Merge remote-tracking branch 'origin/master' into drm-misc-fixes

Pick up 4.14-rc1

Signed-off-by: Sean Paul <seanpaul@chromium.org>

5740 files changed, 424794 insertions, 133456 deletions

diff --git a/drivers/Makefile b/drivers/Makefile
index dfdcda00bfe3..d90fdc413648 100644
--- a/drivers/Makefile
+++ b/drivers/Makefile
@@ -125,7 +125,6 @@ obj-$(CONFIG_ACCESSIBILITY)	+= accessibility/
 obj-$(CONFIG_ISDN)		+= isdn/
 obj-$(CONFIG_EDAC)		+= edac/
 obj-$(CONFIG_EISA)		+= eisa/
-obj-y				+= lguest/
 obj-$(CONFIG_CPU_FREQ)		+= cpufreq/
 obj-$(CONFIG_CPU_IDLE)		+= cpuidle/
 obj-y				+= mmc/
diff --git a/drivers/acpi/Makefile b/drivers/acpi/Makefile
index b1aacfc62b1f..90265ab4437a 100644
--- a/drivers/acpi/Makefile
+++ b/drivers/acpi/Makefile
@@ -50,6 +50,7 @@ acpi-$(CONFIG_ACPI_REDUCED_HARDWARE_ONLY) += evged.o
 acpi-y				+= sysfs.o
 acpi-y				+= property.o
 acpi-$(CONFIG_X86)		+= acpi_cmos_rtc.o
+acpi-$(CONFIG_X86)		+= x86/apple.o
 acpi-$(CONFIG_X86)		+= x86/utils.o
 acpi-$(CONFIG_DEBUG_FS)		+= debugfs.o
 acpi-$(CONFIG_ACPI_NUMA)	+= numa.o
diff --git a/drivers/acpi/acpi_lpat.c b/drivers/acpi/acpi_lpat.c
index c1c4877ca96c..2cd9f738812b 100644
--- a/drivers/acpi/acpi_lpat.c
+++ b/drivers/acpi/acpi_lpat.c
@@ -25,7 +25,7 @@
  * @raw: the raw value, used as a key to get the temerature from the
  *       above mapping table
  *
- * A positive converted temperarure value will be returned on success,
+ * A positive converted temperature value will be returned on success,
  * a negative errno will be returned in error cases.
  */
 int acpi_lpat_raw_to_temp(struct acpi_lpat_conversion_table *lpat_table,
@@ -55,11 +55,11 @@ EXPORT_SYMBOL_GPL(acpi_lpat_raw_to_temp);
  * acpi_lpat_temp_to_raw(): Return raw value from temperature through
  * LPAT conversion table
  *
- * @lpat: the temperature_raw mapping table
+ * @lpat_table: the temperature_raw mapping table
  * @temp: the temperature, used as a key to get the raw value from the
  *        above mapping table
  *
- * A positive converted temperature value will be returned on success,
+ * The raw value will be returned on success,
  * a negative errno will be returned in error cases.
  */
 int acpi_lpat_temp_to_raw(struct acpi_lpat_conversion_table *lpat_table,
diff --git a/drivers/acpi/acpi_lpss.c b/drivers/acpi/acpi_lpss.c
index f88caf5aab76..032ae44710e5 100644
--- a/drivers/acpi/acpi_lpss.c
+++ b/drivers/acpi/acpi_lpss.c
@@ -465,7 +465,8 @@ static int acpi_lpss_create_device(struct acpi_device *adev,
 	acpi_dev_free_resource_list(&resource_list);
 
 	if (!pdata->mmio_base) {
-		ret = -ENOMEM;
+		/* Skip the device, but continue the namespace scan. */
+		ret = 0;
 		goto err_out;
 	}
 
diff --git a/drivers/acpi/acpi_processor.c b/drivers/acpi/acpi_processor.c
index f098e25b6b41..86c10599d9f8 100644
--- a/drivers/acpi/acpi_processor.c
+++ b/drivers/acpi/acpi_processor.c
@@ -670,7 +670,7 @@ err:
 
 }
 
-void __init acpi_processor_check_duplicates(void)
+static void __init acpi_processor_check_duplicates(void)
 {
 	/* check the correctness for all processors in ACPI namespace */
 	acpi_walk_namespace(ACPI_TYPE_PROCESSOR, ACPI_ROOT_OBJECT,
diff --git a/drivers/acpi/acpi_video.c b/drivers/acpi/acpi_video.c
index e88fe3632dd6..0972ec0e2eb8 100644
--- a/drivers/acpi/acpi_video.c
+++ b/drivers/acpi/acpi_video.c
@@ -418,7 +418,7 @@ static int video_set_report_key_events(const struct dmi_system_id *id)
 	return 0;
 }
 
-static struct dmi_system_id video_dmi_table[] = {
+static const struct dmi_system_id video_dmi_table[] = {
 	/*
 	 * Broken _BQC workaround http://bugzilla.kernel.org/show_bug.cgi?id=13121
 	 */
diff --git a/drivers/acpi/acpica/Makefile b/drivers/acpi/acpica/Makefile
index b125bdd3d58b..1709551bc4aa 100644
--- a/drivers/acpi/acpica/Makefile
+++ b/drivers/acpi/acpica/Makefile
@@ -18,6 +18,7 @@ acpi-y :=		\
 	dsmthdat.o	\
 	dsobject.o	\
 	dsopcode.o	\
+	dspkginit.o	\
 	dsutils.o	\
 	dswexec.o	\
 	dswload.o	\
diff --git a/drivers/acpi/acpica/acapps.h b/drivers/acpi/acpica/acapps.h
index bb6a84b0b4b3..7a1a68b5ac5c 100644
--- a/drivers/acpi/acpica/acapps.h
+++ b/drivers/acpi/acpica/acapps.h
@@ -114,6 +114,8 @@ ac_get_all_tables_from_file(char *filename,
 			    u8 get_only_aml_tables,
 			    struct acpi_new_table_desc **return_list_head);
 
+void ac_delete_table_list(struct acpi_new_table_desc *list_head);
+
 u8 ac_is_file_binary(FILE * file);
 
 acpi_status ac_validate_table_header(FILE * file, long table_offset);
diff --git a/drivers/acpi/acpica/acdispat.h b/drivers/acpi/acpica/acdispat.h
index 0d95c85cce06..f8f3a6e74128 100644
--- a/drivers/acpi/acpica/acdispat.h
+++ b/drivers/acpi/acpica/acdispat.h
@@ -237,6 +237,11 @@ acpi_ds_initialize_objects(u32 table_index,
  * dsobject - Parser/Interpreter interface - object initialization and conversion
  */
 acpi_status
+acpi_ds_build_internal_object(struct acpi_walk_state *walk_state,
+			      union acpi_parse_object *op,
+			      union acpi_operand_object **obj_desc_ptr);
+
+acpi_status
 acpi_ds_build_internal_buffer_obj(struct acpi_walk_state *walk_state,
 				  union acpi_parse_object *op,
 				  u32 buffer_length,
@@ -259,6 +264,14 @@ acpi_ds_create_node(struct acpi_walk_state *walk_state,
 		    union acpi_parse_object *op);
 
 /*
+ * dspkginit - Package object initialization
+ */
+acpi_status
+acpi_ds_init_package_element(u8 object_type,
+			     union acpi_operand_object *source_object,
+			     union acpi_generic_state *state, void *context);
+
+/*
  * dsutils - Parser/Interpreter interface utility routines
  */
 void acpi_ds_clear_implicit_return(struct acpi_walk_state *walk_state);
diff --git a/drivers/acpi/acpica/aclocal.h b/drivers/acpi/acpica/aclocal.h
index 8ddd3b20e0c6..0d45b8bb1678 100644
--- a/drivers/acpi/acpica/aclocal.h
+++ b/drivers/acpi/acpica/aclocal.h
@@ -199,6 +199,7 @@ struct acpi_namespace_node {
 #define ANOBJ_EVALUATED                 0x20	/* Set on first evaluation of node */
 #define ANOBJ_ALLOCATED_BUFFER          0x40	/* Method AML buffer is dynamic (install_method) */
 
+#define IMPLICIT_EXTERNAL               0x02	/* iASL only: This object created implicitly via External */
 #define ANOBJ_IS_EXTERNAL               0x08	/* iASL only: This object created via External() */
 #define ANOBJ_METHOD_NO_RETVAL          0x10	/* iASL only: Method has no return value */
 #define ANOBJ_METHOD_SOME_NO_RETVAL     0x20	/* iASL only: Method has at least one return value */
@@ -604,7 +605,7 @@ struct acpi_update_state {
  * Pkg state - used to traverse nested package structures
  */
 struct acpi_pkg_state {
-	ACPI_STATE_COMMON u16 index;
+	ACPI_STATE_COMMON u32 index;
 	union acpi_operand_object *source_object;
 	union acpi_operand_object *dest_object;
 	struct acpi_walk_state *walk_state;
@@ -867,7 +868,7 @@ struct acpi_parse_obj_named {
 
 /* This version is used by the iASL compiler only */
 
-#define ACPI_MAX_PARSEOP_NAME   20
+#define ACPI_MAX_PARSEOP_NAME       20
 
 struct acpi_parse_obj_asl {
 	ACPI_PARSE_COMMON union acpi_parse_object *child;
@@ -907,7 +908,7 @@ union acpi_parse_object {
 struct asl_comment_state {
 	u8 comment_type;
 	u32 spaces_before;
-	union acpi_parse_object *latest_parse_node;
+	union acpi_parse_object *latest_parse_op;
 	union acpi_parse_object *parsing_paren_brace_node;
 	u8 capture_comments;
 };
diff --git a/drivers/acpi/acpica/acobject.h b/drivers/acpi/acpica/acobject.h
index 27c3f982d810..5226146190bf 100644
--- a/drivers/acpi/acpica/acobject.h
+++ b/drivers/acpi/acpica/acobject.h
@@ -122,7 +122,9 @@ struct acpi_object_integer {
 	_type                           *pointer; \
 	u32                             length;
 
-struct acpi_object_string {	/* Null terminated, ASCII characters only */
+/* Null terminated, ASCII characters only */
+
+struct acpi_object_string {
 	ACPI_OBJECT_COMMON_HEADER ACPI_COMMON_BUFFER_INFO(char)	/* String in AML stream or allocated string */
 };
 
@@ -211,7 +213,9 @@ struct acpi_object_method {
 	union acpi_operand_object       *notify_list[2];    /* Handlers for system/device notifies */\
 	union acpi_operand_object       *handler;	/* Handler for Address space */
 
-struct acpi_object_notify_common {	/* COMMON NOTIFY for POWER, PROCESSOR, DEVICE, and THERMAL */
+/* COMMON NOTIFY for POWER, PROCESSOR, DEVICE, and THERMAL */
+
+struct acpi_object_notify_common {
 ACPI_OBJECT_COMMON_HEADER ACPI_COMMON_NOTIFY_INFO};
 
 struct acpi_object_device {
@@ -258,7 +262,9 @@ ACPI_OBJECT_COMMON_HEADER ACPI_COMMON_NOTIFY_INFO};
 	u8                              access_length;	/* For serial regions/fields */
 
 
-struct acpi_object_field_common {	/* COMMON FIELD (for BUFFER, REGION, BANK, and INDEX fields) */
+/* COMMON FIELD (for BUFFER, REGION, BANK, and INDEX fields) */
+
+struct acpi_object_field_common {
 	ACPI_OBJECT_COMMON_HEADER ACPI_COMMON_FIELD_INFO union acpi_operand_object *region_obj;	/* Parent Operation Region object (REGION/BANK fields only) */
 };
 
@@ -333,11 +339,12 @@ struct acpi_object_addr_handler {
 struct acpi_object_reference {
 	ACPI_OBJECT_COMMON_HEADER u8 class;	/* Reference Class */
 	u8 target_type;		/* Used for Index Op */
-	u8 reserved;
+	u8 resolved;		/* Reference has been resolved to a value */
 	void *object;		/* name_op=>HANDLE to obj, index_op=>union acpi_operand_object */
 	struct acpi_namespace_node *node;	/* ref_of or Namepath */
 	union acpi_operand_object **where;	/* Target of Index */
 	u8 *index_pointer;	/* Used for Buffers and Strings */
+	u8 *aml;		/* Used for deferred resolution of the ref */
 	u32 value;		/* Used for Local/Arg/Index/ddb_handle */
 };
 
diff --git a/drivers/acpi/acpica/actables.h b/drivers/acpi/acpica/actables.h
index c8da453bd960..84a3ceb6e384 100644
--- a/drivers/acpi/acpica/actables.h
+++ b/drivers/acpi/acpica/actables.h
@@ -76,7 +76,8 @@ void acpi_tb_release_temp_table(struct acpi_table_desc *table_desc);
 acpi_status acpi_tb_validate_temp_table(struct acpi_table_desc *table_desc);
 
 acpi_status
-acpi_tb_verify_temp_table(struct acpi_table_desc *table_desc, char *signature);
+acpi_tb_verify_temp_table(struct acpi_table_desc *table_desc,
+			  char *signature, u32 *table_index);
 
 u8 acpi_tb_is_table_loaded(u32 table_index);
 
@@ -132,6 +133,8 @@ acpi_tb_install_and_load_table(acpi_physical_address address,
 
 acpi_status acpi_tb_unload_table(u32 table_index);
 
+void acpi_tb_notify_table(u32 event, void *table);
+
 void acpi_tb_terminate(void);
 
 acpi_status acpi_tb_delete_namespace_by_owner(u32 table_index);
diff --git a/drivers/acpi/acpica/acutils.h b/drivers/acpi/acpica/acutils.h
index 2a3cc4296481..745134ade35f 100644
--- a/drivers/acpi/acpica/acutils.h
+++ b/drivers/acpi/acpica/acutils.h
@@ -516,7 +516,7 @@ union acpi_generic_state *acpi_ut_create_update_state(union acpi_operand_object
 
 union acpi_generic_state *acpi_ut_create_pkg_state(void *internal_object,
 						   void *external_object,
-						   u16 index);
+						   u32 index);
 
 acpi_status
 acpi_ut_create_update_state_and_push(union acpi_operand_object *object,
@@ -538,6 +538,13 @@ acpi_status
 acpi_ut_short_divide(u64 in_dividend,
 		     u32 divisor, u64 *out_quotient, u32 *out_remainder);
 
+acpi_status
+acpi_ut_short_multiply(u64 in_multiplicand, u32 multiplier, u64 *outproduct);
+
+acpi_status acpi_ut_short_shift_left(u64 operand, u32 count, u64 *out_result);
+
+acpi_status acpi_ut_short_shift_right(u64 operand, u32 count, u64 *out_result);
+
 /*
  * utmisc
  */
diff --git a/drivers/acpi/acpica/dbdisply.c b/drivers/acpi/acpica/dbdisply.c
index 46bf270ac525..5a606eac0c22 100644
--- a/drivers/acpi/acpica/dbdisply.c
+++ b/drivers/acpi/acpica/dbdisply.c
@@ -310,7 +310,7 @@ dump_node:
 	}
 
 	else {
-		acpi_os_printf("Object (%p) Pathname: %s\n",
+		acpi_os_printf("Object %p: Namespace Node - Pathname: %s\n",
 			       node, (char *)ret_buf.pointer);
 	}
 
@@ -326,7 +326,7 @@ dump_node:
 
 	obj_desc = acpi_ns_get_attached_object(node);
 	if (obj_desc) {
-		acpi_os_printf("\nAttached Object (%p):\n", obj_desc);
+		acpi_os_printf("\nAttached Object %p:", obj_desc);
 		if (!acpi_os_readable
 		    (obj_desc, sizeof(union acpi_operand_object))) {
 			acpi_os_printf
@@ -335,9 +335,36 @@ dump_node:
 			return;
 		}
 
-		acpi_ut_debug_dump_buffer((void *)obj_desc,
-					  sizeof(union acpi_operand_object),
-					  display, ACPI_UINT32_MAX);
+		if (ACPI_GET_DESCRIPTOR_TYPE(((struct acpi_namespace_node *)
+					      obj_desc)) ==
+		    ACPI_DESC_TYPE_NAMED) {
+			acpi_os_printf(" Namespace Node - ");
+			status =
+			    acpi_get_name((struct acpi_namespace_node *)
+					  obj_desc,
+					  ACPI_FULL_PATHNAME_NO_TRAILING,
+					  &ret_buf);
+			if (ACPI_FAILURE(status)) {
+				acpi_os_printf
+				    ("Could not convert name to pathname\n");
+			} else {
+				acpi_os_printf("Pathname: %s",
+					       (char *)ret_buf.pointer);
+			}
+
+			acpi_os_printf("\n");
+			acpi_ut_debug_dump_buffer((void *)obj_desc,
+						  sizeof(struct
+							 acpi_namespace_node),
+						  display, ACPI_UINT32_MAX);
+		} else {
+			acpi_os_printf("\n");
+			acpi_ut_debug_dump_buffer((void *)obj_desc,
+						  sizeof(union
+							 acpi_operand_object),
+						  display, ACPI_UINT32_MAX);
+		}
+
 		acpi_ex_dump_object_descriptor(obj_desc, 1);
 	}
 }
diff --git a/drivers/acpi/acpica/dsfield.c b/drivers/acpi/acpica/dsfield.c
index c5dccc54307d..7bcf5f5ea029 100644
--- a/drivers/acpi/acpica/dsfield.c
+++ b/drivers/acpi/acpica/dsfield.c
@@ -184,6 +184,7 @@ acpi_ds_create_buffer_field(union acpi_parse_object *op,
 		/* Execute flag should always be set when this function is entered */
 
 		if (!(walk_state->parse_flags & ACPI_PARSE_EXECUTE)) {
+			ACPI_ERROR((AE_INFO, "Parse execute mode is not set"));
 			return_ACPI_STATUS(AE_AML_INTERNAL);
 		}
 
@@ -556,6 +557,7 @@ acpi_ds_init_field_objects(union acpi_parse_object *op,
 			return_ACPI_STATUS(AE_OK);
 		}
 
+		ACPI_ERROR((AE_INFO, "Parse deferred mode is not set"));
 		return_ACPI_STATUS(AE_AML_INTERNAL);
 	}
 
diff --git a/drivers/acpi/acpica/dsobject.c b/drivers/acpi/acpica/dsobject.c
index 7df3152ed856..82448551781b 100644
--- a/drivers/acpi/acpica/dsobject.c
+++ b/drivers/acpi/acpica/dsobject.c
@@ -52,12 +52,6 @@
 #define _COMPONENT          ACPI_DISPATCHER
 ACPI_MODULE_NAME("dsobject")
 
-/* Local prototypes */
-static acpi_status
-acpi_ds_build_internal_object(struct acpi_walk_state *walk_state,
-			      union acpi_parse_object *op,
-			      union acpi_operand_object **obj_desc_ptr);
-
 #ifndef ACPI_NO_METHOD_EXECUTION
 /*******************************************************************************
  *
@@ -73,15 +67,13 @@ acpi_ds_build_internal_object(struct acpi_walk_state *walk_state,
  *              Simple objects are any objects other than a package object!
  *
  ******************************************************************************/
-
-static acpi_status
+acpi_status
 acpi_ds_build_internal_object(struct acpi_walk_state *walk_state,
 			      union acpi_parse_object *op,
 			      union acpi_operand_object **obj_desc_ptr)
 {
 	union acpi_operand_object *obj_desc;
 	acpi_status status;
-	acpi_object_type type;
 
 	ACPI_FUNCTION_TRACE(ds_build_internal_object);
 
@@ -89,140 +81,47 @@ acpi_ds_build_internal_object(struct acpi_walk_state *walk_state,
 	if (op->common.aml_opcode == AML_INT_NAMEPATH_OP) {
 		/*
 		 * This is a named object reference. If this name was
-		 * previously looked up in the namespace, it was stored in this op.
-		 * Otherwise, go ahead and look it up now
+		 * previously looked up in the namespace, it was stored in
+		 * this op. Otherwise, go ahead and look it up now
 		 */
 		if (!op->common.node) {
-			status = acpi_ns_lookup(walk_state->scope_info,
-						op->common.value.string,
-						ACPI_TYPE_ANY,
-						ACPI_IMODE_EXECUTE,
-						ACPI_NS_SEARCH_PARENT |
-						ACPI_NS_DONT_OPEN_SCOPE, NULL,
-						ACPI_CAST_INDIRECT_PTR(struct
-								       acpi_namespace_node,
-								       &(op->
-									 common.
-									 node)));
-			if (ACPI_FAILURE(status)) {
-
-				/* Check if we are resolving a named reference within a package */
-
-				if ((status == AE_NOT_FOUND)
-				    && (acpi_gbl_enable_interpreter_slack)
-				    &&
-				    ((op->common.parent->common.aml_opcode ==
-				      AML_PACKAGE_OP)
-				     || (op->common.parent->common.aml_opcode ==
-					 AML_VARIABLE_PACKAGE_OP))) {
-					/*
-					 * We didn't find the target and we are populating elements
-					 * of a package - ignore if slack enabled. Some ASL code
-					 * contains dangling invalid references in packages and
-					 * expects that no exception will be issued. Leave the
-					 * element as a null element. It cannot be used, but it
-					 * can be overwritten by subsequent ASL code - this is
-					 * typically the case.
-					 */
-					ACPI_DEBUG_PRINT((ACPI_DB_INFO,
-							  "Ignoring unresolved reference in package [%4.4s]\n",
-							  walk_state->
-							  scope_info->scope.
-							  node->name.ascii));
-
-					return_ACPI_STATUS(AE_OK);
-				} else {
-					ACPI_ERROR_NAMESPACE(op->common.value.
-							     string, status);
-				}
-
-				return_ACPI_STATUS(status);
-			}
-		}
-
-		/* Special object resolution for elements of a package */
-
-		if ((op->common.parent->common.aml_opcode == AML_PACKAGE_OP) ||
-		    (op->common.parent->common.aml_opcode ==
-		     AML_VARIABLE_PACKAGE_OP)) {
-			/*
-			 * Attempt to resolve the node to a value before we insert it into
-			 * the package. If this is a reference to a common data type,
-			 * resolve it immediately. According to the ACPI spec, package
-			 * elements can only be "data objects" or method references.
-			 * Attempt to resolve to an Integer, Buffer, String or Package.
-			 * If cannot, return the named reference (for things like Devices,
-			 * Methods, etc.) Buffer Fields and Fields will resolve to simple
-			 * objects (int/buf/str/pkg).
-			 *
-			 * NOTE: References to things like Devices, Methods, Mutexes, etc.
-			 * will remain as named references. This behavior is not described
-			 * in the ACPI spec, but it appears to be an oversight.
-			 */
-			obj_desc =
-			    ACPI_CAST_PTR(union acpi_operand_object,
-					  op->common.node);
-
-			status =
-			    acpi_ex_resolve_node_to_value(ACPI_CAST_INDIRECT_PTR
-							  (struct
-							   acpi_namespace_node,
-							   &obj_desc),
-							  walk_state);
-			if (ACPI_FAILURE(status)) {
-				return_ACPI_STATUS(status);
-			}
-
-			/*
-			 * Special handling for Alias objects. We need to setup the type
-			 * and the Op->Common.Node to point to the Alias target. Note,
-			 * Alias has at most one level of indirection internally.
-			 */
-			type = op->common.node->type;
-			if (type == ACPI_TYPE_LOCAL_ALIAS) {
-				type = obj_desc->common.type;
-				op->common.node =
-				    ACPI_CAST_PTR(struct acpi_namespace_node,
-						  op->common.node->object);
-			}
-
-			switch (type) {
-				/*
-				 * For these types, we need the actual node, not the subobject.
-				 * However, the subobject did not get an extra reference count above.
-				 *
-				 * TBD: should ex_resolve_node_to_value be changed to fix this?
-				 */
-			case ACPI_TYPE_DEVICE:
-			case ACPI_TYPE_THERMAL:
-
-				acpi_ut_add_reference(op->common.node->object);
 
-				/*lint -fallthrough */
-				/*
-				 * For these types, we need the actual node, not the subobject.
-				 * The subobject got an extra reference count in ex_resolve_node_to_value.
-				 */
-			case ACPI_TYPE_MUTEX:
-			case ACPI_TYPE_METHOD:
-			case ACPI_TYPE_POWER:
-			case ACPI_TYPE_PROCESSOR:
-			case ACPI_TYPE_EVENT:
-			case ACPI_TYPE_REGION:
-
-				/* We will create a reference object for these types below */
-				break;
+			/* Check if we are resolving a named reference within a package */
 
-			default:
+			if ((op->common.parent->common.aml_opcode ==
+			     AML_PACKAGE_OP)
+			    || (op->common.parent->common.aml_opcode ==
+				AML_VARIABLE_PACKAGE_OP)) {
 				/*
-				 * All other types - the node was resolved to an actual
-				 * object, we are done.
+				 * We won't resolve package elements here, we will do this
+				 * after all ACPI tables are loaded into the namespace. This
+				 * behavior supports both forward references to named objects
+				 * and external references to objects in other tables.
 				 */
-				goto exit;
+				goto create_new_object;
+			} else {
+				status = acpi_ns_lookup(walk_state->scope_info,
+							op->common.value.string,
+							ACPI_TYPE_ANY,
+							ACPI_IMODE_EXECUTE,
+							ACPI_NS_SEARCH_PARENT |
+							ACPI_NS_DONT_OPEN_SCOPE,
+							NULL,
+							ACPI_CAST_INDIRECT_PTR
+							(struct
+							 acpi_namespace_node,
+							 &(op->common.node)));
+				if (ACPI_FAILURE(status)) {
+					ACPI_ERROR_NAMESPACE(op->common.value.
+							     string, status);
+					return_ACPI_STATUS(status);
+				}
 			}
 		}
 	}
 
+create_new_object:
+
 	/* Create and init a new internal ACPI object */
 
 	obj_desc = acpi_ut_create_internal_object((acpi_ps_get_opcode_info
@@ -240,7 +139,27 @@ acpi_ds_build_internal_object(struct acpi_walk_state *walk_state,
 		return_ACPI_STATUS(status);
 	}
 
-exit:
+	/*
+	 * Handling for unresolved package reference elements.
+	 * These are elements that are namepaths.
+	 */
+	if ((op->common.parent->common.aml_opcode == AML_PACKAGE_OP) ||
+	    (op->common.parent->common.aml_opcode == AML_VARIABLE_PACKAGE_OP)) {
+		obj_desc->reference.resolved = TRUE;
+
+		if ((op->common.aml_opcode == AML_INT_NAMEPATH_OP) &&
+		    !obj_desc->reference.node) {
+			/*
+			 * Name was unresolved above.
+			 * Get the prefix node for later lookup
+			 */
+			obj_desc->reference.node =
+			    walk_state->scope_info->scope.node;
+			obj_desc->reference.aml = op->common.aml;
+			obj_desc->reference.resolved = FALSE;
+		}
+	}
+
 	*obj_desc_ptr = obj_desc;
 	return_ACPI_STATUS(status);
 }
@@ -351,200 +270,6 @@ acpi_ds_build_internal_buffer_obj(struct acpi_walk_state *walk_state,
 
 /*******************************************************************************
  *
- * FUNCTION:    acpi_ds_build_internal_package_obj
- *
- * PARAMETERS:  walk_state      - Current walk state
- *              op              - Parser object to be translated
- *              element_count   - Number of elements in the package - this is
- *                                the num_elements argument to Package()
- *              obj_desc_ptr    - Where the ACPI internal object is returned
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Translate a parser Op package object to the equivalent
- *              namespace object
- *
- * NOTE: The number of elements in the package will be always be the num_elements
- * count, regardless of the number of elements in the package list. If
- * num_elements is smaller, only that many package list elements are used.
- * if num_elements is larger, the Package object is padded out with
- * objects of type Uninitialized (as per ACPI spec.)
- *
- * Even though the ASL compilers do not allow num_elements to be smaller
- * than the Package list length (for the fixed length package opcode), some
- * BIOS code modifies the AML on the fly to adjust the num_elements, and
- * this code compensates for that. This also provides compatibility with
- * other AML interpreters.
- *
- ******************************************************************************/
-
-acpi_status
-acpi_ds_build_internal_package_obj(struct acpi_walk_state *walk_state,
-				   union acpi_parse_object *op,
-				   u32 element_count,
-				   union acpi_operand_object **obj_desc_ptr)
-{
-	union acpi_parse_object *arg;
-	union acpi_parse_object *parent;
-	union acpi_operand_object *obj_desc = NULL;
-	acpi_status status = AE_OK;
-	u32 i;
-	u16 index;
-	u16 reference_count;
-
-	ACPI_FUNCTION_TRACE(ds_build_internal_package_obj);
-
-	/* Find the parent of a possibly nested package */
-
-	parent = op->common.parent;
-	while ((parent->common.aml_opcode == AML_PACKAGE_OP) ||
-	       (parent->common.aml_opcode == AML_VARIABLE_PACKAGE_OP)) {
-		parent = parent->common.parent;
-	}
-
-	/*
-	 * If we are evaluating a Named package object "Name (xxxx, Package)",
-	 * the package object already exists, otherwise it must be created.
-	 */
-	obj_desc = *obj_desc_ptr;
-	if (!obj_desc) {
-		obj_desc = acpi_ut_create_internal_object(ACPI_TYPE_PACKAGE);
-		*obj_desc_ptr = obj_desc;
-		if (!obj_desc) {
-			return_ACPI_STATUS(AE_NO_MEMORY);
-		}
-
-		obj_desc->package.node = parent->common.node;
-	}
-
-	/*
-	 * Allocate the element array (array of pointers to the individual
-	 * objects) based on the num_elements parameter. Add an extra pointer slot
-	 * so that the list is always null terminated.
-	 */
-	obj_desc->package.elements = ACPI_ALLOCATE_ZEROED(((acpi_size)
-							   element_count +
-							   1) * sizeof(void *));
-
-	if (!obj_desc->package.elements) {
-		acpi_ut_delete_object_desc(obj_desc);
-		return_ACPI_STATUS(AE_NO_MEMORY);
-	}
-
-	obj_desc->package.count = element_count;
-
-	/*
-	 * Initialize the elements of the package, up to the num_elements count.
-	 * Package is automatically padded with uninitialized (NULL) elements
-	 * if num_elements is greater than the package list length. Likewise,
-	 * Package is truncated if num_elements is less than the list length.
-	 */
-	arg = op->common.value.arg;
-	arg = arg->common.next;
-	for (i = 0; arg && (i < element_count); i++) {
-		if (arg->common.aml_opcode == AML_INT_RETURN_VALUE_OP) {
-			if (arg->common.node->type == ACPI_TYPE_METHOD) {
-				/*
-				 * A method reference "looks" to the parser to be a method
-				 * invocation, so we special case it here
-				 */
-				arg->common.aml_opcode = AML_INT_NAMEPATH_OP;
-				status =
-				    acpi_ds_build_internal_object(walk_state,
-								  arg,
-								  &obj_desc->
-								  package.
-								  elements[i]);
-			} else {
-				/* This package element is already built, just get it */
-
-				obj_desc->package.elements[i] =
-				    ACPI_CAST_PTR(union acpi_operand_object,
-						  arg->common.node);
-			}
-		} else {
-			status =
-			    acpi_ds_build_internal_object(walk_state, arg,
-							  &obj_desc->package.
-							  elements[i]);
-		}
-
-		if (*obj_desc_ptr) {
-
-			/* Existing package, get existing reference count */
-
-			reference_count =
-			    (*obj_desc_ptr)->common.reference_count;
-			if (reference_count > 1) {
-
-				/* Make new element ref count match original ref count */
-
-				for (index = 0; index < (reference_count - 1);
-				     index++) {
-					acpi_ut_add_reference((obj_desc->
-							       package.
-							       elements[i]));
-				}
-			}
-		}
-
-		arg = arg->common.next;
-	}
-
-	/* Check for match between num_elements and actual length of package_list */
-
-	if (arg) {
-		/*
-		 * num_elements was exhausted, but there are remaining elements in the
-		 * package_list. Truncate the package to num_elements.
-		 *
-		 * Note: technically, this is an error, from ACPI spec: "It is an error
-		 * for NumElements to be less than the number of elements in the
-		 * PackageList". However, we just print a message and
-		 * no exception is returned. This provides Windows compatibility. Some
-		 * BIOSs will alter the num_elements on the fly, creating this type
-		 * of ill-formed package object.
-		 */
-		while (arg) {
-			/*
-			 * We must delete any package elements that were created earlier
-			 * and are not going to be used because of the package truncation.
-			 */
-			if (arg->common.node) {
-				acpi_ut_remove_reference(ACPI_CAST_PTR
-							 (union
-							  acpi_operand_object,
-							  arg->common.node));
-				arg->common.node = NULL;
-			}
-
-			/* Find out how many elements there really are */
-
-			i++;
-			arg = arg->common.next;
-		}
-
-		ACPI_INFO(("Actual Package length (%u) is larger than "
-			   "NumElements field (%u), truncated",
-			   i, element_count));
-	} else if (i < element_count) {
-		/*
-		 * Arg list (elements) was exhausted, but we did not reach num_elements count.
-		 * Note: this is not an error, the package is padded out with NULLs.
-		 */
-		ACPI_DEBUG_PRINT((ACPI_DB_INFO,
-				  "Package List length (%u) smaller than NumElements "
-				  "count (%u), padded with null elements\n",
-				  i, element_count));
-	}
-
-	obj_desc->package.flags |= AOPOBJ_DATA_VALID;
-	op->common.node = ACPI_CAST_PTR(struct acpi_namespace_node, obj_desc);
-	return_ACPI_STATUS(status);
-}
-
-/*******************************************************************************
- *
  * FUNCTION:    acpi_ds_create_node
  *
  * PARAMETERS:  walk_state      - Current walk state
@@ -662,11 +387,20 @@ acpi_ds_init_object_from_op(struct acpi_walk_state *walk_state,
 
 	case ACPI_TYPE_PACKAGE:
 		/*
-		 * Defer evaluation of Package term_arg operand
+		 * Defer evaluation of Package term_arg operand and all
+		 * package elements. (01/2017): We defer the element
+		 * resolution to allow forward references from the package
+		 * in order to provide compatibility with other ACPI
+		 * implementations.
 		 */
 		obj_desc->package.node =
 		    ACPI_CAST_PTR(struct acpi_namespace_node,
 				  walk_state->operands[0]);
+
+		if (!op->named.data) {
+			return_ACPI_STATUS(AE_OK);
+		}
+
 		obj_desc->package.aml_start = op->named.data;
 		obj_desc->package.aml_length = op->named.length;
 		break;
@@ -818,9 +552,11 @@ acpi_ds_init_object_from_op(struct acpi_walk_state *walk_state,
 				/* Node was saved in Op */
 
 				obj_desc->reference.node = op->common.node;
-				obj_desc->reference.object =
-				    op->common.node->object;
 				obj_desc->reference.class = ACPI_REFCLASS_NAME;
+				if (op->common.node) {
+					obj_desc->reference.object =
+					    op->common.node->object;
+				}
 				break;
 
 			case AML_DEBUG_OP:
diff --git a/drivers/acpi/acpica/dsopcode.c b/drivers/acpi/acpica/dsopcode.c
index dfc3c25a083d..0336df7ac47d 100644
--- a/drivers/acpi/acpica/dsopcode.c
+++ b/drivers/acpi/acpica/dsopcode.c
@@ -599,6 +599,15 @@ acpi_ds_eval_data_object_operands(struct acpi_walk_state *walk_state,
 	 */
 	walk_state->operand_index = walk_state->num_operands;
 
+	/* Ignore if child is not valid */
+
+	if (!op->common.value.arg) {
+		ACPI_ERROR((AE_INFO,
+			    "Dispatch: Missing child while executing TermArg for %X",
+			    op->common.aml_opcode));
+		return_ACPI_STATUS(AE_OK);
+	}
+
 	status = acpi_ds_create_operand(walk_state, op->common.value.arg, 1);
 	if (ACPI_FAILURE(status)) {
 		return_ACPI_STATUS(status);
diff --git a/drivers/acpi/acpica/dspkginit.c b/drivers/acpi/acpica/dspkginit.c
new file mode 100644
index 000000000000..6d487edfe2de
--- /dev/null
+++ b/drivers/acpi/acpica/dspkginit.c
@@ -0,0 +1,496 @@
+/******************************************************************************
+ *
+ * Module Name: dspkginit - Completion of deferred package initialization
+ *
+ *****************************************************************************/
+
+/*
+ * Copyright (C) 2000 - 2017, Intel Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce at minimum a disclaimer
+ *    substantially similar to the "NO WARRANTY" disclaimer below
+ *    ("Disclaimer") and any redistribution must be conditioned upon
+ *    including a substantially similar Disclaimer requirement for further
+ *    binary redistribution.
+ * 3. Neither the names of the above-listed copyright holders nor the names
+ *    of any contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * NO WARRANTY
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGES.
+ */
+
+#include <acpi/acpi.h>
+#include "accommon.h"
+#include "acnamesp.h"
+#include "amlcode.h"
+#include "acdispat.h"
+#include "acinterp.h"
+
+#define _COMPONENT          ACPI_NAMESPACE
+ACPI_MODULE_NAME("dspkginit")
+
+/* Local prototypes */
+static void
+acpi_ds_resolve_package_element(union acpi_operand_object **element);
+
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_ds_build_internal_package_obj
+ *
+ * PARAMETERS:  walk_state      - Current walk state
+ *              op              - Parser object to be translated
+ *              element_count   - Number of elements in the package - this is
+ *                                the num_elements argument to Package()
+ *              obj_desc_ptr    - Where the ACPI internal object is returned
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Translate a parser Op package object to the equivalent
+ *              namespace object
+ *
+ * NOTE: The number of elements in the package will be always be the num_elements
+ * count, regardless of the number of elements in the package list. If
+ * num_elements is smaller, only that many package list elements are used.
+ * if num_elements is larger, the Package object is padded out with
+ * objects of type Uninitialized (as per ACPI spec.)
+ *
+ * Even though the ASL compilers do not allow num_elements to be smaller
+ * than the Package list length (for the fixed length package opcode), some
+ * BIOS code modifies the AML on the fly to adjust the num_elements, and
+ * this code compensates for that. This also provides compatibility with
+ * other AML interpreters.
+ *
+ ******************************************************************************/
+
+acpi_status
+acpi_ds_build_internal_package_obj(struct acpi_walk_state *walk_state,
+				   union acpi_parse_object *op,
+				   u32 element_count,
+				   union acpi_operand_object **obj_desc_ptr)
+{
+	union acpi_parse_object *arg;
+	union acpi_parse_object *parent;
+	union acpi_operand_object *obj_desc = NULL;
+	acpi_status status = AE_OK;
+	u16 reference_count;
+	u32 index;
+	u32 i;
+
+	ACPI_FUNCTION_TRACE(ds_build_internal_package_obj);
+
+	/* Find the parent of a possibly nested package */
+
+	parent = op->common.parent;
+	while ((parent->common.aml_opcode == AML_PACKAGE_OP) ||
+	       (parent->common.aml_opcode == AML_VARIABLE_PACKAGE_OP)) {
+		parent = parent->common.parent;
+	}
+
+	/*
+	 * If we are evaluating a Named package object of the form:
+	 *      Name (xxxx, Package)
+	 * the package object already exists, otherwise it must be created.
+	 */
+	obj_desc = *obj_desc_ptr;
+	if (!obj_desc) {
+		obj_desc = acpi_ut_create_internal_object(ACPI_TYPE_PACKAGE);
+		*obj_desc_ptr = obj_desc;
+		if (!obj_desc) {
+			return_ACPI_STATUS(AE_NO_MEMORY);
+		}
+
+		obj_desc->package.node = parent->common.node;
+	}
+
+	if (obj_desc->package.flags & AOPOBJ_DATA_VALID) {	/* Just in case */
+		return_ACPI_STATUS(AE_OK);
+	}
+
+	/*
+	 * Allocate the element array (array of pointers to the individual
+	 * objects) based on the num_elements parameter. Add an extra pointer slot
+	 * so that the list is always null terminated.
+	 */
+	obj_desc->package.elements = ACPI_ALLOCATE_ZEROED(((acpi_size)
+							   element_count +
+							   1) * sizeof(void *));
+
+	if (!obj_desc->package.elements) {
+		acpi_ut_delete_object_desc(obj_desc);
+		return_ACPI_STATUS(AE_NO_MEMORY);
+	}
+
+	obj_desc->package.count = element_count;
+	arg = op->common.value.arg;
+	arg = arg->common.next;
+
+	if (arg) {
+		obj_desc->package.flags |= AOPOBJ_DATA_VALID;
+	}
+
+	/*
+	 * Initialize the elements of the package, up to the num_elements count.
+	 * Package is automatically padded with uninitialized (NULL) elements
+	 * if num_elements is greater than the package list length. Likewise,
+	 * Package is truncated if num_elements is less than the list length.
+	 */
+	for (i = 0; arg && (i < element_count); i++) {
+		if (arg->common.aml_opcode == AML_INT_RETURN_VALUE_OP) {
+			if (arg->common.node->type == ACPI_TYPE_METHOD) {
+				/*
+				 * A method reference "looks" to the parser to be a method
+				 * invocation, so we special case it here
+				 */
+				arg->common.aml_opcode = AML_INT_NAMEPATH_OP;
+				status =
+				    acpi_ds_build_internal_object(walk_state,
+								  arg,
+								  &obj_desc->
+								  package.
+								  elements[i]);
+			} else {
+				/* This package element is already built, just get it */
+
+				obj_desc->package.elements[i] =
+				    ACPI_CAST_PTR(union acpi_operand_object,
+						  arg->common.node);
+			}
+		} else {
+			status =
+			    acpi_ds_build_internal_object(walk_state, arg,
+							  &obj_desc->package.
+							  elements[i]);
+			if (status == AE_NOT_FOUND) {
+				ACPI_ERROR((AE_INFO, "%-48s",
+					    "****DS namepath not found"));
+			}
+
+			/*
+			 * Initialize this package element. This function handles the
+			 * resolution of named references within the package.
+			 */
+			acpi_ds_init_package_element(0,
+						     obj_desc->package.
+						     elements[i], NULL,
+						     &obj_desc->package.
+						     elements[i]);
+		}
+
+		if (*obj_desc_ptr) {
+
+			/* Existing package, get existing reference count */
+
+			reference_count =
+			    (*obj_desc_ptr)->common.reference_count;
+			if (reference_count > 1) {
+
+				/* Make new element ref count match original ref count */
+				/* TBD: Probably need an acpi_ut_add_references function */
+
+				for (index = 0;
+				     index < ((u32)reference_count - 1);
+				     index++) {
+					acpi_ut_add_reference((obj_desc->
+							       package.
+							       elements[i]));
+				}
+			}
+		}
+
+		arg = arg->common.next;
+	}
+
+	/* Check for match between num_elements and actual length of package_list */
+
+	if (arg) {
+		/*
+		 * num_elements was exhausted, but there are remaining elements in
+		 * the package_list. Truncate the package to num_elements.
+		 *
+		 * Note: technically, this is an error, from ACPI spec: "It is an
+		 * error for NumElements to be less than the number of elements in
+		 * the PackageList". However, we just print a message and no
+		 * exception is returned. This provides compatibility with other
+		 * ACPI implementations. Some firmware implementations will alter
+		 * the num_elements on the fly, possibly creating this type of
+		 * ill-formed package object.
+		 */
+		while (arg) {
+			/*
+			 * We must delete any package elements that were created earlier
+			 * and are not going to be used because of the package truncation.
+			 */
+			if (arg->common.node) {
+				acpi_ut_remove_reference(ACPI_CAST_PTR
+							 (union
+							  acpi_operand_object,
+							  arg->common.node));
+				arg->common.node = NULL;
+			}
+
+			/* Find out how many elements there really are */
+
+			i++;
+			arg = arg->common.next;
+		}
+
+		ACPI_INFO(("Actual Package length (%u) is larger than "
+			   "NumElements field (%u), truncated",
+			   i, element_count));
+	} else if (i < element_count) {
+		/*
+		 * Arg list (elements) was exhausted, but we did not reach
+		 * num_elements count.
+		 *
+		 * Note: this is not an error, the package is padded out
+		 * with NULLs.
+		 */
+		ACPI_DEBUG_PRINT((ACPI_DB_INFO,
+				  "Package List length (%u) smaller than NumElements "
+				  "count (%u), padded with null elements\n",
+				  i, element_count));
+	}
+
+	obj_desc->package.flags |= AOPOBJ_DATA_VALID;
+	op->common.node = ACPI_CAST_PTR(struct acpi_namespace_node, obj_desc);
+	return_ACPI_STATUS(status);
+}
+
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_ds_init_package_element
+ *
+ * PARAMETERS:  acpi_pkg_callback
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Resolve a named reference element within a package object
+ *
+ ******************************************************************************/
+
+acpi_status
+acpi_ds_init_package_element(u8 object_type,
+			     union acpi_operand_object *source_object,
+			     union acpi_generic_state *state, void *context)
+{
+	union acpi_operand_object **element_ptr;
+
+	if (!source_object) {
+		return (AE_OK);
+	}
+
+	/*
+	 * The following code is a bit of a hack to workaround a (current)
+	 * limitation of the acpi_pkg_callback interface. We need a pointer
+	 * to the location within the element array because a new object
+	 * may be created and stored there.
+	 */
+	if (context) {
+
+		/* A direct call was made to this function */
+
+		element_ptr = (union acpi_operand_object **)context;
+	} else {
+		/* Call came from acpi_ut_walk_package_tree */
+
+		element_ptr = state->pkg.this_target_obj;
+	}
+
+	/* We are only interested in reference objects/elements */
+
+	if (source_object->common.type == ACPI_TYPE_LOCAL_REFERENCE) {
+
+		/* Attempt to resolve the (named) reference to a namespace node */
+
+		acpi_ds_resolve_package_element(element_ptr);
+	} else if (source_object->common.type == ACPI_TYPE_PACKAGE) {
+		source_object->package.flags |= AOPOBJ_DATA_VALID;
+	}
+
+	return (AE_OK);
+}
+
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_ds_resolve_package_element
+ *
+ * PARAMETERS:  element_ptr         - Pointer to a reference object
+ *
+ * RETURN:      Possible new element is stored to the indirect element_ptr
+ *
+ * DESCRIPTION: Resolve a package element that is a reference to a named
+ *              object.
+ *
+ ******************************************************************************/
+
+static void
+acpi_ds_resolve_package_element(union acpi_operand_object **element_ptr)
+{
+	acpi_status status;
+	union acpi_generic_state scope_info;
+	union acpi_operand_object *element = *element_ptr;
+	struct acpi_namespace_node *resolved_node;
+	char *external_path = NULL;
+	acpi_object_type type;
+
+	ACPI_FUNCTION_TRACE(ds_resolve_package_element);
+
+	/* Check if reference element is already resolved */
+
+	if (element->reference.resolved) {
+		return_VOID;
+	}
+
+	/* Element must be a reference object of correct type */
+
+	scope_info.scope.node = element->reference.node;	/* Prefix node */
+
+	status = acpi_ns_lookup(&scope_info, (char *)element->reference.aml,	/* Pointer to AML path */
+				ACPI_TYPE_ANY, ACPI_IMODE_EXECUTE,
+				ACPI_NS_SEARCH_PARENT | ACPI_NS_DONT_OPEN_SCOPE,
+				NULL, &resolved_node);
+	if (ACPI_FAILURE(status)) {
+		status = acpi_ns_externalize_name(ACPI_UINT32_MAX,
+						  (char *)element->reference.
+						  aml, NULL, &external_path);
+
+		ACPI_EXCEPTION((AE_INFO, status,
+				"Could not find/resolve named package element: %s",
+				external_path));
+
+		ACPI_FREE(external_path);
+		*element_ptr = NULL;
+		return_VOID;
+	} else if (resolved_node->type == ACPI_TYPE_ANY) {
+
+		/* Named reference not resolved, return a NULL package element */
+
+		ACPI_ERROR((AE_INFO,
+			    "Could not resolve named package element [%4.4s] in [%4.4s]",
+			    resolved_node->name.ascii,
+			    scope_info.scope.node->name.ascii));
+		*element_ptr = NULL;
+		return_VOID;
+	}
+#if 0
+	else if (resolved_node->flags & ANOBJ_TEMPORARY) {
+		/*
+		 * A temporary node found here indicates that the reference is
+		 * to a node that was created within this method. We are not
+		 * going to allow it (especially if the package is returned
+		 * from the method) -- the temporary node will be deleted out
+		 * from under the method. (05/2017).
+		 */
+		ACPI_ERROR((AE_INFO,
+			    "Package element refers to a temporary name [%4.4s], "
+			    "inserting a NULL element",
+			    resolved_node->name.ascii));
+		*element_ptr = NULL;
+		return_VOID;
+	}
+#endif
+
+	/*
+	 * Special handling for Alias objects. We need resolved_node to point
+	 * to the Alias target. This effectively "resolves" the alias.
+	 */
+	if (resolved_node->type == ACPI_TYPE_LOCAL_ALIAS) {
+		resolved_node = ACPI_CAST_PTR(struct acpi_namespace_node,
+					      resolved_node->object);
+	}
+
+	/* Update the reference object */
+
+	element->reference.resolved = TRUE;
+	element->reference.node = resolved_node;
+	type = element->reference.node->type;
+
+	/*
+	 * Attempt to resolve the node to a value before we insert it into
+	 * the package. If this is a reference to a common data type,
+	 * resolve it immediately. According to the ACPI spec, package
+	 * elements can only be "data objects" or method references.
+	 * Attempt to resolve to an Integer, Buffer, String or Package.
+	 * If cannot, return the named reference (for things like Devices,
+	 * Methods, etc.) Buffer Fields and Fields will resolve to simple
+	 * objects (int/buf/str/pkg).
+	 *
+	 * NOTE: References to things like Devices, Methods, Mutexes, etc.
+	 * will remain as named references. This behavior is not described
+	 * in the ACPI spec, but it appears to be an oversight.
+	 */
+	status = acpi_ex_resolve_node_to_value(&resolved_node, NULL);
+	if (ACPI_FAILURE(status)) {
+		return_VOID;
+	}
+#if 0
+/* TBD - alias support */
+	/*
+	 * Special handling for Alias objects. We need to setup the type
+	 * and the Op->Common.Node to point to the Alias target. Note,
+	 * Alias has at most one level of indirection internally.
+	 */
+	type = op->common.node->type;
+	if (type == ACPI_TYPE_LOCAL_ALIAS) {
+		type = obj_desc->common.type;
+		op->common.node = ACPI_CAST_PTR(struct acpi_namespace_node,
+						op->common.node->object);
+	}
+#endif
+
+	switch (type) {
+		/*
+		 * These object types are a result of named references, so we will
+		 * leave them as reference objects. In other words, these types
+		 * have no intrinsic "value".
+		 */
+	case ACPI_TYPE_DEVICE:
+	case ACPI_TYPE_THERMAL:
+
+		/* TBD: This may not be necesssary */
+
+		acpi_ut_add_reference(resolved_node->object);
+		break;
+
+	case ACPI_TYPE_MUTEX:
+	case ACPI_TYPE_METHOD:
+	case ACPI_TYPE_POWER:
+	case ACPI_TYPE_PROCESSOR:
+	case ACPI_TYPE_EVENT:
+	case ACPI_TYPE_REGION:
+
+		break;
+
+	default:
+		/*
+		 * For all other types - the node was resolved to an actual
+		 * operand object with a value, return the object
+		 */
+		*element_ptr = (union acpi_operand_object *)resolved_node;
+		break;
+	}
+
+	return_VOID;
+}
diff --git a/drivers/acpi/acpica/evgpeblk.c b/drivers/acpi/acpica/evgpeblk.c
index 9c941947a063..3a3cb8624f41 100644
--- a/drivers/acpi/acpica/evgpeblk.c
+++ b/drivers/acpi/acpica/evgpeblk.c
@@ -440,9 +440,11 @@ acpi_ev_initialize_gpe_block(struct acpi_gpe_xrupt_info *gpe_xrupt_info,
 			     void *ignored)
 {
 	acpi_status status;
+	acpi_event_status event_status;
 	struct acpi_gpe_event_info *gpe_event_info;
 	u32 gpe_enabled_count;
 	u32 gpe_index;
+	u32 gpe_number;
 	u32 i;
 	u32 j;
 
@@ -470,30 +472,40 @@ acpi_ev_initialize_gpe_block(struct acpi_gpe_xrupt_info *gpe_xrupt_info,
 
 			gpe_index = (i * ACPI_GPE_REGISTER_WIDTH) + j;
 			gpe_event_info = &gpe_block->event_info[gpe_index];
+			gpe_number = gpe_block->block_base_number + gpe_index;
 
 			/*
 			 * Ignore GPEs that have no corresponding _Lxx/_Exx method
-			 * and GPEs that are used to wake the system
+			 * and GPEs that are used for wakeup
 			 */
-			if ((ACPI_GPE_DISPATCH_TYPE(gpe_event_info->flags) ==
-			     ACPI_GPE_DISPATCH_NONE)
-			    || (ACPI_GPE_DISPATCH_TYPE(gpe_event_info->flags) ==
-				ACPI_GPE_DISPATCH_HANDLER)
-			    || (ACPI_GPE_DISPATCH_TYPE(gpe_event_info->flags) ==
-				ACPI_GPE_DISPATCH_RAW_HANDLER)
+			if ((ACPI_GPE_DISPATCH_TYPE(gpe_event_info->flags) !=
+			     ACPI_GPE_DISPATCH_METHOD)
 			    || (gpe_event_info->flags & ACPI_GPE_CAN_WAKE)) {
 				continue;
 			}
 
+			event_status = 0;
+			(void)acpi_hw_get_gpe_status(gpe_event_info,
+						     &event_status);
+
 			status = acpi_ev_add_gpe_reference(gpe_event_info);
 			if (ACPI_FAILURE(status)) {
 				ACPI_EXCEPTION((AE_INFO, status,
 					"Could not enable GPE 0x%02X",
-					gpe_index +
-					gpe_block->block_base_number));
+					gpe_number));
 				continue;
 			}
 
+			gpe_event_info->flags |= ACPI_GPE_AUTO_ENABLED;
+
+			if (event_status & ACPI_EVENT_FLAG_STATUS_SET) {
+				ACPI_INFO(("GPE 0x%02X active on init",
+					   gpe_number));
+				(void)acpi_ev_gpe_dispatch(gpe_block->node,
+							   gpe_event_info,
+							   gpe_number);
+			}
+
 			gpe_enabled_count++;
 		}
 	}
diff --git a/drivers/acpi/acpica/evxfgpe.c b/drivers/acpi/acpica/evxfgpe.c
index 57718a3e029a..67c7c4ce276c 100644
--- a/drivers/acpi/acpica/evxfgpe.c
+++ b/drivers/acpi/acpica/evxfgpe.c
@@ -435,6 +435,14 @@ acpi_setup_gpe_for_wake(acpi_handle wake_device,
 		 */
 		gpe_event_info->flags =
 		    (ACPI_GPE_DISPATCH_NOTIFY | ACPI_GPE_LEVEL_TRIGGERED);
+	} else if (gpe_event_info->flags & ACPI_GPE_AUTO_ENABLED) {
+		/*
+		 * A reference to this GPE has been added during the GPE block
+		 * initialization, so drop it now to prevent the GPE from being
+		 * permanently enabled and clear its ACPI_GPE_AUTO_ENABLED flag.
+		 */
+		(void)acpi_ev_remove_gpe_reference(gpe_event_info);
+		gpe_event_info->flags &= ~ACPI_GPE_AUTO_ENABLED;
 	}
 
 	/*
diff --git a/drivers/acpi/acpica/excreate.c b/drivers/acpi/acpica/excreate.c
index d43d7da4c734..b8adb11f1b07 100644
--- a/drivers/acpi/acpica/excreate.c
+++ b/drivers/acpi/acpica/excreate.c
@@ -87,68 +87,40 @@ acpi_status acpi_ex_create_alias(struct acpi_walk_state *walk_state)
 				  target_node->object);
 	}
 
-	/*
-	 * For objects that can never change (i.e., the NS node will
-	 * permanently point to the same object), we can simply attach
-	 * the object to the new NS node. For other objects (such as
-	 * Integers, buffers, etc.), we have to point the Alias node
-	 * to the original Node.
-	 */
-	switch (target_node->type) {
+	/* Ensure that the target node is valid */
 
-		/* For these types, the sub-object can change dynamically via a Store */
+	if (!target_node) {
+		return_ACPI_STATUS(AE_NULL_OBJECT);
+	}
 
-	case ACPI_TYPE_INTEGER:
-	case ACPI_TYPE_STRING:
-	case ACPI_TYPE_BUFFER:
-	case ACPI_TYPE_PACKAGE:
-	case ACPI_TYPE_BUFFER_FIELD:
-		/*
-		 * These types open a new scope, so we need the NS node in order to access
-		 * any children.
-		 */
-	case ACPI_TYPE_DEVICE:
-	case ACPI_TYPE_POWER:
-	case ACPI_TYPE_PROCESSOR:
-	case ACPI_TYPE_THERMAL:
-	case ACPI_TYPE_LOCAL_SCOPE:
-		/*
-		 * The new alias has the type ALIAS and points to the original
-		 * NS node, not the object itself.
-		 */
-		alias_node->type = ACPI_TYPE_LOCAL_ALIAS;
-		alias_node->object =
-		    ACPI_CAST_PTR(union acpi_operand_object, target_node);
-		break;
+	/* Construct the alias object (a namespace node) */
 
+	switch (target_node->type) {
 	case ACPI_TYPE_METHOD:
 		/*
-		 * Control method aliases need to be differentiated
+		 * Control method aliases need to be differentiated with
+		 * a special type
 		 */
 		alias_node->type = ACPI_TYPE_LOCAL_METHOD_ALIAS;
-		alias_node->object =
-		    ACPI_CAST_PTR(union acpi_operand_object, target_node);
 		break;
 
 	default:
-
-		/* Attach the original source object to the new Alias Node */
-
 		/*
-		 * The new alias assumes the type of the target, and it points
-		 * to the same object. The reference count of the object has an
-		 * additional reference to prevent deletion out from under either the
-		 * target node or the alias Node
+		 * All other object types.
+		 *
+		 * The new alias has the type ALIAS and points to the original
+		 * NS node, not the object itself.
 		 */
-		status = acpi_ns_attach_object(alias_node,
-					       acpi_ns_get_attached_object
-					       (target_node),
-					       target_node->type);
+		alias_node->type = ACPI_TYPE_LOCAL_ALIAS;
+		alias_node->object =
+		    ACPI_CAST_PTR(union acpi_operand_object, target_node);
 		break;
 	}
 
 	/* Since both operands are Nodes, we don't need to delete them */
 
+	alias_node->object =
+	    ACPI_CAST_PTR(union acpi_operand_object, target_node);
 	return_ACPI_STATUS(status);
 }
 
diff --git a/drivers/acpi/acpica/exdump.c b/drivers/acpi/acpica/exdump.c
index 44092f744477..83398dc4b7c2 100644
--- a/drivers/acpi/acpica/exdump.c
+++ b/drivers/acpi/acpica/exdump.c
@@ -102,7 +102,7 @@ static struct acpi_exdump_info acpi_ex_dump_package[6] = {
 	{ACPI_EXD_INIT, ACPI_EXD_TABLE_SIZE(acpi_ex_dump_package), NULL},
 	{ACPI_EXD_NODE, ACPI_EXD_OFFSET(package.node), "Parent Node"},
 	{ACPI_EXD_UINT8, ACPI_EXD_OFFSET(package.flags), "Flags"},
-	{ACPI_EXD_UINT32, ACPI_EXD_OFFSET(package.count), "Elements"},
+	{ACPI_EXD_UINT32, ACPI_EXD_OFFSET(package.count), "Element Count"},
 	{ACPI_EXD_POINTER, ACPI_EXD_OFFSET(package.elements), "Element List"},
 	{ACPI_EXD_PACKAGE, 0, NULL}
 };
@@ -384,6 +384,10 @@ acpi_ex_dump_object(union acpi_operand_object *obj_desc,
 	count = info->offset;
 
 	while (count) {
+		if (!obj_desc) {
+			return;
+		}
+
 		target = ACPI_ADD_PTR(u8, obj_desc, info->offset);
 		name = info->name;
 
@@ -469,9 +473,9 @@ acpi_ex_dump_object(union acpi_operand_object *obj_desc,
 			start = *ACPI_CAST_PTR(void *, target);
 			next = start;
 
-			acpi_os_printf("%20s : %p", name, next);
+			acpi_os_printf("%20s : %p ", name, next);
 			if (next) {
-				acpi_os_printf("(%s %2.2X)",
+				acpi_os_printf("%s (Type %2.2X)",
 					       acpi_ut_get_object_type_name
 					       (next), next->common.type);
 
@@ -493,6 +497,8 @@ acpi_ex_dump_object(union acpi_operand_object *obj_desc,
 						break;
 					}
 				}
+			} else {
+				acpi_os_printf("- No attached objects");
 			}
 
 			acpi_os_printf("\n");
@@ -1129,7 +1135,9 @@ acpi_ex_dump_package_obj(union acpi_operand_object *obj_desc,
 
 	default:
 
-		acpi_os_printf("[Unknown Type] %X\n", obj_desc->common.type);
+		acpi_os_printf("[%s] Type: %2.2X\n",
+			       acpi_ut_get_type_name(obj_desc->common.type),
+			       obj_desc->common.type);
 		break;
 	}
 }
@@ -1167,11 +1175,17 @@ acpi_ex_dump_object_descriptor(union acpi_operand_object *obj_desc, u32 flags)
 		acpi_ex_dump_namespace_node((struct acpi_namespace_node *)
 					    obj_desc, flags);
 
-		acpi_os_printf("\nAttached Object (%p):\n",
-			       ((struct acpi_namespace_node *)obj_desc)->
-			       object);
-
 		obj_desc = ((struct acpi_namespace_node *)obj_desc)->object;
+		if (!obj_desc) {
+			return_VOID;
+		}
+
+		acpi_os_printf("\nAttached Object %p", obj_desc);
+		if (ACPI_GET_DESCRIPTOR_TYPE(obj_desc) == ACPI_DESC_TYPE_NAMED) {
+			acpi_os_printf(" - Namespace Node");
+		}
+
+		acpi_os_printf(":\n");
 		goto dump_object;
 	}
 
@@ -1191,6 +1205,10 @@ acpi_ex_dump_object_descriptor(union acpi_operand_object *obj_desc, u32 flags)
 
 dump_object:
 
+	if (!obj_desc) {
+		return_VOID;
+	}
+
 	/* Common Fields */
 
 	acpi_ex_dump_object(obj_desc, acpi_ex_dump_common);
diff --git a/drivers/acpi/acpica/exmisc.c b/drivers/acpi/acpica/exmisc.c
index f222a80ca38e..1e7649ce0a7b 100644
--- a/drivers/acpi/acpica/exmisc.c
+++ b/drivers/acpi/acpica/exmisc.c
@@ -265,6 +265,8 @@ acpi_ex_do_logical_numeric_op(u16 opcode,
 
 	default:
 
+		ACPI_ERROR((AE_INFO,
+			    "Invalid numeric logical opcode: %X", opcode));
 		status = AE_AML_INTERNAL;
 		break;
 	}
@@ -345,6 +347,9 @@ acpi_ex_do_logical_op(u16 opcode,
 
 	default:
 
+		ACPI_ERROR((AE_INFO,
+			    "Invalid object type for logical operator: %X",
+			    operand0->common.type));
 		status = AE_AML_INTERNAL;
 		break;
 	}
@@ -388,6 +393,8 @@ acpi_ex_do_logical_op(u16 opcode,
 
 		default:
 
+			ACPI_ERROR((AE_INFO,
+				    "Invalid comparison opcode: %X", opcode));
 			status = AE_AML_INTERNAL;
 			break;
 		}
@@ -456,6 +463,8 @@ acpi_ex_do_logical_op(u16 opcode,
 
 		default:
 
+			ACPI_ERROR((AE_INFO,
+				    "Invalid comparison opcode: %X", opcode));
 			status = AE_AML_INTERNAL;
 			break;
 		}
diff --git a/drivers/acpi/acpica/exoparg2.c b/drivers/acpi/acpica/exoparg2.c
index eecb3bff7fd7..57980b7d3594 100644
--- a/drivers/acpi/acpica/exoparg2.c
+++ b/drivers/acpi/acpica/exoparg2.c
@@ -414,6 +414,9 @@ acpi_status acpi_ex_opcode_2A_1T_1R(struct acpi_walk_state *walk_state)
 
 		default:
 
+			ACPI_ERROR((AE_INFO,
+				    "Invalid object type: %X",
+				    (operand[0])->common.type));
 			status = AE_AML_INTERNAL;
 			goto cleanup;
 		}
diff --git a/drivers/acpi/acpica/hwregs.c b/drivers/acpi/acpica/hwregs.c
index de74a4c25085..acb417b58bbb 100644
--- a/drivers/acpi/acpica/hwregs.c
+++ b/drivers/acpi/acpica/hwregs.c
@@ -107,7 +107,7 @@ acpi_hw_get_access_bit_width(u64 address,
 	    ACPI_IS_ALIGNED(reg->bit_width, 8)) {
 		access_bit_width = reg->bit_width;
 	} else if (reg->access_width) {
-		access_bit_width = (1 << (reg->access_width + 2));
+		access_bit_width = ACPI_ACCESS_BIT_WIDTH(reg->access_width);
 	} else {
 		access_bit_width =
 		    ACPI_ROUND_UP_POWER_OF_TWO_8(reg->bit_offset +
diff --git a/drivers/acpi/acpica/hwxfsleep.c b/drivers/acpi/acpica/hwxfsleep.c
index 7ef13934968f..e5c095ca6083 100644
--- a/drivers/acpi/acpica/hwxfsleep.c
+++ b/drivers/acpi/acpica/hwxfsleep.c
@@ -72,13 +72,16 @@ static acpi_status acpi_hw_sleep_dispatch(u8 sleep_state, u32 function_id);
 static struct acpi_sleep_functions acpi_sleep_dispatch[] = {
 	{ACPI_STRUCT_INIT(legacy_function,
 			  ACPI_HW_OPTIONAL_FUNCTION(acpi_hw_legacy_sleep)),
-	 ACPI_STRUCT_INIT(extended_function, acpi_hw_extended_sleep) },
+	 ACPI_STRUCT_INIT(extended_function,
+			  acpi_hw_extended_sleep)},
 	{ACPI_STRUCT_INIT(legacy_function,
 			  ACPI_HW_OPTIONAL_FUNCTION(acpi_hw_legacy_wake_prep)),
-	 ACPI_STRUCT_INIT(extended_function, acpi_hw_extended_wake_prep) },
+	 ACPI_STRUCT_INIT(extended_function,
+			  acpi_hw_extended_wake_prep)},
 	{ACPI_STRUCT_INIT(legacy_function,
 			  ACPI_HW_OPTIONAL_FUNCTION(acpi_hw_legacy_wake)),
-	 ACPI_STRUCT_INIT(extended_function, acpi_hw_extended_wake) }
+	 ACPI_STRUCT_INIT(extended_function,
+			  acpi_hw_extended_wake)}
 };
 
 /*
diff --git a/drivers/acpi/acpica/nsaccess.c b/drivers/acpi/acpica/nsaccess.c
index e5f4fa496572..f2733f51ca8d 100644
--- a/drivers/acpi/acpica/nsaccess.c
+++ b/drivers/acpi/acpica/nsaccess.c
@@ -292,6 +292,7 @@ acpi_ns_lookup(union acpi_generic_state *scope_info,
 {
 	acpi_status status;
 	char *path = pathname;
+	char *external_path;
 	struct acpi_namespace_node *prefix_node;
 	struct acpi_namespace_node *current_node = NULL;
 	struct acpi_namespace_node *this_node = NULL;
@@ -427,13 +428,22 @@ acpi_ns_lookup(union acpi_generic_state *scope_info,
 				num_carats++;
 				this_node = this_node->parent;
 				if (!this_node) {
+					/*
+					 * Current scope has no parent scope. Externalize
+					 * the internal path for error message.
+					 */
+					status =
+					    acpi_ns_externalize_name
+					    (ACPI_UINT32_MAX, pathname, NULL,
+					     &external_path);
+					if (ACPI_SUCCESS(status)) {
+						ACPI_ERROR((AE_INFO,
+							    "%s: Path has too many parent prefixes (^)",
+							    external_path));
+
+						ACPI_FREE(external_path);
+					}
 
-					/* Current scope has no parent scope */
-
-					ACPI_ERROR((AE_INFO,
-						    "%s: Path has too many parent prefixes (^) "
-						    "- reached beyond root node",
-						    pathname));
 					return_ACPI_STATUS(AE_NOT_FOUND);
 				}
 			}
@@ -634,6 +644,12 @@ acpi_ns_lookup(union acpi_generic_state *scope_info,
 					    this_node->object;
 				}
 			}
+#ifdef ACPI_ASL_COMPILER
+			if (!acpi_gbl_disasm_flag &&
+			    (this_node->flags & ANOBJ_IS_EXTERNAL)) {
+				this_node->flags |= IMPLICIT_EXTERNAL;
+			}
+#endif
 		}
 
 		/* Special handling for the last segment (num_segments == 0) */
diff --git a/drivers/acpi/acpica/nsarguments.c b/drivers/acpi/acpica/nsarguments.c
index 9095d51f6b37..67b7370dcae5 100644
--- a/drivers/acpi/acpica/nsarguments.c
+++ b/drivers/acpi/acpica/nsarguments.c
@@ -69,9 +69,14 @@ void acpi_ns_check_argument_types(struct acpi_evaluate_info *info)
 	u8 user_arg_type;
 	u32 i;
 
-	/* If not a predefined name, cannot typecheck args */
-
-	if (!info->predefined) {
+	/*
+	 * If not a predefined name, cannot typecheck args, because
+	 * we have no idea what argument types are expected.
+	 * Also, ignore typecheck if warnings/errors if this method
+	 * has already been evaluated at least once -- in order
+	 * to suppress repetitive messages.
+	 */
+	if (!info->predefined || (info->node->flags & ANOBJ_EVALUATED)) {
 		return;
 	}
 
@@ -93,6 +98,10 @@ void acpi_ns_check_argument_types(struct acpi_evaluate_info *info)
 					      acpi_ut_get_type_name
 					      (user_arg_type),
 					      acpi_ut_get_type_name(arg_type)));
+
+			/* Prevent any additional typechecking for this method */
+
+			info->node->flags |= ANOBJ_EVALUATED;
 		}
 	}
 }
@@ -121,7 +130,7 @@ acpi_ns_check_acpi_compliance(char *pathname,
 	u32 aml_param_count;
 	u32 required_param_count;
 
-	if (!predefined) {
+	if (!predefined || (node->flags & ANOBJ_EVALUATED)) {
 		return;
 	}
 
@@ -215,6 +224,10 @@ acpi_ns_check_argument_count(char *pathname,
 	u32 aml_param_count;
 	u32 required_param_count;
 
+	if (node->flags & ANOBJ_EVALUATED) {
+		return;
+	}
+
 	if (!predefined) {
 		/*
 		 * Not a predefined name. Check the incoming user argument count
diff --git a/drivers/acpi/acpica/nsinit.c b/drivers/acpi/acpica/nsinit.c
index ce33e7297ea7..9c6297949712 100644
--- a/drivers/acpi/acpica/nsinit.c
+++ b/drivers/acpi/acpica/nsinit.c
@@ -396,6 +396,20 @@ acpi_ns_init_one_object(acpi_handle obj_handle,
 
 		info->package_init++;
 		status = acpi_ds_get_package_arguments(obj_desc);
+		if (ACPI_FAILURE(status)) {
+			break;
+		}
+
+		/*
+		 * Resolve all named references in package objects (and all
+		 * sub-packages). This action has been deferred until the entire
+		 * namespace has been loaded, in order to support external and
+		 * forward references from individual package elements (05/2017).
+		 */
+		status = acpi_ut_walk_package_tree(obj_desc, NULL,
+						   acpi_ds_init_package_element,
+						   NULL);
+		obj_desc->package.flags |= AOPOBJ_DATA_VALID;
 		break;
 
 	default:
diff --git a/drivers/acpi/acpica/nsnames.c b/drivers/acpi/acpica/nsnames.c
index aa16aeaa8937..a410760a0308 100644
--- a/drivers/acpi/acpica/nsnames.c
+++ b/drivers/acpi/acpica/nsnames.c
@@ -89,7 +89,14 @@ acpi_size acpi_ns_get_pathname_length(struct acpi_namespace_node *node)
 {
 	acpi_size size;
 
-	ACPI_FUNCTION_ENTRY();
+	/* Validate the Node */
+
+	if (ACPI_GET_DESCRIPTOR_TYPE(node) != ACPI_DESC_TYPE_NAMED) {
+		ACPI_ERROR((AE_INFO,
+			    "Invalid/cached reference target node: %p, descriptor type %d",
+			    node, ACPI_GET_DESCRIPTOR_TYPE(node)));
+		return (0);
+	}
 
 	size = acpi_ns_build_normalized_path(node, NULL, 0, FALSE);
 	return (size);
diff --git a/drivers/acpi/acpica/nsprepkg.c b/drivers/acpi/acpica/nsprepkg.c
index 4954cb6c9090..a8ea8fb1d299 100644
--- a/drivers/acpi/acpica/nsprepkg.c
+++ b/drivers/acpi/acpica/nsprepkg.c
@@ -614,6 +614,8 @@ acpi_ns_check_package_list(struct acpi_evaluate_info *info,
 
 		default:	/* Should not get here, type was validated by caller */
 
+			ACPI_ERROR((AE_INFO, "Invalid Package type: %X",
+				    package->ret_info.type));
 			return (AE_AML_INTERNAL);
 		}
 
diff --git a/drivers/acpi/acpica/nsxfeval.c b/drivers/acpi/acpica/nsxfeval.c
index 538c61677c10..783f4c838aee 100644
--- a/drivers/acpi/acpica/nsxfeval.c
+++ b/drivers/acpi/acpica/nsxfeval.c
@@ -100,9 +100,13 @@ acpi_evaluate_object_typed(acpi_handle handle,
 		free_buffer_on_error = TRUE;
 	}
 
-	status = acpi_get_handle(handle, pathname, &target_handle);
-	if (ACPI_FAILURE(status)) {
-		return_ACPI_STATUS(status);
+	if (pathname) {
+		status = acpi_get_handle(handle, pathname, &target_handle);
+		if (ACPI_FAILURE(status)) {
+			return_ACPI_STATUS(status);
+		}
+	} else {
+		target_handle = handle;
 	}
 
 	full_pathname = acpi_ns_get_external_pathname(target_handle);
diff --git a/drivers/acpi/acpica/psloop.c b/drivers/acpi/acpica/psloop.c
index b4224005783c..bb04dec168ad 100644
--- a/drivers/acpi/acpica/psloop.c
+++ b/drivers/acpi/acpica/psloop.c
@@ -164,6 +164,11 @@ acpi_ps_get_arguments(struct acpi_walk_state *walk_state,
 			INCREMENT_ARG_LIST(walk_state->arg_types);
 		}
 
+		ACPI_DEBUG_PRINT((ACPI_DB_PARSE,
+				  "Final argument count: %u pass %u\n",
+				  walk_state->arg_count,
+				  walk_state->pass_number));
+
 		/*
 		 * Handle executable code at "module-level". This refers to
 		 * executable opcodes that appear outside of any control method.
@@ -277,6 +282,11 @@ acpi_ps_get_arguments(struct acpi_walk_state *walk_state,
 			     AML_NAME_OP)
 			    && (walk_state->pass_number <=
 				ACPI_IMODE_LOAD_PASS2)) {
+				ACPI_DEBUG_PRINT((ACPI_DB_PARSE,
+						  "Setup Package/Buffer: Pass %u, AML Ptr: %p\n",
+						  walk_state->pass_number,
+						  aml_op_start));
+
 				/*
 				 * Skip parsing of Buffers and Packages because we don't have
 				 * enough info in the first pass to parse them correctly.
@@ -570,6 +580,10 @@ acpi_status acpi_ps_parse_loop(struct acpi_walk_state *walk_state)
 
 		/* Check for arguments that need to be processed */
 
+		ACPI_DEBUG_PRINT((ACPI_DB_PARSE,
+				  "Parseloop: argument count: %u\n",
+				  walk_state->arg_count));
+
 		if (walk_state->arg_count) {
 			/*
 			 * There are arguments (complex ones), push Op and
diff --git a/drivers/acpi/acpica/psobject.c b/drivers/acpi/acpica/psobject.c
index ef6384e374fc..0bef6df71bba 100644
--- a/drivers/acpi/acpica/psobject.c
+++ b/drivers/acpi/acpica/psobject.c
@@ -359,6 +359,32 @@ acpi_ps_create_op(struct acpi_walk_state *walk_state,
 		    acpi_ps_build_named_op(walk_state, aml_op_start, op,
 					   &named_op);
 		acpi_ps_free_op(op);
+
+#ifdef ACPI_ASL_COMPILER
+		if (acpi_gbl_disasm_flag
+		    && walk_state->opcode == AML_EXTERNAL_OP
+		    && status == AE_NOT_FOUND) {
+			/*
+			 * If parsing of AML_EXTERNAL_OP's name path fails, then skip
+			 * past this opcode and keep parsing. This is a much better
+			 * alternative than to abort the entire disassembler. At this
+			 * point, the parser_state is at the end of the namepath of the
+			 * external declaration opcode. Setting walk_state->Aml to
+			 * walk_state->parser_state.Aml + 2 moves increments the
+			 * walk_state->Aml past the object type and the paramcount of the
+			 * external opcode. For the error message, only print the AML
+			 * offset. We could attempt to print the name but this may cause
+			 * a segmentation fault when printing the namepath because the
+			 * AML may be incorrect.
+			 */
+			acpi_os_printf
+			    ("// Invalid external declaration at AML offset 0x%x.\n",
+			     walk_state->aml -
+			     walk_state->parser_state.aml_start);
+			walk_state->aml = walk_state->parser_state.aml + 2;
+			return_ACPI_STATUS(AE_CTRL_PARSE_CONTINUE);
+		}
+#endif
 		if (ACPI_FAILURE(status)) {
 			return_ACPI_STATUS(status);
 		}
diff --git a/drivers/acpi/acpica/rsxface.c b/drivers/acpi/acpica/rsxface.c
index 59a4f9ed06a7..be65e65e216e 100644
--- a/drivers/acpi/acpica/rsxface.c
+++ b/drivers/acpi/acpica/rsxface.c
@@ -615,7 +615,7 @@ ACPI_EXPORT_SYMBOL(acpi_walk_resource_buffer)
  *                                device we are querying
  *              name            - Method name of the resources we want.
  *                                (METHOD_NAME__CRS, METHOD_NAME__PRS, or
- *                                METHOD_NAME__AEI)
+ *                                METHOD_NAME__AEI or METHOD_NAME__DMA)
  *              user_function   - Called for each resource
  *              context         - Passed to user_function
  *
@@ -641,11 +641,12 @@ acpi_walk_resources(acpi_handle device_handle,
 	if (!device_handle || !user_function || !name ||
 	    (!ACPI_COMPARE_NAME(name, METHOD_NAME__CRS) &&
 	     !ACPI_COMPARE_NAME(name, METHOD_NAME__PRS) &&
-	     !ACPI_COMPARE_NAME(name, METHOD_NAME__AEI))) {
+	     !ACPI_COMPARE_NAME(name, METHOD_NAME__AEI) &&
+	     !ACPI_COMPARE_NAME(name, METHOD_NAME__DMA))) {
 		return_ACPI_STATUS(AE_BAD_PARAMETER);
 	}
 
-	/* Get the _CRS/_PRS/_AEI resource list */
+	/* Get the _CRS/_PRS/_AEI/_DMA resource list */
 
 	buffer.length = ACPI_ALLOCATE_LOCAL_BUFFER;
 	status = acpi_rs_get_method_data(device_handle, name, &buffer);
diff --git a/drivers/acpi/acpica/tbdata.c b/drivers/acpi/acpica/tbdata.c
index c9d6fa6d7cc6..b19a2f0ea331 100644
--- a/drivers/acpi/acpica/tbdata.c
+++ b/drivers/acpi/acpica/tbdata.c
@@ -50,6 +50,57 @@
 #define _COMPONENT          ACPI_TABLES
 ACPI_MODULE_NAME("tbdata")
 
+/* Local prototypes */
+static acpi_status
+acpi_tb_check_duplication(struct acpi_table_desc *table_desc, u32 *table_index);
+
+static u8
+acpi_tb_compare_tables(struct acpi_table_desc *table_desc, u32 table_index);
+
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_tb_compare_tables
+ *
+ * PARAMETERS:  table_desc          - Table 1 descriptor to be compared
+ *              table_index         - Index of table 2 to be compared
+ *
+ * RETURN:      TRUE if both tables are identical.
+ *
+ * DESCRIPTION: This function compares a table with another table that has
+ *              already been installed in the root table list.
+ *
+ ******************************************************************************/
+
+static u8
+acpi_tb_compare_tables(struct acpi_table_desc *table_desc, u32 table_index)
+{
+	acpi_status status = AE_OK;
+	u8 is_identical;
+	struct acpi_table_header *table;
+	u32 table_length;
+	u8 table_flags;
+
+	status =
+	    acpi_tb_acquire_table(&acpi_gbl_root_table_list.tables[table_index],
+				  &table, &table_length, &table_flags);
+	if (ACPI_FAILURE(status)) {
+		return (FALSE);
+	}
+
+	/*
+	 * Check for a table match on the entire table length,
+	 * not just the header.
+	 */
+	is_identical = (u8)((table_desc->length != table_length ||
+			     memcmp(table_desc->pointer, table, table_length)) ?
+			    FALSE : TRUE);
+
+	/* Release the acquired table */
+
+	acpi_tb_release_table(table, table_length, table_flags);
+	return (is_identical);
+}
+
 /*******************************************************************************
  *
  * FUNCTION:    acpi_tb_init_table_descriptor
@@ -64,6 +115,7 @@ ACPI_MODULE_NAME("tbdata")
  * DESCRIPTION: Initialize a new table descriptor
  *
  ******************************************************************************/
+
 void
 acpi_tb_init_table_descriptor(struct acpi_table_desc *table_desc,
 			      acpi_physical_address address,
@@ -338,7 +390,7 @@ void acpi_tb_invalidate_table(struct acpi_table_desc *table_desc)
 acpi_status acpi_tb_validate_temp_table(struct acpi_table_desc *table_desc)
 {
 
-	if (!table_desc->pointer && !acpi_gbl_verify_table_checksum) {
+	if (!table_desc->pointer && !acpi_gbl_enable_table_validation) {
 		/*
 		 * Only validates the header of the table.
 		 * Note that Length contains the size of the mapping after invoking
@@ -354,22 +406,100 @@ acpi_status acpi_tb_validate_temp_table(struct acpi_table_desc *table_desc)
 	return (acpi_tb_validate_table(table_desc));
 }
 
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_tb_check_duplication
+ *
+ * PARAMETERS:  table_desc          - Table descriptor
+ *              table_index         - Where the table index is returned
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Avoid installing duplicated tables. However table override and
+ *              user aided dynamic table load is allowed, thus comparing the
+ *              address of the table is not sufficient, and checking the entire
+ *              table content is required.
+ *
+ ******************************************************************************/
+
+static acpi_status
+acpi_tb_check_duplication(struct acpi_table_desc *table_desc, u32 *table_index)
+{
+	u32 i;
+
+	ACPI_FUNCTION_TRACE(tb_check_duplication);
+
+	/* Check if table is already registered */
+
+	for (i = 0; i < acpi_gbl_root_table_list.current_table_count; ++i) {
+
+		/* Do not compare with unverified tables */
+
+		if (!
+		    (acpi_gbl_root_table_list.tables[i].
+		     flags & ACPI_TABLE_IS_VERIFIED)) {
+			continue;
+		}
+
+		/*
+		 * Check for a table match on the entire table length,
+		 * not just the header.
+		 */
+		if (!acpi_tb_compare_tables(table_desc, i)) {
+			continue;
+		}
+
+		/*
+		 * Note: the current mechanism does not unregister a table if it is
+		 * dynamically unloaded. The related namespace entries are deleted,
+		 * but the table remains in the root table list.
+		 *
+		 * The assumption here is that the number of different tables that
+		 * will be loaded is actually small, and there is minimal overhead
+		 * in just keeping the table in case it is needed again.
+		 *
+		 * If this assumption changes in the future (perhaps on large
+		 * machines with many table load/unload operations), tables will
+		 * need to be unregistered when they are unloaded, and slots in the
+		 * root table list should be reused when empty.
+		 */
+		if (acpi_gbl_root_table_list.tables[i].flags &
+		    ACPI_TABLE_IS_LOADED) {
+
+			/* Table is still loaded, this is an error */
+
+			return_ACPI_STATUS(AE_ALREADY_EXISTS);
+		} else {
+			*table_index = i;
+			return_ACPI_STATUS(AE_CTRL_TERMINATE);
+		}
+	}
+
+	/* Indicate no duplication to the caller */
+
+	return_ACPI_STATUS(AE_OK);
+}
+
 /******************************************************************************
  *
  * FUNCTION:    acpi_tb_verify_temp_table
  *
  * PARAMETERS:  table_desc          - Table descriptor
  *              signature           - Table signature to verify
+ *              table_index         - Where the table index is returned
  *
  * RETURN:      Status
  *
  * DESCRIPTION: This function is called to validate and verify the table, the
  *              returned table descriptor is in "VALIDATED" state.
+ *              Note that 'TableIndex' is required to be set to !NULL to
+ *              enable duplication check.
  *
  *****************************************************************************/
 
 acpi_status
-acpi_tb_verify_temp_table(struct acpi_table_desc *table_desc, char *signature)
+acpi_tb_verify_temp_table(struct acpi_table_desc *table_desc,
+			  char *signature, u32 *table_index)
 {
 	acpi_status status = AE_OK;
 
@@ -392,9 +522,10 @@ acpi_tb_verify_temp_table(struct acpi_table_desc *table_desc, char *signature)
 		goto invalidate_and_exit;
 	}
 
-	/* Verify the checksum */
+	if (acpi_gbl_enable_table_validation) {
+
+		/* Verify the checksum */
 
-	if (acpi_gbl_verify_table_checksum) {
 		status =
 		    acpi_tb_verify_checksum(table_desc->pointer,
 					    table_desc->length);
@@ -411,9 +542,34 @@ acpi_tb_verify_temp_table(struct acpi_table_desc *table_desc, char *signature)
 
 			goto invalidate_and_exit;
 		}
+
+		/* Avoid duplications */
+
+		if (table_index) {
+			status =
+			    acpi_tb_check_duplication(table_desc, table_index);
+			if (ACPI_FAILURE(status)) {
+				if (status != AE_CTRL_TERMINATE) {
+					ACPI_EXCEPTION((AE_INFO, AE_NO_MEMORY,
+							"%4.4s 0x%8.8X%8.8X"
+							" Table is duplicated",
+							acpi_ut_valid_nameseg
+							(table_desc->signature.
+							 ascii) ? table_desc->
+							signature.
+							ascii : "????",
+							ACPI_FORMAT_UINT64
+							(table_desc->address)));
+				}
+
+				goto invalidate_and_exit;
+			}
+		}
+
+		table_desc->flags |= ACPI_TABLE_IS_VERIFIED;
 	}
 
-	return_ACPI_STATUS(AE_OK);
+	return_ACPI_STATUS(status);
 
 invalidate_and_exit:
 	acpi_tb_invalidate_table(table_desc);
@@ -436,6 +592,8 @@ acpi_status acpi_tb_resize_root_table_list(void)
 {
 	struct acpi_table_desc *tables;
 	u32 table_count;
+	u32 current_table_count, max_table_count;
+	u32 i;
 
 	ACPI_FUNCTION_TRACE(tb_resize_root_table_list);
 
@@ -455,8 +613,8 @@ acpi_status acpi_tb_resize_root_table_list(void)
 		table_count = acpi_gbl_root_table_list.current_table_count;
 	}
 
-	tables = ACPI_ALLOCATE_ZEROED(((acpi_size)table_count +
-				       ACPI_ROOT_TABLE_SIZE_INCREMENT) *
+	max_table_count = table_count + ACPI_ROOT_TABLE_SIZE_INCREMENT;
+	tables = ACPI_ALLOCATE_ZEROED(((acpi_size)max_table_count) *
 				      sizeof(struct acpi_table_desc));
 	if (!tables) {
 		ACPI_ERROR((AE_INFO,
@@ -466,9 +624,16 @@ acpi_status acpi_tb_resize_root_table_list(void)
 
 	/* Copy and free the previous table array */
 
+	current_table_count = 0;
 	if (acpi_gbl_root_table_list.tables) {
-		memcpy(tables, acpi_gbl_root_table_list.tables,
-		       (acpi_size)table_count * sizeof(struct acpi_table_desc));
+		for (i = 0; i < table_count; i++) {
+			if (acpi_gbl_root_table_list.tables[i].address) {
+				memcpy(tables + current_table_count,
+				       acpi_gbl_root_table_list.tables + i,
+				       sizeof(struct acpi_table_desc));
+				current_table_count++;
+			}
+		}
 
 		if (acpi_gbl_root_table_list.flags & ACPI_ROOT_ORIGIN_ALLOCATED) {
 			ACPI_FREE(acpi_gbl_root_table_list.tables);
@@ -476,8 +641,8 @@ acpi_status acpi_tb_resize_root_table_list(void)
 	}
 
 	acpi_gbl_root_table_list.tables = tables;
-	acpi_gbl_root_table_list.max_table_count =
-	    table_count + ACPI_ROOT_TABLE_SIZE_INCREMENT;
+	acpi_gbl_root_table_list.max_table_count = max_table_count;
+	acpi_gbl_root_table_list.current_table_count = current_table_count;
 	acpi_gbl_root_table_list.flags |= ACPI_ROOT_ORIGIN_ALLOCATED;
 
 	return_ACPI_STATUS(AE_OK);
@@ -818,13 +983,9 @@ acpi_tb_load_table(u32 table_index, struct acpi_namespace_node *parent_node)
 		acpi_ev_update_gpes(owner_id);
 	}
 
-	/* Invoke table handler if present */
-
-	if (acpi_gbl_table_handler) {
-		(void)acpi_gbl_table_handler(ACPI_TABLE_EVENT_LOAD, table,
-					     acpi_gbl_table_handler_context);
-	}
+	/* Invoke table handler */
 
+	acpi_tb_notify_table(ACPI_TABLE_EVENT_LOAD, table);
 	return_ACPI_STATUS(status);
 }
 
@@ -894,15 +1055,11 @@ acpi_status acpi_tb_unload_table(u32 table_index)
 		return_ACPI_STATUS(AE_NOT_EXIST);
 	}
 
-	/* Invoke table handler if present */
+	/* Invoke table handler */
 
-	if (acpi_gbl_table_handler) {
-		status = acpi_get_table_by_index(table_index, &table);
-		if (ACPI_SUCCESS(status)) {
-			(void)acpi_gbl_table_handler(ACPI_TABLE_EVENT_UNLOAD,
-						     table,
-						     acpi_gbl_table_handler_context);
-		}
+	status = acpi_get_table_by_index(table_index, &table);
+	if (ACPI_SUCCESS(status)) {
+		acpi_tb_notify_table(ACPI_TABLE_EVENT_UNLOAD, table);
 	}
 
 	/* Delete the portion of the namespace owned by this table */
@@ -918,3 +1075,26 @@ acpi_status acpi_tb_unload_table(u32 table_index)
 }
 
 ACPI_EXPORT_SYMBOL(acpi_tb_unload_table)
+
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_tb_notify_table
+ *
+ * PARAMETERS:  event               - Table event
+ *              table               - Validated table pointer
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Notify a table event to the users.
+ *
+ ******************************************************************************/
+
+void acpi_tb_notify_table(u32 event, void *table)
+{
+	/* Invoke table handler if present */
+
+	if (acpi_gbl_table_handler) {
+		(void)acpi_gbl_table_handler(event, table,
+					     acpi_gbl_table_handler_context);
+	}
+}
diff --git a/drivers/acpi/acpica/tbinstal.c b/drivers/acpi/acpica/tbinstal.c
index 4620f3c68c13..0dfc0ac3c141 100644
--- a/drivers/acpi/acpica/tbinstal.c
+++ b/drivers/acpi/acpica/tbinstal.c
@@ -48,54 +48,6 @@
 #define _COMPONENT          ACPI_TABLES
 ACPI_MODULE_NAME("tbinstal")
 
-/* Local prototypes */
-static u8
-acpi_tb_compare_tables(struct acpi_table_desc *table_desc, u32 table_index);
-
-/*******************************************************************************
- *
- * FUNCTION:    acpi_tb_compare_tables
- *
- * PARAMETERS:  table_desc          - Table 1 descriptor to be compared
- *              table_index         - Index of table 2 to be compared
- *
- * RETURN:      TRUE if both tables are identical.
- *
- * DESCRIPTION: This function compares a table with another table that has
- *              already been installed in the root table list.
- *
- ******************************************************************************/
-
-static u8
-acpi_tb_compare_tables(struct acpi_table_desc *table_desc, u32 table_index)
-{
-	acpi_status status = AE_OK;
-	u8 is_identical;
-	struct acpi_table_header *table;
-	u32 table_length;
-	u8 table_flags;
-
-	status =
-	    acpi_tb_acquire_table(&acpi_gbl_root_table_list.tables[table_index],
-				  &table, &table_length, &table_flags);
-	if (ACPI_FAILURE(status)) {
-		return (FALSE);
-	}
-
-	/*
-	 * Check for a table match on the entire table length,
-	 * not just the header.
-	 */
-	is_identical = (u8)((table_desc->length != table_length ||
-			     memcmp(table_desc->pointer, table, table_length)) ?
-			    FALSE : TRUE);
-
-	/* Release the acquired table */
-
-	acpi_tb_release_table(table, table_length, table_flags);
-	return (is_identical);
-}
-
 /*******************************************************************************
  *
  * FUNCTION:    acpi_tb_install_table_with_override
@@ -112,7 +64,6 @@ acpi_tb_compare_tables(struct acpi_table_desc *table_desc, u32 table_index)
  *              table array.
  *
  ******************************************************************************/
-
 void
 acpi_tb_install_table_with_override(struct acpi_table_desc *new_table_desc,
 				    u8 override, u32 *table_index)
@@ -210,95 +161,29 @@ acpi_tb_install_standard_table(acpi_physical_address address,
 		goto release_and_exit;
 	}
 
-	/* Validate and verify a table before installation */
-
-	status = acpi_tb_verify_temp_table(&new_table_desc, NULL);
-	if (ACPI_FAILURE(status)) {
-		goto release_and_exit;
-	}
-
 	/* Acquire the table lock */
 
 	(void)acpi_ut_acquire_mutex(ACPI_MTX_TABLES);
 
-	if (reload) {
-		/*
-		 * Validate the incoming table signature.
-		 *
-		 * 1) Originally, we checked the table signature for "SSDT" or "PSDT".
-		 * 2) We added support for OEMx tables, signature "OEM".
-		 * 3) Valid tables were encountered with a null signature, so we just
-		 *    gave up on validating the signature, (05/2008).
-		 * 4) We encountered non-AML tables such as the MADT, which caused
-		 *    interpreter errors and kernel faults. So now, we once again allow
-		 *    only "SSDT", "OEMx", and now, also a null signature. (05/2011).
-		 */
-		if ((new_table_desc.signature.ascii[0] != 0x00) &&
-		    (!ACPI_COMPARE_NAME
-		     (&new_table_desc.signature, ACPI_SIG_SSDT))
-		    && (strncmp(new_table_desc.signature.ascii, "OEM", 3))) {
-			ACPI_BIOS_ERROR((AE_INFO,
-					 "Table has invalid signature [%4.4s] (0x%8.8X), "
-					 "must be SSDT or OEMx",
-					 acpi_ut_valid_nameseg(new_table_desc.
-							       signature.
-							       ascii) ?
-					 new_table_desc.signature.
-					 ascii : "????",
-					 new_table_desc.signature.integer));
-
-			status = AE_BAD_SIGNATURE;
-			goto unlock_and_exit;
-		}
-
-		/* Check if table is already registered */
-
-		for (i = 0; i < acpi_gbl_root_table_list.current_table_count;
-		     ++i) {
-			/*
-			 * Check for a table match on the entire table length,
-			 * not just the header.
-			 */
-			if (!acpi_tb_compare_tables(&new_table_desc, i)) {
-				continue;
-			}
+	/* Validate and verify a table before installation */
 
+	status = acpi_tb_verify_temp_table(&new_table_desc, NULL, &i);
+	if (ACPI_FAILURE(status)) {
+		if (status == AE_CTRL_TERMINATE) {
 			/*
-			 * Note: the current mechanism does not unregister a table if it is
-			 * dynamically unloaded. The related namespace entries are deleted,
-			 * but the table remains in the root table list.
-			 *
-			 * The assumption here is that the number of different tables that
-			 * will be loaded is actually small, and there is minimal overhead
-			 * in just keeping the table in case it is needed again.
-			 *
-			 * If this assumption changes in the future (perhaps on large
-			 * machines with many table load/unload operations), tables will
-			 * need to be unregistered when they are unloaded, and slots in the
-			 * root table list should be reused when empty.
+			 * Table was unloaded, allow it to be reloaded.
+			 * As we are going to return AE_OK to the caller, we should
+			 * take the responsibility of freeing the input descriptor.
+			 * Refill the input descriptor to ensure
+			 * acpi_tb_install_table_with_override() can be called again to
+			 * indicate the re-installation.
 			 */
-			if (acpi_gbl_root_table_list.tables[i].flags &
-			    ACPI_TABLE_IS_LOADED) {
-
-				/* Table is still loaded, this is an error */
-
-				status = AE_ALREADY_EXISTS;
-				goto unlock_and_exit;
-			} else {
-				/*
-				 * Table was unloaded, allow it to be reloaded.
-				 * As we are going to return AE_OK to the caller, we should
-				 * take the responsibility of freeing the input descriptor.
-				 * Refill the input descriptor to ensure
-				 * acpi_tb_install_table_with_override() can be called again to
-				 * indicate the re-installation.
-				 */
-				acpi_tb_uninstall_table(&new_table_desc);
-				(void)acpi_ut_release_mutex(ACPI_MTX_TABLES);
-				*table_index = i;
-				return_ACPI_STATUS(AE_OK);
-			}
+			acpi_tb_uninstall_table(&new_table_desc);
+			(void)acpi_ut_release_mutex(ACPI_MTX_TABLES);
+			*table_index = i;
+			return_ACPI_STATUS(AE_OK);
 		}
+		goto unlock_and_exit;
 	}
 
 	/* Add the table to the global root table list */
@@ -306,14 +191,10 @@ acpi_tb_install_standard_table(acpi_physical_address address,
 	acpi_tb_install_table_with_override(&new_table_desc, override,
 					    table_index);
 
-	/* Invoke table handler if present */
+	/* Invoke table handler */
 
 	(void)acpi_ut_release_mutex(ACPI_MTX_TABLES);
-	if (acpi_gbl_table_handler) {
-		(void)acpi_gbl_table_handler(ACPI_TABLE_EVENT_INSTALL,
-					     new_table_desc.pointer,
-					     acpi_gbl_table_handler_context);
-	}
+	acpi_tb_notify_table(ACPI_TABLE_EVENT_INSTALL, new_table_desc.pointer);
 	(void)acpi_ut_acquire_mutex(ACPI_MTX_TABLES);
 
 unlock_and_exit:
@@ -382,9 +263,11 @@ void acpi_tb_override_table(struct acpi_table_desc *old_table_desc)
 
 finish_override:
 
-	/* Validate and verify a table before overriding */
-
-	status = acpi_tb_verify_temp_table(&new_table_desc, NULL);
+	/*
+	 * Validate and verify a table before overriding, no nested table
+	 * duplication check as it's too complicated and unnecessary.
+	 */
+	status = acpi_tb_verify_temp_table(&new_table_desc, NULL, NULL);
 	if (ACPI_FAILURE(status)) {
 		return;
 	}
diff --git a/drivers/acpi/acpica/tbxface.c b/drivers/acpi/acpica/tbxface.c
index 010b1c43df92..26ad596c973e 100644
--- a/drivers/acpi/acpica/tbxface.c
+++ b/drivers/acpi/acpica/tbxface.c
@@ -167,7 +167,8 @@ ACPI_EXPORT_SYMBOL_INIT(acpi_initialize_tables)
 acpi_status ACPI_INIT_FUNCTION acpi_reallocate_root_table(void)
 {
 	acpi_status status;
-	u32 i;
+	struct acpi_table_desc *table_desc;
+	u32 i, j;
 
 	ACPI_FUNCTION_TRACE(acpi_reallocate_root_table);
 
@@ -179,6 +180,8 @@ acpi_status ACPI_INIT_FUNCTION acpi_reallocate_root_table(void)
 		return_ACPI_STATUS(AE_SUPPORT);
 	}
 
+	(void)acpi_ut_acquire_mutex(ACPI_MTX_TABLES);
+
 	/*
 	 * Ensure OS early boot logic, which is required by some hosts. If the
 	 * table state is reported to be wrong, developers should fix the
@@ -186,17 +189,39 @@ acpi_status ACPI_INIT_FUNCTION acpi_reallocate_root_table(void)
 	 * early stage.
 	 */
 	for (i = 0; i < acpi_gbl_root_table_list.current_table_count; ++i) {
-		if (acpi_gbl_root_table_list.tables[i].pointer) {
+		table_desc = &acpi_gbl_root_table_list.tables[i];
+		if (table_desc->pointer) {
 			ACPI_ERROR((AE_INFO,
 				    "Table [%4.4s] is not invalidated during early boot stage",
-				    acpi_gbl_root_table_list.tables[i].
-				    signature.ascii));
+				    table_desc->signature.ascii));
 		}
 	}
 
-	acpi_gbl_root_table_list.flags |= ACPI_ROOT_ALLOW_RESIZE;
+	if (!acpi_gbl_enable_table_validation) {
+		/*
+		 * Now it's safe to do full table validation. We can do deferred
+		 * table initilization here once the flag is set.
+		 */
+		acpi_gbl_enable_table_validation = TRUE;
+		for (i = 0; i < acpi_gbl_root_table_list.current_table_count;
+		     ++i) {
+			table_desc = &acpi_gbl_root_table_list.tables[i];
+			if (!(table_desc->flags & ACPI_TABLE_IS_VERIFIED)) {
+				status =
+				    acpi_tb_verify_temp_table(table_desc, NULL,
+							      &j);
+				if (ACPI_FAILURE(status)) {
+					acpi_tb_uninstall_table(table_desc);
+				}
+			}
+		}
+	}
 
+	acpi_gbl_root_table_list.flags |= ACPI_ROOT_ALLOW_RESIZE;
 	status = acpi_tb_resize_root_table_list();
+	acpi_gbl_root_table_list.flags |= ACPI_ROOT_ORIGIN_ALLOCATED;
+
+	(void)acpi_ut_release_mutex(ACPI_MTX_TABLES);
 	return_ACPI_STATUS(status);
 }
 
@@ -369,6 +394,10 @@ void acpi_put_table(struct acpi_table_header *table)
 
 	ACPI_FUNCTION_TRACE(acpi_put_table);
 
+	if (!table) {
+		return_VOID;
+	}
+
 	(void)acpi_ut_acquire_mutex(ACPI_MTX_TABLES);
 
 	/* Walk the root table list */
diff --git a/drivers/acpi/acpica/tbxfload.c b/drivers/acpi/acpica/tbxfload.c
index b71ce3b817ea..d81f442228b8 100644
--- a/drivers/acpi/acpica/tbxfload.c
+++ b/drivers/acpi/acpica/tbxfload.c
@@ -206,7 +206,7 @@ acpi_status acpi_tb_load_namespace(void)
 	for (i = 0; i < acpi_gbl_root_table_list.current_table_count; ++i) {
 		table = &acpi_gbl_root_table_list.tables[i];
 
-		if (!acpi_gbl_root_table_list.tables[i].address ||
+		if (!table->address ||
 		    (!ACPI_COMPARE_NAME(table->signature.ascii, ACPI_SIG_SSDT)
 		     && !ACPI_COMPARE_NAME(table->signature.ascii,
 					   ACPI_SIG_PSDT)
diff --git a/drivers/acpi/acpica/uthex.c b/drivers/acpi/acpica/uthex.c
index 6600bc257516..fb406daf47fa 100644
--- a/drivers/acpi/acpica/uthex.c
+++ b/drivers/acpi/acpica/uthex.c
@@ -69,8 +69,10 @@ static const char acpi_gbl_hex_to_ascii[] = {
 
 char acpi_ut_hex_to_ascii_char(u64 integer, u32 position)
 {
+	u64 index;
 
-	return (acpi_gbl_hex_to_ascii[(integer >> position) & 0xF]);
+	acpi_ut_short_shift_right(integer, position, &index);
+	return (acpi_gbl_hex_to_ascii[index & 0xF]);
 }
 
 /*******************************************************************************
diff --git a/drivers/acpi/acpica/utmath.c b/drivers/acpi/acpica/utmath.c
index aa0502d1d019..5f9c680076c4 100644
--- a/drivers/acpi/acpica/utmath.c
+++ b/drivers/acpi/acpica/utmath.c
@@ -47,15 +47,6 @@
 #define _COMPONENT          ACPI_UTILITIES
 ACPI_MODULE_NAME("utmath")
 
-/*
- * Optional support for 64-bit double-precision integer divide. This code
- * is configurable and is implemented in order to support 32-bit kernel
- * environments where a 64-bit double-precision math library is not available.
- *
- * Support for a more normal 64-bit divide/modulo (with check for a divide-
- * by-zero) appears after this optional section of code.
- */
-#ifndef ACPI_USE_NATIVE_DIVIDE
 /* Structures used only for 64-bit divide */
 typedef struct uint64_struct {
 	u32 lo;
@@ -69,6 +60,217 @@ typedef union uint64_overlay {
 
 } uint64_overlay;
 
+/*
+ * Optional support for 64-bit double-precision integer multiply and shift.
+ * This code is configurable and is implemented in order to support 32-bit
+ * kernel environments where a 64-bit double-precision math library is not
+ * available.
+ */
+#ifndef ACPI_USE_NATIVE_MATH64
+
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_ut_short_multiply
+ *
+ * PARAMETERS:  multiplicand        - 64-bit multiplicand
+ *              multiplier          - 32-bit multiplier
+ *              out_product         - Pointer to where the product is returned
+ *
+ * DESCRIPTION: Perform a short multiply.
+ *
+ ******************************************************************************/
+
+acpi_status
+acpi_ut_short_multiply(u64 multiplicand, u32 multiplier, u64 *out_product)
+{
+	union uint64_overlay multiplicand_ovl;
+	union uint64_overlay product;
+	u32 carry32;
+
+	ACPI_FUNCTION_TRACE(ut_short_multiply);
+
+	multiplicand_ovl.full = multiplicand;
+
+	/*
+	 * The Product is 64 bits, the carry is always 32 bits,
+	 * and is generated by the second multiply.
+	 */
+	ACPI_MUL_64_BY_32(0, multiplicand_ovl.part.hi, multiplier,
+			  product.part.hi, carry32);
+
+	ACPI_MUL_64_BY_32(0, multiplicand_ovl.part.lo, multiplier,
+			  product.part.lo, carry32);
+
+	product.part.hi += carry32;
+
+	/* Return only what was requested */
+
+	if (out_product) {
+		*out_product = product.full;
+	}
+
+	return_ACPI_STATUS(AE_OK);
+}
+
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_ut_short_shift_left
+ *
+ * PARAMETERS:  operand             - 64-bit shift operand
+ *              count               - 32-bit shift count
+ *              out_result          - Pointer to where the result is returned
+ *
+ * DESCRIPTION: Perform a short left shift.
+ *
+ ******************************************************************************/
+
+acpi_status acpi_ut_short_shift_left(u64 operand, u32 count, u64 *out_result)
+{
+	union uint64_overlay operand_ovl;
+
+	ACPI_FUNCTION_TRACE(ut_short_shift_left);
+
+	operand_ovl.full = operand;
+
+	if ((count & 63) >= 32) {
+		operand_ovl.part.hi = operand_ovl.part.lo;
+		operand_ovl.part.lo ^= operand_ovl.part.lo;
+		count = (count & 63) - 32;
+	}
+	ACPI_SHIFT_LEFT_64_BY_32(operand_ovl.part.hi,
+				 operand_ovl.part.lo, count);
+
+	/* Return only what was requested */
+
+	if (out_result) {
+		*out_result = operand_ovl.full;
+	}
+
+	return_ACPI_STATUS(AE_OK);
+}
+
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_ut_short_shift_right
+ *
+ * PARAMETERS:  operand             - 64-bit shift operand
+ *              count               - 32-bit shift count
+ *              out_result          - Pointer to where the result is returned
+ *
+ * DESCRIPTION: Perform a short right shift.
+ *
+ ******************************************************************************/
+
+acpi_status acpi_ut_short_shift_right(u64 operand, u32 count, u64 *out_result)
+{
+	union uint64_overlay operand_ovl;
+
+	ACPI_FUNCTION_TRACE(ut_short_shift_right);
+
+	operand_ovl.full = operand;
+
+	if ((count & 63) >= 32) {
+		operand_ovl.part.lo = operand_ovl.part.hi;
+		operand_ovl.part.hi ^= operand_ovl.part.hi;
+		count = (count & 63) - 32;
+	}
+	ACPI_SHIFT_RIGHT_64_BY_32(operand_ovl.part.hi,
+				  operand_ovl.part.lo, count);
+
+	/* Return only what was requested */
+
+	if (out_result) {
+		*out_result = operand_ovl.full;
+	}
+
+	return_ACPI_STATUS(AE_OK);
+}
+#else
+
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_ut_short_multiply
+ *
+ * PARAMETERS:  See function headers above
+ *
+ * DESCRIPTION: Native version of the ut_short_multiply function.
+ *
+ ******************************************************************************/
+
+acpi_status
+acpi_ut_short_multiply(u64 multiplicand, u32 multiplier, u64 *out_product)
+{
+
+	ACPI_FUNCTION_TRACE(ut_short_multiply);
+
+	/* Return only what was requested */
+
+	if (out_product) {
+		*out_product = multiplicand * multiplier;
+	}
+
+	return_ACPI_STATUS(AE_OK);
+}
+
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_ut_short_shift_left
+ *
+ * PARAMETERS:  See function headers above
+ *
+ * DESCRIPTION: Native version of the ut_short_shift_left function.
+ *
+ ******************************************************************************/
+
+acpi_status acpi_ut_short_shift_left(u64 operand, u32 count, u64 *out_result)
+{
+
+	ACPI_FUNCTION_TRACE(ut_short_shift_left);
+
+	/* Return only what was requested */
+
+	if (out_result) {
+		*out_result = operand << count;
+	}
+
+	return_ACPI_STATUS(AE_OK);
+}
+
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_ut_short_shift_right
+ *
+ * PARAMETERS:  See function headers above
+ *
+ * DESCRIPTION: Native version of the ut_short_shift_right function.
+ *
+ ******************************************************************************/
+
+acpi_status acpi_ut_short_shift_right(u64 operand, u32 count, u64 *out_result)
+{
+
+	ACPI_FUNCTION_TRACE(ut_short_shift_right);
+
+	/* Return only what was requested */
+
+	if (out_result) {
+		*out_result = operand >> count;
+	}
+
+	return_ACPI_STATUS(AE_OK);
+}
+#endif
+
+/*
+ * Optional support for 64-bit double-precision integer divide. This code
+ * is configurable and is implemented in order to support 32-bit kernel
+ * environments where a 64-bit double-precision math library is not available.
+ *
+ * Support for a more normal 64-bit divide/modulo (with check for a divide-
+ * by-zero) appears after this optional section of code.
+ */
+#ifndef ACPI_USE_NATIVE_DIVIDE
+
 /*******************************************************************************
  *
  * FUNCTION:    acpi_ut_short_divide
@@ -258,6 +460,7 @@ acpi_ut_divide(u64 in_dividend,
 }
 
 #else
+
 /*******************************************************************************
  *
  * FUNCTION:    acpi_ut_short_divide, acpi_ut_divide
@@ -272,6 +475,7 @@ acpi_ut_divide(u64 in_dividend,
  *                 perform the divide.
  *
  ******************************************************************************/
+
 acpi_status
 acpi_ut_short_divide(u64 in_dividend,
 		     u32 divisor, u64 *out_quotient, u32 *out_remainder)
diff --git a/drivers/acpi/acpica/utmisc.c b/drivers/acpi/acpica/utmisc.c
index 443ffad01209..45c78c2adbf0 100644
--- a/drivers/acpi/acpica/utmisc.c
+++ b/drivers/acpi/acpica/utmisc.c
@@ -224,7 +224,7 @@ acpi_ut_create_update_state_and_push(union acpi_operand_object *object,
  *
  * RETURN:      Status
  *
- * DESCRIPTION: Walk through a package
+ * DESCRIPTION: Walk through a package, including subpackages
  *
  ******************************************************************************/
 
@@ -236,8 +236,8 @@ acpi_ut_walk_package_tree(union acpi_operand_object *source_object,
 	acpi_status status = AE_OK;
 	union acpi_generic_state *state_list = NULL;
 	union acpi_generic_state *state;
-	u32 this_index;
 	union acpi_operand_object *this_source_obj;
+	u32 this_index;
 
 	ACPI_FUNCTION_TRACE(ut_walk_package_tree);
 
@@ -251,8 +251,10 @@ acpi_ut_walk_package_tree(union acpi_operand_object *source_object,
 		/* Get one element of the package */
 
 		this_index = state->pkg.index;
-		this_source_obj = (union acpi_operand_object *)
+		this_source_obj =
 		    state->pkg.source_object->package.elements[this_index];
+		state->pkg.this_target_obj =
+		    &state->pkg.source_object->package.elements[this_index];
 
 		/*
 		 * Check for:
@@ -339,6 +341,8 @@ acpi_ut_walk_package_tree(union acpi_operand_object *source_object,
 
 	/* We should never get here */
 
+	ACPI_ERROR((AE_INFO, "State list did not terminate correctly"));
+
 	return_ACPI_STATUS(AE_AML_INTERNAL);
 }
 
diff --git a/drivers/acpi/acpica/utobject.c b/drivers/acpi/acpica/utobject.c
index 64e6641bfe82..cb3db9fed50d 100644
--- a/drivers/acpi/acpica/utobject.c
+++ b/drivers/acpi/acpica/utobject.c
@@ -483,6 +483,11 @@ acpi_ut_get_simple_object_size(union acpi_operand_object *internal_object,
 
 		/* A namespace node should never get here */
 
+		ACPI_ERROR((AE_INFO,
+			    "Received a namespace node [%4.4s] "
+			    "where an operand object is required",
+			    ACPI_CAST_PTR(struct acpi_namespace_node,
+					  internal_object)->name.ascii));
 		return_ACPI_STATUS(AE_AML_INTERNAL);
 	}
 
diff --git a/drivers/acpi/acpica/utprint.c b/drivers/acpi/acpica/utprint.c
index 7e6e1ae6140f..c008589b41bd 100644
--- a/drivers/acpi/acpica/utprint.c
+++ b/drivers/acpi/acpica/utprint.c
@@ -176,7 +176,7 @@ const char *acpi_ut_scan_number(const char *string, u64 *number_ptr)
 	u64 number = 0;
 
 	while (isdigit((int)*string)) {
-		number *= 10;
+		acpi_ut_short_multiply(number, 10, &number);
 		number += *(string++) - '0';
 	}
 
@@ -286,7 +286,7 @@ static char *acpi_ut_format_number(char *string,
 	/* Generate full string in reverse order */
 
 	pos = acpi_ut_put_number(reversed_string, number, base, upper);
-	i = ACPI_PTR_DIFF(pos, reversed_string);
+	i = (s32)ACPI_PTR_DIFF(pos, reversed_string);
 
 	/* Printing 100 using %2d gives "100", not "00" */
 
@@ -475,7 +475,7 @@ int vsnprintf(char *string, acpi_size size, const char *format, va_list args)
 			if (!s) {
 				s = "<NULL>";
 			}
-			length = acpi_ut_bound_string_length(s, precision);
+			length = (s32)acpi_ut_bound_string_length(s, precision);
 			if (!(type & ACPI_FORMAT_LEFT)) {
 				while (length < width--) {
 					pos =
@@ -579,7 +579,7 @@ int vsnprintf(char *string, acpi_size size, const char *format, va_list args)
 		}
 	}
 
-	return (ACPI_PTR_DIFF(pos, string));
+	return ((int)ACPI_PTR_DIFF(pos, string));
 }
 
 /*******************************************************************************
diff --git a/drivers/acpi/acpica/utresrc.c b/drivers/acpi/acpica/utresrc.c
index 70f78a4bf13b..f9801d13547f 100644
--- a/drivers/acpi/acpica/utresrc.c
+++ b/drivers/acpi/acpica/utresrc.c
@@ -237,6 +237,13 @@ acpi_ut_walk_aml_resources(struct acpi_walk_state *walk_state,
 				return_ACPI_STATUS(AE_AML_NO_RESOURCE_END_TAG);
 			}
 
+			/*
+			 * Don't attempt to perform any validation on the 2nd byte.
+			 * Although all known ASL compilers insert a zero for the 2nd
+			 * byte, it can also be a checksum (as per the ACPI spec),
+			 * and this is occasionally seen in the field. July 2017.
+			 */
+
 			/* Return the pointer to the end_tag if requested */
 
 			if (!user_function) {
diff --git a/drivers/acpi/acpica/utstate.c b/drivers/acpi/acpica/utstate.c
index 64308c304ade..eafabcd2fada 100644
--- a/drivers/acpi/acpica/utstate.c
+++ b/drivers/acpi/acpica/utstate.c
@@ -226,7 +226,7 @@ union acpi_generic_state *acpi_ut_create_update_state(union acpi_operand_object
 
 union acpi_generic_state *acpi_ut_create_pkg_state(void *internal_object,
 						   void *external_object,
-						   u16 index)
+						   u32 index)
 {
 	union acpi_generic_state *state;
 
diff --git a/drivers/acpi/acpica/utstrtoul64.c b/drivers/acpi/acpica/utstrtoul64.c
index f42be01d99fd..9633ee142855 100644
--- a/drivers/acpi/acpica/utstrtoul64.c
+++ b/drivers/acpi/acpica/utstrtoul64.c
@@ -276,8 +276,8 @@ static u64 acpi_ut_strtoul_base10(char *string, u32 flags)
 
 		/* Convert and insert (add) the decimal digit */
 
-		next_value =
-		    (return_value * 10) + (ascii_digit - ACPI_ASCII_ZERO);
+		acpi_ut_short_multiply(return_value, 10, &next_value);
+		next_value += (ascii_digit - ACPI_ASCII_ZERO);
 
 		/* Check for overflow (32 or 64 bit) - return current converted value */
 
@@ -335,9 +335,8 @@ static u64 acpi_ut_strtoul_base16(char *string, u32 flags)
 
 		/* Convert and insert the hex digit */
 
-		return_value =
-		    (return_value << 4) |
-		    acpi_ut_ascii_char_to_hex(ascii_digit);
+		acpi_ut_short_shift_left(return_value, 4, &return_value);
+		return_value |= acpi_ut_ascii_char_to_hex(ascii_digit);
 
 		string++;
 		valid_digits++;
diff --git a/drivers/acpi/acpica/uttrack.c b/drivers/acpi/acpica/uttrack.c
index 9a07a42cae34..3c8de88ecbd5 100644
--- a/drivers/acpi/acpica/uttrack.c
+++ b/drivers/acpi/acpica/uttrack.c
@@ -591,6 +591,10 @@ void acpi_ut_dump_allocations(u32 component, const char *module)
 		return_VOID;
 	}
 
+	if (!acpi_gbl_global_list) {
+		goto exit;
+	}
+
 	element = acpi_gbl_global_list->list_head;
 	while (element) {
 		if ((element->component & component) &&
@@ -602,7 +606,7 @@ void acpi_ut_dump_allocations(u32 component, const char *module)
 
 			if (element->size <
 			    sizeof(struct acpi_common_descriptor)) {
-				acpi_os_printf("%p Length 0x%04X %9.9s-%u "
+				acpi_os_printf("%p Length 0x%04X %9.9s-%4.4u "
 					       "[Not a Descriptor - too small]\n",
 					       descriptor, element->size,
 					       element->module, element->line);
@@ -612,7 +616,7 @@ void acpi_ut_dump_allocations(u32 component, const char *module)
 				if (ACPI_GET_DESCRIPTOR_TYPE(descriptor) !=
 				    ACPI_DESC_TYPE_CACHED) {
 					acpi_os_printf
-					    ("%p Length 0x%04X %9.9s-%u [%s] ",
+					    ("%p Length 0x%04X %9.9s-%4.4u [%s] ",
 					     descriptor, element->size,
 					     element->module, element->line,
 					     acpi_ut_get_descriptor_name
@@ -705,6 +709,7 @@ void acpi_ut_dump_allocations(u32 component, const char *module)
 		element = element->next;
 	}
 
+exit:
 	(void)acpi_ut_release_mutex(ACPI_MTX_MEMORY);
 
 	/* Print summary */
diff --git a/drivers/acpi/apei/apei-internal.h b/drivers/acpi/apei/apei-internal.h
index 6e9f14c0a71b..cb4126051f62 100644
--- a/drivers/acpi/apei/apei-internal.h
+++ b/drivers/acpi/apei/apei-internal.h
@@ -120,11 +120,6 @@ int apei_exec_collect_resources(struct apei_exec_context *ctx,
 struct dentry;
 struct dentry *apei_get_debugfs_dir(void);
 
-#define apei_estatus_for_each_section(estatus, section)			\
-	for (section = (struct acpi_hest_generic_data *)(estatus + 1);	\
-	     (void *)section - (void *)estatus < estatus->data_length;	\
-	     section = (void *)(section+1) + section->error_data_length)
-
 static inline u32 cper_estatus_len(struct acpi_hest_generic_status *estatus)
 {
 	if (estatus->raw_data_length)
diff --git a/drivers/acpi/apei/einj.c b/drivers/acpi/apei/einj.c
index ec50c32ea3da..b38737c83a24 100644
--- a/drivers/acpi/apei/einj.c
+++ b/drivers/acpi/apei/einj.c
@@ -281,7 +281,7 @@ static struct acpi_generic_address *einj_get_trigger_parameter_region(
 		((char *)trigger_tab + sizeof(struct acpi_einj_trigger));
 	for (i = 0; i < trigger_tab->entry_count; i++) {
 		if (entry->action == ACPI_EINJ_TRIGGER_ERROR &&
-		entry->instruction == ACPI_EINJ_WRITE_REGISTER_VALUE &&
+		entry->instruction <= ACPI_EINJ_WRITE_REGISTER_VALUE &&
 		entry->register_region.space_id ==
 			ACPI_ADR_SPACE_SYSTEM_MEMORY &&
 		(entry->register_region.address & param2) == (param1 & param2))
diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c
index d661d452b238..077f9bad6f44 100644
--- a/drivers/acpi/apei/ghes.c
+++ b/drivers/acpi/apei/ghes.c
@@ -1157,7 +1157,8 @@ static int ghes_probe(struct platform_device *ghes_dev)
 			       generic->header.source_id);
 			goto err_edac_unreg;
 		}
-		rc = request_irq(ghes->irq, ghes_irq_func, 0, "GHES IRQ", ghes);
+		rc = request_irq(ghes->irq, ghes_irq_func, IRQF_SHARED,
+				 "GHES IRQ", ghes);
 		if (rc) {
 			pr_err(GHES_PFX "Failed to register IRQ for generic hardware error source: %d\n",
 			       generic->header.source_id);
@@ -1265,9 +1266,14 @@ static int __init ghes_init(void)
 	if (acpi_disabled)
 		return -ENODEV;
 
-	if (hest_disable) {
+	switch (hest_disable) {
+	case HEST_NOT_FOUND:
+		return -ENODEV;
+	case HEST_DISABLED:
 		pr_info(GHES_PFX "HEST is not enabled!\n");
 		return -EINVAL;
+	default:
+		break;
 	}
 
 	if (ghes_disable) {
diff --git a/drivers/acpi/apei/hest.c b/drivers/acpi/apei/hest.c
index 456b488eb1df..9cb74115a43d 100644
--- a/drivers/acpi/apei/hest.c
+++ b/drivers/acpi/apei/hest.c
@@ -37,7 +37,7 @@
 
 #define HEST_PFX "HEST: "
 
-bool hest_disable;
+int hest_disable;
 EXPORT_SYMBOL_GPL(hest_disable);
 
 /* HEST table parsing */
@@ -213,7 +213,7 @@ err:
 
 static int __init setup_hest_disable(char *str)
 {
-	hest_disable = 1;
+	hest_disable = HEST_DISABLED;
 	return 0;
 }
 
@@ -232,9 +232,10 @@ void __init acpi_hest_init(void)
 
 	status = acpi_get_table(ACPI_SIG_HEST, 0,
 				(struct acpi_table_header **)&hest_tab);
-	if (status == AE_NOT_FOUND)
-		goto err;
-	else if (ACPI_FAILURE(status)) {
+	if (status == AE_NOT_FOUND) {
+		hest_disable = HEST_NOT_FOUND;
+		return;
+	} else if (ACPI_FAILURE(status)) {
 		const char *msg = acpi_format_exception(status);
 		pr_err(HEST_PFX "Failed to get table, %s\n", msg);
 		rc = -EINVAL;
@@ -257,5 +258,5 @@ void __init acpi_hest_init(void)
 	pr_info(HEST_PFX "Table parsing has been initialized.\n");
 	return;
 err:
-	hest_disable = 1;
+	hest_disable = HEST_DISABLED;
 }
diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c
index a3215ee671c1..9565d572f8dd 100644
--- a/drivers/acpi/arm64/iort.c
+++ b/drivers/acpi/arm64/iort.c
@@ -588,7 +588,8 @@ void acpi_configure_pmsi_domain(struct device *dev)
 		dev_set_msi_domain(dev, msi_domain);
 }
 
-static int __get_pci_rid(struct pci_dev *pdev, u16 alias, void *data)
+static int __maybe_unused __get_pci_rid(struct pci_dev *pdev, u16 alias,
+					void *data)
 {
 	u32 *rid = data;
 
@@ -633,8 +634,7 @@ int iort_add_device_replay(const struct iommu_ops *ops, struct device *dev)
 {
 	int err = 0;
 
-	if (!IS_ERR_OR_NULL(ops) && ops->add_device && dev->bus &&
-	    !dev->iommu_group)
+	if (ops->add_device && dev->bus && !dev->iommu_group)
 		err = ops->add_device(dev);
 
 	return err;
@@ -648,45 +648,81 @@ int iort_add_device_replay(const struct iommu_ops *ops, struct device *dev)
 { return 0; }
 #endif
 
-static const struct iommu_ops *iort_iommu_xlate(struct device *dev,
-					struct acpi_iort_node *node,
-					u32 streamid)
+static int iort_iommu_xlate(struct device *dev, struct acpi_iort_node *node,
+			    u32 streamid)
 {
-	const struct iommu_ops *ops = NULL;
-	int ret = -ENODEV;
+	const struct iommu_ops *ops;
 	struct fwnode_handle *iort_fwnode;
 
-	if (node) {
-		iort_fwnode = iort_get_fwnode(node);
-		if (!iort_fwnode)
-			return NULL;
+	if (!node)
+		return -ENODEV;
 
-		ops = iommu_ops_from_fwnode(iort_fwnode);
-		/*
-		 * If the ops look-up fails, this means that either
-		 * the SMMU drivers have not been probed yet or that
-		 * the SMMU drivers are not built in the kernel;
-		 * Depending on whether the SMMU drivers are built-in
-		 * in the kernel or not, defer the IOMMU configuration
-		 * or just abort it.
-		 */
-		if (!ops)
-			return iort_iommu_driver_enabled(node->type) ?
-			       ERR_PTR(-EPROBE_DEFER) : NULL;
+	iort_fwnode = iort_get_fwnode(node);
+	if (!iort_fwnode)
+		return -ENODEV;
 
-		ret = arm_smmu_iort_xlate(dev, streamid, iort_fwnode, ops);
-	}
+	/*
+	 * If the ops look-up fails, this means that either
+	 * the SMMU drivers have not been probed yet or that
+	 * the SMMU drivers are not built in the kernel;
+	 * Depending on whether the SMMU drivers are built-in
+	 * in the kernel or not, defer the IOMMU configuration
+	 * or just abort it.
+	 */
+	ops = iommu_ops_from_fwnode(iort_fwnode);
+	if (!ops)
+		return iort_iommu_driver_enabled(node->type) ?
+		       -EPROBE_DEFER : -ENODEV;
+
+	return arm_smmu_iort_xlate(dev, streamid, iort_fwnode, ops);
+}
+
+struct iort_pci_alias_info {
+	struct device *dev;
+	struct acpi_iort_node *node;
+};
+
+static int iort_pci_iommu_init(struct pci_dev *pdev, u16 alias, void *data)
+{
+	struct iort_pci_alias_info *info = data;
+	struct acpi_iort_node *parent;
+	u32 streamid;
 
-	return ret ? NULL : ops;
+	parent = iort_node_map_id(info->node, alias, &streamid,
+				  IORT_IOMMU_TYPE);
+	return iort_iommu_xlate(info->dev, parent, streamid);
+}
+
+static int nc_dma_get_range(struct device *dev, u64 *size)
+{
+	struct acpi_iort_node *node;
+	struct acpi_iort_named_component *ncomp;
+
+	node = iort_scan_node(ACPI_IORT_NODE_NAMED_COMPONENT,
+			      iort_match_node_callback, dev);
+	if (!node)
+		return -ENODEV;
+
+	ncomp = (struct acpi_iort_named_component *)node->node_data;
+
+	*size = ncomp->memory_address_limit >= 64 ? U64_MAX :
+			1ULL<<ncomp->memory_address_limit;
+
+	return 0;
 }
 
 /**
- * iort_set_dma_mask - Set-up dma mask for a device.
+ * iort_dma_setup() - Set-up device DMA parameters.
  *
  * @dev: device to configure
+ * @dma_addr: device DMA address result pointer
+ * @size: DMA range size result pointer
  */
-void iort_set_dma_mask(struct device *dev)
+void iort_dma_setup(struct device *dev, u64 *dma_addr, u64 *dma_size)
 {
+	u64 mask, dmaaddr = 0, size = 0, offset = 0;
+	int ret, msb;
+
 	/*
 	 * Set default coherent_dma_mask to 32 bit.  Drivers are expected to
 	 * setup the correct supported mask.
@@ -700,6 +736,36 @@ void iort_set_dma_mask(struct device *dev)
 	 */
 	if (!dev->dma_mask)
 		dev->dma_mask = &dev->coherent_dma_mask;
+
+	size = max(dev->coherent_dma_mask, dev->coherent_dma_mask + 1);
+
+	if (dev_is_pci(dev))
+		ret = acpi_dma_get_range(dev, &dmaaddr, &offset, &size);
+	else
+		ret = nc_dma_get_range(dev, &size);
+
+	if (!ret) {
+		msb = fls64(dmaaddr + size - 1);
+		/*
+		 * Round-up to the power-of-two mask or set
+		 * the mask to the whole 64-bit address space
+		 * in case the DMA region covers the full
+		 * memory window.
+		 */
+		mask = msb == 64 ? U64_MAX : (1ULL << msb) - 1;
+		/*
+		 * Limit coherent and dma mask based on size
+		 * retrieved from firmware.
+		 */
+		dev->coherent_dma_mask = mask;
+		*dev->dma_mask = mask;
+	}
+
+	*dma_addr = dmaaddr;
+	*dma_size = size;
+
+	dev->dma_pfn_offset = PFN_DOWN(offset);
+	dev_dbg(dev, "dma_pfn_offset(%#08llx)\n", offset);
 }
 
 /**
@@ -713,9 +779,9 @@ void iort_set_dma_mask(struct device *dev)
 const struct iommu_ops *iort_iommu_configure(struct device *dev)
 {
 	struct acpi_iort_node *node, *parent;
-	const struct iommu_ops *ops = NULL;
+	const struct iommu_ops *ops;
 	u32 streamid = 0;
-	int err;
+	int err = -ENODEV;
 
 	/*
 	 * If we already translated the fwspec there
@@ -727,21 +793,16 @@ const struct iommu_ops *iort_iommu_configure(struct device *dev)
 
 	if (dev_is_pci(dev)) {
 		struct pci_bus *bus = to_pci_dev(dev)->bus;
-		u32 rid;
-
-		pci_for_each_dma_alias(to_pci_dev(dev), __get_pci_rid,
-				       &rid);
+		struct iort_pci_alias_info info = { .dev = dev };
 
 		node = iort_scan_node(ACPI_IORT_NODE_PCI_ROOT_COMPLEX,
 				      iort_match_node_callback, &bus->dev);
 		if (!node)
 			return NULL;
 
-		parent = iort_node_map_id(node, rid, &streamid,
-					  IORT_IOMMU_TYPE);
-
-		ops = iort_iommu_xlate(dev, parent, streamid);
-
+		info.node = node;
+		err = pci_for_each_dma_alias(to_pci_dev(dev),
+					     iort_pci_iommu_init, &info);
 	} else {
 		int i = 0;
 
@@ -750,31 +811,30 @@ const struct iommu_ops *iort_iommu_configure(struct device *dev)
 		if (!node)
 			return NULL;
 
-		parent = iort_node_map_platform_id(node, &streamid,
-						   IORT_IOMMU_TYPE, i++);
-
-		while (parent) {
-			ops = iort_iommu_xlate(dev, parent, streamid);
-			if (IS_ERR_OR_NULL(ops))
-				return ops;
-
+		do {
 			parent = iort_node_map_platform_id(node, &streamid,
 							   IORT_IOMMU_TYPE,
 							   i++);
-		}
+
+			if (parent)
+				err = iort_iommu_xlate(dev, parent, streamid);
+		} while (parent && !err);
 	}
 
 	/*
 	 * If we have reason to believe the IOMMU driver missed the initial
 	 * add_device callback for dev, replay it to get things in order.
 	 */
-	err = iort_add_device_replay(ops, dev);
-	if (err)
-		ops = ERR_PTR(err);
+	if (!err) {
+		ops = iort_fwspec_iommu_ops(dev->iommu_fwspec);
+		err = iort_add_device_replay(ops, dev);
+	}
 
 	/* Ignore all other errors apart from EPROBE_DEFER */
-	if (IS_ERR(ops) && (PTR_ERR(ops) != -EPROBE_DEFER)) {
-		dev_dbg(dev, "Adding to IOMMU failed: %ld\n", PTR_ERR(ops));
+	if (err == -EPROBE_DEFER) {
+		ops = ERR_PTR(err);
+	} else if (err) {
+		dev_dbg(dev, "Adding to IOMMU failed: %d\n", err);
 		ops = NULL;
 	}
 
@@ -908,6 +968,27 @@ static bool __init arm_smmu_v3_is_coherent(struct acpi_iort_node *node)
 	return smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE;
 }
 
+#if defined(CONFIG_ACPI_NUMA) && defined(ACPI_IORT_SMMU_V3_PXM_VALID)
+/*
+ * set numa proximity domain for smmuv3 device
+ */
+static void  __init arm_smmu_v3_set_proximity(struct device *dev,
+					      struct acpi_iort_node *node)
+{
+	struct acpi_iort_smmu_v3 *smmu;
+
+	smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
+	if (smmu->flags & ACPI_IORT_SMMU_V3_PXM_VALID) {
+		set_dev_node(dev, acpi_map_pxm_to_node(smmu->pxm));
+		pr_info("SMMU-v3[%llx] Mapped to Proximity domain %d\n",
+			smmu->base_address,
+			smmu->pxm);
+	}
+}
+#else
+#define arm_smmu_v3_set_proximity NULL
+#endif
+
 static int __init arm_smmu_count_resources(struct acpi_iort_node *node)
 {
 	struct acpi_iort_smmu *smmu;
@@ -977,13 +1058,16 @@ struct iort_iommu_config {
 	int (*iommu_count_resources)(struct acpi_iort_node *node);
 	void (*iommu_init_resources)(struct resource *res,
 				     struct acpi_iort_node *node);
+	void (*iommu_set_proximity)(struct device *dev,
+				    struct acpi_iort_node *node);
 };
 
 static const struct iort_iommu_config iort_arm_smmu_v3_cfg __initconst = {
 	.name = "arm-smmu-v3",
 	.iommu_is_coherent = arm_smmu_v3_is_coherent,
 	.iommu_count_resources = arm_smmu_v3_count_resources,
-	.iommu_init_resources = arm_smmu_v3_init_resources
+	.iommu_init_resources = arm_smmu_v3_init_resources,
+	.iommu_set_proximity = arm_smmu_v3_set_proximity,
 };
 
 static const struct iort_iommu_config iort_arm_smmu_cfg __initconst = {
@@ -1028,6 +1112,9 @@ static int __init iort_add_smmu_platform_device(struct acpi_iort_node *node)
 	if (!pdev)
 		return -ENOMEM;
 
+	if (ops->iommu_set_proximity)
+		ops->iommu_set_proximity(&pdev->dev, node);
+
 	count = ops->iommu_count_resources(node);
 
 	r = kcalloc(count, sizeof(*r), GFP_KERNEL);
diff --git a/drivers/acpi/battery.c b/drivers/acpi/battery.c
index 1cbb88d938e5..13e7b56e33ae 100644
--- a/drivers/acpi/battery.c
+++ b/drivers/acpi/battery.c
@@ -620,7 +620,7 @@ static ssize_t acpi_battery_alarm_store(struct device *dev,
 	return count;
 }
 
-static struct device_attribute alarm_attr = {
+static const struct device_attribute alarm_attr = {
 	.attr = {.name = "alarm", .mode = 0644},
 	.show = acpi_battery_alarm_show,
 	.store = acpi_battery_alarm_store,
diff --git a/drivers/acpi/blacklist.c b/drivers/acpi/blacklist.c
index bb542acc0574..995c4d8922b1 100644
--- a/drivers/acpi/blacklist.c
+++ b/drivers/acpi/blacklist.c
@@ -30,30 +30,13 @@
 
 #include "internal.h"
 
-enum acpi_blacklist_predicates {
-	all_versions,
-	less_than_or_equal,
-	equal,
-	greater_than_or_equal,
-};
-
-struct acpi_blacklist_item {
-	char oem_id[7];
-	char oem_table_id[9];
-	u32 oem_revision;
-	char *table;
-	enum acpi_blacklist_predicates oem_revision_predicate;
-	char *reason;
-	u32 is_critical_error;
-};
-
-static struct dmi_system_id acpi_rev_dmi_table[] __initdata;
+static const struct dmi_system_id acpi_rev_dmi_table[] __initconst;
 
 /*
  * POLICY: If *anything* doesn't work, put it on the blacklist.
  *	   If they are critical errors, mark it critical, and abort driver load.
  */
-static struct acpi_blacklist_item acpi_blacklist[] __initdata = {
+static struct acpi_platform_list acpi_blacklist[] __initdata = {
 	/* Compaq Presario 1700 */
 	{"PTLTD ", "  DSDT  ", 0x06040000, ACPI_SIG_DSDT, less_than_or_equal,
 	 "Multiple problems", 1},
@@ -67,65 +50,27 @@ static struct acpi_blacklist_item acpi_blacklist[] __initdata = {
 	{"IBM   ", "TP600E  ", 0x00000105, ACPI_SIG_DSDT, less_than_or_equal,
 	 "Incorrect _ADR", 1},
 
-	{""}
+	{ }
 };
 
 int __init acpi_blacklisted(void)
 {
-	int i = 0;
+	int i;
 	int blacklisted = 0;
-	struct acpi_table_header table_header;
-
-	while (acpi_blacklist[i].oem_id[0] != '\0') {
-		if (acpi_get_table_header(acpi_blacklist[i].table, 0, &table_header)) {
-			i++;
-			continue;
-		}
-
-		if (strncmp(acpi_blacklist[i].oem_id, table_header.oem_id, 6)) {
-			i++;
-			continue;
-		}
-
-		if (strncmp
-		    (acpi_blacklist[i].oem_table_id, table_header.oem_table_id,
-		     8)) {
-			i++;
-			continue;
-		}
-
-		if ((acpi_blacklist[i].oem_revision_predicate == all_versions)
-		    || (acpi_blacklist[i].oem_revision_predicate ==
-			less_than_or_equal
-			&& table_header.oem_revision <=
-			acpi_blacklist[i].oem_revision)
-		    || (acpi_blacklist[i].oem_revision_predicate ==
-			greater_than_or_equal
-			&& table_header.oem_revision >=
-			acpi_blacklist[i].oem_revision)
-		    || (acpi_blacklist[i].oem_revision_predicate == equal
-			&& table_header.oem_revision ==
-			acpi_blacklist[i].oem_revision)) {
 
-			printk(KERN_ERR PREFIX
-			       "Vendor \"%6.6s\" System \"%8.8s\" "
-			       "Revision 0x%x has a known ACPI BIOS problem.\n",
-			       acpi_blacklist[i].oem_id,
-			       acpi_blacklist[i].oem_table_id,
-			       acpi_blacklist[i].oem_revision);
+	i = acpi_match_platform_list(acpi_blacklist);
+	if (i >= 0) {
+		pr_err(PREFIX "Vendor \"%6.6s\" System \"%8.8s\" Revision 0x%x has a known ACPI BIOS problem.\n",
+		       acpi_blacklist[i].oem_id,
+		       acpi_blacklist[i].oem_table_id,
+		       acpi_blacklist[i].oem_revision);
 
-			printk(KERN_ERR PREFIX
-			       "Reason: %s. This is a %s error\n",
-			       acpi_blacklist[i].reason,
-			       (acpi_blacklist[i].
-				is_critical_error ? "non-recoverable" :
-				"recoverable"));
+		pr_err(PREFIX "Reason: %s. This is a %s error\n",
+		       acpi_blacklist[i].reason,
+		       (acpi_blacklist[i].data ?
+			"non-recoverable" : "recoverable"));
 
-			blacklisted = acpi_blacklist[i].is_critical_error;
-			break;
-		} else {
-			i++;
-		}
+		blacklisted = acpi_blacklist[i].data;
 	}
 
 	(void)early_acpi_osi_init();
@@ -144,7 +89,7 @@ static int __init dmi_enable_rev_override(const struct dmi_system_id *d)
 }
 #endif
 
-static struct dmi_system_id acpi_rev_dmi_table[] __initdata = {
+static const struct dmi_system_id acpi_rev_dmi_table[] __initconst = {
 #ifdef CONFIG_ACPI_REV_OVERRIDE_POSSIBLE
 	/*
 	 * DELL XPS 13 (2015) switches sound between HDA and I2S
diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c
index af74b420ec83..4d0979e02a28 100644
--- a/drivers/acpi/bus.c
+++ b/drivers/acpi/bus.c
@@ -67,7 +67,7 @@ static int set_copy_dsdt(const struct dmi_system_id *id)
 }
 #endif
 
-static struct dmi_system_id dsdt_dmi_table[] __initdata = {
+static const struct dmi_system_id dsdt_dmi_table[] __initconst = {
 	/*
 	 * Invoke DSDT corruption work-around on all Toshiba Satellite.
 	 * https://bugzilla.kernel.org/show_bug.cgi?id=14679
@@ -83,7 +83,7 @@ static struct dmi_system_id dsdt_dmi_table[] __initdata = {
 	{}
 };
 #else
-static struct dmi_system_id dsdt_dmi_table[] __initdata = {
+static const struct dmi_system_id dsdt_dmi_table[] __initconst = {
 	{}
 };
 #endif
@@ -995,9 +995,6 @@ void __init acpi_early_init(void)
 
 	printk(KERN_INFO PREFIX "Core revision %08x\n", ACPI_CA_VERSION);
 
-	/* It's safe to verify table checksums during late stage */
-	acpi_gbl_verify_table_checksum = TRUE;
-
 	/* enable workarounds, unless strict ACPI spec. compliance */
 	if (!acpi_strict)
 		acpi_gbl_enable_interpreter_slack = TRUE;
diff --git a/drivers/acpi/device_pm.c b/drivers/acpi/device_pm.c
index 2ed6935d4483..fbcc73f7a099 100644
--- a/drivers/acpi/device_pm.c
+++ b/drivers/acpi/device_pm.c
@@ -401,6 +401,8 @@ static void acpi_pm_notify_handler(acpi_handle handle, u32 val, void *not_used)
 	if (val != ACPI_NOTIFY_DEVICE_WAKE)
 		return;
 
+	acpi_handle_debug(handle, "Wake notify\n");
+
 	adev = acpi_bus_get_acpi_device(handle);
 	if (!adev)
 		return;
@@ -409,8 +411,12 @@ static void acpi_pm_notify_handler(acpi_handle handle, u32 val, void *not_used)
 
 	if (adev->wakeup.flags.notifier_present) {
 		pm_wakeup_ws_event(adev->wakeup.ws, 0, acpi_s2idle_wakeup());
-		if (adev->wakeup.context.func)
+		if (adev->wakeup.context.func) {
+			acpi_handle_debug(handle, "Running %pF for %s\n",
+					  adev->wakeup.context.func,
+					  dev_name(adev->wakeup.context.dev));
 			adev->wakeup.context.func(&adev->wakeup.context);
+		}
 	}
 
 	mutex_unlock(&acpi_pm_notifier_lock);
@@ -682,55 +688,88 @@ static void acpi_pm_notify_work_func(struct acpi_device_wakeup_context *context)
 	}
 }
 
+static DEFINE_MUTEX(acpi_wakeup_lock);
+
+static int __acpi_device_wakeup_enable(struct acpi_device *adev,
+				       u32 target_state, int max_count)
+{
+	struct acpi_device_wakeup *wakeup = &adev->wakeup;
+	acpi_status status;
+	int error = 0;
+
+	mutex_lock(&acpi_wakeup_lock);
+
+	if (wakeup->enable_count >= max_count)
+		goto out;
+
+	if (wakeup->enable_count > 0)
+		goto inc;
+
+	error = acpi_enable_wakeup_device_power(adev, target_state);
+	if (error)
+		goto out;
+
+	status = acpi_enable_gpe(wakeup->gpe_device, wakeup->gpe_number);
+	if (ACPI_FAILURE(status)) {
+		acpi_disable_wakeup_device_power(adev);
+		error = -EIO;
+		goto out;
+	}
+
+inc:
+	wakeup->enable_count++;
+
+out:
+	mutex_unlock(&acpi_wakeup_lock);
+	return error;
+}
+
 /**
- * acpi_device_wakeup - Enable/disable wakeup functionality for device.
- * @adev: ACPI device to enable/disable wakeup functionality for.
+ * acpi_device_wakeup_enable - Enable wakeup functionality for device.
+ * @adev: ACPI device to enable wakeup functionality for.
  * @target_state: State the system is transitioning into.
- * @enable: Whether to enable or disable the wakeup functionality.
  *
- * Enable/disable the GPE associated with @adev so that it can generate
- * wakeup signals for the device in response to external (remote) events and
- * enable/disable device wakeup power.
+ * Enable the GPE associated with @adev so that it can generate wakeup signals
+ * for the device in response to external (remote) events and enable wakeup
+ * power for it.
+ *
+ * Callers must ensure that @adev is a valid ACPI device node before executing
+ * this function.
+ */
+static int acpi_device_wakeup_enable(struct acpi_device *adev, u32 target_state)
+{
+	return __acpi_device_wakeup_enable(adev, target_state, 1);
+}
+
+/**
+ * acpi_device_wakeup_disable - Disable wakeup functionality for device.
+ * @adev: ACPI device to disable wakeup functionality for.
+ *
+ * Disable the GPE associated with @adev and disable wakeup power for it.
  *
  * Callers must ensure that @adev is a valid ACPI device node before executing
  * this function.
  */
-static int acpi_device_wakeup(struct acpi_device *adev, u32 target_state,
-			      bool enable)
+static void acpi_device_wakeup_disable(struct acpi_device *adev)
 {
 	struct acpi_device_wakeup *wakeup = &adev->wakeup;
 
-	if (enable) {
-		acpi_status res;
-		int error;
+	mutex_lock(&acpi_wakeup_lock);
 
-		if (adev->wakeup.flags.enabled)
-			return 0;
+	if (!wakeup->enable_count)
+		goto out;
 
-		error = acpi_enable_wakeup_device_power(adev, target_state);
-		if (error)
-			return error;
+	acpi_disable_gpe(wakeup->gpe_device, wakeup->gpe_number);
+	acpi_disable_wakeup_device_power(adev);
 
-		res = acpi_enable_gpe(wakeup->gpe_device, wakeup->gpe_number);
-		if (ACPI_FAILURE(res)) {
-			acpi_disable_wakeup_device_power(adev);
-			return -EIO;
-		}
-		adev->wakeup.flags.enabled = 1;
-	} else if (adev->wakeup.flags.enabled) {
-		acpi_disable_gpe(wakeup->gpe_device, wakeup->gpe_number);
-		acpi_disable_wakeup_device_power(adev);
-		adev->wakeup.flags.enabled = 0;
-	}
-	return 0;
+	wakeup->enable_count--;
+
+out:
+	mutex_unlock(&acpi_wakeup_lock);
 }
 
-/**
- * acpi_pm_set_device_wakeup - Enable/disable remote wakeup for given device.
- * @dev: Device to enable/disable to generate wakeup events.
- * @enable: Whether to enable or disable the wakeup functionality.
- */
-int acpi_pm_set_device_wakeup(struct device *dev, bool enable)
+static int __acpi_pm_set_device_wakeup(struct device *dev, bool enable,
+				       int max_count)
 {
 	struct acpi_device *adev;
 	int error;
@@ -744,13 +783,41 @@ int acpi_pm_set_device_wakeup(struct device *dev, bool enable)
 	if (!acpi_device_can_wakeup(adev))
 		return -EINVAL;
 
-	error = acpi_device_wakeup(adev, acpi_target_system_state(), enable);
+	if (!enable) {
+		acpi_device_wakeup_disable(adev);
+		dev_dbg(dev, "Wakeup disabled by ACPI\n");
+		return 0;
+	}
+
+	error = __acpi_device_wakeup_enable(adev, acpi_target_system_state(),
+					    max_count);
 	if (!error)
-		dev_dbg(dev, "Wakeup %s by ACPI\n", enable ? "enabled" : "disabled");
+		dev_dbg(dev, "Wakeup enabled by ACPI\n");
 
 	return error;
 }
-EXPORT_SYMBOL(acpi_pm_set_device_wakeup);
+
+/**
+ * acpi_pm_set_device_wakeup - Enable/disable remote wakeup for given device.
+ * @dev: Device to enable/disable to generate wakeup events.
+ * @enable: Whether to enable or disable the wakeup functionality.
+ */
+int acpi_pm_set_device_wakeup(struct device *dev, bool enable)
+{
+	return __acpi_pm_set_device_wakeup(dev, enable, 1);
+}
+EXPORT_SYMBOL_GPL(acpi_pm_set_device_wakeup);
+
+/**
+ * acpi_pm_set_bridge_wakeup - Enable/disable remote wakeup for given bridge.
+ * @dev: Bridge device to enable/disable to generate wakeup events.
+ * @enable: Whether to enable or disable the wakeup functionality.
+ */
+int acpi_pm_set_bridge_wakeup(struct device *dev, bool enable)
+{
+	return __acpi_pm_set_device_wakeup(dev, enable, INT_MAX);
+}
+EXPORT_SYMBOL_GPL(acpi_pm_set_bridge_wakeup);
 
 /**
  * acpi_dev_pm_low_power - Put ACPI device into a low-power state.
@@ -800,13 +867,15 @@ int acpi_dev_runtime_suspend(struct device *dev)
 
 	remote_wakeup = dev_pm_qos_flags(dev, PM_QOS_FLAG_REMOTE_WAKEUP) >
 				PM_QOS_FLAGS_NONE;
-	error = acpi_device_wakeup(adev, ACPI_STATE_S0, remote_wakeup);
-	if (remote_wakeup && error)
-		return -EAGAIN;
+	if (remote_wakeup) {
+		error = acpi_device_wakeup_enable(adev, ACPI_STATE_S0);
+		if (error)
+			return -EAGAIN;
+	}
 
 	error = acpi_dev_pm_low_power(dev, adev, ACPI_STATE_S0);
-	if (error)
-		acpi_device_wakeup(adev, ACPI_STATE_S0, false);
+	if (error && remote_wakeup)
+		acpi_device_wakeup_disable(adev);
 
 	return error;
 }
@@ -829,7 +898,7 @@ int acpi_dev_runtime_resume(struct device *dev)
 		return 0;
 
 	error = acpi_dev_pm_full_power(adev);
-	acpi_device_wakeup(adev, ACPI_STATE_S0, false);
+	acpi_device_wakeup_disable(adev);
 	return error;
 }
 EXPORT_SYMBOL_GPL(acpi_dev_runtime_resume);
@@ -884,13 +953,15 @@ int acpi_dev_suspend_late(struct device *dev)
 
 	target_state = acpi_target_system_state();
 	wakeup = device_may_wakeup(dev) && acpi_device_can_wakeup(adev);
-	error = acpi_device_wakeup(adev, target_state, wakeup);
-	if (wakeup && error)
-		return error;
+	if (wakeup) {
+		error = acpi_device_wakeup_enable(adev, target_state);
+		if (error)
+			return error;
+	}
 
 	error = acpi_dev_pm_low_power(dev, adev, target_state);
-	if (error)
-		acpi_device_wakeup(adev, ACPI_STATE_UNKNOWN, false);
+	if (error && wakeup)
+		acpi_device_wakeup_disable(adev);
 
 	return error;
 }
@@ -913,7 +984,7 @@ int acpi_dev_resume_early(struct device *dev)
 		return 0;
 
 	error = acpi_dev_pm_full_power(adev);
-	acpi_device_wakeup(adev, ACPI_STATE_UNKNOWN, false);
+	acpi_device_wakeup_disable(adev);
 	return error;
 }
 EXPORT_SYMBOL_GPL(acpi_dev_resume_early);
@@ -1056,7 +1127,7 @@ static void acpi_dev_pm_detach(struct device *dev, bool power_off)
 			 */
 			dev_pm_qos_hide_latency_limit(dev);
 			dev_pm_qos_hide_flags(dev);
-			acpi_device_wakeup(adev, ACPI_STATE_S0, false);
+			acpi_device_wakeup_disable(adev);
 			acpi_dev_pm_low_power(dev, adev, ACPI_STATE_S0);
 		}
 	}
@@ -1100,7 +1171,7 @@ int acpi_dev_pm_attach(struct device *dev, bool power_on)
 	dev_pm_domain_set(dev, &acpi_general_pm_domain);
 	if (power_on) {
 		acpi_dev_pm_full_power(adev);
-		acpi_device_wakeup(adev, ACPI_STATE_S0, false);
+		acpi_device_wakeup_disable(adev);
 	}
 
 	dev->pm_domain->detach = acpi_dev_pm_detach;
diff --git a/drivers/acpi/dock.c b/drivers/acpi/dock.c
index 0c00208b423e..2305e1ab978e 100644
--- a/drivers/acpi/dock.c
+++ b/drivers/acpi/dock.c
@@ -585,7 +585,7 @@ static struct attribute *dock_attributes[] = {
 	NULL
 };
 
-static struct attribute_group dock_attribute_group = {
+static const struct attribute_group dock_attribute_group = {
 	.attrs = dock_attributes
 };
 
diff --git a/drivers/acpi/ec.c b/drivers/acpi/ec.c
index 62068a5e814f..236b14324780 100644
--- a/drivers/acpi/ec.c
+++ b/drivers/acpi/ec.c
@@ -112,8 +112,7 @@ enum {
 	EC_FLAGS_EVT_HANDLER_INSTALLED, /* _Qxx handlers installed */
 	EC_FLAGS_STARTED,		/* Driver is started */
 	EC_FLAGS_STOPPED,		/* Driver is stopped */
-	EC_FLAGS_COMMAND_STORM,		/* GPE storms occurred to the
-					 * current command processing */
+	EC_FLAGS_GPE_MASKED,		/* GPE masked */
 };
 
 #define ACPI_EC_COMMAND_POLL		0x01 /* Available for command byte */
@@ -425,19 +424,19 @@ static void acpi_ec_complete_request(struct acpi_ec *ec)
 		wake_up(&ec->wait);
 }
 
-static void acpi_ec_set_storm(struct acpi_ec *ec, u8 flag)
+static void acpi_ec_mask_gpe(struct acpi_ec *ec)
 {
-	if (!test_bit(flag, &ec->flags)) {
+	if (!test_bit(EC_FLAGS_GPE_MASKED, &ec->flags)) {
 		acpi_ec_disable_gpe(ec, false);
 		ec_dbg_drv("Polling enabled");
-		set_bit(flag, &ec->flags);
+		set_bit(EC_FLAGS_GPE_MASKED, &ec->flags);
 	}
 }
 
-static void acpi_ec_clear_storm(struct acpi_ec *ec, u8 flag)
+static void acpi_ec_unmask_gpe(struct acpi_ec *ec)
 {
-	if (test_bit(flag, &ec->flags)) {
-		clear_bit(flag, &ec->flags);
+	if (test_bit(EC_FLAGS_GPE_MASKED, &ec->flags)) {
+		clear_bit(EC_FLAGS_GPE_MASKED, &ec->flags);
 		acpi_ec_enable_gpe(ec, false);
 		ec_dbg_drv("Polling disabled");
 	}
@@ -464,7 +463,7 @@ static bool acpi_ec_submit_flushable_request(struct acpi_ec *ec)
 
 static void acpi_ec_submit_query(struct acpi_ec *ec)
 {
-	acpi_ec_set_storm(ec, EC_FLAGS_COMMAND_STORM);
+	acpi_ec_mask_gpe(ec);
 	if (!acpi_ec_event_enabled(ec))
 		return;
 	if (!test_and_set_bit(EC_FLAGS_QUERY_PENDING, &ec->flags)) {
@@ -480,7 +479,7 @@ static void acpi_ec_complete_query(struct acpi_ec *ec)
 	if (test_and_clear_bit(EC_FLAGS_QUERY_PENDING, &ec->flags))
 		ec_dbg_evt("Command(%s) unblocked",
 			   acpi_ec_cmd_string(ACPI_EC_COMMAND_QUERY));
-	acpi_ec_clear_storm(ec, EC_FLAGS_COMMAND_STORM);
+	acpi_ec_unmask_gpe(ec);
 }
 
 static inline void __acpi_ec_enable_event(struct acpi_ec *ec)
@@ -700,7 +699,7 @@ err:
 				++t->irq_count;
 			/* Allow triggering on 0 threshold */
 			if (t->irq_count == ec_storm_threshold)
-				acpi_ec_set_storm(ec, EC_FLAGS_COMMAND_STORM);
+				acpi_ec_mask_gpe(ec);
 		}
 	}
 out:
@@ -798,7 +797,7 @@ static int acpi_ec_transaction_unlocked(struct acpi_ec *ec,
 
 	spin_lock_irqsave(&ec->lock, tmp);
 	if (t->irq_count == ec_storm_threshold)
-		acpi_ec_clear_storm(ec, EC_FLAGS_COMMAND_STORM);
+		acpi_ec_unmask_gpe(ec);
 	ec_dbg_req("Command(%s) stopped", acpi_ec_cmd_string(t->command));
 	ec->curr = NULL;
 	/* Disable GPE for command processing (IBF=0/OBF=1) */
@@ -1586,9 +1585,7 @@ static bool acpi_is_boot_ec(struct acpi_ec *ec)
 {
 	if (!boot_ec)
 		return false;
-	if (ec->handle == boot_ec->handle &&
-	    ec->gpe == boot_ec->gpe &&
-	    ec->command_addr == boot_ec->command_addr &&
+	if (ec->command_addr == boot_ec->command_addr &&
 	    ec->data_addr == boot_ec->data_addr)
 		return true;
 	return false;
@@ -1613,6 +1610,13 @@ static int acpi_ec_add(struct acpi_device *device)
 
 	if (acpi_is_boot_ec(ec)) {
 		boot_ec_is_ecdt = false;
+		/*
+		 * Trust PNP0C09 namespace location rather than ECDT ID.
+		 *
+		 * But trust ECDT GPE rather than _GPE because of ASUS quirks,
+		 * so do not change boot_ec->gpe to ec->gpe.
+		 */
+		boot_ec->handle = ec->handle;
 		acpi_handle_debug(ec->handle, "duplicated.\n");
 		acpi_ec_free(ec);
 		ec = boot_ec;
@@ -1741,24 +1745,26 @@ error:
  * functioning ECDT EC first in order to handle the events.
  * https://bugzilla.kernel.org/show_bug.cgi?id=115021
  */
-int __init acpi_ec_ecdt_start(void)
+static int __init acpi_ec_ecdt_start(void)
 {
 	acpi_handle handle;
 
 	if (!boot_ec)
 		return -ENODEV;
-	/*
-	 * The DSDT EC should have already been started in
-	 * acpi_ec_add().
-	 */
+	/* In case acpi_ec_ecdt_start() is called after acpi_ec_add() */
 	if (!boot_ec_is_ecdt)
 		return -ENODEV;
 
 	/*
 	 * At this point, the namespace and the GPE is initialized, so
 	 * start to find the namespace objects and handle the events.
+	 *
+	 * Note: ec->handle can be valid if this function is called after
+	 * acpi_ec_add(), hence the fast path.
 	 */
-	if (!acpi_ec_ecdt_get_handle(&handle))
+	if (boot_ec->handle != ACPI_ROOT_OBJECT)
+		handle = boot_ec->handle;
+	else if (!acpi_ec_ecdt_get_handle(&handle))
 		return -ENODEV;
 	return acpi_config_boot_ec(boot_ec, handle, true, true);
 }
@@ -1803,7 +1809,7 @@ static int ec_honor_ecdt_gpe(const struct dmi_system_id *id)
 	return 0;
 }
 
-static struct dmi_system_id ec_dmi_table[] __initdata = {
+static const struct dmi_system_id ec_dmi_table[] __initconst = {
 	{
 	ec_correct_ecdt, "MSI MS-171F", {
 	DMI_MATCH(DMI_SYS_VENDOR, "Micro-Star"),
@@ -2003,20 +2009,17 @@ static inline void acpi_ec_query_exit(void)
 int __init acpi_ec_init(void)
 {
 	int result;
+	int ecdt_fail, dsdt_fail;
 
 	/* register workqueue for _Qxx evaluations */
 	result = acpi_ec_query_init();
 	if (result)
-		goto err_exit;
-	/* Now register the driver for the EC */
-	result = acpi_bus_register_driver(&acpi_ec_driver);
-	if (result)
-		goto err_exit;
+		return result;
 
-err_exit:
-	if (result)
-		acpi_ec_query_exit();
-	return result;
+	/* Drivers must be started after acpi_ec_query_init() */
+	dsdt_fail = acpi_bus_register_driver(&acpi_ec_driver);
+	ecdt_fail = acpi_ec_ecdt_start();
+	return ecdt_fail && dsdt_fail ? -ENODEV : 0;
 }
 
 /* EC driver currently not unloadable */
diff --git a/drivers/acpi/internal.h b/drivers/acpi/internal.h
index 58dd7ab3c653..4361c4415b4f 100644
--- a/drivers/acpi/internal.h
+++ b/drivers/acpi/internal.h
@@ -185,7 +185,6 @@ typedef int (*acpi_ec_query_func) (void *data);
 int acpi_ec_init(void);
 int acpi_ec_ecdt_probe(void);
 int acpi_ec_dsdt_probe(void);
-int acpi_ec_ecdt_start(void);
 void acpi_ec_block_transactions(void);
 void acpi_ec_unblock_transactions(void);
 int acpi_ec_add_query_handler(struct acpi_ec *ec, u8 query_bit,
@@ -233,6 +232,12 @@ static inline void suspend_nvs_restore(void) {}
 void acpi_init_properties(struct acpi_device *adev);
 void acpi_free_properties(struct acpi_device *adev);
 
+#ifdef CONFIG_X86
+void acpi_extract_apple_properties(struct acpi_device *adev);
+#else
+static inline void acpi_extract_apple_properties(struct acpi_device *adev) {}
+#endif
+
 /*--------------------------------------------------------------------------
 				Watchdog
   -------------------------------------------------------------------------- */
diff --git a/drivers/acpi/nfit/Kconfig b/drivers/acpi/nfit/Kconfig
index 6d3351452ea2..929ba4da0b30 100644
--- a/drivers/acpi/nfit/Kconfig
+++ b/drivers/acpi/nfit/Kconfig
@@ -2,7 +2,7 @@ config ACPI_NFIT
 	tristate "ACPI NVDIMM Firmware Interface Table (NFIT)"
 	depends on PHYS_ADDR_T_64BIT
 	depends on BLK_DEV
-	depends on ARCH_HAS_MMIO_FLUSH
+	depends on ARCH_HAS_PMEM_API
 	select LIBNVDIMM
 	help
 	  Infrastructure to probe ACPI 6 compliant platforms for
diff --git a/drivers/acpi/nfit/core.c b/drivers/acpi/nfit/core.c
index 19182d091587..9c2c49b6a240 100644
--- a/drivers/acpi/nfit/core.c
+++ b/drivers/acpi/nfit/core.c
@@ -228,6 +228,10 @@ int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc, struct nvdimm *nvdimm,
 	if (cmd == ND_CMD_CALL) {
 		call_pkg = buf;
 		func = call_pkg->nd_command;
+
+		for (i = 0; i < ARRAY_SIZE(call_pkg->nd_reserved2); i++)
+			if (call_pkg->nd_reserved2[i])
+				return -EINVAL;
 	}
 
 	if (nvdimm) {
@@ -1674,8 +1678,19 @@ static ssize_t range_index_show(struct device *dev,
 }
 static DEVICE_ATTR_RO(range_index);
 
+static ssize_t ecc_unit_size_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct nd_region *nd_region = to_nd_region(dev);
+	struct nfit_spa *nfit_spa = nd_region_provider_data(nd_region);
+
+	return sprintf(buf, "%d\n", nfit_spa->clear_err_unit);
+}
+static DEVICE_ATTR_RO(ecc_unit_size);
+
 static struct attribute *acpi_nfit_region_attributes[] = {
 	&dev_attr_range_index.attr,
+	&dev_attr_ecc_unit_size.attr,
 	NULL,
 };
 
@@ -1804,6 +1819,7 @@ static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc *acpi_desc,
 		struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
 		struct acpi_nfit_memory_map *memdev = memdev_from_spa(acpi_desc,
 				spa->range_index, i);
+		struct acpi_nfit_control_region *dcr = nfit_mem->dcr;
 
 		if (!memdev || !nfit_mem->dcr) {
 			dev_err(dev, "%s: failed to find DCR\n", __func__);
@@ -1811,13 +1827,13 @@ static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc *acpi_desc,
 		}
 
 		map->region_offset = memdev->region_offset;
-		map->serial_number = nfit_mem->dcr->serial_number;
+		map->serial_number = dcr->serial_number;
 
 		map2->region_offset = memdev->region_offset;
-		map2->serial_number = nfit_mem->dcr->serial_number;
-		map2->vendor_id = nfit_mem->dcr->vendor_id;
-		map2->manufacturing_date = nfit_mem->dcr->manufacturing_date;
-		map2->manufacturing_location = nfit_mem->dcr->manufacturing_location;
+		map2->serial_number = dcr->serial_number;
+		map2->vendor_id = dcr->vendor_id;
+		map2->manufacturing_date = dcr->manufacturing_date;
+		map2->manufacturing_location = dcr->manufacturing_location;
 	}
 
 	/* v1.1 namespaces */
@@ -1835,6 +1851,28 @@ static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc *acpi_desc,
 			cmp_map_compat, NULL);
 	nd_set->altcookie = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0);
 
+	/* record the result of the sort for the mapping position */
+	for (i = 0; i < nr; i++) {
+		struct nfit_set_info_map2 *map2 = &info2->mapping[i];
+		int j;
+
+		for (j = 0; j < nr; j++) {
+			struct nd_mapping_desc *mapping = &ndr_desc->mapping[j];
+			struct nvdimm *nvdimm = mapping->nvdimm;
+			struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
+			struct acpi_nfit_control_region *dcr = nfit_mem->dcr;
+
+			if (map2->serial_number == dcr->serial_number &&
+			    map2->vendor_id == dcr->vendor_id &&
+			    map2->manufacturing_date == dcr->manufacturing_date &&
+			    map2->manufacturing_location
+				    == dcr->manufacturing_location) {
+				mapping->position = i;
+				break;
+			}
+		}
+	}
+
 	ndr_desc->nd_set = nd_set;
 	devm_kfree(dev, info);
 	devm_kfree(dev, info2);
@@ -1930,7 +1968,7 @@ static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk,
 			memcpy_flushcache(mmio->addr.aperture + offset, iobuf + copied, c);
 		else {
 			if (nfit_blk->dimm_flags & NFIT_BLK_READ_FLUSH)
-				mmio_flush_range((void __force *)
+				arch_invalidate_pmem((void __force *)
 					mmio->addr.aperture + offset, c);
 
 			memcpy(iobuf + copied, mmio->addr.aperture + offset, c);
@@ -2884,7 +2922,7 @@ static int acpi_nfit_flush_probe(struct nvdimm_bus_descriptor *nd_desc)
 	 * need to be interruptible while waiting.
 	 */
 	INIT_WORK_ONSTACK(&flush.work, flush_probe);
-	COMPLETION_INITIALIZER_ONSTACK(flush.cmp);
+	init_completion(&flush.cmp);
 	queue_work(nfit_wq, &flush.work);
 	mutex_unlock(&acpi_desc->init_mutex);
 
diff --git a/drivers/acpi/osi.c b/drivers/acpi/osi.c
index 723bee58bbcf..76998a51bf99 100644
--- a/drivers/acpi/osi.c
+++ b/drivers/acpi/osi.c
@@ -27,6 +27,7 @@
 #include <linux/kernel.h>
 #include <linux/acpi.h>
 #include <linux/dmi.h>
+#include <linux/platform_data/x86/apple.h>
 
 #include "internal.h"
 
@@ -257,12 +258,11 @@ bool acpi_osi_is_win8(void)
 }
 EXPORT_SYMBOL(acpi_osi_is_win8);
 
-static void __init acpi_osi_dmi_darwin(bool enable,
-				       const struct dmi_system_id *d)
+static void __init acpi_osi_dmi_darwin(void)
 {
-	pr_notice("DMI detected to setup _OSI(\"Darwin\"): %s\n", d->ident);
+	pr_notice("DMI detected to setup _OSI(\"Darwin\"): Apple hardware\n");
 	osi_config.darwin_dmi = 1;
-	__acpi_osi_setup_darwin(enable);
+	__acpi_osi_setup_darwin(true);
 }
 
 static void __init acpi_osi_dmi_linux(bool enable,
@@ -273,13 +273,6 @@ static void __init acpi_osi_dmi_linux(bool enable,
 	__acpi_osi_setup_linux(enable);
 }
 
-static int __init dmi_enable_osi_darwin(const struct dmi_system_id *d)
-{
-	acpi_osi_dmi_darwin(true, d);
-
-	return 0;
-}
-
 static int __init dmi_enable_osi_linux(const struct dmi_system_id *d)
 {
 	acpi_osi_dmi_linux(true, d);
@@ -319,7 +312,7 @@ static int __init dmi_disable_osi_win8(const struct dmi_system_id *d)
  * Note that _OSI("Linux")/_OSI("Darwin") determined here can be overridden
  * by acpi_osi=!Linux/acpi_osi=!Darwin command line options.
  */
-static struct dmi_system_id acpi_osi_dmi_table[] __initdata = {
+static const struct dmi_system_id acpi_osi_dmi_table[] __initconst = {
 	{
 	.callback = dmi_disable_osi_vista,
 	.ident = "Fujitsu Siemens",
@@ -481,30 +474,16 @@ static struct dmi_system_id acpi_osi_dmi_table[] __initdata = {
 		     DMI_MATCH(DMI_PRODUCT_NAME, "1015PX"),
 		},
 	},
-
-	/*
-	 * Enable _OSI("Darwin") for all apple platforms.
-	 */
-	{
-	.callback = dmi_enable_osi_darwin,
-	.ident = "Apple hardware",
-	.matches = {
-		     DMI_MATCH(DMI_SYS_VENDOR, "Apple Inc."),
-		},
-	},
-	{
-	.callback = dmi_enable_osi_darwin,
-	.ident = "Apple hardware",
-	.matches = {
-		     DMI_MATCH(DMI_SYS_VENDOR, "Apple Computer, Inc."),
-		},
-	},
 	{}
 };
 
 static __init void acpi_osi_dmi_blacklisted(void)
 {
 	dmi_check_system(acpi_osi_dmi_table);
+
+	/* Enable _OSI("Darwin") for Apple platforms. */
+	if (x86_apple_machine)
+		acpi_osi_dmi_darwin();
 }
 
 int __init early_acpi_osi_init(void)
diff --git a/drivers/acpi/pci_root.c b/drivers/acpi/pci_root.c
index 9eec3095e6c3..6fc204a52493 100644
--- a/drivers/acpi/pci_root.c
+++ b/drivers/acpi/pci_root.c
@@ -33,6 +33,7 @@
 #include <linux/acpi.h>
 #include <linux/slab.h>
 #include <linux/dmi.h>
+#include <linux/platform_data/x86/apple.h>
 #include <acpi/apei.h>	/* for acpi_hest_init() */
 
 #include "internal.h"
@@ -431,8 +432,7 @@ static void negotiate_os_control(struct acpi_pci_root *root, int *no_aspm)
 	 * been called successfully. We know the feature set supported by the
 	 * platform, so avoid calling _OSC at all
 	 */
-
-	if (dmi_match(DMI_SYS_VENDOR, "Apple Inc.")) {
+	if (x86_apple_machine) {
 		root->osc_control_set = ~OSC_PCI_EXPRESS_PME_CONTROL;
 		decode_osc_control(root, "OS assumes control of",
 				   root->osc_control_set);
diff --git a/drivers/acpi/pci_slot.c b/drivers/acpi/pci_slot.c
index f62c68e24317..e90b61f7d2db 100644
--- a/drivers/acpi/pci_slot.c
+++ b/drivers/acpi/pci_slot.c
@@ -174,7 +174,7 @@ static int do_sta_before_sun(const struct dmi_system_id *d)
 	return 0;
 }
 
-static struct dmi_system_id acpi_pci_slot_dmi_table[] __initdata = {
+static const struct dmi_system_id acpi_pci_slot_dmi_table[] __initconst = {
 	/*
 	 * Fujitsu Primequest machines will return 1023 to indicate an
 	 * error if the _SUN method is evaluated on SxFy objects that
diff --git a/drivers/acpi/pmic/intel_pmic_xpower.c b/drivers/acpi/pmic/intel_pmic_xpower.c
index 3b7d5be5b7ed..6c99d3f81095 100644
--- a/drivers/acpi/pmic/intel_pmic_xpower.c
+++ b/drivers/acpi/pmic/intel_pmic_xpower.c
@@ -27,6 +27,9 @@
 #define GPI1_LDO_ON		(3 << 0)
 #define GPI1_LDO_OFF		(4 << 0)
 
+#define AXP288_ADC_TS_PIN_GPADC	0xf2
+#define AXP288_ADC_TS_PIN_ON	0xf3
+
 static struct pmic_table power_table[] = {
 	{
 		.address = 0x00,
@@ -209,11 +212,23 @@ static int intel_xpower_pmic_update_power(struct regmap *regmap, int reg,
 static int intel_xpower_pmic_get_raw_temp(struct regmap *regmap, int reg)
 {
 	u8 buf[2];
+	int ret;
 
-	if (regmap_bulk_read(regmap, AXP288_GP_ADC_H, buf, 2))
-		return -EIO;
+	ret = regmap_write(regmap, AXP288_ADC_TS_PIN_CTRL,
+			   AXP288_ADC_TS_PIN_GPADC);
+	if (ret)
+		return ret;
+
+	/* After switching to the GPADC pin give things some time to settle */
+	usleep_range(6000, 10000);
+
+	ret = regmap_bulk_read(regmap, AXP288_GP_ADC_H, buf, 2);
+	if (ret == 0)
+		ret = (buf[0] << 4) + ((buf[1] >> 4) & 0x0f);
+
+	regmap_write(regmap, AXP288_ADC_TS_PIN_CTRL, AXP288_ADC_TS_PIN_ON);
 
-	return (buf[0] << 4) + ((buf[1] >> 4) & 0x0F);
+	return ret;
 }
 
 static struct intel_pmic_opregion_data intel_xpower_pmic_opregion_data = {
diff --git a/drivers/acpi/processor_driver.c b/drivers/acpi/processor_driver.c
index 591d1dd3f04e..9d6aff22684e 100644
--- a/drivers/acpi/processor_driver.c
+++ b/drivers/acpi/processor_driver.c
@@ -237,7 +237,7 @@ static int __acpi_processor_start(struct acpi_device *device)
 
 	result = acpi_cppc_processor_probe(pr);
 	if (result && !IS_ENABLED(CONFIG_ACPI_CPU_FREQ_PSS))
-		dev_warn(&device->dev, "CPPC data invalid or not present\n");
+		dev_dbg(&device->dev, "CPPC data invalid or not present\n");
 
 	if (!cpuidle_get_driver() || cpuidle_get_driver() == &acpi_idle_driver)
 		acpi_processor_power_init(pr);
diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c
index 5c8aa9cf62d7..2736e25e9dc6 100644
--- a/drivers/acpi/processor_idle.c
+++ b/drivers/acpi/processor_idle.c
@@ -48,6 +48,8 @@
 #define _COMPONENT              ACPI_PROCESSOR_COMPONENT
 ACPI_MODULE_NAME("processor_idle");
 
+#define ACPI_IDLE_STATE_START	(IS_ENABLED(CONFIG_ARCH_HAS_CPU_RELAX) ? 1 : 0)
+
 static unsigned int max_cstate __read_mostly = ACPI_PROCESSOR_MAX_POWER;
 module_param(max_cstate, uint, 0000);
 static unsigned int nocst __read_mostly;
@@ -708,8 +710,6 @@ static DEFINE_RAW_SPINLOCK(c3_lock);
 static void acpi_idle_enter_bm(struct acpi_processor *pr,
 			       struct acpi_processor_cx *cx, bool timer_bc)
 {
-	acpi_unlazy_tlb(smp_processor_id());
-
 	/*
 	 * Must be done before busmaster disable as we might need to
 	 * access HPET !
@@ -761,7 +761,7 @@ static int acpi_idle_enter(struct cpuidle_device *dev,
 
 	if (cx->type != ACPI_STATE_C1) {
 		if (acpi_idle_fallback_to_c1(pr) && num_online_cpus() > 1) {
-			index = CPUIDLE_DRIVER_STATE_START;
+			index = ACPI_IDLE_STATE_START;
 			cx = per_cpu(acpi_cstate[index], dev->cpu);
 		} else if (cx->type == ACPI_STATE_C3 && pr->flags.bm_check) {
 			if (cx->bm_sts_skip || !acpi_idle_bm_check()) {
@@ -789,7 +789,7 @@ static int acpi_idle_enter(struct cpuidle_device *dev,
 	return index;
 }
 
-static void acpi_idle_enter_freeze(struct cpuidle_device *dev,
+static void acpi_idle_enter_s2idle(struct cpuidle_device *dev,
 				   struct cpuidle_driver *drv, int index)
 {
 	struct acpi_processor_cx *cx = per_cpu(acpi_cstate[index], dev->cpu);
@@ -813,7 +813,7 @@ static void acpi_idle_enter_freeze(struct cpuidle_device *dev,
 static int acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr,
 					   struct cpuidle_device *dev)
 {
-	int i, count = CPUIDLE_DRIVER_STATE_START;
+	int i, count = ACPI_IDLE_STATE_START;
 	struct acpi_processor_cx *cx;
 
 	if (max_cstate == 0)
@@ -840,7 +840,7 @@ static int acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr,
 
 static int acpi_processor_setup_cstates(struct acpi_processor *pr)
 {
-	int i, count = CPUIDLE_DRIVER_STATE_START;
+	int i, count;
 	struct acpi_processor_cx *cx;
 	struct cpuidle_state *state;
 	struct cpuidle_driver *drv = &acpi_idle_driver;
@@ -848,6 +848,13 @@ static int acpi_processor_setup_cstates(struct acpi_processor *pr)
 	if (max_cstate == 0)
 		max_cstate = 1;
 
+	if (IS_ENABLED(CONFIG_ARCH_HAS_CPU_RELAX)) {
+		cpuidle_poll_state_init(drv);
+		count = 1;
+	} else {
+		count = 0;
+	}
+
 	for (i = 1; i < ACPI_PROCESSOR_MAX_POWER && i <= max_cstate; i++) {
 		cx = &pr->power.states[i];
 
@@ -867,14 +874,14 @@ static int acpi_processor_setup_cstates(struct acpi_processor *pr)
 			drv->safe_state_index = count;
 		}
 		/*
-		 * Halt-induced C1 is not good for ->enter_freeze, because it
+		 * Halt-induced C1 is not good for ->enter_s2idle, because it
 		 * re-enables interrupts on exit.  Moreover, C1 is generally not
 		 * particularly interesting from the suspend-to-idle angle, so
 		 * avoid C1 and the situations in which we may need to fall back
 		 * to it altogether.
 		 */
 		if (cx->type != ACPI_STATE_C1 && !acpi_idle_fallback_to_c1(pr))
-			state->enter_freeze = acpi_idle_enter_freeze;
+			state->enter_s2idle = acpi_idle_enter_s2idle;
 
 		count++;
 		if (count == CPUIDLE_STATE_MAX)
@@ -1291,7 +1298,7 @@ static int acpi_processor_setup_cpuidle_states(struct acpi_processor *pr)
 		return -EINVAL;
 
 	drv->safe_state_index = -1;
-	for (i = CPUIDLE_DRIVER_STATE_START; i < CPUIDLE_STATE_MAX; i++) {
+	for (i = ACPI_IDLE_STATE_START; i < CPUIDLE_STATE_MAX; i++) {
 		drv->states[i].name[0] = '\0';
 		drv->states[i].desc[0] = '\0';
 	}
diff --git a/drivers/acpi/processor_pdc.c b/drivers/acpi/processor_pdc.c
index 7cfbda4d7c51..74f738cb6073 100644
--- a/drivers/acpi/processor_pdc.c
+++ b/drivers/acpi/processor_pdc.c
@@ -173,7 +173,7 @@ static int __init set_no_mwait(const struct dmi_system_id *id)
 	return 0;
 }
 
-static struct dmi_system_id processor_idle_dmi_table[] __initdata = {
+static const struct dmi_system_id processor_idle_dmi_table[] __initconst = {
 	{
 	set_no_mwait, "Extensa 5220", {
 	DMI_MATCH(DMI_BIOS_VENDOR, "Phoenix Technologies LTD"),
diff --git a/drivers/acpi/property.c b/drivers/acpi/property.c
index 917c789f953d..c1c216163de3 100644
--- a/drivers/acpi/property.c
+++ b/drivers/acpi/property.c
@@ -19,21 +19,19 @@
 
 #include "internal.h"
 
-static int acpi_data_get_property_array(struct acpi_device_data *data,
+static int acpi_data_get_property_array(const struct acpi_device_data *data,
 					const char *name,
 					acpi_object_type type,
 					const union acpi_object **obj);
 
-/* ACPI _DSD device properties UUID: daffd814-6eba-4d8c-8a91-bc9bbf4aa301 */
-static const u8 prp_uuid[16] = {
-	0x14, 0xd8, 0xff, 0xda, 0xba, 0x6e, 0x8c, 0x4d,
-	0x8a, 0x91, 0xbc, 0x9b, 0xbf, 0x4a, 0xa3, 0x01
-};
-/* ACPI _DSD data subnodes UUID: dbb8e3e6-5886-4ba6-8795-1319f52a966b */
-static const u8 ads_uuid[16] = {
-	0xe6, 0xe3, 0xb8, 0xdb, 0x86, 0x58, 0xa6, 0x4b,
-	0x87, 0x95, 0x13, 0x19, 0xf5, 0x2a, 0x96, 0x6b
-};
+/* ACPI _DSD device properties GUID: daffd814-6eba-4d8c-8a91-bc9bbf4aa301 */
+static const guid_t prp_guid =
+	GUID_INIT(0xdaffd814, 0x6eba, 0x4d8c,
+		  0x8a, 0x91, 0xbc, 0x9b, 0xbf, 0x4a, 0xa3, 0x01);
+/* ACPI _DSD data subnodes GUID: dbb8e3e6-5886-4ba6-8795-1319f52a966b */
+static const guid_t ads_guid =
+	GUID_INIT(0xdbb8e3e6, 0x5886, 0x4ba6,
+		  0x87, 0x95, 0x13, 0x19, 0xf5, 0x2a, 0x96, 0x6b);
 
 static bool acpi_enumerate_nondev_subnodes(acpi_handle scope,
 					   const union acpi_object *desc,
@@ -56,8 +54,7 @@ static bool acpi_nondev_subnode_extract(const union acpi_object *desc,
 		return false;
 
 	dn->name = link->package.elements[0].string.pointer;
-	dn->fwnode.type = FWNODE_ACPI_DATA;
-	dn->fwnode.ops = &acpi_fwnode_ops;
+	dn->fwnode.ops = &acpi_data_fwnode_ops;
 	dn->parent = parent;
 	INIT_LIST_HEAD(&dn->data.subnodes);
 
@@ -190,22 +187,23 @@ static bool acpi_enumerate_nondev_subnodes(acpi_handle scope,
 {
 	int i;
 
-	/* Look for the ACPI data subnodes UUID. */
+	/* Look for the ACPI data subnodes GUID. */
 	for (i = 0; i < desc->package.count; i += 2) {
-		const union acpi_object *uuid, *links;
+		const union acpi_object *guid, *links;
 
-		uuid = &desc->package.elements[i];
+		guid = &desc->package.elements[i];
 		links = &desc->package.elements[i + 1];
 
 		/*
-		 * The first element must be a UUID and the second one must be
+		 * The first element must be a GUID and the second one must be
 		 * a package.
 		 */
-		if (uuid->type != ACPI_TYPE_BUFFER || uuid->buffer.length != 16
-		    || links->type != ACPI_TYPE_PACKAGE)
+		if (guid->type != ACPI_TYPE_BUFFER ||
+		    guid->buffer.length != 16 ||
+		    links->type != ACPI_TYPE_PACKAGE)
 			break;
 
-		if (memcmp(uuid->buffer.pointer, ads_uuid, sizeof(ads_uuid)))
+		if (!guid_equal((guid_t *)guid->buffer.pointer, &ads_guid))
 			continue;
 
 		return acpi_add_nondev_subnodes(scope, links, &data->subnodes,
@@ -298,26 +296,27 @@ static bool acpi_extract_properties(const union acpi_object *desc,
 	if (desc->package.count % 2)
 		return false;
 
-	/* Look for the device properties UUID. */
+	/* Look for the device properties GUID. */
 	for (i = 0; i < desc->package.count; i += 2) {
-		const union acpi_object *uuid, *properties;
+		const union acpi_object *guid, *properties;
 
-		uuid = &desc->package.elements[i];
+		guid = &desc->package.elements[i];
 		properties = &desc->package.elements[i + 1];
 
 		/*
-		 * The first element must be a UUID and the second one must be
+		 * The first element must be a GUID and the second one must be
 		 * a package.
 		 */
-		if (uuid->type != ACPI_TYPE_BUFFER || uuid->buffer.length != 16
-		    || properties->type != ACPI_TYPE_PACKAGE)
+		if (guid->type != ACPI_TYPE_BUFFER ||
+		    guid->buffer.length != 16 ||
+		    properties->type != ACPI_TYPE_PACKAGE)
 			break;
 
-		if (memcmp(uuid->buffer.pointer, prp_uuid, sizeof(prp_uuid)))
+		if (!guid_equal((guid_t *)guid->buffer.pointer, &prp_guid))
 			continue;
 
 		/*
-		 * We found the matching UUID. Now validate the format of the
+		 * We found the matching GUID. Now validate the format of the
 		 * package immediately following it.
 		 */
 		if (!acpi_properties_format_valid(properties))
@@ -339,6 +338,9 @@ void acpi_init_properties(struct acpi_device *adev)
 
 	INIT_LIST_HEAD(&adev->data.subnodes);
 
+	if (!adev->handle)
+		return;
+
 	/*
 	 * Check if ACPI_DT_NAMESPACE_HID is present and inthat case we fill in
 	 * Device Tree compatible properties for this device.
@@ -373,6 +375,9 @@ void acpi_init_properties(struct acpi_device *adev)
 	if (acpi_of && !adev->flags.of_compatible_ok)
 		acpi_handle_info(adev->handle,
 			 ACPI_DT_NAMESPACE_HID " requires 'compatible' property\n");
+
+	if (!adev->data.pointer)
+		acpi_extract_apple_properties(adev);
 }
 
 static void acpi_destroy_nondev_subnodes(struct list_head *list)
@@ -418,7 +423,7 @@ void acpi_free_properties(struct acpi_device *adev)
  *         %-EINVAL if the property doesn't exist,
  *         %-EPROTO if the property value type doesn't match @type.
  */
-static int acpi_data_get_property(struct acpi_device_data *data,
+static int acpi_data_get_property(const struct acpi_device_data *data,
 				  const char *name, acpi_object_type type,
 				  const union acpi_object **obj)
 {
@@ -460,20 +465,21 @@ static int acpi_data_get_property(struct acpi_device_data *data,
  * @type: Expected property type.
  * @obj: Location to store the property value (if not %NULL).
  */
-int acpi_dev_get_property(struct acpi_device *adev, const char *name,
+int acpi_dev_get_property(const struct acpi_device *adev, const char *name,
 			  acpi_object_type type, const union acpi_object **obj)
 {
 	return adev ? acpi_data_get_property(&adev->data, name, type, obj) : -EINVAL;
 }
 EXPORT_SYMBOL_GPL(acpi_dev_get_property);
 
-static struct acpi_device_data *acpi_device_data_of_node(struct fwnode_handle *fwnode)
+static const struct acpi_device_data *
+acpi_device_data_of_node(const struct fwnode_handle *fwnode)
 {
-	if (fwnode->type == FWNODE_ACPI) {
-		struct acpi_device *adev = to_acpi_device_node(fwnode);
+	if (is_acpi_device_node(fwnode)) {
+		const struct acpi_device *adev = to_acpi_device_node(fwnode);
 		return &adev->data;
-	} else if (fwnode->type == FWNODE_ACPI_DATA) {
-		struct acpi_data_node *dn = to_acpi_data_node(fwnode);
+	} else if (is_acpi_data_node(fwnode)) {
+		const struct acpi_data_node *dn = to_acpi_data_node(fwnode);
 		return &dn->data;
 	}
 	return NULL;
@@ -485,8 +491,8 @@ static struct acpi_device_data *acpi_device_data_of_node(struct fwnode_handle *f
  * @propname: Name of the property.
  * @valptr: Location to store a pointer to the property value (if not %NULL).
  */
-int acpi_node_prop_get(struct fwnode_handle *fwnode, const char *propname,
-		       void **valptr)
+int acpi_node_prop_get(const struct fwnode_handle *fwnode,
+		       const char *propname, void **valptr)
 {
 	return acpi_data_get_property(acpi_device_data_of_node(fwnode),
 				      propname, ACPI_TYPE_ANY,
@@ -512,7 +518,7 @@ int acpi_node_prop_get(struct fwnode_handle *fwnode, const char *propname,
  *         %-EPROTO if the property is not a package or the type of its elements
  *           doesn't match @type.
  */
-static int acpi_data_get_property_array(struct acpi_device_data *data,
+static int acpi_data_get_property_array(const struct acpi_device_data *data,
 					const char *name,
 					acpi_object_type type,
 					const union acpi_object **obj)
@@ -572,13 +578,13 @@ static int acpi_data_get_property_array(struct acpi_device_data *data,
  *
  * Return: %0 on success, negative error code on failure.
  */
-int __acpi_node_get_property_reference(struct fwnode_handle *fwnode,
+int __acpi_node_get_property_reference(const struct fwnode_handle *fwnode,
 	const char *propname, size_t index, size_t num_args,
 	struct acpi_reference_args *args)
 {
 	const union acpi_object *element, *end;
 	const union acpi_object *obj;
-	struct acpi_device_data *data;
+	const struct acpi_device_data *data;
 	struct acpi_device *device;
 	int ret, idx = 0;
 
@@ -674,7 +680,7 @@ int __acpi_node_get_property_reference(struct fwnode_handle *fwnode,
 }
 EXPORT_SYMBOL_GPL(__acpi_node_get_property_reference);
 
-static int acpi_data_prop_read_single(struct acpi_device_data *data,
+static int acpi_data_prop_read_single(const struct acpi_device_data *data,
 				      const char *propname,
 				      enum dev_prop_type proptype, void *val)
 {
@@ -813,7 +819,7 @@ static int acpi_copy_property_array_string(const union acpi_object *items,
 	return nval;
 }
 
-static int acpi_data_prop_read(struct acpi_device_data *data,
+static int acpi_data_prop_read(const struct acpi_device_data *data,
 			       const char *propname,
 			       enum dev_prop_type proptype,
 			       void *val, size_t nval)
@@ -867,7 +873,7 @@ static int acpi_data_prop_read(struct acpi_device_data *data,
 	return ret;
 }
 
-int acpi_dev_prop_read(struct acpi_device *adev, const char *propname,
+int acpi_dev_prop_read(const struct acpi_device *adev, const char *propname,
 		       enum dev_prop_type proptype, void *val, size_t nval)
 {
 	return adev ? acpi_data_prop_read(&adev->data, propname, proptype, val, nval) : -EINVAL;
@@ -885,8 +891,9 @@ int acpi_dev_prop_read(struct acpi_device *adev, const char *propname,
  * of the property.  Otherwise, read at most @nval values to the array at the
  * location pointed to by @val.
  */
-int acpi_node_prop_read(struct fwnode_handle *fwnode,  const char *propname,
-		        enum dev_prop_type proptype, void *val, size_t nval)
+int acpi_node_prop_read(const struct fwnode_handle *fwnode,
+			const char *propname, enum dev_prop_type proptype,
+			void *val, size_t nval)
 {
 	return acpi_data_prop_read(acpi_device_data_of_node(fwnode),
 				   propname, proptype, val, nval);
@@ -897,13 +904,15 @@ int acpi_node_prop_read(struct fwnode_handle *fwnode,  const char *propname,
  * @fwnode: Firmware node to find the next child node for.
  * @child: Handle to one of the device's child nodes or a null handle.
  */
-struct fwnode_handle *acpi_get_next_subnode(struct fwnode_handle *fwnode,
+struct fwnode_handle *acpi_get_next_subnode(const struct fwnode_handle *fwnode,
 					    struct fwnode_handle *child)
 {
-	struct acpi_device *adev = to_acpi_device_node(fwnode);
-	struct list_head *head, *next;
+	const struct acpi_device *adev = to_acpi_device_node(fwnode);
+	struct acpi_device *child_adev = NULL;
+	const struct list_head *head;
+	struct list_head *next;
 
-	if (!child || child->type == FWNODE_ACPI) {
+	if (!child || is_acpi_device_node(child)) {
 		if (adev)
 			head = &adev->children;
 		else
@@ -913,26 +922,27 @@ struct fwnode_handle *acpi_get_next_subnode(struct fwnode_handle *fwnode,
 			goto nondev;
 
 		if (child) {
-			adev = to_acpi_device_node(child);
-			next = adev->node.next;
+			child_adev = to_acpi_device_node(child);
+			next = child_adev->node.next;
 			if (next == head) {
 				child = NULL;
 				goto nondev;
 			}
-			adev = list_entry(next, struct acpi_device, node);
+			child_adev = list_entry(next, struct acpi_device, node);
 		} else {
-			adev = list_first_entry(head, struct acpi_device, node);
+			child_adev = list_first_entry(head, struct acpi_device,
+						      node);
 		}
-		return acpi_fwnode_handle(adev);
+		return acpi_fwnode_handle(child_adev);
 	}
 
  nondev:
-	if (!child || child->type == FWNODE_ACPI_DATA) {
-		struct acpi_data_node *data = to_acpi_data_node(fwnode);
+	if (!child || is_acpi_data_node(child)) {
+		const struct acpi_data_node *data = to_acpi_data_node(fwnode);
 		struct acpi_data_node *dn;
 
-		if (adev)
-			head = &adev->data.subnodes;
+		if (child_adev)
+			head = &child_adev->data.subnodes;
 		else if (data)
 			head = &data->data.subnodes;
 		else
@@ -963,7 +973,7 @@ struct fwnode_handle *acpi_get_next_subnode(struct fwnode_handle *fwnode,
  * Returns parent node of an ACPI device or data firmware node or %NULL if
  * not available.
  */
-struct fwnode_handle *acpi_node_get_parent(struct fwnode_handle *fwnode)
+struct fwnode_handle *acpi_node_get_parent(const struct fwnode_handle *fwnode)
 {
 	if (is_acpi_data_node(fwnode)) {
 		/* All data nodes have parent pointer so just return that */
@@ -992,8 +1002,8 @@ struct fwnode_handle *acpi_node_get_parent(struct fwnode_handle *fwnode)
  * %NULL if there is no next endpoint, ERR_PTR() in case of error. In case
  * of success the next endpoint is returned.
  */
-struct fwnode_handle *acpi_graph_get_next_endpoint(struct fwnode_handle *fwnode,
-						   struct fwnode_handle *prev)
+struct fwnode_handle *acpi_graph_get_next_endpoint(
+	const struct fwnode_handle *fwnode, struct fwnode_handle *prev)
 {
 	struct fwnode_handle *port = NULL;
 	struct fwnode_handle *endpoint;
@@ -1040,14 +1050,15 @@ struct fwnode_handle *acpi_graph_get_next_endpoint(struct fwnode_handle *fwnode,
  * the child node on success, NULL otherwise.
  */
 static struct fwnode_handle *acpi_graph_get_child_prop_value(
-	struct fwnode_handle *fwnode, const char *prop_name, unsigned int val)
+	const struct fwnode_handle *fwnode, const char *prop_name,
+	unsigned int val)
 {
 	struct fwnode_handle *child;
 
 	fwnode_for_each_child_node(fwnode, child) {
 		u32 nr;
 
-		if (!fwnode_property_read_u32(fwnode, prop_name, &nr))
+		if (fwnode_property_read_u32(child, prop_name, &nr))
 			continue;
 
 		if (val == nr)
@@ -1069,17 +1080,18 @@ static struct fwnode_handle *acpi_graph_get_child_prop_value(
  * fields requested by the caller. Returns %0 in case of success and
  * negative errno otherwise.
  */
-int acpi_graph_get_remote_endpoint(struct fwnode_handle *fwnode,
+int acpi_graph_get_remote_endpoint(const struct fwnode_handle *__fwnode,
 				   struct fwnode_handle **parent,
 				   struct fwnode_handle **port,
 				   struct fwnode_handle **endpoint)
 {
+	struct fwnode_handle *fwnode;
 	unsigned int port_nr, endpoint_nr;
 	struct acpi_reference_args args;
 	int ret;
 
 	memset(&args, 0, sizeof(args));
-	ret = acpi_node_get_property_reference(fwnode, "remote-endpoint", 0,
+	ret = acpi_node_get_property_reference(__fwnode, "remote-endpoint", 0,
 					       &args);
 	if (ret)
 		return ret;
@@ -1121,7 +1133,7 @@ int acpi_graph_get_remote_endpoint(struct fwnode_handle *fwnode,
 	return 0;
 }
 
-static bool acpi_fwnode_device_is_available(struct fwnode_handle *fwnode)
+static bool acpi_fwnode_device_is_available(const struct fwnode_handle *fwnode)
 {
 	if (!is_acpi_device_node(fwnode))
 		return false;
@@ -1129,16 +1141,17 @@ static bool acpi_fwnode_device_is_available(struct fwnode_handle *fwnode)
 	return acpi_device_is_present(to_acpi_device_node(fwnode));
 }
 
-static bool acpi_fwnode_property_present(struct fwnode_handle *fwnode,
+static bool acpi_fwnode_property_present(const struct fwnode_handle *fwnode,
 					 const char *propname)
 {
 	return !acpi_node_prop_get(fwnode, propname, NULL);
 }
 
-static int acpi_fwnode_property_read_int_array(struct fwnode_handle *fwnode,
-					       const char *propname,
-					       unsigned int elem_size,
-					       void *val, size_t nval)
+static int
+acpi_fwnode_property_read_int_array(const struct fwnode_handle *fwnode,
+				    const char *propname,
+				    unsigned int elem_size, void *val,
+				    size_t nval)
 {
 	enum dev_prop_type type;
 
@@ -1162,16 +1175,17 @@ static int acpi_fwnode_property_read_int_array(struct fwnode_handle *fwnode,
 	return acpi_node_prop_read(fwnode, propname, type, val, nval);
 }
 
-static int acpi_fwnode_property_read_string_array(struct fwnode_handle *fwnode,
-						  const char *propname,
-						  const char **val, size_t nval)
+static int
+acpi_fwnode_property_read_string_array(const struct fwnode_handle *fwnode,
+				       const char *propname, const char **val,
+				       size_t nval)
 {
 	return acpi_node_prop_read(fwnode, propname, DEV_PROP_STRING,
 				   val, nval);
 }
 
 static struct fwnode_handle *
-acpi_fwnode_get_named_child_node(struct fwnode_handle *fwnode,
+acpi_fwnode_get_named_child_node(const struct fwnode_handle *fwnode,
 				 const char *childname)
 {
 	struct fwnode_handle *child;
@@ -1187,8 +1201,34 @@ acpi_fwnode_get_named_child_node(struct fwnode_handle *fwnode,
 	return NULL;
 }
 
+static int
+acpi_fwnode_get_reference_args(const struct fwnode_handle *fwnode,
+			       const char *prop, const char *nargs_prop,
+			       unsigned int args_count, unsigned int index,
+			       struct fwnode_reference_args *args)
+{
+	struct acpi_reference_args acpi_args;
+	unsigned int i;
+	int ret;
+
+	ret = __acpi_node_get_property_reference(fwnode, prop, index,
+						 args_count, &acpi_args);
+	if (ret < 0)
+		return ret;
+	if (!args)
+		return 0;
+
+	args->nargs = acpi_args.nargs;
+	args->fwnode = acpi_fwnode_handle(acpi_args.adev);
+
+	for (i = 0; i < NR_FWNODE_REFERENCE_ARGS; i++)
+		args->args[i] = i < acpi_args.nargs ? acpi_args.args[i] : 0;
+
+	return 0;
+}
+
 static struct fwnode_handle *
-acpi_fwnode_graph_get_next_endpoint(struct fwnode_handle *fwnode,
+acpi_fwnode_graph_get_next_endpoint(const struct fwnode_handle *fwnode,
 				    struct fwnode_handle *prev)
 {
 	struct fwnode_handle *endpoint;
@@ -1201,7 +1241,7 @@ acpi_fwnode_graph_get_next_endpoint(struct fwnode_handle *fwnode,
 }
 
 static struct fwnode_handle *
-acpi_fwnode_graph_get_remote_endpoint(struct fwnode_handle *fwnode)
+acpi_fwnode_graph_get_remote_endpoint(const struct fwnode_handle *fwnode)
 {
 	struct fwnode_handle *endpoint = NULL;
 
@@ -1210,7 +1250,13 @@ acpi_fwnode_graph_get_remote_endpoint(struct fwnode_handle *fwnode)
 	return endpoint;
 }
 
-static int acpi_fwnode_graph_parse_endpoint(struct fwnode_handle *fwnode,
+static struct fwnode_handle *
+acpi_fwnode_get_parent(struct fwnode_handle *fwnode)
+{
+	return acpi_node_get_parent(fwnode);
+}
+
+static int acpi_fwnode_graph_parse_endpoint(const struct fwnode_handle *fwnode,
 					    struct fwnode_endpoint *endpoint)
 {
 	struct fwnode_handle *port_fwnode = fwnode_get_parent(fwnode);
@@ -1223,16 +1269,27 @@ static int acpi_fwnode_graph_parse_endpoint(struct fwnode_handle *fwnode,
 	return 0;
 }
 
-const struct fwnode_operations acpi_fwnode_ops = {
-	.device_is_available = acpi_fwnode_device_is_available,
-	.property_present = acpi_fwnode_property_present,
-	.property_read_int_array = acpi_fwnode_property_read_int_array,
-	.property_read_string_array = acpi_fwnode_property_read_string_array,
-	.get_parent = acpi_node_get_parent,
-	.get_next_child_node = acpi_get_next_subnode,
-	.get_named_child_node = acpi_fwnode_get_named_child_node,
-	.graph_get_next_endpoint = acpi_fwnode_graph_get_next_endpoint,
-	.graph_get_remote_endpoint = acpi_fwnode_graph_get_remote_endpoint,
-	.graph_get_port_parent = acpi_node_get_parent,
-	.graph_parse_endpoint = acpi_fwnode_graph_parse_endpoint,
-};
+#define DECLARE_ACPI_FWNODE_OPS(ops) \
+	const struct fwnode_operations ops = {				\
+		.device_is_available = acpi_fwnode_device_is_available, \
+		.property_present = acpi_fwnode_property_present,	\
+		.property_read_int_array =				\
+			acpi_fwnode_property_read_int_array,		\
+		.property_read_string_array =				\
+			acpi_fwnode_property_read_string_array,		\
+		.get_parent = acpi_node_get_parent,			\
+		.get_next_child_node = acpi_get_next_subnode,		\
+		.get_named_child_node = acpi_fwnode_get_named_child_node, \
+		.get_reference_args = acpi_fwnode_get_reference_args,	\
+		.graph_get_next_endpoint =				\
+			acpi_fwnode_graph_get_next_endpoint,		\
+		.graph_get_remote_endpoint =				\
+			acpi_fwnode_graph_get_remote_endpoint,		\
+		.graph_get_port_parent = acpi_fwnode_get_parent,	\
+		.graph_parse_endpoint = acpi_fwnode_graph_parse_endpoint, \
+	};								\
+	EXPORT_SYMBOL_GPL(ops)
+
+DECLARE_ACPI_FWNODE_OPS(acpi_device_fwnode_ops);
+DECLARE_ACPI_FWNODE_OPS(acpi_data_fwnode_ops);
+const struct fwnode_operations acpi_static_fwnode_ops;
diff --git a/drivers/acpi/resource.c b/drivers/acpi/resource.c
index cd4c4271dc4c..d85e010ee2cc 100644
--- a/drivers/acpi/resource.c
+++ b/drivers/acpi/resource.c
@@ -573,6 +573,35 @@ static acpi_status acpi_dev_process_resource(struct acpi_resource *ares,
 	return AE_OK;
 }
 
+static int __acpi_dev_get_resources(struct acpi_device *adev,
+				    struct list_head *list,
+				    int (*preproc)(struct acpi_resource *, void *),
+				    void *preproc_data, char *method)
+{
+	struct res_proc_context c;
+	acpi_status status;
+
+	if (!adev || !adev->handle || !list_empty(list))
+		return -EINVAL;
+
+	if (!acpi_has_method(adev->handle, method))
+		return 0;
+
+	c.list = list;
+	c.preproc = preproc;
+	c.preproc_data = preproc_data;
+	c.count = 0;
+	c.error = 0;
+	status = acpi_walk_resources(adev->handle, method,
+				     acpi_dev_process_resource, &c);
+	if (ACPI_FAILURE(status)) {
+		acpi_dev_free_resource_list(list);
+		return c.error ? c.error : -EIO;
+	}
+
+	return c.count;
+}
+
 /**
  * acpi_dev_get_resources - Get current resources of a device.
  * @adev: ACPI device node to get the resources for.
@@ -601,30 +630,45 @@ int acpi_dev_get_resources(struct acpi_device *adev, struct list_head *list,
 			   int (*preproc)(struct acpi_resource *, void *),
 			   void *preproc_data)
 {
-	struct res_proc_context c;
-	acpi_status status;
+	return __acpi_dev_get_resources(adev, list, preproc, preproc_data,
+					METHOD_NAME__CRS);
+}
+EXPORT_SYMBOL_GPL(acpi_dev_get_resources);
 
-	if (!adev || !adev->handle || !list_empty(list))
-		return -EINVAL;
+static int is_memory(struct acpi_resource *ares, void *not_used)
+{
+	struct resource_win win;
+	struct resource *res = &win.res;
 
-	if (!acpi_has_method(adev->handle, METHOD_NAME__CRS))
-		return 0;
+	memset(&win, 0, sizeof(win));
 
-	c.list = list;
-	c.preproc = preproc;
-	c.preproc_data = preproc_data;
-	c.count = 0;
-	c.error = 0;
-	status = acpi_walk_resources(adev->handle, METHOD_NAME__CRS,
-				     acpi_dev_process_resource, &c);
-	if (ACPI_FAILURE(status)) {
-		acpi_dev_free_resource_list(list);
-		return c.error ? c.error : -EIO;
-	}
+	return !(acpi_dev_resource_memory(ares, res)
+	       || acpi_dev_resource_address_space(ares, &win)
+	       || acpi_dev_resource_ext_address_space(ares, &win));
+}
 
-	return c.count;
+/**
+ * acpi_dev_get_dma_resources - Get current DMA resources of a device.
+ * @adev: ACPI device node to get the resources for.
+ * @list: Head of the resultant list of resources (must be empty).
+ *
+ * Evaluate the _DMA method for the given device node and process its
+ * output.
+ *
+ * The resultant struct resource objects are put on the list pointed to
+ * by @list, that must be empty initially, as members of struct
+ * resource_entry objects.  Callers of this routine should use
+ * %acpi_dev_free_resource_list() to free that list.
+ *
+ * The number of resources in the output list is returned on success,
+ * an error code reflecting the error condition is returned otherwise.
+ */
+int acpi_dev_get_dma_resources(struct acpi_device *adev, struct list_head *list)
+{
+	return __acpi_dev_get_resources(adev, list, is_memory, NULL,
+					METHOD_NAME__DMA);
 }
-EXPORT_SYMBOL_GPL(acpi_dev_get_resources);
+EXPORT_SYMBOL_GPL(acpi_dev_get_dma_resources);
 
 /**
  * acpi_dev_filter_resource_type - Filter ACPI resource according to resource
diff --git a/drivers/acpi/sbs.c b/drivers/acpi/sbs.c
index ad0b13ad4bbb..a2428e9462dd 100644
--- a/drivers/acpi/sbs.c
+++ b/drivers/acpi/sbs.c
@@ -31,7 +31,7 @@
 #include <linux/jiffies.h>
 #include <linux/delay.h>
 #include <linux/power_supply.h>
-#include <linux/dmi.h>
+#include <linux/platform_data/x86/apple.h>
 
 #include "sbshc.h"
 #include "battery.h"
@@ -58,8 +58,6 @@ static unsigned int cache_time = 1000;
 module_param(cache_time, uint, 0644);
 MODULE_PARM_DESC(cache_time, "cache time in milliseconds");
 
-static bool sbs_manager_broken;
-
 #define MAX_SBS_BAT			4
 #define ACPI_SBS_BLOCK_MAX		32
 
@@ -476,7 +474,7 @@ static ssize_t acpi_battery_alarm_store(struct device *dev,
 	return count;
 }
 
-static struct device_attribute alarm_attr = {
+static const struct device_attribute alarm_attr = {
 	.attr = {.name = "alarm", .mode = 0644},
 	.show = acpi_battery_alarm_show,
 	.store = acpi_battery_alarm_store,
@@ -632,31 +630,12 @@ static void acpi_sbs_callback(void *context)
 	}
 }
 
-static int disable_sbs_manager(const struct dmi_system_id *d)
-{
-	sbs_manager_broken = true;
-	return 0;
-}
-
-static struct dmi_system_id acpi_sbs_dmi_table[] = {
-	{
-		.callback = disable_sbs_manager,
-		.ident = "Apple",
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "Apple Inc.")
-		},
-	},
-	{ },
-};
-
 static int acpi_sbs_add(struct acpi_device *device)
 {
 	struct acpi_sbs *sbs;
 	int result = 0;
 	int id;
 
-	dmi_check_system(acpi_sbs_dmi_table);
-
 	sbs = kzalloc(sizeof(struct acpi_sbs), GFP_KERNEL);
 	if (!sbs) {
 		result = -ENOMEM;
@@ -677,7 +656,7 @@ static int acpi_sbs_add(struct acpi_device *device)
 
 	result = 0;
 
-	if (!sbs_manager_broken) {
+	if (!x86_apple_machine) {
 		result = acpi_manager_get_info(sbs);
 		if (!result) {
 			sbs->manager_present = 1;
diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c
index 33897298f03e..602f8ff212f2 100644
--- a/drivers/acpi/scan.c
+++ b/drivers/acpi/scan.c
@@ -13,6 +13,7 @@
 #include <linux/dmi.h>
 #include <linux/nls.h>
 #include <linux/dma-mapping.h>
+#include <linux/platform_data/x86/apple.h>
 
 #include <asm/pgtable.h>
 
@@ -1360,6 +1361,85 @@ enum dev_dma_attr acpi_get_dma_attr(struct acpi_device *adev)
 }
 
 /**
+ * acpi_dma_get_range() - Get device DMA parameters.
+ *
+ * @dev: device to configure
+ * @dma_addr: pointer device DMA address result
+ * @offset: pointer to the DMA offset result
+ * @size: pointer to DMA range size result
+ *
+ * Evaluate DMA regions and return respectively DMA region start, offset
+ * and size in dma_addr, offset and size on parsing success; it does not
+ * update the passed in values on failure.
+ *
+ * Return 0 on success, < 0 on failure.
+ */
+int acpi_dma_get_range(struct device *dev, u64 *dma_addr, u64 *offset,
+		       u64 *size)
+{
+	struct acpi_device *adev;
+	LIST_HEAD(list);
+	struct resource_entry *rentry;
+	int ret;
+	struct device *dma_dev = dev;
+	u64 len, dma_start = U64_MAX, dma_end = 0, dma_offset = 0;
+
+	/*
+	 * Walk the device tree chasing an ACPI companion with a _DMA
+	 * object while we go. Stop if we find a device with an ACPI
+	 * companion containing a _DMA method.
+	 */
+	do {
+		adev = ACPI_COMPANION(dma_dev);
+		if (adev && acpi_has_method(adev->handle, METHOD_NAME__DMA))
+			break;
+
+		dma_dev = dma_dev->parent;
+	} while (dma_dev);
+
+	if (!dma_dev)
+		return -ENODEV;
+
+	if (!acpi_has_method(adev->handle, METHOD_NAME__CRS)) {
+		acpi_handle_warn(adev->handle, "_DMA is valid only if _CRS is present\n");
+		return -EINVAL;
+	}
+
+	ret = acpi_dev_get_dma_resources(adev, &list);
+	if (ret > 0) {
+		list_for_each_entry(rentry, &list, node) {
+			if (dma_offset && rentry->offset != dma_offset) {
+				ret = -EINVAL;
+				dev_warn(dma_dev, "Can't handle multiple windows with different offsets\n");
+				goto out;
+			}
+			dma_offset = rentry->offset;
+
+			/* Take lower and upper limits */
+			if (rentry->res->start < dma_start)
+				dma_start = rentry->res->start;
+			if (rentry->res->end > dma_end)
+				dma_end = rentry->res->end;
+		}
+
+		if (dma_start >= dma_end) {
+			ret = -EINVAL;
+			dev_dbg(dma_dev, "Invalid DMA regions configuration\n");
+			goto out;
+		}
+
+		*dma_addr = dma_start - dma_offset;
+		len = dma_end - dma_start;
+		*size = max(len, len + 1);
+		*offset = dma_offset;
+	}
+ out:
+	acpi_dev_free_resource_list(&list);
+
+	return ret >= 0 ? 0 : ret;
+}
+
+/**
  * acpi_dma_configure - Set-up DMA configuration for the device.
  * @dev: The pointer to the device
  * @attr: device dma attributes
@@ -1367,20 +1447,16 @@ enum dev_dma_attr acpi_get_dma_attr(struct acpi_device *adev)
 int acpi_dma_configure(struct device *dev, enum dev_dma_attr attr)
 {
 	const struct iommu_ops *iommu;
-	u64 size;
+	u64 dma_addr = 0, size = 0;
 
-	iort_set_dma_mask(dev);
+	iort_dma_setup(dev, &dma_addr, &size);
 
 	iommu = iort_iommu_configure(dev);
 	if (IS_ERR(iommu) && PTR_ERR(iommu) == -EPROBE_DEFER)
 		return -EPROBE_DEFER;
 
-	size = max(dev->coherent_dma_mask, dev->coherent_dma_mask + 1);
-	/*
-	 * Assume dma valid range starts at 0 and covers the whole
-	 * coherent_dma_mask.
-	 */
-	arch_setup_dma_ops(dev, 0, size, iommu, attr == DEV_DMA_COHERENT);
+	arch_setup_dma_ops(dev, dma_addr, size,
+				iommu, attr == DEV_DMA_COHERENT);
 
 	return 0;
 }
@@ -1452,6 +1528,12 @@ static bool acpi_is_spi_i2c_slave(struct acpi_device *device)
 	struct list_head resource_list;
 	bool is_spi_i2c_slave = false;
 
+	/* Macs use device properties in lieu of _CRS resources */
+	if (x86_apple_machine &&
+	    (fwnode_property_present(&device->fwnode, "spiSclkPeriod") ||
+	     fwnode_property_present(&device->fwnode, "i2cAddress")))
+		return true;
+
 	INIT_LIST_HEAD(&resource_list);
 	acpi_dev_get_resources(device, &resource_list, acpi_check_spi_i2c_slave,
 			       &is_spi_i2c_slave);
@@ -1467,8 +1549,7 @@ void acpi_init_device_object(struct acpi_device *device, acpi_handle handle,
 	device->device_type = type;
 	device->handle = handle;
 	device->parent = acpi_bus_get_parent(handle);
-	device->fwnode.type = FWNODE_ACPI;
-	device->fwnode.ops = &acpi_fwnode_ops;
+	device->fwnode.ops = &acpi_device_fwnode_ops;
 	acpi_set_device_status(device, sta);
 	acpi_device_get_busid(device);
 	acpi_set_pnp_ids(handle, &device->pnp, type);
@@ -2058,6 +2139,9 @@ int __init acpi_scan_init(void)
 			acpi_get_spcr_uart_addr();
 	}
 
+	acpi_gpe_apply_masked_gpes();
+	acpi_update_all_gpes();
+
 	mutex_lock(&acpi_scan_lock);
 	/*
 	 * Enumerate devices in the ACPI namespace.
@@ -2082,10 +2166,6 @@ int __init acpi_scan_init(void)
 		}
 	}
 
-	acpi_gpe_apply_masked_gpes();
-	acpi_update_all_gpes();
-	acpi_ec_ecdt_start();
-
 	acpi_scan_initialized = true;
 
  out:
diff --git a/drivers/acpi/sleep.c b/drivers/acpi/sleep.c
index fa8243c5c062..6804ddab3052 100644
--- a/drivers/acpi/sleep.c
+++ b/drivers/acpi/sleep.c
@@ -160,7 +160,7 @@ static int __init init_nvs_nosave(const struct dmi_system_id *d)
 	return 0;
 }
 
-static struct dmi_system_id acpisleep_dmi_table[] __initdata = {
+static const struct dmi_system_id acpisleep_dmi_table[] __initconst = {
 	{
 	.callback = init_old_suspend_ordering,
 	.ident = "Abit KN9 (nForce4 variant)",
@@ -669,6 +669,7 @@ static const struct acpi_device_id lps0_device_ids[] = {
 
 #define ACPI_LPS0_DSM_UUID	"c4eb40a0-6cd2-11e2-bcfd-0800200c9a66"
 
+#define ACPI_LPS0_GET_DEVICE_CONSTRAINTS	1
 #define ACPI_LPS0_SCREEN_OFF	3
 #define ACPI_LPS0_SCREEN_ON	4
 #define ACPI_LPS0_ENTRY		5
@@ -680,6 +681,166 @@ static acpi_handle lps0_device_handle;
 static guid_t lps0_dsm_guid;
 static char lps0_dsm_func_mask;
 
+/* Device constraint entry structure */
+struct lpi_device_info {
+	char *name;
+	int enabled;
+	union acpi_object *package;
+};
+
+/* Constraint package structure */
+struct lpi_device_constraint {
+	int uid;
+	int min_dstate;
+	int function_states;
+};
+
+struct lpi_constraints {
+	acpi_handle handle;
+	int min_dstate;
+};
+
+static struct lpi_constraints *lpi_constraints_table;
+static int lpi_constraints_table_size;
+
+static void lpi_device_get_constraints(void)
+{
+	union acpi_object *out_obj;
+	int i;
+
+	out_obj = acpi_evaluate_dsm_typed(lps0_device_handle, &lps0_dsm_guid,
+					  1, ACPI_LPS0_GET_DEVICE_CONSTRAINTS,
+					  NULL, ACPI_TYPE_PACKAGE);
+
+	acpi_handle_debug(lps0_device_handle, "_DSM function 1 eval %s\n",
+			  out_obj ? "successful" : "failed");
+
+	if (!out_obj)
+		return;
+
+	lpi_constraints_table = kcalloc(out_obj->package.count,
+					sizeof(*lpi_constraints_table),
+					GFP_KERNEL);
+	if (!lpi_constraints_table)
+		goto free_acpi_buffer;
+
+	acpi_handle_debug(lps0_device_handle, "LPI: constraints list begin:\n");
+
+	for (i = 0; i < out_obj->package.count; i++) {
+		struct lpi_constraints *constraint;
+		acpi_status status;
+		union acpi_object *package = &out_obj->package.elements[i];
+		struct lpi_device_info info = { };
+		int package_count = 0, j;
+
+		if (!package)
+			continue;
+
+		for (j = 0; j < package->package.count; ++j) {
+			union acpi_object *element =
+					&(package->package.elements[j]);
+
+			switch (element->type) {
+			case ACPI_TYPE_INTEGER:
+				info.enabled = element->integer.value;
+				break;
+			case ACPI_TYPE_STRING:
+				info.name = element->string.pointer;
+				break;
+			case ACPI_TYPE_PACKAGE:
+				package_count = element->package.count;
+				info.package = element->package.elements;
+				break;
+			}
+		}
+
+		if (!info.enabled || !info.package || !info.name)
+			continue;
+
+		constraint = &lpi_constraints_table[lpi_constraints_table_size];
+
+		status = acpi_get_handle(NULL, info.name, &constraint->handle);
+		if (ACPI_FAILURE(status))
+			continue;
+
+		acpi_handle_debug(lps0_device_handle,
+				  "index:%d Name:%s\n", i, info.name);
+
+		constraint->min_dstate = -1;
+
+		for (j = 0; j < package_count; ++j) {
+			union acpi_object *info_obj = &info.package[j];
+			union acpi_object *cnstr_pkg;
+			union acpi_object *obj;
+			struct lpi_device_constraint dev_info;
+
+			switch (info_obj->type) {
+			case ACPI_TYPE_INTEGER:
+				/* version */
+				break;
+			case ACPI_TYPE_PACKAGE:
+				if (info_obj->package.count < 2)
+					break;
+
+				cnstr_pkg = info_obj->package.elements;
+				obj = &cnstr_pkg[0];
+				dev_info.uid = obj->integer.value;
+				obj = &cnstr_pkg[1];
+				dev_info.min_dstate = obj->integer.value;
+
+				acpi_handle_debug(lps0_device_handle,
+					"uid:%d min_dstate:%s\n",
+					dev_info.uid,
+					acpi_power_state_string(dev_info.min_dstate));
+
+				constraint->min_dstate = dev_info.min_dstate;
+				break;
+			}
+		}
+
+		if (constraint->min_dstate < 0) {
+			acpi_handle_debug(lps0_device_handle,
+					  "Incomplete constraint defined\n");
+			continue;
+		}
+
+		lpi_constraints_table_size++;
+	}
+
+	acpi_handle_debug(lps0_device_handle, "LPI: constraints list end\n");
+
+free_acpi_buffer:
+	ACPI_FREE(out_obj);
+}
+
+static void lpi_check_constraints(void)
+{
+	int i;
+
+	for (i = 0; i < lpi_constraints_table_size; ++i) {
+		struct acpi_device *adev;
+
+		if (acpi_bus_get_device(lpi_constraints_table[i].handle, &adev))
+			continue;
+
+		acpi_handle_debug(adev->handle,
+			"LPI: required min power state:%s current power state:%s\n",
+			acpi_power_state_string(lpi_constraints_table[i].min_dstate),
+			acpi_power_state_string(adev->power.state));
+
+		if (!adev->flags.power_manageable) {
+			acpi_handle_info(adev->handle, "LPI: Device not power manageble\n");
+			continue;
+		}
+
+		if (adev->power.state < lpi_constraints_table[i].min_dstate)
+			acpi_handle_info(adev->handle,
+				"LPI: Constraint not met; min power state:%s current power state:%s\n",
+				acpi_power_state_string(lpi_constraints_table[i].min_dstate),
+				acpi_power_state_string(adev->power.state));
+	}
+}
+
 static void acpi_sleep_run_lps0_dsm(unsigned int func)
 {
 	union acpi_object *out_obj;
@@ -714,6 +875,12 @@ static int lps0_device_attach(struct acpi_device *adev,
 		if ((bitmask & ACPI_S2IDLE_FUNC_MASK) == ACPI_S2IDLE_FUNC_MASK) {
 			lps0_dsm_func_mask = bitmask;
 			lps0_device_handle = adev->handle;
+			/*
+			 * Use suspend-to-idle by default if the default
+			 * suspend mode was not set from the command line.
+			 */
+			if (mem_sleep_default > PM_SUSPEND_MEM)
+				mem_sleep_current = PM_SUSPEND_TO_IDLE;
 		}
 
 		acpi_handle_debug(adev->handle, "_DSM function mask: 0x%x\n",
@@ -723,6 +890,9 @@ static int lps0_device_attach(struct acpi_device *adev,
 				  "_DSM function 0 evaluation failed\n");
 	}
 	ACPI_FREE(out_obj);
+
+	lpi_device_get_constraints();
+
 	return 0;
 }
 
@@ -731,14 +901,14 @@ static struct acpi_scan_handler lps0_handler = {
 	.attach = lps0_device_attach,
 };
 
-static int acpi_freeze_begin(void)
+static int acpi_s2idle_begin(void)
 {
 	acpi_scan_lock_acquire();
 	s2idle_in_progress = true;
 	return 0;
 }
 
-static int acpi_freeze_prepare(void)
+static int acpi_s2idle_prepare(void)
 {
 	if (lps0_device_handle) {
 		acpi_sleep_run_lps0_dsm(ACPI_LPS0_SCREEN_OFF);
@@ -758,8 +928,12 @@ static int acpi_freeze_prepare(void)
 	return 0;
 }
 
-static void acpi_freeze_wake(void)
+static void acpi_s2idle_wake(void)
 {
+
+	if (pm_debug_messages_on)
+		lpi_check_constraints();
+
 	/*
 	 * If IRQD_WAKEUP_ARMED is not set for the SCI at this point, it means
 	 * that the SCI has triggered while suspended, so cancel the wakeup in
@@ -772,7 +946,7 @@ static void acpi_freeze_wake(void)
 	}
 }
 
-static void acpi_freeze_sync(void)
+static void acpi_s2idle_sync(void)
 {
 	/*
 	 * Process all pending events in case there are any wakeup ones.
@@ -785,7 +959,7 @@ static void acpi_freeze_sync(void)
 	s2idle_wakeup = false;
 }
 
-static void acpi_freeze_restore(void)
+static void acpi_s2idle_restore(void)
 {
 	if (acpi_sci_irq_valid())
 		disable_irq_wake(acpi_sci_irq);
@@ -798,19 +972,19 @@ static void acpi_freeze_restore(void)
 	}
 }
 
-static void acpi_freeze_end(void)
+static void acpi_s2idle_end(void)
 {
 	s2idle_in_progress = false;
 	acpi_scan_lock_release();
 }
 
-static const struct platform_freeze_ops acpi_freeze_ops = {
-	.begin = acpi_freeze_begin,
-	.prepare = acpi_freeze_prepare,
-	.wake = acpi_freeze_wake,
-	.sync = acpi_freeze_sync,
-	.restore = acpi_freeze_restore,
-	.end = acpi_freeze_end,
+static const struct platform_s2idle_ops acpi_s2idle_ops = {
+	.begin = acpi_s2idle_begin,
+	.prepare = acpi_s2idle_prepare,
+	.wake = acpi_s2idle_wake,
+	.sync = acpi_s2idle_sync,
+	.restore = acpi_s2idle_restore,
+	.end = acpi_s2idle_end,
 };
 
 static void acpi_sleep_suspend_setup(void)
@@ -825,7 +999,7 @@ static void acpi_sleep_suspend_setup(void)
 		&acpi_suspend_ops_old : &acpi_suspend_ops);
 
 	acpi_scan_add_handler(&lps0_handler);
-	freeze_set_ops(&acpi_freeze_ops);
+	s2idle_set_ops(&acpi_s2idle_ops);
 }
 
 #else /* !CONFIG_SUSPEND */
@@ -870,7 +1044,7 @@ static struct syscore_ops acpi_sleep_syscore_ops = {
 	.resume = acpi_restore_bm_rld,
 };
 
-void acpi_sleep_syscore_init(void)
+static void acpi_sleep_syscore_init(void)
 {
 	register_syscore_ops(&acpi_sleep_syscore_ops);
 }
diff --git a/drivers/acpi/spcr.c b/drivers/acpi/spcr.c
index 98aa8c808a33..324b35bfe781 100644
--- a/drivers/acpi/spcr.c
+++ b/drivers/acpi/spcr.c
@@ -53,17 +53,24 @@ static bool qdf2400_erratum_44_present(struct acpi_table_header *h)
  */
 static bool xgene_8250_erratum_present(struct acpi_table_spcr *tb)
 {
+	bool xgene_8250 = false;
+
 	if (tb->interface_type != ACPI_DBG2_16550_COMPATIBLE)
 		return false;
 
-	if (memcmp(tb->header.oem_id, "APMC0D", ACPI_OEM_ID_SIZE))
+	if (memcmp(tb->header.oem_id, "APMC0D", ACPI_OEM_ID_SIZE) &&
+	    memcmp(tb->header.oem_id, "HPE   ", ACPI_OEM_ID_SIZE))
 		return false;
 
 	if (!memcmp(tb->header.oem_table_id, "XGENESPC",
 	    ACPI_OEM_TABLE_ID_SIZE) && tb->header.oem_revision == 0)
-		return true;
+		xgene_8250 = true;
 
-	return false;
+	if (!memcmp(tb->header.oem_table_id, "ProLiant",
+	    ACPI_OEM_TABLE_ID_SIZE) && tb->header.oem_revision == 1)
+		xgene_8250 = true;
+
+	return xgene_8250;
 }
 
 /**
@@ -105,16 +112,17 @@ int __init parse_spcr(bool earlycon)
 	}
 
 	if (table->serial_port.space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) {
-		switch (table->serial_port.access_width) {
+		switch (ACPI_ACCESS_BIT_WIDTH((
+			table->serial_port.access_width))) {
 		default:
 			pr_err("Unexpected SPCR Access Width.  Defaulting to byte size\n");
-		case ACPI_ACCESS_SIZE_BYTE:
+		case 8:
 			iotype = "mmio";
 			break;
-		case ACPI_ACCESS_SIZE_WORD:
+		case 16:
 			iotype = "mmio16";
 			break;
-		case ACPI_ACCESS_SIZE_DWORD:
+		case 32:
 			iotype = "mmio32";
 			break;
 		}
@@ -181,11 +189,19 @@ int __init parse_spcr(bool earlycon)
 			uart = "qdf2400_e44";
 	}
 
-	if (xgene_8250_erratum_present(table))
+	if (xgene_8250_erratum_present(table)) {
 		iotype = "mmio32";
 
-	snprintf(opts, sizeof(opts), "%s,%s,0x%llx,%d", uart, iotype,
-		 table->serial_port.address, baud_rate);
+		/* for xgene v1 and v2 we don't know the clock rate of the
+		 * UART so don't attempt to change to the baud rate state
+		 * in the table because driver cannot calculate the dividers
+		 */
+		snprintf(opts, sizeof(opts), "%s,%s,0x%llx", uart, iotype,
+			 table->serial_port.address);
+	} else {
+		snprintf(opts, sizeof(opts), "%s,%s,0x%llx,%d", uart, iotype,
+			 table->serial_port.address, baud_rate);
+	}
 
 	pr_info("console: %s\n", opts);
 
diff --git a/drivers/acpi/sysfs.c b/drivers/acpi/sysfs.c
index e414fabf7315..78a5a23010ab 100644
--- a/drivers/acpi/sysfs.c
+++ b/drivers/acpi/sysfs.c
@@ -2,6 +2,8 @@
  * sysfs.c - ACPI sysfs interface to userspace.
  */
 
+#define pr_fmt(fmt) "ACPI: " fmt
+
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/moduleparam.h>
@@ -306,11 +308,13 @@ module_param_call(acpica_version, NULL, param_get_acpica_version, NULL, 0444);
 /*
  * ACPI table sysfs I/F:
  * /sys/firmware/acpi/tables/
+ * /sys/firmware/acpi/tables/data/
  * /sys/firmware/acpi/tables/dynamic/
  */
 
 static LIST_HEAD(acpi_table_attr_list);
 static struct kobject *tables_kobj;
+static struct kobject *tables_data_kobj;
 static struct kobject *dynamic_tables_kobj;
 static struct kobject *hotplug_kobj;
 
@@ -325,6 +329,11 @@ struct acpi_table_attr {
 	struct list_head node;
 };
 
+struct acpi_data_attr {
+	struct bin_attribute attr;
+	u64	addr;
+};
+
 static ssize_t acpi_table_show(struct file *filp, struct kobject *kobj,
 			       struct bin_attribute *bin_attr, char *buf,
 			       loff_t offset, size_t count)
@@ -420,6 +429,70 @@ acpi_status acpi_sysfs_table_handler(u32 event, void *table, void *context)
 	return AE_OK;
 }
 
+static ssize_t acpi_data_show(struct file *filp, struct kobject *kobj,
+			      struct bin_attribute *bin_attr, char *buf,
+			      loff_t offset, size_t count)
+{
+	struct acpi_data_attr *data_attr;
+	void __iomem *base;
+	ssize_t rc;
+
+	data_attr = container_of(bin_attr, struct acpi_data_attr, attr);
+
+	base = acpi_os_map_memory(data_attr->addr, data_attr->attr.size);
+	if (!base)
+		return -ENOMEM;
+	rc = memory_read_from_buffer(buf, count, &offset, base,
+				     data_attr->attr.size);
+	acpi_os_unmap_memory(base, data_attr->attr.size);
+
+	return rc;
+}
+
+static int acpi_bert_data_init(void *th, struct acpi_data_attr *data_attr)
+{
+	struct acpi_table_bert *bert = th;
+
+	if (bert->header.length < sizeof(struct acpi_table_bert) ||
+	    bert->region_length < sizeof(struct acpi_hest_generic_status)) {
+		kfree(data_attr);
+		return -EINVAL;
+	}
+	data_attr->addr = bert->address;
+	data_attr->attr.size = bert->region_length;
+	data_attr->attr.attr.name = "BERT";
+
+	return sysfs_create_bin_file(tables_data_kobj, &data_attr->attr);
+}
+
+static struct acpi_data_obj {
+	char *name;
+	int (*fn)(void *, struct acpi_data_attr *);
+} acpi_data_objs[] = {
+	{ ACPI_SIG_BERT, acpi_bert_data_init },
+};
+
+#define NUM_ACPI_DATA_OBJS ARRAY_SIZE(acpi_data_objs)
+
+static int acpi_table_data_init(struct acpi_table_header *th)
+{
+	struct acpi_data_attr *data_attr;
+	int i;
+
+	for (i = 0; i < NUM_ACPI_DATA_OBJS; i++) {
+		if (ACPI_COMPARE_NAME(th->signature, acpi_data_objs[i].name)) {
+			data_attr = kzalloc(sizeof(*data_attr), GFP_KERNEL);
+			if (!data_attr)
+				return -ENOMEM;
+			sysfs_attr_init(&data_attr->attr.attr);
+			data_attr->attr.read = acpi_data_show;
+			data_attr->attr.attr.mode = 0400;
+			return acpi_data_objs[i].fn(th, data_attr);
+		}
+	}
+	return 0;
+}
+
 static int acpi_tables_sysfs_init(void)
 {
 	struct acpi_table_attr *table_attr;
@@ -432,6 +505,10 @@ static int acpi_tables_sysfs_init(void)
 	if (!tables_kobj)
 		goto err;
 
+	tables_data_kobj = kobject_create_and_add("data", tables_kobj);
+	if (!tables_data_kobj)
+		goto err_tables_data;
+
 	dynamic_tables_kobj = kobject_create_and_add("dynamic", tables_kobj);
 	if (!dynamic_tables_kobj)
 		goto err_dynamic_tables;
@@ -456,13 +533,17 @@ static int acpi_tables_sysfs_init(void)
 			return ret;
 		}
 		list_add_tail(&table_attr->node, &acpi_table_attr_list);
+		acpi_table_data_init(table_header);
 	}
 
 	kobject_uevent(tables_kobj, KOBJ_ADD);
+	kobject_uevent(tables_data_kobj, KOBJ_ADD);
 	kobject_uevent(dynamic_tables_kobj, KOBJ_ADD);
 
 	return 0;
 err_dynamic_tables:
+	kobject_put(tables_data_kobj);
+err_tables_data:
 	kobject_put(tables_kobj);
 err:
 	return -ENOMEM;
@@ -552,11 +633,15 @@ static void fixed_event_count(u32 event_number)
 static void acpi_global_event_handler(u32 event_type, acpi_handle device,
 	u32 event_number, void *context)
 {
-	if (event_type == ACPI_EVENT_TYPE_GPE)
+	if (event_type == ACPI_EVENT_TYPE_GPE) {
 		gpe_count(event_number);
-
-	if (event_type == ACPI_EVENT_TYPE_FIXED)
+		pr_debug("GPE event 0x%02x\n", event_number);
+	} else if (event_type == ACPI_EVENT_TYPE_FIXED) {
 		fixed_event_count(event_number);
+		pr_debug("Fixed event 0x%02x\n", event_number);
+	} else {
+		pr_debug("Other event 0x%02x\n", event_number);
+	}
 }
 
 static int get_status(u32 index, acpi_event_status *status,
diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c
index ff425390bfa8..80ce2a7d224b 100644
--- a/drivers/acpi/tables.c
+++ b/drivers/acpi/tables.c
@@ -740,10 +740,10 @@ int __init acpi_table_init(void)
 
 	if (acpi_verify_table_checksum) {
 		pr_info("Early table checksum verification enabled\n");
-		acpi_gbl_verify_table_checksum = TRUE;
+		acpi_gbl_enable_table_validation = TRUE;
 	} else {
 		pr_info("Early table checksum verification disabled\n");
-		acpi_gbl_verify_table_checksum = FALSE;
+		acpi_gbl_enable_table_validation = FALSE;
 	}
 
 	status = acpi_initialize_tables(initial_tables, ACPI_MAX_TABLES, 0);
diff --git a/drivers/acpi/thermal.c b/drivers/acpi/thermal.c
index 1d0417b87cb7..551b71a24b85 100644
--- a/drivers/acpi/thermal.c
+++ b/drivers/acpi/thermal.c
@@ -1209,7 +1209,7 @@ static int thermal_psv(const struct dmi_system_id *d) {
 	return 0;
 }
 
-static struct dmi_system_id thermal_dmi_table[] __initdata = {
+static const struct dmi_system_id thermal_dmi_table[] __initconst = {
 	/*
 	 * Award BIOS on this AOpen makes thermal control almost worthless.
 	 * http://bugzilla.kernel.org/show_bug.cgi?id=8842
diff --git a/drivers/acpi/utils.c b/drivers/acpi/utils.c
index b9d956c916f5..0a9e5979aaa9 100644
--- a/drivers/acpi/utils.c
+++ b/drivers/acpi/utils.c
@@ -816,3 +816,39 @@ static int __init acpi_backlight(char *str)
 	return 1;
 }
 __setup("acpi_backlight=", acpi_backlight);
+
+/**
+ * acpi_match_platform_list - Check if the system matches with a given list
+ * @plat: pointer to acpi_platform_list table terminated by a NULL entry
+ *
+ * Return the matched index if the system is found in the platform list.
+ * Otherwise, return a negative error code.
+ */
+int acpi_match_platform_list(const struct acpi_platform_list *plat)
+{
+	struct acpi_table_header hdr;
+	int idx = 0;
+
+	if (acpi_disabled)
+		return -ENODEV;
+
+	for (; plat->oem_id[0]; plat++, idx++) {
+		if (ACPI_FAILURE(acpi_get_table_header(plat->table, 0, &hdr)))
+			continue;
+
+		if (strncmp(plat->oem_id, hdr.oem_id, ACPI_OEM_ID_SIZE))
+			continue;
+
+		if (strncmp(plat->oem_table_id, hdr.oem_table_id, ACPI_OEM_TABLE_ID_SIZE))
+			continue;
+
+		if ((plat->pred == all_versions) ||
+		    (plat->pred == less_than_or_equal && hdr.oem_revision <= plat->oem_revision) ||
+		    (plat->pred == greater_than_or_equal && hdr.oem_revision >= plat->oem_revision) ||
+		    (plat->pred == equal && hdr.oem_revision == plat->oem_revision))
+			return idx;
+	}
+
+	return -ENODEV;
+}
+EXPORT_SYMBOL(acpi_match_platform_list);
diff --git a/drivers/acpi/video_detect.c b/drivers/acpi/video_detect.c
index d179e8d9177d..601e5d372887 100644
--- a/drivers/acpi/video_detect.c
+++ b/drivers/acpi/video_detect.c
@@ -103,6 +103,12 @@ static int video_detect_force_native(const struct dmi_system_id *d)
 	return 0;
 }
 
+static int video_detect_force_none(const struct dmi_system_id *d)
+{
+	acpi_backlight_dmi = acpi_backlight_none;
+	return 0;
+}
+
 static const struct dmi_system_id video_detect_dmi_table[] = {
 	/* On Samsung X360, the BIOS will set a flag (VDRV) if generic
 	 * ACPI backlight device is used. This flag will definitively break
@@ -313,6 +319,14 @@ static const struct dmi_system_id video_detect_dmi_table[] = {
 		DMI_MATCH(DMI_PRODUCT_NAME, "Precision 7510"),
 		},
 	},
+	{
+	 .callback = video_detect_force_none,
+	 .ident = "Dell OptiPlex 9020M",
+	 .matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+		DMI_MATCH(DMI_PRODUCT_NAME, "OptiPlex 9020M"),
+		},
+	},
 	{ },
 };
 
diff --git a/drivers/acpi/x86/apple.c b/drivers/acpi/x86/apple.c
new file mode 100644
index 000000000000..51b4cf9f25da
--- /dev/null
+++ b/drivers/acpi/x86/apple.c
@@ -0,0 +1,141 @@
+/*
+ * apple.c - Apple ACPI quirks
+ * Copyright (C) 2017 Lukas Wunner <lukas@wunner.de>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License (version 2) as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/acpi.h>
+#include <linux/bitmap.h>
+#include <linux/platform_data/x86/apple.h>
+#include <linux/uuid.h>
+
+/* Apple _DSM device properties GUID */
+static const guid_t apple_prp_guid =
+	GUID_INIT(0xa0b5b7c6, 0x1318, 0x441c,
+		  0xb0, 0xc9, 0xfe, 0x69, 0x5e, 0xaf, 0x94, 0x9b);
+
+/**
+ * acpi_extract_apple_properties - retrieve and convert Apple _DSM properties
+ * @adev: ACPI device for which to retrieve the properties
+ *
+ * Invoke Apple's custom _DSM once to check the protocol version and once more
+ * to retrieve the properties.  They are marshalled up in a single package as
+ * alternating key/value elements, unlike _DSD which stores them as a package
+ * of 2-element packages.  Convert to _DSD format and make them available under
+ * the primary fwnode.
+ */
+void acpi_extract_apple_properties(struct acpi_device *adev)
+{
+	unsigned int i, j = 0, newsize = 0, numprops, numvalid;
+	union acpi_object *props, *newprops;
+	unsigned long *valid = NULL;
+	void *free_space;
+
+	if (!x86_apple_machine)
+		return;
+
+	props = acpi_evaluate_dsm_typed(adev->handle, &apple_prp_guid, 1, 0,
+					NULL, ACPI_TYPE_BUFFER);
+	if (!props)
+		return;
+
+	if (!props->buffer.length)
+		goto out_free;
+
+	if (props->buffer.pointer[0] != 3) {
+		acpi_handle_info(adev->handle, FW_INFO
+				 "unsupported properties version %*ph\n",
+				 props->buffer.length, props->buffer.pointer);
+		goto out_free;
+	}
+
+	ACPI_FREE(props);
+	props = acpi_evaluate_dsm_typed(adev->handle, &apple_prp_guid, 1, 1,
+					NULL, ACPI_TYPE_PACKAGE);
+	if (!props)
+		return;
+
+	numprops = props->package.count / 2;
+	if (!numprops)
+		goto out_free;
+
+	valid = kcalloc(BITS_TO_LONGS(numprops), sizeof(long), GFP_KERNEL);
+	if (!valid)
+		goto out_free;
+
+	/* newsize = key length + value length of each tuple */
+	for (i = 0; i < numprops; i++) {
+		union acpi_object *key = &props->package.elements[i * 2];
+		union acpi_object *val = &props->package.elements[i * 2 + 1];
+
+		if ( key->type != ACPI_TYPE_STRING ||
+		    (val->type != ACPI_TYPE_INTEGER &&
+		     val->type != ACPI_TYPE_BUFFER))
+			continue; /* skip invalid properties */
+
+		__set_bit(i, valid);
+		newsize += key->string.length + 1;
+		if ( val->type == ACPI_TYPE_BUFFER)
+			newsize += val->buffer.length;
+	}
+
+	numvalid = bitmap_weight(valid, numprops);
+	if (numprops > numvalid)
+		acpi_handle_info(adev->handle, FW_INFO
+				 "skipped %u properties: wrong type\n",
+				 numprops - numvalid);
+	if (numvalid == 0)
+		goto out_free;
+
+	/* newsize += top-level package + 3 objects for each key/value tuple */
+	newsize	+= (1 + 3 * numvalid) * sizeof(union acpi_object);
+	newprops = ACPI_ALLOCATE_ZEROED(newsize);
+	if (!newprops)
+		goto out_free;
+
+	/* layout: top-level package | packages | key/value tuples | strings */
+	newprops->type = ACPI_TYPE_PACKAGE;
+	newprops->package.count = numvalid;
+	newprops->package.elements = &newprops[1];
+	free_space = &newprops[1 + 3 * numvalid];
+
+	for_each_set_bit(i, valid, numprops) {
+		union acpi_object *key = &props->package.elements[i * 2];
+		union acpi_object *val = &props->package.elements[i * 2 + 1];
+		unsigned int k = 1 + numvalid + j * 2; /* index into newprops */
+		unsigned int v = k + 1;
+
+		newprops[1 + j].type = ACPI_TYPE_PACKAGE;
+		newprops[1 + j].package.count = 2;
+		newprops[1 + j].package.elements = &newprops[k];
+
+		newprops[k].type = ACPI_TYPE_STRING;
+		newprops[k].string.length = key->string.length;
+		newprops[k].string.pointer = free_space;
+		memcpy(free_space, key->string.pointer, key->string.length);
+		free_space += key->string.length + 1;
+
+		newprops[v].type = val->type;
+		if (val->type == ACPI_TYPE_INTEGER) {
+			newprops[v].integer.value = val->integer.value;
+		} else {
+			newprops[v].buffer.length = val->buffer.length;
+			newprops[v].buffer.pointer = free_space;
+			memcpy(free_space, val->buffer.pointer,
+			       val->buffer.length);
+			free_space += val->buffer.length;
+		}
+		j++; /* count valid properties */
+	}
+	WARN_ON(free_space != (void *)newprops + newsize);
+
+	adev->data.properties = newprops;
+	adev->data.pointer = newprops;
+
+out_free:
+	ACPI_FREE(props);
+	kfree(valid);
+}
diff --git a/drivers/android/Kconfig b/drivers/android/Kconfig
index 832e885349b1..9801d852bd56 100644
--- a/drivers/android/Kconfig
+++ b/drivers/android/Kconfig
@@ -22,7 +22,7 @@ config ANDROID_BINDER_IPC
 config ANDROID_BINDER_DEVICES
 	string "Android Binder devices"
 	depends on ANDROID_BINDER_IPC
-	default "binder,hwbinder"
+	default "binder,hwbinder,vndbinder"
 	---help---
 	  Default value for the binder.devices parameter.
 
@@ -32,7 +32,7 @@ config ANDROID_BINDER_DEVICES
 	  therefore logically separated from the other devices.
 
 config ANDROID_BINDER_IPC_32BIT
-	bool
+	bool "Use old (Android 4.4 and earlier) 32-bit binder API"
 	depends on !64BIT && ANDROID_BINDER_IPC
 	default y
 	---help---
@@ -44,6 +44,16 @@ config ANDROID_BINDER_IPC_32BIT
 
 	  Note that enabling this will break newer Android user-space.
 
+config ANDROID_BINDER_IPC_SELFTEST
+	bool "Android Binder IPC Driver Selftest"
+	depends on ANDROID_BINDER_IPC
+	---help---
+	  This feature allows binder selftest to run.
+
+	  Binder selftest checks the allocation and free of binder buffers
+	  exhaustively with combinations of various buffer sizes and
+	  alignments.
+
 endif # if ANDROID
 
 endmenu
diff --git a/drivers/android/Makefile b/drivers/android/Makefile
index 3b7e4b072c58..a01254c43ee3 100644
--- a/drivers/android/Makefile
+++ b/drivers/android/Makefile
@@ -1,3 +1,4 @@
 ccflags-y += -I$(src)			# needed for trace events
 
-obj-$(CONFIG_ANDROID_BINDER_IPC)	+= binder.o
+obj-$(CONFIG_ANDROID_BINDER_IPC)	+= binder.o binder_alloc.o
+obj-$(CONFIG_ANDROID_BINDER_IPC_SELFTEST) += binder_alloc_selftest.o
diff --git a/drivers/android/binder.c b/drivers/android/binder.c
index f7665c31feca..d055b3f2a207 100644
--- a/drivers/android/binder.c
+++ b/drivers/android/binder.c
@@ -15,6 +15,40 @@
  *
  */
 
+/*
+ * Locking overview
+ *
+ * There are 3 main spinlocks which must be acquired in the
+ * order shown:
+ *
+ * 1) proc->outer_lock : protects binder_ref
+ *    binder_proc_lock() and binder_proc_unlock() are
+ *    used to acq/rel.
+ * 2) node->lock : protects most fields of binder_node.
+ *    binder_node_lock() and binder_node_unlock() are
+ *    used to acq/rel
+ * 3) proc->inner_lock : protects the thread and node lists
+ *    (proc->threads, proc->waiting_threads, proc->nodes)
+ *    and all todo lists associated with the binder_proc
+ *    (proc->todo, thread->todo, proc->delivered_death and
+ *    node->async_todo), as well as thread->transaction_stack
+ *    binder_inner_proc_lock() and binder_inner_proc_unlock()
+ *    are used to acq/rel
+ *
+ * Any lock under procA must never be nested under any lock at the same
+ * level or below on procB.
+ *
+ * Functions that require a lock held on entry indicate which lock
+ * in the suffix of the function name:
+ *
+ * foo_olocked() : requires node->outer_lock
+ * foo_nlocked() : requires node->lock
+ * foo_ilocked() : requires proc->inner_lock
+ * foo_oilocked(): requires proc->outer_lock and proc->inner_lock
+ * foo_nilocked(): requires node->lock and proc->inner_lock
+ * ...
+ */
+
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include <asm/cacheflush.h>
@@ -24,7 +58,6 @@
 #include <linux/fs.h>
 #include <linux/list.h>
 #include <linux/miscdevice.h>
-#include <linux/mm.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/nsproxy.h>
@@ -35,30 +68,31 @@
 #include <linux/sched/mm.h>
 #include <linux/seq_file.h>
 #include <linux/uaccess.h>
-#include <linux/vmalloc.h>
-#include <linux/slab.h>
 #include <linux/pid_namespace.h>
 #include <linux/security.h>
+#include <linux/spinlock.h>
 
 #ifdef CONFIG_ANDROID_BINDER_IPC_32BIT
 #define BINDER_IPC_32BIT 1
 #endif
 
 #include <uapi/linux/android/binder.h>
+#include "binder_alloc.h"
 #include "binder_trace.h"
 
-static DEFINE_MUTEX(binder_main_lock);
+static HLIST_HEAD(binder_deferred_list);
 static DEFINE_MUTEX(binder_deferred_lock);
-static DEFINE_MUTEX(binder_mmap_lock);
 
 static HLIST_HEAD(binder_devices);
 static HLIST_HEAD(binder_procs);
-static HLIST_HEAD(binder_deferred_list);
+static DEFINE_MUTEX(binder_procs_lock);
+
 static HLIST_HEAD(binder_dead_nodes);
+static DEFINE_SPINLOCK(binder_dead_nodes_lock);
 
 static struct dentry *binder_debugfs_dir_entry_root;
 static struct dentry *binder_debugfs_dir_entry_proc;
-static int binder_last_id;
+static atomic_t binder_last_id;
 
 #define BINDER_DEBUG_ENTRY(name) \
 static int binder_##name##_open(struct inode *inode, struct file *file) \
@@ -88,8 +122,6 @@ BINDER_DEBUG_ENTRY(proc);
 
 #define FORBIDDEN_MMAP_FLAGS                (VM_WRITE)
 
-#define BINDER_SMALL_BUF_SIZE (PAGE_SIZE * 64)
-
 enum {
 	BINDER_DEBUG_USER_ERROR             = 1U << 0,
 	BINDER_DEBUG_FAILED_TRANSACTION     = 1U << 1,
@@ -104,17 +136,13 @@ enum {
 	BINDER_DEBUG_TRANSACTION_COMPLETE   = 1U << 10,
 	BINDER_DEBUG_FREE_BUFFER            = 1U << 11,
 	BINDER_DEBUG_INTERNAL_REFS          = 1U << 12,
-	BINDER_DEBUG_BUFFER_ALLOC           = 1U << 13,
-	BINDER_DEBUG_PRIORITY_CAP           = 1U << 14,
-	BINDER_DEBUG_BUFFER_ALLOC_ASYNC     = 1U << 15,
+	BINDER_DEBUG_PRIORITY_CAP           = 1U << 13,
+	BINDER_DEBUG_SPINLOCKS              = 1U << 14,
 };
 static uint32_t binder_debug_mask = BINDER_DEBUG_USER_ERROR |
 	BINDER_DEBUG_FAILED_TRANSACTION | BINDER_DEBUG_DEAD_TRANSACTION;
 module_param_named(debug_mask, binder_debug_mask, uint, S_IWUSR | S_IRUGO);
 
-static bool binder_debug_no_lock;
-module_param_named(proc_no_lock, binder_debug_no_lock, bool, S_IWUSR | S_IRUGO);
-
 static char *binder_devices_param = CONFIG_ANDROID_BINDER_DEVICES;
 module_param_named(devices, binder_devices_param, charp, 0444);
 
@@ -171,26 +199,27 @@ enum binder_stat_types {
 };
 
 struct binder_stats {
-	int br[_IOC_NR(BR_FAILED_REPLY) + 1];
-	int bc[_IOC_NR(BC_REPLY_SG) + 1];
-	int obj_created[BINDER_STAT_COUNT];
-	int obj_deleted[BINDER_STAT_COUNT];
+	atomic_t br[_IOC_NR(BR_FAILED_REPLY) + 1];
+	atomic_t bc[_IOC_NR(BC_REPLY_SG) + 1];
+	atomic_t obj_created[BINDER_STAT_COUNT];
+	atomic_t obj_deleted[BINDER_STAT_COUNT];
 };
 
 static struct binder_stats binder_stats;
 
 static inline void binder_stats_deleted(enum binder_stat_types type)
 {
-	binder_stats.obj_deleted[type]++;
+	atomic_inc(&binder_stats.obj_deleted[type]);
 }
 
 static inline void binder_stats_created(enum binder_stat_types type)
 {
-	binder_stats.obj_created[type]++;
+	atomic_inc(&binder_stats.obj_created[type]);
 }
 
 struct binder_transaction_log_entry {
 	int debug_id;
+	int debug_id_done;
 	int call_type;
 	int from_proc;
 	int from_thread;
@@ -200,11 +229,14 @@ struct binder_transaction_log_entry {
 	int to_node;
 	int data_size;
 	int offsets_size;
+	int return_error_line;
+	uint32_t return_error;
+	uint32_t return_error_param;
 	const char *context_name;
 };
 struct binder_transaction_log {
-	int next;
-	int full;
+	atomic_t cur;
+	bool full;
 	struct binder_transaction_log_entry entry[32];
 };
 static struct binder_transaction_log binder_transaction_log;
@@ -214,19 +246,26 @@ static struct binder_transaction_log_entry *binder_transaction_log_add(
 	struct binder_transaction_log *log)
 {
 	struct binder_transaction_log_entry *e;
+	unsigned int cur = atomic_inc_return(&log->cur);
 
-	e = &log->entry[log->next];
-	memset(e, 0, sizeof(*e));
-	log->next++;
-	if (log->next == ARRAY_SIZE(log->entry)) {
-		log->next = 0;
+	if (cur >= ARRAY_SIZE(log->entry))
 		log->full = 1;
-	}
+	e = &log->entry[cur % ARRAY_SIZE(log->entry)];
+	WRITE_ONCE(e->debug_id_done, 0);
+	/*
+	 * write-barrier to synchronize access to e->debug_id_done.
+	 * We make sure the initialized 0 value is seen before
+	 * memset() other fields are zeroed by memset.
+	 */
+	smp_wmb();
+	memset(e, 0, sizeof(*e));
 	return e;
 }
 
 struct binder_context {
 	struct binder_node *binder_context_mgr_node;
+	struct mutex context_mgr_node_lock;
+
 	kuid_t binder_context_mgr_uid;
 	const char *name;
 };
@@ -237,11 +276,20 @@ struct binder_device {
 	struct binder_context context;
 };
 
+/**
+ * struct binder_work - work enqueued on a worklist
+ * @entry:             node enqueued on list
+ * @type:              type of work to be performed
+ *
+ * There are separate work lists for proc, thread, and node (async).
+ */
 struct binder_work {
 	struct list_head entry;
+
 	enum {
 		BINDER_WORK_TRANSACTION = 1,
 		BINDER_WORK_TRANSACTION_COMPLETE,
+		BINDER_WORK_RETURN_ERROR,
 		BINDER_WORK_NODE,
 		BINDER_WORK_DEAD_BINDER,
 		BINDER_WORK_DEAD_BINDER_AND_CLEAR,
@@ -249,8 +297,72 @@ struct binder_work {
 	} type;
 };
 
+struct binder_error {
+	struct binder_work work;
+	uint32_t cmd;
+};
+
+/**
+ * struct binder_node - binder node bookkeeping
+ * @debug_id:             unique ID for debugging
+ *                        (invariant after initialized)
+ * @lock:                 lock for node fields
+ * @work:                 worklist element for node work
+ *                        (protected by @proc->inner_lock)
+ * @rb_node:              element for proc->nodes tree
+ *                        (protected by @proc->inner_lock)
+ * @dead_node:            element for binder_dead_nodes list
+ *                        (protected by binder_dead_nodes_lock)
+ * @proc:                 binder_proc that owns this node
+ *                        (invariant after initialized)
+ * @refs:                 list of references on this node
+ *                        (protected by @lock)
+ * @internal_strong_refs: used to take strong references when
+ *                        initiating a transaction
+ *                        (protected by @proc->inner_lock if @proc
+ *                        and by @lock)
+ * @local_weak_refs:      weak user refs from local process
+ *                        (protected by @proc->inner_lock if @proc
+ *                        and by @lock)
+ * @local_strong_refs:    strong user refs from local process
+ *                        (protected by @proc->inner_lock if @proc
+ *                        and by @lock)
+ * @tmp_refs:             temporary kernel refs
+ *                        (protected by @proc->inner_lock while @proc
+ *                        is valid, and by binder_dead_nodes_lock
+ *                        if @proc is NULL. During inc/dec and node release
+ *                        it is also protected by @lock to provide safety
+ *                        as the node dies and @proc becomes NULL)
+ * @ptr:                  userspace pointer for node
+ *                        (invariant, no lock needed)
+ * @cookie:               userspace cookie for node
+ *                        (invariant, no lock needed)
+ * @has_strong_ref:       userspace notified of strong ref
+ *                        (protected by @proc->inner_lock if @proc
+ *                        and by @lock)
+ * @pending_strong_ref:   userspace has acked notification of strong ref
+ *                        (protected by @proc->inner_lock if @proc
+ *                        and by @lock)
+ * @has_weak_ref:         userspace notified of weak ref
+ *                        (protected by @proc->inner_lock if @proc
+ *                        and by @lock)
+ * @pending_weak_ref:     userspace has acked notification of weak ref
+ *                        (protected by @proc->inner_lock if @proc
+ *                        and by @lock)
+ * @has_async_transaction: async transaction to node in progress
+ *                        (protected by @lock)
+ * @accept_fds:           file descriptor operations supported for node
+ *                        (invariant after initialized)
+ * @min_priority:         minimum scheduling priority
+ *                        (invariant after initialized)
+ * @async_todo:           list of async work items
+ *                        (protected by @proc->inner_lock)
+ *
+ * Bookkeeping structure for binder nodes.
+ */
 struct binder_node {
 	int debug_id;
+	spinlock_t lock;
 	struct binder_work work;
 	union {
 		struct rb_node rb_node;
@@ -261,88 +373,167 @@ struct binder_node {
 	int internal_strong_refs;
 	int local_weak_refs;
 	int local_strong_refs;
+	int tmp_refs;
 	binder_uintptr_t ptr;
 	binder_uintptr_t cookie;
-	unsigned has_strong_ref:1;
-	unsigned pending_strong_ref:1;
-	unsigned has_weak_ref:1;
-	unsigned pending_weak_ref:1;
-	unsigned has_async_transaction:1;
-	unsigned accept_fds:1;
-	unsigned min_priority:8;
+	struct {
+		/*
+		 * bitfield elements protected by
+		 * proc inner_lock
+		 */
+		u8 has_strong_ref:1;
+		u8 pending_strong_ref:1;
+		u8 has_weak_ref:1;
+		u8 pending_weak_ref:1;
+	};
+	struct {
+		/*
+		 * invariant after initialization
+		 */
+		u8 accept_fds:1;
+		u8 min_priority;
+	};
+	bool has_async_transaction;
 	struct list_head async_todo;
 };
 
 struct binder_ref_death {
+	/**
+	 * @work: worklist element for death notifications
+	 *        (protected by inner_lock of the proc that
+	 *        this ref belongs to)
+	 */
 	struct binder_work work;
 	binder_uintptr_t cookie;
 };
 
+/**
+ * struct binder_ref_data - binder_ref counts and id
+ * @debug_id:        unique ID for the ref
+ * @desc:            unique userspace handle for ref
+ * @strong:          strong ref count (debugging only if not locked)
+ * @weak:            weak ref count (debugging only if not locked)
+ *
+ * Structure to hold ref count and ref id information. Since
+ * the actual ref can only be accessed with a lock, this structure
+ * is used to return information about the ref to callers of
+ * ref inc/dec functions.
+ */
+struct binder_ref_data {
+	int debug_id;
+	uint32_t desc;
+	int strong;
+	int weak;
+};
+
+/**
+ * struct binder_ref - struct to track references on nodes
+ * @data:        binder_ref_data containing id, handle, and current refcounts
+ * @rb_node_desc: node for lookup by @data.desc in proc's rb_tree
+ * @rb_node_node: node for lookup by @node in proc's rb_tree
+ * @node_entry:  list entry for node->refs list in target node
+ *               (protected by @node->lock)
+ * @proc:        binder_proc containing ref
+ * @node:        binder_node of target node. When cleaning up a
+ *               ref for deletion in binder_cleanup_ref, a non-NULL
+ *               @node indicates the node must be freed
+ * @death:       pointer to death notification (ref_death) if requested
+ *               (protected by @node->lock)
+ *
+ * Structure to track references from procA to target node (on procB). This
+ * structure is unsafe to access without holding @proc->outer_lock.
+ */
 struct binder_ref {
 	/* Lookups needed: */
 	/*   node + proc => ref (transaction) */
 	/*   desc + proc => ref (transaction, inc/dec ref) */
 	/*   node => refs + procs (proc exit) */
-	int debug_id;
+	struct binder_ref_data data;
 	struct rb_node rb_node_desc;
 	struct rb_node rb_node_node;
 	struct hlist_node node_entry;
 	struct binder_proc *proc;
 	struct binder_node *node;
-	uint32_t desc;
-	int strong;
-	int weak;
 	struct binder_ref_death *death;
 };
 
-struct binder_buffer {
-	struct list_head entry; /* free and allocated entries by address */
-	struct rb_node rb_node; /* free entry by size or allocated entry */
-				/* by address */
-	unsigned free:1;
-	unsigned allow_user_free:1;
-	unsigned async_transaction:1;
-	unsigned debug_id:29;
-
-	struct binder_transaction *transaction;
-
-	struct binder_node *target_node;
-	size_t data_size;
-	size_t offsets_size;
-	size_t extra_buffers_size;
-	uint8_t data[0];
-};
-
 enum binder_deferred_state {
 	BINDER_DEFERRED_PUT_FILES    = 0x01,
 	BINDER_DEFERRED_FLUSH        = 0x02,
 	BINDER_DEFERRED_RELEASE      = 0x04,
 };
 
+/**
+ * struct binder_proc - binder process bookkeeping
+ * @proc_node:            element for binder_procs list
+ * @threads:              rbtree of binder_threads in this proc
+ *                        (protected by @inner_lock)
+ * @nodes:                rbtree of binder nodes associated with
+ *                        this proc ordered by node->ptr
+ *                        (protected by @inner_lock)
+ * @refs_by_desc:         rbtree of refs ordered by ref->desc
+ *                        (protected by @outer_lock)
+ * @refs_by_node:         rbtree of refs ordered by ref->node
+ *                        (protected by @outer_lock)
+ * @waiting_threads:      threads currently waiting for proc work
+ *                        (protected by @inner_lock)
+ * @pid                   PID of group_leader of process
+ *                        (invariant after initialized)
+ * @tsk                   task_struct for group_leader of process
+ *                        (invariant after initialized)
+ * @files                 files_struct for process
+ *                        (invariant after initialized)
+ * @deferred_work_node:   element for binder_deferred_list
+ *                        (protected by binder_deferred_lock)
+ * @deferred_work:        bitmap of deferred work to perform
+ *                        (protected by binder_deferred_lock)
+ * @is_dead:              process is dead and awaiting free
+ *                        when outstanding transactions are cleaned up
+ *                        (protected by @inner_lock)
+ * @todo:                 list of work for this process
+ *                        (protected by @inner_lock)
+ * @wait:                 wait queue head to wait for proc work
+ *                        (invariant after initialized)
+ * @stats:                per-process binder statistics
+ *                        (atomics, no lock needed)
+ * @delivered_death:      list of delivered death notification
+ *                        (protected by @inner_lock)
+ * @max_threads:          cap on number of binder threads
+ *                        (protected by @inner_lock)
+ * @requested_threads:    number of binder threads requested but not
+ *                        yet started. In current implementation, can
+ *                        only be 0 or 1.
+ *                        (protected by @inner_lock)
+ * @requested_threads_started: number binder threads started
+ *                        (protected by @inner_lock)
+ * @tmp_ref:              temporary reference to indicate proc is in use
+ *                        (protected by @inner_lock)
+ * @default_priority:     default scheduler priority
+ *                        (invariant after initialized)
+ * @debugfs_entry:        debugfs node
+ * @alloc:                binder allocator bookkeeping
+ * @context:              binder_context for this proc
+ *                        (invariant after initialized)
+ * @inner_lock:           can nest under outer_lock and/or node lock
+ * @outer_lock:           no nesting under innor or node lock
+ *                        Lock order: 1) outer, 2) node, 3) inner
+ *
+ * Bookkeeping structure for binder processes
+ */
 struct binder_proc {
 	struct hlist_node proc_node;
 	struct rb_root threads;
 	struct rb_root nodes;
 	struct rb_root refs_by_desc;
 	struct rb_root refs_by_node;
+	struct list_head waiting_threads;
 	int pid;
-	struct vm_area_struct *vma;
-	struct mm_struct *vma_vm_mm;
 	struct task_struct *tsk;
 	struct files_struct *files;
 	struct hlist_node deferred_work_node;
 	int deferred_work;
-	void *buffer;
-	ptrdiff_t user_buffer_offset;
-
-	struct list_head buffers;
-	struct rb_root free_buffers;
-	struct rb_root allocated_buffers;
-	size_t free_async_space;
+	bool is_dead;
 
-	struct page **pages;
-	size_t buffer_size;
-	uint32_t buffer_free;
 	struct list_head todo;
 	wait_queue_head_t wait;
 	struct binder_stats stats;
@@ -350,10 +541,13 @@ struct binder_proc {
 	int max_threads;
 	int requested_threads;
 	int requested_threads_started;
-	int ready_threads;
+	int tmp_ref;
 	long default_priority;
 	struct dentry *debugfs_entry;
+	struct binder_alloc alloc;
 	struct binder_context *context;
+	spinlock_t inner_lock;
+	spinlock_t outer_lock;
 };
 
 enum {
@@ -362,22 +556,58 @@ enum {
 	BINDER_LOOPER_STATE_EXITED      = 0x04,
 	BINDER_LOOPER_STATE_INVALID     = 0x08,
 	BINDER_LOOPER_STATE_WAITING     = 0x10,
-	BINDER_LOOPER_STATE_NEED_RETURN = 0x20
+	BINDER_LOOPER_STATE_POLL        = 0x20,
 };
 
+/**
+ * struct binder_thread - binder thread bookkeeping
+ * @proc:                 binder process for this thread
+ *                        (invariant after initialization)
+ * @rb_node:              element for proc->threads rbtree
+ *                        (protected by @proc->inner_lock)
+ * @waiting_thread_node:  element for @proc->waiting_threads list
+ *                        (protected by @proc->inner_lock)
+ * @pid:                  PID for this thread
+ *                        (invariant after initialization)
+ * @looper:               bitmap of looping state
+ *                        (only accessed by this thread)
+ * @looper_needs_return:  looping thread needs to exit driver
+ *                        (no lock needed)
+ * @transaction_stack:    stack of in-progress transactions for this thread
+ *                        (protected by @proc->inner_lock)
+ * @todo:                 list of work to do for this thread
+ *                        (protected by @proc->inner_lock)
+ * @return_error:         transaction errors reported by this thread
+ *                        (only accessed by this thread)
+ * @reply_error:          transaction errors reported by target thread
+ *                        (protected by @proc->inner_lock)
+ * @wait:                 wait queue for thread work
+ * @stats:                per-thread statistics
+ *                        (atomics, no lock needed)
+ * @tmp_ref:              temporary reference to indicate thread is in use
+ *                        (atomic since @proc->inner_lock cannot
+ *                        always be acquired)
+ * @is_dead:              thread is dead and awaiting free
+ *                        when outstanding transactions are cleaned up
+ *                        (protected by @proc->inner_lock)
+ *
+ * Bookkeeping structure for binder threads.
+ */
 struct binder_thread {
 	struct binder_proc *proc;
 	struct rb_node rb_node;
+	struct list_head waiting_thread_node;
 	int pid;
-	int looper;
+	int looper;              /* only modified by this thread */
+	bool looper_need_return; /* can be written by other thread */
 	struct binder_transaction *transaction_stack;
 	struct list_head todo;
-	uint32_t return_error; /* Write failed, return error code in read buf */
-	uint32_t return_error2; /* Write failed, return error code in read */
-		/* buffer. Used when sending a reply to a dead process that */
-		/* we are also waiting on */
+	struct binder_error return_error;
+	struct binder_error reply_error;
 	wait_queue_head_t wait;
 	struct binder_stats stats;
+	atomic_t tmp_ref;
+	bool is_dead;
 };
 
 struct binder_transaction {
@@ -397,10 +627,253 @@ struct binder_transaction {
 	long	priority;
 	long	saved_priority;
 	kuid_t	sender_euid;
+	/**
+	 * @lock:  protects @from, @to_proc, and @to_thread
+	 *
+	 * @from, @to_proc, and @to_thread can be set to NULL
+	 * during thread teardown
+	 */
+	spinlock_t lock;
 };
 
+/**
+ * binder_proc_lock() - Acquire outer lock for given binder_proc
+ * @proc:         struct binder_proc to acquire
+ *
+ * Acquires proc->outer_lock. Used to protect binder_ref
+ * structures associated with the given proc.
+ */
+#define binder_proc_lock(proc) _binder_proc_lock(proc, __LINE__)
+static void
+_binder_proc_lock(struct binder_proc *proc, int line)
+{
+	binder_debug(BINDER_DEBUG_SPINLOCKS,
+		     "%s: line=%d\n", __func__, line);
+	spin_lock(&proc->outer_lock);
+}
+
+/**
+ * binder_proc_unlock() - Release spinlock for given binder_proc
+ * @proc:         struct binder_proc to acquire
+ *
+ * Release lock acquired via binder_proc_lock()
+ */
+#define binder_proc_unlock(_proc) _binder_proc_unlock(_proc, __LINE__)
+static void
+_binder_proc_unlock(struct binder_proc *proc, int line)
+{
+	binder_debug(BINDER_DEBUG_SPINLOCKS,
+		     "%s: line=%d\n", __func__, line);
+	spin_unlock(&proc->outer_lock);
+}
+
+/**
+ * binder_inner_proc_lock() - Acquire inner lock for given binder_proc
+ * @proc:         struct binder_proc to acquire
+ *
+ * Acquires proc->inner_lock. Used to protect todo lists
+ */
+#define binder_inner_proc_lock(proc) _binder_inner_proc_lock(proc, __LINE__)
+static void
+_binder_inner_proc_lock(struct binder_proc *proc, int line)
+{
+	binder_debug(BINDER_DEBUG_SPINLOCKS,
+		     "%s: line=%d\n", __func__, line);
+	spin_lock(&proc->inner_lock);
+}
+
+/**
+ * binder_inner_proc_unlock() - Release inner lock for given binder_proc
+ * @proc:         struct binder_proc to acquire
+ *
+ * Release lock acquired via binder_inner_proc_lock()
+ */
+#define binder_inner_proc_unlock(proc) _binder_inner_proc_unlock(proc, __LINE__)
+static void
+_binder_inner_proc_unlock(struct binder_proc *proc, int line)
+{
+	binder_debug(BINDER_DEBUG_SPINLOCKS,
+		     "%s: line=%d\n", __func__, line);
+	spin_unlock(&proc->inner_lock);
+}
+
+/**
+ * binder_node_lock() - Acquire spinlock for given binder_node
+ * @node:         struct binder_node to acquire
+ *
+ * Acquires node->lock. Used to protect binder_node fields
+ */
+#define binder_node_lock(node) _binder_node_lock(node, __LINE__)
+static void
+_binder_node_lock(struct binder_node *node, int line)
+{
+	binder_debug(BINDER_DEBUG_SPINLOCKS,
+		     "%s: line=%d\n", __func__, line);
+	spin_lock(&node->lock);
+}
+
+/**
+ * binder_node_unlock() - Release spinlock for given binder_proc
+ * @node:         struct binder_node to acquire
+ *
+ * Release lock acquired via binder_node_lock()
+ */
+#define binder_node_unlock(node) _binder_node_unlock(node, __LINE__)
+static void
+_binder_node_unlock(struct binder_node *node, int line)
+{
+	binder_debug(BINDER_DEBUG_SPINLOCKS,
+		     "%s: line=%d\n", __func__, line);
+	spin_unlock(&node->lock);
+}
+
+/**
+ * binder_node_inner_lock() - Acquire node and inner locks
+ * @node:         struct binder_node to acquire
+ *
+ * Acquires node->lock. If node->proc also acquires
+ * proc->inner_lock. Used to protect binder_node fields
+ */
+#define binder_node_inner_lock(node) _binder_node_inner_lock(node, __LINE__)
+static void
+_binder_node_inner_lock(struct binder_node *node, int line)
+{
+	binder_debug(BINDER_DEBUG_SPINLOCKS,
+		     "%s: line=%d\n", __func__, line);
+	spin_lock(&node->lock);
+	if (node->proc)
+		binder_inner_proc_lock(node->proc);
+}
+
+/**
+ * binder_node_unlock() - Release node and inner locks
+ * @node:         struct binder_node to acquire
+ *
+ * Release lock acquired via binder_node_lock()
+ */
+#define binder_node_inner_unlock(node) _binder_node_inner_unlock(node, __LINE__)
+static void
+_binder_node_inner_unlock(struct binder_node *node, int line)
+{
+	struct binder_proc *proc = node->proc;
+
+	binder_debug(BINDER_DEBUG_SPINLOCKS,
+		     "%s: line=%d\n", __func__, line);
+	if (proc)
+		binder_inner_proc_unlock(proc);
+	spin_unlock(&node->lock);
+}
+
+static bool binder_worklist_empty_ilocked(struct list_head *list)
+{
+	return list_empty(list);
+}
+
+/**
+ * binder_worklist_empty() - Check if no items on the work list
+ * @proc:       binder_proc associated with list
+ * @list:	list to check
+ *
+ * Return: true if there are no items on list, else false
+ */
+static bool binder_worklist_empty(struct binder_proc *proc,
+				  struct list_head *list)
+{
+	bool ret;
+
+	binder_inner_proc_lock(proc);
+	ret = binder_worklist_empty_ilocked(list);
+	binder_inner_proc_unlock(proc);
+	return ret;
+}
+
+static void
+binder_enqueue_work_ilocked(struct binder_work *work,
+			   struct list_head *target_list)
+{
+	BUG_ON(target_list == NULL);
+	BUG_ON(work->entry.next && !list_empty(&work->entry));
+	list_add_tail(&work->entry, target_list);
+}
+
+/**
+ * binder_enqueue_work() - Add an item to the work list
+ * @proc:         binder_proc associated with list
+ * @work:         struct binder_work to add to list
+ * @target_list:  list to add work to
+ *
+ * Adds the work to the specified list. Asserts that work
+ * is not already on a list.
+ */
+static void
+binder_enqueue_work(struct binder_proc *proc,
+		    struct binder_work *work,
+		    struct list_head *target_list)
+{
+	binder_inner_proc_lock(proc);
+	binder_enqueue_work_ilocked(work, target_list);
+	binder_inner_proc_unlock(proc);
+}
+
+static void
+binder_dequeue_work_ilocked(struct binder_work *work)
+{
+	list_del_init(&work->entry);
+}
+
+/**
+ * binder_dequeue_work() - Removes an item from the work list
+ * @proc:         binder_proc associated with list
+ * @work:         struct binder_work to remove from list
+ *
+ * Removes the specified work item from whatever list it is on.
+ * Can safely be called if work is not on any list.
+ */
+static void
+binder_dequeue_work(struct binder_proc *proc, struct binder_work *work)
+{
+	binder_inner_proc_lock(proc);
+	binder_dequeue_work_ilocked(work);
+	binder_inner_proc_unlock(proc);
+}
+
+static struct binder_work *binder_dequeue_work_head_ilocked(
+					struct list_head *list)
+{
+	struct binder_work *w;
+
+	w = list_first_entry_or_null(list, struct binder_work, entry);
+	if (w)
+		list_del_init(&w->entry);
+	return w;
+}
+
+/**
+ * binder_dequeue_work_head() - Dequeues the item at head of list
+ * @proc:         binder_proc associated with list
+ * @list:         list to dequeue head
+ *
+ * Removes the head of the list if there are items on the list
+ *
+ * Return: pointer dequeued binder_work, NULL if list was empty
+ */
+static struct binder_work *binder_dequeue_work_head(
+					struct binder_proc *proc,
+					struct list_head *list)
+{
+	struct binder_work *w;
+
+	binder_inner_proc_lock(proc);
+	w = binder_dequeue_work_head_ilocked(list);
+	binder_inner_proc_unlock(proc);
+	return w;
+}
+
 static void
 binder_defer_work(struct binder_proc *proc, enum binder_deferred_state defer);
+static void binder_free_thread(struct binder_thread *thread);
+static void binder_free_proc(struct binder_proc *proc);
+static void binder_inc_node_tmpref_ilocked(struct binder_node *node);
 
 static int task_get_unused_fd_flags(struct binder_proc *proc, int flags)
 {
@@ -451,462 +924,159 @@ static long task_close_fd(struct binder_proc *proc, unsigned int fd)
 	return retval;
 }
 
-static inline void binder_lock(const char *tag)
-{
-	trace_binder_lock(tag);
-	mutex_lock(&binder_main_lock);
-	trace_binder_locked(tag);
-}
-
-static inline void binder_unlock(const char *tag)
-{
-	trace_binder_unlock(tag);
-	mutex_unlock(&binder_main_lock);
-}
-
-static void binder_set_nice(long nice)
-{
-	long min_nice;
-
-	if (can_nice(current, nice)) {
-		set_user_nice(current, nice);
-		return;
-	}
-	min_nice = rlimit_to_nice(current->signal->rlim[RLIMIT_NICE].rlim_cur);
-	binder_debug(BINDER_DEBUG_PRIORITY_CAP,
-		     "%d: nice value %ld not allowed use %ld instead\n",
-		      current->pid, nice, min_nice);
-	set_user_nice(current, min_nice);
-	if (min_nice <= MAX_NICE)
-		return;
-	binder_user_error("%d RLIMIT_NICE not set\n", current->pid);
-}
-
-static size_t binder_buffer_size(struct binder_proc *proc,
-				 struct binder_buffer *buffer)
+static bool binder_has_work_ilocked(struct binder_thread *thread,
+				    bool do_proc_work)
 {
-	if (list_is_last(&buffer->entry, &proc->buffers))
-		return proc->buffer + proc->buffer_size - (void *)buffer->data;
-	return (size_t)list_entry(buffer->entry.next,
-			  struct binder_buffer, entry) - (size_t)buffer->data;
+	return !binder_worklist_empty_ilocked(&thread->todo) ||
+		thread->looper_need_return ||
+		(do_proc_work &&
+		 !binder_worklist_empty_ilocked(&thread->proc->todo));
 }
 
-static void binder_insert_free_buffer(struct binder_proc *proc,
-				      struct binder_buffer *new_buffer)
+static bool binder_has_work(struct binder_thread *thread, bool do_proc_work)
 {
-	struct rb_node **p = &proc->free_buffers.rb_node;
-	struct rb_node *parent = NULL;
-	struct binder_buffer *buffer;
-	size_t buffer_size;
-	size_t new_buffer_size;
+	bool has_work;
 
-	BUG_ON(!new_buffer->free);
+	binder_inner_proc_lock(thread->proc);
+	has_work = binder_has_work_ilocked(thread, do_proc_work);
+	binder_inner_proc_unlock(thread->proc);
 
-	new_buffer_size = binder_buffer_size(proc, new_buffer);
-
-	binder_debug(BINDER_DEBUG_BUFFER_ALLOC,
-		     "%d: add free buffer, size %zd, at %p\n",
-		      proc->pid, new_buffer_size, new_buffer);
-
-	while (*p) {
-		parent = *p;
-		buffer = rb_entry(parent, struct binder_buffer, rb_node);
-		BUG_ON(!buffer->free);
-
-		buffer_size = binder_buffer_size(proc, buffer);
-
-		if (new_buffer_size < buffer_size)
-			p = &parent->rb_left;
-		else
-			p = &parent->rb_right;
-	}
-	rb_link_node(&new_buffer->rb_node, parent, p);
-	rb_insert_color(&new_buffer->rb_node, &proc->free_buffers);
+	return has_work;
 }
 
-static void binder_insert_allocated_buffer(struct binder_proc *proc,
-					   struct binder_buffer *new_buffer)
+static bool binder_available_for_proc_work_ilocked(struct binder_thread *thread)
 {
-	struct rb_node **p = &proc->allocated_buffers.rb_node;
-	struct rb_node *parent = NULL;
-	struct binder_buffer *buffer;
-
-	BUG_ON(new_buffer->free);
-
-	while (*p) {
-		parent = *p;
-		buffer = rb_entry(parent, struct binder_buffer, rb_node);
-		BUG_ON(buffer->free);
-
-		if (new_buffer < buffer)
-			p = &parent->rb_left;
-		else if (new_buffer > buffer)
-			p = &parent->rb_right;
-		else
-			BUG();
-	}
-	rb_link_node(&new_buffer->rb_node, parent, p);
-	rb_insert_color(&new_buffer->rb_node, &proc->allocated_buffers);
+	return !thread->transaction_stack &&
+		binder_worklist_empty_ilocked(&thread->todo) &&
+		(thread->looper & (BINDER_LOOPER_STATE_ENTERED |
+				   BINDER_LOOPER_STATE_REGISTERED));
 }
 
-static struct binder_buffer *binder_buffer_lookup(struct binder_proc *proc,
-						  uintptr_t user_ptr)
+static void binder_wakeup_poll_threads_ilocked(struct binder_proc *proc,
+					       bool sync)
 {
-	struct rb_node *n = proc->allocated_buffers.rb_node;
-	struct binder_buffer *buffer;
-	struct binder_buffer *kern_ptr;
-
-	kern_ptr = (struct binder_buffer *)(user_ptr - proc->user_buffer_offset
-		- offsetof(struct binder_buffer, data));
-
-	while (n) {
-		buffer = rb_entry(n, struct binder_buffer, rb_node);
-		BUG_ON(buffer->free);
+	struct rb_node *n;
+	struct binder_thread *thread;
 
-		if (kern_ptr < buffer)
-			n = n->rb_left;
-		else if (kern_ptr > buffer)
-			n = n->rb_right;
-		else
-			return buffer;
+	for (n = rb_first(&proc->threads); n != NULL; n = rb_next(n)) {
+		thread = rb_entry(n, struct binder_thread, rb_node);
+		if (thread->looper & BINDER_LOOPER_STATE_POLL &&
+		    binder_available_for_proc_work_ilocked(thread)) {
+			if (sync)
+				wake_up_interruptible_sync(&thread->wait);
+			else
+				wake_up_interruptible(&thread->wait);
+		}
 	}
-	return NULL;
 }
 
-static int binder_update_page_range(struct binder_proc *proc, int allocate,
-				    void *start, void *end,
-				    struct vm_area_struct *vma)
+/**
+ * binder_select_thread_ilocked() - selects a thread for doing proc work.
+ * @proc:	process to select a thread from
+ *
+ * Note that calling this function moves the thread off the waiting_threads
+ * list, so it can only be woken up by the caller of this function, or a
+ * signal. Therefore, callers *should* always wake up the thread this function
+ * returns.
+ *
+ * Return:	If there's a thread currently waiting for process work,
+ *		returns that thread. Otherwise returns NULL.
+ */
+static struct binder_thread *
+binder_select_thread_ilocked(struct binder_proc *proc)
 {
-	void *page_addr;
-	unsigned long user_page_addr;
-	struct page **page;
-	struct mm_struct *mm;
-
-	binder_debug(BINDER_DEBUG_BUFFER_ALLOC,
-		     "%d: %s pages %p-%p\n", proc->pid,
-		     allocate ? "allocate" : "free", start, end);
-
-	if (end <= start)
-		return 0;
-
-	trace_binder_update_page_range(proc, allocate, start, end);
-
-	if (vma)
-		mm = NULL;
-	else
-		mm = get_task_mm(proc->tsk);
-
-	if (mm) {
-		down_write(&mm->mmap_sem);
-		vma = proc->vma;
-		if (vma && mm != proc->vma_vm_mm) {
-			pr_err("%d: vma mm and task mm mismatch\n",
-				proc->pid);
-			vma = NULL;
-		}
-	}
-
-	if (allocate == 0)
-		goto free_range;
-
-	if (vma == NULL) {
-		pr_err("%d: binder_alloc_buf failed to map pages in userspace, no vma\n",
-			proc->pid);
-		goto err_no_vma;
-	}
-
-	for (page_addr = start; page_addr < end; page_addr += PAGE_SIZE) {
-		int ret;
+	struct binder_thread *thread;
 
-		page = &proc->pages[(page_addr - proc->buffer) / PAGE_SIZE];
+	assert_spin_locked(&proc->inner_lock);
+	thread = list_first_entry_or_null(&proc->waiting_threads,
+					  struct binder_thread,
+					  waiting_thread_node);
 
-		BUG_ON(*page);
-		*page = alloc_page(GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
-		if (*page == NULL) {
-			pr_err("%d: binder_alloc_buf failed for page at %p\n",
-				proc->pid, page_addr);
-			goto err_alloc_page_failed;
-		}
-		ret = map_kernel_range_noflush((unsigned long)page_addr,
-					PAGE_SIZE, PAGE_KERNEL, page);
-		flush_cache_vmap((unsigned long)page_addr,
-				(unsigned long)page_addr + PAGE_SIZE);
-		if (ret != 1) {
-			pr_err("%d: binder_alloc_buf failed to map page at %p in kernel\n",
-			       proc->pid, page_addr);
-			goto err_map_kernel_failed;
-		}
-		user_page_addr =
-			(uintptr_t)page_addr + proc->user_buffer_offset;
-		ret = vm_insert_page(vma, user_page_addr, page[0]);
-		if (ret) {
-			pr_err("%d: binder_alloc_buf failed to map page at %lx in userspace\n",
-			       proc->pid, user_page_addr);
-			goto err_vm_insert_page_failed;
-		}
-		/* vm_insert_page does not seem to increment the refcount */
-	}
-	if (mm) {
-		up_write(&mm->mmap_sem);
-		mmput(mm);
-	}
-	return 0;
+	if (thread)
+		list_del_init(&thread->waiting_thread_node);
 
-free_range:
-	for (page_addr = end - PAGE_SIZE; page_addr >= start;
-	     page_addr -= PAGE_SIZE) {
-		page = &proc->pages[(page_addr - proc->buffer) / PAGE_SIZE];
-		if (vma)
-			zap_page_range(vma, (uintptr_t)page_addr +
-				proc->user_buffer_offset, PAGE_SIZE);
-err_vm_insert_page_failed:
-		unmap_kernel_range((unsigned long)page_addr, PAGE_SIZE);
-err_map_kernel_failed:
-		__free_page(*page);
-		*page = NULL;
-err_alloc_page_failed:
-		;
-	}
-err_no_vma:
-	if (mm) {
-		up_write(&mm->mmap_sem);
-		mmput(mm);
-	}
-	return -ENOMEM;
+	return thread;
 }
 
-static struct binder_buffer *binder_alloc_buf(struct binder_proc *proc,
-					      size_t data_size,
-					      size_t offsets_size,
-					      size_t extra_buffers_size,
-					      int is_async)
+/**
+ * binder_wakeup_thread_ilocked() - wakes up a thread for doing proc work.
+ * @proc:	process to wake up a thread in
+ * @thread:	specific thread to wake-up (may be NULL)
+ * @sync:	whether to do a synchronous wake-up
+ *
+ * This function wakes up a thread in the @proc process.
+ * The caller may provide a specific thread to wake-up in
+ * the @thread parameter. If @thread is NULL, this function
+ * will wake up threads that have called poll().
+ *
+ * Note that for this function to work as expected, callers
+ * should first call binder_select_thread() to find a thread
+ * to handle the work (if they don't have a thread already),
+ * and pass the result into the @thread parameter.
+ */
+static void binder_wakeup_thread_ilocked(struct binder_proc *proc,
+					 struct binder_thread *thread,
+					 bool sync)
 {
-	struct rb_node *n = proc->free_buffers.rb_node;
-	struct binder_buffer *buffer;
-	size_t buffer_size;
-	struct rb_node *best_fit = NULL;
-	void *has_page_addr;
-	void *end_page_addr;
-	size_t size, data_offsets_size;
-
-	if (proc->vma == NULL) {
-		pr_err("%d: binder_alloc_buf, no vma\n",
-		       proc->pid);
-		return NULL;
-	}
-
-	data_offsets_size = ALIGN(data_size, sizeof(void *)) +
-		ALIGN(offsets_size, sizeof(void *));
-
-	if (data_offsets_size < data_size || data_offsets_size < offsets_size) {
-		binder_user_error("%d: got transaction with invalid size %zd-%zd\n",
-				proc->pid, data_size, offsets_size);
-		return NULL;
-	}
-	size = data_offsets_size + ALIGN(extra_buffers_size, sizeof(void *));
-	if (size < data_offsets_size || size < extra_buffers_size) {
-		binder_user_error("%d: got transaction with invalid extra_buffers_size %zd\n",
-				  proc->pid, extra_buffers_size);
-		return NULL;
-	}
-	if (is_async &&
-	    proc->free_async_space < size + sizeof(struct binder_buffer)) {
-		binder_debug(BINDER_DEBUG_BUFFER_ALLOC,
-			     "%d: binder_alloc_buf size %zd failed, no async space left\n",
-			      proc->pid, size);
-		return NULL;
-	}
+	assert_spin_locked(&proc->inner_lock);
 
-	while (n) {
-		buffer = rb_entry(n, struct binder_buffer, rb_node);
-		BUG_ON(!buffer->free);
-		buffer_size = binder_buffer_size(proc, buffer);
-
-		if (size < buffer_size) {
-			best_fit = n;
-			n = n->rb_left;
-		} else if (size > buffer_size)
-			n = n->rb_right;
-		else {
-			best_fit = n;
-			break;
-		}
-	}
-	if (best_fit == NULL) {
-		pr_err("%d: binder_alloc_buf size %zd failed, no address space\n",
-			proc->pid, size);
-		return NULL;
-	}
-	if (n == NULL) {
-		buffer = rb_entry(best_fit, struct binder_buffer, rb_node);
-		buffer_size = binder_buffer_size(proc, buffer);
-	}
-
-	binder_debug(BINDER_DEBUG_BUFFER_ALLOC,
-		     "%d: binder_alloc_buf size %zd got buffer %p size %zd\n",
-		      proc->pid, size, buffer, buffer_size);
-
-	has_page_addr =
-		(void *)(((uintptr_t)buffer->data + buffer_size) & PAGE_MASK);
-	if (n == NULL) {
-		if (size + sizeof(struct binder_buffer) + 4 >= buffer_size)
-			buffer_size = size; /* no room for other buffers */
+	if (thread) {
+		if (sync)
+			wake_up_interruptible_sync(&thread->wait);
 		else
-			buffer_size = size + sizeof(struct binder_buffer);
-	}
-	end_page_addr =
-		(void *)PAGE_ALIGN((uintptr_t)buffer->data + buffer_size);
-	if (end_page_addr > has_page_addr)
-		end_page_addr = has_page_addr;
-	if (binder_update_page_range(proc, 1,
-	    (void *)PAGE_ALIGN((uintptr_t)buffer->data), end_page_addr, NULL))
-		return NULL;
-
-	rb_erase(best_fit, &proc->free_buffers);
-	buffer->free = 0;
-	binder_insert_allocated_buffer(proc, buffer);
-	if (buffer_size != size) {
-		struct binder_buffer *new_buffer = (void *)buffer->data + size;
-
-		list_add(&new_buffer->entry, &buffer->entry);
-		new_buffer->free = 1;
-		binder_insert_free_buffer(proc, new_buffer);
-	}
-	binder_debug(BINDER_DEBUG_BUFFER_ALLOC,
-		     "%d: binder_alloc_buf size %zd got %p\n",
-		      proc->pid, size, buffer);
-	buffer->data_size = data_size;
-	buffer->offsets_size = offsets_size;
-	buffer->extra_buffers_size = extra_buffers_size;
-	buffer->async_transaction = is_async;
-	if (is_async) {
-		proc->free_async_space -= size + sizeof(struct binder_buffer);
-		binder_debug(BINDER_DEBUG_BUFFER_ALLOC_ASYNC,
-			     "%d: binder_alloc_buf size %zd async free %zd\n",
-			      proc->pid, size, proc->free_async_space);
+			wake_up_interruptible(&thread->wait);
+		return;
 	}
 
-	return buffer;
-}
-
-static void *buffer_start_page(struct binder_buffer *buffer)
-{
-	return (void *)((uintptr_t)buffer & PAGE_MASK);
+	/* Didn't find a thread waiting for proc work; this can happen
+	 * in two scenarios:
+	 * 1. All threads are busy handling transactions
+	 *    In that case, one of those threads should call back into
+	 *    the kernel driver soon and pick up this work.
+	 * 2. Threads are using the (e)poll interface, in which case
+	 *    they may be blocked on the waitqueue without having been
+	 *    added to waiting_threads. For this case, we just iterate
+	 *    over all threads not handling transaction work, and
+	 *    wake them all up. We wake all because we don't know whether
+	 *    a thread that called into (e)poll is handling non-binder
+	 *    work currently.
+	 */
+	binder_wakeup_poll_threads_ilocked(proc, sync);
 }
 
-static void *buffer_end_page(struct binder_buffer *buffer)
+static void binder_wakeup_proc_ilocked(struct binder_proc *proc)
 {
-	return (void *)(((uintptr_t)(buffer + 1) - 1) & PAGE_MASK);
-}
+	struct binder_thread *thread = binder_select_thread_ilocked(proc);
 
-static void binder_delete_free_buffer(struct binder_proc *proc,
-				      struct binder_buffer *buffer)
-{
-	struct binder_buffer *prev, *next = NULL;
-	int free_page_end = 1;
-	int free_page_start = 1;
-
-	BUG_ON(proc->buffers.next == &buffer->entry);
-	prev = list_entry(buffer->entry.prev, struct binder_buffer, entry);
-	BUG_ON(!prev->free);
-	if (buffer_end_page(prev) == buffer_start_page(buffer)) {
-		free_page_start = 0;
-		if (buffer_end_page(prev) == buffer_end_page(buffer))
-			free_page_end = 0;
-		binder_debug(BINDER_DEBUG_BUFFER_ALLOC,
-			     "%d: merge free, buffer %p share page with %p\n",
-			      proc->pid, buffer, prev);
-	}
-
-	if (!list_is_last(&buffer->entry, &proc->buffers)) {
-		next = list_entry(buffer->entry.next,
-				  struct binder_buffer, entry);
-		if (buffer_start_page(next) == buffer_end_page(buffer)) {
-			free_page_end = 0;
-			if (buffer_start_page(next) ==
-			    buffer_start_page(buffer))
-				free_page_start = 0;
-			binder_debug(BINDER_DEBUG_BUFFER_ALLOC,
-				     "%d: merge free, buffer %p share page with %p\n",
-				      proc->pid, buffer, prev);
-		}
-	}
-	list_del(&buffer->entry);
-	if (free_page_start || free_page_end) {
-		binder_debug(BINDER_DEBUG_BUFFER_ALLOC,
-			     "%d: merge free, buffer %p do not share page%s%s with %p or %p\n",
-			     proc->pid, buffer, free_page_start ? "" : " end",
-			     free_page_end ? "" : " start", prev, next);
-		binder_update_page_range(proc, 0, free_page_start ?
-			buffer_start_page(buffer) : buffer_end_page(buffer),
-			(free_page_end ? buffer_end_page(buffer) :
-			buffer_start_page(buffer)) + PAGE_SIZE, NULL);
-	}
+	binder_wakeup_thread_ilocked(proc, thread, /* sync = */false);
 }
 
-static void binder_free_buf(struct binder_proc *proc,
-			    struct binder_buffer *buffer)
+static void binder_set_nice(long nice)
 {
-	size_t size, buffer_size;
-
-	buffer_size = binder_buffer_size(proc, buffer);
-
-	size = ALIGN(buffer->data_size, sizeof(void *)) +
-		ALIGN(buffer->offsets_size, sizeof(void *)) +
-		ALIGN(buffer->extra_buffers_size, sizeof(void *));
-
-	binder_debug(BINDER_DEBUG_BUFFER_ALLOC,
-		     "%d: binder_free_buf %p size %zd buffer_size %zd\n",
-		      proc->pid, buffer, size, buffer_size);
-
-	BUG_ON(buffer->free);
-	BUG_ON(size > buffer_size);
-	BUG_ON(buffer->transaction != NULL);
-	BUG_ON((void *)buffer < proc->buffer);
-	BUG_ON((void *)buffer > proc->buffer + proc->buffer_size);
-
-	if (buffer->async_transaction) {
-		proc->free_async_space += size + sizeof(struct binder_buffer);
-
-		binder_debug(BINDER_DEBUG_BUFFER_ALLOC_ASYNC,
-			     "%d: binder_free_buf size %zd async free %zd\n",
-			      proc->pid, size, proc->free_async_space);
-	}
+	long min_nice;
 
-	binder_update_page_range(proc, 0,
-		(void *)PAGE_ALIGN((uintptr_t)buffer->data),
-		(void *)(((uintptr_t)buffer->data + buffer_size) & PAGE_MASK),
-		NULL);
-	rb_erase(&buffer->rb_node, &proc->allocated_buffers);
-	buffer->free = 1;
-	if (!list_is_last(&buffer->entry, &proc->buffers)) {
-		struct binder_buffer *next = list_entry(buffer->entry.next,
-						struct binder_buffer, entry);
-
-		if (next->free) {
-			rb_erase(&next->rb_node, &proc->free_buffers);
-			binder_delete_free_buffer(proc, next);
-		}
-	}
-	if (proc->buffers.next != &buffer->entry) {
-		struct binder_buffer *prev = list_entry(buffer->entry.prev,
-						struct binder_buffer, entry);
-
-		if (prev->free) {
-			binder_delete_free_buffer(proc, buffer);
-			rb_erase(&prev->rb_node, &proc->free_buffers);
-			buffer = prev;
-		}
+	if (can_nice(current, nice)) {
+		set_user_nice(current, nice);
+		return;
 	}
-	binder_insert_free_buffer(proc, buffer);
+	min_nice = rlimit_to_nice(rlimit(RLIMIT_NICE));
+	binder_debug(BINDER_DEBUG_PRIORITY_CAP,
+		     "%d: nice value %ld not allowed use %ld instead\n",
+		      current->pid, nice, min_nice);
+	set_user_nice(current, min_nice);
+	if (min_nice <= MAX_NICE)
+		return;
+	binder_user_error("%d RLIMIT_NICE not set\n", current->pid);
 }
 
-static struct binder_node *binder_get_node(struct binder_proc *proc,
-					   binder_uintptr_t ptr)
+static struct binder_node *binder_get_node_ilocked(struct binder_proc *proc,
+						   binder_uintptr_t ptr)
 {
 	struct rb_node *n = proc->nodes.rb_node;
 	struct binder_node *node;
 
+	assert_spin_locked(&proc->inner_lock);
+
 	while (n) {
 		node = rb_entry(n, struct binder_node, rb_node);
 
@@ -914,21 +1084,46 @@ static struct binder_node *binder_get_node(struct binder_proc *proc,
 			n = n->rb_left;
 		else if (ptr > node->ptr)
 			n = n->rb_right;
-		else
+		else {
+			/*
+			 * take an implicit weak reference
+			 * to ensure node stays alive until
+			 * call to binder_put_node()
+			 */
+			binder_inc_node_tmpref_ilocked(node);
 			return node;
+		}
 	}
 	return NULL;
 }
 
-static struct binder_node *binder_new_node(struct binder_proc *proc,
-					   binder_uintptr_t ptr,
-					   binder_uintptr_t cookie)
+static struct binder_node *binder_get_node(struct binder_proc *proc,
+					   binder_uintptr_t ptr)
+{
+	struct binder_node *node;
+
+	binder_inner_proc_lock(proc);
+	node = binder_get_node_ilocked(proc, ptr);
+	binder_inner_proc_unlock(proc);
+	return node;
+}
+
+static struct binder_node *binder_init_node_ilocked(
+						struct binder_proc *proc,
+						struct binder_node *new_node,
+						struct flat_binder_object *fp)
 {
 	struct rb_node **p = &proc->nodes.rb_node;
 	struct rb_node *parent = NULL;
 	struct binder_node *node;
+	binder_uintptr_t ptr = fp ? fp->binder : 0;
+	binder_uintptr_t cookie = fp ? fp->cookie : 0;
+	__u32 flags = fp ? fp->flags : 0;
+
+	assert_spin_locked(&proc->inner_lock);
 
 	while (*p) {
+
 		parent = *p;
 		node = rb_entry(parent, struct binder_node, rb_node);
 
@@ -936,33 +1131,74 @@ static struct binder_node *binder_new_node(struct binder_proc *proc,
 			p = &(*p)->rb_left;
 		else if (ptr > node->ptr)
 			p = &(*p)->rb_right;
-		else
-			return NULL;
+		else {
+			/*
+			 * A matching node is already in
+			 * the rb tree. Abandon the init
+			 * and return it.
+			 */
+			binder_inc_node_tmpref_ilocked(node);
+			return node;
+		}
 	}
-
-	node = kzalloc(sizeof(*node), GFP_KERNEL);
-	if (node == NULL)
-		return NULL;
+	node = new_node;
 	binder_stats_created(BINDER_STAT_NODE);
+	node->tmp_refs++;
 	rb_link_node(&node->rb_node, parent, p);
 	rb_insert_color(&node->rb_node, &proc->nodes);
-	node->debug_id = ++binder_last_id;
+	node->debug_id = atomic_inc_return(&binder_last_id);
 	node->proc = proc;
 	node->ptr = ptr;
 	node->cookie = cookie;
 	node->work.type = BINDER_WORK_NODE;
+	node->min_priority = flags & FLAT_BINDER_FLAG_PRIORITY_MASK;
+	node->accept_fds = !!(flags & FLAT_BINDER_FLAG_ACCEPTS_FDS);
+	spin_lock_init(&node->lock);
 	INIT_LIST_HEAD(&node->work.entry);
 	INIT_LIST_HEAD(&node->async_todo);
 	binder_debug(BINDER_DEBUG_INTERNAL_REFS,
 		     "%d:%d node %d u%016llx c%016llx created\n",
 		     proc->pid, current->pid, node->debug_id,
 		     (u64)node->ptr, (u64)node->cookie);
+
 	return node;
 }
 
-static int binder_inc_node(struct binder_node *node, int strong, int internal,
-			   struct list_head *target_list)
+static struct binder_node *binder_new_node(struct binder_proc *proc,
+					   struct flat_binder_object *fp)
 {
+	struct binder_node *node;
+	struct binder_node *new_node = kzalloc(sizeof(*node), GFP_KERNEL);
+
+	if (!new_node)
+		return NULL;
+	binder_inner_proc_lock(proc);
+	node = binder_init_node_ilocked(proc, new_node, fp);
+	binder_inner_proc_unlock(proc);
+	if (node != new_node)
+		/*
+		 * The node was already added by another thread
+		 */
+		kfree(new_node);
+
+	return node;
+}
+
+static void binder_free_node(struct binder_node *node)
+{
+	kfree(node);
+	binder_stats_deleted(BINDER_STAT_NODE);
+}
+
+static int binder_inc_node_nilocked(struct binder_node *node, int strong,
+				    int internal,
+				    struct list_head *target_list)
+{
+	struct binder_proc *proc = node->proc;
+
+	assert_spin_locked(&node->lock);
+	if (proc)
+		assert_spin_locked(&proc->inner_lock);
 	if (strong) {
 		if (internal) {
 			if (target_list == NULL &&
@@ -978,8 +1214,8 @@ static int binder_inc_node(struct binder_node *node, int strong, int internal,
 		} else
 			node->local_strong_refs++;
 		if (!node->has_strong_ref && target_list) {
-			list_del_init(&node->work.entry);
-			list_add_tail(&node->work.entry, target_list);
+			binder_dequeue_work_ilocked(&node->work);
+			binder_enqueue_work_ilocked(&node->work, target_list);
 		}
 	} else {
 		if (!internal)
@@ -990,58 +1226,169 @@ static int binder_inc_node(struct binder_node *node, int strong, int internal,
 					node->debug_id);
 				return -EINVAL;
 			}
-			list_add_tail(&node->work.entry, target_list);
+			binder_enqueue_work_ilocked(&node->work, target_list);
 		}
 	}
 	return 0;
 }
 
-static int binder_dec_node(struct binder_node *node, int strong, int internal)
+static int binder_inc_node(struct binder_node *node, int strong, int internal,
+			   struct list_head *target_list)
+{
+	int ret;
+
+	binder_node_inner_lock(node);
+	ret = binder_inc_node_nilocked(node, strong, internal, target_list);
+	binder_node_inner_unlock(node);
+
+	return ret;
+}
+
+static bool binder_dec_node_nilocked(struct binder_node *node,
+				     int strong, int internal)
 {
+	struct binder_proc *proc = node->proc;
+
+	assert_spin_locked(&node->lock);
+	if (proc)
+		assert_spin_locked(&proc->inner_lock);
 	if (strong) {
 		if (internal)
 			node->internal_strong_refs--;
 		else
 			node->local_strong_refs--;
 		if (node->local_strong_refs || node->internal_strong_refs)
-			return 0;
+			return false;
 	} else {
 		if (!internal)
 			node->local_weak_refs--;
-		if (node->local_weak_refs || !hlist_empty(&node->refs))
-			return 0;
+		if (node->local_weak_refs || node->tmp_refs ||
+				!hlist_empty(&node->refs))
+			return false;
 	}
-	if (node->proc && (node->has_strong_ref || node->has_weak_ref)) {
+
+	if (proc && (node->has_strong_ref || node->has_weak_ref)) {
 		if (list_empty(&node->work.entry)) {
-			list_add_tail(&node->work.entry, &node->proc->todo);
-			wake_up_interruptible(&node->proc->wait);
+			binder_enqueue_work_ilocked(&node->work, &proc->todo);
+			binder_wakeup_proc_ilocked(proc);
 		}
 	} else {
 		if (hlist_empty(&node->refs) && !node->local_strong_refs &&
-		    !node->local_weak_refs) {
-			list_del_init(&node->work.entry);
-			if (node->proc) {
-				rb_erase(&node->rb_node, &node->proc->nodes);
+		    !node->local_weak_refs && !node->tmp_refs) {
+			if (proc) {
+				binder_dequeue_work_ilocked(&node->work);
+				rb_erase(&node->rb_node, &proc->nodes);
 				binder_debug(BINDER_DEBUG_INTERNAL_REFS,
 					     "refless node %d deleted\n",
 					     node->debug_id);
 			} else {
+				BUG_ON(!list_empty(&node->work.entry));
+				spin_lock(&binder_dead_nodes_lock);
+				/*
+				 * tmp_refs could have changed so
+				 * check it again
+				 */
+				if (node->tmp_refs) {
+					spin_unlock(&binder_dead_nodes_lock);
+					return false;
+				}
 				hlist_del(&node->dead_node);
+				spin_unlock(&binder_dead_nodes_lock);
 				binder_debug(BINDER_DEBUG_INTERNAL_REFS,
 					     "dead node %d deleted\n",
 					     node->debug_id);
 			}
-			kfree(node);
-			binder_stats_deleted(BINDER_STAT_NODE);
+			return true;
 		}
 	}
+	return false;
+}
 
-	return 0;
+static void binder_dec_node(struct binder_node *node, int strong, int internal)
+{
+	bool free_node;
+
+	binder_node_inner_lock(node);
+	free_node = binder_dec_node_nilocked(node, strong, internal);
+	binder_node_inner_unlock(node);
+	if (free_node)
+		binder_free_node(node);
+}
+
+static void binder_inc_node_tmpref_ilocked(struct binder_node *node)
+{
+	/*
+	 * No call to binder_inc_node() is needed since we
+	 * don't need to inform userspace of any changes to
+	 * tmp_refs
+	 */
+	node->tmp_refs++;
+}
+
+/**
+ * binder_inc_node_tmpref() - take a temporary reference on node
+ * @node:	node to reference
+ *
+ * Take reference on node to prevent the node from being freed
+ * while referenced only by a local variable. The inner lock is
+ * needed to serialize with the node work on the queue (which
+ * isn't needed after the node is dead). If the node is dead
+ * (node->proc is NULL), use binder_dead_nodes_lock to protect
+ * node->tmp_refs against dead-node-only cases where the node
+ * lock cannot be acquired (eg traversing the dead node list to
+ * print nodes)
+ */
+static void binder_inc_node_tmpref(struct binder_node *node)
+{
+	binder_node_lock(node);
+	if (node->proc)
+		binder_inner_proc_lock(node->proc);
+	else
+		spin_lock(&binder_dead_nodes_lock);
+	binder_inc_node_tmpref_ilocked(node);
+	if (node->proc)
+		binder_inner_proc_unlock(node->proc);
+	else
+		spin_unlock(&binder_dead_nodes_lock);
+	binder_node_unlock(node);
+}
+
+/**
+ * binder_dec_node_tmpref() - remove a temporary reference on node
+ * @node:	node to reference
+ *
+ * Release temporary reference on node taken via binder_inc_node_tmpref()
+ */
+static void binder_dec_node_tmpref(struct binder_node *node)
+{
+	bool free_node;
+
+	binder_node_inner_lock(node);
+	if (!node->proc)
+		spin_lock(&binder_dead_nodes_lock);
+	node->tmp_refs--;
+	BUG_ON(node->tmp_refs < 0);
+	if (!node->proc)
+		spin_unlock(&binder_dead_nodes_lock);
+	/*
+	 * Call binder_dec_node() to check if all refcounts are 0
+	 * and cleanup is needed. Calling with strong=0 and internal=1
+	 * causes no actual reference to be released in binder_dec_node().
+	 * If that changes, a change is needed here too.
+	 */
+	free_node = binder_dec_node_nilocked(node, 0, 1);
+	binder_node_inner_unlock(node);
+	if (free_node)
+		binder_free_node(node);
 }
 
+static void binder_put_node(struct binder_node *node)
+{
+	binder_dec_node_tmpref(node);
+}
 
-static struct binder_ref *binder_get_ref(struct binder_proc *proc,
-					 u32 desc, bool need_strong_ref)
+static struct binder_ref *binder_get_ref_olocked(struct binder_proc *proc,
+						 u32 desc, bool need_strong_ref)
 {
 	struct rb_node *n = proc->refs_by_desc.rb_node;
 	struct binder_ref *ref;
@@ -1049,11 +1396,11 @@ static struct binder_ref *binder_get_ref(struct binder_proc *proc,
 	while (n) {
 		ref = rb_entry(n, struct binder_ref, rb_node_desc);
 
-		if (desc < ref->desc) {
+		if (desc < ref->data.desc) {
 			n = n->rb_left;
-		} else if (desc > ref->desc) {
+		} else if (desc > ref->data.desc) {
 			n = n->rb_right;
-		} else if (need_strong_ref && !ref->strong) {
+		} else if (need_strong_ref && !ref->data.strong) {
 			binder_user_error("tried to use weak ref as strong ref\n");
 			return NULL;
 		} else {
@@ -1063,14 +1410,34 @@ static struct binder_ref *binder_get_ref(struct binder_proc *proc,
 	return NULL;
 }
 
-static struct binder_ref *binder_get_ref_for_node(struct binder_proc *proc,
-						  struct binder_node *node)
+/**
+ * binder_get_ref_for_node_olocked() - get the ref associated with given node
+ * @proc:	binder_proc that owns the ref
+ * @node:	binder_node of target
+ * @new_ref:	newly allocated binder_ref to be initialized or %NULL
+ *
+ * Look up the ref for the given node and return it if it exists
+ *
+ * If it doesn't exist and the caller provides a newly allocated
+ * ref, initialize the fields of the newly allocated ref and insert
+ * into the given proc rb_trees and node refs list.
+ *
+ * Return:	the ref for node. It is possible that another thread
+ *		allocated/initialized the ref first in which case the
+ *		returned ref would be different than the passed-in
+ *		new_ref. new_ref must be kfree'd by the caller in
+ *		this case.
+ */
+static struct binder_ref *binder_get_ref_for_node_olocked(
+					struct binder_proc *proc,
+					struct binder_node *node,
+					struct binder_ref *new_ref)
 {
-	struct rb_node *n;
+	struct binder_context *context = proc->context;
 	struct rb_node **p = &proc->refs_by_node.rb_node;
 	struct rb_node *parent = NULL;
-	struct binder_ref *ref, *new_ref;
-	struct binder_context *context = proc->context;
+	struct binder_ref *ref;
+	struct rb_node *n;
 
 	while (*p) {
 		parent = *p;
@@ -1083,22 +1450,22 @@ static struct binder_ref *binder_get_ref_for_node(struct binder_proc *proc,
 		else
 			return ref;
 	}
-	new_ref = kzalloc(sizeof(*ref), GFP_KERNEL);
-	if (new_ref == NULL)
+	if (!new_ref)
 		return NULL;
+
 	binder_stats_created(BINDER_STAT_REF);
-	new_ref->debug_id = ++binder_last_id;
+	new_ref->data.debug_id = atomic_inc_return(&binder_last_id);
 	new_ref->proc = proc;
 	new_ref->node = node;
 	rb_link_node(&new_ref->rb_node_node, parent, p);
 	rb_insert_color(&new_ref->rb_node_node, &proc->refs_by_node);
 
-	new_ref->desc = (node == context->binder_context_mgr_node) ? 0 : 1;
+	new_ref->data.desc = (node == context->binder_context_mgr_node) ? 0 : 1;
 	for (n = rb_first(&proc->refs_by_desc); n != NULL; n = rb_next(n)) {
 		ref = rb_entry(n, struct binder_ref, rb_node_desc);
-		if (ref->desc > new_ref->desc)
+		if (ref->data.desc > new_ref->data.desc)
 			break;
-		new_ref->desc = ref->desc + 1;
+		new_ref->data.desc = ref->data.desc + 1;
 	}
 
 	p = &proc->refs_by_desc.rb_node;
@@ -1106,121 +1473,423 @@ static struct binder_ref *binder_get_ref_for_node(struct binder_proc *proc,
 		parent = *p;
 		ref = rb_entry(parent, struct binder_ref, rb_node_desc);
 
-		if (new_ref->desc < ref->desc)
+		if (new_ref->data.desc < ref->data.desc)
 			p = &(*p)->rb_left;
-		else if (new_ref->desc > ref->desc)
+		else if (new_ref->data.desc > ref->data.desc)
 			p = &(*p)->rb_right;
 		else
 			BUG();
 	}
 	rb_link_node(&new_ref->rb_node_desc, parent, p);
 	rb_insert_color(&new_ref->rb_node_desc, &proc->refs_by_desc);
-	if (node) {
-		hlist_add_head(&new_ref->node_entry, &node->refs);
 
-		binder_debug(BINDER_DEBUG_INTERNAL_REFS,
-			     "%d new ref %d desc %d for node %d\n",
-			      proc->pid, new_ref->debug_id, new_ref->desc,
-			      node->debug_id);
-	} else {
-		binder_debug(BINDER_DEBUG_INTERNAL_REFS,
-			     "%d new ref %d desc %d for dead node\n",
-			      proc->pid, new_ref->debug_id, new_ref->desc);
-	}
+	binder_node_lock(node);
+	hlist_add_head(&new_ref->node_entry, &node->refs);
+
+	binder_debug(BINDER_DEBUG_INTERNAL_REFS,
+		     "%d new ref %d desc %d for node %d\n",
+		      proc->pid, new_ref->data.debug_id, new_ref->data.desc,
+		      node->debug_id);
+	binder_node_unlock(node);
 	return new_ref;
 }
 
-static void binder_delete_ref(struct binder_ref *ref)
+static void binder_cleanup_ref_olocked(struct binder_ref *ref)
 {
+	bool delete_node = false;
+
 	binder_debug(BINDER_DEBUG_INTERNAL_REFS,
 		     "%d delete ref %d desc %d for node %d\n",
-		      ref->proc->pid, ref->debug_id, ref->desc,
+		      ref->proc->pid, ref->data.debug_id, ref->data.desc,
 		      ref->node->debug_id);
 
 	rb_erase(&ref->rb_node_desc, &ref->proc->refs_by_desc);
 	rb_erase(&ref->rb_node_node, &ref->proc->refs_by_node);
-	if (ref->strong)
-		binder_dec_node(ref->node, 1, 1);
+
+	binder_node_inner_lock(ref->node);
+	if (ref->data.strong)
+		binder_dec_node_nilocked(ref->node, 1, 1);
+
 	hlist_del(&ref->node_entry);
-	binder_dec_node(ref->node, 0, 1);
+	delete_node = binder_dec_node_nilocked(ref->node, 0, 1);
+	binder_node_inner_unlock(ref->node);
+	/*
+	 * Clear ref->node unless we want the caller to free the node
+	 */
+	if (!delete_node) {
+		/*
+		 * The caller uses ref->node to determine
+		 * whether the node needs to be freed. Clear
+		 * it since the node is still alive.
+		 */
+		ref->node = NULL;
+	}
+
 	if (ref->death) {
 		binder_debug(BINDER_DEBUG_DEAD_BINDER,
 			     "%d delete ref %d desc %d has death notification\n",
-			      ref->proc->pid, ref->debug_id, ref->desc);
-		list_del(&ref->death->work.entry);
-		kfree(ref->death);
+			      ref->proc->pid, ref->data.debug_id,
+			      ref->data.desc);
+		binder_dequeue_work(ref->proc, &ref->death->work);
 		binder_stats_deleted(BINDER_STAT_DEATH);
 	}
-	kfree(ref);
 	binder_stats_deleted(BINDER_STAT_REF);
 }
 
-static int binder_inc_ref(struct binder_ref *ref, int strong,
-			  struct list_head *target_list)
+/**
+ * binder_inc_ref_olocked() - increment the ref for given handle
+ * @ref:         ref to be incremented
+ * @strong:      if true, strong increment, else weak
+ * @target_list: list to queue node work on
+ *
+ * Increment the ref. @ref->proc->outer_lock must be held on entry
+ *
+ * Return: 0, if successful, else errno
+ */
+static int binder_inc_ref_olocked(struct binder_ref *ref, int strong,
+				  struct list_head *target_list)
 {
 	int ret;
 
 	if (strong) {
-		if (ref->strong == 0) {
+		if (ref->data.strong == 0) {
 			ret = binder_inc_node(ref->node, 1, 1, target_list);
 			if (ret)
 				return ret;
 		}
-		ref->strong++;
+		ref->data.strong++;
 	} else {
-		if (ref->weak == 0) {
+		if (ref->data.weak == 0) {
 			ret = binder_inc_node(ref->node, 0, 1, target_list);
 			if (ret)
 				return ret;
 		}
-		ref->weak++;
+		ref->data.weak++;
 	}
 	return 0;
 }
 
-
-static int binder_dec_ref(struct binder_ref *ref, int strong)
+/**
+ * binder_dec_ref() - dec the ref for given handle
+ * @ref:	ref to be decremented
+ * @strong:	if true, strong decrement, else weak
+ *
+ * Decrement the ref.
+ *
+ * Return: true if ref is cleaned up and ready to be freed
+ */
+static bool binder_dec_ref_olocked(struct binder_ref *ref, int strong)
 {
 	if (strong) {
-		if (ref->strong == 0) {
+		if (ref->data.strong == 0) {
 			binder_user_error("%d invalid dec strong, ref %d desc %d s %d w %d\n",
-					  ref->proc->pid, ref->debug_id,
-					  ref->desc, ref->strong, ref->weak);
-			return -EINVAL;
-		}
-		ref->strong--;
-		if (ref->strong == 0) {
-			int ret;
-
-			ret = binder_dec_node(ref->node, strong, 1);
-			if (ret)
-				return ret;
+					  ref->proc->pid, ref->data.debug_id,
+					  ref->data.desc, ref->data.strong,
+					  ref->data.weak);
+			return false;
 		}
+		ref->data.strong--;
+		if (ref->data.strong == 0)
+			binder_dec_node(ref->node, strong, 1);
 	} else {
-		if (ref->weak == 0) {
+		if (ref->data.weak == 0) {
 			binder_user_error("%d invalid dec weak, ref %d desc %d s %d w %d\n",
-					  ref->proc->pid, ref->debug_id,
-					  ref->desc, ref->strong, ref->weak);
-			return -EINVAL;
+					  ref->proc->pid, ref->data.debug_id,
+					  ref->data.desc, ref->data.strong,
+					  ref->data.weak);
+			return false;
 		}
-		ref->weak--;
+		ref->data.weak--;
 	}
-	if (ref->strong == 0 && ref->weak == 0)
-		binder_delete_ref(ref);
-	return 0;
+	if (ref->data.strong == 0 && ref->data.weak == 0) {
+		binder_cleanup_ref_olocked(ref);
+		return true;
+	}
+	return false;
 }
 
-static void binder_pop_transaction(struct binder_thread *target_thread,
-				   struct binder_transaction *t)
+/**
+ * binder_get_node_from_ref() - get the node from the given proc/desc
+ * @proc:	proc containing the ref
+ * @desc:	the handle associated with the ref
+ * @need_strong_ref: if true, only return node if ref is strong
+ * @rdata:	the id/refcount data for the ref
+ *
+ * Given a proc and ref handle, return the associated binder_node
+ *
+ * Return: a binder_node or NULL if not found or not strong when strong required
+ */
+static struct binder_node *binder_get_node_from_ref(
+		struct binder_proc *proc,
+		u32 desc, bool need_strong_ref,
+		struct binder_ref_data *rdata)
 {
-	if (target_thread) {
-		BUG_ON(target_thread->transaction_stack != t);
-		BUG_ON(target_thread->transaction_stack->from != target_thread);
-		target_thread->transaction_stack =
-			target_thread->transaction_stack->from_parent;
-		t->from = NULL;
+	struct binder_node *node;
+	struct binder_ref *ref;
+
+	binder_proc_lock(proc);
+	ref = binder_get_ref_olocked(proc, desc, need_strong_ref);
+	if (!ref)
+		goto err_no_ref;
+	node = ref->node;
+	/*
+	 * Take an implicit reference on the node to ensure
+	 * it stays alive until the call to binder_put_node()
+	 */
+	binder_inc_node_tmpref(node);
+	if (rdata)
+		*rdata = ref->data;
+	binder_proc_unlock(proc);
+
+	return node;
+
+err_no_ref:
+	binder_proc_unlock(proc);
+	return NULL;
+}
+
+/**
+ * binder_free_ref() - free the binder_ref
+ * @ref:	ref to free
+ *
+ * Free the binder_ref. Free the binder_node indicated by ref->node
+ * (if non-NULL) and the binder_ref_death indicated by ref->death.
+ */
+static void binder_free_ref(struct binder_ref *ref)
+{
+	if (ref->node)
+		binder_free_node(ref->node);
+	kfree(ref->death);
+	kfree(ref);
+}
+
+/**
+ * binder_update_ref_for_handle() - inc/dec the ref for given handle
+ * @proc:	proc containing the ref
+ * @desc:	the handle associated with the ref
+ * @increment:	true=inc reference, false=dec reference
+ * @strong:	true=strong reference, false=weak reference
+ * @rdata:	the id/refcount data for the ref
+ *
+ * Given a proc and ref handle, increment or decrement the ref
+ * according to "increment" arg.
+ *
+ * Return: 0 if successful, else errno
+ */
+static int binder_update_ref_for_handle(struct binder_proc *proc,
+		uint32_t desc, bool increment, bool strong,
+		struct binder_ref_data *rdata)
+{
+	int ret = 0;
+	struct binder_ref *ref;
+	bool delete_ref = false;
+
+	binder_proc_lock(proc);
+	ref = binder_get_ref_olocked(proc, desc, strong);
+	if (!ref) {
+		ret = -EINVAL;
+		goto err_no_ref;
+	}
+	if (increment)
+		ret = binder_inc_ref_olocked(ref, strong, NULL);
+	else
+		delete_ref = binder_dec_ref_olocked(ref, strong);
+
+	if (rdata)
+		*rdata = ref->data;
+	binder_proc_unlock(proc);
+
+	if (delete_ref)
+		binder_free_ref(ref);
+	return ret;
+
+err_no_ref:
+	binder_proc_unlock(proc);
+	return ret;
+}
+
+/**
+ * binder_dec_ref_for_handle() - dec the ref for given handle
+ * @proc:	proc containing the ref
+ * @desc:	the handle associated with the ref
+ * @strong:	true=strong reference, false=weak reference
+ * @rdata:	the id/refcount data for the ref
+ *
+ * Just calls binder_update_ref_for_handle() to decrement the ref.
+ *
+ * Return: 0 if successful, else errno
+ */
+static int binder_dec_ref_for_handle(struct binder_proc *proc,
+		uint32_t desc, bool strong, struct binder_ref_data *rdata)
+{
+	return binder_update_ref_for_handle(proc, desc, false, strong, rdata);
+}
+
+
+/**
+ * binder_inc_ref_for_node() - increment the ref for given proc/node
+ * @proc:	 proc containing the ref
+ * @node:	 target node
+ * @strong:	 true=strong reference, false=weak reference
+ * @target_list: worklist to use if node is incremented
+ * @rdata:	 the id/refcount data for the ref
+ *
+ * Given a proc and node, increment the ref. Create the ref if it
+ * doesn't already exist
+ *
+ * Return: 0 if successful, else errno
+ */
+static int binder_inc_ref_for_node(struct binder_proc *proc,
+			struct binder_node *node,
+			bool strong,
+			struct list_head *target_list,
+			struct binder_ref_data *rdata)
+{
+	struct binder_ref *ref;
+	struct binder_ref *new_ref = NULL;
+	int ret = 0;
+
+	binder_proc_lock(proc);
+	ref = binder_get_ref_for_node_olocked(proc, node, NULL);
+	if (!ref) {
+		binder_proc_unlock(proc);
+		new_ref = kzalloc(sizeof(*ref), GFP_KERNEL);
+		if (!new_ref)
+			return -ENOMEM;
+		binder_proc_lock(proc);
+		ref = binder_get_ref_for_node_olocked(proc, node, new_ref);
+	}
+	ret = binder_inc_ref_olocked(ref, strong, target_list);
+	*rdata = ref->data;
+	binder_proc_unlock(proc);
+	if (new_ref && ref != new_ref)
+		/*
+		 * Another thread created the ref first so
+		 * free the one we allocated
+		 */
+		kfree(new_ref);
+	return ret;
+}
+
+static void binder_pop_transaction_ilocked(struct binder_thread *target_thread,
+					   struct binder_transaction *t)
+{
+	BUG_ON(!target_thread);
+	assert_spin_locked(&target_thread->proc->inner_lock);
+	BUG_ON(target_thread->transaction_stack != t);
+	BUG_ON(target_thread->transaction_stack->from != target_thread);
+	target_thread->transaction_stack =
+		target_thread->transaction_stack->from_parent;
+	t->from = NULL;
+}
+
+/**
+ * binder_thread_dec_tmpref() - decrement thread->tmp_ref
+ * @thread:	thread to decrement
+ *
+ * A thread needs to be kept alive while being used to create or
+ * handle a transaction. binder_get_txn_from() is used to safely
+ * extract t->from from a binder_transaction and keep the thread
+ * indicated by t->from from being freed. When done with that
+ * binder_thread, this function is called to decrement the
+ * tmp_ref and free if appropriate (thread has been released
+ * and no transaction being processed by the driver)
+ */
+static void binder_thread_dec_tmpref(struct binder_thread *thread)
+{
+	/*
+	 * atomic is used to protect the counter value while
+	 * it cannot reach zero or thread->is_dead is false
+	 */
+	binder_inner_proc_lock(thread->proc);
+	atomic_dec(&thread->tmp_ref);
+	if (thread->is_dead && !atomic_read(&thread->tmp_ref)) {
+		binder_inner_proc_unlock(thread->proc);
+		binder_free_thread(thread);
+		return;
+	}
+	binder_inner_proc_unlock(thread->proc);
+}
+
+/**
+ * binder_proc_dec_tmpref() - decrement proc->tmp_ref
+ * @proc:	proc to decrement
+ *
+ * A binder_proc needs to be kept alive while being used to create or
+ * handle a transaction. proc->tmp_ref is incremented when
+ * creating a new transaction or the binder_proc is currently in-use
+ * by threads that are being released. When done with the binder_proc,
+ * this function is called to decrement the counter and free the
+ * proc if appropriate (proc has been released, all threads have
+ * been released and not currenly in-use to process a transaction).
+ */
+static void binder_proc_dec_tmpref(struct binder_proc *proc)
+{
+	binder_inner_proc_lock(proc);
+	proc->tmp_ref--;
+	if (proc->is_dead && RB_EMPTY_ROOT(&proc->threads) &&
+			!proc->tmp_ref) {
+		binder_inner_proc_unlock(proc);
+		binder_free_proc(proc);
+		return;
+	}
+	binder_inner_proc_unlock(proc);
+}
+
+/**
+ * binder_get_txn_from() - safely extract the "from" thread in transaction
+ * @t:	binder transaction for t->from
+ *
+ * Atomically return the "from" thread and increment the tmp_ref
+ * count for the thread to ensure it stays alive until
+ * binder_thread_dec_tmpref() is called.
+ *
+ * Return: the value of t->from
+ */
+static struct binder_thread *binder_get_txn_from(
+		struct binder_transaction *t)
+{
+	struct binder_thread *from;
+
+	spin_lock(&t->lock);
+	from = t->from;
+	if (from)
+		atomic_inc(&from->tmp_ref);
+	spin_unlock(&t->lock);
+	return from;
+}
+
+/**
+ * binder_get_txn_from_and_acq_inner() - get t->from and acquire inner lock
+ * @t:	binder transaction for t->from
+ *
+ * Same as binder_get_txn_from() except it also acquires the proc->inner_lock
+ * to guarantee that the thread cannot be released while operating on it.
+ * The caller must call binder_inner_proc_unlock() to release the inner lock
+ * as well as call binder_dec_thread_txn() to release the reference.
+ *
+ * Return: the value of t->from
+ */
+static struct binder_thread *binder_get_txn_from_and_acq_inner(
+		struct binder_transaction *t)
+{
+	struct binder_thread *from;
+
+	from = binder_get_txn_from(t);
+	if (!from)
+		return NULL;
+	binder_inner_proc_lock(from->proc);
+	if (t->from) {
+		BUG_ON(from != t->from);
+		return from;
 	}
-	t->need_reply = 0;
+	binder_inner_proc_unlock(from->proc);
+	binder_thread_dec_tmpref(from);
+	return NULL;
+}
+
+static void binder_free_transaction(struct binder_transaction *t)
+{
 	if (t->buffer)
 		t->buffer->transaction = NULL;
 	kfree(t);
@@ -1235,30 +1904,28 @@ static void binder_send_failed_reply(struct binder_transaction *t,
 
 	BUG_ON(t->flags & TF_ONE_WAY);
 	while (1) {
-		target_thread = t->from;
+		target_thread = binder_get_txn_from_and_acq_inner(t);
 		if (target_thread) {
-			if (target_thread->return_error != BR_OK &&
-			   target_thread->return_error2 == BR_OK) {
-				target_thread->return_error2 =
-					target_thread->return_error;
-				target_thread->return_error = BR_OK;
-			}
-			if (target_thread->return_error == BR_OK) {
-				binder_debug(BINDER_DEBUG_FAILED_TRANSACTION,
-					     "send failed reply for transaction %d to %d:%d\n",
-					      t->debug_id,
-					      target_thread->proc->pid,
-					      target_thread->pid);
-
-				binder_pop_transaction(target_thread, t);
-				target_thread->return_error = error_code;
+			binder_debug(BINDER_DEBUG_FAILED_TRANSACTION,
+				     "send failed reply for transaction %d to %d:%d\n",
+				      t->debug_id,
+				      target_thread->proc->pid,
+				      target_thread->pid);
+
+			binder_pop_transaction_ilocked(target_thread, t);
+			if (target_thread->reply_error.cmd == BR_OK) {
+				target_thread->reply_error.cmd = error_code;
+				binder_enqueue_work_ilocked(
+					&target_thread->reply_error.work,
+					&target_thread->todo);
 				wake_up_interruptible(&target_thread->wait);
 			} else {
-				pr_err("reply failed, target thread, %d:%d, has error code %d already\n",
-					target_thread->proc->pid,
-					target_thread->pid,
-					target_thread->return_error);
+				WARN(1, "Unexpected reply error: %u\n",
+						target_thread->reply_error.cmd);
 			}
+			binder_inner_proc_unlock(target_thread->proc);
+			binder_thread_dec_tmpref(target_thread);
+			binder_free_transaction(t);
 			return;
 		}
 		next = t->from_parent;
@@ -1267,7 +1934,7 @@ static void binder_send_failed_reply(struct binder_transaction *t,
 			     "send failed reply for transaction %d, target dead\n",
 			     t->debug_id);
 
-		binder_pop_transaction(target_thread, t);
+		binder_free_transaction(t);
 		if (next == NULL) {
 			binder_debug(BINDER_DEBUG_DEAD_BINDER,
 				     "reply failed, no target thread at root\n");
@@ -1476,24 +2143,26 @@ static void binder_transaction_buffer_release(struct binder_proc *proc,
 				     node->debug_id, (u64)node->ptr);
 			binder_dec_node(node, hdr->type == BINDER_TYPE_BINDER,
 					0);
+			binder_put_node(node);
 		} break;
 		case BINDER_TYPE_HANDLE:
 		case BINDER_TYPE_WEAK_HANDLE: {
 			struct flat_binder_object *fp;
-			struct binder_ref *ref;
+			struct binder_ref_data rdata;
+			int ret;
 
 			fp = to_flat_binder_object(hdr);
-			ref = binder_get_ref(proc, fp->handle,
-					     hdr->type == BINDER_TYPE_HANDLE);
-			if (ref == NULL) {
-				pr_err("transaction release %d bad handle %d\n",
-				 debug_id, fp->handle);
+			ret = binder_dec_ref_for_handle(proc, fp->handle,
+				hdr->type == BINDER_TYPE_HANDLE, &rdata);
+
+			if (ret) {
+				pr_err("transaction release %d bad handle %d, ret = %d\n",
+				 debug_id, fp->handle, ret);
 				break;
 			}
 			binder_debug(BINDER_DEBUG_TRANSACTION,
-				     "        ref %d desc %d (node %d)\n",
-				     ref->debug_id, ref->desc, ref->node->debug_id);
-			binder_dec_ref(ref, hdr->type == BINDER_TYPE_HANDLE);
+				     "        ref %d desc %d\n",
+				     rdata.debug_id, rdata.desc);
 		} break;
 
 		case BINDER_TYPE_FD: {
@@ -1532,7 +2201,8 @@ static void binder_transaction_buffer_release(struct binder_proc *proc,
 			 * back to kernel address space to access it
 			 */
 			parent_buffer = parent->buffer -
-				proc->user_buffer_offset;
+				binder_alloc_get_user_buffer_offset(
+						&proc->alloc);
 
 			fd_buf_size = sizeof(u32) * fda->num_fds;
 			if (fda->num_fds >= SIZE_MAX / sizeof(u32)) {
@@ -1564,102 +2234,122 @@ static int binder_translate_binder(struct flat_binder_object *fp,
 				   struct binder_thread *thread)
 {
 	struct binder_node *node;
-	struct binder_ref *ref;
 	struct binder_proc *proc = thread->proc;
 	struct binder_proc *target_proc = t->to_proc;
+	struct binder_ref_data rdata;
+	int ret = 0;
 
 	node = binder_get_node(proc, fp->binder);
 	if (!node) {
-		node = binder_new_node(proc, fp->binder, fp->cookie);
+		node = binder_new_node(proc, fp);
 		if (!node)
 			return -ENOMEM;
-
-		node->min_priority = fp->flags & FLAT_BINDER_FLAG_PRIORITY_MASK;
-		node->accept_fds = !!(fp->flags & FLAT_BINDER_FLAG_ACCEPTS_FDS);
 	}
 	if (fp->cookie != node->cookie) {
 		binder_user_error("%d:%d sending u%016llx node %d, cookie mismatch %016llx != %016llx\n",
 				  proc->pid, thread->pid, (u64)fp->binder,
 				  node->debug_id, (u64)fp->cookie,
 				  (u64)node->cookie);
-		return -EINVAL;
+		ret = -EINVAL;
+		goto done;
+	}
+	if (security_binder_transfer_binder(proc->tsk, target_proc->tsk)) {
+		ret = -EPERM;
+		goto done;
 	}
-	if (security_binder_transfer_binder(proc->tsk, target_proc->tsk))
-		return -EPERM;
 
-	ref = binder_get_ref_for_node(target_proc, node);
-	if (!ref)
-		return -EINVAL;
+	ret = binder_inc_ref_for_node(target_proc, node,
+			fp->hdr.type == BINDER_TYPE_BINDER,
+			&thread->todo, &rdata);
+	if (ret)
+		goto done;
 
 	if (fp->hdr.type == BINDER_TYPE_BINDER)
 		fp->hdr.type = BINDER_TYPE_HANDLE;
 	else
 		fp->hdr.type = BINDER_TYPE_WEAK_HANDLE;
 	fp->binder = 0;
-	fp->handle = ref->desc;
+	fp->handle = rdata.desc;
 	fp->cookie = 0;
-	binder_inc_ref(ref, fp->hdr.type == BINDER_TYPE_HANDLE, &thread->todo);
 
-	trace_binder_transaction_node_to_ref(t, node, ref);
+	trace_binder_transaction_node_to_ref(t, node, &rdata);
 	binder_debug(BINDER_DEBUG_TRANSACTION,
 		     "        node %d u%016llx -> ref %d desc %d\n",
 		     node->debug_id, (u64)node->ptr,
-		     ref->debug_id, ref->desc);
-
-	return 0;
+		     rdata.debug_id, rdata.desc);
+done:
+	binder_put_node(node);
+	return ret;
 }
 
 static int binder_translate_handle(struct flat_binder_object *fp,
 				   struct binder_transaction *t,
 				   struct binder_thread *thread)
 {
-	struct binder_ref *ref;
 	struct binder_proc *proc = thread->proc;
 	struct binder_proc *target_proc = t->to_proc;
+	struct binder_node *node;
+	struct binder_ref_data src_rdata;
+	int ret = 0;
 
-	ref = binder_get_ref(proc, fp->handle,
-			     fp->hdr.type == BINDER_TYPE_HANDLE);
-	if (!ref) {
+	node = binder_get_node_from_ref(proc, fp->handle,
+			fp->hdr.type == BINDER_TYPE_HANDLE, &src_rdata);
+	if (!node) {
 		binder_user_error("%d:%d got transaction with invalid handle, %d\n",
 				  proc->pid, thread->pid, fp->handle);
 		return -EINVAL;
 	}
-	if (security_binder_transfer_binder(proc->tsk, target_proc->tsk))
-		return -EPERM;
+	if (security_binder_transfer_binder(proc->tsk, target_proc->tsk)) {
+		ret = -EPERM;
+		goto done;
+	}
 
-	if (ref->node->proc == target_proc) {
+	binder_node_lock(node);
+	if (node->proc == target_proc) {
 		if (fp->hdr.type == BINDER_TYPE_HANDLE)
 			fp->hdr.type = BINDER_TYPE_BINDER;
 		else
 			fp->hdr.type = BINDER_TYPE_WEAK_BINDER;
-		fp->binder = ref->node->ptr;
-		fp->cookie = ref->node->cookie;
-		binder_inc_node(ref->node, fp->hdr.type == BINDER_TYPE_BINDER,
-				0, NULL);
-		trace_binder_transaction_ref_to_node(t, ref);
+		fp->binder = node->ptr;
+		fp->cookie = node->cookie;
+		if (node->proc)
+			binder_inner_proc_lock(node->proc);
+		binder_inc_node_nilocked(node,
+					 fp->hdr.type == BINDER_TYPE_BINDER,
+					 0, NULL);
+		if (node->proc)
+			binder_inner_proc_unlock(node->proc);
+		trace_binder_transaction_ref_to_node(t, node, &src_rdata);
 		binder_debug(BINDER_DEBUG_TRANSACTION,
 			     "        ref %d desc %d -> node %d u%016llx\n",
-			     ref->debug_id, ref->desc, ref->node->debug_id,
-			     (u64)ref->node->ptr);
+			     src_rdata.debug_id, src_rdata.desc, node->debug_id,
+			     (u64)node->ptr);
+		binder_node_unlock(node);
 	} else {
-		struct binder_ref *new_ref;
+		int ret;
+		struct binder_ref_data dest_rdata;
 
-		new_ref = binder_get_ref_for_node(target_proc, ref->node);
-		if (!new_ref)
-			return -EINVAL;
+		binder_node_unlock(node);
+		ret = binder_inc_ref_for_node(target_proc, node,
+				fp->hdr.type == BINDER_TYPE_HANDLE,
+				NULL, &dest_rdata);
+		if (ret)
+			goto done;
 
 		fp->binder = 0;
-		fp->handle = new_ref->desc;
+		fp->handle = dest_rdata.desc;
 		fp->cookie = 0;
-		binder_inc_ref(new_ref, fp->hdr.type == BINDER_TYPE_HANDLE,
-			       NULL);
-		trace_binder_transaction_ref_to_ref(t, ref, new_ref);
+		trace_binder_transaction_ref_to_ref(t, node, &src_rdata,
+						    &dest_rdata);
 		binder_debug(BINDER_DEBUG_TRANSACTION,
 			     "        ref %d desc %d -> ref %d desc %d (node %d)\n",
-			     ref->debug_id, ref->desc, new_ref->debug_id,
-			     new_ref->desc, ref->node->debug_id);
+			     src_rdata.debug_id, src_rdata.desc,
+			     dest_rdata.debug_id, dest_rdata.desc,
+			     node->debug_id);
 	}
-	return 0;
+done:
+	binder_put_node(node);
+	return ret;
 }
 
 static int binder_translate_fd(int fd,
@@ -1750,7 +2440,8 @@ static int binder_translate_fd_array(struct binder_fd_array_object *fda,
 	 * Since the parent was already fixed up, convert it
 	 * back to the kernel address space to access it
 	 */
-	parent_buffer = parent->buffer - target_proc->user_buffer_offset;
+	parent_buffer = parent->buffer -
+		binder_alloc_get_user_buffer_offset(&target_proc->alloc);
 	fd_array = (u32 *)(parent_buffer + fda->parent_offset);
 	if (!IS_ALIGNED((unsigned long)fd_array, sizeof(u32))) {
 		binder_user_error("%d:%d parent offset not aligned correctly.\n",
@@ -1818,12 +2509,80 @@ static int binder_fixup_parent(struct binder_transaction *t,
 		return -EINVAL;
 	}
 	parent_buffer = (u8 *)(parent->buffer -
-			       target_proc->user_buffer_offset);
+			binder_alloc_get_user_buffer_offset(
+				&target_proc->alloc));
 	*(binder_uintptr_t *)(parent_buffer + bp->parent_offset) = bp->buffer;
 
 	return 0;
 }
 
+/**
+ * binder_proc_transaction() - sends a transaction to a process and wakes it up
+ * @t:		transaction to send
+ * @proc:	process to send the transaction to
+ * @thread:	thread in @proc to send the transaction to (may be NULL)
+ *
+ * This function queues a transaction to the specified process. It will try
+ * to find a thread in the target process to handle the transaction and
+ * wake it up. If no thread is found, the work is queued to the proc
+ * waitqueue.
+ *
+ * If the @thread parameter is not NULL, the transaction is always queued
+ * to the waitlist of that specific thread.
+ *
+ * Return:	true if the transactions was successfully queued
+ *		false if the target process or thread is dead
+ */
+static bool binder_proc_transaction(struct binder_transaction *t,
+				    struct binder_proc *proc,
+				    struct binder_thread *thread)
+{
+	struct list_head *target_list = NULL;
+	struct binder_node *node = t->buffer->target_node;
+	bool oneway = !!(t->flags & TF_ONE_WAY);
+	bool wakeup = true;
+
+	BUG_ON(!node);
+	binder_node_lock(node);
+	if (oneway) {
+		BUG_ON(thread);
+		if (node->has_async_transaction) {
+			target_list = &node->async_todo;
+			wakeup = false;
+		} else {
+			node->has_async_transaction = 1;
+		}
+	}
+
+	binder_inner_proc_lock(proc);
+
+	if (proc->is_dead || (thread && thread->is_dead)) {
+		binder_inner_proc_unlock(proc);
+		binder_node_unlock(node);
+		return false;
+	}
+
+	if (!thread && !target_list)
+		thread = binder_select_thread_ilocked(proc);
+
+	if (thread)
+		target_list = &thread->todo;
+	else if (!target_list)
+		target_list = &proc->todo;
+	else
+		BUG_ON(target_list != &node->async_todo);
+
+	binder_enqueue_work_ilocked(&t->work, target_list);
+
+	if (wakeup)
+		binder_wakeup_thread_ilocked(proc, thread, !oneway /* sync */);
+
+	binder_inner_proc_unlock(proc);
+	binder_node_unlock(node);
+
+	return true;
+}
+
 static void binder_transaction(struct binder_proc *proc,
 			       struct binder_thread *thread,
 			       struct binder_transaction_data *tr, int reply,
@@ -1835,19 +2594,21 @@ static void binder_transaction(struct binder_proc *proc,
 	binder_size_t *offp, *off_end, *off_start;
 	binder_size_t off_min;
 	u8 *sg_bufp, *sg_buf_end;
-	struct binder_proc *target_proc;
+	struct binder_proc *target_proc = NULL;
 	struct binder_thread *target_thread = NULL;
 	struct binder_node *target_node = NULL;
-	struct list_head *target_list;
-	wait_queue_head_t *target_wait;
 	struct binder_transaction *in_reply_to = NULL;
 	struct binder_transaction_log_entry *e;
-	uint32_t return_error;
+	uint32_t return_error = 0;
+	uint32_t return_error_param = 0;
+	uint32_t return_error_line = 0;
 	struct binder_buffer_object *last_fixup_obj = NULL;
 	binder_size_t last_fixup_min_off = 0;
 	struct binder_context *context = proc->context;
+	int t_debug_id = atomic_inc_return(&binder_last_id);
 
 	e = binder_transaction_log_add(&binder_transaction_log);
+	e->debug_id = t_debug_id;
 	e->call_type = reply ? 2 : !!(tr->flags & TF_ONE_WAY);
 	e->from_proc = proc->pid;
 	e->from_thread = thread->pid;
@@ -1857,29 +2618,40 @@ static void binder_transaction(struct binder_proc *proc,
 	e->context_name = proc->context->name;
 
 	if (reply) {
+		binder_inner_proc_lock(proc);
 		in_reply_to = thread->transaction_stack;
 		if (in_reply_to == NULL) {
+			binder_inner_proc_unlock(proc);
 			binder_user_error("%d:%d got reply transaction with no transaction stack\n",
 					  proc->pid, thread->pid);
 			return_error = BR_FAILED_REPLY;
+			return_error_param = -EPROTO;
+			return_error_line = __LINE__;
 			goto err_empty_call_stack;
 		}
-		binder_set_nice(in_reply_to->saved_priority);
 		if (in_reply_to->to_thread != thread) {
+			spin_lock(&in_reply_to->lock);
 			binder_user_error("%d:%d got reply transaction with bad transaction stack, transaction %d has target %d:%d\n",
 				proc->pid, thread->pid, in_reply_to->debug_id,
 				in_reply_to->to_proc ?
 				in_reply_to->to_proc->pid : 0,
 				in_reply_to->to_thread ?
 				in_reply_to->to_thread->pid : 0);
+			spin_unlock(&in_reply_to->lock);
+			binder_inner_proc_unlock(proc);
 			return_error = BR_FAILED_REPLY;
+			return_error_param = -EPROTO;
+			return_error_line = __LINE__;
 			in_reply_to = NULL;
 			goto err_bad_call_stack;
 		}
 		thread->transaction_stack = in_reply_to->to_parent;
-		target_thread = in_reply_to->from;
+		binder_inner_proc_unlock(proc);
+		binder_set_nice(in_reply_to->saved_priority);
+		target_thread = binder_get_txn_from_and_acq_inner(in_reply_to);
 		if (target_thread == NULL) {
 			return_error = BR_DEAD_REPLY;
+			return_error_line = __LINE__;
 			goto err_dead_binder;
 		}
 		if (target_thread->transaction_stack != in_reply_to) {
@@ -1888,89 +2660,137 @@ static void binder_transaction(struct binder_proc *proc,
 				target_thread->transaction_stack ?
 				target_thread->transaction_stack->debug_id : 0,
 				in_reply_to->debug_id);
+			binder_inner_proc_unlock(target_thread->proc);
 			return_error = BR_FAILED_REPLY;
+			return_error_param = -EPROTO;
+			return_error_line = __LINE__;
 			in_reply_to = NULL;
 			target_thread = NULL;
 			goto err_dead_binder;
 		}
 		target_proc = target_thread->proc;
+		target_proc->tmp_ref++;
+		binder_inner_proc_unlock(target_thread->proc);
 	} else {
 		if (tr->target.handle) {
 			struct binder_ref *ref;
 
-			ref = binder_get_ref(proc, tr->target.handle, true);
-			if (ref == NULL) {
+			/*
+			 * There must already be a strong ref
+			 * on this node. If so, do a strong
+			 * increment on the node to ensure it
+			 * stays alive until the transaction is
+			 * done.
+			 */
+			binder_proc_lock(proc);
+			ref = binder_get_ref_olocked(proc, tr->target.handle,
+						     true);
+			if (ref) {
+				binder_inc_node(ref->node, 1, 0, NULL);
+				target_node = ref->node;
+			}
+			binder_proc_unlock(proc);
+			if (target_node == NULL) {
 				binder_user_error("%d:%d got transaction to invalid handle\n",
 					proc->pid, thread->pid);
 				return_error = BR_FAILED_REPLY;
+				return_error_param = -EINVAL;
+				return_error_line = __LINE__;
 				goto err_invalid_target_handle;
 			}
-			target_node = ref->node;
 		} else {
+			mutex_lock(&context->context_mgr_node_lock);
 			target_node = context->binder_context_mgr_node;
 			if (target_node == NULL) {
 				return_error = BR_DEAD_REPLY;
+				mutex_unlock(&context->context_mgr_node_lock);
+				return_error_line = __LINE__;
 				goto err_no_context_mgr_node;
 			}
+			binder_inc_node(target_node, 1, 0, NULL);
+			mutex_unlock(&context->context_mgr_node_lock);
 		}
 		e->to_node = target_node->debug_id;
+		binder_node_lock(target_node);
 		target_proc = target_node->proc;
 		if (target_proc == NULL) {
+			binder_node_unlock(target_node);
 			return_error = BR_DEAD_REPLY;
+			return_error_line = __LINE__;
 			goto err_dead_binder;
 		}
+		binder_inner_proc_lock(target_proc);
+		target_proc->tmp_ref++;
+		binder_inner_proc_unlock(target_proc);
+		binder_node_unlock(target_node);
 		if (security_binder_transaction(proc->tsk,
 						target_proc->tsk) < 0) {
 			return_error = BR_FAILED_REPLY;
+			return_error_param = -EPERM;
+			return_error_line = __LINE__;
 			goto err_invalid_target_handle;
 		}
+		binder_inner_proc_lock(proc);
 		if (!(tr->flags & TF_ONE_WAY) && thread->transaction_stack) {
 			struct binder_transaction *tmp;
 
 			tmp = thread->transaction_stack;
 			if (tmp->to_thread != thread) {
+				spin_lock(&tmp->lock);
 				binder_user_error("%d:%d got new transaction with bad transaction stack, transaction %d has target %d:%d\n",
 					proc->pid, thread->pid, tmp->debug_id,
 					tmp->to_proc ? tmp->to_proc->pid : 0,
 					tmp->to_thread ?
 					tmp->to_thread->pid : 0);
+				spin_unlock(&tmp->lock);
+				binder_inner_proc_unlock(proc);
 				return_error = BR_FAILED_REPLY;
+				return_error_param = -EPROTO;
+				return_error_line = __LINE__;
 				goto err_bad_call_stack;
 			}
 			while (tmp) {
-				if (tmp->from && tmp->from->proc == target_proc)
-					target_thread = tmp->from;
+				struct binder_thread *from;
+
+				spin_lock(&tmp->lock);
+				from = tmp->from;
+				if (from && from->proc == target_proc) {
+					atomic_inc(&from->tmp_ref);
+					target_thread = from;
+					spin_unlock(&tmp->lock);
+					break;
+				}
+				spin_unlock(&tmp->lock);
 				tmp = tmp->from_parent;
 			}
 		}
+		binder_inner_proc_unlock(proc);
 	}
-	if (target_thread) {
+	if (target_thread)
 		e->to_thread = target_thread->pid;
-		target_list = &target_thread->todo;
-		target_wait = &target_thread->wait;
-	} else {
-		target_list = &target_proc->todo;
-		target_wait = &target_proc->wait;
-	}
 	e->to_proc = target_proc->pid;
 
 	/* TODO: reuse incoming transaction for reply */
 	t = kzalloc(sizeof(*t), GFP_KERNEL);
 	if (t == NULL) {
 		return_error = BR_FAILED_REPLY;
+		return_error_param = -ENOMEM;
+		return_error_line = __LINE__;
 		goto err_alloc_t_failed;
 	}
 	binder_stats_created(BINDER_STAT_TRANSACTION);
+	spin_lock_init(&t->lock);
 
 	tcomplete = kzalloc(sizeof(*tcomplete), GFP_KERNEL);
 	if (tcomplete == NULL) {
 		return_error = BR_FAILED_REPLY;
+		return_error_param = -ENOMEM;
+		return_error_line = __LINE__;
 		goto err_alloc_tcomplete_failed;
 	}
 	binder_stats_created(BINDER_STAT_TRANSACTION_COMPLETE);
 
-	t->debug_id = ++binder_last_id;
-	e->debug_id = t->debug_id;
+	t->debug_id = t_debug_id;
 
 	if (reply)
 		binder_debug(BINDER_DEBUG_TRANSACTION,
@@ -2004,11 +2824,18 @@ static void binder_transaction(struct binder_proc *proc,
 
 	trace_binder_transaction(reply, t, target_node);
 
-	t->buffer = binder_alloc_buf(target_proc, tr->data_size,
+	t->buffer = binder_alloc_new_buf(&target_proc->alloc, tr->data_size,
 		tr->offsets_size, extra_buffers_size,
 		!reply && (t->flags & TF_ONE_WAY));
-	if (t->buffer == NULL) {
-		return_error = BR_FAILED_REPLY;
+	if (IS_ERR(t->buffer)) {
+		/*
+		 * -ESRCH indicates VMA cleared. The target is dying.
+		 */
+		return_error_param = PTR_ERR(t->buffer);
+		return_error = return_error_param == -ESRCH ?
+			BR_DEAD_REPLY : BR_FAILED_REPLY;
+		return_error_line = __LINE__;
+		t->buffer = NULL;
 		goto err_binder_alloc_buf_failed;
 	}
 	t->buffer->allow_user_free = 0;
@@ -2016,9 +2843,6 @@ static void binder_transaction(struct binder_proc *proc,
 	t->buffer->transaction = t;
 	t->buffer->target_node = target_node;
 	trace_binder_transaction_alloc_buf(t->buffer);
-	if (target_node)
-		binder_inc_node(target_node, 1, 0, NULL);
-
 	off_start = (binder_size_t *)(t->buffer->data +
 				      ALIGN(tr->data_size, sizeof(void *)));
 	offp = off_start;
@@ -2028,6 +2852,8 @@ static void binder_transaction(struct binder_proc *proc,
 		binder_user_error("%d:%d got transaction with invalid data ptr\n",
 				proc->pid, thread->pid);
 		return_error = BR_FAILED_REPLY;
+		return_error_param = -EFAULT;
+		return_error_line = __LINE__;
 		goto err_copy_data_failed;
 	}
 	if (copy_from_user(offp, (const void __user *)(uintptr_t)
@@ -2035,12 +2861,16 @@ static void binder_transaction(struct binder_proc *proc,
 		binder_user_error("%d:%d got transaction with invalid offsets ptr\n",
 				proc->pid, thread->pid);
 		return_error = BR_FAILED_REPLY;
+		return_error_param = -EFAULT;
+		return_error_line = __LINE__;
 		goto err_copy_data_failed;
 	}
 	if (!IS_ALIGNED(tr->offsets_size, sizeof(binder_size_t))) {
 		binder_user_error("%d:%d got transaction with invalid offsets size, %lld\n",
 				proc->pid, thread->pid, (u64)tr->offsets_size);
 		return_error = BR_FAILED_REPLY;
+		return_error_param = -EINVAL;
+		return_error_line = __LINE__;
 		goto err_bad_offset;
 	}
 	if (!IS_ALIGNED(extra_buffers_size, sizeof(u64))) {
@@ -2048,6 +2878,8 @@ static void binder_transaction(struct binder_proc *proc,
 				  proc->pid, thread->pid,
 				  (u64)extra_buffers_size);
 		return_error = BR_FAILED_REPLY;
+		return_error_param = -EINVAL;
+		return_error_line = __LINE__;
 		goto err_bad_offset;
 	}
 	off_end = (void *)off_start + tr->offsets_size;
@@ -2064,6 +2896,8 @@ static void binder_transaction(struct binder_proc *proc,
 					  (u64)off_min,
 					  (u64)t->buffer->data_size);
 			return_error = BR_FAILED_REPLY;
+			return_error_param = -EINVAL;
+			return_error_line = __LINE__;
 			goto err_bad_offset;
 		}
 
@@ -2078,6 +2912,8 @@ static void binder_transaction(struct binder_proc *proc,
 			ret = binder_translate_binder(fp, t, thread);
 			if (ret < 0) {
 				return_error = BR_FAILED_REPLY;
+				return_error_param = ret;
+				return_error_line = __LINE__;
 				goto err_translate_failed;
 			}
 		} break;
@@ -2089,6 +2925,8 @@ static void binder_transaction(struct binder_proc *proc,
 			ret = binder_translate_handle(fp, t, thread);
 			if (ret < 0) {
 				return_error = BR_FAILED_REPLY;
+				return_error_param = ret;
+				return_error_line = __LINE__;
 				goto err_translate_failed;
 			}
 		} break;
@@ -2100,6 +2938,8 @@ static void binder_transaction(struct binder_proc *proc,
 
 			if (target_fd < 0) {
 				return_error = BR_FAILED_REPLY;
+				return_error_param = target_fd;
+				return_error_line = __LINE__;
 				goto err_translate_failed;
 			}
 			fp->pad_binder = 0;
@@ -2116,6 +2956,8 @@ static void binder_transaction(struct binder_proc *proc,
 				binder_user_error("%d:%d got transaction with invalid parent offset or type\n",
 						  proc->pid, thread->pid);
 				return_error = BR_FAILED_REPLY;
+				return_error_param = -EINVAL;
+				return_error_line = __LINE__;
 				goto err_bad_parent;
 			}
 			if (!binder_validate_fixup(t->buffer, off_start,
@@ -2125,12 +2967,16 @@ static void binder_transaction(struct binder_proc *proc,
 				binder_user_error("%d:%d got transaction with out-of-order buffer fixup\n",
 						  proc->pid, thread->pid);
 				return_error = BR_FAILED_REPLY;
+				return_error_param = -EINVAL;
+				return_error_line = __LINE__;
 				goto err_bad_parent;
 			}
 			ret = binder_translate_fd_array(fda, parent, t, thread,
 							in_reply_to);
 			if (ret < 0) {
 				return_error = BR_FAILED_REPLY;
+				return_error_param = ret;
+				return_error_line = __LINE__;
 				goto err_translate_failed;
 			}
 			last_fixup_obj = parent;
@@ -2146,6 +2992,8 @@ static void binder_transaction(struct binder_proc *proc,
 				binder_user_error("%d:%d got transaction with too large buffer\n",
 						  proc->pid, thread->pid);
 				return_error = BR_FAILED_REPLY;
+				return_error_param = -EINVAL;
+				return_error_line = __LINE__;
 				goto err_bad_offset;
 			}
 			if (copy_from_user(sg_bufp,
@@ -2153,12 +3001,15 @@ static void binder_transaction(struct binder_proc *proc,
 					   bp->buffer, bp->length)) {
 				binder_user_error("%d:%d got transaction with invalid offsets ptr\n",
 						  proc->pid, thread->pid);
+				return_error_param = -EFAULT;
 				return_error = BR_FAILED_REPLY;
+				return_error_line = __LINE__;
 				goto err_copy_data_failed;
 			}
 			/* Fixup buffer pointer to target proc address space */
 			bp->buffer = (uintptr_t)sg_bufp +
-				target_proc->user_buffer_offset;
+				binder_alloc_get_user_buffer_offset(
+						&target_proc->alloc);
 			sg_bufp += ALIGN(bp->length, sizeof(u64));
 
 			ret = binder_fixup_parent(t, thread, bp, off_start,
@@ -2167,6 +3018,8 @@ static void binder_transaction(struct binder_proc *proc,
 						  last_fixup_min_off);
 			if (ret < 0) {
 				return_error = BR_FAILED_REPLY;
+				return_error_param = ret;
+				return_error_line = __LINE__;
 				goto err_translate_failed;
 			}
 			last_fixup_obj = bp;
@@ -2176,38 +3029,60 @@ static void binder_transaction(struct binder_proc *proc,
 			binder_user_error("%d:%d got transaction with invalid object type, %x\n",
 				proc->pid, thread->pid, hdr->type);
 			return_error = BR_FAILED_REPLY;
+			return_error_param = -EINVAL;
+			return_error_line = __LINE__;
 			goto err_bad_object_type;
 		}
 	}
+	tcomplete->type = BINDER_WORK_TRANSACTION_COMPLETE;
+	binder_enqueue_work(proc, tcomplete, &thread->todo);
+	t->work.type = BINDER_WORK_TRANSACTION;
+
 	if (reply) {
+		binder_inner_proc_lock(target_proc);
+		if (target_thread->is_dead) {
+			binder_inner_proc_unlock(target_proc);
+			goto err_dead_proc_or_thread;
+		}
 		BUG_ON(t->buffer->async_transaction != 0);
-		binder_pop_transaction(target_thread, in_reply_to);
+		binder_pop_transaction_ilocked(target_thread, in_reply_to);
+		binder_enqueue_work_ilocked(&t->work, &target_thread->todo);
+		binder_inner_proc_unlock(target_proc);
+		wake_up_interruptible_sync(&target_thread->wait);
+		binder_free_transaction(in_reply_to);
 	} else if (!(t->flags & TF_ONE_WAY)) {
 		BUG_ON(t->buffer->async_transaction != 0);
+		binder_inner_proc_lock(proc);
 		t->need_reply = 1;
 		t->from_parent = thread->transaction_stack;
 		thread->transaction_stack = t;
+		binder_inner_proc_unlock(proc);
+		if (!binder_proc_transaction(t, target_proc, target_thread)) {
+			binder_inner_proc_lock(proc);
+			binder_pop_transaction_ilocked(thread, t);
+			binder_inner_proc_unlock(proc);
+			goto err_dead_proc_or_thread;
+		}
 	} else {
 		BUG_ON(target_node == NULL);
 		BUG_ON(t->buffer->async_transaction != 1);
-		if (target_node->has_async_transaction) {
-			target_list = &target_node->async_todo;
-			target_wait = NULL;
-		} else
-			target_node->has_async_transaction = 1;
-	}
-	t->work.type = BINDER_WORK_TRANSACTION;
-	list_add_tail(&t->work.entry, target_list);
-	tcomplete->type = BINDER_WORK_TRANSACTION_COMPLETE;
-	list_add_tail(&tcomplete->entry, &thread->todo);
-	if (target_wait) {
-		if (reply || !(t->flags & TF_ONE_WAY))
-			wake_up_interruptible_sync(target_wait);
-		else
-			wake_up_interruptible(target_wait);
+		if (!binder_proc_transaction(t, target_proc, NULL))
+			goto err_dead_proc_or_thread;
 	}
+	if (target_thread)
+		binder_thread_dec_tmpref(target_thread);
+	binder_proc_dec_tmpref(target_proc);
+	/*
+	 * write barrier to synchronize with initialization
+	 * of log entry
+	 */
+	smp_wmb();
+	WRITE_ONCE(e->debug_id_done, t_debug_id);
 	return;
 
+err_dead_proc_or_thread:
+	return_error = BR_DEAD_REPLY;
+	return_error_line = __LINE__;
 err_translate_failed:
 err_bad_object_type:
 err_bad_offset:
@@ -2215,8 +3090,9 @@ err_bad_parent:
 err_copy_data_failed:
 	trace_binder_transaction_failed_buffer_release(t->buffer);
 	binder_transaction_buffer_release(target_proc, t->buffer, offp);
+	target_node = NULL;
 	t->buffer->transaction = NULL;
-	binder_free_buf(target_proc, t->buffer);
+	binder_alloc_free_buf(&target_proc->alloc, t->buffer);
 err_binder_alloc_buf_failed:
 	kfree(tcomplete);
 	binder_stats_deleted(BINDER_STAT_TRANSACTION_COMPLETE);
@@ -2229,24 +3105,49 @@ err_empty_call_stack:
 err_dead_binder:
 err_invalid_target_handle:
 err_no_context_mgr_node:
+	if (target_thread)
+		binder_thread_dec_tmpref(target_thread);
+	if (target_proc)
+		binder_proc_dec_tmpref(target_proc);
+	if (target_node)
+		binder_dec_node(target_node, 1, 0);
+
 	binder_debug(BINDER_DEBUG_FAILED_TRANSACTION,
-		     "%d:%d transaction failed %d, size %lld-%lld\n",
-		     proc->pid, thread->pid, return_error,
-		     (u64)tr->data_size, (u64)tr->offsets_size);
+		     "%d:%d transaction failed %d/%d, size %lld-%lld line %d\n",
+		     proc->pid, thread->pid, return_error, return_error_param,
+		     (u64)tr->data_size, (u64)tr->offsets_size,
+		     return_error_line);
 
 	{
 		struct binder_transaction_log_entry *fe;
 
+		e->return_error = return_error;
+		e->return_error_param = return_error_param;
+		e->return_error_line = return_error_line;
 		fe = binder_transaction_log_add(&binder_transaction_log_failed);
 		*fe = *e;
+		/*
+		 * write barrier to synchronize with initialization
+		 * of log entry
+		 */
+		smp_wmb();
+		WRITE_ONCE(e->debug_id_done, t_debug_id);
+		WRITE_ONCE(fe->debug_id_done, t_debug_id);
 	}
 
-	BUG_ON(thread->return_error != BR_OK);
+	BUG_ON(thread->return_error.cmd != BR_OK);
 	if (in_reply_to) {
-		thread->return_error = BR_TRANSACTION_COMPLETE;
+		thread->return_error.cmd = BR_TRANSACTION_COMPLETE;
+		binder_enqueue_work(thread->proc,
+				    &thread->return_error.work,
+				    &thread->todo);
 		binder_send_failed_reply(in_reply_to, return_error);
-	} else
-		thread->return_error = return_error;
+	} else {
+		thread->return_error.cmd = return_error;
+		binder_enqueue_work(thread->proc,
+				    &thread->return_error.work,
+				    &thread->todo);
+	}
 }
 
 static int binder_thread_write(struct binder_proc *proc,
@@ -2260,15 +3161,17 @@ static int binder_thread_write(struct binder_proc *proc,
 	void __user *ptr = buffer + *consumed;
 	void __user *end = buffer + size;
 
-	while (ptr < end && thread->return_error == BR_OK) {
+	while (ptr < end && thread->return_error.cmd == BR_OK) {
+		int ret;
+
 		if (get_user(cmd, (uint32_t __user *)ptr))
 			return -EFAULT;
 		ptr += sizeof(uint32_t);
 		trace_binder_command(cmd);
 		if (_IOC_NR(cmd) < ARRAY_SIZE(binder_stats.bc)) {
-			binder_stats.bc[_IOC_NR(cmd)]++;
-			proc->stats.bc[_IOC_NR(cmd)]++;
-			thread->stats.bc[_IOC_NR(cmd)]++;
+			atomic_inc(&binder_stats.bc[_IOC_NR(cmd)]);
+			atomic_inc(&proc->stats.bc[_IOC_NR(cmd)]);
+			atomic_inc(&thread->stats.bc[_IOC_NR(cmd)]);
 		}
 		switch (cmd) {
 		case BC_INCREFS:
@@ -2276,53 +3179,61 @@ static int binder_thread_write(struct binder_proc *proc,
 		case BC_RELEASE:
 		case BC_DECREFS: {
 			uint32_t target;
-			struct binder_ref *ref;
 			const char *debug_string;
+			bool strong = cmd == BC_ACQUIRE || cmd == BC_RELEASE;
+			bool increment = cmd == BC_INCREFS || cmd == BC_ACQUIRE;
+			struct binder_ref_data rdata;
 
 			if (get_user(target, (uint32_t __user *)ptr))
 				return -EFAULT;
+
 			ptr += sizeof(uint32_t);
-			if (target == 0 && context->binder_context_mgr_node &&
-			    (cmd == BC_INCREFS || cmd == BC_ACQUIRE)) {
-				ref = binder_get_ref_for_node(proc,
-					context->binder_context_mgr_node);
-				if (ref->desc != target) {
-					binder_user_error("%d:%d tried to acquire reference to desc 0, got %d instead\n",
-						proc->pid, thread->pid,
-						ref->desc);
-				}
-			} else
-				ref = binder_get_ref(proc, target,
-						     cmd == BC_ACQUIRE ||
-						     cmd == BC_RELEASE);
-			if (ref == NULL) {
-				binder_user_error("%d:%d refcount change on invalid ref %d\n",
-					proc->pid, thread->pid, target);
-				break;
+			ret = -1;
+			if (increment && !target) {
+				struct binder_node *ctx_mgr_node;
+				mutex_lock(&context->context_mgr_node_lock);
+				ctx_mgr_node = context->binder_context_mgr_node;
+				if (ctx_mgr_node)
+					ret = binder_inc_ref_for_node(
+							proc, ctx_mgr_node,
+							strong, NULL, &rdata);
+				mutex_unlock(&context->context_mgr_node_lock);
+			}
+			if (ret)
+				ret = binder_update_ref_for_handle(
+						proc, target, increment, strong,
+						&rdata);
+			if (!ret && rdata.desc != target) {
+				binder_user_error("%d:%d tried to acquire reference to desc %d, got %d instead\n",
+					proc->pid, thread->pid,
+					target, rdata.desc);
 			}
 			switch (cmd) {
 			case BC_INCREFS:
 				debug_string = "IncRefs";
-				binder_inc_ref(ref, 0, NULL);
 				break;
 			case BC_ACQUIRE:
 				debug_string = "Acquire";
-				binder_inc_ref(ref, 1, NULL);
 				break;
 			case BC_RELEASE:
 				debug_string = "Release";
-				binder_dec_ref(ref, 1);
 				break;
 			case BC_DECREFS:
 			default:
 				debug_string = "DecRefs";
-				binder_dec_ref(ref, 0);
+				break;
+			}
+			if (ret) {
+				binder_user_error("%d:%d %s %d refcount change on invalid ref %d ret %d\n",
+					proc->pid, thread->pid, debug_string,
+					strong, target, ret);
 				break;
 			}
 			binder_debug(BINDER_DEBUG_USER_REFS,
-				     "%d:%d %s ref %d desc %d s %d w %d for node %d\n",
-				     proc->pid, thread->pid, debug_string, ref->debug_id,
-				     ref->desc, ref->strong, ref->weak, ref->node->debug_id);
+				     "%d:%d %s ref %d desc %d s %d w %d\n",
+				     proc->pid, thread->pid, debug_string,
+				     rdata.debug_id, rdata.desc, rdata.strong,
+				     rdata.weak);
 			break;
 		}
 		case BC_INCREFS_DONE:
@@ -2330,6 +3241,7 @@ static int binder_thread_write(struct binder_proc *proc,
 			binder_uintptr_t node_ptr;
 			binder_uintptr_t cookie;
 			struct binder_node *node;
+			bool free_node;
 
 			if (get_user(node_ptr, (binder_uintptr_t __user *)ptr))
 				return -EFAULT;
@@ -2354,13 +3266,17 @@ static int binder_thread_write(struct binder_proc *proc,
 					"BC_INCREFS_DONE" : "BC_ACQUIRE_DONE",
 					(u64)node_ptr, node->debug_id,
 					(u64)cookie, (u64)node->cookie);
+				binder_put_node(node);
 				break;
 			}
+			binder_node_inner_lock(node);
 			if (cmd == BC_ACQUIRE_DONE) {
 				if (node->pending_strong_ref == 0) {
 					binder_user_error("%d:%d BC_ACQUIRE_DONE node %d has no pending acquire request\n",
 						proc->pid, thread->pid,
 						node->debug_id);
+					binder_node_inner_unlock(node);
+					binder_put_node(node);
 					break;
 				}
 				node->pending_strong_ref = 0;
@@ -2369,16 +3285,23 @@ static int binder_thread_write(struct binder_proc *proc,
 					binder_user_error("%d:%d BC_INCREFS_DONE node %d has no pending increfs request\n",
 						proc->pid, thread->pid,
 						node->debug_id);
+					binder_node_inner_unlock(node);
+					binder_put_node(node);
 					break;
 				}
 				node->pending_weak_ref = 0;
 			}
-			binder_dec_node(node, cmd == BC_ACQUIRE_DONE, 0);
+			free_node = binder_dec_node_nilocked(node,
+					cmd == BC_ACQUIRE_DONE, 0);
+			WARN_ON(free_node);
 			binder_debug(BINDER_DEBUG_USER_REFS,
-				     "%d:%d %s node %d ls %d lw %d\n",
+				     "%d:%d %s node %d ls %d lw %d tr %d\n",
 				     proc->pid, thread->pid,
 				     cmd == BC_INCREFS_DONE ? "BC_INCREFS_DONE" : "BC_ACQUIRE_DONE",
-				     node->debug_id, node->local_strong_refs, node->local_weak_refs);
+				     node->debug_id, node->local_strong_refs,
+				     node->local_weak_refs, node->tmp_refs);
+			binder_node_inner_unlock(node);
+			binder_put_node(node);
 			break;
 		}
 		case BC_ATTEMPT_ACQUIRE:
@@ -2396,7 +3319,8 @@ static int binder_thread_write(struct binder_proc *proc,
 				return -EFAULT;
 			ptr += sizeof(binder_uintptr_t);
 
-			buffer = binder_buffer_lookup(proc, data_ptr);
+			buffer = binder_alloc_prepare_to_free(&proc->alloc,
+							      data_ptr);
 			if (buffer == NULL) {
 				binder_user_error("%d:%d BC_FREE_BUFFER u%016llx no match\n",
 					proc->pid, thread->pid, (u64)data_ptr);
@@ -2418,15 +3342,27 @@ static int binder_thread_write(struct binder_proc *proc,
 				buffer->transaction = NULL;
 			}
 			if (buffer->async_transaction && buffer->target_node) {
-				BUG_ON(!buffer->target_node->has_async_transaction);
-				if (list_empty(&buffer->target_node->async_todo))
-					buffer->target_node->has_async_transaction = 0;
-				else
-					list_move_tail(buffer->target_node->async_todo.next, &thread->todo);
+				struct binder_node *buf_node;
+				struct binder_work *w;
+
+				buf_node = buffer->target_node;
+				binder_node_inner_lock(buf_node);
+				BUG_ON(!buf_node->has_async_transaction);
+				BUG_ON(buf_node->proc != proc);
+				w = binder_dequeue_work_head_ilocked(
+						&buf_node->async_todo);
+				if (!w) {
+					buf_node->has_async_transaction = 0;
+				} else {
+					binder_enqueue_work_ilocked(
+							w, &proc->todo);
+					binder_wakeup_proc_ilocked(proc);
+				}
+				binder_node_inner_unlock(buf_node);
 			}
 			trace_binder_transaction_buffer_release(buffer);
 			binder_transaction_buffer_release(proc, buffer, NULL);
-			binder_free_buf(proc, buffer);
+			binder_alloc_free_buf(&proc->alloc, buffer);
 			break;
 		}
 
@@ -2457,6 +3393,7 @@ static int binder_thread_write(struct binder_proc *proc,
 			binder_debug(BINDER_DEBUG_THREADS,
 				     "%d:%d BC_REGISTER_LOOPER\n",
 				     proc->pid, thread->pid);
+			binder_inner_proc_lock(proc);
 			if (thread->looper & BINDER_LOOPER_STATE_ENTERED) {
 				thread->looper |= BINDER_LOOPER_STATE_INVALID;
 				binder_user_error("%d:%d ERROR: BC_REGISTER_LOOPER called after BC_ENTER_LOOPER\n",
@@ -2470,6 +3407,7 @@ static int binder_thread_write(struct binder_proc *proc,
 				proc->requested_threads_started++;
 			}
 			thread->looper |= BINDER_LOOPER_STATE_REGISTERED;
+			binder_inner_proc_unlock(proc);
 			break;
 		case BC_ENTER_LOOPER:
 			binder_debug(BINDER_DEBUG_THREADS,
@@ -2494,7 +3432,7 @@ static int binder_thread_write(struct binder_proc *proc,
 			uint32_t target;
 			binder_uintptr_t cookie;
 			struct binder_ref *ref;
-			struct binder_ref_death *death;
+			struct binder_ref_death *death = NULL;
 
 			if (get_user(target, (uint32_t __user *)ptr))
 				return -EFAULT;
@@ -2502,7 +3440,29 @@ static int binder_thread_write(struct binder_proc *proc,
 			if (get_user(cookie, (binder_uintptr_t __user *)ptr))
 				return -EFAULT;
 			ptr += sizeof(binder_uintptr_t);
-			ref = binder_get_ref(proc, target, false);
+			if (cmd == BC_REQUEST_DEATH_NOTIFICATION) {
+				/*
+				 * Allocate memory for death notification
+				 * before taking lock
+				 */
+				death = kzalloc(sizeof(*death), GFP_KERNEL);
+				if (death == NULL) {
+					WARN_ON(thread->return_error.cmd !=
+						BR_OK);
+					thread->return_error.cmd = BR_ERROR;
+					binder_enqueue_work(
+						thread->proc,
+						&thread->return_error.work,
+						&thread->todo);
+					binder_debug(
+						BINDER_DEBUG_FAILED_TRANSACTION,
+						"%d:%d BC_REQUEST_DEATH_NOTIFICATION failed\n",
+						proc->pid, thread->pid);
+					break;
+				}
+			}
+			binder_proc_lock(proc);
+			ref = binder_get_ref_olocked(proc, target, false);
 			if (ref == NULL) {
 				binder_user_error("%d:%d %s invalid ref %d\n",
 					proc->pid, thread->pid,
@@ -2510,6 +3470,8 @@ static int binder_thread_write(struct binder_proc *proc,
 					"BC_REQUEST_DEATH_NOTIFICATION" :
 					"BC_CLEAR_DEATH_NOTIFICATION",
 					target);
+				binder_proc_unlock(proc);
+				kfree(death);
 				break;
 			}
 
@@ -2519,21 +3481,18 @@ static int binder_thread_write(struct binder_proc *proc,
 				     cmd == BC_REQUEST_DEATH_NOTIFICATION ?
 				     "BC_REQUEST_DEATH_NOTIFICATION" :
 				     "BC_CLEAR_DEATH_NOTIFICATION",
-				     (u64)cookie, ref->debug_id, ref->desc,
-				     ref->strong, ref->weak, ref->node->debug_id);
+				     (u64)cookie, ref->data.debug_id,
+				     ref->data.desc, ref->data.strong,
+				     ref->data.weak, ref->node->debug_id);
 
+			binder_node_lock(ref->node);
 			if (cmd == BC_REQUEST_DEATH_NOTIFICATION) {
 				if (ref->death) {
 					binder_user_error("%d:%d BC_REQUEST_DEATH_NOTIFICATION death notification already set\n",
 						proc->pid, thread->pid);
-					break;
-				}
-				death = kzalloc(sizeof(*death), GFP_KERNEL);
-				if (death == NULL) {
-					thread->return_error = BR_ERROR;
-					binder_debug(BINDER_DEBUG_FAILED_TRANSACTION,
-						     "%d:%d BC_REQUEST_DEATH_NOTIFICATION failed\n",
-						     proc->pid, thread->pid);
+					binder_node_unlock(ref->node);
+					binder_proc_unlock(proc);
+					kfree(death);
 					break;
 				}
 				binder_stats_created(BINDER_STAT_DEATH);
@@ -2542,17 +3501,19 @@ static int binder_thread_write(struct binder_proc *proc,
 				ref->death = death;
 				if (ref->node->proc == NULL) {
 					ref->death->work.type = BINDER_WORK_DEAD_BINDER;
-					if (thread->looper & (BINDER_LOOPER_STATE_REGISTERED | BINDER_LOOPER_STATE_ENTERED)) {
-						list_add_tail(&ref->death->work.entry, &thread->todo);
-					} else {
-						list_add_tail(&ref->death->work.entry, &proc->todo);
-						wake_up_interruptible(&proc->wait);
-					}
+
+					binder_inner_proc_lock(proc);
+					binder_enqueue_work_ilocked(
+						&ref->death->work, &proc->todo);
+					binder_wakeup_proc_ilocked(proc);
+					binder_inner_proc_unlock(proc);
 				}
 			} else {
 				if (ref->death == NULL) {
 					binder_user_error("%d:%d BC_CLEAR_DEATH_NOTIFICATION death notification not active\n",
 						proc->pid, thread->pid);
+					binder_node_unlock(ref->node);
+					binder_proc_unlock(proc);
 					break;
 				}
 				death = ref->death;
@@ -2561,22 +3522,35 @@ static int binder_thread_write(struct binder_proc *proc,
 						proc->pid, thread->pid,
 						(u64)death->cookie,
 						(u64)cookie);
+					binder_node_unlock(ref->node);
+					binder_proc_unlock(proc);
 					break;
 				}
 				ref->death = NULL;
+				binder_inner_proc_lock(proc);
 				if (list_empty(&death->work.entry)) {
 					death->work.type = BINDER_WORK_CLEAR_DEATH_NOTIFICATION;
-					if (thread->looper & (BINDER_LOOPER_STATE_REGISTERED | BINDER_LOOPER_STATE_ENTERED)) {
-						list_add_tail(&death->work.entry, &thread->todo);
-					} else {
-						list_add_tail(&death->work.entry, &proc->todo);
-						wake_up_interruptible(&proc->wait);
+					if (thread->looper &
+					    (BINDER_LOOPER_STATE_REGISTERED |
+					     BINDER_LOOPER_STATE_ENTERED))
+						binder_enqueue_work_ilocked(
+								&death->work,
+								&thread->todo);
+					else {
+						binder_enqueue_work_ilocked(
+								&death->work,
+								&proc->todo);
+						binder_wakeup_proc_ilocked(
+								proc);
 					}
 				} else {
 					BUG_ON(death->work.type != BINDER_WORK_DEAD_BINDER);
 					death->work.type = BINDER_WORK_DEAD_BINDER_AND_CLEAR;
 				}
+				binder_inner_proc_unlock(proc);
 			}
+			binder_node_unlock(ref->node);
+			binder_proc_unlock(proc);
 		} break;
 		case BC_DEAD_BINDER_DONE: {
 			struct binder_work *w;
@@ -2587,8 +3561,13 @@ static int binder_thread_write(struct binder_proc *proc,
 				return -EFAULT;
 
 			ptr += sizeof(cookie);
-			list_for_each_entry(w, &proc->delivered_death, entry) {
-				struct binder_ref_death *tmp_death = container_of(w, struct binder_ref_death, work);
+			binder_inner_proc_lock(proc);
+			list_for_each_entry(w, &proc->delivered_death,
+					    entry) {
+				struct binder_ref_death *tmp_death =
+					container_of(w,
+						     struct binder_ref_death,
+						     work);
 
 				if (tmp_death->cookie == cookie) {
 					death = tmp_death;
@@ -2602,19 +3581,25 @@ static int binder_thread_write(struct binder_proc *proc,
 			if (death == NULL) {
 				binder_user_error("%d:%d BC_DEAD_BINDER_DONE %016llx not found\n",
 					proc->pid, thread->pid, (u64)cookie);
+				binder_inner_proc_unlock(proc);
 				break;
 			}
-
-			list_del_init(&death->work.entry);
+			binder_dequeue_work_ilocked(&death->work);
 			if (death->work.type == BINDER_WORK_DEAD_BINDER_AND_CLEAR) {
 				death->work.type = BINDER_WORK_CLEAR_DEATH_NOTIFICATION;
-				if (thread->looper & (BINDER_LOOPER_STATE_REGISTERED | BINDER_LOOPER_STATE_ENTERED)) {
-					list_add_tail(&death->work.entry, &thread->todo);
-				} else {
-					list_add_tail(&death->work.entry, &proc->todo);
-					wake_up_interruptible(&proc->wait);
+				if (thread->looper &
+					(BINDER_LOOPER_STATE_REGISTERED |
+					 BINDER_LOOPER_STATE_ENTERED))
+					binder_enqueue_work_ilocked(
+						&death->work, &thread->todo);
+				else {
+					binder_enqueue_work_ilocked(
+							&death->work,
+							&proc->todo);
+					binder_wakeup_proc_ilocked(proc);
 				}
 			}
+			binder_inner_proc_unlock(proc);
 		} break;
 
 		default:
@@ -2632,23 +3617,79 @@ static void binder_stat_br(struct binder_proc *proc,
 {
 	trace_binder_return(cmd);
 	if (_IOC_NR(cmd) < ARRAY_SIZE(binder_stats.br)) {
-		binder_stats.br[_IOC_NR(cmd)]++;
-		proc->stats.br[_IOC_NR(cmd)]++;
-		thread->stats.br[_IOC_NR(cmd)]++;
+		atomic_inc(&binder_stats.br[_IOC_NR(cmd)]);
+		atomic_inc(&proc->stats.br[_IOC_NR(cmd)]);
+		atomic_inc(&thread->stats.br[_IOC_NR(cmd)]);
 	}
 }
 
-static int binder_has_proc_work(struct binder_proc *proc,
-				struct binder_thread *thread)
+static int binder_has_thread_work(struct binder_thread *thread)
 {
-	return !list_empty(&proc->todo) ||
-		(thread->looper & BINDER_LOOPER_STATE_NEED_RETURN);
+	return !binder_worklist_empty(thread->proc, &thread->todo) ||
+		thread->looper_need_return;
 }
 
-static int binder_has_thread_work(struct binder_thread *thread)
+static int binder_put_node_cmd(struct binder_proc *proc,
+			       struct binder_thread *thread,
+			       void __user **ptrp,
+			       binder_uintptr_t node_ptr,
+			       binder_uintptr_t node_cookie,
+			       int node_debug_id,
+			       uint32_t cmd, const char *cmd_name)
 {
-	return !list_empty(&thread->todo) || thread->return_error != BR_OK ||
-		(thread->looper & BINDER_LOOPER_STATE_NEED_RETURN);
+	void __user *ptr = *ptrp;
+
+	if (put_user(cmd, (uint32_t __user *)ptr))
+		return -EFAULT;
+	ptr += sizeof(uint32_t);
+
+	if (put_user(node_ptr, (binder_uintptr_t __user *)ptr))
+		return -EFAULT;
+	ptr += sizeof(binder_uintptr_t);
+
+	if (put_user(node_cookie, (binder_uintptr_t __user *)ptr))
+		return -EFAULT;
+	ptr += sizeof(binder_uintptr_t);
+
+	binder_stat_br(proc, thread, cmd);
+	binder_debug(BINDER_DEBUG_USER_REFS, "%d:%d %s %d u%016llx c%016llx\n",
+		     proc->pid, thread->pid, cmd_name, node_debug_id,
+		     (u64)node_ptr, (u64)node_cookie);
+
+	*ptrp = ptr;
+	return 0;
+}
+
+static int binder_wait_for_work(struct binder_thread *thread,
+				bool do_proc_work)
+{
+	DEFINE_WAIT(wait);
+	struct binder_proc *proc = thread->proc;
+	int ret = 0;
+
+	freezer_do_not_count();
+	binder_inner_proc_lock(proc);
+	for (;;) {
+		prepare_to_wait(&thread->wait, &wait, TASK_INTERRUPTIBLE);
+		if (binder_has_work_ilocked(thread, do_proc_work))
+			break;
+		if (do_proc_work)
+			list_add(&thread->waiting_thread_node,
+				 &proc->waiting_threads);
+		binder_inner_proc_unlock(proc);
+		schedule();
+		binder_inner_proc_lock(proc);
+		list_del_init(&thread->waiting_thread_node);
+		if (signal_pending(current)) {
+			ret = -ERESTARTSYS;
+			break;
+		}
+	}
+	finish_wait(&thread->wait, &wait);
+	binder_inner_proc_unlock(proc);
+	freezer_count();
+
+	return ret;
 }
 
 static int binder_thread_read(struct binder_proc *proc,
@@ -2670,37 +3711,15 @@ static int binder_thread_read(struct binder_proc *proc,
 	}
 
 retry:
-	wait_for_proc_work = thread->transaction_stack == NULL &&
-				list_empty(&thread->todo);
-
-	if (thread->return_error != BR_OK && ptr < end) {
-		if (thread->return_error2 != BR_OK) {
-			if (put_user(thread->return_error2, (uint32_t __user *)ptr))
-				return -EFAULT;
-			ptr += sizeof(uint32_t);
-			binder_stat_br(proc, thread, thread->return_error2);
-			if (ptr == end)
-				goto done;
-			thread->return_error2 = BR_OK;
-		}
-		if (put_user(thread->return_error, (uint32_t __user *)ptr))
-			return -EFAULT;
-		ptr += sizeof(uint32_t);
-		binder_stat_br(proc, thread, thread->return_error);
-		thread->return_error = BR_OK;
-		goto done;
-	}
-
+	binder_inner_proc_lock(proc);
+	wait_for_proc_work = binder_available_for_proc_work_ilocked(thread);
+	binder_inner_proc_unlock(proc);
 
 	thread->looper |= BINDER_LOOPER_STATE_WAITING;
-	if (wait_for_proc_work)
-		proc->ready_threads++;
-
-	binder_unlock(__func__);
 
 	trace_binder_wait_for_work(wait_for_proc_work,
 				   !!thread->transaction_stack,
-				   !list_empty(&thread->todo));
+				   !binder_worklist_empty(proc, &thread->todo));
 	if (wait_for_proc_work) {
 		if (!(thread->looper & (BINDER_LOOPER_STATE_REGISTERED |
 					BINDER_LOOPER_STATE_ENTERED))) {
@@ -2710,23 +3729,15 @@ retry:
 						 binder_stop_on_user_error < 2);
 		}
 		binder_set_nice(proc->default_priority);
-		if (non_block) {
-			if (!binder_has_proc_work(proc, thread))
-				ret = -EAGAIN;
-		} else
-			ret = wait_event_freezable_exclusive(proc->wait, binder_has_proc_work(proc, thread));
-	} else {
-		if (non_block) {
-			if (!binder_has_thread_work(thread))
-				ret = -EAGAIN;
-		} else
-			ret = wait_event_freezable(thread->wait, binder_has_thread_work(thread));
 	}
 
-	binder_lock(__func__);
+	if (non_block) {
+		if (!binder_has_work(thread, wait_for_proc_work))
+			ret = -EAGAIN;
+	} else {
+		ret = binder_wait_for_work(thread, wait_for_proc_work);
+	}
 
-	if (wait_for_proc_work)
-		proc->ready_threads--;
 	thread->looper &= ~BINDER_LOOPER_STATE_WAITING;
 
 	if (ret)
@@ -2735,31 +3746,52 @@ retry:
 	while (1) {
 		uint32_t cmd;
 		struct binder_transaction_data tr;
-		struct binder_work *w;
+		struct binder_work *w = NULL;
+		struct list_head *list = NULL;
 		struct binder_transaction *t = NULL;
+		struct binder_thread *t_from;
+
+		binder_inner_proc_lock(proc);
+		if (!binder_worklist_empty_ilocked(&thread->todo))
+			list = &thread->todo;
+		else if (!binder_worklist_empty_ilocked(&proc->todo) &&
+			   wait_for_proc_work)
+			list = &proc->todo;
+		else {
+			binder_inner_proc_unlock(proc);
 
-		if (!list_empty(&thread->todo)) {
-			w = list_first_entry(&thread->todo, struct binder_work,
-					     entry);
-		} else if (!list_empty(&proc->todo) && wait_for_proc_work) {
-			w = list_first_entry(&proc->todo, struct binder_work,
-					     entry);
-		} else {
 			/* no data added */
-			if (ptr - buffer == 4 &&
-			    !(thread->looper & BINDER_LOOPER_STATE_NEED_RETURN))
+			if (ptr - buffer == 4 && !thread->looper_need_return)
 				goto retry;
 			break;
 		}
 
-		if (end - ptr < sizeof(tr) + 4)
+		if (end - ptr < sizeof(tr) + 4) {
+			binder_inner_proc_unlock(proc);
 			break;
+		}
+		w = binder_dequeue_work_head_ilocked(list);
 
 		switch (w->type) {
 		case BINDER_WORK_TRANSACTION: {
+			binder_inner_proc_unlock(proc);
 			t = container_of(w, struct binder_transaction, work);
 		} break;
+		case BINDER_WORK_RETURN_ERROR: {
+			struct binder_error *e = container_of(
+					w, struct binder_error, work);
+
+			WARN_ON(e->cmd == BR_OK);
+			binder_inner_proc_unlock(proc);
+			if (put_user(e->cmd, (uint32_t __user *)ptr))
+				return -EFAULT;
+			e->cmd = BR_OK;
+			ptr += sizeof(uint32_t);
+
+			binder_stat_br(proc, thread, e->cmd);
+		} break;
 		case BINDER_WORK_TRANSACTION_COMPLETE: {
+			binder_inner_proc_unlock(proc);
 			cmd = BR_TRANSACTION_COMPLETE;
 			if (put_user(cmd, (uint32_t __user *)ptr))
 				return -EFAULT;
@@ -2769,113 +3801,134 @@ retry:
 			binder_debug(BINDER_DEBUG_TRANSACTION_COMPLETE,
 				     "%d:%d BR_TRANSACTION_COMPLETE\n",
 				     proc->pid, thread->pid);
-
-			list_del(&w->entry);
 			kfree(w);
 			binder_stats_deleted(BINDER_STAT_TRANSACTION_COMPLETE);
 		} break;
 		case BINDER_WORK_NODE: {
 			struct binder_node *node = container_of(w, struct binder_node, work);
-			uint32_t cmd = BR_NOOP;
-			const char *cmd_name;
-			int strong = node->internal_strong_refs || node->local_strong_refs;
-			int weak = !hlist_empty(&node->refs) || node->local_weak_refs || strong;
-
-			if (weak && !node->has_weak_ref) {
-				cmd = BR_INCREFS;
-				cmd_name = "BR_INCREFS";
+			int strong, weak;
+			binder_uintptr_t node_ptr = node->ptr;
+			binder_uintptr_t node_cookie = node->cookie;
+			int node_debug_id = node->debug_id;
+			int has_weak_ref;
+			int has_strong_ref;
+			void __user *orig_ptr = ptr;
+
+			BUG_ON(proc != node->proc);
+			strong = node->internal_strong_refs ||
+					node->local_strong_refs;
+			weak = !hlist_empty(&node->refs) ||
+					node->local_weak_refs ||
+					node->tmp_refs || strong;
+			has_strong_ref = node->has_strong_ref;
+			has_weak_ref = node->has_weak_ref;
+
+			if (weak && !has_weak_ref) {
 				node->has_weak_ref = 1;
 				node->pending_weak_ref = 1;
 				node->local_weak_refs++;
-			} else if (strong && !node->has_strong_ref) {
-				cmd = BR_ACQUIRE;
-				cmd_name = "BR_ACQUIRE";
+			}
+			if (strong && !has_strong_ref) {
 				node->has_strong_ref = 1;
 				node->pending_strong_ref = 1;
 				node->local_strong_refs++;
-			} else if (!strong && node->has_strong_ref) {
-				cmd = BR_RELEASE;
-				cmd_name = "BR_RELEASE";
+			}
+			if (!strong && has_strong_ref)
 				node->has_strong_ref = 0;
-			} else if (!weak && node->has_weak_ref) {
-				cmd = BR_DECREFS;
-				cmd_name = "BR_DECREFS";
+			if (!weak && has_weak_ref)
 				node->has_weak_ref = 0;
-			}
-			if (cmd != BR_NOOP) {
-				if (put_user(cmd, (uint32_t __user *)ptr))
-					return -EFAULT;
-				ptr += sizeof(uint32_t);
-				if (put_user(node->ptr,
-					     (binder_uintptr_t __user *)ptr))
-					return -EFAULT;
-				ptr += sizeof(binder_uintptr_t);
-				if (put_user(node->cookie,
-					     (binder_uintptr_t __user *)ptr))
-					return -EFAULT;
-				ptr += sizeof(binder_uintptr_t);
-
-				binder_stat_br(proc, thread, cmd);
-				binder_debug(BINDER_DEBUG_USER_REFS,
-					     "%d:%d %s %d u%016llx c%016llx\n",
-					     proc->pid, thread->pid, cmd_name,
-					     node->debug_id,
-					     (u64)node->ptr, (u64)node->cookie);
-			} else {
-				list_del_init(&w->entry);
-				if (!weak && !strong) {
-					binder_debug(BINDER_DEBUG_INTERNAL_REFS,
-						     "%d:%d node %d u%016llx c%016llx deleted\n",
-						     proc->pid, thread->pid,
-						     node->debug_id,
-						     (u64)node->ptr,
-						     (u64)node->cookie);
-					rb_erase(&node->rb_node, &proc->nodes);
-					kfree(node);
-					binder_stats_deleted(BINDER_STAT_NODE);
-				} else {
-					binder_debug(BINDER_DEBUG_INTERNAL_REFS,
-						     "%d:%d node %d u%016llx c%016llx state unchanged\n",
-						     proc->pid, thread->pid,
-						     node->debug_id,
-						     (u64)node->ptr,
-						     (u64)node->cookie);
-				}
-			}
+			if (!weak && !strong) {
+				binder_debug(BINDER_DEBUG_INTERNAL_REFS,
+					     "%d:%d node %d u%016llx c%016llx deleted\n",
+					     proc->pid, thread->pid,
+					     node_debug_id,
+					     (u64)node_ptr,
+					     (u64)node_cookie);
+				rb_erase(&node->rb_node, &proc->nodes);
+				binder_inner_proc_unlock(proc);
+				binder_node_lock(node);
+				/*
+				 * Acquire the node lock before freeing the
+				 * node to serialize with other threads that
+				 * may have been holding the node lock while
+				 * decrementing this node (avoids race where
+				 * this thread frees while the other thread
+				 * is unlocking the node after the final
+				 * decrement)
+				 */
+				binder_node_unlock(node);
+				binder_free_node(node);
+			} else
+				binder_inner_proc_unlock(proc);
+
+			if (weak && !has_weak_ref)
+				ret = binder_put_node_cmd(
+						proc, thread, &ptr, node_ptr,
+						node_cookie, node_debug_id,
+						BR_INCREFS, "BR_INCREFS");
+			if (!ret && strong && !has_strong_ref)
+				ret = binder_put_node_cmd(
+						proc, thread, &ptr, node_ptr,
+						node_cookie, node_debug_id,
+						BR_ACQUIRE, "BR_ACQUIRE");
+			if (!ret && !strong && has_strong_ref)
+				ret = binder_put_node_cmd(
+						proc, thread, &ptr, node_ptr,
+						node_cookie, node_debug_id,
+						BR_RELEASE, "BR_RELEASE");
+			if (!ret && !weak && has_weak_ref)
+				ret = binder_put_node_cmd(
+						proc, thread, &ptr, node_ptr,
+						node_cookie, node_debug_id,
+						BR_DECREFS, "BR_DECREFS");
+			if (orig_ptr == ptr)
+				binder_debug(BINDER_DEBUG_INTERNAL_REFS,
+					     "%d:%d node %d u%016llx c%016llx state unchanged\n",
+					     proc->pid, thread->pid,
+					     node_debug_id,
+					     (u64)node_ptr,
+					     (u64)node_cookie);
+			if (ret)
+				return ret;
 		} break;
 		case BINDER_WORK_DEAD_BINDER:
 		case BINDER_WORK_DEAD_BINDER_AND_CLEAR:
 		case BINDER_WORK_CLEAR_DEATH_NOTIFICATION: {
 			struct binder_ref_death *death;
 			uint32_t cmd;
+			binder_uintptr_t cookie;
 
 			death = container_of(w, struct binder_ref_death, work);
 			if (w->type == BINDER_WORK_CLEAR_DEATH_NOTIFICATION)
 				cmd = BR_CLEAR_DEATH_NOTIFICATION_DONE;
 			else
 				cmd = BR_DEAD_BINDER;
-			if (put_user(cmd, (uint32_t __user *)ptr))
-				return -EFAULT;
-			ptr += sizeof(uint32_t);
-			if (put_user(death->cookie,
-				     (binder_uintptr_t __user *)ptr))
-				return -EFAULT;
-			ptr += sizeof(binder_uintptr_t);
-			binder_stat_br(proc, thread, cmd);
+			cookie = death->cookie;
+
 			binder_debug(BINDER_DEBUG_DEATH_NOTIFICATION,
 				     "%d:%d %s %016llx\n",
 				      proc->pid, thread->pid,
 				      cmd == BR_DEAD_BINDER ?
 				      "BR_DEAD_BINDER" :
 				      "BR_CLEAR_DEATH_NOTIFICATION_DONE",
-				      (u64)death->cookie);
-
+				      (u64)cookie);
 			if (w->type == BINDER_WORK_CLEAR_DEATH_NOTIFICATION) {
-				list_del(&w->entry);
+				binder_inner_proc_unlock(proc);
 				kfree(death);
 				binder_stats_deleted(BINDER_STAT_DEATH);
-			} else
-				list_move(&w->entry, &proc->delivered_death);
+			} else {
+				binder_enqueue_work_ilocked(
+						w, &proc->delivered_death);
+				binder_inner_proc_unlock(proc);
+			}
+			if (put_user(cmd, (uint32_t __user *)ptr))
+				return -EFAULT;
+			ptr += sizeof(uint32_t);
+			if (put_user(cookie,
+				     (binder_uintptr_t __user *)ptr))
+				return -EFAULT;
+			ptr += sizeof(binder_uintptr_t);
+			binder_stat_br(proc, thread, cmd);
 			if (cmd == BR_DEAD_BINDER)
 				goto done; /* DEAD_BINDER notifications can cause transactions */
 		} break;
@@ -2907,8 +3960,9 @@ retry:
 		tr.flags = t->flags;
 		tr.sender_euid = from_kuid(current_user_ns(), t->sender_euid);
 
-		if (t->from) {
-			struct task_struct *sender = t->from->proc->tsk;
+		t_from = binder_get_txn_from(t);
+		if (t_from) {
+			struct task_struct *sender = t_from->proc->tsk;
 
 			tr.sender_pid = task_tgid_nr_ns(sender,
 							task_active_pid_ns(current));
@@ -2918,18 +3972,24 @@ retry:
 
 		tr.data_size = t->buffer->data_size;
 		tr.offsets_size = t->buffer->offsets_size;
-		tr.data.ptr.buffer = (binder_uintptr_t)(
-					(uintptr_t)t->buffer->data +
-					proc->user_buffer_offset);
+		tr.data.ptr.buffer = (binder_uintptr_t)
+			((uintptr_t)t->buffer->data +
+			binder_alloc_get_user_buffer_offset(&proc->alloc));
 		tr.data.ptr.offsets = tr.data.ptr.buffer +
 					ALIGN(t->buffer->data_size,
 					    sizeof(void *));
 
-		if (put_user(cmd, (uint32_t __user *)ptr))
+		if (put_user(cmd, (uint32_t __user *)ptr)) {
+			if (t_from)
+				binder_thread_dec_tmpref(t_from);
 			return -EFAULT;
+		}
 		ptr += sizeof(uint32_t);
-		if (copy_to_user(ptr, &tr, sizeof(tr)))
+		if (copy_to_user(ptr, &tr, sizeof(tr))) {
+			if (t_from)
+				binder_thread_dec_tmpref(t_from);
 			return -EFAULT;
+		}
 		ptr += sizeof(tr);
 
 		trace_binder_transaction_received(t);
@@ -2939,21 +3999,22 @@ retry:
 			     proc->pid, thread->pid,
 			     (cmd == BR_TRANSACTION) ? "BR_TRANSACTION" :
 			     "BR_REPLY",
-			     t->debug_id, t->from ? t->from->proc->pid : 0,
-			     t->from ? t->from->pid : 0, cmd,
+			     t->debug_id, t_from ? t_from->proc->pid : 0,
+			     t_from ? t_from->pid : 0, cmd,
 			     t->buffer->data_size, t->buffer->offsets_size,
 			     (u64)tr.data.ptr.buffer, (u64)tr.data.ptr.offsets);
 
-		list_del(&t->work.entry);
+		if (t_from)
+			binder_thread_dec_tmpref(t_from);
 		t->buffer->allow_user_free = 1;
 		if (cmd == BR_TRANSACTION && !(t->flags & TF_ONE_WAY)) {
+			binder_inner_proc_lock(thread->proc);
 			t->to_parent = thread->transaction_stack;
 			t->to_thread = thread;
 			thread->transaction_stack = t;
+			binder_inner_proc_unlock(thread->proc);
 		} else {
-			t->buffer->transaction = NULL;
-			kfree(t);
-			binder_stats_deleted(BINDER_STAT_TRANSACTION);
+			binder_free_transaction(t);
 		}
 		break;
 	}
@@ -2961,29 +4022,36 @@ retry:
 done:
 
 	*consumed = ptr - buffer;
-	if (proc->requested_threads + proc->ready_threads == 0 &&
+	binder_inner_proc_lock(proc);
+	if (proc->requested_threads == 0 &&
+	    list_empty(&thread->proc->waiting_threads) &&
 	    proc->requested_threads_started < proc->max_threads &&
 	    (thread->looper & (BINDER_LOOPER_STATE_REGISTERED |
 	     BINDER_LOOPER_STATE_ENTERED)) /* the user-space code fails to */
 	     /*spawn a new thread if we leave this out */) {
 		proc->requested_threads++;
+		binder_inner_proc_unlock(proc);
 		binder_debug(BINDER_DEBUG_THREADS,
 			     "%d:%d BR_SPAWN_LOOPER\n",
 			     proc->pid, thread->pid);
 		if (put_user(BR_SPAWN_LOOPER, (uint32_t __user *)buffer))
 			return -EFAULT;
 		binder_stat_br(proc, thread, BR_SPAWN_LOOPER);
-	}
+	} else
+		binder_inner_proc_unlock(proc);
 	return 0;
 }
 
-static void binder_release_work(struct list_head *list)
+static void binder_release_work(struct binder_proc *proc,
+				struct list_head *list)
 {
 	struct binder_work *w;
 
-	while (!list_empty(list)) {
-		w = list_first_entry(list, struct binder_work, entry);
-		list_del_init(&w->entry);
+	while (1) {
+		w = binder_dequeue_work_head(proc, list);
+		if (!w)
+			return;
+
 		switch (w->type) {
 		case BINDER_WORK_TRANSACTION: {
 			struct binder_transaction *t;
@@ -2996,11 +4064,17 @@ static void binder_release_work(struct list_head *list)
 				binder_debug(BINDER_DEBUG_DEAD_TRANSACTION,
 					"undelivered transaction %d\n",
 					t->debug_id);
-				t->buffer->transaction = NULL;
-				kfree(t);
-				binder_stats_deleted(BINDER_STAT_TRANSACTION);
+				binder_free_transaction(t);
 			}
 		} break;
+		case BINDER_WORK_RETURN_ERROR: {
+			struct binder_error *e = container_of(
+					w, struct binder_error, work);
+
+			binder_debug(BINDER_DEBUG_DEAD_TRANSACTION,
+				"undelivered TRANSACTION_ERROR: %u\n",
+				e->cmd);
+		} break;
 		case BINDER_WORK_TRANSACTION_COMPLETE: {
 			binder_debug(BINDER_DEBUG_DEAD_TRANSACTION,
 				"undelivered TRANSACTION_COMPLETE\n");
@@ -3027,7 +4101,8 @@ static void binder_release_work(struct list_head *list)
 
 }
 
-static struct binder_thread *binder_get_thread(struct binder_proc *proc)
+static struct binder_thread *binder_get_thread_ilocked(
+		struct binder_proc *proc, struct binder_thread *new_thread)
 {
 	struct binder_thread *thread = NULL;
 	struct rb_node *parent = NULL;
@@ -3042,38 +4117,99 @@ static struct binder_thread *binder_get_thread(struct binder_proc *proc)
 		else if (current->pid > thread->pid)
 			p = &(*p)->rb_right;
 		else
-			break;
+			return thread;
 	}
-	if (*p == NULL) {
-		thread = kzalloc(sizeof(*thread), GFP_KERNEL);
-		if (thread == NULL)
+	if (!new_thread)
+		return NULL;
+	thread = new_thread;
+	binder_stats_created(BINDER_STAT_THREAD);
+	thread->proc = proc;
+	thread->pid = current->pid;
+	atomic_set(&thread->tmp_ref, 0);
+	init_waitqueue_head(&thread->wait);
+	INIT_LIST_HEAD(&thread->todo);
+	rb_link_node(&thread->rb_node, parent, p);
+	rb_insert_color(&thread->rb_node, &proc->threads);
+	thread->looper_need_return = true;
+	thread->return_error.work.type = BINDER_WORK_RETURN_ERROR;
+	thread->return_error.cmd = BR_OK;
+	thread->reply_error.work.type = BINDER_WORK_RETURN_ERROR;
+	thread->reply_error.cmd = BR_OK;
+	INIT_LIST_HEAD(&new_thread->waiting_thread_node);
+	return thread;
+}
+
+static struct binder_thread *binder_get_thread(struct binder_proc *proc)
+{
+	struct binder_thread *thread;
+	struct binder_thread *new_thread;
+
+	binder_inner_proc_lock(proc);
+	thread = binder_get_thread_ilocked(proc, NULL);
+	binder_inner_proc_unlock(proc);
+	if (!thread) {
+		new_thread = kzalloc(sizeof(*thread), GFP_KERNEL);
+		if (new_thread == NULL)
 			return NULL;
-		binder_stats_created(BINDER_STAT_THREAD);
-		thread->proc = proc;
-		thread->pid = current->pid;
-		init_waitqueue_head(&thread->wait);
-		INIT_LIST_HEAD(&thread->todo);
-		rb_link_node(&thread->rb_node, parent, p);
-		rb_insert_color(&thread->rb_node, &proc->threads);
-		thread->looper |= BINDER_LOOPER_STATE_NEED_RETURN;
-		thread->return_error = BR_OK;
-		thread->return_error2 = BR_OK;
+		binder_inner_proc_lock(proc);
+		thread = binder_get_thread_ilocked(proc, new_thread);
+		binder_inner_proc_unlock(proc);
+		if (thread != new_thread)
+			kfree(new_thread);
 	}
 	return thread;
 }
 
-static int binder_free_thread(struct binder_proc *proc,
-			      struct binder_thread *thread)
+static void binder_free_proc(struct binder_proc *proc)
+{
+	BUG_ON(!list_empty(&proc->todo));
+	BUG_ON(!list_empty(&proc->delivered_death));
+	binder_alloc_deferred_release(&proc->alloc);
+	put_task_struct(proc->tsk);
+	binder_stats_deleted(BINDER_STAT_PROC);
+	kfree(proc);
+}
+
+static void binder_free_thread(struct binder_thread *thread)
+{
+	BUG_ON(!list_empty(&thread->todo));
+	binder_stats_deleted(BINDER_STAT_THREAD);
+	binder_proc_dec_tmpref(thread->proc);
+	kfree(thread);
+}
+
+static int binder_thread_release(struct binder_proc *proc,
+				 struct binder_thread *thread)
 {
 	struct binder_transaction *t;
 	struct binder_transaction *send_reply = NULL;
 	int active_transactions = 0;
+	struct binder_transaction *last_t = NULL;
 
+	binder_inner_proc_lock(thread->proc);
+	/*
+	 * take a ref on the proc so it survives
+	 * after we remove this thread from proc->threads.
+	 * The corresponding dec is when we actually
+	 * free the thread in binder_free_thread()
+	 */
+	proc->tmp_ref++;
+	/*
+	 * take a ref on this thread to ensure it
+	 * survives while we are releasing it
+	 */
+	atomic_inc(&thread->tmp_ref);
 	rb_erase(&thread->rb_node, &proc->threads);
 	t = thread->transaction_stack;
-	if (t && t->to_thread == thread)
-		send_reply = t;
+	if (t) {
+		spin_lock(&t->lock);
+		if (t->to_thread == thread)
+			send_reply = t;
+	}
+	thread->is_dead = true;
+
 	while (t) {
+		last_t = t;
 		active_transactions++;
 		binder_debug(BINDER_DEBUG_DEAD_TRANSACTION,
 			     "release %d:%d transaction %d %s, still active\n",
@@ -3094,12 +4230,16 @@ static int binder_free_thread(struct binder_proc *proc,
 			t = t->from_parent;
 		} else
 			BUG();
+		spin_unlock(&last_t->lock);
+		if (t)
+			spin_lock(&t->lock);
 	}
+	binder_inner_proc_unlock(thread->proc);
+
 	if (send_reply)
 		binder_send_failed_reply(send_reply, BR_DEAD_REPLY);
-	binder_release_work(&thread->todo);
-	kfree(thread);
-	binder_stats_deleted(BINDER_STAT_THREAD);
+	binder_release_work(proc, &thread->todo);
+	binder_thread_dec_tmpref(thread);
 	return active_transactions;
 }
 
@@ -3108,30 +4248,24 @@ static unsigned int binder_poll(struct file *filp,
 {
 	struct binder_proc *proc = filp->private_data;
 	struct binder_thread *thread = NULL;
-	int wait_for_proc_work;
-
-	binder_lock(__func__);
+	bool wait_for_proc_work;
 
 	thread = binder_get_thread(proc);
 
-	wait_for_proc_work = thread->transaction_stack == NULL &&
-		list_empty(&thread->todo) && thread->return_error == BR_OK;
+	binder_inner_proc_lock(thread->proc);
+	thread->looper |= BINDER_LOOPER_STATE_POLL;
+	wait_for_proc_work = binder_available_for_proc_work_ilocked(thread);
 
-	binder_unlock(__func__);
+	binder_inner_proc_unlock(thread->proc);
+
+	if (binder_has_work(thread, wait_for_proc_work))
+		return POLLIN;
+
+	poll_wait(filp, &thread->wait, wait);
+
+	if (binder_has_thread_work(thread))
+		return POLLIN;
 
-	if (wait_for_proc_work) {
-		if (binder_has_proc_work(proc, thread))
-			return POLLIN;
-		poll_wait(filp, &proc->wait, wait);
-		if (binder_has_proc_work(proc, thread))
-			return POLLIN;
-	} else {
-		if (binder_has_thread_work(thread))
-			return POLLIN;
-		poll_wait(filp, &thread->wait, wait);
-		if (binder_has_thread_work(thread))
-			return POLLIN;
-	}
 	return 0;
 }
 
@@ -3178,8 +4312,10 @@ static int binder_ioctl_write_read(struct file *filp,
 					 &bwr.read_consumed,
 					 filp->f_flags & O_NONBLOCK);
 		trace_binder_read_done(ret);
-		if (!list_empty(&proc->todo))
-			wake_up_interruptible(&proc->wait);
+		binder_inner_proc_lock(proc);
+		if (!binder_worklist_empty_ilocked(&proc->todo))
+			binder_wakeup_proc_ilocked(proc);
+		binder_inner_proc_unlock(proc);
 		if (ret < 0) {
 			if (copy_to_user(ubuf, &bwr, sizeof(bwr)))
 				ret = -EFAULT;
@@ -3204,9 +4340,10 @@ static int binder_ioctl_set_ctx_mgr(struct file *filp)
 	int ret = 0;
 	struct binder_proc *proc = filp->private_data;
 	struct binder_context *context = proc->context;
-
+	struct binder_node *new_node;
 	kuid_t curr_euid = current_euid();
 
+	mutex_lock(&context->context_mgr_node_lock);
 	if (context->binder_context_mgr_node) {
 		pr_err("BINDER_SET_CONTEXT_MGR already set\n");
 		ret = -EBUSY;
@@ -3227,19 +4364,49 @@ static int binder_ioctl_set_ctx_mgr(struct file *filp)
 	} else {
 		context->binder_context_mgr_uid = curr_euid;
 	}
-	context->binder_context_mgr_node = binder_new_node(proc, 0, 0);
-	if (!context->binder_context_mgr_node) {
+	new_node = binder_new_node(proc, NULL);
+	if (!new_node) {
 		ret = -ENOMEM;
 		goto out;
 	}
-	context->binder_context_mgr_node->local_weak_refs++;
-	context->binder_context_mgr_node->local_strong_refs++;
-	context->binder_context_mgr_node->has_strong_ref = 1;
-	context->binder_context_mgr_node->has_weak_ref = 1;
+	binder_node_lock(new_node);
+	new_node->local_weak_refs++;
+	new_node->local_strong_refs++;
+	new_node->has_strong_ref = 1;
+	new_node->has_weak_ref = 1;
+	context->binder_context_mgr_node = new_node;
+	binder_node_unlock(new_node);
+	binder_put_node(new_node);
 out:
+	mutex_unlock(&context->context_mgr_node_lock);
 	return ret;
 }
 
+static int binder_ioctl_get_node_debug_info(struct binder_proc *proc,
+				struct binder_node_debug_info *info)
+{
+	struct rb_node *n;
+	binder_uintptr_t ptr = info->ptr;
+
+	memset(info, 0, sizeof(*info));
+
+	binder_inner_proc_lock(proc);
+	for (n = rb_first(&proc->nodes); n != NULL; n = rb_next(n)) {
+		struct binder_node *node = rb_entry(n, struct binder_node,
+						    rb_node);
+		if (node->ptr > ptr) {
+			info->ptr = node->ptr;
+			info->cookie = node->cookie;
+			info->has_strong_ref = node->has_strong_ref;
+			info->has_weak_ref = node->has_weak_ref;
+			break;
+		}
+	}
+	binder_inner_proc_unlock(proc);
+
+	return 0;
+}
+
 static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 {
 	int ret;
@@ -3251,13 +4418,14 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 	/*pr_info("binder_ioctl: %d:%d %x %lx\n",
 			proc->pid, current->pid, cmd, arg);*/
 
+	binder_selftest_alloc(&proc->alloc);
+
 	trace_binder_ioctl(cmd, arg);
 
 	ret = wait_event_interruptible(binder_user_error_wait, binder_stop_on_user_error < 2);
 	if (ret)
 		goto err_unlocked;
 
-	binder_lock(__func__);
 	thread = binder_get_thread(proc);
 	if (thread == NULL) {
 		ret = -ENOMEM;
@@ -3270,12 +4438,19 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 		if (ret)
 			goto err;
 		break;
-	case BINDER_SET_MAX_THREADS:
-		if (copy_from_user(&proc->max_threads, ubuf, sizeof(proc->max_threads))) {
+	case BINDER_SET_MAX_THREADS: {
+		int max_threads;
+
+		if (copy_from_user(&max_threads, ubuf,
+				   sizeof(max_threads))) {
 			ret = -EINVAL;
 			goto err;
 		}
+		binder_inner_proc_lock(proc);
+		proc->max_threads = max_threads;
+		binder_inner_proc_unlock(proc);
 		break;
+	}
 	case BINDER_SET_CONTEXT_MGR:
 		ret = binder_ioctl_set_ctx_mgr(filp);
 		if (ret)
@@ -3284,7 +4459,7 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 	case BINDER_THREAD_EXIT:
 		binder_debug(BINDER_DEBUG_THREADS, "%d:%d exit\n",
 			     proc->pid, thread->pid);
-		binder_free_thread(proc, thread);
+		binder_thread_release(proc, thread);
 		thread = NULL;
 		break;
 	case BINDER_VERSION: {
@@ -3301,6 +4476,24 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 		}
 		break;
 	}
+	case BINDER_GET_NODE_DEBUG_INFO: {
+		struct binder_node_debug_info info;
+
+		if (copy_from_user(&info, ubuf, sizeof(info))) {
+			ret = -EFAULT;
+			goto err;
+		}
+
+		ret = binder_ioctl_get_node_debug_info(proc, &info);
+		if (ret < 0)
+			goto err;
+
+		if (copy_to_user(ubuf, &info, sizeof(info))) {
+			ret = -EFAULT;
+			goto err;
+		}
+		break;
+	}
 	default:
 		ret = -EINVAL;
 		goto err;
@@ -3308,8 +4501,7 @@ static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 	ret = 0;
 err:
 	if (thread)
-		thread->looper &= ~BINDER_LOOPER_STATE_NEED_RETURN;
-	binder_unlock(__func__);
+		thread->looper_need_return = false;
 	wait_event_interruptible(binder_user_error_wait, binder_stop_on_user_error < 2);
 	if (ret && ret != -ERESTARTSYS)
 		pr_info("%d:%d ioctl %x %lx returned %d\n", proc->pid, current->pid, cmd, arg, ret);
@@ -3338,8 +4530,7 @@ static void binder_vma_close(struct vm_area_struct *vma)
 		     proc->pid, vma->vm_start, vma->vm_end,
 		     (vma->vm_end - vma->vm_start) / SZ_1K, vma->vm_flags,
 		     (unsigned long)pgprot_val(vma->vm_page_prot));
-	proc->vma = NULL;
-	proc->vma_vm_mm = NULL;
+	binder_alloc_vma_close(&proc->alloc);
 	binder_defer_work(proc, BINDER_DEFERRED_PUT_FILES);
 }
 
@@ -3357,20 +4548,18 @@ static const struct vm_operations_struct binder_vm_ops = {
 static int binder_mmap(struct file *filp, struct vm_area_struct *vma)
 {
 	int ret;
-	struct vm_struct *area;
 	struct binder_proc *proc = filp->private_data;
 	const char *failure_string;
-	struct binder_buffer *buffer;
 
-	if (proc->tsk != current)
+	if (proc->tsk != current->group_leader)
 		return -EINVAL;
 
 	if ((vma->vm_end - vma->vm_start) > SZ_4M)
 		vma->vm_end = vma->vm_start + SZ_4M;
 
 	binder_debug(BINDER_DEBUG_OPEN_CLOSE,
-		     "binder_mmap: %d %lx-%lx (%ld K) vma %lx pagep %lx\n",
-		     proc->pid, vma->vm_start, vma->vm_end,
+		     "%s: %d %lx-%lx (%ld K) vma %lx pagep %lx\n",
+		     __func__, proc->pid, vma->vm_start, vma->vm_end,
 		     (vma->vm_end - vma->vm_start) / SZ_1K, vma->vm_flags,
 		     (unsigned long)pgprot_val(vma->vm_page_prot));
 
@@ -3380,73 +4569,15 @@ static int binder_mmap(struct file *filp, struct vm_area_struct *vma)
 		goto err_bad_arg;
 	}
 	vma->vm_flags = (vma->vm_flags | VM_DONTCOPY) & ~VM_MAYWRITE;
-
-	mutex_lock(&binder_mmap_lock);
-	if (proc->buffer) {
-		ret = -EBUSY;
-		failure_string = "already mapped";
-		goto err_already_mapped;
-	}
-
-	area = get_vm_area(vma->vm_end - vma->vm_start, VM_IOREMAP);
-	if (area == NULL) {
-		ret = -ENOMEM;
-		failure_string = "get_vm_area";
-		goto err_get_vm_area_failed;
-	}
-	proc->buffer = area->addr;
-	proc->user_buffer_offset = vma->vm_start - (uintptr_t)proc->buffer;
-	mutex_unlock(&binder_mmap_lock);
-
-#ifdef CONFIG_CPU_CACHE_VIPT
-	if (cache_is_vipt_aliasing()) {
-		while (CACHE_COLOUR((vma->vm_start ^ (uint32_t)proc->buffer))) {
-			pr_info("binder_mmap: %d %lx-%lx maps %p bad alignment\n", proc->pid, vma->vm_start, vma->vm_end, proc->buffer);
-			vma->vm_start += PAGE_SIZE;
-		}
-	}
-#endif
-	proc->pages = kzalloc(sizeof(proc->pages[0]) * ((vma->vm_end - vma->vm_start) / PAGE_SIZE), GFP_KERNEL);
-	if (proc->pages == NULL) {
-		ret = -ENOMEM;
-		failure_string = "alloc page array";
-		goto err_alloc_pages_failed;
-	}
-	proc->buffer_size = vma->vm_end - vma->vm_start;
-
 	vma->vm_ops = &binder_vm_ops;
 	vma->vm_private_data = proc;
 
-	if (binder_update_page_range(proc, 1, proc->buffer, proc->buffer + PAGE_SIZE, vma)) {
-		ret = -ENOMEM;
-		failure_string = "alloc small buf";
-		goto err_alloc_small_buf_failed;
-	}
-	buffer = proc->buffer;
-	INIT_LIST_HEAD(&proc->buffers);
-	list_add(&buffer->entry, &proc->buffers);
-	buffer->free = 1;
-	binder_insert_free_buffer(proc, buffer);
-	proc->free_async_space = proc->buffer_size / 2;
-	barrier();
+	ret = binder_alloc_mmap_handler(&proc->alloc, vma);
+	if (ret)
+		return ret;
 	proc->files = get_files_struct(current);
-	proc->vma = vma;
-	proc->vma_vm_mm = vma->vm_mm;
-
-	/*pr_info("binder_mmap: %d %lx-%lx maps %p\n",
-		 proc->pid, vma->vm_start, vma->vm_end, proc->buffer);*/
 	return 0;
 
-err_alloc_small_buf_failed:
-	kfree(proc->pages);
-	proc->pages = NULL;
-err_alloc_pages_failed:
-	mutex_lock(&binder_mmap_lock);
-	vfree(proc->buffer);
-	proc->buffer = NULL;
-err_get_vm_area_failed:
-err_already_mapped:
-	mutex_unlock(&binder_mmap_lock);
 err_bad_arg:
 	pr_err("binder_mmap: %d %lx-%lx %s failed %d\n",
 	       proc->pid, vma->vm_start, vma->vm_end, failure_string, ret);
@@ -3464,24 +4595,26 @@ static int binder_open(struct inode *nodp, struct file *filp)
 	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
 	if (proc == NULL)
 		return -ENOMEM;
+	spin_lock_init(&proc->inner_lock);
+	spin_lock_init(&proc->outer_lock);
 	get_task_struct(current->group_leader);
 	proc->tsk = current->group_leader;
 	INIT_LIST_HEAD(&proc->todo);
-	init_waitqueue_head(&proc->wait);
 	proc->default_priority = task_nice(current);
 	binder_dev = container_of(filp->private_data, struct binder_device,
 				  miscdev);
 	proc->context = &binder_dev->context;
-
-	binder_lock(__func__);
+	binder_alloc_init(&proc->alloc);
 
 	binder_stats_created(BINDER_STAT_PROC);
-	hlist_add_head(&proc->proc_node, &binder_procs);
 	proc->pid = current->group_leader->pid;
 	INIT_LIST_HEAD(&proc->delivered_death);
+	INIT_LIST_HEAD(&proc->waiting_threads);
 	filp->private_data = proc;
 
-	binder_unlock(__func__);
+	mutex_lock(&binder_procs_lock);
+	hlist_add_head(&proc->proc_node, &binder_procs);
+	mutex_unlock(&binder_procs_lock);
 
 	if (binder_debugfs_dir_entry_proc) {
 		char strbuf[11];
@@ -3517,16 +4650,17 @@ static void binder_deferred_flush(struct binder_proc *proc)
 	struct rb_node *n;
 	int wake_count = 0;
 
+	binder_inner_proc_lock(proc);
 	for (n = rb_first(&proc->threads); n != NULL; n = rb_next(n)) {
 		struct binder_thread *thread = rb_entry(n, struct binder_thread, rb_node);
 
-		thread->looper |= BINDER_LOOPER_STATE_NEED_RETURN;
+		thread->looper_need_return = true;
 		if (thread->looper & BINDER_LOOPER_STATE_WAITING) {
 			wake_up_interruptible(&thread->wait);
 			wake_count++;
 		}
 	}
-	wake_up_interruptible_all(&proc->wait);
+	binder_inner_proc_unlock(proc);
 
 	binder_debug(BINDER_DEBUG_OPEN_CLOSE,
 		     "binder_flush: %d woke %d threads\n", proc->pid,
@@ -3547,13 +4681,21 @@ static int binder_node_release(struct binder_node *node, int refs)
 {
 	struct binder_ref *ref;
 	int death = 0;
+	struct binder_proc *proc = node->proc;
 
-	list_del_init(&node->work.entry);
-	binder_release_work(&node->async_todo);
+	binder_release_work(proc, &node->async_todo);
 
-	if (hlist_empty(&node->refs)) {
-		kfree(node);
-		binder_stats_deleted(BINDER_STAT_NODE);
+	binder_node_lock(node);
+	binder_inner_proc_lock(proc);
+	binder_dequeue_work_ilocked(&node->work);
+	/*
+	 * The caller must have taken a temporary ref on the node,
+	 */
+	BUG_ON(!node->tmp_refs);
+	if (hlist_empty(&node->refs) && node->tmp_refs == 1) {
+		binder_inner_proc_unlock(proc);
+		binder_node_unlock(node);
+		binder_free_node(node);
 
 		return refs;
 	}
@@ -3561,45 +4703,58 @@ static int binder_node_release(struct binder_node *node, int refs)
 	node->proc = NULL;
 	node->local_strong_refs = 0;
 	node->local_weak_refs = 0;
+	binder_inner_proc_unlock(proc);
+
+	spin_lock(&binder_dead_nodes_lock);
 	hlist_add_head(&node->dead_node, &binder_dead_nodes);
+	spin_unlock(&binder_dead_nodes_lock);
 
 	hlist_for_each_entry(ref, &node->refs, node_entry) {
 		refs++;
-
-		if (!ref->death)
+		/*
+		 * Need the node lock to synchronize
+		 * with new notification requests and the
+		 * inner lock to synchronize with queued
+		 * death notifications.
+		 */
+		binder_inner_proc_lock(ref->proc);
+		if (!ref->death) {
+			binder_inner_proc_unlock(ref->proc);
 			continue;
+		}
 
 		death++;
 
-		if (list_empty(&ref->death->work.entry)) {
-			ref->death->work.type = BINDER_WORK_DEAD_BINDER;
-			list_add_tail(&ref->death->work.entry,
-				      &ref->proc->todo);
-			wake_up_interruptible(&ref->proc->wait);
-		} else
-			BUG();
+		BUG_ON(!list_empty(&ref->death->work.entry));
+		ref->death->work.type = BINDER_WORK_DEAD_BINDER;
+		binder_enqueue_work_ilocked(&ref->death->work,
+					    &ref->proc->todo);
+		binder_wakeup_proc_ilocked(ref->proc);
+		binder_inner_proc_unlock(ref->proc);
 	}
 
 	binder_debug(BINDER_DEBUG_DEAD_BINDER,
 		     "node %d now dead, refs %d, death %d\n",
 		     node->debug_id, refs, death);
+	binder_node_unlock(node);
+	binder_put_node(node);
 
 	return refs;
 }
 
 static void binder_deferred_release(struct binder_proc *proc)
 {
-	struct binder_transaction *t;
 	struct binder_context *context = proc->context;
 	struct rb_node *n;
-	int threads, nodes, incoming_refs, outgoing_refs, buffers,
-		active_transactions, page_count;
+	int threads, nodes, incoming_refs, outgoing_refs, active_transactions;
 
-	BUG_ON(proc->vma);
 	BUG_ON(proc->files);
 
+	mutex_lock(&binder_procs_lock);
 	hlist_del(&proc->proc_node);
+	mutex_unlock(&binder_procs_lock);
 
+	mutex_lock(&context->context_mgr_node_lock);
 	if (context->binder_context_mgr_node &&
 	    context->binder_context_mgr_node->proc == proc) {
 		binder_debug(BINDER_DEBUG_DEAD_BINDER,
@@ -3607,15 +4762,25 @@ static void binder_deferred_release(struct binder_proc *proc)
 			     __func__, proc->pid);
 		context->binder_context_mgr_node = NULL;
 	}
+	mutex_unlock(&context->context_mgr_node_lock);
+	binder_inner_proc_lock(proc);
+	/*
+	 * Make sure proc stays alive after we
+	 * remove all the threads
+	 */
+	proc->tmp_ref++;
 
+	proc->is_dead = true;
 	threads = 0;
 	active_transactions = 0;
 	while ((n = rb_first(&proc->threads))) {
 		struct binder_thread *thread;
 
 		thread = rb_entry(n, struct binder_thread, rb_node);
+		binder_inner_proc_unlock(proc);
 		threads++;
-		active_transactions += binder_free_thread(proc, thread);
+		active_transactions += binder_thread_release(proc, thread);
+		binder_inner_proc_lock(proc);
 	}
 
 	nodes = 0;
@@ -3625,73 +4790,42 @@ static void binder_deferred_release(struct binder_proc *proc)
 
 		node = rb_entry(n, struct binder_node, rb_node);
 		nodes++;
+		/*
+		 * take a temporary ref on the node before
+		 * calling binder_node_release() which will either
+		 * kfree() the node or call binder_put_node()
+		 */
+		binder_inc_node_tmpref_ilocked(node);
 		rb_erase(&node->rb_node, &proc->nodes);
+		binder_inner_proc_unlock(proc);
 		incoming_refs = binder_node_release(node, incoming_refs);
+		binder_inner_proc_lock(proc);
 	}
+	binder_inner_proc_unlock(proc);
 
 	outgoing_refs = 0;
+	binder_proc_lock(proc);
 	while ((n = rb_first(&proc->refs_by_desc))) {
 		struct binder_ref *ref;
 
 		ref = rb_entry(n, struct binder_ref, rb_node_desc);
 		outgoing_refs++;
-		binder_delete_ref(ref);
+		binder_cleanup_ref_olocked(ref);
+		binder_proc_unlock(proc);
+		binder_free_ref(ref);
+		binder_proc_lock(proc);
 	}
+	binder_proc_unlock(proc);
 
-	binder_release_work(&proc->todo);
-	binder_release_work(&proc->delivered_death);
-
-	buffers = 0;
-	while ((n = rb_first(&proc->allocated_buffers))) {
-		struct binder_buffer *buffer;
-
-		buffer = rb_entry(n, struct binder_buffer, rb_node);
-
-		t = buffer->transaction;
-		if (t) {
-			t->buffer = NULL;
-			buffer->transaction = NULL;
-			pr_err("release proc %d, transaction %d, not freed\n",
-			       proc->pid, t->debug_id);
-			/*BUG();*/
-		}
-
-		binder_free_buf(proc, buffer);
-		buffers++;
-	}
-
-	binder_stats_deleted(BINDER_STAT_PROC);
-
-	page_count = 0;
-	if (proc->pages) {
-		int i;
-
-		for (i = 0; i < proc->buffer_size / PAGE_SIZE; i++) {
-			void *page_addr;
-
-			if (!proc->pages[i])
-				continue;
-
-			page_addr = proc->buffer + i * PAGE_SIZE;
-			binder_debug(BINDER_DEBUG_BUFFER_ALLOC,
-				     "%s: %d: page %d at %p not freed\n",
-				     __func__, proc->pid, i, page_addr);
-			unmap_kernel_range((unsigned long)page_addr, PAGE_SIZE);
-			__free_page(proc->pages[i]);
-			page_count++;
-		}
-		kfree(proc->pages);
-		vfree(proc->buffer);
-	}
-
-	put_task_struct(proc->tsk);
+	binder_release_work(proc, &proc->todo);
+	binder_release_work(proc, &proc->delivered_death);
 
 	binder_debug(BINDER_DEBUG_OPEN_CLOSE,
-		     "%s: %d threads %d, nodes %d (ref %d), refs %d, active transactions %d, buffers %d, pages %d\n",
+		     "%s: %d threads %d, nodes %d (ref %d), refs %d, active transactions %d\n",
 		     __func__, proc->pid, threads, nodes, incoming_refs,
-		     outgoing_refs, active_transactions, buffers, page_count);
+		     outgoing_refs, active_transactions);
 
-	kfree(proc);
+	binder_proc_dec_tmpref(proc);
 }
 
 static void binder_deferred_func(struct work_struct *work)
@@ -3702,7 +4836,6 @@ static void binder_deferred_func(struct work_struct *work)
 	int defer;
 
 	do {
-		binder_lock(__func__);
 		mutex_lock(&binder_deferred_lock);
 		if (!hlist_empty(&binder_deferred_list)) {
 			proc = hlist_entry(binder_deferred_list.first,
@@ -3729,7 +4862,6 @@ static void binder_deferred_func(struct work_struct *work)
 		if (defer & BINDER_DEFERRED_RELEASE)
 			binder_deferred_release(proc); /* frees proc */
 
-		binder_unlock(__func__);
 		if (files)
 			put_files_struct(files);
 	} while (proc);
@@ -3749,41 +4881,51 @@ binder_defer_work(struct binder_proc *proc, enum binder_deferred_state defer)
 	mutex_unlock(&binder_deferred_lock);
 }
 
-static void print_binder_transaction(struct seq_file *m, const char *prefix,
-				     struct binder_transaction *t)
+static void print_binder_transaction_ilocked(struct seq_file *m,
+					     struct binder_proc *proc,
+					     const char *prefix,
+					     struct binder_transaction *t)
 {
+	struct binder_proc *to_proc;
+	struct binder_buffer *buffer = t->buffer;
+
+	spin_lock(&t->lock);
+	to_proc = t->to_proc;
 	seq_printf(m,
 		   "%s %d: %p from %d:%d to %d:%d code %x flags %x pri %ld r%d",
 		   prefix, t->debug_id, t,
 		   t->from ? t->from->proc->pid : 0,
 		   t->from ? t->from->pid : 0,
-		   t->to_proc ? t->to_proc->pid : 0,
+		   to_proc ? to_proc->pid : 0,
 		   t->to_thread ? t->to_thread->pid : 0,
 		   t->code, t->flags, t->priority, t->need_reply);
-	if (t->buffer == NULL) {
+	spin_unlock(&t->lock);
+
+	if (proc != to_proc) {
+		/*
+		 * Can only safely deref buffer if we are holding the
+		 * correct proc inner lock for this node
+		 */
+		seq_puts(m, "\n");
+		return;
+	}
+
+	if (buffer == NULL) {
 		seq_puts(m, " buffer free\n");
 		return;
 	}
-	if (t->buffer->target_node)
-		seq_printf(m, " node %d",
-			   t->buffer->target_node->debug_id);
+	if (buffer->target_node)
+		seq_printf(m, " node %d", buffer->target_node->debug_id);
 	seq_printf(m, " size %zd:%zd data %p\n",
-		   t->buffer->data_size, t->buffer->offsets_size,
-		   t->buffer->data);
-}
-
-static void print_binder_buffer(struct seq_file *m, const char *prefix,
-				struct binder_buffer *buffer)
-{
-	seq_printf(m, "%s %d: %p size %zd:%zd %s\n",
-		   prefix, buffer->debug_id, buffer->data,
 		   buffer->data_size, buffer->offsets_size,
-		   buffer->transaction ? "active" : "delivered");
+		   buffer->data);
 }
 
-static void print_binder_work(struct seq_file *m, const char *prefix,
-			      const char *transaction_prefix,
-			      struct binder_work *w)
+static void print_binder_work_ilocked(struct seq_file *m,
+				     struct binder_proc *proc,
+				     const char *prefix,
+				     const char *transaction_prefix,
+				     struct binder_work *w)
 {
 	struct binder_node *node;
 	struct binder_transaction *t;
@@ -3791,8 +4933,16 @@ static void print_binder_work(struct seq_file *m, const char *prefix,
 	switch (w->type) {
 	case BINDER_WORK_TRANSACTION:
 		t = container_of(w, struct binder_transaction, work);
-		print_binder_transaction(m, transaction_prefix, t);
+		print_binder_transaction_ilocked(
+				m, proc, transaction_prefix, t);
 		break;
+	case BINDER_WORK_RETURN_ERROR: {
+		struct binder_error *e = container_of(
+				w, struct binder_error, work);
+
+		seq_printf(m, "%stransaction error: %u\n",
+			   prefix, e->cmd);
+	} break;
 	case BINDER_WORK_TRANSACTION_COMPLETE:
 		seq_printf(m, "%stransaction complete\n", prefix);
 		break;
@@ -3817,40 +4967,46 @@ static void print_binder_work(struct seq_file *m, const char *prefix,
 	}
 }
 
-static void print_binder_thread(struct seq_file *m,
-				struct binder_thread *thread,
-				int print_always)
+static void print_binder_thread_ilocked(struct seq_file *m,
+					struct binder_thread *thread,
+					int print_always)
 {
 	struct binder_transaction *t;
 	struct binder_work *w;
 	size_t start_pos = m->count;
 	size_t header_pos;
 
-	seq_printf(m, "  thread %d: l %02x\n", thread->pid, thread->looper);
+	seq_printf(m, "  thread %d: l %02x need_return %d tr %d\n",
+			thread->pid, thread->looper,
+			thread->looper_need_return,
+			atomic_read(&thread->tmp_ref));
 	header_pos = m->count;
 	t = thread->transaction_stack;
 	while (t) {
 		if (t->from == thread) {
-			print_binder_transaction(m,
-						 "    outgoing transaction", t);
+			print_binder_transaction_ilocked(m, thread->proc,
+					"    outgoing transaction", t);
 			t = t->from_parent;
 		} else if (t->to_thread == thread) {
-			print_binder_transaction(m,
+			print_binder_transaction_ilocked(m, thread->proc,
 						 "    incoming transaction", t);
 			t = t->to_parent;
 		} else {
-			print_binder_transaction(m, "    bad transaction", t);
+			print_binder_transaction_ilocked(m, thread->proc,
+					"    bad transaction", t);
 			t = NULL;
 		}
 	}
 	list_for_each_entry(w, &thread->todo, entry) {
-		print_binder_work(m, "    ", "    pending transaction", w);
+		print_binder_work_ilocked(m, thread->proc, "    ",
+					  "    pending transaction", w);
 	}
 	if (!print_always && m->count == header_pos)
 		m->count = start_pos;
 }
 
-static void print_binder_node(struct seq_file *m, struct binder_node *node)
+static void print_binder_node_nilocked(struct seq_file *m,
+				       struct binder_node *node)
 {
 	struct binder_ref *ref;
 	struct binder_work *w;
@@ -3860,27 +5016,34 @@ static void print_binder_node(struct seq_file *m, struct binder_node *node)
 	hlist_for_each_entry(ref, &node->refs, node_entry)
 		count++;
 
-	seq_printf(m, "  node %d: u%016llx c%016llx hs %d hw %d ls %d lw %d is %d iw %d",
+	seq_printf(m, "  node %d: u%016llx c%016llx hs %d hw %d ls %d lw %d is %d iw %d tr %d",
 		   node->debug_id, (u64)node->ptr, (u64)node->cookie,
 		   node->has_strong_ref, node->has_weak_ref,
 		   node->local_strong_refs, node->local_weak_refs,
-		   node->internal_strong_refs, count);
+		   node->internal_strong_refs, count, node->tmp_refs);
 	if (count) {
 		seq_puts(m, " proc");
 		hlist_for_each_entry(ref, &node->refs, node_entry)
 			seq_printf(m, " %d", ref->proc->pid);
 	}
 	seq_puts(m, "\n");
-	list_for_each_entry(w, &node->async_todo, entry)
-		print_binder_work(m, "    ",
-				  "    pending async transaction", w);
+	if (node->proc) {
+		list_for_each_entry(w, &node->async_todo, entry)
+			print_binder_work_ilocked(m, node->proc, "    ",
+					  "    pending async transaction", w);
+	}
 }
 
-static void print_binder_ref(struct seq_file *m, struct binder_ref *ref)
+static void print_binder_ref_olocked(struct seq_file *m,
+				     struct binder_ref *ref)
 {
-	seq_printf(m, "  ref %d: desc %d %snode %d s %d w %d d %p\n",
-		   ref->debug_id, ref->desc, ref->node->proc ? "" : "dead ",
-		   ref->node->debug_id, ref->strong, ref->weak, ref->death);
+	binder_node_lock(ref->node);
+	seq_printf(m, "  ref %d: desc %d %snode %d s %d w %d d %pK\n",
+		   ref->data.debug_id, ref->data.desc,
+		   ref->node->proc ? "" : "dead ",
+		   ref->node->debug_id, ref->data.strong,
+		   ref->data.weak, ref->death);
+	binder_node_unlock(ref->node);
 }
 
 static void print_binder_proc(struct seq_file *m,
@@ -3890,36 +5053,60 @@ static void print_binder_proc(struct seq_file *m,
 	struct rb_node *n;
 	size_t start_pos = m->count;
 	size_t header_pos;
+	struct binder_node *last_node = NULL;
 
 	seq_printf(m, "proc %d\n", proc->pid);
 	seq_printf(m, "context %s\n", proc->context->name);
 	header_pos = m->count;
 
+	binder_inner_proc_lock(proc);
 	for (n = rb_first(&proc->threads); n != NULL; n = rb_next(n))
-		print_binder_thread(m, rb_entry(n, struct binder_thread,
+		print_binder_thread_ilocked(m, rb_entry(n, struct binder_thread,
 						rb_node), print_all);
+
 	for (n = rb_first(&proc->nodes); n != NULL; n = rb_next(n)) {
 		struct binder_node *node = rb_entry(n, struct binder_node,
 						    rb_node);
-		if (print_all || node->has_async_transaction)
-			print_binder_node(m, node);
-	}
+		/*
+		 * take a temporary reference on the node so it
+		 * survives and isn't removed from the tree
+		 * while we print it.
+		 */
+		binder_inc_node_tmpref_ilocked(node);
+		/* Need to drop inner lock to take node lock */
+		binder_inner_proc_unlock(proc);
+		if (last_node)
+			binder_put_node(last_node);
+		binder_node_inner_lock(node);
+		print_binder_node_nilocked(m, node);
+		binder_node_inner_unlock(node);
+		last_node = node;
+		binder_inner_proc_lock(proc);
+	}
+	binder_inner_proc_unlock(proc);
+	if (last_node)
+		binder_put_node(last_node);
+
 	if (print_all) {
+		binder_proc_lock(proc);
 		for (n = rb_first(&proc->refs_by_desc);
 		     n != NULL;
 		     n = rb_next(n))
-			print_binder_ref(m, rb_entry(n, struct binder_ref,
-						     rb_node_desc));
+			print_binder_ref_olocked(m, rb_entry(n,
+							    struct binder_ref,
+							    rb_node_desc));
+		binder_proc_unlock(proc);
 	}
-	for (n = rb_first(&proc->allocated_buffers); n != NULL; n = rb_next(n))
-		print_binder_buffer(m, "  buffer",
-				    rb_entry(n, struct binder_buffer, rb_node));
+	binder_alloc_print_allocated(m, &proc->alloc);
+	binder_inner_proc_lock(proc);
 	list_for_each_entry(w, &proc->todo, entry)
-		print_binder_work(m, "  ", "  pending transaction", w);
+		print_binder_work_ilocked(m, proc, "  ",
+					  "  pending transaction", w);
 	list_for_each_entry(w, &proc->delivered_death, entry) {
 		seq_puts(m, "  has delivered dead binder\n");
 		break;
 	}
+	binder_inner_proc_unlock(proc);
 	if (!print_all && m->count == header_pos)
 		m->count = start_pos;
 }
@@ -3985,17 +5172,21 @@ static void print_binder_stats(struct seq_file *m, const char *prefix,
 	BUILD_BUG_ON(ARRAY_SIZE(stats->bc) !=
 		     ARRAY_SIZE(binder_command_strings));
 	for (i = 0; i < ARRAY_SIZE(stats->bc); i++) {
-		if (stats->bc[i])
+		int temp = atomic_read(&stats->bc[i]);
+
+		if (temp)
 			seq_printf(m, "%s%s: %d\n", prefix,
-				   binder_command_strings[i], stats->bc[i]);
+				   binder_command_strings[i], temp);
 	}
 
 	BUILD_BUG_ON(ARRAY_SIZE(stats->br) !=
 		     ARRAY_SIZE(binder_return_strings));
 	for (i = 0; i < ARRAY_SIZE(stats->br); i++) {
-		if (stats->br[i])
+		int temp = atomic_read(&stats->br[i]);
+
+		if (temp)
 			seq_printf(m, "%s%s: %d\n", prefix,
-				   binder_return_strings[i], stats->br[i]);
+				   binder_return_strings[i], temp);
 	}
 
 	BUILD_BUG_ON(ARRAY_SIZE(stats->obj_created) !=
@@ -4003,11 +5194,15 @@ static void print_binder_stats(struct seq_file *m, const char *prefix,
 	BUILD_BUG_ON(ARRAY_SIZE(stats->obj_created) !=
 		     ARRAY_SIZE(stats->obj_deleted));
 	for (i = 0; i < ARRAY_SIZE(stats->obj_created); i++) {
-		if (stats->obj_created[i] || stats->obj_deleted[i])
-			seq_printf(m, "%s%s: active %d total %d\n", prefix,
+		int created = atomic_read(&stats->obj_created[i]);
+		int deleted = atomic_read(&stats->obj_deleted[i]);
+
+		if (created || deleted)
+			seq_printf(m, "%s%s: active %d total %d\n",
+				prefix,
 				binder_objstat_strings[i],
-				stats->obj_created[i] - stats->obj_deleted[i],
-				stats->obj_created[i]);
+				created - deleted,
+				created);
 	}
 }
 
@@ -4015,51 +5210,61 @@ static void print_binder_proc_stats(struct seq_file *m,
 				    struct binder_proc *proc)
 {
 	struct binder_work *w;
+	struct binder_thread *thread;
 	struct rb_node *n;
-	int count, strong, weak;
+	int count, strong, weak, ready_threads;
+	size_t free_async_space =
+		binder_alloc_get_free_async_space(&proc->alloc);
 
 	seq_printf(m, "proc %d\n", proc->pid);
 	seq_printf(m, "context %s\n", proc->context->name);
 	count = 0;
+	ready_threads = 0;
+	binder_inner_proc_lock(proc);
 	for (n = rb_first(&proc->threads); n != NULL; n = rb_next(n))
 		count++;
+
+	list_for_each_entry(thread, &proc->waiting_threads, waiting_thread_node)
+		ready_threads++;
+
 	seq_printf(m, "  threads: %d\n", count);
 	seq_printf(m, "  requested threads: %d+%d/%d\n"
 			"  ready threads %d\n"
 			"  free async space %zd\n", proc->requested_threads,
 			proc->requested_threads_started, proc->max_threads,
-			proc->ready_threads, proc->free_async_space);
+			ready_threads,
+			free_async_space);
 	count = 0;
 	for (n = rb_first(&proc->nodes); n != NULL; n = rb_next(n))
 		count++;
+	binder_inner_proc_unlock(proc);
 	seq_printf(m, "  nodes: %d\n", count);
 	count = 0;
 	strong = 0;
 	weak = 0;
+	binder_proc_lock(proc);
 	for (n = rb_first(&proc->refs_by_desc); n != NULL; n = rb_next(n)) {
 		struct binder_ref *ref = rb_entry(n, struct binder_ref,
 						  rb_node_desc);
 		count++;
-		strong += ref->strong;
-		weak += ref->weak;
+		strong += ref->data.strong;
+		weak += ref->data.weak;
 	}
+	binder_proc_unlock(proc);
 	seq_printf(m, "  refs: %d s %d w %d\n", count, strong, weak);
 
-	count = 0;
-	for (n = rb_first(&proc->allocated_buffers); n != NULL; n = rb_next(n))
-		count++;
+	count = binder_alloc_get_allocated_count(&proc->alloc);
 	seq_printf(m, "  buffers: %d\n", count);
 
+	binder_alloc_print_pages(m, &proc->alloc);
+
 	count = 0;
+	binder_inner_proc_lock(proc);
 	list_for_each_entry(w, &proc->todo, entry) {
-		switch (w->type) {
-		case BINDER_WORK_TRANSACTION:
+		if (w->type == BINDER_WORK_TRANSACTION)
 			count++;
-			break;
-		default:
-			break;
-		}
 	}
+	binder_inner_proc_unlock(proc);
 	seq_printf(m, "  pending transactions: %d\n", count);
 
 	print_binder_stats(m, "  ", &proc->stats);
@@ -4070,57 +5275,67 @@ static int binder_state_show(struct seq_file *m, void *unused)
 {
 	struct binder_proc *proc;
 	struct binder_node *node;
-	int do_lock = !binder_debug_no_lock;
-
-	if (do_lock)
-		binder_lock(__func__);
+	struct binder_node *last_node = NULL;
 
 	seq_puts(m, "binder state:\n");
 
+	spin_lock(&binder_dead_nodes_lock);
 	if (!hlist_empty(&binder_dead_nodes))
 		seq_puts(m, "dead nodes:\n");
-	hlist_for_each_entry(node, &binder_dead_nodes, dead_node)
-		print_binder_node(m, node);
-
+	hlist_for_each_entry(node, &binder_dead_nodes, dead_node) {
+		/*
+		 * take a temporary reference on the node so it
+		 * survives and isn't removed from the list
+		 * while we print it.
+		 */
+		node->tmp_refs++;
+		spin_unlock(&binder_dead_nodes_lock);
+		if (last_node)
+			binder_put_node(last_node);
+		binder_node_lock(node);
+		print_binder_node_nilocked(m, node);
+		binder_node_unlock(node);
+		last_node = node;
+		spin_lock(&binder_dead_nodes_lock);
+	}
+	spin_unlock(&binder_dead_nodes_lock);
+	if (last_node)
+		binder_put_node(last_node);
+
+	mutex_lock(&binder_procs_lock);
 	hlist_for_each_entry(proc, &binder_procs, proc_node)
 		print_binder_proc(m, proc, 1);
-	if (do_lock)
-		binder_unlock(__func__);
+	mutex_unlock(&binder_procs_lock);
+
 	return 0;
 }
 
 static int binder_stats_show(struct seq_file *m, void *unused)
 {
 	struct binder_proc *proc;
-	int do_lock = !binder_debug_no_lock;
-
-	if (do_lock)
-		binder_lock(__func__);
 
 	seq_puts(m, "binder stats:\n");
 
 	print_binder_stats(m, "", &binder_stats);
 
+	mutex_lock(&binder_procs_lock);
 	hlist_for_each_entry(proc, &binder_procs, proc_node)
 		print_binder_proc_stats(m, proc);
-	if (do_lock)
-		binder_unlock(__func__);
+	mutex_unlock(&binder_procs_lock);
+
 	return 0;
 }
 
 static int binder_transactions_show(struct seq_file *m, void *unused)
 {
 	struct binder_proc *proc;
-	int do_lock = !binder_debug_no_lock;
-
-	if (do_lock)
-		binder_lock(__func__);
 
 	seq_puts(m, "binder transactions:\n");
+	mutex_lock(&binder_procs_lock);
 	hlist_for_each_entry(proc, &binder_procs, proc_node)
 		print_binder_proc(m, proc, 0);
-	if (do_lock)
-		binder_unlock(__func__);
+	mutex_unlock(&binder_procs_lock);
+
 	return 0;
 }
 
@@ -4128,44 +5343,63 @@ static int binder_proc_show(struct seq_file *m, void *unused)
 {
 	struct binder_proc *itr;
 	int pid = (unsigned long)m->private;
-	int do_lock = !binder_debug_no_lock;
-
-	if (do_lock)
-		binder_lock(__func__);
 
+	mutex_lock(&binder_procs_lock);
 	hlist_for_each_entry(itr, &binder_procs, proc_node) {
 		if (itr->pid == pid) {
 			seq_puts(m, "binder proc state:\n");
 			print_binder_proc(m, itr, 1);
 		}
 	}
-	if (do_lock)
-		binder_unlock(__func__);
+	mutex_unlock(&binder_procs_lock);
+
 	return 0;
 }
 
 static void print_binder_transaction_log_entry(struct seq_file *m,
 					struct binder_transaction_log_entry *e)
 {
+	int debug_id = READ_ONCE(e->debug_id_done);
+	/*
+	 * read barrier to guarantee debug_id_done read before
+	 * we print the log values
+	 */
+	smp_rmb();
 	seq_printf(m,
-		   "%d: %s from %d:%d to %d:%d context %s node %d handle %d size %d:%d\n",
+		   "%d: %s from %d:%d to %d:%d context %s node %d handle %d size %d:%d ret %d/%d l=%d",
 		   e->debug_id, (e->call_type == 2) ? "reply" :
 		   ((e->call_type == 1) ? "async" : "call "), e->from_proc,
 		   e->from_thread, e->to_proc, e->to_thread, e->context_name,
-		   e->to_node, e->target_handle, e->data_size, e->offsets_size);
+		   e->to_node, e->target_handle, e->data_size, e->offsets_size,
+		   e->return_error, e->return_error_param,
+		   e->return_error_line);
+	/*
+	 * read-barrier to guarantee read of debug_id_done after
+	 * done printing the fields of the entry
+	 */
+	smp_rmb();
+	seq_printf(m, debug_id && debug_id == READ_ONCE(e->debug_id_done) ?
+			"\n" : " (incomplete)\n");
 }
 
 static int binder_transaction_log_show(struct seq_file *m, void *unused)
 {
 	struct binder_transaction_log *log = m->private;
+	unsigned int log_cur = atomic_read(&log->cur);
+	unsigned int count;
+	unsigned int cur;
 	int i;
 
-	if (log->full) {
-		for (i = log->next; i < ARRAY_SIZE(log->entry); i++)
-			print_binder_transaction_log_entry(m, &log->entry[i]);
+	count = log_cur + 1;
+	cur = count < ARRAY_SIZE(log->entry) && !log->full ?
+		0 : count % ARRAY_SIZE(log->entry);
+	if (count > ARRAY_SIZE(log->entry) || log->full)
+		count = ARRAY_SIZE(log->entry);
+	for (i = 0; i < count; i++) {
+		unsigned int index = cur++ % ARRAY_SIZE(log->entry);
+
+		print_binder_transaction_log_entry(m, &log->entry[index]);
 	}
-	for (i = 0; i < log->next; i++)
-		print_binder_transaction_log_entry(m, &log->entry[i]);
 	return 0;
 }
 
@@ -4200,6 +5434,7 @@ static int __init init_binder_device(const char *name)
 
 	binder_device->context.binder_context_mgr_uid = INVALID_UID;
 	binder_device->context.name = name;
+	mutex_init(&binder_device->context.context_mgr_node_lock);
 
 	ret = misc_register(&binder_device->miscdev);
 	if (ret < 0) {
@@ -4215,10 +5450,15 @@ static int __init init_binder_device(const char *name)
 static int __init binder_init(void)
 {
 	int ret;
-	char *device_name, *device_names;
+	char *device_name, *device_names, *device_tmp;
 	struct binder_device *device;
 	struct hlist_node *tmp;
 
+	binder_alloc_shrinker_init();
+
+	atomic_set(&binder_transaction_log.cur, ~0U);
+	atomic_set(&binder_transaction_log_failed.cur, ~0U);
+
 	binder_debugfs_dir_entry_root = debugfs_create_dir("binder", NULL);
 	if (binder_debugfs_dir_entry_root)
 		binder_debugfs_dir_entry_proc = debugfs_create_dir("proc",
@@ -4263,7 +5503,8 @@ static int __init binder_init(void)
 	}
 	strcpy(device_names, binder_devices_param);
 
-	while ((device_name = strsep(&device_names, ","))) {
+	device_tmp = device_names;
+	while ((device_name = strsep(&device_tmp, ","))) {
 		ret = init_binder_device(device_name);
 		if (ret)
 			goto err_init_binder_device_failed;
@@ -4277,6 +5518,9 @@ err_init_binder_device_failed:
 		hlist_del(&device->hlist);
 		kfree(device);
 	}
+
+	kfree(device_names);
+
 err_alloc_device_names_failed:
 	debugfs_remove_recursive(binder_debugfs_dir_entry_root);
 
diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c
new file mode 100644
index 000000000000..8fe165844e47
--- /dev/null
+++ b/drivers/android/binder_alloc.c
@@ -0,0 +1,1009 @@
+/* binder_alloc.c
+ *
+ * Android IPC Subsystem
+ *
+ * Copyright (C) 2007-2017 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <asm/cacheflush.h>
+#include <linux/list.h>
+#include <linux/sched/mm.h>
+#include <linux/module.h>
+#include <linux/rtmutex.h>
+#include <linux/rbtree.h>
+#include <linux/seq_file.h>
+#include <linux/vmalloc.h>
+#include <linux/slab.h>
+#include <linux/sched.h>
+#include <linux/list_lru.h>
+#include "binder_alloc.h"
+#include "binder_trace.h"
+
+struct list_lru binder_alloc_lru;
+
+static DEFINE_MUTEX(binder_alloc_mmap_lock);
+
+enum {
+	BINDER_DEBUG_OPEN_CLOSE             = 1U << 1,
+	BINDER_DEBUG_BUFFER_ALLOC           = 1U << 2,
+	BINDER_DEBUG_BUFFER_ALLOC_ASYNC     = 1U << 3,
+};
+static uint32_t binder_alloc_debug_mask;
+
+module_param_named(debug_mask, binder_alloc_debug_mask,
+		   uint, 0644);
+
+#define binder_alloc_debug(mask, x...) \
+	do { \
+		if (binder_alloc_debug_mask & mask) \
+			pr_info(x); \
+	} while (0)
+
+static struct binder_buffer *binder_buffer_next(struct binder_buffer *buffer)
+{
+	return list_entry(buffer->entry.next, struct binder_buffer, entry);
+}
+
+static struct binder_buffer *binder_buffer_prev(struct binder_buffer *buffer)
+{
+	return list_entry(buffer->entry.prev, struct binder_buffer, entry);
+}
+
+static size_t binder_alloc_buffer_size(struct binder_alloc *alloc,
+				       struct binder_buffer *buffer)
+{
+	if (list_is_last(&buffer->entry, &alloc->buffers))
+		return (u8 *)alloc->buffer +
+			alloc->buffer_size - (u8 *)buffer->data;
+	return (u8 *)binder_buffer_next(buffer)->data - (u8 *)buffer->data;
+}
+
+static void binder_insert_free_buffer(struct binder_alloc *alloc,
+				      struct binder_buffer *new_buffer)
+{
+	struct rb_node **p = &alloc->free_buffers.rb_node;
+	struct rb_node *parent = NULL;
+	struct binder_buffer *buffer;
+	size_t buffer_size;
+	size_t new_buffer_size;
+
+	BUG_ON(!new_buffer->free);
+
+	new_buffer_size = binder_alloc_buffer_size(alloc, new_buffer);
+
+	binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
+		     "%d: add free buffer, size %zd, at %pK\n",
+		      alloc->pid, new_buffer_size, new_buffer);
+
+	while (*p) {
+		parent = *p;
+		buffer = rb_entry(parent, struct binder_buffer, rb_node);
+		BUG_ON(!buffer->free);
+
+		buffer_size = binder_alloc_buffer_size(alloc, buffer);
+
+		if (new_buffer_size < buffer_size)
+			p = &parent->rb_left;
+		else
+			p = &parent->rb_right;
+	}
+	rb_link_node(&new_buffer->rb_node, parent, p);
+	rb_insert_color(&new_buffer->rb_node, &alloc->free_buffers);
+}
+
+static void binder_insert_allocated_buffer_locked(
+		struct binder_alloc *alloc, struct binder_buffer *new_buffer)
+{
+	struct rb_node **p = &alloc->allocated_buffers.rb_node;
+	struct rb_node *parent = NULL;
+	struct binder_buffer *buffer;
+
+	BUG_ON(new_buffer->free);
+
+	while (*p) {
+		parent = *p;
+		buffer = rb_entry(parent, struct binder_buffer, rb_node);
+		BUG_ON(buffer->free);
+
+		if (new_buffer->data < buffer->data)
+			p = &parent->rb_left;
+		else if (new_buffer->data > buffer->data)
+			p = &parent->rb_right;
+		else
+			BUG();
+	}
+	rb_link_node(&new_buffer->rb_node, parent, p);
+	rb_insert_color(&new_buffer->rb_node, &alloc->allocated_buffers);
+}
+
+static struct binder_buffer *binder_alloc_prepare_to_free_locked(
+		struct binder_alloc *alloc,
+		uintptr_t user_ptr)
+{
+	struct rb_node *n = alloc->allocated_buffers.rb_node;
+	struct binder_buffer *buffer;
+	void *kern_ptr;
+
+	kern_ptr = (void *)(user_ptr - alloc->user_buffer_offset);
+
+	while (n) {
+		buffer = rb_entry(n, struct binder_buffer, rb_node);
+		BUG_ON(buffer->free);
+
+		if (kern_ptr < buffer->data)
+			n = n->rb_left;
+		else if (kern_ptr > buffer->data)
+			n = n->rb_right;
+		else {
+			/*
+			 * Guard against user threads attempting to
+			 * free the buffer twice
+			 */
+			if (buffer->free_in_progress) {
+				pr_err("%d:%d FREE_BUFFER u%016llx user freed buffer twice\n",
+				       alloc->pid, current->pid, (u64)user_ptr);
+				return NULL;
+			}
+			buffer->free_in_progress = 1;
+			return buffer;
+		}
+	}
+	return NULL;
+}
+
+/**
+ * binder_alloc_buffer_lookup() - get buffer given user ptr
+ * @alloc:	binder_alloc for this proc
+ * @user_ptr:	User pointer to buffer data
+ *
+ * Validate userspace pointer to buffer data and return buffer corresponding to
+ * that user pointer. Search the rb tree for buffer that matches user data
+ * pointer.
+ *
+ * Return:	Pointer to buffer or NULL
+ */
+struct binder_buffer *binder_alloc_prepare_to_free(struct binder_alloc *alloc,
+						   uintptr_t user_ptr)
+{
+	struct binder_buffer *buffer;
+
+	mutex_lock(&alloc->mutex);
+	buffer = binder_alloc_prepare_to_free_locked(alloc, user_ptr);
+	mutex_unlock(&alloc->mutex);
+	return buffer;
+}
+
+static int binder_update_page_range(struct binder_alloc *alloc, int allocate,
+				    void *start, void *end,
+				    struct vm_area_struct *vma)
+{
+	void *page_addr;
+	unsigned long user_page_addr;
+	struct binder_lru_page *page;
+	struct mm_struct *mm = NULL;
+	bool need_mm = false;
+
+	binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
+		     "%d: %s pages %pK-%pK\n", alloc->pid,
+		     allocate ? "allocate" : "free", start, end);
+
+	if (end <= start)
+		return 0;
+
+	trace_binder_update_page_range(alloc, allocate, start, end);
+
+	if (allocate == 0)
+		goto free_range;
+
+	for (page_addr = start; page_addr < end; page_addr += PAGE_SIZE) {
+		page = &alloc->pages[(page_addr - alloc->buffer) / PAGE_SIZE];
+		if (!page->page_ptr) {
+			need_mm = true;
+			break;
+		}
+	}
+
+	if (!vma && need_mm)
+		mm = get_task_mm(alloc->tsk);
+
+	if (mm) {
+		down_write(&mm->mmap_sem);
+		vma = alloc->vma;
+		if (vma && mm != alloc->vma_vm_mm) {
+			pr_err("%d: vma mm and task mm mismatch\n",
+				alloc->pid);
+			vma = NULL;
+		}
+	}
+
+	if (!vma && need_mm) {
+		pr_err("%d: binder_alloc_buf failed to map pages in userspace, no vma\n",
+			alloc->pid);
+		goto err_no_vma;
+	}
+
+	for (page_addr = start; page_addr < end; page_addr += PAGE_SIZE) {
+		int ret;
+		bool on_lru;
+		size_t index;
+
+		index = (page_addr - alloc->buffer) / PAGE_SIZE;
+		page = &alloc->pages[index];
+
+		if (page->page_ptr) {
+			trace_binder_alloc_lru_start(alloc, index);
+
+			on_lru = list_lru_del(&binder_alloc_lru, &page->lru);
+			WARN_ON(!on_lru);
+
+			trace_binder_alloc_lru_end(alloc, index);
+			continue;
+		}
+
+		if (WARN_ON(!vma))
+			goto err_page_ptr_cleared;
+
+		trace_binder_alloc_page_start(alloc, index);
+		page->page_ptr = alloc_page(GFP_KERNEL |
+					    __GFP_HIGHMEM |
+					    __GFP_ZERO);
+		if (!page->page_ptr) {
+			pr_err("%d: binder_alloc_buf failed for page at %pK\n",
+				alloc->pid, page_addr);
+			goto err_alloc_page_failed;
+		}
+		page->alloc = alloc;
+		INIT_LIST_HEAD(&page->lru);
+
+		ret = map_kernel_range_noflush((unsigned long)page_addr,
+					       PAGE_SIZE, PAGE_KERNEL,
+					       &page->page_ptr);
+		flush_cache_vmap((unsigned long)page_addr,
+				(unsigned long)page_addr + PAGE_SIZE);
+		if (ret != 1) {
+			pr_err("%d: binder_alloc_buf failed to map page at %pK in kernel\n",
+			       alloc->pid, page_addr);
+			goto err_map_kernel_failed;
+		}
+		user_page_addr =
+			(uintptr_t)page_addr + alloc->user_buffer_offset;
+		ret = vm_insert_page(vma, user_page_addr, page[0].page_ptr);
+		if (ret) {
+			pr_err("%d: binder_alloc_buf failed to map page at %lx in userspace\n",
+			       alloc->pid, user_page_addr);
+			goto err_vm_insert_page_failed;
+		}
+
+		trace_binder_alloc_page_end(alloc, index);
+		/* vm_insert_page does not seem to increment the refcount */
+	}
+	if (mm) {
+		up_write(&mm->mmap_sem);
+		mmput(mm);
+	}
+	return 0;
+
+free_range:
+	for (page_addr = end - PAGE_SIZE; page_addr >= start;
+	     page_addr -= PAGE_SIZE) {
+		bool ret;
+		size_t index;
+
+		index = (page_addr - alloc->buffer) / PAGE_SIZE;
+		page = &alloc->pages[index];
+
+		trace_binder_free_lru_start(alloc, index);
+
+		ret = list_lru_add(&binder_alloc_lru, &page->lru);
+		WARN_ON(!ret);
+
+		trace_binder_free_lru_end(alloc, index);
+		continue;
+
+err_vm_insert_page_failed:
+		unmap_kernel_range((unsigned long)page_addr, PAGE_SIZE);
+err_map_kernel_failed:
+		__free_page(page->page_ptr);
+		page->page_ptr = NULL;
+err_alloc_page_failed:
+err_page_ptr_cleared:
+		;
+	}
+err_no_vma:
+	if (mm) {
+		up_write(&mm->mmap_sem);
+		mmput(mm);
+	}
+	return vma ? -ENOMEM : -ESRCH;
+}
+
+struct binder_buffer *binder_alloc_new_buf_locked(struct binder_alloc *alloc,
+						  size_t data_size,
+						  size_t offsets_size,
+						  size_t extra_buffers_size,
+						  int is_async)
+{
+	struct rb_node *n = alloc->free_buffers.rb_node;
+	struct binder_buffer *buffer;
+	size_t buffer_size;
+	struct rb_node *best_fit = NULL;
+	void *has_page_addr;
+	void *end_page_addr;
+	size_t size, data_offsets_size;
+	int ret;
+
+	if (alloc->vma == NULL) {
+		pr_err("%d: binder_alloc_buf, no vma\n",
+		       alloc->pid);
+		return ERR_PTR(-ESRCH);
+	}
+
+	data_offsets_size = ALIGN(data_size, sizeof(void *)) +
+		ALIGN(offsets_size, sizeof(void *));
+
+	if (data_offsets_size < data_size || data_offsets_size < offsets_size) {
+		binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
+				"%d: got transaction with invalid size %zd-%zd\n",
+				alloc->pid, data_size, offsets_size);
+		return ERR_PTR(-EINVAL);
+	}
+	size = data_offsets_size + ALIGN(extra_buffers_size, sizeof(void *));
+	if (size < data_offsets_size || size < extra_buffers_size) {
+		binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
+				"%d: got transaction with invalid extra_buffers_size %zd\n",
+				alloc->pid, extra_buffers_size);
+		return ERR_PTR(-EINVAL);
+	}
+	if (is_async &&
+	    alloc->free_async_space < size + sizeof(struct binder_buffer)) {
+		binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
+			     "%d: binder_alloc_buf size %zd failed, no async space left\n",
+			      alloc->pid, size);
+		return ERR_PTR(-ENOSPC);
+	}
+
+	/* Pad 0-size buffers so they get assigned unique addresses */
+	size = max(size, sizeof(void *));
+
+	while (n) {
+		buffer = rb_entry(n, struct binder_buffer, rb_node);
+		BUG_ON(!buffer->free);
+		buffer_size = binder_alloc_buffer_size(alloc, buffer);
+
+		if (size < buffer_size) {
+			best_fit = n;
+			n = n->rb_left;
+		} else if (size > buffer_size)
+			n = n->rb_right;
+		else {
+			best_fit = n;
+			break;
+		}
+	}
+	if (best_fit == NULL) {
+		size_t allocated_buffers = 0;
+		size_t largest_alloc_size = 0;
+		size_t total_alloc_size = 0;
+		size_t free_buffers = 0;
+		size_t largest_free_size = 0;
+		size_t total_free_size = 0;
+
+		for (n = rb_first(&alloc->allocated_buffers); n != NULL;
+		     n = rb_next(n)) {
+			buffer = rb_entry(n, struct binder_buffer, rb_node);
+			buffer_size = binder_alloc_buffer_size(alloc, buffer);
+			allocated_buffers++;
+			total_alloc_size += buffer_size;
+			if (buffer_size > largest_alloc_size)
+				largest_alloc_size = buffer_size;
+		}
+		for (n = rb_first(&alloc->free_buffers); n != NULL;
+		     n = rb_next(n)) {
+			buffer = rb_entry(n, struct binder_buffer, rb_node);
+			buffer_size = binder_alloc_buffer_size(alloc, buffer);
+			free_buffers++;
+			total_free_size += buffer_size;
+			if (buffer_size > largest_free_size)
+				largest_free_size = buffer_size;
+		}
+		pr_err("%d: binder_alloc_buf size %zd failed, no address space\n",
+			alloc->pid, size);
+		pr_err("allocated: %zd (num: %zd largest: %zd), free: %zd (num: %zd largest: %zd)\n",
+		       total_alloc_size, allocated_buffers, largest_alloc_size,
+		       total_free_size, free_buffers, largest_free_size);
+		return ERR_PTR(-ENOSPC);
+	}
+	if (n == NULL) {
+		buffer = rb_entry(best_fit, struct binder_buffer, rb_node);
+		buffer_size = binder_alloc_buffer_size(alloc, buffer);
+	}
+
+	binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
+		     "%d: binder_alloc_buf size %zd got buffer %pK size %zd\n",
+		      alloc->pid, size, buffer, buffer_size);
+
+	has_page_addr =
+		(void *)(((uintptr_t)buffer->data + buffer_size) & PAGE_MASK);
+	WARN_ON(n && buffer_size != size);
+	end_page_addr =
+		(void *)PAGE_ALIGN((uintptr_t)buffer->data + size);
+	if (end_page_addr > has_page_addr)
+		end_page_addr = has_page_addr;
+	ret = binder_update_page_range(alloc, 1,
+	    (void *)PAGE_ALIGN((uintptr_t)buffer->data), end_page_addr, NULL);
+	if (ret)
+		return ERR_PTR(ret);
+
+	if (buffer_size != size) {
+		struct binder_buffer *new_buffer;
+
+		new_buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
+		if (!new_buffer) {
+			pr_err("%s: %d failed to alloc new buffer struct\n",
+			       __func__, alloc->pid);
+			goto err_alloc_buf_struct_failed;
+		}
+		new_buffer->data = (u8 *)buffer->data + size;
+		list_add(&new_buffer->entry, &buffer->entry);
+		new_buffer->free = 1;
+		binder_insert_free_buffer(alloc, new_buffer);
+	}
+
+	rb_erase(best_fit, &alloc->free_buffers);
+	buffer->free = 0;
+	buffer->free_in_progress = 0;
+	binder_insert_allocated_buffer_locked(alloc, buffer);
+	binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
+		     "%d: binder_alloc_buf size %zd got %pK\n",
+		      alloc->pid, size, buffer);
+	buffer->data_size = data_size;
+	buffer->offsets_size = offsets_size;
+	buffer->async_transaction = is_async;
+	buffer->extra_buffers_size = extra_buffers_size;
+	if (is_async) {
+		alloc->free_async_space -= size + sizeof(struct binder_buffer);
+		binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC_ASYNC,
+			     "%d: binder_alloc_buf size %zd async free %zd\n",
+			      alloc->pid, size, alloc->free_async_space);
+	}
+	return buffer;
+
+err_alloc_buf_struct_failed:
+	binder_update_page_range(alloc, 0,
+				 (void *)PAGE_ALIGN((uintptr_t)buffer->data),
+				 end_page_addr, NULL);
+	return ERR_PTR(-ENOMEM);
+}
+
+/**
+ * binder_alloc_new_buf() - Allocate a new binder buffer
+ * @alloc:              binder_alloc for this proc
+ * @data_size:          size of user data buffer
+ * @offsets_size:       user specified buffer offset
+ * @extra_buffers_size: size of extra space for meta-data (eg, security context)
+ * @is_async:           buffer for async transaction
+ *
+ * Allocate a new buffer given the requested sizes. Returns
+ * the kernel version of the buffer pointer. The size allocated
+ * is the sum of the three given sizes (each rounded up to
+ * pointer-sized boundary)
+ *
+ * Return:	The allocated buffer or %NULL if error
+ */
+struct binder_buffer *binder_alloc_new_buf(struct binder_alloc *alloc,
+					   size_t data_size,
+					   size_t offsets_size,
+					   size_t extra_buffers_size,
+					   int is_async)
+{
+	struct binder_buffer *buffer;
+
+	mutex_lock(&alloc->mutex);
+	buffer = binder_alloc_new_buf_locked(alloc, data_size, offsets_size,
+					     extra_buffers_size, is_async);
+	mutex_unlock(&alloc->mutex);
+	return buffer;
+}
+
+static void *buffer_start_page(struct binder_buffer *buffer)
+{
+	return (void *)((uintptr_t)buffer->data & PAGE_MASK);
+}
+
+static void *prev_buffer_end_page(struct binder_buffer *buffer)
+{
+	return (void *)(((uintptr_t)(buffer->data) - 1) & PAGE_MASK);
+}
+
+static void binder_delete_free_buffer(struct binder_alloc *alloc,
+				      struct binder_buffer *buffer)
+{
+	struct binder_buffer *prev, *next = NULL;
+	bool to_free = true;
+	BUG_ON(alloc->buffers.next == &buffer->entry);
+	prev = binder_buffer_prev(buffer);
+	BUG_ON(!prev->free);
+	if (prev_buffer_end_page(prev) == buffer_start_page(buffer)) {
+		to_free = false;
+		binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
+				   "%d: merge free, buffer %pK share page with %pK\n",
+				   alloc->pid, buffer->data, prev->data);
+	}
+
+	if (!list_is_last(&buffer->entry, &alloc->buffers)) {
+		next = binder_buffer_next(buffer);
+		if (buffer_start_page(next) == buffer_start_page(buffer)) {
+			to_free = false;
+			binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
+					   "%d: merge free, buffer %pK share page with %pK\n",
+					   alloc->pid,
+					   buffer->data,
+					   next->data);
+		}
+	}
+
+	if (PAGE_ALIGNED(buffer->data)) {
+		binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
+				   "%d: merge free, buffer start %pK is page aligned\n",
+				   alloc->pid, buffer->data);
+		to_free = false;
+	}
+
+	if (to_free) {
+		binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
+				   "%d: merge free, buffer %pK do not share page with %pK or %pK\n",
+				   alloc->pid, buffer->data,
+				   prev->data, next->data);
+		binder_update_page_range(alloc, 0, buffer_start_page(buffer),
+					 buffer_start_page(buffer) + PAGE_SIZE,
+					 NULL);
+	}
+	list_del(&buffer->entry);
+	kfree(buffer);
+}
+
+static void binder_free_buf_locked(struct binder_alloc *alloc,
+				   struct binder_buffer *buffer)
+{
+	size_t size, buffer_size;
+
+	buffer_size = binder_alloc_buffer_size(alloc, buffer);
+
+	size = ALIGN(buffer->data_size, sizeof(void *)) +
+		ALIGN(buffer->offsets_size, sizeof(void *)) +
+		ALIGN(buffer->extra_buffers_size, sizeof(void *));
+
+	binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
+		     "%d: binder_free_buf %pK size %zd buffer_size %zd\n",
+		      alloc->pid, buffer, size, buffer_size);
+
+	BUG_ON(buffer->free);
+	BUG_ON(size > buffer_size);
+	BUG_ON(buffer->transaction != NULL);
+	BUG_ON(buffer->data < alloc->buffer);
+	BUG_ON(buffer->data > alloc->buffer + alloc->buffer_size);
+
+	if (buffer->async_transaction) {
+		alloc->free_async_space += size + sizeof(struct binder_buffer);
+
+		binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC_ASYNC,
+			     "%d: binder_free_buf size %zd async free %zd\n",
+			      alloc->pid, size, alloc->free_async_space);
+	}
+
+	binder_update_page_range(alloc, 0,
+		(void *)PAGE_ALIGN((uintptr_t)buffer->data),
+		(void *)(((uintptr_t)buffer->data + buffer_size) & PAGE_MASK),
+		NULL);
+
+	rb_erase(&buffer->rb_node, &alloc->allocated_buffers);
+	buffer->free = 1;
+	if (!list_is_last(&buffer->entry, &alloc->buffers)) {
+		struct binder_buffer *next = binder_buffer_next(buffer);
+
+		if (next->free) {
+			rb_erase(&next->rb_node, &alloc->free_buffers);
+			binder_delete_free_buffer(alloc, next);
+		}
+	}
+	if (alloc->buffers.next != &buffer->entry) {
+		struct binder_buffer *prev = binder_buffer_prev(buffer);
+
+		if (prev->free) {
+			binder_delete_free_buffer(alloc, buffer);
+			rb_erase(&prev->rb_node, &alloc->free_buffers);
+			buffer = prev;
+		}
+	}
+	binder_insert_free_buffer(alloc, buffer);
+}
+
+/**
+ * binder_alloc_free_buf() - free a binder buffer
+ * @alloc:	binder_alloc for this proc
+ * @buffer:	kernel pointer to buffer
+ *
+ * Free the buffer allocated via binder_alloc_new_buffer()
+ */
+void binder_alloc_free_buf(struct binder_alloc *alloc,
+			    struct binder_buffer *buffer)
+{
+	mutex_lock(&alloc->mutex);
+	binder_free_buf_locked(alloc, buffer);
+	mutex_unlock(&alloc->mutex);
+}
+
+/**
+ * binder_alloc_mmap_handler() - map virtual address space for proc
+ * @alloc:	alloc structure for this proc
+ * @vma:	vma passed to mmap()
+ *
+ * Called by binder_mmap() to initialize the space specified in
+ * vma for allocating binder buffers
+ *
+ * Return:
+ *      0 = success
+ *      -EBUSY = address space already mapped
+ *      -ENOMEM = failed to map memory to given address space
+ */
+int binder_alloc_mmap_handler(struct binder_alloc *alloc,
+			      struct vm_area_struct *vma)
+{
+	int ret;
+	struct vm_struct *area;
+	const char *failure_string;
+	struct binder_buffer *buffer;
+
+	mutex_lock(&binder_alloc_mmap_lock);
+	if (alloc->buffer) {
+		ret = -EBUSY;
+		failure_string = "already mapped";
+		goto err_already_mapped;
+	}
+
+	area = get_vm_area(vma->vm_end - vma->vm_start, VM_IOREMAP);
+	if (area == NULL) {
+		ret = -ENOMEM;
+		failure_string = "get_vm_area";
+		goto err_get_vm_area_failed;
+	}
+	alloc->buffer = area->addr;
+	alloc->user_buffer_offset =
+		vma->vm_start - (uintptr_t)alloc->buffer;
+	mutex_unlock(&binder_alloc_mmap_lock);
+
+#ifdef CONFIG_CPU_CACHE_VIPT
+	if (cache_is_vipt_aliasing()) {
+		while (CACHE_COLOUR(
+				(vma->vm_start ^ (uint32_t)alloc->buffer))) {
+			pr_info("%s: %d %lx-%lx maps %pK bad alignment\n",
+				__func__, alloc->pid, vma->vm_start,
+				vma->vm_end, alloc->buffer);
+			vma->vm_start += PAGE_SIZE;
+		}
+	}
+#endif
+	alloc->pages = kzalloc(sizeof(alloc->pages[0]) *
+				   ((vma->vm_end - vma->vm_start) / PAGE_SIZE),
+			       GFP_KERNEL);
+	if (alloc->pages == NULL) {
+		ret = -ENOMEM;
+		failure_string = "alloc page array";
+		goto err_alloc_pages_failed;
+	}
+	alloc->buffer_size = vma->vm_end - vma->vm_start;
+
+	buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
+	if (!buffer) {
+		ret = -ENOMEM;
+		failure_string = "alloc buffer struct";
+		goto err_alloc_buf_struct_failed;
+	}
+
+	buffer->data = alloc->buffer;
+	list_add(&buffer->entry, &alloc->buffers);
+	buffer->free = 1;
+	binder_insert_free_buffer(alloc, buffer);
+	alloc->free_async_space = alloc->buffer_size / 2;
+	barrier();
+	alloc->vma = vma;
+	alloc->vma_vm_mm = vma->vm_mm;
+
+	return 0;
+
+err_alloc_buf_struct_failed:
+	kfree(alloc->pages);
+	alloc->pages = NULL;
+err_alloc_pages_failed:
+	mutex_lock(&binder_alloc_mmap_lock);
+	vfree(alloc->buffer);
+	alloc->buffer = NULL;
+err_get_vm_area_failed:
+err_already_mapped:
+	mutex_unlock(&binder_alloc_mmap_lock);
+	pr_err("%s: %d %lx-%lx %s failed %d\n", __func__,
+	       alloc->pid, vma->vm_start, vma->vm_end, failure_string, ret);
+	return ret;
+}
+
+
+void binder_alloc_deferred_release(struct binder_alloc *alloc)
+{
+	struct rb_node *n;
+	int buffers, page_count;
+	struct binder_buffer *buffer;
+
+	BUG_ON(alloc->vma);
+
+	buffers = 0;
+	mutex_lock(&alloc->mutex);
+	while ((n = rb_first(&alloc->allocated_buffers))) {
+		buffer = rb_entry(n, struct binder_buffer, rb_node);
+
+		/* Transaction should already have been freed */
+		BUG_ON(buffer->transaction);
+
+		binder_free_buf_locked(alloc, buffer);
+		buffers++;
+	}
+
+	while (!list_empty(&alloc->buffers)) {
+		buffer = list_first_entry(&alloc->buffers,
+					  struct binder_buffer, entry);
+		WARN_ON(!buffer->free);
+
+		list_del(&buffer->entry);
+		WARN_ON_ONCE(!list_empty(&alloc->buffers));
+		kfree(buffer);
+	}
+
+	page_count = 0;
+	if (alloc->pages) {
+		int i;
+
+		for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) {
+			void *page_addr;
+			bool on_lru;
+
+			if (!alloc->pages[i].page_ptr)
+				continue;
+
+			on_lru = list_lru_del(&binder_alloc_lru,
+					      &alloc->pages[i].lru);
+			page_addr = alloc->buffer + i * PAGE_SIZE;
+			binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
+				     "%s: %d: page %d at %pK %s\n",
+				     __func__, alloc->pid, i, page_addr,
+				     on_lru ? "on lru" : "active");
+			unmap_kernel_range((unsigned long)page_addr, PAGE_SIZE);
+			__free_page(alloc->pages[i].page_ptr);
+			page_count++;
+		}
+		kfree(alloc->pages);
+		vfree(alloc->buffer);
+	}
+	mutex_unlock(&alloc->mutex);
+
+	binder_alloc_debug(BINDER_DEBUG_OPEN_CLOSE,
+		     "%s: %d buffers %d, pages %d\n",
+		     __func__, alloc->pid, buffers, page_count);
+}
+
+static void print_binder_buffer(struct seq_file *m, const char *prefix,
+				struct binder_buffer *buffer)
+{
+	seq_printf(m, "%s %d: %pK size %zd:%zd:%zd %s\n",
+		   prefix, buffer->debug_id, buffer->data,
+		   buffer->data_size, buffer->offsets_size,
+		   buffer->extra_buffers_size,
+		   buffer->transaction ? "active" : "delivered");
+}
+
+/**
+ * binder_alloc_print_allocated() - print buffer info
+ * @m:     seq_file for output via seq_printf()
+ * @alloc: binder_alloc for this proc
+ *
+ * Prints information about every buffer associated with
+ * the binder_alloc state to the given seq_file
+ */
+void binder_alloc_print_allocated(struct seq_file *m,
+				  struct binder_alloc *alloc)
+{
+	struct rb_node *n;
+
+	mutex_lock(&alloc->mutex);
+	for (n = rb_first(&alloc->allocated_buffers); n != NULL; n = rb_next(n))
+		print_binder_buffer(m, "  buffer",
+				    rb_entry(n, struct binder_buffer, rb_node));
+	mutex_unlock(&alloc->mutex);
+}
+
+/**
+ * binder_alloc_print_pages() - print page usage
+ * @m:     seq_file for output via seq_printf()
+ * @alloc: binder_alloc for this proc
+ */
+void binder_alloc_print_pages(struct seq_file *m,
+			      struct binder_alloc *alloc)
+{
+	struct binder_lru_page *page;
+	int i;
+	int active = 0;
+	int lru = 0;
+	int free = 0;
+
+	mutex_lock(&alloc->mutex);
+	for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) {
+		page = &alloc->pages[i];
+		if (!page->page_ptr)
+			free++;
+		else if (list_empty(&page->lru))
+			active++;
+		else
+			lru++;
+	}
+	mutex_unlock(&alloc->mutex);
+	seq_printf(m, "  pages: %d:%d:%d\n", active, lru, free);
+}
+
+/**
+ * binder_alloc_get_allocated_count() - return count of buffers
+ * @alloc: binder_alloc for this proc
+ *
+ * Return: count of allocated buffers
+ */
+int binder_alloc_get_allocated_count(struct binder_alloc *alloc)
+{
+	struct rb_node *n;
+	int count = 0;
+
+	mutex_lock(&alloc->mutex);
+	for (n = rb_first(&alloc->allocated_buffers); n != NULL; n = rb_next(n))
+		count++;
+	mutex_unlock(&alloc->mutex);
+	return count;
+}
+
+
+/**
+ * binder_alloc_vma_close() - invalidate address space
+ * @alloc: binder_alloc for this proc
+ *
+ * Called from binder_vma_close() when releasing address space.
+ * Clears alloc->vma to prevent new incoming transactions from
+ * allocating more buffers.
+ */
+void binder_alloc_vma_close(struct binder_alloc *alloc)
+{
+	WRITE_ONCE(alloc->vma, NULL);
+	WRITE_ONCE(alloc->vma_vm_mm, NULL);
+}
+
+/**
+ * binder_alloc_free_page() - shrinker callback to free pages
+ * @item:   item to free
+ * @lock:   lock protecting the item
+ * @cb_arg: callback argument
+ *
+ * Called from list_lru_walk() in binder_shrink_scan() to free
+ * up pages when the system is under memory pressure.
+ */
+enum lru_status binder_alloc_free_page(struct list_head *item,
+				       struct list_lru_one *lru,
+				       spinlock_t *lock,
+				       void *cb_arg)
+{
+	struct mm_struct *mm = NULL;
+	struct binder_lru_page *page = container_of(item,
+						    struct binder_lru_page,
+						    lru);
+	struct binder_alloc *alloc;
+	uintptr_t page_addr;
+	size_t index;
+
+	alloc = page->alloc;
+	if (!mutex_trylock(&alloc->mutex))
+		goto err_get_alloc_mutex_failed;
+
+	if (!page->page_ptr)
+		goto err_page_already_freed;
+
+	index = page - alloc->pages;
+	page_addr = (uintptr_t)alloc->buffer + index * PAGE_SIZE;
+	if (alloc->vma) {
+		mm = get_task_mm(alloc->tsk);
+		if (!mm)
+			goto err_get_task_mm_failed;
+		if (!down_write_trylock(&mm->mmap_sem))
+			goto err_down_write_mmap_sem_failed;
+
+		trace_binder_unmap_user_start(alloc, index);
+
+		zap_page_range(alloc->vma,
+			       page_addr + alloc->user_buffer_offset,
+			       PAGE_SIZE);
+
+		trace_binder_unmap_user_end(alloc, index);
+
+		up_write(&mm->mmap_sem);
+		mmput(mm);
+	}
+
+	trace_binder_unmap_kernel_start(alloc, index);
+
+	unmap_kernel_range(page_addr, PAGE_SIZE);
+	__free_page(page->page_ptr);
+	page->page_ptr = NULL;
+
+	trace_binder_unmap_kernel_end(alloc, index);
+
+	list_lru_isolate(lru, item);
+
+	mutex_unlock(&alloc->mutex);
+	return LRU_REMOVED;
+
+err_down_write_mmap_sem_failed:
+	mmput(mm);
+err_get_task_mm_failed:
+err_page_already_freed:
+	mutex_unlock(&alloc->mutex);
+err_get_alloc_mutex_failed:
+	return LRU_SKIP;
+}
+
+static unsigned long
+binder_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
+{
+	unsigned long ret = list_lru_count(&binder_alloc_lru);
+	return ret;
+}
+
+static unsigned long
+binder_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
+{
+	unsigned long ret;
+
+	ret = list_lru_walk(&binder_alloc_lru, binder_alloc_free_page,
+			    NULL, sc->nr_to_scan);
+	return ret;
+}
+
+struct shrinker binder_shrinker = {
+	.count_objects = binder_shrink_count,
+	.scan_objects = binder_shrink_scan,
+	.seeks = DEFAULT_SEEKS,
+};
+
+/**
+ * binder_alloc_init() - called by binder_open() for per-proc initialization
+ * @alloc: binder_alloc for this proc
+ *
+ * Called from binder_open() to initialize binder_alloc fields for
+ * new binder proc
+ */
+void binder_alloc_init(struct binder_alloc *alloc)
+{
+	alloc->tsk = current->group_leader;
+	alloc->pid = current->group_leader->pid;
+	mutex_init(&alloc->mutex);
+	INIT_LIST_HEAD(&alloc->buffers);
+}
+
+void binder_alloc_shrinker_init(void)
+{
+	list_lru_init(&binder_alloc_lru);
+	register_shrinker(&binder_shrinker);
+}
diff --git a/drivers/android/binder_alloc.h b/drivers/android/binder_alloc.h
new file mode 100644
index 000000000000..a3a3602c689c
--- /dev/null
+++ b/drivers/android/binder_alloc.h
@@ -0,0 +1,187 @@
+/*
+ * Copyright (C) 2017 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#ifndef _LINUX_BINDER_ALLOC_H
+#define _LINUX_BINDER_ALLOC_H
+
+#include <linux/rbtree.h>
+#include <linux/list.h>
+#include <linux/mm.h>
+#include <linux/rtmutex.h>
+#include <linux/vmalloc.h>
+#include <linux/slab.h>
+#include <linux/list_lru.h>
+
+extern struct list_lru binder_alloc_lru;
+struct binder_transaction;
+
+/**
+ * struct binder_buffer - buffer used for binder transactions
+ * @entry:              entry alloc->buffers
+ * @rb_node:            node for allocated_buffers/free_buffers rb trees
+ * @free:               true if buffer is free
+ * @allow_user_free:    describe the second member of struct blah,
+ * @async_transaction:  describe the second member of struct blah,
+ * @debug_id:           describe the second member of struct blah,
+ * @transaction:        describe the second member of struct blah,
+ * @target_node:        describe the second member of struct blah,
+ * @data_size:          describe the second member of struct blah,
+ * @offsets_size:       describe the second member of struct blah,
+ * @extra_buffers_size: describe the second member of struct blah,
+ * @data:i              describe the second member of struct blah,
+ *
+ * Bookkeeping structure for binder transaction buffers
+ */
+struct binder_buffer {
+	struct list_head entry; /* free and allocated entries by address */
+	struct rb_node rb_node; /* free entry by size or allocated entry */
+				/* by address */
+	unsigned free:1;
+	unsigned allow_user_free:1;
+	unsigned async_transaction:1;
+	unsigned free_in_progress:1;
+	unsigned debug_id:28;
+
+	struct binder_transaction *transaction;
+
+	struct binder_node *target_node;
+	size_t data_size;
+	size_t offsets_size;
+	size_t extra_buffers_size;
+	void *data;
+};
+
+/**
+ * struct binder_lru_page - page object used for binder shrinker
+ * @page_ptr: pointer to physical page in mmap'd space
+ * @lru:      entry in binder_alloc_lru
+ * @alloc:    binder_alloc for a proc
+ */
+struct binder_lru_page {
+	struct list_head lru;
+	struct page *page_ptr;
+	struct binder_alloc *alloc;
+};
+
+/**
+ * struct binder_alloc - per-binder proc state for binder allocator
+ * @vma:                vm_area_struct passed to mmap_handler
+ *                      (invarient after mmap)
+ * @tsk:                tid for task that called init for this proc
+ *                      (invariant after init)
+ * @vma_vm_mm:          copy of vma->vm_mm (invarient after mmap)
+ * @buffer:             base of per-proc address space mapped via mmap
+ * @user_buffer_offset: offset between user and kernel VAs for buffer
+ * @buffers:            list of all buffers for this proc
+ * @free_buffers:       rb tree of buffers available for allocation
+ *                      sorted by size
+ * @allocated_buffers:  rb tree of allocated buffers sorted by address
+ * @free_async_space:   VA space available for async buffers. This is
+ *                      initialized at mmap time to 1/2 the full VA space
+ * @pages:              array of binder_lru_page
+ * @buffer_size:        size of address space specified via mmap
+ * @pid:                pid for associated binder_proc (invariant after init)
+ *
+ * Bookkeeping structure for per-proc address space management for binder
+ * buffers. It is normally initialized during binder_init() and binder_mmap()
+ * calls. The address space is used for both user-visible buffers and for
+ * struct binder_buffer objects used to track the user buffers
+ */
+struct binder_alloc {
+	struct mutex mutex;
+	struct task_struct *tsk;
+	struct vm_area_struct *vma;
+	struct mm_struct *vma_vm_mm;
+	void *buffer;
+	ptrdiff_t user_buffer_offset;
+	struct list_head buffers;
+	struct rb_root free_buffers;
+	struct rb_root allocated_buffers;
+	size_t free_async_space;
+	struct binder_lru_page *pages;
+	size_t buffer_size;
+	uint32_t buffer_free;
+	int pid;
+};
+
+#ifdef CONFIG_ANDROID_BINDER_IPC_SELFTEST
+void binder_selftest_alloc(struct binder_alloc *alloc);
+#else
+static inline void binder_selftest_alloc(struct binder_alloc *alloc) {}
+#endif
+enum lru_status binder_alloc_free_page(struct list_head *item,
+				       struct list_lru_one *lru,
+				       spinlock_t *lock, void *cb_arg);
+extern struct binder_buffer *binder_alloc_new_buf(struct binder_alloc *alloc,
+						  size_t data_size,
+						  size_t offsets_size,
+						  size_t extra_buffers_size,
+						  int is_async);
+extern void binder_alloc_init(struct binder_alloc *alloc);
+void binder_alloc_shrinker_init(void);
+extern void binder_alloc_vma_close(struct binder_alloc *alloc);
+extern struct binder_buffer *
+binder_alloc_prepare_to_free(struct binder_alloc *alloc,
+			     uintptr_t user_ptr);
+extern void binder_alloc_free_buf(struct binder_alloc *alloc,
+				  struct binder_buffer *buffer);
+extern int binder_alloc_mmap_handler(struct binder_alloc *alloc,
+				     struct vm_area_struct *vma);
+extern void binder_alloc_deferred_release(struct binder_alloc *alloc);
+extern int binder_alloc_get_allocated_count(struct binder_alloc *alloc);
+extern void binder_alloc_print_allocated(struct seq_file *m,
+					 struct binder_alloc *alloc);
+void binder_alloc_print_pages(struct seq_file *m,
+			      struct binder_alloc *alloc);
+
+/**
+ * binder_alloc_get_free_async_space() - get free space available for async
+ * @alloc:	binder_alloc for this proc
+ *
+ * Return:	the bytes remaining in the address-space for async transactions
+ */
+static inline size_t
+binder_alloc_get_free_async_space(struct binder_alloc *alloc)
+{
+	size_t free_async_space;
+
+	mutex_lock(&alloc->mutex);
+	free_async_space = alloc->free_async_space;
+	mutex_unlock(&alloc->mutex);
+	return free_async_space;
+}
+
+/**
+ * binder_alloc_get_user_buffer_offset() - get offset between kernel/user addrs
+ * @alloc:	binder_alloc for this proc
+ *
+ * Return:	the offset between kernel and user-space addresses to use for
+ * virtual address conversion
+ */
+static inline ptrdiff_t
+binder_alloc_get_user_buffer_offset(struct binder_alloc *alloc)
+{
+	/*
+	 * user_buffer_offset is constant if vma is set and
+	 * undefined if vma is not set. It is possible to
+	 * get here with !alloc->vma if the target process
+	 * is dying while a transaction is being initiated.
+	 * Returning the old value is ok in this case and
+	 * the transaction will fail.
+	 */
+	return alloc->user_buffer_offset;
+}
+
+#endif /* _LINUX_BINDER_ALLOC_H */
+
diff --git a/drivers/android/binder_alloc_selftest.c b/drivers/android/binder_alloc_selftest.c
new file mode 100644
index 000000000000..8bd7bcef967d
--- /dev/null
+++ b/drivers/android/binder_alloc_selftest.c
@@ -0,0 +1,310 @@
+/* binder_alloc_selftest.c
+ *
+ * Android IPC Subsystem
+ *
+ * Copyright (C) 2017 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/mm_types.h>
+#include <linux/err.h>
+#include "binder_alloc.h"
+
+#define BUFFER_NUM 5
+#define BUFFER_MIN_SIZE (PAGE_SIZE / 8)
+
+static bool binder_selftest_run = true;
+static int binder_selftest_failures;
+static DEFINE_MUTEX(binder_selftest_lock);
+
+/**
+ * enum buf_end_align_type - Page alignment of a buffer
+ * end with regard to the end of the previous buffer.
+ *
+ * In the pictures below, buf2 refers to the buffer we
+ * are aligning. buf1 refers to previous buffer by addr.
+ * Symbol [ means the start of a buffer, ] means the end
+ * of a buffer, and | means page boundaries.
+ */
+enum buf_end_align_type {
+	/**
+	 * @SAME_PAGE_UNALIGNED: The end of this buffer is on
+	 * the same page as the end of the previous buffer and
+	 * is not page aligned. Examples:
+	 * buf1 ][ buf2 ][ ...
+	 * buf1 ]|[ buf2 ][ ...
+	 */
+	SAME_PAGE_UNALIGNED = 0,
+	/**
+	 * @SAME_PAGE_ALIGNED: When the end of the previous buffer
+	 * is not page aligned, the end of this buffer is on the
+	 * same page as the end of the previous buffer and is page
+	 * aligned. When the previous buffer is page aligned, the
+	 * end of this buffer is aligned to the next page boundary.
+	 * Examples:
+	 * buf1 ][ buf2 ]| ...
+	 * buf1 ]|[ buf2 ]| ...
+	 */
+	SAME_PAGE_ALIGNED,
+	/**
+	 * @NEXT_PAGE_UNALIGNED: The end of this buffer is on
+	 * the page next to the end of the previous buffer and
+	 * is not page aligned. Examples:
+	 * buf1 ][ buf2 | buf2 ][ ...
+	 * buf1 ]|[ buf2 | buf2 ][ ...
+	 */
+	NEXT_PAGE_UNALIGNED,
+	/**
+	 * @NEXT_PAGE_ALIGNED: The end of this buffer is on
+	 * the page next to the end of the previous buffer and
+	 * is page aligned. Examples:
+	 * buf1 ][ buf2 | buf2 ]| ...
+	 * buf1 ]|[ buf2 | buf2 ]| ...
+	 */
+	NEXT_PAGE_ALIGNED,
+	/**
+	 * @NEXT_NEXT_UNALIGNED: The end of this buffer is on
+	 * the page that follows the page after the end of the
+	 * previous buffer and is not page aligned. Examples:
+	 * buf1 ][ buf2 | buf2 | buf2 ][ ...
+	 * buf1 ]|[ buf2 | buf2 | buf2 ][ ...
+	 */
+	NEXT_NEXT_UNALIGNED,
+	LOOP_END,
+};
+
+static void pr_err_size_seq(size_t *sizes, int *seq)
+{
+	int i;
+
+	pr_err("alloc sizes: ");
+	for (i = 0; i < BUFFER_NUM; i++)
+		pr_cont("[%zu]", sizes[i]);
+	pr_cont("\n");
+	pr_err("free seq: ");
+	for (i = 0; i < BUFFER_NUM; i++)
+		pr_cont("[%d]", seq[i]);
+	pr_cont("\n");
+}
+
+static bool check_buffer_pages_allocated(struct binder_alloc *alloc,
+					 struct binder_buffer *buffer,
+					 size_t size)
+{
+	void *page_addr, *end;
+	int page_index;
+
+	end = (void *)PAGE_ALIGN((uintptr_t)buffer->data + size);
+	page_addr = buffer->data;
+	for (; page_addr < end; page_addr += PAGE_SIZE) {
+		page_index = (page_addr - alloc->buffer) / PAGE_SIZE;
+		if (!alloc->pages[page_index].page_ptr ||
+		    !list_empty(&alloc->pages[page_index].lru)) {
+			pr_err("expect alloc but is %s at page index %d\n",
+			       alloc->pages[page_index].page_ptr ?
+			       "lru" : "free", page_index);
+			return false;
+		}
+	}
+	return true;
+}
+
+static void binder_selftest_alloc_buf(struct binder_alloc *alloc,
+				      struct binder_buffer *buffers[],
+				      size_t *sizes, int *seq)
+{
+	int i;
+
+	for (i = 0; i < BUFFER_NUM; i++) {
+		buffers[i] = binder_alloc_new_buf(alloc, sizes[i], 0, 0, 0);
+		if (IS_ERR(buffers[i]) ||
+		    !check_buffer_pages_allocated(alloc, buffers[i],
+						  sizes[i])) {
+			pr_err_size_seq(sizes, seq);
+			binder_selftest_failures++;
+		}
+	}
+}
+
+static void binder_selftest_free_buf(struct binder_alloc *alloc,
+				     struct binder_buffer *buffers[],
+				     size_t *sizes, int *seq, size_t end)
+{
+	int i;
+
+	for (i = 0; i < BUFFER_NUM; i++)
+		binder_alloc_free_buf(alloc, buffers[seq[i]]);
+
+	for (i = 0; i < end / PAGE_SIZE; i++) {
+		/**
+		 * Error message on a free page can be false positive
+		 * if binder shrinker ran during binder_alloc_free_buf
+		 * calls above.
+		 */
+		if (list_empty(&alloc->pages[i].lru)) {
+			pr_err_size_seq(sizes, seq);
+			pr_err("expect lru but is %s at page index %d\n",
+			       alloc->pages[i].page_ptr ? "alloc" : "free", i);
+			binder_selftest_failures++;
+		}
+	}
+}
+
+static void binder_selftest_free_page(struct binder_alloc *alloc)
+{
+	int i;
+	unsigned long count;
+
+	while ((count = list_lru_count(&binder_alloc_lru))) {
+		list_lru_walk(&binder_alloc_lru, binder_alloc_free_page,
+			      NULL, count);
+	}
+
+	for (i = 0; i < (alloc->buffer_size / PAGE_SIZE); i++) {
+		if (alloc->pages[i].page_ptr) {
+			pr_err("expect free but is %s at page index %d\n",
+			       list_empty(&alloc->pages[i].lru) ?
+			       "alloc" : "lru", i);
+			binder_selftest_failures++;
+		}
+	}
+}
+
+static void binder_selftest_alloc_free(struct binder_alloc *alloc,
+				       size_t *sizes, int *seq, size_t end)
+{
+	struct binder_buffer *buffers[BUFFER_NUM];
+
+	binder_selftest_alloc_buf(alloc, buffers, sizes, seq);
+	binder_selftest_free_buf(alloc, buffers, sizes, seq, end);
+
+	/* Allocate from lru. */
+	binder_selftest_alloc_buf(alloc, buffers, sizes, seq);
+	if (list_lru_count(&binder_alloc_lru))
+		pr_err("lru list should be empty but is not\n");
+
+	binder_selftest_free_buf(alloc, buffers, sizes, seq, end);
+	binder_selftest_free_page(alloc);
+}
+
+static bool is_dup(int *seq, int index, int val)
+{
+	int i;
+
+	for (i = 0; i < index; i++) {
+		if (seq[i] == val)
+			return true;
+	}
+	return false;
+}
+
+/* Generate BUFFER_NUM factorial free orders. */
+static void binder_selftest_free_seq(struct binder_alloc *alloc,
+				     size_t *sizes, int *seq,
+				     int index, size_t end)
+{
+	int i;
+
+	if (index == BUFFER_NUM) {
+		binder_selftest_alloc_free(alloc, sizes, seq, end);
+		return;
+	}
+	for (i = 0; i < BUFFER_NUM; i++) {
+		if (is_dup(seq, index, i))
+			continue;
+		seq[index] = i;
+		binder_selftest_free_seq(alloc, sizes, seq, index + 1, end);
+	}
+}
+
+static void binder_selftest_alloc_size(struct binder_alloc *alloc,
+				       size_t *end_offset)
+{
+	int i;
+	int seq[BUFFER_NUM] = {0};
+	size_t front_sizes[BUFFER_NUM];
+	size_t back_sizes[BUFFER_NUM];
+	size_t last_offset, offset = 0;
+
+	for (i = 0; i < BUFFER_NUM; i++) {
+		last_offset = offset;
+		offset = end_offset[i];
+		front_sizes[i] = offset - last_offset;
+		back_sizes[BUFFER_NUM - i - 1] = front_sizes[i];
+	}
+	/*
+	 * Buffers share the first or last few pages.
+	 * Only BUFFER_NUM - 1 buffer sizes are adjustable since
+	 * we need one giant buffer before getting to the last page.
+	 */
+	back_sizes[0] += alloc->buffer_size - end_offset[BUFFER_NUM - 1];
+	binder_selftest_free_seq(alloc, front_sizes, seq, 0,
+				 end_offset[BUFFER_NUM - 1]);
+	binder_selftest_free_seq(alloc, back_sizes, seq, 0, alloc->buffer_size);
+}
+
+static void binder_selftest_alloc_offset(struct binder_alloc *alloc,
+					 size_t *end_offset, int index)
+{
+	int align;
+	size_t end, prev;
+
+	if (index == BUFFER_NUM) {
+		binder_selftest_alloc_size(alloc, end_offset);
+		return;
+	}
+	prev = index == 0 ? 0 : end_offset[index - 1];
+	end = prev;
+
+	BUILD_BUG_ON(BUFFER_MIN_SIZE * BUFFER_NUM >= PAGE_SIZE);
+
+	for (align = SAME_PAGE_UNALIGNED; align < LOOP_END; align++) {
+		if (align % 2)
+			end = ALIGN(end, PAGE_SIZE);
+		else
+			end += BUFFER_MIN_SIZE;
+		end_offset[index] = end;
+		binder_selftest_alloc_offset(alloc, end_offset, index + 1);
+	}
+}
+
+/**
+ * binder_selftest_alloc() - Test alloc and free of buffer pages.
+ * @alloc: Pointer to alloc struct.
+ *
+ * Allocate BUFFER_NUM buffers to cover all page alignment cases,
+ * then free them in all orders possible. Check that pages are
+ * correctly allocated, put onto lru when buffers are freed, and
+ * are freed when binder_alloc_free_page is called.
+ */
+void binder_selftest_alloc(struct binder_alloc *alloc)
+{
+	size_t end_offset[BUFFER_NUM];
+
+	if (!binder_selftest_run)
+		return;
+	mutex_lock(&binder_selftest_lock);
+	if (!binder_selftest_run || !alloc->vma)
+		goto done;
+	pr_info("STARTED\n");
+	binder_selftest_alloc_offset(alloc, end_offset, 0);
+	binder_selftest_run = false;
+	if (binder_selftest_failures > 0)
+		pr_info("%d tests FAILED\n", binder_selftest_failures);
+	else
+		pr_info("PASSED\n");
+
+done:
+	mutex_unlock(&binder_selftest_lock);
+}
diff --git a/drivers/android/binder_trace.h b/drivers/android/binder_trace.h
index 7f20f3dc8369..76e3b9c8a8a2 100644
--- a/drivers/android/binder_trace.h
+++ b/drivers/android/binder_trace.h
@@ -23,7 +23,8 @@
 struct binder_buffer;
 struct binder_node;
 struct binder_proc;
-struct binder_ref;
+struct binder_alloc;
+struct binder_ref_data;
 struct binder_thread;
 struct binder_transaction;
 
@@ -146,8 +147,8 @@ TRACE_EVENT(binder_transaction_received,
 
 TRACE_EVENT(binder_transaction_node_to_ref,
 	TP_PROTO(struct binder_transaction *t, struct binder_node *node,
-		 struct binder_ref *ref),
-	TP_ARGS(t, node, ref),
+		 struct binder_ref_data *rdata),
+	TP_ARGS(t, node, rdata),
 
 	TP_STRUCT__entry(
 		__field(int, debug_id)
@@ -160,8 +161,8 @@ TRACE_EVENT(binder_transaction_node_to_ref,
 		__entry->debug_id = t->debug_id;
 		__entry->node_debug_id = node->debug_id;
 		__entry->node_ptr = node->ptr;
-		__entry->ref_debug_id = ref->debug_id;
-		__entry->ref_desc = ref->desc;
+		__entry->ref_debug_id = rdata->debug_id;
+		__entry->ref_desc = rdata->desc;
 	),
 	TP_printk("transaction=%d node=%d src_ptr=0x%016llx ==> dest_ref=%d dest_desc=%d",
 		  __entry->debug_id, __entry->node_debug_id,
@@ -170,8 +171,9 @@ TRACE_EVENT(binder_transaction_node_to_ref,
 );
 
 TRACE_EVENT(binder_transaction_ref_to_node,
-	TP_PROTO(struct binder_transaction *t, struct binder_ref *ref),
-	TP_ARGS(t, ref),
+	TP_PROTO(struct binder_transaction *t, struct binder_node *node,
+		 struct binder_ref_data *rdata),
+	TP_ARGS(t, node, rdata),
 
 	TP_STRUCT__entry(
 		__field(int, debug_id)
@@ -182,10 +184,10 @@ TRACE_EVENT(binder_transaction_ref_to_node,
 	),
 	TP_fast_assign(
 		__entry->debug_id = t->debug_id;
-		__entry->ref_debug_id = ref->debug_id;
-		__entry->ref_desc = ref->desc;
-		__entry->node_debug_id = ref->node->debug_id;
-		__entry->node_ptr = ref->node->ptr;
+		__entry->ref_debug_id = rdata->debug_id;
+		__entry->ref_desc = rdata->desc;
+		__entry->node_debug_id = node->debug_id;
+		__entry->node_ptr = node->ptr;
 	),
 	TP_printk("transaction=%d node=%d src_ref=%d src_desc=%d ==> dest_ptr=0x%016llx",
 		  __entry->debug_id, __entry->node_debug_id,
@@ -194,9 +196,10 @@ TRACE_EVENT(binder_transaction_ref_to_node,
 );
 
 TRACE_EVENT(binder_transaction_ref_to_ref,
-	TP_PROTO(struct binder_transaction *t, struct binder_ref *src_ref,
-		 struct binder_ref *dest_ref),
-	TP_ARGS(t, src_ref, dest_ref),
+	TP_PROTO(struct binder_transaction *t, struct binder_node *node,
+		 struct binder_ref_data *src_ref,
+		 struct binder_ref_data *dest_ref),
+	TP_ARGS(t, node, src_ref, dest_ref),
 
 	TP_STRUCT__entry(
 		__field(int, debug_id)
@@ -208,7 +211,7 @@ TRACE_EVENT(binder_transaction_ref_to_ref,
 	),
 	TP_fast_assign(
 		__entry->debug_id = t->debug_id;
-		__entry->node_debug_id = src_ref->node->debug_id;
+		__entry->node_debug_id = node->debug_id;
 		__entry->src_ref_debug_id = src_ref->debug_id;
 		__entry->src_ref_desc = src_ref->desc;
 		__entry->dest_ref_debug_id = dest_ref->debug_id;
@@ -268,9 +271,9 @@ DEFINE_EVENT(binder_buffer_class, binder_transaction_failed_buffer_release,
 	TP_ARGS(buffer));
 
 TRACE_EVENT(binder_update_page_range,
-	TP_PROTO(struct binder_proc *proc, bool allocate,
+	TP_PROTO(struct binder_alloc *alloc, bool allocate,
 		 void *start, void *end),
-	TP_ARGS(proc, allocate, start, end),
+	TP_ARGS(alloc, allocate, start, end),
 	TP_STRUCT__entry(
 		__field(int, proc)
 		__field(bool, allocate)
@@ -278,9 +281,9 @@ TRACE_EVENT(binder_update_page_range,
 		__field(size_t, size)
 	),
 	TP_fast_assign(
-		__entry->proc = proc->pid;
+		__entry->proc = alloc->pid;
 		__entry->allocate = allocate;
-		__entry->offset = start - proc->buffer;
+		__entry->offset = start - alloc->buffer;
 		__entry->size = end - start;
 	),
 	TP_printk("proc=%d allocate=%d offset=%zu size=%zu",
@@ -288,6 +291,61 @@ TRACE_EVENT(binder_update_page_range,
 		  __entry->offset, __entry->size)
 );
 
+DECLARE_EVENT_CLASS(binder_lru_page_class,
+	TP_PROTO(const struct binder_alloc *alloc, size_t page_index),
+	TP_ARGS(alloc, page_index),
+	TP_STRUCT__entry(
+		__field(int, proc)
+		__field(size_t, page_index)
+	),
+	TP_fast_assign(
+		__entry->proc = alloc->pid;
+		__entry->page_index = page_index;
+	),
+	TP_printk("proc=%d page_index=%zu",
+		  __entry->proc, __entry->page_index)
+);
+
+DEFINE_EVENT(binder_lru_page_class, binder_alloc_lru_start,
+	TP_PROTO(const struct binder_alloc *alloc, size_t page_index),
+	TP_ARGS(alloc, page_index));
+
+DEFINE_EVENT(binder_lru_page_class, binder_alloc_lru_end,
+	TP_PROTO(const struct binder_alloc *alloc, size_t page_index),
+	TP_ARGS(alloc, page_index));
+
+DEFINE_EVENT(binder_lru_page_class, binder_free_lru_start,
+	TP_PROTO(const struct binder_alloc *alloc, size_t page_index),
+	TP_ARGS(alloc, page_index));
+
+DEFINE_EVENT(binder_lru_page_class, binder_free_lru_end,
+	TP_PROTO(const struct binder_alloc *alloc, size_t page_index),
+	TP_ARGS(alloc, page_index));
+
+DEFINE_EVENT(binder_lru_page_class, binder_alloc_page_start,
+	TP_PROTO(const struct binder_alloc *alloc, size_t page_index),
+	TP_ARGS(alloc, page_index));
+
+DEFINE_EVENT(binder_lru_page_class, binder_alloc_page_end,
+	TP_PROTO(const struct binder_alloc *alloc, size_t page_index),
+	TP_ARGS(alloc, page_index));
+
+DEFINE_EVENT(binder_lru_page_class, binder_unmap_user_start,
+	TP_PROTO(const struct binder_alloc *alloc, size_t page_index),
+	TP_ARGS(alloc, page_index));
+
+DEFINE_EVENT(binder_lru_page_class, binder_unmap_user_end,
+	TP_PROTO(const struct binder_alloc *alloc, size_t page_index),
+	TP_ARGS(alloc, page_index));
+
+DEFINE_EVENT(binder_lru_page_class, binder_unmap_kernel_start,
+	TP_PROTO(const struct binder_alloc *alloc, size_t page_index),
+	TP_ARGS(alloc, page_index));
+
+DEFINE_EVENT(binder_lru_page_class, binder_unmap_kernel_end,
+	TP_PROTO(const struct binder_alloc *alloc, size_t page_index),
+	TP_ARGS(alloc, page_index));
+
 TRACE_EVENT(binder_command,
 	TP_PROTO(uint32_t cmd),
 	TP_ARGS(cmd),
diff --git a/drivers/ata/Kconfig b/drivers/ata/Kconfig
index 363fc5330c21..488c93724220 100644
--- a/drivers/ata/Kconfig
+++ b/drivers/ata/Kconfig
@@ -153,6 +153,16 @@ config AHCI_CEVA
 
 	  If unsure, say N.
 
+config AHCI_MTK
+	tristate "MediaTek AHCI SATA support"
+	depends on ARCH_MEDIATEK
+	select MFD_SYSCON
+	help
+	  This option enables support for the MediaTek SoC's
+	  onboard AHCI SATA controller.
+
+	  If unsure, say N.
+
 config AHCI_MVEBU
 	tristate "Marvell EBU AHCI SATA support"
 	depends on ARCH_MVEBU
diff --git a/drivers/ata/Makefile b/drivers/ata/Makefile
index a26ef5a93919..ff9cd2e37458 100644
--- a/drivers/ata/Makefile
+++ b/drivers/ata/Makefile
@@ -17,6 +17,7 @@ obj-$(CONFIG_AHCI_CEVA)		+= ahci_ceva.o libahci.o libahci_platform.o
 obj-$(CONFIG_AHCI_DA850)	+= ahci_da850.o libahci.o libahci_platform.o
 obj-$(CONFIG_AHCI_DM816)	+= ahci_dm816.o libahci.o libahci_platform.o
 obj-$(CONFIG_AHCI_IMX)		+= ahci_imx.o libahci.o libahci_platform.o
+obj-$(CONFIG_AHCI_MTK)		+= ahci_mtk.o libahci.o libahci_platform.o
 obj-$(CONFIG_AHCI_MVEBU)	+= ahci_mvebu.o libahci.o libahci_platform.o
 obj-$(CONFIG_AHCI_OCTEON)	+= ahci_octeon.o
 obj-$(CONFIG_AHCI_SUNXI)	+= ahci_sunxi.o libahci.o libahci_platform.o
diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c
index 5a5fd0b404eb..cb9b0e9090e3 100644
--- a/drivers/ata/ahci.c
+++ b/drivers/ata/ahci.c
@@ -1469,7 +1469,14 @@ static void ahci_remap_check(struct pci_dev *pdev, int bar,
 		return;
 
 	dev_warn(&pdev->dev, "Found %d remapped NVMe devices.\n", count);
-	dev_warn(&pdev->dev, "Switch your BIOS from RAID to AHCI mode to use them.\n");
+	dev_warn(&pdev->dev,
+		 "Switch your BIOS from RAID to AHCI mode to use them.\n");
+
+	/*
+	 * Don't rely on the msi-x capability in the remap case,
+	 * share the legacy interrupt across ahci and remapped devices.
+	 */
+	hpriv->flags |= AHCI_HFLAG_NO_MSI;
 }
 
 static int ahci_get_irq_vector(struct ata_host *host, int port)
diff --git a/drivers/ata/ahci_da850.c b/drivers/ata/ahci_da850.c
index 1a50cd3b4233..9b34dff64536 100644
--- a/drivers/ata/ahci_da850.c
+++ b/drivers/ata/ahci_da850.c
@@ -216,12 +216,16 @@ static int ahci_da850_probe(struct platform_device *pdev)
 		return rc;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
-	if (!res)
+	if (!res) {
+		rc = -ENODEV;
 		goto disable_resources;
+	}
 
 	pwrdn_reg = devm_ioremap(dev, res->start, resource_size(res));
-	if (!pwrdn_reg)
+	if (!pwrdn_reg) {
+		rc = -ENOMEM;
 		goto disable_resources;
+	}
 
 	da850_sata_init(dev, pwrdn_reg, hpriv->mmio, mpy);
 
diff --git a/drivers/ata/ahci_mtk.c b/drivers/ata/ahci_mtk.c
new file mode 100644
index 000000000000..80854f71559a
--- /dev/null
+++ b/drivers/ata/ahci_mtk.c
@@ -0,0 +1,196 @@
+/*
+ * MeidaTek AHCI SATA driver
+ *
+ * Copyright (c) 2017 MediaTek Inc.
+ * Author: Ryder Lee <ryder.lee@mediatek.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/ahci_platform.h>
+#include <linux/kernel.h>
+#include <linux/libata.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+#include "ahci.h"
+
+#define DRV_NAME		"ahci"
+
+#define SYS_CFG			0x14
+#define SYS_CFG_SATA_MSK	GENMASK(31, 30)
+#define SYS_CFG_SATA_EN		BIT(31)
+
+struct mtk_ahci_plat {
+	struct regmap *mode;
+	struct reset_control *axi_rst;
+	struct reset_control *sw_rst;
+	struct reset_control *reg_rst;
+};
+
+static const struct ata_port_info ahci_port_info = {
+	.flags		= AHCI_FLAG_COMMON,
+	.pio_mask	= ATA_PIO4,
+	.udma_mask	= ATA_UDMA6,
+	.port_ops	= &ahci_platform_ops,
+};
+
+static struct scsi_host_template ahci_platform_sht = {
+	AHCI_SHT(DRV_NAME),
+};
+
+static int mtk_ahci_platform_resets(struct ahci_host_priv *hpriv,
+				    struct device *dev)
+{
+	struct mtk_ahci_plat *plat = hpriv->plat_data;
+	int err;
+
+	/* reset AXI bus and PHY part */
+	plat->axi_rst = devm_reset_control_get_optional_exclusive(dev, "axi");
+	if (PTR_ERR(plat->axi_rst) == -EPROBE_DEFER)
+		return PTR_ERR(plat->axi_rst);
+
+	plat->sw_rst = devm_reset_control_get_optional_exclusive(dev, "sw");
+	if (PTR_ERR(plat->sw_rst) == -EPROBE_DEFER)
+		return PTR_ERR(plat->sw_rst);
+
+	plat->reg_rst = devm_reset_control_get_optional_exclusive(dev, "reg");
+	if (PTR_ERR(plat->reg_rst) == -EPROBE_DEFER)
+		return PTR_ERR(plat->reg_rst);
+
+	err = reset_control_assert(plat->axi_rst);
+	if (err) {
+		dev_err(dev, "failed to assert AXI bus\n");
+		return err;
+	}
+
+	err = reset_control_assert(plat->sw_rst);
+	if (err) {
+		dev_err(dev, "failed to assert PHY digital part\n");
+		return err;
+	}
+
+	err = reset_control_assert(plat->reg_rst);
+	if (err) {
+		dev_err(dev, "failed to assert PHY register part\n");
+		return err;
+	}
+
+	err = reset_control_deassert(plat->reg_rst);
+	if (err) {
+		dev_err(dev, "failed to deassert PHY register part\n");
+		return err;
+	}
+
+	err = reset_control_deassert(plat->sw_rst);
+	if (err) {
+		dev_err(dev, "failed to deassert PHY digital part\n");
+		return err;
+	}
+
+	err = reset_control_deassert(plat->axi_rst);
+	if (err) {
+		dev_err(dev, "failed to deassert AXI bus\n");
+		return err;
+	}
+
+	return 0;
+}
+
+static int mtk_ahci_parse_property(struct ahci_host_priv *hpriv,
+				   struct device *dev)
+{
+	struct mtk_ahci_plat *plat = hpriv->plat_data;
+	struct device_node *np = dev->of_node;
+
+	/* enable SATA function if needed */
+	if (of_find_property(np, "mediatek,phy-mode", NULL)) {
+		plat->mode = syscon_regmap_lookup_by_phandle(
+					np, "mediatek,phy-mode");
+		if (IS_ERR(plat->mode)) {
+			dev_err(dev, "missing phy-mode phandle\n");
+			return PTR_ERR(plat->mode);
+		}
+
+		regmap_update_bits(plat->mode, SYS_CFG, SYS_CFG_SATA_MSK,
+				   SYS_CFG_SATA_EN);
+	}
+
+	of_property_read_u32(np, "ports-implemented", &hpriv->force_port_map);
+
+	return 0;
+}
+
+static int mtk_ahci_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct mtk_ahci_plat *plat;
+	struct ahci_host_priv *hpriv;
+	int err;
+
+	plat = devm_kzalloc(dev, sizeof(*plat), GFP_KERNEL);
+	if (!plat)
+		return -ENOMEM;
+
+	hpriv = ahci_platform_get_resources(pdev);
+	if (IS_ERR(hpriv))
+		return PTR_ERR(hpriv);
+
+	hpriv->plat_data = plat;
+
+	err = mtk_ahci_parse_property(hpriv, dev);
+	if (err)
+		return err;
+
+	err = mtk_ahci_platform_resets(hpriv, dev);
+	if (err)
+		return err;
+
+	err = ahci_platform_enable_resources(hpriv);
+	if (err)
+		return err;
+
+	err = ahci_platform_init_host(pdev, hpriv, &ahci_port_info,
+				      &ahci_platform_sht);
+	if (err)
+		goto disable_resources;
+
+	return 0;
+
+disable_resources:
+	ahci_platform_disable_resources(hpriv);
+	return err;
+}
+
+static SIMPLE_DEV_PM_OPS(ahci_pm_ops, ahci_platform_suspend,
+			 ahci_platform_resume);
+
+static const struct of_device_id ahci_of_match[] = {
+	{ .compatible = "mediatek,mtk-ahci", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, ahci_of_match);
+
+static struct platform_driver mtk_ahci_driver = {
+	.probe = mtk_ahci_probe,
+	.remove = ata_platform_remove_one,
+	.driver = {
+		.name = DRV_NAME,
+		.of_match_table = ahci_of_match,
+		.pm = &ahci_pm_ops,
+	},
+};
+module_platform_driver(mtk_ahci_driver);
+
+MODULE_DESCRIPTION("MeidaTek SATA AHCI Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/ata/ahci_platform.c b/drivers/ata/ahci_platform.c
index 62a04c8fb5c9..99f9a895a459 100644
--- a/drivers/ata/ahci_platform.c
+++ b/drivers/ata/ahci_platform.c
@@ -93,6 +93,7 @@ MODULE_DEVICE_TABLE(acpi, ahci_acpi_match);
 static struct platform_driver ahci_driver = {
 	.probe = ahci_probe,
 	.remove = ata_platform_remove_one,
+	.shutdown = ahci_platform_shutdown,
 	.driver = {
 		.name = DRV_NAME,
 		.of_match_table = ahci_of_match,
diff --git a/drivers/ata/libahci_platform.c b/drivers/ata/libahci_platform.c
index cd2eab6aa92e..a270a1173c8c 100644
--- a/drivers/ata/libahci_platform.c
+++ b/drivers/ata/libahci_platform.c
@@ -602,6 +602,40 @@ static void ahci_host_stop(struct ata_host *host)
 	ahci_platform_disable_resources(hpriv);
 }
 
+/**
+ * ahci_platform_shutdown - Disable interrupts and stop DMA for host ports
+ * @dev: platform device pointer for the host
+ *
+ * This function is called during system shutdown and performs the minimal
+ * deconfiguration required to ensure that an ahci_platform host cannot
+ * corrupt or otherwise interfere with a new kernel being started with kexec.
+ */
+void ahci_platform_shutdown(struct platform_device *pdev)
+{
+	struct ata_host *host = platform_get_drvdata(pdev);
+	struct ahci_host_priv *hpriv = host->private_data;
+	void __iomem *mmio = hpriv->mmio;
+	int i;
+
+	for (i = 0; i < host->n_ports; i++) {
+		struct ata_port *ap = host->ports[i];
+
+		/* Disable port interrupts */
+		if (ap->ops->freeze)
+			ap->ops->freeze(ap);
+
+		/* Stop the port DMA engines */
+		if (ap->ops->port_stop)
+			ap->ops->port_stop(ap);
+	}
+
+	/* Disable and clear host interrupts */
+	writel(readl(mmio + HOST_CTL) & ~HOST_IRQ_EN, mmio + HOST_CTL);
+	readl(mmio + HOST_CTL); /* flush */
+	writel(GENMASK(host->n_ports, 0), mmio + HOST_IRQ_STAT);
+}
+EXPORT_SYMBOL_GPL(ahci_platform_shutdown);
+
 #ifdef CONFIG_PM_SLEEP
 /**
  * ahci_platform_suspend_host - Suspend an ahci-platform host
diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c
index fa7dd4394c02..1945a8ea2099 100644
--- a/drivers/ata/libata-core.c
+++ b/drivers/ata/libata-core.c
@@ -2411,6 +2411,9 @@ static void ata_dev_config_trusted(struct ata_device *dev)
 	u64 trusted_cap;
 	unsigned int err;
 
+	if (!ata_id_has_trusted(dev->id))
+		return;
+
 	if (!ata_identify_page_supported(dev, ATA_LOG_SECURITY)) {
 		ata_dev_warn(dev,
 			     "Security Log not supported\n");
diff --git a/drivers/ata/libata-eh.c b/drivers/ata/libata-eh.c
index 3dbd05532c09..e4effef0c83f 100644
--- a/drivers/ata/libata-eh.c
+++ b/drivers/ata/libata-eh.c
@@ -645,12 +645,11 @@ void ata_scsi_cmd_error_handler(struct Scsi_Host *host, struct ata_port *ap,
 	 * completions are honored.  A scmd is determined to have
 	 * timed out iff its associated qc is active and not failed.
 	 */
+	spin_lock_irqsave(ap->lock, flags);
 	if (ap->ops->error_handler) {
 		struct scsi_cmnd *scmd, *tmp;
 		int nr_timedout = 0;
 
-		spin_lock_irqsave(ap->lock, flags);
-
 		/* This must occur under the ap->lock as we don't want
 		   a polled recovery to race the real interrupt handler
 
@@ -700,12 +699,11 @@ void ata_scsi_cmd_error_handler(struct Scsi_Host *host, struct ata_port *ap,
 		if (nr_timedout)
 			__ata_port_freeze(ap);
 
-		spin_unlock_irqrestore(ap->lock, flags);
 
 		/* initialize eh_tries */
 		ap->eh_tries = ATA_EH_MAX_TRIES;
-	} else
-		spin_unlock_wait(ap->lock);
+	}
+	spin_unlock_irqrestore(ap->lock, flags);
 
 }
 EXPORT_SYMBOL(ata_scsi_cmd_error_handler);
diff --git a/drivers/ata/libata-zpodd.c b/drivers/ata/libata-zpodd.c
index 8a01d09ac4db..23a62e4015d0 100644
--- a/drivers/ata/libata-zpodd.c
+++ b/drivers/ata/libata-zpodd.c
@@ -34,7 +34,7 @@ struct zpodd {
 static int eject_tray(struct ata_device *dev)
 {
 	struct ata_taskfile tf;
-	const char cdb[] = {  GPCMD_START_STOP_UNIT,
+	static const char cdb[] = {  GPCMD_START_STOP_UNIT,
 		0, 0, 0,
 		0x02,     /* LoEj */
 		0, 0, 0, 0, 0, 0, 0,
@@ -55,7 +55,7 @@ static enum odd_mech_type zpodd_get_mech_type(struct ata_device *dev)
 	unsigned int ret;
 	struct rm_feature_desc *desc = (void *)(buf + 8);
 	struct ata_taskfile tf;
-	char cdb[] = {  GPCMD_GET_CONFIGURATION,
+	static const char cdb[] = {  GPCMD_GET_CONFIGURATION,
 			2,      /* only 1 feature descriptor requested */
 			0, 3,   /* 3, removable medium feature */
 			0, 0, 0,/* reserved */
diff --git a/drivers/ata/pata_amd.c b/drivers/ata/pata_amd.c
index 8d4d959a821c..8706533db57b 100644
--- a/drivers/ata/pata_amd.c
+++ b/drivers/ata/pata_amd.c
@@ -616,6 +616,7 @@ static const struct pci_device_id amd[] = {
 	{ PCI_VDEVICE(NVIDIA,	PCI_DEVICE_ID_NVIDIA_NFORCE_MCP73_IDE),	8 },
 	{ PCI_VDEVICE(NVIDIA,	PCI_DEVICE_ID_NVIDIA_NFORCE_MCP77_IDE),	8 },
 	{ PCI_VDEVICE(AMD,	PCI_DEVICE_ID_AMD_CS5536_IDE),		9 },
+	{ PCI_VDEVICE(AMD,	PCI_DEVICE_ID_AMD_CS5536_DEV_IDE),	9 },
 
 	{ },
 };
diff --git a/drivers/ata/pata_cs5536.c b/drivers/ata/pata_cs5536.c
index 6c15a554efbe..dc1255294628 100644
--- a/drivers/ata/pata_cs5536.c
+++ b/drivers/ata/pata_cs5536.c
@@ -289,6 +289,7 @@ static int cs5536_init_one(struct pci_dev *dev, const struct pci_device_id *id)
 
 static const struct pci_device_id cs5536[] = {
 	{ PCI_VDEVICE(AMD,	PCI_DEVICE_ID_AMD_CS5536_IDE), },
+	{ PCI_VDEVICE(AMD,	PCI_DEVICE_ID_AMD_CS5536_DEV_IDE), },
 	{ },
 };
 
diff --git a/drivers/ata/pata_octeon_cf.c b/drivers/ata/pata_octeon_cf.c
index 1ba03d6df951..d3d851b014a3 100644
--- a/drivers/ata/pata_octeon_cf.c
+++ b/drivers/ata/pata_octeon_cf.c
@@ -840,7 +840,6 @@ static int octeon_cf_probe(struct platform_device *pdev)
 	struct property *reg_prop;
 	int n_addr, n_size, reg_len;
 	struct device_node *node;
-	const void *prop;
 	void __iomem *cs0;
 	void __iomem *cs1 = NULL;
 	struct ata_host *host;
@@ -850,7 +849,7 @@ static int octeon_cf_probe(struct platform_device *pdev)
 	void __iomem *base;
 	struct octeon_cf_port *cf_port;
 	int rv = -ENOMEM;
-
+	u32 bus_width;
 
 	node = pdev->dev.of_node;
 	if (node == NULL)
@@ -860,11 +859,10 @@ static int octeon_cf_probe(struct platform_device *pdev)
 	if (!cf_port)
 		return -ENOMEM;
 
-	cf_port->is_true_ide = (of_find_property(node, "cavium,true-ide", NULL) != NULL);
+	cf_port->is_true_ide = of_property_read_bool(node, "cavium,true-ide");
 
-	prop = of_get_property(node, "cavium,bus-width", NULL);
-	if (prop)
-		is_16bit = (be32_to_cpup(prop) == 16);
+	if (of_property_read_u32(node, "cavium,bus-width", &bus_width) == 0)
+		is_16bit = (bus_width == 16);
 	else
 		is_16bit = false;
 
diff --git a/drivers/ata/sata_gemini.c b/drivers/ata/sata_gemini.c
index 8c704523bae7..46950e0267e0 100644
--- a/drivers/ata/sata_gemini.c
+++ b/drivers/ata/sata_gemini.c
@@ -15,6 +15,7 @@
 #include <linux/of_device.h>
 #include <linux/clk.h>
 #include <linux/io.h>
+#include <linux/pinctrl/consumer.h>
 #include "sata_gemini.h"
 
 #define DRV_NAME "gemini_sata_bridge"
@@ -43,17 +44,6 @@ struct sata_gemini {
 	struct clk *sata1_pclk;
 };
 
-/* Global IDE PAD Skew Control Register */
-#define GEMINI_GLOBAL_IDE_SKEW_CTRL		0x18
-#define GEMINI_IDE1_HOST_STROBE_DELAY_SHIFT	28
-#define GEMINI_IDE1_DEVICE_STROBE_DELAY_SHIFT	24
-#define GEMINI_IDE1_OUTPUT_IO_SKEW_SHIFT	20
-#define GEMINI_IDE1_INPUT_IO_SKEW_SHIFT		16
-#define GEMINI_IDE0_HOST_STROBE_DELAY_SHIFT	12
-#define GEMINI_IDE0_DEVICE_STROBE_DELAY_SHIFT	8
-#define GEMINI_IDE0_OUTPUT_IO_SKEW_SHIFT	4
-#define GEMINI_IDE0_INPUT_IO_SKEW_SHIFT		0
-
 /* Miscellaneous Control Register */
 #define GEMINI_GLOBAL_MISC_CTRL		0x30
 /*
@@ -91,8 +81,6 @@ struct sata_gemini {
 #define GEMINI_IDE_IOMUX_MODE2			(2 << 24)
 #define GEMINI_IDE_IOMUX_MODE3			(3 << 24)
 #define GEMINI_IDE_IOMUX_SHIFT			(24)
-#define GEMINI_IDE_PADS_ENABLE			BIT(4)
-#define GEMINI_PFLASH_PADS_DISABLE		BIT(1)
 
 /*
  * Registers directly controlling the PATA<->SATA adapters
@@ -274,14 +262,14 @@ static int gemini_sata_bridge_init(struct sata_gemini *sg)
 		return ret;
 	}
 
-	sg->sata0_reset = devm_reset_control_get(dev, "sata0");
+	sg->sata0_reset = devm_reset_control_get_exclusive(dev, "sata0");
 	if (IS_ERR(sg->sata0_reset)) {
 		dev_err(dev, "no SATA0 reset controller\n");
 		clk_disable_unprepare(sg->sata1_pclk);
 		clk_disable_unprepare(sg->sata0_pclk);
 		return PTR_ERR(sg->sata0_reset);
 	}
-	sg->sata1_reset = devm_reset_control_get(dev, "sata1");
+	sg->sata1_reset = devm_reset_control_get_exclusive(dev, "sata1");
 	if (IS_ERR(sg->sata1_reset)) {
 		dev_err(dev, "no SATA1 reset controller\n");
 		clk_disable_unprepare(sg->sata1_pclk);
@@ -300,17 +288,39 @@ static int gemini_sata_bridge_init(struct sata_gemini *sg)
 	return 0;
 }
 
+static int gemini_setup_ide_pins(struct device *dev)
+{
+	struct pinctrl *p;
+	struct pinctrl_state *ide_state;
+	int ret;
+
+	p = devm_pinctrl_get(dev);
+	if (IS_ERR(p))
+		return PTR_ERR(p);
+
+	ide_state = pinctrl_lookup_state(p, "ide");
+	if (IS_ERR(ide_state))
+		return PTR_ERR(ide_state);
+
+	ret = pinctrl_select_state(p, ide_state);
+	if (ret) {
+		dev_err(dev, "could not select IDE state\n");
+		return ret;
+	}
+
+	return 0;
+}
+
 static int gemini_sata_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct device_node *np = dev->of_node;
 	struct sata_gemini *sg;
-	static struct regmap *map;
+	struct regmap *map;
 	struct resource *res;
 	enum gemini_muxmode muxmode;
 	u32 gmode;
 	u32 gmask;
-	u32 val;
 	int ret;
 
 	sg = devm_kzalloc(dev, sizeof(*sg), GFP_KERNEL);
@@ -362,16 +372,6 @@ static int gemini_sata_probe(struct platform_device *pdev)
 	gmask = GEMINI_IDE_IOMUX_MASK;
 	gmode = (muxmode << GEMINI_IDE_IOMUX_SHIFT);
 
-	/*
-	 * If we mux out the IDE, parallel flash must be disabled.
-	 * SATA0 and SATA1 have dedicated pins and may coexist with
-	 * parallel flash.
-	 */
-	if (sg->ide_pins)
-		gmode |= GEMINI_IDE_PADS_ENABLE | GEMINI_PFLASH_PADS_DISABLE;
-	else
-		gmask |= GEMINI_IDE_PADS_ENABLE;
-
 	ret = regmap_update_bits(map, GEMINI_GLOBAL_MISC_CTRL, gmask, gmode);
 	if (ret) {
 		dev_err(dev, "unable to set up IDE muxing\n");
@@ -379,14 +379,15 @@ static int gemini_sata_probe(struct platform_device *pdev)
 		goto out_unprep_clk;
 	}
 
-	/* FIXME: add more elaborate IDE skew control handling */
+	/*
+	 * Route out the IDE pins if desired.
+	 * This is done by looking up a special pin control state called
+	 * "ide" that will route out the IDE pins.
+	 */
 	if (sg->ide_pins) {
-		ret = regmap_read(map, GEMINI_GLOBAL_IDE_SKEW_CTRL, &val);
-		if (ret) {
-			dev_err(dev, "cannot read IDE skew control register\n");
+		ret = gemini_setup_ide_pins(dev);
+		if (ret)
 			return ret;
-		}
-		dev_info(dev, "IDE skew control: %08x\n", val);
 	}
 
 	dev_info(dev, "set up the Gemini IDE/SATA nexus\n");
diff --git a/drivers/ata/sata_svw.c b/drivers/ata/sata_svw.c
index 0fd6ac7e57ba..a9d692c6c182 100644
--- a/drivers/ata/sata_svw.c
+++ b/drivers/ata/sata_svw.c
@@ -339,7 +339,7 @@ static int k2_sata_show_info(struct seq_file *m, struct Scsi_Host *shost)
 		if (!reg)
 			continue;
 		if (index == *reg) {
-			seq_printf(m, "devspec: %s\n", np->full_name);
+			seq_printf(m, "devspec: %pOF\n", np);
 			break;
 		}
 	}
diff --git a/drivers/atm/adummy.c b/drivers/atm/adummy.c
index 1fd25e872ece..8d98130ecd40 100644
--- a/drivers/atm/adummy.c
+++ b/drivers/atm/adummy.c
@@ -71,7 +71,7 @@ static struct attribute *adummy_attrs[] = {
 	NULL
 };
 
-static struct attribute_group adummy_group_attrs = {
+static const struct attribute_group adummy_group_attrs = {
 	.name = NULL, /* We want them in dev's root folder */
 	.attrs = adummy_attrs
 };
@@ -130,7 +130,7 @@ adummy_proc_read(struct atm_dev *dev, loff_t *pos, char *page)
 	return 0;
 }
 
-static struct atmdev_ops adummy_ops =
+static const struct atmdev_ops adummy_ops =
 {
 	.open =		adummy_open,
 	.close =	adummy_close,	
diff --git a/drivers/atm/ambassador.c b/drivers/atm/ambassador.c
index 906705e5f776..acf16c323e38 100644
--- a/drivers/atm/ambassador.c
+++ b/drivers/atm/ambassador.c
@@ -2374,7 +2374,7 @@ MODULE_PARM_DESC(pci_lat, "PCI latency in bus cycles");
 
 /********** module entry **********/
 
-static struct pci_device_id amb_pci_tbl[] = {
+static const struct pci_device_id amb_pci_tbl[] = {
 	{ PCI_VDEVICE(MADGE, PCI_DEVICE_ID_MADGE_AMBASSADOR), 0 },
 	{ PCI_VDEVICE(MADGE, PCI_DEVICE_ID_MADGE_AMBASSADOR_BAD), 0 },
 	{ 0, }
diff --git a/drivers/atm/atmtcp.c b/drivers/atm/atmtcp.c
index 56fa16c85ebf..afebeb1c3e1e 100644
--- a/drivers/atm/atmtcp.c
+++ b/drivers/atm/atmtcp.c
@@ -342,7 +342,7 @@ static struct atmdev_ops atmtcp_v_dev_ops = {
  */
 
 
-static struct atmdev_ops atmtcp_c_dev_ops = {
+static const struct atmdev_ops atmtcp_c_dev_ops = {
 	.close		= atmtcp_c_close,
 	.send		= atmtcp_c_send
 };
diff --git a/drivers/atm/eni.c b/drivers/atm/eni.c
index b042ec458544..ce47eb17901d 100644
--- a/drivers/atm/eni.c
+++ b/drivers/atm/eni.c
@@ -2292,7 +2292,7 @@ err_disable:
 }
 
 
-static struct pci_device_id eni_pci_tbl[] = {
+static const struct pci_device_id eni_pci_tbl[] = {
 	{ PCI_VDEVICE(EF, PCI_DEVICE_ID_EF_ATM_FPGA), 0 /* FPGA */ },
 	{ PCI_VDEVICE(EF, PCI_DEVICE_ID_EF_ATM_ASIC), 1 /* ASIC */ },
 	{ 0, }
diff --git a/drivers/atm/firestream.c b/drivers/atm/firestream.c
index 22dcab952a24..6b6368a56526 100644
--- a/drivers/atm/firestream.c
+++ b/drivers/atm/firestream.c
@@ -2030,7 +2030,7 @@ static void firestream_remove_one(struct pci_dev *pdev)
 	func_exit ();
 }
 
-static struct pci_device_id firestream_pci_tbl[] = {
+static const struct pci_device_id firestream_pci_tbl[] = {
 	{ PCI_VDEVICE(FUJITSU_ME, PCI_DEVICE_ID_FUJITSU_FS50), FS_IS50},
 	{ PCI_VDEVICE(FUJITSU_ME, PCI_DEVICE_ID_FUJITSU_FS155), FS_IS155},
 	{ 0, }
diff --git a/drivers/atm/fore200e.c b/drivers/atm/fore200e.c
index f0433adcd8fc..f8b7e86907cc 100644
--- a/drivers/atm/fore200e.c
+++ b/drivers/atm/fore200e.c
@@ -2757,7 +2757,7 @@ static void fore200e_pca_remove_one(struct pci_dev *pci_dev)
 }
 
 
-static struct pci_device_id fore200e_pca_tbl[] = {
+static const struct pci_device_id fore200e_pca_tbl[] = {
     { PCI_VENDOR_ID_FORE, PCI_DEVICE_ID_FORE_PCA200E, PCI_ANY_ID, PCI_ANY_ID,
       0, 0, (unsigned long) &fore200e_bus[0] },
     { 0, }
diff --git a/drivers/atm/he.c b/drivers/atm/he.c
index 37ee21c5a5ca..e58538c29377 100644
--- a/drivers/atm/he.c
+++ b/drivers/atm/he.c
@@ -161,7 +161,7 @@ static unsigned int clocktab[] = {
 	CLK_LOW
 };     
 
-static struct atmdev_ops he_ops =
+static const struct atmdev_ops he_ops =
 {
 	.open =		he_open,
 	.close =	he_close,	
@@ -2851,7 +2851,7 @@ MODULE_PARM_DESC(irq_coalesce, "use interrupt coalescing (default 1)");
 module_param(sdh, bool, 0);
 MODULE_PARM_DESC(sdh, "use SDH framing (default 0)");
 
-static struct pci_device_id he_pci_tbl[] = {
+static const struct pci_device_id he_pci_tbl[] = {
 	{ PCI_VDEVICE(FORE, PCI_DEVICE_ID_FORE_HE), 0 },
 	{ 0, }
 };
diff --git a/drivers/atm/horizon.c b/drivers/atm/horizon.c
index 0f18480b33b5..7e76b35f422c 100644
--- a/drivers/atm/horizon.c
+++ b/drivers/atm/horizon.c
@@ -2867,7 +2867,7 @@ MODULE_PARM_DESC(max_tx_size, "maximum size of TX AAL5 frames");
 MODULE_PARM_DESC(max_rx_size, "maximum size of RX AAL5 frames");
 MODULE_PARM_DESC(pci_lat, "PCI latency in bus cycles");
 
-static struct pci_device_id hrz_pci_tbl[] = {
+static const struct pci_device_id hrz_pci_tbl[] = {
 	{ PCI_VENDOR_ID_MADGE, PCI_DEVICE_ID_MADGE_HORIZON, PCI_ANY_ID, PCI_ANY_ID,
 	  0, 0, 0 },
 	{ 0, }
diff --git a/drivers/atm/idt77252.c b/drivers/atm/idt77252.c
index 60bacba03d17..47f3c4ae0594 100644
--- a/drivers/atm/idt77252.c
+++ b/drivers/atm/idt77252.c
@@ -134,7 +134,7 @@ static int idt77252_proc_read(struct atm_dev *dev, loff_t * pos,
 static void idt77252_softint(struct work_struct *work);
 
 
-static struct atmdev_ops idt77252_ops =
+static const struct atmdev_ops idt77252_ops =
 {
 	.dev_close	= idt77252_dev_close,
 	.open		= idt77252_open,
@@ -3725,7 +3725,7 @@ err_out_disable_pdev:
 	return err;
 }
 
-static struct pci_device_id idt77252_pci_tbl[] =
+static const struct pci_device_id idt77252_pci_tbl[] =
 {
 	{ PCI_VDEVICE(IDT, PCI_DEVICE_ID_IDT_IDT77252), 0 },
 	{ 0, }
diff --git a/drivers/atm/iphase.c b/drivers/atm/iphase.c
index a4fa6c82261e..fc72b763fdd7 100644
--- a/drivers/atm/iphase.c
+++ b/drivers/atm/iphase.c
@@ -3266,7 +3266,7 @@ static void ia_remove_one(struct pci_dev *pdev)
       	kfree(iadev);
 }
 
-static struct pci_device_id ia_pci_tbl[] = {
+static const struct pci_device_id ia_pci_tbl[] = {
 	{ PCI_VENDOR_ID_IPHASE, 0x0008, PCI_ANY_ID, PCI_ANY_ID, },
 	{ PCI_VENDOR_ID_IPHASE, 0x0009, PCI_ANY_ID, PCI_ANY_ID, },
 	{ 0,}
diff --git a/drivers/atm/lanai.c b/drivers/atm/lanai.c
index 1a9bc51284b0..2351dad78ff5 100644
--- a/drivers/atm/lanai.c
+++ b/drivers/atm/lanai.c
@@ -2589,7 +2589,7 @@ static int lanai_init_one(struct pci_dev *pci,
 	return result;
 }
 
-static struct pci_device_id lanai_pci_tbl[] = {
+static const struct pci_device_id lanai_pci_tbl[] = {
 	{ PCI_VDEVICE(EF, PCI_DEVICE_ID_EF_ATM_LANAI2) },
 	{ PCI_VDEVICE(EF, PCI_DEVICE_ID_EF_ATM_LANAIHB) },
 	{ 0, }	/* terminal entry */
diff --git a/drivers/atm/nicstar.c b/drivers/atm/nicstar.c
index d879f3bca107..a9702836cbae 100644
--- a/drivers/atm/nicstar.c
+++ b/drivers/atm/nicstar.c
@@ -154,7 +154,7 @@ static unsigned char ns_phy_get(struct atm_dev *dev, unsigned long addr);
 
 static struct ns_dev *cards[NS_MAX_CARDS];
 static unsigned num_cards;
-static struct atmdev_ops atm_ops = {
+static const struct atmdev_ops atm_ops = {
 	.open = ns_open,
 	.close = ns_close,
 	.ioctl = ns_ioctl,
@@ -253,7 +253,7 @@ static void nicstar_remove_one(struct pci_dev *pcidev)
 	kfree(card);
 }
 
-static struct pci_device_id nicstar_pci_tbl[] = {
+static const struct pci_device_id nicstar_pci_tbl[] = {
 	{ PCI_VDEVICE(IDT, PCI_DEVICE_ID_IDT_IDT77201), 0 },
 	{0,}			/* terminate list */
 };
diff --git a/drivers/atm/solos-pci.c b/drivers/atm/solos-pci.c
index c8f2ca6d8b29..0df1a1c80b00 100644
--- a/drivers/atm/solos-pci.c
+++ b/drivers/atm/solos-pci.c
@@ -611,7 +611,7 @@ static struct attribute *solos_attrs[] = {
 	NULL
 };
 
-static struct attribute_group solos_attr_group = {
+static const struct attribute_group solos_attr_group = {
 	.attrs = solos_attrs,
 	.name = "parameters",
 };
@@ -628,7 +628,7 @@ static struct attribute *gpio_attrs[] = {
 	NULL
 };
 
-static struct attribute_group gpio_attr_group = {
+static const struct attribute_group gpio_attr_group = {
 	.attrs = gpio_attrs,
 	.name = "gpio",
 };
@@ -1187,7 +1187,7 @@ static int psend(struct atm_vcc *vcc, struct sk_buff *skb)
 	return 0;
 }
 
-static struct atmdev_ops fpga_ops = {
+static const struct atmdev_ops fpga_ops = {
 	.open =		popen,
 	.close =	pclose,
 	.ioctl =	NULL,
@@ -1476,7 +1476,7 @@ static void fpga_remove(struct pci_dev *dev)
 	kfree(card);
 }
 
-static struct pci_device_id fpga_pci_tbl[] = {
+static const struct pci_device_id fpga_pci_tbl[] = {
 	{ 0x10ee, 0x0300, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
 	{ 0, }
 };
diff --git a/drivers/atm/zatm.c b/drivers/atm/zatm.c
index 07bdd51b3b9a..1ef67db03c8e 100644
--- a/drivers/atm/zatm.c
+++ b/drivers/atm/zatm.c
@@ -1642,7 +1642,7 @@ out_free:
 
 MODULE_LICENSE("GPL");
 
-static struct pci_device_id zatm_pci_tbl[] = {
+static const struct pci_device_id zatm_pci_tbl[] = {
 	{ PCI_VDEVICE(ZEITNET, PCI_DEVICE_ID_ZEITNET_1221), ZATM_COPPER },
 	{ PCI_VDEVICE(ZEITNET, PCI_DEVICE_ID_ZEITNET_1225), 0 },
 	{ 0, }
diff --git a/drivers/auxdisplay/panel.c b/drivers/auxdisplay/panel.c
index 7a8b8fb2f572..df126dcdaf18 100644
--- a/drivers/auxdisplay/panel.c
+++ b/drivers/auxdisplay/panel.c
@@ -877,21 +877,21 @@ static void lcd_clear_fast_tilcd(struct charlcd *charlcd)
 	spin_unlock_irq(&pprt_lock);
 }
 
-static struct charlcd_ops charlcd_serial_ops = {
+static const struct charlcd_ops charlcd_serial_ops = {
 	.write_cmd	= lcd_write_cmd_s,
 	.write_data	= lcd_write_data_s,
 	.clear_fast	= lcd_clear_fast_s,
 	.backlight	= lcd_backlight,
 };
 
-static struct charlcd_ops charlcd_parallel_ops = {
+static const struct charlcd_ops charlcd_parallel_ops = {
 	.write_cmd	= lcd_write_cmd_p8,
 	.write_data	= lcd_write_data_p8,
 	.clear_fast	= lcd_clear_fast_p8,
 	.backlight	= lcd_backlight,
 };
 
-static struct charlcd_ops charlcd_tilcd_ops = {
+static const struct charlcd_ops charlcd_tilcd_ops = {
 	.write_cmd	= lcd_write_cmd_tilcd,
 	.write_data	= lcd_write_data_tilcd,
 	.clear_fast	= lcd_clear_fast_tilcd,
diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig
index f046d21de57d..1a5f6a157a57 100644
--- a/drivers/base/Kconfig
+++ b/drivers/base/Kconfig
@@ -140,13 +140,10 @@ config EXTRA_FIRMWARE
 config EXTRA_FIRMWARE_DIR
 	string "Firmware blobs root directory"
 	depends on EXTRA_FIRMWARE != ""
-	default "firmware"
+	default "/lib/firmware"
 	help
 	  This option controls the directory in which the kernel build system
 	  looks for the firmware files listed in the EXTRA_FIRMWARE option.
-	  The default is firmware/ in the kernel source tree, but by changing
-	  this option you can point it elsewhere, such as /lib/firmware/ or
-	  some other directory containing the firmware files.
 
 config FW_LOADER_USER_HELPER
 	bool
diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c
index d1c33a85059e..41be9ff7d70a 100644
--- a/drivers/base/arch_topology.c
+++ b/drivers/base/arch_topology.c
@@ -41,8 +41,7 @@ static ssize_t cpu_capacity_show(struct device *dev,
 {
 	struct cpu *cpu = container_of(dev, struct cpu, dev);
 
-	return sprintf(buf, "%lu\n",
-			topology_get_cpu_scale(NULL, cpu->dev.id));
+	return sprintf(buf, "%lu\n", topology_get_cpu_scale(NULL, cpu->dev.id));
 }
 
 static ssize_t cpu_capacity_store(struct device *dev,
@@ -96,14 +95,21 @@ subsys_initcall(register_cpu_capacity_sysctl);
 
 static u32 capacity_scale;
 static u32 *raw_capacity;
-static bool cap_parsing_failed;
+
+static int __init free_raw_capacity(void)
+{
+	kfree(raw_capacity);
+	raw_capacity = NULL;
+
+	return 0;
+}
 
 void topology_normalize_cpu_scale(void)
 {
 	u64 capacity;
 	int cpu;
 
-	if (!raw_capacity || cap_parsing_failed)
+	if (!raw_capacity)
 		return;
 
 	pr_debug("cpu_capacity: capacity_scale=%u\n", capacity_scale);
@@ -120,16 +126,16 @@ void topology_normalize_cpu_scale(void)
 	mutex_unlock(&cpu_scale_mutex);
 }
 
-int __init topology_parse_cpu_capacity(struct device_node *cpu_node, int cpu)
+bool __init topology_parse_cpu_capacity(struct device_node *cpu_node, int cpu)
 {
-	int ret = 1;
+	static bool cap_parsing_failed;
+	int ret;
 	u32 cpu_capacity;
 
 	if (cap_parsing_failed)
-		return !ret;
+		return false;
 
-	ret = of_property_read_u32(cpu_node,
-				   "capacity-dmips-mhz",
+	ret = of_property_read_u32(cpu_node, "capacity-dmips-mhz",
 				   &cpu_capacity);
 	if (!ret) {
 		if (!raw_capacity) {
@@ -139,21 +145,21 @@ int __init topology_parse_cpu_capacity(struct device_node *cpu_node, int cpu)
 			if (!raw_capacity) {
 				pr_err("cpu_capacity: failed to allocate memory for raw capacities\n");
 				cap_parsing_failed = true;
-				return 0;
+				return false;
 			}
 		}
 		capacity_scale = max(cpu_capacity, capacity_scale);
 		raw_capacity[cpu] = cpu_capacity;
-		pr_debug("cpu_capacity: %s cpu_capacity=%u (raw)\n",
-			cpu_node->full_name, raw_capacity[cpu]);
+		pr_debug("cpu_capacity: %pOF cpu_capacity=%u (raw)\n",
+			cpu_node, raw_capacity[cpu]);
 	} else {
 		if (raw_capacity) {
-			pr_err("cpu_capacity: missing %s raw capacity\n",
-				cpu_node->full_name);
+			pr_err("cpu_capacity: missing %pOF raw capacity\n",
+				cpu_node);
 			pr_err("cpu_capacity: partial information: fallback to 1024 for all CPUs\n");
 		}
 		cap_parsing_failed = true;
-		kfree(raw_capacity);
+		free_raw_capacity();
 	}
 
 	return !ret;
@@ -161,7 +167,6 @@ int __init topology_parse_cpu_capacity(struct device_node *cpu_node, int cpu)
 
 #ifdef CONFIG_CPU_FREQ
 static cpumask_var_t cpus_to_visit;
-static bool cap_parsing_done;
 static void parsing_done_workfn(struct work_struct *work);
 static DECLARE_WORK(parsing_done_work, parsing_done_workfn);
 
@@ -173,30 +178,31 @@ init_cpu_capacity_callback(struct notifier_block *nb,
 	struct cpufreq_policy *policy = data;
 	int cpu;
 
-	if (cap_parsing_failed || cap_parsing_done)
+	if (!raw_capacity)
 		return 0;
 
-	switch (val) {
-	case CPUFREQ_NOTIFY:
-		pr_debug("cpu_capacity: init cpu capacity for CPUs [%*pbl] (to_visit=%*pbl)\n",
-				cpumask_pr_args(policy->related_cpus),
-				cpumask_pr_args(cpus_to_visit));
-		cpumask_andnot(cpus_to_visit,
-			       cpus_to_visit,
-			       policy->related_cpus);
-		for_each_cpu(cpu, policy->related_cpus) {
-			raw_capacity[cpu] = topology_get_cpu_scale(NULL, cpu) *
-					    policy->cpuinfo.max_freq / 1000UL;
-			capacity_scale = max(raw_capacity[cpu], capacity_scale);
-		}
-		if (cpumask_empty(cpus_to_visit)) {
-			topology_normalize_cpu_scale();
-			kfree(raw_capacity);
-			pr_debug("cpu_capacity: parsing done\n");
-			cap_parsing_done = true;
-			schedule_work(&parsing_done_work);
-		}
+	if (val != CPUFREQ_NOTIFY)
+		return 0;
+
+	pr_debug("cpu_capacity: init cpu capacity for CPUs [%*pbl] (to_visit=%*pbl)\n",
+		 cpumask_pr_args(policy->related_cpus),
+		 cpumask_pr_args(cpus_to_visit));
+
+	cpumask_andnot(cpus_to_visit, cpus_to_visit, policy->related_cpus);
+
+	for_each_cpu(cpu, policy->related_cpus) {
+		raw_capacity[cpu] = topology_get_cpu_scale(NULL, cpu) *
+				    policy->cpuinfo.max_freq / 1000UL;
+		capacity_scale = max(raw_capacity[cpu], capacity_scale);
+	}
+
+	if (cpumask_empty(cpus_to_visit)) {
+		topology_normalize_cpu_scale();
+		free_raw_capacity();
+		pr_debug("cpu_capacity: parsing done\n");
+		schedule_work(&parsing_done_work);
 	}
+
 	return 0;
 }
 
@@ -233,11 +239,5 @@ static void parsing_done_workfn(struct work_struct *work)
 }
 
 #else
-static int __init free_raw_capacity(void)
-{
-	kfree(raw_capacity);
-
-	return 0;
-}
 core_initcall(free_raw_capacity);
 #endif
diff --git a/drivers/base/base.h b/drivers/base/base.h
index e19b1008e5fb..539432a14b5c 100644
--- a/drivers/base/base.h
+++ b/drivers/base/base.h
@@ -126,11 +126,6 @@ extern int driver_add_groups(struct device_driver *drv,
 extern void driver_remove_groups(struct device_driver *drv,
 				 const struct attribute_group **groups);
 
-extern int device_add_groups(struct device *dev,
-			     const struct attribute_group **groups);
-extern void device_remove_groups(struct device *dev,
-				 const struct attribute_group **groups);
-
 extern char *make_class_name(const char *name, struct kobject *kobj);
 
 extern int devres_release_all(struct device *dev);
diff --git a/drivers/base/bus.c b/drivers/base/bus.c
index e162c9a789ba..22a64fd3309b 100644
--- a/drivers/base/bus.c
+++ b/drivers/base/bus.c
@@ -698,7 +698,7 @@ int bus_add_driver(struct device_driver *drv)
 
 out_unregister:
 	kobject_put(&priv->kobj);
-	kfree(drv->p);
+	/* drv->p is freed in driver_release()  */
 	drv->p = NULL;
 out_put_bus:
 	bus_put(bus);
diff --git a/drivers/base/core.c b/drivers/base/core.c
index 755451f684bc..12ebd055724c 100644
--- a/drivers/base/core.c
+++ b/drivers/base/core.c
@@ -1023,12 +1023,144 @@ int device_add_groups(struct device *dev, const struct attribute_group **groups)
 {
 	return sysfs_create_groups(&dev->kobj, groups);
 }
+EXPORT_SYMBOL_GPL(device_add_groups);
 
 void device_remove_groups(struct device *dev,
 			  const struct attribute_group **groups)
 {
 	sysfs_remove_groups(&dev->kobj, groups);
 }
+EXPORT_SYMBOL_GPL(device_remove_groups);
+
+union device_attr_group_devres {
+	const struct attribute_group *group;
+	const struct attribute_group **groups;
+};
+
+static int devm_attr_group_match(struct device *dev, void *res, void *data)
+{
+	return ((union device_attr_group_devres *)res)->group == data;
+}
+
+static void devm_attr_group_remove(struct device *dev, void *res)
+{
+	union device_attr_group_devres *devres = res;
+	const struct attribute_group *group = devres->group;
+
+	dev_dbg(dev, "%s: removing group %p\n", __func__, group);
+	sysfs_remove_group(&dev->kobj, group);
+}
+
+static void devm_attr_groups_remove(struct device *dev, void *res)
+{
+	union device_attr_group_devres *devres = res;
+	const struct attribute_group **groups = devres->groups;
+
+	dev_dbg(dev, "%s: removing groups %p\n", __func__, groups);
+	sysfs_remove_groups(&dev->kobj, groups);
+}
+
+/**
+ * devm_device_add_group - given a device, create a managed attribute group
+ * @dev:	The device to create the group for
+ * @grp:	The attribute group to create
+ *
+ * This function creates a group for the first time.  It will explicitly
+ * warn and error if any of the attribute files being created already exist.
+ *
+ * Returns 0 on success or error code on failure.
+ */
+int devm_device_add_group(struct device *dev, const struct attribute_group *grp)
+{
+	union device_attr_group_devres *devres;
+	int error;
+
+	devres = devres_alloc(devm_attr_group_remove,
+			      sizeof(*devres), GFP_KERNEL);
+	if (!devres)
+		return -ENOMEM;
+
+	error = sysfs_create_group(&dev->kobj, grp);
+	if (error) {
+		devres_free(devres);
+		return error;
+	}
+
+	devres->group = grp;
+	devres_add(dev, devres);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(devm_device_add_group);
+
+/**
+ * devm_device_remove_group: remove a managed group from a device
+ * @dev:	device to remove the group from
+ * @grp:	group to remove
+ *
+ * This function removes a group of attributes from a device. The attributes
+ * previously have to have been created for this group, otherwise it will fail.
+ */
+void devm_device_remove_group(struct device *dev,
+			      const struct attribute_group *grp)
+{
+	WARN_ON(devres_release(dev, devm_attr_group_remove,
+			       devm_attr_group_match,
+			       /* cast away const */ (void *)grp));
+}
+EXPORT_SYMBOL_GPL(devm_device_remove_group);
+
+/**
+ * devm_device_add_groups - create a bunch of managed attribute groups
+ * @dev:	The device to create the group for
+ * @groups:	The attribute groups to create, NULL terminated
+ *
+ * This function creates a bunch of managed attribute groups.  If an error
+ * occurs when creating a group, all previously created groups will be
+ * removed, unwinding everything back to the original state when this
+ * function was called.  It will explicitly warn and error if any of the
+ * attribute files being created already exist.
+ *
+ * Returns 0 on success or error code from sysfs_create_group on failure.
+ */
+int devm_device_add_groups(struct device *dev,
+			   const struct attribute_group **groups)
+{
+	union device_attr_group_devres *devres;
+	int error;
+
+	devres = devres_alloc(devm_attr_groups_remove,
+			      sizeof(*devres), GFP_KERNEL);
+	if (!devres)
+		return -ENOMEM;
+
+	error = sysfs_create_groups(&dev->kobj, groups);
+	if (error) {
+		devres_free(devres);
+		return error;
+	}
+
+	devres->groups = groups;
+	devres_add(dev, devres);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(devm_device_add_groups);
+
+/**
+ * devm_device_remove_groups - remove a list of managed groups
+ *
+ * @dev:	The device for the groups to be removed from
+ * @groups:	NULL terminated list of groups to be removed
+ *
+ * If groups is not NULL, remove the specified groups from the device.
+ */
+void devm_device_remove_groups(struct device *dev,
+			       const struct attribute_group **groups)
+{
+	WARN_ON(devres_release(dev, devm_attr_groups_remove,
+			       devm_attr_group_match,
+			       /* cast away const */ (void *)groups));
+}
+EXPORT_SYMBOL_GPL(devm_device_remove_groups);
 
 static int device_add_attrs(struct device *dev)
 {
@@ -2664,11 +2796,12 @@ void device_shutdown(void)
 		pm_runtime_get_noresume(dev);
 		pm_runtime_barrier(dev);
 
-		if (dev->class && dev->class->shutdown) {
+		if (dev->class && dev->class->shutdown_pre) {
 			if (initcall_debug)
-				dev_info(dev, "shutdown\n");
-			dev->class->shutdown(dev);
-		} else if (dev->bus && dev->bus->shutdown) {
+				dev_info(dev, "shutdown_pre\n");
+			dev->class->shutdown_pre(dev);
+		}
+		if (dev->bus && dev->bus->shutdown) {
 			if (initcall_debug)
 				dev_info(dev, "shutdown\n");
 			dev->bus->shutdown(dev);
diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c
index 2c3b359b3536..321cd7b4d817 100644
--- a/drivers/base/cpu.c
+++ b/drivers/base/cpu.c
@@ -256,9 +256,9 @@ static ssize_t print_cpus_offline(struct device *dev,
 			buf[n++] = ',';
 
 		if (nr_cpu_ids == total_cpus-1)
-			n += snprintf(&buf[n], len - n, "%d", nr_cpu_ids);
+			n += snprintf(&buf[n], len - n, "%u", nr_cpu_ids);
 		else
-			n += snprintf(&buf[n], len - n, "%d-%d",
+			n += snprintf(&buf[n], len - n, "%u-%d",
 						      nr_cpu_ids, total_cpus-1);
 	}
 
diff --git a/drivers/base/dd.c b/drivers/base/dd.c
index 4882f06d12df..ad44b40fe284 100644
--- a/drivers/base/dd.c
+++ b/drivers/base/dd.c
@@ -20,6 +20,7 @@
 #include <linux/device.h>
 #include <linux/delay.h>
 #include <linux/dma-mapping.h>
+#include <linux/init.h>
 #include <linux/module.h>
 #include <linux/kthread.h>
 #include <linux/wait.h>
@@ -53,6 +54,7 @@ static DEFINE_MUTEX(deferred_probe_mutex);
 static LIST_HEAD(deferred_probe_pending_list);
 static LIST_HEAD(deferred_probe_active_list);
 static atomic_t deferred_trigger_count = ATOMIC_INIT(0);
+static bool initcalls_done;
 
 /*
  * In some cases, like suspend to RAM or hibernation, It might be reasonable
@@ -62,6 +64,26 @@ static atomic_t deferred_trigger_count = ATOMIC_INIT(0);
 static bool defer_all_probes;
 
 /*
+ * For initcall_debug, show the deferred probes executed in late_initcall
+ * processing.
+ */
+static void deferred_probe_debug(struct device *dev)
+{
+	ktime_t calltime, delta, rettime;
+	unsigned long long duration;
+
+	printk(KERN_DEBUG "deferred probe %s @ %i\n", dev_name(dev),
+	       task_pid_nr(current));
+	calltime = ktime_get();
+	bus_probe_device(dev);
+	rettime = ktime_get();
+	delta = ktime_sub(rettime, calltime);
+	duration = (unsigned long long) ktime_to_ns(delta) >> 10;
+	printk(KERN_DEBUG "deferred probe %s returned after %lld usecs\n",
+	       dev_name(dev), duration);
+}
+
+/*
  * deferred_probe_work_func() - Retry probing devices in the active list.
  */
 static void deferred_probe_work_func(struct work_struct *work)
@@ -106,7 +128,10 @@ static void deferred_probe_work_func(struct work_struct *work)
 		device_pm_unlock();
 
 		dev_dbg(dev, "Retrying from deferred list\n");
-		bus_probe_device(dev);
+		if (initcall_debug && !initcalls_done)
+			deferred_probe_debug(dev);
+		else
+			bus_probe_device(dev);
 
 		mutex_lock(&deferred_probe_mutex);
 
@@ -215,6 +240,7 @@ static int deferred_probe_initcall(void)
 	driver_deferred_probe_trigger();
 	/* Sort as many dependencies as possible before exiting initcalls */
 	flush_work(&deferred_probe_work);
+	initcalls_done = true;
 	return 0;
 }
 late_initcall(deferred_probe_initcall);
@@ -259,6 +285,8 @@ static void driver_bound(struct device *dev)
 	if (dev->bus)
 		blocking_notifier_call_chain(&dev->bus->p->bus_notifier,
 					     BUS_NOTIFY_BOUND_DRIVER, dev);
+
+	kobject_uevent(&dev->kobj, KOBJ_BIND);
 }
 
 static int driver_sysfs_add(struct device *dev)
@@ -848,6 +876,8 @@ static void __device_release_driver(struct device *dev, struct device *parent)
 			blocking_notifier_call_chain(&dev->bus->p->bus_notifier,
 						     BUS_NOTIFY_UNBOUND_DRIVER,
 						     dev);
+
+		kobject_uevent(&dev->kobj, KOBJ_UNBIND);
 	}
 }
 
diff --git a/drivers/base/dma-coherent.c b/drivers/base/dma-coherent.c
index 1c152aed6b82..744f64f43454 100644
--- a/drivers/base/dma-coherent.c
+++ b/drivers/base/dma-coherent.c
@@ -37,7 +37,7 @@ static inline dma_addr_t dma_get_device_base(struct device *dev,
 		return mem->device_base;
 }
 
-static bool dma_init_coherent_memory(
+static int dma_init_coherent_memory(
 	phys_addr_t phys_addr, dma_addr_t device_addr, size_t size, int flags,
 	struct dma_coherent_mem **mem)
 {
@@ -45,25 +45,28 @@ static bool dma_init_coherent_memory(
 	void __iomem *mem_base = NULL;
 	int pages = size >> PAGE_SHIFT;
 	int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long);
+	int ret;
 
-	if ((flags & (DMA_MEMORY_MAP | DMA_MEMORY_IO)) == 0)
-		goto out;
-	if (!size)
+	if (!size) {
+		ret = -EINVAL;
 		goto out;
+	}
 
-	if (flags & DMA_MEMORY_MAP)
-		mem_base = memremap(phys_addr, size, MEMREMAP_WC);
-	else
-		mem_base = ioremap(phys_addr, size);
-	if (!mem_base)
+	mem_base = memremap(phys_addr, size, MEMREMAP_WC);
+	if (!mem_base) {
+		ret = -EINVAL;
 		goto out;
-
+	}
 	dma_mem = kzalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL);
-	if (!dma_mem)
+	if (!dma_mem) {
+		ret = -ENOMEM;
 		goto out;
+	}
 	dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
-	if (!dma_mem->bitmap)
+	if (!dma_mem->bitmap) {
+		ret = -ENOMEM;
 		goto out;
+	}
 
 	dma_mem->virt_base = mem_base;
 	dma_mem->device_base = device_addr;
@@ -73,17 +76,13 @@ static bool dma_init_coherent_memory(
 	spin_lock_init(&dma_mem->spinlock);
 
 	*mem = dma_mem;
-	return true;
+	return 0;
 
 out:
 	kfree(dma_mem);
-	if (mem_base) {
-		if (flags & DMA_MEMORY_MAP)
-			memunmap(mem_base);
-		else
-			iounmap(mem_base);
-	}
-	return false;
+	if (mem_base)
+		memunmap(mem_base);
+	return ret;
 }
 
 static void dma_release_coherent_memory(struct dma_coherent_mem *mem)
@@ -91,10 +90,7 @@ static void dma_release_coherent_memory(struct dma_coherent_mem *mem)
 	if (!mem)
 		return;
 
-	if (mem->flags & DMA_MEMORY_MAP)
-		memunmap(mem->virt_base);
-	else
-		iounmap(mem->virt_base);
+	memunmap(mem->virt_base);
 	kfree(mem->bitmap);
 	kfree(mem);
 }
@@ -109,8 +105,6 @@ static int dma_assign_coherent_memory(struct device *dev,
 		return -EBUSY;
 
 	dev->dma_mem = mem;
-	/* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */
-
 	return 0;
 }
 
@@ -118,16 +112,16 @@ int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
 				dma_addr_t device_addr, size_t size, int flags)
 {
 	struct dma_coherent_mem *mem;
+	int ret;
 
-	if (!dma_init_coherent_memory(phys_addr, device_addr, size, flags,
-				      &mem))
-		return 0;
-
-	if (dma_assign_coherent_memory(dev, mem) == 0)
-		return flags & DMA_MEMORY_MAP ? DMA_MEMORY_MAP : DMA_MEMORY_IO;
+	ret = dma_init_coherent_memory(phys_addr, device_addr, size, flags, &mem);
+	if (ret)
+		return ret;
 
-	dma_release_coherent_memory(mem);
-	return 0;
+	ret = dma_assign_coherent_memory(dev, mem);
+	if (ret)
+		dma_release_coherent_memory(mem);
+	return ret;
 }
 EXPORT_SYMBOL(dma_declare_coherent_memory);
 
@@ -171,7 +165,6 @@ static void *__dma_alloc_from_coherent(struct dma_coherent_mem *mem,
 	int order = get_order(size);
 	unsigned long flags;
 	int pageno;
-	int dma_memory_map;
 	void *ret;
 
 	spin_lock_irqsave(&mem->spinlock, flags);
@@ -188,15 +181,9 @@ static void *__dma_alloc_from_coherent(struct dma_coherent_mem *mem,
 	 */
 	*dma_handle = mem->device_base + (pageno << PAGE_SHIFT);
 	ret = mem->virt_base + (pageno << PAGE_SHIFT);
-	dma_memory_map = (mem->flags & DMA_MEMORY_MAP);
 	spin_unlock_irqrestore(&mem->spinlock, flags);
-	if (dma_memory_map)
-		memset(ret, 0, size);
-	else
-		memset_io(ret, 0, size);
-
+	memset(ret, 0, size);
 	return ret;
-
 err:
 	spin_unlock_irqrestore(&mem->spinlock, flags);
 	return NULL;
@@ -359,14 +346,17 @@ static struct reserved_mem *dma_reserved_default_memory __initdata;
 static int rmem_dma_device_init(struct reserved_mem *rmem, struct device *dev)
 {
 	struct dma_coherent_mem *mem = rmem->priv;
+	int ret;
 
-	if (!mem &&
-	    !dma_init_coherent_memory(rmem->base, rmem->base, rmem->size,
-				      DMA_MEMORY_MAP | DMA_MEMORY_EXCLUSIVE,
-				      &mem)) {
-		pr_err("Reserved memory: failed to init DMA memory pool at %pa, size %ld MiB\n",
-			&rmem->base, (unsigned long)rmem->size / SZ_1M);
-		return -ENODEV;
+	if (!mem) {
+		ret = dma_init_coherent_memory(rmem->base, rmem->base,
+					       rmem->size,
+					       DMA_MEMORY_EXCLUSIVE, &mem);
+		if (ret) {
+			pr_err("Reserved memory: failed to init DMA memory pool at %pa, size %ld MiB\n",
+				&rmem->base, (unsigned long)rmem->size / SZ_1M);
+			return ret;
+		}
 	}
 	mem->use_dev_dma_pfn_offset = true;
 	rmem->priv = mem;
diff --git a/drivers/base/dma-mapping.c b/drivers/base/dma-mapping.c
index b555ff9dd8fc..e584eddef0a7 100644
--- a/drivers/base/dma-mapping.c
+++ b/drivers/base/dma-mapping.c
@@ -176,13 +176,10 @@ int dmam_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
 
 	rc = dma_declare_coherent_memory(dev, phys_addr, device_addr, size,
 					 flags);
-	if (rc) {
+	if (!rc)
 		devres_add(dev, res);
-		rc = 0;
-	} else {
+	else
 		devres_free(res);
-		rc = -ENOMEM;
-	}
 
 	return rc;
 }
diff --git a/drivers/base/firmware_class.c b/drivers/base/firmware_class.c
index bfbe1e154128..4b57cf5bc81d 100644
--- a/drivers/base/firmware_class.c
+++ b/drivers/base/firmware_class.c
@@ -7,6 +7,8 @@
  *
  */
 
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
 #include <linux/capability.h>
 #include <linux/device.h>
 #include <linux/module.h>
@@ -256,38 +258,6 @@ static int fw_cache_piggyback_on_request(const char *name);
  * guarding for corner cases a global lock should be OK */
 static DEFINE_MUTEX(fw_lock);
 
-static bool __enable_firmware = false;
-
-static void enable_firmware(void)
-{
-	mutex_lock(&fw_lock);
-	__enable_firmware = true;
-	mutex_unlock(&fw_lock);
-}
-
-static void disable_firmware(void)
-{
-	mutex_lock(&fw_lock);
-	__enable_firmware = false;
-	mutex_unlock(&fw_lock);
-}
-
-/*
- * When disabled only the built-in firmware and the firmware cache will be
- * used to look for firmware.
- */
-static bool firmware_enabled(void)
-{
-	bool enabled = false;
-
-	mutex_lock(&fw_lock);
-	if (__enable_firmware)
-		enabled = true;
-	mutex_unlock(&fw_lock);
-
-	return enabled;
-}
-
 static struct firmware_cache fw_cache;
 
 static struct firmware_buf *__allocate_fw_buf(const char *fw_name,
@@ -331,6 +301,7 @@ static struct firmware_buf *__fw_lookup_buf(const char *fw_name)
 	return NULL;
 }
 
+/* Returns 1 for batching firmware requests with the same name */
 static int fw_lookup_and_allocate_buf(const char *fw_name,
 				      struct firmware_cache *fwc,
 				      struct firmware_buf **buf, void *dbuf,
@@ -344,6 +315,7 @@ static int fw_lookup_and_allocate_buf(const char *fw_name,
 		kref_get(&tmp->ref);
 		spin_unlock(&fwc->lock);
 		*buf = tmp;
+		pr_debug("batched request - sharing the same struct firmware_buf and lookup for multiple requests\n");
 		return 1;
 	}
 	tmp = __allocate_fw_buf(fw_name, fwc, dbuf, size);
@@ -1085,9 +1057,12 @@ static int _request_firmware_load(struct firmware_priv *fw_priv,
 		mutex_unlock(&fw_lock);
 	}
 
-	if (fw_state_is_aborted(&buf->fw_st))
-		retval = -EAGAIN;
-	else if (buf->is_paged_buf && !buf->data)
+	if (fw_state_is_aborted(&buf->fw_st)) {
+		if (retval == -ERESTARTSYS)
+			retval = -EINTR;
+		else
+			retval = -EAGAIN;
+	} else if (buf->is_paged_buf && !buf->data)
 		retval = -ENOMEM;
 
 	device_del(f_dev);
@@ -1239,12 +1214,6 @@ _request_firmware(const struct firmware **firmware_p, const char *name,
 	if (ret <= 0) /* error or already assigned */
 		goto out;
 
-	if (!firmware_enabled()) {
-		WARN(1, "firmware request while host is not available\n");
-		ret = -EHOSTDOWN;
-		goto out;
-	}
-
 	ret = fw_get_filesystem_firmware(device, fw->priv);
 	if (ret) {
 		if (!(opt_flags & FW_OPT_NO_WARN))
@@ -1755,62 +1724,6 @@ static void device_uncache_fw_images_delay(unsigned long delay)
 			   msecs_to_jiffies(delay));
 }
 
-/**
- * fw_pm_notify - notifier for suspend/resume
- * @notify_block: unused
- * @mode: mode we are switching to
- * @unused: unused
- *
- * Used to modify the firmware_class state as we move in between states.
- * The firmware_class implements a firmware cache to enable device driver
- * to fetch firmware upon resume before the root filesystem is ready. We
- * disable API calls which do not use the built-in firmware or the firmware
- * cache when we know these calls will not work.
- *
- * The inner logic behind all this is a bit complex so it is worth summarizing
- * the kernel's own suspend/resume process with context and focus on how this
- * can impact the firmware API.
- *
- * First a review on how we go to suspend::
- *
- *	pm_suspend() --> enter_state() -->
- *	sys_sync()
- *	suspend_prepare() -->
- *		__pm_notifier_call_chain(PM_SUSPEND_PREPARE, ...);
- *		suspend_freeze_processes() -->
- *			freeze_processes() -->
- *				__usermodehelper_set_disable_depth(UMH_DISABLED);
- *				freeze all tasks ...
- *			freeze_kernel_threads()
- *	suspend_devices_and_enter() -->
- *		dpm_suspend_start() -->
- *				dpm_prepare()
- *				dpm_suspend()
- *		suspend_enter()  -->
- *			platform_suspend_prepare()
- *			dpm_suspend_late()
- *			freeze_enter()
- *			syscore_suspend()
- *
- * When we resume we bail out of a loop from suspend_devices_and_enter() and
- * unwind back out to the caller enter_state() where we were before as follows::
- *
- * 	enter_state() -->
- *	suspend_devices_and_enter() --> (bail from loop)
- *		dpm_resume_end() -->
- *			dpm_resume()
- *			dpm_complete()
- *	suspend_finish() -->
- *		suspend_thaw_processes() -->
- *			thaw_processes() -->
- *				__usermodehelper_set_disable_depth(UMH_FREEZING);
- *				thaw_workqueues();
- *				thaw all processes ...
- *				usermodehelper_enable();
- *		pm_notifier_call_chain(PM_POST_SUSPEND);
- *
- * fw_pm_notify() works through pm_notifier_call_chain().
- */
 static int fw_pm_notify(struct notifier_block *notify_block,
 			unsigned long mode, void *unused)
 {
@@ -1824,7 +1737,6 @@ static int fw_pm_notify(struct notifier_block *notify_block,
 		 */
 		kill_pending_fw_fallback_reqs(true);
 		device_cache_fw_images();
-		disable_firmware();
 		break;
 
 	case PM_POST_SUSPEND:
@@ -1837,7 +1749,6 @@ static int fw_pm_notify(struct notifier_block *notify_block,
 		mutex_lock(&fw_lock);
 		fw_cache.state = FW_LOADER_NO_CACHE;
 		mutex_unlock(&fw_lock);
-		enable_firmware();
 
 		device_uncache_fw_images_delay(10 * MSEC_PER_SEC);
 		break;
@@ -1886,7 +1797,6 @@ static void __init fw_cache_init(void)
 static int fw_shutdown_notify(struct notifier_block *unused1,
 			      unsigned long unused2, void *unused3)
 {
-	disable_firmware();
 	/*
 	 * Kill all pending fallback requests to avoid both stalling shutdown,
 	 * and avoid a deadlock with the usermode_lock.
@@ -1902,7 +1812,6 @@ static struct notifier_block fw_shutdown_nb = {
 
 static int __init firmware_class_init(void)
 {
-	enable_firmware();
 	fw_cache_init();
 	register_reboot_notifier(&fw_shutdown_nb);
 #ifdef CONFIG_FW_LOADER_USER_HELPER
@@ -1914,7 +1823,6 @@ static int __init firmware_class_init(void)
 
 static void __exit firmware_class_exit(void)
 {
-	disable_firmware();
 #ifdef CONFIG_PM_SLEEP
 	unregister_syscore_ops(&fw_syscore_ops);
 	unregister_pm_notifier(&fw_cache.pm_notify);
diff --git a/drivers/base/memory.c b/drivers/base/memory.c
index c7c4e0325cdb..4e3b61cda520 100644
--- a/drivers/base/memory.c
+++ b/drivers/base/memory.c
@@ -388,6 +388,19 @@ static ssize_t show_phys_device(struct device *dev,
 }
 
 #ifdef CONFIG_MEMORY_HOTREMOVE
+static void print_allowed_zone(char *buf, int nid, unsigned long start_pfn,
+		unsigned long nr_pages, int online_type,
+		struct zone *default_zone)
+{
+	struct zone *zone;
+
+	zone = zone_for_pfn_range(online_type, nid, start_pfn, nr_pages);
+	if (zone != default_zone) {
+		strcat(buf, " ");
+		strcat(buf, zone->name);
+	}
+}
+
 static ssize_t show_valid_zones(struct device *dev,
 				struct device_attribute *attr, char *buf)
 {
@@ -395,7 +408,7 @@ static ssize_t show_valid_zones(struct device *dev,
 	unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr);
 	unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block;
 	unsigned long valid_start_pfn, valid_end_pfn;
-	bool append = false;
+	struct zone *default_zone;
 	int nid;
 
 	/*
@@ -418,16 +431,13 @@ static ssize_t show_valid_zones(struct device *dev,
 	}
 
 	nid = pfn_to_nid(start_pfn);
-	if (allow_online_pfn_range(nid, start_pfn, nr_pages, MMOP_ONLINE_KERNEL)) {
-		strcat(buf, default_zone_for_pfn(nid, start_pfn, nr_pages)->name);
-		append = true;
-	}
+	default_zone = zone_for_pfn_range(MMOP_ONLINE_KEEP, nid, start_pfn, nr_pages);
+	strcat(buf, default_zone->name);
 
-	if (allow_online_pfn_range(nid, start_pfn, nr_pages, MMOP_ONLINE_MOVABLE)) {
-		if (append)
-			strcat(buf, " ");
-		strcat(buf, NODE_DATA(nid)->node_zones[ZONE_MOVABLE].name);
-	}
+	print_allowed_zone(buf, nid, start_pfn, nr_pages, MMOP_ONLINE_KERNEL,
+			default_zone);
+	print_allowed_zone(buf, nid, start_pfn, nr_pages, MMOP_ONLINE_MOVABLE,
+			default_zone);
 out:
 	strcat(buf, "\n");
 
diff --git a/drivers/base/node.c b/drivers/base/node.c
index d8dc83017d8d..3855902f2c5b 100644
--- a/drivers/base/node.c
+++ b/drivers/base/node.c
@@ -160,12 +160,12 @@ static ssize_t node_read_numastat(struct device *dev,
 		       "interleave_hit %lu\n"
 		       "local_node %lu\n"
 		       "other_node %lu\n",
-		       sum_zone_node_page_state(dev->id, NUMA_HIT),
-		       sum_zone_node_page_state(dev->id, NUMA_MISS),
-		       sum_zone_node_page_state(dev->id, NUMA_FOREIGN),
-		       sum_zone_node_page_state(dev->id, NUMA_INTERLEAVE_HIT),
-		       sum_zone_node_page_state(dev->id, NUMA_LOCAL),
-		       sum_zone_node_page_state(dev->id, NUMA_OTHER));
+		       sum_zone_numa_state(dev->id, NUMA_HIT),
+		       sum_zone_numa_state(dev->id, NUMA_MISS),
+		       sum_zone_numa_state(dev->id, NUMA_FOREIGN),
+		       sum_zone_numa_state(dev->id, NUMA_INTERLEAVE_HIT),
+		       sum_zone_numa_state(dev->id, NUMA_LOCAL),
+		       sum_zone_numa_state(dev->id, NUMA_OTHER));
 }
 static DEVICE_ATTR(numastat, S_IRUGO, node_read_numastat, NULL);
 
@@ -181,9 +181,17 @@ static ssize_t node_read_vmstat(struct device *dev,
 		n += sprintf(buf+n, "%s %lu\n", vmstat_text[i],
 			     sum_zone_node_page_state(nid, i));
 
-	for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
+#ifdef CONFIG_NUMA
+	for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++)
 		n += sprintf(buf+n, "%s %lu\n",
 			     vmstat_text[i + NR_VM_ZONE_STAT_ITEMS],
+			     sum_zone_numa_state(nid, i));
+#endif
+
+	for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
+		n += sprintf(buf+n, "%s %lu\n",
+			     vmstat_text[i + NR_VM_ZONE_STAT_ITEMS +
+			     NR_VM_NUMA_STAT_ITEMS],
 			     node_page_state(pgdat, i));
 
 	return n;
diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c
index 60303aa28587..e8ca5e2cf1e5 100644
--- a/drivers/base/power/domain.c
+++ b/drivers/base/power/domain.c
@@ -209,6 +209,34 @@ static void genpd_sd_counter_inc(struct generic_pm_domain *genpd)
 	smp_mb__after_atomic();
 }
 
+#ifdef CONFIG_DEBUG_FS
+static void genpd_update_accounting(struct generic_pm_domain *genpd)
+{
+	ktime_t delta, now;
+
+	now = ktime_get();
+	delta = ktime_sub(now, genpd->accounting_time);
+
+	/*
+	 * If genpd->status is active, it means we are just
+	 * out of off and so update the idle time and vice
+	 * versa.
+	 */
+	if (genpd->status == GPD_STATE_ACTIVE) {
+		int state_idx = genpd->state_idx;
+
+		genpd->states[state_idx].idle_time =
+			ktime_add(genpd->states[state_idx].idle_time, delta);
+	} else {
+		genpd->on_time = ktime_add(genpd->on_time, delta);
+	}
+
+	genpd->accounting_time = now;
+}
+#else
+static inline void genpd_update_accounting(struct generic_pm_domain *genpd) {}
+#endif
+
 static int _genpd_power_on(struct generic_pm_domain *genpd, bool timed)
 {
 	unsigned int state_idx = genpd->state_idx;
@@ -361,6 +389,7 @@ static int genpd_power_off(struct generic_pm_domain *genpd, bool one_dev_on,
 	}
 
 	genpd->status = GPD_STATE_POWER_OFF;
+	genpd_update_accounting(genpd);
 
 	list_for_each_entry(link, &genpd->slave_links, slave_node) {
 		genpd_sd_counter_dec(link->master);
@@ -413,6 +442,8 @@ static int genpd_power_on(struct generic_pm_domain *genpd, unsigned int depth)
 		goto err;
 
 	genpd->status = GPD_STATE_ACTIVE;
+	genpd_update_accounting(genpd);
+
 	return 0;
 
  err:
@@ -1540,6 +1571,7 @@ int pm_genpd_init(struct generic_pm_domain *genpd,
 	genpd->max_off_time_changed = true;
 	genpd->provider = NULL;
 	genpd->has_provider = false;
+	genpd->accounting_time = ktime_get();
 	genpd->domain.ops.runtime_suspend = genpd_runtime_suspend;
 	genpd->domain.ops.runtime_resume = genpd_runtime_resume;
 	genpd->domain.ops.prepare = pm_genpd_prepare;
@@ -1743,7 +1775,7 @@ static int genpd_add_provider(struct device_node *np, genpd_xlate_t xlate,
 	mutex_lock(&of_genpd_mutex);
 	list_add(&cp->link, &of_genpd_providers);
 	mutex_unlock(&of_genpd_mutex);
-	pr_debug("Added domain provider from %s\n", np->full_name);
+	pr_debug("Added domain provider from %pOF\n", np);
 
 	return 0;
 }
@@ -2149,16 +2181,16 @@ static int genpd_parse_state(struct genpd_power_state *genpd_state,
 	err = of_property_read_u32(state_node, "entry-latency-us",
 						&entry_latency);
 	if (err) {
-		pr_debug(" * %s missing entry-latency-us property\n",
-						state_node->full_name);
+		pr_debug(" * %pOF missing entry-latency-us property\n",
+						state_node);
 		return -EINVAL;
 	}
 
 	err = of_property_read_u32(state_node, "exit-latency-us",
 						&exit_latency);
 	if (err) {
-		pr_debug(" * %s missing exit-latency-us property\n",
-						state_node->full_name);
+		pr_debug(" * %pOF missing exit-latency-us property\n",
+						state_node);
 		return -EINVAL;
 	}
 
@@ -2212,8 +2244,8 @@ int of_genpd_parse_idle_states(struct device_node *dn,
 		ret = genpd_parse_state(&st[i++], np);
 		if (ret) {
 			pr_err
-			("Parsing idle state node %s failed with err %d\n",
-							np->full_name, ret);
+			("Parsing idle state node %pOF failed with err %d\n",
+							np, ret);
 			of_node_put(np);
 			kfree(st);
 			return ret;
@@ -2327,7 +2359,7 @@ exit:
 	return 0;
 }
 
-static int pm_genpd_summary_show(struct seq_file *s, void *data)
+static int genpd_summary_show(struct seq_file *s, void *data)
 {
 	struct generic_pm_domain *genpd;
 	int ret = 0;
@@ -2350,21 +2382,187 @@ static int pm_genpd_summary_show(struct seq_file *s, void *data)
 	return ret;
 }
 
-static int pm_genpd_summary_open(struct inode *inode, struct file *file)
+static int genpd_status_show(struct seq_file *s, void *data)
 {
-	return single_open(file, pm_genpd_summary_show, NULL);
+	static const char * const status_lookup[] = {
+		[GPD_STATE_ACTIVE] = "on",
+		[GPD_STATE_POWER_OFF] = "off"
+	};
+
+	struct generic_pm_domain *genpd = s->private;
+	int ret = 0;
+
+	ret = genpd_lock_interruptible(genpd);
+	if (ret)
+		return -ERESTARTSYS;
+
+	if (WARN_ON_ONCE(genpd->status >= ARRAY_SIZE(status_lookup)))
+		goto exit;
+
+	if (genpd->status == GPD_STATE_POWER_OFF)
+		seq_printf(s, "%s-%u\n", status_lookup[genpd->status],
+			genpd->state_idx);
+	else
+		seq_printf(s, "%s\n", status_lookup[genpd->status]);
+exit:
+	genpd_unlock(genpd);
+	return ret;
 }
 
-static const struct file_operations pm_genpd_summary_fops = {
-	.open = pm_genpd_summary_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-};
+static int genpd_sub_domains_show(struct seq_file *s, void *data)
+{
+	struct generic_pm_domain *genpd = s->private;
+	struct gpd_link *link;
+	int ret = 0;
+
+	ret = genpd_lock_interruptible(genpd);
+	if (ret)
+		return -ERESTARTSYS;
+
+	list_for_each_entry(link, &genpd->master_links, master_node)
+		seq_printf(s, "%s\n", link->slave->name);
+
+	genpd_unlock(genpd);
+	return ret;
+}
+
+static int genpd_idle_states_show(struct seq_file *s, void *data)
+{
+	struct generic_pm_domain *genpd = s->private;
+	unsigned int i;
+	int ret = 0;
+
+	ret = genpd_lock_interruptible(genpd);
+	if (ret)
+		return -ERESTARTSYS;
+
+	seq_puts(s, "State          Time Spent(ms)\n");
+
+	for (i = 0; i < genpd->state_count; i++) {
+		ktime_t delta = 0;
+		s64 msecs;
+
+		if ((genpd->status == GPD_STATE_POWER_OFF) &&
+				(genpd->state_idx == i))
+			delta = ktime_sub(ktime_get(), genpd->accounting_time);
+
+		msecs = ktime_to_ms(
+			ktime_add(genpd->states[i].idle_time, delta));
+		seq_printf(s, "S%-13i %lld\n", i, msecs);
+	}
+
+	genpd_unlock(genpd);
+	return ret;
+}
+
+static int genpd_active_time_show(struct seq_file *s, void *data)
+{
+	struct generic_pm_domain *genpd = s->private;
+	ktime_t delta = 0;
+	int ret = 0;
+
+	ret = genpd_lock_interruptible(genpd);
+	if (ret)
+		return -ERESTARTSYS;
+
+	if (genpd->status == GPD_STATE_ACTIVE)
+		delta = ktime_sub(ktime_get(), genpd->accounting_time);
+
+	seq_printf(s, "%lld ms\n", ktime_to_ms(
+				ktime_add(genpd->on_time, delta)));
+
+	genpd_unlock(genpd);
+	return ret;
+}
+
+static int genpd_total_idle_time_show(struct seq_file *s, void *data)
+{
+	struct generic_pm_domain *genpd = s->private;
+	ktime_t delta = 0, total = 0;
+	unsigned int i;
+	int ret = 0;
+
+	ret = genpd_lock_interruptible(genpd);
+	if (ret)
+		return -ERESTARTSYS;
+
+	for (i = 0; i < genpd->state_count; i++) {
+
+		if ((genpd->status == GPD_STATE_POWER_OFF) &&
+				(genpd->state_idx == i))
+			delta = ktime_sub(ktime_get(), genpd->accounting_time);
+
+		total = ktime_add(total, genpd->states[i].idle_time);
+	}
+	total = ktime_add(total, delta);
+
+	seq_printf(s, "%lld ms\n", ktime_to_ms(total));
+
+	genpd_unlock(genpd);
+	return ret;
+}
+
+
+static int genpd_devices_show(struct seq_file *s, void *data)
+{
+	struct generic_pm_domain *genpd = s->private;
+	struct pm_domain_data *pm_data;
+	const char *kobj_path;
+	int ret = 0;
+
+	ret = genpd_lock_interruptible(genpd);
+	if (ret)
+		return -ERESTARTSYS;
+
+	list_for_each_entry(pm_data, &genpd->dev_list, list_node) {
+		kobj_path = kobject_get_path(&pm_data->dev->kobj,
+				genpd_is_irq_safe(genpd) ?
+				GFP_ATOMIC : GFP_KERNEL);
+		if (kobj_path == NULL)
+			continue;
+
+		seq_printf(s, "%s\n", kobj_path);
+		kfree(kobj_path);
+	}
+
+	genpd_unlock(genpd);
+	return ret;
+}
+
+#define define_genpd_open_function(name) \
+static int genpd_##name##_open(struct inode *inode, struct file *file) \
+{ \
+	return single_open(file, genpd_##name##_show, inode->i_private); \
+}
+
+define_genpd_open_function(summary);
+define_genpd_open_function(status);
+define_genpd_open_function(sub_domains);
+define_genpd_open_function(idle_states);
+define_genpd_open_function(active_time);
+define_genpd_open_function(total_idle_time);
+define_genpd_open_function(devices);
+
+#define define_genpd_debugfs_fops(name) \
+static const struct file_operations genpd_##name##_fops = { \
+	.open = genpd_##name##_open, \
+	.read = seq_read, \
+	.llseek = seq_lseek, \
+	.release = single_release, \
+}
+
+define_genpd_debugfs_fops(summary);
+define_genpd_debugfs_fops(status);
+define_genpd_debugfs_fops(sub_domains);
+define_genpd_debugfs_fops(idle_states);
+define_genpd_debugfs_fops(active_time);
+define_genpd_debugfs_fops(total_idle_time);
+define_genpd_debugfs_fops(devices);
 
 static int __init pm_genpd_debug_init(void)
 {
 	struct dentry *d;
+	struct generic_pm_domain *genpd;
 
 	pm_genpd_debugfs_dir = debugfs_create_dir("pm_genpd", NULL);
 
@@ -2372,10 +2570,29 @@ static int __init pm_genpd_debug_init(void)
 		return -ENOMEM;
 
 	d = debugfs_create_file("pm_genpd_summary", S_IRUGO,
-			pm_genpd_debugfs_dir, NULL, &pm_genpd_summary_fops);
+			pm_genpd_debugfs_dir, NULL, &genpd_summary_fops);
 	if (!d)
 		return -ENOMEM;
 
+	list_for_each_entry(genpd, &gpd_list, gpd_list_node) {
+		d = debugfs_create_dir(genpd->name, pm_genpd_debugfs_dir);
+		if (!d)
+			return -ENOMEM;
+
+		debugfs_create_file("current_state", 0444,
+				d, genpd, &genpd_status_fops);
+		debugfs_create_file("sub_domains", 0444,
+				d, genpd, &genpd_sub_domains_fops);
+		debugfs_create_file("idle_states", 0444,
+				d, genpd, &genpd_idle_states_fops);
+		debugfs_create_file("active_time", 0444,
+				d, genpd, &genpd_active_time_fops);
+		debugfs_create_file("total_idle_time", 0444,
+				d, genpd, &genpd_total_idle_time_fops);
+		debugfs_create_file("devices", 0444,
+				d, genpd, &genpd_devices_fops);
+	}
+
 	return 0;
 }
 late_initcall(pm_genpd_debug_init);
diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c
index c99f8730de82..ea1732ed7a9d 100644
--- a/drivers/base/power/main.c
+++ b/drivers/base/power/main.c
@@ -418,8 +418,7 @@ static void pm_dev_err(struct device *dev, pm_message_t state, const char *info,
 		dev_name(dev), pm_verb(state.event), info, error);
 }
 
-#ifdef CONFIG_PM_DEBUG
-static void dpm_show_time(ktime_t starttime, pm_message_t state,
+static void dpm_show_time(ktime_t starttime, pm_message_t state, int error,
 			  const char *info)
 {
 	ktime_t calltime;
@@ -432,14 +431,12 @@ static void dpm_show_time(ktime_t starttime, pm_message_t state,
 	usecs = usecs64;
 	if (usecs == 0)
 		usecs = 1;
-	pr_info("PM: %s%s%s of devices complete after %ld.%03ld msecs\n",
-		info ?: "", info ? " " : "", pm_verb(state.event),
-		usecs / USEC_PER_MSEC, usecs % USEC_PER_MSEC);
+
+	pm_pr_dbg("%s%s%s of devices %s after %ld.%03ld msecs\n",
+		  info ?: "", info ? " " : "", pm_verb(state.event),
+		  error ? "aborted" : "complete",
+		  usecs / USEC_PER_MSEC, usecs % USEC_PER_MSEC);
 }
-#else
-static inline void dpm_show_time(ktime_t starttime, pm_message_t state,
-				 const char *info) {}
-#endif /* CONFIG_PM_DEBUG */
 
 static int dpm_run_callback(pm_callback_t cb, struct device *dev,
 			    pm_message_t state, const char *info)
@@ -602,14 +599,7 @@ static void async_resume_noirq(void *data, async_cookie_t cookie)
 	put_device(dev);
 }
 
-/**
- * dpm_resume_noirq - Execute "noirq resume" callbacks for all devices.
- * @state: PM transition of the system being carried out.
- *
- * Call the "noirq" resume handlers for all devices in dpm_noirq_list and
- * enable device drivers to receive interrupts.
- */
-void dpm_resume_noirq(pm_message_t state)
+void dpm_noirq_resume_devices(pm_message_t state)
 {
 	struct device *dev;
 	ktime_t starttime = ktime_get();
@@ -654,11 +644,28 @@ void dpm_resume_noirq(pm_message_t state)
 	}
 	mutex_unlock(&dpm_list_mtx);
 	async_synchronize_full();
-	dpm_show_time(starttime, state, "noirq");
+	dpm_show_time(starttime, state, 0, "noirq");
+	trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, false);
+}
+
+void dpm_noirq_end(void)
+{
 	resume_device_irqs();
 	device_wakeup_disarm_wake_irqs();
 	cpuidle_resume();
-	trace_suspend_resume(TPS("dpm_resume_noirq"), state.event, false);
+}
+
+/**
+ * dpm_resume_noirq - Execute "noirq resume" callbacks for all devices.
+ * @state: PM transition of the system being carried out.
+ *
+ * Invoke the "noirq" resume callbacks for all devices in dpm_noirq_list and
+ * allow device drivers' interrupt handlers to be called.
+ */
+void dpm_resume_noirq(pm_message_t state)
+{
+	dpm_noirq_resume_devices(state);
+	dpm_noirq_end();
 }
 
 /**
@@ -776,7 +783,7 @@ void dpm_resume_early(pm_message_t state)
 	}
 	mutex_unlock(&dpm_list_mtx);
 	async_synchronize_full();
-	dpm_show_time(starttime, state, "early");
+	dpm_show_time(starttime, state, 0, "early");
 	trace_suspend_resume(TPS("dpm_resume_early"), state.event, false);
 }
 
@@ -948,7 +955,7 @@ void dpm_resume(pm_message_t state)
 	}
 	mutex_unlock(&dpm_list_mtx);
 	async_synchronize_full();
-	dpm_show_time(starttime, state, NULL);
+	dpm_show_time(starttime, state, 0, NULL);
 
 	cpufreq_resume();
 	trace_suspend_resume(TPS("dpm_resume"), state.event, false);
@@ -1098,6 +1105,11 @@ static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool a
 	if (async_error)
 		goto Complete;
 
+	if (pm_wakeup_pending()) {
+		async_error = -EBUSY;
+		goto Complete;
+	}
+
 	if (dev->power.syscore || dev->power.direct_complete)
 		goto Complete;
 
@@ -1158,22 +1170,19 @@ static int device_suspend_noirq(struct device *dev)
 	return __device_suspend_noirq(dev, pm_transition, false);
 }
 
-/**
- * dpm_suspend_noirq - Execute "noirq suspend" callbacks for all devices.
- * @state: PM transition of the system being carried out.
- *
- * Prevent device drivers from receiving interrupts and call the "noirq" suspend
- * handlers for all non-sysdev devices.
- */
-int dpm_suspend_noirq(pm_message_t state)
+void dpm_noirq_begin(void)
+{
+	cpuidle_pause();
+	device_wakeup_arm_wake_irqs();
+	suspend_device_irqs();
+}
+
+int dpm_noirq_suspend_devices(pm_message_t state)
 {
 	ktime_t starttime = ktime_get();
 	int error = 0;
 
 	trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, true);
-	cpuidle_pause();
-	device_wakeup_arm_wake_irqs();
-	suspend_device_irqs();
 	mutex_lock(&dpm_list_mtx);
 	pm_transition = state;
 	async_error = 0;
@@ -1208,15 +1217,32 @@ int dpm_suspend_noirq(pm_message_t state)
 	if (error) {
 		suspend_stats.failed_suspend_noirq++;
 		dpm_save_failed_step(SUSPEND_SUSPEND_NOIRQ);
-		dpm_resume_noirq(resume_event(state));
-	} else {
-		dpm_show_time(starttime, state, "noirq");
 	}
+	dpm_show_time(starttime, state, error, "noirq");
 	trace_suspend_resume(TPS("dpm_suspend_noirq"), state.event, false);
 	return error;
 }
 
 /**
+ * dpm_suspend_noirq - Execute "noirq suspend" callbacks for all devices.
+ * @state: PM transition of the system being carried out.
+ *
+ * Prevent device drivers' interrupt handlers from being called and invoke
+ * "noirq" suspend callbacks for all non-sysdev devices.
+ */
+int dpm_suspend_noirq(pm_message_t state)
+{
+	int ret;
+
+	dpm_noirq_begin();
+	ret = dpm_noirq_suspend_devices(state);
+	if (ret)
+		dpm_resume_noirq(resume_event(state));
+
+	return ret;
+}
+
+/**
  * device_suspend_late - Execute a "late suspend" callback for given device.
  * @dev: Device to handle.
  * @state: PM transition of the system being carried out.
@@ -1350,9 +1376,8 @@ int dpm_suspend_late(pm_message_t state)
 		suspend_stats.failed_suspend_late++;
 		dpm_save_failed_step(SUSPEND_SUSPEND_LATE);
 		dpm_resume_early(resume_event(state));
-	} else {
-		dpm_show_time(starttime, state, "late");
 	}
+	dpm_show_time(starttime, state, error, "late");
 	trace_suspend_resume(TPS("dpm_suspend_late"), state.event, false);
 	return error;
 }
@@ -1618,8 +1643,8 @@ int dpm_suspend(pm_message_t state)
 	if (error) {
 		suspend_stats.failed_suspend++;
 		dpm_save_failed_step(SUSPEND_SUSPEND);
-	} else
-		dpm_show_time(starttime, state, NULL);
+	}
+	dpm_show_time(starttime, state, error, NULL);
 	trace_suspend_resume(TPS("dpm_suspend"), state.event, false);
 	return error;
 }
diff --git a/drivers/base/power/opp/of.c b/drivers/base/power/opp/of.c
index 57eec1ca0569..0b718886479b 100644
--- a/drivers/base/power/opp/of.c
+++ b/drivers/base/power/opp/of.c
@@ -248,15 +248,22 @@ void dev_pm_opp_of_remove_table(struct device *dev)
 }
 EXPORT_SYMBOL_GPL(dev_pm_opp_of_remove_table);
 
-/* Returns opp descriptor node for a device, caller must do of_node_put() */
-struct device_node *dev_pm_opp_of_get_opp_desc_node(struct device *dev)
+/* Returns opp descriptor node for a device node, caller must
+ * do of_node_put() */
+static struct device_node *_opp_of_get_opp_desc_node(struct device_node *np)
 {
 	/*
 	 * There should be only ONE phandle present in "operating-points-v2"
 	 * property.
 	 */
 
-	return of_parse_phandle(dev->of_node, "operating-points-v2", 0);
+	return of_parse_phandle(np, "operating-points-v2", 0);
+}
+
+/* Returns opp descriptor node for a device, caller must do of_node_put() */
+struct device_node *dev_pm_opp_of_get_opp_desc_node(struct device *dev)
+{
+	return _opp_of_get_opp_desc_node(dev->of_node);
 }
 EXPORT_SYMBOL_GPL(dev_pm_opp_of_get_opp_desc_node);
 
@@ -539,8 +546,12 @@ int dev_pm_opp_of_cpumask_add_table(const struct cpumask *cpumask)
 
 		ret = dev_pm_opp_of_add_table(cpu_dev);
 		if (ret) {
-			pr_err("%s: couldn't find opp table for cpu:%d, %d\n",
-			       __func__, cpu, ret);
+			/*
+			 * OPP may get registered dynamically, don't print error
+			 * message here.
+			 */
+			pr_debug("%s: couldn't find opp table for cpu:%d, %d\n",
+				 __func__, cpu, ret);
 
 			/* Free all other OPPs */
 			dev_pm_opp_of_cpumask_remove_table(cpumask);
@@ -572,8 +583,7 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_of_cpumask_add_table);
 int dev_pm_opp_of_get_sharing_cpus(struct device *cpu_dev,
 				   struct cpumask *cpumask)
 {
-	struct device_node *np, *tmp_np;
-	struct device *tcpu_dev;
+	struct device_node *np, *tmp_np, *cpu_np;
 	int cpu, ret = 0;
 
 	/* Get OPP descriptor node */
@@ -593,19 +603,18 @@ int dev_pm_opp_of_get_sharing_cpus(struct device *cpu_dev,
 		if (cpu == cpu_dev->id)
 			continue;
 
-		tcpu_dev = get_cpu_device(cpu);
-		if (!tcpu_dev) {
-			dev_err(cpu_dev, "%s: failed to get cpu%d device\n",
+		cpu_np = of_get_cpu_node(cpu, NULL);
+		if (!cpu_np) {
+			dev_err(cpu_dev, "%s: failed to get cpu%d node\n",
 				__func__, cpu);
-			ret = -ENODEV;
+			ret = -ENOENT;
 			goto put_cpu_node;
 		}
 
 		/* Get OPP descriptor node */
-		tmp_np = dev_pm_opp_of_get_opp_desc_node(tcpu_dev);
+		tmp_np = _opp_of_get_opp_desc_node(cpu_np);
 		if (!tmp_np) {
-			dev_err(tcpu_dev, "%s: Couldn't find opp node.\n",
-				__func__);
+			pr_err("%pOF: Couldn't find opp node\n", cpu_np);
 			ret = -ENOENT;
 			goto put_cpu_node;
 		}
diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c
index 144e6d8fafc8..cdd6f256da59 100644
--- a/drivers/base/power/wakeup.c
+++ b/drivers/base/power/wakeup.c
@@ -412,15 +412,17 @@ void device_set_wakeup_capable(struct device *dev, bool capable)
 	if (!!dev->power.can_wakeup == !!capable)
 		return;
 
+	dev->power.can_wakeup = capable;
 	if (device_is_registered(dev) && !list_empty(&dev->power.entry)) {
 		if (capable) {
-			if (wakeup_sysfs_add(dev))
-				return;
+			int ret = wakeup_sysfs_add(dev);
+
+			if (ret)
+				dev_info(dev, "Wakeup sysfs attributes not added\n");
 		} else {
 			wakeup_sysfs_remove(dev);
 		}
 	}
-	dev->power.can_wakeup = capable;
 }
 EXPORT_SYMBOL_GPL(device_set_wakeup_capable);
 
@@ -863,7 +865,7 @@ bool pm_wakeup_pending(void)
 void pm_system_wakeup(void)
 {
 	atomic_inc(&pm_abort_suspend);
-	freeze_wake();
+	s2idle_wake();
 }
 EXPORT_SYMBOL_GPL(pm_system_wakeup);
 
diff --git a/drivers/base/property.c b/drivers/base/property.c
index edf02c1b5845..d0b65bbe7e15 100644
--- a/drivers/base/property.c
+++ b/drivers/base/property.c
@@ -25,19 +25,25 @@ struct property_set {
 	const struct property_entry *properties;
 };
 
-static inline bool is_pset_node(struct fwnode_handle *fwnode)
-{
-	return !IS_ERR_OR_NULL(fwnode) && fwnode->type == FWNODE_PDATA;
-}
+static const struct fwnode_operations pset_fwnode_ops;
 
-static inline struct property_set *to_pset_node(struct fwnode_handle *fwnode)
+static inline bool is_pset_node(const struct fwnode_handle *fwnode)
 {
-	return is_pset_node(fwnode) ?
-		container_of(fwnode, struct property_set, fwnode) : NULL;
+	return !IS_ERR_OR_NULL(fwnode) && fwnode->ops == &pset_fwnode_ops;
 }
 
-static const struct property_entry *pset_prop_get(struct property_set *pset,
-						  const char *name)
+#define to_pset_node(__fwnode)						\
+	({								\
+		typeof(__fwnode) __to_pset_node_fwnode = __fwnode;	\
+									\
+		is_pset_node(__to_pset_node_fwnode) ?			\
+			container_of(__to_pset_node_fwnode,		\
+				     struct property_set, fwnode) :	\
+			NULL;						\
+	})
+
+static const struct property_entry *
+pset_prop_get(const struct property_set *pset, const char *name)
 {
 	const struct property_entry *prop;
 
@@ -51,7 +57,7 @@ static const struct property_entry *pset_prop_get(struct property_set *pset,
 	return NULL;
 }
 
-static const void *pset_prop_find(struct property_set *pset,
+static const void *pset_prop_find(const struct property_set *pset,
 				  const char *propname, size_t length)
 {
 	const struct property_entry *prop;
@@ -71,7 +77,7 @@ static const void *pset_prop_find(struct property_set *pset,
 	return pointer;
 }
 
-static int pset_prop_read_u8_array(struct property_set *pset,
+static int pset_prop_read_u8_array(const struct property_set *pset,
 				   const char *propname,
 				   u8 *values, size_t nval)
 {
@@ -86,7 +92,7 @@ static int pset_prop_read_u8_array(struct property_set *pset,
 	return 0;
 }
 
-static int pset_prop_read_u16_array(struct property_set *pset,
+static int pset_prop_read_u16_array(const struct property_set *pset,
 				    const char *propname,
 				    u16 *values, size_t nval)
 {
@@ -101,7 +107,7 @@ static int pset_prop_read_u16_array(struct property_set *pset,
 	return 0;
 }
 
-static int pset_prop_read_u32_array(struct property_set *pset,
+static int pset_prop_read_u32_array(const struct property_set *pset,
 				    const char *propname,
 				    u32 *values, size_t nval)
 {
@@ -116,7 +122,7 @@ static int pset_prop_read_u32_array(struct property_set *pset,
 	return 0;
 }
 
-static int pset_prop_read_u64_array(struct property_set *pset,
+static int pset_prop_read_u64_array(const struct property_set *pset,
 				    const char *propname,
 				    u64 *values, size_t nval)
 {
@@ -131,7 +137,7 @@ static int pset_prop_read_u64_array(struct property_set *pset,
 	return 0;
 }
 
-static int pset_prop_count_elems_of_size(struct property_set *pset,
+static int pset_prop_count_elems_of_size(const struct property_set *pset,
 					 const char *propname, size_t length)
 {
 	const struct property_entry *prop;
@@ -143,7 +149,7 @@ static int pset_prop_count_elems_of_size(struct property_set *pset,
 	return prop->length / length;
 }
 
-static int pset_prop_read_string_array(struct property_set *pset,
+static int pset_prop_read_string_array(const struct property_set *pset,
 				       const char *propname,
 				       const char **strings, size_t nval)
 {
@@ -187,18 +193,18 @@ struct fwnode_handle *dev_fwnode(struct device *dev)
 }
 EXPORT_SYMBOL_GPL(dev_fwnode);
 
-static bool pset_fwnode_property_present(struct fwnode_handle *fwnode,
+static bool pset_fwnode_property_present(const struct fwnode_handle *fwnode,
 					 const char *propname)
 {
 	return !!pset_prop_get(to_pset_node(fwnode), propname);
 }
 
-static int pset_fwnode_read_int_array(struct fwnode_handle *fwnode,
+static int pset_fwnode_read_int_array(const struct fwnode_handle *fwnode,
 				      const char *propname,
 				      unsigned int elem_size, void *val,
 				      size_t nval)
 {
-	struct property_set *node = to_pset_node(fwnode);
+	const struct property_set *node = to_pset_node(fwnode);
 
 	if (!val)
 		return pset_prop_count_elems_of_size(node, propname, elem_size);
@@ -217,9 +223,10 @@ static int pset_fwnode_read_int_array(struct fwnode_handle *fwnode,
 	return -ENXIO;
 }
 
-static int pset_fwnode_property_read_string_array(struct fwnode_handle *fwnode,
-						  const char *propname,
-						  const char **val, size_t nval)
+static int
+pset_fwnode_property_read_string_array(const struct fwnode_handle *fwnode,
+				       const char *propname,
+				       const char **val, size_t nval)
 {
 	return pset_prop_read_string_array(to_pset_node(fwnode), propname,
 					   val, nval);
@@ -249,7 +256,8 @@ EXPORT_SYMBOL_GPL(device_property_present);
  * @fwnode: Firmware node whose property to check
  * @propname: Name of the property
  */
-bool fwnode_property_present(struct fwnode_handle *fwnode, const char *propname)
+bool fwnode_property_present(const struct fwnode_handle *fwnode,
+			     const char *propname)
 {
 	bool ret;
 
@@ -431,7 +439,7 @@ int device_property_match_string(struct device *dev, const char *propname,
 }
 EXPORT_SYMBOL_GPL(device_property_match_string);
 
-static int fwnode_property_read_int_array(struct fwnode_handle *fwnode,
+static int fwnode_property_read_int_array(const struct fwnode_handle *fwnode,
 					  const char *propname,
 					  unsigned int elem_size, void *val,
 					  size_t nval)
@@ -467,7 +475,7 @@ static int fwnode_property_read_int_array(struct fwnode_handle *fwnode,
  *	   %-EOVERFLOW if the size of the property is not as expected,
  *	   %-ENXIO if no suitable firmware interface is present.
  */
-int fwnode_property_read_u8_array(struct fwnode_handle *fwnode,
+int fwnode_property_read_u8_array(const struct fwnode_handle *fwnode,
 				  const char *propname, u8 *val, size_t nval)
 {
 	return fwnode_property_read_int_array(fwnode, propname, sizeof(u8),
@@ -493,7 +501,7 @@ EXPORT_SYMBOL_GPL(fwnode_property_read_u8_array);
  *	   %-EOVERFLOW if the size of the property is not as expected,
  *	   %-ENXIO if no suitable firmware interface is present.
  */
-int fwnode_property_read_u16_array(struct fwnode_handle *fwnode,
+int fwnode_property_read_u16_array(const struct fwnode_handle *fwnode,
 				   const char *propname, u16 *val, size_t nval)
 {
 	return fwnode_property_read_int_array(fwnode, propname, sizeof(u16),
@@ -519,7 +527,7 @@ EXPORT_SYMBOL_GPL(fwnode_property_read_u16_array);
  *	   %-EOVERFLOW if the size of the property is not as expected,
  *	   %-ENXIO if no suitable firmware interface is present.
  */
-int fwnode_property_read_u32_array(struct fwnode_handle *fwnode,
+int fwnode_property_read_u32_array(const struct fwnode_handle *fwnode,
 				   const char *propname, u32 *val, size_t nval)
 {
 	return fwnode_property_read_int_array(fwnode, propname, sizeof(u32),
@@ -545,7 +553,7 @@ EXPORT_SYMBOL_GPL(fwnode_property_read_u32_array);
  *	   %-EOVERFLOW if the size of the property is not as expected,
  *	   %-ENXIO if no suitable firmware interface is present.
  */
-int fwnode_property_read_u64_array(struct fwnode_handle *fwnode,
+int fwnode_property_read_u64_array(const struct fwnode_handle *fwnode,
 				   const char *propname, u64 *val, size_t nval)
 {
 	return fwnode_property_read_int_array(fwnode, propname, sizeof(u64),
@@ -571,7 +579,7 @@ EXPORT_SYMBOL_GPL(fwnode_property_read_u64_array);
  *	   %-EOVERFLOW if the size of the property is not as expected,
  *	   %-ENXIO if no suitable firmware interface is present.
  */
-int fwnode_property_read_string_array(struct fwnode_handle *fwnode,
+int fwnode_property_read_string_array(const struct fwnode_handle *fwnode,
 				      const char *propname, const char **val,
 				      size_t nval)
 {
@@ -603,7 +611,7 @@ EXPORT_SYMBOL_GPL(fwnode_property_read_string_array);
  *	   %-EPROTO or %-EILSEQ if the property is not a string,
  *	   %-ENXIO if no suitable firmware interface is present.
  */
-int fwnode_property_read_string(struct fwnode_handle *fwnode,
+int fwnode_property_read_string(const struct fwnode_handle *fwnode,
 				const char *propname, const char **val)
 {
 	int ret = fwnode_property_read_string_array(fwnode, propname, val, 1);
@@ -627,7 +635,7 @@ EXPORT_SYMBOL_GPL(fwnode_property_read_string);
  *	   %-EPROTO if the property is not an array of strings,
  *	   %-ENXIO if no suitable firmware interface is present.
  */
-int fwnode_property_match_string(struct fwnode_handle *fwnode,
+int fwnode_property_match_string(const struct fwnode_handle *fwnode,
 	const char *propname, const char *string)
 {
 	const char **values;
@@ -657,6 +665,34 @@ out:
 }
 EXPORT_SYMBOL_GPL(fwnode_property_match_string);
 
+/**
+ * fwnode_property_get_reference_args() - Find a reference with arguments
+ * @fwnode:	Firmware node where to look for the reference
+ * @prop:	The name of the property
+ * @nargs_prop:	The name of the property telling the number of
+ *		arguments in the referred node. NULL if @nargs is known,
+ *		otherwise @nargs is ignored. Only relevant on OF.
+ * @nargs:	Number of arguments. Ignored if @nargs_prop is non-NULL.
+ * @index:	Index of the reference, from zero onwards.
+ * @args:	Result structure with reference and integer arguments.
+ *
+ * Obtain a reference based on a named property in an fwnode, with
+ * integer arguments.
+ *
+ * Caller is responsible to call fwnode_handle_put() on the returned
+ * args->fwnode pointer.
+ *
+ */
+int fwnode_property_get_reference_args(const struct fwnode_handle *fwnode,
+				       const char *prop, const char *nargs_prop,
+				       unsigned int nargs, unsigned int index,
+				       struct fwnode_reference_args *args)
+{
+	return fwnode_call_int_op(fwnode, get_reference_args, prop, nargs_prop,
+				  nargs, index, args);
+}
+EXPORT_SYMBOL_GPL(fwnode_property_get_reference_args);
+
 static int property_copy_string_array(struct property_entry *dst,
 				      const struct property_entry *src)
 {
@@ -900,7 +936,6 @@ int device_add_properties(struct device *dev,
 	if (IS_ERR(p))
 		return PTR_ERR(p);
 
-	p->fwnode.type = FWNODE_PDATA;
 	p->fwnode.ops = &pset_fwnode_ops;
 	set_secondary_fwnode(dev, &p->fwnode);
 	return 0;
@@ -935,7 +970,7 @@ EXPORT_SYMBOL_GPL(fwnode_get_next_parent);
  * Return parent firmware node of the given node if possible or %NULL if no
  * parent was available.
  */
-struct fwnode_handle *fwnode_get_parent(struct fwnode_handle *fwnode)
+struct fwnode_handle *fwnode_get_parent(const struct fwnode_handle *fwnode)
 {
 	return fwnode_call_ptr_op(fwnode, get_parent);
 }
@@ -946,8 +981,9 @@ EXPORT_SYMBOL_GPL(fwnode_get_parent);
  * @fwnode: Firmware node to find the next child node for.
  * @child: Handle to one of the node's child nodes or a %NULL handle.
  */
-struct fwnode_handle *fwnode_get_next_child_node(struct fwnode_handle *fwnode,
-						 struct fwnode_handle *child)
+struct fwnode_handle *
+fwnode_get_next_child_node(const struct fwnode_handle *fwnode,
+			   struct fwnode_handle *child)
 {
 	return fwnode_call_ptr_op(fwnode, get_next_child_node, child);
 }
@@ -978,8 +1014,9 @@ EXPORT_SYMBOL_GPL(device_get_next_child_node);
  * @fwnode: Firmware node to find the named child node for.
  * @childname: String to match child node name against.
  */
-struct fwnode_handle *fwnode_get_named_child_node(struct fwnode_handle *fwnode,
-						  const char *childname)
+struct fwnode_handle *
+fwnode_get_named_child_node(const struct fwnode_handle *fwnode,
+			    const char *childname)
 {
 	return fwnode_call_ptr_op(fwnode, get_named_child_node, childname);
 }
@@ -1025,7 +1062,7 @@ EXPORT_SYMBOL_GPL(fwnode_handle_put);
  * fwnode_device_is_available - check if a device is available for use
  * @fwnode: Pointer to the fwnode of the device.
  */
-bool fwnode_device_is_available(struct fwnode_handle *fwnode)
+bool fwnode_device_is_available(const struct fwnode_handle *fwnode)
 {
 	return fwnode_call_bool_op(fwnode, device_is_available);
 }
@@ -1163,7 +1200,7 @@ EXPORT_SYMBOL(device_get_mac_address);
  * are available.
  */
 struct fwnode_handle *
-fwnode_graph_get_next_endpoint(struct fwnode_handle *fwnode,
+fwnode_graph_get_next_endpoint(const struct fwnode_handle *fwnode,
 			       struct fwnode_handle *prev)
 {
 	return fwnode_call_ptr_op(fwnode, graph_get_next_endpoint, prev);
@@ -1177,7 +1214,7 @@ EXPORT_SYMBOL_GPL(fwnode_graph_get_next_endpoint);
  * Return: the firmware node of the device the @endpoint belongs to.
  */
 struct fwnode_handle *
-fwnode_graph_get_port_parent(struct fwnode_handle *endpoint)
+fwnode_graph_get_port_parent(const struct fwnode_handle *endpoint)
 {
 	struct fwnode_handle *port, *parent;
 
@@ -1197,7 +1234,7 @@ EXPORT_SYMBOL_GPL(fwnode_graph_get_port_parent);
  * Extracts firmware node of a remote device the @fwnode points to.
  */
 struct fwnode_handle *
-fwnode_graph_get_remote_port_parent(struct fwnode_handle *fwnode)
+fwnode_graph_get_remote_port_parent(const struct fwnode_handle *fwnode)
 {
 	struct fwnode_handle *endpoint, *parent;
 
@@ -1216,7 +1253,8 @@ EXPORT_SYMBOL_GPL(fwnode_graph_get_remote_port_parent);
  *
  * Extracts firmware node of a remote port the @fwnode points to.
  */
-struct fwnode_handle *fwnode_graph_get_remote_port(struct fwnode_handle *fwnode)
+struct fwnode_handle *
+fwnode_graph_get_remote_port(const struct fwnode_handle *fwnode)
 {
 	return fwnode_get_next_parent(fwnode_graph_get_remote_endpoint(fwnode));
 }
@@ -1229,7 +1267,7 @@ EXPORT_SYMBOL_GPL(fwnode_graph_get_remote_port);
  * Extracts firmware node of a remote endpoint the @fwnode points to.
  */
 struct fwnode_handle *
-fwnode_graph_get_remote_endpoint(struct fwnode_handle *fwnode)
+fwnode_graph_get_remote_endpoint(const struct fwnode_handle *fwnode)
 {
 	return fwnode_call_ptr_op(fwnode, graph_get_remote_endpoint);
 }
@@ -1244,8 +1282,9 @@ EXPORT_SYMBOL_GPL(fwnode_graph_get_remote_endpoint);
  * Return: Remote fwnode handle associated with remote endpoint node linked
  *	   to @node. Use fwnode_node_put() on it when done.
  */
-struct fwnode_handle *fwnode_graph_get_remote_node(struct fwnode_handle *fwnode,
-						   u32 port_id, u32 endpoint_id)
+struct fwnode_handle *
+fwnode_graph_get_remote_node(const struct fwnode_handle *fwnode, u32 port_id,
+			     u32 endpoint_id)
 {
 	struct fwnode_handle *endpoint = NULL;
 
@@ -1281,7 +1320,7 @@ EXPORT_SYMBOL_GPL(fwnode_graph_get_remote_node);
  * information in @endpoint. The caller must hold a reference to
  * @fwnode.
  */
-int fwnode_graph_parse_endpoint(struct fwnode_handle *fwnode,
+int fwnode_graph_parse_endpoint(const struct fwnode_handle *fwnode,
 				struct fwnode_endpoint *endpoint)
 {
 	memset(endpoint, 0, sizeof(*endpoint));
diff --git a/drivers/base/topology.c b/drivers/base/topology.c
index d6ec1c546f5b..d936fcf9f1fb 100644
--- a/drivers/base/topology.c
+++ b/drivers/base/topology.c
@@ -105,7 +105,7 @@ static struct attribute *default_attrs[] = {
 	NULL
 };
 
-static struct attribute_group topology_attr_group = {
+static const struct attribute_group topology_attr_group = {
 	.attrs = default_attrs,
 	.name = "topology"
 };
diff --git a/drivers/bcma/Kconfig b/drivers/bcma/Kconfig
index b5c48a8d485f..54f81c554815 100644
--- a/drivers/bcma/Kconfig
+++ b/drivers/bcma/Kconfig
@@ -3,11 +3,8 @@ config BCMA_POSSIBLE
 	depends on HAS_IOMEM && HAS_DMA
 	default y
 
-menu "Broadcom specific AMBA"
-	depends on BCMA_POSSIBLE
-
-config BCMA
-	tristate "BCMA support"
+menuconfig BCMA
+	tristate "Broadcom specific AMBA"
 	depends on BCMA_POSSIBLE
 	help
 	  Bus driver for Broadcom specific Advanced Microcontroller Bus
@@ -117,5 +114,3 @@ config BCMA_DEBUG
 	  This turns on additional debugging messages.
 
 	  If unsure, say N
-
-endmenu
diff --git a/drivers/bcma/driver_gpio.c b/drivers/bcma/driver_gpio.c
index 7bde8d7a2816..982d5781d3ce 100644
--- a/drivers/bcma/driver_gpio.c
+++ b/drivers/bcma/driver_gpio.c
@@ -191,6 +191,7 @@ int bcma_gpio_init(struct bcma_drv_cc *cc)
 	case BCMA_CHIP_ID_BCM4707:
 	case BCMA_CHIP_ID_BCM5357:
 	case BCMA_CHIP_ID_BCM53572:
+	case BCMA_CHIP_ID_BCM53573:
 	case BCMA_CHIP_ID_BCM47094:
 		chip->ngpio	= 32;
 		break;
diff --git a/drivers/block/DAC960.c b/drivers/block/DAC960.c
index 245a879b036e..255591ab3716 100644
--- a/drivers/block/DAC960.c
+++ b/drivers/block/DAC960.c
@@ -1678,9 +1678,12 @@ static bool DAC960_V1_ReadControllerConfiguration(DAC960_Controller_T
       Enquiry2->FirmwareID.FirmwareType = '0';
       Enquiry2->FirmwareID.TurnID = 0;
     }
-  sprintf(Controller->FirmwareVersion, "%d.%02d-%c-%02d",
-	  Enquiry2->FirmwareID.MajorVersion, Enquiry2->FirmwareID.MinorVersion,
-	  Enquiry2->FirmwareID.FirmwareType, Enquiry2->FirmwareID.TurnID);
+  snprintf(Controller->FirmwareVersion, sizeof(Controller->FirmwareVersion),
+	   "%d.%02d-%c-%02d",
+	   Enquiry2->FirmwareID.MajorVersion,
+	   Enquiry2->FirmwareID.MinorVersion,
+	   Enquiry2->FirmwareID.FirmwareType,
+	   Enquiry2->FirmwareID.TurnID);
   if (!((Controller->FirmwareVersion[0] == '5' &&
 	 strcmp(Controller->FirmwareVersion, "5.06") >= 0) ||
 	(Controller->FirmwareVersion[0] == '4' &&
@@ -6588,7 +6591,8 @@ static void DAC960_CreateProcEntries(DAC960_Controller_T *Controller)
 			    &dac960_proc_fops);
 	}
 
-	sprintf(Controller->ControllerName, "c%d", Controller->ControllerNumber);
+	snprintf(Controller->ControllerName, sizeof(Controller->ControllerName),
+		 "c%d", Controller->ControllerNumber);
 	ControllerProcEntry = proc_mkdir(Controller->ControllerName,
 					 DAC960_ProcDirectoryEntry);
 	proc_create_data("initial_status", 0, ControllerProcEntry, &dac960_initial_status_proc_fops, Controller);
diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig
index 8ddc98279c8f..4a438b8abe27 100644
--- a/drivers/block/Kconfig
+++ b/drivers/block/Kconfig
@@ -17,6 +17,7 @@ if BLK_DEV
 
 config BLK_DEV_NULL_BLK
 	tristate "Null test block driver"
+	depends on CONFIGFS_FS
 
 config BLK_DEV_FD
 	tristate "Normal floppy disk support"
@@ -111,33 +112,6 @@ source "drivers/block/mtip32xx/Kconfig"
 
 source "drivers/block/zram/Kconfig"
 
-config BLK_CPQ_CISS_DA
-	tristate "Compaq Smart Array 5xxx support"
-	depends on PCI
-	select CHECK_SIGNATURE
-	select BLK_SCSI_REQUEST
-	help
-	  This is the driver for Compaq Smart Array 5xxx controllers.
-	  Everyone using these boards should say Y here.
-	  See <file:Documentation/blockdev/cciss.txt> for the current list of
-	  boards supported by this driver, and for further information
-	  on the use of this driver.
-
-config CISS_SCSI_TAPE
-	bool "SCSI tape drive support for Smart Array 5xxx"
-	depends on BLK_CPQ_CISS_DA && PROC_FS
-	depends on SCSI=y || SCSI=BLK_CPQ_CISS_DA
-	help
-	  When enabled (Y), this option allows SCSI tape drives and SCSI medium
-	  changers (tape robots) to be accessed via a Compaq 5xxx array 
-	  controller.  (See <file:Documentation/blockdev/cciss.txt> for more details.)
-
-	  "SCSI support" and "SCSI tape support" must also be enabled for this 
-	  option to work.
-
-	  When this option is disabled (N), the SCSI portion of the driver 
-	  is not compiled.
-
 config BLK_DEV_DAC960
 	tristate "Mylex DAC960/DAC1100 PCI RAID Controller support"
 	depends on PCI
@@ -470,7 +444,7 @@ config VIRTIO_BLK
 	depends on VIRTIO
 	---help---
 	  This is the virtual block driver for virtio.  It can be used with
-          lguest or QEMU based VMMs (like KVM or Xen).  Say Y or M.
+          QEMU based VMMs (like KVM or Xen).  Say Y or M.
 
 config VIRTIO_BLK_SCSI
 	bool "SCSI passthrough request for the Virtio block driver"
diff --git a/drivers/block/Makefile b/drivers/block/Makefile
index ec8c36897b75..1f456d86a190 100644
--- a/drivers/block/Makefile
+++ b/drivers/block/Makefile
@@ -15,7 +15,6 @@ obj-$(CONFIG_ATARI_FLOPPY)	+= ataflop.o
 obj-$(CONFIG_AMIGA_Z2RAM)	+= z2ram.o
 obj-$(CONFIG_BLK_DEV_RAM)	+= brd.o
 obj-$(CONFIG_BLK_DEV_LOOP)	+= loop.o
-obj-$(CONFIG_BLK_CPQ_CISS_DA)  += cciss.o
 obj-$(CONFIG_BLK_DEV_DAC960)	+= DAC960.o
 obj-$(CONFIG_XILINX_SYSACE)	+= xsysace.o
 obj-$(CONFIG_CDROM_PKTCDVD)	+= pktcdvd.o
diff --git a/drivers/block/brd.c b/drivers/block/brd.c
index 104b71c0490d..bbd0d186cfc0 100644
--- a/drivers/block/brd.c
+++ b/drivers/block/brd.c
@@ -294,14 +294,13 @@ out:
 
 static blk_qc_t brd_make_request(struct request_queue *q, struct bio *bio)
 {
-	struct block_device *bdev = bio->bi_bdev;
-	struct brd_device *brd = bdev->bd_disk->private_data;
+	struct brd_device *brd = bio->bi_disk->private_data;
 	struct bio_vec bvec;
 	sector_t sector;
 	struct bvec_iter iter;
 
 	sector = bio->bi_iter.bi_sector;
-	if (bio_end_sector(bio) > get_capacity(bdev->bd_disk))
+	if (bio_end_sector(bio) > get_capacity(bio->bi_disk))
 		goto io_error;
 
 	bio_for_each_segment(bvec, bio, iter) {
@@ -326,7 +325,11 @@ static int brd_rw_page(struct block_device *bdev, sector_t sector,
 		       struct page *page, bool is_write)
 {
 	struct brd_device *brd = bdev->bd_disk->private_data;
-	int err = brd_do_bvec(brd, page, PAGE_SIZE, 0, is_write, sector);
+	int err;
+
+	if (PageTransHuge(page))
+		return -ENOTSUPP;
+	err = brd_do_bvec(brd, page, PAGE_SIZE, 0, is_write, sector);
 	page_endio(page, is_write, err);
 	return err;
 }
diff --git a/drivers/block/cciss.c b/drivers/block/cciss.c
deleted file mode 100644
index 678af946be30..000000000000
--- a/drivers/block/cciss.c
+++ /dev/null
@@ -1,5415 +0,0 @@
-/*
- *    Disk Array driver for HP Smart Array controllers.
- *    (C) Copyright 2000, 2007 Hewlett-Packard Development Company, L.P.
- *
- *    This program is free software; you can redistribute it and/or modify
- *    it under the terms of the GNU General Public License as published by
- *    the Free Software Foundation; version 2 of the License.
- *
- *    This program is distributed in the hope that it will be useful,
- *    but WITHOUT ANY WARRANTY; without even the implied warranty of
- *    MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- *    General Public License for more details.
- *
- *    You should have received a copy of the GNU General Public License
- *    along with this program; if not, write to the Free Software
- *    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
- *    02111-1307, USA.
- *
- *    Questions/Comments/Bugfixes to iss_storagedev@hp.com
- *
- */
-
-#include <linux/module.h>
-#include <linux/interrupt.h>
-#include <linux/types.h>
-#include <linux/pci.h>
-#include <linux/pci-aspm.h>
-#include <linux/kernel.h>
-#include <linux/slab.h>
-#include <linux/delay.h>
-#include <linux/major.h>
-#include <linux/fs.h>
-#include <linux/bio.h>
-#include <linux/blkpg.h>
-#include <linux/timer.h>
-#include <linux/proc_fs.h>
-#include <linux/seq_file.h>
-#include <linux/init.h>
-#include <linux/jiffies.h>
-#include <linux/hdreg.h>
-#include <linux/spinlock.h>
-#include <linux/compat.h>
-#include <linux/mutex.h>
-#include <linux/bitmap.h>
-#include <linux/io.h>
-#include <linux/uaccess.h>
-
-#include <linux/dma-mapping.h>
-#include <linux/blkdev.h>
-#include <linux/genhd.h>
-#include <linux/completion.h>
-#include <scsi/scsi.h>
-#include <scsi/sg.h>
-#include <scsi/scsi_ioctl.h>
-#include <scsi/scsi_request.h>
-#include <linux/cdrom.h>
-#include <linux/scatterlist.h>
-#include <linux/kthread.h>
-
-#define CCISS_DRIVER_VERSION(maj,min,submin) ((maj<<16)|(min<<8)|(submin))
-#define DRIVER_NAME "HP CISS Driver (v 3.6.26)"
-#define DRIVER_VERSION CCISS_DRIVER_VERSION(3, 6, 26)
-
-/* Embedded module documentation macros - see modules.h */
-MODULE_AUTHOR("Hewlett-Packard Company");
-MODULE_DESCRIPTION("Driver for HP Smart Array Controllers");
-MODULE_SUPPORTED_DEVICE("HP Smart Array Controllers");
-MODULE_VERSION("3.6.26");
-MODULE_LICENSE("GPL");
-static int cciss_tape_cmds = 6;
-module_param(cciss_tape_cmds, int, 0644);
-MODULE_PARM_DESC(cciss_tape_cmds,
-	"number of commands to allocate for tape devices (default: 6)");
-static int cciss_simple_mode;
-module_param(cciss_simple_mode, int, S_IRUGO|S_IWUSR);
-MODULE_PARM_DESC(cciss_simple_mode,
-	"Use 'simple mode' rather than 'performant mode'");
-
-static int cciss_allow_hpsa;
-module_param(cciss_allow_hpsa, int, S_IRUGO|S_IWUSR);
-MODULE_PARM_DESC(cciss_allow_hpsa,
-	"Prevent cciss driver from accessing hardware known to be "
-	" supported by the hpsa driver");
-
-static DEFINE_MUTEX(cciss_mutex);
-static struct proc_dir_entry *proc_cciss;
-
-#include "cciss_cmd.h"
-#include "cciss.h"
-#include <linux/cciss_ioctl.h>
-
-/* define the PCI info for the cards we can control */
-static const struct pci_device_id cciss_pci_device_id[] = {
-	{PCI_VENDOR_ID_COMPAQ, PCI_DEVICE_ID_COMPAQ_CISS,  0x0E11, 0x4070},
-	{PCI_VENDOR_ID_COMPAQ, PCI_DEVICE_ID_COMPAQ_CISSB, 0x0E11, 0x4080},
-	{PCI_VENDOR_ID_COMPAQ, PCI_DEVICE_ID_COMPAQ_CISSB, 0x0E11, 0x4082},
-	{PCI_VENDOR_ID_COMPAQ, PCI_DEVICE_ID_COMPAQ_CISSB, 0x0E11, 0x4083},
-	{PCI_VENDOR_ID_COMPAQ, PCI_DEVICE_ID_COMPAQ_CISSC, 0x0E11, 0x4091},
-	{PCI_VENDOR_ID_COMPAQ, PCI_DEVICE_ID_COMPAQ_CISSC, 0x0E11, 0x409A},
-	{PCI_VENDOR_ID_COMPAQ, PCI_DEVICE_ID_COMPAQ_CISSC, 0x0E11, 0x409B},
-	{PCI_VENDOR_ID_COMPAQ, PCI_DEVICE_ID_COMPAQ_CISSC, 0x0E11, 0x409C},
-	{PCI_VENDOR_ID_COMPAQ, PCI_DEVICE_ID_COMPAQ_CISSC, 0x0E11, 0x409D},
-	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSA,     0x103C, 0x3225},
-	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSC,     0x103C, 0x3223},
-	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSC,     0x103C, 0x3234},
-	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSC,     0x103C, 0x3235},
-	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSD,     0x103C, 0x3211},
-	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSD,     0x103C, 0x3212},
-	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSD,     0x103C, 0x3213},
-	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSD,     0x103C, 0x3214},
-	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSD,     0x103C, 0x3215},
-	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSC,     0x103C, 0x3237},
-	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSC,     0x103C, 0x323D},
-	{0,}
-};
-
-MODULE_DEVICE_TABLE(pci, cciss_pci_device_id);
-
-/*  board_id = Subsystem Device ID & Vendor ID
- *  product = Marketing Name for the board
- *  access = Address of the struct of function pointers
- */
-static struct board_type products[] = {
-	{0x40700E11, "Smart Array 5300", &SA5_access},
-	{0x40800E11, "Smart Array 5i", &SA5B_access},
-	{0x40820E11, "Smart Array 532", &SA5B_access},
-	{0x40830E11, "Smart Array 5312", &SA5B_access},
-	{0x409A0E11, "Smart Array 641", &SA5_access},
-	{0x409B0E11, "Smart Array 642", &SA5_access},
-	{0x409C0E11, "Smart Array 6400", &SA5_access},
-	{0x409D0E11, "Smart Array 6400 EM", &SA5_access},
-	{0x40910E11, "Smart Array 6i", &SA5_access},
-	{0x3225103C, "Smart Array P600", &SA5_access},
-	{0x3223103C, "Smart Array P800", &SA5_access},
-	{0x3234103C, "Smart Array P400", &SA5_access},
-	{0x3235103C, "Smart Array P400i", &SA5_access},
-	{0x3211103C, "Smart Array E200i", &SA5_access},
-	{0x3212103C, "Smart Array E200", &SA5_access},
-	{0x3213103C, "Smart Array E200i", &SA5_access},
-	{0x3214103C, "Smart Array E200i", &SA5_access},
-	{0x3215103C, "Smart Array E200i", &SA5_access},
-	{0x3237103C, "Smart Array E500", &SA5_access},
-	{0x323D103C, "Smart Array P700m", &SA5_access},
-};
-
-/* How long to wait (in milliseconds) for board to go into simple mode */
-#define MAX_CONFIG_WAIT 30000
-#define MAX_IOCTL_CONFIG_WAIT 1000
-
-/*define how many times we will try a command because of bus resets */
-#define MAX_CMD_RETRIES 3
-
-#define MAX_CTLR	32
-
-/* Originally cciss driver only supports 8 major numbers */
-#define MAX_CTLR_ORIG 	8
-
-static ctlr_info_t *hba[MAX_CTLR];
-
-static struct task_struct *cciss_scan_thread;
-static DEFINE_MUTEX(scan_mutex);
-static LIST_HEAD(scan_q);
-
-static void do_cciss_request(struct request_queue *q);
-static irqreturn_t do_cciss_intx(int irq, void *dev_id);
-static irqreturn_t do_cciss_msix_intr(int irq, void *dev_id);
-static int cciss_open(struct block_device *bdev, fmode_t mode);
-static int cciss_unlocked_open(struct block_device *bdev, fmode_t mode);
-static void cciss_release(struct gendisk *disk, fmode_t mode);
-static int cciss_ioctl(struct block_device *bdev, fmode_t mode,
-		       unsigned int cmd, unsigned long arg);
-static int cciss_getgeo(struct block_device *bdev, struct hd_geometry *geo);
-
-static int cciss_revalidate(struct gendisk *disk);
-static int rebuild_lun_table(ctlr_info_t *h, int first_time, int via_ioctl);
-static int deregister_disk(ctlr_info_t *h, int drv_index,
-			   int clear_all, int via_ioctl);
-
-static void cciss_read_capacity(ctlr_info_t *h, int logvol,
-			sector_t *total_size, unsigned int *block_size);
-static void cciss_read_capacity_16(ctlr_info_t *h, int logvol,
-			sector_t *total_size, unsigned int *block_size);
-static void cciss_geometry_inquiry(ctlr_info_t *h, int logvol,
-			sector_t total_size,
-			unsigned int block_size, InquiryData_struct *inq_buff,
-				   drive_info_struct *drv);
-static void cciss_interrupt_mode(ctlr_info_t *);
-static int cciss_enter_simple_mode(struct ctlr_info *h);
-static void start_io(ctlr_info_t *h);
-static int sendcmd_withirq(ctlr_info_t *h, __u8 cmd, void *buff, size_t size,
-			__u8 page_code, unsigned char scsi3addr[],
-			int cmd_type);
-static int sendcmd_withirq_core(ctlr_info_t *h, CommandList_struct *c,
-	int attempt_retry);
-static int process_sendcmd_error(ctlr_info_t *h, CommandList_struct *c);
-
-static int add_to_scan_list(struct ctlr_info *h);
-static int scan_thread(void *data);
-static int check_for_unit_attention(ctlr_info_t *h, CommandList_struct *c);
-static void cciss_hba_release(struct device *dev);
-static void cciss_device_release(struct device *dev);
-static void cciss_free_gendisk(ctlr_info_t *h, int drv_index);
-static void cciss_free_drive_info(ctlr_info_t *h, int drv_index);
-static inline u32 next_command(ctlr_info_t *h);
-static int cciss_find_cfg_addrs(struct pci_dev *pdev, void __iomem *vaddr,
-				u32 *cfg_base_addr, u64 *cfg_base_addr_index,
-				u64 *cfg_offset);
-static int cciss_pci_find_memory_BAR(struct pci_dev *pdev,
-				     unsigned long *memory_bar);
-static inline u32 cciss_tag_discard_error_bits(ctlr_info_t *h, u32 tag);
-static int write_driver_ver_to_cfgtable(CfgTable_struct __iomem *cfgtable);
-
-/* performant mode helper functions */
-static void  calc_bucket_map(int *bucket, int num_buckets, int nsgs,
-				int *bucket_map);
-static void cciss_put_controller_into_performant_mode(ctlr_info_t *h);
-
-#ifdef CONFIG_PROC_FS
-static void cciss_procinit(ctlr_info_t *h);
-#else
-static void cciss_procinit(ctlr_info_t *h)
-{
-}
-#endif				/* CONFIG_PROC_FS */
-
-#ifdef CONFIG_COMPAT
-static int cciss_compat_ioctl(struct block_device *, fmode_t,
-			      unsigned, unsigned long);
-#endif
-
-static const struct block_device_operations cciss_fops = {
-	.owner = THIS_MODULE,
-	.open = cciss_unlocked_open,
-	.release = cciss_release,
-	.ioctl = cciss_ioctl,
-	.getgeo = cciss_getgeo,
-#ifdef CONFIG_COMPAT
-	.compat_ioctl = cciss_compat_ioctl,
-#endif
-	.revalidate_disk = cciss_revalidate,
-};
-
-/* set_performant_mode: Modify the tag for cciss performant
- * set bit 0 for pull model, bits 3-1 for block fetch
- * register number
- */
-static void set_performant_mode(ctlr_info_t *h, CommandList_struct *c)
-{
-	if (likely(h->transMethod & CFGTBL_Trans_Performant))
-		c->busaddr |= 1 | (h->blockFetchTable[c->Header.SGList] << 1);
-}
-
-/*
- * Enqueuing and dequeuing functions for cmdlists.
- */
-static inline void addQ(struct list_head *list, CommandList_struct *c)
-{
-	list_add_tail(&c->list, list);
-}
-
-static inline void removeQ(CommandList_struct *c)
-{
-	/*
-	 * After kexec/dump some commands might still
-	 * be in flight, which the firmware will try
-	 * to complete. Resetting the firmware doesn't work
-	 * with old fw revisions, so we have to mark
-	 * them off as 'stale' to prevent the driver from
-	 * falling over.
-	 */
-	if (WARN_ON(list_empty(&c->list))) {
-		c->cmd_type = CMD_MSG_STALE;
-		return;
-	}
-
-	list_del_init(&c->list);
-}
-
-static void enqueue_cmd_and_start_io(ctlr_info_t *h,
-	CommandList_struct *c)
-{
-	unsigned long flags;
-	set_performant_mode(h, c);
-	spin_lock_irqsave(&h->lock, flags);
-	addQ(&h->reqQ, c);
-	h->Qdepth++;
-	if (h->Qdepth > h->maxQsinceinit)
-		h->maxQsinceinit = h->Qdepth;
-	start_io(h);
-	spin_unlock_irqrestore(&h->lock, flags);
-}
-
-static void cciss_free_sg_chain_blocks(SGDescriptor_struct **cmd_sg_list,
-	int nr_cmds)
-{
-	int i;
-
-	if (!cmd_sg_list)
-		return;
-	for (i = 0; i < nr_cmds; i++) {
-		kfree(cmd_sg_list[i]);
-		cmd_sg_list[i] = NULL;
-	}
-	kfree(cmd_sg_list);
-}
-
-static SGDescriptor_struct **cciss_allocate_sg_chain_blocks(
-	ctlr_info_t *h, int chainsize, int nr_cmds)
-{
-	int j;
-	SGDescriptor_struct **cmd_sg_list;
-
-	if (chainsize <= 0)
-		return NULL;
-
-	cmd_sg_list = kmalloc(sizeof(*cmd_sg_list) * nr_cmds, GFP_KERNEL);
-	if (!cmd_sg_list)
-		return NULL;
-
-	/* Build up chain blocks for each command */
-	for (j = 0; j < nr_cmds; j++) {
-		/* Need a block of chainsized s/g elements. */
-		cmd_sg_list[j] = kmalloc((chainsize *
-			sizeof(*cmd_sg_list[j])), GFP_KERNEL);
-		if (!cmd_sg_list[j]) {
-			dev_err(&h->pdev->dev, "Cannot get memory "
-				"for s/g chains.\n");
-			goto clean;
-		}
-	}
-	return cmd_sg_list;
-clean:
-	cciss_free_sg_chain_blocks(cmd_sg_list, nr_cmds);
-	return NULL;
-}
-
-static void cciss_unmap_sg_chain_block(ctlr_info_t *h, CommandList_struct *c)
-{
-	SGDescriptor_struct *chain_sg;
-	u64bit temp64;
-
-	if (c->Header.SGTotal <= h->max_cmd_sgentries)
-		return;
-
-	chain_sg = &c->SG[h->max_cmd_sgentries - 1];
-	temp64.val32.lower = chain_sg->Addr.lower;
-	temp64.val32.upper = chain_sg->Addr.upper;
-	pci_unmap_single(h->pdev, temp64.val, chain_sg->Len, PCI_DMA_TODEVICE);
-}
-
-static int cciss_map_sg_chain_block(ctlr_info_t *h, CommandList_struct *c,
-	SGDescriptor_struct *chain_block, int len)
-{
-	SGDescriptor_struct *chain_sg;
-	u64bit temp64;
-
-	chain_sg = &c->SG[h->max_cmd_sgentries - 1];
-	chain_sg->Ext = CCISS_SG_CHAIN;
-	chain_sg->Len = len;
-	temp64.val = pci_map_single(h->pdev, chain_block, len,
-				PCI_DMA_TODEVICE);
-	if (dma_mapping_error(&h->pdev->dev, temp64.val)) {
-		dev_warn(&h->pdev->dev,
-			"%s: error mapping chain block for DMA\n",
-			__func__);
-		return -1;
-	}
-	chain_sg->Addr.lower = temp64.val32.lower;
-	chain_sg->Addr.upper = temp64.val32.upper;
-
-	return 0;
-}
-
-#include "cciss_scsi.c"		/* For SCSI tape support */
-
-static const char *raid_label[] = { "0", "4", "1(1+0)", "5", "5+1", "ADG",
-	"UNKNOWN"
-};
-#define RAID_UNKNOWN (ARRAY_SIZE(raid_label)-1)
-
-#ifdef CONFIG_PROC_FS
-
-/*
- * Report information about this controller.
- */
-#define ENG_GIG 1000000000
-#define ENG_GIG_FACTOR (ENG_GIG/512)
-#define ENGAGE_SCSI	"engage scsi"
-
-static void cciss_seq_show_header(struct seq_file *seq)
-{
-	ctlr_info_t *h = seq->private;
-
-	seq_printf(seq, "%s: HP %s Controller\n"
-		"Board ID: 0x%08lx\n"
-		"Firmware Version: %c%c%c%c\n"
-		"IRQ: %d\n"
-		"Logical drives: %d\n"
-		"Current Q depth: %d\n"
-		"Current # commands on controller: %d\n"
-		"Max Q depth since init: %d\n"
-		"Max # commands on controller since init: %d\n"
-		"Max SG entries since init: %d\n",
-		h->devname,
-		h->product_name,
-		(unsigned long)h->board_id,
-		h->firm_ver[0], h->firm_ver[1], h->firm_ver[2],
-		h->firm_ver[3], (unsigned int)h->intr[h->intr_mode],
-		h->num_luns,
-		h->Qdepth, h->commands_outstanding,
-		h->maxQsinceinit, h->max_outstanding, h->maxSG);
-
-#ifdef CONFIG_CISS_SCSI_TAPE
-	cciss_seq_tape_report(seq, h);
-#endif /* CONFIG_CISS_SCSI_TAPE */
-}
-
-static void *cciss_seq_start(struct seq_file *seq, loff_t *pos)
-{
-	ctlr_info_t *h = seq->private;
-	unsigned long flags;
-
-	/* prevent displaying bogus info during configuration
-	 * or deconfiguration of a logical volume
-	 */
-	spin_lock_irqsave(&h->lock, flags);
-	if (h->busy_configuring) {
-		spin_unlock_irqrestore(&h->lock, flags);
-		return ERR_PTR(-EBUSY);
-	}
-	h->busy_configuring = 1;
-	spin_unlock_irqrestore(&h->lock, flags);
-
-	if (*pos == 0)
-		cciss_seq_show_header(seq);
-
-	return pos;
-}
-
-static int cciss_seq_show(struct seq_file *seq, void *v)
-{
-	sector_t vol_sz, vol_sz_frac;
-	ctlr_info_t *h = seq->private;
-	unsigned ctlr = h->ctlr;
-	loff_t *pos = v;
-	drive_info_struct *drv = h->drv[*pos];
-
-	if (*pos > h->highest_lun)
-		return 0;
-
-	if (drv == NULL) /* it's possible for h->drv[] to have holes. */
-		return 0;
-
-	if (drv->heads == 0)
-		return 0;
-
-	vol_sz = drv->nr_blocks;
-	vol_sz_frac = sector_div(vol_sz, ENG_GIG_FACTOR);
-	vol_sz_frac *= 100;
-	sector_div(vol_sz_frac, ENG_GIG_FACTOR);
-
-	if (drv->raid_level < 0 || drv->raid_level > RAID_UNKNOWN)
-		drv->raid_level = RAID_UNKNOWN;
-	seq_printf(seq, "cciss/c%dd%d:"
-			"\t%4u.%02uGB\tRAID %s\n",
-			ctlr, (int) *pos, (int)vol_sz, (int)vol_sz_frac,
-			raid_label[drv->raid_level]);
-	return 0;
-}
-
-static void *cciss_seq_next(struct seq_file *seq, void *v, loff_t *pos)
-{
-	ctlr_info_t *h = seq->private;
-
-	if (*pos > h->highest_lun)
-		return NULL;
-	*pos += 1;
-
-	return pos;
-}
-
-static void cciss_seq_stop(struct seq_file *seq, void *v)
-{
-	ctlr_info_t *h = seq->private;
-
-	/* Only reset h->busy_configuring if we succeeded in setting
-	 * it during cciss_seq_start. */
-	if (v == ERR_PTR(-EBUSY))
-		return;
-
-	h->busy_configuring = 0;
-}
-
-static const struct seq_operations cciss_seq_ops = {
-	.start = cciss_seq_start,
-	.show  = cciss_seq_show,
-	.next  = cciss_seq_next,
-	.stop  = cciss_seq_stop,
-};
-
-static int cciss_seq_open(struct inode *inode, struct file *file)
-{
-	int ret = seq_open(file, &cciss_seq_ops);
-	struct seq_file *seq = file->private_data;
-
-	if (!ret)
-		seq->private = PDE_DATA(inode);
-
-	return ret;
-}
-
-static ssize_t
-cciss_proc_write(struct file *file, const char __user *buf,
-		 size_t length, loff_t *ppos)
-{
-	int err;
-	char *buffer;
-
-#ifndef CONFIG_CISS_SCSI_TAPE
-	return -EINVAL;
-#endif
-
-	if (!buf || length > PAGE_SIZE - 1)
-		return -EINVAL;
-
-	buffer = memdup_user_nul(buf, length);
-	if (IS_ERR(buffer))
-		return PTR_ERR(buffer);
-
-#ifdef CONFIG_CISS_SCSI_TAPE
-	if (strncmp(ENGAGE_SCSI, buffer, sizeof ENGAGE_SCSI - 1) == 0) {
-		struct seq_file *seq = file->private_data;
-		ctlr_info_t *h = seq->private;
-
-		err = cciss_engage_scsi(h);
-		if (err == 0)
-			err = length;
-	} else
-#endif /* CONFIG_CISS_SCSI_TAPE */
-		err = -EINVAL;
-	/* might be nice to have "disengage" too, but it's not
-	   safely possible. (only 1 module use count, lock issues.) */
-
-	kfree(buffer);
-	return err;
-}
-
-static const struct file_operations cciss_proc_fops = {
-	.owner	 = THIS_MODULE,
-	.open    = cciss_seq_open,
-	.read    = seq_read,
-	.llseek  = seq_lseek,
-	.release = seq_release,
-	.write	 = cciss_proc_write,
-};
-
-static void cciss_procinit(ctlr_info_t *h)
-{
-	struct proc_dir_entry *pde;
-
-	if (proc_cciss == NULL)
-		proc_cciss = proc_mkdir("driver/cciss", NULL);
-	if (!proc_cciss)
-		return;
-	pde = proc_create_data(h->devname, S_IWUSR | S_IRUSR | S_IRGRP |
-					S_IROTH, proc_cciss,
-					&cciss_proc_fops, h);
-}
-#endif				/* CONFIG_PROC_FS */
-
-#define MAX_PRODUCT_NAME_LEN 19
-
-#define to_hba(n) container_of(n, struct ctlr_info, dev)
-#define to_drv(n) container_of(n, drive_info_struct, dev)
-
-/* List of controllers which cannot be hard reset on kexec with reset_devices */
-static u32 unresettable_controller[] = {
-	0x3223103C, /* Smart Array P800 */
-	0x3234103C, /* Smart Array P400 */
-	0x3235103C, /* Smart Array P400i */
-	0x3211103C, /* Smart Array E200i */
-	0x3212103C, /* Smart Array E200 */
-	0x3213103C, /* Smart Array E200i */
-	0x3214103C, /* Smart Array E200i */
-	0x3215103C, /* Smart Array E200i */
-	0x3237103C, /* Smart Array E500 */
-	0x323D103C, /* Smart Array P700m */
-	0x40800E11, /* Smart Array 5i */
-	0x409C0E11, /* Smart Array 6400 */
-	0x409D0E11, /* Smart Array 6400 EM */
-	0x40700E11, /* Smart Array 5300 */
-	0x40820E11, /* Smart Array 532 */
-	0x40830E11, /* Smart Array 5312 */
-	0x409A0E11, /* Smart Array 641 */
-	0x409B0E11, /* Smart Array 642 */
-	0x40910E11, /* Smart Array 6i */
-};
-
-/* List of controllers which cannot even be soft reset */
-static u32 soft_unresettable_controller[] = {
-	0x40800E11, /* Smart Array 5i */
-	0x40700E11, /* Smart Array 5300 */
-	0x40820E11, /* Smart Array 532 */
-	0x40830E11, /* Smart Array 5312 */
-	0x409A0E11, /* Smart Array 641 */
-	0x409B0E11, /* Smart Array 642 */
-	0x40910E11, /* Smart Array 6i */
-	/* Exclude 640x boards.  These are two pci devices in one slot
-	 * which share a battery backed cache module.  One controls the
-	 * cache, the other accesses the cache through the one that controls
-	 * it.  If we reset the one controlling the cache, the other will
-	 * likely not be happy.  Just forbid resetting this conjoined mess.
-	 */
-	0x409C0E11, /* Smart Array 6400 */
-	0x409D0E11, /* Smart Array 6400 EM */
-};
-
-static int ctlr_is_hard_resettable(u32 board_id)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(unresettable_controller); i++)
-		if (unresettable_controller[i] == board_id)
-			return 0;
-	return 1;
-}
-
-static int ctlr_is_soft_resettable(u32 board_id)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(soft_unresettable_controller); i++)
-		if (soft_unresettable_controller[i] == board_id)
-			return 0;
-	return 1;
-}
-
-static int ctlr_is_resettable(u32 board_id)
-{
-	return ctlr_is_hard_resettable(board_id) ||
-		ctlr_is_soft_resettable(board_id);
-}
-
-static ssize_t host_show_resettable(struct device *dev,
-				    struct device_attribute *attr,
-				    char *buf)
-{
-	struct ctlr_info *h = to_hba(dev);
-
-	return snprintf(buf, 20, "%d\n", ctlr_is_resettable(h->board_id));
-}
-static DEVICE_ATTR(resettable, S_IRUGO, host_show_resettable, NULL);
-
-static ssize_t host_store_rescan(struct device *dev,
-				 struct device_attribute *attr,
-				 const char *buf, size_t count)
-{
-	struct ctlr_info *h = to_hba(dev);
-
-	add_to_scan_list(h);
-	wake_up_process(cciss_scan_thread);
-	wait_for_completion_interruptible(&h->scan_wait);
-
-	return count;
-}
-static DEVICE_ATTR(rescan, S_IWUSR, NULL, host_store_rescan);
-
-static ssize_t host_show_transport_mode(struct device *dev,
-				 struct device_attribute *attr,
-				 char *buf)
-{
-	struct ctlr_info *h = to_hba(dev);
-
-	return snprintf(buf, 20, "%s\n",
-		h->transMethod & CFGTBL_Trans_Performant ?
-			"performant" : "simple");
-}
-static DEVICE_ATTR(transport_mode, S_IRUGO, host_show_transport_mode, NULL);
-
-static ssize_t dev_show_unique_id(struct device *dev,
-				 struct device_attribute *attr,
-				 char *buf)
-{
-	drive_info_struct *drv = to_drv(dev);
-	struct ctlr_info *h = to_hba(drv->dev.parent);
-	__u8 sn[16];
-	unsigned long flags;
-	int ret = 0;
-
-	spin_lock_irqsave(&h->lock, flags);
-	if (h->busy_configuring)
-		ret = -EBUSY;
-	else
-		memcpy(sn, drv->serial_no, sizeof(sn));
-	spin_unlock_irqrestore(&h->lock, flags);
-
-	if (ret)
-		return ret;
-	else
-		return snprintf(buf, 16 * 2 + 2,
-				"%02X%02X%02X%02X%02X%02X%02X%02X"
-				"%02X%02X%02X%02X%02X%02X%02X%02X\n",
-				sn[0], sn[1], sn[2], sn[3],
-				sn[4], sn[5], sn[6], sn[7],
-				sn[8], sn[9], sn[10], sn[11],
-				sn[12], sn[13], sn[14], sn[15]);
-}
-static DEVICE_ATTR(unique_id, S_IRUGO, dev_show_unique_id, NULL);
-
-static ssize_t dev_show_vendor(struct device *dev,
-			       struct device_attribute *attr,
-			       char *buf)
-{
-	drive_info_struct *drv = to_drv(dev);
-	struct ctlr_info *h = to_hba(drv->dev.parent);
-	char vendor[VENDOR_LEN + 1];
-	unsigned long flags;
-	int ret = 0;
-
-	spin_lock_irqsave(&h->lock, flags);
-	if (h->busy_configuring)
-		ret = -EBUSY;
-	else
-		memcpy(vendor, drv->vendor, VENDOR_LEN + 1);
-	spin_unlock_irqrestore(&h->lock, flags);
-
-	if (ret)
-		return ret;
-	else
-		return snprintf(buf, sizeof(vendor) + 1, "%s\n", drv->vendor);
-}
-static DEVICE_ATTR(vendor, S_IRUGO, dev_show_vendor, NULL);
-
-static ssize_t dev_show_model(struct device *dev,
-			      struct device_attribute *attr,
-			      char *buf)
-{
-	drive_info_struct *drv = to_drv(dev);
-	struct ctlr_info *h = to_hba(drv->dev.parent);
-	char model[MODEL_LEN + 1];
-	unsigned long flags;
-	int ret = 0;
-
-	spin_lock_irqsave(&h->lock, flags);
-	if (h->busy_configuring)
-		ret = -EBUSY;
-	else
-		memcpy(model, drv->model, MODEL_LEN + 1);
-	spin_unlock_irqrestore(&h->lock, flags);
-
-	if (ret)
-		return ret;
-	else
-		return snprintf(buf, sizeof(model) + 1, "%s\n", drv->model);
-}
-static DEVICE_ATTR(model, S_IRUGO, dev_show_model, NULL);
-
-static ssize_t dev_show_rev(struct device *dev,
-			    struct device_attribute *attr,
-			    char *buf)
-{
-	drive_info_struct *drv = to_drv(dev);
-	struct ctlr_info *h = to_hba(drv->dev.parent);
-	char rev[REV_LEN + 1];
-	unsigned long flags;
-	int ret = 0;
-
-	spin_lock_irqsave(&h->lock, flags);
-	if (h->busy_configuring)
-		ret = -EBUSY;
-	else
-		memcpy(rev, drv->rev, REV_LEN + 1);
-	spin_unlock_irqrestore(&h->lock, flags);
-
-	if (ret)
-		return ret;
-	else
-		return snprintf(buf, sizeof(rev) + 1, "%s\n", drv->rev);
-}
-static DEVICE_ATTR(rev, S_IRUGO, dev_show_rev, NULL);
-
-static ssize_t cciss_show_lunid(struct device *dev,
-				struct device_attribute *attr, char *buf)
-{
-	drive_info_struct *drv = to_drv(dev);
-	struct ctlr_info *h = to_hba(drv->dev.parent);
-	unsigned long flags;
-	unsigned char lunid[8];
-
-	spin_lock_irqsave(&h->lock, flags);
-	if (h->busy_configuring) {
-		spin_unlock_irqrestore(&h->lock, flags);
-		return -EBUSY;
-	}
-	if (!drv->heads) {
-		spin_unlock_irqrestore(&h->lock, flags);
-		return -ENOTTY;
-	}
-	memcpy(lunid, drv->LunID, sizeof(lunid));
-	spin_unlock_irqrestore(&h->lock, flags);
-	return snprintf(buf, 20, "0x%02x%02x%02x%02x%02x%02x%02x%02x\n",
-		lunid[0], lunid[1], lunid[2], lunid[3],
-		lunid[4], lunid[5], lunid[6], lunid[7]);
-}
-static DEVICE_ATTR(lunid, S_IRUGO, cciss_show_lunid, NULL);
-
-static ssize_t cciss_show_raid_level(struct device *dev,
-				     struct device_attribute *attr, char *buf)
-{
-	drive_info_struct *drv = to_drv(dev);
-	struct ctlr_info *h = to_hba(drv->dev.parent);
-	int raid;
-	unsigned long flags;
-
-	spin_lock_irqsave(&h->lock, flags);
-	if (h->busy_configuring) {
-		spin_unlock_irqrestore(&h->lock, flags);
-		return -EBUSY;
-	}
-	raid = drv->raid_level;
-	spin_unlock_irqrestore(&h->lock, flags);
-	if (raid < 0 || raid > RAID_UNKNOWN)
-		raid = RAID_UNKNOWN;
-
-	return snprintf(buf, strlen(raid_label[raid]) + 7, "RAID %s\n",
-			raid_label[raid]);
-}
-static DEVICE_ATTR(raid_level, S_IRUGO, cciss_show_raid_level, NULL);
-
-static ssize_t cciss_show_usage_count(struct device *dev,
-				      struct device_attribute *attr, char *buf)
-{
-	drive_info_struct *drv = to_drv(dev);
-	struct ctlr_info *h = to_hba(drv->dev.parent);
-	unsigned long flags;
-	int count;
-
-	spin_lock_irqsave(&h->lock, flags);
-	if (h->busy_configuring) {
-		spin_unlock_irqrestore(&h->lock, flags);
-		return -EBUSY;
-	}
-	count = drv->usage_count;
-	spin_unlock_irqrestore(&h->lock, flags);
-	return snprintf(buf, 20, "%d\n", count);
-}
-static DEVICE_ATTR(usage_count, S_IRUGO, cciss_show_usage_count, NULL);
-
-static struct attribute *cciss_host_attrs[] = {
-	&dev_attr_rescan.attr,
-	&dev_attr_resettable.attr,
-	&dev_attr_transport_mode.attr,
-	NULL
-};
-
-static struct attribute_group cciss_host_attr_group = {
-	.attrs = cciss_host_attrs,
-};
-
-static const struct attribute_group *cciss_host_attr_groups[] = {
-	&cciss_host_attr_group,
-	NULL
-};
-
-static struct device_type cciss_host_type = {
-	.name		= "cciss_host",
-	.groups		= cciss_host_attr_groups,
-	.release	= cciss_hba_release,
-};
-
-static struct attribute *cciss_dev_attrs[] = {
-	&dev_attr_unique_id.attr,
-	&dev_attr_model.attr,
-	&dev_attr_vendor.attr,
-	&dev_attr_rev.attr,
-	&dev_attr_lunid.attr,
-	&dev_attr_raid_level.attr,
-	&dev_attr_usage_count.attr,
-	NULL
-};
-
-static struct attribute_group cciss_dev_attr_group = {
-	.attrs = cciss_dev_attrs,
-};
-
-static const struct attribute_group *cciss_dev_attr_groups[] = {
-	&cciss_dev_attr_group,
-	NULL
-};
-
-static struct device_type cciss_dev_type = {
-	.name		= "cciss_device",
-	.groups		= cciss_dev_attr_groups,
-	.release	= cciss_device_release,
-};
-
-static struct bus_type cciss_bus_type = {
-	.name		= "cciss",
-};
-
-/*
- * cciss_hba_release is called when the reference count
- * of h->dev goes to zero.
- */
-static void cciss_hba_release(struct device *dev)
-{
-	/*
-	 * nothing to do, but need this to avoid a warning
-	 * about not having a release handler from lib/kref.c.
-	 */
-}
-
-/*
- * Initialize sysfs entry for each controller.  This sets up and registers
- * the 'cciss#' directory for each individual controller under
- * /sys/bus/pci/devices/<dev>/.
- */
-static int cciss_create_hba_sysfs_entry(struct ctlr_info *h)
-{
-	device_initialize(&h->dev);
-	h->dev.type = &cciss_host_type;
-	h->dev.bus = &cciss_bus_type;
-	dev_set_name(&h->dev, "%s", h->devname);
-	h->dev.parent = &h->pdev->dev;
-
-	return device_add(&h->dev);
-}
-
-/*
- * Remove sysfs entries for an hba.
- */
-static void cciss_destroy_hba_sysfs_entry(struct ctlr_info *h)
-{
-	device_del(&h->dev);
-	put_device(&h->dev); /* final put. */
-}
-
-/* cciss_device_release is called when the reference count
- * of h->drv[x]dev goes to zero.
- */
-static void cciss_device_release(struct device *dev)
-{
-	drive_info_struct *drv = to_drv(dev);
-	kfree(drv);
-}
-
-/*
- * Initialize sysfs for each logical drive.  This sets up and registers
- * the 'c#d#' directory for each individual logical drive under
- * /sys/bus/pci/devices/<dev/ccis#/. We also create a link from
- * /sys/block/cciss!c#d# to this entry.
- */
-static long cciss_create_ld_sysfs_entry(struct ctlr_info *h,
-				       int drv_index)
-{
-	struct device *dev;
-
-	if (h->drv[drv_index]->device_initialized)
-		return 0;
-
-	dev = &h->drv[drv_index]->dev;
-	device_initialize(dev);
-	dev->type = &cciss_dev_type;
-	dev->bus = &cciss_bus_type;
-	dev_set_name(dev, "c%dd%d", h->ctlr, drv_index);
-	dev->parent = &h->dev;
-	h->drv[drv_index]->device_initialized = 1;
-	return device_add(dev);
-}
-
-/*
- * Remove sysfs entries for a logical drive.
- */
-static void cciss_destroy_ld_sysfs_entry(struct ctlr_info *h, int drv_index,
-	int ctlr_exiting)
-{
-	struct device *dev = &h->drv[drv_index]->dev;
-
-	/* special case for c*d0, we only destroy it on controller exit */
-	if (drv_index == 0 && !ctlr_exiting)
-		return;
-
-	device_del(dev);
-	put_device(dev); /* the "final" put. */
-	h->drv[drv_index] = NULL;
-}
-
-/*
- * For operations that cannot sleep, a command block is allocated at init,
- * and managed by cmd_alloc() and cmd_free() using a simple bitmap to track
- * which ones are free or in use.
- */
-static CommandList_struct *cmd_alloc(ctlr_info_t *h)
-{
-	CommandList_struct *c;
-	int i;
-	u64bit temp64;
-	dma_addr_t cmd_dma_handle, err_dma_handle;
-
-	do {
-		i = find_first_zero_bit(h->cmd_pool_bits, h->nr_cmds);
-		if (i == h->nr_cmds)
-			return NULL;
-	} while (test_and_set_bit(i, h->cmd_pool_bits) != 0);
-	c = h->cmd_pool + i;
-	memset(c, 0, sizeof(CommandList_struct));
-	cmd_dma_handle = h->cmd_pool_dhandle + i * sizeof(CommandList_struct);
-	c->err_info = h->errinfo_pool + i;
-	memset(c->err_info, 0, sizeof(ErrorInfo_struct));
-	err_dma_handle = h->errinfo_pool_dhandle
-	    + i * sizeof(ErrorInfo_struct);
-	h->nr_allocs++;
-
-	c->cmdindex = i;
-
-	INIT_LIST_HEAD(&c->list);
-	c->busaddr = (__u32) cmd_dma_handle;
-	temp64.val = (__u64) err_dma_handle;
-	c->ErrDesc.Addr.lower = temp64.val32.lower;
-	c->ErrDesc.Addr.upper = temp64.val32.upper;
-	c->ErrDesc.Len = sizeof(ErrorInfo_struct);
-
-	c->ctlr = h->ctlr;
-	return c;
-}
-
-/* allocate a command using pci_alloc_consistent, used for ioctls,
- * etc., not for the main i/o path.
- */
-static CommandList_struct *cmd_special_alloc(ctlr_info_t *h)
-{
-	CommandList_struct *c;
-	u64bit temp64;
-	dma_addr_t cmd_dma_handle, err_dma_handle;
-
-	c = pci_zalloc_consistent(h->pdev, sizeof(CommandList_struct),
-				  &cmd_dma_handle);
-	if (c == NULL)
-		return NULL;
-
-	c->cmdindex = -1;
-
-	c->err_info = pci_zalloc_consistent(h->pdev, sizeof(ErrorInfo_struct),
-					    &err_dma_handle);
-
-	if (c->err_info == NULL) {
-		pci_free_consistent(h->pdev,
-			sizeof(CommandList_struct), c, cmd_dma_handle);
-		return NULL;
-	}
-
-	INIT_LIST_HEAD(&c->list);
-	c->busaddr = (__u32) cmd_dma_handle;
-	temp64.val = (__u64) err_dma_handle;
-	c->ErrDesc.Addr.lower = temp64.val32.lower;
-	c->ErrDesc.Addr.upper = temp64.val32.upper;
-	c->ErrDesc.Len = sizeof(ErrorInfo_struct);
-
-	c->ctlr = h->ctlr;
-	return c;
-}
-
-static void cmd_free(ctlr_info_t *h, CommandList_struct *c)
-{
-	int i;
-
-	i = c - h->cmd_pool;
-	clear_bit(i, h->cmd_pool_bits);
-	h->nr_frees++;
-}
-
-static void cmd_special_free(ctlr_info_t *h, CommandList_struct *c)
-{
-	u64bit temp64;
-
-	temp64.val32.lower = c->ErrDesc.Addr.lower;
-	temp64.val32.upper = c->ErrDesc.Addr.upper;
-	pci_free_consistent(h->pdev, sizeof(ErrorInfo_struct),
-			    c->err_info, (dma_addr_t) temp64.val);
-	pci_free_consistent(h->pdev, sizeof(CommandList_struct), c,
-		(dma_addr_t) cciss_tag_discard_error_bits(h, (u32) c->busaddr));
-}
-
-static inline ctlr_info_t *get_host(struct gendisk *disk)
-{
-	return disk->queue->queuedata;
-}
-
-static inline drive_info_struct *get_drv(struct gendisk *disk)
-{
-	return disk->private_data;
-}
-
-/*
- * Open.  Make sure the device is really there.
- */
-static int cciss_open(struct block_device *bdev, fmode_t mode)
-{
-	ctlr_info_t *h = get_host(bdev->bd_disk);
-	drive_info_struct *drv = get_drv(bdev->bd_disk);
-
-	dev_dbg(&h->pdev->dev, "cciss_open %s\n", bdev->bd_disk->disk_name);
-	if (drv->busy_configuring)
-		return -EBUSY;
-	/*
-	 * Root is allowed to open raw volume zero even if it's not configured
-	 * so array config can still work. Root is also allowed to open any
-	 * volume that has a LUN ID, so it can issue IOCTL to reread the
-	 * disk information.  I don't think I really like this
-	 * but I'm already using way to many device nodes to claim another one
-	 * for "raw controller".
-	 */
-	if (drv->heads == 0) {
-		if (MINOR(bdev->bd_dev) != 0) {	/* not node 0? */
-			/* if not node 0 make sure it is a partition = 0 */
-			if (MINOR(bdev->bd_dev) & 0x0f) {
-				return -ENXIO;
-				/* if it is, make sure we have a LUN ID */
-			} else if (memcmp(drv->LunID, CTLR_LUNID,
-				sizeof(drv->LunID))) {
-				return -ENXIO;
-			}
-		}
-		if (!capable(CAP_SYS_ADMIN))
-			return -EPERM;
-	}
-	drv->usage_count++;
-	h->usage_count++;
-	return 0;
-}
-
-static int cciss_unlocked_open(struct block_device *bdev, fmode_t mode)
-{
-	int ret;
-
-	mutex_lock(&cciss_mutex);
-	ret = cciss_open(bdev, mode);
-	mutex_unlock(&cciss_mutex);
-
-	return ret;
-}
-
-/*
- * Close.  Sync first.
- */
-static void cciss_release(struct gendisk *disk, fmode_t mode)
-{
-	ctlr_info_t *h;
-	drive_info_struct *drv;
-
-	mutex_lock(&cciss_mutex);
-	h = get_host(disk);
-	drv = get_drv(disk);
-	dev_dbg(&h->pdev->dev, "cciss_release %s\n", disk->disk_name);
-	drv->usage_count--;
-	h->usage_count--;
-	mutex_unlock(&cciss_mutex);
-}
-
-#ifdef CONFIG_COMPAT
-
-static int cciss_ioctl32_passthru(struct block_device *bdev, fmode_t mode,
-				  unsigned cmd, unsigned long arg);
-static int cciss_ioctl32_big_passthru(struct block_device *bdev, fmode_t mode,
-				      unsigned cmd, unsigned long arg);
-
-static int cciss_compat_ioctl(struct block_device *bdev, fmode_t mode,
-			      unsigned cmd, unsigned long arg)
-{
-	switch (cmd) {
-	case CCISS_GETPCIINFO:
-	case CCISS_GETINTINFO:
-	case CCISS_SETINTINFO:
-	case CCISS_GETNODENAME:
-	case CCISS_SETNODENAME:
-	case CCISS_GETHEARTBEAT:
-	case CCISS_GETBUSTYPES:
-	case CCISS_GETFIRMVER:
-	case CCISS_GETDRIVVER:
-	case CCISS_REVALIDVOLS:
-	case CCISS_DEREGDISK:
-	case CCISS_REGNEWDISK:
-	case CCISS_REGNEWD:
-	case CCISS_RESCANDISK:
-	case CCISS_GETLUNINFO:
-		return cciss_ioctl(bdev, mode, cmd, arg);
-
-	case CCISS_PASSTHRU32:
-		return cciss_ioctl32_passthru(bdev, mode, cmd, arg);
-	case CCISS_BIG_PASSTHRU32:
-		return cciss_ioctl32_big_passthru(bdev, mode, cmd, arg);
-
-	default:
-		return -ENOIOCTLCMD;
-	}
-}
-
-static int cciss_ioctl32_passthru(struct block_device *bdev, fmode_t mode,
-				  unsigned cmd, unsigned long arg)
-{
-	IOCTL32_Command_struct __user *arg32 =
-	    (IOCTL32_Command_struct __user *) arg;
-	IOCTL_Command_struct arg64;
-	IOCTL_Command_struct __user *p = compat_alloc_user_space(sizeof(arg64));
-	int err;
-	u32 cp;
-
-	memset(&arg64, 0, sizeof(arg64));
-	err = 0;
-	err |=
-	    copy_from_user(&arg64.LUN_info, &arg32->LUN_info,
-			   sizeof(arg64.LUN_info));
-	err |=
-	    copy_from_user(&arg64.Request, &arg32->Request,
-			   sizeof(arg64.Request));
-	err |=
-	    copy_from_user(&arg64.error_info, &arg32->error_info,
-			   sizeof(arg64.error_info));
-	err |= get_user(arg64.buf_size, &arg32->buf_size);
-	err |= get_user(cp, &arg32->buf);
-	arg64.buf = compat_ptr(cp);
-	err |= copy_to_user(p, &arg64, sizeof(arg64));
-
-	if (err)
-		return -EFAULT;
-
-	err = cciss_ioctl(bdev, mode, CCISS_PASSTHRU, (unsigned long)p);
-	if (err)
-		return err;
-	err |=
-	    copy_in_user(&arg32->error_info, &p->error_info,
-			 sizeof(arg32->error_info));
-	if (err)
-		return -EFAULT;
-	return err;
-}
-
-static int cciss_ioctl32_big_passthru(struct block_device *bdev, fmode_t mode,
-				      unsigned cmd, unsigned long arg)
-{
-	BIG_IOCTL32_Command_struct __user *arg32 =
-	    (BIG_IOCTL32_Command_struct __user *) arg;
-	BIG_IOCTL_Command_struct arg64;
-	BIG_IOCTL_Command_struct __user *p =
-	    compat_alloc_user_space(sizeof(arg64));
-	int err;
-	u32 cp;
-
-	memset(&arg64, 0, sizeof(arg64));
-	err = 0;
-	err |=
-	    copy_from_user(&arg64.LUN_info, &arg32->LUN_info,
-			   sizeof(arg64.LUN_info));
-	err |=
-	    copy_from_user(&arg64.Request, &arg32->Request,
-			   sizeof(arg64.Request));
-	err |=
-	    copy_from_user(&arg64.error_info, &arg32->error_info,
-			   sizeof(arg64.error_info));
-	err |= get_user(arg64.buf_size, &arg32->buf_size);
-	err |= get_user(arg64.malloc_size, &arg32->malloc_size);
-	err |= get_user(cp, &arg32->buf);
-	arg64.buf = compat_ptr(cp);
-	err |= copy_to_user(p, &arg64, sizeof(arg64));
-
-	if (err)
-		return -EFAULT;
-
-	err = cciss_ioctl(bdev, mode, CCISS_BIG_PASSTHRU, (unsigned long)p);
-	if (err)
-		return err;
-	err |=
-	    copy_in_user(&arg32->error_info, &p->error_info,
-			 sizeof(arg32->error_info));
-	if (err)
-		return -EFAULT;
-	return err;
-}
-#endif
-
-static int cciss_getgeo(struct block_device *bdev, struct hd_geometry *geo)
-{
-	drive_info_struct *drv = get_drv(bdev->bd_disk);
-
-	if (!drv->cylinders)
-		return -ENXIO;
-
-	geo->heads = drv->heads;
-	geo->sectors = drv->sectors;
-	geo->cylinders = drv->cylinders;
-	return 0;
-}
-
-static void check_ioctl_unit_attention(ctlr_info_t *h, CommandList_struct *c)
-{
-	if (c->err_info->CommandStatus == CMD_TARGET_STATUS &&
-			c->err_info->ScsiStatus != SAM_STAT_CHECK_CONDITION)
-		(void)check_for_unit_attention(h, c);
-}
-
-static int cciss_getpciinfo(ctlr_info_t *h, void __user *argp)
-{
-	cciss_pci_info_struct pciinfo;
-
-	if (!argp)
-		return -EINVAL;
-	pciinfo.domain = pci_domain_nr(h->pdev->bus);
-	pciinfo.bus = h->pdev->bus->number;
-	pciinfo.dev_fn = h->pdev->devfn;
-	pciinfo.board_id = h->board_id;
-	if (copy_to_user(argp, &pciinfo, sizeof(cciss_pci_info_struct)))
-		return -EFAULT;
-	return 0;
-}
-
-static int cciss_getintinfo(ctlr_info_t *h, void __user *argp)
-{
-	cciss_coalint_struct intinfo;
-	unsigned long flags;
-
-	if (!argp)
-		return -EINVAL;
-	spin_lock_irqsave(&h->lock, flags);
-	intinfo.delay = readl(&h->cfgtable->HostWrite.CoalIntDelay);
-	intinfo.count = readl(&h->cfgtable->HostWrite.CoalIntCount);
-	spin_unlock_irqrestore(&h->lock, flags);
-	if (copy_to_user
-	    (argp, &intinfo, sizeof(cciss_coalint_struct)))
-		return -EFAULT;
-	return 0;
-}
-
-static int cciss_setintinfo(ctlr_info_t *h, void __user *argp)
-{
-	cciss_coalint_struct intinfo;
-	unsigned long flags;
-	int i;
-
-	if (!argp)
-		return -EINVAL;
-	if (!capable(CAP_SYS_ADMIN))
-		return -EPERM;
-	if (copy_from_user(&intinfo, argp, sizeof(intinfo)))
-		return -EFAULT;
-	if ((intinfo.delay == 0) && (intinfo.count == 0))
-		return -EINVAL;
-	spin_lock_irqsave(&h->lock, flags);
-	/* Update the field, and then ring the doorbell */
-	writel(intinfo.delay, &(h->cfgtable->HostWrite.CoalIntDelay));
-	writel(intinfo.count, &(h->cfgtable->HostWrite.CoalIntCount));
-	writel(CFGTBL_ChangeReq, h->vaddr + SA5_DOORBELL);
-
-	for (i = 0; i < MAX_IOCTL_CONFIG_WAIT; i++) {
-		if (!(readl(h->vaddr + SA5_DOORBELL) & CFGTBL_ChangeReq))
-			break;
-		udelay(1000); /* delay and try again */
-	}
-	spin_unlock_irqrestore(&h->lock, flags);
-	if (i >= MAX_IOCTL_CONFIG_WAIT)
-		return -EAGAIN;
-	return 0;
-}
-
-static int cciss_getnodename(ctlr_info_t *h, void __user *argp)
-{
-	NodeName_type NodeName;
-	unsigned long flags;
-	int i;
-
-	if (!argp)
-		return -EINVAL;
-	spin_lock_irqsave(&h->lock, flags);
-	for (i = 0; i < 16; i++)
-		NodeName[i] = readb(&h->cfgtable->ServerName[i]);
-	spin_unlock_irqrestore(&h->lock, flags);
-	if (copy_to_user(argp, NodeName, sizeof(NodeName_type)))
-		return -EFAULT;
-	return 0;
-}
-
-static int cciss_setnodename(ctlr_info_t *h, void __user *argp)
-{
-	NodeName_type NodeName;
-	unsigned long flags;
-	int i;
-
-	if (!argp)
-		return -EINVAL;
-	if (!capable(CAP_SYS_ADMIN))
-		return -EPERM;
-	if (copy_from_user(NodeName, argp, sizeof(NodeName_type)))
-		return -EFAULT;
-	spin_lock_irqsave(&h->lock, flags);
-	/* Update the field, and then ring the doorbell */
-	for (i = 0; i < 16; i++)
-		writeb(NodeName[i], &h->cfgtable->ServerName[i]);
-	writel(CFGTBL_ChangeReq, h->vaddr + SA5_DOORBELL);
-	for (i = 0; i < MAX_IOCTL_CONFIG_WAIT; i++) {
-		if (!(readl(h->vaddr + SA5_DOORBELL) & CFGTBL_ChangeReq))
-			break;
-		udelay(1000); /* delay and try again */
-	}
-	spin_unlock_irqrestore(&h->lock, flags);
-	if (i >= MAX_IOCTL_CONFIG_WAIT)
-		return -EAGAIN;
-	return 0;
-}
-
-static int cciss_getheartbeat(ctlr_info_t *h, void __user *argp)
-{
-	Heartbeat_type heartbeat;
-	unsigned long flags;
-
-	if (!argp)
-		return -EINVAL;
-	spin_lock_irqsave(&h->lock, flags);
-	heartbeat = readl(&h->cfgtable->HeartBeat);
-	spin_unlock_irqrestore(&h->lock, flags);
-	if (copy_to_user(argp, &heartbeat, sizeof(Heartbeat_type)))
-		return -EFAULT;
-	return 0;
-}
-
-static int cciss_getbustypes(ctlr_info_t *h, void __user *argp)
-{
-	BusTypes_type BusTypes;
-	unsigned long flags;
-
-	if (!argp)
-		return -EINVAL;
-	spin_lock_irqsave(&h->lock, flags);
-	BusTypes = readl(&h->cfgtable->BusTypes);
-	spin_unlock_irqrestore(&h->lock, flags);
-	if (copy_to_user(argp, &BusTypes, sizeof(BusTypes_type)))
-		return -EFAULT;
-	return 0;
-}
-
-static int cciss_getfirmver(ctlr_info_t *h, void __user *argp)
-{
-	FirmwareVer_type firmware;
-
-	if (!argp)
-		return -EINVAL;
-	memcpy(firmware, h->firm_ver, 4);
-
-	if (copy_to_user
-	    (argp, firmware, sizeof(FirmwareVer_type)))
-		return -EFAULT;
-	return 0;
-}
-
-static int cciss_getdrivver(ctlr_info_t *h, void __user *argp)
-{
-	DriverVer_type DriverVer = DRIVER_VERSION;
-
-	if (!argp)
-		return -EINVAL;
-	if (copy_to_user(argp, &DriverVer, sizeof(DriverVer_type)))
-		return -EFAULT;
-	return 0;
-}
-
-static int cciss_getluninfo(ctlr_info_t *h,
-	struct gendisk *disk, void __user *argp)
-{
-	LogvolInfo_struct luninfo;
-	drive_info_struct *drv = get_drv(disk);
-
-	if (!argp)
-		return -EINVAL;
-	memcpy(&luninfo.LunID, drv->LunID, sizeof(luninfo.LunID));
-	luninfo.num_opens = drv->usage_count;
-	luninfo.num_parts = 0;
-	if (copy_to_user(argp, &luninfo, sizeof(LogvolInfo_struct)))
-		return -EFAULT;
-	return 0;
-}
-
-static int cciss_passthru(ctlr_info_t *h, void __user *argp)
-{
-	IOCTL_Command_struct iocommand;
-	CommandList_struct *c;
-	char *buff = NULL;
-	u64bit temp64;
-	DECLARE_COMPLETION_ONSTACK(wait);
-
-	if (!argp)
-		return -EINVAL;
-
-	if (!capable(CAP_SYS_RAWIO))
-		return -EPERM;
-
-	if (copy_from_user
-	    (&iocommand, argp, sizeof(IOCTL_Command_struct)))
-		return -EFAULT;
-	if ((iocommand.buf_size < 1) &&
-	    (iocommand.Request.Type.Direction != XFER_NONE)) {
-		return -EINVAL;
-	}
-	if (iocommand.buf_size > 0) {
-		buff = kmalloc(iocommand.buf_size, GFP_KERNEL);
-		if (buff == NULL)
-			return -EFAULT;
-	}
-	if (iocommand.Request.Type.Direction == XFER_WRITE) {
-		/* Copy the data into the buffer we created */
-		if (copy_from_user(buff, iocommand.buf, iocommand.buf_size)) {
-			kfree(buff);
-			return -EFAULT;
-		}
-	} else {
-		memset(buff, 0, iocommand.buf_size);
-	}
-	c = cmd_special_alloc(h);
-	if (!c) {
-		kfree(buff);
-		return -ENOMEM;
-	}
-	/* Fill in the command type */
-	c->cmd_type = CMD_IOCTL_PEND;
-	/* Fill in Command Header */
-	c->Header.ReplyQueue = 0;   /* unused in simple mode */
-	if (iocommand.buf_size > 0) { /* buffer to fill */
-		c->Header.SGList = 1;
-		c->Header.SGTotal = 1;
-	} else { /* no buffers to fill */
-		c->Header.SGList = 0;
-		c->Header.SGTotal = 0;
-	}
-	c->Header.LUN = iocommand.LUN_info;
-	/* use the kernel address the cmd block for tag */
-	c->Header.Tag.lower = c->busaddr;
-
-	/* Fill in Request block */
-	c->Request = iocommand.Request;
-
-	/* Fill in the scatter gather information */
-	if (iocommand.buf_size > 0) {
-		temp64.val = pci_map_single(h->pdev, buff,
-			iocommand.buf_size, PCI_DMA_BIDIRECTIONAL);
-		c->SG[0].Addr.lower = temp64.val32.lower;
-		c->SG[0].Addr.upper = temp64.val32.upper;
-		c->SG[0].Len = iocommand.buf_size;
-		c->SG[0].Ext = 0;  /* we are not chaining */
-	}
-	c->waiting = &wait;
-
-	enqueue_cmd_and_start_io(h, c);
-	wait_for_completion(&wait);
-
-	/* unlock the buffers from DMA */
-	temp64.val32.lower = c->SG[0].Addr.lower;
-	temp64.val32.upper = c->SG[0].Addr.upper;
-	pci_unmap_single(h->pdev, (dma_addr_t) temp64.val, iocommand.buf_size,
-			 PCI_DMA_BIDIRECTIONAL);
-	check_ioctl_unit_attention(h, c);
-
-	/* Copy the error information out */
-	iocommand.error_info = *(c->err_info);
-	if (copy_to_user(argp, &iocommand, sizeof(IOCTL_Command_struct))) {
-		kfree(buff);
-		cmd_special_free(h, c);
-		return -EFAULT;
-	}
-
-	if (iocommand.Request.Type.Direction == XFER_READ) {
-		/* Copy the data out of the buffer we created */
-		if (copy_to_user(iocommand.buf, buff, iocommand.buf_size)) {
-			kfree(buff);
-			cmd_special_free(h, c);
-			return -EFAULT;
-		}
-	}
-	kfree(buff);
-	cmd_special_free(h, c);
-	return 0;
-}
-
-static int cciss_bigpassthru(ctlr_info_t *h, void __user *argp)
-{
-	BIG_IOCTL_Command_struct *ioc;
-	CommandList_struct *c;
-	unsigned char **buff = NULL;
-	int *buff_size = NULL;
-	u64bit temp64;
-	BYTE sg_used = 0;
-	int status = 0;
-	int i;
-	DECLARE_COMPLETION_ONSTACK(wait);
-	__u32 left;
-	__u32 sz;
-	BYTE __user *data_ptr;
-
-	if (!argp)
-		return -EINVAL;
-	if (!capable(CAP_SYS_RAWIO))
-		return -EPERM;
-	ioc = kmalloc(sizeof(*ioc), GFP_KERNEL);
-	if (!ioc) {
-		status = -ENOMEM;
-		goto cleanup1;
-	}
-	if (copy_from_user(ioc, argp, sizeof(*ioc))) {
-		status = -EFAULT;
-		goto cleanup1;
-	}
-	if ((ioc->buf_size < 1) &&
-	    (ioc->Request.Type.Direction != XFER_NONE)) {
-		status = -EINVAL;
-		goto cleanup1;
-	}
-	/* Check kmalloc limits  using all SGs */
-	if (ioc->malloc_size > MAX_KMALLOC_SIZE) {
-		status = -EINVAL;
-		goto cleanup1;
-	}
-	if (ioc->buf_size > ioc->malloc_size * MAXSGENTRIES) {
-		status = -EINVAL;
-		goto cleanup1;
-	}
-	buff = kzalloc(MAXSGENTRIES * sizeof(char *), GFP_KERNEL);
-	if (!buff) {
-		status = -ENOMEM;
-		goto cleanup1;
-	}
-	buff_size = kmalloc(MAXSGENTRIES * sizeof(int), GFP_KERNEL);
-	if (!buff_size) {
-		status = -ENOMEM;
-		goto cleanup1;
-	}
-	left = ioc->buf_size;
-	data_ptr = ioc->buf;
-	while (left) {
-		sz = (left > ioc->malloc_size) ? ioc->malloc_size : left;
-		buff_size[sg_used] = sz;
-		buff[sg_used] = kmalloc(sz, GFP_KERNEL);
-		if (buff[sg_used] == NULL) {
-			status = -ENOMEM;
-			goto cleanup1;
-		}
-		if (ioc->Request.Type.Direction == XFER_WRITE) {
-			if (copy_from_user(buff[sg_used], data_ptr, sz)) {
-				status = -EFAULT;
-				goto cleanup1;
-			}
-		} else {
-			memset(buff[sg_used], 0, sz);
-		}
-		left -= sz;
-		data_ptr += sz;
-		sg_used++;
-	}
-	c = cmd_special_alloc(h);
-	if (!c) {
-		status = -ENOMEM;
-		goto cleanup1;
-	}
-	c->cmd_type = CMD_IOCTL_PEND;
-	c->Header.ReplyQueue = 0;
-	c->Header.SGList = sg_used;
-	c->Header.SGTotal = sg_used;
-	c->Header.LUN = ioc->LUN_info;
-	c->Header.Tag.lower = c->busaddr;
-
-	c->Request = ioc->Request;
-	for (i = 0; i < sg_used; i++) {
-		temp64.val = pci_map_single(h->pdev, buff[i], buff_size[i],
-				    PCI_DMA_BIDIRECTIONAL);
-		c->SG[i].Addr.lower = temp64.val32.lower;
-		c->SG[i].Addr.upper = temp64.val32.upper;
-		c->SG[i].Len = buff_size[i];
-		c->SG[i].Ext = 0;	/* we are not chaining */
-	}
-	c->waiting = &wait;
-	enqueue_cmd_and_start_io(h, c);
-	wait_for_completion(&wait);
-	/* unlock the buffers from DMA */
-	for (i = 0; i < sg_used; i++) {
-		temp64.val32.lower = c->SG[i].Addr.lower;
-		temp64.val32.upper = c->SG[i].Addr.upper;
-		pci_unmap_single(h->pdev,
-			(dma_addr_t) temp64.val, buff_size[i],
-			PCI_DMA_BIDIRECTIONAL);
-	}
-	check_ioctl_unit_attention(h, c);
-	/* Copy the error information out */
-	ioc->error_info = *(c->err_info);
-	if (copy_to_user(argp, ioc, sizeof(*ioc))) {
-		cmd_special_free(h, c);
-		status = -EFAULT;
-		goto cleanup1;
-	}
-	if (ioc->Request.Type.Direction == XFER_READ) {
-		/* Copy the data out of the buffer we created */
-		BYTE __user *ptr = ioc->buf;
-		for (i = 0; i < sg_used; i++) {
-			if (copy_to_user(ptr, buff[i], buff_size[i])) {
-				cmd_special_free(h, c);
-				status = -EFAULT;
-				goto cleanup1;
-			}
-			ptr += buff_size[i];
-		}
-	}
-	cmd_special_free(h, c);
-	status = 0;
-cleanup1:
-	if (buff) {
-		for (i = 0; i < sg_used; i++)
-			kfree(buff[i]);
-		kfree(buff);
-	}
-	kfree(buff_size);
-	kfree(ioc);
-	return status;
-}
-
-static int cciss_ioctl(struct block_device *bdev, fmode_t mode,
-	unsigned int cmd, unsigned long arg)
-{
-	struct gendisk *disk = bdev->bd_disk;
-	ctlr_info_t *h = get_host(disk);
-	void __user *argp = (void __user *)arg;
-
-	dev_dbg(&h->pdev->dev, "cciss_ioctl: Called with cmd=%x %lx\n",
-		cmd, arg);
-	switch (cmd) {
-	case CCISS_GETPCIINFO:
-		return cciss_getpciinfo(h, argp);
-	case CCISS_GETINTINFO:
-		return cciss_getintinfo(h, argp);
-	case CCISS_SETINTINFO:
-		return cciss_setintinfo(h, argp);
-	case CCISS_GETNODENAME:
-		return cciss_getnodename(h, argp);
-	case CCISS_SETNODENAME:
-		return cciss_setnodename(h, argp);
-	case CCISS_GETHEARTBEAT:
-		return cciss_getheartbeat(h, argp);
-	case CCISS_GETBUSTYPES:
-		return cciss_getbustypes(h, argp);
-	case CCISS_GETFIRMVER:
-		return cciss_getfirmver(h, argp);
-	case CCISS_GETDRIVVER:
-		return cciss_getdrivver(h, argp);
-	case CCISS_DEREGDISK:
-	case CCISS_REGNEWD:
-	case CCISS_REVALIDVOLS:
-		return rebuild_lun_table(h, 0, 1);
-	case CCISS_GETLUNINFO:
-		return cciss_getluninfo(h, disk, argp);
-	case CCISS_PASSTHRU:
-		return cciss_passthru(h, argp);
-	case CCISS_BIG_PASSTHRU:
-		return cciss_bigpassthru(h, argp);
-
-	/* scsi_cmd_blk_ioctl handles these, below, though some are not */
-	/* very meaningful for cciss.  SG_IO is the main one people want. */
-
-	case SG_GET_VERSION_NUM:
-	case SG_SET_TIMEOUT:
-	case SG_GET_TIMEOUT:
-	case SG_GET_RESERVED_SIZE:
-	case SG_SET_RESERVED_SIZE:
-	case SG_EMULATED_HOST:
-	case SG_IO:
-	case SCSI_IOCTL_SEND_COMMAND:
-		return scsi_cmd_blk_ioctl(bdev, mode, cmd, argp);
-
-	/* scsi_cmd_blk_ioctl would normally handle these, below, but */
-	/* they aren't a good fit for cciss, as CD-ROMs are */
-	/* not supported, and we don't have any bus/target/lun */
-	/* which we present to the kernel. */
-
-	case CDROM_SEND_PACKET:
-	case CDROMCLOSETRAY:
-	case CDROMEJECT:
-	case SCSI_IOCTL_GET_IDLUN:
-	case SCSI_IOCTL_GET_BUS_NUMBER:
-	default:
-		return -ENOTTY;
-	}
-}
-
-static void cciss_check_queues(ctlr_info_t *h)
-{
-	int start_queue = h->next_to_run;
-	int i;
-
-	/* check to see if we have maxed out the number of commands that can
-	 * be placed on the queue.  If so then exit.  We do this check here
-	 * in case the interrupt we serviced was from an ioctl and did not
-	 * free any new commands.
-	 */
-	if ((find_first_zero_bit(h->cmd_pool_bits, h->nr_cmds)) == h->nr_cmds)
-		return;
-
-	/* We have room on the queue for more commands.  Now we need to queue
-	 * them up.  We will also keep track of the next queue to run so
-	 * that every queue gets a chance to be started first.
-	 */
-	for (i = 0; i < h->highest_lun + 1; i++) {
-		int curr_queue = (start_queue + i) % (h->highest_lun + 1);
-		/* make sure the disk has been added and the drive is real
-		 * because this can be called from the middle of init_one.
-		 */
-		if (!h->drv[curr_queue])
-			continue;
-		if (!(h->drv[curr_queue]->queue) ||
-			!(h->drv[curr_queue]->heads))
-			continue;
-		blk_start_queue(h->gendisk[curr_queue]->queue);
-
-		/* check to see if we have maxed out the number of commands
-		 * that can be placed on the queue.
-		 */
-		if ((find_first_zero_bit(h->cmd_pool_bits, h->nr_cmds)) == h->nr_cmds) {
-			if (curr_queue == start_queue) {
-				h->next_to_run =
-				    (start_queue + 1) % (h->highest_lun + 1);
-				break;
-			} else {
-				h->next_to_run = curr_queue;
-				break;
-			}
-		}
-	}
-}
-
-static void cciss_softirq_done(struct request *rq)
-{
-	CommandList_struct *c = rq->completion_data;
-	ctlr_info_t *h = hba[c->ctlr];
-	SGDescriptor_struct *curr_sg = c->SG;
-	u64bit temp64;
-	unsigned long flags;
-	int i, ddir;
-	int sg_index = 0;
-
-	if (c->Request.Type.Direction == XFER_READ)
-		ddir = PCI_DMA_FROMDEVICE;
-	else
-		ddir = PCI_DMA_TODEVICE;
-
-	/* command did not need to be retried */
-	/* unmap the DMA mapping for all the scatter gather elements */
-	for (i = 0; i < c->Header.SGList; i++) {
-		if (curr_sg[sg_index].Ext == CCISS_SG_CHAIN) {
-			cciss_unmap_sg_chain_block(h, c);
-			/* Point to the next block */
-			curr_sg = h->cmd_sg_list[c->cmdindex];
-			sg_index = 0;
-		}
-		temp64.val32.lower = curr_sg[sg_index].Addr.lower;
-		temp64.val32.upper = curr_sg[sg_index].Addr.upper;
-		pci_unmap_page(h->pdev, temp64.val, curr_sg[sg_index].Len,
-				ddir);
-		++sg_index;
-	}
-
-	dev_dbg(&h->pdev->dev, "Done with %p\n", rq);
-
-	/* set the residual count for pc requests */
-	if (blk_rq_is_passthrough(rq))
-		scsi_req(rq)->resid_len = c->err_info->ResidualCnt;
-	blk_end_request_all(rq, scsi_req(rq)->result ?
-			BLK_STS_IOERR : BLK_STS_OK);
-
-	spin_lock_irqsave(&h->lock, flags);
-	cmd_free(h, c);
-	cciss_check_queues(h);
-	spin_unlock_irqrestore(&h->lock, flags);
-}
-
-static inline void log_unit_to_scsi3addr(ctlr_info_t *h,
-	unsigned char scsi3addr[], uint32_t log_unit)
-{
-	memcpy(scsi3addr, h->drv[log_unit]->LunID,
-		sizeof(h->drv[log_unit]->LunID));
-}
-
-/* This function gets the SCSI vendor, model, and revision of a logical drive
- * via the inquiry page 0.  Model, vendor, and rev are set to empty strings if
- * they cannot be read.
- */
-static void cciss_get_device_descr(ctlr_info_t *h, int logvol,
-				   char *vendor, char *model, char *rev)
-{
-	int rc;
-	InquiryData_struct *inq_buf;
-	unsigned char scsi3addr[8];
-
-	*vendor = '\0';
-	*model = '\0';
-	*rev = '\0';
-
-	inq_buf = kzalloc(sizeof(InquiryData_struct), GFP_KERNEL);
-	if (!inq_buf)
-		return;
-
-	log_unit_to_scsi3addr(h, scsi3addr, logvol);
-	rc = sendcmd_withirq(h, CISS_INQUIRY, inq_buf, sizeof(*inq_buf), 0,
-			scsi3addr, TYPE_CMD);
-	if (rc == IO_OK) {
-		memcpy(vendor, &inq_buf->data_byte[8], VENDOR_LEN);
-		vendor[VENDOR_LEN] = '\0';
-		memcpy(model, &inq_buf->data_byte[16], MODEL_LEN);
-		model[MODEL_LEN] = '\0';
-		memcpy(rev, &inq_buf->data_byte[32], REV_LEN);
-		rev[REV_LEN] = '\0';
-	}
-
-	kfree(inq_buf);
-	return;
-}
-
-/* This function gets the serial number of a logical drive via
- * inquiry page 0x83.  Serial no. is 16 bytes.  If the serial
- * number cannot be had, for whatever reason, 16 bytes of 0xff
- * are returned instead.
- */
-static void cciss_get_serial_no(ctlr_info_t *h, int logvol,
-				unsigned char *serial_no, int buflen)
-{
-#define PAGE_83_INQ_BYTES 64
-	int rc;
-	unsigned char *buf;
-	unsigned char scsi3addr[8];
-
-	if (buflen > 16)
-		buflen = 16;
-	memset(serial_no, 0xff, buflen);
-	buf = kzalloc(PAGE_83_INQ_BYTES, GFP_KERNEL);
-	if (!buf)
-		return;
-	memset(serial_no, 0, buflen);
-	log_unit_to_scsi3addr(h, scsi3addr, logvol);
-	rc = sendcmd_withirq(h, CISS_INQUIRY, buf,
-		PAGE_83_INQ_BYTES, 0x83, scsi3addr, TYPE_CMD);
-	if (rc == IO_OK)
-		memcpy(serial_no, &buf[8], buflen);
-	kfree(buf);
-	return;
-}
-
-static void cciss_initialize_rq(struct request *rq)
-{
-	struct scsi_request *sreq = blk_mq_rq_to_pdu(rq);
-
-	scsi_req_init(sreq);
-}
-
-/*
- * cciss_add_disk sets up the block device queue for a logical drive
- */
-static int cciss_add_disk(ctlr_info_t *h, struct gendisk *disk,
-				int drv_index)
-{
-	disk->queue = blk_alloc_queue(GFP_KERNEL);
-	if (!disk->queue)
-		goto init_queue_failure;
-
-	disk->queue->cmd_size = sizeof(struct scsi_request);
-	disk->queue->request_fn = do_cciss_request;
-	disk->queue->initialize_rq_fn = cciss_initialize_rq;
-	disk->queue->queue_lock = &h->lock;
-	queue_flag_set_unlocked(QUEUE_FLAG_SCSI_PASSTHROUGH, disk->queue);
-	if (blk_init_allocated_queue(disk->queue) < 0)
-		goto cleanup_queue;
-
-	sprintf(disk->disk_name, "cciss/c%dd%d", h->ctlr, drv_index);
-	disk->major = h->major;
-	disk->first_minor = drv_index << NWD_SHIFT;
-	disk->fops = &cciss_fops;
-	if (cciss_create_ld_sysfs_entry(h, drv_index))
-		goto cleanup_queue;
-	disk->private_data = h->drv[drv_index];
-
-	/* Set up queue information */
-	blk_queue_bounce_limit(disk->queue, h->pdev->dma_mask);
-
-	/* This is a hardware imposed limit. */
-	blk_queue_max_segments(disk->queue, h->maxsgentries);
-
-	blk_queue_max_hw_sectors(disk->queue, h->cciss_max_sectors);
-
-	blk_queue_softirq_done(disk->queue, cciss_softirq_done);
-
-	disk->queue->queuedata = h;
-
-	blk_queue_logical_block_size(disk->queue,
-				     h->drv[drv_index]->block_size);
-
-	/* Make sure all queue data is written out before */
-	/* setting h->drv[drv_index]->queue, as setting this */
-	/* allows the interrupt handler to start the queue */
-	wmb();
-	h->drv[drv_index]->queue = disk->queue;
-	device_add_disk(&h->drv[drv_index]->dev, disk);
-	return 0;
-
-cleanup_queue:
-	blk_cleanup_queue(disk->queue);
-	disk->queue = NULL;
-init_queue_failure:
-	return -1;
-}
-
-/* This function will check the usage_count of the drive to be updated/added.
- * If the usage_count is zero and it is a heretofore unknown drive, or,
- * the drive's capacity, geometry, or serial number has changed,
- * then the drive information will be updated and the disk will be
- * re-registered with the kernel.  If these conditions don't hold,
- * then it will be left alone for the next reboot.  The exception to this
- * is disk 0 which will always be left registered with the kernel since it
- * is also the controller node.  Any changes to disk 0 will show up on
- * the next reboot.
- */
-static void cciss_update_drive_info(ctlr_info_t *h, int drv_index,
-	int first_time, int via_ioctl)
-{
-	struct gendisk *disk;
-	InquiryData_struct *inq_buff = NULL;
-	unsigned int block_size;
-	sector_t total_size;
-	unsigned long flags = 0;
-	int ret = 0;
-	drive_info_struct *drvinfo;
-
-	/* Get information about the disk and modify the driver structure */
-	inq_buff = kmalloc(sizeof(InquiryData_struct), GFP_KERNEL);
-	drvinfo = kzalloc(sizeof(*drvinfo), GFP_KERNEL);
-	if (inq_buff == NULL || drvinfo == NULL)
-		goto mem_msg;
-
-	/* testing to see if 16-byte CDBs are already being used */
-	if (h->cciss_read == CCISS_READ_16) {
-		cciss_read_capacity_16(h, drv_index,
-			&total_size, &block_size);
-
-	} else {
-		cciss_read_capacity(h, drv_index, &total_size, &block_size);
-		/* if read_capacity returns all F's this volume is >2TB */
-		/* in size so we switch to 16-byte CDB's for all */
-		/* read/write ops */
-		if (total_size == 0xFFFFFFFFULL) {
-			cciss_read_capacity_16(h, drv_index,
-			&total_size, &block_size);
-			h->cciss_read = CCISS_READ_16;
-			h->cciss_write = CCISS_WRITE_16;
-		} else {
-			h->cciss_read = CCISS_READ_10;
-			h->cciss_write = CCISS_WRITE_10;
-		}
-	}
-
-	cciss_geometry_inquiry(h, drv_index, total_size, block_size,
-			       inq_buff, drvinfo);
-	drvinfo->block_size = block_size;
-	drvinfo->nr_blocks = total_size + 1;
-
-	cciss_get_device_descr(h, drv_index, drvinfo->vendor,
-				drvinfo->model, drvinfo->rev);
-	cciss_get_serial_no(h, drv_index, drvinfo->serial_no,
-			sizeof(drvinfo->serial_no));
-	/* Save the lunid in case we deregister the disk, below. */
-	memcpy(drvinfo->LunID, h->drv[drv_index]->LunID,
-		sizeof(drvinfo->LunID));
-
-	/* Is it the same disk we already know, and nothing's changed? */
-	if (h->drv[drv_index]->raid_level != -1 &&
-		((memcmp(drvinfo->serial_no,
-				h->drv[drv_index]->serial_no, 16) == 0) &&
-		drvinfo->block_size == h->drv[drv_index]->block_size &&
-		drvinfo->nr_blocks == h->drv[drv_index]->nr_blocks &&
-		drvinfo->heads == h->drv[drv_index]->heads &&
-		drvinfo->sectors == h->drv[drv_index]->sectors &&
-		drvinfo->cylinders == h->drv[drv_index]->cylinders))
-			/* The disk is unchanged, nothing to update */
-			goto freeret;
-
-	/* If we get here it's not the same disk, or something's changed,
-	 * so we need to * deregister it, and re-register it, if it's not
-	 * in use.
-	 * If the disk already exists then deregister it before proceeding
-	 * (unless it's the first disk (for the controller node).
-	 */
-	if (h->drv[drv_index]->raid_level != -1 && drv_index != 0) {
-		dev_warn(&h->pdev->dev, "disk %d has changed.\n", drv_index);
-		spin_lock_irqsave(&h->lock, flags);
-		h->drv[drv_index]->busy_configuring = 1;
-		spin_unlock_irqrestore(&h->lock, flags);
-
-		/* deregister_disk sets h->drv[drv_index]->queue = NULL
-		 * which keeps the interrupt handler from starting
-		 * the queue.
-		 */
-		ret = deregister_disk(h, drv_index, 0, via_ioctl);
-	}
-
-	/* If the disk is in use return */
-	if (ret)
-		goto freeret;
-
-	/* Save the new information from cciss_geometry_inquiry
-	 * and serial number inquiry.  If the disk was deregistered
-	 * above, then h->drv[drv_index] will be NULL.
-	 */
-	if (h->drv[drv_index] == NULL) {
-		drvinfo->device_initialized = 0;
-		h->drv[drv_index] = drvinfo;
-		drvinfo = NULL; /* so it won't be freed below. */
-	} else {
-		/* special case for cxd0 */
-		h->drv[drv_index]->block_size = drvinfo->block_size;
-		h->drv[drv_index]->nr_blocks = drvinfo->nr_blocks;
-		h->drv[drv_index]->heads = drvinfo->heads;
-		h->drv[drv_index]->sectors = drvinfo->sectors;
-		h->drv[drv_index]->cylinders = drvinfo->cylinders;
-		h->drv[drv_index]->raid_level = drvinfo->raid_level;
-		memcpy(h->drv[drv_index]->serial_no, drvinfo->serial_no, 16);
-		memcpy(h->drv[drv_index]->vendor, drvinfo->vendor,
-			VENDOR_LEN + 1);
-		memcpy(h->drv[drv_index]->model, drvinfo->model, MODEL_LEN + 1);
-		memcpy(h->drv[drv_index]->rev, drvinfo->rev, REV_LEN + 1);
-	}
-
-	++h->num_luns;
-	disk = h->gendisk[drv_index];
-	set_capacity(disk, h->drv[drv_index]->nr_blocks);
-
-	/* If it's not disk 0 (drv_index != 0)
-	 * or if it was disk 0, but there was previously
-	 * no actual corresponding configured logical drive
-	 * (raid_leve == -1) then we want to update the
-	 * logical drive's information.
-	 */
-	if (drv_index || first_time) {
-		if (cciss_add_disk(h, disk, drv_index) != 0) {
-			cciss_free_gendisk(h, drv_index);
-			cciss_free_drive_info(h, drv_index);
-			dev_warn(&h->pdev->dev, "could not update disk %d\n",
-				drv_index);
-			--h->num_luns;
-		}
-	}
-
-freeret:
-	kfree(inq_buff);
-	kfree(drvinfo);
-	return;
-mem_msg:
-	dev_err(&h->pdev->dev, "out of memory\n");
-	goto freeret;
-}
-
-/* This function will find the first index of the controllers drive array
- * that has a null drv pointer and allocate the drive info struct and
- * will return that index   This is where new drives will be added.
- * If the index to be returned is greater than the highest_lun index for
- * the controller then highest_lun is set * to this new index.
- * If there are no available indexes or if tha allocation fails, then -1
- * is returned.  * "controller_node" is used to know if this is a real
- * logical drive, or just the controller node, which determines if this
- * counts towards highest_lun.
- */
-static int cciss_alloc_drive_info(ctlr_info_t *h, int controller_node)
-{
-	int i;
-	drive_info_struct *drv;
-
-	/* Search for an empty slot for our drive info */
-	for (i = 0; i < CISS_MAX_LUN; i++) {
-
-		/* if not cxd0 case, and it's occupied, skip it. */
-		if (h->drv[i] && i != 0)
-			continue;
-		/*
-		 * If it's cxd0 case, and drv is alloc'ed already, and a
-		 * disk is configured there, skip it.
-		 */
-		if (i == 0 && h->drv[i] && h->drv[i]->raid_level != -1)
-			continue;
-
-		/*
-		 * We've found an empty slot.  Update highest_lun
-		 * provided this isn't just the fake cxd0 controller node.
-		 */
-		if (i > h->highest_lun && !controller_node)
-			h->highest_lun = i;
-
-		/* If adding a real disk at cxd0, and it's already alloc'ed */
-		if (i == 0 && h->drv[i] != NULL)
-			return i;
-
-		/*
-		 * Found an empty slot, not already alloc'ed.  Allocate it.
-		 * Mark it with raid_level == -1, so we know it's new later on.
-		 */
-		drv = kzalloc(sizeof(*drv), GFP_KERNEL);
-		if (!drv)
-			return -1;
-		drv->raid_level = -1; /* so we know it's new */
-		h->drv[i] = drv;
-		return i;
-	}
-	return -1;
-}
-
-static void cciss_free_drive_info(ctlr_info_t *h, int drv_index)
-{
-	kfree(h->drv[drv_index]);
-	h->drv[drv_index] = NULL;
-}
-
-static void cciss_free_gendisk(ctlr_info_t *h, int drv_index)
-{
-	put_disk(h->gendisk[drv_index]);
-	h->gendisk[drv_index] = NULL;
-}
-
-/* cciss_add_gendisk finds a free hba[]->drv structure
- * and allocates a gendisk if needed, and sets the lunid
- * in the drvinfo structure.   It returns the index into
- * the ->drv[] array, or -1 if none are free.
- * is_controller_node indicates whether highest_lun should
- * count this disk, or if it's only being added to provide
- * a means to talk to the controller in case no logical
- * drives have yet been configured.
- */
-static int cciss_add_gendisk(ctlr_info_t *h, unsigned char lunid[],
-	int controller_node)
-{
-	int drv_index;
-
-	drv_index = cciss_alloc_drive_info(h, controller_node);
-	if (drv_index == -1)
-		return -1;
-
-	/*Check if the gendisk needs to be allocated */
-	if (!h->gendisk[drv_index]) {
-		h->gendisk[drv_index] =
-			alloc_disk(1 << NWD_SHIFT);
-		if (!h->gendisk[drv_index]) {
-			dev_err(&h->pdev->dev,
-				"could not allocate a new disk %d\n",
-				drv_index);
-			goto err_free_drive_info;
-		}
-	}
-	memcpy(h->drv[drv_index]->LunID, lunid,
-		sizeof(h->drv[drv_index]->LunID));
-	if (cciss_create_ld_sysfs_entry(h, drv_index))
-		goto err_free_disk;
-	/* Don't need to mark this busy because nobody */
-	/* else knows about this disk yet to contend */
-	/* for access to it. */
-	h->drv[drv_index]->busy_configuring = 0;
-	wmb();
-	return drv_index;
-
-err_free_disk:
-	cciss_free_gendisk(h, drv_index);
-err_free_drive_info:
-	cciss_free_drive_info(h, drv_index);
-	return -1;
-}
-
-/* This is for the special case of a controller which
- * has no logical drives.  In this case, we still need
- * to register a disk so the controller can be accessed
- * by the Array Config Utility.
- */
-static void cciss_add_controller_node(ctlr_info_t *h)
-{
-	struct gendisk *disk;
-	int drv_index;
-
-	if (h->gendisk[0] != NULL) /* already did this? Then bail. */
-		return;
-
-	drv_index = cciss_add_gendisk(h, CTLR_LUNID, 1);
-	if (drv_index == -1)
-		goto error;
-	h->drv[drv_index]->block_size = 512;
-	h->drv[drv_index]->nr_blocks = 0;
-	h->drv[drv_index]->heads = 0;
-	h->drv[drv_index]->sectors = 0;
-	h->drv[drv_index]->cylinders = 0;
-	h->drv[drv_index]->raid_level = -1;
-	memset(h->drv[drv_index]->serial_no, 0, 16);
-	disk = h->gendisk[drv_index];
-	if (cciss_add_disk(h, disk, drv_index) == 0)
-		return;
-	cciss_free_gendisk(h, drv_index);
-	cciss_free_drive_info(h, drv_index);
-error:
-	dev_warn(&h->pdev->dev, "could not add disk 0.\n");
-	return;
-}
-
-/* This function will add and remove logical drives from the Logical
- * drive array of the controller and maintain persistency of ordering
- * so that mount points are preserved until the next reboot.  This allows
- * for the removal of logical drives in the middle of the drive array
- * without a re-ordering of those drives.
- * INPUT
- * h		= The controller to perform the operations on
- */
-static int rebuild_lun_table(ctlr_info_t *h, int first_time,
-	int via_ioctl)
-{
-	int num_luns;
-	ReportLunData_struct *ld_buff = NULL;
-	int return_code;
-	int listlength = 0;
-	int i;
-	int drv_found;
-	int drv_index = 0;
-	unsigned char lunid[8] = CTLR_LUNID;
-	unsigned long flags;
-
-	if (!capable(CAP_SYS_RAWIO))
-		return -EPERM;
-
-	/* Set busy_configuring flag for this operation */
-	spin_lock_irqsave(&h->lock, flags);
-	if (h->busy_configuring) {
-		spin_unlock_irqrestore(&h->lock, flags);
-		return -EBUSY;
-	}
-	h->busy_configuring = 1;
-	spin_unlock_irqrestore(&h->lock, flags);
-
-	ld_buff = kzalloc(sizeof(ReportLunData_struct), GFP_KERNEL);
-	if (ld_buff == NULL)
-		goto mem_msg;
-
-	return_code = sendcmd_withirq(h, CISS_REPORT_LOG, ld_buff,
-				      sizeof(ReportLunData_struct),
-				      0, CTLR_LUNID, TYPE_CMD);
-
-	if (return_code == IO_OK)
-		listlength = be32_to_cpu(*(__be32 *) ld_buff->LUNListLength);
-	else {	/* reading number of logical volumes failed */
-		dev_warn(&h->pdev->dev,
-			"report logical volume command failed\n");
-		listlength = 0;
-		goto freeret;
-	}
-
-	num_luns = listlength / 8;	/* 8 bytes per entry */
-	if (num_luns > CISS_MAX_LUN) {
-		num_luns = CISS_MAX_LUN;
-		dev_warn(&h->pdev->dev, "more luns configured"
-		       " on controller than can be handled by"
-		       " this driver.\n");
-	}
-
-	if (num_luns == 0)
-		cciss_add_controller_node(h);
-
-	/* Compare controller drive array to driver's drive array
-	 * to see if any drives are missing on the controller due
-	 * to action of Array Config Utility (user deletes drive)
-	 * and deregister logical drives which have disappeared.
-	 */
-	for (i = 0; i <= h->highest_lun; i++) {
-		int j;
-		drv_found = 0;
-
-		/* skip holes in the array from already deleted drives */
-		if (h->drv[i] == NULL)
-			continue;
-
-		for (j = 0; j < num_luns; j++) {
-			memcpy(lunid, &ld_buff->LUN[j][0], sizeof(lunid));
-			if (memcmp(h->drv[i]->LunID, lunid,
-				sizeof(lunid)) == 0) {
-				drv_found = 1;
-				break;
-			}
-		}
-		if (!drv_found) {
-			/* Deregister it from the OS, it's gone. */
-			spin_lock_irqsave(&h->lock, flags);
-			h->drv[i]->busy_configuring = 1;
-			spin_unlock_irqrestore(&h->lock, flags);
-			return_code = deregister_disk(h, i, 1, via_ioctl);
-			if (h->drv[i] != NULL)
-				h->drv[i]->busy_configuring = 0;
-		}
-	}
-
-	/* Compare controller drive array to driver's drive array.
-	 * Check for updates in the drive information and any new drives
-	 * on the controller due to ACU adding logical drives, or changing
-	 * a logical drive's size, etc.  Reregister any new/changed drives
-	 */
-	for (i = 0; i < num_luns; i++) {
-		int j;
-
-		drv_found = 0;
-
-		memcpy(lunid, &ld_buff->LUN[i][0], sizeof(lunid));
-		/* Find if the LUN is already in the drive array
-		 * of the driver.  If so then update its info
-		 * if not in use.  If it does not exist then find
-		 * the first free index and add it.
-		 */
-		for (j = 0; j <= h->highest_lun; j++) {
-			if (h->drv[j] != NULL &&
-				memcmp(h->drv[j]->LunID, lunid,
-					sizeof(h->drv[j]->LunID)) == 0) {
-				drv_index = j;
-				drv_found = 1;
-				break;
-			}
-		}
-
-		/* check if the drive was found already in the array */
-		if (!drv_found) {
-			drv_index = cciss_add_gendisk(h, lunid, 0);
-			if (drv_index == -1)
-				goto freeret;
-		}
-		cciss_update_drive_info(h, drv_index, first_time, via_ioctl);
-	}		/* end for */
-
-freeret:
-	kfree(ld_buff);
-	h->busy_configuring = 0;
-	/* We return -1 here to tell the ACU that we have registered/updated
-	 * all of the drives that we can and to keep it from calling us
-	 * additional times.
-	 */
-	return -1;
-mem_msg:
-	dev_err(&h->pdev->dev, "out of memory\n");
-	h->busy_configuring = 0;
-	goto freeret;
-}
-
-static void cciss_clear_drive_info(drive_info_struct *drive_info)
-{
-	/* zero out the disk size info */
-	drive_info->nr_blocks = 0;
-	drive_info->block_size = 0;
-	drive_info->heads = 0;
-	drive_info->sectors = 0;
-	drive_info->cylinders = 0;
-	drive_info->raid_level = -1;
-	memset(drive_info->serial_no, 0, sizeof(drive_info->serial_no));
-	memset(drive_info->model, 0, sizeof(drive_info->model));
-	memset(drive_info->rev, 0, sizeof(drive_info->rev));
-	memset(drive_info->vendor, 0, sizeof(drive_info->vendor));
-	/*
-	 * don't clear the LUNID though, we need to remember which
-	 * one this one is.
-	 */
-}
-
-/* This function will deregister the disk and it's queue from the
- * kernel.  It must be called with the controller lock held and the
- * drv structures busy_configuring flag set.  It's parameters are:
- *
- * disk = This is the disk to be deregistered
- * drv  = This is the drive_info_struct associated with the disk to be
- *        deregistered.  It contains information about the disk used
- *        by the driver.
- * clear_all = This flag determines whether or not the disk information
- *             is going to be completely cleared out and the highest_lun
- *             reset.  Sometimes we want to clear out information about
- *             the disk in preparation for re-adding it.  In this case
- *             the highest_lun should be left unchanged and the LunID
- *             should not be cleared.
- * via_ioctl
- *    This indicates whether we've reached this path via ioctl.
- *    This affects the maximum usage count allowed for c0d0 to be messed with.
- *    If this path is reached via ioctl(), then the max_usage_count will
- *    be 1, as the process calling ioctl() has got to have the device open.
- *    If we get here via sysfs, then the max usage count will be zero.
-*/
-static int deregister_disk(ctlr_info_t *h, int drv_index,
-			   int clear_all, int via_ioctl)
-{
-	int i;
-	struct gendisk *disk;
-	drive_info_struct *drv;
-	int recalculate_highest_lun;
-
-	if (!capable(CAP_SYS_RAWIO))
-		return -EPERM;
-
-	drv = h->drv[drv_index];
-	disk = h->gendisk[drv_index];
-
-	/* make sure logical volume is NOT is use */
-	if (clear_all || (h->gendisk[0] == disk)) {
-		if (drv->usage_count > via_ioctl)
-			return -EBUSY;
-	} else if (drv->usage_count > 0)
-		return -EBUSY;
-
-	recalculate_highest_lun = (drv == h->drv[h->highest_lun]);
-
-	/* invalidate the devices and deregister the disk.  If it is disk
-	 * zero do not deregister it but just zero out it's values.  This
-	 * allows us to delete disk zero but keep the controller registered.
-	 */
-	if (h->gendisk[0] != disk) {
-		struct request_queue *q = disk->queue;
-		if (disk->flags & GENHD_FL_UP) {
-			cciss_destroy_ld_sysfs_entry(h, drv_index, 0);
-			del_gendisk(disk);
-		}
-		if (q)
-			blk_cleanup_queue(q);
-		/* If clear_all is set then we are deleting the logical
-		 * drive, not just refreshing its info.  For drives
-		 * other than disk 0 we will call put_disk.  We do not
-		 * do this for disk 0 as we need it to be able to
-		 * configure the controller.
-		 */
-		if (clear_all){
-			/* This isn't pretty, but we need to find the
-			 * disk in our array and NULL our the pointer.
-			 * This is so that we will call alloc_disk if
-			 * this index is used again later.
-			 */
-			for (i=0; i < CISS_MAX_LUN; i++){
-				if (h->gendisk[i] == disk) {
-					h->gendisk[i] = NULL;
-					break;
-				}
-			}
-			put_disk(disk);
-		}
-	} else {
-		set_capacity(disk, 0);
-		cciss_clear_drive_info(drv);
-	}
-
-	--h->num_luns;
-
-	/* if it was the last disk, find the new hightest lun */
-	if (clear_all && recalculate_highest_lun) {
-		int newhighest = -1;
-		for (i = 0; i <= h->highest_lun; i++) {
-			/* if the disk has size > 0, it is available */
-			if (h->drv[i] && h->drv[i]->heads)
-				newhighest = i;
-		}
-		h->highest_lun = newhighest;
-	}
-	return 0;
-}
-
-static int fill_cmd(ctlr_info_t *h, CommandList_struct *c, __u8 cmd, void *buff,
-		size_t size, __u8 page_code, unsigned char *scsi3addr,
-		int cmd_type)
-{
-	u64bit buff_dma_handle;
-	int status = IO_OK;
-
-	c->cmd_type = CMD_IOCTL_PEND;
-	c->Header.ReplyQueue = 0;
-	if (buff != NULL) {
-		c->Header.SGList = 1;
-		c->Header.SGTotal = 1;
-	} else {
-		c->Header.SGList = 0;
-		c->Header.SGTotal = 0;
-	}
-	c->Header.Tag.lower = c->busaddr;
-	memcpy(c->Header.LUN.LunAddrBytes, scsi3addr, 8);
-
-	c->Request.Type.Type = cmd_type;
-	if (cmd_type == TYPE_CMD) {
-		switch (cmd) {
-		case CISS_INQUIRY:
-			/* are we trying to read a vital product page */
-			if (page_code != 0) {
-				c->Request.CDB[1] = 0x01;
-				c->Request.CDB[2] = page_code;
-			}
-			c->Request.CDBLen = 6;
-			c->Request.Type.Attribute = ATTR_SIMPLE;
-			c->Request.Type.Direction = XFER_READ;
-			c->Request.Timeout = 0;
-			c->Request.CDB[0] = CISS_INQUIRY;
-			c->Request.CDB[4] = size & 0xFF;
-			break;
-		case CISS_REPORT_LOG:
-		case CISS_REPORT_PHYS:
-			/* Talking to controller so It's a physical command
-			   mode = 00 target = 0.  Nothing to write.
-			 */
-			c->Request.CDBLen = 12;
-			c->Request.Type.Attribute = ATTR_SIMPLE;
-			c->Request.Type.Direction = XFER_READ;
-			c->Request.Timeout = 0;
-			c->Request.CDB[0] = cmd;
-			c->Request.CDB[6] = (size >> 24) & 0xFF; /* MSB */
-			c->Request.CDB[7] = (size >> 16) & 0xFF;
-			c->Request.CDB[8] = (size >> 8) & 0xFF;
-			c->Request.CDB[9] = size & 0xFF;
-			break;
-
-		case CCISS_READ_CAPACITY:
-			c->Request.CDBLen = 10;
-			c->Request.Type.Attribute = ATTR_SIMPLE;
-			c->Request.Type.Direction = XFER_READ;
-			c->Request.Timeout = 0;
-			c->Request.CDB[0] = cmd;
-			break;
-		case CCISS_READ_CAPACITY_16:
-			c->Request.CDBLen = 16;
-			c->Request.Type.Attribute = ATTR_SIMPLE;
-			c->Request.Type.Direction = XFER_READ;
-			c->Request.Timeout = 0;
-			c->Request.CDB[0] = cmd;
-			c->Request.CDB[1] = 0x10;
-			c->Request.CDB[10] = (size >> 24) & 0xFF;
-			c->Request.CDB[11] = (size >> 16) & 0xFF;
-			c->Request.CDB[12] = (size >> 8) & 0xFF;
-			c->Request.CDB[13] = size & 0xFF;
-			c->Request.Timeout = 0;
-			c->Request.CDB[0] = cmd;
-			break;
-		case CCISS_CACHE_FLUSH:
-			c->Request.CDBLen = 12;
-			c->Request.Type.Attribute = ATTR_SIMPLE;
-			c->Request.Type.Direction = XFER_WRITE;
-			c->Request.Timeout = 0;
-			c->Request.CDB[0] = BMIC_WRITE;
-			c->Request.CDB[6] = BMIC_CACHE_FLUSH;
-			c->Request.CDB[7] = (size >> 8) & 0xFF;
-			c->Request.CDB[8] = size & 0xFF;
-			break;
-		case TEST_UNIT_READY:
-			c->Request.CDBLen = 6;
-			c->Request.Type.Attribute = ATTR_SIMPLE;
-			c->Request.Type.Direction = XFER_NONE;
-			c->Request.Timeout = 0;
-			break;
-		default:
-			dev_warn(&h->pdev->dev, "Unknown Command 0x%c\n", cmd);
-			return IO_ERROR;
-		}
-	} else if (cmd_type == TYPE_MSG) {
-		switch (cmd) {
-		case CCISS_ABORT_MSG:
-			c->Request.CDBLen = 12;
-			c->Request.Type.Attribute = ATTR_SIMPLE;
-			c->Request.Type.Direction = XFER_WRITE;
-			c->Request.Timeout = 0;
-			c->Request.CDB[0] = cmd;	/* abort */
-			c->Request.CDB[1] = 0;	/* abort a command */
-			/* buff contains the tag of the command to abort */
-			memcpy(&c->Request.CDB[4], buff, 8);
-			break;
-		case CCISS_RESET_MSG:
-			c->Request.CDBLen = 16;
-			c->Request.Type.Attribute = ATTR_SIMPLE;
-			c->Request.Type.Direction = XFER_NONE;
-			c->Request.Timeout = 0;
-			memset(&c->Request.CDB[0], 0, sizeof(c->Request.CDB));
-			c->Request.CDB[0] = cmd;	/* reset */
-			c->Request.CDB[1] = CCISS_RESET_TYPE_TARGET;
-			break;
-		case CCISS_NOOP_MSG:
-			c->Request.CDBLen = 1;
-			c->Request.Type.Attribute = ATTR_SIMPLE;
-			c->Request.Type.Direction = XFER_WRITE;
-			c->Request.Timeout = 0;
-			c->Request.CDB[0] = cmd;
-			break;
-		default:
-			dev_warn(&h->pdev->dev,
-				"unknown message type %d\n", cmd);
-			return IO_ERROR;
-		}
-	} else {
-		dev_warn(&h->pdev->dev, "unknown command type %d\n", cmd_type);
-		return IO_ERROR;
-	}
-	/* Fill in the scatter gather information */
-	if (size > 0) {
-		buff_dma_handle.val = (__u64) pci_map_single(h->pdev,
-							     buff, size,
-							     PCI_DMA_BIDIRECTIONAL);
-		c->SG[0].Addr.lower = buff_dma_handle.val32.lower;
-		c->SG[0].Addr.upper = buff_dma_handle.val32.upper;
-		c->SG[0].Len = size;
-		c->SG[0].Ext = 0;	/* we are not chaining */
-	}
-	return status;
-}
-
-static int cciss_send_reset(ctlr_info_t *h, unsigned char *scsi3addr,
-			    u8 reset_type)
-{
-	CommandList_struct *c;
-	int return_status;
-
-	c = cmd_alloc(h);
-	if (!c)
-		return -ENOMEM;
-	return_status = fill_cmd(h, c, CCISS_RESET_MSG, NULL, 0, 0,
-		CTLR_LUNID, TYPE_MSG);
-	c->Request.CDB[1] = reset_type; /* fill_cmd defaults to target reset */
-	if (return_status != IO_OK) {
-		cmd_special_free(h, c);
-		return return_status;
-	}
-	c->waiting = NULL;
-	enqueue_cmd_and_start_io(h, c);
-	/* Don't wait for completion, the reset won't complete.  Don't free
-	 * the command either.  This is the last command we will send before
-	 * re-initializing everything, so it doesn't matter and won't leak.
-	 */
-	return 0;
-}
-
-static int check_target_status(ctlr_info_t *h, CommandList_struct *c)
-{
-	switch (c->err_info->ScsiStatus) {
-	case SAM_STAT_GOOD:
-		return IO_OK;
-	case SAM_STAT_CHECK_CONDITION:
-		switch (0xf & c->err_info->SenseInfo[2]) {
-		case 0: return IO_OK; /* no sense */
-		case 1: return IO_OK; /* recovered error */
-		default:
-			if (check_for_unit_attention(h, c))
-				return IO_NEEDS_RETRY;
-			dev_warn(&h->pdev->dev, "cmd 0x%02x "
-				"check condition, sense key = 0x%02x\n",
-				c->Request.CDB[0], c->err_info->SenseInfo[2]);
-		}
-		break;
-	default:
-		dev_warn(&h->pdev->dev, "cmd 0x%02x"
-			"scsi status = 0x%02x\n",
-			c->Request.CDB[0], c->err_info->ScsiStatus);
-		break;
-	}
-	return IO_ERROR;
-}
-
-static int process_sendcmd_error(ctlr_info_t *h, CommandList_struct *c)
-{
-	int return_status = IO_OK;
-
-	if (c->err_info->CommandStatus == CMD_SUCCESS)
-		return IO_OK;
-
-	switch (c->err_info->CommandStatus) {
-	case CMD_TARGET_STATUS:
-		return_status = check_target_status(h, c);
-		break;
-	case CMD_DATA_UNDERRUN:
-	case CMD_DATA_OVERRUN:
-		/* expected for inquiry and report lun commands */
-		break;
-	case CMD_INVALID:
-		dev_warn(&h->pdev->dev, "cmd 0x%02x is "
-		       "reported invalid\n", c->Request.CDB[0]);
-		return_status = IO_ERROR;
-		break;
-	case CMD_PROTOCOL_ERR:
-		dev_warn(&h->pdev->dev, "cmd 0x%02x has "
-		       "protocol error\n", c->Request.CDB[0]);
-		return_status = IO_ERROR;
-		break;
-	case CMD_HARDWARE_ERR:
-		dev_warn(&h->pdev->dev, "cmd 0x%02x had "
-		       " hardware error\n", c->Request.CDB[0]);
-		return_status = IO_ERROR;
-		break;
-	case CMD_CONNECTION_LOST:
-		dev_warn(&h->pdev->dev, "cmd 0x%02x had "
-		       "connection lost\n", c->Request.CDB[0]);
-		return_status = IO_ERROR;
-		break;
-	case CMD_ABORTED:
-		dev_warn(&h->pdev->dev, "cmd 0x%02x was "
-		       "aborted\n", c->Request.CDB[0]);
-		return_status = IO_ERROR;
-		break;
-	case CMD_ABORT_FAILED:
-		dev_warn(&h->pdev->dev, "cmd 0x%02x reports "
-		       "abort failed\n", c->Request.CDB[0]);
-		return_status = IO_ERROR;
-		break;
-	case CMD_UNSOLICITED_ABORT:
-		dev_warn(&h->pdev->dev, "unsolicited abort 0x%02x\n",
-			c->Request.CDB[0]);
-		return_status = IO_NEEDS_RETRY;
-		break;
-	case CMD_UNABORTABLE:
-		dev_warn(&h->pdev->dev, "cmd unabortable\n");
-		return_status = IO_ERROR;
-		break;
-	default:
-		dev_warn(&h->pdev->dev, "cmd 0x%02x returned "
-		       "unknown status %x\n", c->Request.CDB[0],
-		       c->err_info->CommandStatus);
-		return_status = IO_ERROR;
-	}
-	return return_status;
-}
-
-static int sendcmd_withirq_core(ctlr_info_t *h, CommandList_struct *c,
-	int attempt_retry)
-{
-	DECLARE_COMPLETION_ONSTACK(wait);
-	u64bit buff_dma_handle;
-	int return_status = IO_OK;
-
-resend_cmd2:
-	c->waiting = &wait;
-	enqueue_cmd_and_start_io(h, c);
-
-	wait_for_completion(&wait);
-
-	if (c->err_info->CommandStatus == 0 || !attempt_retry)
-		goto command_done;
-
-	return_status = process_sendcmd_error(h, c);
-
-	if (return_status == IO_NEEDS_RETRY &&
-		c->retry_count < MAX_CMD_RETRIES) {
-		dev_warn(&h->pdev->dev, "retrying 0x%02x\n",
-			c->Request.CDB[0]);
-		c->retry_count++;
-		/* erase the old error information */
-		memset(c->err_info, 0, sizeof(ErrorInfo_struct));
-		return_status = IO_OK;
-		reinit_completion(&wait);
-		goto resend_cmd2;
-	}
-
-command_done:
-	/* unlock the buffers from DMA */
-	buff_dma_handle.val32.lower = c->SG[0].Addr.lower;
-	buff_dma_handle.val32.upper = c->SG[0].Addr.upper;
-	pci_unmap_single(h->pdev, (dma_addr_t) buff_dma_handle.val,
-			 c->SG[0].Len, PCI_DMA_BIDIRECTIONAL);
-	return return_status;
-}
-
-static int sendcmd_withirq(ctlr_info_t *h, __u8 cmd, void *buff, size_t size,
-			   __u8 page_code, unsigned char scsi3addr[],
-			int cmd_type)
-{
-	CommandList_struct *c;
-	int return_status;
-
-	c = cmd_special_alloc(h);
-	if (!c)
-		return -ENOMEM;
-	return_status = fill_cmd(h, c, cmd, buff, size, page_code,
-		scsi3addr, cmd_type);
-	if (return_status == IO_OK)
-		return_status = sendcmd_withirq_core(h, c, 1);
-
-	cmd_special_free(h, c);
-	return return_status;
-}
-
-static void cciss_geometry_inquiry(ctlr_info_t *h, int logvol,
-				   sector_t total_size,
-				   unsigned int block_size,
-				   InquiryData_struct *inq_buff,
-				   drive_info_struct *drv)
-{
-	int return_code;
-	unsigned long t;
-	unsigned char scsi3addr[8];
-
-	memset(inq_buff, 0, sizeof(InquiryData_struct));
-	log_unit_to_scsi3addr(h, scsi3addr, logvol);
-	return_code = sendcmd_withirq(h, CISS_INQUIRY, inq_buff,
-			sizeof(*inq_buff), 0xC1, scsi3addr, TYPE_CMD);
-	if (return_code == IO_OK) {
-		if (inq_buff->data_byte[8] == 0xFF) {
-			dev_warn(&h->pdev->dev,
-			       "reading geometry failed, volume "
-			       "does not support reading geometry\n");
-			drv->heads = 255;
-			drv->sectors = 32;	/* Sectors per track */
-			drv->cylinders = total_size + 1;
-			drv->raid_level = RAID_UNKNOWN;
-		} else {
-			drv->heads = inq_buff->data_byte[6];
-			drv->sectors = inq_buff->data_byte[7];
-			drv->cylinders = (inq_buff->data_byte[4] & 0xff) << 8;
-			drv->cylinders += inq_buff->data_byte[5];
-			drv->raid_level = inq_buff->data_byte[8];
-		}
-		drv->block_size = block_size;
-		drv->nr_blocks = total_size + 1;
-		t = drv->heads * drv->sectors;
-		if (t > 1) {
-			sector_t real_size = total_size + 1;
-			unsigned long rem = sector_div(real_size, t);
-			if (rem)
-				real_size++;
-			drv->cylinders = real_size;
-		}
-	} else {		/* Get geometry failed */
-		dev_warn(&h->pdev->dev, "reading geometry failed\n");
-	}
-}
-
-static void
-cciss_read_capacity(ctlr_info_t *h, int logvol, sector_t *total_size,
-		    unsigned int *block_size)
-{
-	ReadCapdata_struct *buf;
-	int return_code;
-	unsigned char scsi3addr[8];
-
-	buf = kzalloc(sizeof(ReadCapdata_struct), GFP_KERNEL);
-	if (!buf) {
-		dev_warn(&h->pdev->dev, "out of memory\n");
-		return;
-	}
-
-	log_unit_to_scsi3addr(h, scsi3addr, logvol);
-	return_code = sendcmd_withirq(h, CCISS_READ_CAPACITY, buf,
-		sizeof(ReadCapdata_struct), 0, scsi3addr, TYPE_CMD);
-	if (return_code == IO_OK) {
-		*total_size = be32_to_cpu(*(__be32 *) buf->total_size);
-		*block_size = be32_to_cpu(*(__be32 *) buf->block_size);
-	} else {		/* read capacity command failed */
-		dev_warn(&h->pdev->dev, "read capacity failed\n");
-		*total_size = 0;
-		*block_size = BLOCK_SIZE;
-	}
-	kfree(buf);
-}
-
-static void cciss_read_capacity_16(ctlr_info_t *h, int logvol,
-	sector_t *total_size, unsigned int *block_size)
-{
-	ReadCapdata_struct_16 *buf;
-	int return_code;
-	unsigned char scsi3addr[8];
-
-	buf = kzalloc(sizeof(ReadCapdata_struct_16), GFP_KERNEL);
-	if (!buf) {
-		dev_warn(&h->pdev->dev, "out of memory\n");
-		return;
-	}
-
-	log_unit_to_scsi3addr(h, scsi3addr, logvol);
-	return_code = sendcmd_withirq(h, CCISS_READ_CAPACITY_16,
-		buf, sizeof(ReadCapdata_struct_16),
-			0, scsi3addr, TYPE_CMD);
-	if (return_code == IO_OK) {
-		*total_size = be64_to_cpu(*(__be64 *) buf->total_size);
-		*block_size = be32_to_cpu(*(__be32 *) buf->block_size);
-	} else {		/* read capacity command failed */
-		dev_warn(&h->pdev->dev, "read capacity failed\n");
-		*total_size = 0;
-		*block_size = BLOCK_SIZE;
-	}
-	dev_info(&h->pdev->dev, "      blocks= %llu block_size= %d\n",
-	       (unsigned long long)*total_size+1, *block_size);
-	kfree(buf);
-}
-
-static int cciss_revalidate(struct gendisk *disk)
-{
-	ctlr_info_t *h = get_host(disk);
-	drive_info_struct *drv = get_drv(disk);
-	int logvol;
-	int FOUND = 0;
-	unsigned int block_size;
-	sector_t total_size;
-	InquiryData_struct *inq_buff = NULL;
-
-	for (logvol = 0; logvol <= h->highest_lun; logvol++) {
-		if (!h->drv[logvol])
-			continue;
-		if (memcmp(h->drv[logvol]->LunID, drv->LunID,
-			sizeof(drv->LunID)) == 0) {
-			FOUND = 1;
-			break;
-		}
-	}
-
-	if (!FOUND)
-		return 1;
-
-	inq_buff = kmalloc(sizeof(InquiryData_struct), GFP_KERNEL);
-	if (inq_buff == NULL) {
-		dev_warn(&h->pdev->dev, "out of memory\n");
-		return 1;
-	}
-	if (h->cciss_read == CCISS_READ_10) {
-		cciss_read_capacity(h, logvol,
-					&total_size, &block_size);
-	} else {
-		cciss_read_capacity_16(h, logvol,
-					&total_size, &block_size);
-	}
-	cciss_geometry_inquiry(h, logvol, total_size, block_size,
-			       inq_buff, drv);
-
-	blk_queue_logical_block_size(drv->queue, drv->block_size);
-	set_capacity(disk, drv->nr_blocks);
-
-	kfree(inq_buff);
-	return 0;
-}
-
-/*
- * Map (physical) PCI mem into (virtual) kernel space
- */
-static void __iomem *remap_pci_mem(ulong base, ulong size)
-{
-	ulong page_base = ((ulong) base) & PAGE_MASK;
-	ulong page_offs = ((ulong) base) - page_base;
-	void __iomem *page_remapped = ioremap(page_base, page_offs + size);
-
-	return page_remapped ? (page_remapped + page_offs) : NULL;
-}
-
-/*
- * Takes jobs of the Q and sends them to the hardware, then puts it on
- * the Q to wait for completion.
- */
-static void start_io(ctlr_info_t *h)
-{
-	CommandList_struct *c;
-
-	while (!list_empty(&h->reqQ)) {
-		c = list_entry(h->reqQ.next, CommandList_struct, list);
-		/* can't do anything if fifo is full */
-		if ((h->access.fifo_full(h))) {
-			dev_warn(&h->pdev->dev, "fifo full\n");
-			break;
-		}
-
-		/* Get the first entry from the Request Q */
-		removeQ(c);
-		h->Qdepth--;
-
-		/* Tell the controller execute command */
-		h->access.submit_command(h, c);
-
-		/* Put job onto the completed Q */
-		addQ(&h->cmpQ, c);
-	}
-}
-
-/* Assumes that h->lock is held. */
-/* Zeros out the error record and then resends the command back */
-/* to the controller */
-static inline void resend_cciss_cmd(ctlr_info_t *h, CommandList_struct *c)
-{
-	/* erase the old error information */
-	memset(c->err_info, 0, sizeof(ErrorInfo_struct));
-
-	/* add it to software queue and then send it to the controller */
-	addQ(&h->reqQ, c);
-	h->Qdepth++;
-	if (h->Qdepth > h->maxQsinceinit)
-		h->maxQsinceinit = h->Qdepth;
-
-	start_io(h);
-}
-
-static inline unsigned int make_status_bytes(unsigned int scsi_status_byte,
-	unsigned int msg_byte, unsigned int host_byte,
-	unsigned int driver_byte)
-{
-	/* inverse of macros in scsi.h */
-	return (scsi_status_byte & 0xff) |
-		((msg_byte & 0xff) << 8) |
-		((host_byte & 0xff) << 16) |
-		((driver_byte & 0xff) << 24);
-}
-
-static inline int evaluate_target_status(ctlr_info_t *h,
-			CommandList_struct *cmd, int *retry_cmd)
-{
-	unsigned char sense_key;
-	unsigned char status_byte, msg_byte, host_byte, driver_byte;
-	int error_value;
-
-	*retry_cmd = 0;
-	/* If we get in here, it means we got "target status", that is, scsi status */
-	status_byte = cmd->err_info->ScsiStatus;
-	driver_byte = DRIVER_OK;
-	msg_byte = cmd->err_info->CommandStatus; /* correct?  seems too device specific */
-
-	if (blk_rq_is_passthrough(cmd->rq))
-		host_byte = DID_PASSTHROUGH;
-	else
-		host_byte = DID_OK;
-
-	error_value = make_status_bytes(status_byte, msg_byte,
-		host_byte, driver_byte);
-
-	if (cmd->err_info->ScsiStatus != SAM_STAT_CHECK_CONDITION) {
-		if (!blk_rq_is_passthrough(cmd->rq))
-			dev_warn(&h->pdev->dev, "cmd %p "
-			       "has SCSI Status 0x%x\n",
-			       cmd, cmd->err_info->ScsiStatus);
-		return error_value;
-	}
-
-	/* check the sense key */
-	sense_key = 0xf & cmd->err_info->SenseInfo[2];
-	/* no status or recovered error */
-	if (((sense_key == 0x0) || (sense_key == 0x1)) &&
-	    !blk_rq_is_passthrough(cmd->rq))
-		error_value = 0;
-
-	if (check_for_unit_attention(h, cmd)) {
-		*retry_cmd = !blk_rq_is_passthrough(cmd->rq);
-		return 0;
-	}
-
-	/* Not SG_IO or similar? */
-	if (!blk_rq_is_passthrough(cmd->rq)) {
-		if (error_value != 0)
-			dev_warn(&h->pdev->dev, "cmd %p has CHECK CONDITION"
-			       " sense key = 0x%x\n", cmd, sense_key);
-		return error_value;
-	}
-
-	scsi_req(cmd->rq)->sense_len = cmd->err_info->SenseLen;
-	return error_value;
-}
-
-/* checks the status of the job and calls complete buffers to mark all
- * buffers for the completed job. Note that this function does not need
- * to hold the hba/queue lock.
- */
-static inline void complete_command(ctlr_info_t *h, CommandList_struct *cmd,
-				    int timeout)
-{
-	int retry_cmd = 0;
-	struct request *rq = cmd->rq;
-	struct scsi_request *sreq = scsi_req(rq);
-
-	sreq->result = 0;
-
-	if (timeout)
-		sreq->result = make_status_bytes(0, 0, 0, DRIVER_TIMEOUT);
-
-	if (cmd->err_info->CommandStatus == 0)	/* no error has occurred */
-		goto after_error_processing;
-
-	switch (cmd->err_info->CommandStatus) {
-	case CMD_TARGET_STATUS:
-		sreq->result = evaluate_target_status(h, cmd, &retry_cmd);
-		break;
-	case CMD_DATA_UNDERRUN:
-		if (!blk_rq_is_passthrough(cmd->rq)) {
-			dev_warn(&h->pdev->dev, "cmd %p has"
-			       " completed with data underrun "
-			       "reported\n", cmd);
-		}
-		break;
-	case CMD_DATA_OVERRUN:
-		if (!blk_rq_is_passthrough(cmd->rq))
-			dev_warn(&h->pdev->dev, "cciss: cmd %p has"
-			       " completed with data overrun "
-			       "reported\n", cmd);
-		break;
-	case CMD_INVALID:
-		dev_warn(&h->pdev->dev, "cciss: cmd %p is "
-		       "reported invalid\n", cmd);
-		sreq->result = make_status_bytes(SAM_STAT_GOOD,
-			cmd->err_info->CommandStatus, DRIVER_OK,
-			blk_rq_is_passthrough(cmd->rq) ?
-				DID_PASSTHROUGH : DID_ERROR);
-		break;
-	case CMD_PROTOCOL_ERR:
-		dev_warn(&h->pdev->dev, "cciss: cmd %p has "
-		       "protocol error\n", cmd);
-		sreq->result = make_status_bytes(SAM_STAT_GOOD,
-			cmd->err_info->CommandStatus, DRIVER_OK,
-			blk_rq_is_passthrough(cmd->rq) ?
-				DID_PASSTHROUGH : DID_ERROR);
-		break;
-	case CMD_HARDWARE_ERR:
-		dev_warn(&h->pdev->dev, "cciss: cmd %p had "
-		       " hardware error\n", cmd);
-		sreq->result = make_status_bytes(SAM_STAT_GOOD,
-			cmd->err_info->CommandStatus, DRIVER_OK,
-			blk_rq_is_passthrough(cmd->rq) ?
-				DID_PASSTHROUGH : DID_ERROR);
-		break;
-	case CMD_CONNECTION_LOST:
-		dev_warn(&h->pdev->dev, "cciss: cmd %p had "
-		       "connection lost\n", cmd);
-		sreq->result = make_status_bytes(SAM_STAT_GOOD,
-			cmd->err_info->CommandStatus, DRIVER_OK,
-			blk_rq_is_passthrough(cmd->rq) ?
-				DID_PASSTHROUGH : DID_ERROR);
-		break;
-	case CMD_ABORTED:
-		dev_warn(&h->pdev->dev, "cciss: cmd %p was "
-		       "aborted\n", cmd);
-		sreq->result = make_status_bytes(SAM_STAT_GOOD,
-			cmd->err_info->CommandStatus, DRIVER_OK,
-			blk_rq_is_passthrough(cmd->rq) ?
-				DID_PASSTHROUGH : DID_ABORT);
-		break;
-	case CMD_ABORT_FAILED:
-		dev_warn(&h->pdev->dev, "cciss: cmd %p reports "
-		       "abort failed\n", cmd);
-		sreq->result = make_status_bytes(SAM_STAT_GOOD,
-			cmd->err_info->CommandStatus, DRIVER_OK,
-			blk_rq_is_passthrough(cmd->rq) ?
-				DID_PASSTHROUGH : DID_ERROR);
-		break;
-	case CMD_UNSOLICITED_ABORT:
-		dev_warn(&h->pdev->dev, "cciss%d: unsolicited "
-		       "abort %p\n", h->ctlr, cmd);
-		if (cmd->retry_count < MAX_CMD_RETRIES) {
-			retry_cmd = 1;
-			dev_warn(&h->pdev->dev, "retrying %p\n", cmd);
-			cmd->retry_count++;
-		} else
-			dev_warn(&h->pdev->dev,
-				"%p retried too many times\n", cmd);
-		sreq->result = make_status_bytes(SAM_STAT_GOOD,
-			cmd->err_info->CommandStatus, DRIVER_OK,
-			blk_rq_is_passthrough(cmd->rq) ?
-				DID_PASSTHROUGH : DID_ABORT);
-		break;
-	case CMD_TIMEOUT:
-		dev_warn(&h->pdev->dev, "cmd %p timedout\n", cmd);
-		sreq->result = make_status_bytes(SAM_STAT_GOOD,
-			cmd->err_info->CommandStatus, DRIVER_OK,
-			blk_rq_is_passthrough(cmd->rq) ?
-				DID_PASSTHROUGH : DID_ERROR);
-		break;
-	case CMD_UNABORTABLE:
-		dev_warn(&h->pdev->dev, "cmd %p unabortable\n", cmd);
-		sreq->result = make_status_bytes(SAM_STAT_GOOD,
-			cmd->err_info->CommandStatus, DRIVER_OK,
-			blk_rq_is_passthrough(cmd->rq) ?
-				DID_PASSTHROUGH : DID_ERROR);
-		break;
-	default:
-		dev_warn(&h->pdev->dev, "cmd %p returned "
-		       "unknown status %x\n", cmd,
-		       cmd->err_info->CommandStatus);
-		sreq->result = make_status_bytes(SAM_STAT_GOOD,
-			cmd->err_info->CommandStatus, DRIVER_OK,
-			blk_rq_is_passthrough(cmd->rq) ?
-				DID_PASSTHROUGH : DID_ERROR);
-	}
-
-after_error_processing:
-
-	/* We need to return this command */
-	if (retry_cmd) {
-		resend_cciss_cmd(h, cmd);
-		return;
-	}
-	cmd->rq->completion_data = cmd;
-	blk_complete_request(cmd->rq);
-}
-
-static inline u32 cciss_tag_contains_index(u32 tag)
-{
-#define DIRECT_LOOKUP_BIT 0x10
-	return tag & DIRECT_LOOKUP_BIT;
-}
-
-static inline u32 cciss_tag_to_index(u32 tag)
-{
-#define DIRECT_LOOKUP_SHIFT 5
-	return tag >> DIRECT_LOOKUP_SHIFT;
-}
-
-static inline u32 cciss_tag_discard_error_bits(ctlr_info_t *h, u32 tag)
-{
-#define CCISS_PERF_ERROR_BITS ((1 << DIRECT_LOOKUP_SHIFT) - 1)
-#define CCISS_SIMPLE_ERROR_BITS 0x03
-	if (likely(h->transMethod & CFGTBL_Trans_Performant))
-		return tag & ~CCISS_PERF_ERROR_BITS;
-	return tag & ~CCISS_SIMPLE_ERROR_BITS;
-}
-
-static inline void cciss_mark_tag_indexed(u32 *tag)
-{
-	*tag |= DIRECT_LOOKUP_BIT;
-}
-
-static inline void cciss_set_tag_index(u32 *tag, u32 index)
-{
-	*tag |= (index << DIRECT_LOOKUP_SHIFT);
-}
-
-/*
- * Get a request and submit it to the controller.
- */
-static void do_cciss_request(struct request_queue *q)
-{
-	ctlr_info_t *h = q->queuedata;
-	CommandList_struct *c;
-	sector_t start_blk;
-	int seg;
-	struct request *creq;
-	u64bit temp64;
-	struct scatterlist *tmp_sg;
-	SGDescriptor_struct *curr_sg;
-	drive_info_struct *drv;
-	int i, dir;
-	int sg_index = 0;
-	int chained = 0;
-
-      queue:
-	creq = blk_peek_request(q);
-	if (!creq)
-		goto startio;
-
-	BUG_ON(creq->nr_phys_segments > h->maxsgentries);
-
-	c = cmd_alloc(h);
-	if (!c)
-		goto full;
-
-	blk_start_request(creq);
-
-	tmp_sg = h->scatter_list[c->cmdindex];
-	spin_unlock_irq(q->queue_lock);
-
-	c->cmd_type = CMD_RWREQ;
-	c->rq = creq;
-
-	/* fill in the request */
-	drv = creq->rq_disk->private_data;
-	c->Header.ReplyQueue = 0;	/* unused in simple mode */
-	/* got command from pool, so use the command block index instead */
-	/* for direct lookups. */
-	/* The first 2 bits are reserved for controller error reporting. */
-	cciss_set_tag_index(&c->Header.Tag.lower, c->cmdindex);
-	cciss_mark_tag_indexed(&c->Header.Tag.lower);
-	memcpy(&c->Header.LUN, drv->LunID, sizeof(drv->LunID));
-	c->Request.CDBLen = 10;	/* 12 byte commands not in FW yet; */
-	c->Request.Type.Type = TYPE_CMD;	/* It is a command. */
-	c->Request.Type.Attribute = ATTR_SIMPLE;
-	c->Request.Type.Direction =
-	    (rq_data_dir(creq) == READ) ? XFER_READ : XFER_WRITE;
-	c->Request.Timeout = 0;	/* Don't time out */
-	c->Request.CDB[0] =
-	    (rq_data_dir(creq) == READ) ? h->cciss_read : h->cciss_write;
-	start_blk = blk_rq_pos(creq);
-	dev_dbg(&h->pdev->dev, "sector =%d nr_sectors=%d\n",
-	       (int)blk_rq_pos(creq), (int)blk_rq_sectors(creq));
-	sg_init_table(tmp_sg, h->maxsgentries);
-	seg = blk_rq_map_sg(q, creq, tmp_sg);
-
-	/* get the DMA records for the setup */
-	if (c->Request.Type.Direction == XFER_READ)
-		dir = PCI_DMA_FROMDEVICE;
-	else
-		dir = PCI_DMA_TODEVICE;
-
-	curr_sg = c->SG;
-	sg_index = 0;
-	chained = 0;
-
-	for (i = 0; i < seg; i++) {
-		if (((sg_index+1) == (h->max_cmd_sgentries)) &&
-			!chained && ((seg - i) > 1)) {
-			/* Point to next chain block. */
-			curr_sg = h->cmd_sg_list[c->cmdindex];
-			sg_index = 0;
-			chained = 1;
-		}
-		curr_sg[sg_index].Len = tmp_sg[i].length;
-		temp64.val = (__u64) pci_map_page(h->pdev, sg_page(&tmp_sg[i]),
-						tmp_sg[i].offset,
-						tmp_sg[i].length, dir);
-		if (dma_mapping_error(&h->pdev->dev, temp64.val)) {
-			dev_warn(&h->pdev->dev,
-				"%s: error mapping page for DMA\n", __func__);
-			scsi_req(creq)->result =
-				make_status_bytes(SAM_STAT_GOOD, 0, DRIVER_OK,
-						  DID_SOFT_ERROR);
-			cmd_free(h, c);
-			return;
-		}
-		curr_sg[sg_index].Addr.lower = temp64.val32.lower;
-		curr_sg[sg_index].Addr.upper = temp64.val32.upper;
-		curr_sg[sg_index].Ext = 0;  /* we are not chaining */
-		++sg_index;
-	}
-	if (chained) {
-		if (cciss_map_sg_chain_block(h, c, h->cmd_sg_list[c->cmdindex],
-			(seg - (h->max_cmd_sgentries - 1)) *
-				sizeof(SGDescriptor_struct))) {
-			scsi_req(creq)->result =
-				make_status_bytes(SAM_STAT_GOOD, 0, DRIVER_OK,
-						  DID_SOFT_ERROR);
-			cmd_free(h, c);
-			return;
-		}
-	}
-
-	/* track how many SG entries we are using */
-	if (seg > h->maxSG)
-		h->maxSG = seg;
-
-	dev_dbg(&h->pdev->dev, "Submitting %u sectors in %d segments "
-			"chained[%d]\n",
-			blk_rq_sectors(creq), seg, chained);
-
-	c->Header.SGTotal = seg + chained;
-	if (seg <= h->max_cmd_sgentries)
-		c->Header.SGList = c->Header.SGTotal;
-	else
-		c->Header.SGList = h->max_cmd_sgentries;
-	set_performant_mode(h, c);
-
-	switch (req_op(creq)) {
-	case REQ_OP_READ:
-	case REQ_OP_WRITE:
-		if(h->cciss_read == CCISS_READ_10) {
-			c->Request.CDB[1] = 0;
-			c->Request.CDB[2] = (start_blk >> 24) & 0xff; /* MSB */
-			c->Request.CDB[3] = (start_blk >> 16) & 0xff;
-			c->Request.CDB[4] = (start_blk >> 8) & 0xff;
-			c->Request.CDB[5] = start_blk & 0xff;
-			c->Request.CDB[6] = 0; /* (sect >> 24) & 0xff; MSB */
-			c->Request.CDB[7] = (blk_rq_sectors(creq) >> 8) & 0xff;
-			c->Request.CDB[8] = blk_rq_sectors(creq) & 0xff;
-			c->Request.CDB[9] = c->Request.CDB[11] = c->Request.CDB[12] = 0;
-		} else {
-			u32 upper32 = upper_32_bits(start_blk);
-
-			c->Request.CDBLen = 16;
-			c->Request.CDB[1]= 0;
-			c->Request.CDB[2]= (upper32 >> 24) & 0xff; /* MSB */
-			c->Request.CDB[3]= (upper32 >> 16) & 0xff;
-			c->Request.CDB[4]= (upper32 >>  8) & 0xff;
-			c->Request.CDB[5]= upper32 & 0xff;
-			c->Request.CDB[6]= (start_blk >> 24) & 0xff;
-			c->Request.CDB[7]= (start_blk >> 16) & 0xff;
-			c->Request.CDB[8]= (start_blk >>  8) & 0xff;
-			c->Request.CDB[9]= start_blk & 0xff;
-			c->Request.CDB[10]= (blk_rq_sectors(creq) >> 24) & 0xff;
-			c->Request.CDB[11]= (blk_rq_sectors(creq) >> 16) & 0xff;
-			c->Request.CDB[12]= (blk_rq_sectors(creq) >>  8) & 0xff;
-			c->Request.CDB[13]= blk_rq_sectors(creq) & 0xff;
-			c->Request.CDB[14] = c->Request.CDB[15] = 0;
-		}
-		break;
-	case REQ_OP_SCSI_IN:
-	case REQ_OP_SCSI_OUT:
-		c->Request.CDBLen = scsi_req(creq)->cmd_len;
-		memcpy(c->Request.CDB, scsi_req(creq)->cmd, BLK_MAX_CDB);
-		scsi_req(creq)->sense = c->err_info->SenseInfo;
-		break;
-	default:
-		dev_warn(&h->pdev->dev, "bad request type %d\n",
-			creq->cmd_flags);
-		BUG();
-	}
-
-	spin_lock_irq(q->queue_lock);
-
-	addQ(&h->reqQ, c);
-	h->Qdepth++;
-	if (h->Qdepth > h->maxQsinceinit)
-		h->maxQsinceinit = h->Qdepth;
-
-	goto queue;
-full:
-	blk_stop_queue(q);
-startio:
-	/* We will already have the driver lock here so not need
-	 * to lock it.
-	 */
-	start_io(h);
-}
-
-static inline unsigned long get_next_completion(ctlr_info_t *h)
-{
-	return h->access.command_completed(h);
-}
-
-static inline int interrupt_pending(ctlr_info_t *h)
-{
-	return h->access.intr_pending(h);
-}
-
-static inline long interrupt_not_for_us(ctlr_info_t *h)
-{
-	return ((h->access.intr_pending(h) == 0) ||
-		(h->interrupts_enabled == 0));
-}
-
-static inline int bad_tag(ctlr_info_t *h, u32 tag_index,
-			u32 raw_tag)
-{
-	if (unlikely(tag_index >= h->nr_cmds)) {
-		dev_warn(&h->pdev->dev, "bad tag 0x%08x ignored.\n", raw_tag);
-		return 1;
-	}
-	return 0;
-}
-
-static inline void finish_cmd(ctlr_info_t *h, CommandList_struct *c,
-				u32 raw_tag)
-{
-	removeQ(c);
-	if (likely(c->cmd_type == CMD_RWREQ))
-		complete_command(h, c, 0);
-	else if (c->cmd_type == CMD_IOCTL_PEND)
-		complete(c->waiting);
-#ifdef CONFIG_CISS_SCSI_TAPE
-	else if (c->cmd_type == CMD_SCSI)
-		complete_scsi_command(c, 0, raw_tag);
-#endif
-}
-
-static inline u32 next_command(ctlr_info_t *h)
-{
-	u32 a;
-
-	if (unlikely(!(h->transMethod & CFGTBL_Trans_Performant)))
-		return h->access.command_completed(h);
-
-	if ((*(h->reply_pool_head) & 1) == (h->reply_pool_wraparound)) {
-		a = *(h->reply_pool_head); /* Next cmd in ring buffer */
-		(h->reply_pool_head)++;
-		h->commands_outstanding--;
-	} else {
-		a = FIFO_EMPTY;
-	}
-	/* Check for wraparound */
-	if (h->reply_pool_head == (h->reply_pool + h->max_commands)) {
-		h->reply_pool_head = h->reply_pool;
-		h->reply_pool_wraparound ^= 1;
-	}
-	return a;
-}
-
-/* process completion of an indexed ("direct lookup") command */
-static inline u32 process_indexed_cmd(ctlr_info_t *h, u32 raw_tag)
-{
-	u32 tag_index;
-	CommandList_struct *c;
-
-	tag_index = cciss_tag_to_index(raw_tag);
-	if (bad_tag(h, tag_index, raw_tag))
-		return next_command(h);
-	c = h->cmd_pool + tag_index;
-	finish_cmd(h, c, raw_tag);
-	return next_command(h);
-}
-
-/* process completion of a non-indexed command */
-static inline u32 process_nonindexed_cmd(ctlr_info_t *h, u32 raw_tag)
-{
-	CommandList_struct *c = NULL;
-	__u32 busaddr_masked, tag_masked;
-
-	tag_masked = cciss_tag_discard_error_bits(h, raw_tag);
-	list_for_each_entry(c, &h->cmpQ, list) {
-		busaddr_masked = cciss_tag_discard_error_bits(h, c->busaddr);
-		if (busaddr_masked == tag_masked) {
-			finish_cmd(h, c, raw_tag);
-			return next_command(h);
-		}
-	}
-	bad_tag(h, h->nr_cmds + 1, raw_tag);
-	return next_command(h);
-}
-
-/* Some controllers, like p400, will give us one interrupt
- * after a soft reset, even if we turned interrupts off.
- * Only need to check for this in the cciss_xxx_discard_completions
- * functions.
- */
-static int ignore_bogus_interrupt(ctlr_info_t *h)
-{
-	if (likely(!reset_devices))
-		return 0;
-
-	if (likely(h->interrupts_enabled))
-		return 0;
-
-	dev_info(&h->pdev->dev, "Received interrupt while interrupts disabled "
-		"(known firmware bug.)  Ignoring.\n");
-
-	return 1;
-}
-
-static irqreturn_t cciss_intx_discard_completions(int irq, void *dev_id)
-{
-	ctlr_info_t *h = dev_id;
-	unsigned long flags;
-	u32 raw_tag;
-
-	if (ignore_bogus_interrupt(h))
-		return IRQ_NONE;
-
-	if (interrupt_not_for_us(h))
-		return IRQ_NONE;
-	spin_lock_irqsave(&h->lock, flags);
-	while (interrupt_pending(h)) {
-		raw_tag = get_next_completion(h);
-		while (raw_tag != FIFO_EMPTY)
-			raw_tag = next_command(h);
-	}
-	spin_unlock_irqrestore(&h->lock, flags);
-	return IRQ_HANDLED;
-}
-
-static irqreturn_t cciss_msix_discard_completions(int irq, void *dev_id)
-{
-	ctlr_info_t *h = dev_id;
-	unsigned long flags;
-	u32 raw_tag;
-
-	if (ignore_bogus_interrupt(h))
-		return IRQ_NONE;
-
-	spin_lock_irqsave(&h->lock, flags);
-	raw_tag = get_next_completion(h);
-	while (raw_tag != FIFO_EMPTY)
-		raw_tag = next_command(h);
-	spin_unlock_irqrestore(&h->lock, flags);
-	return IRQ_HANDLED;
-}
-
-static irqreturn_t do_cciss_intx(int irq, void *dev_id)
-{
-	ctlr_info_t *h = dev_id;
-	unsigned long flags;
-	u32 raw_tag;
-
-	if (interrupt_not_for_us(h))
-		return IRQ_NONE;
-	spin_lock_irqsave(&h->lock, flags);
-	while (interrupt_pending(h)) {
-		raw_tag = get_next_completion(h);
-		while (raw_tag != FIFO_EMPTY) {
-			if (cciss_tag_contains_index(raw_tag))
-				raw_tag = process_indexed_cmd(h, raw_tag);
-			else
-				raw_tag = process_nonindexed_cmd(h, raw_tag);
-		}
-	}
-	spin_unlock_irqrestore(&h->lock, flags);
-	return IRQ_HANDLED;
-}
-
-/* Add a second interrupt handler for MSI/MSI-X mode. In this mode we never
- * check the interrupt pending register because it is not set.
- */
-static irqreturn_t do_cciss_msix_intr(int irq, void *dev_id)
-{
-	ctlr_info_t *h = dev_id;
-	unsigned long flags;
-	u32 raw_tag;
-
-	spin_lock_irqsave(&h->lock, flags);
-	raw_tag = get_next_completion(h);
-	while (raw_tag != FIFO_EMPTY) {
-		if (cciss_tag_contains_index(raw_tag))
-			raw_tag = process_indexed_cmd(h, raw_tag);
-		else
-			raw_tag = process_nonindexed_cmd(h, raw_tag);
-	}
-	spin_unlock_irqrestore(&h->lock, flags);
-	return IRQ_HANDLED;
-}
-
-/**
- * add_to_scan_list() - add controller to rescan queue
- * @h:		      Pointer to the controller.
- *
- * Adds the controller to the rescan queue if not already on the queue.
- *
- * returns 1 if added to the queue, 0 if skipped (could be on the
- * queue already, or the controller could be initializing or shutting
- * down).
- **/
-static int add_to_scan_list(struct ctlr_info *h)
-{
-	struct ctlr_info *test_h;
-	int found = 0;
-	int ret = 0;
-
-	if (h->busy_initializing)
-		return 0;
-
-	if (!mutex_trylock(&h->busy_shutting_down))
-		return 0;
-
-	mutex_lock(&scan_mutex);
-	list_for_each_entry(test_h, &scan_q, scan_list) {
-		if (test_h == h) {
-			found = 1;
-			break;
-		}
-	}
-	if (!found && !h->busy_scanning) {
-		reinit_completion(&h->scan_wait);
-		list_add_tail(&h->scan_list, &scan_q);
-		ret = 1;
-	}
-	mutex_unlock(&scan_mutex);
-	mutex_unlock(&h->busy_shutting_down);
-
-	return ret;
-}
-
-/**
- * remove_from_scan_list() - remove controller from rescan queue
- * @h:			   Pointer to the controller.
- *
- * Removes the controller from the rescan queue if present. Blocks if
- * the controller is currently conducting a rescan.  The controller
- * can be in one of three states:
- * 1. Doesn't need a scan
- * 2. On the scan list, but not scanning yet (we remove it)
- * 3. Busy scanning (and not on the list). In this case we want to wait for
- *    the scan to complete to make sure the scanning thread for this
- *    controller is completely idle.
- **/
-static void remove_from_scan_list(struct ctlr_info *h)
-{
-	struct ctlr_info *test_h, *tmp_h;
-
-	mutex_lock(&scan_mutex);
-	list_for_each_entry_safe(test_h, tmp_h, &scan_q, scan_list) {
-		if (test_h == h) { /* state 2. */
-			list_del(&h->scan_list);
-			complete_all(&h->scan_wait);
-			mutex_unlock(&scan_mutex);
-			return;
-		}
-	}
-	if (h->busy_scanning) { /* state 3. */
-		mutex_unlock(&scan_mutex);
-		wait_for_completion(&h->scan_wait);
-	} else { /* state 1, nothing to do. */
-		mutex_unlock(&scan_mutex);
-	}
-}
-
-/**
- * scan_thread() - kernel thread used to rescan controllers
- * @data:	 Ignored.
- *
- * A kernel thread used scan for drive topology changes on
- * controllers. The thread processes only one controller at a time
- * using a queue.  Controllers are added to the queue using
- * add_to_scan_list() and removed from the queue either after done
- * processing or using remove_from_scan_list().
- *
- * returns 0.
- **/
-static int scan_thread(void *data)
-{
-	struct ctlr_info *h;
-
-	while (1) {
-		set_current_state(TASK_INTERRUPTIBLE);
-		schedule();
-		if (kthread_should_stop())
-			break;
-
-		while (1) {
-			mutex_lock(&scan_mutex);
-			if (list_empty(&scan_q)) {
-				mutex_unlock(&scan_mutex);
-				break;
-			}
-
-			h = list_entry(scan_q.next,
-				       struct ctlr_info,
-				       scan_list);
-			list_del(&h->scan_list);
-			h->busy_scanning = 1;
-			mutex_unlock(&scan_mutex);
-
-			rebuild_lun_table(h, 0, 0);
-			complete_all(&h->scan_wait);
-			mutex_lock(&scan_mutex);
-			h->busy_scanning = 0;
-			mutex_unlock(&scan_mutex);
-		}
-	}
-
-	return 0;
-}
-
-static int check_for_unit_attention(ctlr_info_t *h, CommandList_struct *c)
-{
-	if (c->err_info->SenseInfo[2] != UNIT_ATTENTION)
-		return 0;
-
-	switch (c->err_info->SenseInfo[12]) {
-	case STATE_CHANGED:
-		dev_warn(&h->pdev->dev, "a state change "
-			"detected, command retried\n");
-		return 1;
-	break;
-	case LUN_FAILED:
-		dev_warn(&h->pdev->dev, "LUN failure "
-			"detected, action required\n");
-		return 1;
-	break;
-	case REPORT_LUNS_CHANGED:
-		dev_warn(&h->pdev->dev, "report LUN data changed\n");
-	/*
-	 * Here, we could call add_to_scan_list and wake up the scan thread,
-	 * except that it's quite likely that we will get more than one
-	 * REPORT_LUNS_CHANGED condition in quick succession, which means
-	 * that those which occur after the first one will likely happen
-	 * *during* the scan_thread's rescan.  And the rescan code is not
-	 * robust enough to restart in the middle, undoing what it has already
-	 * done, and it's not clear that it's even possible to do this, since
-	 * part of what it does is notify the block layer, which starts
-	 * doing it's own i/o to read partition tables and so on, and the
-	 * driver doesn't have visibility to know what might need undoing.
-	 * In any event, if possible, it is horribly complicated to get right
-	 * so we just don't do it for now.
-	 *
-	 * Note: this REPORT_LUNS_CHANGED condition only occurs on the MSA2012.
-	 */
-		return 1;
-	break;
-	case POWER_OR_RESET:
-		dev_warn(&h->pdev->dev,
-			"a power on or device reset detected\n");
-		return 1;
-	break;
-	case UNIT_ATTENTION_CLEARED:
-		dev_warn(&h->pdev->dev,
-			"unit attention cleared by another initiator\n");
-		return 1;
-	break;
-	default:
-		dev_warn(&h->pdev->dev, "unknown unit attention detected\n");
-		return 1;
-	}
-}
-
-/*
- *  We cannot read the structure directly, for portability we must use
- *   the io functions.
- *   This is for debug only.
- */
-static void print_cfg_table(ctlr_info_t *h)
-{
-	int i;
-	char temp_name[17];
-	CfgTable_struct *tb = h->cfgtable;
-
-	dev_dbg(&h->pdev->dev, "Controller Configuration information\n");
-	dev_dbg(&h->pdev->dev, "------------------------------------\n");
-	for (i = 0; i < 4; i++)
-		temp_name[i] = readb(&(tb->Signature[i]));
-	temp_name[4] = '\0';
-	dev_dbg(&h->pdev->dev, "   Signature = %s\n", temp_name);
-	dev_dbg(&h->pdev->dev, "   Spec Number = %d\n",
-		readl(&(tb->SpecValence)));
-	dev_dbg(&h->pdev->dev, "   Transport methods supported = 0x%x\n",
-	       readl(&(tb->TransportSupport)));
-	dev_dbg(&h->pdev->dev, "   Transport methods active = 0x%x\n",
-	       readl(&(tb->TransportActive)));
-	dev_dbg(&h->pdev->dev, "   Requested transport Method = 0x%x\n",
-	       readl(&(tb->HostWrite.TransportRequest)));
-	dev_dbg(&h->pdev->dev, "   Coalesce Interrupt Delay = 0x%x\n",
-	       readl(&(tb->HostWrite.CoalIntDelay)));
-	dev_dbg(&h->pdev->dev, "   Coalesce Interrupt Count = 0x%x\n",
-	       readl(&(tb->HostWrite.CoalIntCount)));
-	dev_dbg(&h->pdev->dev, "   Max outstanding commands = 0x%x\n",
-	       readl(&(tb->CmdsOutMax)));
-	dev_dbg(&h->pdev->dev, "   Bus Types = 0x%x\n",
-		readl(&(tb->BusTypes)));
-	for (i = 0; i < 16; i++)
-		temp_name[i] = readb(&(tb->ServerName[i]));
-	temp_name[16] = '\0';
-	dev_dbg(&h->pdev->dev, "   Server Name = %s\n", temp_name);
-	dev_dbg(&h->pdev->dev, "   Heartbeat Counter = 0x%x\n\n\n",
-		readl(&(tb->HeartBeat)));
-}
-
-static int find_PCI_BAR_index(struct pci_dev *pdev, unsigned long pci_bar_addr)
-{
-	int i, offset, mem_type, bar_type;
-	if (pci_bar_addr == PCI_BASE_ADDRESS_0)	/* looking for BAR zero? */
-		return 0;
-	offset = 0;
-	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
-		bar_type = pci_resource_flags(pdev, i) & PCI_BASE_ADDRESS_SPACE;
-		if (bar_type == PCI_BASE_ADDRESS_SPACE_IO)
-			offset += 4;
-		else {
-			mem_type = pci_resource_flags(pdev, i) &
-			    PCI_BASE_ADDRESS_MEM_TYPE_MASK;
-			switch (mem_type) {
-			case PCI_BASE_ADDRESS_MEM_TYPE_32:
-			case PCI_BASE_ADDRESS_MEM_TYPE_1M:
-				offset += 4;	/* 32 bit */
-				break;
-			case PCI_BASE_ADDRESS_MEM_TYPE_64:
-				offset += 8;
-				break;
-			default:	/* reserved in PCI 2.2 */
-				dev_warn(&pdev->dev,
-				       "Base address is invalid\n");
-				return -1;
-				break;
-			}
-		}
-		if (offset == pci_bar_addr - PCI_BASE_ADDRESS_0)
-			return i + 1;
-	}
-	return -1;
-}
-
-/* Fill in bucket_map[], given nsgs (the max number of
- * scatter gather elements supported) and bucket[],
- * which is an array of 8 integers.  The bucket[] array
- * contains 8 different DMA transfer sizes (in 16
- * byte increments) which the controller uses to fetch
- * commands.  This function fills in bucket_map[], which
- * maps a given number of scatter gather elements to one of
- * the 8 DMA transfer sizes.  The point of it is to allow the
- * controller to only do as much DMA as needed to fetch the
- * command, with the DMA transfer size encoded in the lower
- * bits of the command address.
- */
-static void  calc_bucket_map(int bucket[], int num_buckets,
-	int nsgs, int *bucket_map)
-{
-	int i, j, b, size;
-
-	/* even a command with 0 SGs requires 4 blocks */
-#define MINIMUM_TRANSFER_BLOCKS 4
-#define NUM_BUCKETS 8
-	/* Note, bucket_map must have nsgs+1 entries. */
-	for (i = 0; i <= nsgs; i++) {
-		/* Compute size of a command with i SG entries */
-		size = i + MINIMUM_TRANSFER_BLOCKS;
-		b = num_buckets; /* Assume the biggest bucket */
-		/* Find the bucket that is just big enough */
-		for (j = 0; j < 8; j++) {
-			if (bucket[j] >= size) {
-				b = j;
-				break;
-			}
-		}
-		/* for a command with i SG entries, use bucket b. */
-		bucket_map[i] = b;
-	}
-}
-
-static void cciss_wait_for_mode_change_ack(ctlr_info_t *h)
-{
-	int i;
-
-	/* under certain very rare conditions, this can take awhile.
-	 * (e.g.: hot replace a failed 144GB drive in a RAID 5 set right
-	 * as we enter this code.) */
-	for (i = 0; i < MAX_CONFIG_WAIT; i++) {
-		if (!(readl(h->vaddr + SA5_DOORBELL) & CFGTBL_ChangeReq))
-			break;
-		usleep_range(10000, 20000);
-	}
-}
-
-static void cciss_enter_performant_mode(ctlr_info_t *h, u32 use_short_tags)
-{
-	/* This is a bit complicated.  There are 8 registers on
-	 * the controller which we write to to tell it 8 different
-	 * sizes of commands which there may be.  It's a way of
-	 * reducing the DMA done to fetch each command.  Encoded into
-	 * each command's tag are 3 bits which communicate to the controller
-	 * which of the eight sizes that command fits within.  The size of
-	 * each command depends on how many scatter gather entries there are.
-	 * Each SG entry requires 16 bytes.  The eight registers are programmed
-	 * with the number of 16-byte blocks a command of that size requires.
-	 * The smallest command possible requires 5 such 16 byte blocks.
-	 * the largest command possible requires MAXSGENTRIES + 4 16-byte
-	 * blocks.  Note, this only extends to the SG entries contained
-	 * within the command block, and does not extend to chained blocks
-	 * of SG elements.   bft[] contains the eight values we write to
-	 * the registers.  They are not evenly distributed, but have more
-	 * sizes for small commands, and fewer sizes for larger commands.
-	 */
-	__u32 trans_offset;
-	int bft[8] = { 5, 6, 8, 10, 12, 20, 28, MAXSGENTRIES + 4};
-			/*
-			 *  5 = 1 s/g entry or 4k
-			 *  6 = 2 s/g entry or 8k
-			 *  8 = 4 s/g entry or 16k
-			 * 10 = 6 s/g entry or 24k
-			 */
-	unsigned long register_value;
-	BUILD_BUG_ON(28 > MAXSGENTRIES + 4);
-
-	h->reply_pool_wraparound = 1; /* spec: init to 1 */
-
-	/* Controller spec: zero out this buffer. */
-	memset(h->reply_pool, 0, h->max_commands * sizeof(__u64));
-	h->reply_pool_head = h->reply_pool;
-
-	trans_offset = readl(&(h->cfgtable->TransMethodOffset));
-	calc_bucket_map(bft, ARRAY_SIZE(bft), h->maxsgentries,
-				h->blockFetchTable);
-	writel(bft[0], &h->transtable->BlockFetch0);
-	writel(bft[1], &h->transtable->BlockFetch1);
-	writel(bft[2], &h->transtable->BlockFetch2);
-	writel(bft[3], &h->transtable->BlockFetch3);
-	writel(bft[4], &h->transtable->BlockFetch4);
-	writel(bft[5], &h->transtable->BlockFetch5);
-	writel(bft[6], &h->transtable->BlockFetch6);
-	writel(bft[7], &h->transtable->BlockFetch7);
-
-	/* size of controller ring buffer */
-	writel(h->max_commands, &h->transtable->RepQSize);
-	writel(1, &h->transtable->RepQCount);
-	writel(0, &h->transtable->RepQCtrAddrLow32);
-	writel(0, &h->transtable->RepQCtrAddrHigh32);
-	writel(h->reply_pool_dhandle, &h->transtable->RepQAddr0Low32);
-	writel(0, &h->transtable->RepQAddr0High32);
-	writel(CFGTBL_Trans_Performant | use_short_tags,
-			&(h->cfgtable->HostWrite.TransportRequest));
-
-	writel(CFGTBL_ChangeReq, h->vaddr + SA5_DOORBELL);
-	cciss_wait_for_mode_change_ack(h);
-	register_value = readl(&(h->cfgtable->TransportActive));
-	if (!(register_value & CFGTBL_Trans_Performant))
-		dev_warn(&h->pdev->dev, "cciss: unable to get board into"
-					" performant mode\n");
-}
-
-static void cciss_put_controller_into_performant_mode(ctlr_info_t *h)
-{
-	__u32 trans_support;
-
-	if (cciss_simple_mode)
-		return;
-
-	dev_dbg(&h->pdev->dev, "Trying to put board into Performant mode\n");
-	/* Attempt to put controller into performant mode if supported */
-	/* Does board support performant mode? */
-	trans_support = readl(&(h->cfgtable->TransportSupport));
-	if (!(trans_support & PERFORMANT_MODE))
-		return;
-
-	dev_dbg(&h->pdev->dev, "Placing controller into performant mode\n");
-	/* Performant mode demands commands on a 32 byte boundary
-	 * pci_alloc_consistent aligns on page boundarys already.
-	 * Just need to check if divisible by 32
-	 */
-	if ((sizeof(CommandList_struct) % 32) != 0) {
-		dev_warn(&h->pdev->dev, "%s %d %s\n",
-			"cciss info: command size[",
-			(int)sizeof(CommandList_struct),
-			"] not divisible by 32, no performant mode..\n");
-		return;
-	}
-
-	/* Performant mode ring buffer and supporting data structures */
-	h->reply_pool = (__u64 *)pci_alloc_consistent(
-		h->pdev, h->max_commands * sizeof(__u64),
-		&(h->reply_pool_dhandle));
-
-	/* Need a block fetch table for performant mode */
-	h->blockFetchTable = kmalloc(((h->maxsgentries+1) *
-		sizeof(__u32)), GFP_KERNEL);
-
-	if ((h->reply_pool == NULL) || (h->blockFetchTable == NULL))
-		goto clean_up;
-
-	cciss_enter_performant_mode(h,
-		trans_support & CFGTBL_Trans_use_short_tags);
-
-	/* Change the access methods to the performant access methods */
-	h->access = SA5_performant_access;
-	h->transMethod = CFGTBL_Trans_Performant;
-
-	return;
-clean_up:
-	kfree(h->blockFetchTable);
-	if (h->reply_pool)
-		pci_free_consistent(h->pdev,
-				h->max_commands * sizeof(__u64),
-				h->reply_pool,
-				h->reply_pool_dhandle);
-	return;
-
-} /* cciss_put_controller_into_performant_mode */
-
-/* If MSI/MSI-X is supported by the kernel we will try to enable it on
- * controllers that are capable. If not, we use IO-APIC mode.
- */
-
-static void cciss_interrupt_mode(ctlr_info_t *h)
-{
-	int ret;
-
-	/* Some boards advertise MSI but don't really support it */
-	if ((h->board_id == 0x40700E11) || (h->board_id == 0x40800E11) ||
-	    (h->board_id == 0x40820E11) || (h->board_id == 0x40830E11))
-		goto default_int_mode;
-
-	ret = pci_alloc_irq_vectors(h->pdev, 4, 4, PCI_IRQ_MSIX);
-	if (ret >= 0)   {
-		h->intr[0] = pci_irq_vector(h->pdev, 0);
-		h->intr[1] = pci_irq_vector(h->pdev, 1);
-		h->intr[2] = pci_irq_vector(h->pdev, 2);
-		h->intr[3] = pci_irq_vector(h->pdev, 3);
-		return;
-	}
-
-	ret = pci_alloc_irq_vectors(h->pdev, 1, 1, PCI_IRQ_MSI);
-
-default_int_mode:
-	/* if we get here we're going to use the default interrupt mode */
-	h->intr[h->intr_mode] = pci_irq_vector(h->pdev, 0);
-	return;
-}
-
-static int cciss_lookup_board_id(struct pci_dev *pdev, u32 *board_id)
-{
-	int i;
-	u32 subsystem_vendor_id, subsystem_device_id;
-
-	subsystem_vendor_id = pdev->subsystem_vendor;
-	subsystem_device_id = pdev->subsystem_device;
-	*board_id = ((subsystem_device_id << 16) & 0xffff0000) |
-			subsystem_vendor_id;
-
-	for (i = 0; i < ARRAY_SIZE(products); i++) {
-		/* Stand aside for hpsa driver on request */
-		if (cciss_allow_hpsa)
-			return -ENODEV;
-		if (*board_id == products[i].board_id)
-			return i;
-	}
-	dev_warn(&pdev->dev, "unrecognized board ID: 0x%08x, ignoring.\n",
-		*board_id);
-	return -ENODEV;
-}
-
-static inline bool cciss_board_disabled(ctlr_info_t *h)
-{
-	u16 command;
-
-	(void) pci_read_config_word(h->pdev, PCI_COMMAND, &command);
-	return ((command & PCI_COMMAND_MEMORY) == 0);
-}
-
-static int cciss_pci_find_memory_BAR(struct pci_dev *pdev,
-				     unsigned long *memory_bar)
-{
-	int i;
-
-	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++)
-		if (pci_resource_flags(pdev, i) & IORESOURCE_MEM) {
-			/* addressing mode bits already removed */
-			*memory_bar = pci_resource_start(pdev, i);
-			dev_dbg(&pdev->dev, "memory BAR = %lx\n",
-				*memory_bar);
-			return 0;
-		}
-	dev_warn(&pdev->dev, "no memory BAR found\n");
-	return -ENODEV;
-}
-
-static int cciss_wait_for_board_state(struct pci_dev *pdev,
-				      void __iomem *vaddr, int wait_for_ready)
-#define BOARD_READY 1
-#define BOARD_NOT_READY 0
-{
-	int i, iterations;
-	u32 scratchpad;
-
-	if (wait_for_ready)
-		iterations = CCISS_BOARD_READY_ITERATIONS;
-	else
-		iterations = CCISS_BOARD_NOT_READY_ITERATIONS;
-
-	for (i = 0; i < iterations; i++) {
-		scratchpad = readl(vaddr + SA5_SCRATCHPAD_OFFSET);
-		if (wait_for_ready) {
-			if (scratchpad == CCISS_FIRMWARE_READY)
-				return 0;
-		} else {
-			if (scratchpad != CCISS_FIRMWARE_READY)
-				return 0;
-		}
-		msleep(CCISS_BOARD_READY_POLL_INTERVAL_MSECS);
-	}
-	dev_warn(&pdev->dev, "board not ready, timed out.\n");
-	return -ENODEV;
-}
-
-static int cciss_find_cfg_addrs(struct pci_dev *pdev, void __iomem *vaddr,
-				u32 *cfg_base_addr, u64 *cfg_base_addr_index,
-				u64 *cfg_offset)
-{
-	*cfg_base_addr = readl(vaddr + SA5_CTCFG_OFFSET);
-	*cfg_offset = readl(vaddr + SA5_CTMEM_OFFSET);
-	*cfg_base_addr &= (u32) 0x0000ffff;
-	*cfg_base_addr_index = find_PCI_BAR_index(pdev, *cfg_base_addr);
-	if (*cfg_base_addr_index == -1) {
-		dev_warn(&pdev->dev, "cannot find cfg_base_addr_index, "
-			"*cfg_base_addr = 0x%08x\n", *cfg_base_addr);
-		return -ENODEV;
-	}
-	return 0;
-}
-
-static int cciss_find_cfgtables(ctlr_info_t *h)
-{
-	u64 cfg_offset;
-	u32 cfg_base_addr;
-	u64 cfg_base_addr_index;
-	u32 trans_offset;
-	int rc;
-
-	rc = cciss_find_cfg_addrs(h->pdev, h->vaddr, &cfg_base_addr,
-		&cfg_base_addr_index, &cfg_offset);
-	if (rc)
-		return rc;
-	h->cfgtable = remap_pci_mem(pci_resource_start(h->pdev,
-		cfg_base_addr_index) + cfg_offset, sizeof(*h->cfgtable));
-	if (!h->cfgtable)
-		return -ENOMEM;
-	rc = write_driver_ver_to_cfgtable(h->cfgtable);
-	if (rc)
-		return rc;
-	/* Find performant mode table. */
-	trans_offset = readl(&h->cfgtable->TransMethodOffset);
-	h->transtable = remap_pci_mem(pci_resource_start(h->pdev,
-				cfg_base_addr_index)+cfg_offset+trans_offset,
-				sizeof(*h->transtable));
-	if (!h->transtable)
-		return -ENOMEM;
-	return 0;
-}
-
-static void cciss_get_max_perf_mode_cmds(struct ctlr_info *h)
-{
-	h->max_commands = readl(&(h->cfgtable->MaxPerformantModeCommands));
-
-	/* Limit commands in memory limited kdump scenario. */
-	if (reset_devices && h->max_commands > 32)
-		h->max_commands = 32;
-
-	if (h->max_commands < 16) {
-		dev_warn(&h->pdev->dev, "Controller reports "
-			"max supported commands of %d, an obvious lie. "
-			"Using 16.  Ensure that firmware is up to date.\n",
-			h->max_commands);
-		h->max_commands = 16;
-	}
-}
-
-/* Interrogate the hardware for some limits:
- * max commands, max SG elements without chaining, and with chaining,
- * SG chain block size, etc.
- */
-static void cciss_find_board_params(ctlr_info_t *h)
-{
-	cciss_get_max_perf_mode_cmds(h);
-	h->nr_cmds = h->max_commands - 4 - cciss_tape_cmds;
-	h->maxsgentries = readl(&(h->cfgtable->MaxSGElements));
-	/*
-	 * The P600 may exhibit poor performnace under some workloads
-	 * if we use the value in the configuration table. Limit this
-	 * controller to MAXSGENTRIES (32) instead.
-	 */
-	if (h->board_id == 0x3225103C)
-		h->maxsgentries = MAXSGENTRIES;
-	/*
-	 * Limit in-command s/g elements to 32 save dma'able memory.
-	 * Howvever spec says if 0, use 31
-	 */
-	h->max_cmd_sgentries = 31;
-	if (h->maxsgentries > 512) {
-		h->max_cmd_sgentries = 32;
-		h->chainsize = h->maxsgentries - h->max_cmd_sgentries + 1;
-		h->maxsgentries--; /* save one for chain pointer */
-	} else {
-		h->maxsgentries = 31; /* default to traditional values */
-		h->chainsize = 0;
-	}
-}
-
-static inline bool CISS_signature_present(ctlr_info_t *h)
-{
-	if (!check_signature(h->cfgtable->Signature, "CISS", 4)) {
-		dev_warn(&h->pdev->dev, "not a valid CISS config table\n");
-		return false;
-	}
-	return true;
-}
-
-/* Need to enable prefetch in the SCSI core for 6400 in x86 */
-static inline void cciss_enable_scsi_prefetch(ctlr_info_t *h)
-{
-#ifdef CONFIG_X86
-	u32 prefetch;
-
-	prefetch = readl(&(h->cfgtable->SCSI_Prefetch));
-	prefetch |= 0x100;
-	writel(prefetch, &(h->cfgtable->SCSI_Prefetch));
-#endif
-}
-
-/* Disable DMA prefetch for the P600.  Otherwise an ASIC bug may result
- * in a prefetch beyond physical memory.
- */
-static inline void cciss_p600_dma_prefetch_quirk(ctlr_info_t *h)
-{
-	u32 dma_prefetch;
-	__u32 dma_refetch;
-
-	if (h->board_id != 0x3225103C)
-		return;
-	dma_prefetch = readl(h->vaddr + I2O_DMA1_CFG);
-	dma_prefetch |= 0x8000;
-	writel(dma_prefetch, h->vaddr + I2O_DMA1_CFG);
-	pci_read_config_dword(h->pdev, PCI_COMMAND_PARITY, &dma_refetch);
-	dma_refetch |= 0x1;
-	pci_write_config_dword(h->pdev, PCI_COMMAND_PARITY, dma_refetch);
-}
-
-static int cciss_pci_init(ctlr_info_t *h)
-{
-	int prod_index, err;
-
-	prod_index = cciss_lookup_board_id(h->pdev, &h->board_id);
-	if (prod_index < 0)
-		return -ENODEV;
-	h->product_name = products[prod_index].product_name;
-	h->access = *(products[prod_index].access);
-
-	if (cciss_board_disabled(h)) {
-		dev_warn(&h->pdev->dev, "controller appears to be disabled\n");
-		return -ENODEV;
-	}
-
-	pci_disable_link_state(h->pdev, PCIE_LINK_STATE_L0S |
-				PCIE_LINK_STATE_L1 | PCIE_LINK_STATE_CLKPM);
-
-	err = pci_enable_device(h->pdev);
-	if (err) {
-		dev_warn(&h->pdev->dev, "Unable to Enable PCI device\n");
-		return err;
-	}
-
-	err = pci_request_regions(h->pdev, "cciss");
-	if (err) {
-		dev_warn(&h->pdev->dev,
-			"Cannot obtain PCI resources, aborting\n");
-		return err;
-	}
-
-	dev_dbg(&h->pdev->dev, "irq = %x\n", h->pdev->irq);
-	dev_dbg(&h->pdev->dev, "board_id = %x\n", h->board_id);
-
-/* If the kernel supports MSI/MSI-X we will try to enable that functionality,
- * else we use the IO-APIC interrupt assigned to us by system ROM.
- */
-	cciss_interrupt_mode(h);
-	err = cciss_pci_find_memory_BAR(h->pdev, &h->paddr);
-	if (err)
-		goto err_out_free_res;
-	h->vaddr = remap_pci_mem(h->paddr, 0x250);
-	if (!h->vaddr) {
-		err = -ENOMEM;
-		goto err_out_free_res;
-	}
-	err = cciss_wait_for_board_state(h->pdev, h->vaddr, BOARD_READY);
-	if (err)
-		goto err_out_free_res;
-	err = cciss_find_cfgtables(h);
-	if (err)
-		goto err_out_free_res;
-	print_cfg_table(h);
-	cciss_find_board_params(h);
-
-	if (!CISS_signature_present(h)) {
-		err = -ENODEV;
-		goto err_out_free_res;
-	}
-	cciss_enable_scsi_prefetch(h);
-	cciss_p600_dma_prefetch_quirk(h);
-	err = cciss_enter_simple_mode(h);
-	if (err)
-		goto err_out_free_res;
-	cciss_put_controller_into_performant_mode(h);
-	return 0;
-
-err_out_free_res:
-	/*
-	 * Deliberately omit pci_disable_device(): it does something nasty to
-	 * Smart Array controllers that pci_enable_device does not undo
-	 */
-	if (h->transtable)
-		iounmap(h->transtable);
-	if (h->cfgtable)
-		iounmap(h->cfgtable);
-	if (h->vaddr)
-		iounmap(h->vaddr);
-	pci_release_regions(h->pdev);
-	return err;
-}
-
-/* Function to find the first free pointer into our hba[] array
- * Returns -1 if no free entries are left.
- */
-static int alloc_cciss_hba(struct pci_dev *pdev)
-{
-	int i;
-
-	for (i = 0; i < MAX_CTLR; i++) {
-		if (!hba[i]) {
-			ctlr_info_t *h;
-
-			h = kzalloc(sizeof(ctlr_info_t), GFP_KERNEL);
-			if (!h)
-				goto Enomem;
-			hba[i] = h;
-			return i;
-		}
-	}
-	dev_warn(&pdev->dev, "This driver supports a maximum"
-	       " of %d controllers.\n", MAX_CTLR);
-	return -1;
-Enomem:
-	dev_warn(&pdev->dev, "out of memory.\n");
-	return -1;
-}
-
-static void free_hba(ctlr_info_t *h)
-{
-	int i;
-
-	hba[h->ctlr] = NULL;
-	for (i = 0; i < h->highest_lun + 1; i++)
-		if (h->gendisk[i] != NULL)
-			put_disk(h->gendisk[i]);
-	kfree(h);
-}
-
-/* Send a message CDB to the firmware. */
-static int cciss_message(struct pci_dev *pdev, unsigned char opcode,
-			 unsigned char type)
-{
-	typedef struct {
-		CommandListHeader_struct CommandHeader;
-		RequestBlock_struct Request;
-		ErrDescriptor_struct ErrorDescriptor;
-	} Command;
-	static const size_t cmd_sz = sizeof(Command) + sizeof(ErrorInfo_struct);
-	Command *cmd;
-	dma_addr_t paddr64;
-	uint32_t paddr32, tag;
-	void __iomem *vaddr;
-	int i, err;
-
-	vaddr = ioremap_nocache(pci_resource_start(pdev, 0), pci_resource_len(pdev, 0));
-	if (vaddr == NULL)
-		return -ENOMEM;
-
-	/* The Inbound Post Queue only accepts 32-bit physical addresses for the
-	   CCISS commands, so they must be allocated from the lower 4GiB of
-	   memory. */
-	err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
-	if (err) {
-		iounmap(vaddr);
-		return -ENOMEM;
-	}
-
-	cmd = pci_alloc_consistent(pdev, cmd_sz, &paddr64);
-	if (cmd == NULL) {
-		iounmap(vaddr);
-		return -ENOMEM;
-	}
-
-	/* This must fit, because of the 32-bit consistent DMA mask.  Also,
-	   although there's no guarantee, we assume that the address is at
-	   least 4-byte aligned (most likely, it's page-aligned). */
-	paddr32 = paddr64;
-
-	cmd->CommandHeader.ReplyQueue = 0;
-	cmd->CommandHeader.SGList = 0;
-	cmd->CommandHeader.SGTotal = 0;
-	cmd->CommandHeader.Tag.lower = paddr32;
-	cmd->CommandHeader.Tag.upper = 0;
-	memset(&cmd->CommandHeader.LUN.LunAddrBytes, 0, 8);
-
-	cmd->Request.CDBLen = 16;
-	cmd->Request.Type.Type = TYPE_MSG;
-	cmd->Request.Type.Attribute = ATTR_HEADOFQUEUE;
-	cmd->Request.Type.Direction = XFER_NONE;
-	cmd->Request.Timeout = 0; /* Don't time out */
-	cmd->Request.CDB[0] = opcode;
-	cmd->Request.CDB[1] = type;
-	memset(&cmd->Request.CDB[2], 0, 14); /* the rest of the CDB is reserved */
-
-	cmd->ErrorDescriptor.Addr.lower = paddr32 + sizeof(Command);
-	cmd->ErrorDescriptor.Addr.upper = 0;
-	cmd->ErrorDescriptor.Len = sizeof(ErrorInfo_struct);
-
-	writel(paddr32, vaddr + SA5_REQUEST_PORT_OFFSET);
-
-	for (i = 0; i < 10; i++) {
-		tag = readl(vaddr + SA5_REPLY_PORT_OFFSET);
-		if ((tag & ~3) == paddr32)
-			break;
-		msleep(CCISS_POST_RESET_NOOP_TIMEOUT_MSECS);
-	}
-
-	iounmap(vaddr);
-
-	/* we leak the DMA buffer here ... no choice since the controller could
-	   still complete the command. */
-	if (i == 10) {
-		dev_err(&pdev->dev,
-			"controller message %02x:%02x timed out\n",
-			opcode, type);
-		return -ETIMEDOUT;
-	}
-
-	pci_free_consistent(pdev, cmd_sz, cmd, paddr64);
-
-	if (tag & 2) {
-		dev_err(&pdev->dev, "controller message %02x:%02x failed\n",
-			opcode, type);
-		return -EIO;
-	}
-
-	dev_info(&pdev->dev, "controller message %02x:%02x succeeded\n",
-		opcode, type);
-	return 0;
-}
-
-#define cciss_noop(p) cciss_message(p, 3, 0)
-
-static int cciss_controller_hard_reset(struct pci_dev *pdev,
-	void * __iomem vaddr, u32 use_doorbell)
-{
-	u16 pmcsr;
-	int pos;
-
-	if (use_doorbell) {
-		/* For everything after the P600, the PCI power state method
-		 * of resetting the controller doesn't work, so we have this
-		 * other way using the doorbell register.
-		 */
-		dev_info(&pdev->dev, "using doorbell to reset controller\n");
-		writel(use_doorbell, vaddr + SA5_DOORBELL);
-	} else { /* Try to do it the PCI power state way */
-
-		/* Quoting from the Open CISS Specification: "The Power
-		 * Management Control/Status Register (CSR) controls the power
-		 * state of the device.  The normal operating state is D0,
-		 * CSR=00h.  The software off state is D3, CSR=03h.  To reset
-		 * the controller, place the interface device in D3 then to D0,
-		 * this causes a secondary PCI reset which will reset the
-		 * controller." */
-
-		pos = pci_find_capability(pdev, PCI_CAP_ID_PM);
-		if (pos == 0) {
-			dev_err(&pdev->dev,
-				"cciss_controller_hard_reset: "
-				"PCI PM not supported\n");
-			return -ENODEV;
-		}
-		dev_info(&pdev->dev, "using PCI PM to reset controller\n");
-		/* enter the D3hot power management state */
-		pci_read_config_word(pdev, pos + PCI_PM_CTRL, &pmcsr);
-		pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
-		pmcsr |= PCI_D3hot;
-		pci_write_config_word(pdev, pos + PCI_PM_CTRL, pmcsr);
-
-		msleep(500);
-
-		/* enter the D0 power management state */
-		pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
-		pmcsr |= PCI_D0;
-		pci_write_config_word(pdev, pos + PCI_PM_CTRL, pmcsr);
-
-		/*
-		 * The P600 requires a small delay when changing states.
-		 * Otherwise we may think the board did not reset and we bail.
-		 * This for kdump only and is particular to the P600.
-		 */
-		msleep(500);
-	}
-	return 0;
-}
-
-static void init_driver_version(char *driver_version, int len)
-{
-	memset(driver_version, 0, len);
-	strncpy(driver_version, "cciss " DRIVER_NAME, len - 1);
-}
-
-static int write_driver_ver_to_cfgtable(CfgTable_struct __iomem *cfgtable)
-{
-	char *driver_version;
-	int i, size = sizeof(cfgtable->driver_version);
-
-	driver_version = kmalloc(size, GFP_KERNEL);
-	if (!driver_version)
-		return -ENOMEM;
-
-	init_driver_version(driver_version, size);
-	for (i = 0; i < size; i++)
-		writeb(driver_version[i], &cfgtable->driver_version[i]);
-	kfree(driver_version);
-	return 0;
-}
-
-static void read_driver_ver_from_cfgtable(CfgTable_struct __iomem *cfgtable,
-					  unsigned char *driver_ver)
-{
-	int i;
-
-	for (i = 0; i < sizeof(cfgtable->driver_version); i++)
-		driver_ver[i] = readb(&cfgtable->driver_version[i]);
-}
-
-static int controller_reset_failed(CfgTable_struct __iomem *cfgtable)
-{
-
-	char *driver_ver, *old_driver_ver;
-	int rc, size = sizeof(cfgtable->driver_version);
-
-	old_driver_ver = kmalloc(2 * size, GFP_KERNEL);
-	if (!old_driver_ver)
-		return -ENOMEM;
-	driver_ver = old_driver_ver + size;
-
-	/* After a reset, the 32 bytes of "driver version" in the cfgtable
-	 * should have been changed, otherwise we know the reset failed.
-	 */
-	init_driver_version(old_driver_ver, size);
-	read_driver_ver_from_cfgtable(cfgtable, driver_ver);
-	rc = !memcmp(driver_ver, old_driver_ver, size);
-	kfree(old_driver_ver);
-	return rc;
-}
-
-/* This does a hard reset of the controller using PCI power management
- * states or using the doorbell register. */
-static int cciss_kdump_hard_reset_controller(struct pci_dev *pdev)
-{
-	u64 cfg_offset;
-	u32 cfg_base_addr;
-	u64 cfg_base_addr_index;
-	void __iomem *vaddr;
-	unsigned long paddr;
-	u32 misc_fw_support;
-	int rc;
-	CfgTable_struct __iomem *cfgtable;
-	u32 use_doorbell;
-	u32 board_id;
-	u16 command_register;
-
-	/* For controllers as old a the p600, this is very nearly
-	 * the same thing as
-	 *
-	 * pci_save_state(pci_dev);
-	 * pci_set_power_state(pci_dev, PCI_D3hot);
-	 * pci_set_power_state(pci_dev, PCI_D0);
-	 * pci_restore_state(pci_dev);
-	 *
-	 * For controllers newer than the P600, the pci power state
-	 * method of resetting doesn't work so we have another way
-	 * using the doorbell register.
-	 */
-
-	/* Exclude 640x boards.  These are two pci devices in one slot
-	 * which share a battery backed cache module.  One controls the
-	 * cache, the other accesses the cache through the one that controls
-	 * it.  If we reset the one controlling the cache, the other will
-	 * likely not be happy.  Just forbid resetting this conjoined mess.
-	 */
-	cciss_lookup_board_id(pdev, &board_id);
-	if (!ctlr_is_resettable(board_id)) {
-		dev_warn(&pdev->dev, "Controller not resettable\n");
-		return -ENODEV;
-	}
-
-	/* if controller is soft- but not hard resettable... */
-	if (!ctlr_is_hard_resettable(board_id))
-		return -ENOTSUPP; /* try soft reset later. */
-
-	/* Save the PCI command register */
-	pci_read_config_word(pdev, 4, &command_register);
-	/* Turn the board off.  This is so that later pci_restore_state()
-	 * won't turn the board on before the rest of config space is ready.
-	 */
-	pci_disable_device(pdev);
-	pci_save_state(pdev);
-
-	/* find the first memory BAR, so we can find the cfg table */
-	rc = cciss_pci_find_memory_BAR(pdev, &paddr);
-	if (rc)
-		return rc;
-	vaddr = remap_pci_mem(paddr, 0x250);
-	if (!vaddr)
-		return -ENOMEM;
-
-	/* find cfgtable in order to check if reset via doorbell is supported */
-	rc = cciss_find_cfg_addrs(pdev, vaddr, &cfg_base_addr,
-					&cfg_base_addr_index, &cfg_offset);
-	if (rc)
-		goto unmap_vaddr;
-	cfgtable = remap_pci_mem(pci_resource_start(pdev,
-		       cfg_base_addr_index) + cfg_offset, sizeof(*cfgtable));
-	if (!cfgtable) {
-		rc = -ENOMEM;
-		goto unmap_vaddr;
-	}
-	rc = write_driver_ver_to_cfgtable(cfgtable);
-	if (rc)
-		goto unmap_vaddr;
-
-	/* If reset via doorbell register is supported, use that.
-	 * There are two such methods.  Favor the newest method.
-	 */
-	misc_fw_support = readl(&cfgtable->misc_fw_support);
-	use_doorbell = misc_fw_support & MISC_FW_DOORBELL_RESET2;
-	if (use_doorbell) {
-		use_doorbell = DOORBELL_CTLR_RESET2;
-	} else {
-		use_doorbell = misc_fw_support & MISC_FW_DOORBELL_RESET;
-		if (use_doorbell) {
-			dev_warn(&pdev->dev, "Controller claims that "
-				"'Bit 2 doorbell reset' is "
-				"supported, but not 'bit 5 doorbell reset'.  "
-				"Firmware update is recommended.\n");
-			rc = -ENOTSUPP; /* use the soft reset */
-			goto unmap_cfgtable;
-		}
-	}
-
-	rc = cciss_controller_hard_reset(pdev, vaddr, use_doorbell);
-	if (rc)
-		goto unmap_cfgtable;
-	pci_restore_state(pdev);
-	rc = pci_enable_device(pdev);
-	if (rc) {
-		dev_warn(&pdev->dev, "failed to enable device.\n");
-		goto unmap_cfgtable;
-	}
-	pci_write_config_word(pdev, 4, command_register);
-
-	/* Some devices (notably the HP Smart Array 5i Controller)
-	   need a little pause here */
-	msleep(CCISS_POST_RESET_PAUSE_MSECS);
-
-	/* Wait for board to become not ready, then ready. */
-	dev_info(&pdev->dev, "Waiting for board to reset.\n");
-	rc = cciss_wait_for_board_state(pdev, vaddr, BOARD_NOT_READY);
-	if (rc) {
-		dev_warn(&pdev->dev, "Failed waiting for board to hard reset."
-				"  Will try soft reset.\n");
-		rc = -ENOTSUPP; /* Not expected, but try soft reset later */
-		goto unmap_cfgtable;
-	}
-	rc = cciss_wait_for_board_state(pdev, vaddr, BOARD_READY);
-	if (rc) {
-		dev_warn(&pdev->dev,
-			"failed waiting for board to become ready "
-			"after hard reset\n");
-		goto unmap_cfgtable;
-	}
-
-	rc = controller_reset_failed(vaddr);
-	if (rc < 0)
-		goto unmap_cfgtable;
-	if (rc) {
-		dev_warn(&pdev->dev, "Unable to successfully hard reset "
-			"controller. Will try soft reset.\n");
-		rc = -ENOTSUPP; /* Not expected, but try soft reset later */
-	} else {
-		dev_info(&pdev->dev, "Board ready after hard reset.\n");
-	}
-
-unmap_cfgtable:
-	iounmap(cfgtable);
-
-unmap_vaddr:
-	iounmap(vaddr);
-	return rc;
-}
-
-static int cciss_init_reset_devices(struct pci_dev *pdev)
-{
-	int rc, i;
-
-	if (!reset_devices)
-		return 0;
-
-	/* Reset the controller with a PCI power-cycle or via doorbell */
-	rc = cciss_kdump_hard_reset_controller(pdev);
-
-	/* -ENOTSUPP here means we cannot reset the controller
-	 * but it's already (and still) up and running in
-	 * "performant mode".  Or, it might be 640x, which can't reset
-	 * due to concerns about shared bbwc between 6402/6404 pair.
-	 */
-	if (rc == -ENOTSUPP)
-		return rc; /* just try to do the kdump anyhow. */
-	if (rc)
-		return -ENODEV;
-
-	/* Now try to get the controller to respond to a no-op */
-	dev_warn(&pdev->dev, "Waiting for controller to respond to no-op\n");
-	for (i = 0; i < CCISS_POST_RESET_NOOP_RETRIES; i++) {
-		if (cciss_noop(pdev) == 0)
-			break;
-		else
-			dev_warn(&pdev->dev, "no-op failed%s\n",
-				(i < CCISS_POST_RESET_NOOP_RETRIES - 1 ?
-					"; re-trying" : ""));
-		msleep(CCISS_POST_RESET_NOOP_INTERVAL_MSECS);
-	}
-	return 0;
-}
-
-static int cciss_allocate_cmd_pool(ctlr_info_t *h)
-{
-	h->cmd_pool_bits = kmalloc(BITS_TO_LONGS(h->nr_cmds) *
-		sizeof(unsigned long), GFP_KERNEL);
-	h->cmd_pool = pci_alloc_consistent(h->pdev,
-		h->nr_cmds * sizeof(CommandList_struct),
-		&(h->cmd_pool_dhandle));
-	h->errinfo_pool = pci_alloc_consistent(h->pdev,
-		h->nr_cmds * sizeof(ErrorInfo_struct),
-		&(h->errinfo_pool_dhandle));
-	if ((h->cmd_pool_bits == NULL)
-		|| (h->cmd_pool == NULL)
-		|| (h->errinfo_pool == NULL)) {
-		dev_err(&h->pdev->dev, "out of memory");
-		return -ENOMEM;
-	}
-	return 0;
-}
-
-static int cciss_allocate_scatterlists(ctlr_info_t *h)
-{
-	int i;
-
-	/* zero it, so that on free we need not know how many were alloc'ed */
-	h->scatter_list = kzalloc(h->max_commands *
-				sizeof(struct scatterlist *), GFP_KERNEL);
-	if (!h->scatter_list)
-		return -ENOMEM;
-
-	for (i = 0; i < h->nr_cmds; i++) {
-		h->scatter_list[i] = kmalloc(sizeof(struct scatterlist) *
-						h->maxsgentries, GFP_KERNEL);
-		if (h->scatter_list[i] == NULL) {
-			dev_err(&h->pdev->dev, "could not allocate "
-				"s/g lists\n");
-			return -ENOMEM;
-		}
-	}
-	return 0;
-}
-
-static void cciss_free_scatterlists(ctlr_info_t *h)
-{
-	int i;
-
-	if (h->scatter_list) {
-		for (i = 0; i < h->nr_cmds; i++)
-			kfree(h->scatter_list[i]);
-		kfree(h->scatter_list);
-	}
-}
-
-static void cciss_free_cmd_pool(ctlr_info_t *h)
-{
-	kfree(h->cmd_pool_bits);
-	if (h->cmd_pool)
-		pci_free_consistent(h->pdev,
-			h->nr_cmds * sizeof(CommandList_struct),
-			h->cmd_pool, h->cmd_pool_dhandle);
-	if (h->errinfo_pool)
-		pci_free_consistent(h->pdev,
-			h->nr_cmds * sizeof(ErrorInfo_struct),
-			h->errinfo_pool, h->errinfo_pool_dhandle);
-}
-
-static int cciss_request_irq(ctlr_info_t *h,
-	irqreturn_t (*msixhandler)(int, void *),
-	irqreturn_t (*intxhandler)(int, void *))
-{
-	if (h->pdev->msi_enabled || h->pdev->msix_enabled) {
-		if (!request_irq(h->intr[h->intr_mode], msixhandler,
-				0, h->devname, h))
-			return 0;
-		dev_err(&h->pdev->dev, "Unable to get msi irq %d"
-			" for %s\n", h->intr[h->intr_mode],
-			h->devname);
-		return -1;
-	}
-
-	if (!request_irq(h->intr[h->intr_mode], intxhandler,
-			IRQF_SHARED, h->devname, h))
-		return 0;
-	dev_err(&h->pdev->dev, "Unable to get irq %d for %s\n",
-		h->intr[h->intr_mode], h->devname);
-	return -1;
-}
-
-static int cciss_kdump_soft_reset(ctlr_info_t *h)
-{
-	if (cciss_send_reset(h, CTLR_LUNID, CCISS_RESET_TYPE_CONTROLLER)) {
-		dev_warn(&h->pdev->dev, "Resetting array controller failed.\n");
-		return -EIO;
-	}
-
-	dev_info(&h->pdev->dev, "Waiting for board to soft reset.\n");
-	if (cciss_wait_for_board_state(h->pdev, h->vaddr, BOARD_NOT_READY)) {
-		dev_warn(&h->pdev->dev, "Soft reset had no effect.\n");
-		return -1;
-	}
-
-	dev_info(&h->pdev->dev, "Board reset, awaiting READY status.\n");
-	if (cciss_wait_for_board_state(h->pdev, h->vaddr, BOARD_READY)) {
-		dev_warn(&h->pdev->dev, "Board failed to become ready "
-			"after soft reset.\n");
-		return -1;
-	}
-
-	return 0;
-}
-
-static void cciss_undo_allocations_after_kdump_soft_reset(ctlr_info_t *h)
-{
-	int ctlr = h->ctlr;
-
-	free_irq(h->intr[h->intr_mode], h);
-	pci_free_irq_vectors(h->pdev);
-	cciss_free_sg_chain_blocks(h->cmd_sg_list, h->nr_cmds);
-	cciss_free_scatterlists(h);
-	cciss_free_cmd_pool(h);
-	kfree(h->blockFetchTable);
-	if (h->reply_pool)
-		pci_free_consistent(h->pdev, h->max_commands * sizeof(__u64),
-				h->reply_pool, h->reply_pool_dhandle);
-	if (h->transtable)
-		iounmap(h->transtable);
-	if (h->cfgtable)
-		iounmap(h->cfgtable);
-	if (h->vaddr)
-		iounmap(h->vaddr);
-	unregister_blkdev(h->major, h->devname);
-	cciss_destroy_hba_sysfs_entry(h);
-	pci_release_regions(h->pdev);
-	kfree(h);
-	hba[ctlr] = NULL;
-}
-
-/*
- *  This is it.  Find all the controllers and register them.  I really hate
- *  stealing all these major device numbers.
- *  returns the number of block devices registered.
- */
-static int cciss_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
-{
-	int i;
-	int j = 0;
-	int rc;
-	int try_soft_reset = 0;
-	int dac, return_code;
-	InquiryData_struct *inq_buff;
-	ctlr_info_t *h;
-	unsigned long flags;
-
-	/*
-	 * By default the cciss driver is used for all older HP Smart Array
-	 * controllers. There are module paramaters that allow a user to
-	 * override this behavior and instead use the hpsa SCSI driver. If
-	 * this is the case cciss may be loaded first from the kdump initrd
-	 * image and cause a kernel panic. So if reset_devices is true and
-	 * cciss_allow_hpsa is set just bail.
-	 */
-	if ((reset_devices) && (cciss_allow_hpsa == 1))
-		return -ENODEV;
-	rc = cciss_init_reset_devices(pdev);
-	if (rc) {
-		if (rc != -ENOTSUPP)
-			return rc;
-		/* If the reset fails in a particular way (it has no way to do
-		 * a proper hard reset, so returns -ENOTSUPP) we can try to do
-		 * a soft reset once we get the controller configured up to the
-		 * point that it can accept a command.
-		 */
-		try_soft_reset = 1;
-		rc = 0;
-	}
-
-reinit_after_soft_reset:
-
-	i = alloc_cciss_hba(pdev);
-	if (i < 0)
-		return -ENOMEM;
-
-	h = hba[i];
-	h->pdev = pdev;
-	h->busy_initializing = 1;
-	h->intr_mode = cciss_simple_mode ? SIMPLE_MODE_INT : PERF_MODE_INT;
-	INIT_LIST_HEAD(&h->cmpQ);
-	INIT_LIST_HEAD(&h->reqQ);
-	mutex_init(&h->busy_shutting_down);
-
-	if (cciss_pci_init(h) != 0)
-		goto clean_no_release_regions;
-
-	sprintf(h->devname, "cciss%d", i);
-	h->ctlr = i;
-
-	if (cciss_tape_cmds < 2)
-		cciss_tape_cmds = 2;
-	if (cciss_tape_cmds > 16)
-		cciss_tape_cmds = 16;
-
-	init_completion(&h->scan_wait);
-
-	if (cciss_create_hba_sysfs_entry(h))
-		goto clean0;
-
-	/* configure PCI DMA stuff */
-	if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64)))
-		dac = 1;
-	else if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))
-		dac = 0;
-	else {
-		dev_err(&h->pdev->dev, "no suitable DMA available\n");
-		goto clean1;
-	}
-
-	/*
-	 * register with the major number, or get a dynamic major number
-	 * by passing 0 as argument.  This is done for greater than
-	 * 8 controller support.
-	 */
-	if (i < MAX_CTLR_ORIG)
-		h->major = COMPAQ_CISS_MAJOR + i;
-	rc = register_blkdev(h->major, h->devname);
-	if (rc == -EBUSY || rc == -EINVAL) {
-		dev_err(&h->pdev->dev,
-		       "Unable to get major number %d for %s "
-		       "on hba %d\n", h->major, h->devname, i);
-		goto clean1;
-	} else {
-		if (i >= MAX_CTLR_ORIG)
-			h->major = rc;
-	}
-
-	/* make sure the board interrupts are off */
-	h->access.set_intr_mask(h, CCISS_INTR_OFF);
-	rc = cciss_request_irq(h, do_cciss_msix_intr, do_cciss_intx);
-	if (rc)
-		goto clean2;
-
-	dev_info(&h->pdev->dev, "%s: <0x%x> at PCI %s IRQ %d%s using DAC\n",
-	       h->devname, pdev->device, pci_name(pdev),
-	       h->intr[h->intr_mode], dac ? "" : " not");
-
-	if (cciss_allocate_cmd_pool(h))
-		goto clean4;
-
-	if (cciss_allocate_scatterlists(h))
-		goto clean4;
-
-	h->cmd_sg_list = cciss_allocate_sg_chain_blocks(h,
-		h->chainsize, h->nr_cmds);
-	if (!h->cmd_sg_list && h->chainsize > 0)
-		goto clean4;
-
-	spin_lock_init(&h->lock);
-
-	/* Initialize the pdev driver private data.
-	   have it point to h.  */
-	pci_set_drvdata(pdev, h);
-	/* command and error info recs zeroed out before
-	   they are used */
-	bitmap_zero(h->cmd_pool_bits, h->nr_cmds);
-
-	h->num_luns = 0;
-	h->highest_lun = -1;
-	for (j = 0; j < CISS_MAX_LUN; j++) {
-		h->drv[j] = NULL;
-		h->gendisk[j] = NULL;
-	}
-
-	/* At this point, the controller is ready to take commands.
-	 * Now, if reset_devices and the hard reset didn't work, try
-	 * the soft reset and see if that works.
-	 */
-	if (try_soft_reset) {
-
-		/* This is kind of gross.  We may or may not get a completion
-		 * from the soft reset command, and if we do, then the value
-		 * from the fifo may or may not be valid.  So, we wait 10 secs
-		 * after the reset throwing away any completions we get during
-		 * that time.  Unregister the interrupt handler and register
-		 * fake ones to scoop up any residual completions.
-		 */
-		spin_lock_irqsave(&h->lock, flags);
-		h->access.set_intr_mask(h, CCISS_INTR_OFF);
-		spin_unlock_irqrestore(&h->lock, flags);
-		free_irq(h->intr[h->intr_mode], h);
-		rc = cciss_request_irq(h, cciss_msix_discard_completions,
-					cciss_intx_discard_completions);
-		if (rc) {
-			dev_warn(&h->pdev->dev, "Failed to request_irq after "
-				"soft reset.\n");
-			goto clean4;
-		}
-
-		rc = cciss_kdump_soft_reset(h);
-		if (rc) {
-			dev_warn(&h->pdev->dev, "Soft reset failed.\n");
-			goto clean4;
-		}
-
-		dev_info(&h->pdev->dev, "Board READY.\n");
-		dev_info(&h->pdev->dev,
-			"Waiting for stale completions to drain.\n");
-		h->access.set_intr_mask(h, CCISS_INTR_ON);
-		msleep(10000);
-		h->access.set_intr_mask(h, CCISS_INTR_OFF);
-
-		rc = controller_reset_failed(h->cfgtable);
-		if (rc)
-			dev_info(&h->pdev->dev,
-				"Soft reset appears to have failed.\n");
-
-		/* since the controller's reset, we have to go back and re-init
-		 * everything.  Easiest to just forget what we've done and do it
-		 * all over again.
-		 */
-		cciss_undo_allocations_after_kdump_soft_reset(h);
-		try_soft_reset = 0;
-		if (rc)
-			/* don't go to clean4, we already unallocated */
-			return -ENODEV;
-
-		goto reinit_after_soft_reset;
-	}
-
-	cciss_scsi_setup(h);
-
-	/* Turn the interrupts on so we can service requests */
-	h->access.set_intr_mask(h, CCISS_INTR_ON);
-
-	/* Get the firmware version */
-	inq_buff = kzalloc(sizeof(InquiryData_struct), GFP_KERNEL);
-	if (inq_buff == NULL) {
-		dev_err(&h->pdev->dev, "out of memory\n");
-		goto clean4;
-	}
-
-	return_code = sendcmd_withirq(h, CISS_INQUIRY, inq_buff,
-		sizeof(InquiryData_struct), 0, CTLR_LUNID, TYPE_CMD);
-	if (return_code == IO_OK) {
-		h->firm_ver[0] = inq_buff->data_byte[32];
-		h->firm_ver[1] = inq_buff->data_byte[33];
-		h->firm_ver[2] = inq_buff->data_byte[34];
-		h->firm_ver[3] = inq_buff->data_byte[35];
-	} else {	 /* send command failed */
-		dev_warn(&h->pdev->dev, "unable to determine firmware"
-			" version of controller\n");
-	}
-	kfree(inq_buff);
-
-	cciss_procinit(h);
-
-	h->cciss_max_sectors = 8192;
-
-	rebuild_lun_table(h, 1, 0);
-	cciss_engage_scsi(h);
-	h->busy_initializing = 0;
-	return 0;
-
-clean4:
-	cciss_free_cmd_pool(h);
-	cciss_free_scatterlists(h);
-	cciss_free_sg_chain_blocks(h->cmd_sg_list, h->nr_cmds);
-	free_irq(h->intr[h->intr_mode], h);
-clean2:
-	unregister_blkdev(h->major, h->devname);
-clean1:
-	cciss_destroy_hba_sysfs_entry(h);
-clean0:
-	pci_release_regions(pdev);
-clean_no_release_regions:
-	h->busy_initializing = 0;
-
-	/*
-	 * Deliberately omit pci_disable_device(): it does something nasty to
-	 * Smart Array controllers that pci_enable_device does not undo
-	 */
-	pci_set_drvdata(pdev, NULL);
-	free_hba(h);
-	return -ENODEV;
-}
-
-static void cciss_shutdown(struct pci_dev *pdev)
-{
-	ctlr_info_t *h;
-	char *flush_buf;
-	int return_code;
-
-	h = pci_get_drvdata(pdev);
-	flush_buf = kzalloc(4, GFP_KERNEL);
-	if (!flush_buf) {
-		dev_warn(&h->pdev->dev, "cache not flushed, out of memory.\n");
-		return;
-	}
-	/* write all data in the battery backed cache to disk */
-	return_code = sendcmd_withirq(h, CCISS_CACHE_FLUSH, flush_buf,
-		4, 0, CTLR_LUNID, TYPE_CMD);
-	kfree(flush_buf);
-	if (return_code != IO_OK)
-		dev_warn(&h->pdev->dev, "Error flushing cache\n");
-	h->access.set_intr_mask(h, CCISS_INTR_OFF);
-	free_irq(h->intr[h->intr_mode], h);
-}
-
-static int cciss_enter_simple_mode(struct ctlr_info *h)
-{
-	u32 trans_support;
-
-	trans_support = readl(&(h->cfgtable->TransportSupport));
-	if (!(trans_support & SIMPLE_MODE))
-		return -ENOTSUPP;
-
-	h->max_commands = readl(&(h->cfgtable->CmdsOutMax));
-	writel(CFGTBL_Trans_Simple, &(h->cfgtable->HostWrite.TransportRequest));
-	writel(CFGTBL_ChangeReq, h->vaddr + SA5_DOORBELL);
-	cciss_wait_for_mode_change_ack(h);
-	print_cfg_table(h);
-	if (!(readl(&(h->cfgtable->TransportActive)) & CFGTBL_Trans_Simple)) {
-		dev_warn(&h->pdev->dev, "unable to get board into simple mode\n");
-		return -ENODEV;
-	}
-	h->transMethod = CFGTBL_Trans_Simple;
-	return 0;
-}
-
-
-static void cciss_remove_one(struct pci_dev *pdev)
-{
-	ctlr_info_t *h;
-	int i, j;
-
-	if (pci_get_drvdata(pdev) == NULL) {
-		dev_err(&pdev->dev, "Unable to remove device\n");
-		return;
-	}
-
-	h = pci_get_drvdata(pdev);
-	i = h->ctlr;
-	if (hba[i] == NULL) {
-		dev_err(&pdev->dev, "device appears to already be removed\n");
-		return;
-	}
-
-	mutex_lock(&h->busy_shutting_down);
-
-	remove_from_scan_list(h);
-	remove_proc_entry(h->devname, proc_cciss);
-	unregister_blkdev(h->major, h->devname);
-
-	/* remove it from the disk list */
-	for (j = 0; j < CISS_MAX_LUN; j++) {
-		struct gendisk *disk = h->gendisk[j];
-		if (disk) {
-			struct request_queue *q = disk->queue;
-
-			if (disk->flags & GENHD_FL_UP) {
-				cciss_destroy_ld_sysfs_entry(h, j, 1);
-				del_gendisk(disk);
-			}
-			if (q)
-				blk_cleanup_queue(q);
-		}
-	}
-
-#ifdef CONFIG_CISS_SCSI_TAPE
-	cciss_unregister_scsi(h);	/* unhook from SCSI subsystem */
-#endif
-
-	cciss_shutdown(pdev);
-
-	pci_free_irq_vectors(h->pdev);
-
-	iounmap(h->transtable);
-	iounmap(h->cfgtable);
-	iounmap(h->vaddr);
-
-	cciss_free_cmd_pool(h);
-	/* Free up sg elements */
-	for (j = 0; j < h->nr_cmds; j++)
-		kfree(h->scatter_list[j]);
-	kfree(h->scatter_list);
-	cciss_free_sg_chain_blocks(h->cmd_sg_list, h->nr_cmds);
-	kfree(h->blockFetchTable);
-	if (h->reply_pool)
-		pci_free_consistent(h->pdev, h->max_commands * sizeof(__u64),
-				h->reply_pool, h->reply_pool_dhandle);
-	/*
-	 * Deliberately omit pci_disable_device(): it does something nasty to
-	 * Smart Array controllers that pci_enable_device does not undo
-	 */
-	pci_release_regions(pdev);
-	pci_set_drvdata(pdev, NULL);
-	cciss_destroy_hba_sysfs_entry(h);
-	mutex_unlock(&h->busy_shutting_down);
-	free_hba(h);
-}
-
-static struct pci_driver cciss_pci_driver = {
-	.name = "cciss",
-	.probe = cciss_init_one,
-	.remove = cciss_remove_one,
-	.id_table = cciss_pci_device_id,	/* id_table */
-	.shutdown = cciss_shutdown,
-};
-
-/*
- *  This is it.  Register the PCI driver information for the cards we control
- *  the OS will call our registered routines when it finds one of our cards.
- */
-static int __init cciss_init(void)
-{
-	int err;
-
-	/*
-	 * The hardware requires that commands are aligned on a 64-bit
-	 * boundary. Given that we use pci_alloc_consistent() to allocate an
-	 * array of them, the size must be a multiple of 8 bytes.
-	 */
-	BUILD_BUG_ON(sizeof(CommandList_struct) % COMMANDLIST_ALIGNMENT);
-	printk(KERN_INFO DRIVER_NAME "\n");
-
-	err = bus_register(&cciss_bus_type);
-	if (err)
-		return err;
-
-	/* Start the scan thread */
-	cciss_scan_thread = kthread_run(scan_thread, NULL, "cciss_scan");
-	if (IS_ERR(cciss_scan_thread)) {
-		err = PTR_ERR(cciss_scan_thread);
-		goto err_bus_unregister;
-	}
-
-	/* Register for our PCI devices */
-	err = pci_register_driver(&cciss_pci_driver);
-	if (err)
-		goto err_thread_stop;
-
-	return err;
-
-err_thread_stop:
-	kthread_stop(cciss_scan_thread);
-err_bus_unregister:
-	bus_unregister(&cciss_bus_type);
-
-	return err;
-}
-
-static void __exit cciss_cleanup(void)
-{
-	int i;
-
-	pci_unregister_driver(&cciss_pci_driver);
-	/* double check that all controller entrys have been removed */
-	for (i = 0; i < MAX_CTLR; i++) {
-		if (hba[i] != NULL) {
-			dev_warn(&hba[i]->pdev->dev,
-				"had to remove controller\n");
-			cciss_remove_one(hba[i]->pdev);
-		}
-	}
-	kthread_stop(cciss_scan_thread);
-	if (proc_cciss)
-		remove_proc_entry("driver/cciss", NULL);
-	bus_unregister(&cciss_bus_type);
-}
-
-module_init(cciss_init);
-module_exit(cciss_cleanup);
diff --git a/drivers/block/cciss.h b/drivers/block/cciss.h
deleted file mode 100644
index 24b5fd75501a..000000000000
--- a/drivers/block/cciss.h
+++ /dev/null
@@ -1,433 +0,0 @@
-#ifndef CCISS_H
-#define CCISS_H
-
-#include <linux/genhd.h>
-#include <linux/mutex.h>
-
-#include "cciss_cmd.h"
-
-
-#define NWD_SHIFT	4
-#define MAX_PART	(1 << NWD_SHIFT)
-
-#define IO_OK		0
-#define IO_ERROR	1
-#define IO_NEEDS_RETRY  3
-
-#define VENDOR_LEN	8
-#define MODEL_LEN	16
-#define REV_LEN		4
-
-struct ctlr_info;
-typedef struct ctlr_info ctlr_info_t;
-
-struct access_method {
-	void (*submit_command)(ctlr_info_t *h, CommandList_struct *c);
-	void (*set_intr_mask)(ctlr_info_t *h, unsigned long val);
-	unsigned long (*fifo_full)(ctlr_info_t *h);
-	bool (*intr_pending)(ctlr_info_t *h);
-	unsigned long (*command_completed)(ctlr_info_t *h);
-};
-typedef struct _drive_info_struct
-{
-	unsigned char LunID[8];
-	int 	usage_count;
-	struct request_queue *queue;
-	sector_t nr_blocks;
-	int	block_size;
-	int 	heads;
-	int	sectors;
-	int 	cylinders;
-	int	raid_level; /* set to -1 to indicate that
-			     * the drive is not in use/configured
-			     */
-	int	busy_configuring; /* This is set when a drive is being removed
-				   * to prevent it from being opened or it's
-				   * queue from being started.
-				   */
-	struct	device dev;
-	__u8 serial_no[16]; /* from inquiry page 0x83,
-			     * not necc. null terminated.
-			     */
-	char vendor[VENDOR_LEN + 1]; /* SCSI vendor string */
-	char model[MODEL_LEN + 1];   /* SCSI model string */
-	char rev[REV_LEN + 1];       /* SCSI revision string */
-	char device_initialized;     /* indicates whether dev is initialized */
-} drive_info_struct;
-
-struct ctlr_info
-{
-	int	ctlr;
-	char	devname[8];
-	char    *product_name;
-	char	firm_ver[4]; /* Firmware version */
-	struct pci_dev *pdev;
-	__u32	board_id;
-	void __iomem *vaddr;
-	unsigned long paddr;
-	int 	nr_cmds; /* Number of commands allowed on this controller */
-	CfgTable_struct __iomem *cfgtable;
-	int	interrupts_enabled;
-	int	major;
-	int 	max_commands;
-	int	commands_outstanding;
-	int 	max_outstanding; /* Debug */ 
-	int	num_luns;
-	int 	highest_lun;
-	int	usage_count;  /* number of opens all all minor devices */
-	/* Need space for temp sg list
-	 * number of scatter/gathers supported
-	 * number of scatter/gathers in chained block
-	 */
-	struct	scatterlist **scatter_list;
-	int	maxsgentries;
-	int	chainsize;
-	int	max_cmd_sgentries;
-	SGDescriptor_struct **cmd_sg_list;
-
-#	define PERF_MODE_INT	0
-#	define DOORBELL_INT	1
-#	define SIMPLE_MODE_INT	2
-#	define MEMQ_MODE_INT	3
-	unsigned int intr[4];
-	int	intr_mode;
-	int 	cciss_max_sectors;
-	BYTE	cciss_read;
-	BYTE	cciss_write;
-	BYTE	cciss_read_capacity;
-
-	/* information about each logical volume */
-	drive_info_struct *drv[CISS_MAX_LUN];
-
-	struct access_method access;
-
-	/* queue and queue Info */ 
-	struct list_head reqQ;
-	struct list_head cmpQ;
-	unsigned int Qdepth;
-	unsigned int maxQsinceinit;
-	unsigned int maxSG;
-	spinlock_t lock;
-
-	/* pointers to command and error info pool */
-	CommandList_struct 	*cmd_pool;
-	dma_addr_t		cmd_pool_dhandle; 
-	ErrorInfo_struct 	*errinfo_pool;
-	dma_addr_t		errinfo_pool_dhandle; 
-        unsigned long  		*cmd_pool_bits;
-	int			nr_allocs;
-	int			nr_frees; 
-	int			busy_configuring;
-	int			busy_initializing;
-	int			busy_scanning;
-	struct mutex		busy_shutting_down;
-
-	/* This element holds the zero based queue number of the last
-	 * queue to be started.  It is used for fairness.
-	*/
-	int			next_to_run;
-
-	/* Disk structures we need to pass back */
-	struct gendisk   *gendisk[CISS_MAX_LUN];
-#ifdef CONFIG_CISS_SCSI_TAPE
-	struct cciss_scsi_adapter_data_t *scsi_ctlr;
-#endif
-	unsigned char alive;
-	struct list_head scan_list;
-	struct completion scan_wait;
-	struct device dev;
-	/*
-	 * Performant mode tables.
-	 */
-	u32 trans_support;
-	u32 trans_offset;
-	struct TransTable_struct *transtable;
-	unsigned long transMethod;
-
-	/*
-	 * Performant mode completion buffer
-	 */
-	u64 *reply_pool;
-	dma_addr_t reply_pool_dhandle;
-	u64 *reply_pool_head;
-	size_t reply_pool_size;
-	unsigned char reply_pool_wraparound;
-	u32 *blockFetchTable;
-};
-
-/*  Defining the diffent access_methods
- *
- * Memory mapped FIFO interface (SMART 53xx cards)
- */
-#define SA5_DOORBELL	0x20
-#define SA5_REQUEST_PORT_OFFSET	0x40
-#define SA5_REPLY_INTR_MASK_OFFSET	0x34
-#define SA5_REPLY_PORT_OFFSET		0x44
-#define SA5_INTR_STATUS		0x30
-#define SA5_SCRATCHPAD_OFFSET	0xB0
-
-#define SA5_CTCFG_OFFSET	0xB4
-#define SA5_CTMEM_OFFSET	0xB8
-
-#define SA5_INTR_OFF		0x08
-#define SA5B_INTR_OFF		0x04
-#define SA5_INTR_PENDING	0x08
-#define SA5B_INTR_PENDING	0x04
-#define FIFO_EMPTY		0xffffffff	
-#define CCISS_FIRMWARE_READY	0xffff0000 /* value in scratchpad register */
-/* Perf. mode flags */
-#define SA5_PERF_INTR_PENDING	0x04
-#define SA5_PERF_INTR_OFF	0x05
-#define SA5_OUTDB_STATUS_PERF_BIT	0x01
-#define SA5_OUTDB_CLEAR_PERF_BIT	0x01
-#define SA5_OUTDB_CLEAR         0xA0
-#define SA5_OUTDB_CLEAR_PERF_BIT        0x01
-#define SA5_OUTDB_STATUS        0x9C
-
-
-#define  CISS_ERROR_BIT		0x02
-
-#define CCISS_INTR_ON 	1 
-#define CCISS_INTR_OFF	0
-
-
-/* CCISS_BOARD_READY_WAIT_SECS is how long to wait for a board
- * to become ready, in seconds, before giving up on it.
- * CCISS_BOARD_READY_POLL_INTERVAL_MSECS * is how long to wait
- * between polling the board to see if it is ready, in
- * milliseconds.  CCISS_BOARD_READY_ITERATIONS is derived
- * the above.
- */
-#define CCISS_BOARD_READY_WAIT_SECS (120)
-#define CCISS_BOARD_NOT_READY_WAIT_SECS (100)
-#define CCISS_BOARD_READY_POLL_INTERVAL_MSECS (100)
-#define CCISS_BOARD_READY_ITERATIONS \
-	((CCISS_BOARD_READY_WAIT_SECS * 1000) / \
-		CCISS_BOARD_READY_POLL_INTERVAL_MSECS)
-#define CCISS_BOARD_NOT_READY_ITERATIONS \
-	((CCISS_BOARD_NOT_READY_WAIT_SECS * 1000) / \
-		CCISS_BOARD_READY_POLL_INTERVAL_MSECS)
-#define CCISS_POST_RESET_PAUSE_MSECS (3000)
-#define CCISS_POST_RESET_NOOP_INTERVAL_MSECS (4000)
-#define CCISS_POST_RESET_NOOP_RETRIES (12)
-#define CCISS_POST_RESET_NOOP_TIMEOUT_MSECS (10000)
-
-/* 
-	Send the command to the hardware 
-*/
-static void SA5_submit_command( ctlr_info_t *h, CommandList_struct *c) 
-{
-#ifdef CCISS_DEBUG
-	printk(KERN_WARNING "cciss%d: Sending %08x - down to controller\n",
-			h->ctlr, c->busaddr);
-#endif /* CCISS_DEBUG */
-         writel(c->busaddr, h->vaddr + SA5_REQUEST_PORT_OFFSET);
-	readl(h->vaddr + SA5_SCRATCHPAD_OFFSET);
-	 h->commands_outstanding++;
-	 if ( h->commands_outstanding > h->max_outstanding)
-		h->max_outstanding = h->commands_outstanding;
-}
-
-/*  
- *  This card is the opposite of the other cards.  
- *   0 turns interrupts on... 
- *   0x08 turns them off... 
- */
-static void SA5_intr_mask(ctlr_info_t *h, unsigned long val)
-{
-	if (val) 
-	{ /* Turn interrupts on */
-		h->interrupts_enabled = 1;
-		writel(0, h->vaddr + SA5_REPLY_INTR_MASK_OFFSET);
-		(void) readl(h->vaddr + SA5_REPLY_INTR_MASK_OFFSET);
-	} else /* Turn them off */
-	{
-		h->interrupts_enabled = 0;
-        	writel( SA5_INTR_OFF, 
-			h->vaddr + SA5_REPLY_INTR_MASK_OFFSET);
-		(void) readl(h->vaddr + SA5_REPLY_INTR_MASK_OFFSET);
-	}
-}
-/*
- *  This card is the opposite of the other cards.
- *   0 turns interrupts on...
- *   0x04 turns them off...
- */
-static void SA5B_intr_mask(ctlr_info_t *h, unsigned long val)
-{
-        if (val)
-        { /* Turn interrupts on */
-		h->interrupts_enabled = 1;
-                writel(0, h->vaddr + SA5_REPLY_INTR_MASK_OFFSET);
-		(void) readl(h->vaddr + SA5_REPLY_INTR_MASK_OFFSET);
-        } else /* Turn them off */
-        {
-		h->interrupts_enabled = 0;
-                writel( SA5B_INTR_OFF,
-                        h->vaddr + SA5_REPLY_INTR_MASK_OFFSET);
-		(void) readl(h->vaddr + SA5_REPLY_INTR_MASK_OFFSET);
-        }
-}
-
-/* Performant mode intr_mask */
-static void SA5_performant_intr_mask(ctlr_info_t *h, unsigned long val)
-{
-	if (val) { /* turn on interrupts */
-		h->interrupts_enabled = 1;
-		writel(0, h->vaddr + SA5_REPLY_INTR_MASK_OFFSET);
-		(void) readl(h->vaddr + SA5_REPLY_INTR_MASK_OFFSET);
-	} else {
-		h->interrupts_enabled = 0;
-		writel(SA5_PERF_INTR_OFF,
-				h->vaddr + SA5_REPLY_INTR_MASK_OFFSET);
-		(void) readl(h->vaddr + SA5_REPLY_INTR_MASK_OFFSET);
-	}
-}
-
-/*
- *  Returns true if fifo is full.  
- * 
- */ 
-static unsigned long SA5_fifo_full(ctlr_info_t *h)
-{
-	if( h->commands_outstanding >= h->max_commands)
-		return(1);
-	else 
-		return(0);
-
-}
-/* 
- *   returns value read from hardware. 
- *     returns FIFO_EMPTY if there is nothing to read 
- */ 
-static unsigned long SA5_completed(ctlr_info_t *h)
-{
-	unsigned long register_value 
-		= readl(h->vaddr + SA5_REPLY_PORT_OFFSET);
-	if(register_value != FIFO_EMPTY)
-	{
-		h->commands_outstanding--;
-#ifdef CCISS_DEBUG
-		printk("cciss:  Read %lx back from board\n", register_value);
-#endif /* CCISS_DEBUG */ 
-	} 
-#ifdef CCISS_DEBUG
-	else
-	{
-		printk("cciss:  FIFO Empty read\n");
-	}
-#endif 
-	return ( register_value); 
-
-}
-
-/* Performant mode command completed */
-static unsigned long SA5_performant_completed(ctlr_info_t *h)
-{
-	unsigned long register_value = FIFO_EMPTY;
-
-	/* flush the controller write of the reply queue by reading
-	 * outbound doorbell status register.
-	 */
-	register_value = readl(h->vaddr + SA5_OUTDB_STATUS);
-	/* msi auto clears the interrupt pending bit. */
-	if (!(h->pdev->msi_enabled || h->pdev->msix_enabled)) {
-		writel(SA5_OUTDB_CLEAR_PERF_BIT, h->vaddr + SA5_OUTDB_CLEAR);
-		/* Do a read in order to flush the write to the controller
-		 * (as per spec.)
-		 */
-		register_value = readl(h->vaddr + SA5_OUTDB_STATUS);
-	}
-
-	if ((*(h->reply_pool_head) & 1) == (h->reply_pool_wraparound)) {
-		register_value = *(h->reply_pool_head);
-		(h->reply_pool_head)++;
-		h->commands_outstanding--;
-	} else {
-		register_value = FIFO_EMPTY;
-	}
-	/* Check for wraparound */
-	if (h->reply_pool_head == (h->reply_pool + h->max_commands)) {
-		h->reply_pool_head = h->reply_pool;
-		h->reply_pool_wraparound ^= 1;
-	}
-
-	return register_value;
-}
-/*
- *	Returns true if an interrupt is pending.. 
- */
-static bool SA5_intr_pending(ctlr_info_t *h)
-{
-	unsigned long register_value  = 
-		readl(h->vaddr + SA5_INTR_STATUS);
-#ifdef CCISS_DEBUG
-	printk("cciss: intr_pending %lx\n", register_value);
-#endif  /* CCISS_DEBUG */
-	if( register_value &  SA5_INTR_PENDING) 
-		return  1;	
-	return 0 ;
-}
-
-/*
- *      Returns true if an interrupt is pending..
- */
-static bool SA5B_intr_pending(ctlr_info_t *h)
-{
-        unsigned long register_value  =
-                readl(h->vaddr + SA5_INTR_STATUS);
-#ifdef CCISS_DEBUG
-        printk("cciss: intr_pending %lx\n", register_value);
-#endif  /* CCISS_DEBUG */
-        if( register_value &  SA5B_INTR_PENDING)
-                return  1;
-        return 0 ;
-}
-
-static bool SA5_performant_intr_pending(ctlr_info_t *h)
-{
-	unsigned long register_value = readl(h->vaddr + SA5_INTR_STATUS);
-
-	if (!register_value)
-		return false;
-
-	if (h->pdev->msi_enabled || h->pdev->msix_enabled)
-		return true;
-
-	/* Read outbound doorbell to flush */
-	register_value = readl(h->vaddr + SA5_OUTDB_STATUS);
-	return register_value & SA5_OUTDB_STATUS_PERF_BIT;
-}
-
-static struct access_method SA5_access = {
-	.submit_command = SA5_submit_command,
-	.set_intr_mask = SA5_intr_mask,
-	.fifo_full = SA5_fifo_full,
-	.intr_pending = SA5_intr_pending,
-	.command_completed = SA5_completed,
-};
-
-static struct access_method SA5B_access = {
-	.submit_command = SA5_submit_command,
-	.set_intr_mask = SA5B_intr_mask,
-	.fifo_full = SA5_fifo_full,
-	.intr_pending = SA5B_intr_pending,
-	.command_completed = SA5_completed,
-};
-
-static struct access_method SA5_performant_access = {
-	.submit_command = SA5_submit_command,
-	.set_intr_mask = SA5_performant_intr_mask,
-	.fifo_full = SA5_fifo_full,
-	.intr_pending = SA5_performant_intr_pending,
-	.command_completed = SA5_performant_completed,
-};
-
-struct board_type {
-	__u32	board_id;
-	char	*product_name;
-	struct access_method *access;
-	int nr_cmds; /* Max cmds this kind of ctlr can handle. */
-};
-
-#endif /* CCISS_H */
diff --git a/drivers/block/cciss_cmd.h b/drivers/block/cciss_cmd.h
deleted file mode 100644
index d9be6b4d49a6..000000000000
--- a/drivers/block/cciss_cmd.h
+++ /dev/null
@@ -1,269 +0,0 @@
-#ifndef CCISS_CMD_H
-#define CCISS_CMD_H
-
-#include <linux/cciss_defs.h>
-
-/* DEFINES */
-#define CISS_VERSION "1.00"
-
-/* general boundary definitions */
-#define MAXSGENTRIES            32
-#define CCISS_SG_CHAIN          0x80000000
-#define MAXREPLYQS              256
-
-/* Unit Attentions ASC's as defined for the MSA2012sa */
-#define POWER_OR_RESET			0x29
-#define STATE_CHANGED			0x2a
-#define UNIT_ATTENTION_CLEARED		0x2f
-#define LUN_FAILED			0x3e
-#define REPORT_LUNS_CHANGED		0x3f
-
-/* Unit Attentions ASCQ's as defined for the MSA2012sa */
-
-	/* These ASCQ's defined for ASC = POWER_OR_RESET */
-#define POWER_ON_RESET			0x00
-#define POWER_ON_REBOOT			0x01
-#define SCSI_BUS_RESET			0x02
-#define MSA_TARGET_RESET		0x03
-#define CONTROLLER_FAILOVER		0x04
-#define TRANSCEIVER_SE			0x05
-#define TRANSCEIVER_LVD			0x06
-
-	/* These ASCQ's defined for ASC = STATE_CHANGED */
-#define RESERVATION_PREEMPTED		0x03
-#define ASYM_ACCESS_CHANGED		0x06
-#define LUN_CAPACITY_CHANGED		0x09
-
-/* config space register offsets */
-#define CFG_VENDORID            0x00
-#define CFG_DEVICEID            0x02
-#define CFG_I2OBAR              0x10
-#define CFG_MEM1BAR             0x14
-
-/* i2o space register offsets */
-#define I2O_IBDB_SET            0x20
-#define I2O_IBDB_CLEAR          0x70
-#define I2O_INT_STATUS          0x30
-#define I2O_INT_MASK            0x34
-#define I2O_IBPOST_Q            0x40
-#define I2O_OBPOST_Q            0x44
-#define I2O_DMA1_CFG		0x214
-
-/* Configuration Table */
-#define CFGTBL_ChangeReq        0x00000001l
-#define CFGTBL_AccCmds          0x00000001l
-#define DOORBELL_CTLR_RESET     0x00000004l
-#define DOORBELL_CTLR_RESET2    0x00000020l
-
-#define CFGTBL_Trans_Simple     0x00000002l
-#define CFGTBL_Trans_Performant 0x00000004l
-#define CFGTBL_Trans_use_short_tags 0x20000000l
-
-#define CFGTBL_BusType_Ultra2   0x00000001l
-#define CFGTBL_BusType_Ultra3   0x00000002l
-#define CFGTBL_BusType_Fibre1G  0x00000100l
-#define CFGTBL_BusType_Fibre2G  0x00000200l
-typedef struct _vals32
-{
-        __u32   lower;
-        __u32   upper;
-} vals32;
-
-typedef union _u64bit
-{
-   vals32	val32;
-   __u64	val;
-} u64bit;
-
-/* Type defs used in the following structs */
-#define QWORD vals32 
-
-/* STRUCTURES */
-#define CISS_MAX_PHYS_LUN	1024
-/* SCSI-3 Cmmands */
-
-#pragma pack(1)	
-
-#define CISS_INQUIRY 0x12
-/* Date returned */
-typedef struct _InquiryData_struct
-{
-  BYTE data_byte[36];
-} InquiryData_struct;
-
-#define CISS_REPORT_LOG 0xc2    /* Report Logical LUNs */
-#define CISS_REPORT_PHYS 0xc3   /* Report Physical LUNs */
-/* Data returned */
-typedef struct _ReportLUNdata_struct
-{
-  BYTE LUNListLength[4];
-  DWORD reserved;
-  BYTE LUN[CISS_MAX_LUN][8];
-} ReportLunData_struct;
-
-#define CCISS_READ_CAPACITY 0x25 /* Read Capacity */ 
-typedef struct _ReadCapdata_struct
-{
-  BYTE total_size[4];	/* Total size in blocks */
-  BYTE block_size[4];	/* Size of blocks in bytes */
-} ReadCapdata_struct;
-
-#define CCISS_READ_CAPACITY_16 0x9e /* Read Capacity 16 */
-
-/* service action to differentiate a 16 byte read capacity from
-   other commands that use the 0x9e SCSI op code */
-
-#define CCISS_READ_CAPACITY_16_SERVICE_ACT 0x10
-
-typedef struct _ReadCapdata_struct_16
-{
-	BYTE total_size[8];   /* Total size in blocks */
-	BYTE block_size[4];   /* Size of blocks in bytes */
-	BYTE prot_en:1;       /* protection enable bit */
-	BYTE rto_en:1;        /* reference tag own enable bit */
-	BYTE reserved:6;      /* reserved bits */
-	BYTE reserved2[18];   /* reserved bytes per spec */
-} ReadCapdata_struct_16;
-
-/* Define the supported read/write commands for cciss based controllers */
-
-#define CCISS_READ_10   0x28    /* Read(10)  */
-#define CCISS_WRITE_10  0x2a    /* Write(10) */
-#define CCISS_READ_16   0x88    /* Read(16)  */
-#define CCISS_WRITE_16  0x8a    /* Write(16) */
-
-/* Define the CDB lengths supported by cciss based controllers */
-
-#define CDB_LEN10	10
-#define CDB_LEN16	16
-
-/* BMIC commands */
-#define BMIC_READ 0x26
-#define BMIC_WRITE 0x27
-#define BMIC_CACHE_FLUSH 0xc2
-#define CCISS_CACHE_FLUSH 0x01	/* C2 was already being used by CCISS */
-
-#define CCISS_ABORT_MSG 0x00
-#define CCISS_RESET_MSG 0x01
-#define CCISS_RESET_TYPE_CONTROLLER 0x00
-#define CCISS_RESET_TYPE_BUS 0x01
-#define CCISS_RESET_TYPE_TARGET 0x03
-#define CCISS_RESET_TYPE_LUN 0x04
-#define CCISS_NOOP_MSG 0x03
-
-/* Command List Structure */
-#define CTLR_LUNID "\0\0\0\0\0\0\0\0"
-
-typedef struct _CommandListHeader_struct {
-  BYTE              ReplyQueue;
-  BYTE              SGList;
-  HWORD             SGTotal;
-  QWORD             Tag;
-  LUNAddr_struct    LUN;
-} CommandListHeader_struct;
-typedef struct _ErrDescriptor_struct {
-  QWORD  Addr;
-  DWORD  Len;
-} ErrDescriptor_struct;
-typedef struct _SGDescriptor_struct {
-  QWORD  Addr;
-  DWORD  Len;
-  DWORD  Ext;
-} SGDescriptor_struct;
-
-/* Command types */
-#define CMD_RWREQ       0x00
-#define CMD_IOCTL_PEND  0x01
-#define CMD_SCSI	0x03
-#define CMD_MSG_DONE	0x04
-#define CMD_MSG_TIMEOUT 0x05
-#define CMD_MSG_STALE	0xff
-
-/* This structure needs to be divisible by COMMANDLIST_ALIGNMENT
- * because low bits of the address are used to to indicate that
- * whether the tag contains an index or an address.  PAD_32 and
- * PAD_64 can be adjusted independently as needed for 32-bit
- * and 64-bits systems.
- */
-#define COMMANDLIST_ALIGNMENT (32)
-#define IS_64_BIT ((sizeof(long) - 4)/4)
-#define IS_32_BIT (!IS_64_BIT)
-#define PAD_32 (0)
-#define PAD_64 (4)
-#define PADSIZE (IS_32_BIT * PAD_32 + IS_64_BIT * PAD_64)
-#define DIRECT_LOOKUP_BIT 0x10
-#define DIRECT_LOOKUP_SHIFT 5
-
-typedef struct _CommandList_struct {
-  CommandListHeader_struct Header;
-  RequestBlock_struct      Request;
-  ErrDescriptor_struct     ErrDesc;
-  SGDescriptor_struct      SG[MAXSGENTRIES];
-	/* information associated with the command */ 
-  __u32			   busaddr; /* physical address of this record */
-  ErrorInfo_struct * 	   err_info; /* pointer to the allocated mem */ 
-  int			   ctlr;
-  int			   cmd_type; 
-  long			   cmdindex;
-  struct list_head list;
-  struct request *	   rq;
-  struct completion *waiting;
-  int	 retry_count;
-  void * scsi_cmd;
-  char pad[PADSIZE];
-} CommandList_struct;
-
-/* Configuration Table Structure */
-typedef struct _HostWrite_struct {
-  DWORD TransportRequest;
-  DWORD Reserved;
-  DWORD CoalIntDelay;
-  DWORD CoalIntCount;
-} HostWrite_struct;
-
-typedef struct _CfgTable_struct {
-  BYTE             Signature[4];
-  DWORD            SpecValence;
-#define SIMPLE_MODE	0x02
-#define PERFORMANT_MODE	0x04
-#define MEMQ_MODE	0x08
-  DWORD            TransportSupport;
-  DWORD            TransportActive;
-  HostWrite_struct HostWrite;
-  DWORD            CmdsOutMax;
-  DWORD            BusTypes;
-  DWORD            TransMethodOffset;
-  BYTE             ServerName[16];
-  DWORD            HeartBeat;
-  DWORD            SCSI_Prefetch;
-  DWORD            MaxSGElements;
-  DWORD            MaxLogicalUnits;
-  DWORD            MaxPhysicalDrives;
-  DWORD            MaxPhysicalDrivesPerLogicalUnit;
-  DWORD            MaxPerformantModeCommands;
-  u8		   reserved[0x78 - 0x58];
-  u32		   misc_fw_support; /* offset 0x78 */
-#define MISC_FW_DOORBELL_RESET (0x02)
-#define MISC_FW_DOORBELL_RESET2 (0x10)
-	u8	   driver_version[32];
-} CfgTable_struct;
-
-struct TransTable_struct {
-  u32 BlockFetch0;
-  u32 BlockFetch1;
-  u32 BlockFetch2;
-  u32 BlockFetch3;
-  u32 BlockFetch4;
-  u32 BlockFetch5;
-  u32 BlockFetch6;
-  u32 BlockFetch7;
-  u32 RepQSize;
-  u32 RepQCount;
-  u32 RepQCtrAddrLow32;
-  u32 RepQCtrAddrHigh32;
-  u32 RepQAddr0Low32;
-  u32 RepQAddr0High32;
-};
-
-#pragma pack()	 
-#endif /* CCISS_CMD_H */
diff --git a/drivers/block/cciss_scsi.c b/drivers/block/cciss_scsi.c
deleted file mode 100644
index 01a1f7e24978..000000000000
--- a/drivers/block/cciss_scsi.c
+++ /dev/null
@@ -1,1653 +0,0 @@
-/*
- *    Disk Array driver for HP Smart Array controllers, SCSI Tape module.
- *    (C) Copyright 2001, 2007 Hewlett-Packard Development Company, L.P.
- *
- *    This program is free software; you can redistribute it and/or modify
- *    it under the terms of the GNU General Public License as published by
- *    the Free Software Foundation; version 2 of the License.
- *
- *    This program is distributed in the hope that it will be useful,
- *    but WITHOUT ANY WARRANTY; without even the implied warranty of
- *    MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- *    General Public License for more details.
- *
- *    You should have received a copy of the GNU General Public License
- *    along with this program; if not, write to the Free Software
- *    Foundation, Inc., 59 Temple Place, Suite 300, Boston, MA
- *    02111-1307, USA.
- *
- *    Questions/Comments/Bugfixes to iss_storagedev@hp.com
- *
- *    Author: Stephen M. Cameron
- */
-#ifdef CONFIG_CISS_SCSI_TAPE
-
-/* Here we have code to present the driver as a scsi driver
-   as it is simultaneously presented as a block driver.  The
-   reason for doing this is to allow access to SCSI tape drives
-   through the array controller.  Note in particular, neither
-   physical nor logical disks are presented through the scsi layer. */
-
-#include <linux/timer.h>
-#include <linux/completion.h>
-#include <linux/slab.h>
-#include <linux/string.h>
-
-#include <linux/atomic.h>
-
-#include <scsi/scsi_cmnd.h>
-#include <scsi/scsi_device.h>
-#include <scsi/scsi_host.h>
-
-#include "cciss_scsi.h"
-
-#define CCISS_ABORT_MSG 0x00
-#define CCISS_RESET_MSG 0x01
-
-static int fill_cmd(ctlr_info_t *h, CommandList_struct *c, __u8 cmd, void *buff,
-	size_t size,
-	__u8 page_code, unsigned char *scsi3addr,
-	int cmd_type);
-
-static CommandList_struct *cmd_alloc(ctlr_info_t *h);
-static CommandList_struct *cmd_special_alloc(ctlr_info_t *h);
-static void cmd_free(ctlr_info_t *h, CommandList_struct *c);
-static void cmd_special_free(ctlr_info_t *h, CommandList_struct *c);
-
-static int cciss_scsi_write_info(struct Scsi_Host *sh,
-		char *buffer, /* data buffer */
-		int length); 	   /* length of data in buffer */
-static int cciss_scsi_show_info(struct seq_file *m,
-				struct Scsi_Host *sh);
-
-static int cciss_scsi_queue_command (struct Scsi_Host *h,
-				     struct scsi_cmnd *cmd);
-static int cciss_eh_device_reset_handler(struct scsi_cmnd *);
-static int cciss_eh_abort_handler(struct scsi_cmnd *);
-
-static struct cciss_scsi_hba_t ccissscsi[MAX_CTLR] = {
-	{ .name = "cciss0", .ndevices = 0 },
-	{ .name = "cciss1", .ndevices = 0 },
-	{ .name = "cciss2", .ndevices = 0 },
-	{ .name = "cciss3", .ndevices = 0 },
-	{ .name = "cciss4", .ndevices = 0 },
-	{ .name = "cciss5", .ndevices = 0 },
-	{ .name = "cciss6", .ndevices = 0 },
-	{ .name = "cciss7", .ndevices = 0 },
-};
-
-static struct scsi_host_template cciss_driver_template = {
-	.module			= THIS_MODULE,
-	.name			= "cciss",
-	.proc_name		= "cciss",
-	.write_info		= cciss_scsi_write_info,
-	.show_info		= cciss_scsi_show_info,
-	.queuecommand		= cciss_scsi_queue_command,
-	.this_id		= 7,
-	.use_clustering		= DISABLE_CLUSTERING,
-	/* Can't have eh_bus_reset_handler or eh_host_reset_handler for cciss */
-	.eh_device_reset_handler= cciss_eh_device_reset_handler,
-	.eh_abort_handler	= cciss_eh_abort_handler,
-};
-
-#pragma pack(1)
-
-#define SCSI_PAD_32 8
-#define SCSI_PAD_64 8
-
-struct cciss_scsi_cmd_stack_elem_t {
-	CommandList_struct cmd;
-	ErrorInfo_struct Err;
-	__u32 busaddr;
-	int cmdindex;
-	u8 pad[IS_32_BIT * SCSI_PAD_32 + IS_64_BIT * SCSI_PAD_64];
-};
-
-#pragma pack()
-
-#pragma pack(1)
-struct cciss_scsi_cmd_stack_t {
-	struct cciss_scsi_cmd_stack_elem_t *pool;
-	struct cciss_scsi_cmd_stack_elem_t **elem;
-	dma_addr_t cmd_pool_handle;
-	int top;
-	int nelems;
-};
-#pragma pack()
-
-struct cciss_scsi_adapter_data_t {
-	struct Scsi_Host *scsi_host;
-	struct cciss_scsi_cmd_stack_t cmd_stack;
-	SGDescriptor_struct **cmd_sg_list;
-	int registered;
-	spinlock_t lock; // to protect ccissscsi[ctlr];
-};
-
-#define CPQ_TAPE_LOCK(h, flags) spin_lock_irqsave( \
-	&h->scsi_ctlr->lock, flags);
-#define CPQ_TAPE_UNLOCK(h, flags) spin_unlock_irqrestore( \
-	&h->scsi_ctlr->lock, flags);
-
-static CommandList_struct *
-scsi_cmd_alloc(ctlr_info_t *h)
-{
-	/* assume only one process in here at a time, locking done by caller. */
-	/* use h->lock */
-	/* might be better to rewrite how we allocate scsi commands in a way that */
-	/* needs no locking at all. */
-
-	/* take the top memory chunk off the stack and return it, if any. */
-	struct cciss_scsi_cmd_stack_elem_t *c;
-	struct cciss_scsi_adapter_data_t *sa;
-	struct cciss_scsi_cmd_stack_t *stk;
-	u64bit temp64;
-
-	sa = h->scsi_ctlr;
-	stk = &sa->cmd_stack;
-
-	if (stk->top < 0)
-		return NULL;
-	c = stk->elem[stk->top];
-	/* memset(c, 0, sizeof(*c)); */
-	memset(&c->cmd, 0, sizeof(c->cmd));
-	memset(&c->Err, 0, sizeof(c->Err));
-	/* set physical addr of cmd and addr of scsi parameters */
-	c->cmd.busaddr = c->busaddr;
-	c->cmd.cmdindex = c->cmdindex;
-	/* (__u32) (stk->cmd_pool_handle +
-		(sizeof(struct cciss_scsi_cmd_stack_elem_t)*stk->top)); */
-
-	temp64.val = (__u64) (c->busaddr + sizeof(CommandList_struct));
-	/* (__u64) (stk->cmd_pool_handle +
-		(sizeof(struct cciss_scsi_cmd_stack_elem_t)*stk->top) +
-		 sizeof(CommandList_struct)); */
-	stk->top--;
-	c->cmd.ErrDesc.Addr.lower = temp64.val32.lower;
-	c->cmd.ErrDesc.Addr.upper = temp64.val32.upper;
-	c->cmd.ErrDesc.Len = sizeof(ErrorInfo_struct);
-
-	c->cmd.ctlr = h->ctlr;
-	c->cmd.err_info = &c->Err;
-
-	return (CommandList_struct *) c;
-}
-
-static void
-scsi_cmd_free(ctlr_info_t *h, CommandList_struct *c)
-{
-	/* assume only one process in here at a time, locking done by caller. */
-	/* use h->lock */
-	/* drop the free memory chunk on top of the stack. */
-
-	struct cciss_scsi_adapter_data_t *sa;
-	struct cciss_scsi_cmd_stack_t *stk;
-
-	sa = h->scsi_ctlr;
-	stk = &sa->cmd_stack;
-	stk->top++;
-	if (stk->top >= stk->nelems) {
-		dev_err(&h->pdev->dev,
-			"scsi_cmd_free called too many times.\n");
-		BUG();
-	}
-	stk->elem[stk->top] = (struct cciss_scsi_cmd_stack_elem_t *) c;
-}
-
-static int
-scsi_cmd_stack_setup(ctlr_info_t *h, struct cciss_scsi_adapter_data_t *sa)
-{
-	int i;
-	struct cciss_scsi_cmd_stack_t *stk;
-	size_t size;
-
-	stk = &sa->cmd_stack;
-	stk->nelems = cciss_tape_cmds + 2;
-	sa->cmd_sg_list = cciss_allocate_sg_chain_blocks(h,
-		h->chainsize, stk->nelems);
-	if (!sa->cmd_sg_list && h->chainsize > 0)
-		return -ENOMEM;
-
-	size = sizeof(struct cciss_scsi_cmd_stack_elem_t) * stk->nelems;
-
-	/* Check alignment, see cciss_cmd.h near CommandList_struct def. */
-	BUILD_BUG_ON((sizeof(*stk->pool) % COMMANDLIST_ALIGNMENT) != 0);
-	/* pci_alloc_consistent guarantees 32-bit DMA address will be used */
-	stk->pool = (struct cciss_scsi_cmd_stack_elem_t *)
-		pci_alloc_consistent(h->pdev, size, &stk->cmd_pool_handle);
-
-	if (stk->pool == NULL) {
-		cciss_free_sg_chain_blocks(sa->cmd_sg_list, stk->nelems);
-		sa->cmd_sg_list = NULL;
-		return -ENOMEM;
-	}
-	stk->elem = kmalloc(sizeof(stk->elem[0]) * stk->nelems, GFP_KERNEL);
-	if (!stk->elem) {
-		pci_free_consistent(h->pdev, size, stk->pool,
-		stk->cmd_pool_handle);
-		return -1;
-	}
-	for (i = 0; i < stk->nelems; i++) {
-		stk->elem[i] = &stk->pool[i];
-		stk->elem[i]->busaddr = (__u32) (stk->cmd_pool_handle +
-			(sizeof(struct cciss_scsi_cmd_stack_elem_t) * i));
-		stk->elem[i]->cmdindex = i;
-	}
-	stk->top = stk->nelems-1;
-	return 0;
-}
-
-static void
-scsi_cmd_stack_free(ctlr_info_t *h)
-{
-	struct cciss_scsi_adapter_data_t *sa;
-	struct cciss_scsi_cmd_stack_t *stk;
-	size_t size;
-
-	sa = h->scsi_ctlr;
-	stk = &sa->cmd_stack;
-	if (stk->top != stk->nelems-1) {
-		dev_warn(&h->pdev->dev,
-			"bug: %d scsi commands are still outstanding.\n",
-			stk->nelems - stk->top);
-	}
-	size = sizeof(struct cciss_scsi_cmd_stack_elem_t) * stk->nelems;
-
-	pci_free_consistent(h->pdev, size, stk->pool, stk->cmd_pool_handle);
-	stk->pool = NULL;
-	cciss_free_sg_chain_blocks(sa->cmd_sg_list, stk->nelems);
-	kfree(stk->elem);
-	stk->elem = NULL;
-}
-
-#if 0
-static void
-print_cmd(CommandList_struct *cp)
-{
-	printk("queue:%d\n", cp->Header.ReplyQueue);
-	printk("sglist:%d\n", cp->Header.SGList);
-	printk("sgtot:%d\n", cp->Header.SGTotal);
-	printk("Tag:0x%08x/0x%08x\n", cp->Header.Tag.upper,
-			cp->Header.Tag.lower);
-	printk("LUN:0x%8phN\n", cp->Header.LUN.LunAddrBytes);
-	printk("CDBLen:%d\n", cp->Request.CDBLen);
-	printk("Type:%d\n",cp->Request.Type.Type);
-	printk("Attr:%d\n",cp->Request.Type.Attribute);
-	printk(" Dir:%d\n",cp->Request.Type.Direction);
-	printk("Timeout:%d\n",cp->Request.Timeout);
-	printk("CDB: %16ph\n", cp->Request.CDB);
-	printk("edesc.Addr: 0x%08x/0%08x, Len  = %d\n",
-		cp->ErrDesc.Addr.upper, cp->ErrDesc.Addr.lower,
-			cp->ErrDesc.Len);
-	printk("sgs..........Errorinfo:\n");
-	printk("scsistatus:%d\n", cp->err_info->ScsiStatus);
-	printk("senselen:%d\n", cp->err_info->SenseLen);
-	printk("cmd status:%d\n", cp->err_info->CommandStatus);
-	printk("resid cnt:%d\n", cp->err_info->ResidualCnt);
-	printk("offense size:%d\n", cp->err_info->MoreErrInfo.Invalid_Cmd.offense_size);
-	printk("offense byte:%d\n", cp->err_info->MoreErrInfo.Invalid_Cmd.offense_num);
-	printk("offense value:%d\n", cp->err_info->MoreErrInfo.Invalid_Cmd.offense_value);
-}
-#endif
-
-static int
-find_bus_target_lun(ctlr_info_t *h, int *bus, int *target, int *lun)
-{
-	/* finds an unused bus, target, lun for a new device */
-	/* assumes h->scsi_ctlr->lock is held */
-	int i, found=0;
-	unsigned char target_taken[CCISS_MAX_SCSI_DEVS_PER_HBA];
-
-	memset(&target_taken[0], 0, CCISS_MAX_SCSI_DEVS_PER_HBA);
-
-	target_taken[SELF_SCSI_ID] = 1;
-	for (i = 0; i < ccissscsi[h->ctlr].ndevices; i++)
-		target_taken[ccissscsi[h->ctlr].dev[i].target] = 1;
-
-	for (i = 0; i < CCISS_MAX_SCSI_DEVS_PER_HBA; i++) {
-		if (!target_taken[i]) {
-			*bus = 0; *target=i; *lun = 0; found=1;
-			break;
-		}
-	}
-	return (!found);
-}
-struct scsi2map {
-	char scsi3addr[8];
-	int bus, target, lun;
-};
-
-static int
-cciss_scsi_add_entry(ctlr_info_t *h, int hostno,
-		struct cciss_scsi_dev_t *device,
-		struct scsi2map *added, int *nadded)
-{
-	/* assumes h->scsi_ctlr->lock is held */
-	int n = ccissscsi[h->ctlr].ndevices;
-	struct cciss_scsi_dev_t *sd;
-	int i, bus, target, lun;
-	unsigned char addr1[8], addr2[8];
-
-	if (n >= CCISS_MAX_SCSI_DEVS_PER_HBA) {
-		dev_warn(&h->pdev->dev, "Too many devices, "
-			"some will be inaccessible.\n");
-		return -1;
-	}
-
-	bus = target = -1;
-	lun = 0;
-	/* Is this device a non-zero lun of a multi-lun device */
-	/* byte 4 of the 8-byte LUN addr will contain the logical unit no. */
-	if (device->scsi3addr[4] != 0) {
-		/* Search through our list and find the device which */
-		/* has the same 8 byte LUN address, excepting byte 4. */
-		/* Assign the same bus and target for this new LUN. */
-		/* Use the logical unit number from the firmware. */
-		memcpy(addr1, device->scsi3addr, 8);
-		addr1[4] = 0;
-		for (i = 0; i < n; i++) {
-			sd = &ccissscsi[h->ctlr].dev[i];
-			memcpy(addr2, sd->scsi3addr, 8);
-			addr2[4] = 0;
-			/* differ only in byte 4? */
-			if (memcmp(addr1, addr2, 8) == 0) {
-				bus = sd->bus;
-				target = sd->target;
-				lun = device->scsi3addr[4];
-				break;
-			}
-		}
-	}
-
-	sd = &ccissscsi[h->ctlr].dev[n];
-	if (lun == 0) {
-		if (find_bus_target_lun(h,
-			&sd->bus, &sd->target, &sd->lun) != 0)
-			return -1;
-	} else {
-		sd->bus = bus;
-		sd->target = target;
-		sd->lun = lun;
-	}
-	added[*nadded].bus = sd->bus;
-	added[*nadded].target = sd->target;
-	added[*nadded].lun = sd->lun;
-	(*nadded)++;
-
-	memcpy(sd->scsi3addr, device->scsi3addr, 8);
-	memcpy(sd->vendor, device->vendor, sizeof(sd->vendor));
-	memcpy(sd->revision, device->revision, sizeof(sd->revision));
-	memcpy(sd->device_id, device->device_id, sizeof(sd->device_id));
-	sd->devtype = device->devtype;
-
-	ccissscsi[h->ctlr].ndevices++;
-
-	/* initially, (before registering with scsi layer) we don't
-	   know our hostno and we don't want to print anything first
-	   time anyway (the scsi layer's inquiries will show that info) */
-	if (hostno != -1)
-		dev_info(&h->pdev->dev, "%s device c%db%dt%dl%d added.\n",
-			scsi_device_type(sd->devtype), hostno,
-			sd->bus, sd->target, sd->lun);
-	return 0;
-}
-
-static void
-cciss_scsi_remove_entry(ctlr_info_t *h, int hostno, int entry,
-	struct scsi2map *removed, int *nremoved)
-{
-	/* assumes h->ctlr]->scsi_ctlr->lock is held */
-	int i;
-	struct cciss_scsi_dev_t sd;
-
-	if (entry < 0 || entry >= CCISS_MAX_SCSI_DEVS_PER_HBA) return;
-	sd = ccissscsi[h->ctlr].dev[entry];
-	removed[*nremoved].bus    = sd.bus;
-	removed[*nremoved].target = sd.target;
-	removed[*nremoved].lun    = sd.lun;
-	(*nremoved)++;
-	for (i = entry; i < ccissscsi[h->ctlr].ndevices-1; i++)
-		ccissscsi[h->ctlr].dev[i] = ccissscsi[h->ctlr].dev[i+1];
-	ccissscsi[h->ctlr].ndevices--;
-	dev_info(&h->pdev->dev, "%s device c%db%dt%dl%d removed.\n",
-		scsi_device_type(sd.devtype), hostno,
-			sd.bus, sd.target, sd.lun);
-}
-
-
-#define SCSI3ADDR_EQ(a,b) ( \
-	(a)[7] == (b)[7] && \
-	(a)[6] == (b)[6] && \
-	(a)[5] == (b)[5] && \
-	(a)[4] == (b)[4] && \
-	(a)[3] == (b)[3] && \
-	(a)[2] == (b)[2] && \
-	(a)[1] == (b)[1] && \
-	(a)[0] == (b)[0])
-
-static void fixup_botched_add(ctlr_info_t *h, char *scsi3addr)
-{
-	/* called when scsi_add_device fails in order to re-adjust */
-	/* ccissscsi[] to match the mid layer's view. */
-	unsigned long flags;
-	int i, j;
-	CPQ_TAPE_LOCK(h, flags);
-	for (i = 0; i < ccissscsi[h->ctlr].ndevices; i++) {
-		if (memcmp(scsi3addr,
-				ccissscsi[h->ctlr].dev[i].scsi3addr, 8) == 0) {
-			for (j = i; j < ccissscsi[h->ctlr].ndevices-1; j++)
-				ccissscsi[h->ctlr].dev[j] =
-					ccissscsi[h->ctlr].dev[j+1];
-			ccissscsi[h->ctlr].ndevices--;
-			break;
-		}
-	}
-	CPQ_TAPE_UNLOCK(h, flags);
-}
-
-static int device_is_the_same(struct cciss_scsi_dev_t *dev1,
-	struct cciss_scsi_dev_t *dev2)
-{
-	return dev1->devtype == dev2->devtype &&
-		memcmp(dev1->scsi3addr, dev2->scsi3addr,
-			sizeof(dev1->scsi3addr)) == 0 &&
-		memcmp(dev1->device_id, dev2->device_id,
-			sizeof(dev1->device_id)) == 0 &&
-		memcmp(dev1->vendor, dev2->vendor,
-			sizeof(dev1->vendor)) == 0 &&
-		memcmp(dev1->model, dev2->model,
-			sizeof(dev1->model)) == 0 &&
-		memcmp(dev1->revision, dev2->revision,
-			sizeof(dev1->revision)) == 0;
-}
-
-static int
-adjust_cciss_scsi_table(ctlr_info_t *h, int hostno,
-	struct cciss_scsi_dev_t sd[], int nsds)
-{
-	/* sd contains scsi3 addresses and devtypes, but
-	   bus target and lun are not filled in.  This funciton
-	   takes what's in sd to be the current and adjusts
-	   ccissscsi[] to be in line with what's in sd. */
-
-	int i,j, found, changes=0;
-	struct cciss_scsi_dev_t *csd;
-	unsigned long flags;
-	struct scsi2map *added, *removed;
-	int nadded, nremoved;
-	struct Scsi_Host *sh = NULL;
-
-	added = kzalloc(sizeof(*added) * CCISS_MAX_SCSI_DEVS_PER_HBA,
-			GFP_KERNEL);
-	removed = kzalloc(sizeof(*removed) * CCISS_MAX_SCSI_DEVS_PER_HBA,
-			GFP_KERNEL);
-
-	if (!added || !removed) {
-		dev_warn(&h->pdev->dev,
-			"Out of memory in adjust_cciss_scsi_table\n");
-		goto free_and_out;
-	}
-
-	CPQ_TAPE_LOCK(h, flags);
-
-	if (hostno != -1)  /* if it's not the first time... */
-		sh = h->scsi_ctlr->scsi_host;
-
-	/* find any devices in ccissscsi[] that are not in
-	   sd[] and remove them from ccissscsi[] */
-
-	i = 0;
-	nremoved = 0;
-	nadded = 0;
-	while (i < ccissscsi[h->ctlr].ndevices) {
-		csd = &ccissscsi[h->ctlr].dev[i];
-		found=0;
-		for (j=0;j<nsds;j++) {
-			if (SCSI3ADDR_EQ(sd[j].scsi3addr,
-				csd->scsi3addr)) {
-				if (device_is_the_same(&sd[j], csd))
-					found=2;
-				else
-					found=1;
-				break;
-			}
-		}
-
-		if (found == 0) { /* device no longer present. */
-			changes++;
-			cciss_scsi_remove_entry(h, hostno, i,
-				removed, &nremoved);
-			/* remove ^^^, hence i not incremented */
-		} else if (found == 1) { /* device is different in some way */
-			changes++;
-			dev_info(&h->pdev->dev,
-				"device c%db%dt%dl%d has changed.\n",
-				hostno, csd->bus, csd->target, csd->lun);
-			cciss_scsi_remove_entry(h, hostno, i,
-				removed, &nremoved);
-			/* remove ^^^, hence i not incremented */
-			if (cciss_scsi_add_entry(h, hostno, &sd[j],
-				added, &nadded) != 0)
-				/* we just removed one, so add can't fail. */
-					BUG();
-			csd->devtype = sd[j].devtype;
-			memcpy(csd->device_id, sd[j].device_id,
-				sizeof(csd->device_id));
-			memcpy(csd->vendor, sd[j].vendor,
-				sizeof(csd->vendor));
-			memcpy(csd->model, sd[j].model,
-				sizeof(csd->model));
-			memcpy(csd->revision, sd[j].revision,
-				sizeof(csd->revision));
-		} else 		/* device is same as it ever was, */
-			i++;	/* so just move along. */
-	}
-
-	/* Now, make sure every device listed in sd[] is also
- 	   listed in ccissscsi[], adding them if they aren't found */
-
-	for (i=0;i<nsds;i++) {
-		found=0;
-		for (j = 0; j < ccissscsi[h->ctlr].ndevices; j++) {
-			csd = &ccissscsi[h->ctlr].dev[j];
-			if (SCSI3ADDR_EQ(sd[i].scsi3addr,
-				csd->scsi3addr)) {
-				if (device_is_the_same(&sd[i], csd))
-					found=2;	/* found device */
-				else
-					found=1; 	/* found a bug. */
-				break;
-			}
-		}
-		if (!found) {
-			changes++;
-			if (cciss_scsi_add_entry(h, hostno, &sd[i],
-				added, &nadded) != 0)
-				break;
-		} else if (found == 1) {
-			/* should never happen... */
-			changes++;
-			dev_warn(&h->pdev->dev,
-				"device unexpectedly changed\n");
-			/* but if it does happen, we just ignore that device */
-		}
-	}
-	CPQ_TAPE_UNLOCK(h, flags);
-
-	/* Don't notify scsi mid layer of any changes the first time through */
-	/* (or if there are no changes) scsi_scan_host will do it later the */
-	/* first time through. */
-	if (hostno == -1 || !changes)
-		goto free_and_out;
-
-	/* Notify scsi mid layer of any removed devices */
-	for (i = 0; i < nremoved; i++) {
-		struct scsi_device *sdev =
-			scsi_device_lookup(sh, removed[i].bus,
-				removed[i].target, removed[i].lun);
-		if (sdev != NULL) {
-			scsi_remove_device(sdev);
-			scsi_device_put(sdev);
-		} else {
-			/* We don't expect to get here. */
-			/* future cmds to this device will get selection */
-			/* timeout as if the device was gone. */
-			dev_warn(&h->pdev->dev, "didn't find "
-				"c%db%dt%dl%d\n for removal.",
-				hostno, removed[i].bus,
-				removed[i].target, removed[i].lun);
-		}
-	}
-
-	/* Notify scsi mid layer of any added devices */
-	for (i = 0; i < nadded; i++) {
-		int rc;
-		rc = scsi_add_device(sh, added[i].bus,
-			added[i].target, added[i].lun);
-		if (rc == 0)
-			continue;
-		dev_warn(&h->pdev->dev, "scsi_add_device "
-			"c%db%dt%dl%d failed, device not added.\n",
-			hostno, added[i].bus, added[i].target, added[i].lun);
-		/* now we have to remove it from ccissscsi, */
-		/* since it didn't get added to scsi mid layer */
-		fixup_botched_add(h, added[i].scsi3addr);
-	}
-
-free_and_out:
-	kfree(added);
-	kfree(removed);
-	return 0;
-}
-
-static int
-lookup_scsi3addr(ctlr_info_t *h, int bus, int target, int lun, char *scsi3addr)
-{
-	int i;
-	struct cciss_scsi_dev_t *sd;
-	unsigned long flags;
-
-	CPQ_TAPE_LOCK(h, flags);
-	for (i = 0; i < ccissscsi[h->ctlr].ndevices; i++) {
-		sd = &ccissscsi[h->ctlr].dev[i];
-		if (sd->bus == bus &&
-		    sd->target == target &&
-		    sd->lun == lun) {
-			memcpy(scsi3addr, &sd->scsi3addr[0], 8);
-			CPQ_TAPE_UNLOCK(h, flags);
-			return 0;
-		}
-	}
-	CPQ_TAPE_UNLOCK(h, flags);
-	return -1;
-}
-
-static void
-cciss_scsi_setup(ctlr_info_t *h)
-{
-	struct cciss_scsi_adapter_data_t * shba;
-
-	ccissscsi[h->ctlr].ndevices = 0;
-	shba = kmalloc(sizeof(*shba), GFP_KERNEL);
-	if (shba == NULL)
-		return;
-	shba->scsi_host = NULL;
-	spin_lock_init(&shba->lock);
-	shba->registered = 0;
-	if (scsi_cmd_stack_setup(h, shba) != 0) {
-		kfree(shba);
-		shba = NULL;
-	}
-	h->scsi_ctlr = shba;
-	return;
-}
-
-static void complete_scsi_command(CommandList_struct *c, int timeout,
-	__u32 tag)
-{
-	struct scsi_cmnd *cmd;
-	ctlr_info_t *h;
-	ErrorInfo_struct *ei;
-
-	ei = c->err_info;
-
-	/* First, see if it was a message rather than a command */
-	if (c->Request.Type.Type == TYPE_MSG)  {
-		c->cmd_type = CMD_MSG_DONE;
-		return;
-	}
-
-	cmd = (struct scsi_cmnd *) c->scsi_cmd;
-	h = hba[c->ctlr];
-
-	scsi_dma_unmap(cmd);
-	if (c->Header.SGTotal > h->max_cmd_sgentries)
-		cciss_unmap_sg_chain_block(h, c);
-
-	cmd->result = (DID_OK << 16); 		/* host byte */
-	cmd->result |= (COMMAND_COMPLETE << 8);	/* msg byte */
-	/* cmd->result |= (GOOD < 1); */		/* status byte */
-
-	cmd->result |= (ei->ScsiStatus);
-	/* printk("Scsistatus is 0x%02x\n", ei->ScsiStatus);  */
-
-	/* copy the sense data whether we need to or not. */
-
-	memcpy(cmd->sense_buffer, ei->SenseInfo,
-		ei->SenseLen > SCSI_SENSE_BUFFERSIZE ?
-			SCSI_SENSE_BUFFERSIZE :
-			ei->SenseLen);
-	scsi_set_resid(cmd, ei->ResidualCnt);
-
-	if (ei->CommandStatus != 0) { /* an error has occurred */
-		switch (ei->CommandStatus) {
-			case CMD_TARGET_STATUS:
-				/* Pass it up to the upper layers... */
-				if (!ei->ScsiStatus) {
-
-	/* Ordinarily, this case should never happen, but there is a bug
-	   in some released firmware revisions that allows it to happen
-	   if, for example, a 4100 backplane loses power and the tape
-	   drive is in it.  We assume that it's a fatal error of some
-	   kind because we can't show that it wasn't. We will make it
-	   look like selection timeout since that is the most common
-	   reason for this to occur, and it's severe enough. */
-
-					cmd->result = DID_NO_CONNECT << 16;
-				}
-			break;
-			case CMD_DATA_UNDERRUN: /* let mid layer handle it. */
-			break;
-			case CMD_DATA_OVERRUN:
-				dev_warn(&h->pdev->dev, "%p has"
-					" completed with data overrun "
-					"reported\n", c);
-			break;
-			case CMD_INVALID: {
-				/*
-				print_hex_dump(KERN_INFO, "", DUMP_PREFIX_OFFSET, 16, 1, c, sizeof(*c), false);
-				print_cmd(c);
-				 */
-     /* We get CMD_INVALID if you address a non-existent tape drive instead
-	of a selection timeout (no response).  You will see this if you yank
-	out a tape drive, then try to access it. This is kind of a shame
-	because it means that any other CMD_INVALID (e.g. driver bug) will
-	get interpreted as a missing target. */
-				cmd->result = DID_NO_CONNECT << 16;
-				}
-			break;
-			case CMD_PROTOCOL_ERR:
-				cmd->result = DID_ERROR << 16;
-				dev_warn(&h->pdev->dev,
-					"%p has protocol error\n", c);
-                        break;
-			case CMD_HARDWARE_ERR:
-				cmd->result = DID_ERROR << 16;
-				dev_warn(&h->pdev->dev,
-					"%p had hardware error\n", c);
-                        break;
-			case CMD_CONNECTION_LOST:
-				cmd->result = DID_ERROR << 16;
-				dev_warn(&h->pdev->dev,
-					"%p had connection lost\n", c);
-			break;
-			case CMD_ABORTED:
-				cmd->result = DID_ABORT << 16;
-				dev_warn(&h->pdev->dev, "%p was aborted\n", c);
-			break;
-			case CMD_ABORT_FAILED:
-				cmd->result = DID_ERROR << 16;
-				dev_warn(&h->pdev->dev,
-					"%p reports abort failed\n", c);
-			break;
-			case CMD_UNSOLICITED_ABORT:
-				cmd->result = DID_ABORT << 16;
-				dev_warn(&h->pdev->dev, "%p aborted due to an "
-					"unsolicited abort\n", c);
-			break;
-			case CMD_TIMEOUT:
-				cmd->result = DID_TIME_OUT << 16;
-				dev_warn(&h->pdev->dev, "%p timedout\n", c);
-			break;
-			case CMD_UNABORTABLE:
-				cmd->result = DID_ERROR << 16;
-				dev_warn(&h->pdev->dev, "c %p command "
-					"unabortable\n", c);
-			break;
-			default:
-				cmd->result = DID_ERROR << 16;
-				dev_warn(&h->pdev->dev,
-					"%p returned unknown status %x\n", c,
-						ei->CommandStatus);
-		}
-	}
-	cmd->scsi_done(cmd);
-	scsi_cmd_free(h, c);
-}
-
-static int
-cciss_scsi_detect(ctlr_info_t *h)
-{
-	struct Scsi_Host *sh;
-	int error;
-
-	sh = scsi_host_alloc(&cciss_driver_template, sizeof(struct ctlr_info *));
-	if (sh == NULL)
-		goto fail;
-	sh->io_port = 0;	// good enough?  FIXME,
-	sh->n_io_port = 0;	// I don't think we use these two...
-	sh->this_id = SELF_SCSI_ID;
-	sh->can_queue = cciss_tape_cmds;
-	sh->sg_tablesize = h->maxsgentries;
-	sh->max_cmd_len = MAX_COMMAND_SIZE;
-	sh->max_sectors = h->cciss_max_sectors;
-
-	((struct cciss_scsi_adapter_data_t *)
-		h->scsi_ctlr)->scsi_host = sh;
-	sh->hostdata[0] = (unsigned long) h;
-	sh->irq = h->intr[SIMPLE_MODE_INT];
-	sh->unique_id = sh->irq;
-	error = scsi_add_host(sh, &h->pdev->dev);
-	if (error)
-		goto fail_host_put;
-	scsi_scan_host(sh);
-	return 1;
-
- fail_host_put:
-	scsi_host_put(sh);
- fail:
-	return 0;
-}
-
-static void
-cciss_unmap_one(struct pci_dev *pdev,
-		CommandList_struct *c,
-		size_t buflen,
-		int data_direction)
-{
-	u64bit addr64;
-
-	addr64.val32.lower = c->SG[0].Addr.lower;
-	addr64.val32.upper = c->SG[0].Addr.upper;
-	pci_unmap_single(pdev, (dma_addr_t) addr64.val, buflen, data_direction);
-}
-
-static void
-cciss_map_one(struct pci_dev *pdev,
-		CommandList_struct *c,
-		unsigned char *buf,
-		size_t buflen,
-		int data_direction)
-{
-	__u64 addr64;
-
-	addr64 = (__u64) pci_map_single(pdev, buf, buflen, data_direction);
-	c->SG[0].Addr.lower =
-	  (__u32) (addr64 & (__u64) 0x00000000FFFFFFFF);
-	c->SG[0].Addr.upper =
-	  (__u32) ((addr64 >> 32) & (__u64) 0x00000000FFFFFFFF);
-	c->SG[0].Len = buflen;
-	c->Header.SGList = (__u8) 1;   /* no. SGs contig in this cmd */
-	c->Header.SGTotal = (__u16) 1; /* total sgs in this cmd list */
-}
-
-static int
-cciss_scsi_do_simple_cmd(ctlr_info_t *h,
-			CommandList_struct *c,
-			unsigned char *scsi3addr,
-			unsigned char *cdb,
-			unsigned char cdblen,
-			unsigned char *buf, int bufsize,
-			int direction)
-{
-	DECLARE_COMPLETION_ONSTACK(wait);
-
-	c->cmd_type = CMD_IOCTL_PEND; /* treat this like an ioctl */
-	c->scsi_cmd = NULL;
-	c->Header.ReplyQueue = 0;  /* unused in simple mode */
-	memcpy(&c->Header.LUN, scsi3addr, sizeof(c->Header.LUN));
-	c->Header.Tag.lower = c->busaddr;  /* Use k. address of cmd as tag */
-	// Fill in the request block...
-
-	/* printk("Using scsi3addr 0x%02x%0x2%0x2%0x2%0x2%0x2%0x2%0x2\n",
-		scsi3addr[0], scsi3addr[1], scsi3addr[2], scsi3addr[3],
-		scsi3addr[4], scsi3addr[5], scsi3addr[6], scsi3addr[7]); */
-
-	memset(c->Request.CDB, 0, sizeof(c->Request.CDB));
-	memcpy(c->Request.CDB, cdb, cdblen);
-	c->Request.Timeout = 0;
-	c->Request.CDBLen = cdblen;
-	c->Request.Type.Type = TYPE_CMD;
-	c->Request.Type.Attribute = ATTR_SIMPLE;
-	c->Request.Type.Direction = direction;
-
-	/* Fill in the SG list and do dma mapping */
-	cciss_map_one(h->pdev, c, (unsigned char *) buf,
-			bufsize, DMA_FROM_DEVICE);
-
-	c->waiting = &wait;
-	enqueue_cmd_and_start_io(h, c);
-	wait_for_completion(&wait);
-
-	/* undo the dma mapping */
-	cciss_unmap_one(h->pdev, c, bufsize, DMA_FROM_DEVICE);
-	return(0);
-}
-
-static void
-cciss_scsi_interpret_error(ctlr_info_t *h, CommandList_struct *c)
-{
-	ErrorInfo_struct *ei;
-
-	ei = c->err_info;
-	switch (ei->CommandStatus) {
-		case CMD_TARGET_STATUS:
-			dev_warn(&h->pdev->dev,
-				"cmd %p has completed with errors\n", c);
-			dev_warn(&h->pdev->dev,
-				"cmd %p has SCSI Status = %x\n",
-				c, ei->ScsiStatus);
-			if (ei->ScsiStatus == 0)
-				dev_warn(&h->pdev->dev,
-				"SCSI status is abnormally zero.  "
-				"(probably indicates selection timeout "
-				"reported incorrectly due to a known "
-				"firmware bug, circa July, 2001.)\n");
-		break;
-		case CMD_DATA_UNDERRUN: /* let mid layer handle it. */
-			dev_info(&h->pdev->dev, "UNDERRUN\n");
-		break;
-		case CMD_DATA_OVERRUN:
-			dev_warn(&h->pdev->dev, "%p has"
-				" completed with data overrun "
-				"reported\n", c);
-		break;
-		case CMD_INVALID: {
-			/* controller unfortunately reports SCSI passthru's */
-			/* to non-existent targets as invalid commands. */
-			dev_warn(&h->pdev->dev,
-				"%p is reported invalid (probably means "
-				"target device no longer present)\n", c);
-			/*
-			print_hex_dump(KERN_INFO, "", DUMP_PREFIX_OFFSET, 16, 1, c, sizeof(*c), false);
-			print_cmd(c);
-			 */
-			}
-		break;
-		case CMD_PROTOCOL_ERR:
-			dev_warn(&h->pdev->dev, "%p has protocol error\n", c);
-		break;
-		case CMD_HARDWARE_ERR:
-			/* cmd->result = DID_ERROR << 16; */
-			dev_warn(&h->pdev->dev, "%p had hardware error\n", c);
-		break;
-		case CMD_CONNECTION_LOST:
-			dev_warn(&h->pdev->dev, "%p had connection lost\n", c);
-		break;
-		case CMD_ABORTED:
-			dev_warn(&h->pdev->dev, "%p was aborted\n", c);
-		break;
-		case CMD_ABORT_FAILED:
-			dev_warn(&h->pdev->dev,
-				"%p reports abort failed\n", c);
-		break;
-		case CMD_UNSOLICITED_ABORT:
-			dev_warn(&h->pdev->dev,
-				"%p aborted due to an unsolicited abort\n", c);
-		break;
-		case CMD_TIMEOUT:
-			dev_warn(&h->pdev->dev, "%p timedout\n", c);
-		break;
-		case CMD_UNABORTABLE:
-			dev_warn(&h->pdev->dev,
-				"%p unabortable\n", c);
-		break;
-		default:
-			dev_warn(&h->pdev->dev,
-				"%p returned unknown status %x\n",
-				c, ei->CommandStatus);
-	}
-}
-
-static int
-cciss_scsi_do_inquiry(ctlr_info_t *h, unsigned char *scsi3addr,
-	unsigned char page, unsigned char *buf,
-	unsigned char bufsize)
-{
-	int rc;
-	CommandList_struct *c;
-	char cdb[6];
-	ErrorInfo_struct *ei;
-	unsigned long flags;
-
-	spin_lock_irqsave(&h->lock, flags);
-	c = scsi_cmd_alloc(h);
-	spin_unlock_irqrestore(&h->lock, flags);
-
-	if (c == NULL) {			/* trouble... */
-		printk("cmd_alloc returned NULL!\n");
-		return -1;
-	}
-
-	ei = c->err_info;
-
-	cdb[0] = CISS_INQUIRY;
-	cdb[1] = (page != 0);
-	cdb[2] = page;
-	cdb[3] = 0;
-	cdb[4] = bufsize;
-	cdb[5] = 0;
-	rc = cciss_scsi_do_simple_cmd(h, c, scsi3addr, cdb,
-				6, buf, bufsize, XFER_READ);
-
-	if (rc != 0) return rc; /* something went wrong */
-
-	if (ei->CommandStatus != 0 &&
-	    ei->CommandStatus != CMD_DATA_UNDERRUN) {
-		cciss_scsi_interpret_error(h, c);
-		rc = -1;
-	}
-	spin_lock_irqsave(&h->lock, flags);
-	scsi_cmd_free(h, c);
-	spin_unlock_irqrestore(&h->lock, flags);
-	return rc;
-}
-
-/* Get the device id from inquiry page 0x83 */
-static int cciss_scsi_get_device_id(ctlr_info_t *h, unsigned char *scsi3addr,
-	unsigned char *device_id, int buflen)
-{
-	int rc;
-	unsigned char *buf;
-
-	if (buflen > 16)
-		buflen = 16;
-	buf = kzalloc(64, GFP_KERNEL);
-	if (!buf)
-		return -1;
-	rc = cciss_scsi_do_inquiry(h, scsi3addr, 0x83, buf, 64);
-	if (rc == 0)
-		memcpy(device_id, &buf[8], buflen);
-	kfree(buf);
-	return rc != 0;
-}
-
-static int
-cciss_scsi_do_report_phys_luns(ctlr_info_t *h,
-		ReportLunData_struct *buf, int bufsize)
-{
-	int rc;
-	CommandList_struct *c;
-	unsigned char cdb[12];
-	unsigned char scsi3addr[8];
-	ErrorInfo_struct *ei;
-	unsigned long flags;
-
-	spin_lock_irqsave(&h->lock, flags);
-	c = scsi_cmd_alloc(h);
-	spin_unlock_irqrestore(&h->lock, flags);
-	if (c == NULL) {			/* trouble... */
-		printk("cmd_alloc returned NULL!\n");
-		return -1;
-	}
-
-	memset(&scsi3addr[0], 0, 8); /* address the controller */
-	cdb[0] = CISS_REPORT_PHYS;
-	cdb[1] = 0;
-	cdb[2] = 0;
-	cdb[3] = 0;
-	cdb[4] = 0;
-	cdb[5] = 0;
-	cdb[6] = (bufsize >> 24) & 0xFF;  //MSB
-	cdb[7] = (bufsize >> 16) & 0xFF;
-	cdb[8] = (bufsize >> 8) & 0xFF;
-	cdb[9] = bufsize & 0xFF;
-	cdb[10] = 0;
-	cdb[11] = 0;
-
-	rc = cciss_scsi_do_simple_cmd(h, c, scsi3addr,
-				cdb, 12,
-				(unsigned char *) buf,
-				bufsize, XFER_READ);
-
-	if (rc != 0) return rc; /* something went wrong */
-
-	ei = c->err_info;
-	if (ei->CommandStatus != 0 &&
-	    ei->CommandStatus != CMD_DATA_UNDERRUN) {
-		cciss_scsi_interpret_error(h, c);
-		rc = -1;
-	}
-	spin_lock_irqsave(&h->lock, flags);
-	scsi_cmd_free(h, c);
-	spin_unlock_irqrestore(&h->lock, flags);
-	return rc;
-}
-
-static void
-cciss_update_non_disk_devices(ctlr_info_t *h, int hostno)
-{
-	/* the idea here is we could get notified from /proc
-	   that some devices have changed, so we do a report
-	   physical luns cmd, and adjust our list of devices
-	   accordingly.  (We can't rely on the scsi-mid layer just
-	   doing inquiries, because the "busses" that the scsi
-	   mid-layer probes are totally fabricated by this driver,
-	   so new devices wouldn't show up.
-
-	   the scsi3addr's of devices won't change so long as the
-	   adapter is not reset.  That means we can rescan and
-	   tell which devices we already know about, vs. new
-	   devices, vs.  disappearing devices.
-
-	   Also, if you yank out a tape drive, then put in a disk
-	   in it's place, (say, a configured volume from another
-	   array controller for instance)  _don't_ poke this driver
-           (so it thinks it's still a tape, but _do_ poke the scsi
-           mid layer, so it does an inquiry... the scsi mid layer
-           will see the physical disk.  This would be bad.  Need to
-	   think about how to prevent that.  One idea would be to
-	   snoop all scsi responses and if an inquiry repsonse comes
-	   back that reports a disk, chuck it an return selection
-	   timeout instead and adjust our table...  Not sure i like
-	   that though.
-
-	 */
-#define OBDR_TAPE_INQ_SIZE 49
-#define OBDR_TAPE_SIG "$DR-10"
-	ReportLunData_struct *ld_buff;
-	unsigned char *inq_buff;
-	unsigned char scsi3addr[8];
-	__u32 num_luns=0;
-	unsigned char *ch;
-	struct cciss_scsi_dev_t *currentsd, *this_device;
-	int ncurrent=0;
-	int reportlunsize = sizeof(*ld_buff) + CISS_MAX_PHYS_LUN * 8;
-	int i;
-
-	ld_buff = kzalloc(reportlunsize, GFP_KERNEL);
-	inq_buff = kmalloc(OBDR_TAPE_INQ_SIZE, GFP_KERNEL);
-	currentsd = kzalloc(sizeof(*currentsd) *
-			(CCISS_MAX_SCSI_DEVS_PER_HBA+1), GFP_KERNEL);
-	if (ld_buff == NULL || inq_buff == NULL || currentsd == NULL) {
-		printk(KERN_ERR "cciss: out of memory\n");
-		goto out;
-	}
-	this_device = &currentsd[CCISS_MAX_SCSI_DEVS_PER_HBA];
-	if (cciss_scsi_do_report_phys_luns(h, ld_buff, reportlunsize) == 0) {
-		ch = &ld_buff->LUNListLength[0];
-		num_luns = ((ch[0]<<24) | (ch[1]<<16) | (ch[2]<<8) | ch[3]) / 8;
-		if (num_luns > CISS_MAX_PHYS_LUN) {
-			printk(KERN_WARNING
-				"cciss: Maximum physical LUNs (%d) exceeded.  "
-				"%d LUNs ignored.\n", CISS_MAX_PHYS_LUN,
-				num_luns - CISS_MAX_PHYS_LUN);
-			num_luns = CISS_MAX_PHYS_LUN;
-		}
-	}
-	else {
-		printk(KERN_ERR  "cciss: Report physical LUNs failed.\n");
-		goto out;
-	}
-
-
-	/* adjust our table of devices */
-	for (i = 0; i < num_luns; i++) {
-		/* for each physical lun, do an inquiry */
-		if (ld_buff->LUN[i][3] & 0xC0) continue;
-		memset(inq_buff, 0, OBDR_TAPE_INQ_SIZE);
-		memcpy(&scsi3addr[0], &ld_buff->LUN[i][0], 8);
-
-		if (cciss_scsi_do_inquiry(h, scsi3addr, 0, inq_buff,
-			(unsigned char) OBDR_TAPE_INQ_SIZE) != 0)
-			/* Inquiry failed (msg printed already) */
-			continue; /* so we will skip this device. */
-
-		this_device->devtype = (inq_buff[0] & 0x1f);
-		this_device->bus = -1;
-		this_device->target = -1;
-		this_device->lun = -1;
-		memcpy(this_device->scsi3addr, scsi3addr, 8);
-		memcpy(this_device->vendor, &inq_buff[8],
-			sizeof(this_device->vendor));
-		memcpy(this_device->model, &inq_buff[16],
-			sizeof(this_device->model));
-		memcpy(this_device->revision, &inq_buff[32],
-			sizeof(this_device->revision));
-		memset(this_device->device_id, 0,
-			sizeof(this_device->device_id));
-		cciss_scsi_get_device_id(h, scsi3addr,
-			this_device->device_id, sizeof(this_device->device_id));
-
-		switch (this_device->devtype) {
-		  case 0x05: /* CD-ROM */ {
-
-			/* We don't *really* support actual CD-ROM devices,
-			 * just this "One Button Disaster Recovery" tape drive
-			 * which temporarily pretends to be a CD-ROM drive.
-			 * So we check that the device is really an OBDR tape
-			 * device by checking for "$DR-10" in bytes 43-48 of
-			 * the inquiry data.
-			 */
-				char obdr_sig[7];
-
-				strncpy(obdr_sig, &inq_buff[43], 6);
-				obdr_sig[6] = '\0';
-				if (strncmp(obdr_sig, OBDR_TAPE_SIG, 6) != 0)
-					/* Not OBDR device, ignore it. */
-					break;
-			}
-			/* fall through . . . */
-		  case 0x01: /* sequential access, (tape) */
-		  case 0x08: /* medium changer */
-			if (ncurrent >= CCISS_MAX_SCSI_DEVS_PER_HBA) {
-				printk(KERN_INFO "cciss%d: %s ignored, "
-					"too many devices.\n", h->ctlr,
-					scsi_device_type(this_device->devtype));
-				break;
-			}
-			currentsd[ncurrent] = *this_device;
-			ncurrent++;
-			break;
-		  default:
-			break;
-		}
-	}
-
-	adjust_cciss_scsi_table(h, hostno, currentsd, ncurrent);
-out:
-	kfree(inq_buff);
-	kfree(ld_buff);
-	kfree(currentsd);
-	return;
-}
-
-static int
-is_keyword(char *ptr, int len, char *verb)  // Thanks to ncr53c8xx.c
-{
-	int verb_len = strlen(verb);
-	if (len >= verb_len && !memcmp(verb,ptr,verb_len))
-		return verb_len;
-	else
-		return 0;
-}
-
-static int
-cciss_scsi_user_command(ctlr_info_t *h, int hostno, char *buffer, int length)
-{
-	int arg_len;
-
-	if ((arg_len = is_keyword(buffer, length, "rescan")) != 0)
-		cciss_update_non_disk_devices(h, hostno);
-	else
-		return -EINVAL;
-	return length;
-}
-
-static int
-cciss_scsi_write_info(struct Scsi_Host *sh,
-		char *buffer, /* data buffer */
-		int length) 	   /* length of data in buffer */
-{
-	ctlr_info_t *h = (ctlr_info_t *) sh->hostdata[0];
-	if (h == NULL)  /* This really shouldn't ever happen. */
-		return -EINVAL;
-
-	return cciss_scsi_user_command(h, sh->host_no,
-			buffer, length);
-}
-
-static int
-cciss_scsi_show_info(struct seq_file *m, struct Scsi_Host *sh)
-{
-
-	ctlr_info_t *h = (ctlr_info_t *) sh->hostdata[0];
-	int i;
-
-	if (h == NULL)  /* This really shouldn't ever happen. */
-		return -EINVAL;
-
-	seq_printf(m, "cciss%d: SCSI host: %d\n",
-			h->ctlr, sh->host_no);
-
-	/* this information is needed by apps to know which cciss
-	   device corresponds to which scsi host number without
-	   having to open a scsi target device node.  The device
-	   information is not a duplicate of /proc/scsi/scsi because
-	   the two may be out of sync due to scsi hotplug, rather
-	   this info is for an app to be able to use to know how to
-	   get them back in sync. */
-
-	for (i = 0; i < ccissscsi[h->ctlr].ndevices; i++) {
-		struct cciss_scsi_dev_t *sd =
-			&ccissscsi[h->ctlr].dev[i];
-		seq_printf(m, "c%db%dt%dl%d %02d "
-			"0x%02x%02x%02x%02x%02x%02x%02x%02x\n",
-			sh->host_no, sd->bus, sd->target, sd->lun,
-			sd->devtype,
-			sd->scsi3addr[0], sd->scsi3addr[1],
-			sd->scsi3addr[2], sd->scsi3addr[3],
-			sd->scsi3addr[4], sd->scsi3addr[5],
-			sd->scsi3addr[6], sd->scsi3addr[7]);
-	}
-	return 0;
-}
-
-/* cciss_scatter_gather takes a struct scsi_cmnd, (cmd), and does the pci
-   dma mapping  and fills in the scatter gather entries of the
-   cciss command, c. */
-
-static void cciss_scatter_gather(ctlr_info_t *h, CommandList_struct *c,
-	struct scsi_cmnd *cmd)
-{
-	unsigned int len;
-	struct scatterlist *sg;
-	__u64 addr64;
-	int request_nsgs, i, chained, sg_index;
-	struct cciss_scsi_adapter_data_t *sa = h->scsi_ctlr;
-	SGDescriptor_struct *curr_sg;
-
-	BUG_ON(scsi_sg_count(cmd) > h->maxsgentries);
-
-	chained = 0;
-	sg_index = 0;
-	curr_sg = c->SG;
-	request_nsgs = scsi_dma_map(cmd);
-	if (request_nsgs) {
-		scsi_for_each_sg(cmd, sg, request_nsgs, i) {
-			if (sg_index + 1 == h->max_cmd_sgentries &&
-				!chained && request_nsgs - i > 1) {
-				chained = 1;
-				sg_index = 0;
-				curr_sg = sa->cmd_sg_list[c->cmdindex];
-			}
-			addr64 = (__u64) sg_dma_address(sg);
-			len  = sg_dma_len(sg);
-			curr_sg[sg_index].Addr.lower =
-				(__u32) (addr64 & 0x0FFFFFFFFULL);
-			curr_sg[sg_index].Addr.upper =
-				(__u32) ((addr64 >> 32) & 0x0FFFFFFFFULL);
-			curr_sg[sg_index].Len = len;
-			curr_sg[sg_index].Ext = 0;
-			++sg_index;
-		}
-		if (chained)
-			cciss_map_sg_chain_block(h, c,
-				sa->cmd_sg_list[c->cmdindex],
-				(request_nsgs - (h->max_cmd_sgentries - 1)) *
-					sizeof(SGDescriptor_struct));
-	}
-	/* track how many SG entries we are using */
-	if (request_nsgs > h->maxSG)
-		h->maxSG = request_nsgs;
-	c->Header.SGTotal = (u16) request_nsgs + chained;
-	if (request_nsgs > h->max_cmd_sgentries)
-		c->Header.SGList = h->max_cmd_sgentries;
-	else
-		c->Header.SGList = c->Header.SGTotal;
-	return;
-}
-
-
-static int
-cciss_scsi_queue_command_lck(struct scsi_cmnd *cmd, void (*done)(struct scsi_cmnd *))
-{
-	ctlr_info_t *h;
-	int rc;
-	unsigned char scsi3addr[8];
-	CommandList_struct *c;
-	unsigned long flags;
-
-	// Get the ptr to our adapter structure (hba[i]) out of cmd->host.
-	// We violate cmd->host privacy here.  (Is there another way?)
-	h = (ctlr_info_t *) cmd->device->host->hostdata[0];
-
-	rc = lookup_scsi3addr(h, cmd->device->channel, cmd->device->id,
-			cmd->device->lun, scsi3addr);
-	if (rc != 0) {
-		/* the scsi nexus does not match any that we presented... */
-		/* pretend to mid layer that we got selection timeout */
-		cmd->result = DID_NO_CONNECT << 16;
-		done(cmd);
-		/* we might want to think about registering controller itself
-		   as a processor device on the bus so sg binds to it. */
-		return 0;
-	}
-
-	/* Ok, we have a reasonable scsi nexus, so send the cmd down, and
-           see what the device thinks of it. */
-
-	spin_lock_irqsave(&h->lock, flags);
-	c = scsi_cmd_alloc(h);
-	spin_unlock_irqrestore(&h->lock, flags);
-	if (c == NULL) {			/* trouble... */
-		dev_warn(&h->pdev->dev, "scsi_cmd_alloc returned NULL!\n");
-		/* FIXME: next 3 lines are -> BAD! <- */
-		cmd->result = DID_NO_CONNECT << 16;
-		done(cmd);
-		return 0;
-	}
-
-	// Fill in the command list header
-
-	cmd->scsi_done = done;    // save this for use by completion code
-
-	/* save c in case we have to abort it */
-	cmd->host_scribble = (unsigned char *) c;
-
-	c->cmd_type = CMD_SCSI;
-	c->scsi_cmd = cmd;
-	c->Header.ReplyQueue = 0;  /* unused in simple mode */
-	memcpy(&c->Header.LUN.LunAddrBytes[0], &scsi3addr[0], 8);
-	c->Header.Tag.lower = c->busaddr;  /* Use k. address of cmd as tag */
-
-	// Fill in the request block...
-
-	c->Request.Timeout = 0;
-	memset(c->Request.CDB, 0, sizeof(c->Request.CDB));
-	BUG_ON(cmd->cmd_len > sizeof(c->Request.CDB));
-	c->Request.CDBLen = cmd->cmd_len;
-	memcpy(c->Request.CDB, cmd->cmnd, cmd->cmd_len);
-	c->Request.Type.Type = TYPE_CMD;
-	c->Request.Type.Attribute = ATTR_SIMPLE;
-	switch (cmd->sc_data_direction) {
-	  case DMA_TO_DEVICE:
-		c->Request.Type.Direction = XFER_WRITE;
-		break;
-	  case DMA_FROM_DEVICE:
-		c->Request.Type.Direction = XFER_READ;
-		break;
-	  case DMA_NONE:
-		c->Request.Type.Direction = XFER_NONE;
-		break;
-	  case DMA_BIDIRECTIONAL:
-		// This can happen if a buggy application does a scsi passthru
-		// and sets both inlen and outlen to non-zero. ( see
-		// ../scsi/scsi_ioctl.c:scsi_ioctl_send_command() )
-
-		c->Request.Type.Direction = XFER_RSVD;
-		// This is technically wrong, and cciss controllers should
-		// reject it with CMD_INVALID, which is the most correct
-		// response, but non-fibre backends appear to let it
-		// slide by, and give the same results as if this field
-		// were set correctly.  Either way is acceptable for
-		// our purposes here.
-
-		break;
-
-	  default:
-		dev_warn(&h->pdev->dev, "unknown data direction: %d\n",
-			cmd->sc_data_direction);
-		BUG();
-		break;
-	}
-	cciss_scatter_gather(h, c, cmd);
-	enqueue_cmd_and_start_io(h, c);
-	/* the cmd'll come back via intr handler in complete_scsi_command()  */
-	return 0;
-}
-
-static DEF_SCSI_QCMD(cciss_scsi_queue_command)
-
-static void cciss_unregister_scsi(ctlr_info_t *h)
-{
-	struct cciss_scsi_adapter_data_t *sa;
-	struct cciss_scsi_cmd_stack_t *stk;
-	unsigned long flags;
-
-	/* we are being forcibly unloaded, and may not refuse. */
-
-	spin_lock_irqsave(&h->lock, flags);
-	sa = h->scsi_ctlr;
-	stk = &sa->cmd_stack;
-
-	/* if we weren't ever actually registered, don't unregister */
-	if (sa->registered) {
-		spin_unlock_irqrestore(&h->lock, flags);
-		scsi_remove_host(sa->scsi_host);
-		scsi_host_put(sa->scsi_host);
-		spin_lock_irqsave(&h->lock, flags);
-	}
-
-	/* set scsi_host to NULL so our detect routine will
-	   find us on register */
-	sa->scsi_host = NULL;
-	spin_unlock_irqrestore(&h->lock, flags);
-	scsi_cmd_stack_free(h);
-	kfree(sa);
-}
-
-static int cciss_engage_scsi(ctlr_info_t *h)
-{
-	struct cciss_scsi_adapter_data_t *sa;
-	struct cciss_scsi_cmd_stack_t *stk;
-	unsigned long flags;
-
-	spin_lock_irqsave(&h->lock, flags);
-	sa = h->scsi_ctlr;
-	stk = &sa->cmd_stack;
-
-	if (sa->registered) {
-		dev_info(&h->pdev->dev, "SCSI subsystem already engaged.\n");
-		spin_unlock_irqrestore(&h->lock, flags);
-		return -ENXIO;
-	}
-	sa->registered = 1;
-	spin_unlock_irqrestore(&h->lock, flags);
-	cciss_update_non_disk_devices(h, -1);
-	cciss_scsi_detect(h);
-	return 0;
-}
-
-static void
-cciss_seq_tape_report(struct seq_file *seq, ctlr_info_t *h)
-{
-	unsigned long flags;
-
-	CPQ_TAPE_LOCK(h, flags);
-	seq_printf(seq,
-		"Sequential access devices: %d\n\n",
-			ccissscsi[h->ctlr].ndevices);
-	CPQ_TAPE_UNLOCK(h, flags);
-}
-
-static int wait_for_device_to_become_ready(ctlr_info_t *h,
-	unsigned char lunaddr[])
-{
-	int rc;
-	int count = 0;
-	int waittime = HZ;
-	CommandList_struct *c;
-
-	c = cmd_alloc(h);
-	if (!c) {
-		dev_warn(&h->pdev->dev, "out of memory in "
-			"wait_for_device_to_become_ready.\n");
-		return IO_ERROR;
-	}
-
-	/* Send test unit ready until device ready, or give up. */
-	while (count < 20) {
-
-		/* Wait for a bit.  do this first, because if we send
-		 * the TUR right away, the reset will just abort it.
-		 */
-		schedule_timeout_uninterruptible(waittime);
-		count++;
-
-		/* Increase wait time with each try, up to a point. */
-		if (waittime < (HZ * 30))
-			waittime = waittime * 2;
-
-		/* Send the Test Unit Ready */
-		rc = fill_cmd(h, c, TEST_UNIT_READY, NULL, 0, 0,
-			lunaddr, TYPE_CMD);
-		if (rc == 0)
-			rc = sendcmd_withirq_core(h, c, 0);
-
-		(void) process_sendcmd_error(h, c);
-
-		if (rc != 0)
-			goto retry_tur;
-
-		if (c->err_info->CommandStatus == CMD_SUCCESS)
-			break;
-
-		if (c->err_info->CommandStatus == CMD_TARGET_STATUS &&
-			c->err_info->ScsiStatus == SAM_STAT_CHECK_CONDITION) {
-			if (c->err_info->SenseInfo[2] == NO_SENSE)
-				break;
-			if (c->err_info->SenseInfo[2] == UNIT_ATTENTION) {
-				unsigned char asc;
-				asc = c->err_info->SenseInfo[12];
-				check_for_unit_attention(h, c);
-				if (asc == POWER_OR_RESET)
-					break;
-			}
-		}
-retry_tur:
-		dev_warn(&h->pdev->dev, "Waiting %d secs "
-			"for device to become ready.\n",
-			waittime / HZ);
-		rc = 1; /* device not ready. */
-	}
-
-	if (rc)
-		dev_warn(&h->pdev->dev, "giving up on device.\n");
-	else
-		dev_warn(&h->pdev->dev, "device is ready.\n");
-
-	cmd_free(h, c);
-	return rc;
-}
-
-/* Need at least one of these error handlers to keep ../scsi/hosts.c from
- * complaining.  Doing a host- or bus-reset can't do anything good here.
- * Despite what it might say in scsi_error.c, there may well be commands
- * on the controller, as the cciss driver registers twice, once as a block
- * device for the logical drives, and once as a scsi device, for any tape
- * drives.  So we know there are no commands out on the tape drives, but we
- * don't know there are no commands on the controller, and it is likely
- * that there probably are, as the cciss block device is most commonly used
- * as a boot device (embedded controller on HP/Compaq systems.)
-*/
-
-static int cciss_eh_device_reset_handler(struct scsi_cmnd *scsicmd)
-{
-	int rc;
-	CommandList_struct *cmd_in_trouble;
-	unsigned char lunaddr[8];
-	ctlr_info_t *h;
-
-	/* find the controller to which the command to be aborted was sent */
-	h = (ctlr_info_t *) scsicmd->device->host->hostdata[0];
-	if (h == NULL) /* paranoia */
-		return FAILED;
-	dev_warn(&h->pdev->dev, "resetting tape drive or medium changer.\n");
-	/* find the command that's giving us trouble */
-	cmd_in_trouble = (CommandList_struct *) scsicmd->host_scribble;
-	if (cmd_in_trouble == NULL) /* paranoia */
-		return FAILED;
-	memcpy(lunaddr, &cmd_in_trouble->Header.LUN.LunAddrBytes[0], 8);
-	/* send a reset to the SCSI LUN which the command was sent to */
-	rc = sendcmd_withirq(h, CCISS_RESET_MSG, NULL, 0, 0, lunaddr,
-		TYPE_MSG);
-	if (rc == 0 && wait_for_device_to_become_ready(h, lunaddr) == 0)
-		return SUCCESS;
-	dev_warn(&h->pdev->dev, "resetting device failed.\n");
-	return FAILED;
-}
-
-static int  cciss_eh_abort_handler(struct scsi_cmnd *scsicmd)
-{
-	int rc;
-	CommandList_struct *cmd_to_abort;
-	unsigned char lunaddr[8];
-	ctlr_info_t *h;
-
-	/* find the controller to which the command to be aborted was sent */
-	h = (ctlr_info_t *) scsicmd->device->host->hostdata[0];
-	if (h == NULL) /* paranoia */
-		return FAILED;
-	dev_warn(&h->pdev->dev, "aborting tardy SCSI cmd\n");
-
-	/* find the command to be aborted */
-	cmd_to_abort = (CommandList_struct *) scsicmd->host_scribble;
-	if (cmd_to_abort == NULL) /* paranoia */
-		return FAILED;
-	memcpy(lunaddr, &cmd_to_abort->Header.LUN.LunAddrBytes[0], 8);
-	rc = sendcmd_withirq(h, CCISS_ABORT_MSG, &cmd_to_abort->Header.Tag,
-		0, 0, lunaddr, TYPE_MSG);
-	if (rc == 0)
-		return SUCCESS;
-	return FAILED;
-
-}
-
-#else /* no CONFIG_CISS_SCSI_TAPE */
-
-/* If no tape support, then these become defined out of existence */
-
-#define cciss_scsi_setup(cntl_num)
-#define cciss_engage_scsi(h)
-
-#endif /* CONFIG_CISS_SCSI_TAPE */
diff --git a/drivers/block/cciss_scsi.h b/drivers/block/cciss_scsi.h
deleted file mode 100644
index e71d986727ca..000000000000
--- a/drivers/block/cciss_scsi.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- *    Disk Array driver for HP Smart Array controllers, SCSI Tape module.
- *    (C) Copyright 2001, 2007 Hewlett-Packard Development Company, L.P.
- *
- *    This program is free software; you can redistribute it and/or modify
- *    it under the terms of the GNU General Public License as published by
- *    the Free Software Foundation; version 2 of the License.
- *
- *    This program is distributed in the hope that it will be useful,
- *    but WITHOUT ANY WARRANTY; without even the implied warranty of
- *    MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- *    General Public License for more details.
- *
- *    You should have received a copy of the GNU General Public License
- *    along with this program; if not, write to the Free Software
- *    Foundation, Inc., 59 Temple Place, Suite 300, Boston, MA
- *    02111-1307, USA.
- *
- *    Questions/Comments/Bugfixes to iss_storagedev@hp.com
- *
- */
-#ifdef CONFIG_CISS_SCSI_TAPE
-#ifndef _CCISS_SCSI_H_
-#define _CCISS_SCSI_H_
-
-#include <scsi/scsicam.h> /* possibly irrelevant, since we don't show disks */
-
-		/* the scsi id of the adapter... */
-#define SELF_SCSI_ID 15
-		/* 15 is somewhat arbitrary, since the scsi-2 bus
-		   that's presented by the driver to the OS is
-		   fabricated.  The "real" scsi-3 bus the
-		   hardware presents is fabricated too.
-		   The actual, honest-to-goodness physical
-		   bus that the devices are attached to is not
-		   addressible natively, and may in fact turn
-		   out to be not scsi at all. */
-
-
-/* 
-
-If the upper scsi layer tries to track how many commands we have 
-outstanding, it will be operating under the misapprehension that it is
-the only one sending us requests.  We also have the block interface,
-which is where most requests must surely come from, so the upper layer's
-notion of how many requests we have outstanding will be wrong most or
-all of the time. 
-
-Note, the normal SCSI mid-layer error handling doesn't work well
-for this driver because 1) it takes the io_request_lock before
-calling error handlers and uses a local variable to store flags,
-so the io_request_lock cannot be released and interrupts enabled
-inside the error handlers, and, the error handlers cannot poll
-for command completion because they might get commands from the
-block half of the driver completing, and not know what to do
-with them.  That's what we get for making a hybrid scsi/block
-driver, I suppose.
-
-*/
-
-struct cciss_scsi_dev_t {
-	int devtype;
-	int bus, target, lun;		/* as presented to the OS */
-	unsigned char scsi3addr[8];	/* as presented to the HW */
-	unsigned char device_id[16];	/* from inquiry pg. 0x83 */
-	unsigned char vendor[8];	/* bytes 8-15 of inquiry data */
-	unsigned char model[16];	/* bytes 16-31 of inquiry data */
-	unsigned char revision[4];	/* bytes 32-35 of inquiry data */
-};
-
-struct cciss_scsi_hba_t {
-	char *name;
-	int ndevices;
-#define CCISS_MAX_SCSI_DEVS_PER_HBA 16
-	struct cciss_scsi_dev_t dev[CCISS_MAX_SCSI_DEVS_PER_HBA];
-};
-
-#endif /* _CCISS_SCSI_H_ */
-#endif /* CONFIG_CISS_SCSI_TAPE */
diff --git a/drivers/block/drbd/drbd_actlog.c b/drivers/block/drbd/drbd_actlog.c
index e02c45cd3c5a..5f0eaee8c8a7 100644
--- a/drivers/block/drbd/drbd_actlog.c
+++ b/drivers/block/drbd/drbd_actlog.c
@@ -151,7 +151,7 @@ static int _drbd_md_sync_page_io(struct drbd_device *device,
 	op_flags |= REQ_SYNC;
 
 	bio = bio_alloc_drbd(GFP_NOIO);
-	bio->bi_bdev = bdev->md_bdev;
+	bio_set_dev(bio, bdev->md_bdev);
 	bio->bi_iter.bi_sector = sector;
 	err = -EIO;
 	if (bio_add_page(bio, device->md_io.page, size, 0) != size)
diff --git a/drivers/block/drbd/drbd_bitmap.c b/drivers/block/drbd/drbd_bitmap.c
index 809fd245c3dc..bd97908c766f 100644
--- a/drivers/block/drbd/drbd_bitmap.c
+++ b/drivers/block/drbd/drbd_bitmap.c
@@ -1019,7 +1019,7 @@ static void bm_page_io_async(struct drbd_bm_aio_ctx *ctx, int page_nr) __must_ho
 		bm_store_page_idx(page, page_nr);
 	} else
 		page = b->bm_pages[page_nr];
-	bio->bi_bdev = device->ldev->md_bdev;
+	bio_set_dev(bio, device->ldev->md_bdev);
 	bio->bi_iter.bi_sector = on_disk_sector;
 	/* bio_add_page of a single page to an empty bio will always succeed,
 	 * according to api.  Do we want to assert that? */
diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index d17b6e6393c7..7e8589ce631c 100644
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -63,19 +63,15 @@
 # define __must_hold(x)
 #endif
 
-/* module parameter, defined in drbd_main.c */
-extern unsigned int minor_count;
-extern bool disable_sendpage;
-extern bool allow_oos;
-void tl_abort_disk_io(struct drbd_device *device);
-
+/* shared module parameters, defined in drbd_main.c */
 #ifdef CONFIG_DRBD_FAULT_INJECTION
-extern int enable_faults;
-extern int fault_rate;
-extern int fault_devs;
+extern int drbd_enable_faults;
+extern int drbd_fault_rate;
 #endif
 
-extern char usermode_helper[];
+extern unsigned int drbd_minor_count;
+extern char drbd_usermode_helper[];
+extern int drbd_proc_details;
 
 
 /* This is used to stop/restart our threads.
@@ -181,8 +177,8 @@ _drbd_insert_fault(struct drbd_device *device, unsigned int type);
 static inline int
 drbd_insert_fault(struct drbd_device *device, unsigned int type) {
 #ifdef CONFIG_DRBD_FAULT_INJECTION
-	return fault_rate &&
-		(enable_faults & (1<<type)) &&
+	return drbd_fault_rate &&
+		(drbd_enable_faults & (1<<type)) &&
 		_drbd_insert_fault(device, type);
 #else
 	return 0;
@@ -745,6 +741,8 @@ struct drbd_connection {
 	unsigned current_tle_writes;	/* writes seen within this tl epoch */
 
 	unsigned long last_reconnect_jif;
+	/* empty member on older kernels without blk_start_plug() */
+	struct blk_plug receiver_plug;
 	struct drbd_thread receiver;
 	struct drbd_thread worker;
 	struct drbd_thread ack_receiver;
@@ -1131,7 +1129,8 @@ extern void conn_send_sr_reply(struct drbd_connection *connection, enum drbd_sta
 extern int drbd_send_rs_deallocated(struct drbd_peer_device *, struct drbd_peer_request *);
 extern void drbd_backing_dev_free(struct drbd_device *device, struct drbd_backing_dev *ldev);
 extern void drbd_device_cleanup(struct drbd_device *device);
-void drbd_print_uuids(struct drbd_device *device, const char *text);
+extern void drbd_print_uuids(struct drbd_device *device, const char *text);
+extern void drbd_queue_unplug(struct drbd_device *device);
 
 extern void conn_md_sync(struct drbd_connection *connection);
 extern void drbd_md_write(struct drbd_device *device, void *buffer);
@@ -1463,8 +1462,6 @@ extern struct drbd_resource *drbd_find_resource(const char *name);
 extern void drbd_destroy_resource(struct kref *kref);
 extern void conn_free_crypto(struct drbd_connection *connection);
 
-extern int proc_details;
-
 /* drbd_req */
 extern void do_submit(struct work_struct *ws);
 extern void __drbd_make_request(struct drbd_device *, struct bio *, unsigned long);
@@ -1628,8 +1625,8 @@ static inline void drbd_generic_make_request(struct drbd_device *device,
 					     int fault_type, struct bio *bio)
 {
 	__release(local);
-	if (!bio->bi_bdev) {
-		drbd_err(device, "drbd_generic_make_request: bio->bi_bdev == NULL\n");
+	if (!bio->bi_disk) {
+		drbd_err(device, "drbd_generic_make_request: bio->bi_disk == NULL\n");
 		bio->bi_status = BLK_STS_IOERR;
 		bio_endio(bio);
 		return;
diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c
index e2ed28d45ce1..8cb3791898ae 100644
--- a/drivers/block/drbd/drbd_main.c
+++ b/drivers/block/drbd/drbd_main.c
@@ -77,41 +77,41 @@ MODULE_PARM_DESC(minor_count, "Approximate number of drbd devices ("
 MODULE_ALIAS_BLOCKDEV_MAJOR(DRBD_MAJOR);
 
 #include <linux/moduleparam.h>
-/* allow_open_on_secondary */
-MODULE_PARM_DESC(allow_oos, "DONT USE!");
 /* thanks to these macros, if compiled into the kernel (not-module),
- * this becomes the boot parameter drbd.minor_count */
-module_param(minor_count, uint, 0444);
-module_param(disable_sendpage, bool, 0644);
-module_param(allow_oos, bool, 0);
-module_param(proc_details, int, 0644);
+ * these become boot parameters (e.g., drbd.minor_count) */
 
 #ifdef CONFIG_DRBD_FAULT_INJECTION
-int enable_faults;
-int fault_rate;
-static int fault_count;
-int fault_devs;
+int drbd_enable_faults;
+int drbd_fault_rate;
+static int drbd_fault_count;
+static int drbd_fault_devs;
 /* bitmap of enabled faults */
-module_param(enable_faults, int, 0664);
+module_param_named(enable_faults, drbd_enable_faults, int, 0664);
 /* fault rate % value - applies to all enabled faults */
-module_param(fault_rate, int, 0664);
+module_param_named(fault_rate, drbd_fault_rate, int, 0664);
 /* count of faults inserted */
-module_param(fault_count, int, 0664);
+module_param_named(fault_count, drbd_fault_count, int, 0664);
 /* bitmap of devices to insert faults on */
-module_param(fault_devs, int, 0644);
+module_param_named(fault_devs, drbd_fault_devs, int, 0644);
 #endif
 
-/* module parameter, defined */
-unsigned int minor_count = DRBD_MINOR_COUNT_DEF;
-bool disable_sendpage;
-bool allow_oos;
-int proc_details;       /* Detail level in proc drbd*/
-
+/* module parameters we can keep static */
+static bool drbd_allow_oos; /* allow_open_on_secondary */
+static bool drbd_disable_sendpage;
+MODULE_PARM_DESC(allow_oos, "DONT USE!");
+module_param_named(allow_oos, drbd_allow_oos, bool, 0);
+module_param_named(disable_sendpage, drbd_disable_sendpage, bool, 0644);
+
+/* module parameters we share */
+int drbd_proc_details; /* Detail level in proc drbd*/
+module_param_named(proc_details, drbd_proc_details, int, 0644);
+/* module parameters shared with defaults */
+unsigned int drbd_minor_count = DRBD_MINOR_COUNT_DEF;
 /* Module parameter for setting the user mode helper program
  * to run. Default is /sbin/drbdadm */
-char usermode_helper[80] = "/sbin/drbdadm";
-
-module_param_string(usermode_helper, usermode_helper, sizeof(usermode_helper), 0644);
+char drbd_usermode_helper[80] = "/sbin/drbdadm";
+module_param_named(minor_count, drbd_minor_count, uint, 0444);
+module_param_string(usermode_helper, drbd_usermode_helper, sizeof(drbd_usermode_helper), 0644);
 
 /* in 2.6.x, our device mapping and config info contains our virtual gendisks
  * as member "struct gendisk *vdisk;"
@@ -923,7 +923,9 @@ void drbd_gen_and_send_sync_uuid(struct drbd_peer_device *peer_device)
 }
 
 /* communicated if (agreed_features & DRBD_FF_WSAME) */
-void assign_p_sizes_qlim(struct drbd_device *device, struct p_sizes *p, struct request_queue *q)
+static void
+assign_p_sizes_qlim(struct drbd_device *device, struct p_sizes *p,
+					struct request_queue *q)
 {
 	if (q) {
 		p->qlim->physical_block_size = cpu_to_be32(queue_physical_block_size(q));
@@ -1560,7 +1562,7 @@ static int _drbd_send_page(struct drbd_peer_device *peer_device, struct page *pa
 	 * put_page(); and would cause either a VM_BUG directly, or
 	 * __page_cache_release a page that would actually still be referenced
 	 * by someone, leading to some obscure delayed Oops somewhere else. */
-	if (disable_sendpage || (page_count(page) < 1) || PageSlab(page))
+	if (drbd_disable_sendpage || (page_count(page) < 1) || PageSlab(page))
 		return _drbd_no_send_page(peer_device, page, offset, size, msg_flags);
 
 	msg_flags |= MSG_NOSIGNAL;
@@ -1932,7 +1934,7 @@ static int drbd_open(struct block_device *bdev, fmode_t mode)
 	if (device->state.role != R_PRIMARY) {
 		if (mode & FMODE_WRITE)
 			rv = -EROFS;
-		else if (!allow_oos)
+		else if (!drbd_allow_oos)
 			rv = -EMEDIUMTYPE;
 	}
 
@@ -1952,6 +1954,19 @@ static void drbd_release(struct gendisk *gd, fmode_t mode)
 	mutex_unlock(&drbd_main_mutex);
 }
 
+/* need to hold resource->req_lock */
+void drbd_queue_unplug(struct drbd_device *device)
+{
+	if (device->state.pdsk >= D_INCONSISTENT && device->state.conn >= C_CONNECTED) {
+		D_ASSERT(device, device->state.role == R_PRIMARY);
+		if (test_and_clear_bit(UNPLUG_REMOTE, &device->flags)) {
+			drbd_queue_work_if_unqueued(
+				&first_peer_device(device)->connection->sender_work,
+				&device->unplug_work);
+		}
+	}
+}
+
 static void drbd_set_defaults(struct drbd_device *device)
 {
 	/* Beware! The actual layout differs
@@ -2008,18 +2023,14 @@ void drbd_init_set_defaults(struct drbd_device *device)
 	device->unplug_work.cb  = w_send_write_hint;
 	device->bm_io_work.w.cb = w_bitmap_io;
 
-	init_timer(&device->resync_timer);
-	init_timer(&device->md_sync_timer);
-	init_timer(&device->start_resync_timer);
-	init_timer(&device->request_timer);
-	device->resync_timer.function = resync_timer_fn;
-	device->resync_timer.data = (unsigned long) device;
-	device->md_sync_timer.function = md_sync_timer_fn;
-	device->md_sync_timer.data = (unsigned long) device;
-	device->start_resync_timer.function = start_resync_timer_fn;
-	device->start_resync_timer.data = (unsigned long) device;
-	device->request_timer.function = request_timer_fn;
-	device->request_timer.data = (unsigned long) device;
+	setup_timer(&device->resync_timer, resync_timer_fn,
+			(unsigned long)device);
+	setup_timer(&device->md_sync_timer, md_sync_timer_fn,
+			(unsigned long)device);
+	setup_timer(&device->start_resync_timer, start_resync_timer_fn,
+			(unsigned long)device);
+	setup_timer(&device->request_timer, request_timer_fn,
+			(unsigned long)device);
 
 	init_waitqueue_head(&device->misc_wait);
 	init_waitqueue_head(&device->state_wait);
@@ -2131,7 +2142,7 @@ static void drbd_destroy_mempools(void)
 static int drbd_create_mempools(void)
 {
 	struct page *page;
-	const int number = (DRBD_MAX_BIO_SIZE/PAGE_SIZE) * minor_count;
+	const int number = (DRBD_MAX_BIO_SIZE/PAGE_SIZE) * drbd_minor_count;
 	int i;
 
 	/* prepare our caches and mempools */
@@ -2167,13 +2178,12 @@ static int drbd_create_mempools(void)
 		goto Enomem;
 
 	/* mempools */
-	drbd_io_bio_set = bioset_create(BIO_POOL_SIZE, 0, BIOSET_NEED_RESCUER);
+	drbd_io_bio_set = bioset_create(BIO_POOL_SIZE, 0, 0);
 	if (drbd_io_bio_set == NULL)
 		goto Enomem;
 
 	drbd_md_io_bio_set = bioset_create(DRBD_MIN_POOL_PAGES, 0,
-					   BIOSET_NEED_BVECS |
-					   BIOSET_NEED_RESCUER);
+					   BIOSET_NEED_BVECS);
 	if (drbd_md_io_bio_set == NULL)
 		goto Enomem;
 
@@ -2409,7 +2419,6 @@ static void drbd_cleanup(void)
 		destroy_workqueue(retry.wq);
 
 	drbd_genl_unregister();
-	drbd_debugfs_cleanup();
 
 	idr_for_each_entry(&drbd_devices, device, i)
 		drbd_delete_device(device);
@@ -2420,6 +2429,8 @@ static void drbd_cleanup(void)
 		drbd_free_resource(resource);
 	}
 
+	drbd_debugfs_cleanup();
+
 	drbd_destroy_mempools();
 	unregister_blkdev(DRBD_MAJOR, "drbd");
 
@@ -2972,12 +2983,12 @@ static int __init drbd_init(void)
 {
 	int err;
 
-	if (minor_count < DRBD_MINOR_COUNT_MIN || minor_count > DRBD_MINOR_COUNT_MAX) {
-		pr_err("invalid minor_count (%d)\n", minor_count);
+	if (drbd_minor_count < DRBD_MINOR_COUNT_MIN || drbd_minor_count > DRBD_MINOR_COUNT_MAX) {
+		pr_err("invalid minor_count (%d)\n", drbd_minor_count);
 #ifdef MODULE
 		return -EINVAL;
 #else
-		minor_count = DRBD_MINOR_COUNT_DEF;
+		drbd_minor_count = DRBD_MINOR_COUNT_DEF;
 #endif
 	}
 
@@ -3900,12 +3911,12 @@ _drbd_insert_fault(struct drbd_device *device, unsigned int type)
 	static struct fault_random_state rrs = {0, 0};
 
 	unsigned int ret = (
-		(fault_devs == 0 ||
-			((1 << device_to_minor(device)) & fault_devs) != 0) &&
-		(((_drbd_fault_random(&rrs) % 100) + 1) <= fault_rate));
+		(drbd_fault_devs == 0 ||
+			((1 << device_to_minor(device)) & drbd_fault_devs) != 0) &&
+		(((_drbd_fault_random(&rrs) % 100) + 1) <= drbd_fault_rate));
 
 	if (ret) {
-		fault_count++;
+		drbd_fault_count++;
 
 		if (__ratelimit(&drbd_ratelimit_state))
 			drbd_warn(device, "***Simulating %s failure\n",
diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c
index ad0fcb43e45c..a12f77e6891e 100644
--- a/drivers/block/drbd/drbd_nl.c
+++ b/drivers/block/drbd/drbd_nl.c
@@ -344,7 +344,7 @@ int drbd_khelper(struct drbd_device *device, char *cmd)
 			 (char[60]) { }, /* address */
 			NULL };
 	char mb[14];
-	char *argv[] = {usermode_helper, cmd, mb, NULL };
+	char *argv[] = {drbd_usermode_helper, cmd, mb, NULL };
 	struct drbd_connection *connection = first_peer_device(device)->connection;
 	struct sib_info sib;
 	int ret;
@@ -359,19 +359,19 @@ int drbd_khelper(struct drbd_device *device, char *cmd)
 	 * write out any unsynced meta data changes now */
 	drbd_md_sync(device);
 
-	drbd_info(device, "helper command: %s %s %s\n", usermode_helper, cmd, mb);
+	drbd_info(device, "helper command: %s %s %s\n", drbd_usermode_helper, cmd, mb);
 	sib.sib_reason = SIB_HELPER_PRE;
 	sib.helper_name = cmd;
 	drbd_bcast_event(device, &sib);
 	notify_helper(NOTIFY_CALL, device, connection, cmd, 0);
-	ret = call_usermodehelper(usermode_helper, argv, envp, UMH_WAIT_PROC);
+	ret = call_usermodehelper(drbd_usermode_helper, argv, envp, UMH_WAIT_PROC);
 	if (ret)
 		drbd_warn(device, "helper command: %s %s %s exit code %u (0x%x)\n",
-				usermode_helper, cmd, mb,
+				drbd_usermode_helper, cmd, mb,
 				(ret >> 8) & 0xff, ret);
 	else
 		drbd_info(device, "helper command: %s %s %s exit code %u (0x%x)\n",
-				usermode_helper, cmd, mb,
+				drbd_usermode_helper, cmd, mb,
 				(ret >> 8) & 0xff, ret);
 	sib.sib_reason = SIB_HELPER_POST;
 	sib.helper_exit_code = ret;
@@ -396,24 +396,24 @@ enum drbd_peer_state conn_khelper(struct drbd_connection *connection, char *cmd)
 			 (char[60]) { }, /* address */
 			NULL };
 	char *resource_name = connection->resource->name;
-	char *argv[] = {usermode_helper, cmd, resource_name, NULL };
+	char *argv[] = {drbd_usermode_helper, cmd, resource_name, NULL };
 	int ret;
 
 	setup_khelper_env(connection, envp);
 	conn_md_sync(connection);
 
-	drbd_info(connection, "helper command: %s %s %s\n", usermode_helper, cmd, resource_name);
+	drbd_info(connection, "helper command: %s %s %s\n", drbd_usermode_helper, cmd, resource_name);
 	/* TODO: conn_bcast_event() ?? */
 	notify_helper(NOTIFY_CALL, NULL, connection, cmd, 0);
 
-	ret = call_usermodehelper(usermode_helper, argv, envp, UMH_WAIT_PROC);
+	ret = call_usermodehelper(drbd_usermode_helper, argv, envp, UMH_WAIT_PROC);
 	if (ret)
 		drbd_warn(connection, "helper command: %s %s %s exit code %u (0x%x)\n",
-			  usermode_helper, cmd, resource_name,
+			  drbd_usermode_helper, cmd, resource_name,
 			  (ret >> 8) & 0xff, ret);
 	else
 		drbd_info(connection, "helper command: %s %s %s exit code %u (0x%x)\n",
-			  usermode_helper, cmd, resource_name,
+			  drbd_usermode_helper, cmd, resource_name,
 			  (ret >> 8) & 0xff, ret);
 	/* TODO: conn_bcast_event() ?? */
 	notify_helper(NOTIFY_RESPONSE, NULL, connection, cmd, ret);
@@ -1236,12 +1236,18 @@ static void fixup_discard_if_not_supported(struct request_queue *q)
 
 static void decide_on_write_same_support(struct drbd_device *device,
 			struct request_queue *q,
-			struct request_queue *b, struct o_qlim *o)
+			struct request_queue *b, struct o_qlim *o,
+			bool disable_write_same)
 {
 	struct drbd_peer_device *peer_device = first_peer_device(device);
 	struct drbd_connection *connection = peer_device->connection;
 	bool can_do = b ? b->limits.max_write_same_sectors : true;
 
+	if (can_do && disable_write_same) {
+		can_do = false;
+		drbd_info(peer_device, "WRITE_SAME disabled by config\n");
+	}
+
 	if (can_do && connection->cstate >= C_CONNECTED && !(connection->agreed_features & DRBD_FF_WSAME)) {
 		can_do = false;
 		drbd_info(peer_device, "peer does not support WRITE_SAME\n");
@@ -1302,6 +1308,7 @@ static void drbd_setup_queue_param(struct drbd_device *device, struct drbd_backi
 	struct request_queue *b = NULL;
 	struct disk_conf *dc;
 	bool discard_zeroes_if_aligned = true;
+	bool disable_write_same = false;
 
 	if (bdev) {
 		b = bdev->backing_bdev->bd_disk->queue;
@@ -1311,6 +1318,7 @@ static void drbd_setup_queue_param(struct drbd_device *device, struct drbd_backi
 		dc = rcu_dereference(device->ldev->disk_conf);
 		max_segments = dc->max_bio_bvecs;
 		discard_zeroes_if_aligned = dc->discard_zeroes_if_aligned;
+		disable_write_same = dc->disable_write_same;
 		rcu_read_unlock();
 
 		blk_set_stacking_limits(&q->limits);
@@ -1321,7 +1329,7 @@ static void drbd_setup_queue_param(struct drbd_device *device, struct drbd_backi
 	blk_queue_max_segments(q, max_segments ? max_segments : BLK_MAX_SEGMENTS);
 	blk_queue_segment_boundary(q, PAGE_SIZE-1);
 	decide_on_discard_support(device, q, b, discard_zeroes_if_aligned);
-	decide_on_write_same_support(device, q, b, o);
+	decide_on_write_same_support(device, q, b, o, disable_write_same);
 
 	if (b) {
 		blk_queue_stack_limits(q, b);
@@ -1612,7 +1620,8 @@ int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info)
 	if (write_ordering_changed(old_disk_conf, new_disk_conf))
 		drbd_bump_write_ordering(device->resource, NULL, WO_BDEV_FLUSH);
 
-	if (old_disk_conf->discard_zeroes_if_aligned != new_disk_conf->discard_zeroes_if_aligned)
+	if (old_disk_conf->discard_zeroes_if_aligned != new_disk_conf->discard_zeroes_if_aligned
+	||  old_disk_conf->disable_write_same != new_disk_conf->disable_write_same)
 		drbd_reconsider_queue_parameters(device, device->ldev, NULL);
 
 	drbd_md_sync(device);
@@ -2140,34 +2149,13 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
 
 static int adm_detach(struct drbd_device *device, int force)
 {
-	enum drbd_state_rv retcode;
-	void *buffer;
-	int ret;
-
 	if (force) {
 		set_bit(FORCE_DETACH, &device->flags);
 		drbd_force_state(device, NS(disk, D_FAILED));
-		retcode = SS_SUCCESS;
-		goto out;
+		return SS_SUCCESS;
 	}
 
-	drbd_suspend_io(device); /* so no-one is stuck in drbd_al_begin_io */
-	buffer = drbd_md_get_buffer(device, __func__); /* make sure there is no in-flight meta-data IO */
-	if (buffer) {
-		retcode = drbd_request_state(device, NS(disk, D_FAILED));
-		drbd_md_put_buffer(device);
-	} else /* already <= D_FAILED */
-		retcode = SS_NOTHING_TO_DO;
-	/* D_FAILED will transition to DISKLESS. */
-	drbd_resume_io(device);
-	ret = wait_event_interruptible(device->misc_wait,
-			device->state.disk != D_FAILED);
-	if ((int)retcode == (int)SS_IS_DISKLESS)
-		retcode = SS_NOTHING_TO_DO;
-	if (ret)
-		retcode = ERR_INTR;
-out:
-	return retcode;
+	return drbd_request_detach_interruptible(device);
 }
 
 /* Detaching the disk is a process in multiple stages.  First we need to lock
diff --git a/drivers/block/drbd/drbd_proc.c b/drivers/block/drbd/drbd_proc.c
index 8378142f7a55..582caeb0de86 100644
--- a/drivers/block/drbd/drbd_proc.c
+++ b/drivers/block/drbd/drbd_proc.c
@@ -127,7 +127,7 @@ static void drbd_syncer_progress(struct drbd_device *device, struct seq_file *se
 		seq_putc(seq, '=');
 	seq_putc(seq, '>');
 	for (i = 0; i < y; i++)
-		seq_printf(seq, ".");
+		seq_putc(seq, '.');
 	seq_puts(seq, "] ");
 
 	if (state.conn == C_VERIFY_S || state.conn == C_VERIFY_T)
@@ -179,7 +179,7 @@ static void drbd_syncer_progress(struct drbd_device *device, struct seq_file *se
 	seq_printf_with_thousands_grouping(seq, dbdt);
 	seq_puts(seq, " (");
 	/* ------------------------- ~3s average ------------------------ */
-	if (proc_details >= 1) {
+	if (drbd_proc_details >= 1) {
 		/* this is what drbd_rs_should_slow_down() uses */
 		i = (device->rs_last_mark + DRBD_SYNC_MARKS-1) % DRBD_SYNC_MARKS;
 		dt = (jiffies - device->rs_mark_time[i]) / HZ;
@@ -209,7 +209,7 @@ static void drbd_syncer_progress(struct drbd_device *device, struct seq_file *se
 	}
 	seq_printf(seq, " K/sec%s\n", stalled ? " (stalled)" : "");
 
-	if (proc_details >= 1) {
+	if (drbd_proc_details >= 1) {
 		/* 64 bit:
 		 * we convert to sectors in the display below. */
 		unsigned long bm_bits = drbd_bm_bits(device);
@@ -332,13 +332,13 @@ static int drbd_seq_show(struct seq_file *seq, void *v)
 		    state.conn == C_VERIFY_T)
 			drbd_syncer_progress(device, seq, state);
 
-		if (proc_details >= 1 && get_ldev_if_state(device, D_FAILED)) {
+		if (drbd_proc_details >= 1 && get_ldev_if_state(device, D_FAILED)) {
 			lc_seq_printf_stats(seq, device->resync);
 			lc_seq_printf_stats(seq, device->act_log);
 			put_ldev(device);
 		}
 
-		if (proc_details >= 2)
+		if (drbd_proc_details >= 2)
 			seq_printf(seq, "\tblocked on activity log: %d\n", atomic_read(&device->ap_actlog_cnt));
 	}
 	rcu_read_unlock();
diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c
index c7e95e6380fb..796eaf347dc0 100644
--- a/drivers/block/drbd/drbd_receiver.c
+++ b/drivers/block/drbd/drbd_receiver.c
@@ -332,7 +332,7 @@ static void drbd_free_pages(struct drbd_device *device, struct page *page, int i
 	if (page == NULL)
 		return;
 
-	if (drbd_pp_vacant > (DRBD_MAX_BIO_SIZE/PAGE_SIZE) * minor_count)
+	if (drbd_pp_vacant > (DRBD_MAX_BIO_SIZE/PAGE_SIZE) * drbd_minor_count)
 		i = page_chain_free(page);
 	else {
 		struct page *tmp;
@@ -1100,7 +1100,10 @@ randomize:
 	idr_for_each_entry(&connection->peer_devices, peer_device, vnr)
 		mutex_lock(peer_device->device->state_mutex);
 
+	/* avoid a race with conn_request_state( C_DISCONNECTING ) */
+	spin_lock_irq(&connection->resource->req_lock);
 	set_bit(STATE_SENT, &connection->flags);
+	spin_unlock_irq(&connection->resource->req_lock);
 
 	idr_for_each_entry(&connection->peer_devices, peer_device, vnr)
 		mutex_unlock(peer_device->device->state_mutex);
@@ -1194,6 +1197,14 @@ static int decode_header(struct drbd_connection *connection, void *header, struc
 	return 0;
 }
 
+static void drbd_unplug_all_devices(struct drbd_connection *connection)
+{
+	if (current->plug == &connection->receiver_plug) {
+		blk_finish_plug(&connection->receiver_plug);
+		blk_start_plug(&connection->receiver_plug);
+	} /* else: maybe just schedule() ?? */
+}
+
 static int drbd_recv_header(struct drbd_connection *connection, struct packet_info *pi)
 {
 	void *buffer = connection->data.rbuf;
@@ -1209,6 +1220,36 @@ static int drbd_recv_header(struct drbd_connection *connection, struct packet_in
 	return err;
 }
 
+static int drbd_recv_header_maybe_unplug(struct drbd_connection *connection, struct packet_info *pi)
+{
+	void *buffer = connection->data.rbuf;
+	unsigned int size = drbd_header_size(connection);
+	int err;
+
+	err = drbd_recv_short(connection->data.socket, buffer, size, MSG_NOSIGNAL|MSG_DONTWAIT);
+	if (err != size) {
+		/* If we have nothing in the receive buffer now, to reduce
+		 * application latency, try to drain the backend queues as
+		 * quickly as possible, and let remote TCP know what we have
+		 * received so far. */
+		if (err == -EAGAIN) {
+			drbd_tcp_quickack(connection->data.socket);
+			drbd_unplug_all_devices(connection);
+		}
+		if (err > 0) {
+			buffer += err;
+			size -= err;
+		}
+		err = drbd_recv_all_warn(connection, buffer, size);
+		if (err)
+			return err;
+	}
+
+	err = decode_header(connection, connection->data.rbuf, pi);
+	connection->last_received = jiffies;
+
+	return err;
+}
 /* This is blkdev_issue_flush, but asynchronous.
  * We want to submit to all component volumes in parallel,
  * then wait for all completions.
@@ -1223,7 +1264,7 @@ struct one_flush_context {
 	struct issue_flush_context *ctx;
 };
 
-void one_flush_endio(struct bio *bio)
+static void one_flush_endio(struct bio *bio)
 {
 	struct one_flush_context *octx = bio->bi_private;
 	struct drbd_device *device = octx->device;
@@ -1265,7 +1306,7 @@ static void submit_one_flush(struct drbd_device *device, struct issue_flush_cont
 
 	octx->device = device;
 	octx->ctx = ctx;
-	bio->bi_bdev = device->ldev->backing_bdev;
+	bio_set_dev(bio, device->ldev->backing_bdev);
 	bio->bi_private = octx;
 	bio->bi_end_io = one_flush_endio;
 	bio->bi_opf = REQ_OP_FLUSH | REQ_PREFLUSH;
@@ -1548,7 +1589,7 @@ next_bio:
 	}
 	/* > peer_req->i.sector, unless this is the first bio */
 	bio->bi_iter.bi_sector = sector;
-	bio->bi_bdev = device->ldev->backing_bdev;
+	bio_set_dev(bio, device->ldev->backing_bdev);
 	bio_set_op_attrs(bio, op, op_flags);
 	bio->bi_private = peer_req;
 	bio->bi_end_io = drbd_peer_request_endio;
@@ -4085,7 +4126,7 @@ static int receive_uuids(struct drbd_connection *connection, struct packet_info
 		return config_unknown_volume(connection, pi);
 	device = peer_device->device;
 
-	p_uuid = kmalloc(sizeof(u64)*UI_EXTENDED_SIZE, GFP_NOIO);
+	p_uuid = kmalloc_array(UI_EXTENDED_SIZE, sizeof(*p_uuid), GFP_NOIO);
 	if (!p_uuid) {
 		drbd_err(device, "kmalloc of p_uuid failed\n");
 		return false;
@@ -4882,8 +4923,8 @@ static void drbdd(struct drbd_connection *connection)
 		struct data_cmd const *cmd;
 
 		drbd_thread_current_set_cpu(&connection->receiver);
-		update_receiver_timing_details(connection, drbd_recv_header);
-		if (drbd_recv_header(connection, &pi))
+		update_receiver_timing_details(connection, drbd_recv_header_maybe_unplug);
+		if (drbd_recv_header_maybe_unplug(connection, &pi))
 			goto err_out;
 
 		cmd = &drbd_cmd_handler[pi.cmd];
@@ -5375,8 +5416,11 @@ int drbd_receiver(struct drbd_thread *thi)
 		}
 	} while (h == 0);
 
-	if (h > 0)
+	if (h > 0) {
+		blk_start_plug(&connection->receiver_plug);
 		drbdd(connection);
+		blk_finish_plug(&connection->receiver_plug);
+	}
 
 	conn_disconnect(connection);
 
diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c
index f6e865b2d543..de8566e55334 100644
--- a/drivers/block/drbd/drbd_req.c
+++ b/drivers/block/drbd/drbd_req.c
@@ -36,14 +36,18 @@ static bool drbd_may_do_local_read(struct drbd_device *device, sector_t sector,
 /* Update disk stats at start of I/O request */
 static void _drbd_start_io_acct(struct drbd_device *device, struct drbd_request *req)
 {
-	generic_start_io_acct(bio_data_dir(req->master_bio), req->i.size >> 9,
-			      &device->vdisk->part0);
+	struct request_queue *q = device->rq_queue;
+
+	generic_start_io_acct(q, bio_data_dir(req->master_bio),
+				req->i.size >> 9, &device->vdisk->part0);
 }
 
 /* Update disk stats when completing request upwards */
 static void _drbd_end_io_acct(struct drbd_device *device, struct drbd_request *req)
 {
-	generic_end_io_acct(bio_data_dir(req->master_bio),
+	struct request_queue *q = device->rq_queue;
+
+	generic_end_io_acct(q, bio_data_dir(req->master_bio),
 			    &device->vdisk->part0, req->start_jif);
 }
 
@@ -1175,7 +1179,7 @@ drbd_submit_req_private_bio(struct drbd_request *req)
 	else
 		type = DRBD_FAULT_DT_RD;
 
-	bio->bi_bdev = device->ldev->backing_bdev;
+	bio_set_dev(bio, device->ldev->backing_bdev);
 
 	/* State may have changed since we grabbed our reference on the
 	 * ->ldev member. Double check, and short-circuit to endio.
@@ -1275,6 +1279,57 @@ static bool may_do_writes(struct drbd_device *device)
 	return s.disk == D_UP_TO_DATE || s.pdsk == D_UP_TO_DATE;
 }
 
+struct drbd_plug_cb {
+	struct blk_plug_cb cb;
+	struct drbd_request *most_recent_req;
+	/* do we need more? */
+};
+
+static void drbd_unplug(struct blk_plug_cb *cb, bool from_schedule)
+{
+	struct drbd_plug_cb *plug = container_of(cb, struct drbd_plug_cb, cb);
+	struct drbd_resource *resource = plug->cb.data;
+	struct drbd_request *req = plug->most_recent_req;
+
+	kfree(cb);
+	if (!req)
+		return;
+
+	spin_lock_irq(&resource->req_lock);
+	/* In case the sender did not process it yet, raise the flag to
+	 * have it followed with P_UNPLUG_REMOTE just after. */
+	req->rq_state |= RQ_UNPLUG;
+	/* but also queue a generic unplug */
+	drbd_queue_unplug(req->device);
+	kref_put(&req->kref, drbd_req_destroy);
+	spin_unlock_irq(&resource->req_lock);
+}
+
+static struct drbd_plug_cb* drbd_check_plugged(struct drbd_resource *resource)
+{
+	/* A lot of text to say
+	 * return (struct drbd_plug_cb*)blk_check_plugged(); */
+	struct drbd_plug_cb *plug;
+	struct blk_plug_cb *cb = blk_check_plugged(drbd_unplug, resource, sizeof(*plug));
+
+	if (cb)
+		plug = container_of(cb, struct drbd_plug_cb, cb);
+	else
+		plug = NULL;
+	return plug;
+}
+
+static void drbd_update_plug(struct drbd_plug_cb *plug, struct drbd_request *req)
+{
+	struct drbd_request *tmp = plug->most_recent_req;
+	/* Will be sent to some peer.
+	 * Remember to tag it with UNPLUG_REMOTE on unplug */
+	kref_get(&req->kref);
+	plug->most_recent_req = req;
+	if (tmp)
+		kref_put(&tmp->kref, drbd_req_destroy);
+}
+
 static void drbd_send_and_submit(struct drbd_device *device, struct drbd_request *req)
 {
 	struct drbd_resource *resource = device->resource;
@@ -1347,6 +1402,12 @@ static void drbd_send_and_submit(struct drbd_device *device, struct drbd_request
 			no_remote = true;
 	}
 
+	if (no_remote == false) {
+		struct drbd_plug_cb *plug = drbd_check_plugged(resource);
+		if (plug)
+			drbd_update_plug(plug, req);
+	}
+
 	/* If it took the fast path in drbd_request_prepare, add it here.
 	 * The slow path has added it already. */
 	if (list_empty(&req->req_pending_master_completion))
@@ -1395,7 +1456,10 @@ void __drbd_make_request(struct drbd_device *device, struct bio *bio, unsigned l
 
 static void submit_fast_path(struct drbd_device *device, struct list_head *incoming)
 {
+	struct blk_plug plug;
 	struct drbd_request *req, *tmp;
+
+	blk_start_plug(&plug);
 	list_for_each_entry_safe(req, tmp, incoming, tl_requests) {
 		const int rw = bio_data_dir(req->master_bio);
 
@@ -1413,6 +1477,7 @@ static void submit_fast_path(struct drbd_device *device, struct list_head *incom
 		list_del_init(&req->tl_requests);
 		drbd_send_and_submit(device, req);
 	}
+	blk_finish_plug(&plug);
 }
 
 static bool prepare_al_transaction_nonblock(struct drbd_device *device,
@@ -1420,12 +1485,12 @@ static bool prepare_al_transaction_nonblock(struct drbd_device *device,
 					    struct list_head *pending,
 					    struct list_head *later)
 {
-	struct drbd_request *req, *tmp;
+	struct drbd_request *req;
 	int wake = 0;
 	int err;
 
 	spin_lock_irq(&device->al_lock);
-	list_for_each_entry_safe(req, tmp, incoming, tl_requests) {
+	while ((req = list_first_entry_or_null(incoming, struct drbd_request, tl_requests))) {
 		err = drbd_al_begin_io_nonblock(device, &req->i);
 		if (err == -ENOBUFS)
 			break;
@@ -1442,17 +1507,20 @@ static bool prepare_al_transaction_nonblock(struct drbd_device *device,
 	return !list_empty(pending);
 }
 
-void send_and_submit_pending(struct drbd_device *device, struct list_head *pending)
+static void send_and_submit_pending(struct drbd_device *device, struct list_head *pending)
 {
-	struct drbd_request *req, *tmp;
+	struct blk_plug plug;
+	struct drbd_request *req;
 
-	list_for_each_entry_safe(req, tmp, pending, tl_requests) {
+	blk_start_plug(&plug);
+	while ((req = list_first_entry_or_null(pending, struct drbd_request, tl_requests))) {
 		req->rq_state |= RQ_IN_ACT_LOG;
 		req->in_actlog_jif = jiffies;
 		atomic_dec(&device->ap_actlog_cnt);
 		list_del_init(&req->tl_requests);
 		drbd_send_and_submit(device, req);
 	}
+	blk_finish_plug(&plug);
 }
 
 void do_submit(struct work_struct *ws)
diff --git a/drivers/block/drbd/drbd_req.h b/drivers/block/drbd/drbd_req.h
index 9e1866ab238f..a2254f825601 100644
--- a/drivers/block/drbd/drbd_req.h
+++ b/drivers/block/drbd/drbd_req.h
@@ -212,6 +212,11 @@ enum drbd_req_state_bits {
 	/* Should call drbd_al_complete_io() for this request... */
 	__RQ_IN_ACT_LOG,
 
+	/* This was the most recent request during some blk_finish_plug()
+	 * or its implicit from-schedule equivalent.
+	 * We may use it as hint to send a P_UNPLUG_REMOTE */
+	__RQ_UNPLUG,
+
 	/* The peer has sent a retry ACK */
 	__RQ_POSTPONED,
 
@@ -249,6 +254,7 @@ enum drbd_req_state_bits {
 #define RQ_WSAME           (1UL << __RQ_WSAME)
 #define RQ_UNMAP           (1UL << __RQ_UNMAP)
 #define RQ_IN_ACT_LOG      (1UL << __RQ_IN_ACT_LOG)
+#define RQ_UNPLUG          (1UL << __RQ_UNPLUG)
 #define RQ_POSTPONED	   (1UL << __RQ_POSTPONED)
 #define RQ_COMPLETION_SUSP (1UL << __RQ_COMPLETION_SUSP)
 #define RQ_EXP_RECEIVE_ACK (1UL << __RQ_EXP_RECEIVE_ACK)
diff --git a/drivers/block/drbd/drbd_state.c b/drivers/block/drbd/drbd_state.c
index eea0c4aec978..0813c654c893 100644
--- a/drivers/block/drbd/drbd_state.c
+++ b/drivers/block/drbd/drbd_state.c
@@ -346,7 +346,7 @@ static enum drbd_role min_role(enum drbd_role role1, enum drbd_role role2)
 
 enum drbd_role conn_highest_role(struct drbd_connection *connection)
 {
-	enum drbd_role role = R_UNKNOWN;
+	enum drbd_role role = R_SECONDARY;
 	struct drbd_peer_device *peer_device;
 	int vnr;
 
@@ -579,11 +579,14 @@ drbd_req_state(struct drbd_device *device, union drbd_state mask,
 	unsigned long flags;
 	union drbd_state os, ns;
 	enum drbd_state_rv rv;
+	void *buffer = NULL;
 
 	init_completion(&done);
 
 	if (f & CS_SERIALIZE)
 		mutex_lock(device->state_mutex);
+	if (f & CS_INHIBIT_MD_IO)
+		buffer = drbd_md_get_buffer(device, __func__);
 
 	spin_lock_irqsave(&device->resource->req_lock, flags);
 	os = drbd_read_state(device);
@@ -636,6 +639,8 @@ drbd_req_state(struct drbd_device *device, union drbd_state mask,
 	}
 
 abort:
+	if (buffer)
+		drbd_md_put_buffer(device);
 	if (f & CS_SERIALIZE)
 		mutex_unlock(device->state_mutex);
 
@@ -664,6 +669,47 @@ _drbd_request_state(struct drbd_device *device, union drbd_state mask,
 	return rv;
 }
 
+/*
+ * We grab drbd_md_get_buffer(), because we don't want to "fail" the disk while
+ * there is IO in-flight: the transition into D_FAILED for detach purposes
+ * may get misinterpreted as actual IO error in a confused endio function.
+ *
+ * We wrap it all into wait_event(), to retry in case the drbd_req_state()
+ * returns SS_IN_TRANSIENT_STATE.
+ *
+ * To avoid potential deadlock with e.g. the receiver thread trying to grab
+ * drbd_md_get_buffer() while trying to get out of the "transient state", we
+ * need to grab and release the meta data buffer inside of that wait_event loop.
+ */
+static enum drbd_state_rv
+request_detach(struct drbd_device *device)
+{
+	return drbd_req_state(device, NS(disk, D_FAILED),
+			CS_VERBOSE | CS_ORDERED | CS_INHIBIT_MD_IO);
+}
+
+enum drbd_state_rv
+drbd_request_detach_interruptible(struct drbd_device *device)
+{
+	enum drbd_state_rv rv;
+	int ret;
+
+	drbd_suspend_io(device); /* so no-one is stuck in drbd_al_begin_io */
+	wait_event_interruptible(device->state_wait,
+		(rv = request_detach(device)) != SS_IN_TRANSIENT_STATE);
+	drbd_resume_io(device);
+
+	ret = wait_event_interruptible(device->misc_wait,
+			device->state.disk != D_FAILED);
+
+	if (rv == SS_IS_DISKLESS)
+		rv = SS_NOTHING_TO_DO;
+	if (ret)
+		rv = ERR_INTR;
+
+	return rv;
+}
+
 enum drbd_state_rv
 _drbd_request_state_holding_state_mutex(struct drbd_device *device, union drbd_state mask,
 		    union drbd_state val, enum chg_state_flags f)
diff --git a/drivers/block/drbd/drbd_state.h b/drivers/block/drbd/drbd_state.h
index 6c9d5d4a8a75..0276c98fbbdd 100644
--- a/drivers/block/drbd/drbd_state.h
+++ b/drivers/block/drbd/drbd_state.h
@@ -71,6 +71,10 @@ enum chg_state_flags {
 	CS_DC_SUSP       = 1 << 10,
 	CS_DC_MASK       = CS_DC_ROLE + CS_DC_PEER + CS_DC_CONN + CS_DC_DISK + CS_DC_PDSK,
 	CS_IGN_OUTD_FAIL = 1 << 11,
+
+	/* Make sure no meta data IO is in flight, by calling
+	 * drbd_md_get_buffer().  Used for graceful detach. */
+	CS_INHIBIT_MD_IO = 1 << 12,
 };
 
 /* drbd_dev_state and drbd_state are different types. This is to stress the
@@ -156,6 +160,10 @@ static inline int drbd_request_state(struct drbd_device *device,
 	return _drbd_request_state(device, mask, val, CS_VERBOSE + CS_ORDERED);
 }
 
+/* for use in adm_detach() (drbd_adm_detach(), drbd_adm_down()) */
+enum drbd_state_rv
+drbd_request_detach_interruptible(struct drbd_device *device);
+
 enum drbd_role conn_highest_role(struct drbd_connection *connection);
 enum drbd_role conn_highest_peer(struct drbd_connection *connection);
 enum drbd_disk_state conn_highest_disk(struct drbd_connection *connection);
diff --git a/drivers/block/drbd/drbd_worker.c b/drivers/block/drbd/drbd_worker.c
index 1d8726a8df34..03471b3fce86 100644
--- a/drivers/block/drbd/drbd_worker.c
+++ b/drivers/block/drbd/drbd_worker.c
@@ -65,6 +65,11 @@ void drbd_md_endio(struct bio *bio)
 	device = bio->bi_private;
 	device->md_io.error = blk_status_to_errno(bio->bi_status);
 
+	/* special case: drbd_md_read() during drbd_adm_attach() */
+	if (device->ldev)
+		put_ldev(device);
+	bio_put(bio);
+
 	/* We grabbed an extra reference in _drbd_md_sync_page_io() to be able
 	 * to timeout on the lower level device, and eventually detach from it.
 	 * If this io completion runs after that timeout expired, this
@@ -79,9 +84,6 @@ void drbd_md_endio(struct bio *bio)
 	drbd_md_put_buffer(device);
 	device->md_io.done = 1;
 	wake_up(&device->misc_wait);
-	bio_put(bio);
-	if (device->ldev) /* special case: drbd_md_read() during drbd_adm_attach() */
-		put_ldev(device);
 }
 
 /* reads on behalf of the partner,
@@ -128,6 +130,14 @@ void drbd_endio_write_sec_final(struct drbd_peer_request *peer_req) __releases(l
 	block_id = peer_req->block_id;
 	peer_req->flags &= ~EE_CALL_AL_COMPLETE_IO;
 
+	if (peer_req->flags & EE_WAS_ERROR) {
+		/* In protocol != C, we usually do not send write acks.
+		 * In case of a write error, send the neg ack anyways. */
+		if (!__test_and_set_bit(__EE_SEND_WRITE_ACK, &peer_req->flags))
+			inc_unacked(device);
+		drbd_set_out_of_sync(device, peer_req->i.sector, peer_req->i.size);
+	}
+
 	spin_lock_irqsave(&device->resource->req_lock, flags);
 	device->writ_cnt += peer_req->i.size >> 9;
 	list_move_tail(&peer_req->w.list, &device->done_ee);
@@ -195,7 +205,8 @@ void drbd_peer_request_endio(struct bio *bio)
 	}
 }
 
-void drbd_panic_after_delayed_completion_of_aborted_request(struct drbd_device *device)
+static void
+drbd_panic_after_delayed_completion_of_aborted_request(struct drbd_device *device)
 {
 	panic("drbd%u %s/%u potential random memory corruption caused by delayed completion of aborted local request\n",
 		device->minor, device->resource->name, device->vnr);
@@ -1382,18 +1393,22 @@ static int drbd_send_barrier(struct drbd_connection *connection)
 	return conn_send_command(connection, sock, P_BARRIER, sizeof(*p), NULL, 0);
 }
 
+static int pd_send_unplug_remote(struct drbd_peer_device *pd)
+{
+	struct drbd_socket *sock = &pd->connection->data;
+	if (!drbd_prepare_command(pd, sock))
+		return -EIO;
+	return drbd_send_command(pd, sock, P_UNPLUG_REMOTE, 0, NULL, 0);
+}
+
 int w_send_write_hint(struct drbd_work *w, int cancel)
 {
 	struct drbd_device *device =
 		container_of(w, struct drbd_device, unplug_work);
-	struct drbd_socket *sock;
 
 	if (cancel)
 		return 0;
-	sock = &first_peer_device(device)->connection->data;
-	if (!drbd_prepare_command(first_peer_device(device), sock))
-		return -EIO;
-	return drbd_send_command(first_peer_device(device), sock, P_UNPLUG_REMOTE, 0, NULL, 0);
+	return pd_send_unplug_remote(first_peer_device(device));
 }
 
 static void re_init_if_first_write(struct drbd_connection *connection, unsigned int epoch)
@@ -1455,6 +1470,7 @@ int w_send_dblock(struct drbd_work *w, int cancel)
 	struct drbd_device *device = req->device;
 	struct drbd_peer_device *const peer_device = first_peer_device(device);
 	struct drbd_connection *connection = peer_device->connection;
+	bool do_send_unplug = req->rq_state & RQ_UNPLUG;
 	int err;
 
 	if (unlikely(cancel)) {
@@ -1470,6 +1486,9 @@ int w_send_dblock(struct drbd_work *w, int cancel)
 	err = drbd_send_dblock(peer_device, req);
 	req_mod(req, err ? SEND_FAILED : HANDED_OVER_TO_NETWORK);
 
+	if (do_send_unplug && !err)
+		pd_send_unplug_remote(peer_device);
+
 	return err;
 }
 
@@ -1484,6 +1503,7 @@ int w_send_read_req(struct drbd_work *w, int cancel)
 	struct drbd_device *device = req->device;
 	struct drbd_peer_device *const peer_device = first_peer_device(device);
 	struct drbd_connection *connection = peer_device->connection;
+	bool do_send_unplug = req->rq_state & RQ_UNPLUG;
 	int err;
 
 	if (unlikely(cancel)) {
@@ -1501,6 +1521,9 @@ int w_send_read_req(struct drbd_work *w, int cancel)
 
 	req_mod(req, err ? SEND_FAILED : HANDED_OVER_TO_NETWORK);
 
+	if (do_send_unplug && !err)
+		pd_send_unplug_remote(peer_device);
+
 	return err;
 }
 
@@ -1513,7 +1536,7 @@ int w_restart_disk_io(struct drbd_work *w, int cancel)
 		drbd_al_begin_io(device, &req->i);
 
 	drbd_req_make_private_bio(req, req->master_bio);
-	req->private_bio->bi_bdev = device->ldev->backing_bdev;
+	bio_set_dev(req->private_bio, device->ldev->backing_bdev);
 	generic_make_request(req->private_bio);
 
 	return 0;
@@ -1733,6 +1756,11 @@ void drbd_start_resync(struct drbd_device *device, enum drbd_conns side)
 		return;
 	}
 
+	if (!connection) {
+		drbd_err(device, "No connection to peer, aborting!\n");
+		return;
+	}
+
 	if (!test_bit(B_RS_H_DONE, &device->flags)) {
 		if (side == C_SYNC_TARGET) {
 			/* Since application IO was locked out during C_WF_BITMAP_T and
diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c
index 9c00f29e40c1..60c086a53609 100644
--- a/drivers/block/floppy.c
+++ b/drivers/block/floppy.c
@@ -4134,7 +4134,7 @@ static int __floppy_read_block_0(struct block_device *bdev, int drive)
 	cbdata.drive = drive;
 
 	bio_init(&bio, &bio_vec, 1);
-	bio.bi_bdev = bdev;
+	bio_set_dev(&bio, bdev);
 	bio_add_page(&bio, page, size, 0);
 
 	bio.bi_iter.bi_sector = 0;
diff --git a/drivers/block/loop.c b/drivers/block/loop.c
index ef8334949b42..85de67334695 100644
--- a/drivers/block/loop.c
+++ b/drivers/block/loop.c
@@ -213,16 +213,18 @@ static void __loop_update_dio(struct loop_device *lo, bool dio)
 	 */
 	blk_mq_freeze_queue(lo->lo_queue);
 	lo->use_dio = use_dio;
-	if (use_dio)
+	if (use_dio) {
+		queue_flag_clear_unlocked(QUEUE_FLAG_NOMERGES, lo->lo_queue);
 		lo->lo_flags |= LO_FLAGS_DIRECT_IO;
-	else
+	} else {
+		queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, lo->lo_queue);
 		lo->lo_flags &= ~LO_FLAGS_DIRECT_IO;
+	}
 	blk_mq_unfreeze_queue(lo->lo_queue);
 }
 
 static int
-figure_loop_size(struct loop_device *lo, loff_t offset, loff_t sizelimit,
-		 loff_t logical_blocksize)
+figure_loop_size(struct loop_device *lo, loff_t offset, loff_t sizelimit)
 {
 	loff_t size = get_size(offset, sizelimit, lo->lo_backing_file);
 	sector_t x = (sector_t)size;
@@ -234,12 +236,6 @@ figure_loop_size(struct loop_device *lo, loff_t offset, loff_t sizelimit,
 		lo->lo_offset = offset;
 	if (lo->lo_sizelimit != sizelimit)
 		lo->lo_sizelimit = sizelimit;
-	if (lo->lo_flags & LO_FLAGS_BLOCKSIZE) {
-		lo->lo_logical_blocksize = logical_blocksize;
-		blk_queue_physical_block_size(lo->lo_queue, lo->lo_blocksize);
-		blk_queue_logical_block_size(lo->lo_queue,
-					     lo->lo_logical_blocksize);
-	}
 	set_capacity(lo->lo_disk, x);
 	bd_set_size(bdev, (loff_t)get_capacity(bdev->bd_disk) << 9);
 	/* let user-space know about the new size */
@@ -467,12 +463,21 @@ static void lo_complete_rq(struct request *rq)
 	blk_mq_end_request(rq, cmd->ret < 0 ? BLK_STS_IOERR : BLK_STS_OK);
 }
 
+static void lo_rw_aio_do_completion(struct loop_cmd *cmd)
+{
+	if (!atomic_dec_and_test(&cmd->ref))
+		return;
+	kfree(cmd->bvec);
+	cmd->bvec = NULL;
+	blk_mq_complete_request(cmd->rq);
+}
+
 static void lo_rw_aio_complete(struct kiocb *iocb, long ret, long ret2)
 {
 	struct loop_cmd *cmd = container_of(iocb, struct loop_cmd, iocb);
 
 	cmd->ret = ret;
-	blk_mq_complete_request(cmd->rq);
+	lo_rw_aio_do_completion(cmd);
 }
 
 static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd,
@@ -480,22 +485,51 @@ static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd,
 {
 	struct iov_iter iter;
 	struct bio_vec *bvec;
-	struct bio *bio = cmd->rq->bio;
+	struct request *rq = cmd->rq;
+	struct bio *bio = rq->bio;
 	struct file *file = lo->lo_backing_file;
+	unsigned int offset;
+	int segments = 0;
 	int ret;
 
-	/* nomerge for loop request queue */
-	WARN_ON(cmd->rq->bio != cmd->rq->biotail);
+	if (rq->bio != rq->biotail) {
+		struct req_iterator iter;
+		struct bio_vec tmp;
+
+		__rq_for_each_bio(bio, rq)
+			segments += bio_segments(bio);
+		bvec = kmalloc(sizeof(struct bio_vec) * segments, GFP_NOIO);
+		if (!bvec)
+			return -EIO;
+		cmd->bvec = bvec;
+
+		/*
+		 * The bios of the request may be started from the middle of
+		 * the 'bvec' because of bio splitting, so we can't directly
+		 * copy bio->bi_iov_vec to new bvec. The rq_for_each_segment
+		 * API will take care of all details for us.
+		 */
+		rq_for_each_segment(tmp, rq, iter) {
+			*bvec = tmp;
+			bvec++;
+		}
+		bvec = cmd->bvec;
+		offset = 0;
+	} else {
+		/*
+		 * Same here, this bio may be started from the middle of the
+		 * 'bvec' because of bio splitting, so offset from the bvec
+		 * must be passed to iov iterator
+		 */
+		offset = bio->bi_iter.bi_bvec_done;
+		bvec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
+		segments = bio_segments(bio);
+	}
+	atomic_set(&cmd->ref, 2);
 
-	bvec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
 	iov_iter_bvec(&iter, ITER_BVEC | rw, bvec,
-		      bio_segments(bio), blk_rq_bytes(cmd->rq));
-	/*
-	 * This bio may be started from the middle of the 'bvec'
-	 * because of bio splitting, so offset from the bvec must
-	 * be passed to iov iterator
-	 */
-	iter.iov_offset = bio->bi_iter.bi_bvec_done;
+		      segments, blk_rq_bytes(rq));
+	iter.iov_offset = offset;
 
 	cmd->iocb.ki_pos = pos;
 	cmd->iocb.ki_filp = file;
@@ -507,6 +541,8 @@ static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd,
 	else
 		ret = call_read_iter(file, &cmd->iocb, &iter);
 
+	lo_rw_aio_do_completion(cmd);
+
 	if (ret != -EIOCBQUEUED)
 		cmd->iocb.ki_complete(&cmd->iocb, ret, 0);
 	return 0;
@@ -553,74 +589,12 @@ static int do_req_filebacked(struct loop_device *lo, struct request *rq)
 	}
 }
 
-struct switch_request {
-	struct file *file;
-	struct completion wait;
-};
-
 static inline void loop_update_dio(struct loop_device *lo)
 {
 	__loop_update_dio(lo, io_is_direct(lo->lo_backing_file) |
 			lo->use_dio);
 }
 
-/*
- * Do the actual switch; called from the BIO completion routine
- */
-static void do_loop_switch(struct loop_device *lo, struct switch_request *p)
-{
-	struct file *file = p->file;
-	struct file *old_file = lo->lo_backing_file;
-	struct address_space *mapping;
-
-	/* if no new file, only flush of queued bios requested */
-	if (!file)
-		return;
-
-	mapping = file->f_mapping;
-	mapping_set_gfp_mask(old_file->f_mapping, lo->old_gfp_mask);
-	lo->lo_backing_file = file;
-	lo->lo_blocksize = S_ISBLK(mapping->host->i_mode) ?
-		mapping->host->i_bdev->bd_block_size : PAGE_SIZE;
-	lo->old_gfp_mask = mapping_gfp_mask(mapping);
-	mapping_set_gfp_mask(mapping, lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS));
-	loop_update_dio(lo);
-}
-
-/*
- * loop_switch performs the hard work of switching a backing store.
- * First it needs to flush existing IO, it does this by sending a magic
- * BIO down the pipe. The completion of this BIO does the actual switch.
- */
-static int loop_switch(struct loop_device *lo, struct file *file)
-{
-	struct switch_request w;
-
-	w.file = file;
-
-	/* freeze queue and wait for completion of scheduled requests */
-	blk_mq_freeze_queue(lo->lo_queue);
-
-	/* do the switch action */
-	do_loop_switch(lo, &w);
-
-	/* unfreeze */
-	blk_mq_unfreeze_queue(lo->lo_queue);
-
-	return 0;
-}
-
-/*
- * Helper to flush the IOs in loop, but keeping loop thread running
- */
-static int loop_flush(struct loop_device *lo)
-{
-	/* loop not yet configured, no running thread, nothing to flush */
-	if (lo->lo_state != Lo_bound)
-		return 0;
-	return loop_switch(lo, NULL);
-}
-
 static void loop_reread_partitions(struct loop_device *lo,
 				   struct block_device *bdev)
 {
@@ -685,9 +659,14 @@ static int loop_change_fd(struct loop_device *lo, struct block_device *bdev,
 		goto out_putf;
 
 	/* and ... switch */
-	error = loop_switch(lo, file);
-	if (error)
-		goto out_putf;
+	blk_mq_freeze_queue(lo->lo_queue);
+	mapping_set_gfp_mask(old_file->f_mapping, lo->old_gfp_mask);
+	lo->lo_backing_file = file;
+	lo->old_gfp_mask = mapping_gfp_mask(file->f_mapping);
+	mapping_set_gfp_mask(file->f_mapping,
+			     lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS));
+	loop_update_dio(lo);
+	blk_mq_unfreeze_queue(lo->lo_queue);
 
 	fput(old_file);
 	if (lo->lo_flags & LO_FLAGS_PARTSCAN)
@@ -820,7 +799,6 @@ static void loop_config_discard(struct loop_device *lo)
 	struct file *file = lo->lo_backing_file;
 	struct inode *inode = file->f_mapping->host;
 	struct request_queue *q = lo->lo_queue;
-	int lo_bits = 9;
 
 	/*
 	 * We use punch hole to reclaim the free space used by the
@@ -840,11 +818,9 @@ static void loop_config_discard(struct loop_device *lo)
 
 	q->limits.discard_granularity = inode->i_sb->s_blocksize;
 	q->limits.discard_alignment = 0;
-	if (lo->lo_flags & LO_FLAGS_BLOCKSIZE)
-		lo_bits = blksize_bits(lo->lo_logical_blocksize);
 
-	blk_queue_max_discard_sectors(q, UINT_MAX >> lo_bits);
-	blk_queue_max_write_zeroes_sectors(q, UINT_MAX >> lo_bits);
+	blk_queue_max_discard_sectors(q, UINT_MAX >> 9);
+	blk_queue_max_write_zeroes_sectors(q, UINT_MAX >> 9);
 	queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q);
 }
 
@@ -877,7 +853,6 @@ static int loop_set_fd(struct loop_device *lo, fmode_t mode,
 	struct file	*file, *f;
 	struct inode	*inode;
 	struct address_space *mapping;
-	unsigned lo_blocksize;
 	int		lo_flags = 0;
 	int		error;
 	loff_t		size;
@@ -921,9 +896,6 @@ static int loop_set_fd(struct loop_device *lo, fmode_t mode,
 	    !file->f_op->write_iter)
 		lo_flags |= LO_FLAGS_READ_ONLY;
 
-	lo_blocksize = S_ISBLK(inode->i_mode) ?
-		inode->i_bdev->bd_block_size : PAGE_SIZE;
-
 	error = -EFBIG;
 	size = get_loop_size(lo, file);
 	if ((loff_t)(sector_t)size != size)
@@ -937,8 +909,6 @@ static int loop_set_fd(struct loop_device *lo, fmode_t mode,
 	set_device_ro(bdev, (lo_flags & LO_FLAGS_READ_ONLY) != 0);
 
 	lo->use_dio = false;
-	lo->lo_blocksize = lo_blocksize;
-	lo->lo_logical_blocksize = 512;
 	lo->lo_device = bdev;
 	lo->lo_flags = lo_flags;
 	lo->lo_backing_file = file;
@@ -958,7 +928,8 @@ static int loop_set_fd(struct loop_device *lo, fmode_t mode,
 	/* let user-space know about the new size */
 	kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE);
 
-	set_blocksize(bdev, lo_blocksize);
+	set_blocksize(bdev, S_ISBLK(inode->i_mode) ?
+		      block_size(inode->i_bdev) : PAGE_SIZE);
 
 	lo->lo_state = Lo_bound;
 	if (part_shift)
@@ -1064,6 +1035,9 @@ static int loop_clr_fd(struct loop_device *lo)
 	memset(lo->lo_encrypt_key, 0, LO_KEY_SIZE);
 	memset(lo->lo_crypt_name, 0, LO_NAME_SIZE);
 	memset(lo->lo_file_name, 0, LO_NAME_SIZE);
+	blk_queue_logical_block_size(lo->lo_queue, 512);
+	blk_queue_physical_block_size(lo->lo_queue, 512);
+	blk_queue_io_min(lo->lo_queue, 512);
 	if (bdev) {
 		bdput(bdev);
 		invalidate_bdev(bdev);
@@ -1104,7 +1078,6 @@ loop_set_status(struct loop_device *lo, const struct loop_info64 *info)
 	int err;
 	struct loop_func_table *xfer;
 	kuid_t uid = current_uid();
-	int lo_flags = lo->lo_flags;
 
 	if (lo->lo_encrypt_key_size &&
 	    !uid_eq(lo->lo_key_owner, uid) &&
@@ -1137,26 +1110,9 @@ loop_set_status(struct loop_device *lo, const struct loop_info64 *info)
 	if (err)
 		goto exit;
 
-	if (info->lo_flags & LO_FLAGS_BLOCKSIZE) {
-		if (!(lo->lo_flags & LO_FLAGS_BLOCKSIZE))
-			lo->lo_logical_blocksize = 512;
-		lo->lo_flags |= LO_FLAGS_BLOCKSIZE;
-		if (LO_INFO_BLOCKSIZE(info) != 512 &&
-		    LO_INFO_BLOCKSIZE(info) != 1024 &&
-		    LO_INFO_BLOCKSIZE(info) != 2048 &&
-		    LO_INFO_BLOCKSIZE(info) != 4096)
-			return -EINVAL;
-		if (LO_INFO_BLOCKSIZE(info) > lo->lo_blocksize)
-			return -EINVAL;
-	}
-
 	if (lo->lo_offset != info->lo_offset ||
-	    lo->lo_sizelimit != info->lo_sizelimit ||
-	    lo->lo_flags != lo_flags ||
-	    ((lo->lo_flags & LO_FLAGS_BLOCKSIZE) &&
-	     lo->lo_logical_blocksize != LO_INFO_BLOCKSIZE(info))) {
-		if (figure_loop_size(lo, info->lo_offset, info->lo_sizelimit,
-				     LO_INFO_BLOCKSIZE(info))) {
+	    lo->lo_sizelimit != info->lo_sizelimit) {
+		if (figure_loop_size(lo, info->lo_offset, info->lo_sizelimit)) {
 			err = -EFBIG;
 			goto exit;
 		}
@@ -1348,8 +1304,7 @@ static int loop_set_capacity(struct loop_device *lo)
 	if (unlikely(lo->lo_state != Lo_bound))
 		return -ENXIO;
 
-	return figure_loop_size(lo, lo->lo_offset, lo->lo_sizelimit,
-				lo->lo_logical_blocksize);
+	return figure_loop_size(lo, lo->lo_offset, lo->lo_sizelimit);
 }
 
 static int loop_set_dio(struct loop_device *lo, unsigned long arg)
@@ -1366,6 +1321,26 @@ static int loop_set_dio(struct loop_device *lo, unsigned long arg)
 	return error;
 }
 
+static int loop_set_block_size(struct loop_device *lo, unsigned long arg)
+{
+	if (lo->lo_state != Lo_bound)
+		return -ENXIO;
+
+	if (arg < 512 || arg > PAGE_SIZE || !is_power_of_2(arg))
+		return -EINVAL;
+
+	blk_mq_freeze_queue(lo->lo_queue);
+
+	blk_queue_logical_block_size(lo->lo_queue, arg);
+	blk_queue_physical_block_size(lo->lo_queue, arg);
+	blk_queue_io_min(lo->lo_queue, arg);
+	loop_update_dio(lo);
+
+	blk_mq_unfreeze_queue(lo->lo_queue);
+
+	return 0;
+}
+
 static int lo_ioctl(struct block_device *bdev, fmode_t mode,
 	unsigned int cmd, unsigned long arg)
 {
@@ -1414,6 +1389,11 @@ static int lo_ioctl(struct block_device *bdev, fmode_t mode,
 		if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN))
 			err = loop_set_dio(lo, arg);
 		break;
+	case LOOP_SET_BLOCK_SIZE:
+		err = -EPERM;
+		if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN))
+			err = loop_set_block_size(lo, arg);
+		break;
 	default:
 		err = lo->ioctl ? lo->ioctl(lo, cmd, arg) : -EINVAL;
 	}
@@ -1613,12 +1593,13 @@ static void lo_release(struct gendisk *disk, fmode_t mode)
 		err = loop_clr_fd(lo);
 		if (!err)
 			return;
-	} else {
+	} else if (lo->lo_state == Lo_bound) {
 		/*
 		 * Otherwise keep thread (if running) and config,
 		 * but flush possible ongoing bios in thread.
 		 */
-		loop_flush(lo);
+		blk_mq_freeze_queue(lo->lo_queue);
+		blk_mq_unfreeze_queue(lo->lo_queue);
 	}
 
 	mutex_unlock(&lo->lo_ctl_mutex);
@@ -1800,9 +1781,13 @@ static int loop_add(struct loop_device **l, int i)
 	}
 	lo->lo_queue->queuedata = lo;
 
+	blk_queue_max_hw_sectors(lo->lo_queue, BLK_DEF_MAX_SECTORS);
+
 	/*
-	 * It doesn't make sense to enable merge because the I/O
-	 * submitted to backing file is handled page by page.
+	 * By default, we do buffer IO, so it doesn't make sense to enable
+	 * merge because the I/O submitted to backing file is handled page by
+	 * page. For directio mode, merge does help to dispatch bigger request
+	 * to underlayer disk. We will enable merge once directio is enabled.
 	 */
 	queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, lo->lo_queue);
 
@@ -1996,10 +1981,6 @@ static int __init loop_init(void)
 	struct loop_device *lo;
 	int err;
 
-	err = misc_register(&loop_misc);
-	if (err < 0)
-		return err;
-
 	part_shift = 0;
 	if (max_part > 0) {
 		part_shift = fls(max_part);
@@ -2017,12 +1998,12 @@ static int __init loop_init(void)
 
 	if ((1UL << part_shift) > DISK_MAX_PARTS) {
 		err = -EINVAL;
-		goto misc_out;
+		goto err_out;
 	}
 
 	if (max_loop > 1UL << (MINORBITS - part_shift)) {
 		err = -EINVAL;
-		goto misc_out;
+		goto err_out;
 	}
 
 	/*
@@ -2041,6 +2022,11 @@ static int __init loop_init(void)
 		range = 1UL << MINORBITS;
 	}
 
+	err = misc_register(&loop_misc);
+	if (err < 0)
+		goto err_out;
+
+
 	if (register_blkdev(LOOP_MAJOR, "loop")) {
 		err = -EIO;
 		goto misc_out;
@@ -2060,6 +2046,7 @@ static int __init loop_init(void)
 
 misc_out:
 	misc_deregister(&loop_misc);
+err_out:
 	return err;
 }
 
diff --git a/drivers/block/loop.h b/drivers/block/loop.h
index 2c096b9a17b8..f68c1d50802f 100644
--- a/drivers/block/loop.h
+++ b/drivers/block/loop.h
@@ -48,8 +48,6 @@ struct loop_device {
 
 	struct file *	lo_backing_file;
 	struct block_device *lo_device;
-	unsigned	lo_blocksize;
-	unsigned	lo_logical_blocksize;
 	void		*key_data; 
 
 	gfp_t		old_gfp_mask;
@@ -69,10 +67,13 @@ struct loop_device {
 struct loop_cmd {
 	struct kthread_work work;
 	struct request *rq;
-	struct list_head list;
-	bool use_aio;           /* use AIO interface to handle I/O */
+	union {
+		bool use_aio; /* use AIO interface to handle I/O */
+		atomic_t ref; /* only for aio */
+	};
 	long ret;
 	struct kiocb iocb;
+	struct bio_vec *bvec;
 };
 
 /* Support for loadable transfer modules */
diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c
index 5bdf923294a5..2aa87cbdede0 100644
--- a/drivers/block/nbd.c
+++ b/drivers/block/nbd.c
@@ -128,7 +128,7 @@ static struct dentry *nbd_dbg_dir;
 #define NBD_MAGIC 0x68797548
 
 static unsigned int nbds_max = 16;
-static int max_part;
+static int max_part = 16;
 static struct workqueue_struct *recv_workqueue;
 static int part_shift;
 
@@ -165,7 +165,7 @@ static ssize_t pid_show(struct device *dev,
 	return sprintf(buf, "%d\n", task_pid_nr(nbd->task_recv));
 }
 
-static struct device_attribute pid_attr = {
+static const struct device_attribute pid_attr = {
 	.attr = { .name = "pid", .mode = S_IRUGO},
 	.show = pid_show,
 };
@@ -1584,6 +1584,15 @@ again:
 		}
 	} else {
 		nbd = idr_find(&nbd_index_idr, index);
+		if (!nbd) {
+			ret = nbd_dev_add(index);
+			if (ret < 0) {
+				mutex_unlock(&nbd_index_mutex);
+				printk(KERN_ERR "nbd: failed to add new device\n");
+				return ret;
+			}
+			nbd = idr_find(&nbd_index_idr, index);
+		}
 	}
 	if (!nbd) {
 		printk(KERN_ERR "nbd: couldn't find device at index %d\n",
@@ -2137,4 +2146,4 @@ MODULE_LICENSE("GPL");
 module_param(nbds_max, int, 0444);
 MODULE_PARM_DESC(nbds_max, "number of network block devices to initialize (default: 16)");
 module_param(max_part, int, 0444);
-MODULE_PARM_DESC(max_part, "number of partitions per device (default: 0)");
+MODULE_PARM_DESC(max_part, "number of partitions per device (default: 16)");
diff --git a/drivers/block/null_blk.c b/drivers/block/null_blk.c
index 85c24cace973..8042c26ea9e6 100644
--- a/drivers/block/null_blk.c
+++ b/drivers/block/null_blk.c
@@ -1,3 +1,7 @@
+/*
+ * Add configfs and memory store: Kyungchan Koh <kkc6196@fb.com> and
+ * Shaohua Li <shli@fb.com>
+ */
 #include <linux/module.h>
 
 #include <linux/moduleparam.h>
@@ -9,27 +13,110 @@
 #include <linux/blk-mq.h>
 #include <linux/hrtimer.h>
 #include <linux/lightnvm.h>
+#include <linux/configfs.h>
+#include <linux/badblocks.h>
+
+#define SECTOR_SHIFT		9
+#define PAGE_SECTORS_SHIFT	(PAGE_SHIFT - SECTOR_SHIFT)
+#define PAGE_SECTORS		(1 << PAGE_SECTORS_SHIFT)
+#define SECTOR_SIZE		(1 << SECTOR_SHIFT)
+#define SECTOR_MASK		(PAGE_SECTORS - 1)
+
+#define FREE_BATCH		16
+
+#define TICKS_PER_SEC		50ULL
+#define TIMER_INTERVAL		(NSEC_PER_SEC / TICKS_PER_SEC)
+
+static inline u64 mb_per_tick(int mbps)
+{
+	return (1 << 20) / TICKS_PER_SEC * ((u64) mbps);
+}
 
 struct nullb_cmd {
 	struct list_head list;
 	struct llist_node ll_list;
-	struct call_single_data csd;
+	call_single_data_t csd;
 	struct request *rq;
 	struct bio *bio;
 	unsigned int tag;
 	struct nullb_queue *nq;
 	struct hrtimer timer;
+	blk_status_t error;
 };
 
 struct nullb_queue {
 	unsigned long *tag_map;
 	wait_queue_head_t wait;
 	unsigned int queue_depth;
+	struct nullb_device *dev;
 
 	struct nullb_cmd *cmds;
 };
 
+/*
+ * Status flags for nullb_device.
+ *
+ * CONFIGURED:	Device has been configured and turned on. Cannot reconfigure.
+ * UP:		Device is currently on and visible in userspace.
+ * THROTTLED:	Device is being throttled.
+ * CACHE:	Device is using a write-back cache.
+ */
+enum nullb_device_flags {
+	NULLB_DEV_FL_CONFIGURED	= 0,
+	NULLB_DEV_FL_UP		= 1,
+	NULLB_DEV_FL_THROTTLED	= 2,
+	NULLB_DEV_FL_CACHE	= 3,
+};
+
+/*
+ * nullb_page is a page in memory for nullb devices.
+ *
+ * @page:	The page holding the data.
+ * @bitmap:	The bitmap represents which sector in the page has data.
+ *		Each bit represents one block size. For example, sector 8
+ *		will use the 7th bit
+ * The highest 2 bits of bitmap are for special purpose. LOCK means the cache
+ * page is being flushing to storage. FREE means the cache page is freed and
+ * should be skipped from flushing to storage. Please see
+ * null_make_cache_space
+ */
+struct nullb_page {
+	struct page *page;
+	unsigned long bitmap;
+};
+#define NULLB_PAGE_LOCK (sizeof(unsigned long) * 8 - 1)
+#define NULLB_PAGE_FREE (sizeof(unsigned long) * 8 - 2)
+
+struct nullb_device {
+	struct nullb *nullb;
+	struct config_item item;
+	struct radix_tree_root data; /* data stored in the disk */
+	struct radix_tree_root cache; /* disk cache data */
+	unsigned long flags; /* device flags */
+	unsigned int curr_cache;
+	struct badblocks badblocks;
+
+	unsigned long size; /* device size in MB */
+	unsigned long completion_nsec; /* time in ns to complete a request */
+	unsigned long cache_size; /* disk cache size in MB */
+	unsigned int submit_queues; /* number of submission queues */
+	unsigned int home_node; /* home node for the device */
+	unsigned int queue_mode; /* block interface */
+	unsigned int blocksize; /* block size */
+	unsigned int irqmode; /* IRQ completion handler */
+	unsigned int hw_queue_depth; /* queue depth */
+	unsigned int index; /* index of the disk, only valid with a disk */
+	unsigned int mbps; /* Bandwidth throttle cap (in MB/s) */
+	bool use_lightnvm; /* register as a LightNVM device */
+	bool blocking; /* blocking blk-mq device */
+	bool use_per_node_hctx; /* use per-node allocation for hardware context */
+	bool power; /* power on/off the device */
+	bool memory_backed; /* if data is stored in memory */
+	bool discard; /* if support discard */
+};
+
 struct nullb {
+	struct nullb_device *dev;
 	struct list_head list;
 	unsigned int index;
 	struct request_queue *q;
@@ -37,8 +124,10 @@ struct nullb {
 	struct nvm_dev *ndev;
 	struct blk_mq_tag_set *tag_set;
 	struct blk_mq_tag_set __tag_set;
-	struct hrtimer timer;
 	unsigned int queue_depth;
+	atomic_long_t cur_bytes;
+	struct hrtimer bw_timer;
+	unsigned long cache_flush_pos;
 	spinlock_t lock;
 
 	struct nullb_queue *queues;
@@ -49,7 +138,7 @@ struct nullb {
 static LIST_HEAD(nullb_list);
 static struct mutex lock;
 static int null_major;
-static int nullb_indexes;
+static DEFINE_IDA(nullb_indexes);
 static struct kmem_cache *ppa_cache;
 static struct blk_mq_tag_set tag_set;
 
@@ -65,15 +154,15 @@ enum {
 	NULL_Q_MQ		= 2,
 };
 
-static int submit_queues;
-module_param(submit_queues, int, S_IRUGO);
+static int g_submit_queues = 1;
+module_param_named(submit_queues, g_submit_queues, int, S_IRUGO);
 MODULE_PARM_DESC(submit_queues, "Number of submission queues");
 
-static int home_node = NUMA_NO_NODE;
-module_param(home_node, int, S_IRUGO);
+static int g_home_node = NUMA_NO_NODE;
+module_param_named(home_node, g_home_node, int, S_IRUGO);
 MODULE_PARM_DESC(home_node, "Home node for the device");
 
-static int queue_mode = NULL_Q_MQ;
+static int g_queue_mode = NULL_Q_MQ;
 
 static int null_param_store_val(const char *str, int *val, int min, int max)
 {
@@ -92,7 +181,7 @@ static int null_param_store_val(const char *str, int *val, int min, int max)
 
 static int null_set_queue_mode(const char *str, const struct kernel_param *kp)
 {
-	return null_param_store_val(str, &queue_mode, NULL_Q_BIO, NULL_Q_MQ);
+	return null_param_store_val(str, &g_queue_mode, NULL_Q_BIO, NULL_Q_MQ);
 }
 
 static const struct kernel_param_ops null_queue_mode_param_ops = {
@@ -100,38 +189,38 @@ static const struct kernel_param_ops null_queue_mode_param_ops = {
 	.get	= param_get_int,
 };
 
-device_param_cb(queue_mode, &null_queue_mode_param_ops, &queue_mode, S_IRUGO);
+device_param_cb(queue_mode, &null_queue_mode_param_ops, &g_queue_mode, S_IRUGO);
 MODULE_PARM_DESC(queue_mode, "Block interface to use (0=bio,1=rq,2=multiqueue)");
 
-static int gb = 250;
-module_param(gb, int, S_IRUGO);
+static int g_gb = 250;
+module_param_named(gb, g_gb, int, S_IRUGO);
 MODULE_PARM_DESC(gb, "Size in GB");
 
-static int bs = 512;
-module_param(bs, int, S_IRUGO);
+static int g_bs = 512;
+module_param_named(bs, g_bs, int, S_IRUGO);
 MODULE_PARM_DESC(bs, "Block size (in bytes)");
 
 static int nr_devices = 1;
 module_param(nr_devices, int, S_IRUGO);
 MODULE_PARM_DESC(nr_devices, "Number of devices to register");
 
-static bool use_lightnvm;
-module_param(use_lightnvm, bool, S_IRUGO);
+static bool g_use_lightnvm;
+module_param_named(use_lightnvm, g_use_lightnvm, bool, S_IRUGO);
 MODULE_PARM_DESC(use_lightnvm, "Register as a LightNVM device");
 
-static bool blocking;
-module_param(blocking, bool, S_IRUGO);
+static bool g_blocking;
+module_param_named(blocking, g_blocking, bool, S_IRUGO);
 MODULE_PARM_DESC(blocking, "Register as a blocking blk-mq driver device");
 
 static bool shared_tags;
 module_param(shared_tags, bool, S_IRUGO);
 MODULE_PARM_DESC(shared_tags, "Share tag set between devices for blk-mq");
 
-static int irqmode = NULL_IRQ_SOFTIRQ;
+static int g_irqmode = NULL_IRQ_SOFTIRQ;
 
 static int null_set_irqmode(const char *str, const struct kernel_param *kp)
 {
-	return null_param_store_val(str, &irqmode, NULL_IRQ_NONE,
+	return null_param_store_val(str, &g_irqmode, NULL_IRQ_NONE,
 					NULL_IRQ_TIMER);
 }
 
@@ -140,21 +229,358 @@ static const struct kernel_param_ops null_irqmode_param_ops = {
 	.get	= param_get_int,
 };
 
-device_param_cb(irqmode, &null_irqmode_param_ops, &irqmode, S_IRUGO);
+device_param_cb(irqmode, &null_irqmode_param_ops, &g_irqmode, S_IRUGO);
 MODULE_PARM_DESC(irqmode, "IRQ completion handler. 0-none, 1-softirq, 2-timer");
 
-static unsigned long completion_nsec = 10000;
-module_param(completion_nsec, ulong, S_IRUGO);
+static unsigned long g_completion_nsec = 10000;
+module_param_named(completion_nsec, g_completion_nsec, ulong, S_IRUGO);
 MODULE_PARM_DESC(completion_nsec, "Time in ns to complete a request in hardware. Default: 10,000ns");
 
-static int hw_queue_depth = 64;
-module_param(hw_queue_depth, int, S_IRUGO);
+static int g_hw_queue_depth = 64;
+module_param_named(hw_queue_depth, g_hw_queue_depth, int, S_IRUGO);
 MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: 64");
 
-static bool use_per_node_hctx = false;
-module_param(use_per_node_hctx, bool, S_IRUGO);
+static bool g_use_per_node_hctx;
+module_param_named(use_per_node_hctx, g_use_per_node_hctx, bool, S_IRUGO);
 MODULE_PARM_DESC(use_per_node_hctx, "Use per-node allocation for hardware context queues. Default: false");
 
+static struct nullb_device *null_alloc_dev(void);
+static void null_free_dev(struct nullb_device *dev);
+static void null_del_dev(struct nullb *nullb);
+static int null_add_dev(struct nullb_device *dev);
+static void null_free_device_storage(struct nullb_device *dev, bool is_cache);
+
+static inline struct nullb_device *to_nullb_device(struct config_item *item)
+{
+	return item ? container_of(item, struct nullb_device, item) : NULL;
+}
+
+static inline ssize_t nullb_device_uint_attr_show(unsigned int val, char *page)
+{
+	return snprintf(page, PAGE_SIZE, "%u\n", val);
+}
+
+static inline ssize_t nullb_device_ulong_attr_show(unsigned long val,
+	char *page)
+{
+	return snprintf(page, PAGE_SIZE, "%lu\n", val);
+}
+
+static inline ssize_t nullb_device_bool_attr_show(bool val, char *page)
+{
+	return snprintf(page, PAGE_SIZE, "%u\n", val);
+}
+
+static ssize_t nullb_device_uint_attr_store(unsigned int *val,
+	const char *page, size_t count)
+{
+	unsigned int tmp;
+	int result;
+
+	result = kstrtouint(page, 0, &tmp);
+	if (result)
+		return result;
+
+	*val = tmp;
+	return count;
+}
+
+static ssize_t nullb_device_ulong_attr_store(unsigned long *val,
+	const char *page, size_t count)
+{
+	int result;
+	unsigned long tmp;
+
+	result = kstrtoul(page, 0, &tmp);
+	if (result)
+		return result;
+
+	*val = tmp;
+	return count;
+}
+
+static ssize_t nullb_device_bool_attr_store(bool *val, const char *page,
+	size_t count)
+{
+	bool tmp;
+	int result;
+
+	result = kstrtobool(page,  &tmp);
+	if (result)
+		return result;
+
+	*val = tmp;
+	return count;
+}
+
+/* The following macro should only be used with TYPE = {uint, ulong, bool}. */
+#define NULLB_DEVICE_ATTR(NAME, TYPE)						\
+static ssize_t									\
+nullb_device_##NAME##_show(struct config_item *item, char *page)		\
+{										\
+	return nullb_device_##TYPE##_attr_show(					\
+				to_nullb_device(item)->NAME, page);		\
+}										\
+static ssize_t									\
+nullb_device_##NAME##_store(struct config_item *item, const char *page,		\
+			    size_t count)					\
+{										\
+	if (test_bit(NULLB_DEV_FL_CONFIGURED, &to_nullb_device(item)->flags))	\
+		return -EBUSY;							\
+	return nullb_device_##TYPE##_attr_store(				\
+			&to_nullb_device(item)->NAME, page, count);		\
+}										\
+CONFIGFS_ATTR(nullb_device_, NAME);
+
+NULLB_DEVICE_ATTR(size, ulong);
+NULLB_DEVICE_ATTR(completion_nsec, ulong);
+NULLB_DEVICE_ATTR(submit_queues, uint);
+NULLB_DEVICE_ATTR(home_node, uint);
+NULLB_DEVICE_ATTR(queue_mode, uint);
+NULLB_DEVICE_ATTR(blocksize, uint);
+NULLB_DEVICE_ATTR(irqmode, uint);
+NULLB_DEVICE_ATTR(hw_queue_depth, uint);
+NULLB_DEVICE_ATTR(index, uint);
+NULLB_DEVICE_ATTR(use_lightnvm, bool);
+NULLB_DEVICE_ATTR(blocking, bool);
+NULLB_DEVICE_ATTR(use_per_node_hctx, bool);
+NULLB_DEVICE_ATTR(memory_backed, bool);
+NULLB_DEVICE_ATTR(discard, bool);
+NULLB_DEVICE_ATTR(mbps, uint);
+NULLB_DEVICE_ATTR(cache_size, ulong);
+
+static ssize_t nullb_device_power_show(struct config_item *item, char *page)
+{
+	return nullb_device_bool_attr_show(to_nullb_device(item)->power, page);
+}
+
+static ssize_t nullb_device_power_store(struct config_item *item,
+				     const char *page, size_t count)
+{
+	struct nullb_device *dev = to_nullb_device(item);
+	bool newp = false;
+	ssize_t ret;
+
+	ret = nullb_device_bool_attr_store(&newp, page, count);
+	if (ret < 0)
+		return ret;
+
+	if (!dev->power && newp) {
+		if (test_and_set_bit(NULLB_DEV_FL_UP, &dev->flags))
+			return count;
+		if (null_add_dev(dev)) {
+			clear_bit(NULLB_DEV_FL_UP, &dev->flags);
+			return -ENOMEM;
+		}
+
+		set_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags);
+		dev->power = newp;
+	} else if (dev->power && !newp) {
+		mutex_lock(&lock);
+		dev->power = newp;
+		null_del_dev(dev->nullb);
+		mutex_unlock(&lock);
+		clear_bit(NULLB_DEV_FL_UP, &dev->flags);
+	}
+
+	return count;
+}
+
+CONFIGFS_ATTR(nullb_device_, power);
+
+static ssize_t nullb_device_badblocks_show(struct config_item *item, char *page)
+{
+	struct nullb_device *t_dev = to_nullb_device(item);
+
+	return badblocks_show(&t_dev->badblocks, page, 0);
+}
+
+static ssize_t nullb_device_badblocks_store(struct config_item *item,
+				     const char *page, size_t count)
+{
+	struct nullb_device *t_dev = to_nullb_device(item);
+	char *orig, *buf, *tmp;
+	u64 start, end;
+	int ret;
+
+	orig = kstrndup(page, count, GFP_KERNEL);
+	if (!orig)
+		return -ENOMEM;
+
+	buf = strstrip(orig);
+
+	ret = -EINVAL;
+	if (buf[0] != '+' && buf[0] != '-')
+		goto out;
+	tmp = strchr(&buf[1], '-');
+	if (!tmp)
+		goto out;
+	*tmp = '\0';
+	ret = kstrtoull(buf + 1, 0, &start);
+	if (ret)
+		goto out;
+	ret = kstrtoull(tmp + 1, 0, &end);
+	if (ret)
+		goto out;
+	ret = -EINVAL;
+	if (start > end)
+		goto out;
+	/* enable badblocks */
+	cmpxchg(&t_dev->badblocks.shift, -1, 0);
+	if (buf[0] == '+')
+		ret = badblocks_set(&t_dev->badblocks, start,
+			end - start + 1, 1);
+	else
+		ret = badblocks_clear(&t_dev->badblocks, start,
+			end - start + 1);
+	if (ret == 0)
+		ret = count;
+out:
+	kfree(orig);
+	return ret;
+}
+CONFIGFS_ATTR(nullb_device_, badblocks);
+
+static struct configfs_attribute *nullb_device_attrs[] = {
+	&nullb_device_attr_size,
+	&nullb_device_attr_completion_nsec,
+	&nullb_device_attr_submit_queues,
+	&nullb_device_attr_home_node,
+	&nullb_device_attr_queue_mode,
+	&nullb_device_attr_blocksize,
+	&nullb_device_attr_irqmode,
+	&nullb_device_attr_hw_queue_depth,
+	&nullb_device_attr_index,
+	&nullb_device_attr_use_lightnvm,
+	&nullb_device_attr_blocking,
+	&nullb_device_attr_use_per_node_hctx,
+	&nullb_device_attr_power,
+	&nullb_device_attr_memory_backed,
+	&nullb_device_attr_discard,
+	&nullb_device_attr_mbps,
+	&nullb_device_attr_cache_size,
+	&nullb_device_attr_badblocks,
+	NULL,
+};
+
+static void nullb_device_release(struct config_item *item)
+{
+	struct nullb_device *dev = to_nullb_device(item);
+
+	badblocks_exit(&dev->badblocks);
+	null_free_device_storage(dev, false);
+	null_free_dev(dev);
+}
+
+static struct configfs_item_operations nullb_device_ops = {
+	.release	= nullb_device_release,
+};
+
+static struct config_item_type nullb_device_type = {
+	.ct_item_ops	= &nullb_device_ops,
+	.ct_attrs	= nullb_device_attrs,
+	.ct_owner	= THIS_MODULE,
+};
+
+static struct
+config_item *nullb_group_make_item(struct config_group *group, const char *name)
+{
+	struct nullb_device *dev;
+
+	dev = null_alloc_dev();
+	if (!dev)
+		return ERR_PTR(-ENOMEM);
+
+	config_item_init_type_name(&dev->item, name, &nullb_device_type);
+
+	return &dev->item;
+}
+
+static void
+nullb_group_drop_item(struct config_group *group, struct config_item *item)
+{
+	struct nullb_device *dev = to_nullb_device(item);
+
+	if (test_and_clear_bit(NULLB_DEV_FL_UP, &dev->flags)) {
+		mutex_lock(&lock);
+		dev->power = false;
+		null_del_dev(dev->nullb);
+		mutex_unlock(&lock);
+	}
+
+	config_item_put(item);
+}
+
+static ssize_t memb_group_features_show(struct config_item *item, char *page)
+{
+	return snprintf(page, PAGE_SIZE, "memory_backed,discard,bandwidth,cache,badblocks\n");
+}
+
+CONFIGFS_ATTR_RO(memb_group_, features);
+
+static struct configfs_attribute *nullb_group_attrs[] = {
+	&memb_group_attr_features,
+	NULL,
+};
+
+static struct configfs_group_operations nullb_group_ops = {
+	.make_item	= nullb_group_make_item,
+	.drop_item	= nullb_group_drop_item,
+};
+
+static struct config_item_type nullb_group_type = {
+	.ct_group_ops	= &nullb_group_ops,
+	.ct_attrs	= nullb_group_attrs,
+	.ct_owner	= THIS_MODULE,
+};
+
+static struct configfs_subsystem nullb_subsys = {
+	.su_group = {
+		.cg_item = {
+			.ci_namebuf = "nullb",
+			.ci_type = &nullb_group_type,
+		},
+	},
+};
+
+static inline int null_cache_active(struct nullb *nullb)
+{
+	return test_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);
+}
+
+static struct nullb_device *null_alloc_dev(void)
+{
+	struct nullb_device *dev;
+
+	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+	if (!dev)
+		return NULL;
+	INIT_RADIX_TREE(&dev->data, GFP_ATOMIC);
+	INIT_RADIX_TREE(&dev->cache, GFP_ATOMIC);
+	if (badblocks_init(&dev->badblocks, 0)) {
+		kfree(dev);
+		return NULL;
+	}
+
+	dev->size = g_gb * 1024;
+	dev->completion_nsec = g_completion_nsec;
+	dev->submit_queues = g_submit_queues;
+	dev->home_node = g_home_node;
+	dev->queue_mode = g_queue_mode;
+	dev->blocksize = g_bs;
+	dev->irqmode = g_irqmode;
+	dev->hw_queue_depth = g_hw_queue_depth;
+	dev->use_lightnvm = g_use_lightnvm;
+	dev->blocking = g_blocking;
+	dev->use_per_node_hctx = g_use_per_node_hctx;
+	return dev;
+}
+
+static void null_free_dev(struct nullb_device *dev)
+{
+	kfree(dev);
+}
+
 static void put_tag(struct nullb_queue *nq, unsigned int tag)
 {
 	clear_bit_unlock(tag, nq->tag_map);
@@ -193,7 +619,7 @@ static struct nullb_cmd *__alloc_cmd(struct nullb_queue *nq)
 		cmd = &nq->cmds[tag];
 		cmd->tag = tag;
 		cmd->nq = nq;
-		if (irqmode == NULL_IRQ_TIMER) {
+		if (nq->dev->irqmode == NULL_IRQ_TIMER) {
 			hrtimer_init(&cmd->timer, CLOCK_MONOTONIC,
 				     HRTIMER_MODE_REL);
 			cmd->timer.function = null_cmd_timer_expired;
@@ -229,19 +655,21 @@ static struct nullb_cmd *alloc_cmd(struct nullb_queue *nq, int can_wait)
 static void end_cmd(struct nullb_cmd *cmd)
 {
 	struct request_queue *q = NULL;
+	int queue_mode = cmd->nq->dev->queue_mode;
 
 	if (cmd->rq)
 		q = cmd->rq->q;
 
 	switch (queue_mode)  {
 	case NULL_Q_MQ:
-		blk_mq_end_request(cmd->rq, BLK_STS_OK);
+		blk_mq_end_request(cmd->rq, cmd->error);
 		return;
 	case NULL_Q_RQ:
 		INIT_LIST_HEAD(&cmd->rq->queuelist);
-		blk_end_request_all(cmd->rq, BLK_STS_OK);
+		blk_end_request_all(cmd->rq, cmd->error);
 		break;
 	case NULL_Q_BIO:
+		cmd->bio->bi_status = cmd->error;
 		bio_endio(cmd->bio);
 		break;
 	}
@@ -267,25 +695,582 @@ static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer)
 
 static void null_cmd_end_timer(struct nullb_cmd *cmd)
 {
-	ktime_t kt = completion_nsec;
+	ktime_t kt = cmd->nq->dev->completion_nsec;
 
 	hrtimer_start(&cmd->timer, kt, HRTIMER_MODE_REL);
 }
 
 static void null_softirq_done_fn(struct request *rq)
 {
-	if (queue_mode == NULL_Q_MQ)
+	struct nullb *nullb = rq->q->queuedata;
+
+	if (nullb->dev->queue_mode == NULL_Q_MQ)
 		end_cmd(blk_mq_rq_to_pdu(rq));
 	else
 		end_cmd(rq->special);
 }
 
-static inline void null_handle_cmd(struct nullb_cmd *cmd)
+static struct nullb_page *null_alloc_page(gfp_t gfp_flags)
+{
+	struct nullb_page *t_page;
+
+	t_page = kmalloc(sizeof(struct nullb_page), gfp_flags);
+	if (!t_page)
+		goto out;
+
+	t_page->page = alloc_pages(gfp_flags, 0);
+	if (!t_page->page)
+		goto out_freepage;
+
+	t_page->bitmap = 0;
+	return t_page;
+out_freepage:
+	kfree(t_page);
+out:
+	return NULL;
+}
+
+static void null_free_page(struct nullb_page *t_page)
+{
+	__set_bit(NULLB_PAGE_FREE, &t_page->bitmap);
+	if (test_bit(NULLB_PAGE_LOCK, &t_page->bitmap))
+		return;
+	__free_page(t_page->page);
+	kfree(t_page);
+}
+
+static void null_free_sector(struct nullb *nullb, sector_t sector,
+	bool is_cache)
+{
+	unsigned int sector_bit;
+	u64 idx;
+	struct nullb_page *t_page, *ret;
+	struct radix_tree_root *root;
+
+	root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
+	idx = sector >> PAGE_SECTORS_SHIFT;
+	sector_bit = (sector & SECTOR_MASK);
+
+	t_page = radix_tree_lookup(root, idx);
+	if (t_page) {
+		__clear_bit(sector_bit, &t_page->bitmap);
+
+		if (!t_page->bitmap) {
+			ret = radix_tree_delete_item(root, idx, t_page);
+			WARN_ON(ret != t_page);
+			null_free_page(ret);
+			if (is_cache)
+				nullb->dev->curr_cache -= PAGE_SIZE;
+		}
+	}
+}
+
+static struct nullb_page *null_radix_tree_insert(struct nullb *nullb, u64 idx,
+	struct nullb_page *t_page, bool is_cache)
+{
+	struct radix_tree_root *root;
+
+	root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
+
+	if (radix_tree_insert(root, idx, t_page)) {
+		null_free_page(t_page);
+		t_page = radix_tree_lookup(root, idx);
+		WARN_ON(!t_page || t_page->page->index != idx);
+	} else if (is_cache)
+		nullb->dev->curr_cache += PAGE_SIZE;
+
+	return t_page;
+}
+
+static void null_free_device_storage(struct nullb_device *dev, bool is_cache)
+{
+	unsigned long pos = 0;
+	int nr_pages;
+	struct nullb_page *ret, *t_pages[FREE_BATCH];
+	struct radix_tree_root *root;
+
+	root = is_cache ? &dev->cache : &dev->data;
+
+	do {
+		int i;
+
+		nr_pages = radix_tree_gang_lookup(root,
+				(void **)t_pages, pos, FREE_BATCH);
+
+		for (i = 0; i < nr_pages; i++) {
+			pos = t_pages[i]->page->index;
+			ret = radix_tree_delete_item(root, pos, t_pages[i]);
+			WARN_ON(ret != t_pages[i]);
+			null_free_page(ret);
+		}
+
+		pos++;
+	} while (nr_pages == FREE_BATCH);
+
+	if (is_cache)
+		dev->curr_cache = 0;
+}
+
+static struct nullb_page *__null_lookup_page(struct nullb *nullb,
+	sector_t sector, bool for_write, bool is_cache)
+{
+	unsigned int sector_bit;
+	u64 idx;
+	struct nullb_page *t_page;
+	struct radix_tree_root *root;
+
+	idx = sector >> PAGE_SECTORS_SHIFT;
+	sector_bit = (sector & SECTOR_MASK);
+
+	root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
+	t_page = radix_tree_lookup(root, idx);
+	WARN_ON(t_page && t_page->page->index != idx);
+
+	if (t_page && (for_write || test_bit(sector_bit, &t_page->bitmap)))
+		return t_page;
+
+	return NULL;
+}
+
+static struct nullb_page *null_lookup_page(struct nullb *nullb,
+	sector_t sector, bool for_write, bool ignore_cache)
+{
+	struct nullb_page *page = NULL;
+
+	if (!ignore_cache)
+		page = __null_lookup_page(nullb, sector, for_write, true);
+	if (page)
+		return page;
+	return __null_lookup_page(nullb, sector, for_write, false);
+}
+
+static struct nullb_page *null_insert_page(struct nullb *nullb,
+	sector_t sector, bool ignore_cache)
+{
+	u64 idx;
+	struct nullb_page *t_page;
+
+	t_page = null_lookup_page(nullb, sector, true, ignore_cache);
+	if (t_page)
+		return t_page;
+
+	spin_unlock_irq(&nullb->lock);
+
+	t_page = null_alloc_page(GFP_NOIO);
+	if (!t_page)
+		goto out_lock;
+
+	if (radix_tree_preload(GFP_NOIO))
+		goto out_freepage;
+
+	spin_lock_irq(&nullb->lock);
+	idx = sector >> PAGE_SECTORS_SHIFT;
+	t_page->page->index = idx;
+	t_page = null_radix_tree_insert(nullb, idx, t_page, !ignore_cache);
+	radix_tree_preload_end();
+
+	return t_page;
+out_freepage:
+	null_free_page(t_page);
+out_lock:
+	spin_lock_irq(&nullb->lock);
+	return null_lookup_page(nullb, sector, true, ignore_cache);
+}
+
+static int null_flush_cache_page(struct nullb *nullb, struct nullb_page *c_page)
+{
+	int i;
+	unsigned int offset;
+	u64 idx;
+	struct nullb_page *t_page, *ret;
+	void *dst, *src;
+
+	idx = c_page->page->index;
+
+	t_page = null_insert_page(nullb, idx << PAGE_SECTORS_SHIFT, true);
+
+	__clear_bit(NULLB_PAGE_LOCK, &c_page->bitmap);
+	if (test_bit(NULLB_PAGE_FREE, &c_page->bitmap)) {
+		null_free_page(c_page);
+		if (t_page && t_page->bitmap == 0) {
+			ret = radix_tree_delete_item(&nullb->dev->data,
+				idx, t_page);
+			null_free_page(t_page);
+		}
+		return 0;
+	}
+
+	if (!t_page)
+		return -ENOMEM;
+
+	src = kmap_atomic(c_page->page);
+	dst = kmap_atomic(t_page->page);
+
+	for (i = 0; i < PAGE_SECTORS;
+			i += (nullb->dev->blocksize >> SECTOR_SHIFT)) {
+		if (test_bit(i, &c_page->bitmap)) {
+			offset = (i << SECTOR_SHIFT);
+			memcpy(dst + offset, src + offset,
+				nullb->dev->blocksize);
+			__set_bit(i, &t_page->bitmap);
+		}
+	}
+
+	kunmap_atomic(dst);
+	kunmap_atomic(src);
+
+	ret = radix_tree_delete_item(&nullb->dev->cache, idx, c_page);
+	null_free_page(ret);
+	nullb->dev->curr_cache -= PAGE_SIZE;
+
+	return 0;
+}
+
+static int null_make_cache_space(struct nullb *nullb, unsigned long n)
 {
+	int i, err, nr_pages;
+	struct nullb_page *c_pages[FREE_BATCH];
+	unsigned long flushed = 0, one_round;
+
+again:
+	if ((nullb->dev->cache_size * 1024 * 1024) >
+	     nullb->dev->curr_cache + n || nullb->dev->curr_cache == 0)
+		return 0;
+
+	nr_pages = radix_tree_gang_lookup(&nullb->dev->cache,
+			(void **)c_pages, nullb->cache_flush_pos, FREE_BATCH);
+	/*
+	 * nullb_flush_cache_page could unlock before using the c_pages. To
+	 * avoid race, we don't allow page free
+	 */
+	for (i = 0; i < nr_pages; i++) {
+		nullb->cache_flush_pos = c_pages[i]->page->index;
+		/*
+		 * We found the page which is being flushed to disk by other
+		 * threads
+		 */
+		if (test_bit(NULLB_PAGE_LOCK, &c_pages[i]->bitmap))
+			c_pages[i] = NULL;
+		else
+			__set_bit(NULLB_PAGE_LOCK, &c_pages[i]->bitmap);
+	}
+
+	one_round = 0;
+	for (i = 0; i < nr_pages; i++) {
+		if (c_pages[i] == NULL)
+			continue;
+		err = null_flush_cache_page(nullb, c_pages[i]);
+		if (err)
+			return err;
+		one_round++;
+	}
+	flushed += one_round << PAGE_SHIFT;
+
+	if (n > flushed) {
+		if (nr_pages == 0)
+			nullb->cache_flush_pos = 0;
+		if (one_round == 0) {
+			/* give other threads a chance */
+			spin_unlock_irq(&nullb->lock);
+			spin_lock_irq(&nullb->lock);
+		}
+		goto again;
+	}
+	return 0;
+}
+
+static int copy_to_nullb(struct nullb *nullb, struct page *source,
+	unsigned int off, sector_t sector, size_t n, bool is_fua)
+{
+	size_t temp, count = 0;
+	unsigned int offset;
+	struct nullb_page *t_page;
+	void *dst, *src;
+
+	while (count < n) {
+		temp = min_t(size_t, nullb->dev->blocksize, n - count);
+
+		if (null_cache_active(nullb) && !is_fua)
+			null_make_cache_space(nullb, PAGE_SIZE);
+
+		offset = (sector & SECTOR_MASK) << SECTOR_SHIFT;
+		t_page = null_insert_page(nullb, sector,
+			!null_cache_active(nullb) || is_fua);
+		if (!t_page)
+			return -ENOSPC;
+
+		src = kmap_atomic(source);
+		dst = kmap_atomic(t_page->page);
+		memcpy(dst + offset, src + off + count, temp);
+		kunmap_atomic(dst);
+		kunmap_atomic(src);
+
+		__set_bit(sector & SECTOR_MASK, &t_page->bitmap);
+
+		if (is_fua)
+			null_free_sector(nullb, sector, true);
+
+		count += temp;
+		sector += temp >> SECTOR_SHIFT;
+	}
+	return 0;
+}
+
+static int copy_from_nullb(struct nullb *nullb, struct page *dest,
+	unsigned int off, sector_t sector, size_t n)
+{
+	size_t temp, count = 0;
+	unsigned int offset;
+	struct nullb_page *t_page;
+	void *dst, *src;
+
+	while (count < n) {
+		temp = min_t(size_t, nullb->dev->blocksize, n - count);
+
+		offset = (sector & SECTOR_MASK) << SECTOR_SHIFT;
+		t_page = null_lookup_page(nullb, sector, false,
+			!null_cache_active(nullb));
+
+		dst = kmap_atomic(dest);
+		if (!t_page) {
+			memset(dst + off + count, 0, temp);
+			goto next;
+		}
+		src = kmap_atomic(t_page->page);
+		memcpy(dst + off + count, src + offset, temp);
+		kunmap_atomic(src);
+next:
+		kunmap_atomic(dst);
+
+		count += temp;
+		sector += temp >> SECTOR_SHIFT;
+	}
+	return 0;
+}
+
+static void null_handle_discard(struct nullb *nullb, sector_t sector, size_t n)
+{
+	size_t temp;
+
+	spin_lock_irq(&nullb->lock);
+	while (n > 0) {
+		temp = min_t(size_t, n, nullb->dev->blocksize);
+		null_free_sector(nullb, sector, false);
+		if (null_cache_active(nullb))
+			null_free_sector(nullb, sector, true);
+		sector += temp >> SECTOR_SHIFT;
+		n -= temp;
+	}
+	spin_unlock_irq(&nullb->lock);
+}
+
+static int null_handle_flush(struct nullb *nullb)
+{
+	int err;
+
+	if (!null_cache_active(nullb))
+		return 0;
+
+	spin_lock_irq(&nullb->lock);
+	while (true) {
+		err = null_make_cache_space(nullb,
+			nullb->dev->cache_size * 1024 * 1024);
+		if (err || nullb->dev->curr_cache == 0)
+			break;
+	}
+
+	WARN_ON(!radix_tree_empty(&nullb->dev->cache));
+	spin_unlock_irq(&nullb->lock);
+	return err;
+}
+
+static int null_transfer(struct nullb *nullb, struct page *page,
+	unsigned int len, unsigned int off, bool is_write, sector_t sector,
+	bool is_fua)
+{
+	int err = 0;
+
+	if (!is_write) {
+		err = copy_from_nullb(nullb, page, off, sector, len);
+		flush_dcache_page(page);
+	} else {
+		flush_dcache_page(page);
+		err = copy_to_nullb(nullb, page, off, sector, len, is_fua);
+	}
+
+	return err;
+}
+
+static int null_handle_rq(struct nullb_cmd *cmd)
+{
+	struct request *rq = cmd->rq;
+	struct nullb *nullb = cmd->nq->dev->nullb;
+	int err;
+	unsigned int len;
+	sector_t sector;
+	struct req_iterator iter;
+	struct bio_vec bvec;
+
+	sector = blk_rq_pos(rq);
+
+	if (req_op(rq) == REQ_OP_DISCARD) {
+		null_handle_discard(nullb, sector, blk_rq_bytes(rq));
+		return 0;
+	}
+
+	spin_lock_irq(&nullb->lock);
+	rq_for_each_segment(bvec, rq, iter) {
+		len = bvec.bv_len;
+		err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset,
+				     op_is_write(req_op(rq)), sector,
+				     req_op(rq) & REQ_FUA);
+		if (err) {
+			spin_unlock_irq(&nullb->lock);
+			return err;
+		}
+		sector += len >> SECTOR_SHIFT;
+	}
+	spin_unlock_irq(&nullb->lock);
+
+	return 0;
+}
+
+static int null_handle_bio(struct nullb_cmd *cmd)
+{
+	struct bio *bio = cmd->bio;
+	struct nullb *nullb = cmd->nq->dev->nullb;
+	int err;
+	unsigned int len;
+	sector_t sector;
+	struct bio_vec bvec;
+	struct bvec_iter iter;
+
+	sector = bio->bi_iter.bi_sector;
+
+	if (bio_op(bio) == REQ_OP_DISCARD) {
+		null_handle_discard(nullb, sector,
+			bio_sectors(bio) << SECTOR_SHIFT);
+		return 0;
+	}
+
+	spin_lock_irq(&nullb->lock);
+	bio_for_each_segment(bvec, bio, iter) {
+		len = bvec.bv_len;
+		err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset,
+				     op_is_write(bio_op(bio)), sector,
+				     bio_op(bio) & REQ_FUA);
+		if (err) {
+			spin_unlock_irq(&nullb->lock);
+			return err;
+		}
+		sector += len >> SECTOR_SHIFT;
+	}
+	spin_unlock_irq(&nullb->lock);
+	return 0;
+}
+
+static void null_stop_queue(struct nullb *nullb)
+{
+	struct request_queue *q = nullb->q;
+
+	if (nullb->dev->queue_mode == NULL_Q_MQ)
+		blk_mq_stop_hw_queues(q);
+	else {
+		spin_lock_irq(q->queue_lock);
+		blk_stop_queue(q);
+		spin_unlock_irq(q->queue_lock);
+	}
+}
+
+static void null_restart_queue_async(struct nullb *nullb)
+{
+	struct request_queue *q = nullb->q;
+	unsigned long flags;
+
+	if (nullb->dev->queue_mode == NULL_Q_MQ)
+		blk_mq_start_stopped_hw_queues(q, true);
+	else {
+		spin_lock_irqsave(q->queue_lock, flags);
+		blk_start_queue_async(q);
+		spin_unlock_irqrestore(q->queue_lock, flags);
+	}
+}
+
+static blk_status_t null_handle_cmd(struct nullb_cmd *cmd)
+{
+	struct nullb_device *dev = cmd->nq->dev;
+	struct nullb *nullb = dev->nullb;
+	int err = 0;
+
+	if (test_bit(NULLB_DEV_FL_THROTTLED, &dev->flags)) {
+		struct request *rq = cmd->rq;
+
+		if (!hrtimer_active(&nullb->bw_timer))
+			hrtimer_restart(&nullb->bw_timer);
+
+		if (atomic_long_sub_return(blk_rq_bytes(rq),
+				&nullb->cur_bytes) < 0) {
+			null_stop_queue(nullb);
+			/* race with timer */
+			if (atomic_long_read(&nullb->cur_bytes) > 0)
+				null_restart_queue_async(nullb);
+			if (dev->queue_mode == NULL_Q_RQ) {
+				struct request_queue *q = nullb->q;
+
+				spin_lock_irq(q->queue_lock);
+				rq->rq_flags |= RQF_DONTPREP;
+				blk_requeue_request(q, rq);
+				spin_unlock_irq(q->queue_lock);
+				return BLK_STS_OK;
+			} else
+				/* requeue request */
+				return BLK_STS_RESOURCE;
+		}
+	}
+
+	if (nullb->dev->badblocks.shift != -1) {
+		int bad_sectors;
+		sector_t sector, size, first_bad;
+		bool is_flush = true;
+
+		if (dev->queue_mode == NULL_Q_BIO &&
+				bio_op(cmd->bio) != REQ_OP_FLUSH) {
+			is_flush = false;
+			sector = cmd->bio->bi_iter.bi_sector;
+			size = bio_sectors(cmd->bio);
+		}
+		if (dev->queue_mode != NULL_Q_BIO &&
+				req_op(cmd->rq) != REQ_OP_FLUSH) {
+			is_flush = false;
+			sector = blk_rq_pos(cmd->rq);
+			size = blk_rq_sectors(cmd->rq);
+		}
+		if (!is_flush && badblocks_check(&nullb->dev->badblocks, sector,
+				size, &first_bad, &bad_sectors)) {
+			cmd->error = BLK_STS_IOERR;
+			goto out;
+		}
+	}
+
+	if (dev->memory_backed) {
+		if (dev->queue_mode == NULL_Q_BIO) {
+			if (bio_op(cmd->bio) == REQ_OP_FLUSH)
+				err = null_handle_flush(nullb);
+			else
+				err = null_handle_bio(cmd);
+		} else {
+			if (req_op(cmd->rq) == REQ_OP_FLUSH)
+				err = null_handle_flush(nullb);
+			else
+				err = null_handle_rq(cmd);
+		}
+	}
+	cmd->error = errno_to_blk_status(err);
+out:
 	/* Complete IO by inline, softirq or timer */
-	switch (irqmode) {
+	switch (dev->irqmode) {
 	case NULL_IRQ_SOFTIRQ:
-		switch (queue_mode)  {
+		switch (dev->queue_mode)  {
 		case NULL_Q_MQ:
 			blk_mq_complete_request(cmd->rq);
 			break;
@@ -307,6 +1292,34 @@ static inline void null_handle_cmd(struct nullb_cmd *cmd)
 		null_cmd_end_timer(cmd);
 		break;
 	}
+	return BLK_STS_OK;
+}
+
+static enum hrtimer_restart nullb_bwtimer_fn(struct hrtimer *timer)
+{
+	struct nullb *nullb = container_of(timer, struct nullb, bw_timer);
+	ktime_t timer_interval = ktime_set(0, TIMER_INTERVAL);
+	unsigned int mbps = nullb->dev->mbps;
+
+	if (atomic_long_read(&nullb->cur_bytes) == mb_per_tick(mbps))
+		return HRTIMER_NORESTART;
+
+	atomic_long_set(&nullb->cur_bytes, mb_per_tick(mbps));
+	null_restart_queue_async(nullb);
+
+	hrtimer_forward_now(&nullb->bw_timer, timer_interval);
+
+	return HRTIMER_RESTART;
+}
+
+static void nullb_setup_bwtimer(struct nullb *nullb)
+{
+	ktime_t timer_interval = ktime_set(0, TIMER_INTERVAL);
+
+	hrtimer_init(&nullb->bw_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+	nullb->bw_timer.function = nullb_bwtimer_fn;
+	atomic_long_set(&nullb->cur_bytes, mb_per_tick(nullb->dev->mbps));
+	hrtimer_start(&nullb->bw_timer, timer_interval, HRTIMER_MODE_REL);
 }
 
 static struct nullb_queue *nullb_to_queue(struct nullb *nullb)
@@ -366,20 +1379,20 @@ static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx,
 			 const struct blk_mq_queue_data *bd)
 {
 	struct nullb_cmd *cmd = blk_mq_rq_to_pdu(bd->rq);
+	struct nullb_queue *nq = hctx->driver_data;
 
 	might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);
 
-	if (irqmode == NULL_IRQ_TIMER) {
+	if (nq->dev->irqmode == NULL_IRQ_TIMER) {
 		hrtimer_init(&cmd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 		cmd->timer.function = null_cmd_timer_expired;
 	}
 	cmd->rq = bd->rq;
-	cmd->nq = hctx->driver_data;
+	cmd->nq = nq;
 
 	blk_mq_start_request(bd->rq);
 
-	null_handle_cmd(cmd);
-	return BLK_STS_OK;
+	return null_handle_cmd(cmd);
 }
 
 static const struct blk_mq_ops null_mq_ops = {
@@ -438,7 +1451,8 @@ static int null_lnvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd)
 
 static int null_lnvm_id(struct nvm_dev *dev, struct nvm_id *id)
 {
-	sector_t size = gb * 1024 * 1024 * 1024ULL;
+	struct nullb *nullb = dev->q->queuedata;
+	sector_t size = (sector_t)nullb->dev->size * 1024 * 1024ULL;
 	sector_t blksize;
 	struct nvm_id_group *grp;
 
@@ -460,7 +1474,7 @@ static int null_lnvm_id(struct nvm_dev *dev, struct nvm_id *id)
 	id->ppaf.ch_offset = 56;
 	id->ppaf.ch_len = 8;
 
-	sector_div(size, bs); /* convert size to pages */
+	sector_div(size, nullb->dev->blocksize); /* convert size to pages */
 	size >>= 8; /* concert size to pgs pr blk */
 	grp = &id->grp;
 	grp->mtype = 0;
@@ -474,8 +1488,8 @@ static int null_lnvm_id(struct nvm_dev *dev, struct nvm_id *id)
 	grp->num_blk = blksize;
 	grp->num_pln = 1;
 
-	grp->fpg_sz = bs;
-	grp->csecs = bs;
+	grp->fpg_sz = nullb->dev->blocksize;
+	grp->csecs = nullb->dev->blocksize;
 	grp->trdt = 25000;
 	grp->trdm = 25000;
 	grp->tprt = 500000;
@@ -483,7 +1497,7 @@ static int null_lnvm_id(struct nvm_dev *dev, struct nvm_id *id)
 	grp->tbet = 1500000;
 	grp->tbem = 1500000;
 	grp->mpos = 0x010101; /* single plane rwe */
-	grp->cpar = hw_queue_depth;
+	grp->cpar = nullb->dev->hw_queue_depth;
 
 	return 0;
 }
@@ -568,19 +1582,44 @@ static void null_nvm_unregister(struct nullb *nullb) {}
 
 static void null_del_dev(struct nullb *nullb)
 {
+	struct nullb_device *dev = nullb->dev;
+
+	ida_simple_remove(&nullb_indexes, nullb->index);
+
 	list_del_init(&nullb->list);
 
-	if (use_lightnvm)
+	if (dev->use_lightnvm)
 		null_nvm_unregister(nullb);
 	else
 		del_gendisk(nullb->disk);
+
+	if (test_bit(NULLB_DEV_FL_THROTTLED, &nullb->dev->flags)) {
+		hrtimer_cancel(&nullb->bw_timer);
+		atomic_long_set(&nullb->cur_bytes, LONG_MAX);
+		null_restart_queue_async(nullb);
+	}
+
 	blk_cleanup_queue(nullb->q);
-	if (queue_mode == NULL_Q_MQ && nullb->tag_set == &nullb->__tag_set)
+	if (dev->queue_mode == NULL_Q_MQ &&
+	    nullb->tag_set == &nullb->__tag_set)
 		blk_mq_free_tag_set(nullb->tag_set);
-	if (!use_lightnvm)
+	if (!dev->use_lightnvm)
 		put_disk(nullb->disk);
 	cleanup_queues(nullb);
+	if (null_cache_active(nullb))
+		null_free_device_storage(nullb->dev, true);
 	kfree(nullb);
+	dev->nullb = NULL;
+}
+
+static void null_config_discard(struct nullb *nullb)
+{
+	if (nullb->dev->discard == false)
+		return;
+	nullb->q->limits.discard_granularity = nullb->dev->blocksize;
+	nullb->q->limits.discard_alignment = nullb->dev->blocksize;
+	blk_queue_max_discard_sectors(nullb->q, UINT_MAX >> 9);
+	queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, nullb->q);
 }
 
 static int null_open(struct block_device *bdev, fmode_t mode)
@@ -605,6 +1644,7 @@ static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq)
 
 	init_waitqueue_head(&nq->wait);
 	nq->queue_depth = nullb->queue_depth;
+	nq->dev = nullb->dev;
 }
 
 static void null_init_queues(struct nullb *nullb)
@@ -652,13 +1692,13 @@ static int setup_commands(struct nullb_queue *nq)
 
 static int setup_queues(struct nullb *nullb)
 {
-	nullb->queues = kzalloc(submit_queues * sizeof(struct nullb_queue),
-								GFP_KERNEL);
+	nullb->queues = kzalloc(nullb->dev->submit_queues *
+		sizeof(struct nullb_queue), GFP_KERNEL);
 	if (!nullb->queues)
 		return -ENOMEM;
 
 	nullb->nr_queues = 0;
-	nullb->queue_depth = hw_queue_depth;
+	nullb->queue_depth = nullb->dev->hw_queue_depth;
 
 	return 0;
 }
@@ -668,7 +1708,7 @@ static int init_driver_queues(struct nullb *nullb)
 	struct nullb_queue *nq;
 	int i, ret = 0;
 
-	for (i = 0; i < submit_queues; i++) {
+	for (i = 0; i < nullb->dev->submit_queues; i++) {
 		nq = &nullb->queues[i];
 
 		null_init_queue(nullb, nq);
@@ -686,10 +1726,10 @@ static int null_gendisk_register(struct nullb *nullb)
 	struct gendisk *disk;
 	sector_t size;
 
-	disk = nullb->disk = alloc_disk_node(1, home_node);
+	disk = nullb->disk = alloc_disk_node(1, nullb->dev->home_node);
 	if (!disk)
 		return -ENOMEM;
-	size = gb * 1024 * 1024 * 1024ULL;
+	size = (sector_t)nullb->dev->size * 1024 * 1024ULL;
 	set_capacity(disk, size >> 9);
 
 	disk->flags |= GENHD_FL_EXT_DEVT | GENHD_FL_SUPPRESS_PARTITION_INFO;
@@ -704,49 +1744,86 @@ static int null_gendisk_register(struct nullb *nullb)
 	return 0;
 }
 
-static int null_init_tag_set(struct blk_mq_tag_set *set)
+static int null_init_tag_set(struct nullb *nullb, struct blk_mq_tag_set *set)
 {
 	set->ops = &null_mq_ops;
-	set->nr_hw_queues = submit_queues;
-	set->queue_depth = hw_queue_depth;
-	set->numa_node = home_node;
+	set->nr_hw_queues = nullb ? nullb->dev->submit_queues :
+						g_submit_queues;
+	set->queue_depth = nullb ? nullb->dev->hw_queue_depth :
+						g_hw_queue_depth;
+	set->numa_node = nullb ? nullb->dev->home_node : g_home_node;
 	set->cmd_size	= sizeof(struct nullb_cmd);
 	set->flags = BLK_MQ_F_SHOULD_MERGE;
 	set->driver_data = NULL;
 
-	if (blocking)
+	if ((nullb && nullb->dev->blocking) || g_blocking)
 		set->flags |= BLK_MQ_F_BLOCKING;
 
 	return blk_mq_alloc_tag_set(set);
 }
 
-static int null_add_dev(void)
+static void null_validate_conf(struct nullb_device *dev)
+{
+	dev->blocksize = round_down(dev->blocksize, 512);
+	dev->blocksize = clamp_t(unsigned int, dev->blocksize, 512, 4096);
+	if (dev->use_lightnvm && dev->blocksize != 4096)
+		dev->blocksize = 4096;
+
+	if (dev->use_lightnvm && dev->queue_mode != NULL_Q_MQ)
+		dev->queue_mode = NULL_Q_MQ;
+
+	if (dev->queue_mode == NULL_Q_MQ && dev->use_per_node_hctx) {
+		if (dev->submit_queues != nr_online_nodes)
+			dev->submit_queues = nr_online_nodes;
+	} else if (dev->submit_queues > nr_cpu_ids)
+		dev->submit_queues = nr_cpu_ids;
+	else if (dev->submit_queues == 0)
+		dev->submit_queues = 1;
+
+	dev->queue_mode = min_t(unsigned int, dev->queue_mode, NULL_Q_MQ);
+	dev->irqmode = min_t(unsigned int, dev->irqmode, NULL_IRQ_TIMER);
+
+	/* Do memory allocation, so set blocking */
+	if (dev->memory_backed)
+		dev->blocking = true;
+	else /* cache is meaningless */
+		dev->cache_size = 0;
+	dev->cache_size = min_t(unsigned long, ULONG_MAX / 1024 / 1024,
+						dev->cache_size);
+	dev->mbps = min_t(unsigned int, 1024 * 40, dev->mbps);
+	/* can not stop a queue */
+	if (dev->queue_mode == NULL_Q_BIO)
+		dev->mbps = 0;
+}
+
+static int null_add_dev(struct nullb_device *dev)
 {
 	struct nullb *nullb;
 	int rv;
 
-	nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, home_node);
+	null_validate_conf(dev);
+
+	nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, dev->home_node);
 	if (!nullb) {
 		rv = -ENOMEM;
 		goto out;
 	}
+	nullb->dev = dev;
+	dev->nullb = nullb;
 
 	spin_lock_init(&nullb->lock);
 
-	if (queue_mode == NULL_Q_MQ && use_per_node_hctx)
-		submit_queues = nr_online_nodes;
-
 	rv = setup_queues(nullb);
 	if (rv)
 		goto out_free_nullb;
 
-	if (queue_mode == NULL_Q_MQ) {
+	if (dev->queue_mode == NULL_Q_MQ) {
 		if (shared_tags) {
 			nullb->tag_set = &tag_set;
 			rv = 0;
 		} else {
 			nullb->tag_set = &nullb->__tag_set;
-			rv = null_init_tag_set(nullb->tag_set);
+			rv = null_init_tag_set(nullb, nullb->tag_set);
 		}
 
 		if (rv)
@@ -758,8 +1835,8 @@ static int null_add_dev(void)
 			goto out_cleanup_tags;
 		}
 		null_init_queues(nullb);
-	} else if (queue_mode == NULL_Q_BIO) {
-		nullb->q = blk_alloc_queue_node(GFP_KERNEL, home_node);
+	} else if (dev->queue_mode == NULL_Q_BIO) {
+		nullb->q = blk_alloc_queue_node(GFP_KERNEL, dev->home_node);
 		if (!nullb->q) {
 			rv = -ENOMEM;
 			goto out_cleanup_queues;
@@ -769,7 +1846,8 @@ static int null_add_dev(void)
 		if (rv)
 			goto out_cleanup_blk_queue;
 	} else {
-		nullb->q = blk_init_queue_node(null_request_fn, &nullb->lock, home_node);
+		nullb->q = blk_init_queue_node(null_request_fn, &nullb->lock,
+						dev->home_node);
 		if (!nullb->q) {
 			rv = -ENOMEM;
 			goto out_cleanup_queues;
@@ -781,20 +1859,34 @@ static int null_add_dev(void)
 			goto out_cleanup_blk_queue;
 	}
 
+	if (dev->mbps) {
+		set_bit(NULLB_DEV_FL_THROTTLED, &dev->flags);
+		nullb_setup_bwtimer(nullb);
+	}
+
+	if (dev->cache_size > 0) {
+		set_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);
+		blk_queue_write_cache(nullb->q, true, true);
+		blk_queue_flush_queueable(nullb->q, true);
+	}
+
 	nullb->q->queuedata = nullb;
 	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, nullb->q);
 	queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, nullb->q);
 
 	mutex_lock(&lock);
-	nullb->index = nullb_indexes++;
+	nullb->index = ida_simple_get(&nullb_indexes, 0, 0, GFP_KERNEL);
+	dev->index = nullb->index;
 	mutex_unlock(&lock);
 
-	blk_queue_logical_block_size(nullb->q, bs);
-	blk_queue_physical_block_size(nullb->q, bs);
+	blk_queue_logical_block_size(nullb->q, dev->blocksize);
+	blk_queue_physical_block_size(nullb->q, dev->blocksize);
+
+	null_config_discard(nullb);
 
 	sprintf(nullb->disk_name, "nullb%d", nullb->index);
 
-	if (use_lightnvm)
+	if (dev->use_lightnvm)
 		rv = null_nvm_register(nullb);
 	else
 		rv = null_gendisk_register(nullb);
@@ -810,7 +1902,7 @@ static int null_add_dev(void)
 out_cleanup_blk_queue:
 	blk_cleanup_queue(nullb->q);
 out_cleanup_tags:
-	if (queue_mode == NULL_Q_MQ && nullb->tag_set == &nullb->__tag_set)
+	if (dev->queue_mode == NULL_Q_MQ && nullb->tag_set == &nullb->__tag_set)
 		blk_mq_free_tag_set(nullb->tag_set);
 out_cleanup_queues:
 	cleanup_queues(nullb);
@@ -825,51 +1917,63 @@ static int __init null_init(void)
 	int ret = 0;
 	unsigned int i;
 	struct nullb *nullb;
+	struct nullb_device *dev;
+
+	/* check for nullb_page.bitmap */
+	if (sizeof(unsigned long) * 8 - 2 < (PAGE_SIZE >> SECTOR_SHIFT))
+		return -EINVAL;
 
-	if (bs > PAGE_SIZE) {
+	if (g_bs > PAGE_SIZE) {
 		pr_warn("null_blk: invalid block size\n");
 		pr_warn("null_blk: defaults block size to %lu\n", PAGE_SIZE);
-		bs = PAGE_SIZE;
+		g_bs = PAGE_SIZE;
 	}
 
-	if (use_lightnvm && bs != 4096) {
+	if (g_use_lightnvm && g_bs != 4096) {
 		pr_warn("null_blk: LightNVM only supports 4k block size\n");
 		pr_warn("null_blk: defaults block size to 4k\n");
-		bs = 4096;
+		g_bs = 4096;
 	}
 
-	if (use_lightnvm && queue_mode != NULL_Q_MQ) {
+	if (g_use_lightnvm && g_queue_mode != NULL_Q_MQ) {
 		pr_warn("null_blk: LightNVM only supported for blk-mq\n");
 		pr_warn("null_blk: defaults queue mode to blk-mq\n");
-		queue_mode = NULL_Q_MQ;
+		g_queue_mode = NULL_Q_MQ;
 	}
 
-	if (queue_mode == NULL_Q_MQ && use_per_node_hctx) {
-		if (submit_queues < nr_online_nodes) {
-			pr_warn("null_blk: submit_queues param is set to %u.",
+	if (g_queue_mode == NULL_Q_MQ && g_use_per_node_hctx) {
+		if (g_submit_queues != nr_online_nodes) {
+			pr_warn("null_blk: submit_queues param is set to %u.\n",
 							nr_online_nodes);
-			submit_queues = nr_online_nodes;
+			g_submit_queues = nr_online_nodes;
 		}
-	} else if (submit_queues > nr_cpu_ids)
-		submit_queues = nr_cpu_ids;
-	else if (!submit_queues)
-		submit_queues = 1;
+	} else if (g_submit_queues > nr_cpu_ids)
+		g_submit_queues = nr_cpu_ids;
+	else if (g_submit_queues <= 0)
+		g_submit_queues = 1;
 
-	if (queue_mode == NULL_Q_MQ && shared_tags) {
-		ret = null_init_tag_set(&tag_set);
+	if (g_queue_mode == NULL_Q_MQ && shared_tags) {
+		ret = null_init_tag_set(NULL, &tag_set);
 		if (ret)
 			return ret;
 	}
 
+	config_group_init(&nullb_subsys.su_group);
+	mutex_init(&nullb_subsys.su_mutex);
+
+	ret = configfs_register_subsystem(&nullb_subsys);
+	if (ret)
+		goto err_tagset;
+
 	mutex_init(&lock);
 
 	null_major = register_blkdev(0, "nullb");
 	if (null_major < 0) {
 		ret = null_major;
-		goto err_tagset;
+		goto err_conf;
 	}
 
-	if (use_lightnvm) {
+	if (g_use_lightnvm) {
 		ppa_cache = kmem_cache_create("ppa_cache", 64 * sizeof(u64),
 								0, 0, NULL);
 		if (!ppa_cache) {
@@ -880,9 +1984,14 @@ static int __init null_init(void)
 	}
 
 	for (i = 0; i < nr_devices; i++) {
-		ret = null_add_dev();
-		if (ret)
+		dev = null_alloc_dev();
+		if (!dev)
+			goto err_dev;
+		ret = null_add_dev(dev);
+		if (ret) {
+			null_free_dev(dev);
 			goto err_dev;
+		}
 	}
 
 	pr_info("null: module loaded\n");
@@ -891,13 +2000,17 @@ static int __init null_init(void)
 err_dev:
 	while (!list_empty(&nullb_list)) {
 		nullb = list_entry(nullb_list.next, struct nullb, list);
+		dev = nullb->dev;
 		null_del_dev(nullb);
+		null_free_dev(dev);
 	}
 	kmem_cache_destroy(ppa_cache);
 err_ppa:
 	unregister_blkdev(null_major, "nullb");
+err_conf:
+	configfs_unregister_subsystem(&nullb_subsys);
 err_tagset:
-	if (queue_mode == NULL_Q_MQ && shared_tags)
+	if (g_queue_mode == NULL_Q_MQ && shared_tags)
 		blk_mq_free_tag_set(&tag_set);
 	return ret;
 }
@@ -906,16 +2019,22 @@ static void __exit null_exit(void)
 {
 	struct nullb *nullb;
 
+	configfs_unregister_subsystem(&nullb_subsys);
+
 	unregister_blkdev(null_major, "nullb");
 
 	mutex_lock(&lock);
 	while (!list_empty(&nullb_list)) {
+		struct nullb_device *dev;
+
 		nullb = list_entry(nullb_list.next, struct nullb, list);
+		dev = nullb->dev;
 		null_del_dev(nullb);
+		null_free_dev(dev);
 	}
 	mutex_unlock(&lock);
 
-	if (queue_mode == NULL_Q_MQ && shared_tags)
+	if (g_queue_mode == NULL_Q_MQ && shared_tags)
 		blk_mq_free_tag_set(&tag_set);
 
 	kmem_cache_destroy(ppa_cache);
@@ -924,5 +2043,5 @@ static void __exit null_exit(void)
 module_init(null_init);
 module_exit(null_exit);
 
-MODULE_AUTHOR("Jens Axboe <jaxboe@fusionio.com>");
+MODULE_AUTHOR("Jens Axboe <axboe@kernel.dk>");
 MODULE_LICENSE("GPL");
diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c
index 6b8b097abbb9..67974796c350 100644
--- a/drivers/block/pktcdvd.c
+++ b/drivers/block/pktcdvd.c
@@ -1028,7 +1028,7 @@ static void pkt_gather_data(struct pktcdvd_device *pd, struct packet_data *pkt)
 		bio = pkt->r_bios[f];
 		bio_reset(bio);
 		bio->bi_iter.bi_sector = pkt->sector + f * (CD_FRAMESIZE >> 9);
-		bio->bi_bdev = pd->bdev;
+		bio_set_dev(bio, pd->bdev);
 		bio->bi_end_io = pkt_end_io_read;
 		bio->bi_private = pkt;
 
@@ -1122,7 +1122,7 @@ static int pkt_start_recovery(struct packet_data *pkt)
 	pkt->sector = new_sector;
 
 	bio_reset(pkt->bio);
-	pkt->bio->bi_bdev = pd->bdev;
+	bio_set_set(pkt->bio, pd->bdev);
 	bio_set_op_attrs(pkt->bio, REQ_OP_WRITE, 0);
 	pkt->bio->bi_iter.bi_sector = new_sector;
 	pkt->bio->bi_iter.bi_size = pkt->frames * CD_FRAMESIZE;
@@ -1267,7 +1267,7 @@ static void pkt_start_write(struct pktcdvd_device *pd, struct packet_data *pkt)
 
 	bio_reset(pkt->w_bio);
 	pkt->w_bio->bi_iter.bi_sector = pkt->sector;
-	pkt->w_bio->bi_bdev = pd->bdev;
+	bio_set_dev(pkt->w_bio, pd->bdev);
 	pkt->w_bio->bi_end_io = pkt_end_io_packet_write;
 	pkt->w_bio->bi_private = pkt;
 
@@ -2314,7 +2314,7 @@ static void pkt_make_request_read(struct pktcdvd_device *pd, struct bio *bio)
 
 	psd->pd = pd;
 	psd->bio = bio;
-	cloned_bio->bi_bdev = pd->bdev;
+	bio_set_dev(cloned_bio, pd->bdev);
 	cloned_bio->bi_private = psd;
 	cloned_bio->bi_end_io = pkt_end_io_read_cloned;
 	pd->stats.secs_r += bio_sectors(bio);
@@ -2415,8 +2415,7 @@ static blk_qc_t pkt_make_request(struct request_queue *q, struct bio *bio)
 
 	pd = q->queuedata;
 	if (!pd) {
-		pr_err("%s incorrect request queue\n",
-		       bdevname(bio->bi_bdev, b));
+		pr_err("%s incorrect request queue\n", bio_devname(bio, b));
 		goto end_io;
 	}
 
diff --git a/drivers/block/ps3vram.c b/drivers/block/ps3vram.c
index e0e81cacd781..6a55959cbf78 100644
--- a/drivers/block/ps3vram.c
+++ b/drivers/block/ps3vram.c
@@ -409,10 +409,8 @@ static int ps3vram_cache_init(struct ps3_system_bus_device *dev)
 	priv->cache.page_size = CACHE_PAGE_SIZE;
 	priv->cache.tags = kzalloc(sizeof(struct ps3vram_tag) *
 				   CACHE_PAGE_COUNT, GFP_KERNEL);
-	if (priv->cache.tags == NULL) {
-		dev_err(&dev->core, "Could not allocate cache tags\n");
+	if (!priv->cache.tags)
 		return -ENOMEM;
-	}
 
 	dev_info(&dev->core, "Created ram cache: %d entries, %d KiB each\n",
 		CACHE_PAGE_COUNT, CACHE_PAGE_SIZE / 1024);
@@ -743,7 +741,11 @@ static int ps3vram_probe(struct ps3_system_bus_device *dev)
 		goto out_unmap_reports;
 	}
 
-	ps3vram_cache_init(dev);
+	error = ps3vram_cache_init(dev);
+	if (error < 0) {
+		goto out_unmap_reports;
+	}
+
 	ps3vram_proc_init(dev);
 
 	queue = blk_alloc_queue(GFP_KERNEL);
diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c
index b008b6a98098..b640ad8a6d20 100644
--- a/drivers/block/rbd.c
+++ b/drivers/block/rbd.c
@@ -3435,7 +3435,7 @@ static void rbd_acquire_lock(struct work_struct *work)
 	struct rbd_device *rbd_dev = container_of(to_delayed_work(work),
 					    struct rbd_device, lock_dwork);
 	enum rbd_lock_state lock_state;
-	int ret;
+	int ret = 0;
 
 	dout("%s rbd_dev %p\n", __func__, rbd_dev);
 again:
diff --git a/drivers/block/rsxx/dev.c b/drivers/block/rsxx/dev.c
index 7f4acebf4657..e397d3ee7308 100644
--- a/drivers/block/rsxx/dev.c
+++ b/drivers/block/rsxx/dev.c
@@ -112,7 +112,7 @@ static const struct block_device_operations rsxx_fops = {
 
 static void disk_stats_start(struct rsxx_cardinfo *card, struct bio *bio)
 {
-	generic_start_io_acct(bio_data_dir(bio), bio_sectors(bio),
+	generic_start_io_acct(card->queue, bio_data_dir(bio), bio_sectors(bio),
 			     &card->gendisk->part0);
 }
 
@@ -120,8 +120,8 @@ static void disk_stats_complete(struct rsxx_cardinfo *card,
 				struct bio *bio,
 				unsigned long start_time)
 {
-	generic_end_io_acct(bio_data_dir(bio), &card->gendisk->part0,
-			   start_time);
+	generic_end_io_acct(card->queue, bio_data_dir(bio),
+				&card->gendisk->part0, start_time);
 }
 
 static void bio_dma_done_cb(struct rsxx_cardinfo *card,
diff --git a/drivers/block/skd_main.c b/drivers/block/skd_main.c
index d0368682bd43..7cedb4295e9d 100644
--- a/drivers/block/skd_main.c
+++ b/drivers/block/skd_main.c
@@ -1,19 +1,12 @@
-/* Copyright 2012 STEC, Inc.
+/*
+ * Driver for sTec s1120 PCIe SSDs. sTec was acquired in 2013 by HGST and HGST
+ * was acquired by Western Digital in 2012.
+ *
+ * Copyright 2012 sTec, Inc.
+ * Copyright (c) 2017 Western Digital Corporation or its affiliates.
  *
- * This file is licensed under the terms of the 3-clause
- * BSD License (http://opensource.org/licenses/BSD-3-Clause)
- * or the GNU GPL-2.0 (http://www.gnu.org/licenses/gpl-2.0.html),
- * at your option. Both licenses are also available in the LICENSE file
- * distributed with this project. This file may not be copied, modified,
- * or distributed except in accordance with those terms.
- * Gordoni Waidhofer <gwaidhofer@stec-inc.com>
- * Initial Driver Design!
- * Thomas Swann <tswann@stec-inc.com>
- * Interrupt handling.
- * Ramprasad Chinthekindi <rchinthekindi@stec-inc.com>
- * biomode implementation.
- * Akhil Bhansali <abhansali@stec-inc.com>
- * Added support for DISCARD / FLUSH and FUA.
+ * This file is part of the Linux kernel, and is made available under
+ * the terms of the GNU General Public License version 2.
  */
 
 #include <linux/kernel.h>
@@ -23,11 +16,11 @@
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/blkdev.h>
+#include <linux/blk-mq.h>
 #include <linux/sched.h>
 #include <linux/interrupt.h>
 #include <linux/compiler.h>
 #include <linux/workqueue.h>
-#include <linux/bitops.h>
 #include <linux/delay.h>
 #include <linux/time.h>
 #include <linux/hdreg.h>
@@ -37,9 +30,9 @@
 #include <linux/version.h>
 #include <linux/err.h>
 #include <linux/aer.h>
-#include <linux/ctype.h>
 #include <linux/wait.h>
-#include <linux/uio.h>
+#include <linux/stringify.h>
+#include <linux/slab_def.h>
 #include <scsi/scsi.h>
 #include <scsi/sg.h>
 #include <linux/io.h>
@@ -51,19 +44,6 @@
 static int skd_dbg_level;
 static int skd_isr_comp_limit = 4;
 
-enum {
-	STEC_LINK_2_5GTS = 0,
-	STEC_LINK_5GTS = 1,
-	STEC_LINK_8GTS = 2,
-	STEC_LINK_UNKNOWN = 0xFF
-};
-
-enum {
-	SKD_FLUSH_INITIALIZER,
-	SKD_FLUSH_ZERO_SIZE_FIRST,
-	SKD_FLUSH_DATA_SECOND,
-};
-
 #define SKD_ASSERT(expr) \
 	do { \
 		if (unlikely(!(expr))) { \
@@ -73,17 +53,11 @@ enum {
 	} while (0)
 
 #define DRV_NAME "skd"
-#define DRV_VERSION "2.2.1"
-#define DRV_BUILD_ID "0260"
 #define PFX DRV_NAME ": "
-#define DRV_BIN_VERSION 0x100
-#define DRV_VER_COMPL   "2.2.1." DRV_BUILD_ID
 
-MODULE_AUTHOR("bug-reports: support@stec-inc.com");
-MODULE_LICENSE("Dual BSD/GPL");
+MODULE_LICENSE("GPL");
 
-MODULE_DESCRIPTION("STEC s1120 PCIe SSD block driver (b" DRV_BUILD_ID ")");
-MODULE_VERSION(DRV_VERSION "-" DRV_BUILD_ID);
+MODULE_DESCRIPTION("STEC s1120 PCIe SSD block driver");
 
 #define PCI_VENDOR_ID_STEC      0x1B39
 #define PCI_DEVICE_ID_S1120     0x0001
@@ -96,34 +70,32 @@ MODULE_VERSION(DRV_VERSION "-" DRV_BUILD_ID);
 #define SKD_PAUSE_TIMEOUT       (5 * 1000)
 
 #define SKD_N_FITMSG_BYTES      (512u)
+#define SKD_MAX_REQ_PER_MSG	14
 
-#define SKD_N_SPECIAL_CONTEXT   32u
 #define SKD_N_SPECIAL_FITMSG_BYTES      (128u)
 
 /* SG elements are 32 bytes, so we can make this 4096 and still be under the
  * 128KB limit.  That allows 4096*4K = 16M xfer size
  */
 #define SKD_N_SG_PER_REQ_DEFAULT 256u
-#define SKD_N_SG_PER_SPECIAL    256u
 
 #define SKD_N_COMPLETION_ENTRY  256u
 #define SKD_N_READ_CAP_BYTES    (8u)
 
 #define SKD_N_INTERNAL_BYTES    (512u)
 
+#define SKD_SKCOMP_SIZE							\
+	((sizeof(struct fit_completion_entry_v1) +			\
+	  sizeof(struct fit_comp_error_info)) * SKD_N_COMPLETION_ENTRY)
+
 /* 5 bits of uniqifier, 0xF800 */
-#define SKD_ID_INCR             (0x400)
 #define SKD_ID_TABLE_MASK       (3u << 8u)
 #define  SKD_ID_RW_REQUEST      (0u << 8u)
 #define  SKD_ID_INTERNAL        (1u << 8u)
-#define  SKD_ID_SPECIAL_REQUEST (2u << 8u)
 #define  SKD_ID_FIT_MSG         (3u << 8u)
 #define SKD_ID_SLOT_MASK        0x00FFu
 #define SKD_ID_SLOT_AND_TABLE_MASK 0x03FFu
 
-#define SKD_N_TIMEOUT_SLOT      4u
-#define SKD_TIMEOUT_SLOT_MASK   3u
-
 #define SKD_N_MAX_SECTORS 2048u
 
 #define SKD_MAX_RETRIES 2u
@@ -141,7 +113,6 @@ enum skd_drvr_state {
 	SKD_DRVR_STATE_ONLINE,
 	SKD_DRVR_STATE_PAUSING,
 	SKD_DRVR_STATE_PAUSED,
-	SKD_DRVR_STATE_DRAINING_TIMEOUT,
 	SKD_DRVR_STATE_RESTARTING,
 	SKD_DRVR_STATE_RESUMING,
 	SKD_DRVR_STATE_STOPPING,
@@ -158,7 +129,6 @@ enum skd_drvr_state {
 #define SKD_WAIT_BOOT_TIMO      SKD_TIMER_SECONDS(90u)
 #define SKD_STARTING_TIMO       SKD_TIMER_SECONDS(8u)
 #define SKD_RESTARTING_TIMO     SKD_TIMER_MINUTES(4u)
-#define SKD_DRAINING_TIMO       SKD_TIMER_SECONDS(6u)
 #define SKD_BUSY_TIMO           SKD_TIMER_MINUTES(20u)
 #define SKD_STARTED_BUSY_TIMO   SKD_TIMER_SECONDS(60u)
 #define SKD_START_WAIT_SECONDS  90u
@@ -169,12 +139,6 @@ enum skd_req_state {
 	SKD_REQ_STATE_BUSY,
 	SKD_REQ_STATE_COMPLETED,
 	SKD_REQ_STATE_TIMEOUT,
-	SKD_REQ_STATE_ABORTED,
-};
-
-enum skd_fit_msg_state {
-	SKD_MSG_STATE_IDLE,
-	SKD_MSG_STATE_BUSY,
 };
 
 enum skd_check_status_action {
@@ -185,34 +149,29 @@ enum skd_check_status_action {
 	SKD_CHECK_STATUS_BUSY_IMMINENT,
 };
 
-struct skd_fitmsg_context {
-	enum skd_fit_msg_state state;
-
-	struct skd_fitmsg_context *next;
+struct skd_msg_buf {
+	struct fit_msg_hdr	fmh;
+	struct skd_scsi_request	scsi[SKD_MAX_REQ_PER_MSG];
+};
 
+struct skd_fitmsg_context {
 	u32 id;
-	u16 outstanding;
 
 	u32 length;
-	u32 offset;
 
-	u8 *msg_buf;
+	struct skd_msg_buf *msg_buf;
 	dma_addr_t mb_dma_address;
 };
 
 struct skd_request_context {
 	enum skd_req_state state;
 
-	struct skd_request_context *next;
-
 	u16 id;
 	u32 fitmsg_id;
 
-	struct request *req;
 	u8 flush_cmd;
 
-	u32 timeout_stamp;
-	u8 sg_data_dir;
+	enum dma_data_direction data_dir;
 	struct scatterlist *sg;
 	u32 n_sg;
 	u32 sg_byte_count;
@@ -224,38 +183,19 @@ struct skd_request_context {
 
 	struct fit_comp_error_info err_info;
 
+	blk_status_t status;
 };
-#define SKD_DATA_DIR_HOST_TO_CARD       1
-#define SKD_DATA_DIR_CARD_TO_HOST       2
 
 struct skd_special_context {
 	struct skd_request_context req;
 
-	u8 orphaned;
-
 	void *data_buf;
 	dma_addr_t db_dma_address;
 
-	u8 *msg_buf;
+	struct skd_msg_buf *msg_buf;
 	dma_addr_t mb_dma_address;
 };
 
-struct skd_sg_io {
-	fmode_t mode;
-	void __user *argp;
-
-	struct sg_io_hdr sg;
-
-	u8 cdb[16];
-
-	u32 dxfer_len;
-	u32 iovcnt;
-	struct sg_iovec *iov;
-	struct sg_iovec no_iov_iov;
-
-	struct skd_special_context *skspcl;
-};
-
 typedef enum skd_irq_type {
 	SKD_IRQ_LEGACY,
 	SKD_IRQ_MSI,
@@ -265,7 +205,7 @@ typedef enum skd_irq_type {
 #define SKD_MAX_BARS                    2
 
 struct skd_device {
-	volatile void __iomem *mem_map[SKD_MAX_BARS];
+	void __iomem *mem_map[SKD_MAX_BARS];
 	resource_size_t mem_phys[SKD_MAX_BARS];
 	u32 mem_size[SKD_MAX_BARS];
 
@@ -276,21 +216,20 @@ struct skd_device {
 
 	spinlock_t lock;
 	struct gendisk *disk;
+	struct blk_mq_tag_set tag_set;
 	struct request_queue *queue;
+	struct skd_fitmsg_context *skmsg;
 	struct device *class_dev;
 	int gendisk_on;
 	int sync_done;
 
-	atomic_t device_count;
 	u32 devno;
 	u32 major;
-	char name[32];
 	char isr_name[30];
 
 	enum skd_drvr_state state;
 	u32 drive_state;
 
-	u32 in_flight;
 	u32 cur_max_queue_depth;
 	u32 queue_low_water_mark;
 	u32 dev_max_queue_depth;
@@ -298,27 +237,20 @@ struct skd_device {
 	u32 num_fitmsg_context;
 	u32 num_req_context;
 
-	u32 timeout_slot[SKD_N_TIMEOUT_SLOT];
-	u32 timeout_stamp;
-	struct skd_fitmsg_context *skmsg_free_list;
 	struct skd_fitmsg_context *skmsg_table;
 
-	struct skd_request_context *skreq_free_list;
-	struct skd_request_context *skreq_table;
-
-	struct skd_special_context *skspcl_free_list;
-	struct skd_special_context *skspcl_table;
-
 	struct skd_special_context internal_skspcl;
 	u32 read_cap_blocksize;
 	u32 read_cap_last_lba;
 	int read_cap_is_valid;
 	int inquiry_is_valid;
 	u8 inq_serial_num[13];  /*12 chars plus null term */
-	u8 id_str[80];          /* holds a composite name (pci + sernum) */
 
 	u8 skcomp_cycle;
 	u32 skcomp_ix;
+	struct kmem_cache *msgbuf_cache;
+	struct kmem_cache *sglist_cache;
+	struct kmem_cache *databuf_cache;
 	struct fit_completion_entry_v1 *skcomp_table;
 	struct fit_comp_error_info *skerr_table;
 	dma_addr_t cq_dma_address;
@@ -329,7 +261,6 @@ struct skd_device {
 	u32 timer_countdown;
 	u32 timer_substate;
 
-	int n_special;
 	int sgs_per_request;
 	u32 last_mtd;
 
@@ -343,7 +274,7 @@ struct skd_device {
 
 	u32 timo_slot;
 
-
+	struct work_struct start_queue;
 	struct work_struct completion_worker;
 };
 
@@ -353,53 +284,32 @@ struct skd_device {
 
 static inline u32 skd_reg_read32(struct skd_device *skdev, u32 offset)
 {
-	u32 val;
-
-	if (likely(skdev->dbg_level < 2))
-		return readl(skdev->mem_map[1] + offset);
-	else {
-		barrier();
-		val = readl(skdev->mem_map[1] + offset);
-		barrier();
-		pr_debug("%s:%s:%d offset %x = %x\n",
-			 skdev->name, __func__, __LINE__, offset, val);
-		return val;
-	}
+	u32 val = readl(skdev->mem_map[1] + offset);
 
+	if (unlikely(skdev->dbg_level >= 2))
+		dev_dbg(&skdev->pdev->dev, "offset %x = %x\n", offset, val);
+	return val;
 }
 
 static inline void skd_reg_write32(struct skd_device *skdev, u32 val,
 				   u32 offset)
 {
-	if (likely(skdev->dbg_level < 2)) {
-		writel(val, skdev->mem_map[1] + offset);
-		barrier();
-	} else {
-		barrier();
-		writel(val, skdev->mem_map[1] + offset);
-		barrier();
-		pr_debug("%s:%s:%d offset %x = %x\n",
-			 skdev->name, __func__, __LINE__, offset, val);
-	}
+	writel(val, skdev->mem_map[1] + offset);
+	if (unlikely(skdev->dbg_level >= 2))
+		dev_dbg(&skdev->pdev->dev, "offset %x = %x\n", offset, val);
 }
 
 static inline void skd_reg_write64(struct skd_device *skdev, u64 val,
 				   u32 offset)
 {
-	if (likely(skdev->dbg_level < 2)) {
-		writeq(val, skdev->mem_map[1] + offset);
-		barrier();
-	} else {
-		barrier();
-		writeq(val, skdev->mem_map[1] + offset);
-		barrier();
-		pr_debug("%s:%s:%d offset %x = %016llx\n",
-			 skdev->name, __func__, __LINE__, offset, val);
-	}
+	writeq(val, skdev->mem_map[1] + offset);
+	if (unlikely(skdev->dbg_level >= 2))
+		dev_dbg(&skdev->pdev->dev, "offset %x = %016llx\n", offset,
+			val);
 }
 
 
-#define SKD_IRQ_DEFAULT SKD_IRQ_MSI
+#define SKD_IRQ_DEFAULT SKD_IRQ_MSIX
 static int skd_isr_type = SKD_IRQ_DEFAULT;
 
 module_param(skd_isr_type, int, 0444);
@@ -412,7 +322,7 @@ static int skd_max_req_per_msg = SKD_MAX_REQ_PER_MSG_DEFAULT;
 module_param(skd_max_req_per_msg, int, 0444);
 MODULE_PARM_DESC(skd_max_req_per_msg,
 		 "Maximum SCSI requests packed in a single message."
-		 " (1-14, default==1)");
+		 " (1-" __stringify(SKD_MAX_REQ_PER_MSG) ", default==1)");
 
 #define SKD_MAX_QUEUE_DEPTH_DEFAULT 64
 #define SKD_MAX_QUEUE_DEPTH_DEFAULT_STR "64"
@@ -429,10 +339,10 @@ MODULE_PARM_DESC(skd_sgs_per_request,
 		 "Maximum SG elements per block request."
 		 " (1-4096, default==256)");
 
-static int skd_max_pass_thru = SKD_N_SPECIAL_CONTEXT;
+static int skd_max_pass_thru = 1;
 module_param(skd_max_pass_thru, int, 0444);
 MODULE_PARM_DESC(skd_max_pass_thru,
-		 "Maximum SCSI pass-thru at a time." " (1-50, default==32)");
+		 "Maximum SCSI pass-thru at a time. IGNORED");
 
 module_param(skd_dbg_level, int, 0444);
 MODULE_PARM_DESC(skd_dbg_level, "s1120 debug level (0,1,2)");
@@ -449,9 +359,6 @@ static void skd_send_fitmsg(struct skd_device *skdev,
 			    struct skd_fitmsg_context *skmsg);
 static void skd_send_special_fitmsg(struct skd_device *skdev,
 				    struct skd_special_context *skspcl);
-static void skd_request_fn(struct request_queue *rq);
-static void skd_end_request(struct skd_device *skdev,
-		struct skd_request_context *skreq, blk_status_t status);
 static bool skd_preop_sg_list(struct skd_device *skdev,
 			     struct skd_request_context *skreq);
 static void skd_postop_sg_list(struct skd_device *skdev,
@@ -460,19 +367,14 @@ static void skd_postop_sg_list(struct skd_device *skdev,
 static void skd_restart_device(struct skd_device *skdev);
 static int skd_quiesce_dev(struct skd_device *skdev);
 static int skd_unquiesce_dev(struct skd_device *skdev);
-static void skd_release_special(struct skd_device *skdev,
-				struct skd_special_context *skspcl);
 static void skd_disable_interrupts(struct skd_device *skdev);
 static void skd_isr_fwstate(struct skd_device *skdev);
-static void skd_recover_requests(struct skd_device *skdev, int requeue);
+static void skd_recover_requests(struct skd_device *skdev);
 static void skd_soft_reset(struct skd_device *skdev);
 
-static const char *skd_name(struct skd_device *skdev);
 const char *skd_drive_state_to_str(int state);
 const char *skd_skdev_state_to_str(enum skd_drvr_state state);
 static void skd_log_skdev(struct skd_device *skdev, const char *event);
-static void skd_log_skmsg(struct skd_device *skdev,
-			  struct skd_fitmsg_context *skmsg, const char *event);
 static void skd_log_skreq(struct skd_device *skdev,
 			  struct skd_request_context *skreq, const char *event);
 
@@ -481,18 +383,20 @@ static void skd_log_skreq(struct skd_device *skdev,
  * READ/WRITE REQUESTS
  *****************************************************************************
  */
-static void skd_fail_all_pending(struct skd_device *skdev)
+static void skd_inc_in_flight(struct request *rq, void *data, bool reserved)
 {
-	struct request_queue *q = skdev->queue;
-	struct request *req;
+	int *count = data;
 
-	for (;; ) {
-		req = blk_peek_request(q);
-		if (req == NULL)
-			break;
-		blk_start_request(req);
-		__blk_end_request_all(req, BLK_STS_IOERR);
-	}
+	count++;
+}
+
+static int skd_in_flight(struct skd_device *skdev)
+{
+	int count = 0;
+
+	blk_mq_tagset_busy_iter(&skdev->tag_set, skd_inc_in_flight, &count);
+
+	return count;
 }
 
 static void
@@ -501,9 +405,9 @@ skd_prep_rw_cdb(struct skd_scsi_request *scsi_req,
 		unsigned count)
 {
 	if (data_dir == READ)
-		scsi_req->cdb[0] = 0x28;
+		scsi_req->cdb[0] = READ_10;
 	else
-		scsi_req->cdb[0] = 0x2a;
+		scsi_req->cdb[0] = WRITE_10;
 
 	scsi_req->cdb[1] = 0;
 	scsi_req->cdb[2] = (lba & 0xff000000) >> 24;
@@ -522,7 +426,7 @@ skd_prep_zerosize_flush_cdb(struct skd_scsi_request *scsi_req,
 {
 	skreq->flush_cmd = 1;
 
-	scsi_req->cdb[0] = 0x35;
+	scsi_req->cdb[0] = SYNCHRONIZE_CACHE;
 	scsi_req->cdb[1] = 0;
 	scsi_req->cdb[2] = 0;
 	scsi_req->cdb[3] = 0;
@@ -534,307 +438,194 @@ skd_prep_zerosize_flush_cdb(struct skd_scsi_request *scsi_req,
 	scsi_req->cdb[9] = 0;
 }
 
-static void skd_request_fn_not_online(struct request_queue *q);
-
-static void skd_request_fn(struct request_queue *q)
+/*
+ * Return true if and only if all pending requests should be failed.
+ */
+static bool skd_fail_all(struct request_queue *q)
 {
 	struct skd_device *skdev = q->queuedata;
-	struct skd_fitmsg_context *skmsg = NULL;
-	struct fit_msg_hdr *fmh = NULL;
-	struct skd_request_context *skreq;
-	struct request *req = NULL;
-	struct skd_scsi_request *scsi_req;
-	unsigned long io_flags;
-	u32 lba;
-	u32 count;
-	int data_dir;
-	u32 be_lba;
-	u32 be_count;
-	u64 be_dmaa;
-	u64 cmdctxt;
-	u32 timo_slot;
-	void *cmd_ptr;
-	int flush, fua;
-
-	if (skdev->state != SKD_DRVR_STATE_ONLINE) {
-		skd_request_fn_not_online(q);
-		return;
-	}
 
-	if (blk_queue_stopped(skdev->queue)) {
-		if (skdev->skmsg_free_list == NULL ||
-		    skdev->skreq_free_list == NULL ||
-		    skdev->in_flight >= skdev->queue_low_water_mark)
-			/* There is still some kind of shortage */
-			return;
-
-		queue_flag_clear(QUEUE_FLAG_STOPPED, skdev->queue);
-	}
+	SKD_ASSERT(skdev->state != SKD_DRVR_STATE_ONLINE);
 
-	/*
-	 * Stop conditions:
-	 *  - There are no more native requests
-	 *  - There are already the maximum number of requests in progress
-	 *  - There are no more skd_request_context entries
-	 *  - There are no more FIT msg buffers
+	skd_log_skdev(skdev, "req_not_online");
+	switch (skdev->state) {
+	case SKD_DRVR_STATE_PAUSING:
+	case SKD_DRVR_STATE_PAUSED:
+	case SKD_DRVR_STATE_STARTING:
+	case SKD_DRVR_STATE_RESTARTING:
+	case SKD_DRVR_STATE_WAIT_BOOT:
+	/* In case of starting, we haven't started the queue,
+	 * so we can't get here... but requests are
+	 * possibly hanging out waiting for us because we
+	 * reported the dev/skd0 already.  They'll wait
+	 * forever if connect doesn't complete.
+	 * What to do??? delay dev/skd0 ??
 	 */
-	for (;; ) {
-
-		flush = fua = 0;
-
-		req = blk_peek_request(q);
-
-		/* Are there any native requests to start? */
-		if (req == NULL)
-			break;
-
-		lba = (u32)blk_rq_pos(req);
-		count = blk_rq_sectors(req);
-		data_dir = rq_data_dir(req);
-		io_flags = req->cmd_flags;
-
-		if (req_op(req) == REQ_OP_FLUSH)
-			flush++;
-
-		if (io_flags & REQ_FUA)
-			fua++;
-
-		pr_debug("%s:%s:%d new req=%p lba=%u(0x%x) "
-			 "count=%u(0x%x) dir=%d\n",
-			 skdev->name, __func__, __LINE__,
-			 req, lba, lba, count, count, data_dir);
-
-		/* At this point we know there is a request */
+	case SKD_DRVR_STATE_BUSY:
+	case SKD_DRVR_STATE_BUSY_IMMINENT:
+	case SKD_DRVR_STATE_BUSY_ERASE:
+		return false;
 
-		/* Are too many requets already in progress? */
-		if (skdev->in_flight >= skdev->cur_max_queue_depth) {
-			pr_debug("%s:%s:%d qdepth %d, limit %d\n",
-				 skdev->name, __func__, __LINE__,
-				 skdev->in_flight, skdev->cur_max_queue_depth);
-			break;
-		}
+	case SKD_DRVR_STATE_BUSY_SANITIZE:
+	case SKD_DRVR_STATE_STOPPING:
+	case SKD_DRVR_STATE_SYNCING:
+	case SKD_DRVR_STATE_FAULT:
+	case SKD_DRVR_STATE_DISAPPEARED:
+	default:
+		return true;
+	}
+}
 
-		/* Is a skd_request_context available? */
-		skreq = skdev->skreq_free_list;
-		if (skreq == NULL) {
-			pr_debug("%s:%s:%d Out of req=%p\n",
-				 skdev->name, __func__, __LINE__, q);
-			break;
-		}
-		SKD_ASSERT(skreq->state == SKD_REQ_STATE_IDLE);
-		SKD_ASSERT((skreq->id & SKD_ID_INCR) == 0);
-
-		/* Now we check to see if we can get a fit msg */
-		if (skmsg == NULL) {
-			if (skdev->skmsg_free_list == NULL) {
-				pr_debug("%s:%s:%d Out of msg\n",
-					 skdev->name, __func__, __LINE__);
-				break;
-			}
-		}
+static blk_status_t skd_mq_queue_rq(struct blk_mq_hw_ctx *hctx,
+				    const struct blk_mq_queue_data *mqd)
+{
+	struct request *const req = mqd->rq;
+	struct request_queue *const q = req->q;
+	struct skd_device *skdev = q->queuedata;
+	struct skd_fitmsg_context *skmsg;
+	struct fit_msg_hdr *fmh;
+	const u32 tag = blk_mq_unique_tag(req);
+	struct skd_request_context *const skreq = blk_mq_rq_to_pdu(req);
+	struct skd_scsi_request *scsi_req;
+	unsigned long flags = 0;
+	const u32 lba = blk_rq_pos(req);
+	const u32 count = blk_rq_sectors(req);
+	const int data_dir = rq_data_dir(req);
 
-		skreq->flush_cmd = 0;
-		skreq->n_sg = 0;
-		skreq->sg_byte_count = 0;
+	if (unlikely(skdev->state != SKD_DRVR_STATE_ONLINE))
+		return skd_fail_all(q) ? BLK_STS_IOERR : BLK_STS_RESOURCE;
 
-		/*
-		 * OK to now dequeue request from q.
-		 *
-		 * At this point we are comitted to either start or reject
-		 * the native request. Note that skd_request_context is
-		 * available but is still at the head of the free list.
-		 */
-		blk_start_request(req);
-		skreq->req = req;
-		skreq->fitmsg_id = 0;
-
-		/* Either a FIT msg is in progress or we have to start one. */
-		if (skmsg == NULL) {
-			/* Are there any FIT msg buffers available? */
-			skmsg = skdev->skmsg_free_list;
-			if (skmsg == NULL) {
-				pr_debug("%s:%s:%d Out of msg skdev=%p\n",
-					 skdev->name, __func__, __LINE__,
-					 skdev);
-				break;
-			}
-			SKD_ASSERT(skmsg->state == SKD_MSG_STATE_IDLE);
-			SKD_ASSERT((skmsg->id & SKD_ID_INCR) == 0);
+	blk_mq_start_request(req);
 
-			skdev->skmsg_free_list = skmsg->next;
+	WARN_ONCE(tag >= skd_max_queue_depth, "%#x > %#x (nr_requests = %lu)\n",
+		  tag, skd_max_queue_depth, q->nr_requests);
 
-			skmsg->state = SKD_MSG_STATE_BUSY;
-			skmsg->id += SKD_ID_INCR;
+	SKD_ASSERT(skreq->state == SKD_REQ_STATE_IDLE);
 
-			/* Initialize the FIT msg header */
-			fmh = (struct fit_msg_hdr *)skmsg->msg_buf;
-			memset(fmh, 0, sizeof(*fmh));
-			fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
-			skmsg->length = sizeof(*fmh);
-		}
+	dev_dbg(&skdev->pdev->dev,
+		"new req=%p lba=%u(0x%x) count=%u(0x%x) dir=%d\n", req, lba,
+		lba, count, count, data_dir);
 
-		skreq->fitmsg_id = skmsg->id;
+	skreq->id = tag + SKD_ID_RW_REQUEST;
+	skreq->flush_cmd = 0;
+	skreq->n_sg = 0;
+	skreq->sg_byte_count = 0;
 
-		/*
-		 * Note that a FIT msg may have just been started
-		 * but contains no SoFIT requests yet.
-		 */
+	skreq->fitmsg_id = 0;
 
-		/*
-		 * Transcode the request, checking as we go. The outcome of
-		 * the transcoding is represented by the error variable.
-		 */
-		cmd_ptr = &skmsg->msg_buf[skmsg->length];
-		memset(cmd_ptr, 0, 32);
+	skreq->data_dir = data_dir == READ ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
 
-		be_lba = cpu_to_be32(lba);
-		be_count = cpu_to_be32(count);
-		be_dmaa = cpu_to_be64((u64)skreq->sksg_dma_address);
-		cmdctxt = skreq->id + SKD_ID_INCR;
+	if (req->bio && !skd_preop_sg_list(skdev, skreq)) {
+		dev_dbg(&skdev->pdev->dev, "error Out\n");
+		skreq->status = BLK_STS_RESOURCE;
+		blk_mq_complete_request(req);
+		return BLK_STS_OK;
+	}
 
-		scsi_req = cmd_ptr;
-		scsi_req->hdr.tag = cmdctxt;
-		scsi_req->hdr.sg_list_dma_address = be_dmaa;
+	dma_sync_single_for_device(&skdev->pdev->dev, skreq->sksg_dma_address,
+				   skreq->n_sg *
+				   sizeof(struct fit_sg_descriptor),
+				   DMA_TO_DEVICE);
 
-		if (data_dir == READ)
-			skreq->sg_data_dir = SKD_DATA_DIR_CARD_TO_HOST;
-		else
-			skreq->sg_data_dir = SKD_DATA_DIR_HOST_TO_CARD;
+	/* Either a FIT msg is in progress or we have to start one. */
+	if (skd_max_req_per_msg == 1) {
+		skmsg = NULL;
+	} else {
+		spin_lock_irqsave(&skdev->lock, flags);
+		skmsg = skdev->skmsg;
+	}
+	if (!skmsg) {
+		skmsg = &skdev->skmsg_table[tag];
+		skdev->skmsg = skmsg;
 
-		if (flush == SKD_FLUSH_ZERO_SIZE_FIRST) {
-			skd_prep_zerosize_flush_cdb(scsi_req, skreq);
-			SKD_ASSERT(skreq->flush_cmd == 1);
+		/* Initialize the FIT msg header */
+		fmh = &skmsg->msg_buf->fmh;
+		memset(fmh, 0, sizeof(*fmh));
+		fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
+		skmsg->length = sizeof(*fmh);
+	} else {
+		fmh = &skmsg->msg_buf->fmh;
+	}
 
-		} else {
-			skd_prep_rw_cdb(scsi_req, data_dir, lba, count);
-		}
+	skreq->fitmsg_id = skmsg->id;
 
-		if (fua)
-			scsi_req->cdb[1] |= SKD_FUA_NV;
+	scsi_req = &skmsg->msg_buf->scsi[fmh->num_protocol_cmds_coalesced];
+	memset(scsi_req, 0, sizeof(*scsi_req));
 
-		if (!req->bio)
-			goto skip_sg;
+	scsi_req->hdr.tag = skreq->id;
+	scsi_req->hdr.sg_list_dma_address =
+		cpu_to_be64(skreq->sksg_dma_address);
 
-		if (!skd_preop_sg_list(skdev, skreq)) {
-			/*
-			 * Complete the native request with error.
-			 * Note that the request context is still at the
-			 * head of the free list, and that the SoFIT request
-			 * was encoded into the FIT msg buffer but the FIT
-			 * msg length has not been updated. In short, the
-			 * only resource that has been allocated but might
-			 * not be used is that the FIT msg could be empty.
-			 */
-			pr_debug("%s:%s:%d error Out\n",
-				 skdev->name, __func__, __LINE__);
-			skd_end_request(skdev, skreq, BLK_STS_RESOURCE);
-			continue;
-		}
+	if (req_op(req) == REQ_OP_FLUSH) {
+		skd_prep_zerosize_flush_cdb(scsi_req, skreq);
+		SKD_ASSERT(skreq->flush_cmd == 1);
+	} else {
+		skd_prep_rw_cdb(scsi_req, data_dir, lba, count);
+	}
 
-skip_sg:
-		scsi_req->hdr.sg_list_len_bytes =
-			cpu_to_be32(skreq->sg_byte_count);
+	if (req->cmd_flags & REQ_FUA)
+		scsi_req->cdb[1] |= SKD_FUA_NV;
 
-		/* Complete resource allocations. */
-		skdev->skreq_free_list = skreq->next;
-		skreq->state = SKD_REQ_STATE_BUSY;
-		skreq->id += SKD_ID_INCR;
+	scsi_req->hdr.sg_list_len_bytes = cpu_to_be32(skreq->sg_byte_count);
 
-		skmsg->length += sizeof(struct skd_scsi_request);
-		fmh->num_protocol_cmds_coalesced++;
+	/* Complete resource allocations. */
+	skreq->state = SKD_REQ_STATE_BUSY;
 
-		/*
-		 * Update the active request counts.
-		 * Capture the timeout timestamp.
-		 */
-		skreq->timeout_stamp = skdev->timeout_stamp;
-		timo_slot = skreq->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
-		skdev->timeout_slot[timo_slot]++;
-		skdev->in_flight++;
-		pr_debug("%s:%s:%d req=0x%x busy=%d\n",
-			 skdev->name, __func__, __LINE__,
-			 skreq->id, skdev->in_flight);
+	skmsg->length += sizeof(struct skd_scsi_request);
+	fmh->num_protocol_cmds_coalesced++;
 
-		/*
-		 * If the FIT msg buffer is full send it.
-		 */
-		if (skmsg->length >= SKD_N_FITMSG_BYTES ||
-		    fmh->num_protocol_cmds_coalesced >= skd_max_req_per_msg) {
-			skd_send_fitmsg(skdev, skmsg);
-			skmsg = NULL;
-			fmh = NULL;
-		}
-	}
+	dev_dbg(&skdev->pdev->dev, "req=0x%x busy=%d\n", skreq->id,
+		skd_in_flight(skdev));
 
 	/*
-	 * Is a FIT msg in progress? If it is empty put the buffer back
-	 * on the free list. If it is non-empty send what we got.
-	 * This minimizes latency when there are fewer requests than
-	 * what fits in a FIT msg.
+	 * If the FIT msg buffer is full send it.
 	 */
-	if (skmsg != NULL) {
-		/* Bigger than just a FIT msg header? */
-		if (skmsg->length > sizeof(struct fit_msg_hdr)) {
-			pr_debug("%s:%s:%d sending msg=%p, len %d\n",
-				 skdev->name, __func__, __LINE__,
-				 skmsg, skmsg->length);
+	if (skd_max_req_per_msg == 1) {
+		skd_send_fitmsg(skdev, skmsg);
+	} else {
+		if (mqd->last ||
+		    fmh->num_protocol_cmds_coalesced >= skd_max_req_per_msg) {
 			skd_send_fitmsg(skdev, skmsg);
-		} else {
-			/*
-			 * The FIT msg is empty. It means we got started
-			 * on the msg, but the requests were rejected.
-			 */
-			skmsg->state = SKD_MSG_STATE_IDLE;
-			skmsg->id += SKD_ID_INCR;
-			skmsg->next = skdev->skmsg_free_list;
-			skdev->skmsg_free_list = skmsg;
+			skdev->skmsg = NULL;
 		}
-		skmsg = NULL;
-		fmh = NULL;
+		spin_unlock_irqrestore(&skdev->lock, flags);
 	}
 
-	/*
-	 * If req is non-NULL it means there is something to do but
-	 * we are out of a resource.
-	 */
-	if (req)
-		blk_stop_queue(skdev->queue);
+	return BLK_STS_OK;
 }
 
-static void skd_end_request(struct skd_device *skdev,
-		struct skd_request_context *skreq, blk_status_t error)
+static enum blk_eh_timer_return skd_timed_out(struct request *req,
+					      bool reserved)
 {
-	if (unlikely(error)) {
-		struct request *req = skreq->req;
-		char *cmd = (rq_data_dir(req) == READ) ? "read" : "write";
-		u32 lba = (u32)blk_rq_pos(req);
-		u32 count = blk_rq_sectors(req);
-
-		pr_err("(%s): Error cmd=%s sect=%u count=%u id=0x%x\n",
-		       skd_name(skdev), cmd, lba, count, skreq->id);
-	} else
-		pr_debug("%s:%s:%d id=0x%x error=%d\n",
-			 skdev->name, __func__, __LINE__, skreq->id, error);
+	struct skd_device *skdev = req->q->queuedata;
+
+	dev_err(&skdev->pdev->dev, "request with tag %#x timed out\n",
+		blk_mq_unique_tag(req));
 
-	__blk_end_request_all(skreq->req, error);
+	return BLK_EH_RESET_TIMER;
+}
+
+static void skd_complete_rq(struct request *req)
+{
+	struct skd_request_context *skreq = blk_mq_rq_to_pdu(req);
+
+	blk_mq_end_request(req, skreq->status);
 }
 
 static bool skd_preop_sg_list(struct skd_device *skdev,
 			     struct skd_request_context *skreq)
 {
-	struct request *req = skreq->req;
-	int writing = skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD;
-	int pci_dir = writing ? PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE;
-	struct scatterlist *sg = &skreq->sg[0];
+	struct request *req = blk_mq_rq_from_pdu(skreq);
+	struct scatterlist *sgl = &skreq->sg[0], *sg;
 	int n_sg;
 	int i;
 
 	skreq->sg_byte_count = 0;
 
-	/* SKD_ASSERT(skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD ||
-		   skreq->sg_data_dir == SKD_DATA_DIR_CARD_TO_HOST); */
+	WARN_ON_ONCE(skreq->data_dir != DMA_TO_DEVICE &&
+		     skreq->data_dir != DMA_FROM_DEVICE);
 
-	n_sg = blk_rq_map_sg(skdev->queue, req, sg);
+	n_sg = blk_rq_map_sg(skdev->queue, req, sgl);
 	if (n_sg <= 0)
 		return false;
 
@@ -842,7 +633,7 @@ static bool skd_preop_sg_list(struct skd_device *skdev,
 	 * Map scatterlist to PCI bus addresses.
 	 * Note PCI might change the number of entries.
 	 */
-	n_sg = pci_map_sg(skdev->pdev, sg, n_sg, pci_dir);
+	n_sg = pci_map_sg(skdev->pdev, sgl, n_sg, skreq->data_dir);
 	if (n_sg <= 0)
 		return false;
 
@@ -850,10 +641,10 @@ static bool skd_preop_sg_list(struct skd_device *skdev,
 
 	skreq->n_sg = n_sg;
 
-	for (i = 0; i < n_sg; i++) {
+	for_each_sg(sgl, sg, n_sg, i) {
 		struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
-		u32 cnt = sg_dma_len(&sg[i]);
-		uint64_t dma_addr = sg_dma_address(&sg[i]);
+		u32 cnt = sg_dma_len(sg);
+		uint64_t dma_addr = sg_dma_address(sg);
 
 		sgd->control = FIT_SGD_CONTROL_NOT_LAST;
 		sgd->byte_count = cnt;
@@ -866,16 +657,16 @@ static bool skd_preop_sg_list(struct skd_device *skdev,
 	skreq->sksg_list[n_sg - 1].control = FIT_SGD_CONTROL_LAST;
 
 	if (unlikely(skdev->dbg_level > 1)) {
-		pr_debug("%s:%s:%d skreq=%x sksg_list=%p sksg_dma=%llx\n",
-			 skdev->name, __func__, __LINE__,
-			 skreq->id, skreq->sksg_list, skreq->sksg_dma_address);
+		dev_dbg(&skdev->pdev->dev,
+			"skreq=%x sksg_list=%p sksg_dma=%llx\n",
+			skreq->id, skreq->sksg_list, skreq->sksg_dma_address);
 		for (i = 0; i < n_sg; i++) {
 			struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
-			pr_debug("%s:%s:%d   sg[%d] count=%u ctrl=0x%x "
-				 "addr=0x%llx next=0x%llx\n",
-				 skdev->name, __func__, __LINE__,
-				 i, sgd->byte_count, sgd->control,
-				 sgd->host_side_addr, sgd->next_desc_ptr);
+
+			dev_dbg(&skdev->pdev->dev,
+				"  sg[%d] count=%u ctrl=0x%x addr=0x%llx next=0x%llx\n",
+				i, sgd->byte_count, sgd->control,
+				sgd->host_side_addr, sgd->next_desc_ptr);
 		}
 	}
 
@@ -885,9 +676,6 @@ static bool skd_preop_sg_list(struct skd_device *skdev,
 static void skd_postop_sg_list(struct skd_device *skdev,
 			       struct skd_request_context *skreq)
 {
-	int writing = skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD;
-	int pci_dir = writing ? PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE;
-
 	/*
 	 * restore the next ptr for next IO request so we
 	 * don't have to set it every time.
@@ -895,51 +683,7 @@ static void skd_postop_sg_list(struct skd_device *skdev,
 	skreq->sksg_list[skreq->n_sg - 1].next_desc_ptr =
 		skreq->sksg_dma_address +
 		((skreq->n_sg) * sizeof(struct fit_sg_descriptor));
-	pci_unmap_sg(skdev->pdev, &skreq->sg[0], skreq->n_sg, pci_dir);
-}
-
-static void skd_request_fn_not_online(struct request_queue *q)
-{
-	struct skd_device *skdev = q->queuedata;
-	int error;
-
-	SKD_ASSERT(skdev->state != SKD_DRVR_STATE_ONLINE);
-
-	skd_log_skdev(skdev, "req_not_online");
-	switch (skdev->state) {
-	case SKD_DRVR_STATE_PAUSING:
-	case SKD_DRVR_STATE_PAUSED:
-	case SKD_DRVR_STATE_STARTING:
-	case SKD_DRVR_STATE_RESTARTING:
-	case SKD_DRVR_STATE_WAIT_BOOT:
-	/* In case of starting, we haven't started the queue,
-	 * so we can't get here... but requests are
-	 * possibly hanging out waiting for us because we
-	 * reported the dev/skd0 already.  They'll wait
-	 * forever if connect doesn't complete.
-	 * What to do??? delay dev/skd0 ??
-	 */
-	case SKD_DRVR_STATE_BUSY:
-	case SKD_DRVR_STATE_BUSY_IMMINENT:
-	case SKD_DRVR_STATE_BUSY_ERASE:
-	case SKD_DRVR_STATE_DRAINING_TIMEOUT:
-		return;
-
-	case SKD_DRVR_STATE_BUSY_SANITIZE:
-	case SKD_DRVR_STATE_STOPPING:
-	case SKD_DRVR_STATE_SYNCING:
-	case SKD_DRVR_STATE_FAULT:
-	case SKD_DRVR_STATE_DISAPPEARED:
-	default:
-		error = -EIO;
-		break;
-	}
-
-	/* If we get here, terminate all pending block requeusts
-	 * with EIO and any scsi pass thru with appropriate sense
-	 */
-
-	skd_fail_all_pending(skdev);
+	pci_unmap_sg(skdev->pdev, &skreq->sg[0], skreq->n_sg, skreq->data_dir);
 }
 
 /*
@@ -950,12 +694,22 @@ static void skd_request_fn_not_online(struct request_queue *q)
 
 static void skd_timer_tick_not_online(struct skd_device *skdev);
 
+static void skd_start_queue(struct work_struct *work)
+{
+	struct skd_device *skdev = container_of(work, typeof(*skdev),
+						start_queue);
+
+	/*
+	 * Although it is safe to call blk_start_queue() from interrupt
+	 * context, blk_mq_start_hw_queues() must not be called from
+	 * interrupt context.
+	 */
+	blk_mq_start_hw_queues(skdev->queue);
+}
+
 static void skd_timer_tick(ulong arg)
 {
 	struct skd_device *skdev = (struct skd_device *)arg;
-
-	u32 timo_slot;
-	u32 overdue_timestamp;
 	unsigned long reqflags;
 	u32 state;
 
@@ -972,37 +726,9 @@ static void skd_timer_tick(ulong arg)
 	if (state != skdev->drive_state)
 		skd_isr_fwstate(skdev);
 
-	if (skdev->state != SKD_DRVR_STATE_ONLINE) {
+	if (skdev->state != SKD_DRVR_STATE_ONLINE)
 		skd_timer_tick_not_online(skdev);
-		goto timer_func_out;
-	}
-	skdev->timeout_stamp++;
-	timo_slot = skdev->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
-
-	/*
-	 * All requests that happened during the previous use of
-	 * this slot should be done by now. The previous use was
-	 * over 7 seconds ago.
-	 */
-	if (skdev->timeout_slot[timo_slot] == 0)
-		goto timer_func_out;
-
-	/* Something is overdue */
-	overdue_timestamp = skdev->timeout_stamp - SKD_N_TIMEOUT_SLOT;
-
-	pr_debug("%s:%s:%d found %d timeouts, draining busy=%d\n",
-		 skdev->name, __func__, __LINE__,
-		 skdev->timeout_slot[timo_slot], skdev->in_flight);
-	pr_err("(%s): Overdue IOs (%d), busy %d\n",
-	       skd_name(skdev), skdev->timeout_slot[timo_slot],
-	       skdev->in_flight);
 
-	skdev->timer_countdown = SKD_DRAINING_TIMO;
-	skdev->state = SKD_DRVR_STATE_DRAINING_TIMEOUT;
-	skdev->timo_slot = timo_slot;
-	blk_stop_queue(skdev->queue);
-
-timer_func_out:
 	mod_timer(&skdev->timer, (jiffies + HZ));
 
 	spin_unlock_irqrestore(&skdev->lock, reqflags);
@@ -1015,9 +741,9 @@ static void skd_timer_tick_not_online(struct skd_device *skdev)
 	case SKD_DRVR_STATE_LOAD:
 		break;
 	case SKD_DRVR_STATE_BUSY_SANITIZE:
-		pr_debug("%s:%s:%d drive busy sanitize[%x], driver[%x]\n",
-			 skdev->name, __func__, __LINE__,
-			 skdev->drive_state, skdev->state);
+		dev_dbg(&skdev->pdev->dev,
+			"drive busy sanitize[%x], driver[%x]\n",
+			skdev->drive_state, skdev->state);
 		/* If we've been in sanitize for 3 seconds, we figure we're not
 		 * going to get anymore completions, so recover requests now
 		 */
@@ -1025,22 +751,21 @@ static void skd_timer_tick_not_online(struct skd_device *skdev)
 			skdev->timer_countdown--;
 			return;
 		}
-		skd_recover_requests(skdev, 0);
+		skd_recover_requests(skdev);
 		break;
 
 	case SKD_DRVR_STATE_BUSY:
 	case SKD_DRVR_STATE_BUSY_IMMINENT:
 	case SKD_DRVR_STATE_BUSY_ERASE:
-		pr_debug("%s:%s:%d busy[%x], countdown=%d\n",
-			 skdev->name, __func__, __LINE__,
-			 skdev->state, skdev->timer_countdown);
+		dev_dbg(&skdev->pdev->dev, "busy[%x], countdown=%d\n",
+			skdev->state, skdev->timer_countdown);
 		if (skdev->timer_countdown > 0) {
 			skdev->timer_countdown--;
 			return;
 		}
-		pr_debug("%s:%s:%d busy[%x], timedout=%d, restarting device.",
-			 skdev->name, __func__, __LINE__,
-			 skdev->state, skdev->timer_countdown);
+		dev_dbg(&skdev->pdev->dev,
+			"busy[%x], timedout=%d, restarting device.",
+			skdev->state, skdev->timer_countdown);
 		skd_restart_device(skdev);
 		break;
 
@@ -1054,12 +779,12 @@ static void skd_timer_tick_not_online(struct skd_device *skdev)
 		 * revcover at some point. */
 		skdev->state = SKD_DRVR_STATE_FAULT;
 
-		pr_err("(%s): DriveFault Connect Timeout (%x)\n",
-		       skd_name(skdev), skdev->drive_state);
+		dev_err(&skdev->pdev->dev, "DriveFault Connect Timeout (%x)\n",
+			skdev->drive_state);
 
 		/*start the queue so we can respond with error to requests */
 		/* wakeup anyone waiting for startup complete */
-		blk_start_queue(skdev->queue);
+		schedule_work(&skdev->start_queue);
 		skdev->gendisk_on = -1;
 		wake_up_interruptible(&skdev->waitq);
 		break;
@@ -1072,29 +797,6 @@ static void skd_timer_tick_not_online(struct skd_device *skdev)
 	case SKD_DRVR_STATE_PAUSED:
 		break;
 
-	case SKD_DRVR_STATE_DRAINING_TIMEOUT:
-		pr_debug("%s:%s:%d "
-			 "draining busy [%d] tick[%d] qdb[%d] tmls[%d]\n",
-			 skdev->name, __func__, __LINE__,
-			 skdev->timo_slot,
-			 skdev->timer_countdown,
-			 skdev->in_flight,
-			 skdev->timeout_slot[skdev->timo_slot]);
-		/* if the slot has cleared we can let the I/O continue */
-		if (skdev->timeout_slot[skdev->timo_slot] == 0) {
-			pr_debug("%s:%s:%d Slot drained, starting queue.\n",
-				 skdev->name, __func__, __LINE__);
-			skdev->state = SKD_DRVR_STATE_ONLINE;
-			blk_start_queue(skdev->queue);
-			return;
-		}
-		if (skdev->timer_countdown > 0) {
-			skdev->timer_countdown--;
-			return;
-		}
-		skd_restart_device(skdev);
-		break;
-
 	case SKD_DRVR_STATE_RESTARTING:
 		if (skdev->timer_countdown > 0) {
 			skdev->timer_countdown--;
@@ -1103,8 +805,9 @@ static void skd_timer_tick_not_online(struct skd_device *skdev)
 		/* For now, we fault the drive. Could attempt resets to
 		 * revcover at some point. */
 		skdev->state = SKD_DRVR_STATE_FAULT;
-		pr_err("(%s): DriveFault Reconnect Timeout (%x)\n",
-		       skd_name(skdev), skdev->drive_state);
+		dev_err(&skdev->pdev->dev,
+			"DriveFault Reconnect Timeout (%x)\n",
+			skdev->drive_state);
 
 		/*
 		 * Recovering does two things:
@@ -1124,18 +827,18 @@ static void skd_timer_tick_not_online(struct skd_device *skdev)
 			/* It never came out of soft reset. Try to
 			 * recover the requests and then let them
 			 * fail. This is to mitigate hung processes. */
-			skd_recover_requests(skdev, 0);
+			skd_recover_requests(skdev);
 		else {
-			pr_err("(%s): Disable BusMaster (%x)\n",
-			       skd_name(skdev), skdev->drive_state);
+			dev_err(&skdev->pdev->dev, "Disable BusMaster (%x)\n",
+				skdev->drive_state);
 			pci_disable_device(skdev->pdev);
 			skd_disable_interrupts(skdev);
-			skd_recover_requests(skdev, 0);
+			skd_recover_requests(skdev);
 		}
 
 		/*start the queue so we can respond with error to requests */
 		/* wakeup anyone waiting for startup complete */
-		blk_start_queue(skdev->queue);
+		schedule_work(&skdev->start_queue);
 		skdev->gendisk_on = -1;
 		wake_up_interruptible(&skdev->waitq);
 		break;
@@ -1154,13 +857,11 @@ static int skd_start_timer(struct skd_device *skdev)
 {
 	int rc;
 
-	init_timer(&skdev->timer);
 	setup_timer(&skdev->timer, skd_timer_tick, (ulong)skdev);
 
 	rc = mod_timer(&skdev->timer, (jiffies + HZ));
 	if (rc)
-		pr_err("%s: failed to start timer %d\n",
-		       __func__, rc);
+		dev_err(&skdev->pdev->dev, "failed to start timer %d\n", rc);
 	return rc;
 }
 
@@ -1171,634 +872,6 @@ static void skd_kill_timer(struct skd_device *skdev)
 
 /*
  *****************************************************************************
- * IOCTL
- *****************************************************************************
- */
-static int skd_ioctl_sg_io(struct skd_device *skdev,
-			   fmode_t mode, void __user *argp);
-static int skd_sg_io_get_and_check_args(struct skd_device *skdev,
-					struct skd_sg_io *sksgio);
-static int skd_sg_io_obtain_skspcl(struct skd_device *skdev,
-				   struct skd_sg_io *sksgio);
-static int skd_sg_io_prep_buffering(struct skd_device *skdev,
-				    struct skd_sg_io *sksgio);
-static int skd_sg_io_copy_buffer(struct skd_device *skdev,
-				 struct skd_sg_io *sksgio, int dxfer_dir);
-static int skd_sg_io_send_fitmsg(struct skd_device *skdev,
-				 struct skd_sg_io *sksgio);
-static int skd_sg_io_await(struct skd_device *skdev, struct skd_sg_io *sksgio);
-static int skd_sg_io_release_skspcl(struct skd_device *skdev,
-				    struct skd_sg_io *sksgio);
-static int skd_sg_io_put_status(struct skd_device *skdev,
-				struct skd_sg_io *sksgio);
-
-static void skd_complete_special(struct skd_device *skdev,
-				 volatile struct fit_completion_entry_v1
-				 *skcomp,
-				 volatile struct fit_comp_error_info *skerr,
-				 struct skd_special_context *skspcl);
-
-static int skd_bdev_ioctl(struct block_device *bdev, fmode_t mode,
-			  uint cmd_in, ulong arg)
-{
-	static const int sg_version_num = 30527;
-	int rc = 0, timeout;
-	struct gendisk *disk = bdev->bd_disk;
-	struct skd_device *skdev = disk->private_data;
-	int __user *p = (int __user *)arg;
-
-	pr_debug("%s:%s:%d %s: CMD[%s] ioctl  mode 0x%x, cmd 0x%x arg %0lx\n",
-		 skdev->name, __func__, __LINE__,
-		 disk->disk_name, current->comm, mode, cmd_in, arg);
-
-	if (!capable(CAP_SYS_ADMIN))
-		return -EPERM;
-
-	switch (cmd_in) {
-	case SG_SET_TIMEOUT:
-		rc = get_user(timeout, p);
-		if (!rc)
-			disk->queue->sg_timeout = clock_t_to_jiffies(timeout);
-		break;
-	case SG_GET_TIMEOUT:
-		rc = jiffies_to_clock_t(disk->queue->sg_timeout);
-		break;
-	case SG_GET_VERSION_NUM:
-		rc = put_user(sg_version_num, p);
-		break;
-	case SG_IO:
-		rc = skd_ioctl_sg_io(skdev, mode, (void __user *)arg);
-		break;
-
-	default:
-		rc = -ENOTTY;
-		break;
-	}
-
-	pr_debug("%s:%s:%d %s:  completion rc %d\n",
-		 skdev->name, __func__, __LINE__, disk->disk_name, rc);
-	return rc;
-}
-
-static int skd_ioctl_sg_io(struct skd_device *skdev, fmode_t mode,
-			   void __user *argp)
-{
-	int rc;
-	struct skd_sg_io sksgio;
-
-	memset(&sksgio, 0, sizeof(sksgio));
-	sksgio.mode = mode;
-	sksgio.argp = argp;
-	sksgio.iov = &sksgio.no_iov_iov;
-
-	switch (skdev->state) {
-	case SKD_DRVR_STATE_ONLINE:
-	case SKD_DRVR_STATE_BUSY_IMMINENT:
-		break;
-
-	default:
-		pr_debug("%s:%s:%d drive not online\n",
-			 skdev->name, __func__, __LINE__);
-		rc = -ENXIO;
-		goto out;
-	}
-
-	rc = skd_sg_io_get_and_check_args(skdev, &sksgio);
-	if (rc)
-		goto out;
-
-	rc = skd_sg_io_obtain_skspcl(skdev, &sksgio);
-	if (rc)
-		goto out;
-
-	rc = skd_sg_io_prep_buffering(skdev, &sksgio);
-	if (rc)
-		goto out;
-
-	rc = skd_sg_io_copy_buffer(skdev, &sksgio, SG_DXFER_TO_DEV);
-	if (rc)
-		goto out;
-
-	rc = skd_sg_io_send_fitmsg(skdev, &sksgio);
-	if (rc)
-		goto out;
-
-	rc = skd_sg_io_await(skdev, &sksgio);
-	if (rc)
-		goto out;
-
-	rc = skd_sg_io_copy_buffer(skdev, &sksgio, SG_DXFER_FROM_DEV);
-	if (rc)
-		goto out;
-
-	rc = skd_sg_io_put_status(skdev, &sksgio);
-	if (rc)
-		goto out;
-
-	rc = 0;
-
-out:
-	skd_sg_io_release_skspcl(skdev, &sksgio);
-
-	if (sksgio.iov != NULL && sksgio.iov != &sksgio.no_iov_iov)
-		kfree(sksgio.iov);
-	return rc;
-}
-
-static int skd_sg_io_get_and_check_args(struct skd_device *skdev,
-					struct skd_sg_io *sksgio)
-{
-	struct sg_io_hdr *sgp = &sksgio->sg;
-	int i, acc;
-
-	if (!access_ok(VERIFY_WRITE, sksgio->argp, sizeof(sg_io_hdr_t))) {
-		pr_debug("%s:%s:%d access sg failed %p\n",
-			 skdev->name, __func__, __LINE__, sksgio->argp);
-		return -EFAULT;
-	}
-
-	if (__copy_from_user(sgp, sksgio->argp, sizeof(sg_io_hdr_t))) {
-		pr_debug("%s:%s:%d copy_from_user sg failed %p\n",
-			 skdev->name, __func__, __LINE__, sksgio->argp);
-		return -EFAULT;
-	}
-
-	if (sgp->interface_id != SG_INTERFACE_ID_ORIG) {
-		pr_debug("%s:%s:%d interface_id invalid 0x%x\n",
-			 skdev->name, __func__, __LINE__, sgp->interface_id);
-		return -EINVAL;
-	}
-
-	if (sgp->cmd_len > sizeof(sksgio->cdb)) {
-		pr_debug("%s:%s:%d cmd_len invalid %d\n",
-			 skdev->name, __func__, __LINE__, sgp->cmd_len);
-		return -EINVAL;
-	}
-
-	if (sgp->iovec_count > 256) {
-		pr_debug("%s:%s:%d iovec_count invalid %d\n",
-			 skdev->name, __func__, __LINE__, sgp->iovec_count);
-		return -EINVAL;
-	}
-
-	if (sgp->dxfer_len > (PAGE_SIZE * SKD_N_SG_PER_SPECIAL)) {
-		pr_debug("%s:%s:%d dxfer_len invalid %d\n",
-			 skdev->name, __func__, __LINE__, sgp->dxfer_len);
-		return -EINVAL;
-	}
-
-	switch (sgp->dxfer_direction) {
-	case SG_DXFER_NONE:
-		acc = -1;
-		break;
-
-	case SG_DXFER_TO_DEV:
-		acc = VERIFY_READ;
-		break;
-
-	case SG_DXFER_FROM_DEV:
-	case SG_DXFER_TO_FROM_DEV:
-		acc = VERIFY_WRITE;
-		break;
-
-	default:
-		pr_debug("%s:%s:%d dxfer_dir invalid %d\n",
-			 skdev->name, __func__, __LINE__, sgp->dxfer_direction);
-		return -EINVAL;
-	}
-
-	if (copy_from_user(sksgio->cdb, sgp->cmdp, sgp->cmd_len)) {
-		pr_debug("%s:%s:%d copy_from_user cmdp failed %p\n",
-			 skdev->name, __func__, __LINE__, sgp->cmdp);
-		return -EFAULT;
-	}
-
-	if (sgp->mx_sb_len != 0) {
-		if (!access_ok(VERIFY_WRITE, sgp->sbp, sgp->mx_sb_len)) {
-			pr_debug("%s:%s:%d access sbp failed %p\n",
-				 skdev->name, __func__, __LINE__, sgp->sbp);
-			return -EFAULT;
-		}
-	}
-
-	if (sgp->iovec_count == 0) {
-		sksgio->iov[0].iov_base = sgp->dxferp;
-		sksgio->iov[0].iov_len = sgp->dxfer_len;
-		sksgio->iovcnt = 1;
-		sksgio->dxfer_len = sgp->dxfer_len;
-	} else {
-		struct sg_iovec *iov;
-		uint nbytes = sizeof(*iov) * sgp->iovec_count;
-		size_t iov_data_len;
-
-		iov = kmalloc(nbytes, GFP_KERNEL);
-		if (iov == NULL) {
-			pr_debug("%s:%s:%d alloc iovec failed %d\n",
-				 skdev->name, __func__, __LINE__,
-				 sgp->iovec_count);
-			return -ENOMEM;
-		}
-		sksgio->iov = iov;
-		sksgio->iovcnt = sgp->iovec_count;
-
-		if (copy_from_user(iov, sgp->dxferp, nbytes)) {
-			pr_debug("%s:%s:%d copy_from_user iovec failed %p\n",
-				 skdev->name, __func__, __LINE__, sgp->dxferp);
-			return -EFAULT;
-		}
-
-		/*
-		 * Sum up the vecs, making sure they don't overflow
-		 */
-		iov_data_len = 0;
-		for (i = 0; i < sgp->iovec_count; i++) {
-			if (iov_data_len + iov[i].iov_len < iov_data_len)
-				return -EINVAL;
-			iov_data_len += iov[i].iov_len;
-		}
-
-		/* SG_IO howto says that the shorter of the two wins */
-		if (sgp->dxfer_len < iov_data_len) {
-			sksgio->iovcnt = iov_shorten((struct iovec *)iov,
-						     sgp->iovec_count,
-						     sgp->dxfer_len);
-			sksgio->dxfer_len = sgp->dxfer_len;
-		} else
-			sksgio->dxfer_len = iov_data_len;
-	}
-
-	if (sgp->dxfer_direction != SG_DXFER_NONE) {
-		struct sg_iovec *iov = sksgio->iov;
-		for (i = 0; i < sksgio->iovcnt; i++, iov++) {
-			if (!access_ok(acc, iov->iov_base, iov->iov_len)) {
-				pr_debug("%s:%s:%d access data failed %p/%d\n",
-					 skdev->name, __func__, __LINE__,
-					 iov->iov_base, (int)iov->iov_len);
-				return -EFAULT;
-			}
-		}
-	}
-
-	return 0;
-}
-
-static int skd_sg_io_obtain_skspcl(struct skd_device *skdev,
-				   struct skd_sg_io *sksgio)
-{
-	struct skd_special_context *skspcl = NULL;
-	int rc;
-
-	for (;;) {
-		ulong flags;
-
-		spin_lock_irqsave(&skdev->lock, flags);
-		skspcl = skdev->skspcl_free_list;
-		if (skspcl != NULL) {
-			skdev->skspcl_free_list =
-				(struct skd_special_context *)skspcl->req.next;
-			skspcl->req.id += SKD_ID_INCR;
-			skspcl->req.state = SKD_REQ_STATE_SETUP;
-			skspcl->orphaned = 0;
-			skspcl->req.n_sg = 0;
-		}
-		spin_unlock_irqrestore(&skdev->lock, flags);
-
-		if (skspcl != NULL) {
-			rc = 0;
-			break;
-		}
-
-		pr_debug("%s:%s:%d blocking\n",
-			 skdev->name, __func__, __LINE__);
-
-		rc = wait_event_interruptible_timeout(
-				skdev->waitq,
-				(skdev->skspcl_free_list != NULL),
-				msecs_to_jiffies(sksgio->sg.timeout));
-
-		pr_debug("%s:%s:%d unblocking, rc=%d\n",
-			 skdev->name, __func__, __LINE__, rc);
-
-		if (rc <= 0) {
-			if (rc == 0)
-				rc = -ETIMEDOUT;
-			else
-				rc = -EINTR;
-			break;
-		}
-		/*
-		 * If we get here rc > 0 meaning the timeout to
-		 * wait_event_interruptible_timeout() had time left, hence the
-		 * sought event -- non-empty free list -- happened.
-		 * Retry the allocation.
-		 */
-	}
-	sksgio->skspcl = skspcl;
-
-	return rc;
-}
-
-static int skd_skreq_prep_buffering(struct skd_device *skdev,
-				    struct skd_request_context *skreq,
-				    u32 dxfer_len)
-{
-	u32 resid = dxfer_len;
-
-	/*
-	 * The DMA engine must have aligned addresses and byte counts.
-	 */
-	resid += (-resid) & 3;
-	skreq->sg_byte_count = resid;
-
-	skreq->n_sg = 0;
-
-	while (resid > 0) {
-		u32 nbytes = PAGE_SIZE;
-		u32 ix = skreq->n_sg;
-		struct scatterlist *sg = &skreq->sg[ix];
-		struct fit_sg_descriptor *sksg = &skreq->sksg_list[ix];
-		struct page *page;
-
-		if (nbytes > resid)
-			nbytes = resid;
-
-		page = alloc_page(GFP_KERNEL);
-		if (page == NULL)
-			return -ENOMEM;
-
-		sg_set_page(sg, page, nbytes, 0);
-
-		/* TODO: This should be going through a pci_???()
-		 * routine to do proper mapping. */
-		sksg->control = FIT_SGD_CONTROL_NOT_LAST;
-		sksg->byte_count = nbytes;
-
-		sksg->host_side_addr = sg_phys(sg);
-
-		sksg->dev_side_addr = 0;
-		sksg->next_desc_ptr = skreq->sksg_dma_address +
-				      (ix + 1) * sizeof(*sksg);
-
-		skreq->n_sg++;
-		resid -= nbytes;
-	}
-
-	if (skreq->n_sg > 0) {
-		u32 ix = skreq->n_sg - 1;
-		struct fit_sg_descriptor *sksg = &skreq->sksg_list[ix];
-
-		sksg->control = FIT_SGD_CONTROL_LAST;
-		sksg->next_desc_ptr = 0;
-	}
-
-	if (unlikely(skdev->dbg_level > 1)) {
-		u32 i;
-
-		pr_debug("%s:%s:%d skreq=%x sksg_list=%p sksg_dma=%llx\n",
-			 skdev->name, __func__, __LINE__,
-			 skreq->id, skreq->sksg_list, skreq->sksg_dma_address);
-		for (i = 0; i < skreq->n_sg; i++) {
-			struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
-
-			pr_debug("%s:%s:%d   sg[%d] count=%u ctrl=0x%x "
-				 "addr=0x%llx next=0x%llx\n",
-				 skdev->name, __func__, __LINE__,
-				 i, sgd->byte_count, sgd->control,
-				 sgd->host_side_addr, sgd->next_desc_ptr);
-		}
-	}
-
-	return 0;
-}
-
-static int skd_sg_io_prep_buffering(struct skd_device *skdev,
-				    struct skd_sg_io *sksgio)
-{
-	struct skd_special_context *skspcl = sksgio->skspcl;
-	struct skd_request_context *skreq = &skspcl->req;
-	u32 dxfer_len = sksgio->dxfer_len;
-	int rc;
-
-	rc = skd_skreq_prep_buffering(skdev, skreq, dxfer_len);
-	/*
-	 * Eventually, errors or not, skd_release_special() is called
-	 * to recover allocations including partial allocations.
-	 */
-	return rc;
-}
-
-static int skd_sg_io_copy_buffer(struct skd_device *skdev,
-				 struct skd_sg_io *sksgio, int dxfer_dir)
-{
-	struct skd_special_context *skspcl = sksgio->skspcl;
-	u32 iov_ix = 0;
-	struct sg_iovec curiov;
-	u32 sksg_ix = 0;
-	u8 *bufp = NULL;
-	u32 buf_len = 0;
-	u32 resid = sksgio->dxfer_len;
-	int rc;
-
-	curiov.iov_len = 0;
-	curiov.iov_base = NULL;
-
-	if (dxfer_dir != sksgio->sg.dxfer_direction) {
-		if (dxfer_dir != SG_DXFER_TO_DEV ||
-		    sksgio->sg.dxfer_direction != SG_DXFER_TO_FROM_DEV)
-			return 0;
-	}
-
-	while (resid > 0) {
-		u32 nbytes = PAGE_SIZE;
-
-		if (curiov.iov_len == 0) {
-			curiov = sksgio->iov[iov_ix++];
-			continue;
-		}
-
-		if (buf_len == 0) {
-			struct page *page;
-			page = sg_page(&skspcl->req.sg[sksg_ix++]);
-			bufp = page_address(page);
-			buf_len = PAGE_SIZE;
-		}
-
-		nbytes = min_t(u32, nbytes, resid);
-		nbytes = min_t(u32, nbytes, curiov.iov_len);
-		nbytes = min_t(u32, nbytes, buf_len);
-
-		if (dxfer_dir == SG_DXFER_TO_DEV)
-			rc = __copy_from_user(bufp, curiov.iov_base, nbytes);
-		else
-			rc = __copy_to_user(curiov.iov_base, bufp, nbytes);
-
-		if (rc)
-			return -EFAULT;
-
-		resid -= nbytes;
-		curiov.iov_len -= nbytes;
-		curiov.iov_base += nbytes;
-		buf_len -= nbytes;
-	}
-
-	return 0;
-}
-
-static int skd_sg_io_send_fitmsg(struct skd_device *skdev,
-				 struct skd_sg_io *sksgio)
-{
-	struct skd_special_context *skspcl = sksgio->skspcl;
-	struct fit_msg_hdr *fmh = (struct fit_msg_hdr *)skspcl->msg_buf;
-	struct skd_scsi_request *scsi_req = (struct skd_scsi_request *)&fmh[1];
-
-	memset(skspcl->msg_buf, 0, SKD_N_SPECIAL_FITMSG_BYTES);
-
-	/* Initialize the FIT msg header */
-	fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
-	fmh->num_protocol_cmds_coalesced = 1;
-
-	/* Initialize the SCSI request */
-	if (sksgio->sg.dxfer_direction != SG_DXFER_NONE)
-		scsi_req->hdr.sg_list_dma_address =
-			cpu_to_be64(skspcl->req.sksg_dma_address);
-	scsi_req->hdr.tag = skspcl->req.id;
-	scsi_req->hdr.sg_list_len_bytes =
-		cpu_to_be32(skspcl->req.sg_byte_count);
-	memcpy(scsi_req->cdb, sksgio->cdb, sizeof(scsi_req->cdb));
-
-	skspcl->req.state = SKD_REQ_STATE_BUSY;
-	skd_send_special_fitmsg(skdev, skspcl);
-
-	return 0;
-}
-
-static int skd_sg_io_await(struct skd_device *skdev, struct skd_sg_io *sksgio)
-{
-	unsigned long flags;
-	int rc;
-
-	rc = wait_event_interruptible_timeout(skdev->waitq,
-					      (sksgio->skspcl->req.state !=
-					       SKD_REQ_STATE_BUSY),
-					      msecs_to_jiffies(sksgio->sg.
-							       timeout));
-
-	spin_lock_irqsave(&skdev->lock, flags);
-
-	if (sksgio->skspcl->req.state == SKD_REQ_STATE_ABORTED) {
-		pr_debug("%s:%s:%d skspcl %p aborted\n",
-			 skdev->name, __func__, __LINE__, sksgio->skspcl);
-
-		/* Build check cond, sense and let command finish. */
-		/* For a timeout, we must fabricate completion and sense
-		 * data to complete the command */
-		sksgio->skspcl->req.completion.status =
-			SAM_STAT_CHECK_CONDITION;
-
-		memset(&sksgio->skspcl->req.err_info, 0,
-		       sizeof(sksgio->skspcl->req.err_info));
-		sksgio->skspcl->req.err_info.type = 0x70;
-		sksgio->skspcl->req.err_info.key = ABORTED_COMMAND;
-		sksgio->skspcl->req.err_info.code = 0x44;
-		sksgio->skspcl->req.err_info.qual = 0;
-		rc = 0;
-	} else if (sksgio->skspcl->req.state != SKD_REQ_STATE_BUSY)
-		/* No longer on the adapter. We finish. */
-		rc = 0;
-	else {
-		/* Something's gone wrong. Still busy. Timeout or
-		 * user interrupted (control-C). Mark as an orphan
-		 * so it will be disposed when completed. */
-		sksgio->skspcl->orphaned = 1;
-		sksgio->skspcl = NULL;
-		if (rc == 0) {
-			pr_debug("%s:%s:%d timed out %p (%u ms)\n",
-				 skdev->name, __func__, __LINE__,
-				 sksgio, sksgio->sg.timeout);
-			rc = -ETIMEDOUT;
-		} else {
-			pr_debug("%s:%s:%d cntlc %p\n",
-				 skdev->name, __func__, __LINE__, sksgio);
-			rc = -EINTR;
-		}
-	}
-
-	spin_unlock_irqrestore(&skdev->lock, flags);
-
-	return rc;
-}
-
-static int skd_sg_io_put_status(struct skd_device *skdev,
-				struct skd_sg_io *sksgio)
-{
-	struct sg_io_hdr *sgp = &sksgio->sg;
-	struct skd_special_context *skspcl = sksgio->skspcl;
-	int resid = 0;
-
-	u32 nb = be32_to_cpu(skspcl->req.completion.num_returned_bytes);
-
-	sgp->status = skspcl->req.completion.status;
-	resid = sksgio->dxfer_len - nb;
-
-	sgp->masked_status = sgp->status & STATUS_MASK;
-	sgp->msg_status = 0;
-	sgp->host_status = 0;
-	sgp->driver_status = 0;
-	sgp->resid = resid;
-	if (sgp->masked_status || sgp->host_status || sgp->driver_status)
-		sgp->info |= SG_INFO_CHECK;
-
-	pr_debug("%s:%s:%d status %x masked %x resid 0x%x\n",
-		 skdev->name, __func__, __LINE__,
-		 sgp->status, sgp->masked_status, sgp->resid);
-
-	if (sgp->masked_status == SAM_STAT_CHECK_CONDITION) {
-		if (sgp->mx_sb_len > 0) {
-			struct fit_comp_error_info *ei = &skspcl->req.err_info;
-			u32 nbytes = sizeof(*ei);
-
-			nbytes = min_t(u32, nbytes, sgp->mx_sb_len);
-
-			sgp->sb_len_wr = nbytes;
-
-			if (__copy_to_user(sgp->sbp, ei, nbytes)) {
-				pr_debug("%s:%s:%d copy_to_user sense failed %p\n",
-					 skdev->name, __func__, __LINE__,
-					 sgp->sbp);
-				return -EFAULT;
-			}
-		}
-	}
-
-	if (__copy_to_user(sksgio->argp, sgp, sizeof(sg_io_hdr_t))) {
-		pr_debug("%s:%s:%d copy_to_user sg failed %p\n",
-			 skdev->name, __func__, __LINE__, sksgio->argp);
-		return -EFAULT;
-	}
-
-	return 0;
-}
-
-static int skd_sg_io_release_skspcl(struct skd_device *skdev,
-				    struct skd_sg_io *sksgio)
-{
-	struct skd_special_context *skspcl = sksgio->skspcl;
-
-	if (skspcl != NULL) {
-		ulong flags;
-
-		sksgio->skspcl = NULL;
-
-		spin_lock_irqsave(&skdev->lock, flags);
-		skd_release_special(skdev, skspcl);
-		spin_unlock_irqrestore(&skdev->lock, flags);
-	}
-
-	return 0;
-}
-
-/*
- *****************************************************************************
  * INTERNAL REQUESTS -- generated by driver itself
  *****************************************************************************
  */
@@ -1811,14 +884,15 @@ static int skd_format_internal_skspcl(struct skd_device *skdev)
 	uint64_t dma_address;
 	struct skd_scsi_request *scsi;
 
-	fmh = (struct fit_msg_hdr *)&skspcl->msg_buf[0];
+	fmh = &skspcl->msg_buf->fmh;
 	fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
 	fmh->num_protocol_cmds_coalesced = 1;
 
-	scsi = (struct skd_scsi_request *)&skspcl->msg_buf[64];
+	scsi = &skspcl->msg_buf->scsi[0];
 	memset(scsi, 0, sizeof(*scsi));
 	dma_address = skspcl->req.sksg_dma_address;
 	scsi->hdr.sg_list_dma_address = cpu_to_be64(dma_address);
+	skspcl->req.n_sg = 1;
 	sgd->control = FIT_SGD_CONTROL_LAST;
 	sgd->byte_count = 0;
 	sgd->host_side_addr = skspcl->db_dma_address;
@@ -1846,11 +920,9 @@ static void skd_send_internal_skspcl(struct skd_device *skdev,
 		 */
 		return;
 
-	SKD_ASSERT((skspcl->req.id & SKD_ID_INCR) == 0);
 	skspcl->req.state = SKD_REQ_STATE_BUSY;
-	skspcl->req.id += SKD_ID_INCR;
 
-	scsi = (struct skd_scsi_request *)&skspcl->msg_buf[64];
+	scsi = &skspcl->msg_buf->scsi[0];
 	scsi->hdr.tag = skspcl->req.id;
 
 	memset(scsi->cdb, 0, sizeof(scsi->cdb));
@@ -1940,32 +1012,35 @@ static void skd_log_check_status(struct skd_device *skdev, u8 status, u8 key,
 	/* If the check condition is of special interest, log a message */
 	if ((status == SAM_STAT_CHECK_CONDITION) && (key == 0x02)
 	    && (code == 0x04) && (qual == 0x06)) {
-		pr_err("(%s): *** LOST_WRITE_DATA ERROR *** key/asc/"
-		       "ascq/fruc %02x/%02x/%02x/%02x\n",
-		       skd_name(skdev), key, code, qual, fruc);
+		dev_err(&skdev->pdev->dev,
+			"*** LOST_WRITE_DATA ERROR *** key/asc/ascq/fruc %02x/%02x/%02x/%02x\n",
+			key, code, qual, fruc);
 	}
 }
 
 static void skd_complete_internal(struct skd_device *skdev,
-				  volatile struct fit_completion_entry_v1
-				  *skcomp,
-				  volatile struct fit_comp_error_info *skerr,
+				  struct fit_completion_entry_v1 *skcomp,
+				  struct fit_comp_error_info *skerr,
 				  struct skd_special_context *skspcl)
 {
 	u8 *buf = skspcl->data_buf;
 	u8 status;
 	int i;
-	struct skd_scsi_request *scsi =
-		(struct skd_scsi_request *)&skspcl->msg_buf[64];
+	struct skd_scsi_request *scsi = &skspcl->msg_buf->scsi[0];
+
+	lockdep_assert_held(&skdev->lock);
 
 	SKD_ASSERT(skspcl == &skdev->internal_skspcl);
 
-	pr_debug("%s:%s:%d complete internal %x\n",
-		 skdev->name, __func__, __LINE__, scsi->cdb[0]);
+	dev_dbg(&skdev->pdev->dev, "complete internal %x\n", scsi->cdb[0]);
+
+	dma_sync_single_for_cpu(&skdev->pdev->dev,
+				skspcl->db_dma_address,
+				skspcl->req.sksg_list[0].byte_count,
+				DMA_BIDIRECTIONAL);
 
 	skspcl->req.completion = *skcomp;
 	skspcl->req.state = SKD_REQ_STATE_IDLE;
-	skspcl->req.id += SKD_ID_INCR;
 
 	status = skspcl->req.completion.status;
 
@@ -1981,14 +1056,15 @@ static void skd_complete_internal(struct skd_device *skdev,
 			skd_send_internal_skspcl(skdev, skspcl, WRITE_BUFFER);
 		else {
 			if (skdev->state == SKD_DRVR_STATE_STOPPING) {
-				pr_debug("%s:%s:%d TUR failed, don't send anymore state 0x%x\n",
-					 skdev->name, __func__, __LINE__,
-					 skdev->state);
+				dev_dbg(&skdev->pdev->dev,
+					"TUR failed, don't send anymore state 0x%x\n",
+					skdev->state);
 				return;
 			}
-			pr_debug("%s:%s:%d **** TUR failed, retry skerr\n",
-				 skdev->name, __func__, __LINE__);
-			skd_send_internal_skspcl(skdev, skspcl, 0x00);
+			dev_dbg(&skdev->pdev->dev,
+				"**** TUR failed, retry skerr\n");
+			skd_send_internal_skspcl(skdev, skspcl,
+						 TEST_UNIT_READY);
 		}
 		break;
 
@@ -1997,14 +1073,15 @@ static void skd_complete_internal(struct skd_device *skdev,
 			skd_send_internal_skspcl(skdev, skspcl, READ_BUFFER);
 		else {
 			if (skdev->state == SKD_DRVR_STATE_STOPPING) {
-				pr_debug("%s:%s:%d write buffer failed, don't send anymore state 0x%x\n",
-					 skdev->name, __func__, __LINE__,
-					 skdev->state);
+				dev_dbg(&skdev->pdev->dev,
+					"write buffer failed, don't send anymore state 0x%x\n",
+					skdev->state);
 				return;
 			}
-			pr_debug("%s:%s:%d **** write buffer failed, retry skerr\n",
-				 skdev->name, __func__, __LINE__);
-			skd_send_internal_skspcl(skdev, skspcl, 0x00);
+			dev_dbg(&skdev->pdev->dev,
+				"**** write buffer failed, retry skerr\n");
+			skd_send_internal_skspcl(skdev, skspcl,
+						 TEST_UNIT_READY);
 		}
 		break;
 
@@ -2014,33 +1091,31 @@ static void skd_complete_internal(struct skd_device *skdev,
 				skd_send_internal_skspcl(skdev, skspcl,
 							 READ_CAPACITY);
 			else {
-				pr_err(
-				       "(%s):*** W/R Buffer mismatch %d ***\n",
-				       skd_name(skdev), skdev->connect_retries);
+				dev_err(&skdev->pdev->dev,
+					"*** W/R Buffer mismatch %d ***\n",
+					skdev->connect_retries);
 				if (skdev->connect_retries <
 				    SKD_MAX_CONNECT_RETRIES) {
 					skdev->connect_retries++;
 					skd_soft_reset(skdev);
 				} else {
-					pr_err(
-					       "(%s): W/R Buffer Connect Error\n",
-					       skd_name(skdev));
+					dev_err(&skdev->pdev->dev,
+						"W/R Buffer Connect Error\n");
 					return;
 				}
 			}
 
 		} else {
 			if (skdev->state == SKD_DRVR_STATE_STOPPING) {
-				pr_debug("%s:%s:%d "
-					 "read buffer failed, don't send anymore state 0x%x\n",
-					 skdev->name, __func__, __LINE__,
-					 skdev->state);
+				dev_dbg(&skdev->pdev->dev,
+					"read buffer failed, don't send anymore state 0x%x\n",
+					skdev->state);
 				return;
 			}
-			pr_debug("%s:%s:%d "
-				 "**** read buffer failed, retry skerr\n",
-				 skdev->name, __func__, __LINE__);
-			skd_send_internal_skspcl(skdev, skspcl, 0x00);
+			dev_dbg(&skdev->pdev->dev,
+				"**** read buffer failed, retry skerr\n");
+			skd_send_internal_skspcl(skdev, skspcl,
+						 TEST_UNIT_READY);
 		}
 		break;
 
@@ -2054,10 +1129,9 @@ static void skd_complete_internal(struct skd_device *skdev,
 				(buf[4] << 24) | (buf[5] << 16) |
 				(buf[6] << 8) | buf[7];
 
-			pr_debug("%s:%s:%d last lba %d, bs %d\n",
-				 skdev->name, __func__, __LINE__,
-				 skdev->read_cap_last_lba,
-				 skdev->read_cap_blocksize);
+			dev_dbg(&skdev->pdev->dev, "last lba %d, bs %d\n",
+				skdev->read_cap_last_lba,
+				skdev->read_cap_blocksize);
 
 			set_capacity(skdev->disk, skdev->read_cap_last_lba + 1);
 
@@ -2068,13 +1142,10 @@ static void skd_complete_internal(struct skd_device *skdev,
 			   (skerr->key == MEDIUM_ERROR)) {
 			skdev->read_cap_last_lba = ~0;
 			set_capacity(skdev->disk, skdev->read_cap_last_lba + 1);
-			pr_debug("%s:%s:%d "
-				 "**** MEDIUM ERROR caused READCAP to fail, ignore failure and continue to inquiry\n",
-				 skdev->name, __func__, __LINE__);
+			dev_dbg(&skdev->pdev->dev, "**** MEDIUM ERROR caused READCAP to fail, ignore failure and continue to inquiry\n");
 			skd_send_internal_skspcl(skdev, skspcl, INQUIRY);
 		} else {
-			pr_debug("%s:%s:%d **** READCAP failed, retry TUR\n",
-				 skdev->name, __func__, __LINE__);
+			dev_dbg(&skdev->pdev->dev, "**** READCAP failed, retry TUR\n");
 			skd_send_internal_skspcl(skdev, skspcl,
 						 TEST_UNIT_READY);
 		}
@@ -2091,8 +1162,7 @@ static void skd_complete_internal(struct skd_device *skdev,
 		}
 
 		if (skd_unquiesce_dev(skdev) < 0)
-			pr_debug("%s:%s:%d **** failed, to ONLINE device\n",
-				 skdev->name, __func__, __LINE__);
+			dev_dbg(&skdev->pdev->dev, "**** failed, to ONLINE device\n");
 		 /* connection is complete */
 		skdev->connect_retries = 0;
 		break;
@@ -2120,27 +1190,20 @@ static void skd_send_fitmsg(struct skd_device *skdev,
 			    struct skd_fitmsg_context *skmsg)
 {
 	u64 qcmd;
-	struct fit_msg_hdr *fmh;
 
-	pr_debug("%s:%s:%d dma address 0x%llx, busy=%d\n",
-		 skdev->name, __func__, __LINE__,
-		 skmsg->mb_dma_address, skdev->in_flight);
-	pr_debug("%s:%s:%d msg_buf 0x%p, offset %x\n",
-		 skdev->name, __func__, __LINE__,
-		 skmsg->msg_buf, skmsg->offset);
+	dev_dbg(&skdev->pdev->dev, "dma address 0x%llx, busy=%d\n",
+		skmsg->mb_dma_address, skd_in_flight(skdev));
+	dev_dbg(&skdev->pdev->dev, "msg_buf %p\n", skmsg->msg_buf);
 
 	qcmd = skmsg->mb_dma_address;
 	qcmd |= FIT_QCMD_QID_NORMAL;
 
-	fmh = (struct fit_msg_hdr *)skmsg->msg_buf;
-	skmsg->outstanding = fmh->num_protocol_cmds_coalesced;
-
 	if (unlikely(skdev->dbg_level > 1)) {
 		u8 *bp = (u8 *)skmsg->msg_buf;
 		int i;
 		for (i = 0; i < skmsg->length; i += 8) {
-			pr_debug("%s:%s:%d msg[%2d] %8ph\n",
-				 skdev->name, __func__, __LINE__, i, &bp[i]);
+			dev_dbg(&skdev->pdev->dev, "msg[%2d] %8ph\n", i,
+				&bp[i]);
 			if (i == 0)
 				i = 64 - 8;
 		}
@@ -2160,6 +1223,12 @@ static void skd_send_fitmsg(struct skd_device *skdev,
 		 */
 		qcmd |= FIT_QCMD_MSGSIZE_64;
 
+	dma_sync_single_for_device(&skdev->pdev->dev, skmsg->mb_dma_address,
+				   skmsg->length, DMA_TO_DEVICE);
+
+	/* Make sure skd_msg_buf is written before the doorbell is triggered. */
+	smp_wmb();
+
 	SKD_WRITEQ(skdev, qcmd, FIT_Q_COMMAND);
 }
 
@@ -2168,30 +1237,31 @@ static void skd_send_special_fitmsg(struct skd_device *skdev,
 {
 	u64 qcmd;
 
+	WARN_ON_ONCE(skspcl->req.n_sg != 1);
+
 	if (unlikely(skdev->dbg_level > 1)) {
 		u8 *bp = (u8 *)skspcl->msg_buf;
 		int i;
 
 		for (i = 0; i < SKD_N_SPECIAL_FITMSG_BYTES; i += 8) {
-			pr_debug("%s:%s:%d  spcl[%2d] %8ph\n",
-				 skdev->name, __func__, __LINE__, i, &bp[i]);
+			dev_dbg(&skdev->pdev->dev, " spcl[%2d] %8ph\n", i,
+				&bp[i]);
 			if (i == 0)
 				i = 64 - 8;
 		}
 
-		pr_debug("%s:%s:%d skspcl=%p id=%04x sksg_list=%p sksg_dma=%llx\n",
-			 skdev->name, __func__, __LINE__,
-			 skspcl, skspcl->req.id, skspcl->req.sksg_list,
-			 skspcl->req.sksg_dma_address);
+		dev_dbg(&skdev->pdev->dev,
+			"skspcl=%p id=%04x sksg_list=%p sksg_dma=%llx\n",
+			skspcl, skspcl->req.id, skspcl->req.sksg_list,
+			skspcl->req.sksg_dma_address);
 		for (i = 0; i < skspcl->req.n_sg; i++) {
 			struct fit_sg_descriptor *sgd =
 				&skspcl->req.sksg_list[i];
 
-			pr_debug("%s:%s:%d   sg[%d] count=%u ctrl=0x%x "
-				 "addr=0x%llx next=0x%llx\n",
-				 skdev->name, __func__, __LINE__,
-				 i, sgd->byte_count, sgd->control,
-				 sgd->host_side_addr, sgd->next_desc_ptr);
+			dev_dbg(&skdev->pdev->dev,
+				"  sg[%d] count=%u ctrl=0x%x addr=0x%llx next=0x%llx\n",
+				i, sgd->byte_count, sgd->control,
+				sgd->host_side_addr, sgd->next_desc_ptr);
 		}
 	}
 
@@ -2202,6 +1272,20 @@ static void skd_send_special_fitmsg(struct skd_device *skdev,
 	qcmd = skspcl->mb_dma_address;
 	qcmd |= FIT_QCMD_QID_NORMAL + FIT_QCMD_MSGSIZE_128;
 
+	dma_sync_single_for_device(&skdev->pdev->dev, skspcl->mb_dma_address,
+				   SKD_N_SPECIAL_FITMSG_BYTES, DMA_TO_DEVICE);
+	dma_sync_single_for_device(&skdev->pdev->dev,
+				   skspcl->req.sksg_dma_address,
+				   1 * sizeof(struct fit_sg_descriptor),
+				   DMA_TO_DEVICE);
+	dma_sync_single_for_device(&skdev->pdev->dev,
+				   skspcl->db_dma_address,
+				   skspcl->req.sksg_list[0].byte_count,
+				   DMA_BIDIRECTIONAL);
+
+	/* Make sure skd_msg_buf is written before the doorbell is triggered. */
+	smp_wmb();
+
 	SKD_WRITEQ(skdev, qcmd, FIT_Q_COMMAND);
 }
 
@@ -2212,8 +1296,8 @@ static void skd_send_special_fitmsg(struct skd_device *skdev,
  */
 
 static void skd_complete_other(struct skd_device *skdev,
-			       volatile struct fit_completion_entry_v1 *skcomp,
-			       volatile struct fit_comp_error_info *skerr);
+			       struct fit_completion_entry_v1 *skcomp,
+			       struct fit_comp_error_info *skerr);
 
 struct sns_info {
 	u8 type;
@@ -2262,21 +1346,20 @@ static struct sns_info skd_chkstat_table[] = {
 
 static enum skd_check_status_action
 skd_check_status(struct skd_device *skdev,
-		 u8 cmp_status, volatile struct fit_comp_error_info *skerr)
+		 u8 cmp_status, struct fit_comp_error_info *skerr)
 {
-	int i, n;
+	int i;
 
-	pr_err("(%s): key/asc/ascq/fruc %02x/%02x/%02x/%02x\n",
-	       skd_name(skdev), skerr->key, skerr->code, skerr->qual,
-	       skerr->fruc);
+	dev_err(&skdev->pdev->dev, "key/asc/ascq/fruc %02x/%02x/%02x/%02x\n",
+		skerr->key, skerr->code, skerr->qual, skerr->fruc);
 
-	pr_debug("%s:%s:%d stat: t=%02x stat=%02x k=%02x c=%02x q=%02x fruc=%02x\n",
-		 skdev->name, __func__, __LINE__, skerr->type, cmp_status,
-		 skerr->key, skerr->code, skerr->qual, skerr->fruc);
+	dev_dbg(&skdev->pdev->dev,
+		"stat: t=%02x stat=%02x k=%02x c=%02x q=%02x fruc=%02x\n",
+		skerr->type, cmp_status, skerr->key, skerr->code, skerr->qual,
+		skerr->fruc);
 
 	/* Does the info match an entry in the good category? */
-	n = sizeof(skd_chkstat_table) / sizeof(skd_chkstat_table[0]);
-	for (i = 0; i < n; i++) {
+	for (i = 0; i < ARRAY_SIZE(skd_chkstat_table); i++) {
 		struct sns_info *sns = &skd_chkstat_table[i];
 
 		if (sns->mask & 0x10)
@@ -2300,10 +1383,9 @@ skd_check_status(struct skd_device *skdev,
 				continue;
 
 		if (sns->action == SKD_CHECK_STATUS_REPORT_SMART_ALERT) {
-			pr_err("(%s): SMART Alert: sense key/asc/ascq "
-			       "%02x/%02x/%02x\n",
-			       skd_name(skdev), skerr->key,
-			       skerr->code, skerr->qual);
+			dev_err(&skdev->pdev->dev,
+				"SMART Alert: sense key/asc/ascq %02x/%02x/%02x\n",
+				skerr->key, skerr->code, skerr->qual);
 		}
 		return sns->action;
 	}
@@ -2312,335 +1394,80 @@ skd_check_status(struct skd_device *skdev,
 	 * zero status means good
 	 */
 	if (cmp_status) {
-		pr_debug("%s:%s:%d status check: error\n",
-			 skdev->name, __func__, __LINE__);
+		dev_dbg(&skdev->pdev->dev, "status check: error\n");
 		return SKD_CHECK_STATUS_REPORT_ERROR;
 	}
 
-	pr_debug("%s:%s:%d status check good default\n",
-		 skdev->name, __func__, __LINE__);
+	dev_dbg(&skdev->pdev->dev, "status check good default\n");
 	return SKD_CHECK_STATUS_REPORT_GOOD;
 }
 
 static void skd_resolve_req_exception(struct skd_device *skdev,
-				      struct skd_request_context *skreq)
+				      struct skd_request_context *skreq,
+				      struct request *req)
 {
 	u8 cmp_status = skreq->completion.status;
 
 	switch (skd_check_status(skdev, cmp_status, &skreq->err_info)) {
 	case SKD_CHECK_STATUS_REPORT_GOOD:
 	case SKD_CHECK_STATUS_REPORT_SMART_ALERT:
-		skd_end_request(skdev, skreq, BLK_STS_OK);
+		skreq->status = BLK_STS_OK;
+		blk_mq_complete_request(req);
 		break;
 
 	case SKD_CHECK_STATUS_BUSY_IMMINENT:
 		skd_log_skreq(skdev, skreq, "retry(busy)");
-		blk_requeue_request(skdev->queue, skreq->req);
-		pr_info("(%s) drive BUSY imminent\n", skd_name(skdev));
+		blk_requeue_request(skdev->queue, req);
+		dev_info(&skdev->pdev->dev, "drive BUSY imminent\n");
 		skdev->state = SKD_DRVR_STATE_BUSY_IMMINENT;
 		skdev->timer_countdown = SKD_TIMER_MINUTES(20);
 		skd_quiesce_dev(skdev);
 		break;
 
 	case SKD_CHECK_STATUS_REQUEUE_REQUEST:
-		if ((unsigned long) ++skreq->req->special < SKD_MAX_RETRIES) {
+		if ((unsigned long) ++req->special < SKD_MAX_RETRIES) {
 			skd_log_skreq(skdev, skreq, "retry");
-			blk_requeue_request(skdev->queue, skreq->req);
+			blk_requeue_request(skdev->queue, req);
 			break;
 		}
-	/* fall through to report error */
+		/* fall through */
 
 	case SKD_CHECK_STATUS_REPORT_ERROR:
 	default:
-		skd_end_request(skdev, skreq, BLK_STS_IOERR);
+		skreq->status = BLK_STS_IOERR;
+		blk_mq_complete_request(req);
 		break;
 	}
 }
 
-/* assume spinlock is already held */
 static void skd_release_skreq(struct skd_device *skdev,
 			      struct skd_request_context *skreq)
 {
-	u32 msg_slot;
-	struct skd_fitmsg_context *skmsg;
-
-	u32 timo_slot;
-
-	/*
-	 * Reclaim the FIT msg buffer if this is
-	 * the first of the requests it carried to
-	 * be completed. The FIT msg buffer used to
-	 * send this request cannot be reused until
-	 * we are sure the s1120 card has copied
-	 * it to its memory. The FIT msg might have
-	 * contained several requests. As soon as
-	 * any of them are completed we know that
-	 * the entire FIT msg was transferred.
-	 * Only the first completed request will
-	 * match the FIT msg buffer id. The FIT
-	 * msg buffer id is immediately updated.
-	 * When subsequent requests complete the FIT
-	 * msg buffer id won't match, so we know
-	 * quite cheaply that it is already done.
-	 */
-	msg_slot = skreq->fitmsg_id & SKD_ID_SLOT_MASK;
-	SKD_ASSERT(msg_slot < skdev->num_fitmsg_context);
-
-	skmsg = &skdev->skmsg_table[msg_slot];
-	if (skmsg->id == skreq->fitmsg_id) {
-		SKD_ASSERT(skmsg->state == SKD_MSG_STATE_BUSY);
-		SKD_ASSERT(skmsg->outstanding > 0);
-		skmsg->outstanding--;
-		if (skmsg->outstanding == 0) {
-			skmsg->state = SKD_MSG_STATE_IDLE;
-			skmsg->id += SKD_ID_INCR;
-			skmsg->next = skdev->skmsg_free_list;
-			skdev->skmsg_free_list = skmsg;
-		}
-	}
-
-	/*
-	 * Decrease the number of active requests.
-	 * Also decrements the count in the timeout slot.
-	 */
-	SKD_ASSERT(skdev->in_flight > 0);
-	skdev->in_flight -= 1;
-
-	timo_slot = skreq->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
-	SKD_ASSERT(skdev->timeout_slot[timo_slot] > 0);
-	skdev->timeout_slot[timo_slot] -= 1;
-
-	/*
-	 * Reset backpointer
-	 */
-	skreq->req = NULL;
-
 	/*
 	 * Reclaim the skd_request_context
 	 */
 	skreq->state = SKD_REQ_STATE_IDLE;
-	skreq->id += SKD_ID_INCR;
-	skreq->next = skdev->skreq_free_list;
-	skdev->skreq_free_list = skreq;
 }
 
-#define DRIVER_INQ_EVPD_PAGE_CODE   0xDA
-
-static void skd_do_inq_page_00(struct skd_device *skdev,
-			       volatile struct fit_completion_entry_v1 *skcomp,
-			       volatile struct fit_comp_error_info *skerr,
-			       uint8_t *cdb, uint8_t *buf)
-{
-	uint16_t insert_pt, max_bytes, drive_pages, drive_bytes, new_size;
-
-	/* Caller requested "supported pages".  The driver needs to insert
-	 * its page.
-	 */
-	pr_debug("%s:%s:%d skd_do_driver_inquiry: modify supported pages.\n",
-		 skdev->name, __func__, __LINE__);
-
-	/* If the device rejected the request because the CDB was
-	 * improperly formed, then just leave.
-	 */
-	if (skcomp->status == SAM_STAT_CHECK_CONDITION &&
-	    skerr->key == ILLEGAL_REQUEST && skerr->code == 0x24)
-		return;
-
-	/* Get the amount of space the caller allocated */
-	max_bytes = (cdb[3] << 8) | cdb[4];
-
-	/* Get the number of pages actually returned by the device */
-	drive_pages = (buf[2] << 8) | buf[3];
-	drive_bytes = drive_pages + 4;
-	new_size = drive_pages + 1;
-
-	/* Supported pages must be in numerical order, so find where
-	 * the driver page needs to be inserted into the list of
-	 * pages returned by the device.
-	 */
-	for (insert_pt = 4; insert_pt < drive_bytes; insert_pt++) {
-		if (buf[insert_pt] == DRIVER_INQ_EVPD_PAGE_CODE)
-			return; /* Device using this page code. abort */
-		else if (buf[insert_pt] > DRIVER_INQ_EVPD_PAGE_CODE)
-			break;
-	}
-
-	if (insert_pt < max_bytes) {
-		uint16_t u;
-
-		/* Shift everything up one byte to make room. */
-		for (u = new_size + 3; u > insert_pt; u--)
-			buf[u] = buf[u - 1];
-		buf[insert_pt] = DRIVER_INQ_EVPD_PAGE_CODE;
-
-		/* SCSI byte order increment of num_returned_bytes by 1 */
-		skcomp->num_returned_bytes =
-			be32_to_cpu(skcomp->num_returned_bytes) + 1;
-		skcomp->num_returned_bytes =
-			be32_to_cpu(skcomp->num_returned_bytes);
-	}
-
-	/* update page length field to reflect the driver's page too */
-	buf[2] = (uint8_t)((new_size >> 8) & 0xFF);
-	buf[3] = (uint8_t)((new_size >> 0) & 0xFF);
-}
-
-static void skd_get_link_info(struct pci_dev *pdev, u8 *speed, u8 *width)
-{
-	int pcie_reg;
-	u16 pci_bus_speed;
-	u8 pci_lanes;
-
-	pcie_reg = pci_find_capability(pdev, PCI_CAP_ID_EXP);
-	if (pcie_reg) {
-		u16 linksta;
-		pci_read_config_word(pdev, pcie_reg + PCI_EXP_LNKSTA, &linksta);
-
-		pci_bus_speed = linksta & 0xF;
-		pci_lanes = (linksta & 0x3F0) >> 4;
-	} else {
-		*speed = STEC_LINK_UNKNOWN;
-		*width = 0xFF;
-		return;
-	}
-
-	switch (pci_bus_speed) {
-	case 1:
-		*speed = STEC_LINK_2_5GTS;
-		break;
-	case 2:
-		*speed = STEC_LINK_5GTS;
-		break;
-	case 3:
-		*speed = STEC_LINK_8GTS;
-		break;
-	default:
-		*speed = STEC_LINK_UNKNOWN;
-		break;
-	}
-
-	if (pci_lanes <= 0x20)
-		*width = pci_lanes;
-	else
-		*width = 0xFF;
-}
-
-static void skd_do_inq_page_da(struct skd_device *skdev,
-			       volatile struct fit_completion_entry_v1 *skcomp,
-			       volatile struct fit_comp_error_info *skerr,
-			       uint8_t *cdb, uint8_t *buf)
-{
-	struct pci_dev *pdev = skdev->pdev;
-	unsigned max_bytes;
-	struct driver_inquiry_data inq;
-	u16 val;
-
-	pr_debug("%s:%s:%d skd_do_driver_inquiry: return driver page\n",
-		 skdev->name, __func__, __LINE__);
-
-	memset(&inq, 0, sizeof(inq));
-
-	inq.page_code = DRIVER_INQ_EVPD_PAGE_CODE;
-
-	skd_get_link_info(pdev, &inq.pcie_link_speed, &inq.pcie_link_lanes);
-	inq.pcie_bus_number = cpu_to_be16(pdev->bus->number);
-	inq.pcie_device_number = PCI_SLOT(pdev->devfn);
-	inq.pcie_function_number = PCI_FUNC(pdev->devfn);
-
-	pci_read_config_word(pdev, PCI_VENDOR_ID, &val);
-	inq.pcie_vendor_id = cpu_to_be16(val);
-
-	pci_read_config_word(pdev, PCI_DEVICE_ID, &val);
-	inq.pcie_device_id = cpu_to_be16(val);
-
-	pci_read_config_word(pdev, PCI_SUBSYSTEM_VENDOR_ID, &val);
-	inq.pcie_subsystem_vendor_id = cpu_to_be16(val);
-
-	pci_read_config_word(pdev, PCI_SUBSYSTEM_ID, &val);
-	inq.pcie_subsystem_device_id = cpu_to_be16(val);
-
-	/* Driver version, fixed lenth, padded with spaces on the right */
-	inq.driver_version_length = sizeof(inq.driver_version);
-	memset(&inq.driver_version, ' ', sizeof(inq.driver_version));
-	memcpy(inq.driver_version, DRV_VER_COMPL,
-	       min(sizeof(inq.driver_version), strlen(DRV_VER_COMPL)));
-
-	inq.page_length = cpu_to_be16((sizeof(inq) - 4));
-
-	/* Clear the error set by the device */
-	skcomp->status = SAM_STAT_GOOD;
-	memset((void *)skerr, 0, sizeof(*skerr));
-
-	/* copy response into output buffer */
-	max_bytes = (cdb[3] << 8) | cdb[4];
-	memcpy(buf, &inq, min_t(unsigned, max_bytes, sizeof(inq)));
-
-	skcomp->num_returned_bytes =
-		be32_to_cpu(min_t(uint16_t, max_bytes, sizeof(inq)));
-}
-
-static void skd_do_driver_inq(struct skd_device *skdev,
-			      volatile struct fit_completion_entry_v1 *skcomp,
-			      volatile struct fit_comp_error_info *skerr,
-			      uint8_t *cdb, uint8_t *buf)
-{
-	if (!buf)
-		return;
-	else if (cdb[0] != INQUIRY)
-		return;         /* Not an INQUIRY */
-	else if ((cdb[1] & 1) == 0)
-		return;         /* EVPD not set */
-	else if (cdb[2] == 0)
-		/* Need to add driver's page to supported pages list */
-		skd_do_inq_page_00(skdev, skcomp, skerr, cdb, buf);
-	else if (cdb[2] == DRIVER_INQ_EVPD_PAGE_CODE)
-		/* Caller requested driver's page */
-		skd_do_inq_page_da(skdev, skcomp, skerr, cdb, buf);
-}
-
-static unsigned char *skd_sg_1st_page_ptr(struct scatterlist *sg)
-{
-	if (!sg)
-		return NULL;
-	if (!sg_page(sg))
-		return NULL;
-	return sg_virt(sg);
-}
-
-static void skd_process_scsi_inq(struct skd_device *skdev,
-				 volatile struct fit_completion_entry_v1
-				 *skcomp,
-				 volatile struct fit_comp_error_info *skerr,
-				 struct skd_special_context *skspcl)
-{
-	uint8_t *buf;
-	struct fit_msg_hdr *fmh = (struct fit_msg_hdr *)skspcl->msg_buf;
-	struct skd_scsi_request *scsi_req = (struct skd_scsi_request *)&fmh[1];
-
-	dma_sync_sg_for_cpu(skdev->class_dev, skspcl->req.sg, skspcl->req.n_sg,
-			    skspcl->req.sg_data_dir);
-	buf = skd_sg_1st_page_ptr(skspcl->req.sg);
-
-	if (buf)
-		skd_do_driver_inq(skdev, skcomp, skerr, scsi_req->cdb, buf);
-}
-
-
 static int skd_isr_completion_posted(struct skd_device *skdev,
 					int limit, int *enqueued)
 {
-	volatile struct fit_completion_entry_v1 *skcmp = NULL;
-	volatile struct fit_comp_error_info *skerr;
+	struct fit_completion_entry_v1 *skcmp;
+	struct fit_comp_error_info *skerr;
 	u16 req_id;
-	u32 req_slot;
+	u32 tag;
+	u16 hwq = 0;
+	struct request *rq;
 	struct skd_request_context *skreq;
-	u16 cmp_cntxt = 0;
-	u8 cmp_status = 0;
-	u8 cmp_cycle = 0;
-	u32 cmp_bytes = 0;
+	u16 cmp_cntxt;
+	u8 cmp_status;
+	u8 cmp_cycle;
+	u32 cmp_bytes;
 	int rc = 0;
 	int processed = 0;
 
+	lockdep_assert_held(&skdev->lock);
+
 	for (;; ) {
 		SKD_ASSERT(skdev->skcomp_ix < SKD_N_COMPLETION_ENTRY);
 
@@ -2652,16 +1479,14 @@ static int skd_isr_completion_posted(struct skd_device *skdev,
 
 		skerr = &skdev->skerr_table[skdev->skcomp_ix];
 
-		pr_debug("%s:%s:%d "
-			 "cycle=%d ix=%d got cycle=%d cmdctxt=0x%x stat=%d "
-			 "busy=%d rbytes=0x%x proto=%d\n",
-			 skdev->name, __func__, __LINE__, skdev->skcomp_cycle,
-			 skdev->skcomp_ix, cmp_cycle, cmp_cntxt, cmp_status,
-			 skdev->in_flight, cmp_bytes, skdev->proto_ver);
+		dev_dbg(&skdev->pdev->dev,
+			"cycle=%d ix=%d got cycle=%d cmdctxt=0x%x stat=%d busy=%d rbytes=0x%x proto=%d\n",
+			skdev->skcomp_cycle, skdev->skcomp_ix, cmp_cycle,
+			cmp_cntxt, cmp_status, skd_in_flight(skdev),
+			cmp_bytes, skdev->proto_ver);
 
 		if (cmp_cycle != skdev->skcomp_cycle) {
-			pr_debug("%s:%s:%d end of completions\n",
-				 skdev->name, __func__, __LINE__);
+			dev_dbg(&skdev->pdev->dev, "end of completions\n");
 			break;
 		}
 		/*
@@ -2680,49 +1505,38 @@ static int skd_isr_completion_posted(struct skd_device *skdev,
 		 * r/w request (see skd_start() above) or a special request.
 		 */
 		req_id = cmp_cntxt;
-		req_slot = req_id & SKD_ID_SLOT_AND_TABLE_MASK;
+		tag = req_id & SKD_ID_SLOT_AND_TABLE_MASK;
 
 		/* Is this other than a r/w request? */
-		if (req_slot >= skdev->num_req_context) {
+		if (tag >= skdev->num_req_context) {
 			/*
 			 * This is not a completion for a r/w request.
 			 */
+			WARN_ON_ONCE(blk_mq_tag_to_rq(skdev->tag_set.tags[hwq],
+						      tag));
 			skd_complete_other(skdev, skcmp, skerr);
 			continue;
 		}
 
-		skreq = &skdev->skreq_table[req_slot];
+		rq = blk_mq_tag_to_rq(skdev->tag_set.tags[hwq], tag);
+		if (WARN(!rq, "No request for tag %#x -> %#x\n", cmp_cntxt,
+			 tag))
+			continue;
+		skreq = blk_mq_rq_to_pdu(rq);
 
 		/*
 		 * Make sure the request ID for the slot matches.
 		 */
 		if (skreq->id != req_id) {
-			pr_debug("%s:%s:%d mismatch comp_id=0x%x req_id=0x%x\n",
-				 skdev->name, __func__, __LINE__,
-				 req_id, skreq->id);
-			{
-				u16 new_id = cmp_cntxt;
-				pr_err("(%s): Completion mismatch "
-				       "comp_id=0x%04x skreq=0x%04x new=0x%04x\n",
-				       skd_name(skdev), req_id,
-				       skreq->id, new_id);
+			dev_err(&skdev->pdev->dev,
+				"Completion mismatch comp_id=0x%04x skreq=0x%04x new=0x%04x\n",
+				req_id, skreq->id, cmp_cntxt);
 
-				continue;
-			}
+			continue;
 		}
 
 		SKD_ASSERT(skreq->state == SKD_REQ_STATE_BUSY);
 
-		if (skreq->state == SKD_REQ_STATE_ABORTED) {
-			pr_debug("%s:%s:%d reclaim req %p id=%04x\n",
-				 skdev->name, __func__, __LINE__,
-				 skreq, skreq->id);
-			/* a previously timed out command can
-			 * now be cleaned up */
-			skd_release_skreq(skdev, skreq);
-			continue;
-		}
-
 		skreq->completion = *skcmp;
 		if (unlikely(cmp_status == SAM_STAT_CHECK_CONDITION)) {
 			skreq->err_info = *skerr;
@@ -2734,27 +1548,17 @@ static int skd_isr_completion_posted(struct skd_device *skdev,
 		if (skreq->n_sg > 0)
 			skd_postop_sg_list(skdev, skreq);
 
-		if (!skreq->req) {
-			pr_debug("%s:%s:%d NULL backptr skdreq %p, "
-				 "req=0x%x req_id=0x%x\n",
-				 skdev->name, __func__, __LINE__,
-				 skreq, skreq->id, req_id);
-		} else {
-			/*
-			 * Capture the outcome and post it back to the
-			 * native request.
-			 */
-			if (likely(cmp_status == SAM_STAT_GOOD))
-				skd_end_request(skdev, skreq, BLK_STS_OK);
-			else
-				skd_resolve_req_exception(skdev, skreq);
-		}
+		skd_release_skreq(skdev, skreq);
 
 		/*
-		 * Release the skreq, its FIT msg (if one), timeout slot,
-		 * and queue depth.
+		 * Capture the outcome and post it back to the native request.
 		 */
-		skd_release_skreq(skdev, skreq);
+		if (likely(cmp_status == SAM_STAT_GOOD)) {
+			skreq->status = BLK_STS_OK;
+			blk_mq_complete_request(rq);
+		} else {
+			skd_resolve_req_exception(skdev, skreq, rq);
+		}
 
 		/* skd_isr_comp_limit equal zero means no limit */
 		if (limit) {
@@ -2765,8 +1569,8 @@ static int skd_isr_completion_posted(struct skd_device *skdev,
 		}
 	}
 
-	if ((skdev->state == SKD_DRVR_STATE_PAUSING)
-		&& (skdev->in_flight) == 0) {
+	if (skdev->state == SKD_DRVR_STATE_PAUSING &&
+	    skd_in_flight(skdev) == 0) {
 		skdev->state = SKD_DRVR_STATE_PAUSED;
 		wake_up_interruptible(&skdev->waitq);
 	}
@@ -2775,21 +1579,22 @@ static int skd_isr_completion_posted(struct skd_device *skdev,
 }
 
 static void skd_complete_other(struct skd_device *skdev,
-			       volatile struct fit_completion_entry_v1 *skcomp,
-			       volatile struct fit_comp_error_info *skerr)
+			       struct fit_completion_entry_v1 *skcomp,
+			       struct fit_comp_error_info *skerr)
 {
 	u32 req_id = 0;
 	u32 req_table;
 	u32 req_slot;
 	struct skd_special_context *skspcl;
 
+	lockdep_assert_held(&skdev->lock);
+
 	req_id = skcomp->tag;
 	req_table = req_id & SKD_ID_TABLE_MASK;
 	req_slot = req_id & SKD_ID_SLOT_MASK;
 
-	pr_debug("%s:%s:%d table=0x%x id=0x%x slot=%d\n",
-		 skdev->name, __func__, __LINE__,
-		 req_table, req_id, req_slot);
+	dev_dbg(&skdev->pdev->dev, "table=0x%x id=0x%x slot=%d\n", req_table,
+		req_id, req_slot);
 
 	/*
 	 * Based on the request id, determine how to dispatch this completion.
@@ -2799,28 +1604,12 @@ static void skd_complete_other(struct skd_device *skdev,
 	switch (req_table) {
 	case SKD_ID_RW_REQUEST:
 		/*
-		 * The caller, skd_completion_posted_isr() above,
+		 * The caller, skd_isr_completion_posted() above,
 		 * handles r/w requests. The only way we get here
 		 * is if the req_slot is out of bounds.
 		 */
 		break;
 
-	case SKD_ID_SPECIAL_REQUEST:
-		/*
-		 * Make sure the req_slot is in bounds and that the id
-		 * matches.
-		 */
-		if (req_slot < skdev->n_special) {
-			skspcl = &skdev->skspcl_table[req_slot];
-			if (skspcl->req.id == req_id &&
-			    skspcl->req.state == SKD_REQ_STATE_BUSY) {
-				skd_complete_special(skdev,
-						     skcomp, skerr, skspcl);
-				return;
-			}
-		}
-		break;
-
 	case SKD_ID_INTERNAL:
 		if (req_slot == 0) {
 			skspcl = &skdev->internal_skspcl;
@@ -2851,72 +1640,9 @@ static void skd_complete_other(struct skd_device *skdev,
 	 */
 }
 
-static void skd_complete_special(struct skd_device *skdev,
-				 volatile struct fit_completion_entry_v1
-				 *skcomp,
-				 volatile struct fit_comp_error_info *skerr,
-				 struct skd_special_context *skspcl)
-{
-	pr_debug("%s:%s:%d  completing special request %p\n",
-		 skdev->name, __func__, __LINE__, skspcl);
-	if (skspcl->orphaned) {
-		/* Discard orphaned request */
-		/* ?: Can this release directly or does it need
-		 * to use a worker? */
-		pr_debug("%s:%s:%d release orphaned %p\n",
-			 skdev->name, __func__, __LINE__, skspcl);
-		skd_release_special(skdev, skspcl);
-		return;
-	}
-
-	skd_process_scsi_inq(skdev, skcomp, skerr, skspcl);
-
-	skspcl->req.state = SKD_REQ_STATE_COMPLETED;
-	skspcl->req.completion = *skcomp;
-	skspcl->req.err_info = *skerr;
-
-	skd_log_check_status(skdev, skspcl->req.completion.status, skerr->key,
-			     skerr->code, skerr->qual, skerr->fruc);
-
-	wake_up_interruptible(&skdev->waitq);
-}
-
-/* assume spinlock is already held */
-static void skd_release_special(struct skd_device *skdev,
-				struct skd_special_context *skspcl)
-{
-	int i, was_depleted;
-
-	for (i = 0; i < skspcl->req.n_sg; i++) {
-		struct page *page = sg_page(&skspcl->req.sg[i]);
-		__free_page(page);
-	}
-
-	was_depleted = (skdev->skspcl_free_list == NULL);
-
-	skspcl->req.state = SKD_REQ_STATE_IDLE;
-	skspcl->req.id += SKD_ID_INCR;
-	skspcl->req.next =
-		(struct skd_request_context *)skdev->skspcl_free_list;
-	skdev->skspcl_free_list = (struct skd_special_context *)skspcl;
-
-	if (was_depleted) {
-		pr_debug("%s:%s:%d skspcl was depleted\n",
-			 skdev->name, __func__, __LINE__);
-		/* Free list was depleted. Their might be waiters. */
-		wake_up_interruptible(&skdev->waitq);
-	}
-}
-
 static void skd_reset_skcomp(struct skd_device *skdev)
 {
-	u32 nbytes;
-	struct fit_completion_entry_v1 *skcomp;
-
-	nbytes = sizeof(*skcomp) * SKD_N_COMPLETION_ENTRY;
-	nbytes += sizeof(struct fit_comp_error_info) * SKD_N_COMPLETION_ENTRY;
-
-	memset(skdev->skcomp_table, 0, nbytes);
+	memset(skdev->skcomp_table, 0, SKD_SKCOMP_SIZE);
 
 	skdev->skcomp_ix = 0;
 	skdev->skcomp_cycle = 1;
@@ -2941,7 +1667,7 @@ static void skd_completion_worker(struct work_struct *work)
 	 * process everything in compq
 	 */
 	skd_isr_completion_posted(skdev, 0, &flush_enqueued);
-	skd_request_fn(skdev->queue);
+	schedule_work(&skdev->start_queue);
 
 	spin_unlock_irqrestore(&skdev->lock, flags);
 }
@@ -2951,14 +1677,13 @@ static void skd_isr_msg_from_dev(struct skd_device *skdev);
 static irqreturn_t
 skd_isr(int irq, void *ptr)
 {
-	struct skd_device *skdev;
+	struct skd_device *skdev = ptr;
 	u32 intstat;
 	u32 ack;
 	int rc = 0;
 	int deferred = 0;
 	int flush_enqueued = 0;
 
-	skdev = (struct skd_device *)ptr;
 	spin_lock(&skdev->lock);
 
 	for (;; ) {
@@ -2967,8 +1692,8 @@ skd_isr(int irq, void *ptr)
 		ack = FIT_INT_DEF_MASK;
 		ack &= intstat;
 
-		pr_debug("%s:%s:%d intstat=0x%x ack=0x%x\n",
-			 skdev->name, __func__, __LINE__, intstat, ack);
+		dev_dbg(&skdev->pdev->dev, "intstat=0x%x ack=0x%x\n", intstat,
+			ack);
 
 		/* As long as there is an int pending on device, keep
 		 * running loop.  When none, get out, but if we've never
@@ -3018,12 +1743,12 @@ skd_isr(int irq, void *ptr)
 	}
 
 	if (unlikely(flush_enqueued))
-		skd_request_fn(skdev->queue);
+		schedule_work(&skdev->start_queue);
 
 	if (deferred)
 		schedule_work(&skdev->completion_worker);
 	else if (!flush_enqueued)
-		skd_request_fn(skdev->queue);
+		schedule_work(&skdev->start_queue);
 
 	spin_unlock(&skdev->lock);
 
@@ -3033,13 +1758,13 @@ skd_isr(int irq, void *ptr)
 static void skd_drive_fault(struct skd_device *skdev)
 {
 	skdev->state = SKD_DRVR_STATE_FAULT;
-	pr_err("(%s): Drive FAULT\n", skd_name(skdev));
+	dev_err(&skdev->pdev->dev, "Drive FAULT\n");
 }
 
 static void skd_drive_disappeared(struct skd_device *skdev)
 {
 	skdev->state = SKD_DRVR_STATE_DISAPPEARED;
-	pr_err("(%s): Drive DISAPPEARED\n", skd_name(skdev));
+	dev_err(&skdev->pdev->dev, "Drive DISAPPEARED\n");
 }
 
 static void skd_isr_fwstate(struct skd_device *skdev)
@@ -3052,10 +1777,9 @@ static void skd_isr_fwstate(struct skd_device *skdev)
 	sense = SKD_READL(skdev, FIT_STATUS);
 	state = sense & FIT_SR_DRIVE_STATE_MASK;
 
-	pr_err("(%s): s1120 state %s(%d)=>%s(%d)\n",
-	       skd_name(skdev),
-	       skd_drive_state_to_str(skdev->drive_state), skdev->drive_state,
-	       skd_drive_state_to_str(state), state);
+	dev_err(&skdev->pdev->dev, "s1120 state %s(%d)=>%s(%d)\n",
+		skd_drive_state_to_str(skdev->drive_state), skdev->drive_state,
+		skd_drive_state_to_str(state), state);
 
 	skdev->drive_state = state;
 
@@ -3066,7 +1790,7 @@ static void skd_isr_fwstate(struct skd_device *skdev)
 			break;
 		}
 		if (skdev->state == SKD_DRVR_STATE_RESTARTING)
-			skd_recover_requests(skdev, 0);
+			skd_recover_requests(skdev);
 		if (skdev->state == SKD_DRVR_STATE_WAIT_BOOT) {
 			skdev->timer_countdown = SKD_STARTING_TIMO;
 			skdev->state = SKD_DRVR_STATE_STARTING;
@@ -3087,11 +1811,11 @@ static void skd_isr_fwstate(struct skd_device *skdev)
 			skdev->cur_max_queue_depth * 2 / 3 + 1;
 		if (skdev->queue_low_water_mark < 1)
 			skdev->queue_low_water_mark = 1;
-		pr_info(
-		       "(%s): Queue depth limit=%d dev=%d lowat=%d\n",
-		       skd_name(skdev),
-		       skdev->cur_max_queue_depth,
-		       skdev->dev_max_queue_depth, skdev->queue_low_water_mark);
+		dev_info(&skdev->pdev->dev,
+			 "Queue depth limit=%d dev=%d lowat=%d\n",
+			 skdev->cur_max_queue_depth,
+			 skdev->dev_max_queue_depth,
+			 skdev->queue_low_water_mark);
 
 		skd_refresh_device_data(skdev);
 		break;
@@ -3107,7 +1831,7 @@ static void skd_isr_fwstate(struct skd_device *skdev)
 		 */
 		skdev->state = SKD_DRVR_STATE_BUSY_SANITIZE;
 		skdev->timer_countdown = SKD_TIMER_SECONDS(3);
-		blk_start_queue(skdev->queue);
+		schedule_work(&skdev->start_queue);
 		break;
 	case FIT_SR_DRIVE_BUSY_ERASE:
 		skdev->state = SKD_DRVR_STATE_BUSY_ERASE;
@@ -3128,8 +1852,7 @@ static void skd_isr_fwstate(struct skd_device *skdev)
 		}
 		break;
 	case FIT_SR_DRIVE_FW_BOOTING:
-		pr_debug("%s:%s:%d ISR FIT_SR_DRIVE_FW_BOOTING %s\n",
-			 skdev->name, __func__, __LINE__, skdev->name);
+		dev_dbg(&skdev->pdev->dev, "ISR FIT_SR_DRIVE_FW_BOOTING\n");
 		skdev->state = SKD_DRVR_STATE_WAIT_BOOT;
 		skdev->timer_countdown = SKD_WAIT_BOOT_TIMO;
 		break;
@@ -3141,17 +1864,17 @@ static void skd_isr_fwstate(struct skd_device *skdev)
 
 	case FIT_SR_DRIVE_FAULT:
 		skd_drive_fault(skdev);
-		skd_recover_requests(skdev, 0);
-		blk_start_queue(skdev->queue);
+		skd_recover_requests(skdev);
+		schedule_work(&skdev->start_queue);
 		break;
 
 	/* PCIe bus returned all Fs? */
 	case 0xFF:
-		pr_info("(%s): state=0x%x sense=0x%x\n",
-		       skd_name(skdev), state, sense);
+		dev_info(&skdev->pdev->dev, "state=0x%x sense=0x%x\n", state,
+			 sense);
 		skd_drive_disappeared(skdev);
-		skd_recover_requests(skdev, 0);
-		blk_start_queue(skdev->queue);
+		skd_recover_requests(skdev);
+		schedule_work(&skdev->start_queue);
 		break;
 	default:
 		/*
@@ -3159,92 +1882,33 @@ static void skd_isr_fwstate(struct skd_device *skdev)
 		 */
 		break;
 	}
-	pr_err("(%s): Driver state %s(%d)=>%s(%d)\n",
-	       skd_name(skdev),
-	       skd_skdev_state_to_str(prev_driver_state), prev_driver_state,
-	       skd_skdev_state_to_str(skdev->state), skdev->state);
+	dev_err(&skdev->pdev->dev, "Driver state %s(%d)=>%s(%d)\n",
+		skd_skdev_state_to_str(prev_driver_state), prev_driver_state,
+		skd_skdev_state_to_str(skdev->state), skdev->state);
 }
 
-static void skd_recover_requests(struct skd_device *skdev, int requeue)
+static void skd_recover_request(struct request *req, void *data, bool reserved)
 {
-	int i;
-
-	for (i = 0; i < skdev->num_req_context; i++) {
-		struct skd_request_context *skreq = &skdev->skreq_table[i];
-
-		if (skreq->state == SKD_REQ_STATE_BUSY) {
-			skd_log_skreq(skdev, skreq, "recover");
-
-			SKD_ASSERT((skreq->id & SKD_ID_INCR) != 0);
-			SKD_ASSERT(skreq->req != NULL);
-
-			/* Release DMA resources for the request. */
-			if (skreq->n_sg > 0)
-				skd_postop_sg_list(skdev, skreq);
-
-			if (requeue &&
-			    (unsigned long) ++skreq->req->special <
-			    SKD_MAX_RETRIES)
-				blk_requeue_request(skdev->queue, skreq->req);
-			else
-				skd_end_request(skdev, skreq, BLK_STS_IOERR);
-
-			skreq->req = NULL;
-
-			skreq->state = SKD_REQ_STATE_IDLE;
-			skreq->id += SKD_ID_INCR;
-		}
-		if (i > 0)
-			skreq[-1].next = skreq;
-		skreq->next = NULL;
-	}
-	skdev->skreq_free_list = skdev->skreq_table;
+	struct skd_device *const skdev = data;
+	struct skd_request_context *skreq = blk_mq_rq_to_pdu(req);
 
-	for (i = 0; i < skdev->num_fitmsg_context; i++) {
-		struct skd_fitmsg_context *skmsg = &skdev->skmsg_table[i];
+	if (skreq->state != SKD_REQ_STATE_BUSY)
+		return;
 
-		if (skmsg->state == SKD_MSG_STATE_BUSY) {
-			skd_log_skmsg(skdev, skmsg, "salvaged");
-			SKD_ASSERT((skmsg->id & SKD_ID_INCR) != 0);
-			skmsg->state = SKD_MSG_STATE_IDLE;
-			skmsg->id += SKD_ID_INCR;
-		}
-		if (i > 0)
-			skmsg[-1].next = skmsg;
-		skmsg->next = NULL;
-	}
-	skdev->skmsg_free_list = skdev->skmsg_table;
+	skd_log_skreq(skdev, skreq, "recover");
 
-	for (i = 0; i < skdev->n_special; i++) {
-		struct skd_special_context *skspcl = &skdev->skspcl_table[i];
+	/* Release DMA resources for the request. */
+	if (skreq->n_sg > 0)
+		skd_postop_sg_list(skdev, skreq);
 
-		/* If orphaned, reclaim it because it has already been reported
-		 * to the process as an error (it was just waiting for
-		 * a completion that didn't come, and now it will never come)
-		 * If busy, change to a state that will cause it to error
-		 * out in the wait routine and let it do the normal
-		 * reporting and reclaiming
-		 */
-		if (skspcl->req.state == SKD_REQ_STATE_BUSY) {
-			if (skspcl->orphaned) {
-				pr_debug("%s:%s:%d orphaned %p\n",
-					 skdev->name, __func__, __LINE__,
-					 skspcl);
-				skd_release_special(skdev, skspcl);
-			} else {
-				pr_debug("%s:%s:%d not orphaned %p\n",
-					 skdev->name, __func__, __LINE__,
-					 skspcl);
-				skspcl->req.state = SKD_REQ_STATE_ABORTED;
-			}
-		}
-	}
-	skdev->skspcl_free_list = skdev->skspcl_table;
-
-	for (i = 0; i < SKD_N_TIMEOUT_SLOT; i++)
-		skdev->timeout_slot[i] = 0;
+	skreq->state = SKD_REQ_STATE_IDLE;
+	skreq->status = BLK_STS_IOERR;
+	blk_mq_complete_request(req);
+}
 
-	skdev->in_flight = 0;
+static void skd_recover_requests(struct skd_device *skdev)
+{
+	blk_mq_tagset_busy_iter(&skdev->tag_set, skd_recover_request, skdev);
 }
 
 static void skd_isr_msg_from_dev(struct skd_device *skdev)
@@ -3255,8 +1919,8 @@ static void skd_isr_msg_from_dev(struct skd_device *skdev)
 
 	mfd = SKD_READL(skdev, FIT_MSG_FROM_DEVICE);
 
-	pr_debug("%s:%s:%d mfd=0x%x last_mtd=0x%x\n",
-		 skdev->name, __func__, __LINE__, mfd, skdev->last_mtd);
+	dev_dbg(&skdev->pdev->dev, "mfd=0x%x last_mtd=0x%x\n", mfd,
+		skdev->last_mtd);
 
 	/* ignore any mtd that is an ack for something we didn't send */
 	if (FIT_MXD_TYPE(mfd) != FIT_MXD_TYPE(skdev->last_mtd))
@@ -3267,13 +1931,10 @@ static void skd_isr_msg_from_dev(struct skd_device *skdev)
 		skdev->proto_ver = FIT_PROTOCOL_MAJOR_VER(mfd);
 
 		if (skdev->proto_ver != FIT_PROTOCOL_VERSION_1) {
-			pr_err("(%s): protocol mismatch\n",
-			       skdev->name);
-			pr_err("(%s):   got=%d support=%d\n",
-			       skdev->name, skdev->proto_ver,
-			       FIT_PROTOCOL_VERSION_1);
-			pr_err("(%s):   please upgrade driver\n",
-			       skdev->name);
+			dev_err(&skdev->pdev->dev, "protocol mismatch\n");
+			dev_err(&skdev->pdev->dev, "  got=%d support=%d\n",
+				skdev->proto_ver, FIT_PROTOCOL_VERSION_1);
+			dev_err(&skdev->pdev->dev, "  please upgrade driver\n");
 			skdev->state = SKD_DRVR_STATE_PROTOCOL_MISMATCH;
 			skd_soft_reset(skdev);
 			break;
@@ -3327,9 +1988,8 @@ static void skd_isr_msg_from_dev(struct skd_device *skdev)
 		SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
 		skdev->last_mtd = mtd;
 
-		pr_err("(%s): Time sync driver=0x%x device=0x%x\n",
-		       skd_name(skdev),
-		       skdev->connect_time_stamp, skdev->drive_jiffies);
+		dev_err(&skdev->pdev->dev, "Time sync driver=0x%x device=0x%x\n",
+			skdev->connect_time_stamp, skdev->drive_jiffies);
 		break;
 
 	case FIT_MTD_ARM_QUEUE:
@@ -3351,8 +2011,7 @@ static void skd_disable_interrupts(struct skd_device *skdev)
 	sense = SKD_READL(skdev, FIT_CONTROL);
 	sense &= ~FIT_CR_ENABLE_INTERRUPTS;
 	SKD_WRITEL(skdev, sense, FIT_CONTROL);
-	pr_debug("%s:%s:%d sense 0x%x\n",
-		 skdev->name, __func__, __LINE__, sense);
+	dev_dbg(&skdev->pdev->dev, "sense 0x%x\n", sense);
 
 	/* Note that the 1s is written. A 1-bit means
 	 * disable, a 0 means enable.
@@ -3371,13 +2030,11 @@ static void skd_enable_interrupts(struct skd_device *skdev)
 	/* Note that the compliment of mask is written. A 1-bit means
 	 * disable, a 0 means enable. */
 	SKD_WRITEL(skdev, ~val, FIT_INT_MASK_HOST);
-	pr_debug("%s:%s:%d interrupt mask=0x%x\n",
-		 skdev->name, __func__, __LINE__, ~val);
+	dev_dbg(&skdev->pdev->dev, "interrupt mask=0x%x\n", ~val);
 
 	val = SKD_READL(skdev, FIT_CONTROL);
 	val |= FIT_CR_ENABLE_INTERRUPTS;
-	pr_debug("%s:%s:%d control=0x%x\n",
-		 skdev->name, __func__, __LINE__, val);
+	dev_dbg(&skdev->pdev->dev, "control=0x%x\n", val);
 	SKD_WRITEL(skdev, val, FIT_CONTROL);
 }
 
@@ -3393,8 +2050,7 @@ static void skd_soft_reset(struct skd_device *skdev)
 
 	val = SKD_READL(skdev, FIT_CONTROL);
 	val |= (FIT_CR_SOFT_RESET);
-	pr_debug("%s:%s:%d control=0x%x\n",
-		 skdev->name, __func__, __LINE__, val);
+	dev_dbg(&skdev->pdev->dev, "control=0x%x\n", val);
 	SKD_WRITEL(skdev, val, FIT_CONTROL);
 }
 
@@ -3411,8 +2067,7 @@ static void skd_start_device(struct skd_device *skdev)
 
 	sense = SKD_READL(skdev, FIT_STATUS);
 
-	pr_debug("%s:%s:%d initial status=0x%x\n",
-		 skdev->name, __func__, __LINE__, sense);
+	dev_dbg(&skdev->pdev->dev, "initial status=0x%x\n", sense);
 
 	state = sense & FIT_SR_DRIVE_STATE_MASK;
 	skdev->drive_state = state;
@@ -3425,25 +2080,23 @@ static void skd_start_device(struct skd_device *skdev)
 
 	switch (skdev->drive_state) {
 	case FIT_SR_DRIVE_OFFLINE:
-		pr_err("(%s): Drive offline...\n", skd_name(skdev));
+		dev_err(&skdev->pdev->dev, "Drive offline...\n");
 		break;
 
 	case FIT_SR_DRIVE_FW_BOOTING:
-		pr_debug("%s:%s:%d FIT_SR_DRIVE_FW_BOOTING %s\n",
-			 skdev->name, __func__, __LINE__, skdev->name);
+		dev_dbg(&skdev->pdev->dev, "FIT_SR_DRIVE_FW_BOOTING\n");
 		skdev->state = SKD_DRVR_STATE_WAIT_BOOT;
 		skdev->timer_countdown = SKD_WAIT_BOOT_TIMO;
 		break;
 
 	case FIT_SR_DRIVE_BUSY_SANITIZE:
-		pr_info("(%s): Start: BUSY_SANITIZE\n",
-		       skd_name(skdev));
+		dev_info(&skdev->pdev->dev, "Start: BUSY_SANITIZE\n");
 		skdev->state = SKD_DRVR_STATE_BUSY_SANITIZE;
 		skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
 		break;
 
 	case FIT_SR_DRIVE_BUSY_ERASE:
-		pr_info("(%s): Start: BUSY_ERASE\n", skd_name(skdev));
+		dev_info(&skdev->pdev->dev, "Start: BUSY_ERASE\n");
 		skdev->state = SKD_DRVR_STATE_BUSY_ERASE;
 		skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
 		break;
@@ -3454,14 +2107,13 @@ static void skd_start_device(struct skd_device *skdev)
 		break;
 
 	case FIT_SR_DRIVE_BUSY:
-		pr_err("(%s): Drive Busy...\n", skd_name(skdev));
+		dev_err(&skdev->pdev->dev, "Drive Busy...\n");
 		skdev->state = SKD_DRVR_STATE_BUSY;
 		skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
 		break;
 
 	case FIT_SR_DRIVE_SOFT_RESET:
-		pr_err("(%s) drive soft reset in prog\n",
-		       skd_name(skdev));
+		dev_err(&skdev->pdev->dev, "drive soft reset in prog\n");
 		break;
 
 	case FIT_SR_DRIVE_FAULT:
@@ -3471,9 +2123,8 @@ static void skd_start_device(struct skd_device *skdev)
 		 */
 		skd_drive_fault(skdev);
 		/*start the queue so we can respond with error to requests */
-		pr_debug("%s:%s:%d starting %s queue\n",
-			 skdev->name, __func__, __LINE__, skdev->name);
-		blk_start_queue(skdev->queue);
+		dev_dbg(&skdev->pdev->dev, "starting queue\n");
+		schedule_work(&skdev->start_queue);
 		skdev->gendisk_on = -1;
 		wake_up_interruptible(&skdev->waitq);
 		break;
@@ -3483,38 +2134,33 @@ static void skd_start_device(struct skd_device *skdev)
 		 * to the BAR1 addresses. */
 		skd_drive_disappeared(skdev);
 		/*start the queue so we can respond with error to requests */
-		pr_debug("%s:%s:%d starting %s queue to error-out reqs\n",
-			 skdev->name, __func__, __LINE__, skdev->name);
-		blk_start_queue(skdev->queue);
+		dev_dbg(&skdev->pdev->dev,
+			"starting queue to error-out reqs\n");
+		schedule_work(&skdev->start_queue);
 		skdev->gendisk_on = -1;
 		wake_up_interruptible(&skdev->waitq);
 		break;
 
 	default:
-		pr_err("(%s) Start: unknown state %x\n",
-		       skd_name(skdev), skdev->drive_state);
+		dev_err(&skdev->pdev->dev, "Start: unknown state %x\n",
+			skdev->drive_state);
 		break;
 	}
 
 	state = SKD_READL(skdev, FIT_CONTROL);
-	pr_debug("%s:%s:%d FIT Control Status=0x%x\n",
-		 skdev->name, __func__, __LINE__, state);
+	dev_dbg(&skdev->pdev->dev, "FIT Control Status=0x%x\n", state);
 
 	state = SKD_READL(skdev, FIT_INT_STATUS_HOST);
-	pr_debug("%s:%s:%d Intr Status=0x%x\n",
-		 skdev->name, __func__, __LINE__, state);
+	dev_dbg(&skdev->pdev->dev, "Intr Status=0x%x\n", state);
 
 	state = SKD_READL(skdev, FIT_INT_MASK_HOST);
-	pr_debug("%s:%s:%d Intr Mask=0x%x\n",
-		 skdev->name, __func__, __LINE__, state);
+	dev_dbg(&skdev->pdev->dev, "Intr Mask=0x%x\n", state);
 
 	state = SKD_READL(skdev, FIT_MSG_FROM_DEVICE);
-	pr_debug("%s:%s:%d Msg from Dev=0x%x\n",
-		 skdev->name, __func__, __LINE__, state);
+	dev_dbg(&skdev->pdev->dev, "Msg from Dev=0x%x\n", state);
 
 	state = SKD_READL(skdev, FIT_HW_VERSION);
-	pr_debug("%s:%s:%d HW version=0x%x\n",
-		 skdev->name, __func__, __LINE__, state);
+	dev_dbg(&skdev->pdev->dev, "HW version=0x%x\n", state);
 
 	spin_unlock_irqrestore(&skdev->lock, flags);
 }
@@ -3529,14 +2175,12 @@ static void skd_stop_device(struct skd_device *skdev)
 	spin_lock_irqsave(&skdev->lock, flags);
 
 	if (skdev->state != SKD_DRVR_STATE_ONLINE) {
-		pr_err("(%s): skd_stop_device not online no sync\n",
-		       skd_name(skdev));
+		dev_err(&skdev->pdev->dev, "%s not online no sync\n", __func__);
 		goto stop_out;
 	}
 
 	if (skspcl->req.state != SKD_REQ_STATE_IDLE) {
-		pr_err("(%s): skd_stop_device no special\n",
-		       skd_name(skdev));
+		dev_err(&skdev->pdev->dev, "%s no special\n", __func__);
 		goto stop_out;
 	}
 
@@ -3554,16 +2198,13 @@ static void skd_stop_device(struct skd_device *skdev)
 
 	switch (skdev->sync_done) {
 	case 0:
-		pr_err("(%s): skd_stop_device no sync\n",
-		       skd_name(skdev));
+		dev_err(&skdev->pdev->dev, "%s no sync\n", __func__);
 		break;
 	case 1:
-		pr_err("(%s): skd_stop_device sync done\n",
-		       skd_name(skdev));
+		dev_err(&skdev->pdev->dev, "%s sync done\n", __func__);
 		break;
 	default:
-		pr_err("(%s): skd_stop_device sync error\n",
-		       skd_name(skdev));
+		dev_err(&skdev->pdev->dev, "%s sync error\n", __func__);
 	}
 
 stop_out:
@@ -3593,8 +2234,8 @@ stop_out:
 	}
 
 	if (dev_state != FIT_SR_DRIVE_INIT)
-		pr_err("(%s): skd_stop_device state error 0x%02x\n",
-		       skd_name(skdev), dev_state);
+		dev_err(&skdev->pdev->dev, "%s state error 0x%02x\n", __func__,
+			dev_state);
 }
 
 /* assume spinlock is held */
@@ -3607,8 +2248,7 @@ static void skd_restart_device(struct skd_device *skdev)
 
 	state = SKD_READL(skdev, FIT_STATUS);
 
-	pr_debug("%s:%s:%d drive status=0x%x\n",
-		 skdev->name, __func__, __LINE__, state);
+	dev_dbg(&skdev->pdev->dev, "drive status=0x%x\n", state);
 
 	state &= FIT_SR_DRIVE_STATE_MASK;
 	skdev->drive_state = state;
@@ -3628,9 +2268,8 @@ static int skd_quiesce_dev(struct skd_device *skdev)
 	switch (skdev->state) {
 	case SKD_DRVR_STATE_BUSY:
 	case SKD_DRVR_STATE_BUSY_IMMINENT:
-		pr_debug("%s:%s:%d stopping %s queue\n",
-			 skdev->name, __func__, __LINE__, skdev->name);
-		blk_stop_queue(skdev->queue);
+		dev_dbg(&skdev->pdev->dev, "stopping queue\n");
+		blk_mq_stop_hw_queues(skdev->queue);
 		break;
 	case SKD_DRVR_STATE_ONLINE:
 	case SKD_DRVR_STATE_STOPPING:
@@ -3642,8 +2281,8 @@ static int skd_quiesce_dev(struct skd_device *skdev)
 	case SKD_DRVR_STATE_RESUMING:
 	default:
 		rc = -EINVAL;
-		pr_debug("%s:%s:%d state [%d] not implemented\n",
-			 skdev->name, __func__, __LINE__, skdev->state);
+		dev_dbg(&skdev->pdev->dev, "state [%d] not implemented\n",
+			skdev->state);
 	}
 	return rc;
 }
@@ -3655,8 +2294,7 @@ static int skd_unquiesce_dev(struct skd_device *skdev)
 
 	skd_log_skdev(skdev, "unquiesce");
 	if (skdev->state == SKD_DRVR_STATE_ONLINE) {
-		pr_debug("%s:%s:%d **** device already ONLINE\n",
-			 skdev->name, __func__, __LINE__);
+		dev_dbg(&skdev->pdev->dev, "**** device already ONLINE\n");
 		return 0;
 	}
 	if (skdev->drive_state != FIT_SR_DRIVE_ONLINE) {
@@ -3669,8 +2307,7 @@ static int skd_unquiesce_dev(struct skd_device *skdev)
 		 * to become available.
 		 */
 		skdev->state = SKD_DRVR_STATE_BUSY;
-		pr_debug("%s:%s:%d drive BUSY state\n",
-			 skdev->name, __func__, __LINE__);
+		dev_dbg(&skdev->pdev->dev, "drive BUSY state\n");
 		return 0;
 	}
 
@@ -3689,26 +2326,24 @@ static int skd_unquiesce_dev(struct skd_device *skdev)
 	case SKD_DRVR_STATE_IDLE:
 	case SKD_DRVR_STATE_LOAD:
 		skdev->state = SKD_DRVR_STATE_ONLINE;
-		pr_err("(%s): Driver state %s(%d)=>%s(%d)\n",
-		       skd_name(skdev),
-		       skd_skdev_state_to_str(prev_driver_state),
-		       prev_driver_state, skd_skdev_state_to_str(skdev->state),
-		       skdev->state);
-		pr_debug("%s:%s:%d **** device ONLINE...starting block queue\n",
-			 skdev->name, __func__, __LINE__);
-		pr_debug("%s:%s:%d starting %s queue\n",
-			 skdev->name, __func__, __LINE__, skdev->name);
-		pr_info("(%s): STEC s1120 ONLINE\n", skd_name(skdev));
-		blk_start_queue(skdev->queue);
+		dev_err(&skdev->pdev->dev, "Driver state %s(%d)=>%s(%d)\n",
+			skd_skdev_state_to_str(prev_driver_state),
+			prev_driver_state, skd_skdev_state_to_str(skdev->state),
+			skdev->state);
+		dev_dbg(&skdev->pdev->dev,
+			"**** device ONLINE...starting block queue\n");
+		dev_dbg(&skdev->pdev->dev, "starting queue\n");
+		dev_info(&skdev->pdev->dev, "STEC s1120 ONLINE\n");
+		schedule_work(&skdev->start_queue);
 		skdev->gendisk_on = 1;
 		wake_up_interruptible(&skdev->waitq);
 		break;
 
 	case SKD_DRVR_STATE_DISAPPEARED:
 	default:
-		pr_debug("%s:%s:%d **** driver state %d, not implemented \n",
-			 skdev->name, __func__, __LINE__,
-			 skdev->state);
+		dev_dbg(&skdev->pdev->dev,
+			"**** driver state %d, not implemented\n",
+			skdev->state);
 		return -EBUSY;
 	}
 	return 0;
@@ -3726,11 +2361,10 @@ static irqreturn_t skd_reserved_isr(int irq, void *skd_host_data)
 	unsigned long flags;
 
 	spin_lock_irqsave(&skdev->lock, flags);
-	pr_debug("%s:%s:%d MSIX = 0x%x\n",
-		 skdev->name, __func__, __LINE__,
-		 SKD_READL(skdev, FIT_INT_STATUS_HOST));
-	pr_err("(%s): MSIX reserved irq %d = 0x%x\n", skd_name(skdev),
-	       irq, SKD_READL(skdev, FIT_INT_STATUS_HOST));
+	dev_dbg(&skdev->pdev->dev, "MSIX = 0x%x\n",
+		SKD_READL(skdev, FIT_INT_STATUS_HOST));
+	dev_err(&skdev->pdev->dev, "MSIX reserved irq %d = 0x%x\n", irq,
+		SKD_READL(skdev, FIT_INT_STATUS_HOST));
 	SKD_WRITEL(skdev, FIT_INT_RESERVED_MASK, FIT_INT_STATUS_HOST);
 	spin_unlock_irqrestore(&skdev->lock, flags);
 	return IRQ_HANDLED;
@@ -3742,9 +2376,8 @@ static irqreturn_t skd_statec_isr(int irq, void *skd_host_data)
 	unsigned long flags;
 
 	spin_lock_irqsave(&skdev->lock, flags);
-	pr_debug("%s:%s:%d MSIX = 0x%x\n",
-		 skdev->name, __func__, __LINE__,
-		 SKD_READL(skdev, FIT_INT_STATUS_HOST));
+	dev_dbg(&skdev->pdev->dev, "MSIX = 0x%x\n",
+		SKD_READL(skdev, FIT_INT_STATUS_HOST));
 	SKD_WRITEL(skdev, FIT_ISH_FW_STATE_CHANGE, FIT_INT_STATUS_HOST);
 	skd_isr_fwstate(skdev);
 	spin_unlock_irqrestore(&skdev->lock, flags);
@@ -3759,19 +2392,18 @@ static irqreturn_t skd_comp_q(int irq, void *skd_host_data)
 	int deferred;
 
 	spin_lock_irqsave(&skdev->lock, flags);
-	pr_debug("%s:%s:%d MSIX = 0x%x\n",
-		 skdev->name, __func__, __LINE__,
-		 SKD_READL(skdev, FIT_INT_STATUS_HOST));
+	dev_dbg(&skdev->pdev->dev, "MSIX = 0x%x\n",
+		SKD_READL(skdev, FIT_INT_STATUS_HOST));
 	SKD_WRITEL(skdev, FIT_ISH_COMPLETION_POSTED, FIT_INT_STATUS_HOST);
 	deferred = skd_isr_completion_posted(skdev, skd_isr_comp_limit,
 						&flush_enqueued);
 	if (flush_enqueued)
-		skd_request_fn(skdev->queue);
+		schedule_work(&skdev->start_queue);
 
 	if (deferred)
 		schedule_work(&skdev->completion_worker);
 	else if (!flush_enqueued)
-		skd_request_fn(skdev->queue);
+		schedule_work(&skdev->start_queue);
 
 	spin_unlock_irqrestore(&skdev->lock, flags);
 
@@ -3784,9 +2416,8 @@ static irqreturn_t skd_msg_isr(int irq, void *skd_host_data)
 	unsigned long flags;
 
 	spin_lock_irqsave(&skdev->lock, flags);
-	pr_debug("%s:%s:%d MSIX = 0x%x\n",
-		 skdev->name, __func__, __LINE__,
-		 SKD_READL(skdev, FIT_INT_STATUS_HOST));
+	dev_dbg(&skdev->pdev->dev, "MSIX = 0x%x\n",
+		SKD_READL(skdev, FIT_INT_STATUS_HOST));
 	SKD_WRITEL(skdev, FIT_ISH_MSG_FROM_DEV, FIT_INT_STATUS_HOST);
 	skd_isr_msg_from_dev(skdev);
 	spin_unlock_irqrestore(&skdev->lock, flags);
@@ -3799,9 +2430,8 @@ static irqreturn_t skd_qfull_isr(int irq, void *skd_host_data)
 	unsigned long flags;
 
 	spin_lock_irqsave(&skdev->lock, flags);
-	pr_debug("%s:%s:%d MSIX = 0x%x\n",
-		 skdev->name, __func__, __LINE__,
-		 SKD_READL(skdev, FIT_INT_STATUS_HOST));
+	dev_dbg(&skdev->pdev->dev, "MSIX = 0x%x\n",
+		SKD_READL(skdev, FIT_INT_STATUS_HOST));
 	SKD_WRITEL(skdev, FIT_INT_QUEUE_FULL, FIT_INT_STATUS_HOST);
 	spin_unlock_irqrestore(&skdev->lock, flags);
 	return IRQ_HANDLED;
@@ -3850,8 +2480,7 @@ static int skd_acquire_msix(struct skd_device *skdev)
 	rc = pci_alloc_irq_vectors(pdev, SKD_MAX_MSIX_COUNT, SKD_MAX_MSIX_COUNT,
 			PCI_IRQ_MSIX);
 	if (rc < 0) {
-		pr_err("(%s): failed to enable MSI-X %d\n",
-		       skd_name(skdev), rc);
+		dev_err(&skdev->pdev->dev, "failed to enable MSI-X %d\n", rc);
 		goto out;
 	}
 
@@ -3859,8 +2488,7 @@ static int skd_acquire_msix(struct skd_device *skdev)
 			sizeof(struct skd_msix_entry), GFP_KERNEL);
 	if (!skdev->msix_entries) {
 		rc = -ENOMEM;
-		pr_err("(%s): msix table allocation error\n",
-		       skd_name(skdev));
+		dev_err(&skdev->pdev->dev, "msix table allocation error\n");
 		goto out;
 	}
 
@@ -3877,16 +2505,15 @@ static int skd_acquire_msix(struct skd_device *skdev)
 				msix_entries[i].handler, 0,
 				qentry->isr_name, skdev);
 		if (rc) {
-			pr_err("(%s): Unable to register(%d) MSI-X "
-			       "handler %d: %s\n",
-			       skd_name(skdev), rc, i, qentry->isr_name);
+			dev_err(&skdev->pdev->dev,
+				"Unable to register(%d) MSI-X handler %d: %s\n",
+				rc, i, qentry->isr_name);
 			goto msix_out;
 		}
 	}
 
-	pr_debug("%s:%s:%d %s: <%s> msix %d irq(s) enabled\n",
-		 skdev->name, __func__, __LINE__,
-		 pci_name(pdev), skdev->name, SKD_MAX_MSIX_COUNT);
+	dev_dbg(&skdev->pdev->dev, "%d msix irq(s) enabled\n",
+		SKD_MAX_MSIX_COUNT);
 	return 0;
 
 msix_out:
@@ -3909,8 +2536,8 @@ static int skd_acquire_irq(struct skd_device *skdev)
 		if (!rc)
 			return 0;
 
-		pr_err("(%s): failed to enable MSI-X, re-trying with MSI %d\n",
-		       skd_name(skdev), rc);
+		dev_err(&skdev->pdev->dev,
+			"failed to enable MSI-X, re-trying with MSI %d\n", rc);
 	}
 
 	snprintf(skdev->isr_name, sizeof(skdev->isr_name), "%s%d", DRV_NAME,
@@ -3920,8 +2547,8 @@ static int skd_acquire_irq(struct skd_device *skdev)
 		irq_flag |= PCI_IRQ_MSI;
 	rc = pci_alloc_irq_vectors(pdev, 1, 1, irq_flag);
 	if (rc < 0) {
-		pr_err("(%s): failed to allocate the MSI interrupt %d\n",
-			skd_name(skdev), rc);
+		dev_err(&skdev->pdev->dev,
+			"failed to allocate the MSI interrupt %d\n", rc);
 		return rc;
 	}
 
@@ -3930,8 +2557,8 @@ static int skd_acquire_irq(struct skd_device *skdev)
 			skdev->isr_name, skdev);
 	if (rc) {
 		pci_free_irq_vectors(pdev);
-		pr_err("(%s): failed to allocate interrupt %d\n",
-			skd_name(skdev), rc);
+		dev_err(&skdev->pdev->dev, "failed to allocate interrupt %d\n",
+			rc);
 		return rc;
 	}
 
@@ -3965,20 +2592,45 @@ static void skd_release_irq(struct skd_device *skdev)
  *****************************************************************************
  */
 
+static void *skd_alloc_dma(struct skd_device *skdev, struct kmem_cache *s,
+			   dma_addr_t *dma_handle, gfp_t gfp,
+			   enum dma_data_direction dir)
+{
+	struct device *dev = &skdev->pdev->dev;
+	void *buf;
+
+	buf = kmem_cache_alloc(s, gfp);
+	if (!buf)
+		return NULL;
+	*dma_handle = dma_map_single(dev, buf, s->size, dir);
+	if (dma_mapping_error(dev, *dma_handle)) {
+		kfree(buf);
+		buf = NULL;
+	}
+	return buf;
+}
+
+static void skd_free_dma(struct skd_device *skdev, struct kmem_cache *s,
+			 void *vaddr, dma_addr_t dma_handle,
+			 enum dma_data_direction dir)
+{
+	if (!vaddr)
+		return;
+
+	dma_unmap_single(&skdev->pdev->dev, dma_handle, s->size, dir);
+	kmem_cache_free(s, vaddr);
+}
+
 static int skd_cons_skcomp(struct skd_device *skdev)
 {
 	int rc = 0;
 	struct fit_completion_entry_v1 *skcomp;
-	u32 nbytes;
 
-	nbytes = sizeof(*skcomp) * SKD_N_COMPLETION_ENTRY;
-	nbytes += sizeof(struct fit_comp_error_info) * SKD_N_COMPLETION_ENTRY;
+	dev_dbg(&skdev->pdev->dev,
+		"comp pci_alloc, total bytes %zd entries %d\n",
+		SKD_SKCOMP_SIZE, SKD_N_COMPLETION_ENTRY);
 
-	pr_debug("%s:%s:%d comp pci_alloc, total bytes %d entries %d\n",
-		 skdev->name, __func__, __LINE__,
-		 nbytes, SKD_N_COMPLETION_ENTRY);
-
-	skcomp = pci_zalloc_consistent(skdev->pdev, nbytes,
+	skcomp = pci_zalloc_consistent(skdev->pdev, SKD_SKCOMP_SIZE,
 				       &skdev->cq_dma_address);
 
 	if (skcomp == NULL) {
@@ -4000,14 +2652,14 @@ static int skd_cons_skmsg(struct skd_device *skdev)
 	int rc = 0;
 	u32 i;
 
-	pr_debug("%s:%s:%d skmsg_table kzalloc, struct %lu, count %u total %lu\n",
-		 skdev->name, __func__, __LINE__,
-		 sizeof(struct skd_fitmsg_context),
-		 skdev->num_fitmsg_context,
-		 sizeof(struct skd_fitmsg_context) * skdev->num_fitmsg_context);
+	dev_dbg(&skdev->pdev->dev,
+		"skmsg_table kcalloc, struct %lu, count %u total %lu\n",
+		sizeof(struct skd_fitmsg_context), skdev->num_fitmsg_context,
+		sizeof(struct skd_fitmsg_context) * skdev->num_fitmsg_context);
 
-	skdev->skmsg_table = kzalloc(sizeof(struct skd_fitmsg_context)
-				     *skdev->num_fitmsg_context, GFP_KERNEL);
+	skdev->skmsg_table = kcalloc(skdev->num_fitmsg_context,
+				     sizeof(struct skd_fitmsg_context),
+				     GFP_KERNEL);
 	if (skdev->skmsg_table == NULL) {
 		rc = -ENOMEM;
 		goto err_out;
@@ -4020,9 +2672,8 @@ static int skd_cons_skmsg(struct skd_device *skdev)
 
 		skmsg->id = i + SKD_ID_FIT_MSG;
 
-		skmsg->state = SKD_MSG_STATE_IDLE;
 		skmsg->msg_buf = pci_alloc_consistent(skdev->pdev,
-						      SKD_N_FITMSG_BYTES + 64,
+						      SKD_N_FITMSG_BYTES,
 						      &skmsg->mb_dma_address);
 
 		if (skmsg->msg_buf == NULL) {
@@ -4030,22 +2681,13 @@ static int skd_cons_skmsg(struct skd_device *skdev)
 			goto err_out;
 		}
 
-		skmsg->offset = (u32)((u64)skmsg->msg_buf &
-				      (~FIT_QCMD_BASE_ADDRESS_MASK));
-		skmsg->msg_buf += ~FIT_QCMD_BASE_ADDRESS_MASK;
-		skmsg->msg_buf = (u8 *)((u64)skmsg->msg_buf &
-				       FIT_QCMD_BASE_ADDRESS_MASK);
-		skmsg->mb_dma_address += ~FIT_QCMD_BASE_ADDRESS_MASK;
-		skmsg->mb_dma_address &= FIT_QCMD_BASE_ADDRESS_MASK;
+		WARN(((uintptr_t)skmsg->msg_buf | skmsg->mb_dma_address) &
+		     (FIT_QCMD_ALIGN - 1),
+		     "not aligned: msg_buf %p mb_dma_address %#llx\n",
+		     skmsg->msg_buf, skmsg->mb_dma_address);
 		memset(skmsg->msg_buf, 0, SKD_N_FITMSG_BYTES);
-
-		skmsg->next = &skmsg[1];
 	}
 
-	/* Free list is in order starting with the 0th entry. */
-	skdev->skmsg_table[i - 1].next = NULL;
-	skdev->skmsg_free_list = skdev->skmsg_table;
-
 err_out:
 	return rc;
 }
@@ -4055,18 +2697,14 @@ static struct fit_sg_descriptor *skd_cons_sg_list(struct skd_device *skdev,
 						  dma_addr_t *ret_dma_addr)
 {
 	struct fit_sg_descriptor *sg_list;
-	u32 nbytes;
-
-	nbytes = sizeof(*sg_list) * n_sg;
 
-	sg_list = pci_alloc_consistent(skdev->pdev, nbytes, ret_dma_addr);
+	sg_list = skd_alloc_dma(skdev, skdev->sglist_cache, ret_dma_addr,
+				GFP_DMA | __GFP_ZERO, DMA_TO_DEVICE);
 
 	if (sg_list != NULL) {
 		uint64_t dma_address = *ret_dma_addr;
 		u32 i;
 
-		memset(sg_list, 0, nbytes);
-
 		for (i = 0; i < n_sg - 1; i++) {
 			uint64_t ndp_off;
 			ndp_off = (i + 1) * sizeof(struct fit_sg_descriptor);
@@ -4079,153 +2717,63 @@ static struct fit_sg_descriptor *skd_cons_sg_list(struct skd_device *skdev,
 	return sg_list;
 }
 
-static int skd_cons_skreq(struct skd_device *skdev)
+static void skd_free_sg_list(struct skd_device *skdev,
+			     struct fit_sg_descriptor *sg_list,
+			     dma_addr_t dma_addr)
 {
-	int rc = 0;
-	u32 i;
-
-	pr_debug("%s:%s:%d skreq_table kzalloc, struct %lu, count %u total %lu\n",
-		 skdev->name, __func__, __LINE__,
-		 sizeof(struct skd_request_context),
-		 skdev->num_req_context,
-		 sizeof(struct skd_request_context) * skdev->num_req_context);
-
-	skdev->skreq_table = kzalloc(sizeof(struct skd_request_context)
-				     * skdev->num_req_context, GFP_KERNEL);
-	if (skdev->skreq_table == NULL) {
-		rc = -ENOMEM;
-		goto err_out;
-	}
-
-	pr_debug("%s:%s:%d alloc sg_table sg_per_req %u scatlist %lu total %lu\n",
-		 skdev->name, __func__, __LINE__,
-		 skdev->sgs_per_request, sizeof(struct scatterlist),
-		 skdev->sgs_per_request * sizeof(struct scatterlist));
-
-	for (i = 0; i < skdev->num_req_context; i++) {
-		struct skd_request_context *skreq;
-
-		skreq = &skdev->skreq_table[i];
-
-		skreq->id = i + SKD_ID_RW_REQUEST;
-		skreq->state = SKD_REQ_STATE_IDLE;
-
-		skreq->sg = kzalloc(sizeof(struct scatterlist) *
-				    skdev->sgs_per_request, GFP_KERNEL);
-		if (skreq->sg == NULL) {
-			rc = -ENOMEM;
-			goto err_out;
-		}
-		sg_init_table(skreq->sg, skdev->sgs_per_request);
-
-		skreq->sksg_list = skd_cons_sg_list(skdev,
-						    skdev->sgs_per_request,
-						    &skreq->sksg_dma_address);
-
-		if (skreq->sksg_list == NULL) {
-			rc = -ENOMEM;
-			goto err_out;
-		}
-
-		skreq->next = &skreq[1];
-	}
-
-	/* Free list is in order starting with the 0th entry. */
-	skdev->skreq_table[i - 1].next = NULL;
-	skdev->skreq_free_list = skdev->skreq_table;
+	if (WARN_ON_ONCE(!sg_list))
+		return;
 
-err_out:
-	return rc;
+	skd_free_dma(skdev, skdev->sglist_cache, sg_list, dma_addr,
+		     DMA_TO_DEVICE);
 }
 
-static int skd_cons_skspcl(struct skd_device *skdev)
+static int skd_init_request(struct blk_mq_tag_set *set, struct request *rq,
+			    unsigned int hctx_idx, unsigned int numa_node)
 {
-	int rc = 0;
-	u32 i, nbytes;
-
-	pr_debug("%s:%s:%d skspcl_table kzalloc, struct %lu, count %u total %lu\n",
-		 skdev->name, __func__, __LINE__,
-		 sizeof(struct skd_special_context),
-		 skdev->n_special,
-		 sizeof(struct skd_special_context) * skdev->n_special);
-
-	skdev->skspcl_table = kzalloc(sizeof(struct skd_special_context)
-				      * skdev->n_special, GFP_KERNEL);
-	if (skdev->skspcl_table == NULL) {
-		rc = -ENOMEM;
-		goto err_out;
-	}
+	struct skd_device *skdev = set->driver_data;
+	struct skd_request_context *skreq = blk_mq_rq_to_pdu(rq);
 
-	for (i = 0; i < skdev->n_special; i++) {
-		struct skd_special_context *skspcl;
-
-		skspcl = &skdev->skspcl_table[i];
-
-		skspcl->req.id = i + SKD_ID_SPECIAL_REQUEST;
-		skspcl->req.state = SKD_REQ_STATE_IDLE;
-
-		skspcl->req.next = &skspcl[1].req;
-
-		nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
-
-		skspcl->msg_buf =
-			pci_zalloc_consistent(skdev->pdev, nbytes,
-					      &skspcl->mb_dma_address);
-		if (skspcl->msg_buf == NULL) {
-			rc = -ENOMEM;
-			goto err_out;
-		}
-
-		skspcl->req.sg = kzalloc(sizeof(struct scatterlist) *
-					 SKD_N_SG_PER_SPECIAL, GFP_KERNEL);
-		if (skspcl->req.sg == NULL) {
-			rc = -ENOMEM;
-			goto err_out;
-		}
-
-		skspcl->req.sksg_list = skd_cons_sg_list(skdev,
-							 SKD_N_SG_PER_SPECIAL,
-							 &skspcl->req.
-							 sksg_dma_address);
-		if (skspcl->req.sksg_list == NULL) {
-			rc = -ENOMEM;
-			goto err_out;
-		}
-	}
+	skreq->state = SKD_REQ_STATE_IDLE;
+	skreq->sg = (void *)(skreq + 1);
+	sg_init_table(skreq->sg, skd_sgs_per_request);
+	skreq->sksg_list = skd_cons_sg_list(skdev, skd_sgs_per_request,
+					    &skreq->sksg_dma_address);
 
-	/* Free list is in order starting with the 0th entry. */
-	skdev->skspcl_table[i - 1].req.next = NULL;
-	skdev->skspcl_free_list = skdev->skspcl_table;
+	return skreq->sksg_list ? 0 : -ENOMEM;
+}
 
-	return rc;
+static void skd_exit_request(struct blk_mq_tag_set *set, struct request *rq,
+			     unsigned int hctx_idx)
+{
+	struct skd_device *skdev = set->driver_data;
+	struct skd_request_context *skreq = blk_mq_rq_to_pdu(rq);
 
-err_out:
-	return rc;
+	skd_free_sg_list(skdev, skreq->sksg_list, skreq->sksg_dma_address);
 }
 
 static int skd_cons_sksb(struct skd_device *skdev)
 {
 	int rc = 0;
 	struct skd_special_context *skspcl;
-	u32 nbytes;
 
 	skspcl = &skdev->internal_skspcl;
 
 	skspcl->req.id = 0 + SKD_ID_INTERNAL;
 	skspcl->req.state = SKD_REQ_STATE_IDLE;
 
-	nbytes = SKD_N_INTERNAL_BYTES;
-
-	skspcl->data_buf = pci_zalloc_consistent(skdev->pdev, nbytes,
-						 &skspcl->db_dma_address);
+	skspcl->data_buf = skd_alloc_dma(skdev, skdev->databuf_cache,
+					 &skspcl->db_dma_address,
+					 GFP_DMA | __GFP_ZERO,
+					 DMA_BIDIRECTIONAL);
 	if (skspcl->data_buf == NULL) {
 		rc = -ENOMEM;
 		goto err_out;
 	}
 
-	nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
-	skspcl->msg_buf = pci_zalloc_consistent(skdev->pdev, nbytes,
-						&skspcl->mb_dma_address);
+	skspcl->msg_buf = skd_alloc_dma(skdev, skdev->msgbuf_cache,
+					&skspcl->mb_dma_address,
+					GFP_DMA | __GFP_ZERO, DMA_TO_DEVICE);
 	if (skspcl->msg_buf == NULL) {
 		rc = -ENOMEM;
 		goto err_out;
@@ -4247,6 +2795,14 @@ err_out:
 	return rc;
 }
 
+static const struct blk_mq_ops skd_mq_ops = {
+	.queue_rq	= skd_mq_queue_rq,
+	.complete	= skd_complete_rq,
+	.timeout	= skd_timed_out,
+	.init_request	= skd_init_request,
+	.exit_request	= skd_exit_request,
+};
+
 static int skd_cons_disk(struct skd_device *skdev)
 {
 	int rc = 0;
@@ -4268,31 +2824,46 @@ static int skd_cons_disk(struct skd_device *skdev)
 	disk->fops = &skd_blockdev_ops;
 	disk->private_data = skdev;
 
-	q = blk_init_queue(skd_request_fn, &skdev->lock);
-	if (!q) {
-		rc = -ENOMEM;
+	memset(&skdev->tag_set, 0, sizeof(skdev->tag_set));
+	skdev->tag_set.ops = &skd_mq_ops;
+	skdev->tag_set.nr_hw_queues = 1;
+	skdev->tag_set.queue_depth = skd_max_queue_depth;
+	skdev->tag_set.cmd_size = sizeof(struct skd_request_context) +
+		skdev->sgs_per_request * sizeof(struct scatterlist);
+	skdev->tag_set.numa_node = NUMA_NO_NODE;
+	skdev->tag_set.flags = BLK_MQ_F_SHOULD_MERGE |
+		BLK_MQ_F_SG_MERGE |
+		BLK_ALLOC_POLICY_TO_MQ_FLAG(BLK_TAG_ALLOC_FIFO);
+	skdev->tag_set.driver_data = skdev;
+	rc = blk_mq_alloc_tag_set(&skdev->tag_set);
+	if (rc)
+		goto err_out;
+	q = blk_mq_init_queue(&skdev->tag_set);
+	if (IS_ERR(q)) {
+		blk_mq_free_tag_set(&skdev->tag_set);
+		rc = PTR_ERR(q);
 		goto err_out;
 	}
-	blk_queue_bounce_limit(q, BLK_BOUNCE_HIGH);
+	q->queuedata = skdev;
 
 	skdev->queue = q;
 	disk->queue = q;
-	q->queuedata = skdev;
 
 	blk_queue_write_cache(q, true, true);
 	blk_queue_max_segments(q, skdev->sgs_per_request);
 	blk_queue_max_hw_sectors(q, SKD_N_MAX_SECTORS);
 
-	/* set sysfs ptimal_io_size to 8K */
+	/* set optimal I/O size to 8KB */
 	blk_queue_io_opt(q, 8192);
 
 	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q);
 	queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, q);
 
+	blk_queue_rq_timeout(q, 8 * HZ);
+
 	spin_lock_irqsave(&skdev->lock, flags);
-	pr_debug("%s:%s:%d stopping %s queue\n",
-		 skdev->name, __func__, __LINE__, skdev->name);
-	blk_stop_queue(skdev->queue);
+	dev_dbg(&skdev->pdev->dev, "stopping queue\n");
+	blk_mq_stop_hw_queues(skdev->queue);
 	spin_unlock_irqrestore(&skdev->lock, flags);
 
 err_out:
@@ -4306,13 +2877,13 @@ static struct skd_device *skd_construct(struct pci_dev *pdev)
 {
 	struct skd_device *skdev;
 	int blk_major = skd_major;
+	size_t size;
 	int rc;
 
 	skdev = kzalloc(sizeof(*skdev), GFP_KERNEL);
 
 	if (!skdev) {
-		pr_err(PFX "(%s): memory alloc failure\n",
-		       pci_name(pdev));
+		dev_err(&pdev->dev, "memory alloc failure\n");
 		return NULL;
 	}
 
@@ -4320,60 +2891,71 @@ static struct skd_device *skd_construct(struct pci_dev *pdev)
 	skdev->pdev = pdev;
 	skdev->devno = skd_next_devno++;
 	skdev->major = blk_major;
-	sprintf(skdev->name, DRV_NAME "%d", skdev->devno);
 	skdev->dev_max_queue_depth = 0;
 
 	skdev->num_req_context = skd_max_queue_depth;
 	skdev->num_fitmsg_context = skd_max_queue_depth;
-	skdev->n_special = skd_max_pass_thru;
 	skdev->cur_max_queue_depth = 1;
 	skdev->queue_low_water_mark = 1;
 	skdev->proto_ver = 99;
 	skdev->sgs_per_request = skd_sgs_per_request;
 	skdev->dbg_level = skd_dbg_level;
 
-	atomic_set(&skdev->device_count, 0);
-
 	spin_lock_init(&skdev->lock);
 
+	INIT_WORK(&skdev->start_queue, skd_start_queue);
 	INIT_WORK(&skdev->completion_worker, skd_completion_worker);
 
-	pr_debug("%s:%s:%d skcomp\n", skdev->name, __func__, __LINE__);
-	rc = skd_cons_skcomp(skdev);
-	if (rc < 0)
+	size = max(SKD_N_FITMSG_BYTES, SKD_N_SPECIAL_FITMSG_BYTES);
+	skdev->msgbuf_cache = kmem_cache_create("skd-msgbuf", size, 0,
+						SLAB_HWCACHE_ALIGN, NULL);
+	if (!skdev->msgbuf_cache)
 		goto err_out;
-
-	pr_debug("%s:%s:%d skmsg\n", skdev->name, __func__, __LINE__);
-	rc = skd_cons_skmsg(skdev);
-	if (rc < 0)
+	WARN_ONCE(kmem_cache_size(skdev->msgbuf_cache) < size,
+		  "skd-msgbuf: %d < %zd\n",
+		  kmem_cache_size(skdev->msgbuf_cache), size);
+	size = skd_sgs_per_request * sizeof(struct fit_sg_descriptor);
+	skdev->sglist_cache = kmem_cache_create("skd-sglist", size, 0,
+						SLAB_HWCACHE_ALIGN, NULL);
+	if (!skdev->sglist_cache)
+		goto err_out;
+	WARN_ONCE(kmem_cache_size(skdev->sglist_cache) < size,
+		  "skd-sglist: %d < %zd\n",
+		  kmem_cache_size(skdev->sglist_cache), size);
+	size = SKD_N_INTERNAL_BYTES;
+	skdev->databuf_cache = kmem_cache_create("skd-databuf", size, 0,
+						 SLAB_HWCACHE_ALIGN, NULL);
+	if (!skdev->databuf_cache)
 		goto err_out;
+	WARN_ONCE(kmem_cache_size(skdev->databuf_cache) < size,
+		  "skd-databuf: %d < %zd\n",
+		  kmem_cache_size(skdev->databuf_cache), size);
 
-	pr_debug("%s:%s:%d skreq\n", skdev->name, __func__, __LINE__);
-	rc = skd_cons_skreq(skdev);
+	dev_dbg(&skdev->pdev->dev, "skcomp\n");
+	rc = skd_cons_skcomp(skdev);
 	if (rc < 0)
 		goto err_out;
 
-	pr_debug("%s:%s:%d skspcl\n", skdev->name, __func__, __LINE__);
-	rc = skd_cons_skspcl(skdev);
+	dev_dbg(&skdev->pdev->dev, "skmsg\n");
+	rc = skd_cons_skmsg(skdev);
 	if (rc < 0)
 		goto err_out;
 
-	pr_debug("%s:%s:%d sksb\n", skdev->name, __func__, __LINE__);
+	dev_dbg(&skdev->pdev->dev, "sksb\n");
 	rc = skd_cons_sksb(skdev);
 	if (rc < 0)
 		goto err_out;
 
-	pr_debug("%s:%s:%d disk\n", skdev->name, __func__, __LINE__);
+	dev_dbg(&skdev->pdev->dev, "disk\n");
 	rc = skd_cons_disk(skdev);
 	if (rc < 0)
 		goto err_out;
 
-	pr_debug("%s:%s:%d VICTORY\n", skdev->name, __func__, __LINE__);
+	dev_dbg(&skdev->pdev->dev, "VICTORY\n");
 	return skdev;
 
 err_out:
-	pr_debug("%s:%s:%d construct failed\n",
-		 skdev->name, __func__, __LINE__);
+	dev_dbg(&skdev->pdev->dev, "construct failed\n");
 	skd_destruct(skdev);
 	return NULL;
 }
@@ -4386,14 +2968,9 @@ err_out:
 
 static void skd_free_skcomp(struct skd_device *skdev)
 {
-	if (skdev->skcomp_table != NULL) {
-		u32 nbytes;
-
-		nbytes = sizeof(skdev->skcomp_table[0]) *
-			 SKD_N_COMPLETION_ENTRY;
-		pci_free_consistent(skdev->pdev, nbytes,
+	if (skdev->skcomp_table)
+		pci_free_consistent(skdev->pdev, SKD_SKCOMP_SIZE,
 				    skdev->skcomp_table, skdev->cq_dma_address);
-	}
 
 	skdev->skcomp_table = NULL;
 	skdev->cq_dma_address = 0;
@@ -4412,8 +2989,6 @@ static void skd_free_skmsg(struct skd_device *skdev)
 		skmsg = &skdev->skmsg_table[i];
 
 		if (skmsg->msg_buf != NULL) {
-			skmsg->msg_buf += skmsg->offset;
-			skmsg->mb_dma_address += skmsg->offset;
 			pci_free_consistent(skdev->pdev, SKD_N_FITMSG_BYTES,
 					    skmsg->msg_buf,
 					    skmsg->mb_dma_address);
@@ -4426,109 +3001,23 @@ static void skd_free_skmsg(struct skd_device *skdev)
 	skdev->skmsg_table = NULL;
 }
 
-static void skd_free_sg_list(struct skd_device *skdev,
-			     struct fit_sg_descriptor *sg_list,
-			     u32 n_sg, dma_addr_t dma_addr)
-{
-	if (sg_list != NULL) {
-		u32 nbytes;
-
-		nbytes = sizeof(*sg_list) * n_sg;
-
-		pci_free_consistent(skdev->pdev, nbytes, sg_list, dma_addr);
-	}
-}
-
-static void skd_free_skreq(struct skd_device *skdev)
-{
-	u32 i;
-
-	if (skdev->skreq_table == NULL)
-		return;
-
-	for (i = 0; i < skdev->num_req_context; i++) {
-		struct skd_request_context *skreq;
-
-		skreq = &skdev->skreq_table[i];
-
-		skd_free_sg_list(skdev, skreq->sksg_list,
-				 skdev->sgs_per_request,
-				 skreq->sksg_dma_address);
-
-		skreq->sksg_list = NULL;
-		skreq->sksg_dma_address = 0;
-
-		kfree(skreq->sg);
-	}
-
-	kfree(skdev->skreq_table);
-	skdev->skreq_table = NULL;
-}
-
-static void skd_free_skspcl(struct skd_device *skdev)
-{
-	u32 i;
-	u32 nbytes;
-
-	if (skdev->skspcl_table == NULL)
-		return;
-
-	for (i = 0; i < skdev->n_special; i++) {
-		struct skd_special_context *skspcl;
-
-		skspcl = &skdev->skspcl_table[i];
-
-		if (skspcl->msg_buf != NULL) {
-			nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
-			pci_free_consistent(skdev->pdev, nbytes,
-					    skspcl->msg_buf,
-					    skspcl->mb_dma_address);
-		}
-
-		skspcl->msg_buf = NULL;
-		skspcl->mb_dma_address = 0;
-
-		skd_free_sg_list(skdev, skspcl->req.sksg_list,
-				 SKD_N_SG_PER_SPECIAL,
-				 skspcl->req.sksg_dma_address);
-
-		skspcl->req.sksg_list = NULL;
-		skspcl->req.sksg_dma_address = 0;
-
-		kfree(skspcl->req.sg);
-	}
-
-	kfree(skdev->skspcl_table);
-	skdev->skspcl_table = NULL;
-}
-
 static void skd_free_sksb(struct skd_device *skdev)
 {
-	struct skd_special_context *skspcl;
-	u32 nbytes;
-
-	skspcl = &skdev->internal_skspcl;
-
-	if (skspcl->data_buf != NULL) {
-		nbytes = SKD_N_INTERNAL_BYTES;
+	struct skd_special_context *skspcl = &skdev->internal_skspcl;
 
-		pci_free_consistent(skdev->pdev, nbytes,
-				    skspcl->data_buf, skspcl->db_dma_address);
-	}
+	skd_free_dma(skdev, skdev->databuf_cache, skspcl->data_buf,
+		     skspcl->db_dma_address, DMA_BIDIRECTIONAL);
 
 	skspcl->data_buf = NULL;
 	skspcl->db_dma_address = 0;
 
-	if (skspcl->msg_buf != NULL) {
-		nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
-		pci_free_consistent(skdev->pdev, nbytes,
-				    skspcl->msg_buf, skspcl->mb_dma_address);
-	}
+	skd_free_dma(skdev, skdev->msgbuf_cache, skspcl->msg_buf,
+		     skspcl->mb_dma_address, DMA_TO_DEVICE);
 
 	skspcl->msg_buf = NULL;
 	skspcl->mb_dma_address = 0;
 
-	skd_free_sg_list(skdev, skspcl->req.sksg_list, 1,
+	skd_free_sg_list(skdev, skspcl->req.sksg_list,
 			 skspcl->req.sksg_dma_address);
 
 	skspcl->req.sksg_list = NULL;
@@ -4539,15 +3028,20 @@ static void skd_free_disk(struct skd_device *skdev)
 {
 	struct gendisk *disk = skdev->disk;
 
-	if (disk != NULL) {
-		struct request_queue *q = disk->queue;
+	if (disk && (disk->flags & GENHD_FL_UP))
+		del_gendisk(disk);
 
-		if (disk->flags & GENHD_FL_UP)
-			del_gendisk(disk);
-		if (q)
-			blk_cleanup_queue(q);
-		put_disk(disk);
+	if (skdev->queue) {
+		blk_cleanup_queue(skdev->queue);
+		skdev->queue = NULL;
+		if (disk)
+			disk->queue = NULL;
 	}
+
+	if (skdev->tag_set.tags)
+		blk_mq_free_tag_set(&skdev->tag_set);
+
+	put_disk(disk);
 	skdev->disk = NULL;
 }
 
@@ -4556,26 +3050,25 @@ static void skd_destruct(struct skd_device *skdev)
 	if (skdev == NULL)
 		return;
 
+	cancel_work_sync(&skdev->start_queue);
 
-	pr_debug("%s:%s:%d disk\n", skdev->name, __func__, __LINE__);
+	dev_dbg(&skdev->pdev->dev, "disk\n");
 	skd_free_disk(skdev);
 
-	pr_debug("%s:%s:%d sksb\n", skdev->name, __func__, __LINE__);
+	dev_dbg(&skdev->pdev->dev, "sksb\n");
 	skd_free_sksb(skdev);
 
-	pr_debug("%s:%s:%d skspcl\n", skdev->name, __func__, __LINE__);
-	skd_free_skspcl(skdev);
-
-	pr_debug("%s:%s:%d skreq\n", skdev->name, __func__, __LINE__);
-	skd_free_skreq(skdev);
-
-	pr_debug("%s:%s:%d skmsg\n", skdev->name, __func__, __LINE__);
+	dev_dbg(&skdev->pdev->dev, "skmsg\n");
 	skd_free_skmsg(skdev);
 
-	pr_debug("%s:%s:%d skcomp\n", skdev->name, __func__, __LINE__);
+	dev_dbg(&skdev->pdev->dev, "skcomp\n");
 	skd_free_skcomp(skdev);
 
-	pr_debug("%s:%s:%d skdev\n", skdev->name, __func__, __LINE__);
+	kmem_cache_destroy(skdev->databuf_cache);
+	kmem_cache_destroy(skdev->sglist_cache);
+	kmem_cache_destroy(skdev->msgbuf_cache);
+
+	dev_dbg(&skdev->pdev->dev, "skdev\n");
 	kfree(skdev);
 }
 
@@ -4592,9 +3085,8 @@ static int skd_bdev_getgeo(struct block_device *bdev, struct hd_geometry *geo)
 
 	skdev = bdev->bd_disk->private_data;
 
-	pr_debug("%s:%s:%d %s: CMD[%s] getgeo device\n",
-		 skdev->name, __func__, __LINE__,
-		 bdev->bd_disk->disk_name, current->comm);
+	dev_dbg(&skdev->pdev->dev, "%s: CMD[%s] getgeo device\n",
+		bdev->bd_disk->disk_name, current->comm);
 
 	if (skdev->read_cap_is_valid) {
 		capacity = get_capacity(skdev->disk);
@@ -4609,18 +3101,16 @@ static int skd_bdev_getgeo(struct block_device *bdev, struct hd_geometry *geo)
 
 static int skd_bdev_attach(struct device *parent, struct skd_device *skdev)
 {
-	pr_debug("%s:%s:%d add_disk\n", skdev->name, __func__, __LINE__);
+	dev_dbg(&skdev->pdev->dev, "add_disk\n");
 	device_add_disk(parent, skdev->disk);
 	return 0;
 }
 
 static const struct block_device_operations skd_blockdev_ops = {
 	.owner		= THIS_MODULE,
-	.ioctl		= skd_bdev_ioctl,
 	.getgeo		= skd_bdev_getgeo,
 };
 
-
 /*
  *****************************************************************************
  * PCIe DRIVER GLUE
@@ -4671,10 +3161,8 @@ static int skd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	char pci_str[32];
 	struct skd_device *skdev;
 
-	pr_info("STEC s1120 Driver(%s) version %s-b%s\n",
-	       DRV_NAME, DRV_VERSION, DRV_BUILD_ID);
-	pr_info("(skd?:??:[%s]): vendor=%04X device=%04x\n",
-	       pci_name(pdev), pdev->vendor, pdev->device);
+	dev_dbg(&pdev->dev, "vendor=%04X device=%04x\n", pdev->vendor,
+		pdev->device);
 
 	rc = pci_enable_device(pdev);
 	if (rc)
@@ -4685,16 +3173,13 @@ static int skd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
 	if (!rc) {
 		if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
-
-			pr_err("(%s): consistent DMA mask error %d\n",
-			       pci_name(pdev), rc);
+			dev_err(&pdev->dev, "consistent DMA mask error %d\n",
+				rc);
 		}
 	} else {
-		(rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)));
+		rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
 		if (rc) {
-
-			pr_err("(%s): DMA mask error %d\n",
-			       pci_name(pdev), rc);
+			dev_err(&pdev->dev, "DMA mask error %d\n", rc);
 			goto err_out_regions;
 		}
 	}
@@ -4714,19 +3199,17 @@ static int skd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	}
 
 	skd_pci_info(skdev, pci_str);
-	pr_info("(%s): %s 64bit\n", skd_name(skdev), pci_str);
+	dev_info(&pdev->dev, "%s 64bit\n", pci_str);
 
 	pci_set_master(pdev);
 	rc = pci_enable_pcie_error_reporting(pdev);
 	if (rc) {
-		pr_err(
-		       "(%s): bad enable of PCIe error reporting rc=%d\n",
-		       skd_name(skdev), rc);
+		dev_err(&pdev->dev,
+			"bad enable of PCIe error reporting rc=%d\n", rc);
 		skdev->pcie_error_reporting_is_enabled = 0;
 	} else
 		skdev->pcie_error_reporting_is_enabled = 1;
 
-
 	pci_set_drvdata(pdev, skdev);
 
 	for (i = 0; i < SKD_MAX_BARS; i++) {
@@ -4735,21 +3218,19 @@ static int skd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		skdev->mem_map[i] = ioremap(skdev->mem_phys[i],
 					    skdev->mem_size[i]);
 		if (!skdev->mem_map[i]) {
-			pr_err("(%s): Unable to map adapter memory!\n",
-			       skd_name(skdev));
+			dev_err(&pdev->dev,
+				"Unable to map adapter memory!\n");
 			rc = -ENODEV;
 			goto err_out_iounmap;
 		}
-		pr_debug("%s:%s:%d mem_map=%p, phyd=%016llx, size=%d\n",
-			 skdev->name, __func__, __LINE__,
-			 skdev->mem_map[i],
-			 (uint64_t)skdev->mem_phys[i], skdev->mem_size[i]);
+		dev_dbg(&pdev->dev, "mem_map=%p, phyd=%016llx, size=%d\n",
+			skdev->mem_map[i], (uint64_t)skdev->mem_phys[i],
+			skdev->mem_size[i]);
 	}
 
 	rc = skd_acquire_irq(skdev);
 	if (rc) {
-		pr_err("(%s): interrupt resource error %d\n",
-		       skd_name(skdev), rc);
+		dev_err(&pdev->dev, "interrupt resource error %d\n", rc);
 		goto err_out_iounmap;
 	}
 
@@ -4771,29 +3252,14 @@ static int skd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	} else {
 		/* we timed out, something is wrong with the device,
 		   don't add the disk structure */
-		pr_err(
-		       "(%s): error: waiting for s1120 timed out %d!\n",
-		       skd_name(skdev), rc);
+		dev_err(&pdev->dev, "error: waiting for s1120 timed out %d!\n",
+			rc);
 		/* in case of no error; we timeout with ENXIO */
 		if (!rc)
 			rc = -ENXIO;
 		goto err_out_timer;
 	}
 
-
-#ifdef SKD_VMK_POLL_HANDLER
-	if (skdev->irq_type == SKD_IRQ_MSIX) {
-		/* MSIX completion handler is being used for coredump */
-		vmklnx_scsi_register_poll_handler(skdev->scsi_host,
-						  skdev->msix_entries[5].vector,
-						  skd_comp_q, skdev);
-	} else {
-		vmklnx_scsi_register_poll_handler(skdev->scsi_host,
-						  skdev->pdev->irq, skd_isr,
-						  skdev);
-	}
-#endif  /* SKD_VMK_POLL_HANDLER */
-
 	return rc;
 
 err_out_timer:
@@ -4826,7 +3292,7 @@ static void skd_pci_remove(struct pci_dev *pdev)
 
 	skdev = pci_get_drvdata(pdev);
 	if (!skdev) {
-		pr_err("%s: no device data for PCI\n", pci_name(pdev));
+		dev_err(&pdev->dev, "no device data for PCI\n");
 		return;
 	}
 	skd_stop_device(skdev);
@@ -4834,7 +3300,7 @@ static void skd_pci_remove(struct pci_dev *pdev)
 
 	for (i = 0; i < SKD_MAX_BARS; i++)
 		if (skdev->mem_map[i])
-			iounmap((u32 *)skdev->mem_map[i]);
+			iounmap(skdev->mem_map[i]);
 
 	if (skdev->pcie_error_reporting_is_enabled)
 		pci_disable_pcie_error_reporting(pdev);
@@ -4855,7 +3321,7 @@ static int skd_pci_suspend(struct pci_dev *pdev, pm_message_t state)
 
 	skdev = pci_get_drvdata(pdev);
 	if (!skdev) {
-		pr_err("%s: no device data for PCI\n", pci_name(pdev));
+		dev_err(&pdev->dev, "no device data for PCI\n");
 		return -EIO;
 	}
 
@@ -4865,7 +3331,7 @@ static int skd_pci_suspend(struct pci_dev *pdev, pm_message_t state)
 
 	for (i = 0; i < SKD_MAX_BARS; i++)
 		if (skdev->mem_map[i])
-			iounmap((u32 *)skdev->mem_map[i]);
+			iounmap(skdev->mem_map[i]);
 
 	if (skdev->pcie_error_reporting_is_enabled)
 		pci_disable_pcie_error_reporting(pdev);
@@ -4885,7 +3351,7 @@ static int skd_pci_resume(struct pci_dev *pdev)
 
 	skdev = pci_get_drvdata(pdev);
 	if (!skdev) {
-		pr_err("%s: no device data for PCI\n", pci_name(pdev));
+		dev_err(&pdev->dev, "no device data for PCI\n");
 		return -1;
 	}
 
@@ -4903,15 +3369,14 @@ static int skd_pci_resume(struct pci_dev *pdev)
 	if (!rc) {
 		if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
 
-			pr_err("(%s): consistent DMA mask error %d\n",
-			       pci_name(pdev), rc);
+			dev_err(&pdev->dev, "consistent DMA mask error %d\n",
+				rc);
 		}
 	} else {
 		rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
 		if (rc) {
 
-			pr_err("(%s): DMA mask error %d\n",
-			       pci_name(pdev), rc);
+			dev_err(&pdev->dev, "DMA mask error %d\n", rc);
 			goto err_out_regions;
 		}
 	}
@@ -4919,8 +3384,8 @@ static int skd_pci_resume(struct pci_dev *pdev)
 	pci_set_master(pdev);
 	rc = pci_enable_pcie_error_reporting(pdev);
 	if (rc) {
-		pr_err("(%s): bad enable of PCIe error reporting rc=%d\n",
-		       skdev->name, rc);
+		dev_err(&pdev->dev,
+			"bad enable of PCIe error reporting rc=%d\n", rc);
 		skdev->pcie_error_reporting_is_enabled = 0;
 	} else
 		skdev->pcie_error_reporting_is_enabled = 1;
@@ -4932,21 +3397,17 @@ static int skd_pci_resume(struct pci_dev *pdev)
 		skdev->mem_map[i] = ioremap(skdev->mem_phys[i],
 					    skdev->mem_size[i]);
 		if (!skdev->mem_map[i]) {
-			pr_err("(%s): Unable to map adapter memory!\n",
-			       skd_name(skdev));
+			dev_err(&pdev->dev, "Unable to map adapter memory!\n");
 			rc = -ENODEV;
 			goto err_out_iounmap;
 		}
-		pr_debug("%s:%s:%d mem_map=%p, phyd=%016llx, size=%d\n",
-			 skdev->name, __func__, __LINE__,
-			 skdev->mem_map[i],
-			 (uint64_t)skdev->mem_phys[i], skdev->mem_size[i]);
+		dev_dbg(&pdev->dev, "mem_map=%p, phyd=%016llx, size=%d\n",
+			skdev->mem_map[i], (uint64_t)skdev->mem_phys[i],
+			skdev->mem_size[i]);
 	}
 	rc = skd_acquire_irq(skdev);
 	if (rc) {
-
-		pr_err("(%s): interrupt resource error %d\n",
-		       pci_name(pdev), rc);
+		dev_err(&pdev->dev, "interrupt resource error %d\n", rc);
 		goto err_out_iounmap;
 	}
 
@@ -4984,15 +3445,15 @@ static void skd_pci_shutdown(struct pci_dev *pdev)
 {
 	struct skd_device *skdev;
 
-	pr_err("skd_pci_shutdown called\n");
+	dev_err(&pdev->dev, "%s called\n", __func__);
 
 	skdev = pci_get_drvdata(pdev);
 	if (!skdev) {
-		pr_err("%s: no device data for PCI\n", pci_name(pdev));
+		dev_err(&pdev->dev, "no device data for PCI\n");
 		return;
 	}
 
-	pr_err("%s: calling stop\n", skd_name(skdev));
+	dev_err(&pdev->dev, "calling stop\n");
 	skd_stop_device(skdev);
 }
 
@@ -5012,21 +3473,6 @@ static struct pci_driver skd_driver = {
  *****************************************************************************
  */
 
-static const char *skd_name(struct skd_device *skdev)
-{
-	memset(skdev->id_str, 0, sizeof(skdev->id_str));
-
-	if (skdev->inquiry_is_valid)
-		snprintf(skdev->id_str, sizeof(skdev->id_str), "%s:%s:[%s]",
-			 skdev->name, skdev->inq_serial_num,
-			 pci_name(skdev->pdev));
-	else
-		snprintf(skdev->id_str, sizeof(skdev->id_str), "%s:??:[%s]",
-			 skdev->name, pci_name(skdev->pdev));
-
-	return skdev->id_str;
-}
-
 const char *skd_drive_state_to_str(int state)
 {
 	switch (state) {
@@ -5078,8 +3524,6 @@ const char *skd_skdev_state_to_str(enum skd_drvr_state state)
 		return "PAUSING";
 	case SKD_DRVR_STATE_PAUSED:
 		return "PAUSED";
-	case SKD_DRVR_STATE_DRAINING_TIMEOUT:
-		return "DRAINING_TIMEOUT";
 	case SKD_DRVR_STATE_RESTARTING:
 		return "RESTARTING";
 	case SKD_DRVR_STATE_RESUMING:
@@ -5106,18 +3550,6 @@ const char *skd_skdev_state_to_str(enum skd_drvr_state state)
 	}
 }
 
-static const char *skd_skmsg_state_to_str(enum skd_fit_msg_state state)
-{
-	switch (state) {
-	case SKD_MSG_STATE_IDLE:
-		return "IDLE";
-	case SKD_MSG_STATE_BUSY:
-		return "BUSY";
-	default:
-		return "???";
-	}
-}
-
 static const char *skd_skreq_state_to_str(enum skd_req_state state)
 {
 	switch (state) {
@@ -5131,8 +3563,6 @@ static const char *skd_skreq_state_to_str(enum skd_req_state state)
 		return "COMPLETED";
 	case SKD_REQ_STATE_TIMEOUT:
 		return "TIMEOUT";
-	case SKD_REQ_STATE_ABORTED:
-		return "ABORTED";
 	default:
 		return "???";
 	}
@@ -5140,58 +3570,34 @@ static const char *skd_skreq_state_to_str(enum skd_req_state state)
 
 static void skd_log_skdev(struct skd_device *skdev, const char *event)
 {
-	pr_debug("%s:%s:%d (%s) skdev=%p event='%s'\n",
-		 skdev->name, __func__, __LINE__, skdev->name, skdev, event);
-	pr_debug("%s:%s:%d   drive_state=%s(%d) driver_state=%s(%d)\n",
-		 skdev->name, __func__, __LINE__,
-		 skd_drive_state_to_str(skdev->drive_state), skdev->drive_state,
-		 skd_skdev_state_to_str(skdev->state), skdev->state);
-	pr_debug("%s:%s:%d   busy=%d limit=%d dev=%d lowat=%d\n",
-		 skdev->name, __func__, __LINE__,
-		 skdev->in_flight, skdev->cur_max_queue_depth,
-		 skdev->dev_max_queue_depth, skdev->queue_low_water_mark);
-	pr_debug("%s:%s:%d   timestamp=0x%x cycle=%d cycle_ix=%d\n",
-		 skdev->name, __func__, __LINE__,
-		 skdev->timeout_stamp, skdev->skcomp_cycle, skdev->skcomp_ix);
-}
-
-static void skd_log_skmsg(struct skd_device *skdev,
-			  struct skd_fitmsg_context *skmsg, const char *event)
-{
-	pr_debug("%s:%s:%d (%s) skmsg=%p event='%s'\n",
-		 skdev->name, __func__, __LINE__, skdev->name, skmsg, event);
-	pr_debug("%s:%s:%d   state=%s(%d) id=0x%04x length=%d\n",
-		 skdev->name, __func__, __LINE__,
-		 skd_skmsg_state_to_str(skmsg->state), skmsg->state,
-		 skmsg->id, skmsg->length);
+	dev_dbg(&skdev->pdev->dev, "skdev=%p event='%s'\n", skdev, event);
+	dev_dbg(&skdev->pdev->dev, "  drive_state=%s(%d) driver_state=%s(%d)\n",
+		skd_drive_state_to_str(skdev->drive_state), skdev->drive_state,
+		skd_skdev_state_to_str(skdev->state), skdev->state);
+	dev_dbg(&skdev->pdev->dev, "  busy=%d limit=%d dev=%d lowat=%d\n",
+		skd_in_flight(skdev), skdev->cur_max_queue_depth,
+		skdev->dev_max_queue_depth, skdev->queue_low_water_mark);
+	dev_dbg(&skdev->pdev->dev, "  cycle=%d cycle_ix=%d\n",
+		skdev->skcomp_cycle, skdev->skcomp_ix);
 }
 
 static void skd_log_skreq(struct skd_device *skdev,
 			  struct skd_request_context *skreq, const char *event)
 {
-	pr_debug("%s:%s:%d (%s) skreq=%p event='%s'\n",
-		 skdev->name, __func__, __LINE__, skdev->name, skreq, event);
-	pr_debug("%s:%s:%d   state=%s(%d) id=0x%04x fitmsg=0x%04x\n",
-		 skdev->name, __func__, __LINE__,
-		 skd_skreq_state_to_str(skreq->state), skreq->state,
-		 skreq->id, skreq->fitmsg_id);
-	pr_debug("%s:%s:%d   timo=0x%x sg_dir=%d n_sg=%d\n",
-		 skdev->name, __func__, __LINE__,
-		 skreq->timeout_stamp, skreq->sg_data_dir, skreq->n_sg);
-
-	if (skreq->req != NULL) {
-		struct request *req = skreq->req;
-		u32 lba = (u32)blk_rq_pos(req);
-		u32 count = blk_rq_sectors(req);
-
-		pr_debug("%s:%s:%d "
-			 "req=%p lba=%u(0x%x) count=%u(0x%x) dir=%d\n",
-			 skdev->name, __func__, __LINE__,
-			 req, lba, lba, count, count,
-			 (int)rq_data_dir(req));
-	} else
-		pr_debug("%s:%s:%d req=NULL\n",
-			 skdev->name, __func__, __LINE__);
+	struct request *req = blk_mq_rq_from_pdu(skreq);
+	u32 lba = blk_rq_pos(req);
+	u32 count = blk_rq_sectors(req);
+
+	dev_dbg(&skdev->pdev->dev, "skreq=%p event='%s'\n", skreq, event);
+	dev_dbg(&skdev->pdev->dev, "  state=%s(%d) id=0x%04x fitmsg=0x%04x\n",
+		skd_skreq_state_to_str(skreq->state), skreq->state, skreq->id,
+		skreq->fitmsg_id);
+	dev_dbg(&skdev->pdev->dev, "  sg_dir=%d n_sg=%d\n",
+		skreq->data_dir, skreq->n_sg);
+
+	dev_dbg(&skdev->pdev->dev,
+		"req=%p lba=%u(0x%x) count=%u(0x%x) dir=%d\n", req, lba, lba,
+		count, count, (int)rq_data_dir(req));
 }
 
 /*
@@ -5202,7 +3608,14 @@ static void skd_log_skreq(struct skd_device *skdev,
 
 static int __init skd_init(void)
 {
-	pr_info(PFX " v%s-b%s loaded\n", DRV_VERSION, DRV_BUILD_ID);
+	BUILD_BUG_ON(sizeof(struct fit_completion_entry_v1) != 8);
+	BUILD_BUG_ON(sizeof(struct fit_comp_error_info) != 32);
+	BUILD_BUG_ON(sizeof(struct skd_command_header) != 16);
+	BUILD_BUG_ON(sizeof(struct skd_scsi_request) != 32);
+	BUILD_BUG_ON(sizeof(struct driver_inquiry_data) != 44);
+	BUILD_BUG_ON(offsetof(struct skd_msg_buf, fmh) != 0);
+	BUILD_BUG_ON(offsetof(struct skd_msg_buf, scsi) != 64);
+	BUILD_BUG_ON(sizeof(struct skd_msg_buf) != SKD_N_FITMSG_BYTES);
 
 	switch (skd_isr_type) {
 	case SKD_IRQ_LEGACY:
@@ -5222,7 +3635,8 @@ static int __init skd_init(void)
 		skd_max_queue_depth = SKD_MAX_QUEUE_DEPTH_DEFAULT;
 	}
 
-	if (skd_max_req_per_msg < 1 || skd_max_req_per_msg > 14) {
+	if (skd_max_req_per_msg < 1 ||
+	    skd_max_req_per_msg > SKD_MAX_REQ_PER_MSG) {
 		pr_err(PFX "skd_max_req_per_msg %d invalid, re-set to %d\n",
 		       skd_max_req_per_msg, SKD_MAX_REQ_PER_MSG_DEFAULT);
 		skd_max_req_per_msg = SKD_MAX_REQ_PER_MSG_DEFAULT;
@@ -5246,19 +3660,11 @@ static int __init skd_init(void)
 		skd_isr_comp_limit = 0;
 	}
 
-	if (skd_max_pass_thru < 1 || skd_max_pass_thru > 50) {
-		pr_err(PFX "skd_max_pass_thru %d invalid, re-set to %d\n",
-		       skd_max_pass_thru, SKD_N_SPECIAL_CONTEXT);
-		skd_max_pass_thru = SKD_N_SPECIAL_CONTEXT;
-	}
-
 	return pci_register_driver(&skd_driver);
 }
 
 static void __exit skd_exit(void)
 {
-	pr_info(PFX " v%s-b%s unloading\n", DRV_VERSION, DRV_BUILD_ID);
-
 	pci_unregister_driver(&skd_driver);
 
 	if (skd_major)
diff --git a/drivers/block/skd_s1120.h b/drivers/block/skd_s1120.h
index 61c757ff0161..de35f47e953c 100644
--- a/drivers/block/skd_s1120.h
+++ b/drivers/block/skd_s1120.h
@@ -1,19 +1,15 @@
-/* Copyright 2012 STEC, Inc.
+/*
+ * Copyright 2012 STEC, Inc.
+ * Copyright (c) 2017 Western Digital Corporation or its affiliates.
  *
- * This file is licensed under the terms of the 3-clause
- * BSD License (http://opensource.org/licenses/BSD-3-Clause)
- * or the GNU GPL-2.0 (http://www.gnu.org/licenses/gpl-2.0.html),
- * at your option. Both licenses are also available in the LICENSE file
- * distributed with this project. This file may not be copied, modified,
- * or distributed except in accordance with those terms.
+ * This file is part of the Linux kernel, and is made available under
+ * the terms of the GNU General Public License version 2.
  */
 
 
 #ifndef SKD_S1120_H
 #define SKD_S1120_H
 
-#pragma pack(push, s1120_h, 1)
-
 /*
  * Q-channel, 64-bit r/w
  */
@@ -30,7 +26,7 @@
 #define  FIT_QCMD_MSGSIZE_128		(0x1 << 4)
 #define  FIT_QCMD_MSGSIZE_256		(0x2 << 4)
 #define  FIT_QCMD_MSGSIZE_512		(0x3 << 4)
-#define  FIT_QCMD_BASE_ADDRESS_MASK	(0xFFFFFFFFFFFFFFC0ull)
+#define  FIT_QCMD_ALIGN			L1_CACHE_BYTES
 
 /*
  * Control, 32-bit r/w
@@ -250,7 +246,7 @@ struct fit_msg_hdr {
  *  20-23 of the FIT_MTD_FITFW_INIT response.
  */
 struct fit_completion_entry_v1 {
-	uint32_t	num_returned_bytes;
+	__be32		num_returned_bytes;
 	uint16_t	tag;
 	uint8_t		status;  /* SCSI status */
 	uint8_t		cycle;
@@ -278,7 +274,7 @@ struct fit_comp_error_info {
 	uint16_t	sks_low; /* 10: Sense Key Specific (LSW) */
 	uint16_t	reserved3; /* 12: Part of additional sense bytes (unused) */
 	uint16_t	uec; /* 14: Additional Sense Bytes */
-	uint64_t	per; /* 16: Additional Sense Bytes */
+	uint64_t	per __packed; /* 16: Additional Sense Bytes */
 	uint8_t		reserved4[2]; /* 1E: Additional Sense Bytes (unused) */
 };
 
@@ -292,11 +288,11 @@ struct fit_comp_error_info {
  * Version one has the last 32 bits sg_list_len_bytes;
  */
 struct skd_command_header {
-	uint64_t	sg_list_dma_address;
+	__be64		sg_list_dma_address;
 	uint16_t	tag;
 	uint8_t		attribute;
 	uint8_t		add_cdb_len;     /* In 32 bit words */
-	uint32_t	sg_list_len_bytes;
+	__be32		sg_list_len_bytes;
 };
 
 struct skd_scsi_request {
@@ -309,22 +305,20 @@ struct driver_inquiry_data {
 	uint8_t		peripheral_device_type:5;
 	uint8_t		qualifier:3;
 	uint8_t		page_code;
-	uint16_t	page_length;
-	uint16_t	pcie_bus_number;
+	__be16		page_length;
+	__be16		pcie_bus_number;
 	uint8_t		pcie_device_number;
 	uint8_t		pcie_function_number;
 	uint8_t		pcie_link_speed;
 	uint8_t		pcie_link_lanes;
-	uint16_t	pcie_vendor_id;
-	uint16_t	pcie_device_id;
-	uint16_t	pcie_subsystem_vendor_id;
-	uint16_t	pcie_subsystem_device_id;
+	__be16		pcie_vendor_id;
+	__be16		pcie_device_id;
+	__be16		pcie_subsystem_vendor_id;
+	__be16		pcie_subsystem_device_id;
 	uint8_t		reserved1[2];
 	uint8_t		reserved2[3];
 	uint8_t		driver_version_length;
 	uint8_t		driver_version[0x14];
 };
 
-#pragma pack(pop, s1120_h)
-
 #endif /* SKD_S1120_H */
diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c
index 1498b899a593..34e17ee799be 100644
--- a/drivers/block/virtio_blk.c
+++ b/drivers/block/virtio_blk.c
@@ -265,7 +265,7 @@ static blk_status_t virtio_queue_rq(struct blk_mq_hw_ctx *hctx,
 	}
 
 	spin_lock_irqsave(&vblk->vqs[qid].lock, flags);
-	if (req_op(req) == REQ_OP_SCSI_IN || req_op(req) == REQ_OP_SCSI_OUT)
+	if (blk_rq_is_scsi(req))
 		err = virtblk_add_req_scsi(vblk->vqs[qid].vq, vbr, vbr->sg, num);
 	else
 		err = virtblk_add_req(vblk->vqs[qid].vq, vbr, vbr->sg, num);
@@ -381,6 +381,7 @@ static void virtblk_config_changed_work(struct work_struct *work)
 	struct request_queue *q = vblk->disk->queue;
 	char cap_str_2[10], cap_str_10[10];
 	char *envp[] = { "RESIZE=1", NULL };
+	unsigned long long nblocks;
 	u64 capacity;
 
 	/* Host must always specify the capacity. */
@@ -393,16 +394,19 @@ static void virtblk_config_changed_work(struct work_struct *work)
 		capacity = (sector_t)-1;
 	}
 
-	string_get_size(capacity, queue_logical_block_size(q),
+	nblocks = DIV_ROUND_UP_ULL(capacity, queue_logical_block_size(q) >> 9);
+
+	string_get_size(nblocks, queue_logical_block_size(q),
 			STRING_UNITS_2, cap_str_2, sizeof(cap_str_2));
-	string_get_size(capacity, queue_logical_block_size(q),
+	string_get_size(nblocks, queue_logical_block_size(q),
 			STRING_UNITS_10, cap_str_10, sizeof(cap_str_10));
 
 	dev_notice(&vdev->dev,
-		  "new size: %llu %d-byte logical blocks (%s/%s)\n",
-		  (unsigned long long)capacity,
-		  queue_logical_block_size(q),
-		  cap_str_10, cap_str_2);
+		   "new size: %llu %d-byte logical blocks (%s/%s)\n",
+		   nblocks,
+		   queue_logical_block_size(q),
+		   cap_str_10,
+		   cap_str_2);
 
 	set_capacity(vblk->disk, capacity);
 	revalidate_disk(vblk->disk);
diff --git a/drivers/block/xen-blkback/blkback.c b/drivers/block/xen-blkback/blkback.c
index fe7cd58c43d0..987d665e82de 100644
--- a/drivers/block/xen-blkback/blkback.c
+++ b/drivers/block/xen-blkback/blkback.c
@@ -705,9 +705,9 @@ static unsigned int xen_blkbk_unmap_prepare(
 				    GNTMAP_host_map, pages[i]->handle);
 		pages[i]->handle = BLKBACK_INVALID_HANDLE;
 		invcount++;
-       }
+	}
 
-       return invcount;
+	return invcount;
 }
 
 static void xen_blkbk_unmap_and_respond_callback(int result, struct gntab_unmap_queue_data *data)
@@ -1251,6 +1251,7 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring,
 		break;
 	case BLKIF_OP_WRITE_BARRIER:
 		drain = true;
+		/* fall through */
 	case BLKIF_OP_FLUSH_DISKCACHE:
 		ring->st_f_req++;
 		operation = REQ_OP_WRITE;
@@ -1362,7 +1363,7 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring,
 				goto fail_put_bio;
 
 			biolist[nbio++] = bio;
-			bio->bi_bdev    = preq.bdev;
+			bio_set_dev(bio, preq.bdev);
 			bio->bi_private = pending_req;
 			bio->bi_end_io  = end_block_io_op;
 			bio->bi_iter.bi_sector  = preq.sector_number;
@@ -1381,7 +1382,7 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring,
 			goto fail_put_bio;
 
 		biolist[nbio++] = bio;
-		bio->bi_bdev    = preq.bdev;
+		bio_set_dev(bio, preq.bdev);
 		bio->bi_private = pending_req;
 		bio->bi_end_io  = end_block_io_op;
 		bio_set_op_attrs(bio, operation, operation_flags);
diff --git a/drivers/block/xen-blkback/xenbus.c b/drivers/block/xen-blkback/xenbus.c
index 792da683e70d..21c1be1eb226 100644
--- a/drivers/block/xen-blkback/xenbus.c
+++ b/drivers/block/xen-blkback/xenbus.c
@@ -244,6 +244,7 @@ static int xen_blkif_disconnect(struct xen_blkif *blkif)
 {
 	struct pending_req *req, *n;
 	unsigned int j, r;
+	bool busy = false;
 
 	for (r = 0; r < blkif->nr_rings; r++) {
 		struct xen_blkif_ring *ring = &blkif->rings[r];
@@ -261,8 +262,10 @@ static int xen_blkif_disconnect(struct xen_blkif *blkif)
 		 * don't have any discard_io or other_io requests. So, checking
 		 * for inflight IO is enough.
 		 */
-		if (atomic_read(&ring->inflight) > 0)
-			return -EBUSY;
+		if (atomic_read(&ring->inflight) > 0) {
+			busy = true;
+			continue;
+		}
 
 		if (ring->irq) {
 			unbind_from_irqhandler(ring->irq, ring);
@@ -300,6 +303,9 @@ static int xen_blkif_disconnect(struct xen_blkif *blkif)
 		WARN_ON(i != (XEN_BLKIF_REQS_PER_PAGE * blkif->nr_ring_pages));
 		ring->active = false;
 	}
+	if (busy)
+		return -EBUSY;
+
 	blkif->nr_ring_pages = 0;
 	/*
 	 * blkif->rings was allocated in connect_ring, so we should free it in
@@ -810,7 +816,8 @@ static void frontend_changed(struct xenbus_device *dev,
 		xenbus_switch_state(dev, XenbusStateClosed);
 		if (xenbus_dev_is_online(dev))
 			break;
-		/* fall through if not online */
+		/* fall through */
+		/* if not online */
 	case XenbusStateUnknown:
 		/* implies xen_blkif_disconnect() via xen_blkbk_remove() */
 		device_unregister(&dev->dev);
diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c
index 98e34e4c62b8..891265acb10e 100644
--- a/drivers/block/xen-blkfront.c
+++ b/drivers/block/xen-blkfront.c
@@ -2075,9 +2075,9 @@ static int blkfront_resume(struct xenbus_device *dev)
 			/*
 			 * Get the bios in the request so we can re-queue them.
 			 */
-			if (req_op(shadow[i].request) == REQ_OP_FLUSH ||
-			    req_op(shadow[i].request) == REQ_OP_DISCARD ||
-			    req_op(shadow[i].request) == REQ_OP_SECURE_ERASE ||
+			if (req_op(shadow[j].request) == REQ_OP_FLUSH ||
+			    req_op(shadow[j].request) == REQ_OP_DISCARD ||
+			    req_op(shadow[j].request) == REQ_OP_SECURE_ERASE ||
 			    shadow[j].request->cmd_flags & REQ_FUA) {
 				/*
 				 * Flush operations don't contain bios, so
@@ -2456,7 +2456,7 @@ static void blkback_changed(struct xenbus_device *dev,
 	case XenbusStateClosed:
 		if (dev->state == XenbusStateClosed)
 			break;
-		/* Missed the backend's Closing state -- fallthrough */
+		/* fall through */
 	case XenbusStateClosing:
 		if (info)
 			blkfront_closing(info);
diff --git a/drivers/block/zram/Kconfig b/drivers/block/zram/Kconfig
index b8ecba6dcd3b..7cd4a8ec3c8f 100644
--- a/drivers/block/zram/Kconfig
+++ b/drivers/block/zram/Kconfig
@@ -13,3 +13,15 @@ config ZRAM
 	  disks and maybe many more.
 
 	  See zram.txt for more information.
+
+config ZRAM_WRITEBACK
+       bool "Write back incompressible page to backing device"
+       depends on ZRAM
+       default n
+       help
+	 With incompressible page, there is no memory saving to keep it
+	 in memory. Instead, write it out to backing device.
+	 For this feature, admin should set up backing device via
+	 /sys/block/zramX/backing_dev.
+
+	 See zram.txt for more infomration.
diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c
index 3b1b6340ba13..2981c27d3aae 100644
--- a/drivers/block/zram/zram_drv.c
+++ b/drivers/block/zram/zram_drv.c
@@ -175,20 +175,11 @@ static inline void update_used_max(struct zram *zram,
 	} while (old_max != cur_max);
 }
 
-static inline void zram_fill_page(char *ptr, unsigned long len,
+static inline void zram_fill_page(void *ptr, unsigned long len,
 					unsigned long value)
 {
-	int i;
-	unsigned long *page = (unsigned long *)ptr;
-
 	WARN_ON_ONCE(!IS_ALIGNED(len, sizeof(unsigned long)));
-
-	if (likely(value == 0)) {
-		memset(ptr, 0, len);
-	} else {
-		for (i = 0; i < len / sizeof(*page); i++)
-			page[i] = value;
-	}
+	memset_l(ptr, value, len / sizeof(unsigned long));
 }
 
 static bool page_same_filled(void *ptr, unsigned long *element)
@@ -270,6 +261,349 @@ static ssize_t mem_used_max_store(struct device *dev,
 	return len;
 }
 
+#ifdef CONFIG_ZRAM_WRITEBACK
+static bool zram_wb_enabled(struct zram *zram)
+{
+	return zram->backing_dev;
+}
+
+static void reset_bdev(struct zram *zram)
+{
+	struct block_device *bdev;
+
+	if (!zram_wb_enabled(zram))
+		return;
+
+	bdev = zram->bdev;
+	if (zram->old_block_size)
+		set_blocksize(bdev, zram->old_block_size);
+	blkdev_put(bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);
+	/* hope filp_close flush all of IO */
+	filp_close(zram->backing_dev, NULL);
+	zram->backing_dev = NULL;
+	zram->old_block_size = 0;
+	zram->bdev = NULL;
+
+	kvfree(zram->bitmap);
+	zram->bitmap = NULL;
+}
+
+static ssize_t backing_dev_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct zram *zram = dev_to_zram(dev);
+	struct file *file = zram->backing_dev;
+	char *p;
+	ssize_t ret;
+
+	down_read(&zram->init_lock);
+	if (!zram_wb_enabled(zram)) {
+		memcpy(buf, "none\n", 5);
+		up_read(&zram->init_lock);
+		return 5;
+	}
+
+	p = file_path(file, buf, PAGE_SIZE - 1);
+	if (IS_ERR(p)) {
+		ret = PTR_ERR(p);
+		goto out;
+	}
+
+	ret = strlen(p);
+	memmove(buf, p, ret);
+	buf[ret++] = '\n';
+out:
+	up_read(&zram->init_lock);
+	return ret;
+}
+
+static ssize_t backing_dev_store(struct device *dev,
+		struct device_attribute *attr, const char *buf, size_t len)
+{
+	char *file_name;
+	struct file *backing_dev = NULL;
+	struct inode *inode;
+	struct address_space *mapping;
+	unsigned int bitmap_sz, old_block_size = 0;
+	unsigned long nr_pages, *bitmap = NULL;
+	struct block_device *bdev = NULL;
+	int err;
+	struct zram *zram = dev_to_zram(dev);
+
+	file_name = kmalloc(PATH_MAX, GFP_KERNEL);
+	if (!file_name)
+		return -ENOMEM;
+
+	down_write(&zram->init_lock);
+	if (init_done(zram)) {
+		pr_info("Can't setup backing device for initialized device\n");
+		err = -EBUSY;
+		goto out;
+	}
+
+	strlcpy(file_name, buf, len);
+
+	backing_dev = filp_open(file_name, O_RDWR|O_LARGEFILE, 0);
+	if (IS_ERR(backing_dev)) {
+		err = PTR_ERR(backing_dev);
+		backing_dev = NULL;
+		goto out;
+	}
+
+	mapping = backing_dev->f_mapping;
+	inode = mapping->host;
+
+	/* Support only block device in this moment */
+	if (!S_ISBLK(inode->i_mode)) {
+		err = -ENOTBLK;
+		goto out;
+	}
+
+	bdev = bdgrab(I_BDEV(inode));
+	err = blkdev_get(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL, zram);
+	if (err < 0)
+		goto out;
+
+	nr_pages = i_size_read(inode) >> PAGE_SHIFT;
+	bitmap_sz = BITS_TO_LONGS(nr_pages) * sizeof(long);
+	bitmap = kvzalloc(bitmap_sz, GFP_KERNEL);
+	if (!bitmap) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	old_block_size = block_size(bdev);
+	err = set_blocksize(bdev, PAGE_SIZE);
+	if (err)
+		goto out;
+
+	reset_bdev(zram);
+	spin_lock_init(&zram->bitmap_lock);
+
+	zram->old_block_size = old_block_size;
+	zram->bdev = bdev;
+	zram->backing_dev = backing_dev;
+	zram->bitmap = bitmap;
+	zram->nr_pages = nr_pages;
+	up_write(&zram->init_lock);
+
+	pr_info("setup backing device %s\n", file_name);
+	kfree(file_name);
+
+	return len;
+out:
+	if (bitmap)
+		kvfree(bitmap);
+
+	if (bdev)
+		blkdev_put(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
+
+	if (backing_dev)
+		filp_close(backing_dev, NULL);
+
+	up_write(&zram->init_lock);
+
+	kfree(file_name);
+
+	return err;
+}
+
+static unsigned long get_entry_bdev(struct zram *zram)
+{
+	unsigned long entry;
+
+	spin_lock(&zram->bitmap_lock);
+	/* skip 0 bit to confuse zram.handle = 0 */
+	entry = find_next_zero_bit(zram->bitmap, zram->nr_pages, 1);
+	if (entry == zram->nr_pages) {
+		spin_unlock(&zram->bitmap_lock);
+		return 0;
+	}
+
+	set_bit(entry, zram->bitmap);
+	spin_unlock(&zram->bitmap_lock);
+
+	return entry;
+}
+
+static void put_entry_bdev(struct zram *zram, unsigned long entry)
+{
+	int was_set;
+
+	spin_lock(&zram->bitmap_lock);
+	was_set = test_and_clear_bit(entry, zram->bitmap);
+	spin_unlock(&zram->bitmap_lock);
+	WARN_ON_ONCE(!was_set);
+}
+
+void zram_page_end_io(struct bio *bio)
+{
+	struct page *page = bio->bi_io_vec[0].bv_page;
+
+	page_endio(page, op_is_write(bio_op(bio)),
+			blk_status_to_errno(bio->bi_status));
+	bio_put(bio);
+}
+
+/*
+ * Returns 1 if the submission is successful.
+ */
+static int read_from_bdev_async(struct zram *zram, struct bio_vec *bvec,
+			unsigned long entry, struct bio *parent)
+{
+	struct bio *bio;
+
+	bio = bio_alloc(GFP_ATOMIC, 1);
+	if (!bio)
+		return -ENOMEM;
+
+	bio->bi_iter.bi_sector = entry * (PAGE_SIZE >> 9);
+	bio_set_dev(bio, zram->bdev);
+	if (!bio_add_page(bio, bvec->bv_page, bvec->bv_len, bvec->bv_offset)) {
+		bio_put(bio);
+		return -EIO;
+	}
+
+	if (!parent) {
+		bio->bi_opf = REQ_OP_READ;
+		bio->bi_end_io = zram_page_end_io;
+	} else {
+		bio->bi_opf = parent->bi_opf;
+		bio_chain(bio, parent);
+	}
+
+	submit_bio(bio);
+	return 1;
+}
+
+struct zram_work {
+	struct work_struct work;
+	struct zram *zram;
+	unsigned long entry;
+	struct bio *bio;
+};
+
+#if PAGE_SIZE != 4096
+static void zram_sync_read(struct work_struct *work)
+{
+	struct bio_vec bvec;
+	struct zram_work *zw = container_of(work, struct zram_work, work);
+	struct zram *zram = zw->zram;
+	unsigned long entry = zw->entry;
+	struct bio *bio = zw->bio;
+
+	read_from_bdev_async(zram, &bvec, entry, bio);
+}
+
+/*
+ * Block layer want one ->make_request_fn to be active at a time
+ * so if we use chained IO with parent IO in same context,
+ * it's a deadlock. To avoid, it, it uses worker thread context.
+ */
+static int read_from_bdev_sync(struct zram *zram, struct bio_vec *bvec,
+				unsigned long entry, struct bio *bio)
+{
+	struct zram_work work;
+
+	work.zram = zram;
+	work.entry = entry;
+	work.bio = bio;
+
+	INIT_WORK_ONSTACK(&work.work, zram_sync_read);
+	queue_work(system_unbound_wq, &work.work);
+	flush_work(&work.work);
+	destroy_work_on_stack(&work.work);
+
+	return 1;
+}
+#else
+static int read_from_bdev_sync(struct zram *zram, struct bio_vec *bvec,
+				unsigned long entry, struct bio *bio)
+{
+	WARN_ON(1);
+	return -EIO;
+}
+#endif
+
+static int read_from_bdev(struct zram *zram, struct bio_vec *bvec,
+			unsigned long entry, struct bio *parent, bool sync)
+{
+	if (sync)
+		return read_from_bdev_sync(zram, bvec, entry, parent);
+	else
+		return read_from_bdev_async(zram, bvec, entry, parent);
+}
+
+static int write_to_bdev(struct zram *zram, struct bio_vec *bvec,
+					u32 index, struct bio *parent,
+					unsigned long *pentry)
+{
+	struct bio *bio;
+	unsigned long entry;
+
+	bio = bio_alloc(GFP_ATOMIC, 1);
+	if (!bio)
+		return -ENOMEM;
+
+	entry = get_entry_bdev(zram);
+	if (!entry) {
+		bio_put(bio);
+		return -ENOSPC;
+	}
+
+	bio->bi_iter.bi_sector = entry * (PAGE_SIZE >> 9);
+	bio_set_dev(bio, zram->bdev);
+	if (!bio_add_page(bio, bvec->bv_page, bvec->bv_len,
+					bvec->bv_offset)) {
+		bio_put(bio);
+		put_entry_bdev(zram, entry);
+		return -EIO;
+	}
+
+	if (!parent) {
+		bio->bi_opf = REQ_OP_WRITE | REQ_SYNC;
+		bio->bi_end_io = zram_page_end_io;
+	} else {
+		bio->bi_opf = parent->bi_opf;
+		bio_chain(bio, parent);
+	}
+
+	submit_bio(bio);
+	*pentry = entry;
+
+	return 0;
+}
+
+static void zram_wb_clear(struct zram *zram, u32 index)
+{
+	unsigned long entry;
+
+	zram_clear_flag(zram, index, ZRAM_WB);
+	entry = zram_get_element(zram, index);
+	zram_set_element(zram, index, 0);
+	put_entry_bdev(zram, entry);
+}
+
+#else
+static bool zram_wb_enabled(struct zram *zram) { return false; }
+static inline void reset_bdev(struct zram *zram) {};
+static int write_to_bdev(struct zram *zram, struct bio_vec *bvec,
+					u32 index, struct bio *parent,
+					unsigned long *pentry)
+
+{
+	return -EIO;
+}
+
+static int read_from_bdev(struct zram *zram, struct bio_vec *bvec,
+			unsigned long entry, struct bio *parent, bool sync)
+{
+	return -EIO;
+}
+static void zram_wb_clear(struct zram *zram, u32 index) {}
+#endif
+
+
 /*
  * We switched to per-cpu streams and this attr is not needed anymore.
  * However, we will keep it around for some time, because:
@@ -453,30 +787,6 @@ static bool zram_same_page_read(struct zram *zram, u32 index,
 	return false;
 }
 
-static bool zram_same_page_write(struct zram *zram, u32 index,
-					struct page *page)
-{
-	unsigned long element;
-	void *mem = kmap_atomic(page);
-
-	if (page_same_filled(mem, &element)) {
-		kunmap_atomic(mem);
-		/* Free memory associated with this sector now. */
-		zram_slot_lock(zram, index);
-		zram_free_page(zram, index);
-		zram_set_flag(zram, index, ZRAM_SAME);
-		zram_set_element(zram, index, element);
-		zram_slot_unlock(zram, index);
-
-		atomic64_inc(&zram->stats.same_pages);
-		atomic64_inc(&zram->stats.pages_stored);
-		return true;
-	}
-	kunmap_atomic(mem);
-
-	return false;
-}
-
 static void zram_meta_free(struct zram *zram, u64 disksize)
 {
 	size_t num_pages = disksize >> PAGE_SHIFT;
@@ -515,7 +825,13 @@ static bool zram_meta_alloc(struct zram *zram, u64 disksize)
  */
 static void zram_free_page(struct zram *zram, size_t index)
 {
-	unsigned long handle = zram_get_handle(zram, index);
+	unsigned long handle;
+
+	if (zram_wb_enabled(zram) && zram_test_flag(zram, index, ZRAM_WB)) {
+		zram_wb_clear(zram, index);
+		atomic64_dec(&zram->stats.pages_stored);
+		return;
+	}
 
 	/*
 	 * No memory is allocated for same element filled pages.
@@ -529,6 +845,7 @@ static void zram_free_page(struct zram *zram, size_t index)
 		return;
 	}
 
+	handle = zram_get_handle(zram, index);
 	if (!handle)
 		return;
 
@@ -542,13 +859,31 @@ static void zram_free_page(struct zram *zram, size_t index)
 	zram_set_obj_size(zram, index, 0);
 }
 
-static int zram_decompress_page(struct zram *zram, struct page *page, u32 index)
+static int __zram_bvec_read(struct zram *zram, struct page *page, u32 index,
+				struct bio *bio, bool partial_io)
 {
 	int ret;
 	unsigned long handle;
 	unsigned int size;
 	void *src, *dst;
 
+	if (zram_wb_enabled(zram)) {
+		zram_slot_lock(zram, index);
+		if (zram_test_flag(zram, index, ZRAM_WB)) {
+			struct bio_vec bvec;
+
+			zram_slot_unlock(zram, index);
+
+			bvec.bv_page = page;
+			bvec.bv_len = PAGE_SIZE;
+			bvec.bv_offset = 0;
+			return read_from_bdev(zram, &bvec,
+					zram_get_element(zram, index),
+					bio, partial_io);
+		}
+		zram_slot_unlock(zram, index);
+	}
+
 	if (zram_same_page_read(zram, index, page, 0, PAGE_SIZE))
 		return 0;
 
@@ -581,7 +916,7 @@ static int zram_decompress_page(struct zram *zram, struct page *page, u32 index)
 }
 
 static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
-				u32 index, int offset)
+				u32 index, int offset, struct bio *bio)
 {
 	int ret;
 	struct page *page;
@@ -594,7 +929,7 @@ static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
 			return -ENOMEM;
 	}
 
-	ret = zram_decompress_page(zram, page, index);
+	ret = __zram_bvec_read(zram, page, index, bio, is_partial_io(bvec));
 	if (unlikely(ret))
 		goto out;
 
@@ -613,30 +948,57 @@ out:
 	return ret;
 }
 
-static int zram_compress(struct zram *zram, struct zcomp_strm **zstrm,
-			struct page *page,
-			unsigned long *out_handle, unsigned int *out_comp_len)
+static int __zram_bvec_write(struct zram *zram, struct bio_vec *bvec,
+				u32 index, struct bio *bio)
 {
-	int ret;
-	unsigned int comp_len;
-	void *src;
+	int ret = 0;
 	unsigned long alloced_pages;
 	unsigned long handle = 0;
+	unsigned int comp_len = 0;
+	void *src, *dst, *mem;
+	struct zcomp_strm *zstrm;
+	struct page *page = bvec->bv_page;
+	unsigned long element = 0;
+	enum zram_pageflags flags = 0;
+	bool allow_wb = true;
+
+	mem = kmap_atomic(page);
+	if (page_same_filled(mem, &element)) {
+		kunmap_atomic(mem);
+		/* Free memory associated with this sector now. */
+		flags = ZRAM_SAME;
+		atomic64_inc(&zram->stats.same_pages);
+		goto out;
+	}
+	kunmap_atomic(mem);
 
 compress_again:
+	zstrm = zcomp_stream_get(zram->comp);
 	src = kmap_atomic(page);
-	ret = zcomp_compress(*zstrm, src, &comp_len);
+	ret = zcomp_compress(zstrm, src, &comp_len);
 	kunmap_atomic(src);
 
 	if (unlikely(ret)) {
+		zcomp_stream_put(zram->comp);
 		pr_err("Compression failed! err=%d\n", ret);
-		if (handle)
-			zs_free(zram->mem_pool, handle);
+		zs_free(zram->mem_pool, handle);
 		return ret;
 	}
 
-	if (unlikely(comp_len > max_zpage_size))
+	if (unlikely(comp_len > max_zpage_size)) {
+		if (zram_wb_enabled(zram) && allow_wb) {
+			zcomp_stream_put(zram->comp);
+			ret = write_to_bdev(zram, bvec, index, bio, &element);
+			if (!ret) {
+				flags = ZRAM_WB;
+				ret = 1;
+				goto out;
+			}
+			allow_wb = false;
+			goto compress_again;
+		}
 		comp_len = PAGE_SIZE;
+	}
 
 	/*
 	 * handle allocation has 2 paths:
@@ -663,7 +1025,6 @@ compress_again:
 		handle = zs_malloc(zram->mem_pool, comp_len,
 				GFP_NOIO | __GFP_HIGHMEM |
 				__GFP_MOVABLE);
-		*zstrm = zcomp_stream_get(zram->comp);
 		if (handle)
 			goto compress_again;
 		return -ENOMEM;
@@ -673,34 +1034,11 @@ compress_again:
 	update_used_max(zram, alloced_pages);
 
 	if (zram->limit_pages && alloced_pages > zram->limit_pages) {
+		zcomp_stream_put(zram->comp);
 		zs_free(zram->mem_pool, handle);
 		return -ENOMEM;
 	}
 
-	*out_handle = handle;
-	*out_comp_len = comp_len;
-	return 0;
-}
-
-static int __zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index)
-{
-	int ret;
-	unsigned long handle;
-	unsigned int comp_len;
-	void *src, *dst;
-	struct zcomp_strm *zstrm;
-	struct page *page = bvec->bv_page;
-
-	if (zram_same_page_write(zram, index, page))
-		return 0;
-
-	zstrm = zcomp_stream_get(zram->comp);
-	ret = zram_compress(zram, &zstrm, page, &handle, &comp_len);
-	if (ret) {
-		zcomp_stream_put(zram->comp);
-		return ret;
-	}
-
 	dst = zs_map_object(zram->mem_pool, handle, ZS_MM_WO);
 
 	src = zstrm->buffer;
@@ -712,25 +1050,31 @@ static int __zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index)
 
 	zcomp_stream_put(zram->comp);
 	zs_unmap_object(zram->mem_pool, handle);
-
+	atomic64_add(comp_len, &zram->stats.compr_data_size);
+out:
 	/*
 	 * Free memory associated with this sector
 	 * before overwriting unused sectors.
 	 */
 	zram_slot_lock(zram, index);
 	zram_free_page(zram, index);
-	zram_set_handle(zram, index, handle);
-	zram_set_obj_size(zram, index, comp_len);
+
+	if (flags) {
+		zram_set_flag(zram, index, flags);
+		zram_set_element(zram, index, element);
+	}  else {
+		zram_set_handle(zram, index, handle);
+		zram_set_obj_size(zram, index, comp_len);
+	}
 	zram_slot_unlock(zram, index);
 
 	/* Update stats */
-	atomic64_add(comp_len, &zram->stats.compr_data_size);
 	atomic64_inc(&zram->stats.pages_stored);
-	return 0;
+	return ret;
 }
 
 static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec,
-				u32 index, int offset)
+				u32 index, int offset, struct bio *bio)
 {
 	int ret;
 	struct page *page = NULL;
@@ -748,7 +1092,7 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec,
 		if (!page)
 			return -ENOMEM;
 
-		ret = zram_decompress_page(zram, page, index);
+		ret = __zram_bvec_read(zram, page, index, bio, true);
 		if (ret)
 			goto out;
 
@@ -763,7 +1107,7 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec,
 		vec.bv_offset = 0;
 	}
 
-	ret = __zram_bvec_write(zram, &vec, index);
+	ret = __zram_bvec_write(zram, &vec, index, bio);
 out:
 	if (is_partial_io(bvec))
 		__free_page(page);
@@ -808,28 +1152,34 @@ static void zram_bio_discard(struct zram *zram, u32 index,
 	}
 }
 
+/*
+ * Returns errno if it has some problem. Otherwise return 0 or 1.
+ * Returns 0 if IO request was done synchronously
+ * Returns 1 if IO request was successfully submitted.
+ */
 static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index,
-			int offset, bool is_write)
+			int offset, bool is_write, struct bio *bio)
 {
 	unsigned long start_time = jiffies;
 	int rw_acct = is_write ? REQ_OP_WRITE : REQ_OP_READ;
+	struct request_queue *q = zram->disk->queue;
 	int ret;
 
-	generic_start_io_acct(rw_acct, bvec->bv_len >> SECTOR_SHIFT,
+	generic_start_io_acct(q, rw_acct, bvec->bv_len >> SECTOR_SHIFT,
 			&zram->disk->part0);
 
 	if (!is_write) {
 		atomic64_inc(&zram->stats.num_reads);
-		ret = zram_bvec_read(zram, bvec, index, offset);
+		ret = zram_bvec_read(zram, bvec, index, offset, bio);
 		flush_dcache_page(bvec->bv_page);
 	} else {
 		atomic64_inc(&zram->stats.num_writes);
-		ret = zram_bvec_write(zram, bvec, index, offset);
+		ret = zram_bvec_write(zram, bvec, index, offset, bio);
 	}
 
-	generic_end_io_acct(rw_acct, &zram->disk->part0, start_time);
+	generic_end_io_acct(q, rw_acct, &zram->disk->part0, start_time);
 
-	if (unlikely(ret)) {
+	if (unlikely(ret < 0)) {
 		if (!is_write)
 			atomic64_inc(&zram->stats.failed_reads);
 		else
@@ -868,7 +1218,7 @@ static void __zram_make_request(struct zram *zram, struct bio *bio)
 			bv.bv_len = min_t(unsigned int, PAGE_SIZE - offset,
 							unwritten);
 			if (zram_bvec_rw(zram, &bv, index, offset,
-					op_is_write(bio_op(bio))) < 0)
+					op_is_write(bio_op(bio)), bio) < 0)
 				goto out;
 
 			bv.bv_offset += bv.bv_len;
@@ -922,16 +1272,18 @@ static void zram_slot_free_notify(struct block_device *bdev,
 static int zram_rw_page(struct block_device *bdev, sector_t sector,
 		       struct page *page, bool is_write)
 {
-	int offset, err = -EIO;
+	int offset, ret;
 	u32 index;
 	struct zram *zram;
 	struct bio_vec bv;
 
+	if (PageTransHuge(page))
+		return -ENOTSUPP;
 	zram = bdev->bd_disk->private_data;
 
 	if (!valid_io_request(zram, sector, PAGE_SIZE)) {
 		atomic64_inc(&zram->stats.invalid_io);
-		err = -EINVAL;
+		ret = -EINVAL;
 		goto out;
 	}
 
@@ -942,7 +1294,7 @@ static int zram_rw_page(struct block_device *bdev, sector_t sector,
 	bv.bv_len = PAGE_SIZE;
 	bv.bv_offset = 0;
 
-	err = zram_bvec_rw(zram, &bv, index, offset, is_write);
+	ret = zram_bvec_rw(zram, &bv, index, offset, is_write, NULL);
 out:
 	/*
 	 * If I/O fails, just return error(ie, non-zero) without
@@ -952,9 +1304,20 @@ out:
 	 * bio->bi_end_io does things to handle the error
 	 * (e.g., SetPageError, set_page_dirty and extra works).
 	 */
-	if (err == 0)
+	if (unlikely(ret < 0))
+		return ret;
+
+	switch (ret) {
+	case 0:
 		page_endio(page, is_write, 0);
-	return err;
+		break;
+	case 1:
+		ret = 0;
+		break;
+	default:
+		WARN_ON(1);
+	}
+	return ret;
 }
 
 static void zram_reset_device(struct zram *zram)
@@ -983,6 +1346,7 @@ static void zram_reset_device(struct zram *zram)
 	zram_meta_free(zram, disksize);
 	memset(&zram->stats, 0, sizeof(zram->stats));
 	zcomp_destroy(comp);
+	reset_bdev(zram);
 }
 
 static ssize_t disksize_store(struct device *dev,
@@ -1108,6 +1472,9 @@ static DEVICE_ATTR_WO(mem_limit);
 static DEVICE_ATTR_WO(mem_used_max);
 static DEVICE_ATTR_RW(max_comp_streams);
 static DEVICE_ATTR_RW(comp_algorithm);
+#ifdef CONFIG_ZRAM_WRITEBACK
+static DEVICE_ATTR_RW(backing_dev);
+#endif
 
 static struct attribute *zram_disk_attrs[] = {
 	&dev_attr_disksize.attr,
@@ -1118,6 +1485,9 @@ static struct attribute *zram_disk_attrs[] = {
 	&dev_attr_mem_used_max.attr,
 	&dev_attr_max_comp_streams.attr,
 	&dev_attr_comp_algorithm.attr,
+#ifdef CONFIG_ZRAM_WRITEBACK
+	&dev_attr_backing_dev.attr,
+#endif
 	&dev_attr_io_stat.attr,
 	&dev_attr_mm_stat.attr,
 	&dev_attr_debug_stat.attr,
diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h
index e34e44d02e3e..31762db861e3 100644
--- a/drivers/block/zram/zram_drv.h
+++ b/drivers/block/zram/zram_drv.h
@@ -60,9 +60,10 @@ static const size_t max_zpage_size = PAGE_SIZE / 4 * 3;
 
 /* Flags for zram pages (table[page_no].value) */
 enum zram_pageflags {
-	/* Page consists entirely of zeros */
+	/* Page consists the same element */
 	ZRAM_SAME = ZRAM_FLAG_SHIFT,
 	ZRAM_ACCESS,	/* page is now accessed */
+	ZRAM_WB,	/* page is stored on backing_device */
 
 	__NR_ZRAM_PAGEFLAGS,
 };
@@ -115,5 +116,13 @@ struct zram {
 	 * zram is claimed so open request will be failed
 	 */
 	bool claim; /* Protected by bdev->bd_mutex */
+#ifdef CONFIG_ZRAM_WRITEBACK
+	struct file *backing_dev;
+	struct block_device *bdev;
+	unsigned int old_block_size;
+	unsigned long *bitmap;
+	unsigned long nr_pages;
+	spinlock_t bitmap_lock;
+#endif
 };
 #endif
diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig
index 35952a94875e..fae5a74dc737 100644
--- a/drivers/bluetooth/Kconfig
+++ b/drivers/bluetooth/Kconfig
@@ -98,6 +98,7 @@ config BT_HCIUART_NOKIA
 	depends on BT_HCIUART_SERDEV
 	depends on PM
 	select BT_HCIUART_H4
+	select BT_BCM
 	help
 	  Nokia H4+ is serial protocol for communication between Bluetooth
 	  device and host. This protocol is required for Bluetooth devices
@@ -167,6 +168,7 @@ config BT_HCIUART_INTEL
 config BT_HCIUART_BCM
 	bool "Broadcom protocol support"
 	depends on BT_HCIUART
+	depends on BT_HCIUART_SERDEV
 	select BT_HCIUART_H4
 	select BT_BCM
 	help
diff --git a/drivers/bluetooth/ath3k.c b/drivers/bluetooth/ath3k.c
index b793853ff05f..204afe66de92 100644
--- a/drivers/bluetooth/ath3k.c
+++ b/drivers/bluetooth/ath3k.c
@@ -140,7 +140,8 @@ MODULE_DEVICE_TABLE(usb, ath3k_table);
 
 #define BTUSB_ATH3012		0x80
 /* This table is to load patch and sysconfig files
- * for AR3012 */
+ * for AR3012
+ */
 static const struct usb_device_id ath3k_blist_tbl[] = {
 
 	/* Atheros AR3012 with sflash firmware*/
diff --git a/drivers/bluetooth/bluecard_cs.c b/drivers/bluetooth/bluecard_cs.c
index d4b0b655dde6..b07ca9565291 100644
--- a/drivers/bluetooth/bluecard_cs.c
+++ b/drivers/bluetooth/bluecard_cs.c
@@ -93,6 +93,7 @@ static void bluecard_detach(struct pcmcia_device *p_dev);
 
 /* Hardware states */
 #define CARD_READY             1
+#define CARD_ACTIVITY	       2
 #define CARD_HAS_PCCARD_ID     4
 #define CARD_HAS_POWER_LED     5
 #define CARD_HAS_ACTIVITY_LED  6
@@ -160,16 +161,14 @@ static void bluecard_activity_led_timeout(u_long arg)
 	struct bluecard_info *info = (struct bluecard_info *)arg;
 	unsigned int iobase = info->p_dev->resource[0]->start;
 
-	if (!test_bit(CARD_HAS_PCCARD_ID, &(info->hw_state)))
-		return;
-
-	if (test_bit(CARD_HAS_ACTIVITY_LED, &(info->hw_state))) {
-		/* Disable activity LED */
-		outb(0x08 | 0x20, iobase + 0x30);
-	} else {
-		/* Disable power LED */
-		outb(0x00, iobase + 0x30);
+	if (test_bit(CARD_ACTIVITY, &(info->hw_state))) {
+		/* leave LED in inactive state for HZ/10 for blink effect */
+		clear_bit(CARD_ACTIVITY, &(info->hw_state));
+		mod_timer(&(info->timer), jiffies + HZ / 10);
 	}
+
+	/* Disable activity LED, enable power LED */
+	outb(0x08 | 0x20, iobase + 0x30);
 }
 
 
@@ -177,22 +176,22 @@ static void bluecard_enable_activity_led(struct bluecard_info *info)
 {
 	unsigned int iobase = info->p_dev->resource[0]->start;
 
-	if (!test_bit(CARD_HAS_PCCARD_ID, &(info->hw_state)))
+	/* don't disturb running blink timer */
+	if (timer_pending(&(info->timer)))
 		return;
 
-	if (test_bit(CARD_HAS_ACTIVITY_LED, &(info->hw_state))) {
-		/* Enable activity LED */
-		outb(0x10 | 0x40, iobase + 0x30);
+	set_bit(CARD_ACTIVITY, &(info->hw_state));
 
-		/* Stop the LED after HZ/4 */
-		mod_timer(&(info->timer), jiffies + HZ / 4);
+	if (test_bit(CARD_HAS_ACTIVITY_LED, &(info->hw_state))) {
+		/* Enable activity LED, keep power LED enabled */
+		outb(0x18 | 0x60, iobase + 0x30);
 	} else {
-		/* Enable power LED */
-		outb(0x08 | 0x20, iobase + 0x30);
-
-		/* Stop the LED after HZ/2 */
-		mod_timer(&(info->timer), jiffies + HZ / 2);
+		/* Disable power LED */
+		outb(0x00, iobase + 0x30);
 	}
+
+	/* Stop the LED after HZ/10 */
+	mod_timer(&(info->timer), jiffies + HZ / 10);
 }
 
 
@@ -625,16 +624,13 @@ static int bluecard_hci_flush(struct hci_dev *hdev)
 static int bluecard_hci_open(struct hci_dev *hdev)
 {
 	struct bluecard_info *info = hci_get_drvdata(hdev);
+	unsigned int iobase = info->p_dev->resource[0]->start;
 
 	if (test_bit(CARD_HAS_PCCARD_ID, &(info->hw_state)))
 		bluecard_hci_set_baud_rate(hdev, DEFAULT_BAUD_RATE);
 
-	if (test_bit(CARD_HAS_PCCARD_ID, &(info->hw_state))) {
-		unsigned int iobase = info->p_dev->resource[0]->start;
-
-		/* Enable LED */
-		outb(0x08 | 0x20, iobase + 0x30);
-	}
+	/* Enable power LED */
+	outb(0x08 | 0x20, iobase + 0x30);
 
 	return 0;
 }
@@ -643,15 +639,15 @@ static int bluecard_hci_open(struct hci_dev *hdev)
 static int bluecard_hci_close(struct hci_dev *hdev)
 {
 	struct bluecard_info *info = hci_get_drvdata(hdev);
+	unsigned int iobase = info->p_dev->resource[0]->start;
 
 	bluecard_hci_flush(hdev);
 
-	if (test_bit(CARD_HAS_PCCARD_ID, &(info->hw_state))) {
-		unsigned int iobase = info->p_dev->resource[0]->start;
+	/* Stop LED timer */
+	del_timer_sync(&(info->timer));
 
-		/* Disable LED */
-		outb(0x00, iobase + 0x30);
-	}
+	/* Disable power LED */
+	outb(0x00, iobase + 0x30);
 
 	return 0;
 }
diff --git a/drivers/bluetooth/bt3c_cs.c b/drivers/bluetooth/bt3c_cs.c
index 32dcac017395..194788739a83 100644
--- a/drivers/bluetooth/bt3c_cs.c
+++ b/drivers/bluetooth/bt3c_cs.c
@@ -684,14 +684,16 @@ static int bt3c_config(struct pcmcia_device *link)
 	unsigned long try;
 
 	/* First pass: look for a config entry that looks normal.
-	   Two tries: without IO aliases, then with aliases */
+	 * Two tries: without IO aliases, then with aliases
+	 */
 	for (try = 0; try < 2; try++)
 		if (!pcmcia_loop_config(link, bt3c_check_config, (void *) try))
 			goto found_port;
 
 	/* Second pass: try to find an entry that isn't picky about
-	   its base address, then try to grab any standard serial port
-	   address, and finally try to get any free port. */
+	 * its base address, then try to grab any standard serial port
+	 * address, and finally try to get any free port.
+	 */
 	if (!pcmcia_loop_config(link, bt3c_check_config_notpicky, NULL))
 		goto found_port;
 
diff --git a/drivers/bluetooth/btbcm.c b/drivers/bluetooth/btbcm.c
index 9ab6cfbb831d..cc4bdefa6648 100644
--- a/drivers/bluetooth/btbcm.c
+++ b/drivers/bluetooth/btbcm.c
@@ -287,6 +287,37 @@ static struct sk_buff *btbcm_read_usb_product(struct hci_dev *hdev)
 	return skb;
 }
 
+static int btbcm_read_info(struct hci_dev *hdev)
+{
+	struct sk_buff *skb;
+
+	/* Read Verbose Config Version Info */
+	skb = btbcm_read_verbose_config(hdev);
+	if (IS_ERR(skb))
+		return PTR_ERR(skb);
+
+	BT_INFO("%s: BCM: chip id %u", hdev->name, skb->data[1]);
+	kfree_skb(skb);
+
+	/* Read Controller Features */
+	skb = btbcm_read_controller_features(hdev);
+	if (IS_ERR(skb))
+		return PTR_ERR(skb);
+
+	BT_INFO("%s: BCM: features 0x%2.2x", hdev->name, skb->data[1]);
+	kfree_skb(skb);
+
+	/* Read Local Name */
+	skb = btbcm_read_local_name(hdev);
+	if (IS_ERR(skb))
+		return PTR_ERR(skb);
+
+	BT_INFO("%s: %s", hdev->name, (char *)(skb->data + 1));
+	kfree_skb(skb);
+
+	return 0;
+}
+
 static const struct {
 	u16 subver;
 	const char *name;
@@ -322,13 +353,10 @@ int btbcm_initialize(struct hci_dev *hdev, char *fw_name, size_t len)
 	subver = le16_to_cpu(ver->lmp_subver);
 	kfree_skb(skb);
 
-	/* Read Verbose Config Version Info */
-	skb = btbcm_read_verbose_config(hdev);
-	if (IS_ERR(skb))
-		return PTR_ERR(skb);
-
-	BT_INFO("%s: BCM: chip id %u", hdev->name, skb->data[1]);
-	kfree_skb(skb);
+	/* Read controller information */
+	err = btbcm_read_info(hdev);
+	if (err)
+		return err;
 
 	switch ((rev & 0xf000) >> 12) {
 	case 0:
@@ -431,29 +459,10 @@ int btbcm_setup_patchram(struct hci_dev *hdev)
 	subver = le16_to_cpu(ver->lmp_subver);
 	kfree_skb(skb);
 
-	/* Read Verbose Config Version Info */
-	skb = btbcm_read_verbose_config(hdev);
-	if (IS_ERR(skb))
-		return PTR_ERR(skb);
-
-	BT_INFO("%s: BCM: chip id %u", hdev->name, skb->data[1]);
-	kfree_skb(skb);
-
-	/* Read Controller Features */
-	skb = btbcm_read_controller_features(hdev);
-	if (IS_ERR(skb))
-		return PTR_ERR(skb);
-
-	BT_INFO("%s: BCM: features 0x%2.2x", hdev->name, skb->data[1]);
-	kfree_skb(skb);
-
-	/* Read Local Name */
-	skb = btbcm_read_local_name(hdev);
-	if (IS_ERR(skb))
-		return PTR_ERR(skb);
-
-	BT_INFO("%s: %s", hdev->name, (char *)(skb->data + 1));
-	kfree_skb(skb);
+	/* Read controller information */
+	err = btbcm_read_info(hdev);
+	if (err)
+		return err;
 
 	switch ((rev & 0xf000) >> 12) {
 	case 0:
diff --git a/drivers/bluetooth/btmrvl_sdio.c b/drivers/bluetooth/btmrvl_sdio.c
index eb794f08b238..03341ce98c32 100644
--- a/drivers/bluetooth/btmrvl_sdio.c
+++ b/drivers/bluetooth/btmrvl_sdio.c
@@ -1455,7 +1455,8 @@ done:
 	fw_dump_ptr = fw_dump_data;
 
 	/* Dump all the memory data into single file, a userspace script will
-	   be used to split all the memory data to multiple files*/
+	 * be used to split all the memory data to multiple files
+	 */
 	BT_INFO("== btmrvl firmware dump to /sys/class/devcoredump start");
 	for (idx = 0; idx < dump_num; idx++) {
 		struct memory_type_mapping *entry = &mem_type_mapping_tbl[idx];
@@ -1482,7 +1483,8 @@ done:
 	}
 
 	/* fw_dump_data will be free in device coredump release function
-	   after 5 min*/
+	 * after 5 min
+	 */
 	dev_coredumpv(&card->func->dev, fw_dump_data, fw_dump_len, GFP_KERNEL);
 	BT_INFO("== btmrvl firmware dump to /sys/class/devcoredump end");
 }
diff --git a/drivers/bluetooth/btqca.c b/drivers/bluetooth/btqca.c
index 28afd5d585f9..0bbdfcef2aa8 100644
--- a/drivers/bluetooth/btqca.c
+++ b/drivers/bluetooth/btqca.c
@@ -81,7 +81,7 @@ static int rome_patch_ver_req(struct hci_dev *hdev, u32 *rome_version)
 	 * and lower 2 bytes from patch will be used.
 	 */
 	*rome_version = (le32_to_cpu(ver->soc_id) << 16) |
-		        (le16_to_cpu(ver->rome_ver) & 0x0000ffff);
+			(le16_to_cpu(ver->rome_ver) & 0x0000ffff);
 
 out:
 	kfree_skb(skb);
diff --git a/drivers/bluetooth/btrtl.c b/drivers/bluetooth/btrtl.c
index 8279094dd713..d9a99b4302ea 100644
--- a/drivers/bluetooth/btrtl.c
+++ b/drivers/bluetooth/btrtl.c
@@ -279,6 +279,8 @@ static int rtl_load_config(struct hci_dev *hdev, const char *name, u8 **buff)
 		return ret;
 	ret = fw->size;
 	*buff = kmemdup(fw->data, ret, GFP_KERNEL);
+	if (!*buff)
+		ret = -ENOMEM;
 
 	release_firmware(fw);
 
diff --git a/drivers/bluetooth/btsdio.c b/drivers/bluetooth/btsdio.c
index 1cb958e199eb..c8e945d19ffe 100644
--- a/drivers/bluetooth/btsdio.c
+++ b/drivers/bluetooth/btsdio.c
@@ -144,7 +144,8 @@ static int btsdio_rx_packet(struct btsdio_data *data)
 	if (!skb) {
 		/* Out of memory. Prepare a read retry and just
 		 * return with the expectation that the next time
-		 * we're called we'll have more memory. */
+		 * we're called we'll have more memory.
+		 */
 		return -ENOMEM;
 	}
 
diff --git a/drivers/bluetooth/btuart_cs.c b/drivers/bluetooth/btuart_cs.c
index 7df79bb12350..310e9c2e09b6 100644
--- a/drivers/bluetooth/btuart_cs.c
+++ b/drivers/bluetooth/btuart_cs.c
@@ -614,14 +614,16 @@ static int btuart_config(struct pcmcia_device *link)
 	int try;
 
 	/* First pass: look for a config entry that looks normal.
-	   Two tries: without IO aliases, then with aliases */
+	 * Two tries: without IO aliases, then with aliases
+	 */
 	for (try = 0; try < 2; try++)
 		if (!pcmcia_loop_config(link, btuart_check_config, &try))
 			goto found_port;
 
 	/* Second pass: try to find an entry that isn't picky about
-	   its base address, then try to grab any standard serial port
-	   address, and finally try to get any free port. */
+	 * its base address, then try to grab any standard serial port
+	 * address, and finally try to get any free port.
+	 */
 	if (!pcmcia_loop_config(link, btuart_check_config_notpicky, NULL))
 		goto found_port;
 
diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c
index fa24d693af24..7a5c06aaa181 100644
--- a/drivers/bluetooth/btusb.c
+++ b/drivers/bluetooth/btusb.c
@@ -66,6 +66,7 @@ static struct usb_driver btusb_driver;
 #define BTUSB_BCM2045		0x40000
 #define BTUSB_IFNUM_2		0x80000
 #define BTUSB_CW6622		0x100000
+#define BTUSB_BCM_NO_PRODID	0x200000
 
 static const struct usb_device_id btusb_table[] = {
 	/* Generic Bluetooth USB device */
@@ -131,7 +132,8 @@ static const struct usb_device_id btusb_table[] = {
 	{ USB_DEVICE(0x19ff, 0x0239), .driver_info = BTUSB_BCM_PATCHRAM },
 
 	/* Broadcom BCM43142A0 (Foxconn/Lenovo) */
-	{ USB_DEVICE(0x105b, 0xe065), .driver_info = BTUSB_BCM_PATCHRAM },
+	{ USB_VENDOR_AND_INTERFACE_INFO(0x105b, 0xff, 0x01, 0x01),
+	  .driver_info = BTUSB_BCM_PATCHRAM },
 
 	/* Broadcom BCM920703 (HTC Vive) */
 	{ USB_VENDOR_AND_INTERFACE_INFO(0x0bb4, 0xff, 0x01, 0x01),
@@ -169,6 +171,10 @@ static const struct usb_device_id btusb_table[] = {
 	{ USB_VENDOR_AND_INTERFACE_INFO(0x0930, 0xff, 0x01, 0x01),
 	  .driver_info = BTUSB_BCM_PATCHRAM },
 
+	/* Broadcom devices with missing product id */
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x0000, 0x0000, 0xff, 0x01, 0x01),
+	  .driver_info = BTUSB_BCM_PATCHRAM | BTUSB_BCM_NO_PRODID },
+
 	/* Intel Bluetooth USB Bootloader (RAM module) */
 	{ USB_DEVICE(0x8087, 0x0a5a),
 	  .driver_info = BTUSB_INTEL_BOOT | BTUSB_BROKEN_ISOC },
@@ -268,6 +274,7 @@ static const struct usb_device_id blacklist_table[] = {
 	{ USB_DEVICE(0x0489, 0xe092), .driver_info = BTUSB_QCA_ROME },
 	{ USB_DEVICE(0x0489, 0xe0a2), .driver_info = BTUSB_QCA_ROME },
 	{ USB_DEVICE(0x04ca, 0x3011), .driver_info = BTUSB_QCA_ROME },
+	{ USB_DEVICE(0x04ca, 0x3016), .driver_info = BTUSB_QCA_ROME },
 
 	/* Broadcom BCM2035 */
 	{ USB_DEVICE(0x0a5c, 0x2009), .driver_info = BTUSB_BCM92035 },
@@ -357,6 +364,7 @@ static const struct usb_device_id blacklist_table[] = {
 	{ USB_DEVICE(0x13d3, 0x3410), .driver_info = BTUSB_REALTEK },
 	{ USB_DEVICE(0x13d3, 0x3416), .driver_info = BTUSB_REALTEK },
 	{ USB_DEVICE(0x13d3, 0x3459), .driver_info = BTUSB_REALTEK },
+	{ USB_DEVICE(0x13d3, 0x3494), .driver_info = BTUSB_REALTEK },
 
 	/* Additional Realtek 8821AE Bluetooth devices */
 	{ USB_DEVICE(0x0b05, 0x17dc), .driver_info = BTUSB_REALTEK },
@@ -656,7 +664,8 @@ static void btusb_intr_complete(struct urb *urb)
 	err = usb_submit_urb(urb, GFP_ATOMIC);
 	if (err < 0) {
 		/* -EPERM: urb is being killed;
-		 * -ENODEV: device got disconnected */
+		 * -ENODEV: device got disconnected
+		 */
 		if (err != -EPERM && err != -ENODEV)
 			BT_ERR("%s urb %p failed to resubmit (%d)",
 			       hdev->name, urb, -err);
@@ -745,7 +754,8 @@ static void btusb_bulk_complete(struct urb *urb)
 	err = usb_submit_urb(urb, GFP_ATOMIC);
 	if (err < 0) {
 		/* -EPERM: urb is being killed;
-		 * -ENODEV: device got disconnected */
+		 * -ENODEV: device got disconnected
+		 */
 		if (err != -EPERM && err != -ENODEV)
 			BT_ERR("%s urb %p failed to resubmit (%d)",
 			       hdev->name, urb, -err);
@@ -840,7 +850,8 @@ static void btusb_isoc_complete(struct urb *urb)
 	err = usb_submit_urb(urb, GFP_ATOMIC);
 	if (err < 0) {
 		/* -EPERM: urb is being killed;
-		 * -ENODEV: device got disconnected */
+		 * -ENODEV: device got disconnected
+		 */
 		if (err != -EPERM && err != -ENODEV)
 			BT_ERR("%s urb %p failed to resubmit (%d)",
 			       hdev->name, urb, -err);
@@ -952,7 +963,8 @@ static void btusb_diag_complete(struct urb *urb)
 	err = usb_submit_urb(urb, GFP_ATOMIC);
 	if (err < 0) {
 		/* -EPERM: urb is being killed;
-		 * -ENODEV: device got disconnected */
+		 * -ENODEV: device got disconnected
+		 */
 		if (err != -EPERM && err != -ENODEV)
 			BT_ERR("%s urb %p failed to resubmit (%d)",
 			       hdev->name, urb, -err);
@@ -1076,6 +1088,10 @@ static int btusb_open(struct hci_dev *hdev)
 	}
 
 	data->intf->needs_remote_wakeup = 1;
+	/* device specific wakeup source enabled and required for USB
+	 * remote wakeup while host is suspended
+	 */
+	device_wakeup_enable(&data->udev->dev);
 
 	if (test_and_set_bit(BTUSB_INTR_RUNNING, &data->flags))
 		goto done;
@@ -1139,6 +1155,7 @@ static int btusb_close(struct hci_dev *hdev)
 		goto failed;
 
 	data->intf->needs_remote_wakeup = 0;
+	device_wakeup_disable(&data->udev->dev);
 	usb_autopm_put_interface(data->intf);
 
 failed:
@@ -2892,11 +2909,25 @@ static int btusb_probe(struct usb_interface *intf,
 	if (id->driver_info == BTUSB_IGNORE)
 		return -ENODEV;
 
+	if (id->driver_info & BTUSB_BCM_NO_PRODID) {
+		struct usb_device *udev = interface_to_usbdev(intf);
+
+		/* For the broken Broadcom devices that show 0000:0000
+		 * as USB vendor and product information, check that the
+		 * manufacturer string identifies them as Broadcom based
+		 * devices.
+		 */
+		if (!udev->manufacturer ||
+		    strcmp(udev->manufacturer, "Broadcom Corp"))
+			return -ENODEV;
+	}
+
 	if (id->driver_info & BTUSB_ATH3012) {
 		struct usb_device *udev = interface_to_usbdev(intf);
 
 		/* Old firmware would otherwise let ath3k driver load
-		 * patch and sysconfig files */
+		 * patch and sysconfig files
+		 */
 		if (le16_to_cpu(udev->descriptor.bcdDevice) <= 0x0001)
 			return -ENODEV;
 	}
@@ -3067,6 +3098,12 @@ static int btusb_probe(struct usb_interface *intf,
 	if (id->driver_info & BTUSB_QCA_ROME) {
 		data->setup_on_usb = btusb_setup_qca;
 		hdev->set_bdaddr = btusb_set_bdaddr_ath3012;
+
+		/* QCA Rome devices lose their updated firmware over suspend,
+		 * but the USB hub doesn't notice any status change.
+		 * Explicitly request a device reset on resume.
+		 */
+		set_bit(BTUSB_RESET_RESUME, &data->flags);
 	}
 
 #ifdef CONFIG_BT_HCIBTUSB_RTL
@@ -3259,13 +3296,28 @@ static void play_deferred(struct btusb_data *data)
 	int err;
 
 	while ((urb = usb_get_from_anchor(&data->deferred))) {
+		usb_anchor_urb(urb, &data->tx_anchor);
+
 		err = usb_submit_urb(urb, GFP_ATOMIC);
-		if (err < 0)
+		if (err < 0) {
+			if (err != -EPERM && err != -ENODEV)
+				BT_ERR("%s urb %p submission failed (%d)",
+				       data->hdev->name, urb, -err);
+			kfree(urb->setup_packet);
+			usb_unanchor_urb(urb);
+			usb_free_urb(urb);
 			break;
+		}
 
 		data->tx_in_flight++;
+		usb_free_urb(urb);
+	}
+
+	/* Cleanup the rest deferred urbs. */
+	while ((urb = usb_get_from_anchor(&data->deferred))) {
+		kfree(urb->setup_packet);
+		usb_free_urb(urb);
 	}
-	usb_scuttle_anchored_urbs(&data->deferred);
 }
 
 static int btusb_resume(struct usb_interface *intf)
diff --git a/drivers/bluetooth/btwilink.c b/drivers/bluetooth/btwilink.c
index 85a3978b064f..5ef8000f90a9 100644
--- a/drivers/bluetooth/btwilink.c
+++ b/drivers/bluetooth/btwilink.c
@@ -93,8 +93,7 @@ static void st_reg_completion_cb(void *priv_data, int data)
 	complete(&lhst->wait_reg_completion);
 }
 
-/* Called by Shared Transport layer when receive data is
- * available */
+/* Called by Shared Transport layer when receive data is available */
 static long st_receive(void *priv_data, struct sk_buff *skb)
 {
 	struct ti_st *lhst = priv_data;
@@ -198,7 +197,8 @@ static int ti_st_open(struct hci_dev *hdev)
 		}
 
 		/* Is ST registration callback
-		 * called with ERROR status? */
+		 * called with ERROR status?
+		 */
 		if (hst->reg_status != 0) {
 			BT_ERR("ST registration completed with invalid "
 					"status %d", hst->reg_status);
@@ -276,7 +276,7 @@ static int ti_st_send_frame(struct hci_dev *hdev, struct sk_buff *skb)
 
 static int bt_ti_probe(struct platform_device *pdev)
 {
-	static struct ti_st *hst;
+	struct ti_st *hst;
 	struct hci_dev *hdev;
 	int err;
 
diff --git a/drivers/bluetooth/hci_bcm.c b/drivers/bluetooth/hci_bcm.c
index 6a662d0161b4..e2540113d0da 100644
--- a/drivers/bluetooth/hci_bcm.c
+++ b/drivers/bluetooth/hci_bcm.c
@@ -27,6 +27,8 @@
 #include <linux/firmware.h>
 #include <linux/module.h>
 #include <linux/acpi.h>
+#include <linux/of.h>
+#include <linux/property.h>
 #include <linux/platform_device.h>
 #include <linux/clk.h>
 #include <linux/gpio/consumer.h>
@@ -34,6 +36,7 @@
 #include <linux/interrupt.h>
 #include <linux/dmi.h>
 #include <linux/pm_runtime.h>
+#include <linux/serdev.h>
 
 #include <net/bluetooth/bluetooth.h>
 #include <net/bluetooth/hci_core.h>
@@ -41,11 +44,15 @@
 #include "btbcm.h"
 #include "hci_uart.h"
 
+#define BCM_NULL_PKT 0x00
+#define BCM_NULL_SIZE 0
+
 #define BCM_LM_DIAG_PKT 0x07
 #define BCM_LM_DIAG_SIZE 63
 
 #define BCM_AUTOSUSPEND_DELAY	5000 /* default autosleep delay */
 
+/* platform device driver resources */
 struct bcm_device {
 	struct list_head	list;
 
@@ -59,6 +66,7 @@ struct bcm_device {
 	bool			clk_enabled;
 
 	u32			init_speed;
+	u32			oper_speed;
 	int			irq;
 	u8			irq_polarity;
 
@@ -68,6 +76,12 @@ struct bcm_device {
 #endif
 };
 
+/* serdev driver resources */
+struct bcm_serdev {
+	struct hci_uart hu;
+};
+
+/* generic bcm uart resources */
 struct bcm_data {
 	struct sk_buff		*rx_skb;
 	struct sk_buff_head	txq;
@@ -79,6 +93,14 @@ struct bcm_data {
 static DEFINE_MUTEX(bcm_device_lock);
 static LIST_HEAD(bcm_device_list);
 
+static inline void host_set_baudrate(struct hci_uart *hu, unsigned int speed)
+{
+	if (hu->serdev)
+		serdev_device_set_baudrate(hu->serdev, speed);
+	else
+		hci_uart_set_baudrate(hu, speed);
+}
+
 static int bcm_set_baudrate(struct hci_uart *hu, unsigned int speed)
 {
 	struct hci_dev *hdev = hu->hdev;
@@ -176,7 +198,7 @@ static irqreturn_t bcm_host_wake(int irq, void *data)
 static int bcm_request_irq(struct bcm_data *bcm)
 {
 	struct bcm_device *bdev = bcm->dev;
-	int err = 0;
+	int err;
 
 	/* If this is not a platform device, do not enable PM functionalities */
 	mutex_lock(&bcm_device_lock);
@@ -185,21 +207,23 @@ static int bcm_request_irq(struct bcm_data *bcm)
 		goto unlock;
 	}
 
-	if (bdev->irq > 0) {
-		err = devm_request_irq(&bdev->pdev->dev, bdev->irq,
-				       bcm_host_wake, IRQF_TRIGGER_RISING,
-				       "host_wake", bdev);
-		if (err)
-			goto unlock;
+	if (bdev->irq <= 0) {
+		err = -EOPNOTSUPP;
+		goto unlock;
+	}
+
+	err = devm_request_irq(&bdev->pdev->dev, bdev->irq, bcm_host_wake,
+			       IRQF_TRIGGER_RISING, "host_wake", bdev);
+	if (err)
+		goto unlock;
 
-		device_init_wakeup(&bdev->pdev->dev, true);
+	device_init_wakeup(&bdev->pdev->dev, true);
 
-		pm_runtime_set_autosuspend_delay(&bdev->pdev->dev,
-						 BCM_AUTOSUSPEND_DELAY);
-		pm_runtime_use_autosuspend(&bdev->pdev->dev);
-		pm_runtime_set_active(&bdev->pdev->dev);
-		pm_runtime_enable(&bdev->pdev->dev);
-	}
+	pm_runtime_set_autosuspend_delay(&bdev->pdev->dev,
+					 BCM_AUTOSUSPEND_DELAY);
+	pm_runtime_use_autosuspend(&bdev->pdev->dev);
+	pm_runtime_set_active(&bdev->pdev->dev);
+	pm_runtime_enable(&bdev->pdev->dev);
 
 unlock:
 	mutex_unlock(&bcm_device_lock);
@@ -287,6 +311,14 @@ static int bcm_open(struct hci_uart *hu)
 
 	hu->priv = bcm;
 
+	/* If this is a serdev defined device, then only use
+	 * serdev open primitive and skip the rest.
+	 */
+	if (hu->serdev) {
+		serdev_device_open(hu->serdev);
+		goto out;
+	}
+
 	if (!hu->tty->dev)
 		goto out;
 
@@ -301,6 +333,7 @@ static int bcm_open(struct hci_uart *hu)
 		if (hu->tty->dev->parent == dev->pdev->dev.parent) {
 			bcm->dev = dev;
 			hu->init_speed = dev->init_speed;
+			hu->oper_speed = dev->oper_speed;
 #ifdef CONFIG_PM
 			dev->hu = hu;
 #endif
@@ -321,6 +354,12 @@ static int bcm_close(struct hci_uart *hu)
 
 	bt_dev_dbg(hu->hdev, "hu %p", hu);
 
+	/* If this is a serdev defined device, only use serdev
+	 * close primitive and then continue as usual.
+	 */
+	if (hu->serdev)
+		serdev_device_close(hu->serdev);
+
 	/* Protect bcm->dev against removal of the device or driver */
 	mutex_lock(&bcm_device_lock);
 	if (bcm_device_exists(bdev)) {
@@ -396,7 +435,7 @@ static int bcm_setup(struct hci_uart *hu)
 		speed = 0;
 
 	if (speed)
-		hci_uart_set_baudrate(hu, speed);
+		host_set_baudrate(hu, speed);
 
 	/* Operational speed if any */
 	if (hu->oper_speed)
@@ -409,7 +448,7 @@ static int bcm_setup(struct hci_uart *hu)
 	if (speed) {
 		err = bcm_set_baudrate(hu, speed);
 		if (!err)
-			hci_uart_set_baudrate(hu, speed);
+			host_set_baudrate(hu, speed);
 	}
 
 finalize:
@@ -432,11 +471,19 @@ finalize:
 	.lsize = 0, \
 	.maxlen = BCM_LM_DIAG_SIZE
 
+#define BCM_RECV_NULL \
+	.type = BCM_NULL_PKT, \
+	.hlen = BCM_NULL_SIZE, \
+	.loff = 0, \
+	.lsize = 0, \
+	.maxlen = BCM_NULL_SIZE
+
 static const struct h4_recv_pkt bcm_recv_pkts[] = {
 	{ H4_RECV_ACL,      .recv = hci_recv_frame },
 	{ H4_RECV_SCO,      .recv = hci_recv_frame },
 	{ H4_RECV_EVENT,    .recv = hci_recv_frame },
 	{ BCM_RECV_LM_DIAG, .recv = hci_recv_diag  },
+	{ BCM_RECV_NULL,    .recv = hci_recv_diag  },
 };
 
 static int bcm_recv(struct hci_uart *hu, const void *data, int count)
@@ -697,8 +744,10 @@ static int bcm_resource(struct acpi_resource *ares, void *data)
 
 	case ACPI_RESOURCE_TYPE_SERIAL_BUS:
 		sb = &ares->data.uart_serial_bus;
-		if (sb->type == ACPI_RESOURCE_SERIAL_TYPE_UART)
+		if (sb->type == ACPI_RESOURCE_SERIAL_TYPE_UART) {
 			dev->init_speed = sb->default_baud_rate;
+			dev->oper_speed = 4000000;
+		}
 		break;
 
 	default:
@@ -851,7 +900,6 @@ static const struct hci_uart_proto bcm_proto = {
 	.name		= "Broadcom",
 	.manufacturer	= 15,
 	.init_speed	= 115200,
-	.oper_speed	= 4000000,
 	.open		= bcm_open,
 	.close		= bcm_close,
 	.flush		= bcm_flush,
@@ -901,9 +949,57 @@ static struct platform_driver bcm_driver = {
 	},
 };
 
+static int bcm_serdev_probe(struct serdev_device *serdev)
+{
+	struct bcm_serdev *bcmdev;
+	u32 speed;
+	int err;
+
+	bcmdev = devm_kzalloc(&serdev->dev, sizeof(*bcmdev), GFP_KERNEL);
+	if (!bcmdev)
+		return -ENOMEM;
+
+	bcmdev->hu.serdev = serdev;
+	serdev_device_set_drvdata(serdev, bcmdev);
+
+	err = device_property_read_u32(&serdev->dev, "max-speed", &speed);
+	if (!err)
+		bcmdev->hu.oper_speed = speed;
+
+	return hci_uart_register_device(&bcmdev->hu, &bcm_proto);
+}
+
+static void bcm_serdev_remove(struct serdev_device *serdev)
+{
+	struct bcm_serdev *bcmdev = serdev_device_get_drvdata(serdev);
+
+	hci_uart_unregister_device(&bcmdev->hu);
+}
+
+#ifdef CONFIG_OF
+static const struct of_device_id bcm_bluetooth_of_match[] = {
+	{ .compatible = "brcm,bcm43438-bt" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, bcm_bluetooth_of_match);
+#endif
+
+static struct serdev_device_driver bcm_serdev_driver = {
+	.probe = bcm_serdev_probe,
+	.remove = bcm_serdev_remove,
+	.driver = {
+		.name = "hci_uart_bcm",
+		.of_match_table = of_match_ptr(bcm_bluetooth_of_match),
+	},
+};
+
 int __init bcm_init(void)
 {
+	/* For now, we need to keep both platform device
+	 * driver (ACPI generated) and serdev driver (DT).
+	 */
 	platform_driver_register(&bcm_driver);
+	serdev_device_driver_register(&bcm_serdev_driver);
 
 	return hci_uart_register_proto(&bcm_proto);
 }
@@ -911,6 +1007,7 @@ int __init bcm_init(void)
 int __exit bcm_deinit(void)
 {
 	platform_driver_unregister(&bcm_driver);
+	serdev_device_driver_unregister(&bcm_serdev_driver);
 
 	return hci_uart_unregister_proto(&bcm_proto);
 }
diff --git a/drivers/bluetooth/hci_h4.c b/drivers/bluetooth/hci_h4.c
index 4e328d7d47bb..3b82a87224a9 100644
--- a/drivers/bluetooth/hci_h4.c
+++ b/drivers/bluetooth/hci_h4.c
@@ -172,7 +172,7 @@ struct sk_buff *h4_recv_buf(struct hci_dev *hdev, struct sk_buff *skb,
 			    const struct h4_recv_pkt *pkts, int pkts_count)
 {
 	struct hci_uart *hu = hci_get_drvdata(hdev);
-	u8 alignment = hu->alignment;
+	u8 alignment = hu->alignment ? hu->alignment : 1;
 
 	while (count) {
 		int i, len;
diff --git a/drivers/bluetooth/hci_ldisc.c b/drivers/bluetooth/hci_ldisc.c
index 8397b716fa65..a746627e784e 100644
--- a/drivers/bluetooth/hci_ldisc.c
+++ b/drivers/bluetooth/hci_ldisc.c
@@ -457,7 +457,8 @@ static int hci_uart_tty_open(struct tty_struct *tty)
 	BT_DBG("tty %p", tty);
 
 	/* Error if the tty has no write op instead of leaving an exploitable
-	   hole */
+	 * hole
+	 */
 	if (tty->ops->write == NULL)
 		return -EOPNOTSUPP;
 
diff --git a/drivers/bluetooth/hci_ll.c b/drivers/bluetooth/hci_ll.c
index c982943f0747..424c15aa7bb7 100644
--- a/drivers/bluetooth/hci_ll.c
+++ b/drivers/bluetooth/hci_ll.c
@@ -622,7 +622,8 @@ static int download_firmware(struct ll_device *lldev)
 			cmd = (struct hci_command *)action_ptr;
 			if (cmd->opcode == 0xff36) {
 				/* ignore remote change
-				 * baud rate HCI VS command */
+				 * baud rate HCI VS command
+				 */
 				bt_dev_warn(lldev->hu.hdev, "change remote baud rate command in firmware");
 				break;
 			}
@@ -742,14 +743,8 @@ static int hci_ti_probe(struct serdev_device *serdev)
 static void hci_ti_remove(struct serdev_device *serdev)
 {
 	struct ll_device *lldev = serdev_device_get_drvdata(serdev);
-	struct hci_uart *hu = &lldev->hu;
-	struct hci_dev *hdev = hu->hdev;
 
-	cancel_work_sync(&hu->write_work);
-
-	hci_unregister_dev(hdev);
-	hci_free_dev(hdev);
-	hu->proto->close(hu);
+	hci_uart_unregister_device(&lldev->hu);
 }
 
 static const struct of_device_id hci_ti_of_match[] = {
diff --git a/drivers/bluetooth/hci_nokia.c b/drivers/bluetooth/hci_nokia.c
index 181a15b549e5..3539fd03f47e 100644
--- a/drivers/bluetooth/hci_nokia.c
+++ b/drivers/bluetooth/hci_nokia.c
@@ -767,16 +767,8 @@ static int nokia_bluetooth_serdev_probe(struct serdev_device *serdev)
 static void nokia_bluetooth_serdev_remove(struct serdev_device *serdev)
 {
 	struct nokia_bt_dev *btdev = serdev_device_get_drvdata(serdev);
-	struct hci_uart *hu = &btdev->hu;
-	struct hci_dev *hdev = hu->hdev;
 
-	cancel_work_sync(&hu->write_work);
-
-	hci_unregister_dev(hdev);
-	hci_free_dev(hdev);
-	hu->proto->close(hu);
-
-	pm_runtime_disable(&btdev->serdev->dev);
+	hci_uart_unregister_device(&btdev->hu);
 }
 
 static int nokia_bluetooth_runtime_suspend(struct device *dev)
diff --git a/drivers/bluetooth/hci_serdev.c b/drivers/bluetooth/hci_serdev.c
index aea930101dd2..b725ac4f7ff6 100644
--- a/drivers/bluetooth/hci_serdev.c
+++ b/drivers/bluetooth/hci_serdev.c
@@ -354,3 +354,16 @@ err_alloc:
 	return err;
 }
 EXPORT_SYMBOL_GPL(hci_uart_register_device);
+
+void hci_uart_unregister_device(struct hci_uart *hu)
+{
+	struct hci_dev *hdev = hu->hdev;
+
+	hci_unregister_dev(hdev);
+	hci_free_dev(hdev);
+
+	cancel_work_sync(&hu->write_work);
+
+	hu->proto->close(hu);
+}
+EXPORT_SYMBOL_GPL(hci_uart_unregister_device);
diff --git a/drivers/bluetooth/hci_uart.h b/drivers/bluetooth/hci_uart.h
index c6e9e1cf63f8..d9cd95d81149 100644
--- a/drivers/bluetooth/hci_uart.h
+++ b/drivers/bluetooth/hci_uart.h
@@ -112,6 +112,7 @@ struct hci_uart {
 int hci_uart_register_proto(const struct hci_uart_proto *p);
 int hci_uart_unregister_proto(const struct hci_uart_proto *p);
 int hci_uart_register_device(struct hci_uart *hu, const struct hci_uart_proto *p);
+void hci_uart_unregister_device(struct hci_uart *hu);
 
 int hci_uart_tx_wakeup(struct hci_uart *hu);
 int hci_uart_init_ready(struct hci_uart *hu);
diff --git a/drivers/bus/Kconfig b/drivers/bus/Kconfig
index 2408ea38a39c..ae3d8f3444b9 100644
--- a/drivers/bus/Kconfig
+++ b/drivers/bus/Kconfig
@@ -132,7 +132,7 @@ config SIMPLE_PM_BUS
 
 config SUNXI_RSB
 	tristate "Allwinner sunXi Reduced Serial Bus Driver"
-	  default MACH_SUN8I || MACH_SUN9I
+	  default MACH_SUN8I || MACH_SUN9I || ARM64
 	  depends on ARCH_SUNXI
 	  select REGMAP
 	  help
diff --git a/drivers/bus/arm-cci.c b/drivers/bus/arm-cci.c
index c49da15d9790..3c29d36702a8 100644
--- a/drivers/bus/arm-cci.c
+++ b/drivers/bus/arm-cci.c
@@ -2124,8 +2124,8 @@ int notrace __cci_control_port_by_device(struct device_node *dn, bool enable)
 		return -ENODEV;
 
 	port = __cci_ace_get_port(dn, ACE_LITE_PORT);
-	if (WARN_ONCE(port < 0, "node %s ACE lite port look-up failure\n",
-				dn->full_name))
+	if (WARN_ONCE(port < 0, "node %pOF ACE lite port look-up failure\n",
+				dn))
 		return -ENODEV;
 	cci_port_control(port, enable);
 	return 0;
@@ -2200,14 +2200,14 @@ static int cci_probe_ports(struct device_node *np)
 
 		if (of_property_read_string(cp, "interface-type",
 					&match_str)) {
-			WARN(1, "node %s missing interface-type property\n",
-				  cp->full_name);
+			WARN(1, "node %pOF missing interface-type property\n",
+				  cp);
 			continue;
 		}
 		is_ace = strcmp(match_str, "ace") == 0;
 		if (!is_ace && strcmp(match_str, "ace-lite")) {
-			WARN(1, "node %s containing invalid interface-type property, skipping it\n",
-					cp->full_name);
+			WARN(1, "node %pOF containing invalid interface-type property, skipping it\n",
+					cp);
 			continue;
 		}
 
diff --git a/drivers/bus/imx-weim.c b/drivers/bus/imx-weim.c
index 4bd361d64270..3d56ebcda720 100644
--- a/drivers/bus/imx-weim.c
+++ b/drivers/bus/imx-weim.c
@@ -156,8 +156,8 @@ static int __init weim_parse_dt(struct platform_device *pdev,
 
 		ret = weim_timing_setup(child, base, devtype);
 		if (ret)
-			dev_warn(&pdev->dev, "%s set timing failed.\n",
-				child->full_name);
+			dev_warn(&pdev->dev, "%pOF set timing failed.\n",
+				child);
 		else
 			have_child = 1;
 	}
@@ -166,8 +166,8 @@ static int __init weim_parse_dt(struct platform_device *pdev,
 		ret = of_platform_default_populate(pdev->dev.of_node,
 						   NULL, &pdev->dev);
 	if (ret)
-		dev_err(&pdev->dev, "%s fail to create devices.\n",
-			pdev->dev.of_node->full_name);
+		dev_err(&pdev->dev, "%pOF fail to create devices.\n",
+			pdev->dev.of_node);
 	return ret;
 }
 
diff --git a/drivers/bus/omap-ocp2scp.c b/drivers/bus/omap-ocp2scp.c
index bf500e0e7362..77791f3dcfc6 100644
--- a/drivers/bus/omap-ocp2scp.c
+++ b/drivers/bus/omap-ocp2scp.c
@@ -70,8 +70,10 @@ static int omap_ocp2scp_probe(struct platform_device *pdev)
 	if (!of_device_is_compatible(np, "ti,am437x-ocp2scp")) {
 		res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 		regs = devm_ioremap_resource(&pdev->dev, res);
-		if (IS_ERR(regs))
-			goto err0;
+		if (IS_ERR(regs)) {
+			ret = PTR_ERR(regs);
+			goto err1;
+		}
 
 		pm_runtime_get_sync(&pdev->dev);
 		reg = readl_relaxed(regs + OCP2SCP_TIMING);
@@ -83,6 +85,9 @@ static int omap_ocp2scp_probe(struct platform_device *pdev)
 
 	return 0;
 
+err1:
+	pm_runtime_disable(&pdev->dev);
+
 err0:
 	device_for_each_child(&pdev->dev, NULL, ocp2scp_remove_devices);
 
diff --git a/drivers/bus/sunxi-rsb.c b/drivers/bus/sunxi-rsb.c
index 795c9d9c96a6..328ca93781cf 100644
--- a/drivers/bus/sunxi-rsb.c
+++ b/drivers/bus/sunxi-rsb.c
@@ -556,20 +556,20 @@ static int of_rsb_register_devices(struct sunxi_rsb *rsb)
 
 	/* Runtime addresses for all slaves should be set first */
 	for_each_available_child_of_node(np, child) {
-		dev_dbg(dev, "setting child %s runtime address\n",
-			child->full_name);
+		dev_dbg(dev, "setting child %pOF runtime address\n",
+			child);
 
 		ret = of_property_read_u32(child, "reg", &hwaddr);
 		if (ret) {
-			dev_err(dev, "%s: invalid 'reg' property: %d\n",
-				child->full_name, ret);
+			dev_err(dev, "%pOF: invalid 'reg' property: %d\n",
+				child, ret);
 			continue;
 		}
 
 		rtaddr = sunxi_rsb_get_rtaddr(hwaddr);
 		if (!rtaddr) {
-			dev_err(dev, "%s: unknown hardware device address\n",
-				child->full_name);
+			dev_err(dev, "%pOF: unknown hardware device address\n",
+				child);
 			continue;
 		}
 
@@ -586,15 +586,15 @@ static int of_rsb_register_devices(struct sunxi_rsb *rsb)
 		/* send command */
 		ret = _sunxi_rsb_run_xfer(rsb);
 		if (ret)
-			dev_warn(dev, "%s: set runtime address failed: %d\n",
-				 child->full_name, ret);
+			dev_warn(dev, "%pOF: set runtime address failed: %d\n",
+				 child, ret);
 	}
 
 	/* Then we start adding devices and probing them */
 	for_each_available_child_of_node(np, child) {
 		struct sunxi_rsb_device *rdev;
 
-		dev_dbg(dev, "adding child %s\n", child->full_name);
+		dev_dbg(dev, "adding child %pOF\n", child);
 
 		ret = of_property_read_u32(child, "reg", &hwaddr);
 		if (ret)
@@ -606,8 +606,8 @@ static int of_rsb_register_devices(struct sunxi_rsb *rsb)
 
 		rdev = sunxi_rsb_device_create(rsb, child, hwaddr, rtaddr);
 		if (IS_ERR(rdev))
-			dev_err(dev, "failed to add child device %s: %ld\n",
-				child->full_name, PTR_ERR(rdev));
+			dev_err(dev, "failed to add child device %pOF: %ld\n",
+				child, PTR_ERR(rdev));
 	}
 
 	return 0;
diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig
index ccd239ab879f..623714344600 100644
--- a/drivers/char/Kconfig
+++ b/drivers/char/Kconfig
@@ -161,7 +161,7 @@ config VIRTIO_CONSOLE
 	depends on VIRTIO && TTY
 	select HVC_DRIVER
 	help
-	  Virtio console for use with lguest and other hypervisors.
+	  Virtio console for use with hypervisors.
 
 	  Also serves as a general-purpose serial device for data
 	  transfer between the guest and host.  Character devices at
diff --git a/drivers/char/agp/ali-agp.c b/drivers/char/agp/ali-agp.c
index dcbbb4ea3cc1..89527bae4602 100644
--- a/drivers/char/agp/ali-agp.c
+++ b/drivers/char/agp/ali-agp.c
@@ -381,7 +381,7 @@ static void agp_ali_remove(struct pci_dev *pdev)
 	agp_put_bridge(bridge);
 }
 
-static struct pci_device_id agp_ali_pci_table[] = {
+static const struct pci_device_id agp_ali_pci_table[] = {
 	{
 	.class		= (PCI_CLASS_BRIDGE_HOST << 8),
 	.class_mask	= ~0,
diff --git a/drivers/char/agp/amd-k7-agp.c b/drivers/char/agp/amd-k7-agp.c
index 5fbd333e4c6d..b450544dcaf0 100644
--- a/drivers/char/agp/amd-k7-agp.c
+++ b/drivers/char/agp/amd-k7-agp.c
@@ -21,7 +21,7 @@
 #define AMD_TLBFLUSH	0x0c	/* In mmio region (32-bit register) */
 #define AMD_CACHEENTRY	0x10	/* In mmio region (32-bit register) */
 
-static struct pci_device_id agp_amdk7_pci_table[];
+static const struct pci_device_id agp_amdk7_pci_table[];
 
 struct amd_page_map {
 	unsigned long *real;
@@ -508,7 +508,7 @@ static int agp_amdk7_resume(struct pci_dev *pdev)
 #endif /* CONFIG_PM */
 
 /* must be the same order as name table above */
-static struct pci_device_id agp_amdk7_pci_table[] = {
+static const struct pci_device_id agp_amdk7_pci_table[] = {
 	{
 	.class		= (PCI_CLASS_BRIDGE_HOST << 8),
 	.class_mask	= ~0,
diff --git a/drivers/char/agp/amd64-agp.c b/drivers/char/agp/amd64-agp.c
index c99cd19d9147..e50c29c97ca7 100644
--- a/drivers/char/agp/amd64-agp.c
+++ b/drivers/char/agp/amd64-agp.c
@@ -610,7 +610,7 @@ static int agp_amd64_resume(struct pci_dev *pdev)
 
 #endif /* CONFIG_PM */
 
-static struct pci_device_id agp_amd64_pci_table[] = {
+static const struct pci_device_id agp_amd64_pci_table[] = {
 	{
 	.class		= (PCI_CLASS_BRIDGE_HOST << 8),
 	.class_mask	= ~0,
diff --git a/drivers/char/agp/ati-agp.c b/drivers/char/agp/ati-agp.c
index 0b5ec7af2414..88b4cbee4dac 100644
--- a/drivers/char/agp/ati-agp.c
+++ b/drivers/char/agp/ati-agp.c
@@ -540,7 +540,7 @@ static void agp_ati_remove(struct pci_dev *pdev)
 	agp_put_bridge(bridge);
 }
 
-static struct pci_device_id agp_ati_pci_table[] = {
+static const struct pci_device_id agp_ati_pci_table[] = {
 	{
 	.class		= (PCI_CLASS_BRIDGE_HOST << 8),
 	.class_mask	= ~0,
diff --git a/drivers/char/agp/efficeon-agp.c b/drivers/char/agp/efficeon-agp.c
index 533cb6d229b8..7f88490b5479 100644
--- a/drivers/char/agp/efficeon-agp.c
+++ b/drivers/char/agp/efficeon-agp.c
@@ -427,7 +427,7 @@ static int agp_efficeon_resume(struct pci_dev *pdev)
 }
 #endif
 
-static struct pci_device_id agp_efficeon_pci_table[] = {
+static const struct pci_device_id agp_efficeon_pci_table[] = {
 	{
 	.class		= (PCI_CLASS_BRIDGE_HOST << 8),
 	.class_mask	= ~0,
diff --git a/drivers/char/agp/intel-agp.c b/drivers/char/agp/intel-agp.c
index 0a21daed5b62..9e4f27a6cb5a 100644
--- a/drivers/char/agp/intel-agp.c
+++ b/drivers/char/agp/intel-agp.c
@@ -828,7 +828,7 @@ static int agp_intel_resume(struct pci_dev *pdev)
 }
 #endif
 
-static struct pci_device_id agp_intel_pci_table[] = {
+static const struct pci_device_id agp_intel_pci_table[] = {
 #define ID(x)						\
 	{						\
 	.class		= (PCI_CLASS_BRIDGE_HOST << 8),	\
diff --git a/drivers/char/agp/nvidia-agp.c b/drivers/char/agp/nvidia-agp.c
index 6c8d39cb566e..828b34445203 100644
--- a/drivers/char/agp/nvidia-agp.c
+++ b/drivers/char/agp/nvidia-agp.c
@@ -420,7 +420,7 @@ static int agp_nvidia_resume(struct pci_dev *pdev)
 #endif
 
 
-static struct pci_device_id agp_nvidia_pci_table[] = {
+static const struct pci_device_id agp_nvidia_pci_table[] = {
 	{
 	.class		= (PCI_CLASS_BRIDGE_HOST << 8),
 	.class_mask	= ~0,
diff --git a/drivers/char/agp/sis-agp.c b/drivers/char/agp/sis-agp.c
index 2c74038da459..14909fc5d767 100644
--- a/drivers/char/agp/sis-agp.c
+++ b/drivers/char/agp/sis-agp.c
@@ -237,7 +237,7 @@ static int agp_sis_resume(struct pci_dev *pdev)
 
 #endif /* CONFIG_PM */
 
-static struct pci_device_id agp_sis_pci_table[] = {
+static const struct pci_device_id agp_sis_pci_table[] = {
 	{
 		.class		= (PCI_CLASS_BRIDGE_HOST << 8),
 		.class_mask	= ~0,
diff --git a/drivers/char/agp/uninorth-agp.c b/drivers/char/agp/uninorth-agp.c
index fdced547ad59..c381c8e396fc 100644
--- a/drivers/char/agp/uninorth-agp.c
+++ b/drivers/char/agp/uninorth-agp.c
@@ -679,7 +679,7 @@ static void agp_uninorth_remove(struct pci_dev *pdev)
 	agp_put_bridge(bridge);
 }
 
-static struct pci_device_id agp_uninorth_pci_table[] = {
+static const struct pci_device_id agp_uninorth_pci_table[] = {
 	{
 	.class		= (PCI_CLASS_BRIDGE_HOST << 8),
 	.class_mask	= ~0,
diff --git a/drivers/char/applicom.c b/drivers/char/applicom.c
index b67263d6e34b..c0a5b1f3a986 100644
--- a/drivers/char/applicom.c
+++ b/drivers/char/applicom.c
@@ -67,7 +67,7 @@ static char *applicom_pci_devnames[] = {
 	"PCI2000PFB"
 };
 
-static struct pci_device_id applicom_pci_tbl[] = {
+static const struct pci_device_id applicom_pci_tbl[] = {
 	{ PCI_VDEVICE(APPLICOM, PCI_DEVICE_ID_APPLICOM_PCIGENERIC) },
 	{ PCI_VDEVICE(APPLICOM, PCI_DEVICE_ID_APPLICOM_PCI2000IBS_CAN) },
 	{ PCI_VDEVICE(APPLICOM, PCI_DEVICE_ID_APPLICOM_PCI2000PFB) },
diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig
index 1b223c32a8ae..95a031e9eced 100644
--- a/drivers/char/hw_random/Kconfig
+++ b/drivers/char/hw_random/Kconfig
@@ -13,10 +13,8 @@ menuconfig HW_RANDOM
 	  that's usually called /dev/hwrng, and which exposes one
 	  of possibly several hardware random number generators.
 
-	  These hardware random number generators do not feed directly
-	  into the kernel's random number generator.  That is usually
-	  handled by the "rngd" daemon.  Documentation/hw_random.txt
-	  has more information.
+	  These hardware random number generators do feed into the
+	  kernel's random number generator entropy pool.
 
 	  If unsure, say Y.
 
@@ -255,6 +253,20 @@ config HW_RANDOM_MXC_RNGA
 
 	  If unsure, say Y.
 
+config HW_RANDOM_IMX_RNGC
+	tristate "Freescale i.MX RNGC Random Number Generator"
+	depends on ARCH_MXC
+	default HW_RANDOM
+	---help---
+	  This driver provides kernel-side support for the Random Number
+	  Generator Version C hardware found on some Freescale i.MX
+	  processors. Version B is also supported by this driver.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called imx-rngc.
+
+	  If unsure, say Y.
+
 config HW_RANDOM_NOMADIK
 	tristate "ST-Ericsson Nomadik Random Number Generator support"
 	depends on ARCH_NOMADIK
diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile
index b085975ec1d2..39a67defac67 100644
--- a/drivers/char/hw_random/Makefile
+++ b/drivers/char/hw_random/Makefile
@@ -20,6 +20,7 @@ obj-$(CONFIG_HW_RANDOM_PASEMI) += pasemi-rng.o
 obj-$(CONFIG_HW_RANDOM_VIRTIO) += virtio-rng.o
 obj-$(CONFIG_HW_RANDOM_TX4939) += tx4939-rng.o
 obj-$(CONFIG_HW_RANDOM_MXC_RNGA) += mxc-rnga.o
+obj-$(CONFIG_HW_RANDOM_IMX_RNGC) += imx-rngc.o
 obj-$(CONFIG_HW_RANDOM_OCTEON) += octeon-rng.o
 obj-$(CONFIG_HW_RANDOM_NOMADIK) += nomadik-rng.o
 obj-$(CONFIG_HW_RANDOM_PSERIES) += pseries-rng.o
diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c
index 503a41dfa193..9701ac7d8b47 100644
--- a/drivers/char/hw_random/core.c
+++ b/drivers/char/hw_random/core.c
@@ -28,7 +28,10 @@
 #define RNG_MODULE_NAME		"hw_random"
 
 static struct hwrng *current_rng;
+/* the current rng has been explicitly chosen by user via sysfs */
+static int cur_rng_set_by_user;
 static struct task_struct *hwrng_fill;
+/* list of registered rngs, sorted decending by quality */
 static LIST_HEAD(rng_list);
 /* Protects rng_list and current_rng */
 static DEFINE_MUTEX(rng_mutex);
@@ -303,6 +306,7 @@ static ssize_t hwrng_attr_current_store(struct device *dev,
 	list_for_each_entry(rng, &rng_list, list) {
 		if (sysfs_streq(rng->name, buf)) {
 			err = 0;
+			cur_rng_set_by_user = 1;
 			if (rng != current_rng)
 				err = set_current_rng(rng);
 			break;
@@ -351,16 +355,27 @@ static ssize_t hwrng_attr_available_show(struct device *dev,
 	return strlen(buf);
 }
 
+static ssize_t hwrng_attr_selected_show(struct device *dev,
+					struct device_attribute *attr,
+					char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "%d\n", cur_rng_set_by_user);
+}
+
 static DEVICE_ATTR(rng_current, S_IRUGO | S_IWUSR,
 		   hwrng_attr_current_show,
 		   hwrng_attr_current_store);
 static DEVICE_ATTR(rng_available, S_IRUGO,
 		   hwrng_attr_available_show,
 		   NULL);
+static DEVICE_ATTR(rng_selected, S_IRUGO,
+		   hwrng_attr_selected_show,
+		   NULL);
 
 static struct attribute *rng_dev_attrs[] = {
 	&dev_attr_rng_current.attr,
 	&dev_attr_rng_available.attr,
+	&dev_attr_rng_selected.attr,
 	NULL
 };
 
@@ -417,6 +432,7 @@ int hwrng_register(struct hwrng *rng)
 {
 	int err = -EINVAL;
 	struct hwrng *old_rng, *tmp;
+	struct list_head *rng_list_ptr;
 
 	if (!rng->name || (!rng->data_read && !rng->read))
 		goto out;
@@ -432,14 +448,27 @@ int hwrng_register(struct hwrng *rng)
 	init_completion(&rng->cleanup_done);
 	complete(&rng->cleanup_done);
 
+	/* rng_list is sorted by decreasing quality */
+	list_for_each(rng_list_ptr, &rng_list) {
+		tmp = list_entry(rng_list_ptr, struct hwrng, list);
+		if (tmp->quality < rng->quality)
+			break;
+	}
+	list_add_tail(&rng->list, rng_list_ptr);
+
 	old_rng = current_rng;
 	err = 0;
-	if (!old_rng) {
+	if (!old_rng ||
+	    (!cur_rng_set_by_user && rng->quality > old_rng->quality)) {
+		/*
+		 * Set new rng as current as the new rng source
+		 * provides better entropy quality and was not
+		 * chosen by userspace.
+		 */
 		err = set_current_rng(rng);
 		if (err)
 			goto out_unlock;
 	}
-	list_add_tail(&rng->list, &rng_list);
 
 	if (old_rng && !rng->init) {
 		/*
@@ -466,12 +495,13 @@ void hwrng_unregister(struct hwrng *rng)
 	list_del(&rng->list);
 	if (current_rng == rng) {
 		drop_current_rng();
+		cur_rng_set_by_user = 0;
+		/* rng_list is sorted by quality, use the best (=first) one */
 		if (!list_empty(&rng_list)) {
-			struct hwrng *tail;
-
-			tail = list_entry(rng_list.prev, struct hwrng, list);
+			struct hwrng *new_rng;
 
-			set_current_rng(tail);
+			new_rng = list_entry(rng_list.next, struct hwrng, list);
+			set_current_rng(new_rng);
 		}
 	}
 
diff --git a/drivers/char/hw_random/imx-rngc.c b/drivers/char/hw_random/imx-rngc.c
new file mode 100644
index 000000000000..88db42d30760
--- /dev/null
+++ b/drivers/char/hw_random/imx-rngc.c
@@ -0,0 +1,331 @@
+/*
+ * RNG driver for Freescale RNGC
+ *
+ * Copyright (C) 2008-2012 Freescale Semiconductor, Inc.
+ * Copyright (C) 2017 Martin Kaiser <martin@kaiser.cx>
+ *
+ * The code contained herein is licensed under the GNU General Public
+ * License. You may obtain a copy of the GNU General Public License
+ * Version 2 or later at the following locations:
+ *
+ * http://www.opensource.org/licenses/gpl-license.html
+ * http://www.gnu.org/copyleft/gpl.html
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/platform_device.h>
+#include <linux/interrupt.h>
+#include <linux/hw_random.h>
+#include <linux/completion.h>
+#include <linux/io.h>
+
+#define RNGC_COMMAND			0x0004
+#define RNGC_CONTROL			0x0008
+#define RNGC_STATUS			0x000C
+#define RNGC_ERROR			0x0010
+#define RNGC_FIFO			0x0014
+
+#define RNGC_CMD_CLR_ERR		0x00000020
+#define RNGC_CMD_CLR_INT		0x00000010
+#define RNGC_CMD_SEED			0x00000002
+#define RNGC_CMD_SELF_TEST		0x00000001
+
+#define RNGC_CTRL_MASK_ERROR		0x00000040
+#define RNGC_CTRL_MASK_DONE		0x00000020
+
+#define RNGC_STATUS_ERROR		0x00010000
+#define RNGC_STATUS_FIFO_LEVEL_MASK	0x00000f00
+#define RNGC_STATUS_FIFO_LEVEL_SHIFT	8
+#define RNGC_STATUS_SEED_DONE		0x00000020
+#define RNGC_STATUS_ST_DONE		0x00000010
+
+#define RNGC_ERROR_STATUS_STAT_ERR	0x00000008
+
+#define RNGC_TIMEOUT  3000 /* 3 sec */
+
+
+static bool self_test = true;
+module_param(self_test, bool, 0);
+
+struct imx_rngc {
+	struct device		*dev;
+	struct clk		*clk;
+	void __iomem		*base;
+	struct hwrng		rng;
+	struct completion	rng_op_done;
+	/*
+	 * err_reg is written only by the irq handler and read only
+	 * when interrupts are masked, we need no spinlock
+	 */
+	u32			err_reg;
+};
+
+
+static inline void imx_rngc_irq_mask_clear(struct imx_rngc *rngc)
+{
+	u32 ctrl, cmd;
+
+	/* mask interrupts */
+	ctrl = readl(rngc->base + RNGC_CONTROL);
+	ctrl |= RNGC_CTRL_MASK_DONE | RNGC_CTRL_MASK_ERROR;
+	writel(ctrl, rngc->base + RNGC_CONTROL);
+
+	/*
+	 * CLR_INT clears the interrupt only if there's no error
+	 * CLR_ERR clear the interrupt and the error register if there
+	 * is an error
+	 */
+	cmd = readl(rngc->base + RNGC_COMMAND);
+	cmd |= RNGC_CMD_CLR_INT | RNGC_CMD_CLR_ERR;
+	writel(cmd, rngc->base + RNGC_COMMAND);
+}
+
+static inline void imx_rngc_irq_unmask(struct imx_rngc *rngc)
+{
+	u32 ctrl;
+
+	ctrl = readl(rngc->base + RNGC_CONTROL);
+	ctrl &= ~(RNGC_CTRL_MASK_DONE | RNGC_CTRL_MASK_ERROR);
+	writel(ctrl, rngc->base + RNGC_CONTROL);
+}
+
+static int imx_rngc_self_test(struct imx_rngc *rngc)
+{
+	u32 cmd;
+	int ret;
+
+	imx_rngc_irq_unmask(rngc);
+
+	/* run self test */
+	cmd = readl(rngc->base + RNGC_COMMAND);
+	writel(cmd | RNGC_CMD_SELF_TEST, rngc->base + RNGC_COMMAND);
+
+	ret = wait_for_completion_timeout(&rngc->rng_op_done, RNGC_TIMEOUT);
+	if (!ret) {
+		imx_rngc_irq_mask_clear(rngc);
+		return -ETIMEDOUT;
+	}
+
+	if (rngc->err_reg != 0)
+		return -EIO;
+
+	return 0;
+}
+
+static int imx_rngc_read(struct hwrng *rng, void *data, size_t max, bool wait)
+{
+	struct imx_rngc *rngc = container_of(rng, struct imx_rngc, rng);
+	unsigned int status;
+	unsigned int level;
+	int retval = 0;
+
+	while (max >= sizeof(u32)) {
+		status = readl(rngc->base + RNGC_STATUS);
+
+		/* is there some error while reading this random number? */
+		if (status & RNGC_STATUS_ERROR)
+			break;
+
+		/* how many random numbers are in FIFO? [0-16] */
+		level = (status & RNGC_STATUS_FIFO_LEVEL_MASK) >>
+			RNGC_STATUS_FIFO_LEVEL_SHIFT;
+
+		if (level) {
+			/* retrieve a random number from FIFO */
+			*(u32 *)data = readl(rngc->base + RNGC_FIFO);
+
+			retval += sizeof(u32);
+			data += sizeof(u32);
+			max -= sizeof(u32);
+		}
+	}
+
+	return retval ? retval : -EIO;
+}
+
+static irqreturn_t imx_rngc_irq(int irq, void *priv)
+{
+	struct imx_rngc *rngc = (struct imx_rngc *)priv;
+	u32 status;
+
+	/*
+	 * clearing the interrupt will also clear the error register
+	 * read error and status before clearing
+	 */
+	status = readl(rngc->base + RNGC_STATUS);
+	rngc->err_reg = readl(rngc->base + RNGC_ERROR);
+
+	imx_rngc_irq_mask_clear(rngc);
+
+	if (status & (RNGC_STATUS_SEED_DONE | RNGC_STATUS_ST_DONE))
+		complete(&rngc->rng_op_done);
+
+	return IRQ_HANDLED;
+}
+
+static int imx_rngc_init(struct hwrng *rng)
+{
+	struct imx_rngc *rngc = container_of(rng, struct imx_rngc, rng);
+	u32 cmd;
+	int ret;
+
+	/* clear error */
+	cmd = readl(rngc->base + RNGC_COMMAND);
+	writel(cmd | RNGC_CMD_CLR_ERR, rngc->base + RNGC_COMMAND);
+
+	/* create seed, repeat while there is some statistical error */
+	do {
+		imx_rngc_irq_unmask(rngc);
+
+		/* seed creation */
+		cmd = readl(rngc->base + RNGC_COMMAND);
+		writel(cmd | RNGC_CMD_SEED, rngc->base + RNGC_COMMAND);
+
+		ret = wait_for_completion_timeout(&rngc->rng_op_done,
+				RNGC_TIMEOUT);
+
+		if (!ret) {
+			imx_rngc_irq_mask_clear(rngc);
+			return -ETIMEDOUT;
+		}
+
+	} while (rngc->err_reg == RNGC_ERROR_STATUS_STAT_ERR);
+
+	return rngc->err_reg ? -EIO : 0;
+}
+
+static int imx_rngc_probe(struct platform_device *pdev)
+{
+	struct imx_rngc *rngc;
+	struct resource *res;
+	int ret;
+	int irq;
+
+	rngc = devm_kzalloc(&pdev->dev, sizeof(*rngc), GFP_KERNEL);
+	if (!rngc)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	rngc->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(rngc->base))
+		return PTR_ERR(rngc->base);
+
+	rngc->clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(rngc->clk)) {
+		dev_err(&pdev->dev, "Can not get rng_clk\n");
+		return PTR_ERR(rngc->clk);
+	}
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq <= 0) {
+		dev_err(&pdev->dev, "Couldn't get irq %d\n", irq);
+		return irq;
+	}
+
+	ret = clk_prepare_enable(rngc->clk);
+	if (ret)
+		return ret;
+
+	ret = devm_request_irq(&pdev->dev,
+			irq, imx_rngc_irq, 0, pdev->name, (void *)rngc);
+	if (ret) {
+		dev_err(rngc->dev, "Can't get interrupt working.\n");
+		goto err;
+	}
+
+	init_completion(&rngc->rng_op_done);
+
+	rngc->rng.name = pdev->name;
+	rngc->rng.init = imx_rngc_init;
+	rngc->rng.read = imx_rngc_read;
+
+	rngc->dev = &pdev->dev;
+	platform_set_drvdata(pdev, rngc);
+
+	imx_rngc_irq_mask_clear(rngc);
+
+	if (self_test) {
+		ret = imx_rngc_self_test(rngc);
+		if (ret) {
+			dev_err(rngc->dev, "FSL RNGC self test failed.\n");
+			goto err;
+		}
+	}
+
+	ret = hwrng_register(&rngc->rng);
+	if (ret) {
+		dev_err(&pdev->dev, "FSL RNGC registering failed (%d)\n", ret);
+		goto err;
+	}
+
+	dev_info(&pdev->dev, "Freescale RNGC registered.\n");
+	return 0;
+
+err:
+	clk_disable_unprepare(rngc->clk);
+
+	return ret;
+}
+
+static int __exit imx_rngc_remove(struct platform_device *pdev)
+{
+	struct imx_rngc *rngc = platform_get_drvdata(pdev);
+
+	hwrng_unregister(&rngc->rng);
+
+	clk_disable_unprepare(rngc->clk);
+
+	return 0;
+}
+
+#ifdef CONFIG_PM
+static int imx_rngc_suspend(struct device *dev)
+{
+	struct imx_rngc *rngc = dev_get_drvdata(dev);
+
+	clk_disable_unprepare(rngc->clk);
+
+	return 0;
+}
+
+static int imx_rngc_resume(struct device *dev)
+{
+	struct imx_rngc *rngc = dev_get_drvdata(dev);
+
+	clk_prepare_enable(rngc->clk);
+
+	return 0;
+}
+
+static const struct dev_pm_ops imx_rngc_pm_ops = {
+	.suspend	= imx_rngc_suspend,
+	.resume		= imx_rngc_resume,
+};
+#endif
+
+static const struct of_device_id imx_rngc_dt_ids[] = {
+	{ .compatible = "fsl,imx25-rngb", .data = NULL, },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, imx_rngc_dt_ids);
+
+static struct platform_driver imx_rngc_driver = {
+	.driver = {
+		.name = "imx_rngc",
+#ifdef CONFIG_PM
+		.pm = &imx_rngc_pm_ops,
+#endif
+		.of_match_table = imx_rngc_dt_ids,
+	},
+	.remove = __exit_p(imx_rngc_remove),
+};
+
+module_platform_driver_probe(imx_rngc_driver, imx_rngc_probe);
+
+MODULE_AUTHOR("Freescale Semiconductor, Inc.");
+MODULE_DESCRIPTION("H/W RNGC driver for i.MX");
+MODULE_LICENSE("GPL");
diff --git a/drivers/char/ipmi/ipmi_si_intf.c b/drivers/char/ipmi/ipmi_si_intf.c
index 985973855005..36f47e8d06a3 100644
--- a/drivers/char/ipmi/ipmi_si_intf.c
+++ b/drivers/char/ipmi/ipmi_si_intf.c
@@ -2812,7 +2812,7 @@ static struct platform_driver ipmi_driver = {
 };
 
 #ifdef CONFIG_PARISC
-static int ipmi_parisc_probe(struct parisc_device *dev)
+static int __init ipmi_parisc_probe(struct parisc_device *dev)
 {
 	struct smi_info *info;
 	int rv;
@@ -2850,22 +2850,24 @@ static int ipmi_parisc_probe(struct parisc_device *dev)
 	return 0;
 }
 
-static int ipmi_parisc_remove(struct parisc_device *dev)
+static int __exit ipmi_parisc_remove(struct parisc_device *dev)
 {
 	cleanup_one_si(dev_get_drvdata(&dev->dev));
 	return 0;
 }
 
-static const struct parisc_device_id ipmi_parisc_tbl[] = {
+static const struct parisc_device_id ipmi_parisc_tbl[] __initconst = {
 	{ HPHW_MC, HVERSION_REV_ANY_ID, 0x004, 0xC0 },
 	{ 0, }
 };
 
-static struct parisc_driver ipmi_parisc_driver = {
+MODULE_DEVICE_TABLE(parisc, ipmi_parisc_tbl);
+
+static struct parisc_driver ipmi_parisc_driver __refdata = {
 	.name =		"ipmi",
 	.id_table =	ipmi_parisc_tbl,
 	.probe =	ipmi_parisc_probe,
-	.remove =	ipmi_parisc_remove,
+	.remove =	__exit_p(ipmi_parisc_remove),
 };
 #endif /* CONFIG_PARISC */
 
diff --git a/drivers/char/mwave/smapi.c b/drivers/char/mwave/smapi.c
index 8c5411a8f33f..691f5898bb32 100644
--- a/drivers/char/mwave/smapi.c
+++ b/drivers/char/mwave/smapi.c
@@ -128,10 +128,11 @@ int smapi_query_DSP_cfg(SMAPI_DSP_SETTINGS * pSettings)
 {
 	int bRC = -EIO;
 	unsigned short usAX, usBX, usCX, usDX, usDI, usSI;
-	unsigned short ausDspBases[] = { 0x0030, 0x4E30, 0x8E30, 0xCE30, 0x0130, 0x0350, 0x0070, 0x0DB0 };
-	unsigned short ausUartBases[] = { 0x03F8, 0x02F8, 0x03E8, 0x02E8 };
-	unsigned short numDspBases = 8;
-	unsigned short numUartBases = 4;
+	static const unsigned short ausDspBases[] = {
+		0x0030, 0x4E30, 0x8E30, 0xCE30,
+		0x0130, 0x0350, 0x0070, 0x0DB0 };
+	static const unsigned short ausUartBases[] = {
+		0x03F8, 0x02F8, 0x03E8, 0x02E8 };
 
 	PRINTK_1(TRACE_SMAPI, "smapi::smapi_query_DSP_cfg entry\n");
 
@@ -148,7 +149,7 @@ int smapi_query_DSP_cfg(SMAPI_DSP_SETTINGS * pSettings)
 	pSettings->bDSPEnabled = ((usCX & 0x0001) != 0);
 	pSettings->usDspIRQ = usSI & 0x00FF;
 	pSettings->usDspDMA = (usSI & 0xFF00) >> 8;
-	if ((usDI & 0x00FF) < numDspBases) {
+	if ((usDI & 0x00FF) < ARRAY_SIZE(ausDspBases)) {
 		pSettings->usDspBaseIO = ausDspBases[usDI & 0x00FF];
 	} else {
 		pSettings->usDspBaseIO = 0;
@@ -176,7 +177,7 @@ int smapi_query_DSP_cfg(SMAPI_DSP_SETTINGS * pSettings)
 
 	pSettings->bModemEnabled = ((usCX & 0x0001) != 0);
 	pSettings->usUartIRQ = usSI & 0x000F;
-	if (((usSI & 0xFF00) >> 8) < numUartBases) {
+	if (((usSI & 0xFF00) >> 8) < ARRAY_SIZE(ausUartBases)) {
 		pSettings->usUartBaseIO = ausUartBases[(usSI & 0xFF00) >> 8];
 	} else {
 		pSettings->usUartBaseIO = 0;
@@ -205,15 +206,16 @@ int smapi_set_DSP_cfg(void)
 	int bRC = -EIO;
 	int i;
 	unsigned short usAX, usBX, usCX, usDX, usDI, usSI;
-	unsigned short ausDspBases[] = { 0x0030, 0x4E30, 0x8E30, 0xCE30, 0x0130, 0x0350, 0x0070, 0x0DB0 };
-	unsigned short ausUartBases[] = { 0x03F8, 0x02F8, 0x03E8, 0x02E8 };
-	unsigned short ausDspIrqs[] = { 5, 7, 10, 11, 15 };
-	unsigned short ausUartIrqs[] = { 3, 4 };
-
-	unsigned short numDspBases = 8;
-	unsigned short numUartBases = 4;
-	unsigned short numDspIrqs = 5;
-	unsigned short numUartIrqs = 2;
+	static const unsigned short ausDspBases[] = {
+		0x0030, 0x4E30, 0x8E30, 0xCE30,
+		0x0130, 0x0350, 0x0070, 0x0DB0 };
+	static const unsigned short ausUartBases[] = {
+		0x03F8, 0x02F8, 0x03E8, 0x02E8 };
+	static const unsigned short ausDspIrqs[] = {
+		5, 7, 10, 11, 15 };
+	static const unsigned short ausUartIrqs[] = {
+		3, 4 };
+
 	unsigned short dspio_index = 0, uartio_index = 0;
 
 	PRINTK_5(TRACE_SMAPI,
@@ -221,11 +223,11 @@ int smapi_set_DSP_cfg(void)
 		mwave_3780i_irq, mwave_3780i_io, mwave_uart_irq, mwave_uart_io);
 
 	if (mwave_3780i_io) {
-		for (i = 0; i < numDspBases; i++) {
+		for (i = 0; i < ARRAY_SIZE(ausDspBases); i++) {
 			if (mwave_3780i_io == ausDspBases[i])
 				break;
 		}
-		if (i == numDspBases) {
+		if (i == ARRAY_SIZE(ausDspBases)) {
 			PRINTK_ERROR(KERN_ERR_MWAVE "smapi::smapi_set_DSP_cfg: Error: Invalid mwave_3780i_io address %x. Aborting.\n", mwave_3780i_io);
 			return bRC;
 		}
@@ -233,22 +235,22 @@ int smapi_set_DSP_cfg(void)
 	}
 
 	if (mwave_3780i_irq) {
-		for (i = 0; i < numDspIrqs; i++) {
+		for (i = 0; i < ARRAY_SIZE(ausDspIrqs); i++) {
 			if (mwave_3780i_irq == ausDspIrqs[i])
 				break;
 		}
-		if (i == numDspIrqs) {
+		if (i == ARRAY_SIZE(ausDspIrqs)) {
 			PRINTK_ERROR(KERN_ERR_MWAVE "smapi::smapi_set_DSP_cfg: Error: Invalid mwave_3780i_irq %x. Aborting.\n", mwave_3780i_irq);
 			return bRC;
 		}
 	}
 
 	if (mwave_uart_io) {
-		for (i = 0; i < numUartBases; i++) {
+		for (i = 0; i < ARRAY_SIZE(ausUartBases); i++) {
 			if (mwave_uart_io == ausUartBases[i])
 				break;
 		}
-		if (i == numUartBases) {
+		if (i == ARRAY_SIZE(ausUartBases)) {
 			PRINTK_ERROR(KERN_ERR_MWAVE "smapi::smapi_set_DSP_cfg: Error: Invalid mwave_uart_io address %x. Aborting.\n", mwave_uart_io);
 			return bRC;
 		}
@@ -257,11 +259,11 @@ int smapi_set_DSP_cfg(void)
 
 
 	if (mwave_uart_irq) {
-		for (i = 0; i < numUartIrqs; i++) {
+		for (i = 0; i < ARRAY_SIZE(ausUartIrqs); i++) {
 			if (mwave_uart_irq == ausUartIrqs[i])
 				break;
 		}
-		if (i == numUartIrqs) {
+		if (i == ARRAY_SIZE(ausUartIrqs)) {
 			PRINTK_ERROR(KERN_ERR_MWAVE "smapi::smapi_set_DSP_cfg: Error: Invalid mwave_uart_irq %x. Aborting.\n", mwave_uart_irq);
 			return bRC;
 		}
diff --git a/drivers/char/ppdev.c b/drivers/char/ppdev.c
index 3e73bcdf9e65..d256110ba672 100644
--- a/drivers/char/ppdev.c
+++ b/drivers/char/ppdev.c
@@ -101,9 +101,6 @@ static DEFINE_IDA(ida_index);
 #define PP_BUFFER_SIZE 1024
 #define PARDEVICE_MAX 8
 
-/* ROUND_UP macro from fs/select.c */
-#define ROUND_UP(x,y) (((x)+(y)-1)/(y))
-
 static DEFINE_MUTEX(pp_do_mutex);
 
 /* define fixed sized ioctl cmd for y2038 migration */
diff --git a/drivers/char/sonypi.c b/drivers/char/sonypi.c
index f4f866ee54bc..d3a979e25724 100644
--- a/drivers/char/sonypi.c
+++ b/drivers/char/sonypi.c
@@ -1491,7 +1491,7 @@ static struct platform_driver sonypi_driver = {
 
 static struct platform_device *sonypi_platform_device;
 
-static struct dmi_system_id __initdata sonypi_dmi_table[] = {
+static const struct dmi_system_id sonypi_dmi_table[] __initconst = {
 	{
 		.ident = "Sony Vaio",
 		.matches = {
diff --git a/drivers/char/tlclk.c b/drivers/char/tlclk.c
index 572a51704e67..6210bff46341 100644
--- a/drivers/char/tlclk.c
+++ b/drivers/char/tlclk.c
@@ -766,7 +766,7 @@ static struct attribute *tlclk_sysfs_entries[] = {
 	NULL
 };
 
-static struct attribute_group tlclk_attribute_group = {
+static const struct attribute_group tlclk_attribute_group = {
 	.name = NULL,		/* put in device directory */
 	.attrs = tlclk_sysfs_entries,
 };
diff --git a/drivers/char/tpm/tpm-chip.c b/drivers/char/tpm/tpm-chip.c
index 67ec9d3d04f5..0eca20c5a80c 100644
--- a/drivers/char/tpm/tpm-chip.c
+++ b/drivers/char/tpm/tpm-chip.c
@@ -164,14 +164,7 @@ static int tpm_class_shutdown(struct device *dev)
 		chip->ops = NULL;
 		up_write(&chip->ops_sem);
 	}
-	/* Allow bus- and device-specific code to run. Note: since chip->ops
-	 * is NULL, more-specific shutdown code will not be able to issue TPM
-	 * commands.
-	 */
-	if (dev->bus && dev->bus->shutdown)
-		dev->bus->shutdown(dev);
-	else if (dev->driver && dev->driver->shutdown)
-		dev->driver->shutdown(dev);
+
 	return 0;
 }
 
@@ -214,7 +207,7 @@ struct tpm_chip *tpm_chip_alloc(struct device *pdev,
 	device_initialize(&chip->devs);
 
 	chip->dev.class = tpm_class;
-	chip->dev.class->shutdown = tpm_class_shutdown;
+	chip->dev.class->shutdown_pre = tpm_class_shutdown;
 	chip->dev.release = tpm_dev_release;
 	chip->dev.parent = pdev;
 	chip->dev.groups = chip->groups;
diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c
index ad843eb02ae7..d1aed2513bd9 100644
--- a/drivers/char/virtio_console.c
+++ b/drivers/char/virtio_console.c
@@ -451,9 +451,6 @@ static struct port_buffer *alloc_buf(struct virtqueue *vq, size_t buf_size,
 		 * device is created by remoteproc, the DMA memory is
 		 * associated with the grandparent device:
 		 * vdev => rproc => platform-dev.
-		 * The code here would have been less quirky if
-		 * DMA_MEMORY_INCLUDES_CHILDREN had been supported
-		 * in dma-coherent.c
 		 */
 		if (!vq->vdev->dev.parent || !vq->vdev->dev.parent->parent)
 			goto free_buf;
@@ -1130,7 +1127,7 @@ static const struct file_operations port_fops = {
  * We turn the characters into a scatter-gather list, add it to the
  * output queue and then kick the Host.  Then we sit here waiting for
  * it to finish: inefficient in theory, but in practice
- * implementations will do it immediately (lguest's Launcher does).
+ * implementations will do it immediately.
  */
 static int put_chars(u32 vtermno, const char *buf, int count)
 {
@@ -1308,7 +1305,7 @@ static struct attribute *port_sysfs_entries[] = {
 	NULL
 };
 
-static struct attribute_group port_attribute_group = {
+static const struct attribute_group port_attribute_group = {
 	.name = NULL,		/* put in device directory */
 	.attrs = port_sysfs_entries,
 };
diff --git a/drivers/char/xilinx_hwicap/xilinx_hwicap.c b/drivers/char/xilinx_hwicap/xilinx_hwicap.c
index 3e6b23c3453c..067396bedf22 100644
--- a/drivers/char/xilinx_hwicap/xilinx_hwicap.c
+++ b/drivers/char/xilinx_hwicap/xilinx_hwicap.c
@@ -86,8 +86,7 @@
 #include <linux/cdev.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
-
-#include <asm/io.h>
+#include <linux/io.h>
 #include <linux/uaccess.h>
 
 #ifdef CONFIG_OF
@@ -222,6 +221,8 @@ static const struct config_registers v6_config_registers = {
  * hwicap_command_desync - Send a DESYNC command to the ICAP port.
  * @drvdata: a pointer to the drvdata.
  *
+ * Returns: '0' on success and failure value on error
+ *
  * This command desynchronizes the ICAP After this command, a
  * bitstream containing a NULL packet, followed by a SYNCH packet is
  * required before the ICAP will recognize commands.
@@ -251,10 +252,12 @@ static int hwicap_command_desync(struct hwicap_drvdata *drvdata)
  * hwicap_get_configuration_register - Query a configuration register.
  * @drvdata: a pointer to the drvdata.
  * @reg: a constant which represents the configuration
- *		register value to be returned.
- * 		Examples:  XHI_IDCODE, XHI_FLR.
+ * register value to be returned.
+ * Examples: XHI_IDCODE, XHI_FLR.
  * @reg_data: returns the value of the register.
  *
+ * Returns: '0' on success and failure value on error
+ *
  * Sends a query packet to the ICAP and then receives the response.
  * The icap is left in Synched state.
  */
@@ -320,7 +323,8 @@ static int hwicap_initialize_hwicap(struct hwicap_drvdata *drvdata)
 	dev_dbg(drvdata->dev, "initializing\n");
 
 	/* Abort any current transaction, to make sure we have the
-	 * ICAP in a good state. */
+	 * ICAP in a good state.
+	 */
 	dev_dbg(drvdata->dev, "Reset...\n");
 	drvdata->config->reset(drvdata);
 
@@ -632,7 +636,6 @@ static int hwicap_setup(struct device *dev, int id,
 
 	drvdata = kzalloc(sizeof(struct hwicap_drvdata), GFP_KERNEL);
 	if (!drvdata) {
-		dev_err(dev, "Couldn't allocate device private record\n");
 		retval = -ENOMEM;
 		goto failed0;
 	}
@@ -759,20 +762,20 @@ static int hwicap_of_probe(struct platform_device *op,
 	id = of_get_property(op->dev.of_node, "port-number", NULL);
 
 	/* It's most likely that we're using V4, if the family is not
-	   specified */
+	 * specified
+	 */
 	regs = &v4_config_registers;
 	family = of_get_property(op->dev.of_node, "xlnx,family", NULL);
 
 	if (family) {
-		if (!strcmp(family, "virtex2p")) {
+		if (!strcmp(family, "virtex2p"))
 			regs = &v2_config_registers;
-		} else if (!strcmp(family, "virtex4")) {
+		else if (!strcmp(family, "virtex4"))
 			regs = &v4_config_registers;
-		} else if (!strcmp(family, "virtex5")) {
+		else if (!strcmp(family, "virtex5"))
 			regs = &v5_config_registers;
-		} else if (!strcmp(family, "virtex6")) {
+		else if (!strcmp(family, "virtex6"))
 			regs = &v6_config_registers;
-		}
 	}
 	return hwicap_setup(&op->dev, id ? *id : -1, &res, config,
 			regs);
@@ -802,20 +805,20 @@ static int hwicap_drv_probe(struct platform_device *pdev)
 		return -ENODEV;
 
 	/* It's most likely that we're using V4, if the family is not
-	   specified */
+	 * specified
+	 */
 	regs = &v4_config_registers;
 	family = pdev->dev.platform_data;
 
 	if (family) {
-		if (!strcmp(family, "virtex2p")) {
+		if (!strcmp(family, "virtex2p"))
 			regs = &v2_config_registers;
-		} else if (!strcmp(family, "virtex4")) {
+		else if (!strcmp(family, "virtex4"))
 			regs = &v4_config_registers;
-		} else if (!strcmp(family, "virtex5")) {
+		else if (!strcmp(family, "virtex5"))
 			regs = &v5_config_registers;
-		} else if (!strcmp(family, "virtex6")) {
+		else if (!strcmp(family, "virtex6"))
 			regs = &v6_config_registers;
-		}
 	}
 
 	return hwicap_setup(&pdev->dev, pdev->id, res,
diff --git a/drivers/char/xilinx_hwicap/xilinx_hwicap.h b/drivers/char/xilinx_hwicap/xilinx_hwicap.h
index 38b145eaf24d..6b963d1c8ba3 100644
--- a/drivers/char/xilinx_hwicap/xilinx_hwicap.h
+++ b/drivers/char/xilinx_hwicap/xilinx_hwicap.h
@@ -62,11 +62,13 @@ struct hwicap_drvdata {
 
 struct hwicap_driver_config {
 	/* Read configuration data given by size into the data buffer.
-	   Return 0 if successful. */
+	 * Return 0 if successful.
+	 */
 	int (*get_configuration)(struct hwicap_drvdata *drvdata, u32 *data,
 			u32 size);
 	/* Write configuration data given by size from the data buffer.
-	   Return 0 if successful. */
+	 * Return 0 if successful.
+	 */
 	int (*set_configuration)(struct hwicap_drvdata *drvdata, u32 *data,
 			u32 size);
 	/* Get the status register, bit pattern given by:
@@ -193,11 +195,12 @@ struct config_registers {
  * hwicap_type_1_read - Generates a Type 1 read packet header.
  * @reg: is the address of the register to be read back.
  *
+ * Return:
  * Generates a Type 1 read packet header, which is used to indirectly
  * read registers in the configuration logic.  This packet must then
  * be sent through the icap device, and a return packet received with
  * the information.
- **/
+ */
 static inline u32 hwicap_type_1_read(u32 reg)
 {
 	return (XHI_TYPE_1 << XHI_TYPE_SHIFT) |
@@ -208,7 +211,9 @@ static inline u32 hwicap_type_1_read(u32 reg)
 /**
  * hwicap_type_1_write - Generates a Type 1 write packet header
  * @reg: is the address of the register to be read back.
- **/
+ *
+ * Return: Type 1 write packet header
+ */
 static inline u32 hwicap_type_1_write(u32 reg)
 {
 	return (XHI_TYPE_1 << XHI_TYPE_SHIFT) |
diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig
index 68ca2d9fcd73..1c4e1aa6767e 100644
--- a/drivers/clk/Kconfig
+++ b/drivers/clk/Kconfig
@@ -31,6 +31,13 @@ config COMMON_CLK_WM831X
 
 source "drivers/clk/versatile/Kconfig"
 
+config CLK_HSDK
+	bool "PLL Driver for HSDK platform"
+	depends on OF || COMPILE_TEST
+	---help---
+	  This driver supports the HSDK core, system, ddr, tunnel and hdmi PLLs
+	  control.
+
 config COMMON_CLK_MAX77686
 	tristate "Clock driver for Maxim 77620/77686/77802 MFD"
 	depends on MFD_MAX77686 || MFD_MAX77620 || COMPILE_TEST
@@ -39,10 +46,10 @@ config COMMON_CLK_MAX77686
 	  clock.
 
 config COMMON_CLK_RK808
-	tristate "Clock driver for RK808/RK818"
+	tristate "Clock driver for RK805/RK808/RK818"
 	depends on MFD_RK808
 	---help---
-	  This driver supports RK808 and RK818 crystal oscillator clock. These
+	  This driver supports RK805, RK808 and RK818 crystal oscillator clock. These
 	  multi-function devices have two fixed-rate oscillators,
 	  clocked at 32KHz each. Clkout1 is always on, Clkout2 can off
 	  by control register.
@@ -210,14 +217,14 @@ config COMMON_CLK_OXNAS
 	  Support for the OXNAS SoC Family clocks.
 
 config COMMON_CLK_VC5
-	tristate "Clock driver for IDT VersaClock5 devices"
+	tristate "Clock driver for IDT VersaClock 5,6 devices"
 	depends on I2C
 	depends on OF
 	select REGMAP_I2C
 	help
 	---help---
-	  This driver supports the IDT VersaClock5 programmable clock
-	  generator.
+	  This driver supports the IDT VersaClock 5 and VersaClock 6
+	  programmable clock generators.
 
 source "drivers/clk/bcm/Kconfig"
 source "drivers/clk/hisilicon/Kconfig"
diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile
index cd376b3fb47a..c99f363826f0 100644
--- a/drivers/clk/Makefile
+++ b/drivers/clk/Makefile
@@ -27,8 +27,8 @@ obj-$(CONFIG_COMMON_CLK_CS2000_CP)	+= clk-cs2000-cp.o
 obj-$(CONFIG_ARCH_EFM32)		+= clk-efm32gg.o
 obj-$(CONFIG_COMMON_CLK_GEMINI)		+= clk-gemini.o
 obj-$(CONFIG_ARCH_HIGHBANK)		+= clk-highbank.o
+obj-$(CONFIG_CLK_HSDK)			+= clk-hsdk-pll.o
 obj-$(CONFIG_COMMON_CLK_MAX77686)	+= clk-max77686.o
-obj-$(CONFIG_ARCH_MB86S7X)		+= clk-mb86s7x.o
 obj-$(CONFIG_ARCH_MOXART)		+= clk-moxart.o
 obj-$(CONFIG_ARCH_NOMADIK)		+= clk-nomadik.o
 obj-$(CONFIG_ARCH_NSPIRE)		+= clk-nspire.o
@@ -44,6 +44,7 @@ obj-$(CONFIG_COMMON_CLK_SI5351)		+= clk-si5351.o
 obj-$(CONFIG_COMMON_CLK_SI514)		+= clk-si514.o
 obj-$(CONFIG_COMMON_CLK_SI570)		+= clk-si570.o
 obj-$(CONFIG_ARCH_STM32)		+= clk-stm32f4.o
+obj-$(CONFIG_ARCH_STM32)		+= clk-stm32h7.o
 obj-$(CONFIG_ARCH_TANGO)		+= clk-tango4.o
 obj-$(CONFIG_CLK_TWL6040)		+= clk-twl6040.o
 obj-$(CONFIG_ARCH_U300)			+= clk-u300.o
diff --git a/drivers/clk/at91/Makefile b/drivers/clk/at91/Makefile
index 13e67bd35cff..c68947b65a4c 100644
--- a/drivers/clk/at91/Makefile
+++ b/drivers/clk/at91/Makefile
@@ -6,6 +6,7 @@ obj-y += pmc.o sckc.o
 obj-y += clk-slow.o clk-main.o clk-pll.o clk-plldiv.o clk-master.o
 obj-y += clk-system.o clk-peripheral.o clk-programmable.o
 
+obj-$(CONFIG_HAVE_AT91_AUDIO_PLL)	+= clk-audio-pll.o
 obj-$(CONFIG_HAVE_AT91_UTMI)		+= clk-utmi.o
 obj-$(CONFIG_HAVE_AT91_USB_CLK)		+= clk-usb.o
 obj-$(CONFIG_HAVE_AT91_SMD)		+= clk-smd.o
diff --git a/drivers/clk/at91/clk-audio-pll.c b/drivers/clk/at91/clk-audio-pll.c
new file mode 100644
index 000000000000..da7bafcfbe70
--- /dev/null
+++ b/drivers/clk/at91/clk-audio-pll.c
@@ -0,0 +1,536 @@
+/*
+ *  Copyright (C) 2016 Atmel Corporation,
+ *		       Songjun Wu <songjun.wu@atmel.com>,
+ *                     Nicolas Ferre <nicolas.ferre@atmel.com>
+ *  Copyright (C) 2017 Free Electrons,
+ *		       Quentin Schulz <quentin.schulz@free-electrons.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * The Sama5d2 SoC has two audio PLLs (PMC and PAD) that shares the same parent
+ * (FRAC). FRAC can output between 620 and 700MHz and only multiply the rate of
+ * its own parent. PMC and PAD can then divide the FRAC rate to best match the
+ * asked rate.
+ *
+ * Traits of FRAC clock:
+ * enable - clk_enable writes nd, fracr parameters and enables PLL
+ * rate - rate is adjustable.
+ *        clk->rate = parent->rate * ((nd + 1) + (fracr / 2^22))
+ * parent - fixed parent.  No clk_set_parent support
+ *
+ * Traits of PMC clock:
+ * enable - clk_enable writes qdpmc, and enables PMC output
+ * rate - rate is adjustable.
+ *        clk->rate = parent->rate / (qdpmc + 1)
+ * parent - fixed parent.  No clk_set_parent support
+ *
+ * Traits of PAD clock:
+ * enable - clk_enable writes divisors and enables PAD output
+ * rate - rate is adjustable.
+ *        clk->rate = parent->rate / (qdaudio * div))
+ * parent - fixed parent.  No clk_set_parent support
+ *
+ */
+
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/clk/at91_pmc.h>
+#include <linux/of.h>
+#include <linux/mfd/syscon.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+
+#define AUDIO_PLL_DIV_FRAC	BIT(22)
+#define AUDIO_PLL_ND_MAX	(AT91_PMC_AUDIO_PLL_ND_MASK >> \
+					AT91_PMC_AUDIO_PLL_ND_OFFSET)
+
+#define AUDIO_PLL_QDPAD(qd, div)	((AT91_PMC_AUDIO_PLL_QDPAD_EXTDIV(qd) & \
+					  AT91_PMC_AUDIO_PLL_QDPAD_EXTDIV_MASK) | \
+					 (AT91_PMC_AUDIO_PLL_QDPAD_DIV(div) & \
+					  AT91_PMC_AUDIO_PLL_QDPAD_DIV_MASK))
+
+#define AUDIO_PLL_QDPMC_MAX		(AT91_PMC_AUDIO_PLL_QDPMC_MASK >> \
+						AT91_PMC_AUDIO_PLL_QDPMC_OFFSET)
+
+#define AUDIO_PLL_FOUT_MIN	620000000UL
+#define AUDIO_PLL_FOUT_MAX	700000000UL
+
+struct clk_audio_frac {
+	struct clk_hw hw;
+	struct regmap *regmap;
+	u32 fracr;
+	u8 nd;
+};
+
+struct clk_audio_pad {
+	struct clk_hw hw;
+	struct regmap *regmap;
+	u8 qdaudio;
+	u8 div;
+};
+
+struct clk_audio_pmc {
+	struct clk_hw hw;
+	struct regmap *regmap;
+	u8 qdpmc;
+};
+
+#define to_clk_audio_frac(hw) container_of(hw, struct clk_audio_frac, hw)
+#define to_clk_audio_pad(hw) container_of(hw, struct clk_audio_pad, hw)
+#define to_clk_audio_pmc(hw) container_of(hw, struct clk_audio_pmc, hw)
+
+static int clk_audio_pll_frac_enable(struct clk_hw *hw)
+{
+	struct clk_audio_frac *frac = to_clk_audio_frac(hw);
+
+	regmap_update_bits(frac->regmap, AT91_PMC_AUDIO_PLL0,
+			   AT91_PMC_AUDIO_PLL_RESETN, 0);
+	regmap_update_bits(frac->regmap, AT91_PMC_AUDIO_PLL0,
+			   AT91_PMC_AUDIO_PLL_RESETN,
+			   AT91_PMC_AUDIO_PLL_RESETN);
+	regmap_update_bits(frac->regmap, AT91_PMC_AUDIO_PLL1,
+			   AT91_PMC_AUDIO_PLL_FRACR_MASK, frac->fracr);
+
+	/*
+	 * reset and enable have to be done in 2 separated writes
+	 * for AT91_PMC_AUDIO_PLL0
+	 */
+	regmap_update_bits(frac->regmap, AT91_PMC_AUDIO_PLL0,
+			   AT91_PMC_AUDIO_PLL_PLLEN |
+			   AT91_PMC_AUDIO_PLL_ND_MASK,
+			   AT91_PMC_AUDIO_PLL_PLLEN |
+			   AT91_PMC_AUDIO_PLL_ND(frac->nd));
+
+	return 0;
+}
+
+static int clk_audio_pll_pad_enable(struct clk_hw *hw)
+{
+	struct clk_audio_pad *apad_ck = to_clk_audio_pad(hw);
+
+	regmap_update_bits(apad_ck->regmap, AT91_PMC_AUDIO_PLL1,
+			   AT91_PMC_AUDIO_PLL_QDPAD_MASK,
+			   AUDIO_PLL_QDPAD(apad_ck->qdaudio, apad_ck->div));
+	regmap_update_bits(apad_ck->regmap, AT91_PMC_AUDIO_PLL0,
+			   AT91_PMC_AUDIO_PLL_PADEN, AT91_PMC_AUDIO_PLL_PADEN);
+
+	return 0;
+}
+
+static int clk_audio_pll_pmc_enable(struct clk_hw *hw)
+{
+	struct clk_audio_pmc *apmc_ck = to_clk_audio_pmc(hw);
+
+	regmap_update_bits(apmc_ck->regmap, AT91_PMC_AUDIO_PLL0,
+			   AT91_PMC_AUDIO_PLL_PMCEN |
+			   AT91_PMC_AUDIO_PLL_QDPMC_MASK,
+			   AT91_PMC_AUDIO_PLL_PMCEN |
+			   AT91_PMC_AUDIO_PLL_QDPMC(apmc_ck->qdpmc));
+	return 0;
+}
+
+static void clk_audio_pll_frac_disable(struct clk_hw *hw)
+{
+	struct clk_audio_frac *frac = to_clk_audio_frac(hw);
+
+	regmap_update_bits(frac->regmap, AT91_PMC_AUDIO_PLL0,
+			   AT91_PMC_AUDIO_PLL_PLLEN, 0);
+	/* do it in 2 separated writes */
+	regmap_update_bits(frac->regmap, AT91_PMC_AUDIO_PLL0,
+			   AT91_PMC_AUDIO_PLL_RESETN, 0);
+}
+
+static void clk_audio_pll_pad_disable(struct clk_hw *hw)
+{
+	struct clk_audio_pad *apad_ck = to_clk_audio_pad(hw);
+
+	regmap_update_bits(apad_ck->regmap, AT91_PMC_AUDIO_PLL0,
+			   AT91_PMC_AUDIO_PLL_PADEN, 0);
+}
+
+static void clk_audio_pll_pmc_disable(struct clk_hw *hw)
+{
+	struct clk_audio_pmc *apmc_ck = to_clk_audio_pmc(hw);
+
+	regmap_update_bits(apmc_ck->regmap, AT91_PMC_AUDIO_PLL0,
+			   AT91_PMC_AUDIO_PLL_PMCEN, 0);
+}
+
+static unsigned long clk_audio_pll_fout(unsigned long parent_rate,
+					unsigned long nd, unsigned long fracr)
+{
+	unsigned long long fr = (unsigned long long)parent_rate * fracr;
+
+	pr_debug("A PLL: %s, fr = %llu\n", __func__, fr);
+
+	fr = DIV_ROUND_CLOSEST_ULL(fr, AUDIO_PLL_DIV_FRAC);
+
+	pr_debug("A PLL: %s, fr = %llu\n", __func__, fr);
+
+	return parent_rate * (nd + 1) + fr;
+}
+
+static unsigned long clk_audio_pll_frac_recalc_rate(struct clk_hw *hw,
+						    unsigned long parent_rate)
+{
+	struct clk_audio_frac *frac = to_clk_audio_frac(hw);
+	unsigned long fout;
+
+	fout = clk_audio_pll_fout(parent_rate, frac->nd, frac->fracr);
+
+	pr_debug("A PLL: %s, fout = %lu (nd = %u, fracr = %lu)\n", __func__,
+		 fout, frac->nd, (unsigned long)frac->fracr);
+
+	return fout;
+}
+
+static unsigned long clk_audio_pll_pad_recalc_rate(struct clk_hw *hw,
+						   unsigned long parent_rate)
+{
+	struct clk_audio_pad *apad_ck = to_clk_audio_pad(hw);
+	unsigned long apad_rate = 0;
+
+	if (apad_ck->qdaudio && apad_ck->div)
+		apad_rate = parent_rate / (apad_ck->qdaudio * apad_ck->div);
+
+	pr_debug("A PLL/PAD: %s, apad_rate = %lu (div = %u, qdaudio = %u)\n",
+		 __func__, apad_rate, apad_ck->div, apad_ck->qdaudio);
+
+	return apad_rate;
+}
+
+static unsigned long clk_audio_pll_pmc_recalc_rate(struct clk_hw *hw,
+						   unsigned long parent_rate)
+{
+	struct clk_audio_pmc *apmc_ck = to_clk_audio_pmc(hw);
+	unsigned long apmc_rate = 0;
+
+	apmc_rate = parent_rate / (apmc_ck->qdpmc + 1);
+
+	pr_debug("A PLL/PMC: %s, apmc_rate = %lu (qdpmc = %u)\n", __func__,
+		 apmc_rate, apmc_ck->qdpmc);
+
+	return apmc_rate;
+}
+
+static int clk_audio_pll_frac_compute_frac(unsigned long rate,
+					   unsigned long parent_rate,
+					   unsigned long *nd,
+					   unsigned long *fracr)
+{
+	unsigned long long tmp, rem;
+
+	if (!rate)
+		return -EINVAL;
+
+	tmp = rate;
+	rem = do_div(tmp, parent_rate);
+	if (!tmp || tmp >= AUDIO_PLL_ND_MAX)
+		return -EINVAL;
+
+	*nd = tmp - 1;
+
+	tmp = rem * AUDIO_PLL_DIV_FRAC;
+	tmp = DIV_ROUND_CLOSEST_ULL(tmp, parent_rate);
+	if (tmp > AT91_PMC_AUDIO_PLL_FRACR_MASK)
+		return -EINVAL;
+
+	/* we can cast here as we verified the bounds just above */
+	*fracr = (unsigned long)tmp;
+
+	return 0;
+}
+
+static int clk_audio_pll_frac_determine_rate(struct clk_hw *hw,
+					     struct clk_rate_request *req)
+{
+	unsigned long fracr, nd;
+	int ret;
+
+	pr_debug("A PLL: %s, rate = %lu (parent_rate = %lu)\n", __func__,
+		 req->rate, req->best_parent_rate);
+
+	req->rate = clamp(req->rate, AUDIO_PLL_FOUT_MIN, AUDIO_PLL_FOUT_MAX);
+
+	req->min_rate = max(req->min_rate, AUDIO_PLL_FOUT_MIN);
+	req->max_rate = min(req->max_rate, AUDIO_PLL_FOUT_MAX);
+
+	ret = clk_audio_pll_frac_compute_frac(req->rate, req->best_parent_rate,
+					      &nd, &fracr);
+	if (ret)
+		return ret;
+
+	req->rate = clk_audio_pll_fout(req->best_parent_rate, nd, fracr);
+
+	req->best_parent_hw = clk_hw_get_parent(hw);
+
+	pr_debug("A PLL: %s, best_rate = %lu (nd = %lu, fracr = %lu)\n",
+		 __func__, req->rate, nd, fracr);
+
+	return 0;
+}
+
+static long clk_audio_pll_pad_round_rate(struct clk_hw *hw, unsigned long rate,
+					 unsigned long *parent_rate)
+{
+	struct clk_hw *pclk = clk_hw_get_parent(hw);
+	long best_rate = -EINVAL;
+	unsigned long best_parent_rate;
+	unsigned long tmp_qd;
+	u32 div;
+	long tmp_rate;
+	int tmp_diff;
+	int best_diff = -1;
+
+	pr_debug("A PLL/PAD: %s, rate = %lu (parent_rate = %lu)\n", __func__,
+		 rate, *parent_rate);
+
+	/*
+	 * Rate divisor is actually made of two different divisors, multiplied
+	 * between themselves before dividing the rate.
+	 * tmp_qd goes from 1 to 31 and div is either 2 or 3.
+	 * In order to avoid testing twice the rate divisor (e.g. divisor 12 can
+	 * be found with (tmp_qd, div) = (2, 6) or (3, 4)), we remove any loop
+	 * for a rate divisor when div is 2 and tmp_qd is a multiple of 3.
+	 * We cannot inverse it (condition div is 3 and tmp_qd is even) or we
+	 * would miss some rate divisor that aren't reachable with div being 2
+	 * (e.g. rate divisor 90 is made with div = 3 and tmp_qd = 30, thus
+	 * tmp_qd is even so we skip it because we think div 2 could make this
+	 * rate divisor which isn't possible since tmp_qd has to be <= 31).
+	 */
+	for (tmp_qd = 1; tmp_qd < AT91_PMC_AUDIO_PLL_QDPAD_EXTDIV_MAX; tmp_qd++)
+		for (div = 2; div <= 3; div++) {
+			if (div == 2 && tmp_qd % 3 == 0)
+				continue;
+
+			best_parent_rate = clk_hw_round_rate(pclk,
+							rate * tmp_qd * div);
+			tmp_rate = best_parent_rate / (div * tmp_qd);
+			tmp_diff = abs(rate - tmp_rate);
+
+			if (best_diff < 0 || best_diff > tmp_diff) {
+				*parent_rate = best_parent_rate;
+				best_rate = tmp_rate;
+				best_diff = tmp_diff;
+			}
+		}
+
+	pr_debug("A PLL/PAD: %s, best_rate = %ld, best_parent_rate = %lu\n",
+		 __func__, best_rate, best_parent_rate);
+
+	return best_rate;
+}
+
+static long clk_audio_pll_pmc_round_rate(struct clk_hw *hw, unsigned long rate,
+					 unsigned long *parent_rate)
+{
+	struct clk_hw *pclk = clk_hw_get_parent(hw);
+	long best_rate = -EINVAL;
+	unsigned long best_parent_rate = 0;
+	u32 tmp_qd = 0, div;
+	long tmp_rate;
+	int tmp_diff;
+	int best_diff = -1;
+
+	pr_debug("A PLL/PMC: %s, rate = %lu (parent_rate = %lu)\n", __func__,
+		 rate, *parent_rate);
+
+	for (div = 1; div <= AUDIO_PLL_QDPMC_MAX; div++) {
+		best_parent_rate = clk_round_rate(pclk->clk, rate * div);
+		tmp_rate = best_parent_rate / div;
+		tmp_diff = abs(rate - tmp_rate);
+
+		if (best_diff < 0 || best_diff > tmp_diff) {
+			*parent_rate = best_parent_rate;
+			best_rate = tmp_rate;
+			best_diff = tmp_diff;
+			tmp_qd = div;
+		}
+	}
+
+	pr_debug("A PLL/PMC: %s, best_rate = %ld, best_parent_rate = %lu (qd = %d)\n",
+		 __func__, best_rate, *parent_rate, tmp_qd - 1);
+
+	return best_rate;
+}
+
+static int clk_audio_pll_frac_set_rate(struct clk_hw *hw, unsigned long rate,
+				       unsigned long parent_rate)
+{
+	struct clk_audio_frac *frac = to_clk_audio_frac(hw);
+	unsigned long fracr, nd;
+	int ret;
+
+	pr_debug("A PLL: %s, rate = %lu (parent_rate = %lu)\n", __func__, rate,
+		 parent_rate);
+
+	if (rate < AUDIO_PLL_FOUT_MIN || rate > AUDIO_PLL_FOUT_MAX)
+		return -EINVAL;
+
+	ret = clk_audio_pll_frac_compute_frac(rate, parent_rate, &nd, &fracr);
+	if (ret)
+		return ret;
+
+	frac->nd = nd;
+	frac->fracr = fracr;
+
+	return 0;
+}
+
+static int clk_audio_pll_pad_set_rate(struct clk_hw *hw, unsigned long rate,
+				      unsigned long parent_rate)
+{
+	struct clk_audio_pad *apad_ck = to_clk_audio_pad(hw);
+	u8 tmp_div;
+
+	pr_debug("A PLL/PAD: %s, rate = %lu (parent_rate = %lu)\n", __func__,
+		 rate, parent_rate);
+
+	if (!rate)
+		return -EINVAL;
+
+	tmp_div = parent_rate / rate;
+	if (tmp_div % 3 == 0) {
+		apad_ck->qdaudio = tmp_div / 3;
+		apad_ck->div = 3;
+	} else {
+		apad_ck->qdaudio = tmp_div / 2;
+		apad_ck->div = 2;
+	}
+
+	return 0;
+}
+
+static int clk_audio_pll_pmc_set_rate(struct clk_hw *hw, unsigned long rate,
+				      unsigned long parent_rate)
+{
+	struct clk_audio_pmc *apmc_ck = to_clk_audio_pmc(hw);
+
+	if (!rate)
+		return -EINVAL;
+
+	pr_debug("A PLL/PMC: %s, rate = %lu (parent_rate = %lu)\n", __func__,
+		 rate, parent_rate);
+
+	apmc_ck->qdpmc = parent_rate / rate - 1;
+
+	return 0;
+}
+
+static const struct clk_ops audio_pll_frac_ops = {
+	.enable = clk_audio_pll_frac_enable,
+	.disable = clk_audio_pll_frac_disable,
+	.recalc_rate = clk_audio_pll_frac_recalc_rate,
+	.determine_rate = clk_audio_pll_frac_determine_rate,
+	.set_rate = clk_audio_pll_frac_set_rate,
+};
+
+static const struct clk_ops audio_pll_pad_ops = {
+	.enable = clk_audio_pll_pad_enable,
+	.disable = clk_audio_pll_pad_disable,
+	.recalc_rate = clk_audio_pll_pad_recalc_rate,
+	.round_rate = clk_audio_pll_pad_round_rate,
+	.set_rate = clk_audio_pll_pad_set_rate,
+};
+
+static const struct clk_ops audio_pll_pmc_ops = {
+	.enable = clk_audio_pll_pmc_enable,
+	.disable = clk_audio_pll_pmc_disable,
+	.recalc_rate = clk_audio_pll_pmc_recalc_rate,
+	.round_rate = clk_audio_pll_pmc_round_rate,
+	.set_rate = clk_audio_pll_pmc_set_rate,
+};
+
+static int of_sama5d2_clk_audio_pll_setup(struct device_node *np,
+					  struct clk_init_data *init,
+					  struct clk_hw *hw,
+					  struct regmap **clk_audio_regmap)
+{
+	struct regmap *regmap;
+	const char *parent_names[1];
+	int ret;
+
+	regmap = syscon_node_to_regmap(of_get_parent(np));
+	if (IS_ERR(regmap))
+		return PTR_ERR(regmap);
+
+	init->name = np->name;
+	of_clk_parent_fill(np, parent_names, 1);
+	init->parent_names = parent_names;
+	init->num_parents = 1;
+
+	hw->init = init;
+	*clk_audio_regmap = regmap;
+
+	ret = clk_hw_register(NULL, hw);
+	if (ret)
+		return ret;
+
+	return of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw);
+}
+
+static void __init of_sama5d2_clk_audio_pll_frac_setup(struct device_node *np)
+{
+	struct clk_audio_frac *frac_ck;
+	struct clk_init_data init = {};
+
+	frac_ck = kzalloc(sizeof(*frac_ck), GFP_KERNEL);
+	if (!frac_ck)
+		return;
+
+	init.ops = &audio_pll_frac_ops;
+	init.flags = CLK_SET_RATE_GATE;
+
+	if (of_sama5d2_clk_audio_pll_setup(np, &init, &frac_ck->hw,
+					   &frac_ck->regmap))
+		kfree(frac_ck);
+}
+
+static void __init of_sama5d2_clk_audio_pll_pad_setup(struct device_node *np)
+{
+	struct clk_audio_pad *apad_ck;
+	struct clk_init_data init = {};
+
+	apad_ck = kzalloc(sizeof(*apad_ck), GFP_KERNEL);
+	if (!apad_ck)
+		return;
+
+	init.ops = &audio_pll_pad_ops;
+	init.flags = CLK_SET_RATE_GATE | CLK_SET_PARENT_GATE |
+		CLK_SET_RATE_PARENT;
+
+	if (of_sama5d2_clk_audio_pll_setup(np, &init, &apad_ck->hw,
+					   &apad_ck->regmap))
+		kfree(apad_ck);
+}
+
+static void __init of_sama5d2_clk_audio_pll_pmc_setup(struct device_node *np)
+{
+	struct clk_audio_pad *apmc_ck;
+	struct clk_init_data init = {};
+
+	apmc_ck = kzalloc(sizeof(*apmc_ck), GFP_KERNEL);
+	if (!apmc_ck)
+		return;
+
+	init.ops = &audio_pll_pmc_ops;
+	init.flags = CLK_SET_RATE_GATE | CLK_SET_PARENT_GATE |
+		CLK_SET_RATE_PARENT;
+
+	if (of_sama5d2_clk_audio_pll_setup(np, &init, &apmc_ck->hw,
+					   &apmc_ck->regmap))
+		kfree(apmc_ck);
+}
+
+CLK_OF_DECLARE(of_sama5d2_clk_audio_pll_frac_setup,
+	       "atmel,sama5d2-clk-audio-pll-frac",
+	       of_sama5d2_clk_audio_pll_frac_setup);
+CLK_OF_DECLARE(of_sama5d2_clk_audio_pll_pad_setup,
+	       "atmel,sama5d2-clk-audio-pll-pad",
+	       of_sama5d2_clk_audio_pll_pad_setup);
+CLK_OF_DECLARE(of_sama5d2_clk_audio_pll_pmc_setup,
+	       "atmel,sama5d2-clk-audio-pll-pmc",
+	       of_sama5d2_clk_audio_pll_pmc_setup);
diff --git a/drivers/clk/at91/clk-generated.c b/drivers/clk/at91/clk-generated.c
index f0b7ae904ce2..33481368740e 100644
--- a/drivers/clk/at91/clk-generated.c
+++ b/drivers/clk/at91/clk-generated.c
@@ -26,6 +26,13 @@
 #define GENERATED_SOURCE_MAX	6
 #define GENERATED_MAX_DIV	255
 
+#define GCK_ID_SSC0		43
+#define GCK_ID_SSC1		44
+#define GCK_ID_I2S0		54
+#define GCK_ID_I2S1		55
+#define GCK_ID_CLASSD		59
+#define GCK_INDEX_DT_AUDIO_PLL	5
+
 struct clk_generated {
 	struct clk_hw hw;
 	struct regmap *regmap;
@@ -34,6 +41,7 @@ struct clk_generated {
 	u32 id;
 	u32 gckdiv;
 	u8 parent_id;
+	bool audio_pll_allowed;
 };
 
 #define to_clk_generated(hw) \
@@ -99,21 +107,41 @@ clk_generated_recalc_rate(struct clk_hw *hw,
 	return DIV_ROUND_CLOSEST(parent_rate, gck->gckdiv + 1);
 }
 
+static void clk_generated_best_diff(struct clk_rate_request *req,
+				    struct clk_hw *parent,
+				    unsigned long parent_rate, u32 div,
+				    int *best_diff, long *best_rate)
+{
+	unsigned long tmp_rate;
+	int tmp_diff;
+
+	if (!div)
+		tmp_rate = parent_rate;
+	else
+		tmp_rate = parent_rate / div;
+	tmp_diff = abs(req->rate - tmp_rate);
+
+	if (*best_diff < 0 || *best_diff > tmp_diff) {
+		*best_rate = tmp_rate;
+		*best_diff = tmp_diff;
+		req->best_parent_rate = parent_rate;
+		req->best_parent_hw = parent;
+	}
+}
+
 static int clk_generated_determine_rate(struct clk_hw *hw,
 					struct clk_rate_request *req)
 {
 	struct clk_generated *gck = to_clk_generated(hw);
 	struct clk_hw *parent = NULL;
+	struct clk_rate_request req_parent = *req;
 	long best_rate = -EINVAL;
-	unsigned long tmp_rate, min_rate;
+	unsigned long min_rate, parent_rate;
 	int best_diff = -1;
-	int tmp_diff;
 	int i;
+	u32 div;
 
-	for (i = 0; i < clk_hw_get_num_parents(hw); i++) {
-		u32 div;
-		unsigned long parent_rate;
-
+	for (i = 0; i < clk_hw_get_num_parents(hw) - 1; i++) {
 		parent = clk_hw_get_parent_by_index(hw, i);
 		if (!parent)
 			continue;
@@ -124,25 +152,43 @@ static int clk_generated_determine_rate(struct clk_hw *hw,
 		    (gck->range.max && min_rate > gck->range.max))
 			continue;
 
-		for (div = 1; div < GENERATED_MAX_DIV + 2; div++) {
-			tmp_rate = DIV_ROUND_CLOSEST(parent_rate, div);
-			tmp_diff = abs(req->rate - tmp_rate);
+		div = DIV_ROUND_CLOSEST(parent_rate, req->rate);
 
-			if (best_diff < 0 || best_diff > tmp_diff) {
-				best_rate = tmp_rate;
-				best_diff = tmp_diff;
-				req->best_parent_rate = parent_rate;
-				req->best_parent_hw = parent;
-			}
+		clk_generated_best_diff(req, parent, parent_rate, div,
+					&best_diff, &best_rate);
 
-			if (!best_diff || tmp_rate < req->rate)
-				break;
-		}
+		if (!best_diff)
+			break;
+	}
+
+	/*
+	 * The audio_pll rate can be modified, unlike the five others clocks
+	 * that should never be altered.
+	 * The audio_pll can technically be used by multiple consumers. However,
+	 * with the rate locking, the first consumer to enable to clock will be
+	 * the one definitely setting the rate of the clock.
+	 * Since audio IPs are most likely to request the same rate, we enforce
+	 * that the only clks able to modify gck rate are those of audio IPs.
+	 */
+
+	if (!gck->audio_pll_allowed)
+		goto end;
+
+	parent = clk_hw_get_parent_by_index(hw, GCK_INDEX_DT_AUDIO_PLL);
+	if (!parent)
+		goto end;
+
+	for (div = 1; div < GENERATED_MAX_DIV + 2; div++) {
+		req_parent.rate = req->rate * div;
+		__clk_determine_rate(parent, &req_parent);
+		clk_generated_best_diff(req, parent, req_parent.rate, div,
+					&best_diff, &best_rate);
 
 		if (!best_diff)
 			break;
 	}
 
+end:
 	pr_debug("GCLK: %s, best_rate = %ld, parent clk: %s @ %ld\n",
 		 __func__, best_rate,
 		 __clk_get_name((req->best_parent_hw)->clk),
@@ -252,7 +298,8 @@ at91_clk_register_generated(struct regmap *regmap, spinlock_t *lock,
 	init.ops = &generated_ops;
 	init.parent_names = parent_names;
 	init.num_parents = num_parents;
-	init.flags = CLK_SET_RATE_GATE | CLK_SET_PARENT_GATE;
+	init.flags = CLK_SET_RATE_GATE | CLK_SET_PARENT_GATE |
+		CLK_SET_RATE_PARENT;
 
 	gck->id = id;
 	gck->hw.init = &init;
@@ -284,6 +331,7 @@ static void __init of_sama5d2_clk_generated_setup(struct device_node *np)
 	struct device_node *gcknp;
 	struct clk_range range = CLK_RANGE(0, 0);
 	struct regmap *regmap;
+	struct clk_generated *gck;
 
 	num_parents = of_clk_get_parent_count(np);
 	if (num_parents == 0 || num_parents > GENERATED_SOURCE_MAX)
@@ -315,6 +363,21 @@ static void __init of_sama5d2_clk_generated_setup(struct device_node *np)
 		hw = at91_clk_register_generated(regmap, &pmc_pcr_lock, name,
 						  parent_names, num_parents,
 						  id, &range);
+
+		gck = to_clk_generated(hw);
+
+		if (of_device_is_compatible(np,
+					    "atmel,sama5d2-clk-generated")) {
+			if (gck->id == GCK_ID_SSC0 || gck->id == GCK_ID_SSC1 ||
+			    gck->id == GCK_ID_I2S0 || gck->id == GCK_ID_I2S1 ||
+			    gck->id == GCK_ID_CLASSD)
+				gck->audio_pll_allowed = true;
+			else
+				gck->audio_pll_allowed = false;
+		} else {
+			gck->audio_pll_allowed = false;
+		}
+
 		if (IS_ERR(hw))
 			continue;
 
diff --git a/drivers/clk/axs10x/Makefile b/drivers/clk/axs10x/Makefile
index 01996b871b06..d747deafbf1e 100644
--- a/drivers/clk/axs10x/Makefile
+++ b/drivers/clk/axs10x/Makefile
@@ -1 +1,2 @@
 obj-y += i2s_pll_clock.o
+obj-y += pll_clock.o
diff --git a/drivers/clk/axs10x/pll_clock.c b/drivers/clk/axs10x/pll_clock.c
new file mode 100644
index 000000000000..25d8c240ddfb
--- /dev/null
+++ b/drivers/clk/axs10x/pll_clock.c
@@ -0,0 +1,346 @@
+/*
+ * Synopsys AXS10X SDP Generic PLL clock driver
+ *
+ * Copyright (C) 2017 Synopsys
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2. This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#include <linux/platform_device.h>
+#include <linux/module.h>
+#include <linux/clk-provider.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/device.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/slab.h>
+#include <linux/of.h>
+
+/* PLL registers addresses */
+#define PLL_REG_IDIV	0x0
+#define PLL_REG_FBDIV	0x4
+#define PLL_REG_ODIV	0x8
+
+/*
+ * Bit fields of the PLL IDIV/FBDIV/ODIV registers:
+ *  ________________________________________________________________________
+ * |31                15|    14    |   13   |  12  |11         6|5         0|
+ * |-------RESRVED------|-NOUPDATE-|-BYPASS-|-EDGE-|--HIGHTIME--|--LOWTIME--|
+ * |____________________|__________|________|______|____________|___________|
+ *
+ * Following macros determine the way of access to these registers
+ * They should be set up only using the macros.
+ * reg should be an u32 variable.
+ */
+
+#define PLL_REG_GET_LOW(reg)			\
+	(((reg) & (0x3F << 0)) >> 0)
+#define PLL_REG_GET_HIGH(reg)			\
+	(((reg) & (0x3F << 6)) >> 6)
+#define PLL_REG_GET_EDGE(reg)			\
+	(((reg) & (BIT(12))) ? 1 : 0)
+#define PLL_REG_GET_BYPASS(reg)			\
+	(((reg) & (BIT(13))) ? 1 : 0)
+#define PLL_REG_GET_NOUPD(reg)			\
+	(((reg) & (BIT(14))) ? 1 : 0)
+#define PLL_REG_GET_PAD(reg)			\
+	(((reg) & (0x1FFFF << 15)) >> 15)
+
+#define PLL_REG_SET_LOW(reg, value)		\
+	{ reg |= (((value) & 0x3F) << 0); }
+#define PLL_REG_SET_HIGH(reg, value)		\
+	{ reg |= (((value) & 0x3F) << 6); }
+#define PLL_REG_SET_EDGE(reg, value)		\
+	{ reg |= (((value) & 0x01) << 12); }
+#define PLL_REG_SET_BYPASS(reg, value)		\
+	{ reg |= (((value) & 0x01) << 13); }
+#define PLL_REG_SET_NOUPD(reg, value)		\
+	{ reg |= (((value) & 0x01) << 14); }
+#define PLL_REG_SET_PAD(reg, value)		\
+	{ reg |= (((value) & 0x1FFFF) << 15); }
+
+#define PLL_LOCK	BIT(0)
+#define PLL_ERROR	BIT(1)
+#define PLL_MAX_LOCK_TIME 100 /* 100 us */
+
+struct axs10x_pll_cfg {
+	u32 rate;
+	u32 idiv;
+	u32 fbdiv;
+	u32 odiv;
+};
+
+static const struct axs10x_pll_cfg arc_pll_cfg[] = {
+	{ 33333333,  1, 1,  1 },
+	{ 50000000,  1, 30, 20 },
+	{ 75000000,  2, 45, 10 },
+	{ 90000000,  2, 54, 10 },
+	{ 100000000, 1, 30, 10 },
+	{ 125000000, 2, 45, 6 },
+	{}
+};
+
+static const struct axs10x_pll_cfg pgu_pll_cfg[] = {
+	{ 25200000, 1, 84, 90 },
+	{ 50000000, 1, 100, 54 },
+	{ 74250000, 1, 44, 16 },
+	{}
+};
+
+struct axs10x_pll_clk {
+	struct clk_hw hw;
+	void __iomem *base;
+	void __iomem *lock;
+	const struct axs10x_pll_cfg *pll_cfg;
+	struct device *dev;
+};
+
+static inline void axs10x_pll_write(struct axs10x_pll_clk *clk, u32 reg,
+				    u32 val)
+{
+	iowrite32(val, clk->base + reg);
+}
+
+static inline u32 axs10x_pll_read(struct axs10x_pll_clk *clk, u32 reg)
+{
+	return ioread32(clk->base + reg);
+}
+
+static inline struct axs10x_pll_clk *to_axs10x_pll_clk(struct clk_hw *hw)
+{
+	return container_of(hw, struct axs10x_pll_clk, hw);
+}
+
+static inline u32 axs10x_div_get_value(u32 reg)
+{
+	if (PLL_REG_GET_BYPASS(reg))
+		return 1;
+
+	return PLL_REG_GET_HIGH(reg) + PLL_REG_GET_LOW(reg);
+}
+
+static inline u32 axs10x_encode_div(unsigned int id, int upd)
+{
+	u32 div = 0;
+
+	PLL_REG_SET_LOW(div, (id % 2 == 0) ? id >> 1 : (id >> 1) + 1);
+	PLL_REG_SET_HIGH(div, id >> 1);
+	PLL_REG_SET_EDGE(div, id % 2);
+	PLL_REG_SET_BYPASS(div, id == 1 ? 1 : 0);
+	PLL_REG_SET_NOUPD(div, upd == 0 ? 1 : 0);
+
+	return div;
+}
+
+static unsigned long axs10x_pll_recalc_rate(struct clk_hw *hw,
+					    unsigned long parent_rate)
+{
+	u64 rate;
+	u32 idiv, fbdiv, odiv;
+	struct axs10x_pll_clk *clk = to_axs10x_pll_clk(hw);
+
+	idiv = axs10x_div_get_value(axs10x_pll_read(clk, PLL_REG_IDIV));
+	fbdiv = axs10x_div_get_value(axs10x_pll_read(clk, PLL_REG_FBDIV));
+	odiv = axs10x_div_get_value(axs10x_pll_read(clk, PLL_REG_ODIV));
+
+	rate = (u64)parent_rate * fbdiv;
+	do_div(rate, idiv * odiv);
+
+	return rate;
+}
+
+static long axs10x_pll_round_rate(struct clk_hw *hw, unsigned long rate,
+				  unsigned long *prate)
+{
+	int i;
+	long best_rate;
+	struct axs10x_pll_clk *clk = to_axs10x_pll_clk(hw);
+	const struct axs10x_pll_cfg *pll_cfg = clk->pll_cfg;
+
+	if (pll_cfg[0].rate == 0)
+		return -EINVAL;
+
+	best_rate = pll_cfg[0].rate;
+
+	for (i = 1; pll_cfg[i].rate != 0; i++) {
+		if (abs(rate - pll_cfg[i].rate) < abs(rate - best_rate))
+			best_rate = pll_cfg[i].rate;
+	}
+
+	return best_rate;
+}
+
+static int axs10x_pll_set_rate(struct clk_hw *hw, unsigned long rate,
+			       unsigned long parent_rate)
+{
+	int i;
+	struct axs10x_pll_clk *clk = to_axs10x_pll_clk(hw);
+	const struct axs10x_pll_cfg *pll_cfg = clk->pll_cfg;
+
+	for (i = 0; pll_cfg[i].rate != 0; i++) {
+		if (pll_cfg[i].rate == rate) {
+			axs10x_pll_write(clk, PLL_REG_IDIV,
+					 axs10x_encode_div(pll_cfg[i].idiv, 0));
+			axs10x_pll_write(clk, PLL_REG_FBDIV,
+					 axs10x_encode_div(pll_cfg[i].fbdiv, 0));
+			axs10x_pll_write(clk, PLL_REG_ODIV,
+					 axs10x_encode_div(pll_cfg[i].odiv, 1));
+
+			/*
+			 * Wait until CGU relocks and check error status.
+			 * If after timeout CGU is unlocked yet return error
+			 */
+			udelay(PLL_MAX_LOCK_TIME);
+			if (!(ioread32(clk->lock) & PLL_LOCK))
+				return -ETIMEDOUT;
+
+			if (ioread32(clk->lock) & PLL_ERROR)
+				return -EINVAL;
+
+			return 0;
+		}
+	}
+
+	dev_err(clk->dev, "invalid rate=%ld, parent_rate=%ld\n", rate,
+			parent_rate);
+	return -EINVAL;
+}
+
+static const struct clk_ops axs10x_pll_ops = {
+	.recalc_rate = axs10x_pll_recalc_rate,
+	.round_rate = axs10x_pll_round_rate,
+	.set_rate = axs10x_pll_set_rate,
+};
+
+static int axs10x_pll_clk_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	const char *parent_name;
+	struct axs10x_pll_clk *pll_clk;
+	struct resource *mem;
+	struct clk_init_data init = { };
+	int ret;
+
+	pll_clk = devm_kzalloc(dev, sizeof(*pll_clk), GFP_KERNEL);
+	if (!pll_clk)
+		return -ENOMEM;
+
+	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	pll_clk->base = devm_ioremap_resource(dev, mem);
+	if (IS_ERR(pll_clk->base))
+		return PTR_ERR(pll_clk->base);
+
+	mem = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+	pll_clk->lock = devm_ioremap_resource(dev, mem);
+	if (IS_ERR(pll_clk->lock))
+		return PTR_ERR(pll_clk->lock);
+
+	init.name = dev->of_node->name;
+	init.ops = &axs10x_pll_ops;
+	parent_name = of_clk_get_parent_name(dev->of_node, 0);
+	init.parent_names = &parent_name;
+	init.num_parents = 1;
+	pll_clk->hw.init = &init;
+	pll_clk->dev = dev;
+	pll_clk->pll_cfg = of_device_get_match_data(dev);
+
+	if (!pll_clk->pll_cfg) {
+		dev_err(dev, "No OF match data provided\n");
+		return -EINVAL;
+	}
+
+	ret = devm_clk_hw_register(dev, &pll_clk->hw);
+	if (ret) {
+		dev_err(dev, "failed to register %s clock\n", init.name);
+		return ret;
+	}
+
+	return of_clk_add_hw_provider(dev->of_node, of_clk_hw_simple_get,
+			&pll_clk->hw);
+}
+
+static int axs10x_pll_clk_remove(struct platform_device *pdev)
+{
+	of_clk_del_provider(pdev->dev.of_node);
+	return 0;
+}
+
+static void __init of_axs10x_pll_clk_setup(struct device_node *node)
+{
+	const char *parent_name;
+	struct axs10x_pll_clk *pll_clk;
+	struct clk_init_data init = { };
+	int ret;
+
+	pll_clk = kzalloc(sizeof(*pll_clk), GFP_KERNEL);
+	if (!pll_clk)
+		return;
+
+	pll_clk->base = of_iomap(node, 0);
+	if (!pll_clk->base) {
+		pr_err("failed to map pll div registers\n");
+		goto err_free_pll_clk;
+	}
+
+	pll_clk->lock = of_iomap(node, 1);
+	if (!pll_clk->lock) {
+		pr_err("failed to map pll lock register\n");
+		goto err_unmap_base;
+	}
+
+	init.name = node->name;
+	init.ops = &axs10x_pll_ops;
+	parent_name = of_clk_get_parent_name(node, 0);
+	init.parent_names = &parent_name;
+	init.num_parents = parent_name ? 1 : 0;
+	pll_clk->hw.init = &init;
+	pll_clk->pll_cfg = arc_pll_cfg;
+
+	ret = clk_hw_register(NULL, &pll_clk->hw);
+	if (ret) {
+		pr_err("failed to register %s clock\n", node->name);
+		goto err_unmap_lock;
+	}
+
+	ret = of_clk_add_hw_provider(node, of_clk_hw_simple_get, &pll_clk->hw);
+	if (ret) {
+		pr_err("failed to add hw provider for %s clock\n", node->name);
+		goto err_unregister_clk;
+	}
+
+	return;
+
+err_unregister_clk:
+	clk_hw_unregister(&pll_clk->hw);
+err_unmap_lock:
+	iounmap(pll_clk->lock);
+err_unmap_base:
+	iounmap(pll_clk->base);
+err_free_pll_clk:
+	kfree(pll_clk);
+}
+CLK_OF_DECLARE(axs10x_pll_clock, "snps,axs10x-arc-pll-clock",
+	       of_axs10x_pll_clk_setup);
+
+static const struct of_device_id axs10x_pll_clk_id[] = {
+	{ .compatible = "snps,axs10x-pgu-pll-clock", .data = &pgu_pll_cfg},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, axs10x_pll_clk_id);
+
+static struct platform_driver axs10x_pll_clk_driver = {
+	.driver = {
+		.name = "axs10x-pll-clock",
+		.of_match_table = axs10x_pll_clk_id,
+	},
+	.probe = axs10x_pll_clk_probe,
+	.remove = axs10x_pll_clk_remove,
+};
+builtin_platform_driver(axs10x_pll_clk_driver);
+
+MODULE_AUTHOR("Vlad Zakharov <vzakhar@synopsys.com>");
+MODULE_DESCRIPTION("Synopsys AXS10X SDP Generic PLL Clock Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/berlin/bg2.c b/drivers/clk/berlin/bg2.c
index 1d99292e2039..e7331ace0337 100644
--- a/drivers/clk/berlin/bg2.c
+++ b/drivers/clk/berlin/bg2.c
@@ -679,8 +679,7 @@ static void __init berlin2_clock_setup(struct device_node *np)
 		if (!IS_ERR(hws[n]))
 			continue;
 
-		pr_err("%s: Unable to register leaf clock %d\n",
-		       np->full_name, n);
+		pr_err("%pOF: Unable to register leaf clock %d\n", np, n);
 		goto bg2_fail;
 	}
 
diff --git a/drivers/clk/berlin/bg2q.c b/drivers/clk/berlin/bg2q.c
index 3b784b593afd..67c270b143f7 100644
--- a/drivers/clk/berlin/bg2q.c
+++ b/drivers/clk/berlin/bg2q.c
@@ -304,14 +304,14 @@ static void __init berlin2q_clock_setup(struct device_node *np)
 
 	gbase = of_iomap(parent_np, 0);
 	if (!gbase) {
-		pr_err("%s: Unable to map global base\n", np->full_name);
+		pr_err("%pOF: Unable to map global base\n", np);
 		return;
 	}
 
 	/* BG2Q CPU PLL is not part of global registers */
 	cpupll_base = of_iomap(parent_np, 1);
 	if (!cpupll_base) {
-		pr_err("%s: Unable to map cpupll base\n", np->full_name);
+		pr_err("%pOF: Unable to map cpupll base\n", np);
 		iounmap(gbase);
 		return;
 	}
@@ -376,8 +376,7 @@ static void __init berlin2q_clock_setup(struct device_node *np)
 		if (!IS_ERR(hws[n]))
 			continue;
 
-		pr_err("%s: Unable to register leaf clock %d\n",
-		       np->full_name, n);
+		pr_err("%pOF: Unable to register leaf clock %d\n", np, n);
 		goto bg2q_fail;
 	}
 
diff --git a/drivers/clk/clk-asm9260.c b/drivers/clk/clk-asm9260.c
index ea8568536193..bf0582cbbf38 100644
--- a/drivers/clk/clk-asm9260.c
+++ b/drivers/clk/clk-asm9260.c
@@ -338,8 +338,8 @@ static void __init asm9260_acc_init(struct device_node *np)
 		if (!IS_ERR(hws[n]))
 			continue;
 
-		pr_err("%s: Unable to register leaf clock %d\n",
-				np->full_name, n);
+		pr_err("%pOF: Unable to register leaf clock %d\n",
+				np, n);
 		goto fail;
 	}
 
diff --git a/drivers/clk/clk-conf.c b/drivers/clk/clk-conf.c
index 7ec36722f8ab..49819b546134 100644
--- a/drivers/clk/clk-conf.c
+++ b/drivers/clk/clk-conf.c
@@ -23,8 +23,8 @@ static int __set_clk_parents(struct device_node *node, bool clk_supplier)
 	num_parents = of_count_phandle_with_args(node, "assigned-clock-parents",
 						 "#clock-cells");
 	if (num_parents == -EINVAL)
-		pr_err("clk: invalid value of clock-parents property at %s\n",
-		       node->full_name);
+		pr_err("clk: invalid value of clock-parents property at %pOF\n",
+		       node);
 
 	for (index = 0; index < num_parents; index++) {
 		rc = of_parse_phandle_with_args(node, "assigned-clock-parents",
@@ -41,8 +41,8 @@ static int __set_clk_parents(struct device_node *node, bool clk_supplier)
 		pclk = of_clk_get_from_provider(&clkspec);
 		if (IS_ERR(pclk)) {
 			if (PTR_ERR(pclk) != -EPROBE_DEFER)
-				pr_warn("clk: couldn't get parent clock %d for %s\n",
-					index, node->full_name);
+				pr_warn("clk: couldn't get parent clock %d for %pOF\n",
+					index, node);
 			return PTR_ERR(pclk);
 		}
 
@@ -57,8 +57,8 @@ static int __set_clk_parents(struct device_node *node, bool clk_supplier)
 		clk = of_clk_get_from_provider(&clkspec);
 		if (IS_ERR(clk)) {
 			if (PTR_ERR(clk) != -EPROBE_DEFER)
-				pr_warn("clk: couldn't get assigned clock %d for %s\n",
-					index, node->full_name);
+				pr_warn("clk: couldn't get assigned clock %d for %pOF\n",
+					index, node);
 			rc = PTR_ERR(clk);
 			goto err;
 		}
@@ -102,8 +102,8 @@ static int __set_clk_rates(struct device_node *node, bool clk_supplier)
 			clk = of_clk_get_from_provider(&clkspec);
 			if (IS_ERR(clk)) {
 				if (PTR_ERR(clk) != -EPROBE_DEFER)
-					pr_warn("clk: couldn't get clock %d for %s\n",
-						index, node->full_name);
+					pr_warn("clk: couldn't get clock %d for %pOF\n",
+						index, node);
 				return PTR_ERR(clk);
 			}
 
diff --git a/drivers/clk/clk-cs2000-cp.c b/drivers/clk/clk-cs2000-cp.c
index c54baede4d68..e8ea81c30f0c 100644
--- a/drivers/clk/clk-cs2000-cp.c
+++ b/drivers/clk/clk-cs2000-cp.c
@@ -343,6 +343,15 @@ static int cs2000_set_rate(struct clk_hw *hw,
 	return __cs2000_set_rate(priv, ch, rate, parent_rate);
 }
 
+static int cs2000_set_saved_rate(struct cs2000_priv *priv)
+{
+	int ch = 0; /* it uses ch0 only at this point */
+
+	return __cs2000_set_rate(priv, ch,
+				 priv->saved_rate,
+				 priv->saved_parent_rate);
+}
+
 static int cs2000_enable(struct clk_hw *hw)
 {
 	struct cs2000_priv *priv = hw_to_priv(hw);
@@ -535,11 +544,8 @@ probe_err:
 static int cs2000_resume(struct device *dev)
 {
 	struct cs2000_priv *priv = dev_get_drvdata(dev);
-	int ch = 0; /* it uses ch0 only at this point */
 
-	return __cs2000_set_rate(priv, ch,
-				 priv->saved_rate,
-				 priv->saved_parent_rate);
+	return cs2000_set_saved_rate(priv);
 }
 
 static const struct dev_pm_ops cs2000_pm_ops = {
diff --git a/drivers/clk/clk-divider.c b/drivers/clk/clk-divider.c
index 9bb472cccca6..4ed516cb7276 100644
--- a/drivers/clk/clk-divider.c
+++ b/drivers/clk/clk-divider.c
@@ -385,12 +385,14 @@ static int clk_divider_set_rate(struct clk_hw *hw, unsigned long rate,
 				unsigned long parent_rate)
 {
 	struct clk_divider *divider = to_clk_divider(hw);
-	unsigned int value;
+	int value;
 	unsigned long flags = 0;
 	u32 val;
 
 	value = divider_get_val(rate, parent_rate, divider->table,
 				divider->width, divider->flags);
+	if (value < 0)
+		return value;
 
 	if (divider->lock)
 		spin_lock_irqsave(divider->lock, flags);
@@ -403,7 +405,7 @@ static int clk_divider_set_rate(struct clk_hw *hw, unsigned long rate,
 		val = clk_readl(divider->reg);
 		val &= ~(div_mask(divider->width) << divider->shift);
 	}
-	val |= value << divider->shift;
+	val |= (u32)value << divider->shift;
 	clk_writel(val, divider->reg);
 
 	if (divider->lock)
diff --git a/drivers/clk/clk-fractional-divider.c b/drivers/clk/clk-fractional-divider.c
index aab904618eb6..fdf625fb10fa 100644
--- a/drivers/clk/clk-fractional-divider.c
+++ b/drivers/clk/clk-fractional-divider.c
@@ -49,16 +49,12 @@ static unsigned long clk_fd_recalc_rate(struct clk_hw *hw,
 	return ret;
 }
 
-static long clk_fd_round_rate(struct clk_hw *hw, unsigned long rate,
-			      unsigned long *parent_rate)
+static void clk_fd_general_approximation(struct clk_hw *hw, unsigned long rate,
+					 unsigned long *parent_rate,
+					 unsigned long *m, unsigned long *n)
 {
 	struct clk_fractional_divider *fd = to_clk_fd(hw);
 	unsigned long scale;
-	unsigned long m, n;
-	u64 ret;
-
-	if (!rate || rate >= *parent_rate)
-		return *parent_rate;
 
 	/*
 	 * Get rate closer to *parent_rate to guarantee there is no overflow
@@ -71,7 +67,23 @@ static long clk_fd_round_rate(struct clk_hw *hw, unsigned long rate,
 
 	rational_best_approximation(rate, *parent_rate,
 			GENMASK(fd->mwidth - 1, 0), GENMASK(fd->nwidth - 1, 0),
-			&m, &n);
+			m, n);
+}
+
+static long clk_fd_round_rate(struct clk_hw *hw, unsigned long rate,
+			      unsigned long *parent_rate)
+{
+	struct clk_fractional_divider *fd = to_clk_fd(hw);
+	unsigned long m, n;
+	u64 ret;
+
+	if (!rate || rate >= *parent_rate)
+		return *parent_rate;
+
+	if (fd->approximation)
+		fd->approximation(hw, rate, parent_rate, &m, &n);
+	else
+		clk_fd_general_approximation(hw, rate, parent_rate, &m, &n);
 
 	ret = (u64)*parent_rate * m;
 	do_div(ret, n);
diff --git a/drivers/clk/clk-gate.c b/drivers/clk/clk-gate.c
index 4e0c054a787c..dd82485e09a1 100644
--- a/drivers/clk/clk-gate.c
+++ b/drivers/clk/clk-gate.c
@@ -86,7 +86,7 @@ static void clk_gate_disable(struct clk_hw *hw)
 	clk_gate_endisable(hw, 0);
 }
 
-static int clk_gate_is_enabled(struct clk_hw *hw)
+int clk_gate_is_enabled(struct clk_hw *hw)
 {
 	u32 reg;
 	struct clk_gate *gate = to_clk_gate(hw);
@@ -101,6 +101,7 @@ static int clk_gate_is_enabled(struct clk_hw *hw)
 
 	return reg ? 1 : 0;
 }
+EXPORT_SYMBOL_GPL(clk_gate_is_enabled);
 
 const struct clk_ops clk_gate_ops = {
 	.enable = clk_gate_enable,
diff --git a/drivers/clk/clk-gemini.c b/drivers/clk/clk-gemini.c
index b4cf2f699a21..f940e5af845b 100644
--- a/drivers/clk/clk-gemini.c
+++ b/drivers/clk/clk-gemini.c
@@ -37,7 +37,6 @@ static DEFINE_SPINLOCK(gemini_clk_lock);
 
 #define GEMINI_GLOBAL_MISC_CONTROL	0x30
 #define PCI_CLK_66MHZ			BIT(18)
-#define PCI_CLK_OE			BIT(17)
 
 #define GEMINI_GLOBAL_CLOCK_CONTROL	0x34
 #define PCI_CLKRUN_EN			BIT(16)
@@ -159,9 +158,6 @@ static int gemini_pci_enable(struct clk_hw *hw)
 
 	regmap_update_bits(pciclk->map, GEMINI_GLOBAL_CLOCK_CONTROL,
 			   0, PCI_CLKRUN_EN);
-	regmap_update_bits(pciclk->map,
-			   GEMINI_GLOBAL_MISC_CONTROL,
-			   0, PCI_CLK_OE);
 	return 0;
 }
 
@@ -169,9 +165,6 @@ static void gemini_pci_disable(struct clk_hw *hw)
 {
 	struct clk_gemini_pci *pciclk = to_pciclk(hw);
 
-	regmap_update_bits(pciclk->map,
-			   GEMINI_GLOBAL_MISC_CONTROL,
-			   PCI_CLK_OE, 0);
 	regmap_update_bits(pciclk->map, GEMINI_GLOBAL_CLOCK_CONTROL,
 			   PCI_CLKRUN_EN, 0);
 }
diff --git a/drivers/clk/clk-hsdk-pll.c b/drivers/clk/clk-hsdk-pll.c
new file mode 100644
index 000000000000..bbf237173b37
--- /dev/null
+++ b/drivers/clk/clk-hsdk-pll.c
@@ -0,0 +1,431 @@
+/*
+ * Synopsys HSDK SDP Generic PLL clock driver
+ *
+ * Copyright (C) 2017 Synopsys
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2. This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#define CGU_PLL_CTRL	0x000 /* ARC PLL control register */
+#define CGU_PLL_STATUS	0x004 /* ARC PLL status register */
+#define CGU_PLL_FMEAS	0x008 /* ARC PLL frequency measurement register */
+#define CGU_PLL_MON	0x00C /* ARC PLL monitor register */
+
+#define CGU_PLL_CTRL_ODIV_SHIFT		2
+#define CGU_PLL_CTRL_IDIV_SHIFT		4
+#define CGU_PLL_CTRL_FBDIV_SHIFT	9
+#define CGU_PLL_CTRL_BAND_SHIFT		20
+
+#define CGU_PLL_CTRL_ODIV_MASK		GENMASK(3, CGU_PLL_CTRL_ODIV_SHIFT)
+#define CGU_PLL_CTRL_IDIV_MASK		GENMASK(8, CGU_PLL_CTRL_IDIV_SHIFT)
+#define CGU_PLL_CTRL_FBDIV_MASK		GENMASK(15, CGU_PLL_CTRL_FBDIV_SHIFT)
+
+#define CGU_PLL_CTRL_PD			BIT(0)
+#define CGU_PLL_CTRL_BYPASS		BIT(1)
+
+#define CGU_PLL_STATUS_LOCK		BIT(0)
+#define CGU_PLL_STATUS_ERR		BIT(1)
+
+#define HSDK_PLL_MAX_LOCK_TIME		100 /* 100 us */
+
+#define CGU_PLL_SOURCE_MAX		1
+
+#define CORE_IF_CLK_THRESHOLD_HZ	500000000
+#define CREG_CORE_IF_CLK_DIV_1		0x0
+#define CREG_CORE_IF_CLK_DIV_2		0x1
+
+struct hsdk_pll_cfg {
+	u32 rate;
+	u32 idiv;
+	u32 fbdiv;
+	u32 odiv;
+	u32 band;
+};
+
+static const struct hsdk_pll_cfg asdt_pll_cfg[] = {
+	{ 100000000,  0, 11, 3, 0 },
+	{ 133000000,  0, 15, 3, 0 },
+	{ 200000000,  1, 47, 3, 0 },
+	{ 233000000,  1, 27, 2, 0 },
+	{ 300000000,  1, 35, 2, 0 },
+	{ 333000000,  1, 39, 2, 0 },
+	{ 400000000,  1, 47, 2, 0 },
+	{ 500000000,  0, 14, 1, 0 },
+	{ 600000000,  0, 17, 1, 0 },
+	{ 700000000,  0, 20, 1, 0 },
+	{ 800000000,  0, 23, 1, 0 },
+	{ 900000000,  1, 26, 0, 0 },
+	{ 1000000000, 1, 29, 0, 0 },
+	{ 1100000000, 1, 32, 0, 0 },
+	{ 1200000000, 1, 35, 0, 0 },
+	{ 1300000000, 1, 38, 0, 0 },
+	{ 1400000000, 1, 41, 0, 0 },
+	{ 1500000000, 1, 44, 0, 0 },
+	{ 1600000000, 1, 47, 0, 0 },
+	{}
+};
+
+static const struct hsdk_pll_cfg hdmi_pll_cfg[] = {
+	{ 297000000,  0, 21, 2, 0 },
+	{ 540000000,  0, 19, 1, 0 },
+	{ 594000000,  0, 21, 1, 0 },
+	{}
+};
+
+struct hsdk_pll_clk {
+	struct clk_hw hw;
+	void __iomem *regs;
+	void __iomem *spec_regs;
+	const struct hsdk_pll_devdata *pll_devdata;
+	struct device *dev;
+};
+
+struct hsdk_pll_devdata {
+	const struct hsdk_pll_cfg *pll_cfg;
+	int (*update_rate)(struct hsdk_pll_clk *clk, unsigned long rate,
+			   const struct hsdk_pll_cfg *cfg);
+};
+
+static int hsdk_pll_core_update_rate(struct hsdk_pll_clk *, unsigned long,
+				     const struct hsdk_pll_cfg *);
+static int hsdk_pll_comm_update_rate(struct hsdk_pll_clk *, unsigned long,
+				     const struct hsdk_pll_cfg *);
+
+static const struct hsdk_pll_devdata core_pll_devdata = {
+	.pll_cfg = asdt_pll_cfg,
+	.update_rate = hsdk_pll_core_update_rate,
+};
+
+static const struct hsdk_pll_devdata sdt_pll_devdata = {
+	.pll_cfg = asdt_pll_cfg,
+	.update_rate = hsdk_pll_comm_update_rate,
+};
+
+static const struct hsdk_pll_devdata hdmi_pll_devdata = {
+	.pll_cfg = hdmi_pll_cfg,
+	.update_rate = hsdk_pll_comm_update_rate,
+};
+
+static inline void hsdk_pll_write(struct hsdk_pll_clk *clk, u32 reg, u32 val)
+{
+	iowrite32(val, clk->regs + reg);
+}
+
+static inline u32 hsdk_pll_read(struct hsdk_pll_clk *clk, u32 reg)
+{
+	return ioread32(clk->regs + reg);
+}
+
+static inline void hsdk_pll_set_cfg(struct hsdk_pll_clk *clk,
+				    const struct hsdk_pll_cfg *cfg)
+{
+	u32 val = 0;
+
+	/* Powerdown and Bypass bits should be cleared */
+	val |= cfg->idiv << CGU_PLL_CTRL_IDIV_SHIFT;
+	val |= cfg->fbdiv << CGU_PLL_CTRL_FBDIV_SHIFT;
+	val |= cfg->odiv << CGU_PLL_CTRL_ODIV_SHIFT;
+	val |= cfg->band << CGU_PLL_CTRL_BAND_SHIFT;
+
+	dev_dbg(clk->dev, "write configurarion: %#x\n", val);
+
+	hsdk_pll_write(clk, CGU_PLL_CTRL, val);
+}
+
+static inline bool hsdk_pll_is_locked(struct hsdk_pll_clk *clk)
+{
+	return !!(hsdk_pll_read(clk, CGU_PLL_STATUS) & CGU_PLL_STATUS_LOCK);
+}
+
+static inline bool hsdk_pll_is_err(struct hsdk_pll_clk *clk)
+{
+	return !!(hsdk_pll_read(clk, CGU_PLL_STATUS) & CGU_PLL_STATUS_ERR);
+}
+
+static inline struct hsdk_pll_clk *to_hsdk_pll_clk(struct clk_hw *hw)
+{
+	return container_of(hw, struct hsdk_pll_clk, hw);
+}
+
+static unsigned long hsdk_pll_recalc_rate(struct clk_hw *hw,
+					  unsigned long parent_rate)
+{
+	u32 val;
+	u64 rate;
+	u32 idiv, fbdiv, odiv;
+	struct hsdk_pll_clk *clk = to_hsdk_pll_clk(hw);
+
+	val = hsdk_pll_read(clk, CGU_PLL_CTRL);
+
+	dev_dbg(clk->dev, "current configurarion: %#x\n", val);
+
+	/* Check if PLL is disabled */
+	if (val & CGU_PLL_CTRL_PD)
+		return 0;
+
+	/* Check if PLL is bypassed */
+	if (val & CGU_PLL_CTRL_BYPASS)
+		return parent_rate;
+
+	/* input divider = reg.idiv + 1 */
+	idiv = 1 + ((val & CGU_PLL_CTRL_IDIV_MASK) >> CGU_PLL_CTRL_IDIV_SHIFT);
+	/* fb divider = 2*(reg.fbdiv + 1) */
+	fbdiv = 2 * (1 + ((val & CGU_PLL_CTRL_FBDIV_MASK) >> CGU_PLL_CTRL_FBDIV_SHIFT));
+	/* output divider = 2^(reg.odiv) */
+	odiv = 1 << ((val & CGU_PLL_CTRL_ODIV_MASK) >> CGU_PLL_CTRL_ODIV_SHIFT);
+
+	rate = (u64)parent_rate * fbdiv;
+	do_div(rate, idiv * odiv);
+
+	return rate;
+}
+
+static long hsdk_pll_round_rate(struct clk_hw *hw, unsigned long rate,
+				unsigned long *prate)
+{
+	int i;
+	unsigned long best_rate;
+	struct hsdk_pll_clk *clk = to_hsdk_pll_clk(hw);
+	const struct hsdk_pll_cfg *pll_cfg = clk->pll_devdata->pll_cfg;
+
+	if (pll_cfg[0].rate == 0)
+		return -EINVAL;
+
+	best_rate = pll_cfg[0].rate;
+
+	for (i = 1; pll_cfg[i].rate != 0; i++) {
+		if (abs(rate - pll_cfg[i].rate) < abs(rate - best_rate))
+			best_rate = pll_cfg[i].rate;
+	}
+
+	dev_dbg(clk->dev, "chosen best rate: %lu\n", best_rate);
+
+	return best_rate;
+}
+
+static int hsdk_pll_comm_update_rate(struct hsdk_pll_clk *clk,
+				     unsigned long rate,
+				     const struct hsdk_pll_cfg *cfg)
+{
+	hsdk_pll_set_cfg(clk, cfg);
+
+	/*
+	 * Wait until CGU relocks and check error status.
+	 * If after timeout CGU is unlocked yet return error.
+	 */
+	udelay(HSDK_PLL_MAX_LOCK_TIME);
+	if (!hsdk_pll_is_locked(clk))
+		return -ETIMEDOUT;
+
+	if (hsdk_pll_is_err(clk))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int hsdk_pll_core_update_rate(struct hsdk_pll_clk *clk,
+				     unsigned long rate,
+				     const struct hsdk_pll_cfg *cfg)
+{
+	/*
+	 * When core clock exceeds 500MHz, the divider for the interface
+	 * clock must be programmed to div-by-2.
+	 */
+	if (rate > CORE_IF_CLK_THRESHOLD_HZ)
+		iowrite32(CREG_CORE_IF_CLK_DIV_2, clk->spec_regs);
+
+	hsdk_pll_set_cfg(clk, cfg);
+
+	/*
+	 * Wait until CGU relocks and check error status.
+	 * If after timeout CGU is unlocked yet return error.
+	 */
+	udelay(HSDK_PLL_MAX_LOCK_TIME);
+	if (!hsdk_pll_is_locked(clk))
+		return -ETIMEDOUT;
+
+	if (hsdk_pll_is_err(clk))
+		return -EINVAL;
+
+	/*
+	 * Program divider to div-by-1 if we succesfuly set core clock below
+	 * 500MHz threshold.
+	 */
+	if (rate <= CORE_IF_CLK_THRESHOLD_HZ)
+		iowrite32(CREG_CORE_IF_CLK_DIV_1, clk->spec_regs);
+
+	return 0;
+}
+
+static int hsdk_pll_set_rate(struct clk_hw *hw, unsigned long rate,
+			     unsigned long parent_rate)
+{
+	int i;
+	struct hsdk_pll_clk *clk = to_hsdk_pll_clk(hw);
+	const struct hsdk_pll_cfg *pll_cfg = clk->pll_devdata->pll_cfg;
+
+	for (i = 0; pll_cfg[i].rate != 0; i++) {
+		if (pll_cfg[i].rate == rate) {
+			return clk->pll_devdata->update_rate(clk, rate,
+							     &pll_cfg[i]);
+		}
+	}
+
+	dev_err(clk->dev, "invalid rate=%ld, parent_rate=%ld\n", rate,
+			parent_rate);
+
+	return -EINVAL;
+}
+
+static const struct clk_ops hsdk_pll_ops = {
+	.recalc_rate = hsdk_pll_recalc_rate,
+	.round_rate = hsdk_pll_round_rate,
+	.set_rate = hsdk_pll_set_rate,
+};
+
+static int hsdk_pll_clk_probe(struct platform_device *pdev)
+{
+	int ret;
+	struct resource *mem;
+	const char *parent_name;
+	unsigned int num_parents;
+	struct hsdk_pll_clk *pll_clk;
+	struct clk_init_data init = { };
+	struct device *dev = &pdev->dev;
+
+	pll_clk = devm_kzalloc(dev, sizeof(*pll_clk), GFP_KERNEL);
+	if (!pll_clk)
+		return -ENOMEM;
+
+	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	pll_clk->regs = devm_ioremap_resource(dev, mem);
+	if (IS_ERR(pll_clk->regs))
+		return PTR_ERR(pll_clk->regs);
+
+	init.name = dev->of_node->name;
+	init.ops = &hsdk_pll_ops;
+	parent_name = of_clk_get_parent_name(dev->of_node, 0);
+	init.parent_names = &parent_name;
+	num_parents = of_clk_get_parent_count(dev->of_node);
+	if (num_parents == 0 || num_parents > CGU_PLL_SOURCE_MAX) {
+		dev_err(dev, "wrong clock parents number: %u\n", num_parents);
+		return -EINVAL;
+	}
+	init.num_parents = num_parents;
+
+	pll_clk->hw.init = &init;
+	pll_clk->dev = dev;
+	pll_clk->pll_devdata = of_device_get_match_data(dev);
+
+	if (!pll_clk->pll_devdata) {
+		dev_err(dev, "No OF match data provided\n");
+		return -EINVAL;
+	}
+
+	ret = devm_clk_hw_register(dev, &pll_clk->hw);
+	if (ret) {
+		dev_err(dev, "failed to register %s clock\n", init.name);
+		return ret;
+	}
+
+	return of_clk_add_hw_provider(dev->of_node, of_clk_hw_simple_get,
+			&pll_clk->hw);
+}
+
+static int hsdk_pll_clk_remove(struct platform_device *pdev)
+{
+	of_clk_del_provider(pdev->dev.of_node);
+	return 0;
+}
+
+static void __init of_hsdk_pll_clk_setup(struct device_node *node)
+{
+	int ret;
+	const char *parent_name;
+	unsigned int num_parents;
+	struct hsdk_pll_clk *pll_clk;
+	struct clk_init_data init = { };
+
+	pll_clk = kzalloc(sizeof(*pll_clk), GFP_KERNEL);
+	if (!pll_clk)
+		return;
+
+	pll_clk->regs = of_iomap(node, 0);
+	if (!pll_clk->regs) {
+		pr_err("failed to map pll registers\n");
+		goto err_free_pll_clk;
+	}
+
+	pll_clk->spec_regs = of_iomap(node, 1);
+	if (!pll_clk->spec_regs) {
+		pr_err("failed to map pll registers\n");
+		goto err_unmap_comm_regs;
+	}
+
+	init.name = node->name;
+	init.ops = &hsdk_pll_ops;
+	parent_name = of_clk_get_parent_name(node, 0);
+	init.parent_names = &parent_name;
+	num_parents = of_clk_get_parent_count(node);
+	if (num_parents > CGU_PLL_SOURCE_MAX) {
+		pr_err("too much clock parents: %u\n", num_parents);
+		goto err_unmap_spec_regs;
+	}
+	init.num_parents = num_parents;
+
+	pll_clk->hw.init = &init;
+	pll_clk->pll_devdata = &core_pll_devdata;
+
+	ret = clk_hw_register(NULL, &pll_clk->hw);
+	if (ret) {
+		pr_err("failed to register %s clock\n", node->name);
+		goto err_unmap_spec_regs;
+	}
+
+	ret = of_clk_add_hw_provider(node, of_clk_hw_simple_get, &pll_clk->hw);
+	if (ret) {
+		pr_err("failed to add hw provider for %s clock\n", node->name);
+		goto err_unmap_spec_regs;
+	}
+
+	return;
+
+err_unmap_spec_regs:
+	iounmap(pll_clk->spec_regs);
+err_unmap_comm_regs:
+	iounmap(pll_clk->regs);
+err_free_pll_clk:
+	kfree(pll_clk);
+}
+
+/* Core PLL needed early for ARC cpus timers */
+CLK_OF_DECLARE(hsdk_pll_clock, "snps,hsdk-core-pll-clock",
+of_hsdk_pll_clk_setup);
+
+static const struct of_device_id hsdk_pll_clk_id[] = {
+	{ .compatible = "snps,hsdk-gp-pll-clock", .data = &sdt_pll_devdata},
+	{ .compatible = "snps,hsdk-hdmi-pll-clock", .data = &hdmi_pll_devdata},
+	{ }
+};
+
+static struct platform_driver hsdk_pll_clk_driver = {
+	.driver = {
+		.name = "hsdk-gp-pll-clock",
+		.of_match_table = hsdk_pll_clk_id,
+	},
+	.probe = hsdk_pll_clk_probe,
+	.remove = hsdk_pll_clk_remove,
+};
+builtin_platform_driver(hsdk_pll_clk_driver);
diff --git a/drivers/clk/clk-mb86s7x.c b/drivers/clk/clk-mb86s7x.c
deleted file mode 100644
index 2a83a3ff1d09..000000000000
--- a/drivers/clk/clk-mb86s7x.c
+++ /dev/null
@@ -1,390 +0,0 @@
-/*
- * Copyright (C) 2013-2015 FUJITSU SEMICONDUCTOR LIMITED
- * Copyright (C) 2015 Linaro Ltd.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, version 2 of the License.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
-
-#include <linux/clkdev.h>
-#include <linux/err.h>
-#include <linux/io.h>
-#include <linux/of.h>
-#include <linux/cpu.h>
-#include <linux/clk-provider.h>
-#include <linux/spinlock.h>
-#include <linux/module.h>
-#include <linux/topology.h>
-#include <linux/mailbox_client.h>
-#include <linux/platform_device.h>
-
-#include <soc/mb86s7x/scb_mhu.h>
-
-#define to_crg_clk(p) container_of(p, struct crg_clk, hw)
-#define to_clc_clk(p) container_of(p, struct cl_clk, hw)
-
-struct mb86s7x_peri_clk {
-	u32 payload_size;
-	u32 cntrlr;
-	u32 domain;
-	u32 port;
-	u32 en;
-	u64 frequency;
-} __packed __aligned(4);
-
-struct hack_rate {
-	unsigned clk_id;
-	unsigned long rate;
-	int gated;
-};
-
-struct crg_clk {
-	struct clk_hw hw;
-	u8 cntrlr, domain, port;
-};
-
-static int crg_gate_control(struct clk_hw *hw, int en)
-{
-	struct crg_clk *crgclk = to_crg_clk(hw);
-	struct mb86s7x_peri_clk cmd;
-	int ret;
-
-	cmd.payload_size = sizeof(cmd);
-	cmd.cntrlr = crgclk->cntrlr;
-	cmd.domain = crgclk->domain;
-	cmd.port = crgclk->port;
-	cmd.en = en;
-
-	/* Port is UngatedCLK */
-	if (cmd.port == 8)
-		return en ? 0 : -EINVAL;
-
-	pr_debug("%s:%d CMD Cntrlr-%u Dom-%u Port-%u En-%u}\n",
-		 __func__, __LINE__, cmd.cntrlr,
-		 cmd.domain, cmd.port, cmd.en);
-
-	ret = mb86s7x_send_packet(CMD_PERI_CLOCK_GATE_SET_REQ,
-				  &cmd, sizeof(cmd));
-	if (ret < 0) {
-		pr_err("%s:%d failed!\n", __func__, __LINE__);
-		return ret;
-	}
-
-	pr_debug("%s:%d REP Cntrlr-%u Dom-%u Port-%u En-%u}\n",
-		 __func__, __LINE__, cmd.cntrlr,
-		 cmd.domain, cmd.port, cmd.en);
-
-	/* If the request was rejected */
-	if (cmd.en != en)
-		ret = -EINVAL;
-	else
-		ret = 0;
-
-	return ret;
-}
-
-static int crg_port_prepare(struct clk_hw *hw)
-{
-	return crg_gate_control(hw, 1);
-}
-
-static void crg_port_unprepare(struct clk_hw *hw)
-{
-	crg_gate_control(hw, 0);
-}
-
-static int
-crg_rate_control(struct clk_hw *hw, int set, unsigned long *rate)
-{
-	struct crg_clk *crgclk = to_crg_clk(hw);
-	struct mb86s7x_peri_clk cmd;
-	int code, ret;
-
-	cmd.payload_size = sizeof(cmd);
-	cmd.cntrlr = crgclk->cntrlr;
-	cmd.domain = crgclk->domain;
-	cmd.port = crgclk->port;
-	cmd.frequency = *rate;
-
-	if (set) {
-		code = CMD_PERI_CLOCK_RATE_SET_REQ;
-		pr_debug("%s:%d CMD Cntrlr-%u Dom-%u Port-%u Rate-SET %lluHz}\n",
-			 __func__, __LINE__, cmd.cntrlr,
-			 cmd.domain, cmd.port, cmd.frequency);
-	} else {
-		code = CMD_PERI_CLOCK_RATE_GET_REQ;
-		pr_debug("%s:%d CMD Cntrlr-%u Dom-%u Port-%u Rate-GET}\n",
-			 __func__, __LINE__, cmd.cntrlr,
-			 cmd.domain, cmd.port);
-	}
-
-	ret = mb86s7x_send_packet(code, &cmd, sizeof(cmd));
-	if (ret < 0) {
-		pr_err("%s:%d failed!\n", __func__, __LINE__);
-		return ret;
-	}
-
-	if (set)
-		pr_debug("%s:%d REP Cntrlr-%u Dom-%u Port-%u Rate-SET %lluHz}\n",
-			 __func__, __LINE__, cmd.cntrlr,
-			 cmd.domain, cmd.port, cmd.frequency);
-	else
-		pr_debug("%s:%d REP Cntrlr-%u Dom-%u Port-%u Rate-GOT %lluHz}\n",
-			 __func__, __LINE__, cmd.cntrlr,
-			 cmd.domain, cmd.port, cmd.frequency);
-
-	*rate = cmd.frequency;
-	return 0;
-}
-
-static unsigned long
-crg_port_recalc_rate(struct clk_hw *hw,	unsigned long parent_rate)
-{
-	unsigned long rate;
-
-	crg_rate_control(hw, 0, &rate);
-
-	return rate;
-}
-
-static long
-crg_port_round_rate(struct clk_hw *hw,
-		    unsigned long rate, unsigned long *pr)
-{
-	return rate;
-}
-
-static int
-crg_port_set_rate(struct clk_hw *hw,
-		  unsigned long rate, unsigned long parent_rate)
-{
-	return crg_rate_control(hw, 1, &rate);
-}
-
-const struct clk_ops crg_port_ops = {
-	.prepare = crg_port_prepare,
-	.unprepare = crg_port_unprepare,
-	.recalc_rate = crg_port_recalc_rate,
-	.round_rate = crg_port_round_rate,
-	.set_rate = crg_port_set_rate,
-};
-
-struct mb86s70_crg11 {
-	struct mutex lock; /* protects CLK populating and searching */
-};
-
-static struct clk *crg11_get(struct of_phandle_args *clkspec, void *data)
-{
-	struct mb86s70_crg11 *crg11 = data;
-	struct clk_init_data init;
-	u32 cntrlr, domain, port;
-	struct crg_clk *crgclk;
-	struct clk *clk;
-	char clkp[20];
-
-	if (clkspec->args_count != 3)
-		return ERR_PTR(-EINVAL);
-
-	cntrlr = clkspec->args[0];
-	domain = clkspec->args[1];
-	port = clkspec->args[2];
-
-	if (port > 7)
-		snprintf(clkp, 20, "UngatedCLK%d_%X", cntrlr, domain);
-	else
-		snprintf(clkp, 20, "CLK%d_%X_%d", cntrlr, domain, port);
-
-	mutex_lock(&crg11->lock);
-
-	clk = __clk_lookup(clkp);
-	if (clk) {
-		mutex_unlock(&crg11->lock);
-		return clk;
-	}
-
-	crgclk = kzalloc(sizeof(*crgclk), GFP_KERNEL);
-	if (!crgclk) {
-		mutex_unlock(&crg11->lock);
-		return ERR_PTR(-ENOMEM);
-	}
-
-	init.name = clkp;
-	init.num_parents = 0;
-	init.ops = &crg_port_ops;
-	init.flags = 0;
-	crgclk->hw.init = &init;
-	crgclk->cntrlr = cntrlr;
-	crgclk->domain = domain;
-	crgclk->port = port;
-	clk = clk_register(NULL, &crgclk->hw);
-	if (IS_ERR(clk))
-		pr_err("%s:%d Error!\n", __func__, __LINE__);
-	else
-		pr_debug("Registered %s\n", clkp);
-
-	clk_register_clkdev(clk, clkp, NULL);
-	mutex_unlock(&crg11->lock);
-	return clk;
-}
-
-static void __init crg_port_init(struct device_node *node)
-{
-	struct mb86s70_crg11 *crg11;
-
-	crg11 = kzalloc(sizeof(*crg11), GFP_KERNEL);
-	if (!crg11)
-		return;
-
-	mutex_init(&crg11->lock);
-
-	of_clk_add_provider(node, crg11_get, crg11);
-}
-CLK_OF_DECLARE(crg11_gate, "fujitsu,mb86s70-crg11", crg_port_init);
-
-struct cl_clk {
-	struct clk_hw hw;
-	int cluster;
-};
-
-struct mb86s7x_cpu_freq {
-	u32 payload_size;
-	u32 cluster_class;
-	u32 cluster_id;
-	u32 cpu_id;
-	u64 frequency;
-};
-
-static void mhu_cluster_rate(struct clk_hw *hw, unsigned long *rate, int get)
-{
-	struct cl_clk *clc = to_clc_clk(hw);
-	struct mb86s7x_cpu_freq cmd;
-	int code, ret;
-
-	cmd.payload_size = sizeof(cmd);
-	cmd.cluster_class = 0;
-	cmd.cluster_id = clc->cluster;
-	cmd.cpu_id = 0;
-	cmd.frequency = *rate;
-
-	if (get)
-		code = CMD_CPU_CLOCK_RATE_GET_REQ;
-	else
-		code = CMD_CPU_CLOCK_RATE_SET_REQ;
-
-	pr_debug("%s:%d CMD Cl_Class-%u CL_ID-%u CPU_ID-%u Freq-%llu}\n",
-		 __func__, __LINE__, cmd.cluster_class,
-		 cmd.cluster_id, cmd.cpu_id, cmd.frequency);
-
-	ret = mb86s7x_send_packet(code, &cmd, sizeof(cmd));
-	if (ret < 0) {
-		pr_err("%s:%d failed!\n", __func__, __LINE__);
-		return;
-	}
-
-	pr_debug("%s:%d REP Cl_Class-%u CL_ID-%u CPU_ID-%u Freq-%llu}\n",
-		 __func__, __LINE__, cmd.cluster_class,
-		 cmd.cluster_id, cmd.cpu_id, cmd.frequency);
-
-	*rate = cmd.frequency;
-}
-
-static unsigned long
-clc_recalc_rate(struct clk_hw *hw, unsigned long unused)
-{
-	unsigned long rate;
-
-	mhu_cluster_rate(hw, &rate, 1);
-	return rate;
-}
-
-static long
-clc_round_rate(struct clk_hw *hw, unsigned long rate,
-	       unsigned long *unused)
-{
-	return rate;
-}
-
-static int
-clc_set_rate(struct clk_hw *hw, unsigned long rate,
-	     unsigned long unused)
-{
-	unsigned long res = rate;
-
-	mhu_cluster_rate(hw, &res, 0);
-
-	return (res == rate) ? 0 : -EINVAL;
-}
-
-static struct clk_ops clk_clc_ops = {
-	.recalc_rate = clc_recalc_rate,
-	.round_rate = clc_round_rate,
-	.set_rate = clc_set_rate,
-};
-
-static struct clk_hw *mb86s7x_clclk_register(struct device *cpu_dev)
-{
-	struct clk_init_data init;
-	struct cl_clk *clc;
-	int ret;
-
-	clc = kzalloc(sizeof(*clc), GFP_KERNEL);
-	if (!clc)
-		return ERR_PTR(-ENOMEM);
-
-	clc->hw.init = &init;
-	clc->cluster = topology_physical_package_id(cpu_dev->id);
-
-	init.name = dev_name(cpu_dev);
-	init.ops = &clk_clc_ops;
-	init.flags = CLK_GET_RATE_NOCACHE;
-	init.num_parents = 0;
-
-	ret = devm_clk_hw_register(cpu_dev, &clc->hw);
-	if (ret)
-		return ERR_PTR(ret);
-	return &clc->hw;
-}
-
-static int mb86s7x_clclk_of_init(void)
-{
-	int cpu, ret = -ENODEV;
-	struct device_node *np;
-	struct clk_hw *hw;
-
-	np = of_find_compatible_node(NULL, NULL, "fujitsu,mb86s70-scb-1.0");
-	if (!np || !of_device_is_available(np))
-		goto exit;
-
-	for_each_possible_cpu(cpu) {
-		struct device *cpu_dev = get_cpu_device(cpu);
-
-		if (!cpu_dev) {
-			pr_err("failed to get cpu%d device\n", cpu);
-			continue;
-		}
-
-		hw = mb86s7x_clclk_register(cpu_dev);
-		if (IS_ERR(hw)) {
-			pr_err("failed to register cpu%d clock\n", cpu);
-			continue;
-		}
-		if (clk_hw_register_clkdev(hw, NULL, dev_name(cpu_dev))) {
-			pr_err("failed to register cpu%d clock lookup\n", cpu);
-			continue;
-		}
-		pr_debug("registered clk for %s\n", dev_name(cpu_dev));
-	}
-	ret = 0;
-
-	platform_device_register_simple("arm-bL-cpufreq-dt", -1, NULL, 0);
-exit:
-	of_node_put(np);
-	return ret;
-}
-module_init(mb86s7x_clclk_of_init);
diff --git a/drivers/clk/clk-moxart.c b/drivers/clk/clk-moxart.c
index b86dac851116..58428d0043fd 100644
--- a/drivers/clk/clk-moxart.c
+++ b/drivers/clk/clk-moxart.c
@@ -18,7 +18,7 @@
 
 static void __init moxart_of_pll_clk_init(struct device_node *node)
 {
-	static void __iomem *base;
+	void __iomem *base;
 	struct clk_hw *hw;
 	struct clk *ref_clk;
 	unsigned int mul;
@@ -30,7 +30,7 @@ static void __init moxart_of_pll_clk_init(struct device_node *node)
 
 	base = of_iomap(node, 0);
 	if (!base) {
-		pr_err("%s: of_iomap failed\n", node->full_name);
+		pr_err("%pOF: of_iomap failed\n", node);
 		return;
 	}
 
@@ -39,13 +39,13 @@ static void __init moxart_of_pll_clk_init(struct device_node *node)
 
 	ref_clk = of_clk_get(node, 0);
 	if (IS_ERR(ref_clk)) {
-		pr_err("%s: of_clk_get failed\n", node->full_name);
+		pr_err("%pOF: of_clk_get failed\n", node);
 		return;
 	}
 
 	hw = clk_hw_register_fixed_factor(NULL, name, parent_name, 0, mul, 1);
 	if (IS_ERR(hw)) {
-		pr_err("%s: failed to register clock\n", node->full_name);
+		pr_err("%pOF: failed to register clock\n", node);
 		return;
 	}
 
@@ -57,7 +57,7 @@ CLK_OF_DECLARE(moxart_pll_clock, "moxa,moxart-pll-clock",
 
 static void __init moxart_of_apb_clk_init(struct device_node *node)
 {
-	static void __iomem *base;
+	void __iomem *base;
 	struct clk_hw *hw;
 	struct clk *pll_clk;
 	unsigned int div, val;
@@ -70,7 +70,7 @@ static void __init moxart_of_apb_clk_init(struct device_node *node)
 
 	base = of_iomap(node, 0);
 	if (!base) {
-		pr_err("%s: of_iomap failed\n", node->full_name);
+		pr_err("%pOF: of_iomap failed\n", node);
 		return;
 	}
 
@@ -83,13 +83,13 @@ static void __init moxart_of_apb_clk_init(struct device_node *node)
 
 	pll_clk = of_clk_get(node, 0);
 	if (IS_ERR(pll_clk)) {
-		pr_err("%s: of_clk_get failed\n", node->full_name);
+		pr_err("%pOF: of_clk_get failed\n", node);
 		return;
 	}
 
 	hw = clk_hw_register_fixed_factor(NULL, name, parent_name, 0, 1, div);
 	if (IS_ERR(hw)) {
-		pr_err("%s: failed to register clock\n", node->full_name);
+		pr_err("%pOF: failed to register clock\n", node);
 		return;
 	}
 
diff --git a/drivers/clk/clk-qoriq.c b/drivers/clk/clk-qoriq.c
index f3931e38fac0..b0ea753b8709 100644
--- a/drivers/clk/clk-qoriq.c
+++ b/drivers/clk/clk-qoriq.c
@@ -12,6 +12,7 @@
 
 #include <linux/clk.h>
 #include <linux/clk-provider.h>
+#include <linux/clkdev.h>
 #include <linux/fsl/guts.h>
 #include <linux/io.h>
 #include <linux/kernel.h>
@@ -537,6 +538,17 @@ static const struct clockgen_chipinfo chipinfo[] = {
 		.flags = CG_PLL_8BIT,
 	},
 	{
+		.compat = "fsl,ls1088a-clockgen",
+		.cmux_groups = {
+			&clockgen2_cmux_cga12
+		},
+		.cmux_to_group = {
+			0, 0, -1
+		},
+		.pll_mask = 0x07,
+		.flags = CG_VER3 | CG_LITTLE_ENDIAN,
+	},
+	{
 		.compat = "fsl,ls1012a-clockgen",
 		.cmux_groups = {
 			&ls1012a_cmux
@@ -1113,6 +1125,7 @@ static void __init create_one_pll(struct clockgen *cg, int idx)
 
 	for (i = 0; i < ARRAY_SIZE(pll->div); i++) {
 		struct clk *clk;
+		int ret;
 
 		snprintf(pll->div[i].name, sizeof(pll->div[i].name),
 			 "cg-pll%d-div%d", idx, i + 1);
@@ -1126,6 +1139,11 @@ static void __init create_one_pll(struct clockgen *cg, int idx)
 		}
 
 		pll->div[i].clk = clk;
+		ret = clk_register_clkdev(clk, pll->div[i].name, NULL);
+		if (ret != 0)
+			pr_err("%s: %s: register to lookup table failed %ld\n",
+			       __func__, pll->div[i].name, PTR_ERR(clk));
+
 	}
 }
 
@@ -1348,8 +1366,7 @@ static void __init clockgen_init(struct device_node *np)
 	}
 
 	if (i == ARRAY_SIZE(chipinfo)) {
-		pr_err("%s: unknown clockgen node %s\n", __func__,
-		       np->full_name);
+		pr_err("%s: unknown clockgen node %pOF\n", __func__, np);
 		goto err;
 	}
 	clockgen.info = chipinfo[i];
@@ -1362,8 +1379,8 @@ static void __init clockgen_init(struct device_node *np)
 		if (guts) {
 			clockgen.guts = of_iomap(guts, 0);
 			if (!clockgen.guts) {
-				pr_err("%s: Couldn't map %s regs\n", __func__,
-				       guts->full_name);
+				pr_err("%s: Couldn't map %pOF regs\n", __func__,
+				       guts);
 			}
 		}
 
@@ -1398,6 +1415,7 @@ CLK_OF_DECLARE(qoriq_clockgen_ls1012a, "fsl,ls1012a-clockgen", clockgen_init);
 CLK_OF_DECLARE(qoriq_clockgen_ls1021a, "fsl,ls1021a-clockgen", clockgen_init);
 CLK_OF_DECLARE(qoriq_clockgen_ls1043a, "fsl,ls1043a-clockgen", clockgen_init);
 CLK_OF_DECLARE(qoriq_clockgen_ls1046a, "fsl,ls1046a-clockgen", clockgen_init);
+CLK_OF_DECLARE(qoriq_clockgen_ls1088a, "fsl,ls1088a-clockgen", clockgen_init);
 CLK_OF_DECLARE(qoriq_clockgen_ls2080a, "fsl,ls2080a-clockgen", clockgen_init);
 
 /* Legacy nodes */
diff --git a/drivers/clk/clk-si5351.c b/drivers/clk/clk-si5351.c
index 2492442eea77..20d90769cced 100644
--- a/drivers/clk/clk-si5351.c
+++ b/drivers/clk/clk-si5351.c
@@ -519,6 +519,11 @@ static int si5351_pll_set_rate(struct clk_hw *hw, unsigned long rate,
 		SI5351_CLK_INTEGER_MODE,
 		(hwdata->params.p2 == 0) ? SI5351_CLK_INTEGER_MODE : 0);
 
+	/* Do a pll soft reset on the affected pll */
+	si5351_reg_write(hwdata->drvdata, SI5351_PLL_RESET,
+			 hwdata->num == 0 ? SI5351_PLL_RESET_A :
+					    SI5351_PLL_RESET_B);
+
 	dev_dbg(&hwdata->drvdata->client->dev,
 		"%s - %s: p1 = %lu, p2 = %lu, p3 = %lu, parent_rate = %lu, rate = %lu\n",
 		__func__, clk_hw_get_name(hw),
@@ -1091,13 +1096,6 @@ static int si5351_clkout_set_rate(struct clk_hw *hw, unsigned long rate,
 	si5351_set_bits(hwdata->drvdata, SI5351_CLK0_CTRL + hwdata->num,
 			SI5351_CLK_POWERDOWN, 0);
 
-	/*
-	 * Do a pll soft reset on both plls, needed in some cases to get
-	 * all outputs running.
-	 */
-	si5351_reg_write(hwdata->drvdata, SI5351_PLL_RESET,
-			 SI5351_PLL_RESET_A | SI5351_PLL_RESET_B);
-
 	dev_dbg(&hwdata->drvdata->client->dev,
 		"%s - %s: rdiv = %u, parent_rate = %lu, rate = %lu\n",
 		__func__, clk_hw_get_name(hw), (1 << rdiv),
diff --git a/drivers/clk/clk-stm32f4.c b/drivers/clk/clk-stm32f4.c
index 68e2a4e499f1..96c6b6bc8f0e 100644
--- a/drivers/clk/clk-stm32f4.c
+++ b/drivers/clk/clk-stm32f4.c
@@ -1541,8 +1541,8 @@ static void __init stm32f4_rcc_init(struct device_node *np)
 		    base + gd->offset, gd->bit_idx, 0, &stm32f4_clk_lock);
 
 		if (IS_ERR(clks[idx])) {
-			pr_err("%s: Unable to register leaf clock %s\n",
-			       np->full_name, gd->name);
+			pr_err("%pOF: Unable to register leaf clock %s\n",
+			       np, gd->name);
 			goto fail;
 		}
 	}
diff --git a/drivers/clk/clk-stm32h7.c b/drivers/clk/clk-stm32h7.c
new file mode 100644
index 000000000000..a94c3f56c590
--- /dev/null
+++ b/drivers/clk/clk-stm32h7.c
@@ -0,0 +1,1410 @@
+/*
+ * Copyright (C) Gabriel Fernandez 2017
+ * Author: Gabriel Fernandez <gabriel.fernandez@st.com>
+ *
+ * License terms: GPL V2.0.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/mfd/syscon.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/stm32h7-clks.h>
+
+/* Reset Clock Control Registers */
+#define RCC_CR		0x00
+#define RCC_CFGR	0x10
+#define RCC_D1CFGR	0x18
+#define RCC_D2CFGR	0x1C
+#define RCC_D3CFGR	0x20
+#define RCC_PLLCKSELR	0x28
+#define RCC_PLLCFGR	0x2C
+#define RCC_PLL1DIVR	0x30
+#define RCC_PLL1FRACR	0x34
+#define RCC_PLL2DIVR	0x38
+#define RCC_PLL2FRACR	0x3C
+#define RCC_PLL3DIVR	0x40
+#define RCC_PLL3FRACR	0x44
+#define RCC_D1CCIPR	0x4C
+#define RCC_D2CCIP1R	0x50
+#define RCC_D2CCIP2R	0x54
+#define RCC_D3CCIPR	0x58
+#define RCC_BDCR	0x70
+#define RCC_CSR		0x74
+#define RCC_AHB3ENR	0xD4
+#define RCC_AHB1ENR	0xD8
+#define RCC_AHB2ENR	0xDC
+#define RCC_AHB4ENR	0xE0
+#define RCC_APB3ENR	0xE4
+#define RCC_APB1LENR	0xE8
+#define RCC_APB1HENR	0xEC
+#define RCC_APB2ENR	0xF0
+#define RCC_APB4ENR	0xF4
+
+static DEFINE_SPINLOCK(stm32rcc_lock);
+
+static void __iomem *base;
+static struct clk_hw **hws;
+
+/* System clock parent */
+static const char * const sys_src[] = {
+	"hsi_ck", "csi_ck", "hse_ck", "pll1_p" };
+
+static const char * const tracein_src[] = {
+	"hsi_ck", "csi_ck", "hse_ck", "pll1_r" };
+
+static const char * const per_src[] = {
+	"hsi_ker", "csi_ker", "hse_ck", "disabled" };
+
+static const char * const pll_src[] = {
+	"hsi_ck", "csi_ck", "hse_ck", "no clock" };
+
+static const char * const sdmmc_src[] = { "pll1_q", "pll2_r" };
+
+static const char * const dsi_src[] = { "ck_dsi_phy", "pll2_q" };
+
+static const char * const qspi_src[] = {
+	"hclk", "pll1_q", "pll2_r", "per_ck" };
+
+static const char * const fmc_src[] = {
+	"hclk", "pll1_q", "pll2_r", "per_ck" };
+
+/* Kernel clock parent */
+static const char * const swp_src[] = {	"pclk1", "hsi_ker" };
+
+static const char * const fdcan_src[] = { "hse_ck", "pll1_q", "pll2_q" };
+
+static const char * const dfsdm1_src[] = { "pclk2", "sys_ck" };
+
+static const char * const spdifrx_src[] = {
+	"pll1_q", "pll2_r", "pll3_r", "hsi_ker" };
+
+static const char *spi_src1[5] = {
+	"pll1_q", "pll2_p", "pll3_p", NULL, "per_ck" };
+
+static const char * const spi_src2[] = {
+	"pclk2", "pll2_q", "pll3_q", "hsi_ker", "csi_ker", "hse_ck" };
+
+static const char * const spi_src3[] = {
+	"pclk4", "pll2_q", "pll3_q", "hsi_ker", "csi_ker", "hse_ck" };
+
+static const char * const lptim_src1[] = {
+	"pclk1", "pll2_p", "pll3_r", "lse_ck", "lsi_ck", "per_ck" };
+
+static const char * const lptim_src2[] = {
+	"pclk4", "pll2_p", "pll3_r", "lse_ck", "lsi_ck", "per_ck" };
+
+static const char * const cec_src[] = {"lse_ck", "lsi_ck", "csi_ker_div122" };
+
+static const char * const usbotg_src[] = {"pll1_q", "pll3_q", "rc48_ck" };
+
+/* i2c 1,2,3 src */
+static const char * const i2c_src1[] = {
+	"pclk1", "pll3_r", "hsi_ker", "csi_ker" };
+
+static const char * const i2c_src2[] = {
+	"pclk4", "pll3_r", "hsi_ker", "csi_ker" };
+
+static const char * const rng_src[] = {
+	"rc48_ck", "pll1_q", "lse_ck", "lsi_ck" };
+
+/* usart 1,6 src */
+static const char * const usart_src1[] = {
+	"pclk2", "pll2_q", "pll3_q", "hsi_ker", "csi_ker", "lse_ck" };
+
+/* usart 2,3,4,5,7,8 src */
+static const char * const usart_src2[] = {
+	"pclk1", "pll2_q", "pll3_q", "hsi_ker", "csi_ker", "lse_ck" };
+
+static const char *sai_src[5] = {
+	"pll1_q", "pll2_p", "pll3_p", NULL, "per_ck" };
+
+static const char * const adc_src[] = { "pll2_p", "pll3_r", "per_ck" };
+
+/* lptim 2,3,4,5 src */
+static const char * const lpuart1_src[] = {
+	"pclk3", "pll2_q", "pll3_q", "csi_ker", "lse_ck" };
+
+static const char * const hrtim_src[] = { "tim2_ker", "d1cpre" };
+
+/* RTC clock parent */
+static const char * const rtc_src[] = { "off", "lse_ck", "lsi_ck", "hse_1M" };
+
+/* Micro-controller output clock parent */
+static const char * const mco_src1[] = {
+	"hsi_ck", "lse_ck", "hse_ck", "pll1_q",	"rc48_ck" };
+
+static const char * const mco_src2[] = {
+	"sys_ck", "pll2_p", "hse_ck", "pll1_p", "csi_ck", "lsi_ck" };
+
+/* LCD clock */
+static const char * const ltdc_src[] = {"pll3_r"};
+
+/* Gate clock with ready bit and backup domain management */
+struct stm32_ready_gate {
+	struct	clk_gate gate;
+	u8	bit_rdy;
+};
+
+#define to_ready_gate_clk(_rgate) container_of(_rgate, struct stm32_ready_gate,\
+		gate)
+
+#define RGATE_TIMEOUT 10000
+
+static int ready_gate_clk_enable(struct clk_hw *hw)
+{
+	struct clk_gate *gate = to_clk_gate(hw);
+	struct stm32_ready_gate *rgate = to_ready_gate_clk(gate);
+	int bit_status;
+	unsigned int timeout = RGATE_TIMEOUT;
+
+	if (clk_gate_ops.is_enabled(hw))
+		return 0;
+
+	clk_gate_ops.enable(hw);
+
+	/* We can't use readl_poll_timeout() because we can blocked if
+	 * someone enables this clock before clocksource changes.
+	 * Only jiffies counter is available. Jiffies are incremented by
+	 * interruptions and enable op does not allow to be interrupted.
+	 */
+	do {
+		bit_status = !(readl(gate->reg) & BIT(rgate->bit_rdy));
+
+		if (bit_status)
+			udelay(100);
+
+	} while (bit_status && --timeout);
+
+	return bit_status;
+}
+
+static void ready_gate_clk_disable(struct clk_hw *hw)
+{
+	struct clk_gate *gate = to_clk_gate(hw);
+	struct stm32_ready_gate *rgate = to_ready_gate_clk(gate);
+	int bit_status;
+	unsigned int timeout = RGATE_TIMEOUT;
+
+	if (!clk_gate_ops.is_enabled(hw))
+		return;
+
+	clk_gate_ops.disable(hw);
+
+	do {
+		bit_status = !!(readl(gate->reg) & BIT(rgate->bit_rdy));
+
+		if (bit_status)
+			udelay(100);
+
+	} while (bit_status && --timeout);
+}
+
+static const struct clk_ops ready_gate_clk_ops = {
+	.enable		= ready_gate_clk_enable,
+	.disable	= ready_gate_clk_disable,
+	.is_enabled	= clk_gate_is_enabled,
+};
+
+static struct clk_hw *clk_register_ready_gate(struct device *dev,
+		const char *name, const char *parent_name,
+		void __iomem *reg, u8 bit_idx, u8 bit_rdy,
+		unsigned long flags, spinlock_t *lock)
+{
+	struct stm32_ready_gate *rgate;
+	struct clk_init_data init = { NULL };
+	struct clk_hw *hw;
+	int ret;
+
+	rgate = kzalloc(sizeof(*rgate), GFP_KERNEL);
+	if (!rgate)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = name;
+	init.ops = &ready_gate_clk_ops;
+	init.flags = flags;
+	init.parent_names = &parent_name;
+	init.num_parents = 1;
+
+	rgate->bit_rdy = bit_rdy;
+	rgate->gate.lock = lock;
+	rgate->gate.reg = reg;
+	rgate->gate.bit_idx = bit_idx;
+	rgate->gate.hw.init = &init;
+
+	hw = &rgate->gate.hw;
+	ret = clk_hw_register(dev, hw);
+	if (ret) {
+		kfree(rgate);
+		hw = ERR_PTR(ret);
+	}
+
+	return hw;
+}
+
+struct gate_cfg {
+	u32 offset;
+	u8  bit_idx;
+};
+
+struct muxdiv_cfg {
+	u32 offset;
+	u8 shift;
+	u8 width;
+};
+
+struct composite_clk_cfg {
+	struct gate_cfg *gate;
+	struct muxdiv_cfg *mux;
+	struct muxdiv_cfg *div;
+	const char *name;
+	const char * const *parent_name;
+	int num_parents;
+	u32 flags;
+};
+
+struct composite_clk_gcfg_t {
+	u8 flags;
+	const struct clk_ops *ops;
+};
+
+/*
+ * General config definition of a composite clock (only clock diviser for rate)
+ */
+struct composite_clk_gcfg {
+	struct composite_clk_gcfg_t *mux;
+	struct composite_clk_gcfg_t *div;
+	struct composite_clk_gcfg_t *gate;
+};
+
+#define M_CFG_MUX(_mux_ops, _mux_flags)\
+	.mux = &(struct composite_clk_gcfg_t) { _mux_flags, _mux_ops}
+
+#define M_CFG_DIV(_rate_ops, _rate_flags)\
+	.div = &(struct composite_clk_gcfg_t) {_rate_flags, _rate_ops}
+
+#define M_CFG_GATE(_gate_ops, _gate_flags)\
+	.gate = &(struct composite_clk_gcfg_t) { _gate_flags, _gate_ops}
+
+static struct clk_mux *_get_cmux(void __iomem *reg, u8 shift, u8 width,
+		u32 flags, spinlock_t *lock)
+{
+	struct clk_mux *mux;
+
+	mux = kzalloc(sizeof(*mux), GFP_KERNEL);
+	if (!mux)
+		return ERR_PTR(-ENOMEM);
+
+	mux->reg	= reg;
+	mux->shift	= shift;
+	mux->mask	= (1 << width) - 1;
+	mux->flags	= flags;
+	mux->lock	= lock;
+
+	return mux;
+}
+
+static struct clk_divider *_get_cdiv(void __iomem *reg, u8 shift, u8 width,
+		u32 flags, spinlock_t *lock)
+{
+	struct clk_divider *div;
+
+	div = kzalloc(sizeof(*div), GFP_KERNEL);
+
+	if (!div)
+		return ERR_PTR(-ENOMEM);
+
+	div->reg   = reg;
+	div->shift = shift;
+	div->width = width;
+	div->flags = flags;
+	div->lock  = lock;
+
+	return div;
+}
+
+static struct clk_gate *_get_cgate(void __iomem *reg, u8 bit_idx, u32 flags,
+		spinlock_t *lock)
+{
+	struct clk_gate *gate;
+
+	gate = kzalloc(sizeof(*gate), GFP_KERNEL);
+	if (!gate)
+		return ERR_PTR(-ENOMEM);
+
+	gate->reg	= reg;
+	gate->bit_idx	= bit_idx;
+	gate->flags	= flags;
+	gate->lock	= lock;
+
+	return gate;
+}
+
+struct composite_cfg {
+	struct clk_hw *mux_hw;
+	struct clk_hw *div_hw;
+	struct clk_hw *gate_hw;
+
+	const struct clk_ops *mux_ops;
+	const struct clk_ops *div_ops;
+	const struct clk_ops *gate_ops;
+};
+
+static void get_cfg_composite_div(const struct composite_clk_gcfg *gcfg,
+		const struct composite_clk_cfg *cfg,
+		struct composite_cfg *composite, spinlock_t *lock)
+{
+	struct clk_mux     *mux = NULL;
+	struct clk_divider *div = NULL;
+	struct clk_gate    *gate = NULL;
+	const struct clk_ops *mux_ops, *div_ops, *gate_ops;
+	struct clk_hw *mux_hw;
+	struct clk_hw *div_hw;
+	struct clk_hw *gate_hw;
+
+	mux_ops = div_ops = gate_ops = NULL;
+	mux_hw = div_hw = gate_hw = NULL;
+
+	if (gcfg->mux && gcfg->mux) {
+		mux = _get_cmux(base + cfg->mux->offset,
+				cfg->mux->shift,
+				cfg->mux->width,
+				gcfg->mux->flags, lock);
+
+		if (!IS_ERR(mux)) {
+			mux_hw = &mux->hw;
+			mux_ops = gcfg->mux->ops ?
+				  gcfg->mux->ops : &clk_mux_ops;
+		}
+	}
+
+	if (gcfg->div && cfg->div) {
+		div = _get_cdiv(base + cfg->div->offset,
+				cfg->div->shift,
+				cfg->div->width,
+				gcfg->div->flags, lock);
+
+		if (!IS_ERR(div)) {
+			div_hw = &div->hw;
+			div_ops = gcfg->div->ops ?
+				  gcfg->div->ops : &clk_divider_ops;
+		}
+	}
+
+	if (gcfg->gate && gcfg->gate) {
+		gate = _get_cgate(base + cfg->gate->offset,
+				cfg->gate->bit_idx,
+				gcfg->gate->flags, lock);
+
+		if (!IS_ERR(gate)) {
+			gate_hw = &gate->hw;
+			gate_ops = gcfg->gate->ops ?
+				   gcfg->gate->ops : &clk_gate_ops;
+		}
+	}
+
+	composite->mux_hw = mux_hw;
+	composite->mux_ops = mux_ops;
+
+	composite->div_hw = div_hw;
+	composite->div_ops = div_ops;
+
+	composite->gate_hw = gate_hw;
+	composite->gate_ops = gate_ops;
+}
+
+/* Kernel Timer */
+struct timer_ker {
+	u8 dppre_shift;
+	struct clk_hw hw;
+	spinlock_t *lock;
+};
+
+#define to_timer_ker(_hw) container_of(_hw, struct timer_ker, hw)
+
+static unsigned long timer_ker_recalc_rate(struct clk_hw *hw,
+		unsigned long parent_rate)
+{
+	struct timer_ker *clk_elem = to_timer_ker(hw);
+	u32 timpre;
+	u32 dppre_shift = clk_elem->dppre_shift;
+	u32 prescaler;
+	u32 mul;
+
+	timpre = (readl(base + RCC_CFGR) >> 15) & 0x01;
+
+	prescaler = (readl(base + RCC_D2CFGR) >> dppre_shift) & 0x03;
+
+	mul = 2;
+
+	if (prescaler < 4)
+		mul = 1;
+
+	else if (timpre && prescaler > 4)
+		mul = 4;
+
+	return parent_rate * mul;
+}
+
+static const struct clk_ops timer_ker_ops = {
+	.recalc_rate = timer_ker_recalc_rate,
+};
+
+static struct clk_hw *clk_register_stm32_timer_ker(struct device *dev,
+		const char *name, const char *parent_name,
+		unsigned long flags,
+		u8 dppre_shift,
+		spinlock_t *lock)
+{
+	struct timer_ker *element;
+	struct clk_init_data init;
+	struct clk_hw *hw;
+	int err;
+
+	element = kzalloc(sizeof(*element), GFP_KERNEL);
+	if (!element)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = name;
+	init.ops = &timer_ker_ops;
+	init.flags = flags;
+	init.parent_names = &parent_name;
+	init.num_parents = 1;
+
+	element->hw.init = &init;
+	element->lock = lock;
+	element->dppre_shift = dppre_shift;
+
+	hw = &element->hw;
+	err = clk_hw_register(dev, hw);
+
+	if (err) {
+		kfree(element);
+		return ERR_PTR(err);
+	}
+
+	return hw;
+}
+
+static const struct clk_div_table d1cpre_div_table[] = {
+	{ 0, 1 }, { 1, 1 }, { 2, 1 }, { 3, 1},
+	{ 4, 1 }, { 5, 1 }, { 6, 1 }, { 7, 1},
+	{ 8, 2 }, { 9, 4 }, { 10, 8 }, { 11, 16 },
+	{ 12, 64 }, { 13, 128 }, { 14, 256 },
+	{ 15, 512 },
+	{ 0 },
+};
+
+static const struct clk_div_table ppre_div_table[] = {
+	{ 0, 1 }, { 1, 1 }, { 2, 1 }, { 3, 1},
+	{ 4, 2 }, { 5, 4 }, { 6, 8 }, { 7, 16 },
+	{ 0 },
+};
+
+static void register_core_and_bus_clocks(void)
+{
+	/* CORE AND BUS */
+	hws[SYS_D1CPRE] = clk_hw_register_divider_table(NULL, "d1cpre",
+			"sys_ck", CLK_IGNORE_UNUSED, base + RCC_D1CFGR, 8, 4, 0,
+			d1cpre_div_table, &stm32rcc_lock);
+
+	hws[HCLK] = clk_hw_register_divider_table(NULL, "hclk", "d1cpre",
+			CLK_IGNORE_UNUSED, base + RCC_D1CFGR, 0, 4, 0,
+			d1cpre_div_table, &stm32rcc_lock);
+
+	/* D1 DOMAIN */
+	/* * CPU Systick */
+	hws[CPU_SYSTICK] = clk_hw_register_fixed_factor(NULL, "systick",
+			"d1cpre", 0, 1, 8);
+
+	/* * APB3 peripheral */
+	hws[PCLK3] = clk_hw_register_divider_table(NULL, "pclk3", "hclk", 0,
+			base + RCC_D1CFGR, 4, 3, 0,
+			ppre_div_table, &stm32rcc_lock);
+
+	/* D2 DOMAIN */
+	/* * APB1 peripheral */
+	hws[PCLK1] = clk_hw_register_divider_table(NULL, "pclk1", "hclk", 0,
+			base + RCC_D2CFGR, 4, 3, 0,
+			ppre_div_table, &stm32rcc_lock);
+
+	/* Timers prescaler clocks */
+	clk_register_stm32_timer_ker(NULL, "tim1_ker", "pclk1", 0,
+			4, &stm32rcc_lock);
+
+	/* * APB2 peripheral */
+	hws[PCLK2] = clk_hw_register_divider_table(NULL, "pclk2", "hclk", 0,
+			base + RCC_D2CFGR, 8, 3, 0, ppre_div_table,
+			&stm32rcc_lock);
+
+	clk_register_stm32_timer_ker(NULL, "tim2_ker", "pclk2", 0, 8,
+			&stm32rcc_lock);
+
+	/* D3 DOMAIN */
+	/* * APB4 peripheral */
+	hws[PCLK4] = clk_hw_register_divider_table(NULL, "pclk4", "hclk", 0,
+			base + RCC_D3CFGR, 4, 3, 0,
+			ppre_div_table, &stm32rcc_lock);
+}
+
+/* MUX clock configuration */
+struct stm32_mux_clk {
+	const char *name;
+	const char * const *parents;
+	u8 num_parents;
+	u32 offset;
+	u8 shift;
+	u8 width;
+	u32 flags;
+};
+
+#define M_MCLOCF(_name, _parents, _mux_offset, _mux_shift, _mux_width, _flags)\
+{\
+	.name		= _name,\
+	.parents	= _parents,\
+	.num_parents	= ARRAY_SIZE(_parents),\
+	.offset		= _mux_offset,\
+	.shift		= _mux_shift,\
+	.width		= _mux_width,\
+	.flags		= _flags,\
+}
+
+#define M_MCLOC(_name, _parents, _mux_offset, _mux_shift, _mux_width)\
+	M_MCLOCF(_name, _parents, _mux_offset, _mux_shift, _mux_width, 0)\
+
+static const struct stm32_mux_clk stm32_mclk[] __initconst = {
+	M_MCLOC("per_ck",	per_src,	RCC_D1CCIPR,	28, 3),
+	M_MCLOC("pllsrc",	pll_src,	RCC_PLLCKSELR,	 0, 3),
+	M_MCLOC("sys_ck",	sys_src,	RCC_CFGR,	 0, 3),
+	M_MCLOC("tracein_ck",	tracein_src,	RCC_CFGR,	 0, 3),
+};
+
+/* Oscillary clock configuration */
+struct stm32_osc_clk {
+	const char *name;
+	const char *parent;
+	u32 gate_offset;
+	u8 bit_idx;
+	u8 bit_rdy;
+	u32 flags;
+};
+
+#define OSC_CLKF(_name, _parent, _gate_offset, _bit_idx, _bit_rdy, _flags)\
+{\
+	.name		= _name,\
+	.parent		= _parent,\
+	.gate_offset	= _gate_offset,\
+	.bit_idx	= _bit_idx,\
+	.bit_rdy	= _bit_rdy,\
+	.flags		= _flags,\
+}
+
+#define OSC_CLK(_name, _parent, _gate_offset, _bit_idx, _bit_rdy)\
+	OSC_CLKF(_name, _parent, _gate_offset, _bit_idx, _bit_rdy, 0)
+
+static const struct stm32_osc_clk stm32_oclk[] __initconst = {
+	OSC_CLKF("hsi_ck",  "hsidiv",   RCC_CR,   0,  2, CLK_IGNORE_UNUSED),
+	OSC_CLKF("hsi_ker", "hsidiv",   RCC_CR,   1,  2, CLK_IGNORE_UNUSED),
+	OSC_CLKF("csi_ck",  "clk-csi",  RCC_CR,   7,  8, CLK_IGNORE_UNUSED),
+	OSC_CLKF("csi_ker", "clk-csi",  RCC_CR,   9,  8, CLK_IGNORE_UNUSED),
+	OSC_CLKF("rc48_ck", "clk-rc48", RCC_CR,  12, 13, CLK_IGNORE_UNUSED),
+	OSC_CLKF("lsi_ck",  "clk-lsi",  RCC_CSR,  0,  1, CLK_IGNORE_UNUSED),
+};
+
+/* PLL configuration */
+struct st32h7_pll_cfg {
+	u8 bit_idx;
+	u32 offset_divr;
+	u8 bit_frac_en;
+	u32 offset_frac;
+	u8 divm;
+};
+
+struct stm32_pll_data {
+	const char *name;
+	const char *parent_name;
+	unsigned long flags;
+	const struct st32h7_pll_cfg *cfg;
+};
+
+static const struct st32h7_pll_cfg stm32h7_pll1 = {
+	.bit_idx = 24,
+	.offset_divr = RCC_PLL1DIVR,
+	.bit_frac_en = 0,
+	.offset_frac = RCC_PLL1FRACR,
+	.divm = 4,
+};
+
+static const struct st32h7_pll_cfg stm32h7_pll2 = {
+	.bit_idx = 26,
+	.offset_divr = RCC_PLL2DIVR,
+	.bit_frac_en = 4,
+	.offset_frac = RCC_PLL2FRACR,
+	.divm = 12,
+};
+
+static const struct st32h7_pll_cfg stm32h7_pll3 = {
+	.bit_idx = 28,
+	.offset_divr = RCC_PLL3DIVR,
+	.bit_frac_en = 8,
+	.offset_frac = RCC_PLL3FRACR,
+	.divm = 20,
+};
+
+static const struct stm32_pll_data stm32_pll[] = {
+	{ "vco1", "pllsrc", CLK_IGNORE_UNUSED, &stm32h7_pll1 },
+	{ "vco2", "pllsrc", 0, &stm32h7_pll2 },
+	{ "vco3", "pllsrc", 0, &stm32h7_pll3 },
+};
+
+struct stm32_fractional_divider {
+	void __iomem	*mreg;
+	u8		mshift;
+	u8		mwidth;
+	u32		mmask;
+
+	void __iomem	*nreg;
+	u8		nshift;
+	u8		nwidth;
+
+	void __iomem	*freg_status;
+	u8		freg_bit;
+	void __iomem	*freg_value;
+	u8		fshift;
+	u8		fwidth;
+
+	u8		flags;
+	struct clk_hw	hw;
+	spinlock_t	*lock;
+};
+
+struct stm32_pll_obj {
+	spinlock_t *lock;
+	struct stm32_fractional_divider div;
+	struct stm32_ready_gate rgate;
+	struct clk_hw hw;
+};
+
+#define to_pll(_hw) container_of(_hw, struct stm32_pll_obj, hw)
+
+static int pll_is_enabled(struct clk_hw *hw)
+{
+	struct stm32_pll_obj *clk_elem = to_pll(hw);
+	struct clk_hw *_hw = &clk_elem->rgate.gate.hw;
+
+	__clk_hw_set_clk(_hw, hw);
+
+	return ready_gate_clk_ops.is_enabled(_hw);
+}
+
+static int pll_enable(struct clk_hw *hw)
+{
+	struct stm32_pll_obj *clk_elem = to_pll(hw);
+	struct clk_hw *_hw = &clk_elem->rgate.gate.hw;
+
+	__clk_hw_set_clk(_hw, hw);
+
+	return ready_gate_clk_ops.enable(_hw);
+}
+
+static void pll_disable(struct clk_hw *hw)
+{
+	struct stm32_pll_obj *clk_elem = to_pll(hw);
+	struct clk_hw *_hw = &clk_elem->rgate.gate.hw;
+
+	__clk_hw_set_clk(_hw, hw);
+
+	ready_gate_clk_ops.disable(_hw);
+}
+
+static int pll_frac_is_enabled(struct clk_hw *hw)
+{
+	struct stm32_pll_obj *clk_elem = to_pll(hw);
+	struct stm32_fractional_divider *fd = &clk_elem->div;
+
+	return (readl(fd->freg_status) >> fd->freg_bit) & 0x01;
+}
+
+static unsigned long pll_read_frac(struct clk_hw *hw)
+{
+	struct stm32_pll_obj *clk_elem = to_pll(hw);
+	struct stm32_fractional_divider *fd = &clk_elem->div;
+
+	return (readl(fd->freg_value) >> fd->fshift) &
+		GENMASK(fd->fwidth - 1, 0);
+}
+
+static unsigned long pll_fd_recalc_rate(struct clk_hw *hw,
+		unsigned long parent_rate)
+{
+	struct stm32_pll_obj *clk_elem = to_pll(hw);
+	struct stm32_fractional_divider *fd = &clk_elem->div;
+	unsigned long m, n;
+	u32 val, mask;
+	u64 rate, rate1 = 0;
+
+	val = readl(fd->mreg);
+	mask = GENMASK(fd->mwidth - 1, 0) << fd->mshift;
+	m = (val & mask) >> fd->mshift;
+
+	val = readl(fd->nreg);
+	mask = GENMASK(fd->nwidth - 1, 0) << fd->nshift;
+	n = ((val & mask) >> fd->nshift) + 1;
+
+	if (!n || !m)
+		return parent_rate;
+
+	rate = (u64)parent_rate * n;
+	do_div(rate, m);
+
+	if (pll_frac_is_enabled(hw)) {
+		val = pll_read_frac(hw);
+		rate1 = (u64)parent_rate * (u64)val;
+		do_div(rate1, (m * 8191));
+	}
+
+	return rate + rate1;
+}
+
+static const struct clk_ops pll_ops = {
+	.enable		= pll_enable,
+	.disable	= pll_disable,
+	.is_enabled	= pll_is_enabled,
+	.recalc_rate	= pll_fd_recalc_rate,
+};
+
+static struct clk_hw *clk_register_stm32_pll(struct device *dev,
+		const char *name,
+		const char *parent,
+		unsigned long flags,
+		const struct st32h7_pll_cfg *cfg,
+		spinlock_t *lock)
+{
+	struct stm32_pll_obj *pll;
+	struct clk_init_data init = { NULL };
+	struct clk_hw *hw;
+	int ret;
+	struct stm32_fractional_divider *div = NULL;
+	struct stm32_ready_gate *rgate;
+
+	pll = kzalloc(sizeof(*pll), GFP_KERNEL);
+	if (!pll)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = name;
+	init.ops = &pll_ops;
+	init.flags = flags;
+	init.parent_names = &parent;
+	init.num_parents = 1;
+	pll->hw.init = &init;
+
+	hw = &pll->hw;
+	rgate = &pll->rgate;
+
+	rgate->bit_rdy = cfg->bit_idx + 1;
+	rgate->gate.lock = lock;
+	rgate->gate.reg = base + RCC_CR;
+	rgate->gate.bit_idx = cfg->bit_idx;
+
+	div = &pll->div;
+	div->flags = 0;
+	div->mreg = base + RCC_PLLCKSELR;
+	div->mshift = cfg->divm;
+	div->mwidth = 6;
+	div->nreg = base +  cfg->offset_divr;
+	div->nshift = 0;
+	div->nwidth = 9;
+
+	div->freg_status = base + RCC_PLLCFGR;
+	div->freg_bit = cfg->bit_frac_en;
+	div->freg_value = base +  cfg->offset_frac;
+	div->fshift = 3;
+	div->fwidth = 13;
+
+	div->lock = lock;
+
+	ret = clk_hw_register(dev, hw);
+	if (ret) {
+		kfree(pll);
+		hw = ERR_PTR(ret);
+	}
+
+	return hw;
+}
+
+/* ODF CLOCKS */
+static unsigned long odf_divider_recalc_rate(struct clk_hw *hw,
+		unsigned long parent_rate)
+{
+	return clk_divider_ops.recalc_rate(hw, parent_rate);
+}
+
+static long odf_divider_round_rate(struct clk_hw *hw, unsigned long rate,
+		unsigned long *prate)
+{
+	return clk_divider_ops.round_rate(hw, rate, prate);
+}
+
+static int odf_divider_set_rate(struct clk_hw *hw, unsigned long rate,
+		unsigned long parent_rate)
+{
+	struct clk_hw *hwp;
+	int pll_status;
+	int ret;
+
+	hwp = clk_hw_get_parent(hw);
+
+	pll_status = pll_is_enabled(hwp);
+
+	if (pll_status)
+		pll_disable(hwp);
+
+	ret = clk_divider_ops.set_rate(hw, rate, parent_rate);
+
+	if (pll_status)
+		pll_enable(hwp);
+
+	return ret;
+}
+
+static const struct clk_ops odf_divider_ops = {
+	.recalc_rate	= odf_divider_recalc_rate,
+	.round_rate	= odf_divider_round_rate,
+	.set_rate	= odf_divider_set_rate,
+};
+
+static int odf_gate_enable(struct clk_hw *hw)
+{
+	struct clk_hw *hwp;
+	int pll_status;
+	int ret;
+
+	if (clk_gate_ops.is_enabled(hw))
+		return 0;
+
+	hwp = clk_hw_get_parent(hw);
+
+	pll_status = pll_is_enabled(hwp);
+
+	if (pll_status)
+		pll_disable(hwp);
+
+	ret = clk_gate_ops.enable(hw);
+
+	if (pll_status)
+		pll_enable(hwp);
+
+	return ret;
+}
+
+static void odf_gate_disable(struct clk_hw *hw)
+{
+	struct clk_hw *hwp;
+	int pll_status;
+
+	if (!clk_gate_ops.is_enabled(hw))
+		return;
+
+	hwp = clk_hw_get_parent(hw);
+
+	pll_status = pll_is_enabled(hwp);
+
+	if (pll_status)
+		pll_disable(hwp);
+
+	clk_gate_ops.disable(hw);
+
+	if (pll_status)
+		pll_enable(hwp);
+}
+
+static const struct clk_ops odf_gate_ops = {
+	.enable		= odf_gate_enable,
+	.disable	= odf_gate_disable,
+	.is_enabled	= clk_gate_is_enabled,
+};
+
+static struct composite_clk_gcfg odf_clk_gcfg = {
+	M_CFG_DIV(&odf_divider_ops, 0),
+	M_CFG_GATE(&odf_gate_ops, 0),
+};
+
+#define M_ODF_F(_name, _parent, _gate_offset,  _bit_idx, _rate_offset,\
+		_rate_shift, _rate_width, _flags)\
+{\
+	.mux = NULL,\
+	.div = &(struct muxdiv_cfg) {_rate_offset, _rate_shift, _rate_width},\
+	.gate = &(struct gate_cfg) {_gate_offset, _bit_idx },\
+	.name = _name,\
+	.parent_name = &(const char *) {_parent},\
+	.num_parents = 1,\
+	.flags = _flags,\
+}
+
+#define M_ODF(_name, _parent, _gate_offset,  _bit_idx, _rate_offset,\
+		_rate_shift, _rate_width)\
+M_ODF_F(_name, _parent, _gate_offset,  _bit_idx, _rate_offset,\
+		_rate_shift, _rate_width, 0)\
+
+static const struct composite_clk_cfg stm32_odf[3][3] = {
+	{
+		M_ODF_F("pll1_p", "vco1", RCC_PLLCFGR, 16, RCC_PLL1DIVR,  9, 7,
+				CLK_IGNORE_UNUSED),
+		M_ODF_F("pll1_q", "vco1", RCC_PLLCFGR, 17, RCC_PLL1DIVR, 16, 7,
+				CLK_IGNORE_UNUSED),
+		M_ODF_F("pll1_r", "vco1", RCC_PLLCFGR, 18, RCC_PLL1DIVR, 24, 7,
+				CLK_IGNORE_UNUSED),
+	},
+
+	{
+		M_ODF("pll2_p", "vco2", RCC_PLLCFGR, 19, RCC_PLL2DIVR,  9, 7),
+		M_ODF("pll2_q", "vco2", RCC_PLLCFGR, 20, RCC_PLL2DIVR, 16, 7),
+		M_ODF("pll2_r", "vco2", RCC_PLLCFGR, 21, RCC_PLL2DIVR, 24, 7),
+	},
+	{
+		M_ODF("pll3_p", "vco3", RCC_PLLCFGR, 22, RCC_PLL3DIVR,  9, 7),
+		M_ODF("pll3_q", "vco3", RCC_PLLCFGR, 23, RCC_PLL3DIVR, 16, 7),
+		M_ODF("pll3_r", "vco3", RCC_PLLCFGR, 24, RCC_PLL3DIVR, 24, 7),
+	}
+};
+
+/* PERIF CLOCKS */
+struct pclk_t {
+	u32 gate_offset;
+	u8 bit_idx;
+	const char *name;
+	const char *parent;
+	u32 flags;
+};
+
+#define PER_CLKF(_gate_offset, _bit_idx, _name, _parent, _flags)\
+{\
+	.gate_offset	= _gate_offset,\
+	.bit_idx	= _bit_idx,\
+	.name		= _name,\
+	.parent		= _parent,\
+	.flags		= _flags,\
+}
+
+#define PER_CLK(_gate_offset, _bit_idx, _name, _parent)\
+	PER_CLKF(_gate_offset, _bit_idx, _name, _parent, 0)
+
+static const struct pclk_t pclk[] = {
+	PER_CLK(RCC_AHB3ENR, 31, "d1sram1", "hclk"),
+	PER_CLK(RCC_AHB3ENR, 30, "itcm", "hclk"),
+	PER_CLK(RCC_AHB3ENR, 29, "dtcm2", "hclk"),
+	PER_CLK(RCC_AHB3ENR, 28, "dtcm1", "hclk"),
+	PER_CLK(RCC_AHB3ENR, 8, "flitf", "hclk"),
+	PER_CLK(RCC_AHB3ENR, 5, "jpgdec", "hclk"),
+	PER_CLK(RCC_AHB3ENR, 4, "dma2d", "hclk"),
+	PER_CLK(RCC_AHB3ENR, 0, "mdma", "hclk"),
+	PER_CLK(RCC_AHB1ENR, 28, "usb2ulpi", "hclk"),
+	PER_CLK(RCC_AHB1ENR, 26, "usb1ulpi", "hclk"),
+	PER_CLK(RCC_AHB1ENR, 17, "eth1rx", "hclk"),
+	PER_CLK(RCC_AHB1ENR, 16, "eth1tx", "hclk"),
+	PER_CLK(RCC_AHB1ENR, 15, "eth1mac", "hclk"),
+	PER_CLK(RCC_AHB1ENR, 14, "art", "hclk"),
+	PER_CLK(RCC_AHB1ENR, 1, "dma2", "hclk"),
+	PER_CLK(RCC_AHB1ENR, 0, "dma1", "hclk"),
+	PER_CLK(RCC_AHB2ENR, 31, "d2sram3", "hclk"),
+	PER_CLK(RCC_AHB2ENR, 30, "d2sram2", "hclk"),
+	PER_CLK(RCC_AHB2ENR, 29, "d2sram1", "hclk"),
+	PER_CLK(RCC_AHB2ENR, 5, "hash", "hclk"),
+	PER_CLK(RCC_AHB2ENR, 4, "crypt", "hclk"),
+	PER_CLK(RCC_AHB2ENR, 0, "camitf", "hclk"),
+	PER_CLK(RCC_AHB4ENR, 28, "bkpram", "hclk"),
+	PER_CLK(RCC_AHB4ENR, 25, "hsem", "hclk"),
+	PER_CLK(RCC_AHB4ENR, 21, "bdma", "hclk"),
+	PER_CLK(RCC_AHB4ENR, 19, "crc", "hclk"),
+	PER_CLK(RCC_AHB4ENR, 10, "gpiok", "hclk"),
+	PER_CLK(RCC_AHB4ENR, 9, "gpioj", "hclk"),
+	PER_CLK(RCC_AHB4ENR, 8, "gpioi", "hclk"),
+	PER_CLK(RCC_AHB4ENR, 7, "gpioh", "hclk"),
+	PER_CLK(RCC_AHB4ENR, 6, "gpiog", "hclk"),
+	PER_CLK(RCC_AHB4ENR, 5, "gpiof", "hclk"),
+	PER_CLK(RCC_AHB4ENR, 4, "gpioe", "hclk"),
+	PER_CLK(RCC_AHB4ENR, 3, "gpiod", "hclk"),
+	PER_CLK(RCC_AHB4ENR, 2, "gpioc", "hclk"),
+	PER_CLK(RCC_AHB4ENR, 1, "gpiob", "hclk"),
+	PER_CLK(RCC_AHB4ENR, 0, "gpioa", "hclk"),
+	PER_CLK(RCC_APB3ENR, 6, "wwdg1", "pclk3"),
+	PER_CLK(RCC_APB1LENR, 29, "dac12", "pclk1"),
+	PER_CLK(RCC_APB1LENR, 11, "wwdg2", "pclk1"),
+	PER_CLK(RCC_APB1LENR, 8, "tim14", "tim1_ker"),
+	PER_CLK(RCC_APB1LENR, 7, "tim13", "tim1_ker"),
+	PER_CLK(RCC_APB1LENR, 6, "tim12", "tim1_ker"),
+	PER_CLK(RCC_APB1LENR, 5, "tim7", "tim1_ker"),
+	PER_CLK(RCC_APB1LENR, 4, "tim6", "tim1_ker"),
+	PER_CLK(RCC_APB1LENR, 3, "tim5", "tim1_ker"),
+	PER_CLK(RCC_APB1LENR, 2, "tim4", "tim1_ker"),
+	PER_CLK(RCC_APB1LENR, 1, "tim3", "tim1_ker"),
+	PER_CLK(RCC_APB1LENR, 0, "tim2", "tim1_ker"),
+	PER_CLK(RCC_APB1HENR, 5, "mdios", "pclk1"),
+	PER_CLK(RCC_APB1HENR, 4, "opamp", "pclk1"),
+	PER_CLK(RCC_APB1HENR, 1, "crs", "pclk1"),
+	PER_CLK(RCC_APB2ENR, 18, "tim17", "tim2_ker"),
+	PER_CLK(RCC_APB2ENR, 17, "tim16", "tim2_ker"),
+	PER_CLK(RCC_APB2ENR, 16, "tim15", "tim2_ker"),
+	PER_CLK(RCC_APB2ENR, 1, "tim8", "tim2_ker"),
+	PER_CLK(RCC_APB2ENR, 0, "tim1", "tim2_ker"),
+	PER_CLK(RCC_APB4ENR, 26, "tmpsens", "pclk4"),
+	PER_CLK(RCC_APB4ENR, 16, "rtcapb", "pclk4"),
+	PER_CLK(RCC_APB4ENR, 15, "vref", "pclk4"),
+	PER_CLK(RCC_APB4ENR, 14, "comp12", "pclk4"),
+	PER_CLK(RCC_APB4ENR, 1, "syscfg", "pclk4"),
+};
+
+/* KERNEL CLOCKS */
+#define KER_CLKF(_gate_offset, _bit_idx,\
+		_mux_offset, _mux_shift, _mux_width,\
+		_name, _parent_name,\
+		_flags) \
+{ \
+	.gate = &(struct gate_cfg) {_gate_offset, _bit_idx},\
+	.mux = &(struct muxdiv_cfg) {_mux_offset, _mux_shift, _mux_width },\
+	.name = _name, \
+	.parent_name = _parent_name, \
+	.num_parents = ARRAY_SIZE(_parent_name),\
+	.flags = _flags,\
+}
+
+#define KER_CLK(_gate_offset, _bit_idx, _mux_offset, _mux_shift, _mux_width,\
+		_name, _parent_name) \
+KER_CLKF(_gate_offset, _bit_idx, _mux_offset, _mux_shift, _mux_width,\
+		_name, _parent_name, 0)\
+
+#define KER_CLKF_NOMUX(_gate_offset, _bit_idx,\
+		_name, _parent_name,\
+		_flags) \
+{ \
+	.gate = &(struct gate_cfg) {_gate_offset, _bit_idx},\
+	.mux = NULL,\
+	.name = _name, \
+	.parent_name = _parent_name, \
+	.num_parents = 1,\
+	.flags = _flags,\
+}
+
+static const struct composite_clk_cfg kclk[] = {
+	KER_CLK(RCC_AHB3ENR,  16, RCC_D1CCIPR,	16, 1, "sdmmc1", sdmmc_src),
+	KER_CLKF(RCC_AHB3ENR, 14, RCC_D1CCIPR,	 4, 2, "quadspi", qspi_src,
+			CLK_IGNORE_UNUSED),
+	KER_CLKF(RCC_AHB3ENR, 12, RCC_D1CCIPR,	 0, 2, "fmc", fmc_src,
+			CLK_IGNORE_UNUSED),
+	KER_CLK(RCC_AHB1ENR,  27, RCC_D2CCIP2R,	20, 2, "usb2otg", usbotg_src),
+	KER_CLK(RCC_AHB1ENR,  25, RCC_D2CCIP2R, 20, 2, "usb1otg", usbotg_src),
+	KER_CLK(RCC_AHB1ENR,   5, RCC_D3CCIPR,	16, 2, "adc12", adc_src),
+	KER_CLK(RCC_AHB2ENR,   9, RCC_D1CCIPR,	16, 1, "sdmmc2", sdmmc_src),
+	KER_CLK(RCC_AHB2ENR,   6, RCC_D2CCIP2R,	 8, 2, "rng", rng_src),
+	KER_CLK(RCC_AHB4ENR,  24, RCC_D3CCIPR,  16, 2, "adc3", adc_src),
+	KER_CLKF(RCC_APB3ENR,   4, RCC_D1CCIPR,	 8, 1, "dsi", dsi_src,
+			CLK_SET_RATE_PARENT),
+	KER_CLKF_NOMUX(RCC_APB3ENR, 3, "ltdc", ltdc_src, CLK_SET_RATE_PARENT),
+	KER_CLK(RCC_APB1LENR, 31, RCC_D2CCIP2R,  0, 3, "usart8", usart_src2),
+	KER_CLK(RCC_APB1LENR, 30, RCC_D2CCIP2R,  0, 3, "usart7", usart_src2),
+	KER_CLK(RCC_APB1LENR, 27, RCC_D2CCIP2R, 22, 2, "hdmicec", cec_src),
+	KER_CLK(RCC_APB1LENR, 23, RCC_D2CCIP2R, 12, 2, "i2c3", i2c_src1),
+	KER_CLK(RCC_APB1LENR, 22, RCC_D2CCIP2R, 12, 2, "i2c2", i2c_src1),
+	KER_CLK(RCC_APB1LENR, 21, RCC_D2CCIP2R, 12, 2, "i2c1", i2c_src1),
+	KER_CLK(RCC_APB1LENR, 20, RCC_D2CCIP2R,	 0, 3, "uart5", usart_src2),
+	KER_CLK(RCC_APB1LENR, 19, RCC_D2CCIP2R,  0, 3, "uart4", usart_src2),
+	KER_CLK(RCC_APB1LENR, 18, RCC_D2CCIP2R,  0, 3, "usart3", usart_src2),
+	KER_CLK(RCC_APB1LENR, 17, RCC_D2CCIP2R,  0, 3, "usart2", usart_src2),
+	KER_CLK(RCC_APB1LENR, 16, RCC_D2CCIP1R, 20, 2, "spdifrx", spdifrx_src),
+	KER_CLK(RCC_APB1LENR, 15, RCC_D2CCIP1R, 16, 3, "spi3", spi_src1),
+	KER_CLK(RCC_APB1LENR, 14, RCC_D2CCIP1R, 16, 3, "spi2", spi_src1),
+	KER_CLK(RCC_APB1LENR,  9, RCC_D2CCIP2R, 28, 3, "lptim1", lptim_src1),
+	KER_CLK(RCC_APB1HENR,  8, RCC_D2CCIP1R, 28, 2, "fdcan", fdcan_src),
+	KER_CLK(RCC_APB1HENR,  2, RCC_D2CCIP1R, 31, 1, "swp", swp_src),
+	KER_CLK(RCC_APB2ENR,  29, RCC_CFGR,	14, 1, "hrtim", hrtim_src),
+	KER_CLK(RCC_APB2ENR,  28, RCC_D2CCIP1R, 24, 1, "dfsdm1", dfsdm1_src),
+	KER_CLKF(RCC_APB2ENR,  24, RCC_D2CCIP1R,  6, 3, "sai3", sai_src,
+		 CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT),
+	KER_CLKF(RCC_APB2ENR,  23, RCC_D2CCIP1R,  6, 3, "sai2", sai_src,
+		 CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT),
+	KER_CLKF(RCC_APB2ENR,  22, RCC_D2CCIP1R,  0, 3, "sai1", sai_src,
+		 CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT),
+	KER_CLK(RCC_APB2ENR,  20, RCC_D2CCIP1R, 16, 3, "spi5", spi_src2),
+	KER_CLK(RCC_APB2ENR,  13, RCC_D2CCIP1R, 16, 3, "spi4", spi_src2),
+	KER_CLK(RCC_APB2ENR,  12, RCC_D2CCIP1R, 16, 3, "spi1", spi_src1),
+	KER_CLK(RCC_APB2ENR,   5, RCC_D2CCIP2R,  3, 3, "usart6", usart_src1),
+	KER_CLK(RCC_APB2ENR,   4, RCC_D2CCIP2R,  3, 3, "usart1", usart_src1),
+	KER_CLK(RCC_APB4ENR,  21, RCC_D3CCIPR,	24, 3, "sai4b", sai_src),
+	KER_CLK(RCC_APB4ENR,  21, RCC_D3CCIPR,	21, 3, "sai4a", sai_src),
+	KER_CLK(RCC_APB4ENR,  12, RCC_D3CCIPR,	13, 3, "lptim5", lptim_src2),
+	KER_CLK(RCC_APB4ENR,  11, RCC_D3CCIPR,	13, 3, "lptim4", lptim_src2),
+	KER_CLK(RCC_APB4ENR,  10, RCC_D3CCIPR,	13, 3, "lptim3", lptim_src2),
+	KER_CLK(RCC_APB4ENR,   9, RCC_D3CCIPR,	10, 3, "lptim2", lptim_src2),
+	KER_CLK(RCC_APB4ENR,   7, RCC_D3CCIPR,	 8, 2, "i2c4", i2c_src2),
+	KER_CLK(RCC_APB4ENR,   5, RCC_D3CCIPR,	28, 3, "spi6", spi_src3),
+	KER_CLK(RCC_APB4ENR,   3, RCC_D3CCIPR,	 0, 3, "lpuart1", lpuart1_src),
+};
+
+static struct composite_clk_gcfg kernel_clk_cfg = {
+	M_CFG_MUX(NULL, 0),
+	M_CFG_GATE(NULL, 0),
+};
+
+/* RTC clock */
+/*
+ * RTC & LSE registers are protected against parasitic write access.
+ * PWR_CR_DBP bit must be set to enable write access to RTC registers.
+ */
+/* STM32_PWR_CR */
+#define PWR_CR				0x00
+/* STM32_PWR_CR bit field */
+#define PWR_CR_DBP			BIT(8)
+
+static struct composite_clk_gcfg rtc_clk_cfg = {
+	M_CFG_MUX(NULL, 0),
+	M_CFG_GATE(NULL, 0),
+};
+
+static const struct composite_clk_cfg rtc_clk =
+	KER_CLK(RCC_BDCR, 15, RCC_BDCR, 8, 2, "rtc_ck", rtc_src);
+
+/* Micro-controller output clock */
+static struct composite_clk_gcfg mco_clk_cfg = {
+	M_CFG_MUX(NULL, 0),
+	M_CFG_DIV(NULL,	CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO),
+};
+
+#define M_MCO_F(_name, _parents, _mux_offset,  _mux_shift, _mux_width,\
+		_rate_offset, _rate_shift, _rate_width,\
+		_flags)\
+{\
+	.mux = &(struct muxdiv_cfg) {_mux_offset, _mux_shift, _mux_width },\
+	.div = &(struct muxdiv_cfg) {_rate_offset, _rate_shift, _rate_width},\
+	.gate = NULL,\
+	.name = _name,\
+	.parent_name = _parents,\
+	.num_parents = ARRAY_SIZE(_parents),\
+	.flags = _flags,\
+}
+
+static const struct composite_clk_cfg mco_clk[] = {
+	M_MCO_F("mco1", mco_src1, RCC_CFGR, 22, 4, RCC_CFGR, 18, 4, 0),
+	M_MCO_F("mco2", mco_src2, RCC_CFGR, 29, 3, RCC_CFGR, 25, 4, 0),
+};
+
+static void __init stm32h7_rcc_init(struct device_node *np)
+{
+	struct clk_hw_onecell_data *clk_data;
+	struct composite_cfg c_cfg;
+	int n;
+	const char *hse_clk, *lse_clk, *i2s_clk;
+	struct regmap *pdrm;
+
+	clk_data = kzalloc(sizeof(*clk_data) +
+			sizeof(*clk_data->hws) * STM32H7_MAX_CLKS,
+			GFP_KERNEL);
+	if (!clk_data)
+		return;
+
+	clk_data->num = STM32H7_MAX_CLKS;
+
+	hws = clk_data->hws;
+
+	for (n = 0; n < STM32H7_MAX_CLKS; n++)
+		hws[n] = ERR_PTR(-ENOENT);
+
+	/* get RCC base @ from DT */
+	base = of_iomap(np, 0);
+	if (!base) {
+		pr_err("%s: unable to map resource", np->name);
+		goto err_free_clks;
+	}
+
+	pdrm = syscon_regmap_lookup_by_phandle(np, "st,syscfg");
+	if (IS_ERR(pdrm))
+		pr_warn("%s: Unable to get syscfg\n", __func__);
+	else
+		/* In any case disable backup domain write protection
+		 * and will never be enabled.
+		 * Needed by LSE & RTC clocks.
+		 */
+		regmap_update_bits(pdrm, PWR_CR, PWR_CR_DBP, PWR_CR_DBP);
+
+	/* Put parent names from DT */
+	hse_clk = of_clk_get_parent_name(np, 0);
+	lse_clk = of_clk_get_parent_name(np, 1);
+	i2s_clk = of_clk_get_parent_name(np, 2);
+
+	sai_src[3] = i2s_clk;
+	spi_src1[3] = i2s_clk;
+
+	/* Register Internal oscillators */
+	clk_hw_register_fixed_rate(NULL, "clk-hsi", NULL, 0, 64000000);
+	clk_hw_register_fixed_rate(NULL, "clk-csi", NULL, 0, 4000000);
+	clk_hw_register_fixed_rate(NULL, "clk-lsi", NULL, 0, 32000);
+	clk_hw_register_fixed_rate(NULL, "clk-rc48", NULL, 0, 48000);
+
+	/* This clock is coming from outside. Frequencies unknown */
+	hws[CK_DSI_PHY] = clk_hw_register_fixed_rate(NULL, "ck_dsi_phy", NULL,
+			0, 0);
+
+	hws[HSI_DIV] = clk_hw_register_divider(NULL, "hsidiv", "clk-hsi", 0,
+			base + RCC_CR, 3, 2, CLK_DIVIDER_POWER_OF_TWO,
+			&stm32rcc_lock);
+
+	hws[HSE_1M] = clk_hw_register_divider(NULL, "hse_1M", "hse_ck",	0,
+			base + RCC_CFGR, 8, 6, CLK_DIVIDER_ONE_BASED |
+			CLK_DIVIDER_ALLOW_ZERO,
+			&stm32rcc_lock);
+
+	/* Mux system clocks */
+	for (n = 0; n < ARRAY_SIZE(stm32_mclk); n++)
+		hws[MCLK_BANK + n] = clk_hw_register_mux(NULL,
+				stm32_mclk[n].name,
+				stm32_mclk[n].parents,
+				stm32_mclk[n].num_parents,
+				stm32_mclk[n].flags,
+				stm32_mclk[n].offset + base,
+				stm32_mclk[n].shift,
+				stm32_mclk[n].width,
+				0,
+				&stm32rcc_lock);
+
+	register_core_and_bus_clocks();
+
+	/* Oscillary clocks */
+	for (n = 0; n < ARRAY_SIZE(stm32_oclk); n++)
+		hws[OSC_BANK + n] = clk_register_ready_gate(NULL,
+				stm32_oclk[n].name,
+				stm32_oclk[n].parent,
+				stm32_oclk[n].gate_offset + base,
+				stm32_oclk[n].bit_idx,
+				stm32_oclk[n].bit_rdy,
+				stm32_oclk[n].flags,
+				&stm32rcc_lock);
+
+	hws[HSE_CK] = clk_register_ready_gate(NULL,
+				"hse_ck",
+				hse_clk,
+				RCC_CR + base,
+				16, 17,
+				0,
+				&stm32rcc_lock);
+
+	hws[LSE_CK] = clk_register_ready_gate(NULL,
+				"lse_ck",
+				lse_clk,
+				RCC_BDCR + base,
+				0, 1,
+				0,
+				&stm32rcc_lock);
+
+	hws[CSI_KER_DIV122 + n] = clk_hw_register_fixed_factor(NULL,
+			"csi_ker_div122", "csi_ker", 0, 1, 122);
+
+	/* PLLs */
+	for (n = 0; n < ARRAY_SIZE(stm32_pll); n++) {
+		int odf;
+
+		/* Register the VCO */
+		clk_register_stm32_pll(NULL, stm32_pll[n].name,
+				stm32_pll[n].parent_name, stm32_pll[n].flags,
+				stm32_pll[n].cfg,
+				&stm32rcc_lock);
+
+		/* Register the 3 output dividers */
+		for (odf = 0; odf < 3; odf++) {
+			int idx = n * 3 + odf;
+
+			get_cfg_composite_div(&odf_clk_gcfg, &stm32_odf[n][odf],
+					&c_cfg,	&stm32rcc_lock);
+
+			hws[ODF_BANK + idx] = clk_hw_register_composite(NULL,
+					stm32_odf[n][odf].name,
+					stm32_odf[n][odf].parent_name,
+					stm32_odf[n][odf].num_parents,
+					c_cfg.mux_hw, c_cfg.mux_ops,
+					c_cfg.div_hw, c_cfg.div_ops,
+					c_cfg.gate_hw, c_cfg.gate_ops,
+					stm32_odf[n][odf].flags);
+		}
+	}
+
+	/* Peripheral clocks */
+	for (n = 0; n < ARRAY_SIZE(pclk); n++)
+		hws[PERIF_BANK + n] = clk_hw_register_gate(NULL, pclk[n].name,
+				pclk[n].parent,
+				pclk[n].flags, base + pclk[n].gate_offset,
+				pclk[n].bit_idx, pclk[n].flags, &stm32rcc_lock);
+
+	/* Kernel clocks */
+	for (n = 0; n < ARRAY_SIZE(kclk); n++) {
+		get_cfg_composite_div(&kernel_clk_cfg, &kclk[n], &c_cfg,
+				&stm32rcc_lock);
+
+		hws[KERN_BANK + n] = clk_hw_register_composite(NULL,
+				kclk[n].name,
+				kclk[n].parent_name,
+				kclk[n].num_parents,
+				c_cfg.mux_hw, c_cfg.mux_ops,
+				c_cfg.div_hw, c_cfg.div_ops,
+				c_cfg.gate_hw, c_cfg.gate_ops,
+				kclk[n].flags);
+	}
+
+	/* RTC clock (default state is off) */
+	clk_hw_register_fixed_rate(NULL, "off", NULL, 0, 0);
+
+	get_cfg_composite_div(&rtc_clk_cfg, &rtc_clk, &c_cfg, &stm32rcc_lock);
+
+	hws[RTC_CK] = clk_hw_register_composite(NULL,
+			rtc_clk.name,
+			rtc_clk.parent_name,
+			rtc_clk.num_parents,
+			c_cfg.mux_hw, c_cfg.mux_ops,
+			c_cfg.div_hw, c_cfg.div_ops,
+			c_cfg.gate_hw, c_cfg.gate_ops,
+			rtc_clk.flags);
+
+	/* Micro-controller clocks */
+	for (n = 0; n < ARRAY_SIZE(mco_clk); n++) {
+		get_cfg_composite_div(&mco_clk_cfg, &mco_clk[n], &c_cfg,
+				&stm32rcc_lock);
+
+		hws[MCO_BANK + n] = clk_hw_register_composite(NULL,
+				mco_clk[n].name,
+				mco_clk[n].parent_name,
+				mco_clk[n].num_parents,
+				c_cfg.mux_hw, c_cfg.mux_ops,
+				c_cfg.div_hw, c_cfg.div_ops,
+				c_cfg.gate_hw, c_cfg.gate_ops,
+				mco_clk[n].flags);
+	}
+
+	of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_data);
+
+	return;
+
+err_free_clks:
+	kfree(clk_data);
+}
+
+/* The RCC node is a clock and reset controller, and these
+ * functionalities are supported by different drivers that
+ * matches the same compatible strings.
+ */
+CLK_OF_DECLARE_DRIVER(stm32h7_rcc, "st,stm32h743-rcc", stm32h7_rcc_init);
diff --git a/drivers/clk/clk-versaclock5.c b/drivers/clk/clk-versaclock5.c
index ea7d552a2f2b..decffb3826ec 100644
--- a/drivers/clk/clk-versaclock5.c
+++ b/drivers/clk/clk-versaclock5.c
@@ -57,6 +57,7 @@
 #define VC5_PRIM_SRC_SHDN			0x10
 #define VC5_PRIM_SRC_SHDN_EN_XTAL		BIT(7)
 #define VC5_PRIM_SRC_SHDN_EN_CLKIN		BIT(6)
+#define VC5_PRIM_SRC_SHDN_EN_DOUBLE_XTAL_FREQ	BIT(3)
 #define VC5_PRIM_SRC_SHDN_SP			BIT(1)
 #define VC5_PRIM_SRC_SHDN_EN_GBL_SHDN		BIT(0)
 
@@ -122,12 +123,16 @@
 /* flags to describe chip features */
 /* chip has built-in oscilator */
 #define VC5_HAS_INTERNAL_XTAL	BIT(0)
+/* chip has PFD requency doubler */
+#define VC5_HAS_PFD_FREQ_DBL	BIT(1)
 
 /* Supported IDT VC5 models. */
 enum vc5_model {
 	IDT_VC5_5P49V5923,
+	IDT_VC5_5P49V5925,
 	IDT_VC5_5P49V5933,
 	IDT_VC5_5P49V5935,
+	IDT_VC6_5P49V6901,
 };
 
 /* Structure to describe features of a particular VC5 model */
@@ -157,6 +162,8 @@ struct vc5_driver_data {
 	struct clk		*pin_clkin;
 	unsigned char		clk_mux_ins;
 	struct clk_hw		clk_mux;
+	struct clk_hw		clk_mul;
+	struct clk_hw		clk_pfd;
 	struct vc5_hw_data	clk_pll;
 	struct vc5_hw_data	clk_fod[VC5_MAX_FOD_NUM];
 	struct vc5_hw_data	clk_out[VC5_MAX_CLK_OUT_NUM];
@@ -166,6 +173,14 @@ static const char * const vc5_mux_names[] = {
 	"mux"
 };
 
+static const char * const vc5_dbl_names[] = {
+	"dbl"
+};
+
+static const char * const vc5_pfd_names[] = {
+	"pfd"
+};
+
 static const char * const vc5_pll_names[] = {
 	"pll"
 };
@@ -254,11 +269,64 @@ static int vc5_mux_set_parent(struct clk_hw *hw, u8 index)
 	return regmap_update_bits(vc5->regmap, VC5_PRIM_SRC_SHDN, mask, src);
 }
 
-static unsigned long vc5_mux_recalc_rate(struct clk_hw *hw,
+static const struct clk_ops vc5_mux_ops = {
+	.set_parent	= vc5_mux_set_parent,
+	.get_parent	= vc5_mux_get_parent,
+};
+
+static unsigned long vc5_dbl_recalc_rate(struct clk_hw *hw,
 					 unsigned long parent_rate)
 {
 	struct vc5_driver_data *vc5 =
-		container_of(hw, struct vc5_driver_data, clk_mux);
+		container_of(hw, struct vc5_driver_data, clk_mul);
+	unsigned int premul;
+
+	regmap_read(vc5->regmap, VC5_PRIM_SRC_SHDN, &premul);
+	if (premul & VC5_PRIM_SRC_SHDN_EN_DOUBLE_XTAL_FREQ)
+		parent_rate *= 2;
+
+	return parent_rate;
+}
+
+static long vc5_dbl_round_rate(struct clk_hw *hw, unsigned long rate,
+			       unsigned long *parent_rate)
+{
+	if ((*parent_rate == rate) || ((*parent_rate * 2) == rate))
+		return rate;
+	else
+		return -EINVAL;
+}
+
+static int vc5_dbl_set_rate(struct clk_hw *hw, unsigned long rate,
+			    unsigned long parent_rate)
+{
+	struct vc5_driver_data *vc5 =
+		container_of(hw, struct vc5_driver_data, clk_mul);
+	u32 mask;
+
+	if ((parent_rate * 2) == rate)
+		mask = VC5_PRIM_SRC_SHDN_EN_DOUBLE_XTAL_FREQ;
+	else
+		mask = 0;
+
+	regmap_update_bits(vc5->regmap, VC5_PRIM_SRC_SHDN,
+			   VC5_PRIM_SRC_SHDN_EN_DOUBLE_XTAL_FREQ,
+			   mask);
+
+	return 0;
+}
+
+static const struct clk_ops vc5_dbl_ops = {
+	.recalc_rate	= vc5_dbl_recalc_rate,
+	.round_rate	= vc5_dbl_round_rate,
+	.set_rate	= vc5_dbl_set_rate,
+};
+
+static unsigned long vc5_pfd_recalc_rate(struct clk_hw *hw,
+					 unsigned long parent_rate)
+{
+	struct vc5_driver_data *vc5 =
+		container_of(hw, struct vc5_driver_data, clk_pfd);
 	unsigned int prediv, div;
 
 	regmap_read(vc5->regmap, VC5_VCO_CTRL_AND_PREDIV, &prediv);
@@ -276,7 +344,7 @@ static unsigned long vc5_mux_recalc_rate(struct clk_hw *hw,
 		return parent_rate / VC5_REF_DIVIDER_REF_DIV(div);
 }
 
-static long vc5_mux_round_rate(struct clk_hw *hw, unsigned long rate,
+static long vc5_pfd_round_rate(struct clk_hw *hw, unsigned long rate,
 			       unsigned long *parent_rate)
 {
 	unsigned long idiv;
@@ -296,11 +364,11 @@ static long vc5_mux_round_rate(struct clk_hw *hw, unsigned long rate,
 	return *parent_rate / idiv;
 }
 
-static int vc5_mux_set_rate(struct clk_hw *hw, unsigned long rate,
+static int vc5_pfd_set_rate(struct clk_hw *hw, unsigned long rate,
 			    unsigned long parent_rate)
 {
 	struct vc5_driver_data *vc5 =
-		container_of(hw, struct vc5_driver_data, clk_mux);
+		container_of(hw, struct vc5_driver_data, clk_pfd);
 	unsigned long idiv;
 	u8 div;
 
@@ -328,12 +396,10 @@ static int vc5_mux_set_rate(struct clk_hw *hw, unsigned long rate,
 	return 0;
 }
 
-static const struct clk_ops vc5_mux_ops = {
-	.set_parent	= vc5_mux_set_parent,
-	.get_parent	= vc5_mux_get_parent,
-	.recalc_rate	= vc5_mux_recalc_rate,
-	.round_rate	= vc5_mux_round_rate,
-	.set_rate	= vc5_mux_set_rate,
+static const struct clk_ops vc5_pfd_ops = {
+	.recalc_rate	= vc5_pfd_recalc_rate,
+	.round_rate	= vc5_pfd_round_rate,
+	.set_rate	= vc5_pfd_set_rate,
 };
 
 /*
@@ -426,6 +492,10 @@ static unsigned long vc5_fod_recalc_rate(struct clk_hw *hw,
 	div_frc = (od_frc[0] << 22) | (od_frc[1] << 14) |
 		  (od_frc[2] << 6) | (od_frc[3] >> 2);
 
+	/* Avoid division by zero if the output is not configured. */
+	if (div_int == 0 && div_frc == 0)
+		return 0;
+
 	/* The PLL divider has 12 integer bits and 30 fractional bits */
 	return div64_u64((u64)f_in << 24ULL, ((u64)div_int << 24ULL) + div_frc);
 }
@@ -503,6 +573,25 @@ static int vc5_clk_out_prepare(struct clk_hw *hw)
 {
 	struct vc5_hw_data *hwdata = container_of(hw, struct vc5_hw_data, hw);
 	struct vc5_driver_data *vc5 = hwdata->vc5;
+	const u8 mask = VC5_OUT_DIV_CONTROL_SELB_NORM |
+			VC5_OUT_DIV_CONTROL_SEL_EXT |
+			VC5_OUT_DIV_CONTROL_EN_FOD;
+	unsigned int src;
+	int ret;
+
+	/*
+	 * If the input mux is disabled, enable it first and
+	 * select source from matching FOD.
+	 */
+	regmap_read(vc5->regmap, VC5_OUT_DIV_CONTROL(hwdata->num), &src);
+	if ((src & mask) == 0) {
+		src = VC5_OUT_DIV_CONTROL_RESET | VC5_OUT_DIV_CONTROL_EN_FOD;
+		ret = regmap_update_bits(vc5->regmap,
+					 VC5_OUT_DIV_CONTROL(hwdata->num),
+					 mask | VC5_OUT_DIV_CONTROL_RESET, src);
+		if (ret)
+			return ret;
+	}
 
 	/* Enable the clock buffer */
 	regmap_update_bits(vc5->regmap, VC5_CLK_OUTPUT_CFG(hwdata->num, 1),
@@ -516,7 +605,7 @@ static void vc5_clk_out_unprepare(struct clk_hw *hw)
 	struct vc5_hw_data *hwdata = container_of(hw, struct vc5_hw_data, hw);
 	struct vc5_driver_data *vc5 = hwdata->vc5;
 
-	/* Enable the clock buffer */
+	/* Disable the clock buffer */
 	regmap_update_bits(vc5->regmap, VC5_CLK_OUTPUT_CFG(hwdata->num, 1),
 			   VC5_CLK_OUTPUT_CFG1_EN_CLKBUF, 0);
 }
@@ -537,6 +626,9 @@ static unsigned char vc5_clk_out_get_parent(struct clk_hw *hw)
 	regmap_read(vc5->regmap, VC5_OUT_DIV_CONTROL(hwdata->num), &src);
 	src &= mask;
 
+	if (src == 0)	/* Input mux set to DISABLED */
+		return 0;
+
 	if ((src & fodclkmask) == VC5_OUT_DIV_CONTROL_EN_FOD)
 		return 0;
 
@@ -595,7 +687,9 @@ static int vc5_map_index_to_output(const enum vc5_model model,
 	case IDT_VC5_5P49V5933:
 		return (n == 0) ? 0 : 3;
 	case IDT_VC5_5P49V5923:
+	case IDT_VC5_5P49V5925:
 	case IDT_VC5_5P49V5935:
+	case IDT_VC6_5P49V6901:
 	default:
 		return n;
 	}
@@ -672,12 +766,46 @@ static int vc5_probe(struct i2c_client *client,
 		goto err_clk;
 	}
 
+	if (vc5->chip_info->flags & VC5_HAS_PFD_FREQ_DBL) {
+		/* Register frequency doubler */
+		memset(&init, 0, sizeof(init));
+		init.name = vc5_dbl_names[0];
+		init.ops = &vc5_dbl_ops;
+		init.flags = CLK_SET_RATE_PARENT;
+		init.parent_names = vc5_mux_names;
+		init.num_parents = 1;
+		vc5->clk_mul.init = &init;
+		ret = devm_clk_hw_register(&client->dev, &vc5->clk_mul);
+		if (ret) {
+			dev_err(&client->dev, "unable to register %s\n",
+				init.name);
+			goto err_clk;
+		}
+	}
+
+	/* Register PFD */
+	memset(&init, 0, sizeof(init));
+	init.name = vc5_pfd_names[0];
+	init.ops = &vc5_pfd_ops;
+	init.flags = CLK_SET_RATE_PARENT;
+	if (vc5->chip_info->flags & VC5_HAS_PFD_FREQ_DBL)
+		init.parent_names = vc5_dbl_names;
+	else
+		init.parent_names = vc5_mux_names;
+	init.num_parents = 1;
+	vc5->clk_pfd.init = &init;
+	ret = devm_clk_hw_register(&client->dev, &vc5->clk_pfd);
+	if (ret) {
+		dev_err(&client->dev, "unable to register %s\n", init.name);
+		goto err_clk;
+	}
+
 	/* Register PLL */
 	memset(&init, 0, sizeof(init));
 	init.name = vc5_pll_names[0];
 	init.ops = &vc5_pll_ops;
 	init.flags = CLK_SET_RATE_PARENT;
-	init.parent_names = vc5_mux_names;
+	init.parent_names = vc5_pfd_names;
 	init.num_parents = 1;
 	vc5->clk_pll.num = 0;
 	vc5->clk_pll.vc5 = vc5;
@@ -785,6 +913,13 @@ static const struct vc5_chip_info idt_5p49v5923_info = {
 	.flags = 0,
 };
 
+static const struct vc5_chip_info idt_5p49v5925_info = {
+	.model = IDT_VC5_5P49V5925,
+	.clk_fod_cnt = 4,
+	.clk_out_cnt = 5,
+	.flags = 0,
+};
+
 static const struct vc5_chip_info idt_5p49v5933_info = {
 	.model = IDT_VC5_5P49V5933,
 	.clk_fod_cnt = 2,
@@ -799,18 +934,29 @@ static const struct vc5_chip_info idt_5p49v5935_info = {
 	.flags = VC5_HAS_INTERNAL_XTAL,
 };
 
+static const struct vc5_chip_info idt_5p49v6901_info = {
+	.model = IDT_VC6_5P49V6901,
+	.clk_fod_cnt = 4,
+	.clk_out_cnt = 5,
+	.flags = VC5_HAS_PFD_FREQ_DBL,
+};
+
 static const struct i2c_device_id vc5_id[] = {
 	{ "5p49v5923", .driver_data = IDT_VC5_5P49V5923 },
+	{ "5p49v5925", .driver_data = IDT_VC5_5P49V5925 },
 	{ "5p49v5933", .driver_data = IDT_VC5_5P49V5933 },
 	{ "5p49v5935", .driver_data = IDT_VC5_5P49V5935 },
+	{ "5p49v6901", .driver_data = IDT_VC6_5P49V6901 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, vc5_id);
 
 static const struct of_device_id clk_vc5_of_match[] = {
 	{ .compatible = "idt,5p49v5923", .data = &idt_5p49v5923_info },
+	{ .compatible = "idt,5p49v5925", .data = &idt_5p49v5925_info },
 	{ .compatible = "idt,5p49v5933", .data = &idt_5p49v5933_info },
 	{ .compatible = "idt,5p49v5935", .data = &idt_5p49v5935_info },
+	{ .compatible = "idt,5p49v6901", .data = &idt_5p49v6901_info },
 	{ },
 };
 MODULE_DEVICE_TABLE(of, clk_vc5_of_match);
diff --git a/drivers/clk/clk-xgene.c b/drivers/clk/clk-xgene.c
index bc37030e38ba..4c75821a3933 100644
--- a/drivers/clk/clk-xgene.c
+++ b/drivers/clk/clk-xgene.c
@@ -192,7 +192,7 @@ static void xgene_pllclk_init(struct device_node *np, enum xgene_pll_type pll_ty
 
 	reg = of_iomap(np, 0);
 	if (reg == NULL) {
-		pr_err("Unable to map CSR register for %s\n", np->full_name);
+		pr_err("Unable to map CSR register for %pOF\n", np);
 		return;
 	}
 	of_property_read_string(np, "clock-output-names", &clk_name);
@@ -409,12 +409,12 @@ static void xgene_pmdclk_init(struct device_node *np)
 	/* Parse the DTS register for resource */
 	rc = of_address_to_resource(np, 0, &res);
 	if (rc != 0) {
-		pr_err("no DTS register for %s\n", np->full_name);
+		pr_err("no DTS register for %pOF\n", np);
 		return;
 	}
 	csr_reg = of_iomap(np, 0);
 	if (!csr_reg) {
-		pr_err("Unable to map resource for %s\n", np->full_name);
+		pr_err("Unable to map resource for %pOF\n", np);
 		return;
 	}
 	of_property_read_string(np, "clock-output-names", &clk_name);
@@ -703,16 +703,14 @@ static void __init xgene_devclk_init(struct device_node *np)
 		rc = of_address_to_resource(np, i, &res);
 		if (rc != 0) {
 			if (i == 0) {
-				pr_err("no DTS register for %s\n",
-					np->full_name);
+				pr_err("no DTS register for %pOF\n", np);
 				return;
 			}
 			break;
 		}
 		map_res = of_iomap(np, i);
 		if (map_res == NULL) {
-			pr_err("Unable to map resource %d for %s\n",
-				i, np->full_name);
+			pr_err("Unable to map resource %d for %pOF\n", i, np);
 			goto err;
 		}
 		if (strcmp(res.name, "div-reg") == 0)
@@ -747,8 +745,7 @@ static void __init xgene_devclk_init(struct device_node *np)
 	pr_debug("Add %s clock\n", clk_name);
 	rc = of_clk_add_provider(np, of_clk_src_simple_get, clk);
 	if (rc != 0)
-		pr_err("%s: could register provider clk %s\n", __func__,
-			np->full_name);
+		pr_err("%s: could register provider clk %pOF\n", __func__, np);
 
 	return;
 
diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c
index fc58c52a26b4..c8d83acda006 100644
--- a/drivers/clk/clk.c
+++ b/drivers/clk/clk.c
@@ -3132,7 +3132,7 @@ int of_clk_add_provider(struct device_node *np,
 	mutex_lock(&of_clk_mutex);
 	list_add(&cp->link, &of_clk_providers);
 	mutex_unlock(&of_clk_mutex);
-	pr_debug("Added clock from %s\n", np->full_name);
+	pr_debug("Added clock from %pOF\n", np);
 
 	ret = of_clk_set_defaults(np, true);
 	if (ret < 0)
@@ -3167,7 +3167,7 @@ int of_clk_add_hw_provider(struct device_node *np,
 	mutex_lock(&of_clk_mutex);
 	list_add(&cp->link, &of_clk_providers);
 	mutex_unlock(&of_clk_mutex);
-	pr_debug("Added clk_hw provider from %s\n", np->full_name);
+	pr_debug("Added clk_hw provider from %pOF\n", np);
 
 	ret = of_clk_set_defaults(np, true);
 	if (ret < 0)
diff --git a/drivers/clk/clkdev.c b/drivers/clk/clkdev.c
index bb8a77a5985f..6b2f29df3f70 100644
--- a/drivers/clk/clkdev.c
+++ b/drivers/clk/clkdev.c
@@ -77,8 +77,8 @@ static struct clk *__of_clk_get_by_name(struct device_node *np,
 			break;
 		} else if (name && index >= 0) {
 			if (PTR_ERR(clk) != -EPROBE_DEFER)
-				pr_err("ERROR: could not get clock %s:%s(%i)\n",
-					np->full_name, name ? name : "", index);
+				pr_err("ERROR: could not get clock %pOF:%s(%i)\n",
+					np, name ? name : "", index);
 			return clk;
 		}
 
diff --git a/drivers/clk/hisilicon/clk-hi6220.c b/drivers/clk/hisilicon/clk-hi6220.c
index 4181b6808545..e786d717f75d 100644
--- a/drivers/clk/hisilicon/clk-hi6220.c
+++ b/drivers/clk/hisilicon/clk-hi6220.c
@@ -55,9 +55,9 @@ static struct hisi_fixed_factor_clock hi6220_fixed_factor_clks[] __initdata = {
 };
 
 static struct hisi_gate_clock hi6220_separated_gate_clks_ao[] __initdata = {
-	{ HI6220_WDT0_PCLK,   "wdt0_pclk",   "clk_tcxo", CLK_SET_RATE_PARENT|CLK_IGNORE_UNUSED, 0x630, 12, 0, },
-	{ HI6220_WDT1_PCLK,   "wdt1_pclk",   "clk_tcxo", CLK_SET_RATE_PARENT|CLK_IGNORE_UNUSED, 0x630, 13, 0, },
-	{ HI6220_WDT2_PCLK,   "wdt2_pclk",   "clk_tcxo", CLK_SET_RATE_PARENT|CLK_IGNORE_UNUSED, 0x630, 14, 0, },
+	{ HI6220_WDT0_PCLK,   "wdt0_pclk",   "ref32k",   CLK_SET_RATE_PARENT|CLK_IGNORE_UNUSED, 0x630, 12, 0, },
+	{ HI6220_WDT1_PCLK,   "wdt1_pclk",   "ref32k",   CLK_SET_RATE_PARENT|CLK_IGNORE_UNUSED, 0x630, 13, 0, },
+	{ HI6220_WDT2_PCLK,   "wdt2_pclk",   "ref32k",   CLK_SET_RATE_PARENT|CLK_IGNORE_UNUSED, 0x630, 14, 0, },
 	{ HI6220_TIMER0_PCLK, "timer0_pclk", "clk_tcxo", CLK_SET_RATE_PARENT|CLK_IGNORE_UNUSED, 0x630, 15, 0, },
 	{ HI6220_TIMER1_PCLK, "timer1_pclk", "clk_tcxo", CLK_SET_RATE_PARENT|CLK_IGNORE_UNUSED, 0x630, 16, 0, },
 	{ HI6220_TIMER2_PCLK, "timer2_pclk", "clk_tcxo", CLK_SET_RATE_PARENT|CLK_IGNORE_UNUSED, 0x630, 17, 0, },
diff --git a/drivers/clk/imx/clk-imx51-imx53.c b/drivers/clk/imx/clk-imx51-imx53.c
index 1e3c9ea5f9dc..7bcaf270db11 100644
--- a/drivers/clk/imx/clk-imx51-imx53.c
+++ b/drivers/clk/imx/clk-imx51-imx53.c
@@ -416,10 +416,10 @@ static void __init mx51_clocks_init(struct device_node *np)
 
 	clk[IMX5_CLK_LP_APM]		= imx_clk_mux("lp_apm", MXC_CCM_CCSR, 9, 1,
 						lp_apm_sel, ARRAY_SIZE(lp_apm_sel));
-	clk[IMX5_CLK_IPU_DI0_SEL]	= imx_clk_mux("ipu_di0_sel", MXC_CCM_CSCMR2, 26, 3,
-						mx51_ipu_di0_sel, ARRAY_SIZE(mx51_ipu_di0_sel));
-	clk[IMX5_CLK_IPU_DI1_SEL]	= imx_clk_mux("ipu_di1_sel", MXC_CCM_CSCMR2, 29, 3,
-						mx51_ipu_di1_sel, ARRAY_SIZE(mx51_ipu_di1_sel));
+	clk[IMX5_CLK_IPU_DI0_SEL]	= imx_clk_mux_flags("ipu_di0_sel", MXC_CCM_CSCMR2, 26, 3,
+						mx51_ipu_di0_sel, ARRAY_SIZE(mx51_ipu_di0_sel), CLK_SET_RATE_PARENT);
+	clk[IMX5_CLK_IPU_DI1_SEL]	= imx_clk_mux_flags("ipu_di1_sel", MXC_CCM_CSCMR2, 29, 3,
+						mx51_ipu_di1_sel, ARRAY_SIZE(mx51_ipu_di1_sel), CLK_SET_RATE_PARENT);
 	clk[IMX5_CLK_TVE_EXT_SEL]	= imx_clk_mux_flags("tve_ext_sel", MXC_CCM_CSCMR1, 6, 1,
 						mx51_tve_ext_sel, ARRAY_SIZE(mx51_tve_ext_sel), CLK_SET_RATE_PARENT);
 	clk[IMX5_CLK_TVE_SEL]		= imx_clk_mux("tve_sel", MXC_CCM_CSCMR1, 7, 1,
diff --git a/drivers/clk/imx/clk-imx6sl.c b/drivers/clk/imx/clk-imx6sl.c
index 5fd4ddac1bf1..9642cdf0fb88 100644
--- a/drivers/clk/imx/clk-imx6sl.c
+++ b/drivers/clk/imx/clk-imx6sl.c
@@ -71,7 +71,7 @@ static const char *pll5_bypass_sels[]	= { "pll5", "pll5_bypass_src", };
 static const char *pll6_bypass_sels[]	= { "pll6", "pll6_bypass_src", };
 static const char *pll7_bypass_sels[]	= { "pll7", "pll7_bypass_src", };
 
-static struct clk_div_table clk_enet_ref_table[] = {
+static const struct clk_div_table clk_enet_ref_table[] = {
 	{ .val = 0, .div = 20, },
 	{ .val = 1, .div = 10, },
 	{ .val = 2, .div = 5, },
@@ -79,14 +79,14 @@ static struct clk_div_table clk_enet_ref_table[] = {
 	{ }
 };
 
-static struct clk_div_table post_div_table[] = {
+static const struct clk_div_table post_div_table[] = {
 	{ .val = 2, .div = 1, },
 	{ .val = 1, .div = 2, },
 	{ .val = 0, .div = 4, },
 	{ }
 };
 
-static struct clk_div_table video_div_table[] = {
+static const struct clk_div_table video_div_table[] = {
 	{ .val = 0, .div = 1, },
 	{ .val = 1, .div = 2, },
 	{ .val = 2, .div = 1, },
diff --git a/drivers/clk/imx/clk-imx6sx.c b/drivers/clk/imx/clk-imx6sx.c
index b5c96de41ccf..e6d389e333d7 100644
--- a/drivers/clk/imx/clk-imx6sx.c
+++ b/drivers/clk/imx/clk-imx6sx.c
@@ -105,7 +105,7 @@ static int const clks_init_on[] __initconst = {
 	IMX6SX_CLK_EPIT2,
 };
 
-static struct clk_div_table clk_enet_ref_table[] = {
+static const struct clk_div_table clk_enet_ref_table[] = {
 	{ .val = 0, .div = 20, },
 	{ .val = 1, .div = 10, },
 	{ .val = 2, .div = 5, },
@@ -113,14 +113,14 @@ static struct clk_div_table clk_enet_ref_table[] = {
 	{ }
 };
 
-static struct clk_div_table post_div_table[] = {
+static const struct clk_div_table post_div_table[] = {
 	{ .val = 2, .div = 1, },
 	{ .val = 1, .div = 2, },
 	{ .val = 0, .div = 4, },
 	{ }
 };
 
-static struct clk_div_table video_div_table[] = {
+static const struct clk_div_table video_div_table[] = {
 	{ .val = 0, .div = 1, },
 	{ .val = 1, .div = 2, },
 	{ .val = 2, .div = 1, },
diff --git a/drivers/clk/imx/clk-imx6ul.c b/drivers/clk/imx/clk-imx6ul.c
index b4e0dff3c8c2..5e8c18afce9a 100644
--- a/drivers/clk/imx/clk-imx6ul.c
+++ b/drivers/clk/imx/clk-imx6ul.c
@@ -78,7 +78,7 @@ static int const clks_init_on[] __initconst = {
 	IMX6UL_CLK_MMDC_P0_FAST, IMX6UL_CLK_MMDC_P0_IPG,
 };
 
-static struct clk_div_table clk_enet_ref_table[] = {
+static const struct clk_div_table clk_enet_ref_table[] = {
 	{ .val = 0, .div = 20, },
 	{ .val = 1, .div = 10, },
 	{ .val = 2, .div = 5, },
@@ -86,14 +86,14 @@ static struct clk_div_table clk_enet_ref_table[] = {
 	{ }
 };
 
-static struct clk_div_table post_div_table[] = {
+static const struct clk_div_table post_div_table[] = {
 	{ .val = 2, .div = 1, },
 	{ .val = 1, .div = 2, },
 	{ .val = 0, .div = 4, },
 	{ }
 };
 
-static struct clk_div_table video_div_table[] = {
+static const struct clk_div_table video_div_table[] = {
 	{ .val = 0, .div = 1, },
 	{ .val = 1, .div = 2, },
 	{ .val = 2, .div = 1, },
diff --git a/drivers/clk/imx/clk-imx7d.c b/drivers/clk/imx/clk-imx7d.c
index 3da121826b1b..2305699db467 100644
--- a/drivers/clk/imx/clk-imx7d.c
+++ b/drivers/clk/imx/clk-imx7d.c
@@ -27,7 +27,7 @@ static u32 share_count_sai2;
 static u32 share_count_sai3;
 static u32 share_count_nand;
 
-static struct clk_div_table test_div_table[] = {
+static const struct clk_div_table test_div_table[] = {
 	{ .val = 3, .div = 1, },
 	{ .val = 2, .div = 1, },
 	{ .val = 1, .div = 2, },
@@ -35,7 +35,7 @@ static struct clk_div_table test_div_table[] = {
 	{ }
 };
 
-static struct clk_div_table post_div_table[] = {
+static const struct clk_div_table post_div_table[] = {
 	{ .val = 3, .div = 4, },
 	{ .val = 2, .div = 1, },
 	{ .val = 1, .div = 2, },
diff --git a/drivers/clk/imx/clk-vf610.c b/drivers/clk/imx/clk-vf610.c
index 59b1863deb88..6dae54325a91 100644
--- a/drivers/clk/imx/clk-vf610.c
+++ b/drivers/clk/imx/clk-vf610.c
@@ -102,7 +102,7 @@ static const char *ftm_ext_sels[]	= {"sirc_128k", "sxosc", "fxosc_half", "audio_
 static const char *ftm_fix_sels[]	= { "sxosc", "ipg_bus", };
 
 
-static struct clk_div_table pll4_audio_div_table[] = {
+static const struct clk_div_table pll4_audio_div_table[] = {
 	{ .val = 0, .div = 1 },
 	{ .val = 1, .div = 2 },
 	{ .val = 2, .div = 6 },
diff --git a/drivers/clk/mediatek/clk-cpumux.c b/drivers/clk/mediatek/clk-cpumux.c
index edd8e6918050..16e56772d280 100644
--- a/drivers/clk/mediatek/clk-cpumux.c
+++ b/drivers/clk/mediatek/clk-cpumux.c
@@ -27,7 +27,6 @@ static inline struct mtk_clk_cpumux *to_mtk_clk_cpumux(struct clk_hw *_hw)
 static u8 clk_cpumux_get_parent(struct clk_hw *hw)
 {
 	struct mtk_clk_cpumux *mux = to_mtk_clk_cpumux(hw);
-	int num_parents = clk_hw_get_num_parents(hw);
 	unsigned int val;
 
 	regmap_read(mux->regmap, mux->reg, &val);
@@ -35,9 +34,6 @@ static u8 clk_cpumux_get_parent(struct clk_hw *hw)
 	val >>= mux->shift;
 	val &= mux->mask;
 
-	if (val >= num_parents)
-		return -EINVAL;
-
 	return val;
 }
 
@@ -98,7 +94,7 @@ int __init mtk_clk_register_cpumuxes(struct device_node *node,
 
 	regmap = syscon_node_to_regmap(node);
 	if (IS_ERR(regmap)) {
-		pr_err("Cannot find regmap for %s: %ld\n", node->full_name,
+		pr_err("Cannot find regmap for %pOF: %ld\n", node,
 		       PTR_ERR(regmap));
 		return PTR_ERR(regmap);
 	}
diff --git a/drivers/clk/mediatek/clk-mtk.c b/drivers/clk/mediatek/clk-mtk.c
index 0541df78141c..9c0ae4278a94 100644
--- a/drivers/clk/mediatek/clk-mtk.c
+++ b/drivers/clk/mediatek/clk-mtk.c
@@ -114,7 +114,7 @@ int mtk_clk_register_gates(struct device_node *node,
 
 	regmap = syscon_node_to_regmap(node);
 	if (IS_ERR(regmap)) {
-		pr_err("Cannot find regmap for %s: %ld\n", node->full_name,
+		pr_err("Cannot find regmap for %pOF: %ld\n", node,
 				PTR_ERR(regmap));
 		return PTR_ERR(regmap);
 	}
diff --git a/drivers/clk/mediatek/reset.c b/drivers/clk/mediatek/reset.c
index 309049d41f1b..d3551d5efef2 100644
--- a/drivers/clk/mediatek/reset.c
+++ b/drivers/clk/mediatek/reset.c
@@ -72,7 +72,7 @@ void mtk_register_reset_controller(struct device_node *np,
 
 	regmap = syscon_node_to_regmap(np);
 	if (IS_ERR(regmap)) {
-		pr_err("Cannot find regmap for %s: %ld\n", np->full_name,
+		pr_err("Cannot find regmap for %pOF: %ld\n", np,
 				PTR_ERR(regmap));
 		return;
 	}
diff --git a/drivers/clk/meson/Kconfig b/drivers/clk/meson/Kconfig
index 5588f75a8414..d2d0174a6eca 100644
--- a/drivers/clk/meson/Kconfig
+++ b/drivers/clk/meson/Kconfig
@@ -6,6 +6,7 @@ config COMMON_CLK_AMLOGIC
 config COMMON_CLK_MESON8B
 	bool
 	depends on COMMON_CLK_AMLOGIC
+	select RESET_CONTROLLER
 	help
 	  Support for the clock controller on AmLogic S802 (Meson8),
 	  S805 (Meson8b) and S812 (Meson8m2) devices. Say Y if you
diff --git a/drivers/clk/meson/Makefile b/drivers/clk/meson/Makefile
index 83b6d9d65aa1..b139d41b25da 100644
--- a/drivers/clk/meson/Makefile
+++ b/drivers/clk/meson/Makefile
@@ -4,4 +4,4 @@
 
 obj-$(CONFIG_COMMON_CLK_AMLOGIC) += clk-pll.o clk-cpu.o clk-mpll.o clk-audio-divider.o
 obj-$(CONFIG_COMMON_CLK_MESON8B) += meson8b.o
-obj-$(CONFIG_COMMON_CLK_GXBB)	 += gxbb.o gxbb-aoclk.o
+obj-$(CONFIG_COMMON_CLK_GXBB)	 += gxbb.o gxbb-aoclk.o gxbb-aoclk-regmap.o gxbb-aoclk-32k.o
diff --git a/drivers/clk/meson/gxbb-aoclk-32k.c b/drivers/clk/meson/gxbb-aoclk-32k.c
new file mode 100644
index 000000000000..491634dbc985
--- /dev/null
+++ b/drivers/clk/meson/gxbb-aoclk-32k.c
@@ -0,0 +1,194 @@
+/*
+ * Copyright (c) 2017 BayLibre, SAS.
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/bitfield.h>
+#include <linux/regmap.h>
+#include "gxbb-aoclk.h"
+
+/*
+ * The AO Domain embeds a dual/divider to generate a more precise
+ * 32,768KHz clock for low-power suspend mode and CEC.
+ *                      ______   ______
+ *                     |      | |      |
+ *         ______      | Div1 |-| Cnt1 |       ______
+ *        |      |    /|______| |______|\     |      |
+ * Xtal-->| Gate |---|  ______   ______  X-X--| Gate |-->
+ *        |______| |  \|      | |      |/  |  |______|
+ *                 |   | Div2 |-| Cnt2 |   |
+ *                 |   |______| |______|   |
+ *                 |_______________________|
+ *
+ * The dividing can be switched to single or dual, with a counter
+ * for each divider to set when the switching is done.
+ * The entire dividing mechanism can be also bypassed.
+ */
+
+#define CLK_CNTL0_N1_MASK	GENMASK(11, 0)
+#define CLK_CNTL0_N2_MASK	GENMASK(23, 12)
+#define CLK_CNTL0_DUALDIV_EN	BIT(28)
+#define CLK_CNTL0_OUT_GATE_EN	BIT(30)
+#define CLK_CNTL0_IN_GATE_EN	BIT(31)
+
+#define CLK_CNTL1_M1_MASK	GENMASK(11, 0)
+#define CLK_CNTL1_M2_MASK	GENMASK(23, 12)
+#define CLK_CNTL1_BYPASS_EN	BIT(24)
+#define CLK_CNTL1_SELECT_OSC	BIT(27)
+
+#define PWR_CNTL_ALT_32K_SEL	GENMASK(13, 10)
+
+struct cec_32k_freq_table {
+	unsigned long parent_rate;
+	unsigned long target_rate;
+	bool dualdiv;
+	unsigned int n1;
+	unsigned int n2;
+	unsigned int m1;
+	unsigned int m2;
+};
+
+static const struct cec_32k_freq_table aoclk_cec_32k_table[] = {
+	[0] = {
+		.parent_rate = 24000000,
+		.target_rate = 32768,
+		.dualdiv = true,
+		.n1 = 733,
+		.n2 = 732,
+		.m1 = 8,
+		.m2 = 11,
+	},
+};
+
+/*
+ * If CLK_CNTL0_DUALDIV_EN == 0
+ *  - will use N1 divider only
+ * If CLK_CNTL0_DUALDIV_EN == 1
+ *  - hold M1 cycles of N1 divider then changes to N2
+ *  - hold M2 cycles of N2 divider then changes to N1
+ * Then we can get more accurate division.
+ */
+static unsigned long aoclk_cec_32k_recalc_rate(struct clk_hw *hw,
+					       unsigned long parent_rate)
+{
+	struct aoclk_cec_32k *cec_32k = to_aoclk_cec_32k(hw);
+	unsigned long n1;
+	u32 reg0, reg1;
+
+	regmap_read(cec_32k->regmap, AO_RTC_ALT_CLK_CNTL0, &reg0);
+	regmap_read(cec_32k->regmap, AO_RTC_ALT_CLK_CNTL1, &reg1);
+
+	if (reg1 & CLK_CNTL1_BYPASS_EN)
+		return parent_rate;
+
+	if (reg0 & CLK_CNTL0_DUALDIV_EN) {
+		unsigned long n2, m1, m2, f1, f2, p1, p2;
+
+		n1 = FIELD_GET(CLK_CNTL0_N1_MASK, reg0) + 1;
+		n2 = FIELD_GET(CLK_CNTL0_N2_MASK, reg0) + 1;
+
+		m1 = FIELD_GET(CLK_CNTL1_M1_MASK, reg1) + 1;
+		m2 = FIELD_GET(CLK_CNTL1_M2_MASK, reg1) + 1;
+
+		f1 = DIV_ROUND_CLOSEST(parent_rate, n1);
+		f2 = DIV_ROUND_CLOSEST(parent_rate, n2);
+
+		p1 = DIV_ROUND_CLOSEST(100000000 * m1, f1 * (m1 + m2));
+		p2 = DIV_ROUND_CLOSEST(100000000 * m2, f2 * (m1 + m2));
+
+		return DIV_ROUND_UP(100000000, p1 + p2);
+	}
+
+	n1 = FIELD_GET(CLK_CNTL0_N1_MASK, reg0) + 1;
+
+	return DIV_ROUND_CLOSEST(parent_rate, n1);
+}
+
+static const struct cec_32k_freq_table *find_cec_32k_freq(unsigned long rate,
+							  unsigned long prate)
+{
+	int i;
+
+	for (i = 0 ; i < ARRAY_SIZE(aoclk_cec_32k_table) ; ++i)
+		if (aoclk_cec_32k_table[i].parent_rate == prate &&
+		    aoclk_cec_32k_table[i].target_rate == rate)
+			return &aoclk_cec_32k_table[i];
+
+	return NULL;
+}
+
+static long aoclk_cec_32k_round_rate(struct clk_hw *hw, unsigned long rate,
+				     unsigned long *prate)
+{
+	const struct cec_32k_freq_table *freq = find_cec_32k_freq(rate,
+								  *prate);
+
+	/* If invalid return first one */
+	if (!freq)
+		return aoclk_cec_32k_table[0].target_rate;
+
+	return freq->target_rate;
+}
+
+/*
+ * From the Amlogic init procedure, the IN and OUT gates needs to be handled
+ * in the init procedure to avoid any glitches.
+ */
+
+static int aoclk_cec_32k_set_rate(struct clk_hw *hw, unsigned long rate,
+				  unsigned long parent_rate)
+{
+	const struct cec_32k_freq_table *freq = find_cec_32k_freq(rate,
+								  parent_rate);
+	struct aoclk_cec_32k *cec_32k = to_aoclk_cec_32k(hw);
+	u32 reg = 0;
+
+	if (!freq)
+		return -EINVAL;
+
+	/* Disable clock */
+	regmap_update_bits(cec_32k->regmap, AO_RTC_ALT_CLK_CNTL0,
+			   CLK_CNTL0_IN_GATE_EN | CLK_CNTL0_OUT_GATE_EN, 0);
+
+	reg = FIELD_PREP(CLK_CNTL0_N1_MASK, freq->n1 - 1);
+	if (freq->dualdiv)
+		reg |= CLK_CNTL0_DUALDIV_EN |
+		       FIELD_PREP(CLK_CNTL0_N2_MASK, freq->n2 - 1);
+
+	regmap_write(cec_32k->regmap, AO_RTC_ALT_CLK_CNTL0, reg);
+
+	reg = FIELD_PREP(CLK_CNTL1_M1_MASK, freq->m1 - 1);
+	if (freq->dualdiv)
+		reg |= FIELD_PREP(CLK_CNTL1_M2_MASK, freq->m2 - 1);
+
+	regmap_write(cec_32k->regmap, AO_RTC_ALT_CLK_CNTL1, reg);
+
+	/* Enable clock */
+	regmap_update_bits(cec_32k->regmap, AO_RTC_ALT_CLK_CNTL0,
+			   CLK_CNTL0_IN_GATE_EN, CLK_CNTL0_IN_GATE_EN);
+
+	udelay(200);
+
+	regmap_update_bits(cec_32k->regmap, AO_RTC_ALT_CLK_CNTL0,
+			   CLK_CNTL0_OUT_GATE_EN, CLK_CNTL0_OUT_GATE_EN);
+
+	regmap_update_bits(cec_32k->regmap, AO_CRT_CLK_CNTL1,
+			   CLK_CNTL1_SELECT_OSC, CLK_CNTL1_SELECT_OSC);
+
+	/* Select 32k from XTAL */
+	regmap_update_bits(cec_32k->regmap,
+			  AO_RTI_PWR_CNTL_REG0,
+			  PWR_CNTL_ALT_32K_SEL,
+			  FIELD_PREP(PWR_CNTL_ALT_32K_SEL, 4));
+
+	return 0;
+}
+
+const struct clk_ops meson_aoclk_cec_32k_ops = {
+	.recalc_rate = aoclk_cec_32k_recalc_rate,
+	.round_rate = aoclk_cec_32k_round_rate,
+	.set_rate = aoclk_cec_32k_set_rate,
+};
diff --git a/drivers/clk/meson/gxbb-aoclk-regmap.c b/drivers/clk/meson/gxbb-aoclk-regmap.c
new file mode 100644
index 000000000000..2515fbfa0467
--- /dev/null
+++ b/drivers/clk/meson/gxbb-aoclk-regmap.c
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2017 BayLibre, SAS.
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/bitfield.h>
+#include <linux/regmap.h>
+#include "gxbb-aoclk.h"
+
+static int aoclk_gate_regmap_enable(struct clk_hw *hw)
+{
+	struct aoclk_gate_regmap *gate = to_aoclk_gate_regmap(hw);
+
+	return regmap_update_bits(gate->regmap, AO_RTI_GEN_CNTL_REG0,
+				  BIT(gate->bit_idx), BIT(gate->bit_idx));
+}
+
+static void aoclk_gate_regmap_disable(struct clk_hw *hw)
+{
+	struct aoclk_gate_regmap *gate = to_aoclk_gate_regmap(hw);
+
+	regmap_update_bits(gate->regmap, AO_RTI_GEN_CNTL_REG0,
+			   BIT(gate->bit_idx), 0);
+}
+
+static int aoclk_gate_regmap_is_enabled(struct clk_hw *hw)
+{
+	struct aoclk_gate_regmap *gate = to_aoclk_gate_regmap(hw);
+	unsigned int val;
+	int ret;
+
+	ret = regmap_read(gate->regmap, AO_RTI_GEN_CNTL_REG0, &val);
+	if (ret)
+		return ret;
+
+	return (val & BIT(gate->bit_idx)) != 0;
+}
+
+const struct clk_ops meson_aoclk_gate_regmap_ops = {
+	.enable = aoclk_gate_regmap_enable,
+	.disable = aoclk_gate_regmap_disable,
+	.is_enabled = aoclk_gate_regmap_is_enabled,
+};
diff --git a/drivers/clk/meson/gxbb-aoclk.c b/drivers/clk/meson/gxbb-aoclk.c
index b45c5fba7e35..6c161e0a8e59 100644
--- a/drivers/clk/meson/gxbb-aoclk.c
+++ b/drivers/clk/meson/gxbb-aoclk.c
@@ -56,16 +56,20 @@
 #include <linux/of_address.h>
 #include <linux/platform_device.h>
 #include <linux/reset-controller.h>
+#include <linux/mfd/syscon.h>
+#include <linux/regmap.h>
 #include <linux/init.h>
+#include <linux/delay.h>
 #include <dt-bindings/clock/gxbb-aoclkc.h>
 #include <dt-bindings/reset/gxbb-aoclkc.h>
+#include "gxbb-aoclk.h"
 
 static DEFINE_SPINLOCK(gxbb_aoclk_lock);
 
 struct gxbb_aoclk_reset_controller {
 	struct reset_controller_dev reset;
 	unsigned int *data;
-	void __iomem *base;
+	struct regmap *regmap;
 };
 
 static int gxbb_aoclk_do_reset(struct reset_controller_dev *rcdev,
@@ -74,9 +78,8 @@ static int gxbb_aoclk_do_reset(struct reset_controller_dev *rcdev,
 	struct gxbb_aoclk_reset_controller *reset =
 		container_of(rcdev, struct gxbb_aoclk_reset_controller, reset);
 
-	writel(BIT(reset->data[id]), reset->base);
-
-	return 0;
+	return regmap_write(reset->regmap, AO_RTI_GEN_CNTL_REG0,
+			    BIT(reset->data[id]));
 }
 
 static const struct reset_control_ops gxbb_aoclk_reset_ops = {
@@ -84,13 +87,12 @@ static const struct reset_control_ops gxbb_aoclk_reset_ops = {
 };
 
 #define GXBB_AO_GATE(_name, _bit)					\
-static struct clk_gate _name##_ao = {					\
-	.reg = (void __iomem *)0,					\
+static struct aoclk_gate_regmap _name##_ao = {				\
 	.bit_idx = (_bit),						\
 	.lock = &gxbb_aoclk_lock,					\
 	.hw.init = &(struct clk_init_data) {				\
 		.name = #_name "_ao",					\
-		.ops = &clk_gate_ops,					\
+		.ops = &meson_aoclk_gate_regmap_ops,			\
 		.parent_names = (const char *[]){ "clk81" },		\
 		.num_parents = 1,					\
 		.flags = (CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED),	\
@@ -104,6 +106,17 @@ GXBB_AO_GATE(uart1, 3);
 GXBB_AO_GATE(uart2, 5);
 GXBB_AO_GATE(ir_blaster, 6);
 
+static struct aoclk_cec_32k cec_32k_ao = {
+	.lock = &gxbb_aoclk_lock,
+	.hw.init = &(struct clk_init_data) {
+		.name = "cec_32k_ao",
+		.ops = &meson_aoclk_cec_32k_ops,
+		.parent_names = (const char *[]){ "xtal" },
+		.num_parents = 1,
+		.flags = CLK_IGNORE_UNUSED,
+	},
+};
+
 static unsigned int gxbb_aoclk_reset[] = {
 	[RESET_AO_REMOTE] = 16,
 	[RESET_AO_I2C_MASTER] = 18,
@@ -113,7 +126,7 @@ static unsigned int gxbb_aoclk_reset[] = {
 	[RESET_AO_IR_BLASTER] = 23,
 };
 
-static struct clk_gate *gxbb_aoclk_gate[] = {
+static struct aoclk_gate_regmap *gxbb_aoclk_gate[] = {
 	[CLKID_AO_REMOTE] = &remote_ao,
 	[CLKID_AO_I2C_MASTER] = &i2c_master_ao,
 	[CLKID_AO_I2C_SLAVE] = &i2c_slave_ao,
@@ -130,30 +143,30 @@ static struct clk_hw_onecell_data gxbb_aoclk_onecell_data = {
 		[CLKID_AO_UART1] = &uart1_ao.hw,
 		[CLKID_AO_UART2] = &uart2_ao.hw,
 		[CLKID_AO_IR_BLASTER] = &ir_blaster_ao.hw,
+		[CLKID_AO_CEC_32K] = &cec_32k_ao.hw,
 	},
-	.num = ARRAY_SIZE(gxbb_aoclk_gate),
+	.num = 7,
 };
 
 static int gxbb_aoclkc_probe(struct platform_device *pdev)
 {
-	struct resource *res;
-	void __iomem *base;
-	int ret, clkid;
-	struct device *dev = &pdev->dev;
 	struct gxbb_aoclk_reset_controller *rstc;
+	struct device *dev = &pdev->dev;
+	struct regmap *regmap;
+	int ret, clkid;
 
 	rstc = devm_kzalloc(dev, sizeof(*rstc), GFP_KERNEL);
 	if (!rstc)
 		return -ENOMEM;
 
-	/* Generic clocks */
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	base = devm_ioremap_resource(dev, res);
-	if (IS_ERR(base))
-		return PTR_ERR(base);
+	regmap = syscon_node_to_regmap(of_get_parent(dev->of_node));
+	if (IS_ERR(regmap)) {
+		dev_err(dev, "failed to get regmap\n");
+		return -ENODEV;
+	}
 
 	/* Reset Controller */
-	rstc->base = base;
+	rstc->regmap = regmap;
 	rstc->data = gxbb_aoclk_reset;
 	rstc->reset.ops = &gxbb_aoclk_reset_ops;
 	rstc->reset.nr_resets = ARRAY_SIZE(gxbb_aoclk_reset);
@@ -161,10 +174,10 @@ static int gxbb_aoclkc_probe(struct platform_device *pdev)
 	ret = devm_reset_controller_register(dev, &rstc->reset);
 
 	/*
-	 * Populate base address and register all clks
+	 * Populate regmap and register all clks
 	 */
-	for (clkid = 0; clkid < gxbb_aoclk_onecell_data.num; clkid++) {
-		gxbb_aoclk_gate[clkid]->reg = base;
+	for (clkid = 0; clkid < ARRAY_SIZE(gxbb_aoclk_gate); clkid++) {
+		gxbb_aoclk_gate[clkid]->regmap = regmap;
 
 		ret = devm_clk_hw_register(dev,
 					gxbb_aoclk_onecell_data.hws[clkid]);
@@ -172,12 +185,18 @@ static int gxbb_aoclkc_probe(struct platform_device *pdev)
 			return ret;
 	}
 
+	/* Specific clocks */
+	cec_32k_ao.regmap = regmap;
+	ret = devm_clk_hw_register(dev, &cec_32k_ao.hw);
+	if (ret)
+		return ret;
+
 	return of_clk_add_hw_provider(dev->of_node, of_clk_hw_onecell_get,
 			&gxbb_aoclk_onecell_data);
 }
 
 static const struct of_device_id gxbb_aoclkc_match_table[] = {
-	{ .compatible = "amlogic,gxbb-aoclkc" },
+	{ .compatible = "amlogic,meson-gx-aoclkc" },
 	{ }
 };
 
diff --git a/drivers/clk/meson/gxbb-aoclk.h b/drivers/clk/meson/gxbb-aoclk.h
new file mode 100644
index 000000000000..e8604c8f7eee
--- /dev/null
+++ b/drivers/clk/meson/gxbb-aoclk.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2017 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ */
+
+#ifndef __GXBB_AOCLKC_H
+#define __GXBB_AOCLKC_H
+
+/* AO Configuration Clock registers offsets */
+#define AO_RTI_PWR_CNTL_REG1	0x0c
+#define AO_RTI_PWR_CNTL_REG0	0x10
+#define AO_RTI_GEN_CNTL_REG0	0x40
+#define AO_OSCIN_CNTL		0x58
+#define AO_CRT_CLK_CNTL1	0x68
+#define AO_RTC_ALT_CLK_CNTL0	0x94
+#define AO_RTC_ALT_CLK_CNTL1	0x98
+
+struct aoclk_gate_regmap {
+	struct clk_hw hw;
+	unsigned bit_idx;
+	struct regmap *regmap;
+	spinlock_t *lock;
+};
+
+#define to_aoclk_gate_regmap(_hw) \
+	container_of(_hw, struct aoclk_gate_regmap, hw)
+
+extern const struct clk_ops meson_aoclk_gate_regmap_ops;
+
+struct aoclk_cec_32k {
+	struct clk_hw hw;
+	struct regmap *regmap;
+	spinlock_t *lock;
+};
+
+#define to_aoclk_cec_32k(_hw) container_of(_hw, struct aoclk_cec_32k, hw)
+
+extern const struct clk_ops meson_aoclk_cec_32k_ops;
+
+#endif /* __GXBB_AOCLKC_H */
diff --git a/drivers/clk/meson/gxbb.c b/drivers/clk/meson/gxbb.c
index a7ea5f3da89d..b2d1e8ed7152 100644
--- a/drivers/clk/meson/gxbb.c
+++ b/drivers/clk/meson/gxbb.c
@@ -850,13 +850,14 @@ static struct meson_clk_audio_divider gxbb_cts_amclk_div = {
 		.shift   = 0,
 		.width   = 8,
 	},
+	.flags = CLK_DIVIDER_ROUND_CLOSEST,
 	.lock = &clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "cts_amclk_div",
 		.ops = &meson_clk_audio_divider_ops,
 		.parent_names = (const char *[]){ "cts_amclk_sel" },
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_PARENT | CLK_DIVIDER_ROUND_CLOSEST,
+		.flags = CLK_SET_RATE_PARENT,
 	},
 };
 
@@ -880,7 +881,7 @@ static struct clk_mux gxbb_cts_mclk_i958_sel = {
 	/* Default parent unknown (register reset value: 0) */
 	.table = (u32[]){ 1, 2, 3 },
 	.lock = &clk_lock,
-		.hw.init = &(struct clk_init_data){
+	.hw.init = &(struct clk_init_data) {
 		.name = "cts_mclk_i958_sel",
 		.ops = &clk_mux_ops,
 		.parent_names = (const char *[]){ "mpll0", "mpll1", "mpll2" },
@@ -894,12 +895,13 @@ static struct clk_divider gxbb_cts_mclk_i958_div = {
 	.shift = 16,
 	.width = 8,
 	.lock = &clk_lock,
-	.hw.init = &(struct clk_init_data){
+	.flags = CLK_DIVIDER_ROUND_CLOSEST,
+	.hw.init = &(struct clk_init_data) {
 		.name = "cts_mclk_i958_div",
 		.ops = &clk_divider_ops,
 		.parent_names = (const char *[]){ "cts_mclk_i958_sel" },
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_PARENT | CLK_DIVIDER_ROUND_CLOSEST,
+		.flags = CLK_SET_RATE_PARENT,
 	},
 };
 
@@ -979,6 +981,156 @@ static struct clk_mux gxbb_32k_clk_sel = {
 	},
 };
 
+static const char * const gxbb_sd_emmc_clk0_parent_names[] = {
+	"xtal", "fclk_div2", "fclk_div3", "fclk_div5", "fclk_div7",
+
+	/*
+	 * Following these parent clocks, we should also have had mpll2, mpll3
+	 * and gp0_pll but these clocks are too precious to be used here. All
+	 * the necessary rates for MMC and NAND operation can be acheived using
+	 * xtal or fclk_div clocks
+	 */
+};
+
+/* SDIO clock */
+static struct clk_mux gxbb_sd_emmc_a_clk0_sel = {
+	.reg = (void *)HHI_SD_EMMC_CLK_CNTL,
+	.mask = 0x7,
+	.shift = 9,
+	.lock = &clk_lock,
+	.hw.init = &(struct clk_init_data) {
+		.name = "sd_emmc_a_clk0_sel",
+		.ops = &clk_mux_ops,
+		.parent_names = gxbb_sd_emmc_clk0_parent_names,
+		.num_parents = ARRAY_SIZE(gxbb_sd_emmc_clk0_parent_names),
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_divider gxbb_sd_emmc_a_clk0_div = {
+	.reg = (void *)HHI_SD_EMMC_CLK_CNTL,
+	.shift = 0,
+	.width = 7,
+	.lock = &clk_lock,
+	.flags = CLK_DIVIDER_ROUND_CLOSEST,
+	.hw.init = &(struct clk_init_data) {
+		.name = "sd_emmc_a_clk0_div",
+		.ops = &clk_divider_ops,
+		.parent_names = (const char *[]){ "sd_emmc_a_clk0_sel" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_gate gxbb_sd_emmc_a_clk0 = {
+	.reg = (void *)HHI_SD_EMMC_CLK_CNTL,
+	.bit_idx = 7,
+	.lock = &clk_lock,
+	.hw.init = &(struct clk_init_data){
+		.name = "sd_emmc_a_clk0",
+		.ops = &clk_gate_ops,
+		.parent_names = (const char *[]){ "sd_emmc_a_clk0_div" },
+		.num_parents = 1,
+
+		/*
+		 * FIXME:
+		 * We need CLK_IGNORE_UNUSED because mmc DT node point to xtal
+		 * instead of this clock. CCF would gate this on boot, killing
+		 * the mmc controller. Please remove this flag once DT properly
+		 * point to this clock instead of xtal
+		 *
+		 * Same goes for emmc B and C clocks
+		 */
+		.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
+	},
+};
+
+/* SDcard clock */
+static struct clk_mux gxbb_sd_emmc_b_clk0_sel = {
+	.reg = (void *)HHI_SD_EMMC_CLK_CNTL,
+	.mask = 0x7,
+	.shift = 25,
+	.lock = &clk_lock,
+	.hw.init = &(struct clk_init_data) {
+		.name = "sd_emmc_b_clk0_sel",
+		.ops = &clk_mux_ops,
+		.parent_names = gxbb_sd_emmc_clk0_parent_names,
+		.num_parents = ARRAY_SIZE(gxbb_sd_emmc_clk0_parent_names),
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_divider gxbb_sd_emmc_b_clk0_div = {
+	.reg = (void *)HHI_SD_EMMC_CLK_CNTL,
+	.shift = 16,
+	.width = 7,
+	.lock = &clk_lock,
+	.flags = CLK_DIVIDER_ROUND_CLOSEST,
+	.hw.init = &(struct clk_init_data) {
+		.name = "sd_emmc_b_clk0_div",
+		.ops = &clk_divider_ops,
+		.parent_names = (const char *[]){ "sd_emmc_b_clk0_sel" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_gate gxbb_sd_emmc_b_clk0 = {
+	.reg = (void *)HHI_SD_EMMC_CLK_CNTL,
+	.bit_idx = 23,
+	.lock = &clk_lock,
+	.hw.init = &(struct clk_init_data){
+		.name = "sd_emmc_b_clk0",
+		.ops = &clk_gate_ops,
+		.parent_names = (const char *[]){ "sd_emmc_b_clk0_div" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
+	},
+};
+
+/* EMMC/NAND clock */
+static struct clk_mux gxbb_sd_emmc_c_clk0_sel = {
+	.reg = (void *)HHI_NAND_CLK_CNTL,
+	.mask = 0x7,
+	.shift = 9,
+	.lock = &clk_lock,
+	.hw.init = &(struct clk_init_data) {
+		.name = "sd_emmc_c_clk0_sel",
+		.ops = &clk_mux_ops,
+		.parent_names = gxbb_sd_emmc_clk0_parent_names,
+		.num_parents = ARRAY_SIZE(gxbb_sd_emmc_clk0_parent_names),
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_divider gxbb_sd_emmc_c_clk0_div = {
+	.reg = (void *)HHI_NAND_CLK_CNTL,
+	.shift = 0,
+	.width = 7,
+	.lock = &clk_lock,
+	.flags = CLK_DIVIDER_ROUND_CLOSEST,
+	.hw.init = &(struct clk_init_data) {
+		.name = "sd_emmc_c_clk0_div",
+		.ops = &clk_divider_ops,
+		.parent_names = (const char *[]){ "sd_emmc_c_clk0_sel" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_gate gxbb_sd_emmc_c_clk0 = {
+	.reg = (void *)HHI_NAND_CLK_CNTL,
+	.bit_idx = 7,
+	.lock = &clk_lock,
+	.hw.init = &(struct clk_init_data){
+		.name = "sd_emmc_c_clk0",
+		.ops = &clk_gate_ops,
+		.parent_names = (const char *[]){ "sd_emmc_c_clk0_div" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
+	},
+};
+
 /* Everything Else (EE) domain gates */
 static MESON_GATE(gxbb_ddr, HHI_GCLK_MPEG0, 0);
 static MESON_GATE(gxbb_dos, HHI_GCLK_MPEG0, 1);
@@ -1188,6 +1340,16 @@ static struct clk_hw_onecell_data gxbb_hw_onecell_data = {
 		[CLKID_32K_CLK]		    = &gxbb_32k_clk.hw,
 		[CLKID_32K_CLK_SEL]	    = &gxbb_32k_clk_sel.hw,
 		[CLKID_32K_CLK_DIV]	    = &gxbb_32k_clk_div.hw,
+		[CLKID_SD_EMMC_A_CLK0_SEL]  = &gxbb_sd_emmc_a_clk0_sel.hw,
+		[CLKID_SD_EMMC_A_CLK0_DIV]  = &gxbb_sd_emmc_a_clk0_div.hw,
+		[CLKID_SD_EMMC_A_CLK0]	    = &gxbb_sd_emmc_a_clk0.hw,
+		[CLKID_SD_EMMC_B_CLK0_SEL]  = &gxbb_sd_emmc_b_clk0_sel.hw,
+		[CLKID_SD_EMMC_B_CLK0_DIV]  = &gxbb_sd_emmc_b_clk0_div.hw,
+		[CLKID_SD_EMMC_B_CLK0]	    = &gxbb_sd_emmc_b_clk0.hw,
+		[CLKID_SD_EMMC_C_CLK0_SEL]  = &gxbb_sd_emmc_c_clk0_sel.hw,
+		[CLKID_SD_EMMC_C_CLK0_DIV]  = &gxbb_sd_emmc_c_clk0_div.hw,
+		[CLKID_SD_EMMC_C_CLK0]	    = &gxbb_sd_emmc_c_clk0.hw,
+		[NR_CLKS]		    = NULL,
 	},
 	.num = NR_CLKS,
 };
@@ -1310,6 +1472,16 @@ static struct clk_hw_onecell_data gxl_hw_onecell_data = {
 		[CLKID_32K_CLK]		    = &gxbb_32k_clk.hw,
 		[CLKID_32K_CLK_SEL]	    = &gxbb_32k_clk_sel.hw,
 		[CLKID_32K_CLK_DIV]	    = &gxbb_32k_clk_div.hw,
+		[CLKID_SD_EMMC_A_CLK0_SEL]  = &gxbb_sd_emmc_a_clk0_sel.hw,
+		[CLKID_SD_EMMC_A_CLK0_DIV]  = &gxbb_sd_emmc_a_clk0_div.hw,
+		[CLKID_SD_EMMC_A_CLK0]	    = &gxbb_sd_emmc_a_clk0.hw,
+		[CLKID_SD_EMMC_B_CLK0_SEL]  = &gxbb_sd_emmc_b_clk0_sel.hw,
+		[CLKID_SD_EMMC_B_CLK0_DIV]  = &gxbb_sd_emmc_b_clk0_div.hw,
+		[CLKID_SD_EMMC_B_CLK0]	    = &gxbb_sd_emmc_b_clk0.hw,
+		[CLKID_SD_EMMC_C_CLK0_SEL]  = &gxbb_sd_emmc_c_clk0_sel.hw,
+		[CLKID_SD_EMMC_C_CLK0_DIV]  = &gxbb_sd_emmc_c_clk0_div.hw,
+		[CLKID_SD_EMMC_C_CLK0]	    = &gxbb_sd_emmc_c_clk0.hw,
+		[NR_CLKS]		    = NULL,
 	},
 	.num = NR_CLKS,
 };
@@ -1425,6 +1597,9 @@ static struct clk_gate *const gxbb_clk_gates[] = {
 	&gxbb_cts_amclk,
 	&gxbb_cts_mclk_i958,
 	&gxbb_32k_clk,
+	&gxbb_sd_emmc_a_clk0,
+	&gxbb_sd_emmc_b_clk0,
+	&gxbb_sd_emmc_c_clk0,
 };
 
 static struct clk_mux *const gxbb_clk_muxes[] = {
@@ -1437,6 +1612,9 @@ static struct clk_mux *const gxbb_clk_muxes[] = {
 	&gxbb_cts_mclk_i958_sel,
 	&gxbb_cts_i958,
 	&gxbb_32k_clk_sel,
+	&gxbb_sd_emmc_a_clk0_sel,
+	&gxbb_sd_emmc_b_clk0_sel,
+	&gxbb_sd_emmc_c_clk0_sel,
 };
 
 static struct clk_divider *const gxbb_clk_dividers[] = {
@@ -1446,6 +1624,9 @@ static struct clk_divider *const gxbb_clk_dividers[] = {
 	&gxbb_mali_1_div,
 	&gxbb_cts_mclk_i958_div,
 	&gxbb_32k_clk_div,
+	&gxbb_sd_emmc_a_clk0_div,
+	&gxbb_sd_emmc_b_clk0_div,
+	&gxbb_sd_emmc_c_clk0_div,
 };
 
 static struct meson_clk_audio_divider *const gxbb_audio_dividers[] = {
diff --git a/drivers/clk/meson/gxbb.h b/drivers/clk/meson/gxbb.h
index d63e77e8433d..5b1d4b374d1c 100644
--- a/drivers/clk/meson/gxbb.h
+++ b/drivers/clk/meson/gxbb.h
@@ -167,130 +167,33 @@
  * CLKID index values
  *
  * These indices are entirely contrived and do not map onto the hardware.
- * Migrate them out of this header and into the DT header file when they need
- * to be exposed to client nodes in DT: include/dt-bindings/clock/gxbb-clkc.h
+ * It has now been decided to expose everything by default in the DT header:
+ * include/dt-bindings/clock/gxbb-clkc.h. Only the clocks ids we don't want
+ * to expose, such as the internal muxes and dividers of composite clocks,
+ * will remain defined here.
  */
-#define CLKID_SYS_PLL		  0
 /* ID 1 is unused (it was used by the non-existing CLKID_CPUCLK before) */
-/* CLKID_HDMI_PLL */
-#define CLKID_FIXED_PLL		  3
-/* CLKID_FCLK_DIV2 */
-/* CLKID_FCLK_DIV3 */
-/* CLKID_FCLK_DIV4 */
-#define CLKID_FCLK_DIV5		  7
-#define CLKID_FCLK_DIV7		  8
-/* CLKID_GP0_PLL */
 #define CLKID_MPEG_SEL		  10
 #define CLKID_MPEG_DIV		  11
-/* CLKID_CLK81 */
-#define CLKID_MPLL0		  13
-#define CLKID_MPLL1		  14
-/* CLKID_MPLL2 */
-#define CLKID_DDR		  16
-#define CLKID_DOS		  17
-#define CLKID_ISA		  18
-#define CLKID_PL301		  19
-#define CLKID_PERIPHS		  20
-/* CLKID_SPICC */
-/* CLKID_I2C */
-/* #define CLKID_SAR_ADC */
-#define CLKID_SMART_CARD	  24
-/* CLKID_RNG0 */
-/* CLKID_UART0 */
-#define CLKID_SDHC		  27
-#define CLKID_STREAM		  28
-#define CLKID_ASYNC_FIFO	  29
-#define CLKID_SDIO		  30
-#define CLKID_ABUF		  31
-#define CLKID_HIU_IFACE		  32
-#define CLKID_ASSIST_MISC	  33
-/* CLKID_SPI */
-#define CLKID_I2S_SPDIF		  35
-/* CLKID_ETH */
-#define CLKID_DEMUX		  37
-/* CLKID_AIU_GLUE */
-/* CLKID_IEC958 */
-/* CLKID_I2S_OUT */
-#define CLKID_AMCLK		  41
-#define CLKID_AIFIFO2		  42
-#define CLKID_MIXER		  43
-/* CLKID_MIXER_IFACE */
-#define CLKID_ADC		  45
-#define CLKID_BLKMV		  46
-/* CLKID_AIU */
-/* CLKID_UART1 */
-#define CLKID_G2D		  49
-/* CLKID_USB0 */
-/* CLKID_USB1 */
-#define CLKID_RESET		  52
-#define CLKID_NAND		  53
-#define CLKID_DOS_PARSER	  54
-/* CLKID_USB */
-#define CLKID_VDIN1		  56
-#define CLKID_AHB_ARB0		  57
-#define CLKID_EFUSE		  58
-#define CLKID_BOOT_ROM		  59
-#define CLKID_AHB_DATA_BUS	  60
-#define CLKID_AHB_CTRL_BUS	  61
-#define CLKID_HDMI_INTR_SYNC	  62
-/* CLKID_HDMI_PCLK */
-/* CLKID_USB1_DDR_BRIDGE */
-/* CLKID_USB0_DDR_BRIDGE */
-#define CLKID_MMC_PCLK		  66
-#define CLKID_DVIN		  67
-/* CLKID_UART2 */
-/* #define CLKID_SANA */
-#define CLKID_VPU_INTR		  70
-#define CLKID_SEC_AHB_AHB3_BRIDGE 71
-#define CLKID_CLK81_A53		  72
-#define CLKID_VCLK2_VENCI0	  73
-#define CLKID_VCLK2_VENCI1	  74
-#define CLKID_VCLK2_VENCP0	  75
-#define CLKID_VCLK2_VENCP1	  76
-/* CLKID_GCLK_VENCI_INT0 */
-#define CLKID_GCLK_VENCI_INT	  78
-#define CLKID_DAC_CLK		  79
-/* CLKID_AOCLK_GATE */
-/* CLKID_IEC958_GATE */
-#define CLKID_ENC480P		  82
-#define CLKID_RNG1		  83
-#define CLKID_GCLK_VENCI_INT1	  84
-#define CLKID_VCLK2_VENCLMCC	  85
-#define CLKID_VCLK2_VENCL	  86
-#define CLKID_VCLK_OTHER	  87
-#define CLKID_EDP		  88
-#define CLKID_AO_MEDIA_CPU	  89
-#define CLKID_AO_AHB_SRAM	  90
-#define CLKID_AO_AHB_BUS	  91
-#define CLKID_AO_IFACE		  92
-/* CLKID_AO_I2C */
-/* CLKID_SD_EMMC_A */
-/* CLKID_SD_EMMC_B */
-/* CLKID_SD_EMMC_C */
-/* CLKID_SAR_ADC_CLK */
-/* CLKID_SAR_ADC_SEL */
 #define CLKID_SAR_ADC_DIV	  99
-/* CLKID_MALI_0_SEL */
-#define CLKID_MALI_0_DIV	 101
-/* CLKID_MALI_0	*/
-/* CLKID_MALI_1_SEL */
-#define CLKID_MALI_1_DIV	 104
-/* CLKID_MALI_1	*/
-/* CLKID_MALI	*/
-/* CLKID_CTS_AMCLK */
+#define CLKID_MALI_0_DIV	  101
+#define CLKID_MALI_1_DIV	  104
 #define CLKID_CTS_AMCLK_SEL	  108
 #define CLKID_CTS_AMCLK_DIV	  109
-/* CLKID_CTS_MCLK_I958 */
 #define CLKID_CTS_MCLK_I958_SEL	  111
 #define CLKID_CTS_MCLK_I958_DIV	  112
-/* CLKID_CTS_I958 */
-#define CLKID_32K_CLK		  114
 #define CLKID_32K_CLK_SEL	  115
 #define CLKID_32K_CLK_DIV	  116
+#define CLKID_SD_EMMC_A_CLK0_SEL  117
+#define CLKID_SD_EMMC_A_CLK0_DIV  118
+#define CLKID_SD_EMMC_B_CLK0_SEL  120
+#define CLKID_SD_EMMC_B_CLK0_DIV  121
+#define CLKID_SD_EMMC_C_CLK0_SEL  123
+#define CLKID_SD_EMMC_C_CLK0_DIV  124
 
-#define NR_CLKS			  117
+#define NR_CLKS			  126
 
-/* include the CLKIDs that have been made part of the stable DT binding */
+/* include the CLKIDs that have been made part of the DT binding */
 #include <dt-bindings/clock/gxbb-clkc.h>
 
 #endif /* __GXBB_H */
diff --git a/drivers/clk/meson/meson8b.c b/drivers/clk/meson/meson8b.c
index 6ec512ad2598..20ab7190d328 100644
--- a/drivers/clk/meson/meson8b.c
+++ b/drivers/clk/meson/meson8b.c
@@ -25,6 +25,8 @@
 #include <linux/clk-provider.h>
 #include <linux/of_address.h>
 #include <linux/platform_device.h>
+#include <linux/reset-controller.h>
+#include <linux/slab.h>
 #include <linux/init.h>
 
 #include "clkc.h"
@@ -32,6 +34,13 @@
 
 static DEFINE_SPINLOCK(clk_lock);
 
+static void __iomem *clk_base;
+
+struct meson8b_clk_reset {
+	struct reset_controller_dev reset;
+	void __iomem *base;
+};
+
 static const struct pll_rate_table sys_pll_rate_table[] = {
 	PLL_RATE(312000000, 52, 1, 2),
 	PLL_RATE(336000000, 56, 1, 2),
@@ -590,6 +599,7 @@ static struct clk_hw_onecell_data meson8b_hw_onecell_data = {
 		[CLKID_MPLL0]		    = &meson8b_mpll0.hw,
 		[CLKID_MPLL1]		    = &meson8b_mpll1.hw,
 		[CLKID_MPLL2]		    = &meson8b_mpll2.hw,
+		[CLK_NR_CLKS]		    = NULL,
 	},
 	.num = CLK_NR_CLKS,
 };
@@ -695,20 +705,114 @@ static struct clk_divider *const meson8b_clk_dividers[] = {
 	&meson8b_mpeg_clk_div,
 };
 
+static const struct meson8b_clk_reset_line {
+	u32 reg;
+	u8 bit_idx;
+} meson8b_clk_reset_bits[] = {
+	[CLKC_RESET_L2_CACHE_SOFT_RESET] = {
+		.reg = HHI_SYS_CPU_CLK_CNTL0, .bit_idx = 30
+	},
+	[CLKC_RESET_AXI_64_TO_128_BRIDGE_A5_SOFT_RESET] = {
+		.reg = HHI_SYS_CPU_CLK_CNTL0, .bit_idx = 29
+	},
+	[CLKC_RESET_SCU_SOFT_RESET] = {
+		.reg = HHI_SYS_CPU_CLK_CNTL0, .bit_idx = 28
+	},
+	[CLKC_RESET_CPU3_SOFT_RESET] = {
+		.reg = HHI_SYS_CPU_CLK_CNTL0, .bit_idx = 27
+	},
+	[CLKC_RESET_CPU2_SOFT_RESET] = {
+		.reg = HHI_SYS_CPU_CLK_CNTL0, .bit_idx = 26
+	},
+	[CLKC_RESET_CPU1_SOFT_RESET] = {
+		.reg = HHI_SYS_CPU_CLK_CNTL0, .bit_idx = 25
+	},
+	[CLKC_RESET_CPU0_SOFT_RESET] = {
+		.reg = HHI_SYS_CPU_CLK_CNTL0, .bit_idx = 24
+	},
+	[CLKC_RESET_A5_GLOBAL_RESET] = {
+		.reg = HHI_SYS_CPU_CLK_CNTL0, .bit_idx = 18
+	},
+	[CLKC_RESET_A5_AXI_SOFT_RESET] = {
+		.reg = HHI_SYS_CPU_CLK_CNTL0, .bit_idx = 17
+	},
+	[CLKC_RESET_A5_ABP_SOFT_RESET] = {
+		.reg = HHI_SYS_CPU_CLK_CNTL0, .bit_idx = 16
+	},
+	[CLKC_RESET_AXI_64_TO_128_BRIDGE_MMC_SOFT_RESET] = {
+		.reg = HHI_SYS_CPU_CLK_CNTL1, .bit_idx = 30
+	},
+	[CLKC_RESET_VID_CLK_CNTL_SOFT_RESET] = {
+		.reg = HHI_VID_CLK_CNTL, .bit_idx = 15
+	},
+	[CLKC_RESET_VID_DIVIDER_CNTL_SOFT_RESET_POST] = {
+		.reg = HHI_VID_DIVIDER_CNTL, .bit_idx = 7
+	},
+	[CLKC_RESET_VID_DIVIDER_CNTL_SOFT_RESET_PRE] = {
+		.reg = HHI_VID_DIVIDER_CNTL, .bit_idx = 3
+	},
+	[CLKC_RESET_VID_DIVIDER_CNTL_RESET_N_POST] = {
+		.reg = HHI_VID_DIVIDER_CNTL, .bit_idx = 1
+	},
+	[CLKC_RESET_VID_DIVIDER_CNTL_RESET_N_PRE] = {
+		.reg = HHI_VID_DIVIDER_CNTL, .bit_idx = 0
+	},
+};
+
+static int meson8b_clk_reset_update(struct reset_controller_dev *rcdev,
+				    unsigned long id, bool assert)
+{
+	struct meson8b_clk_reset *meson8b_clk_reset =
+		container_of(rcdev, struct meson8b_clk_reset, reset);
+	unsigned long flags;
+	const struct meson8b_clk_reset_line *reset;
+	u32 val;
+
+	if (id >= ARRAY_SIZE(meson8b_clk_reset_bits))
+		return -EINVAL;
+
+	reset = &meson8b_clk_reset_bits[id];
+
+	spin_lock_irqsave(&clk_lock, flags);
+
+	val = readl(meson8b_clk_reset->base + reset->reg);
+	if (assert)
+		val |= BIT(reset->bit_idx);
+	else
+		val &= ~BIT(reset->bit_idx);
+	writel(val, meson8b_clk_reset->base + reset->reg);
+
+	spin_unlock_irqrestore(&clk_lock, flags);
+
+	return 0;
+}
+
+static int meson8b_clk_reset_assert(struct reset_controller_dev *rcdev,
+				     unsigned long id)
+{
+	return meson8b_clk_reset_update(rcdev, id, true);
+}
+
+static int meson8b_clk_reset_deassert(struct reset_controller_dev *rcdev,
+				       unsigned long id)
+{
+	return meson8b_clk_reset_update(rcdev, id, false);
+}
+
+static const struct reset_control_ops meson8b_clk_reset_ops = {
+	.assert = meson8b_clk_reset_assert,
+	.deassert = meson8b_clk_reset_deassert,
+};
+
 static int meson8b_clkc_probe(struct platform_device *pdev)
 {
-	void __iomem *clk_base;
 	int ret, clkid, i;
 	struct clk_hw *parent_hw;
 	struct clk *parent_clk;
 	struct device *dev = &pdev->dev;
 
-	/*  Generic clocks and PLLs */
-	clk_base = of_iomap(dev->of_node, 1);
-	if (!clk_base) {
-		pr_err("%s: Unable to map clk base\n", __func__);
+	if (!clk_base)
 		return -ENXIO;
-	}
 
 	/* Populate base address for PLLs */
 	for (i = 0; i < ARRAY_SIZE(meson8b_clk_plls); i++)
@@ -748,7 +852,7 @@ static int meson8b_clkc_probe(struct platform_device *pdev)
 		/* FIXME convert to devm_clk_register */
 		ret = devm_clk_hw_register(dev, meson8b_hw_onecell_data.hws[clkid]);
 		if (ret)
-			goto iounmap;
+			return ret;
 	}
 
 	/*
@@ -771,15 +875,11 @@ static int meson8b_clkc_probe(struct platform_device *pdev)
 	if (ret) {
 		pr_err("%s: failed to register clock notifier for cpu_clk\n",
 				__func__);
-		goto iounmap;
+		return ret;
 	}
 
 	return of_clk_add_hw_provider(dev->of_node, of_clk_hw_onecell_get,
 			&meson8b_hw_onecell_data);
-
-iounmap:
-	iounmap(clk_base);
-	return ret;
 }
 
 static const struct of_device_id meson8b_clkc_match_table[] = {
@@ -798,3 +898,39 @@ static struct platform_driver meson8b_driver = {
 };
 
 builtin_platform_driver(meson8b_driver);
+
+static void __init meson8b_clkc_reset_init(struct device_node *np)
+{
+	struct meson8b_clk_reset *rstc;
+	int ret;
+
+	/* Generic clocks, PLLs and some of the reset-bits */
+	clk_base = of_iomap(np, 1);
+	if (!clk_base) {
+		pr_err("%s: Unable to map clk base\n", __func__);
+		return;
+	}
+
+	rstc = kzalloc(sizeof(*rstc), GFP_KERNEL);
+	if (!rstc)
+		return;
+
+	/* Reset Controller */
+	rstc->base = clk_base;
+	rstc->reset.ops = &meson8b_clk_reset_ops;
+	rstc->reset.nr_resets = ARRAY_SIZE(meson8b_clk_reset_bits);
+	rstc->reset.of_node = np;
+	ret = reset_controller_register(&rstc->reset);
+	if (ret) {
+		pr_err("%s: Failed to register clkc reset controller: %d\n",
+		       __func__, ret);
+		return;
+	}
+}
+
+CLK_OF_DECLARE_DRIVER(meson8_clkc, "amlogic,meson8-clkc",
+		      meson8b_clkc_reset_init);
+CLK_OF_DECLARE_DRIVER(meson8b_clkc, "amlogic,meson8b-clkc",
+		      meson8b_clkc_reset_init);
+CLK_OF_DECLARE_DRIVER(meson8m2_clkc, "amlogic,meson8m2-clkc",
+		      meson8b_clkc_reset_init);
diff --git a/drivers/clk/meson/meson8b.h b/drivers/clk/meson/meson8b.h
index a687e02547dc..2eaf8a52e7dd 100644
--- a/drivers/clk/meson/meson8b.h
+++ b/drivers/clk/meson/meson8b.h
@@ -37,6 +37,9 @@
 #define HHI_GCLK_AO			0x154 /* 0x55 offset in data sheet */
 #define HHI_SYS_CPU_CLK_CNTL1		0x15c /* 0x57 offset in data sheet */
 #define HHI_MPEG_CLK_CNTL		0x174 /* 0x5d offset in data sheet */
+#define HHI_VID_CLK_CNTL		0x17c /* 0x5f offset in data sheet */
+#define HHI_VID_DIVIDER_CNTL		0x198 /* 0x66 offset in data sheet */
+#define HHI_SYS_CPU_CLK_CNTL0		0x19c /* 0x67 offset in data sheet */
 #define HHI_MPLL_CNTL			0x280 /* 0xa0 offset in data sheet */
 #define HHI_SYS_PLL_CNTL		0x300 /* 0xc0 offset in data sheet */
 #define HHI_VID_PLL_CNTL		0x320 /* 0xc8 offset in data sheet */
@@ -60,110 +63,19 @@
  * CLKID index values
  *
  * These indices are entirely contrived and do not map onto the hardware.
- * Migrate them out of this header and into the DT header file when they need
- * to be exposed to client nodes in DT: include/dt-bindings/clock/meson8b-clkc.h
+ * It has now been decided to expose everything by default in the DT header:
+ * include/dt-bindings/clock/gxbb-clkc.h. Only the clocks ids we don't want
+ * to expose, such as the internal muxes and dividers of composite clocks,
+ * will remain defined here.
  */
 
-/* CLKID_UNUSED */
-/* CLKID_XTAL */
-/* CLKID_PLL_FIXED */
-/* CLKID_PLL_VID */
-/* CLKID_PLL_SYS */
-/* CLKID_FCLK_DIV2 */
-/* CLKID_FCLK_DIV3 */
-/* CLKID_FCLK_DIV4 */
-/* CLKID_FCLK_DIV5 */
-/* CLKID_FCLK_DIV7 */
-/* CLKID_CLK81 */
-/* CLKID_MALI */
-/* CLKID_CPUCLK */
-/* CLKID_ZERO */
-/* CLKID_MPEG_SEL */
-/* CLKID_MPEG_DIV */
-#define CLKID_DDR		16
-#define CLKID_DOS		17
-#define CLKID_ISA		18
-#define CLKID_PL301		19
-#define CLKID_PERIPHS		20
-#define CLKID_SPICC		21
-#define CLKID_I2C		22
-/* #define CLKID_SAR_ADC */
-#define CLKID_SMART_CARD	24
-/* #define CLKID_RNG0 */
-#define CLKID_UART0		26
-#define CLKID_SDHC		27
-#define CLKID_STREAM		28
-#define CLKID_ASYNC_FIFO	29
-/* #define CLKID_SDIO */
-#define CLKID_ABUF		31
-#define CLKID_HIU_IFACE		32
-#define CLKID_ASSIST_MISC	33
-#define CLKID_SPI		34
-#define CLKID_I2S_SPDIF		35
-/* #define CLKID_ETH */
-#define CLKID_DEMUX		37
-#define CLKID_AIU_GLUE		38
-#define CLKID_IEC958		39
-#define CLKID_I2S_OUT		40
-#define CLKID_AMCLK		41
-#define CLKID_AIFIFO2		42
-#define CLKID_MIXER		43
-#define CLKID_MIXER_IFACE	44
-#define CLKID_ADC		45
-#define CLKID_BLKMV		46
-#define CLKID_AIU		47
-#define CLKID_UART1		48
-#define CLKID_G2D		49
-/* #define CLKID_USB0 */
-/* #define CLKID_USB1 */
-#define CLKID_RESET		52
-#define CLKID_NAND		53
-#define CLKID_DOS_PARSER	54
-/* #define CLKID_USB */
-#define CLKID_VDIN1		56
-#define CLKID_AHB_ARB0		57
-#define CLKID_EFUSE		58
-#define CLKID_BOOT_ROM		59
-#define CLKID_AHB_DATA_BUS	60
-#define CLKID_AHB_CTRL_BUS	61
-#define CLKID_HDMI_INTR_SYNC	62
-#define CLKID_HDMI_PCLK		63
-/* CLKID_USB1_DDR_BRIDGE */
-/* CLKID_USB0_DDR_BRIDGE */
-#define CLKID_MMC_PCLK		66
-#define CLKID_DVIN		67
-#define CLKID_UART2		68
-/* #define CLKID_SANA */
-#define CLKID_VPU_INTR		70
-#define CLKID_SEC_AHB_AHB3_BRIDGE	71
-#define CLKID_CLK81_A9		72
-#define CLKID_VCLK2_VENCI0	73
-#define CLKID_VCLK2_VENCI1	74
-#define CLKID_VCLK2_VENCP0	75
-#define CLKID_VCLK2_VENCP1	76
-#define CLKID_GCLK_VENCI_INT	77
-#define CLKID_GCLK_VENCP_INT	78
-#define CLKID_DAC_CLK		79
-#define CLKID_AOCLK_GATE	80
-#define CLKID_IEC958_GATE	81
-#define CLKID_ENC480P		82
-#define CLKID_RNG1		83
-#define CLKID_GCLK_VENCL_INT	84
-#define CLKID_VCLK2_VENCLMCC	85
-#define CLKID_VCLK2_VENCL	86
-#define CLKID_VCLK2_OTHER	87
-#define CLKID_EDP		88
-#define CLKID_AO_MEDIA_CPU	89
-#define CLKID_AO_AHB_SRAM	90
-#define CLKID_AO_AHB_BUS	91
-#define CLKID_AO_IFACE		92
-#define CLKID_MPLL0		93
-#define CLKID_MPLL1		94
-#define CLKID_MPLL2		95
-
 #define CLK_NR_CLKS		96
 
-/* include the CLKIDs that have been made part of the stable DT binding */
+/*
+ * include the CLKID and RESETID that have
+ * been made part of the stable DT binding
+ */
 #include <dt-bindings/clock/meson8b-clkc.h>
+#include <dt-bindings/reset/amlogic,meson8b-clkc-reset.h>
 
 #endif /* __MESON8B_H */
diff --git a/drivers/clk/mmp/clk.c b/drivers/clk/mmp/clk.c
index 61893fe73251..089927e4cda2 100644
--- a/drivers/clk/mmp/clk.c
+++ b/drivers/clk/mmp/clk.c
@@ -9,7 +9,7 @@
 void mmp_clk_init(struct device_node *np, struct mmp_clk_unit *unit,
 		int nr_clks)
 {
-	static struct clk **clk_table;
+	struct clk **clk_table;
 
 	clk_table = kcalloc(nr_clks, sizeof(struct clk *), GFP_KERNEL);
 	if (!clk_table)
diff --git a/drivers/clk/nxp/clk-lpc32xx.c b/drivers/clk/nxp/clk-lpc32xx.c
index 5b98ff9076f3..7b359afd620e 100644
--- a/drivers/clk/nxp/clk-lpc32xx.c
+++ b/drivers/clk/nxp/clk-lpc32xx.c
@@ -885,7 +885,7 @@ static const struct clk_ops clk_usb_i2c_ops = {
 	.recalc_rate = clk_usb_i2c_recalc_rate,
 };
 
-static int clk_gate_enable(struct clk_hw *hw)
+static int lpc32xx_clk_gate_enable(struct clk_hw *hw)
 {
 	struct lpc32xx_clk_gate *clk = to_lpc32xx_gate(hw);
 	u32 mask = BIT(clk->bit_idx);
@@ -894,7 +894,7 @@ static int clk_gate_enable(struct clk_hw *hw)
 	return regmap_update_bits(clk_regmap, clk->reg, mask, val);
 }
 
-static void clk_gate_disable(struct clk_hw *hw)
+static void lpc32xx_clk_gate_disable(struct clk_hw *hw)
 {
 	struct lpc32xx_clk_gate *clk = to_lpc32xx_gate(hw);
 	u32 mask = BIT(clk->bit_idx);
@@ -903,7 +903,7 @@ static void clk_gate_disable(struct clk_hw *hw)
 	regmap_update_bits(clk_regmap, clk->reg, mask, val);
 }
 
-static int clk_gate_is_enabled(struct clk_hw *hw)
+static int lpc32xx_clk_gate_is_enabled(struct clk_hw *hw)
 {
 	struct lpc32xx_clk_gate *clk = to_lpc32xx_gate(hw);
 	u32 val;
@@ -916,9 +916,9 @@ static int clk_gate_is_enabled(struct clk_hw *hw)
 }
 
 static const struct clk_ops lpc32xx_clk_gate_ops = {
-	.enable = clk_gate_enable,
-	.disable = clk_gate_disable,
-	.is_enabled = clk_gate_is_enabled,
+	.enable = lpc32xx_clk_gate_enable,
+	.disable = lpc32xx_clk_gate_disable,
+	.is_enabled = lpc32xx_clk_gate_is_enabled,
 };
 
 #define div_mask(width)	((1 << (width)) - 1)
diff --git a/drivers/clk/qcom/clk-smd-rpm.c b/drivers/clk/qcom/clk-smd-rpm.c
index d990fe44aef3..cc03d5508627 100644
--- a/drivers/clk/qcom/clk-smd-rpm.c
+++ b/drivers/clk/qcom/clk-smd-rpm.c
@@ -412,8 +412,6 @@ static const struct clk_ops clk_smd_rpm_ops = {
 static const struct clk_ops clk_smd_rpm_branch_ops = {
 	.prepare	= clk_smd_rpm_prepare,
 	.unprepare	= clk_smd_rpm_unprepare,
-	.round_rate	= clk_smd_rpm_round_rate,
-	.recalc_rate	= clk_smd_rpm_recalc_rate,
 };
 
 /* msm8916 */
diff --git a/drivers/clk/qcom/gcc-msm8916.c b/drivers/clk/qcom/gcc-msm8916.c
index 2cfe7000fc60..3410ee68d4bc 100644
--- a/drivers/clk/qcom/gcc-msm8916.c
+++ b/drivers/clk/qcom/gcc-msm8916.c
@@ -1176,7 +1176,7 @@ static struct clk_rcg2 bimc_gpu_clk_src = {
 		.parent_names = gcc_xo_gpll0_bimc,
 		.num_parents = 3,
 		.flags = CLK_GET_RATE_NOCACHE,
-		.ops = &clk_rcg2_shared_ops,
+		.ops = &clk_rcg2_ops,
 	},
 };
 
diff --git a/drivers/clk/qcom/gcc-msm8996.c b/drivers/clk/qcom/gcc-msm8996.c
index 8abc200d4fd3..7ddec886fcd3 100644
--- a/drivers/clk/qcom/gcc-msm8996.c
+++ b/drivers/clk/qcom/gcc-msm8996.c
@@ -2730,6 +2730,32 @@ static struct clk_fixed_factor ufs_rx_cfg_clk_src = {
 	},
 };
 
+static struct clk_branch gcc_hlos1_vote_lpass_core_smmu_clk = {
+	.halt_reg = 0x7d010,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x7d010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "hlos1_vote_lpass_core_smmu_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_hlos1_vote_lpass_adsp_smmu_clk = {
+	.halt_reg = 0x7d014,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x7d014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "hlos1_vote_lpass_adsp_smmu_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
 static struct clk_branch gcc_ufs_rx_cfg_clk = {
 	.halt_reg = 0x75014,
 	.clkr = {
@@ -3307,6 +3333,8 @@ static struct clk_regmap *gcc_msm8996_clocks[] = {
 	[GCC_UFS_AHB_CLK] = &gcc_ufs_ahb_clk.clkr,
 	[GCC_UFS_TX_CFG_CLK] = &gcc_ufs_tx_cfg_clk.clkr,
 	[GCC_UFS_RX_CFG_CLK] = &gcc_ufs_rx_cfg_clk.clkr,
+	[GCC_HLOS1_VOTE_LPASS_CORE_SMMU_CLK] = &gcc_hlos1_vote_lpass_core_smmu_clk.clkr,
+	[GCC_HLOS1_VOTE_LPASS_ADSP_SMMU_CLK] = &gcc_hlos1_vote_lpass_adsp_smmu_clk.clkr,
 	[GCC_UFS_TX_SYMBOL_0_CLK] = &gcc_ufs_tx_symbol_0_clk.clkr,
 	[GCC_UFS_RX_SYMBOL_0_CLK] = &gcc_ufs_rx_symbol_0_clk.clkr,
 	[GCC_UFS_RX_SYMBOL_1_CLK] = &gcc_ufs_rx_symbol_1_clk.clkr,
diff --git a/drivers/clk/renesas/Kconfig b/drivers/clk/renesas/Kconfig
index 78d1df9112ba..acbb38151ba1 100644
--- a/drivers/clk/renesas/Kconfig
+++ b/drivers/clk/renesas/Kconfig
@@ -15,6 +15,7 @@ config CLK_RENESAS
 	select CLK_R8A7794 if ARCH_R8A7794
 	select CLK_R8A7795 if ARCH_R8A7795
 	select CLK_R8A7796 if ARCH_R8A7796
+	select CLK_R8A77995 if ARCH_R8A77995
 	select CLK_SH73A0 if ARCH_SH73A0
 
 if CLK_RENESAS
@@ -34,94 +35,103 @@ config CLK_EMEV2
 	bool "Emma Mobile EV2 clock support" if COMPILE_TEST
 
 config CLK_RZA1
-	bool
+	bool "RZ/A1H clock support" if COMPILE_TEST
 	select CLK_RENESAS_CPG_MSTP
 
 config CLK_R8A73A4
-	bool
+	bool "R-Mobile APE6 clock support" if COMPILE_TEST
 	select CLK_RENESAS_CPG_MSTP
 	select CLK_RENESAS_DIV6
 
 config CLK_R8A7740
-	bool
+	bool "R-Mobile A1 clock support" if COMPILE_TEST
 	select CLK_RENESAS_CPG_MSTP
 	select CLK_RENESAS_DIV6
 
 config CLK_R8A7743
-	bool
+	bool "RZ/G1M clock support" if COMPILE_TEST
 	select CLK_RCAR_GEN2_CPG
 
 config CLK_R8A7745
-	bool
+	bool "RZ/G1E clock support" if COMPILE_TEST
 	select CLK_RCAR_GEN2_CPG
 
 config CLK_R8A7778
-	bool
+	bool "R-Car M1A clock support" if COMPILE_TEST
 	select CLK_RENESAS_CPG_MSTP
 
 config CLK_R8A7779
-	bool
+	bool "R-Car H1 clock support" if COMPILE_TEST
 	select CLK_RENESAS_CPG_MSTP
 
 config CLK_R8A7790
-	bool
+	bool "R-Car H2 clock support" if COMPILE_TEST
 	select CLK_RCAR_GEN2 if CLK_RENESAS_LEGACY
 	select CLK_RCAR_GEN2_CPG
 	select CLK_RENESAS_DIV6
 
 config CLK_R8A7791
-	bool
+	bool "R-Car M2-W/N clock support" if COMPILE_TEST
 	select CLK_RCAR_GEN2 if CLK_RENESAS_LEGACY
 	select CLK_RCAR_GEN2_CPG
 	select CLK_RENESAS_DIV6
 
 config CLK_R8A7792
-	bool
+	bool "R-Car V2H clock support" if COMPILE_TEST
 	select CLK_RCAR_GEN2 if CLK_RENESAS_LEGACY
 	select CLK_RCAR_GEN2_CPG
 
 config CLK_R8A7794
-	bool
+	bool "R-Car E2 clock support" if COMPILE_TEST
 	select CLK_RCAR_GEN2 if CLK_RENESAS_LEGACY
 	select CLK_RCAR_GEN2_CPG
 	select CLK_RENESAS_DIV6
 
 config CLK_R8A7795
-	bool
+	bool "R-Car H3 clock support" if COMPILE_TEST
 	select CLK_RCAR_GEN3_CPG
 
 config CLK_R8A7796
-	bool
+	bool "R-Car M3-W clock support" if COMPILE_TEST
+	select CLK_RCAR_GEN3_CPG
+
+config CLK_R8A77995
+	bool "R-Car D3 clock support" if COMPILE_TEST
 	select CLK_RCAR_GEN3_CPG
 
 config CLK_SH73A0
-	bool
+	bool "SH-Mobile AG5 clock support" if COMPILE_TEST
 	select CLK_RENESAS_CPG_MSTP
 	select CLK_RENESAS_DIV6
 
 
 # Family
 config CLK_RCAR_GEN2
-	bool
+	bool "R-Car Gen2 legacy clock support" if COMPILE_TEST
 	select CLK_RENESAS_CPG_MSTP
 	select CLK_RENESAS_DIV6
 
 config CLK_RCAR_GEN2_CPG
-	bool
+	bool "R-Car Gen2 CPG clock support" if COMPILE_TEST
 	select CLK_RENESAS_CPG_MSSR
 
 config CLK_RCAR_GEN3_CPG
-	bool
+	bool "R-Car Gen3 CPG clock support" if COMPILE_TEST
 	select CLK_RENESAS_CPG_MSSR
 
+config CLK_RCAR_USB2_CLOCK_SEL
+	bool "Renesas R-Car USB2 clock selector support"
+	depends on ARCH_RENESAS || COMPILE_TEST
+	help
+	  This is a driver for R-Car USB2 clock selector
 
 # Generic
 config CLK_RENESAS_CPG_MSSR
-	bool
+	bool "CPG/MSSR clock support" if COMPILE_TEST
 	select CLK_RENESAS_DIV6
 
 config CLK_RENESAS_CPG_MSTP
-	bool
+	bool "MSTP clock support" if COMPILE_TEST
 
 config CLK_RENESAS_DIV6
 	bool "DIV6 clock support" if COMPILE_TEST
diff --git a/drivers/clk/renesas/Makefile b/drivers/clk/renesas/Makefile
index 02d04124371f..9bda3ec5b199 100644
--- a/drivers/clk/renesas/Makefile
+++ b/drivers/clk/renesas/Makefile
@@ -13,12 +13,14 @@ obj-$(CONFIG_CLK_R8A7792)		+= r8a7792-cpg-mssr.o
 obj-$(CONFIG_CLK_R8A7794)		+= r8a7794-cpg-mssr.o
 obj-$(CONFIG_CLK_R8A7795)		+= r8a7795-cpg-mssr.o
 obj-$(CONFIG_CLK_R8A7796)		+= r8a7796-cpg-mssr.o
+obj-$(CONFIG_CLK_R8A77995)		+= r8a77995-cpg-mssr.o
 obj-$(CONFIG_CLK_SH73A0)		+= clk-sh73a0.o
 
 # Family
 obj-$(CONFIG_CLK_RCAR_GEN2)		+= clk-rcar-gen2.o
 obj-$(CONFIG_CLK_RCAR_GEN2_CPG)		+= rcar-gen2-cpg.o
 obj-$(CONFIG_CLK_RCAR_GEN3_CPG)		+= rcar-gen3-cpg.o
+obj-$(CONFIG_CLK_RCAR_USB2_CLOCK_SEL)	+= rcar-usb2-clock-sel.o
 
 # Generic
 obj-$(CONFIG_CLK_RENESAS_CPG_MSSR)	+= renesas-cpg-mssr.o
diff --git a/drivers/clk/renesas/clk-div6.c b/drivers/clk/renesas/clk-div6.c
index 0627860233cb..3e0040c0ac87 100644
--- a/drivers/clk/renesas/clk-div6.c
+++ b/drivers/clk/renesas/clk-div6.c
@@ -29,6 +29,9 @@
  * @hw: handle between common and hardware-specific interfaces
  * @reg: IO-remapped register
  * @div: divisor value (1-64)
+ * @src_shift: Shift to access the register bits to select the parent clock
+ * @src_width: Number of register bits to select the parent clock (may be 0)
+ * @parents: Array to map from valid parent clocks indices to hardware indices
  */
 struct div6_clock {
 	struct clk_hw hw;
diff --git a/drivers/clk/renesas/clk-mstp.c b/drivers/clk/renesas/clk-mstp.c
index f1617dd044cb..500a9e4e03c4 100644
--- a/drivers/clk/renesas/clk-mstp.c
+++ b/drivers/clk/renesas/clk-mstp.c
@@ -335,7 +335,7 @@ void __init cpg_mstp_add_clk_domain(struct device_node *np)
 	u32 ncells;
 
 	if (of_property_read_u32(np, "#power-domain-cells", &ncells)) {
-		pr_warn("%s lacks #power-domain-cells\n", np->full_name);
+		pr_warn("%pOF lacks #power-domain-cells\n", np);
 		return;
 	}
 
diff --git a/drivers/clk/renesas/clk-rcar-gen2.c b/drivers/clk/renesas/clk-rcar-gen2.c
index 51a2479ed5d7..0b2e56d0d94b 100644
--- a/drivers/clk/renesas/clk-rcar-gen2.c
+++ b/drivers/clk/renesas/clk-rcar-gen2.c
@@ -407,8 +407,7 @@ static void __init rcar_gen2_cpg_clocks_init(struct device_node *np)
 
 	if (rcar_rst_read_mode_pins(&cpg_mode)) {
 		/* Backward-compatibility with old DT */
-		pr_warn("%s: failed to obtain mode pins from RST\n",
-			np->full_name);
+		pr_warn("%pOF: failed to obtain mode pins from RST\n", np);
 		cpg_mode = rcar_gen2_read_mode_pins();
 	}
 
diff --git a/drivers/clk/renesas/r8a7792-cpg-mssr.c b/drivers/clk/renesas/r8a7792-cpg-mssr.c
index a832b9b6f7b0..7f85bbf20bf7 100644
--- a/drivers/clk/renesas/r8a7792-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a7792-cpg-mssr.c
@@ -118,6 +118,13 @@ static const struct mssr_mod_clk r8a7792_mod_clks[] __initconst = {
 	DEF_MOD("vin1",			 810,	R8A7792_CLK_ZG),
 	DEF_MOD("vin0",			 811,	R8A7792_CLK_ZG),
 	DEF_MOD("etheravb",		 812,	R8A7792_CLK_HP),
+	DEF_MOD("imr-lx3",		 821,	R8A7792_CLK_ZG),
+	DEF_MOD("imr-lsx3-1",		 822,	R8A7792_CLK_ZG),
+	DEF_MOD("imr-lsx3-0",		 823,	R8A7792_CLK_ZG),
+	DEF_MOD("imr-lsx3-5",		 825,	R8A7792_CLK_ZG),
+	DEF_MOD("imr-lsx3-4",		 826,	R8A7792_CLK_ZG),
+	DEF_MOD("imr-lsx3-3",		 827,	R8A7792_CLK_ZG),
+	DEF_MOD("imr-lsx3-2",		 828,	R8A7792_CLK_ZG),
 	DEF_MOD("gyro-adc",		 901,	R8A7792_CLK_P),
 	DEF_MOD("gpio7",		 904,	R8A7792_CLK_CP),
 	DEF_MOD("gpio6",		 905,	R8A7792_CLK_CP),
diff --git a/drivers/clk/renesas/r8a7795-cpg-mssr.c b/drivers/clk/renesas/r8a7795-cpg-mssr.c
index c091a8e024b8..762b2f8824f1 100644
--- a/drivers/clk/renesas/r8a7795-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a7795-cpg-mssr.c
@@ -305,23 +305,23 @@ static const unsigned int r8a7795_crit_mod_clks[] __initconst = {
 					 (((md) & BIT(17)) >> 17))
 
 static const struct rcar_gen3_cpg_pll_config cpg_pll_configs[16] __initconst = {
-	/* EXTAL div	PLL1 mult	PLL3 mult */
-	{ 1,		192,		192,	},
-	{ 1,		192,		128,	},
-	{ 0, /* Prohibited setting */		},
-	{ 1,		192,		192,	},
-	{ 1,		160,		160,	},
-	{ 1,		160,		106,	},
-	{ 0, /* Prohibited setting */		},
-	{ 1,		160,		160,	},
-	{ 1,		128,		128,	},
-	{ 1,		128,		84,	},
-	{ 0, /* Prohibited setting */		},
-	{ 1,		128,		128,	},
-	{ 2,		192,		192,	},
-	{ 2,		192,		128,	},
-	{ 0, /* Prohibited setting */		},
-	{ 2,		192,		192,	},
+	/* EXTAL div	PLL1 mult/div	PLL3 mult/div */
+	{ 1,		192,	1,	192,	1,	},
+	{ 1,		192,	1,	128,	1,	},
+	{ 0, /* Prohibited setting */			},
+	{ 1,		192,	1,	192,	1,	},
+	{ 1,		160,	1,	160,	1,	},
+	{ 1,		160,	1,	106,	1,	},
+	{ 0, /* Prohibited setting */			},
+	{ 1,		160,	1,	160,	1,	},
+	{ 1,		128,	1,	128,	1,	},
+	{ 1,		128,	1,	84,	1,	},
+	{ 0, /* Prohibited setting */			},
+	{ 1,		128,	1,	128,	1,	},
+	{ 2,		192,	1,	192,	1,	},
+	{ 2,		192,	1,	128,	1,	},
+	{ 0, /* Prohibited setting */			},
+	{ 2,		192,	1,	192,	1,	},
 };
 
 static const struct soc_device_attribute r8a7795es1[] __initconst = {
diff --git a/drivers/clk/renesas/r8a7796-cpg-mssr.c b/drivers/clk/renesas/r8a7796-cpg-mssr.c
index acc6d0f153e1..e5e7fb212288 100644
--- a/drivers/clk/renesas/r8a7796-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a7796-cpg-mssr.c
@@ -138,6 +138,7 @@ static const struct mssr_mod_clk r8a7796_mod_clks[] __initconst = {
 	DEF_MOD("sdif0",		 314,	R8A7796_CLK_SD0),
 	DEF_MOD("pcie1",		 318,	R8A7796_CLK_S3D1),
 	DEF_MOD("pcie0",		 319,	R8A7796_CLK_S3D1),
+	DEF_MOD("usb3-if0",		 328,	R8A7796_CLK_S3D1),
 	DEF_MOD("usb-dmac0",		 330,	R8A7796_CLK_S3D1),
 	DEF_MOD("usb-dmac1",		 331,	R8A7796_CLK_S3D1),
 	DEF_MOD("rwdt",			 402,	R8A7796_CLK_R),
@@ -277,23 +278,23 @@ static const unsigned int r8a7796_crit_mod_clks[] __initconst = {
 					 (((md) & BIT(17)) >> 17))
 
 static const struct rcar_gen3_cpg_pll_config cpg_pll_configs[16] __initconst = {
-	/* EXTAL div	PLL1 mult	PLL3 mult */
-	{ 1,		192,		192,	},
-	{ 1,		192,		128,	},
-	{ 0, /* Prohibited setting */		},
-	{ 1,		192,		192,	},
-	{ 1,		160,		160,	},
-	{ 1,		160,		106,	},
-	{ 0, /* Prohibited setting */		},
-	{ 1,		160,		160,	},
-	{ 1,		128,		128,	},
-	{ 1,		128,		84,	},
-	{ 0, /* Prohibited setting */		},
-	{ 1,		128,		128,	},
-	{ 2,		192,		192,	},
-	{ 2,		192,		128,	},
-	{ 0, /* Prohibited setting */		},
-	{ 2,		192,		192,	},
+	/* EXTAL div	PLL1 mult/div	PLL3 mult/div */
+	{ 1,		192,	1,	192,	1,	},
+	{ 1,		192,	1,	128,	1,	},
+	{ 0, /* Prohibited setting */			},
+	{ 1,		192,	1,	192,	1,	},
+	{ 1,		160,	1,	160,	1,	},
+	{ 1,		160,	1,	106,	1,	},
+	{ 0, /* Prohibited setting */			},
+	{ 1,		160,	1,	160,	1,	},
+	{ 1,		128,	1,	128,	1,	},
+	{ 1,		128,	1,	84,	1,	},
+	{ 0, /* Prohibited setting */			},
+	{ 1,		128,	1,	128,	1,	},
+	{ 2,		192,	1,	192,	1,	},
+	{ 2,		192,	1,	128,	1,	},
+	{ 0, /* Prohibited setting */			},
+	{ 2,		192,	1,	192,	1,	},
 };
 
 static int __init r8a7796_cpg_mssr_init(struct device *dev)
diff --git a/drivers/clk/renesas/r8a77995-cpg-mssr.c b/drivers/clk/renesas/r8a77995-cpg-mssr.c
new file mode 100644
index 000000000000..e594cf8ee63b
--- /dev/null
+++ b/drivers/clk/renesas/r8a77995-cpg-mssr.c
@@ -0,0 +1,236 @@
+/*
+ * r8a77995 Clock Pulse Generator / Module Standby and Software Reset
+ *
+ * Copyright (C) 2017 Glider bvba
+ *
+ * Based on r8a7795-cpg-mssr.c
+ *
+ * Copyright (C) 2015 Glider bvba
+ * Copyright (C) 2015 Renesas Electronics Corp.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ */
+
+#include <linux/device.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/soc/renesas/rcar-rst.h>
+
+#include <dt-bindings/clock/r8a77995-cpg-mssr.h>
+
+#include "renesas-cpg-mssr.h"
+#include "rcar-gen3-cpg.h"
+
+enum clk_ids {
+	/* Core Clock Outputs exported to DT */
+	LAST_DT_CORE_CLK = R8A77995_CLK_CP,
+
+	/* External Input Clocks */
+	CLK_EXTAL,
+
+	/* Internal Core Clocks */
+	CLK_MAIN,
+	CLK_PLL0,
+	CLK_PLL1,
+	CLK_PLL3,
+	CLK_PLL0D2,
+	CLK_PLL0D3,
+	CLK_PLL0D5,
+	CLK_PLL1D2,
+	CLK_PE,
+	CLK_S0,
+	CLK_S1,
+	CLK_S2,
+	CLK_S3,
+	CLK_SDSRC,
+	CLK_SSPSRC,
+
+	/* Module Clocks */
+	MOD_CLK_BASE
+};
+
+static const struct cpg_core_clk r8a77995_core_clks[] __initconst = {
+	/* External Clock Inputs */
+	DEF_INPUT("extal",     CLK_EXTAL),
+
+	/* Internal Core Clocks */
+	DEF_BASE(".main",      CLK_MAIN, CLK_TYPE_GEN3_MAIN,       CLK_EXTAL),
+	DEF_BASE(".pll1",      CLK_PLL1, CLK_TYPE_GEN3_PLL1,       CLK_MAIN),
+	DEF_BASE(".pll3",      CLK_PLL3, CLK_TYPE_GEN3_PLL3,       CLK_MAIN),
+
+	DEF_FIXED(".pll0",     CLK_PLL0,           CLK_MAIN,	   4, 250),
+	DEF_FIXED(".pll0d2",   CLK_PLL0D2,         CLK_PLL0,       2, 1),
+	DEF_FIXED(".pll0d3",   CLK_PLL0D3,         CLK_PLL0,       3, 1),
+	DEF_FIXED(".pll0d5",   CLK_PLL0D5,         CLK_PLL0,       5, 1),
+	DEF_FIXED(".pll1d2",   CLK_PLL1D2,         CLK_PLL1,       2, 1),
+	DEF_FIXED(".pe",       CLK_PE,             CLK_PLL0D3,     4, 1),
+	DEF_FIXED(".s0",       CLK_S0,             CLK_PLL1,       2, 1),
+	DEF_FIXED(".s1",       CLK_S1,             CLK_PLL1,       3, 1),
+	DEF_FIXED(".s2",       CLK_S2,             CLK_PLL1,       4, 1),
+	DEF_FIXED(".s3",       CLK_S3,             CLK_PLL1,       6, 1),
+	DEF_FIXED(".sdsrc",    CLK_SDSRC,          CLK_PLL1,       2, 1),
+
+	/* Core Clock Outputs */
+	DEF_FIXED("z2",        R8A77995_CLK_Z2,    CLK_PLL0D3,     1, 1),
+	DEF_FIXED("ztr",       R8A77995_CLK_ZTR,   CLK_PLL1,       6, 1),
+	DEF_FIXED("zt",        R8A77995_CLK_ZT,    CLK_PLL1,       4, 1),
+	DEF_FIXED("zx",        R8A77995_CLK_ZX,    CLK_PLL1,       3, 1),
+	DEF_FIXED("s0d1",      R8A77995_CLK_S0D1,  CLK_S0,         1, 1),
+	DEF_FIXED("s1d1",      R8A77995_CLK_S1D1,  CLK_S1,         1, 1),
+	DEF_FIXED("s1d2",      R8A77995_CLK_S1D2,  CLK_S1,         2, 1),
+	DEF_FIXED("s1d4",      R8A77995_CLK_S1D4,  CLK_S1,         4, 1),
+	DEF_FIXED("s2d1",      R8A77995_CLK_S2D1,  CLK_S2,         1, 1),
+	DEF_FIXED("s2d2",      R8A77995_CLK_S2D2,  CLK_S2,         2, 1),
+	DEF_FIXED("s2d4",      R8A77995_CLK_S2D4,  CLK_S2,         4, 1),
+	DEF_FIXED("s3d1",      R8A77995_CLK_S3D1,  CLK_S3,         1, 1),
+	DEF_FIXED("s3d2",      R8A77995_CLK_S3D2,  CLK_S3,         2, 1),
+	DEF_FIXED("s3d4",      R8A77995_CLK_S3D4,  CLK_S3,         4, 1),
+
+	DEF_FIXED("cl",        R8A77995_CLK_CL,    CLK_PLL1,      48, 1),
+	DEF_FIXED("cp",        R8A77995_CLK_CP,    CLK_EXTAL,      2, 1),
+	DEF_FIXED("osc",       R8A77995_CLK_OSC,   CLK_EXTAL,    384, 1),
+	DEF_FIXED("r",         R8A77995_CLK_R,     CLK_EXTAL,   1536, 1),
+
+	DEF_GEN3_PE("s1d4c",   R8A77995_CLK_S1D4C, CLK_S1, 4, CLK_PE, 2),
+	DEF_GEN3_PE("s3d1c",   R8A77995_CLK_S3D1C, CLK_S3, 1, CLK_PE, 1),
+	DEF_GEN3_PE("s3d2c",   R8A77995_CLK_S3D2C, CLK_S3, 2, CLK_PE, 2),
+	DEF_GEN3_PE("s3d4c",   R8A77995_CLK_S3D4C, CLK_S3, 4, CLK_PE, 4),
+
+	DEF_GEN3_SD("sd0",     R8A77995_CLK_SD0,   CLK_SDSRC,     0x268),
+
+	DEF_DIV6P1("canfd",    R8A77995_CLK_CANFD, CLK_PLL0D3,    0x244),
+	DEF_DIV6P1("mso",      R8A77995_CLK_MSO,   CLK_PLL1D2,    0x014),
+};
+
+static const struct mssr_mod_clk r8a77995_mod_clks[] __initconst = {
+	DEF_MOD("scif5",		 202,	R8A77995_CLK_S3D4C),
+	DEF_MOD("scif4",		 203,	R8A77995_CLK_S3D4C),
+	DEF_MOD("scif3",		 204,	R8A77995_CLK_S3D4C),
+	DEF_MOD("scif1",		 206,	R8A77995_CLK_S3D4C),
+	DEF_MOD("scif0",		 207,	R8A77995_CLK_S3D4C),
+	DEF_MOD("msiof3",		 208,	R8A77995_CLK_MSO),
+	DEF_MOD("msiof2",		 209,	R8A77995_CLK_MSO),
+	DEF_MOD("msiof1",		 210,	R8A77995_CLK_MSO),
+	DEF_MOD("msiof0",		 211,	R8A77995_CLK_MSO),
+	DEF_MOD("sys-dmac2",		 217,	R8A77995_CLK_S3D1),
+	DEF_MOD("sys-dmac1",		 218,	R8A77995_CLK_S3D1),
+	DEF_MOD("sys-dmac0",		 219,	R8A77995_CLK_S3D1),
+	DEF_MOD("cmt3",			 300,	R8A77995_CLK_R),
+	DEF_MOD("cmt2",			 301,	R8A77995_CLK_R),
+	DEF_MOD("cmt1",			 302,	R8A77995_CLK_R),
+	DEF_MOD("cmt0",			 303,	R8A77995_CLK_R),
+	DEF_MOD("scif2",		 310,	R8A77995_CLK_S3D4C),
+	DEF_MOD("emmc0",		 312,	R8A77995_CLK_SD0),
+	DEF_MOD("usb-dmac0",		 330,	R8A77995_CLK_S3D1),
+	DEF_MOD("usb-dmac1",		 331,	R8A77995_CLK_S3D1),
+	DEF_MOD("rwdt",			 402,	R8A77995_CLK_R),
+	DEF_MOD("intc-ex",		 407,	R8A77995_CLK_CP),
+	DEF_MOD("intc-ap",		 408,	R8A77995_CLK_S3D1),
+	DEF_MOD("audmac0",		 502,	R8A77995_CLK_S3D1),
+	DEF_MOD("hscif3",		 517,	R8A77995_CLK_S3D1C),
+	DEF_MOD("hscif0",		 520,	R8A77995_CLK_S3D1C),
+	DEF_MOD("thermal",		 522,	R8A77995_CLK_CP),
+	DEF_MOD("pwm",			 523,	R8A77995_CLK_S3D4C),
+	DEF_MOD("fcpvd1",		 602,	R8A77995_CLK_S1D2),
+	DEF_MOD("fcpvd0",		 603,	R8A77995_CLK_S1D2),
+	DEF_MOD("fcpvbs",		 607,	R8A77995_CLK_S0D1),
+	DEF_MOD("vspd1",		 622,	R8A77995_CLK_S1D2),
+	DEF_MOD("vspd0",		 623,	R8A77995_CLK_S1D2),
+	DEF_MOD("vspbs",		 627,	R8A77995_CLK_S0D1),
+	DEF_MOD("ehci0",		 703,	R8A77995_CLK_S3D2),
+	DEF_MOD("hsusb",		 704,	R8A77995_CLK_S3D2),
+	DEF_MOD("du1",			 723,	R8A77995_CLK_S2D1),
+	DEF_MOD("du0",			 724,	R8A77995_CLK_S2D1),
+	DEF_MOD("lvds",			 727,	R8A77995_CLK_S2D1),
+	DEF_MOD("vin7",			 804,	R8A77995_CLK_S1D2),
+	DEF_MOD("vin6",			 805,	R8A77995_CLK_S1D2),
+	DEF_MOD("vin5",			 806,	R8A77995_CLK_S1D2),
+	DEF_MOD("vin4",			 807,	R8A77995_CLK_S1D2),
+	DEF_MOD("etheravb",		 812,	R8A77995_CLK_S3D2),
+	DEF_MOD("imr0",			 823,	R8A77995_CLK_S1D2),
+	DEF_MOD("gpio6",		 906,	R8A77995_CLK_S3D4),
+	DEF_MOD("gpio5",		 907,	R8A77995_CLK_S3D4),
+	DEF_MOD("gpio4",		 908,	R8A77995_CLK_S3D4),
+	DEF_MOD("gpio3",		 909,	R8A77995_CLK_S3D4),
+	DEF_MOD("gpio2",		 910,	R8A77995_CLK_S3D4),
+	DEF_MOD("gpio1",		 911,	R8A77995_CLK_S3D4),
+	DEF_MOD("gpio0",		 912,	R8A77995_CLK_S3D4),
+	DEF_MOD("can-fd",		 914,	R8A77995_CLK_S3D2),
+	DEF_MOD("can-if1",		 915,	R8A77995_CLK_S3D4),
+	DEF_MOD("can-if0",		 916,	R8A77995_CLK_S3D4),
+	DEF_MOD("i2c3",			 928,	R8A77995_CLK_S3D2),
+	DEF_MOD("i2c2",			 929,	R8A77995_CLK_S3D2),
+	DEF_MOD("i2c1",			 930,	R8A77995_CLK_S3D2),
+	DEF_MOD("i2c0",			 931,	R8A77995_CLK_S3D2),
+	DEF_MOD("ssi-all",		1005,	R8A77995_CLK_S3D4),
+	DEF_MOD("ssi4",			1011,	MOD_CLK_ID(1005)),
+	DEF_MOD("ssi3",			1012,	MOD_CLK_ID(1005)),
+	DEF_MOD("scu-all",		1017,	R8A77995_CLK_S3D4),
+	DEF_MOD("scu-dvc1",		1018,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-dvc0",		1019,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-ctu1-mix1",	1020,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-ctu0-mix0",	1021,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src6",		1025,	MOD_CLK_ID(1017)),
+	DEF_MOD("scu-src5",		1026,	MOD_CLK_ID(1017)),
+};
+
+static const unsigned int r8a77995_crit_mod_clks[] __initconst = {
+	MOD_CLK_ID(408),	/* INTC-AP (GIC) */
+};
+
+
+/*
+ * CPG Clock Data
+ */
+
+/*
+ * MD19		EXTAL (MHz)	PLL0		PLL1		PLL3
+ *--------------------------------------------------------------------
+ * 0		48 x 1		x250/4		x100/3		x100/3
+ * 1		48 x 1		x250/4		x100/3		x116/6
+ */
+#define CPG_PLL_CONFIG_INDEX(md)	(((md) & BIT(19)) >> 19)
+
+static const struct rcar_gen3_cpg_pll_config cpg_pll_configs[2] __initconst = {
+	/* EXTAL div	PLL1 mult/div	PLL3 mult/div */
+	{ 1,		100,	3,	100,	3,	},
+	{ 1,		100,	3,	116,	6,	},
+};
+
+static int __init r8a77995_cpg_mssr_init(struct device *dev)
+{
+	const struct rcar_gen3_cpg_pll_config *cpg_pll_config;
+	u32 cpg_mode;
+	int error;
+
+	error = rcar_rst_read_mode_pins(&cpg_mode);
+	if (error)
+		return error;
+
+	cpg_pll_config = &cpg_pll_configs[CPG_PLL_CONFIG_INDEX(cpg_mode)];
+
+	return rcar_gen3_cpg_init(cpg_pll_config, 0, cpg_mode);
+}
+
+const struct cpg_mssr_info r8a77995_cpg_mssr_info __initconst = {
+	/* Core Clocks */
+	.core_clks = r8a77995_core_clks,
+	.num_core_clks = ARRAY_SIZE(r8a77995_core_clks),
+	.last_dt_core_clk = LAST_DT_CORE_CLK,
+	.num_total_core_clks = MOD_CLK_BASE,
+
+	/* Module Clocks */
+	.mod_clks = r8a77995_mod_clks,
+	.num_mod_clks = ARRAY_SIZE(r8a77995_mod_clks),
+	.num_hw_mod_clks = 12 * 32,
+
+	/* Critical Module Clocks */
+	.crit_mod_clks = r8a77995_crit_mod_clks,
+	.num_crit_mod_clks = ARRAY_SIZE(r8a77995_crit_mod_clks),
+
+	/* Callbacks */
+	.init = r8a77995_cpg_mssr_init,
+	.cpg_clk_register = rcar_gen3_cpg_clk_register,
+};
diff --git a/drivers/clk/renesas/rcar-gen3-cpg.c b/drivers/clk/renesas/rcar-gen3-cpg.c
index 3dee900522b7..951105816547 100644
--- a/drivers/clk/renesas/rcar-gen3-cpg.c
+++ b/drivers/clk/renesas/rcar-gen3-cpg.c
@@ -60,6 +60,7 @@ struct sd_clock {
 	unsigned int div_num;
 	unsigned int div_min;
 	unsigned int div_max;
+	unsigned int cur_div_idx;
 };
 
 /* SDn divider
@@ -96,21 +97,10 @@ static const struct sd_div_table cpg_sd_div_table[] = {
 static int cpg_sd_clock_enable(struct clk_hw *hw)
 {
 	struct sd_clock *clock = to_sd_clock(hw);
-	u32 val, sd_fc;
-	unsigned int i;
-
-	val = readl(clock->reg);
-
-	sd_fc = val & CPG_SD_FC_MASK;
-	for (i = 0; i < clock->div_num; i++)
-		if (sd_fc == (clock->div_table[i].val & CPG_SD_FC_MASK))
-			break;
-
-	if (i >= clock->div_num)
-		return -EINVAL;
+	u32 val = readl(clock->reg);
 
 	val &= ~(CPG_SD_STP_MASK);
-	val |= clock->div_table[i].val & CPG_SD_STP_MASK;
+	val |= clock->div_table[clock->cur_div_idx].val & CPG_SD_STP_MASK;
 
 	writel(val, clock->reg);
 
@@ -135,21 +125,9 @@ static unsigned long cpg_sd_clock_recalc_rate(struct clk_hw *hw,
 						unsigned long parent_rate)
 {
 	struct sd_clock *clock = to_sd_clock(hw);
-	unsigned long rate = parent_rate;
-	u32 val, sd_fc;
-	unsigned int i;
 
-	val = readl(clock->reg);
-
-	sd_fc = val & CPG_SD_FC_MASK;
-	for (i = 0; i < clock->div_num; i++)
-		if (sd_fc == (clock->div_table[i].val & CPG_SD_FC_MASK))
-			break;
-
-	if (i >= clock->div_num)
-		return -EINVAL;
-
-	return DIV_ROUND_CLOSEST(rate, clock->div_table[i].div);
+	return DIV_ROUND_CLOSEST(parent_rate,
+				 clock->div_table[clock->cur_div_idx].div);
 }
 
 static unsigned int cpg_sd_clock_calc_div(struct sd_clock *clock,
@@ -190,6 +168,8 @@ static int cpg_sd_clock_set_rate(struct clk_hw *hw, unsigned long rate,
 	if (i >= clock->div_num)
 		return -EINVAL;
 
+	clock->cur_div_idx = i;
+
 	val = readl(clock->reg);
 	val &= ~(CPG_SD_STP_MASK | CPG_SD_FC_MASK);
 	val |= clock->div_table[i].val & (CPG_SD_STP_MASK | CPG_SD_FC_MASK);
@@ -215,6 +195,7 @@ static struct clk * __init cpg_sd_clk_register(const struct cpg_core_clk *core,
 	struct sd_clock *clock;
 	struct clk *clk;
 	unsigned int i;
+	u32 sd_fc;
 
 	clock = kzalloc(sizeof(*clock), GFP_KERNEL);
 	if (!clock)
@@ -231,6 +212,18 @@ static struct clk * __init cpg_sd_clk_register(const struct cpg_core_clk *core,
 	clock->div_table = cpg_sd_div_table;
 	clock->div_num = ARRAY_SIZE(cpg_sd_div_table);
 
+	sd_fc = readl(clock->reg) & CPG_SD_FC_MASK;
+	for (i = 0; i < clock->div_num; i++)
+		if (sd_fc == (clock->div_table[i].val & CPG_SD_FC_MASK))
+			break;
+
+	if (WARN_ON(i >= clock->div_num)) {
+		kfree(clock);
+		return ERR_PTR(-EINVAL);
+	}
+
+	clock->cur_div_idx = i;
+
 	clock->div_max = clock->div_table[0].div;
 	clock->div_min = clock->div_max;
 	for (i = 1; i < clock->div_num; i++) {
@@ -279,7 +272,7 @@ struct clk * __init rcar_gen3_cpg_clk_register(struct device *dev,
 	unsigned int div = 1;
 	u32 value;
 
-	parent = clks[core->parent];
+	parent = clks[core->parent & 0xffff];	/* CLK_TYPE_PE uses high bits */
 	if (IS_ERR(parent))
 		return ERR_CAST(parent);
 
@@ -303,6 +296,7 @@ struct clk * __init rcar_gen3_cpg_clk_register(struct device *dev,
 
 	case CLK_TYPE_GEN3_PLL1:
 		mult = cpg_pll_config->pll1_mult;
+		div = cpg_pll_config->pll1_div;
 		break;
 
 	case CLK_TYPE_GEN3_PLL2:
@@ -320,6 +314,7 @@ struct clk * __init rcar_gen3_cpg_clk_register(struct device *dev,
 
 	case CLK_TYPE_GEN3_PLL3:
 		mult = cpg_pll_config->pll3_mult;
+		div = cpg_pll_config->pll3_div;
 		break;
 
 	case CLK_TYPE_GEN3_PLL4:
@@ -360,6 +355,24 @@ struct clk * __init rcar_gen3_cpg_clk_register(struct device *dev,
 			parent = clks[cpg_clk_extalr];
 		break;
 
+	case CLK_TYPE_GEN3_PE:
+		/*
+		 * Peripheral clock with a fixed divider, selectable between
+		 * clean and spread spectrum parents using MD12
+		 */
+		if (cpg_mode & BIT(12)) {
+			/* Clean */
+			div = core->div & 0xffff;
+		} else {
+			/* SCCG */
+			parent = clks[core->parent >> 16];
+			if (IS_ERR(parent))
+				return ERR_CAST(parent);
+			div = core->div >> 16;
+		}
+		mult = 1;
+		break;
+
 	default:
 		return ERR_PTR(-EINVAL);
 	}
diff --git a/drivers/clk/renesas/rcar-gen3-cpg.h b/drivers/clk/renesas/rcar-gen3-cpg.h
index 073be54b5d03..d756ef8b78eb 100644
--- a/drivers/clk/renesas/rcar-gen3-cpg.h
+++ b/drivers/clk/renesas/rcar-gen3-cpg.h
@@ -20,15 +20,24 @@ enum rcar_gen3_clk_types {
 	CLK_TYPE_GEN3_PLL4,
 	CLK_TYPE_GEN3_SD,
 	CLK_TYPE_GEN3_R,
+	CLK_TYPE_GEN3_PE,
 };
 
 #define DEF_GEN3_SD(_name, _id, _parent, _offset)	\
 	DEF_BASE(_name, _id, CLK_TYPE_GEN3_SD, _parent, .offset = _offset)
 
+#define DEF_GEN3_PE(_name, _id, _parent_sscg, _div_sscg, _parent_clean, \
+		    _div_clean) \
+	DEF_BASE(_name, _id, CLK_TYPE_GEN3_PE,			\
+		 (_parent_sscg) << 16 | (_parent_clean),	\
+		 .div = (_div_sscg) << 16 | (_div_clean))
+
 struct rcar_gen3_cpg_pll_config {
-	unsigned int extal_div;
-	unsigned int pll1_mult;
-	unsigned int pll3_mult;
+	u8 extal_div;
+	u8 pll1_mult;
+	u8 pll1_div;
+	u8 pll3_mult;
+	u8 pll3_div;
 };
 
 #define CPG_RCKCR	0x240
diff --git a/drivers/clk/renesas/rcar-usb2-clock-sel.c b/drivers/clk/renesas/rcar-usb2-clock-sel.c
new file mode 100644
index 000000000000..6cd030a58964
--- /dev/null
+++ b/drivers/clk/renesas/rcar-usb2-clock-sel.c
@@ -0,0 +1,188 @@
+/*
+ * Renesas R-Car USB2.0 clock selector
+ *
+ * Copyright (C) 2017 Renesas Electronics Corp.
+ *
+ * Based on renesas-cpg-mssr.c
+ *
+ * Copyright (C) 2015 Glider bvba
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ */
+
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/device.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/pm.h>
+#include <linux/pm_runtime.h>
+#include <linux/slab.h>
+
+#define USB20_CLKSET0		0x00
+#define CLKSET0_INTCLK_EN	BIT(11)
+#define CLKSET0_PRIVATE		BIT(0)
+#define CLKSET0_EXTAL_ONLY	(CLKSET0_INTCLK_EN | CLKSET0_PRIVATE)
+
+struct usb2_clock_sel_priv {
+	void __iomem *base;
+	struct clk_hw hw;
+	bool extal;
+	bool xtal;
+};
+#define to_priv(_hw)	container_of(_hw, struct usb2_clock_sel_priv, hw)
+
+static void usb2_clock_sel_enable_extal_only(struct usb2_clock_sel_priv *priv)
+{
+	u16 val = readw(priv->base + USB20_CLKSET0);
+
+	pr_debug("%s: enter %d %d %x\n", __func__,
+		 priv->extal, priv->xtal, val);
+
+	if (priv->extal && !priv->xtal && val != CLKSET0_EXTAL_ONLY)
+		writew(CLKSET0_EXTAL_ONLY, priv->base + USB20_CLKSET0);
+}
+
+static void usb2_clock_sel_disable_extal_only(struct usb2_clock_sel_priv *priv)
+{
+	if (priv->extal && !priv->xtal)
+		writew(CLKSET0_PRIVATE, priv->base + USB20_CLKSET0);
+}
+
+static int usb2_clock_sel_enable(struct clk_hw *hw)
+{
+	usb2_clock_sel_enable_extal_only(to_priv(hw));
+
+	return 0;
+}
+
+static void usb2_clock_sel_disable(struct clk_hw *hw)
+{
+	usb2_clock_sel_disable_extal_only(to_priv(hw));
+}
+
+/*
+ * This module seems a mux, but this driver assumes a gate because
+ * ehci/ohci platform drivers don't support clk_set_parent() for now.
+ * If this driver acts as a gate, ehci/ohci-platform drivers don't need
+ * any modification.
+ */
+static const struct clk_ops usb2_clock_sel_clock_ops = {
+	.enable = usb2_clock_sel_enable,
+	.disable = usb2_clock_sel_disable,
+};
+
+static const struct of_device_id rcar_usb2_clock_sel_match[] = {
+	{ .compatible = "renesas,rcar-gen3-usb2-clock-sel" },
+	{ }
+};
+
+static int rcar_usb2_clock_sel_suspend(struct device *dev)
+{
+	struct usb2_clock_sel_priv *priv = dev_get_drvdata(dev);
+
+	usb2_clock_sel_disable_extal_only(priv);
+	pm_runtime_put(dev);
+
+	return 0;
+}
+
+static int rcar_usb2_clock_sel_resume(struct device *dev)
+{
+	struct usb2_clock_sel_priv *priv = dev_get_drvdata(dev);
+
+	pm_runtime_get_sync(dev);
+	usb2_clock_sel_enable_extal_only(priv);
+
+	return 0;
+}
+
+static int rcar_usb2_clock_sel_remove(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct usb2_clock_sel_priv *priv = platform_get_drvdata(pdev);
+
+	of_clk_del_provider(dev->of_node);
+	clk_hw_unregister(&priv->hw);
+	pm_runtime_put(dev);
+	pm_runtime_disable(dev);
+
+	return 0;
+}
+
+static int rcar_usb2_clock_sel_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct usb2_clock_sel_priv *priv;
+	struct resource *res;
+	struct clk *clk;
+	struct clk_init_data init;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	priv->base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(priv->base))
+		return PTR_ERR(priv->base);
+
+	pm_runtime_enable(dev);
+	pm_runtime_get_sync(dev);
+
+	clk = devm_clk_get(dev, "usb_extal");
+	if (!IS_ERR(clk) && !clk_prepare_enable(clk)) {
+		priv->extal = !!clk_get_rate(clk);
+		clk_disable_unprepare(clk);
+	}
+	clk = devm_clk_get(dev, "usb_xtal");
+	if (!IS_ERR(clk) && !clk_prepare_enable(clk)) {
+		priv->xtal = !!clk_get_rate(clk);
+		clk_disable_unprepare(clk);
+	}
+
+	if (!priv->extal && !priv->xtal) {
+		dev_err(dev, "This driver needs usb_extal or usb_xtal\n");
+		return -ENOENT;
+	}
+
+	platform_set_drvdata(pdev, priv);
+	dev_set_drvdata(dev, priv);
+
+	init.name = "rcar_usb2_clock_sel";
+	init.ops = &usb2_clock_sel_clock_ops;
+	init.flags = 0;
+	init.parent_names = NULL;
+	init.num_parents = 0;
+	priv->hw.init = &init;
+
+	clk = clk_register(NULL, &priv->hw);
+	if (IS_ERR(clk))
+		return PTR_ERR(clk);
+
+	return of_clk_add_hw_provider(np, of_clk_hw_simple_get, &priv->hw);
+}
+
+static const struct dev_pm_ops rcar_usb2_clock_sel_pm_ops = {
+	.suspend	= rcar_usb2_clock_sel_suspend,
+	.resume		= rcar_usb2_clock_sel_resume,
+};
+
+static struct platform_driver rcar_usb2_clock_sel_driver = {
+	.driver		= {
+		.name	= "rcar-usb2-clock-sel",
+		.of_match_table = rcar_usb2_clock_sel_match,
+		.pm	= &rcar_usb2_clock_sel_pm_ops,
+	},
+	.probe		= rcar_usb2_clock_sel_probe,
+	.remove		= rcar_usb2_clock_sel_remove,
+};
+builtin_platform_driver(rcar_usb2_clock_sel_driver);
+
+MODULE_DESCRIPTION("Renesas R-Car USB2 clock selector Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/renesas/renesas-cpg-mssr.c b/drivers/clk/renesas/renesas-cpg-mssr.c
index 1f607c806f9b..e580a5e6346c 100644
--- a/drivers/clk/renesas/renesas-cpg-mssr.c
+++ b/drivers/clk/renesas/renesas-cpg-mssr.c
@@ -680,6 +680,12 @@ static const struct of_device_id cpg_mssr_match[] = {
 		.data = &r8a7796_cpg_mssr_info,
 	},
 #endif
+#ifdef CONFIG_CLK_R8A77995
+	{
+		.compatible = "renesas,r8a77995-cpg-mssr",
+		.data = &r8a77995_cpg_mssr_info,
+	},
+#endif
 	{ /* sentinel */ }
 };
 
diff --git a/drivers/clk/renesas/renesas-cpg-mssr.h b/drivers/clk/renesas/renesas-cpg-mssr.h
index 43d7c7f6832d..94b9071d1061 100644
--- a/drivers/clk/renesas/renesas-cpg-mssr.h
+++ b/drivers/clk/renesas/renesas-cpg-mssr.h
@@ -138,6 +138,7 @@ extern const struct cpg_mssr_info r8a7792_cpg_mssr_info;
 extern const struct cpg_mssr_info r8a7794_cpg_mssr_info;
 extern const struct cpg_mssr_info r8a7795_cpg_mssr_info;
 extern const struct cpg_mssr_info r8a7796_cpg_mssr_info;
+extern const struct cpg_mssr_info r8a77995_cpg_mssr_info;
 
 
     /*
diff --git a/drivers/clk/rockchip/clk-rk3128.c b/drivers/clk/rockchip/clk-rk3128.c
index e243f2eae68f..62d7854e4b87 100644
--- a/drivers/clk/rockchip/clk-rk3128.c
+++ b/drivers/clk/rockchip/clk-rk3128.c
@@ -201,7 +201,7 @@ static struct rockchip_clk_branch rk3128_uart2_fracmux __initdata =
 	MUX(SCLK_UART2, "sclk_uart2", mux_uart2_p, CLK_SET_RATE_PARENT,
 			RK2928_CLKSEL_CON(15), 8, 2, MFLAGS);
 
-static struct rockchip_clk_branch rk3128_clk_branches[] __initdata = {
+static struct rockchip_clk_branch common_clk_branches[] __initdata = {
 	/*
 	 * Clock-Architecture Diagram 1
 	 */
@@ -459,10 +459,6 @@ static struct rockchip_clk_branch rk3128_clk_branches[] __initdata = {
 			RK2928_CLKSEL_CON(2), 14, 2, MFLAGS, 8, 5, DFLAGS,
 			RK2928_CLKGATE_CON(10), 15, GFLAGS),
 
-	COMPOSITE(SCLK_SFC, "sclk_sfc", mux_sclk_sfc_src_p, 0,
-			RK2928_CLKSEL_CON(11), 14, 2, MFLAGS, 8, 5, DFLAGS,
-			RK2928_CLKGATE_CON(3), 15, GFLAGS),
-
 	COMPOSITE_NOMUX(PCLK_PMU_PRE, "pclk_pmu_pre", "cpll", 0,
 			RK2928_CLKSEL_CON(29), 8, 6, DFLAGS,
 			RK2928_CLKGATE_CON(1), 0, GFLAGS),
@@ -495,7 +491,6 @@ static struct rockchip_clk_branch rk3128_clk_branches[] __initdata = {
 	GATE(ACLK_DMAC, "aclk_dmac", "aclk_peri", 0, RK2928_CLKGATE_CON(5), 1, GFLAGS),
 	GATE(0, "aclk_peri_niu", "aclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(9), 15, GFLAGS),
 	GATE(0, "aclk_cpu_to_peri", "aclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(4), 2, GFLAGS),
-	GATE(HCLK_GPS, "hclk_gps", "aclk_peri", 0, RK2928_CLKGATE_CON(3), 14, GFLAGS),
 
 	GATE(HCLK_I2S_8CH, "hclk_i2s_8ch", "hclk_peri", 0, RK2928_CLKGATE_CON(7), 4, GFLAGS),
 	GATE(0, "hclk_peri_matrix", "hclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(4), 0, GFLAGS),
@@ -541,7 +536,6 @@ static struct rockchip_clk_branch rk3128_clk_branches[] __initdata = {
 	GATE(0, "hclk_rom", "hclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(5), 6, GFLAGS),
 	GATE(HCLK_CRYPTO, "hclk_crypto", "hclk_cpu", 0, RK2928_CLKGATE_CON(3), 5, GFLAGS),
 
-	GATE(PCLK_HDMI, "pclk_hdmi", "pclk_cpu", 0, RK2928_CLKGATE_CON(3), 8, GFLAGS),
 	GATE(PCLK_ACODEC, "pclk_acodec", "pclk_cpu", 0, RK2928_CLKGATE_CON(5), 14, GFLAGS),
 	GATE(0, "pclk_ddrupctl", "pclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(5), 7, GFLAGS),
 	GATE(0, "pclk_grf", "pclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(5), 4, GFLAGS),
@@ -561,6 +555,21 @@ static struct rockchip_clk_branch rk3128_clk_branches[] __initdata = {
 	MMC(SCLK_EMMC_SAMPLE,  "emmc_sample",  "sclk_emmc",  RK3228_EMMC_CON1,  0),
 };
 
+static struct rockchip_clk_branch rk3126_clk_branches[] __initdata = {
+	GATE(0, "pclk_stimer", "pclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(3), 15, GFLAGS),
+	GATE(0, "pclk_s_efuse", "pclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(3), 14, GFLAGS),
+	GATE(0, "pclk_sgrf", "pclk_cpu", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(3), 8, GFLAGS),
+};
+
+static struct rockchip_clk_branch rk3128_clk_branches[] __initdata = {
+	COMPOSITE(SCLK_SFC, "sclk_sfc", mux_sclk_sfc_src_p, 0,
+			RK2928_CLKSEL_CON(11), 14, 2, MFLAGS, 8, 5, DFLAGS,
+			RK2928_CLKGATE_CON(3), 15, GFLAGS),
+
+	GATE(HCLK_GPS, "hclk_gps", "aclk_peri", 0, RK2928_CLKGATE_CON(3), 14, GFLAGS),
+	GATE(PCLK_HDMI, "pclk_hdmi", "pclk_cpu", 0, RK2928_CLKGATE_CON(3), 8, GFLAGS),
+};
+
 static const char *const rk3128_critical_clocks[] __initconst = {
 	"aclk_cpu",
 	"hclk_cpu",
@@ -570,7 +579,7 @@ static const char *const rk3128_critical_clocks[] __initconst = {
 	"pclk_peri",
 };
 
-static void __init rk3128_clk_init(struct device_node *np)
+static struct rockchip_clk_provider *__init rk3128_common_clk_init(struct device_node *np)
 {
 	struct rockchip_clk_provider *ctx;
 	void __iomem *reg_base;
@@ -578,23 +587,21 @@ static void __init rk3128_clk_init(struct device_node *np)
 	reg_base = of_iomap(np, 0);
 	if (!reg_base) {
 		pr_err("%s: could not map cru region\n", __func__);
-		return;
+		return ERR_PTR(-ENOMEM);
 	}
 
 	ctx = rockchip_clk_init(np, reg_base, CLK_NR_CLKS);
 	if (IS_ERR(ctx)) {
 		pr_err("%s: rockchip clk init failed\n", __func__);
 		iounmap(reg_base);
-		return;
+		return ERR_PTR(-ENOMEM);
 	}
 
 	rockchip_clk_register_plls(ctx, rk3128_pll_clks,
 				   ARRAY_SIZE(rk3128_pll_clks),
 				   RK3128_GRF_SOC_STATUS0);
-	rockchip_clk_register_branches(ctx, rk3128_clk_branches,
-				  ARRAY_SIZE(rk3128_clk_branches));
-	rockchip_clk_protect_critical(rk3128_critical_clocks,
-				      ARRAY_SIZE(rk3128_critical_clocks));
+	rockchip_clk_register_branches(ctx, common_clk_branches,
+				  ARRAY_SIZE(common_clk_branches));
 
 	rockchip_clk_register_armclk(ctx, ARMCLK, "armclk",
 			mux_armclk_p, ARRAY_SIZE(mux_armclk_p),
@@ -606,6 +613,40 @@ static void __init rk3128_clk_init(struct device_node *np)
 
 	rockchip_register_restart_notifier(ctx, RK2928_GLB_SRST_FST, NULL);
 
+	return ctx;
+}
+
+static void __init rk3126_clk_init(struct device_node *np)
+{
+	struct rockchip_clk_provider *ctx;
+
+	ctx = rk3128_common_clk_init(np);
+	if (IS_ERR(ctx))
+		return;
+
+	rockchip_clk_register_branches(ctx, rk3126_clk_branches,
+				       ARRAY_SIZE(rk3126_clk_branches));
+	rockchip_clk_protect_critical(rk3128_critical_clocks,
+				      ARRAY_SIZE(rk3128_critical_clocks));
+
+	rockchip_clk_of_add_provider(np, ctx);
+}
+
+CLK_OF_DECLARE(rk3126_cru, "rockchip,rk3126-cru", rk3126_clk_init);
+
+static void __init rk3128_clk_init(struct device_node *np)
+{
+	struct rockchip_clk_provider *ctx;
+
+	ctx = rk3128_common_clk_init(np);
+	if (IS_ERR(ctx))
+		return;
+
+	rockchip_clk_register_branches(ctx, rk3128_clk_branches,
+				       ARRAY_SIZE(rk3128_clk_branches));
+	rockchip_clk_protect_critical(rk3128_critical_clocks,
+				      ARRAY_SIZE(rk3128_critical_clocks));
+
 	rockchip_clk_of_add_provider(np, ctx);
 }
 
diff --git a/drivers/clk/rockchip/clk-rk3228.c b/drivers/clk/rockchip/clk-rk3228.c
index bb405d9044a3..11e7f2d1c054 100644
--- a/drivers/clk/rockchip/clk-rk3228.c
+++ b/drivers/clk/rockchip/clk-rk3228.c
@@ -391,7 +391,7 @@ static struct rockchip_clk_branch rk3228_clk_branches[] __initdata = {
 			RK2928_CLKSEL_CON(11), 8, 2, MFLAGS, 0, 8, DFLAGS,
 			RK2928_CLKGATE_CON(2), 11, GFLAGS),
 
-	COMPOSITE_NODIV(0, "sclk_sdio_src", mux_mmc_src_p, 0,
+	COMPOSITE_NODIV(SCLK_SDIO_SRC, "sclk_sdio_src", mux_mmc_src_p, 0,
 			RK2928_CLKSEL_CON(11), 10, 2, MFLAGS,
 			RK2928_CLKGATE_CON(2), 13, GFLAGS),
 	DIV(SCLK_SDIO, "sclk_sdio", "sclk_sdio_src", 0,
diff --git a/drivers/clk/rockchip/clk-rv1108.c b/drivers/clk/rockchip/clk-rv1108.c
index 7c05ab366348..089cb17925e5 100644
--- a/drivers/clk/rockchip/clk-rv1108.c
+++ b/drivers/clk/rockchip/clk-rv1108.c
@@ -93,9 +93,24 @@ static struct rockchip_pll_rate_table rv1108_pll_rates[] = {
 	}
 
 static struct rockchip_cpuclk_rate_table rv1108_cpuclk_rates[] __initdata = {
-	RV1108_CPUCLK_RATE(816000000, 4),
-	RV1108_CPUCLK_RATE(600000000, 4),
-	RV1108_CPUCLK_RATE(312000000, 4),
+	RV1108_CPUCLK_RATE(1608000000, 7),
+	RV1108_CPUCLK_RATE(1512000000, 7),
+	RV1108_CPUCLK_RATE(1488000000, 5),
+	RV1108_CPUCLK_RATE(1416000000, 5),
+	RV1108_CPUCLK_RATE(1392000000, 5),
+	RV1108_CPUCLK_RATE(1296000000, 5),
+	RV1108_CPUCLK_RATE(1200000000, 5),
+	RV1108_CPUCLK_RATE(1104000000, 5),
+	RV1108_CPUCLK_RATE(1008000000, 5),
+	RV1108_CPUCLK_RATE(912000000, 5),
+	RV1108_CPUCLK_RATE(816000000, 3),
+	RV1108_CPUCLK_RATE(696000000, 3),
+	RV1108_CPUCLK_RATE(600000000, 3),
+	RV1108_CPUCLK_RATE(500000000, 3),
+	RV1108_CPUCLK_RATE(408000000, 1),
+	RV1108_CPUCLK_RATE(312000000, 1),
+	RV1108_CPUCLK_RATE(216000000, 1),
+	RV1108_CPUCLK_RATE(96000000, 1),
 };
 
 static const struct rockchip_cpuclk_reg_data rv1108_cpuclk_data = {
@@ -105,7 +120,7 @@ static const struct rockchip_cpuclk_reg_data rv1108_cpuclk_data = {
 	.mux_core_alt = 1,
 	.mux_core_main = 0,
 	.mux_core_shift = 8,
-	.mux_core_mask = 0x1,
+	.mux_core_mask = 0x3,
 };
 
 PNAME(mux_pll_p)		= { "xin24m", "xin24m"};
@@ -114,30 +129,42 @@ PNAME(mux_armclk_p)		= { "apll_core", "gpll_core", "dpll_core" };
 PNAME(mux_usb480m_pre_p)	= { "usbphy", "xin24m" };
 PNAME(mux_hdmiphy_phy_p)	= { "hdmiphy", "xin24m" };
 PNAME(mux_dclk_hdmiphy_pre_p)	= { "dclk_hdmiphy_src_gpll", "dclk_hdmiphy_src_dpll" };
-PNAME(mux_pll_src_4plls_p)	= { "dpll", "hdmiphy", "gpll", "usb480m" };
+PNAME(mux_pll_src_4plls_p)	= { "dpll", "gpll", "hdmiphy", "usb480m" };
 PNAME(mux_pll_src_3plls_p)	= { "apll", "gpll", "dpll" };
 PNAME(mux_pll_src_2plls_p)	= { "dpll", "gpll" };
 PNAME(mux_pll_src_apll_gpll_p)	= { "apll", "gpll" };
-PNAME(mux_aclk_peri_src_p)	= { "aclk_peri_src_dpll", "aclk_peri_src_gpll" };
+PNAME(mux_aclk_peri_src_p)	= { "aclk_peri_src_gpll", "aclk_peri_src_dpll" };
 PNAME(mux_aclk_bus_src_p)	= { "aclk_bus_src_gpll", "aclk_bus_src_apll", "aclk_bus_src_dpll" };
 PNAME(mux_mmc_src_p)		= { "dpll", "gpll", "xin24m", "usb480m" };
 PNAME(mux_pll_src_dpll_gpll_usb480m_p)	= { "dpll", "gpll", "usb480m" };
 PNAME(mux_uart0_p)		= { "uart0_src", "uart0_frac", "xin24m" };
 PNAME(mux_uart1_p)		= { "uart1_src", "uart1_frac", "xin24m" };
 PNAME(mux_uart2_p)		= { "uart2_src", "uart2_frac", "xin24m" };
-PNAME(mux_sclk_macphy_p)	= { "sclk_macphy_pre", "ext_gmac" };
+PNAME(mux_sclk_mac_p)		= { "sclk_mac_pre", "ext_gmac" };
 PNAME(mux_i2s0_pre_p)		= { "i2s0_src", "i2s0_frac", "ext_i2s", "xin12m" };
 PNAME(mux_i2s_out_p)		= { "i2s0_pre", "xin12m" };
-PNAME(mux_i2s1_p)		= { "i2s1_src", "i2s1_frac", "xin12m" };
-PNAME(mux_i2s2_p)		= { "i2s2_src", "i2s2_frac", "xin12m" };
+PNAME(mux_i2s1_p)		= { "i2s1_src", "i2s1_frac", "dummy", "xin12m" };
+PNAME(mux_i2s2_p)		= { "i2s2_src", "i2s2_frac", "dummy", "xin12m" };
+PNAME(mux_wifi_src_p)		= { "gpll", "xin24m" };
+PNAME(mux_cifout_src_p)	= { "hdmiphy", "gpll" };
+PNAME(mux_cifout_p)		= { "sclk_cifout_src", "xin24m" };
+PNAME(mux_sclk_cif0_src_p)	= { "pclk_vip", "clk_cif0_chn_out", "pclkin_cvbs2cif" };
+PNAME(mux_sclk_cif1_src_p)	= { "pclk_vip", "clk_cif1_chn_out", "pclkin_cvbs2cif" };
+PNAME(mux_sclk_cif2_src_p)	= { "pclk_vip", "clk_cif2_chn_out", "pclkin_cvbs2cif" };
+PNAME(mux_sclk_cif3_src_p)	= { "pclk_vip", "clk_cif3_chn_out", "pclkin_cvbs2cif" };
+PNAME(mux_dsp_src_p)		= { "dpll", "gpll", "apll", "usb480m" };
+PNAME(mux_dclk_hdmiphy_p)	= { "hdmiphy", "xin24m" };
+PNAME(mux_dclk_vop_p)		= { "dclk_hdmiphy", "dclk_vop_src" };
+PNAME(mux_hdmi_cec_src_p)		= { "dpll", "gpll", "xin24m" };
+PNAME(mux_cvbs_src_p)		= { "apll", "io_cvbs_clkin", "hdmiphy", "gpll" };
 
 static struct rockchip_pll_clock rv1108_pll_clks[] __initdata = {
 	[apll] = PLL(pll_rk3399, PLL_APLL, "apll", mux_pll_p, 0, RV1108_PLL_CON(0),
-		     RV1108_PLL_CON(3), 8, 31, 0, rv1108_pll_rates),
+		     RV1108_PLL_CON(3), 8, 0, 0, rv1108_pll_rates),
 	[dpll] = PLL(pll_rk3399, PLL_DPLL, "dpll", mux_pll_p, 0, RV1108_PLL_CON(8),
-		     RV1108_PLL_CON(11), 8, 31, 0, NULL),
+		     RV1108_PLL_CON(11), 8, 1, 0, NULL),
 	[gpll] = PLL(pll_rk3399, PLL_GPLL, "gpll", mux_pll_p, 0, RV1108_PLL_CON(16),
-		     RV1108_PLL_CON(19), 8, 31, ROCKCHIP_PLL_SYNC_RATE, rv1108_pll_rates),
+		     RV1108_PLL_CON(19), 8, 2, 0, rv1108_pll_rates),
 };
 
 #define MFLAGS CLK_MUX_HIWORD_MASK
@@ -170,10 +197,10 @@ static struct rockchip_clk_branch rv1108_i2s2_fracmux __initdata =
 			RV1108_CLKSEL_CON(7), 12, 2, MFLAGS);
 
 static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = {
-	MUX(0, "hdmi_phy", mux_hdmiphy_phy_p, CLK_SET_RATE_PARENT,
-			RV1108_MISC_CON, 13, 2, MFLAGS),
+	MUX(0, "hdmiphy", mux_hdmiphy_phy_p, CLK_SET_RATE_PARENT,
+			RV1108_MISC_CON, 13, 1, MFLAGS),
 	MUX(0, "usb480m", mux_usb480m_pre_p, CLK_SET_RATE_PARENT,
-			RV1108_MISC_CON, 15, 2, MFLAGS),
+			RV1108_MISC_CON, 15, 1, MFLAGS),
 	/*
 	 * Clock-Architecture Diagram 2
 	 */
@@ -197,50 +224,212 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = {
 			RV1108_CLKGATE_CON(11), 1, GFLAGS),
 
 	/* PD_RKVENC */
+	COMPOSITE(0, "aclk_rkvenc_pre", mux_pll_src_4plls_p, 0,
+			RV1108_CLKSEL_CON(37), 6, 2, MFLAGS, 0, 5, DFLAGS,
+			RV1108_CLKGATE_CON(8), 8, GFLAGS),
+	FACTOR_GATE(0, "hclk_rkvenc_pre", "aclk_rkvenc_pre", 0, 1, 4,
+			RV1108_CLKGATE_CON(8), 10, GFLAGS),
+	COMPOSITE(SCLK_VENC_CORE, "clk_venc_core", mux_pll_src_4plls_p, 0,
+			RV1108_CLKSEL_CON(37), 14, 2, MFLAGS, 8, 5, DFLAGS,
+			RV1108_CLKGATE_CON(8), 9, GFLAGS),
+	GATE(ACLK_RKVENC, "aclk_rkvenc", "aclk_rkvenc_pre", 0,
+			RV1108_CLKGATE_CON(19), 8, GFLAGS),
+	GATE(HCLK_RKVENC, "hclk_rkvenc", "hclk_rkvenc_pre", 0,
+			RV1108_CLKGATE_CON(19), 9, GFLAGS),
+	GATE(0, "aclk_rkvenc_niu", "aclk_rkvenc_pre", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(19), 11, GFLAGS),
+	GATE(0, "hclk_rkvenc_niu", "hclk_rkvenc_pre", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(19), 10, GFLAGS),
 
 	/* PD_RKVDEC */
+	COMPOSITE(SCLK_HEVC_CORE, "sclk_hevc_core", mux_pll_src_4plls_p, 0,
+			RV1108_CLKSEL_CON(36), 6, 2, MFLAGS, 0, 5, DFLAGS,
+			RV1108_CLKGATE_CON(8), 2, GFLAGS),
+	FACTOR_GATE(0, "hclk_rkvdec_pre", "sclk_hevc_core", 0, 1, 4,
+			RV1108_CLKGATE_CON(8), 10, GFLAGS),
+	COMPOSITE(SCLK_HEVC_CABAC, "clk_hevc_cabac", mux_pll_src_4plls_p, 0,
+			RV1108_CLKSEL_CON(35), 14, 2, MFLAGS, 8, 5, DFLAGS,
+			RV1108_CLKGATE_CON(8), 1, GFLAGS),
+
+	COMPOSITE(0, "aclk_rkvdec_pre", mux_pll_src_4plls_p, 0,
+			RV1108_CLKSEL_CON(35), 6, 2, MFLAGS, 0, 5, DFLAGS,
+			RV1108_CLKGATE_CON(8), 0, GFLAGS),
+	COMPOSITE(0, "aclk_vpu_pre", mux_pll_src_4plls_p, 0,
+			RV1108_CLKSEL_CON(36), 14, 2, MFLAGS, 8, 5, DFLAGS,
+			RV1108_CLKGATE_CON(8), 3, GFLAGS),
+	GATE(ACLK_RKVDEC, "aclk_rkvdec", "aclk_rkvdec_pre", 0,
+			RV1108_CLKGATE_CON(19), 0, GFLAGS),
+	GATE(ACLK_VPU, "aclk_vpu", "aclk_vpu_pre", 0,
+			RV1108_CLKGATE_CON(19), 1, GFLAGS),
+	GATE(HCLK_RKVDEC, "hclk_rkvdec", "hclk_rkvdec_pre", 0,
+			RV1108_CLKGATE_CON(19), 2, GFLAGS),
+	GATE(HCLK_VPU, "hclk_vpu", "hclk_rkvdec_pre", 0,
+			RV1108_CLKGATE_CON(19), 3, GFLAGS),
+	GATE(0, "aclk_rkvdec_niu", "aclk_rkvdec_pre", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(19), 4, GFLAGS),
+	GATE(0, "hclk_rkvdec_niu", "hclk_rkvdec_pre", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(19), 5, GFLAGS),
+	GATE(0, "aclk_vpu_niu", "aclk_vpu_pre", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(19), 6, GFLAGS),
 
 	/* PD_PMU_wrapper */
 	COMPOSITE_NOMUX(0, "pmu_24m_ena", "gpll", CLK_IGNORE_UNUSED,
 			RV1108_CLKSEL_CON(38), 0, 5, DFLAGS,
 			RV1108_CLKGATE_CON(8), 12, GFLAGS),
-	GATE(0, "pmu", "pmu_24m_ena", CLK_IGNORE_UNUSED,
+	GATE(0, "pclk_pmu", "pmu_24m_ena", CLK_IGNORE_UNUSED,
 			RV1108_CLKGATE_CON(10), 0, GFLAGS),
-	GATE(0, "intmem1", "pmu_24m_ena", CLK_IGNORE_UNUSED,
+	GATE(0, "pclk_intmem1", "pmu_24m_ena", CLK_IGNORE_UNUSED,
 			RV1108_CLKGATE_CON(10), 1, GFLAGS),
-	GATE(0, "gpio0_pmu", "pmu_24m_ena", CLK_IGNORE_UNUSED,
+	GATE(PCLK_GPIO0_PMU, "pclk_gpio0_pmu", "pmu_24m_ena", 0,
 			RV1108_CLKGATE_CON(10), 2, GFLAGS),
-	GATE(0, "pmugrf", "pmu_24m_ena", CLK_IGNORE_UNUSED,
+	GATE(0, "pclk_pmugrf", "pmu_24m_ena", CLK_IGNORE_UNUSED,
 			RV1108_CLKGATE_CON(10), 3, GFLAGS),
-	GATE(0, "pmu_noc", "pmu_24m_ena", CLK_IGNORE_UNUSED,
+	GATE(0, "pclk_pmu_niu", "pmu_24m_ena", CLK_IGNORE_UNUSED,
 			RV1108_CLKGATE_CON(10), 4, GFLAGS),
-	GATE(0, "i2c0_pmu_pclk", "pmu_24m_ena", CLK_IGNORE_UNUSED,
+	GATE(PCLK_I2C0_PMU, "pclk_i2c0_pmu", "pmu_24m_ena", 0,
 			RV1108_CLKGATE_CON(10), 5, GFLAGS),
-	GATE(0, "pwm0_pmu_pclk", "pmu_24m_ena", CLK_IGNORE_UNUSED,
+	GATE(PCLK_PWM0_PMU, "pclk_pwm0_pmu", "pmu_24m_ena", 0,
 			RV1108_CLKGATE_CON(10), 6, GFLAGS),
-	COMPOSITE(0, "pwm0_pmu_clk", mux_pll_src_2plls_p, CLK_IGNORE_UNUSED,
+	COMPOSITE(SCLK_PWM0_PMU, "sclk_pwm0_pmu", mux_pll_src_2plls_p, 0,
 			RV1108_CLKSEL_CON(12), 7, 1, MFLAGS, 0, 7, DFLAGS,
 			RV1108_CLKGATE_CON(8), 15, GFLAGS),
-	COMPOSITE(0, "i2c0_pmu_clk", mux_pll_src_2plls_p, CLK_IGNORE_UNUSED,
+	COMPOSITE(SCLK_I2C0_PMU, "sclk_i2c0_pmu", mux_pll_src_2plls_p, 0,
 			RV1108_CLKSEL_CON(19), 7, 1, MFLAGS, 0, 7, DFLAGS,
 			RV1108_CLKGATE_CON(8), 14, GFLAGS),
 	GATE(0, "pvtm_pmu", "xin24m", CLK_IGNORE_UNUSED,
 			RV1108_CLKGATE_CON(8), 13, GFLAGS),
 
 	/*
+	 * Clock-Architecture Diagram 3
+	 */
+	COMPOSITE(SCLK_WIFI, "sclk_wifi", mux_wifi_src_p, 0,
+			RV1108_CLKSEL_CON(28), 15, 1, MFLAGS, 8, 6, DFLAGS,
+			RV1108_CLKGATE_CON(9), 8, GFLAGS),
+	COMPOSITE_NODIV(0, "sclk_cifout_src", mux_cifout_src_p, 0,
+			RV1108_CLKSEL_CON(40), 8, 1, MFLAGS,
+			RV1108_CLKGATE_CON(9), 11, GFLAGS),
+	COMPOSITE_NOGATE(SCLK_CIFOUT, "sclk_cifout", mux_cifout_p, 0,
+			RV1108_CLKSEL_CON(40), 12, 1, MFLAGS, 0, 5, DFLAGS),
+	COMPOSITE_NOMUX(SCLK_MIPI_CSI_OUT, "sclk_mipi_csi_out", "xin24m", 0,
+			RV1108_CLKSEL_CON(41), 0, 5, DFLAGS,
+			RV1108_CLKGATE_CON(9), 12, GFLAGS),
+
+	GATE(0, "pclk_acodecphy", "pclk_top_pre", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(14), 6, GFLAGS),
+	GATE(0, "pclk_usbgrf", "pclk_top_pre", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(14), 14, GFLAGS),
+
+	GATE(ACLK_CIF0, "aclk_cif0", "aclk_vio1_pre", 0,
+			RV1108_CLKGATE_CON(18), 10, GFLAGS),
+	GATE(HCLK_CIF0, "hclk_cif0", "hclk_vio_pre", 0,
+			RV1108_CLKGATE_CON(18), 10, GFLAGS),
+	COMPOSITE_NODIV(SCLK_CIF0, "sclk_cif0", mux_sclk_cif0_src_p, 0,
+			RV1108_CLKSEL_CON(31), 0, 2, MFLAGS,
+			RV1108_CLKGATE_CON(7), 9, GFLAGS),
+	GATE(ACLK_CIF1, "aclk_cif1", "aclk_vio1_pre", 0,
+			RV1108_CLKGATE_CON(17), 6, GFLAGS),
+	GATE(HCLK_CIF1, "hclk_cif1", "hclk_vio_pre", 0,
+			RV1108_CLKGATE_CON(17), 7, GFLAGS),
+	COMPOSITE_NODIV(SCLK_CIF1, "sclk_cif1", mux_sclk_cif1_src_p, 0,
+			RV1108_CLKSEL_CON(31), 2, 2, MFLAGS,
+			RV1108_CLKGATE_CON(7), 10, GFLAGS),
+	GATE(ACLK_CIF2, "aclk_cif2", "aclk_vio1_pre", 0,
+			RV1108_CLKGATE_CON(17), 8, GFLAGS),
+	GATE(HCLK_CIF2, "hclk_cif2", "hclk_vio_pre", 0,
+			RV1108_CLKGATE_CON(17), 9, GFLAGS),
+	COMPOSITE_NODIV(SCLK_CIF2, "sclk_cif2", mux_sclk_cif2_src_p, 0,
+			RV1108_CLKSEL_CON(31), 4, 2, MFLAGS,
+			RV1108_CLKGATE_CON(7), 11, GFLAGS),
+	GATE(ACLK_CIF3, "aclk_cif3", "aclk_vio1_pre", 0,
+			RV1108_CLKGATE_CON(17), 10, GFLAGS),
+	GATE(HCLK_CIF3, "hclk_cif3", "hclk_vio_pre", 0,
+			RV1108_CLKGATE_CON(17), 11, GFLAGS),
+	COMPOSITE_NODIV(SCLK_CIF3, "sclk_cif3", mux_sclk_cif3_src_p, 0,
+			RV1108_CLKSEL_CON(31), 6, 2, MFLAGS,
+			RV1108_CLKGATE_CON(7), 12, GFLAGS),
+	GATE(0, "pclk_cif1to4", "pclk_vip", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(7), 8, GFLAGS),
+
+	/* PD_DSP_wrapper */
+	COMPOSITE(SCLK_DSP, "sclk_dsp", mux_dsp_src_p, 0,
+			RV1108_CLKSEL_CON(42), 8, 2, MFLAGS, 0, 5, DFLAGS,
+			RV1108_CLKGATE_CON(9), 0, GFLAGS),
+	GATE(0, "clk_dsp_sys_wd", "sclk_dsp", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(16), 0, GFLAGS),
+	GATE(0, "clk_dsp_epp_wd", "sclk_dsp", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(16), 1, GFLAGS),
+	GATE(0, "clk_dsp_edp_wd", "sclk_dsp", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(16), 2, GFLAGS),
+	GATE(0, "clk_dsp_iop_wd", "sclk_dsp", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(16), 3, GFLAGS),
+	GATE(0, "clk_dsp_free", "sclk_dsp", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(16), 13, GFLAGS),
+	COMPOSITE_NOMUX(SCLK_DSP_IOP, "sclk_dsp_iop", "sclk_dsp", 0,
+			RV1108_CLKSEL_CON(44), 0, 5, DFLAGS,
+			RV1108_CLKGATE_CON(9), 1, GFLAGS),
+	COMPOSITE_NOMUX(SCLK_DSP_EPP, "sclk_dsp_epp", "sclk_dsp", 0,
+			RV1108_CLKSEL_CON(44), 8, 5, DFLAGS,
+			RV1108_CLKGATE_CON(9), 2, GFLAGS),
+	COMPOSITE_NOMUX(SCLK_DSP_EDP, "sclk_dsp_edp", "sclk_dsp", 0,
+			RV1108_CLKSEL_CON(45), 0, 5, DFLAGS,
+			RV1108_CLKGATE_CON(9), 3, GFLAGS),
+	COMPOSITE_NOMUX(SCLK_DSP_EDAP, "sclk_dsp_edap", "sclk_dsp", 0,
+			RV1108_CLKSEL_CON(45), 8, 5, DFLAGS,
+			RV1108_CLKGATE_CON(9), 4, GFLAGS),
+	GATE(0, "pclk_dsp_iop_niu", "sclk_dsp_iop", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(16), 4, GFLAGS),
+	GATE(0, "aclk_dsp_epp_niu", "sclk_dsp_epp", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(16), 5, GFLAGS),
+	GATE(0, "aclk_dsp_edp_niu", "sclk_dsp_edp", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(16), 6, GFLAGS),
+	GATE(0, "pclk_dsp_dbg_niu", "sclk_dsp", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(16), 7, GFLAGS),
+	GATE(0, "aclk_dsp_edap_niu", "sclk_dsp_edap", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(16), 14, GFLAGS),
+	COMPOSITE_NOMUX(SCLK_DSP_PFM, "sclk_dsp_pfm", "sclk_dsp", 0,
+			RV1108_CLKSEL_CON(43), 0, 5, DFLAGS,
+			RV1108_CLKGATE_CON(9), 5, GFLAGS),
+	COMPOSITE_NOMUX(PCLK_DSP_CFG, "pclk_dsp_cfg", "sclk_dsp", 0,
+			RV1108_CLKSEL_CON(43), 8, 5, DFLAGS,
+			RV1108_CLKGATE_CON(9), 6, GFLAGS),
+	GATE(0, "pclk_dsp_cfg_niu", "pclk_dsp_cfg", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(16), 8, GFLAGS),
+	GATE(0, "pclk_dsp_pfm_mon", "pclk_dsp_cfg", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(16), 9, GFLAGS),
+	GATE(0, "pclk_intc", "pclk_dsp_cfg", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(16), 10, GFLAGS),
+	GATE(0, "pclk_dsp_grf", "pclk_dsp_cfg", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(16), 11, GFLAGS),
+	GATE(0, "pclk_mailbox", "pclk_dsp_cfg", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(16), 12, GFLAGS),
+	GATE(0, "aclk_dsp_epp_perf", "sclk_dsp_epp", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(16), 15, GFLAGS),
+	GATE(0, "aclk_dsp_edp_perf", "sclk_dsp_edp", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(11), 8, GFLAGS),
+
+	/*
 	 * Clock-Architecture Diagram 4
 	 */
-	COMPOSITE(0, "aclk_vio0_2wrap_occ", mux_pll_src_4plls_p, CLK_IGNORE_UNUSED,
+	COMPOSITE(0, "aclk_vio0_pre", mux_pll_src_4plls_p, CLK_IGNORE_UNUSED,
 			RV1108_CLKSEL_CON(28), 6, 2, MFLAGS, 0, 5, DFLAGS,
 			RV1108_CLKGATE_CON(6), 0, GFLAGS),
-	GATE(0, "aclk_vio0_pre", "aclk_vio0_2wrap_occ", CLK_IGNORE_UNUSED,
+	GATE(ACLK_VIO0, "aclk_vio0", "aclk_vio0_pre", 0,
 			RV1108_CLKGATE_CON(17), 0, GFLAGS),
 	COMPOSITE_NOMUX(0, "hclk_vio_pre", "aclk_vio0_pre", 0,
 			RV1108_CLKSEL_CON(29), 0, 5, DFLAGS,
 			RV1108_CLKGATE_CON(7), 2, GFLAGS),
+	GATE(HCLK_VIO, "hclk_vio", "hclk_vio_pre", 0,
+			RV1108_CLKGATE_CON(17), 2, GFLAGS),
 	COMPOSITE_NOMUX(0, "pclk_vio_pre", "aclk_vio0_pre", 0,
 			RV1108_CLKSEL_CON(29), 8, 5, DFLAGS,
 			RV1108_CLKGATE_CON(7), 3, GFLAGS),
+	GATE(PCLK_VIO, "pclk_vio", "pclk_vio_pre", 0,
+			RV1108_CLKGATE_CON(17), 3, GFLAGS),
+	COMPOSITE(0, "aclk_vio1_pre", mux_pll_src_4plls_p, CLK_IGNORE_UNUSED,
+			RV1108_CLKSEL_CON(28), 14, 2, MFLAGS, 8, 5, DFLAGS,
+			RV1108_CLKGATE_CON(6), 1, GFLAGS),
+	GATE(ACLK_VIO1, "aclk_vio1", "aclk_vio1_pre", 0,
+			RV1108_CLKGATE_CON(17), 1, GFLAGS),
 
 	INVERTER(0, "pclk_vip", "ext_vip",
 			RV1108_CLKSEL_CON(31), 8, IFLAGS),
@@ -252,8 +441,63 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = {
 			RV1108_CLKGATE_CON(6), 5, GFLAGS),
 	GATE(0, "dclk_hdmiphy_src_dpll", "dpll", CLK_IGNORE_UNUSED,
 			RV1108_CLKGATE_CON(6), 4, GFLAGS),
-	COMPOSITE_NOGATE(0, "dclk_hdmiphy", mux_dclk_hdmiphy_pre_p, 0,
-			RV1108_CLKSEL_CON(32), 6, 2, MFLAGS, 8, 6, DFLAGS),
+	COMPOSITE_NOGATE(0, "dclk_hdmiphy_pre", mux_dclk_hdmiphy_pre_p, 0,
+			RV1108_CLKSEL_CON(32), 6, 1, MFLAGS, 8, 6, DFLAGS),
+	COMPOSITE_NOGATE(DCLK_VOP_SRC, "dclk_vop_src", mux_dclk_hdmiphy_pre_p, 0,
+			RV1108_CLKSEL_CON(32), 6, 1, MFLAGS, 0, 6, DFLAGS),
+	MUX(DCLK_HDMIPHY, "dclk_hdmiphy", mux_dclk_hdmiphy_p, CLK_SET_RATE_PARENT,
+			RV1108_CLKSEL_CON(32), 15, 1, MFLAGS),
+	MUX(DCLK_VOP, "dclk_vop", mux_dclk_vop_p, CLK_SET_RATE_PARENT,
+			RV1108_CLKSEL_CON(32), 7, 1, MFLAGS),
+	GATE(ACLK_VOP, "aclk_vop", "aclk_vio0_pre", 0,
+			RV1108_CLKGATE_CON(18), 0, GFLAGS),
+	GATE(HCLK_VOP, "hclk_vop", "hclk_vio_pre", 0,
+			RV1108_CLKGATE_CON(18), 1, GFLAGS),
+	GATE(ACLK_IEP, "aclk_iep", "aclk_vio0_pre", 0,
+			RV1108_CLKGATE_CON(18), 2, GFLAGS),
+	GATE(HCLK_IEP, "hclk_iep", "hclk_vio_pre", 0,
+			RV1108_CLKGATE_CON(18), 3, GFLAGS),
+
+	GATE(ACLK_RGA, "aclk_rga", "aclk_vio1_pre", 0,
+			RV1108_CLKGATE_CON(18), 4, GFLAGS),
+	GATE(HCLK_RGA, "hclk_rga", "hclk_vio_pre", 0,
+			RV1108_CLKGATE_CON(18), 5, GFLAGS),
+	COMPOSITE(SCLK_RGA, "sclk_rga", mux_pll_src_4plls_p, 0,
+			RV1108_CLKSEL_CON(33), 6, 2, MFLAGS, 0, 5, DFLAGS,
+			RV1108_CLKGATE_CON(6), 6, GFLAGS),
+
+	COMPOSITE(SCLK_CVBS_HOST, "sclk_cvbs_host", mux_cvbs_src_p, 0,
+			RV1108_CLKSEL_CON(33), 13, 2, MFLAGS, 8, 5, DFLAGS,
+			RV1108_CLKGATE_CON(6), 7, GFLAGS),
+	FACTOR(0, "sclk_cvbs_27m", "sclk_cvbs_host", 0, 1, 2),
+
+	GATE(SCLK_HDMI_SFR, "sclk_hdmi_sfr", "xin24m", 0,
+			RV1108_CLKGATE_CON(6), 8, GFLAGS),
+
+	COMPOSITE(SCLK_HDMI_CEC, "sclk_hdmi_cec", mux_hdmi_cec_src_p, 0,
+			RV1108_CLKSEL_CON(34), 14, 2, MFLAGS, 0, 14, DFLAGS,
+			RV1108_CLKGATE_CON(6), 9, GFLAGS),
+	GATE(PCLK_MIPI_DSI, "pclk_mipi_dsi", "pclk_vio_pre", 0,
+			RV1108_CLKGATE_CON(18), 8, GFLAGS),
+	GATE(PCLK_HDMI_CTRL, "pclk_hdmi_ctrl", "pclk_vio_pre", 0,
+			RV1108_CLKGATE_CON(18), 9, GFLAGS),
+
+	GATE(ACLK_ISP, "aclk_isp", "aclk_vio1_pre", 0,
+			RV1108_CLKGATE_CON(18), 12, GFLAGS),
+	GATE(HCLK_ISP, "hclk_isp", "hclk_vio_pre", 0,
+			RV1108_CLKGATE_CON(18), 11, GFLAGS),
+	COMPOSITE(SCLK_ISP, "sclk_isp", mux_pll_src_4plls_p, 0,
+			RV1108_CLKSEL_CON(30), 14, 2, MFLAGS, 8, 5, DFLAGS,
+			RV1108_CLKGATE_CON(6), 3, GFLAGS),
+
+	GATE(0, "clk_dsiphy24m", "xin24m", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(9), 10, GFLAGS),
+	GATE(0, "pclk_vdacphy", "pclk_top_pre", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(14), 9, GFLAGS),
+	GATE(0, "pclk_mipi_dsiphy", "pclk_top_pre", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(14), 11, GFLAGS),
+	GATE(0, "pclk_mipi_csiphy", "pclk_top_pre", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(14), 12, GFLAGS),
 
 	/*
 	 * Clock-Architecture Diagram 5
@@ -261,10 +505,11 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = {
 
 	FACTOR(0, "xin12m", "xin24m", 0, 1, 2),
 
-	COMPOSITE(0, "i2s0_src", mux_pll_src_2plls_p, 0,
+
+	COMPOSITE(SCLK_I2S0_SRC, "i2s0_src", mux_pll_src_2plls_p, 0,
 			RV1108_CLKSEL_CON(5), 8, 1, MFLAGS, 0, 7, DFLAGS,
 			RV1108_CLKGATE_CON(2), 0, GFLAGS),
-	COMPOSITE_FRACMUX(0, "i2s1_frac", "i2s1_src", CLK_SET_RATE_PARENT,
+	COMPOSITE_FRACMUX(0, "i2s0_frac", "i2s0_src", CLK_SET_RATE_PARENT,
 			RV1108_CLKSEL_CON(8), 0,
 			RV1108_CLKGATE_CON(2), 1, GFLAGS,
 			&rv1108_i2s0_fracmux),
@@ -274,7 +519,7 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = {
 			RV1108_CLKSEL_CON(5), 15, 1, MFLAGS,
 			RV1108_CLKGATE_CON(2), 3, GFLAGS),
 
-	COMPOSITE(0, "i2s1_src", mux_pll_src_2plls_p, 0,
+	COMPOSITE(SCLK_I2S1_SRC, "i2s1_src", mux_pll_src_2plls_p, 0,
 			RV1108_CLKSEL_CON(6), 8, 1, MFLAGS, 0, 7, DFLAGS,
 			RV1108_CLKGATE_CON(2), 4, GFLAGS),
 	COMPOSITE_FRACMUX(0, "i2s1_frac", "i2s1_src", CLK_SET_RATE_PARENT,
@@ -284,7 +529,7 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = {
 	GATE(SCLK_I2S1, "sclk_i2s1", "i2s1_pre", CLK_SET_RATE_PARENT,
 			RV1108_CLKGATE_CON(2), 6, GFLAGS),
 
-	COMPOSITE(0, "i2s2_src", mux_pll_src_2plls_p, 0,
+	COMPOSITE(SCLK_I2S2_SRC, "i2s2_src", mux_pll_src_2plls_p, 0,
 			RV1108_CLKSEL_CON(7), 8, 1, MFLAGS, 0, 7, DFLAGS,
 			RV1108_CLKGATE_CON(3), 8, GFLAGS),
 	COMPOSITE_FRACMUX(0, "i2s2_frac", "i2s2_src", CLK_SET_RATE_PARENT,
@@ -303,32 +548,53 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = {
 			RV1108_CLKGATE_CON(1), 2, GFLAGS),
 	COMPOSITE_NOGATE(ACLK_PRE, "aclk_bus_pre", mux_aclk_bus_src_p, 0,
 			RV1108_CLKSEL_CON(2), 8, 2, MFLAGS, 0, 5, DFLAGS),
-	COMPOSITE_NOMUX(0, "hclk_bus_pre", "aclk_bus_2wrap_occ", 0,
+	COMPOSITE_NOMUX(HCLK_BUS, "hclk_bus_pre", "aclk_bus_pre", 0,
 			RV1108_CLKSEL_CON(3), 0, 5, DFLAGS,
 			RV1108_CLKGATE_CON(1), 4, GFLAGS),
-	COMPOSITE_NOMUX(0, "pclken_bus", "aclk_bus_2wrap_occ", 0,
+	COMPOSITE_NOMUX(0, "pclk_bus_pre", "aclk_bus_pre", 0,
 			RV1108_CLKSEL_CON(3), 8, 5, DFLAGS,
 			RV1108_CLKGATE_CON(1), 5, GFLAGS),
-	GATE(0, "pclk_bus_pre", "pclken_bus", CLK_IGNORE_UNUSED,
+	GATE(PCLK_BUS, "pclk_bus", "pclk_bus_pre", 0,
 			RV1108_CLKGATE_CON(1), 6, GFLAGS),
-	GATE(0, "pclk_top_pre", "pclken_bus", CLK_IGNORE_UNUSED,
+	GATE(0, "pclk_top_pre", "pclk_bus_pre", CLK_IGNORE_UNUSED,
 			RV1108_CLKGATE_CON(1), 7, GFLAGS),
-	GATE(0, "pclk_ddr_pre", "pclken_bus", CLK_IGNORE_UNUSED,
+	GATE(0, "pclk_ddr_pre", "pclk_bus_pre", CLK_IGNORE_UNUSED,
 			RV1108_CLKGATE_CON(1), 8, GFLAGS),
-	GATE(0, "clk_timer0", "mux_pll_p", CLK_IGNORE_UNUSED,
+	GATE(SCLK_TIMER0, "clk_timer0", "xin24m", 0,
 			RV1108_CLKGATE_CON(1), 9, GFLAGS),
-	GATE(0, "clk_timer1", "mux_pll_p", CLK_IGNORE_UNUSED,
+	GATE(SCLK_TIMER1, "clk_timer1", "xin24m", CLK_IGNORE_UNUSED,
 			RV1108_CLKGATE_CON(1), 10, GFLAGS),
-	GATE(0, "pclk_timer", "pclk_bus_pre", CLK_IGNORE_UNUSED,
+	GATE(PCLK_TIMER, "pclk_timer", "pclk_bus_pre", CLK_IGNORE_UNUSED,
 			RV1108_CLKGATE_CON(13), 4, GFLAGS),
 
-	COMPOSITE(0, "uart0_src", mux_pll_src_dpll_gpll_usb480m_p, CLK_IGNORE_UNUSED,
+	GATE(HCLK_I2S0_8CH, "hclk_i2s0_8ch", "hclk_bus_pre", 0,
+			RV1108_CLKGATE_CON(12), 7, GFLAGS),
+	GATE(HCLK_I2S1_2CH, "hclk_i2s1_2ch", "hclk_bus_pre", 0,
+			RV1108_CLKGATE_CON(12), 8, GFLAGS),
+	GATE(HCLK_I2S2_2CH, "hclk_i2s2_2ch", "hclk_bus_pre", 0,
+			RV1108_CLKGATE_CON(12), 9, GFLAGS),
+
+	GATE(HCLK_CRYPTO_MST, "hclk_crypto_mst", "hclk_bus_pre", 0,
+			RV1108_CLKGATE_CON(12), 10, GFLAGS),
+	GATE(HCLK_CRYPTO_SLV, "hclk_crypto_slv", "hclk_bus_pre", 0,
+			RV1108_CLKGATE_CON(12), 11, GFLAGS),
+	COMPOSITE(SCLK_CRYPTO, "sclk_crypto", mux_pll_src_2plls_p, 0,
+			RV1108_CLKSEL_CON(11), 7, 1, MFLAGS, 0, 5, DFLAGS,
+			RV1108_CLKGATE_CON(2), 12, GFLAGS),
+
+	COMPOSITE(SCLK_SPI, "sclk_spi", mux_pll_src_2plls_p, 0,
+			RV1108_CLKSEL_CON(11), 15, 1, MFLAGS, 8, 5, DFLAGS,
+			RV1108_CLKGATE_CON(3), 0, GFLAGS),
+	GATE(PCLK_SPI, "pclk_spi", "pclk_bus_pre", 0,
+			RV1108_CLKGATE_CON(13), 5, GFLAGS),
+
+	COMPOSITE(SCLK_UART0_SRC, "uart0_src", mux_pll_src_dpll_gpll_usb480m_p, CLK_IGNORE_UNUSED,
 			RV1108_CLKSEL_CON(13), 12, 2, MFLAGS, 0, 7, DFLAGS,
 			RV1108_CLKGATE_CON(3), 1, GFLAGS),
-	COMPOSITE(0, "uart1_src", mux_pll_src_dpll_gpll_usb480m_p, CLK_IGNORE_UNUSED,
+	COMPOSITE(SCLK_UART1_SRC, "uart1_src", mux_pll_src_dpll_gpll_usb480m_p, CLK_IGNORE_UNUSED,
 			RV1108_CLKSEL_CON(14), 12, 2, MFLAGS, 0, 7, DFLAGS,
 			RV1108_CLKGATE_CON(3), 3, GFLAGS),
-	COMPOSITE(0, "uart21_src", mux_pll_src_dpll_gpll_usb480m_p, CLK_IGNORE_UNUSED,
+	COMPOSITE(SCLK_UART2_SRC, "uart2_src", mux_pll_src_dpll_gpll_usb480m_p, CLK_IGNORE_UNUSED,
 			RV1108_CLKSEL_CON(15), 12, 2, MFLAGS, 0, 7, DFLAGS,
 			RV1108_CLKGATE_CON(3), 5, GFLAGS),
 
@@ -344,44 +610,58 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = {
 			RV1108_CLKSEL_CON(18), 0,
 			RV1108_CLKGATE_CON(3), 6, GFLAGS,
 			&rv1108_uart2_fracmux),
-	GATE(PCLK_UART0, "pclk_uart0", "pclk_bus_pre", CLK_IGNORE_UNUSED,
+	GATE(PCLK_UART0, "pclk_uart0", "pclk_bus_pre", 0,
 			RV1108_CLKGATE_CON(13), 10, GFLAGS),
-	GATE(PCLK_UART1, "pclk_uart1", "pclk_bus_pre", CLK_IGNORE_UNUSED,
+	GATE(PCLK_UART1, "pclk_uart1", "pclk_bus_pre", 0,
 			RV1108_CLKGATE_CON(13), 11, GFLAGS),
-	GATE(PCLK_UART2, "pclk_uart2", "pclk_bus_pre", CLK_IGNORE_UNUSED,
+	GATE(PCLK_UART2, "pclk_uart2", "pclk_bus_pre", 0,
 			RV1108_CLKGATE_CON(13), 12, GFLAGS),
 
-	COMPOSITE(0, "clk_i2c1", mux_pll_src_2plls_p, CLK_IGNORE_UNUSED,
-			RV1108_CLKSEL_CON(19), 15, 2, MFLAGS, 8, 7, DFLAGS,
+	COMPOSITE(SCLK_I2C1, "clk_i2c1", mux_pll_src_2plls_p, 0,
+			RV1108_CLKSEL_CON(19), 15, 1, MFLAGS, 8, 7, DFLAGS,
 			RV1108_CLKGATE_CON(3), 7, GFLAGS),
-	COMPOSITE(0, "clk_i2c2", mux_pll_src_2plls_p, CLK_IGNORE_UNUSED,
-			RV1108_CLKSEL_CON(20), 7, 2, MFLAGS, 0, 7, DFLAGS,
+	COMPOSITE(SCLK_I2C2, "clk_i2c2", mux_pll_src_2plls_p, 0,
+			RV1108_CLKSEL_CON(20), 7, 1, MFLAGS, 0, 7, DFLAGS,
 			RV1108_CLKGATE_CON(3), 8, GFLAGS),
-	COMPOSITE(0, "clk_i2c3", mux_pll_src_2plls_p, CLK_IGNORE_UNUSED,
-			RV1108_CLKSEL_CON(20), 15, 2, MFLAGS, 8, 7, DFLAGS,
+	COMPOSITE(SCLK_I2C3, "clk_i2c3", mux_pll_src_2plls_p, 0,
+			RV1108_CLKSEL_CON(20), 15, 1, MFLAGS, 8, 7, DFLAGS,
 			RV1108_CLKGATE_CON(3), 9, GFLAGS),
-	GATE(0, "pclk_i2c1", "pclk_bus_pre", CLK_IGNORE_UNUSED,
+	GATE(PCLK_I2C1, "pclk_i2c1", "pclk_bus_pre", 0,
 			RV1108_CLKGATE_CON(13), 0, GFLAGS),
-	GATE(0, "pclk_i2c2", "pclk_bus_pre", CLK_IGNORE_UNUSED,
+	GATE(PCLK_I2C2, "pclk_i2c2", "pclk_bus_pre", 0,
 			RV1108_CLKGATE_CON(13), 1, GFLAGS),
-	GATE(0, "pclk_i2c3", "pclk_bus_pre", CLK_IGNORE_UNUSED,
+	GATE(PCLK_I2C3, "pclk_i2c3", "pclk_bus_pre", 0,
 			RV1108_CLKGATE_CON(13), 2, GFLAGS),
-	COMPOSITE(0, "clk_pwm1", mux_pll_src_2plls_p, CLK_IGNORE_UNUSED,
+	COMPOSITE(SCLK_PWM, "clk_pwm", mux_pll_src_2plls_p, 0,
 			RV1108_CLKSEL_CON(12), 15, 2, MFLAGS, 8, 7, DFLAGS,
 			RV1108_CLKGATE_CON(3), 10, GFLAGS),
-	GATE(0, "pclk_pwm1", "pclk_bus_pre", CLK_IGNORE_UNUSED,
+	GATE(PCLK_PWM, "pclk_pwm", "pclk_bus_pre", 0,
 			RV1108_CLKGATE_CON(13), 6, GFLAGS),
-	GATE(0, "pclk_wdt", "pclk_bus_pre", CLK_IGNORE_UNUSED,
+	GATE(PCLK_WDT, "pclk_wdt", "pclk_bus_pre", 0,
 			RV1108_CLKGATE_CON(13), 3, GFLAGS),
-	GATE(0, "pclk_gpio1", "pclk_bus_pre", CLK_IGNORE_UNUSED,
+	GATE(PCLK_GPIO1, "pclk_gpio1", "pclk_bus_pre", 0,
 			RV1108_CLKGATE_CON(13), 7, GFLAGS),
-	GATE(0, "pclk_gpio2", "pclk_bus_pre", CLK_IGNORE_UNUSED,
+	GATE(PCLK_GPIO2, "pclk_gpio2", "pclk_bus_pre", 0,
 			RV1108_CLKGATE_CON(13), 8, GFLAGS),
-	GATE(0, "pclk_gpio3", "pclk_bus_pre", CLK_IGNORE_UNUSED,
+	GATE(PCLK_GPIO3, "pclk_gpio3", "pclk_bus_pre", 0,
 			RV1108_CLKGATE_CON(13), 9, GFLAGS),
 
 	GATE(0, "pclk_grf", "pclk_bus_pre", CLK_IGNORE_UNUSED,
 			RV1108_CLKGATE_CON(14), 0, GFLAGS),
+	GATE(PCLK_EFUSE0, "pclk_efuse0", "pclk_bus_pre", 0,
+			RV1108_CLKGATE_CON(12), 12, GFLAGS),
+	GATE(PCLK_EFUSE1, "pclk_efuse1", "pclk_bus_pre", 0,
+			RV1108_CLKGATE_CON(12), 13, GFLAGS),
+	GATE(PCLK_TSADC, "pclk_tsadc", "pclk_bus_pre", 0,
+			RV1108_CLKGATE_CON(13), 13, GFLAGS),
+	COMPOSITE_NOMUX(SCLK_TSADC, "sclk_tsadc", "xin24m", 0,
+			RV1108_CLKSEL_CON(21), 0, 10, DFLAGS,
+			RV1108_CLKGATE_CON(3), 11, GFLAGS),
+	GATE(PCLK_SARADC, "pclk_saradc", "pclk_bus_pre", 0,
+			RV1108_CLKGATE_CON(13), 14, GFLAGS),
+	COMPOSITE_NOMUX(SCLK_SARADC, "sclk_saradc", "xin24m", 0,
+			RV1108_CLKSEL_CON(22), 0, 10, DFLAGS,
+			RV1108_CLKGATE_CON(3), 12, GFLAGS),
 
 	GATE(ACLK_DMAC, "aclk_dmac", "aclk_bus_pre", 0,
 	     RV1108_CLKGATE_CON(12), 2, GFLAGS),
@@ -397,18 +677,24 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = {
 			RV1108_CLKGATE_CON(0), 9, GFLAGS),
 	GATE(0, "gpll_ddr", "gpll", CLK_IGNORE_UNUSED,
 			RV1108_CLKGATE_CON(0), 10, GFLAGS),
-	COMPOSITE(0, "ddrphy4x", mux_ddrphy_p, CLK_IGNORE_UNUSED,
+	COMPOSITE_NOGATE(0, "clk_ddrphy_src", mux_ddrphy_p, CLK_IGNORE_UNUSED,
 			RV1108_CLKSEL_CON(4), 8, 2, MFLAGS, 0, 3,
-			DFLAGS | CLK_DIVIDER_POWER_OF_TWO,
+			DFLAGS | CLK_DIVIDER_POWER_OF_TWO),
+	FACTOR(0, "clk_ddr", "clk_ddrphy_src", 0, 1, 2),
+	GATE(0, "clk_ddrphy4x", "clk_ddr", CLK_IGNORE_UNUSED,
 			RV1108_CLKGATE_CON(10), 9, GFLAGS),
-	GATE(0, "ddrupctl", "ddrphy_pre", CLK_IGNORE_UNUSED,
+	GATE(0, "pclk_ddrupctl", "pclk_ddr_pre", CLK_IGNORE_UNUSED,
 			RV1108_CLKGATE_CON(12), 4, GFLAGS),
-	GATE(0, "ddrc", "ddrphy", CLK_IGNORE_UNUSED,
+	GATE(0, "nclk_ddrupctl", "clk_ddr", CLK_IGNORE_UNUSED,
 			RV1108_CLKGATE_CON(12), 5, GFLAGS),
-	GATE(0, "ddrmon", "ddrphy_pre", CLK_IGNORE_UNUSED,
+	GATE(0, "pclk_ddrmon", "pclk_ddr_pre", CLK_IGNORE_UNUSED,
 			RV1108_CLKGATE_CON(12), 6, GFLAGS),
 	GATE(0, "timer_clk", "xin24m", CLK_IGNORE_UNUSED,
 			RV1108_CLKGATE_CON(0), 11, GFLAGS),
+	GATE(0, "pclk_mschniu", "pclk_ddr_pre", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(14), 2, GFLAGS),
+	GATE(0, "pclk_ddrphy", "pclk_ddr_pre", CLK_IGNORE_UNUSED,
+			RV1108_CLKGATE_CON(14), 4, GFLAGS),
 
 	/*
 	 * Clock-Architecture Diagram 6
@@ -418,23 +704,23 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = {
 	COMPOSITE_NOMUX(0, "pclk_periph_pre", "gpll", 0,
 			RV1108_CLKSEL_CON(23), 10, 5, DFLAGS,
 			RV1108_CLKGATE_CON(4), 5, GFLAGS),
-	GATE(0, "pclk_periph", "pclk_periph_pre", CLK_IGNORE_UNUSED,
+	GATE(PCLK_PERI, "pclk_periph", "pclk_periph_pre", CLK_IGNORE_UNUSED,
 			RV1108_CLKGATE_CON(15), 13, GFLAGS),
 	COMPOSITE_NOMUX(0, "hclk_periph_pre", "gpll", 0,
 			RV1108_CLKSEL_CON(23), 5, 5, DFLAGS,
 			RV1108_CLKGATE_CON(4), 4, GFLAGS),
-	GATE(0, "hclk_periph", "hclk_periph_pre", CLK_IGNORE_UNUSED,
+	GATE(HCLK_PERI, "hclk_periph", "hclk_periph_pre", CLK_IGNORE_UNUSED,
 			RV1108_CLKGATE_CON(15), 12, GFLAGS),
 
 	GATE(0, "aclk_peri_src_dpll", "dpll", CLK_IGNORE_UNUSED,
 			RV1108_CLKGATE_CON(4), 1, GFLAGS),
 	GATE(0, "aclk_peri_src_gpll", "gpll", CLK_IGNORE_UNUSED,
 			RV1108_CLKGATE_CON(4), 2, GFLAGS),
-	COMPOSITE(0, "aclk_periph", mux_aclk_peri_src_p, CLK_IGNORE_UNUSED,
-			RV1108_CLKSEL_CON(23), 15, 2, MFLAGS, 0, 5, DFLAGS,
+	COMPOSITE(ACLK_PERI, "aclk_periph", mux_aclk_peri_src_p, 0,
+			RV1108_CLKSEL_CON(23), 15, 1, MFLAGS, 0, 5, DFLAGS,
 			RV1108_CLKGATE_CON(15), 11, GFLAGS),
 
-	COMPOSITE(SCLK_SDMMC, "sclk_sdmmc0", mux_mmc_src_p, 0,
+	COMPOSITE(SCLK_SDMMC, "sclk_sdmmc", mux_mmc_src_p, 0,
 			RV1108_CLKSEL_CON(25), 8, 2, MFLAGS, 0, 8, DFLAGS,
 			RV1108_CLKGATE_CON(5), 0, GFLAGS),
 
@@ -454,23 +740,31 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = {
 	GATE(HCLK_EMMC, "hclk_emmc", "hclk_periph", 0, RV1108_CLKGATE_CON(15), 2, GFLAGS),
 
 	COMPOSITE(SCLK_NANDC, "sclk_nandc", mux_pll_src_2plls_p, 0,
-			RV1108_CLKSEL_CON(27), 14, 2, MFLAGS, 8, 5, DFLAGS,
+			RV1108_CLKSEL_CON(27), 14, 1, MFLAGS, 8, 5, DFLAGS,
 			RV1108_CLKGATE_CON(5), 3, GFLAGS),
 	GATE(HCLK_NANDC, "hclk_nandc", "hclk_periph", 0, RV1108_CLKGATE_CON(15), 3, GFLAGS),
 
+	GATE(HCLK_HOST0, "hclk_host0", "hclk_periph", 0, RV1108_CLKGATE_CON(15), 6, GFLAGS),
+	GATE(0, "hclk_host0_arb", "hclk_periph", CLK_IGNORE_UNUSED, RV1108_CLKGATE_CON(15), 7, GFLAGS),
+	GATE(HCLK_OTG, "hclk_otg", "hclk_periph", 0, RV1108_CLKGATE_CON(15), 8, GFLAGS),
+	GATE(0, "hclk_otg_pmu", "hclk_periph", CLK_IGNORE_UNUSED, RV1108_CLKGATE_CON(15), 9, GFLAGS),
+	GATE(SCLK_USBPHY, "clk_usbphy", "xin24m", CLK_IGNORE_UNUSED, RV1108_CLKGATE_CON(5), 5, GFLAGS),
+
 	COMPOSITE(SCLK_SFC, "sclk_sfc", mux_pll_src_2plls_p, 0,
-			RV1108_CLKSEL_CON(27), 7, 2, MFLAGS, 0, 7, DFLAGS,
+			RV1108_CLKSEL_CON(27), 7, 1, MFLAGS, 0, 7, DFLAGS,
 			RV1108_CLKGATE_CON(5), 4, GFLAGS),
 	GATE(HCLK_SFC, "hclk_sfc", "hclk_periph", 0, RV1108_CLKGATE_CON(15), 10, GFLAGS),
 
-	COMPOSITE(0, "sclk_macphy_pre", mux_pll_src_apll_gpll_p, 0,
-			RV1108_CLKSEL_CON(24), 12, 2, MFLAGS, 0, 5, DFLAGS,
+	COMPOSITE(SCLK_MAC_PRE, "sclk_mac_pre", mux_pll_src_apll_gpll_p, 0,
+			RV1108_CLKSEL_CON(24), 12, 1, MFLAGS, 0, 5, DFLAGS,
 			RV1108_CLKGATE_CON(4), 10, GFLAGS),
-	MUX(0, "sclk_macphy", mux_sclk_macphy_p, CLK_SET_RATE_PARENT,
-			RV1108_CLKSEL_CON(24), 8, 2, MFLAGS),
-	GATE(0, "sclk_macphy_rx", "sclk_macphy", 0, RV1108_CLKGATE_CON(4), 8, GFLAGS),
-	GATE(0, "sclk_mac_ref", "sclk_macphy", 0, RV1108_CLKGATE_CON(4), 6, GFLAGS),
-	GATE(0, "sclk_mac_refout", "sclk_macphy", 0, RV1108_CLKGATE_CON(4), 7, GFLAGS),
+	MUX(SCLK_MAC, "sclk_mac", mux_sclk_mac_p, CLK_SET_RATE_PARENT,
+			RV1108_CLKSEL_CON(24), 8, 1, MFLAGS),
+	GATE(SCLK_MAC_RX, "sclk_mac_rx", "sclk_mac", 0, RV1108_CLKGATE_CON(4), 8, GFLAGS),
+	GATE(SCLK_MAC_REF, "sclk_mac_ref", "sclk_mac", 0, RV1108_CLKGATE_CON(4), 6, GFLAGS),
+	GATE(SCLK_MAC_REFOUT, "sclk_mac_refout", "sclk_mac", 0, RV1108_CLKGATE_CON(4), 7, GFLAGS),
+	GATE(ACLK_GMAC, "aclk_gmac", "aclk_periph", 0, RV1108_CLKGATE_CON(15), 4, GFLAGS),
+	GATE(PCLK_GMAC, "pclk_gmac", "pclk_periph", 0, RV1108_CLKGATE_CON(15), 5, GFLAGS),
 
 	MMC(SCLK_SDMMC_DRV,    "sdmmc_drv",    "sclk_sdmmc", RV1108_SDMMC_CON0, 1),
 	MMC(SCLK_SDMMC_SAMPLE, "sdmmc_sample", "sclk_sdmmc", RV1108_SDMMC_CON1, 1),
@@ -484,10 +778,16 @@ static struct rockchip_clk_branch rv1108_clk_branches[] __initdata = {
 
 static const char *const rv1108_critical_clocks[] __initconst = {
 	"aclk_core",
-	"aclk_bus_src_gpll",
+	"aclk_bus",
+	"hclk_bus",
+	"pclk_bus",
 	"aclk_periph",
 	"hclk_periph",
 	"pclk_periph",
+	"nclk_ddrupctl",
+	"pclk_ddrmon",
+	"pclk_acodecphy",
+	"pclk_pmu",
 };
 
 static void __init rv1108_clk_init(struct device_node *np)
diff --git a/drivers/clk/rockchip/clk.c b/drivers/clk/rockchip/clk.c
index fe1d393cf678..35dbd63c2f49 100644
--- a/drivers/clk/rockchip/clk.c
+++ b/drivers/clk/rockchip/clk.c
@@ -29,6 +29,7 @@
 #include <linux/mfd/syscon.h>
 #include <linux/regmap.h>
 #include <linux/reboot.h>
+#include <linux/rational.h>
 #include "clk.h"
 
 /**
@@ -164,6 +165,40 @@ static int rockchip_clk_frac_notifier_cb(struct notifier_block *nb,
 	return notifier_from_errno(ret);
 }
 
+/**
+ * fractional divider must set that denominator is 20 times larger than
+ * numerator to generate precise clock frequency.
+ */
+static void rockchip_fractional_approximation(struct clk_hw *hw,
+		unsigned long rate, unsigned long *parent_rate,
+		unsigned long *m, unsigned long *n)
+{
+	struct clk_fractional_divider *fd = to_clk_fd(hw);
+	unsigned long p_rate, p_parent_rate;
+	struct clk_hw *p_parent;
+	unsigned long scale;
+
+	p_rate = clk_hw_get_rate(clk_hw_get_parent(hw));
+	if ((rate * 20 > p_rate) && (p_rate % rate != 0)) {
+		p_parent = clk_hw_get_parent(clk_hw_get_parent(hw));
+		p_parent_rate = clk_hw_get_rate(p_parent);
+		*parent_rate = p_parent_rate;
+	}
+
+	/*
+	 * Get rate closer to *parent_rate to guarantee there is no overflow
+	 * for m and n. In the result it will be the nearest rate left shifted
+	 * by (scale - fd->nwidth) bits.
+	 */
+	scale = fls_long(*parent_rate / rate - 1);
+	if (scale > fd->nwidth)
+		rate <<= scale - fd->nwidth;
+
+	rational_best_approximation(rate, *parent_rate,
+			GENMASK(fd->mwidth - 1, 0), GENMASK(fd->nwidth - 1, 0),
+			m, n);
+}
+
 static struct clk *rockchip_clk_register_frac_branch(
 		struct rockchip_clk_provider *ctx, const char *name,
 		const char *const *parent_names, u8 num_parents,
@@ -210,6 +245,7 @@ static struct clk *rockchip_clk_register_frac_branch(
 	div->nwidth = 16;
 	div->nmask = GENMASK(div->nwidth - 1, 0) << div->nshift;
 	div->lock = lock;
+	div->approximation = rockchip_fractional_approximation;
 	div_ops = &clk_fractional_divider_ops;
 
 	clk = clk_register_composite(NULL, name, parent_names, num_parents,
diff --git a/drivers/clk/samsung/clk-exynos-audss.c b/drivers/clk/samsung/clk-exynos-audss.c
index 1fab56f396d4..b117783ed404 100644
--- a/drivers/clk/samsung/clk-exynos-audss.c
+++ b/drivers/clk/samsung/clk-exynos-audss.c
@@ -180,7 +180,7 @@ static int exynos_audss_clk_probe(struct platform_device *pdev)
 	}
 	clk_table[EXYNOS_MOUT_AUDSS] = clk_hw_register_mux(NULL, "mout_audss",
 				mout_audss_p, ARRAY_SIZE(mout_audss_p),
-				CLK_SET_RATE_NO_REPARENT,
+				CLK_SET_RATE_NO_REPARENT | CLK_SET_RATE_PARENT,
 				reg_base + ASS_CLK_SRC, 0, 1, 0, &lock);
 
 	cdclk = devm_clk_get(&pdev->dev, "cdclk");
@@ -195,11 +195,11 @@ static int exynos_audss_clk_probe(struct platform_device *pdev)
 				reg_base + ASS_CLK_SRC, 2, 2, 0, &lock);
 
 	clk_table[EXYNOS_DOUT_SRP] = clk_hw_register_divider(NULL, "dout_srp",
-				"mout_audss", 0, reg_base + ASS_CLK_DIV, 0, 4,
-				0, &lock);
+				"mout_audss", CLK_SET_RATE_PARENT,
+				reg_base + ASS_CLK_DIV, 0, 4, 0, &lock);
 
 	clk_table[EXYNOS_DOUT_AUD_BUS] = clk_hw_register_divider(NULL,
-				"dout_aud_bus", "dout_srp", 0,
+				"dout_aud_bus", "dout_srp", CLK_SET_RATE_PARENT,
 				reg_base + ASS_CLK_DIV, 4, 4, 0, &lock);
 
 	clk_table[EXYNOS_DOUT_I2S] = clk_hw_register_divider(NULL, "dout_i2s",
diff --git a/drivers/clk/samsung/clk-exynos5420.c b/drivers/clk/samsung/clk-exynos5420.c
index 9a6476aa7d81..25601967d1cd 100644
--- a/drivers/clk/samsung/clk-exynos5420.c
+++ b/drivers/clk/samsung/clk-exynos5420.c
@@ -537,8 +537,8 @@ static const struct samsung_mux_clock exynos5800_mux_clks[] __initconst = {
 
 	MUX(CLK_MOUT_MX_MSPLL_CCORE, "mout_mx_mspll_ccore",
 			mout_mx_mspll_ccore_p, SRC_TOP7, 16, 2),
-	MUX(CLK_MOUT_MAU_EPLL, "mout_mau_epll_clk", mout_mau_epll_clk_5800_p,
-							SRC_TOP7, 20, 2),
+	MUX_F(CLK_MOUT_MAU_EPLL, "mout_mau_epll_clk", mout_mau_epll_clk_5800_p,
+			SRC_TOP7, 20, 2, CLK_SET_RATE_PARENT, 0),
 	MUX(0, "sclk_bpll", mout_bpll_p, SRC_TOP7, 24, 1),
 	MUX(0, "mout_epll2", mout_epll2_5800_p, SRC_TOP7, 28, 1),
 
@@ -547,8 +547,8 @@ static const struct samsung_mux_clock exynos5800_mux_clks[] __initconst = {
 	MUX(0, "mout_aclk432_cam", mout_group6_5800_p, SRC_TOP8, 24, 2),
 	MUX(0, "mout_aclk432_scaler", mout_group6_5800_p, SRC_TOP8, 28, 2),
 
-	MUX(CLK_MOUT_USER_MAU_EPLL, "mout_user_mau_epll", mout_group16_5800_p,
-							SRC_TOP9, 8, 1),
+	MUX_F(CLK_MOUT_USER_MAU_EPLL, "mout_user_mau_epll", mout_group16_5800_p,
+			SRC_TOP9, 8, 1, CLK_SET_RATE_PARENT, 0),
 	MUX(0, "mout_user_aclk550_cam", mout_group15_5800_p,
 							SRC_TOP9, 16, 1),
 	MUX(0, "mout_user_aclkfl1_550_cam", mout_group13_5800_p,
@@ -590,6 +590,8 @@ static const struct samsung_gate_clock exynos5800_gate_clks[] __initconst = {
 				GATE_BUS_TOP, 24, 0, 0),
 	GATE(CLK_ACLK432_SCALER, "aclk432_scaler", "mout_user_aclk432_scaler",
 				GATE_BUS_TOP, 27, CLK_IS_CRITICAL, 0),
+	GATE(CLK_MAU_EPLL, "mau_epll", "mout_user_mau_epll",
+			SRC_MASK_TOP7, 20, CLK_SET_RATE_PARENT, 0),
 };
 
 static const struct samsung_mux_clock exynos5420_mux_clks[] __initconst = {
@@ -629,6 +631,11 @@ static const struct samsung_div_clock exynos5420_div_clks[] __initconst = {
 			"mout_aclk400_wcore_bpll", DIV_TOP0, 16, 3),
 };
 
+static const struct samsung_gate_clock exynos5420_gate_clks[] __initconst = {
+	GATE(CLK_MAU_EPLL, "mau_epll", "mout_mau_epll_clk",
+			SRC_MASK_TOP7, 20, CLK_SET_RATE_PARENT, 0),
+};
+
 static const struct samsung_mux_clock exynos5x_mux_clks[] __initconst = {
 	MUX(0, "mout_user_pclk66_gpio", mout_user_pclk66_gpio_p,
 			SRC_TOP7, 4, 1),
@@ -706,7 +713,8 @@ static const struct samsung_mux_clock exynos5x_mux_clks[] __initconst = {
 	MUX(0, "mout_sclk_spll", mout_spll_p, SRC_TOP6, 8, 1),
 	MUX(0, "mout_sclk_ipll", mout_ipll_p, SRC_TOP6, 12, 1),
 	MUX(0, "mout_sclk_rpll", mout_rpll_p, SRC_TOP6, 16, 1),
-	MUX(CLK_MOUT_EPLL, "mout_sclk_epll", mout_epll_p, SRC_TOP6, 20, 1),
+	MUX_F(CLK_MOUT_EPLL, "mout_sclk_epll", mout_epll_p, SRC_TOP6, 20, 1,
+			CLK_SET_RATE_PARENT, 0),
 	MUX(0, "mout_sclk_dpll", mout_dpll_p, SRC_TOP6, 24, 1),
 	MUX(0, "mout_sclk_cpll", mout_cpll_p, SRC_TOP6, 28, 1),
 
@@ -1001,9 +1009,6 @@ static const struct samsung_gate_clock exynos5x_gate_clks[] __initconst = {
 	GATE(0, "aclk300_disp1", "mout_user_aclk300_disp1",
 			SRC_MASK_TOP2, 24, CLK_IS_CRITICAL, 0),
 
-	GATE(CLK_MAU_EPLL, "mau_epll", "mout_mau_epll_clk",
-			SRC_MASK_TOP7, 20, 0, 0),
-
 	/* sclk */
 	GATE(CLK_SCLK_UART0, "sclk_uart0", "dout_uart0",
 		GATE_TOP_SCLK_PERIC, 0, CLK_SET_RATE_PARENT, 0),
@@ -1440,6 +1445,8 @@ static void __init exynos5x_clk_init(struct device_node *np,
 				ARRAY_SIZE(exynos5420_mux_clks));
 		samsung_clk_register_div(ctx, exynos5420_div_clks,
 				ARRAY_SIZE(exynos5420_div_clks));
+		samsung_clk_register_gate(ctx, exynos5420_gate_clks,
+				ARRAY_SIZE(exynos5420_gate_clks));
 	} else {
 		samsung_clk_register_fixed_factor(
 				ctx, exynos5800_fixed_factor_clks,
diff --git a/drivers/clk/sunxi-ng/Kconfig b/drivers/clk/sunxi-ng/Kconfig
index 7342928c35cd..6427d0ebe2de 100644
--- a/drivers/clk/sunxi-ng/Kconfig
+++ b/drivers/clk/sunxi-ng/Kconfig
@@ -11,6 +11,19 @@ config SUN50I_A64_CCU
 	default ARM64 && ARCH_SUNXI
 	depends on (ARM64 && ARCH_SUNXI) || COMPILE_TEST
 
+config SUN4I_A10_CCU
+	bool "Support for the Allwinner A10/A20 CCU"
+	select SUNXI_CCU_DIV
+	select SUNXI_CCU_MULT
+	select SUNXI_CCU_NK
+	select SUNXI_CCU_NKM
+	select SUNXI_CCU_NM
+	select SUNXI_CCU_MP
+	select SUNXI_CCU_PHASE
+	default MACH_SUN4I
+	default MACH_SUN7I
+	depends on MACH_SUN4I || MACH_SUN7I || COMPILE_TEST
+
 config SUN5I_CCU
 	bool "Support for the Allwinner sun5i family CCM"
 	default MACH_SUN5I
@@ -48,6 +61,11 @@ config SUN8I_V3S_CCU
 config SUN8I_DE2_CCU
 	bool "Support for the Allwinner SoCs DE2 CCU"
 
+config SUN8I_R40_CCU
+	bool "Support for the Allwinner R40 CCU"
+	default MACH_SUN8I
+	depends on MACH_SUN8I || COMPILE_TEST
+
 config SUN9I_A80_CCU
 	bool "Support for the Allwinner A80 CCU"
 	default MACH_SUN9I
diff --git a/drivers/clk/sunxi-ng/Makefile b/drivers/clk/sunxi-ng/Makefile
index 0c45fa50283d..85a0633c1eac 100644
--- a/drivers/clk/sunxi-ng/Makefile
+++ b/drivers/clk/sunxi-ng/Makefile
@@ -1,5 +1,6 @@
 # Common objects
 lib-$(CONFIG_SUNXI_CCU)		+= ccu_common.o
+lib-$(CONFIG_SUNXI_CCU)		+= ccu_mmc_timing.o
 lib-$(CONFIG_SUNXI_CCU)		+= ccu_reset.o
 
 # Base clock types
@@ -19,6 +20,7 @@ lib-$(CONFIG_SUNXI_CCU)		+= ccu_mp.o
 
 # SoC support
 obj-$(CONFIG_SUN50I_A64_CCU)	+= ccu-sun50i-a64.o
+obj-$(CONFIG_SUN4I_A10_CCU)	+= ccu-sun4i-a10.o
 obj-$(CONFIG_SUN5I_CCU)		+= ccu-sun5i.o
 obj-$(CONFIG_SUN6I_A31_CCU)	+= ccu-sun6i-a31.o
 obj-$(CONFIG_SUN8I_A23_CCU)	+= ccu-sun8i-a23.o
@@ -28,6 +30,7 @@ obj-$(CONFIG_SUN8I_H3_CCU)	+= ccu-sun8i-h3.o
 obj-$(CONFIG_SUN8I_V3S_CCU)	+= ccu-sun8i-v3s.o
 obj-$(CONFIG_SUN8I_DE2_CCU)	+= ccu-sun8i-de2.o
 obj-$(CONFIG_SUN8I_R_CCU)	+= ccu-sun8i-r.o
+obj-$(CONFIG_SUN8I_R40_CCU)	+= ccu-sun8i-r40.o
 obj-$(CONFIG_SUN9I_A80_CCU)	+= ccu-sun9i-a80.o
 obj-$(CONFIG_SUN9I_A80_CCU)	+= ccu-sun9i-a80-de.o
 obj-$(CONFIG_SUN9I_A80_CCU)	+= ccu-sun9i-a80-usb.o
diff --git a/drivers/clk/sunxi-ng/ccu-sun4i-a10.c b/drivers/clk/sunxi-ng/ccu-sun4i-a10.c
new file mode 100644
index 000000000000..286b0049b7b6
--- /dev/null
+++ b/drivers/clk/sunxi-ng/ccu-sun4i-a10.c
@@ -0,0 +1,1456 @@
+/*
+ * Copyright (c) 2017 Priit Laes <plaes@plaes.org>.
+ * Copyright (c) 2017 Maxime Ripard.
+ * Copyright (c) 2017 Jonathan Liu.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/of_address.h>
+
+#include "ccu_common.h"
+#include "ccu_reset.h"
+
+#include "ccu_div.h"
+#include "ccu_gate.h"
+#include "ccu_mp.h"
+#include "ccu_mult.h"
+#include "ccu_nk.h"
+#include "ccu_nkm.h"
+#include "ccu_nkmp.h"
+#include "ccu_nm.h"
+#include "ccu_phase.h"
+
+#include "ccu-sun4i-a10.h"
+
+static struct ccu_nkmp pll_core_clk = {
+	.enable		= BIT(31),
+	.n		= _SUNXI_CCU_MULT_OFFSET(8, 5, 0),
+	.k		= _SUNXI_CCU_MULT(4, 2),
+	.m		= _SUNXI_CCU_DIV(0, 2),
+	.p		= _SUNXI_CCU_DIV(16, 2),
+	.common		= {
+		.reg		= 0x000,
+		.hw.init	= CLK_HW_INIT("pll-core",
+					      "hosc",
+					      &ccu_nkmp_ops,
+					      0),
+	},
+};
+
+/*
+ * The Audio PLL is supposed to have 4 outputs: 3 fixed factors from
+ * the base (2x, 4x and 8x), and one variable divider (the one true
+ * pll audio).
+ *
+ * We don't have any need for the variable divider for now, so we just
+ * hardcode it to match with the clock names.
+ */
+#define SUN4I_PLL_AUDIO_REG	0x008
+static struct ccu_nm pll_audio_base_clk = {
+	.enable		= BIT(31),
+	.n		= _SUNXI_CCU_MULT_OFFSET(8, 7, 0),
+	.m		= _SUNXI_CCU_DIV_OFFSET(0, 5, 0),
+	.common		= {
+		.reg		= 0x008,
+		.hw.init	= CLK_HW_INIT("pll-audio-base",
+					      "hosc",
+					      &ccu_nm_ops,
+					      0),
+	},
+
+};
+
+static struct ccu_mult pll_video0_clk = {
+	.enable		= BIT(31),
+	.mult		= _SUNXI_CCU_MULT_OFFSET_MIN_MAX(0, 7, 0, 9, 127),
+	.frac		= _SUNXI_CCU_FRAC(BIT(15), BIT(14),
+					  270000000, 297000000),
+	.common		= {
+		.reg		= 0x010,
+		.features	= (CCU_FEATURE_FRACTIONAL |
+				   CCU_FEATURE_ALL_PREDIV),
+		.prediv		= 8,
+		.hw.init	= CLK_HW_INIT("pll-video0",
+					      "hosc",
+					      &ccu_mult_ops,
+					      0),
+	},
+};
+
+static struct ccu_nkmp pll_ve_sun4i_clk = {
+	.enable		= BIT(31),
+	.n		= _SUNXI_CCU_MULT_OFFSET(8, 5, 0),
+	.k		= _SUNXI_CCU_MULT(4, 2),
+	.m		= _SUNXI_CCU_DIV(0, 2),
+	.p		= _SUNXI_CCU_DIV(16, 2),
+	.common		= {
+		.reg		= 0x018,
+		.hw.init	= CLK_HW_INIT("pll-ve",
+					      "hosc",
+					      &ccu_nkmp_ops,
+					      0),
+	},
+};
+
+static struct ccu_nk pll_ve_sun7i_clk = {
+	.enable		= BIT(31),
+	.n		= _SUNXI_CCU_MULT_OFFSET(8, 5, 0),
+	.k		= _SUNXI_CCU_MULT(4, 2),
+	.common		= {
+		.reg		= 0x018,
+		.hw.init	= CLK_HW_INIT("pll-ve",
+					      "hosc",
+					      &ccu_nk_ops,
+					      0),
+	},
+};
+
+static struct ccu_nk pll_ddr_base_clk = {
+	.enable		= BIT(31),
+	.n		= _SUNXI_CCU_MULT_OFFSET(8, 5, 0),
+	.k		= _SUNXI_CCU_MULT(4, 2),
+	.common		= {
+		.reg		= 0x020,
+		.hw.init	= CLK_HW_INIT("pll-ddr-base",
+					      "hosc",
+					      &ccu_nk_ops,
+					      0),
+	},
+};
+
+static SUNXI_CCU_M(pll_ddr_clk, "pll-ddr", "pll-ddr-base", 0x020, 0, 2,
+		   CLK_IS_CRITICAL);
+
+static struct ccu_div pll_ddr_other_clk = {
+	.div		= _SUNXI_CCU_DIV_FLAGS(16, 2, CLK_DIVIDER_POWER_OF_TWO),
+	.common		= {
+		.reg		= 0x020,
+		.hw.init	= CLK_HW_INIT("pll-ddr-other", "pll-ddr-base",
+					      &ccu_div_ops,
+					      0),
+	},
+};
+
+static struct ccu_nk pll_periph_base_clk = {
+	.enable		= BIT(31),
+	.n		= _SUNXI_CCU_MULT_OFFSET(8, 5, 0),
+	.k		= _SUNXI_CCU_MULT(4, 2),
+	.common		= {
+		.reg		= 0x028,
+		.hw.init	= CLK_HW_INIT("pll-periph-base",
+					      "hosc",
+					      &ccu_nk_ops,
+					      0),
+	},
+};
+
+static CLK_FIXED_FACTOR(pll_periph_clk, "pll-periph", "pll-periph-base",
+			2, 1, CLK_SET_RATE_PARENT);
+
+/* Not documented on A10 */
+static struct ccu_div pll_periph_sata_clk = {
+	.enable		= BIT(14),
+	.div		= _SUNXI_CCU_DIV(0, 2),
+	.fixed_post_div	= 6,
+	.common		= {
+		.reg		= 0x028,
+		.features	= CCU_FEATURE_FIXED_POSTDIV,
+		.hw.init	= CLK_HW_INIT("pll-periph-sata",
+					      "pll-periph-base",
+					      &ccu_div_ops, 0),
+	},
+};
+
+static struct ccu_mult pll_video1_clk = {
+	.enable		= BIT(31),
+	.mult		= _SUNXI_CCU_MULT_OFFSET_MIN_MAX(0, 7, 0, 9, 127),
+	.frac		= _SUNXI_CCU_FRAC(BIT(15), BIT(14),
+				  270000000, 297000000),
+	.common		= {
+		.reg		= 0x030,
+		.features	= (CCU_FEATURE_FRACTIONAL |
+				   CCU_FEATURE_ALL_PREDIV),
+		.prediv		= 8,
+		.hw.init	= CLK_HW_INIT("pll-video1",
+					      "hosc",
+					      &ccu_mult_ops,
+					      0),
+	},
+};
+
+/* Not present on A10 */
+static struct ccu_nk pll_gpu_clk = {
+	.enable		= BIT(31),
+	.n		= _SUNXI_CCU_MULT_OFFSET(8, 5, 0),
+	.k		= _SUNXI_CCU_MULT(4, 2),
+	.common		= {
+		.reg		= 0x040,
+		.hw.init	= CLK_HW_INIT("pll-gpu",
+					      "hosc",
+					      &ccu_nk_ops,
+					      0),
+	},
+};
+
+static SUNXI_CCU_GATE(hosc_clk,	"hosc",	"osc24M", 0x050, BIT(0), 0);
+
+static const char *const cpu_parents[] = { "osc32k", "hosc",
+					   "pll-core", "pll-periph" };
+static const struct ccu_mux_fixed_prediv cpu_predivs[] = {
+	{ .index = 3, .div = 3, },
+};
+
+#define SUN4I_AHB_REG		0x054
+static struct ccu_mux cpu_clk = {
+	.mux		= {
+		.shift		= 16,
+		.width		= 2,
+		.fixed_predivs	= cpu_predivs,
+		.n_predivs	= ARRAY_SIZE(cpu_predivs),
+	},
+	.common		= {
+		.reg		= 0x054,
+		.features	= CCU_FEATURE_FIXED_PREDIV,
+		.hw.init	= CLK_HW_INIT_PARENTS("cpu",
+						      cpu_parents,
+						      &ccu_mux_ops,
+						      CLK_IS_CRITICAL),
+	}
+};
+
+static SUNXI_CCU_M(axi_clk, "axi", "cpu", 0x054, 0, 2, 0);
+
+static struct ccu_div ahb_sun4i_clk = {
+	.div		= _SUNXI_CCU_DIV_FLAGS(4, 2, CLK_DIVIDER_POWER_OF_TWO),
+	.common		= {
+		.reg		= 0x054,
+		.hw.init	= CLK_HW_INIT("ahb", "axi", &ccu_div_ops, 0),
+	},
+};
+
+static const char *const ahb_sun7i_parents[] = { "axi", "pll-periph",
+						 "pll-periph" };
+static const struct ccu_mux_fixed_prediv ahb_sun7i_predivs[] = {
+	{ .index = 1, .div = 2, },
+	{ /* Sentinel */ },
+};
+static struct ccu_div ahb_sun7i_clk = {
+	.div		= _SUNXI_CCU_DIV_FLAGS(4, 2, CLK_DIVIDER_POWER_OF_TWO),
+	.mux		= {
+		.shift		= 6,
+		.width		= 2,
+		.fixed_predivs	= ahb_sun7i_predivs,
+		.n_predivs	= ARRAY_SIZE(ahb_sun7i_predivs),
+	},
+
+	.common		= {
+		.reg		= 0x054,
+		.hw.init	= CLK_HW_INIT_PARENTS("ahb",
+						      ahb_sun7i_parents,
+						      &ccu_div_ops,
+						      0),
+	},
+};
+
+static struct clk_div_table apb0_div_table[] = {
+	{ .val = 0, .div = 2 },
+	{ .val = 1, .div = 2 },
+	{ .val = 2, .div = 4 },
+	{ .val = 3, .div = 8 },
+	{ /* Sentinel */ },
+};
+static SUNXI_CCU_DIV_TABLE(apb0_clk, "apb0", "ahb",
+			   0x054, 8, 2, apb0_div_table, 0);
+
+static const char *const apb1_parents[] = { "hosc", "pll-periph", "osc32k" };
+static SUNXI_CCU_MP_WITH_MUX(apb1_clk, "apb1", apb1_parents, 0x058,
+			     0, 5,	/* M */
+			     16, 2,	/* P */
+			     24, 2,	/* mux */
+			     0);
+
+/* Not present on A20 */
+static SUNXI_CCU_GATE(axi_dram_clk,	"axi-dram",	"ahb",
+		      0x05c, BIT(31), 0);
+
+static SUNXI_CCU_GATE(ahb_otg_clk,	"ahb-otg",	"ahb",
+		      0x060, BIT(0), 0);
+static SUNXI_CCU_GATE(ahb_ehci0_clk,	"ahb-ehci0",	"ahb",
+		      0x060, BIT(1), 0);
+static SUNXI_CCU_GATE(ahb_ohci0_clk,	"ahb-ohci0",	"ahb",
+		      0x060, BIT(2), 0);
+static SUNXI_CCU_GATE(ahb_ehci1_clk,	"ahb-ehci1",	"ahb",
+		      0x060, BIT(3), 0);
+static SUNXI_CCU_GATE(ahb_ohci1_clk,	"ahb-ohci1",	"ahb",
+		      0x060, BIT(4), 0);
+static SUNXI_CCU_GATE(ahb_ss_clk,	"ahb-ss",	"ahb",
+		      0x060, BIT(5), 0);
+static SUNXI_CCU_GATE(ahb_dma_clk,	"ahb-dma",	"ahb",
+		      0x060, BIT(6), 0);
+static SUNXI_CCU_GATE(ahb_bist_clk,	"ahb-bist",	"ahb",
+		      0x060, BIT(7), 0);
+static SUNXI_CCU_GATE(ahb_mmc0_clk,	"ahb-mmc0",	"ahb",
+		      0x060, BIT(8), 0);
+static SUNXI_CCU_GATE(ahb_mmc1_clk,	"ahb-mmc1",	"ahb",
+		      0x060, BIT(9), 0);
+static SUNXI_CCU_GATE(ahb_mmc2_clk,	"ahb-mmc2",	"ahb",
+		      0x060, BIT(10), 0);
+static SUNXI_CCU_GATE(ahb_mmc3_clk,	"ahb-mmc3",	"ahb",
+		      0x060, BIT(11), 0);
+static SUNXI_CCU_GATE(ahb_ms_clk,	"ahb-ms",	"ahb",
+		      0x060, BIT(12), 0);
+static SUNXI_CCU_GATE(ahb_nand_clk,	"ahb-nand",	"ahb",
+		      0x060, BIT(13), 0);
+static SUNXI_CCU_GATE(ahb_sdram_clk,	"ahb-sdram",	"ahb",
+		      0x060, BIT(14), CLK_IS_CRITICAL);
+
+static SUNXI_CCU_GATE(ahb_ace_clk,	"ahb-ace",	"ahb",
+		      0x060, BIT(16), 0);
+static SUNXI_CCU_GATE(ahb_emac_clk,	"ahb-emac",	"ahb",
+		      0x060, BIT(17), 0);
+static SUNXI_CCU_GATE(ahb_ts_clk,	"ahb-ts",	"ahb",
+		      0x060, BIT(18), 0);
+static SUNXI_CCU_GATE(ahb_spi0_clk,	"ahb-spi0",	"ahb",
+		      0x060, BIT(20), 0);
+static SUNXI_CCU_GATE(ahb_spi1_clk,	"ahb-spi1",	"ahb",
+		      0x060, BIT(21), 0);
+static SUNXI_CCU_GATE(ahb_spi2_clk,	"ahb-spi2",	"ahb",
+		      0x060, BIT(22), 0);
+static SUNXI_CCU_GATE(ahb_spi3_clk,	"ahb-spi3",	"ahb",
+		      0x060, BIT(23), 0);
+static SUNXI_CCU_GATE(ahb_pata_clk,	"ahb-pata",	"ahb",
+		      0x060, BIT(24), 0);
+/* Not documented on A20 */
+static SUNXI_CCU_GATE(ahb_sata_clk,	"ahb-sata",	"ahb",
+		      0x060, BIT(25), 0);
+/* Not present on A20 */
+static SUNXI_CCU_GATE(ahb_gps_clk,	"ahb-gps",	"ahb",
+		      0x060, BIT(26), 0);
+/* Not present on A10 */
+static SUNXI_CCU_GATE(ahb_hstimer_clk,	"ahb-hstimer",	"ahb",
+		      0x060, BIT(28), 0);
+
+static SUNXI_CCU_GATE(ahb_ve_clk,	"ahb-ve",	"ahb",
+		      0x064, BIT(0), 0);
+static SUNXI_CCU_GATE(ahb_tvd_clk,	"ahb-tvd",	"ahb",
+		      0x064, BIT(1), 0);
+static SUNXI_CCU_GATE(ahb_tve0_clk,	"ahb-tve0",	"ahb",
+		      0x064, BIT(2), 0);
+static SUNXI_CCU_GATE(ahb_tve1_clk,	"ahb-tve1",	"ahb",
+		      0x064, BIT(3), 0);
+static SUNXI_CCU_GATE(ahb_lcd0_clk,	"ahb-lcd0",	"ahb",
+		      0x064, BIT(4), 0);
+static SUNXI_CCU_GATE(ahb_lcd1_clk,	"ahb-lcd1",	"ahb",
+		      0x064, BIT(5), 0);
+static SUNXI_CCU_GATE(ahb_csi0_clk,	"ahb-csi0",	"ahb",
+		      0x064, BIT(8), 0);
+static SUNXI_CCU_GATE(ahb_csi1_clk,	"ahb-csi1",	"ahb",
+		      0x064, BIT(9), 0);
+/* Not present on A10 */
+static SUNXI_CCU_GATE(ahb_hdmi1_clk,	"ahb-hdmi1",	"ahb",
+		      0x064, BIT(10), 0);
+static SUNXI_CCU_GATE(ahb_hdmi0_clk,	"ahb-hdmi0",	"ahb",
+		      0x064, BIT(11), 0);
+static SUNXI_CCU_GATE(ahb_de_be0_clk,	"ahb-de-be0",	"ahb",
+		      0x064, BIT(12), 0);
+static SUNXI_CCU_GATE(ahb_de_be1_clk,	"ahb-de-be1",	"ahb",
+		      0x064, BIT(13), 0);
+static SUNXI_CCU_GATE(ahb_de_fe0_clk,	"ahb-de-fe0",	"ahb",
+		      0x064, BIT(14), 0);
+static SUNXI_CCU_GATE(ahb_de_fe1_clk,	"ahb-de-fe1",	"ahb",
+		      0x064, BIT(15), 0);
+/* Not present on A10 */
+static SUNXI_CCU_GATE(ahb_gmac_clk,	"ahb-gmac",	"ahb",
+		      0x064, BIT(17), 0);
+static SUNXI_CCU_GATE(ahb_mp_clk,	"ahb-mp",	"ahb",
+		      0x064, BIT(18), 0);
+static SUNXI_CCU_GATE(ahb_gpu_clk,	"ahb-gpu",	"ahb",
+		      0x064, BIT(20), 0);
+
+static SUNXI_CCU_GATE(apb0_codec_clk,	"apb0-codec",	"apb0",
+		      0x068, BIT(0), 0);
+static SUNXI_CCU_GATE(apb0_spdif_clk,	"apb0-spdif",	"apb0",
+		      0x068, BIT(1), 0);
+static SUNXI_CCU_GATE(apb0_ac97_clk,	"apb0-ac97",	"apb0",
+		      0x068, BIT(2), 0);
+static SUNXI_CCU_GATE(apb0_i2s0_clk,	"apb0-i2s0",	"apb0",
+		      0x068, BIT(3), 0);
+/* Not present on A10 */
+static SUNXI_CCU_GATE(apb0_i2s1_clk,	"apb0-i2s1",	"apb0",
+		      0x068, BIT(4), 0);
+static SUNXI_CCU_GATE(apb0_pio_clk,	"apb0-pio",	"apb0",
+		      0x068, BIT(5), 0);
+static SUNXI_CCU_GATE(apb0_ir0_clk,	"apb0-ir0",	"apb0",
+		      0x068, BIT(6), 0);
+static SUNXI_CCU_GATE(apb0_ir1_clk,	"apb0-ir1",	"apb0",
+		      0x068, BIT(7), 0);
+/* Not present on A10 */
+static SUNXI_CCU_GATE(apb0_i2s2_clk,	"apb0-i2s2",	"apb0",
+		      0x068, BIT(8), 0);
+static SUNXI_CCU_GATE(apb0_keypad_clk,	"apb0-keypad",	"apb0",
+		      0x068, BIT(10), 0);
+
+static SUNXI_CCU_GATE(apb1_i2c0_clk,	"apb1-i2c0",	"apb1",
+		      0x06c, BIT(0), 0);
+static SUNXI_CCU_GATE(apb1_i2c1_clk,	"apb1-i2c1",	"apb1",
+		      0x06c, BIT(1), 0);
+static SUNXI_CCU_GATE(apb1_i2c2_clk,	"apb1-i2c2",	"apb1",
+		      0x06c, BIT(2), 0);
+/* Not present on A10 */
+static SUNXI_CCU_GATE(apb1_i2c3_clk,	"apb1-i2c3",	"apb1",
+		      0x06c, BIT(3), 0);
+static SUNXI_CCU_GATE(apb1_can_clk,	"apb1-can",	"apb1",
+		      0x06c, BIT(4), 0);
+static SUNXI_CCU_GATE(apb1_scr_clk,	"apb1-scr",	"apb1",
+		      0x06c, BIT(5), 0);
+static SUNXI_CCU_GATE(apb1_ps20_clk,	"apb1-ps20",	"apb1",
+		      0x06c, BIT(6), 0);
+static SUNXI_CCU_GATE(apb1_ps21_clk,	"apb1-ps21",	"apb1",
+		      0x06c, BIT(7), 0);
+/* Not present on A10 */
+static SUNXI_CCU_GATE(apb1_i2c4_clk,	"apb1-i2c4",	"apb1",
+		      0x06c, BIT(15), 0);
+static SUNXI_CCU_GATE(apb1_uart0_clk,	"apb1-uart0",	"apb1",
+		      0x06c, BIT(16), 0);
+static SUNXI_CCU_GATE(apb1_uart1_clk,	"apb1-uart1",	"apb1",
+		      0x06c, BIT(17), 0);
+static SUNXI_CCU_GATE(apb1_uart2_clk,	"apb1-uart2",	"apb1",
+		      0x06c, BIT(18), 0);
+static SUNXI_CCU_GATE(apb1_uart3_clk,	"apb1-uart3",	"apb1",
+		      0x06c, BIT(19), 0);
+static SUNXI_CCU_GATE(apb1_uart4_clk,	"apb1-uart4",	"apb1",
+		      0x06c, BIT(20), 0);
+static SUNXI_CCU_GATE(apb1_uart5_clk,	"apb1-uart5",	"apb1",
+		      0x06c, BIT(21), 0);
+static SUNXI_CCU_GATE(apb1_uart6_clk,	"apb1-uart6",	"apb1",
+		      0x06c, BIT(22), 0);
+static SUNXI_CCU_GATE(apb1_uart7_clk,	"apb1-uart7",	"apb1",
+		      0x06c, BIT(23), 0);
+
+static const char *const mod0_default_parents[] = { "hosc", "pll-periph",
+						     "pll-ddr-other" };
+static SUNXI_CCU_MP_WITH_MUX_GATE(nand_clk, "nand", mod0_default_parents, 0x080,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+/* Undocumented on A10 */
+static SUNXI_CCU_MP_WITH_MUX_GATE(ms_clk, "ms", mod0_default_parents, 0x084,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(mmc0_clk, "mmc0", mod0_default_parents, 0x088,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+/* MMC output and sample clocks are not present on A10 */
+static SUNXI_CCU_PHASE(mmc0_output_clk, "mmc0_output", "mmc0",
+		       0x088, 8, 3, 0);
+static SUNXI_CCU_PHASE(mmc0_sample_clk, "mmc0_sample", "mmc0",
+		       0x088, 20, 3, 0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(mmc1_clk, "mmc1", mod0_default_parents, 0x08c,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+/* MMC output and sample clocks are not present on A10 */
+static SUNXI_CCU_PHASE(mmc1_output_clk, "mmc1_output", "mmc1",
+		       0x08c, 8, 3, 0);
+static SUNXI_CCU_PHASE(mmc1_sample_clk, "mmc1_sample", "mmc1",
+		       0x08c, 20, 3, 0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(mmc2_clk, "mmc2", mod0_default_parents, 0x090,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+/* MMC output and sample clocks are not present on A10 */
+static SUNXI_CCU_PHASE(mmc2_output_clk, "mmc2_output", "mmc2",
+		       0x090, 8, 3, 0);
+static SUNXI_CCU_PHASE(mmc2_sample_clk, "mmc2_sample", "mmc2",
+		       0x090, 20, 3, 0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(mmc3_clk, "mmc3", mod0_default_parents, 0x094,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+/* MMC output and sample clocks are not present on A10 */
+static SUNXI_CCU_PHASE(mmc3_output_clk, "mmc3_output", "mmc3",
+		       0x094, 8, 3, 0);
+static SUNXI_CCU_PHASE(mmc3_sample_clk, "mmc3_sample", "mmc3",
+		       0x094, 20, 3, 0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(ts_clk, "ts", mod0_default_parents, 0x098,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(ss_clk, "ss", mod0_default_parents, 0x09c,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(spi0_clk, "spi0", mod0_default_parents, 0x0a0,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(spi1_clk, "spi1", mod0_default_parents, 0x0a4,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(spi2_clk, "spi2", mod0_default_parents, 0x0a8,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+/* Undocumented on A10 */
+static SUNXI_CCU_MP_WITH_MUX_GATE(pata_clk, "pata", mod0_default_parents, 0x0ac,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+/* TODO: Check whether A10 actually supports osc32k as 4th parent? */
+static const char *const ir_parents_sun4i[] = { "hosc", "pll-periph",
+						"pll-ddr-other" };
+static SUNXI_CCU_MP_WITH_MUX_GATE(ir0_sun4i_clk, "ir0", ir_parents_sun4i, 0x0b0,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(ir1_sun4i_clk, "ir1", ir_parents_sun4i, 0x0b4,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+static const char *const ir_parents_sun7i[] = { "hosc", "pll-periph",
+						"pll-ddr-other", "osc32k" };
+static SUNXI_CCU_MP_WITH_MUX_GATE(ir0_sun7i_clk, "ir0", ir_parents_sun7i, 0x0b0,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(ir1_sun7i_clk, "ir1", ir_parents_sun7i, 0x0b4,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static const char *const audio_parents[] = { "pll-audio-8x", "pll-audio-4x",
+					      "pll-audio-2x", "pll-audio" };
+static SUNXI_CCU_MUX_WITH_GATE(i2s0_clk, "i2s0", audio_parents,
+			       0x0b8, 16, 2, BIT(31), CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_MUX_WITH_GATE(ac97_clk, "ac97", audio_parents,
+			       0x0bc, 16, 2, BIT(31), CLK_SET_RATE_PARENT);
+
+/* Undocumented on A10 */
+static SUNXI_CCU_MUX_WITH_GATE(spdif_clk, "spdif", audio_parents,
+			       0x0c0, 16, 2, BIT(31), CLK_SET_RATE_PARENT);
+
+static const char *const keypad_parents[] = { "hosc", "losc"};
+static const u8 keypad_table[] = { 0, 2 };
+static struct ccu_mp keypad_clk = {
+	.enable		= BIT(31),
+	.m		= _SUNXI_CCU_DIV(0, 5),
+	.p		= _SUNXI_CCU_DIV(16, 2),
+	.mux		= _SUNXI_CCU_MUX_TABLE(24, 2, keypad_table),
+	.common		= {
+		.reg		= 0x0c4,
+		.hw.init	= CLK_HW_INIT_PARENTS("keypad",
+						      keypad_parents,
+						      &ccu_mp_ops,
+						      0),
+	},
+};
+
+/*
+ * SATA supports external clock as parent via BIT(24) and is probably an
+ * optional crystal or oscillator that can be connected to the
+ * SATA-CLKM / SATA-CLKP pins.
+ */
+static const char *const sata_parents[] = {"pll-periph-sata", "sata-ext"};
+static SUNXI_CCU_MUX_WITH_GATE(sata_clk, "sata", sata_parents,
+			       0x0c8, 24, 1, BIT(31), CLK_SET_RATE_PARENT);
+
+
+static SUNXI_CCU_GATE(usb_ohci0_clk,	"usb-ohci0",	"pll-periph",
+		      0x0cc, BIT(6), 0);
+static SUNXI_CCU_GATE(usb_ohci1_clk,	"usb-ohci1",	"pll-periph",
+		      0x0cc, BIT(7), 0);
+static SUNXI_CCU_GATE(usb_phy_clk,	"usb-phy",	"pll-periph",
+		      0x0cc, BIT(8), 0);
+
+/* TODO: GPS CLK 0x0d0 */
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(spi3_clk, "spi3", mod0_default_parents, 0x0d4,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+/* Not present on A10 */
+static SUNXI_CCU_MUX_WITH_GATE(i2s1_clk, "i2s1", audio_parents,
+			       0x0d8, 16, 2, BIT(31), CLK_SET_RATE_PARENT);
+
+/* Not present on A10 */
+static SUNXI_CCU_MUX_WITH_GATE(i2s2_clk, "i2s2", audio_parents,
+			       0x0dc, 16, 2, BIT(31), CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_GATE(dram_ve_clk,	"dram-ve",	"pll-ddr",
+		      0x100, BIT(0), 0);
+static SUNXI_CCU_GATE(dram_csi0_clk,	"dram-csi0",	"pll-ddr",
+		      0x100, BIT(1), 0);
+static SUNXI_CCU_GATE(dram_csi1_clk,	"dram-csi1",	"pll-ddr",
+		      0x100, BIT(2), 0);
+static SUNXI_CCU_GATE(dram_ts_clk,	"dram-ts",	"pll-ddr",
+		      0x100, BIT(3), 0);
+static SUNXI_CCU_GATE(dram_tvd_clk,	"dram-tvd",	"pll-ddr",
+		      0x100, BIT(4), 0);
+static SUNXI_CCU_GATE(dram_tve0_clk,	"dram-tve0",	"pll-ddr",
+		      0x100, BIT(5), 0);
+static SUNXI_CCU_GATE(dram_tve1_clk,	"dram-tve1",	"pll-ddr",
+		      0x100, BIT(6), 0);
+
+/* Clock seems to be critical only on sun4i */
+static SUNXI_CCU_GATE(dram_out_clk,	"dram-out",	"pll-ddr",
+		      0x100, BIT(15), CLK_IS_CRITICAL);
+static SUNXI_CCU_GATE(dram_de_fe1_clk,	"dram-de-fe1",	"pll-ddr",
+		      0x100, BIT(24), 0);
+static SUNXI_CCU_GATE(dram_de_fe0_clk,	"dram-de-fe0",	"pll-ddr",
+		      0x100, BIT(25), 0);
+static SUNXI_CCU_GATE(dram_de_be0_clk,	"dram-de-be0",	"pll-ddr",
+		      0x100, BIT(26), 0);
+static SUNXI_CCU_GATE(dram_de_be1_clk,	"dram-de-be1",	"pll-ddr",
+		      0x100, BIT(27), 0);
+static SUNXI_CCU_GATE(dram_mp_clk,	"dram-mp",	"pll-ddr",
+		      0x100, BIT(28), 0);
+static SUNXI_CCU_GATE(dram_ace_clk,	"dram-ace",	"pll-ddr",
+		      0x100, BIT(29), 0);
+
+static const char *const de_parents[] = { "pll-video0", "pll-video1",
+					   "pll-ddr-other" };
+static SUNXI_CCU_M_WITH_MUX_GATE(de_be0_clk, "de-be0", de_parents,
+				 0x104, 0, 4, 24, 2, BIT(31), 0);
+
+static SUNXI_CCU_M_WITH_MUX_GATE(de_be1_clk, "de-be1", de_parents,
+				 0x108, 0, 4, 24, 2, BIT(31), 0);
+
+static SUNXI_CCU_M_WITH_MUX_GATE(de_fe0_clk, "de-fe0", de_parents,
+				 0x10c, 0, 4, 24, 2, BIT(31), 0);
+
+static SUNXI_CCU_M_WITH_MUX_GATE(de_fe1_clk, "de-fe1", de_parents,
+				 0x110, 0, 4, 24, 2, BIT(31), 0);
+
+/* Undocumented on A10 */
+static SUNXI_CCU_M_WITH_MUX_GATE(de_mp_clk, "de-mp", de_parents,
+				 0x114, 0, 4, 24, 2, BIT(31), 0);
+
+static const char *const disp_parents[] = { "pll-video0", "pll-video1",
+					    "pll-video0-2x", "pll-video1-2x" };
+static SUNXI_CCU_MUX_WITH_GATE(tcon0_ch0_clk, "tcon0-ch0-sclk", disp_parents,
+			       0x118, 24, 2, BIT(31), CLK_SET_RATE_PARENT);
+static SUNXI_CCU_MUX_WITH_GATE(tcon1_ch0_clk, "tcon1-ch0-sclk", disp_parents,
+			       0x11c, 24, 2, BIT(31), CLK_SET_RATE_PARENT);
+
+static const char *const csi_sclk_parents[] = { "pll-video0", "pll-ve",
+						"pll-ddr-other", "pll-periph" };
+
+static SUNXI_CCU_M_WITH_MUX_GATE(csi_sclk_clk, "csi-sclk",
+				 csi_sclk_parents,
+				 0x120, 0, 4, 24, 2, BIT(31), 0);
+
+/* TVD clock setup for A10 */
+static const char *const tvd_parents[] = { "pll-video0", "pll-video1" };
+static SUNXI_CCU_MUX_WITH_GATE(tvd_sun4i_clk, "tvd", tvd_parents,
+			       0x128, 24, 1, BIT(31), 0);
+
+/* TVD clock setup for A20 */
+static SUNXI_CCU_MP_WITH_MUX_GATE(tvd_sclk2_sun7i_clk,
+				  "tvd-sclk2", tvd_parents,
+				  0x128,
+				  0, 4,		/* M */
+				  16, 4,	/* P */
+				  8, 1,		/* mux */
+				  BIT(15),	/* gate */
+				  0);
+
+static SUNXI_CCU_M_WITH_GATE(tvd_sclk1_sun7i_clk, "tvd-sclk1", "tvd-sclk2",
+			     0x128, 0, 4, BIT(31), 0);
+
+static SUNXI_CCU_M_WITH_MUX_GATE(tcon0_ch1_sclk2_clk, "tcon0-ch1-sclk2",
+				 disp_parents,
+				 0x12c, 0, 4, 24, 2, BIT(31),
+				 CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_M_WITH_GATE(tcon0_ch1_clk,
+			     "tcon0-ch1-sclk1", "tcon0-ch1-sclk2",
+			     0x12c, 11, 1, BIT(15),
+			     CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_M_WITH_MUX_GATE(tcon1_ch1_sclk2_clk, "tcon1-ch1-sclk2",
+				 disp_parents,
+				 0x130, 0, 4, 24, 2, BIT(31),
+				 CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_M_WITH_GATE(tcon1_ch1_clk,
+			     "tcon1-ch1-sclk1", "tcon1-ch1-sclk2",
+			     0x130, 11, 1, BIT(15),
+			     CLK_SET_RATE_PARENT);
+
+static const char *const csi_parents[] = { "hosc", "pll-video0", "pll-video1",
+					   "pll-video0-2x", "pll-video1-2x"};
+static const u8 csi_table[] = { 0, 1, 2, 5, 6};
+static SUNXI_CCU_M_WITH_MUX_TABLE_GATE(csi0_clk, "csi0",
+				       csi_parents, csi_table,
+				       0x134, 0, 5, 24, 3, BIT(31), 0);
+
+static SUNXI_CCU_M_WITH_MUX_TABLE_GATE(csi1_clk, "csi1",
+				       csi_parents, csi_table,
+				       0x138, 0, 5, 24, 3, BIT(31), 0);
+
+static SUNXI_CCU_M_WITH_GATE(ve_clk, "ve", "pll-ve", 0x13c, 16, 8, BIT(31), 0);
+
+static SUNXI_CCU_GATE(codec_clk, "codec", "pll-audio",
+		      0x140, BIT(31), CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_GATE(avs_clk, "avs", "hosc", 0x144, BIT(31), 0);
+
+static const char *const ace_parents[] = { "pll-ve", "pll-ddr-other" };
+static SUNXI_CCU_M_WITH_MUX_GATE(ace_clk, "ace", ace_parents,
+				 0x148, 0, 4, 24, 1, BIT(31), 0);
+
+static SUNXI_CCU_M_WITH_MUX_GATE(hdmi_clk, "hdmi", disp_parents,
+				 0x150, 0, 4, 24, 2, BIT(31),
+				 CLK_SET_RATE_PARENT);
+
+static const char *const gpu_parents_sun4i[] = { "pll-video0", "pll-ve",
+						 "pll-ddr-other",
+						 "pll-video1" };
+static SUNXI_CCU_M_WITH_MUX_GATE(gpu_sun4i_clk, "gpu", gpu_parents_sun4i,
+				 0x154, 0, 4, 24, 2, BIT(31),
+				 CLK_SET_RATE_PARENT);
+
+static const char *const gpu_parents_sun7i[] = { "pll-video0", "pll-ve",
+						 "pll-ddr-other", "pll-video1",
+						 "pll-gpu" };
+static const u8 gpu_table_sun7i[] = { 0, 1, 2, 3, 4 };
+static SUNXI_CCU_M_WITH_MUX_TABLE_GATE(gpu_sun7i_clk, "gpu",
+				       gpu_parents_sun7i, gpu_table_sun7i,
+				       0x154, 0, 4, 24, 3, BIT(31),
+				       CLK_SET_RATE_PARENT);
+
+static const char *const mbus_sun4i_parents[] = { "hosc", "pll-periph",
+						  "pll-ddr-other" };
+static SUNXI_CCU_MP_WITH_MUX_GATE(mbus_sun4i_clk, "mbus", mbus_sun4i_parents,
+				  0x15c, 0, 4, 16, 2, 24, 2, BIT(31),
+				  0);
+static const char *const mbus_sun7i_parents[] = { "hosc", "pll-periph-base",
+						  "pll-ddr-other" };
+static SUNXI_CCU_MP_WITH_MUX_GATE(mbus_sun7i_clk, "mbus", mbus_sun7i_parents,
+				  0x15c, 0, 4, 16, 2, 24, 2, BIT(31),
+				  CLK_IS_CRITICAL);
+
+static SUNXI_CCU_GATE(hdmi1_slow_clk, "hdmi1-slow", "hosc", 0x178, BIT(31), 0);
+
+static const char *const hdmi1_parents[] = { "pll-video0", "pll-video1" };
+static const u8 hdmi1_table[] = { 0, 1};
+static SUNXI_CCU_M_WITH_MUX_TABLE_GATE(hdmi1_clk, "hdmi1",
+				       hdmi1_parents, hdmi1_table,
+				       0x17c, 0, 4, 24, 2, BIT(31),
+				       CLK_SET_RATE_PARENT);
+
+static const char *const out_parents[] = { "hosc", "osc32k", "hosc" };
+static const struct ccu_mux_fixed_prediv clk_out_predivs[] = {
+	{ .index = 0, .div = 750, },
+};
+
+static struct ccu_mp out_a_clk = {
+	.enable		= BIT(31),
+	.m		= _SUNXI_CCU_DIV(8, 5),
+	.p		= _SUNXI_CCU_DIV(20, 2),
+	.mux		= {
+		.shift		= 24,
+		.width		= 2,
+		.fixed_predivs	= clk_out_predivs,
+		.n_predivs	= ARRAY_SIZE(clk_out_predivs),
+	},
+	.common		= {
+		.reg		= 0x1f0,
+		.features	= CCU_FEATURE_FIXED_PREDIV,
+		.hw.init	= CLK_HW_INIT_PARENTS("out-a",
+						      out_parents,
+						      &ccu_mp_ops,
+						      0),
+	},
+};
+static struct ccu_mp out_b_clk = {
+	.enable		= BIT(31),
+	.m		= _SUNXI_CCU_DIV(8, 5),
+	.p		= _SUNXI_CCU_DIV(20, 2),
+	.mux		= {
+		.shift		= 24,
+		.width		= 2,
+		.fixed_predivs	= clk_out_predivs,
+		.n_predivs	= ARRAY_SIZE(clk_out_predivs),
+	},
+	.common		= {
+		.reg		= 0x1f4,
+		.features	= CCU_FEATURE_FIXED_PREDIV,
+		.hw.init	= CLK_HW_INIT_PARENTS("out-b",
+						      out_parents,
+						      &ccu_mp_ops,
+						      0),
+	},
+};
+
+static struct ccu_common *sun4i_sun7i_ccu_clks[] = {
+	&hosc_clk.common,
+	&pll_core_clk.common,
+	&pll_audio_base_clk.common,
+	&pll_video0_clk.common,
+	&pll_ve_sun4i_clk.common,
+	&pll_ve_sun7i_clk.common,
+	&pll_ddr_base_clk.common,
+	&pll_ddr_clk.common,
+	&pll_ddr_other_clk.common,
+	&pll_periph_base_clk.common,
+	&pll_periph_sata_clk.common,
+	&pll_video1_clk.common,
+	&pll_gpu_clk.common,
+	&cpu_clk.common,
+	&axi_clk.common,
+	&axi_dram_clk.common,
+	&ahb_sun4i_clk.common,
+	&ahb_sun7i_clk.common,
+	&apb0_clk.common,
+	&apb1_clk.common,
+	&ahb_otg_clk.common,
+	&ahb_ehci0_clk.common,
+	&ahb_ohci0_clk.common,
+	&ahb_ehci1_clk.common,
+	&ahb_ohci1_clk.common,
+	&ahb_ss_clk.common,
+	&ahb_dma_clk.common,
+	&ahb_bist_clk.common,
+	&ahb_mmc0_clk.common,
+	&ahb_mmc1_clk.common,
+	&ahb_mmc2_clk.common,
+	&ahb_mmc3_clk.common,
+	&ahb_ms_clk.common,
+	&ahb_nand_clk.common,
+	&ahb_sdram_clk.common,
+	&ahb_ace_clk.common,
+	&ahb_emac_clk.common,
+	&ahb_ts_clk.common,
+	&ahb_spi0_clk.common,
+	&ahb_spi1_clk.common,
+	&ahb_spi2_clk.common,
+	&ahb_spi3_clk.common,
+	&ahb_pata_clk.common,
+	&ahb_sata_clk.common,
+	&ahb_gps_clk.common,
+	&ahb_hstimer_clk.common,
+	&ahb_ve_clk.common,
+	&ahb_tvd_clk.common,
+	&ahb_tve0_clk.common,
+	&ahb_tve1_clk.common,
+	&ahb_lcd0_clk.common,
+	&ahb_lcd1_clk.common,
+	&ahb_csi0_clk.common,
+	&ahb_csi1_clk.common,
+	&ahb_hdmi1_clk.common,
+	&ahb_hdmi0_clk.common,
+	&ahb_de_be0_clk.common,
+	&ahb_de_be1_clk.common,
+	&ahb_de_fe0_clk.common,
+	&ahb_de_fe1_clk.common,
+	&ahb_gmac_clk.common,
+	&ahb_mp_clk.common,
+	&ahb_gpu_clk.common,
+	&apb0_codec_clk.common,
+	&apb0_spdif_clk.common,
+	&apb0_ac97_clk.common,
+	&apb0_i2s0_clk.common,
+	&apb0_i2s1_clk.common,
+	&apb0_pio_clk.common,
+	&apb0_ir0_clk.common,
+	&apb0_ir1_clk.common,
+	&apb0_i2s2_clk.common,
+	&apb0_keypad_clk.common,
+	&apb1_i2c0_clk.common,
+	&apb1_i2c1_clk.common,
+	&apb1_i2c2_clk.common,
+	&apb1_i2c3_clk.common,
+	&apb1_can_clk.common,
+	&apb1_scr_clk.common,
+	&apb1_ps20_clk.common,
+	&apb1_ps21_clk.common,
+	&apb1_i2c4_clk.common,
+	&apb1_uart0_clk.common,
+	&apb1_uart1_clk.common,
+	&apb1_uart2_clk.common,
+	&apb1_uart3_clk.common,
+	&apb1_uart4_clk.common,
+	&apb1_uart5_clk.common,
+	&apb1_uart6_clk.common,
+	&apb1_uart7_clk.common,
+	&nand_clk.common,
+	&ms_clk.common,
+	&mmc0_clk.common,
+	&mmc0_output_clk.common,
+	&mmc0_sample_clk.common,
+	&mmc1_clk.common,
+	&mmc1_output_clk.common,
+	&mmc1_sample_clk.common,
+	&mmc2_clk.common,
+	&mmc2_output_clk.common,
+	&mmc2_sample_clk.common,
+	&mmc3_clk.common,
+	&mmc3_output_clk.common,
+	&mmc3_sample_clk.common,
+	&ts_clk.common,
+	&ss_clk.common,
+	&spi0_clk.common,
+	&spi1_clk.common,
+	&spi2_clk.common,
+	&pata_clk.common,
+	&ir0_sun4i_clk.common,
+	&ir1_sun4i_clk.common,
+	&ir0_sun7i_clk.common,
+	&ir1_sun7i_clk.common,
+	&i2s0_clk.common,
+	&ac97_clk.common,
+	&spdif_clk.common,
+	&keypad_clk.common,
+	&sata_clk.common,
+	&usb_ohci0_clk.common,
+	&usb_ohci1_clk.common,
+	&usb_phy_clk.common,
+	&spi3_clk.common,
+	&i2s1_clk.common,
+	&i2s2_clk.common,
+	&dram_ve_clk.common,
+	&dram_csi0_clk.common,
+	&dram_csi1_clk.common,
+	&dram_ts_clk.common,
+	&dram_tvd_clk.common,
+	&dram_tve0_clk.common,
+	&dram_tve1_clk.common,
+	&dram_out_clk.common,
+	&dram_de_fe1_clk.common,
+	&dram_de_fe0_clk.common,
+	&dram_de_be0_clk.common,
+	&dram_de_be1_clk.common,
+	&dram_mp_clk.common,
+	&dram_ace_clk.common,
+	&de_be0_clk.common,
+	&de_be1_clk.common,
+	&de_fe0_clk.common,
+	&de_fe1_clk.common,
+	&de_mp_clk.common,
+	&tcon0_ch0_clk.common,
+	&tcon1_ch0_clk.common,
+	&csi_sclk_clk.common,
+	&tvd_sun4i_clk.common,
+	&tvd_sclk1_sun7i_clk.common,
+	&tvd_sclk2_sun7i_clk.common,
+	&tcon0_ch1_sclk2_clk.common,
+	&tcon0_ch1_clk.common,
+	&tcon1_ch1_sclk2_clk.common,
+	&tcon1_ch1_clk.common,
+	&csi0_clk.common,
+	&csi1_clk.common,
+	&ve_clk.common,
+	&codec_clk.common,
+	&avs_clk.common,
+	&ace_clk.common,
+	&hdmi_clk.common,
+	&gpu_sun4i_clk.common,
+	&gpu_sun7i_clk.common,
+	&mbus_sun4i_clk.common,
+	&mbus_sun7i_clk.common,
+	&hdmi1_slow_clk.common,
+	&hdmi1_clk.common,
+	&out_a_clk.common,
+	&out_b_clk.common
+};
+
+/* Post-divider for pll-audio is hardcoded to 4 */
+static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio",
+			"pll-audio-base", 4, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x",
+			"pll-audio-base", 2, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x",
+			"pll-audio-base", 1, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x",
+			"pll-audio-base", 1, 2, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_video0_2x_clk, "pll-video0-2x",
+			"pll-video0", 1, 2, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_video1_2x_clk, "pll-video1-2x",
+			"pll-video1", 1, 2, CLK_SET_RATE_PARENT);
+
+
+static struct clk_hw_onecell_data sun4i_a10_hw_clks = {
+	.hws	= {
+		[CLK_HOSC]		= &hosc_clk.common.hw,
+		[CLK_PLL_CORE]		= &pll_core_clk.common.hw,
+		[CLK_PLL_AUDIO_BASE]	= &pll_audio_base_clk.common.hw,
+		[CLK_PLL_AUDIO]		= &pll_audio_clk.hw,
+		[CLK_PLL_AUDIO_2X]	= &pll_audio_2x_clk.hw,
+		[CLK_PLL_AUDIO_4X]	= &pll_audio_4x_clk.hw,
+		[CLK_PLL_AUDIO_8X]	= &pll_audio_8x_clk.hw,
+		[CLK_PLL_VIDEO0]	= &pll_video0_clk.common.hw,
+		[CLK_PLL_VIDEO0_2X]	= &pll_video0_2x_clk.hw,
+		[CLK_PLL_VE]		= &pll_ve_sun4i_clk.common.hw,
+		[CLK_PLL_DDR_BASE]	= &pll_ddr_base_clk.common.hw,
+		[CLK_PLL_DDR]		= &pll_ddr_clk.common.hw,
+		[CLK_PLL_DDR_OTHER]	= &pll_ddr_other_clk.common.hw,
+		[CLK_PLL_PERIPH_BASE]	= &pll_periph_base_clk.common.hw,
+		[CLK_PLL_PERIPH]	= &pll_periph_clk.hw,
+		[CLK_PLL_PERIPH_SATA]	= &pll_periph_sata_clk.common.hw,
+		[CLK_PLL_VIDEO1]	= &pll_video1_clk.common.hw,
+		[CLK_PLL_VIDEO1_2X]	= &pll_video1_2x_clk.hw,
+		[CLK_CPU]		= &cpu_clk.common.hw,
+		[CLK_AXI]		= &axi_clk.common.hw,
+		[CLK_AXI_DRAM]		= &axi_dram_clk.common.hw,
+		[CLK_AHB]		= &ahb_sun4i_clk.common.hw,
+		[CLK_APB0]		= &apb0_clk.common.hw,
+		[CLK_APB1]		= &apb1_clk.common.hw,
+		[CLK_AHB_OTG]		= &ahb_otg_clk.common.hw,
+		[CLK_AHB_EHCI0]		= &ahb_ehci0_clk.common.hw,
+		[CLK_AHB_OHCI0]		= &ahb_ohci0_clk.common.hw,
+		[CLK_AHB_EHCI1]		= &ahb_ehci1_clk.common.hw,
+		[CLK_AHB_OHCI1]		= &ahb_ohci1_clk.common.hw,
+		[CLK_AHB_SS]		= &ahb_ss_clk.common.hw,
+		[CLK_AHB_DMA]		= &ahb_dma_clk.common.hw,
+		[CLK_AHB_BIST]		= &ahb_bist_clk.common.hw,
+		[CLK_AHB_MMC0]		= &ahb_mmc0_clk.common.hw,
+		[CLK_AHB_MMC1]		= &ahb_mmc1_clk.common.hw,
+		[CLK_AHB_MMC2]		= &ahb_mmc2_clk.common.hw,
+		[CLK_AHB_MMC3]		= &ahb_mmc3_clk.common.hw,
+		[CLK_AHB_MS]		= &ahb_ms_clk.common.hw,
+		[CLK_AHB_NAND]		= &ahb_nand_clk.common.hw,
+		[CLK_AHB_SDRAM]		= &ahb_sdram_clk.common.hw,
+		[CLK_AHB_ACE]		= &ahb_ace_clk.common.hw,
+		[CLK_AHB_EMAC]		= &ahb_emac_clk.common.hw,
+		[CLK_AHB_TS]		= &ahb_ts_clk.common.hw,
+		[CLK_AHB_SPI0]		= &ahb_spi0_clk.common.hw,
+		[CLK_AHB_SPI1]		= &ahb_spi1_clk.common.hw,
+		[CLK_AHB_SPI2]		= &ahb_spi2_clk.common.hw,
+		[CLK_AHB_SPI3]		= &ahb_spi3_clk.common.hw,
+		[CLK_AHB_PATA]		= &ahb_pata_clk.common.hw,
+		[CLK_AHB_SATA]		= &ahb_sata_clk.common.hw,
+		[CLK_AHB_GPS]		= &ahb_gps_clk.common.hw,
+		[CLK_AHB_VE]		= &ahb_ve_clk.common.hw,
+		[CLK_AHB_TVD]		= &ahb_tvd_clk.common.hw,
+		[CLK_AHB_TVE0]		= &ahb_tve0_clk.common.hw,
+		[CLK_AHB_TVE1]		= &ahb_tve1_clk.common.hw,
+		[CLK_AHB_LCD0]		= &ahb_lcd0_clk.common.hw,
+		[CLK_AHB_LCD1]		= &ahb_lcd1_clk.common.hw,
+		[CLK_AHB_CSI0]		= &ahb_csi0_clk.common.hw,
+		[CLK_AHB_CSI1]		= &ahb_csi1_clk.common.hw,
+		[CLK_AHB_HDMI0]		= &ahb_hdmi0_clk.common.hw,
+		[CLK_AHB_DE_BE0]	= &ahb_de_be0_clk.common.hw,
+		[CLK_AHB_DE_BE1]	= &ahb_de_be1_clk.common.hw,
+		[CLK_AHB_DE_FE0]	= &ahb_de_fe0_clk.common.hw,
+		[CLK_AHB_DE_FE1]	= &ahb_de_fe1_clk.common.hw,
+		[CLK_AHB_MP]		= &ahb_mp_clk.common.hw,
+		[CLK_AHB_GPU]		= &ahb_gpu_clk.common.hw,
+		[CLK_APB0_CODEC]	= &apb0_codec_clk.common.hw,
+		[CLK_APB0_SPDIF]	= &apb0_spdif_clk.common.hw,
+		[CLK_APB0_AC97]		= &apb0_ac97_clk.common.hw,
+		[CLK_APB0_I2S0]		= &apb0_i2s0_clk.common.hw,
+		[CLK_APB0_PIO]		= &apb0_pio_clk.common.hw,
+		[CLK_APB0_IR0]		= &apb0_ir0_clk.common.hw,
+		[CLK_APB0_IR1]		= &apb0_ir1_clk.common.hw,
+		[CLK_APB0_KEYPAD]	= &apb0_keypad_clk.common.hw,
+		[CLK_APB1_I2C0]		= &apb1_i2c0_clk.common.hw,
+		[CLK_APB1_I2C1]		= &apb1_i2c1_clk.common.hw,
+		[CLK_APB1_I2C2]		= &apb1_i2c2_clk.common.hw,
+		[CLK_APB1_CAN]		= &apb1_can_clk.common.hw,
+		[CLK_APB1_SCR]		= &apb1_scr_clk.common.hw,
+		[CLK_APB1_PS20]		= &apb1_ps20_clk.common.hw,
+		[CLK_APB1_PS21]		= &apb1_ps21_clk.common.hw,
+		[CLK_APB1_UART0]	= &apb1_uart0_clk.common.hw,
+		[CLK_APB1_UART1]	= &apb1_uart1_clk.common.hw,
+		[CLK_APB1_UART2]	= &apb1_uart2_clk.common.hw,
+		[CLK_APB1_UART3]	= &apb1_uart3_clk.common.hw,
+		[CLK_APB1_UART4]	= &apb1_uart4_clk.common.hw,
+		[CLK_APB1_UART5]	= &apb1_uart5_clk.common.hw,
+		[CLK_APB1_UART6]	= &apb1_uart6_clk.common.hw,
+		[CLK_APB1_UART7]	= &apb1_uart7_clk.common.hw,
+		[CLK_NAND]		= &nand_clk.common.hw,
+		[CLK_MS]		= &ms_clk.common.hw,
+		[CLK_MMC0]		= &mmc0_clk.common.hw,
+		[CLK_MMC1]		= &mmc1_clk.common.hw,
+		[CLK_MMC2]		= &mmc2_clk.common.hw,
+		[CLK_MMC3]		= &mmc3_clk.common.hw,
+		[CLK_TS]		= &ts_clk.common.hw,
+		[CLK_SS]		= &ss_clk.common.hw,
+		[CLK_SPI0]		= &spi0_clk.common.hw,
+		[CLK_SPI1]		= &spi1_clk.common.hw,
+		[CLK_SPI2]		= &spi2_clk.common.hw,
+		[CLK_PATA]		= &pata_clk.common.hw,
+		[CLK_IR0]		= &ir0_sun4i_clk.common.hw,
+		[CLK_IR1]		= &ir1_sun4i_clk.common.hw,
+		[CLK_I2S0]		= &i2s0_clk.common.hw,
+		[CLK_AC97]		= &ac97_clk.common.hw,
+		[CLK_SPDIF]		= &spdif_clk.common.hw,
+		[CLK_KEYPAD]		= &keypad_clk.common.hw,
+		[CLK_SATA]		= &sata_clk.common.hw,
+		[CLK_USB_OHCI0]		= &usb_ohci0_clk.common.hw,
+		[CLK_USB_OHCI1]		= &usb_ohci1_clk.common.hw,
+		[CLK_USB_PHY]		= &usb_phy_clk.common.hw,
+		/* CLK_GPS is unimplemented */
+		[CLK_SPI3]		= &spi3_clk.common.hw,
+		[CLK_DRAM_VE]		= &dram_ve_clk.common.hw,
+		[CLK_DRAM_CSI0]		= &dram_csi0_clk.common.hw,
+		[CLK_DRAM_CSI1]		= &dram_csi1_clk.common.hw,
+		[CLK_DRAM_TS]		= &dram_ts_clk.common.hw,
+		[CLK_DRAM_TVD]		= &dram_tvd_clk.common.hw,
+		[CLK_DRAM_TVE0]		= &dram_tve0_clk.common.hw,
+		[CLK_DRAM_TVE1]		= &dram_tve1_clk.common.hw,
+		[CLK_DRAM_OUT]		= &dram_out_clk.common.hw,
+		[CLK_DRAM_DE_FE1]	= &dram_de_fe1_clk.common.hw,
+		[CLK_DRAM_DE_FE0]	= &dram_de_fe0_clk.common.hw,
+		[CLK_DRAM_DE_BE0]	= &dram_de_be0_clk.common.hw,
+		[CLK_DRAM_DE_BE1]	= &dram_de_be1_clk.common.hw,
+		[CLK_DRAM_MP]		= &dram_mp_clk.common.hw,
+		[CLK_DRAM_ACE]		= &dram_ace_clk.common.hw,
+		[CLK_DE_BE0]		= &de_be0_clk.common.hw,
+		[CLK_DE_BE1]		= &de_be1_clk.common.hw,
+		[CLK_DE_FE0]		= &de_fe0_clk.common.hw,
+		[CLK_DE_FE1]		= &de_fe1_clk.common.hw,
+		[CLK_DE_MP]		= &de_mp_clk.common.hw,
+		[CLK_TCON0_CH0]		= &tcon0_ch0_clk.common.hw,
+		[CLK_TCON1_CH0]		= &tcon1_ch0_clk.common.hw,
+		[CLK_CSI_SCLK]		= &csi_sclk_clk.common.hw,
+		[CLK_TVD]		= &tvd_sun4i_clk.common.hw,
+		[CLK_TCON0_CH1_SCLK2]	= &tcon0_ch1_sclk2_clk.common.hw,
+		[CLK_TCON0_CH1]		= &tcon0_ch1_clk.common.hw,
+		[CLK_TCON1_CH1_SCLK2]	= &tcon1_ch1_sclk2_clk.common.hw,
+		[CLK_TCON1_CH1]		= &tcon1_ch1_clk.common.hw,
+		[CLK_CSI0]		= &csi0_clk.common.hw,
+		[CLK_CSI1]		= &csi1_clk.common.hw,
+		[CLK_VE]		= &ve_clk.common.hw,
+		[CLK_CODEC]		= &codec_clk.common.hw,
+		[CLK_AVS]		= &avs_clk.common.hw,
+		[CLK_ACE]		= &ace_clk.common.hw,
+		[CLK_HDMI]		= &hdmi_clk.common.hw,
+		[CLK_GPU]		= &gpu_sun7i_clk.common.hw,
+		[CLK_MBUS]		= &mbus_sun4i_clk.common.hw,
+	},
+	.num	= CLK_NUMBER_SUN4I,
+};
+static struct clk_hw_onecell_data sun7i_a20_hw_clks = {
+	.hws	= {
+		[CLK_HOSC]		= &hosc_clk.common.hw,
+		[CLK_PLL_CORE]		= &pll_core_clk.common.hw,
+		[CLK_PLL_AUDIO_BASE]	= &pll_audio_base_clk.common.hw,
+		[CLK_PLL_AUDIO]		= &pll_audio_clk.hw,
+		[CLK_PLL_AUDIO_2X]	= &pll_audio_2x_clk.hw,
+		[CLK_PLL_AUDIO_4X]	= &pll_audio_4x_clk.hw,
+		[CLK_PLL_AUDIO_8X]	= &pll_audio_8x_clk.hw,
+		[CLK_PLL_VIDEO0]	= &pll_video0_clk.common.hw,
+		[CLK_PLL_VIDEO0_2X]	= &pll_video0_2x_clk.hw,
+		[CLK_PLL_VE]		= &pll_ve_sun7i_clk.common.hw,
+		[CLK_PLL_DDR_BASE]	= &pll_ddr_base_clk.common.hw,
+		[CLK_PLL_DDR]		= &pll_ddr_clk.common.hw,
+		[CLK_PLL_DDR_OTHER]	= &pll_ddr_other_clk.common.hw,
+		[CLK_PLL_PERIPH_BASE]	= &pll_periph_base_clk.common.hw,
+		[CLK_PLL_PERIPH]	= &pll_periph_clk.hw,
+		[CLK_PLL_PERIPH_SATA]	= &pll_periph_sata_clk.common.hw,
+		[CLK_PLL_VIDEO1]	= &pll_video1_clk.common.hw,
+		[CLK_PLL_VIDEO1_2X]	= &pll_video1_2x_clk.hw,
+		[CLK_PLL_GPU]		= &pll_gpu_clk.common.hw,
+		[CLK_CPU]		= &cpu_clk.common.hw,
+		[CLK_AXI]		= &axi_clk.common.hw,
+		[CLK_AHB]		= &ahb_sun7i_clk.common.hw,
+		[CLK_APB0]		= &apb0_clk.common.hw,
+		[CLK_APB1]		= &apb1_clk.common.hw,
+		[CLK_AHB_OTG]		= &ahb_otg_clk.common.hw,
+		[CLK_AHB_EHCI0]		= &ahb_ehci0_clk.common.hw,
+		[CLK_AHB_OHCI0]		= &ahb_ohci0_clk.common.hw,
+		[CLK_AHB_EHCI1]		= &ahb_ehci1_clk.common.hw,
+		[CLK_AHB_OHCI1]		= &ahb_ohci1_clk.common.hw,
+		[CLK_AHB_SS]		= &ahb_ss_clk.common.hw,
+		[CLK_AHB_DMA]		= &ahb_dma_clk.common.hw,
+		[CLK_AHB_BIST]		= &ahb_bist_clk.common.hw,
+		[CLK_AHB_MMC0]		= &ahb_mmc0_clk.common.hw,
+		[CLK_AHB_MMC1]		= &ahb_mmc1_clk.common.hw,
+		[CLK_AHB_MMC2]		= &ahb_mmc2_clk.common.hw,
+		[CLK_AHB_MMC3]		= &ahb_mmc3_clk.common.hw,
+		[CLK_AHB_MS]		= &ahb_ms_clk.common.hw,
+		[CLK_AHB_NAND]		= &ahb_nand_clk.common.hw,
+		[CLK_AHB_SDRAM]		= &ahb_sdram_clk.common.hw,
+		[CLK_AHB_ACE]		= &ahb_ace_clk.common.hw,
+		[CLK_AHB_EMAC]		= &ahb_emac_clk.common.hw,
+		[CLK_AHB_TS]		= &ahb_ts_clk.common.hw,
+		[CLK_AHB_SPI0]		= &ahb_spi0_clk.common.hw,
+		[CLK_AHB_SPI1]		= &ahb_spi1_clk.common.hw,
+		[CLK_AHB_SPI2]		= &ahb_spi2_clk.common.hw,
+		[CLK_AHB_SPI3]		= &ahb_spi3_clk.common.hw,
+		[CLK_AHB_PATA]		= &ahb_pata_clk.common.hw,
+		[CLK_AHB_SATA]		= &ahb_sata_clk.common.hw,
+		[CLK_AHB_HSTIMER]	= &ahb_hstimer_clk.common.hw,
+		[CLK_AHB_VE]		= &ahb_ve_clk.common.hw,
+		[CLK_AHB_TVD]		= &ahb_tvd_clk.common.hw,
+		[CLK_AHB_TVE0]		= &ahb_tve0_clk.common.hw,
+		[CLK_AHB_TVE1]		= &ahb_tve1_clk.common.hw,
+		[CLK_AHB_LCD0]		= &ahb_lcd0_clk.common.hw,
+		[CLK_AHB_LCD1]		= &ahb_lcd1_clk.common.hw,
+		[CLK_AHB_CSI0]		= &ahb_csi0_clk.common.hw,
+		[CLK_AHB_CSI1]		= &ahb_csi1_clk.common.hw,
+		[CLK_AHB_HDMI1]		= &ahb_hdmi1_clk.common.hw,
+		[CLK_AHB_HDMI0]		= &ahb_hdmi0_clk.common.hw,
+		[CLK_AHB_DE_BE0]	= &ahb_de_be0_clk.common.hw,
+		[CLK_AHB_DE_BE1]	= &ahb_de_be1_clk.common.hw,
+		[CLK_AHB_DE_FE0]	= &ahb_de_fe0_clk.common.hw,
+		[CLK_AHB_DE_FE1]	= &ahb_de_fe1_clk.common.hw,
+		[CLK_AHB_GMAC]		= &ahb_gmac_clk.common.hw,
+		[CLK_AHB_MP]		= &ahb_mp_clk.common.hw,
+		[CLK_AHB_GPU]		= &ahb_gpu_clk.common.hw,
+		[CLK_APB0_CODEC]	= &apb0_codec_clk.common.hw,
+		[CLK_APB0_SPDIF]	= &apb0_spdif_clk.common.hw,
+		[CLK_APB0_AC97]		= &apb0_ac97_clk.common.hw,
+		[CLK_APB0_I2S0]		= &apb0_i2s0_clk.common.hw,
+		[CLK_APB0_I2S1]		= &apb0_i2s1_clk.common.hw,
+		[CLK_APB0_PIO]		= &apb0_pio_clk.common.hw,
+		[CLK_APB0_IR0]		= &apb0_ir0_clk.common.hw,
+		[CLK_APB0_IR1]		= &apb0_ir1_clk.common.hw,
+		[CLK_APB0_I2S2]		= &apb0_i2s2_clk.common.hw,
+		[CLK_APB0_KEYPAD]	= &apb0_keypad_clk.common.hw,
+		[CLK_APB1_I2C0]		= &apb1_i2c0_clk.common.hw,
+		[CLK_APB1_I2C1]		= &apb1_i2c1_clk.common.hw,
+		[CLK_APB1_I2C2]		= &apb1_i2c2_clk.common.hw,
+		[CLK_APB1_I2C3]		= &apb1_i2c3_clk.common.hw,
+		[CLK_APB1_CAN]		= &apb1_can_clk.common.hw,
+		[CLK_APB1_SCR]		= &apb1_scr_clk.common.hw,
+		[CLK_APB1_PS20]		= &apb1_ps20_clk.common.hw,
+		[CLK_APB1_PS21]		= &apb1_ps21_clk.common.hw,
+		[CLK_APB1_I2C4]		= &apb1_i2c4_clk.common.hw,
+		[CLK_APB1_UART0]	= &apb1_uart0_clk.common.hw,
+		[CLK_APB1_UART1]	= &apb1_uart1_clk.common.hw,
+		[CLK_APB1_UART2]	= &apb1_uart2_clk.common.hw,
+		[CLK_APB1_UART3]	= &apb1_uart3_clk.common.hw,
+		[CLK_APB1_UART4]	= &apb1_uart4_clk.common.hw,
+		[CLK_APB1_UART5]	= &apb1_uart5_clk.common.hw,
+		[CLK_APB1_UART6]	= &apb1_uart6_clk.common.hw,
+		[CLK_APB1_UART7]	= &apb1_uart7_clk.common.hw,
+		[CLK_NAND]		= &nand_clk.common.hw,
+		[CLK_MS]		= &ms_clk.common.hw,
+		[CLK_MMC0]		= &mmc0_clk.common.hw,
+		[CLK_MMC0_OUTPUT]	= &mmc0_output_clk.common.hw,
+		[CLK_MMC0_SAMPLE]	= &mmc0_sample_clk.common.hw,
+		[CLK_MMC1]		= &mmc1_clk.common.hw,
+		[CLK_MMC1_OUTPUT]	= &mmc1_output_clk.common.hw,
+		[CLK_MMC1_SAMPLE]	= &mmc1_sample_clk.common.hw,
+		[CLK_MMC2]		= &mmc2_clk.common.hw,
+		[CLK_MMC2_OUTPUT]	= &mmc2_output_clk.common.hw,
+		[CLK_MMC2_SAMPLE]	= &mmc2_sample_clk.common.hw,
+		[CLK_MMC3]		= &mmc3_clk.common.hw,
+		[CLK_MMC3_OUTPUT]	= &mmc3_output_clk.common.hw,
+		[CLK_MMC3_SAMPLE]	= &mmc3_sample_clk.common.hw,
+		[CLK_TS]		= &ts_clk.common.hw,
+		[CLK_SS]		= &ss_clk.common.hw,
+		[CLK_SPI0]		= &spi0_clk.common.hw,
+		[CLK_SPI1]		= &spi1_clk.common.hw,
+		[CLK_SPI2]		= &spi2_clk.common.hw,
+		[CLK_PATA]		= &pata_clk.common.hw,
+		[CLK_IR0]		= &ir0_sun7i_clk.common.hw,
+		[CLK_IR1]		= &ir1_sun7i_clk.common.hw,
+		[CLK_I2S0]		= &i2s0_clk.common.hw,
+		[CLK_AC97]		= &ac97_clk.common.hw,
+		[CLK_SPDIF]		= &spdif_clk.common.hw,
+		[CLK_KEYPAD]		= &keypad_clk.common.hw,
+		[CLK_SATA]		= &sata_clk.common.hw,
+		[CLK_USB_OHCI0]		= &usb_ohci0_clk.common.hw,
+		[CLK_USB_OHCI1]		= &usb_ohci1_clk.common.hw,
+		[CLK_USB_PHY]		= &usb_phy_clk.common.hw,
+		/* CLK_GPS is unimplemented */
+		[CLK_SPI3]		= &spi3_clk.common.hw,
+		[CLK_I2S1]		= &i2s1_clk.common.hw,
+		[CLK_I2S2]		= &i2s2_clk.common.hw,
+		[CLK_DRAM_VE]		= &dram_ve_clk.common.hw,
+		[CLK_DRAM_CSI0]		= &dram_csi0_clk.common.hw,
+		[CLK_DRAM_CSI1]		= &dram_csi1_clk.common.hw,
+		[CLK_DRAM_TS]		= &dram_ts_clk.common.hw,
+		[CLK_DRAM_TVD]		= &dram_tvd_clk.common.hw,
+		[CLK_DRAM_TVE0]		= &dram_tve0_clk.common.hw,
+		[CLK_DRAM_TVE1]		= &dram_tve1_clk.common.hw,
+		[CLK_DRAM_OUT]		= &dram_out_clk.common.hw,
+		[CLK_DRAM_DE_FE1]	= &dram_de_fe1_clk.common.hw,
+		[CLK_DRAM_DE_FE0]	= &dram_de_fe0_clk.common.hw,
+		[CLK_DRAM_DE_BE0]	= &dram_de_be0_clk.common.hw,
+		[CLK_DRAM_DE_BE1]	= &dram_de_be1_clk.common.hw,
+		[CLK_DRAM_MP]		= &dram_mp_clk.common.hw,
+		[CLK_DRAM_ACE]		= &dram_ace_clk.common.hw,
+		[CLK_DE_BE0]		= &de_be0_clk.common.hw,
+		[CLK_DE_BE1]		= &de_be1_clk.common.hw,
+		[CLK_DE_FE0]		= &de_fe0_clk.common.hw,
+		[CLK_DE_FE1]		= &de_fe1_clk.common.hw,
+		[CLK_DE_MP]		= &de_mp_clk.common.hw,
+		[CLK_TCON0_CH0]		= &tcon0_ch0_clk.common.hw,
+		[CLK_TCON1_CH0]		= &tcon1_ch0_clk.common.hw,
+		[CLK_CSI_SCLK]		= &csi_sclk_clk.common.hw,
+		[CLK_TVD_SCLK2]		= &tvd_sclk2_sun7i_clk.common.hw,
+		[CLK_TVD]		= &tvd_sclk1_sun7i_clk.common.hw,
+		[CLK_TCON0_CH1_SCLK2]	= &tcon0_ch1_sclk2_clk.common.hw,
+		[CLK_TCON0_CH1]		= &tcon0_ch1_clk.common.hw,
+		[CLK_TCON1_CH1_SCLK2]	= &tcon1_ch1_sclk2_clk.common.hw,
+		[CLK_TCON1_CH1]		= &tcon1_ch1_clk.common.hw,
+		[CLK_CSI0]		= &csi0_clk.common.hw,
+		[CLK_CSI1]		= &csi1_clk.common.hw,
+		[CLK_VE]		= &ve_clk.common.hw,
+		[CLK_CODEC]		= &codec_clk.common.hw,
+		[CLK_AVS]		= &avs_clk.common.hw,
+		[CLK_ACE]		= &ace_clk.common.hw,
+		[CLK_HDMI]		= &hdmi_clk.common.hw,
+		[CLK_GPU]		= &gpu_sun7i_clk.common.hw,
+		[CLK_MBUS]		= &mbus_sun7i_clk.common.hw,
+		[CLK_HDMI1_SLOW]	= &hdmi1_slow_clk.common.hw,
+		[CLK_HDMI1]		= &hdmi1_clk.common.hw,
+		[CLK_OUT_A]		= &out_a_clk.common.hw,
+		[CLK_OUT_B]		= &out_b_clk.common.hw,
+	},
+	.num	= CLK_NUMBER_SUN7I,
+};
+
+static struct ccu_reset_map sunxi_a10_a20_ccu_resets[] = {
+	[RST_USB_PHY0]		= { 0x0cc, BIT(0) },
+	[RST_USB_PHY1]		= { 0x0cc, BIT(1) },
+	[RST_USB_PHY2]		= { 0x0cc, BIT(2) },
+	[RST_GPS]		= { 0x0d0, BIT(0) },
+	[RST_DE_BE0]		= { 0x104, BIT(30) },
+	[RST_DE_BE1]		= { 0x108, BIT(30) },
+	[RST_DE_FE0]		= { 0x10c, BIT(30) },
+	[RST_DE_FE1]		= { 0x110, BIT(30) },
+	[RST_DE_MP]		= { 0x114, BIT(30) },
+	[RST_TVE0]		= { 0x118, BIT(29) },
+	[RST_TCON0]		= { 0x118, BIT(30) },
+	[RST_TVE1]		= { 0x11c, BIT(29) },
+	[RST_TCON1]		= { 0x11c, BIT(30) },
+	[RST_CSI0]		= { 0x134, BIT(30) },
+	[RST_CSI1]		= { 0x138, BIT(30) },
+	[RST_VE]		= { 0x13c, BIT(0) },
+	[RST_ACE]		= { 0x148, BIT(16) },
+	[RST_LVDS]		= { 0x14c, BIT(0) },
+	[RST_GPU]		= { 0x154, BIT(30) },
+	[RST_HDMI_H]		= { 0x170, BIT(0) },
+	[RST_HDMI_SYS]		= { 0x170, BIT(1) },
+	[RST_HDMI_AUDIO_DMA]	= { 0x170, BIT(2) },
+};
+
+static const struct sunxi_ccu_desc sun4i_a10_ccu_desc = {
+	.ccu_clks	= sun4i_sun7i_ccu_clks,
+	.num_ccu_clks	= ARRAY_SIZE(sun4i_sun7i_ccu_clks),
+
+	.hw_clks	= &sun4i_a10_hw_clks,
+
+	.resets		= sunxi_a10_a20_ccu_resets,
+	.num_resets	= ARRAY_SIZE(sunxi_a10_a20_ccu_resets),
+};
+
+static const struct sunxi_ccu_desc sun7i_a20_ccu_desc = {
+	.ccu_clks	= sun4i_sun7i_ccu_clks,
+	.num_ccu_clks	= ARRAY_SIZE(sun4i_sun7i_ccu_clks),
+
+	.hw_clks	= &sun7i_a20_hw_clks,
+
+	.resets		= sunxi_a10_a20_ccu_resets,
+	.num_resets	= ARRAY_SIZE(sunxi_a10_a20_ccu_resets),
+};
+
+static void __init sun4i_ccu_init(struct device_node *node,
+				  const struct sunxi_ccu_desc *desc)
+{
+	void __iomem *reg;
+	u32 val;
+
+	reg = of_io_request_and_map(node, 0, of_node_full_name(node));
+	if (IS_ERR(reg)) {
+		pr_err("%s: Could not map the clock registers\n",
+		       of_node_full_name(node));
+		return;
+	}
+
+	/* Force the PLL-Audio-1x divider to 4 */
+	val = readl(reg + SUN4I_PLL_AUDIO_REG);
+	val &= ~GENMASK(29, 26);
+	writel(val | (4 << 26), reg + SUN4I_PLL_AUDIO_REG);
+
+	/*
+	 * Use the peripheral PLL6 as the AHB parent, instead of CPU /
+	 * AXI which have rate changes due to cpufreq.
+	 *
+	 * This is especially a big deal for the HS timer whose parent
+	 * clock is AHB.
+	 *
+	 * NB! These bits are undocumented in A10 manual.
+	 */
+	val = readl(reg + SUN4I_AHB_REG);
+	val &= ~GENMASK(7, 6);
+	writel(val | (2 << 6), reg + SUN4I_AHB_REG);
+
+	sunxi_ccu_probe(node, reg, desc);
+}
+
+static void __init sun4i_a10_ccu_setup(struct device_node *node)
+{
+	sun4i_ccu_init(node, &sun4i_a10_ccu_desc);
+}
+CLK_OF_DECLARE(sun4i_a10_ccu, "allwinner,sun4i-a10-ccu",
+	       sun4i_a10_ccu_setup);
+
+static void __init sun7i_a20_ccu_setup(struct device_node *node)
+{
+	sun4i_ccu_init(node, &sun7i_a20_ccu_desc);
+}
+CLK_OF_DECLARE(sun7i_a20_ccu, "allwinner,sun7i-a20-ccu",
+	       sun7i_a20_ccu_setup);
diff --git a/drivers/clk/sunxi-ng/ccu-sun4i-a10.h b/drivers/clk/sunxi-ng/ccu-sun4i-a10.h
new file mode 100644
index 000000000000..c5947c7c050e
--- /dev/null
+++ b/drivers/clk/sunxi-ng/ccu-sun4i-a10.h
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2017 Priit Laes
+ *
+ * Priit Laes <plaes@plaes.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _CCU_SUN4I_A10_H_
+#define _CCU_SUN4I_A10_H_
+
+#include <dt-bindings/clock/sun4i-a10-ccu.h>
+#include <dt-bindings/clock/sun7i-a20-ccu.h>
+#include <dt-bindings/reset/sun4i-a10-ccu.h>
+
+/* The HOSC is exported */
+#define CLK_PLL_CORE		2
+#define CLK_PLL_AUDIO_BASE	3
+#define CLK_PLL_AUDIO		4
+#define CLK_PLL_AUDIO_2X	5
+#define CLK_PLL_AUDIO_4X	6
+#define CLK_PLL_AUDIO_8X	7
+#define CLK_PLL_VIDEO0		8
+#define CLK_PLL_VIDEO0_2X	9
+#define CLK_PLL_VE		10
+#define CLK_PLL_DDR_BASE	11
+#define CLK_PLL_DDR		12
+#define CLK_PLL_DDR_OTHER	13
+#define CLK_PLL_PERIPH_BASE	14
+#define CLK_PLL_PERIPH		15
+#define CLK_PLL_PERIPH_SATA	16
+#define CLK_PLL_VIDEO1		17
+#define CLK_PLL_VIDEO1_2X	18
+#define CLK_PLL_GPU		19
+
+/* The CPU clock is exported */
+#define CLK_AXI			21
+#define CLK_AXI_DRAM		22
+#define CLK_AHB			23
+#define CLK_APB0		24
+#define CLK_APB1		25
+
+/* AHB gates are exported (23..68) */
+/* APB0 gates are exported (69..78) */
+/* APB1 gates are exported (79..95) */
+/* IP module clocks are exported (96..128) */
+/* DRAM gates are exported (129..142)*/
+/* Media (display engine clocks & etc) are exported (143..169) */
+
+#define CLK_NUMBER_SUN4I	(CLK_MBUS + 1)
+#define CLK_NUMBER_SUN7I	(CLK_OUT_B + 1)
+
+#endif /* _CCU_SUN4I_A10_H_ */
diff --git a/drivers/clk/sunxi-ng/ccu-sun5i.c b/drivers/clk/sunxi-ng/ccu-sun5i.c
index 31d7ffda9aab..ab9e850b3707 100644
--- a/drivers/clk/sunxi-ng/ccu-sun5i.c
+++ b/drivers/clk/sunxi-ng/ccu-sun5i.c
@@ -976,8 +976,7 @@ static void __init sun5i_ccu_init(struct device_node *node,
 
 	reg = of_io_request_and_map(node, 0, of_node_full_name(node));
 	if (IS_ERR(reg)) {
-		pr_err("%s: Could not map the clock registers\n",
-		       of_node_full_name(node));
+		pr_err("%pOF: Could not map the clock registers\n", node);
 		return;
 	}
 
diff --git a/drivers/clk/sunxi-ng/ccu-sun6i-a31.c b/drivers/clk/sunxi-ng/ccu-sun6i-a31.c
index 4d6078fca9ac..8af434815fba 100644
--- a/drivers/clk/sunxi-ng/ccu-sun6i-a31.c
+++ b/drivers/clk/sunxi-ng/ccu-sun6i-a31.c
@@ -1217,8 +1217,7 @@ static void __init sun6i_a31_ccu_setup(struct device_node *node)
 
 	reg = of_io_request_and_map(node, 0, of_node_full_name(node));
 	if (IS_ERR(reg)) {
-		pr_err("%s: Could not map the clock registers\n",
-		       of_node_full_name(node));
+		pr_err("%pOF: Could not map the clock registers\n", node);
 		return;
 	}
 
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a23.c b/drivers/clk/sunxi-ng/ccu-sun8i-a23.c
index 8a753ed0426d..d93b452f0df9 100644
--- a/drivers/clk/sunxi-ng/ccu-sun8i-a23.c
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-a23.c
@@ -716,8 +716,7 @@ static void __init sun8i_a23_ccu_setup(struct device_node *node)
 
 	reg = of_io_request_and_map(node, 0, of_node_full_name(node));
 	if (IS_ERR(reg)) {
-		pr_err("%s: Could not map the clock registers\n",
-		       of_node_full_name(node));
+		pr_err("%pOF: Could not map the clock registers\n", node);
 		return;
 	}
 
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a33.c b/drivers/clk/sunxi-ng/ccu-sun8i-a33.c
index 10b38dc46f75..13eb5b23c5e7 100644
--- a/drivers/clk/sunxi-ng/ccu-sun8i-a33.c
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-a33.c
@@ -777,8 +777,7 @@ static void __init sun8i_a33_ccu_setup(struct device_node *node)
 
 	reg = of_io_request_and_map(node, 0, of_node_full_name(node));
 	if (IS_ERR(reg)) {
-		pr_err("%s: Could not map the clock registers\n",
-		       of_node_full_name(node));
+		pr_err("%pOF: Could not map the clock registers\n", node);
 		return;
 	}
 
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c b/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c
index 947f9f6e05d2..e43acebdfbcd 100644
--- a/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c
@@ -418,14 +418,8 @@ static SUNXI_CCU_PHASE(mmc1_sample_clk, "mmc1-sample", "mmc1",
 static SUNXI_CCU_PHASE(mmc1_output_clk, "mmc1-output", "mmc1",
 		       0x08c, 8, 3, 0);
 
-/* TODO Support MMC2 clock's new timing mode. */
-static SUNXI_CCU_MP_WITH_MUX_GATE(mmc2_clk, "mmc2", mod0_default_parents,
-				  0x090,
-				  0, 4,		/* M */
-				  16, 2,	/* P */
-				  24, 2,	/* mux */
-				  BIT(31),	/* gate */
-				  0);
+static SUNXI_CCU_MP_MMC_WITH_MUX_GATE(mmc2_clk, "mmc2", mod0_default_parents,
+				      0x090, 0);
 
 static SUNXI_CCU_PHASE(mmc2_sample_clk, "mmc2-sample", "mmc2",
 		       0x090, 20, 3, 0);
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-h3.c b/drivers/clk/sunxi-ng/ccu-sun8i-h3.c
index 62e4f0d2b2fc..1729ff6a5aae 100644
--- a/drivers/clk/sunxi-ng/ccu-sun8i-h3.c
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-h3.c
@@ -135,7 +135,7 @@ static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_de_clk, "pll-de",
 static const char * const cpux_parents[] = { "osc32k", "osc24M",
 					     "pll-cpux" , "pll-cpux" };
 static SUNXI_CCU_MUX(cpux_clk, "cpux", cpux_parents,
-		     0x050, 16, 2, CLK_IS_CRITICAL);
+		     0x050, 16, 2, CLK_IS_CRITICAL | CLK_SET_RATE_PARENT);
 
 static SUNXI_CCU_M(axi_clk, "axi", "cpux", 0x050, 0, 2, 0);
 
@@ -1103,6 +1103,13 @@ static const struct sunxi_ccu_desc sun50i_h5_ccu_desc = {
 	.num_resets	= ARRAY_SIZE(sun50i_h5_ccu_resets),
 };
 
+static struct ccu_pll_nb sun8i_h3_pll_cpu_nb = {
+	.common	= &pll_cpux_clk.common,
+	/* copy from pll_cpux_clk */
+	.enable	= BIT(31),
+	.lock	= BIT(28),
+};
+
 static struct ccu_mux_nb sun8i_h3_cpu_nb = {
 	.common		= &cpux_clk.common,
 	.cm		= &cpux_clk.mux,
@@ -1118,8 +1125,7 @@ static void __init sunxi_h3_h5_ccu_init(struct device_node *node,
 
 	reg = of_io_request_and_map(node, 0, of_node_full_name(node));
 	if (IS_ERR(reg)) {
-		pr_err("%s: Could not map the clock registers\n",
-		       of_node_full_name(node));
+		pr_err("%pOF: Could not map the clock registers\n", node);
 		return;
 	}
 
@@ -1130,6 +1136,10 @@ static void __init sunxi_h3_h5_ccu_init(struct device_node *node,
 
 	sunxi_ccu_probe(node, reg, desc);
 
+	/* Gate then ungate PLL CPU after any rate changes */
+	ccu_pll_notifier_register(&sun8i_h3_pll_cpu_nb);
+
+	/* Reparent CPU during PLL CPU rate changes */
 	ccu_mux_notifier_register(pll_cpux_clk.common.hw.clk,
 				  &sun8i_h3_cpu_nb);
 }
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-r.c b/drivers/clk/sunxi-ng/ccu-sun8i-r.c
index e54816ec1dbe..71feb7b24e8a 100644
--- a/drivers/clk/sunxi-ng/ccu-sun8i-r.c
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-r.c
@@ -290,8 +290,7 @@ static void __init sunxi_r_ccu_init(struct device_node *node,
 
 	reg = of_io_request_and_map(node, 0, of_node_full_name(node));
 	if (IS_ERR(reg)) {
-		pr_err("%s: Could not map the clock registers\n",
-		       of_node_full_name(node));
+		pr_err("%pOF: Could not map the clock registers\n", node);
 		return;
 	}
 
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-r.h b/drivers/clk/sunxi-ng/ccu-sun8i-r.h
index a7a407f12b56..fb01bffb929d 100644
--- a/drivers/clk/sunxi-ng/ccu-sun8i-r.h
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-r.h
@@ -13,7 +13,7 @@
  */
 
 #ifndef _CCU_SUN8I_R_H
-#define _CCU_SUN8I_R_H_
+#define _CCU_SUN8I_R_H
 
 #include <dt-bindings/clock/sun8i-r-ccu.h>
 #include <dt-bindings/reset/sun8i-r-ccu.h>
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-r40.c b/drivers/clk/sunxi-ng/ccu-sun8i-r40.c
new file mode 100644
index 000000000000..933f2e68f42a
--- /dev/null
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-r40.c
@@ -0,0 +1,1290 @@
+/*
+ * Copyright (c) 2017 Icenowy Zheng <icenowy@aosc.io>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/of_address.h>
+
+#include "ccu_common.h"
+#include "ccu_reset.h"
+
+#include "ccu_div.h"
+#include "ccu_gate.h"
+#include "ccu_mp.h"
+#include "ccu_mult.h"
+#include "ccu_nk.h"
+#include "ccu_nkm.h"
+#include "ccu_nkmp.h"
+#include "ccu_nm.h"
+#include "ccu_phase.h"
+
+#include "ccu-sun8i-r40.h"
+
+/* TODO: The result of N*K is required to be in [10, 88] range. */
+static struct ccu_nkmp pll_cpu_clk = {
+	.enable		= BIT(31),
+	.lock		= BIT(28),
+	.n		= _SUNXI_CCU_MULT(8, 5),
+	.k		= _SUNXI_CCU_MULT(4, 2),
+	.m		= _SUNXI_CCU_DIV(0, 2),
+	.p		= _SUNXI_CCU_DIV_MAX(16, 2, 4),
+	.common		= {
+		.reg		= 0x000,
+		.hw.init	= CLK_HW_INIT("pll-cpu",
+					      "osc24M",
+					      &ccu_nkmp_ops,
+					      CLK_SET_RATE_UNGATE),
+	},
+};
+
+/*
+ * The Audio PLL is supposed to have 4 outputs: 3 fixed factors from
+ * the base (2x, 4x and 8x), and one variable divider (the one true
+ * pll audio).
+ *
+ * We don't have any need for the variable divider for now, so we just
+ * hardcode it to match with the clock names
+ */
+#define SUN8I_R40_PLL_AUDIO_REG	0x008
+
+static SUNXI_CCU_NM_WITH_GATE_LOCK(pll_audio_base_clk, "pll-audio-base",
+				   "osc24M", 0x008,
+				   8, 7,	/* N */
+				   0, 5,	/* M */
+				   BIT(31),	/* gate */
+				   BIT(28),	/* lock */
+				   CLK_SET_RATE_UNGATE);
+
+/* TODO: The result of N/M is required to be in [8, 25] range. */
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_video0_clk, "pll-video0",
+					"osc24M", 0x0010,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+/* TODO: The result of N/M is required to be in [8, 25] range. */
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_ve_clk, "pll-ve",
+					"osc24M", 0x0018,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+/* TODO: The result of N*K is required to be in [10, 77] range. */
+static SUNXI_CCU_NKM_WITH_GATE_LOCK(pll_ddr0_clk, "pll-ddr0",
+				    "osc24M", 0x020,
+				    8, 5,	/* N */
+				    4, 2,	/* K */
+				    0, 2,	/* M */
+				    BIT(31),	/* gate */
+				    BIT(28),	/* lock */
+				    CLK_SET_RATE_UNGATE);
+
+/* TODO: The result of N*K is required to be in [21, 58] range. */
+static struct ccu_nk pll_periph0_clk = {
+	.enable		= BIT(31),
+	.lock		= BIT(28),
+	.n		= _SUNXI_CCU_MULT(8, 5),
+	.k		= _SUNXI_CCU_MULT(4, 2),
+	.fixed_post_div	= 2,
+	.common		= {
+		.reg		= 0x028,
+		.features	= CCU_FEATURE_FIXED_POSTDIV,
+		.hw.init	= CLK_HW_INIT("pll-periph0", "osc24M",
+					      &ccu_nk_ops,
+					      CLK_SET_RATE_UNGATE),
+	},
+};
+
+static struct ccu_div pll_periph0_sata_clk = {
+	.enable		= BIT(24),
+	.div		= _SUNXI_CCU_DIV(0, 2),
+	/*
+	 * The formula of pll-periph0 (1x) is 24MHz*N*K/2, and the formula
+	 * of pll-periph0-sata is 24MHz*N*K/M/6, so the postdiv here is
+	 * 6/2 = 3.
+	 */
+	.fixed_post_div	= 3,
+	.common		= {
+		.reg		= 0x028,
+		.features	= CCU_FEATURE_FIXED_POSTDIV,
+		.hw.init	= CLK_HW_INIT("pll-periph0-sata",
+					      "pll-periph0",
+					      &ccu_div_ops, 0),
+	},
+};
+
+/* TODO: The result of N*K is required to be in [21, 58] range. */
+static struct ccu_nk pll_periph1_clk = {
+	.enable		= BIT(31),
+	.lock		= BIT(28),
+	.n		= _SUNXI_CCU_MULT(8, 5),
+	.k		= _SUNXI_CCU_MULT(4, 2),
+	.fixed_post_div	= 2,
+	.common		= {
+		.reg		= 0x02c,
+		.features	= CCU_FEATURE_FIXED_POSTDIV,
+		.hw.init	= CLK_HW_INIT("pll-periph1", "osc24M",
+					      &ccu_nk_ops,
+					      CLK_SET_RATE_UNGATE),
+	},
+};
+
+/* TODO: The result of N/M is required to be in [8, 25] range. */
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_video1_clk, "pll-video1",
+					"osc24M", 0x030,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+static struct ccu_nkm pll_sata_clk = {
+	.enable		= BIT(31),
+	.lock		= BIT(28),
+	.n		= _SUNXI_CCU_MULT(8, 5),
+	.k		= _SUNXI_CCU_MULT(4, 2),
+	.m		= _SUNXI_CCU_DIV(0, 2),
+	.fixed_post_div	= 6,
+	.common		= {
+		.reg		= 0x034,
+		.features	= CCU_FEATURE_FIXED_POSTDIV,
+		.hw.init	= CLK_HW_INIT("pll-sata", "osc24M",
+					      &ccu_nkm_ops,
+					      CLK_SET_RATE_UNGATE),
+	},
+};
+
+static const char * const pll_sata_out_parents[] = { "pll-sata",
+						     "pll-periph0-sata" };
+static SUNXI_CCU_MUX_WITH_GATE(pll_sata_out_clk, "pll-sata-out",
+			       pll_sata_out_parents, 0x034,
+			       30, 1,	/* mux */
+			       BIT(14),	/* gate */
+			       CLK_SET_RATE_PARENT);
+
+/* TODO: The result of N/M is required to be in [8, 25] range. */
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_gpu_clk, "pll-gpu",
+					"osc24M", 0x038,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+/*
+ * The MIPI PLL has 2 modes: "MIPI" and "HDMI".
+ *
+ * The MIPI mode is a standard NKM-style clock. The HDMI mode is an
+ * integer / fractional clock with switchable multipliers and dividers.
+ * This is not supported here. We hardcode the PLL to MIPI mode.
+ *
+ * TODO: In the MIPI mode, M/N is required to be equal or lesser than 3,
+ * which cannot be implemented now.
+ */
+#define SUN8I_R40_PLL_MIPI_REG	0x040
+
+static const char * const pll_mipi_parents[] = { "pll-video0" };
+static struct ccu_nkm pll_mipi_clk = {
+	.enable	= BIT(31) | BIT(23) | BIT(22),
+	.lock	= BIT(28),
+	.n	= _SUNXI_CCU_MULT(8, 4),
+	.k	= _SUNXI_CCU_MULT_MIN(4, 2, 2),
+	.m	= _SUNXI_CCU_DIV(0, 4),
+	.mux	= _SUNXI_CCU_MUX(21, 1),
+	.common	= {
+		.reg		= 0x040,
+		.hw.init	= CLK_HW_INIT_PARENTS("pll-mipi",
+						      pll_mipi_parents,
+						      &ccu_nkm_ops,
+						      CLK_SET_RATE_UNGATE)
+	},
+};
+
+/* TODO: The result of N/M is required to be in [8, 25] range. */
+static SUNXI_CCU_NM_WITH_FRAC_GATE_LOCK(pll_de_clk, "pll-de",
+					"osc24M", 0x048,
+					8, 7,		/* N */
+					0, 4,		/* M */
+					BIT(24),	/* frac enable */
+					BIT(25),	/* frac select */
+					270000000,	/* frac rate 0 */
+					297000000,	/* frac rate 1 */
+					BIT(31),	/* gate */
+					BIT(28),	/* lock */
+					CLK_SET_RATE_UNGATE);
+
+/* TODO: The N factor is required to be in [16, 75] range. */
+static SUNXI_CCU_NM_WITH_GATE_LOCK(pll_ddr1_clk, "pll-ddr1",
+				   "osc24M", 0x04c,
+				   8, 7,	/* N */
+				   0, 2,	/* M */
+				   BIT(31),	/* gate */
+				   BIT(28),	/* lock */
+				   CLK_SET_RATE_UNGATE);
+
+static const char * const cpu_parents[] = { "osc32k", "osc24M",
+					     "pll-cpu", "pll-cpu" };
+static SUNXI_CCU_MUX(cpu_clk, "cpu", cpu_parents,
+		     0x050, 16, 2, CLK_IS_CRITICAL | CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_M(axi_clk, "axi", "cpu", 0x050, 0, 2, 0);
+
+static const char * const ahb1_parents[] = { "osc32k", "osc24M",
+					     "axi", "pll-periph0" };
+static const struct ccu_mux_var_prediv ahb1_predivs[] = {
+	{ .index = 3, .shift = 6, .width = 2 },
+};
+static struct ccu_div ahb1_clk = {
+	.div		= _SUNXI_CCU_DIV_FLAGS(4, 2, CLK_DIVIDER_POWER_OF_TWO),
+
+	.mux		= {
+		.shift	= 12,
+		.width	= 2,
+
+		.var_predivs	= ahb1_predivs,
+		.n_var_predivs	= ARRAY_SIZE(ahb1_predivs),
+	},
+
+	.common		= {
+		.reg		= 0x054,
+		.features	= CCU_FEATURE_VARIABLE_PREDIV,
+		.hw.init	= CLK_HW_INIT_PARENTS("ahb1",
+						      ahb1_parents,
+						      &ccu_div_ops,
+						      0),
+	},
+};
+
+static struct clk_div_table apb1_div_table[] = {
+	{ .val = 0, .div = 2 },
+	{ .val = 1, .div = 2 },
+	{ .val = 2, .div = 4 },
+	{ .val = 3, .div = 8 },
+	{ /* Sentinel */ },
+};
+static SUNXI_CCU_DIV_TABLE(apb1_clk, "apb1", "ahb1",
+			   0x054, 8, 2, apb1_div_table, 0);
+
+static const char * const apb2_parents[] = { "osc32k", "osc24M",
+					     "pll-periph0-2x",
+					     "pll-periph0-2x" };
+static SUNXI_CCU_MP_WITH_MUX(apb2_clk, "apb2", apb2_parents, 0x058,
+			     0, 5,	/* M */
+			     16, 2,	/* P */
+			     24, 2,	/* mux */
+			     0);
+
+static SUNXI_CCU_GATE(bus_mipi_dsi_clk,	"bus-mipi-dsi",	"ahb1",
+		      0x060, BIT(1), 0);
+static SUNXI_CCU_GATE(bus_ce_clk,	"bus-ce",	"ahb1",
+		      0x060, BIT(5), 0);
+static SUNXI_CCU_GATE(bus_dma_clk,	"bus-dma",	"ahb1",
+		      0x060, BIT(6), 0);
+static SUNXI_CCU_GATE(bus_mmc0_clk,	"bus-mmc0",	"ahb1",
+		      0x060, BIT(8), 0);
+static SUNXI_CCU_GATE(bus_mmc1_clk,	"bus-mmc1",	"ahb1",
+		      0x060, BIT(9), 0);
+static SUNXI_CCU_GATE(bus_mmc2_clk,	"bus-mmc2",	"ahb1",
+		      0x060, BIT(10), 0);
+static SUNXI_CCU_GATE(bus_mmc3_clk,	"bus-mmc3",	"ahb1",
+		      0x060, BIT(11), 0);
+static SUNXI_CCU_GATE(bus_nand_clk,	"bus-nand",	"ahb1",
+		      0x060, BIT(13), 0);
+static SUNXI_CCU_GATE(bus_dram_clk,	"bus-dram",	"ahb1",
+		      0x060, BIT(14), 0);
+static SUNXI_CCU_GATE(bus_emac_clk,	"bus-emac",	"ahb1",
+		      0x060, BIT(17), 0);
+static SUNXI_CCU_GATE(bus_ts_clk,	"bus-ts",	"ahb1",
+		      0x060, BIT(18), 0);
+static SUNXI_CCU_GATE(bus_hstimer_clk,	"bus-hstimer",	"ahb1",
+		      0x060, BIT(19), 0);
+static SUNXI_CCU_GATE(bus_spi0_clk,	"bus-spi0",	"ahb1",
+		      0x060, BIT(20), 0);
+static SUNXI_CCU_GATE(bus_spi1_clk,	"bus-spi1",	"ahb1",
+		      0x060, BIT(21), 0);
+static SUNXI_CCU_GATE(bus_spi2_clk,	"bus-spi2",	"ahb1",
+		      0x060, BIT(22), 0);
+static SUNXI_CCU_GATE(bus_spi3_clk,	"bus-spi3",	"ahb1",
+		      0x060, BIT(23), 0);
+static SUNXI_CCU_GATE(bus_sata_clk,	"bus-sata",	"ahb1",
+		      0x060, BIT(24), 0);
+static SUNXI_CCU_GATE(bus_otg_clk,	"bus-otg",	"ahb1",
+		      0x060, BIT(25), 0);
+static SUNXI_CCU_GATE(bus_ehci0_clk,	"bus-ehci0",	"ahb1",
+		      0x060, BIT(26), 0);
+static SUNXI_CCU_GATE(bus_ehci1_clk,	"bus-ehci1",	"ahb1",
+		      0x060, BIT(27), 0);
+static SUNXI_CCU_GATE(bus_ehci2_clk,	"bus-ehci2",	"ahb1",
+		      0x060, BIT(28), 0);
+static SUNXI_CCU_GATE(bus_ohci0_clk,	"bus-ohci0",	"ahb1",
+		      0x060, BIT(29), 0);
+static SUNXI_CCU_GATE(bus_ohci1_clk,	"bus-ohci1",	"ahb1",
+		      0x060, BIT(30), 0);
+static SUNXI_CCU_GATE(bus_ohci2_clk,	"bus-ohci2",	"ahb1",
+		      0x060, BIT(31), 0);
+
+static SUNXI_CCU_GATE(bus_ve_clk,	"bus-ve",	"ahb1",
+		      0x064, BIT(0), 0);
+static SUNXI_CCU_GATE(bus_mp_clk,	"bus-mp",	"ahb1",
+		      0x064, BIT(2), 0);
+static SUNXI_CCU_GATE(bus_deinterlace_clk,	"bus-deinterlace",	"ahb1",
+		      0x064, BIT(5), 0);
+static SUNXI_CCU_GATE(bus_csi0_clk,	"bus-csi0",	"ahb1",
+		      0x064, BIT(8), 0);
+static SUNXI_CCU_GATE(bus_csi1_clk,	"bus-csi1",	"ahb1",
+		      0x064, BIT(9), 0);
+static SUNXI_CCU_GATE(bus_hdmi0_clk,	"bus-hdmi0",	"ahb1",
+		      0x064, BIT(10), 0);
+static SUNXI_CCU_GATE(bus_hdmi1_clk,	"bus-hdmi1",	"ahb1",
+		      0x064, BIT(11), 0);
+static SUNXI_CCU_GATE(bus_de_clk,	"bus-de",	"ahb1",
+		      0x064, BIT(12), 0);
+static SUNXI_CCU_GATE(bus_tve0_clk,	"bus-tve0",	"ahb1",
+		      0x064, BIT(13), 0);
+static SUNXI_CCU_GATE(bus_tve1_clk,	"bus-tve1",	"ahb1",
+		      0x064, BIT(14), 0);
+static SUNXI_CCU_GATE(bus_tve_top_clk,	"bus-tve-top",	"ahb1",
+		      0x064, BIT(15), 0);
+static SUNXI_CCU_GATE(bus_gmac_clk,	"bus-gmac",	"ahb1",
+		      0x064, BIT(17), 0);
+static SUNXI_CCU_GATE(bus_gpu_clk,	"bus-gpu",	"ahb1",
+		      0x064, BIT(20), 0);
+static SUNXI_CCU_GATE(bus_tvd0_clk,	"bus-tvd0",	"ahb1",
+		      0x064, BIT(21), 0);
+static SUNXI_CCU_GATE(bus_tvd1_clk,	"bus-tvd1",	"ahb1",
+		      0x064, BIT(22), 0);
+static SUNXI_CCU_GATE(bus_tvd2_clk,	"bus-tvd2",	"ahb1",
+		      0x064, BIT(23), 0);
+static SUNXI_CCU_GATE(bus_tvd3_clk,	"bus-tvd3",	"ahb1",
+		      0x064, BIT(24), 0);
+static SUNXI_CCU_GATE(bus_tvd_top_clk,	"bus-tvd-top",	"ahb1",
+		      0x064, BIT(25), 0);
+static SUNXI_CCU_GATE(bus_tcon_lcd0_clk,	"bus-tcon-lcd0",	"ahb1",
+		      0x064, BIT(26), 0);
+static SUNXI_CCU_GATE(bus_tcon_lcd1_clk,	"bus-tcon-lcd1",	"ahb1",
+		      0x064, BIT(27), 0);
+static SUNXI_CCU_GATE(bus_tcon_tv0_clk,	"bus-tcon-tv0",	"ahb1",
+		      0x064, BIT(28), 0);
+static SUNXI_CCU_GATE(bus_tcon_tv1_clk,	"bus-tcon-tv1",	"ahb1",
+		      0x064, BIT(29), 0);
+static SUNXI_CCU_GATE(bus_tcon_top_clk,	"bus-tcon-top",	"ahb1",
+		      0x064, BIT(30), 0);
+
+static SUNXI_CCU_GATE(bus_codec_clk,	"bus-codec",	"apb1",
+		      0x068, BIT(0), 0);
+static SUNXI_CCU_GATE(bus_spdif_clk,	"bus-spdif",	"apb1",
+		      0x068, BIT(1), 0);
+static SUNXI_CCU_GATE(bus_ac97_clk,	"bus-ac97",	"apb1",
+		      0x068, BIT(2), 0);
+static SUNXI_CCU_GATE(bus_pio_clk,	"bus-pio",	"apb1",
+		      0x068, BIT(5), 0);
+static SUNXI_CCU_GATE(bus_ir0_clk,	"bus-ir0",	"apb1",
+		      0x068, BIT(6), 0);
+static SUNXI_CCU_GATE(bus_ir1_clk,	"bus-ir1",	"apb1",
+		      0x068, BIT(7), 0);
+static SUNXI_CCU_GATE(bus_ths_clk,	"bus-ths",	"apb1",
+		      0x068, BIT(8), 0);
+static SUNXI_CCU_GATE(bus_keypad_clk,	"bus-keypad",	"apb1",
+		      0x068, BIT(10), 0);
+static SUNXI_CCU_GATE(bus_i2s0_clk,	"bus-i2s0",	"apb1",
+		      0x068, BIT(12), 0);
+static SUNXI_CCU_GATE(bus_i2s1_clk,	"bus-i2s1",	"apb1",
+		      0x068, BIT(13), 0);
+static SUNXI_CCU_GATE(bus_i2s2_clk,	"bus-i2s2",	"apb1",
+		      0x068, BIT(14), 0);
+
+static SUNXI_CCU_GATE(bus_i2c0_clk,	"bus-i2c0",	"apb2",
+		      0x06c, BIT(0), 0);
+static SUNXI_CCU_GATE(bus_i2c1_clk,	"bus-i2c1",	"apb2",
+		      0x06c, BIT(1), 0);
+static SUNXI_CCU_GATE(bus_i2c2_clk,	"bus-i2c2",	"apb2",
+		      0x06c, BIT(2), 0);
+static SUNXI_CCU_GATE(bus_i2c3_clk,	"bus-i2c3",	"apb2",
+		      0x06c, BIT(3), 0);
+/*
+ * In datasheet here's "Reserved", however the gate exists in BSP soucre
+ * code.
+ */
+static SUNXI_CCU_GATE(bus_can_clk,	"bus-can",	"apb2",
+		      0x06c, BIT(4), 0);
+static SUNXI_CCU_GATE(bus_scr_clk,	"bus-scr",	"apb2",
+		      0x06c, BIT(5), 0);
+static SUNXI_CCU_GATE(bus_ps20_clk,	"bus-ps20",	"apb2",
+		      0x06c, BIT(6), 0);
+static SUNXI_CCU_GATE(bus_ps21_clk,	"bus-ps21",	"apb2",
+		      0x06c, BIT(7), 0);
+static SUNXI_CCU_GATE(bus_i2c4_clk,	"bus-i2c4",	"apb2",
+		      0x06c, BIT(15), 0);
+static SUNXI_CCU_GATE(bus_uart0_clk,	"bus-uart0",	"apb2",
+		      0x06c, BIT(16), 0);
+static SUNXI_CCU_GATE(bus_uart1_clk,	"bus-uart1",	"apb2",
+		      0x06c, BIT(17), 0);
+static SUNXI_CCU_GATE(bus_uart2_clk,	"bus-uart2",	"apb2",
+		      0x06c, BIT(18), 0);
+static SUNXI_CCU_GATE(bus_uart3_clk,	"bus-uart3",	"apb2",
+		      0x06c, BIT(19), 0);
+static SUNXI_CCU_GATE(bus_uart4_clk,	"bus-uart4",	"apb2",
+		      0x06c, BIT(20), 0);
+static SUNXI_CCU_GATE(bus_uart5_clk,	"bus-uart5",	"apb2",
+		      0x06c, BIT(21), 0);
+static SUNXI_CCU_GATE(bus_uart6_clk,	"bus-uart6",	"apb2",
+		      0x06c, BIT(22), 0);
+static SUNXI_CCU_GATE(bus_uart7_clk,	"bus-uart7",	"apb2",
+		      0x06c, BIT(23), 0);
+
+static SUNXI_CCU_GATE(bus_dbg_clk,	"bus-dbg",	"ahb1",
+		      0x070, BIT(7), 0);
+
+static const char * const ths_parents[] = { "osc24M" };
+static struct ccu_div ths_clk = {
+	.enable	= BIT(31),
+	.div	= _SUNXI_CCU_DIV_FLAGS(0, 2, CLK_DIVIDER_POWER_OF_TWO),
+	.mux	= _SUNXI_CCU_MUX(24, 2),
+	.common	= {
+		.reg		= 0x074,
+		.hw.init	= CLK_HW_INIT_PARENTS("ths",
+						      ths_parents,
+						      &ccu_div_ops,
+						      0),
+	},
+};
+
+static const char * const mod0_default_parents[] = { "osc24M", "pll-periph0",
+						     "pll-periph1" };
+static SUNXI_CCU_MP_WITH_MUX_GATE(nand_clk, "nand", mod0_default_parents, 0x080,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(mmc0_clk, "mmc0", mod0_default_parents, 0x088,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(mmc1_clk, "mmc1", mod0_default_parents, 0x08c,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(mmc2_clk, "mmc2", mod0_default_parents, 0x090,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(mmc3_clk, "mmc3", mod0_default_parents, 0x094,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static const char * const ts_parents[] = { "osc24M", "pll-periph0", };
+static SUNXI_CCU_MP_WITH_MUX_GATE(ts_clk, "ts", ts_parents, 0x098,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 4,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static const char * const ce_parents[] = { "osc24M", "pll-periph0-2x",
+					   "pll-periph1-2x" };
+static SUNXI_CCU_MP_WITH_MUX_GATE(ce_clk, "ce", ce_parents, 0x09c,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(spi0_clk, "spi0", mod0_default_parents, 0x0a0,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(spi1_clk, "spi1", mod0_default_parents, 0x0a4,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(spi2_clk, "spi2", mod0_default_parents, 0x0a8,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(spi3_clk, "spi3", mod0_default_parents, 0x0ac,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static const char * const i2s_parents[] = { "pll-audio-8x", "pll-audio-4x",
+					    "pll-audio-2x", "pll-audio" };
+static SUNXI_CCU_MUX_WITH_GATE(i2s0_clk, "i2s0", i2s_parents,
+			       0x0b0, 16, 2, BIT(31), CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_MUX_WITH_GATE(i2s1_clk, "i2s1", i2s_parents,
+			       0x0b4, 16, 2, BIT(31), CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_MUX_WITH_GATE(i2s2_clk, "i2s2", i2s_parents,
+			       0x0b8, 16, 2, BIT(31), CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_MUX_WITH_GATE(ac97_clk, "ac97", i2s_parents,
+			       0x0bc, 16, 2, BIT(31), CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_MUX_WITH_GATE(spdif_clk, "spdif", i2s_parents,
+			       0x0c0, 16, 2, BIT(31), CLK_SET_RATE_PARENT);
+
+static const char * const keypad_parents[] = { "osc24M", "osc32k" };
+static const u8 keypad_table[] = { 0, 2 };
+static struct ccu_mp keypad_clk = {
+	.enable	= BIT(31),
+	.m	= _SUNXI_CCU_DIV(0, 5),
+	.p	= _SUNXI_CCU_DIV(16, 2),
+	.mux	= _SUNXI_CCU_MUX_TABLE(24, 2, keypad_table),
+	.common	= {
+		.reg		= 0x0c4,
+		.hw.init	= CLK_HW_INIT_PARENTS("keypad",
+						      keypad_parents,
+						      &ccu_mp_ops,
+						      0),
+	}
+};
+
+static const char * const sata_parents[] = { "pll-sata-out", "sata-ext" };
+static SUNXI_CCU_MUX_WITH_GATE(sata_clk, "sata", sata_parents,
+			       0x0c8, 24, 1, BIT(31), CLK_SET_RATE_PARENT);
+
+/*
+ * There are 3 OHCI 12M clock source selection bits in this register.
+ * We will force them to 0 (12M divided from 48M).
+ */
+#define SUN8I_R40_USB_CLK_REG	0x0cc
+
+static SUNXI_CCU_GATE(usb_phy0_clk,	"usb-phy0",	"osc24M",
+		      0x0cc, BIT(8), 0);
+static SUNXI_CCU_GATE(usb_phy1_clk,	"usb-phy1",	"osc24M",
+		      0x0cc, BIT(9), 0);
+static SUNXI_CCU_GATE(usb_phy2_clk,	"usb-phy2",	"osc24M",
+		      0x0cc, BIT(10), 0);
+static SUNXI_CCU_GATE(usb_ohci0_clk,	"usb-ohci0",	"osc12M",
+		      0x0cc, BIT(16), 0);
+static SUNXI_CCU_GATE(usb_ohci1_clk,	"usb-ohci1",	"osc12M",
+		      0x0cc, BIT(17), 0);
+static SUNXI_CCU_GATE(usb_ohci2_clk,	"usb-ohci2",	"osc12M",
+		      0x0cc, BIT(18), 0);
+
+static const char * const ir_parents[] = { "osc24M", "pll-periph0",
+					   "pll-periph1", "osc32k" };
+static SUNXI_CCU_MP_WITH_MUX_GATE(ir0_clk, "ir0", ir_parents, 0x0d0,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE(ir1_clk, "ir1", ir_parents, 0x0d4,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  0);
+
+static const char * const dram_parents[] = { "pll-ddr0", "pll-ddr1" };
+static SUNXI_CCU_M_WITH_MUX(dram_clk, "dram", dram_parents,
+			    0x0f4, 0, 2, 20, 2, CLK_IS_CRITICAL);
+
+static SUNXI_CCU_GATE(dram_ve_clk,	"dram-ve",	"dram",
+		      0x100, BIT(0), 0);
+static SUNXI_CCU_GATE(dram_csi0_clk,	"dram-csi0",	"dram",
+		      0x100, BIT(1), 0);
+static SUNXI_CCU_GATE(dram_csi1_clk,	"dram-csi1",	"dram",
+		      0x100, BIT(2), 0);
+static SUNXI_CCU_GATE(dram_ts_clk,	"dram-ts",	"dram",
+		      0x100, BIT(3), 0);
+static SUNXI_CCU_GATE(dram_tvd_clk,	"dram-tvd",	"dram",
+		      0x100, BIT(4), 0);
+static SUNXI_CCU_GATE(dram_mp_clk,	"dram-mp",	"dram",
+		      0x100, BIT(5), 0);
+static SUNXI_CCU_GATE(dram_deinterlace_clk,	"dram-deinterlace",	"dram",
+		      0x100, BIT(6), 0);
+
+static const char * const de_parents[] = { "pll-periph0-2x", "pll-de" };
+static SUNXI_CCU_M_WITH_MUX_GATE(de_clk, "de", de_parents,
+				 0x104, 0, 4, 24, 3, BIT(31), 0);
+static SUNXI_CCU_M_WITH_MUX_GATE(mp_clk, "mp", de_parents,
+				 0x108, 0, 4, 24, 3, BIT(31), 0);
+
+static const char * const tcon_parents[] = { "pll-video0", "pll-video1",
+					     "pll-video0-2x", "pll-video1-2x",
+					     "pll-mipi" };
+static SUNXI_CCU_MUX_WITH_GATE(tcon_lcd0_clk, "tcon-lcd0", tcon_parents,
+			       0x110, 24, 3, BIT(31), CLK_SET_RATE_PARENT);
+static SUNXI_CCU_MUX_WITH_GATE(tcon_lcd1_clk, "tcon-lcd1", tcon_parents,
+			       0x114, 24, 3, BIT(31), CLK_SET_RATE_PARENT);
+static SUNXI_CCU_M_WITH_MUX_GATE(tcon_tv0_clk, "tcon-tv0", tcon_parents,
+				 0x118, 0, 4, 24, 3, BIT(31), 0);
+static SUNXI_CCU_M_WITH_MUX_GATE(tcon_tv1_clk, "tcon-tv1", tcon_parents,
+				 0x11c, 0, 4, 24, 3, BIT(31), 0);
+
+static const char * const deinterlace_parents[] = { "pll-periph0",
+						    "pll-periph1" };
+static SUNXI_CCU_M_WITH_MUX_GATE(deinterlace_clk, "deinterlace",
+				 deinterlace_parents, 0x124, 0, 4, 24, 3,
+				 BIT(31), 0);
+
+static const char * const csi_mclk_parents[] = { "osc24M", "pll-video1",
+						 "pll-periph1" };
+static SUNXI_CCU_M_WITH_MUX_GATE(csi1_mclk_clk, "csi1-mclk", csi_mclk_parents,
+				 0x130, 0, 5, 8, 3, BIT(15), 0);
+
+static const char * const csi_sclk_parents[] = { "pll-periph0", "pll-periph1" };
+static SUNXI_CCU_M_WITH_MUX_GATE(csi_sclk_clk, "csi-sclk", csi_sclk_parents,
+				 0x134, 16, 4, 24, 3, BIT(31), 0);
+
+static SUNXI_CCU_M_WITH_MUX_GATE(csi0_mclk_clk, "csi0-mclk", csi_mclk_parents,
+				 0x134, 0, 5, 8, 3, BIT(15), 0);
+
+static SUNXI_CCU_M_WITH_GATE(ve_clk, "ve", "pll-ve",
+			     0x13c, 16, 3, BIT(31), CLK_SET_RATE_PARENT);
+
+static SUNXI_CCU_GATE(codec_clk,	"codec",	"pll-audio",
+		      0x140, BIT(31), CLK_SET_RATE_PARENT);
+static SUNXI_CCU_GATE(avs_clk,		"avs",		"osc24M",
+		      0x144, BIT(31), 0);
+
+static const char * const hdmi_parents[] = { "pll-video0", "pll-video1" };
+static SUNXI_CCU_M_WITH_MUX_GATE(hdmi_clk, "hdmi", hdmi_parents,
+				 0x150, 0, 4, 24, 2, BIT(31), 0);
+
+static SUNXI_CCU_GATE(hdmi_slow_clk,	"hdmi-slow",	"osc24M",
+		      0x154, BIT(31), 0);
+
+/*
+ * In the SoC's user manual, the P factor is mentioned, but not used in
+ * the frequency formula.
+ *
+ * Here the factor is included, according to the BSP kernel source,
+ * which contains the P factor of this clock.
+ */
+static const char * const mbus_parents[] = { "osc24M", "pll-periph0-2x",
+					     "pll-ddr0" };
+static SUNXI_CCU_MP_WITH_MUX_GATE(mbus_clk, "mbus", mbus_parents, 0x15c,
+				  0, 4,		/* M */
+				  16, 2,	/* P */
+				  24, 2,	/* mux */
+				  BIT(31),	/* gate */
+				  CLK_IS_CRITICAL);
+
+static const char * const dsi_dphy_parents[] = { "pll-video0", "pll-video1",
+						 "pll-periph0" };
+static SUNXI_CCU_M_WITH_MUX_GATE(dsi_dphy_clk, "dsi-dphy", dsi_dphy_parents,
+				 0x168, 0, 4, 8, 2, BIT(15), 0);
+
+static SUNXI_CCU_M_WITH_MUX_GATE(tve0_clk, "tve0", tcon_parents,
+				 0x180, 0, 4, 24, 3, BIT(31), 0);
+static SUNXI_CCU_M_WITH_MUX_GATE(tve1_clk, "tve1", tcon_parents,
+				 0x184, 0, 4, 24, 3, BIT(31), 0);
+
+static const char * const tvd_parents[] = { "pll-video0", "pll-video1",
+					    "pll-video0-2x", "pll-video1-2x" };
+static SUNXI_CCU_M_WITH_MUX_GATE(tvd0_clk, "tvd0", tvd_parents,
+				 0x188, 0, 4, 24, 3, BIT(31), 0);
+static SUNXI_CCU_M_WITH_MUX_GATE(tvd1_clk, "tvd1", tvd_parents,
+				 0x18c, 0, 4, 24, 3, BIT(31), 0);
+static SUNXI_CCU_M_WITH_MUX_GATE(tvd2_clk, "tvd2", tvd_parents,
+				 0x190, 0, 4, 24, 3, BIT(31), 0);
+static SUNXI_CCU_M_WITH_MUX_GATE(tvd3_clk, "tvd3", tvd_parents,
+				 0x194, 0, 4, 24, 3, BIT(31), 0);
+
+static SUNXI_CCU_M_WITH_GATE(gpu_clk, "gpu", "pll-gpu",
+			     0x1a0, 0, 3, BIT(31), CLK_SET_RATE_PARENT);
+
+static const char * const out_parents[] = { "osc24M", "osc32k", "osc24M" };
+static const struct ccu_mux_fixed_prediv out_predivs[] = {
+	{ .index = 0, .div = 750, },
+};
+
+static struct ccu_mp outa_clk = {
+	.enable	= BIT(31),
+	.m	= _SUNXI_CCU_DIV(8, 5),
+	.p	= _SUNXI_CCU_DIV(20, 2),
+	.mux	= {
+		.shift		= 24,
+		.width		= 2,
+		.fixed_predivs	= out_predivs,
+		.n_predivs	= ARRAY_SIZE(out_predivs),
+	},
+	.common	= {
+		.reg		= 0x1f0,
+		.features	= CCU_FEATURE_FIXED_PREDIV,
+		.hw.init	= CLK_HW_INIT_PARENTS("outa", out_parents,
+						      &ccu_mp_ops, 0),
+	}
+};
+
+static struct ccu_mp outb_clk = {
+	.enable	= BIT(31),
+	.m	= _SUNXI_CCU_DIV(8, 5),
+	.p	= _SUNXI_CCU_DIV(20, 2),
+	.mux	= {
+		.shift		= 24,
+		.width		= 2,
+		.fixed_predivs	= out_predivs,
+		.n_predivs	= ARRAY_SIZE(out_predivs),
+	},
+	.common	= {
+		.reg		= 0x1f4,
+		.features	= CCU_FEATURE_FIXED_PREDIV,
+		.hw.init	= CLK_HW_INIT_PARENTS("outb", out_parents,
+						      &ccu_mp_ops, 0),
+	}
+};
+
+static struct ccu_common *sun8i_r40_ccu_clks[] = {
+	&pll_cpu_clk.common,
+	&pll_audio_base_clk.common,
+	&pll_video0_clk.common,
+	&pll_ve_clk.common,
+	&pll_ddr0_clk.common,
+	&pll_periph0_clk.common,
+	&pll_periph0_sata_clk.common,
+	&pll_periph1_clk.common,
+	&pll_video1_clk.common,
+	&pll_sata_clk.common,
+	&pll_sata_out_clk.common,
+	&pll_gpu_clk.common,
+	&pll_mipi_clk.common,
+	&pll_de_clk.common,
+	&pll_ddr1_clk.common,
+	&cpu_clk.common,
+	&axi_clk.common,
+	&ahb1_clk.common,
+	&apb1_clk.common,
+	&apb2_clk.common,
+	&bus_mipi_dsi_clk.common,
+	&bus_ce_clk.common,
+	&bus_dma_clk.common,
+	&bus_mmc0_clk.common,
+	&bus_mmc1_clk.common,
+	&bus_mmc2_clk.common,
+	&bus_mmc3_clk.common,
+	&bus_nand_clk.common,
+	&bus_dram_clk.common,
+	&bus_emac_clk.common,
+	&bus_ts_clk.common,
+	&bus_hstimer_clk.common,
+	&bus_spi0_clk.common,
+	&bus_spi1_clk.common,
+	&bus_spi2_clk.common,
+	&bus_spi3_clk.common,
+	&bus_sata_clk.common,
+	&bus_otg_clk.common,
+	&bus_ehci0_clk.common,
+	&bus_ehci1_clk.common,
+	&bus_ehci2_clk.common,
+	&bus_ohci0_clk.common,
+	&bus_ohci1_clk.common,
+	&bus_ohci2_clk.common,
+	&bus_ve_clk.common,
+	&bus_mp_clk.common,
+	&bus_deinterlace_clk.common,
+	&bus_csi0_clk.common,
+	&bus_csi1_clk.common,
+	&bus_hdmi0_clk.common,
+	&bus_hdmi1_clk.common,
+	&bus_de_clk.common,
+	&bus_tve0_clk.common,
+	&bus_tve1_clk.common,
+	&bus_tve_top_clk.common,
+	&bus_gmac_clk.common,
+	&bus_gpu_clk.common,
+	&bus_tvd0_clk.common,
+	&bus_tvd1_clk.common,
+	&bus_tvd2_clk.common,
+	&bus_tvd3_clk.common,
+	&bus_tvd_top_clk.common,
+	&bus_tcon_lcd0_clk.common,
+	&bus_tcon_lcd1_clk.common,
+	&bus_tcon_tv0_clk.common,
+	&bus_tcon_tv1_clk.common,
+	&bus_tcon_top_clk.common,
+	&bus_codec_clk.common,
+	&bus_spdif_clk.common,
+	&bus_ac97_clk.common,
+	&bus_pio_clk.common,
+	&bus_ir0_clk.common,
+	&bus_ir1_clk.common,
+	&bus_ths_clk.common,
+	&bus_keypad_clk.common,
+	&bus_i2s0_clk.common,
+	&bus_i2s1_clk.common,
+	&bus_i2s2_clk.common,
+	&bus_i2c0_clk.common,
+	&bus_i2c1_clk.common,
+	&bus_i2c2_clk.common,
+	&bus_i2c3_clk.common,
+	&bus_can_clk.common,
+	&bus_scr_clk.common,
+	&bus_ps20_clk.common,
+	&bus_ps21_clk.common,
+	&bus_i2c4_clk.common,
+	&bus_uart0_clk.common,
+	&bus_uart1_clk.common,
+	&bus_uart2_clk.common,
+	&bus_uart3_clk.common,
+	&bus_uart4_clk.common,
+	&bus_uart5_clk.common,
+	&bus_uart6_clk.common,
+	&bus_uart7_clk.common,
+	&bus_dbg_clk.common,
+	&ths_clk.common,
+	&nand_clk.common,
+	&mmc0_clk.common,
+	&mmc1_clk.common,
+	&mmc2_clk.common,
+	&mmc3_clk.common,
+	&ts_clk.common,
+	&ce_clk.common,
+	&spi0_clk.common,
+	&spi1_clk.common,
+	&spi2_clk.common,
+	&spi3_clk.common,
+	&i2s0_clk.common,
+	&i2s1_clk.common,
+	&i2s2_clk.common,
+	&ac97_clk.common,
+	&spdif_clk.common,
+	&keypad_clk.common,
+	&sata_clk.common,
+	&usb_phy0_clk.common,
+	&usb_phy1_clk.common,
+	&usb_phy2_clk.common,
+	&usb_ohci0_clk.common,
+	&usb_ohci1_clk.common,
+	&usb_ohci2_clk.common,
+	&ir0_clk.common,
+	&ir1_clk.common,
+	&dram_clk.common,
+	&dram_ve_clk.common,
+	&dram_csi0_clk.common,
+	&dram_csi1_clk.common,
+	&dram_ts_clk.common,
+	&dram_tvd_clk.common,
+	&dram_mp_clk.common,
+	&dram_deinterlace_clk.common,
+	&de_clk.common,
+	&mp_clk.common,
+	&tcon_lcd0_clk.common,
+	&tcon_lcd1_clk.common,
+	&tcon_tv0_clk.common,
+	&tcon_tv1_clk.common,
+	&deinterlace_clk.common,
+	&csi1_mclk_clk.common,
+	&csi_sclk_clk.common,
+	&csi0_mclk_clk.common,
+	&ve_clk.common,
+	&codec_clk.common,
+	&avs_clk.common,
+	&hdmi_clk.common,
+	&hdmi_slow_clk.common,
+	&mbus_clk.common,
+	&dsi_dphy_clk.common,
+	&tve0_clk.common,
+	&tve1_clk.common,
+	&tvd0_clk.common,
+	&tvd1_clk.common,
+	&tvd2_clk.common,
+	&tvd3_clk.common,
+	&gpu_clk.common,
+	&outa_clk.common,
+	&outb_clk.common,
+};
+
+/* Fixed Factor clocks */
+static CLK_FIXED_FACTOR(osc12M_clk, "osc12M", "osc24M", 2, 1, 0);
+
+/* We hardcode the divider to 4 for now */
+static CLK_FIXED_FACTOR(pll_audio_clk, "pll-audio",
+			"pll-audio-base", 4, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_audio_2x_clk, "pll-audio-2x",
+			"pll-audio-base", 2, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_audio_4x_clk, "pll-audio-4x",
+			"pll-audio-base", 1, 1, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_audio_8x_clk, "pll-audio-8x",
+			"pll-audio-base", 1, 2, CLK_SET_RATE_PARENT);
+static CLK_FIXED_FACTOR(pll_periph0_2x_clk, "pll-periph0-2x",
+			"pll-periph0", 1, 2, 0);
+static CLK_FIXED_FACTOR(pll_periph1_2x_clk, "pll-periph1-2x",
+			"pll-periph1", 1, 2, 0);
+static CLK_FIXED_FACTOR(pll_video0_2x_clk, "pll-video0-2x",
+			"pll-video0", 1, 2, 0);
+static CLK_FIXED_FACTOR(pll_video1_2x_clk, "pll-video1-2x",
+			"pll-video1", 1, 2, 0);
+
+static struct clk_hw_onecell_data sun8i_r40_hw_clks = {
+	.hws	= {
+		[CLK_OSC_12M]		= &osc12M_clk.hw,
+		[CLK_PLL_CPU]		= &pll_cpu_clk.common.hw,
+		[CLK_PLL_AUDIO_BASE]	= &pll_audio_base_clk.common.hw,
+		[CLK_PLL_AUDIO]		= &pll_audio_clk.hw,
+		[CLK_PLL_AUDIO_2X]	= &pll_audio_2x_clk.hw,
+		[CLK_PLL_AUDIO_4X]	= &pll_audio_4x_clk.hw,
+		[CLK_PLL_AUDIO_8X]	= &pll_audio_8x_clk.hw,
+		[CLK_PLL_VIDEO0]	= &pll_video0_clk.common.hw,
+		[CLK_PLL_VIDEO0_2X]	= &pll_video0_2x_clk.hw,
+		[CLK_PLL_VE]		= &pll_ve_clk.common.hw,
+		[CLK_PLL_DDR0]		= &pll_ddr0_clk.common.hw,
+		[CLK_PLL_PERIPH0]	= &pll_periph0_clk.common.hw,
+		[CLK_PLL_PERIPH0_SATA]	= &pll_periph0_sata_clk.common.hw,
+		[CLK_PLL_PERIPH0_2X]	= &pll_periph0_2x_clk.hw,
+		[CLK_PLL_PERIPH1]	= &pll_periph1_clk.common.hw,
+		[CLK_PLL_PERIPH1_2X]	= &pll_periph1_2x_clk.hw,
+		[CLK_PLL_VIDEO1]	= &pll_video1_clk.common.hw,
+		[CLK_PLL_VIDEO1_2X]	= &pll_video1_2x_clk.hw,
+		[CLK_PLL_SATA]		= &pll_sata_clk.common.hw,
+		[CLK_PLL_SATA_OUT]	= &pll_sata_out_clk.common.hw,
+		[CLK_PLL_GPU]		= &pll_gpu_clk.common.hw,
+		[CLK_PLL_MIPI]		= &pll_mipi_clk.common.hw,
+		[CLK_PLL_DE]		= &pll_de_clk.common.hw,
+		[CLK_PLL_DDR1]		= &pll_ddr1_clk.common.hw,
+		[CLK_CPU]		= &cpu_clk.common.hw,
+		[CLK_AXI]		= &axi_clk.common.hw,
+		[CLK_AHB1]		= &ahb1_clk.common.hw,
+		[CLK_APB1]		= &apb1_clk.common.hw,
+		[CLK_APB2]		= &apb2_clk.common.hw,
+		[CLK_BUS_MIPI_DSI]	= &bus_mipi_dsi_clk.common.hw,
+		[CLK_BUS_CE]		= &bus_ce_clk.common.hw,
+		[CLK_BUS_DMA]		= &bus_dma_clk.common.hw,
+		[CLK_BUS_MMC0]		= &bus_mmc0_clk.common.hw,
+		[CLK_BUS_MMC1]		= &bus_mmc1_clk.common.hw,
+		[CLK_BUS_MMC2]		= &bus_mmc2_clk.common.hw,
+		[CLK_BUS_MMC3]		= &bus_mmc3_clk.common.hw,
+		[CLK_BUS_NAND]		= &bus_nand_clk.common.hw,
+		[CLK_BUS_DRAM]		= &bus_dram_clk.common.hw,
+		[CLK_BUS_EMAC]		= &bus_emac_clk.common.hw,
+		[CLK_BUS_TS]		= &bus_ts_clk.common.hw,
+		[CLK_BUS_HSTIMER]	= &bus_hstimer_clk.common.hw,
+		[CLK_BUS_SPI0]		= &bus_spi0_clk.common.hw,
+		[CLK_BUS_SPI1]		= &bus_spi1_clk.common.hw,
+		[CLK_BUS_SPI2]		= &bus_spi2_clk.common.hw,
+		[CLK_BUS_SPI3]		= &bus_spi3_clk.common.hw,
+		[CLK_BUS_SATA]		= &bus_sata_clk.common.hw,
+		[CLK_BUS_OTG]		= &bus_otg_clk.common.hw,
+		[CLK_BUS_EHCI0]		= &bus_ehci0_clk.common.hw,
+		[CLK_BUS_EHCI1]		= &bus_ehci1_clk.common.hw,
+		[CLK_BUS_EHCI2]		= &bus_ehci2_clk.common.hw,
+		[CLK_BUS_OHCI0]		= &bus_ohci0_clk.common.hw,
+		[CLK_BUS_OHCI1]		= &bus_ohci1_clk.common.hw,
+		[CLK_BUS_OHCI2]		= &bus_ohci2_clk.common.hw,
+		[CLK_BUS_VE]		= &bus_ve_clk.common.hw,
+		[CLK_BUS_MP]		= &bus_mp_clk.common.hw,
+		[CLK_BUS_DEINTERLACE]	= &bus_deinterlace_clk.common.hw,
+		[CLK_BUS_CSI0]		= &bus_csi0_clk.common.hw,
+		[CLK_BUS_CSI1]		= &bus_csi1_clk.common.hw,
+		[CLK_BUS_HDMI0]		= &bus_hdmi0_clk.common.hw,
+		[CLK_BUS_HDMI1]		= &bus_hdmi1_clk.common.hw,
+		[CLK_BUS_DE]		= &bus_de_clk.common.hw,
+		[CLK_BUS_TVE0]		= &bus_tve0_clk.common.hw,
+		[CLK_BUS_TVE1]		= &bus_tve1_clk.common.hw,
+		[CLK_BUS_TVE_TOP]	= &bus_tve_top_clk.common.hw,
+		[CLK_BUS_GMAC]		= &bus_gmac_clk.common.hw,
+		[CLK_BUS_GPU]		= &bus_gpu_clk.common.hw,
+		[CLK_BUS_TVD0]		= &bus_tvd0_clk.common.hw,
+		[CLK_BUS_TVD1]		= &bus_tvd1_clk.common.hw,
+		[CLK_BUS_TVD2]		= &bus_tvd2_clk.common.hw,
+		[CLK_BUS_TVD3]		= &bus_tvd3_clk.common.hw,
+		[CLK_BUS_TVD_TOP]	= &bus_tvd_top_clk.common.hw,
+		[CLK_BUS_TCON_LCD0]	= &bus_tcon_lcd0_clk.common.hw,
+		[CLK_BUS_TCON_LCD1]	= &bus_tcon_lcd1_clk.common.hw,
+		[CLK_BUS_TCON_TV0]	= &bus_tcon_tv0_clk.common.hw,
+		[CLK_BUS_TCON_TV1]	= &bus_tcon_tv1_clk.common.hw,
+		[CLK_BUS_TCON_TOP]	= &bus_tcon_top_clk.common.hw,
+		[CLK_BUS_CODEC]		= &bus_codec_clk.common.hw,
+		[CLK_BUS_SPDIF]		= &bus_spdif_clk.common.hw,
+		[CLK_BUS_AC97]		= &bus_ac97_clk.common.hw,
+		[CLK_BUS_PIO]		= &bus_pio_clk.common.hw,
+		[CLK_BUS_IR0]		= &bus_ir0_clk.common.hw,
+		[CLK_BUS_IR1]		= &bus_ir1_clk.common.hw,
+		[CLK_BUS_THS]		= &bus_ths_clk.common.hw,
+		[CLK_BUS_KEYPAD]	= &bus_keypad_clk.common.hw,
+		[CLK_BUS_I2S0]		= &bus_i2s0_clk.common.hw,
+		[CLK_BUS_I2S1]		= &bus_i2s1_clk.common.hw,
+		[CLK_BUS_I2S2]		= &bus_i2s2_clk.common.hw,
+		[CLK_BUS_I2C0]		= &bus_i2c0_clk.common.hw,
+		[CLK_BUS_I2C1]		= &bus_i2c1_clk.common.hw,
+		[CLK_BUS_I2C2]		= &bus_i2c2_clk.common.hw,
+		[CLK_BUS_I2C3]		= &bus_i2c3_clk.common.hw,
+		[CLK_BUS_CAN]		= &bus_can_clk.common.hw,
+		[CLK_BUS_SCR]		= &bus_scr_clk.common.hw,
+		[CLK_BUS_PS20]		= &bus_ps20_clk.common.hw,
+		[CLK_BUS_PS21]		= &bus_ps21_clk.common.hw,
+		[CLK_BUS_I2C4]		= &bus_i2c4_clk.common.hw,
+		[CLK_BUS_UART0]		= &bus_uart0_clk.common.hw,
+		[CLK_BUS_UART1]		= &bus_uart1_clk.common.hw,
+		[CLK_BUS_UART2]		= &bus_uart2_clk.common.hw,
+		[CLK_BUS_UART3]		= &bus_uart3_clk.common.hw,
+		[CLK_BUS_UART4]		= &bus_uart4_clk.common.hw,
+		[CLK_BUS_UART5]		= &bus_uart5_clk.common.hw,
+		[CLK_BUS_UART6]		= &bus_uart6_clk.common.hw,
+		[CLK_BUS_UART7]		= &bus_uart7_clk.common.hw,
+		[CLK_BUS_DBG]		= &bus_dbg_clk.common.hw,
+		[CLK_THS]		= &ths_clk.common.hw,
+		[CLK_NAND]		= &nand_clk.common.hw,
+		[CLK_MMC0]		= &mmc0_clk.common.hw,
+		[CLK_MMC1]		= &mmc1_clk.common.hw,
+		[CLK_MMC2]		= &mmc2_clk.common.hw,
+		[CLK_MMC3]		= &mmc3_clk.common.hw,
+		[CLK_TS]		= &ts_clk.common.hw,
+		[CLK_CE]		= &ce_clk.common.hw,
+		[CLK_SPI0]		= &spi0_clk.common.hw,
+		[CLK_SPI1]		= &spi1_clk.common.hw,
+		[CLK_SPI2]		= &spi2_clk.common.hw,
+		[CLK_SPI3]		= &spi3_clk.common.hw,
+		[CLK_I2S0]		= &i2s0_clk.common.hw,
+		[CLK_I2S1]		= &i2s1_clk.common.hw,
+		[CLK_I2S2]		= &i2s2_clk.common.hw,
+		[CLK_AC97]		= &ac97_clk.common.hw,
+		[CLK_SPDIF]		= &spdif_clk.common.hw,
+		[CLK_KEYPAD]		= &keypad_clk.common.hw,
+		[CLK_SATA]		= &sata_clk.common.hw,
+		[CLK_USB_PHY0]		= &usb_phy0_clk.common.hw,
+		[CLK_USB_PHY1]		= &usb_phy1_clk.common.hw,
+		[CLK_USB_PHY2]		= &usb_phy2_clk.common.hw,
+		[CLK_USB_OHCI0]		= &usb_ohci0_clk.common.hw,
+		[CLK_USB_OHCI1]		= &usb_ohci1_clk.common.hw,
+		[CLK_USB_OHCI2]		= &usb_ohci2_clk.common.hw,
+		[CLK_IR0]		= &ir0_clk.common.hw,
+		[CLK_IR1]		= &ir1_clk.common.hw,
+		[CLK_DRAM]		= &dram_clk.common.hw,
+		[CLK_DRAM_VE]		= &dram_ve_clk.common.hw,
+		[CLK_DRAM_CSI0]		= &dram_csi0_clk.common.hw,
+		[CLK_DRAM_CSI1]		= &dram_csi1_clk.common.hw,
+		[CLK_DRAM_TS]		= &dram_ts_clk.common.hw,
+		[CLK_DRAM_TVD]		= &dram_tvd_clk.common.hw,
+		[CLK_DRAM_MP]		= &dram_mp_clk.common.hw,
+		[CLK_DRAM_DEINTERLACE]	= &dram_deinterlace_clk.common.hw,
+		[CLK_DE]		= &de_clk.common.hw,
+		[CLK_MP]		= &mp_clk.common.hw,
+		[CLK_TCON_LCD0]		= &tcon_lcd0_clk.common.hw,
+		[CLK_TCON_LCD1]		= &tcon_lcd1_clk.common.hw,
+		[CLK_TCON_TV0]		= &tcon_tv0_clk.common.hw,
+		[CLK_TCON_TV1]		= &tcon_tv1_clk.common.hw,
+		[CLK_DEINTERLACE]	= &deinterlace_clk.common.hw,
+		[CLK_CSI1_MCLK]		= &csi1_mclk_clk.common.hw,
+		[CLK_CSI_SCLK]		= &csi_sclk_clk.common.hw,
+		[CLK_CSI0_MCLK]		= &csi0_mclk_clk.common.hw,
+		[CLK_VE]		= &ve_clk.common.hw,
+		[CLK_CODEC]		= &codec_clk.common.hw,
+		[CLK_AVS]		= &avs_clk.common.hw,
+		[CLK_HDMI]		= &hdmi_clk.common.hw,
+		[CLK_HDMI_SLOW]		= &hdmi_slow_clk.common.hw,
+		[CLK_MBUS]		= &mbus_clk.common.hw,
+		[CLK_DSI_DPHY]		= &dsi_dphy_clk.common.hw,
+		[CLK_TVE0]		= &tve0_clk.common.hw,
+		[CLK_TVE1]		= &tve1_clk.common.hw,
+		[CLK_TVD0]		= &tvd0_clk.common.hw,
+		[CLK_TVD1]		= &tvd1_clk.common.hw,
+		[CLK_TVD2]		= &tvd2_clk.common.hw,
+		[CLK_TVD3]		= &tvd3_clk.common.hw,
+		[CLK_GPU]		= &gpu_clk.common.hw,
+		[CLK_OUTA]		= &outa_clk.common.hw,
+		[CLK_OUTB]		= &outb_clk.common.hw,
+	},
+	.num	= CLK_NUMBER,
+};
+
+static struct ccu_reset_map sun8i_r40_ccu_resets[] = {
+	[RST_USB_PHY0]		=  { 0x0cc, BIT(0) },
+	[RST_USB_PHY1]		=  { 0x0cc, BIT(1) },
+	[RST_USB_PHY2]		=  { 0x0cc, BIT(2) },
+
+	[RST_DRAM]		=  { 0x0f4, BIT(31) },
+	[RST_MBUS]		=  { 0x0fc, BIT(31) },
+
+	[RST_BUS_MIPI_DSI]	=  { 0x2c0, BIT(1) },
+	[RST_BUS_CE]		=  { 0x2c0, BIT(5) },
+	[RST_BUS_DMA]		=  { 0x2c0, BIT(6) },
+	[RST_BUS_MMC0]		=  { 0x2c0, BIT(8) },
+	[RST_BUS_MMC1]		=  { 0x2c0, BIT(9) },
+	[RST_BUS_MMC2]		=  { 0x2c0, BIT(10) },
+	[RST_BUS_MMC3]		=  { 0x2c0, BIT(11) },
+	[RST_BUS_NAND]		=  { 0x2c0, BIT(13) },
+	[RST_BUS_DRAM]		=  { 0x2c0, BIT(14) },
+	[RST_BUS_EMAC]		=  { 0x2c0, BIT(17) },
+	[RST_BUS_TS]		=  { 0x2c0, BIT(18) },
+	[RST_BUS_HSTIMER]	=  { 0x2c0, BIT(19) },
+	[RST_BUS_SPI0]		=  { 0x2c0, BIT(20) },
+	[RST_BUS_SPI1]		=  { 0x2c0, BIT(21) },
+	[RST_BUS_SPI2]		=  { 0x2c0, BIT(22) },
+	[RST_BUS_SPI3]		=  { 0x2c0, BIT(23) },
+	[RST_BUS_SATA]		=  { 0x2c0, BIT(24) },
+	[RST_BUS_OTG]		=  { 0x2c0, BIT(25) },
+	[RST_BUS_EHCI0]		=  { 0x2c0, BIT(26) },
+	[RST_BUS_EHCI1]		=  { 0x2c0, BIT(27) },
+	[RST_BUS_EHCI2]		=  { 0x2c0, BIT(28) },
+	[RST_BUS_OHCI0]		=  { 0x2c0, BIT(29) },
+	[RST_BUS_OHCI1]		=  { 0x2c0, BIT(30) },
+	[RST_BUS_OHCI2]		=  { 0x2c0, BIT(31) },
+
+	[RST_BUS_VE]		=  { 0x2c4, BIT(0) },
+	[RST_BUS_MP]		=  { 0x2c4, BIT(2) },
+	[RST_BUS_DEINTERLACE]	=  { 0x2c4, BIT(5) },
+	[RST_BUS_CSI0]		=  { 0x2c4, BIT(8) },
+	[RST_BUS_CSI1]		=  { 0x2c4, BIT(9) },
+	[RST_BUS_HDMI0]		=  { 0x2c4, BIT(10) },
+	[RST_BUS_HDMI1]		=  { 0x2c4, BIT(11) },
+	[RST_BUS_DE]		=  { 0x2c4, BIT(12) },
+	[RST_BUS_TVE0]		=  { 0x2c4, BIT(13) },
+	[RST_BUS_TVE1]		=  { 0x2c4, BIT(14) },
+	[RST_BUS_TVE_TOP]	=  { 0x2c4, BIT(15) },
+	[RST_BUS_GMAC]		=  { 0x2c4, BIT(17) },
+	[RST_BUS_GPU]		=  { 0x2c4, BIT(20) },
+	[RST_BUS_TVD0]		=  { 0x2c4, BIT(21) },
+	[RST_BUS_TVD1]		=  { 0x2c4, BIT(22) },
+	[RST_BUS_TVD2]		=  { 0x2c4, BIT(23) },
+	[RST_BUS_TVD3]		=  { 0x2c4, BIT(24) },
+	[RST_BUS_TVD_TOP]	=  { 0x2c4, BIT(25) },
+	[RST_BUS_TCON_LCD0]	=  { 0x2c4, BIT(26) },
+	[RST_BUS_TCON_LCD1]	=  { 0x2c4, BIT(27) },
+	[RST_BUS_TCON_TV0]	=  { 0x2c4, BIT(28) },
+	[RST_BUS_TCON_TV1]	=  { 0x2c4, BIT(29) },
+	[RST_BUS_TCON_TOP]	=  { 0x2c4, BIT(30) },
+	[RST_BUS_DBG]		=  { 0x2c4, BIT(31) },
+
+	[RST_BUS_LVDS]		=  { 0x2c8, BIT(0) },
+
+	[RST_BUS_CODEC]		=  { 0x2d0, BIT(0) },
+	[RST_BUS_SPDIF]		=  { 0x2d0, BIT(1) },
+	[RST_BUS_AC97]		=  { 0x2d0, BIT(2) },
+	[RST_BUS_IR0]		=  { 0x2d0, BIT(6) },
+	[RST_BUS_IR1]		=  { 0x2d0, BIT(7) },
+	[RST_BUS_THS]		=  { 0x2d0, BIT(8) },
+	[RST_BUS_KEYPAD]	=  { 0x2d0, BIT(10) },
+	[RST_BUS_I2S0]		=  { 0x2d0, BIT(12) },
+	[RST_BUS_I2S1]		=  { 0x2d0, BIT(13) },
+	[RST_BUS_I2S2]		=  { 0x2d0, BIT(14) },
+
+	[RST_BUS_I2C0]		=  { 0x2d8, BIT(0) },
+	[RST_BUS_I2C1]		=  { 0x2d8, BIT(1) },
+	[RST_BUS_I2C2]		=  { 0x2d8, BIT(2) },
+	[RST_BUS_I2C3]		=  { 0x2d8, BIT(3) },
+	[RST_BUS_CAN]		=  { 0x2d8, BIT(4) },
+	[RST_BUS_SCR]		=  { 0x2d8, BIT(5) },
+	[RST_BUS_PS20]		=  { 0x2d8, BIT(6) },
+	[RST_BUS_PS21]		=  { 0x2d8, BIT(7) },
+	[RST_BUS_I2C4]		=  { 0x2d8, BIT(15) },
+	[RST_BUS_UART0]		=  { 0x2d8, BIT(16) },
+	[RST_BUS_UART1]		=  { 0x2d8, BIT(17) },
+	[RST_BUS_UART2]		=  { 0x2d8, BIT(18) },
+	[RST_BUS_UART3]		=  { 0x2d8, BIT(19) },
+	[RST_BUS_UART4]		=  { 0x2d8, BIT(20) },
+	[RST_BUS_UART5]		=  { 0x2d8, BIT(21) },
+	[RST_BUS_UART6]		=  { 0x2d8, BIT(22) },
+	[RST_BUS_UART7]		=  { 0x2d8, BIT(23) },
+};
+
+static const struct sunxi_ccu_desc sun8i_r40_ccu_desc = {
+	.ccu_clks	= sun8i_r40_ccu_clks,
+	.num_ccu_clks	= ARRAY_SIZE(sun8i_r40_ccu_clks),
+
+	.hw_clks	= &sun8i_r40_hw_clks,
+
+	.resets		= sun8i_r40_ccu_resets,
+	.num_resets	= ARRAY_SIZE(sun8i_r40_ccu_resets),
+};
+
+static struct ccu_pll_nb sun8i_r40_pll_cpu_nb = {
+	.common	= &pll_cpu_clk.common,
+	/* copy from pll_cpu_clk */
+	.enable	= BIT(31),
+	.lock	= BIT(28),
+};
+
+static struct ccu_mux_nb sun8i_r40_cpu_nb = {
+	.common		= &cpu_clk.common,
+	.cm		= &cpu_clk.mux,
+	.delay_us	= 1, /* > 8 clock cycles at 24 MHz */
+	.bypass_index	= 1, /* index of 24 MHz oscillator */
+};
+
+static void __init sun8i_r40_ccu_setup(struct device_node *node)
+{
+	void __iomem *reg;
+	u32 val;
+
+	reg = of_io_request_and_map(node, 0, of_node_full_name(node));
+	if (IS_ERR(reg)) {
+		pr_err("%s: Could not map the clock registers\n",
+		       of_node_full_name(node));
+		return;
+	}
+
+	/* Force the PLL-Audio-1x divider to 4 */
+	val = readl(reg + SUN8I_R40_PLL_AUDIO_REG);
+	val &= ~GENMASK(19, 16);
+	writel(val | (3 << 16), reg + SUN8I_R40_PLL_AUDIO_REG);
+
+	/* Force PLL-MIPI to MIPI mode */
+	val = readl(reg + SUN8I_R40_PLL_MIPI_REG);
+	val &= ~BIT(16);
+	writel(val, reg + SUN8I_R40_PLL_MIPI_REG);
+
+	/* Force OHCI 12M parent to 12M divided from 48M */
+	val = readl(reg + SUN8I_R40_USB_CLK_REG);
+	val &= ~GENMASK(25, 20);
+	writel(val, reg + SUN8I_R40_USB_CLK_REG);
+
+	sunxi_ccu_probe(node, reg, &sun8i_r40_ccu_desc);
+
+	/* Gate then ungate PLL CPU after any rate changes */
+	ccu_pll_notifier_register(&sun8i_r40_pll_cpu_nb);
+
+	/* Reparent CPU during PLL CPU rate changes */
+	ccu_mux_notifier_register(pll_cpu_clk.common.hw.clk,
+				  &sun8i_r40_cpu_nb);
+}
+CLK_OF_DECLARE(sun8i_r40_ccu, "allwinner,sun8i-r40-ccu",
+	       sun8i_r40_ccu_setup);
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-r40.h b/drivers/clk/sunxi-ng/ccu-sun8i-r40.h
new file mode 100644
index 000000000000..0db8e1e97af8
--- /dev/null
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-r40.h
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2017 Icenowy Zheng <icenowy@aosc.io>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _CCU_SUN8I_R40_H_
+#define _CCU_SUN8I_R40_H_
+
+#include <dt-bindings/clock/sun8i-r40-ccu.h>
+#include <dt-bindings/reset/sun8i-r40-ccu.h>
+
+#define CLK_OSC_12M		0
+#define CLK_PLL_CPU		1
+#define CLK_PLL_AUDIO_BASE	2
+#define CLK_PLL_AUDIO		3
+#define CLK_PLL_AUDIO_2X	4
+#define CLK_PLL_AUDIO_4X	5
+#define CLK_PLL_AUDIO_8X	6
+#define CLK_PLL_VIDEO0		7
+#define CLK_PLL_VIDEO0_2X	8
+#define CLK_PLL_VE		9
+#define CLK_PLL_DDR0		10
+#define CLK_PLL_PERIPH0		11
+#define CLK_PLL_PERIPH0_SATA	12
+#define CLK_PLL_PERIPH0_2X	13
+#define CLK_PLL_PERIPH1		14
+#define CLK_PLL_PERIPH1_2X	15
+#define CLK_PLL_VIDEO1		16
+#define CLK_PLL_VIDEO1_2X	17
+#define CLK_PLL_SATA		18
+#define CLK_PLL_SATA_OUT	19
+#define CLK_PLL_GPU		20
+#define CLK_PLL_MIPI		21
+#define CLK_PLL_DE		22
+#define CLK_PLL_DDR1		23
+
+/* The CPU clock is exported */
+
+#define CLK_AXI			25
+#define CLK_AHB1		26
+#define CLK_APB1		27
+#define CLK_APB2		28
+
+/* All the bus gates are exported */
+
+/* The first bunch of module clocks are exported */
+
+#define CLK_DRAM		132
+
+/* All the DRAM gates are exported */
+
+/* Some more module clocks are exported */
+
+#define CLK_MBUS		155
+
+/* Another bunch of module clocks are exported */
+
+#define CLK_NUMBER		(CLK_OUTB + 1)
+
+#endif /* _CCU_SUN8I_R40_H_ */
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c b/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c
index a34a78d7fb28..621b1cd996db 100644
--- a/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c
@@ -575,8 +575,7 @@ static void __init sun8i_v3s_ccu_setup(struct device_node *node)
 
 	reg = of_io_request_and_map(node, 0, of_node_full_name(node));
 	if (IS_ERR(reg)) {
-		pr_err("%s: Could not map the clock registers\n",
-		       of_node_full_name(node));
+		pr_err("%pOF: Could not map the clock registers\n", node);
 		return;
 	}
 
diff --git a/drivers/clk/sunxi-ng/ccu_common.h b/drivers/clk/sunxi-ng/ccu_common.h
index d6fdd7a789aa..cadd1a9f93b6 100644
--- a/drivers/clk/sunxi-ng/ccu_common.h
+++ b/drivers/clk/sunxi-ng/ccu_common.h
@@ -23,6 +23,10 @@
 #define CCU_FEATURE_FIXED_POSTDIV	BIT(3)
 #define CCU_FEATURE_ALL_PREDIV		BIT(4)
 #define CCU_FEATURE_LOCK_REG		BIT(5)
+#define CCU_FEATURE_MMC_TIMING_SWITCH	BIT(6)
+
+/* MMC timing mode switch bit */
+#define CCU_MMC_NEW_TIMING_MODE		BIT(30)
 
 struct device_node;
 
diff --git a/drivers/clk/sunxi-ng/ccu_div.c b/drivers/clk/sunxi-ng/ccu_div.c
index c0e5c10d0091..baa3cf96507b 100644
--- a/drivers/clk/sunxi-ng/ccu_div.c
+++ b/drivers/clk/sunxi-ng/ccu_div.c
@@ -21,10 +21,18 @@ static unsigned long ccu_div_round_rate(struct ccu_mux_internal *mux,
 {
 	struct ccu_div *cd = data;
 
-	return divider_round_rate_parent(&cd->common.hw, parent,
+	if (cd->common.features & CCU_FEATURE_FIXED_POSTDIV)
+		rate *= cd->fixed_post_div;
+
+	rate = divider_round_rate_parent(&cd->common.hw, parent,
 					 rate, parent_rate,
 					 cd->div.table, cd->div.width,
 					 cd->div.flags);
+
+	if (cd->common.features & CCU_FEATURE_FIXED_POSTDIV)
+		rate /= cd->fixed_post_div;
+
+	return rate;
 }
 
 static void ccu_div_disable(struct clk_hw *hw)
@@ -62,8 +70,13 @@ static unsigned long ccu_div_recalc_rate(struct clk_hw *hw,
 	parent_rate = ccu_mux_helper_apply_prediv(&cd->common, &cd->mux, -1,
 						  parent_rate);
 
-	return divider_recalc_rate(hw, parent_rate, val, cd->div.table,
-				   cd->div.flags);
+	val = divider_recalc_rate(hw, parent_rate, val, cd->div.table,
+				  cd->div.flags);
+
+	if (cd->common.features & CCU_FEATURE_FIXED_POSTDIV)
+		val /= cd->fixed_post_div;
+
+	return val;
 }
 
 static int ccu_div_determine_rate(struct clk_hw *hw,
@@ -86,6 +99,9 @@ static int ccu_div_set_rate(struct clk_hw *hw, unsigned long rate,
 	parent_rate = ccu_mux_helper_apply_prediv(&cd->common, &cd->mux, -1,
 						  parent_rate);
 
+	if (cd->common.features & CCU_FEATURE_FIXED_POSTDIV)
+		rate *= cd->fixed_post_div;
+
 	val = divider_get_val(rate, parent_rate, cd->div.table, cd->div.width,
 			      cd->div.flags);
 
diff --git a/drivers/clk/sunxi-ng/ccu_div.h b/drivers/clk/sunxi-ng/ccu_div.h
index 08d074451204..f3a5028dcd14 100644
--- a/drivers/clk/sunxi-ng/ccu_div.h
+++ b/drivers/clk/sunxi-ng/ccu_div.h
@@ -86,9 +86,10 @@ struct ccu_div_internal {
 struct ccu_div {
 	u32			enable;
 
-	struct ccu_div_internal		div;
+	struct ccu_div_internal	div;
 	struct ccu_mux_internal	mux;
 	struct ccu_common	common;
+	unsigned int		fixed_post_div;
 };
 
 #define SUNXI_CCU_DIV_TABLE_WITH_GATE(_struct, _name, _parent, _reg,	\
diff --git a/drivers/clk/sunxi-ng/ccu_frac.c b/drivers/clk/sunxi-ng/ccu_frac.c
index 8b5eb7756bf7..d1d168d4c4f0 100644
--- a/drivers/clk/sunxi-ng/ccu_frac.c
+++ b/drivers/clk/sunxi-ng/ccu_frac.c
@@ -67,25 +67,25 @@ unsigned long ccu_frac_helper_read_rate(struct ccu_common *common,
 {
 	u32 reg;
 
-	printk("%s: Read fractional\n", clk_hw_get_name(&common->hw));
+	pr_debug("%s: Read fractional\n", clk_hw_get_name(&common->hw));
 
 	if (!(common->features & CCU_FEATURE_FRACTIONAL))
 		return 0;
 
-	printk("%s: clock is fractional (rates %lu and %lu)\n",
-	       clk_hw_get_name(&common->hw), cf->rates[0], cf->rates[1]);
+	pr_debug("%s: clock is fractional (rates %lu and %lu)\n",
+		 clk_hw_get_name(&common->hw), cf->rates[0], cf->rates[1]);
 
 	reg = readl(common->base + common->reg);
 
-	printk("%s: clock reg is 0x%x (select is 0x%x)\n",
-	       clk_hw_get_name(&common->hw), reg, cf->select);
+	pr_debug("%s: clock reg is 0x%x (select is 0x%x)\n",
+		 clk_hw_get_name(&common->hw), reg, cf->select);
 
 	return (reg & cf->select) ? cf->rates[1] : cf->rates[0];
 }
 
 int ccu_frac_helper_set_rate(struct ccu_common *common,
 			     struct ccu_frac_internal *cf,
-			     unsigned long rate)
+			     unsigned long rate, u32 lock)
 {
 	unsigned long flags;
 	u32 reg, sel;
@@ -106,5 +106,7 @@ int ccu_frac_helper_set_rate(struct ccu_common *common,
 	writel(reg | sel, common->base + common->reg);
 	spin_unlock_irqrestore(common->lock, flags);
 
+	ccu_helper_wait_for_lock(common, lock);
+
 	return 0;
 }
diff --git a/drivers/clk/sunxi-ng/ccu_frac.h b/drivers/clk/sunxi-ng/ccu_frac.h
index 7b1ee380156f..efe2dd6bac01 100644
--- a/drivers/clk/sunxi-ng/ccu_frac.h
+++ b/drivers/clk/sunxi-ng/ccu_frac.h
@@ -48,6 +48,6 @@ unsigned long ccu_frac_helper_read_rate(struct ccu_common *common,
 
 int ccu_frac_helper_set_rate(struct ccu_common *common,
 			     struct ccu_frac_internal *cf,
-			     unsigned long rate);
+			     unsigned long rate, u32 lock);
 
 #endif /* _CCU_FRAC_H_ */
diff --git a/drivers/clk/sunxi-ng/ccu_mmc_timing.c b/drivers/clk/sunxi-ng/ccu_mmc_timing.c
new file mode 100644
index 000000000000..f9869f7353c0
--- /dev/null
+++ b/drivers/clk/sunxi-ng/ccu_mmc_timing.c
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2017 Chen-Yu Tsai. All rights reserved.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/clk/sunxi-ng.h>
+
+#include "ccu_common.h"
+
+/**
+ * sunxi_ccu_set_mmc_timing_mode: Configure the MMC clock timing mode
+ * @clk: clock to be configured
+ * @new_mode: true for new timing mode introduced in A83T and later
+ *
+ * Returns 0 on success, -ENOTSUPP if the clock does not support
+ * switching modes.
+ */
+int sunxi_ccu_set_mmc_timing_mode(struct clk *clk, bool new_mode)
+{
+	struct clk_hw *hw = __clk_get_hw(clk);
+	struct ccu_common *cm = hw_to_ccu_common(hw);
+	unsigned long flags;
+	u32 val;
+
+	if (!(cm->features & CCU_FEATURE_MMC_TIMING_SWITCH))
+		return -ENOTSUPP;
+
+	spin_lock_irqsave(cm->lock, flags);
+
+	val = readl(cm->base + cm->reg);
+	if (new_mode)
+		val |= CCU_MMC_NEW_TIMING_MODE;
+	else
+		val &= ~CCU_MMC_NEW_TIMING_MODE;
+	writel(val, cm->base + cm->reg);
+
+	spin_unlock_irqrestore(cm->lock, flags);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(sunxi_ccu_set_mmc_timing_mode);
+
+/**
+ * sunxi_ccu_set_mmc_timing_mode: Get the current MMC clock timing mode
+ * @clk: clock to query
+ *
+ * Returns 0 if the clock is in old timing mode, > 0 if it is in
+ * new timing mode, and -ENOTSUPP if the clock does not support
+ * this function.
+ */
+int sunxi_ccu_get_mmc_timing_mode(struct clk *clk)
+{
+	struct clk_hw *hw = __clk_get_hw(clk);
+	struct ccu_common *cm = hw_to_ccu_common(hw);
+
+	if (!(cm->features & CCU_FEATURE_MMC_TIMING_SWITCH))
+		return -ENOTSUPP;
+
+	return !!(readl(cm->base + cm->reg) & CCU_MMC_NEW_TIMING_MODE);
+}
+EXPORT_SYMBOL_GPL(sunxi_ccu_get_mmc_timing_mode);
diff --git a/drivers/clk/sunxi-ng/ccu_mp.c b/drivers/clk/sunxi-ng/ccu_mp.c
index b917ad7a386c..688855e7dc8c 100644
--- a/drivers/clk/sunxi-ng/ccu_mp.c
+++ b/drivers/clk/sunxi-ng/ccu_mp.c
@@ -172,3 +172,83 @@ const struct clk_ops ccu_mp_ops = {
 	.recalc_rate	= ccu_mp_recalc_rate,
 	.set_rate	= ccu_mp_set_rate,
 };
+
+/*
+ * Support for MMC timing mode switching
+ *
+ * The MMC clocks on some SoCs support switching between old and
+ * new timing modes. A platform specific API is provided to query
+ * and set the timing mode on supported SoCs.
+ *
+ * In addition, a special class of ccu_mp_ops is provided, which
+ * takes in to account the timing mode switch. When the new timing
+ * mode is active, the clock output rate is halved. This new class
+ * is a wrapper around the generic ccu_mp_ops. When clock rates
+ * are passed through to ccu_mp_ops callbacks, they are doubled
+ * if the new timing mode bit is set, to account for the post
+ * divider. Conversely, when clock rates are passed back, they
+ * are halved if the mode bit is set.
+ */
+
+static unsigned long ccu_mp_mmc_recalc_rate(struct clk_hw *hw,
+					    unsigned long parent_rate)
+{
+	unsigned long rate = ccu_mp_recalc_rate(hw, parent_rate);
+	struct ccu_common *cm = hw_to_ccu_common(hw);
+	u32 val = readl(cm->base + cm->reg);
+
+	if (val & CCU_MMC_NEW_TIMING_MODE)
+		return rate / 2;
+	return rate;
+}
+
+static int ccu_mp_mmc_determine_rate(struct clk_hw *hw,
+				     struct clk_rate_request *req)
+{
+	struct ccu_common *cm = hw_to_ccu_common(hw);
+	u32 val = readl(cm->base + cm->reg);
+	int ret;
+
+	/* adjust the requested clock rate */
+	if (val & CCU_MMC_NEW_TIMING_MODE) {
+		req->rate *= 2;
+		req->min_rate *= 2;
+		req->max_rate *= 2;
+	}
+
+	ret = ccu_mp_determine_rate(hw, req);
+
+	/* re-adjust the requested clock rate back */
+	if (val & CCU_MMC_NEW_TIMING_MODE) {
+		req->rate /= 2;
+		req->min_rate /= 2;
+		req->max_rate /= 2;
+	}
+
+	return ret;
+}
+
+static int ccu_mp_mmc_set_rate(struct clk_hw *hw, unsigned long rate,
+			       unsigned long parent_rate)
+{
+	struct ccu_common *cm = hw_to_ccu_common(hw);
+	u32 val = readl(cm->base + cm->reg);
+
+	if (val & CCU_MMC_NEW_TIMING_MODE)
+		rate *= 2;
+
+	return ccu_mp_set_rate(hw, rate, parent_rate);
+}
+
+const struct clk_ops ccu_mp_mmc_ops = {
+	.disable	= ccu_mp_disable,
+	.enable		= ccu_mp_enable,
+	.is_enabled	= ccu_mp_is_enabled,
+
+	.get_parent	= ccu_mp_get_parent,
+	.set_parent	= ccu_mp_set_parent,
+
+	.determine_rate	= ccu_mp_mmc_determine_rate,
+	.recalc_rate	= ccu_mp_mmc_recalc_rate,
+	.set_rate	= ccu_mp_mmc_set_rate,
+};
diff --git a/drivers/clk/sunxi-ng/ccu_mp.h b/drivers/clk/sunxi-ng/ccu_mp.h
index 915625e97d98..aaef11d747ea 100644
--- a/drivers/clk/sunxi-ng/ccu_mp.h
+++ b/drivers/clk/sunxi-ng/ccu_mp.h
@@ -14,6 +14,7 @@
 #ifndef _CCU_MP_H_
 #define _CCU_MP_H_
 
+#include <linux/bitops.h>
 #include <linux/clk-provider.h>
 
 #include "ccu_common.h"
@@ -74,4 +75,33 @@ static inline struct ccu_mp *hw_to_ccu_mp(struct clk_hw *hw)
 
 extern const struct clk_ops ccu_mp_ops;
 
+/*
+ * Special class of M-P clock that supports MMC timing modes
+ *
+ * Since the MMC clock registers all follow the same layout, we can
+ * simplify the macro for this particular case. In addition, as
+ * switching modes also affects the output clock rate, we need to
+ * have CLK_GET_RATE_NOCACHE for all these types of clocks.
+ */
+
+#define SUNXI_CCU_MP_MMC_WITH_MUX_GATE(_struct, _name, _parents, _reg,	\
+				       _flags)				\
+	struct ccu_mp _struct = {					\
+		.enable	= BIT(31),					\
+		.m	= _SUNXI_CCU_DIV(0, 4),				\
+		.p	= _SUNXI_CCU_DIV(16, 2),			\
+		.mux	= _SUNXI_CCU_MUX(24, 2),			\
+		.common	= {						\
+			.reg		= _reg,				\
+			.features	= CCU_FEATURE_MMC_TIMING_SWITCH, \
+			.hw.init	= CLK_HW_INIT_PARENTS(_name,	\
+							      _parents, \
+							      &ccu_mp_mmc_ops, \
+							      CLK_GET_RATE_NOCACHE | \
+							      _flags),	\
+		}							\
+	}
+
+extern const struct clk_ops ccu_mp_mmc_ops;
+
 #endif /* _CCU_MP_H_ */
diff --git a/drivers/clk/sunxi-ng/ccu_mult.c b/drivers/clk/sunxi-ng/ccu_mult.c
index 20d0300867f2..12e0783caee6 100644
--- a/drivers/clk/sunxi-ng/ccu_mult.c
+++ b/drivers/clk/sunxi-ng/ccu_mult.c
@@ -111,10 +111,14 @@ static int ccu_mult_set_rate(struct clk_hw *hw, unsigned long rate,
 	unsigned long flags;
 	u32 reg;
 
-	if (ccu_frac_helper_has_rate(&cm->common, &cm->frac, rate))
-		return ccu_frac_helper_set_rate(&cm->common, &cm->frac, rate);
-	else
+	if (ccu_frac_helper_has_rate(&cm->common, &cm->frac, rate)) {
+		ccu_frac_helper_enable(&cm->common, &cm->frac);
+
+		return ccu_frac_helper_set_rate(&cm->common, &cm->frac,
+						rate, cm->lock);
+	} else {
 		ccu_frac_helper_disable(&cm->common, &cm->frac);
+	}
 
 	parent_rate = ccu_mux_helper_apply_prediv(&cm->common, &cm->mux, -1,
 						  parent_rate);
diff --git a/drivers/clk/sunxi-ng/ccu_nkm.c b/drivers/clk/sunxi-ng/ccu_nkm.c
index 44b16dc8fea6..841840e35e61 100644
--- a/drivers/clk/sunxi-ng/ccu_nkm.c
+++ b/drivers/clk/sunxi-ng/ccu_nkm.c
@@ -75,7 +75,7 @@ static unsigned long ccu_nkm_recalc_rate(struct clk_hw *hw,
 					unsigned long parent_rate)
 {
 	struct ccu_nkm *nkm = hw_to_ccu_nkm(hw);
-	unsigned long n, m, k;
+	unsigned long n, m, k, rate;
 	u32 reg;
 
 	reg = readl(nkm->common.base + nkm->common.reg);
@@ -98,7 +98,12 @@ static unsigned long ccu_nkm_recalc_rate(struct clk_hw *hw,
 	if (!m)
 		m++;
 
-	return parent_rate * n  * k / m;
+	rate = parent_rate * n  * k / m;
+
+	if (nkm->common.features & CCU_FEATURE_FIXED_POSTDIV)
+		rate /= nkm->fixed_post_div;
+
+	return rate;
 }
 
 static unsigned long ccu_nkm_round_rate(struct ccu_mux_internal *mux,
@@ -117,9 +122,17 @@ static unsigned long ccu_nkm_round_rate(struct ccu_mux_internal *mux,
 	_nkm.min_m = 1;
 	_nkm.max_m = nkm->m.max ?: 1 << nkm->m.width;
 
+	if (nkm->common.features & CCU_FEATURE_FIXED_POSTDIV)
+		rate *= nkm->fixed_post_div;
+
 	ccu_nkm_find_best(*parent_rate, rate, &_nkm);
 
-	return *parent_rate * _nkm.n * _nkm.k / _nkm.m;
+	rate = *parent_rate * _nkm.n * _nkm.k / _nkm.m;
+
+	if (nkm->common.features & CCU_FEATURE_FIXED_POSTDIV)
+		rate /= nkm->fixed_post_div;
+
+	return rate;
 }
 
 static int ccu_nkm_determine_rate(struct clk_hw *hw,
@@ -139,6 +152,9 @@ static int ccu_nkm_set_rate(struct clk_hw *hw, unsigned long rate,
 	unsigned long flags;
 	u32 reg;
 
+	if (nkm->common.features & CCU_FEATURE_FIXED_POSTDIV)
+		rate *= nkm->fixed_post_div;
+
 	_nkm.min_n = nkm->n.min ?: 1;
 	_nkm.max_n = nkm->n.max ?: 1 << nkm->n.width;
 	_nkm.min_k = nkm->k.min ?: 1;
diff --git a/drivers/clk/sunxi-ng/ccu_nkm.h b/drivers/clk/sunxi-ng/ccu_nkm.h
index 34580894f4d1..cc6efb70a102 100644
--- a/drivers/clk/sunxi-ng/ccu_nkm.h
+++ b/drivers/clk/sunxi-ng/ccu_nkm.h
@@ -34,6 +34,8 @@ struct ccu_nkm {
 	struct ccu_div_internal		m;
 	struct ccu_mux_internal	mux;
 
+	unsigned int		fixed_post_div;
+
 	struct ccu_common	common;
 };
 
diff --git a/drivers/clk/sunxi-ng/ccu_nm.c b/drivers/clk/sunxi-ng/ccu_nm.c
index 5e5e90a4a50c..a32158e8f2e3 100644
--- a/drivers/clk/sunxi-ng/ccu_nm.c
+++ b/drivers/clk/sunxi-ng/ccu_nm.c
@@ -117,10 +117,23 @@ static int ccu_nm_set_rate(struct clk_hw *hw, unsigned long rate,
 	unsigned long flags;
 	u32 reg;
 
-	if (ccu_frac_helper_has_rate(&nm->common, &nm->frac, rate))
-		return ccu_frac_helper_set_rate(&nm->common, &nm->frac, rate);
-	else
+	if (ccu_frac_helper_has_rate(&nm->common, &nm->frac, rate)) {
+		spin_lock_irqsave(nm->common.lock, flags);
+
+		/* most SoCs require M to be 0 if fractional mode is used */
+		reg = readl(nm->common.base + nm->common.reg);
+		reg &= ~GENMASK(nm->m.width + nm->m.shift - 1, nm->m.shift);
+		writel(reg, nm->common.base + nm->common.reg);
+
+		spin_unlock_irqrestore(nm->common.lock, flags);
+
+		ccu_frac_helper_enable(&nm->common, &nm->frac);
+
+		return ccu_frac_helper_set_rate(&nm->common, &nm->frac,
+						rate, nm->lock);
+	} else {
 		ccu_frac_helper_disable(&nm->common, &nm->frac);
+	}
 
 	_nm.min_n = nm->n.min ?: 1;
 	_nm.max_n = nm->n.max ?: 1 << nm->n.width;
diff --git a/drivers/clk/sunxi-ng/ccu_reset.c b/drivers/clk/sunxi-ng/ccu_reset.c
index 6c31d48783a7..1dc4e98ea802 100644
--- a/drivers/clk/sunxi-ng/ccu_reset.c
+++ b/drivers/clk/sunxi-ng/ccu_reset.c
@@ -8,6 +8,7 @@
  * the License, or (at your option) any later version.
  */
 
+#include <linux/delay.h>
 #include <linux/io.h>
 #include <linux/reset-controller.h>
 
@@ -49,7 +50,18 @@ static int ccu_reset_deassert(struct reset_controller_dev *rcdev,
 	return 0;
 }
 
+static int ccu_reset_reset(struct reset_controller_dev *rcdev,
+			   unsigned long id)
+{
+	ccu_reset_assert(rcdev, id);
+	udelay(10);
+	ccu_reset_deassert(rcdev, id);
+
+	return 0;
+}
+
 const struct reset_control_ops ccu_reset_ops = {
 	.assert		= ccu_reset_assert,
 	.deassert	= ccu_reset_deassert,
+	.reset		= ccu_reset_reset,
 };
diff --git a/drivers/clk/sunxi/clk-sun8i-bus-gates.c b/drivers/clk/sunxi/clk-sun8i-bus-gates.c
index 63fdb790df29..bee305bdddbe 100644
--- a/drivers/clk/sunxi/clk-sun8i-bus-gates.c
+++ b/drivers/clk/sunxi/clk-sun8i-bus-gates.c
@@ -78,6 +78,10 @@ static void __init sun8i_h3_bus_gates_init(struct device_node *node)
 			clk_parent = APB1;
 		else if (index >= 96 && index <= 127)
 			clk_parent = APB2;
+		else {
+			WARN_ON(true);
+			continue;
+		}
 
 		clk_reg = reg + 4 * (index / 32);
 		clk_bit = index % 32;
diff --git a/drivers/clk/sunxi/clk-sunxi.c b/drivers/clk/sunxi/clk-sunxi.c
index f2c9274b8bd5..aa4add580516 100644
--- a/drivers/clk/sunxi/clk-sunxi.c
+++ b/drivers/clk/sunxi/clk-sunxi.c
@@ -666,15 +666,14 @@ static struct clk * __init sunxi_mux_clk_setup(struct device_node *node,
 
 	reg = of_iomap(node, 0);
 	if (!reg) {
-		pr_err("Could not map registers for mux-clk: %s\n",
-		       of_node_full_name(node));
+		pr_err("Could not map registers for mux-clk: %pOF\n", node);
 		return NULL;
 	}
 
 	i = of_clk_parent_fill(node, parents, SUNXI_MAX_PARENTS);
 	if (of_property_read_string(node, "clock-output-names", &clk_name)) {
-		pr_err("%s: could not read clock-output-names from \"%s\"\n",
-		       __func__, of_node_full_name(node));
+		pr_err("%s: could not read clock-output-names from \"%pOF\"\n",
+		       __func__, node);
 		goto out_unmap;
 	}
 
@@ -797,16 +796,15 @@ static void __init sunxi_divider_clk_setup(struct device_node *node,
 
 	reg = of_iomap(node, 0);
 	if (!reg) {
-		pr_err("Could not map registers for mux-clk: %s\n",
-		       of_node_full_name(node));
+		pr_err("Could not map registers for mux-clk: %pOF\n", node);
 		return;
 	}
 
 	clk_parent = of_clk_get_parent_name(node, 0);
 
 	if (of_property_read_string(node, "clock-output-names", &clk_name)) {
-		pr_err("%s: could not read clock-output-names from \"%s\"\n",
-		       __func__, of_node_full_name(node));
+		pr_err("%s: could not read clock-output-names from \"%pOF\"\n",
+		       __func__, node);
 		goto out_unmap;
 	}
 
@@ -1010,8 +1008,7 @@ static struct clk ** __init sunxi_divs_clk_setup(struct device_node *node,
 
 	reg = of_iomap(node, 0);
 	if (!reg) {
-		pr_err("Could not map registers for divs-clk: %s\n",
-		       of_node_full_name(node));
+		pr_err("Could not map registers for divs-clk: %pOF\n", node);
 		return NULL;
 	}
 
diff --git a/drivers/clk/tegra/clk-emc.c b/drivers/clk/tegra/clk-emc.c
index 74e7544f861b..11a5066e5c27 100644
--- a/drivers/clk/tegra/clk-emc.c
+++ b/drivers/clk/tegra/clk-emc.c
@@ -378,7 +378,7 @@ static int load_one_timing_from_dt(struct tegra_clk_emc *tegra,
 
 	err = of_property_read_u32(node, "clock-frequency", &tmp);
 	if (err) {
-		pr_err("timing %s: failed to read rate\n", node->full_name);
+		pr_err("timing %pOF: failed to read rate\n", node);
 		return err;
 	}
 
@@ -386,8 +386,7 @@ static int load_one_timing_from_dt(struct tegra_clk_emc *tegra,
 
 	err = of_property_read_u32(node, "nvidia,parent-clock-frequency", &tmp);
 	if (err) {
-		pr_err("timing %s: failed to read parent rate\n",
-		       node->full_name);
+		pr_err("timing %pOF: failed to read parent rate\n", node);
 		return err;
 	}
 
@@ -395,8 +394,7 @@ static int load_one_timing_from_dt(struct tegra_clk_emc *tegra,
 
 	timing->parent = of_clk_get_by_name(node, "emc-parent");
 	if (IS_ERR(timing->parent)) {
-		pr_err("timing %s: failed to get parent clock\n",
-		       node->full_name);
+		pr_err("timing %pOF: failed to get parent clock\n", node);
 		return PTR_ERR(timing->parent);
 	}
 
@@ -409,8 +407,8 @@ static int load_one_timing_from_dt(struct tegra_clk_emc *tegra,
 		}
 	}
 	if (timing->parent_index == 0xff) {
-		pr_err("timing %s: %s is not a valid parent\n",
-		       node->full_name, __clk_get_name(timing->parent));
+		pr_err("timing %pOF: %s is not a valid parent\n",
+		       node, __clk_get_name(timing->parent));
 		clk_put(timing->parent);
 		return -EINVAL;
 	}
diff --git a/drivers/clk/tegra/clk-pll.c b/drivers/clk/tegra/clk-pll.c
index 159a854779e6..7c369e21c91c 100644
--- a/drivers/clk/tegra/clk-pll.c
+++ b/drivers/clk/tegra/clk-pll.c
@@ -363,7 +363,7 @@ static void _clk_pll_enable(struct clk_hw *hw)
 		val = pll_readl(pll->params->iddq_reg, pll);
 		val &= ~BIT(pll->params->iddq_bit_idx);
 		pll_writel(val, pll->params->iddq_reg, pll);
-		udelay(2);
+		udelay(5);
 	}
 
 	if (pll->params->reset_reg) {
@@ -418,6 +418,26 @@ static void _clk_pll_disable(struct clk_hw *hw)
 	}
 }
 
+static void pll_clk_start_ss(struct tegra_clk_pll *pll)
+{
+	if (pll->params->defaults_set && pll->params->ssc_ctrl_reg) {
+		u32 val = pll_readl(pll->params->ssc_ctrl_reg, pll);
+
+		val |= pll->params->ssc_ctrl_en_mask;
+		pll_writel(val, pll->params->ssc_ctrl_reg, pll);
+	}
+}
+
+static void pll_clk_stop_ss(struct tegra_clk_pll *pll)
+{
+	if (pll->params->defaults_set && pll->params->ssc_ctrl_reg) {
+		u32 val = pll_readl(pll->params->ssc_ctrl_reg, pll);
+
+		val &= ~pll->params->ssc_ctrl_en_mask;
+		pll_writel(val, pll->params->ssc_ctrl_reg, pll);
+	}
+}
+
 static int clk_pll_enable(struct clk_hw *hw)
 {
 	struct tegra_clk_pll *pll = to_clk_pll(hw);
@@ -431,6 +451,8 @@ static int clk_pll_enable(struct clk_hw *hw)
 
 	ret = clk_pll_wait_for_lock(pll);
 
+	pll_clk_start_ss(pll);
+
 	if (pll->lock)
 		spin_unlock_irqrestore(pll->lock, flags);
 
@@ -445,6 +467,8 @@ static void clk_pll_disable(struct clk_hw *hw)
 	if (pll->lock)
 		spin_lock_irqsave(pll->lock, flags);
 
+	pll_clk_stop_ss(pll);
+
 	_clk_pll_disable(hw);
 
 	if (pll->lock)
@@ -666,6 +690,8 @@ static void _get_pll_mnp(struct tegra_clk_pll *pll,
 	struct tegra_clk_pll_params *params = pll->params;
 	struct div_nmp *div_nmp = params->div_nmp;
 
+	*cfg = (struct tegra_clk_pll_freq_table) { };
+
 	if ((params->flags & (TEGRA_PLLM | TEGRA_PLLMB)) &&
 		(pll_override_readl(PMC_PLLP_WB0_OVERRIDE, pll) &
 			PMC_PLLP_WB0_OVERRIDE_PLLM_OVERRIDE)) {
@@ -716,26 +742,6 @@ static void _update_pll_cpcon(struct tegra_clk_pll *pll,
 	pll_writel_misc(val, pll);
 }
 
-static void pll_clk_start_ss(struct tegra_clk_pll *pll)
-{
-	if (pll->params->defaults_set && pll->params->ssc_ctrl_reg) {
-		u32 val = pll_readl(pll->params->ssc_ctrl_reg, pll);
-
-		val |= pll->params->ssc_ctrl_en_mask;
-		pll_writel(val, pll->params->ssc_ctrl_reg, pll);
-	}
-}
-
-static void pll_clk_stop_ss(struct tegra_clk_pll *pll)
-{
-	if (pll->params->defaults_set && pll->params->ssc_ctrl_reg) {
-		u32 val = pll_readl(pll->params->ssc_ctrl_reg, pll);
-
-		val &= ~pll->params->ssc_ctrl_en_mask;
-		pll_writel(val, pll->params->ssc_ctrl_reg, pll);
-	}
-}
-
 static int _program_pll(struct clk_hw *hw, struct tegra_clk_pll_freq_table *cfg,
 			unsigned long rate)
 {
@@ -2251,7 +2257,7 @@ tegra_clk_register_pllu_tegra114(const char *name, const char *parent_name,
 }
 #endif
 
-#if defined(CONFIG_ARCH_TEGRA_124_SOC) || defined(CONFIG_ARCH_TEGRA_132_SOC)
+#if defined(CONFIG_ARCH_TEGRA_124_SOC) || defined(CONFIG_ARCH_TEGRA_132_SOC) || defined(CONFIG_ARCH_TEGRA_210_SOC)
 static const struct clk_ops tegra_clk_pllss_ops = {
 	.is_enabled = clk_pll_is_enabled,
 	.enable = clk_pll_enable,
@@ -2349,7 +2355,6 @@ struct clk *tegra_clk_register_pllre_tegra210(const char *name,
 			  struct tegra_clk_pll_params *pll_params,
 			  spinlock_t *lock, unsigned long parent_rate)
 {
-	u32 val;
 	struct tegra_clk_pll *pll;
 	struct clk *clk;
 
@@ -2363,26 +2368,8 @@ struct clk *tegra_clk_register_pllre_tegra210(const char *name,
 	if (IS_ERR(pll))
 		return ERR_CAST(pll);
 
-	/* program minimum rate by default */
-
-	val = pll_readl_base(pll);
-	if (val & PLL_BASE_ENABLE)
-		WARN_ON(readl_relaxed(clk_base + pll_params->iddq_reg) &
-				BIT(pll_params->iddq_bit_idx));
-	else {
-		val = 0x4 << divm_shift(pll);
-		val |= 0x41 << divn_shift(pll);
-		pll_writel_base(val, pll);
-	}
-
-	/* disable lock override */
-
-	val = pll_readl_misc(pll);
-	val &= ~BIT(29);
-	pll_writel_misc(val, pll);
-
 	clk = _tegra_clk_register_pll(pll, name, parent_name, flags,
-				      &tegra_clk_pllre_ops);
+				      &tegra_clk_pll_ops);
 	if (IS_ERR(clk))
 		kfree(pll);
 
@@ -2604,46 +2591,6 @@ struct clk *tegra_clk_register_pllc_tegra210(const char *name,
 	return clk;
 }
 
-struct clk *tegra_clk_register_pllxc_tegra210(const char *name,
-			const char *parent_name, void __iomem *clk_base,
-			void __iomem *pmc, unsigned long flags,
-			struct tegra_clk_pll_params *pll_params,
-			spinlock_t *lock)
-{
-	struct tegra_clk_pll *pll;
-	struct clk *clk, *parent;
-	unsigned long parent_rate;
-
-	parent = __clk_lookup(parent_name);
-	if (!parent) {
-		WARN(1, "parent clk %s of %s must be registered first\n",
-			name, parent_name);
-		return ERR_PTR(-EINVAL);
-	}
-
-	if (!pll_params->pdiv_tohw)
-		return ERR_PTR(-EINVAL);
-
-	parent_rate = clk_get_rate(parent);
-
-	pll_params->vco_min = _clip_vco_min(pll_params->vco_min, parent_rate);
-
-	if (pll_params->adjust_vco)
-		pll_params->vco_min = pll_params->adjust_vco(pll_params,
-							     parent_rate);
-
-	pll = _tegra_init_pll(clk_base, pmc, pll_params, lock);
-	if (IS_ERR(pll))
-		return ERR_CAST(pll);
-
-	clk = _tegra_clk_register_pll(pll, name, parent_name, flags,
-				      &tegra_clk_pll_ops);
-	if (IS_ERR(clk))
-		kfree(pll);
-
-	return clk;
-}
-
 struct clk *tegra_clk_register_pllss_tegra210(const char *name,
 				const char *parent_name, void __iomem *clk_base,
 				unsigned long flags,
@@ -2652,10 +2599,8 @@ struct clk *tegra_clk_register_pllss_tegra210(const char *name,
 {
 	struct tegra_clk_pll *pll;
 	struct clk *clk, *parent;
-	struct tegra_clk_pll_freq_table cfg;
 	unsigned long parent_rate;
 	u32 val;
-	int i;
 
 	if (!pll_params->div_nmp)
 		return ERR_PTR(-EINVAL);
@@ -2667,13 +2612,11 @@ struct clk *tegra_clk_register_pllss_tegra210(const char *name,
 		return ERR_PTR(-EINVAL);
 	}
 
-	pll = _tegra_init_pll(clk_base, NULL, pll_params, lock);
-	if (IS_ERR(pll))
-		return ERR_CAST(pll);
-
-	val = pll_readl_base(pll);
-	val &= ~PLLSS_REF_SRC_SEL_MASK;
-	pll_writel_base(val, pll);
+	val = readl_relaxed(clk_base + pll_params->base_reg);
+	if (val & PLLSS_REF_SRC_SEL_MASK) {
+		WARN(1, "not supported reference clock for %s\n", name);
+		return ERR_PTR(-EINVAL);
+	}
 
 	parent_rate = clk_get_rate(parent);
 
@@ -2683,36 +2626,10 @@ struct clk *tegra_clk_register_pllss_tegra210(const char *name,
 		pll_params->vco_min = pll_params->adjust_vco(pll_params,
 							     parent_rate);
 
-	/* initialize PLL to minimum rate */
-
-	cfg.m = _pll_fixed_mdiv(pll_params, parent_rate);
-	cfg.n = cfg.m * pll_params->vco_min / parent_rate;
-
-	for (i = 0; pll_params->pdiv_tohw[i].pdiv; i++)
-		;
-	if (!i) {
-		kfree(pll);
-		return ERR_PTR(-EINVAL);
-	}
-
-	cfg.p = pll_params->pdiv_tohw[i-1].hw_val;
-
-	_update_pll_mnp(pll, &cfg);
-
-	pll_writel_misc(PLLSS_MISC_DEFAULT, pll);
-
-	val = pll_readl_base(pll);
-	if (val & PLL_BASE_ENABLE) {
-		if (val & BIT(pll_params->iddq_bit_idx)) {
-			WARN(1, "%s is on but IDDQ set\n", name);
-			kfree(pll);
-			return ERR_PTR(-EINVAL);
-		}
-	} else
-		val |= BIT(pll_params->iddq_bit_idx);
-
-	val &= ~PLLSS_LOCK_OVERRIDE;
-	pll_writel_base(val, pll);
+	pll_params->flags |= TEGRA_PLL_BYPASS;
+	pll = _tegra_init_pll(clk_base, NULL, pll_params, lock);
+	if (IS_ERR(pll))
+		return ERR_CAST(pll);
 
 	clk = _tegra_clk_register_pll(pll, name, parent_name, flags,
 					&tegra_clk_pll_ops);
diff --git a/drivers/clk/tegra/clk-tegra-periph.c b/drivers/clk/tegra/clk-tegra-periph.c
index 294bfe40a4f5..848255cc0209 100644
--- a/drivers/clk/tegra/clk-tegra-periph.c
+++ b/drivers/clk/tegra/clk-tegra-periph.c
@@ -216,7 +216,8 @@
 			     _clk_num, _clk_id)			\
 	TEGRA_INIT_DATA_TABLE(_name, NULL, NULL, _parents, _offset,\
 			30, MASK(2), 0, 0, 16, 0, TEGRA_DIVIDER_ROUND_UP,\
-			_clk_num, 0, _clk_id, _parents##_idx, 0, NULL)
+			_clk_num, TEGRA_PERIPH_ON_APB, _clk_id, \
+			_parents##_idx, 0, NULL)
 
 #define XUSB(_name, _parents, _offset, \
 			     _clk_num, _gate_flags, _clk_id)	 \
diff --git a/drivers/clk/tegra/clk-tegra-super-gen4.c b/drivers/clk/tegra/clk-tegra-super-gen4.c
index 474de0f0c26d..4f6fd307cb70 100644
--- a/drivers/clk/tegra/clk-tegra-super-gen4.c
+++ b/drivers/clk/tegra/clk-tegra-super-gen4.c
@@ -232,8 +232,15 @@ static void __init tegra_super_clk_init(void __iomem *clk_base,
 	if (!dt_clk)
 		return;
 
-	clk = tegra_clk_register_pllxc("pll_x", "pll_ref", clk_base,
-			pmc_base, CLK_IGNORE_UNUSED, params, NULL);
+#if defined(CONFIG_ARCH_TEGRA_210_SOC)
+	if (gen_info->gen == gen5)
+		clk = tegra_clk_register_pllc_tegra210("pll_x", "pll_ref",
+			clk_base, pmc_base, CLK_IGNORE_UNUSED, params, NULL);
+	else
+#endif
+		clk = tegra_clk_register_pllxc("pll_x", "pll_ref", clk_base,
+				pmc_base, CLK_IGNORE_UNUSED, params, NULL);
+
 	*dt_clk = clk;
 
 	/* PLLX_OUT0 */
diff --git a/drivers/clk/tegra/clk-tegra210.c b/drivers/clk/tegra/clk-tegra210.c
index 1024e853ea65..6d7a613f2656 100644
--- a/drivers/clk/tegra/clk-tegra210.c
+++ b/drivers/clk/tegra/clk-tegra210.c
@@ -146,7 +146,7 @@
 #define PLLD_SDM_EN_MASK BIT(16)
 
 #define PLLD2_SDM_EN_MASK BIT(31)
-#define PLLD2_SSC_EN_MASK BIT(30)
+#define PLLD2_SSC_EN_MASK 0
 
 #define PLLDP_SS_CFG	0x598
 #define PLLDP_SDM_EN_MASK BIT(31)
@@ -241,6 +241,9 @@
 #define PLL_SDM_COEFF BIT(13)
 #define sdin_din_to_data(din)	((u16)((din) ? : 0xFFFFU))
 #define sdin_data_to_din(dat)	(((dat) == 0xFFFFU) ? 0 : (s16)dat)
+/* This macro returns ndiv effective scaled to SDM range */
+#define sdin_get_n_eff(cfg)	((cfg)->n * PLL_SDM_COEFF + ((cfg)->sdm_data ? \
+		(PLL_SDM_COEFF/2 + sdin_data_to_din((cfg)->sdm_data)) : 0))
 
 /* Tegra CPU clock and reset control regs */
 #define CLK_RST_CONTROLLER_CPU_CMPLX_STATUS	0x470
@@ -715,8 +718,6 @@ static void plldss_defaults(const char *pll_name, struct tegra_clk_pll *plldss,
 	plldss->params->defaults_set = true;
 
 	if (val & PLL_ENABLE) {
-		pr_warn("%s already enabled. Postponing set full defaults\n",
-			 pll_name);
 
 		/*
 		 * PLL is ON: check if defaults already set, then set those
@@ -755,6 +756,10 @@ static void plldss_defaults(const char *pll_name, struct tegra_clk_pll *plldss,
 				(~PLLDSS_MISC1_CFG_EN_SDM));
 		}
 
+		if (!plldss->params->defaults_set)
+			pr_warn("%s already enabled. Postponing set full defaults\n",
+				 pll_name);
+
 		/* Enable lock detect */
 		if (val & PLLDSS_BASE_LOCK_OVERRIDE) {
 			val &= ~PLLDSS_BASE_LOCK_OVERRIDE;
@@ -1288,8 +1293,7 @@ static int tegra210_pll_fixed_mdiv_cfg(struct clk_hw *hw,
 			s -= PLL_SDM_COEFF / 2;
 			cfg->sdm_data = sdin_din_to_data(s);
 		}
-		cfg->output_rate *= cfg->n * PLL_SDM_COEFF + PLL_SDM_COEFF/2 +
-					sdin_data_to_din(cfg->sdm_data);
+		cfg->output_rate *= sdin_get_n_eff(cfg);
 		cfg->output_rate /= p * cfg->m * PLL_SDM_COEFF;
 	} else {
 		cfg->output_rate *= cfg->n;
@@ -1314,8 +1318,7 @@ static int tegra210_pll_fixed_mdiv_cfg(struct clk_hw *hw,
  */
 static void tegra210_clk_pll_set_gain(struct tegra_clk_pll_freq_table *cfg)
 {
-	cfg->n = cfg->n * PLL_SDM_COEFF + PLL_SDM_COEFF/2 +
-			sdin_data_to_din(cfg->sdm_data);
+	cfg->n = sdin_get_n_eff(cfg);
 	cfg->m *= PLL_SDM_COEFF;
 }
 
@@ -2204,7 +2207,6 @@ static struct tegra_clk tegra210_clks[tegra_clk_max] __initdata = {
 	[tegra_clk_gpu] = { .dt_id = TEGRA210_CLK_GPU, .present = true },
 	[tegra_clk_pll_g_ref] = { .dt_id = TEGRA210_CLK_PLL_G_REF, .present = true, },
 	[tegra_clk_uartb_8] = { .dt_id = TEGRA210_CLK_UARTB, .present = true },
-	[tegra_clk_vfir] = { .dt_id = TEGRA210_CLK_VFIR, .present = true },
 	[tegra_clk_spdif_in_8] = { .dt_id = TEGRA210_CLK_SPDIF_IN, .present = true },
 	[tegra_clk_spdif_out] = { .dt_id = TEGRA210_CLK_SPDIF_OUT, .present = true },
 	[tegra_clk_vi_10] = { .dt_id = TEGRA210_CLK_VI, .present = true },
@@ -2470,15 +2472,14 @@ static void tegra210_utmi_param_configure(void)
 	reg |= UTMIP_PLL_CFG2_STABLE_COUNT(utmi_parameters[i].stable_count);
 
 	reg &= ~UTMIP_PLL_CFG2_ACTIVE_DLY_COUNT(~0);
-
 	reg |=
 	UTMIP_PLL_CFG2_ACTIVE_DLY_COUNT(utmi_parameters[i].active_delay_count);
 	writel_relaxed(reg, clk_base + UTMIP_PLL_CFG2);
 
 	/* Program UTMIP PLL delay and oscillator frequency counts */
 	reg = readl_relaxed(clk_base + UTMIP_PLL_CFG1);
-	reg &= ~UTMIP_PLL_CFG1_ENABLE_DLY_COUNT(~0);
 
+	reg &= ~UTMIP_PLL_CFG1_ENABLE_DLY_COUNT(~0);
 	reg |=
 	UTMIP_PLL_CFG1_ENABLE_DLY_COUNT(utmi_parameters[i].enable_delay_count);
 
@@ -2494,7 +2495,8 @@ static void tegra210_utmi_param_configure(void)
 	reg &= ~UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERDOWN;
 	reg |= UTMIP_PLL_CFG1_FORCE_PLL_ENABLE_POWERUP;
 	writel_relaxed(reg, clk_base + UTMIP_PLL_CFG1);
-	udelay(1);
+
+	udelay(20);
 
 	/* Enable samplers for SNPS, XUSB_HOST, XUSB_DEV */
 	reg = readl_relaxed(clk_base + UTMIP_PLL_CFG2);
@@ -2552,6 +2554,7 @@ static int tegra210_enable_pllu(void)
 	reg = readl_relaxed(clk_base + pllu.params->ext_misc_reg[0]);
 	reg &= ~BIT(pllu.params->iddq_bit_idx);
 	writel_relaxed(reg, clk_base + pllu.params->ext_misc_reg[0]);
+	udelay(5);
 
 	reg = readl_relaxed(clk_base + PLLU_BASE);
 	reg &= ~GENMASK(20, 0);
@@ -2559,6 +2562,7 @@ static int tegra210_enable_pllu(void)
 	reg |= fentry->n << 8;
 	reg |= fentry->p << 16;
 	writel(reg, clk_base + PLLU_BASE);
+	udelay(1);
 	reg |= PLL_ENABLE;
 	writel(reg, clk_base + PLLU_BASE);
 
@@ -2699,7 +2703,7 @@ static void __init tegra210_pll_init(void __iomem *clk_base,
 	struct clk *clk;
 
 	/* PLLC */
-	clk = tegra_clk_register_pllxc_tegra210("pll_c", "pll_ref", clk_base,
+	clk = tegra_clk_register_pllc_tegra210("pll_c", "pll_ref", clk_base,
 			pmc, 0, &pll_c_params, NULL);
 	if (!WARN_ON(IS_ERR(clk)))
 		clk_register_clkdev(clk, "pll_c", NULL);
@@ -2798,14 +2802,14 @@ static void __init tegra210_pll_init(void __iomem *clk_base,
 	/* PLLU_60M */
 	clk = clk_register_gate(NULL, "pll_u_60M", "pll_u_out2",
 				CLK_SET_RATE_PARENT, clk_base + PLLU_BASE,
-				23, 0, NULL);
+				23, 0, &pll_u_lock);
 	clk_register_clkdev(clk, "pll_u_60M", NULL);
 	clks[TEGRA210_CLK_PLL_U_60M] = clk;
 
 	/* PLLU_48M */
 	clk = clk_register_gate(NULL, "pll_u_48M", "pll_u_out1",
 				CLK_SET_RATE_PARENT, clk_base + PLLU_BASE,
-				25, 0, NULL);
+				25, 0, &pll_u_lock);
 	clk_register_clkdev(clk, "pll_u_48M", NULL);
 	clks[TEGRA210_CLK_PLL_U_48M] = clk;
 
diff --git a/drivers/clk/tegra/clk.h b/drivers/clk/tegra/clk.h
index 945b07093afa..872f1189ad7f 100644
--- a/drivers/clk/tegra/clk.h
+++ b/drivers/clk/tegra/clk.h
@@ -362,12 +362,6 @@ struct clk *tegra_clk_register_pllxc(const char *name, const char *parent_name,
 			    struct tegra_clk_pll_params *pll_params,
 			    spinlock_t *lock);
 
-struct clk *tegra_clk_register_pllxc_tegra210(const char *name,
-			const char *parent_name, void __iomem *clk_base,
-			void __iomem *pmc, unsigned long flags,
-			struct tegra_clk_pll_params *pll_params,
-			spinlock_t *lock);
-
 struct clk *tegra_clk_register_pllm(const char *name, const char *parent_name,
 			   void __iomem *clk_base, void __iomem *pmc,
 			   unsigned long flags,
diff --git a/drivers/clk/ti/adpll.c b/drivers/clk/ti/adpll.c
index 255cafb18336..d6036c788fab 100644
--- a/drivers/clk/ti/adpll.c
+++ b/drivers/clk/ti/adpll.c
@@ -222,7 +222,7 @@ static int ti_adpll_setup_clock(struct ti_adpll_data *d, struct clk *clock,
 
 	/* Separate con_id in format "pll040dcoclkldo" to fit MAX_CON_ID */
 	postfix = strrchr(name, '.');
-	if (strlen(postfix) > 1) {
+	if (postfix && strlen(postfix) > 1) {
 		if (strlen(postfix) > ADPLL_MAX_CON_ID)
 			dev_warn(d->dev, "clock %s con_id lookup may fail\n",
 				 name);
@@ -486,7 +486,7 @@ static u8 ti_adpll_get_parent(struct clk_hw *hw)
 	return 0;
 }
 
-static struct clk_ops ti_adpll_ops = {
+static const struct clk_ops ti_adpll_ops = {
 	.prepare = ti_adpll_prepare,
 	.unprepare = ti_adpll_unprepare,
 	.is_prepared = ti_adpll_is_prepared,
diff --git a/drivers/clk/ti/apll.c b/drivers/clk/ti/apll.c
index 06f486b3488c..83b148f8037c 100644
--- a/drivers/clk/ti/apll.c
+++ b/drivers/clk/ti/apll.c
@@ -304,7 +304,7 @@ static void omap2_apll_disable(struct clk_hw *hw)
 	ti_clk_ll_ops->clk_writel(v, &ad->control_reg);
 }
 
-static struct clk_ops omap2_apll_ops = {
+static const struct clk_ops omap2_apll_ops = {
 	.enable		= &omap2_apll_enable,
 	.disable	= &omap2_apll_disable,
 	.is_enabled	= &omap2_apll_is_enabled,
diff --git a/drivers/clk/ti/clockdomain.c b/drivers/clk/ti/clockdomain.c
index fbedc6a9fed0..07a805125e98 100644
--- a/drivers/clk/ti/clockdomain.c
+++ b/drivers/clk/ti/clockdomain.c
@@ -138,8 +138,8 @@ static void __init of_ti_clockdomain_setup(struct device_node *node)
 	for (i = 0; i < num_clks; i++) {
 		clk = of_clk_get(node, i);
 		if (IS_ERR(clk)) {
-			pr_err("%s: Failed get %s' clock nr %d (%ld)\n",
-			       __func__, node->full_name, i, PTR_ERR(clk));
+			pr_err("%s: Failed get %pOF' clock nr %d (%ld)\n",
+			       __func__, node, i, PTR_ERR(clk));
 			continue;
 		}
 		clk_hw = __clk_get_hw(clk);
diff --git a/drivers/clk/ti/fapll.c b/drivers/clk/ti/fapll.c
index 66a0d0ed8b55..071af44b1ba8 100644
--- a/drivers/clk/ti/fapll.c
+++ b/drivers/clk/ti/fapll.c
@@ -268,7 +268,7 @@ static int ti_fapll_set_rate(struct clk_hw *hw, unsigned long rate,
 	return 0;
 }
 
-static struct clk_ops ti_fapll_ops = {
+static const struct clk_ops ti_fapll_ops = {
 	.enable = ti_fapll_enable,
 	.disable = ti_fapll_disable,
 	.is_enabled = ti_fapll_is_enabled,
@@ -478,7 +478,7 @@ static int ti_fapll_synth_set_rate(struct clk_hw *hw, unsigned long rate,
 	return 0;
 }
 
-static struct clk_ops ti_fapll_synt_ops = {
+static const struct clk_ops ti_fapll_synt_ops = {
 	.enable = ti_fapll_synth_enable,
 	.disable = ti_fapll_synth_disable,
 	.is_enabled = ti_fapll_synth_is_enabled,
diff --git a/drivers/clk/uniphier/clk-uniphier-core.c b/drivers/clk/uniphier/clk-uniphier-core.c
index 2cf386347f0c..e09f3dd46318 100644
--- a/drivers/clk/uniphier/clk-uniphier-core.c
+++ b/drivers/clk/uniphier/clk-uniphier-core.c
@@ -111,10 +111,6 @@ static int uniphier_clk_remove(struct platform_device *pdev)
 static const struct of_device_id uniphier_clk_match[] = {
 	/* System clock */
 	{
-		.compatible = "socionext,uniphier-sld3-clock",
-		.data = uniphier_sld3_sys_clk_data,
-	},
-	{
 		.compatible = "socionext,uniphier-ld4-clock",
 		.data = uniphier_ld4_sys_clk_data,
 	},
@@ -142,22 +138,22 @@ static const struct of_device_id uniphier_clk_match[] = {
 		.compatible = "socionext,uniphier-ld20-clock",
 		.data = uniphier_ld20_sys_clk_data,
 	},
-	/* Media I/O clock, SD clock */
 	{
-		.compatible = "socionext,uniphier-sld3-mio-clock",
-		.data = uniphier_sld3_mio_clk_data,
+		.compatible = "socionext,uniphier-pxs3-clock",
+		.data = uniphier_pxs3_sys_clk_data,
 	},
+	/* Media I/O clock, SD clock */
 	{
 		.compatible = "socionext,uniphier-ld4-mio-clock",
-		.data = uniphier_sld3_mio_clk_data,
+		.data = uniphier_ld4_mio_clk_data,
 	},
 	{
 		.compatible = "socionext,uniphier-pro4-mio-clock",
-		.data = uniphier_sld3_mio_clk_data,
+		.data = uniphier_ld4_mio_clk_data,
 	},
 	{
 		.compatible = "socionext,uniphier-sld8-mio-clock",
-		.data = uniphier_sld3_mio_clk_data,
+		.data = uniphier_ld4_mio_clk_data,
 	},
 	{
 		.compatible = "socionext,uniphier-pro5-sd-clock",
@@ -169,12 +165,16 @@ static const struct of_device_id uniphier_clk_match[] = {
 	},
 	{
 		.compatible = "socionext,uniphier-ld11-mio-clock",
-		.data = uniphier_sld3_mio_clk_data,
+		.data = uniphier_ld4_mio_clk_data,
 	},
 	{
 		.compatible = "socionext,uniphier-ld20-sd-clock",
 		.data = uniphier_pro5_sd_clk_data,
 	},
+	{
+		.compatible = "socionext,uniphier-pxs3-sd-clock",
+		.data = uniphier_pro5_sd_clk_data,
+	},
 	/* Peripheral clock */
 	{
 		.compatible = "socionext,uniphier-ld4-peri-clock",
@@ -204,6 +204,10 @@ static const struct of_device_id uniphier_clk_match[] = {
 		.compatible = "socionext,uniphier-ld20-peri-clock",
 		.data = uniphier_pro4_peri_clk_data,
 	},
+	{
+		.compatible = "socionext,uniphier-pxs3-peri-clock",
+		.data = uniphier_pro4_peri_clk_data,
+	},
 	{ /* sentinel */ }
 };
 
diff --git a/drivers/clk/uniphier/clk-uniphier-mio.c b/drivers/clk/uniphier/clk-uniphier-mio.c
index 218d20f099ce..16e4d303f535 100644
--- a/drivers/clk/uniphier/clk-uniphier-mio.c
+++ b/drivers/clk/uniphier/clk-uniphier-mio.c
@@ -76,7 +76,7 @@
 #define UNIPHIER_MIO_CLK_DMAC(idx)					\
 	UNIPHIER_CLK_GATE("miodmac", (idx), "stdmac", 0x20, 25)
 
-const struct uniphier_clk_data uniphier_sld3_mio_clk_data[] = {
+const struct uniphier_clk_data uniphier_ld4_mio_clk_data[] = {
 	UNIPHIER_MIO_CLK_SD_FIXED,
 	UNIPHIER_MIO_CLK_SD(0, 0),
 	UNIPHIER_MIO_CLK_SD(1, 1),
@@ -85,11 +85,9 @@ const struct uniphier_clk_data uniphier_sld3_mio_clk_data[] = {
 	UNIPHIER_MIO_CLK_USB2(8, 0),
 	UNIPHIER_MIO_CLK_USB2(9, 1),
 	UNIPHIER_MIO_CLK_USB2(10, 2),
-	UNIPHIER_MIO_CLK_USB2(11, 3),
 	UNIPHIER_MIO_CLK_USB2_PHY(12, 0),
 	UNIPHIER_MIO_CLK_USB2_PHY(13, 1),
 	UNIPHIER_MIO_CLK_USB2_PHY(14, 2),
-	UNIPHIER_MIO_CLK_USB2_PHY(15, 3),
 	{ /* sentinel */ }
 };
 
diff --git a/drivers/clk/uniphier/clk-uniphier-sys.c b/drivers/clk/uniphier/clk-uniphier-sys.c
index ad0218182a9f..0e396f3da526 100644
--- a/drivers/clk/uniphier/clk-uniphier-sys.c
+++ b/drivers/clk/uniphier/clk-uniphier-sys.c
@@ -17,7 +17,7 @@
 
 #include "clk-uniphier.h"
 
-#define UNIPHIER_SLD3_SYS_CLK_SD					\
+#define UNIPHIER_LD4_SYS_CLK_SD					\
 	UNIPHIER_CLK_FACTOR("sd-200m", -1, "spll", 1, 8),		\
 	UNIPHIER_CLK_FACTOR("sd-133m", -1, "vpll27a", 1, 2)
 
@@ -30,7 +30,7 @@
 	UNIPHIER_CLK_FACTOR("sd-133m", -1, "spll", 1, 15)
 
 /* Denali driver requires clk_x rate (clk: 50MHz, clk_x & ecc_clk: 200MHz) */
-#define UNIPHIER_SLD3_SYS_CLK_NAND(idx)					\
+#define UNIPHIER_LD4_SYS_CLK_NAND(idx)					\
 	UNIPHIER_CLK_FACTOR("nand-200m", -1, "spll", 1, 8),		\
 	UNIPHIER_CLK_GATE("nand", (idx), "nand-200m", 0x2104, 2)
 
@@ -45,7 +45,7 @@
 #define UNIPHIER_LD11_SYS_CLK_EMMC(idx)					\
 	UNIPHIER_CLK_GATE("emmc", (idx), NULL, 0x210c, 2)
 
-#define UNIPHIER_SLD3_SYS_CLK_STDMAC(idx)				\
+#define UNIPHIER_LD4_SYS_CLK_STDMAC(idx)				\
 	UNIPHIER_CLK_GATE("stdmac", (idx), NULL, 0x2104, 10)
 
 #define UNIPHIER_LD11_SYS_CLK_STDMAC(idx)				\
@@ -57,19 +57,23 @@
 #define UNIPHIER_PRO4_SYS_CLK_USB3(idx, ch)				\
 	UNIPHIER_CLK_GATE("usb3" #ch, (idx), NULL, 0x2104, 16 + (ch))
 
-const struct uniphier_clk_data uniphier_sld3_sys_clk_data[] = {
-	UNIPHIER_CLK_FACTOR("spll", -1, "ref", 65, 1),		/* 1597.44 MHz */
-	UNIPHIER_CLK_FACTOR("upll", -1, "ref", 6000, 512),	/* 288 MHz */
-	UNIPHIER_CLK_FACTOR("a2pll", -1, "ref", 24, 1),		/* 589.824 MHz */
-	UNIPHIER_CLK_FACTOR("vpll27a", -1, "ref", 5625, 512),	/* 270 MHz */
-	UNIPHIER_CLK_FACTOR("uart", 0, "a2pll", 1, 16),
-	UNIPHIER_CLK_FACTOR("i2c", 1, "spll", 1, 16),
-	UNIPHIER_SLD3_SYS_CLK_NAND(2),
-	UNIPHIER_SLD3_SYS_CLK_SD,
-	UNIPHIER_CLK_FACTOR("usb2", -1, "upll", 1, 12),
-	UNIPHIER_SLD3_SYS_CLK_STDMAC(8),
-	{ /* sentinel */ }
-};
+#define UNIPHIER_LD11_SYS_CLK_AIO(idx)					\
+	UNIPHIER_CLK_FACTOR("aio-io200m", -1, "spll", 1, 10),		\
+	UNIPHIER_CLK_GATE("aio", (idx), "aio-io200m", 0x2108, 0)
+
+#define UNIPHIER_LD11_SYS_CLK_EVEA(idx)					\
+	UNIPHIER_CLK_FACTOR("evea-io100m", -1, "spll", 1, 20),		\
+	UNIPHIER_CLK_GATE("evea", (idx), "evea-io100m", 0x2108, 1)
+
+#define UNIPHIER_LD11_SYS_CLK_EXIV(idx)					\
+	UNIPHIER_CLK_FACTOR("exiv-io200m", -1, "spll", 1, 10),		\
+	UNIPHIER_CLK_GATE("exiv", (idx), "exiv-io200m", 0x2110, 2)
+
+#define UNIPHIER_PRO4_SYS_CLK_ETHER(idx)				\
+	UNIPHIER_CLK_GATE("ether", (idx), NULL, 0x2104, 12)
+
+#define UNIPHIER_LD11_SYS_CLK_ETHER(idx)				\
+	UNIPHIER_CLK_GATE("ether", (idx), NULL, 0x210c, 6)
 
 const struct uniphier_clk_data uniphier_ld4_sys_clk_data[] = {
 	UNIPHIER_CLK_FACTOR("spll", -1, "ref", 65, 1),		/* 1597.44 MHz */
@@ -78,10 +82,10 @@ const struct uniphier_clk_data uniphier_ld4_sys_clk_data[] = {
 	UNIPHIER_CLK_FACTOR("vpll27a", -1, "ref", 5625, 512),	/* 270 MHz */
 	UNIPHIER_CLK_FACTOR("uart", 0, "a2pll", 1, 16),
 	UNIPHIER_CLK_FACTOR("i2c", 1, "spll", 1, 16),
-	UNIPHIER_SLD3_SYS_CLK_NAND(2),
-	UNIPHIER_SLD3_SYS_CLK_SD,
+	UNIPHIER_LD4_SYS_CLK_NAND(2),
+	UNIPHIER_LD4_SYS_CLK_SD,
 	UNIPHIER_CLK_FACTOR("usb2", -1, "upll", 1, 12),
-	UNIPHIER_SLD3_SYS_CLK_STDMAC(8),		/* Ether, HSC, MIO */
+	UNIPHIER_LD4_SYS_CLK_STDMAC(8),			/* Ether, HSC, MIO */
 	{ /* sentinel */ }
 };
 
@@ -92,10 +96,11 @@ const struct uniphier_clk_data uniphier_pro4_sys_clk_data[] = {
 	UNIPHIER_CLK_FACTOR("vpll27a", -1, "ref", 270, 25),	/* 270 MHz */
 	UNIPHIER_CLK_FACTOR("uart", 0, "a2pll", 1, 8),
 	UNIPHIER_CLK_FACTOR("i2c", 1, "spll", 1, 32),
-	UNIPHIER_SLD3_SYS_CLK_NAND(2),
-	UNIPHIER_SLD3_SYS_CLK_SD,
+	UNIPHIER_LD4_SYS_CLK_NAND(2),
+	UNIPHIER_LD4_SYS_CLK_SD,
 	UNIPHIER_CLK_FACTOR("usb2", -1, "upll", 1, 12),
-	UNIPHIER_SLD3_SYS_CLK_STDMAC(8),		/* HSC, MIO, RLE */
+	UNIPHIER_PRO4_SYS_CLK_ETHER(6),
+	UNIPHIER_LD4_SYS_CLK_STDMAC(8),			/* HSC, MIO, RLE */
 	UNIPHIER_PRO4_SYS_CLK_GIO(12),			/* Ether, SATA, USB3 */
 	UNIPHIER_PRO4_SYS_CLK_USB3(14, 0),
 	UNIPHIER_PRO4_SYS_CLK_USB3(15, 1),
@@ -108,10 +113,10 @@ const struct uniphier_clk_data uniphier_sld8_sys_clk_data[] = {
 	UNIPHIER_CLK_FACTOR("vpll27a", -1, "ref", 270, 25),	/* 270 MHz */
 	UNIPHIER_CLK_FACTOR("uart", 0, "spll", 1, 20),
 	UNIPHIER_CLK_FACTOR("i2c", 1, "spll", 1, 16),
-	UNIPHIER_SLD3_SYS_CLK_NAND(2),
-	UNIPHIER_SLD3_SYS_CLK_SD,
+	UNIPHIER_LD4_SYS_CLK_NAND(2),
+	UNIPHIER_LD4_SYS_CLK_SD,
 	UNIPHIER_CLK_FACTOR("usb2", -1, "upll", 1, 12),
-	UNIPHIER_SLD3_SYS_CLK_STDMAC(8),		/* Ether, HSC, MIO */
+	UNIPHIER_LD4_SYS_CLK_STDMAC(8),			/* Ether, HSC, MIO */
 	{ /* sentinel */ }
 };
 
@@ -123,7 +128,7 @@ const struct uniphier_clk_data uniphier_pro5_sys_clk_data[] = {
 	UNIPHIER_CLK_FACTOR("i2c", 1, "spll", 1, 48),
 	UNIPHIER_PRO5_SYS_CLK_NAND(2),
 	UNIPHIER_PRO5_SYS_CLK_SD,
-	UNIPHIER_SLD3_SYS_CLK_STDMAC(8),			/* HSC */
+	UNIPHIER_LD4_SYS_CLK_STDMAC(8),				/* HSC */
 	UNIPHIER_PRO4_SYS_CLK_GIO(12),				/* PCIe, USB3 */
 	UNIPHIER_PRO4_SYS_CLK_USB3(14, 0),
 	UNIPHIER_PRO4_SYS_CLK_USB3(15, 1),
@@ -136,7 +141,8 @@ const struct uniphier_clk_data uniphier_pxs2_sys_clk_data[] = {
 	UNIPHIER_CLK_FACTOR("i2c", 1, "spll", 1, 48),
 	UNIPHIER_PRO5_SYS_CLK_NAND(2),
 	UNIPHIER_PRO5_SYS_CLK_SD,
-	UNIPHIER_SLD3_SYS_CLK_STDMAC(8),			/* HSC, RLE */
+	UNIPHIER_PRO4_SYS_CLK_ETHER(6),
+	UNIPHIER_LD4_SYS_CLK_STDMAC(8),				/* HSC, RLE */
 	/* GIO is always clock-enabled: no function for 0x2104 bit6 */
 	UNIPHIER_PRO4_SYS_CLK_USB3(14, 0),
 	UNIPHIER_PRO4_SYS_CLK_USB3(15, 1),
@@ -156,8 +162,12 @@ const struct uniphier_clk_data uniphier_ld11_sys_clk_data[] = {
 	UNIPHIER_LD11_SYS_CLK_NAND(2),
 	UNIPHIER_LD11_SYS_CLK_EMMC(4),
 	/* Index 5 reserved for eMMC PHY */
+	UNIPHIER_LD11_SYS_CLK_ETHER(6),
 	UNIPHIER_LD11_SYS_CLK_STDMAC(8),			/* HSC, MIO */
 	UNIPHIER_CLK_FACTOR("usb2", -1, "ref", 24, 25),
+	UNIPHIER_LD11_SYS_CLK_AIO(40),
+	UNIPHIER_LD11_SYS_CLK_EVEA(41),
+	UNIPHIER_LD11_SYS_CLK_EXIV(42),
 	/* CPU gears */
 	UNIPHIER_CLK_DIV4("cpll", 2, 3, 4, 8),
 	UNIPHIER_CLK_DIV4("mpll", 2, 3, 4, 8),
@@ -185,6 +195,7 @@ const struct uniphier_clk_data uniphier_ld20_sys_clk_data[] = {
 	UNIPHIER_LD11_SYS_CLK_EMMC(4),
 	/* Index 5 reserved for eMMC PHY */
 	UNIPHIER_LD20_SYS_CLK_SD,
+	UNIPHIER_LD11_SYS_CLK_ETHER(6),
 	UNIPHIER_LD11_SYS_CLK_STDMAC(8),			/* HSC */
 	/* GIO is always clock-enabled: no function for 0x210c bit5 */
 	/*
@@ -194,6 +205,9 @@ const struct uniphier_clk_data uniphier_ld20_sys_clk_data[] = {
 	UNIPHIER_CLK_GATE("usb30", 14, NULL, 0x210c, 14),
 	UNIPHIER_CLK_GATE("usb30-phy0", 16, NULL, 0x210c, 12),
 	UNIPHIER_CLK_GATE("usb30-phy1", 17, NULL, 0x210c, 13),
+	UNIPHIER_LD11_SYS_CLK_AIO(40),
+	UNIPHIER_LD11_SYS_CLK_EVEA(41),
+	UNIPHIER_LD11_SYS_CLK_EXIV(42),
 	/* CPU gears */
 	UNIPHIER_CLK_DIV4("cpll", 2, 3, 4, 8),
 	UNIPHIER_CLK_DIV4("spll", 2, 3, 4, 8),
@@ -209,3 +223,33 @@ const struct uniphier_clk_data uniphier_ld20_sys_clk_data[] = {
 			     "spll/4", "spll/8", "s2pll/4", "s2pll/8"),
 	{ /* sentinel */ }
 };
+
+const struct uniphier_clk_data uniphier_pxs3_sys_clk_data[] = {
+	UNIPHIER_CLK_FACTOR("cpll", -1, "ref", 104, 1),		/* ARM: 2600 MHz */
+	UNIPHIER_CLK_FACTOR("spll", -1, "ref", 80, 1),		/* 2000 MHz */
+	UNIPHIER_CLK_FACTOR("s2pll", -1, "ref", 88, 1),		/* IPP: 2400 MHz */
+	UNIPHIER_CLK_FACTOR("uart", 0, "spll", 1, 34),
+	UNIPHIER_CLK_FACTOR("i2c", 1, "spll", 1, 40),
+	UNIPHIER_LD20_SYS_CLK_SD,
+	UNIPHIER_LD11_SYS_CLK_NAND(2),
+	UNIPHIER_LD11_SYS_CLK_EMMC(4),
+	UNIPHIER_CLK_GATE("usb30", 12, NULL, 0x2104, 4),	/* =GIO0 */
+	UNIPHIER_CLK_GATE("usb31-0", 13, NULL, 0x2104, 5),	/* =GIO1 */
+	UNIPHIER_CLK_GATE("usb31-1", 14, NULL, 0x2104, 6),	/* =GIO1-1 */
+	UNIPHIER_CLK_GATE("usb30-phy0", 16, NULL, 0x210c, 16),
+	UNIPHIER_CLK_GATE("usb30-phy1", 17, NULL, 0x210c, 18),
+	UNIPHIER_CLK_GATE("usb30-phy2", 18, NULL, 0x210c, 20),
+	UNIPHIER_CLK_GATE("usb31-phy0", 20, NULL, 0x210c, 17),
+	UNIPHIER_CLK_GATE("usb31-phy1", 21, NULL, 0x210c, 19),
+	/* CPU gears */
+	UNIPHIER_CLK_DIV4("cpll", 2, 3, 4, 8),
+	UNIPHIER_CLK_DIV4("spll", 2, 3, 4, 8),
+	UNIPHIER_CLK_DIV4("s2pll", 2, 3, 4, 8),
+	UNIPHIER_CLK_CPUGEAR("cpu-ca53", 33, 0x8080, 0xf, 8,
+			     "cpll/2", "spll/2", "cpll/3", "spll/3",
+			     "spll/4", "spll/8", "cpll/4", "cpll/8"),
+	UNIPHIER_CLK_CPUGEAR("cpu-ipp", 34, 0x8100, 0xf, 8,
+			     "s2pll/2", "spll/2", "s2pll/3", "spll/3",
+			     "spll/4", "spll/8", "s2pll/4", "s2pll/8"),
+	{ /* sentinel */ }
+};
diff --git a/drivers/clk/uniphier/clk-uniphier.h b/drivers/clk/uniphier/clk-uniphier.h
index 01c16ecec48f..d10a009ada96 100644
--- a/drivers/clk/uniphier/clk-uniphier.h
+++ b/drivers/clk/uniphier/clk-uniphier.h
@@ -147,7 +147,6 @@ struct clk_hw *uniphier_clk_register_mux(struct device *dev,
 					 const char *name,
 				const struct uniphier_clk_mux_data *data);
 
-extern const struct uniphier_clk_data uniphier_sld3_sys_clk_data[];
 extern const struct uniphier_clk_data uniphier_ld4_sys_clk_data[];
 extern const struct uniphier_clk_data uniphier_pro4_sys_clk_data[];
 extern const struct uniphier_clk_data uniphier_sld8_sys_clk_data[];
@@ -155,7 +154,8 @@ extern const struct uniphier_clk_data uniphier_pro5_sys_clk_data[];
 extern const struct uniphier_clk_data uniphier_pxs2_sys_clk_data[];
 extern const struct uniphier_clk_data uniphier_ld11_sys_clk_data[];
 extern const struct uniphier_clk_data uniphier_ld20_sys_clk_data[];
-extern const struct uniphier_clk_data uniphier_sld3_mio_clk_data[];
+extern const struct uniphier_clk_data uniphier_pxs3_sys_clk_data[];
+extern const struct uniphier_clk_data uniphier_ld4_mio_clk_data[];
 extern const struct uniphier_clk_data uniphier_pro5_sd_clk_data[];
 extern const struct uniphier_clk_data uniphier_ld4_peri_clk_data[];
 extern const struct uniphier_clk_data uniphier_pro4_peri_clk_data[];
diff --git a/drivers/clk/ux500/clk-prcc.c b/drivers/clk/ux500/clk-prcc.c
index 0e950769ed03..f50592775c9d 100644
--- a/drivers/clk/ux500/clk-prcc.c
+++ b/drivers/clk/ux500/clk-prcc.c
@@ -79,13 +79,13 @@ static int clk_prcc_is_enabled(struct clk_hw *hw)
 	return clk->is_enabled;
 }
 
-static struct clk_ops clk_prcc_pclk_ops = {
+static const struct clk_ops clk_prcc_pclk_ops = {
 	.enable = clk_prcc_pclk_enable,
 	.disable = clk_prcc_pclk_disable,
 	.is_enabled = clk_prcc_is_enabled,
 };
 
-static struct clk_ops clk_prcc_kclk_ops = {
+static const struct clk_ops clk_prcc_kclk_ops = {
 	.enable = clk_prcc_kclk_enable,
 	.disable = clk_prcc_kclk_disable,
 	.is_enabled = clk_prcc_is_enabled,
@@ -96,7 +96,7 @@ static struct clk *clk_reg_prcc(const char *name,
 				resource_size_t phy_base,
 				u32 cg_sel,
 				unsigned long flags,
-				struct clk_ops *clk_prcc_ops)
+				const struct clk_ops *clk_prcc_ops)
 {
 	struct clk_prcc *clk;
 	struct clk_init_data clk_prcc_init;
diff --git a/drivers/clk/ux500/clk-prcmu.c b/drivers/clk/ux500/clk-prcmu.c
index 7f343821f4e4..6e3e16b2e5ca 100644
--- a/drivers/clk/ux500/clk-prcmu.c
+++ b/drivers/clk/ux500/clk-prcmu.c
@@ -186,7 +186,7 @@ static void clk_prcmu_opp_volt_unprepare(struct clk_hw *hw)
 	clk->is_prepared = 0;
 }
 
-static struct clk_ops clk_prcmu_scalable_ops = {
+static const struct clk_ops clk_prcmu_scalable_ops = {
 	.prepare = clk_prcmu_prepare,
 	.unprepare = clk_prcmu_unprepare,
 	.is_prepared = clk_prcmu_is_prepared,
@@ -198,7 +198,7 @@ static struct clk_ops clk_prcmu_scalable_ops = {
 	.set_rate = clk_prcmu_set_rate,
 };
 
-static struct clk_ops clk_prcmu_gate_ops = {
+static const struct clk_ops clk_prcmu_gate_ops = {
 	.prepare = clk_prcmu_prepare,
 	.unprepare = clk_prcmu_unprepare,
 	.is_prepared = clk_prcmu_is_prepared,
@@ -208,19 +208,19 @@ static struct clk_ops clk_prcmu_gate_ops = {
 	.recalc_rate = clk_prcmu_recalc_rate,
 };
 
-static struct clk_ops clk_prcmu_scalable_rate_ops = {
+static const struct clk_ops clk_prcmu_scalable_rate_ops = {
 	.is_enabled = clk_prcmu_is_enabled,
 	.recalc_rate = clk_prcmu_recalc_rate,
 	.round_rate = clk_prcmu_round_rate,
 	.set_rate = clk_prcmu_set_rate,
 };
 
-static struct clk_ops clk_prcmu_rate_ops = {
+static const struct clk_ops clk_prcmu_rate_ops = {
 	.is_enabled = clk_prcmu_is_enabled,
 	.recalc_rate = clk_prcmu_recalc_rate,
 };
 
-static struct clk_ops clk_prcmu_opp_gate_ops = {
+static const struct clk_ops clk_prcmu_opp_gate_ops = {
 	.prepare = clk_prcmu_opp_prepare,
 	.unprepare = clk_prcmu_opp_unprepare,
 	.is_prepared = clk_prcmu_is_prepared,
@@ -230,7 +230,7 @@ static struct clk_ops clk_prcmu_opp_gate_ops = {
 	.recalc_rate = clk_prcmu_recalc_rate,
 };
 
-static struct clk_ops clk_prcmu_opp_volt_scalable_ops = {
+static const struct clk_ops clk_prcmu_opp_volt_scalable_ops = {
 	.prepare = clk_prcmu_opp_volt_prepare,
 	.unprepare = clk_prcmu_opp_volt_unprepare,
 	.is_prepared = clk_prcmu_is_prepared,
@@ -247,7 +247,7 @@ static struct clk *clk_reg_prcmu(const char *name,
 				 u8 cg_sel,
 				 unsigned long rate,
 				 unsigned long flags,
-				 struct clk_ops *clk_prcmu_ops)
+				 const struct clk_ops *clk_prcmu_ops)
 {
 	struct clk_prcmu *clk;
 	struct clk_init_data clk_prcmu_init;
diff --git a/drivers/clk/ux500/clk-sysctrl.c b/drivers/clk/ux500/clk-sysctrl.c
index 266ddea630d2..8a4e93ce1e42 100644
--- a/drivers/clk/ux500/clk-sysctrl.c
+++ b/drivers/clk/ux500/clk-sysctrl.c
@@ -98,18 +98,18 @@ static u8 clk_sysctrl_get_parent(struct clk_hw *hw)
 	return clk->parent_index;
 }
 
-static struct clk_ops clk_sysctrl_gate_ops = {
+static const struct clk_ops clk_sysctrl_gate_ops = {
 	.prepare = clk_sysctrl_prepare,
 	.unprepare = clk_sysctrl_unprepare,
 };
 
-static struct clk_ops clk_sysctrl_gate_fixed_rate_ops = {
+static const struct clk_ops clk_sysctrl_gate_fixed_rate_ops = {
 	.prepare = clk_sysctrl_prepare,
 	.unprepare = clk_sysctrl_unprepare,
 	.recalc_rate = clk_sysctrl_recalc_rate,
 };
 
-static struct clk_ops clk_sysctrl_set_parent_ops = {
+static const struct clk_ops clk_sysctrl_set_parent_ops = {
 	.set_parent = clk_sysctrl_set_parent,
 	.get_parent = clk_sysctrl_get_parent,
 };
@@ -124,7 +124,7 @@ static struct clk *clk_reg_sysctrl(struct device *dev,
 				unsigned long rate,
 				unsigned long enable_delay_us,
 				unsigned long flags,
-				struct clk_ops *clk_sysctrl_ops)
+				const struct clk_ops *clk_sysctrl_ops)
 {
 	struct clk_sysctrl *clk;
 	struct clk_init_data clk_sysctrl_init;
diff --git a/drivers/clk/versatile/clk-vexpress-osc.c b/drivers/clk/versatile/clk-vexpress-osc.c
index 7e5add7d7752..e7a868b83fe5 100644
--- a/drivers/clk/versatile/clk-vexpress-osc.c
+++ b/drivers/clk/versatile/clk-vexpress-osc.c
@@ -61,7 +61,7 @@ static int vexpress_osc_set_rate(struct clk_hw *hw, unsigned long rate,
 	return regmap_write(osc->reg, 0, rate);
 }
 
-static struct clk_ops vexpress_osc_ops = {
+static const struct clk_ops vexpress_osc_ops = {
 	.recalc_rate = vexpress_osc_recalc_rate,
 	.round_rate = vexpress_osc_round_rate,
 	.set_rate = vexpress_osc_set_rate,
diff --git a/drivers/clk/zte/clk-zx296718.c b/drivers/clk/zte/clk-zx296718.c
index 27f853d4c76b..354dd508c516 100644
--- a/drivers/clk/zte/clk-zx296718.c
+++ b/drivers/clk/zte/clk-zx296718.c
@@ -451,7 +451,7 @@ static struct zx_clk_fixed_factor top_ffactor_clk[] = {
 	FFACTOR(0, "emmc_mux_div2",	"emmc_mux", 1, 2, CLK_SET_RATE_PARENT),
 };
 
-static struct clk_div_table noc_div_table[] = {
+static const struct clk_div_table noc_div_table[] = {
 	{ .val = 1, .div = 2, },
 	{ .val = 3, .div = 4, },
 };
@@ -644,7 +644,7 @@ static int __init top_clocks_init(struct device_node *np)
 	return 0;
 }
 
-static struct clk_div_table common_even_div_table[] = {
+static const struct clk_div_table common_even_div_table[] = {
 	{ .val = 0, .div = 1, },
 	{ .val = 1, .div = 2, },
 	{ .val = 3, .div = 4, },
@@ -656,7 +656,7 @@ static struct clk_div_table common_even_div_table[] = {
 	{ .val = 15, .div = 16, },
 };
 
-static struct clk_div_table common_div_table[] = {
+static const struct clk_div_table common_div_table[] = {
 	{ .val = 0, .div = 1, },
 	{ .val = 1, .div = 2, },
 	{ .val = 2, .div = 3, },
diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig
index fcae5ca6ac92..cc6062049170 100644
--- a/drivers/clocksource/Kconfig
+++ b/drivers/clocksource/Kconfig
@@ -262,7 +262,7 @@ config CLKSRC_LPC32XX
 
 config CLKSRC_PISTACHIO
 	bool "Clocksource for Pistachio SoC" if COMPILE_TEST
-	depends on HAS_IOMEM
+	depends on GENERIC_CLOCKEVENTS && HAS_IOMEM
 	select TIMER_OF
 	help
 	  Enables the clocksource for the Pistachio SoC.
@@ -598,6 +598,14 @@ config CLKSRC_IMX_GPT
 	depends on ARM && CLKDEV_LOOKUP
 	select CLKSRC_MMIO
 
+config CLKSRC_IMX_TPM
+	bool "Clocksource using i.MX TPM" if COMPILE_TEST
+	depends on ARM && CLKDEV_LOOKUP && GENERIC_CLOCKEVENTS
+	select CLKSRC_MMIO
+	help
+	  Enable this option to use IMX Timer/PWM Module (TPM) timer as
+	  clocksource.
+
 config CLKSRC_ST_LPC
 	bool "Low power clocksource found in the LPC" if COMPILE_TEST
 	select TIMER_OF if OF
diff --git a/drivers/clocksource/Makefile b/drivers/clocksource/Makefile
index 6df949402dfc..dbc1ad14515e 100644
--- a/drivers/clocksource/Makefile
+++ b/drivers/clocksource/Makefile
@@ -67,6 +67,7 @@ obj-$(CONFIG_CLKSRC_VERSATILE)		+= versatile.o
 obj-$(CONFIG_CLKSRC_MIPS_GIC)		+= mips-gic-timer.o
 obj-$(CONFIG_CLKSRC_TANGO_XTAL)		+= tango_xtal.o
 obj-$(CONFIG_CLKSRC_IMX_GPT)		+= timer-imx-gpt.o
+obj-$(CONFIG_CLKSRC_IMX_TPM)		+= timer-imx-tpm.o
 obj-$(CONFIG_ASM9260_TIMER)		+= asm9260_timer.o
 obj-$(CONFIG_H8300_TMR8)		+= h8300_timer8.o
 obj-$(CONFIG_H8300_TMR16)		+= h8300_timer16.o
diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c
index aae87c4c546e..fd4b7f684bd0 100644
--- a/drivers/clocksource/arm_arch_timer.c
+++ b/drivers/clocksource/arm_arch_timer.c
@@ -455,7 +455,11 @@ void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa
 			per_cpu(timer_unstable_counter_workaround, i) = wa;
 	}
 
-	static_branch_enable(&arch_timer_read_ool_enabled);
+	/*
+	 * Use the locked version, as we're called from the CPU
+	 * hotplug framework. Otherwise, we end-up in deadlock-land.
+	 */
+	static_branch_enable_cpuslocked(&arch_timer_read_ool_enabled);
 
 	/*
 	 * Don't use the vdso fastpath if errata require using the
@@ -1440,7 +1444,7 @@ static int __init arch_timer_mem_acpi_init(int platform_timer_count)
 	 * While unlikely, it's theoretically possible that none of the frames
 	 * in a timer expose the combination of feature we want.
 	 */
-	for (i = i; i < timer_count; i++) {
+	for (i = 0; i < timer_count; i++) {
 		timer = &timers[i];
 
 		frame = arch_timer_mem_find_best_frame(timer);
diff --git a/drivers/clocksource/bcm2835_timer.c b/drivers/clocksource/bcm2835_timer.c
index 82828d3a4739..39e489a96ad7 100644
--- a/drivers/clocksource/bcm2835_timer.c
+++ b/drivers/clocksource/bcm2835_timer.c
@@ -114,7 +114,6 @@ static int __init bcm2835_timer_init(struct device_node *node)
 
 	timer = kzalloc(sizeof(*timer), GFP_KERNEL);
 	if (!timer) {
-		pr_err("Can't allocate timer struct\n");
 		ret = -ENOMEM;
 		goto err_iounmap;
 	}
diff --git a/drivers/clocksource/em_sti.c b/drivers/clocksource/em_sti.c
index bc48cbf6a795..269db74a0658 100644
--- a/drivers/clocksource/em_sti.c
+++ b/drivers/clocksource/em_sti.c
@@ -305,7 +305,7 @@ static int em_sti_probe(struct platform_device *pdev)
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
 		dev_err(&pdev->dev, "failed to get irq\n");
-		return -EINVAL;
+		return irq;
 	}
 
 	/* map memory, let base point to the STI instance */
@@ -314,11 +314,12 @@ static int em_sti_probe(struct platform_device *pdev)
 	if (IS_ERR(p->base))
 		return PTR_ERR(p->base);
 
-	if (devm_request_irq(&pdev->dev, irq, em_sti_interrupt,
-			     IRQF_TIMER | IRQF_IRQPOLL | IRQF_NOBALANCING,
-			     dev_name(&pdev->dev), p)) {
+	ret = devm_request_irq(&pdev->dev, irq, em_sti_interrupt,
+			       IRQF_TIMER | IRQF_IRQPOLL | IRQF_NOBALANCING,
+			       dev_name(&pdev->dev), p);
+	if (ret) {
 		dev_err(&pdev->dev, "failed to request low IRQ\n");
-		return -ENOENT;
+		return ret;
 	}
 
 	/* get hold of clock */
diff --git a/drivers/clocksource/mips-gic-timer.c b/drivers/clocksource/mips-gic-timer.c
index 17b861ea2626..ae3167c28b12 100644
--- a/drivers/clocksource/mips-gic-timer.c
+++ b/drivers/clocksource/mips-gic-timer.c
@@ -10,25 +10,45 @@
 #include <linux/cpu.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
-#include <linux/irqchip/mips-gic.h>
 #include <linux/notifier.h>
 #include <linux/of_irq.h>
 #include <linux/percpu.h>
 #include <linux/smp.h>
 #include <linux/time.h>
+#include <asm/mips-cps.h>
 
 static DEFINE_PER_CPU(struct clock_event_device, gic_clockevent_device);
 static int gic_timer_irq;
 static unsigned int gic_frequency;
 
+static u64 notrace gic_read_count(void)
+{
+	unsigned int hi, hi2, lo;
+
+	if (mips_cm_is64)
+		return read_gic_counter();
+
+	do {
+		hi = read_gic_counter_32h();
+		lo = read_gic_counter_32l();
+		hi2 = read_gic_counter_32h();
+	} while (hi2 != hi);
+
+	return (((u64) hi) << 32) + lo;
+}
+
 static int gic_next_event(unsigned long delta, struct clock_event_device *evt)
 {
+	unsigned long flags;
 	u64 cnt;
 	int res;
 
 	cnt = gic_read_count();
 	cnt += (u64)delta;
-	gic_write_cpu_compare(cnt, cpumask_first(evt->cpumask));
+	local_irq_save(flags);
+	write_gic_vl_other(mips_cm_vp_id(cpumask_first(evt->cpumask)));
+	write_gic_vo_compare(cnt);
+	local_irq_restore(flags);
 	res = ((int)(gic_read_count() - cnt) >= 0) ? -ETIME : 0;
 	return res;
 }
@@ -37,7 +57,7 @@ static irqreturn_t gic_compare_interrupt(int irq, void *dev_id)
 {
 	struct clock_event_device *cd = dev_id;
 
-	gic_write_compare(gic_read_compare());
+	write_gic_vl_compare(read_gic_vl_compare());
 	cd->event_handler(cd);
 	return IRQ_HANDLED;
 }
@@ -139,10 +159,15 @@ static struct clocksource gic_clocksource = {
 
 static int __init __gic_clocksource_init(void)
 {
+	unsigned int count_width;
 	int ret;
 
 	/* Set clocksource mask. */
-	gic_clocksource.mask = CLOCKSOURCE_MASK(gic_get_count_width());
+	count_width = read_gic_config() & GIC_CONFIG_COUNTBITS;
+	count_width >>= __fls(GIC_CONFIG_COUNTBITS);
+	count_width *= 4;
+	count_width += 32;
+	gic_clocksource.mask = CLOCKSOURCE_MASK(count_width);
 
 	/* Calculate a somewhat reasonable rating value. */
 	gic_clocksource.rating = 200 + gic_frequency / 10000000;
@@ -159,7 +184,7 @@ static int __init gic_clocksource_of_init(struct device_node *node)
 	struct clk *clk;
 	int ret;
 
-	if (!gic_present || !node->parent ||
+	if (!mips_gic_present() || !node->parent ||
 	    !of_device_is_compatible(node->parent, "mti,gic")) {
 		pr_warn("No DT definition for the mips gic driver\n");
 		return -ENXIO;
@@ -197,7 +222,7 @@ static int __init gic_clocksource_of_init(struct device_node *node)
 	}
 
 	/* And finally start the counter */
-	gic_start_count();
+	clear_gic_config(GIC_CONFIG_COUNTSTOP);
 
 	return 0;
 }
diff --git a/drivers/clocksource/tango_xtal.c b/drivers/clocksource/tango_xtal.c
index c4e1c2e6046f..6a8d9838ce33 100644
--- a/drivers/clocksource/tango_xtal.c
+++ b/drivers/clocksource/tango_xtal.c
@@ -26,13 +26,13 @@ static int __init tango_clocksource_init(struct device_node *np)
 
 	xtal_in_cnt = of_iomap(np, 0);
 	if (xtal_in_cnt == NULL) {
-		pr_err("%s: invalid address\n", np->full_name);
+		pr_err("%pOF: invalid address\n", np);
 		return -ENXIO;
 	}
 
 	clk = of_clk_get(np, 0);
 	if (IS_ERR(clk)) {
-		pr_err("%s: invalid clock\n", np->full_name);
+		pr_err("%pOF: invalid clock\n", np);
 		return PTR_ERR(clk);
 	}
 
@@ -43,7 +43,7 @@ static int __init tango_clocksource_init(struct device_node *np)
 	ret = clocksource_mmio_init(xtal_in_cnt, "tango-xtal", xtal_freq, 350,
 				    32, clocksource_mmio_readl_up);
 	if (ret) {
-		pr_err("%s: registration failed\n", np->full_name);
+		pr_err("%pOF: registration failed\n", np);
 		return ret;
 	}
 
diff --git a/drivers/clocksource/timer-imx-tpm.c b/drivers/clocksource/timer-imx-tpm.c
new file mode 100644
index 000000000000..21bffdcb2f20
--- /dev/null
+++ b/drivers/clocksource/timer-imx-tpm.c
@@ -0,0 +1,239 @@
+/*
+ * Copyright 2016 Freescale Semiconductor, Inc.
+ * Copyright 2017 NXP
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ */
+
+#include <linux/clk.h>
+#include <linux/clockchips.h>
+#include <linux/clocksource.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/sched_clock.h>
+
+#define TPM_SC				0x10
+#define TPM_SC_CMOD_INC_PER_CNT		(0x1 << 3)
+#define TPM_SC_CMOD_DIV_DEFAULT		0x3
+#define TPM_CNT				0x14
+#define TPM_MOD				0x18
+#define TPM_STATUS			0x1c
+#define TPM_STATUS_CH0F			BIT(0)
+#define TPM_C0SC			0x20
+#define TPM_C0SC_CHIE			BIT(6)
+#define TPM_C0SC_MODE_SHIFT		2
+#define TPM_C0SC_MODE_MASK		0x3c
+#define TPM_C0SC_MODE_SW_COMPARE	0x4
+#define TPM_C0V				0x24
+
+static void __iomem *timer_base;
+static struct clock_event_device clockevent_tpm;
+
+static inline void tpm_timer_disable(void)
+{
+	unsigned int val;
+
+	/* channel disable */
+	val = readl(timer_base + TPM_C0SC);
+	val &= ~(TPM_C0SC_MODE_MASK | TPM_C0SC_CHIE);
+	writel(val, timer_base + TPM_C0SC);
+}
+
+static inline void tpm_timer_enable(void)
+{
+	unsigned int val;
+
+	/* channel enabled in sw compare mode */
+	val = readl(timer_base + TPM_C0SC);
+	val |= (TPM_C0SC_MODE_SW_COMPARE << TPM_C0SC_MODE_SHIFT) |
+	       TPM_C0SC_CHIE;
+	writel(val, timer_base + TPM_C0SC);
+}
+
+static inline void tpm_irq_acknowledge(void)
+{
+	writel(TPM_STATUS_CH0F, timer_base + TPM_STATUS);
+}
+
+static struct delay_timer tpm_delay_timer;
+
+static inline unsigned long tpm_read_counter(void)
+{
+	return readl(timer_base + TPM_CNT);
+}
+
+static unsigned long tpm_read_current_timer(void)
+{
+	return tpm_read_counter();
+}
+
+static u64 notrace tpm_read_sched_clock(void)
+{
+	return tpm_read_counter();
+}
+
+static int __init tpm_clocksource_init(unsigned long rate)
+{
+	tpm_delay_timer.read_current_timer = &tpm_read_current_timer;
+	tpm_delay_timer.freq = rate;
+	register_current_timer_delay(&tpm_delay_timer);
+
+	sched_clock_register(tpm_read_sched_clock, 32, rate);
+
+	return clocksource_mmio_init(timer_base + TPM_CNT, "imx-tpm",
+				     rate, 200, 32, clocksource_mmio_readl_up);
+}
+
+static int tpm_set_next_event(unsigned long delta,
+				struct clock_event_device *evt)
+{
+	unsigned long next, now;
+
+	next = tpm_read_counter();
+	next += delta;
+	writel(next, timer_base + TPM_C0V);
+	now = tpm_read_counter();
+
+	/*
+	 * NOTE: We observed in a very small probability, the bus fabric
+	 * contention between GPU and A7 may results a few cycles delay
+	 * of writing CNT registers which may cause the min_delta event got
+	 * missed, so we need add a ETIME check here in case it happened.
+	 */
+	return (int)((next - now) <= 0) ? -ETIME : 0;
+}
+
+static int tpm_set_state_oneshot(struct clock_event_device *evt)
+{
+	tpm_timer_enable();
+
+	return 0;
+}
+
+static int tpm_set_state_shutdown(struct clock_event_device *evt)
+{
+	tpm_timer_disable();
+
+	return 0;
+}
+
+static irqreturn_t tpm_timer_interrupt(int irq, void *dev_id)
+{
+	struct clock_event_device *evt = dev_id;
+
+	tpm_irq_acknowledge();
+
+	evt->event_handler(evt);
+
+	return IRQ_HANDLED;
+}
+
+static struct clock_event_device clockevent_tpm = {
+	.name			= "i.MX7ULP TPM Timer",
+	.features		= CLOCK_EVT_FEAT_ONESHOT,
+	.set_state_oneshot	= tpm_set_state_oneshot,
+	.set_next_event		= tpm_set_next_event,
+	.set_state_shutdown	= tpm_set_state_shutdown,
+	.rating			= 200,
+};
+
+static int __init tpm_clockevent_init(unsigned long rate, int irq)
+{
+	int ret;
+
+	ret = request_irq(irq, tpm_timer_interrupt, IRQF_TIMER | IRQF_IRQPOLL,
+			  "i.MX7ULP TPM Timer", &clockevent_tpm);
+
+	clockevent_tpm.cpumask = cpumask_of(0);
+	clockevent_tpm.irq = irq;
+	clockevents_config_and_register(&clockevent_tpm,
+					rate, 300, 0xfffffffe);
+
+	return ret;
+}
+
+static int __init tpm_timer_init(struct device_node *np)
+{
+	struct clk *ipg, *per;
+	int irq, ret;
+	u32 rate;
+
+	timer_base = of_iomap(np, 0);
+	if (!timer_base) {
+		pr_err("tpm: failed to get base address\n");
+		return -ENXIO;
+	}
+
+	irq = irq_of_parse_and_map(np, 0);
+	if (!irq) {
+		pr_err("tpm: failed to get irq\n");
+		ret = -ENOENT;
+		goto err_iomap;
+	}
+
+	ipg = of_clk_get_by_name(np, "ipg");
+	per = of_clk_get_by_name(np, "per");
+	if (IS_ERR(ipg) || IS_ERR(per)) {
+		pr_err("tpm: failed to get igp or per clk\n");
+		ret = -ENODEV;
+		goto err_clk_get;
+	}
+
+	/* enable clk before accessing registers */
+	ret = clk_prepare_enable(ipg);
+	if (ret) {
+		pr_err("tpm: ipg clock enable failed (%d)\n", ret);
+		goto err_clk_get;
+	}
+
+	ret = clk_prepare_enable(per);
+	if (ret) {
+		pr_err("tpm: per clock enable failed (%d)\n", ret);
+		goto err_per_clk_enable;
+	}
+
+	/*
+	 * Initialize tpm module to a known state
+	 * 1) Counter disabled
+	 * 2) TPM counter operates in up counting mode
+	 * 3) Timer Overflow Interrupt disabled
+	 * 4) Channel0 disabled
+	 * 5) DMA transfers disabled
+	 */
+	writel(0, timer_base + TPM_SC);
+	writel(0, timer_base + TPM_CNT);
+	writel(0, timer_base + TPM_C0SC);
+
+	/* increase per cnt, div 8 by default */
+	writel(TPM_SC_CMOD_INC_PER_CNT | TPM_SC_CMOD_DIV_DEFAULT,
+		     timer_base + TPM_SC);
+
+	/* set MOD register to maximum for free running mode */
+	writel(0xffffffff, timer_base + TPM_MOD);
+
+	rate = clk_get_rate(per) >> 3;
+	ret = tpm_clocksource_init(rate);
+	if (ret)
+		goto err_per_clk_enable;
+
+	ret = tpm_clockevent_init(rate, irq);
+	if (ret)
+		goto err_per_clk_enable;
+
+	return 0;
+
+err_per_clk_enable:
+	clk_disable_unprepare(ipg);
+err_clk_get:
+	clk_put(per);
+	clk_put(ipg);
+err_iomap:
+	iounmap(timer_base);
+	return ret;
+}
+TIMER_OF_DECLARE(imx7ulp, "fsl,imx7ulp-tpm", tpm_timer_init);
diff --git a/drivers/clocksource/timer-of.c b/drivers/clocksource/timer-of.c
index d509b500a7b5..c79122d8e10d 100644
--- a/drivers/clocksource/timer-of.c
+++ b/drivers/clocksource/timer-of.c
@@ -52,7 +52,7 @@ static __init int timer_irq_init(struct device_node *np,
 		of_irq->irq = irq_of_parse_and_map(np, of_irq->index);
 	}
 	if (!of_irq->irq) {
-		pr_err("Failed to map interrupt for %s\n", np->full_name);
+		pr_err("Failed to map interrupt for %pOF\n", np);
 		return -EINVAL;
 	}
 
@@ -63,8 +63,7 @@ static __init int timer_irq_init(struct device_node *np,
 			    of_irq->flags ? of_irq->flags : IRQF_TIMER,
 			    np->full_name, clkevt);
 	if (ret) {
-		pr_err("Failed to request irq %d for %s\n", of_irq->irq,
-		       np->full_name);
+		pr_err("Failed to request irq %d for %pOF\n", of_irq->irq, np);
 		return ret;
 	}
 
@@ -88,20 +87,20 @@ static __init int timer_clk_init(struct device_node *np,
 	of_clk->clk = of_clk->name ? of_clk_get_by_name(np, of_clk->name) :
 		of_clk_get(np, of_clk->index);
 	if (IS_ERR(of_clk->clk)) {
-		pr_err("Failed to get clock for %s\n", np->full_name);
+		pr_err("Failed to get clock for %pOF\n", np);
 		return PTR_ERR(of_clk->clk);
 	}
 
 	ret = clk_prepare_enable(of_clk->clk);
 	if (ret) {
-		pr_err("Failed for enable clock for %s\n", np->full_name);
+		pr_err("Failed for enable clock for %pOF\n", np);
 		goto out_clk_put;
 	}
 
 	of_clk->rate = clk_get_rate(of_clk->clk);
 	if (!of_clk->rate) {
 		ret = -EINVAL;
-		pr_err("Failed to get clock rate for %s\n", np->full_name);
+		pr_err("Failed to get clock rate for %pOF\n", np);
 		goto out_clk_disable;
 	}
 
@@ -128,9 +127,9 @@ static __init int timer_base_init(struct device_node *np,
 	const char *name = of_base->name ? of_base->name : np->full_name;
 
 	of_base->base = of_io_request_and_map(np, of_base->index, name);
-	if (!of_base->base) {
+	if (IS_ERR(of_base->base)) {
 		pr_err("Failed to iomap (%s)\n", name);
-		return -ENXIO;
+		return PTR_ERR(of_base->base);
 	}
 
 	return 0;
diff --git a/drivers/clocksource/timer-probe.c b/drivers/clocksource/timer-probe.c
index da81e5de74fe..028075720334 100644
--- a/drivers/clocksource/timer-probe.c
+++ b/drivers/clocksource/timer-probe.c
@@ -40,8 +40,7 @@ void __init timer_probe(void)
 
 		ret = init_func_ret(np);
 		if (ret) {
-			pr_err("Failed to initialize '%s': %d\n",
-			       of_node_full_name(np), ret);
+			pr_err("Failed to initialize '%pOF': %d\n", np, ret);
 			continue;
 		}
 
diff --git a/drivers/clocksource/timer-stm32.c b/drivers/clocksource/timer-stm32.c
index 174d1243ea93..8f2423789ba9 100644
--- a/drivers/clocksource/timer-stm32.c
+++ b/drivers/clocksource/timer-stm32.c
@@ -138,7 +138,7 @@ static int __init stm32_clockevent_init(struct device_node *np)
 	irq = irq_of_parse_and_map(np, 0);
 	if (!irq) {
 		ret = -EINVAL;
-		pr_err("%s: failed to get irq.\n", np->full_name);
+		pr_err("%pOF: failed to get irq.\n", np);
 		goto err_get_irq;
 	}
 
@@ -168,12 +168,12 @@ static int __init stm32_clockevent_init(struct device_node *np)
 	ret = request_irq(irq, stm32_clock_event_handler, IRQF_TIMER,
 			"stm32 clockevent", data);
 	if (ret) {
-		pr_err("%s: failed to request irq.\n", np->full_name);
+		pr_err("%pOF: failed to request irq.\n", np);
 		goto err_get_irq;
 	}
 
-	pr_info("%s: STM32 clockevent driver initialized (%d bits)\n",
-			np->full_name, bits);
+	pr_info("%pOF: STM32 clockevent driver initialized (%d bits)\n",
+			np, bits);
 
 	return ret;
 
diff --git a/drivers/cpufreq/Kconfig.arm b/drivers/cpufreq/Kconfig.arm
index 2011fec2d6ad..bdce4488ded1 100644
--- a/drivers/cpufreq/Kconfig.arm
+++ b/drivers/cpufreq/Kconfig.arm
@@ -71,15 +71,6 @@ config ARM_HIGHBANK_CPUFREQ
 
 	  If in doubt, say N.
 
-config ARM_DB8500_CPUFREQ
-	tristate "ST-Ericsson DB8500 cpufreq" if COMPILE_TEST && !ARCH_U8500
-	default ARCH_U8500
-	depends on HAS_IOMEM
-	depends on !CPU_THERMAL || THERMAL
-	help
-	  This adds the CPUFreq driver for ST-Ericsson Ux500 (DB8500) SoC
-	  series.
-
 config ARM_IMX6Q_CPUFREQ
 	tristate "Freescale i.MX6 cpufreq support"
 	depends on ARCH_MXC
@@ -96,14 +87,13 @@ config ARM_KIRKWOOD_CPUFREQ
 	  This adds the CPUFreq driver for Marvell Kirkwood
 	  SoCs.
 
-config ARM_MT8173_CPUFREQ
-	tristate "Mediatek MT8173 CPUFreq support"
+config ARM_MEDIATEK_CPUFREQ
+	tristate "CPU Frequency scaling support for MediaTek SoCs"
 	depends on ARCH_MEDIATEK && REGULATOR
-	depends on ARM64 || (ARM_CPU_TOPOLOGY && COMPILE_TEST)
 	depends on !CPU_THERMAL || THERMAL
 	select PM_OPP
 	help
-	  This adds the CPUFreq driver support for Mediatek MT8173 SoC.
+	  This adds the CPUFreq driver support for MediaTek SoCs.
 
 config ARM_OMAP2PLUS_CPUFREQ
 	bool "TI OMAP2+"
@@ -242,6 +232,11 @@ config ARM_STI_CPUFREQ
 	  this config option if you wish to add CPUFreq support for STi based
 	  SoCs.
 
+config ARM_TANGO_CPUFREQ
+	bool
+	depends on CPUFREQ_DT && ARCH_TANGO
+	default y
+
 config ARM_TEGRA20_CPUFREQ
 	bool "Tegra20 CPUFreq support"
 	depends on ARCH_TEGRA
diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile
index ab3a42cd29ef..c7af9b2a255e 100644
--- a/drivers/cpufreq/Makefile
+++ b/drivers/cpufreq/Makefile
@@ -53,12 +53,11 @@ obj-$(CONFIG_ARM_DT_BL_CPUFREQ)		+= arm_big_little_dt.o
 
 obj-$(CONFIG_ARM_BRCMSTB_AVS_CPUFREQ)	+= brcmstb-avs-cpufreq.o
 obj-$(CONFIG_ARCH_DAVINCI)		+= davinci-cpufreq.o
-obj-$(CONFIG_ARM_DB8500_CPUFREQ)	+= dbx500-cpufreq.o
 obj-$(CONFIG_ARM_EXYNOS5440_CPUFREQ)	+= exynos5440-cpufreq.o
 obj-$(CONFIG_ARM_HIGHBANK_CPUFREQ)	+= highbank-cpufreq.o
 obj-$(CONFIG_ARM_IMX6Q_CPUFREQ)		+= imx6q-cpufreq.o
 obj-$(CONFIG_ARM_KIRKWOOD_CPUFREQ)	+= kirkwood-cpufreq.o
-obj-$(CONFIG_ARM_MT8173_CPUFREQ)	+= mt8173-cpufreq.o
+obj-$(CONFIG_ARM_MEDIATEK_CPUFREQ)	+= mediatek-cpufreq.o
 obj-$(CONFIG_ARM_OMAP2PLUS_CPUFREQ)	+= omap-cpufreq.o
 obj-$(CONFIG_ARM_PXA2xx_CPUFREQ)	+= pxa2xx-cpufreq.o
 obj-$(CONFIG_PXA3xx)			+= pxa3xx-cpufreq.o
@@ -75,6 +74,7 @@ obj-$(CONFIG_ARM_SA1110_CPUFREQ)	+= sa1110-cpufreq.o
 obj-$(CONFIG_ARM_SCPI_CPUFREQ)		+= scpi-cpufreq.o
 obj-$(CONFIG_ARM_SPEAR_CPUFREQ)		+= spear-cpufreq.o
 obj-$(CONFIG_ARM_STI_CPUFREQ)		+= sti-cpufreq.o
+obj-$(CONFIG_ARM_TANGO_CPUFREQ)		+= tango-cpufreq.o
 obj-$(CONFIG_ARM_TEGRA20_CPUFREQ)	+= tegra20-cpufreq.o
 obj-$(CONFIG_ARM_TEGRA124_CPUFREQ)	+= tegra124-cpufreq.o
 obj-$(CONFIG_ARM_TEGRA186_CPUFREQ)	+= tegra186-cpufreq.o
diff --git a/drivers/cpufreq/arm_big_little.c b/drivers/cpufreq/arm_big_little.c
index ea6d62547b10..17504129fd77 100644
--- a/drivers/cpufreq/arm_big_little.c
+++ b/drivers/cpufreq/arm_big_little.c
@@ -483,11 +483,8 @@ static int bL_cpufreq_init(struct cpufreq_policy *policy)
 		return ret;
 	}
 
-	if (arm_bL_ops->get_transition_latency)
-		policy->cpuinfo.transition_latency =
-			arm_bL_ops->get_transition_latency(cpu_dev);
-	else
-		policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
+	policy->cpuinfo.transition_latency =
+				arm_bL_ops->get_transition_latency(cpu_dev);
 
 	if (is_bL_switching_enabled())
 		per_cpu(cpu_last_req_freq, policy->cpu) = clk_get_cpu_rate(policy->cpu);
@@ -622,7 +619,8 @@ int bL_cpufreq_register(struct cpufreq_arm_bL_ops *ops)
 		return -EBUSY;
 	}
 
-	if (!ops || !strlen(ops->name) || !ops->init_opp_table) {
+	if (!ops || !strlen(ops->name) || !ops->init_opp_table ||
+	    !ops->get_transition_latency) {
 		pr_err("%s: Invalid arm_bL_ops, exiting\n", __func__);
 		return -ENODEV;
 	}
diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c
index 10be285c9055..a1c3025f9df7 100644
--- a/drivers/cpufreq/cppc_cpufreq.c
+++ b/drivers/cpufreq/cppc_cpufreq.c
@@ -172,7 +172,6 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy)
 		return -EFAULT;
 	}
 
-	cpumask_set_cpu(policy->cpu, policy->cpus);
 	cpu->cur_policy = policy;
 
 	/* Set policy->cur to max now. The governors will adjust later. */
diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c
index 1c262923fe58..a020da7940d6 100644
--- a/drivers/cpufreq/cpufreq-dt-platdev.c
+++ b/drivers/cpufreq/cpufreq-dt-platdev.c
@@ -9,11 +9,16 @@
 
 #include <linux/err.h>
 #include <linux/of.h>
+#include <linux/of_device.h>
 #include <linux/platform_device.h>
 
 #include "cpufreq-dt.h"
 
-static const struct of_device_id machines[] __initconst = {
+/*
+ * Machines for which the cpufreq device is *always* created, mostly used for
+ * platforms using "operating-points" (V1) property.
+ */
+static const struct of_device_id whitelist[] __initconst = {
 	{ .compatible = "allwinner,sun4i-a10", },
 	{ .compatible = "allwinner,sun5i-a10s", },
 	{ .compatible = "allwinner,sun5i-a13", },
@@ -22,7 +27,6 @@ static const struct of_device_id machines[] __initconst = {
 	{ .compatible = "allwinner,sun6i-a31s", },
 	{ .compatible = "allwinner,sun7i-a20", },
 	{ .compatible = "allwinner,sun8i-a23", },
-	{ .compatible = "allwinner,sun8i-a33", },
 	{ .compatible = "allwinner,sun8i-a83t", },
 	{ .compatible = "allwinner,sun8i-h3", },
 
@@ -32,7 +36,6 @@ static const struct of_device_id machines[] __initconst = {
 	{ .compatible = "arm,integrator-cp", },
 
 	{ .compatible = "hisilicon,hi3660", },
-	{ .compatible = "hisilicon,hi6220", },
 
 	{ .compatible = "fsl,imx27", },
 	{ .compatible = "fsl,imx51", },
@@ -46,11 +49,8 @@ static const struct of_device_id machines[] __initconst = {
 	{ .compatible = "samsung,exynos3250", },
 	{ .compatible = "samsung,exynos4210", },
 	{ .compatible = "samsung,exynos4212", },
-	{ .compatible = "samsung,exynos4412", },
 	{ .compatible = "samsung,exynos5250", },
 #ifndef CONFIG_BL_SWITCHER
-	{ .compatible = "samsung,exynos5420", },
-	{ .compatible = "samsung,exynos5433", },
 	{ .compatible = "samsung,exynos5800", },
 #endif
 
@@ -67,6 +67,8 @@ static const struct of_device_id machines[] __initconst = {
 	{ .compatible = "renesas,r8a7792", },
 	{ .compatible = "renesas,r8a7793", },
 	{ .compatible = "renesas,r8a7794", },
+	{ .compatible = "renesas,r8a7795", },
+	{ .compatible = "renesas,r8a7796", },
 	{ .compatible = "renesas,sh73a0", },
 
 	{ .compatible = "rockchip,rk2928", },
@@ -76,17 +78,17 @@ static const struct of_device_id machines[] __initconst = {
 	{ .compatible = "rockchip,rk3188", },
 	{ .compatible = "rockchip,rk3228", },
 	{ .compatible = "rockchip,rk3288", },
+	{ .compatible = "rockchip,rk3328", },
 	{ .compatible = "rockchip,rk3366", },
 	{ .compatible = "rockchip,rk3368", },
 	{ .compatible = "rockchip,rk3399", },
 
-	{ .compatible = "sigma,tango4" },
-
-	{ .compatible = "socionext,uniphier-pro5", },
-	{ .compatible = "socionext,uniphier-pxs2", },
 	{ .compatible = "socionext,uniphier-ld6b", },
-	{ .compatible = "socionext,uniphier-ld11", },
-	{ .compatible = "socionext,uniphier-ld20", },
+
+	{ .compatible = "st-ericsson,u8500", },
+	{ .compatible = "st-ericsson,u8540", },
+	{ .compatible = "st-ericsson,u9500", },
+	{ .compatible = "st-ericsson,u9540", },
 
 	{ .compatible = "ti,omap2", },
 	{ .compatible = "ti,omap3", },
@@ -94,27 +96,56 @@ static const struct of_device_id machines[] __initconst = {
 	{ .compatible = "ti,omap5", },
 
 	{ .compatible = "xlnx,zynq-7000", },
+	{ .compatible = "xlnx,zynqmp", },
 
-	{ .compatible = "zte,zx296718", },
+	{ }
+};
 
+/*
+ * Machines for which the cpufreq device is *not* created, mostly used for
+ * platforms using "operating-points-v2" property.
+ */
+static const struct of_device_id blacklist[] __initconst = {
 	{ }
 };
 
+static bool __init cpu0_node_has_opp_v2_prop(void)
+{
+	struct device_node *np = of_cpu_device_node_get(0);
+	bool ret = false;
+
+	if (of_get_property(np, "operating-points-v2", NULL))
+		ret = true;
+
+	of_node_put(np);
+	return ret;
+}
+
 static int __init cpufreq_dt_platdev_init(void)
 {
 	struct device_node *np = of_find_node_by_path("/");
 	const struct of_device_id *match;
+	const void *data = NULL;
 
 	if (!np)
 		return -ENODEV;
 
-	match = of_match_node(machines, np);
+	match = of_match_node(whitelist, np);
+	if (match) {
+		data = match->data;
+		goto create_pdev;
+	}
+
+	if (cpu0_node_has_opp_v2_prop() && !of_match_node(blacklist, np))
+		goto create_pdev;
+
 	of_node_put(np);
-	if (!match)
-		return -ENODEV;
+	return -ENODEV;
 
+create_pdev:
+	of_node_put(np);
 	return PTR_ERR_OR_ZERO(platform_device_register_data(NULL, "cpufreq-dt",
-			       -1, match->data,
+			       -1, data,
 			       sizeof(struct cpufreq_dt_platform_data)));
 }
 device_initcall(cpufreq_dt_platdev_init);
diff --git a/drivers/cpufreq/cpufreq-dt.c b/drivers/cpufreq/cpufreq-dt.c
index fef3c2160691..d83ab94d041a 100644
--- a/drivers/cpufreq/cpufreq-dt.c
+++ b/drivers/cpufreq/cpufreq-dt.c
@@ -274,6 +274,7 @@ static int cpufreq_init(struct cpufreq_policy *policy)
 		transition_latency = CPUFREQ_ETERNAL;
 
 	policy->cpuinfo.transition_latency = transition_latency;
+	policy->dvfs_possible_from_any_cpu = true;
 
 	return 0;
 
diff --git a/drivers/cpufreq/cpufreq-nforce2.c b/drivers/cpufreq/cpufreq-nforce2.c
index 5503d491b016..dbf82f36d270 100644
--- a/drivers/cpufreq/cpufreq-nforce2.c
+++ b/drivers/cpufreq/cpufreq-nforce2.c
@@ -357,7 +357,6 @@ static int nforce2_cpu_init(struct cpufreq_policy *policy)
 	/* cpuinfo and default policy values */
 	policy->min = policy->cpuinfo.min_freq = min_fsb * fid * 100;
 	policy->max = policy->cpuinfo.max_freq = max_fsb * fid * 100;
-	policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
 
 	return 0;
 }
@@ -369,6 +368,7 @@ static int nforce2_cpu_exit(struct cpufreq_policy *policy)
 
 static struct cpufreq_driver nforce2_driver = {
 	.name = "nforce2",
+	.flags = CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING,
 	.verify = nforce2_verify,
 	.target = nforce2_target,
 	.get = nforce2_get,
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index 9bf97a366029..ea43b147a7fe 100644
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -524,6 +524,32 @@ unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy,
 }
 EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq);
 
+unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy)
+{
+	unsigned int latency;
+
+	if (policy->transition_delay_us)
+		return policy->transition_delay_us;
+
+	latency = policy->cpuinfo.transition_latency / NSEC_PER_USEC;
+	if (latency) {
+		/*
+		 * For platforms that can change the frequency very fast (< 10
+		 * us), the above formula gives a decent transition delay. But
+		 * for platforms where transition_latency is in milliseconds, it
+		 * ends up giving unrealistic values.
+		 *
+		 * Cap the default transition delay to 10 ms, which seems to be
+		 * a reasonable amount of time after which we should reevaluate
+		 * the frequency.
+		 */
+		return min(latency * LATENCY_MULTIPLIER, (unsigned int)10000);
+	}
+
+	return LATENCY_MULTIPLIER;
+}
+EXPORT_SYMBOL_GPL(cpufreq_policy_transition_delay_us);
+
 /*********************************************************************
  *                          SYSFS INTERFACE                          *
  *********************************************************************/
@@ -1817,9 +1843,10 @@ EXPORT_SYMBOL(cpufreq_unregister_notifier);
  * twice in parallel for the same policy and that it will never be called in
  * parallel with either ->target() or ->target_index() for the same policy.
  *
- * If CPUFREQ_ENTRY_INVALID is returned by the driver's ->fast_switch()
- * callback to indicate an error condition, the hardware configuration must be
- * preserved.
+ * Returns the actual frequency set for the CPU.
+ *
+ * If 0 is returned by the driver's ->fast_switch() callback to indicate an
+ * error condition, the hardware configuration must be preserved.
  */
 unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy,
 					unsigned int target_freq)
@@ -1988,13 +2015,13 @@ static int cpufreq_init_governor(struct cpufreq_policy *policy)
 	if (!policy->governor)
 		return -EINVAL;
 
-	if (policy->governor->max_transition_latency &&
-	    policy->cpuinfo.transition_latency >
-	    policy->governor->max_transition_latency) {
+	/* Platform doesn't want dynamic frequency switching ? */
+	if (policy->governor->dynamic_switching &&
+	    cpufreq_driver->flags & CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING) {
 		struct cpufreq_governor *gov = cpufreq_fallback_governor();
 
 		if (gov) {
-			pr_warn("%s governor failed, too long transition latency of HW, fallback to %s governor\n",
+			pr_warn("Can't use %s governor as dynamic switching is disallowed. Fallback to %s governor\n",
 				policy->governor->name, gov->name);
 			policy->governor = gov;
 		} else {
diff --git a/drivers/cpufreq/cpufreq_conservative.c b/drivers/cpufreq/cpufreq_conservative.c
index 88220ff3e1c2..f20f20a77d4d 100644
--- a/drivers/cpufreq/cpufreq_conservative.c
+++ b/drivers/cpufreq/cpufreq_conservative.c
@@ -246,7 +246,6 @@ gov_show_one_common(sampling_rate);
 gov_show_one_common(sampling_down_factor);
 gov_show_one_common(up_threshold);
 gov_show_one_common(ignore_nice_load);
-gov_show_one_common(min_sampling_rate);
 gov_show_one(cs, down_threshold);
 gov_show_one(cs, freq_step);
 
@@ -254,12 +253,10 @@ gov_attr_rw(sampling_rate);
 gov_attr_rw(sampling_down_factor);
 gov_attr_rw(up_threshold);
 gov_attr_rw(ignore_nice_load);
-gov_attr_ro(min_sampling_rate);
 gov_attr_rw(down_threshold);
 gov_attr_rw(freq_step);
 
 static struct attribute *cs_attributes[] = {
-	&min_sampling_rate.attr,
 	&sampling_rate.attr,
 	&sampling_down_factor.attr,
 	&up_threshold.attr,
@@ -297,10 +294,7 @@ static int cs_init(struct dbs_data *dbs_data)
 	dbs_data->up_threshold = DEF_FREQUENCY_UP_THRESHOLD;
 	dbs_data->sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR;
 	dbs_data->ignore_nice_load = 0;
-
 	dbs_data->tuners = tuners;
-	dbs_data->min_sampling_rate = MIN_SAMPLING_RATE_RATIO *
-		jiffies_to_usecs(10);
 
 	return 0;
 }
diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c
index 47e24b5384b3..58d4f4e1ad6a 100644
--- a/drivers/cpufreq/cpufreq_governor.c
+++ b/drivers/cpufreq/cpufreq_governor.c
@@ -47,14 +47,11 @@ ssize_t store_sampling_rate(struct gov_attr_set *attr_set, const char *buf,
 {
 	struct dbs_data *dbs_data = to_dbs_data(attr_set);
 	struct policy_dbs_info *policy_dbs;
-	unsigned int rate;
 	int ret;
-	ret = sscanf(buf, "%u", &rate);
+	ret = sscanf(buf, "%u", &dbs_data->sampling_rate);
 	if (ret != 1)
 		return -EINVAL;
 
-	dbs_data->sampling_rate = max(rate, dbs_data->min_sampling_rate);
-
 	/*
 	 * We are operating under dbs_data->mutex and so the list and its
 	 * entries can't be freed concurrently.
@@ -275,6 +272,9 @@ static void dbs_update_util_handler(struct update_util_data *data, u64 time,
 	struct policy_dbs_info *policy_dbs = cdbs->policy_dbs;
 	u64 delta_ns, lst;
 
+	if (!cpufreq_can_do_remote_dvfs(policy_dbs->policy))
+		return;
+
 	/*
 	 * The work may not be allowed to be queued up right now.
 	 * Possible reasons:
@@ -392,7 +392,6 @@ int cpufreq_dbs_governor_init(struct cpufreq_policy *policy)
 	struct dbs_governor *gov = dbs_governor_of(policy);
 	struct dbs_data *dbs_data;
 	struct policy_dbs_info *policy_dbs;
-	unsigned int latency;
 	int ret = 0;
 
 	/* State should be equivalent to EXIT */
@@ -431,16 +430,7 @@ int cpufreq_dbs_governor_init(struct cpufreq_policy *policy)
 	if (ret)
 		goto free_policy_dbs_info;
 
-	/* policy latency is in ns. Convert it to us first */
-	latency = policy->cpuinfo.transition_latency / 1000;
-	if (latency == 0)
-		latency = 1;
-
-	/* Bring kernel and HW constraints together */
-	dbs_data->min_sampling_rate = max(dbs_data->min_sampling_rate,
-					  MIN_LATENCY_MULTIPLIER * latency);
-	dbs_data->sampling_rate = max(dbs_data->min_sampling_rate,
-				      LATENCY_MULTIPLIER * latency);
+	dbs_data->sampling_rate = cpufreq_policy_transition_delay_us(policy);
 
 	if (!have_governor_per_policy())
 		gov->gdbs_data = dbs_data;
diff --git a/drivers/cpufreq/cpufreq_governor.h b/drivers/cpufreq/cpufreq_governor.h
index 0236ec2cd654..8463f5def0f5 100644
--- a/drivers/cpufreq/cpufreq_governor.h
+++ b/drivers/cpufreq/cpufreq_governor.h
@@ -41,7 +41,6 @@ enum {OD_NORMAL_SAMPLE, OD_SUB_SAMPLE};
 struct dbs_data {
 	struct gov_attr_set attr_set;
 	void *tuners;
-	unsigned int min_sampling_rate;
 	unsigned int ignore_nice_load;
 	unsigned int sampling_rate;
 	unsigned int sampling_down_factor;
@@ -160,7 +159,7 @@ void cpufreq_dbs_governor_limits(struct cpufreq_policy *policy);
 #define CPUFREQ_DBS_GOVERNOR_INITIALIZER(_name_)			\
 	{								\
 		.name = _name_,						\
-		.max_transition_latency	= TRANSITION_LATENCY_LIMIT,	\
+		.dynamic_switching = true,				\
 		.owner = THIS_MODULE,					\
 		.init = cpufreq_dbs_governor_init,			\
 		.exit = cpufreq_dbs_governor_exit,			\
diff --git a/drivers/cpufreq/cpufreq_ondemand.c b/drivers/cpufreq/cpufreq_ondemand.c
index 3937acf7e026..6b423eebfd5d 100644
--- a/drivers/cpufreq/cpufreq_ondemand.c
+++ b/drivers/cpufreq/cpufreq_ondemand.c
@@ -319,7 +319,6 @@ gov_show_one_common(sampling_rate);
 gov_show_one_common(up_threshold);
 gov_show_one_common(sampling_down_factor);
 gov_show_one_common(ignore_nice_load);
-gov_show_one_common(min_sampling_rate);
 gov_show_one_common(io_is_busy);
 gov_show_one(od, powersave_bias);
 
@@ -329,10 +328,8 @@ gov_attr_rw(up_threshold);
 gov_attr_rw(sampling_down_factor);
 gov_attr_rw(ignore_nice_load);
 gov_attr_rw(powersave_bias);
-gov_attr_ro(min_sampling_rate);
 
 static struct attribute *od_attributes[] = {
-	&min_sampling_rate.attr,
 	&sampling_rate.attr,
 	&up_threshold.attr,
 	&sampling_down_factor.attr,
@@ -373,17 +370,8 @@ static int od_init(struct dbs_data *dbs_data)
 	if (idle_time != -1ULL) {
 		/* Idle micro accounting is supported. Use finer thresholds */
 		dbs_data->up_threshold = MICRO_FREQUENCY_UP_THRESHOLD;
-		/*
-		 * In nohz/micro accounting case we set the minimum frequency
-		 * not depending on HZ, but fixed (very low).
-		*/
-		dbs_data->min_sampling_rate = MICRO_FREQUENCY_MIN_SAMPLE_RATE;
 	} else {
 		dbs_data->up_threshold = DEF_FREQUENCY_UP_THRESHOLD;
-
-		/* For correct statistics, we need 10 ticks for each measure */
-		dbs_data->min_sampling_rate = MIN_SAMPLING_RATE_RATIO *
-			jiffies_to_usecs(10);
 	}
 
 	dbs_data->sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR;
diff --git a/drivers/cpufreq/dbx500-cpufreq.c b/drivers/cpufreq/dbx500-cpufreq.c
deleted file mode 100644
index 4ee0431579c1..000000000000
--- a/drivers/cpufreq/dbx500-cpufreq.c
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Copyright (C) STMicroelectronics 2009
- * Copyright (C) ST-Ericsson SA 2010-2012
- *
- * License Terms: GNU General Public License v2
- * Author: Sundar Iyer <sundar.iyer@stericsson.com>
- * Author: Martin Persson <martin.persson@stericsson.com>
- * Author: Jonas Aaberg <jonas.aberg@stericsson.com>
- */
-
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/cpufreq.h>
-#include <linux/cpu_cooling.h>
-#include <linux/delay.h>
-#include <linux/slab.h>
-#include <linux/platform_device.h>
-#include <linux/clk.h>
-
-static struct cpufreq_frequency_table *freq_table;
-static struct clk *armss_clk;
-static struct thermal_cooling_device *cdev;
-
-static int dbx500_cpufreq_target(struct cpufreq_policy *policy,
-				unsigned int index)
-{
-	/* update armss clk frequency */
-	return clk_set_rate(armss_clk, freq_table[index].frequency * 1000);
-}
-
-static int dbx500_cpufreq_init(struct cpufreq_policy *policy)
-{
-	policy->clk = armss_clk;
-	return cpufreq_generic_init(policy, freq_table, 20 * 1000);
-}
-
-static int dbx500_cpufreq_exit(struct cpufreq_policy *policy)
-{
-	if (!IS_ERR(cdev))
-		cpufreq_cooling_unregister(cdev);
-	return 0;
-}
-
-static void dbx500_cpufreq_ready(struct cpufreq_policy *policy)
-{
-	cdev = cpufreq_cooling_register(policy);
-	if (IS_ERR(cdev))
-		pr_err("Failed to register cooling device %ld\n", PTR_ERR(cdev));
-	else
-		pr_info("Cooling device registered: %s\n", cdev->type);
-}
-
-static struct cpufreq_driver dbx500_cpufreq_driver = {
-	.flags  = CPUFREQ_STICKY | CPUFREQ_CONST_LOOPS |
-			CPUFREQ_NEED_INITIAL_FREQ_CHECK,
-	.verify = cpufreq_generic_frequency_table_verify,
-	.target_index = dbx500_cpufreq_target,
-	.get    = cpufreq_generic_get,
-	.init   = dbx500_cpufreq_init,
-	.exit  = dbx500_cpufreq_exit,
-	.ready  = dbx500_cpufreq_ready,
-	.name   = "DBX500",
-	.attr   = cpufreq_generic_attr,
-};
-
-static int dbx500_cpufreq_probe(struct platform_device *pdev)
-{
-	struct cpufreq_frequency_table *pos;
-
-	freq_table = dev_get_platdata(&pdev->dev);
-	if (!freq_table) {
-		pr_err("dbx500-cpufreq: Failed to fetch cpufreq table\n");
-		return -ENODEV;
-	}
-
-	armss_clk = clk_get(&pdev->dev, "armss");
-	if (IS_ERR(armss_clk)) {
-		pr_err("dbx500-cpufreq: Failed to get armss clk\n");
-		return PTR_ERR(armss_clk);
-	}
-
-	pr_info("dbx500-cpufreq: Available frequencies:\n");
-	cpufreq_for_each_entry(pos, freq_table)
-		pr_info("  %d Mhz\n", pos->frequency / 1000);
-
-	return cpufreq_register_driver(&dbx500_cpufreq_driver);
-}
-
-static struct platform_driver dbx500_cpufreq_plat_driver = {
-	.driver = {
-		.name = "cpufreq-ux500",
-	},
-	.probe = dbx500_cpufreq_probe,
-};
-
-static int __init dbx500_cpufreq_register(void)
-{
-	return platform_driver_register(&dbx500_cpufreq_plat_driver);
-}
-device_initcall(dbx500_cpufreq_register);
-
-MODULE_LICENSE("GPL v2");
-MODULE_DESCRIPTION("cpufreq driver for DBX500");
diff --git a/drivers/cpufreq/elanfreq.c b/drivers/cpufreq/elanfreq.c
index bfce11cba1df..45e2ca62515e 100644
--- a/drivers/cpufreq/elanfreq.c
+++ b/drivers/cpufreq/elanfreq.c
@@ -165,9 +165,6 @@ static int elanfreq_cpu_init(struct cpufreq_policy *policy)
 		if (pos->frequency > max_freq)
 			pos->frequency = CPUFREQ_ENTRY_INVALID;
 
-	/* cpuinfo and default policy values */
-	policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
-
 	return cpufreq_table_validate_and_show(policy, elanfreq_table);
 }
 
@@ -196,6 +193,7 @@ __setup("elanfreq=", elanfreq_setup);
 
 static struct cpufreq_driver elanfreq_driver = {
 	.get		= elanfreq_get_cpu_frequency,
+	.flags		= CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING,
 	.verify		= cpufreq_generic_frequency_table_verify,
 	.target_index	= elanfreq_target,
 	.init		= elanfreq_cpu_init,
diff --git a/drivers/cpufreq/gx-suspmod.c b/drivers/cpufreq/gx-suspmod.c
index 3488c9c175eb..8f52a06664e3 100644
--- a/drivers/cpufreq/gx-suspmod.c
+++ b/drivers/cpufreq/gx-suspmod.c
@@ -428,7 +428,6 @@ static int cpufreq_gx_cpu_init(struct cpufreq_policy *policy)
 	policy->max = maxfreq;
 	policy->cpuinfo.min_freq = maxfreq / max_duration;
 	policy->cpuinfo.max_freq = maxfreq;
-	policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
 
 	return 0;
 }
@@ -438,6 +437,7 @@ static int cpufreq_gx_cpu_init(struct cpufreq_policy *policy)
  *   MediaGX/Geode GX initialize cpufreq driver
  */
 static struct cpufreq_driver gx_suspmod_driver = {
+	.flags		= CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING,
 	.get		= gx_get_cpuspeed,
 	.verify		= cpufreq_gx_verify,
 	.target		= cpufreq_gx_target,
diff --git a/drivers/cpufreq/imx6q-cpufreq.c b/drivers/cpufreq/imx6q-cpufreq.c
index b6edd3ccaa55..14466a9b01c0 100644
--- a/drivers/cpufreq/imx6q-cpufreq.c
+++ b/drivers/cpufreq/imx6q-cpufreq.c
@@ -47,6 +47,7 @@ static int imx6q_set_target(struct cpufreq_policy *policy, unsigned int index)
 	struct dev_pm_opp *opp;
 	unsigned long freq_hz, volt, volt_old;
 	unsigned int old_freq, new_freq;
+	bool pll1_sys_temp_enabled = false;
 	int ret;
 
 	new_freq = freq_table[index].frequency;
@@ -124,6 +125,10 @@ static int imx6q_set_target(struct cpufreq_policy *policy, unsigned int index)
 		if (freq_hz > clk_get_rate(pll2_pfd2_396m_clk)) {
 			clk_set_rate(pll1_sys_clk, new_freq * 1000);
 			clk_set_parent(pll1_sw_clk, pll1_sys_clk);
+		} else {
+			/* pll1_sys needs to be enabled for divider rate change to work. */
+			pll1_sys_temp_enabled = true;
+			clk_prepare_enable(pll1_sys_clk);
 		}
 	}
 
@@ -135,6 +140,10 @@ static int imx6q_set_target(struct cpufreq_policy *policy, unsigned int index)
 		return ret;
 	}
 
+	/* PLL1 is only needed until after ARM-PODF is set. */
+	if (pll1_sys_temp_enabled)
+		clk_disable_unprepare(pll1_sys_clk);
+
 	/* scaling down?  scale voltage after frequency */
 	if (new_freq < old_freq) {
 		ret = regulator_set_voltage_tol(arm_reg, volt, 0);
diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index 0566455f233e..93a0e88bef76 100644
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -37,8 +37,7 @@
 #include <asm/cpufeature.h>
 #include <asm/intel-family.h>
 
-#define INTEL_PSTATE_DEFAULT_SAMPLING_INTERVAL	(10 * NSEC_PER_MSEC)
-#define INTEL_PSTATE_HWP_SAMPLING_INTERVAL	(50 * NSEC_PER_MSEC)
+#define INTEL_PSTATE_SAMPLING_INTERVAL	(10 * NSEC_PER_MSEC)
 
 #define INTEL_CPUFREQ_TRANSITION_LATENCY	20000
 #define INTEL_CPUFREQ_TRANSITION_DELAY		500
@@ -173,28 +172,6 @@ struct vid_data {
 };
 
 /**
- * struct _pid -	Stores PID data
- * @setpoint:		Target set point for busyness or performance
- * @integral:		Storage for accumulated error values
- * @p_gain:		PID proportional gain
- * @i_gain:		PID integral gain
- * @d_gain:		PID derivative gain
- * @deadband:		PID deadband
- * @last_err:		Last error storage for integral part of PID calculation
- *
- * Stores PID coefficients and last error for PID controller.
- */
-struct _pid {
-	int setpoint;
-	int32_t integral;
-	int32_t p_gain;
-	int32_t i_gain;
-	int32_t d_gain;
-	int deadband;
-	int32_t last_err;
-};
-
-/**
  * struct global_params - Global parameters, mostly tunable via sysfs.
  * @no_turbo:		Whether or not to use turbo P-states.
  * @turbo_disabled:	Whethet or not turbo P-states are available at all,
@@ -223,7 +200,6 @@ struct global_params {
  * @last_update:	Time of the last update.
  * @pstate:		Stores P state limits for this CPU
  * @vid:		Stores VID limits for this CPU
- * @pid:		Stores PID parameters for this CPU
  * @last_sample_time:	Last Sample time
  * @aperf_mperf_shift:	Number of clock cycles after aperf, merf is incremented
  *			This shift is a multiplier to mperf delta to
@@ -258,7 +234,6 @@ struct cpudata {
 
 	struct pstate_data pstate;
 	struct vid_data vid;
-	struct _pid pid;
 
 	u64	last_update;
 	u64	last_sample_time;
@@ -284,28 +259,6 @@ struct cpudata {
 static struct cpudata **all_cpu_data;
 
 /**
- * struct pstate_adjust_policy - Stores static PID configuration data
- * @sample_rate_ms:	PID calculation sample rate in ms
- * @sample_rate_ns:	Sample rate calculation in ns
- * @deadband:		PID deadband
- * @setpoint:		PID Setpoint
- * @p_gain_pct:		PID proportional gain
- * @i_gain_pct:		PID integral gain
- * @d_gain_pct:		PID derivative gain
- *
- * Stores per CPU model static PID configuration data.
- */
-struct pstate_adjust_policy {
-	int sample_rate_ms;
-	s64 sample_rate_ns;
-	int deadband;
-	int setpoint;
-	int p_gain_pct;
-	int d_gain_pct;
-	int i_gain_pct;
-};
-
-/**
  * struct pstate_funcs - Per CPU model specific callbacks
  * @get_max:		Callback to get maximum non turbo effective P state
  * @get_max_physical:	Callback to get maximum non turbo physical P state
@@ -314,7 +267,6 @@ struct pstate_adjust_policy {
  * @get_scaling:	Callback to get frequency scaling factor
  * @get_val:		Callback to convert P state to actual MSR write value
  * @get_vid:		Callback to get VID data for Atom platforms
- * @update_util:	Active mode utilization update callback.
  *
  * Core and Atom CPU models have different way to get P State limits. This
  * structure is used to store those callbacks.
@@ -328,20 +280,9 @@ struct pstate_funcs {
 	int (*get_aperf_mperf_shift)(void);
 	u64 (*get_val)(struct cpudata*, int pstate);
 	void (*get_vid)(struct cpudata *);
-	void (*update_util)(struct update_util_data *data, u64 time,
-			    unsigned int flags);
 };
 
 static struct pstate_funcs pstate_funcs __read_mostly;
-static struct pstate_adjust_policy pid_params __read_mostly = {
-	.sample_rate_ms = 10,
-	.sample_rate_ns = 10 * NSEC_PER_MSEC,
-	.deadband = 0,
-	.setpoint = 97,
-	.p_gain_pct = 20,
-	.d_gain_pct = 0,
-	.i_gain_pct = 0,
-};
 
 static int hwp_active __read_mostly;
 static bool per_cpu_limits __read_mostly;
@@ -509,56 +450,6 @@ static inline void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
 }
 #endif
 
-static signed int pid_calc(struct _pid *pid, int32_t busy)
-{
-	signed int result;
-	int32_t pterm, dterm, fp_error;
-	int32_t integral_limit;
-
-	fp_error = pid->setpoint - busy;
-
-	if (abs(fp_error) <= pid->deadband)
-		return 0;
-
-	pterm = mul_fp(pid->p_gain, fp_error);
-
-	pid->integral += fp_error;
-
-	/*
-	 * We limit the integral here so that it will never
-	 * get higher than 30.  This prevents it from becoming
-	 * too large an input over long periods of time and allows
-	 * it to get factored out sooner.
-	 *
-	 * The value of 30 was chosen through experimentation.
-	 */
-	integral_limit = int_tofp(30);
-	if (pid->integral > integral_limit)
-		pid->integral = integral_limit;
-	if (pid->integral < -integral_limit)
-		pid->integral = -integral_limit;
-
-	dterm = mul_fp(pid->d_gain, fp_error - pid->last_err);
-	pid->last_err = fp_error;
-
-	result = pterm + mul_fp(pid->integral, pid->i_gain) + dterm;
-	result = result + (1 << (FRAC_BITS-1));
-	return (signed int)fp_toint(result);
-}
-
-static inline void intel_pstate_pid_reset(struct cpudata *cpu)
-{
-	struct _pid *pid = &cpu->pid;
-
-	pid->p_gain = percent_fp(pid_params.p_gain_pct);
-	pid->d_gain = percent_fp(pid_params.d_gain_pct);
-	pid->i_gain = percent_fp(pid_params.i_gain_pct);
-	pid->setpoint = int_tofp(pid_params.setpoint);
-	pid->last_err  = pid->setpoint - int_tofp(100);
-	pid->deadband  = int_tofp(pid_params.deadband);
-	pid->integral  = 0;
-}
-
 static inline void update_turbo_state(void)
 {
 	u64 misc_en;
@@ -911,82 +802,6 @@ static void intel_pstate_update_policies(void)
 		cpufreq_update_policy(cpu);
 }
 
-/************************** debugfs begin ************************/
-static int pid_param_set(void *data, u64 val)
-{
-	unsigned int cpu;
-
-	*(u32 *)data = val;
-	pid_params.sample_rate_ns = pid_params.sample_rate_ms * NSEC_PER_MSEC;
-	for_each_possible_cpu(cpu)
-		if (all_cpu_data[cpu])
-			intel_pstate_pid_reset(all_cpu_data[cpu]);
-
-	return 0;
-}
-
-static int pid_param_get(void *data, u64 *val)
-{
-	*val = *(u32 *)data;
-	return 0;
-}
-DEFINE_SIMPLE_ATTRIBUTE(fops_pid_param, pid_param_get, pid_param_set, "%llu\n");
-
-static struct dentry *debugfs_parent;
-
-struct pid_param {
-	char *name;
-	void *value;
-	struct dentry *dentry;
-};
-
-static struct pid_param pid_files[] = {
-	{"sample_rate_ms", &pid_params.sample_rate_ms, },
-	{"d_gain_pct", &pid_params.d_gain_pct, },
-	{"i_gain_pct", &pid_params.i_gain_pct, },
-	{"deadband", &pid_params.deadband, },
-	{"setpoint", &pid_params.setpoint, },
-	{"p_gain_pct", &pid_params.p_gain_pct, },
-	{NULL, NULL, }
-};
-
-static void intel_pstate_debug_expose_params(void)
-{
-	int i;
-
-	debugfs_parent = debugfs_create_dir("pstate_snb", NULL);
-	if (IS_ERR_OR_NULL(debugfs_parent))
-		return;
-
-	for (i = 0; pid_files[i].name; i++) {
-		struct dentry *dentry;
-
-		dentry = debugfs_create_file(pid_files[i].name, 0660,
-					     debugfs_parent, pid_files[i].value,
-					     &fops_pid_param);
-		if (!IS_ERR(dentry))
-			pid_files[i].dentry = dentry;
-	}
-}
-
-static void intel_pstate_debug_hide_params(void)
-{
-	int i;
-
-	if (IS_ERR_OR_NULL(debugfs_parent))
-		return;
-
-	for (i = 0; pid_files[i].name; i++) {
-		debugfs_remove(pid_files[i].dentry);
-		pid_files[i].dentry = NULL;
-	}
-
-	debugfs_remove(debugfs_parent);
-	debugfs_parent = NULL;
-}
-
-/************************** debugfs end ************************/
-
 /************************** sysfs begin ************************/
 #define show_one(file_name, object)					\
 	static ssize_t show_##file_name					\
@@ -1613,8 +1428,7 @@ static inline bool intel_pstate_sample(struct cpudata *cpu, u64 time)
 
 static inline int32_t get_avg_frequency(struct cpudata *cpu)
 {
-	return mul_ext_fp(cpu->sample.core_avg_perf,
-			  cpu->pstate.max_pstate_physical * cpu->pstate.scaling);
+	return mul_ext_fp(cpu->sample.core_avg_perf, cpu_khz);
 }
 
 static inline int32_t get_avg_pstate(struct cpudata *cpu)
@@ -1623,7 +1437,7 @@ static inline int32_t get_avg_pstate(struct cpudata *cpu)
 			  cpu->sample.core_avg_perf);
 }
 
-static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu)
+static inline int32_t get_target_pstate(struct cpudata *cpu)
 {
 	struct sample *sample = &cpu->sample;
 	int32_t busy_frac, boost;
@@ -1661,44 +1475,6 @@ static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu)
 	return target;
 }
 
-static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu)
-{
-	int32_t perf_scaled, max_pstate, current_pstate, sample_ratio;
-	u64 duration_ns;
-
-	/*
-	 * perf_scaled is the ratio of the average P-state during the last
-	 * sampling period to the P-state requested last time (in percent).
-	 *
-	 * That measures the system's response to the previous P-state
-	 * selection.
-	 */
-	max_pstate = cpu->pstate.max_pstate_physical;
-	current_pstate = cpu->pstate.current_pstate;
-	perf_scaled = mul_ext_fp(cpu->sample.core_avg_perf,
-			       div_fp(100 * max_pstate, current_pstate));
-
-	/*
-	 * Since our utilization update callback will not run unless we are
-	 * in C0, check if the actual elapsed time is significantly greater (3x)
-	 * than our sample interval.  If it is, then we were idle for a long
-	 * enough period of time to adjust our performance metric.
-	 */
-	duration_ns = cpu->sample.time - cpu->last_sample_time;
-	if ((s64)duration_ns > pid_params.sample_rate_ns * 3) {
-		sample_ratio = div_fp(pid_params.sample_rate_ns, duration_ns);
-		perf_scaled = mul_fp(perf_scaled, sample_ratio);
-	} else {
-		sample_ratio = div_fp(100 * (cpu->sample.mperf << cpu->aperf_mperf_shift),
-				      cpu->sample.tsc);
-		if (sample_ratio < int_tofp(1))
-			perf_scaled = 0;
-	}
-
-	cpu->sample.busy_scaled = perf_scaled;
-	return cpu->pstate.current_pstate - pid_calc(&cpu->pid, perf_scaled);
-}
-
 static int intel_pstate_prepare_request(struct cpudata *cpu, int pstate)
 {
 	int max_pstate = intel_pstate_get_base_pstate(cpu);
@@ -1718,13 +1494,15 @@ static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate)
 	wrmsrl(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate));
 }
 
-static void intel_pstate_adjust_pstate(struct cpudata *cpu, int target_pstate)
+static void intel_pstate_adjust_pstate(struct cpudata *cpu)
 {
 	int from = cpu->pstate.current_pstate;
 	struct sample *sample;
+	int target_pstate;
 
 	update_turbo_state();
 
+	target_pstate = get_target_pstate(cpu);
 	target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
 	trace_cpu_frequency(target_pstate * cpu->pstate.scaling, cpu->cpu);
 	intel_pstate_update_pstate(cpu, target_pstate);
@@ -1741,31 +1519,27 @@ static void intel_pstate_adjust_pstate(struct cpudata *cpu, int target_pstate)
 		fp_toint(cpu->iowait_boost * 100));
 }
 
-static void intel_pstate_update_util_pid(struct update_util_data *data,
-					 u64 time, unsigned int flags)
-{
-	struct cpudata *cpu = container_of(data, struct cpudata, update_util);
-	u64 delta_ns = time - cpu->sample.time;
-
-	if ((s64)delta_ns < pid_params.sample_rate_ns)
-		return;
-
-	if (intel_pstate_sample(cpu, time)) {
-		int target_pstate;
-
-		target_pstate = get_target_pstate_use_performance(cpu);
-		intel_pstate_adjust_pstate(cpu, target_pstate);
-	}
-}
-
 static void intel_pstate_update_util(struct update_util_data *data, u64 time,
 				     unsigned int flags)
 {
 	struct cpudata *cpu = container_of(data, struct cpudata, update_util);
 	u64 delta_ns;
 
+	/* Don't allow remote callbacks */
+	if (smp_processor_id() != cpu->cpu)
+		return;
+
 	if (flags & SCHED_CPUFREQ_IOWAIT) {
 		cpu->iowait_boost = int_tofp(1);
+		cpu->last_update = time;
+		/*
+		 * The last time the busy was 100% so P-state was max anyway
+		 * so avoid overhead of computation.
+		 */
+		if (fp_toint(cpu->sample.busy_scaled) == 100)
+			return;
+
+		goto set_pstate;
 	} else if (cpu->iowait_boost) {
 		/* Clear iowait_boost if the CPU may have been idle. */
 		delta_ns = time - cpu->last_update;
@@ -1774,15 +1548,12 @@ static void intel_pstate_update_util(struct update_util_data *data, u64 time,
 	}
 	cpu->last_update = time;
 	delta_ns = time - cpu->sample.time;
-	if ((s64)delta_ns < INTEL_PSTATE_DEFAULT_SAMPLING_INTERVAL)
+	if ((s64)delta_ns < INTEL_PSTATE_SAMPLING_INTERVAL)
 		return;
 
-	if (intel_pstate_sample(cpu, time)) {
-		int target_pstate;
-
-		target_pstate = get_target_pstate_use_cpu_load(cpu);
-		intel_pstate_adjust_pstate(cpu, target_pstate);
-	}
+set_pstate:
+	if (intel_pstate_sample(cpu, time))
+		intel_pstate_adjust_pstate(cpu);
 }
 
 static struct pstate_funcs core_funcs = {
@@ -1792,7 +1563,6 @@ static struct pstate_funcs core_funcs = {
 	.get_turbo = core_get_turbo_pstate,
 	.get_scaling = core_get_scaling,
 	.get_val = core_get_val,
-	.update_util = intel_pstate_update_util_pid,
 };
 
 static const struct pstate_funcs silvermont_funcs = {
@@ -1803,7 +1573,6 @@ static const struct pstate_funcs silvermont_funcs = {
 	.get_val = atom_get_val,
 	.get_scaling = silvermont_get_scaling,
 	.get_vid = atom_get_vid,
-	.update_util = intel_pstate_update_util,
 };
 
 static const struct pstate_funcs airmont_funcs = {
@@ -1814,7 +1583,6 @@ static const struct pstate_funcs airmont_funcs = {
 	.get_val = atom_get_val,
 	.get_scaling = airmont_get_scaling,
 	.get_vid = atom_get_vid,
-	.update_util = intel_pstate_update_util,
 };
 
 static const struct pstate_funcs knl_funcs = {
@@ -1825,7 +1593,6 @@ static const struct pstate_funcs knl_funcs = {
 	.get_aperf_mperf_shift = knl_get_aperf_mperf_shift,
 	.get_scaling = core_get_scaling,
 	.get_val = core_get_val,
-	.update_util = intel_pstate_update_util_pid,
 };
 
 static const struct pstate_funcs bxt_funcs = {
@@ -1835,7 +1602,6 @@ static const struct pstate_funcs bxt_funcs = {
 	.get_turbo = core_get_turbo_pstate,
 	.get_scaling = core_get_scaling,
 	.get_val = core_get_val,
-	.update_util = intel_pstate_update_util,
 };
 
 #define ICPU(model, policy) \
@@ -1879,8 +1645,6 @@ static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[] = {
 	{}
 };
 
-static bool pid_in_use(void);
-
 static int intel_pstate_init_cpu(unsigned int cpunum)
 {
 	struct cpudata *cpu;
@@ -1911,8 +1675,6 @@ static int intel_pstate_init_cpu(unsigned int cpunum)
 			intel_pstate_disable_ee(cpunum);
 
 		intel_pstate_hwp_enable(cpu);
-	} else if (pid_in_use()) {
-		intel_pstate_pid_reset(cpu);
 	}
 
 	intel_pstate_get_cpu_pstates(cpu);
@@ -1935,7 +1697,7 @@ static void intel_pstate_set_update_util_hook(unsigned int cpu_num)
 	/* Prevent intel_pstate_update_util() from using stale data. */
 	cpu->sample.time = 0;
 	cpufreq_add_update_util_hook(cpu_num, &cpu->update_util,
-				     pstate_funcs.update_util);
+				     intel_pstate_update_util);
 	cpu->update_util_set = true;
 }
 
@@ -2133,7 +1895,6 @@ static int __intel_pstate_cpu_init(struct cpufreq_policy *policy)
 	policy->cpuinfo.max_freq *= cpu->pstate.scaling;
 
 	intel_pstate_init_acpi_perf_limits(policy);
-	cpumask_set_cpu(policy->cpu, policy->cpus);
 
 	policy->fast_switch_possible = true;
 
@@ -2147,7 +1908,6 @@ static int intel_pstate_cpu_init(struct cpufreq_policy *policy)
 	if (ret)
 		return ret;
 
-	policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
 	if (IS_ENABLED(CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE))
 		policy->policy = CPUFREQ_POLICY_PERFORMANCE;
 	else
@@ -2262,12 +2022,6 @@ static struct cpufreq_driver intel_cpufreq = {
 
 static struct cpufreq_driver *default_driver = &intel_pstate;
 
-static bool pid_in_use(void)
-{
-	return intel_pstate_driver == &intel_pstate &&
-		pstate_funcs.update_util == intel_pstate_update_util_pid;
-}
-
 static void intel_pstate_driver_cleanup(void)
 {
 	unsigned int cpu;
@@ -2302,9 +2056,6 @@ static int intel_pstate_register_driver(struct cpufreq_driver *driver)
 
 	global.min_perf_pct = min_perf_pct_min();
 
-	if (pid_in_use())
-		intel_pstate_debug_expose_params();
-
 	return 0;
 }
 
@@ -2313,9 +2064,6 @@ static int intel_pstate_unregister_driver(void)
 	if (hwp_active)
 		return -EBUSY;
 
-	if (pid_in_use())
-		intel_pstate_debug_hide_params();
-
 	cpufreq_unregister_driver(intel_pstate_driver);
 	intel_pstate_driver_cleanup();
 
@@ -2383,24 +2131,6 @@ static int __init intel_pstate_msrs_not_valid(void)
 	return 0;
 }
 
-#ifdef CONFIG_ACPI
-static void intel_pstate_use_acpi_profile(void)
-{
-	switch (acpi_gbl_FADT.preferred_profile) {
-	case PM_MOBILE:
-	case PM_TABLET:
-	case PM_APPLIANCE_PC:
-	case PM_DESKTOP:
-	case PM_WORKSTATION:
-		pstate_funcs.update_util = intel_pstate_update_util;
-	}
-}
-#else
-static void intel_pstate_use_acpi_profile(void)
-{
-}
-#endif
-
 static void __init copy_cpu_funcs(struct pstate_funcs *funcs)
 {
 	pstate_funcs.get_max   = funcs->get_max;
@@ -2410,10 +2140,7 @@ static void __init copy_cpu_funcs(struct pstate_funcs *funcs)
 	pstate_funcs.get_scaling = funcs->get_scaling;
 	pstate_funcs.get_val   = funcs->get_val;
 	pstate_funcs.get_vid   = funcs->get_vid;
-	pstate_funcs.update_util = funcs->update_util;
 	pstate_funcs.get_aperf_mperf_shift = funcs->get_aperf_mperf_shift;
-
-	intel_pstate_use_acpi_profile();
 }
 
 #ifdef CONFIG_ACPI
@@ -2467,39 +2194,31 @@ enum {
 	PPC,
 };
 
-struct hw_vendor_info {
-	u16  valid;
-	char oem_id[ACPI_OEM_ID_SIZE];
-	char oem_table_id[ACPI_OEM_TABLE_ID_SIZE];
-	int  oem_pwr_table;
-};
-
 /* Hardware vendor-specific info that has its own power management modes */
-static struct hw_vendor_info vendor_info[] __initdata = {
-	{1, "HP    ", "ProLiant", PSS},
-	{1, "ORACLE", "X4-2    ", PPC},
-	{1, "ORACLE", "X4-2L   ", PPC},
-	{1, "ORACLE", "X4-2B   ", PPC},
-	{1, "ORACLE", "X3-2    ", PPC},
-	{1, "ORACLE", "X3-2L   ", PPC},
-	{1, "ORACLE", "X3-2B   ", PPC},
-	{1, "ORACLE", "X4470M2 ", PPC},
-	{1, "ORACLE", "X4270M3 ", PPC},
-	{1, "ORACLE", "X4270M2 ", PPC},
-	{1, "ORACLE", "X4170M2 ", PPC},
-	{1, "ORACLE", "X4170 M3", PPC},
-	{1, "ORACLE", "X4275 M3", PPC},
-	{1, "ORACLE", "X6-2    ", PPC},
-	{1, "ORACLE", "Sudbury ", PPC},
-	{0, "", ""},
+static struct acpi_platform_list plat_info[] __initdata = {
+	{"HP    ", "ProLiant", 0, ACPI_SIG_FADT, all_versions, 0, PSS},
+	{"ORACLE", "X4-2    ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
+	{"ORACLE", "X4-2L   ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
+	{"ORACLE", "X4-2B   ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
+	{"ORACLE", "X3-2    ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
+	{"ORACLE", "X3-2L   ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
+	{"ORACLE", "X3-2B   ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
+	{"ORACLE", "X4470M2 ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
+	{"ORACLE", "X4270M3 ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
+	{"ORACLE", "X4270M2 ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
+	{"ORACLE", "X4170M2 ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
+	{"ORACLE", "X4170 M3", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
+	{"ORACLE", "X4275 M3", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
+	{"ORACLE", "X6-2    ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
+	{"ORACLE", "Sudbury ", 0, ACPI_SIG_FADT, all_versions, 0, PPC},
+	{ } /* End */
 };
 
 static bool __init intel_pstate_platform_pwr_mgmt_exists(void)
 {
-	struct acpi_table_header hdr;
-	struct hw_vendor_info *v_info;
 	const struct x86_cpu_id *id;
 	u64 misc_pwr;
+	int idx;
 
 	id = x86_match_cpu(intel_pstate_cpu_oob_ids);
 	if (id) {
@@ -2508,21 +2227,15 @@ static bool __init intel_pstate_platform_pwr_mgmt_exists(void)
 			return true;
 	}
 
-	if (acpi_disabled ||
-	    ACPI_FAILURE(acpi_get_table_header(ACPI_SIG_FADT, 0, &hdr)))
+	idx = acpi_match_platform_list(plat_info);
+	if (idx < 0)
 		return false;
 
-	for (v_info = vendor_info; v_info->valid; v_info++) {
-		if (!strncmp(hdr.oem_id, v_info->oem_id, ACPI_OEM_ID_SIZE) &&
-			!strncmp(hdr.oem_table_id, v_info->oem_table_id,
-						ACPI_OEM_TABLE_ID_SIZE))
-			switch (v_info->oem_pwr_table) {
-			case PSS:
-				return intel_pstate_no_acpi_pss();
-			case PPC:
-				return intel_pstate_has_acpi_ppc() &&
-					(!force_load);
-			}
+	switch (plat_info[idx].data) {
+	case PSS:
+		return intel_pstate_no_acpi_pss();
+	case PPC:
+		return intel_pstate_has_acpi_ppc() && !force_load;
 	}
 
 	return false;
@@ -2557,9 +2270,7 @@ static int __init intel_pstate_init(void)
 
 	if (x86_match_cpu(hwp_support_ids)) {
 		copy_cpu_funcs(&core_funcs);
-		if (no_hwp) {
-			pstate_funcs.update_util = intel_pstate_update_util;
-		} else {
+		if (!no_hwp) {
 			hwp_active++;
 			intel_pstate.attr = hwp_cpufreq_attrs;
 			goto hwp_cpu_matched;
diff --git a/drivers/cpufreq/longrun.c b/drivers/cpufreq/longrun.c
index 074971b12635..542aa9adba1a 100644
--- a/drivers/cpufreq/longrun.c
+++ b/drivers/cpufreq/longrun.c
@@ -270,7 +270,6 @@ static int longrun_cpu_init(struct cpufreq_policy *policy)
 	/* cpuinfo and default policy values */
 	policy->cpuinfo.min_freq = longrun_low_freq;
 	policy->cpuinfo.max_freq = longrun_high_freq;
-	policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
 	longrun_get_policy(policy);
 
 	return 0;
diff --git a/drivers/cpufreq/loongson2_cpufreq.c b/drivers/cpufreq/loongson2_cpufreq.c
index 9ac27b22476c..da344696beed 100644
--- a/drivers/cpufreq/loongson2_cpufreq.c
+++ b/drivers/cpufreq/loongson2_cpufreq.c
@@ -114,7 +114,7 @@ static struct cpufreq_driver loongson2_cpufreq_driver = {
 	.attr = cpufreq_generic_attr,
 };
 
-static struct platform_device_id platform_device_ids[] = {
+static const struct platform_device_id platform_device_ids[] = {
 	{
 		.name = "loongson2_cpufreq",
 	},
diff --git a/drivers/cpufreq/mt8173-cpufreq.c b/drivers/cpufreq/mediatek-cpufreq.c
index f9f00fb4bc3a..18c4bd9a5c65 100644
--- a/drivers/cpufreq/mt8173-cpufreq.c
+++ b/drivers/cpufreq/mediatek-cpufreq.c
@@ -507,7 +507,7 @@ static int mtk_cpufreq_exit(struct cpufreq_policy *policy)
 	return 0;
 }
 
-static struct cpufreq_driver mt8173_cpufreq_driver = {
+static struct cpufreq_driver mtk_cpufreq_driver = {
 	.flags = CPUFREQ_STICKY | CPUFREQ_NEED_INITIAL_FREQ_CHECK |
 		 CPUFREQ_HAVE_GOVERNOR_PER_POLICY,
 	.verify = cpufreq_generic_frequency_table_verify,
@@ -520,7 +520,7 @@ static struct cpufreq_driver mt8173_cpufreq_driver = {
 	.attr = cpufreq_generic_attr,
 };
 
-static int mt8173_cpufreq_probe(struct platform_device *pdev)
+static int mtk_cpufreq_probe(struct platform_device *pdev)
 {
 	struct mtk_cpu_dvfs_info *info, *tmp;
 	int cpu, ret;
@@ -547,7 +547,7 @@ static int mt8173_cpufreq_probe(struct platform_device *pdev)
 		list_add(&info->list_head, &dvfs_info_list);
 	}
 
-	ret = cpufreq_register_driver(&mt8173_cpufreq_driver);
+	ret = cpufreq_register_driver(&mtk_cpufreq_driver);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to register mtk cpufreq driver\n");
 		goto release_dvfs_info_list;
@@ -564,15 +564,18 @@ release_dvfs_info_list:
 	return ret;
 }
 
-static struct platform_driver mt8173_cpufreq_platdrv = {
+static struct platform_driver mtk_cpufreq_platdrv = {
 	.driver = {
-		.name	= "mt8173-cpufreq",
+		.name	= "mtk-cpufreq",
 	},
-	.probe		= mt8173_cpufreq_probe,
+	.probe		= mtk_cpufreq_probe,
 };
 
 /* List of machines supported by this driver */
-static const struct of_device_id mt8173_cpufreq_machines[] __initconst = {
+static const struct of_device_id mtk_cpufreq_machines[] __initconst = {
+	{ .compatible = "mediatek,mt2701", },
+	{ .compatible = "mediatek,mt7622", },
+	{ .compatible = "mediatek,mt7623", },
 	{ .compatible = "mediatek,mt817x", },
 	{ .compatible = "mediatek,mt8173", },
 	{ .compatible = "mediatek,mt8176", },
@@ -580,7 +583,7 @@ static const struct of_device_id mt8173_cpufreq_machines[] __initconst = {
 	{ }
 };
 
-static int __init mt8173_cpufreq_driver_init(void)
+static int __init mtk_cpufreq_driver_init(void)
 {
 	struct device_node *np;
 	const struct of_device_id *match;
@@ -591,14 +594,14 @@ static int __init mt8173_cpufreq_driver_init(void)
 	if (!np)
 		return -ENODEV;
 
-	match = of_match_node(mt8173_cpufreq_machines, np);
+	match = of_match_node(mtk_cpufreq_machines, np);
 	of_node_put(np);
 	if (!match) {
-		pr_warn("Machine is not compatible with mt8173-cpufreq\n");
+		pr_warn("Machine is not compatible with mtk-cpufreq\n");
 		return -ENODEV;
 	}
 
-	err = platform_driver_register(&mt8173_cpufreq_platdrv);
+	err = platform_driver_register(&mtk_cpufreq_platdrv);
 	if (err)
 		return err;
 
@@ -608,7 +611,7 @@ static int __init mt8173_cpufreq_driver_init(void)
 	 * and the device registration codes are put here to handle defer
 	 * probing.
 	 */
-	pdev = platform_device_register_simple("mt8173-cpufreq", -1, NULL, 0);
+	pdev = platform_device_register_simple("mtk-cpufreq", -1, NULL, 0);
 	if (IS_ERR(pdev)) {
 		pr_err("failed to register mtk-cpufreq platform device\n");
 		return PTR_ERR(pdev);
@@ -616,4 +619,4 @@ static int __init mt8173_cpufreq_driver_init(void)
 
 	return 0;
 }
-device_initcall(mt8173_cpufreq_driver_init);
+device_initcall(mtk_cpufreq_driver_init);
diff --git a/drivers/cpufreq/pmac32-cpufreq.c b/drivers/cpufreq/pmac32-cpufreq.c
index ff44016ea031..61ae06ca008e 100644
--- a/drivers/cpufreq/pmac32-cpufreq.c
+++ b/drivers/cpufreq/pmac32-cpufreq.c
@@ -442,7 +442,8 @@ static struct cpufreq_driver pmac_cpufreq_driver = {
 	.init		= pmac_cpufreq_cpu_init,
 	.suspend	= pmac_cpufreq_suspend,
 	.resume		= pmac_cpufreq_resume,
-	.flags		= CPUFREQ_PM_NO_WARN,
+	.flags		= CPUFREQ_PM_NO_WARN |
+			  CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING,
 	.attr		= cpufreq_generic_attr,
 	.name		= "powermac",
 };
@@ -626,14 +627,16 @@ static int __init pmac_cpufreq_setup(void)
 	if (!value)
 		goto out;
 	cur_freq = (*value) / 1000;
-	transition_latency = CPUFREQ_ETERNAL;
 
 	/*  Check for 7447A based MacRISC3 */
 	if (of_machine_is_compatible("MacRISC3") &&
 	    of_get_property(cpunode, "dynamic-power-step", NULL) &&
 	    PVR_VER(mfspr(SPRN_PVR)) == 0x8003) {
 		pmac_cpufreq_init_7447A(cpunode);
+
+		/* Allow dynamic switching */
 		transition_latency = 8000000;
+		pmac_cpufreq_driver.flags &= ~CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING;
 	/* Check for other MacRISC3 machines */
 	} else if (of_machine_is_compatible("PowerBook3,4") ||
 		   of_machine_is_compatible("PowerBook3,5") ||
diff --git a/drivers/cpufreq/pmac64-cpufreq.c b/drivers/cpufreq/pmac64-cpufreq.c
index 267e0894c62d..be623dd7b9f2 100644
--- a/drivers/cpufreq/pmac64-cpufreq.c
+++ b/drivers/cpufreq/pmac64-cpufreq.c
@@ -516,7 +516,7 @@ static int __init g5_pm72_cpufreq_init(struct device_node *cpunode)
 		goto bail;
 	}
 
-	DBG("cpufreq: i2c clock chip found: %s\n", hwclock->full_name);
+	DBG("cpufreq: i2c clock chip found: %pOF\n", hwclock);
 
 	/* Now get all the platform functions */
 	pfunc_cpu_getfreq =
diff --git a/drivers/cpufreq/powernow-k7.c b/drivers/cpufreq/powernow-k7.c
index 9f013ed42977..80ac313e6c59 100644
--- a/drivers/cpufreq/powernow-k7.c
+++ b/drivers/cpufreq/powernow-k7.c
@@ -578,7 +578,7 @@ static int acer_cpufreq_pst(const struct dmi_system_id *d)
  * A BIOS update is all that can save them.
  * Mention this, and disable cpufreq.
  */
-static struct dmi_system_id powernow_dmi_table[] = {
+static const struct dmi_system_id powernow_dmi_table[] = {
 	{
 		.callback = acer_cpufreq_pst,
 		.ident = "Acer Aspire",
diff --git a/drivers/cpufreq/s5pv210-cpufreq.c b/drivers/cpufreq/s5pv210-cpufreq.c
index f82074eea779..5d31c2db12a3 100644
--- a/drivers/cpufreq/s5pv210-cpufreq.c
+++ b/drivers/cpufreq/s5pv210-cpufreq.c
@@ -602,6 +602,7 @@ static int s5pv210_cpufreq_probe(struct platform_device *pdev)
 	}
 
 	clk_base = of_iomap(np, 0);
+	of_node_put(np);
 	if (!clk_base) {
 		pr_err("%s: failed to map clock registers\n", __func__);
 		return -EFAULT;
@@ -612,6 +613,7 @@ static int s5pv210_cpufreq_probe(struct platform_device *pdev)
 		if (id < 0 || id >= ARRAY_SIZE(dmc_base)) {
 			pr_err("%s: failed to get alias of dmc node '%s'\n",
 				__func__, np->name);
+			of_node_put(np);
 			return id;
 		}
 
@@ -619,6 +621,7 @@ static int s5pv210_cpufreq_probe(struct platform_device *pdev)
 		if (!dmc_base[id]) {
 			pr_err("%s: failed to map dmc%d registers\n",
 				__func__, id);
+			of_node_put(np);
 			return -EFAULT;
 		}
 	}
diff --git a/drivers/cpufreq/sa1100-cpufreq.c b/drivers/cpufreq/sa1100-cpufreq.c
index 728eab77e8e0..e2d8a77c36d5 100644
--- a/drivers/cpufreq/sa1100-cpufreq.c
+++ b/drivers/cpufreq/sa1100-cpufreq.c
@@ -197,11 +197,12 @@ static int sa1100_target(struct cpufreq_policy *policy, unsigned int ppcr)
 
 static int __init sa1100_cpu_init(struct cpufreq_policy *policy)
 {
-	return cpufreq_generic_init(policy, sa11x0_freq_table, CPUFREQ_ETERNAL);
+	return cpufreq_generic_init(policy, sa11x0_freq_table, 0);
 }
 
 static struct cpufreq_driver sa1100_driver __refdata = {
-	.flags		= CPUFREQ_STICKY | CPUFREQ_NEED_INITIAL_FREQ_CHECK,
+	.flags		= CPUFREQ_STICKY | CPUFREQ_NEED_INITIAL_FREQ_CHECK |
+			  CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING,
 	.verify		= cpufreq_generic_frequency_table_verify,
 	.target_index	= sa1100_target,
 	.get		= sa11x0_getspeed,
diff --git a/drivers/cpufreq/sa1110-cpufreq.c b/drivers/cpufreq/sa1110-cpufreq.c
index 2bac9b6cfeea..66e5fb088ecc 100644
--- a/drivers/cpufreq/sa1110-cpufreq.c
+++ b/drivers/cpufreq/sa1110-cpufreq.c
@@ -306,13 +306,14 @@ static int sa1110_target(struct cpufreq_policy *policy, unsigned int ppcr)
 
 static int __init sa1110_cpu_init(struct cpufreq_policy *policy)
 {
-	return cpufreq_generic_init(policy, sa11x0_freq_table, CPUFREQ_ETERNAL);
+	return cpufreq_generic_init(policy, sa11x0_freq_table, 0);
 }
 
 /* sa1110_driver needs __refdata because it must remain after init registers
  * it with cpufreq_register_driver() */
 static struct cpufreq_driver sa1110_driver __refdata = {
-	.flags		= CPUFREQ_STICKY | CPUFREQ_NEED_INITIAL_FREQ_CHECK,
+	.flags		= CPUFREQ_STICKY | CPUFREQ_NEED_INITIAL_FREQ_CHECK |
+			  CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING,
 	.verify		= cpufreq_generic_frequency_table_verify,
 	.target_index	= sa1110_target,
 	.get		= sa11x0_getspeed,
diff --git a/drivers/cpufreq/sh-cpufreq.c b/drivers/cpufreq/sh-cpufreq.c
index 719c3d9f07fb..28893d435cf5 100644
--- a/drivers/cpufreq/sh-cpufreq.c
+++ b/drivers/cpufreq/sh-cpufreq.c
@@ -137,8 +137,6 @@ static int sh_cpufreq_cpu_init(struct cpufreq_policy *policy)
 			(clk_round_rate(cpuclk, ~0UL) + 500) / 1000;
 	}
 
-	policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
-
 	dev_info(dev, "CPU Frequencies - Minimum %u.%03u MHz, "
 	       "Maximum %u.%03u MHz.\n",
 	       policy->min / 1000, policy->min % 1000,
@@ -159,6 +157,7 @@ static int sh_cpufreq_cpu_exit(struct cpufreq_policy *policy)
 
 static struct cpufreq_driver sh_cpufreq_driver = {
 	.name		= "sh",
+	.flags		= CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING,
 	.get		= sh_cpufreq_get,
 	.target		= sh_cpufreq_target,
 	.verify		= sh_cpufreq_verify,
diff --git a/drivers/cpufreq/speedstep-ich.c b/drivers/cpufreq/speedstep-ich.c
index b86953a3ddc4..0412a246a785 100644
--- a/drivers/cpufreq/speedstep-ich.c
+++ b/drivers/cpufreq/speedstep-ich.c
@@ -207,7 +207,7 @@ static unsigned int speedstep_detect_chipset(void)
 		 * 8100 which use a pretty old revision of the 82815
 		 * host bridge. Abort on these systems.
 		 */
-		static struct pci_dev *hostbridge;
+		struct pci_dev *hostbridge;
 
 		hostbridge  = pci_get_subsys(PCI_VENDOR_ID_INTEL,
 			      PCI_DEVICE_ID_INTEL_82815_MC,
diff --git a/drivers/cpufreq/speedstep-lib.c b/drivers/cpufreq/speedstep-lib.c
index 1b8062182c81..ccab452a4ef5 100644
--- a/drivers/cpufreq/speedstep-lib.c
+++ b/drivers/cpufreq/speedstep-lib.c
@@ -35,7 +35,7 @@ static int relaxed_check;
 static unsigned int pentium3_get_frequency(enum speedstep_processor processor)
 {
 	/* See table 14 of p3_ds.pdf and table 22 of 29834003.pdf */
-	struct {
+	static const struct {
 		unsigned int ratio;	/* Frequency Multiplier (x10) */
 		u8 bitmap;		/* power on configuration bits
 					[27, 25:22] (in MSR 0x2a) */
@@ -58,7 +58,7 @@ static unsigned int pentium3_get_frequency(enum speedstep_processor processor)
 	};
 
 	/* PIII(-M) FSB settings: see table b1-b of 24547206.pdf */
-	struct {
+	static const struct {
 		unsigned int value;	/* Front Side Bus speed in MHz */
 		u8 bitmap;		/* power on configuration bits [18: 19]
 					(in MSR 0x2a) */
diff --git a/drivers/cpufreq/speedstep-smi.c b/drivers/cpufreq/speedstep-smi.c
index 37b30071c220..d23f24ccff38 100644
--- a/drivers/cpufreq/speedstep-smi.c
+++ b/drivers/cpufreq/speedstep-smi.c
@@ -266,7 +266,6 @@ static int speedstep_cpu_init(struct cpufreq_policy *policy)
 			pr_debug("workaround worked.\n");
 	}
 
-	policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
 	return cpufreq_table_validate_and_show(policy, speedstep_freqs);
 }
 
@@ -290,6 +289,7 @@ static int speedstep_resume(struct cpufreq_policy *policy)
 
 static struct cpufreq_driver speedstep_driver = {
 	.name		= "speedstep-smi",
+	.flags		= CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING,
 	.verify		= cpufreq_generic_frequency_table_verify,
 	.target_index	= speedstep_target,
 	.init		= speedstep_cpu_init,
diff --git a/drivers/cpufreq/sti-cpufreq.c b/drivers/cpufreq/sti-cpufreq.c
index d2d0430d09d4..47105735df12 100644
--- a/drivers/cpufreq/sti-cpufreq.c
+++ b/drivers/cpufreq/sti-cpufreq.c
@@ -65,8 +65,8 @@ static int sti_cpufreq_fetch_major(void) {
 	ret = of_property_read_u32_index(np, "st,syscfg",
 					 MAJOR_ID_INDEX, &major_offset);
 	if (ret) {
-		dev_err(dev, "No major number offset provided in %s [%d]\n",
-			np->full_name, ret);
+		dev_err(dev, "No major number offset provided in %pOF [%d]\n",
+			np, ret);
 		return ret;
 	}
 
@@ -92,8 +92,8 @@ static int sti_cpufreq_fetch_minor(void)
 					 MINOR_ID_INDEX, &minor_offset);
 	if (ret) {
 		dev_err(dev,
-			"No minor number offset provided %s [%d]\n",
-			np->full_name, ret);
+			"No minor number offset provided %pOF [%d]\n",
+			np, ret);
 		return ret;
 	}
 
diff --git a/drivers/cpufreq/tango-cpufreq.c b/drivers/cpufreq/tango-cpufreq.c
new file mode 100644
index 000000000000..89a7f860bfe8
--- /dev/null
+++ b/drivers/cpufreq/tango-cpufreq.c
@@ -0,0 +1,38 @@
+#include <linux/of.h>
+#include <linux/cpu.h>
+#include <linux/clk.h>
+#include <linux/pm_opp.h>
+#include <linux/platform_device.h>
+
+static const struct of_device_id machines[] __initconst = {
+	{ .compatible = "sigma,tango4" },
+	{ /* sentinel */ }
+};
+
+static int __init tango_cpufreq_init(void)
+{
+	struct device *cpu_dev = get_cpu_device(0);
+	unsigned long max_freq;
+	struct clk *cpu_clk;
+	void *res;
+
+	if (!of_match_node(machines, of_root))
+		return -ENODEV;
+
+	cpu_clk = clk_get(cpu_dev, NULL);
+	if (IS_ERR(cpu_clk))
+		return -ENODEV;
+
+	max_freq = clk_get_rate(cpu_clk);
+
+	dev_pm_opp_add(cpu_dev, max_freq / 1, 0);
+	dev_pm_opp_add(cpu_dev, max_freq / 2, 0);
+	dev_pm_opp_add(cpu_dev, max_freq / 3, 0);
+	dev_pm_opp_add(cpu_dev, max_freq / 5, 0);
+	dev_pm_opp_add(cpu_dev, max_freq / 9, 0);
+
+	res = platform_device_register_data(NULL, "cpufreq-dt", -1, NULL, 0);
+
+	return PTR_ERR_OR_ZERO(res);
+}
+device_initcall(tango_cpufreq_init);
diff --git a/drivers/cpufreq/ti-cpufreq.c b/drivers/cpufreq/ti-cpufreq.c
index a7b5658c0460..b29cd3398463 100644
--- a/drivers/cpufreq/ti-cpufreq.c
+++ b/drivers/cpufreq/ti-cpufreq.c
@@ -245,8 +245,6 @@ static int ti_cpufreq_init(void)
 	if (ret)
 		goto fail_put_node;
 
-	of_node_put(opp_data->opp_node);
-
 	ret = PTR_ERR_OR_ZERO(dev_pm_opp_set_supported_hw(opp_data->cpu_dev,
 							  version, VERSION_COUNT));
 	if (ret) {
@@ -255,6 +253,8 @@ static int ti_cpufreq_init(void)
 		goto fail_put_node;
 	}
 
+	of_node_put(opp_data->opp_node);
+
 register_cpufreq_dt:
 	platform_device_register_simple("cpufreq-dt", -1, NULL, 0);
 
diff --git a/drivers/cpufreq/unicore2-cpufreq.c b/drivers/cpufreq/unicore2-cpufreq.c
index 6f9dfa80563a..db62d9844751 100644
--- a/drivers/cpufreq/unicore2-cpufreq.c
+++ b/drivers/cpufreq/unicore2-cpufreq.c
@@ -58,13 +58,12 @@ static int __init ucv2_cpu_init(struct cpufreq_policy *policy)
 
 	policy->min = policy->cpuinfo.min_freq = 250000;
 	policy->max = policy->cpuinfo.max_freq = 1000000;
-	policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
 	policy->clk = clk_get(NULL, "MAIN_CLK");
 	return PTR_ERR_OR_ZERO(policy->clk);
 }
 
 static struct cpufreq_driver ucv2_driver = {
-	.flags		= CPUFREQ_STICKY,
+	.flags		= CPUFREQ_STICKY | CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING,
 	.verify		= ucv2_verify_speed,
 	.target		= ucv2_target,
 	.get		= cpufreq_generic_get,
diff --git a/drivers/cpuidle/Makefile b/drivers/cpuidle/Makefile
index 3ba81b1dffad..0b67a05a7aae 100644
--- a/drivers/cpuidle/Makefile
+++ b/drivers/cpuidle/Makefile
@@ -5,6 +5,7 @@
 obj-y += cpuidle.o driver.o governor.o sysfs.o governors/
 obj-$(CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED) += coupled.o
 obj-$(CONFIG_DT_IDLE_STATES)		  += dt_idle_states.o
+obj-$(CONFIG_ARCH_HAS_CPU_RELAX)	  += poll_state.o
 
 ##################################################################################
 # ARM SoC drivers
diff --git a/drivers/cpuidle/coupled.c b/drivers/cpuidle/coupled.c
index 71e586d7df71..147f38ea0fcd 100644
--- a/drivers/cpuidle/coupled.c
+++ b/drivers/cpuidle/coupled.c
@@ -119,13 +119,13 @@ struct cpuidle_coupled {
 
 #define CPUIDLE_COUPLED_NOT_IDLE	(-1)
 
-static DEFINE_PER_CPU(struct call_single_data, cpuidle_coupled_poke_cb);
+static DEFINE_PER_CPU(call_single_data_t, cpuidle_coupled_poke_cb);
 
 /*
  * The cpuidle_coupled_poke_pending mask is used to avoid calling
- * __smp_call_function_single with the per cpu call_single_data struct already
+ * __smp_call_function_single with the per cpu call_single_data_t struct already
  * in use.  This prevents a deadlock where two cpus are waiting for each others
- * call_single_data struct to be available
+ * call_single_data_t struct to be available
  */
 static cpumask_t cpuidle_coupled_poke_pending;
 
@@ -339,7 +339,7 @@ static void cpuidle_coupled_handle_poke(void *info)
  */
 static void cpuidle_coupled_poke(int cpu)
 {
-	struct call_single_data *csd = &per_cpu(cpuidle_coupled_poke_cb, cpu);
+	call_single_data_t *csd = &per_cpu(cpuidle_coupled_poke_cb, cpu);
 
 	if (!cpumask_test_and_set_cpu(cpu, &cpuidle_coupled_poke_pending))
 		smp_call_function_single_async(cpu, csd);
@@ -651,7 +651,7 @@ int cpuidle_coupled_register_device(struct cpuidle_device *dev)
 {
 	int cpu;
 	struct cpuidle_device *other_dev;
-	struct call_single_data *csd;
+	call_single_data_t *csd;
 	struct cpuidle_coupled *coupled;
 
 	if (cpumask_empty(&dev->coupled_cpus))
diff --git a/drivers/cpuidle/cpuidle-cps.c b/drivers/cpuidle/cpuidle-cps.c
index 12b9145913de..72b5e47286b4 100644
--- a/drivers/cpuidle/cpuidle-cps.c
+++ b/drivers/cpuidle/cpuidle-cps.c
@@ -37,7 +37,7 @@ static int cps_nc_enter(struct cpuidle_device *dev,
 	 * TODO: don't treat core 0 specially, just prevent the final core
 	 * TODO: remap interrupt affinity temporarily
 	 */
-	if (!cpu_data[dev->cpu].core && (index > STATE_NC_WAIT))
+	if (cpus_are_siblings(0, dev->cpu) && (index > STATE_NC_WAIT))
 		index = STATE_NC_WAIT;
 
 	/* Select the appropriate cps_pm_state */
diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c
index 60bb64f4329d..484cc8909d5c 100644
--- a/drivers/cpuidle/cpuidle.c
+++ b/drivers/cpuidle/cpuidle.c
@@ -77,7 +77,7 @@ static int find_deepest_state(struct cpuidle_driver *drv,
 			      struct cpuidle_device *dev,
 			      unsigned int max_latency,
 			      unsigned int forbidden_flags,
-			      bool freeze)
+			      bool s2idle)
 {
 	unsigned int latency_req = 0;
 	int i, ret = 0;
@@ -89,7 +89,7 @@ static int find_deepest_state(struct cpuidle_driver *drv,
 		if (s->disabled || su->disable || s->exit_latency <= latency_req
 		    || s->exit_latency > max_latency
 		    || (s->flags & forbidden_flags)
-		    || (freeze && !s->enter_freeze))
+		    || (s2idle && !s->enter_s2idle))
 			continue;
 
 		latency_req = s->exit_latency;
@@ -128,7 +128,7 @@ int cpuidle_find_deepest_state(struct cpuidle_driver *drv,
 }
 
 #ifdef CONFIG_SUSPEND
-static void enter_freeze_proper(struct cpuidle_driver *drv,
+static void enter_s2idle_proper(struct cpuidle_driver *drv,
 				struct cpuidle_device *dev, int index)
 {
 	/*
@@ -143,7 +143,7 @@ static void enter_freeze_proper(struct cpuidle_driver *drv,
 	 * suspended is generally unsafe.
 	 */
 	stop_critical_timings();
-	drv->states[index].enter_freeze(dev, drv, index);
+	drv->states[index].enter_s2idle(dev, drv, index);
 	WARN_ON(!irqs_disabled());
 	/*
 	 * timekeeping_resume() that will be called by tick_unfreeze() for the
@@ -155,25 +155,25 @@ static void enter_freeze_proper(struct cpuidle_driver *drv,
 }
 
 /**
- * cpuidle_enter_freeze - Enter an idle state suitable for suspend-to-idle.
+ * cpuidle_enter_s2idle - Enter an idle state suitable for suspend-to-idle.
  * @drv: cpuidle driver for the given CPU.
  * @dev: cpuidle device for the given CPU.
  *
- * If there are states with the ->enter_freeze callback, find the deepest of
+ * If there are states with the ->enter_s2idle callback, find the deepest of
  * them and enter it with frozen tick.
  */
-int cpuidle_enter_freeze(struct cpuidle_driver *drv, struct cpuidle_device *dev)
+int cpuidle_enter_s2idle(struct cpuidle_driver *drv, struct cpuidle_device *dev)
 {
 	int index;
 
 	/*
-	 * Find the deepest state with ->enter_freeze present, which guarantees
+	 * Find the deepest state with ->enter_s2idle present, which guarantees
 	 * that interrupts won't be enabled when it exits and allows the tick to
 	 * be frozen safely.
 	 */
 	index = find_deepest_state(drv, dev, UINT_MAX, 0, true);
 	if (index > 0)
-		enter_freeze_proper(drv, dev, index);
+		enter_s2idle_proper(drv, dev, index);
 
 	return index;
 }
diff --git a/drivers/cpuidle/driver.c b/drivers/cpuidle/driver.c
index e53fb861beb0..dc32f34e68d9 100644
--- a/drivers/cpuidle/driver.c
+++ b/drivers/cpuidle/driver.c
@@ -179,36 +179,6 @@ static void __cpuidle_driver_init(struct cpuidle_driver *drv)
 	}
 }
 
-#ifdef CONFIG_ARCH_HAS_CPU_RELAX
-static int __cpuidle poll_idle(struct cpuidle_device *dev,
-			       struct cpuidle_driver *drv, int index)
-{
-	local_irq_enable();
-	if (!current_set_polling_and_test()) {
-		while (!need_resched())
-			cpu_relax();
-	}
-	current_clr_polling();
-
-	return index;
-}
-
-static void poll_idle_init(struct cpuidle_driver *drv)
-{
-	struct cpuidle_state *state = &drv->states[0];
-
-	snprintf(state->name, CPUIDLE_NAME_LEN, "POLL");
-	snprintf(state->desc, CPUIDLE_DESC_LEN, "CPUIDLE CORE POLL IDLE");
-	state->exit_latency = 0;
-	state->target_residency = 0;
-	state->power_usage = -1;
-	state->enter = poll_idle;
-	state->disabled = false;
-}
-#else
-static void poll_idle_init(struct cpuidle_driver *drv) {}
-#endif /* !CONFIG_ARCH_HAS_CPU_RELAX */
-
 /**
  * __cpuidle_register_driver: register the driver
  * @drv: a valid pointer to a struct cpuidle_driver
@@ -246,8 +216,6 @@ static int __cpuidle_register_driver(struct cpuidle_driver *drv)
 		on_each_cpu_mask(drv->cpumask, cpuidle_setup_broadcast_timer,
 				 (void *)1, 1);
 
-	poll_idle_init(drv);
-
 	return 0;
 }
 
diff --git a/drivers/cpuidle/dt_idle_states.c b/drivers/cpuidle/dt_idle_states.c
index ae8eb0359889..53342b7f1010 100644
--- a/drivers/cpuidle/dt_idle_states.c
+++ b/drivers/cpuidle/dt_idle_states.c
@@ -41,9 +41,9 @@ static int init_state_node(struct cpuidle_state *idle_state,
 	/*
 	 * Since this is not a "coupled" state, it's safe to assume interrupts
 	 * won't be enabled when it exits allowing the tick to be frozen
-	 * safely. So enter() can be also enter_freeze() callback.
+	 * safely. So enter() can be also enter_s2idle() callback.
 	 */
-	idle_state->enter_freeze = match_id->data;
+	idle_state->enter_s2idle = match_id->data;
 
 	err = of_property_read_u32(state_node, "wakeup-latency-us",
 				   &idle_state->exit_latency);
@@ -53,16 +53,16 @@ static int init_state_node(struct cpuidle_state *idle_state,
 		err = of_property_read_u32(state_node, "entry-latency-us",
 					   &entry_latency);
 		if (err) {
-			pr_debug(" * %s missing entry-latency-us property\n",
-				 state_node->full_name);
+			pr_debug(" * %pOF missing entry-latency-us property\n",
+				 state_node);
 			return -EINVAL;
 		}
 
 		err = of_property_read_u32(state_node, "exit-latency-us",
 					   &exit_latency);
 		if (err) {
-			pr_debug(" * %s missing exit-latency-us property\n",
-				 state_node->full_name);
+			pr_debug(" * %pOF missing exit-latency-us property\n",
+				 state_node);
 			return -EINVAL;
 		}
 		/*
@@ -75,8 +75,8 @@ static int init_state_node(struct cpuidle_state *idle_state,
 	err = of_property_read_u32(state_node, "min-residency-us",
 				   &idle_state->target_residency);
 	if (err) {
-		pr_debug(" * %s missing min-residency-us property\n",
-			     state_node->full_name);
+		pr_debug(" * %pOF missing min-residency-us property\n",
+			     state_node);
 		return -EINVAL;
 	}
 
@@ -186,8 +186,8 @@ int dt_init_idle_driver(struct cpuidle_driver *drv,
 		}
 
 		if (!idle_state_valid(state_node, i, cpumask)) {
-			pr_warn("%s idle state not valid, bailing out\n",
-				state_node->full_name);
+			pr_warn("%pOF idle state not valid, bailing out\n",
+				state_node);
 			err = -EINVAL;
 			break;
 		}
@@ -200,8 +200,8 @@ int dt_init_idle_driver(struct cpuidle_driver *drv,
 		idle_state = &drv->states[state_idx++];
 		err = init_state_node(idle_state, matches, state_node);
 		if (err) {
-			pr_err("Parsing idle state node %s failed with err %d\n",
-			       state_node->full_name, err);
+			pr_err("Parsing idle state node %pOF failed with err %d\n",
+			       state_node, err);
 			err = -EINVAL;
 			break;
 		}
diff --git a/drivers/cpuidle/governors/ladder.c b/drivers/cpuidle/governors/ladder.c
index ac321f09e717..ce1a2ffffb2a 100644
--- a/drivers/cpuidle/governors/ladder.c
+++ b/drivers/cpuidle/governors/ladder.c
@@ -69,6 +69,7 @@ static int ladder_select_state(struct cpuidle_driver *drv,
 	struct ladder_device *ldev = this_cpu_ptr(&ladder_devices);
 	struct ladder_device_state *last_state;
 	int last_residency, last_idx = ldev->last_state_idx;
+	int first_idx = drv->states[0].flags & CPUIDLE_FLAG_POLLING ? 1 : 0;
 	int latency_req = pm_qos_request(PM_QOS_CPU_DMA_LATENCY);
 
 	/* Special case when user has set very strict latency requirement */
@@ -96,13 +97,13 @@ static int ladder_select_state(struct cpuidle_driver *drv,
 	}
 
 	/* consider demotion */
-	if (last_idx > CPUIDLE_DRIVER_STATE_START &&
+	if (last_idx > first_idx &&
 	    (drv->states[last_idx].disabled ||
 	    dev->states_usage[last_idx].disable ||
 	    drv->states[last_idx].exit_latency > latency_req)) {
 		int i;
 
-		for (i = last_idx - 1; i > CPUIDLE_DRIVER_STATE_START; i--) {
+		for (i = last_idx - 1; i > first_idx; i--) {
 			if (drv->states[i].exit_latency <= latency_req)
 				break;
 		}
@@ -110,7 +111,7 @@ static int ladder_select_state(struct cpuidle_driver *drv,
 		return i;
 	}
 
-	if (last_idx > CPUIDLE_DRIVER_STATE_START &&
+	if (last_idx > first_idx &&
 	    last_residency < last_state->threshold.demotion_time) {
 		last_state->stats.demotion_count++;
 		last_state->stats.promotion_count = 0;
@@ -133,13 +134,14 @@ static int ladder_enable_device(struct cpuidle_driver *drv,
 				struct cpuidle_device *dev)
 {
 	int i;
+	int first_idx = drv->states[0].flags & CPUIDLE_FLAG_POLLING ? 1 : 0;
 	struct ladder_device *ldev = &per_cpu(ladder_devices, dev->cpu);
 	struct ladder_device_state *lstate;
 	struct cpuidle_state *state;
 
-	ldev->last_state_idx = CPUIDLE_DRIVER_STATE_START;
+	ldev->last_state_idx = first_idx;
 
-	for (i = CPUIDLE_DRIVER_STATE_START; i < drv->state_count; i++) {
+	for (i = first_idx; i < drv->state_count; i++) {
 		state = &drv->states[i];
 		lstate = &ldev->states[i];
 
@@ -151,7 +153,7 @@ static int ladder_enable_device(struct cpuidle_driver *drv,
 
 		if (i < drv->state_count - 1)
 			lstate->threshold.promotion_time = state->exit_latency;
-		if (i > CPUIDLE_DRIVER_STATE_START)
+		if (i > first_idx)
 			lstate->threshold.demotion_time = state->exit_latency;
 	}
 
diff --git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c
index 61b64c2b2cb8..48eaf2879228 100644
--- a/drivers/cpuidle/governors/menu.c
+++ b/drivers/cpuidle/governors/menu.c
@@ -324,8 +324,9 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev)
 	expected_interval = get_typical_interval(data);
 	expected_interval = min(expected_interval, data->next_timer_us);
 
-	if (CPUIDLE_DRIVER_STATE_START > 0) {
-		struct cpuidle_state *s = &drv->states[CPUIDLE_DRIVER_STATE_START];
+	first_idx = 0;
+	if (drv->states[0].flags & CPUIDLE_FLAG_POLLING) {
+		struct cpuidle_state *s = &drv->states[1];
 		unsigned int polling_threshold;
 
 		/*
@@ -336,12 +337,8 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev)
 		polling_threshold = max_t(unsigned int, 20, s->target_residency);
 		if (data->next_timer_us > polling_threshold &&
 		    latency_req > s->exit_latency && !s->disabled &&
-		    !dev->states_usage[CPUIDLE_DRIVER_STATE_START].disable)
-			first_idx = CPUIDLE_DRIVER_STATE_START;
-		else
-			first_idx = CPUIDLE_DRIVER_STATE_START - 1;
-	} else {
-		first_idx = 0;
+		    !dev->states_usage[1].disable)
+			first_idx = 1;
 	}
 
 	/*
diff --git a/drivers/cpuidle/poll_state.c b/drivers/cpuidle/poll_state.c
new file mode 100644
index 000000000000..7416b16287de
--- /dev/null
+++ b/drivers/cpuidle/poll_state.c
@@ -0,0 +1,37 @@
+/*
+ * poll_state.c - Polling idle state
+ *
+ * This file is released under the GPLv2.
+ */
+
+#include <linux/cpuidle.h>
+#include <linux/sched.h>
+#include <linux/sched/idle.h>
+
+static int __cpuidle poll_idle(struct cpuidle_device *dev,
+			       struct cpuidle_driver *drv, int index)
+{
+	local_irq_enable();
+	if (!current_set_polling_and_test()) {
+		while (!need_resched())
+			cpu_relax();
+	}
+	current_clr_polling();
+
+	return index;
+}
+
+void cpuidle_poll_state_init(struct cpuidle_driver *drv)
+{
+	struct cpuidle_state *state = &drv->states[0];
+
+	snprintf(state->name, CPUIDLE_NAME_LEN, "POLL");
+	snprintf(state->desc, CPUIDLE_DESC_LEN, "CPUIDLE CORE POLL IDLE");
+	state->exit_latency = 0;
+	state->target_residency = 0;
+	state->power_usage = -1;
+	state->enter = poll_idle;
+	state->disabled = false;
+	state->flags = CPUIDLE_FLAG_POLLING;
+}
+EXPORT_SYMBOL_GPL(cpuidle_poll_state_init);
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index 4b75084fabad..fe33c199fc1a 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -525,12 +525,26 @@ config CRYPTO_DEV_ATMEL_SHA
 	  To compile this driver as a module, choose M here: the module
 	  will be called atmel-sha.
 
+config CRYPTO_DEV_ATMEL_ECC
+	tristate "Support for Microchip / Atmel ECC hw accelerator"
+	depends on ARCH_AT91 || COMPILE_TEST
+	depends on I2C
+	select CRYPTO_ECDH
+	select CRC16
+	help
+	  Microhip / Atmel ECC hw accelerator.
+	  Select this if you want to use the Microchip / Atmel module for
+	  ECDH algorithm.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called atmel-ecc.
+
 config CRYPTO_DEV_CCP
-	bool "Support for AMD Cryptographic Coprocessor"
+	bool "Support for AMD Secure Processor"
 	depends on ((X86 && PCI) || (ARM64 && (OF_ADDRESS || ACPI))) && HAS_IOMEM
 	help
-	  The AMD Cryptographic Coprocessor provides hardware offload support
-	  for encryption, hashing and related operations.
+	  The AMD Secure Processor provides support for the Cryptographic Coprocessor
+	  (CCP) and the Platform Security Processor (PSP) devices.
 
 if CRYPTO_DEV_CCP
 	source "drivers/crypto/ccp/Kconfig"
@@ -616,6 +630,14 @@ config CRYPTO_DEV_SUN4I_SS
 	  To compile this driver as a module, choose M here: the module
 	  will be called sun4i-ss.
 
+config CRYPTO_DEV_SUN4I_SS_PRNG
+	bool "Support for Allwinner Security System PRNG"
+	depends on CRYPTO_DEV_SUN4I_SS
+	select CRYPTO_RNG
+	help
+	  Select this option if you want to provide kernel-side support for
+	  the Pseudo-Random Number Generator found in the Security System.
+
 config CRYPTO_DEV_ROCKCHIP
 	tristate "Rockchip's Cryptographic Engine driver"
 	depends on OF && ARCH_ROCKCHIP
@@ -686,4 +708,25 @@ config CRYPTO_DEV_SAFEXCEL
 	  chain mode, AES cipher mode and SHA1/SHA224/SHA256/SHA512 hash
 	  algorithms.
 
+config CRYPTO_DEV_ARTPEC6
+	tristate "Support for Axis ARTPEC-6/7 hardware crypto acceleration."
+	depends on ARM && (ARCH_ARTPEC || COMPILE_TEST)
+	depends on HAS_DMA
+	depends on OF
+	select CRYPTO_AEAD
+	select CRYPTO_AES
+	select CRYPTO_ALGAPI
+	select CRYPTO_BLKCIPHER
+	select CRYPTO_CTR
+	select CRYPTO_HASH
+	select CRYPTO_SHA1
+	select CRYPTO_SHA256
+	select CRYPTO_SHA384
+	select CRYPTO_SHA512
+	help
+	  Enables the driver for the on-chip crypto accelerator
+	  of Axis ARTPEC SoCs.
+
+	  To compile this driver as a module, choose M here.
+
 endif # CRYPTO_HW
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index 2c555a3393b2..808432b44c6b 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -1,6 +1,7 @@
 obj-$(CONFIG_CRYPTO_DEV_ATMEL_AES) += atmel-aes.o
 obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA) += atmel-sha.o
 obj-$(CONFIG_CRYPTO_DEV_ATMEL_TDES) += atmel-tdes.o
+obj-$(CONFIG_CRYPTO_DEV_ATMEL_ECC) += atmel-ecc.o
 obj-$(CONFIG_CRYPTO_DEV_BFIN_CRC) += bfin_crc.o
 obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += cavium/
 obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/
@@ -35,7 +36,7 @@ obj-$(CONFIG_CRYPTO_DEV_QCE) += qce/
 obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP) += rockchip/
 obj-$(CONFIG_CRYPTO_DEV_S5P) += s5p-sss.o
 obj-$(CONFIG_CRYPTO_DEV_SAHARA) += sahara.o
-obj-$(CONFIG_CRYPTO_DEV_STM32) += stm32/
+obj-$(CONFIG_ARCH_STM32) += stm32/
 obj-$(CONFIG_CRYPTO_DEV_SUN4I_SS) += sunxi-ss/
 obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o
 obj-$(CONFIG_CRYPTO_DEV_UX500) += ux500/
@@ -43,3 +44,4 @@ obj-$(CONFIG_CRYPTO_DEV_VIRTIO) += virtio/
 obj-$(CONFIG_CRYPTO_DEV_VMX) += vmx/
 obj-$(CONFIG_CRYPTO_DEV_BCM_SPU) += bcm/
 obj-$(CONFIG_CRYPTO_DEV_SAFEXCEL) += inside-secure/
+obj-$(CONFIG_CRYPTO_DEV_ARTPEC6) += axis/
diff --git a/drivers/crypto/atmel-ecc.c b/drivers/crypto/atmel-ecc.c
new file mode 100644
index 000000000000..e66f18a0ddd0
--- /dev/null
+++ b/drivers/crypto/atmel-ecc.c
@@ -0,0 +1,781 @@
+/*
+ * Microchip / Atmel ECC (I2C) driver.
+ *
+ * Copyright (c) 2017, Microchip Technology Inc.
+ * Author: Tudor Ambarus <tudor.ambarus@microchip.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/bitrev.h>
+#include <linux/crc16.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/workqueue.h>
+#include <crypto/internal/kpp.h>
+#include <crypto/ecdh.h>
+#include <crypto/kpp.h>
+#include "atmel-ecc.h"
+
+/* Used for binding tfm objects to i2c clients. */
+struct atmel_ecc_driver_data {
+	struct list_head i2c_client_list;
+	spinlock_t i2c_list_lock;
+} ____cacheline_aligned;
+
+static struct atmel_ecc_driver_data driver_data;
+
+/**
+ * atmel_ecc_i2c_client_priv - i2c_client private data
+ * @client              : pointer to i2c client device
+ * @i2c_client_list_node: part of i2c_client_list
+ * @lock                : lock for sending i2c commands
+ * @wake_token          : wake token array of zeros
+ * @wake_token_sz       : size in bytes of the wake_token
+ * @tfm_count           : number of active crypto transformations on i2c client
+ *
+ * Reads and writes from/to the i2c client are sequential. The first byte
+ * transmitted to the device is treated as the byte size. Any attempt to send
+ * more than this number of bytes will cause the device to not ACK those bytes.
+ * After the host writes a single command byte to the input buffer, reads are
+ * prohibited until after the device completes command execution. Use a mutex
+ * when sending i2c commands.
+ */
+struct atmel_ecc_i2c_client_priv {
+	struct i2c_client *client;
+	struct list_head i2c_client_list_node;
+	struct mutex lock;
+	u8 wake_token[WAKE_TOKEN_MAX_SIZE];
+	size_t wake_token_sz;
+	atomic_t tfm_count ____cacheline_aligned;
+};
+
+/**
+ * atmel_ecdh_ctx - transformation context
+ * @client     : pointer to i2c client device
+ * @fallback   : used for unsupported curves or when user wants to use its own
+ *               private key.
+ * @public_key : generated when calling set_secret(). It's the responsibility
+ *               of the user to not call set_secret() while
+ *               generate_public_key() or compute_shared_secret() are in flight.
+ * @curve_id   : elliptic curve id
+ * @n_sz       : size in bytes of the n prime
+ * @do_fallback: true when the device doesn't support the curve or when the user
+ *               wants to use its own private key.
+ */
+struct atmel_ecdh_ctx {
+	struct i2c_client *client;
+	struct crypto_kpp *fallback;
+	const u8 *public_key;
+	unsigned int curve_id;
+	size_t n_sz;
+	bool do_fallback;
+};
+
+/**
+ * atmel_ecc_work_data - data structure representing the work
+ * @ctx : transformation context.
+ * @cbk : pointer to a callback function to be invoked upon completion of this
+ *        request. This has the form:
+ *        callback(struct atmel_ecc_work_data *work_data, void *areq, u8 status)
+ *        where:
+ *        @work_data: data structure representing the work
+ *        @areq     : optional pointer to an argument passed with the original
+ *                    request.
+ *        @status   : status returned from the i2c client device or i2c error.
+ * @areq: optional pointer to a user argument for use at callback time.
+ * @work: describes the task to be executed.
+ * @cmd : structure used for communicating with the device.
+ */
+struct atmel_ecc_work_data {
+	struct atmel_ecdh_ctx *ctx;
+	void (*cbk)(struct atmel_ecc_work_data *work_data, void *areq,
+		    int status);
+	void *areq;
+	struct work_struct work;
+	struct atmel_ecc_cmd cmd;
+};
+
+static u16 atmel_ecc_crc16(u16 crc, const u8 *buffer, size_t len)
+{
+	return cpu_to_le16(bitrev16(crc16(crc, buffer, len)));
+}
+
+/**
+ * atmel_ecc_checksum() - Generate 16-bit CRC as required by ATMEL ECC.
+ * CRC16 verification of the count, opcode, param1, param2 and data bytes.
+ * The checksum is saved in little-endian format in the least significant
+ * two bytes of the command. CRC polynomial is 0x8005 and the initial register
+ * value should be zero.
+ *
+ * @cmd : structure used for communicating with the device.
+ */
+static void atmel_ecc_checksum(struct atmel_ecc_cmd *cmd)
+{
+	u8 *data = &cmd->count;
+	size_t len = cmd->count - CRC_SIZE;
+	u16 *crc16 = (u16 *)(data + len);
+
+	*crc16 = atmel_ecc_crc16(0, data, len);
+}
+
+static void atmel_ecc_init_read_cmd(struct atmel_ecc_cmd *cmd)
+{
+	cmd->word_addr = COMMAND;
+	cmd->opcode = OPCODE_READ;
+	/*
+	 * Read the word from Configuration zone that contains the lock bytes
+	 * (UserExtra, Selector, LockValue, LockConfig).
+	 */
+	cmd->param1 = CONFIG_ZONE;
+	cmd->param2 = DEVICE_LOCK_ADDR;
+	cmd->count = READ_COUNT;
+
+	atmel_ecc_checksum(cmd);
+
+	cmd->msecs = MAX_EXEC_TIME_READ;
+	cmd->rxsize = READ_RSP_SIZE;
+}
+
+static void atmel_ecc_init_genkey_cmd(struct atmel_ecc_cmd *cmd, u16 keyid)
+{
+	cmd->word_addr = COMMAND;
+	cmd->count = GENKEY_COUNT;
+	cmd->opcode = OPCODE_GENKEY;
+	cmd->param1 = GENKEY_MODE_PRIVATE;
+	/* a random private key will be generated and stored in slot keyID */
+	cmd->param2 = cpu_to_le16(keyid);
+
+	atmel_ecc_checksum(cmd);
+
+	cmd->msecs = MAX_EXEC_TIME_GENKEY;
+	cmd->rxsize = GENKEY_RSP_SIZE;
+}
+
+static int atmel_ecc_init_ecdh_cmd(struct atmel_ecc_cmd *cmd,
+				   struct scatterlist *pubkey)
+{
+	size_t copied;
+
+	cmd->word_addr = COMMAND;
+	cmd->count = ECDH_COUNT;
+	cmd->opcode = OPCODE_ECDH;
+	cmd->param1 = ECDH_PREFIX_MODE;
+	/* private key slot */
+	cmd->param2 = cpu_to_le16(DATA_SLOT_2);
+
+	/*
+	 * The device only supports NIST P256 ECC keys. The public key size will
+	 * always be the same. Use a macro for the key size to avoid unnecessary
+	 * computations.
+	 */
+	copied = sg_copy_to_buffer(pubkey, 1, cmd->data, ATMEL_ECC_PUBKEY_SIZE);
+	if (copied != ATMEL_ECC_PUBKEY_SIZE)
+		return -EINVAL;
+
+	atmel_ecc_checksum(cmd);
+
+	cmd->msecs = MAX_EXEC_TIME_ECDH;
+	cmd->rxsize = ECDH_RSP_SIZE;
+
+	return 0;
+}
+
+/*
+ * After wake and after execution of a command, there will be error, status, or
+ * result bytes in the device's output register that can be retrieved by the
+ * system. When the length of that group is four bytes, the codes returned are
+ * detailed in error_list.
+ */
+static int atmel_ecc_status(struct device *dev, u8 *status)
+{
+	size_t err_list_len = ARRAY_SIZE(error_list);
+	int i;
+	u8 err_id = status[1];
+
+	if (*status != STATUS_SIZE)
+		return 0;
+
+	if (err_id == STATUS_WAKE_SUCCESSFUL || err_id == STATUS_NOERR)
+		return 0;
+
+	for (i = 0; i < err_list_len; i++)
+		if (error_list[i].value == err_id)
+			break;
+
+	/* if err_id is not in the error_list then ignore it */
+	if (i != err_list_len) {
+		dev_err(dev, "%02x: %s:\n", err_id, error_list[i].error_text);
+		return err_id;
+	}
+
+	return 0;
+}
+
+static int atmel_ecc_wakeup(struct i2c_client *client)
+{
+	struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
+	u8 status[STATUS_RSP_SIZE];
+	int ret;
+
+	/*
+	 * The device ignores any levels or transitions on the SCL pin when the
+	 * device is idle, asleep or during waking up. Don't check for error
+	 * when waking up the device.
+	 */
+	i2c_master_send(client, i2c_priv->wake_token, i2c_priv->wake_token_sz);
+
+	/*
+	 * Wait to wake the device. Typical execution times for ecdh and genkey
+	 * are around tens of milliseconds. Delta is chosen to 50 microseconds.
+	 */
+	usleep_range(TWHI_MIN, TWHI_MAX);
+
+	ret = i2c_master_recv(client, status, STATUS_SIZE);
+	if (ret < 0)
+		return ret;
+
+	return atmel_ecc_status(&client->dev, status);
+}
+
+static int atmel_ecc_sleep(struct i2c_client *client)
+{
+	u8 sleep = SLEEP_TOKEN;
+
+	return i2c_master_send(client, &sleep, 1);
+}
+
+static void atmel_ecdh_done(struct atmel_ecc_work_data *work_data, void *areq,
+			    int status)
+{
+	struct kpp_request *req = areq;
+	struct atmel_ecdh_ctx *ctx = work_data->ctx;
+	struct atmel_ecc_cmd *cmd = &work_data->cmd;
+	size_t copied;
+	size_t n_sz = ctx->n_sz;
+
+	if (status)
+		goto free_work_data;
+
+	/* copy the shared secret */
+	copied = sg_copy_from_buffer(req->dst, 1, &cmd->data[RSP_DATA_IDX],
+				     n_sz);
+	if (copied != n_sz)
+		status = -EINVAL;
+
+	/* fall through */
+free_work_data:
+	kzfree(work_data);
+	kpp_request_complete(req, status);
+}
+
+/*
+ * atmel_ecc_send_receive() - send a command to the device and receive its
+ *                            response.
+ * @client: i2c client device
+ * @cmd   : structure used to communicate with the device
+ *
+ * After the device receives a Wake token, a watchdog counter starts within the
+ * device. After the watchdog timer expires, the device enters sleep mode
+ * regardless of whether some I/O transmission or command execution is in
+ * progress. If a command is attempted when insufficient time remains prior to
+ * watchdog timer execution, the device will return the watchdog timeout error
+ * code without attempting to execute the command. There is no way to reset the
+ * counter other than to put the device into sleep or idle mode and then
+ * wake it up again.
+ */
+static int atmel_ecc_send_receive(struct i2c_client *client,
+				  struct atmel_ecc_cmd *cmd)
+{
+	struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
+	int ret;
+
+	mutex_lock(&i2c_priv->lock);
+
+	ret = atmel_ecc_wakeup(client);
+	if (ret)
+		goto err;
+
+	/* send the command */
+	ret = i2c_master_send(client, (u8 *)cmd, cmd->count + WORD_ADDR_SIZE);
+	if (ret < 0)
+		goto err;
+
+	/* delay the appropriate amount of time for command to execute */
+	msleep(cmd->msecs);
+
+	/* receive the response */
+	ret = i2c_master_recv(client, cmd->data, cmd->rxsize);
+	if (ret < 0)
+		goto err;
+
+	/* put the device into low-power mode */
+	ret = atmel_ecc_sleep(client);
+	if (ret < 0)
+		goto err;
+
+	mutex_unlock(&i2c_priv->lock);
+	return atmel_ecc_status(&client->dev, cmd->data);
+err:
+	mutex_unlock(&i2c_priv->lock);
+	return ret;
+}
+
+static void atmel_ecc_work_handler(struct work_struct *work)
+{
+	struct atmel_ecc_work_data *work_data =
+			container_of(work, struct atmel_ecc_work_data, work);
+	struct atmel_ecc_cmd *cmd = &work_data->cmd;
+	struct i2c_client *client = work_data->ctx->client;
+	int status;
+
+	status = atmel_ecc_send_receive(client, cmd);
+	work_data->cbk(work_data, work_data->areq, status);
+}
+
+static void atmel_ecc_enqueue(struct atmel_ecc_work_data *work_data,
+			      void (*cbk)(struct atmel_ecc_work_data *work_data,
+					  void *areq, int status),
+			      void *areq)
+{
+	work_data->cbk = (void *)cbk;
+	work_data->areq = areq;
+
+	INIT_WORK(&work_data->work, atmel_ecc_work_handler);
+	schedule_work(&work_data->work);
+}
+
+static unsigned int atmel_ecdh_supported_curve(unsigned int curve_id)
+{
+	if (curve_id == ECC_CURVE_NIST_P256)
+		return ATMEL_ECC_NIST_P256_N_SIZE;
+
+	return 0;
+}
+
+/*
+ * A random private key is generated and stored in the device. The device
+ * returns the pair public key.
+ */
+static int atmel_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
+				 unsigned int len)
+{
+	struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm);
+	struct atmel_ecc_cmd *cmd;
+	void *public_key;
+	struct ecdh params;
+	int ret = -ENOMEM;
+
+	/* free the old public key, if any */
+	kfree(ctx->public_key);
+	/* make sure you don't free the old public key twice */
+	ctx->public_key = NULL;
+
+	if (crypto_ecdh_decode_key(buf, len, &params) < 0) {
+		dev_err(&ctx->client->dev, "crypto_ecdh_decode_key failed\n");
+		return -EINVAL;
+	}
+
+	ctx->n_sz = atmel_ecdh_supported_curve(params.curve_id);
+	if (!ctx->n_sz || params.key_size) {
+		/* fallback to ecdh software implementation */
+		ctx->do_fallback = true;
+		return crypto_kpp_set_secret(ctx->fallback, buf, len);
+	}
+
+	cmd = kmalloc(sizeof(*cmd), GFP_KERNEL);
+	if (!cmd)
+		return -ENOMEM;
+
+	/*
+	 * The device only supports NIST P256 ECC keys. The public key size will
+	 * always be the same. Use a macro for the key size to avoid unnecessary
+	 * computations.
+	 */
+	public_key = kmalloc(ATMEL_ECC_PUBKEY_SIZE, GFP_KERNEL);
+	if (!public_key)
+		goto free_cmd;
+
+	ctx->do_fallback = false;
+	ctx->curve_id = params.curve_id;
+
+	atmel_ecc_init_genkey_cmd(cmd, DATA_SLOT_2);
+
+	ret = atmel_ecc_send_receive(ctx->client, cmd);
+	if (ret)
+		goto free_public_key;
+
+	/* save the public key */
+	memcpy(public_key, &cmd->data[RSP_DATA_IDX], ATMEL_ECC_PUBKEY_SIZE);
+	ctx->public_key = public_key;
+
+	kfree(cmd);
+	return 0;
+
+free_public_key:
+	kfree(public_key);
+free_cmd:
+	kfree(cmd);
+	return ret;
+}
+
+static int atmel_ecdh_generate_public_key(struct kpp_request *req)
+{
+	struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
+	struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm);
+	size_t copied;
+	int ret = 0;
+
+	if (ctx->do_fallback) {
+		kpp_request_set_tfm(req, ctx->fallback);
+		return crypto_kpp_generate_public_key(req);
+	}
+
+	/* public key was saved at private key generation */
+	copied = sg_copy_from_buffer(req->dst, 1, ctx->public_key,
+				     ATMEL_ECC_PUBKEY_SIZE);
+	if (copied != ATMEL_ECC_PUBKEY_SIZE)
+		ret = -EINVAL;
+
+	return ret;
+}
+
+static int atmel_ecdh_compute_shared_secret(struct kpp_request *req)
+{
+	struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
+	struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm);
+	struct atmel_ecc_work_data *work_data;
+	gfp_t gfp;
+	int ret;
+
+	if (ctx->do_fallback) {
+		kpp_request_set_tfm(req, ctx->fallback);
+		return crypto_kpp_compute_shared_secret(req);
+	}
+
+	gfp = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL :
+							     GFP_ATOMIC;
+
+	work_data = kmalloc(sizeof(*work_data), gfp);
+	if (!work_data)
+		return -ENOMEM;
+
+	work_data->ctx = ctx;
+
+	ret = atmel_ecc_init_ecdh_cmd(&work_data->cmd, req->src);
+	if (ret)
+		goto free_work_data;
+
+	atmel_ecc_enqueue(work_data, atmel_ecdh_done, req);
+
+	return -EINPROGRESS;
+
+free_work_data:
+	kfree(work_data);
+	return ret;
+}
+
+static struct i2c_client *atmel_ecc_i2c_client_alloc(void)
+{
+	struct atmel_ecc_i2c_client_priv *i2c_priv, *min_i2c_priv = NULL;
+	struct i2c_client *client = ERR_PTR(-ENODEV);
+	int min_tfm_cnt = INT_MAX;
+	int tfm_cnt;
+
+	spin_lock(&driver_data.i2c_list_lock);
+
+	if (list_empty(&driver_data.i2c_client_list)) {
+		spin_unlock(&driver_data.i2c_list_lock);
+		return ERR_PTR(-ENODEV);
+	}
+
+	list_for_each_entry(i2c_priv, &driver_data.i2c_client_list,
+			    i2c_client_list_node) {
+		tfm_cnt = atomic_read(&i2c_priv->tfm_count);
+		if (tfm_cnt < min_tfm_cnt) {
+			min_tfm_cnt = tfm_cnt;
+			min_i2c_priv = i2c_priv;
+		}
+		if (!min_tfm_cnt)
+			break;
+	}
+
+	if (min_i2c_priv) {
+		atomic_inc(&min_i2c_priv->tfm_count);
+		client = min_i2c_priv->client;
+	}
+
+	spin_unlock(&driver_data.i2c_list_lock);
+
+	return client;
+}
+
+static void atmel_ecc_i2c_client_free(struct i2c_client *client)
+{
+	struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
+
+	atomic_dec(&i2c_priv->tfm_count);
+}
+
+static int atmel_ecdh_init_tfm(struct crypto_kpp *tfm)
+{
+	const char *alg = kpp_alg_name(tfm);
+	struct crypto_kpp *fallback;
+	struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm);
+
+	ctx->client = atmel_ecc_i2c_client_alloc();
+	if (IS_ERR(ctx->client)) {
+		pr_err("tfm - i2c_client binding failed\n");
+		return PTR_ERR(ctx->client);
+	}
+
+	fallback = crypto_alloc_kpp(alg, 0, CRYPTO_ALG_NEED_FALLBACK);
+	if (IS_ERR(fallback)) {
+		dev_err(&ctx->client->dev, "Failed to allocate transformation for '%s': %ld\n",
+			alg, PTR_ERR(fallback));
+		return PTR_ERR(fallback);
+	}
+
+	crypto_kpp_set_flags(fallback, crypto_kpp_get_flags(tfm));
+
+	dev_info(&ctx->client->dev, "Using '%s' as fallback implementation.\n",
+		 crypto_tfm_alg_driver_name(crypto_kpp_tfm(fallback)));
+
+	ctx->fallback = fallback;
+
+	return 0;
+}
+
+static void atmel_ecdh_exit_tfm(struct crypto_kpp *tfm)
+{
+	struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm);
+
+	kfree(ctx->public_key);
+	crypto_free_kpp(ctx->fallback);
+	atmel_ecc_i2c_client_free(ctx->client);
+}
+
+static unsigned int atmel_ecdh_max_size(struct crypto_kpp *tfm)
+{
+	struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm);
+
+	if (ctx->fallback)
+		return crypto_kpp_maxsize(ctx->fallback);
+
+	/*
+	 * The device only supports NIST P256 ECC keys. The public key size will
+	 * always be the same. Use a macro for the key size to avoid unnecessary
+	 * computations.
+	 */
+	return ATMEL_ECC_PUBKEY_SIZE;
+}
+
+static struct kpp_alg atmel_ecdh = {
+	.set_secret = atmel_ecdh_set_secret,
+	.generate_public_key = atmel_ecdh_generate_public_key,
+	.compute_shared_secret = atmel_ecdh_compute_shared_secret,
+	.init = atmel_ecdh_init_tfm,
+	.exit = atmel_ecdh_exit_tfm,
+	.max_size = atmel_ecdh_max_size,
+	.base = {
+		.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
+		.cra_name = "ecdh",
+		.cra_driver_name = "atmel-ecdh",
+		.cra_priority = ATMEL_ECC_PRIORITY,
+		.cra_module = THIS_MODULE,
+		.cra_ctxsize = sizeof(struct atmel_ecdh_ctx),
+	},
+};
+
+static inline size_t atmel_ecc_wake_token_sz(u32 bus_clk_rate)
+{
+	u32 no_of_bits = DIV_ROUND_UP(TWLO_USEC * bus_clk_rate, USEC_PER_SEC);
+
+	/* return the size of the wake_token in bytes */
+	return DIV_ROUND_UP(no_of_bits, 8);
+}
+
+static int device_sanity_check(struct i2c_client *client)
+{
+	struct atmel_ecc_cmd *cmd;
+	int ret;
+
+	cmd = kmalloc(sizeof(*cmd), GFP_KERNEL);
+	if (!cmd)
+		return -ENOMEM;
+
+	atmel_ecc_init_read_cmd(cmd);
+
+	ret = atmel_ecc_send_receive(client, cmd);
+	if (ret)
+		goto free_cmd;
+
+	/*
+	 * It is vital that the Configuration, Data and OTP zones be locked
+	 * prior to release into the field of the system containing the device.
+	 * Failure to lock these zones may permit modification of any secret
+	 * keys and may lead to other security problems.
+	 */
+	if (cmd->data[LOCK_CONFIG_IDX] || cmd->data[LOCK_VALUE_IDX]) {
+		dev_err(&client->dev, "Configuration or Data and OTP zones are unlocked!\n");
+		ret = -ENOTSUPP;
+	}
+
+	/* fall through */
+free_cmd:
+	kfree(cmd);
+	return ret;
+}
+
+static int atmel_ecc_probe(struct i2c_client *client,
+			   const struct i2c_device_id *id)
+{
+	struct atmel_ecc_i2c_client_priv *i2c_priv;
+	struct device *dev = &client->dev;
+	int ret;
+	u32 bus_clk_rate;
+
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) {
+		dev_err(dev, "I2C_FUNC_I2C not supported\n");
+		return -ENODEV;
+	}
+
+	ret = of_property_read_u32(client->adapter->dev.of_node,
+				   "clock-frequency", &bus_clk_rate);
+	if (ret) {
+		dev_err(dev, "of: failed to read clock-frequency property\n");
+		return ret;
+	}
+
+	if (bus_clk_rate > 1000000L) {
+		dev_err(dev, "%d exceeds maximum supported clock frequency (1MHz)\n",
+			bus_clk_rate);
+		return -EINVAL;
+	}
+
+	i2c_priv = devm_kmalloc(dev, sizeof(*i2c_priv), GFP_KERNEL);
+	if (!i2c_priv)
+		return -ENOMEM;
+
+	i2c_priv->client = client;
+	mutex_init(&i2c_priv->lock);
+
+	/*
+	 * WAKE_TOKEN_MAX_SIZE was calculated for the maximum bus_clk_rate -
+	 * 1MHz. The previous bus_clk_rate check ensures us that wake_token_sz
+	 * will always be smaller than or equal to WAKE_TOKEN_MAX_SIZE.
+	 */
+	i2c_priv->wake_token_sz = atmel_ecc_wake_token_sz(bus_clk_rate);
+
+	memset(i2c_priv->wake_token, 0, sizeof(i2c_priv->wake_token));
+
+	atomic_set(&i2c_priv->tfm_count, 0);
+
+	i2c_set_clientdata(client, i2c_priv);
+
+	ret = device_sanity_check(client);
+	if (ret)
+		return ret;
+
+	spin_lock(&driver_data.i2c_list_lock);
+	list_add_tail(&i2c_priv->i2c_client_list_node,
+		      &driver_data.i2c_client_list);
+	spin_unlock(&driver_data.i2c_list_lock);
+
+	ret = crypto_register_kpp(&atmel_ecdh);
+	if (ret) {
+		spin_lock(&driver_data.i2c_list_lock);
+		list_del(&i2c_priv->i2c_client_list_node);
+		spin_unlock(&driver_data.i2c_list_lock);
+
+		dev_err(dev, "%s alg registration failed\n",
+			atmel_ecdh.base.cra_driver_name);
+	} else {
+		dev_info(dev, "atmel ecc algorithms registered in /proc/crypto\n");
+	}
+
+	return ret;
+}
+
+static int atmel_ecc_remove(struct i2c_client *client)
+{
+	struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
+
+	/* Return EBUSY if i2c client already allocated. */
+	if (atomic_read(&i2c_priv->tfm_count)) {
+		dev_err(&client->dev, "Device is busy\n");
+		return -EBUSY;
+	}
+
+	crypto_unregister_kpp(&atmel_ecdh);
+
+	spin_lock(&driver_data.i2c_list_lock);
+	list_del(&i2c_priv->i2c_client_list_node);
+	spin_unlock(&driver_data.i2c_list_lock);
+
+	return 0;
+}
+
+#ifdef CONFIG_OF
+static const struct of_device_id atmel_ecc_dt_ids[] = {
+	{
+		.compatible = "atmel,atecc508a",
+	}, {
+		/* sentinel */
+	}
+};
+MODULE_DEVICE_TABLE(of, atmel_ecc_dt_ids);
+#endif
+
+static const struct i2c_device_id atmel_ecc_id[] = {
+	{ "atecc508a", 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, atmel_ecc_id);
+
+static struct i2c_driver atmel_ecc_driver = {
+	.driver = {
+		.name	= "atmel-ecc",
+		.of_match_table = of_match_ptr(atmel_ecc_dt_ids),
+	},
+	.probe		= atmel_ecc_probe,
+	.remove		= atmel_ecc_remove,
+	.id_table	= atmel_ecc_id,
+};
+
+static int __init atmel_ecc_init(void)
+{
+	spin_lock_init(&driver_data.i2c_list_lock);
+	INIT_LIST_HEAD(&driver_data.i2c_client_list);
+	return i2c_add_driver(&atmel_ecc_driver);
+}
+
+static void __exit atmel_ecc_exit(void)
+{
+	flush_scheduled_work();
+	i2c_del_driver(&atmel_ecc_driver);
+}
+
+module_init(atmel_ecc_init);
+module_exit(atmel_ecc_exit);
+
+MODULE_AUTHOR("Tudor Ambarus <tudor.ambarus@microchip.com>");
+MODULE_DESCRIPTION("Microchip / Atmel ECC (I2C) driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/crypto/atmel-ecc.h b/drivers/crypto/atmel-ecc.h
new file mode 100644
index 000000000000..25232c8abcc2
--- /dev/null
+++ b/drivers/crypto/atmel-ecc.h
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2017, Microchip Technology Inc.
+ * Author: Tudor Ambarus <tudor.ambarus@microchip.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef __ATMEL_ECC_H__
+#define __ATMEL_ECC_H__
+
+#define ATMEL_ECC_PRIORITY		300
+
+#define COMMAND				0x03 /* packet function */
+#define SLEEP_TOKEN			0x01
+#define WAKE_TOKEN_MAX_SIZE		8
+
+/* Definitions of Data and Command sizes */
+#define WORD_ADDR_SIZE			1
+#define COUNT_SIZE			1
+#define CRC_SIZE			2
+#define CMD_OVERHEAD_SIZE		(COUNT_SIZE + CRC_SIZE)
+
+/* size in bytes of the n prime */
+#define ATMEL_ECC_NIST_P256_N_SIZE	32
+#define ATMEL_ECC_PUBKEY_SIZE		(2 * ATMEL_ECC_NIST_P256_N_SIZE)
+
+#define STATUS_RSP_SIZE			4
+#define ECDH_RSP_SIZE			(32 + CMD_OVERHEAD_SIZE)
+#define GENKEY_RSP_SIZE			(ATMEL_ECC_PUBKEY_SIZE + \
+					 CMD_OVERHEAD_SIZE)
+#define READ_RSP_SIZE			(4 + CMD_OVERHEAD_SIZE)
+#define MAX_RSP_SIZE			GENKEY_RSP_SIZE
+
+/**
+ * atmel_ecc_cmd - structure used for communicating with the device.
+ * @word_addr: indicates the function of the packet sent to the device. This
+ *             byte should have a value of COMMAND for normal operation.
+ * @count    : number of bytes to be transferred to (or from) the device.
+ * @opcode   : the command code.
+ * @param1   : the first parameter; always present.
+ * @param2   : the second parameter; always present.
+ * @data     : optional remaining input data. Includes a 2-byte CRC.
+ * @rxsize   : size of the data received from i2c client.
+ * @msecs    : command execution time in milliseconds
+ */
+struct atmel_ecc_cmd {
+	u8 word_addr;
+	u8 count;
+	u8 opcode;
+	u8 param1;
+	u16 param2;
+	u8 data[MAX_RSP_SIZE];
+	u8 msecs;
+	u16 rxsize;
+} __packed;
+
+/* Status/Error codes */
+#define STATUS_SIZE			0x04
+#define STATUS_NOERR			0x00
+#define STATUS_WAKE_SUCCESSFUL		0x11
+
+static const struct {
+	u8 value;
+	const char *error_text;
+} error_list[] = {
+	{ 0x01, "CheckMac or Verify miscompare" },
+	{ 0x03, "Parse Error" },
+	{ 0x05, "ECC Fault" },
+	{ 0x0F, "Execution Error" },
+	{ 0xEE, "Watchdog about to expire" },
+	{ 0xFF, "CRC or other communication error" },
+};
+
+/* Definitions for eeprom organization */
+#define CONFIG_ZONE			0
+
+/* Definitions for Indexes common to all commands */
+#define RSP_DATA_IDX			1 /* buffer index of data in response */
+#define DATA_SLOT_2			2 /* used for ECDH private key */
+
+/* Definitions for the device lock state */
+#define DEVICE_LOCK_ADDR		0x15
+#define LOCK_VALUE_IDX			(RSP_DATA_IDX + 2)
+#define LOCK_CONFIG_IDX			(RSP_DATA_IDX + 3)
+
+/*
+ * Wake High delay to data communication (microseconds). SDA should be stable
+ * high for this entire duration.
+ */
+#define TWHI_MIN			1500
+#define TWHI_MAX			1550
+
+/* Wake Low duration */
+#define TWLO_USEC			60
+
+/* Command execution time (milliseconds) */
+#define MAX_EXEC_TIME_ECDH		58
+#define MAX_EXEC_TIME_GENKEY		115
+#define MAX_EXEC_TIME_READ		1
+
+/* Command opcode */
+#define OPCODE_ECDH			0x43
+#define OPCODE_GENKEY			0x40
+#define OPCODE_READ			0x02
+
+/* Definitions for the READ Command */
+#define READ_COUNT			7
+
+/* Definitions for the GenKey Command */
+#define GENKEY_COUNT			7
+#define GENKEY_MODE_PRIVATE		0x04
+
+/* Definitions for the ECDH Command */
+#define ECDH_COUNT			71
+#define ECDH_PREFIX_MODE		0x00
+
+#endif /* __ATMEL_ECC_H__ */
diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c
index dad4e5bad827..3e2f41b3eaf3 100644
--- a/drivers/crypto/atmel-sha.c
+++ b/drivers/crypto/atmel-sha.c
@@ -2883,7 +2883,7 @@ sha_dd_err:
 
 static int atmel_sha_remove(struct platform_device *pdev)
 {
-	static struct atmel_sha_dev *sha_dd;
+	struct atmel_sha_dev *sha_dd;
 
 	sha_dd = platform_get_drvdata(pdev);
 	if (!sha_dd)
diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c
index b25f1b3c981f..f4b335dda568 100644
--- a/drivers/crypto/atmel-tdes.c
+++ b/drivers/crypto/atmel-tdes.c
@@ -1487,7 +1487,7 @@ tdes_dd_err:
 
 static int atmel_tdes_remove(struct platform_device *pdev)
 {
-	static struct atmel_tdes_dev *tdes_dd;
+	struct atmel_tdes_dev *tdes_dd;
 
 	tdes_dd = platform_get_drvdata(pdev);
 	if (!tdes_dd)
diff --git a/drivers/crypto/axis/Makefile b/drivers/crypto/axis/Makefile
new file mode 100644
index 000000000000..be9a84a4b667
--- /dev/null
+++ b/drivers/crypto/axis/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_CRYPTO_DEV_ARTPEC6) := artpec6_crypto.o
diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c
new file mode 100644
index 000000000000..d9fbbf01062b
--- /dev/null
+++ b/drivers/crypto/axis/artpec6_crypto.c
@@ -0,0 +1,3192 @@
+/*
+ *   Driver for ARTPEC-6 crypto block using the kernel asynchronous crypto api.
+ *
+ *    Copyright (C) 2014-2017  Axis Communications AB
+ */
+#define pr_fmt(fmt)     KBUILD_MODNAME ": " fmt
+
+#include <linux/bitfield.h>
+#include <linux/crypto.h>
+#include <linux/debugfs.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/fault-inject.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+
+#include <crypto/aes.h>
+#include <crypto/internal/aead.h>
+#include <crypto/internal/hash.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/scatterwalk.h>
+#include <crypto/sha.h>
+#include <crypto/xts.h>
+
+/* Max length of a line in all cache levels for Artpec SoCs. */
+#define ARTPEC_CACHE_LINE_MAX	32
+
+#define PDMA_OUT_CFG		0x0000
+#define PDMA_OUT_BUF_CFG	0x0004
+#define PDMA_OUT_CMD		0x0008
+#define PDMA_OUT_DESCRQ_PUSH	0x0010
+#define PDMA_OUT_DESCRQ_STAT	0x0014
+
+#define A6_PDMA_IN_CFG		0x0028
+#define A6_PDMA_IN_BUF_CFG	0x002c
+#define A6_PDMA_IN_CMD		0x0030
+#define A6_PDMA_IN_STATQ_PUSH	0x0038
+#define A6_PDMA_IN_DESCRQ_PUSH	0x0044
+#define A6_PDMA_IN_DESCRQ_STAT	0x0048
+#define A6_PDMA_INTR_MASK	0x0068
+#define A6_PDMA_ACK_INTR	0x006c
+#define A6_PDMA_MASKED_INTR	0x0074
+
+#define A7_PDMA_IN_CFG		0x002c
+#define A7_PDMA_IN_BUF_CFG	0x0030
+#define A7_PDMA_IN_CMD		0x0034
+#define A7_PDMA_IN_STATQ_PUSH	0x003c
+#define A7_PDMA_IN_DESCRQ_PUSH	0x0048
+#define A7_PDMA_IN_DESCRQ_STAT	0x004C
+#define A7_PDMA_INTR_MASK	0x006c
+#define A7_PDMA_ACK_INTR	0x0070
+#define A7_PDMA_MASKED_INTR	0x0078
+
+#define PDMA_OUT_CFG_EN				BIT(0)
+
+#define PDMA_OUT_BUF_CFG_DATA_BUF_SIZE		GENMASK(4, 0)
+#define PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE		GENMASK(9, 5)
+
+#define PDMA_OUT_CMD_START			BIT(0)
+#define A6_PDMA_OUT_CMD_STOP			BIT(3)
+#define A7_PDMA_OUT_CMD_STOP			BIT(2)
+
+#define PDMA_OUT_DESCRQ_PUSH_LEN		GENMASK(5, 0)
+#define PDMA_OUT_DESCRQ_PUSH_ADDR		GENMASK(31, 6)
+
+#define PDMA_OUT_DESCRQ_STAT_LEVEL		GENMASK(3, 0)
+#define PDMA_OUT_DESCRQ_STAT_SIZE		GENMASK(7, 4)
+
+#define PDMA_IN_CFG_EN				BIT(0)
+
+#define PDMA_IN_BUF_CFG_DATA_BUF_SIZE		GENMASK(4, 0)
+#define PDMA_IN_BUF_CFG_DESCR_BUF_SIZE		GENMASK(9, 5)
+#define PDMA_IN_BUF_CFG_STAT_BUF_SIZE		GENMASK(14, 10)
+
+#define PDMA_IN_CMD_START			BIT(0)
+#define A6_PDMA_IN_CMD_FLUSH_STAT		BIT(2)
+#define A6_PDMA_IN_CMD_STOP			BIT(3)
+#define A7_PDMA_IN_CMD_FLUSH_STAT		BIT(1)
+#define A7_PDMA_IN_CMD_STOP			BIT(2)
+
+#define PDMA_IN_STATQ_PUSH_LEN			GENMASK(5, 0)
+#define PDMA_IN_STATQ_PUSH_ADDR			GENMASK(31, 6)
+
+#define PDMA_IN_DESCRQ_PUSH_LEN			GENMASK(5, 0)
+#define PDMA_IN_DESCRQ_PUSH_ADDR		GENMASK(31, 6)
+
+#define PDMA_IN_DESCRQ_STAT_LEVEL		GENMASK(3, 0)
+#define PDMA_IN_DESCRQ_STAT_SIZE		GENMASK(7, 4)
+
+#define A6_PDMA_INTR_MASK_IN_DATA		BIT(2)
+#define A6_PDMA_INTR_MASK_IN_EOP		BIT(3)
+#define A6_PDMA_INTR_MASK_IN_EOP_FLUSH		BIT(4)
+
+#define A7_PDMA_INTR_MASK_IN_DATA		BIT(3)
+#define A7_PDMA_INTR_MASK_IN_EOP		BIT(4)
+#define A7_PDMA_INTR_MASK_IN_EOP_FLUSH		BIT(5)
+
+#define A6_CRY_MD_OPER		GENMASK(19, 16)
+
+#define A6_CRY_MD_HASH_SEL_CTX	GENMASK(21, 20)
+#define A6_CRY_MD_HASH_HMAC_FIN	BIT(23)
+
+#define A6_CRY_MD_CIPHER_LEN	GENMASK(21, 20)
+#define A6_CRY_MD_CIPHER_DECR	BIT(22)
+#define A6_CRY_MD_CIPHER_TWEAK	BIT(23)
+#define A6_CRY_MD_CIPHER_DSEQ	BIT(24)
+
+#define A7_CRY_MD_OPER		GENMASK(11, 8)
+
+#define A7_CRY_MD_HASH_SEL_CTX	GENMASK(13, 12)
+#define A7_CRY_MD_HASH_HMAC_FIN	BIT(15)
+
+#define A7_CRY_MD_CIPHER_LEN	GENMASK(13, 12)
+#define A7_CRY_MD_CIPHER_DECR	BIT(14)
+#define A7_CRY_MD_CIPHER_TWEAK	BIT(15)
+#define A7_CRY_MD_CIPHER_DSEQ	BIT(16)
+
+/* DMA metadata constants */
+#define regk_crypto_aes_cbc     0x00000002
+#define regk_crypto_aes_ctr     0x00000003
+#define regk_crypto_aes_ecb     0x00000001
+#define regk_crypto_aes_gcm     0x00000004
+#define regk_crypto_aes_xts     0x00000005
+#define regk_crypto_cache       0x00000002
+#define a6_regk_crypto_dlkey    0x0000000a
+#define a7_regk_crypto_dlkey    0x0000000e
+#define regk_crypto_ext         0x00000001
+#define regk_crypto_hmac_sha1   0x00000007
+#define regk_crypto_hmac_sha256 0x00000009
+#define regk_crypto_hmac_sha384 0x0000000b
+#define regk_crypto_hmac_sha512 0x0000000d
+#define regk_crypto_init        0x00000000
+#define regk_crypto_key_128     0x00000000
+#define regk_crypto_key_192     0x00000001
+#define regk_crypto_key_256     0x00000002
+#define regk_crypto_null        0x00000000
+#define regk_crypto_sha1        0x00000006
+#define regk_crypto_sha256      0x00000008
+#define regk_crypto_sha384      0x0000000a
+#define regk_crypto_sha512      0x0000000c
+
+/* DMA descriptor structures */
+struct pdma_descr_ctrl  {
+	unsigned char short_descr : 1;
+	unsigned char pad1        : 1;
+	unsigned char eop         : 1;
+	unsigned char intr        : 1;
+	unsigned char short_len   : 3;
+	unsigned char pad2        : 1;
+} __packed;
+
+struct pdma_data_descr {
+	unsigned int len : 24;
+	unsigned int buf : 32;
+} __packed;
+
+struct pdma_short_descr {
+	unsigned char data[7];
+} __packed;
+
+struct pdma_descr {
+	struct pdma_descr_ctrl ctrl;
+	union {
+		struct pdma_data_descr   data;
+		struct pdma_short_descr  shrt;
+	};
+};
+
+struct pdma_stat_descr {
+	unsigned char pad1        : 1;
+	unsigned char pad2        : 1;
+	unsigned char eop         : 1;
+	unsigned char pad3        : 5;
+	unsigned int  len         : 24;
+};
+
+/* Each descriptor array can hold max 64 entries */
+#define PDMA_DESCR_COUNT	64
+
+#define MODULE_NAME   "Artpec-6 CA"
+
+/* Hash modes (including HMAC variants) */
+#define ARTPEC6_CRYPTO_HASH_SHA1	1
+#define ARTPEC6_CRYPTO_HASH_SHA256	2
+#define ARTPEC6_CRYPTO_HASH_SHA384	3
+#define ARTPEC6_CRYPTO_HASH_SHA512	4
+
+/* Crypto modes */
+#define ARTPEC6_CRYPTO_CIPHER_AES_ECB	1
+#define ARTPEC6_CRYPTO_CIPHER_AES_CBC	2
+#define ARTPEC6_CRYPTO_CIPHER_AES_CTR	3
+#define ARTPEC6_CRYPTO_CIPHER_AES_XTS	5
+
+/* The PDMA is a DMA-engine tightly coupled with a ciphering engine.
+ * It operates on a descriptor array with up to 64 descriptor entries.
+ * The arrays must be 64 byte aligned in memory.
+ *
+ * The ciphering unit has no registers and is completely controlled by
+ * a 4-byte metadata that is inserted at the beginning of each dma packet.
+ *
+ * A dma packet is a sequence of descriptors terminated by setting the .eop
+ * field in the final descriptor of the packet.
+ *
+ * Multiple packets are used for providing context data, key data and
+ * the plain/ciphertext.
+ *
+ *   PDMA Descriptors (Array)
+ *  +------+------+------+~~+-------+------+----
+ *  |  0   |  1   |  2   |~~| 11 EOP|  12  |  ....
+ *  +--+---+--+---+----+-+~~+-------+----+-+----
+ *     |      |        |       |         |
+ *     |      |        |       |         |
+ *   __|__  +-------++-------++-------+ +----+
+ *  | MD  | |Payload||Payload||Payload| | MD |
+ *  +-----+ +-------++-------++-------+ +----+
+ */
+
+struct artpec6_crypto_bounce_buffer {
+	struct list_head list;
+	size_t length;
+	struct scatterlist *sg;
+	size_t offset;
+	/* buf is aligned to ARTPEC_CACHE_LINE_MAX and
+	 * holds up to ARTPEC_CACHE_LINE_MAX bytes data.
+	 */
+	void *buf;
+};
+
+struct artpec6_crypto_dma_map {
+	dma_addr_t dma_addr;
+	size_t size;
+	enum dma_data_direction dir;
+};
+
+struct artpec6_crypto_dma_descriptors {
+	struct pdma_descr out[PDMA_DESCR_COUNT] __aligned(64);
+	struct pdma_descr in[PDMA_DESCR_COUNT] __aligned(64);
+	u32 stat[PDMA_DESCR_COUNT] __aligned(64);
+	struct list_head bounce_buffers;
+	/* Enough maps for all out/in buffers, and all three descr. arrays */
+	struct artpec6_crypto_dma_map maps[PDMA_DESCR_COUNT * 2 + 2];
+	dma_addr_t out_dma_addr;
+	dma_addr_t in_dma_addr;
+	dma_addr_t stat_dma_addr;
+	size_t out_cnt;
+	size_t in_cnt;
+	size_t map_count;
+};
+
+enum artpec6_crypto_variant {
+	ARTPEC6_CRYPTO,
+	ARTPEC7_CRYPTO,
+};
+
+struct artpec6_crypto {
+	void __iomem *base;
+	spinlock_t queue_lock;
+	struct list_head queue; /* waiting for pdma fifo space */
+	struct list_head pending; /* submitted to pdma fifo */
+	struct tasklet_struct task;
+	struct kmem_cache *dma_cache;
+	int pending_count;
+	struct timer_list timer;
+	enum artpec6_crypto_variant variant;
+	void *pad_buffer; /* cache-aligned block padding buffer */
+	void *zero_buffer;
+};
+
+enum artpec6_crypto_hash_flags {
+	HASH_FLAG_INIT_CTX = 2,
+	HASH_FLAG_UPDATE = 4,
+	HASH_FLAG_FINALIZE = 8,
+	HASH_FLAG_HMAC = 16,
+	HASH_FLAG_UPDATE_KEY = 32,
+};
+
+struct artpec6_crypto_req_common {
+	struct list_head list;
+	struct artpec6_crypto_dma_descriptors *dma;
+	struct crypto_async_request *req;
+	void (*complete)(struct crypto_async_request *req);
+	gfp_t gfp_flags;
+};
+
+struct artpec6_hash_request_context {
+	char partial_buffer[SHA512_BLOCK_SIZE];
+	char partial_buffer_out[SHA512_BLOCK_SIZE];
+	char key_buffer[SHA512_BLOCK_SIZE];
+	char pad_buffer[SHA512_BLOCK_SIZE + 32];
+	unsigned char digeststate[SHA512_DIGEST_SIZE];
+	size_t partial_bytes;
+	u64 digcnt;
+	u32 key_md;
+	u32 hash_md;
+	enum artpec6_crypto_hash_flags hash_flags;
+	struct artpec6_crypto_req_common common;
+};
+
+struct artpec6_hash_export_state {
+	char partial_buffer[SHA512_BLOCK_SIZE];
+	unsigned char digeststate[SHA512_DIGEST_SIZE];
+	size_t partial_bytes;
+	u64 digcnt;
+	int oper;
+	unsigned int hash_flags;
+};
+
+struct artpec6_hashalg_context {
+	char hmac_key[SHA512_BLOCK_SIZE];
+	size_t hmac_key_length;
+	struct crypto_shash *child_hash;
+};
+
+struct artpec6_crypto_request_context {
+	u32 cipher_md;
+	bool decrypt;
+	struct artpec6_crypto_req_common common;
+};
+
+struct artpec6_cryptotfm_context {
+	unsigned char aes_key[2*AES_MAX_KEY_SIZE];
+	size_t key_length;
+	u32 key_md;
+	int crypto_type;
+	struct crypto_skcipher *fallback;
+};
+
+struct artpec6_crypto_aead_hw_ctx {
+	__be64	aad_length_bits;
+	__be64  text_length_bits;
+	__u8	J0[AES_BLOCK_SIZE];
+};
+
+struct artpec6_crypto_aead_req_ctx {
+	struct artpec6_crypto_aead_hw_ctx hw_ctx;
+	u32 cipher_md;
+	bool decrypt;
+	struct artpec6_crypto_req_common common;
+	__u8 decryption_tag[AES_BLOCK_SIZE] ____cacheline_aligned;
+};
+
+/* The crypto framework makes it hard to avoid this global. */
+static struct device *artpec6_crypto_dev;
+
+static struct dentry *dbgfs_root;
+
+#ifdef CONFIG_FAULT_INJECTION
+static DECLARE_FAULT_ATTR(artpec6_crypto_fail_status_read);
+static DECLARE_FAULT_ATTR(artpec6_crypto_fail_dma_array_full);
+#endif
+
+enum {
+	ARTPEC6_CRYPTO_PREPARE_HASH_NO_START,
+	ARTPEC6_CRYPTO_PREPARE_HASH_START,
+};
+
+static int artpec6_crypto_prepare_aead(struct aead_request *areq);
+static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq);
+static int artpec6_crypto_prepare_hash(struct ahash_request *areq);
+
+static void
+artpec6_crypto_complete_crypto(struct crypto_async_request *req);
+static void
+artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req);
+static void
+artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req);
+static void
+artpec6_crypto_complete_aead(struct crypto_async_request *req);
+static void
+artpec6_crypto_complete_hash(struct crypto_async_request *req);
+
+static int
+artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common);
+
+static void
+artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common);
+
+struct artpec6_crypto_walk {
+	struct scatterlist *sg;
+	size_t offset;
+};
+
+static void artpec6_crypto_walk_init(struct artpec6_crypto_walk *awalk,
+				     struct scatterlist *sg)
+{
+	awalk->sg = sg;
+	awalk->offset = 0;
+}
+
+static size_t artpec6_crypto_walk_advance(struct artpec6_crypto_walk *awalk,
+					  size_t nbytes)
+{
+	while (nbytes && awalk->sg) {
+		size_t piece;
+
+		WARN_ON(awalk->offset > awalk->sg->length);
+
+		piece = min(nbytes, (size_t)awalk->sg->length - awalk->offset);
+		nbytes -= piece;
+		awalk->offset += piece;
+		if (awalk->offset == awalk->sg->length) {
+			awalk->sg = sg_next(awalk->sg);
+			awalk->offset = 0;
+		}
+
+	}
+
+	return nbytes;
+}
+
+static size_t
+artpec6_crypto_walk_chunklen(const struct artpec6_crypto_walk *awalk)
+{
+	WARN_ON(awalk->sg->length == awalk->offset);
+
+	return awalk->sg->length - awalk->offset;
+}
+
+static dma_addr_t
+artpec6_crypto_walk_chunk_phys(const struct artpec6_crypto_walk *awalk)
+{
+	return sg_phys(awalk->sg) + awalk->offset;
+}
+
+static void
+artpec6_crypto_copy_bounce_buffers(struct artpec6_crypto_req_common *common)
+{
+	struct artpec6_crypto_dma_descriptors *dma = common->dma;
+	struct artpec6_crypto_bounce_buffer *b;
+	struct artpec6_crypto_bounce_buffer *next;
+
+	list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) {
+		pr_debug("bounce entry %p: %zu bytes @ %zu from %p\n",
+			 b, b->length, b->offset, b->buf);
+		sg_pcopy_from_buffer(b->sg,
+				   1,
+				   b->buf,
+				   b->length,
+				   b->offset);
+
+		list_del(&b->list);
+		kfree(b);
+	}
+}
+
+static inline bool artpec6_crypto_busy(void)
+{
+	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
+	int fifo_count = ac->pending_count;
+
+	return fifo_count > 6;
+}
+
+static int artpec6_crypto_submit(struct artpec6_crypto_req_common *req)
+{
+	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
+	int ret = -EBUSY;
+
+	spin_lock_bh(&ac->queue_lock);
+
+	if (!artpec6_crypto_busy()) {
+		list_add_tail(&req->list, &ac->pending);
+		artpec6_crypto_start_dma(req);
+		ret = -EINPROGRESS;
+	} else if (req->req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) {
+		list_add_tail(&req->list, &ac->queue);
+	} else {
+		artpec6_crypto_common_destroy(req);
+	}
+
+	spin_unlock_bh(&ac->queue_lock);
+
+	return ret;
+}
+
+static void artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common)
+{
+	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
+	enum artpec6_crypto_variant variant = ac->variant;
+	void __iomem *base = ac->base;
+	struct artpec6_crypto_dma_descriptors *dma = common->dma;
+	u32 ind, statd, outd;
+
+	/* Make descriptor content visible to the DMA before starting it. */
+	wmb();
+
+	ind = FIELD_PREP(PDMA_IN_DESCRQ_PUSH_LEN, dma->in_cnt - 1) |
+	      FIELD_PREP(PDMA_IN_DESCRQ_PUSH_ADDR, dma->in_dma_addr >> 6);
+
+	statd = FIELD_PREP(PDMA_IN_STATQ_PUSH_LEN, dma->in_cnt - 1) |
+		FIELD_PREP(PDMA_IN_STATQ_PUSH_ADDR, dma->stat_dma_addr >> 6);
+
+	outd = FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_LEN, dma->out_cnt - 1) |
+	       FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_ADDR, dma->out_dma_addr >> 6);
+
+	if (variant == ARTPEC6_CRYPTO) {
+		writel_relaxed(ind, base + A6_PDMA_IN_DESCRQ_PUSH);
+		writel_relaxed(statd, base + A6_PDMA_IN_STATQ_PUSH);
+		writel_relaxed(PDMA_IN_CMD_START, base + A6_PDMA_IN_CMD);
+	} else {
+		writel_relaxed(ind, base + A7_PDMA_IN_DESCRQ_PUSH);
+		writel_relaxed(statd, base + A7_PDMA_IN_STATQ_PUSH);
+		writel_relaxed(PDMA_IN_CMD_START, base + A7_PDMA_IN_CMD);
+	}
+
+	writel_relaxed(outd, base + PDMA_OUT_DESCRQ_PUSH);
+	writel_relaxed(PDMA_OUT_CMD_START, base + PDMA_OUT_CMD);
+
+	ac->pending_count++;
+}
+
+static void
+artpec6_crypto_init_dma_operation(struct artpec6_crypto_req_common *common)
+{
+	struct artpec6_crypto_dma_descriptors *dma = common->dma;
+
+	dma->out_cnt = 0;
+	dma->in_cnt = 0;
+	dma->map_count = 0;
+	INIT_LIST_HEAD(&dma->bounce_buffers);
+}
+
+static bool fault_inject_dma_descr(void)
+{
+#ifdef CONFIG_FAULT_INJECTION
+	return should_fail(&artpec6_crypto_fail_dma_array_full, 1);
+#else
+	return false;
+#endif
+}
+
+/** artpec6_crypto_setup_out_descr_phys - Setup an out channel with a
+ *                                        physical address
+ *
+ * @addr: The physical address of the data buffer
+ * @len:  The length of the data buffer
+ * @eop:  True if this is the last buffer in the packet
+ *
+ * @return 0 on success or -ENOSPC if there are no more descriptors available
+ */
+static int
+artpec6_crypto_setup_out_descr_phys(struct artpec6_crypto_req_common *common,
+				    dma_addr_t addr, size_t len, bool eop)
+{
+	struct artpec6_crypto_dma_descriptors *dma = common->dma;
+	struct pdma_descr *d;
+
+	if (dma->out_cnt >= PDMA_DESCR_COUNT ||
+	    fault_inject_dma_descr()) {
+		pr_err("No free OUT DMA descriptors available!\n");
+		return -ENOSPC;
+	}
+
+	d = &dma->out[dma->out_cnt++];
+	memset(d, 0, sizeof(*d));
+
+	d->ctrl.short_descr = 0;
+	d->ctrl.eop = eop;
+	d->data.len = len;
+	d->data.buf = addr;
+	return 0;
+}
+
+/** artpec6_crypto_setup_out_descr_short - Setup a short out descriptor
+ *
+ * @dst: The virtual address of the data
+ * @len: The length of the data, must be between 1 to 7 bytes
+ * @eop: True if this is the last buffer in the packet
+ *
+ * @return 0 on success
+ *	-ENOSPC if no more descriptors are available
+ *	-EINVAL if the data length exceeds 7 bytes
+ */
+static int
+artpec6_crypto_setup_out_descr_short(struct artpec6_crypto_req_common *common,
+				     void *dst, unsigned int len, bool eop)
+{
+	struct artpec6_crypto_dma_descriptors *dma = common->dma;
+	struct pdma_descr *d;
+
+	if (dma->out_cnt >= PDMA_DESCR_COUNT ||
+	    fault_inject_dma_descr()) {
+		pr_err("No free OUT DMA descriptors available!\n");
+		return -ENOSPC;
+	} else if (len > 7 || len < 1) {
+		return -EINVAL;
+	}
+	d = &dma->out[dma->out_cnt++];
+	memset(d, 0, sizeof(*d));
+
+	d->ctrl.short_descr = 1;
+	d->ctrl.short_len = len;
+	d->ctrl.eop = eop;
+	memcpy(d->shrt.data, dst, len);
+	return 0;
+}
+
+static int artpec6_crypto_dma_map_page(struct artpec6_crypto_req_common *common,
+				      struct page *page, size_t offset,
+				      size_t size,
+				      enum dma_data_direction dir,
+				      dma_addr_t *dma_addr_out)
+{
+	struct artpec6_crypto_dma_descriptors *dma = common->dma;
+	struct device *dev = artpec6_crypto_dev;
+	struct artpec6_crypto_dma_map *map;
+	dma_addr_t dma_addr;
+
+	*dma_addr_out = 0;
+
+	if (dma->map_count >= ARRAY_SIZE(dma->maps))
+		return -ENOMEM;
+
+	dma_addr = dma_map_page(dev, page, offset, size, dir);
+	if (dma_mapping_error(dev, dma_addr))
+		return -ENOMEM;
+
+	map = &dma->maps[dma->map_count++];
+	map->size = size;
+	map->dma_addr = dma_addr;
+	map->dir = dir;
+
+	*dma_addr_out = dma_addr;
+
+	return 0;
+}
+
+static int
+artpec6_crypto_dma_map_single(struct artpec6_crypto_req_common *common,
+			      void *ptr, size_t size,
+			      enum dma_data_direction dir,
+			      dma_addr_t *dma_addr_out)
+{
+	struct page *page = virt_to_page(ptr);
+	size_t offset = (uintptr_t)ptr & ~PAGE_MASK;
+
+	return artpec6_crypto_dma_map_page(common, page, offset, size, dir,
+					  dma_addr_out);
+}
+
+static int
+artpec6_crypto_dma_map_descs(struct artpec6_crypto_req_common *common)
+{
+	struct artpec6_crypto_dma_descriptors *dma = common->dma;
+	int ret;
+
+	ret = artpec6_crypto_dma_map_single(common, dma->in,
+				sizeof(dma->in[0]) * dma->in_cnt,
+				DMA_TO_DEVICE, &dma->in_dma_addr);
+	if (ret)
+		return ret;
+
+	ret = artpec6_crypto_dma_map_single(common, dma->out,
+				sizeof(dma->out[0]) * dma->out_cnt,
+				DMA_TO_DEVICE, &dma->out_dma_addr);
+	if (ret)
+		return ret;
+
+	/* We only read one stat descriptor */
+	dma->stat[dma->in_cnt - 1] = 0;
+
+	/*
+	 * DMA_BIDIRECTIONAL since we need our zeroing of the stat descriptor
+	 * to be written.
+	 */
+	return artpec6_crypto_dma_map_single(common,
+				dma->stat + dma->in_cnt - 1,
+				sizeof(dma->stat[0]),
+				DMA_BIDIRECTIONAL,
+				&dma->stat_dma_addr);
+}
+
+static void
+artpec6_crypto_dma_unmap_all(struct artpec6_crypto_req_common *common)
+{
+	struct artpec6_crypto_dma_descriptors *dma = common->dma;
+	struct device *dev = artpec6_crypto_dev;
+	int i;
+
+	for (i = 0; i < dma->map_count; i++) {
+		struct artpec6_crypto_dma_map *map = &dma->maps[i];
+
+		dma_unmap_page(dev, map->dma_addr, map->size, map->dir);
+	}
+
+	dma->map_count = 0;
+}
+
+/** artpec6_crypto_setup_out_descr - Setup an out descriptor
+ *
+ * @dst: The virtual address of the data
+ * @len: The length of the data
+ * @eop: True if this is the last buffer in the packet
+ * @use_short: If this is true and the data length is 7 bytes or less then
+ *	a short descriptor will be used
+ *
+ * @return 0 on success
+ *	Any errors from artpec6_crypto_setup_out_descr_short() or
+ *	setup_out_descr_phys()
+ */
+static int
+artpec6_crypto_setup_out_descr(struct artpec6_crypto_req_common *common,
+			       void *dst, unsigned int len, bool eop,
+			       bool use_short)
+{
+	if (use_short && len < 7) {
+		return artpec6_crypto_setup_out_descr_short(common, dst, len,
+							    eop);
+	} else {
+		int ret;
+		dma_addr_t dma_addr;
+
+		ret = artpec6_crypto_dma_map_single(common, dst, len,
+						   DMA_TO_DEVICE,
+						   &dma_addr);
+		if (ret)
+			return ret;
+
+		return artpec6_crypto_setup_out_descr_phys(common, dma_addr,
+							   len, eop);
+	}
+}
+
+/** artpec6_crypto_setup_in_descr_phys - Setup an in channel with a
+ *                                       physical address
+ *
+ * @addr: The physical address of the data buffer
+ * @len:  The length of the data buffer
+ * @intr: True if an interrupt should be fired after HW processing of this
+ *	  descriptor
+ *
+ */
+static int
+artpec6_crypto_setup_in_descr_phys(struct artpec6_crypto_req_common *common,
+			       dma_addr_t addr, unsigned int len, bool intr)
+{
+	struct artpec6_crypto_dma_descriptors *dma = common->dma;
+	struct pdma_descr *d;
+
+	if (dma->in_cnt >= PDMA_DESCR_COUNT ||
+	    fault_inject_dma_descr()) {
+		pr_err("No free IN DMA descriptors available!\n");
+		return -ENOSPC;
+	}
+	d = &dma->in[dma->in_cnt++];
+	memset(d, 0, sizeof(*d));
+
+	d->ctrl.intr = intr;
+	d->data.len = len;
+	d->data.buf = addr;
+	return 0;
+}
+
+/** artpec6_crypto_setup_in_descr - Setup an in channel descriptor
+ *
+ * @buffer: The virtual address to of the data buffer
+ * @len:    The length of the data buffer
+ * @last:   If this is the last data buffer in the request (i.e. an interrupt
+ *	    is needed
+ *
+ * Short descriptors are not used for the in channel
+ */
+static int
+artpec6_crypto_setup_in_descr(struct artpec6_crypto_req_common *common,
+			  void *buffer, unsigned int len, bool last)
+{
+	dma_addr_t dma_addr;
+	int ret;
+
+	ret = artpec6_crypto_dma_map_single(common, buffer, len,
+					   DMA_FROM_DEVICE, &dma_addr);
+	if (ret)
+		return ret;
+
+	return artpec6_crypto_setup_in_descr_phys(common, dma_addr, len, last);
+}
+
+static struct artpec6_crypto_bounce_buffer *
+artpec6_crypto_alloc_bounce(gfp_t flags)
+{
+	void *base;
+	size_t alloc_size = sizeof(struct artpec6_crypto_bounce_buffer) +
+			    2 * ARTPEC_CACHE_LINE_MAX;
+	struct artpec6_crypto_bounce_buffer *bbuf = kzalloc(alloc_size, flags);
+
+	if (!bbuf)
+		return NULL;
+
+	base = bbuf + 1;
+	bbuf->buf = PTR_ALIGN(base, ARTPEC_CACHE_LINE_MAX);
+	return bbuf;
+}
+
+static int setup_bounce_buffer_in(struct artpec6_crypto_req_common *common,
+				  struct artpec6_crypto_walk *walk, size_t size)
+{
+	struct artpec6_crypto_bounce_buffer *bbuf;
+	int ret;
+
+	bbuf = artpec6_crypto_alloc_bounce(common->gfp_flags);
+	if (!bbuf)
+		return -ENOMEM;
+
+	bbuf->length = size;
+	bbuf->sg = walk->sg;
+	bbuf->offset = walk->offset;
+
+	ret =  artpec6_crypto_setup_in_descr(common, bbuf->buf, size, false);
+	if (ret) {
+		kfree(bbuf);
+		return ret;
+	}
+
+	pr_debug("BOUNCE %zu offset %zu\n", size, walk->offset);
+	list_add_tail(&bbuf->list, &common->dma->bounce_buffers);
+	return 0;
+}
+
+static int
+artpec6_crypto_setup_sg_descrs_in(struct artpec6_crypto_req_common *common,
+				  struct artpec6_crypto_walk *walk,
+				  size_t count)
+{
+	size_t chunk;
+	int ret;
+	dma_addr_t addr;
+
+	while (walk->sg && count) {
+		chunk = min(count, artpec6_crypto_walk_chunklen(walk));
+		addr = artpec6_crypto_walk_chunk_phys(walk);
+
+		/* When destination buffers are not aligned to the cache line
+		 * size we need bounce buffers. The DMA-API requires that the
+		 * entire line is owned by the DMA buffer and this holds also
+		 * for the case when coherent DMA is used.
+		 */
+		if (!IS_ALIGNED(addr, ARTPEC_CACHE_LINE_MAX)) {
+			chunk = min_t(dma_addr_t, chunk,
+				      ALIGN(addr, ARTPEC_CACHE_LINE_MAX) -
+				      addr);
+
+			pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk);
+			ret = setup_bounce_buffer_in(common, walk, chunk);
+		} else if (chunk < ARTPEC_CACHE_LINE_MAX) {
+			pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk);
+			ret = setup_bounce_buffer_in(common, walk, chunk);
+		} else {
+			dma_addr_t dma_addr;
+
+			chunk = chunk & ~(ARTPEC_CACHE_LINE_MAX-1);
+
+			pr_debug("CHUNK %pad:%zu\n", &addr, chunk);
+
+			ret = artpec6_crypto_dma_map_page(common,
+							 sg_page(walk->sg),
+							 walk->sg->offset +
+							 walk->offset,
+							 chunk,
+							 DMA_FROM_DEVICE,
+							 &dma_addr);
+			if (ret)
+				return ret;
+
+			ret = artpec6_crypto_setup_in_descr_phys(common,
+								 dma_addr,
+								 chunk, false);
+		}
+
+		if (ret)
+			return ret;
+
+		count = count - chunk;
+		artpec6_crypto_walk_advance(walk, chunk);
+	}
+
+	if (count)
+		pr_err("EOL unexpected %zu bytes left\n", count);
+
+	return count ? -EINVAL : 0;
+}
+
+static int
+artpec6_crypto_setup_sg_descrs_out(struct artpec6_crypto_req_common *common,
+				   struct artpec6_crypto_walk *walk,
+				   size_t count)
+{
+	size_t chunk;
+	int ret;
+	dma_addr_t addr;
+
+	while (walk->sg && count) {
+		chunk = min(count, artpec6_crypto_walk_chunklen(walk));
+		addr = artpec6_crypto_walk_chunk_phys(walk);
+
+		pr_debug("OUT-CHUNK %pad:%zu\n", &addr, chunk);
+
+		if (addr & 3) {
+			char buf[3];
+
+			chunk = min_t(size_t, chunk, (4-(addr&3)));
+
+			sg_pcopy_to_buffer(walk->sg, 1, buf, chunk,
+					   walk->offset);
+
+			ret = artpec6_crypto_setup_out_descr_short(common, buf,
+								   chunk,
+								   false);
+		} else {
+			dma_addr_t dma_addr;
+
+			ret = artpec6_crypto_dma_map_page(common,
+							 sg_page(walk->sg),
+							 walk->sg->offset +
+							 walk->offset,
+							 chunk,
+							 DMA_TO_DEVICE,
+							 &dma_addr);
+			if (ret)
+				return ret;
+
+			ret = artpec6_crypto_setup_out_descr_phys(common,
+								 dma_addr,
+								 chunk, false);
+		}
+
+		if (ret)
+			return ret;
+
+		count = count - chunk;
+		artpec6_crypto_walk_advance(walk, chunk);
+	}
+
+	if (count)
+		pr_err("EOL unexpected %zu bytes left\n", count);
+
+	return count ? -EINVAL : 0;
+}
+
+
+/** artpec6_crypto_terminate_out_descrs - Set the EOP on the last out descriptor
+ *
+ * If the out descriptor list is non-empty, then the eop flag on the
+ * last used out descriptor will be set.
+ *
+ * @return  0 on success
+ *	-EINVAL if the out descriptor is empty or has overflown
+ */
+static int
+artpec6_crypto_terminate_out_descrs(struct artpec6_crypto_req_common *common)
+{
+	struct artpec6_crypto_dma_descriptors *dma = common->dma;
+	struct pdma_descr *d;
+
+	if (!dma->out_cnt || dma->out_cnt > PDMA_DESCR_COUNT) {
+		pr_err("%s: OUT descriptor list is %s\n",
+			MODULE_NAME, dma->out_cnt ? "empty" : "full");
+		return -EINVAL;
+
+	}
+
+	d = &dma->out[dma->out_cnt-1];
+	d->ctrl.eop = 1;
+
+	return 0;
+}
+
+/** artpec6_crypto_terminate_in_descrs - Set the interrupt flag on the last
+ *                                       in descriptor
+ *
+ * See artpec6_crypto_terminate_out_descrs() for return values
+ */
+static int
+artpec6_crypto_terminate_in_descrs(struct artpec6_crypto_req_common *common)
+{
+	struct artpec6_crypto_dma_descriptors *dma = common->dma;
+	struct pdma_descr *d;
+
+	if (!dma->in_cnt || dma->in_cnt > PDMA_DESCR_COUNT) {
+		pr_err("%s: IN descriptor list is %s\n",
+			MODULE_NAME, dma->in_cnt ? "empty" : "full");
+		return -EINVAL;
+	}
+
+	d = &dma->in[dma->in_cnt-1];
+	d->ctrl.intr = 1;
+	return 0;
+}
+
+/** create_hash_pad - Create a Secure Hash conformant pad
+ *
+ * @dst:      The destination buffer to write the pad. Must be at least 64 bytes
+ * @dgstlen:  The total length of the hash digest in bytes
+ * @bitcount: The total length of the digest in bits
+ *
+ * @return The total number of padding bytes written to @dst
+ */
+static size_t
+create_hash_pad(int oper, unsigned char *dst, u64 dgstlen, u64 bitcount)
+{
+	unsigned int mod, target, diff, pad_bytes, size_bytes;
+	__be64 bits = __cpu_to_be64(bitcount);
+
+	switch (oper) {
+	case regk_crypto_sha1:
+	case regk_crypto_sha256:
+	case regk_crypto_hmac_sha1:
+	case regk_crypto_hmac_sha256:
+		target = 448 / 8;
+		mod = 512 / 8;
+		size_bytes = 8;
+		break;
+	default:
+		target = 896 / 8;
+		mod = 1024 / 8;
+		size_bytes = 16;
+		break;
+	}
+
+	target -= 1;
+	diff = dgstlen & (mod - 1);
+	pad_bytes = diff > target ? target + mod - diff : target - diff;
+
+	memset(dst + 1, 0, pad_bytes);
+	dst[0] = 0x80;
+
+	if (size_bytes == 16) {
+		memset(dst + 1 + pad_bytes, 0, 8);
+		memcpy(dst + 1 + pad_bytes + 8, &bits, 8);
+	} else {
+		memcpy(dst + 1 + pad_bytes, &bits, 8);
+	}
+
+	return pad_bytes + size_bytes + 1;
+}
+
+static int artpec6_crypto_common_init(struct artpec6_crypto_req_common *common,
+		struct crypto_async_request *parent,
+		void (*complete)(struct crypto_async_request *req),
+		struct scatterlist *dstsg, unsigned int nbytes)
+{
+	gfp_t flags;
+	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
+
+	flags = (parent->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
+		 GFP_KERNEL : GFP_ATOMIC;
+
+	common->gfp_flags = flags;
+	common->dma = kmem_cache_alloc(ac->dma_cache, flags);
+	if (!common->dma)
+		return -ENOMEM;
+
+	common->req = parent;
+	common->complete = complete;
+	return 0;
+}
+
+static void
+artpec6_crypto_bounce_destroy(struct artpec6_crypto_dma_descriptors *dma)
+{
+	struct artpec6_crypto_bounce_buffer *b;
+	struct artpec6_crypto_bounce_buffer *next;
+
+	list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) {
+		kfree(b);
+	}
+}
+
+static int
+artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common)
+{
+	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
+
+	artpec6_crypto_dma_unmap_all(common);
+	artpec6_crypto_bounce_destroy(common->dma);
+	kmem_cache_free(ac->dma_cache, common->dma);
+	common->dma = NULL;
+	return 0;
+}
+
+/*
+ * Ciphering functions.
+ */
+static int artpec6_crypto_encrypt(struct skcipher_request *req)
+{
+	struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
+	struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher);
+	struct artpec6_crypto_request_context *req_ctx = NULL;
+	void (*complete)(struct crypto_async_request *req);
+	int ret;
+
+	req_ctx = skcipher_request_ctx(req);
+
+	switch (ctx->crypto_type) {
+	case ARTPEC6_CRYPTO_CIPHER_AES_CBC:
+	case ARTPEC6_CRYPTO_CIPHER_AES_ECB:
+	case ARTPEC6_CRYPTO_CIPHER_AES_XTS:
+		req_ctx->decrypt = 0;
+		break;
+	default:
+		break;
+	}
+
+	switch (ctx->crypto_type) {
+	case ARTPEC6_CRYPTO_CIPHER_AES_CBC:
+		complete = artpec6_crypto_complete_cbc_encrypt;
+		break;
+	default:
+		complete = artpec6_crypto_complete_crypto;
+		break;
+	}
+
+	ret = artpec6_crypto_common_init(&req_ctx->common,
+				  &req->base,
+				  complete,
+				  req->dst, req->cryptlen);
+	if (ret)
+		return ret;
+
+	ret = artpec6_crypto_prepare_crypto(req);
+	if (ret) {
+		artpec6_crypto_common_destroy(&req_ctx->common);
+		return ret;
+	}
+
+	return artpec6_crypto_submit(&req_ctx->common);
+}
+
+static int artpec6_crypto_decrypt(struct skcipher_request *req)
+{
+	int ret;
+	struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
+	struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher);
+	struct artpec6_crypto_request_context *req_ctx = NULL;
+	void (*complete)(struct crypto_async_request *req);
+
+	req_ctx = skcipher_request_ctx(req);
+
+	switch (ctx->crypto_type) {
+	case ARTPEC6_CRYPTO_CIPHER_AES_CBC:
+	case ARTPEC6_CRYPTO_CIPHER_AES_ECB:
+	case ARTPEC6_CRYPTO_CIPHER_AES_XTS:
+		req_ctx->decrypt = 1;
+		break;
+	default:
+		break;
+	}
+
+
+	switch (ctx->crypto_type) {
+	case ARTPEC6_CRYPTO_CIPHER_AES_CBC:
+		complete = artpec6_crypto_complete_cbc_decrypt;
+		break;
+	default:
+		complete = artpec6_crypto_complete_crypto;
+		break;
+	}
+
+	ret = artpec6_crypto_common_init(&req_ctx->common, &req->base,
+				  complete,
+				  req->dst, req->cryptlen);
+	if (ret)
+		return ret;
+
+	ret = artpec6_crypto_prepare_crypto(req);
+	if (ret) {
+		artpec6_crypto_common_destroy(&req_ctx->common);
+		return ret;
+	}
+
+	return artpec6_crypto_submit(&req_ctx->common);
+}
+
+static int
+artpec6_crypto_ctr_crypt(struct skcipher_request *req, bool encrypt)
+{
+	struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
+	struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher);
+	size_t iv_len = crypto_skcipher_ivsize(cipher);
+	unsigned int counter = be32_to_cpup((__be32 *)
+					    (req->iv + iv_len - 4));
+	unsigned int nblks = ALIGN(req->cryptlen, AES_BLOCK_SIZE) /
+			     AES_BLOCK_SIZE;
+
+	/*
+	 * The hardware uses only the last 32-bits as the counter while the
+	 * kernel tests (aes_ctr_enc_tv_template[4] for example) expect that
+	 * the whole IV is a counter.  So fallback if the counter is going to
+	 * overlow.
+	 */
+	if (counter + nblks < counter) {
+		int ret;
+
+		pr_debug("counter %x will overflow (nblks %u), falling back\n",
+			 counter, counter + nblks);
+
+		ret = crypto_skcipher_setkey(ctx->fallback, ctx->aes_key,
+					     ctx->key_length);
+		if (ret)
+			return ret;
+
+		{
+			SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
+
+			skcipher_request_set_tfm(subreq, ctx->fallback);
+			skcipher_request_set_callback(subreq, req->base.flags,
+						      NULL, NULL);
+			skcipher_request_set_crypt(subreq, req->src, req->dst,
+						   req->cryptlen, req->iv);
+			ret = encrypt ? crypto_skcipher_encrypt(subreq)
+				      : crypto_skcipher_decrypt(subreq);
+			skcipher_request_zero(subreq);
+		}
+		return ret;
+	}
+
+	return encrypt ? artpec6_crypto_encrypt(req)
+		       : artpec6_crypto_decrypt(req);
+}
+
+static int artpec6_crypto_ctr_encrypt(struct skcipher_request *req)
+{
+	return artpec6_crypto_ctr_crypt(req, true);
+}
+
+static int artpec6_crypto_ctr_decrypt(struct skcipher_request *req)
+{
+	return artpec6_crypto_ctr_crypt(req, false);
+}
+
+/*
+ * AEAD functions
+ */
+static int artpec6_crypto_aead_init(struct crypto_aead *tfm)
+{
+	struct artpec6_cryptotfm_context *tfm_ctx = crypto_aead_ctx(tfm);
+
+	memset(tfm_ctx, 0, sizeof(*tfm_ctx));
+
+	crypto_aead_set_reqsize(tfm,
+				sizeof(struct artpec6_crypto_aead_req_ctx));
+
+	return 0;
+}
+
+static int artpec6_crypto_aead_set_key(struct crypto_aead *tfm, const u8 *key,
+			       unsigned int len)
+{
+	struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(&tfm->base);
+
+	if (len != 16 && len != 24 && len != 32) {
+		crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+		return -1;
+	}
+
+	ctx->key_length = len;
+
+	memcpy(ctx->aes_key, key, len);
+	return 0;
+}
+
+static int artpec6_crypto_aead_encrypt(struct aead_request *req)
+{
+	int ret;
+	struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req);
+
+	req_ctx->decrypt = false;
+	ret = artpec6_crypto_common_init(&req_ctx->common, &req->base,
+				  artpec6_crypto_complete_aead,
+				  NULL, 0);
+	if (ret)
+		return ret;
+
+	ret = artpec6_crypto_prepare_aead(req);
+	if (ret) {
+		artpec6_crypto_common_destroy(&req_ctx->common);
+		return ret;
+	}
+
+	return artpec6_crypto_submit(&req_ctx->common);
+}
+
+static int artpec6_crypto_aead_decrypt(struct aead_request *req)
+{
+	int ret;
+	struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req);
+
+	req_ctx->decrypt = true;
+	if (req->cryptlen < AES_BLOCK_SIZE)
+		return -EINVAL;
+
+	ret = artpec6_crypto_common_init(&req_ctx->common,
+				  &req->base,
+				  artpec6_crypto_complete_aead,
+				  NULL, 0);
+	if (ret)
+		return ret;
+
+	ret = artpec6_crypto_prepare_aead(req);
+	if (ret) {
+		artpec6_crypto_common_destroy(&req_ctx->common);
+		return ret;
+	}
+
+	return artpec6_crypto_submit(&req_ctx->common);
+}
+
+static int artpec6_crypto_prepare_hash(struct ahash_request *areq)
+{
+	struct artpec6_hashalg_context *ctx = crypto_tfm_ctx(areq->base.tfm);
+	struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(areq);
+	size_t digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq));
+	size_t contextsize = digestsize == SHA384_DIGEST_SIZE ?
+		SHA512_DIGEST_SIZE : digestsize;
+	size_t blocksize = crypto_tfm_alg_blocksize(
+		crypto_ahash_tfm(crypto_ahash_reqtfm(areq)));
+	struct artpec6_crypto_req_common *common = &req_ctx->common;
+	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
+	enum artpec6_crypto_variant variant = ac->variant;
+	u32 sel_ctx;
+	bool ext_ctx = false;
+	bool run_hw = false;
+	int error = 0;
+
+	artpec6_crypto_init_dma_operation(common);
+
+	/* Upload HMAC key, must be first the first packet */
+	if (req_ctx->hash_flags & HASH_FLAG_HMAC) {
+		if (variant == ARTPEC6_CRYPTO) {
+			req_ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER,
+						     a6_regk_crypto_dlkey);
+		} else {
+			req_ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER,
+						     a7_regk_crypto_dlkey);
+		}
+
+		/* Copy and pad up the key */
+		memcpy(req_ctx->key_buffer, ctx->hmac_key,
+		       ctx->hmac_key_length);
+		memset(req_ctx->key_buffer + ctx->hmac_key_length, 0,
+		       blocksize - ctx->hmac_key_length);
+
+		error = artpec6_crypto_setup_out_descr(common,
+					(void *)&req_ctx->key_md,
+					sizeof(req_ctx->key_md), false, false);
+		if (error)
+			return error;
+
+		error = artpec6_crypto_setup_out_descr(common,
+					req_ctx->key_buffer, blocksize,
+					true, false);
+		if (error)
+			return error;
+	}
+
+	if (!(req_ctx->hash_flags & HASH_FLAG_INIT_CTX)) {
+		/* Restore context */
+		sel_ctx = regk_crypto_ext;
+		ext_ctx = true;
+	} else {
+		sel_ctx = regk_crypto_init;
+	}
+
+	if (variant == ARTPEC6_CRYPTO) {
+		req_ctx->hash_md &= ~A6_CRY_MD_HASH_SEL_CTX;
+		req_ctx->hash_md |= FIELD_PREP(A6_CRY_MD_HASH_SEL_CTX, sel_ctx);
+
+		/* If this is the final round, set the final flag */
+		if (req_ctx->hash_flags & HASH_FLAG_FINALIZE)
+			req_ctx->hash_md |= A6_CRY_MD_HASH_HMAC_FIN;
+	} else {
+		req_ctx->hash_md &= ~A7_CRY_MD_HASH_SEL_CTX;
+		req_ctx->hash_md |= FIELD_PREP(A7_CRY_MD_HASH_SEL_CTX, sel_ctx);
+
+		/* If this is the final round, set the final flag */
+		if (req_ctx->hash_flags & HASH_FLAG_FINALIZE)
+			req_ctx->hash_md |= A7_CRY_MD_HASH_HMAC_FIN;
+	}
+
+	/* Setup up metadata descriptors */
+	error = artpec6_crypto_setup_out_descr(common,
+				(void *)&req_ctx->hash_md,
+				sizeof(req_ctx->hash_md), false, false);
+	if (error)
+		return error;
+
+	error = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false);
+	if (error)
+		return error;
+
+	if (ext_ctx) {
+		error = artpec6_crypto_setup_out_descr(common,
+					req_ctx->digeststate,
+					contextsize, false, false);
+
+		if (error)
+			return error;
+	}
+
+	if (req_ctx->hash_flags & HASH_FLAG_UPDATE) {
+		size_t done_bytes = 0;
+		size_t total_bytes = areq->nbytes + req_ctx->partial_bytes;
+		size_t ready_bytes = round_down(total_bytes, blocksize);
+		struct artpec6_crypto_walk walk;
+
+		run_hw = ready_bytes > 0;
+		if (req_ctx->partial_bytes && ready_bytes) {
+			/* We have a partial buffer and will at least some bytes
+			 * to the HW. Empty this partial buffer before tackling
+			 * the SG lists
+			 */
+			memcpy(req_ctx->partial_buffer_out,
+				req_ctx->partial_buffer,
+				req_ctx->partial_bytes);
+
+			error = artpec6_crypto_setup_out_descr(common,
+						req_ctx->partial_buffer_out,
+						req_ctx->partial_bytes,
+						false, true);
+			if (error)
+				return error;
+
+			/* Reset partial buffer */
+			done_bytes += req_ctx->partial_bytes;
+			req_ctx->partial_bytes = 0;
+		}
+
+		artpec6_crypto_walk_init(&walk, areq->src);
+
+		error = artpec6_crypto_setup_sg_descrs_out(common, &walk,
+							   ready_bytes -
+							   done_bytes);
+		if (error)
+			return error;
+
+		if (walk.sg) {
+			size_t sg_skip = ready_bytes - done_bytes;
+			size_t sg_rem = areq->nbytes - sg_skip;
+
+			sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
+					   req_ctx->partial_buffer +
+					   req_ctx->partial_bytes,
+					   sg_rem, sg_skip);
+
+			req_ctx->partial_bytes += sg_rem;
+		}
+
+		req_ctx->digcnt += ready_bytes;
+		req_ctx->hash_flags &= ~(HASH_FLAG_UPDATE);
+	}
+
+	/* Finalize */
+	if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) {
+		bool needtrim = contextsize != digestsize;
+		size_t hash_pad_len;
+		u64 digest_bits;
+		u32 oper;
+
+		if (variant == ARTPEC6_CRYPTO)
+			oper = FIELD_GET(A6_CRY_MD_OPER, req_ctx->hash_md);
+		else
+			oper = FIELD_GET(A7_CRY_MD_OPER, req_ctx->hash_md);
+
+		/* Write out the partial buffer if present */
+		if (req_ctx->partial_bytes) {
+			memcpy(req_ctx->partial_buffer_out,
+			       req_ctx->partial_buffer,
+			       req_ctx->partial_bytes);
+			error = artpec6_crypto_setup_out_descr(common,
+						req_ctx->partial_buffer_out,
+						req_ctx->partial_bytes,
+						false, true);
+			if (error)
+				return error;
+
+			req_ctx->digcnt += req_ctx->partial_bytes;
+			req_ctx->partial_bytes = 0;
+		}
+
+		if (req_ctx->hash_flags & HASH_FLAG_HMAC)
+			digest_bits = 8 * (req_ctx->digcnt + blocksize);
+		else
+			digest_bits = 8 * req_ctx->digcnt;
+
+		/* Add the hash pad */
+		hash_pad_len = create_hash_pad(oper, req_ctx->pad_buffer,
+					       req_ctx->digcnt, digest_bits);
+		error = artpec6_crypto_setup_out_descr(common,
+						      req_ctx->pad_buffer,
+						      hash_pad_len, false,
+						      true);
+		req_ctx->digcnt = 0;
+
+		if (error)
+			return error;
+
+		/* Descriptor for the final result */
+		error = artpec6_crypto_setup_in_descr(common, areq->result,
+						      digestsize,
+						      !needtrim);
+		if (error)
+			return error;
+
+		if (needtrim) {
+			/* Discard the extra context bytes for SHA-384 */
+			error = artpec6_crypto_setup_in_descr(common,
+					req_ctx->partial_buffer,
+					digestsize - contextsize, true);
+			if (error)
+				return error;
+		}
+
+	} else { /* This is not the final operation for this request */
+		if (!run_hw)
+			return ARTPEC6_CRYPTO_PREPARE_HASH_NO_START;
+
+		/* Save the result to the context */
+		error = artpec6_crypto_setup_in_descr(common,
+						      req_ctx->digeststate,
+						      contextsize, false);
+		if (error)
+			return error;
+		/* fall through */
+	}
+
+	req_ctx->hash_flags &= ~(HASH_FLAG_INIT_CTX | HASH_FLAG_UPDATE |
+				 HASH_FLAG_FINALIZE);
+
+	error = artpec6_crypto_terminate_in_descrs(common);
+	if (error)
+		return error;
+
+	error = artpec6_crypto_terminate_out_descrs(common);
+	if (error)
+		return error;
+
+	error = artpec6_crypto_dma_map_descs(common);
+	if (error)
+		return error;
+
+	return ARTPEC6_CRYPTO_PREPARE_HASH_START;
+}
+
+
+static int artpec6_crypto_aes_ecb_init(struct crypto_skcipher *tfm)
+{
+	struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
+
+	tfm->reqsize = sizeof(struct artpec6_crypto_request_context);
+	ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_ECB;
+
+	return 0;
+}
+
+static int artpec6_crypto_aes_ctr_init(struct crypto_skcipher *tfm)
+{
+	struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
+
+	ctx->fallback = crypto_alloc_skcipher(crypto_tfm_alg_name(&tfm->base),
+					      0,
+					      CRYPTO_ALG_ASYNC |
+					      CRYPTO_ALG_NEED_FALLBACK);
+	if (IS_ERR(ctx->fallback))
+		return PTR_ERR(ctx->fallback);
+
+	tfm->reqsize = sizeof(struct artpec6_crypto_request_context);
+	ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CTR;
+
+	return 0;
+}
+
+static int artpec6_crypto_aes_cbc_init(struct crypto_skcipher *tfm)
+{
+	struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
+
+	tfm->reqsize = sizeof(struct artpec6_crypto_request_context);
+	ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CBC;
+
+	return 0;
+}
+
+static int artpec6_crypto_aes_xts_init(struct crypto_skcipher *tfm)
+{
+	struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
+
+	tfm->reqsize = sizeof(struct artpec6_crypto_request_context);
+	ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_XTS;
+
+	return 0;
+}
+
+static void artpec6_crypto_aes_exit(struct crypto_skcipher *tfm)
+{
+	struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
+
+	memset(ctx, 0, sizeof(*ctx));
+}
+
+static void artpec6_crypto_aes_ctr_exit(struct crypto_skcipher *tfm)
+{
+	struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
+
+	crypto_free_skcipher(ctx->fallback);
+	artpec6_crypto_aes_exit(tfm);
+}
+
+static int
+artpec6_crypto_cipher_set_key(struct crypto_skcipher *cipher, const u8 *key,
+			      unsigned int keylen)
+{
+	struct artpec6_cryptotfm_context *ctx =
+		crypto_skcipher_ctx(cipher);
+
+	switch (keylen) {
+	case 16:
+	case 24:
+	case 32:
+		break;
+	default:
+		crypto_skcipher_set_flags(cipher,
+					  CRYPTO_TFM_RES_BAD_KEY_LEN);
+		return -EINVAL;
+	}
+
+	memcpy(ctx->aes_key, key, keylen);
+	ctx->key_length = keylen;
+	return 0;
+}
+
+static int
+artpec6_crypto_xts_set_key(struct crypto_skcipher *cipher, const u8 *key,
+			      unsigned int keylen)
+{
+	struct artpec6_cryptotfm_context *ctx =
+		crypto_skcipher_ctx(cipher);
+	int ret;
+
+	ret = xts_check_key(&cipher->base, key, keylen);
+	if (ret)
+		return ret;
+
+	switch (keylen) {
+	case 32:
+	case 48:
+	case 64:
+		break;
+	default:
+		crypto_skcipher_set_flags(cipher,
+					  CRYPTO_TFM_RES_BAD_KEY_LEN);
+		return -EINVAL;
+	}
+
+	memcpy(ctx->aes_key, key, keylen);
+	ctx->key_length = keylen;
+	return 0;
+}
+
+/** artpec6_crypto_process_crypto - Prepare an async block cipher crypto request
+ *
+ * @req: The asynch request to process
+ *
+ * @return 0 if the dma job was successfully prepared
+ *	  <0 on error
+ *
+ * This function sets up the PDMA descriptors for a block cipher request.
+ *
+ * The required padding is added for AES-CTR using a statically defined
+ * buffer.
+ *
+ * The PDMA descriptor list will be as follows:
+ *
+ * OUT: [KEY_MD][KEY][EOP]<CIPHER_MD>[IV]<data_0>...[data_n][AES-CTR_pad]<eop>
+ * IN:  <CIPHER_MD><data_0>...[data_n]<intr>
+ *
+ */
+static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq)
+{
+	int ret;
+	struct artpec6_crypto_walk walk;
+	struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
+	struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher);
+	struct artpec6_crypto_request_context *req_ctx = NULL;
+	size_t iv_len = crypto_skcipher_ivsize(cipher);
+	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
+	enum artpec6_crypto_variant variant = ac->variant;
+	struct artpec6_crypto_req_common *common;
+	bool cipher_decr = false;
+	size_t cipher_klen;
+	u32 cipher_len = 0; /* Same as regk_crypto_key_128 for NULL crypto */
+	u32 oper;
+
+	req_ctx = skcipher_request_ctx(areq);
+	common = &req_ctx->common;
+
+	artpec6_crypto_init_dma_operation(common);
+
+	if (variant == ARTPEC6_CRYPTO)
+		ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, a6_regk_crypto_dlkey);
+	else
+		ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, a7_regk_crypto_dlkey);
+
+	ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md,
+					     sizeof(ctx->key_md), false, false);
+	if (ret)
+		return ret;
+
+	ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key,
+					      ctx->key_length, true, false);
+	if (ret)
+		return ret;
+
+	req_ctx->cipher_md = 0;
+
+	if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS)
+		cipher_klen = ctx->key_length/2;
+	else
+		cipher_klen =  ctx->key_length;
+
+	/* Metadata */
+	switch (cipher_klen) {
+	case 16:
+		cipher_len = regk_crypto_key_128;
+		break;
+	case 24:
+		cipher_len = regk_crypto_key_192;
+		break;
+	case 32:
+		cipher_len = regk_crypto_key_256;
+		break;
+	default:
+		pr_err("%s: Invalid key length %d!\n",
+			MODULE_NAME, ctx->key_length);
+		return -EINVAL;
+	}
+
+	switch (ctx->crypto_type) {
+	case ARTPEC6_CRYPTO_CIPHER_AES_ECB:
+		oper = regk_crypto_aes_ecb;
+		cipher_decr = req_ctx->decrypt;
+		break;
+
+	case ARTPEC6_CRYPTO_CIPHER_AES_CBC:
+		oper = regk_crypto_aes_cbc;
+		cipher_decr = req_ctx->decrypt;
+		break;
+
+	case ARTPEC6_CRYPTO_CIPHER_AES_CTR:
+		oper = regk_crypto_aes_ctr;
+		cipher_decr = false;
+		break;
+
+	case ARTPEC6_CRYPTO_CIPHER_AES_XTS:
+		oper = regk_crypto_aes_xts;
+		cipher_decr = req_ctx->decrypt;
+
+		if (variant == ARTPEC6_CRYPTO)
+			req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DSEQ;
+		else
+			req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DSEQ;
+		break;
+
+	default:
+		pr_err("%s: Invalid cipher mode %d!\n",
+			MODULE_NAME, ctx->crypto_type);
+		return -EINVAL;
+	}
+
+	if (variant == ARTPEC6_CRYPTO) {
+		req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER, oper);
+		req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN,
+						 cipher_len);
+		if (cipher_decr)
+			req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR;
+	} else {
+		req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER, oper);
+		req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN,
+						 cipher_len);
+		if (cipher_decr)
+			req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR;
+	}
+
+	ret = artpec6_crypto_setup_out_descr(common,
+					    &req_ctx->cipher_md,
+					    sizeof(req_ctx->cipher_md),
+					    false, false);
+	if (ret)
+		return ret;
+
+	ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false);
+	if (ret)
+		return ret;
+
+	if (iv_len) {
+		ret = artpec6_crypto_setup_out_descr(common, areq->iv, iv_len,
+						     false, false);
+		if (ret)
+			return ret;
+	}
+	/* Data out */
+	artpec6_crypto_walk_init(&walk, areq->src);
+	ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, areq->cryptlen);
+	if (ret)
+		return ret;
+
+	/* Data in */
+	artpec6_crypto_walk_init(&walk, areq->dst);
+	ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, areq->cryptlen);
+	if (ret)
+		return ret;
+
+	/* CTR-mode padding required by the HW. */
+	if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_CTR ||
+	    ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS) {
+		size_t pad = ALIGN(areq->cryptlen, AES_BLOCK_SIZE) -
+			     areq->cryptlen;
+
+		if (pad) {
+			ret = artpec6_crypto_setup_out_descr(common,
+							     ac->pad_buffer,
+							     pad, false, false);
+			if (ret)
+				return ret;
+
+			ret = artpec6_crypto_setup_in_descr(common,
+							    ac->pad_buffer, pad,
+							    false);
+			if (ret)
+				return ret;
+		}
+	}
+
+	ret = artpec6_crypto_terminate_out_descrs(common);
+	if (ret)
+		return ret;
+
+	ret = artpec6_crypto_terminate_in_descrs(common);
+	if (ret)
+		return ret;
+
+	return artpec6_crypto_dma_map_descs(common);
+}
+
+static int artpec6_crypto_prepare_aead(struct aead_request *areq)
+{
+	size_t count;
+	int ret;
+	size_t input_length;
+	struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(areq->base.tfm);
+	struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq);
+	struct crypto_aead *cipher = crypto_aead_reqtfm(areq);
+	struct artpec6_crypto_req_common *common = &req_ctx->common;
+	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
+	enum artpec6_crypto_variant variant = ac->variant;
+	u32 md_cipher_len;
+
+	artpec6_crypto_init_dma_operation(common);
+
+	/* Key */
+	if (variant == ARTPEC6_CRYPTO) {
+		ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER,
+					 a6_regk_crypto_dlkey);
+	} else {
+		ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER,
+					 a7_regk_crypto_dlkey);
+	}
+	ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md,
+					     sizeof(ctx->key_md), false, false);
+	if (ret)
+		return ret;
+
+	ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key,
+					     ctx->key_length, true, false);
+	if (ret)
+		return ret;
+
+	req_ctx->cipher_md = 0;
+
+	switch (ctx->key_length) {
+	case 16:
+		md_cipher_len = regk_crypto_key_128;
+		break;
+	case 24:
+		md_cipher_len = regk_crypto_key_192;
+		break;
+	case 32:
+		md_cipher_len = regk_crypto_key_256;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (variant == ARTPEC6_CRYPTO) {
+		req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER,
+						 regk_crypto_aes_gcm);
+		req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN,
+						 md_cipher_len);
+		if (req_ctx->decrypt)
+			req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR;
+	} else {
+		req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER,
+						 regk_crypto_aes_gcm);
+		req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN,
+						 md_cipher_len);
+		if (req_ctx->decrypt)
+			req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR;
+	}
+
+	ret = artpec6_crypto_setup_out_descr(common,
+					    (void *) &req_ctx->cipher_md,
+					    sizeof(req_ctx->cipher_md), false,
+					    false);
+	if (ret)
+		return ret;
+
+	ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false);
+	if (ret)
+		return ret;
+
+	/* For the decryption, cryptlen includes the tag. */
+	input_length = areq->cryptlen;
+	if (req_ctx->decrypt)
+		input_length -= AES_BLOCK_SIZE;
+
+	/* Prepare the context buffer */
+	req_ctx->hw_ctx.aad_length_bits =
+		__cpu_to_be64(8*areq->assoclen);
+
+	req_ctx->hw_ctx.text_length_bits =
+		__cpu_to_be64(8*input_length);
+
+	memcpy(req_ctx->hw_ctx.J0, areq->iv, crypto_aead_ivsize(cipher));
+	// The HW omits the initial increment of the counter field.
+	crypto_inc(req_ctx->hw_ctx.J0+12, 4);
+
+	ret = artpec6_crypto_setup_out_descr(common, &req_ctx->hw_ctx,
+		sizeof(struct artpec6_crypto_aead_hw_ctx), false, false);
+	if (ret)
+		return ret;
+
+	{
+		struct artpec6_crypto_walk walk;
+
+		artpec6_crypto_walk_init(&walk, areq->src);
+
+		/* Associated data */
+		count = areq->assoclen;
+		ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count);
+		if (ret)
+			return ret;
+
+		if (!IS_ALIGNED(areq->assoclen, 16)) {
+			size_t assoc_pad = 16 - (areq->assoclen % 16);
+			/* The HW mandates zero padding here */
+			ret = artpec6_crypto_setup_out_descr(common,
+							     ac->zero_buffer,
+							     assoc_pad, false,
+							     false);
+			if (ret)
+				return ret;
+		}
+
+		/* Data to crypto */
+		count = input_length;
+		ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count);
+		if (ret)
+			return ret;
+
+		if (!IS_ALIGNED(input_length, 16)) {
+			size_t crypto_pad = 16 - (input_length % 16);
+			/* The HW mandates zero padding here */
+			ret = artpec6_crypto_setup_out_descr(common,
+							     ac->zero_buffer,
+							     crypto_pad,
+							     false,
+							     false);
+			if (ret)
+				return ret;
+		}
+	}
+
+	/* Data from crypto */
+	{
+		struct artpec6_crypto_walk walk;
+		size_t output_len = areq->cryptlen;
+
+		if (req_ctx->decrypt)
+			output_len -= AES_BLOCK_SIZE;
+
+		artpec6_crypto_walk_init(&walk, areq->dst);
+
+		/* skip associated data in the output */
+		count = artpec6_crypto_walk_advance(&walk, areq->assoclen);
+		if (count)
+			return -EINVAL;
+
+		count = output_len;
+		ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, count);
+		if (ret)
+			return ret;
+
+		/* Put padding between the cryptotext and the auth tag */
+		if (!IS_ALIGNED(output_len, 16)) {
+			size_t crypto_pad = 16 - (output_len % 16);
+
+			ret = artpec6_crypto_setup_in_descr(common,
+							    ac->pad_buffer,
+							    crypto_pad, false);
+			if (ret)
+				return ret;
+		}
+
+		/* The authentication tag shall follow immediately after
+		 * the output ciphertext. For decryption it is put in a context
+		 * buffer for later compare against the input tag.
+		 */
+		count = AES_BLOCK_SIZE;
+
+		if (req_ctx->decrypt) {
+			ret = artpec6_crypto_setup_in_descr(common,
+				req_ctx->decryption_tag, count, false);
+			if (ret)
+				return ret;
+
+		} else {
+			ret = artpec6_crypto_setup_sg_descrs_in(common, &walk,
+								count);
+			if (ret)
+				return ret;
+		}
+
+	}
+
+	ret = artpec6_crypto_terminate_in_descrs(common);
+	if (ret)
+		return ret;
+
+	ret = artpec6_crypto_terminate_out_descrs(common);
+	if (ret)
+		return ret;
+
+	return artpec6_crypto_dma_map_descs(common);
+}
+
+static void artpec6_crypto_process_queue(struct artpec6_crypto *ac)
+{
+	struct artpec6_crypto_req_common *req;
+
+	while (!list_empty(&ac->queue) && !artpec6_crypto_busy()) {
+		req = list_first_entry(&ac->queue,
+				       struct artpec6_crypto_req_common,
+				       list);
+		list_move_tail(&req->list, &ac->pending);
+		artpec6_crypto_start_dma(req);
+
+		req->req->complete(req->req, -EINPROGRESS);
+	}
+
+	/*
+	 * In some cases, the hardware can raise an in_eop_flush interrupt
+	 * before actually updating the status, so we have an timer which will
+	 * recheck the status on timeout.  Since the cases are expected to be
+	 * very rare, we use a relatively large timeout value.  There should be
+	 * no noticeable negative effect if we timeout spuriously.
+	 */
+	if (ac->pending_count)
+		mod_timer(&ac->timer, jiffies + msecs_to_jiffies(100));
+	else
+		del_timer(&ac->timer);
+}
+
+static void artpec6_crypto_timeout(unsigned long data)
+{
+	struct artpec6_crypto *ac = (struct artpec6_crypto *) data;
+
+	dev_info_ratelimited(artpec6_crypto_dev, "timeout\n");
+
+	tasklet_schedule(&ac->task);
+}
+
+static void artpec6_crypto_task(unsigned long data)
+{
+	struct artpec6_crypto *ac = (struct artpec6_crypto *)data;
+	struct artpec6_crypto_req_common *req;
+	struct artpec6_crypto_req_common *n;
+
+	if (list_empty(&ac->pending)) {
+		pr_debug("Spurious IRQ\n");
+		return;
+	}
+
+	spin_lock_bh(&ac->queue_lock);
+
+	list_for_each_entry_safe(req, n, &ac->pending, list) {
+		struct artpec6_crypto_dma_descriptors *dma = req->dma;
+		u32 stat;
+
+		dma_sync_single_for_cpu(artpec6_crypto_dev, dma->stat_dma_addr,
+					sizeof(dma->stat[0]),
+					DMA_BIDIRECTIONAL);
+
+		stat = req->dma->stat[req->dma->in_cnt-1];
+
+		/* A non-zero final status descriptor indicates
+		 * this job has finished.
+		 */
+		pr_debug("Request %p status is %X\n", req, stat);
+		if (!stat)
+			break;
+
+		/* Allow testing of timeout handling with fault injection */
+#ifdef CONFIG_FAULT_INJECTION
+		if (should_fail(&artpec6_crypto_fail_status_read, 1))
+			continue;
+#endif
+
+		pr_debug("Completing request %p\n", req);
+
+		list_del(&req->list);
+
+		artpec6_crypto_dma_unmap_all(req);
+		artpec6_crypto_copy_bounce_buffers(req);
+
+		ac->pending_count--;
+		artpec6_crypto_common_destroy(req);
+		req->complete(req->req);
+	}
+
+	artpec6_crypto_process_queue(ac);
+
+	spin_unlock_bh(&ac->queue_lock);
+}
+
+static void artpec6_crypto_complete_crypto(struct crypto_async_request *req)
+{
+	req->complete(req, 0);
+}
+
+static void
+artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req)
+{
+	struct skcipher_request *cipher_req = container_of(req,
+		struct skcipher_request, base);
+
+	scatterwalk_map_and_copy(cipher_req->iv, cipher_req->src,
+				 cipher_req->cryptlen - AES_BLOCK_SIZE,
+				 AES_BLOCK_SIZE, 0);
+	req->complete(req, 0);
+}
+
+static void
+artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req)
+{
+	struct skcipher_request *cipher_req = container_of(req,
+		struct skcipher_request, base);
+
+	scatterwalk_map_and_copy(cipher_req->iv, cipher_req->dst,
+				 cipher_req->cryptlen - AES_BLOCK_SIZE,
+				 AES_BLOCK_SIZE, 0);
+	req->complete(req, 0);
+}
+
+static void artpec6_crypto_complete_aead(struct crypto_async_request *req)
+{
+	int result = 0;
+
+	/* Verify GCM hashtag. */
+	struct aead_request *areq = container_of(req,
+		struct aead_request, base);
+	struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq);
+
+	if (req_ctx->decrypt) {
+		u8 input_tag[AES_BLOCK_SIZE];
+
+		sg_pcopy_to_buffer(areq->src,
+				   sg_nents(areq->src),
+				   input_tag,
+				   AES_BLOCK_SIZE,
+				   areq->assoclen + areq->cryptlen -
+				   AES_BLOCK_SIZE);
+
+		if (memcmp(req_ctx->decryption_tag,
+			   input_tag,
+			   AES_BLOCK_SIZE)) {
+			pr_debug("***EBADMSG:\n");
+			print_hex_dump_debug("ref:", DUMP_PREFIX_ADDRESS, 32, 1,
+					     input_tag, AES_BLOCK_SIZE, true);
+			print_hex_dump_debug("out:", DUMP_PREFIX_ADDRESS, 32, 1,
+					     req_ctx->decryption_tag,
+					     AES_BLOCK_SIZE, true);
+
+			result = -EBADMSG;
+		}
+	}
+
+	req->complete(req, result);
+}
+
+static void artpec6_crypto_complete_hash(struct crypto_async_request *req)
+{
+	req->complete(req, 0);
+}
+
+
+/*------------------- Hash functions -----------------------------------------*/
+static int
+artpec6_crypto_hash_set_key(struct crypto_ahash *tfm,
+		    const u8 *key, unsigned int keylen)
+{
+	struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(&tfm->base);
+	size_t blocksize;
+	int ret;
+
+	if (!keylen) {
+		pr_err("Invalid length (%d) of HMAC key\n",
+			keylen);
+		return -EINVAL;
+	}
+
+	memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key));
+
+	blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
+
+	if (keylen > blocksize) {
+		SHASH_DESC_ON_STACK(hdesc, tfm_ctx->child_hash);
+
+		hdesc->tfm = tfm_ctx->child_hash;
+		hdesc->flags = crypto_ahash_get_flags(tfm) &
+			       CRYPTO_TFM_REQ_MAY_SLEEP;
+
+		tfm_ctx->hmac_key_length = blocksize;
+		ret = crypto_shash_digest(hdesc, key, keylen,
+					  tfm_ctx->hmac_key);
+		if (ret)
+			return ret;
+
+	} else {
+		memcpy(tfm_ctx->hmac_key, key, keylen);
+		tfm_ctx->hmac_key_length = keylen;
+	}
+
+	return 0;
+}
+
+static int
+artpec6_crypto_init_hash(struct ahash_request *req, u8 type, int hmac)
+{
+	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
+	enum artpec6_crypto_variant variant = ac->variant;
+	struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
+	u32 oper;
+
+	memset(req_ctx, 0, sizeof(*req_ctx));
+
+	req_ctx->hash_flags = HASH_FLAG_INIT_CTX;
+	if (hmac)
+		req_ctx->hash_flags |= (HASH_FLAG_HMAC | HASH_FLAG_UPDATE_KEY);
+
+	switch (type) {
+	case ARTPEC6_CRYPTO_HASH_SHA1:
+		oper = hmac ? regk_crypto_hmac_sha1 : regk_crypto_sha1;
+		break;
+	case ARTPEC6_CRYPTO_HASH_SHA256:
+		oper = hmac ? regk_crypto_hmac_sha256 : regk_crypto_sha256;
+		break;
+	case ARTPEC6_CRYPTO_HASH_SHA384:
+		oper = hmac ? regk_crypto_hmac_sha384 : regk_crypto_sha384;
+		break;
+	case ARTPEC6_CRYPTO_HASH_SHA512:
+		oper = hmac ? regk_crypto_hmac_sha512 : regk_crypto_sha512;
+		break;
+
+	default:
+		pr_err("%s: Unsupported hash type 0x%x\n", MODULE_NAME, type);
+		return -EINVAL;
+	}
+
+	if (variant == ARTPEC6_CRYPTO)
+		req_ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, oper);
+	else
+		req_ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, oper);
+
+	return 0;
+}
+
+static int artpec6_crypto_prepare_submit_hash(struct ahash_request *req)
+{
+	struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
+	int ret;
+
+	if (!req_ctx->common.dma) {
+		ret = artpec6_crypto_common_init(&req_ctx->common,
+					  &req->base,
+					  artpec6_crypto_complete_hash,
+					  NULL, 0);
+
+		if (ret)
+			return ret;
+	}
+
+	ret = artpec6_crypto_prepare_hash(req);
+	switch (ret) {
+	case ARTPEC6_CRYPTO_PREPARE_HASH_START:
+		ret = artpec6_crypto_submit(&req_ctx->common);
+		break;
+
+	case ARTPEC6_CRYPTO_PREPARE_HASH_NO_START:
+		ret = 0;
+		/* Fallthrough */
+
+	default:
+		artpec6_crypto_common_destroy(&req_ctx->common);
+		break;
+	}
+
+	return ret;
+}
+
+static int artpec6_crypto_hash_final(struct ahash_request *req)
+{
+	struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
+
+	req_ctx->hash_flags |= HASH_FLAG_FINALIZE;
+
+	return artpec6_crypto_prepare_submit_hash(req);
+}
+
+static int artpec6_crypto_hash_update(struct ahash_request *req)
+{
+	struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
+
+	req_ctx->hash_flags |= HASH_FLAG_UPDATE;
+
+	return artpec6_crypto_prepare_submit_hash(req);
+}
+
+static int artpec6_crypto_sha1_init(struct ahash_request *req)
+{
+	return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0);
+}
+
+static int artpec6_crypto_sha1_digest(struct ahash_request *req)
+{
+	struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
+
+	artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0);
+
+	req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
+
+	return artpec6_crypto_prepare_submit_hash(req);
+}
+
+static int artpec6_crypto_sha256_init(struct ahash_request *req)
+{
+	return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0);
+}
+
+static int artpec6_crypto_sha256_digest(struct ahash_request *req)
+{
+	struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
+
+	artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0);
+	req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
+
+	return artpec6_crypto_prepare_submit_hash(req);
+}
+
+static int __maybe_unused artpec6_crypto_sha384_init(struct ahash_request *req)
+{
+	return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 0);
+}
+
+static int __maybe_unused
+artpec6_crypto_sha384_digest(struct ahash_request *req)
+{
+	struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
+
+	artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 0);
+	req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
+
+	return artpec6_crypto_prepare_submit_hash(req);
+}
+
+static int artpec6_crypto_sha512_init(struct ahash_request *req)
+{
+	return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 0);
+}
+
+static int artpec6_crypto_sha512_digest(struct ahash_request *req)
+{
+	struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
+
+	artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 0);
+	req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
+
+	return artpec6_crypto_prepare_submit_hash(req);
+}
+
+static int artpec6_crypto_hmac_sha256_init(struct ahash_request *req)
+{
+	return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1);
+}
+
+static int __maybe_unused
+artpec6_crypto_hmac_sha384_init(struct ahash_request *req)
+{
+	return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 1);
+}
+
+static int artpec6_crypto_hmac_sha512_init(struct ahash_request *req)
+{
+	return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 1);
+}
+
+static int artpec6_crypto_hmac_sha256_digest(struct ahash_request *req)
+{
+	struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
+
+	artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1);
+	req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
+
+	return artpec6_crypto_prepare_submit_hash(req);
+}
+
+static int __maybe_unused
+artpec6_crypto_hmac_sha384_digest(struct ahash_request *req)
+{
+	struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
+
+	artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 1);
+	req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
+
+	return artpec6_crypto_prepare_submit_hash(req);
+}
+
+static int artpec6_crypto_hmac_sha512_digest(struct ahash_request *req)
+{
+	struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
+
+	artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 1);
+	req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
+
+	return artpec6_crypto_prepare_submit_hash(req);
+}
+
+static int artpec6_crypto_ahash_init_common(struct crypto_tfm *tfm,
+				    const char *base_hash_name)
+{
+	struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm);
+
+	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+				 sizeof(struct artpec6_hash_request_context));
+	memset(tfm_ctx, 0, sizeof(*tfm_ctx));
+
+	if (base_hash_name) {
+		struct crypto_shash *child;
+
+		child = crypto_alloc_shash(base_hash_name, 0,
+					   CRYPTO_ALG_NEED_FALLBACK);
+
+		if (IS_ERR(child))
+			return PTR_ERR(child);
+
+		tfm_ctx->child_hash = child;
+	}
+
+	return 0;
+}
+
+static int artpec6_crypto_ahash_init(struct crypto_tfm *tfm)
+{
+	return artpec6_crypto_ahash_init_common(tfm, NULL);
+}
+
+static int artpec6_crypto_ahash_init_hmac_sha256(struct crypto_tfm *tfm)
+{
+	return artpec6_crypto_ahash_init_common(tfm, "sha256");
+}
+
+static int __maybe_unused
+artpec6_crypto_ahash_init_hmac_sha384(struct crypto_tfm *tfm)
+{
+	return artpec6_crypto_ahash_init_common(tfm, "sha384");
+}
+
+static int artpec6_crypto_ahash_init_hmac_sha512(struct crypto_tfm *tfm)
+{
+	return artpec6_crypto_ahash_init_common(tfm, "sha512");
+}
+
+static void artpec6_crypto_ahash_exit(struct crypto_tfm *tfm)
+{
+	struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm);
+
+	if (tfm_ctx->child_hash)
+		crypto_free_shash(tfm_ctx->child_hash);
+
+	memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key));
+	tfm_ctx->hmac_key_length = 0;
+}
+
+static int artpec6_crypto_hash_export(struct ahash_request *req, void *out)
+{
+	const struct artpec6_hash_request_context *ctx = ahash_request_ctx(req);
+	struct artpec6_hash_export_state *state = out;
+	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
+	enum artpec6_crypto_variant variant = ac->variant;
+
+	BUILD_BUG_ON(sizeof(state->partial_buffer) !=
+		     sizeof(ctx->partial_buffer));
+	BUILD_BUG_ON(sizeof(state->digeststate) != sizeof(ctx->digeststate));
+
+	state->digcnt = ctx->digcnt;
+	state->partial_bytes = ctx->partial_bytes;
+	state->hash_flags = ctx->hash_flags;
+
+	if (variant == ARTPEC6_CRYPTO)
+		state->oper = FIELD_GET(A6_CRY_MD_OPER, ctx->hash_md);
+	else
+		state->oper = FIELD_GET(A7_CRY_MD_OPER, ctx->hash_md);
+
+	memcpy(state->partial_buffer, ctx->partial_buffer,
+	       sizeof(state->partial_buffer));
+	memcpy(state->digeststate, ctx->digeststate,
+	       sizeof(state->digeststate));
+
+	return 0;
+}
+
+static int artpec6_crypto_hash_import(struct ahash_request *req, const void *in)
+{
+	struct artpec6_hash_request_context *ctx = ahash_request_ctx(req);
+	const struct artpec6_hash_export_state *state = in;
+	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
+	enum artpec6_crypto_variant variant = ac->variant;
+
+	memset(ctx, 0, sizeof(*ctx));
+
+	ctx->digcnt = state->digcnt;
+	ctx->partial_bytes = state->partial_bytes;
+	ctx->hash_flags = state->hash_flags;
+
+	if (variant == ARTPEC6_CRYPTO)
+		ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, state->oper);
+	else
+		ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, state->oper);
+
+	memcpy(ctx->partial_buffer, state->partial_buffer,
+	       sizeof(state->partial_buffer));
+	memcpy(ctx->digeststate, state->digeststate,
+	       sizeof(state->digeststate));
+
+	return 0;
+}
+
+static int init_crypto_hw(struct artpec6_crypto *ac)
+{
+	enum artpec6_crypto_variant variant = ac->variant;
+	void __iomem *base = ac->base;
+	u32 out_descr_buf_size;
+	u32 out_data_buf_size;
+	u32 in_data_buf_size;
+	u32 in_descr_buf_size;
+	u32 in_stat_buf_size;
+	u32 in, out;
+
+	/*
+	 * The PDMA unit contains 1984 bytes of internal memory for the OUT
+	 * channels and 1024 bytes for the IN channel. This is an elastic
+	 * memory used to internally store the descriptors and data. The values
+	 * ares specified in 64 byte incremements.  Trustzone buffers are not
+	 * used at this stage.
+	 */
+	out_data_buf_size = 16;  /* 1024 bytes for data */
+	out_descr_buf_size = 15; /* 960 bytes for descriptors */
+	in_data_buf_size = 8;    /* 512 bytes for data */
+	in_descr_buf_size = 4;   /* 256 bytes for descriptors */
+	in_stat_buf_size = 4;   /* 256 bytes for stat descrs */
+
+	BUILD_BUG_ON_MSG((out_data_buf_size
+				+ out_descr_buf_size) * 64 > 1984,
+			  "Invalid OUT configuration");
+
+	BUILD_BUG_ON_MSG((in_data_buf_size
+				+ in_descr_buf_size
+				+ in_stat_buf_size) * 64 > 1024,
+			  "Invalid IN configuration");
+
+	in = FIELD_PREP(PDMA_IN_BUF_CFG_DATA_BUF_SIZE, in_data_buf_size) |
+	     FIELD_PREP(PDMA_IN_BUF_CFG_DESCR_BUF_SIZE, in_descr_buf_size) |
+	     FIELD_PREP(PDMA_IN_BUF_CFG_STAT_BUF_SIZE, in_stat_buf_size);
+
+	out = FIELD_PREP(PDMA_OUT_BUF_CFG_DATA_BUF_SIZE, out_data_buf_size) |
+	      FIELD_PREP(PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE, out_descr_buf_size);
+
+	writel_relaxed(out, base + PDMA_OUT_BUF_CFG);
+	writel_relaxed(PDMA_OUT_CFG_EN, base + PDMA_OUT_CFG);
+
+	if (variant == ARTPEC6_CRYPTO) {
+		writel_relaxed(in, base + A6_PDMA_IN_BUF_CFG);
+		writel_relaxed(PDMA_IN_CFG_EN, base + A6_PDMA_IN_CFG);
+		writel_relaxed(A6_PDMA_INTR_MASK_IN_DATA |
+			       A6_PDMA_INTR_MASK_IN_EOP_FLUSH,
+			       base + A6_PDMA_INTR_MASK);
+	} else {
+		writel_relaxed(in, base + A7_PDMA_IN_BUF_CFG);
+		writel_relaxed(PDMA_IN_CFG_EN, base + A7_PDMA_IN_CFG);
+		writel_relaxed(A7_PDMA_INTR_MASK_IN_DATA |
+			       A7_PDMA_INTR_MASK_IN_EOP_FLUSH,
+			       base + A7_PDMA_INTR_MASK);
+	}
+
+	return 0;
+}
+
+static void artpec6_crypto_disable_hw(struct artpec6_crypto *ac)
+{
+	enum artpec6_crypto_variant variant = ac->variant;
+	void __iomem *base = ac->base;
+
+	if (variant == ARTPEC6_CRYPTO) {
+		writel_relaxed(A6_PDMA_IN_CMD_STOP, base + A6_PDMA_IN_CMD);
+		writel_relaxed(0, base + A6_PDMA_IN_CFG);
+		writel_relaxed(A6_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD);
+	} else {
+		writel_relaxed(A7_PDMA_IN_CMD_STOP, base + A7_PDMA_IN_CMD);
+		writel_relaxed(0, base + A7_PDMA_IN_CFG);
+		writel_relaxed(A7_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD);
+	}
+
+	writel_relaxed(0, base + PDMA_OUT_CFG);
+
+}
+
+static irqreturn_t artpec6_crypto_irq(int irq, void *dev_id)
+{
+	struct artpec6_crypto *ac = dev_id;
+	enum artpec6_crypto_variant variant = ac->variant;
+	void __iomem *base = ac->base;
+	u32 mask_in_data, mask_in_eop_flush;
+	u32 in_cmd_flush_stat, in_cmd_reg;
+	u32 ack_intr_reg;
+	u32 ack = 0;
+	u32 intr;
+
+	if (variant == ARTPEC6_CRYPTO) {
+		intr = readl_relaxed(base + A6_PDMA_MASKED_INTR);
+		mask_in_data = A6_PDMA_INTR_MASK_IN_DATA;
+		mask_in_eop_flush = A6_PDMA_INTR_MASK_IN_EOP_FLUSH;
+		in_cmd_flush_stat = A6_PDMA_IN_CMD_FLUSH_STAT;
+		in_cmd_reg = A6_PDMA_IN_CMD;
+		ack_intr_reg = A6_PDMA_ACK_INTR;
+	} else {
+		intr = readl_relaxed(base + A7_PDMA_MASKED_INTR);
+		mask_in_data = A7_PDMA_INTR_MASK_IN_DATA;
+		mask_in_eop_flush = A7_PDMA_INTR_MASK_IN_EOP_FLUSH;
+		in_cmd_flush_stat = A7_PDMA_IN_CMD_FLUSH_STAT;
+		in_cmd_reg = A7_PDMA_IN_CMD;
+		ack_intr_reg = A7_PDMA_ACK_INTR;
+	}
+
+	/* We get two interrupt notifications from each job.
+	 * The in_data means all data was sent to memory and then
+	 * we request a status flush command to write the per-job
+	 * status to its status vector. This ensures that the
+	 * tasklet can detect exactly how many submitted jobs
+	 * that have finished.
+	 */
+	if (intr & mask_in_data)
+		ack |= mask_in_data;
+
+	if (intr & mask_in_eop_flush)
+		ack |= mask_in_eop_flush;
+	else
+		writel_relaxed(in_cmd_flush_stat, base + in_cmd_reg);
+
+	writel_relaxed(ack, base + ack_intr_reg);
+
+	if (intr & mask_in_eop_flush)
+		tasklet_schedule(&ac->task);
+
+	return IRQ_HANDLED;
+}
+
+/*------------------- Algorithm definitions ----------------------------------*/
+
+/* Hashes */
+static struct ahash_alg hash_algos[] = {
+	/* SHA-1 */
+	{
+		.init = artpec6_crypto_sha1_init,
+		.update = artpec6_crypto_hash_update,
+		.final = artpec6_crypto_hash_final,
+		.digest = artpec6_crypto_sha1_digest,
+		.import = artpec6_crypto_hash_import,
+		.export = artpec6_crypto_hash_export,
+		.halg.digestsize = SHA1_DIGEST_SIZE,
+		.halg.statesize = sizeof(struct artpec6_hash_export_state),
+		.halg.base = {
+			.cra_name = "sha1",
+			.cra_driver_name = "artpec-sha1",
+			.cra_priority = 300,
+			.cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
+			.cra_blocksize = SHA1_BLOCK_SIZE,
+			.cra_ctxsize = sizeof(struct artpec6_hashalg_context),
+			.cra_alignmask = 3,
+			.cra_module = THIS_MODULE,
+			.cra_init = artpec6_crypto_ahash_init,
+			.cra_exit = artpec6_crypto_ahash_exit,
+		}
+	},
+	/* SHA-256 */
+	{
+		.init = artpec6_crypto_sha256_init,
+		.update = artpec6_crypto_hash_update,
+		.final = artpec6_crypto_hash_final,
+		.digest = artpec6_crypto_sha256_digest,
+		.import = artpec6_crypto_hash_import,
+		.export = artpec6_crypto_hash_export,
+		.halg.digestsize = SHA256_DIGEST_SIZE,
+		.halg.statesize = sizeof(struct artpec6_hash_export_state),
+		.halg.base = {
+			.cra_name = "sha256",
+			.cra_driver_name = "artpec-sha256",
+			.cra_priority = 300,
+			.cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
+			.cra_blocksize = SHA256_BLOCK_SIZE,
+			.cra_ctxsize = sizeof(struct artpec6_hashalg_context),
+			.cra_alignmask = 3,
+			.cra_module = THIS_MODULE,
+			.cra_init = artpec6_crypto_ahash_init,
+			.cra_exit = artpec6_crypto_ahash_exit,
+		}
+	},
+	/* HMAC SHA-256 */
+	{
+		.init = artpec6_crypto_hmac_sha256_init,
+		.update = artpec6_crypto_hash_update,
+		.final = artpec6_crypto_hash_final,
+		.digest = artpec6_crypto_hmac_sha256_digest,
+		.import = artpec6_crypto_hash_import,
+		.export = artpec6_crypto_hash_export,
+		.setkey = artpec6_crypto_hash_set_key,
+		.halg.digestsize = SHA256_DIGEST_SIZE,
+		.halg.statesize = sizeof(struct artpec6_hash_export_state),
+		.halg.base = {
+			.cra_name = "hmac(sha256)",
+			.cra_driver_name = "artpec-hmac-sha256",
+			.cra_priority = 300,
+			.cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
+			.cra_blocksize = SHA256_BLOCK_SIZE,
+			.cra_ctxsize = sizeof(struct artpec6_hashalg_context),
+			.cra_alignmask = 3,
+			.cra_module = THIS_MODULE,
+			.cra_init = artpec6_crypto_ahash_init_hmac_sha256,
+			.cra_exit = artpec6_crypto_ahash_exit,
+		}
+	},
+};
+
+static struct ahash_alg artpec7_hash_algos[] = {
+	/* SHA-384 */
+	{
+		.init = artpec6_crypto_sha384_init,
+		.update = artpec6_crypto_hash_update,
+		.final = artpec6_crypto_hash_final,
+		.digest = artpec6_crypto_sha384_digest,
+		.import = artpec6_crypto_hash_import,
+		.export = artpec6_crypto_hash_export,
+		.halg.digestsize = SHA384_DIGEST_SIZE,
+		.halg.statesize = sizeof(struct artpec6_hash_export_state),
+		.halg.base = {
+			.cra_name = "sha384",
+			.cra_driver_name = "artpec-sha384",
+			.cra_priority = 300,
+			.cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
+			.cra_blocksize = SHA384_BLOCK_SIZE,
+			.cra_ctxsize = sizeof(struct artpec6_hashalg_context),
+			.cra_alignmask = 3,
+			.cra_module = THIS_MODULE,
+			.cra_init = artpec6_crypto_ahash_init,
+			.cra_exit = artpec6_crypto_ahash_exit,
+		}
+	},
+	/* HMAC SHA-384 */
+	{
+		.init = artpec6_crypto_hmac_sha384_init,
+		.update = artpec6_crypto_hash_update,
+		.final = artpec6_crypto_hash_final,
+		.digest = artpec6_crypto_hmac_sha384_digest,
+		.import = artpec6_crypto_hash_import,
+		.export = artpec6_crypto_hash_export,
+		.setkey = artpec6_crypto_hash_set_key,
+		.halg.digestsize = SHA384_DIGEST_SIZE,
+		.halg.statesize = sizeof(struct artpec6_hash_export_state),
+		.halg.base = {
+			.cra_name = "hmac(sha384)",
+			.cra_driver_name = "artpec-hmac-sha384",
+			.cra_priority = 300,
+			.cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
+			.cra_blocksize = SHA384_BLOCK_SIZE,
+			.cra_ctxsize = sizeof(struct artpec6_hashalg_context),
+			.cra_alignmask = 3,
+			.cra_module = THIS_MODULE,
+			.cra_init = artpec6_crypto_ahash_init_hmac_sha384,
+			.cra_exit = artpec6_crypto_ahash_exit,
+		}
+	},
+	/* SHA-512 */
+	{
+		.init = artpec6_crypto_sha512_init,
+		.update = artpec6_crypto_hash_update,
+		.final = artpec6_crypto_hash_final,
+		.digest = artpec6_crypto_sha512_digest,
+		.import = artpec6_crypto_hash_import,
+		.export = artpec6_crypto_hash_export,
+		.halg.digestsize = SHA512_DIGEST_SIZE,
+		.halg.statesize = sizeof(struct artpec6_hash_export_state),
+		.halg.base = {
+			.cra_name = "sha512",
+			.cra_driver_name = "artpec-sha512",
+			.cra_priority = 300,
+			.cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
+			.cra_blocksize = SHA512_BLOCK_SIZE,
+			.cra_ctxsize = sizeof(struct artpec6_hashalg_context),
+			.cra_alignmask = 3,
+			.cra_module = THIS_MODULE,
+			.cra_init = artpec6_crypto_ahash_init,
+			.cra_exit = artpec6_crypto_ahash_exit,
+		}
+	},
+	/* HMAC SHA-512 */
+	{
+		.init = artpec6_crypto_hmac_sha512_init,
+		.update = artpec6_crypto_hash_update,
+		.final = artpec6_crypto_hash_final,
+		.digest = artpec6_crypto_hmac_sha512_digest,
+		.import = artpec6_crypto_hash_import,
+		.export = artpec6_crypto_hash_export,
+		.setkey = artpec6_crypto_hash_set_key,
+		.halg.digestsize = SHA512_DIGEST_SIZE,
+		.halg.statesize = sizeof(struct artpec6_hash_export_state),
+		.halg.base = {
+			.cra_name = "hmac(sha512)",
+			.cra_driver_name = "artpec-hmac-sha512",
+			.cra_priority = 300,
+			.cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
+			.cra_blocksize = SHA512_BLOCK_SIZE,
+			.cra_ctxsize = sizeof(struct artpec6_hashalg_context),
+			.cra_alignmask = 3,
+			.cra_module = THIS_MODULE,
+			.cra_init = artpec6_crypto_ahash_init_hmac_sha512,
+			.cra_exit = artpec6_crypto_ahash_exit,
+		}
+	},
+};
+
+/* Crypto */
+static struct skcipher_alg crypto_algos[] = {
+	/* AES - ECB */
+	{
+		.base = {
+			.cra_name = "ecb(aes)",
+			.cra_driver_name = "artpec6-ecb-aes",
+			.cra_priority = 300,
+			.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
+				     CRYPTO_ALG_ASYNC,
+			.cra_blocksize = AES_BLOCK_SIZE,
+			.cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
+			.cra_alignmask = 3,
+			.cra_module = THIS_MODULE,
+		},
+		.min_keysize = AES_MIN_KEY_SIZE,
+		.max_keysize = AES_MAX_KEY_SIZE,
+		.setkey = artpec6_crypto_cipher_set_key,
+		.encrypt = artpec6_crypto_encrypt,
+		.decrypt = artpec6_crypto_decrypt,
+		.init = artpec6_crypto_aes_ecb_init,
+		.exit = artpec6_crypto_aes_exit,
+	},
+	/* AES - CTR */
+	{
+		.base = {
+			.cra_name = "ctr(aes)",
+			.cra_driver_name = "artpec6-ctr-aes",
+			.cra_priority = 300,
+			.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
+				     CRYPTO_ALG_ASYNC |
+				     CRYPTO_ALG_NEED_FALLBACK,
+			.cra_blocksize = 1,
+			.cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
+			.cra_alignmask = 3,
+			.cra_module = THIS_MODULE,
+		},
+		.min_keysize = AES_MIN_KEY_SIZE,
+		.max_keysize = AES_MAX_KEY_SIZE,
+		.ivsize = AES_BLOCK_SIZE,
+		.setkey = artpec6_crypto_cipher_set_key,
+		.encrypt = artpec6_crypto_ctr_encrypt,
+		.decrypt = artpec6_crypto_ctr_decrypt,
+		.init = artpec6_crypto_aes_ctr_init,
+		.exit = artpec6_crypto_aes_ctr_exit,
+	},
+	/* AES - CBC */
+	{
+		.base = {
+			.cra_name = "cbc(aes)",
+			.cra_driver_name = "artpec6-cbc-aes",
+			.cra_priority = 300,
+			.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
+				     CRYPTO_ALG_ASYNC,
+			.cra_blocksize = AES_BLOCK_SIZE,
+			.cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
+			.cra_alignmask = 3,
+			.cra_module = THIS_MODULE,
+		},
+		.min_keysize = AES_MIN_KEY_SIZE,
+		.max_keysize = AES_MAX_KEY_SIZE,
+		.ivsize = AES_BLOCK_SIZE,
+		.setkey = artpec6_crypto_cipher_set_key,
+		.encrypt = artpec6_crypto_encrypt,
+		.decrypt = artpec6_crypto_decrypt,
+		.init = artpec6_crypto_aes_cbc_init,
+		.exit = artpec6_crypto_aes_exit
+	},
+	/* AES - XTS */
+	{
+		.base = {
+			.cra_name = "xts(aes)",
+			.cra_driver_name = "artpec6-xts-aes",
+			.cra_priority = 300,
+			.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
+				     CRYPTO_ALG_ASYNC,
+			.cra_blocksize = 1,
+			.cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
+			.cra_alignmask = 3,
+			.cra_module = THIS_MODULE,
+		},
+		.min_keysize = 2*AES_MIN_KEY_SIZE,
+		.max_keysize = 2*AES_MAX_KEY_SIZE,
+		.ivsize = 16,
+		.setkey = artpec6_crypto_xts_set_key,
+		.encrypt = artpec6_crypto_encrypt,
+		.decrypt = artpec6_crypto_decrypt,
+		.init = artpec6_crypto_aes_xts_init,
+		.exit = artpec6_crypto_aes_exit,
+	},
+};
+
+static struct aead_alg aead_algos[] = {
+	{
+		.init   = artpec6_crypto_aead_init,
+		.setkey = artpec6_crypto_aead_set_key,
+		.encrypt = artpec6_crypto_aead_encrypt,
+		.decrypt = artpec6_crypto_aead_decrypt,
+		.ivsize = AES_BLOCK_SIZE,
+		.maxauthsize = AES_BLOCK_SIZE,
+
+		.base = {
+			.cra_name = "gcm(aes)",
+			.cra_driver_name = "artpec-gcm-aes",
+			.cra_priority = 300,
+			.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC |
+				     CRYPTO_ALG_KERN_DRIVER_ONLY,
+			.cra_blocksize = 1,
+			.cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
+			.cra_alignmask = 3,
+			.cra_module = THIS_MODULE,
+		},
+	}
+};
+
+#ifdef CONFIG_DEBUG_FS
+
+struct dbgfs_u32 {
+	char *name;
+	mode_t mode;
+	u32 *flag;
+	char *desc;
+};
+
+static void artpec6_crypto_init_debugfs(void)
+{
+	dbgfs_root = debugfs_create_dir("artpec6_crypto", NULL);
+
+	if (!dbgfs_root || IS_ERR(dbgfs_root)) {
+		dbgfs_root = NULL;
+		pr_err("%s: Could not initialise debugfs!\n", MODULE_NAME);
+		return;
+	}
+
+#ifdef CONFIG_FAULT_INJECTION
+	fault_create_debugfs_attr("fail_status_read", dbgfs_root,
+				  &artpec6_crypto_fail_status_read);
+
+	fault_create_debugfs_attr("fail_dma_array_full", dbgfs_root,
+				  &artpec6_crypto_fail_dma_array_full);
+#endif
+}
+
+static void artpec6_crypto_free_debugfs(void)
+{
+	if (!dbgfs_root)
+		return;
+
+	debugfs_remove_recursive(dbgfs_root);
+	dbgfs_root = NULL;
+}
+#endif
+
+static const struct of_device_id artpec6_crypto_of_match[] = {
+	{ .compatible = "axis,artpec6-crypto", .data = (void *)ARTPEC6_CRYPTO },
+	{ .compatible = "axis,artpec7-crypto", .data = (void *)ARTPEC7_CRYPTO },
+	{}
+};
+MODULE_DEVICE_TABLE(of, artpec6_crypto_of_match);
+
+static int artpec6_crypto_probe(struct platform_device *pdev)
+{
+	const struct of_device_id *match;
+	enum artpec6_crypto_variant variant;
+	struct artpec6_crypto *ac;
+	struct device *dev = &pdev->dev;
+	void __iomem *base;
+	struct resource *res;
+	int irq;
+	int err;
+
+	if (artpec6_crypto_dev)
+		return -ENODEV;
+
+	match = of_match_node(artpec6_crypto_of_match, dev->of_node);
+	if (!match)
+		return -EINVAL;
+
+	variant = (enum artpec6_crypto_variant)match->data;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res)
+		return -ENODEV;
+
+	base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return -ENODEV;
+
+	ac = devm_kzalloc(&pdev->dev, sizeof(struct artpec6_crypto),
+			  GFP_KERNEL);
+	if (!ac)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, ac);
+	ac->variant = variant;
+
+	spin_lock_init(&ac->queue_lock);
+	INIT_LIST_HEAD(&ac->queue);
+	INIT_LIST_HEAD(&ac->pending);
+	setup_timer(&ac->timer, artpec6_crypto_timeout, (unsigned long) ac);
+
+	ac->base = base;
+
+	ac->dma_cache = kmem_cache_create("artpec6_crypto_dma",
+		sizeof(struct artpec6_crypto_dma_descriptors),
+		64,
+		0,
+		NULL);
+	if (!ac->dma_cache)
+		return -ENOMEM;
+
+#ifdef CONFIG_DEBUG_FS
+	artpec6_crypto_init_debugfs();
+#endif
+
+	tasklet_init(&ac->task, artpec6_crypto_task,
+		     (unsigned long)ac);
+
+	ac->pad_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX,
+				      GFP_KERNEL);
+	if (!ac->pad_buffer)
+		return -ENOMEM;
+	ac->pad_buffer = PTR_ALIGN(ac->pad_buffer, ARTPEC_CACHE_LINE_MAX);
+
+	ac->zero_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX,
+				      GFP_KERNEL);
+	if (!ac->zero_buffer)
+		return -ENOMEM;
+	ac->zero_buffer = PTR_ALIGN(ac->zero_buffer, ARTPEC_CACHE_LINE_MAX);
+
+	err = init_crypto_hw(ac);
+	if (err)
+		goto free_cache;
+
+	err = devm_request_irq(&pdev->dev, irq, artpec6_crypto_irq, 0,
+			       "artpec6-crypto", ac);
+	if (err)
+		goto disable_hw;
+
+	artpec6_crypto_dev = &pdev->dev;
+
+	err = crypto_register_ahashes(hash_algos, ARRAY_SIZE(hash_algos));
+	if (err) {
+		dev_err(dev, "Failed to register ahashes\n");
+		goto disable_hw;
+	}
+
+	if (variant != ARTPEC6_CRYPTO) {
+		err = crypto_register_ahashes(artpec7_hash_algos,
+					      ARRAY_SIZE(artpec7_hash_algos));
+		if (err) {
+			dev_err(dev, "Failed to register ahashes\n");
+			goto unregister_ahashes;
+		}
+	}
+
+	err = crypto_register_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos));
+	if (err) {
+		dev_err(dev, "Failed to register ciphers\n");
+		goto unregister_a7_ahashes;
+	}
+
+	err = crypto_register_aeads(aead_algos, ARRAY_SIZE(aead_algos));
+	if (err) {
+		dev_err(dev, "Failed to register aeads\n");
+		goto unregister_algs;
+	}
+
+	return 0;
+
+unregister_algs:
+	crypto_unregister_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos));
+unregister_a7_ahashes:
+	if (variant != ARTPEC6_CRYPTO)
+		crypto_unregister_ahashes(artpec7_hash_algos,
+					  ARRAY_SIZE(artpec7_hash_algos));
+unregister_ahashes:
+	crypto_unregister_ahashes(hash_algos, ARRAY_SIZE(hash_algos));
+disable_hw:
+	artpec6_crypto_disable_hw(ac);
+free_cache:
+	kmem_cache_destroy(ac->dma_cache);
+	return err;
+}
+
+static int artpec6_crypto_remove(struct platform_device *pdev)
+{
+	struct artpec6_crypto *ac = platform_get_drvdata(pdev);
+	int irq = platform_get_irq(pdev, 0);
+
+	crypto_unregister_ahashes(hash_algos, ARRAY_SIZE(hash_algos));
+	if (ac->variant != ARTPEC6_CRYPTO)
+		crypto_unregister_ahashes(artpec7_hash_algos,
+					  ARRAY_SIZE(artpec7_hash_algos));
+	crypto_unregister_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos));
+	crypto_unregister_aeads(aead_algos, ARRAY_SIZE(aead_algos));
+
+	tasklet_disable(&ac->task);
+	devm_free_irq(&pdev->dev, irq, ac);
+	tasklet_kill(&ac->task);
+	del_timer_sync(&ac->timer);
+
+	artpec6_crypto_disable_hw(ac);
+
+	kmem_cache_destroy(ac->dma_cache);
+#ifdef CONFIG_DEBUG_FS
+	artpec6_crypto_free_debugfs();
+#endif
+	return 0;
+}
+
+static struct platform_driver artpec6_crypto_driver = {
+	.probe   = artpec6_crypto_probe,
+	.remove  = artpec6_crypto_remove,
+	.driver  = {
+		.name  = "artpec6-crypto",
+		.owner = THIS_MODULE,
+		.of_match_table = artpec6_crypto_of_match,
+	},
+};
+
+module_platform_driver(artpec6_crypto_driver);
+
+MODULE_AUTHOR("Axis Communications AB");
+MODULE_DESCRIPTION("ARTPEC-6 Crypto driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c
index 9cfd36c1bcb6..8685c7e4debd 100644
--- a/drivers/crypto/bcm/cipher.c
+++ b/drivers/crypto/bcm/cipher.c
@@ -90,8 +90,6 @@ static int aead_pri = 150;
 module_param(aead_pri, int, 0644);
 MODULE_PARM_DESC(aead_pri, "Priority for AEAD algos");
 
-#define MAX_SPUS 16
-
 /* A type 3 BCM header, expected to precede the SPU header for SPU-M.
  * Bits 3 and 4 in the first byte encode the channel number (the dma ringset).
  * 0x60 - ring 0
@@ -120,7 +118,7 @@ static u8 select_channel(void)
 {
 	u8 chan_idx = atomic_inc_return(&iproc_priv.next_chan);
 
-	return chan_idx % iproc_priv.spu.num_spu;
+	return chan_idx % iproc_priv.spu.num_chan;
 }
 
 /**
@@ -4528,8 +4526,13 @@ static void spu_functions_register(struct device *dev,
  */
 static int spu_mb_init(struct device *dev)
 {
-	struct mbox_client *mcl = &iproc_priv.mcl[iproc_priv.spu.num_spu];
-	int err;
+	struct mbox_client *mcl = &iproc_priv.mcl;
+	int err, i;
+
+	iproc_priv.mbox = devm_kcalloc(dev, iproc_priv.spu.num_chan,
+				  sizeof(struct mbox_chan *), GFP_KERNEL);
+	if (!iproc_priv.mbox)
+		return -ENOMEM;
 
 	mcl->dev = dev;
 	mcl->tx_block = false;
@@ -4538,25 +4541,33 @@ static int spu_mb_init(struct device *dev)
 	mcl->rx_callback = spu_rx_callback;
 	mcl->tx_done = NULL;
 
-	iproc_priv.mbox[iproc_priv.spu.num_spu] =
-			mbox_request_channel(mcl, 0);
-	if (IS_ERR(iproc_priv.mbox[iproc_priv.spu.num_spu])) {
-		err = (int)PTR_ERR(iproc_priv.mbox[iproc_priv.spu.num_spu]);
-		dev_err(dev,
-			"Mbox channel %d request failed with err %d",
-			iproc_priv.spu.num_spu, err);
-		iproc_priv.mbox[iproc_priv.spu.num_spu] = NULL;
-		return err;
+	for (i = 0; i < iproc_priv.spu.num_chan; i++) {
+		iproc_priv.mbox[i] = mbox_request_channel(mcl, i);
+		if (IS_ERR(iproc_priv.mbox[i])) {
+			err = (int)PTR_ERR(iproc_priv.mbox[i]);
+			dev_err(dev,
+				"Mbox channel %d request failed with err %d",
+				i, err);
+			iproc_priv.mbox[i] = NULL;
+			goto free_channels;
+		}
 	}
 
 	return 0;
+free_channels:
+	for (i = 0; i < iproc_priv.spu.num_chan; i++) {
+		if (iproc_priv.mbox[i])
+			mbox_free_channel(iproc_priv.mbox[i]);
+	}
+
+	return err;
 }
 
 static void spu_mb_release(struct platform_device *pdev)
 {
 	int i;
 
-	for (i = 0; i < iproc_priv.spu.num_spu; i++)
+	for (i = 0; i < iproc_priv.spu.num_chan; i++)
 		mbox_free_channel(iproc_priv.mbox[i]);
 }
 
@@ -4567,7 +4578,7 @@ static void spu_counters_init(void)
 
 	atomic_set(&iproc_priv.session_count, 0);
 	atomic_set(&iproc_priv.stream_count, 0);
-	atomic_set(&iproc_priv.next_chan, (int)iproc_priv.spu.num_spu);
+	atomic_set(&iproc_priv.next_chan, (int)iproc_priv.spu.num_chan);
 	atomic64_set(&iproc_priv.bytes_in, 0);
 	atomic64_set(&iproc_priv.bytes_out, 0);
 	for (i = 0; i < SPU_OP_NUM; i++) {
@@ -4809,47 +4820,38 @@ static int spu_dt_read(struct platform_device *pdev)
 	struct resource *spu_ctrl_regs;
 	const struct of_device_id *match;
 	const struct spu_type_subtype *matched_spu_type;
-	void __iomem *spu_reg_vbase[MAX_SPUS];
-	int err;
+	struct device_node *dn = pdev->dev.of_node;
+	int err, i;
 
-	match = of_match_device(of_match_ptr(bcm_spu_dt_ids), dev);
-	matched_spu_type = match->data;
+	/* Count number of mailbox channels */
+	spu->num_chan = of_count_phandle_with_args(dn, "mboxes", "#mbox-cells");
 
-	if (iproc_priv.spu.num_spu > 1) {
-		/* If this is 2nd or later SPU, make sure it's same type */
-		if ((spu->spu_type != matched_spu_type->type) ||
-		    (spu->spu_subtype != matched_spu_type->subtype)) {
-			err = -EINVAL;
-			dev_err(&pdev->dev, "Multiple SPU types not allowed");
-			return err;
-		}
-	} else {
-		/* Record type of first SPU */
-		spu->spu_type = matched_spu_type->type;
-		spu->spu_subtype = matched_spu_type->subtype;
+	match = of_match_device(of_match_ptr(bcm_spu_dt_ids), dev);
+	if (!match) {
+		dev_err(&pdev->dev, "Failed to match device\n");
+		return -ENODEV;
 	}
 
-	/* Get and map SPU registers */
-	spu_ctrl_regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!spu_ctrl_regs) {
-		err = -EINVAL;
-		dev_err(&pdev->dev, "Invalid/missing registers for SPU\n");
-		return err;
-	}
+	matched_spu_type = match->data;
 
-	spu_reg_vbase[iproc_priv.spu.num_spu] =
-				devm_ioremap_resource(dev, spu_ctrl_regs);
-	if (IS_ERR(spu_reg_vbase[iproc_priv.spu.num_spu])) {
-		err = PTR_ERR(spu_reg_vbase[iproc_priv.spu.num_spu]);
-		dev_err(&pdev->dev, "Failed to map registers: %d\n",
-			err);
-		spu_reg_vbase[iproc_priv.spu.num_spu] = NULL;
-		return err;
-	}
+	spu->spu_type = matched_spu_type->type;
+	spu->spu_subtype = matched_spu_type->subtype;
 
-	dev_dbg(dev, "SPU %d detected.", iproc_priv.spu.num_spu);
+	i = 0;
+	for (i = 0; (i < MAX_SPUS) && ((spu_ctrl_regs =
+		platform_get_resource(pdev, IORESOURCE_MEM, i)) != NULL); i++) {
 
-	spu->reg_vbase[iproc_priv.spu.num_spu] = spu_reg_vbase;
+		spu->reg_vbase[i] = devm_ioremap_resource(dev, spu_ctrl_regs);
+		if (IS_ERR(spu->reg_vbase[i])) {
+			err = PTR_ERR(spu->reg_vbase[i]);
+			dev_err(&pdev->dev, "Failed to map registers: %d\n",
+				err);
+			spu->reg_vbase[i] = NULL;
+			return err;
+		}
+	}
+	spu->num_spu = i;
+	dev_dbg(dev, "Device has %d SPUs", spu->num_spu);
 
 	return 0;
 }
@@ -4860,8 +4862,8 @@ int bcm_spu_probe(struct platform_device *pdev)
 	struct spu_hw *spu = &iproc_priv.spu;
 	int err = 0;
 
-	iproc_priv.pdev[iproc_priv.spu.num_spu] = pdev;
-	platform_set_drvdata(iproc_priv.pdev[iproc_priv.spu.num_spu],
+	iproc_priv.pdev  = pdev;
+	platform_set_drvdata(iproc_priv.pdev,
 			     &iproc_priv);
 
 	err = spu_dt_read(pdev);
@@ -4872,12 +4874,6 @@ int bcm_spu_probe(struct platform_device *pdev)
 	if (err < 0)
 		goto failure;
 
-	iproc_priv.spu.num_spu++;
-
-	/* If already initialized, we've just added another SPU and are done */
-	if (iproc_priv.inited)
-		return 0;
-
 	if (spu->spu_type == SPU_TYPE_SPUM)
 		iproc_priv.bcm_hdr_len = 8;
 	else if (spu->spu_type == SPU_TYPE_SPU2)
@@ -4893,8 +4889,6 @@ int bcm_spu_probe(struct platform_device *pdev)
 	if (err < 0)
 		goto fail_reg;
 
-	iproc_priv.inited = true;
-
 	return 0;
 
 fail_reg:
diff --git a/drivers/crypto/bcm/cipher.h b/drivers/crypto/bcm/cipher.h
index 51dca529ce8f..57a55eb2a255 100644
--- a/drivers/crypto/bcm/cipher.h
+++ b/drivers/crypto/bcm/cipher.h
@@ -427,10 +427,13 @@ struct spu_hw {
 
 	/* The number of SPUs on this platform */
 	u32 num_spu;
+
+	/* The number of SPU channels on this platform */
+	u32 num_chan;
 };
 
 struct device_private {
-	struct platform_device *pdev[MAX_SPUS];
+	struct platform_device *pdev;
 
 	struct spu_hw spu;
 
@@ -470,12 +473,10 @@ struct device_private {
 	/* Number of ICV check failures for AEAD messages */
 	atomic_t bad_icv;
 
-	struct mbox_client mcl[MAX_SPUS];
-	/* Array of mailbox channel pointers, one for each channel */
-	struct mbox_chan *mbox[MAX_SPUS];
+	struct mbox_client mcl;
 
-	/* Driver initialized */
-	bool inited;
+	/* Array of mailbox channel pointers, one for each channel */
+	struct mbox_chan **mbox;
 };
 
 extern struct device_private iproc_priv;
diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
index 0488b7f81dcf..54f3b375a453 100644
--- a/drivers/crypto/caam/caamalg.c
+++ b/drivers/crypto/caam/caamalg.c
@@ -81,40 +81,6 @@
 #define debug(format, arg...)
 #endif
 
-#ifdef DEBUG
-#include <linux/highmem.h>
-
-static void dbg_dump_sg(const char *level, const char *prefix_str,
-			int prefix_type, int rowsize, int groupsize,
-			struct scatterlist *sg, size_t tlen, bool ascii)
-{
-	struct scatterlist *it;
-	void *it_page;
-	size_t len;
-	void *buf;
-
-	for (it = sg; it != NULL && tlen > 0 ; it = sg_next(sg)) {
-		/*
-		 * make sure the scatterlist's page
-		 * has a valid virtual memory mapping
-		 */
-		it_page = kmap_atomic(sg_page(it));
-		if (unlikely(!it_page)) {
-			printk(KERN_ERR "dbg_dump_sg: kmap failed\n");
-			return;
-		}
-
-		buf = it_page + it->offset;
-		len = min_t(size_t, tlen, it->length);
-		print_hex_dump(level, prefix_str, prefix_type, rowsize,
-			       groupsize, buf, len, ascii);
-		tlen -= len;
-
-		kunmap_atomic(it_page);
-	}
-}
-#endif
-
 static struct list_head alg_list;
 
 struct caam_alg_entry {
@@ -898,10 +864,10 @@ static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
 	print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
 		       DUMP_PREFIX_ADDRESS, 16, 4, req->info,
 		       edesc->src_nents > 1 ? 100 : ivsize, 1);
-	dbg_dump_sg(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
-		    DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
-		    edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
 #endif
+	caam_dump_sg(KERN_ERR, "dst    @" __stringify(__LINE__)": ",
+		     DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
+		     edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
 
 	ablkcipher_unmap(jrdev, edesc, req);
 
@@ -937,10 +903,10 @@ static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
 	print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
 		       DUMP_PREFIX_ADDRESS, 16, 4, req->info,
 		       ivsize, 1);
-	dbg_dump_sg(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
-		    DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
-		    edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
 #endif
+	caam_dump_sg(KERN_ERR, "dst    @" __stringify(__LINE__)": ",
+		     DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
+		     edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
 
 	ablkcipher_unmap(jrdev, edesc, req);
 
@@ -1107,10 +1073,10 @@ static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
 		       ivsize, 1);
 	pr_err("asked=%d, nbytes%d\n",
 	       (int)edesc->src_nents > 1 ? 100 : req->nbytes, req->nbytes);
-	dbg_dump_sg(KERN_ERR, "src    @"__stringify(__LINE__)": ",
-		    DUMP_PREFIX_ADDRESS, 16, 4, req->src,
-		    edesc->src_nents > 1 ? 100 : req->nbytes, 1);
 #endif
+	caam_dump_sg(KERN_ERR, "src    @" __stringify(__LINE__)": ",
+		     DUMP_PREFIX_ADDRESS, 16, 4, req->src,
+		     edesc->src_nents > 1 ? 100 : req->nbytes, 1);
 
 	len = desc_len(sh_desc);
 	init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
@@ -1164,10 +1130,10 @@ static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr,
 	print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ",
 		       DUMP_PREFIX_ADDRESS, 16, 4, req->info,
 		       ivsize, 1);
-	dbg_dump_sg(KERN_ERR, "src    @" __stringify(__LINE__) ": ",
-		    DUMP_PREFIX_ADDRESS, 16, 4, req->src,
-		    edesc->src_nents > 1 ? 100 : req->nbytes, 1);
 #endif
+	caam_dump_sg(KERN_ERR, "src    @" __stringify(__LINE__) ": ",
+		     DUMP_PREFIX_ADDRESS, 16, 4, req->src,
+		     edesc->src_nents > 1 ? 100 : req->nbytes, 1);
 
 	len = desc_len(sh_desc);
 	init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
@@ -1449,11 +1415,9 @@ static int aead_decrypt(struct aead_request *req)
 	u32 *desc;
 	int ret = 0;
 
-#ifdef DEBUG
-	dbg_dump_sg(KERN_ERR, "dec src@"__stringify(__LINE__)": ",
-		    DUMP_PREFIX_ADDRESS, 16, 4, req->src,
-		    req->assoclen + req->cryptlen, 1);
-#endif
+	caam_dump_sg(KERN_ERR, "dec src@" __stringify(__LINE__)": ",
+		     DUMP_PREFIX_ADDRESS, 16, 4, req->src,
+		     req->assoclen + req->cryptlen, 1);
 
 	/* allocate extended descriptor */
 	edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN,
diff --git a/drivers/crypto/caam/caamalg_desc.c b/drivers/crypto/caam/caamalg_desc.c
index 6f9c7ec0e339..530c14ee32de 100644
--- a/drivers/crypto/caam/caamalg_desc.c
+++ b/drivers/crypto/caam/caamalg_desc.c
@@ -599,7 +599,7 @@ void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata,
 
 	/* skip key loading if they are loaded due to sharing */
 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
-				   JUMP_COND_SHRD | JUMP_COND_SELF);
+				   JUMP_COND_SHRD);
 	if (cdata->key_inline)
 		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
 				  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
@@ -688,8 +688,7 @@ void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata,
 
 	/* skip key loading if they are loaded due to sharing */
 	key_jump_cmd = append_jump(desc, JUMP_JSL |
-				   JUMP_TEST_ALL | JUMP_COND_SHRD |
-				   JUMP_COND_SELF);
+				   JUMP_TEST_ALL | JUMP_COND_SHRD);
 	if (cdata->key_inline)
 		append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
 				  cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c
index 78c4c0485c58..2eefc4a26bc2 100644
--- a/drivers/crypto/caam/caamalg_qi.c
+++ b/drivers/crypto/caam/caamalg_qi.c
@@ -12,7 +12,6 @@
 #include "intern.h"
 #include "desc_constr.h"
 #include "error.h"
-#include "sg_sw_sec4.h"
 #include "sg_sw_qm.h"
 #include "key_gen.h"
 #include "qi.h"
@@ -399,6 +398,7 @@ badkey:
  * @iv_dma: dma address of iv for checking continuity and link table
  * @qm_sg_bytes: length of dma mapped h/w link table
  * @qm_sg_dma: bus physical mapped address of h/w link table
+ * @assoclen: associated data length, in CAAM endianness
  * @assoclen_dma: bus physical mapped address of req->assoclen
  * @drv_req: driver-specific request structure
  * @sgt: the h/w link table
@@ -409,8 +409,12 @@ struct aead_edesc {
 	dma_addr_t iv_dma;
 	int qm_sg_bytes;
 	dma_addr_t qm_sg_dma;
+	unsigned int assoclen;
 	dma_addr_t assoclen_dma;
 	struct caam_drv_req drv_req;
+#define CAAM_QI_MAX_AEAD_SG						\
+	((CAAM_QI_MEMCACHE_SIZE - offsetof(struct aead_edesc, sgt)) /	\
+	 sizeof(struct qm_sg_entry))
 	struct qm_sg_entry sgt[0];
 };
 
@@ -431,6 +435,9 @@ struct ablkcipher_edesc {
 	int qm_sg_bytes;
 	dma_addr_t qm_sg_dma;
 	struct caam_drv_req drv_req;
+#define CAAM_QI_MAX_ABLKCIPHER_SG					    \
+	((CAAM_QI_MEMCACHE_SIZE - offsetof(struct ablkcipher_edesc, sgt)) / \
+	 sizeof(struct qm_sg_entry))
 	struct qm_sg_entry sgt[0];
 };
 
@@ -660,6 +667,14 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
 	 */
 	qm_sg_ents = 1 + !!ivsize + mapped_src_nents +
 		     (mapped_dst_nents > 1 ? mapped_dst_nents : 0);
+	if (unlikely(qm_sg_ents > CAAM_QI_MAX_AEAD_SG)) {
+		dev_err(qidev, "Insufficient S/G entries: %d > %lu\n",
+			qm_sg_ents, CAAM_QI_MAX_AEAD_SG);
+		caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
+			   iv_dma, ivsize, op_type, 0, 0);
+		qi_cache_free(edesc);
+		return ERR_PTR(-ENOMEM);
+	}
 	sg_table = &edesc->sgt[0];
 	qm_sg_bytes = qm_sg_ents * sizeof(*sg_table);
 
@@ -670,7 +685,8 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
 	edesc->drv_req.cbk = aead_done;
 	edesc->drv_req.drv_ctx = drv_ctx;
 
-	edesc->assoclen_dma = dma_map_single(qidev, &req->assoclen, 4,
+	edesc->assoclen = cpu_to_caam32(req->assoclen);
+	edesc->assoclen_dma = dma_map_single(qidev, &edesc->assoclen, 4,
 					     DMA_TO_DEVICE);
 	if (dma_mapping_error(qidev, edesc->assoclen_dma)) {
 		dev_err(qidev, "unable to map assoclen\n");
@@ -776,9 +792,9 @@ static void ablkcipher_done(struct caam_drv_req *drv_req, u32 status)
 	struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
 	struct caam_ctx *caam_ctx = crypto_ablkcipher_ctx(ablkcipher);
 	struct device *qidev = caam_ctx->qidev;
-#ifdef DEBUG
 	int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
 
+#ifdef DEBUG
 	dev_err(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status);
 #endif
 
@@ -791,14 +807,21 @@ static void ablkcipher_done(struct caam_drv_req *drv_req, u32 status)
 	print_hex_dump(KERN_ERR, "dstiv  @" __stringify(__LINE__)": ",
 		       DUMP_PREFIX_ADDRESS, 16, 4, req->info,
 		       edesc->src_nents > 1 ? 100 : ivsize, 1);
-	dbg_dump_sg(KERN_ERR, "dst    @" __stringify(__LINE__)": ",
-		    DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
-		    edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
+	caam_dump_sg(KERN_ERR, "dst    @" __stringify(__LINE__)": ",
+		     DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
+		     edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
 #endif
 
 	ablkcipher_unmap(qidev, edesc, req);
 	qi_cache_free(edesc);
 
+	/*
+	 * The crypto API expects us to set the IV (req->info) to the last
+	 * ciphertext block. This is used e.g. by the CTS mode.
+	 */
+	scatterwalk_map_and_copy(req->info, req->dst, req->nbytes - ivsize,
+				 ivsize, 0);
+
 	ablkcipher_request_complete(req, status);
 }
 
@@ -880,6 +903,15 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
 	}
 	dst_sg_idx = qm_sg_ents;
 
+	qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
+	if (unlikely(qm_sg_ents > CAAM_QI_MAX_ABLKCIPHER_SG)) {
+		dev_err(qidev, "Insufficient S/G entries: %d > %lu\n",
+			qm_sg_ents, CAAM_QI_MAX_ABLKCIPHER_SG);
+		caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
+			   iv_dma, ivsize, op_type, 0, 0);
+		return ERR_PTR(-ENOMEM);
+	}
+
 	/* allocate space for base edesc and link tables */
 	edesc = qi_cache_alloc(GFP_DMA | flags);
 	if (unlikely(!edesc)) {
@@ -892,7 +924,6 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
 	edesc->src_nents = src_nents;
 	edesc->dst_nents = dst_nents;
 	edesc->iv_dma = iv_dma;
-	qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
 	sg_table = &edesc->sgt[0];
 	edesc->qm_sg_bytes = qm_sg_ents * sizeof(*sg_table);
 	edesc->drv_req.app_ctx = req;
@@ -1026,6 +1057,14 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
 		qm_sg_ents += 1 + mapped_dst_nents;
 	}
 
+	if (unlikely(qm_sg_ents > CAAM_QI_MAX_ABLKCIPHER_SG)) {
+		dev_err(qidev, "Insufficient S/G entries: %d > %lu\n",
+			qm_sg_ents, CAAM_QI_MAX_ABLKCIPHER_SG);
+		caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
+			   iv_dma, ivsize, GIVENCRYPT, 0, 0);
+		return ERR_PTR(-ENOMEM);
+	}
+
 	/* allocate space for base edesc and link tables */
 	edesc = qi_cache_alloc(GFP_DMA | flags);
 	if (!edesc) {
@@ -1968,7 +2007,7 @@ static struct caam_aead_alg driver_aeads[] = {
 				.cra_name = "echainiv(authenc(hmac(sha256),"
 					    "cbc(des)))",
 				.cra_driver_name = "echainiv-authenc-"
-						   "hmac-sha256-cbc-desi-"
+						   "hmac-sha256-cbc-des-"
 						   "caam-qi",
 				.cra_blocksize = DES_BLOCK_SIZE,
 			},
diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c
index 910ec61cae09..698580b60b2f 100644
--- a/drivers/crypto/caam/caamhash.c
+++ b/drivers/crypto/caam/caamhash.c
@@ -791,8 +791,8 @@ static int ahash_update_ctx(struct ahash_request *req)
 							 to_hash - *buflen,
 							 *next_buflen, 0);
 		} else {
-			(edesc->sec4_sg + sec4_sg_src_index - 1)->len |=
-				cpu_to_caam32(SEC4_SG_LEN_FIN);
+			sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index -
+					    1);
 		}
 
 		desc = edesc->hw_desc;
@@ -882,8 +882,7 @@ static int ahash_final_ctx(struct ahash_request *req)
 	if (ret)
 		goto unmap_ctx;
 
-	(edesc->sec4_sg + sec4_sg_src_index - 1)->len |=
-		cpu_to_caam32(SEC4_SG_LEN_FIN);
+	sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index - 1);
 
 	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
 					    sec4_sg_bytes, DMA_TO_DEVICE);
diff --git a/drivers/crypto/caam/caamrng.c b/drivers/crypto/caam/caamrng.c
index 41398da3edf4..fde07d4ff019 100644
--- a/drivers/crypto/caam/caamrng.c
+++ b/drivers/crypto/caam/caamrng.c
@@ -285,11 +285,7 @@ static int caam_init_rng(struct caam_rng_ctx *ctx, struct device *jrdev)
 	if (err)
 		return err;
 
-	err = caam_init_buf(ctx, 1);
-	if (err)
-		return err;
-
-	return 0;
+	return caam_init_buf(ctx, 1);
 }
 
 static struct hwrng caam_rng = {
diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c
index dd353e342c12..dacb53fb690e 100644
--- a/drivers/crypto/caam/ctrl.c
+++ b/drivers/crypto/caam/ctrl.c
@@ -17,6 +17,8 @@
 
 bool caam_little_end;
 EXPORT_SYMBOL(caam_little_end);
+bool caam_dpaa2;
+EXPORT_SYMBOL(caam_dpaa2);
 
 #ifdef CONFIG_CAAM_QI
 #include "qi.h"
@@ -319,8 +321,11 @@ static int caam_remove(struct platform_device *pdev)
 		caam_qi_shutdown(ctrlpriv->qidev);
 #endif
 
-	/* De-initialize RNG state handles initialized by this driver. */
-	if (ctrlpriv->rng4_sh_init)
+	/*
+	 * De-initialize RNG state handles initialized by this driver.
+	 * In case of DPAA 2.x, RNG is managed by MC firmware.
+	 */
+	if (!caam_dpaa2 && ctrlpriv->rng4_sh_init)
 		deinstantiate_rng(ctrldev, ctrlpriv->rng4_sh_init);
 
 	/* Shut down debug views */
@@ -444,7 +449,6 @@ static int caam_probe(struct platform_device *pdev)
 
 	dev = &pdev->dev;
 	dev_set_drvdata(dev, ctrlpriv);
-	ctrlpriv->pdev = pdev;
 	nprop = pdev->dev.of_node;
 
 	/* Enable clocking */
@@ -553,12 +557,17 @@ static int caam_probe(struct platform_device *pdev)
 
 	/*
 	 * Enable DECO watchdogs and, if this is a PHYS_ADDR_T_64BIT kernel,
-	 * long pointers in master configuration register
+	 * long pointers in master configuration register.
+	 * In case of DPAA 2.x, Management Complex firmware performs
+	 * the configuration.
 	 */
-	clrsetbits_32(&ctrl->mcr, MCFGR_AWCACHE_MASK | MCFGR_LONG_PTR,
-		      MCFGR_AWCACHE_CACH | MCFGR_AWCACHE_BUFF |
-		      MCFGR_WDENABLE | MCFGR_LARGE_BURST |
-		      (sizeof(dma_addr_t) == sizeof(u64) ? MCFGR_LONG_PTR : 0));
+	caam_dpaa2 = !!(comp_params & CTPR_MS_DPAA2);
+	if (!caam_dpaa2)
+		clrsetbits_32(&ctrl->mcr, MCFGR_AWCACHE_MASK | MCFGR_LONG_PTR,
+			      MCFGR_AWCACHE_CACH | MCFGR_AWCACHE_BUFF |
+			      MCFGR_WDENABLE | MCFGR_LARGE_BURST |
+			      (sizeof(dma_addr_t) == sizeof(u64) ?
+			       MCFGR_LONG_PTR : 0));
 
 	/*
 	 *  Read the Compile Time paramters and SCFGR to determine
@@ -587,7 +596,9 @@ static int caam_probe(struct platform_device *pdev)
 			      JRSTART_JR3_START);
 
 	if (sizeof(dma_addr_t) == sizeof(u64)) {
-		if (of_device_is_compatible(nprop, "fsl,sec-v5.0"))
+		if (caam_dpaa2)
+			ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49));
+		else if (of_device_is_compatible(nprop, "fsl,sec-v5.0"))
 			ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40));
 		else
 			ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(36));
@@ -630,11 +641,9 @@ static int caam_probe(struct platform_device *pdev)
 			ring++;
 		}
 
-	/* Check to see if QI present. If so, enable */
-	ctrlpriv->qi_present =
-			!!(rd_reg32(&ctrl->perfmon.comp_parms_ms) &
-			   CTPR_MS_QI_MASK);
-	if (ctrlpriv->qi_present) {
+	/* Check to see if (DPAA 1.x) QI present. If so, enable */
+	ctrlpriv->qi_present = !!(comp_params & CTPR_MS_QI_MASK);
+	if (ctrlpriv->qi_present && !caam_dpaa2) {
 		ctrlpriv->qi = (struct caam_queue_if __iomem __force *)
 			       ((__force uint8_t *)ctrl +
 				 BLOCK_OFFSET * QI_BLOCK_NUMBER
@@ -662,8 +671,10 @@ static int caam_probe(struct platform_device *pdev)
 	/*
 	 * If SEC has RNG version >= 4 and RNG state handle has not been
 	 * already instantiated, do RNG instantiation
+	 * In case of DPAA 2.x, RNG is managed by MC firmware.
 	 */
-	if ((cha_vid_ls & CHA_ID_LS_RNG_MASK) >> CHA_ID_LS_RNG_SHIFT >= 4) {
+	if (!caam_dpaa2 &&
+	    (cha_vid_ls & CHA_ID_LS_RNG_MASK) >> CHA_ID_LS_RNG_SHIFT >= 4) {
 		ctrlpriv->rng4_sh_init =
 			rd_reg32(&ctrl->r4tst[0].rdsta);
 		/*
@@ -731,63 +742,43 @@ static int caam_probe(struct platform_device *pdev)
 	/* Report "alive" for developer to see */
 	dev_info(dev, "device ID = 0x%016llx (Era %d)\n", caam_id,
 		 caam_get_era());
-	dev_info(dev, "job rings = %d, qi = %d\n",
-		 ctrlpriv->total_jobrs, ctrlpriv->qi_present);
+	dev_info(dev, "job rings = %d, qi = %d, dpaa2 = %s\n",
+		 ctrlpriv->total_jobrs, ctrlpriv->qi_present,
+		 caam_dpaa2 ? "yes" : "no");
 
 #ifdef CONFIG_DEBUG_FS
-
-	ctrlpriv->ctl_rq_dequeued =
-		debugfs_create_file("rq_dequeued",
-				    S_IRUSR | S_IRGRP | S_IROTH,
-				    ctrlpriv->ctl, &perfmon->req_dequeued,
-				    &caam_fops_u64_ro);
-	ctrlpriv->ctl_ob_enc_req =
-		debugfs_create_file("ob_rq_encrypted",
-				    S_IRUSR | S_IRGRP | S_IROTH,
-				    ctrlpriv->ctl, &perfmon->ob_enc_req,
-				    &caam_fops_u64_ro);
-	ctrlpriv->ctl_ib_dec_req =
-		debugfs_create_file("ib_rq_decrypted",
-				    S_IRUSR | S_IRGRP | S_IROTH,
-				    ctrlpriv->ctl, &perfmon->ib_dec_req,
-				    &caam_fops_u64_ro);
-	ctrlpriv->ctl_ob_enc_bytes =
-		debugfs_create_file("ob_bytes_encrypted",
-				    S_IRUSR | S_IRGRP | S_IROTH,
-				    ctrlpriv->ctl, &perfmon->ob_enc_bytes,
-				    &caam_fops_u64_ro);
-	ctrlpriv->ctl_ob_prot_bytes =
-		debugfs_create_file("ob_bytes_protected",
-				    S_IRUSR | S_IRGRP | S_IROTH,
-				    ctrlpriv->ctl, &perfmon->ob_prot_bytes,
-				    &caam_fops_u64_ro);
-	ctrlpriv->ctl_ib_dec_bytes =
-		debugfs_create_file("ib_bytes_decrypted",
-				    S_IRUSR | S_IRGRP | S_IROTH,
-				    ctrlpriv->ctl, &perfmon->ib_dec_bytes,
-				    &caam_fops_u64_ro);
-	ctrlpriv->ctl_ib_valid_bytes =
-		debugfs_create_file("ib_bytes_validated",
-				    S_IRUSR | S_IRGRP | S_IROTH,
-				    ctrlpriv->ctl, &perfmon->ib_valid_bytes,
-				    &caam_fops_u64_ro);
+	debugfs_create_file("rq_dequeued", S_IRUSR | S_IRGRP | S_IROTH,
+			    ctrlpriv->ctl, &perfmon->req_dequeued,
+			    &caam_fops_u64_ro);
+	debugfs_create_file("ob_rq_encrypted", S_IRUSR | S_IRGRP | S_IROTH,
+			    ctrlpriv->ctl, &perfmon->ob_enc_req,
+			    &caam_fops_u64_ro);
+	debugfs_create_file("ib_rq_decrypted", S_IRUSR | S_IRGRP | S_IROTH,
+			    ctrlpriv->ctl, &perfmon->ib_dec_req,
+			    &caam_fops_u64_ro);
+	debugfs_create_file("ob_bytes_encrypted", S_IRUSR | S_IRGRP | S_IROTH,
+			    ctrlpriv->ctl, &perfmon->ob_enc_bytes,
+			    &caam_fops_u64_ro);
+	debugfs_create_file("ob_bytes_protected", S_IRUSR | S_IRGRP | S_IROTH,
+			    ctrlpriv->ctl, &perfmon->ob_prot_bytes,
+			    &caam_fops_u64_ro);
+	debugfs_create_file("ib_bytes_decrypted", S_IRUSR | S_IRGRP | S_IROTH,
+			    ctrlpriv->ctl, &perfmon->ib_dec_bytes,
+			    &caam_fops_u64_ro);
+	debugfs_create_file("ib_bytes_validated", S_IRUSR | S_IRGRP | S_IROTH,
+			    ctrlpriv->ctl, &perfmon->ib_valid_bytes,
+			    &caam_fops_u64_ro);
 
 	/* Controller level - global status values */
-	ctrlpriv->ctl_faultaddr =
-		debugfs_create_file("fault_addr",
-				    S_IRUSR | S_IRGRP | S_IROTH,
-				    ctrlpriv->ctl, &perfmon->faultaddr,
-				    &caam_fops_u32_ro);
-	ctrlpriv->ctl_faultdetail =
-		debugfs_create_file("fault_detail",
-				    S_IRUSR | S_IRGRP | S_IROTH,
-				    ctrlpriv->ctl, &perfmon->faultdetail,
-				    &caam_fops_u32_ro);
-	ctrlpriv->ctl_faultstatus =
-		debugfs_create_file("fault_status",
-				    S_IRUSR | S_IRGRP | S_IROTH,
-				    ctrlpriv->ctl, &perfmon->status,
-				    &caam_fops_u32_ro);
+	debugfs_create_file("fault_addr", S_IRUSR | S_IRGRP | S_IROTH,
+			    ctrlpriv->ctl, &perfmon->faultaddr,
+			    &caam_fops_u32_ro);
+	debugfs_create_file("fault_detail", S_IRUSR | S_IRGRP | S_IROTH,
+			    ctrlpriv->ctl, &perfmon->faultdetail,
+			    &caam_fops_u32_ro);
+	debugfs_create_file("fault_status", S_IRUSR | S_IRGRP | S_IROTH,
+			    ctrlpriv->ctl, &perfmon->status,
+			    &caam_fops_u32_ro);
 
 	/* Internal covering keys (useful in non-secure mode only) */
 	ctrlpriv->ctl_kek_wrap.data = (__force void *)&ctrlpriv->ctrl->kek[0];
diff --git a/drivers/crypto/caam/ctrl.h b/drivers/crypto/caam/ctrl.h
index cac5402a46eb..7e7bf68c9ef5 100644
--- a/drivers/crypto/caam/ctrl.h
+++ b/drivers/crypto/caam/ctrl.h
@@ -10,4 +10,6 @@
 /* Prototypes for backend-level services exposed to APIs */
 int caam_get_era(void);
 
+extern bool caam_dpaa2;
+
 #endif /* CTRL_H */
diff --git a/drivers/crypto/caam/error.c b/drivers/crypto/caam/error.c
index 6f44ccb55c63..3d639f3b45aa 100644
--- a/drivers/crypto/caam/error.c
+++ b/drivers/crypto/caam/error.c
@@ -9,6 +9,46 @@
 #include "desc.h"
 #include "error.h"
 
+#ifdef DEBUG
+#include <linux/highmem.h>
+
+void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type,
+		  int rowsize, int groupsize, struct scatterlist *sg,
+		  size_t tlen, bool ascii)
+{
+	struct scatterlist *it;
+	void *it_page;
+	size_t len;
+	void *buf;
+
+	for (it = sg; it && tlen > 0 ; it = sg_next(sg)) {
+		/*
+		 * make sure the scatterlist's page
+		 * has a valid virtual memory mapping
+		 */
+		it_page = kmap_atomic(sg_page(it));
+		if (unlikely(!it_page)) {
+			pr_err("caam_dump_sg: kmap failed\n");
+			return;
+		}
+
+		buf = it_page + it->offset;
+		len = min_t(size_t, tlen, it->length);
+		print_hex_dump(level, prefix_str, prefix_type, rowsize,
+			       groupsize, buf, len, ascii);
+		tlen -= len;
+
+		kunmap_atomic(it_page);
+	}
+}
+#else
+void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type,
+		  int rowsize, int groupsize, struct scatterlist *sg,
+		  size_t tlen, bool ascii)
+{}
+#endif /* DEBUG */
+EXPORT_SYMBOL(caam_dump_sg);
+
 static const struct {
 	u8 value;
 	const char *error_text;
diff --git a/drivers/crypto/caam/error.h b/drivers/crypto/caam/error.h
index b6350b0d9153..250e1a21c473 100644
--- a/drivers/crypto/caam/error.h
+++ b/drivers/crypto/caam/error.h
@@ -8,4 +8,8 @@
 #define CAAM_ERROR_H
 #define CAAM_ERROR_STR_MAX 302
 void caam_jr_strstatus(struct device *jrdev, u32 status);
+
+void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type,
+		  int rowsize, int groupsize, struct scatterlist *sg,
+		  size_t tlen, bool ascii);
 #endif /* CAAM_ERROR_H */
diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h
index 85b6c5835b8f..a52361258d3a 100644
--- a/drivers/crypto/caam/intern.h
+++ b/drivers/crypto/caam/intern.h
@@ -64,12 +64,9 @@ struct caam_drv_private_jr {
  * Driver-private storage for a single CAAM block instance
  */
 struct caam_drv_private {
-
-	struct device *dev;
 #ifdef CONFIG_CAAM_QI
 	struct device *qidev;
 #endif
-	struct platform_device *pdev;
 
 	/* Physical-presence section */
 	struct caam_ctrl __iomem *ctrl; /* controller region */
@@ -105,16 +102,8 @@ struct caam_drv_private {
 #ifdef CONFIG_DEBUG_FS
 	struct dentry *dfs_root;
 	struct dentry *ctl; /* controller dir */
-	struct dentry *ctl_rq_dequeued, *ctl_ob_enc_req, *ctl_ib_dec_req;
-	struct dentry *ctl_ob_enc_bytes, *ctl_ob_prot_bytes;
-	struct dentry *ctl_ib_dec_bytes, *ctl_ib_valid_bytes;
-	struct dentry *ctl_faultaddr, *ctl_faultdetail, *ctl_faultstatus;
-
 	struct debugfs_blob_wrapper ctl_kek_wrap, ctl_tkek_wrap, ctl_tdsk_wrap;
 	struct dentry *ctl_kek, *ctl_tkek, *ctl_tdsk;
-#ifdef CONFIG_CAAM_QI
-	struct dentry *qi_congested;
-#endif
 #endif
 };
 
diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c
index 1ccfb317d468..d258953ff488 100644
--- a/drivers/crypto/caam/jr.c
+++ b/drivers/crypto/caam/jr.c
@@ -9,6 +9,7 @@
 #include <linux/of_address.h>
 
 #include "compat.h"
+#include "ctrl.h"
 #include "regs.h"
 #include "jr.h"
 #include "desc.h"
@@ -499,7 +500,11 @@ static int caam_jr_probe(struct platform_device *pdev)
 	jrpriv->rregs = (struct caam_job_ring __iomem __force *)ctrl;
 
 	if (sizeof(dma_addr_t) == sizeof(u64)) {
-		if (of_device_is_compatible(nprop, "fsl,sec-v5.0-job-ring"))
+		if (caam_dpaa2)
+			error = dma_set_mask_and_coherent(jrdev,
+							  DMA_BIT_MASK(49));
+		else if (of_device_is_compatible(nprop,
+						 "fsl,sec-v5.0-job-ring"))
 			error = dma_set_mask_and_coherent(jrdev,
 							  DMA_BIT_MASK(40));
 		else
diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c
index 1990ed460c46..e4cf00014233 100644
--- a/drivers/crypto/caam/qi.c
+++ b/drivers/crypto/caam/qi.c
@@ -24,9 +24,6 @@
  */
 #define MAX_RSP_FQ_BACKLOG_PER_CPU	256
 
-/* Length of a single buffer in the QI driver memory cache */
-#define CAAM_QI_MEMCACHE_SIZE	512
-
 #define CAAM_QI_ENQUEUE_RETRIES	10000
 
 #define CAAM_NAPI_WEIGHT	63
@@ -55,6 +52,7 @@ struct caam_qi_pcpu_priv {
 } ____cacheline_aligned;
 
 static DEFINE_PER_CPU(struct caam_qi_pcpu_priv, pcpu_qipriv);
+static DEFINE_PER_CPU(int, last_cpu);
 
 /*
  * caam_qi_priv - CAAM QI backend private params
@@ -203,8 +201,8 @@ static struct qman_fq *create_caam_req_fq(struct device *qidev,
 		goto init_req_fq_fail;
 	}
 
-	dev_info(qidev, "Allocated request FQ %u for CPU %u\n", req_fq->fqid,
-		 smp_processor_id());
+	dev_dbg(qidev, "Allocated request FQ %u for CPU %u\n", req_fq->fqid,
+		smp_processor_id());
 	return req_fq;
 
 init_req_fq_fail:
@@ -277,6 +275,7 @@ empty_fq:
 		dev_err(qidev, "OOS of FQID: %u failed\n", fq->fqid);
 
 	qman_destroy_fq(fq);
+	kfree(fq);
 
 	return ret;
 }
@@ -342,8 +341,7 @@ int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc)
 		drv_ctx->req_fq = old_fq;
 
 		if (kill_fq(qidev, new_fq))
-			dev_warn(qidev, "New CAAM FQ: %u kill failed\n",
-				 new_fq->fqid);
+			dev_warn(qidev, "New CAAM FQ kill failed\n");
 
 		return ret;
 	}
@@ -373,10 +371,9 @@ int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc)
 		drv_ctx->req_fq = old_fq;
 
 		if (kill_fq(qidev, new_fq))
-			dev_warn(qidev, "New CAAM FQ: %u kill failed\n",
-				 new_fq->fqid);
+			dev_warn(qidev, "New CAAM FQ kill failed\n");
 	} else if (kill_fq(qidev, old_fq)) {
-		dev_warn(qidev, "Old CAAM FQ: %u kill failed\n", old_fq->fqid);
+		dev_warn(qidev, "Old CAAM FQ kill failed\n");
 	}
 
 	return 0;
@@ -392,7 +389,6 @@ struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev,
 	dma_addr_t hwdesc;
 	struct caam_drv_ctx *drv_ctx;
 	const cpumask_t *cpus = qman_affine_cpus();
-	static DEFINE_PER_CPU(int, last_cpu);
 
 	num_words = desc_len(sh_desc);
 	if (num_words > MAX_SDLEN) {
@@ -511,7 +507,6 @@ int caam_qi_shutdown(struct device *qidev)
 
 		if (kill_fq(qidev, per_cpu(pcpu_qipriv.rsp_fq, i)))
 			dev_err(qidev, "Rsp FQ kill failed, cpu: %d\n", i);
-		kfree(per_cpu(pcpu_qipriv.rsp_fq, i));
 	}
 
 	/*
@@ -646,7 +641,7 @@ static int alloc_rsp_fq_cpu(struct device *qidev, unsigned int cpu)
 
 	per_cpu(pcpu_qipriv.rsp_fq, cpu) = fq;
 
-	dev_info(qidev, "Allocated response FQ %u for CPU %u", fq->fqid, cpu);
+	dev_dbg(qidev, "Allocated response FQ %u for CPU %u", fq->fqid, cpu);
 	return 0;
 }
 
@@ -679,7 +674,7 @@ static int init_cgr(struct device *qidev)
 		return ret;
 	}
 
-	dev_info(qidev, "Congestion threshold set to %llu\n", val);
+	dev_dbg(qidev, "Congestion threshold set to %llu\n", val);
 	return 0;
 }
 
@@ -737,6 +732,7 @@ int caam_qi_init(struct platform_device *caam_pdev)
 	qi_pdev = platform_device_register_full(&qi_pdev_info);
 	if (IS_ERR(qi_pdev))
 		return PTR_ERR(qi_pdev);
+	set_dma_ops(&qi_pdev->dev, get_dma_ops(ctrldev));
 
 	ctrlpriv = dev_get_drvdata(ctrldev);
 	qidev = &qi_pdev->dev;
@@ -795,10 +791,8 @@ int caam_qi_init(struct platform_device *caam_pdev)
 	/* Done with the CGRs; restore the cpus allowed mask */
 	set_cpus_allowed_ptr(current, &old_cpumask);
 #ifdef CONFIG_DEBUG_FS
-	ctrlpriv->qi_congested = debugfs_create_file("qi_congested", 0444,
-						     ctrlpriv->ctl,
-						     &times_congested,
-						     &caam_fops_u64_ro);
+	debugfs_create_file("qi_congested", 0444, ctrlpriv->ctl,
+			    &times_congested, &caam_fops_u64_ro);
 #endif
 	dev_info(qidev, "Linux CAAM Queue I/F driver initialised\n");
 	return 0;
diff --git a/drivers/crypto/caam/qi.h b/drivers/crypto/caam/qi.h
index 33b0433f5f22..ecb21f207637 100644
--- a/drivers/crypto/caam/qi.h
+++ b/drivers/crypto/caam/qi.h
@@ -39,6 +39,9 @@
  */
 #define MAX_SDLEN	((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) / CAAM_CMD_SZ)
 
+/* Length of a single buffer in the QI driver memory cache */
+#define CAAM_QI_MEMCACHE_SIZE	768
+
 extern bool caam_congested __read_mostly;
 
 /*
diff --git a/drivers/crypto/caam/regs.h b/drivers/crypto/caam/regs.h
index 84d2f838a063..2b5efff9ec3c 100644
--- a/drivers/crypto/caam/regs.h
+++ b/drivers/crypto/caam/regs.h
@@ -293,6 +293,7 @@ struct caam_perfmon {
 	u32 cha_rev_ls;		/* CRNR - CHA Rev No. Least significant half*/
 #define CTPR_MS_QI_SHIFT	25
 #define CTPR_MS_QI_MASK		(0x1ull << CTPR_MS_QI_SHIFT)
+#define CTPR_MS_DPAA2		BIT(13)
 #define CTPR_MS_VIRT_EN_INCL	0x00000001
 #define CTPR_MS_VIRT_EN_POR	0x00000002
 #define CTPR_MS_PG_SZ_MASK	0x10
diff --git a/drivers/crypto/caam/sg_sw_qm2.h b/drivers/crypto/caam/sg_sw_qm2.h
new file mode 100644
index 000000000000..31b440757146
--- /dev/null
+++ b/drivers/crypto/caam/sg_sw_qm2.h
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2015-2016 Freescale Semiconductor, Inc.
+ * Copyright 2017 NXP
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *	 notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *	 notice, this list of conditions and the following disclaimer in the
+ *	 documentation and/or other materials provided with the distribution.
+ *     * Neither the names of the above-listed copyright holders nor the
+ *	 names of any contributors may be used to endorse or promote products
+ *	 derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _SG_SW_QM2_H_
+#define _SG_SW_QM2_H_
+
+#include "../../../drivers/staging/fsl-mc/include/dpaa2-fd.h"
+
+static inline void dma_to_qm_sg_one(struct dpaa2_sg_entry *qm_sg_ptr,
+				    dma_addr_t dma, u32 len, u16 offset)
+{
+	dpaa2_sg_set_addr(qm_sg_ptr, dma);
+	dpaa2_sg_set_format(qm_sg_ptr, dpaa2_sg_single);
+	dpaa2_sg_set_final(qm_sg_ptr, false);
+	dpaa2_sg_set_len(qm_sg_ptr, len);
+	dpaa2_sg_set_bpid(qm_sg_ptr, 0);
+	dpaa2_sg_set_offset(qm_sg_ptr, offset);
+}
+
+/*
+ * convert scatterlist to h/w link table format
+ * but does not have final bit; instead, returns last entry
+ */
+static inline struct dpaa2_sg_entry *
+sg_to_qm_sg(struct scatterlist *sg, int sg_count,
+	    struct dpaa2_sg_entry *qm_sg_ptr, u16 offset)
+{
+	while (sg_count && sg) {
+		dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg),
+				 sg_dma_len(sg), offset);
+		qm_sg_ptr++;
+		sg = sg_next(sg);
+		sg_count--;
+	}
+	return qm_sg_ptr - 1;
+}
+
+/*
+ * convert scatterlist to h/w link table format
+ * scatterlist must have been previously dma mapped
+ */
+static inline void sg_to_qm_sg_last(struct scatterlist *sg, int sg_count,
+				    struct dpaa2_sg_entry *qm_sg_ptr,
+				    u16 offset)
+{
+	qm_sg_ptr = sg_to_qm_sg(sg, sg_count, qm_sg_ptr, offset);
+	dpaa2_sg_set_final(qm_sg_ptr, true);
+}
+
+#endif /* _SG_SW_QM2_H_ */
diff --git a/drivers/crypto/caam/sg_sw_sec4.h b/drivers/crypto/caam/sg_sw_sec4.h
index c6adad09c972..936b1b630058 100644
--- a/drivers/crypto/caam/sg_sw_sec4.h
+++ b/drivers/crypto/caam/sg_sw_sec4.h
@@ -5,7 +5,13 @@
  *
  */
 
+#ifndef _SG_SW_SEC4_H_
+#define _SG_SW_SEC4_H_
+
+#include "ctrl.h"
 #include "regs.h"
+#include "sg_sw_qm2.h"
+#include "../../../drivers/staging/fsl-mc/include/dpaa2-fd.h"
 
 struct sec4_sg_entry {
 	u64 ptr;
@@ -19,9 +25,15 @@ struct sec4_sg_entry {
 static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr,
 				      dma_addr_t dma, u32 len, u16 offset)
 {
-	sec4_sg_ptr->ptr = cpu_to_caam_dma64(dma);
-	sec4_sg_ptr->len = cpu_to_caam32(len);
-	sec4_sg_ptr->bpid_offset = cpu_to_caam32(offset & SEC4_SG_OFFSET_MASK);
+	if (caam_dpaa2) {
+		dma_to_qm_sg_one((struct dpaa2_sg_entry *)sec4_sg_ptr, dma, len,
+				 offset);
+	} else {
+		sec4_sg_ptr->ptr = cpu_to_caam_dma64(dma);
+		sec4_sg_ptr->len = cpu_to_caam32(len);
+		sec4_sg_ptr->bpid_offset = cpu_to_caam32(offset &
+							 SEC4_SG_OFFSET_MASK);
+	}
 #ifdef DEBUG
 	print_hex_dump(KERN_ERR, "sec4_sg_ptr@: ",
 		       DUMP_PREFIX_ADDRESS, 16, 4, sec4_sg_ptr,
@@ -47,6 +59,14 @@ sg_to_sec4_sg(struct scatterlist *sg, int sg_count,
 	return sec4_sg_ptr - 1;
 }
 
+static inline void sg_to_sec4_set_last(struct sec4_sg_entry *sec4_sg_ptr)
+{
+	if (caam_dpaa2)
+		dpaa2_sg_set_final((struct dpaa2_sg_entry *)sec4_sg_ptr, true);
+	else
+		sec4_sg_ptr->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
+}
+
 /*
  * convert scatterlist to h/w link table format
  * scatterlist must have been previously dma mapped
@@ -56,20 +76,7 @@ static inline void sg_to_sec4_sg_last(struct scatterlist *sg, int sg_count,
 				      u16 offset)
 {
 	sec4_sg_ptr = sg_to_sec4_sg(sg, sg_count, sec4_sg_ptr, offset);
-	sec4_sg_ptr->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
+	sg_to_sec4_set_last(sec4_sg_ptr);
 }
 
-static inline struct sec4_sg_entry *sg_to_sec4_sg_len(
-	struct scatterlist *sg, unsigned int total,
-	struct sec4_sg_entry *sec4_sg_ptr)
-{
-	do {
-		unsigned int len = min(sg_dma_len(sg), total);
-
-		dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg), len, 0);
-		sec4_sg_ptr++;
-		sg = sg_next(sg);
-		total -= len;
-	} while (total);
-	return sec4_sg_ptr - 1;
-}
+#endif /* _SG_SW_SEC4_H_ */
diff --git a/drivers/crypto/cavium/cpt/cptpf_main.c b/drivers/crypto/cavium/cpt/cptpf_main.c
index 4119c40e7c4b..34a6d8bf229e 100644
--- a/drivers/crypto/cavium/cpt/cptpf_main.c
+++ b/drivers/crypto/cavium/cpt/cptpf_main.c
@@ -268,8 +268,10 @@ static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae)
 	mcode = &cpt->mcode[cpt->next_mc_idx];
 	memcpy(mcode->version, (u8 *)fw_entry->data, CPT_UCODE_VERSION_SZ);
 	mcode->code_size = ntohl(ucode->code_length) * 2;
-	if (!mcode->code_size)
-		return -EINVAL;
+	if (!mcode->code_size) {
+		ret = -EINVAL;
+		goto fw_release;
+	}
 
 	mcode->is_ae = is_ae;
 	mcode->core_mask = 0ULL;
@@ -280,7 +282,8 @@ static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae)
 					  &mcode->phys_base, GFP_KERNEL);
 	if (!mcode->code) {
 		dev_err(dev, "Unable to allocate space for microcode");
-		return -ENOMEM;
+		ret = -ENOMEM;
+		goto fw_release;
 	}
 
 	memcpy((void *)mcode->code, (void *)(fw_entry->data + sizeof(*ucode)),
@@ -302,12 +305,14 @@ static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae)
 	ret = do_cpt_init(cpt, mcode);
 	if (ret) {
 		dev_err(dev, "do_cpt_init failed with ret: %d\n", ret);
-		return ret;
+		goto fw_release;
 	}
 
 	dev_info(dev, "Microcode Loaded %s\n", mcode->version);
 	mcode->is_mc_valid = 1;
 	cpt->next_mc_idx++;
+
+fw_release:
 	release_firmware(fw_entry);
 
 	return ret;
diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c
index 9ccefb9b7232..fee7cb2ce747 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_main.c
+++ b/drivers/crypto/cavium/nitrox/nitrox_main.c
@@ -513,8 +513,10 @@ static int nitrox_probe(struct pci_dev *pdev,
 	pci_set_master(pdev);
 
 	ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
-	if (!ndev)
+	if (!ndev) {
+		err = -ENOMEM;
 		goto ndev_fail;
+	}
 
 	pci_set_drvdata(pdev, ndev);
 	ndev->pdev = pdev;
diff --git a/drivers/crypto/ccp/Kconfig b/drivers/crypto/ccp/Kconfig
index 2238f77aa248..6d626606b9c5 100644
--- a/drivers/crypto/ccp/Kconfig
+++ b/drivers/crypto/ccp/Kconfig
@@ -1,25 +1,33 @@
 config CRYPTO_DEV_CCP_DD
-	tristate "Cryptographic Coprocessor device driver"
-	depends on CRYPTO_DEV_CCP
+	tristate "Secure Processor device driver"
 	default m
+	help
+	  Provides AMD Secure Processor device driver.
+	  If you choose 'M' here, this module will be called ccp.
+
+config CRYPTO_DEV_SP_CCP
+	bool "Cryptographic Coprocessor device"
+	default y
+	depends on CRYPTO_DEV_CCP_DD
 	select HW_RANDOM
 	select DMA_ENGINE
 	select DMADEVICES
 	select CRYPTO_SHA1
 	select CRYPTO_SHA256
 	help
-	  Provides the interface to use the AMD Cryptographic Coprocessor
-	  which can be used to offload encryption operations such as SHA,
-	  AES and more. If you choose 'M' here, this module will be called
-	  ccp.
+	  Provides the support for AMD Cryptographic Coprocessor (CCP) device
+	  which can be used to offload encryption operations such as SHA, AES
+	  and more.
 
 config CRYPTO_DEV_CCP_CRYPTO
 	tristate "Encryption and hashing offload support"
-	depends on CRYPTO_DEV_CCP_DD
 	default m
+	depends on CRYPTO_DEV_CCP_DD
+	depends on CRYPTO_DEV_SP_CCP
 	select CRYPTO_HASH
 	select CRYPTO_BLKCIPHER
 	select CRYPTO_AUTHENC
+	select CRYPTO_RSA
 	help
 	  Support for using the cryptographic API with the AMD Cryptographic
 	  Coprocessor. This module supports offload of SHA and AES algorithms.
diff --git a/drivers/crypto/ccp/Makefile b/drivers/crypto/ccp/Makefile
index 59493fd3a751..57f8debfcfb3 100644
--- a/drivers/crypto/ccp/Makefile
+++ b/drivers/crypto/ccp/Makefile
@@ -1,12 +1,12 @@
 obj-$(CONFIG_CRYPTO_DEV_CCP_DD) += ccp.o
-ccp-objs := ccp-dev.o \
+ccp-objs  := sp-dev.o sp-platform.o
+ccp-$(CONFIG_CRYPTO_DEV_SP_CCP) += ccp-dev.o \
 	    ccp-ops.o \
 	    ccp-dev-v3.o \
 	    ccp-dev-v5.o \
-	    ccp-platform.o \
 	    ccp-dmaengine.o \
 	    ccp-debugfs.o
-ccp-$(CONFIG_PCI) += ccp-pci.o
+ccp-$(CONFIG_PCI) += sp-pci.o
 
 obj-$(CONFIG_CRYPTO_DEV_CCP_CRYPTO) += ccp-crypto.o
 ccp-crypto-objs := ccp-crypto-main.o \
@@ -15,4 +15,5 @@ ccp-crypto-objs := ccp-crypto-main.o \
 		   ccp-crypto-aes-xts.o \
 		   ccp-crypto-aes-galois.o \
 		   ccp-crypto-des3.o \
+		   ccp-crypto-rsa.o \
 		   ccp-crypto-sha.o
diff --git a/drivers/crypto/ccp/ccp-crypto-aes-galois.c b/drivers/crypto/ccp/ccp-crypto-aes-galois.c
index 38ee6f348ea9..52313524a4dd 100644
--- a/drivers/crypto/ccp/ccp-crypto-aes-galois.c
+++ b/drivers/crypto/ccp/ccp-crypto-aes-galois.c
@@ -1,7 +1,7 @@
 /*
  * AMD Cryptographic Coprocessor (CCP) AES GCM crypto API support
  *
- * Copyright (C) 2016 Advanced Micro Devices, Inc.
+ * Copyright (C) 2016,2017 Advanced Micro Devices, Inc.
  *
  * Author: Gary R Hook <gary.hook@amd.com>
  *
diff --git a/drivers/crypto/ccp/ccp-crypto-aes-xts.c b/drivers/crypto/ccp/ccp-crypto-aes-xts.c
index 58a4244b4752..94b5bcf5b628 100644
--- a/drivers/crypto/ccp/ccp-crypto-aes-xts.c
+++ b/drivers/crypto/ccp/ccp-crypto-aes-xts.c
@@ -1,8 +1,9 @@
 /*
  * AMD Cryptographic Coprocessor (CCP) AES XTS crypto API support
  *
- * Copyright (C) 2013 Advanced Micro Devices, Inc.
+ * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
  *
+ * Author: Gary R Hook <gary.hook@amd.com>
  * Author: Tom Lendacky <thomas.lendacky@amd.com>
  *
  * This program is free software; you can redistribute it and/or modify
@@ -15,6 +16,7 @@
 #include <linux/delay.h>
 #include <linux/scatterlist.h>
 #include <crypto/aes.h>
+#include <crypto/xts.h>
 #include <crypto/internal/skcipher.h>
 #include <crypto/scatterwalk.h>
 
@@ -37,46 +39,26 @@ struct ccp_unit_size_map {
 	u32 value;
 };
 
-static struct ccp_unit_size_map unit_size_map[] = {
+static struct ccp_unit_size_map xts_unit_sizes[] = {
 	{
-		.size	= 4096,
-		.value	= CCP_XTS_AES_UNIT_SIZE_4096,
-	},
-	{
-		.size	= 2048,
-		.value	= CCP_XTS_AES_UNIT_SIZE_2048,
-	},
-	{
-		.size	= 1024,
-		.value	= CCP_XTS_AES_UNIT_SIZE_1024,
+		.size   = 16,
+		.value	= CCP_XTS_AES_UNIT_SIZE_16,
 	},
 	{
-		.size	= 512,
+		.size   = 512,
 		.value	= CCP_XTS_AES_UNIT_SIZE_512,
 	},
 	{
-		.size	= 256,
-		.value	= CCP_XTS_AES_UNIT_SIZE__LAST,
-	},
-	{
-		.size	= 128,
-		.value	= CCP_XTS_AES_UNIT_SIZE__LAST,
-	},
-	{
-		.size	= 64,
-		.value	= CCP_XTS_AES_UNIT_SIZE__LAST,
-	},
-	{
-		.size	= 32,
-		.value	= CCP_XTS_AES_UNIT_SIZE__LAST,
+		.size   = 1024,
+		.value	= CCP_XTS_AES_UNIT_SIZE_1024,
 	},
 	{
-		.size	= 16,
-		.value	= CCP_XTS_AES_UNIT_SIZE_16,
+		.size   = 2048,
+		.value	= CCP_XTS_AES_UNIT_SIZE_2048,
 	},
 	{
-		.size	= 1,
-		.value	= CCP_XTS_AES_UNIT_SIZE__LAST,
+		.size   = 4096,
+		.value	= CCP_XTS_AES_UNIT_SIZE_4096,
 	},
 };
 
@@ -96,15 +78,26 @@ static int ccp_aes_xts_complete(struct crypto_async_request *async_req, int ret)
 static int ccp_aes_xts_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 			      unsigned int key_len)
 {
-	struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ablkcipher_tfm(tfm));
+	struct crypto_tfm *xfm = crypto_ablkcipher_tfm(tfm);
+	struct ccp_ctx *ctx = crypto_tfm_ctx(xfm);
+	unsigned int ccpversion = ccp_version();
+	int ret;
 
-	/* Only support 128-bit AES key with a 128-bit Tweak key,
-	 * otherwise use the fallback
+	ret = xts_check_key(xfm, key, key_len);
+	if (ret)
+		return ret;
+
+	/* Version 3 devices support 128-bit keys; version 5 devices can
+	 * accommodate 128- and 256-bit keys.
 	 */
 	switch (key_len) {
 	case AES_KEYSIZE_128 * 2:
 		memcpy(ctx->u.aes.key, key, key_len);
 		break;
+	case AES_KEYSIZE_256 * 2:
+		if (ccpversion > CCP_VERSION(3, 0))
+			memcpy(ctx->u.aes.key, key, key_len);
+		break;
 	}
 	ctx->u.aes.key_len = key_len / 2;
 	sg_init_one(&ctx->u.aes.key_sg, ctx->u.aes.key, key_len);
@@ -117,6 +110,8 @@ static int ccp_aes_xts_crypt(struct ablkcipher_request *req,
 {
 	struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
 	struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req);
+	unsigned int ccpversion = ccp_version();
+	unsigned int fallback = 0;
 	unsigned int unit;
 	u32 unit_size;
 	int ret;
@@ -130,18 +125,32 @@ static int ccp_aes_xts_crypt(struct ablkcipher_request *req,
 	if (!req->info)
 		return -EINVAL;
 
+	/* Check conditions under which the CCP can fulfill a request. The
+	 * device can handle input plaintext of a length that is a multiple
+	 * of the unit_size, bug the crypto implementation only supports
+	 * the unit_size being equal to the input length. This limits the
+	 * number of scenarios we can handle.
+	 */
 	unit_size = CCP_XTS_AES_UNIT_SIZE__LAST;
-	if (req->nbytes <= unit_size_map[0].size) {
-		for (unit = 0; unit < ARRAY_SIZE(unit_size_map); unit++) {
-			if (!(req->nbytes & (unit_size_map[unit].size - 1))) {
-				unit_size = unit_size_map[unit].value;
-				break;
-			}
+	for (unit = 0; unit < ARRAY_SIZE(xts_unit_sizes); unit++) {
+		if (req->nbytes == xts_unit_sizes[unit].size) {
+			unit_size = unit;
+			break;
 		}
 	}
-
-	if ((unit_size == CCP_XTS_AES_UNIT_SIZE__LAST) ||
-	    (ctx->u.aes.key_len != AES_KEYSIZE_128)) {
+	/* The CCP has restrictions on block sizes. Also, a version 3 device
+	 * only supports AES-128 operations; version 5 CCPs support both
+	 * AES-128 and -256 operations.
+	 */
+	if (unit_size == CCP_XTS_AES_UNIT_SIZE__LAST)
+		fallback = 1;
+	if ((ccpversion < CCP_VERSION(5, 0)) &&
+	    (ctx->u.aes.key_len != AES_KEYSIZE_128))
+		fallback = 1;
+	if ((ctx->u.aes.key_len != AES_KEYSIZE_128) &&
+	    (ctx->u.aes.key_len != AES_KEYSIZE_256))
+		fallback = 1;
+	if (fallback) {
 		SKCIPHER_REQUEST_ON_STACK(subreq, ctx->u.aes.tfm_skcipher);
 
 		/* Use the fallback to process the request for any
@@ -164,6 +173,7 @@ static int ccp_aes_xts_crypt(struct ablkcipher_request *req,
 	memset(&rctx->cmd, 0, sizeof(rctx->cmd));
 	INIT_LIST_HEAD(&rctx->cmd.entry);
 	rctx->cmd.engine = CCP_ENGINE_XTS_AES_128;
+	rctx->cmd.u.xts.type = CCP_AES_TYPE_128;
 	rctx->cmd.u.xts.action = (encrypt) ? CCP_AES_ACTION_ENCRYPT
 					   : CCP_AES_ACTION_DECRYPT;
 	rctx->cmd.u.xts.unit_size = unit_size;
diff --git a/drivers/crypto/ccp/ccp-crypto-des3.c b/drivers/crypto/ccp/ccp-crypto-des3.c
index 5af7347ae03c..ae87b741f9d5 100644
--- a/drivers/crypto/ccp/ccp-crypto-des3.c
+++ b/drivers/crypto/ccp/ccp-crypto-des3.c
@@ -1,7 +1,7 @@
 /*
  * AMD Cryptographic Coprocessor (CCP) DES3 crypto API support
  *
- * Copyright (C) 2016 Advanced Micro Devices, Inc.
+ * Copyright (C) 2016,2017 Advanced Micro Devices, Inc.
  *
  * Author: Gary R Hook <ghook@amd.com>
  *
diff --git a/drivers/crypto/ccp/ccp-crypto-main.c b/drivers/crypto/ccp/ccp-crypto-main.c
index 8dccbddabef1..35a9de7fd475 100644
--- a/drivers/crypto/ccp/ccp-crypto-main.c
+++ b/drivers/crypto/ccp/ccp-crypto-main.c
@@ -1,7 +1,7 @@
 /*
  * AMD Cryptographic Coprocessor (CCP) crypto API support
  *
- * Copyright (C) 2013 Advanced Micro Devices, Inc.
+ * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
  *
  * Author: Tom Lendacky <thomas.lendacky@amd.com>
  *
@@ -17,6 +17,7 @@
 #include <linux/ccp.h>
 #include <linux/scatterlist.h>
 #include <crypto/internal/hash.h>
+#include <crypto/internal/akcipher.h>
 
 #include "ccp-crypto.h"
 
@@ -37,10 +38,15 @@ static unsigned int des3_disable;
 module_param(des3_disable, uint, 0444);
 MODULE_PARM_DESC(des3_disable, "Disable use of 3DES - any non-zero value");
 
+static unsigned int rsa_disable;
+module_param(rsa_disable, uint, 0444);
+MODULE_PARM_DESC(rsa_disable, "Disable use of RSA - any non-zero value");
+
 /* List heads for the supported algorithms */
 static LIST_HEAD(hash_algs);
 static LIST_HEAD(cipher_algs);
 static LIST_HEAD(aead_algs);
+static LIST_HEAD(akcipher_algs);
 
 /* For any tfm, requests for that tfm must be returned on the order
  * received.  With multiple queues available, the CCP can process more
@@ -358,6 +364,12 @@ static int ccp_register_algs(void)
 			return ret;
 	}
 
+	if (!rsa_disable) {
+		ret = ccp_register_rsa_algs(&akcipher_algs);
+		if (ret)
+			return ret;
+	}
+
 	return 0;
 }
 
@@ -366,6 +378,7 @@ static void ccp_unregister_algs(void)
 	struct ccp_crypto_ahash_alg *ahash_alg, *ahash_tmp;
 	struct ccp_crypto_ablkcipher_alg *ablk_alg, *ablk_tmp;
 	struct ccp_crypto_aead *aead_alg, *aead_tmp;
+	struct ccp_crypto_akcipher_alg *akc_alg, *akc_tmp;
 
 	list_for_each_entry_safe(ahash_alg, ahash_tmp, &hash_algs, entry) {
 		crypto_unregister_ahash(&ahash_alg->alg);
@@ -384,6 +397,12 @@ static void ccp_unregister_algs(void)
 		list_del(&aead_alg->entry);
 		kfree(aead_alg);
 	}
+
+	list_for_each_entry_safe(akc_alg, akc_tmp, &akcipher_algs, entry) {
+		crypto_unregister_akcipher(&akc_alg->alg);
+		list_del(&akc_alg->entry);
+		kfree(akc_alg);
+	}
 }
 
 static int ccp_crypto_init(void)
diff --git a/drivers/crypto/ccp/ccp-crypto-rsa.c b/drivers/crypto/ccp/ccp-crypto-rsa.c
new file mode 100644
index 000000000000..e6db8672d89c
--- /dev/null
+++ b/drivers/crypto/ccp/ccp-crypto-rsa.c
@@ -0,0 +1,299 @@
+/*
+ * AMD Cryptographic Coprocessor (CCP) RSA crypto API support
+ *
+ * Copyright (C) 2017 Advanced Micro Devices, Inc.
+ *
+ * Author: Gary R Hook <gary.hook@amd.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/module.h>
+#include <linux/sched.h>
+#include <linux/scatterlist.h>
+#include <linux/crypto.h>
+#include <crypto/algapi.h>
+#include <crypto/internal/rsa.h>
+#include <crypto/internal/akcipher.h>
+#include <crypto/akcipher.h>
+#include <crypto/scatterwalk.h>
+
+#include "ccp-crypto.h"
+
+static inline struct akcipher_request *akcipher_request_cast(
+	struct crypto_async_request *req)
+{
+	return container_of(req, struct akcipher_request, base);
+}
+
+static inline int ccp_copy_and_save_keypart(u8 **kpbuf, unsigned int *kplen,
+					    const u8 *buf, size_t sz)
+{
+	int nskip;
+
+	for (nskip = 0; nskip < sz; nskip++)
+		if (buf[nskip])
+			break;
+	*kplen = sz - nskip;
+	*kpbuf = kzalloc(*kplen, GFP_KERNEL);
+	if (!*kpbuf)
+		return -ENOMEM;
+	memcpy(*kpbuf, buf + nskip, *kplen);
+
+	return 0;
+}
+
+static int ccp_rsa_complete(struct crypto_async_request *async_req, int ret)
+{
+	struct akcipher_request *req = akcipher_request_cast(async_req);
+	struct ccp_rsa_req_ctx *rctx = akcipher_request_ctx(req);
+
+	if (ret)
+		return ret;
+
+	req->dst_len = rctx->cmd.u.rsa.key_size >> 3;
+
+	return 0;
+}
+
+static unsigned int ccp_rsa_maxsize(struct crypto_akcipher *tfm)
+{
+	if (ccp_version() > CCP_VERSION(3, 0))
+		return CCP5_RSA_MAXMOD;
+	else
+		return CCP_RSA_MAXMOD;
+}
+
+static int ccp_rsa_crypt(struct akcipher_request *req, bool encrypt)
+{
+	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
+	struct ccp_ctx *ctx = akcipher_tfm_ctx(tfm);
+	struct ccp_rsa_req_ctx *rctx = akcipher_request_ctx(req);
+	int ret = 0;
+
+	memset(&rctx->cmd, 0, sizeof(rctx->cmd));
+	INIT_LIST_HEAD(&rctx->cmd.entry);
+	rctx->cmd.engine = CCP_ENGINE_RSA;
+
+	rctx->cmd.u.rsa.key_size = ctx->u.rsa.key_len; /* in bits */
+	if (encrypt) {
+		rctx->cmd.u.rsa.exp = &ctx->u.rsa.e_sg;
+		rctx->cmd.u.rsa.exp_len = ctx->u.rsa.e_len;
+	} else {
+		rctx->cmd.u.rsa.exp = &ctx->u.rsa.d_sg;
+		rctx->cmd.u.rsa.exp_len = ctx->u.rsa.d_len;
+	}
+	rctx->cmd.u.rsa.mod = &ctx->u.rsa.n_sg;
+	rctx->cmd.u.rsa.mod_len = ctx->u.rsa.n_len;
+	rctx->cmd.u.rsa.src = req->src;
+	rctx->cmd.u.rsa.src_len = req->src_len;
+	rctx->cmd.u.rsa.dst = req->dst;
+
+	ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd);
+
+	return ret;
+}
+
+static int ccp_rsa_encrypt(struct akcipher_request *req)
+{
+	return ccp_rsa_crypt(req, true);
+}
+
+static int ccp_rsa_decrypt(struct akcipher_request *req)
+{
+	return ccp_rsa_crypt(req, false);
+}
+
+static int ccp_check_key_length(unsigned int len)
+{
+	/* In bits */
+	if (len < 8 || len > 4096)
+		return -EINVAL;
+	return 0;
+}
+
+static void ccp_rsa_free_key_bufs(struct ccp_ctx *ctx)
+{
+	/* Clean up old key data */
+	kzfree(ctx->u.rsa.e_buf);
+	ctx->u.rsa.e_buf = NULL;
+	ctx->u.rsa.e_len = 0;
+	kzfree(ctx->u.rsa.n_buf);
+	ctx->u.rsa.n_buf = NULL;
+	ctx->u.rsa.n_len = 0;
+	kzfree(ctx->u.rsa.d_buf);
+	ctx->u.rsa.d_buf = NULL;
+	ctx->u.rsa.d_len = 0;
+}
+
+static int ccp_rsa_setkey(struct crypto_akcipher *tfm, const void *key,
+			  unsigned int keylen, bool private)
+{
+	struct ccp_ctx *ctx = akcipher_tfm_ctx(tfm);
+	struct rsa_key raw_key;
+	int ret;
+
+	ccp_rsa_free_key_bufs(ctx);
+	memset(&raw_key, 0, sizeof(raw_key));
+
+	/* Code borrowed from crypto/rsa.c */
+	if (private)
+		ret = rsa_parse_priv_key(&raw_key, key, keylen);
+	else
+		ret = rsa_parse_pub_key(&raw_key, key, keylen);
+	if (ret)
+		goto n_key;
+
+	ret = ccp_copy_and_save_keypart(&ctx->u.rsa.n_buf, &ctx->u.rsa.n_len,
+					raw_key.n, raw_key.n_sz);
+	if (ret)
+		goto key_err;
+	sg_init_one(&ctx->u.rsa.n_sg, ctx->u.rsa.n_buf, ctx->u.rsa.n_len);
+
+	ctx->u.rsa.key_len = ctx->u.rsa.n_len << 3; /* convert to bits */
+	if (ccp_check_key_length(ctx->u.rsa.key_len)) {
+		ret = -EINVAL;
+		goto key_err;
+	}
+
+	ret = ccp_copy_and_save_keypart(&ctx->u.rsa.e_buf, &ctx->u.rsa.e_len,
+					raw_key.e, raw_key.e_sz);
+	if (ret)
+		goto key_err;
+	sg_init_one(&ctx->u.rsa.e_sg, ctx->u.rsa.e_buf, ctx->u.rsa.e_len);
+
+	if (private) {
+		ret = ccp_copy_and_save_keypart(&ctx->u.rsa.d_buf,
+						&ctx->u.rsa.d_len,
+						raw_key.d, raw_key.d_sz);
+		if (ret)
+			goto key_err;
+		sg_init_one(&ctx->u.rsa.d_sg,
+			    ctx->u.rsa.d_buf, ctx->u.rsa.d_len);
+	}
+
+	return 0;
+
+key_err:
+	ccp_rsa_free_key_bufs(ctx);
+
+n_key:
+	return ret;
+}
+
+static int ccp_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key,
+			      unsigned int keylen)
+{
+	return ccp_rsa_setkey(tfm, key, keylen, true);
+}
+
+static int ccp_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key,
+			     unsigned int keylen)
+{
+	return ccp_rsa_setkey(tfm, key, keylen, false);
+}
+
+static int ccp_rsa_init_tfm(struct crypto_akcipher *tfm)
+{
+	struct ccp_ctx *ctx = akcipher_tfm_ctx(tfm);
+
+	akcipher_set_reqsize(tfm, sizeof(struct ccp_rsa_req_ctx));
+	ctx->complete = ccp_rsa_complete;
+
+	return 0;
+}
+
+static void ccp_rsa_exit_tfm(struct crypto_akcipher *tfm)
+{
+	struct ccp_ctx *ctx = crypto_tfm_ctx(&tfm->base);
+
+	ccp_rsa_free_key_bufs(ctx);
+}
+
+static struct akcipher_alg ccp_rsa_defaults = {
+	.encrypt = ccp_rsa_encrypt,
+	.decrypt = ccp_rsa_decrypt,
+	.sign = ccp_rsa_decrypt,
+	.verify = ccp_rsa_encrypt,
+	.set_pub_key = ccp_rsa_setpubkey,
+	.set_priv_key = ccp_rsa_setprivkey,
+	.max_size = ccp_rsa_maxsize,
+	.init = ccp_rsa_init_tfm,
+	.exit = ccp_rsa_exit_tfm,
+	.base = {
+		.cra_name = "rsa",
+		.cra_driver_name = "rsa-ccp",
+		.cra_priority = CCP_CRA_PRIORITY,
+		.cra_module = THIS_MODULE,
+		.cra_ctxsize = 2 * sizeof(struct ccp_ctx),
+	},
+};
+
+struct ccp_rsa_def {
+	unsigned int version;
+	const char *name;
+	const char *driver_name;
+	unsigned int reqsize;
+	struct akcipher_alg *alg_defaults;
+};
+
+static struct ccp_rsa_def rsa_algs[] = {
+	{
+		.version	= CCP_VERSION(3, 0),
+		.name		= "rsa",
+		.driver_name	= "rsa-ccp",
+		.reqsize	= sizeof(struct ccp_rsa_req_ctx),
+		.alg_defaults	= &ccp_rsa_defaults,
+	}
+};
+
+int ccp_register_rsa_alg(struct list_head *head, const struct ccp_rsa_def *def)
+{
+	struct ccp_crypto_akcipher_alg *ccp_alg;
+	struct akcipher_alg *alg;
+	int ret;
+
+	ccp_alg = kzalloc(sizeof(*ccp_alg), GFP_KERNEL);
+	if (!ccp_alg)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&ccp_alg->entry);
+
+	alg = &ccp_alg->alg;
+	*alg = *def->alg_defaults;
+	snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name);
+	snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
+		 def->driver_name);
+	ret = crypto_register_akcipher(alg);
+	if (ret) {
+		pr_err("%s akcipher algorithm registration error (%d)\n",
+		       alg->base.cra_name, ret);
+		kfree(ccp_alg);
+		return ret;
+	}
+
+	list_add(&ccp_alg->entry, head);
+
+	return 0;
+}
+
+int ccp_register_rsa_algs(struct list_head *head)
+{
+	int i, ret;
+	unsigned int ccpversion = ccp_version();
+
+	/* Register the RSA algorithm in standard mode
+	 * This works for CCP v3 and later
+	 */
+	for (i = 0; i < ARRAY_SIZE(rsa_algs); i++) {
+		if (rsa_algs[i].version > ccpversion)
+			continue;
+		ret = ccp_register_rsa_alg(head, &rsa_algs[i]);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
diff --git a/drivers/crypto/ccp/ccp-crypto-sha.c b/drivers/crypto/ccp/ccp-crypto-sha.c
index ce97b3868f4a..8b9b16d433f7 100644
--- a/drivers/crypto/ccp/ccp-crypto-sha.c
+++ b/drivers/crypto/ccp/ccp-crypto-sha.c
@@ -1,7 +1,7 @@
 /*
  * AMD Cryptographic Coprocessor (CCP) SHA crypto API support
  *
- * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
+ * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
  *
  * Author: Tom Lendacky <thomas.lendacky@amd.com>
  * Author: Gary R Hook <gary.hook@amd.com>
diff --git a/drivers/crypto/ccp/ccp-crypto.h b/drivers/crypto/ccp/ccp-crypto.h
index dd5bf15f06e5..b9fd090c46c2 100644
--- a/drivers/crypto/ccp/ccp-crypto.h
+++ b/drivers/crypto/ccp/ccp-crypto.h
@@ -1,7 +1,7 @@
 /*
  * AMD Cryptographic Coprocessor (CCP) crypto API support
  *
- * Copyright (C) 2013 Advanced Micro Devices, Inc.
+ * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
  *
  * Author: Tom Lendacky <thomas.lendacky@amd.com>
  *
@@ -24,6 +24,8 @@
 #include <crypto/ctr.h>
 #include <crypto/hash.h>
 #include <crypto/sha.h>
+#include <crypto/akcipher.h>
+#include <crypto/internal/rsa.h>
 
 #define	CCP_LOG_LEVEL	KERN_INFO
 
@@ -58,6 +60,12 @@ struct ccp_crypto_ahash_alg {
 	struct ahash_alg alg;
 };
 
+struct ccp_crypto_akcipher_alg {
+	struct list_head entry;
+
+	struct akcipher_alg alg;
+};
+
 static inline struct ccp_crypto_ablkcipher_alg *
 	ccp_crypto_ablkcipher_alg(struct crypto_tfm *tfm)
 {
@@ -91,7 +99,7 @@ struct ccp_aes_ctx {
 
 	struct scatterlist key_sg;
 	unsigned int key_len;
-	u8 key[AES_MAX_KEY_SIZE];
+	u8 key[AES_MAX_KEY_SIZE * 2];
 
 	u8 nonce[CTR_RFC3686_NONCE_SIZE];
 
@@ -227,12 +235,35 @@ struct ccp_sha_exp_ctx {
 	u8 buf[MAX_SHA_BLOCK_SIZE];
 };
 
+/***** RSA related defines *****/
+
+struct ccp_rsa_ctx {
+	unsigned int key_len; /* in bits */
+	struct scatterlist e_sg;
+	u8 *e_buf;
+	unsigned int e_len;
+	struct scatterlist n_sg;
+	u8 *n_buf;
+	unsigned int n_len;
+	struct scatterlist d_sg;
+	u8 *d_buf;
+	unsigned int d_len;
+};
+
+struct ccp_rsa_req_ctx {
+	struct ccp_cmd cmd;
+};
+
+#define	CCP_RSA_MAXMOD	(4 * 1024 / 8)
+#define	CCP5_RSA_MAXMOD	(16 * 1024 / 8)
+
 /***** Common Context Structure *****/
 struct ccp_ctx {
 	int (*complete)(struct crypto_async_request *req, int ret);
 
 	union {
 		struct ccp_aes_ctx aes;
+		struct ccp_rsa_ctx rsa;
 		struct ccp_sha_ctx sha;
 		struct ccp_des3_ctx des3;
 	} u;
@@ -249,5 +280,6 @@ int ccp_register_aes_xts_algs(struct list_head *head);
 int ccp_register_aes_aeads(struct list_head *head);
 int ccp_register_sha_algs(struct list_head *head);
 int ccp_register_des3_algs(struct list_head *head);
+int ccp_register_rsa_algs(struct list_head *head);
 
 #endif
diff --git a/drivers/crypto/ccp/ccp-debugfs.c b/drivers/crypto/ccp/ccp-debugfs.c
index 3cd6c83754e0..59d4ca4e72d8 100644
--- a/drivers/crypto/ccp/ccp-debugfs.c
+++ b/drivers/crypto/ccp/ccp-debugfs.c
@@ -305,19 +305,19 @@ void ccp5_debugfs_setup(struct ccp_device *ccp)
 
 	ccp->debugfs_instance = debugfs_create_dir(ccp->name, ccp_debugfs_dir);
 	if (!ccp->debugfs_instance)
-		return;
+		goto err;
 
 	debugfs_info = debugfs_create_file("info", 0400,
 					   ccp->debugfs_instance, ccp,
 					   &ccp_debugfs_info_ops);
 	if (!debugfs_info)
-		return;
+		goto err;
 
 	debugfs_stats = debugfs_create_file("stats", 0600,
 					    ccp->debugfs_instance, ccp,
 					    &ccp_debugfs_stats_ops);
 	if (!debugfs_stats)
-		return;
+		goto err;
 
 	for (i = 0; i < ccp->cmd_q_count; i++) {
 		cmd_q = &ccp->cmd_q[i];
@@ -327,15 +327,20 @@ void ccp5_debugfs_setup(struct ccp_device *ccp)
 		debugfs_q_instance =
 			debugfs_create_dir(name, ccp->debugfs_instance);
 		if (!debugfs_q_instance)
-			return;
+			goto err;
 
 		debugfs_q_stats =
 			debugfs_create_file("stats", 0600,
 					    debugfs_q_instance, cmd_q,
 					    &ccp_debugfs_queue_ops);
 		if (!debugfs_q_stats)
-			return;
+			goto err;
 	}
+
+	return;
+
+err:
+	debugfs_remove_recursive(ccp->debugfs_instance);
 }
 
 void ccp5_debugfs_destroy(void)
diff --git a/drivers/crypto/ccp/ccp-dev-v3.c b/drivers/crypto/ccp/ccp-dev-v3.c
index 367c2e30656f..240bebbcb8ac 100644
--- a/drivers/crypto/ccp/ccp-dev-v3.c
+++ b/drivers/crypto/ccp/ccp-dev-v3.c
@@ -1,7 +1,7 @@
 /*
  * AMD Cryptographic Coprocessor (CCP) driver
  *
- * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
+ * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
  *
  * Author: Tom Lendacky <thomas.lendacky@amd.com>
  * Author: Gary R Hook <gary.hook@amd.com>
@@ -359,8 +359,7 @@ static void ccp_irq_bh(unsigned long data)
 
 static irqreturn_t ccp_irq_handler(int irq, void *data)
 {
-	struct device *dev = data;
-	struct ccp_device *ccp = dev_get_drvdata(dev);
+	struct ccp_device *ccp = (struct ccp_device *)data;
 
 	ccp_disable_queue_interrupts(ccp);
 	if (ccp->use_tasklet)
@@ -454,7 +453,7 @@ static int ccp_init(struct ccp_device *ccp)
 	iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG);
 
 	/* Request an irq */
-	ret = ccp->get_irq(ccp);
+	ret = sp_request_ccp_irq(ccp->sp, ccp_irq_handler, ccp->name, ccp);
 	if (ret) {
 		dev_err(dev, "unable to allocate an IRQ\n");
 		goto e_pool;
@@ -511,7 +510,7 @@ e_kthread:
 		if (ccp->cmd_q[i].kthread)
 			kthread_stop(ccp->cmd_q[i].kthread);
 
-	ccp->free_irq(ccp);
+	sp_free_ccp_irq(ccp->sp, ccp);
 
 e_pool:
 	for (i = 0; i < ccp->cmd_q_count; i++)
@@ -550,7 +549,7 @@ static void ccp_destroy(struct ccp_device *ccp)
 		if (ccp->cmd_q[i].kthread)
 			kthread_stop(ccp->cmd_q[i].kthread);
 
-	ccp->free_irq(ccp);
+	sp_free_ccp_irq(ccp->sp, ccp);
 
 	for (i = 0; i < ccp->cmd_q_count; i++)
 		dma_pool_destroy(ccp->cmd_q[i].dma_pool);
@@ -586,10 +585,17 @@ static const struct ccp_actions ccp3_actions = {
 	.irqhandler = ccp_irq_handler,
 };
 
+const struct ccp_vdata ccpv3_platform = {
+	.version = CCP_VERSION(3, 0),
+	.setup = NULL,
+	.perform = &ccp3_actions,
+	.offset = 0,
+};
+
 const struct ccp_vdata ccpv3 = {
 	.version = CCP_VERSION(3, 0),
 	.setup = NULL,
 	.perform = &ccp3_actions,
-	.bar = 2,
 	.offset = 0x20000,
+	.rsamax = CCP_RSA_MAX_WIDTH,
 };
diff --git a/drivers/crypto/ccp/ccp-dev-v5.c b/drivers/crypto/ccp/ccp-dev-v5.c
index b10d2d2075cb..65604fc65e8f 100644
--- a/drivers/crypto/ccp/ccp-dev-v5.c
+++ b/drivers/crypto/ccp/ccp-dev-v5.c
@@ -1,7 +1,7 @@
 /*
  * AMD Cryptographic Coprocessor (CCP) driver
  *
- * Copyright (C) 2016 Advanced Micro Devices, Inc.
+ * Copyright (C) 2016,2017 Advanced Micro Devices, Inc.
  *
  * Author: Gary R Hook <gary.hook@amd.com>
  *
@@ -145,6 +145,7 @@ union ccp_function {
 #define	CCP_AES_MODE(p)		((p)->aes.mode)
 #define	CCP_AES_TYPE(p)		((p)->aes.type)
 #define	CCP_XTS_SIZE(p)		((p)->aes_xts.size)
+#define	CCP_XTS_TYPE(p)		((p)->aes_xts.type)
 #define	CCP_XTS_ENCRYPT(p)	((p)->aes_xts.encrypt)
 #define	CCP_DES3_SIZE(p)	((p)->des3.size)
 #define	CCP_DES3_ENCRYPT(p)	((p)->des3.encrypt)
@@ -344,6 +345,7 @@ static int ccp5_perform_xts_aes(struct ccp_op *op)
 	CCP5_CMD_PROT(&desc) = 0;
 
 	function.raw = 0;
+	CCP_XTS_TYPE(&function) = op->u.xts.type;
 	CCP_XTS_ENCRYPT(&function) = op->u.xts.action;
 	CCP_XTS_SIZE(&function) = op->u.xts.unit_size;
 	CCP5_CMD_FUNCTION(&desc) = function.raw;
@@ -469,7 +471,7 @@ static int ccp5_perform_rsa(struct ccp_op *op)
 	CCP5_CMD_PROT(&desc) = 0;
 
 	function.raw = 0;
-	CCP_RSA_SIZE(&function) = op->u.rsa.mod_size >> 3;
+	CCP_RSA_SIZE(&function) = (op->u.rsa.mod_size + 7) >> 3;
 	CCP5_CMD_FUNCTION(&desc) = function.raw;
 
 	CCP5_CMD_LEN(&desc) = op->u.rsa.input_len;
@@ -484,10 +486,10 @@ static int ccp5_perform_rsa(struct ccp_op *op)
 	CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
 	CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
 
-	/* Exponent is in LSB memory */
-	CCP5_CMD_KEY_LO(&desc) = op->sb_key * LSB_ITEM_SIZE;
-	CCP5_CMD_KEY_HI(&desc) = 0;
-	CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
+	/* Key (Exponent) is in external memory */
+	CCP5_CMD_KEY_LO(&desc) = ccp_addr_lo(&op->exp.u.dma);
+	CCP5_CMD_KEY_HI(&desc) = ccp_addr_hi(&op->exp.u.dma);
+	CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
 
 	return ccp5_do_cmd(&desc, op->cmd_q);
 }
@@ -769,8 +771,7 @@ static void ccp5_irq_bh(unsigned long data)
 
 static irqreturn_t ccp5_irq_handler(int irq, void *data)
 {
-	struct device *dev = data;
-	struct ccp_device *ccp = dev_get_drvdata(dev);
+	struct ccp_device *ccp = (struct ccp_device *)data;
 
 	ccp5_disable_queue_interrupts(ccp);
 	ccp->total_interrupts++;
@@ -881,7 +882,7 @@ static int ccp5_init(struct ccp_device *ccp)
 
 	dev_dbg(dev, "Requesting an IRQ...\n");
 	/* Request an irq */
-	ret = ccp->get_irq(ccp);
+	ret = sp_request_ccp_irq(ccp->sp, ccp5_irq_handler, ccp->name, ccp);
 	if (ret) {
 		dev_err(dev, "unable to allocate an IRQ\n");
 		goto e_pool;
@@ -987,7 +988,7 @@ e_kthread:
 			kthread_stop(ccp->cmd_q[i].kthread);
 
 e_irq:
-	ccp->free_irq(ccp);
+	sp_free_ccp_irq(ccp->sp, ccp);
 
 e_pool:
 	for (i = 0; i < ccp->cmd_q_count; i++)
@@ -1037,7 +1038,7 @@ static void ccp5_destroy(struct ccp_device *ccp)
 		if (ccp->cmd_q[i].kthread)
 			kthread_stop(ccp->cmd_q[i].kthread);
 
-	ccp->free_irq(ccp);
+	sp_free_ccp_irq(ccp->sp, ccp);
 
 	for (i = 0; i < ccp->cmd_q_count; i++) {
 		cmd_q = &ccp->cmd_q[i];
@@ -1106,15 +1107,14 @@ static const struct ccp_actions ccp5_actions = {
 	.init = ccp5_init,
 	.destroy = ccp5_destroy,
 	.get_free_slots = ccp5_get_free_slots,
-	.irqhandler = ccp5_irq_handler,
 };
 
 const struct ccp_vdata ccpv5a = {
 	.version = CCP_VERSION(5, 0),
 	.setup = ccp5_config,
 	.perform = &ccp5_actions,
-	.bar = 2,
 	.offset = 0x0,
+	.rsamax = CCP5_RSA_MAX_WIDTH,
 };
 
 const struct ccp_vdata ccpv5b = {
@@ -1122,6 +1122,6 @@ const struct ccp_vdata ccpv5b = {
 	.dma_chan_attr = DMA_PRIVATE,
 	.setup = ccp5other_config,
 	.perform = &ccp5_actions,
-	.bar = 2,
 	.offset = 0x0,
+	.rsamax = CCP5_RSA_MAX_WIDTH,
 };
diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c
index 2506b5025700..4e029b176641 100644
--- a/drivers/crypto/ccp/ccp-dev.c
+++ b/drivers/crypto/ccp/ccp-dev.c
@@ -1,7 +1,7 @@
 /*
  * AMD Cryptographic Coprocessor (CCP) driver
  *
- * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
+ * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
  *
  * Author: Tom Lendacky <thomas.lendacky@amd.com>
  * Author: Gary R Hook <gary.hook@amd.com>
@@ -11,7 +11,6 @@
  * published by the Free Software Foundation.
  */
 
-#include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/kthread.h>
 #include <linux/sched.h>
@@ -30,12 +29,6 @@
 
 #include "ccp-dev.h"
 
-MODULE_AUTHOR("Tom Lendacky <thomas.lendacky@amd.com>");
-MODULE_AUTHOR("Gary R Hook <gary.hook@amd.com>");
-MODULE_LICENSE("GPL");
-MODULE_VERSION("1.1.0");
-MODULE_DESCRIPTION("AMD Cryptographic Coprocessor driver");
-
 struct ccp_tasklet_data {
 	struct completion completion;
 	struct ccp_cmd *cmd;
@@ -111,13 +104,6 @@ static LIST_HEAD(ccp_units);
 static DEFINE_SPINLOCK(ccp_rr_lock);
 static struct ccp_device *ccp_rr;
 
-/* Ever-increasing value to produce unique unit numbers */
-static atomic_t ccp_unit_ordinal;
-static unsigned int ccp_increment_unit_ordinal(void)
-{
-	return atomic_inc_return(&ccp_unit_ordinal);
-}
-
 /**
  * ccp_add_device - add a CCP device to the list
  *
@@ -415,6 +401,7 @@ static void ccp_do_cmd_complete(unsigned long data)
 	struct ccp_cmd *cmd = tdata->cmd;
 
 	cmd->callback(cmd->data, cmd->ret);
+
 	complete(&tdata->completion);
 }
 
@@ -464,14 +451,17 @@ int ccp_cmd_queue_thread(void *data)
  *
  * @dev: device struct of the CCP
  */
-struct ccp_device *ccp_alloc_struct(struct device *dev)
+struct ccp_device *ccp_alloc_struct(struct sp_device *sp)
 {
+	struct device *dev = sp->dev;
 	struct ccp_device *ccp;
 
 	ccp = devm_kzalloc(dev, sizeof(*ccp), GFP_KERNEL);
 	if (!ccp)
 		return NULL;
 	ccp->dev = dev;
+	ccp->sp = sp;
+	ccp->axcache = sp->axcache;
 
 	INIT_LIST_HEAD(&ccp->cmd);
 	INIT_LIST_HEAD(&ccp->backlog);
@@ -486,9 +476,8 @@ struct ccp_device *ccp_alloc_struct(struct device *dev)
 	init_waitqueue_head(&ccp->sb_queue);
 	init_waitqueue_head(&ccp->suspend_queue);
 
-	ccp->ord = ccp_increment_unit_ordinal();
-	snprintf(ccp->name, MAX_CCP_NAME_LEN, "ccp-%u", ccp->ord);
-	snprintf(ccp->rngname, MAX_CCP_NAME_LEN, "ccp-%u-rng", ccp->ord);
+	snprintf(ccp->name, MAX_CCP_NAME_LEN, "ccp-%u", sp->ord);
+	snprintf(ccp->rngname, MAX_CCP_NAME_LEN, "ccp-%u-rng", sp->ord);
 
 	return ccp;
 }
@@ -538,55 +527,100 @@ bool ccp_queues_suspended(struct ccp_device *ccp)
 
 	return ccp->cmd_q_count == suspended;
 }
-#endif
 
-static int __init ccp_mod_init(void)
+int ccp_dev_suspend(struct sp_device *sp, pm_message_t state)
 {
-#ifdef CONFIG_X86
-	int ret;
+	struct ccp_device *ccp = sp->ccp_data;
+	unsigned long flags;
+	unsigned int i;
 
-	ret = ccp_pci_init();
-	if (ret)
-		return ret;
+	spin_lock_irqsave(&ccp->cmd_lock, flags);
 
-	/* Don't leave the driver loaded if init failed */
-	if (ccp_present() != 0) {
-		ccp_pci_exit();
-		return -ENODEV;
+	ccp->suspending = 1;
+
+	/* Wake all the queue kthreads to prepare for suspend */
+	for (i = 0; i < ccp->cmd_q_count; i++)
+		wake_up_process(ccp->cmd_q[i].kthread);
+
+	spin_unlock_irqrestore(&ccp->cmd_lock, flags);
+
+	/* Wait for all queue kthreads to say they're done */
+	while (!ccp_queues_suspended(ccp))
+		wait_event_interruptible(ccp->suspend_queue,
+					 ccp_queues_suspended(ccp));
+
+	return 0;
+}
+
+int ccp_dev_resume(struct sp_device *sp)
+{
+	struct ccp_device *ccp = sp->ccp_data;
+	unsigned long flags;
+	unsigned int i;
+
+	spin_lock_irqsave(&ccp->cmd_lock, flags);
+
+	ccp->suspending = 0;
+
+	/* Wake up all the kthreads */
+	for (i = 0; i < ccp->cmd_q_count; i++) {
+		ccp->cmd_q[i].suspended = 0;
+		wake_up_process(ccp->cmd_q[i].kthread);
 	}
 
+	spin_unlock_irqrestore(&ccp->cmd_lock, flags);
+
 	return 0;
+}
 #endif
 
-#ifdef CONFIG_ARM64
+int ccp_dev_init(struct sp_device *sp)
+{
+	struct device *dev = sp->dev;
+	struct ccp_device *ccp;
 	int ret;
 
-	ret = ccp_platform_init();
+	ret = -ENOMEM;
+	ccp = ccp_alloc_struct(sp);
+	if (!ccp)
+		goto e_err;
+	sp->ccp_data = ccp;
+
+	ccp->vdata = (struct ccp_vdata *)sp->dev_vdata->ccp_vdata;
+	if (!ccp->vdata || !ccp->vdata->version) {
+		ret = -ENODEV;
+		dev_err(dev, "missing driver data\n");
+		goto e_err;
+	}
+
+	ccp->use_tasklet = sp->use_tasklet;
+
+	ccp->io_regs = sp->io_map + ccp->vdata->offset;
+	if (ccp->vdata->setup)
+		ccp->vdata->setup(ccp);
+
+	ret = ccp->vdata->perform->init(ccp);
 	if (ret)
-		return ret;
+		goto e_err;
 
-	/* Don't leave the driver loaded if init failed */
-	if (ccp_present() != 0) {
-		ccp_platform_exit();
-		return -ENODEV;
-	}
+	dev_notice(dev, "ccp enabled\n");
 
 	return 0;
-#endif
 
-	return -ENODEV;
+e_err:
+	sp->ccp_data = NULL;
+
+	dev_notice(dev, "ccp initialization failed\n");
+
+	return ret;
 }
 
-static void __exit ccp_mod_exit(void)
+void ccp_dev_destroy(struct sp_device *sp)
 {
-#ifdef CONFIG_X86
-	ccp_pci_exit();
-#endif
+	struct ccp_device *ccp = sp->ccp_data;
 
-#ifdef CONFIG_ARM64
-	ccp_platform_exit();
-#endif
-}
+	if (!ccp)
+		return;
 
-module_init(ccp_mod_init);
-module_exit(ccp_mod_exit);
+	ccp->vdata->perform->destroy(ccp);
+}
diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h
index a70154ac7405..6810b65c1939 100644
--- a/drivers/crypto/ccp/ccp-dev.h
+++ b/drivers/crypto/ccp/ccp-dev.h
@@ -1,7 +1,7 @@
 /*
  * AMD Cryptographic Coprocessor (CCP) driver
  *
- * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
+ * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
  *
  * Author: Tom Lendacky <thomas.lendacky@amd.com>
  * Author: Gary R Hook <gary.hook@amd.com>
@@ -27,6 +27,8 @@
 #include <linux/irqreturn.h>
 #include <linux/dmaengine.h>
 
+#include "sp-dev.h"
+
 #define MAX_CCP_NAME_LEN		16
 #define MAX_DMAPOOL_NAME_LEN		32
 
@@ -192,6 +194,7 @@
 #define CCP_AES_CTX_SB_COUNT		1
 
 #define CCP_XTS_AES_KEY_SB_COUNT	1
+#define CCP5_XTS_AES_KEY_SB_COUNT	2
 #define CCP_XTS_AES_CTX_SB_COUNT	1
 
 #define CCP_DES3_KEY_SB_COUNT		1
@@ -200,6 +203,7 @@
 #define CCP_SHA_SB_COUNT		1
 
 #define CCP_RSA_MAX_WIDTH		4096
+#define CCP5_RSA_MAX_WIDTH		16384
 
 #define CCP_PASSTHRU_BLOCKSIZE		256
 #define CCP_PASSTHRU_MASKSIZE		32
@@ -344,12 +348,11 @@ struct ccp_device {
 	char rngname[MAX_CCP_NAME_LEN];
 
 	struct device *dev;
+	struct sp_device *sp;
 
 	/* Bus specific device information
 	 */
 	void *dev_specific;
-	int (*get_irq)(struct ccp_device *ccp);
-	void (*free_irq)(struct ccp_device *ccp);
 	unsigned int qim;
 	unsigned int irq;
 	bool use_tasklet;
@@ -362,7 +365,6 @@ struct ccp_device {
 	 *   them.
 	 */
 	struct mutex req_mutex ____cacheline_aligned;
-	void __iomem *io_map;
 	void __iomem *io_regs;
 
 	/* Master lists that all cmds are queued on. Because there can be
@@ -497,6 +499,7 @@ struct ccp_aes_op {
 };
 
 struct ccp_xts_aes_op {
+	enum ccp_aes_type type;
 	enum ccp_aes_action action;
 	enum ccp_xts_aes_unit_size unit_size;
 };
@@ -626,18 +629,12 @@ struct ccp5_desc {
 	struct dword7 dw7;
 };
 
-int ccp_pci_init(void);
-void ccp_pci_exit(void);
-
-int ccp_platform_init(void);
-void ccp_platform_exit(void);
-
 void ccp_add_device(struct ccp_device *ccp);
 void ccp_del_device(struct ccp_device *ccp);
 
 extern void ccp_log_error(struct ccp_device *, int);
 
-struct ccp_device *ccp_alloc_struct(struct device *dev);
+struct ccp_device *ccp_alloc_struct(struct sp_device *sp);
 bool ccp_queues_suspended(struct ccp_device *ccp);
 int ccp_cmd_queue_thread(void *data);
 int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait);
@@ -669,16 +666,7 @@ struct ccp_actions {
 	irqreturn_t (*irqhandler)(int, void *);
 };
 
-/* Structure to hold CCP version-specific values */
-struct ccp_vdata {
-	const unsigned int version;
-	const unsigned int dma_chan_attr;
-	void (*setup)(struct ccp_device *);
-	const struct ccp_actions *perform;
-	const unsigned int bar;
-	const unsigned int offset;
-};
-
+extern const struct ccp_vdata ccpv3_platform;
 extern const struct ccp_vdata ccpv3;
 extern const struct ccp_vdata ccpv5a;
 extern const struct ccp_vdata ccpv5b;
diff --git a/drivers/crypto/ccp/ccp-dmaengine.c b/drivers/crypto/ccp/ccp-dmaengine.c
index e00be01fbf5a..d608043c0280 100644
--- a/drivers/crypto/ccp/ccp-dmaengine.c
+++ b/drivers/crypto/ccp/ccp-dmaengine.c
@@ -1,7 +1,7 @@
 /*
  * AMD Cryptographic Coprocessor (CCP) driver
  *
- * Copyright (C) 2016 Advanced Micro Devices, Inc.
+ * Copyright (C) 2016,2017 Advanced Micro Devices, Inc.
  *
  * Author: Gary R Hook <gary.hook@amd.com>
  *
@@ -502,27 +502,6 @@ static struct dma_async_tx_descriptor *ccp_prep_dma_memcpy(
 	return &desc->tx_desc;
 }
 
-static struct dma_async_tx_descriptor *ccp_prep_dma_sg(
-	struct dma_chan *dma_chan, struct scatterlist *dst_sg,
-	unsigned int dst_nents, struct scatterlist *src_sg,
-	unsigned int src_nents, unsigned long flags)
-{
-	struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
-						 dma_chan);
-	struct ccp_dma_desc *desc;
-
-	dev_dbg(chan->ccp->dev,
-		"%s - src=%p, src_nents=%u dst=%p, dst_nents=%u, flags=%#lx\n",
-		__func__, src_sg, src_nents, dst_sg, dst_nents, flags);
-
-	desc = ccp_create_desc(dma_chan, dst_sg, dst_nents, src_sg, src_nents,
-			       flags);
-	if (!desc)
-		return NULL;
-
-	return &desc->tx_desc;
-}
-
 static struct dma_async_tx_descriptor *ccp_prep_dma_interrupt(
 	struct dma_chan *dma_chan, unsigned long flags)
 {
@@ -704,7 +683,6 @@ int ccp_dmaengine_register(struct ccp_device *ccp)
 	dma_dev->directions = DMA_MEM_TO_MEM;
 	dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
 	dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
-	dma_cap_set(DMA_SG, dma_dev->cap_mask);
 	dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask);
 
 	/* The DMA channels for this device can be set to public or private,
@@ -740,7 +718,6 @@ int ccp_dmaengine_register(struct ccp_device *ccp)
 
 	dma_dev->device_free_chan_resources = ccp_free_chan_resources;
 	dma_dev->device_prep_dma_memcpy = ccp_prep_dma_memcpy;
-	dma_dev->device_prep_dma_sg = ccp_prep_dma_sg;
 	dma_dev->device_prep_dma_interrupt = ccp_prep_dma_interrupt;
 	dma_dev->device_issue_pending = ccp_issue_pending;
 	dma_dev->device_tx_status = ccp_tx_status;
diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c
index c0dfdacbdff5..406b95329b3d 100644
--- a/drivers/crypto/ccp/ccp-ops.c
+++ b/drivers/crypto/ccp/ccp-ops.c
@@ -1,7 +1,7 @@
 /*
  * AMD Cryptographic Coprocessor (CCP) driver
  *
- * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
+ * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
  *
  * Author: Tom Lendacky <thomas.lendacky@amd.com>
  * Author: Gary R Hook <gary.hook@amd.com>
@@ -168,7 +168,7 @@ static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
 
 		wa->dma.address = dma_map_single(wa->dev, wa->address, len,
 						 dir);
-		if (!wa->dma.address)
+		if (dma_mapping_error(wa->dev, wa->dma.address))
 			return -ENOMEM;
 
 		wa->dma.length = len;
@@ -1038,6 +1038,8 @@ static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
 	struct ccp_op op;
 	unsigned int unit_size, dm_offset;
 	bool in_place = false;
+	unsigned int sb_count;
+	enum ccp_aes_type aestype;
 	int ret;
 
 	switch (xts->unit_size) {
@@ -1061,7 +1063,11 @@ static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
 		return -EINVAL;
 	}
 
-	if (xts->key_len != AES_KEYSIZE_128)
+	if (xts->key_len == AES_KEYSIZE_128)
+		aestype = CCP_AES_TYPE_128;
+	else if (xts->key_len == AES_KEYSIZE_256)
+		aestype = CCP_AES_TYPE_256;
+	else
 		return -EINVAL;
 
 	if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
@@ -1083,23 +1089,44 @@ static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
 	op.sb_key = cmd_q->sb_key;
 	op.sb_ctx = cmd_q->sb_ctx;
 	op.init = 1;
+	op.u.xts.type = aestype;
 	op.u.xts.action = xts->action;
 	op.u.xts.unit_size = xts->unit_size;
 
-	/* All supported key sizes fit in a single (32-byte) SB entry
-	 * and must be in little endian format. Use the 256-bit byte
-	 * swap passthru option to convert from big endian to little
-	 * endian.
+	/* A version 3 device only supports 128-bit keys, which fits into a
+	 * single SB entry. A version 5 device uses a 512-bit vector, so two
+	 * SB entries.
 	 */
+	if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
+		sb_count = CCP_XTS_AES_KEY_SB_COUNT;
+	else
+		sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
 	ret = ccp_init_dm_workarea(&key, cmd_q,
-				   CCP_XTS_AES_KEY_SB_COUNT * CCP_SB_BYTES,
+				   sb_count * CCP_SB_BYTES,
 				   DMA_TO_DEVICE);
 	if (ret)
 		return ret;
 
-	dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
-	ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
-	ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len);
+	if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
+		/* All supported key sizes must be in little endian format.
+		 * Use the 256-bit byte swap passthru option to convert from
+		 * big endian to little endian.
+		 */
+		dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
+		ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
+		ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len);
+	} else {
+		/* Version 5 CCPs use a 512-bit space for the key: each portion
+		 * occupies 256 bits, or one entire slot, and is zero-padded.
+		 */
+		unsigned int pad;
+
+		dm_offset = CCP_SB_BYTES;
+		pad = dm_offset - xts->key_len;
+		ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len);
+		ccp_set_dm_area(&key, dm_offset + pad, xts->key, xts->key_len,
+				xts->key_len);
+	}
 	ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 			     CCP_PASSTHRU_BYTESWAP_256BIT);
 	if (ret) {
@@ -1731,42 +1758,53 @@ e_ctx:
 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 {
 	struct ccp_rsa_engine *rsa = &cmd->u.rsa;
-	struct ccp_dm_workarea exp, src;
-	struct ccp_data dst;
+	struct ccp_dm_workarea exp, src, dst;
 	struct ccp_op op;
 	unsigned int sb_count, i_len, o_len;
 	int ret;
 
-	if (rsa->key_size > CCP_RSA_MAX_WIDTH)
+	/* Check against the maximum allowable size, in bits */
+	if (rsa->key_size > cmd_q->ccp->vdata->rsamax)
 		return -EINVAL;
 
 	if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
 		return -EINVAL;
 
+	memset(&op, 0, sizeof(op));
+	op.cmd_q = cmd_q;
+	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
+
 	/* The RSA modulus must precede the message being acted upon, so
 	 * it must be copied to a DMA area where the message and the
 	 * modulus can be concatenated.  Therefore the input buffer
 	 * length required is twice the output buffer length (which
-	 * must be a multiple of 256-bits).
+	 * must be a multiple of 256-bits).  Compute o_len, i_len in bytes.
+	 * Buffer sizes must be a multiple of 32 bytes; rounding up may be
+	 * required.
 	 */
-	o_len = ((rsa->key_size + 255) / 256) * 32;
+	o_len = 32 * ((rsa->key_size + 255) / 256);
 	i_len = o_len * 2;
 
-	sb_count = o_len / CCP_SB_BYTES;
-
-	memset(&op, 0, sizeof(op));
-	op.cmd_q = cmd_q;
-	op.jobid = ccp_gen_jobid(cmd_q->ccp);
-	op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q, sb_count);
-
-	if (!op.sb_key)
-		return -EIO;
+	sb_count = 0;
+	if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
+		/* sb_count is the number of storage block slots required
+		 * for the modulus.
+		 */
+		sb_count = o_len / CCP_SB_BYTES;
+		op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q,
+								sb_count);
+		if (!op.sb_key)
+			return -EIO;
+	} else {
+		/* A version 5 device allows a modulus size that will not fit
+		 * in the LSB, so the command will transfer it from memory.
+		 * Set the sb key to the default, even though it's not used.
+		 */
+		op.sb_key = cmd_q->sb_key;
+	}
 
-	/* The RSA exponent may span multiple (32-byte) SB entries and must
-	 * be in little endian format. Reverse copy each 32-byte chunk
-	 * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk)
-	 * and each byte within that chunk and do not perform any byte swap
-	 * operations on the passthru operation.
+	/* The RSA exponent must be in little endian format. Reverse its
+	 * byte order.
 	 */
 	ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
 	if (ret)
@@ -1775,11 +1813,22 @@ static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 	ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len);
 	if (ret)
 		goto e_exp;
-	ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
-			     CCP_PASSTHRU_BYTESWAP_NOOP);
-	if (ret) {
-		cmd->engine_error = cmd_q->cmd_error;
-		goto e_exp;
+
+	if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
+		/* Copy the exponent to the local storage block, using
+		 * as many 32-byte blocks as were allocated above. It's
+		 * already little endian, so no further change is required.
+		 */
+		ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
+				     CCP_PASSTHRU_BYTESWAP_NOOP);
+		if (ret) {
+			cmd->engine_error = cmd_q->cmd_error;
+			goto e_exp;
+		}
+	} else {
+		/* The exponent can be retrieved from memory via DMA. */
+		op.exp.u.dma.address = exp.dma.address;
+		op.exp.u.dma.offset = 0;
 	}
 
 	/* Concatenate the modulus and the message. Both the modulus and
@@ -1798,8 +1847,7 @@ static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 		goto e_src;
 
 	/* Prepare the output area for the operation */
-	ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len,
-			    o_len, DMA_FROM_DEVICE);
+	ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE);
 	if (ret)
 		goto e_src;
 
@@ -1807,7 +1855,7 @@ static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 	op.src.u.dma.address = src.dma.address;
 	op.src.u.dma.offset = 0;
 	op.src.u.dma.length = i_len;
-	op.dst.u.dma.address = dst.dm_wa.dma.address;
+	op.dst.u.dma.address = dst.dma.address;
 	op.dst.u.dma.offset = 0;
 	op.dst.u.dma.length = o_len;
 
@@ -1820,10 +1868,10 @@ static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 		goto e_dst;
 	}
 
-	ccp_reverse_get_dm_area(&dst.dm_wa, 0, rsa->dst, 0, rsa->mod_len);
+	ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len);
 
 e_dst:
-	ccp_free_data(&dst, cmd_q);
+	ccp_dm_free(&dst);
 
 e_src:
 	ccp_dm_free(&src);
@@ -1832,7 +1880,8 @@ e_exp:
 	ccp_dm_free(&exp);
 
 e_sb:
-	cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
+	if (sb_count)
+		cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
 
 	return ret;
 }
@@ -1992,7 +2041,7 @@ static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
 
 	memset(&op, 0, sizeof(op));
 	op.cmd_q = cmd_q;
-	op.jobid = ccp_gen_jobid(cmd_q->ccp);
+	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 
 	if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
 		/* Load the mask */
diff --git a/drivers/crypto/ccp/ccp-pci.c b/drivers/crypto/ccp/ccp-pci.c
deleted file mode 100644
index e880d4cf4ada..000000000000
--- a/drivers/crypto/ccp/ccp-pci.c
+++ /dev/null
@@ -1,356 +0,0 @@
-/*
- * AMD Cryptographic Coprocessor (CCP) driver
- *
- * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
- *
- * Author: Tom Lendacky <thomas.lendacky@amd.com>
- * Author: Gary R Hook <gary.hook@amd.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/device.h>
-#include <linux/pci.h>
-#include <linux/pci_ids.h>
-#include <linux/dma-mapping.h>
-#include <linux/kthread.h>
-#include <linux/sched.h>
-#include <linux/interrupt.h>
-#include <linux/spinlock.h>
-#include <linux/delay.h>
-#include <linux/ccp.h>
-
-#include "ccp-dev.h"
-
-#define MSIX_VECTORS			2
-
-struct ccp_msix {
-	u32 vector;
-	char name[16];
-};
-
-struct ccp_pci {
-	int msix_count;
-	struct ccp_msix msix[MSIX_VECTORS];
-};
-
-static int ccp_get_msix_irqs(struct ccp_device *ccp)
-{
-	struct ccp_pci *ccp_pci = ccp->dev_specific;
-	struct device *dev = ccp->dev;
-	struct pci_dev *pdev = to_pci_dev(dev);
-	struct msix_entry msix_entry[MSIX_VECTORS];
-	unsigned int name_len = sizeof(ccp_pci->msix[0].name) - 1;
-	int v, ret;
-
-	for (v = 0; v < ARRAY_SIZE(msix_entry); v++)
-		msix_entry[v].entry = v;
-
-	ret = pci_enable_msix_range(pdev, msix_entry, 1, v);
-	if (ret < 0)
-		return ret;
-
-	ccp_pci->msix_count = ret;
-	for (v = 0; v < ccp_pci->msix_count; v++) {
-		/* Set the interrupt names and request the irqs */
-		snprintf(ccp_pci->msix[v].name, name_len, "%s-%u",
-			 ccp->name, v);
-		ccp_pci->msix[v].vector = msix_entry[v].vector;
-		ret = request_irq(ccp_pci->msix[v].vector,
-				  ccp->vdata->perform->irqhandler,
-				  0, ccp_pci->msix[v].name, dev);
-		if (ret) {
-			dev_notice(dev, "unable to allocate MSI-X IRQ (%d)\n",
-				   ret);
-			goto e_irq;
-		}
-	}
-	ccp->use_tasklet = true;
-
-	return 0;
-
-e_irq:
-	while (v--)
-		free_irq(ccp_pci->msix[v].vector, dev);
-
-	pci_disable_msix(pdev);
-
-	ccp_pci->msix_count = 0;
-
-	return ret;
-}
-
-static int ccp_get_msi_irq(struct ccp_device *ccp)
-{
-	struct device *dev = ccp->dev;
-	struct pci_dev *pdev = to_pci_dev(dev);
-	int ret;
-
-	ret = pci_enable_msi(pdev);
-	if (ret)
-		return ret;
-
-	ccp->irq = pdev->irq;
-	ret = request_irq(ccp->irq, ccp->vdata->perform->irqhandler, 0,
-			  ccp->name, dev);
-	if (ret) {
-		dev_notice(dev, "unable to allocate MSI IRQ (%d)\n", ret);
-		goto e_msi;
-	}
-	ccp->use_tasklet = true;
-
-	return 0;
-
-e_msi:
-	pci_disable_msi(pdev);
-
-	return ret;
-}
-
-static int ccp_get_irqs(struct ccp_device *ccp)
-{
-	struct device *dev = ccp->dev;
-	int ret;
-
-	ret = ccp_get_msix_irqs(ccp);
-	if (!ret)
-		return 0;
-
-	/* Couldn't get MSI-X vectors, try MSI */
-	dev_notice(dev, "could not enable MSI-X (%d), trying MSI\n", ret);
-	ret = ccp_get_msi_irq(ccp);
-	if (!ret)
-		return 0;
-
-	/* Couldn't get MSI interrupt */
-	dev_notice(dev, "could not enable MSI (%d)\n", ret);
-
-	return ret;
-}
-
-static void ccp_free_irqs(struct ccp_device *ccp)
-{
-	struct ccp_pci *ccp_pci = ccp->dev_specific;
-	struct device *dev = ccp->dev;
-	struct pci_dev *pdev = to_pci_dev(dev);
-
-	if (ccp_pci->msix_count) {
-		while (ccp_pci->msix_count--)
-			free_irq(ccp_pci->msix[ccp_pci->msix_count].vector,
-				 dev);
-		pci_disable_msix(pdev);
-	} else if (ccp->irq) {
-		free_irq(ccp->irq, dev);
-		pci_disable_msi(pdev);
-	}
-	ccp->irq = 0;
-}
-
-static int ccp_find_mmio_area(struct ccp_device *ccp)
-{
-	struct device *dev = ccp->dev;
-	struct pci_dev *pdev = to_pci_dev(dev);
-	resource_size_t io_len;
-	unsigned long io_flags;
-
-	io_flags = pci_resource_flags(pdev, ccp->vdata->bar);
-	io_len = pci_resource_len(pdev, ccp->vdata->bar);
-	if ((io_flags & IORESOURCE_MEM) &&
-	    (io_len >= (ccp->vdata->offset + 0x800)))
-		return ccp->vdata->bar;
-
-	return -EIO;
-}
-
-static int ccp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
-{
-	struct ccp_device *ccp;
-	struct ccp_pci *ccp_pci;
-	struct device *dev = &pdev->dev;
-	unsigned int bar;
-	int ret;
-
-	ret = -ENOMEM;
-	ccp = ccp_alloc_struct(dev);
-	if (!ccp)
-		goto e_err;
-
-	ccp_pci = devm_kzalloc(dev, sizeof(*ccp_pci), GFP_KERNEL);
-	if (!ccp_pci)
-		goto e_err;
-
-	ccp->dev_specific = ccp_pci;
-	ccp->vdata = (struct ccp_vdata *)id->driver_data;
-	if (!ccp->vdata || !ccp->vdata->version) {
-		ret = -ENODEV;
-		dev_err(dev, "missing driver data\n");
-		goto e_err;
-	}
-	ccp->get_irq = ccp_get_irqs;
-	ccp->free_irq = ccp_free_irqs;
-
-	ret = pci_request_regions(pdev, "ccp");
-	if (ret) {
-		dev_err(dev, "pci_request_regions failed (%d)\n", ret);
-		goto e_err;
-	}
-
-	ret = pci_enable_device(pdev);
-	if (ret) {
-		dev_err(dev, "pci_enable_device failed (%d)\n", ret);
-		goto e_regions;
-	}
-
-	pci_set_master(pdev);
-
-	ret = ccp_find_mmio_area(ccp);
-	if (ret < 0)
-		goto e_device;
-	bar = ret;
-
-	ret = -EIO;
-	ccp->io_map = pci_iomap(pdev, bar, 0);
-	if (!ccp->io_map) {
-		dev_err(dev, "pci_iomap failed\n");
-		goto e_device;
-	}
-	ccp->io_regs = ccp->io_map + ccp->vdata->offset;
-
-	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
-	if (ret) {
-		ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
-		if (ret) {
-			dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n",
-				ret);
-			goto e_iomap;
-		}
-	}
-
-	dev_set_drvdata(dev, ccp);
-
-	if (ccp->vdata->setup)
-		ccp->vdata->setup(ccp);
-
-	ret = ccp->vdata->perform->init(ccp);
-	if (ret)
-		goto e_iomap;
-
-	dev_notice(dev, "enabled\n");
-
-	return 0;
-
-e_iomap:
-	pci_iounmap(pdev, ccp->io_map);
-
-e_device:
-	pci_disable_device(pdev);
-
-e_regions:
-	pci_release_regions(pdev);
-
-e_err:
-	dev_notice(dev, "initialization failed\n");
-	return ret;
-}
-
-static void ccp_pci_remove(struct pci_dev *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct ccp_device *ccp = dev_get_drvdata(dev);
-
-	if (!ccp)
-		return;
-
-	ccp->vdata->perform->destroy(ccp);
-
-	pci_iounmap(pdev, ccp->io_map);
-
-	pci_disable_device(pdev);
-
-	pci_release_regions(pdev);
-
-	dev_notice(dev, "disabled\n");
-}
-
-#ifdef CONFIG_PM
-static int ccp_pci_suspend(struct pci_dev *pdev, pm_message_t state)
-{
-	struct device *dev = &pdev->dev;
-	struct ccp_device *ccp = dev_get_drvdata(dev);
-	unsigned long flags;
-	unsigned int i;
-
-	spin_lock_irqsave(&ccp->cmd_lock, flags);
-
-	ccp->suspending = 1;
-
-	/* Wake all the queue kthreads to prepare for suspend */
-	for (i = 0; i < ccp->cmd_q_count; i++)
-		wake_up_process(ccp->cmd_q[i].kthread);
-
-	spin_unlock_irqrestore(&ccp->cmd_lock, flags);
-
-	/* Wait for all queue kthreads to say they're done */
-	while (!ccp_queues_suspended(ccp))
-		wait_event_interruptible(ccp->suspend_queue,
-					 ccp_queues_suspended(ccp));
-
-	return 0;
-}
-
-static int ccp_pci_resume(struct pci_dev *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct ccp_device *ccp = dev_get_drvdata(dev);
-	unsigned long flags;
-	unsigned int i;
-
-	spin_lock_irqsave(&ccp->cmd_lock, flags);
-
-	ccp->suspending = 0;
-
-	/* Wake up all the kthreads */
-	for (i = 0; i < ccp->cmd_q_count; i++) {
-		ccp->cmd_q[i].suspended = 0;
-		wake_up_process(ccp->cmd_q[i].kthread);
-	}
-
-	spin_unlock_irqrestore(&ccp->cmd_lock, flags);
-
-	return 0;
-}
-#endif
-
-static const struct pci_device_id ccp_pci_table[] = {
-	{ PCI_VDEVICE(AMD, 0x1537), (kernel_ulong_t)&ccpv3 },
-	{ PCI_VDEVICE(AMD, 0x1456), (kernel_ulong_t)&ccpv5a },
-	{ PCI_VDEVICE(AMD, 0x1468), (kernel_ulong_t)&ccpv5b },
-	/* Last entry must be zero */
-	{ 0, }
-};
-MODULE_DEVICE_TABLE(pci, ccp_pci_table);
-
-static struct pci_driver ccp_pci_driver = {
-	.name = "ccp",
-	.id_table = ccp_pci_table,
-	.probe = ccp_pci_probe,
-	.remove = ccp_pci_remove,
-#ifdef CONFIG_PM
-	.suspend = ccp_pci_suspend,
-	.resume = ccp_pci_resume,
-#endif
-};
-
-int ccp_pci_init(void)
-{
-	return pci_register_driver(&ccp_pci_driver);
-}
-
-void ccp_pci_exit(void)
-{
-	pci_unregister_driver(&ccp_pci_driver);
-}
diff --git a/drivers/crypto/ccp/ccp-platform.c b/drivers/crypto/ccp/ccp-platform.c
deleted file mode 100644
index e26969e601ad..000000000000
--- a/drivers/crypto/ccp/ccp-platform.c
+++ /dev/null
@@ -1,293 +0,0 @@
-/*
- * AMD Cryptographic Coprocessor (CCP) driver
- *
- * Copyright (C) 2014,2016 Advanced Micro Devices, Inc.
- *
- * Author: Tom Lendacky <thomas.lendacky@amd.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/device.h>
-#include <linux/platform_device.h>
-#include <linux/ioport.h>
-#include <linux/dma-mapping.h>
-#include <linux/kthread.h>
-#include <linux/sched.h>
-#include <linux/interrupt.h>
-#include <linux/spinlock.h>
-#include <linux/delay.h>
-#include <linux/ccp.h>
-#include <linux/of.h>
-#include <linux/of_address.h>
-#include <linux/acpi.h>
-
-#include "ccp-dev.h"
-
-struct ccp_platform {
-	int coherent;
-};
-
-static const struct acpi_device_id ccp_acpi_match[];
-static const struct of_device_id ccp_of_match[];
-
-static struct ccp_vdata *ccp_get_of_version(struct platform_device *pdev)
-{
-#ifdef CONFIG_OF
-	const struct of_device_id *match;
-
-	match = of_match_node(ccp_of_match, pdev->dev.of_node);
-	if (match && match->data)
-		return (struct ccp_vdata *)match->data;
-#endif
-	return NULL;
-}
-
-static struct ccp_vdata *ccp_get_acpi_version(struct platform_device *pdev)
-{
-#ifdef CONFIG_ACPI
-	const struct acpi_device_id *match;
-
-	match = acpi_match_device(ccp_acpi_match, &pdev->dev);
-	if (match && match->driver_data)
-		return (struct ccp_vdata *)match->driver_data;
-#endif
-	return NULL;
-}
-
-static int ccp_get_irq(struct ccp_device *ccp)
-{
-	struct device *dev = ccp->dev;
-	struct platform_device *pdev = to_platform_device(dev);
-	int ret;
-
-	ret = platform_get_irq(pdev, 0);
-	if (ret < 0)
-		return ret;
-
-	ccp->irq = ret;
-	ret = request_irq(ccp->irq, ccp->vdata->perform->irqhandler, 0,
-			  ccp->name, dev);
-	if (ret) {
-		dev_notice(dev, "unable to allocate IRQ (%d)\n", ret);
-		return ret;
-	}
-
-	return 0;
-}
-
-static int ccp_get_irqs(struct ccp_device *ccp)
-{
-	struct device *dev = ccp->dev;
-	int ret;
-
-	ret = ccp_get_irq(ccp);
-	if (!ret)
-		return 0;
-
-	/* Couldn't get an interrupt */
-	dev_notice(dev, "could not enable interrupts (%d)\n", ret);
-
-	return ret;
-}
-
-static void ccp_free_irqs(struct ccp_device *ccp)
-{
-	struct device *dev = ccp->dev;
-
-	free_irq(ccp->irq, dev);
-}
-
-static struct resource *ccp_find_mmio_area(struct ccp_device *ccp)
-{
-	struct device *dev = ccp->dev;
-	struct platform_device *pdev = to_platform_device(dev);
-	struct resource *ior;
-
-	ior = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (ior && (resource_size(ior) >= 0x800))
-		return ior;
-
-	return NULL;
-}
-
-static int ccp_platform_probe(struct platform_device *pdev)
-{
-	struct ccp_device *ccp;
-	struct ccp_platform *ccp_platform;
-	struct device *dev = &pdev->dev;
-	enum dev_dma_attr attr;
-	struct resource *ior;
-	int ret;
-
-	ret = -ENOMEM;
-	ccp = ccp_alloc_struct(dev);
-	if (!ccp)
-		goto e_err;
-
-	ccp_platform = devm_kzalloc(dev, sizeof(*ccp_platform), GFP_KERNEL);
-	if (!ccp_platform)
-		goto e_err;
-
-	ccp->dev_specific = ccp_platform;
-	ccp->vdata = pdev->dev.of_node ? ccp_get_of_version(pdev)
-					 : ccp_get_acpi_version(pdev);
-	if (!ccp->vdata || !ccp->vdata->version) {
-		ret = -ENODEV;
-		dev_err(dev, "missing driver data\n");
-		goto e_err;
-	}
-	ccp->get_irq = ccp_get_irqs;
-	ccp->free_irq = ccp_free_irqs;
-
-	ior = ccp_find_mmio_area(ccp);
-	ccp->io_map = devm_ioremap_resource(dev, ior);
-	if (IS_ERR(ccp->io_map)) {
-		ret = PTR_ERR(ccp->io_map);
-		goto e_err;
-	}
-	ccp->io_regs = ccp->io_map;
-
-	attr = device_get_dma_attr(dev);
-	if (attr == DEV_DMA_NOT_SUPPORTED) {
-		dev_err(dev, "DMA is not supported");
-		goto e_err;
-	}
-
-	ccp_platform->coherent = (attr == DEV_DMA_COHERENT);
-	if (ccp_platform->coherent)
-		ccp->axcache = CACHE_WB_NO_ALLOC;
-	else
-		ccp->axcache = CACHE_NONE;
-
-	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
-	if (ret) {
-		dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", ret);
-		goto e_err;
-	}
-
-	dev_set_drvdata(dev, ccp);
-
-	ret = ccp->vdata->perform->init(ccp);
-	if (ret)
-		goto e_err;
-
-	dev_notice(dev, "enabled\n");
-
-	return 0;
-
-e_err:
-	dev_notice(dev, "initialization failed\n");
-	return ret;
-}
-
-static int ccp_platform_remove(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct ccp_device *ccp = dev_get_drvdata(dev);
-
-	ccp->vdata->perform->destroy(ccp);
-
-	dev_notice(dev, "disabled\n");
-
-	return 0;
-}
-
-#ifdef CONFIG_PM
-static int ccp_platform_suspend(struct platform_device *pdev,
-				pm_message_t state)
-{
-	struct device *dev = &pdev->dev;
-	struct ccp_device *ccp = dev_get_drvdata(dev);
-	unsigned long flags;
-	unsigned int i;
-
-	spin_lock_irqsave(&ccp->cmd_lock, flags);
-
-	ccp->suspending = 1;
-
-	/* Wake all the queue kthreads to prepare for suspend */
-	for (i = 0; i < ccp->cmd_q_count; i++)
-		wake_up_process(ccp->cmd_q[i].kthread);
-
-	spin_unlock_irqrestore(&ccp->cmd_lock, flags);
-
-	/* Wait for all queue kthreads to say they're done */
-	while (!ccp_queues_suspended(ccp))
-		wait_event_interruptible(ccp->suspend_queue,
-					 ccp_queues_suspended(ccp));
-
-	return 0;
-}
-
-static int ccp_platform_resume(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct ccp_device *ccp = dev_get_drvdata(dev);
-	unsigned long flags;
-	unsigned int i;
-
-	spin_lock_irqsave(&ccp->cmd_lock, flags);
-
-	ccp->suspending = 0;
-
-	/* Wake up all the kthreads */
-	for (i = 0; i < ccp->cmd_q_count; i++) {
-		ccp->cmd_q[i].suspended = 0;
-		wake_up_process(ccp->cmd_q[i].kthread);
-	}
-
-	spin_unlock_irqrestore(&ccp->cmd_lock, flags);
-
-	return 0;
-}
-#endif
-
-#ifdef CONFIG_ACPI
-static const struct acpi_device_id ccp_acpi_match[] = {
-	{ "AMDI0C00", (kernel_ulong_t)&ccpv3 },
-	{ },
-};
-MODULE_DEVICE_TABLE(acpi, ccp_acpi_match);
-#endif
-
-#ifdef CONFIG_OF
-static const struct of_device_id ccp_of_match[] = {
-	{ .compatible = "amd,ccp-seattle-v1a",
-	  .data = (const void *)&ccpv3 },
-	{ },
-};
-MODULE_DEVICE_TABLE(of, ccp_of_match);
-#endif
-
-static struct platform_driver ccp_platform_driver = {
-	.driver = {
-		.name = "ccp",
-#ifdef CONFIG_ACPI
-		.acpi_match_table = ccp_acpi_match,
-#endif
-#ifdef CONFIG_OF
-		.of_match_table = ccp_of_match,
-#endif
-	},
-	.probe = ccp_platform_probe,
-	.remove = ccp_platform_remove,
-#ifdef CONFIG_PM
-	.suspend = ccp_platform_suspend,
-	.resume = ccp_platform_resume,
-#endif
-};
-
-int ccp_platform_init(void)
-{
-	return platform_driver_register(&ccp_platform_driver);
-}
-
-void ccp_platform_exit(void)
-{
-	platform_driver_unregister(&ccp_platform_driver);
-}
diff --git a/drivers/crypto/ccp/sp-dev.c b/drivers/crypto/ccp/sp-dev.c
new file mode 100644
index 000000000000..bef387c8abfd
--- /dev/null
+++ b/drivers/crypto/ccp/sp-dev.c
@@ -0,0 +1,277 @@
+/*
+ * AMD Secure Processor driver
+ *
+ * Copyright (C) 2017 Advanced Micro Devices, Inc.
+ *
+ * Author: Tom Lendacky <thomas.lendacky@amd.com>
+ * Author: Gary R Hook <gary.hook@amd.com>
+ * Author: Brijesh Singh <brijesh.singh@amd.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/kthread.h>
+#include <linux/sched.h>
+#include <linux/interrupt.h>
+#include <linux/spinlock.h>
+#include <linux/spinlock_types.h>
+#include <linux/types.h>
+#include <linux/ccp.h>
+
+#include "ccp-dev.h"
+#include "sp-dev.h"
+
+MODULE_AUTHOR("Tom Lendacky <thomas.lendacky@amd.com>");
+MODULE_AUTHOR("Gary R Hook <gary.hook@amd.com>");
+MODULE_LICENSE("GPL");
+MODULE_VERSION("1.1.0");
+MODULE_DESCRIPTION("AMD Secure Processor driver");
+
+/* List of SPs, SP count, read-write access lock, and access functions
+ *
+ * Lock structure: get sp_unit_lock for reading whenever we need to
+ * examine the SP list.
+ */
+static DEFINE_RWLOCK(sp_unit_lock);
+static LIST_HEAD(sp_units);
+
+/* Ever-increasing value to produce unique unit numbers */
+static atomic_t sp_ordinal;
+
+static void sp_add_device(struct sp_device *sp)
+{
+	unsigned long flags;
+
+	write_lock_irqsave(&sp_unit_lock, flags);
+
+	list_add_tail(&sp->entry, &sp_units);
+
+	write_unlock_irqrestore(&sp_unit_lock, flags);
+}
+
+static void sp_del_device(struct sp_device *sp)
+{
+	unsigned long flags;
+
+	write_lock_irqsave(&sp_unit_lock, flags);
+
+	list_del(&sp->entry);
+
+	write_unlock_irqrestore(&sp_unit_lock, flags);
+}
+
+static irqreturn_t sp_irq_handler(int irq, void *data)
+{
+	struct sp_device *sp = data;
+
+	if (sp->ccp_irq_handler)
+		sp->ccp_irq_handler(irq, sp->ccp_irq_data);
+
+	if (sp->psp_irq_handler)
+		sp->psp_irq_handler(irq, sp->psp_irq_data);
+
+	return IRQ_HANDLED;
+}
+
+int sp_request_ccp_irq(struct sp_device *sp, irq_handler_t handler,
+		       const char *name, void *data)
+{
+	int ret;
+
+	if ((sp->psp_irq == sp->ccp_irq) && sp->dev_vdata->psp_vdata) {
+		/* Need a common routine to manage all interrupts */
+		sp->ccp_irq_data = data;
+		sp->ccp_irq_handler = handler;
+
+		if (!sp->irq_registered) {
+			ret = request_irq(sp->ccp_irq, sp_irq_handler, 0,
+					  sp->name, sp);
+			if (ret)
+				return ret;
+
+			sp->irq_registered = true;
+		}
+	} else {
+		/* Each sub-device can manage it's own interrupt */
+		ret = request_irq(sp->ccp_irq, handler, 0, name, data);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+int sp_request_psp_irq(struct sp_device *sp, irq_handler_t handler,
+		       const char *name, void *data)
+{
+	int ret;
+
+	if ((sp->psp_irq == sp->ccp_irq) && sp->dev_vdata->ccp_vdata) {
+		/* Need a common routine to manage all interrupts */
+		sp->psp_irq_data = data;
+		sp->psp_irq_handler = handler;
+
+		if (!sp->irq_registered) {
+			ret = request_irq(sp->psp_irq, sp_irq_handler, 0,
+					  sp->name, sp);
+			if (ret)
+				return ret;
+
+			sp->irq_registered = true;
+		}
+	} else {
+		/* Each sub-device can manage it's own interrupt */
+		ret = request_irq(sp->psp_irq, handler, 0, name, data);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+void sp_free_ccp_irq(struct sp_device *sp, void *data)
+{
+	if ((sp->psp_irq == sp->ccp_irq) && sp->dev_vdata->psp_vdata) {
+		/* Using common routine to manage all interrupts */
+		if (!sp->psp_irq_handler) {
+			/* Nothing else using it, so free it */
+			free_irq(sp->ccp_irq, sp);
+
+			sp->irq_registered = false;
+		}
+
+		sp->ccp_irq_handler = NULL;
+		sp->ccp_irq_data = NULL;
+	} else {
+		/* Each sub-device can manage it's own interrupt */
+		free_irq(sp->ccp_irq, data);
+	}
+}
+
+void sp_free_psp_irq(struct sp_device *sp, void *data)
+{
+	if ((sp->psp_irq == sp->ccp_irq) && sp->dev_vdata->ccp_vdata) {
+		/* Using common routine to manage all interrupts */
+		if (!sp->ccp_irq_handler) {
+			/* Nothing else using it, so free it */
+			free_irq(sp->psp_irq, sp);
+
+			sp->irq_registered = false;
+		}
+
+		sp->psp_irq_handler = NULL;
+		sp->psp_irq_data = NULL;
+	} else {
+		/* Each sub-device can manage it's own interrupt */
+		free_irq(sp->psp_irq, data);
+	}
+}
+
+/**
+ * sp_alloc_struct - allocate and initialize the sp_device struct
+ *
+ * @dev: device struct of the SP
+ */
+struct sp_device *sp_alloc_struct(struct device *dev)
+{
+	struct sp_device *sp;
+
+	sp = devm_kzalloc(dev, sizeof(*sp), GFP_KERNEL);
+	if (!sp)
+		return NULL;
+
+	sp->dev = dev;
+	sp->ord = atomic_inc_return(&sp_ordinal);
+	snprintf(sp->name, SP_MAX_NAME_LEN, "sp-%u", sp->ord);
+
+	return sp;
+}
+
+int sp_init(struct sp_device *sp)
+{
+	sp_add_device(sp);
+
+	if (sp->dev_vdata->ccp_vdata)
+		ccp_dev_init(sp);
+
+	return 0;
+}
+
+void sp_destroy(struct sp_device *sp)
+{
+	if (sp->dev_vdata->ccp_vdata)
+		ccp_dev_destroy(sp);
+
+	sp_del_device(sp);
+}
+
+#ifdef CONFIG_PM
+int sp_suspend(struct sp_device *sp, pm_message_t state)
+{
+	int ret;
+
+	if (sp->dev_vdata->ccp_vdata) {
+		ret = ccp_dev_suspend(sp, state);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+int sp_resume(struct sp_device *sp)
+{
+	int ret;
+
+	if (sp->dev_vdata->ccp_vdata) {
+		ret = ccp_dev_resume(sp);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+#endif
+
+static int __init sp_mod_init(void)
+{
+#ifdef CONFIG_X86
+	int ret;
+
+	ret = sp_pci_init();
+	if (ret)
+		return ret;
+
+	return 0;
+#endif
+
+#ifdef CONFIG_ARM64
+	int ret;
+
+	ret = sp_platform_init();
+	if (ret)
+		return ret;
+
+	return 0;
+#endif
+
+	return -ENODEV;
+}
+
+static void __exit sp_mod_exit(void)
+{
+#ifdef CONFIG_X86
+	sp_pci_exit();
+#endif
+
+#ifdef CONFIG_ARM64
+	sp_platform_exit();
+#endif
+}
+
+module_init(sp_mod_init);
+module_exit(sp_mod_exit);
diff --git a/drivers/crypto/ccp/sp-dev.h b/drivers/crypto/ccp/sp-dev.h
new file mode 100644
index 000000000000..5ab486ade1ad
--- /dev/null
+++ b/drivers/crypto/ccp/sp-dev.h
@@ -0,0 +1,133 @@
+/*
+ * AMD Secure Processor driver
+ *
+ * Copyright (C) 2017 Advanced Micro Devices, Inc.
+ *
+ * Author: Tom Lendacky <thomas.lendacky@amd.com>
+ * Author: Gary R Hook <gary.hook@amd.com>
+ * Author: Brijesh Singh <brijesh.singh@amd.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef __SP_DEV_H__
+#define __SP_DEV_H__
+
+#include <linux/device.h>
+#include <linux/pci.h>
+#include <linux/spinlock.h>
+#include <linux/mutex.h>
+#include <linux/list.h>
+#include <linux/wait.h>
+#include <linux/dmapool.h>
+#include <linux/hw_random.h>
+#include <linux/bitops.h>
+#include <linux/interrupt.h>
+#include <linux/irqreturn.h>
+
+#define SP_MAX_NAME_LEN		32
+
+#define CACHE_NONE			0x00
+#define CACHE_WB_NO_ALLOC		0xb7
+
+/* Structure to hold CCP device data */
+struct ccp_device;
+struct ccp_vdata {
+	const unsigned int version;
+	const unsigned int dma_chan_attr;
+	void (*setup)(struct ccp_device *);
+	const struct ccp_actions *perform;
+	const unsigned int offset;
+	const unsigned int rsamax;
+};
+/* Structure to hold SP device data */
+struct sp_dev_vdata {
+	const unsigned int bar;
+
+	const struct ccp_vdata *ccp_vdata;
+	void *psp_vdata;
+};
+
+struct sp_device {
+	struct list_head entry;
+
+	struct device *dev;
+
+	struct sp_dev_vdata *dev_vdata;
+	unsigned int ord;
+	char name[SP_MAX_NAME_LEN];
+
+	/* Bus specific device information */
+	void *dev_specific;
+
+	/* I/O area used for device communication. */
+	void __iomem *io_map;
+
+	/* DMA caching attribute support */
+	unsigned int axcache;
+
+	bool irq_registered;
+	bool use_tasklet;
+
+	unsigned int ccp_irq;
+	irq_handler_t ccp_irq_handler;
+	void *ccp_irq_data;
+
+	unsigned int psp_irq;
+	irq_handler_t psp_irq_handler;
+	void *psp_irq_data;
+
+	void *ccp_data;
+	void *psp_data;
+};
+
+int sp_pci_init(void);
+void sp_pci_exit(void);
+
+int sp_platform_init(void);
+void sp_platform_exit(void);
+
+struct sp_device *sp_alloc_struct(struct device *dev);
+
+int sp_init(struct sp_device *sp);
+void sp_destroy(struct sp_device *sp);
+struct sp_device *sp_get_master(void);
+
+int sp_suspend(struct sp_device *sp, pm_message_t state);
+int sp_resume(struct sp_device *sp);
+int sp_request_ccp_irq(struct sp_device *sp, irq_handler_t handler,
+		       const char *name, void *data);
+void sp_free_ccp_irq(struct sp_device *sp, void *data);
+int sp_request_psp_irq(struct sp_device *sp, irq_handler_t handler,
+		       const char *name, void *data);
+void sp_free_psp_irq(struct sp_device *sp, void *data);
+
+#ifdef CONFIG_CRYPTO_DEV_SP_CCP
+
+int ccp_dev_init(struct sp_device *sp);
+void ccp_dev_destroy(struct sp_device *sp);
+
+int ccp_dev_suspend(struct sp_device *sp, pm_message_t state);
+int ccp_dev_resume(struct sp_device *sp);
+
+#else	/* !CONFIG_CRYPTO_DEV_SP_CCP */
+
+static inline int ccp_dev_init(struct sp_device *sp)
+{
+	return 0;
+}
+static inline void ccp_dev_destroy(struct sp_device *sp) { }
+
+static inline int ccp_dev_suspend(struct sp_device *sp, pm_message_t state)
+{
+	return 0;
+}
+static inline int ccp_dev_resume(struct sp_device *sp)
+{
+	return 0;
+}
+#endif	/* CONFIG_CRYPTO_DEV_SP_CCP */
+
+#endif
diff --git a/drivers/crypto/ccp/sp-pci.c b/drivers/crypto/ccp/sp-pci.c
new file mode 100644
index 000000000000..9859aa683a28
--- /dev/null
+++ b/drivers/crypto/ccp/sp-pci.c
@@ -0,0 +1,276 @@
+/*
+ * AMD Secure Processor device driver
+ *
+ * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
+ *
+ * Author: Tom Lendacky <thomas.lendacky@amd.com>
+ * Author: Gary R Hook <gary.hook@amd.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/device.h>
+#include <linux/pci.h>
+#include <linux/pci_ids.h>
+#include <linux/dma-mapping.h>
+#include <linux/kthread.h>
+#include <linux/sched.h>
+#include <linux/interrupt.h>
+#include <linux/spinlock.h>
+#include <linux/delay.h>
+#include <linux/ccp.h>
+
+#include "ccp-dev.h"
+
+#define MSIX_VECTORS			2
+
+struct sp_pci {
+	int msix_count;
+	struct msix_entry msix_entry[MSIX_VECTORS];
+};
+
+static int sp_get_msix_irqs(struct sp_device *sp)
+{
+	struct sp_pci *sp_pci = sp->dev_specific;
+	struct device *dev = sp->dev;
+	struct pci_dev *pdev = to_pci_dev(dev);
+	int v, ret;
+
+	for (v = 0; v < ARRAY_SIZE(sp_pci->msix_entry); v++)
+		sp_pci->msix_entry[v].entry = v;
+
+	ret = pci_enable_msix_range(pdev, sp_pci->msix_entry, 1, v);
+	if (ret < 0)
+		return ret;
+
+	sp_pci->msix_count = ret;
+	sp->use_tasklet = true;
+
+	sp->psp_irq = sp_pci->msix_entry[0].vector;
+	sp->ccp_irq = (sp_pci->msix_count > 1) ? sp_pci->msix_entry[1].vector
+					       : sp_pci->msix_entry[0].vector;
+	return 0;
+}
+
+static int sp_get_msi_irq(struct sp_device *sp)
+{
+	struct device *dev = sp->dev;
+	struct pci_dev *pdev = to_pci_dev(dev);
+	int ret;
+
+	ret = pci_enable_msi(pdev);
+	if (ret)
+		return ret;
+
+	sp->ccp_irq = pdev->irq;
+	sp->psp_irq = pdev->irq;
+
+	return 0;
+}
+
+static int sp_get_irqs(struct sp_device *sp)
+{
+	struct device *dev = sp->dev;
+	int ret;
+
+	ret = sp_get_msix_irqs(sp);
+	if (!ret)
+		return 0;
+
+	/* Couldn't get MSI-X vectors, try MSI */
+	dev_notice(dev, "could not enable MSI-X (%d), trying MSI\n", ret);
+	ret = sp_get_msi_irq(sp);
+	if (!ret)
+		return 0;
+
+	/* Couldn't get MSI interrupt */
+	dev_notice(dev, "could not enable MSI (%d)\n", ret);
+
+	return ret;
+}
+
+static void sp_free_irqs(struct sp_device *sp)
+{
+	struct sp_pci *sp_pci = sp->dev_specific;
+	struct device *dev = sp->dev;
+	struct pci_dev *pdev = to_pci_dev(dev);
+
+	if (sp_pci->msix_count)
+		pci_disable_msix(pdev);
+	else if (sp->psp_irq)
+		pci_disable_msi(pdev);
+
+	sp->ccp_irq = 0;
+	sp->psp_irq = 0;
+}
+
+static int sp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+	struct sp_device *sp;
+	struct sp_pci *sp_pci;
+	struct device *dev = &pdev->dev;
+	void __iomem * const *iomap_table;
+	int bar_mask;
+	int ret;
+
+	ret = -ENOMEM;
+	sp = sp_alloc_struct(dev);
+	if (!sp)
+		goto e_err;
+
+	sp_pci = devm_kzalloc(dev, sizeof(*sp_pci), GFP_KERNEL);
+	if (!sp_pci)
+		goto e_err;
+
+	sp->dev_specific = sp_pci;
+	sp->dev_vdata = (struct sp_dev_vdata *)id->driver_data;
+	if (!sp->dev_vdata) {
+		ret = -ENODEV;
+		dev_err(dev, "missing driver data\n");
+		goto e_err;
+	}
+
+	ret = pcim_enable_device(pdev);
+	if (ret) {
+		dev_err(dev, "pcim_enable_device failed (%d)\n", ret);
+		goto e_err;
+	}
+
+	bar_mask = pci_select_bars(pdev, IORESOURCE_MEM);
+	ret = pcim_iomap_regions(pdev, bar_mask, "ccp");
+	if (ret) {
+		dev_err(dev, "pcim_iomap_regions failed (%d)\n", ret);
+		goto e_err;
+	}
+
+	iomap_table = pcim_iomap_table(pdev);
+	if (!iomap_table) {
+		dev_err(dev, "pcim_iomap_table failed\n");
+		ret = -ENOMEM;
+		goto e_err;
+	}
+
+	sp->io_map = iomap_table[sp->dev_vdata->bar];
+	if (!sp->io_map) {
+		dev_err(dev, "ioremap failed\n");
+		ret = -ENOMEM;
+		goto e_err;
+	}
+
+	ret = sp_get_irqs(sp);
+	if (ret)
+		goto e_err;
+
+	pci_set_master(pdev);
+
+	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
+	if (ret) {
+		ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
+		if (ret) {
+			dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n",
+				ret);
+			goto e_err;
+		}
+	}
+
+	dev_set_drvdata(dev, sp);
+
+	ret = sp_init(sp);
+	if (ret)
+		goto e_err;
+
+	dev_notice(dev, "enabled\n");
+
+	return 0;
+
+e_err:
+	dev_notice(dev, "initialization failed\n");
+	return ret;
+}
+
+static void sp_pci_remove(struct pci_dev *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct sp_device *sp = dev_get_drvdata(dev);
+
+	if (!sp)
+		return;
+
+	sp_destroy(sp);
+
+	sp_free_irqs(sp);
+
+	dev_notice(dev, "disabled\n");
+}
+
+#ifdef CONFIG_PM
+static int sp_pci_suspend(struct pci_dev *pdev, pm_message_t state)
+{
+	struct device *dev = &pdev->dev;
+	struct sp_device *sp = dev_get_drvdata(dev);
+
+	return sp_suspend(sp, state);
+}
+
+static int sp_pci_resume(struct pci_dev *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct sp_device *sp = dev_get_drvdata(dev);
+
+	return sp_resume(sp);
+}
+#endif
+
+static const struct sp_dev_vdata dev_vdata[] = {
+	{
+		.bar = 2,
+#ifdef CONFIG_CRYPTO_DEV_SP_CCP
+		.ccp_vdata = &ccpv3,
+#endif
+	},
+	{
+		.bar = 2,
+#ifdef CONFIG_CRYPTO_DEV_SP_CCP
+		.ccp_vdata = &ccpv5a,
+#endif
+	},
+	{
+		.bar = 2,
+#ifdef CONFIG_CRYPTO_DEV_SP_CCP
+		.ccp_vdata = &ccpv5b,
+#endif
+	},
+};
+static const struct pci_device_id sp_pci_table[] = {
+	{ PCI_VDEVICE(AMD, 0x1537), (kernel_ulong_t)&dev_vdata[0] },
+	{ PCI_VDEVICE(AMD, 0x1456), (kernel_ulong_t)&dev_vdata[1] },
+	{ PCI_VDEVICE(AMD, 0x1468), (kernel_ulong_t)&dev_vdata[2] },
+	/* Last entry must be zero */
+	{ 0, }
+};
+MODULE_DEVICE_TABLE(pci, sp_pci_table);
+
+static struct pci_driver sp_pci_driver = {
+	.name = "ccp",
+	.id_table = sp_pci_table,
+	.probe = sp_pci_probe,
+	.remove = sp_pci_remove,
+#ifdef CONFIG_PM
+	.suspend = sp_pci_suspend,
+	.resume = sp_pci_resume,
+#endif
+};
+
+int sp_pci_init(void)
+{
+	return pci_register_driver(&sp_pci_driver);
+}
+
+void sp_pci_exit(void)
+{
+	pci_unregister_driver(&sp_pci_driver);
+}
diff --git a/drivers/crypto/ccp/sp-platform.c b/drivers/crypto/ccp/sp-platform.c
new file mode 100644
index 000000000000..71734f254fd1
--- /dev/null
+++ b/drivers/crypto/ccp/sp-platform.c
@@ -0,0 +1,256 @@
+/*
+ * AMD Secure Processor device driver
+ *
+ * Copyright (C) 2014,2016 Advanced Micro Devices, Inc.
+ *
+ * Author: Tom Lendacky <thomas.lendacky@amd.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/device.h>
+#include <linux/platform_device.h>
+#include <linux/ioport.h>
+#include <linux/dma-mapping.h>
+#include <linux/kthread.h>
+#include <linux/sched.h>
+#include <linux/interrupt.h>
+#include <linux/spinlock.h>
+#include <linux/delay.h>
+#include <linux/ccp.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/acpi.h>
+
+#include "ccp-dev.h"
+
+struct sp_platform {
+	int coherent;
+	unsigned int irq_count;
+};
+
+static const struct acpi_device_id sp_acpi_match[];
+static const struct of_device_id sp_of_match[];
+
+static struct sp_dev_vdata *sp_get_of_version(struct platform_device *pdev)
+{
+#ifdef CONFIG_OF
+	const struct of_device_id *match;
+
+	match = of_match_node(sp_of_match, pdev->dev.of_node);
+	if (match && match->data)
+		return (struct sp_dev_vdata *)match->data;
+#endif
+	return NULL;
+}
+
+static struct sp_dev_vdata *sp_get_acpi_version(struct platform_device *pdev)
+{
+#ifdef CONFIG_ACPI
+	const struct acpi_device_id *match;
+
+	match = acpi_match_device(sp_acpi_match, &pdev->dev);
+	if (match && match->driver_data)
+		return (struct sp_dev_vdata *)match->driver_data;
+#endif
+	return NULL;
+}
+
+static int sp_get_irqs(struct sp_device *sp)
+{
+	struct sp_platform *sp_platform = sp->dev_specific;
+	struct device *dev = sp->dev;
+	struct platform_device *pdev = to_platform_device(dev);
+	unsigned int i, count;
+	int ret;
+
+	for (i = 0, count = 0; i < pdev->num_resources; i++) {
+		struct resource *res = &pdev->resource[i];
+
+		if (resource_type(res) == IORESOURCE_IRQ)
+			count++;
+	}
+
+	sp_platform->irq_count = count;
+
+	ret = platform_get_irq(pdev, 0);
+	if (ret < 0) {
+		dev_notice(dev, "unable to get IRQ (%d)\n", ret);
+		return ret;
+	}
+
+	sp->psp_irq = ret;
+	if (count == 1) {
+		sp->ccp_irq = ret;
+	} else {
+		ret = platform_get_irq(pdev, 1);
+		if (ret < 0) {
+			dev_notice(dev, "unable to get IRQ (%d)\n", ret);
+			return ret;
+		}
+
+		sp->ccp_irq = ret;
+	}
+
+	return 0;
+}
+
+static int sp_platform_probe(struct platform_device *pdev)
+{
+	struct sp_device *sp;
+	struct sp_platform *sp_platform;
+	struct device *dev = &pdev->dev;
+	enum dev_dma_attr attr;
+	struct resource *ior;
+	int ret;
+
+	ret = -ENOMEM;
+	sp = sp_alloc_struct(dev);
+	if (!sp)
+		goto e_err;
+
+	sp_platform = devm_kzalloc(dev, sizeof(*sp_platform), GFP_KERNEL);
+	if (!sp_platform)
+		goto e_err;
+
+	sp->dev_specific = sp_platform;
+	sp->dev_vdata = pdev->dev.of_node ? sp_get_of_version(pdev)
+					 : sp_get_acpi_version(pdev);
+	if (!sp->dev_vdata) {
+		ret = -ENODEV;
+		dev_err(dev, "missing driver data\n");
+		goto e_err;
+	}
+
+	ior = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	sp->io_map = devm_ioremap_resource(dev, ior);
+	if (IS_ERR(sp->io_map)) {
+		ret = PTR_ERR(sp->io_map);
+		goto e_err;
+	}
+
+	attr = device_get_dma_attr(dev);
+	if (attr == DEV_DMA_NOT_SUPPORTED) {
+		dev_err(dev, "DMA is not supported");
+		goto e_err;
+	}
+
+	sp_platform->coherent = (attr == DEV_DMA_COHERENT);
+	if (sp_platform->coherent)
+		sp->axcache = CACHE_WB_NO_ALLOC;
+	else
+		sp->axcache = CACHE_NONE;
+
+	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
+	if (ret) {
+		dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", ret);
+		goto e_err;
+	}
+
+	ret = sp_get_irqs(sp);
+	if (ret)
+		goto e_err;
+
+	dev_set_drvdata(dev, sp);
+
+	ret = sp_init(sp);
+	if (ret)
+		goto e_err;
+
+	dev_notice(dev, "enabled\n");
+
+	return 0;
+
+e_err:
+	dev_notice(dev, "initialization failed\n");
+	return ret;
+}
+
+static int sp_platform_remove(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct sp_device *sp = dev_get_drvdata(dev);
+
+	sp_destroy(sp);
+
+	dev_notice(dev, "disabled\n");
+
+	return 0;
+}
+
+#ifdef CONFIG_PM
+static int sp_platform_suspend(struct platform_device *pdev,
+				pm_message_t state)
+{
+	struct device *dev = &pdev->dev;
+	struct sp_device *sp = dev_get_drvdata(dev);
+
+	return sp_suspend(sp, state);
+}
+
+static int sp_platform_resume(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct sp_device *sp = dev_get_drvdata(dev);
+
+	return sp_resume(sp);
+}
+#endif
+
+static const struct sp_dev_vdata dev_vdata[] = {
+	{
+		.bar = 0,
+#ifdef CONFIG_CRYPTO_DEV_SP_CCP
+		.ccp_vdata = &ccpv3_platform,
+#endif
+	},
+};
+
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id sp_acpi_match[] = {
+	{ "AMDI0C00", (kernel_ulong_t)&dev_vdata[0] },
+	{ },
+};
+MODULE_DEVICE_TABLE(acpi, sp_acpi_match);
+#endif
+
+#ifdef CONFIG_OF
+static const struct of_device_id sp_of_match[] = {
+	{ .compatible = "amd,ccp-seattle-v1a",
+	  .data = (const void *)&dev_vdata[0] },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, sp_of_match);
+#endif
+
+static struct platform_driver sp_platform_driver = {
+	.driver = {
+		.name = "ccp",
+#ifdef CONFIG_ACPI
+		.acpi_match_table = sp_acpi_match,
+#endif
+#ifdef CONFIG_OF
+		.of_match_table = sp_of_match,
+#endif
+	},
+	.probe = sp_platform_probe,
+	.remove = sp_platform_remove,
+#ifdef CONFIG_PM
+	.suspend = sp_platform_suspend,
+	.resume = sp_platform_resume,
+#endif
+};
+
+int sp_platform_init(void)
+{
+	return platform_driver_register(&sp_platform_driver);
+}
+
+void sp_platform_exit(void)
+{
+	platform_driver_unregister(&sp_platform_driver);
+}
diff --git a/drivers/crypto/geode-aes.c b/drivers/crypto/geode-aes.c
index fe538e5287a5..eb2a0a73cbed 100644
--- a/drivers/crypto/geode-aes.c
+++ b/drivers/crypto/geode-aes.c
@@ -1,10 +1,10 @@
  /* Copyright (C) 2004-2006, Advanced Micro Devices, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+  *
+  * This program is free software; you can redistribute it and/or modify
+  * it under the terms of the GNU General Public License as published by
+  * the Free Software Foundation; either version 2 of the License, or
+  * (at your option) any later version.
+  */
 
 #include <linux/module.h>
 #include <linux/kernel.h>
@@ -30,6 +30,7 @@ static inline void
 _writefield(u32 offset, void *value)
 {
 	int i;
+
 	for (i = 0; i < 4; i++)
 		iowrite32(((u32 *) value)[i], _iobase + offset + (i * 4));
 }
@@ -39,6 +40,7 @@ static inline void
 _readfield(u32 offset, void *value)
 {
 	int i;
+
 	for (i = 0; i < 4; i++)
 		((u32 *) value)[i] = ioread32(_iobase + offset + (i * 4));
 }
@@ -515,6 +517,7 @@ static void geode_aes_remove(struct pci_dev *dev)
 static int geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id)
 {
 	int ret;
+
 	ret = pci_enable_device(dev);
 	if (ret)
 		return ret;
@@ -570,7 +573,7 @@ static int geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id)
 }
 
 static struct pci_device_id geode_aes_tbl[] = {
-	{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_LX_AES), } ,
+	{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_LX_AES), },
 	{ 0, }
 };
 
diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c
index 0c6a917a9ab8..b87000a0a01c 100644
--- a/drivers/crypto/img-hash.c
+++ b/drivers/crypto/img-hash.c
@@ -1054,7 +1054,7 @@ res_err:
 
 static int img_hash_remove(struct platform_device *pdev)
 {
-	static struct img_hash_dev *hdev;
+	struct img_hash_dev *hdev;
 
 	hdev = platform_get_drvdata(pdev);
 	spin_lock(&img_hash.lock);
diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c
index 1fabd4aee81b..89ba9e85c0f3 100644
--- a/drivers/crypto/inside-secure/safexcel.c
+++ b/drivers/crypto/inside-secure/safexcel.c
@@ -839,9 +839,10 @@ static int safexcel_probe(struct platform_device *pdev)
 		snprintf(irq_name, 6, "ring%d", i);
 		irq = safexcel_request_ring_irq(pdev, irq_name, safexcel_irq_ring,
 						ring_irq);
-
-		if (irq < 0)
+		if (irq < 0) {
+			ret = irq;
 			goto err_clk;
+		}
 
 		priv->ring[i].work_data.priv = priv;
 		priv->ring[i].work_data.ring = i;
diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c
index 427cbe012729..dadc4a808df5 100644
--- a/drivers/crypto/ixp4xx_crypto.c
+++ b/drivers/crypto/ixp4xx_crypto.c
@@ -1073,7 +1073,7 @@ static int aead_perform(struct aead_request *req, int encrypt,
 		req_ctx->hmac_virt = dma_pool_alloc(buffer_pool, flags,
 				&crypt->icv_rev_aes);
 		if (unlikely(!req_ctx->hmac_virt))
-			goto free_buf_src;
+			goto free_buf_dst;
 		if (!encrypt) {
 			scatterwalk_map_and_copy(req_ctx->hmac_virt,
 				req->src, cryptlen, authsize, 0);
@@ -1088,10 +1088,10 @@ static int aead_perform(struct aead_request *req, int encrypt,
 	BUG_ON(qmgr_stat_overflow(SEND_QID));
 	return -EINPROGRESS;
 
-free_buf_src:
-	free_buf_chain(dev, req_ctx->src, crypt->src_buf);
 free_buf_dst:
 	free_buf_chain(dev, req_ctx->dst, crypt->dst_buf);
+free_buf_src:
+	free_buf_chain(dev, req_ctx->src, crypt->src_buf);
 	crypt->ctl_flags = CTL_FLAG_UNUSED;
 	return -ENOMEM;
 }
diff --git a/drivers/crypto/mediatek/mtk-platform.c b/drivers/crypto/mediatek/mtk-platform.c
index 000b6500a22d..b182e941b0cd 100644
--- a/drivers/crypto/mediatek/mtk-platform.c
+++ b/drivers/crypto/mediatek/mtk-platform.c
@@ -500,7 +500,7 @@ static int mtk_crypto_probe(struct platform_device *pdev)
 		cryp->irq[i] = platform_get_irq(pdev, i);
 		if (cryp->irq[i] < 0) {
 			dev_err(cryp->dev, "no IRQ:%d resource info\n", i);
-			return -ENXIO;
+			return cryp->irq[i];
 		}
 	}
 
diff --git a/drivers/crypto/mxc-scc.c b/drivers/crypto/mxc-scc.c
index ee4be1b0d30b..e01c46387df8 100644
--- a/drivers/crypto/mxc-scc.c
+++ b/drivers/crypto/mxc-scc.c
@@ -708,8 +708,8 @@ static int mxc_scc_probe(struct platform_device *pdev)
 	for (i = 0; i < 2; i++) {
 		irq = platform_get_irq(pdev, i);
 		if (irq < 0) {
-			dev_err(dev, "failed to get irq resource\n");
-			ret = -EINVAL;
+			dev_err(dev, "failed to get irq resource: %d\n", irq);
+			ret = irq;
 			goto err_out;
 		}
 
diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c
index 625ee50fd78b..764be3e6933c 100644
--- a/drivers/crypto/mxs-dcp.c
+++ b/drivers/crypto/mxs-dcp.c
@@ -908,12 +908,16 @@ static int mxs_dcp_probe(struct platform_device *pdev)
 
 	iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	dcp_vmi_irq = platform_get_irq(pdev, 0);
-	if (dcp_vmi_irq < 0)
+	if (dcp_vmi_irq < 0) {
+		dev_err(dev, "Failed to get IRQ: (%d)!\n", dcp_vmi_irq);
 		return dcp_vmi_irq;
+	}
 
 	dcp_irq = platform_get_irq(pdev, 1);
-	if (dcp_irq < 0)
+	if (dcp_irq < 0) {
+		dev_err(dev, "Failed to get IRQ: (%d)!\n", dcp_irq);
 		return dcp_irq;
+	}
 
 	sdcp = devm_kzalloc(dev, sizeof(*sdcp), GFP_KERNEL);
 	if (!sdcp)
diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c
index 269451375b63..a9fd8b9e86cd 100644
--- a/drivers/crypto/n2_core.c
+++ b/drivers/crypto/n2_core.c
@@ -1730,8 +1730,8 @@ static int spu_mdesc_walk_arcs(struct mdesc_handle *mdesc,
 			continue;
 		id = mdesc_get_property(mdesc, tgt, "id", NULL);
 		if (table[*id] != NULL) {
-			dev_err(&dev->dev, "%s: SPU cpu slot already set.\n",
-				dev->dev.of_node->full_name);
+			dev_err(&dev->dev, "%pOF: SPU cpu slot already set.\n",
+				dev->dev.of_node);
 			return -EINVAL;
 		}
 		cpumask_set_cpu(*id, &p->sharing);
@@ -1751,8 +1751,8 @@ static int handle_exec_unit(struct spu_mdesc_info *ip, struct list_head *list,
 
 	p = kzalloc(sizeof(struct spu_queue), GFP_KERNEL);
 	if (!p) {
-		dev_err(&dev->dev, "%s: Could not allocate SPU queue.\n",
-			dev->dev.of_node->full_name);
+		dev_err(&dev->dev, "%pOF: Could not allocate SPU queue.\n",
+			dev->dev.of_node);
 		return -ENOMEM;
 	}
 
@@ -1981,41 +1981,39 @@ static void n2_spu_driver_version(void)
 static int n2_crypto_probe(struct platform_device *dev)
 {
 	struct mdesc_handle *mdesc;
-	const char *full_name;
 	struct n2_crypto *np;
 	int err;
 
 	n2_spu_driver_version();
 
-	full_name = dev->dev.of_node->full_name;
-	pr_info("Found N2CP at %s\n", full_name);
+	pr_info("Found N2CP at %pOF\n", dev->dev.of_node);
 
 	np = alloc_n2cp();
 	if (!np) {
-		dev_err(&dev->dev, "%s: Unable to allocate n2cp.\n",
-			full_name);
+		dev_err(&dev->dev, "%pOF: Unable to allocate n2cp.\n",
+			dev->dev.of_node);
 		return -ENOMEM;
 	}
 
 	err = grab_global_resources();
 	if (err) {
-		dev_err(&dev->dev, "%s: Unable to grab "
-			"global resources.\n", full_name);
+		dev_err(&dev->dev, "%pOF: Unable to grab global resources.\n",
+			dev->dev.of_node);
 		goto out_free_n2cp;
 	}
 
 	mdesc = mdesc_grab();
 
 	if (!mdesc) {
-		dev_err(&dev->dev, "%s: Unable to grab MDESC.\n",
-			full_name);
+		dev_err(&dev->dev, "%pOF: Unable to grab MDESC.\n",
+			dev->dev.of_node);
 		err = -ENODEV;
 		goto out_free_global;
 	}
 	err = grab_mdesc_irq_props(mdesc, dev, &np->cwq_info, "n2cp");
 	if (err) {
-		dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n",
-			full_name);
+		dev_err(&dev->dev, "%pOF: Unable to grab IRQ props.\n",
+			dev->dev.of_node);
 		mdesc_release(mdesc);
 		goto out_free_global;
 	}
@@ -2026,15 +2024,15 @@ static int n2_crypto_probe(struct platform_device *dev)
 	mdesc_release(mdesc);
 
 	if (err) {
-		dev_err(&dev->dev, "%s: CWQ MDESC scan failed.\n",
-			full_name);
+		dev_err(&dev->dev, "%pOF: CWQ MDESC scan failed.\n",
+			dev->dev.of_node);
 		goto out_free_global;
 	}
 
 	err = n2_register_algs();
 	if (err) {
-		dev_err(&dev->dev, "%s: Unable to register algorithms.\n",
-			full_name);
+		dev_err(&dev->dev, "%pOF: Unable to register algorithms.\n",
+			dev->dev.of_node);
 		goto out_free_spu_list;
 	}
 
@@ -2092,42 +2090,40 @@ static void free_ncp(struct n2_mau *mp)
 static int n2_mau_probe(struct platform_device *dev)
 {
 	struct mdesc_handle *mdesc;
-	const char *full_name;
 	struct n2_mau *mp;
 	int err;
 
 	n2_spu_driver_version();
 
-	full_name = dev->dev.of_node->full_name;
-	pr_info("Found NCP at %s\n", full_name);
+	pr_info("Found NCP at %pOF\n", dev->dev.of_node);
 
 	mp = alloc_ncp();
 	if (!mp) {
-		dev_err(&dev->dev, "%s: Unable to allocate ncp.\n",
-			full_name);
+		dev_err(&dev->dev, "%pOF: Unable to allocate ncp.\n",
+			dev->dev.of_node);
 		return -ENOMEM;
 	}
 
 	err = grab_global_resources();
 	if (err) {
-		dev_err(&dev->dev, "%s: Unable to grab "
-			"global resources.\n", full_name);
+		dev_err(&dev->dev, "%pOF: Unable to grab global resources.\n",
+			dev->dev.of_node);
 		goto out_free_ncp;
 	}
 
 	mdesc = mdesc_grab();
 
 	if (!mdesc) {
-		dev_err(&dev->dev, "%s: Unable to grab MDESC.\n",
-			full_name);
+		dev_err(&dev->dev, "%pOF: Unable to grab MDESC.\n",
+			dev->dev.of_node);
 		err = -ENODEV;
 		goto out_free_global;
 	}
 
 	err = grab_mdesc_irq_props(mdesc, dev, &mp->mau_info, "ncp");
 	if (err) {
-		dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n",
-			full_name);
+		dev_err(&dev->dev, "%pOF: Unable to grab IRQ props.\n",
+			dev->dev.of_node);
 		mdesc_release(mdesc);
 		goto out_free_global;
 	}
@@ -2138,8 +2134,8 @@ static int n2_mau_probe(struct platform_device *dev)
 	mdesc_release(mdesc);
 
 	if (err) {
-		dev_err(&dev->dev, "%s: MAU MDESC scan failed.\n",
-			full_name);
+		dev_err(&dev->dev, "%pOF: MAU MDESC scan failed.\n",
+			dev->dev.of_node);
 		goto out_free_global;
 	}
 
diff --git a/drivers/crypto/nx/Kconfig b/drivers/crypto/nx/Kconfig
index ad7552a6998c..cd5dda9c48f4 100644
--- a/drivers/crypto/nx/Kconfig
+++ b/drivers/crypto/nx/Kconfig
@@ -38,6 +38,7 @@ config CRYPTO_DEV_NX_COMPRESS_PSERIES
 config CRYPTO_DEV_NX_COMPRESS_POWERNV
 	tristate "Compression acceleration support on PowerNV platform"
 	depends on PPC_POWERNV
+	depends on PPC_VAS
 	default y
 	help
 	  Support for PowerPC Nest (NX) compression acceleration. This
diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c
index 1710f80a09ec..874ddf5e9087 100644
--- a/drivers/crypto/nx/nx-842-powernv.c
+++ b/drivers/crypto/nx/nx-842-powernv.c
@@ -22,6 +22,8 @@
 
 #include <asm/prom.h>
 #include <asm/icswx.h>
+#include <asm/vas.h>
+#include <asm/reg.h>
 
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");
@@ -31,6 +33,9 @@ MODULE_ALIAS_CRYPTO("842-nx");
 
 #define WORKMEM_ALIGN	(CRB_ALIGN)
 #define CSB_WAIT_MAX	(5000) /* ms */
+#define VAS_RETRIES	(10)
+/* # of requests allowed per RxFIFO at a time. 0 for unlimited */
+#define MAX_CREDITS_PER_RXFIFO	(1024)
 
 struct nx842_workmem {
 	/* Below fields must be properly aligned */
@@ -41,19 +46,34 @@ struct nx842_workmem {
 
 	ktime_t start;
 
+	struct vas_window *txwin;	/* Used with VAS function */
 	char padding[WORKMEM_ALIGN]; /* unused, to allow alignment */
 } __packed __aligned(WORKMEM_ALIGN);
 
 struct nx842_coproc {
 	unsigned int chip_id;
 	unsigned int ct;
-	unsigned int ci;
+	unsigned int ci;	/* Coprocessor instance, used with icswx */
+	struct {
+		struct vas_window *rxwin;
+		int id;
+	} vas;
 	struct list_head list;
 };
 
+/*
+ * Send the request to NX engine on the chip for the corresponding CPU
+ * where the process is executing. Use with VAS function.
+ */
+static DEFINE_PER_CPU(struct nx842_coproc *, coproc_inst);
+
 /* no cpu hotplug on powernv, so this list never changes after init */
 static LIST_HEAD(nx842_coprocs);
-static unsigned int nx842_ct;
+static unsigned int nx842_ct;	/* used in icswx function */
+
+static int (*nx842_powernv_exec)(const unsigned char *in,
+				unsigned int inlen, unsigned char *out,
+				unsigned int *outlenp, void *workmem, int fc);
 
 /**
  * setup_indirect_dde - Setup an indirect DDE
@@ -238,6 +258,13 @@ static int wait_for_csb(struct nx842_workmem *wmem,
 	case CSB_CC_TEMPL_OVERFLOW:
 		CSB_ERR(csb, "Compressed data template shows data past end");
 		return -EINVAL;
+	case CSB_CC_EXCEED_BYTE_COUNT:	/* P9 or later */
+		/*
+		 * DDE byte count exceeds the limit specified in Maximum
+		 * byte count register.
+		 */
+		CSB_ERR(csb, "DDE byte count exceeds the limit");
+		return -EINVAL;
 
 	/* these should not happen */
 	case CSB_CC_INVALID_ALIGN:
@@ -279,9 +306,17 @@ static int wait_for_csb(struct nx842_workmem *wmem,
 		CSB_ERR(csb, "Too many DDEs in DDL");
 		return -EINVAL;
 	case CSB_CC_TRANSPORT:
+	case CSB_CC_INVALID_CRB:	/* P9 or later */
 		/* shouldn't happen, we setup CRB correctly */
 		CSB_ERR(csb, "Invalid CRB");
 		return -EINVAL;
+	case CSB_CC_INVALID_DDE:	/* P9 or later */
+		/*
+		 * shouldn't happen, setup_direct/indirect_dde creates
+		 * DDE right
+		 */
+		CSB_ERR(csb, "Invalid DDE");
+		return -EINVAL;
 	case CSB_CC_SEGMENTED_DDL:
 		/* shouldn't happen, setup_ddl creates DDL right */
 		CSB_ERR(csb, "Segmented DDL error");
@@ -325,6 +360,9 @@ static int wait_for_csb(struct nx842_workmem *wmem,
 	case CSB_CC_HW:
 		CSB_ERR(csb, "Correctable hardware error");
 		return -EPROTO;
+	case CSB_CC_HW_EXPIRED_TIMER:	/* P9 or later */
+		CSB_ERR(csb, "Job did not finish within allowed time");
+		return -EPROTO;
 
 	default:
 		CSB_ERR(csb, "Invalid CC %d", csb->cc);
@@ -353,8 +391,42 @@ static int wait_for_csb(struct nx842_workmem *wmem,
 	return 0;
 }
 
+static int nx842_config_crb(const unsigned char *in, unsigned int inlen,
+			unsigned char *out, unsigned int outlen,
+			struct nx842_workmem *wmem)
+{
+	struct coprocessor_request_block *crb;
+	struct coprocessor_status_block *csb;
+	u64 csb_addr;
+	int ret;
+
+	crb = &wmem->crb;
+	csb = &crb->csb;
+
+	/* Clear any previous values */
+	memset(crb, 0, sizeof(*crb));
+
+	/* set up DDLs */
+	ret = setup_ddl(&crb->source, wmem->ddl_in,
+			(unsigned char *)in, inlen, true);
+	if (ret)
+		return ret;
+
+	ret = setup_ddl(&crb->target, wmem->ddl_out,
+			out, outlen, false);
+	if (ret)
+		return ret;
+
+	/* set up CRB's CSB addr */
+	csb_addr = nx842_get_pa(csb) & CRB_CSB_ADDRESS;
+	csb_addr |= CRB_CSB_AT; /* Addrs are phys */
+	crb->csb_addr = cpu_to_be64(csb_addr);
+
+	return 0;
+}
+
 /**
- * nx842_powernv_function - compress/decompress data using the 842 algorithm
+ * nx842_exec_icswx - compress/decompress data using the 842 algorithm
  *
  * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems.
  * This compresses or decompresses the provided input buffer into the provided
@@ -384,7 +456,7 @@ static int wait_for_csb(struct nx842_workmem *wmem,
  *   -ETIMEDOUT	hardware did not complete operation in reasonable time
  *   -EINTR	operation was aborted
  */
-static int nx842_powernv_function(const unsigned char *in, unsigned int inlen,
+static int nx842_exec_icswx(const unsigned char *in, unsigned int inlen,
 				  unsigned char *out, unsigned int *outlenp,
 				  void *workmem, int fc)
 {
@@ -392,7 +464,6 @@ static int nx842_powernv_function(const unsigned char *in, unsigned int inlen,
 	struct coprocessor_status_block *csb;
 	struct nx842_workmem *wmem;
 	int ret;
-	u64 csb_addr;
 	u32 ccw;
 	unsigned int outlen = *outlenp;
 
@@ -406,32 +477,18 @@ static int nx842_powernv_function(const unsigned char *in, unsigned int inlen,
 		return -ENODEV;
 	}
 
-	crb = &wmem->crb;
-	csb = &crb->csb;
-
-	/* Clear any previous values */
-	memset(crb, 0, sizeof(*crb));
-
-	/* set up DDLs */
-	ret = setup_ddl(&crb->source, wmem->ddl_in,
-			(unsigned char *)in, inlen, true);
-	if (ret)
-		return ret;
-	ret = setup_ddl(&crb->target, wmem->ddl_out,
-			out, outlen, false);
+	ret = nx842_config_crb(in, inlen, out, outlen, wmem);
 	if (ret)
 		return ret;
 
+	crb = &wmem->crb;
+	csb = &crb->csb;
+
 	/* set up CCW */
 	ccw = 0;
-	ccw = SET_FIELD(ccw, CCW_CT, nx842_ct);
-	ccw = SET_FIELD(ccw, CCW_CI_842, 0); /* use 0 for hw auto-selection */
-	ccw = SET_FIELD(ccw, CCW_FC_842, fc);
-
-	/* set up CRB's CSB addr */
-	csb_addr = nx842_get_pa(csb) & CRB_CSB_ADDRESS;
-	csb_addr |= CRB_CSB_AT; /* Addrs are phys */
-	crb->csb_addr = cpu_to_be64(csb_addr);
+	ccw = SET_FIELD(CCW_CT, ccw, nx842_ct);
+	ccw = SET_FIELD(CCW_CI_842, ccw, 0); /* use 0 for hw auto-selection */
+	ccw = SET_FIELD(CCW_FC_842, ccw, fc);
 
 	wmem->start = ktime_get();
 
@@ -471,6 +528,104 @@ static int nx842_powernv_function(const unsigned char *in, unsigned int inlen,
 }
 
 /**
+ * nx842_exec_vas - compress/decompress data using the 842 algorithm
+ *
+ * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems.
+ * This compresses or decompresses the provided input buffer into the provided
+ * output buffer.
+ *
+ * Upon return from this function @outlen contains the length of the
+ * output data.  If there is an error then @outlen will be 0 and an
+ * error will be specified by the return code from this function.
+ *
+ * The @workmem buffer should only be used by one function call at a time.
+ *
+ * @in: input buffer pointer
+ * @inlen: input buffer size
+ * @out: output buffer pointer
+ * @outlenp: output buffer size pointer
+ * @workmem: working memory buffer pointer, size determined by
+ *           nx842_powernv_driver.workmem_size
+ * @fc: function code, see CCW Function Codes in nx-842.h
+ *
+ * Returns:
+ *   0		Success, output of length @outlenp stored in the buffer
+ *		at @out
+ *   -ENODEV	Hardware unavailable
+ *   -ENOSPC	Output buffer is to small
+ *   -EMSGSIZE	Input buffer too large
+ *   -EINVAL	buffer constraints do not fix nx842_constraints
+ *   -EPROTO	hardware error during operation
+ *   -ETIMEDOUT	hardware did not complete operation in reasonable time
+ *   -EINTR	operation was aborted
+ */
+static int nx842_exec_vas(const unsigned char *in, unsigned int inlen,
+				  unsigned char *out, unsigned int *outlenp,
+				  void *workmem, int fc)
+{
+	struct coprocessor_request_block *crb;
+	struct coprocessor_status_block *csb;
+	struct nx842_workmem *wmem;
+	struct vas_window *txwin;
+	int ret, i = 0;
+	u32 ccw;
+	unsigned int outlen = *outlenp;
+
+	wmem = PTR_ALIGN(workmem, WORKMEM_ALIGN);
+
+	*outlenp = 0;
+
+	crb = &wmem->crb;
+	csb = &crb->csb;
+
+	ret = nx842_config_crb(in, inlen, out, outlen, wmem);
+	if (ret)
+		return ret;
+
+	ccw = 0;
+	ccw = SET_FIELD(CCW_FC_842, ccw, fc);
+	crb->ccw = cpu_to_be32(ccw);
+
+	txwin = wmem->txwin;
+	/* shoudn't happen, we don't load without a coproc */
+	if (!txwin) {
+		pr_err_ratelimited("NX-842 coprocessor is not available");
+		return -ENODEV;
+	}
+
+	do {
+		wmem->start = ktime_get();
+		preempt_disable();
+		/*
+		 * VAS copy CRB into L2 cache. Refer <asm/vas.h>.
+		 * @crb and @offset.
+		 */
+		vas_copy_crb(crb, 0);
+
+		/*
+		 * VAS paste previously copied CRB to NX.
+		 * @txwin, @offset and @last (must be true).
+		 */
+		ret = vas_paste_crb(txwin, 0, 1);
+		preempt_enable();
+		/*
+		 * Retry copy/paste function for VAS failures.
+		 */
+	} while (ret && (i++ < VAS_RETRIES));
+
+	if (ret) {
+		pr_err_ratelimited("VAS copy/paste failed\n");
+		return ret;
+	}
+
+	ret = wait_for_csb(wmem, csb);
+	if (!ret)
+		*outlenp = be32_to_cpu(csb->count);
+
+	return ret;
+}
+
+/**
  * nx842_powernv_compress - Compress data using the 842 algorithm
  *
  * Compression provided by the NX842 coprocessor on IBM PowerNV systems.
@@ -488,13 +643,13 @@ static int nx842_powernv_function(const unsigned char *in, unsigned int inlen,
  * @workmem: working memory buffer pointer, size determined by
  *           nx842_powernv_driver.workmem_size
  *
- * Returns: see @nx842_powernv_function()
+ * Returns: see @nx842_powernv_exec()
  */
 static int nx842_powernv_compress(const unsigned char *in, unsigned int inlen,
 				  unsigned char *out, unsigned int *outlenp,
 				  void *wmem)
 {
-	return nx842_powernv_function(in, inlen, out, outlenp,
+	return nx842_powernv_exec(in, inlen, out, outlenp,
 				      wmem, CCW_FC_842_COMP_CRC);
 }
 
@@ -516,16 +671,219 @@ static int nx842_powernv_compress(const unsigned char *in, unsigned int inlen,
  * @workmem: working memory buffer pointer, size determined by
  *           nx842_powernv_driver.workmem_size
  *
- * Returns: see @nx842_powernv_function()
+ * Returns: see @nx842_powernv_exec()
  */
 static int nx842_powernv_decompress(const unsigned char *in, unsigned int inlen,
 				    unsigned char *out, unsigned int *outlenp,
 				    void *wmem)
 {
-	return nx842_powernv_function(in, inlen, out, outlenp,
+	return nx842_powernv_exec(in, inlen, out, outlenp,
 				      wmem, CCW_FC_842_DECOMP_CRC);
 }
 
+static inline void nx842_add_coprocs_list(struct nx842_coproc *coproc,
+					int chipid)
+{
+	coproc->chip_id = chipid;
+	INIT_LIST_HEAD(&coproc->list);
+	list_add(&coproc->list, &nx842_coprocs);
+}
+
+/*
+ * Identify chip ID for each CPU and save coprocesor adddress for the
+ * corresponding NX engine in percpu coproc_inst.
+ * coproc_inst is used in crypto_init to open send window on the NX instance
+ * for the corresponding CPU / chip where the open request is executed.
+ */
+static void nx842_set_per_cpu_coproc(struct nx842_coproc *coproc)
+{
+	unsigned int i, chip_id;
+
+	for_each_possible_cpu(i) {
+		chip_id = cpu_to_chip_id(i);
+
+		if (coproc->chip_id == chip_id)
+			per_cpu(coproc_inst, i) = coproc;
+	}
+}
+
+
+static struct vas_window *nx842_alloc_txwin(struct nx842_coproc *coproc)
+{
+	struct vas_window *txwin = NULL;
+	struct vas_tx_win_attr txattr;
+
+	/*
+	 * Kernel requests will be high priority. So open send
+	 * windows only for high priority RxFIFO entries.
+	 */
+	vas_init_tx_win_attr(&txattr, coproc->ct);
+	txattr.lpid = 0;	/* lpid is 0 for kernel requests */
+	txattr.pid = 0;		/* pid is 0 for kernel requests */
+
+	/*
+	 * Open a VAS send window which is used to send request to NX.
+	 */
+	txwin = vas_tx_win_open(coproc->vas.id, coproc->ct, &txattr);
+	if (IS_ERR(txwin)) {
+		pr_err("ibm,nx-842: Can not open TX window: %ld\n",
+				PTR_ERR(txwin));
+		return NULL;
+	}
+
+	return txwin;
+}
+
+static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,
+					int vasid)
+{
+	struct vas_window *rxwin = NULL;
+	struct vas_rx_win_attr rxattr;
+	struct nx842_coproc *coproc;
+	u32 lpid, pid, tid, fifo_size;
+	u64 rx_fifo;
+	const char *priority;
+	int ret;
+
+	ret = of_property_read_u64(dn, "rx-fifo-address", &rx_fifo);
+	if (ret) {
+		pr_err("Missing rx-fifo-address property\n");
+		return ret;
+	}
+
+	ret = of_property_read_u32(dn, "rx-fifo-size", &fifo_size);
+	if (ret) {
+		pr_err("Missing rx-fifo-size property\n");
+		return ret;
+	}
+
+	ret = of_property_read_u32(dn, "lpid", &lpid);
+	if (ret) {
+		pr_err("Missing lpid property\n");
+		return ret;
+	}
+
+	ret = of_property_read_u32(dn, "pid", &pid);
+	if (ret) {
+		pr_err("Missing pid property\n");
+		return ret;
+	}
+
+	ret = of_property_read_u32(dn, "tid", &tid);
+	if (ret) {
+		pr_err("Missing tid property\n");
+		return ret;
+	}
+
+	ret = of_property_read_string(dn, "priority", &priority);
+	if (ret) {
+		pr_err("Missing priority property\n");
+		return ret;
+	}
+
+	coproc = kzalloc(sizeof(*coproc), GFP_KERNEL);
+	if (!coproc)
+		return -ENOMEM;
+
+	if (!strcmp(priority, "High"))
+		coproc->ct = VAS_COP_TYPE_842_HIPRI;
+	else if (!strcmp(priority, "Normal"))
+		coproc->ct = VAS_COP_TYPE_842;
+	else {
+		pr_err("Invalid RxFIFO priority value\n");
+		ret =  -EINVAL;
+		goto err_out;
+	}
+
+	vas_init_rx_win_attr(&rxattr, coproc->ct);
+	rxattr.rx_fifo = (void *)rx_fifo;
+	rxattr.rx_fifo_size = fifo_size;
+	rxattr.lnotify_lpid = lpid;
+	rxattr.lnotify_pid = pid;
+	rxattr.lnotify_tid = tid;
+	rxattr.wcreds_max = MAX_CREDITS_PER_RXFIFO;
+
+	/*
+	 * Open a VAS receice window which is used to configure RxFIFO
+	 * for NX.
+	 */
+	rxwin = vas_rx_win_open(vasid, coproc->ct, &rxattr);
+	if (IS_ERR(rxwin)) {
+		ret = PTR_ERR(rxwin);
+		pr_err("setting RxFIFO with VAS failed: %d\n",
+			ret);
+		goto err_out;
+	}
+
+	coproc->vas.rxwin = rxwin;
+	coproc->vas.id = vasid;
+	nx842_add_coprocs_list(coproc, chip_id);
+
+	/*
+	 * Kernel requests use only high priority FIFOs. So save coproc
+	 * info in percpu coproc_inst which will be used to open send
+	 * windows for crypto open requests later.
+	 */
+	if (coproc->ct == VAS_COP_TYPE_842_HIPRI)
+		nx842_set_per_cpu_coproc(coproc);
+
+	return 0;
+
+err_out:
+	kfree(coproc);
+	return ret;
+}
+
+
+static int __init nx842_powernv_probe_vas(struct device_node *pn)
+{
+	struct device_node *dn;
+	int chip_id, vasid, ret = 0;
+	int nx_fifo_found = 0;
+
+	chip_id = of_get_ibm_chip_id(pn);
+	if (chip_id < 0) {
+		pr_err("ibm,chip-id missing\n");
+		return -EINVAL;
+	}
+
+	for_each_compatible_node(dn, NULL, "ibm,power9-vas-x") {
+		if (of_get_ibm_chip_id(dn) == chip_id)
+			break;
+	}
+
+	if (!dn) {
+		pr_err("Missing VAS device node\n");
+		return -EINVAL;
+	}
+
+	if (of_property_read_u32(dn, "ibm,vas-id", &vasid)) {
+		pr_err("Missing ibm,vas-id device property\n");
+		of_node_put(dn);
+		return -EINVAL;
+	}
+
+	of_node_put(dn);
+
+	for_each_child_of_node(pn, dn) {
+		if (of_device_is_compatible(dn, "ibm,p9-nx-842")) {
+			ret = vas_cfg_coproc_info(dn, chip_id, vasid);
+			if (ret) {
+				of_node_put(dn);
+				return ret;
+			}
+			nx_fifo_found++;
+		}
+	}
+
+	if (!nx_fifo_found) {
+		pr_err("NX842 FIFO nodes are missing\n");
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
 static int __init nx842_powernv_probe(struct device_node *dn)
 {
 	struct nx842_coproc *coproc;
@@ -552,11 +910,9 @@ static int __init nx842_powernv_probe(struct device_node *dn)
 	if (!coproc)
 		return -ENOMEM;
 
-	coproc->chip_id = chip_id;
 	coproc->ct = ct;
 	coproc->ci = ci;
-	INIT_LIST_HEAD(&coproc->list);
-	list_add(&coproc->list, &nx842_coprocs);
+	nx842_add_coprocs_list(coproc, chip_id);
 
 	pr_info("coprocessor found on chip %d, CT %d CI %d\n", chip_id, ct, ci);
 
@@ -569,6 +925,19 @@ static int __init nx842_powernv_probe(struct device_node *dn)
 	return 0;
 }
 
+static void nx842_delete_coprocs(void)
+{
+	struct nx842_coproc *coproc, *n;
+
+	list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
+		if (coproc->vas.rxwin)
+			vas_win_close(coproc->vas.rxwin);
+
+		list_del(&coproc->list);
+		kfree(coproc);
+	}
+}
+
 static struct nx842_constraints nx842_powernv_constraints = {
 	.alignment =	DDE_BUFFER_ALIGN,
 	.multiple =	DDE_BUFFER_LAST_MULT,
@@ -585,6 +954,46 @@ static struct nx842_driver nx842_powernv_driver = {
 	.decompress =	nx842_powernv_decompress,
 };
 
+static int nx842_powernv_crypto_init_vas(struct crypto_tfm *tfm)
+{
+	struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct nx842_workmem *wmem;
+	struct nx842_coproc *coproc;
+	int ret;
+
+	ret = nx842_crypto_init(tfm, &nx842_powernv_driver);
+
+	if (ret)
+		return ret;
+
+	wmem = PTR_ALIGN((struct nx842_workmem *)ctx->wmem, WORKMEM_ALIGN);
+	coproc = per_cpu(coproc_inst, smp_processor_id());
+
+	ret = -EINVAL;
+	if (coproc && coproc->vas.rxwin) {
+		wmem->txwin = nx842_alloc_txwin(coproc);
+		if (!IS_ERR(wmem->txwin))
+			return 0;
+
+		ret = PTR_ERR(wmem->txwin);
+	}
+
+	return ret;
+}
+
+void nx842_powernv_crypto_exit_vas(struct crypto_tfm *tfm)
+{
+	struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct nx842_workmem *wmem;
+
+	wmem = PTR_ALIGN((struct nx842_workmem *)ctx->wmem, WORKMEM_ALIGN);
+
+	if (wmem && wmem->txwin)
+		vas_win_close(wmem->txwin);
+
+	nx842_crypto_exit(tfm);
+}
+
 static int nx842_powernv_crypto_init(struct crypto_tfm *tfm)
 {
 	return nx842_crypto_init(tfm, &nx842_powernv_driver);
@@ -618,21 +1027,31 @@ static __init int nx842_powernv_init(void)
 	BUILD_BUG_ON(DDE_BUFFER_ALIGN % DDE_BUFFER_SIZE_MULT);
 	BUILD_BUG_ON(DDE_BUFFER_SIZE_MULT % DDE_BUFFER_LAST_MULT);
 
-	for_each_compatible_node(dn, NULL, "ibm,power-nx")
-		nx842_powernv_probe(dn);
+	for_each_compatible_node(dn, NULL, "ibm,power9-nx") {
+		ret = nx842_powernv_probe_vas(dn);
+		if (ret) {
+			nx842_delete_coprocs();
+			return ret;
+		}
+	}
 
-	if (!nx842_ct)
-		return -ENODEV;
+	if (list_empty(&nx842_coprocs)) {
+		for_each_compatible_node(dn, NULL, "ibm,power-nx")
+			nx842_powernv_probe(dn);
 
-	ret = crypto_register_alg(&nx842_powernv_alg);
-	if (ret) {
-		struct nx842_coproc *coproc, *n;
+		if (!nx842_ct)
+			return -ENODEV;
 
-		list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
-			list_del(&coproc->list);
-			kfree(coproc);
-		}
+		nx842_powernv_exec = nx842_exec_icswx;
+	} else {
+		nx842_powernv_exec = nx842_exec_vas;
+		nx842_powernv_alg.cra_init = nx842_powernv_crypto_init_vas;
+		nx842_powernv_alg.cra_exit = nx842_powernv_crypto_exit_vas;
+	}
 
+	ret = crypto_register_alg(&nx842_powernv_alg);
+	if (ret) {
+		nx842_delete_coprocs();
 		return ret;
 	}
 
@@ -642,13 +1061,8 @@ module_init(nx842_powernv_init);
 
 static void __exit nx842_powernv_exit(void)
 {
-	struct nx842_coproc *coproc, *n;
-
 	crypto_unregister_alg(&nx842_powernv_alg);
 
-	list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
-		list_del(&coproc->list);
-		kfree(coproc);
-	}
+	nx842_delete_coprocs();
 }
 module_exit(nx842_powernv_exit);
diff --git a/drivers/crypto/nx/nx-842.c b/drivers/crypto/nx/nx-842.c
index d94e25df503b..da3cb8c35ec7 100644
--- a/drivers/crypto/nx/nx-842.c
+++ b/drivers/crypto/nx/nx-842.c
@@ -116,7 +116,7 @@ int nx842_crypto_init(struct crypto_tfm *tfm, struct nx842_driver *driver)
 
 	spin_lock_init(&ctx->lock);
 	ctx->driver = driver;
-	ctx->wmem = kmalloc(driver->workmem_size, GFP_KERNEL);
+	ctx->wmem = kzalloc(driver->workmem_size, GFP_KERNEL);
 	ctx->sbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER);
 	ctx->dbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER);
 	if (!ctx->wmem || !ctx->sbounce || !ctx->dbounce) {
diff --git a/drivers/crypto/nx/nx-842.h b/drivers/crypto/nx/nx-842.h
index a4eee3bba937..bb2f31792683 100644
--- a/drivers/crypto/nx/nx-842.h
+++ b/drivers/crypto/nx/nx-842.h
@@ -76,9 +76,17 @@
 #define CSB_CC_DECRYPT_OVERFLOW	(64)
 /* asym crypt codes */
 #define CSB_CC_MINV_OVERFLOW	(128)
+/*
+ * HW error - Job did not finish in the maximum time allowed.
+ * Job terminated.
+ */
+#define CSB_CC_HW_EXPIRED_TIMER		(224)
 /* These are reserved for hypervisor use */
 #define CSB_CC_HYP_RESERVE_START	(240)
 #define CSB_CC_HYP_RESERVE_END		(253)
+#define CSB_CC_HYP_RESERVE_P9_END	(251)
+/* No valid interrupt server (P9 or later). */
+#define CSB_CC_HYP_RESERVE_NO_INTR_SERVER	(252)
 #define CSB_CC_HYP_NO_HW		(254)
 #define CSB_CC_HYP_HANG_ABORTED		(255)
 
@@ -100,11 +108,6 @@ static inline unsigned long nx842_get_pa(void *addr)
 	return page_to_phys(vmalloc_to_page(addr)) + offset_in_page(addr);
 }
 
-/* Get/Set bit fields */
-#define MASK_LSH(m)		(__builtin_ffsl(m) - 1)
-#define GET_FIELD(v, m)		(((v) & (m)) >> MASK_LSH(m))
-#define SET_FIELD(v, m, val)	(((v) & ~(m)) | (((val) << MASK_LSH(m)) & (m)))
-
 /**
  * This provides the driver's constraints.  Different nx842 implementations
  * may have varying requirements.  The constraints are:
diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c
index 5120a17731d0..c376a3ee7c2c 100644
--- a/drivers/crypto/omap-aes.c
+++ b/drivers/crypto/omap-aes.c
@@ -1095,6 +1095,7 @@ static int omap_aes_probe(struct platform_device *pdev)
 		irq = platform_get_irq(pdev, 0);
 		if (irq < 0) {
 			dev_err(dev, "can't get IRQ resource\n");
+			err = irq;
 			goto err_irq;
 		}
 
diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c
index 0bcab00e0ff5..d37c9506c36c 100644
--- a/drivers/crypto/omap-des.c
+++ b/drivers/crypto/omap-des.c
@@ -1023,7 +1023,8 @@ static int omap_des_probe(struct platform_device *pdev)
 
 		irq = platform_get_irq(pdev, 0);
 		if (irq < 0) {
-			dev_err(dev, "can't get IRQ resource\n");
+			dev_err(dev, "can't get IRQ resource: %d\n", irq);
+			err = irq;
 			goto err_irq;
 		}
 
diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c
index 9ad9d399daf1..c40ac30ec002 100644
--- a/drivers/crypto/omap-sham.c
+++ b/drivers/crypto/omap-sham.c
@@ -2133,7 +2133,7 @@ data_err:
 
 static int omap_sham_remove(struct platform_device *pdev)
 {
-	static struct omap_sham_dev *dd;
+	struct omap_sham_dev *dd;
 	int i, j;
 
 	dd = platform_get_drvdata(pdev);
diff --git a/drivers/crypto/qat/qat_common/adf_aer.c b/drivers/crypto/qat/qat_common/adf_aer.c
index d3e25c37dc33..da8a2d3b5e9a 100644
--- a/drivers/crypto/qat/qat_common/adf_aer.c
+++ b/drivers/crypto/qat/qat_common/adf_aer.c
@@ -208,7 +208,7 @@ static pci_ers_result_t adf_slot_reset(struct pci_dev *pdev)
 static void adf_resume(struct pci_dev *pdev)
 {
 	dev_info(&pdev->dev, "Acceleration driver reset completed\n");
-	dev_info(&pdev->dev, "Device is up and runnig\n");
+	dev_info(&pdev->dev, "Device is up and running\n");
 }
 
 static const struct pci_error_handlers adf_err_handler = {
diff --git a/drivers/crypto/rockchip/rk3288_crypto.c b/drivers/crypto/rockchip/rk3288_crypto.c
index d0f80c6241f9..c9d622abd90c 100644
--- a/drivers/crypto/rockchip/rk3288_crypto.c
+++ b/drivers/crypto/rockchip/rk3288_crypto.c
@@ -169,50 +169,82 @@ static irqreturn_t rk_crypto_irq_handle(int irq, void *dev_id)
 {
 	struct rk_crypto_info *dev  = platform_get_drvdata(dev_id);
 	u32 interrupt_status;
-	int err = 0;
 
 	spin_lock(&dev->lock);
 	interrupt_status = CRYPTO_READ(dev, RK_CRYPTO_INTSTS);
 	CRYPTO_WRITE(dev, RK_CRYPTO_INTSTS, interrupt_status);
+
 	if (interrupt_status & 0x0a) {
 		dev_warn(dev->dev, "DMA Error\n");
-		err = -EFAULT;
-	} else if (interrupt_status & 0x05) {
-		err = dev->update(dev);
+		dev->err = -EFAULT;
 	}
-	if (err)
-		dev->complete(dev, err);
+	tasklet_schedule(&dev->done_task);
+
 	spin_unlock(&dev->lock);
 	return IRQ_HANDLED;
 }
 
-static void rk_crypto_tasklet_cb(unsigned long data)
+static int rk_crypto_enqueue(struct rk_crypto_info *dev,
+			      struct crypto_async_request *async_req)
+{
+	unsigned long flags;
+	int ret;
+
+	spin_lock_irqsave(&dev->lock, flags);
+	ret = crypto_enqueue_request(&dev->queue, async_req);
+	if (dev->busy) {
+		spin_unlock_irqrestore(&dev->lock, flags);
+		return ret;
+	}
+	dev->busy = true;
+	spin_unlock_irqrestore(&dev->lock, flags);
+	tasklet_schedule(&dev->queue_task);
+
+	return ret;
+}
+
+static void rk_crypto_queue_task_cb(unsigned long data)
 {
 	struct rk_crypto_info *dev = (struct rk_crypto_info *)data;
 	struct crypto_async_request *async_req, *backlog;
 	unsigned long flags;
 	int err = 0;
 
+	dev->err = 0;
 	spin_lock_irqsave(&dev->lock, flags);
 	backlog   = crypto_get_backlog(&dev->queue);
 	async_req = crypto_dequeue_request(&dev->queue);
-	spin_unlock_irqrestore(&dev->lock, flags);
+
 	if (!async_req) {
-		dev_err(dev->dev, "async_req is NULL !!\n");
+		dev->busy = false;
+		spin_unlock_irqrestore(&dev->lock, flags);
 		return;
 	}
+	spin_unlock_irqrestore(&dev->lock, flags);
+
 	if (backlog) {
 		backlog->complete(backlog, -EINPROGRESS);
 		backlog = NULL;
 	}
 
-	if (crypto_tfm_alg_type(async_req->tfm) == CRYPTO_ALG_TYPE_ABLKCIPHER)
-		dev->ablk_req = ablkcipher_request_cast(async_req);
-	else
-		dev->ahash_req = ahash_request_cast(async_req);
+	dev->async_req = async_req;
 	err = dev->start(dev);
 	if (err)
-		dev->complete(dev, err);
+		dev->complete(dev->async_req, err);
+}
+
+static void rk_crypto_done_task_cb(unsigned long data)
+{
+	struct rk_crypto_info *dev = (struct rk_crypto_info *)data;
+
+	if (dev->err) {
+		dev->complete(dev->async_req, dev->err);
+		return;
+	}
+
+	dev->err = dev->update(dev);
+	if (dev->err)
+		dev->complete(dev->async_req, dev->err);
 }
 
 static struct rk_crypto_tmp *rk_cipher_algs[] = {
@@ -361,14 +393,18 @@ static int rk_crypto_probe(struct platform_device *pdev)
 	crypto_info->dev = &pdev->dev;
 	platform_set_drvdata(pdev, crypto_info);
 
-	tasklet_init(&crypto_info->crypto_tasklet,
-		     rk_crypto_tasklet_cb, (unsigned long)crypto_info);
+	tasklet_init(&crypto_info->queue_task,
+		     rk_crypto_queue_task_cb, (unsigned long)crypto_info);
+	tasklet_init(&crypto_info->done_task,
+		     rk_crypto_done_task_cb, (unsigned long)crypto_info);
 	crypto_init_queue(&crypto_info->queue, 50);
 
 	crypto_info->enable_clk = rk_crypto_enable_clk;
 	crypto_info->disable_clk = rk_crypto_disable_clk;
 	crypto_info->load_data = rk_load_data;
 	crypto_info->unload_data = rk_unload_data;
+	crypto_info->enqueue = rk_crypto_enqueue;
+	crypto_info->busy = false;
 
 	err = rk_crypto_register(crypto_info);
 	if (err) {
@@ -380,7 +416,8 @@ static int rk_crypto_probe(struct platform_device *pdev)
 	return 0;
 
 err_register_alg:
-	tasklet_kill(&crypto_info->crypto_tasklet);
+	tasklet_kill(&crypto_info->queue_task);
+	tasklet_kill(&crypto_info->done_task);
 err_crypto:
 	return err;
 }
@@ -390,7 +427,8 @@ static int rk_crypto_remove(struct platform_device *pdev)
 	struct rk_crypto_info *crypto_tmp = platform_get_drvdata(pdev);
 
 	rk_crypto_unregister();
-	tasklet_kill(&crypto_tmp->crypto_tasklet);
+	tasklet_kill(&crypto_tmp->done_task);
+	tasklet_kill(&crypto_tmp->queue_task);
 	return 0;
 }
 
diff --git a/drivers/crypto/rockchip/rk3288_crypto.h b/drivers/crypto/rockchip/rk3288_crypto.h
index d7b71fea320b..ab6a1b4c40f0 100644
--- a/drivers/crypto/rockchip/rk3288_crypto.h
+++ b/drivers/crypto/rockchip/rk3288_crypto.h
@@ -190,9 +190,10 @@ struct rk_crypto_info {
 	void __iomem			*reg;
 	int				irq;
 	struct crypto_queue		queue;
-	struct tasklet_struct		crypto_tasklet;
-	struct ablkcipher_request	*ablk_req;
-	struct ahash_request		*ahash_req;
+	struct tasklet_struct		queue_task;
+	struct tasklet_struct		done_task;
+	struct crypto_async_request	*async_req;
+	int 				err;
 	/* device lock */
 	spinlock_t			lock;
 
@@ -208,18 +209,20 @@ struct rk_crypto_info {
 	size_t				nents;
 	unsigned int			total;
 	unsigned int			count;
-	u32				mode;
 	dma_addr_t			addr_in;
 	dma_addr_t			addr_out;
+	bool				busy;
 	int (*start)(struct rk_crypto_info *dev);
 	int (*update)(struct rk_crypto_info *dev);
-	void (*complete)(struct rk_crypto_info *dev, int err);
+	void (*complete)(struct crypto_async_request *base, int err);
 	int (*enable_clk)(struct rk_crypto_info *dev);
 	void (*disable_clk)(struct rk_crypto_info *dev);
 	int (*load_data)(struct rk_crypto_info *dev,
 			 struct scatterlist *sg_src,
 			 struct scatterlist *sg_dst);
 	void (*unload_data)(struct rk_crypto_info *dev);
+	int (*enqueue)(struct rk_crypto_info *dev,
+		       struct crypto_async_request *async_req);
 };
 
 /* the private variable of hash */
@@ -232,12 +235,14 @@ struct rk_ahash_ctx {
 /* the privete variable of hash for fallback */
 struct rk_ahash_rctx {
 	struct ahash_request		fallback_req;
+	u32				mode;
 };
 
 /* the private variable of cipher */
 struct rk_cipher_ctx {
 	struct rk_crypto_info		*dev;
 	unsigned int			keylen;
+	u32				mode;
 };
 
 enum alg_type {
diff --git a/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c b/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c
index b5a3afe222e4..639c15c5364b 100644
--- a/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c
+++ b/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c
@@ -15,35 +15,19 @@
 
 #define RK_CRYPTO_DEC			BIT(0)
 
-static void rk_crypto_complete(struct rk_crypto_info *dev, int err)
+static void rk_crypto_complete(struct crypto_async_request *base, int err)
 {
-	if (dev->ablk_req->base.complete)
-		dev->ablk_req->base.complete(&dev->ablk_req->base, err);
+	if (base->complete)
+		base->complete(base, err);
 }
 
 static int rk_handle_req(struct rk_crypto_info *dev,
 			 struct ablkcipher_request *req)
 {
-	unsigned long flags;
-	int err;
-
 	if (!IS_ALIGNED(req->nbytes, dev->align_size))
 		return -EINVAL;
-
-	dev->left_bytes = req->nbytes;
-	dev->total = req->nbytes;
-	dev->sg_src = req->src;
-	dev->first = req->src;
-	dev->nents = sg_nents(req->src);
-	dev->sg_dst = req->dst;
-	dev->aligned = 1;
-	dev->ablk_req = req;
-
-	spin_lock_irqsave(&dev->lock, flags);
-	err = ablkcipher_enqueue_request(&dev->queue, req);
-	spin_unlock_irqrestore(&dev->lock, flags);
-	tasklet_schedule(&dev->crypto_tasklet);
-	return err;
+	else
+		return dev->enqueue(dev, &req->base);
 }
 
 static int rk_aes_setkey(struct crypto_ablkcipher *cipher,
@@ -93,7 +77,7 @@ static int rk_aes_ecb_encrypt(struct ablkcipher_request *req)
 	struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 	struct rk_crypto_info *dev = ctx->dev;
 
-	dev->mode = RK_CRYPTO_AES_ECB_MODE;
+	ctx->mode = RK_CRYPTO_AES_ECB_MODE;
 	return rk_handle_req(dev, req);
 }
 
@@ -103,7 +87,7 @@ static int rk_aes_ecb_decrypt(struct ablkcipher_request *req)
 	struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 	struct rk_crypto_info *dev = ctx->dev;
 
-	dev->mode = RK_CRYPTO_AES_ECB_MODE | RK_CRYPTO_DEC;
+	ctx->mode = RK_CRYPTO_AES_ECB_MODE | RK_CRYPTO_DEC;
 	return rk_handle_req(dev, req);
 }
 
@@ -113,7 +97,7 @@ static int rk_aes_cbc_encrypt(struct ablkcipher_request *req)
 	struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 	struct rk_crypto_info *dev = ctx->dev;
 
-	dev->mode = RK_CRYPTO_AES_CBC_MODE;
+	ctx->mode = RK_CRYPTO_AES_CBC_MODE;
 	return rk_handle_req(dev, req);
 }
 
@@ -123,7 +107,7 @@ static int rk_aes_cbc_decrypt(struct ablkcipher_request *req)
 	struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 	struct rk_crypto_info *dev = ctx->dev;
 
-	dev->mode = RK_CRYPTO_AES_CBC_MODE | RK_CRYPTO_DEC;
+	ctx->mode = RK_CRYPTO_AES_CBC_MODE | RK_CRYPTO_DEC;
 	return rk_handle_req(dev, req);
 }
 
@@ -133,7 +117,7 @@ static int rk_des_ecb_encrypt(struct ablkcipher_request *req)
 	struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 	struct rk_crypto_info *dev = ctx->dev;
 
-	dev->mode = 0;
+	ctx->mode = 0;
 	return rk_handle_req(dev, req);
 }
 
@@ -143,7 +127,7 @@ static int rk_des_ecb_decrypt(struct ablkcipher_request *req)
 	struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 	struct rk_crypto_info *dev = ctx->dev;
 
-	dev->mode = RK_CRYPTO_DEC;
+	ctx->mode = RK_CRYPTO_DEC;
 	return rk_handle_req(dev, req);
 }
 
@@ -153,7 +137,7 @@ static int rk_des_cbc_encrypt(struct ablkcipher_request *req)
 	struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 	struct rk_crypto_info *dev = ctx->dev;
 
-	dev->mode = RK_CRYPTO_TDES_CHAINMODE_CBC;
+	ctx->mode = RK_CRYPTO_TDES_CHAINMODE_CBC;
 	return rk_handle_req(dev, req);
 }
 
@@ -163,7 +147,7 @@ static int rk_des_cbc_decrypt(struct ablkcipher_request *req)
 	struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 	struct rk_crypto_info *dev = ctx->dev;
 
-	dev->mode = RK_CRYPTO_TDES_CHAINMODE_CBC | RK_CRYPTO_DEC;
+	ctx->mode = RK_CRYPTO_TDES_CHAINMODE_CBC | RK_CRYPTO_DEC;
 	return rk_handle_req(dev, req);
 }
 
@@ -173,7 +157,7 @@ static int rk_des3_ede_ecb_encrypt(struct ablkcipher_request *req)
 	struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 	struct rk_crypto_info *dev = ctx->dev;
 
-	dev->mode = RK_CRYPTO_TDES_SELECT;
+	ctx->mode = RK_CRYPTO_TDES_SELECT;
 	return rk_handle_req(dev, req);
 }
 
@@ -183,7 +167,7 @@ static int rk_des3_ede_ecb_decrypt(struct ablkcipher_request *req)
 	struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 	struct rk_crypto_info *dev = ctx->dev;
 
-	dev->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_DEC;
+	ctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_DEC;
 	return rk_handle_req(dev, req);
 }
 
@@ -193,7 +177,7 @@ static int rk_des3_ede_cbc_encrypt(struct ablkcipher_request *req)
 	struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 	struct rk_crypto_info *dev = ctx->dev;
 
-	dev->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC;
+	ctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC;
 	return rk_handle_req(dev, req);
 }
 
@@ -203,15 +187,16 @@ static int rk_des3_ede_cbc_decrypt(struct ablkcipher_request *req)
 	struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 	struct rk_crypto_info *dev = ctx->dev;
 
-	dev->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC |
+	ctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC |
 		    RK_CRYPTO_DEC;
 	return rk_handle_req(dev, req);
 }
 
 static void rk_ablk_hw_init(struct rk_crypto_info *dev)
 {
-	struct crypto_ablkcipher *cipher =
-		crypto_ablkcipher_reqtfm(dev->ablk_req);
+	struct ablkcipher_request *req =
+		ablkcipher_request_cast(dev->async_req);
+	struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(req);
 	struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
 	struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(cipher);
 	u32 ivsize, block, conf_reg = 0;
@@ -220,25 +205,23 @@ static void rk_ablk_hw_init(struct rk_crypto_info *dev)
 	ivsize = crypto_ablkcipher_ivsize(cipher);
 
 	if (block == DES_BLOCK_SIZE) {
-		dev->mode |= RK_CRYPTO_TDES_FIFO_MODE |
+		ctx->mode |= RK_CRYPTO_TDES_FIFO_MODE |
 			     RK_CRYPTO_TDES_BYTESWAP_KEY |
 			     RK_CRYPTO_TDES_BYTESWAP_IV;
-		CRYPTO_WRITE(dev, RK_CRYPTO_TDES_CTRL, dev->mode);
-		memcpy_toio(dev->reg + RK_CRYPTO_TDES_IV_0,
-			    dev->ablk_req->info, ivsize);
+		CRYPTO_WRITE(dev, RK_CRYPTO_TDES_CTRL, ctx->mode);
+		memcpy_toio(dev->reg + RK_CRYPTO_TDES_IV_0, req->info, ivsize);
 		conf_reg = RK_CRYPTO_DESSEL;
 	} else {
-		dev->mode |= RK_CRYPTO_AES_FIFO_MODE |
+		ctx->mode |= RK_CRYPTO_AES_FIFO_MODE |
 			     RK_CRYPTO_AES_KEY_CHANGE |
 			     RK_CRYPTO_AES_BYTESWAP_KEY |
 			     RK_CRYPTO_AES_BYTESWAP_IV;
 		if (ctx->keylen == AES_KEYSIZE_192)
-			dev->mode |= RK_CRYPTO_AES_192BIT_key;
+			ctx->mode |= RK_CRYPTO_AES_192BIT_key;
 		else if (ctx->keylen == AES_KEYSIZE_256)
-			dev->mode |= RK_CRYPTO_AES_256BIT_key;
-		CRYPTO_WRITE(dev, RK_CRYPTO_AES_CTRL, dev->mode);
-		memcpy_toio(dev->reg + RK_CRYPTO_AES_IV_0,
-			    dev->ablk_req->info, ivsize);
+			ctx->mode |= RK_CRYPTO_AES_256BIT_key;
+		CRYPTO_WRITE(dev, RK_CRYPTO_AES_CTRL, ctx->mode);
+		memcpy_toio(dev->reg + RK_CRYPTO_AES_IV_0, req->info, ivsize);
 	}
 	conf_reg |= RK_CRYPTO_BYTESWAP_BTFIFO |
 		    RK_CRYPTO_BYTESWAP_BRFIFO;
@@ -268,8 +251,18 @@ static int rk_set_data_start(struct rk_crypto_info *dev)
 
 static int rk_ablk_start(struct rk_crypto_info *dev)
 {
+	struct ablkcipher_request *req =
+		ablkcipher_request_cast(dev->async_req);
 	unsigned long flags;
-	int err;
+	int err = 0;
+
+	dev->left_bytes = req->nbytes;
+	dev->total = req->nbytes;
+	dev->sg_src = req->src;
+	dev->first = req->src;
+	dev->nents = sg_nents(req->src);
+	dev->sg_dst = req->dst;
+	dev->aligned = 1;
 
 	spin_lock_irqsave(&dev->lock, flags);
 	rk_ablk_hw_init(dev);
@@ -280,15 +273,16 @@ static int rk_ablk_start(struct rk_crypto_info *dev)
 
 static void rk_iv_copyback(struct rk_crypto_info *dev)
 {
-	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(dev->ablk_req);
+	struct ablkcipher_request *req =
+		ablkcipher_request_cast(dev->async_req);
+	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 	u32 ivsize = crypto_ablkcipher_ivsize(tfm);
 
 	if (ivsize == DES_BLOCK_SIZE)
-		memcpy_fromio(dev->ablk_req->info,
-			      dev->reg + RK_CRYPTO_TDES_IV_0, ivsize);
+		memcpy_fromio(req->info, dev->reg + RK_CRYPTO_TDES_IV_0,
+			      ivsize);
 	else if (ivsize == AES_BLOCK_SIZE)
-		memcpy_fromio(dev->ablk_req->info,
-			      dev->reg + RK_CRYPTO_AES_IV_0, ivsize);
+		memcpy_fromio(req->info, dev->reg + RK_CRYPTO_AES_IV_0, ivsize);
 }
 
 /* return:
@@ -298,10 +292,12 @@ static void rk_iv_copyback(struct rk_crypto_info *dev)
 static int rk_ablk_rx(struct rk_crypto_info *dev)
 {
 	int err = 0;
+	struct ablkcipher_request *req =
+		ablkcipher_request_cast(dev->async_req);
 
 	dev->unload_data(dev);
 	if (!dev->aligned) {
-		if (!sg_pcopy_from_buffer(dev->ablk_req->dst, dev->nents,
+		if (!sg_pcopy_from_buffer(req->dst, dev->nents,
 					  dev->addr_vir, dev->count,
 					  dev->total - dev->left_bytes -
 					  dev->count)) {
@@ -324,7 +320,8 @@ static int rk_ablk_rx(struct rk_crypto_info *dev)
 	} else {
 		rk_iv_copyback(dev);
 		/* here show the calculation is over without any err */
-		dev->complete(dev, 0);
+		dev->complete(dev->async_req, 0);
+		tasklet_schedule(&dev->queue_task);
 	}
 out_rx:
 	return err;
diff --git a/drivers/crypto/rockchip/rk3288_crypto_ahash.c b/drivers/crypto/rockchip/rk3288_crypto_ahash.c
index 718588219f75..821a506b9e17 100644
--- a/drivers/crypto/rockchip/rk3288_crypto_ahash.c
+++ b/drivers/crypto/rockchip/rk3288_crypto_ahash.c
@@ -40,14 +40,16 @@ static int zero_message_process(struct ahash_request *req)
 	return 0;
 }
 
-static void rk_ahash_crypto_complete(struct rk_crypto_info *dev, int err)
+static void rk_ahash_crypto_complete(struct crypto_async_request *base, int err)
 {
-	if (dev->ahash_req->base.complete)
-		dev->ahash_req->base.complete(&dev->ahash_req->base, err);
+	if (base->complete)
+		base->complete(base, err);
 }
 
 static void rk_ahash_reg_init(struct rk_crypto_info *dev)
 {
+	struct ahash_request *req = ahash_request_cast(dev->async_req);
+	struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
 	int reg_status = 0;
 
 	reg_status = CRYPTO_READ(dev, RK_CRYPTO_CTRL) |
@@ -67,7 +69,7 @@ static void rk_ahash_reg_init(struct rk_crypto_info *dev)
 	CRYPTO_WRITE(dev, RK_CRYPTO_INTSTS, RK_CRYPTO_HRDMA_ERR_INT |
 					    RK_CRYPTO_HRDMA_DONE_INT);
 
-	CRYPTO_WRITE(dev, RK_CRYPTO_HASH_CTRL, dev->mode |
+	CRYPTO_WRITE(dev, RK_CRYPTO_HASH_CTRL, rctx->mode |
 					       RK_CRYPTO_HASH_SWAP_DO);
 
 	CRYPTO_WRITE(dev, RK_CRYPTO_CONF, RK_CRYPTO_BYTESWAP_HRFIFO |
@@ -164,64 +166,13 @@ static int rk_ahash_export(struct ahash_request *req, void *out)
 
 static int rk_ahash_digest(struct ahash_request *req)
 {
-	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 	struct rk_ahash_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
-	struct rk_crypto_info *dev = NULL;
-	unsigned long flags;
-	int ret;
+	struct rk_crypto_info *dev = tctx->dev;
 
 	if (!req->nbytes)
 		return zero_message_process(req);
-
-	dev = tctx->dev;
-	dev->total = req->nbytes;
-	dev->left_bytes = req->nbytes;
-	dev->aligned = 0;
-	dev->mode = 0;
-	dev->align_size = 4;
-	dev->sg_dst = NULL;
-	dev->sg_src = req->src;
-	dev->first = req->src;
-	dev->nents = sg_nents(req->src);
-
-	switch (crypto_ahash_digestsize(tfm)) {
-	case SHA1_DIGEST_SIZE:
-		dev->mode = RK_CRYPTO_HASH_SHA1;
-		break;
-	case SHA256_DIGEST_SIZE:
-		dev->mode = RK_CRYPTO_HASH_SHA256;
-		break;
-	case MD5_DIGEST_SIZE:
-		dev->mode = RK_CRYPTO_HASH_MD5;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	rk_ahash_reg_init(dev);
-
-	spin_lock_irqsave(&dev->lock, flags);
-	ret = crypto_enqueue_request(&dev->queue, &req->base);
-	spin_unlock_irqrestore(&dev->lock, flags);
-
-	tasklet_schedule(&dev->crypto_tasklet);
-
-	/*
-	 * it will take some time to process date after last dma transmission.
-	 *
-	 * waiting time is relative with the last date len,
-	 * so cannot set a fixed time here.
-	 * 10-50 makes system not call here frequently wasting
-	 * efficiency, and make it response quickly when dma
-	 * complete.
-	 */
-	while (!CRYPTO_READ(dev, RK_CRYPTO_HASH_STS))
-		usleep_range(10, 50);
-
-	memcpy_fromio(req->result, dev->reg + RK_CRYPTO_HASH_DOUT_0,
-		      crypto_ahash_digestsize(tfm));
-
-	return 0;
+	else
+		return dev->enqueue(dev, &req->base);
 }
 
 static void crypto_ahash_dma_start(struct rk_crypto_info *dev)
@@ -244,12 +195,45 @@ static int rk_ahash_set_data_start(struct rk_crypto_info *dev)
 
 static int rk_ahash_start(struct rk_crypto_info *dev)
 {
+	struct ahash_request *req = ahash_request_cast(dev->async_req);
+	struct crypto_ahash *tfm;
+	struct rk_ahash_rctx *rctx;
+
+	dev->total = req->nbytes;
+	dev->left_bytes = req->nbytes;
+	dev->aligned = 0;
+	dev->align_size = 4;
+	dev->sg_dst = NULL;
+	dev->sg_src = req->src;
+	dev->first = req->src;
+	dev->nents = sg_nents(req->src);
+	rctx = ahash_request_ctx(req);
+	rctx->mode = 0;
+
+	tfm = crypto_ahash_reqtfm(req);
+	switch (crypto_ahash_digestsize(tfm)) {
+	case SHA1_DIGEST_SIZE:
+		rctx->mode = RK_CRYPTO_HASH_SHA1;
+		break;
+	case SHA256_DIGEST_SIZE:
+		rctx->mode = RK_CRYPTO_HASH_SHA256;
+		break;
+	case MD5_DIGEST_SIZE:
+		rctx->mode = RK_CRYPTO_HASH_MD5;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	rk_ahash_reg_init(dev);
 	return rk_ahash_set_data_start(dev);
 }
 
 static int rk_ahash_crypto_rx(struct rk_crypto_info *dev)
 {
 	int err = 0;
+	struct ahash_request *req = ahash_request_cast(dev->async_req);
+	struct crypto_ahash *tfm;
 
 	dev->unload_data(dev);
 	if (dev->left_bytes) {
@@ -264,7 +248,24 @@ static int rk_ahash_crypto_rx(struct rk_crypto_info *dev)
 		}
 		err = rk_ahash_set_data_start(dev);
 	} else {
-		dev->complete(dev, 0);
+		/*
+		 * it will take some time to process date after last dma
+		 * transmission.
+		 *
+		 * waiting time is relative with the last date len,
+		 * so cannot set a fixed time here.
+		 * 10us makes system not call here frequently wasting
+		 * efficiency, and make it response quickly when dma
+		 * complete.
+		 */
+		while (!CRYPTO_READ(dev, RK_CRYPTO_HASH_STS))
+			udelay(10);
+
+		tfm = crypto_ahash_reqtfm(req);
+		memcpy_fromio(req->result, dev->reg + RK_CRYPTO_HASH_DOUT_0,
+			      crypto_ahash_digestsize(tfm));
+		dev->complete(dev->async_req, 0);
+		tasklet_schedule(&dev->queue_task);
 	}
 
 out_rx:
diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c
index 1d9ecd368b5b..08e7bdcaa6e3 100644
--- a/drivers/crypto/sahara.c
+++ b/drivers/crypto/sahara.c
@@ -202,7 +202,6 @@ struct sahara_dev {
 	struct completion	dma_completion;
 
 	struct sahara_ctx	*ctx;
-	spinlock_t		lock;
 	struct crypto_queue	queue;
 	unsigned long		flags;
 
@@ -543,10 +542,10 @@ static int sahara_hw_descriptor_create(struct sahara_dev *dev)
 
 unmap_out:
 	dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg,
-		DMA_TO_DEVICE);
+		DMA_FROM_DEVICE);
 unmap_in:
 	dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
-		DMA_FROM_DEVICE);
+		DMA_TO_DEVICE);
 
 	return -EINVAL;
 }
@@ -594,9 +593,9 @@ static int sahara_aes_process(struct ablkcipher_request *req)
 	}
 
 	dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg,
-		DMA_TO_DEVICE);
-	dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
 		DMA_FROM_DEVICE);
+	dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
+		DMA_TO_DEVICE);
 
 	return 0;
 }
@@ -1376,13 +1375,13 @@ static void sahara_unregister_algs(struct sahara_dev *dev)
 			crypto_unregister_ahash(&sha_v4_algs[i]);
 }
 
-static struct platform_device_id sahara_platform_ids[] = {
+static const struct platform_device_id sahara_platform_ids[] = {
 	{ .name = "sahara-imx27" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(platform, sahara_platform_ids);
 
-static struct of_device_id sahara_dt_ids[] = {
+static const struct of_device_id sahara_dt_ids[] = {
 	{ .compatible = "fsl,imx53-sahara" },
 	{ .compatible = "fsl,imx27-sahara" },
 	{ /* sentinel */ }
@@ -1487,7 +1486,6 @@ static int sahara_probe(struct platform_device *pdev)
 
 	crypto_init_queue(&dev->queue, SAHARA_QUEUE_LENGTH);
 
-	spin_lock_init(&dev->lock);
 	mutex_init(&dev->queue_mutex);
 
 	dev_ptr = dev;
diff --git a/drivers/crypto/stm32/Kconfig b/drivers/crypto/stm32/Kconfig
index 09b4ec87c212..602332e02729 100644
--- a/drivers/crypto/stm32/Kconfig
+++ b/drivers/crypto/stm32/Kconfig
@@ -1,7 +1,20 @@
-config CRYPTO_DEV_STM32
-	tristate "Support for STM32 crypto accelerators"
+config CRC_DEV_STM32
+	tristate "Support for STM32 crc accelerators"
 	depends on ARCH_STM32
 	select CRYPTO_HASH
 	help
           This enables support for the CRC32 hw accelerator which can be found
-	  on STMicroelectronis STM32 SOC.
+	  on STMicroelectronics STM32 SOC.
+
+config HASH_DEV_STM32
+	tristate "Support for STM32 hash accelerators"
+	depends on ARCH_STM32
+	depends on HAS_DMA
+	select CRYPTO_HASH
+	select CRYPTO_MD5
+	select CRYPTO_SHA1
+	select CRYPTO_SHA256
+	select CRYPTO_ENGINE
+	help
+          This enables support for the HASH hw accelerator which can be found
+	  on STMicroelectronics STM32 SOC.
diff --git a/drivers/crypto/stm32/Makefile b/drivers/crypto/stm32/Makefile
index 73b4c6e47f5f..73cd56cad0cc 100644
--- a/drivers/crypto/stm32/Makefile
+++ b/drivers/crypto/stm32/Makefile
@@ -1,2 +1,2 @@
-obj-$(CONFIG_CRYPTO_DEV_STM32) += stm32_cryp.o
-stm32_cryp-objs := stm32_crc32.o
+obj-$(CONFIG_CRC_DEV_STM32) += stm32_crc32.o
+obj-$(CONFIG_HASH_DEV_STM32) += stm32-hash.o
\ No newline at end of file
diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c
new file mode 100644
index 000000000000..b585ce54a802
--- /dev/null
+++ b/drivers/crypto/stm32/stm32-hash.c
@@ -0,0 +1,1575 @@
+/*
+ * This file is part of STM32 Crypto driver for Linux.
+ *
+ * Copyright (C) 2017, STMicroelectronics - All Rights Reserved
+ * Author(s): Lionel DEBIEVE <lionel.debieve@st.com> for STMicroelectronics.
+ *
+ * License terms: GPL V2.0.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include <linux/clk.h>
+#include <linux/crypto.h>
+#include <linux/delay.h>
+#include <linux/dmaengine.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+
+#include <crypto/engine.h>
+#include <crypto/hash.h>
+#include <crypto/md5.h>
+#include <crypto/scatterwalk.h>
+#include <crypto/sha.h>
+#include <crypto/internal/hash.h>
+
+#define HASH_CR				0x00
+#define HASH_DIN			0x04
+#define HASH_STR			0x08
+#define HASH_IMR			0x20
+#define HASH_SR				0x24
+#define HASH_CSR(x)			(0x0F8 + ((x) * 0x04))
+#define HASH_HREG(x)			(0x310 + ((x) * 0x04))
+#define HASH_HWCFGR			0x3F0
+#define HASH_VER			0x3F4
+#define HASH_ID				0x3F8
+
+/* Control Register */
+#define HASH_CR_INIT			BIT(2)
+#define HASH_CR_DMAE			BIT(3)
+#define HASH_CR_DATATYPE_POS		4
+#define HASH_CR_MODE			BIT(6)
+#define HASH_CR_MDMAT			BIT(13)
+#define HASH_CR_DMAA			BIT(14)
+#define HASH_CR_LKEY			BIT(16)
+
+#define HASH_CR_ALGO_SHA1		0x0
+#define HASH_CR_ALGO_MD5		0x80
+#define HASH_CR_ALGO_SHA224		0x40000
+#define HASH_CR_ALGO_SHA256		0x40080
+
+/* Interrupt */
+#define HASH_DINIE			BIT(0)
+#define HASH_DCIE			BIT(1)
+
+/* Interrupt Mask */
+#define HASH_MASK_CALC_COMPLETION	BIT(0)
+#define HASH_MASK_DATA_INPUT		BIT(1)
+
+/* Context swap register */
+#define HASH_CSR_REGISTER_NUMBER	53
+
+/* Status Flags */
+#define HASH_SR_DATA_INPUT_READY	BIT(0)
+#define HASH_SR_OUTPUT_READY		BIT(1)
+#define HASH_SR_DMA_ACTIVE		BIT(2)
+#define HASH_SR_BUSY			BIT(3)
+
+/* STR Register */
+#define HASH_STR_NBLW_MASK		GENMASK(4, 0)
+#define HASH_STR_DCAL			BIT(8)
+
+#define HASH_FLAGS_INIT			BIT(0)
+#define HASH_FLAGS_OUTPUT_READY		BIT(1)
+#define HASH_FLAGS_CPU			BIT(2)
+#define HASH_FLAGS_DMA_READY		BIT(3)
+#define HASH_FLAGS_DMA_ACTIVE		BIT(4)
+#define HASH_FLAGS_HMAC_INIT		BIT(5)
+#define HASH_FLAGS_HMAC_FINAL		BIT(6)
+#define HASH_FLAGS_HMAC_KEY		BIT(7)
+
+#define HASH_FLAGS_FINAL		BIT(15)
+#define HASH_FLAGS_FINUP		BIT(16)
+#define HASH_FLAGS_ALGO_MASK		GENMASK(21, 18)
+#define HASH_FLAGS_MD5			BIT(18)
+#define HASH_FLAGS_SHA1			BIT(19)
+#define HASH_FLAGS_SHA224		BIT(20)
+#define HASH_FLAGS_SHA256		BIT(21)
+#define HASH_FLAGS_ERRORS		BIT(22)
+#define HASH_FLAGS_HMAC			BIT(23)
+
+#define HASH_OP_UPDATE			1
+#define HASH_OP_FINAL			2
+
+enum stm32_hash_data_format {
+	HASH_DATA_32_BITS		= 0x0,
+	HASH_DATA_16_BITS		= 0x1,
+	HASH_DATA_8_BITS		= 0x2,
+	HASH_DATA_1_BIT			= 0x3
+};
+
+#define HASH_BUFLEN			256
+#define HASH_LONG_KEY			64
+#define HASH_MAX_KEY_SIZE		(SHA256_BLOCK_SIZE * 8)
+#define HASH_QUEUE_LENGTH		16
+#define HASH_DMA_THRESHOLD		50
+
+struct stm32_hash_ctx {
+	struct stm32_hash_dev	*hdev;
+	unsigned long		flags;
+
+	u8			key[HASH_MAX_KEY_SIZE];
+	int			keylen;
+};
+
+struct stm32_hash_request_ctx {
+	struct stm32_hash_dev	*hdev;
+	unsigned long		flags;
+	unsigned long		op;
+
+	u8 digest[SHA256_DIGEST_SIZE] __aligned(sizeof(u32));
+	size_t			digcnt;
+	size_t			bufcnt;
+	size_t			buflen;
+
+	/* DMA */
+	struct scatterlist	*sg;
+	unsigned int		offset;
+	unsigned int		total;
+	struct scatterlist	sg_key;
+
+	dma_addr_t		dma_addr;
+	size_t			dma_ct;
+	int			nents;
+
+	u8			data_type;
+
+	u8 buffer[HASH_BUFLEN] __aligned(sizeof(u32));
+
+	/* Export Context */
+	u32			*hw_context;
+};
+
+struct stm32_hash_algs_info {
+	struct ahash_alg	*algs_list;
+	size_t			size;
+};
+
+struct stm32_hash_pdata {
+	struct stm32_hash_algs_info	*algs_info;
+	size_t				algs_info_size;
+};
+
+struct stm32_hash_dev {
+	struct list_head	list;
+	struct device		*dev;
+	struct clk		*clk;
+	struct reset_control	*rst;
+	void __iomem		*io_base;
+	phys_addr_t		phys_base;
+	u32			dma_mode;
+	u32			dma_maxburst;
+
+	spinlock_t		lock; /* lock to protect queue */
+
+	struct ahash_request	*req;
+	struct crypto_engine	*engine;
+
+	int			err;
+	unsigned long		flags;
+
+	struct dma_chan		*dma_lch;
+	struct completion	dma_completion;
+
+	const struct stm32_hash_pdata	*pdata;
+};
+
+struct stm32_hash_drv {
+	struct list_head	dev_list;
+	spinlock_t		lock; /* List protection access */
+};
+
+static struct stm32_hash_drv stm32_hash = {
+	.dev_list = LIST_HEAD_INIT(stm32_hash.dev_list),
+	.lock = __SPIN_LOCK_UNLOCKED(stm32_hash.lock),
+};
+
+static void stm32_hash_dma_callback(void *param);
+
+static inline u32 stm32_hash_read(struct stm32_hash_dev *hdev, u32 offset)
+{
+	return readl_relaxed(hdev->io_base + offset);
+}
+
+static inline void stm32_hash_write(struct stm32_hash_dev *hdev,
+				    u32 offset, u32 value)
+{
+	writel_relaxed(value, hdev->io_base + offset);
+}
+
+static inline int stm32_hash_wait_busy(struct stm32_hash_dev *hdev)
+{
+	u32 status;
+
+	return readl_relaxed_poll_timeout(hdev->io_base + HASH_SR, status,
+				   !(status & HASH_SR_BUSY), 10, 10000);
+}
+
+static void stm32_hash_set_nblw(struct stm32_hash_dev *hdev, int length)
+{
+	u32 reg;
+
+	reg = stm32_hash_read(hdev, HASH_STR);
+	reg &= ~(HASH_STR_NBLW_MASK);
+	reg |= (8U * ((length) % 4U));
+	stm32_hash_write(hdev, HASH_STR, reg);
+}
+
+static int stm32_hash_write_key(struct stm32_hash_dev *hdev)
+{
+	struct crypto_ahash *tfm = crypto_ahash_reqtfm(hdev->req);
+	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+	u32 reg;
+	int keylen = ctx->keylen;
+	void *key = ctx->key;
+
+	if (keylen) {
+		stm32_hash_set_nblw(hdev, keylen);
+
+		while (keylen > 0) {
+			stm32_hash_write(hdev, HASH_DIN, *(u32 *)key);
+			keylen -= 4;
+			key += 4;
+		}
+
+		reg = stm32_hash_read(hdev, HASH_STR);
+		reg |= HASH_STR_DCAL;
+		stm32_hash_write(hdev, HASH_STR, reg);
+
+		return -EINPROGRESS;
+	}
+
+	return 0;
+}
+
+static void stm32_hash_write_ctrl(struct stm32_hash_dev *hdev)
+{
+	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
+	struct crypto_ahash *tfm = crypto_ahash_reqtfm(hdev->req);
+	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+
+	u32 reg = HASH_CR_INIT;
+
+	if (!(hdev->flags & HASH_FLAGS_INIT)) {
+		switch (rctx->flags & HASH_FLAGS_ALGO_MASK) {
+		case HASH_FLAGS_MD5:
+			reg |= HASH_CR_ALGO_MD5;
+			break;
+		case HASH_FLAGS_SHA1:
+			reg |= HASH_CR_ALGO_SHA1;
+			break;
+		case HASH_FLAGS_SHA224:
+			reg |= HASH_CR_ALGO_SHA224;
+			break;
+		case HASH_FLAGS_SHA256:
+			reg |= HASH_CR_ALGO_SHA256;
+			break;
+		default:
+			reg |= HASH_CR_ALGO_MD5;
+		}
+
+		reg |= (rctx->data_type << HASH_CR_DATATYPE_POS);
+
+		if (rctx->flags & HASH_FLAGS_HMAC) {
+			hdev->flags |= HASH_FLAGS_HMAC;
+			reg |= HASH_CR_MODE;
+			if (ctx->keylen > HASH_LONG_KEY)
+				reg |= HASH_CR_LKEY;
+		}
+
+		stm32_hash_write(hdev, HASH_IMR, HASH_DCIE);
+
+		stm32_hash_write(hdev, HASH_CR, reg);
+
+		hdev->flags |= HASH_FLAGS_INIT;
+
+		dev_dbg(hdev->dev, "Write Control %x\n", reg);
+	}
+}
+
+static void stm32_hash_append_sg(struct stm32_hash_request_ctx *rctx)
+{
+	size_t count;
+
+	while ((rctx->bufcnt < rctx->buflen) && rctx->total) {
+		count = min(rctx->sg->length - rctx->offset, rctx->total);
+		count = min(count, rctx->buflen - rctx->bufcnt);
+
+		if (count <= 0) {
+			if ((rctx->sg->length == 0) && !sg_is_last(rctx->sg)) {
+				rctx->sg = sg_next(rctx->sg);
+				continue;
+			} else {
+				break;
+			}
+		}
+
+		scatterwalk_map_and_copy(rctx->buffer + rctx->bufcnt, rctx->sg,
+					 rctx->offset, count, 0);
+
+		rctx->bufcnt += count;
+		rctx->offset += count;
+		rctx->total -= count;
+
+		if (rctx->offset == rctx->sg->length) {
+			rctx->sg = sg_next(rctx->sg);
+			if (rctx->sg)
+				rctx->offset = 0;
+			else
+				rctx->total = 0;
+		}
+	}
+}
+
+static int stm32_hash_xmit_cpu(struct stm32_hash_dev *hdev,
+			       const u8 *buf, size_t length, int final)
+{
+	unsigned int count, len32;
+	const u32 *buffer = (const u32 *)buf;
+	u32 reg;
+
+	if (final)
+		hdev->flags |= HASH_FLAGS_FINAL;
+
+	len32 = DIV_ROUND_UP(length, sizeof(u32));
+
+	dev_dbg(hdev->dev, "%s: length: %d, final: %x len32 %i\n",
+		__func__, length, final, len32);
+
+	hdev->flags |= HASH_FLAGS_CPU;
+
+	stm32_hash_write_ctrl(hdev);
+
+	if (stm32_hash_wait_busy(hdev))
+		return -ETIMEDOUT;
+
+	if ((hdev->flags & HASH_FLAGS_HMAC) &&
+	    (hdev->flags & ~HASH_FLAGS_HMAC_KEY)) {
+		hdev->flags |= HASH_FLAGS_HMAC_KEY;
+		stm32_hash_write_key(hdev);
+		if (stm32_hash_wait_busy(hdev))
+			return -ETIMEDOUT;
+	}
+
+	for (count = 0; count < len32; count++)
+		stm32_hash_write(hdev, HASH_DIN, buffer[count]);
+
+	if (final) {
+		stm32_hash_set_nblw(hdev, length);
+		reg = stm32_hash_read(hdev, HASH_STR);
+		reg |= HASH_STR_DCAL;
+		stm32_hash_write(hdev, HASH_STR, reg);
+		if (hdev->flags & HASH_FLAGS_HMAC) {
+			if (stm32_hash_wait_busy(hdev))
+				return -ETIMEDOUT;
+			stm32_hash_write_key(hdev);
+		}
+		return -EINPROGRESS;
+	}
+
+	return 0;
+}
+
+static int stm32_hash_update_cpu(struct stm32_hash_dev *hdev)
+{
+	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
+	int bufcnt, err = 0, final;
+
+	dev_dbg(hdev->dev, "%s flags %lx\n", __func__, rctx->flags);
+
+	final = (rctx->flags & HASH_FLAGS_FINUP);
+
+	while ((rctx->total >= rctx->buflen) ||
+	       (rctx->bufcnt + rctx->total >= rctx->buflen)) {
+		stm32_hash_append_sg(rctx);
+		bufcnt = rctx->bufcnt;
+		rctx->bufcnt = 0;
+		err = stm32_hash_xmit_cpu(hdev, rctx->buffer, bufcnt, 0);
+	}
+
+	stm32_hash_append_sg(rctx);
+
+	if (final) {
+		bufcnt = rctx->bufcnt;
+		rctx->bufcnt = 0;
+		err = stm32_hash_xmit_cpu(hdev, rctx->buffer, bufcnt,
+					  (rctx->flags & HASH_FLAGS_FINUP));
+	}
+
+	return err;
+}
+
+static int stm32_hash_xmit_dma(struct stm32_hash_dev *hdev,
+			       struct scatterlist *sg, int length, int mdma)
+{
+	struct dma_async_tx_descriptor *in_desc;
+	dma_cookie_t cookie;
+	u32 reg;
+	int err;
+
+	in_desc = dmaengine_prep_slave_sg(hdev->dma_lch, sg, 1,
+					  DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT |
+					  DMA_CTRL_ACK);
+	if (!in_desc) {
+		dev_err(hdev->dev, "dmaengine_prep_slave error\n");
+		return -ENOMEM;
+	}
+
+	reinit_completion(&hdev->dma_completion);
+	in_desc->callback = stm32_hash_dma_callback;
+	in_desc->callback_param = hdev;
+
+	hdev->flags |= HASH_FLAGS_FINAL;
+	hdev->flags |= HASH_FLAGS_DMA_ACTIVE;
+
+	reg = stm32_hash_read(hdev, HASH_CR);
+
+	if (mdma)
+		reg |= HASH_CR_MDMAT;
+	else
+		reg &= ~HASH_CR_MDMAT;
+
+	reg |= HASH_CR_DMAE;
+
+	stm32_hash_write(hdev, HASH_CR, reg);
+
+	stm32_hash_set_nblw(hdev, length);
+
+	cookie = dmaengine_submit(in_desc);
+	err = dma_submit_error(cookie);
+	if (err)
+		return -ENOMEM;
+
+	dma_async_issue_pending(hdev->dma_lch);
+
+	if (!wait_for_completion_interruptible_timeout(&hdev->dma_completion,
+						       msecs_to_jiffies(100)))
+		err = -ETIMEDOUT;
+
+	if (dma_async_is_tx_complete(hdev->dma_lch, cookie,
+				     NULL, NULL) != DMA_COMPLETE)
+		err = -ETIMEDOUT;
+
+	if (err) {
+		dev_err(hdev->dev, "DMA Error %i\n", err);
+		dmaengine_terminate_all(hdev->dma_lch);
+		return err;
+	}
+
+	return -EINPROGRESS;
+}
+
+static void stm32_hash_dma_callback(void *param)
+{
+	struct stm32_hash_dev *hdev = param;
+
+	complete(&hdev->dma_completion);
+
+	hdev->flags |= HASH_FLAGS_DMA_READY;
+}
+
+static int stm32_hash_hmac_dma_send(struct stm32_hash_dev *hdev)
+{
+	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
+	struct crypto_ahash *tfm = crypto_ahash_reqtfm(hdev->req);
+	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+	int err;
+
+	if (ctx->keylen < HASH_DMA_THRESHOLD || (hdev->dma_mode == 1)) {
+		err = stm32_hash_write_key(hdev);
+		if (stm32_hash_wait_busy(hdev))
+			return -ETIMEDOUT;
+	} else {
+		if (!(hdev->flags & HASH_FLAGS_HMAC_KEY))
+			sg_init_one(&rctx->sg_key, ctx->key,
+				    ALIGN(ctx->keylen, sizeof(u32)));
+
+		rctx->dma_ct = dma_map_sg(hdev->dev, &rctx->sg_key, 1,
+					  DMA_TO_DEVICE);
+		if (rctx->dma_ct == 0) {
+			dev_err(hdev->dev, "dma_map_sg error\n");
+			return -ENOMEM;
+		}
+
+		err = stm32_hash_xmit_dma(hdev, &rctx->sg_key, ctx->keylen, 0);
+
+		dma_unmap_sg(hdev->dev, &rctx->sg_key, 1, DMA_TO_DEVICE);
+	}
+
+	return err;
+}
+
+static int stm32_hash_dma_init(struct stm32_hash_dev *hdev)
+{
+	struct dma_slave_config dma_conf;
+	int err;
+
+	memset(&dma_conf, 0, sizeof(dma_conf));
+
+	dma_conf.direction = DMA_MEM_TO_DEV;
+	dma_conf.dst_addr = hdev->phys_base + HASH_DIN;
+	dma_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+	dma_conf.src_maxburst = hdev->dma_maxburst;
+	dma_conf.dst_maxburst = hdev->dma_maxburst;
+	dma_conf.device_fc = false;
+
+	hdev->dma_lch = dma_request_slave_channel(hdev->dev, "in");
+	if (!hdev->dma_lch) {
+		dev_err(hdev->dev, "Couldn't acquire a slave DMA channel.\n");
+		return -EBUSY;
+	}
+
+	err = dmaengine_slave_config(hdev->dma_lch, &dma_conf);
+	if (err) {
+		dma_release_channel(hdev->dma_lch);
+		hdev->dma_lch = NULL;
+		dev_err(hdev->dev, "Couldn't configure DMA slave.\n");
+		return err;
+	}
+
+	init_completion(&hdev->dma_completion);
+
+	return 0;
+}
+
+static int stm32_hash_dma_send(struct stm32_hash_dev *hdev)
+{
+	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req);
+	struct scatterlist sg[1], *tsg;
+	int err = 0, len = 0, reg, ncp;
+	unsigned int i;
+	const u32 *buffer = (const u32 *)rctx->buffer;
+
+	rctx->sg = hdev->req->src;
+	rctx->total = hdev->req->nbytes;
+
+	rctx->nents = sg_nents(rctx->sg);
+
+	if (rctx->nents < 0)
+		return -EINVAL;
+
+	stm32_hash_write_ctrl(hdev);
+
+	if (hdev->flags & HASH_FLAGS_HMAC) {
+		err = stm32_hash_hmac_dma_send(hdev);
+		if (err != -EINPROGRESS)
+			return err;
+	}
+
+	for_each_sg(rctx->sg, tsg, rctx->nents, i) {
+		len = sg->length;
+
+		sg[0] = *tsg;
+		if (sg_is_last(sg)) {
+			if (hdev->dma_mode == 1) {
+				len = (ALIGN(sg->length, 16) - 16);
+
+				ncp = sg_pcopy_to_buffer(
+					rctx->sg, rctx->nents,
+					rctx->buffer, sg->length - len,
+					rctx->total - sg->length + len);
+
+				sg->length = len;
+			} else {
+				if (!(IS_ALIGNED(sg->length, sizeof(u32)))) {
+					len = sg->length;
+					sg->length = ALIGN(sg->length,
+							   sizeof(u32));
+				}
+			}
+		}
+
+		rctx->dma_ct = dma_map_sg(hdev->dev, sg, 1,
+					  DMA_TO_DEVICE);
+		if (rctx->dma_ct == 0) {
+			dev_err(hdev->dev, "dma_map_sg error\n");
+			return -ENOMEM;
+		}
+
+		err = stm32_hash_xmit_dma(hdev, sg, len,
+					  !sg_is_last(sg));
+
+		dma_unmap_sg(hdev->dev, sg, 1, DMA_TO_DEVICE);
+
+		if (err == -ENOMEM)
+			return err;
+	}
+
+	if (hdev->dma_mode == 1) {
+		if (stm32_hash_wait_busy(hdev))
+			return -ETIMEDOUT;
+		reg = stm32_hash_read(hdev, HASH_CR);
+		reg &= ~HASH_CR_DMAE;
+		reg |= HASH_CR_DMAA;
+		stm32_hash_write(hdev, HASH_CR, reg);
+
+		for (i = 0; i < DIV_ROUND_UP(ncp, sizeof(u32)); i++)
+			stm32_hash_write(hdev, HASH_DIN, buffer[i]);
+
+		stm32_hash_set_nblw(hdev, ncp);
+		reg = stm32_hash_read(hdev, HASH_STR);
+		reg |= HASH_STR_DCAL;
+		stm32_hash_write(hdev, HASH_STR, reg);
+		err = -EINPROGRESS;
+	}
+
+	if (hdev->flags & HASH_FLAGS_HMAC) {
+		if (stm32_hash_wait_busy(hdev))
+			return -ETIMEDOUT;
+		err = stm32_hash_hmac_dma_send(hdev);
+	}
+
+	return err;
+}
+
+static struct stm32_hash_dev *stm32_hash_find_dev(struct stm32_hash_ctx *ctx)
+{
+	struct stm32_hash_dev *hdev = NULL, *tmp;
+
+	spin_lock_bh(&stm32_hash.lock);
+	if (!ctx->hdev) {
+		list_for_each_entry(tmp, &stm32_hash.dev_list, list) {
+			hdev = tmp;
+			break;
+		}
+		ctx->hdev = hdev;
+	} else {
+		hdev = ctx->hdev;
+	}
+
+	spin_unlock_bh(&stm32_hash.lock);
+
+	return hdev;
+}
+
+static bool stm32_hash_dma_aligned_data(struct ahash_request *req)
+{
+	struct scatterlist *sg;
+	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
+	struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
+	int i;
+
+	if (req->nbytes <= HASH_DMA_THRESHOLD)
+		return false;
+
+	if (sg_nents(req->src) > 1) {
+		if (hdev->dma_mode == 1)
+			return false;
+		for_each_sg(req->src, sg, sg_nents(req->src), i) {
+			if ((!IS_ALIGNED(sg->length, sizeof(u32))) &&
+			    (!sg_is_last(sg)))
+				return false;
+		}
+	}
+
+	if (req->src->offset % 4)
+		return false;
+
+	return true;
+}
+
+static int stm32_hash_init(struct ahash_request *req)
+{
+	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
+	struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
+
+	rctx->hdev = hdev;
+
+	rctx->flags = HASH_FLAGS_CPU;
+
+	rctx->digcnt = crypto_ahash_digestsize(tfm);
+	switch (rctx->digcnt) {
+	case MD5_DIGEST_SIZE:
+		rctx->flags |= HASH_FLAGS_MD5;
+		break;
+	case SHA1_DIGEST_SIZE:
+		rctx->flags |= HASH_FLAGS_SHA1;
+		break;
+	case SHA224_DIGEST_SIZE:
+		rctx->flags |= HASH_FLAGS_SHA224;
+		break;
+	case SHA256_DIGEST_SIZE:
+		rctx->flags |= HASH_FLAGS_SHA256;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	rctx->bufcnt = 0;
+	rctx->buflen = HASH_BUFLEN;
+	rctx->total = 0;
+	rctx->offset = 0;
+	rctx->data_type = HASH_DATA_8_BITS;
+
+	memset(rctx->buffer, 0, HASH_BUFLEN);
+
+	if (ctx->flags & HASH_FLAGS_HMAC)
+		rctx->flags |= HASH_FLAGS_HMAC;
+
+	dev_dbg(hdev->dev, "%s Flags %lx\n", __func__, rctx->flags);
+
+	return 0;
+}
+
+static int stm32_hash_update_req(struct stm32_hash_dev *hdev)
+{
+	return stm32_hash_update_cpu(hdev);
+}
+
+static int stm32_hash_final_req(struct stm32_hash_dev *hdev)
+{
+	struct ahash_request *req = hdev->req;
+	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
+	int err;
+
+	if (!(rctx->flags & HASH_FLAGS_CPU))
+		err = stm32_hash_dma_send(hdev);
+	else
+		err = stm32_hash_xmit_cpu(hdev, rctx->buffer, rctx->bufcnt, 1);
+
+	rctx->bufcnt = 0;
+
+	return err;
+}
+
+static void stm32_hash_copy_hash(struct ahash_request *req)
+{
+	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
+	u32 *hash = (u32 *)rctx->digest;
+	unsigned int i, hashsize;
+
+	switch (rctx->flags & HASH_FLAGS_ALGO_MASK) {
+	case HASH_FLAGS_MD5:
+		hashsize = MD5_DIGEST_SIZE;
+		break;
+	case HASH_FLAGS_SHA1:
+		hashsize = SHA1_DIGEST_SIZE;
+		break;
+	case HASH_FLAGS_SHA224:
+		hashsize = SHA224_DIGEST_SIZE;
+		break;
+	case HASH_FLAGS_SHA256:
+		hashsize = SHA256_DIGEST_SIZE;
+		break;
+	default:
+		return;
+	}
+
+	for (i = 0; i < hashsize / sizeof(u32); i++)
+		hash[i] = be32_to_cpu(stm32_hash_read(rctx->hdev,
+						      HASH_HREG(i)));
+}
+
+static int stm32_hash_finish(struct ahash_request *req)
+{
+	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
+
+	if (!req->result)
+		return -EINVAL;
+
+	memcpy(req->result, rctx->digest, rctx->digcnt);
+
+	return 0;
+}
+
+static void stm32_hash_finish_req(struct ahash_request *req, int err)
+{
+	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
+	struct stm32_hash_dev *hdev = rctx->hdev;
+
+	if (!err && (HASH_FLAGS_FINAL & hdev->flags)) {
+		stm32_hash_copy_hash(req);
+		err = stm32_hash_finish(req);
+		hdev->flags &= ~(HASH_FLAGS_FINAL | HASH_FLAGS_CPU |
+				 HASH_FLAGS_INIT | HASH_FLAGS_DMA_READY |
+				 HASH_FLAGS_OUTPUT_READY | HASH_FLAGS_HMAC |
+				 HASH_FLAGS_HMAC_INIT | HASH_FLAGS_HMAC_FINAL |
+				 HASH_FLAGS_HMAC_KEY);
+	} else {
+		rctx->flags |= HASH_FLAGS_ERRORS;
+	}
+
+	crypto_finalize_hash_request(hdev->engine, req, err);
+}
+
+static int stm32_hash_hw_init(struct stm32_hash_dev *hdev,
+			      struct stm32_hash_request_ctx *rctx)
+{
+	if (!(HASH_FLAGS_INIT & hdev->flags)) {
+		stm32_hash_write(hdev, HASH_CR, HASH_CR_INIT);
+		stm32_hash_write(hdev, HASH_STR, 0);
+		stm32_hash_write(hdev, HASH_DIN, 0);
+		stm32_hash_write(hdev, HASH_IMR, 0);
+		hdev->err = 0;
+	}
+
+	return 0;
+}
+
+static int stm32_hash_handle_queue(struct stm32_hash_dev *hdev,
+				   struct ahash_request *req)
+{
+	return crypto_transfer_hash_request_to_engine(hdev->engine, req);
+}
+
+static int stm32_hash_prepare_req(struct crypto_engine *engine,
+				  struct ahash_request *req)
+{
+	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
+	struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
+	struct stm32_hash_request_ctx *rctx;
+
+	if (!hdev)
+		return -ENODEV;
+
+	hdev->req = req;
+
+	rctx = ahash_request_ctx(req);
+
+	dev_dbg(hdev->dev, "processing new req, op: %lu, nbytes %d\n",
+		rctx->op, req->nbytes);
+
+	return stm32_hash_hw_init(hdev, rctx);
+}
+
+static int stm32_hash_one_request(struct crypto_engine *engine,
+				  struct ahash_request *req)
+{
+	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
+	struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
+	struct stm32_hash_request_ctx *rctx;
+	int err = 0;
+
+	if (!hdev)
+		return -ENODEV;
+
+	hdev->req = req;
+
+	rctx = ahash_request_ctx(req);
+
+	if (rctx->op == HASH_OP_UPDATE)
+		err = stm32_hash_update_req(hdev);
+	else if (rctx->op == HASH_OP_FINAL)
+		err = stm32_hash_final_req(hdev);
+
+	if (err != -EINPROGRESS)
+	/* done task will not finish it, so do it here */
+		stm32_hash_finish_req(req, err);
+
+	return 0;
+}
+
+static int stm32_hash_enqueue(struct ahash_request *req, unsigned int op)
+{
+	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
+	struct stm32_hash_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+	struct stm32_hash_dev *hdev = ctx->hdev;
+
+	rctx->op = op;
+
+	return stm32_hash_handle_queue(hdev, req);
+}
+
+static int stm32_hash_update(struct ahash_request *req)
+{
+	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
+	int ret;
+
+	if (!req->nbytes || !(rctx->flags & HASH_FLAGS_CPU))
+		return 0;
+
+	rctx->total = req->nbytes;
+	rctx->sg = req->src;
+	rctx->offset = 0;
+
+	if ((rctx->bufcnt + rctx->total < rctx->buflen)) {
+		stm32_hash_append_sg(rctx);
+		return 0;
+	}
+
+	ret = stm32_hash_enqueue(req, HASH_OP_UPDATE);
+
+	if (rctx->flags & HASH_FLAGS_FINUP)
+		return ret;
+
+	return 0;
+}
+
+static int stm32_hash_final(struct ahash_request *req)
+{
+	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
+
+	rctx->flags |= HASH_FLAGS_FINUP;
+
+	return stm32_hash_enqueue(req, HASH_OP_FINAL);
+}
+
+static int stm32_hash_finup(struct ahash_request *req)
+{
+	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
+	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
+	struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
+	int err1, err2;
+
+	rctx->flags |= HASH_FLAGS_FINUP;
+
+	if (hdev->dma_lch && stm32_hash_dma_aligned_data(req))
+		rctx->flags &= ~HASH_FLAGS_CPU;
+
+	err1 = stm32_hash_update(req);
+
+	if (err1 == -EINPROGRESS || err1 == -EBUSY)
+		return err1;
+
+	/*
+	 * final() has to be always called to cleanup resources
+	 * even if update() failed, except EINPROGRESS
+	 */
+	err2 = stm32_hash_final(req);
+
+	return err1 ?: err2;
+}
+
+static int stm32_hash_digest(struct ahash_request *req)
+{
+	return stm32_hash_init(req) ?: stm32_hash_finup(req);
+}
+
+static int stm32_hash_export(struct ahash_request *req, void *out)
+{
+	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
+	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
+	struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
+	u32 *preg;
+	unsigned int i;
+
+	while (!(stm32_hash_read(hdev, HASH_SR) & HASH_SR_DATA_INPUT_READY))
+		cpu_relax();
+
+	rctx->hw_context = kmalloc(sizeof(u32) * (3 + HASH_CSR_REGISTER_NUMBER),
+				   GFP_KERNEL);
+
+	preg = rctx->hw_context;
+
+	*preg++ = stm32_hash_read(hdev, HASH_IMR);
+	*preg++ = stm32_hash_read(hdev, HASH_STR);
+	*preg++ = stm32_hash_read(hdev, HASH_CR);
+	for (i = 0; i < HASH_CSR_REGISTER_NUMBER; i++)
+		*preg++ = stm32_hash_read(hdev, HASH_CSR(i));
+
+	memcpy(out, rctx, sizeof(*rctx));
+
+	return 0;
+}
+
+static int stm32_hash_import(struct ahash_request *req, const void *in)
+{
+	struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
+	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
+	struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx);
+	const u32 *preg = in;
+	u32 reg;
+	unsigned int i;
+
+	memcpy(rctx, in, sizeof(*rctx));
+
+	preg = rctx->hw_context;
+
+	stm32_hash_write(hdev, HASH_IMR, *preg++);
+	stm32_hash_write(hdev, HASH_STR, *preg++);
+	stm32_hash_write(hdev, HASH_CR, *preg);
+	reg = *preg++ | HASH_CR_INIT;
+	stm32_hash_write(hdev, HASH_CR, reg);
+
+	for (i = 0; i < HASH_CSR_REGISTER_NUMBER; i++)
+		stm32_hash_write(hdev, HASH_CSR(i), *preg++);
+
+	kfree(rctx->hw_context);
+
+	return 0;
+}
+
+static int stm32_hash_setkey(struct crypto_ahash *tfm,
+			     const u8 *key, unsigned int keylen)
+{
+	struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+
+	if (keylen <= HASH_MAX_KEY_SIZE) {
+		memcpy(ctx->key, key, keylen);
+		ctx->keylen = keylen;
+	} else {
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static int stm32_hash_cra_init_algs(struct crypto_tfm *tfm,
+				    const char *algs_hmac_name)
+{
+	struct stm32_hash_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+				 sizeof(struct stm32_hash_request_ctx));
+
+	ctx->keylen = 0;
+
+	if (algs_hmac_name)
+		ctx->flags |= HASH_FLAGS_HMAC;
+
+	return 0;
+}
+
+static int stm32_hash_cra_init(struct crypto_tfm *tfm)
+{
+	return stm32_hash_cra_init_algs(tfm, NULL);
+}
+
+static int stm32_hash_cra_md5_init(struct crypto_tfm *tfm)
+{
+	return stm32_hash_cra_init_algs(tfm, "md5");
+}
+
+static int stm32_hash_cra_sha1_init(struct crypto_tfm *tfm)
+{
+	return stm32_hash_cra_init_algs(tfm, "sha1");
+}
+
+static int stm32_hash_cra_sha224_init(struct crypto_tfm *tfm)
+{
+	return stm32_hash_cra_init_algs(tfm, "sha224");
+}
+
+static int stm32_hash_cra_sha256_init(struct crypto_tfm *tfm)
+{
+	return stm32_hash_cra_init_algs(tfm, "sha256");
+}
+
+static irqreturn_t stm32_hash_irq_thread(int irq, void *dev_id)
+{
+	struct stm32_hash_dev *hdev = dev_id;
+	int err;
+
+	if (HASH_FLAGS_CPU & hdev->flags) {
+		if (HASH_FLAGS_OUTPUT_READY & hdev->flags) {
+			hdev->flags &= ~HASH_FLAGS_OUTPUT_READY;
+			goto finish;
+		}
+	} else if (HASH_FLAGS_DMA_READY & hdev->flags) {
+		if (HASH_FLAGS_DMA_ACTIVE & hdev->flags) {
+			hdev->flags &= ~HASH_FLAGS_DMA_ACTIVE;
+				goto finish;
+		}
+	}
+
+	return IRQ_HANDLED;
+
+finish:
+	/*Finish current request */
+	stm32_hash_finish_req(hdev->req, err);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t stm32_hash_irq_handler(int irq, void *dev_id)
+{
+	struct stm32_hash_dev *hdev = dev_id;
+	u32 reg;
+
+	reg = stm32_hash_read(hdev, HASH_SR);
+	if (reg & HASH_SR_OUTPUT_READY) {
+		reg &= ~HASH_SR_OUTPUT_READY;
+		stm32_hash_write(hdev, HASH_SR, reg);
+		hdev->flags |= HASH_FLAGS_OUTPUT_READY;
+		return IRQ_WAKE_THREAD;
+	}
+
+	return IRQ_NONE;
+}
+
+static struct ahash_alg algs_md5_sha1[] = {
+	{
+		.init = stm32_hash_init,
+		.update = stm32_hash_update,
+		.final = stm32_hash_final,
+		.finup = stm32_hash_finup,
+		.digest = stm32_hash_digest,
+		.export = stm32_hash_export,
+		.import = stm32_hash_import,
+		.halg = {
+			.digestsize = MD5_DIGEST_SIZE,
+			.statesize = sizeof(struct stm32_hash_request_ctx),
+			.base = {
+				.cra_name = "md5",
+				.cra_driver_name = "stm32-md5",
+				.cra_priority = 200,
+				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
+					CRYPTO_ALG_ASYNC |
+					CRYPTO_ALG_KERN_DRIVER_ONLY,
+				.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
+				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
+				.cra_alignmask = 3,
+				.cra_init = stm32_hash_cra_init,
+				.cra_module = THIS_MODULE,
+			}
+		}
+	},
+	{
+		.init = stm32_hash_init,
+		.update = stm32_hash_update,
+		.final = stm32_hash_final,
+		.finup = stm32_hash_finup,
+		.digest = stm32_hash_digest,
+		.export = stm32_hash_export,
+		.import = stm32_hash_import,
+		.setkey = stm32_hash_setkey,
+		.halg = {
+			.digestsize = MD5_DIGEST_SIZE,
+			.statesize = sizeof(struct stm32_hash_request_ctx),
+			.base = {
+				.cra_name = "hmac(md5)",
+				.cra_driver_name = "stm32-hmac-md5",
+				.cra_priority = 200,
+				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
+					CRYPTO_ALG_ASYNC |
+					CRYPTO_ALG_KERN_DRIVER_ONLY,
+				.cra_blocksize = MD5_HMAC_BLOCK_SIZE,
+				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
+				.cra_alignmask = 3,
+				.cra_init = stm32_hash_cra_md5_init,
+				.cra_module = THIS_MODULE,
+			}
+		}
+	},
+	{
+		.init = stm32_hash_init,
+		.update = stm32_hash_update,
+		.final = stm32_hash_final,
+		.finup = stm32_hash_finup,
+		.digest = stm32_hash_digest,
+		.export = stm32_hash_export,
+		.import = stm32_hash_import,
+		.halg = {
+			.digestsize = SHA1_DIGEST_SIZE,
+			.statesize = sizeof(struct stm32_hash_request_ctx),
+			.base = {
+				.cra_name = "sha1",
+				.cra_driver_name = "stm32-sha1",
+				.cra_priority = 200,
+				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
+					CRYPTO_ALG_ASYNC |
+					CRYPTO_ALG_KERN_DRIVER_ONLY,
+				.cra_blocksize = SHA1_BLOCK_SIZE,
+				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
+				.cra_alignmask = 3,
+				.cra_init = stm32_hash_cra_init,
+				.cra_module = THIS_MODULE,
+			}
+		}
+	},
+	{
+		.init = stm32_hash_init,
+		.update = stm32_hash_update,
+		.final = stm32_hash_final,
+		.finup = stm32_hash_finup,
+		.digest = stm32_hash_digest,
+		.export = stm32_hash_export,
+		.import = stm32_hash_import,
+		.setkey = stm32_hash_setkey,
+		.halg = {
+			.digestsize = SHA1_DIGEST_SIZE,
+			.statesize = sizeof(struct stm32_hash_request_ctx),
+			.base = {
+				.cra_name = "hmac(sha1)",
+				.cra_driver_name = "stm32-hmac-sha1",
+				.cra_priority = 200,
+				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
+					CRYPTO_ALG_ASYNC |
+					CRYPTO_ALG_KERN_DRIVER_ONLY,
+				.cra_blocksize = SHA1_BLOCK_SIZE,
+				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
+				.cra_alignmask = 3,
+				.cra_init = stm32_hash_cra_sha1_init,
+				.cra_module = THIS_MODULE,
+			}
+		}
+	},
+};
+
+static struct ahash_alg algs_sha224_sha256[] = {
+	{
+		.init = stm32_hash_init,
+		.update = stm32_hash_update,
+		.final = stm32_hash_final,
+		.finup = stm32_hash_finup,
+		.digest = stm32_hash_digest,
+		.export = stm32_hash_export,
+		.import = stm32_hash_import,
+		.halg = {
+			.digestsize = SHA224_DIGEST_SIZE,
+			.statesize = sizeof(struct stm32_hash_request_ctx),
+			.base = {
+				.cra_name = "sha224",
+				.cra_driver_name = "stm32-sha224",
+				.cra_priority = 200,
+				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
+					CRYPTO_ALG_ASYNC |
+					CRYPTO_ALG_KERN_DRIVER_ONLY,
+				.cra_blocksize = SHA224_BLOCK_SIZE,
+				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
+				.cra_alignmask = 3,
+				.cra_init = stm32_hash_cra_init,
+				.cra_module = THIS_MODULE,
+			}
+		}
+	},
+	{
+		.init = stm32_hash_init,
+		.update = stm32_hash_update,
+		.final = stm32_hash_final,
+		.finup = stm32_hash_finup,
+		.digest = stm32_hash_digest,
+		.setkey = stm32_hash_setkey,
+		.export = stm32_hash_export,
+		.import = stm32_hash_import,
+		.halg = {
+			.digestsize = SHA224_DIGEST_SIZE,
+			.statesize = sizeof(struct stm32_hash_request_ctx),
+			.base = {
+				.cra_name = "hmac(sha224)",
+				.cra_driver_name = "stm32-hmac-sha224",
+				.cra_priority = 200,
+				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
+					CRYPTO_ALG_ASYNC |
+					CRYPTO_ALG_KERN_DRIVER_ONLY,
+				.cra_blocksize = SHA224_BLOCK_SIZE,
+				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
+				.cra_alignmask = 3,
+				.cra_init = stm32_hash_cra_sha224_init,
+				.cra_module = THIS_MODULE,
+			}
+		}
+	},
+	{
+		.init = stm32_hash_init,
+		.update = stm32_hash_update,
+		.final = stm32_hash_final,
+		.finup = stm32_hash_finup,
+		.digest = stm32_hash_digest,
+		.export = stm32_hash_export,
+		.import = stm32_hash_import,
+		.halg = {
+			.digestsize = SHA256_DIGEST_SIZE,
+			.statesize = sizeof(struct stm32_hash_request_ctx),
+			.base = {
+				.cra_name = "sha256",
+				.cra_driver_name = "stm32-sha256",
+				.cra_priority = 200,
+				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
+					CRYPTO_ALG_ASYNC |
+					CRYPTO_ALG_KERN_DRIVER_ONLY,
+				.cra_blocksize = SHA256_BLOCK_SIZE,
+				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
+				.cra_alignmask = 3,
+				.cra_init = stm32_hash_cra_init,
+				.cra_module = THIS_MODULE,
+			}
+		}
+	},
+	{
+		.init = stm32_hash_init,
+		.update = stm32_hash_update,
+		.final = stm32_hash_final,
+		.finup = stm32_hash_finup,
+		.digest = stm32_hash_digest,
+		.export = stm32_hash_export,
+		.import = stm32_hash_import,
+		.setkey = stm32_hash_setkey,
+		.halg = {
+			.digestsize = SHA256_DIGEST_SIZE,
+			.statesize = sizeof(struct stm32_hash_request_ctx),
+			.base = {
+				.cra_name = "hmac(sha256)",
+				.cra_driver_name = "stm32-hmac-sha256",
+				.cra_priority = 200,
+				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
+					CRYPTO_ALG_ASYNC |
+					CRYPTO_ALG_KERN_DRIVER_ONLY,
+				.cra_blocksize = SHA256_BLOCK_SIZE,
+				.cra_ctxsize = sizeof(struct stm32_hash_ctx),
+				.cra_alignmask = 3,
+				.cra_init = stm32_hash_cra_sha256_init,
+				.cra_module = THIS_MODULE,
+			}
+		}
+	},
+};
+
+static int stm32_hash_register_algs(struct stm32_hash_dev *hdev)
+{
+	unsigned int i, j;
+	int err;
+
+	for (i = 0; i < hdev->pdata->algs_info_size; i++) {
+		for (j = 0; j < hdev->pdata->algs_info[i].size; j++) {
+			err = crypto_register_ahash(
+				&hdev->pdata->algs_info[i].algs_list[j]);
+			if (err)
+				goto err_algs;
+		}
+	}
+
+	return 0;
+err_algs:
+	dev_err(hdev->dev, "Algo %d : %d failed\n", i, j);
+	for (; i--; ) {
+		for (; j--;)
+			crypto_unregister_ahash(
+				&hdev->pdata->algs_info[i].algs_list[j]);
+	}
+
+	return err;
+}
+
+static int stm32_hash_unregister_algs(struct stm32_hash_dev *hdev)
+{
+	unsigned int i, j;
+
+	for (i = 0; i < hdev->pdata->algs_info_size; i++) {
+		for (j = 0; j < hdev->pdata->algs_info[i].size; j++)
+			crypto_unregister_ahash(
+				&hdev->pdata->algs_info[i].algs_list[j]);
+	}
+
+	return 0;
+}
+
+static struct stm32_hash_algs_info stm32_hash_algs_info_stm32f4[] = {
+	{
+		.algs_list	= algs_md5_sha1,
+		.size		= ARRAY_SIZE(algs_md5_sha1),
+	},
+};
+
+static const struct stm32_hash_pdata stm32_hash_pdata_stm32f4 = {
+	.algs_info	= stm32_hash_algs_info_stm32f4,
+	.algs_info_size	= ARRAY_SIZE(stm32_hash_algs_info_stm32f4),
+};
+
+static struct stm32_hash_algs_info stm32_hash_algs_info_stm32f7[] = {
+	{
+		.algs_list	= algs_md5_sha1,
+		.size		= ARRAY_SIZE(algs_md5_sha1),
+	},
+	{
+		.algs_list	= algs_sha224_sha256,
+		.size		= ARRAY_SIZE(algs_sha224_sha256),
+	},
+};
+
+static const struct stm32_hash_pdata stm32_hash_pdata_stm32f7 = {
+	.algs_info	= stm32_hash_algs_info_stm32f7,
+	.algs_info_size	= ARRAY_SIZE(stm32_hash_algs_info_stm32f7),
+};
+
+static const struct of_device_id stm32_hash_of_match[] = {
+	{
+		.compatible = "st,stm32f456-hash",
+		.data = &stm32_hash_pdata_stm32f4,
+	},
+	{
+		.compatible = "st,stm32f756-hash",
+		.data = &stm32_hash_pdata_stm32f7,
+	},
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, stm32_hash_of_match);
+
+static int stm32_hash_get_of_match(struct stm32_hash_dev *hdev,
+				   struct device *dev)
+{
+	const struct of_device_id *match;
+	int err;
+
+	match = of_match_device(stm32_hash_of_match, dev);
+	if (!match) {
+		dev_err(dev, "no compatible OF match\n");
+		return -EINVAL;
+	}
+
+	err = of_property_read_u32(dev->of_node, "dma-maxburst",
+				   &hdev->dma_maxburst);
+
+	hdev->pdata = match->data;
+
+	return err;
+}
+
+static int stm32_hash_probe(struct platform_device *pdev)
+{
+	struct stm32_hash_dev *hdev;
+	struct device *dev = &pdev->dev;
+	struct resource *res;
+	int ret, irq;
+
+	hdev = devm_kzalloc(dev, sizeof(*hdev), GFP_KERNEL);
+	if (!hdev)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	hdev->io_base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(hdev->io_base))
+		return PTR_ERR(hdev->io_base);
+
+	hdev->phys_base = res->start;
+
+	ret = stm32_hash_get_of_match(hdev, dev);
+	if (ret)
+		return ret;
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		dev_err(dev, "Cannot get IRQ resource\n");
+		return irq;
+	}
+
+	ret = devm_request_threaded_irq(dev, irq, stm32_hash_irq_handler,
+					stm32_hash_irq_thread, IRQF_ONESHOT,
+					dev_name(dev), hdev);
+	if (ret) {
+		dev_err(dev, "Cannot grab IRQ\n");
+		return ret;
+	}
+
+	hdev->clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(hdev->clk)) {
+		dev_err(dev, "failed to get clock for hash (%lu)\n",
+			PTR_ERR(hdev->clk));
+		return PTR_ERR(hdev->clk);
+	}
+
+	ret = clk_prepare_enable(hdev->clk);
+	if (ret) {
+		dev_err(dev, "failed to enable hash clock (%d)\n", ret);
+		return ret;
+	}
+
+	hdev->rst = devm_reset_control_get(&pdev->dev, NULL);
+	if (!IS_ERR(hdev->rst)) {
+		reset_control_assert(hdev->rst);
+		udelay(2);
+		reset_control_deassert(hdev->rst);
+	}
+
+	hdev->dev = dev;
+
+	platform_set_drvdata(pdev, hdev);
+
+	ret = stm32_hash_dma_init(hdev);
+	if (ret)
+		dev_dbg(dev, "DMA mode not available\n");
+
+	spin_lock(&stm32_hash.lock);
+	list_add_tail(&hdev->list, &stm32_hash.dev_list);
+	spin_unlock(&stm32_hash.lock);
+
+	/* Initialize crypto engine */
+	hdev->engine = crypto_engine_alloc_init(dev, 1);
+	if (!hdev->engine) {
+		ret = -ENOMEM;
+		goto err_engine;
+	}
+
+	hdev->engine->prepare_hash_request = stm32_hash_prepare_req;
+	hdev->engine->hash_one_request = stm32_hash_one_request;
+
+	ret = crypto_engine_start(hdev->engine);
+	if (ret)
+		goto err_engine_start;
+
+	hdev->dma_mode = stm32_hash_read(hdev, HASH_HWCFGR);
+
+	/* Register algos */
+	ret = stm32_hash_register_algs(hdev);
+	if (ret)
+		goto err_algs;
+
+	dev_info(dev, "Init HASH done HW ver %x DMA mode %u\n",
+		 stm32_hash_read(hdev, HASH_VER), hdev->dma_mode);
+
+	return 0;
+
+err_algs:
+err_engine_start:
+	crypto_engine_exit(hdev->engine);
+err_engine:
+	spin_lock(&stm32_hash.lock);
+	list_del(&hdev->list);
+	spin_unlock(&stm32_hash.lock);
+
+	if (hdev->dma_lch)
+		dma_release_channel(hdev->dma_lch);
+
+	clk_disable_unprepare(hdev->clk);
+
+	return ret;
+}
+
+static int stm32_hash_remove(struct platform_device *pdev)
+{
+	static struct stm32_hash_dev *hdev;
+
+	hdev = platform_get_drvdata(pdev);
+	if (!hdev)
+		return -ENODEV;
+
+	stm32_hash_unregister_algs(hdev);
+
+	crypto_engine_exit(hdev->engine);
+
+	spin_lock(&stm32_hash.lock);
+	list_del(&hdev->list);
+	spin_unlock(&stm32_hash.lock);
+
+	if (hdev->dma_lch)
+		dma_release_channel(hdev->dma_lch);
+
+	clk_disable_unprepare(hdev->clk);
+
+	return 0;
+}
+
+static struct platform_driver stm32_hash_driver = {
+	.probe		= stm32_hash_probe,
+	.remove		= stm32_hash_remove,
+	.driver		= {
+		.name	= "stm32-hash",
+		.of_match_table	= stm32_hash_of_match,
+	}
+};
+
+module_platform_driver(stm32_hash_driver);
+
+MODULE_DESCRIPTION("STM32 SHA1/224/256 & MD5 (HMAC) hw accelerator driver");
+MODULE_AUTHOR("Lionel Debieve <lionel.debieve@st.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/crypto/stm32/stm32_crc32.c b/drivers/crypto/stm32/stm32_crc32.c
index ec83b1e6bfe8..090582baecfe 100644
--- a/drivers/crypto/stm32/stm32_crc32.c
+++ b/drivers/crypto/stm32/stm32_crc32.c
@@ -107,12 +107,12 @@ static int stm32_crc_init(struct shash_desc *desc)
 	spin_unlock_bh(&crc_list.lock);
 
 	/* Reset, set key, poly and configure in bit reverse mode */
-	writel(bitrev32(mctx->key), ctx->crc->regs + CRC_INIT);
-	writel(bitrev32(mctx->poly), ctx->crc->regs + CRC_POL);
-	writel(CRC_CR_RESET | CRC_CR_REVERSE, ctx->crc->regs + CRC_CR);
+	writel_relaxed(bitrev32(mctx->key), ctx->crc->regs + CRC_INIT);
+	writel_relaxed(bitrev32(mctx->poly), ctx->crc->regs + CRC_POL);
+	writel_relaxed(CRC_CR_RESET | CRC_CR_REVERSE, ctx->crc->regs + CRC_CR);
 
 	/* Store partial result */
-	ctx->partial = readl(ctx->crc->regs + CRC_DR);
+	ctx->partial = readl_relaxed(ctx->crc->regs + CRC_DR);
 	ctx->crc->nb_pending_bytes = 0;
 
 	return 0;
@@ -135,7 +135,8 @@ static int stm32_crc_update(struct shash_desc *desc, const u8 *d8,
 
 		if (crc->nb_pending_bytes == sizeof(u32)) {
 			/* Process completed pending data */
-			writel(*(u32 *)crc->pending_data, crc->regs + CRC_DR);
+			writel_relaxed(*(u32 *)crc->pending_data,
+				       crc->regs + CRC_DR);
 			crc->nb_pending_bytes = 0;
 		}
 	}
@@ -143,10 +144,10 @@ static int stm32_crc_update(struct shash_desc *desc, const u8 *d8,
 	d32 = (u32 *)d8;
 	for (i = 0; i < length >> 2; i++)
 		/* Process 32 bits data */
-		writel(*(d32++), crc->regs + CRC_DR);
+		writel_relaxed(*(d32++), crc->regs + CRC_DR);
 
 	/* Store partial result */
-	ctx->partial = readl(crc->regs + CRC_DR);
+	ctx->partial = readl_relaxed(crc->regs + CRC_DR);
 
 	/* Check for pending data (non 32 bits) */
 	length &= 3;
@@ -295,7 +296,7 @@ static int stm32_crc_remove(struct platform_device *pdev)
 	list_del(&crc->list);
 	spin_unlock(&crc_list.lock);
 
-	crypto_unregister_shash(algs);
+	crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
 
 	clk_disable_unprepare(crc->clk);
 
diff --git a/drivers/crypto/sunxi-ss/Makefile b/drivers/crypto/sunxi-ss/Makefile
index 8f4c7a273141..ccb893219079 100644
--- a/drivers/crypto/sunxi-ss/Makefile
+++ b/drivers/crypto/sunxi-ss/Makefile
@@ -1,2 +1,3 @@
 obj-$(CONFIG_CRYPTO_DEV_SUN4I_SS) += sun4i-ss.o
 sun4i-ss-y += sun4i-ss-core.o sun4i-ss-hash.o sun4i-ss-cipher.o
+sun4i-ss-$(CONFIG_CRYPTO_DEV_SUN4I_SS_PRNG) += sun4i-ss-prng.o
diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-core.c b/drivers/crypto/sunxi-ss/sun4i-ss-core.c
index 02ad8256e900..1547cbe13dc2 100644
--- a/drivers/crypto/sunxi-ss/sun4i-ss-core.c
+++ b/drivers/crypto/sunxi-ss/sun4i-ss-core.c
@@ -213,6 +213,23 @@ static struct sun4i_ss_alg_template ss_algs[] = {
 		}
 	}
 },
+#ifdef CONFIG_CRYPTO_DEV_SUN4I_SS_PRNG
+{
+	.type = CRYPTO_ALG_TYPE_RNG,
+	.alg.rng = {
+		.base = {
+			.cra_name		= "stdrng",
+			.cra_driver_name	= "sun4i_ss_rng",
+			.cra_priority		= 300,
+			.cra_ctxsize		= 0,
+			.cra_module		= THIS_MODULE,
+		},
+		.generate               = sun4i_ss_prng_generate,
+		.seed                   = sun4i_ss_prng_seed,
+		.seedsize               = SS_SEED_LEN / BITS_PER_BYTE,
+	}
+},
+#endif
 };
 
 static int sun4i_ss_probe(struct platform_device *pdev)
@@ -355,6 +372,13 @@ static int sun4i_ss_probe(struct platform_device *pdev)
 				goto error_alg;
 			}
 			break;
+		case CRYPTO_ALG_TYPE_RNG:
+			err = crypto_register_rng(&ss_algs[i].alg.rng);
+			if (err) {
+				dev_err(ss->dev, "Fail to register %s\n",
+					ss_algs[i].alg.rng.base.cra_name);
+			}
+			break;
 		}
 	}
 	platform_set_drvdata(pdev, ss);
@@ -369,6 +393,9 @@ error_alg:
 		case CRYPTO_ALG_TYPE_AHASH:
 			crypto_unregister_ahash(&ss_algs[i].alg.hash);
 			break;
+		case CRYPTO_ALG_TYPE_RNG:
+			crypto_unregister_rng(&ss_algs[i].alg.rng);
+			break;
 		}
 	}
 	if (ss->reset)
@@ -393,6 +420,9 @@ static int sun4i_ss_remove(struct platform_device *pdev)
 		case CRYPTO_ALG_TYPE_AHASH:
 			crypto_unregister_ahash(&ss_algs[i].alg.hash);
 			break;
+		case CRYPTO_ALG_TYPE_RNG:
+			crypto_unregister_rng(&ss_algs[i].alg.rng);
+			break;
 		}
 	}
 
diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-prng.c b/drivers/crypto/sunxi-ss/sun4i-ss-prng.c
new file mode 100644
index 000000000000..0d01d1624252
--- /dev/null
+++ b/drivers/crypto/sunxi-ss/sun4i-ss-prng.c
@@ -0,0 +1,56 @@
+#include "sun4i-ss.h"
+
+int sun4i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed,
+		       unsigned int slen)
+{
+	struct sun4i_ss_alg_template *algt;
+	struct rng_alg *alg = crypto_rng_alg(tfm);
+
+	algt = container_of(alg, struct sun4i_ss_alg_template, alg.rng);
+	memcpy(algt->ss->seed, seed, slen);
+
+	return 0;
+}
+
+int sun4i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src,
+			   unsigned int slen, u8 *dst, unsigned int dlen)
+{
+	struct sun4i_ss_alg_template *algt;
+	struct rng_alg *alg = crypto_rng_alg(tfm);
+	int i;
+	u32 v;
+	u32 *data = (u32 *)dst;
+	const u32 mode = SS_OP_PRNG | SS_PRNG_CONTINUE | SS_ENABLED;
+	size_t len;
+	struct sun4i_ss_ctx *ss;
+	unsigned int todo = (dlen / 4) * 4;
+
+	algt = container_of(alg, struct sun4i_ss_alg_template, alg.rng);
+	ss = algt->ss;
+
+	spin_lock(&ss->slock);
+
+	writel(mode, ss->base + SS_CTL);
+
+	while (todo > 0) {
+		/* write the seed */
+		for (i = 0; i < SS_SEED_LEN / BITS_PER_LONG; i++)
+			writel(ss->seed[i], ss->base + SS_KEY0 + i * 4);
+
+		/* Read the random data */
+		len = min_t(size_t, SS_DATA_LEN / BITS_PER_BYTE, todo);
+		readsl(ss->base + SS_TXFIFO, data, len / 4);
+		data += len / 4;
+		todo -= len;
+
+		/* Update the seed */
+		for (i = 0; i < SS_SEED_LEN / BITS_PER_LONG; i++) {
+			v = readl(ss->base + SS_KEY0 + i * 4);
+			ss->seed[i] = v;
+		}
+	}
+
+	writel(0, ss->base + SS_CTL);
+	spin_unlock(&ss->slock);
+	return dlen;
+}
diff --git a/drivers/crypto/sunxi-ss/sun4i-ss.h b/drivers/crypto/sunxi-ss/sun4i-ss.h
index a0e1efc1cb2a..f3ac90692ac6 100644
--- a/drivers/crypto/sunxi-ss/sun4i-ss.h
+++ b/drivers/crypto/sunxi-ss/sun4i-ss.h
@@ -32,6 +32,7 @@
 #include <crypto/aes.h>
 #include <crypto/des.h>
 #include <crypto/internal/rng.h>
+#include <crypto/rng.h>
 
 #define SS_CTL            0x00
 #define SS_KEY0           0x04
@@ -127,6 +128,9 @@
 #define SS_RXFIFO_EMP_INT_ENABLE	(1 << 2)
 #define SS_TXFIFO_AVA_INT_ENABLE	(1 << 0)
 
+#define SS_SEED_LEN 192
+#define SS_DATA_LEN 160
+
 struct sun4i_ss_ctx {
 	void __iomem *base;
 	int irq;
@@ -136,6 +140,9 @@ struct sun4i_ss_ctx {
 	struct device *dev;
 	struct resource *res;
 	spinlock_t slock; /* control the use of the device */
+#ifdef CONFIG_CRYPTO_DEV_SUN4I_SS_PRNG
+	u32 seed[SS_SEED_LEN / BITS_PER_LONG];
+#endif
 };
 
 struct sun4i_ss_alg_template {
@@ -144,6 +151,7 @@ struct sun4i_ss_alg_template {
 	union {
 		struct skcipher_alg crypto;
 		struct ahash_alg hash;
+		struct rng_alg rng;
 	} alg;
 	struct sun4i_ss_ctx *ss;
 };
@@ -201,3 +209,6 @@ int sun4i_ss_des_setkey(struct crypto_skcipher *tfm, const u8 *key,
 			unsigned int keylen);
 int sun4i_ss_des3_setkey(struct crypto_skcipher *tfm, const u8 *key,
 			 unsigned int keylen);
+int sun4i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src,
+			   unsigned int slen, u8 *dst, unsigned int dlen);
+int sun4i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed, unsigned int slen);
diff --git a/drivers/crypto/virtio/virtio_crypto_algs.c b/drivers/crypto/virtio/virtio_crypto_algs.c
index 49defda4e03d..5035b0dc1e40 100644
--- a/drivers/crypto/virtio/virtio_crypto_algs.c
+++ b/drivers/crypto/virtio/virtio_crypto_algs.c
@@ -27,12 +27,68 @@
 #include <uapi/linux/virtio_crypto.h>
 #include "virtio_crypto_common.h"
 
+
+struct virtio_crypto_ablkcipher_ctx {
+	struct virtio_crypto *vcrypto;
+	struct crypto_tfm *tfm;
+
+	struct virtio_crypto_sym_session_info enc_sess_info;
+	struct virtio_crypto_sym_session_info dec_sess_info;
+};
+
+struct virtio_crypto_sym_request {
+	struct virtio_crypto_request base;
+
+	/* Cipher or aead */
+	uint32_t type;
+	struct virtio_crypto_ablkcipher_ctx *ablkcipher_ctx;
+	struct ablkcipher_request *ablkcipher_req;
+	uint8_t *iv;
+	/* Encryption? */
+	bool encrypt;
+};
+
 /*
  * The algs_lock protects the below global virtio_crypto_active_devs
  * and crypto algorithms registion.
  */
 static DEFINE_MUTEX(algs_lock);
 static unsigned int virtio_crypto_active_devs;
+static void virtio_crypto_ablkcipher_finalize_req(
+	struct virtio_crypto_sym_request *vc_sym_req,
+	struct ablkcipher_request *req,
+	int err);
+
+static void virtio_crypto_dataq_sym_callback
+		(struct virtio_crypto_request *vc_req, int len)
+{
+	struct virtio_crypto_sym_request *vc_sym_req =
+		container_of(vc_req, struct virtio_crypto_sym_request, base);
+	struct ablkcipher_request *ablk_req;
+	int error;
+
+	/* Finish the encrypt or decrypt process */
+	if (vc_sym_req->type == VIRTIO_CRYPTO_SYM_OP_CIPHER) {
+		switch (vc_req->status) {
+		case VIRTIO_CRYPTO_OK:
+			error = 0;
+			break;
+		case VIRTIO_CRYPTO_INVSESS:
+		case VIRTIO_CRYPTO_ERR:
+			error = -EINVAL;
+			break;
+		case VIRTIO_CRYPTO_BADMSG:
+			error = -EBADMSG;
+			break;
+		default:
+			error = -EIO;
+			break;
+		}
+		ablk_req = vc_sym_req->ablkcipher_req;
+		virtio_crypto_ablkcipher_finalize_req(vc_sym_req,
+							ablk_req, error);
+	}
+}
 
 static u64 virtio_crypto_alg_sg_nents_length(struct scatterlist *sg)
 {
@@ -286,13 +342,14 @@ static int virtio_crypto_ablkcipher_setkey(struct crypto_ablkcipher *tfm,
 }
 
 static int
-__virtio_crypto_ablkcipher_do_req(struct virtio_crypto_request *vc_req,
+__virtio_crypto_ablkcipher_do_req(struct virtio_crypto_sym_request *vc_sym_req,
 		struct ablkcipher_request *req,
 		struct data_queue *data_vq)
 {
 	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
+	struct virtio_crypto_ablkcipher_ctx *ctx = vc_sym_req->ablkcipher_ctx;
+	struct virtio_crypto_request *vc_req = &vc_sym_req->base;
 	unsigned int ivsize = crypto_ablkcipher_ivsize(tfm);
-	struct virtio_crypto_ablkcipher_ctx *ctx = vc_req->ablkcipher_ctx;
 	struct virtio_crypto *vcrypto = ctx->vcrypto;
 	struct virtio_crypto_op_data_req *req_data;
 	int src_nents, dst_nents;
@@ -326,9 +383,9 @@ __virtio_crypto_ablkcipher_do_req(struct virtio_crypto_request *vc_req,
 	}
 
 	vc_req->req_data = req_data;
-	vc_req->type = VIRTIO_CRYPTO_SYM_OP_CIPHER;
+	vc_sym_req->type = VIRTIO_CRYPTO_SYM_OP_CIPHER;
 	/* Head of operation */
-	if (vc_req->encrypt) {
+	if (vc_sym_req->encrypt) {
 		req_data->header.session_id =
 			cpu_to_le64(ctx->enc_sess_info.session_id);
 		req_data->header.opcode =
@@ -383,7 +440,7 @@ __virtio_crypto_ablkcipher_do_req(struct virtio_crypto_request *vc_req,
 	memcpy(iv, req->info, ivsize);
 	sg_init_one(&iv_sg, iv, ivsize);
 	sgs[num_out++] = &iv_sg;
-	vc_req->iv = iv;
+	vc_sym_req->iv = iv;
 
 	/* Source data */
 	for (i = 0; i < src_nents; i++)
@@ -421,15 +478,18 @@ static int virtio_crypto_ablkcipher_encrypt(struct ablkcipher_request *req)
 {
 	struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
 	struct virtio_crypto_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(atfm);
-	struct virtio_crypto_request *vc_req = ablkcipher_request_ctx(req);
+	struct virtio_crypto_sym_request *vc_sym_req =
+				ablkcipher_request_ctx(req);
+	struct virtio_crypto_request *vc_req = &vc_sym_req->base;
 	struct virtio_crypto *vcrypto = ctx->vcrypto;
 	/* Use the first data virtqueue as default */
 	struct data_queue *data_vq = &vcrypto->data_vq[0];
 
-	vc_req->ablkcipher_ctx = ctx;
-	vc_req->ablkcipher_req = req;
-	vc_req->encrypt = true;
 	vc_req->dataq = data_vq;
+	vc_req->alg_cb = virtio_crypto_dataq_sym_callback;
+	vc_sym_req->ablkcipher_ctx = ctx;
+	vc_sym_req->ablkcipher_req = req;
+	vc_sym_req->encrypt = true;
 
 	return crypto_transfer_cipher_request_to_engine(data_vq->engine, req);
 }
@@ -438,16 +498,18 @@ static int virtio_crypto_ablkcipher_decrypt(struct ablkcipher_request *req)
 {
 	struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
 	struct virtio_crypto_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(atfm);
-	struct virtio_crypto_request *vc_req = ablkcipher_request_ctx(req);
+	struct virtio_crypto_sym_request *vc_sym_req =
+				ablkcipher_request_ctx(req);
+	struct virtio_crypto_request *vc_req = &vc_sym_req->base;
 	struct virtio_crypto *vcrypto = ctx->vcrypto;
 	/* Use the first data virtqueue as default */
 	struct data_queue *data_vq = &vcrypto->data_vq[0];
 
-	vc_req->ablkcipher_ctx = ctx;
-	vc_req->ablkcipher_req = req;
-
-	vc_req->encrypt = false;
 	vc_req->dataq = data_vq;
+	vc_req->alg_cb = virtio_crypto_dataq_sym_callback;
+	vc_sym_req->ablkcipher_ctx = ctx;
+	vc_sym_req->ablkcipher_req = req;
+	vc_sym_req->encrypt = false;
 
 	return crypto_transfer_cipher_request_to_engine(data_vq->engine, req);
 }
@@ -456,7 +518,7 @@ static int virtio_crypto_ablkcipher_init(struct crypto_tfm *tfm)
 {
 	struct virtio_crypto_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
 
-	tfm->crt_ablkcipher.reqsize = sizeof(struct virtio_crypto_request);
+	tfm->crt_ablkcipher.reqsize = sizeof(struct virtio_crypto_sym_request);
 	ctx->tfm = tfm;
 
 	return 0;
@@ -479,11 +541,13 @@ int virtio_crypto_ablkcipher_crypt_req(
 	struct crypto_engine *engine,
 	struct ablkcipher_request *req)
 {
-	struct virtio_crypto_request *vc_req = ablkcipher_request_ctx(req);
+	struct virtio_crypto_sym_request *vc_sym_req =
+				ablkcipher_request_ctx(req);
+	struct virtio_crypto_request *vc_req = &vc_sym_req->base;
 	struct data_queue *data_vq = vc_req->dataq;
 	int ret;
 
-	ret = __virtio_crypto_ablkcipher_do_req(vc_req, req, data_vq);
+	ret = __virtio_crypto_ablkcipher_do_req(vc_sym_req, req, data_vq);
 	if (ret < 0)
 		return ret;
 
@@ -492,14 +556,15 @@ int virtio_crypto_ablkcipher_crypt_req(
 	return 0;
 }
 
-void virtio_crypto_ablkcipher_finalize_req(
-	struct virtio_crypto_request *vc_req,
+static void virtio_crypto_ablkcipher_finalize_req(
+	struct virtio_crypto_sym_request *vc_sym_req,
 	struct ablkcipher_request *req,
 	int err)
 {
-	crypto_finalize_cipher_request(vc_req->dataq->engine, req, err);
-
-	virtcrypto_clear_request(vc_req);
+	crypto_finalize_cipher_request(vc_sym_req->base.dataq->engine,
+					req, err);
+	kzfree(vc_sym_req->iv);
+	virtcrypto_clear_request(&vc_sym_req->base);
 }
 
 static struct crypto_alg virtio_crypto_algs[] = { {
diff --git a/drivers/crypto/virtio/virtio_crypto_common.h b/drivers/crypto/virtio/virtio_crypto_common.h
index da6d8c0ea407..e976539a05d9 100644
--- a/drivers/crypto/virtio/virtio_crypto_common.h
+++ b/drivers/crypto/virtio/virtio_crypto_common.h
@@ -83,26 +83,16 @@ struct virtio_crypto_sym_session_info {
 	__u64 session_id;
 };
 
-struct virtio_crypto_ablkcipher_ctx {
-	struct virtio_crypto *vcrypto;
-	struct crypto_tfm *tfm;
-
-	struct virtio_crypto_sym_session_info enc_sess_info;
-	struct virtio_crypto_sym_session_info dec_sess_info;
-};
+struct virtio_crypto_request;
+typedef void (*virtio_crypto_data_callback)
+		(struct virtio_crypto_request *vc_req, int len);
 
 struct virtio_crypto_request {
-	/* Cipher or aead */
-	uint32_t type;
 	uint8_t status;
-	struct virtio_crypto_ablkcipher_ctx *ablkcipher_ctx;
-	struct ablkcipher_request *ablkcipher_req;
 	struct virtio_crypto_op_data_req *req_data;
 	struct scatterlist **sgs;
-	uint8_t *iv;
-	/* Encryption? */
-	bool encrypt;
 	struct data_queue *dataq;
+	virtio_crypto_data_callback alg_cb;
 };
 
 int virtcrypto_devmgr_add_dev(struct virtio_crypto *vcrypto_dev);
@@ -119,10 +109,6 @@ void virtcrypto_dev_stop(struct virtio_crypto *vcrypto);
 int virtio_crypto_ablkcipher_crypt_req(
 	struct crypto_engine *engine,
 	struct ablkcipher_request *req);
-void virtio_crypto_ablkcipher_finalize_req(
-	struct virtio_crypto_request *vc_req,
-	struct ablkcipher_request *req,
-	int err);
 
 void
 virtcrypto_clear_request(struct virtio_crypto_request *vc_req);
diff --git a/drivers/crypto/virtio/virtio_crypto_core.c b/drivers/crypto/virtio/virtio_crypto_core.c
index a111cd72797b..ff1410a32c2b 100644
--- a/drivers/crypto/virtio/virtio_crypto_core.c
+++ b/drivers/crypto/virtio/virtio_crypto_core.c
@@ -29,7 +29,6 @@ void
 virtcrypto_clear_request(struct virtio_crypto_request *vc_req)
 {
 	if (vc_req) {
-		kzfree(vc_req->iv);
 		kzfree(vc_req->req_data);
 		kfree(vc_req->sgs);
 	}
@@ -41,40 +40,18 @@ static void virtcrypto_dataq_callback(struct virtqueue *vq)
 	struct virtio_crypto_request *vc_req;
 	unsigned long flags;
 	unsigned int len;
-	struct ablkcipher_request *ablk_req;
-	int error;
 	unsigned int qid = vq->index;
 
 	spin_lock_irqsave(&vcrypto->data_vq[qid].lock, flags);
 	do {
 		virtqueue_disable_cb(vq);
 		while ((vc_req = virtqueue_get_buf(vq, &len)) != NULL) {
-			if (vc_req->type == VIRTIO_CRYPTO_SYM_OP_CIPHER) {
-				switch (vc_req->status) {
-				case VIRTIO_CRYPTO_OK:
-					error = 0;
-					break;
-				case VIRTIO_CRYPTO_INVSESS:
-				case VIRTIO_CRYPTO_ERR:
-					error = -EINVAL;
-					break;
-				case VIRTIO_CRYPTO_BADMSG:
-					error = -EBADMSG;
-					break;
-				default:
-					error = -EIO;
-					break;
-				}
-				ablk_req = vc_req->ablkcipher_req;
-
-				spin_unlock_irqrestore(
-					&vcrypto->data_vq[qid].lock, flags);
-				/* Finish the encrypt or decrypt process */
-				virtio_crypto_ablkcipher_finalize_req(vc_req,
-				    ablk_req, error);
-				spin_lock_irqsave(
-					&vcrypto->data_vq[qid].lock, flags);
-			}
+			spin_unlock_irqrestore(
+				&vcrypto->data_vq[qid].lock, flags);
+			if (vc_req->alg_cb)
+				vc_req->alg_cb(vc_req, len);
+			spin_lock_irqsave(
+				&vcrypto->data_vq[qid].lock, flags);
 		}
 	} while (!virtqueue_enable_cb(vq));
 	spin_unlock_irqrestore(&vcrypto->data_vq[qid].lock, flags);
@@ -270,7 +247,7 @@ static int virtcrypto_update_status(struct virtio_crypto *vcrypto)
 
 			return -EPERM;
 		}
-		dev_info(&vcrypto->vdev->dev, "Accelerator is ready\n");
+		dev_info(&vcrypto->vdev->dev, "Accelerator device is ready\n");
 	} else {
 		virtcrypto_dev_stop(vcrypto);
 		dev_info(&vcrypto->vdev->dev, "Accelerator is not ready\n");
diff --git a/drivers/crypto/vmx/aes_ctr.c b/drivers/crypto/vmx/aes_ctr.c
index 9c26d9e8dbea..17d84217dd76 100644
--- a/drivers/crypto/vmx/aes_ctr.c
+++ b/drivers/crypto/vmx/aes_ctr.c
@@ -104,8 +104,7 @@ static void p8_aes_ctr_final(struct p8_aes_ctr_ctx *ctx,
 	pagefault_enable();
 	preempt_enable();
 
-	crypto_xor(keystream, src, nbytes);
-	memcpy(dst, keystream, nbytes);
+	crypto_xor_cpy(dst, keystream, src, nbytes);
 	crypto_inc(ctrblk, AES_BLOCK_SIZE);
 }
 
diff --git a/drivers/dax/super.c b/drivers/dax/super.c
index 938eb4868f7f..557b93703532 100644
--- a/drivers/dax/super.c
+++ b/drivers/dax/super.c
@@ -46,6 +46,8 @@ void dax_read_unlock(int id)
 EXPORT_SYMBOL_GPL(dax_read_unlock);
 
 #ifdef CONFIG_BLOCK
+#include <linux/blkdev.h>
+
 int bdev_dax_pgoff(struct block_device *bdev, sector_t sector, size_t size,
 		pgoff_t *pgoff)
 {
@@ -59,6 +61,16 @@ int bdev_dax_pgoff(struct block_device *bdev, sector_t sector, size_t size,
 }
 EXPORT_SYMBOL(bdev_dax_pgoff);
 
+#if IS_ENABLED(CONFIG_FS_DAX)
+struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev)
+{
+	if (!blk_queue_dax(bdev->bd_queue))
+		return NULL;
+	return fs_dax_get_by_host(bdev->bd_disk->disk_name);
+}
+EXPORT_SYMBOL_GPL(fs_dax_get_by_bdev);
+#endif
+
 /**
  * __bdev_dax_supported() - Check if the device supports dax for filesystem
  * @sb: The superblock of the device
@@ -189,8 +201,10 @@ static umode_t dax_visible(struct kobject *kobj, struct attribute *a, int n)
 	if (!dax_dev)
 		return 0;
 
-	if (a == &dev_attr_write_cache.attr && !dax_dev->ops->flush)
+#ifndef CONFIG_ARCH_HAS_PMEM_API
+	if (a == &dev_attr_write_cache.attr)
 		return 0;
+#endif
 	return a->mode;
 }
 
@@ -255,18 +269,23 @@ size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
 }
 EXPORT_SYMBOL_GPL(dax_copy_from_iter);
 
-void dax_flush(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
-		size_t size)
+#ifdef CONFIG_ARCH_HAS_PMEM_API
+void arch_wb_cache_pmem(void *addr, size_t size);
+void dax_flush(struct dax_device *dax_dev, void *addr, size_t size)
 {
-	if (!dax_alive(dax_dev))
+	if (unlikely(!dax_alive(dax_dev)))
 		return;
 
-	if (!test_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags))
+	if (unlikely(!test_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags)))
 		return;
 
-	if (dax_dev->ops->flush)
-		dax_dev->ops->flush(dax_dev, pgoff, addr, size);
+	arch_wb_cache_pmem(addr, size);
 }
+#else
+void dax_flush(struct dax_device *dax_dev, void *addr, size_t size)
+{
+}
+#endif
 EXPORT_SYMBOL_GPL(dax_flush);
 
 void dax_write_cache(struct dax_device *dax_dev, bool wc)
diff --git a/drivers/devfreq/Kconfig b/drivers/devfreq/Kconfig
index 41254e702f1e..6a172d338f6d 100644
--- a/drivers/devfreq/Kconfig
+++ b/drivers/devfreq/Kconfig
@@ -1,6 +1,7 @@
 menuconfig PM_DEVFREQ
 	bool "Generic Dynamic Voltage and Frequency Scaling (DVFS) support"
 	select SRCU
+	select PM_OPP
 	help
 	  A device may have a list of frequencies and voltages available.
 	  devfreq, a generic DVFS framework can be registered for a device
diff --git a/drivers/devfreq/devfreq-event.c b/drivers/devfreq/devfreq-event.c
index 8648b32ebc89..d67242d87744 100644
--- a/drivers/devfreq/devfreq-event.c
+++ b/drivers/devfreq/devfreq-event.c
@@ -277,8 +277,8 @@ int devfreq_event_get_edev_count(struct device *dev)
 						sizeof(u32));
 	if (count < 0) {
 		dev_err(dev,
-			"failed to get the count of devfreq-event in %s node\n",
-			dev->of_node->full_name);
+			"failed to get the count of devfreq-event in %pOF node\n",
+			dev->of_node);
 		return count;
 	}
 
diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c
index dea04871b50d..a1c4ee818614 100644
--- a/drivers/devfreq/devfreq.c
+++ b/drivers/devfreq/devfreq.c
@@ -564,7 +564,7 @@ struct devfreq *devfreq_add_device(struct device *dev,
 	err = device_register(&devfreq->dev);
 	if (err) {
 		mutex_unlock(&devfreq->lock);
-		goto err_out;
+		goto err_dev;
 	}
 
 	devfreq->trans_table =	devm_kzalloc(&devfreq->dev,
@@ -610,6 +610,9 @@ err_init:
 	mutex_unlock(&devfreq_list_lock);
 
 	device_unregister(&devfreq->dev);
+err_dev:
+	if (devfreq)
+		kfree(devfreq);
 err_out:
 	return ERR_PTR(err);
 }
diff --git a/drivers/devfreq/governor.h b/drivers/devfreq/governor.h
index a4f2fa1091e4..cfc50a61a90d 100644
--- a/drivers/devfreq/governor.h
+++ b/drivers/devfreq/governor.h
@@ -69,4 +69,8 @@ extern int devfreq_remove_governor(struct devfreq_governor *governor);
 
 extern int devfreq_update_status(struct devfreq *devfreq, unsigned long freq);
 
+static inline int devfreq_update_stats(struct devfreq *df)
+{
+	return df->profile->get_dev_status(df->dev.parent, &df->last_status);
+}
 #endif /* _GOVERNOR_H */
diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c
index 56e0a0e1b600..9a302799040e 100644
--- a/drivers/dma-buf/dma-fence.c
+++ b/drivers/dma-buf/dma-fence.c
@@ -48,7 +48,7 @@ static atomic64_t dma_fence_context_counter = ATOMIC64_INIT(0);
  */
 u64 dma_fence_context_alloc(unsigned num)
 {
-	BUG_ON(!num);
+	WARN_ON(!num);
 	return atomic64_add_return(num, &dma_fence_context_counter) - num;
 }
 EXPORT_SYMBOL(dma_fence_context_alloc);
@@ -172,7 +172,7 @@ void dma_fence_release(struct kref *kref)
 
 	trace_dma_fence_destroy(fence);
 
-	BUG_ON(!list_empty(&fence->cb_list));
+	WARN_ON(!list_empty(&fence->cb_list));
 
 	if (fence->ops->release)
 		fence->ops->release(fence);
diff --git a/drivers/dma-buf/reservation.c b/drivers/dma-buf/reservation.c
index 393817e849ed..dec3a815455d 100644
--- a/drivers/dma-buf/reservation.c
+++ b/drivers/dma-buf/reservation.c
@@ -195,8 +195,7 @@ done:
 	if (old)
 		kfree_rcu(old, rcu);
 
-	if (old_fence)
-		dma_fence_put(old_fence);
+	dma_fence_put(old_fence);
 }
 
 /**
@@ -258,12 +257,71 @@ void reservation_object_add_excl_fence(struct reservation_object *obj,
 		dma_fence_put(rcu_dereference_protected(old->shared[i],
 						reservation_object_held(obj)));
 
-	if (old_fence)
-		dma_fence_put(old_fence);
+	dma_fence_put(old_fence);
 }
 EXPORT_SYMBOL(reservation_object_add_excl_fence);
 
 /**
+* reservation_object_copy_fences - Copy all fences from src to dst.
+* @dst: the destination reservation object
+* @src: the source reservation object
+*
+* Copy all fences from src to dst. Both src->lock as well as dst-lock must be
+* held.
+*/
+int reservation_object_copy_fences(struct reservation_object *dst,
+				   struct reservation_object *src)
+{
+	struct reservation_object_list *src_list, *dst_list;
+	struct dma_fence *old, *new;
+	size_t size;
+	unsigned i;
+
+	src_list = reservation_object_get_list(src);
+
+	if (src_list) {
+		size = offsetof(typeof(*src_list),
+				shared[src_list->shared_count]);
+		dst_list = kmalloc(size, GFP_KERNEL);
+		if (!dst_list)
+			return -ENOMEM;
+
+		dst_list->shared_count = src_list->shared_count;
+		dst_list->shared_max = src_list->shared_count;
+		for (i = 0; i < src_list->shared_count; ++i)
+			dst_list->shared[i] =
+				dma_fence_get(src_list->shared[i]);
+	} else {
+		dst_list = NULL;
+	}
+
+	kfree(dst->staged);
+	dst->staged = NULL;
+
+	src_list = reservation_object_get_list(dst);
+
+	old = reservation_object_get_excl(dst);
+	new = reservation_object_get_excl(src);
+
+	dma_fence_get(new);
+
+	preempt_disable();
+	write_seqcount_begin(&dst->seq);
+	/* write_seqcount_begin provides the necessary memory barrier */
+	RCU_INIT_POINTER(dst->fence_excl, new);
+	RCU_INIT_POINTER(dst->fence, dst_list);
+	write_seqcount_end(&dst->seq);
+	preempt_enable();
+
+	if (src_list)
+		kfree_rcu(src_list, rcu);
+	dma_fence_put(old);
+
+	return 0;
+}
+EXPORT_SYMBOL(reservation_object_copy_fences);
+
+/**
  * reservation_object_get_fences_rcu - Get an object's shared and exclusive
  * fences without update side lock held
  * @obj: the reservation object
@@ -373,12 +431,25 @@ long reservation_object_wait_timeout_rcu(struct reservation_object *obj,
 	long ret = timeout ? timeout : 1;
 
 retry:
-	fence = NULL;
 	shared_count = 0;
 	seq = read_seqcount_begin(&obj->seq);
 	rcu_read_lock();
 
-	if (wait_all) {
+	fence = rcu_dereference(obj->fence_excl);
+	if (fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
+		if (!dma_fence_get_rcu(fence))
+			goto unlock_retry;
+
+		if (dma_fence_is_signaled(fence)) {
+			dma_fence_put(fence);
+			fence = NULL;
+		}
+
+	} else {
+		fence = NULL;
+	}
+
+	if (!fence && wait_all) {
 		struct reservation_object_list *fobj =
 						rcu_dereference(obj->fence);
 
@@ -405,22 +476,6 @@ retry:
 		}
 	}
 
-	if (!shared_count) {
-		struct dma_fence *fence_excl = rcu_dereference(obj->fence_excl);
-
-		if (fence_excl &&
-		    !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
-			      &fence_excl->flags)) {
-			if (!dma_fence_get_rcu(fence_excl))
-				goto unlock_retry;
-
-			if (dma_fence_is_signaled(fence_excl))
-				dma_fence_put(fence_excl);
-			else
-				fence = fence_excl;
-		}
-	}
-
 	rcu_read_unlock();
 	if (fence) {
 		if (read_seqcount_retry(&obj->seq, seq)) {
diff --git a/drivers/dma-buf/sw_sync.c b/drivers/dma-buf/sw_sync.c
index 69c5ff36e2f9..38cc7389a6c1 100644
--- a/drivers/dma-buf/sw_sync.c
+++ b/drivers/dma-buf/sw_sync.c
@@ -96,9 +96,9 @@ static struct sync_timeline *sync_timeline_create(const char *name)
 	obj->context = dma_fence_context_alloc(1);
 	strlcpy(obj->name, name, sizeof(obj->name));
 
-	INIT_LIST_HEAD(&obj->child_list_head);
-	INIT_LIST_HEAD(&obj->active_list_head);
-	spin_lock_init(&obj->child_list_lock);
+	obj->pt_tree = RB_ROOT;
+	INIT_LIST_HEAD(&obj->pt_list);
+	spin_lock_init(&obj->lock);
 
 	sync_timeline_debug_add(obj);
 
@@ -125,68 +125,6 @@ static void sync_timeline_put(struct sync_timeline *obj)
 	kref_put(&obj->kref, sync_timeline_free);
 }
 
-/**
- * sync_timeline_signal() - signal a status change on a sync_timeline
- * @obj:	sync_timeline to signal
- * @inc:	num to increment on timeline->value
- *
- * A sync implementation should call this any time one of it's fences
- * has signaled or has an error condition.
- */
-static void sync_timeline_signal(struct sync_timeline *obj, unsigned int inc)
-{
-	unsigned long flags;
-	struct sync_pt *pt, *next;
-
-	trace_sync_timeline(obj);
-
-	spin_lock_irqsave(&obj->child_list_lock, flags);
-
-	obj->value += inc;
-
-	list_for_each_entry_safe(pt, next, &obj->active_list_head,
-				 active_list) {
-		if (dma_fence_is_signaled_locked(&pt->base))
-			list_del_init(&pt->active_list);
-	}
-
-	spin_unlock_irqrestore(&obj->child_list_lock, flags);
-}
-
-/**
- * sync_pt_create() - creates a sync pt
- * @parent:	fence's parent sync_timeline
- * @size:	size to allocate for this pt
- * @inc:	value of the fence
- *
- * Creates a new sync_pt as a child of @parent.  @size bytes will be
- * allocated allowing for implementation specific data to be kept after
- * the generic sync_timeline struct. Returns the sync_pt object or
- * NULL in case of error.
- */
-static struct sync_pt *sync_pt_create(struct sync_timeline *obj, int size,
-			     unsigned int value)
-{
-	unsigned long flags;
-	struct sync_pt *pt;
-
-	if (size < sizeof(*pt))
-		return NULL;
-
-	pt = kzalloc(size, GFP_KERNEL);
-	if (!pt)
-		return NULL;
-
-	spin_lock_irqsave(&obj->child_list_lock, flags);
-	sync_timeline_get(obj);
-	dma_fence_init(&pt->base, &timeline_fence_ops, &obj->child_list_lock,
-		       obj->context, value);
-	list_add_tail(&pt->child_list, &obj->child_list_head);
-	INIT_LIST_HEAD(&pt->active_list);
-	spin_unlock_irqrestore(&obj->child_list_lock, flags);
-	return pt;
-}
-
 static const char *timeline_fence_get_driver_name(struct dma_fence *fence)
 {
 	return "sw_sync";
@@ -203,13 +141,17 @@ static void timeline_fence_release(struct dma_fence *fence)
 {
 	struct sync_pt *pt = dma_fence_to_sync_pt(fence);
 	struct sync_timeline *parent = dma_fence_parent(fence);
-	unsigned long flags;
 
-	spin_lock_irqsave(fence->lock, flags);
-	list_del(&pt->child_list);
-	if (!list_empty(&pt->active_list))
-		list_del(&pt->active_list);
-	spin_unlock_irqrestore(fence->lock, flags);
+	if (!list_empty(&pt->link)) {
+		unsigned long flags;
+
+		spin_lock_irqsave(fence->lock, flags);
+		if (!list_empty(&pt->link)) {
+			list_del(&pt->link);
+			rb_erase(&pt->node, &parent->pt_tree);
+		}
+		spin_unlock_irqrestore(fence->lock, flags);
+	}
 
 	sync_timeline_put(parent);
 	dma_fence_free(fence);
@@ -219,18 +161,11 @@ static bool timeline_fence_signaled(struct dma_fence *fence)
 {
 	struct sync_timeline *parent = dma_fence_parent(fence);
 
-	return (fence->seqno > parent->value) ? false : true;
+	return !__dma_fence_is_later(fence->seqno, parent->value);
 }
 
 static bool timeline_fence_enable_signaling(struct dma_fence *fence)
 {
-	struct sync_pt *pt = dma_fence_to_sync_pt(fence);
-	struct sync_timeline *parent = dma_fence_parent(fence);
-
-	if (timeline_fence_signaled(fence))
-		return false;
-
-	list_add_tail(&pt->active_list, &parent->active_list_head);
 	return true;
 }
 
@@ -259,6 +194,107 @@ static const struct dma_fence_ops timeline_fence_ops = {
 	.timeline_value_str = timeline_fence_timeline_value_str,
 };
 
+/**
+ * sync_timeline_signal() - signal a status change on a sync_timeline
+ * @obj:	sync_timeline to signal
+ * @inc:	num to increment on timeline->value
+ *
+ * A sync implementation should call this any time one of it's fences
+ * has signaled or has an error condition.
+ */
+static void sync_timeline_signal(struct sync_timeline *obj, unsigned int inc)
+{
+	struct sync_pt *pt, *next;
+
+	trace_sync_timeline(obj);
+
+	spin_lock_irq(&obj->lock);
+
+	obj->value += inc;
+
+	list_for_each_entry_safe(pt, next, &obj->pt_list, link) {
+		if (!timeline_fence_signaled(&pt->base))
+			break;
+
+		list_del_init(&pt->link);
+		rb_erase(&pt->node, &obj->pt_tree);
+
+		/*
+		 * A signal callback may release the last reference to this
+		 * fence, causing it to be freed. That operation has to be
+		 * last to avoid a use after free inside this loop, and must
+		 * be after we remove the fence from the timeline in order to
+		 * prevent deadlocking on timeline->lock inside
+		 * timeline_fence_release().
+		 */
+		dma_fence_signal_locked(&pt->base);
+	}
+
+	spin_unlock_irq(&obj->lock);
+}
+
+/**
+ * sync_pt_create() - creates a sync pt
+ * @parent:	fence's parent sync_timeline
+ * @inc:	value of the fence
+ *
+ * Creates a new sync_pt as a child of @parent.  @size bytes will be
+ * allocated allowing for implementation specific data to be kept after
+ * the generic sync_timeline struct. Returns the sync_pt object or
+ * NULL in case of error.
+ */
+static struct sync_pt *sync_pt_create(struct sync_timeline *obj,
+				      unsigned int value)
+{
+	struct sync_pt *pt;
+
+	pt = kzalloc(sizeof(*pt), GFP_KERNEL);
+	if (!pt)
+		return NULL;
+
+	sync_timeline_get(obj);
+	dma_fence_init(&pt->base, &timeline_fence_ops, &obj->lock,
+		       obj->context, value);
+	INIT_LIST_HEAD(&pt->link);
+
+	spin_lock_irq(&obj->lock);
+	if (!dma_fence_is_signaled_locked(&pt->base)) {
+		struct rb_node **p = &obj->pt_tree.rb_node;
+		struct rb_node *parent = NULL;
+
+		while (*p) {
+			struct sync_pt *other;
+			int cmp;
+
+			parent = *p;
+			other = rb_entry(parent, typeof(*pt), node);
+			cmp = value - other->base.seqno;
+			if (cmp > 0) {
+				p = &parent->rb_right;
+			} else if (cmp < 0) {
+				p = &parent->rb_left;
+			} else {
+				if (dma_fence_get_rcu(&other->base)) {
+					dma_fence_put(&pt->base);
+					pt = other;
+					goto unlock;
+				}
+				p = &parent->rb_left;
+			}
+		}
+		rb_link_node(&pt->node, parent, p);
+		rb_insert_color(&pt->node, &obj->pt_tree);
+
+		parent = rb_next(&pt->node);
+		list_add_tail(&pt->link,
+			      parent ? &rb_entry(parent, typeof(*pt), node)->link : &obj->pt_list);
+	}
+unlock:
+	spin_unlock_irq(&obj->lock);
+
+	return pt;
+}
+
 /*
  * *WARNING*
  *
@@ -309,7 +345,7 @@ static long sw_sync_ioctl_create_fence(struct sync_timeline *obj,
 		goto err;
 	}
 
-	pt = sync_pt_create(obj, sizeof(*pt), data.value);
+	pt = sync_pt_create(obj, data.value);
 	if (!pt) {
 		err = -ENOMEM;
 		goto err;
@@ -345,6 +381,11 @@ static long sw_sync_ioctl_inc(struct sync_timeline *obj, unsigned long arg)
 	if (copy_from_user(&value, (void __user *)arg, sizeof(value)))
 		return -EFAULT;
 
+	while (value > INT_MAX)  {
+		sync_timeline_signal(obj, INT_MAX);
+		value -= INT_MAX;
+	}
+
 	sync_timeline_signal(obj, value);
 
 	return 0;
diff --git a/drivers/dma-buf/sync_debug.c b/drivers/dma-buf/sync_debug.c
index 59a3b2f8ee91..c4c8ecb24aa9 100644
--- a/drivers/dma-buf/sync_debug.c
+++ b/drivers/dma-buf/sync_debug.c
@@ -116,17 +116,15 @@ static void sync_print_fence(struct seq_file *s,
 static void sync_print_obj(struct seq_file *s, struct sync_timeline *obj)
 {
 	struct list_head *pos;
-	unsigned long flags;
 
 	seq_printf(s, "%s: %d\n", obj->name, obj->value);
 
-	spin_lock_irqsave(&obj->child_list_lock, flags);
-	list_for_each(pos, &obj->child_list_head) {
-		struct sync_pt *pt =
-			container_of(pos, struct sync_pt, child_list);
+	spin_lock_irq(&obj->lock);
+	list_for_each(pos, &obj->pt_list) {
+		struct sync_pt *pt = container_of(pos, struct sync_pt, link);
 		sync_print_fence(s, &pt->base, false);
 	}
-	spin_unlock_irqrestore(&obj->child_list_lock, flags);
+	spin_unlock_irq(&obj->lock);
 }
 
 static void sync_print_sync_file(struct seq_file *s,
@@ -151,12 +149,11 @@ static void sync_print_sync_file(struct seq_file *s,
 
 static int sync_debugfs_show(struct seq_file *s, void *unused)
 {
-	unsigned long flags;
 	struct list_head *pos;
 
 	seq_puts(s, "objs:\n--------------\n");
 
-	spin_lock_irqsave(&sync_timeline_list_lock, flags);
+	spin_lock_irq(&sync_timeline_list_lock);
 	list_for_each(pos, &sync_timeline_list_head) {
 		struct sync_timeline *obj =
 			container_of(pos, struct sync_timeline,
@@ -165,11 +162,11 @@ static int sync_debugfs_show(struct seq_file *s, void *unused)
 		sync_print_obj(s, obj);
 		seq_putc(s, '\n');
 	}
-	spin_unlock_irqrestore(&sync_timeline_list_lock, flags);
+	spin_unlock_irq(&sync_timeline_list_lock);
 
 	seq_puts(s, "fences:\n--------------\n");
 
-	spin_lock_irqsave(&sync_file_list_lock, flags);
+	spin_lock_irq(&sync_file_list_lock);
 	list_for_each(pos, &sync_file_list_head) {
 		struct sync_file *sync_file =
 			container_of(pos, struct sync_file, sync_file_list);
@@ -177,7 +174,7 @@ static int sync_debugfs_show(struct seq_file *s, void *unused)
 		sync_print_sync_file(s, sync_file);
 		seq_putc(s, '\n');
 	}
-	spin_unlock_irqrestore(&sync_file_list_lock, flags);
+	spin_unlock_irq(&sync_file_list_lock);
 	return 0;
 }
 
diff --git a/drivers/dma-buf/sync_debug.h b/drivers/dma-buf/sync_debug.h
index 26fe8b9907b3..d615a89f774c 100644
--- a/drivers/dma-buf/sync_debug.h
+++ b/drivers/dma-buf/sync_debug.h
@@ -14,6 +14,7 @@
 #define _LINUX_SYNC_H
 
 #include <linux/list.h>
+#include <linux/rbtree.h>
 #include <linux/spinlock.h>
 #include <linux/dma-fence.h>
 
@@ -24,42 +25,41 @@
  * struct sync_timeline - sync object
  * @kref:		reference count on fence.
  * @name:		name of the sync_timeline. Useful for debugging
- * @child_list_head:	list of children sync_pts for this sync_timeline
- * @child_list_lock:	lock protecting @child_list_head and fence.status
- * @active_list_head:	list of active (unsignaled/errored) sync_pts
+ * @lock:		lock protecting @pt_list and @value
+ * @pt_tree:		rbtree of active (unsignaled/errored) sync_pts
+ * @pt_list:		list of active (unsignaled/errored) sync_pts
  * @sync_timeline_list:	membership in global sync_timeline_list
  */
 struct sync_timeline {
 	struct kref		kref;
 	char			name[32];
 
-	/* protected by child_list_lock */
+	/* protected by lock */
 	u64			context;
 	int			value;
 
-	struct list_head	child_list_head;
-	spinlock_t		child_list_lock;
-
-	struct list_head	active_list_head;
+	struct rb_root		pt_tree;
+	struct list_head	pt_list;
+	spinlock_t		lock;
 
 	struct list_head	sync_timeline_list;
 };
 
 static inline struct sync_timeline *dma_fence_parent(struct dma_fence *fence)
 {
-	return container_of(fence->lock, struct sync_timeline, child_list_lock);
+	return container_of(fence->lock, struct sync_timeline, lock);
 }
 
 /**
  * struct sync_pt - sync_pt object
  * @base: base fence object
- * @child_list: sync timeline child's list
- * @active_list: sync timeline active child's list
+ * @link: link on the sync timeline's list
+ * @node: node in the sync timeline's tree
  */
 struct sync_pt {
 	struct dma_fence base;
-	struct list_head child_list;
-	struct list_head active_list;
+	struct list_head link;
+	struct rb_node node;
 };
 
 #ifdef CONFIG_SW_SYNC
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index fa8f9c07ce73..fadc4d8783bd 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -56,6 +56,12 @@ config DMA_OF
 	select DMA_ENGINE
 
 #devices
+config ALTERA_MSGDMA
+	tristate "Altera / Intel mSGDMA Engine"
+	select DMA_ENGINE
+	help
+	  Enable support for Altera / Intel mSGDMA controller.
+
 config AMBA_PL08X
 	bool "ARM PrimeCell PL080 or PL081 support"
 	depends on ARM_AMBA
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile
index d12ab2985ed1..f08f8de1b567 100644
--- a/drivers/dma/Makefile
+++ b/drivers/dma/Makefile
@@ -12,6 +12,7 @@ obj-$(CONFIG_DMA_OF) += of-dma.o
 obj-$(CONFIG_DMATEST) += dmatest.o
 
 #devices
+obj-$(CONFIG_ALTERA_MSGDMA) += altera-msgdma.o
 obj-$(CONFIG_AMBA_PL08X) += amba-pl08x.o
 obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += ppc4xx/
 obj-$(CONFIG_AT_HDMAC) += at_hdmac.o
diff --git a/drivers/dma/altera-msgdma.c b/drivers/dma/altera-msgdma.c
new file mode 100644
index 000000000000..32905d5606ac
--- /dev/null
+++ b/drivers/dma/altera-msgdma.c
@@ -0,0 +1,927 @@
+/*
+ * DMA driver for Altera mSGDMA IP core
+ *
+ * Copyright (C) 2017 Stefan Roese <sr@denx.de>
+ *
+ * Based on drivers/dma/xilinx/zynqmp_dma.c, which is:
+ * Copyright (C) 2016 Xilinx, Inc. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/bitops.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#include "dmaengine.h"
+
+#define MSGDMA_MAX_TRANS_LEN		U32_MAX
+#define MSGDMA_DESC_NUM			1024
+
+/**
+ * struct msgdma_extended_desc - implements an extended descriptor
+ * @read_addr_lo: data buffer source address low bits
+ * @write_addr_lo: data buffer destination address low bits
+ * @len: the number of bytes to transfer per descriptor
+ * @burst_seq_num: bit 31:24 write burst
+ *		   bit 23:16 read burst
+ *		   bit 15:00 sequence number
+ * @stride: bit 31:16 write stride
+ *	    bit 15:00 read stride
+ * @read_addr_hi: data buffer source address high bits
+ * @write_addr_hi: data buffer destination address high bits
+ * @control: characteristics of the transfer
+ */
+struct msgdma_extended_desc {
+	u32 read_addr_lo;
+	u32 write_addr_lo;
+	u32 len;
+	u32 burst_seq_num;
+	u32 stride;
+	u32 read_addr_hi;
+	u32 write_addr_hi;
+	u32 control;
+};
+
+/* mSGDMA descriptor control field bit definitions */
+#define MSGDMA_DESC_CTL_SET_CH(x)	((x) & 0xff)
+#define MSGDMA_DESC_CTL_GEN_SOP		BIT(8)
+#define MSGDMA_DESC_CTL_GEN_EOP		BIT(9)
+#define MSGDMA_DESC_CTL_PARK_READS	BIT(10)
+#define MSGDMA_DESC_CTL_PARK_WRITES	BIT(11)
+#define MSGDMA_DESC_CTL_END_ON_EOP	BIT(12)
+#define MSGDMA_DESC_CTL_END_ON_LEN	BIT(13)
+#define MSGDMA_DESC_CTL_TR_COMP_IRQ	BIT(14)
+#define MSGDMA_DESC_CTL_EARLY_IRQ	BIT(15)
+#define MSGDMA_DESC_CTL_TR_ERR_IRQ	GENMASK(23, 16)
+#define MSGDMA_DESC_CTL_EARLY_DONE	BIT(24)
+
+/*
+ * Writing "1" the "go" bit commits the entire descriptor into the
+ * descriptor FIFO(s)
+ */
+#define MSGDMA_DESC_CTL_GO		BIT(31)
+
+/* Tx buffer control flags */
+#define MSGDMA_DESC_CTL_TX_FIRST	(MSGDMA_DESC_CTL_GEN_SOP |	\
+					 MSGDMA_DESC_CTL_TR_ERR_IRQ |	\
+					 MSGDMA_DESC_CTL_GO)
+
+#define MSGDMA_DESC_CTL_TX_MIDDLE	(MSGDMA_DESC_CTL_TR_ERR_IRQ |	\
+					 MSGDMA_DESC_CTL_GO)
+
+#define MSGDMA_DESC_CTL_TX_LAST		(MSGDMA_DESC_CTL_GEN_EOP |	\
+					 MSGDMA_DESC_CTL_TR_COMP_IRQ |	\
+					 MSGDMA_DESC_CTL_TR_ERR_IRQ |	\
+					 MSGDMA_DESC_CTL_GO)
+
+#define MSGDMA_DESC_CTL_TX_SINGLE	(MSGDMA_DESC_CTL_GEN_SOP |	\
+					 MSGDMA_DESC_CTL_GEN_EOP |	\
+					 MSGDMA_DESC_CTL_TR_COMP_IRQ |	\
+					 MSGDMA_DESC_CTL_TR_ERR_IRQ |	\
+					 MSGDMA_DESC_CTL_GO)
+
+#define MSGDMA_DESC_CTL_RX_SINGLE	(MSGDMA_DESC_CTL_END_ON_EOP |	\
+					 MSGDMA_DESC_CTL_END_ON_LEN |	\
+					 MSGDMA_DESC_CTL_TR_COMP_IRQ |	\
+					 MSGDMA_DESC_CTL_EARLY_IRQ |	\
+					 MSGDMA_DESC_CTL_TR_ERR_IRQ |	\
+					 MSGDMA_DESC_CTL_GO)
+
+/* mSGDMA extended descriptor stride definitions */
+#define MSGDMA_DESC_STRIDE_RD		0x00000001
+#define MSGDMA_DESC_STRIDE_WR		0x00010000
+#define MSGDMA_DESC_STRIDE_RW		0x00010001
+
+/* mSGDMA dispatcher control and status register map */
+#define MSGDMA_CSR_STATUS		0x00	/* Read / Clear */
+#define MSGDMA_CSR_CONTROL		0x04	/* Read / Write */
+#define MSGDMA_CSR_RW_FILL_LEVEL	0x08	/* 31:16 - write fill level */
+						/* 15:00 - read fill level */
+#define MSGDMA_CSR_RESP_FILL_LEVEL	0x0c	/* response FIFO fill level */
+#define MSGDMA_CSR_RW_SEQ_NUM		0x10	/* 31:16 - write seq number */
+						/* 15:00 - read seq number */
+
+/* mSGDMA CSR status register bit definitions */
+#define MSGDMA_CSR_STAT_BUSY			BIT(0)
+#define MSGDMA_CSR_STAT_DESC_BUF_EMPTY		BIT(1)
+#define MSGDMA_CSR_STAT_DESC_BUF_FULL		BIT(2)
+#define MSGDMA_CSR_STAT_RESP_BUF_EMPTY		BIT(3)
+#define MSGDMA_CSR_STAT_RESP_BUF_FULL		BIT(4)
+#define MSGDMA_CSR_STAT_STOPPED			BIT(5)
+#define MSGDMA_CSR_STAT_RESETTING		BIT(6)
+#define MSGDMA_CSR_STAT_STOPPED_ON_ERR		BIT(7)
+#define MSGDMA_CSR_STAT_STOPPED_ON_EARLY	BIT(8)
+#define MSGDMA_CSR_STAT_IRQ			BIT(9)
+#define MSGDMA_CSR_STAT_MASK			GENMASK(9, 0)
+#define MSGDMA_CSR_STAT_MASK_WITHOUT_IRQ	GENMASK(8, 0)
+
+#define DESC_EMPTY	(MSGDMA_CSR_STAT_DESC_BUF_EMPTY | \
+			 MSGDMA_CSR_STAT_RESP_BUF_EMPTY)
+
+/* mSGDMA CSR control register bit definitions */
+#define MSGDMA_CSR_CTL_STOP			BIT(0)
+#define MSGDMA_CSR_CTL_RESET			BIT(1)
+#define MSGDMA_CSR_CTL_STOP_ON_ERR		BIT(2)
+#define MSGDMA_CSR_CTL_STOP_ON_EARLY		BIT(3)
+#define MSGDMA_CSR_CTL_GLOBAL_INTR		BIT(4)
+#define MSGDMA_CSR_CTL_STOP_DESCS		BIT(5)
+
+/* mSGDMA CSR fill level bits */
+#define MSGDMA_CSR_WR_FILL_LEVEL_GET(v)		(((v) & 0xffff0000) >> 16)
+#define MSGDMA_CSR_RD_FILL_LEVEL_GET(v)		((v) & 0x0000ffff)
+#define MSGDMA_CSR_RESP_FILL_LEVEL_GET(v)	((v) & 0x0000ffff)
+
+#define MSGDMA_CSR_SEQ_NUM_GET(v)		(((v) & 0xffff0000) >> 16)
+
+/* mSGDMA response register map */
+#define MSGDMA_RESP_BYTES_TRANSFERRED	0x00
+#define MSGDMA_RESP_STATUS		0x04
+
+/* mSGDMA response register bit definitions */
+#define MSGDMA_RESP_EARLY_TERM	BIT(8)
+#define MSGDMA_RESP_ERR_MASK	0xff
+
+/**
+ * struct msgdma_sw_desc - implements a sw descriptor
+ * @async_tx: support for the async_tx api
+ * @hw_desc: assosiated HW descriptor
+ * @free_list: node of the free SW descriprots list
+ */
+struct msgdma_sw_desc {
+	struct dma_async_tx_descriptor async_tx;
+	struct msgdma_extended_desc hw_desc;
+	struct list_head node;
+	struct list_head tx_list;
+};
+
+/**
+ * struct msgdma_device - DMA device structure
+ */
+struct msgdma_device {
+	spinlock_t lock;
+	struct device *dev;
+	struct tasklet_struct irq_tasklet;
+	struct list_head pending_list;
+	struct list_head free_list;
+	struct list_head active_list;
+	struct list_head done_list;
+	u32 desc_free_cnt;
+	bool idle;
+
+	struct dma_device dmadev;
+	struct dma_chan	dmachan;
+	dma_addr_t hw_desq;
+	struct msgdma_sw_desc *sw_desq;
+	unsigned int npendings;
+
+	struct dma_slave_config slave_cfg;
+
+	int irq;
+
+	/* mSGDMA controller */
+	void __iomem *csr;
+
+	/* mSGDMA descriptors */
+	void __iomem *desc;
+
+	/* mSGDMA response */
+	void __iomem *resp;
+};
+
+#define to_mdev(chan)	container_of(chan, struct msgdma_device, dmachan)
+#define tx_to_desc(tx)	container_of(tx, struct msgdma_sw_desc, async_tx)
+
+/**
+ * msgdma_get_descriptor - Get the sw descriptor from the pool
+ * @mdev: Pointer to the Altera mSGDMA device structure
+ *
+ * Return: The sw descriptor
+ */
+static struct msgdma_sw_desc *msgdma_get_descriptor(struct msgdma_device *mdev)
+{
+	struct msgdma_sw_desc *desc;
+
+	spin_lock_bh(&mdev->lock);
+	desc = list_first_entry(&mdev->free_list, struct msgdma_sw_desc, node);
+	list_del(&desc->node);
+	spin_unlock_bh(&mdev->lock);
+
+	INIT_LIST_HEAD(&desc->tx_list);
+
+	return desc;
+}
+
+/**
+ * msgdma_free_descriptor - Issue pending transactions
+ * @mdev: Pointer to the Altera mSGDMA device structure
+ * @desc: Transaction descriptor pointer
+ */
+static void msgdma_free_descriptor(struct msgdma_device *mdev,
+				   struct msgdma_sw_desc *desc)
+{
+	struct msgdma_sw_desc *child, *next;
+
+	mdev->desc_free_cnt++;
+	list_add_tail(&desc->node, &mdev->free_list);
+	list_for_each_entry_safe(child, next, &desc->tx_list, node) {
+		mdev->desc_free_cnt++;
+		list_move_tail(&child->node, &mdev->free_list);
+	}
+}
+
+/**
+ * msgdma_free_desc_list - Free descriptors list
+ * @mdev: Pointer to the Altera mSGDMA device structure
+ * @list: List to parse and delete the descriptor
+ */
+static void msgdma_free_desc_list(struct msgdma_device *mdev,
+				  struct list_head *list)
+{
+	struct msgdma_sw_desc *desc, *next;
+
+	list_for_each_entry_safe(desc, next, list, node)
+		msgdma_free_descriptor(mdev, desc);
+}
+
+/**
+ * msgdma_desc_config - Configure the descriptor
+ * @desc: Hw descriptor pointer
+ * @dst: Destination buffer address
+ * @src: Source buffer address
+ * @len: Transfer length
+ */
+static void msgdma_desc_config(struct msgdma_extended_desc *desc,
+			       dma_addr_t dst, dma_addr_t src, size_t len,
+			       u32 stride)
+{
+	/* Set lower 32bits of src & dst addresses in the descriptor */
+	desc->read_addr_lo = lower_32_bits(src);
+	desc->write_addr_lo = lower_32_bits(dst);
+
+	/* Set upper 32bits of src & dst addresses in the descriptor */
+	desc->read_addr_hi = upper_32_bits(src);
+	desc->write_addr_hi = upper_32_bits(dst);
+
+	desc->len = len;
+	desc->stride = stride;
+	desc->burst_seq_num = 0;	/* 0 will result in max burst length */
+
+	/*
+	 * Don't set interrupt on xfer end yet, this will be done later
+	 * for the "last" descriptor
+	 */
+	desc->control = MSGDMA_DESC_CTL_TR_ERR_IRQ | MSGDMA_DESC_CTL_GO |
+		MSGDMA_DESC_CTL_END_ON_LEN;
+}
+
+/**
+ * msgdma_desc_config_eod - Mark the descriptor as end descriptor
+ * @desc: Hw descriptor pointer
+ */
+static void msgdma_desc_config_eod(struct msgdma_extended_desc *desc)
+{
+	desc->control |= MSGDMA_DESC_CTL_TR_COMP_IRQ;
+}
+
+/**
+ * msgdma_tx_submit - Submit DMA transaction
+ * @tx: Async transaction descriptor pointer
+ *
+ * Return: cookie value
+ */
+static dma_cookie_t msgdma_tx_submit(struct dma_async_tx_descriptor *tx)
+{
+	struct msgdma_device *mdev = to_mdev(tx->chan);
+	struct msgdma_sw_desc *new;
+	dma_cookie_t cookie;
+
+	new = tx_to_desc(tx);
+	spin_lock_bh(&mdev->lock);
+	cookie = dma_cookie_assign(tx);
+
+	list_add_tail(&new->node, &mdev->pending_list);
+	spin_unlock_bh(&mdev->lock);
+
+	return cookie;
+}
+
+/**
+ * msgdma_prep_memcpy - prepare descriptors for memcpy transaction
+ * @dchan: DMA channel
+ * @dma_dst: Destination buffer address
+ * @dma_src: Source buffer address
+ * @len: Transfer length
+ * @flags: transfer ack flags
+ *
+ * Return: Async transaction descriptor on success and NULL on failure
+ */
+static struct dma_async_tx_descriptor *
+msgdma_prep_memcpy(struct dma_chan *dchan, dma_addr_t dma_dst,
+		   dma_addr_t dma_src, size_t len, ulong flags)
+{
+	struct msgdma_device *mdev = to_mdev(dchan);
+	struct msgdma_sw_desc *new, *first = NULL;
+	struct msgdma_extended_desc *desc;
+	size_t copy;
+	u32 desc_cnt;
+
+	desc_cnt = DIV_ROUND_UP(len, MSGDMA_MAX_TRANS_LEN);
+
+	spin_lock_bh(&mdev->lock);
+	if (desc_cnt > mdev->desc_free_cnt) {
+		spin_unlock_bh(&mdev->lock);
+		dev_dbg(mdev->dev, "mdev %p descs are not available\n", mdev);
+		return NULL;
+	}
+	mdev->desc_free_cnt -= desc_cnt;
+	spin_unlock_bh(&mdev->lock);
+
+	do {
+		/* Allocate and populate the descriptor */
+		new = msgdma_get_descriptor(mdev);
+
+		copy = min_t(size_t, len, MSGDMA_MAX_TRANS_LEN);
+		desc = &new->hw_desc;
+		msgdma_desc_config(desc, dma_dst, dma_src, copy,
+				   MSGDMA_DESC_STRIDE_RW);
+		len -= copy;
+		dma_src += copy;
+		dma_dst += copy;
+		if (!first)
+			first = new;
+		else
+			list_add_tail(&new->node, &first->tx_list);
+	} while (len);
+
+	msgdma_desc_config_eod(desc);
+	async_tx_ack(&first->async_tx);
+	first->async_tx.flags = flags;
+
+	return &first->async_tx;
+}
+
+/**
+ * msgdma_prep_slave_sg - prepare descriptors for a slave sg transaction
+ *
+ * @dchan: DMA channel
+ * @sgl: Destination scatter list
+ * @sg_len: Number of entries in destination scatter list
+ * @dir: DMA transfer direction
+ * @flags: transfer ack flags
+ * @context: transfer context (unused)
+ */
+static struct dma_async_tx_descriptor *
+msgdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
+		     unsigned int sg_len, enum dma_transfer_direction dir,
+		     unsigned long flags, void *context)
+
+{
+	struct msgdma_device *mdev = to_mdev(dchan);
+	struct dma_slave_config *cfg = &mdev->slave_cfg;
+	struct msgdma_sw_desc *new, *first = NULL;
+	void *desc = NULL;
+	size_t len, avail;
+	dma_addr_t dma_dst, dma_src;
+	u32 desc_cnt = 0, i;
+	struct scatterlist *sg;
+	u32 stride;
+
+	for_each_sg(sgl, sg, sg_len, i)
+		desc_cnt += DIV_ROUND_UP(sg_dma_len(sg), MSGDMA_MAX_TRANS_LEN);
+
+	spin_lock_bh(&mdev->lock);
+	if (desc_cnt > mdev->desc_free_cnt) {
+		spin_unlock_bh(&mdev->lock);
+		dev_dbg(mdev->dev, "mdev %p descs are not available\n", mdev);
+		return NULL;
+	}
+	mdev->desc_free_cnt -= desc_cnt;
+	spin_unlock_bh(&mdev->lock);
+
+	avail = sg_dma_len(sgl);
+
+	/* Run until we are out of scatterlist entries */
+	while (true) {
+		/* Allocate and populate the descriptor */
+		new = msgdma_get_descriptor(mdev);
+
+		desc = &new->hw_desc;
+		len = min_t(size_t, avail, MSGDMA_MAX_TRANS_LEN);
+
+		if (dir == DMA_MEM_TO_DEV) {
+			dma_src = sg_dma_address(sgl) + sg_dma_len(sgl) - avail;
+			dma_dst = cfg->dst_addr;
+			stride = MSGDMA_DESC_STRIDE_RD;
+		} else {
+			dma_src = cfg->src_addr;
+			dma_dst = sg_dma_address(sgl) + sg_dma_len(sgl) - avail;
+			stride = MSGDMA_DESC_STRIDE_WR;
+		}
+		msgdma_desc_config(desc, dma_dst, dma_src, len, stride);
+		avail -= len;
+
+		if (!first)
+			first = new;
+		else
+			list_add_tail(&new->node, &first->tx_list);
+
+		/* Fetch the next scatterlist entry */
+		if (avail == 0) {
+			if (sg_len == 0)
+				break;
+			sgl = sg_next(sgl);
+			if (sgl == NULL)
+				break;
+			sg_len--;
+			avail = sg_dma_len(sgl);
+		}
+	}
+
+	msgdma_desc_config_eod(desc);
+	first->async_tx.flags = flags;
+
+	return &first->async_tx;
+}
+
+static int msgdma_dma_config(struct dma_chan *dchan,
+			     struct dma_slave_config *config)
+{
+	struct msgdma_device *mdev = to_mdev(dchan);
+
+	memcpy(&mdev->slave_cfg, config, sizeof(*config));
+
+	return 0;
+}
+
+static void msgdma_reset(struct msgdma_device *mdev)
+{
+	u32 val;
+	int ret;
+
+	/* Reset mSGDMA */
+	iowrite32(MSGDMA_CSR_STAT_MASK, mdev->csr + MSGDMA_CSR_STATUS);
+	iowrite32(MSGDMA_CSR_CTL_RESET, mdev->csr + MSGDMA_CSR_CONTROL);
+
+	ret = readl_poll_timeout(mdev->csr + MSGDMA_CSR_STATUS, val,
+				 (val & MSGDMA_CSR_STAT_RESETTING) == 0,
+				 1, 10000);
+	if (ret)
+		dev_err(mdev->dev, "DMA channel did not reset\n");
+
+	/* Clear all status bits */
+	iowrite32(MSGDMA_CSR_STAT_MASK, mdev->csr + MSGDMA_CSR_STATUS);
+
+	/* Enable the DMA controller including interrupts */
+	iowrite32(MSGDMA_CSR_CTL_STOP_ON_ERR | MSGDMA_CSR_CTL_STOP_ON_EARLY |
+		  MSGDMA_CSR_CTL_GLOBAL_INTR, mdev->csr + MSGDMA_CSR_CONTROL);
+
+	mdev->idle = true;
+};
+
+static void msgdma_copy_one(struct msgdma_device *mdev,
+			    struct msgdma_sw_desc *desc)
+{
+	void __iomem *hw_desc = mdev->desc;
+
+	/*
+	 * Check if the DESC FIFO it not full. If its full, we need to wait
+	 * for at least one entry to become free again
+	 */
+	while (ioread32(mdev->csr + MSGDMA_CSR_STATUS) &
+	       MSGDMA_CSR_STAT_DESC_BUF_FULL)
+		mdelay(1);
+
+	/*
+	 * The descriptor needs to get copied into the descriptor FIFO
+	 * of the DMA controller. The descriptor will get flushed to the
+	 * FIFO, once the last word (control word) is written. Since we
+	 * are not 100% sure that memcpy() writes all word in the "correct"
+	 * oder (address from low to high) on all architectures, we make
+	 * sure this control word is written last by single coding it and
+	 * adding some write-barriers here.
+	 */
+	memcpy((void __force *)hw_desc, &desc->hw_desc,
+	       sizeof(desc->hw_desc) - sizeof(u32));
+
+	/* Write control word last to flush this descriptor into the FIFO */
+	mdev->idle = false;
+	wmb();
+	iowrite32(desc->hw_desc.control, hw_desc +
+		  offsetof(struct msgdma_extended_desc, control));
+	wmb();
+}
+
+/**
+ * msgdma_copy_desc_to_fifo - copy descriptor(s) into controller FIFO
+ * @mdev: Pointer to the Altera mSGDMA device structure
+ * @desc: Transaction descriptor pointer
+ */
+static void msgdma_copy_desc_to_fifo(struct msgdma_device *mdev,
+				     struct msgdma_sw_desc *desc)
+{
+	struct msgdma_sw_desc *sdesc, *next;
+
+	msgdma_copy_one(mdev, desc);
+
+	list_for_each_entry_safe(sdesc, next, &desc->tx_list, node)
+		msgdma_copy_one(mdev, sdesc);
+}
+
+/**
+ * msgdma_start_transfer - Initiate the new transfer
+ * @mdev: Pointer to the Altera mSGDMA device structure
+ */
+static void msgdma_start_transfer(struct msgdma_device *mdev)
+{
+	struct msgdma_sw_desc *desc;
+
+	if (!mdev->idle)
+		return;
+
+	desc = list_first_entry_or_null(&mdev->pending_list,
+					struct msgdma_sw_desc, node);
+	if (!desc)
+		return;
+
+	list_splice_tail_init(&mdev->pending_list, &mdev->active_list);
+	msgdma_copy_desc_to_fifo(mdev, desc);
+}
+
+/**
+ * msgdma_issue_pending - Issue pending transactions
+ * @chan: DMA channel pointer
+ */
+static void msgdma_issue_pending(struct dma_chan *chan)
+{
+	struct msgdma_device *mdev = to_mdev(chan);
+
+	spin_lock_bh(&mdev->lock);
+	msgdma_start_transfer(mdev);
+	spin_unlock_bh(&mdev->lock);
+}
+
+/**
+ * msgdma_chan_desc_cleanup - Cleanup the completed descriptors
+ * @mdev: Pointer to the Altera mSGDMA device structure
+ */
+static void msgdma_chan_desc_cleanup(struct msgdma_device *mdev)
+{
+	struct msgdma_sw_desc *desc, *next;
+
+	list_for_each_entry_safe(desc, next, &mdev->done_list, node) {
+		dma_async_tx_callback callback;
+		void *callback_param;
+
+		list_del(&desc->node);
+
+		callback = desc->async_tx.callback;
+		callback_param = desc->async_tx.callback_param;
+		if (callback) {
+			spin_unlock(&mdev->lock);
+			callback(callback_param);
+			spin_lock(&mdev->lock);
+		}
+
+		/* Run any dependencies, then free the descriptor */
+		msgdma_free_descriptor(mdev, desc);
+	}
+}
+
+/**
+ * msgdma_complete_descriptor - Mark the active descriptor as complete
+ * @mdev: Pointer to the Altera mSGDMA device structure
+ */
+static void msgdma_complete_descriptor(struct msgdma_device *mdev)
+{
+	struct msgdma_sw_desc *desc;
+
+	desc = list_first_entry_or_null(&mdev->active_list,
+					struct msgdma_sw_desc, node);
+	if (!desc)
+		return;
+	list_del(&desc->node);
+	dma_cookie_complete(&desc->async_tx);
+	list_add_tail(&desc->node, &mdev->done_list);
+}
+
+/**
+ * msgdma_free_descriptors - Free channel descriptors
+ * @mdev: Pointer to the Altera mSGDMA device structure
+ */
+static void msgdma_free_descriptors(struct msgdma_device *mdev)
+{
+	msgdma_free_desc_list(mdev, &mdev->active_list);
+	msgdma_free_desc_list(mdev, &mdev->pending_list);
+	msgdma_free_desc_list(mdev, &mdev->done_list);
+}
+
+/**
+ * msgdma_free_chan_resources - Free channel resources
+ * @dchan: DMA channel pointer
+ */
+static void msgdma_free_chan_resources(struct dma_chan *dchan)
+{
+	struct msgdma_device *mdev = to_mdev(dchan);
+
+	spin_lock_bh(&mdev->lock);
+	msgdma_free_descriptors(mdev);
+	spin_unlock_bh(&mdev->lock);
+	kfree(mdev->sw_desq);
+}
+
+/**
+ * msgdma_alloc_chan_resources - Allocate channel resources
+ * @dchan: DMA channel
+ *
+ * Return: Number of descriptors on success and failure value on error
+ */
+static int msgdma_alloc_chan_resources(struct dma_chan *dchan)
+{
+	struct msgdma_device *mdev = to_mdev(dchan);
+	struct msgdma_sw_desc *desc;
+	int i;
+
+	mdev->sw_desq = kcalloc(MSGDMA_DESC_NUM, sizeof(*desc), GFP_NOWAIT);
+	if (!mdev->sw_desq)
+		return -ENOMEM;
+
+	mdev->idle = true;
+	mdev->desc_free_cnt = MSGDMA_DESC_NUM;
+
+	INIT_LIST_HEAD(&mdev->free_list);
+
+	for (i = 0; i < MSGDMA_DESC_NUM; i++) {
+		desc = mdev->sw_desq + i;
+		dma_async_tx_descriptor_init(&desc->async_tx, &mdev->dmachan);
+		desc->async_tx.tx_submit = msgdma_tx_submit;
+		list_add_tail(&desc->node, &mdev->free_list);
+	}
+
+	return MSGDMA_DESC_NUM;
+}
+
+/**
+ * msgdma_tasklet - Schedule completion tasklet
+ * @data: Pointer to the Altera sSGDMA channel structure
+ */
+static void msgdma_tasklet(unsigned long data)
+{
+	struct msgdma_device *mdev = (struct msgdma_device *)data;
+	u32 count;
+	u32 __maybe_unused size;
+	u32 __maybe_unused status;
+
+	spin_lock(&mdev->lock);
+
+	/* Read number of responses that are available */
+	count = ioread32(mdev->csr + MSGDMA_CSR_RESP_FILL_LEVEL);
+	dev_dbg(mdev->dev, "%s (%d): response count=%d\n",
+		__func__, __LINE__, count);
+
+	while (count--) {
+		/*
+		 * Read both longwords to purge this response from the FIFO
+		 * On Avalon-MM implementations, size and status do not
+		 * have any real values, like transferred bytes or error
+		 * bits. So we need to just drop these values.
+		 */
+		size = ioread32(mdev->resp + MSGDMA_RESP_BYTES_TRANSFERRED);
+		status = ioread32(mdev->resp - MSGDMA_RESP_STATUS);
+
+		msgdma_complete_descriptor(mdev);
+		msgdma_chan_desc_cleanup(mdev);
+	}
+
+	spin_unlock(&mdev->lock);
+}
+
+/**
+ * msgdma_irq_handler - Altera mSGDMA Interrupt handler
+ * @irq: IRQ number
+ * @data: Pointer to the Altera mSGDMA device structure
+ *
+ * Return: IRQ_HANDLED/IRQ_NONE
+ */
+static irqreturn_t msgdma_irq_handler(int irq, void *data)
+{
+	struct msgdma_device *mdev = data;
+	u32 status;
+
+	status = ioread32(mdev->csr + MSGDMA_CSR_STATUS);
+	if ((status & MSGDMA_CSR_STAT_BUSY) == 0) {
+		/* Start next transfer if the DMA controller is idle */
+		spin_lock(&mdev->lock);
+		mdev->idle = true;
+		msgdma_start_transfer(mdev);
+		spin_unlock(&mdev->lock);
+	}
+
+	tasklet_schedule(&mdev->irq_tasklet);
+
+	/* Clear interrupt in mSGDMA controller */
+	iowrite32(MSGDMA_CSR_STAT_IRQ, mdev->csr + MSGDMA_CSR_STATUS);
+
+	return IRQ_HANDLED;
+}
+
+/**
+ * msgdma_chan_remove - Channel remove function
+ * @mdev: Pointer to the Altera mSGDMA device structure
+ */
+static void msgdma_dev_remove(struct msgdma_device *mdev)
+{
+	if (!mdev)
+		return;
+
+	devm_free_irq(mdev->dev, mdev->irq, mdev);
+	tasklet_kill(&mdev->irq_tasklet);
+	list_del(&mdev->dmachan.device_node);
+}
+
+static int request_and_map(struct platform_device *pdev, const char *name,
+			   struct resource **res, void __iomem **ptr)
+{
+	struct resource *region;
+	struct device *device = &pdev->dev;
+
+	*res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name);
+	if (*res == NULL) {
+		dev_err(device, "resource %s not defined\n", name);
+		return -ENODEV;
+	}
+
+	region = devm_request_mem_region(device, (*res)->start,
+					 resource_size(*res), dev_name(device));
+	if (region == NULL) {
+		dev_err(device, "unable to request %s\n", name);
+		return -EBUSY;
+	}
+
+	*ptr = devm_ioremap_nocache(device, region->start,
+				    resource_size(region));
+	if (*ptr == NULL) {
+		dev_err(device, "ioremap_nocache of %s failed!", name);
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+/**
+ * msgdma_probe - Driver probe function
+ * @pdev: Pointer to the platform_device structure
+ *
+ * Return: '0' on success and failure value on error
+ */
+static int msgdma_probe(struct platform_device *pdev)
+{
+	struct msgdma_device *mdev;
+	struct dma_device *dma_dev;
+	struct resource *dma_res;
+	int ret;
+
+	mdev = devm_kzalloc(&pdev->dev, sizeof(*mdev), GFP_NOWAIT);
+	if (!mdev)
+		return -ENOMEM;
+
+	mdev->dev = &pdev->dev;
+
+	/* Map CSR space */
+	ret = request_and_map(pdev, "csr", &dma_res, &mdev->csr);
+	if (ret)
+		return ret;
+
+	/* Map (extended) descriptor space */
+	ret = request_and_map(pdev, "desc", &dma_res, &mdev->desc);
+	if (ret)
+		return ret;
+
+	/* Map response space */
+	ret = request_and_map(pdev, "resp", &dma_res, &mdev->resp);
+	if (ret)
+		return ret;
+
+	platform_set_drvdata(pdev, mdev);
+
+	/* Get interrupt nr from platform data */
+	mdev->irq = platform_get_irq(pdev, 0);
+	if (mdev->irq < 0)
+		return -ENXIO;
+
+	ret = devm_request_irq(&pdev->dev, mdev->irq, msgdma_irq_handler,
+			       0, dev_name(&pdev->dev), mdev);
+	if (ret)
+		return ret;
+
+	tasklet_init(&mdev->irq_tasklet, msgdma_tasklet, (unsigned long)mdev);
+
+	dma_cookie_init(&mdev->dmachan);
+
+	spin_lock_init(&mdev->lock);
+
+	INIT_LIST_HEAD(&mdev->active_list);
+	INIT_LIST_HEAD(&mdev->pending_list);
+	INIT_LIST_HEAD(&mdev->done_list);
+	INIT_LIST_HEAD(&mdev->free_list);
+
+	dma_dev = &mdev->dmadev;
+
+	/* Set DMA capabilities */
+	dma_cap_zero(dma_dev->cap_mask);
+	dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
+	dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
+
+	dma_dev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
+	dma_dev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
+	dma_dev->directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM) |
+		BIT(DMA_MEM_TO_MEM);
+	dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
+
+	/* Init DMA link list */
+	INIT_LIST_HEAD(&dma_dev->channels);
+
+	/* Set base routines */
+	dma_dev->device_tx_status = dma_cookie_status;
+	dma_dev->device_issue_pending = msgdma_issue_pending;
+	dma_dev->dev = &pdev->dev;
+
+	dma_dev->copy_align = DMAENGINE_ALIGN_4_BYTES;
+	dma_dev->device_prep_dma_memcpy = msgdma_prep_memcpy;
+	dma_dev->device_prep_slave_sg = msgdma_prep_slave_sg;
+	dma_dev->device_config = msgdma_dma_config;
+
+	dma_dev->device_alloc_chan_resources = msgdma_alloc_chan_resources;
+	dma_dev->device_free_chan_resources = msgdma_free_chan_resources;
+
+	mdev->dmachan.device = dma_dev;
+	list_add_tail(&mdev->dmachan.device_node, &dma_dev->channels);
+
+	/* Set DMA mask to 64 bits */
+	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+	if (ret) {
+		dev_warn(&pdev->dev, "unable to set coherent mask to 64");
+		ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
+		if (ret)
+			goto fail;
+	}
+
+	msgdma_reset(mdev);
+
+	ret = dma_async_device_register(dma_dev);
+	if (ret)
+		goto fail;
+
+	dev_notice(&pdev->dev, "Altera mSGDMA driver probe success\n");
+
+	return 0;
+
+fail:
+	msgdma_dev_remove(mdev);
+
+	return ret;
+}
+
+/**
+ * msgdma_dma_remove - Driver remove function
+ * @pdev: Pointer to the platform_device structure
+ *
+ * Return: Always '0'
+ */
+static int msgdma_remove(struct platform_device *pdev)
+{
+	struct msgdma_device *mdev = platform_get_drvdata(pdev);
+
+	dma_async_device_unregister(&mdev->dmadev);
+	msgdma_dev_remove(mdev);
+
+	dev_notice(&pdev->dev, "Altera mSGDMA driver removed\n");
+
+	return 0;
+}
+
+static struct platform_driver msgdma_driver = {
+	.driver = {
+		.name = "altera-msgdma",
+	},
+	.probe = msgdma_probe,
+	.remove = msgdma_remove,
+};
+
+module_platform_driver(msgdma_driver);
+
+MODULE_ALIAS("platform:altera-msgdma");
+MODULE_DESCRIPTION("Altera mSGDMA driver");
+MODULE_AUTHOR("Stefan Roese <sr@denx.de>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c
index 13cc95c0474c..b52b0d55247e 100644
--- a/drivers/dma/amba-pl08x.c
+++ b/drivers/dma/amba-pl08x.c
@@ -3033,7 +3033,7 @@ static struct vendor_data vendor_ftdmac020 = {
 	.max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
 };
 
-static struct amba_id pl08x_ids[] = {
+static const struct amba_id pl08x_ids[] = {
 	/* Samsung PL080S variant */
 	{
 		.id	= 0x0a141080,
diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c
index 1baf3404a365..fbab271b3bf9 100644
--- a/drivers/dma/at_hdmac.c
+++ b/drivers/dma/at_hdmac.c
@@ -1203,138 +1203,6 @@ err:
 }
 
 /**
- * atc_prep_dma_sg - prepare memory to memory scather-gather operation
- * @chan: the channel to prepare operation on
- * @dst_sg: destination scatterlist
- * @dst_nents: number of destination scatterlist entries
- * @src_sg: source scatterlist
- * @src_nents: number of source scatterlist entries
- * @flags: tx descriptor status flags
- */
-static struct dma_async_tx_descriptor *
-atc_prep_dma_sg(struct dma_chan *chan,
-		struct scatterlist *dst_sg, unsigned int dst_nents,
-		struct scatterlist *src_sg, unsigned int src_nents,
-		unsigned long flags)
-{
-	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
-	struct at_desc		*desc = NULL;
-	struct at_desc		*first = NULL;
-	struct at_desc		*prev = NULL;
-	unsigned int		src_width;
-	unsigned int		dst_width;
-	size_t			xfer_count;
-	u32			ctrla;
-	u32			ctrlb;
-	size_t			dst_len = 0, src_len = 0;
-	dma_addr_t		dst = 0, src = 0;
-	size_t			len = 0, total_len = 0;
-
-	if (unlikely(dst_nents == 0 || src_nents == 0))
-		return NULL;
-
-	if (unlikely(dst_sg == NULL || src_sg == NULL))
-		return NULL;
-
-	ctrlb =   ATC_DEFAULT_CTRLB | ATC_IEN
-		| ATC_SRC_ADDR_MODE_INCR
-		| ATC_DST_ADDR_MODE_INCR
-		| ATC_FC_MEM2MEM;
-
-	/*
-	 * loop until there is either no more source or no more destination
-	 * scatterlist entry
-	 */
-	while (true) {
-
-		/* prepare the next transfer */
-		if (dst_len == 0) {
-
-			/* no more destination scatterlist entries */
-			if (!dst_sg || !dst_nents)
-				break;
-
-			dst = sg_dma_address(dst_sg);
-			dst_len = sg_dma_len(dst_sg);
-
-			dst_sg = sg_next(dst_sg);
-			dst_nents--;
-		}
-
-		if (src_len == 0) {
-
-			/* no more source scatterlist entries */
-			if (!src_sg || !src_nents)
-				break;
-
-			src = sg_dma_address(src_sg);
-			src_len = sg_dma_len(src_sg);
-
-			src_sg = sg_next(src_sg);
-			src_nents--;
-		}
-
-		len = min_t(size_t, src_len, dst_len);
-		if (len == 0)
-			continue;
-
-		/* take care for the alignment */
-		src_width = dst_width = atc_get_xfer_width(src, dst, len);
-
-		ctrla = ATC_SRC_WIDTH(src_width) |
-			ATC_DST_WIDTH(dst_width);
-
-		/*
-		 * The number of transfers to set up refer to the source width
-		 * that depends on the alignment.
-		 */
-		xfer_count = len >> src_width;
-		if (xfer_count > ATC_BTSIZE_MAX) {
-			xfer_count = ATC_BTSIZE_MAX;
-			len = ATC_BTSIZE_MAX << src_width;
-		}
-
-		/* create the transfer */
-		desc = atc_desc_get(atchan);
-		if (!desc)
-			goto err_desc_get;
-
-		desc->lli.saddr = src;
-		desc->lli.daddr = dst;
-		desc->lli.ctrla = ctrla | xfer_count;
-		desc->lli.ctrlb = ctrlb;
-
-		desc->txd.cookie = 0;
-		desc->len = len;
-
-		atc_desc_chain(&first, &prev, desc);
-
-		/* update the lengths and addresses for the next loop cycle */
-		dst_len -= len;
-		src_len -= len;
-		dst += len;
-		src += len;
-
-		total_len += len;
-	}
-
-	/* First descriptor of the chain embedds additional information */
-	first->txd.cookie = -EBUSY;
-	first->total_len = total_len;
-
-	/* set end-of-link to the last link descriptor of list*/
-	set_desc_eol(desc);
-
-	first->txd.flags = flags; /* client is in control of this ack */
-
-	return &first->txd;
-
-err_desc_get:
-	atc_desc_put(atchan, first);
-	return NULL;
-}
-
-/**
  * atc_dma_cyclic_check_values
  * Check for too big/unaligned periods and unaligned DMA buffer
  */
@@ -1933,14 +1801,12 @@ static int __init at_dma_probe(struct platform_device *pdev)
 
 	/* setup platform data for each SoC */
 	dma_cap_set(DMA_MEMCPY, at91sam9rl_config.cap_mask);
-	dma_cap_set(DMA_SG, at91sam9rl_config.cap_mask);
 	dma_cap_set(DMA_INTERLEAVE, at91sam9g45_config.cap_mask);
 	dma_cap_set(DMA_MEMCPY, at91sam9g45_config.cap_mask);
 	dma_cap_set(DMA_MEMSET, at91sam9g45_config.cap_mask);
 	dma_cap_set(DMA_MEMSET_SG, at91sam9g45_config.cap_mask);
 	dma_cap_set(DMA_PRIVATE, at91sam9g45_config.cap_mask);
 	dma_cap_set(DMA_SLAVE, at91sam9g45_config.cap_mask);
-	dma_cap_set(DMA_SG, at91sam9g45_config.cap_mask);
 
 	/* get DMA parameters from controller type */
 	plat_dat = at_dma_get_driver_data(pdev);
@@ -2078,16 +1944,12 @@ static int __init at_dma_probe(struct platform_device *pdev)
 		atdma->dma_common.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
 	}
 
-	if (dma_has_cap(DMA_SG, atdma->dma_common.cap_mask))
-		atdma->dma_common.device_prep_dma_sg = atc_prep_dma_sg;
-
 	dma_writel(atdma, EN, AT_DMA_ENABLE);
 
-	dev_info(&pdev->dev, "Atmel AHB DMA Controller ( %s%s%s%s), %d channels\n",
+	dev_info(&pdev->dev, "Atmel AHB DMA Controller ( %s%s%s), %d channels\n",
 	  dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask) ? "cpy " : "",
 	  dma_has_cap(DMA_MEMSET, atdma->dma_common.cap_mask) ? "set " : "",
 	  dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)  ? "slave " : "",
-	  dma_has_cap(DMA_SG, atdma->dma_common.cap_mask)  ? "sg-cpy " : "",
 	  plat_dat->nr_channels);
 
 	dma_async_device_register(&atdma->dma_common);
diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c
index 7d4e0bcda9af..c00e3923d7d8 100644
--- a/drivers/dma/at_xdmac.c
+++ b/drivers/dma/at_xdmac.c
@@ -875,7 +875,7 @@ at_xdmac_interleaved_queue_desc(struct dma_chan *chan,
 	dwidth = at_xdmac_align_width(chan, src | dst | chunk->size);
 	if (chunk->size >= (AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth)) {
 		dev_dbg(chan2dev(chan),
-			"%s: chunk too big (%d, max size %lu)...\n",
+			"%s: chunk too big (%zu, max size %lu)...\n",
 			__func__, chunk->size,
 			AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth);
 		return NULL;
@@ -956,7 +956,7 @@ at_xdmac_prep_interleaved(struct dma_chan *chan,
 	if ((xt->numf > 1) && (xt->frame_size > 1))
 		return NULL;
 
-	dev_dbg(chan2dev(chan), "%s: src=%pad, dest=%pad, numf=%d, frame_size=%d, flags=0x%lx\n",
+	dev_dbg(chan2dev(chan), "%s: src=%pad, dest=%pad, numf=%zu, frame_size=%zu, flags=0x%lx\n",
 		__func__, &xt->src_start, &xt->dst_start,	xt->numf,
 		xt->frame_size, flags);
 
@@ -990,7 +990,7 @@ at_xdmac_prep_interleaved(struct dma_chan *chan,
 			dst_skip = chunk->size + dst_icg;
 
 			dev_dbg(chan2dev(chan),
-				"%s: chunk size=%d, src icg=%d, dst icg=%d\n",
+				"%s: chunk size=%zu, src icg=%zu, dst icg=%zu\n",
 				__func__, chunk->size, src_icg, dst_icg);
 
 			desc = at_xdmac_interleaved_queue_desc(chan, atchan,
@@ -1207,7 +1207,7 @@ at_xdmac_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value,
 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);
 	struct at_xdmac_desc	*desc;
 
-	dev_dbg(chan2dev(chan), "%s: dest=%pad, len=%d, pattern=0x%x, flags=0x%lx\n",
+	dev_dbg(chan2dev(chan), "%s: dest=%pad, len=%zu, pattern=0x%x, flags=0x%lx\n",
 		__func__, &dest, len, value, flags);
 
 	if (unlikely(!len))
@@ -1883,8 +1883,11 @@ static int atmel_xdmac_resume(struct device *dev)
 	struct at_xdmac_chan	*atchan;
 	struct dma_chan		*chan, *_chan;
 	int			i;
+	int ret;
 
-	clk_prepare_enable(atxdmac->clk);
+	ret = clk_prepare_enable(atxdmac->clk);
+	if (ret)
+		return ret;
 
 	/* Clear pending interrupts. */
 	for (i = 0; i < atxdmac->dma.chancnt; i++) {
diff --git a/drivers/dma/bcm-sba-raid.c b/drivers/dma/bcm-sba-raid.c
index e41bbc7cb094..6c2c44724637 100644
--- a/drivers/dma/bcm-sba-raid.c
+++ b/drivers/dma/bcm-sba-raid.c
@@ -36,6 +36,7 @@
  */
 
 #include <linux/bitops.h>
+#include <linux/debugfs.h>
 #include <linux/dma-mapping.h>
 #include <linux/dmaengine.h>
 #include <linux/list.h>
@@ -48,7 +49,8 @@
 
 #include "dmaengine.h"
 
-/* SBA command related defines */
+/* ====== Driver macros and defines ===== */
+
 #define SBA_TYPE_SHIFT					48
 #define SBA_TYPE_MASK					GENMASK(1, 0)
 #define SBA_TYPE_A					0x0
@@ -82,39 +84,40 @@
 #define SBA_CMD_WRITE_BUFFER				0xc
 #define SBA_CMD_GALOIS					0xe
 
+#define SBA_MAX_REQ_PER_MBOX_CHANNEL			8192
+
 /* Driver helper macros */
 #define to_sba_request(tx)		\
 	container_of(tx, struct sba_request, tx)
 #define to_sba_device(dchan)		\
 	container_of(dchan, struct sba_device, dma_chan)
 
-enum sba_request_state {
-	SBA_REQUEST_STATE_FREE = 1,
-	SBA_REQUEST_STATE_ALLOCED = 2,
-	SBA_REQUEST_STATE_PENDING = 3,
-	SBA_REQUEST_STATE_ACTIVE = 4,
-	SBA_REQUEST_STATE_RECEIVED = 5,
-	SBA_REQUEST_STATE_COMPLETED = 6,
-	SBA_REQUEST_STATE_ABORTED = 7,
+/* ===== Driver data structures ===== */
+
+enum sba_request_flags {
+	SBA_REQUEST_STATE_FREE		= 0x001,
+	SBA_REQUEST_STATE_ALLOCED	= 0x002,
+	SBA_REQUEST_STATE_PENDING	= 0x004,
+	SBA_REQUEST_STATE_ACTIVE	= 0x008,
+	SBA_REQUEST_STATE_ABORTED	= 0x010,
+	SBA_REQUEST_STATE_MASK		= 0x0ff,
+	SBA_REQUEST_FENCE		= 0x100,
 };
 
 struct sba_request {
 	/* Global state */
 	struct list_head node;
 	struct sba_device *sba;
-	enum sba_request_state state;
-	bool fence;
+	u32 flags;
 	/* Chained requests management */
 	struct sba_request *first;
 	struct list_head next;
-	unsigned int next_count;
 	atomic_t next_pending_count;
 	/* BRCM message data */
-	void *resp;
-	dma_addr_t resp_dma;
-	struct brcm_sba_command *cmds;
 	struct brcm_message msg;
 	struct dma_async_tx_descriptor tx;
+	/* SBA commands */
+	struct brcm_sba_command cmds[0];
 };
 
 enum sba_version {
@@ -152,19 +155,18 @@ struct sba_device {
 	void *cmds_base;
 	dma_addr_t cmds_dma_base;
 	spinlock_t reqs_lock;
-	struct sba_request *reqs;
 	bool reqs_fence;
 	struct list_head reqs_alloc_list;
 	struct list_head reqs_pending_list;
 	struct list_head reqs_active_list;
-	struct list_head reqs_received_list;
-	struct list_head reqs_completed_list;
 	struct list_head reqs_aborted_list;
 	struct list_head reqs_free_list;
-	int reqs_free_count;
+	/* DebugFS directory entries */
+	struct dentry *root;
+	struct dentry *stats;
 };
 
-/* ====== SBA command helper routines ===== */
+/* ====== Command helper routines ===== */
 
 static inline u64 __pure sba_cmd_enc(u64 cmd, u32 val, u32 shift, u32 mask)
 {
@@ -196,32 +198,50 @@ static inline u32 __pure sba_cmd_pq_c_mdata(u32 d, u32 b1, u32 b0)
 	       ((d & SBA_C_MDATA_DNUM_MASK) << SBA_C_MDATA_DNUM_SHIFT);
 }
 
-/* ====== Channel resource management routines ===== */
+/* ====== General helper routines ===== */
+
+static void sba_peek_mchans(struct sba_device *sba)
+{
+	int mchan_idx;
+
+	for (mchan_idx = 0; mchan_idx < sba->mchans_count; mchan_idx++)
+		mbox_client_peek_data(sba->mchans[mchan_idx]);
+}
 
 static struct sba_request *sba_alloc_request(struct sba_device *sba)
 {
+	bool found = false;
 	unsigned long flags;
 	struct sba_request *req = NULL;
 
 	spin_lock_irqsave(&sba->reqs_lock, flags);
+	list_for_each_entry(req, &sba->reqs_free_list, node) {
+		if (async_tx_test_ack(&req->tx)) {
+			list_move_tail(&req->node, &sba->reqs_alloc_list);
+			found = true;
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&sba->reqs_lock, flags);
 
-	req = list_first_entry_or_null(&sba->reqs_free_list,
-				       struct sba_request, node);
-	if (req) {
-		list_move_tail(&req->node, &sba->reqs_alloc_list);
-		req->state = SBA_REQUEST_STATE_ALLOCED;
-		req->fence = false;
-		req->first = req;
-		INIT_LIST_HEAD(&req->next);
-		req->next_count = 1;
-		atomic_set(&req->next_pending_count, 1);
-
-		sba->reqs_free_count--;
-
-		dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan);
+	if (!found) {
+		/*
+		 * We have no more free requests so, we peek
+		 * mailbox channels hoping few active requests
+		 * would have completed which will create more
+		 * room for new requests.
+		 */
+		sba_peek_mchans(sba);
+		return NULL;
 	}
 
-	spin_unlock_irqrestore(&sba->reqs_lock, flags);
+	req->flags = SBA_REQUEST_STATE_ALLOCED;
+	req->first = req;
+	INIT_LIST_HEAD(&req->next);
+	atomic_set(&req->next_pending_count, 1);
+
+	dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan);
+	async_tx_ack(&req->tx);
 
 	return req;
 }
@@ -231,7 +251,8 @@ static void _sba_pending_request(struct sba_device *sba,
 				 struct sba_request *req)
 {
 	lockdep_assert_held(&sba->reqs_lock);
-	req->state = SBA_REQUEST_STATE_PENDING;
+	req->flags &= ~SBA_REQUEST_STATE_MASK;
+	req->flags |= SBA_REQUEST_STATE_PENDING;
 	list_move_tail(&req->node, &sba->reqs_pending_list);
 	if (list_empty(&sba->reqs_active_list))
 		sba->reqs_fence = false;
@@ -246,9 +267,10 @@ static bool _sba_active_request(struct sba_device *sba,
 		sba->reqs_fence = false;
 	if (sba->reqs_fence)
 		return false;
-	req->state = SBA_REQUEST_STATE_ACTIVE;
+	req->flags &= ~SBA_REQUEST_STATE_MASK;
+	req->flags |= SBA_REQUEST_STATE_ACTIVE;
 	list_move_tail(&req->node, &sba->reqs_active_list);
-	if (req->fence)
+	if (req->flags & SBA_REQUEST_FENCE)
 		sba->reqs_fence = true;
 	return true;
 }
@@ -258,7 +280,8 @@ static void _sba_abort_request(struct sba_device *sba,
 			       struct sba_request *req)
 {
 	lockdep_assert_held(&sba->reqs_lock);
-	req->state = SBA_REQUEST_STATE_ABORTED;
+	req->flags &= ~SBA_REQUEST_STATE_MASK;
+	req->flags |= SBA_REQUEST_STATE_ABORTED;
 	list_move_tail(&req->node, &sba->reqs_aborted_list);
 	if (list_empty(&sba->reqs_active_list))
 		sba->reqs_fence = false;
@@ -269,42 +292,11 @@ static void _sba_free_request(struct sba_device *sba,
 			      struct sba_request *req)
 {
 	lockdep_assert_held(&sba->reqs_lock);
-	req->state = SBA_REQUEST_STATE_FREE;
+	req->flags &= ~SBA_REQUEST_STATE_MASK;
+	req->flags |= SBA_REQUEST_STATE_FREE;
 	list_move_tail(&req->node, &sba->reqs_free_list);
 	if (list_empty(&sba->reqs_active_list))
 		sba->reqs_fence = false;
-	sba->reqs_free_count++;
-}
-
-static void sba_received_request(struct sba_request *req)
-{
-	unsigned long flags;
-	struct sba_device *sba = req->sba;
-
-	spin_lock_irqsave(&sba->reqs_lock, flags);
-	req->state = SBA_REQUEST_STATE_RECEIVED;
-	list_move_tail(&req->node, &sba->reqs_received_list);
-	spin_unlock_irqrestore(&sba->reqs_lock, flags);
-}
-
-static void sba_complete_chained_requests(struct sba_request *req)
-{
-	unsigned long flags;
-	struct sba_request *nreq;
-	struct sba_device *sba = req->sba;
-
-	spin_lock_irqsave(&sba->reqs_lock, flags);
-
-	req->state = SBA_REQUEST_STATE_COMPLETED;
-	list_move_tail(&req->node, &sba->reqs_completed_list);
-	list_for_each_entry(nreq, &req->next, next) {
-		nreq->state = SBA_REQUEST_STATE_COMPLETED;
-		list_move_tail(&nreq->node, &sba->reqs_completed_list);
-	}
-	if (list_empty(&sba->reqs_active_list))
-		sba->reqs_fence = false;
-
-	spin_unlock_irqrestore(&sba->reqs_lock, flags);
 }
 
 static void sba_free_chained_requests(struct sba_request *req)
@@ -332,8 +324,7 @@ static void sba_chain_request(struct sba_request *first,
 
 	list_add_tail(&req->next, &first->next);
 	req->first = first;
-	first->next_count++;
-	atomic_set(&first->next_pending_count, first->next_count);
+	atomic_inc(&first->next_pending_count);
 
 	spin_unlock_irqrestore(&sba->reqs_lock, flags);
 }
@@ -349,14 +340,6 @@ static void sba_cleanup_nonpending_requests(struct sba_device *sba)
 	list_for_each_entry_safe(req, req1, &sba->reqs_alloc_list, node)
 		_sba_free_request(sba, req);
 
-	/* Freeup all received request */
-	list_for_each_entry_safe(req, req1, &sba->reqs_received_list, node)
-		_sba_free_request(sba, req);
-
-	/* Freeup all completed request */
-	list_for_each_entry_safe(req, req1, &sba->reqs_completed_list, node)
-		_sba_free_request(sba, req);
-
 	/* Set all active requests as aborted */
 	list_for_each_entry_safe(req, req1, &sba->reqs_active_list, node)
 		_sba_abort_request(sba, req);
@@ -383,26 +366,6 @@ static void sba_cleanup_pending_requests(struct sba_device *sba)
 	spin_unlock_irqrestore(&sba->reqs_lock, flags);
 }
 
-/* ====== DMAENGINE callbacks ===== */
-
-static void sba_free_chan_resources(struct dma_chan *dchan)
-{
-	/*
-	 * Channel resources are pre-alloced so we just free-up
-	 * whatever we can so that we can re-use pre-alloced
-	 * channel resources next time.
-	 */
-	sba_cleanup_nonpending_requests(to_sba_device(dchan));
-}
-
-static int sba_device_terminate_all(struct dma_chan *dchan)
-{
-	/* Cleanup all pending requests */
-	sba_cleanup_pending_requests(to_sba_device(dchan));
-
-	return 0;
-}
-
 static int sba_send_mbox_request(struct sba_device *sba,
 				 struct sba_request *req)
 {
@@ -419,42 +382,156 @@ static int sba_send_mbox_request(struct sba_device *sba,
 		dev_err(sba->dev, "send message failed with error %d", ret);
 		return ret;
 	}
+
+	/* Check error returned by mailbox controller */
 	ret = req->msg.error;
 	if (ret < 0) {
 		dev_err(sba->dev, "message error %d", ret);
-		return ret;
 	}
 
-	return 0;
+	/* Signal txdone for mailbox channel */
+	mbox_client_txdone(sba->mchans[mchans_idx], ret);
+
+	return ret;
 }
 
-static void sba_issue_pending(struct dma_chan *dchan)
+/* Note: Must be called with sba->reqs_lock held */
+static void _sba_process_pending_requests(struct sba_device *sba)
 {
 	int ret;
-	unsigned long flags;
-	struct sba_request *req, *req1;
-	struct sba_device *sba = to_sba_device(dchan);
+	u32 count;
+	struct sba_request *req;
 
-	spin_lock_irqsave(&sba->reqs_lock, flags);
+	/*
+	 * Process few pending requests
+	 *
+	 * For now, we process (<number_of_mailbox_channels> * 8)
+	 * number of requests at a time.
+	 */
+	count = sba->mchans_count * 8;
+	while (!list_empty(&sba->reqs_pending_list) && count) {
+		/* Get the first pending request */
+		req = list_first_entry(&sba->reqs_pending_list,
+				       struct sba_request, node);
 
-	/* Process all pending request */
-	list_for_each_entry_safe(req, req1, &sba->reqs_pending_list, node) {
 		/* Try to make request active */
 		if (!_sba_active_request(sba, req))
 			break;
 
 		/* Send request to mailbox channel */
-		spin_unlock_irqrestore(&sba->reqs_lock, flags);
 		ret = sba_send_mbox_request(sba, req);
-		spin_lock_irqsave(&sba->reqs_lock, flags);
-
-		/* If something went wrong then keep request pending */
 		if (ret < 0) {
 			_sba_pending_request(sba, req);
 			break;
 		}
+
+		count--;
+	}
+}
+
+static void sba_process_received_request(struct sba_device *sba,
+					 struct sba_request *req)
+{
+	unsigned long flags;
+	struct dma_async_tx_descriptor *tx;
+	struct sba_request *nreq, *first = req->first;
+
+	/* Process only after all chained requests are received */
+	if (!atomic_dec_return(&first->next_pending_count)) {
+		tx = &first->tx;
+
+		WARN_ON(tx->cookie < 0);
+		if (tx->cookie > 0) {
+			dma_cookie_complete(tx);
+			dmaengine_desc_get_callback_invoke(tx, NULL);
+			dma_descriptor_unmap(tx);
+			tx->callback = NULL;
+			tx->callback_result = NULL;
+		}
+
+		dma_run_dependencies(tx);
+
+		spin_lock_irqsave(&sba->reqs_lock, flags);
+
+		/* Free all requests chained to first request */
+		list_for_each_entry(nreq, &first->next, next)
+			_sba_free_request(sba, nreq);
+		INIT_LIST_HEAD(&first->next);
+
+		/* Free the first request */
+		_sba_free_request(sba, first);
+
+		/* Process pending requests */
+		_sba_process_pending_requests(sba);
+
+		spin_unlock_irqrestore(&sba->reqs_lock, flags);
 	}
+}
+
+static void sba_write_stats_in_seqfile(struct sba_device *sba,
+				       struct seq_file *file)
+{
+	unsigned long flags;
+	struct sba_request *req;
+	u32 free_count = 0, alloced_count = 0;
+	u32 pending_count = 0, active_count = 0, aborted_count = 0;
+
+	spin_lock_irqsave(&sba->reqs_lock, flags);
+
+	list_for_each_entry(req, &sba->reqs_free_list, node)
+		if (async_tx_test_ack(&req->tx))
+			free_count++;
+
+	list_for_each_entry(req, &sba->reqs_alloc_list, node)
+		alloced_count++;
+
+	list_for_each_entry(req, &sba->reqs_pending_list, node)
+		pending_count++;
+
+	list_for_each_entry(req, &sba->reqs_active_list, node)
+		active_count++;
 
+	list_for_each_entry(req, &sba->reqs_aborted_list, node)
+		aborted_count++;
+
+	spin_unlock_irqrestore(&sba->reqs_lock, flags);
+
+	seq_printf(file, "maximum requests   = %d\n", sba->max_req);
+	seq_printf(file, "free requests      = %d\n", free_count);
+	seq_printf(file, "alloced requests   = %d\n", alloced_count);
+	seq_printf(file, "pending requests   = %d\n", pending_count);
+	seq_printf(file, "active requests    = %d\n", active_count);
+	seq_printf(file, "aborted requests   = %d\n", aborted_count);
+}
+
+/* ====== DMAENGINE callbacks ===== */
+
+static void sba_free_chan_resources(struct dma_chan *dchan)
+{
+	/*
+	 * Channel resources are pre-alloced so we just free-up
+	 * whatever we can so that we can re-use pre-alloced
+	 * channel resources next time.
+	 */
+	sba_cleanup_nonpending_requests(to_sba_device(dchan));
+}
+
+static int sba_device_terminate_all(struct dma_chan *dchan)
+{
+	/* Cleanup all pending requests */
+	sba_cleanup_pending_requests(to_sba_device(dchan));
+
+	return 0;
+}
+
+static void sba_issue_pending(struct dma_chan *dchan)
+{
+	unsigned long flags;
+	struct sba_device *sba = to_sba_device(dchan);
+
+	/* Process pending requests */
+	spin_lock_irqsave(&sba->reqs_lock, flags);
+	_sba_process_pending_requests(sba);
 	spin_unlock_irqrestore(&sba->reqs_lock, flags);
 }
 
@@ -486,17 +563,15 @@ static enum dma_status sba_tx_status(struct dma_chan *dchan,
 				     dma_cookie_t cookie,
 				     struct dma_tx_state *txstate)
 {
-	int mchan_idx;
 	enum dma_status ret;
 	struct sba_device *sba = to_sba_device(dchan);
 
-	for (mchan_idx = 0; mchan_idx < sba->mchans_count; mchan_idx++)
-		mbox_client_peek_data(sba->mchans[mchan_idx]);
-
 	ret = dma_cookie_status(dchan, cookie, txstate);
 	if (ret == DMA_COMPLETE)
 		return ret;
 
+	sba_peek_mchans(sba);
+
 	return dma_cookie_status(dchan, cookie, txstate);
 }
 
@@ -506,6 +581,7 @@ static void sba_fillup_interrupt_msg(struct sba_request *req,
 {
 	u64 cmd;
 	u32 c_mdata;
+	dma_addr_t resp_dma = req->tx.phys;
 	struct brcm_sba_command *cmdsp = cmds;
 
 	/* Type-B command to load dummy data into buf0 */
@@ -521,7 +597,7 @@ static void sba_fillup_interrupt_msg(struct sba_request *req,
 	cmdsp->cmd = cmd;
 	*cmdsp->cmd_dma = cpu_to_le64(cmd);
 	cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
-	cmdsp->data = req->resp_dma;
+	cmdsp->data = resp_dma;
 	cmdsp->data_len = req->sba->hw_resp_size;
 	cmdsp++;
 
@@ -542,11 +618,11 @@ static void sba_fillup_interrupt_msg(struct sba_request *req,
 	cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
 	if (req->sba->hw_resp_size) {
 		cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
-		cmdsp->resp = req->resp_dma;
+		cmdsp->resp = resp_dma;
 		cmdsp->resp_len = req->sba->hw_resp_size;
 	}
 	cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
-	cmdsp->data = req->resp_dma;
+	cmdsp->data = resp_dma;
 	cmdsp->data_len = req->sba->hw_resp_size;
 	cmdsp++;
 
@@ -573,7 +649,7 @@ sba_prep_dma_interrupt(struct dma_chan *dchan, unsigned long flags)
 	 * Force fence so that no requests are submitted
 	 * until DMA callback for this request is invoked.
 	 */
-	req->fence = true;
+	req->flags |= SBA_REQUEST_FENCE;
 
 	/* Fillup request message */
 	sba_fillup_interrupt_msg(req, req->cmds, &req->msg);
@@ -593,6 +669,7 @@ static void sba_fillup_memcpy_msg(struct sba_request *req,
 {
 	u64 cmd;
 	u32 c_mdata;
+	dma_addr_t resp_dma = req->tx.phys;
 	struct brcm_sba_command *cmdsp = cmds;
 
 	/* Type-B command to load data into buf0 */
@@ -629,7 +706,7 @@ static void sba_fillup_memcpy_msg(struct sba_request *req,
 	cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
 	if (req->sba->hw_resp_size) {
 		cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
-		cmdsp->resp = req->resp_dma;
+		cmdsp->resp = resp_dma;
 		cmdsp->resp_len = req->sba->hw_resp_size;
 	}
 	cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
@@ -656,7 +733,8 @@ sba_prep_dma_memcpy_req(struct sba_device *sba,
 	req = sba_alloc_request(sba);
 	if (!req)
 		return NULL;
-	req->fence = (flags & DMA_PREP_FENCE) ? true : false;
+	if (flags & DMA_PREP_FENCE)
+		req->flags |= SBA_REQUEST_FENCE;
 
 	/* Fillup request message */
 	sba_fillup_memcpy_msg(req, req->cmds, &req->msg,
@@ -711,6 +789,7 @@ static void sba_fillup_xor_msg(struct sba_request *req,
 	u64 cmd;
 	u32 c_mdata;
 	unsigned int i;
+	dma_addr_t resp_dma = req->tx.phys;
 	struct brcm_sba_command *cmdsp = cmds;
 
 	/* Type-B command to load data into buf0 */
@@ -766,7 +845,7 @@ static void sba_fillup_xor_msg(struct sba_request *req,
 	cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
 	if (req->sba->hw_resp_size) {
 		cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
-		cmdsp->resp = req->resp_dma;
+		cmdsp->resp = resp_dma;
 		cmdsp->resp_len = req->sba->hw_resp_size;
 	}
 	cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
@@ -782,7 +861,7 @@ static void sba_fillup_xor_msg(struct sba_request *req,
 	msg->error = 0;
 }
 
-struct sba_request *
+static struct sba_request *
 sba_prep_dma_xor_req(struct sba_device *sba,
 		     dma_addr_t off, dma_addr_t dst, dma_addr_t *src,
 		     u32 src_cnt, size_t len, unsigned long flags)
@@ -793,7 +872,8 @@ sba_prep_dma_xor_req(struct sba_device *sba,
 	req = sba_alloc_request(sba);
 	if (!req)
 		return NULL;
-	req->fence = (flags & DMA_PREP_FENCE) ? true : false;
+	if (flags & DMA_PREP_FENCE)
+		req->flags |= SBA_REQUEST_FENCE;
 
 	/* Fillup request message */
 	sba_fillup_xor_msg(req, req->cmds, &req->msg,
@@ -854,6 +934,7 @@ static void sba_fillup_pq_msg(struct sba_request *req,
 	u64 cmd;
 	u32 c_mdata;
 	unsigned int i;
+	dma_addr_t resp_dma = req->tx.phys;
 	struct brcm_sba_command *cmdsp = cmds;
 
 	if (pq_continue) {
@@ -947,7 +1028,7 @@ static void sba_fillup_pq_msg(struct sba_request *req,
 		cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
 		if (req->sba->hw_resp_size) {
 			cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
-			cmdsp->resp = req->resp_dma;
+			cmdsp->resp = resp_dma;
 			cmdsp->resp_len = req->sba->hw_resp_size;
 		}
 		cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
@@ -974,7 +1055,7 @@ static void sba_fillup_pq_msg(struct sba_request *req,
 		cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
 		if (req->sba->hw_resp_size) {
 			cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
-			cmdsp->resp = req->resp_dma;
+			cmdsp->resp = resp_dma;
 			cmdsp->resp_len = req->sba->hw_resp_size;
 		}
 		cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
@@ -991,7 +1072,7 @@ static void sba_fillup_pq_msg(struct sba_request *req,
 	msg->error = 0;
 }
 
-struct sba_request *
+static struct sba_request *
 sba_prep_dma_pq_req(struct sba_device *sba, dma_addr_t off,
 		    dma_addr_t *dst_p, dma_addr_t *dst_q, dma_addr_t *src,
 		    u32 src_cnt, const u8 *scf, size_t len, unsigned long flags)
@@ -1002,7 +1083,8 @@ sba_prep_dma_pq_req(struct sba_device *sba, dma_addr_t off,
 	req = sba_alloc_request(sba);
 	if (!req)
 		return NULL;
-	req->fence = (flags & DMA_PREP_FENCE) ? true : false;
+	if (flags & DMA_PREP_FENCE)
+		req->flags |= SBA_REQUEST_FENCE;
 
 	/* Fillup request messages */
 	sba_fillup_pq_msg(req, dmaf_continue(flags),
@@ -1027,6 +1109,7 @@ static void sba_fillup_pq_single_msg(struct sba_request *req,
 	u64 cmd;
 	u32 c_mdata;
 	u8 pos, dpos = raid6_gflog[scf];
+	dma_addr_t resp_dma = req->tx.phys;
 	struct brcm_sba_command *cmdsp = cmds;
 
 	if (!dst_p)
@@ -1105,7 +1188,7 @@ static void sba_fillup_pq_single_msg(struct sba_request *req,
 	cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
 	if (req->sba->hw_resp_size) {
 		cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
-		cmdsp->resp = req->resp_dma;
+		cmdsp->resp = resp_dma;
 		cmdsp->resp_len = req->sba->hw_resp_size;
 	}
 	cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
@@ -1226,7 +1309,7 @@ skip_q_computation:
 	cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
 	if (req->sba->hw_resp_size) {
 		cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
-		cmdsp->resp = req->resp_dma;
+		cmdsp->resp = resp_dma;
 		cmdsp->resp_len = req->sba->hw_resp_size;
 	}
 	cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
@@ -1243,7 +1326,7 @@ skip_q:
 	msg->error = 0;
 }
 
-struct sba_request *
+static struct sba_request *
 sba_prep_dma_pq_single_req(struct sba_device *sba, dma_addr_t off,
 			   dma_addr_t *dst_p, dma_addr_t *dst_q,
 			   dma_addr_t src, u8 scf, size_t len,
@@ -1255,7 +1338,8 @@ sba_prep_dma_pq_single_req(struct sba_device *sba, dma_addr_t off,
 	req = sba_alloc_request(sba);
 	if (!req)
 		return NULL;
-	req->fence = (flags & DMA_PREP_FENCE) ? true : false;
+	if (flags & DMA_PREP_FENCE)
+		req->flags |= SBA_REQUEST_FENCE;
 
 	/* Fillup request messages */
 	sba_fillup_pq_single_msg(req,  dmaf_continue(flags),
@@ -1370,40 +1454,10 @@ fail:
 
 /* ====== Mailbox callbacks ===== */
 
-static void sba_dma_tx_actions(struct sba_request *req)
-{
-	struct dma_async_tx_descriptor *tx = &req->tx;
-
-	WARN_ON(tx->cookie < 0);
-
-	if (tx->cookie > 0) {
-		dma_cookie_complete(tx);
-
-		/*
-		 * Call the callback (must not sleep or submit new
-		 * operations to this channel)
-		 */
-		if (tx->callback)
-			tx->callback(tx->callback_param);
-
-		dma_descriptor_unmap(tx);
-	}
-
-	/* Run dependent operations */
-	dma_run_dependencies(tx);
-
-	/* If waiting for 'ack' then move to completed list */
-	if (!async_tx_test_ack(&req->tx))
-		sba_complete_chained_requests(req);
-	else
-		sba_free_chained_requests(req);
-}
-
 static void sba_receive_message(struct mbox_client *cl, void *msg)
 {
-	unsigned long flags;
 	struct brcm_message *m = msg;
-	struct sba_request *req = m->ctx, *req1;
+	struct sba_request *req = m->ctx;
 	struct sba_device *sba = req->sba;
 
 	/* Error count if message has error */
@@ -1411,52 +1465,37 @@ static void sba_receive_message(struct mbox_client *cl, void *msg)
 		dev_err(sba->dev, "%s got message with error %d",
 			dma_chan_name(&sba->dma_chan), m->error);
 
-	/* Mark request as received */
-	sba_received_request(req);
-
-	/* Wait for all chained requests to be completed */
-	if (atomic_dec_return(&req->first->next_pending_count))
-		goto done;
-
-	/* Point to first request */
-	req = req->first;
-
-	/* Update request */
-	if (req->state == SBA_REQUEST_STATE_RECEIVED)
-		sba_dma_tx_actions(req);
-	else
-		sba_free_chained_requests(req);
+	/* Process received request */
+	sba_process_received_request(sba, req);
+}
 
-	spin_lock_irqsave(&sba->reqs_lock, flags);
+/* ====== Debugfs callbacks ====== */
 
-	/* Re-check all completed request waiting for 'ack' */
-	list_for_each_entry_safe(req, req1, &sba->reqs_completed_list, node) {
-		spin_unlock_irqrestore(&sba->reqs_lock, flags);
-		sba_dma_tx_actions(req);
-		spin_lock_irqsave(&sba->reqs_lock, flags);
-	}
+static int sba_debugfs_stats_show(struct seq_file *file, void *offset)
+{
+	struct platform_device *pdev = to_platform_device(file->private);
+	struct sba_device *sba = platform_get_drvdata(pdev);
 
-	spin_unlock_irqrestore(&sba->reqs_lock, flags);
+	/* Write stats in file */
+	sba_write_stats_in_seqfile(sba, file);
 
-done:
-	/* Try to submit pending request */
-	sba_issue_pending(&sba->dma_chan);
+	return 0;
 }
 
 /* ====== Platform driver routines ===== */
 
 static int sba_prealloc_channel_resources(struct sba_device *sba)
 {
-	int i, j, p, ret = 0;
+	int i, j, ret = 0;
 	struct sba_request *req = NULL;
 
-	sba->resp_base = dma_alloc_coherent(sba->dma_dev.dev,
+	sba->resp_base = dma_alloc_coherent(sba->mbox_dev,
 					    sba->max_resp_pool_size,
 					    &sba->resp_dma_base, GFP_KERNEL);
 	if (!sba->resp_base)
 		return -ENOMEM;
 
-	sba->cmds_base = dma_alloc_coherent(sba->dma_dev.dev,
+	sba->cmds_base = dma_alloc_coherent(sba->mbox_dev,
 					    sba->max_cmds_pool_size,
 					    &sba->cmds_dma_base, GFP_KERNEL);
 	if (!sba->cmds_base) {
@@ -1469,36 +1508,23 @@ static int sba_prealloc_channel_resources(struct sba_device *sba)
 	INIT_LIST_HEAD(&sba->reqs_alloc_list);
 	INIT_LIST_HEAD(&sba->reqs_pending_list);
 	INIT_LIST_HEAD(&sba->reqs_active_list);
-	INIT_LIST_HEAD(&sba->reqs_received_list);
-	INIT_LIST_HEAD(&sba->reqs_completed_list);
 	INIT_LIST_HEAD(&sba->reqs_aborted_list);
 	INIT_LIST_HEAD(&sba->reqs_free_list);
 
-	sba->reqs = devm_kcalloc(sba->dev, sba->max_req,
-				 sizeof(*req), GFP_KERNEL);
-	if (!sba->reqs) {
-		ret = -ENOMEM;
-		goto fail_free_cmds_pool;
-	}
-
-	for (i = 0, p = 0; i < sba->max_req; i++) {
-		req = &sba->reqs[i];
+	for (i = 0; i < sba->max_req; i++) {
+		req = devm_kzalloc(sba->dev,
+				sizeof(*req) +
+				sba->max_cmd_per_req * sizeof(req->cmds[0]),
+				GFP_KERNEL);
+		if (!req) {
+			ret = -ENOMEM;
+			goto fail_free_cmds_pool;
+		}
 		INIT_LIST_HEAD(&req->node);
 		req->sba = sba;
-		req->state = SBA_REQUEST_STATE_FREE;
+		req->flags = SBA_REQUEST_STATE_FREE;
 		INIT_LIST_HEAD(&req->next);
-		req->next_count = 1;
 		atomic_set(&req->next_pending_count, 0);
-		req->fence = false;
-		req->resp = sba->resp_base + p;
-		req->resp_dma = sba->resp_dma_base + p;
-		p += sba->hw_resp_size;
-		req->cmds = devm_kcalloc(sba->dev, sba->max_cmd_per_req,
-					 sizeof(*req->cmds), GFP_KERNEL);
-		if (!req->cmds) {
-			ret = -ENOMEM;
-			goto fail_free_cmds_pool;
-		}
 		for (j = 0; j < sba->max_cmd_per_req; j++) {
 			req->cmds[j].cmd = 0;
 			req->cmds[j].cmd_dma = sba->cmds_base +
@@ -1509,21 +1535,20 @@ static int sba_prealloc_channel_resources(struct sba_device *sba)
 		}
 		memset(&req->msg, 0, sizeof(req->msg));
 		dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan);
+		async_tx_ack(&req->tx);
 		req->tx.tx_submit = sba_tx_submit;
-		req->tx.phys = req->resp_dma;
+		req->tx.phys = sba->resp_dma_base + i * sba->hw_resp_size;
 		list_add_tail(&req->node, &sba->reqs_free_list);
 	}
 
-	sba->reqs_free_count = sba->max_req;
-
 	return 0;
 
 fail_free_cmds_pool:
-	dma_free_coherent(sba->dma_dev.dev,
+	dma_free_coherent(sba->mbox_dev,
 			  sba->max_cmds_pool_size,
 			  sba->cmds_base, sba->cmds_dma_base);
 fail_free_resp_pool:
-	dma_free_coherent(sba->dma_dev.dev,
+	dma_free_coherent(sba->mbox_dev,
 			  sba->max_resp_pool_size,
 			  sba->resp_base, sba->resp_dma_base);
 	return ret;
@@ -1532,9 +1557,9 @@ fail_free_resp_pool:
 static void sba_freeup_channel_resources(struct sba_device *sba)
 {
 	dmaengine_terminate_all(&sba->dma_chan);
-	dma_free_coherent(sba->dma_dev.dev, sba->max_cmds_pool_size,
+	dma_free_coherent(sba->mbox_dev, sba->max_cmds_pool_size,
 			  sba->cmds_base, sba->cmds_dma_base);
-	dma_free_coherent(sba->dma_dev.dev, sba->max_resp_pool_size,
+	dma_free_coherent(sba->mbox_dev, sba->max_resp_pool_size,
 			  sba->resp_base, sba->resp_dma_base);
 	sba->resp_base = NULL;
 	sba->resp_dma_base = 0;
@@ -1625,6 +1650,13 @@ static int sba_probe(struct platform_device *pdev)
 	sba->dev = &pdev->dev;
 	platform_set_drvdata(pdev, sba);
 
+	/* Number of channels equals number of mailbox channels */
+	ret = of_count_phandle_with_args(pdev->dev.of_node,
+					 "mboxes", "#mbox-cells");
+	if (ret <= 0)
+		return -ENODEV;
+	mchans_count = ret;
+
 	/* Determine SBA version from DT compatible string */
 	if (of_device_is_compatible(sba->dev->of_node, "brcm,iproc-sba"))
 		sba->ver = SBA_VER_1;
@@ -1637,14 +1669,12 @@ static int sba_probe(struct platform_device *pdev)
 	/* Derived Configuration parameters */
 	switch (sba->ver) {
 	case SBA_VER_1:
-		sba->max_req = 1024;
 		sba->hw_buf_size = 4096;
 		sba->hw_resp_size = 8;
 		sba->max_pq_coefs = 6;
 		sba->max_pq_srcs = 6;
 		break;
 	case SBA_VER_2:
-		sba->max_req = 1024;
 		sba->hw_buf_size = 4096;
 		sba->hw_resp_size = 8;
 		sba->max_pq_coefs = 30;
@@ -1658,6 +1688,7 @@ static int sba_probe(struct platform_device *pdev)
 	default:
 		return -EINVAL;
 	}
+	sba->max_req = SBA_MAX_REQ_PER_MBOX_CHANNEL * mchans_count;
 	sba->max_cmd_per_req = sba->max_pq_srcs + 3;
 	sba->max_xor_srcs = sba->max_cmd_per_req - 1;
 	sba->max_resp_pool_size = sba->max_req * sba->hw_resp_size;
@@ -1668,25 +1699,17 @@ static int sba_probe(struct platform_device *pdev)
 	sba->client.dev			= &pdev->dev;
 	sba->client.rx_callback		= sba_receive_message;
 	sba->client.tx_block		= false;
-	sba->client.knows_txdone	= false;
+	sba->client.knows_txdone	= true;
 	sba->client.tx_tout		= 0;
 
-	/* Number of channels equals number of mailbox channels */
-	ret = of_count_phandle_with_args(pdev->dev.of_node,
-					 "mboxes", "#mbox-cells");
-	if (ret <= 0)
-		return -ENODEV;
-	mchans_count = ret;
-	sba->mchans_count = 0;
-	atomic_set(&sba->mchans_current, 0);
-
 	/* Allocate mailbox channel array */
-	sba->mchans = devm_kcalloc(&pdev->dev, sba->mchans_count,
+	sba->mchans = devm_kcalloc(&pdev->dev, mchans_count,
 				   sizeof(*sba->mchans), GFP_KERNEL);
 	if (!sba->mchans)
 		return -ENOMEM;
 
 	/* Request mailbox channels */
+	sba->mchans_count = 0;
 	for (i = 0; i < mchans_count; i++) {
 		sba->mchans[i] = mbox_request_channel(&sba->client, i);
 		if (IS_ERR(sba->mchans[i])) {
@@ -1695,6 +1718,7 @@ static int sba_probe(struct platform_device *pdev)
 		}
 		sba->mchans_count++;
 	}
+	atomic_set(&sba->mchans_current, 0);
 
 	/* Find-out underlying mailbox device */
 	ret = of_parse_phandle_with_args(pdev->dev.of_node,
@@ -1723,15 +1747,34 @@ static int sba_probe(struct platform_device *pdev)
 		}
 	}
 
-	/* Register DMA device with linux async framework */
-	ret = sba_async_register(sba);
+	/* Prealloc channel resource */
+	ret = sba_prealloc_channel_resources(sba);
 	if (ret)
 		goto fail_free_mchans;
 
-	/* Prealloc channel resource */
-	ret = sba_prealloc_channel_resources(sba);
+	/* Check availability of debugfs */
+	if (!debugfs_initialized())
+		goto skip_debugfs;
+
+	/* Create debugfs root entry */
+	sba->root = debugfs_create_dir(dev_name(sba->dev), NULL);
+	if (IS_ERR_OR_NULL(sba->root)) {
+		dev_err(sba->dev, "failed to create debugfs root entry\n");
+		sba->root = NULL;
+		goto skip_debugfs;
+	}
+
+	/* Create debugfs stats entry */
+	sba->stats = debugfs_create_devm_seqfile(sba->dev, "stats", sba->root,
+						 sba_debugfs_stats_show);
+	if (IS_ERR_OR_NULL(sba->stats))
+		dev_err(sba->dev, "failed to create debugfs stats file\n");
+skip_debugfs:
+
+	/* Register DMA device with Linux async framework */
+	ret = sba_async_register(sba);
 	if (ret)
-		goto fail_async_dev_unreg;
+		goto fail_free_resources;
 
 	/* Print device info */
 	dev_info(sba->dev, "%s using SBAv%d and %d mailbox channels",
@@ -1740,8 +1783,9 @@ static int sba_probe(struct platform_device *pdev)
 
 	return 0;
 
-fail_async_dev_unreg:
-	dma_async_device_unregister(&sba->dma_dev);
+fail_free_resources:
+	debugfs_remove_recursive(sba->root);
+	sba_freeup_channel_resources(sba);
 fail_free_mchans:
 	for (i = 0; i < sba->mchans_count; i++)
 		mbox_free_channel(sba->mchans[i]);
@@ -1753,10 +1797,12 @@ static int sba_remove(struct platform_device *pdev)
 	int i;
 	struct sba_device *sba = platform_get_drvdata(pdev);
 
-	sba_freeup_channel_resources(sba);
-
 	dma_async_device_unregister(&sba->dma_dev);
 
+	debugfs_remove_recursive(sba->root);
+
+	sba_freeup_channel_resources(sba);
+
 	for (i = 0; i < sba->mchans_count; i++)
 		mbox_free_channel(sba->mchans[i]);
 
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index d9118ec23025..b451354735d3 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -923,30 +923,85 @@ int dma_async_device_register(struct dma_device *device)
 		return -ENODEV;
 
 	/* validate device routines */
-	BUG_ON(dma_has_cap(DMA_MEMCPY, device->cap_mask) &&
-		!device->device_prep_dma_memcpy);
-	BUG_ON(dma_has_cap(DMA_XOR, device->cap_mask) &&
-		!device->device_prep_dma_xor);
-	BUG_ON(dma_has_cap(DMA_XOR_VAL, device->cap_mask) &&
-		!device->device_prep_dma_xor_val);
-	BUG_ON(dma_has_cap(DMA_PQ, device->cap_mask) &&
-		!device->device_prep_dma_pq);
-	BUG_ON(dma_has_cap(DMA_PQ_VAL, device->cap_mask) &&
-		!device->device_prep_dma_pq_val);
-	BUG_ON(dma_has_cap(DMA_MEMSET, device->cap_mask) &&
-		!device->device_prep_dma_memset);
-	BUG_ON(dma_has_cap(DMA_INTERRUPT, device->cap_mask) &&
-		!device->device_prep_dma_interrupt);
-	BUG_ON(dma_has_cap(DMA_SG, device->cap_mask) &&
-		!device->device_prep_dma_sg);
-	BUG_ON(dma_has_cap(DMA_CYCLIC, device->cap_mask) &&
-		!device->device_prep_dma_cyclic);
-	BUG_ON(dma_has_cap(DMA_INTERLEAVE, device->cap_mask) &&
-		!device->device_prep_interleaved_dma);
-
-	BUG_ON(!device->device_tx_status);
-	BUG_ON(!device->device_issue_pending);
-	BUG_ON(!device->dev);
+	if (!device->dev) {
+		pr_err("DMAdevice must have dev\n");
+		return -EIO;
+	}
+
+	if (dma_has_cap(DMA_MEMCPY, device->cap_mask) && !device->device_prep_dma_memcpy) {
+		dev_err(device->dev,
+			"Device claims capability %s, but op is not defined\n",
+			"DMA_MEMCPY");
+		return -EIO;
+	}
+
+	if (dma_has_cap(DMA_XOR, device->cap_mask) && !device->device_prep_dma_xor) {
+		dev_err(device->dev,
+			"Device claims capability %s, but op is not defined\n",
+			"DMA_XOR");
+		return -EIO;
+	}
+
+	if (dma_has_cap(DMA_XOR_VAL, device->cap_mask) && !device->device_prep_dma_xor_val) {
+		dev_err(device->dev,
+			"Device claims capability %s, but op is not defined\n",
+			"DMA_XOR_VAL");
+		return -EIO;
+	}
+
+	if (dma_has_cap(DMA_PQ, device->cap_mask) && !device->device_prep_dma_pq) {
+		dev_err(device->dev,
+			"Device claims capability %s, but op is not defined\n",
+			"DMA_PQ");
+		return -EIO;
+	}
+
+	if (dma_has_cap(DMA_PQ_VAL, device->cap_mask) && !device->device_prep_dma_pq_val) {
+		dev_err(device->dev,
+			"Device claims capability %s, but op is not defined\n",
+			"DMA_PQ_VAL");
+		return -EIO;
+	}
+
+	if (dma_has_cap(DMA_MEMSET, device->cap_mask) && !device->device_prep_dma_memset) {
+		dev_err(device->dev,
+			"Device claims capability %s, but op is not defined\n",
+			"DMA_MEMSET");
+		return -EIO;
+	}
+
+	if (dma_has_cap(DMA_INTERRUPT, device->cap_mask) && !device->device_prep_dma_interrupt) {
+		dev_err(device->dev,
+			"Device claims capability %s, but op is not defined\n",
+			"DMA_INTERRUPT");
+		return -EIO;
+	}
+
+	if (dma_has_cap(DMA_CYCLIC, device->cap_mask) && !device->device_prep_dma_cyclic) {
+		dev_err(device->dev,
+			"Device claims capability %s, but op is not defined\n",
+			"DMA_CYCLIC");
+		return -EIO;
+	}
+
+	if (dma_has_cap(DMA_INTERLEAVE, device->cap_mask) && !device->device_prep_interleaved_dma) {
+		dev_err(device->dev,
+			"Device claims capability %s, but op is not defined\n",
+			"DMA_INTERLEAVE");
+		return -EIO;
+	}
+
+
+	if (!device->device_tx_status) {
+		dev_err(device->dev, "Device tx_status is not defined\n");
+		return -EIO;
+	}
+
+
+	if (!device->device_issue_pending) {
+		dev_err(device->dev, "Device issue_pending is not defined\n");
+		return -EIO;
+	}
 
 	/* note: this only matters in the
 	 * CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH=n case
diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c
index a07ef3d6b3ec..34ff53290b03 100644
--- a/drivers/dma/dmatest.c
+++ b/drivers/dma/dmatest.c
@@ -52,15 +52,10 @@ module_param(iterations, uint, S_IRUGO | S_IWUSR);
 MODULE_PARM_DESC(iterations,
 		"Iterations before stopping test (default: infinite)");
 
-static unsigned int sg_buffers = 1;
-module_param(sg_buffers, uint, S_IRUGO | S_IWUSR);
-MODULE_PARM_DESC(sg_buffers,
-		"Number of scatter gather buffers (default: 1)");
-
 static unsigned int dmatest;
 module_param(dmatest, uint, S_IRUGO | S_IWUSR);
 MODULE_PARM_DESC(dmatest,
-		"dmatest 0-memcpy 1-slave_sg (default: 0)");
+		"dmatest 0-memcpy 1-memset (default: 0)");
 
 static unsigned int xor_sources = 3;
 module_param(xor_sources, uint, S_IRUGO | S_IWUSR);
@@ -158,6 +153,7 @@ MODULE_PARM_DESC(run, "Run the test (default: false)");
 #define PATTERN_COPY		0x40
 #define PATTERN_OVERWRITE	0x20
 #define PATTERN_COUNT_MASK	0x1f
+#define PATTERN_MEMSET_IDX	0x01
 
 struct dmatest_thread {
 	struct list_head	node;
@@ -239,46 +235,62 @@ static unsigned long dmatest_random(void)
 	return buf;
 }
 
+static inline u8 gen_inv_idx(u8 index, bool is_memset)
+{
+	u8 val = is_memset ? PATTERN_MEMSET_IDX : index;
+
+	return ~val & PATTERN_COUNT_MASK;
+}
+
+static inline u8 gen_src_value(u8 index, bool is_memset)
+{
+	return PATTERN_SRC | gen_inv_idx(index, is_memset);
+}
+
+static inline u8 gen_dst_value(u8 index, bool is_memset)
+{
+	return PATTERN_DST | gen_inv_idx(index, is_memset);
+}
+
 static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,
-		unsigned int buf_size)
+		unsigned int buf_size, bool is_memset)
 {
 	unsigned int i;
 	u8 *buf;
 
 	for (; (buf = *bufs); bufs++) {
 		for (i = 0; i < start; i++)
-			buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
+			buf[i] = gen_src_value(i, is_memset);
 		for ( ; i < start + len; i++)
-			buf[i] = PATTERN_SRC | PATTERN_COPY
-				| (~i & PATTERN_COUNT_MASK);
+			buf[i] = gen_src_value(i, is_memset) | PATTERN_COPY;
 		for ( ; i < buf_size; i++)
-			buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
+			buf[i] = gen_src_value(i, is_memset);
 		buf++;
 	}
 }
 
 static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
-		unsigned int buf_size)
+		unsigned int buf_size, bool is_memset)
 {
 	unsigned int i;
 	u8 *buf;
 
 	for (; (buf = *bufs); bufs++) {
 		for (i = 0; i < start; i++)
-			buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
+			buf[i] = gen_dst_value(i, is_memset);
 		for ( ; i < start + len; i++)
-			buf[i] = PATTERN_DST | PATTERN_OVERWRITE
-				| (~i & PATTERN_COUNT_MASK);
+			buf[i] = gen_dst_value(i, is_memset) |
+						PATTERN_OVERWRITE;
 		for ( ; i < buf_size; i++)
-			buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
+			buf[i] = gen_dst_value(i, is_memset);
 	}
 }
 
 static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
-		unsigned int counter, bool is_srcbuf)
+		unsigned int counter, bool is_srcbuf, bool is_memset)
 {
 	u8		diff = actual ^ pattern;
-	u8		expected = pattern | (~counter & PATTERN_COUNT_MASK);
+	u8		expected = pattern | gen_inv_idx(counter, is_memset);
 	const char	*thread_name = current->comm;
 
 	if (is_srcbuf)
@@ -298,7 +310,7 @@ static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
 
 static unsigned int dmatest_verify(u8 **bufs, unsigned int start,
 		unsigned int end, unsigned int counter, u8 pattern,
-		bool is_srcbuf)
+		bool is_srcbuf, bool is_memset)
 {
 	unsigned int i;
 	unsigned int error_count = 0;
@@ -311,11 +323,12 @@ static unsigned int dmatest_verify(u8 **bufs, unsigned int start,
 		counter = counter_orig;
 		for (i = start; i < end; i++) {
 			actual = buf[i];
-			expected = pattern | (~counter & PATTERN_COUNT_MASK);
+			expected = pattern | gen_inv_idx(counter, is_memset);
 			if (actual != expected) {
 				if (error_count < MAX_ERROR_COUNT)
 					dmatest_mismatch(actual, pattern, i,
-							 counter, is_srcbuf);
+							 counter, is_srcbuf,
+							 is_memset);
 				error_count++;
 			}
 			counter++;
@@ -435,6 +448,7 @@ static int dmatest_func(void *data)
 	s64			runtime = 0;
 	unsigned long long	total_len = 0;
 	u8			align = 0;
+	bool			is_memset = false;
 
 	set_freezable();
 
@@ -448,9 +462,10 @@ static int dmatest_func(void *data)
 	if (thread->type == DMA_MEMCPY) {
 		align = dev->copy_align;
 		src_cnt = dst_cnt = 1;
-	} else if (thread->type == DMA_SG) {
-		align = dev->copy_align;
-		src_cnt = dst_cnt = sg_buffers;
+	} else if (thread->type == DMA_MEMSET) {
+		align = dev->fill_align;
+		src_cnt = dst_cnt = 1;
+		is_memset = true;
 	} else if (thread->type == DMA_XOR) {
 		/* force odd to ensure dst = src */
 		src_cnt = min_odd(params->xor_sources | 1, dev->max_xor);
@@ -530,8 +545,6 @@ static int dmatest_func(void *data)
 		dma_addr_t srcs[src_cnt];
 		dma_addr_t *dsts;
 		unsigned int src_off, dst_off, len;
-		struct scatterlist tx_sg[src_cnt];
-		struct scatterlist rx_sg[src_cnt];
 
 		total_tests++;
 
@@ -571,9 +584,9 @@ static int dmatest_func(void *data)
 			dst_off = (dst_off >> align) << align;
 
 			dmatest_init_srcs(thread->srcs, src_off, len,
-					  params->buf_size);
+					  params->buf_size, is_memset);
 			dmatest_init_dsts(thread->dsts, dst_off, len,
-					  params->buf_size);
+					  params->buf_size, is_memset);
 
 			diff = ktime_sub(ktime_get(), start);
 			filltime = ktime_add(filltime, diff);
@@ -627,22 +640,15 @@ static int dmatest_func(void *data)
 			um->bidi_cnt++;
 		}
 
-		sg_init_table(tx_sg, src_cnt);
-		sg_init_table(rx_sg, src_cnt);
-		for (i = 0; i < src_cnt; i++) {
-			sg_dma_address(&rx_sg[i]) = srcs[i];
-			sg_dma_address(&tx_sg[i]) = dsts[i] + dst_off;
-			sg_dma_len(&tx_sg[i]) = len;
-			sg_dma_len(&rx_sg[i]) = len;
-		}
-
 		if (thread->type == DMA_MEMCPY)
 			tx = dev->device_prep_dma_memcpy(chan,
 							 dsts[0] + dst_off,
 							 srcs[0], len, flags);
-		else if (thread->type == DMA_SG)
-			tx = dev->device_prep_dma_sg(chan, tx_sg, src_cnt,
-						     rx_sg, src_cnt, flags);
+		else if (thread->type == DMA_MEMSET)
+			tx = dev->device_prep_dma_memset(chan,
+						dsts[0] + dst_off,
+						*(thread->srcs[0] + src_off),
+						len, flags);
 		else if (thread->type == DMA_XOR)
 			tx = dev->device_prep_dma_xor(chan,
 						      dsts[0] + dst_off,
@@ -722,23 +728,25 @@ static int dmatest_func(void *data)
 		start = ktime_get();
 		pr_debug("%s: verifying source buffer...\n", current->comm);
 		error_count = dmatest_verify(thread->srcs, 0, src_off,
-				0, PATTERN_SRC, true);
+				0, PATTERN_SRC, true, is_memset);
 		error_count += dmatest_verify(thread->srcs, src_off,
 				src_off + len, src_off,
-				PATTERN_SRC | PATTERN_COPY, true);
+				PATTERN_SRC | PATTERN_COPY, true, is_memset);
 		error_count += dmatest_verify(thread->srcs, src_off + len,
 				params->buf_size, src_off + len,
-				PATTERN_SRC, true);
+				PATTERN_SRC, true, is_memset);
 
 		pr_debug("%s: verifying dest buffer...\n", current->comm);
 		error_count += dmatest_verify(thread->dsts, 0, dst_off,
-				0, PATTERN_DST, false);
+				0, PATTERN_DST, false, is_memset);
+
 		error_count += dmatest_verify(thread->dsts, dst_off,
 				dst_off + len, src_off,
-				PATTERN_SRC | PATTERN_COPY, false);
+				PATTERN_SRC | PATTERN_COPY, false, is_memset);
+
 		error_count += dmatest_verify(thread->dsts, dst_off + len,
 				params->buf_size, dst_off + len,
-				PATTERN_DST, false);
+				PATTERN_DST, false, is_memset);
 
 		diff = ktime_sub(ktime_get(), start);
 		comparetime = ktime_add(comparetime, diff);
@@ -821,8 +829,8 @@ static int dmatest_add_threads(struct dmatest_info *info,
 
 	if (type == DMA_MEMCPY)
 		op = "copy";
-	else if (type == DMA_SG)
-		op = "sg";
+	else if (type == DMA_MEMSET)
+		op = "set";
 	else if (type == DMA_XOR)
 		op = "xor";
 	else if (type == DMA_PQ)
@@ -883,9 +891,9 @@ static int dmatest_add_channel(struct dmatest_info *info,
 		}
 	}
 
-	if (dma_has_cap(DMA_SG, dma_dev->cap_mask)) {
+	if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) {
 		if (dmatest == 1) {
-			cnt = dmatest_add_threads(info, dtc, DMA_SG);
+			cnt = dmatest_add_threads(info, dtc, DMA_MEMSET);
 			thread_count += cnt > 0 ? cnt : 0;
 		}
 	}
@@ -961,8 +969,8 @@ static void run_threaded_test(struct dmatest_info *info)
 	params->noverify = noverify;
 
 	request_channels(info, DMA_MEMCPY);
+	request_channels(info, DMA_MEMSET);
 	request_channels(info, DMA_XOR);
-	request_channels(info, DMA_SG);
 	request_channels(info, DMA_PQ);
 }
 
diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c
index 3b8b752ede2d..3eaece888e75 100644
--- a/drivers/dma/fsldma.c
+++ b/drivers/dma/fsldma.c
@@ -825,122 +825,6 @@ fail:
 	return NULL;
 }
 
-static struct dma_async_tx_descriptor *fsl_dma_prep_sg(struct dma_chan *dchan,
-	struct scatterlist *dst_sg, unsigned int dst_nents,
-	struct scatterlist *src_sg, unsigned int src_nents,
-	unsigned long flags)
-{
-	struct fsl_desc_sw *first = NULL, *prev = NULL, *new = NULL;
-	struct fsldma_chan *chan = to_fsl_chan(dchan);
-	size_t dst_avail, src_avail;
-	dma_addr_t dst, src;
-	size_t len;
-
-	/* basic sanity checks */
-	if (dst_nents == 0 || src_nents == 0)
-		return NULL;
-
-	if (dst_sg == NULL || src_sg == NULL)
-		return NULL;
-
-	/*
-	 * TODO: should we check that both scatterlists have the same
-	 * TODO: number of bytes in total? Is that really an error?
-	 */
-
-	/* get prepared for the loop */
-	dst_avail = sg_dma_len(dst_sg);
-	src_avail = sg_dma_len(src_sg);
-
-	/* run until we are out of scatterlist entries */
-	while (true) {
-
-		/* create the largest transaction possible */
-		len = min_t(size_t, src_avail, dst_avail);
-		len = min_t(size_t, len, FSL_DMA_BCR_MAX_CNT);
-		if (len == 0)
-			goto fetch;
-
-		dst = sg_dma_address(dst_sg) + sg_dma_len(dst_sg) - dst_avail;
-		src = sg_dma_address(src_sg) + sg_dma_len(src_sg) - src_avail;
-
-		/* allocate and populate the descriptor */
-		new = fsl_dma_alloc_descriptor(chan);
-		if (!new) {
-			chan_err(chan, "%s\n", msg_ld_oom);
-			goto fail;
-		}
-
-		set_desc_cnt(chan, &new->hw, len);
-		set_desc_src(chan, &new->hw, src);
-		set_desc_dst(chan, &new->hw, dst);
-
-		if (!first)
-			first = new;
-		else
-			set_desc_next(chan, &prev->hw, new->async_tx.phys);
-
-		new->async_tx.cookie = 0;
-		async_tx_ack(&new->async_tx);
-		prev = new;
-
-		/* Insert the link descriptor to the LD ring */
-		list_add_tail(&new->node, &first->tx_list);
-
-		/* update metadata */
-		dst_avail -= len;
-		src_avail -= len;
-
-fetch:
-		/* fetch the next dst scatterlist entry */
-		if (dst_avail == 0) {
-
-			/* no more entries: we're done */
-			if (dst_nents == 0)
-				break;
-
-			/* fetch the next entry: if there are no more: done */
-			dst_sg = sg_next(dst_sg);
-			if (dst_sg == NULL)
-				break;
-
-			dst_nents--;
-			dst_avail = sg_dma_len(dst_sg);
-		}
-
-		/* fetch the next src scatterlist entry */
-		if (src_avail == 0) {
-
-			/* no more entries: we're done */
-			if (src_nents == 0)
-				break;
-
-			/* fetch the next entry: if there are no more: done */
-			src_sg = sg_next(src_sg);
-			if (src_sg == NULL)
-				break;
-
-			src_nents--;
-			src_avail = sg_dma_len(src_sg);
-		}
-	}
-
-	new->async_tx.flags = flags; /* client is in control of this ack */
-	new->async_tx.cookie = -EBUSY;
-
-	/* Set End-of-link to the last link descriptor of new list */
-	set_ld_eol(chan, new);
-
-	return &first->async_tx;
-
-fail:
-	if (!first)
-		return NULL;
-
-	fsldma_free_desc_list_reverse(chan, &first->tx_list);
-	return NULL;
-}
-
 static int fsl_dma_device_terminate_all(struct dma_chan *dchan)
 {
 	struct fsldma_chan *chan;
@@ -1357,12 +1241,10 @@ static int fsldma_of_probe(struct platform_device *op)
 	fdev->irq = irq_of_parse_and_map(op->dev.of_node, 0);
 
 	dma_cap_set(DMA_MEMCPY, fdev->common.cap_mask);
-	dma_cap_set(DMA_SG, fdev->common.cap_mask);
 	dma_cap_set(DMA_SLAVE, fdev->common.cap_mask);
 	fdev->common.device_alloc_chan_resources = fsl_dma_alloc_chan_resources;
 	fdev->common.device_free_chan_resources = fsl_dma_free_chan_resources;
 	fdev->common.device_prep_dma_memcpy = fsl_dma_prep_memcpy;
-	fdev->common.device_prep_dma_sg = fsl_dma_prep_sg;
 	fdev->common.device_tx_status = fsl_tx_status;
 	fdev->common.device_issue_pending = fsl_dma_memcpy_issue_pending;
 	fdev->common.device_config = fsl_dma_device_config;
diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
index a371b07a0981..f70cc74032ea 100644
--- a/drivers/dma/ioat/dma.c
+++ b/drivers/dma/ioat/dma.c
@@ -644,9 +644,13 @@ static void __cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete)
 		mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
 	}
 
-	/* 5 microsecond delay per pending descriptor */
-	writew(min((5 * (active - i)), IOAT_INTRDELAY_MASK),
-	       ioat_chan->ioat_dma->reg_base + IOAT_INTRDELAY_OFFSET);
+	/* microsecond delay by sysfs variable  per pending descriptor */
+	if (ioat_chan->intr_coalesce != ioat_chan->prev_intr_coalesce) {
+		writew(min((ioat_chan->intr_coalesce * (active - i)),
+		       IOAT_INTRDELAY_MASK),
+		       ioat_chan->ioat_dma->reg_base + IOAT_INTRDELAY_OFFSET);
+		ioat_chan->prev_intr_coalesce = ioat_chan->intr_coalesce;
+	}
 }
 
 static void ioat_cleanup(struct ioatdma_chan *ioat_chan)
diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h
index a9bc1a15b0d1..56200eefcf5e 100644
--- a/drivers/dma/ioat/dma.h
+++ b/drivers/dma/ioat/dma.h
@@ -142,11 +142,14 @@ struct ioatdma_chan {
 	spinlock_t prep_lock;
 	struct ioat_descs descs[2];
 	int desc_chunks;
+	int intr_coalesce;
+	int prev_intr_coalesce;
 };
 
 struct ioat_sysfs_entry {
 	struct attribute attr;
 	ssize_t (*show)(struct dma_chan *, char *);
+	ssize_t (*store)(struct dma_chan *, const char *, size_t);
 };
 
 /**
diff --git a/drivers/dma/ioat/init.c b/drivers/dma/ioat/init.c
index ed8ed1192775..93e006c3441d 100644
--- a/drivers/dma/ioat/init.c
+++ b/drivers/dma/ioat/init.c
@@ -39,7 +39,7 @@ MODULE_VERSION(IOAT_DMA_VERSION);
 MODULE_LICENSE("Dual BSD/GPL");
 MODULE_AUTHOR("Intel Corporation");
 
-static struct pci_device_id ioat_pci_tbl[] = {
+static const struct pci_device_id ioat_pci_tbl[] = {
 	/* I/OAT v3 platforms */
 	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG0) },
 	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG1) },
diff --git a/drivers/dma/ioat/sysfs.c b/drivers/dma/ioat/sysfs.c
index cb4a857ee21b..3ac677f29e8f 100644
--- a/drivers/dma/ioat/sysfs.c
+++ b/drivers/dma/ioat/sysfs.c
@@ -64,8 +64,24 @@ ioat_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
 	return entry->show(&ioat_chan->dma_chan, page);
 }
 
+static ssize_t
+ioat_attr_store(struct kobject *kobj, struct attribute *attr,
+const char *page, size_t count)
+{
+	struct ioat_sysfs_entry *entry;
+	struct ioatdma_chan *ioat_chan;
+
+	entry = container_of(attr, struct ioat_sysfs_entry, attr);
+	ioat_chan = container_of(kobj, struct ioatdma_chan, kobj);
+
+	if (!entry->store)
+		return -EIO;
+	return entry->store(&ioat_chan->dma_chan, page, count);
+}
+
 const struct sysfs_ops ioat_sysfs_ops = {
 	.show	= ioat_attr_show,
+	.store  = ioat_attr_store,
 };
 
 void ioat_kobject_add(struct ioatdma_device *ioat_dma, struct kobj_type *type)
@@ -121,11 +137,37 @@ static ssize_t ring_active_show(struct dma_chan *c, char *page)
 }
 static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active);
 
+static ssize_t intr_coalesce_show(struct dma_chan *c, char *page)
+{
+	struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
+
+	return sprintf(page, "%d\n", ioat_chan->intr_coalesce);
+}
+
+static ssize_t intr_coalesce_store(struct dma_chan *c, const char *page,
+size_t count)
+{
+	int intr_coalesce = 0;
+	struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
+
+	if (sscanf(page, "%du", &intr_coalesce) != -1) {
+		if ((intr_coalesce < 0) ||
+		    (intr_coalesce > IOAT_INTRDELAY_MASK))
+			return -EINVAL;
+		ioat_chan->intr_coalesce = intr_coalesce;
+	}
+
+	return count;
+}
+
+static struct ioat_sysfs_entry intr_coalesce_attr = __ATTR_RW(intr_coalesce);
+
 static struct attribute *ioat_attrs[] = {
 	&ring_size_attr.attr,
 	&ring_active_attr.attr,
 	&ioat_cap_attr.attr,
 	&ioat_version_attr.attr,
+	&intr_coalesce_attr.attr,
 	NULL,
 };
 
diff --git a/drivers/dma/k3dma.c b/drivers/dma/k3dma.c
index 01e25c68dd5a..01d2a750a621 100644
--- a/drivers/dma/k3dma.c
+++ b/drivers/dma/k3dma.c
@@ -223,7 +223,6 @@ static irqreturn_t k3_dma_int_handler(int irq, void *dev_id)
 			if (c && (tc1 & BIT(i))) {
 				spin_lock_irqsave(&c->vc.lock, flags);
 				vchan_cookie_complete(&p->ds_run->vd);
-				WARN_ON_ONCE(p->ds_done);
 				p->ds_done = p->ds_run;
 				p->ds_run = NULL;
 				spin_unlock_irqrestore(&c->vc.lock, flags);
@@ -274,13 +273,14 @@ static int k3_dma_start_txd(struct k3_dma_chan *c)
 		 */
 		list_del(&ds->vd.node);
 
-		WARN_ON_ONCE(c->phy->ds_run);
-		WARN_ON_ONCE(c->phy->ds_done);
 		c->phy->ds_run = ds;
+		c->phy->ds_done = NULL;
 		/* start dma */
 		k3_dma_set_desc(c->phy, &ds->desc_hw[0]);
 		return 0;
 	}
+	c->phy->ds_run = NULL;
+	c->phy->ds_done = NULL;
 	return -EAGAIN;
 }
 
@@ -722,11 +722,7 @@ static int k3_dma_terminate_all(struct dma_chan *chan)
 			k3_dma_free_desc(&p->ds_run->vd);
 			p->ds_run = NULL;
 		}
-		if (p->ds_done) {
-			k3_dma_free_desc(&p->ds_done->vd);
-			p->ds_done = NULL;
-		}
-
+		p->ds_done = NULL;
 	}
 	spin_unlock_irqrestore(&c->vc.lock, flags);
 	vchan_dma_desc_free_list(&c->vc, &head);
diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c
index 25bc5b103aa2..1993889003fd 100644
--- a/drivers/dma/mv_xor.c
+++ b/drivers/dma/mv_xor.c
@@ -68,36 +68,6 @@ static void mv_desc_init(struct mv_xor_desc_slot *desc,
 	hw_desc->byte_count = byte_count;
 }
 
-/* Populate the descriptor */
-static void mv_xor_config_sg_ll_desc(struct mv_xor_desc_slot *desc,
-				     dma_addr_t dma_src, dma_addr_t dma_dst,
-				     u32 len, struct mv_xor_desc_slot *prev)
-{
-	struct mv_xor_desc *hw_desc = desc->hw_desc;
-
-	hw_desc->status = XOR_DESC_DMA_OWNED;
-	hw_desc->phy_next_desc = 0;
-	/* Configure for XOR with only one src address -> MEMCPY */
-	hw_desc->desc_command = XOR_DESC_OPERATION_XOR | (0x1 << 0);
-	hw_desc->phy_dest_addr = dma_dst;
-	hw_desc->phy_src_addr[0] = dma_src;
-	hw_desc->byte_count = len;
-
-	if (prev) {
-		struct mv_xor_desc *hw_prev = prev->hw_desc;
-
-		hw_prev->phy_next_desc = desc->async_tx.phys;
-	}
-}
-
-static void mv_xor_desc_config_eod(struct mv_xor_desc_slot *desc)
-{
-	struct mv_xor_desc *hw_desc = desc->hw_desc;
-
-	/* Enable end-of-descriptor interrupt */
-	hw_desc->desc_command |= XOR_DESC_EOD_INT_EN;
-}
-
 static void mv_desc_set_mode(struct mv_xor_desc_slot *desc)
 {
 	struct mv_xor_desc *hw_desc = desc->hw_desc;
@@ -662,132 +632,6 @@ mv_xor_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
 	return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags);
 }
 
-/**
- * mv_xor_prep_dma_sg - prepare descriptors for a memory sg transaction
- * @chan: DMA channel
- * @dst_sg: Destination scatter list
- * @dst_sg_len: Number of entries in destination scatter list
- * @src_sg: Source scatter list
- * @src_sg_len: Number of entries in source scatter list
- * @flags: transfer ack flags
- *
- * Return: Async transaction descriptor on success and NULL on failure
- */
-static struct dma_async_tx_descriptor *
-mv_xor_prep_dma_sg(struct dma_chan *chan, struct scatterlist *dst_sg,
-		   unsigned int dst_sg_len, struct scatterlist *src_sg,
-		   unsigned int src_sg_len, unsigned long flags)
-{
-	struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
-	struct mv_xor_desc_slot *new;
-	struct mv_xor_desc_slot *first = NULL;
-	struct mv_xor_desc_slot *prev = NULL;
-	size_t len, dst_avail, src_avail;
-	dma_addr_t dma_dst, dma_src;
-	int desc_cnt = 0;
-	int ret;
-
-	dev_dbg(mv_chan_to_devp(mv_chan),
-		"%s dst_sg_len: %d src_sg_len: %d flags: %ld\n",
-		__func__, dst_sg_len, src_sg_len, flags);
-
-	dst_avail = sg_dma_len(dst_sg);
-	src_avail = sg_dma_len(src_sg);
-
-	/* Run until we are out of scatterlist entries */
-	while (true) {
-		/* Allocate and populate the descriptor */
-		desc_cnt++;
-		new = mv_chan_alloc_slot(mv_chan);
-		if (!new) {
-			dev_err(mv_chan_to_devp(mv_chan),
-				"Out of descriptors (desc_cnt=%d)!\n",
-				desc_cnt);
-			goto err;
-		}
-
-		len = min_t(size_t, src_avail, dst_avail);
-		len = min_t(size_t, len, MV_XOR_MAX_BYTE_COUNT);
-		if (len == 0)
-			goto fetch;
-
-		if (len < MV_XOR_MIN_BYTE_COUNT) {
-			dev_err(mv_chan_to_devp(mv_chan),
-				"Transfer size of %zu too small!\n", len);
-			goto err;
-		}
-
-		dma_dst = sg_dma_address(dst_sg) + sg_dma_len(dst_sg) -
-			dst_avail;
-		dma_src = sg_dma_address(src_sg) + sg_dma_len(src_sg) -
-			src_avail;
-
-		/* Check if a new window needs to get added for 'dst' */
-		ret = mv_xor_add_io_win(mv_chan, dma_dst);
-		if (ret)
-			goto err;
-
-		/* Check if a new window needs to get added for 'src' */
-		ret = mv_xor_add_io_win(mv_chan, dma_src);
-		if (ret)
-			goto err;
-
-		/* Populate the descriptor */
-		mv_xor_config_sg_ll_desc(new, dma_src, dma_dst, len, prev);
-		prev = new;
-		dst_avail -= len;
-		src_avail -= len;
-
-		if (!first)
-			first = new;
-		else
-			list_move_tail(&new->node, &first->sg_tx_list);
-
-fetch:
-		/* Fetch the next dst scatterlist entry */
-		if (dst_avail == 0) {
-			if (dst_sg_len == 0)
-				break;
-
-			/* Fetch the next entry: if there are no more: done */
-			dst_sg = sg_next(dst_sg);
-			if (dst_sg == NULL)
-				break;
-
-			dst_sg_len--;
-			dst_avail = sg_dma_len(dst_sg);
-		}
-
-		/* Fetch the next src scatterlist entry */
-		if (src_avail == 0) {
-			if (src_sg_len == 0)
-				break;
-
-			/* Fetch the next entry: if there are no more: done */
-			src_sg = sg_next(src_sg);
-			if (src_sg == NULL)
-				break;
-
-			src_sg_len--;
-			src_avail = sg_dma_len(src_sg);
-		}
-	}
-
-	/* Set the EOD flag in the last descriptor */
-	mv_xor_desc_config_eod(new);
-	first->async_tx.flags = flags;
-
-	return &first->async_tx;
-
-err:
-	/* Cleanup: Move all descriptors back into the free list */
-	spin_lock_bh(&mv_chan->lock);
-	mv_desc_clean_slot(first, mv_chan);
-	spin_unlock_bh(&mv_chan->lock);
-
-	return NULL;
-}
-
 static void mv_xor_free_chan_resources(struct dma_chan *chan)
 {
 	struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
@@ -1254,8 +1098,6 @@ mv_xor_channel_add(struct mv_xor_device *xordev,
 		dma_dev->device_prep_dma_interrupt = mv_xor_prep_dma_interrupt;
 	if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
 		dma_dev->device_prep_dma_memcpy = mv_xor_prep_dma_memcpy;
-	if (dma_has_cap(DMA_SG, dma_dev->cap_mask))
-		dma_dev->device_prep_dma_sg = mv_xor_prep_dma_sg;
 	if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
 		dma_dev->max_xor = 8;
 		dma_dev->device_prep_dma_xor = mv_xor_prep_dma_xor;
@@ -1305,11 +1147,10 @@ mv_xor_channel_add(struct mv_xor_device *xordev,
 			goto err_free_irq;
 	}
 
-	dev_info(&pdev->dev, "Marvell XOR (%s): ( %s%s%s%s)\n",
+	dev_info(&pdev->dev, "Marvell XOR (%s): ( %s%s%s)\n",
 		 mv_chan->op_in_desc ? "Descriptor Mode" : "Registers Mode",
 		 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
 		 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
-		 dma_has_cap(DMA_SG, dma_dev->cap_mask) ? "sg " : "",
 		 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
 
 	dma_async_device_register(dma_dev);
@@ -1552,7 +1393,6 @@ static int mv_xor_probe(struct platform_device *pdev)
 
 			dma_cap_zero(cap_mask);
 			dma_cap_set(DMA_MEMCPY, cap_mask);
-			dma_cap_set(DMA_SG, cap_mask);
 			dma_cap_set(DMA_XOR, cap_mask);
 			dma_cap_set(DMA_INTERRUPT, cap_mask);
 
diff --git a/drivers/dma/nbpfaxi.c b/drivers/dma/nbpfaxi.c
index 3f45b9bdf201..d3f918a9ee76 100644
--- a/drivers/dma/nbpfaxi.c
+++ b/drivers/dma/nbpfaxi.c
@@ -1005,21 +1005,6 @@ static struct dma_async_tx_descriptor *nbpf_prep_memcpy(
 			    DMA_MEM_TO_MEM, flags);
 }
 
-static struct dma_async_tx_descriptor *nbpf_prep_memcpy_sg(
-	struct dma_chan *dchan,
-	struct scatterlist *dst_sg, unsigned int dst_nents,
-	struct scatterlist *src_sg, unsigned int src_nents,
-	unsigned long flags)
-{
-	struct nbpf_channel *chan = nbpf_to_chan(dchan);
-
-	if (dst_nents != src_nents)
-		return NULL;
-
-	return nbpf_prep_sg(chan, src_sg, dst_sg, src_nents,
-			    DMA_MEM_TO_MEM, flags);
-}
-
 static struct dma_async_tx_descriptor *nbpf_prep_slave_sg(
 	struct dma_chan *dchan, struct scatterlist *sgl, unsigned int sg_len,
 	enum dma_transfer_direction direction, unsigned long flags, void *context)
@@ -1417,13 +1402,11 @@ static int nbpf_probe(struct platform_device *pdev)
 	dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
 	dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
 	dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask);
-	dma_cap_set(DMA_SG, dma_dev->cap_mask);
 
 	/* Common and MEMCPY operations */
 	dma_dev->device_alloc_chan_resources
 		= nbpf_alloc_chan_resources;
 	dma_dev->device_free_chan_resources = nbpf_free_chan_resources;
-	dma_dev->device_prep_dma_sg = nbpf_prep_memcpy_sg;
 	dma_dev->device_prep_dma_memcpy = nbpf_prep_memcpy;
 	dma_dev->device_tx_status = nbpf_tx_status;
 	dma_dev->device_issue_pending = nbpf_issue_pending;
diff --git a/drivers/dma/of-dma.c b/drivers/dma/of-dma.c
index faae0bfe1109..91fd395c90c4 100644
--- a/drivers/dma/of-dma.c
+++ b/drivers/dma/of-dma.c
@@ -38,8 +38,8 @@ static struct of_dma *of_dma_find_controller(struct of_phandle_args *dma_spec)
 		if (ofdma->of_node == dma_spec->np)
 			return ofdma;
 
-	pr_debug("%s: can't find DMA controller %s\n", __func__,
-		 dma_spec->np->full_name);
+	pr_debug("%s: can't find DMA controller %pOF\n", __func__,
+		 dma_spec->np);
 
 	return NULL;
 }
@@ -255,8 +255,8 @@ struct dma_chan *of_dma_request_slave_channel(struct device_node *np,
 
 	count = of_property_count_strings(np, "dma-names");
 	if (count < 0) {
-		pr_err("%s: dma-names property of node '%s' missing or empty\n",
-			__func__, np->full_name);
+		pr_err("%s: dma-names property of node '%pOF' missing or empty\n",
+			__func__, np);
 		return ERR_PTR(-ENODEV);
 	}
 
diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c
index b19ee04567b5..f122c2a7b9f0 100644
--- a/drivers/dma/pl330.c
+++ b/drivers/dma/pl330.c
@@ -3023,7 +3023,7 @@ static int pl330_remove(struct amba_device *adev)
 	return 0;
 }
 
-static struct amba_id pl330_ids[] = {
+static const struct amba_id pl330_ids[] = {
 	{
 		.id	= 0x00041330,
 		.mask	= 0x000fffff,
diff --git a/drivers/dma/ppc4xx/adma.c b/drivers/dma/ppc4xx/adma.c
index b1535b1fe95c..4cf0d4d0cecf 100644
--- a/drivers/dma/ppc4xx/adma.c
+++ b/drivers/dma/ppc4xx/adma.c
@@ -4040,9 +4040,9 @@ static int ppc440spe_adma_probe(struct platform_device *ofdev)
 		/* it is DMA0 or DMA1 */
 		idx = of_get_property(np, "cell-index", &len);
 		if (!idx || (len != sizeof(u32))) {
-			dev_err(&ofdev->dev, "Device node %s has missing "
+			dev_err(&ofdev->dev, "Device node %pOF has missing "
 				"or invalid cell-index property\n",
-				np->full_name);
+				np);
 			return -EINVAL;
 		}
 		id = *idx;
@@ -4307,7 +4307,7 @@ static int ppc440spe_adma_remove(struct platform_device *ofdev)
  * "poly" allows setting/checking used polynomial (for PPC440SPe only).
  */
 
-static ssize_t show_ppc440spe_devices(struct device_driver *dev, char *buf)
+static ssize_t devices_show(struct device_driver *dev, char *buf)
 {
 	ssize_t size = 0;
 	int i;
@@ -4321,16 +4321,17 @@ static ssize_t show_ppc440spe_devices(struct device_driver *dev, char *buf)
 	}
 	return size;
 }
+static DRIVER_ATTR_RO(devices);
 
-static ssize_t show_ppc440spe_r6enable(struct device_driver *dev, char *buf)
+static ssize_t enable_show(struct device_driver *dev, char *buf)
 {
 	return snprintf(buf, PAGE_SIZE,
 			"PPC440SP(e) RAID-6 capabilities are %sABLED.\n",
 			ppc440spe_r6_enabled ? "EN" : "DIS");
 }
 
-static ssize_t store_ppc440spe_r6enable(struct device_driver *dev,
-					const char *buf, size_t count)
+static ssize_t enable_store(struct device_driver *dev, const char *buf,
+			    size_t count)
 {
 	unsigned long val;
 
@@ -4357,8 +4358,9 @@ static ssize_t store_ppc440spe_r6enable(struct device_driver *dev,
 	}
 	return count;
 }
+static DRIVER_ATTR_RW(enable);
 
-static ssize_t show_ppc440spe_r6poly(struct device_driver *dev, char *buf)
+static ssize_t poly_store(struct device_driver *dev, char *buf)
 {
 	ssize_t size = 0;
 	u32 reg;
@@ -4377,8 +4379,8 @@ static ssize_t show_ppc440spe_r6poly(struct device_driver *dev, char *buf)
 	return size;
 }
 
-static ssize_t store_ppc440spe_r6poly(struct device_driver *dev,
-				      const char *buf, size_t count)
+static ssize_t poly_store(struct device_driver *dev, const char *buf,
+			  size_t count)
 {
 	unsigned long reg, val;
 
@@ -4404,12 +4406,7 @@ static ssize_t store_ppc440spe_r6poly(struct device_driver *dev,
 
 	return count;
 }
-
-static DRIVER_ATTR(devices, S_IRUGO, show_ppc440spe_devices, NULL);
-static DRIVER_ATTR(enable, S_IRUGO | S_IWUSR, show_ppc440spe_r6enable,
-		   store_ppc440spe_r6enable);
-static DRIVER_ATTR(poly, S_IRUGO | S_IWUSR, show_ppc440spe_r6poly,
-		   store_ppc440spe_r6poly);
+static DRIVER_ATTR_RW(poly);
 
 /*
  * Common initialisation for RAID engines; allocate memory for
@@ -4448,8 +4445,7 @@ static int ppc440spe_configure_raid_devices(void)
 	dcr_base = dcr_resource_start(np, 0);
 	dcr_len = dcr_resource_len(np, 0);
 	if (!dcr_base && !dcr_len) {
-		pr_err("%s: can't get DCR registers base/len!\n",
-			np->full_name);
+		pr_err("%pOF: can't get DCR registers base/len!\n", np);
 		of_node_put(np);
 		iounmap(i2o_reg);
 		return -ENODEV;
@@ -4457,7 +4453,7 @@ static int ppc440spe_configure_raid_devices(void)
 
 	i2o_dcr_host = dcr_map(np, dcr_base, dcr_len);
 	if (!DCR_MAP_OK(i2o_dcr_host)) {
-		pr_err("%s: failed to map DCRs!\n", np->full_name);
+		pr_err("%pOF: failed to map DCRs!\n", np);
 		of_node_put(np);
 		iounmap(i2o_reg);
 		return -ENODEV;
@@ -4518,15 +4514,14 @@ static int ppc440spe_configure_raid_devices(void)
 	dcr_base = dcr_resource_start(np, 0);
 	dcr_len = dcr_resource_len(np, 0);
 	if (!dcr_base && !dcr_len) {
-		pr_err("%s: can't get DCR registers base/len!\n",
-			np->full_name);
+		pr_err("%pOF: can't get DCR registers base/len!\n", np);
 		ret = -ENODEV;
 		goto out_mq;
 	}
 
 	ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len);
 	if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) {
-		pr_err("%s: failed to map DCRs!\n", np->full_name);
+		pr_err("%pOF: failed to map DCRs!\n", np);
 		ret = -ENODEV;
 		goto out_mq;
 	}
diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c
index 03c4eb3fd314..6d89fb6a6a92 100644
--- a/drivers/dma/qcom/bam_dma.c
+++ b/drivers/dma/qcom/bam_dma.c
@@ -65,6 +65,7 @@ struct bam_desc_hw {
 #define DESC_FLAG_EOT BIT(14)
 #define DESC_FLAG_EOB BIT(13)
 #define DESC_FLAG_NWD BIT(12)
+#define DESC_FLAG_CMD BIT(11)
 
 struct bam_async_desc {
 	struct virt_dma_desc vd;
@@ -645,6 +646,9 @@ static struct dma_async_tx_descriptor *bam_prep_slave_sg(struct dma_chan *chan,
 		unsigned int curr_offset = 0;
 
 		do {
+			if (flags & DMA_PREP_CMD)
+				desc->flags |= cpu_to_le16(DESC_FLAG_CMD);
+
 			desc->addr = cpu_to_le32(sg_dma_address(sg) +
 						 curr_offset);
 
@@ -960,7 +964,7 @@ static void bam_start_dma(struct bam_chan *bchan)
 
 	/* set any special flags on the last descriptor */
 	if (async_desc->num_desc == async_desc->xfer_len)
-		desc[async_desc->xfer_len - 1].flags =
+		desc[async_desc->xfer_len - 1].flags |=
 					cpu_to_le16(async_desc->flags);
 	else
 		desc[async_desc->xfer_len - 1].flags |=
diff --git a/drivers/dma/qcom/hidma.c b/drivers/dma/qcom/hidma.c
index 34fb6afd229b..e3669850aef4 100644
--- a/drivers/dma/qcom/hidma.c
+++ b/drivers/dma/qcom/hidma.c
@@ -411,7 +411,40 @@ hidma_prep_dma_memcpy(struct dma_chan *dmach, dma_addr_t dest, dma_addr_t src,
 		return NULL;
 
 	hidma_ll_set_transfer_params(mdma->lldev, mdesc->tre_ch,
-				     src, dest, len, flags);
+				     src, dest, len, flags,
+				     HIDMA_TRE_MEMCPY);
+
+	/* Place descriptor in prepared list */
+	spin_lock_irqsave(&mchan->lock, irqflags);
+	list_add_tail(&mdesc->node, &mchan->prepared);
+	spin_unlock_irqrestore(&mchan->lock, irqflags);
+
+	return &mdesc->desc;
+}
+
+static struct dma_async_tx_descriptor *
+hidma_prep_dma_memset(struct dma_chan *dmach, dma_addr_t dest, int value,
+		size_t len, unsigned long flags)
+{
+	struct hidma_chan *mchan = to_hidma_chan(dmach);
+	struct hidma_desc *mdesc = NULL;
+	struct hidma_dev *mdma = mchan->dmadev;
+	unsigned long irqflags;
+
+	/* Get free descriptor */
+	spin_lock_irqsave(&mchan->lock, irqflags);
+	if (!list_empty(&mchan->free)) {
+		mdesc = list_first_entry(&mchan->free, struct hidma_desc, node);
+		list_del(&mdesc->node);
+	}
+	spin_unlock_irqrestore(&mchan->lock, irqflags);
+
+	if (!mdesc)
+		return NULL;
+
+	hidma_ll_set_transfer_params(mdma->lldev, mdesc->tre_ch,
+				     value, dest, len, flags,
+				     HIDMA_TRE_MEMSET);
 
 	/* Place descriptor in prepared list */
 	spin_lock_irqsave(&mchan->lock, irqflags);
@@ -776,6 +809,7 @@ static int hidma_probe(struct platform_device *pdev)
 	pm_runtime_get_sync(dmadev->ddev.dev);
 
 	dma_cap_set(DMA_MEMCPY, dmadev->ddev.cap_mask);
+	dma_cap_set(DMA_MEMSET, dmadev->ddev.cap_mask);
 	if (WARN_ON(!pdev->dev.dma_mask)) {
 		rc = -ENXIO;
 		goto dmafree;
@@ -786,6 +820,7 @@ static int hidma_probe(struct platform_device *pdev)
 	dmadev->dev_trca = trca;
 	dmadev->trca_resource = trca_resource;
 	dmadev->ddev.device_prep_dma_memcpy = hidma_prep_dma_memcpy;
+	dmadev->ddev.device_prep_dma_memset = hidma_prep_dma_memset;
 	dmadev->ddev.device_alloc_chan_resources = hidma_alloc_chan_resources;
 	dmadev->ddev.device_free_chan_resources = hidma_free_chan_resources;
 	dmadev->ddev.device_tx_status = hidma_tx_status;
diff --git a/drivers/dma/qcom/hidma.h b/drivers/dma/qcom/hidma.h
index 41e0aa283828..5f9966e82c0b 100644
--- a/drivers/dma/qcom/hidma.h
+++ b/drivers/dma/qcom/hidma.h
@@ -28,6 +28,11 @@
 #define HIDMA_TRE_DEST_LOW_IDX		4
 #define HIDMA_TRE_DEST_HI_IDX		5
 
+enum tre_type {
+	HIDMA_TRE_MEMCPY = 3,
+	HIDMA_TRE_MEMSET = 4,
+};
+
 struct hidma_tre {
 	atomic_t allocated;		/* if this channel is allocated	    */
 	bool queued;			/* flag whether this is pending     */
@@ -150,7 +155,7 @@ void hidma_ll_start(struct hidma_lldev *llhndl);
 int hidma_ll_disable(struct hidma_lldev *lldev);
 int hidma_ll_enable(struct hidma_lldev *llhndl);
 void hidma_ll_set_transfer_params(struct hidma_lldev *llhndl, u32 tre_ch,
-	dma_addr_t src, dma_addr_t dest, u32 len, u32 flags);
+	dma_addr_t src, dma_addr_t dest, u32 len, u32 flags, u32 txntype);
 void hidma_ll_setup_irq(struct hidma_lldev *lldev, bool msi);
 int hidma_ll_setup(struct hidma_lldev *lldev);
 struct hidma_lldev *hidma_ll_init(struct device *dev, u32 max_channels,
diff --git a/drivers/dma/qcom/hidma_ll.c b/drivers/dma/qcom/hidma_ll.c
index 1530a661518d..4999e266b2de 100644
--- a/drivers/dma/qcom/hidma_ll.c
+++ b/drivers/dma/qcom/hidma_ll.c
@@ -105,10 +105,6 @@ enum ch_state {
 	HIDMA_CH_STOPPED = 4,
 };
 
-enum tre_type {
-	HIDMA_TRE_MEMCPY = 3,
-};
-
 enum err_code {
 	HIDMA_EVRE_STATUS_COMPLETE = 1,
 	HIDMA_EVRE_STATUS_ERROR = 4,
@@ -174,8 +170,7 @@ int hidma_ll_request(struct hidma_lldev *lldev, u32 sig, const char *dev_name,
 	tre->err_info = 0;
 	tre->lldev = lldev;
 	tre_local = &tre->tre_local[0];
-	tre_local[HIDMA_TRE_CFG_IDX] = HIDMA_TRE_MEMCPY;
-	tre_local[HIDMA_TRE_CFG_IDX] |= (lldev->chidx & 0xFF) << 8;
+	tre_local[HIDMA_TRE_CFG_IDX] = (lldev->chidx & 0xFF) << 8;
 	tre_local[HIDMA_TRE_CFG_IDX] |= BIT(16);	/* set IEOB */
 	*tre_ch = i;
 	if (callback)
@@ -607,7 +602,7 @@ int hidma_ll_disable(struct hidma_lldev *lldev)
 
 void hidma_ll_set_transfer_params(struct hidma_lldev *lldev, u32 tre_ch,
 				  dma_addr_t src, dma_addr_t dest, u32 len,
-				  u32 flags)
+				  u32 flags, u32 txntype)
 {
 	struct hidma_tre *tre;
 	u32 *tre_local;
@@ -626,6 +621,8 @@ void hidma_ll_set_transfer_params(struct hidma_lldev *lldev, u32 tre_ch,
 	}
 
 	tre_local = &tre->tre_local[0];
+	tre_local[HIDMA_TRE_CFG_IDX] &= ~GENMASK(7, 0);
+	tre_local[HIDMA_TRE_CFG_IDX] |= txntype;
 	tre_local[HIDMA_TRE_LEN_IDX] = len;
 	tre_local[HIDMA_TRE_SRC_LOW_IDX] = lower_32_bits(src);
 	tre_local[HIDMA_TRE_SRC_HI_IDX] = upper_32_bits(src);
diff --git a/drivers/dma/qcom/hidma_mgmt.c b/drivers/dma/qcom/hidma_mgmt.c
index 5a0991bc4787..7335e2eb9b72 100644
--- a/drivers/dma/qcom/hidma_mgmt.c
+++ b/drivers/dma/qcom/hidma_mgmt.c
@@ -28,7 +28,7 @@
 
 #include "hidma_mgmt.h"
 
-#define HIDMA_QOS_N_OFFSET		0x300
+#define HIDMA_QOS_N_OFFSET		0x700
 #define HIDMA_CFG_OFFSET		0x400
 #define HIDMA_MAX_BUS_REQ_LEN_OFFSET	0x41C
 #define HIDMA_MAX_XACTIONS_OFFSET	0x420
@@ -227,7 +227,8 @@ static int hidma_mgmt_probe(struct platform_device *pdev)
 		goto out;
 	}
 
-	if (max_write_request) {
+	if (max_write_request &&
+			(max_write_request != mgmtdev->max_write_request)) {
 		dev_info(&pdev->dev, "overriding max-write-burst-bytes: %d\n",
 			max_write_request);
 		mgmtdev->max_write_request = max_write_request;
@@ -240,7 +241,8 @@ static int hidma_mgmt_probe(struct platform_device *pdev)
 		dev_err(&pdev->dev, "max-read-burst-bytes missing\n");
 		goto out;
 	}
-	if (max_read_request) {
+	if (max_read_request &&
+			(max_read_request != mgmtdev->max_read_request)) {
 		dev_info(&pdev->dev, "overriding max-read-burst-bytes: %d\n",
 			max_read_request);
 		mgmtdev->max_read_request = max_read_request;
@@ -253,7 +255,8 @@ static int hidma_mgmt_probe(struct platform_device *pdev)
 		dev_err(&pdev->dev, "max-write-transactions missing\n");
 		goto out;
 	}
-	if (max_wr_xactions) {
+	if (max_wr_xactions &&
+			(max_wr_xactions != mgmtdev->max_wr_xactions)) {
 		dev_info(&pdev->dev, "overriding max-write-transactions: %d\n",
 			max_wr_xactions);
 		mgmtdev->max_wr_xactions = max_wr_xactions;
@@ -266,7 +269,8 @@ static int hidma_mgmt_probe(struct platform_device *pdev)
 		dev_err(&pdev->dev, "max-read-transactions missing\n");
 		goto out;
 	}
-	if (max_rd_xactions) {
+	if (max_rd_xactions &&
+			(max_rd_xactions != mgmtdev->max_rd_xactions)) {
 		dev_info(&pdev->dev, "overriding max-read-transactions: %d\n",
 			max_rd_xactions);
 		mgmtdev->max_rd_xactions = max_rd_xactions;
@@ -354,7 +358,7 @@ static int __init hidma_mgmt_of_populate_channels(struct device_node *np)
 	struct platform_device_info pdevinfo;
 	struct of_phandle_args out_irq;
 	struct device_node *child;
-	struct resource *res;
+	struct resource *res = NULL;
 	const __be32 *cell;
 	int ret = 0, size, i, num;
 	u64 addr, addr_size;
diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c
index ffcadca53243..2b2c7db3e480 100644
--- a/drivers/dma/sh/rcar-dmac.c
+++ b/drivers/dma/sh/rcar-dmac.c
@@ -1690,6 +1690,15 @@ static int rcar_dmac_chan_probe(struct rcar_dmac *dmac,
 	if (!irqname)
 		return -ENOMEM;
 
+	/*
+	 * Initialize the DMA engine channel and add it to the DMA engine
+	 * channels list.
+	 */
+	chan->device = &dmac->engine;
+	dma_cookie_init(chan);
+
+	list_add_tail(&chan->device_node, &dmac->engine.channels);
+
 	ret = devm_request_threaded_irq(dmac->dev, rchan->irq,
 					rcar_dmac_isr_channel,
 					rcar_dmac_isr_channel_thread, 0,
@@ -1700,15 +1709,6 @@ static int rcar_dmac_chan_probe(struct rcar_dmac *dmac,
 		return ret;
 	}
 
-	/*
-	 * Initialize the DMA engine channel and add it to the DMA engine
-	 * channels list.
-	 */
-	chan->device = &dmac->engine;
-	dma_cookie_init(chan);
-
-	list_add_tail(&chan->device_node, &dmac->engine.channels);
-
 	return 0;
 }
 
@@ -1794,14 +1794,6 @@ static int rcar_dmac_probe(struct platform_device *pdev)
 	if (!irqname)
 		return -ENOMEM;
 
-	ret = devm_request_irq(&pdev->dev, irq, rcar_dmac_isr_error, 0,
-			       irqname, dmac);
-	if (ret) {
-		dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n",
-			irq, ret);
-		return ret;
-	}
-
 	/* Enable runtime PM and initialize the device. */
 	pm_runtime_enable(&pdev->dev);
 	ret = pm_runtime_get_sync(&pdev->dev);
@@ -1818,8 +1810,32 @@ static int rcar_dmac_probe(struct platform_device *pdev)
 		goto error;
 	}
 
-	/* Initialize the channels. */
-	INIT_LIST_HEAD(&dmac->engine.channels);
+	/* Initialize engine */
+	engine = &dmac->engine;
+
+	dma_cap_set(DMA_MEMCPY, engine->cap_mask);
+	dma_cap_set(DMA_SLAVE, engine->cap_mask);
+
+	engine->dev		= &pdev->dev;
+	engine->copy_align	= ilog2(RCAR_DMAC_MEMCPY_XFER_SIZE);
+
+	engine->src_addr_widths	= widths;
+	engine->dst_addr_widths	= widths;
+	engine->directions	= BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM);
+	engine->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
+
+	engine->device_alloc_chan_resources	= rcar_dmac_alloc_chan_resources;
+	engine->device_free_chan_resources	= rcar_dmac_free_chan_resources;
+	engine->device_prep_dma_memcpy		= rcar_dmac_prep_dma_memcpy;
+	engine->device_prep_slave_sg		= rcar_dmac_prep_slave_sg;
+	engine->device_prep_dma_cyclic		= rcar_dmac_prep_dma_cyclic;
+	engine->device_config			= rcar_dmac_device_config;
+	engine->device_terminate_all		= rcar_dmac_chan_terminate_all;
+	engine->device_tx_status		= rcar_dmac_tx_status;
+	engine->device_issue_pending		= rcar_dmac_issue_pending;
+	engine->device_synchronize		= rcar_dmac_device_synchronize;
+
+	INIT_LIST_HEAD(&engine->channels);
 
 	for (i = 0; i < dmac->n_channels; ++i) {
 		ret = rcar_dmac_chan_probe(dmac, &dmac->channels[i],
@@ -1828,6 +1844,14 @@ static int rcar_dmac_probe(struct platform_device *pdev)
 			goto error;
 	}
 
+	ret = devm_request_irq(&pdev->dev, irq, rcar_dmac_isr_error, 0,
+			       irqname, dmac);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n",
+			irq, ret);
+		return ret;
+	}
+
 	/* Register the DMAC as a DMA provider for DT. */
 	ret = of_dma_controller_register(pdev->dev.of_node, rcar_dmac_of_xlate,
 					 NULL);
@@ -1839,29 +1863,6 @@ static int rcar_dmac_probe(struct platform_device *pdev)
 	 *
 	 * Default transfer size of 32 bytes requires 32-byte alignment.
 	 */
-	engine = &dmac->engine;
-	dma_cap_set(DMA_MEMCPY, engine->cap_mask);
-	dma_cap_set(DMA_SLAVE, engine->cap_mask);
-
-	engine->dev = &pdev->dev;
-	engine->copy_align = ilog2(RCAR_DMAC_MEMCPY_XFER_SIZE);
-
-	engine->src_addr_widths = widths;
-	engine->dst_addr_widths = widths;
-	engine->directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM);
-	engine->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
-
-	engine->device_alloc_chan_resources = rcar_dmac_alloc_chan_resources;
-	engine->device_free_chan_resources = rcar_dmac_free_chan_resources;
-	engine->device_prep_dma_memcpy = rcar_dmac_prep_dma_memcpy;
-	engine->device_prep_slave_sg = rcar_dmac_prep_slave_sg;
-	engine->device_prep_dma_cyclic = rcar_dmac_prep_dma_cyclic;
-	engine->device_config = rcar_dmac_device_config;
-	engine->device_terminate_all = rcar_dmac_chan_terminate_all;
-	engine->device_tx_status = rcar_dmac_tx_status;
-	engine->device_issue_pending = rcar_dmac_issue_pending;
-	engine->device_synchronize = rcar_dmac_device_synchronize;
-
 	ret = dma_async_device_register(engine);
 	if (ret < 0)
 		goto error;
diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c
index c3052fbfd092..c2b089af0420 100644
--- a/drivers/dma/ste_dma40.c
+++ b/drivers/dma/ste_dma40.c
@@ -79,7 +79,7 @@ static int dma40_memcpy_channels[] = {
 };
 
 /* Default configuration for physcial memcpy */
-static struct stedma40_chan_cfg dma40_memcpy_conf_phy = {
+static const struct stedma40_chan_cfg dma40_memcpy_conf_phy = {
 	.mode = STEDMA40_MODE_PHYSICAL,
 	.dir = DMA_MEM_TO_MEM,
 
@@ -93,7 +93,7 @@ static struct stedma40_chan_cfg dma40_memcpy_conf_phy = {
 };
 
 /* Default configuration for logical memcpy */
-static struct stedma40_chan_cfg dma40_memcpy_conf_log = {
+static const struct stedma40_chan_cfg dma40_memcpy_conf_log = {
 	.mode = STEDMA40_MODE_LOGICAL,
 	.dir = DMA_MEM_TO_MEM,
 
@@ -2485,19 +2485,6 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
 }
 
 static struct dma_async_tx_descriptor *
-d40_prep_memcpy_sg(struct dma_chan *chan,
-		   struct scatterlist *dst_sg, unsigned int dst_nents,
-		   struct scatterlist *src_sg, unsigned int src_nents,
-		   unsigned long dma_flags)
-{
-	if (dst_nents != src_nents)
-		return NULL;
-
-	return d40_prep_sg(chan, src_sg, dst_sg, src_nents,
-			   DMA_MEM_TO_MEM, dma_flags);
-}
-
-static struct dma_async_tx_descriptor *
 d40_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 		  unsigned int sg_len, enum dma_transfer_direction direction,
 		  unsigned long dma_flags, void *context)
@@ -2821,9 +2808,6 @@ static void d40_ops_init(struct d40_base *base, struct dma_device *dev)
 		dev->copy_align = DMAENGINE_ALIGN_4_BYTES;
 	}
 
-	if (dma_has_cap(DMA_SG, dev->cap_mask))
-		dev->device_prep_dma_sg = d40_prep_memcpy_sg;
-
 	if (dma_has_cap(DMA_CYCLIC, dev->cap_mask))
 		dev->device_prep_dma_cyclic = dma40_prep_dma_cyclic;
 
@@ -2865,7 +2849,6 @@ static int __init d40_dmaengine_init(struct d40_base *base,
 
 	dma_cap_zero(base->dma_memcpy.cap_mask);
 	dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask);
-	dma_cap_set(DMA_SG, base->dma_memcpy.cap_mask);
 
 	d40_ops_init(base, &base->dma_memcpy);
 
@@ -2883,7 +2866,6 @@ static int __init d40_dmaengine_init(struct d40_base *base,
 	dma_cap_zero(base->dma_both.cap_mask);
 	dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask);
 	dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask);
-	dma_cap_set(DMA_SG, base->dma_both.cap_mask);
 	dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask);
 
 	d40_ops_init(base, &base->dma_both);
diff --git a/drivers/dma/sun6i-dma.c b/drivers/dma/sun6i-dma.c
index a2358780ab2c..bcd496edc70f 100644
--- a/drivers/dma/sun6i-dma.c
+++ b/drivers/dma/sun6i-dma.c
@@ -101,6 +101,17 @@ struct sun6i_dma_config {
 	u32 nr_max_channels;
 	u32 nr_max_requests;
 	u32 nr_max_vchans;
+	/*
+	 * In the datasheets/user manuals of newer Allwinner SoCs, a special
+	 * bit (bit 2 at register 0x20) is present.
+	 * It's named "DMA MCLK interface circuit auto gating bit" in the
+	 * documents, and the footnote of this register says that this bit
+	 * should be set up when initializing the DMA controller.
+	 * Allwinner A23/A33 user manuals do not have this bit documented,
+	 * however these SoCs really have and need this bit, as seen in the
+	 * BSP kernel source code.
+	 */
+	bool gate_needed;
 };
 
 /*
@@ -1009,6 +1020,7 @@ static struct sun6i_dma_config sun8i_a23_dma_cfg = {
 	.nr_max_channels = 8,
 	.nr_max_requests = 24,
 	.nr_max_vchans   = 37,
+	.gate_needed	 = true,
 };
 
 static struct sun6i_dma_config sun8i_a83t_dma_cfg = {
@@ -1028,11 +1040,24 @@ static struct sun6i_dma_config sun8i_h3_dma_cfg = {
 	.nr_max_vchans   = 34,
 };
 
+/*
+ * The V3s have only 8 physical channels, a maximum DRQ port id of 23,
+ * and a total of 24 usable source and destination endpoints.
+ */
+
+static struct sun6i_dma_config sun8i_v3s_dma_cfg = {
+	.nr_max_channels = 8,
+	.nr_max_requests = 23,
+	.nr_max_vchans   = 24,
+	.gate_needed	 = true,
+};
+
 static const struct of_device_id sun6i_dma_match[] = {
 	{ .compatible = "allwinner,sun6i-a31-dma", .data = &sun6i_a31_dma_cfg },
 	{ .compatible = "allwinner,sun8i-a23-dma", .data = &sun8i_a23_dma_cfg },
 	{ .compatible = "allwinner,sun8i-a83t-dma", .data = &sun8i_a83t_dma_cfg },
 	{ .compatible = "allwinner,sun8i-h3-dma", .data = &sun8i_h3_dma_cfg },
+	{ .compatible = "allwinner,sun8i-v3s-dma", .data = &sun8i_v3s_dma_cfg },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, sun6i_dma_match);
@@ -1174,13 +1199,7 @@ static int sun6i_dma_probe(struct platform_device *pdev)
 		goto err_dma_unregister;
 	}
 
-	/*
-	 * sun8i variant requires us to toggle a dma gating register,
-	 * as seen in Allwinner's SDK. This register is not documented
-	 * in the A23 user manual.
-	 */
-	if (of_device_is_compatible(pdev->dev.of_node,
-				    "allwinner,sun8i-a23-dma"))
+	if (sdc->cfg->gate_needed)
 		writel(SUN8I_DMA_GATE_ENABLE, sdc->base + SUN8I_DMA_GATE);
 
 	return 0;
diff --git a/drivers/dma/tegra210-adma.c b/drivers/dma/tegra210-adma.c
index b10cbaa82ff5..b26256f23d67 100644
--- a/drivers/dma/tegra210-adma.c
+++ b/drivers/dma/tegra210-adma.c
@@ -717,8 +717,8 @@ static int tegra_adma_probe(struct platform_device *pdev)
 		tdc->chan_addr = tdma->base_addr + ADMA_CH_REG_OFFSET(i);
 
 		tdc->irq = of_irq_get(pdev->dev.of_node, i);
-		if (tdc->irq < 0) {
-			ret = tdc->irq;
+		if (tdc->irq <= 0) {
+			ret = tdc->irq ?: -ENXIO;
 			goto irq_dispose;
 		}
 
diff --git a/drivers/dma/ti-dma-crossbar.c b/drivers/dma/ti-dma-crossbar.c
index 2403475a37cf..2f65a8fde21d 100644
--- a/drivers/dma/ti-dma-crossbar.c
+++ b/drivers/dma/ti-dma-crossbar.c
@@ -308,7 +308,7 @@ static const struct of_device_id ti_dra7_master_match[] = {
 static inline void ti_dra7_xbar_reserve(int offset, int len, unsigned long *p)
 {
 	for (; len > 0; len--)
-		clear_bit(offset + (len - 1), p);
+		set_bit(offset + (len - 1), p);
 }
 
 static int ti_dra7_xbar_probe(struct platform_device *pdev)
diff --git a/drivers/dma/xgene-dma.c b/drivers/dma/xgene-dma.c
index 8b693b712d0f..1d5988849aa6 100644
--- a/drivers/dma/xgene-dma.c
+++ b/drivers/dma/xgene-dma.c
@@ -391,11 +391,6 @@ static void xgene_dma_set_src_buffer(__le64 *ext8, size_t *len,
 	*paddr += nbytes;
 }
 
-static void xgene_dma_invalidate_buffer(__le64 *ext8)
-{
-	*ext8 |= cpu_to_le64(XGENE_DMA_INVALID_LEN_CODE);
-}
-
 static __le64 *xgene_dma_lookup_ext8(struct xgene_dma_desc_hw *desc, int idx)
 {
 	switch (idx) {
@@ -425,48 +420,6 @@ static void xgene_dma_init_desc(struct xgene_dma_desc_hw *desc,
 				XGENE_DMA_DESC_HOENQ_NUM_POS);
 }
 
-static void xgene_dma_prep_cpy_desc(struct xgene_dma_chan *chan,
-				    struct xgene_dma_desc_sw *desc_sw,
-				    dma_addr_t dst, dma_addr_t src,
-				    size_t len)
-{
-	struct xgene_dma_desc_hw *desc1, *desc2;
-	int i;
-
-	/* Get 1st descriptor */
-	desc1 = &desc_sw->desc1;
-	xgene_dma_init_desc(desc1, chan->tx_ring.dst_ring_num);
-
-	/* Set destination address */
-	desc1->m2 |= cpu_to_le64(XGENE_DMA_DESC_DR_BIT);
-	desc1->m3 |= cpu_to_le64(dst);
-
-	/* Set 1st source address */
-	xgene_dma_set_src_buffer(&desc1->m1, &len, &src);
-
-	if (!len)
-		return;
-
-	/*
-	 * We need to split this source buffer,
-	 * and need to use 2nd descriptor
-	 */
-	desc2 = &desc_sw->desc2;
-	desc1->m0 |= cpu_to_le64(XGENE_DMA_DESC_NV_BIT);
-
-	/* Set 2nd to 5th source address */
-	for (i = 0; i < 4 && len; i++)
-		xgene_dma_set_src_buffer(xgene_dma_lookup_ext8(desc2, i),
-					 &len, &src);
-
-	/* Invalidate unused source address field */
-	for (; i < 4; i++)
-		xgene_dma_invalidate_buffer(xgene_dma_lookup_ext8(desc2, i));
-
-	/* Updated flag that we have prepared 64B descriptor */
-	desc_sw->flags |= XGENE_DMA_FLAG_64B_DESC;
-}
-
 static void xgene_dma_prep_xor_desc(struct xgene_dma_chan *chan,
 				    struct xgene_dma_desc_sw *desc_sw,
 				    dma_addr_t *dst, dma_addr_t *src,
@@ -891,114 +844,6 @@ static void xgene_dma_free_chan_resources(struct dma_chan *dchan)
 	chan->desc_pool = NULL;
 }
 
-static struct dma_async_tx_descriptor *xgene_dma_prep_sg(
-	struct dma_chan *dchan, struct scatterlist *dst_sg,
-	u32 dst_nents, struct scatterlist *src_sg,
-	u32 src_nents, unsigned long flags)
-{
-	struct xgene_dma_desc_sw *first = NULL, *new = NULL;
-	struct xgene_dma_chan *chan;
-	size_t dst_avail, src_avail;
-	dma_addr_t dst, src;
-	size_t len;
-
-	if (unlikely(!dchan))
-		return NULL;
-
-	if (unlikely(!dst_nents || !src_nents))
-		return NULL;
-
-	if (unlikely(!dst_sg || !src_sg))
-		return NULL;
-
-	chan = to_dma_chan(dchan);
-
-	/* Get prepared for the loop */
-	dst_avail = sg_dma_len(dst_sg);
-	src_avail = sg_dma_len(src_sg);
-	dst_nents--;
-	src_nents--;
-
-	/* Run until we are out of scatterlist entries */
-	while (true) {
-		/* Create the largest transaction possible */
-		len = min_t(size_t, src_avail, dst_avail);
-		len = min_t(size_t, len, XGENE_DMA_MAX_64B_DESC_BYTE_CNT);
-		if (len == 0)
-			goto fetch;
-
-		dst = sg_dma_address(dst_sg) + sg_dma_len(dst_sg) - dst_avail;
-		src = sg_dma_address(src_sg) + sg_dma_len(src_sg) - src_avail;
-
-		/* Allocate the link descriptor from DMA pool */
-		new = xgene_dma_alloc_descriptor(chan);
-		if (!new)
-			goto fail;
-
-		/* Prepare DMA descriptor */
-		xgene_dma_prep_cpy_desc(chan, new, dst, src, len);
-
-		if (!first)
-			first = new;
-
-		new->tx.cookie = 0;
-		async_tx_ack(&new->tx);
-
-		/* update metadata */
-		dst_avail -= len;
-		src_avail -= len;
-
-		/* Insert the link descriptor to the LD ring */
-		list_add_tail(&new->node, &first->tx_list);
-
-fetch:
-		/* fetch the next dst scatterlist entry */
-		if (dst_avail == 0) {
-			/* no more entries: we're done */
-			if (dst_nents == 0)
-				break;
-
-			/* fetch the next entry: if there are no more: done */
-			dst_sg = sg_next(dst_sg);
-			if (!dst_sg)
-				break;
-
-			dst_nents--;
-			dst_avail = sg_dma_len(dst_sg);
-		}
-
-		/* fetch the next src scatterlist entry */
-		if (src_avail == 0) {
-			/* no more entries: we're done */
-			if (src_nents == 0)
-				break;
-
-			/* fetch the next entry: if there are no more: done */
-			src_sg = sg_next(src_sg);
-			if (!src_sg)
-				break;
-
-			src_nents--;
-			src_avail = sg_dma_len(src_sg);
-		}
-	}
-
-	if (!new)
-		return NULL;
-
-	new->tx.flags = flags; /* client is in control of this ack */
-	new->tx.cookie = -EBUSY;
-	list_splice(&first->tx_list, &new->tx_list);
-
-	return &new->tx;
-fail:
-	if (!first)
-		return NULL;
-
-	xgene_dma_free_desc_list(chan, &first->tx_list);
-	return NULL;
-}
-
 static struct dma_async_tx_descriptor *xgene_dma_prep_xor(
 	struct dma_chan *dchan, dma_addr_t dst,	dma_addr_t *src,
 	u32 src_cnt, size_t len, unsigned long flags)
@@ -1653,7 +1498,6 @@ static void xgene_dma_set_caps(struct xgene_dma_chan *chan,
 	dma_cap_zero(dma_dev->cap_mask);
 
 	/* Set DMA device capability */
-	dma_cap_set(DMA_SG, dma_dev->cap_mask);
 
 	/* Basically here, the X-Gene SoC DMA engine channel 0 supports XOR
 	 * and channel 1 supports XOR, PQ both. First thing here is we have
@@ -1679,7 +1523,6 @@ static void xgene_dma_set_caps(struct xgene_dma_chan *chan,
 	dma_dev->device_free_chan_resources = xgene_dma_free_chan_resources;
 	dma_dev->device_issue_pending = xgene_dma_issue_pending;
 	dma_dev->device_tx_status = xgene_dma_tx_status;
-	dma_dev->device_prep_dma_sg = xgene_dma_prep_sg;
 
 	if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
 		dma_dev->device_prep_dma_xor = xgene_dma_prep_xor;
@@ -1731,8 +1574,7 @@ static int xgene_dma_async_register(struct xgene_dma *pdma, int id)
 
 	/* DMA capability info */
 	dev_info(pdma->dev,
-		 "%s: CAPABILITY ( %s%s%s)\n", dma_chan_name(&chan->dma_chan),
-		 dma_has_cap(DMA_SG, dma_dev->cap_mask) ? "SGCPY " : "",
+		 "%s: CAPABILITY ( %s%s)\n", dma_chan_name(&chan->dma_chan),
 		 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "XOR " : "",
 		 dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "PQ " : "");
 
diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c
index 8cf87b1a284b..8722bcba489d 100644
--- a/drivers/dma/xilinx/xilinx_dma.c
+++ b/drivers/dma/xilinx/xilinx_dma.c
@@ -2124,7 +2124,7 @@ static int axidma_clk_init(struct platform_device *pdev, struct clk **axi_clk,
 	*axi_clk = devm_clk_get(&pdev->dev, "s_axi_lite_aclk");
 	if (IS_ERR(*axi_clk)) {
 		err = PTR_ERR(*axi_clk);
-		dev_err(&pdev->dev, "failed to get axi_aclk (%u)\n", err);
+		dev_err(&pdev->dev, "failed to get axi_aclk (%d)\n", err);
 		return err;
 	}
 
@@ -2142,25 +2142,25 @@ static int axidma_clk_init(struct platform_device *pdev, struct clk **axi_clk,
 
 	err = clk_prepare_enable(*axi_clk);
 	if (err) {
-		dev_err(&pdev->dev, "failed to enable axi_clk (%u)\n", err);
+		dev_err(&pdev->dev, "failed to enable axi_clk (%d)\n", err);
 		return err;
 	}
 
 	err = clk_prepare_enable(*tx_clk);
 	if (err) {
-		dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err);
+		dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err);
 		goto err_disable_axiclk;
 	}
 
 	err = clk_prepare_enable(*rx_clk);
 	if (err) {
-		dev_err(&pdev->dev, "failed to enable rx_clk (%u)\n", err);
+		dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err);
 		goto err_disable_txclk;
 	}
 
 	err = clk_prepare_enable(*sg_clk);
 	if (err) {
-		dev_err(&pdev->dev, "failed to enable sg_clk (%u)\n", err);
+		dev_err(&pdev->dev, "failed to enable sg_clk (%d)\n", err);
 		goto err_disable_rxclk;
 	}
 
@@ -2189,26 +2189,26 @@ static int axicdma_clk_init(struct platform_device *pdev, struct clk **axi_clk,
 	*axi_clk = devm_clk_get(&pdev->dev, "s_axi_lite_aclk");
 	if (IS_ERR(*axi_clk)) {
 		err = PTR_ERR(*axi_clk);
-		dev_err(&pdev->dev, "failed to get axi_clk (%u)\n", err);
+		dev_err(&pdev->dev, "failed to get axi_clk (%d)\n", err);
 		return err;
 	}
 
 	*dev_clk = devm_clk_get(&pdev->dev, "m_axi_aclk");
 	if (IS_ERR(*dev_clk)) {
 		err = PTR_ERR(*dev_clk);
-		dev_err(&pdev->dev, "failed to get dev_clk (%u)\n", err);
+		dev_err(&pdev->dev, "failed to get dev_clk (%d)\n", err);
 		return err;
 	}
 
 	err = clk_prepare_enable(*axi_clk);
 	if (err) {
-		dev_err(&pdev->dev, "failed to enable axi_clk (%u)\n", err);
+		dev_err(&pdev->dev, "failed to enable axi_clk (%d)\n", err);
 		return err;
 	}
 
 	err = clk_prepare_enable(*dev_clk);
 	if (err) {
-		dev_err(&pdev->dev, "failed to enable dev_clk (%u)\n", err);
+		dev_err(&pdev->dev, "failed to enable dev_clk (%d)\n", err);
 		goto err_disable_axiclk;
 	}
 
@@ -2229,7 +2229,7 @@ static int axivdma_clk_init(struct platform_device *pdev, struct clk **axi_clk,
 	*axi_clk = devm_clk_get(&pdev->dev, "s_axi_lite_aclk");
 	if (IS_ERR(*axi_clk)) {
 		err = PTR_ERR(*axi_clk);
-		dev_err(&pdev->dev, "failed to get axi_aclk (%u)\n", err);
+		dev_err(&pdev->dev, "failed to get axi_aclk (%d)\n", err);
 		return err;
 	}
 
@@ -2251,31 +2251,31 @@ static int axivdma_clk_init(struct platform_device *pdev, struct clk **axi_clk,
 
 	err = clk_prepare_enable(*axi_clk);
 	if (err) {
-		dev_err(&pdev->dev, "failed to enable axi_clk (%u)\n", err);
+		dev_err(&pdev->dev, "failed to enable axi_clk (%d)\n", err);
 		return err;
 	}
 
 	err = clk_prepare_enable(*tx_clk);
 	if (err) {
-		dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err);
+		dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err);
 		goto err_disable_axiclk;
 	}
 
 	err = clk_prepare_enable(*txs_clk);
 	if (err) {
-		dev_err(&pdev->dev, "failed to enable txs_clk (%u)\n", err);
+		dev_err(&pdev->dev, "failed to enable txs_clk (%d)\n", err);
 		goto err_disable_txclk;
 	}
 
 	err = clk_prepare_enable(*rx_clk);
 	if (err) {
-		dev_err(&pdev->dev, "failed to enable rx_clk (%u)\n", err);
+		dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err);
 		goto err_disable_txsclk;
 	}
 
 	err = clk_prepare_enable(*rxs_clk);
 	if (err) {
-		dev_err(&pdev->dev, "failed to enable rxs_clk (%u)\n", err);
+		dev_err(&pdev->dev, "failed to enable rxs_clk (%d)\n", err);
 		goto err_disable_rxclk;
 	}
 
diff --git a/drivers/dma/xilinx/zynqmp_dma.c b/drivers/dma/xilinx/zynqmp_dma.c
index 47f64192d2fd..1ee1241ca797 100644
--- a/drivers/dma/xilinx/zynqmp_dma.c
+++ b/drivers/dma/xilinx/zynqmp_dma.c
@@ -830,98 +830,6 @@ static struct dma_async_tx_descriptor *zynqmp_dma_prep_memcpy(
 }
 
 /**
- * zynqmp_dma_prep_slave_sg - prepare descriptors for a memory sg transaction
- * @dchan: DMA channel
- * @dst_sg: Destination scatter list
- * @dst_sg_len: Number of entries in destination scatter list
- * @src_sg: Source scatter list
- * @src_sg_len: Number of entries in source scatter list
- * @flags: transfer ack flags
- *
- * Return: Async transaction descriptor on success and NULL on failure
- */
-static struct dma_async_tx_descriptor *zynqmp_dma_prep_sg(
-			struct dma_chan *dchan, struct scatterlist *dst_sg,
-			unsigned int dst_sg_len, struct scatterlist *src_sg,
-			unsigned int src_sg_len, unsigned long flags)
-{
-	struct zynqmp_dma_desc_sw *new, *first = NULL;
-	struct zynqmp_dma_chan *chan = to_chan(dchan);
-	void *desc = NULL, *prev = NULL;
-	size_t len, dst_avail, src_avail;
-	dma_addr_t dma_dst, dma_src;
-	u32 desc_cnt = 0, i;
-	struct scatterlist *sg;
-
-	for_each_sg(src_sg, sg, src_sg_len, i)
-		desc_cnt += DIV_ROUND_UP(sg_dma_len(sg),
-					 ZYNQMP_DMA_MAX_TRANS_LEN);
-
-	spin_lock_bh(&chan->lock);
-	if (desc_cnt > chan->desc_free_cnt) {
-		spin_unlock_bh(&chan->lock);
-		dev_dbg(chan->dev, "chan %p descs are not available\n", chan);
-		return NULL;
-	}
-	chan->desc_free_cnt = chan->desc_free_cnt - desc_cnt;
-	spin_unlock_bh(&chan->lock);
-
-	dst_avail = sg_dma_len(dst_sg);
-	src_avail = sg_dma_len(src_sg);
-
-	/* Run until we are out of scatterlist entries */
-	while (true) {
-		/* Allocate and populate the descriptor */
-		new = zynqmp_dma_get_descriptor(chan);
-		desc = (struct zynqmp_dma_desc_ll *)new->src_v;
-		len = min_t(size_t, src_avail, dst_avail);
-		len = min_t(size_t, len, ZYNQMP_DMA_MAX_TRANS_LEN);
-		if (len == 0)
-			goto fetch;
-		dma_dst = sg_dma_address(dst_sg) + sg_dma_len(dst_sg) -
-			dst_avail;
-		dma_src = sg_dma_address(src_sg) + sg_dma_len(src_sg) -
-			src_avail;
-
-		zynqmp_dma_config_sg_ll_desc(chan, desc, dma_src, dma_dst,
-					     len, prev);
-		prev = desc;
-		dst_avail -= len;
-		src_avail -= len;
-
-		if (!first)
-			first = new;
-		else
-			list_add_tail(&new->node, &first->tx_list);
-fetch:
-		/* Fetch the next dst scatterlist entry */
-		if (dst_avail == 0) {
-			if (dst_sg_len == 0)
-				break;
-			dst_sg = sg_next(dst_sg);
-			if (dst_sg == NULL)
-				break;
-			dst_sg_len--;
-			dst_avail = sg_dma_len(dst_sg);
-		}
-		/* Fetch the next src scatterlist entry */
-		if (src_avail == 0) {
-			if (src_sg_len == 0)
-				break;
-			src_sg = sg_next(src_sg);
-			if (src_sg == NULL)
-				break;
-			src_sg_len--;
-			src_avail = sg_dma_len(src_sg);
-		}
-	}
-
-	zynqmp_dma_desc_config_eod(chan, desc);
-	first->async_tx.flags = flags;
-	return &first->async_tx;
-}
-
-/**
  * zynqmp_dma_chan_remove - Channel remove function
  * @chan: ZynqMP DMA channel pointer
  */
@@ -1064,11 +972,9 @@ static int zynqmp_dma_probe(struct platform_device *pdev)
 	INIT_LIST_HEAD(&zdev->common.channels);
 
 	dma_set_mask(&pdev->dev, DMA_BIT_MASK(44));
-	dma_cap_set(DMA_SG, zdev->common.cap_mask);
 	dma_cap_set(DMA_MEMCPY, zdev->common.cap_mask);
 
 	p = &zdev->common;
-	p->device_prep_dma_sg = zynqmp_dma_prep_sg;
 	p->device_prep_dma_memcpy = zynqmp_dma_prep_memcpy;
 	p->device_terminate_all = zynqmp_dma_device_terminate_all;
 	p->device_issue_pending = zynqmp_dma_issue_pending;
diff --git a/drivers/edac/altera_edac.c b/drivers/edac/altera_edac.c
index db75d4b614f7..346c4987b284 100644
--- a/drivers/edac/altera_edac.c
+++ b/drivers/edac/altera_edac.c
@@ -38,7 +38,6 @@
 #include "edac_module.h"
 
 #define EDAC_MOD_STR		"altera_edac"
-#define EDAC_VERSION		"1"
 #define EDAC_DEVICE		"Altera"
 
 static const struct altr_sdram_prv_data c5_data = {
@@ -392,7 +391,6 @@ static int altr_sdram_probe(struct platform_device *pdev)
 	mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED;
 	mci->edac_cap = EDAC_FLAG_SECDED;
 	mci->mod_name = EDAC_MOD_STR;
-	mci->mod_ver = EDAC_VERSION;
 	mci->ctl_name = dev_name(&pdev->dev);
 	mci->scrub_mode = SCRUB_SW_SRC;
 	mci->dev_name = dev_name(&pdev->dev);
@@ -749,8 +747,10 @@ static int altr_edac_device_probe(struct platform_device *pdev)
 	drvdata->edac_dev_name = ecc_name;
 
 	drvdata->base = devm_ioremap(&pdev->dev, r->start, resource_size(r));
-	if (!drvdata->base)
+	if (!drvdata->base) {
+		res = -ENOMEM;
 		goto fail1;
+	}
 
 	/* Get driver specific data for this EDAC device */
 	drvdata->data = of_match_node(altr_edac_device_of_match, np)->data;
diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c
index 3aea55698165..ac2f30295efe 100644
--- a/drivers/edac/amd64_edac.c
+++ b/drivers/edac/amd64_edac.c
@@ -3130,7 +3130,6 @@ static void setup_mci_misc_attrs(struct mem_ctl_info *mci,
 
 	mci->edac_cap		= determine_edac_cap(pvt);
 	mci->mod_name		= EDAC_MOD_STR;
-	mci->mod_ver		= EDAC_AMD64_VERSION;
 	mci->ctl_name		= fam->ctl_name;
 	mci->dev_name		= pci_name(pvt->F3);
 	mci->ctl_page_to_phys	= NULL;
diff --git a/drivers/edac/amd76x_edac.c b/drivers/edac/amd76x_edac.c
index a7450275ad28..9c6e326b4c14 100644
--- a/drivers/edac/amd76x_edac.c
+++ b/drivers/edac/amd76x_edac.c
@@ -19,7 +19,6 @@
 #include <linux/edac.h>
 #include "edac_module.h"
 
-#define AMD76X_REVISION	" Ver: 2.0.2"
 #define EDAC_MOD_STR	"amd76x_edac"
 
 #define amd76x_printk(level, fmt, arg...) \
@@ -263,7 +262,6 @@ static int amd76x_probe1(struct pci_dev *pdev, int dev_idx)
 	mci->edac_cap = ems_mode ?
 		(EDAC_FLAG_EC | EDAC_FLAG_SECDED) : EDAC_FLAG_NONE;
 	mci->mod_name = EDAC_MOD_STR;
-	mci->mod_ver = AMD76X_REVISION;
 	mci->ctl_name = amd76x_devs[dev_idx].ctl_name;
 	mci->dev_name = pci_name(pdev);
 	mci->edac_check = amd76x_check;
diff --git a/drivers/edac/cpc925_edac.c b/drivers/edac/cpc925_edac.c
index 837b62c4993d..2c98e020df05 100644
--- a/drivers/edac/cpc925_edac.c
+++ b/drivers/edac/cpc925_edac.c
@@ -618,7 +618,7 @@ static u32 cpc925_cpu_mask_disabled(void)
 		}
 
 		if (reg == NULL || *reg > 2) {
-			cpc925_printk(KERN_ERR, "Bad reg value at %s\n", cpunode->full_name);
+			cpc925_printk(KERN_ERR, "Bad reg value at %pOF\n", cpunode);
 			continue;
 		}
 
@@ -999,7 +999,6 @@ static int cpc925_probe(struct platform_device *pdev)
 	mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED;
 	mci->edac_cap = EDAC_FLAG_SECDED;
 	mci->mod_name = CPC925_EDAC_MOD_STR;
-	mci->mod_ver = CPC925_EDAC_REVISION;
 	mci->ctl_name = pdev->name;
 
 	if (edac_op_state == EDAC_OPSTATE_POLL)
diff --git a/drivers/edac/e752x_edac.c b/drivers/edac/e752x_edac.c
index 1a352cae1f52..b5de9a13ea3f 100644
--- a/drivers/edac/e752x_edac.c
+++ b/drivers/edac/e752x_edac.c
@@ -26,7 +26,6 @@
 #include <linux/edac.h>
 #include "edac_module.h"
 
-#define E752X_REVISION	" Ver: 2.0.2"
 #define EDAC_MOD_STR	"e752x_edac"
 
 static int report_non_memory_errors;
@@ -1303,7 +1302,6 @@ static int e752x_probe1(struct pci_dev *pdev, int dev_idx)
 		(EDAC_FLAG_NONE | EDAC_FLAG_SECDED | EDAC_FLAG_S4ECD4ED);
 	/* FIXME - what if different memory types are in different csrows? */
 	mci->mod_name = EDAC_MOD_STR;
-	mci->mod_ver = E752X_REVISION;
 	mci->pdev = &pdev->dev;
 
 	edac_dbg(3, "init pvt\n");
diff --git a/drivers/edac/e7xxx_edac.c b/drivers/edac/e7xxx_edac.c
index 67ef07aed923..75d7ce62b3be 100644
--- a/drivers/edac/e7xxx_edac.c
+++ b/drivers/edac/e7xxx_edac.c
@@ -32,7 +32,6 @@
 #include <linux/edac.h>
 #include "edac_module.h"
 
-#define	E7XXX_REVISION " Ver: 2.0.2"
 #define	EDAC_MOD_STR	"e7xxx_edac"
 
 #define e7xxx_printk(level, fmt, arg...) \
@@ -458,7 +457,6 @@ static int e7xxx_probe1(struct pci_dev *pdev, int dev_idx)
 		EDAC_FLAG_S4ECD4ED;
 	/* FIXME - what if different memory types are in different csrows? */
 	mci->mod_name = EDAC_MOD_STR;
-	mci->mod_ver = E7XXX_REVISION;
 	mci->pdev = &pdev->dev;
 	edac_dbg(3, "init pvt\n");
 	pvt = (struct e7xxx_pvt *)mci->pvt_info;
diff --git a/drivers/edac/edac_mc_sysfs.c b/drivers/edac/edac_mc_sysfs.c
index 445862dac273..e4fcfa84fbd3 100644
--- a/drivers/edac/edac_mc_sysfs.c
+++ b/drivers/edac/edac_mc_sysfs.c
@@ -287,7 +287,7 @@ static struct attribute *csrow_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group csrow_attr_grp = {
+static const struct attribute_group csrow_attr_grp = {
 	.attrs	= csrow_attrs,
 };
 
@@ -304,7 +304,7 @@ static void csrow_attr_release(struct device *dev)
 	kfree(csrow);
 }
 
-static struct device_type csrow_attr_type = {
+static const struct device_type csrow_attr_type = {
 	.groups		= csrow_attr_groups,
 	.release	= csrow_attr_release,
 };
@@ -627,7 +627,7 @@ static struct attribute *dimm_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group dimm_attr_grp = {
+static const struct attribute_group dimm_attr_grp = {
 	.attrs	= dimm_attrs,
 };
 
@@ -644,7 +644,7 @@ static void dimm_attr_release(struct device *dev)
 	kfree(dimm);
 }
 
-static struct device_type dimm_attr_type = {
+static const struct device_type dimm_attr_type = {
 	.groups		= dimm_attr_groups,
 	.release	= dimm_attr_release,
 };
@@ -902,7 +902,7 @@ static umode_t mci_attr_is_visible(struct kobject *kobj,
 	return mode;
 }
 
-static struct attribute_group mci_attr_grp = {
+static const struct attribute_group mci_attr_grp = {
 	.attrs	= mci_attrs,
 	.is_visible = mci_attr_is_visible,
 };
@@ -920,7 +920,7 @@ static void mci_attr_release(struct device *dev)
 	kfree(mci);
 }
 
-static struct device_type mci_attr_type = {
+static const struct device_type mci_attr_type = {
 	.groups		= mci_attr_groups,
 	.release	= mci_attr_release,
 };
@@ -1074,7 +1074,7 @@ static void mc_attr_release(struct device *dev)
 	kfree(dev);
 }
 
-static struct device_type mc_attr_type = {
+static const struct device_type mc_attr_type = {
 	.release	= mc_attr_release,
 };
 /*
diff --git a/drivers/edac/ghes_edac.c b/drivers/edac/ghes_edac.c
index 4e61a6229dd2..6f80eb65c26c 100644
--- a/drivers/edac/ghes_edac.c
+++ b/drivers/edac/ghes_edac.c
@@ -17,8 +17,6 @@
 #include "edac_module.h"
 #include <ras/ras_event.h>
 
-#define GHES_EDAC_REVISION " Ver: 1.0.0"
-
 struct ghes_edac_pvt {
 	struct list_head list;
 	struct ghes *ghes;
@@ -451,7 +449,6 @@ int ghes_edac_register(struct ghes *ghes, struct device *dev)
 	mci->edac_ctl_cap = EDAC_FLAG_NONE;
 	mci->edac_cap = EDAC_FLAG_NONE;
 	mci->mod_name = "ghes_edac.c";
-	mci->mod_ver = GHES_EDAC_REVISION;
 	mci->ctl_name = "ghes_edac";
 	mci->dev_name = "ghes";
 
diff --git a/drivers/edac/highbank_mc_edac.c b/drivers/edac/highbank_mc_edac.c
index 0e7e0a404d89..6092e61be605 100644
--- a/drivers/edac/highbank_mc_edac.c
+++ b/drivers/edac/highbank_mc_edac.c
@@ -224,7 +224,6 @@ static int highbank_mc_probe(struct platform_device *pdev)
 	mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED;
 	mci->edac_cap = EDAC_FLAG_SECDED;
 	mci->mod_name = pdev->dev.driver->name;
-	mci->mod_ver = "1";
 	mci->ctl_name = id->compatible;
 	mci->dev_name = dev_name(&pdev->dev);
 	mci->scrub_mode = SCRUB_SW_SRC;
diff --git a/drivers/edac/i3000_edac.c b/drivers/edac/i3000_edac.c
index 5306240570d7..8085a32ec3bd 100644
--- a/drivers/edac/i3000_edac.c
+++ b/drivers/edac/i3000_edac.c
@@ -16,8 +16,6 @@
 #include <linux/edac.h>
 #include "edac_module.h"
 
-#define I3000_REVISION		"1.1"
-
 #define EDAC_MOD_STR		"i3000_edac"
 
 #define I3000_RANKS		8
@@ -375,7 +373,6 @@ static int i3000_probe1(struct pci_dev *pdev, int dev_idx)
 	mci->edac_cap = EDAC_FLAG_SECDED;
 
 	mci->mod_name = EDAC_MOD_STR;
-	mci->mod_ver = I3000_REVISION;
 	mci->ctl_name = i3000_devs[dev_idx].ctl_name;
 	mci->dev_name = pci_name(pdev);
 	mci->edac_check = i3000_check;
diff --git a/drivers/edac/i3200_edac.c b/drivers/edac/i3200_edac.c
index 77c58d201a30..d92d56cee101 100644
--- a/drivers/edac/i3200_edac.c
+++ b/drivers/edac/i3200_edac.c
@@ -17,8 +17,6 @@
 
 #include <linux/io-64-nonatomic-lo-hi.h>
 
-#define I3200_REVISION        "1.1"
-
 #define EDAC_MOD_STR        "i3200_edac"
 
 #define PCI_DEVICE_ID_INTEL_3200_HB    0x29f0
@@ -375,7 +373,6 @@ static int i3200_probe1(struct pci_dev *pdev, int dev_idx)
 	mci->edac_cap = EDAC_FLAG_SECDED;
 
 	mci->mod_name = EDAC_MOD_STR;
-	mci->mod_ver = I3200_REVISION;
 	mci->ctl_name = i3200_devs[dev_idx].ctl_name;
 	mci->dev_name = pci_name(pdev);
 	mci->edac_check = i3200_check;
diff --git a/drivers/edac/i5000_edac.c b/drivers/edac/i5000_edac.c
index 8f5a56e25bd2..53f24b18cd61 100644
--- a/drivers/edac/i5000_edac.c
+++ b/drivers/edac/i5000_edac.c
@@ -1430,7 +1430,6 @@ static int i5000_probe1(struct pci_dev *pdev, int dev_idx)
 	mci->edac_ctl_cap = EDAC_FLAG_NONE;
 	mci->edac_cap = EDAC_FLAG_NONE;
 	mci->mod_name = "i5000_edac.c";
-	mci->mod_ver = I5000_REVISION;
 	mci->ctl_name = i5000_devs[dev_idx].ctl_name;
 	mci->dev_name = pci_name(pdev);
 	mci->ctl_page_to_phys = NULL;
diff --git a/drivers/edac/i5100_edac.c b/drivers/edac/i5100_edac.c
index a8334c4acea7..b506eef6b146 100644
--- a/drivers/edac/i5100_edac.c
+++ b/drivers/edac/i5100_edac.c
@@ -1108,7 +1108,6 @@ static int i5100_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	mci->edac_ctl_cap = EDAC_FLAG_SECDED;
 	mci->edac_cap = EDAC_FLAG_SECDED;
 	mci->mod_name = "i5100_edac.c";
-	mci->mod_ver = "not versioned";
 	mci->ctl_name = "i5100";
 	mci->dev_name = pci_name(pdev);
 	mci->ctl_page_to_phys = NULL;
diff --git a/drivers/edac/i5400_edac.c b/drivers/edac/i5400_edac.c
index cd889edc8516..6f8bcdb9256a 100644
--- a/drivers/edac/i5400_edac.c
+++ b/drivers/edac/i5400_edac.c
@@ -1315,7 +1315,6 @@ static int i5400_probe1(struct pci_dev *pdev, int dev_idx)
 	mci->edac_ctl_cap = EDAC_FLAG_NONE;
 	mci->edac_cap = EDAC_FLAG_NONE;
 	mci->mod_name = "i5400_edac.c";
-	mci->mod_ver = I5400_REVISION;
 	mci->ctl_name = i5400_devs[dev_idx].ctl_name;
 	mci->dev_name = pci_name(pdev);
 	mci->ctl_page_to_phys = NULL;
diff --git a/drivers/edac/i7300_edac.c b/drivers/edac/i7300_edac.c
index e391f5a716be..6b5a554ba8e4 100644
--- a/drivers/edac/i7300_edac.c
+++ b/drivers/edac/i7300_edac.c
@@ -1077,7 +1077,6 @@ static int i7300_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	mci->edac_ctl_cap = EDAC_FLAG_NONE;
 	mci->edac_cap = EDAC_FLAG_NONE;
 	mci->mod_name = "i7300_edac.c";
-	mci->mod_ver = I7300_REVISION;
 	mci->ctl_name = i7300_devs[0].ctl_name;
 	mci->dev_name = pci_name(pdev);
 	mci->ctl_page_to_phys = NULL;
diff --git a/drivers/edac/i7core_edac.c b/drivers/edac/i7core_edac.c
index 75ad847593b7..c16c3b931b3d 100644
--- a/drivers/edac/i7core_edac.c
+++ b/drivers/edac/i7core_edac.c
@@ -1079,7 +1079,7 @@ static struct attribute *i7core_addrmatch_attrs[] = {
 	NULL
 };
 
-static struct attribute_group addrmatch_grp = {
+static const struct attribute_group addrmatch_grp = {
 	.attrs	= i7core_addrmatch_attrs,
 };
 
@@ -1094,7 +1094,7 @@ static void addrmatch_release(struct device *device)
 	kfree(device);
 }
 
-static struct device_type addrmatch_type = {
+static const struct device_type addrmatch_type = {
 	.groups		= addrmatch_groups,
 	.release	= addrmatch_release,
 };
@@ -1110,7 +1110,7 @@ static struct attribute *i7core_udimm_counters_attrs[] = {
 	NULL
 };
 
-static struct attribute_group all_channel_counts_grp = {
+static const struct attribute_group all_channel_counts_grp = {
 	.attrs	= i7core_udimm_counters_attrs,
 };
 
@@ -1125,7 +1125,7 @@ static void all_channel_counts_release(struct device *device)
 	kfree(device);
 }
 
-static struct device_type all_channel_counts_type = {
+static const struct device_type all_channel_counts_type = {
 	.groups		= all_channel_counts_groups,
 	.release	= all_channel_counts_release,
 };
@@ -2159,7 +2159,6 @@ static int i7core_register_mci(struct i7core_dev *i7core_dev)
 	mci->edac_ctl_cap = EDAC_FLAG_NONE;
 	mci->edac_cap = EDAC_FLAG_NONE;
 	mci->mod_name = "i7core_edac.c";
-	mci->mod_ver = I7CORE_REVISION;
 	mci->ctl_name = kasprintf(GFP_KERNEL, "i7 core #%d",
 				  i7core_dev->socket);
 	mci->dev_name = pci_name(i7core_dev->pdev[0]);
diff --git a/drivers/edac/i82443bxgx_edac.c b/drivers/edac/i82443bxgx_edac.c
index cb61a5b7d080..a2ca929e2168 100644
--- a/drivers/edac/i82443bxgx_edac.c
+++ b/drivers/edac/i82443bxgx_edac.c
@@ -31,8 +31,6 @@
 #include <linux/edac.h>
 #include "edac_module.h"
 
-#define I82443_REVISION	"0.1"
-
 #define EDAC_MOD_STR    "i82443bxgx_edac"
 
 /* The 82443BX supports SDRAM, or EDO (EDO for mobile only), "Memory
@@ -320,7 +318,6 @@ static int i82443bxgx_edacmc_probe1(struct pci_dev *pdev, int dev_idx)
 				I82443BXGX_EAP_OFFSET_MBE));
 
 	mci->mod_name = EDAC_MOD_STR;
-	mci->mod_ver = I82443_REVISION;
 	mci->ctl_name = "I82443BXGX";
 	mci->dev_name = pci_name(pdev);
 	mci->edac_check = i82443bxgx_edacmc_check;
diff --git a/drivers/edac/i82860_edac.c b/drivers/edac/i82860_edac.c
index 236c813227fc..3e3a80ffb322 100644
--- a/drivers/edac/i82860_edac.c
+++ b/drivers/edac/i82860_edac.c
@@ -16,7 +16,6 @@
 #include <linux/edac.h>
 #include "edac_module.h"
 
-#define  I82860_REVISION " Ver: 2.0.2"
 #define EDAC_MOD_STR	"i82860_edac"
 
 #define i82860_printk(level, fmt, arg...) \
@@ -216,7 +215,6 @@ static int i82860_probe1(struct pci_dev *pdev, int dev_idx)
 	/* I"m not sure about this but I think that all RDRAM is SECDED */
 	mci->edac_cap = EDAC_FLAG_SECDED;
 	mci->mod_name = EDAC_MOD_STR;
-	mci->mod_ver = I82860_REVISION;
 	mci->ctl_name = i82860_devs[dev_idx].ctl_name;
 	mci->dev_name = pci_name(pdev);
 	mci->edac_check = i82860_check;
diff --git a/drivers/edac/i82875p_edac.c b/drivers/edac/i82875p_edac.c
index e286b7e74c7a..ceac925af38c 100644
--- a/drivers/edac/i82875p_edac.c
+++ b/drivers/edac/i82875p_edac.c
@@ -20,7 +20,6 @@
 #include <linux/edac.h>
 #include "edac_module.h"
 
-#define I82875P_REVISION	" Ver: 2.0.2"
 #define EDAC_MOD_STR		"i82875p_edac"
 
 #define i82875p_printk(level, fmt, arg...) \
@@ -423,7 +422,6 @@ static int i82875p_probe1(struct pci_dev *pdev, int dev_idx)
 	mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED;
 	mci->edac_cap = EDAC_FLAG_UNKNOWN;
 	mci->mod_name = EDAC_MOD_STR;
-	mci->mod_ver = I82875P_REVISION;
 	mci->ctl_name = i82875p_devs[dev_idx].ctl_name;
 	mci->dev_name = pci_name(pdev);
 	mci->edac_check = i82875p_check;
diff --git a/drivers/edac/i82975x_edac.c b/drivers/edac/i82975x_edac.c
index 9dcdab28f665..892815eaa97b 100644
--- a/drivers/edac/i82975x_edac.c
+++ b/drivers/edac/i82975x_edac.c
@@ -16,7 +16,6 @@
 #include <linux/edac.h>
 #include "edac_module.h"
 
-#define I82975X_REVISION	" Ver: 1.0.0"
 #define EDAC_MOD_STR		"i82975x_edac"
 
 #define i82975x_printk(level, fmt, arg...) \
@@ -564,7 +563,6 @@ static int i82975x_probe1(struct pci_dev *pdev, int dev_idx)
 	mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED;
 	mci->edac_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED;
 	mci->mod_name = EDAC_MOD_STR;
-	mci->mod_ver = I82975X_REVISION;
 	mci->ctl_name = i82975x_devs[dev_idx].ctl_name;
 	mci->dev_name = pci_name(pdev);
 	mci->edac_check = i82975x_check;
diff --git a/drivers/edac/ie31200_edac.c b/drivers/edac/ie31200_edac.c
index 4260579e6901..aac9b9b360b8 100644
--- a/drivers/edac/ie31200_edac.c
+++ b/drivers/edac/ie31200_edac.c
@@ -45,7 +45,6 @@
 #include <linux/io-64-nonatomic-lo-hi.h>
 #include "edac_module.h"
 
-#define IE31200_REVISION "1.0"
 #define EDAC_MOD_STR "ie31200_edac"
 
 #define ie31200_printk(level, fmt, arg...) \
@@ -420,7 +419,6 @@ static int ie31200_probe1(struct pci_dev *pdev, int dev_idx)
 	mci->edac_ctl_cap = EDAC_FLAG_SECDED;
 	mci->edac_cap = EDAC_FLAG_SECDED;
 	mci->mod_name = EDAC_MOD_STR;
-	mci->mod_ver = IE31200_REVISION;
 	mci->ctl_name = ie31200_devs[dev_idx].ctl_name;
 	mci->dev_name = pci_name(pdev);
 	mci->edac_check = ie31200_check;
diff --git a/drivers/edac/mce_amd.c b/drivers/edac/mce_amd.c
index 9a2658a256a9..a11a671c7a38 100644
--- a/drivers/edac/mce_amd.c
+++ b/drivers/edac/mce_amd.c
@@ -1,6 +1,8 @@
 #include <linux/module.h>
 #include <linux/slab.h>
 
+#include <asm/cpu.h>
+
 #include "mce_amd.h"
 
 static struct amd_decoder_ops *fam_ops;
@@ -744,7 +746,7 @@ static void decode_mc3_mce(struct mce *m)
 
 static void decode_mc4_mce(struct mce *m)
 {
-	struct cpuinfo_x86 *c = &boot_cpu_data;
+	unsigned int fam = x86_family(m->cpuid);
 	int node_id = amd_get_nb_id(m->extcpu);
 	u16 ec = EC(m->status);
 	u8 xec = XEC(m->status, 0x1f);
@@ -758,7 +760,7 @@ static void decode_mc4_mce(struct mce *m)
 		/* special handling for DRAM ECCs */
 		if (xec == 0x0 || xec == 0x8) {
 			/* no ECCs on F11h */
-			if (c->x86 == 0x11)
+			if (fam == 0x11)
 				goto wrong_mc4_mce;
 
 			pr_cont("%s.\n", mc4_mce_desc[xec]);
@@ -779,7 +781,7 @@ static void decode_mc4_mce(struct mce *m)
 		return;
 
 	case 0x19:
-		if (boot_cpu_data.x86 == 0x15 || boot_cpu_data.x86 == 0x16)
+		if (fam == 0x15 || fam == 0x16)
 			pr_cont("Compute Unit Data Error.\n");
 		else
 			goto wrong_mc4_mce;
@@ -802,11 +804,11 @@ static void decode_mc4_mce(struct mce *m)
 
 static void decode_mc5_mce(struct mce *m)
 {
-	struct cpuinfo_x86 *c = &boot_cpu_data;
+	unsigned int fam = x86_family(m->cpuid);
 	u16 ec = EC(m->status);
 	u8 xec = XEC(m->status, xec_mask);
 
-	if (c->x86 == 0xf || c->x86 == 0x11)
+	if (fam == 0xf || fam == 0x11)
 		goto wrong_mc5_mce;
 
 	pr_emerg(HW_ERR "MC5 Error: ");
@@ -849,7 +851,7 @@ static void decode_mc6_mce(struct mce *m)
 }
 
 /* Decode errors according to Scalable MCA specification */
-static void decode_smca_errors(struct mce *m)
+static void decode_smca_error(struct mce *m)
 {
 	struct smca_hwid *hwid;
 	unsigned int bank_type;
@@ -859,7 +861,7 @@ static void decode_smca_errors(struct mce *m)
 	if (m->bank >= ARRAY_SIZE(smca_banks))
 		return;
 
-	if (boot_cpu_data.x86 >= 0x17 && m->bank == 4)
+	if (x86_family(m->cpuid) >= 0x17 && m->bank == 4)
 		pr_emerg(HW_ERR "Bank 4 is reserved on Fam17h.\n");
 
 	hwid = smca_banks[m->bank].hwid;
@@ -878,12 +880,8 @@ static void decode_smca_errors(struct mce *m)
 		pr_cont("%s.\n", smca_mce_descs[bank_type].descs[xec]);
 	}
 
-	/*
-	 * amd_get_nb_id() returns the last level cache id.
-	 * The last level cache on Fam17h is 1 level below the node.
-	 */
 	if (bank_type == SMCA_UMC && xec == 0 && decode_dram_ecc)
-		decode_dram_ecc(amd_get_nb_id(m->extcpu) >> 1, m);
+		decode_dram_ecc(cpu_to_node(m->extcpu), m);
 }
 
 static inline void amd_decode_err_code(u16 ec)
@@ -915,12 +913,10 @@ static inline void amd_decode_err_code(u16 ec)
  */
 static bool amd_filter_mce(struct mce *m)
 {
-	u8 xec = (m->status >> 16) & 0x1f;
-
 	/*
 	 * NB GART TLB error reporting is disabled by default.
 	 */
-	if (m->bank == 4 && xec == 0x5 && !report_gart_errors)
+	if (m->bank == 4 && XEC(m->status, 0x1f) == 0x5 && !report_gart_errors)
 		return true;
 
 	return false;
@@ -946,7 +942,7 @@ static int
 amd_decode_mce(struct notifier_block *nb, unsigned long val, void *data)
 {
 	struct mce *m = (struct mce *)data;
-	struct cpuinfo_x86 *c = &cpu_data(m->extcpu);
+	unsigned int fam = x86_family(m->cpuid);
 	int ecc;
 
 	if (amd_filter_mce(m))
@@ -956,7 +952,7 @@ amd_decode_mce(struct notifier_block *nb, unsigned long val, void *data)
 
 	pr_emerg(HW_ERR "CPU:%d (%x:%x:%x) MC%d_STATUS[%s|%s|%s|%s|%s",
 		m->extcpu,
-		c->x86, c->x86_model, c->x86_mask,
+		fam, x86_model(m->cpuid), x86_stepping(m->cpuid),
 		m->bank,
 		((m->status & MCI_STATUS_OVER)	? "Over"  : "-"),
 		((m->status & MCI_STATUS_UC)	? "UE"	  :
@@ -965,11 +961,11 @@ amd_decode_mce(struct notifier_block *nb, unsigned long val, void *data)
 		((m->status & MCI_STATUS_PCC)	? "PCC"	  : "-"),
 		((m->status & MCI_STATUS_ADDRV)	? "AddrV" : "-"));
 
-	if (c->x86 >= 0x15) {
+	if (fam >= 0x15) {
 		pr_cont("|%s", (m->status & MCI_STATUS_DEFERRED ? "Deferred" : "-"));
 
 		/* F15h, bank4, bit 43 is part of McaStatSubCache. */
-		if (c->x86 != 0x15 || m->bank != 4)
+		if (fam != 0x15 || m->bank != 4)
 			pr_cont("|%s", (m->status & MCI_STATUS_POISON ? "Poison" : "-"));
 	}
 
@@ -1002,7 +998,7 @@ amd_decode_mce(struct notifier_block *nb, unsigned long val, void *data)
 
 		pr_cont("\n");
 
-		decode_smca_errors(m);
+		decode_smca_error(m);
 		goto err_code;
 	}
 
diff --git a/drivers/edac/mv64x60_edac.c b/drivers/edac/mv64x60_edac.c
index d3650df94fe8..ec5d695bbb72 100644
--- a/drivers/edac/mv64x60_edac.c
+++ b/drivers/edac/mv64x60_edac.c
@@ -766,7 +766,6 @@ static int mv64x60_mc_err_probe(struct platform_device *pdev)
 	mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED;
 	mci->edac_cap = EDAC_FLAG_SECDED;
 	mci->mod_name = EDAC_MOD_STR;
-	mci->mod_ver = MV64x60_REVISION;
 	mci->ctl_name = mv64x60_ctl_name;
 
 	if (edac_op_state == EDAC_OPSTATE_POLL)
diff --git a/drivers/edac/pnd2_edac.c b/drivers/edac/pnd2_edac.c
index 8e599490f6de..4395c84cdcbf 100644
--- a/drivers/edac/pnd2_edac.c
+++ b/drivers/edac/pnd2_edac.c
@@ -129,42 +129,72 @@ static struct mem_ctl_info *pnd2_mci;
 #define GET_BITFIELD(v, lo, hi)	(((v) & GENMASK_ULL(hi, lo)) >> (lo))
 #define U64_LSHIFT(val, s)	((u64)(val) << (s))
 
-#ifdef CONFIG_X86_INTEL_SBI_APL
-#include "linux/platform_data/sbi_apl.h"
-static int sbi_send(int port, int off, int op, u32 *data)
+/*
+ * On Apollo Lake we access memory controller registers via a
+ * side-band mailbox style interface in a hidden PCI device
+ * configuration space.
+ */
+static struct pci_bus	*p2sb_bus;
+#define P2SB_DEVFN	PCI_DEVFN(0xd, 0)
+#define P2SB_ADDR_OFF	0xd0
+#define P2SB_DATA_OFF	0xd4
+#define P2SB_STAT_OFF	0xd8
+#define P2SB_ROUT_OFF	0xda
+#define P2SB_EADD_OFF	0xdc
+#define P2SB_HIDE_OFF	0xe1
+
+#define P2SB_BUSY	1
+
+#define P2SB_READ(size, off, ptr) \
+	pci_bus_read_config_##size(p2sb_bus, P2SB_DEVFN, off, ptr)
+#define P2SB_WRITE(size, off, val) \
+	pci_bus_write_config_##size(p2sb_bus, P2SB_DEVFN, off, val)
+
+static bool p2sb_is_busy(u16 *status)
 {
-	struct sbi_apl_message sbi_arg;
-	int ret, read = 0;
+	P2SB_READ(word, P2SB_STAT_OFF, status);
 
-	memset(&sbi_arg, 0, sizeof(sbi_arg));
+	return !!(*status & P2SB_BUSY);
+}
 
-	if (op == 0 || op == 4 || op == 6)
-		read = 1;
-	else
-		sbi_arg.data = *data;
+static int _apl_rd_reg(int port, int off, int op, u32 *data)
+{
+	int retries = 0xff, ret;
+	u16 status;
+	u8 hidden;
+
+	/* Unhide the P2SB device, if it's hidden */
+	P2SB_READ(byte, P2SB_HIDE_OFF, &hidden);
+	if (hidden)
+		P2SB_WRITE(byte, P2SB_HIDE_OFF, 0);
+
+	if (p2sb_is_busy(&status)) {
+		ret = -EAGAIN;
+		goto out;
+	}
 
-	sbi_arg.opcode = op;
-	sbi_arg.port_address = port;
-	sbi_arg.register_offset = off;
-	ret = sbi_apl_commit(&sbi_arg);
-	if (ret || sbi_arg.status)
-		edac_dbg(2, "sbi_send status=%d ret=%d data=%x\n",
-				 sbi_arg.status, ret, sbi_arg.data);
+	P2SB_WRITE(dword, P2SB_ADDR_OFF, (port << 24) | off);
+	P2SB_WRITE(dword, P2SB_DATA_OFF, 0);
+	P2SB_WRITE(dword, P2SB_EADD_OFF, 0);
+	P2SB_WRITE(word, P2SB_ROUT_OFF, 0);
+	P2SB_WRITE(word, P2SB_STAT_OFF, (op << 8) | P2SB_BUSY);
 
-	if (ret == 0)
-		ret = sbi_arg.status;
+	while (p2sb_is_busy(&status)) {
+		if (retries-- == 0) {
+			ret = -EBUSY;
+			goto out;
+		}
+	}
 
-	if (ret == 0 && read)
-		*data = sbi_arg.data;
+	P2SB_READ(dword, P2SB_DATA_OFF, data);
+	ret = (status >> 1) & 0x3;
+out:
+	/* Hide the P2SB device, if it was hidden before */
+	if (hidden)
+		P2SB_WRITE(byte, P2SB_HIDE_OFF, hidden);
 
 	return ret;
 }
-#else
-static int sbi_send(int port, int off, int op, u32 *data)
-{
-	return -EUNATCH;
-}
-#endif
 
 static int apl_rd_reg(int port, int off, int op, void *data, size_t sz, char *name)
 {
@@ -173,10 +203,10 @@ static int apl_rd_reg(int port, int off, int op, void *data, size_t sz, char *na
 	edac_dbg(2, "Read %s port=%x off=%x op=%x\n", name, port, off, op);
 	switch (sz) {
 	case 8:
-		ret = sbi_send(port, off + 4, op, (u32 *)(data + 4));
+		ret = _apl_rd_reg(port, off + 4, op, (u32 *)(data + 4));
 		/* fall through */
 	case 4:
-		ret |= sbi_send(port, off, op, (u32 *)data);
+		ret |= _apl_rd_reg(port, off, op, (u32 *)data);
 		pnd2_printk(KERN_DEBUG, "%s=%x%08x ret=%d\n", name,
 					sz == 8 ? *((u32 *)(data + 4)) : 0, *((u32 *)data), ret);
 		break;
@@ -212,11 +242,23 @@ static u64 get_sideband_reg_base_addr(void)
 {
 	struct pci_dev *pdev;
 	u32 hi, lo;
+	u8 hidden;
 
 	pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x19dd, NULL);
 	if (pdev) {
+		/* Unhide the P2SB device, if it's hidden */
+		pci_read_config_byte(pdev, 0xe1, &hidden);
+		if (hidden)
+			pci_write_config_byte(pdev, 0xe1, 0);
+
 		pci_read_config_dword(pdev, 0x10, &lo);
 		pci_read_config_dword(pdev, 0x14, &hi);
+		lo &= 0xfffffff0;
+
+		/* Hide the P2SB device, if it was hidden before */
+		if (hidden)
+			pci_write_config_byte(pdev, 0xe1, hidden);
+
 		pci_dev_put(pdev);
 		return (U64_LSHIFT(hi, 32) | U64_LSHIFT(lo, 0));
 	} else {
@@ -1515,6 +1557,12 @@ static int __init pnd2_init(void)
 
 	ops = (struct dunit_ops *)id->driver_data;
 
+	if (ops->type == APL) {
+		p2sb_bus = pci_find_bus(0, 0);
+		if (!p2sb_bus)
+			return -ENODEV;
+	}
+
 	/* Ensure that the OPSTATE is set correctly for POLL or NMI */
 	opstate_init();
 
diff --git a/drivers/edac/ppc4xx_edac.c b/drivers/edac/ppc4xx_edac.c
index e55e92590106..fd3202c30f69 100644
--- a/drivers/edac/ppc4xx_edac.c
+++ b/drivers/edac/ppc4xx_edac.c
@@ -1063,7 +1063,6 @@ static int ppc4xx_edac_mc_init(struct mem_ctl_info *mci,
 	/* Initialize strings */
 
 	mci->mod_name		= PPC4XX_EDAC_MODULE_NAME;
-	mci->mod_ver		= PPC4XX_EDAC_MODULE_REVISION;
 	mci->ctl_name		= ppc4xx_edac_match->compatible,
 	mci->dev_name		= np->full_name;
 
@@ -1267,8 +1266,8 @@ static int ppc4xx_edac_probe(struct platform_device *op)
 	memcheck = (mcopt1 & SDRAM_MCOPT1_MCHK_MASK);
 
 	if (memcheck == SDRAM_MCOPT1_MCHK_NON) {
-		ppc4xx_edac_printk(KERN_INFO, "%s: No ECC memory detected or "
-				   "ECC is disabled.\n", np->full_name);
+		ppc4xx_edac_printk(KERN_INFO, "%pOF: No ECC memory detected or "
+				   "ECC is disabled.\n", np);
 		status = -ENODEV;
 		goto done;
 	}
@@ -1287,9 +1286,9 @@ static int ppc4xx_edac_probe(struct platform_device *op)
 	mci = edac_mc_alloc(ppc4xx_edac_instance, ARRAY_SIZE(layers), layers,
 			    sizeof(struct ppc4xx_edac_pdata));
 	if (mci == NULL) {
-		ppc4xx_edac_printk(KERN_ERR, "%s: "
+		ppc4xx_edac_printk(KERN_ERR, "%pOF: "
 				   "Failed to allocate EDAC MC instance!\n",
-				   np->full_name);
+				   np);
 		status = -ENOMEM;
 		goto done;
 	}
diff --git a/drivers/edac/r82600_edac.c b/drivers/edac/r82600_edac.c
index 978916625ced..851e53e122aa 100644
--- a/drivers/edac/r82600_edac.c
+++ b/drivers/edac/r82600_edac.c
@@ -22,7 +22,6 @@
 #include <linux/edac.h>
 #include "edac_module.h"
 
-#define R82600_REVISION	" Ver: 2.0.2"
 #define EDAC_MOD_STR	"r82600_edac"
 
 #define r82600_printk(level, fmt, arg...) \
@@ -316,7 +315,6 @@ static int r82600_probe1(struct pci_dev *pdev, int dev_idx)
 		mci->edac_cap = EDAC_FLAG_NONE;
 
 	mci->mod_name = EDAC_MOD_STR;
-	mci->mod_ver = R82600_REVISION;
 	mci->ctl_name = "R82600";
 	mci->dev_name = pci_name(pdev);
 	mci->edac_check = r82600_check;
diff --git a/drivers/edac/sb_edac.c b/drivers/edac/sb_edac.c
index 80d860cb0746..dc0591654011 100644
--- a/drivers/edac/sb_edac.c
+++ b/drivers/edac/sb_edac.c
@@ -300,6 +300,12 @@ enum domain {
 	SOCK,
 };
 
+enum mirroring_mode {
+	NON_MIRRORING,
+	ADDR_RANGE_MIRRORING,
+	FULL_MIRRORING,
+};
+
 struct sbridge_pvt;
 struct sbridge_info {
 	enum type	type;
@@ -377,8 +383,9 @@ struct sbridge_pvt {
 	struct sbridge_channel	channel[NUM_CHANNELS];
 
 	/* Memory type detection */
-	bool			is_mirrored, is_lockstep, is_close_pg;
+	bool			is_cur_addr_mirrored, is_lockstep, is_close_pg;
 	bool			is_chan_hash;
+	enum mirroring_mode	mirror_mode;
 
 	/* Memory description */
 	u64			tolm, tohm;
@@ -1648,10 +1655,6 @@ static int get_dimm_config(struct mem_ctl_info *mci)
 	enum edac_type mode;
 	u32 reg;
 
-	if (pvt->info.type == HASWELL || pvt->info.type == BROADWELL) {
-		pci_read_config_dword(pvt->pci_ha, HASWELL_HASYSDEFEATURE2, &reg);
-		pvt->is_chan_hash = GET_BITFIELD(reg, 21, 21);
-	}
 	pvt->sbridge_dev->node_id = pvt->info.get_node_id(pvt);
 	edac_dbg(0, "mc#%d: Node ID: %d, source ID: %d\n",
 		 pvt->sbridge_dev->mc,
@@ -1663,22 +1666,45 @@ static int get_dimm_config(struct mem_ctl_info *mci)
 	 */
 	if (pvt->info.type == KNIGHTS_LANDING) {
 		mode = EDAC_S4ECD4ED;
-		pvt->is_mirrored = false;
+		pvt->mirror_mode = NON_MIRRORING;
+		pvt->is_cur_addr_mirrored = false;
 
 		if (knl_get_dimm_capacity(pvt, knl_mc_sizes) != 0)
 			return -1;
-		pci_read_config_dword(pvt->pci_ta, KNL_MCMTR, &pvt->info.mcmtr);
+		if (pci_read_config_dword(pvt->pci_ta, KNL_MCMTR, &pvt->info.mcmtr)) {
+			edac_dbg(0, "Failed to read KNL_MCMTR register\n");
+			return -ENODEV;
+		}
 	} else {
-		pci_read_config_dword(pvt->pci_ras, RASENABLES, &reg);
+		if (pvt->info.type == HASWELL || pvt->info.type == BROADWELL) {
+			if (pci_read_config_dword(pvt->pci_ha, HASWELL_HASYSDEFEATURE2, &reg)) {
+				edac_dbg(0, "Failed to read HASWELL_HASYSDEFEATURE2 register\n");
+				return -ENODEV;
+			}
+			pvt->is_chan_hash = GET_BITFIELD(reg, 21, 21);
+			if (GET_BITFIELD(reg, 28, 28)) {
+				pvt->mirror_mode = ADDR_RANGE_MIRRORING;
+				edac_dbg(0, "Address range partial memory mirroring is enabled\n");
+				goto next;
+			}
+		}
+		if (pci_read_config_dword(pvt->pci_ras, RASENABLES, &reg)) {
+			edac_dbg(0, "Failed to read RASENABLES register\n");
+			return -ENODEV;
+		}
 		if (IS_MIRROR_ENABLED(reg)) {
-			edac_dbg(0, "Memory mirror is enabled\n");
-			pvt->is_mirrored = true;
+			pvt->mirror_mode = FULL_MIRRORING;
+			edac_dbg(0, "Full memory mirroring is enabled\n");
 		} else {
-			edac_dbg(0, "Memory mirror is disabled\n");
-			pvt->is_mirrored = false;
+			pvt->mirror_mode = NON_MIRRORING;
+			edac_dbg(0, "Memory mirroring is disabled\n");
 		}
 
-		pci_read_config_dword(pvt->pci_ta, MCMTR, &pvt->info.mcmtr);
+next:
+		if (pci_read_config_dword(pvt->pci_ta, MCMTR, &pvt->info.mcmtr)) {
+			edac_dbg(0, "Failed to read MCMTR register\n");
+			return -ENODEV;
+		}
 		if (IS_LOCKSTEP_ENABLED(pvt->info.mcmtr)) {
 			edac_dbg(0, "Lockstep is enabled\n");
 			mode = EDAC_S8ECD8ED;
@@ -2092,7 +2118,8 @@ static int get_memory_error_data(struct mem_ctl_info *mci,
 
 	pci_read_config_dword(pvt->pci_tad[base_ch], tad_ch_nilv_offset[n_tads], &tad_offset);
 
-	if (pvt->is_mirrored) {
+	if (pvt->mirror_mode == FULL_MIRRORING ||
+	    (pvt->mirror_mode == ADDR_RANGE_MIRRORING && n_tads == 0)) {
 		*channel_mask |= 1 << ((base_ch + 2) % 4);
 		switch(ch_way) {
 		case 2:
@@ -2103,8 +2130,12 @@ static int get_memory_error_data(struct mem_ctl_info *mci,
 			sprintf(msg, "Invalid mirror set. Can't decode addr");
 			return -EINVAL;
 		}
-	} else
+
+		pvt->is_cur_addr_mirrored = true;
+	} else {
 		sck_xch = (1 << sck_way) * ch_way;
+		pvt->is_cur_addr_mirrored = false;
+	}
 
 	if (pvt->is_lockstep)
 		*channel_mask |= 1 << ((base_ch + 1) % 4);
@@ -2967,7 +2998,7 @@ static void sbridge_mce_output_error(struct mem_ctl_info *mci,
 	 * EDAC core should be handling the channel mask, in order to point
 	 * to the group of dimm's where the error may be happening.
 	 */
-	if (!pvt->is_lockstep && !pvt->is_mirrored && !pvt->is_close_pg)
+	if (!pvt->is_lockstep && !pvt->is_cur_addr_mirrored && !pvt->is_close_pg)
 		channel = first_channel;
 
 	snprintf(msg, sizeof(msg),
@@ -3125,7 +3156,6 @@ static int sbridge_register_mci(struct sbridge_dev *sbridge_dev, enum type type)
 	mci->edac_ctl_cap = EDAC_FLAG_NONE;
 	mci->edac_cap = EDAC_FLAG_NONE;
 	mci->mod_name = "sb_edac.c";
-	mci->mod_ver = SBRIDGE_REVISION;
 	mci->dev_name = pci_name(pdev);
 	mci->ctl_page_to_phys = NULL;
 
diff --git a/drivers/edac/skx_edac.c b/drivers/edac/skx_edac.c
index 64bef6c9cfb4..16dea97568a1 100644
--- a/drivers/edac/skx_edac.c
+++ b/drivers/edac/skx_edac.c
@@ -31,8 +31,6 @@
 
 #include "edac_module.h"
 
-#define SKX_REVISION    " Ver: 1.0 "
-
 /*
  * Debug macros
  */
@@ -473,7 +471,6 @@ static int skx_register_mci(struct skx_imc *imc)
 	mci->edac_cap = EDAC_FLAG_NONE;
 	mci->mod_name = "skx_edac.c";
 	mci->dev_name = pci_name(imc->chan[0].cdev);
-	mci->mod_ver = SKX_REVISION;
 	mci->ctl_page_to_phys = NULL;
 
 	rc = skx_get_dimm_config(mci);
diff --git a/drivers/edac/synopsys_edac.c b/drivers/edac/synopsys_edac.c
index 1c01dec78ec3..0c9c59e2b5a3 100644
--- a/drivers/edac/synopsys_edac.c
+++ b/drivers/edac/synopsys_edac.c
@@ -413,7 +413,6 @@ static int synps_edac_mc_init(struct mem_ctl_info *mci,
 	mci->ctl_name = "synps_ddr_controller";
 	mci->dev_name = SYNPS_EDAC_MOD_STRING;
 	mci->mod_name = SYNPS_EDAC_MOD_VER;
-	mci->mod_ver = "1";
 
 	edac_op_state = EDAC_OPSTATE_POLL;
 	mci->edac_check = synps_edac_check;
diff --git a/drivers/edac/thunderx_edac.c b/drivers/edac/thunderx_edac.c
index 2d352b40ae1c..f35d87519a3e 100644
--- a/drivers/edac/thunderx_edac.c
+++ b/drivers/edac/thunderx_edac.c
@@ -732,7 +732,6 @@ static int thunderx_lmc_probe(struct pci_dev *pdev,
 	mci->edac_cap = EDAC_FLAG_SECDED;
 
 	mci->mod_name = "thunderx-lmc";
-	mci->mod_ver = "1";
 	mci->ctl_name = "thunderx-lmc";
 	mci->dev_name = dev_name(&pdev->dev);
 	mci->scrub_mode = SCRUB_NONE;
@@ -775,11 +774,10 @@ static int thunderx_lmc_probe(struct pci_dev *pdev,
 
 	lmc->xor_bank = lmc_control & LMC_CONTROL_XOR_BANK;
 
-	l2c_ioaddr = ioremap(L2C_CTL | FIELD_PREP(THUNDERX_NODE, lmc->node),
-			     PAGE_SIZE);
-
+	l2c_ioaddr = ioremap(L2C_CTL | FIELD_PREP(THUNDERX_NODE, lmc->node), PAGE_SIZE);
 	if (!l2c_ioaddr) {
 		dev_err(&pdev->dev, "Cannot map L2C_CTL\n");
+		ret = -ENOMEM;
 		goto err_free;
 	}
 
diff --git a/drivers/edac/x38_edac.c b/drivers/edac/x38_edac.c
index 03c97a4bf590..cc779f3f9e2d 100644
--- a/drivers/edac/x38_edac.c
+++ b/drivers/edac/x38_edac.c
@@ -18,8 +18,6 @@
 #include <linux/io-64-nonatomic-lo-hi.h>
 #include "edac_module.h"
 
-#define X38_REVISION		"1.1"
-
 #define EDAC_MOD_STR		"x38_edac"
 
 #define PCI_DEVICE_ID_INTEL_X38_HB	0x29e0
@@ -357,7 +355,6 @@ static int x38_probe1(struct pci_dev *pdev, int dev_idx)
 	mci->edac_cap = EDAC_FLAG_SECDED;
 
 	mci->mod_name = EDAC_MOD_STR;
-	mci->mod_ver = X38_REVISION;
 	mci->ctl_name = x38_devs[dev_idx].ctl_name;
 	mci->dev_name = pci_name(pdev);
 	mci->edac_check = x38_check;
diff --git a/drivers/edac/xgene_edac.c b/drivers/edac/xgene_edac.c
index 669246056812..e8b81d7ef61f 100644
--- a/drivers/edac/xgene_edac.c
+++ b/drivers/edac/xgene_edac.c
@@ -415,7 +415,6 @@ static int xgene_edac_mc_add(struct xgene_edac *edac, struct device_node *np)
 	mci->edac_ctl_cap = EDAC_FLAG_SECDED;
 	mci->edac_cap = EDAC_FLAG_SECDED;
 	mci->mod_name = EDAC_MOD_STR;
-	mci->mod_ver = "0.1";
 	mci->ctl_page_to_phys = NULL;
 	mci->scrub_cap = SCRUB_FLAG_HW_SRC;
 	mci->scrub_mode = SCRUB_HW_SRC;
diff --git a/drivers/extcon/Kconfig b/drivers/extcon/Kconfig
index 6d50071f07d5..a7bca4207f44 100644
--- a/drivers/extcon/Kconfig
+++ b/drivers/extcon/Kconfig
@@ -150,4 +150,11 @@ config EXTCON_USB_GPIO
 	  Say Y here to enable GPIO based USB cable detection extcon support.
 	  Used typically if GPIO is used for USB ID pin detection.
 
+config EXTCON_USBC_CROS_EC
+	tristate "ChromeOS Embedded Controller EXTCON support"
+	depends on MFD_CROS_EC
+	help
+	  Say Y here to enable USB Type C cable detection extcon support when
+	  using Chrome OS EC based USB Type-C ports.
+
 endif
diff --git a/drivers/extcon/Makefile b/drivers/extcon/Makefile
index ecfa95804427..a73624e76193 100644
--- a/drivers/extcon/Makefile
+++ b/drivers/extcon/Makefile
@@ -20,3 +20,4 @@ obj-$(CONFIG_EXTCON_QCOM_SPMI_MISC) += extcon-qcom-spmi-misc.o
 obj-$(CONFIG_EXTCON_RT8973A)	+= extcon-rt8973a.o
 obj-$(CONFIG_EXTCON_SM5502)	+= extcon-sm5502.o
 obj-$(CONFIG_EXTCON_USB_GPIO)	+= extcon-usb-gpio.o
+obj-$(CONFIG_EXTCON_USBC_CROS_EC) += extcon-usbc-cros-ec.o
diff --git a/drivers/extcon/devres.c b/drivers/extcon/devres.c
index 186fd735eb28..f599aeddf8e5 100644
--- a/drivers/extcon/devres.c
+++ b/drivers/extcon/devres.c
@@ -1,5 +1,5 @@
 /*
- *  drivers/extcon/devres.c - EXTCON device's resource management
+ * drivers/extcon/devres.c - EXTCON device's resource management
  *
  * Copyright (C) 2016 Samsung Electronics
  * Author: Chanwoo Choi <cw00.choi@samsung.com>
@@ -59,10 +59,9 @@ static void devm_extcon_dev_notifier_all_unreg(struct device *dev, void *res)
 
 /**
  * devm_extcon_dev_allocate - Allocate managed extcon device
- * @dev:		device owning the extcon device being created
- * @supported_cable:	Array of supported extcon ending with EXTCON_NONE.
- *			If supported_cable is NULL, cable name related APIs
- *			are disabled.
+ * @dev:		the device owning the extcon device being created
+ * @supported_cable:	the array of the supported external connectors
+ *			ending with EXTCON_NONE.
  *
  * This function manages automatically the memory of extcon device using device
  * resource management and simplify the control of freeing the memory of extcon
@@ -97,8 +96,8 @@ EXPORT_SYMBOL_GPL(devm_extcon_dev_allocate);
 
 /**
  * devm_extcon_dev_free() - Resource-managed extcon_dev_unregister()
- * @dev:	device the extcon belongs to
- * @edev:	the extcon device to unregister
+ * @dev:	the device owning the extcon device being created
+ * @edev:	the extcon device to be freed
  *
  * Free the memory that is allocated with devm_extcon_dev_allocate()
  * function.
@@ -112,10 +111,9 @@ EXPORT_SYMBOL_GPL(devm_extcon_dev_free);
 
 /**
  * devm_extcon_dev_register() - Resource-managed extcon_dev_register()
- * @dev:	device to allocate extcon device
- * @edev:	the new extcon device to register
+ * @dev:	the device owning the extcon device being created
+ * @edev:	the extcon device to be registered
  *
- * Managed extcon_dev_register() function. If extcon device is attached with
  * this function, that extcon device is automatically unregistered on driver
  * detach. Internally this function calls extcon_dev_register() function.
  * To get more information, refer that function.
@@ -149,8 +147,8 @@ EXPORT_SYMBOL_GPL(devm_extcon_dev_register);
 
 /**
  * devm_extcon_dev_unregister() - Resource-managed extcon_dev_unregister()
- * @dev:	device the extcon belongs to
- * @edev:	the extcon device to unregister
+ * @dev:	the device owning the extcon device being created
+ * @edev:	the extcon device to unregistered
  *
  * Unregister extcon device that is registered with devm_extcon_dev_register()
  * function.
@@ -164,10 +162,10 @@ EXPORT_SYMBOL_GPL(devm_extcon_dev_unregister);
 
 /**
  * devm_extcon_register_notifier() - Resource-managed extcon_register_notifier()
- * @dev:	device to allocate extcon device
- * @edev:	the extcon device that has the external connecotr.
- * @id:		the unique id of each external connector in extcon enumeration.
- * @nb:		a notifier block to be registered.
+ * @dev:	the device owning the extcon device being created
+ * @edev:	the extcon device
+ * @id:		the unique id among the extcon enumeration
+ * @nb:		a notifier block to be registered
  *
  * This function manages automatically the notifier of extcon device using
  * device resource management and simplify the control of unregistering
@@ -208,10 +206,10 @@ EXPORT_SYMBOL(devm_extcon_register_notifier);
 /**
  * devm_extcon_unregister_notifier()
 			- Resource-managed extcon_unregister_notifier()
- * @dev:	device to allocate extcon device
- * @edev:	the extcon device that has the external connecotr.
- * @id:		the unique id of each external connector in extcon enumeration.
- * @nb:		a notifier block to be registered.
+ * @dev:	the device owning the extcon device being created
+ * @edev:	the extcon device
+ * @id:		the unique id among the extcon enumeration
+ * @nb:		a notifier block to be registered
  */
 void devm_extcon_unregister_notifier(struct device *dev,
 				struct extcon_dev *edev, unsigned int id,
@@ -225,9 +223,9 @@ EXPORT_SYMBOL(devm_extcon_unregister_notifier);
 /**
  * devm_extcon_register_notifier_all()
  *		- Resource-managed extcon_register_notifier_all()
- * @dev:	device to allocate extcon device
- * @edev:	the extcon device that has the external connecotr.
- * @nb:		a notifier block to be registered.
+ * @dev:	the device owning the extcon device being created
+ * @edev:	the extcon device
+ * @nb:		a notifier block to be registered
  *
  * This function manages automatically the notifier of extcon device using
  * device resource management and simplify the control of unregistering
@@ -263,9 +261,9 @@ EXPORT_SYMBOL(devm_extcon_register_notifier_all);
 /**
  * devm_extcon_unregister_notifier_all()
  *		- Resource-managed extcon_unregister_notifier_all()
- * @dev:	device to allocate extcon device
- * @edev:	the extcon device that has the external connecotr.
- * @nb:		a notifier block to be registered.
+ * @dev:	the device owning the extcon device being created
+ * @edev:	the extcon device
+ * @nb:		a notifier block to be registered
  */
 void devm_extcon_unregister_notifier_all(struct device *dev,
 				struct extcon_dev *edev,
diff --git a/drivers/extcon/extcon-intel-int3496.c b/drivers/extcon/extcon-intel-int3496.c
index d9f9afe45961..1a45e745717d 100644
--- a/drivers/extcon/extcon-intel-int3496.c
+++ b/drivers/extcon/extcon-intel-int3496.c
@@ -171,7 +171,7 @@ static int int3496_remove(struct platform_device *pdev)
 	return 0;
 }
 
-static struct acpi_device_id int3496_acpi_match[] = {
+static const struct acpi_device_id int3496_acpi_match[] = {
 	{ "INT3496" },
 	{ }
 };
diff --git a/drivers/extcon/extcon-max77693.c b/drivers/extcon/extcon-max77693.c
index 62163468f205..7a5856809047 100644
--- a/drivers/extcon/extcon-max77693.c
+++ b/drivers/extcon/extcon-max77693.c
@@ -811,9 +811,8 @@ static int max77693_muic_chg_handler(struct max77693_muic_info *info)
 			 */
 			extcon_set_state_sync(info->edev, EXTCON_CHG_USB_DCP,
 						attached);
-			if (!cable_attached)
-				extcon_set_state_sync(info->edev,
-					EXTCON_DISP_MHL, cable_attached);
+			extcon_set_state_sync(info->edev, EXTCON_DISP_MHL,
+						cable_attached);
 			break;
 		}
 
diff --git a/drivers/extcon/extcon-usbc-cros-ec.c b/drivers/extcon/extcon-usbc-cros-ec.c
new file mode 100644
index 000000000000..598956f1dcae
--- /dev/null
+++ b/drivers/extcon/extcon-usbc-cros-ec.c
@@ -0,0 +1,417 @@
+/**
+ * drivers/extcon/extcon-usbc-cros-ec - ChromeOS Embedded Controller extcon
+ *
+ * Copyright (C) 2017 Google, Inc
+ * Author: Benson Leung <bleung@chromium.org>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/extcon.h>
+#include <linux/kernel.h>
+#include <linux/mfd/cros_ec.h>
+#include <linux/module.h>
+#include <linux/notifier.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/sched.h>
+
+struct cros_ec_extcon_info {
+	struct device *dev;
+	struct extcon_dev *edev;
+
+	int port_id;
+
+	struct cros_ec_device *ec;
+
+	struct notifier_block notifier;
+
+	bool dp; /* DisplayPort enabled */
+	bool mux; /* SuperSpeed (usb3) enabled */
+	unsigned int power_type;
+};
+
+static const unsigned int usb_type_c_cable[] = {
+	EXTCON_DISP_DP,
+	EXTCON_NONE,
+};
+
+/**
+ * cros_ec_pd_command() - Send a command to the EC.
+ * @info: pointer to struct cros_ec_extcon_info
+ * @command: EC command
+ * @version: EC command version
+ * @outdata: EC command output data
+ * @outsize: Size of outdata
+ * @indata: EC command input data
+ * @insize: Size of indata
+ *
+ * Return: 0 on success, <0 on failure.
+ */
+static int cros_ec_pd_command(struct cros_ec_extcon_info *info,
+			      unsigned int command,
+			      unsigned int version,
+			      void *outdata,
+			      unsigned int outsize,
+			      void *indata,
+			      unsigned int insize)
+{
+	struct cros_ec_command *msg;
+	int ret;
+
+	msg = kzalloc(sizeof(*msg) + max(outsize, insize), GFP_KERNEL);
+	if (!msg)
+		return -ENOMEM;
+
+	msg->version = version;
+	msg->command = command;
+	msg->outsize = outsize;
+	msg->insize = insize;
+
+	if (outsize)
+		memcpy(msg->data, outdata, outsize);
+
+	ret = cros_ec_cmd_xfer_status(info->ec, msg);
+	if (ret >= 0 && insize)
+		memcpy(indata, msg->data, insize);
+
+	kfree(msg);
+	return ret;
+}
+
+/**
+ * cros_ec_usb_get_power_type() - Get power type info about PD device attached
+ * to given port.
+ * @info: pointer to struct cros_ec_extcon_info
+ *
+ * Return: power type on success, <0 on failure.
+ */
+static int cros_ec_usb_get_power_type(struct cros_ec_extcon_info *info)
+{
+	struct ec_params_usb_pd_power_info req;
+	struct ec_response_usb_pd_power_info resp;
+	int ret;
+
+	req.port = info->port_id;
+	ret = cros_ec_pd_command(info, EC_CMD_USB_PD_POWER_INFO, 0,
+				 &req, sizeof(req), &resp, sizeof(resp));
+	if (ret < 0)
+		return ret;
+
+	return resp.type;
+}
+
+/**
+ * cros_ec_usb_get_pd_mux_state() - Get PD mux state for given port.
+ * @info: pointer to struct cros_ec_extcon_info
+ *
+ * Return: PD mux state on success, <0 on failure.
+ */
+static int cros_ec_usb_get_pd_mux_state(struct cros_ec_extcon_info *info)
+{
+	struct ec_params_usb_pd_mux_info req;
+	struct ec_response_usb_pd_mux_info resp;
+	int ret;
+
+	req.port = info->port_id;
+	ret = cros_ec_pd_command(info, EC_CMD_USB_PD_MUX_INFO, 0,
+				 &req, sizeof(req),
+				 &resp, sizeof(resp));
+	if (ret < 0)
+		return ret;
+
+	return resp.flags;
+}
+
+/**
+ * cros_ec_usb_get_role() - Get role info about possible PD device attached to a
+ * given port.
+ * @info: pointer to struct cros_ec_extcon_info
+ * @polarity: pointer to cable polarity (return value)
+ *
+ * Return: role info on success, -ENOTCONN if no cable is connected, <0 on
+ * failure.
+ */
+static int cros_ec_usb_get_role(struct cros_ec_extcon_info *info,
+				bool *polarity)
+{
+	struct ec_params_usb_pd_control pd_control;
+	struct ec_response_usb_pd_control_v1 resp;
+	int ret;
+
+	pd_control.port = info->port_id;
+	pd_control.role = USB_PD_CTRL_ROLE_NO_CHANGE;
+	pd_control.mux = USB_PD_CTRL_MUX_NO_CHANGE;
+	ret = cros_ec_pd_command(info, EC_CMD_USB_PD_CONTROL, 1,
+				 &pd_control, sizeof(pd_control),
+				 &resp, sizeof(resp));
+	if (ret < 0)
+		return ret;
+
+	if (!(resp.enabled & PD_CTRL_RESP_ENABLED_CONNECTED))
+		return -ENOTCONN;
+
+	*polarity = resp.polarity;
+
+	return resp.role;
+}
+
+/**
+ * cros_ec_pd_get_num_ports() - Get number of EC charge ports.
+ * @info: pointer to struct cros_ec_extcon_info
+ *
+ * Return: number of ports on success, <0 on failure.
+ */
+static int cros_ec_pd_get_num_ports(struct cros_ec_extcon_info *info)
+{
+	struct ec_response_usb_pd_ports resp;
+	int ret;
+
+	ret = cros_ec_pd_command(info, EC_CMD_USB_PD_PORTS,
+				 0, NULL, 0, &resp, sizeof(resp));
+	if (ret < 0)
+		return ret;
+
+	return resp.num_ports;
+}
+
+static int extcon_cros_ec_detect_cable(struct cros_ec_extcon_info *info,
+				       bool force)
+{
+	struct device *dev = info->dev;
+	int role, power_type;
+	bool polarity = false;
+	bool dp = false;
+	bool mux = false;
+	bool hpd = false;
+
+	power_type = cros_ec_usb_get_power_type(info);
+	if (power_type < 0) {
+		dev_err(dev, "failed getting power type err = %d\n",
+			power_type);
+		return power_type;
+	}
+
+	role = cros_ec_usb_get_role(info, &polarity);
+	if (role < 0) {
+		if (role != -ENOTCONN) {
+			dev_err(dev, "failed getting role err = %d\n", role);
+			return role;
+		}
+	} else {
+		int pd_mux_state;
+
+		pd_mux_state = cros_ec_usb_get_pd_mux_state(info);
+		if (pd_mux_state < 0)
+			pd_mux_state = USB_PD_MUX_USB_ENABLED;
+
+		dp = pd_mux_state & USB_PD_MUX_DP_ENABLED;
+		mux = pd_mux_state & USB_PD_MUX_USB_ENABLED;
+		hpd = pd_mux_state & USB_PD_MUX_HPD_IRQ;
+	}
+
+	if (force || info->dp != dp || info->mux != mux ||
+		info->power_type != power_type) {
+
+		info->dp = dp;
+		info->mux = mux;
+		info->power_type = power_type;
+
+		extcon_set_state(info->edev, EXTCON_DISP_DP, dp);
+
+		extcon_set_property(info->edev, EXTCON_DISP_DP,
+				    EXTCON_PROP_USB_TYPEC_POLARITY,
+				    (union extcon_property_value)(int)polarity);
+		extcon_set_property(info->edev, EXTCON_DISP_DP,
+				    EXTCON_PROP_USB_SS,
+				    (union extcon_property_value)(int)mux);
+		extcon_set_property(info->edev, EXTCON_DISP_DP,
+				    EXTCON_PROP_DISP_HPD,
+				    (union extcon_property_value)(int)hpd);
+
+		extcon_sync(info->edev, EXTCON_DISP_DP);
+
+	} else if (hpd) {
+		extcon_set_property(info->edev, EXTCON_DISP_DP,
+				    EXTCON_PROP_DISP_HPD,
+				    (union extcon_property_value)(int)hpd);
+		extcon_sync(info->edev, EXTCON_DISP_DP);
+	}
+
+	return 0;
+}
+
+static int extcon_cros_ec_event(struct notifier_block *nb,
+				unsigned long queued_during_suspend,
+				void *_notify)
+{
+	struct cros_ec_extcon_info *info;
+	struct cros_ec_device *ec;
+	u32 host_event;
+
+	info = container_of(nb, struct cros_ec_extcon_info, notifier);
+	ec = info->ec;
+
+	host_event = cros_ec_get_host_event(ec);
+	if (host_event & (EC_HOST_EVENT_MASK(EC_HOST_EVENT_PD_MCU) |
+			  EC_HOST_EVENT_MASK(EC_HOST_EVENT_USB_MUX))) {
+		extcon_cros_ec_detect_cable(info, false);
+		return NOTIFY_OK;
+	}
+
+	return NOTIFY_DONE;
+}
+
+static int extcon_cros_ec_probe(struct platform_device *pdev)
+{
+	struct cros_ec_extcon_info *info;
+	struct cros_ec_device *ec = dev_get_drvdata(pdev->dev.parent);
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	int numports, ret;
+
+	info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL);
+	if (!info)
+		return -ENOMEM;
+
+	info->dev = dev;
+	info->ec = ec;
+
+	if (np) {
+		u32 port;
+
+		ret = of_property_read_u32(np, "google,usb-port-id", &port);
+		if (ret < 0) {
+			dev_err(dev, "Missing google,usb-port-id property\n");
+			return ret;
+		}
+		info->port_id = port;
+	} else {
+		info->port_id = pdev->id;
+	}
+
+	numports = cros_ec_pd_get_num_ports(info);
+	if (numports < 0) {
+		dev_err(dev, "failed getting number of ports! ret = %d\n",
+			numports);
+		return numports;
+	}
+
+	if (info->port_id >= numports) {
+		dev_err(dev, "This system only supports %d ports\n", numports);
+		return -ENODEV;
+	}
+
+	info->edev = devm_extcon_dev_allocate(dev, usb_type_c_cable);
+	if (IS_ERR(info->edev)) {
+		dev_err(dev, "failed to allocate extcon device\n");
+		return -ENOMEM;
+	}
+
+	ret = devm_extcon_dev_register(dev, info->edev);
+	if (ret < 0) {
+		dev_err(dev, "failed to register extcon device\n");
+		return ret;
+	}
+
+	extcon_set_property_capability(info->edev, EXTCON_DISP_DP,
+				       EXTCON_PROP_USB_TYPEC_POLARITY);
+	extcon_set_property_capability(info->edev, EXTCON_DISP_DP,
+				       EXTCON_PROP_USB_SS);
+	extcon_set_property_capability(info->edev, EXTCON_DISP_DP,
+				       EXTCON_PROP_DISP_HPD);
+
+	platform_set_drvdata(pdev, info);
+
+	/* Get PD events from the EC */
+	info->notifier.notifier_call = extcon_cros_ec_event;
+	ret = blocking_notifier_chain_register(&info->ec->event_notifier,
+					       &info->notifier);
+	if (ret < 0) {
+		dev_err(dev, "failed to register notifier\n");
+		return ret;
+	}
+
+	/* Perform initial detection */
+	ret = extcon_cros_ec_detect_cable(info, true);
+	if (ret < 0) {
+		dev_err(dev, "failed to detect initial cable state\n");
+		goto unregister_notifier;
+	}
+
+	return 0;
+
+unregister_notifier:
+	blocking_notifier_chain_unregister(&info->ec->event_notifier,
+					   &info->notifier);
+	return ret;
+}
+
+static int extcon_cros_ec_remove(struct platform_device *pdev)
+{
+	struct cros_ec_extcon_info *info = platform_get_drvdata(pdev);
+
+	blocking_notifier_chain_unregister(&info->ec->event_notifier,
+					   &info->notifier);
+
+	return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int extcon_cros_ec_suspend(struct device *dev)
+{
+	return 0;
+}
+
+static int extcon_cros_ec_resume(struct device *dev)
+{
+	int ret;
+	struct cros_ec_extcon_info *info = dev_get_drvdata(dev);
+
+	ret = extcon_cros_ec_detect_cable(info, true);
+	if (ret < 0)
+		dev_err(dev, "failed to detect cable state on resume\n");
+
+	return 0;
+}
+
+static const struct dev_pm_ops extcon_cros_ec_dev_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(extcon_cros_ec_suspend, extcon_cros_ec_resume)
+};
+
+#define DEV_PM_OPS	(&extcon_cros_ec_dev_pm_ops)
+#else
+#define DEV_PM_OPS	NULL
+#endif /* CONFIG_PM_SLEEP */
+
+#ifdef CONFIG_OF
+static const struct of_device_id extcon_cros_ec_of_match[] = {
+	{ .compatible = "google,extcon-usbc-cros-ec" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, extcon_cros_ec_of_match);
+#endif /* CONFIG_OF */
+
+static struct platform_driver extcon_cros_ec_driver = {
+	.driver = {
+		.name  = "extcon-usbc-cros-ec",
+		.of_match_table = of_match_ptr(extcon_cros_ec_of_match),
+		.pm = DEV_PM_OPS,
+	},
+	.remove  = extcon_cros_ec_remove,
+	.probe   = extcon_cros_ec_probe,
+};
+
+module_platform_driver(extcon_cros_ec_driver);
+
+MODULE_DESCRIPTION("ChromeOS Embedded Controller extcon driver");
+MODULE_AUTHOR("Benson Leung <bleung@chromium.org>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/extcon/extcon.c b/drivers/extcon/extcon.c
index 8eccf7b14937..35e9fb885486 100644
--- a/drivers/extcon/extcon.c
+++ b/drivers/extcon/extcon.c
@@ -1,7 +1,5 @@
 /*
- *  drivers/extcon/extcon.c - External Connector (extcon) framework.
- *
- *  External connector (extcon) class driver
+ * drivers/extcon/extcon.c - External Connector (extcon) framework.
  *
  * Copyright (C) 2015 Samsung Electronics
  * Author: Chanwoo Choi <cw00.choi@samsung.com>
@@ -37,7 +35,6 @@
 #include "extcon.h"
 
 #define SUPPORTED_CABLE_MAX	32
-#define CABLE_NAME_MAX		30
 
 struct __extcon_info {
 	unsigned int type;
@@ -200,13 +197,13 @@ struct __extcon_info {
 };
 
 /**
- * struct extcon_cable - An internal data for each cable of extcon device.
- * @edev:		The extcon device
- * @cable_index:	Index of this cable in the edev
- * @attr_g:		Attribute group for the cable
+ * struct extcon_cable - An internal data for an external connector.
+ * @edev:		the extcon device
+ * @cable_index:	the index of this cable in the edev
+ * @attr_g:		the attribute group for the cable
  * @attr_name:		"name" sysfs entry
  * @attr_state:		"state" sysfs entry
- * @attrs:		Array pointing to attr_name and attr_state for attr_g
+ * @attrs:		the array pointing to attr_name and attr_state for attr_g
  */
 struct extcon_cable {
 	struct extcon_dev *edev;
@@ -234,15 +231,6 @@ static struct class *extcon_class;
 static LIST_HEAD(extcon_dev_list);
 static DEFINE_MUTEX(extcon_dev_list_lock);
 
-/**
- * check_mutually_exclusive - Check if new_state violates mutually_exclusive
- *			      condition.
- * @edev:	the extcon device
- * @new_state:	new cable attach status for @edev
- *
- * Returns 0 if nothing violates. Returns the index + 1 for the first
- * violated condition.
- */
 static int check_mutually_exclusive(struct extcon_dev *edev, u32 new_state)
 {
 	int i = 0;
@@ -417,11 +405,13 @@ static ssize_t cable_state_show(struct device *dev,
 }
 
 /**
- * extcon_sync()	- Synchronize the states for both the attached/detached
- * @edev:		the extcon device that has the cable.
+ * extcon_sync() - Synchronize the state for an external connector.
+ * @edev:	the extcon device
+ *
+ * Note that this function send a notification in order to synchronize
+ * the state and property of an external connector.
  *
- * This function send a notification to synchronize the all states of a
- * specific external connector
+ * Returns 0 if success or error number if fail.
  */
 int extcon_sync(struct extcon_dev *edev, unsigned int id)
 {
@@ -497,9 +487,11 @@ int extcon_sync(struct extcon_dev *edev, unsigned int id)
 EXPORT_SYMBOL_GPL(extcon_sync);
 
 /**
- * extcon_get_state() - Get the state of a external connector.
- * @edev:	the extcon device that has the cable.
- * @id:		the unique id of each external connector in extcon enumeration.
+ * extcon_get_state() - Get the state of an external connector.
+ * @edev:	the extcon device
+ * @id:		the unique id indicating an external connector
+ *
+ * Returns 0 if success or error number if fail.
  */
 int extcon_get_state(struct extcon_dev *edev, const unsigned int id)
 {
@@ -522,20 +514,19 @@ int extcon_get_state(struct extcon_dev *edev, const unsigned int id)
 EXPORT_SYMBOL_GPL(extcon_get_state);
 
 /**
- * extcon_set_state() - Set the state of a external connector.
- *			without a notification.
- * @edev:		the extcon device that has the cable.
- * @id:			the unique id of each external connector
- *			in extcon enumeration.
- * @state:		the new cable status. The default semantics is
- *			true: attached / false: detached.
+ * extcon_set_state() - Set the state of an external connector.
+ * @edev:	the extcon device
+ * @id:		the unique id indicating an external connector
+ * @state:	the new state of an external connector.
+ *		the default semantics is true: attached / false: detached.
+ *
+ * Note that this function set the state of an external connector without
+ * a notification. To synchronize the state of an external connector,
+ * have to use extcon_set_state_sync() and extcon_sync().
  *
- * This function only set the state of a external connector without
- * a notification. To synchronize the data of a external connector,
- * use extcon_set_state_sync() and extcon_sync().
+ * Returns 0 if success or error number if fail.
  */
-int extcon_set_state(struct extcon_dev *edev, unsigned int id,
-				bool cable_state)
+int extcon_set_state(struct extcon_dev *edev, unsigned int id, bool state)
 {
 	unsigned long flags;
 	int index, ret = 0;
@@ -550,11 +541,11 @@ int extcon_set_state(struct extcon_dev *edev, unsigned int id,
 	spin_lock_irqsave(&edev->lock, flags);
 
 	/* Check whether the external connector's state is changed. */
-	if (!is_extcon_changed(edev, index, cable_state))
+	if (!is_extcon_changed(edev, index, state))
 		goto out;
 
 	if (check_mutually_exclusive(edev,
-		(edev->state & ~BIT(index)) | (cable_state & BIT(index)))) {
+		(edev->state & ~BIT(index)) | (state & BIT(index)))) {
 		ret = -EPERM;
 		goto out;
 	}
@@ -563,11 +554,11 @@ int extcon_set_state(struct extcon_dev *edev, unsigned int id,
 	 * Initialize the value of extcon property before setting
 	 * the detached state for an external connector.
 	 */
-	if (!cable_state)
+	if (!state)
 		init_property(edev, id, index);
 
-	/* Update the state for a external connector. */
-	if (cable_state)
+	/* Update the state for an external connector. */
+	if (state)
 		edev->state |= BIT(index);
 	else
 		edev->state &= ~(BIT(index));
@@ -579,19 +570,18 @@ out:
 EXPORT_SYMBOL_GPL(extcon_set_state);
 
 /**
- * extcon_set_state_sync() - Set the state of a external connector
- *			with a notification.
- * @edev:		the extcon device that has the cable.
- * @id:			the unique id of each external connector
- *			in extcon enumeration.
- * @state:		the new cable status. The default semantics is
- *			true: attached / false: detached.
+ * extcon_set_state_sync() - Set the state of an external connector with sync.
+ * @edev:	the extcon device
+ * @id:		the unique id indicating an external connector
+ * @state:	the new state of external connector.
+ *		the default semantics is true: attached / false: detached.
+ *
+ * Note that this function set the state of external connector
+ * and synchronize the state by sending a notification.
  *
- * This function set the state of external connector and synchronize the data
- * by usning a notification.
+ * Returns 0 if success or error number if fail.
  */
-int extcon_set_state_sync(struct extcon_dev *edev, unsigned int id,
-				bool cable_state)
+int extcon_set_state_sync(struct extcon_dev *edev, unsigned int id, bool state)
 {
 	int ret, index;
 	unsigned long flags;
@@ -602,12 +592,12 @@ int extcon_set_state_sync(struct extcon_dev *edev, unsigned int id,
 
 	/* Check whether the external connector's state is changed. */
 	spin_lock_irqsave(&edev->lock, flags);
-	ret = is_extcon_changed(edev, index, cable_state);
+	ret = is_extcon_changed(edev, index, state);
 	spin_unlock_irqrestore(&edev->lock, flags);
 	if (!ret)
 		return 0;
 
-	ret = extcon_set_state(edev, id, cable_state);
+	ret = extcon_set_state(edev, id, state);
 	if (ret < 0)
 		return ret;
 
@@ -616,19 +606,18 @@ int extcon_set_state_sync(struct extcon_dev *edev, unsigned int id,
 EXPORT_SYMBOL_GPL(extcon_set_state_sync);
 
 /**
- * extcon_get_property() - Get the property value of a specific cable.
- * @edev:		the extcon device that has the cable.
- * @id:			the unique id of each external connector
- *			in extcon enumeration.
- * @prop:		the property id among enum extcon_property.
- * @prop_val:		the pointer which store the value of property.
+ * extcon_get_property() - Get the property value of an external connector.
+ * @edev:	the extcon device
+ * @id:		the unique id indicating an external connector
+ * @prop:	the property id indicating an extcon property
+ * @prop_val:	the pointer which store the value of extcon property
  *
- * When getting the property value of external connector, the external connector
- * should be attached. If detached state, function just return 0 without
- * property value. Also, the each property should be included in the list of
- * supported properties according to the type of external connectors.
+ * Note that when getting the property value of external connector,
+ * the external connector should be attached. If detached state, function
+ * return 0 without property value. Also, the each property should be
+ * included in the list of supported properties according to extcon type.
  *
- * Returns 0 if success or error number if fail
+ * Returns 0 if success or error number if fail.
  */
 int extcon_get_property(struct extcon_dev *edev, unsigned int id,
 				unsigned int prop,
@@ -698,17 +687,16 @@ int extcon_get_property(struct extcon_dev *edev, unsigned int id,
 EXPORT_SYMBOL_GPL(extcon_get_property);
 
 /**
- * extcon_set_property() - Set the property value of a specific cable.
- * @edev:		the extcon device that has the cable.
- * @id:			the unique id of each external connector
- *			in extcon enumeration.
- * @prop:		the property id among enum extcon_property.
- * @prop_val:		the pointer including the new value of property.
+ * extcon_set_property() - Set the property value of an external connector.
+ * @edev:	the extcon device
+ * @id:		the unique id indicating an external connector
+ * @prop:	the property id indicating an extcon property
+ * @prop_val:	the pointer including the new value of extcon property
  *
- * The each property should be included in the list of supported properties
- * according to the type of external connectors.
+ * Note that each property should be included in the list of supported
+ * properties according to the extcon type.
  *
- * Returns 0 if success or error number if fail
+ * Returns 0 if success or error number if fail.
  */
 int extcon_set_property(struct extcon_dev *edev, unsigned int id,
 				unsigned int prop,
@@ -766,15 +754,14 @@ int extcon_set_property(struct extcon_dev *edev, unsigned int id,
 EXPORT_SYMBOL_GPL(extcon_set_property);
 
 /**
- * extcon_set_property_sync() - Set the property value of a specific cable
-			with a notification.
- * @prop_val:		the pointer including the new value of property.
+ * extcon_set_property_sync() - Set property of an external connector with sync.
+ * @prop_val:	the pointer including the new value of extcon property
  *
- * When setting the property value of external connector, the external connector
- * should be attached. The each property should be included in the list of
- * supported properties according to the type of external connectors.
+ * Note that when setting the property value of external connector,
+ * the external connector should be attached. The each property should
+ * be included in the list of supported properties according to extcon type.
  *
- * Returns 0 if success or error number if fail
+ * Returns 0 if success or error number if fail.
  */
 int extcon_set_property_sync(struct extcon_dev *edev, unsigned int id,
 				unsigned int prop,
@@ -791,12 +778,11 @@ int extcon_set_property_sync(struct extcon_dev *edev, unsigned int id,
 EXPORT_SYMBOL_GPL(extcon_set_property_sync);
 
 /**
- * extcon_get_property_capability() - Get the capability of property
- *			of an external connector.
- * @edev:		the extcon device that has the cable.
- * @id:			the unique id of each external connector
- *			in extcon enumeration.
- * @prop:		the property id among enum extcon_property.
+ * extcon_get_property_capability() - Get the capability of the property
+ *					for an external connector.
+ * @edev:	the extcon device
+ * @id:		the unique id indicating an external connector
+ * @prop:	the property id indicating an extcon property
  *
  * Returns 1 if the property is available or 0 if not available.
  */
@@ -822,18 +808,17 @@ int extcon_get_property_capability(struct extcon_dev *edev, unsigned int id,
 EXPORT_SYMBOL_GPL(extcon_get_property_capability);
 
 /**
- * extcon_set_property_capability() - Set the capability of a property
- *			of an external connector.
- * @edev:		the extcon device that has the cable.
- * @id:			the unique id of each external connector
- *			in extcon enumeration.
- * @prop:		the property id among enum extcon_property.
+ * extcon_set_property_capability() - Set the capability of the property
+ *					for an external connector.
+ * @edev:	the extcon device
+ * @id:		the unique id indicating an external connector
+ * @prop:	the property id indicating an extcon property
  *
- * This function set the capability of a property for an external connector
- * to mark the bit in capability bitmap which mean the available state of
- * a property.
+ * Note that this function set the capability of the property
+ * for an external connector in order to mark the bit in capability
+ * bitmap which mean the available state of the property.
  *
- * Returns 0 if success or error number if fail
+ * Returns 0 if success or error number if fail.
  */
 int extcon_set_property_capability(struct extcon_dev *edev, unsigned int id,
 					unsigned int prop)
@@ -881,8 +866,10 @@ int extcon_set_property_capability(struct extcon_dev *edev, unsigned int id,
 EXPORT_SYMBOL_GPL(extcon_set_property_capability);
 
 /**
- * extcon_get_extcon_dev() - Get the extcon device instance from the name
- * @extcon_name:	The extcon name provided with extcon_dev_register()
+ * extcon_get_extcon_dev() - Get the extcon device instance from the name.
+ * @extcon_name:	the extcon name provided with extcon_dev_register()
+ *
+ * Return the pointer of extcon device if success or ERR_PTR(err) if fail.
  */
 struct extcon_dev *extcon_get_extcon_dev(const char *extcon_name)
 {
@@ -904,15 +891,17 @@ out:
 EXPORT_SYMBOL_GPL(extcon_get_extcon_dev);
 
 /**
- * extcon_register_notifier() - Register a notifiee to get notified by
- *				any attach status changes from the extcon.
- * @edev:	the extcon device that has the external connecotr.
- * @id:		the unique id of each external connector in extcon enumeration.
- * @nb:		a notifier block to be registered.
+ * extcon_register_notifier() - Register a notifier block to get notified by
+ *				any state changes from the extcon.
+ * @edev:	the extcon device
+ * @id:		the unique id indicating an external connector
+ * @nb:		a notifier block to be registered
  *
  * Note that the second parameter given to the callback of nb (val) is
- * "old_state", not the current state. The current state can be retrieved
- * by looking at the third pameter (edev pointer)'s state value.
+ * the current state of an external connector and the third pameter
+ * is the pointer of extcon device.
+ *
+ * Returns 0 if success or error number if fail.
  */
 int extcon_register_notifier(struct extcon_dev *edev, unsigned int id,
 			     struct notifier_block *nb)
@@ -936,10 +925,12 @@ int extcon_register_notifier(struct extcon_dev *edev, unsigned int id,
 EXPORT_SYMBOL_GPL(extcon_register_notifier);
 
 /**
- * extcon_unregister_notifier() - Unregister a notifiee from the extcon device.
- * @edev:	the extcon device that has the external connecotr.
- * @id:		the unique id of each external connector in extcon enumeration.
- * @nb:		a notifier block to be registered.
+ * extcon_unregister_notifier() - Unregister a notifier block from the extcon.
+ * @edev:	the extcon device
+ * @id:		the unique id indicating an external connector
+ * @nb:		a notifier block to be registered
+ *
+ * Returns 0 if success or error number if fail.
  */
 int extcon_unregister_notifier(struct extcon_dev *edev, unsigned int id,
 				struct notifier_block *nb)
@@ -963,16 +954,16 @@ int extcon_unregister_notifier(struct extcon_dev *edev, unsigned int id,
 EXPORT_SYMBOL_GPL(extcon_unregister_notifier);
 
 /**
- * extcon_register_notifier_all() - Register a notifier block for all connectors
- * @edev:	the extcon device that has the external connector.
- * @nb:		a notifier block to be registered.
+ * extcon_register_notifier_all() - Register a notifier block for all connectors.
+ * @edev:	the extcon device
+ * @nb:		a notifier block to be registered
  *
- * This function registers a notifier block in order to receive the state
- * change of all supported external connectors from extcon device.
+ * Note that this function registers a notifier block in order to receive
+ * the state change of all supported external connectors from extcon device.
  * And the second parameter given to the callback of nb (val) is
- * the current state and third parameter is the edev pointer.
+ * the current state and the third pameter is the pointer of extcon device.
  *
- * Returns 0 if success or error number if fail
+ * Returns 0 if success or error number if fail.
  */
 int extcon_register_notifier_all(struct extcon_dev *edev,
 				struct notifier_block *nb)
@@ -993,10 +984,10 @@ EXPORT_SYMBOL_GPL(extcon_register_notifier_all);
 
 /**
  * extcon_unregister_notifier_all() - Unregister a notifier block from extcon.
- * @edev:	the extcon device that has the external connecotr.
- * @nb:		a notifier block to be registered.
+ * @edev:	the extcon device
+ * @nb:		a notifier block to be registered
  *
- * Returns 0 if success or error number if fail
+ * Returns 0 if success or error number if fail.
  */
 int extcon_unregister_notifier_all(struct extcon_dev *edev,
 				struct notifier_block *nb)
@@ -1045,15 +1036,14 @@ static void dummy_sysfs_dev_release(struct device *dev)
 
 /*
  * extcon_dev_allocate() - Allocate the memory of extcon device.
- * @supported_cable:	Array of supported extcon ending with EXTCON_NONE.
- *			If supported_cable is NULL, cable name related APIs
- *			are disabled.
+ * @supported_cable:	the array of the supported external connectors
+ *			ending with EXTCON_NONE.
  *
- * This function allocates the memory for extcon device without allocating
- * memory in each extcon provider driver and initialize default setting for
- * extcon device.
+ * Note that this function allocates the memory for extcon device 
+ * and initialize default setting for the extcon device.
  *
- * Return the pointer of extcon device if success or ERR_PTR(err) if fail
+ * Returns the pointer memory of allocated extcon_dev if success
+ * or ERR_PTR(err) if fail.
  */
 struct extcon_dev *extcon_dev_allocate(const unsigned int *supported_cable)
 {
@@ -1074,7 +1064,7 @@ struct extcon_dev *extcon_dev_allocate(const unsigned int *supported_cable)
 
 /*
  * extcon_dev_free() - Free the memory of extcon device.
- * @edev:	the extcon device to free
+ * @edev:	the extcon device
  */
 void extcon_dev_free(struct extcon_dev *edev)
 {
@@ -1083,13 +1073,18 @@ void extcon_dev_free(struct extcon_dev *edev)
 EXPORT_SYMBOL_GPL(extcon_dev_free);
 
 /**
- * extcon_dev_register() - Register a new extcon device
- * @edev	: the new extcon device (should be allocated before calling)
+ * extcon_dev_register() - Register an new extcon device
+ * @edev:	the extcon device to be registered
  *
  * Among the members of edev struct, please set the "user initializing data"
- * in any case and set the "optional callbacks" if required. However, please
  * do not set the values of "internal data", which are initialized by
  * this function.
+ *
+ * Note that before calling this funciton, have to allocate the memory
+ * of an extcon device by using the extcon_dev_allocate(). And the extcon
+ * dev should include the supported_cable information.
+ *
+ * Returns 0 if success or error number if fail.
  */
 int extcon_dev_register(struct extcon_dev *edev)
 {
@@ -1296,7 +1291,7 @@ EXPORT_SYMBOL_GPL(extcon_dev_register);
 
 /**
  * extcon_dev_unregister() - Unregister the extcon device.
- * @edev:	the extcon device instance to be unregistered.
+ * @edev:	the extcon device to be unregistered.
  *
  * Note that this does not call kfree(edev) because edev was not allocated
  * by this class.
@@ -1342,11 +1337,11 @@ EXPORT_SYMBOL_GPL(extcon_dev_unregister);
 
 #ifdef CONFIG_OF
 /*
- * extcon_get_edev_by_phandle - Get the extcon device from devicetree
- * @dev - instance to the given device
- * @index - index into list of extcon_dev
+ * extcon_get_edev_by_phandle - Get the extcon device from devicetree.
+ * @dev		: the instance to the given device
+ * @index	: the index into list of extcon_dev
  *
- * return the instance of extcon device
+ * Return the pointer of extcon device if success or ERR_PTR(err) if fail.
  */
 struct extcon_dev *extcon_get_edev_by_phandle(struct device *dev, int index)
 {
@@ -1363,8 +1358,8 @@ struct extcon_dev *extcon_get_edev_by_phandle(struct device *dev, int index)
 
 	node = of_parse_phandle(dev->of_node, "extcon", index);
 	if (!node) {
-		dev_dbg(dev, "failed to get phandle in %s node\n",
-			dev->of_node->full_name);
+		dev_dbg(dev, "failed to get phandle in %pOF node\n",
+			dev->of_node);
 		return ERR_PTR(-ENODEV);
 	}
 
@@ -1411,8 +1406,6 @@ static void __exit extcon_class_exit(void)
 module_exit(extcon_class_exit);
 
 MODULE_AUTHOR("Chanwoo Choi <cw00.choi@samsung.com>");
-MODULE_AUTHOR("Mike Lockwood <lockwood@android.com>");
-MODULE_AUTHOR("Donggeun Kim <dg77.kim@samsung.com>");
 MODULE_AUTHOR("MyungJoo Ham <myungjoo.ham@samsung.com>");
-MODULE_DESCRIPTION("External connector (extcon) class driver");
-MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("External Connector (extcon) framework");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/firmware/arm_scpi.c b/drivers/firmware/arm_scpi.c
index 8043e51de897..7da9f1b83ebe 100644
--- a/drivers/firmware/arm_scpi.c
+++ b/drivers/firmware/arm_scpi.c
@@ -357,7 +357,7 @@ struct sensor_value {
 } __packed;
 
 struct dev_pstate_set {
-	u16 dev_id;
+	__le16 dev_id;
 	u8 pstate;
 } __packed;
 
@@ -965,7 +965,7 @@ static int scpi_probe(struct platform_device *pdev)
 
 	count = of_count_phandle_with_args(np, "mboxes", "#mbox-cells");
 	if (count < 0) {
-		dev_err(dev, "no mboxes property in '%s'\n", np->full_name);
+		dev_err(dev, "no mboxes property in '%pOF'\n", np);
 		return -ENODEV;
 	}
 
diff --git a/drivers/firmware/dcdbas.c b/drivers/firmware/dcdbas.c
index 2fe1a130189f..c16600f30611 100644
--- a/drivers/firmware/dcdbas.c
+++ b/drivers/firmware/dcdbas.c
@@ -534,7 +534,7 @@ static struct attribute *dcdbas_dev_attrs[] = {
 	NULL
 };
 
-static struct attribute_group dcdbas_attr_group = {
+static const struct attribute_group dcdbas_attr_group = {
 	.attrs = dcdbas_dev_attrs,
 	.bin_attrs = dcdbas_bin_attrs,
 };
diff --git a/drivers/firmware/dmi-sysfs.c b/drivers/firmware/dmi-sysfs.c
index ef76e5eecf0b..d5de6ee8466d 100644
--- a/drivers/firmware/dmi-sysfs.c
+++ b/drivers/firmware/dmi-sysfs.c
@@ -25,6 +25,7 @@
 #include <linux/slab.h>
 #include <linux/list.h>
 #include <linux/io.h>
+#include <asm/dmi.h>
 
 #define MAX_ENTRY_TYPE 255 /* Most of these aren't used, but we consider
 			      the top entry type is only 8 bits */
@@ -380,7 +381,7 @@ static ssize_t dmi_sel_raw_read_phys32(struct dmi_sysfs_entry *entry,
 	u8 __iomem *mapped;
 	ssize_t wrote = 0;
 
-	mapped = ioremap(sel->access_method_address, sel->area_length);
+	mapped = dmi_remap(sel->access_method_address, sel->area_length);
 	if (!mapped)
 		return -EIO;
 
@@ -390,7 +391,7 @@ static ssize_t dmi_sel_raw_read_phys32(struct dmi_sysfs_entry *entry,
 		wrote++;
 	}
 
-	iounmap(mapped);
+	dmi_unmap(mapped);
 	return wrote;
 }
 
diff --git a/drivers/firmware/efi/Kconfig b/drivers/firmware/efi/Kconfig
index 394db40ed374..2b4c39fdfa91 100644
--- a/drivers/firmware/efi/Kconfig
+++ b/drivers/firmware/efi/Kconfig
@@ -151,6 +151,16 @@ config APPLE_PROPERTIES
 
 	  If unsure, say Y if you have a Mac.  Otherwise N.
 
+config RESET_ATTACK_MITIGATION
+	bool "Reset memory attack mitigation"
+	depends on EFI_STUB
+	help
+	  Request that the firmware clear the contents of RAM after a reboot
+	  using the TCG Platform Reset Attack Mitigation specification. This
+	  protects against an attacker forcibly rebooting the system while it
+	  still contains secrets in RAM, booting another OS and extracting the
+	  secrets.
+
 endmenu
 
 config UEFI_CPER
diff --git a/drivers/firmware/efi/apple-properties.c b/drivers/firmware/efi/apple-properties.c
index c473f4c5ca34..9f6bcf173b0e 100644
--- a/drivers/firmware/efi/apple-properties.c
+++ b/drivers/firmware/efi/apple-properties.c
@@ -18,8 +18,8 @@
 #define pr_fmt(fmt) "apple-properties: " fmt
 
 #include <linux/bootmem.h>
-#include <linux/dmi.h>
 #include <linux/efi.h>
+#include <linux/platform_data/x86/apple.h>
 #include <linux/property.h>
 #include <linux/slab.h>
 #include <linux/ucs2_string.h>
@@ -191,8 +191,7 @@ static int __init map_properties(void)
 	u64 pa_data;
 	int ret;
 
-	if (!dmi_match(DMI_SYS_VENDOR, "Apple Inc.") &&
-	    !dmi_match(DMI_SYS_VENDOR, "Apple Computer, Inc."))
+	if (!x86_apple_machine)
 		return 0;
 
 	pa_data = boot_params.hdr.setup_data;
diff --git a/drivers/firmware/efi/arm-init.c b/drivers/firmware/efi/arm-init.c
index 1027d7b44358..80d1a885def5 100644
--- a/drivers/firmware/efi/arm-init.c
+++ b/drivers/firmware/efi/arm-init.c
@@ -145,6 +145,9 @@ static int __init uefi_init(void)
 					 sizeof(efi_config_table_t),
 					 arch_tables);
 
+	if (!retval)
+		efi.config_table = (unsigned long)efi.systab->tables;
+
 	early_memunmap(config_tables, table_size);
 out:
 	early_memunmap(efi.systab,  sizeof(efi_system_table_t));
@@ -159,6 +162,7 @@ static __init int is_usable_memory(efi_memory_desc_t *md)
 	switch (md->type) {
 	case EFI_LOADER_CODE:
 	case EFI_LOADER_DATA:
+	case EFI_ACPI_RECLAIM_MEMORY:
 	case EFI_BOOT_SERVICES_CODE:
 	case EFI_BOOT_SERVICES_DATA:
 	case EFI_CONVENTIONAL_MEMORY:
@@ -211,6 +215,10 @@ static __init void reserve_regions(void)
 
 			if (!is_usable_memory(md))
 				memblock_mark_nomap(paddr, size);
+
+			/* keep ACPI reclaim memory intact for kexec etc. */
+			if (md->type == EFI_ACPI_RECLAIM_MEMORY)
+				memblock_reserve(paddr, size);
 		}
 	}
 }
diff --git a/drivers/firmware/efi/cper.c b/drivers/firmware/efi/cper.c
index 48a8f69da42a..d2fcafcea07e 100644
--- a/drivers/firmware/efi/cper.c
+++ b/drivers/firmware/efi/cper.c
@@ -534,7 +534,7 @@ static void
 cper_estatus_print_section(const char *pfx, struct acpi_hest_generic_data *gdata,
 			   int sec_no)
 {
-	uuid_le *sec_type = (uuid_le *)gdata->section_type;
+	guid_t *sec_type = (guid_t *)gdata->section_type;
 	__u16 severity;
 	char newpfx[64];
 
@@ -545,12 +545,12 @@ cper_estatus_print_section(const char *pfx, struct acpi_hest_generic_data *gdata
 	printk("%s""Error %d, type: %s\n", pfx, sec_no,
 	       cper_severity_str(severity));
 	if (gdata->validation_bits & CPER_SEC_VALID_FRU_ID)
-		printk("%s""fru_id: %pUl\n", pfx, (uuid_le *)gdata->fru_id);
+		printk("%s""fru_id: %pUl\n", pfx, gdata->fru_id);
 	if (gdata->validation_bits & CPER_SEC_VALID_FRU_TEXT)
 		printk("%s""fru_text: %.20s\n", pfx, gdata->fru_text);
 
 	snprintf(newpfx, sizeof(newpfx), "%s%s", pfx, INDENT_SP);
-	if (!uuid_le_cmp(*sec_type, CPER_SEC_PROC_GENERIC)) {
+	if (guid_equal(sec_type, &CPER_SEC_PROC_GENERIC)) {
 		struct cper_sec_proc_generic *proc_err = acpi_hest_get_payload(gdata);
 
 		printk("%s""section_type: general processor error\n", newpfx);
@@ -558,7 +558,7 @@ cper_estatus_print_section(const char *pfx, struct acpi_hest_generic_data *gdata
 			cper_print_proc_generic(newpfx, proc_err);
 		else
 			goto err_section_too_small;
-	} else if (!uuid_le_cmp(*sec_type, CPER_SEC_PLATFORM_MEM)) {
+	} else if (guid_equal(sec_type, &CPER_SEC_PLATFORM_MEM)) {
 		struct cper_sec_mem_err *mem_err = acpi_hest_get_payload(gdata);
 
 		printk("%s""section_type: memory error\n", newpfx);
@@ -568,7 +568,7 @@ cper_estatus_print_section(const char *pfx, struct acpi_hest_generic_data *gdata
 				       gdata->error_data_length);
 		else
 			goto err_section_too_small;
-	} else if (!uuid_le_cmp(*sec_type, CPER_SEC_PCIE)) {
+	} else if (guid_equal(sec_type, &CPER_SEC_PCIE)) {
 		struct cper_sec_pcie *pcie = acpi_hest_get_payload(gdata);
 
 		printk("%s""section_type: PCIe error\n", newpfx);
@@ -606,7 +606,6 @@ void cper_estatus_print(const char *pfx,
 			const struct acpi_hest_generic_status *estatus)
 {
 	struct acpi_hest_generic_data *gdata;
-	unsigned int data_len;
 	int sec_no = 0;
 	char newpfx[64];
 	__u16 severity;
@@ -617,14 +616,10 @@ void cper_estatus_print(const char *pfx,
 		       "It has been corrected by h/w "
 		       "and requires no further action");
 	printk("%s""event severity: %s\n", pfx, cper_severity_str(severity));
-	data_len = estatus->data_length;
-	gdata = (struct acpi_hest_generic_data *)(estatus + 1);
 	snprintf(newpfx, sizeof(newpfx), "%s%s", pfx, INDENT_SP);
 
-	while (data_len >= acpi_hest_get_size(gdata)) {
+	apei_estatus_for_each_section(estatus, gdata) {
 		cper_estatus_print_section(newpfx, gdata, sec_no);
-		data_len -= acpi_hest_get_record_size(gdata);
-		gdata = acpi_hest_get_next(gdata);
 		sec_no++;
 	}
 }
@@ -653,15 +648,12 @@ int cper_estatus_check(const struct acpi_hest_generic_status *estatus)
 	if (rc)
 		return rc;
 	data_len = estatus->data_length;
-	gdata = (struct acpi_hest_generic_data *)(estatus + 1);
 
-	while (data_len >= acpi_hest_get_size(gdata)) {
+	apei_estatus_for_each_section(estatus, gdata) {
 		gedata_len = acpi_hest_get_error_length(gdata);
 		if (gedata_len > data_len - acpi_hest_get_size(gdata))
 			return -EINVAL;
-
 		data_len -= acpi_hest_get_record_size(gdata);
-		gdata = acpi_hest_get_next(gdata);
 	}
 	if (data_len)
 		return -EINVAL;
diff --git a/drivers/firmware/efi/efi-bgrt.c b/drivers/firmware/efi/efi-bgrt.c
index b58233e4ed71..50793fda7819 100644
--- a/drivers/firmware/efi/efi-bgrt.c
+++ b/drivers/firmware/efi/efi-bgrt.c
@@ -27,26 +27,6 @@ struct bmp_header {
 	u32 size;
 } __packed;
 
-static bool efi_bgrt_addr_valid(u64 addr)
-{
-	efi_memory_desc_t *md;
-
-	for_each_efi_memory_desc(md) {
-		u64 size;
-		u64 end;
-
-		if (md->type != EFI_BOOT_SERVICES_DATA)
-			continue;
-
-		size = md->num_pages << EFI_PAGE_SHIFT;
-		end = md->phys_addr + size;
-		if (addr >= md->phys_addr && addr < end)
-			return true;
-	}
-
-	return false;
-}
-
 void __init efi_bgrt_init(struct acpi_table_header *table)
 {
 	void *image;
@@ -85,7 +65,7 @@ void __init efi_bgrt_init(struct acpi_table_header *table)
 		goto out;
 	}
 
-	if (!efi_bgrt_addr_valid(bgrt->image_address)) {
+	if (efi_mem_type(bgrt->image_address) != EFI_BOOT_SERVICES_DATA) {
 		pr_notice("Ignoring BGRT: invalid image address\n");
 		goto out;
 	}
diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c
index 045d6d311bde..f70febf680c3 100644
--- a/drivers/firmware/efi/efi.c
+++ b/drivers/firmware/efi/efi.c
@@ -55,6 +55,25 @@ struct efi __read_mostly efi = {
 };
 EXPORT_SYMBOL(efi);
 
+static unsigned long *efi_tables[] = {
+	&efi.mps,
+	&efi.acpi,
+	&efi.acpi20,
+	&efi.smbios,
+	&efi.smbios3,
+	&efi.sal_systab,
+	&efi.boot_info,
+	&efi.hcdp,
+	&efi.uga,
+	&efi.uv_systab,
+	&efi.fw_vendor,
+	&efi.runtime,
+	&efi.config_table,
+	&efi.esrt,
+	&efi.properties_table,
+	&efi.mem_attr_table,
+};
+
 static bool disable_runtime;
 static int __init setup_noefi(char *arg)
 {
@@ -179,7 +198,7 @@ static umode_t efi_attr_is_visible(struct kobject *kobj,
 	return attr->mode;
 }
 
-static struct attribute_group efi_subsys_attr_group = {
+static const struct attribute_group efi_subsys_attr_group = {
 	.attrs = efi_subsys_attrs,
 	.is_visible = efi_attr_is_visible,
 };
@@ -522,6 +541,7 @@ int __init efi_config_parse_tables(void *config_tables, int count, int sz,
 			if (seed != NULL) {
 				add_device_randomness(seed->bits, seed->size);
 				early_memunmap(seed, sizeof(*seed) + size);
+				pr_notice("seeding entropy pool\n");
 			} else {
 				pr_err("Could not map UEFI random seed!\n");
 			}
@@ -791,19 +811,19 @@ char * __init efi_md_typeattr_format(char *buf, size_t size,
 }
 
 /*
+ * IA64 has a funky EFI memory map that doesn't work the same way as
+ * other architectures.
+ */
+#ifndef CONFIG_IA64
+/*
  * efi_mem_attributes - lookup memmap attributes for physical address
  * @phys_addr: the physical address to lookup
  *
  * Search in the EFI memory map for the region covering
  * @phys_addr. Returns the EFI memory attributes if the region
  * was found in the memory map, 0 otherwise.
- *
- * Despite being marked __weak, most architectures should *not*
- * override this function. It is __weak solely for the benefit
- * of ia64 which has a funky EFI memory map that doesn't work
- * the same way as other architectures.
  */
-u64 __weak efi_mem_attributes(unsigned long phys_addr)
+u64 efi_mem_attributes(unsigned long phys_addr)
 {
 	efi_memory_desc_t *md;
 
@@ -819,6 +839,31 @@ u64 __weak efi_mem_attributes(unsigned long phys_addr)
 	return 0;
 }
 
+/*
+ * efi_mem_type - lookup memmap type for physical address
+ * @phys_addr: the physical address to lookup
+ *
+ * Search in the EFI memory map for the region covering @phys_addr.
+ * Returns the EFI memory type if the region was found in the memory
+ * map, EFI_RESERVED_TYPE (zero) otherwise.
+ */
+int efi_mem_type(unsigned long phys_addr)
+{
+	const efi_memory_desc_t *md;
+
+	if (!efi_enabled(EFI_MEMMAP))
+		return -ENOTSUPP;
+
+	for_each_efi_memory_desc(md) {
+		if ((md->phys_addr <= phys_addr) &&
+		    (phys_addr < (md->phys_addr +
+				  (md->num_pages << EFI_PAGE_SHIFT))))
+			return md->type;
+	}
+	return -EINVAL;
+}
+#endif
+
 int efi_status_to_err(efi_status_t status)
 {
 	int err;
@@ -855,6 +900,20 @@ int efi_status_to_err(efi_status_t status)
 	return err;
 }
 
+bool efi_is_table_address(unsigned long phys_addr)
+{
+	unsigned int i;
+
+	if (phys_addr == EFI_INVALID_TABLE_ADDR)
+		return false;
+
+	for (i = 0; i < ARRAY_SIZE(efi_tables); i++)
+		if (*(efi_tables[i]) == phys_addr)
+			return true;
+
+	return false;
+}
+
 #ifdef CONFIG_KEXEC
 static int update_efi_random_seed(struct notifier_block *nb,
 				  unsigned long code, void *unused)
@@ -867,7 +926,7 @@ static int update_efi_random_seed(struct notifier_block *nb,
 
 	seed = memremap(efi.rng_seed, sizeof(*seed), MEMREMAP_WB);
 	if (seed != NULL) {
-		size = min(seed->size, 32U);
+		size = min(seed->size, EFI_RANDOM_SEED_SIZE);
 		memunmap(seed);
 	} else {
 		pr_err("Could not map UEFI random seed!\n");
diff --git a/drivers/firmware/efi/esrt.c b/drivers/firmware/efi/esrt.c
index 8554d7aec31c..bd7ed3c1148a 100644
--- a/drivers/firmware/efi/esrt.c
+++ b/drivers/firmware/efi/esrt.c
@@ -230,7 +230,7 @@ static umode_t esrt_attr_is_visible(struct kobject *kobj,
 	return attr->mode;
 }
 
-static struct attribute_group esrt_attr_group = {
+static const struct attribute_group esrt_attr_group = {
 	.attrs = esrt_attrs,
 	.is_visible = esrt_attr_is_visible,
 };
diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile
index 37e24f525162..dedf9bde44db 100644
--- a/drivers/firmware/efi/libstub/Makefile
+++ b/drivers/firmware/efi/libstub/Makefile
@@ -10,7 +10,7 @@ cflags-$(CONFIG_X86)		+= -m$(BITS) -D__KERNEL__ -O2 \
 				   -fPIC -fno-strict-aliasing -mno-red-zone \
 				   -mno-mmx -mno-sse
 
-cflags-$(CONFIG_ARM64)		:= $(subst -pg,,$(KBUILD_CFLAGS))
+cflags-$(CONFIG_ARM64)		:= $(subst -pg,,$(KBUILD_CFLAGS)) -fpie
 cflags-$(CONFIG_ARM)		:= $(subst -pg,,$(KBUILD_CFLAGS)) \
 				   -fno-builtin -fpic -mno-single-pic-base
 
@@ -30,6 +30,7 @@ OBJECT_FILES_NON_STANDARD	:= y
 KCOV_INSTRUMENT			:= n
 
 lib-y				:= efi-stub-helper.o gop.o secureboot.o
+lib-$(CONFIG_RESET_ATTACK_MITIGATION) += tpm.o
 
 # include the stub's generic dependencies from lib/ when building for ARM/arm64
 arm-deps := fdt_rw.c fdt_ro.c fdt_wip.c fdt.c fdt_empty_tree.c fdt_sw.c sort.c
diff --git a/drivers/firmware/efi/libstub/arm-stub.c b/drivers/firmware/efi/libstub/arm-stub.c
index 8181ac179d14..1cb2d1c070c3 100644
--- a/drivers/firmware/efi/libstub/arm-stub.c
+++ b/drivers/firmware/efi/libstub/arm-stub.c
@@ -192,6 +192,9 @@ unsigned long efi_entry(void *handle, efi_system_table_t *sys_table,
 		goto fail_free_cmdline;
 	}
 
+	/* Ask the firmware to clear memory on unclean shutdown */
+	efi_enable_reset_attack_mitigation(sys_table);
+
 	secure_boot = efi_get_secureboot(sys_table);
 
 	/*
diff --git a/drivers/firmware/efi/libstub/arm64-stub.c b/drivers/firmware/efi/libstub/arm64-stub.c
index b4c2589d7c91..b9bd827caa22 100644
--- a/drivers/firmware/efi/libstub/arm64-stub.c
+++ b/drivers/firmware/efi/libstub/arm64-stub.c
@@ -9,9 +9,18 @@
  * published by the Free Software Foundation.
  *
  */
+
+/*
+ * To prevent the compiler from emitting GOT-indirected (and thus absolute)
+ * references to the section markers, override their visibility as 'hidden'
+ */
+#pragma GCC visibility push(hidden)
+#include <asm/sections.h>
+#pragma GCC visibility pop
+
 #include <linux/efi.h>
 #include <asm/efi.h>
-#include <asm/sections.h>
+#include <asm/memory.h>
 #include <asm/sysreg.h>
 
 #include "efistub.h"
@@ -81,9 +90,10 @@ efi_status_t handle_kernel_image(efi_system_table_t *sys_table_arg,
 		/*
 		 * If CONFIG_DEBUG_ALIGN_RODATA is not set, produce a
 		 * displacement in the interval [0, MIN_KIMG_ALIGN) that
-		 * is a multiple of the minimal segment alignment (SZ_64K)
+		 * doesn't violate this kernel's de-facto alignment
+		 * constraints.
 		 */
-		u32 mask = (MIN_KIMG_ALIGN - 1) & ~(SZ_64K - 1);
+		u32 mask = (MIN_KIMG_ALIGN - 1) & ~(EFI_KIMG_ALIGN - 1);
 		u32 offset = !IS_ENABLED(CONFIG_DEBUG_ALIGN_RODATA) ?
 			     (phys_seed >> 32) & mask : TEXT_OFFSET;
 
diff --git a/drivers/firmware/efi/libstub/efi-stub-helper.c b/drivers/firmware/efi/libstub/efi-stub-helper.c
index b0184360efc6..50a9cab5a834 100644
--- a/drivers/firmware/efi/libstub/efi-stub-helper.c
+++ b/drivers/firmware/efi/libstub/efi-stub-helper.c
@@ -205,7 +205,7 @@ again:
 		unsigned long m = (unsigned long)map;
 		u64 start, end;
 
-		desc = (efi_memory_desc_t *)(m + (i * desc_size));
+		desc = efi_early_memdesc_ptr(m, desc_size, i);
 		if (desc->type != EFI_CONVENTIONAL_MEMORY)
 			continue;
 
@@ -298,7 +298,7 @@ efi_status_t efi_low_alloc(efi_system_table_t *sys_table_arg,
 		unsigned long m = (unsigned long)map;
 		u64 start, end;
 
-		desc = (efi_memory_desc_t *)(m + (i * desc_size));
+		desc = efi_early_memdesc_ptr(m, desc_size, i);
 
 		if (desc->type != EFI_CONVENTIONAL_MEMORY)
 			continue;
diff --git a/drivers/firmware/efi/libstub/random.c b/drivers/firmware/efi/libstub/random.c
index 7e72954d5860..e0e603a89aa9 100644
--- a/drivers/firmware/efi/libstub/random.c
+++ b/drivers/firmware/efi/libstub/random.c
@@ -145,8 +145,6 @@ efi_status_t efi_random_alloc(efi_system_table_t *sys_table_arg,
 	return status;
 }
 
-#define RANDOM_SEED_SIZE	32
-
 efi_status_t efi_random_get_seed(efi_system_table_t *sys_table_arg)
 {
 	efi_guid_t rng_proto = EFI_RNG_PROTOCOL_GUID;
@@ -162,25 +160,25 @@ efi_status_t efi_random_get_seed(efi_system_table_t *sys_table_arg)
 		return status;
 
 	status = efi_call_early(allocate_pool, EFI_RUNTIME_SERVICES_DATA,
-				sizeof(*seed) + RANDOM_SEED_SIZE,
+				sizeof(*seed) + EFI_RANDOM_SEED_SIZE,
 				(void **)&seed);
 	if (status != EFI_SUCCESS)
 		return status;
 
-	status = rng->get_rng(rng, &rng_algo_raw, RANDOM_SEED_SIZE,
+	status = rng->get_rng(rng, &rng_algo_raw, EFI_RANDOM_SEED_SIZE,
 			      seed->bits);
 	if (status == EFI_UNSUPPORTED)
 		/*
 		 * Use whatever algorithm we have available if the raw algorithm
 		 * is not implemented.
 		 */
-		status = rng->get_rng(rng, NULL, RANDOM_SEED_SIZE,
+		status = rng->get_rng(rng, NULL, EFI_RANDOM_SEED_SIZE,
 				      seed->bits);
 
 	if (status != EFI_SUCCESS)
 		goto err_freepool;
 
-	seed->size = RANDOM_SEED_SIZE;
+	seed->size = EFI_RANDOM_SEED_SIZE;
 	status = efi_call_early(install_configuration_table, &rng_table_guid,
 				seed);
 	if (status != EFI_SUCCESS)
diff --git a/drivers/firmware/efi/libstub/tpm.c b/drivers/firmware/efi/libstub/tpm.c
new file mode 100644
index 000000000000..6224cdbc9669
--- /dev/null
+++ b/drivers/firmware/efi/libstub/tpm.c
@@ -0,0 +1,58 @@
+/*
+ * TPM handling.
+ *
+ * Copyright (C) 2016 CoreOS, Inc
+ * Copyright (C) 2017 Google, Inc.
+ *     Matthew Garrett <mjg59@google.com>
+ *
+ * This file is part of the Linux kernel, and is made available under the
+ * terms of the GNU General Public License version 2.
+ */
+#include <linux/efi.h>
+#include <asm/efi.h>
+
+#include "efistub.h"
+
+static const efi_char16_t efi_MemoryOverWriteRequest_name[] = {
+	'M', 'e', 'm', 'o', 'r', 'y', 'O', 'v', 'e', 'r', 'w', 'r', 'i', 't',
+	'e', 'R', 'e', 'q', 'u', 'e', 's', 't', 'C', 'o', 'n', 't', 'r', 'o',
+	'l', 0
+};
+
+#define MEMORY_ONLY_RESET_CONTROL_GUID \
+	EFI_GUID(0xe20939be, 0x32d4, 0x41be, 0xa1, 0x50, 0x89, 0x7f, 0x85, 0xd4, 0x98, 0x29)
+
+#define get_efi_var(name, vendor, ...) \
+	efi_call_runtime(get_variable, \
+			 (efi_char16_t *)(name), (efi_guid_t *)(vendor), \
+			 __VA_ARGS__)
+
+#define set_efi_var(name, vendor, ...) \
+	efi_call_runtime(set_variable, \
+			 (efi_char16_t *)(name), (efi_guid_t *)(vendor), \
+			 __VA_ARGS__)
+
+/*
+ * Enable reboot attack mitigation. This requests that the firmware clear the
+ * RAM on next reboot before proceeding with boot, ensuring that any secrets
+ * are cleared. If userland has ensured that all secrets have been removed
+ * from RAM before reboot it can simply reset this variable.
+ */
+void efi_enable_reset_attack_mitigation(efi_system_table_t *sys_table_arg)
+{
+	u8 val = 1;
+	efi_guid_t var_guid = MEMORY_ONLY_RESET_CONTROL_GUID;
+	efi_status_t status;
+	unsigned long datasize = 0;
+
+	status = get_efi_var(efi_MemoryOverWriteRequest_name, &var_guid,
+			     NULL, &datasize, NULL);
+
+	if (status == EFI_NOT_FOUND)
+		return;
+
+	set_efi_var(efi_MemoryOverWriteRequest_name, &var_guid,
+		    EFI_VARIABLE_NON_VOLATILE |
+		    EFI_VARIABLE_BOOTSERVICE_ACCESS |
+		    EFI_VARIABLE_RUNTIME_ACCESS, sizeof(val), &val);
+}
diff --git a/drivers/firmware/efi/reboot.c b/drivers/firmware/efi/reboot.c
index 62ead9b9d871..22874544d301 100644
--- a/drivers/firmware/efi/reboot.c
+++ b/drivers/firmware/efi/reboot.c
@@ -5,6 +5,8 @@
 #include <linux/efi.h>
 #include <linux/reboot.h>
 
+static void (*orig_pm_power_off)(void);
+
 int efi_reboot_quirk_mode = -1;
 
 void efi_reboot(enum reboot_mode reboot_mode, const char *__unused)
@@ -51,6 +53,12 @@ bool __weak efi_poweroff_required(void)
 static void efi_power_off(void)
 {
 	efi.reset_system(EFI_RESET_SHUTDOWN, EFI_SUCCESS, 0, NULL);
+	/*
+	 * The above call should not return, if it does fall back to
+	 * the original power off method (typically ACPI poweroff).
+	 */
+	if (orig_pm_power_off)
+		orig_pm_power_off();
 }
 
 static int __init efi_shutdown_init(void)
@@ -58,8 +66,10 @@ static int __init efi_shutdown_init(void)
 	if (!efi_enabled(EFI_RUNTIME_SERVICES))
 		return -ENODEV;
 
-	if (efi_poweroff_required())
+	if (efi_poweroff_required()) {
+		orig_pm_power_off = pm_power_off;
 		pm_power_off = efi_power_off;
+	}
 
 	return 0;
 }
diff --git a/drivers/firmware/google/gsmi.c b/drivers/firmware/google/gsmi.c
index c46387160976..c8f169bf2e27 100644
--- a/drivers/firmware/google/gsmi.c
+++ b/drivers/firmware/google/gsmi.c
@@ -709,7 +709,7 @@ static u32 __init hash_oem_table_id(char s[8])
 	return local_hash_64(input, 32);
 }
 
-static struct dmi_system_id gsmi_dmi_table[] __initdata = {
+static const struct dmi_system_id gsmi_dmi_table[] __initconst = {
 	{
 		.ident = "Google Board",
 		.matches = {
diff --git a/drivers/firmware/google/memconsole-x86-legacy.c b/drivers/firmware/google/memconsole-x86-legacy.c
index 8c1bf6dbdaa6..19bcbd10855b 100644
--- a/drivers/firmware/google/memconsole-x86-legacy.c
+++ b/drivers/firmware/google/memconsole-x86-legacy.c
@@ -126,7 +126,7 @@ static bool memconsole_ebda_init(void)
 	return false;
 }
 
-static struct dmi_system_id memconsole_dmi_table[] __initdata = {
+static const struct dmi_system_id memconsole_dmi_table[] __initconst = {
 	{
 		.ident = "Google Board",
 		.matches = {
diff --git a/drivers/firmware/google/vpd.c b/drivers/firmware/google/vpd.c
index 78945729388e..35e553b3b190 100644
--- a/drivers/firmware/google/vpd.c
+++ b/drivers/firmware/google/vpd.c
@@ -202,7 +202,7 @@ static int vpd_section_init(const char *name, struct vpd_section *sec,
 	sec->raw_name = kasprintf(GFP_KERNEL, "%s_raw", name);
 	if (!sec->raw_name) {
 		err = -ENOMEM;
-		goto err_iounmap;
+		goto err_memunmap;
 	}
 
 	sysfs_bin_attr_init(&sec->bin_attr);
@@ -233,8 +233,8 @@ err_sysfs_remove:
 	sysfs_remove_bin_file(vpd_kobj, &sec->bin_attr);
 err_free_raw_name:
 	kfree(sec->raw_name);
-err_iounmap:
-	iounmap(sec->baseaddr);
+err_memunmap:
+	memunmap(sec->baseaddr);
 	return err;
 }
 
@@ -245,7 +245,7 @@ static int vpd_section_destroy(struct vpd_section *sec)
 		kobject_put(sec->kobj);
 		sysfs_remove_bin_file(vpd_kobj, &sec->bin_attr);
 		kfree(sec->raw_name);
-		iounmap(sec->baseaddr);
+		memunmap(sec->baseaddr);
 	}
 
 	return 0;
@@ -262,7 +262,7 @@ static int vpd_sections_init(phys_addr_t physaddr)
 		return -ENOMEM;
 
 	memcpy_fromio(&header, temp, sizeof(struct vpd_cbmem));
-	iounmap(temp);
+	memunmap(temp);
 
 	if (header.magic != VPD_CBMEM_MAGIC)
 		return -ENODEV;
diff --git a/drivers/firmware/pcdp.c b/drivers/firmware/pcdp.c
index 75273a251603..e83d6aec0c13 100644
--- a/drivers/firmware/pcdp.c
+++ b/drivers/firmware/pcdp.c
@@ -95,7 +95,7 @@ efi_setup_pcdp_console(char *cmdline)
 	if (efi.hcdp == EFI_INVALID_TABLE_ADDR)
 		return -ENODEV;
 
-	pcdp = early_ioremap(efi.hcdp, 4096);
+	pcdp = early_memremap(efi.hcdp, 4096);
 	printk(KERN_INFO "PCDP: v%d at 0x%lx\n", pcdp->rev, efi.hcdp);
 
 	if (strstr(cmdline, "console=hcdp")) {
@@ -131,6 +131,6 @@ efi_setup_pcdp_console(char *cmdline)
 	}
 
 out:
-	early_iounmap(pcdp, 4096);
+	early_memunmap(pcdp, 4096);
 	return rc;
 }
diff --git a/drivers/firmware/psci.c b/drivers/firmware/psci.c
index 493a56a4cfc4..d687ca3d5049 100644
--- a/drivers/firmware/psci.c
+++ b/drivers/firmware/psci.c
@@ -280,8 +280,8 @@ static int psci_dt_cpu_init_idle(struct device_node *cpu_node, int cpu)
 					   "arm,psci-suspend-param",
 					   &state);
 		if (ret) {
-			pr_warn(" * %s missing arm,psci-suspend-param property\n",
-				state_node->full_name);
+			pr_warn(" * %pOF missing arm,psci-suspend-param property\n",
+				state_node);
 			of_node_put(state_node);
 			goto free_mem;
 		}
diff --git a/drivers/firmware/tegra/bpmp.c b/drivers/firmware/tegra/bpmp.c
index b25179517cc5..73ca55b7b7ec 100644
--- a/drivers/firmware/tegra/bpmp.c
+++ b/drivers/firmware/tegra/bpmp.c
@@ -806,6 +806,8 @@ static int tegra_bpmp_probe(struct platform_device *pdev)
 
 	dev_info(&pdev->dev, "firmware: %s\n", tag);
 
+	platform_set_drvdata(pdev, bpmp);
+
 	err = of_platform_default_populate(pdev->dev.of_node, NULL, &pdev->dev);
 	if (err < 0)
 		goto free_mrq;
@@ -822,8 +824,6 @@ static int tegra_bpmp_probe(struct platform_device *pdev)
 	if (err < 0)
 		goto free_mrq;
 
-	platform_set_drvdata(pdev, bpmp);
-
 	return 0;
 
 free_mrq:
diff --git a/drivers/fmc/Makefile b/drivers/fmc/Makefile
index b9452919739f..e809322e1bac 100644
--- a/drivers/fmc/Makefile
+++ b/drivers/fmc/Makefile
@@ -6,6 +6,7 @@ fmc-y += fmc-match.o
 fmc-y += fmc-sdb.o
 fmc-y += fru-parse.o
 fmc-y += fmc-dump.o
+fmc-y += fmc-debug.o
 
 obj-$(CONFIG_FMC_FAKEDEV) += fmc-fakedev.o
 obj-$(CONFIG_FMC_TRIVIAL) += fmc-trivial.o
diff --git a/drivers/fmc/fmc-chardev.c b/drivers/fmc/fmc-chardev.c
index ace6ef24d15e..5ecf4090a610 100644
--- a/drivers/fmc/fmc-chardev.c
+++ b/drivers/fmc/fmc-chardev.c
@@ -129,8 +129,7 @@ static int fc_probe(struct fmc_device *fmc)
 
 	struct fc_instance *fc;
 
-	if (fmc->op->validate)
-		index = fmc->op->validate(fmc, &fc_drv);
+	index = fmc_validate(fmc, &fc_drv);
 	if (index < 0)
 		return -EINVAL; /* not our device: invalid */
 
diff --git a/drivers/fmc/fmc-core.c b/drivers/fmc/fmc-core.c
index 353fc546fb08..cec3b8db0d69 100644
--- a/drivers/fmc/fmc-core.c
+++ b/drivers/fmc/fmc-core.c
@@ -13,6 +13,9 @@
 #include <linux/init.h>
 #include <linux/device.h>
 #include <linux/fmc.h>
+#include <linux/fmc-sdb.h>
+
+#include "fmc-private.h"
 
 static int fmc_check_version(unsigned long version, const char *name)
 {
@@ -118,6 +121,61 @@ static struct bin_attribute fmc_eeprom_attr = {
 	.write = fmc_write_eeprom,
 };
 
+int fmc_irq_request(struct fmc_device *fmc, irq_handler_t h,
+		    char *name, int flags)
+{
+	if (fmc->op->irq_request)
+		return fmc->op->irq_request(fmc, h, name, flags);
+	return -EPERM;
+}
+EXPORT_SYMBOL(fmc_irq_request);
+
+void fmc_irq_free(struct fmc_device *fmc)
+{
+	if (fmc->op->irq_free)
+		fmc->op->irq_free(fmc);
+}
+EXPORT_SYMBOL(fmc_irq_free);
+
+void fmc_irq_ack(struct fmc_device *fmc)
+{
+	if (likely(fmc->op->irq_ack))
+		fmc->op->irq_ack(fmc);
+}
+EXPORT_SYMBOL(fmc_irq_ack);
+
+int fmc_validate(struct fmc_device *fmc, struct fmc_driver *drv)
+{
+	if (fmc->op->validate)
+		return fmc->op->validate(fmc, drv);
+	return -EPERM;
+}
+EXPORT_SYMBOL(fmc_validate);
+
+int fmc_gpio_config(struct fmc_device *fmc, struct fmc_gpio *gpio, int ngpio)
+{
+	if (fmc->op->gpio_config)
+		return fmc->op->gpio_config(fmc, gpio, ngpio);
+	return -EPERM;
+}
+EXPORT_SYMBOL(fmc_gpio_config);
+
+int fmc_read_ee(struct fmc_device *fmc, int pos, void *d, int l)
+{
+	if (fmc->op->read_ee)
+		return fmc->op->read_ee(fmc, pos, d, l);
+	return -EPERM;
+}
+EXPORT_SYMBOL(fmc_read_ee);
+
+int fmc_write_ee(struct fmc_device *fmc, int pos, const void *d, int l)
+{
+	if (fmc->op->write_ee)
+		return fmc->op->write_ee(fmc, pos, d, l);
+	return -EPERM;
+}
+EXPORT_SYMBOL(fmc_write_ee);
+
 /*
  * Functions for client modules follow
  */
@@ -141,7 +199,8 @@ EXPORT_SYMBOL(fmc_driver_unregister);
  * When a device set is registered, all eeproms must be read
  * and all FRUs must be parsed
  */
-int fmc_device_register_n(struct fmc_device **devs, int n)
+int fmc_device_register_n_gw(struct fmc_device **devs, int n,
+			  struct fmc_gateware *gw)
 {
 	struct fmc_device *fmc, **devarray;
 	uint32_t device_id;
@@ -221,6 +280,21 @@ int fmc_device_register_n(struct fmc_device **devs, int n)
 		else
 			dev_set_name(&fmc->dev, "%s-%04x", fmc->mezzanine_name,
 				     device_id);
+
+		if (gw) {
+			/*
+			 * The carrier already know the bitstream to load
+			 * for this set of FMC mezzanines.
+			 */
+			ret = fmc->op->reprogram_raw(fmc, NULL,
+						     gw->bitstream, gw->len);
+			if (ret) {
+				dev_warn(fmc->hwdev,
+					 "Invalid gateware for FMC mezzanine\n");
+				goto out;
+			}
+		}
+
 		ret = device_add(&fmc->dev);
 		if (ret < 0) {
 			dev_err(fmc->hwdev, "Slot %i: Failed in registering "
@@ -234,18 +308,16 @@ int fmc_device_register_n(struct fmc_device **devs, int n)
 		}
 		/* This device went well, give information to the user */
 		fmc_dump_eeprom(fmc);
-		fmc_dump_sdb(fmc);
+		fmc_debug_init(fmc);
 	}
 	return 0;
 
 out1:
 	device_del(&fmc->dev);
 out:
-	fmc_free_id_info(fmc);
-	put_device(&fmc->dev);
-
 	kfree(devarray);
 	for (i--; i >= 0; i--) {
+		fmc_debug_exit(devs[i]);
 		sysfs_remove_bin_file(&devs[i]->dev.kobj, &fmc_eeprom_attr);
 		device_del(&devs[i]->dev);
 		fmc_free_id_info(devs[i]);
@@ -254,8 +326,20 @@ out:
 	return ret;
 
 }
+EXPORT_SYMBOL(fmc_device_register_n_gw);
+
+int fmc_device_register_n(struct fmc_device **devs, int n)
+{
+	return fmc_device_register_n_gw(devs, n, NULL);
+}
 EXPORT_SYMBOL(fmc_device_register_n);
 
+int fmc_device_register_gw(struct fmc_device *fmc, struct fmc_gateware *gw)
+{
+	return fmc_device_register_n_gw(&fmc, 1, gw);
+}
+EXPORT_SYMBOL(fmc_device_register_gw);
+
 int fmc_device_register(struct fmc_device *fmc)
 {
 	return fmc_device_register_n(&fmc, 1);
@@ -273,6 +357,7 @@ void fmc_device_unregister_n(struct fmc_device **devs, int n)
 	kfree(devs[0]->devarray);
 
 	for (i = 0; i < n; i++) {
+		fmc_debug_exit(devs[i]);
 		sysfs_remove_bin_file(&devs[i]->dev.kobj, &fmc_eeprom_attr);
 		device_del(&devs[i]->dev);
 		fmc_free_id_info(devs[i]);
diff --git a/drivers/fmc/fmc-debug.c b/drivers/fmc/fmc-debug.c
new file mode 100644
index 000000000000..32930722770c
--- /dev/null
+++ b/drivers/fmc/fmc-debug.c
@@ -0,0 +1,173 @@
+/*
+ * Copyright (C) 2015 CERN (www.cern.ch)
+ * Author: Federico Vaga <federico.vaga@cern.ch>
+ *
+ * Released according to the GNU GPL, version 2 or any later version.
+ */
+
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/init.h>
+#include <linux/fs.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#include <asm/byteorder.h>
+
+#include <linux/fmc.h>
+#include <linux/sdb.h>
+#include <linux/fmc-sdb.h>
+
+#define FMC_DBG_SDB_DUMP "dump_sdb"
+
+static char *__strip_trailing_space(char *buf, char *str, int len)
+{
+	int i = len - 1;
+
+	memcpy(buf, str, len);
+	buf[len] = '\0';
+	while (i >= 0 && buf[i] == ' ')
+		buf[i--] = '\0';
+	return buf;
+}
+
+#define __sdb_string(buf, field) ({			\
+	BUILD_BUG_ON(sizeof(buf) < sizeof(field));	\
+	__strip_trailing_space(buf, (void *)(field), sizeof(field));	\
+		})
+
+/**
+ * We do not check seq_printf() errors because we want to see things in any case
+ */
+static void fmc_sdb_dump_recursive(struct fmc_device *fmc, struct seq_file *s,
+				   const struct sdb_array *arr)
+{
+	unsigned long base = arr->baseaddr;
+	int i, j, n = arr->len, level = arr->level;
+	char tmp[64];
+
+	for (i = 0; i < n; i++) {
+		union  sdb_record *r;
+		struct sdb_product *p;
+		struct sdb_component *c;
+
+		r = &arr->record[i];
+		c = &r->dev.sdb_component;
+		p = &c->product;
+
+		for (j = 0; j < level; j++)
+			seq_printf(s, "   ");
+		switch (r->empty.record_type) {
+		case sdb_type_interconnect:
+			seq_printf(s, "%08llx:%08x %.19s\n",
+				   __be64_to_cpu(p->vendor_id),
+				   __be32_to_cpu(p->device_id),
+				   p->name);
+			break;
+		case sdb_type_device:
+			seq_printf(s, "%08llx:%08x %.19s (%08llx-%08llx)\n",
+				   __be64_to_cpu(p->vendor_id),
+				   __be32_to_cpu(p->device_id),
+				   p->name,
+				   __be64_to_cpu(c->addr_first) + base,
+				   __be64_to_cpu(c->addr_last) + base);
+			break;
+		case sdb_type_bridge:
+			seq_printf(s, "%08llx:%08x %.19s (bridge: %08llx)\n",
+				   __be64_to_cpu(p->vendor_id),
+				   __be32_to_cpu(p->device_id),
+				   p->name,
+				   __be64_to_cpu(c->addr_first) + base);
+			if (IS_ERR(arr->subtree[i])) {
+				seq_printf(s, "SDB: (bridge error %li)\n",
+					 PTR_ERR(arr->subtree[i]));
+				break;
+			}
+			fmc_sdb_dump_recursive(fmc, s, arr->subtree[i]);
+			break;
+		case sdb_type_integration:
+			seq_printf(s, "integration\n");
+			break;
+		case sdb_type_repo_url:
+			seq_printf(s, "Synthesis repository: %s\n",
+					  __sdb_string(tmp, r->repo_url.repo_url));
+			break;
+		case sdb_type_synthesis:
+			seq_printf(s, "Bitstream '%s' ",
+					  __sdb_string(tmp, r->synthesis.syn_name));
+			seq_printf(s, "synthesized %08x by %s ",
+					  __be32_to_cpu(r->synthesis.date),
+					  __sdb_string(tmp, r->synthesis.user_name));
+			seq_printf(s, "(%s version %x), ",
+					  __sdb_string(tmp, r->synthesis.tool_name),
+					  __be32_to_cpu(r->synthesis.tool_version));
+			seq_printf(s, "commit %pm\n",
+					  r->synthesis.commit_id);
+			break;
+		case sdb_type_empty:
+			seq_printf(s, "empty\n");
+			break;
+		default:
+			seq_printf(s, "UNKNOWN TYPE 0x%02x\n",
+				   r->empty.record_type);
+			break;
+		}
+	}
+}
+
+static int fmc_sdb_dump(struct seq_file *s, void *offset)
+{
+	struct fmc_device *fmc = s->private;
+
+	if (!fmc->sdb) {
+		seq_printf(s, "no SDB information\n");
+		return 0;
+	}
+
+	seq_printf(s, "FMC: %s (%s), slot %i, device %s\n", dev_name(fmc->hwdev),
+	fmc->carrier_name, fmc->slot_id, dev_name(&fmc->dev));
+	/* Dump SDB information */
+	fmc_sdb_dump_recursive(fmc, s, fmc->sdb);
+
+	return 0;
+}
+
+
+static int fmc_sdb_dump_open(struct inode *inode, struct file *file)
+{
+	struct fmc_device *fmc = inode->i_private;
+
+	return single_open(file, fmc_sdb_dump, fmc);
+}
+
+
+const struct file_operations fmc_dbgfs_sdb_dump = {
+	.owner = THIS_MODULE,
+	.open  = fmc_sdb_dump_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+int fmc_debug_init(struct fmc_device *fmc)
+{
+	fmc->dbg_dir = debugfs_create_dir(dev_name(&fmc->dev), NULL);
+	if (IS_ERR_OR_NULL(fmc->dbg_dir)) {
+		pr_err("FMC: Cannot create debugfs\n");
+		return PTR_ERR(fmc->dbg_dir);
+	}
+
+	fmc->dbg_sdb_dump = debugfs_create_file(FMC_DBG_SDB_DUMP, 0444,
+						fmc->dbg_dir, fmc,
+						&fmc_dbgfs_sdb_dump);
+	if (IS_ERR_OR_NULL(fmc->dbg_sdb_dump))
+		pr_err("FMC: Cannot create debugfs file %s\n",
+		       FMC_DBG_SDB_DUMP);
+
+	return 0;
+}
+
+void fmc_debug_exit(struct fmc_device *fmc)
+{
+	if (fmc->dbg_dir)
+		debugfs_remove_recursive(fmc->dbg_dir);
+}
diff --git a/drivers/fmc/fmc-dump.c b/drivers/fmc/fmc-dump.c
index c91afd6388f6..cd1df475b254 100644
--- a/drivers/fmc/fmc-dump.c
+++ b/drivers/fmc/fmc-dump.c
@@ -15,8 +15,6 @@
 
 static int fmc_must_dump_eeprom;
 module_param_named(dump_eeprom, fmc_must_dump_eeprom, int, 0644);
-static int fmc_must_dump_sdb;
-module_param_named(dump_sdb, fmc_must_dump_sdb, int, 0644);
 
 #define LINELEN 16
 
@@ -59,42 +57,3 @@ void fmc_dump_eeprom(const struct fmc_device *fmc)
 	for (i = 0; i < fmc->eeprom_len; i += LINELEN, line += LINELEN)
 		prev = dump_line(i, line, prev);
 }
-
-void fmc_dump_sdb(const struct fmc_device *fmc)
-{
-	const uint8_t *line, *prev;
-	int i, len;
-
-	if (!fmc->sdb)
-		return;
-	if (!fmc_must_dump_sdb)
-		return;
-
-	/* If the argument is not-zero, do simple dump (== show) */
-	if (fmc_must_dump_sdb > 0)
-		fmc_show_sdb_tree(fmc);
-
-	if (fmc_must_dump_sdb == 1)
-		return;
-
-	/* If bigger than 1, dump it seriously, to help debugging */
-
-	/*
-	 * Here we should really use libsdbfs (which is designed to
-	 * work in kernel space as well) , but it doesn't support
-	 * directories yet, and it requires better intergration (it
-	 * should be used instead of fmc-specific code).
-	 *
-	 * So, lazily, just dump the top-level array
-	 */
-	pr_info("FMC: %s (%s), slot %i, device %s\n", dev_name(fmc->hwdev),
-		fmc->carrier_name, fmc->slot_id, dev_name(&fmc->dev));
-	pr_info("FMC: poor dump of sdb first level:\n");
-
-	len = fmc->sdb->len * sizeof(union sdb_record);
-	line = (void *)fmc->sdb->record;
-	prev = NULL;
-	for (i = 0; i < len; i += LINELEN, line += LINELEN)
-		prev = dump_line(i, line, prev);
-	return;
-}
diff --git a/drivers/fmc/fmc-match.c b/drivers/fmc/fmc-match.c
index 104a5efc2207..a0956d1f7550 100644
--- a/drivers/fmc/fmc-match.c
+++ b/drivers/fmc/fmc-match.c
@@ -63,7 +63,7 @@ int fmc_fill_id_info(struct fmc_device *fmc)
 		if (!fmc->eeprom)
 			return -ENOMEM;
 		allocated = 1;
-		ret = fmc->op->read_ee(fmc, 0, fmc->eeprom, fmc->eeprom_len);
+		ret = fmc_read_ee(fmc, 0, fmc->eeprom, fmc->eeprom_len);
 		if (ret < 0)
 			goto out;
 	}
diff --git a/drivers/fmc/fmc-private.h b/drivers/fmc/fmc-private.h
new file mode 100644
index 000000000000..1e5136643bdc
--- /dev/null
+++ b/drivers/fmc/fmc-private.h
@@ -0,0 +1,9 @@
+/*
+ * Copyright (C) 2015 CERN (www.cern.ch)
+ * Author: Federico Vaga <federico.vaga@cern.ch>
+ *
+ * Released according to the GNU GPL, version 2 or any later version.
+ */
+
+extern int fmc_debug_init(struct fmc_device *fmc);
+extern void fmc_debug_exit(struct fmc_device *fmc);
diff --git a/drivers/fmc/fmc-sdb.c b/drivers/fmc/fmc-sdb.c
index 4603fdb74465..ffdc1762b580 100644
--- a/drivers/fmc/fmc-sdb.c
+++ b/drivers/fmc/fmc-sdb.c
@@ -127,12 +127,12 @@ int fmc_free_sdb_tree(struct fmc_device *fmc)
 EXPORT_SYMBOL(fmc_free_sdb_tree);
 
 /* This helper calls reprogram and inizialized sdb as well */
-int fmc_reprogram(struct fmc_device *fmc, struct fmc_driver *d, char *gw,
-			 int sdb_entry)
+int fmc_reprogram_raw(struct fmc_device *fmc, struct fmc_driver *d,
+		      void *gw, unsigned long len, int sdb_entry)
 {
 	int ret;
 
-	ret = fmc->op->reprogram(fmc, d, gw);
+	ret = fmc->op->reprogram_raw(fmc, d, gw, len);
 	if (ret < 0)
 		return ret;
 	if (sdb_entry < 0)
@@ -145,108 +145,39 @@ int fmc_reprogram(struct fmc_device *fmc, struct fmc_driver *d, char *gw,
 			sdb_entry);
 		return -ENODEV;
 	}
-	fmc_dump_sdb(fmc);
-	return 0;
-}
-EXPORT_SYMBOL(fmc_reprogram);
-
-static char *__strip_trailing_space(char *buf, char *str, int len)
-{
-	int i = len - 1;
 
-	memcpy(buf, str, len);
-	while(i >= 0 && buf[i] == ' ')
-		buf[i--] = '\0';
-	return buf;
+	return 0;
 }
+EXPORT_SYMBOL(fmc_reprogram_raw);
 
-#define __sdb_string(buf, field) ({			\
-	BUILD_BUG_ON(sizeof(buf) < sizeof(field));	\
-	__strip_trailing_space(buf, (void *)(field), sizeof(field));	\
-		})
-
-static void __fmc_show_sdb_tree(const struct fmc_device *fmc,
-				const struct sdb_array *arr)
+/* This helper calls reprogram and inizialized sdb as well */
+int fmc_reprogram(struct fmc_device *fmc, struct fmc_driver *d, char *gw,
+			 int sdb_entry)
 {
-	unsigned long base = arr->baseaddr;
-	int i, j, n = arr->len, level = arr->level;
-	char buf[64];
-
-	for (i = 0; i < n; i++) {
-		union  sdb_record *r;
-		struct sdb_product *p;
-		struct sdb_component *c;
-		r = &arr->record[i];
-		c = &r->dev.sdb_component;
-		p = &c->product;
+	int ret;
 
-		dev_info(&fmc->dev, "SDB: ");
+	ret = fmc->op->reprogram(fmc, d, gw);
+	if (ret < 0)
+		return ret;
+	if (sdb_entry < 0)
+		return ret;
 
-		for (j = 0; j < level; j++)
-			printk(KERN_CONT "   ");
-		switch (r->empty.record_type) {
-		case sdb_type_interconnect:
-			printk(KERN_CONT "%08llx:%08x %.19s\n",
-			       __be64_to_cpu(p->vendor_id),
-			       __be32_to_cpu(p->device_id),
-			       p->name);
-			break;
-		case sdb_type_device:
-			printk(KERN_CONT "%08llx:%08x %.19s (%08llx-%08llx)\n",
-			       __be64_to_cpu(p->vendor_id),
-			       __be32_to_cpu(p->device_id),
-			       p->name,
-			       __be64_to_cpu(c->addr_first) + base,
-			       __be64_to_cpu(c->addr_last) + base);
-			break;
-		case sdb_type_bridge:
-			printk(KERN_CONT "%08llx:%08x %.19s (bridge: %08llx)\n",
-			       __be64_to_cpu(p->vendor_id),
-			       __be32_to_cpu(p->device_id),
-			       p->name,
-			       __be64_to_cpu(c->addr_first) + base);
-			if (IS_ERR(arr->subtree[i])) {
-				dev_info(&fmc->dev, "SDB: (bridge error %li)\n",
-					 PTR_ERR(arr->subtree[i]));
-				break;
-			}
-			__fmc_show_sdb_tree(fmc, arr->subtree[i]);
-			break;
-		case sdb_type_integration:
-			printk(KERN_CONT "integration\n");
-			break;
-		case sdb_type_repo_url:
-			printk(KERN_CONT "Synthesis repository: %s\n",
-			       __sdb_string(buf, r->repo_url.repo_url));
-			break;
-		case sdb_type_synthesis:
-			printk(KERN_CONT "Bitstream '%s' ",
-			       __sdb_string(buf, r->synthesis.syn_name));
-			printk(KERN_CONT "synthesized %08x by %s ",
-			       __be32_to_cpu(r->synthesis.date),
-			       __sdb_string(buf, r->synthesis.user_name));
-			printk(KERN_CONT "(%s version %x), ",
-			       __sdb_string(buf, r->synthesis.tool_name),
-			       __be32_to_cpu(r->synthesis.tool_version));
-			printk(KERN_CONT "commit %pm\n",
-			       r->synthesis.commit_id);
-			break;
-		case sdb_type_empty:
-			printk(KERN_CONT "empty\n");
-			break;
-		default:
-			printk(KERN_CONT "UNKNOWN TYPE 0x%02x\n",
-			       r->empty.record_type);
-			break;
-		}
+	/* We are required to find SDB at a given offset */
+	ret = fmc_scan_sdb_tree(fmc, sdb_entry);
+	if (ret < 0) {
+		dev_err(&fmc->dev, "Can't find SDB at address 0x%x\n",
+			sdb_entry);
+		return -ENODEV;
 	}
+
+	return 0;
 }
+EXPORT_SYMBOL(fmc_reprogram);
 
 void fmc_show_sdb_tree(const struct fmc_device *fmc)
 {
-	if (!fmc->sdb)
-		return;
-	__fmc_show_sdb_tree(fmc, fmc->sdb);
+	pr_err("%s: not supported anymore, use debugfs to dump SDB\n",
+		__func__);
 }
 EXPORT_SYMBOL(fmc_show_sdb_tree);
 
diff --git a/drivers/fmc/fmc-trivial.c b/drivers/fmc/fmc-trivial.c
index 6c590f54c79d..8defdee3e3a3 100644
--- a/drivers/fmc/fmc-trivial.c
+++ b/drivers/fmc/fmc-trivial.c
@@ -24,7 +24,7 @@ static irqreturn_t t_handler(int irq, void *dev_id)
 {
 	struct fmc_device *fmc = dev_id;
 
-	fmc->op->irq_ack(fmc);
+	fmc_irq_ack(fmc);
 	dev_info(&fmc->dev, "received irq %i\n", irq);
 	return IRQ_HANDLED;
 }
@@ -46,25 +46,21 @@ static int t_probe(struct fmc_device *fmc)
 	int ret;
 	int index = 0;
 
-	if (fmc->op->validate)
-		index = fmc->op->validate(fmc, &t_drv);
+	index = fmc_validate(fmc, &t_drv);
 	if (index < 0)
 		return -EINVAL; /* not our device: invalid */
 
-	ret = fmc->op->irq_request(fmc, t_handler, "fmc-trivial", IRQF_SHARED);
+	ret = fmc_irq_request(fmc, t_handler, "fmc-trivial", IRQF_SHARED);
 	if (ret < 0)
 		return ret;
 	/* ignore error code of call below, we really don't care */
-	fmc->op->gpio_config(fmc, t_gpio, ARRAY_SIZE(t_gpio));
+	fmc_gpio_config(fmc, t_gpio, ARRAY_SIZE(t_gpio));
 
-	/* Reprogram, if asked to. ESRCH == no filename specified */
-	ret = -ESRCH;
-	if (fmc->op->reprogram)
-		ret = fmc->op->reprogram(fmc, &t_drv, "");
-	if (ret == -ESRCH)
+	ret = fmc_reprogram(fmc, &t_drv, "", 0);
+	if (ret == -EPERM) /* programming not supported */
 		ret = 0;
 	if (ret < 0)
-		fmc->op->irq_free(fmc);
+		fmc_irq_free(fmc);
 
 	/* FIXME: reprogram LM32 too */
 	return ret;
@@ -72,7 +68,7 @@ static int t_probe(struct fmc_device *fmc)
 
 static int t_remove(struct fmc_device *fmc)
 {
-	fmc->op->irq_free(fmc);
+	fmc_irq_free(fmc);
 	return 0;
 }
 
diff --git a/drivers/fmc/fmc-write-eeprom.c b/drivers/fmc/fmc-write-eeprom.c
index 9bb2cbd5c9f2..3eb81bb1f1fc 100644
--- a/drivers/fmc/fmc-write-eeprom.c
+++ b/drivers/fmc/fmc-write-eeprom.c
@@ -50,7 +50,7 @@ static int fwe_run_tlv(struct fmc_device *fmc, const struct firmware *fw,
 		if (write) {
 			dev_info(&fmc->dev, "write %i bytes at 0x%04x\n",
 				 thislen, thisaddr);
-			err = fmc->op->write_ee(fmc, thisaddr, p + 5, thislen);
+			err = fmc_write_ee(fmc, thisaddr, p + 5, thislen);
 		}
 		if (err < 0) {
 			dev_err(&fmc->dev, "write failure @0x%04x\n",
@@ -70,7 +70,7 @@ static int fwe_run_bin(struct fmc_device *fmc, const struct firmware *fw)
 	int ret;
 
 	dev_info(&fmc->dev, "programming %zi bytes\n", fw->size);
-	ret = fmc->op->write_ee(fmc, 0, (void *)fw->data, fw->size);
+	ret = fmc_write_ee(fmc, 0, (void *)fw->data, fw->size);
 	if (ret < 0) {
 		dev_info(&fmc->dev, "write_eeprom: error %i\n", ret);
 		return ret;
@@ -115,8 +115,8 @@ static int fwe_probe(struct fmc_device *fmc)
 			KBUILD_MODNAME);
 		return -ENODEV;
 	}
-	if (fmc->op->validate)
-		index = fmc->op->validate(fmc, &fwe_drv);
+
+	index = fmc_validate(fmc, &fwe_drv);
 	if (index < 0) {
 		pr_err("%s: refusing device \"%s\"\n", KBUILD_MODNAME,
 		       dev_name(dev));
diff --git a/drivers/fmc/fru-parse.c b/drivers/fmc/fru-parse.c
index cb46263c5da2..eb21480d399f 100644
--- a/drivers/fmc/fru-parse.c
+++ b/drivers/fmc/fru-parse.c
@@ -31,12 +31,11 @@ static char *__fru_alloc_get_tl(struct fru_common_header *header, int nr)
 {
 	struct fru_type_length *tl;
 	char *res;
-	int len;
 
 	tl = __fru_get_board_tl(header, nr);
 	if (!tl)
 		return NULL;
-	len = fru_strlen(tl);
+
 	res = fru_alloc(fru_strlen(tl) + 1);
 	if (!res)
 		return NULL;
diff --git a/drivers/fpga/Kconfig b/drivers/fpga/Kconfig
index 161ba9dccede..ad5448f718b3 100644
--- a/drivers/fpga/Kconfig
+++ b/drivers/fpga/Kconfig
@@ -2,9 +2,7 @@
 # FPGA framework configuration
 #
 
-menu "FPGA Configuration Support"
-
-config FPGA
+menuconfig FPGA
 	tristate "FPGA Configuration Framework"
 	help
 	  Say Y here if you want support for configuring FPGAs from the
@@ -26,6 +24,20 @@ config FPGA_MGR_ICE40_SPI
 	help
 	  FPGA manager driver support for Lattice iCE40 FPGAs over SPI.
 
+config FPGA_MGR_ALTERA_CVP
+	tristate "Altera Arria-V/Cyclone-V/Stratix-V CvP FPGA Manager"
+	depends on PCI
+	help
+	  FPGA manager driver support for Arria-V, Cyclone-V, Stratix-V
+	  and Arria 10 Altera FPGAs using the CvP interface over PCIe.
+
+config FPGA_MGR_ALTERA_PS_SPI
+	tristate "Altera FPGA Passive Serial over SPI"
+	depends on SPI
+	help
+	  FPGA manager driver support for Altera Arria/Cyclone/Stratix
+	  using the passive serial interface over SPI.
+
 config FPGA_MGR_SOCFPGA
 	tristate "Altera SOCFPGA FPGA Manager"
 	depends on ARCH_SOCFPGA || COMPILE_TEST
@@ -106,5 +118,3 @@ config XILINX_PR_DECOUPLER
 	  being reprogrammed during partial reconfig.
 
 endif # FPGA
-
-endmenu
diff --git a/drivers/fpga/Makefile b/drivers/fpga/Makefile
index 2a4f0218145c..e09895f0525b 100644
--- a/drivers/fpga/Makefile
+++ b/drivers/fpga/Makefile
@@ -6,6 +6,8 @@
 obj-$(CONFIG_FPGA)			+= fpga-mgr.o
 
 # FPGA Manager Drivers
+obj-$(CONFIG_FPGA_MGR_ALTERA_CVP)	+= altera-cvp.o
+obj-$(CONFIG_FPGA_MGR_ALTERA_PS_SPI)	+= altera-ps-spi.o
 obj-$(CONFIG_FPGA_MGR_ICE40_SPI)	+= ice40-spi.o
 obj-$(CONFIG_FPGA_MGR_SOCFPGA)		+= socfpga.o
 obj-$(CONFIG_FPGA_MGR_SOCFPGA_A10)	+= socfpga-a10.o
diff --git a/drivers/fpga/altera-cvp.c b/drivers/fpga/altera-cvp.c
new file mode 100644
index 000000000000..08629ee69d11
--- /dev/null
+++ b/drivers/fpga/altera-cvp.c
@@ -0,0 +1,500 @@
+/*
+ * FPGA Manager Driver for Altera Arria/Cyclone/Stratix CvP
+ *
+ * Copyright (C) 2017 DENX Software Engineering
+ *
+ * Anatolij Gustschin <agust@denx.de>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * Manage Altera FPGA firmware using PCIe CvP.
+ * Firmware must be in binary "rbf" format.
+ */
+
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/fpga/fpga-mgr.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/sizes.h>
+
+#define CVP_BAR		0	/* BAR used for data transfer in memory mode */
+#define CVP_DUMMY_WR	244	/* dummy writes to clear CvP state machine */
+#define TIMEOUT_US	2000	/* CVP STATUS timeout for USERMODE polling */
+
+/* Vendor Specific Extended Capability Registers */
+#define VSE_PCIE_EXT_CAP_ID		0x200
+#define VSE_PCIE_EXT_CAP_ID_VAL		0x000b	/* 16bit */
+
+#define VSE_CVP_STATUS			0x21c	/* 32bit */
+#define VSE_CVP_STATUS_CFG_RDY		BIT(18)	/* CVP_CONFIG_READY */
+#define VSE_CVP_STATUS_CFG_ERR		BIT(19)	/* CVP_CONFIG_ERROR */
+#define VSE_CVP_STATUS_CVP_EN		BIT(20)	/* ctrl block is enabling CVP */
+#define VSE_CVP_STATUS_USERMODE		BIT(21)	/* USERMODE */
+#define VSE_CVP_STATUS_CFG_DONE		BIT(23)	/* CVP_CONFIG_DONE */
+#define VSE_CVP_STATUS_PLD_CLK_IN_USE	BIT(24)	/* PLD_CLK_IN_USE */
+
+#define VSE_CVP_MODE_CTRL		0x220	/* 32bit */
+#define VSE_CVP_MODE_CTRL_CVP_MODE	BIT(0)	/* CVP (1) or normal mode (0) */
+#define VSE_CVP_MODE_CTRL_HIP_CLK_SEL	BIT(1) /* PMA (1) or fabric clock (0) */
+#define VSE_CVP_MODE_CTRL_NUMCLKS_OFF	8	/* NUMCLKS bits offset */
+#define VSE_CVP_MODE_CTRL_NUMCLKS_MASK	GENMASK(15, 8)
+
+#define VSE_CVP_DATA			0x228	/* 32bit */
+#define VSE_CVP_PROG_CTRL		0x22c	/* 32bit */
+#define VSE_CVP_PROG_CTRL_CONFIG	BIT(0)
+#define VSE_CVP_PROG_CTRL_START_XFER	BIT(1)
+
+#define VSE_UNCOR_ERR_STATUS		0x234	/* 32bit */
+#define VSE_UNCOR_ERR_CVP_CFG_ERR	BIT(5)	/* CVP_CONFIG_ERROR_LATCHED */
+
+#define DRV_NAME		"altera-cvp"
+#define ALTERA_CVP_MGR_NAME	"Altera CvP FPGA Manager"
+
+/* Optional CvP config error status check for debugging */
+static bool altera_cvp_chkcfg;
+
+struct altera_cvp_conf {
+	struct fpga_manager	*mgr;
+	struct pci_dev		*pci_dev;
+	void __iomem		*map;
+	void			(*write_data)(struct altera_cvp_conf *, u32);
+	char			mgr_name[64];
+	u8			numclks;
+};
+
+static enum fpga_mgr_states altera_cvp_state(struct fpga_manager *mgr)
+{
+	struct altera_cvp_conf *conf = mgr->priv;
+	u32 status;
+
+	pci_read_config_dword(conf->pci_dev, VSE_CVP_STATUS, &status);
+
+	if (status & VSE_CVP_STATUS_CFG_DONE)
+		return FPGA_MGR_STATE_OPERATING;
+
+	if (status & VSE_CVP_STATUS_CVP_EN)
+		return FPGA_MGR_STATE_POWER_UP;
+
+	return FPGA_MGR_STATE_UNKNOWN;
+}
+
+static void altera_cvp_write_data_iomem(struct altera_cvp_conf *conf, u32 val)
+{
+	writel(val, conf->map);
+}
+
+static void altera_cvp_write_data_config(struct altera_cvp_conf *conf, u32 val)
+{
+	pci_write_config_dword(conf->pci_dev, VSE_CVP_DATA, val);
+}
+
+/* switches between CvP clock and internal clock */
+static void altera_cvp_dummy_write(struct altera_cvp_conf *conf)
+{
+	unsigned int i;
+	u32 val;
+
+	/* set 1 CVP clock cycle for every CVP Data Register Write */
+	pci_read_config_dword(conf->pci_dev, VSE_CVP_MODE_CTRL, &val);
+	val &= ~VSE_CVP_MODE_CTRL_NUMCLKS_MASK;
+	val |= 1 << VSE_CVP_MODE_CTRL_NUMCLKS_OFF;
+	pci_write_config_dword(conf->pci_dev, VSE_CVP_MODE_CTRL, val);
+
+	for (i = 0; i < CVP_DUMMY_WR; i++)
+		conf->write_data(conf, 0); /* dummy data, could be any value */
+}
+
+static int altera_cvp_wait_status(struct altera_cvp_conf *conf, u32 status_mask,
+				  u32 status_val, int timeout_us)
+{
+	unsigned int retries;
+	u32 val;
+
+	retries = timeout_us / 10;
+	if (timeout_us % 10)
+		retries++;
+
+	do {
+		pci_read_config_dword(conf->pci_dev, VSE_CVP_STATUS, &val);
+		if ((val & status_mask) == status_val)
+			return 0;
+
+		/* use small usleep value to re-check and break early */
+		usleep_range(10, 11);
+	} while (--retries);
+
+	return -ETIMEDOUT;
+}
+
+static int altera_cvp_teardown(struct fpga_manager *mgr,
+			       struct fpga_image_info *info)
+{
+	struct altera_cvp_conf *conf = mgr->priv;
+	struct pci_dev *pdev = conf->pci_dev;
+	int ret;
+	u32 val;
+
+	/* STEP 12 - reset START_XFER bit */
+	pci_read_config_dword(pdev, VSE_CVP_PROG_CTRL, &val);
+	val &= ~VSE_CVP_PROG_CTRL_START_XFER;
+	pci_write_config_dword(pdev, VSE_CVP_PROG_CTRL, val);
+
+	/* STEP 13 - reset CVP_CONFIG bit */
+	val &= ~VSE_CVP_PROG_CTRL_CONFIG;
+	pci_write_config_dword(pdev, VSE_CVP_PROG_CTRL, val);
+
+	/*
+	 * STEP 14
+	 * - set CVP_NUMCLKS to 1 and then issue CVP_DUMMY_WR dummy
+	 *   writes to the HIP
+	 */
+	altera_cvp_dummy_write(conf); /* from CVP clock to internal clock */
+
+	/* STEP 15 - poll CVP_CONFIG_READY bit for 0 with 10us timeout */
+	ret = altera_cvp_wait_status(conf, VSE_CVP_STATUS_CFG_RDY, 0, 10);
+	if (ret)
+		dev_err(&mgr->dev, "CFG_RDY == 0 timeout\n");
+
+	return ret;
+}
+
+static int altera_cvp_write_init(struct fpga_manager *mgr,
+				 struct fpga_image_info *info,
+				 const char *buf, size_t count)
+{
+	struct altera_cvp_conf *conf = mgr->priv;
+	struct pci_dev *pdev = conf->pci_dev;
+	u32 iflags, val;
+	int ret;
+
+	iflags = info ? info->flags : 0;
+
+	if (iflags & FPGA_MGR_PARTIAL_RECONFIG) {
+		dev_err(&mgr->dev, "Partial reconfiguration not supported.\n");
+		return -EINVAL;
+	}
+
+	/* Determine allowed clock to data ratio */
+	if (iflags & FPGA_MGR_COMPRESSED_BITSTREAM)
+		conf->numclks = 8; /* ratio for all compressed images */
+	else if (iflags & FPGA_MGR_ENCRYPTED_BITSTREAM)
+		conf->numclks = 4; /* for uncompressed and encrypted images */
+	else
+		conf->numclks = 1; /* for uncompressed and unencrypted images */
+
+	/* STEP 1 - read CVP status and check CVP_EN flag */
+	pci_read_config_dword(pdev, VSE_CVP_STATUS, &val);
+	if (!(val & VSE_CVP_STATUS_CVP_EN)) {
+		dev_err(&mgr->dev, "CVP mode off: 0x%04x\n", val);
+		return -ENODEV;
+	}
+
+	if (val & VSE_CVP_STATUS_CFG_RDY) {
+		dev_warn(&mgr->dev, "CvP already started, teardown first\n");
+		ret = altera_cvp_teardown(mgr, info);
+		if (ret)
+			return ret;
+	}
+
+	/*
+	 * STEP 2
+	 * - set HIP_CLK_SEL and CVP_MODE (must be set in the order mentioned)
+	 */
+	/* switch from fabric to PMA clock */
+	pci_read_config_dword(pdev, VSE_CVP_MODE_CTRL, &val);
+	val |= VSE_CVP_MODE_CTRL_HIP_CLK_SEL;
+	pci_write_config_dword(pdev, VSE_CVP_MODE_CTRL, val);
+
+	/* set CVP mode */
+	pci_read_config_dword(pdev, VSE_CVP_MODE_CTRL, &val);
+	val |= VSE_CVP_MODE_CTRL_CVP_MODE;
+	pci_write_config_dword(pdev, VSE_CVP_MODE_CTRL, val);
+
+	/*
+	 * STEP 3
+	 * - set CVP_NUMCLKS to 1 and issue CVP_DUMMY_WR dummy writes to the HIP
+	 */
+	altera_cvp_dummy_write(conf);
+
+	/* STEP 4 - set CVP_CONFIG bit */
+	pci_read_config_dword(pdev, VSE_CVP_PROG_CTRL, &val);
+	/* request control block to begin transfer using CVP */
+	val |= VSE_CVP_PROG_CTRL_CONFIG;
+	pci_write_config_dword(pdev, VSE_CVP_PROG_CTRL, val);
+
+	/* STEP 5 - poll CVP_CONFIG READY for 1 with 10us timeout */
+	ret = altera_cvp_wait_status(conf, VSE_CVP_STATUS_CFG_RDY,
+				     VSE_CVP_STATUS_CFG_RDY, 10);
+	if (ret) {
+		dev_warn(&mgr->dev, "CFG_RDY == 1 timeout\n");
+		return ret;
+	}
+
+	/*
+	 * STEP 6
+	 * - set CVP_NUMCLKS to 1 and issue CVP_DUMMY_WR dummy writes to the HIP
+	 */
+	altera_cvp_dummy_write(conf);
+
+	/* STEP 7 - set START_XFER */
+	pci_read_config_dword(pdev, VSE_CVP_PROG_CTRL, &val);
+	val |= VSE_CVP_PROG_CTRL_START_XFER;
+	pci_write_config_dword(pdev, VSE_CVP_PROG_CTRL, val);
+
+	/* STEP 8 - start transfer (set CVP_NUMCLKS for bitstream) */
+	pci_read_config_dword(pdev, VSE_CVP_MODE_CTRL, &val);
+	val &= ~VSE_CVP_MODE_CTRL_NUMCLKS_MASK;
+	val |= conf->numclks << VSE_CVP_MODE_CTRL_NUMCLKS_OFF;
+	pci_write_config_dword(pdev, VSE_CVP_MODE_CTRL, val);
+
+	return 0;
+}
+
+static inline int altera_cvp_chk_error(struct fpga_manager *mgr, size_t bytes)
+{
+	struct altera_cvp_conf *conf = mgr->priv;
+	u32 val;
+
+	/* STEP 10 (optional) - check CVP_CONFIG_ERROR flag */
+	pci_read_config_dword(conf->pci_dev, VSE_CVP_STATUS, &val);
+	if (val & VSE_CVP_STATUS_CFG_ERR) {
+		dev_err(&mgr->dev, "CVP_CONFIG_ERROR after %zu bytes!\n",
+			bytes);
+		return -EPROTO;
+	}
+	return 0;
+}
+
+static int altera_cvp_write(struct fpga_manager *mgr, const char *buf,
+			    size_t count)
+{
+	struct altera_cvp_conf *conf = mgr->priv;
+	const u32 *data;
+	size_t done, remaining;
+	int status = 0;
+	u32 mask;
+
+	/* STEP 9 - write 32-bit data from RBF file to CVP data register */
+	data = (u32 *)buf;
+	remaining = count;
+	done = 0;
+
+	while (remaining >= 4) {
+		conf->write_data(conf, *data++);
+		done += 4;
+		remaining -= 4;
+
+		/*
+		 * STEP 10 (optional) and STEP 11
+		 * - check error flag
+		 * - loop until data transfer completed
+		 * Config images can be huge (more than 40 MiB), so
+		 * only check after a new 4k data block has been written.
+		 * This reduces the number of checks and speeds up the
+		 * configuration process.
+		 */
+		if (altera_cvp_chkcfg && !(done % SZ_4K)) {
+			status = altera_cvp_chk_error(mgr, done);
+			if (status < 0)
+				return status;
+		}
+	}
+
+	/* write up to 3 trailing bytes, if any */
+	mask = BIT(remaining * 8) - 1;
+	if (mask)
+		conf->write_data(conf, *data & mask);
+
+	if (altera_cvp_chkcfg)
+		status = altera_cvp_chk_error(mgr, count);
+
+	return status;
+}
+
+static int altera_cvp_write_complete(struct fpga_manager *mgr,
+				     struct fpga_image_info *info)
+{
+	struct altera_cvp_conf *conf = mgr->priv;
+	struct pci_dev *pdev = conf->pci_dev;
+	int ret;
+	u32 mask;
+	u32 val;
+
+	ret = altera_cvp_teardown(mgr, info);
+	if (ret)
+		return ret;
+
+	/* STEP 16 - check CVP_CONFIG_ERROR_LATCHED bit */
+	pci_read_config_dword(pdev, VSE_UNCOR_ERR_STATUS, &val);
+	if (val & VSE_UNCOR_ERR_CVP_CFG_ERR) {
+		dev_err(&mgr->dev, "detected CVP_CONFIG_ERROR_LATCHED!\n");
+		return -EPROTO;
+	}
+
+	/* STEP 17 - reset CVP_MODE and HIP_CLK_SEL bit */
+	pci_read_config_dword(pdev, VSE_CVP_MODE_CTRL, &val);
+	val &= ~VSE_CVP_MODE_CTRL_HIP_CLK_SEL;
+	val &= ~VSE_CVP_MODE_CTRL_CVP_MODE;
+	pci_write_config_dword(pdev, VSE_CVP_MODE_CTRL, val);
+
+	/* STEP 18 - poll PLD_CLK_IN_USE and USER_MODE bits */
+	mask = VSE_CVP_STATUS_PLD_CLK_IN_USE | VSE_CVP_STATUS_USERMODE;
+	ret = altera_cvp_wait_status(conf, mask, mask, TIMEOUT_US);
+	if (ret)
+		dev_err(&mgr->dev, "PLD_CLK_IN_USE|USERMODE timeout\n");
+
+	return ret;
+}
+
+static const struct fpga_manager_ops altera_cvp_ops = {
+	.state		= altera_cvp_state,
+	.write_init	= altera_cvp_write_init,
+	.write		= altera_cvp_write,
+	.write_complete	= altera_cvp_write_complete,
+};
+
+static ssize_t show_chkcfg(struct device_driver *dev, char *buf)
+{
+	return snprintf(buf, 3, "%d\n", altera_cvp_chkcfg);
+}
+
+static ssize_t store_chkcfg(struct device_driver *drv, const char *buf,
+			    size_t count)
+{
+	int ret;
+
+	ret = kstrtobool(buf, &altera_cvp_chkcfg);
+	if (ret)
+		return ret;
+
+	return count;
+}
+
+static DRIVER_ATTR(chkcfg, 0600, show_chkcfg, store_chkcfg);
+
+static int altera_cvp_probe(struct pci_dev *pdev,
+			    const struct pci_device_id *dev_id);
+static void altera_cvp_remove(struct pci_dev *pdev);
+
+#define PCI_VENDOR_ID_ALTERA	0x1172
+
+static struct pci_device_id altera_cvp_id_tbl[] = {
+	{ PCI_VDEVICE(ALTERA, PCI_ANY_ID) },
+	{ }
+};
+MODULE_DEVICE_TABLE(pci, altera_cvp_id_tbl);
+
+static struct pci_driver altera_cvp_driver = {
+	.name   = DRV_NAME,
+	.id_table = altera_cvp_id_tbl,
+	.probe  = altera_cvp_probe,
+	.remove = altera_cvp_remove,
+};
+
+static int altera_cvp_probe(struct pci_dev *pdev,
+			    const struct pci_device_id *dev_id)
+{
+	struct altera_cvp_conf *conf;
+	u16 cmd, val;
+	int ret;
+
+	/*
+	 * First check if this is the expected FPGA device. PCI config
+	 * space access works without enabling the PCI device, memory
+	 * space access is enabled further down.
+	 */
+	pci_read_config_word(pdev, VSE_PCIE_EXT_CAP_ID, &val);
+	if (val != VSE_PCIE_EXT_CAP_ID_VAL) {
+		dev_err(&pdev->dev, "Wrong EXT_CAP_ID value 0x%x\n", val);
+		return -ENODEV;
+	}
+
+	conf = devm_kzalloc(&pdev->dev, sizeof(*conf), GFP_KERNEL);
+	if (!conf)
+		return -ENOMEM;
+
+	/*
+	 * Enable memory BAR access. We cannot use pci_enable_device() here
+	 * because it will make the driver unusable with FPGA devices that
+	 * have additional big IOMEM resources (e.g. 4GiB BARs) on 32-bit
+	 * platform. Such BARs will not have an assigned address range and
+	 * pci_enable_device() will fail, complaining about not claimed BAR,
+	 * even if the concerned BAR is not needed for FPGA configuration
+	 * at all. Thus, enable the device via PCI config space command.
+	 */
+	pci_read_config_word(pdev, PCI_COMMAND, &cmd);
+	if (!(cmd & PCI_COMMAND_MEMORY)) {
+		cmd |= PCI_COMMAND_MEMORY;
+		pci_write_config_word(pdev, PCI_COMMAND, cmd);
+	}
+
+	ret = pci_request_region(pdev, CVP_BAR, "CVP");
+	if (ret) {
+		dev_err(&pdev->dev, "Requesting CVP BAR region failed\n");
+		goto err_disable;
+	}
+
+	conf->pci_dev = pdev;
+	conf->write_data = altera_cvp_write_data_iomem;
+
+	conf->map = pci_iomap(pdev, CVP_BAR, 0);
+	if (!conf->map) {
+		dev_warn(&pdev->dev, "Mapping CVP BAR failed\n");
+		conf->write_data = altera_cvp_write_data_config;
+	}
+
+	snprintf(conf->mgr_name, sizeof(conf->mgr_name), "%s @%s",
+		 ALTERA_CVP_MGR_NAME, pci_name(pdev));
+
+	ret = fpga_mgr_register(&pdev->dev, conf->mgr_name,
+				&altera_cvp_ops, conf);
+	if (ret)
+		goto err_unmap;
+
+	ret = driver_create_file(&altera_cvp_driver.driver,
+				 &driver_attr_chkcfg);
+	if (ret) {
+		dev_err(&pdev->dev, "Can't create sysfs chkcfg file\n");
+		fpga_mgr_unregister(&pdev->dev);
+		goto err_unmap;
+	}
+
+	return 0;
+
+err_unmap:
+	pci_iounmap(pdev, conf->map);
+	pci_release_region(pdev, CVP_BAR);
+err_disable:
+	cmd &= ~PCI_COMMAND_MEMORY;
+	pci_write_config_word(pdev, PCI_COMMAND, cmd);
+	return ret;
+}
+
+static void altera_cvp_remove(struct pci_dev *pdev)
+{
+	struct fpga_manager *mgr = pci_get_drvdata(pdev);
+	struct altera_cvp_conf *conf = mgr->priv;
+	u16 cmd;
+
+	driver_remove_file(&altera_cvp_driver.driver, &driver_attr_chkcfg);
+	fpga_mgr_unregister(&pdev->dev);
+	pci_iounmap(pdev, conf->map);
+	pci_release_region(pdev, CVP_BAR);
+	pci_read_config_word(pdev, PCI_COMMAND, &cmd);
+	cmd &= ~PCI_COMMAND_MEMORY;
+	pci_write_config_word(pdev, PCI_COMMAND, cmd);
+}
+
+module_pci_driver(altera_cvp_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Anatolij Gustschin <agust@denx.de>");
+MODULE_DESCRIPTION("Module to load Altera FPGA over CvP");
diff --git a/drivers/fpga/altera-hps2fpga.c b/drivers/fpga/altera-hps2fpga.c
index 3066b805f2d0..406d2f10741f 100644
--- a/drivers/fpga/altera-hps2fpga.c
+++ b/drivers/fpga/altera-hps2fpga.c
@@ -66,7 +66,7 @@ static int alt_hps2fpga_enable_show(struct fpga_bridge *bridge)
 
 /* The L3 REMAP register is write only, so keep a cached value. */
 static unsigned int l3_remap_shadow;
-static spinlock_t l3_remap_lock;
+static DEFINE_SPINLOCK(l3_remap_lock);
 
 static int _alt_hps2fpga_enable_set(struct altera_hps2fpga_data *priv,
 				    bool enable)
@@ -143,9 +143,15 @@ static int alt_fpga_bridge_probe(struct platform_device *pdev)
 	int ret;
 
 	of_id = of_match_device(altera_fpga_of_match, dev);
+	if (!of_id) {
+		dev_err(dev, "failed to match device\n");
+		return -ENODEV;
+	}
+
 	priv = (struct altera_hps2fpga_data *)of_id->data;
 
-	priv->bridge_reset = of_reset_control_get_by_index(dev->of_node, 0);
+	priv->bridge_reset = of_reset_control_get_exclusive_by_index(dev->of_node,
+								     0);
 	if (IS_ERR(priv->bridge_reset)) {
 		dev_err(dev, "Could not get %s reset control\n", priv->name);
 		return PTR_ERR(priv->bridge_reset);
@@ -171,8 +177,6 @@ static int alt_fpga_bridge_probe(struct platform_device *pdev)
 		return -EBUSY;
 	}
 
-	spin_lock_init(&l3_remap_lock);
-
 	if (!of_property_read_u32(dev->of_node, "bridge-enable", &enable)) {
 		if (enable > 1) {
 			dev_warn(dev, "invalid bridge-enable %u > 1\n", enable);
diff --git a/drivers/fpga/altera-ps-spi.c b/drivers/fpga/altera-ps-spi.c
new file mode 100644
index 000000000000..14f14efdf0d5
--- /dev/null
+++ b/drivers/fpga/altera-ps-spi.c
@@ -0,0 +1,308 @@
+/*
+ * Altera Passive Serial SPI Driver
+ *
+ *  Copyright (c) 2017 United Western Technologies, Corporation
+ *
+ *  Joshua Clayton <stillcompiling@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * Manage Altera FPGA firmware that is loaded over SPI using the passive
+ * serial configuration method.
+ * Firmware must be in binary "rbf" format.
+ * Works on Arria 10, Cyclone V and Stratix V. Should work on Cyclone series.
+ * May work on other Altera FPGAs.
+ */
+
+#include <linux/bitrev.h>
+#include <linux/delay.h>
+#include <linux/fpga/fpga-mgr.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/of_gpio.h>
+#include <linux/of_device.h>
+#include <linux/spi/spi.h>
+#include <linux/sizes.h>
+
+enum altera_ps_devtype {
+	CYCLONE5,
+	ARRIA10,
+};
+
+struct altera_ps_data {
+	enum altera_ps_devtype devtype;
+	int status_wait_min_us;
+	int status_wait_max_us;
+	int t_cfg_us;
+	int t_st2ck_us;
+};
+
+struct altera_ps_conf {
+	struct gpio_desc *config;
+	struct gpio_desc *confd;
+	struct gpio_desc *status;
+	struct spi_device *spi;
+	const struct altera_ps_data *data;
+	u32 info_flags;
+	char mgr_name[64];
+};
+
+/*          |   Arria 10  |   Cyclone5  |   Stratix5  |
+ * t_CF2ST0 |     [; 600] |     [; 600] |     [; 600] |ns
+ * t_CFG    |        [2;] |        [2;] |        [2;] |µs
+ * t_STATUS | [268; 3000] | [268; 1506] | [268; 1506] |µs
+ * t_CF2ST1 |    [; 3000] |    [; 1506] |    [; 1506] |µs
+ * t_CF2CK  |     [3010;] |     [1506;] |     [1506;] |µs
+ * t_ST2CK  |       [10;] |        [2;] |        [2;] |µs
+ * t_CD2UM  |  [175; 830] |  [175; 437] |  [175; 437] |µs
+ */
+static struct altera_ps_data c5_data = {
+	/* these values for Cyclone5 are compatible with Stratix5 */
+	.devtype = CYCLONE5,
+	.status_wait_min_us = 268,
+	.status_wait_max_us = 1506,
+	.t_cfg_us = 2,
+	.t_st2ck_us = 2,
+};
+
+static struct altera_ps_data a10_data = {
+	.devtype = ARRIA10,
+	.status_wait_min_us = 268,  /* min(t_STATUS) */
+	.status_wait_max_us = 3000, /* max(t_CF2ST1) */
+	.t_cfg_us = 2,    /* max { min(t_CFG), max(tCF2ST0) } */
+	.t_st2ck_us = 10, /* min(t_ST2CK) */
+};
+
+static const struct of_device_id of_ef_match[] = {
+	{ .compatible = "altr,fpga-passive-serial", .data = &c5_data },
+	{ .compatible = "altr,fpga-arria10-passive-serial", .data = &a10_data },
+	{}
+};
+MODULE_DEVICE_TABLE(of, of_ef_match);
+
+static enum fpga_mgr_states altera_ps_state(struct fpga_manager *mgr)
+{
+	struct altera_ps_conf *conf = mgr->priv;
+
+	if (gpiod_get_value_cansleep(conf->status))
+		return FPGA_MGR_STATE_RESET;
+
+	return FPGA_MGR_STATE_UNKNOWN;
+}
+
+static inline void altera_ps_delay(int delay_us)
+{
+	if (delay_us > 10)
+		usleep_range(delay_us, delay_us + 5);
+	else
+		udelay(delay_us);
+}
+
+static int altera_ps_write_init(struct fpga_manager *mgr,
+				struct fpga_image_info *info,
+				const char *buf, size_t count)
+{
+	struct altera_ps_conf *conf = mgr->priv;
+	int min, max, waits;
+	int i;
+
+	conf->info_flags = info->flags;
+
+	if (info->flags & FPGA_MGR_PARTIAL_RECONFIG) {
+		dev_err(&mgr->dev, "Partial reconfiguration not supported.\n");
+		return -EINVAL;
+	}
+
+	gpiod_set_value_cansleep(conf->config, 1);
+
+	/* wait min reset pulse time */
+	altera_ps_delay(conf->data->t_cfg_us);
+
+	if (!gpiod_get_value_cansleep(conf->status)) {
+		dev_err(&mgr->dev, "Status pin failed to show a reset\n");
+		return -EIO;
+	}
+
+	gpiod_set_value_cansleep(conf->config, 0);
+
+	min = conf->data->status_wait_min_us;
+	max = conf->data->status_wait_max_us;
+	waits = max / min;
+	if (max % min)
+		waits++;
+
+	/* wait for max { max(t_STATUS), max(t_CF2ST1) } */
+	for (i = 0; i < waits; i++) {
+		usleep_range(min, min + 10);
+		if (!gpiod_get_value_cansleep(conf->status)) {
+			/* wait for min(t_ST2CK)*/
+			altera_ps_delay(conf->data->t_st2ck_us);
+			return 0;
+		}
+	}
+
+	dev_err(&mgr->dev, "Status pin not ready.\n");
+	return -EIO;
+}
+
+static void rev_buf(char *buf, size_t len)
+{
+	u32 *fw32 = (u32 *)buf;
+	size_t extra_bytes = (len & 0x03);
+	const u32 *fw_end = (u32 *)(buf + len - extra_bytes);
+
+	/* set buffer to lsb first */
+	while (fw32 < fw_end) {
+		*fw32 = bitrev8x4(*fw32);
+		fw32++;
+	}
+
+	if (extra_bytes) {
+		buf = (char *)fw_end;
+		while (extra_bytes) {
+			*buf = bitrev8(*buf);
+			buf++;
+			extra_bytes--;
+		}
+	}
+}
+
+static int altera_ps_write(struct fpga_manager *mgr, const char *buf,
+			   size_t count)
+{
+	struct altera_ps_conf *conf = mgr->priv;
+	const char *fw_data = buf;
+	const char *fw_data_end = fw_data + count;
+
+	while (fw_data < fw_data_end) {
+		int ret;
+		size_t stride = min_t(size_t, fw_data_end - fw_data, SZ_4K);
+
+		if (!(conf->info_flags & FPGA_MGR_BITSTREAM_LSB_FIRST))
+			rev_buf((char *)fw_data, stride);
+
+		ret = spi_write(conf->spi, fw_data, stride);
+		if (ret) {
+			dev_err(&mgr->dev, "spi error in firmware write: %d\n",
+				ret);
+			return ret;
+		}
+		fw_data += stride;
+	}
+
+	return 0;
+}
+
+static int altera_ps_write_complete(struct fpga_manager *mgr,
+				    struct fpga_image_info *info)
+{
+	struct altera_ps_conf *conf = mgr->priv;
+	const char dummy[] = {0};
+	int ret;
+
+	if (gpiod_get_value_cansleep(conf->status)) {
+		dev_err(&mgr->dev, "Error during configuration.\n");
+		return -EIO;
+	}
+
+	if (!IS_ERR(conf->confd)) {
+		if (!gpiod_get_raw_value_cansleep(conf->confd)) {
+			dev_err(&mgr->dev, "CONF_DONE is inactive!\n");
+			return -EIO;
+		}
+	}
+
+	/*
+	 * After CONF_DONE goes high, send two additional falling edges on DCLK
+	 * to begin initialization and enter user mode
+	 */
+	ret = spi_write(conf->spi, dummy, 1);
+	if (ret) {
+		dev_err(&mgr->dev, "spi error during end sequence: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static const struct fpga_manager_ops altera_ps_ops = {
+	.state = altera_ps_state,
+	.write_init = altera_ps_write_init,
+	.write = altera_ps_write,
+	.write_complete = altera_ps_write_complete,
+};
+
+static int altera_ps_probe(struct spi_device *spi)
+{
+	struct altera_ps_conf *conf;
+	const struct of_device_id *of_id;
+
+	conf = devm_kzalloc(&spi->dev, sizeof(*conf), GFP_KERNEL);
+	if (!conf)
+		return -ENOMEM;
+
+	of_id = of_match_device(of_ef_match, &spi->dev);
+	if (!of_id)
+		return -ENODEV;
+
+	conf->data = of_id->data;
+	conf->spi = spi;
+	conf->config = devm_gpiod_get(&spi->dev, "nconfig", GPIOD_OUT_HIGH);
+	if (IS_ERR(conf->config)) {
+		dev_err(&spi->dev, "Failed to get config gpio: %ld\n",
+			PTR_ERR(conf->config));
+		return PTR_ERR(conf->config);
+	}
+
+	conf->status = devm_gpiod_get(&spi->dev, "nstat", GPIOD_IN);
+	if (IS_ERR(conf->status)) {
+		dev_err(&spi->dev, "Failed to get status gpio: %ld\n",
+			PTR_ERR(conf->status));
+		return PTR_ERR(conf->status);
+	}
+
+	conf->confd = devm_gpiod_get(&spi->dev, "confd", GPIOD_IN);
+	if (IS_ERR(conf->confd)) {
+		dev_warn(&spi->dev, "Not using confd gpio: %ld\n",
+			 PTR_ERR(conf->confd));
+	}
+
+	/* Register manager with unique name */
+	snprintf(conf->mgr_name, sizeof(conf->mgr_name), "%s %s",
+		 dev_driver_string(&spi->dev), dev_name(&spi->dev));
+
+	return fpga_mgr_register(&spi->dev, conf->mgr_name,
+				 &altera_ps_ops, conf);
+}
+
+static int altera_ps_remove(struct spi_device *spi)
+{
+	fpga_mgr_unregister(&spi->dev);
+
+	return 0;
+}
+
+static const struct spi_device_id altera_ps_spi_ids[] = {
+	{"cyclone-ps-spi", 0},
+	{}
+};
+MODULE_DEVICE_TABLE(spi, altera_ps_spi_ids);
+
+static struct spi_driver altera_ps_driver = {
+	.driver = {
+		.name = "altera-ps-spi",
+		.owner = THIS_MODULE,
+		.of_match_table = of_match_ptr(of_ef_match),
+	},
+	.id_table = altera_ps_spi_ids,
+	.probe = altera_ps_probe,
+	.remove = altera_ps_remove,
+};
+
+module_spi_driver(altera_ps_driver)
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Joshua Clayton <stillcompiling@gmail.com>");
+MODULE_DESCRIPTION("Module to load Altera FPGA firmware over SPI");
diff --git a/drivers/fpga/fpga-region.c b/drivers/fpga/fpga-region.c
index 3b6b2f4182a1..d9ab7c75b14f 100644
--- a/drivers/fpga/fpga-region.c
+++ b/drivers/fpga/fpga-region.c
@@ -319,8 +319,8 @@ static int child_regions_with_firmware(struct device_node *overlay)
 	of_node_put(child_region);
 
 	if (ret)
-		pr_err("firmware-name not allowed in child FPGA region: %s",
-		       child_region->full_name);
+		pr_err("firmware-name not allowed in child FPGA region: %pOF",
+		       child_region);
 
 	return ret;
 }
diff --git a/drivers/fsi/fsi-core.c b/drivers/fsi/fsi-core.c
index 06432d84cbf8..4ea63d9bd131 100644
--- a/drivers/fsi/fsi-core.c
+++ b/drivers/fsi/fsi-core.c
@@ -475,7 +475,7 @@ static ssize_t fsi_slave_sysfs_raw_write(struct file *file,
 	return count;
 }
 
-static struct bin_attribute fsi_slave_raw_attr = {
+static const struct bin_attribute fsi_slave_raw_attr = {
 	.attr = {
 		.name = "raw",
 		.mode = 0600,
@@ -499,7 +499,7 @@ static ssize_t fsi_slave_sysfs_term_write(struct file *file,
 	return count;
 }
 
-static struct bin_attribute fsi_slave_term_attr = {
+static const struct bin_attribute fsi_slave_term_attr = {
 	.attr = {
 		.name = "term",
 		.mode = 0200,
diff --git a/drivers/fsi/fsi-scom.c b/drivers/fsi/fsi-scom.c
index 98d062fd353e..e13353a2fd7c 100644
--- a/drivers/fsi/fsi-scom.c
+++ b/drivers/fsi/fsi-scom.c
@@ -57,12 +57,6 @@ static int put_scom(struct scom_device *scom_dev, uint64_t value,
 	int rc;
 	uint32_t data;
 
-	data = cpu_to_be32(SCOM_RESET_CMD);
-	rc = fsi_device_write(scom_dev->fsi_dev, SCOM_RESET_REG, &data,
-				sizeof(uint32_t));
-	if (rc)
-		return rc;
-
 	data = cpu_to_be32((value >> 32) & 0xffffffff);
 	rc = fsi_device_write(scom_dev->fsi_dev, SCOM_DATA0_REG, &data,
 				sizeof(uint32_t));
@@ -186,6 +180,7 @@ static const struct file_operations scom_fops = {
 
 static int scom_probe(struct device *dev)
 {
+	uint32_t data;
 	struct fsi_device *fsi_dev = to_fsi_dev(dev);
 	struct scom_device *scom;
 
@@ -202,6 +197,9 @@ static int scom_probe(struct device *dev)
 	scom->mdev.parent = dev;
 	list_add(&scom->link, &scom_devices);
 
+	data = cpu_to_be32(SCOM_RESET_CMD);
+	fsi_device_write(fsi_dev, SCOM_RESET_REG, &data, sizeof(uint32_t));
+
 	return misc_register(&scom->mdev);
 }
 
diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig
index 461d6fc3688b..3388d54ba114 100644
--- a/drivers/gpio/Kconfig
+++ b/drivers/gpio/Kconfig
@@ -311,7 +311,7 @@ config GPIO_MOCKUP
 	depends on GPIOLIB && SYSFS
 	select GPIO_SYSFS
 	select GPIOLIB_IRQCHIP
-	select IRQ_WORK
+	select IRQ_SIM
 	help
 	  This enables GPIO Testing driver, which provides a way to test GPIO
 	  subsystem through sysfs(or char device) and debugfs. GPIO_SYSFS
@@ -450,6 +450,15 @@ config GPIO_TS4800
 	help
 	  This driver support TS-4800 FPGA GPIO controllers.
 
+config GPIO_THUNDERX
+	tristate "Cavium ThunderX/OCTEON-TX GPIO"
+	depends on ARCH_THUNDER || (64BIT && COMPILE_TEST)
+	depends on PCI_MSI && IRQ_DOMAIN_HIERARCHY
+	select IRQ_FASTEOI_HIERARCHY_HANDLERS
+	help
+	  Say yes here to support the on-chip GPIO lines on the ThunderX
+	  and OCTEON-TX families of SoCs.
+
 config GPIO_TZ1090
 	bool "Toumaz Xenif TZ1090 GPIO support"
 	depends on SOC_TZ1090
@@ -1065,6 +1074,21 @@ config GPIO_TPS65912
 	help
 	  This driver supports TPS65912 gpio chip
 
+config GPIO_TPS68470
+	bool "TPS68470 GPIO"
+	depends on MFD_TPS68470
+	help
+	  Select this option to enable GPIO driver for the TPS68470
+	  chip family.
+	  There are 7 GPIOs and few sensor related GPIOs supported
+	  by the TPS68470. While the 7 GPIOs can be configured as
+	  input or output as appropriate, the sensor related GPIOs
+	  are "output only" GPIOs.
+
+	  This driver config is bool, as the GPIO functionality
+	  of the TPS68470 must be available before dependent
+	  drivers are loaded.
+
 config GPIO_TWL4030
 	tristate "TWL4030, TWL5030, and TPS659x0 GPIOs"
 	depends on TWL4030_CORE
diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile
index a9fda6c55113..aeb70e9de6f2 100644
--- a/drivers/gpio/Makefile
+++ b/drivers/gpio/Makefile
@@ -113,6 +113,7 @@ obj-$(CONFIG_GPIO_SYSCON)	+= gpio-syscon.o
 obj-$(CONFIG_GPIO_TB10X)	+= gpio-tb10x.o
 obj-$(CONFIG_GPIO_TC3589X)	+= gpio-tc3589x.o
 obj-$(CONFIG_GPIO_TEGRA)	+= gpio-tegra.o
+obj-$(CONFIG_GPIO_THUNDERX)	+= gpio-thunderx.o
 obj-$(CONFIG_GPIO_TIMBERDALE)	+= gpio-timberdale.o
 obj-$(CONFIG_GPIO_PALMAS)	+= gpio-palmas.o
 obj-$(CONFIG_GPIO_TPIC2810)	+= gpio-tpic2810.o
@@ -121,6 +122,7 @@ obj-$(CONFIG_GPIO_TPS65218)	+= gpio-tps65218.o
 obj-$(CONFIG_GPIO_TPS6586X)	+= gpio-tps6586x.o
 obj-$(CONFIG_GPIO_TPS65910)	+= gpio-tps65910.o
 obj-$(CONFIG_GPIO_TPS65912)	+= gpio-tps65912.o
+obj-$(CONFIG_GPIO_TPS68470)	+= gpio-tps68470.o
 obj-$(CONFIG_GPIO_TS4800)	+= gpio-ts4800.o
 obj-$(CONFIG_GPIO_TS4900)	+= gpio-ts4900.o
 obj-$(CONFIG_GPIO_TS5500)	+= gpio-ts5500.o
diff --git a/drivers/gpio/devres.c b/drivers/gpio/devres.c
index a75511d1ea5d..afbff155a0ba 100644
--- a/drivers/gpio/devres.c
+++ b/drivers/gpio/devres.c
@@ -132,6 +132,7 @@ EXPORT_SYMBOL(devm_gpiod_get_index);
  * @index:	index of the GPIO to obtain in the consumer
  * @child:	firmware node (child of @dev)
  * @flags:	GPIO initialization flags
+ * @label:	label to attach to the requested GPIO
  *
  * GPIO descriptors returned from this function are automatically disposed on
  * driver detach.
@@ -271,6 +272,7 @@ EXPORT_SYMBOL(devm_gpiod_get_array_optional);
 
 /**
  * devm_gpiod_put - Resource-managed gpiod_put()
+ * @dev:	GPIO consumer
  * @desc:	GPIO descriptor to dispose of
  *
  * Dispose of a GPIO descriptor obtained with devm_gpiod_get() or
@@ -286,6 +288,7 @@ EXPORT_SYMBOL(devm_gpiod_put);
 
 /**
  * devm_gpiod_put_array - Resource-managed gpiod_put_array()
+ * @dev:	GPIO consumer
  * @descs:	GPIO descriptor array to dispose of
  *
  * Dispose of an array of GPIO descriptors obtained with devm_gpiod_get_array().
diff --git a/drivers/gpio/gpio-74x164.c b/drivers/gpio/gpio-74x164.c
index a6607faf2fdf..6b535ec858cc 100644
--- a/drivers/gpio/gpio-74x164.c
+++ b/drivers/gpio/gpio-74x164.c
@@ -9,6 +9,7 @@
  *  published by the Free Software Foundation.
  */
 
+#include <linux/gpio/consumer.h>
 #include <linux/init.h>
 #include <linux/mutex.h>
 #include <linux/spi/spi.h>
@@ -31,6 +32,7 @@ struct gen_74x164_chip {
 	 * numbering, store the bytes in reverse order.
 	 */
 	u8			buffer[0];
+	struct gpio_desc	*gpiod_oe;
 };
 
 static int __gen_74x164_write_config(struct gen_74x164_chip *chip)
@@ -126,6 +128,13 @@ static int gen_74x164_probe(struct spi_device *spi)
 	if (!chip)
 		return -ENOMEM;
 
+	chip->gpiod_oe = devm_gpiod_get_optional(&spi->dev, "enable",
+						 GPIOD_OUT_LOW);
+	if (IS_ERR(chip->gpiod_oe))
+		return PTR_ERR(chip->gpiod_oe);
+
+	gpiod_set_value_cansleep(chip->gpiod_oe, 1);
+
 	spi_set_drvdata(spi, chip);
 
 	chip->gpio_chip.label = spi->modalias;
@@ -164,6 +173,7 @@ static int gen_74x164_remove(struct spi_device *spi)
 {
 	struct gen_74x164_chip *chip = spi_get_drvdata(spi);
 
+	gpiod_set_value_cansleep(chip->gpiod_oe, 0);
 	gpiochip_remove(&chip->gpio_chip);
 	mutex_destroy(&chip->lock);
 
diff --git a/drivers/gpio/gpio-altera-a10sr.c b/drivers/gpio/gpio-altera-a10sr.c
index 16a8951b2bed..6b11f1314248 100644
--- a/drivers/gpio/gpio-altera-a10sr.c
+++ b/drivers/gpio/gpio-altera-a10sr.c
@@ -71,7 +71,7 @@ static int altr_a10sr_gpio_direction_output(struct gpio_chip *gc,
 	return -EINVAL;
 }
 
-static struct gpio_chip altr_a10sr_gc = {
+static const struct gpio_chip altr_a10sr_gc = {
 	.label = "altr_a10sr_gpio",
 	.owner = THIS_MODULE,
 	.get = altr_a10sr_gpio_get,
diff --git a/drivers/gpio/gpio-altera.c b/drivers/gpio/gpio-altera.c
index 17485dc20384..ccc02ed65b3c 100644
--- a/drivers/gpio/gpio-altera.c
+++ b/drivers/gpio/gpio-altera.c
@@ -324,8 +324,8 @@ skip_irq:
 	return 0;
 teardown:
 	of_mm_gpiochip_remove(&altera_gc->mmchip);
-	pr_err("%s: registration failed with status %d\n",
-		node->full_name, ret);
+	pr_err("%pOF: registration failed with status %d\n",
+		node, ret);
 
 	return ret;
 }
diff --git a/drivers/gpio/gpio-aspeed.c b/drivers/gpio/gpio-aspeed.c
index 4ca436e66bdb..bfc53995064a 100644
--- a/drivers/gpio/gpio-aspeed.c
+++ b/drivers/gpio/gpio-aspeed.c
@@ -834,7 +834,7 @@ static int __init aspeed_gpio_probe(struct platform_device *pdev)
 	gpio->clk = of_clk_get(pdev->dev.of_node, 0);
 	if (IS_ERR(gpio->clk)) {
 		dev_warn(&pdev->dev,
-				"No HPLL clock phandle provided, debouncing disabled\n");
+				"Failed to get clock from devicetree, debouncing disabled\n");
 		gpio->clk = NULL;
 	}
 
diff --git a/drivers/gpio/gpio-brcmstb.c b/drivers/gpio/gpio-brcmstb.c
index e6489143721a..dd0308cc8bb0 100644
--- a/drivers/gpio/gpio-brcmstb.c
+++ b/drivers/gpio/gpio-brcmstb.c
@@ -339,6 +339,7 @@ static int brcmstb_gpio_irq_setup(struct platform_device *pdev,
 	struct brcmstb_gpio_priv *priv = bank->parent_priv;
 	struct device *dev = &pdev->dev;
 	struct device_node *np = dev->of_node;
+	int err;
 
 	bank->irq_chip.name = dev_name(dev);
 	bank->irq_chip.irq_mask = brcmstb_gpio_irq_mask;
@@ -355,8 +356,6 @@ static int brcmstb_gpio_irq_setup(struct platform_device *pdev,
 			dev_warn(dev,
 				"Couldn't get wake IRQ - GPIOs will not be able to wake from sleep");
 		} else {
-			int err;
-
 			/*
 			 * Set wakeup capability before requesting wakeup
 			 * interrupt, so we can process boot-time "wakeups"
@@ -383,8 +382,10 @@ static int brcmstb_gpio_irq_setup(struct platform_device *pdev,
 	if (priv->can_wake)
 		bank->irq_chip.irq_set_wake = brcmstb_gpio_irq_set_wake;
 
-	gpiochip_irqchip_add(&bank->gc, &bank->irq_chip, 0,
-			handle_simple_irq, IRQ_TYPE_NONE);
+	err = gpiochip_irqchip_add(&bank->gc, &bank->irq_chip, 0,
+				   handle_simple_irq, IRQ_TYPE_NONE);
+	if (err)
+		return err;
 	gpiochip_set_chained_irqchip(&bank->gc, &bank->irq_chip,
 			priv->parent_irq, brcmstb_gpio_irq_handler);
 
@@ -483,7 +484,7 @@ static int brcmstb_gpio_probe(struct platform_device *pdev)
 
 		gc->of_node = np;
 		gc->owner = THIS_MODULE;
-		gc->label = np->full_name;
+		gc->label = devm_kasprintf(dev, GFP_KERNEL, "%pOF", dev->of_node);
 		gc->base = gpio_base;
 		gc->of_gpio_n_cells = 2;
 		gc->of_xlate = brcmstb_gpio_of_xlate;
diff --git a/drivers/gpio/gpio-davinci.c b/drivers/gpio/gpio-davinci.c
index 65cb359308e3..f75d8443ecaf 100644
--- a/drivers/gpio/gpio-davinci.c
+++ b/drivers/gpio/gpio-davinci.c
@@ -166,7 +166,7 @@ of_err:
 static int davinci_gpio_probe(struct platform_device *pdev)
 {
 	static int ctrl_num, bank_base;
-	int gpio, bank;
+	int gpio, bank, ret = 0;
 	unsigned ngpio, nbank;
 	struct davinci_gpio_controller *chips;
 	struct davinci_gpio_platform_data *pdata;
@@ -232,10 +232,23 @@ static int davinci_gpio_probe(struct platform_device *pdev)
 	for (gpio = 0, bank = 0; gpio < ngpio; gpio += 32, bank++)
 		chips->regs[bank] = gpio_base + offset_array[bank];
 
-	gpiochip_add_data(&chips->chip, chips);
+	ret = devm_gpiochip_add_data(dev, &chips->chip, chips);
+	if (ret)
+		goto err;
+
 	platform_set_drvdata(pdev, chips);
-	davinci_gpio_irq_setup(pdev);
+	ret = davinci_gpio_irq_setup(pdev);
+	if (ret)
+		goto err;
+
 	return 0;
+
+err:
+	/* Revert the static variable increments */
+	ctrl_num--;
+	bank_base -= ngpio;
+
+	return ret;
 }
 
 /*--------------------------------------------------------------------------*/
@@ -477,8 +490,7 @@ static int davinci_gpio_irq_setup(struct platform_device *pdev)
 
 	clk = devm_clk_get(dev, "gpio");
 	if (IS_ERR(clk)) {
-		printk(KERN_ERR "Error %ld getting gpio clock?\n",
-		       PTR_ERR(clk));
+		dev_err(dev, "Error %ld getting gpio clock\n", PTR_ERR(clk));
 		return PTR_ERR(clk);
 	}
 	ret = clk_prepare_enable(clk);
diff --git a/drivers/gpio/gpio-ge.c b/drivers/gpio/gpio-ge.c
index 8650b2916f87..6f5a7fe9787d 100644
--- a/drivers/gpio/gpio-ge.c
+++ b/drivers/gpio/gpio-ge.c
@@ -76,8 +76,7 @@ static int __init gef_gpio_probe(struct platform_device *pdev)
 	}
 
 	/* Setup pointers to chip functions */
-	gc->label = devm_kstrdup(&pdev->dev, pdev->dev.of_node->full_name,
-				     GFP_KERNEL);
+	gc->label = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%pOF", pdev->dev.of_node);
 	if (!gc->label) {
 		ret = -ENOMEM;
 		goto err0;
@@ -96,8 +95,7 @@ static int __init gef_gpio_probe(struct platform_device *pdev)
 	return 0;
 err0:
 	iounmap(regs);
-	pr_err("%s: GPIO chip registration failed\n",
-			pdev->dev.of_node->full_name);
+	pr_err("%pOF: GPIO chip registration failed\n", pdev->dev.of_node);
 	return ret;
 };
 
diff --git a/drivers/gpio/gpio-grgpio.c b/drivers/gpio/gpio-grgpio.c
index 7847dd34f86f..6544a16ab02e 100644
--- a/drivers/gpio/gpio-grgpio.c
+++ b/drivers/gpio/gpio-grgpio.c
@@ -367,7 +367,7 @@ static int grgpio_probe(struct platform_device *ofdev)
 	gc->of_node = np;
 	gc->owner = THIS_MODULE;
 	gc->to_irq = grgpio_to_irq;
-	gc->label = np->full_name;
+	gc->label = devm_kasprintf(&ofdev->dev, GFP_KERNEL, "%pOF", np);
 	gc->base = -1;
 
 	err = of_property_read_u32(np, "nbits", &prop);
diff --git a/drivers/gpio/gpio-it87.c b/drivers/gpio/gpio-it87.c
index 45d29e488dbb..d43d0a2cc4c5 100644
--- a/drivers/gpio/gpio-it87.c
+++ b/drivers/gpio/gpio-it87.c
@@ -2,6 +2,7 @@
  *  GPIO interface for IT87xx Super I/O chips
  *
  *  Author: Diego Elio Pettenò <flameeyes@flameeyes.eu>
+ *  Copyright (c) 2017 Google, Inc.
  *
  *  Based on it87_wdt.c     by Oliver Schuster
  *           gpio-it8761e.c by Denis Turischev
@@ -39,6 +40,7 @@
 #define IT8728_ID	0x8728
 #define IT8732_ID	0x8732
 #define IT8761_ID	0x8761
+#define IT8772_ID	0x8772
 
 /* IO Ports */
 #define REG		0x2e
@@ -314,6 +316,7 @@ static int __init it87_gpio_init(void)
 		break;
 	case IT8728_ID:
 	case IT8732_ID:
+	case IT8772_ID:
 		gpio_ba_reg = 0x62;
 		it87_gpio->io_size = 8;
 		it87_gpio->output_base = 0xc8;
diff --git a/drivers/gpio/gpio-max77620.c b/drivers/gpio/gpio-max77620.c
index 743459d9477d..538bce4b5b42 100644
--- a/drivers/gpio/gpio-max77620.c
+++ b/drivers/gpio/gpio-max77620.c
@@ -82,7 +82,7 @@ static const struct regmap_irq max77620_gpio_irqs[] = {
 	},
 };
 
-static struct regmap_irq_chip max77620_gpio_irq_chip = {
+static const struct regmap_irq_chip max77620_gpio_irq_chip = {
 	.name = "max77620-gpio",
 	.irqs = max77620_gpio_irqs,
 	.num_irqs = ARRAY_SIZE(max77620_gpio_irqs),
diff --git a/drivers/gpio/gpio-mb86s7x.c b/drivers/gpio/gpio-mb86s7x.c
index ffb73f688ae1..94d772677ed6 100644
--- a/drivers/gpio/gpio-mb86s7x.c
+++ b/drivers/gpio/gpio-mb86s7x.c
@@ -168,7 +168,9 @@ static int mb86s70_gpio_probe(struct platform_device *pdev)
 	if (IS_ERR(gchip->clk))
 		return PTR_ERR(gchip->clk);
 
-	clk_prepare_enable(gchip->clk);
+	ret = clk_prepare_enable(gchip->clk);
+	if (ret)
+		return ret;
 
 	spin_lock_init(&gchip->lock);
 
diff --git a/drivers/gpio/gpio-ml-ioh.c b/drivers/gpio/gpio-ml-ioh.c
index 74fdce096c26..4b80e996d976 100644
--- a/drivers/gpio/gpio-ml-ioh.c
+++ b/drivers/gpio/gpio-ml-ioh.c
@@ -391,9 +391,10 @@ static int ioh_gpio_alloc_generic_chip(struct ioh_gpio *chip,
 {
 	struct irq_chip_generic *gc;
 	struct irq_chip_type *ct;
+	int rv;
 
-	gc = irq_alloc_generic_chip("ioh_gpio", 1, irq_start, chip->base,
-				    handle_simple_irq);
+	gc = devm_irq_alloc_generic_chip(chip->dev, "ioh_gpio", 1, irq_start,
+					 chip->base, handle_simple_irq);
 	if (!gc)
 		return -ENOMEM;
 
@@ -406,10 +407,11 @@ static int ioh_gpio_alloc_generic_chip(struct ioh_gpio *chip,
 	ct->chip.irq_disable = ioh_irq_disable;
 	ct->chip.irq_enable = ioh_irq_enable;
 
-	irq_setup_generic_chip(gc, IRQ_MSK(num), IRQ_GC_INIT_MASK_CACHE,
-			       IRQ_NOREQUEST | IRQ_NOPROBE, 0);
+	rv = devm_irq_setup_generic_chip(chip->dev, gc, IRQ_MSK(num),
+					 IRQ_GC_INIT_MASK_CACHE,
+					 IRQ_NOREQUEST | IRQ_NOPROBE, 0);
 
-	return 0;
+	return rv;
 }
 
 static int ioh_gpio_probe(struct pci_dev *pdev,
diff --git a/drivers/gpio/gpio-mockup.c b/drivers/gpio/gpio-mockup.c
index a6565e128f9e..9532d86a82f7 100644
--- a/drivers/gpio/gpio-mockup.c
+++ b/drivers/gpio/gpio-mockup.c
@@ -20,7 +20,7 @@
 #include <linux/slab.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
-#include <linux/irq_work.h>
+#include <linux/irq_sim.h>
 #include <linux/debugfs.h>
 #include <linux/uaccess.h>
 
@@ -47,18 +47,12 @@ enum {
 struct gpio_mockup_line_status {
 	int dir;
 	bool value;
-	bool irq_enabled;
-};
-
-struct gpio_mockup_irq_context {
-	struct irq_work work;
-	int irq;
 };
 
 struct gpio_mockup_chip {
 	struct gpio_chip gc;
 	struct gpio_mockup_line_status *lines;
-	struct gpio_mockup_irq_context irq_ctx;
+	struct irq_sim irqsim;
 	struct dentry *dbg_dir;
 };
 
@@ -144,65 +138,11 @@ static int gpio_mockup_name_lines(struct device *dev,
 	return 0;
 }
 
-static int gpio_mockup_to_irq(struct gpio_chip *chip, unsigned int offset)
-{
-	return chip->irq_base + offset;
-}
-
-static void gpio_mockup_irqmask(struct irq_data *data)
-{
-	struct gpio_chip *gc = irq_data_get_irq_chip_data(data);
-	struct gpio_mockup_chip *chip = gpiochip_get_data(gc);
-
-	chip->lines[data->irq - gc->irq_base].irq_enabled = false;
-}
-
-static void gpio_mockup_irqunmask(struct irq_data *data)
+static int gpio_mockup_to_irq(struct gpio_chip *gc, unsigned int offset)
 {
-	struct gpio_chip *gc = irq_data_get_irq_chip_data(data);
 	struct gpio_mockup_chip *chip = gpiochip_get_data(gc);
 
-	chip->lines[data->irq - gc->irq_base].irq_enabled = true;
-}
-
-static struct irq_chip gpio_mockup_irqchip = {
-	.name		= GPIO_MOCKUP_NAME,
-	.irq_mask	= gpio_mockup_irqmask,
-	.irq_unmask	= gpio_mockup_irqunmask,
-};
-
-static void gpio_mockup_handle_irq(struct irq_work *work)
-{
-	struct gpio_mockup_irq_context *irq_ctx;
-
-	irq_ctx = container_of(work, struct gpio_mockup_irq_context, work);
-	handle_simple_irq(irq_to_desc(irq_ctx->irq));
-}
-
-static int gpio_mockup_irqchip_setup(struct device *dev,
-				     struct gpio_mockup_chip *chip)
-{
-	struct gpio_chip *gc = &chip->gc;
-	int irq_base, i;
-
-	irq_base = devm_irq_alloc_descs(dev, -1, 0, gc->ngpio, 0);
-	if (irq_base < 0)
-		return irq_base;
-
-	gc->irq_base = irq_base;
-	gc->irqchip = &gpio_mockup_irqchip;
-
-	for (i = 0; i < gc->ngpio; i++) {
-		irq_set_chip(irq_base + i, gc->irqchip);
-		irq_set_chip_data(irq_base + i, gc);
-		irq_set_handler(irq_base + i, &handle_simple_irq);
-		irq_modify_status(irq_base + i,
-				  IRQ_NOREQUEST | IRQ_NOAUTOEN, IRQ_NOPROBE);
-	}
-
-	init_irq_work(&chip->irq_ctx.work, gpio_mockup_handle_irq);
-
-	return 0;
+	return irq_sim_irqnum(&chip->irqsim, offset);
 }
 
 static ssize_t gpio_mockup_event_write(struct file *file,
@@ -213,7 +153,6 @@ static ssize_t gpio_mockup_event_write(struct file *file,
 	struct gpio_mockup_chip *chip;
 	struct seq_file *sfile;
 	struct gpio_desc *desc;
-	struct gpio_chip *gc;
 	int rv, val;
 
 	rv = kstrtoint_from_user(usr_buf, size, 0, &val);
@@ -226,13 +165,9 @@ static ssize_t gpio_mockup_event_write(struct file *file,
 	priv = sfile->private;
 	desc = priv->desc;
 	chip = priv->chip;
-	gc = &chip->gc;
 
-	if (chip->lines[priv->offset].irq_enabled) {
-		gpiod_set_value_cansleep(desc, val);
-		priv->chip->irq_ctx.irq = gc->irq_base + priv->offset;
-		irq_work_queue(&priv->chip->irq_ctx.work);
-	}
+	gpiod_set_value_cansleep(desc, val);
+	irq_sim_fire(&chip->irqsim, priv->offset);
 
 	return size;
 }
@@ -319,7 +254,7 @@ static int gpio_mockup_add(struct device *dev,
 			return ret;
 	}
 
-	ret = gpio_mockup_irqchip_setup(dev, chip);
+	ret = devm_irq_sim_init(dev, &chip->irqsim, gc->ngpio);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpio/gpio-mpc8xxx.c b/drivers/gpio/gpio-mpc8xxx.c
index 793518a30afe..8c93dec498fa 100644
--- a/drivers/gpio/gpio-mpc8xxx.c
+++ b/drivers/gpio/gpio-mpc8xxx.c
@@ -348,8 +348,8 @@ static int mpc8xxx_probe(struct platform_device *pdev)
 
 	ret = gpiochip_add_data(gc, mpc8xxx_gc);
 	if (ret) {
-		pr_err("%s: GPIO chip registration failed with status %d\n",
-		       np->full_name, ret);
+		pr_err("%pOF: GPIO chip registration failed with status %d\n",
+		       np, ret);
 		goto err;
 	}
 
diff --git a/drivers/gpio/gpio-msic.c b/drivers/gpio/gpio-msic.c
index 1b7ce7f85886..6cb67595d15f 100644
--- a/drivers/gpio/gpio-msic.c
+++ b/drivers/gpio/gpio-msic.c
@@ -265,8 +265,8 @@ static int platform_msic_gpio_probe(struct platform_device *pdev)
 	int i;
 
 	if (irq < 0) {
-		dev_err(dev, "no IRQ line\n");
-		return -EINVAL;
+		dev_err(dev, "no IRQ line: %d\n", irq);
+		return irq;
 	}
 
 	if (!pdata || !pdata->gpio_base) {
diff --git a/drivers/gpio/gpio-mvebu.c b/drivers/gpio/gpio-mvebu.c
index e338c3743562..45c65f805fd6 100644
--- a/drivers/gpio/gpio-mvebu.c
+++ b/drivers/gpio/gpio-mvebu.c
@@ -557,7 +557,7 @@ static void mvebu_gpio_irq_handler(struct irq_desc *desc)
 	edge_cause = mvebu_gpio_read_edge_cause(mvchip);
 	edge_mask  = mvebu_gpio_read_edge_mask(mvchip);
 
-	cause = (data_in ^ level_mask) | (edge_cause & edge_mask);
+	cause = (data_in & level_mask) | (edge_cause & edge_mask);
 
 	for (i = 0; i < mvchip->chip.ngpio; i++) {
 		int irq;
diff --git a/drivers/gpio/gpio-mxc.c b/drivers/gpio/gpio-mxc.c
index 92692251ade1..5245a2fe62ae 100644
--- a/drivers/gpio/gpio-mxc.c
+++ b/drivers/gpio/gpio-mxc.c
@@ -66,6 +66,7 @@ struct mxc_gpio_port {
 	int irq_high;
 	struct irq_domain *domain;
 	struct gpio_chip gc;
+	struct device *dev;
 	u32 both_edges;
 };
 
@@ -324,29 +325,31 @@ static int gpio_set_wake_irq(struct irq_data *d, u32 enable)
 	struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d);
 	struct mxc_gpio_port *port = gc->private;
 	u32 gpio_idx = d->hwirq;
+	int ret;
 
 	if (enable) {
 		if (port->irq_high && (gpio_idx >= 16))
-			enable_irq_wake(port->irq_high);
+			ret = enable_irq_wake(port->irq_high);
 		else
-			enable_irq_wake(port->irq);
+			ret = enable_irq_wake(port->irq);
 	} else {
 		if (port->irq_high && (gpio_idx >= 16))
-			disable_irq_wake(port->irq_high);
+			ret = disable_irq_wake(port->irq_high);
 		else
-			disable_irq_wake(port->irq);
+			ret = disable_irq_wake(port->irq);
 	}
 
-	return 0;
+	return ret;
 }
 
 static int mxc_gpio_init_gc(struct mxc_gpio_port *port, int irq_base)
 {
 	struct irq_chip_generic *gc;
 	struct irq_chip_type *ct;
+	int rv;
 
-	gc = irq_alloc_generic_chip("gpio-mxc", 1, irq_base,
-				    port->base, handle_level_irq);
+	gc = devm_irq_alloc_generic_chip(port->dev, "gpio-mxc", 1, irq_base,
+					 port->base, handle_level_irq);
 	if (!gc)
 		return -ENOMEM;
 	gc->private = port;
@@ -361,10 +364,11 @@ static int mxc_gpio_init_gc(struct mxc_gpio_port *port, int irq_base)
 	ct->regs.ack = GPIO_ISR;
 	ct->regs.mask = GPIO_IMR;
 
-	irq_setup_generic_chip(gc, IRQ_MSK(32), IRQ_GC_INIT_NESTED_LOCK,
-			       IRQ_NOREQUEST, 0);
+	rv = devm_irq_setup_generic_chip(port->dev, gc, IRQ_MSK(32),
+					 IRQ_GC_INIT_NESTED_LOCK,
+					 IRQ_NOREQUEST, 0);
 
-	return 0;
+	return rv;
 }
 
 static void mxc_gpio_get_hw(struct platform_device *pdev)
@@ -418,6 +422,8 @@ static int mxc_gpio_probe(struct platform_device *pdev)
 	if (!port)
 		return -ENOMEM;
 
+	port->dev = &pdev->dev;
+
 	iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	port->base = devm_ioremap_resource(&pdev->dev, iores);
 	if (IS_ERR(port->base))
@@ -507,6 +513,7 @@ static struct platform_driver mxc_gpio_driver = {
 	.driver		= {
 		.name	= "gpio-mxc",
 		.of_match_table = mxc_gpio_dt_ids,
+		.suppress_bind_attrs = true,
 	},
 	.probe		= mxc_gpio_probe,
 	.id_table	= mxc_gpio_devtype,
diff --git a/drivers/gpio/gpio-mxs.c b/drivers/gpio/gpio-mxs.c
index 6ae583f36733..435def22445d 100644
--- a/drivers/gpio/gpio-mxs.c
+++ b/drivers/gpio/gpio-mxs.c
@@ -66,6 +66,7 @@ struct mxs_gpio_port {
 	int irq;
 	struct irq_domain *domain;
 	struct gpio_chip gc;
+	struct device *dev;
 	enum mxs_gpio_id devid;
 	u32 both_edges;
 };
@@ -209,9 +210,10 @@ static int mxs_gpio_init_gc(struct mxs_gpio_port *port, int irq_base)
 {
 	struct irq_chip_generic *gc;
 	struct irq_chip_type *ct;
+	int rv;
 
-	gc = irq_alloc_generic_chip("gpio-mxs", 2, irq_base,
-				    port->base, handle_level_irq);
+	gc = devm_irq_alloc_generic_chip(port->dev, "gpio-mxs", 2, irq_base,
+					 port->base, handle_level_irq);
 	if (!gc)
 		return -ENOMEM;
 
@@ -242,10 +244,11 @@ static int mxs_gpio_init_gc(struct mxs_gpio_port *port, int irq_base)
 	ct->regs.disable = PINCTRL_IRQEN(port) + MXS_CLR;
 	ct->handler = handle_level_irq;
 
-	irq_setup_generic_chip(gc, IRQ_MSK(32), IRQ_GC_INIT_NESTED_LOCK,
-			       IRQ_NOREQUEST, 0);
+	rv = devm_irq_setup_generic_chip(port->dev, gc, IRQ_MSK(32),
+					 IRQ_GC_INIT_NESTED_LOCK,
+					 IRQ_NOREQUEST, 0);
 
-	return 0;
+	return rv;
 }
 
 static int mxs_gpio_to_irq(struct gpio_chip *gc, unsigned offset)
@@ -304,6 +307,7 @@ static int mxs_gpio_probe(struct platform_device *pdev)
 	if (port->id < 0)
 		return port->id;
 	port->devid = (enum mxs_gpio_id) of_id->data;
+	port->dev = &pdev->dev;
 	port->irq = platform_get_irq(pdev, 0);
 	if (port->irq < 0)
 		return port->irq;
@@ -379,6 +383,7 @@ static struct platform_driver mxs_gpio_driver = {
 	.driver		= {
 		.name	= "gpio-mxs",
 		.of_match_table = mxs_gpio_dt_ids,
+		.suppress_bind_attrs = true,
 	},
 	.probe		= mxs_gpio_probe,
 	.id_table	= mxs_gpio_ids,
diff --git a/drivers/gpio/gpio-omap.c b/drivers/gpio/gpio-omap.c
index f8c550de6c72..dbf869fb63ce 100644
--- a/drivers/gpio/gpio-omap.c
+++ b/drivers/gpio/gpio-omap.c
@@ -1247,6 +1247,8 @@ static int omap_gpio_probe(struct platform_device *pdev)
 	if (ret) {
 		pm_runtime_put_sync(dev);
 		pm_runtime_disable(dev);
+		if (bank->dbck_flag)
+			clk_unprepare(bank->dbck);
 		return ret;
 	}
 
diff --git a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c
index 4c9e21300a26..1b9dbf691ae7 100644
--- a/drivers/gpio/gpio-pca953x.c
+++ b/drivers/gpio/gpio-pca953x.c
@@ -187,10 +187,9 @@ static int pca953x_write_regs_8(struct pca953x_chip *chip, int reg, u8 *val)
 
 static int pca953x_write_regs_16(struct pca953x_chip *chip, int reg, u8 *val)
 {
-	__le16 word = cpu_to_le16(get_unaligned((u16 *)val));
+	u16 word = get_unaligned((u16 *)val);
 
-	return i2c_smbus_write_word_data(chip->client,
-					 reg << 1, (__force u16)word);
+	return i2c_smbus_write_word_data(chip->client, reg << 1, word);
 }
 
 static int pca957x_write_regs_16(struct pca953x_chip *chip, int reg, u8 *val)
@@ -241,8 +240,7 @@ static int pca953x_read_regs_16(struct pca953x_chip *chip, int reg, u8 *val)
 	int ret;
 
 	ret = i2c_smbus_read_word_data(chip->client, reg << 1);
-	val[0] = (u16)ret & 0xFF;
-	val[1] = (u16)ret >> 8;
+	put_unaligned(ret, (u16 *)val);
 
 	return ret;
 }
diff --git a/drivers/gpio/gpio-pch.c b/drivers/gpio/gpio-pch.c
index f6600f8ada52..68c6d0c5a6d1 100644
--- a/drivers/gpio/gpio-pch.c
+++ b/drivers/gpio/gpio-pch.c
@@ -337,9 +337,10 @@ static int pch_gpio_alloc_generic_chip(struct pch_gpio *chip,
 {
 	struct irq_chip_generic *gc;
 	struct irq_chip_type *ct;
+	int rv;
 
-	gc = irq_alloc_generic_chip("pch_gpio", 1, irq_start, chip->base,
-				    handle_simple_irq);
+	gc = devm_irq_alloc_generic_chip(chip->dev, "pch_gpio", 1, irq_start,
+					 chip->base, handle_simple_irq);
 	if (!gc)
 		return -ENOMEM;
 
@@ -351,10 +352,11 @@ static int pch_gpio_alloc_generic_chip(struct pch_gpio *chip,
 	ct->chip.irq_unmask = pch_irq_unmask;
 	ct->chip.irq_set_type = pch_irq_type;
 
-	irq_setup_generic_chip(gc, IRQ_MSK(num), IRQ_GC_INIT_MASK_CACHE,
-			       IRQ_NOREQUEST | IRQ_NOPROBE, 0);
+	rv = devm_irq_setup_generic_chip(chip->dev, gc, IRQ_MSK(num),
+					 IRQ_GC_INIT_MASK_CACHE,
+					 IRQ_NOREQUEST | IRQ_NOPROBE, 0);
 
-	return 0;
+	return rv;
 }
 
 static int pch_gpio_probe(struct pci_dev *pdev,
diff --git a/drivers/gpio/gpio-pl061.c b/drivers/gpio/gpio-pl061.c
index 3d3d6b6645a7..6aaaab79c205 100644
--- a/drivers/gpio/gpio-pl061.c
+++ b/drivers/gpio/gpio-pl061.c
@@ -405,7 +405,7 @@ static const struct dev_pm_ops pl061_dev_pm_ops = {
 };
 #endif
 
-static struct amba_id pl061_ids[] = {
+static const struct amba_id pl061_ids[] = {
 	{
 		.id	= 0x00041061,
 		.mask	= 0x000fffff,
diff --git a/drivers/gpio/gpio-pxa.c b/drivers/gpio/gpio-pxa.c
index 832f3e46ba9f..6029899789f3 100644
--- a/drivers/gpio/gpio-pxa.c
+++ b/drivers/gpio/gpio-pxa.c
@@ -451,7 +451,9 @@ static irqreturn_t pxa_gpio_demux_handler(int in_irq, void *d)
 			for_each_set_bit(n, &gedr, BITS_PER_LONG) {
 				loop = 1;
 
-				generic_handle_irq(gpio_to_irq(gpio + n));
+				generic_handle_irq(
+					irq_find_mapping(pchip->irqdomain,
+							 gpio + n));
 			}
 		}
 		handled += loop;
@@ -465,9 +467,9 @@ static irqreturn_t pxa_gpio_direct_handler(int in_irq, void *d)
 	struct pxa_gpio_chip *pchip = d;
 
 	if (in_irq == pchip->irq0) {
-		generic_handle_irq(gpio_to_irq(0));
+		generic_handle_irq(irq_find_mapping(pchip->irqdomain, 0));
 	} else if (in_irq == pchip->irq1) {
-		generic_handle_irq(gpio_to_irq(1));
+		generic_handle_irq(irq_find_mapping(pchip->irqdomain, 1));
 	} else {
 		pr_err("%s() unknown irq %d\n", __func__, in_irq);
 		return IRQ_NONE;
diff --git a/drivers/gpio/gpio-rcar.c b/drivers/gpio/gpio-rcar.c
index 4a1536a050bc..1f0871553fd2 100644
--- a/drivers/gpio/gpio-rcar.c
+++ b/drivers/gpio/gpio-rcar.c
@@ -371,6 +371,16 @@ static const struct of_device_id gpio_rcar_of_table[] = {
 		/* Gen3 GPIO is identical to Gen2. */
 		.data = &gpio_rcar_info_gen2,
 	}, {
+		.compatible = "renesas,rcar-gen1-gpio",
+		.data = &gpio_rcar_info_gen1,
+	}, {
+		.compatible = "renesas,rcar-gen2-gpio",
+		.data = &gpio_rcar_info_gen2,
+	}, {
+		.compatible = "renesas,rcar-gen3-gpio",
+		/* Gen3 GPIO is identical to Gen2. */
+		.data = &gpio_rcar_info_gen2,
+	}, {
 		.compatible = "renesas,gpio-rcar",
 		.data = &gpio_rcar_info_gen1,
 	}, {
diff --git a/drivers/gpio/gpio-sta2x11.c b/drivers/gpio/gpio-sta2x11.c
index 9e705162da8d..407359da08f9 100644
--- a/drivers/gpio/gpio-sta2x11.c
+++ b/drivers/gpio/gpio-sta2x11.c
@@ -324,9 +324,11 @@ static int gsta_alloc_irq_chip(struct gsta_gpio *chip)
 {
 	struct irq_chip_generic *gc;
 	struct irq_chip_type *ct;
+	int rv;
 
-	gc = irq_alloc_generic_chip(KBUILD_MODNAME, 1, chip->irq_base,
-				     chip->reg_base, handle_simple_irq);
+	gc = devm_irq_alloc_generic_chip(chip->dev, KBUILD_MODNAME, 1,
+					 chip->irq_base,
+					 chip->reg_base, handle_simple_irq);
 	if (!gc)
 		return -ENOMEM;
 
@@ -338,8 +340,11 @@ static int gsta_alloc_irq_chip(struct gsta_gpio *chip)
 	ct->chip.irq_enable = gsta_irq_enable;
 
 	/* FIXME: this makes at most 32 interrupts. Request 0 by now */
-	irq_setup_generic_chip(gc, 0 /* IRQ_MSK(GSTA_GPIO_PER_BLOCK) */, 0,
-			       IRQ_NOREQUEST | IRQ_NOPROBE, 0);
+	rv = devm_irq_setup_generic_chip(chip->dev, gc,
+					 0 /* IRQ_MSK(GSTA_GPIO_PER_BLOCK) */,
+					 0, IRQ_NOREQUEST | IRQ_NOPROBE, 0);
+	if (rv)
+		return rv;
 
 	/* Set up all all 128 interrupts: code from setup_generic_chip */
 	{
@@ -432,6 +437,7 @@ static int gsta_probe(struct platform_device *dev)
 static struct platform_driver sta2x11_gpio_platform_driver = {
 	.driver = {
 		.name	= "sta2x11-gpio",
+		.suppress_bind_attrs = true,
 	},
 	.probe = gsta_probe,
 };
diff --git a/drivers/gpio/gpio-tb10x.c b/drivers/gpio/gpio-tb10x.c
index 80b6959ae995..091ffaaec635 100644
--- a/drivers/gpio/gpio-tb10x.c
+++ b/drivers/gpio/gpio-tb10x.c
@@ -191,7 +191,8 @@ static int tb10x_gpio_probe(struct platform_device *pdev)
 	if (IS_ERR(tb10x_gpio->base))
 		return PTR_ERR(tb10x_gpio->base);
 
-	tb10x_gpio->gc.label		= of_node_full_name(dn);
+	tb10x_gpio->gc.label		=
+		devm_kasprintf(&pdev->dev, GFP_KERNEL, "%pOF", pdev->dev.of_node);
 	tb10x_gpio->gc.parent		= &pdev->dev;
 	tb10x_gpio->gc.owner		= THIS_MODULE;
 	tb10x_gpio->gc.direction_input	= tb10x_gpio_direction_in;
diff --git a/drivers/gpio/gpio-tegra.c b/drivers/gpio/gpio-tegra.c
index 506c6a67c5fc..fbaf974277df 100644
--- a/drivers/gpio/gpio-tegra.c
+++ b/drivers/gpio/gpio-tegra.c
@@ -67,8 +67,8 @@
 struct tegra_gpio_info;
 
 struct tegra_gpio_bank {
-	int bank;
-	int irq;
+	unsigned int bank;
+	unsigned int irq;
 	spinlock_t lvl_lock[4];
 	spinlock_t dbc_lock[4];	/* Lock for updating debounce count register */
 #ifdef CONFIG_PM_SLEEP
@@ -112,13 +112,14 @@ static inline u32 tegra_gpio_readl(struct tegra_gpio_info *tgi, u32 reg)
 	return __raw_readl(tgi->regs + reg);
 }
 
-static int tegra_gpio_compose(int bank, int port, int bit)
+static unsigned int tegra_gpio_compose(unsigned int bank, unsigned int port,
+				       unsigned int bit)
 {
 	return (bank << 5) | ((port & 0x3) << 3) | (bit & 0x7);
 }
 
 static void tegra_gpio_mask_write(struct tegra_gpio_info *tgi, u32 reg,
-				  int gpio, int value)
+				  unsigned int gpio, u32 value)
 {
 	u32 val;
 
@@ -128,22 +129,22 @@ static void tegra_gpio_mask_write(struct tegra_gpio_info *tgi, u32 reg,
 	tegra_gpio_writel(tgi, val, reg);
 }
 
-static void tegra_gpio_enable(struct tegra_gpio_info *tgi, int gpio)
+static void tegra_gpio_enable(struct tegra_gpio_info *tgi, unsigned int gpio)
 {
 	tegra_gpio_mask_write(tgi, GPIO_MSK_CNF(tgi, gpio), gpio, 1);
 }
 
-static void tegra_gpio_disable(struct tegra_gpio_info *tgi, int gpio)
+static void tegra_gpio_disable(struct tegra_gpio_info *tgi, unsigned int gpio)
 {
 	tegra_gpio_mask_write(tgi, GPIO_MSK_CNF(tgi, gpio), gpio, 0);
 }
 
-static int tegra_gpio_request(struct gpio_chip *chip, unsigned offset)
+static int tegra_gpio_request(struct gpio_chip *chip, unsigned int offset)
 {
 	return pinctrl_request_gpio(offset);
 }
 
-static void tegra_gpio_free(struct gpio_chip *chip, unsigned offset)
+static void tegra_gpio_free(struct gpio_chip *chip, unsigned int offset)
 {
 	struct tegra_gpio_info *tgi = gpiochip_get_data(chip);
 
@@ -151,17 +152,18 @@ static void tegra_gpio_free(struct gpio_chip *chip, unsigned offset)
 	tegra_gpio_disable(tgi, offset);
 }
 
-static void tegra_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
+static void tegra_gpio_set(struct gpio_chip *chip, unsigned int offset,
+			   int value)
 {
 	struct tegra_gpio_info *tgi = gpiochip_get_data(chip);
 
 	tegra_gpio_mask_write(tgi, GPIO_MSK_OUT(tgi, offset), offset, value);
 }
 
-static int tegra_gpio_get(struct gpio_chip *chip, unsigned offset)
+static int tegra_gpio_get(struct gpio_chip *chip, unsigned int offset)
 {
 	struct tegra_gpio_info *tgi = gpiochip_get_data(chip);
-	int bval = BIT(GPIO_BIT(offset));
+	unsigned int bval = BIT(GPIO_BIT(offset));
 
 	/* If gpio is in output mode then read from the out value */
 	if (tegra_gpio_readl(tgi, GPIO_OE(tgi, offset)) & bval)
@@ -170,7 +172,8 @@ static int tegra_gpio_get(struct gpio_chip *chip, unsigned offset)
 	return !!(tegra_gpio_readl(tgi, GPIO_IN(tgi, offset)) & bval);
 }
 
-static int tegra_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
+static int tegra_gpio_direction_input(struct gpio_chip *chip,
+				      unsigned int offset)
 {
 	struct tegra_gpio_info *tgi = gpiochip_get_data(chip);
 
@@ -179,8 +182,9 @@ static int tegra_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 	return 0;
 }
 
-static int tegra_gpio_direction_output(struct gpio_chip *chip, unsigned offset,
-					int value)
+static int tegra_gpio_direction_output(struct gpio_chip *chip,
+				       unsigned int offset,
+				       int value)
 {
 	struct tegra_gpio_info *tgi = gpiochip_get_data(chip);
 
@@ -190,7 +194,8 @@ static int tegra_gpio_direction_output(struct gpio_chip *chip, unsigned offset,
 	return 0;
 }
 
-static int tegra_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
+static int tegra_gpio_get_direction(struct gpio_chip *chip,
+				    unsigned int offset)
 {
 	struct tegra_gpio_info *tgi = gpiochip_get_data(chip);
 	u32 pin_mask = BIT(GPIO_BIT(offset));
@@ -212,7 +217,7 @@ static int tegra_gpio_set_debounce(struct gpio_chip *chip, unsigned int offset,
 	struct tegra_gpio_bank *bank = &tgi->bank_info[GPIO_BANK(offset)];
 	unsigned int debounce_ms = DIV_ROUND_UP(debounce, 1000);
 	unsigned long flags;
-	int port;
+	unsigned int port;
 
 	if (!debounce_ms) {
 		tegra_gpio_mask_write(tgi, GPIO_MSK_DBC_EN(tgi, offset),
@@ -250,7 +255,7 @@ static int tegra_gpio_set_config(struct gpio_chip *chip, unsigned int offset,
 	return tegra_gpio_set_debounce(chip, offset, debounce);
 }
 
-static int tegra_gpio_to_irq(struct gpio_chip *chip, unsigned offset)
+static int tegra_gpio_to_irq(struct gpio_chip *chip, unsigned int offset)
 {
 	struct tegra_gpio_info *tgi = gpiochip_get_data(chip);
 
@@ -261,7 +266,7 @@ static void tegra_gpio_irq_ack(struct irq_data *d)
 {
 	struct tegra_gpio_bank *bank = irq_data_get_irq_chip_data(d);
 	struct tegra_gpio_info *tgi = bank->tgi;
-	int gpio = d->hwirq;
+	unsigned int gpio = d->hwirq;
 
 	tegra_gpio_writel(tgi, 1 << GPIO_BIT(gpio), GPIO_INT_CLR(tgi, gpio));
 }
@@ -270,7 +275,7 @@ static void tegra_gpio_irq_mask(struct irq_data *d)
 {
 	struct tegra_gpio_bank *bank = irq_data_get_irq_chip_data(d);
 	struct tegra_gpio_info *tgi = bank->tgi;
-	int gpio = d->hwirq;
+	unsigned int gpio = d->hwirq;
 
 	tegra_gpio_mask_write(tgi, GPIO_MSK_INT_ENB(tgi, gpio), gpio, 0);
 }
@@ -279,20 +284,18 @@ static void tegra_gpio_irq_unmask(struct irq_data *d)
 {
 	struct tegra_gpio_bank *bank = irq_data_get_irq_chip_data(d);
 	struct tegra_gpio_info *tgi = bank->tgi;
-	int gpio = d->hwirq;
+	unsigned int gpio = d->hwirq;
 
 	tegra_gpio_mask_write(tgi, GPIO_MSK_INT_ENB(tgi, gpio), gpio, 1);
 }
 
 static int tegra_gpio_irq_set_type(struct irq_data *d, unsigned int type)
 {
-	int gpio = d->hwirq;
+	unsigned int gpio = d->hwirq, port = GPIO_PORT(gpio), lvl_type;
 	struct tegra_gpio_bank *bank = irq_data_get_irq_chip_data(d);
 	struct tegra_gpio_info *tgi = bank->tgi;
-	int port = GPIO_PORT(gpio);
-	int lvl_type;
-	int val;
 	unsigned long flags;
+	u32 val;
 	int ret;
 
 	switch (type & IRQ_TYPE_SENSE_MASK) {
@@ -323,7 +326,7 @@ static int tegra_gpio_irq_set_type(struct irq_data *d, unsigned int type)
 	ret = gpiochip_lock_as_irq(&tgi->gc, gpio);
 	if (ret) {
 		dev_err(tgi->dev,
-			"unable to lock Tegra GPIO %d as IRQ\n", gpio);
+			"unable to lock Tegra GPIO %u as IRQ\n", gpio);
 		return ret;
 	}
 
@@ -351,17 +354,15 @@ static void tegra_gpio_irq_shutdown(struct irq_data *d)
 {
 	struct tegra_gpio_bank *bank = irq_data_get_irq_chip_data(d);
 	struct tegra_gpio_info *tgi = bank->tgi;
-	int gpio = d->hwirq;
+	unsigned int gpio = d->hwirq;
 
 	gpiochip_unlock_as_irq(&tgi->gc, gpio);
 }
 
 static void tegra_gpio_irq_handler(struct irq_desc *desc)
 {
-	int port;
-	int pin;
+	unsigned int port, pin, gpio;
 	bool unmasked = false;
-	int gpio;
 	u32 lvl;
 	unsigned long sta;
 	struct irq_chip *chip = irq_desc_get_chip(desc);
@@ -389,7 +390,8 @@ static void tegra_gpio_irq_handler(struct irq_desc *desc)
 				chained_irq_exit(chip, desc);
 			}
 
-			generic_handle_irq(gpio_to_irq(gpio + pin));
+			generic_handle_irq(irq_find_mapping(tgi->irq_domain,
+							    gpio + pin));
 		}
 	}
 
@@ -404,8 +406,7 @@ static int tegra_gpio_resume(struct device *dev)
 	struct platform_device *pdev = to_platform_device(dev);
 	struct tegra_gpio_info *tgi = platform_get_drvdata(pdev);
 	unsigned long flags;
-	int b;
-	int p;
+	unsigned int b, p;
 
 	local_irq_save(flags);
 
@@ -413,7 +414,8 @@ static int tegra_gpio_resume(struct device *dev)
 		struct tegra_gpio_bank *bank = &tgi->bank_info[b];
 
 		for (p = 0; p < ARRAY_SIZE(bank->oe); p++) {
-			unsigned int gpio = (b<<5) | (p<<3);
+			unsigned int gpio = (b << 5) | (p << 3);
+
 			tegra_gpio_writel(tgi, bank->cnf[p],
 					  GPIO_CNF(tgi, gpio));
 
@@ -444,15 +446,15 @@ static int tegra_gpio_suspend(struct device *dev)
 	struct platform_device *pdev = to_platform_device(dev);
 	struct tegra_gpio_info *tgi = platform_get_drvdata(pdev);
 	unsigned long flags;
-	int b;
-	int p;
+	unsigned int b, p;
 
 	local_irq_save(flags);
 	for (b = 0; b < tgi->bank_count; b++) {
 		struct tegra_gpio_bank *bank = &tgi->bank_info[b];
 
 		for (p = 0; p < ARRAY_SIZE(bank->oe); p++) {
-			unsigned int gpio = (b<<5) | (p<<3);
+			unsigned int gpio = (b << 5) | (p << 3);
+
 			bank->cnf[p] = tegra_gpio_readl(tgi,
 							GPIO_CNF(tgi, gpio));
 			bank->out[p] = tegra_gpio_readl(tgi,
@@ -483,7 +485,7 @@ static int tegra_gpio_suspend(struct device *dev)
 static int tegra_gpio_irq_set_wake(struct irq_data *d, unsigned int enable)
 {
 	struct tegra_gpio_bank *bank = irq_data_get_irq_chip_data(d);
-	int gpio = d->hwirq;
+	unsigned int gpio = d->hwirq;
 	u32 port, bit, mask;
 
 	port = GPIO_PORT(gpio);
@@ -507,14 +509,14 @@ static int tegra_gpio_irq_set_wake(struct irq_data *d, unsigned int enable)
 static int dbg_gpio_show(struct seq_file *s, void *unused)
 {
 	struct tegra_gpio_info *tgi = s->private;
-	int i;
-	int j;
+	unsigned int i, j;
 
 	for (i = 0; i < tgi->bank_count; i++) {
 		for (j = 0; j < 4; j++) {
-			int gpio = tegra_gpio_compose(i, j, 0);
+			unsigned int gpio = tegra_gpio_compose(i, j, 0);
+
 			seq_printf(s,
-				"%d:%d %02x %02x %02x %02x %02x %02x %06x\n",
+				"%u:%u %02x %02x %02x %02x %02x %02x %06x\n",
 				i, j,
 				tegra_gpio_readl(tgi, GPIO_CNF(tgi, gpio)),
 				tegra_gpio_readl(tgi, GPIO_OE(tgi, gpio)),
@@ -542,7 +544,7 @@ static const struct file_operations debug_fops = {
 
 static void tegra_gpio_debuginit(struct tegra_gpio_info *tgi)
 {
-	(void) debugfs_create_file("tegra_gpio", S_IRUGO,
+	(void) debugfs_create_file("tegra_gpio", 0444,
 					NULL, tgi, &debug_fops);
 }
 
@@ -566,35 +568,25 @@ static struct lock_class_key gpio_lock_class;
 
 static int tegra_gpio_probe(struct platform_device *pdev)
 {
-	const struct tegra_gpio_soc_config *config;
 	struct tegra_gpio_info *tgi;
 	struct resource *res;
 	struct tegra_gpio_bank *bank;
+	unsigned int gpio, i, j;
 	int ret;
-	int gpio;
-	int i;
-	int j;
-
-	config = of_device_get_match_data(&pdev->dev);
-	if (!config) {
-		dev_err(&pdev->dev, "Error: No device match found\n");
-		return -ENODEV;
-	}
 
 	tgi = devm_kzalloc(&pdev->dev, sizeof(*tgi), GFP_KERNEL);
 	if (!tgi)
 		return -ENODEV;
 
-	tgi->soc = config;
+	tgi->soc = of_device_get_match_data(&pdev->dev);
 	tgi->dev = &pdev->dev;
 
-	for (;;) {
-		res = platform_get_resource(pdev, IORESOURCE_IRQ,
-					    tgi->bank_count);
-		if (!res)
-			break;
-		tgi->bank_count++;
-	}
+	ret = platform_irq_count(pdev);
+	if (ret < 0)
+		return ret;
+
+	tgi->bank_count = ret;
+
 	if (!tgi->bank_count) {
 		dev_err(&pdev->dev, "Missing IRQ resource\n");
 		return -ENODEV;
@@ -626,13 +618,13 @@ static int tegra_gpio_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, tgi);
 
-	if (config->debounce_supported)
+	if (tgi->soc->debounce_supported)
 		tgi->gc.set_config = tegra_gpio_set_config;
 
-	tgi->bank_info = devm_kzalloc(&pdev->dev, tgi->bank_count *
+	tgi->bank_info = devm_kcalloc(&pdev->dev, tgi->bank_count,
 				      sizeof(*tgi->bank_info), GFP_KERNEL);
 	if (!tgi->bank_info)
-		return -ENODEV;
+		return -ENOMEM;
 
 	tgi->irq_domain = irq_domain_add_linear(pdev->dev.of_node,
 						tgi->gc.ngpio,
@@ -641,15 +633,15 @@ static int tegra_gpio_probe(struct platform_device *pdev)
 		return -ENODEV;
 
 	for (i = 0; i < tgi->bank_count; i++) {
-		res = platform_get_resource(pdev, IORESOURCE_IRQ, i);
-		if (!res) {
-			dev_err(&pdev->dev, "Missing IRQ resource\n");
-			return -ENODEV;
+		ret = platform_get_irq(pdev, i);
+		if (ret < 0) {
+			dev_err(&pdev->dev, "Missing IRQ resource: %d\n", ret);
+			return ret;
 		}
 
 		bank = &tgi->bank_info[i];
 		bank->bank = i;
-		bank->irq = res->start;
+		bank->irq = ret;
 		bank->tgi = tgi;
 	}
 
@@ -661,6 +653,7 @@ static int tegra_gpio_probe(struct platform_device *pdev)
 	for (i = 0; i < tgi->bank_count; i++) {
 		for (j = 0; j < 4; j++) {
 			int gpio = tegra_gpio_compose(i, j, 0);
+
 			tegra_gpio_writel(tgi, 0x00, GPIO_INT_ENB(tgi, gpio));
 		}
 	}
diff --git a/drivers/gpio/gpio-thunderx.c b/drivers/gpio/gpio-thunderx.c
new file mode 100644
index 000000000000..57efb251f9c4
--- /dev/null
+++ b/drivers/gpio/gpio-thunderx.c
@@ -0,0 +1,639 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 2016, 2017 Cavium Inc.
+ */
+
+#include <linux/bitops.h>
+#include <linux/gpio/driver.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/irq.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/spinlock.h>
+
+
+#define GPIO_RX_DAT	0x0
+#define GPIO_TX_SET	0x8
+#define GPIO_TX_CLR	0x10
+#define GPIO_CONST	0x90
+#define  GPIO_CONST_GPIOS_MASK 0xff
+#define GPIO_BIT_CFG	0x400
+#define  GPIO_BIT_CFG_TX_OE		BIT(0)
+#define  GPIO_BIT_CFG_PIN_XOR		BIT(1)
+#define  GPIO_BIT_CFG_INT_EN		BIT(2)
+#define  GPIO_BIT_CFG_INT_TYPE		BIT(3)
+#define  GPIO_BIT_CFG_FIL_MASK		GENMASK(11, 4)
+#define  GPIO_BIT_CFG_FIL_CNT_SHIFT	4
+#define  GPIO_BIT_CFG_FIL_SEL_SHIFT	8
+#define  GPIO_BIT_CFG_TX_OD		BIT(12)
+#define  GPIO_BIT_CFG_PIN_SEL_MASK	GENMASK(25, 16)
+#define GPIO_INTR	0x800
+#define  GPIO_INTR_INTR			BIT(0)
+#define  GPIO_INTR_INTR_W1S		BIT(1)
+#define  GPIO_INTR_ENA_W1C		BIT(2)
+#define  GPIO_INTR_ENA_W1S		BIT(3)
+#define GPIO_2ND_BANK	0x1400
+
+#define GLITCH_FILTER_400NS ((4u << GPIO_BIT_CFG_FIL_SEL_SHIFT) | \
+			     (9u << GPIO_BIT_CFG_FIL_CNT_SHIFT))
+
+struct thunderx_gpio;
+
+struct thunderx_line {
+	struct thunderx_gpio	*txgpio;
+	unsigned int		line;
+	unsigned int		fil_bits;
+};
+
+struct thunderx_gpio {
+	struct gpio_chip	chip;
+	u8 __iomem		*register_base;
+	struct irq_domain	*irqd;
+	struct msix_entry	*msix_entries;	/* per line MSI-X */
+	struct thunderx_line	*line_entries;	/* per line irq info */
+	raw_spinlock_t		lock;
+	unsigned long		invert_mask[2];
+	unsigned long		od_mask[2];
+	int			base_msi;
+};
+
+static unsigned int bit_cfg_reg(unsigned int line)
+{
+	return 8 * line + GPIO_BIT_CFG;
+}
+
+static unsigned int intr_reg(unsigned int line)
+{
+	return 8 * line + GPIO_INTR;
+}
+
+static bool thunderx_gpio_is_gpio_nowarn(struct thunderx_gpio *txgpio,
+					 unsigned int line)
+{
+	u64 bit_cfg = readq(txgpio->register_base + bit_cfg_reg(line));
+
+	return (bit_cfg & GPIO_BIT_CFG_PIN_SEL_MASK) == 0;
+}
+
+/*
+ * Check (and WARN) that the pin is available for GPIO.  We will not
+ * allow modification of the state of non-GPIO pins from this driver.
+ */
+static bool thunderx_gpio_is_gpio(struct thunderx_gpio *txgpio,
+				  unsigned int line)
+{
+	bool rv = thunderx_gpio_is_gpio_nowarn(txgpio, line);
+
+	WARN_RATELIMIT(!rv, "Pin %d not available for GPIO\n", line);
+
+	return rv;
+}
+
+static int thunderx_gpio_request(struct gpio_chip *chip, unsigned int line)
+{
+	struct thunderx_gpio *txgpio = gpiochip_get_data(chip);
+
+	return thunderx_gpio_is_gpio(txgpio, line) ? 0 : -EIO;
+}
+
+static int thunderx_gpio_dir_in(struct gpio_chip *chip, unsigned int line)
+{
+	struct thunderx_gpio *txgpio = gpiochip_get_data(chip);
+
+	if (!thunderx_gpio_is_gpio(txgpio, line))
+		return -EIO;
+
+	raw_spin_lock(&txgpio->lock);
+	clear_bit(line, txgpio->invert_mask);
+	clear_bit(line, txgpio->od_mask);
+	writeq(txgpio->line_entries[line].fil_bits,
+	       txgpio->register_base + bit_cfg_reg(line));
+	raw_spin_unlock(&txgpio->lock);
+	return 0;
+}
+
+static void thunderx_gpio_set(struct gpio_chip *chip, unsigned int line,
+			      int value)
+{
+	struct thunderx_gpio *txgpio = gpiochip_get_data(chip);
+	int bank = line / 64;
+	int bank_bit = line % 64;
+
+	void __iomem *reg = txgpio->register_base +
+		(bank * GPIO_2ND_BANK) + (value ? GPIO_TX_SET : GPIO_TX_CLR);
+
+	writeq(BIT_ULL(bank_bit), reg);
+}
+
+static int thunderx_gpio_dir_out(struct gpio_chip *chip, unsigned int line,
+				 int value)
+{
+	struct thunderx_gpio *txgpio = gpiochip_get_data(chip);
+	u64 bit_cfg = txgpio->line_entries[line].fil_bits | GPIO_BIT_CFG_TX_OE;
+
+	if (!thunderx_gpio_is_gpio(txgpio, line))
+		return -EIO;
+
+	raw_spin_lock(&txgpio->lock);
+
+	thunderx_gpio_set(chip, line, value);
+
+	if (test_bit(line, txgpio->invert_mask))
+		bit_cfg |= GPIO_BIT_CFG_PIN_XOR;
+
+	if (test_bit(line, txgpio->od_mask))
+		bit_cfg |= GPIO_BIT_CFG_TX_OD;
+
+	writeq(bit_cfg, txgpio->register_base + bit_cfg_reg(line));
+
+	raw_spin_unlock(&txgpio->lock);
+	return 0;
+}
+
+static int thunderx_gpio_get_direction(struct gpio_chip *chip, unsigned int line)
+{
+	struct thunderx_gpio *txgpio = gpiochip_get_data(chip);
+	u64 bit_cfg;
+
+	if (!thunderx_gpio_is_gpio_nowarn(txgpio, line))
+		/*
+		 * Say it is input for now to avoid WARNing on
+		 * gpiochip_add_data().  We will WARN if someone
+		 * requests it or tries to use it.
+		 */
+		return 1;
+
+	bit_cfg = readq(txgpio->register_base + bit_cfg_reg(line));
+
+	return !(bit_cfg & GPIO_BIT_CFG_TX_OE);
+}
+
+static int thunderx_gpio_set_config(struct gpio_chip *chip,
+				    unsigned int line,
+				    unsigned long cfg)
+{
+	bool orig_invert, orig_od, orig_dat, new_invert, new_od;
+	u32 arg, sel;
+	u64 bit_cfg;
+	int bank = line / 64;
+	int bank_bit = line % 64;
+	int ret = -ENOTSUPP;
+	struct thunderx_gpio *txgpio = gpiochip_get_data(chip);
+	void __iomem *reg = txgpio->register_base + (bank * GPIO_2ND_BANK) + GPIO_TX_SET;
+
+	if (!thunderx_gpio_is_gpio(txgpio, line))
+		return -EIO;
+
+	raw_spin_lock(&txgpio->lock);
+	orig_invert = test_bit(line, txgpio->invert_mask);
+	new_invert  = orig_invert;
+	orig_od = test_bit(line, txgpio->od_mask);
+	new_od = orig_od;
+	orig_dat = ((readq(reg) >> bank_bit) & 1) ^ orig_invert;
+	bit_cfg = readq(txgpio->register_base + bit_cfg_reg(line));
+	switch (pinconf_to_config_param(cfg)) {
+	case PIN_CONFIG_DRIVE_OPEN_DRAIN:
+		/*
+		 * Weird, setting open-drain mode causes signal
+		 * inversion.  Note this so we can compensate in the
+		 * dir_out function.
+		 */
+		set_bit(line, txgpio->invert_mask);
+		new_invert  = true;
+		set_bit(line, txgpio->od_mask);
+		new_od = true;
+		ret = 0;
+		break;
+	case PIN_CONFIG_DRIVE_PUSH_PULL:
+		clear_bit(line, txgpio->invert_mask);
+		new_invert  = false;
+		clear_bit(line, txgpio->od_mask);
+		new_od  = false;
+		ret = 0;
+		break;
+	case PIN_CONFIG_INPUT_DEBOUNCE:
+		arg = pinconf_to_config_argument(cfg);
+		if (arg > 1228) { /* 15 * 2^15 * 2.5nS maximum */
+			ret = -EINVAL;
+			break;
+		}
+		arg *= 400; /* scale to 2.5nS clocks. */
+		sel = 0;
+		while (arg > 15) {
+			sel++;
+			arg++; /* always round up */
+			arg >>= 1;
+		}
+		txgpio->line_entries[line].fil_bits =
+			(sel << GPIO_BIT_CFG_FIL_SEL_SHIFT) |
+			(arg << GPIO_BIT_CFG_FIL_CNT_SHIFT);
+		bit_cfg &= ~GPIO_BIT_CFG_FIL_MASK;
+		bit_cfg |= txgpio->line_entries[line].fil_bits;
+		writeq(bit_cfg, txgpio->register_base + bit_cfg_reg(line));
+		ret = 0;
+		break;
+	default:
+		break;
+	}
+	raw_spin_unlock(&txgpio->lock);
+
+	/*
+	 * If currently output and OPEN_DRAIN changed, install the new
+	 * settings
+	 */
+	if ((new_invert != orig_invert || new_od != orig_od) &&
+	    (bit_cfg & GPIO_BIT_CFG_TX_OE))
+		ret = thunderx_gpio_dir_out(chip, line, orig_dat ^ new_invert);
+
+	return ret;
+}
+
+static int thunderx_gpio_get(struct gpio_chip *chip, unsigned int line)
+{
+	struct thunderx_gpio *txgpio = gpiochip_get_data(chip);
+	int bank = line / 64;
+	int bank_bit = line % 64;
+	u64 read_bits = readq(txgpio->register_base + (bank * GPIO_2ND_BANK) + GPIO_RX_DAT);
+	u64 masked_bits = read_bits & BIT_ULL(bank_bit);
+
+	if (test_bit(line, txgpio->invert_mask))
+		return masked_bits == 0;
+	else
+		return masked_bits != 0;
+}
+
+static void thunderx_gpio_set_multiple(struct gpio_chip *chip,
+				       unsigned long *mask,
+				       unsigned long *bits)
+{
+	int bank;
+	u64 set_bits, clear_bits;
+	struct thunderx_gpio *txgpio = gpiochip_get_data(chip);
+
+	for (bank = 0; bank <= chip->ngpio / 64; bank++) {
+		set_bits = bits[bank] & mask[bank];
+		clear_bits = ~bits[bank] & mask[bank];
+		writeq(set_bits, txgpio->register_base + (bank * GPIO_2ND_BANK) + GPIO_TX_SET);
+		writeq(clear_bits, txgpio->register_base + (bank * GPIO_2ND_BANK) + GPIO_TX_CLR);
+	}
+}
+
+static void thunderx_gpio_irq_ack(struct irq_data *data)
+{
+	struct thunderx_line *txline = irq_data_get_irq_chip_data(data);
+
+	writeq(GPIO_INTR_INTR,
+	       txline->txgpio->register_base + intr_reg(txline->line));
+}
+
+static void thunderx_gpio_irq_mask(struct irq_data *data)
+{
+	struct thunderx_line *txline = irq_data_get_irq_chip_data(data);
+
+	writeq(GPIO_INTR_ENA_W1C,
+	       txline->txgpio->register_base + intr_reg(txline->line));
+}
+
+static void thunderx_gpio_irq_mask_ack(struct irq_data *data)
+{
+	struct thunderx_line *txline = irq_data_get_irq_chip_data(data);
+
+	writeq(GPIO_INTR_ENA_W1C | GPIO_INTR_INTR,
+	       txline->txgpio->register_base + intr_reg(txline->line));
+}
+
+static void thunderx_gpio_irq_unmask(struct irq_data *data)
+{
+	struct thunderx_line *txline = irq_data_get_irq_chip_data(data);
+
+	writeq(GPIO_INTR_ENA_W1S,
+	       txline->txgpio->register_base + intr_reg(txline->line));
+}
+
+static int thunderx_gpio_irq_set_type(struct irq_data *data,
+				      unsigned int flow_type)
+{
+	struct thunderx_line *txline = irq_data_get_irq_chip_data(data);
+	struct thunderx_gpio *txgpio = txline->txgpio;
+	u64 bit_cfg;
+
+	irqd_set_trigger_type(data, flow_type);
+
+	bit_cfg = txline->fil_bits | GPIO_BIT_CFG_INT_EN;
+
+	if (flow_type & IRQ_TYPE_EDGE_BOTH) {
+		irq_set_handler_locked(data, handle_fasteoi_ack_irq);
+		bit_cfg |= GPIO_BIT_CFG_INT_TYPE;
+	} else {
+		irq_set_handler_locked(data, handle_fasteoi_mask_irq);
+	}
+
+	raw_spin_lock(&txgpio->lock);
+	if (flow_type & (IRQ_TYPE_EDGE_FALLING | IRQ_TYPE_LEVEL_LOW)) {
+		bit_cfg |= GPIO_BIT_CFG_PIN_XOR;
+		set_bit(txline->line, txgpio->invert_mask);
+	} else {
+		clear_bit(txline->line, txgpio->invert_mask);
+	}
+	clear_bit(txline->line, txgpio->od_mask);
+	writeq(bit_cfg, txgpio->register_base + bit_cfg_reg(txline->line));
+	raw_spin_unlock(&txgpio->lock);
+
+	return IRQ_SET_MASK_OK;
+}
+
+static void thunderx_gpio_irq_enable(struct irq_data *data)
+{
+	irq_chip_enable_parent(data);
+	thunderx_gpio_irq_unmask(data);
+}
+
+static void thunderx_gpio_irq_disable(struct irq_data *data)
+{
+	thunderx_gpio_irq_mask(data);
+	irq_chip_disable_parent(data);
+}
+
+static int thunderx_gpio_irq_request_resources(struct irq_data *data)
+{
+	struct thunderx_line *txline = irq_data_get_irq_chip_data(data);
+	struct thunderx_gpio *txgpio = txline->txgpio;
+	struct irq_data *parent_data = data->parent_data;
+	int r;
+
+	r = gpiochip_lock_as_irq(&txgpio->chip, txline->line);
+	if (r)
+		return r;
+
+	if (parent_data && parent_data->chip->irq_request_resources) {
+		r = parent_data->chip->irq_request_resources(parent_data);
+		if (r)
+			goto error;
+	}
+
+	return 0;
+error:
+	gpiochip_unlock_as_irq(&txgpio->chip, txline->line);
+	return r;
+}
+
+static void thunderx_gpio_irq_release_resources(struct irq_data *data)
+{
+	struct thunderx_line *txline = irq_data_get_irq_chip_data(data);
+	struct thunderx_gpio *txgpio = txline->txgpio;
+	struct irq_data *parent_data = data->parent_data;
+
+	if (parent_data && parent_data->chip->irq_release_resources)
+		parent_data->chip->irq_release_resources(parent_data);
+
+	gpiochip_unlock_as_irq(&txgpio->chip, txline->line);
+}
+
+/*
+ * Interrupts are chained from underlying MSI-X vectors.  We have
+ * these irq_chip functions to be able to handle level triggering
+ * semantics and other acknowledgment tasks associated with the GPIO
+ * mechanism.
+ */
+static struct irq_chip thunderx_gpio_irq_chip = {
+	.name			= "GPIO",
+	.irq_enable		= thunderx_gpio_irq_enable,
+	.irq_disable		= thunderx_gpio_irq_disable,
+	.irq_ack		= thunderx_gpio_irq_ack,
+	.irq_mask		= thunderx_gpio_irq_mask,
+	.irq_mask_ack		= thunderx_gpio_irq_mask_ack,
+	.irq_unmask		= thunderx_gpio_irq_unmask,
+	.irq_eoi		= irq_chip_eoi_parent,
+	.irq_set_affinity	= irq_chip_set_affinity_parent,
+	.irq_request_resources	= thunderx_gpio_irq_request_resources,
+	.irq_release_resources	= thunderx_gpio_irq_release_resources,
+	.irq_set_type		= thunderx_gpio_irq_set_type,
+
+	.flags			= IRQCHIP_SET_TYPE_MASKED
+};
+
+static int thunderx_gpio_irq_map(struct irq_domain *d, unsigned int irq,
+				 irq_hw_number_t hwirq)
+{
+	struct thunderx_gpio *txgpio = d->host_data;
+
+	if (hwirq >= txgpio->chip.ngpio)
+		return -EINVAL;
+	if (!thunderx_gpio_is_gpio_nowarn(txgpio, hwirq))
+		return -EPERM;
+	return 0;
+}
+
+static int thunderx_gpio_irq_translate(struct irq_domain *d,
+				       struct irq_fwspec *fwspec,
+				       irq_hw_number_t *hwirq,
+				       unsigned int *type)
+{
+	struct thunderx_gpio *txgpio = d->host_data;
+
+	if (WARN_ON(fwspec->param_count < 2))
+		return -EINVAL;
+	if (fwspec->param[0] >= txgpio->chip.ngpio)
+		return -EINVAL;
+	*hwirq = fwspec->param[0];
+	*type = fwspec->param[1] & IRQ_TYPE_SENSE_MASK;
+	return 0;
+}
+
+static int thunderx_gpio_irq_alloc(struct irq_domain *d, unsigned int virq,
+				   unsigned int nr_irqs, void *arg)
+{
+	struct thunderx_line *txline = arg;
+
+	return irq_domain_set_hwirq_and_chip(d, virq, txline->line,
+					     &thunderx_gpio_irq_chip, txline);
+}
+
+static const struct irq_domain_ops thunderx_gpio_irqd_ops = {
+	.map		= thunderx_gpio_irq_map,
+	.alloc		= thunderx_gpio_irq_alloc,
+	.translate	= thunderx_gpio_irq_translate
+};
+
+static int thunderx_gpio_to_irq(struct gpio_chip *chip, unsigned int offset)
+{
+	struct thunderx_gpio *txgpio = gpiochip_get_data(chip);
+
+	return irq_find_mapping(txgpio->irqd, offset);
+}
+
+static int thunderx_gpio_probe(struct pci_dev *pdev,
+			       const struct pci_device_id *id)
+{
+	void __iomem * const *tbl;
+	struct device *dev = &pdev->dev;
+	struct thunderx_gpio *txgpio;
+	struct gpio_chip *chip;
+	int ngpio, i;
+	int err = 0;
+
+	txgpio = devm_kzalloc(dev, sizeof(*txgpio), GFP_KERNEL);
+	if (!txgpio)
+		return -ENOMEM;
+
+	raw_spin_lock_init(&txgpio->lock);
+	chip = &txgpio->chip;
+
+	pci_set_drvdata(pdev, txgpio);
+
+	err = pcim_enable_device(pdev);
+	if (err) {
+		dev_err(dev, "Failed to enable PCI device: err %d\n", err);
+		goto out;
+	}
+
+	err = pcim_iomap_regions(pdev, 1 << 0, KBUILD_MODNAME);
+	if (err) {
+		dev_err(dev, "Failed to iomap PCI device: err %d\n", err);
+		goto out;
+	}
+
+	tbl = pcim_iomap_table(pdev);
+	txgpio->register_base = tbl[0];
+	if (!txgpio->register_base) {
+		dev_err(dev, "Cannot map PCI resource\n");
+		err = -ENOMEM;
+		goto out;
+	}
+
+	if (pdev->subsystem_device == 0xa10a) {
+		/* CN88XX has no GPIO_CONST register*/
+		ngpio = 50;
+		txgpio->base_msi = 48;
+	} else {
+		u64 c = readq(txgpio->register_base + GPIO_CONST);
+
+		ngpio = c & GPIO_CONST_GPIOS_MASK;
+		txgpio->base_msi = (c >> 8) & 0xff;
+	}
+
+	txgpio->msix_entries = devm_kzalloc(dev,
+					  sizeof(struct msix_entry) * ngpio,
+					  GFP_KERNEL);
+	if (!txgpio->msix_entries) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	txgpio->line_entries = devm_kzalloc(dev,
+					    sizeof(struct thunderx_line) * ngpio,
+					    GFP_KERNEL);
+	if (!txgpio->line_entries) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	for (i = 0; i < ngpio; i++) {
+		u64 bit_cfg = readq(txgpio->register_base + bit_cfg_reg(i));
+
+		txgpio->msix_entries[i].entry = txgpio->base_msi + (2 * i);
+		txgpio->line_entries[i].line = i;
+		txgpio->line_entries[i].txgpio = txgpio;
+		/*
+		 * If something has already programmed the pin, use
+		 * the existing glitch filter settings, otherwise go
+		 * to 400nS.
+		 */
+		txgpio->line_entries[i].fil_bits = bit_cfg ?
+			(bit_cfg & GPIO_BIT_CFG_FIL_MASK) : GLITCH_FILTER_400NS;
+
+		if ((bit_cfg & GPIO_BIT_CFG_TX_OE) && (bit_cfg & GPIO_BIT_CFG_TX_OD))
+			set_bit(i, txgpio->od_mask);
+		if (bit_cfg & GPIO_BIT_CFG_PIN_XOR)
+			set_bit(i, txgpio->invert_mask);
+	}
+
+
+	/* Enable all MSI-X for interrupts on all possible lines. */
+	err = pci_enable_msix_range(pdev, txgpio->msix_entries, ngpio, ngpio);
+	if (err < 0)
+		goto out;
+
+	/*
+	 * Push GPIO specific irqdomain on hierarchy created as a side
+	 * effect of the pci_enable_msix()
+	 */
+	txgpio->irqd = irq_domain_create_hierarchy(irq_get_irq_data(txgpio->msix_entries[0].vector)->domain,
+						   0, 0, of_node_to_fwnode(dev->of_node),
+						   &thunderx_gpio_irqd_ops, txgpio);
+	if (!txgpio->irqd)
+		goto out;
+
+	/* Push on irq_data and the domain for each line. */
+	for (i = 0; i < ngpio; i++) {
+		err = irq_domain_push_irq(txgpio->irqd,
+					  txgpio->msix_entries[i].vector,
+					  &txgpio->line_entries[i]);
+		if (err < 0)
+			dev_err(dev, "irq_domain_push_irq: %d\n", err);
+	}
+
+	chip->label = KBUILD_MODNAME;
+	chip->parent = dev;
+	chip->owner = THIS_MODULE;
+	chip->request = thunderx_gpio_request;
+	chip->base = -1; /* System allocated */
+	chip->can_sleep = false;
+	chip->ngpio = ngpio;
+	chip->get_direction = thunderx_gpio_get_direction;
+	chip->direction_input = thunderx_gpio_dir_in;
+	chip->get = thunderx_gpio_get;
+	chip->direction_output = thunderx_gpio_dir_out;
+	chip->set = thunderx_gpio_set;
+	chip->set_multiple = thunderx_gpio_set_multiple;
+	chip->set_config = thunderx_gpio_set_config;
+	chip->to_irq = thunderx_gpio_to_irq;
+	err = devm_gpiochip_add_data(dev, chip, txgpio);
+	if (err)
+		goto out;
+
+	dev_info(dev, "ThunderX GPIO: %d lines with base %d.\n",
+		 ngpio, chip->base);
+	return 0;
+out:
+	pci_set_drvdata(pdev, NULL);
+	return err;
+}
+
+static void thunderx_gpio_remove(struct pci_dev *pdev)
+{
+	int i;
+	struct thunderx_gpio *txgpio = pci_get_drvdata(pdev);
+
+	for (i = 0; i < txgpio->chip.ngpio; i++)
+		irq_domain_pop_irq(txgpio->irqd,
+				   txgpio->msix_entries[i].vector);
+
+	irq_domain_remove(txgpio->irqd);
+
+	pci_set_drvdata(pdev, NULL);
+}
+
+static const struct pci_device_id thunderx_gpio_id_table[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, 0xA00A) },
+	{ 0, }	/* end of table */
+};
+
+MODULE_DEVICE_TABLE(pci, thunderx_gpio_id_table);
+
+static struct pci_driver thunderx_gpio_driver = {
+	.name = KBUILD_MODNAME,
+	.id_table = thunderx_gpio_id_table,
+	.probe = thunderx_gpio_probe,
+	.remove = thunderx_gpio_remove,
+};
+
+module_pci_driver(thunderx_gpio_driver);
+
+MODULE_DESCRIPTION("Cavium Inc. ThunderX/OCTEON-TX GPIO Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpio/gpio-tps68470.c b/drivers/gpio/gpio-tps68470.c
new file mode 100644
index 000000000000..fa2662f8b026
--- /dev/null
+++ b/drivers/gpio/gpio-tps68470.c
@@ -0,0 +1,176 @@
+/*
+ * GPIO driver for TPS68470 PMIC
+ *
+ * Copyright (C) 2017 Intel Corporation
+ *
+ * Authors:
+ *	Antti Laakso <antti.laakso@intel.com>
+ *	Tianshu Qiu <tian.shu.qiu@intel.com>
+ *	Jian Xu Zheng <jian.xu.zheng@intel.com>
+ *	Yuning Pu <yuning.pu@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/gpio/driver.h>
+#include <linux/mfd/tps68470.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#define TPS68470_N_LOGIC_OUTPUT	3
+#define TPS68470_N_REGULAR_GPIO	7
+#define TPS68470_N_GPIO	(TPS68470_N_LOGIC_OUTPUT + TPS68470_N_REGULAR_GPIO)
+
+struct tps68470_gpio_data {
+	struct regmap *tps68470_regmap;
+	struct gpio_chip gc;
+};
+
+static int tps68470_gpio_get(struct gpio_chip *gc, unsigned int offset)
+{
+	struct tps68470_gpio_data *tps68470_gpio = gpiochip_get_data(gc);
+	struct regmap *regmap = tps68470_gpio->tps68470_regmap;
+	unsigned int reg = TPS68470_REG_GPDO;
+	int val, ret;
+
+	if (offset >= TPS68470_N_REGULAR_GPIO) {
+		offset -= TPS68470_N_REGULAR_GPIO;
+		reg = TPS68470_REG_SGPO;
+	}
+
+	ret = regmap_read(regmap, reg, &val);
+	if (ret) {
+		dev_err(tps68470_gpio->gc.parent, "reg 0x%x read failed\n",
+			TPS68470_REG_SGPO);
+		return ret;
+	}
+	return !!(val & BIT(offset));
+}
+
+/* Return 0 if output, 1 if input */
+static int tps68470_gpio_get_direction(struct gpio_chip *gc,
+				       unsigned int offset)
+{
+	struct tps68470_gpio_data *tps68470_gpio = gpiochip_get_data(gc);
+	struct regmap *regmap = tps68470_gpio->tps68470_regmap;
+	int val, ret;
+
+	/* rest are always outputs */
+	if (offset >= TPS68470_N_REGULAR_GPIO)
+		return 0;
+
+	ret = regmap_read(regmap, TPS68470_GPIO_CTL_REG_A(offset), &val);
+	if (ret) {
+		dev_err(tps68470_gpio->gc.parent, "reg 0x%x read failed\n",
+			TPS68470_GPIO_CTL_REG_A(offset));
+		return ret;
+	}
+
+	val &= TPS68470_GPIO_MODE_MASK;
+	return val >= TPS68470_GPIO_MODE_OUT_CMOS ? 0 : 1;
+}
+
+static void tps68470_gpio_set(struct gpio_chip *gc, unsigned int offset,
+				int value)
+{
+	struct tps68470_gpio_data *tps68470_gpio = gpiochip_get_data(gc);
+	struct regmap *regmap = tps68470_gpio->tps68470_regmap;
+	unsigned int reg = TPS68470_REG_GPDO;
+
+	if (offset >= TPS68470_N_REGULAR_GPIO) {
+		reg = TPS68470_REG_SGPO;
+		offset -= TPS68470_N_REGULAR_GPIO;
+	}
+
+	regmap_update_bits(regmap, reg, BIT(offset), value ? BIT(offset) : 0);
+}
+
+static int tps68470_gpio_output(struct gpio_chip *gc, unsigned int offset,
+				int value)
+{
+	struct tps68470_gpio_data *tps68470_gpio = gpiochip_get_data(gc);
+	struct regmap *regmap = tps68470_gpio->tps68470_regmap;
+
+	/* rest are always outputs */
+	if (offset >= TPS68470_N_REGULAR_GPIO)
+		return 0;
+
+	/* Set the initial value */
+	tps68470_gpio_set(gc, offset, value);
+
+	return regmap_update_bits(regmap, TPS68470_GPIO_CTL_REG_A(offset),
+				 TPS68470_GPIO_MODE_MASK,
+				 TPS68470_GPIO_MODE_OUT_CMOS);
+}
+
+static int tps68470_gpio_input(struct gpio_chip *gc, unsigned int offset)
+{
+	struct tps68470_gpio_data *tps68470_gpio = gpiochip_get_data(gc);
+	struct regmap *regmap = tps68470_gpio->tps68470_regmap;
+
+	/* rest are always outputs */
+	if (offset >= TPS68470_N_REGULAR_GPIO)
+		return -EINVAL;
+
+	return regmap_update_bits(regmap, TPS68470_GPIO_CTL_REG_A(offset),
+				   TPS68470_GPIO_MODE_MASK, 0x00);
+}
+
+static const char *tps68470_names[TPS68470_N_GPIO] = {
+	"gpio.0", "gpio.1", "gpio.2", "gpio.3",
+	"gpio.4", "gpio.5", "gpio.6",
+	"s_enable", "s_idle", "s_resetn",
+};
+
+static int tps68470_gpio_probe(struct platform_device *pdev)
+{
+	struct tps68470_gpio_data *tps68470_gpio;
+	int ret;
+
+	tps68470_gpio = devm_kzalloc(&pdev->dev, sizeof(*tps68470_gpio),
+				     GFP_KERNEL);
+	if (!tps68470_gpio)
+		return -ENOMEM;
+
+	tps68470_gpio->tps68470_regmap = dev_get_drvdata(pdev->dev.parent);
+	tps68470_gpio->gc.label = "tps68470-gpio";
+	tps68470_gpio->gc.owner = THIS_MODULE;
+	tps68470_gpio->gc.direction_input = tps68470_gpio_input;
+	tps68470_gpio->gc.direction_output = tps68470_gpio_output;
+	tps68470_gpio->gc.get = tps68470_gpio_get;
+	tps68470_gpio->gc.get_direction = tps68470_gpio_get_direction;
+	tps68470_gpio->gc.set = tps68470_gpio_set;
+	tps68470_gpio->gc.can_sleep = true;
+	tps68470_gpio->gc.names = tps68470_names;
+	tps68470_gpio->gc.ngpio = TPS68470_N_GPIO;
+	tps68470_gpio->gc.base = -1;
+	tps68470_gpio->gc.parent = &pdev->dev;
+
+	ret = devm_gpiochip_add_data(&pdev->dev, &tps68470_gpio->gc,
+				     tps68470_gpio);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "Failed to register gpio_chip: %d\n", ret);
+		return ret;
+	}
+
+	platform_set_drvdata(pdev, tps68470_gpio);
+
+	return ret;
+}
+
+static struct platform_driver tps68470_gpio_driver = {
+	.driver = {
+		   .name = "tps68470-gpio",
+	},
+	.probe = tps68470_gpio_probe,
+};
+
+builtin_platform_driver(tps68470_gpio_driver)
diff --git a/drivers/gpio/gpio-twl4030.c b/drivers/gpio/gpio-twl4030.c
index 24f388ed46d4..9b511df5450e 100644
--- a/drivers/gpio/gpio-twl4030.c
+++ b/drivers/gpio/gpio-twl4030.c
@@ -35,7 +35,7 @@
 #include <linux/of.h>
 #include <linux/irqdomain.h>
 
-#include <linux/i2c/twl.h>
+#include <linux/mfd/twl.h>
 
 /*
  * The GPIO "subchip" supports 18 GPIOs which can be configured as
diff --git a/drivers/gpio/gpio-twl6040.c b/drivers/gpio/gpio-twl6040.c
index b780314cdfc9..dadeacf43e0c 100644
--- a/drivers/gpio/gpio-twl6040.c
+++ b/drivers/gpio/gpio-twl6040.c
@@ -32,8 +32,6 @@
 
 #include <linux/mfd/twl6040.h>
 
-static struct gpio_chip twl6040gpo_chip;
-
 static int twl6040gpo_get(struct gpio_chip *chip, unsigned offset)
 {
 	struct twl6040 *twl6040 = dev_get_drvdata(chip->parent->parent);
diff --git a/drivers/gpio/gpio-tz1090.c b/drivers/gpio/gpio-tz1090.c
index ca958e0f6909..22c5be65051f 100644
--- a/drivers/gpio/gpio-tz1090.c
+++ b/drivers/gpio/gpio-tz1090.c
@@ -527,13 +527,12 @@ static void tz1090_gpio_register_banks(struct tz1090_gpio *priv)
 
 		ret = of_property_read_u32(node, "reg", &addr);
 		if (ret) {
-			dev_err(priv->dev, "invalid reg on %s\n",
-				node->full_name);
+			dev_err(priv->dev, "invalid reg on %pOF\n", node);
 			continue;
 		}
 		if (addr >= 3) {
-			dev_err(priv->dev, "index %u in %s out of range\n",
-				addr, node->full_name);
+			dev_err(priv->dev, "index %u in %pOF out of range\n",
+				addr, node);
 			continue;
 		}
 
@@ -543,8 +542,7 @@ static void tz1090_gpio_register_banks(struct tz1090_gpio *priv)
 
 		ret = tz1090_gpio_bank_probe(&info);
 		if (ret) {
-			dev_err(priv->dev, "failure registering %s\n",
-				node->full_name);
+			dev_err(priv->dev, "failure registering %pOF\n", node);
 			of_node_put(node);
 			continue;
 		}
diff --git a/drivers/gpio/gpio-vf610.c b/drivers/gpio/gpio-vf610.c
index 521fbe338589..cbe9e06861de 100644
--- a/drivers/gpio/gpio-vf610.c
+++ b/drivers/gpio/gpio-vf610.c
@@ -30,10 +30,16 @@
 
 #define VF610_GPIO_PER_PORT		32
 
+struct fsl_gpio_soc_data {
+	/* SoCs has a Port Data Direction Register (PDDR) */
+	bool have_paddr;
+};
+
 struct vf610_gpio_port {
 	struct gpio_chip gc;
 	void __iomem *base;
 	void __iomem *gpio_base;
+	const struct fsl_gpio_soc_data *sdata;
 	u8 irqc[VF610_GPIO_PER_PORT];
 	int irq;
 };
@@ -43,6 +49,7 @@ struct vf610_gpio_port {
 #define GPIO_PCOR		0x08
 #define GPIO_PTOR		0x0c
 #define GPIO_PDIR		0x10
+#define GPIO_PDDR		0x14
 
 #define PORT_PCR(n)		((n) * 0x4)
 #define PORT_PCR_IRQC_OFFSET	16
@@ -61,8 +68,13 @@ struct vf610_gpio_port {
 
 static struct irq_chip vf610_gpio_irq_chip;
 
+static const struct fsl_gpio_soc_data imx_data = {
+	.have_paddr = true,
+};
+
 static const struct of_device_id vf610_gpio_dt_ids[] = {
-	{ .compatible = "fsl,vf610-gpio" },
+	{ .compatible = "fsl,vf610-gpio",	.data = NULL, },
+	{ .compatible = "fsl,imx7ulp-gpio",	.data = &imx_data, },
 	{ /* sentinel */ }
 };
 
@@ -79,8 +91,18 @@ static inline u32 vf610_gpio_readl(void __iomem *reg)
 static int vf610_gpio_get(struct gpio_chip *gc, unsigned int gpio)
 {
 	struct vf610_gpio_port *port = gpiochip_get_data(gc);
-
-	return !!(vf610_gpio_readl(port->gpio_base + GPIO_PDIR) & BIT(gpio));
+	unsigned long mask = BIT(gpio);
+	void __iomem *addr;
+
+	if (port->sdata && port->sdata->have_paddr) {
+		mask &= vf610_gpio_readl(port->gpio_base + GPIO_PDDR);
+		addr = mask ? port->gpio_base + GPIO_PDOR :
+			      port->gpio_base + GPIO_PDIR;
+		return !!(vf610_gpio_readl(addr) & BIT(gpio));
+	} else {
+		return !!(vf610_gpio_readl(port->gpio_base + GPIO_PDIR)
+					   & BIT(gpio));
+	}
 }
 
 static void vf610_gpio_set(struct gpio_chip *gc, unsigned int gpio, int val)
@@ -96,12 +118,28 @@ static void vf610_gpio_set(struct gpio_chip *gc, unsigned int gpio, int val)
 
 static int vf610_gpio_direction_input(struct gpio_chip *chip, unsigned gpio)
 {
+	struct vf610_gpio_port *port = gpiochip_get_data(chip);
+	unsigned long mask = BIT(gpio);
+	u32 val;
+
+	if (port->sdata && port->sdata->have_paddr) {
+		val = vf610_gpio_readl(port->gpio_base + GPIO_PDDR);
+		val &= ~mask;
+		vf610_gpio_writel(val, port->gpio_base + GPIO_PDDR);
+	}
+
 	return pinctrl_gpio_direction_input(chip->base + gpio);
 }
 
 static int vf610_gpio_direction_output(struct gpio_chip *chip, unsigned gpio,
 				       int value)
 {
+	struct vf610_gpio_port *port = gpiochip_get_data(chip);
+	unsigned long mask = BIT(gpio);
+
+	if (port->sdata && port->sdata->have_paddr)
+		vf610_gpio_writel(mask, port->gpio_base + GPIO_PDDR);
+
 	vf610_gpio_set(chip, gpio, value);
 
 	return pinctrl_gpio_direction_output(chip->base + gpio);
@@ -216,6 +254,8 @@ static struct irq_chip vf610_gpio_irq_chip = {
 
 static int vf610_gpio_probe(struct platform_device *pdev)
 {
+	const struct of_device_id *of_id = of_match_device(vf610_gpio_dt_ids,
+							   &pdev->dev);
 	struct device *dev = &pdev->dev;
 	struct device_node *np = dev->of_node;
 	struct vf610_gpio_port *port;
@@ -227,6 +267,7 @@ static int vf610_gpio_probe(struct platform_device *pdev)
 	if (!port)
 		return -ENOMEM;
 
+	port->sdata = of_id->data;
 	iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	port->base = devm_ioremap_resource(dev, iores);
 	if (IS_ERR(port->base))
diff --git a/drivers/gpio/gpio-xilinx.c b/drivers/gpio/gpio-xilinx.c
index 14b2a62338ea..e8ec0e33a0a9 100644
--- a/drivers/gpio/gpio-xilinx.c
+++ b/drivers/gpio/gpio-xilinx.c
@@ -360,8 +360,8 @@ static int xgpio_probe(struct platform_device *pdev)
 	/* Call the OF gpio helper to setup and register the GPIO device */
 	status = of_mm_gpiochip_add_data(np, &chip->mmchip, chip);
 	if (status) {
-		pr_err("%s: error in probe function with status %d\n",
-		       np->full_name, status);
+		pr_err("%pOF: error in probe function with status %d\n",
+		       np, status);
 		return status;
 	}
 
diff --git a/drivers/gpio/gpio-zevio.c b/drivers/gpio/gpio-zevio.c
index e23ef7b9451d..3926ce9c2840 100644
--- a/drivers/gpio/gpio-zevio.c
+++ b/drivers/gpio/gpio-zevio.c
@@ -156,7 +156,7 @@ static int zevio_gpio_to_irq(struct gpio_chip *chip, unsigned pin)
 	return -ENXIO;
 }
 
-static struct gpio_chip zevio_gpio_chip = {
+static const struct gpio_chip zevio_gpio_chip = {
 	.direction_input	= zevio_gpio_direction_input,
 	.direction_output	= zevio_gpio_direction_output,
 	.set			= zevio_gpio_set,
diff --git a/drivers/gpio/gpio-zynq.c b/drivers/gpio/gpio-zynq.c
index df0851464006..b3cc948a2d8b 100644
--- a/drivers/gpio/gpio-zynq.c
+++ b/drivers/gpio/gpio-zynq.c
@@ -60,13 +60,13 @@
 #define ZYNQ_GPIO_BANK5_PIN_MAX(str)	(ZYNQ_GPIO_BANK5_PIN_MIN(str) + \
 					ZYNQ##str##_GPIO_BANK5_NGPIO - 1)
 
-
 /* Register offsets for the GPIO device */
 /* LSW Mask & Data -WO */
 #define ZYNQ_GPIO_DATA_LSW_OFFSET(BANK)	(0x000 + (8 * BANK))
 /* MSW Mask & Data -WO */
 #define ZYNQ_GPIO_DATA_MSW_OFFSET(BANK)	(0x004 + (8 * BANK))
 /* Data Register-RW */
+#define ZYNQ_GPIO_DATA_OFFSET(BANK)	(0x040 + (4 * BANK))
 #define ZYNQ_GPIO_DATA_RO_OFFSET(BANK)	(0x060 + (4 * BANK))
 /* Direction mode reg-RW */
 #define ZYNQ_GPIO_DIRM_OFFSET(BANK)	(0x204 + (0x40 * BANK))
@@ -98,6 +98,19 @@
 
 /* set to differentiate zynq from zynqmp, 0=zynqmp, 1=zynq */
 #define ZYNQ_GPIO_QUIRK_IS_ZYNQ	BIT(0)
+#define GPIO_QUIRK_DATA_RO_BUG	BIT(1)
+
+struct gpio_regs {
+	u32 datamsw[ZYNQMP_GPIO_MAX_BANK];
+	u32 datalsw[ZYNQMP_GPIO_MAX_BANK];
+	u32 dirm[ZYNQMP_GPIO_MAX_BANK];
+	u32 outen[ZYNQMP_GPIO_MAX_BANK];
+	u32 int_en[ZYNQMP_GPIO_MAX_BANK];
+	u32 int_dis[ZYNQMP_GPIO_MAX_BANK];
+	u32 int_type[ZYNQMP_GPIO_MAX_BANK];
+	u32 int_polarity[ZYNQMP_GPIO_MAX_BANK];
+	u32 int_any[ZYNQMP_GPIO_MAX_BANK];
+};
 
 /**
  * struct zynq_gpio - gpio device private data structure
@@ -106,6 +119,7 @@
  * @clk:	clock resource for this controller
  * @irq:	interrupt for the GPIO device
  * @p_data:	pointer to platform data
+ * @context:	context registers
  */
 struct zynq_gpio {
 	struct gpio_chip chip;
@@ -113,16 +127,18 @@ struct zynq_gpio {
 	struct clk *clk;
 	int irq;
 	const struct zynq_platform_data *p_data;
+	struct gpio_regs context;
 };
 
 /**
  * struct zynq_platform_data -  zynq gpio platform data structure
  * @label:	string to store in gpio->label
+ * @quirks:	Flags is used to identify the platform
  * @ngpio:	max number of gpio pins
  * @max_bank:	maximum number of gpio banks
  * @bank_min:	this array represents bank's min pin
  * @bank_max:	this array represents bank's max pin
-*/
+ */
 struct zynq_platform_data {
 	const char *label;
 	u32 quirks;
@@ -147,6 +163,17 @@ static int zynq_gpio_is_zynq(struct zynq_gpio *gpio)
 }
 
 /**
+ * gpio_data_ro_bug - test if HW bug exists or not
+ * @gpio:       Pointer to driver data struct
+ *
+ * Return: 0 if bug doesnot exist, 1 if bug exists.
+ */
+static int gpio_data_ro_bug(struct zynq_gpio *gpio)
+{
+	return !!(gpio->p_data->quirks & GPIO_QUIRK_DATA_RO_BUG);
+}
+
+/**
  * zynq_gpio_get_bank_pin - Get the bank number and pin number within that bank
  * for a given pin in the GPIO device
  * @pin_num:	gpio pin number within the device
@@ -154,6 +181,7 @@ static int zynq_gpio_is_zynq(struct zynq_gpio *gpio)
  *		pin
  * @bank_pin_num: an output parameter used to return pin number within a bank
  *		  for the given gpio pin
+ * @gpio:	gpio device data structure
  *
  * Returns the bank number and pin offset within the bank.
  */
@@ -166,11 +194,11 @@ static inline void zynq_gpio_get_bank_pin(unsigned int pin_num,
 
 	for (bank = 0; bank < gpio->p_data->max_bank; bank++) {
 		if ((pin_num >= gpio->p_data->bank_min[bank]) &&
-			(pin_num <= gpio->p_data->bank_max[bank])) {
-				*bank_num = bank;
-				*bank_pin_num = pin_num -
-						gpio->p_data->bank_min[bank];
-				return;
+		    (pin_num <= gpio->p_data->bank_max[bank])) {
+			*bank_num = bank;
+			*bank_pin_num = pin_num -
+					gpio->p_data->bank_min[bank];
+			return;
 		}
 	}
 
@@ -197,9 +225,28 @@ static int zynq_gpio_get_value(struct gpio_chip *chip, unsigned int pin)
 
 	zynq_gpio_get_bank_pin(pin, &bank_num, &bank_pin_num, gpio);
 
-	data = readl_relaxed(gpio->base_addr +
-			     ZYNQ_GPIO_DATA_RO_OFFSET(bank_num));
-
+	if (gpio_data_ro_bug(gpio)) {
+		if (zynq_gpio_is_zynq(gpio)) {
+			if (bank_num <= 1) {
+				data = readl_relaxed(gpio->base_addr +
+					ZYNQ_GPIO_DATA_RO_OFFSET(bank_num));
+			} else {
+				data = readl_relaxed(gpio->base_addr +
+					ZYNQ_GPIO_DATA_OFFSET(bank_num));
+			}
+		} else {
+			if (bank_num <= 2) {
+				data = readl_relaxed(gpio->base_addr +
+					ZYNQ_GPIO_DATA_RO_OFFSET(bank_num));
+			} else {
+				data = readl_relaxed(gpio->base_addr +
+					ZYNQ_GPIO_DATA_OFFSET(bank_num));
+			}
+		}
+	} else {
+		data = readl_relaxed(gpio->base_addr +
+			ZYNQ_GPIO_DATA_RO_OFFSET(bank_num));
+	}
 	return (data >> bank_pin_num) & 1;
 }
 
@@ -263,7 +310,7 @@ static int zynq_gpio_dir_in(struct gpio_chip *chip, unsigned int pin)
 	 * as inputs.
 	 */
 	if (zynq_gpio_is_zynq(gpio) && bank_num == 0 &&
-		(bank_pin_num == 7 || bank_pin_num == 8))
+	    (bank_pin_num == 7 || bank_pin_num == 8))
 		return -EINVAL;
 
 	/* clear the bit in direction mode reg to set the pin as input */
@@ -464,13 +511,14 @@ static int zynq_gpio_set_irq_type(struct irq_data *irq_data, unsigned int type)
 	writel_relaxed(int_any,
 		       gpio->base_addr + ZYNQ_GPIO_INTANY_OFFSET(bank_num));
 
-	if (type & IRQ_TYPE_LEVEL_MASK) {
+	if (type & IRQ_TYPE_LEVEL_MASK)
 		irq_set_chip_handler_name_locked(irq_data,
-			&zynq_gpio_level_irqchip, handle_fasteoi_irq, NULL);
-	} else {
+						 &zynq_gpio_level_irqchip,
+						 handle_fasteoi_irq, NULL);
+	else
 		irq_set_chip_handler_name_locked(irq_data,
-			&zynq_gpio_edge_irqchip, handle_level_irq, NULL);
-	}
+						 &zynq_gpio_edge_irqchip,
+						 handle_level_irq, NULL);
 
 	return 0;
 }
@@ -530,7 +578,6 @@ static void zynq_gpio_handle_bank_irq(struct zynq_gpio *gpio,
 
 /**
  * zynq_gpio_irqhandler - IRQ handler for the gpio banks of a gpio device
- * @irq:	irq number of the gpio bank where interrupt has occurred
  * @desc:	irq descriptor instance of the 'irq'
  *
  * This function reads the Interrupt Status Register of each bank to get the
@@ -560,14 +607,73 @@ static void zynq_gpio_irqhandler(struct irq_desc *desc)
 	chained_irq_exit(irqchip, desc);
 }
 
+static void zynq_gpio_save_context(struct zynq_gpio *gpio)
+{
+	unsigned int bank_num;
+
+	for (bank_num = 0; bank_num < gpio->p_data->max_bank; bank_num++) {
+		gpio->context.datalsw[bank_num] =
+				readl_relaxed(gpio->base_addr +
+				ZYNQ_GPIO_DATA_LSW_OFFSET(bank_num));
+		gpio->context.datamsw[bank_num] =
+				readl_relaxed(gpio->base_addr +
+				ZYNQ_GPIO_DATA_MSW_OFFSET(bank_num));
+		gpio->context.dirm[bank_num] = readl_relaxed(gpio->base_addr +
+				ZYNQ_GPIO_DIRM_OFFSET(bank_num));
+		gpio->context.int_en[bank_num] = readl_relaxed(gpio->base_addr +
+				ZYNQ_GPIO_INTMASK_OFFSET(bank_num));
+		gpio->context.int_type[bank_num] =
+				readl_relaxed(gpio->base_addr +
+				ZYNQ_GPIO_INTTYPE_OFFSET(bank_num));
+		gpio->context.int_polarity[bank_num] =
+				readl_relaxed(gpio->base_addr +
+				ZYNQ_GPIO_INTPOL_OFFSET(bank_num));
+		gpio->context.int_any[bank_num] =
+				readl_relaxed(gpio->base_addr +
+				ZYNQ_GPIO_INTANY_OFFSET(bank_num));
+	}
+}
+
+static void zynq_gpio_restore_context(struct zynq_gpio *gpio)
+{
+	unsigned int bank_num;
+
+	for (bank_num = 0; bank_num < gpio->p_data->max_bank; bank_num++) {
+		writel_relaxed(gpio->context.datalsw[bank_num],
+			       gpio->base_addr +
+			       ZYNQ_GPIO_DATA_LSW_OFFSET(bank_num));
+		writel_relaxed(gpio->context.datamsw[bank_num],
+			       gpio->base_addr +
+			       ZYNQ_GPIO_DATA_MSW_OFFSET(bank_num));
+		writel_relaxed(gpio->context.dirm[bank_num],
+			       gpio->base_addr +
+			       ZYNQ_GPIO_DIRM_OFFSET(bank_num));
+		writel_relaxed(gpio->context.int_en[bank_num],
+			       gpio->base_addr +
+			       ZYNQ_GPIO_INTEN_OFFSET(bank_num));
+		writel_relaxed(gpio->context.int_type[bank_num],
+			       gpio->base_addr +
+			       ZYNQ_GPIO_INTTYPE_OFFSET(bank_num));
+		writel_relaxed(gpio->context.int_polarity[bank_num],
+			       gpio->base_addr +
+			       ZYNQ_GPIO_INTPOL_OFFSET(bank_num));
+		writel_relaxed(gpio->context.int_any[bank_num],
+			       gpio->base_addr +
+			       ZYNQ_GPIO_INTANY_OFFSET(bank_num));
+	}
+}
+
 static int __maybe_unused zynq_gpio_suspend(struct device *dev)
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	int irq = platform_get_irq(pdev, 0);
 	struct irq_data *data = irq_get_irq_data(irq);
+	struct zynq_gpio *gpio = platform_get_drvdata(pdev);
 
-	if (!irqd_is_wakeup_set(data))
+	if (!irqd_is_wakeup_set(data)) {
+		zynq_gpio_save_context(gpio);
 		return pm_runtime_force_suspend(dev);
+	}
 
 	return 0;
 }
@@ -577,9 +683,14 @@ static int __maybe_unused zynq_gpio_resume(struct device *dev)
 	struct platform_device *pdev = to_platform_device(dev);
 	int irq = platform_get_irq(pdev, 0);
 	struct irq_data *data = irq_get_irq_data(irq);
+	struct zynq_gpio *gpio = platform_get_drvdata(pdev);
+	int ret;
 
-	if (!irqd_is_wakeup_set(data))
-		return pm_runtime_force_resume(dev);
+	if (!irqd_is_wakeup_set(data)) {
+		ret = pm_runtime_force_resume(dev);
+		zynq_gpio_restore_context(gpio);
+		return ret;
+	}
 
 	return 0;
 }
@@ -602,7 +713,7 @@ static int __maybe_unused zynq_gpio_runtime_resume(struct device *dev)
 	return clk_prepare_enable(gpio->clk);
 }
 
-static int zynq_gpio_request(struct gpio_chip *chip, unsigned offset)
+static int zynq_gpio_request(struct gpio_chip *chip, unsigned int offset)
 {
 	int ret;
 
@@ -615,7 +726,7 @@ static int zynq_gpio_request(struct gpio_chip *chip, unsigned offset)
 	return ret < 0 ? ret : 0;
 }
 
-static void zynq_gpio_free(struct gpio_chip *chip, unsigned offset)
+static void zynq_gpio_free(struct gpio_chip *chip, unsigned int offset)
 {
 	pm_runtime_put(chip->parent);
 }
@@ -623,11 +734,12 @@ static void zynq_gpio_free(struct gpio_chip *chip, unsigned offset)
 static const struct dev_pm_ops zynq_gpio_dev_pm_ops = {
 	SET_SYSTEM_SLEEP_PM_OPS(zynq_gpio_suspend, zynq_gpio_resume)
 	SET_RUNTIME_PM_OPS(zynq_gpio_runtime_suspend,
-			zynq_gpio_runtime_resume, NULL)
+			   zynq_gpio_runtime_resume, NULL)
 };
 
 static const struct zynq_platform_data zynqmp_gpio_def = {
 	.label = "zynqmp_gpio",
+	.quirks = GPIO_QUIRK_DATA_RO_BUG,
 	.ngpio = ZYNQMP_GPIO_NR_GPIOS,
 	.max_bank = ZYNQMP_GPIO_MAX_BANK,
 	.bank_min[0] = ZYNQ_GPIO_BANK0_PIN_MIN(MP),
@@ -646,7 +758,7 @@ static const struct zynq_platform_data zynqmp_gpio_def = {
 
 static const struct zynq_platform_data zynq_gpio_def = {
 	.label = "zynq_gpio",
-	.quirks = ZYNQ_GPIO_QUIRK_IS_ZYNQ,
+	.quirks = ZYNQ_GPIO_QUIRK_IS_ZYNQ | GPIO_QUIRK_DATA_RO_BUG,
 	.ngpio = ZYNQ_GPIO_NR_GPIOS,
 	.max_bank = ZYNQ_GPIO_MAX_BANK,
 	.bank_min[0] = ZYNQ_GPIO_BANK0_PIN_MIN(),
diff --git a/drivers/gpio/gpiolib-acpi.c b/drivers/gpio/gpiolib-acpi.c
index c9b42dd12dfa..4d2113530735 100644
--- a/drivers/gpio/gpiolib-acpi.c
+++ b/drivers/gpio/gpiolib-acpi.c
@@ -61,7 +61,7 @@ static int acpi_gpiochip_find(struct gpio_chip *gc, void *data)
 #ifdef CONFIG_PINCTRL
 /**
  * acpi_gpiochip_pin_to_gpio_offset() - translates ACPI GPIO to Linux GPIO
- * @chip: GPIO chip
+ * @gdev: GPIO device
  * @pin: ACPI GPIO pin number from GpioIo/GpioInt resource
  *
  * Function takes ACPI GpioIo/GpioInt pin number as a parameter and
@@ -763,7 +763,7 @@ struct gpio_desc *acpi_node_get_gpiod(struct fwnode_handle *fwnode,
  * The function is idempotent, though each time it runs it will configure GPIO
  * pin direction according to the flags in GpioInt resource.
  *
- * Return: Linux IRQ number (>%0) on success, negative errno on failure.
+ * Return: Linux IRQ number (> %0) on success, negative errno on failure.
  */
 int acpi_dev_gpio_irq_get(struct acpi_device *adev, int index)
 {
diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c
index 54ce8dc58ad0..bfcd20699ec8 100644
--- a/drivers/gpio/gpiolib-of.c
+++ b/drivers/gpio/gpiolib-of.c
@@ -78,8 +78,8 @@ struct gpio_desc *of_get_named_gpiod_flags(struct device_node *np,
 	ret = of_parse_phandle_with_args(np, propname, "#gpio-cells", index,
 					 &gpiospec);
 	if (ret) {
-		pr_debug("%s: can't parse '%s' property of node '%s[%d]'\n",
-			__func__, propname, np->full_name, index);
+		pr_debug("%s: can't parse '%s' property of node '%pOF[%d]'\n",
+			__func__, propname, np, index);
 		return ERR_PTR(ret);
 	}
 
@@ -93,8 +93,8 @@ struct gpio_desc *of_get_named_gpiod_flags(struct device_node *np,
 	if (IS_ERR(desc))
 		goto out;
 
-	pr_debug("%s: parsed '%s' property of node '%s[%d]' - status (%d)\n",
-		 __func__, propname, np->full_name, index,
+	pr_debug("%s: parsed '%s' property of node '%pOF[%d]' - status (%d)\n",
+		 __func__, propname, np, index,
 		 PTR_ERR_OR_ZERO(desc));
 
 out:
@@ -273,14 +273,13 @@ static int of_gpiochip_scan_gpios(struct gpio_chip *chip)
 }
 
 /**
- * of_gpio_simple_xlate - translate gpio_spec to the GPIO number and flags
+ * of_gpio_simple_xlate - translate gpiospec to the GPIO number and flags
  * @gc:		pointer to the gpio_chip structure
- * @np:		device node of the GPIO chip
- * @gpio_spec:	gpio specifier as found in the device tree
+ * @gpiospec:	GPIO specifier as found in the device tree
  * @flags:	a flags pointer to fill in
  *
  * This is simple translation function, suitable for the most 1:1 mapped
- * gpio chips. This function performs only one sanity check: whether gpio
+ * GPIO chips. This function performs only one sanity check: whether GPIO
  * is less than ngpios (that is specified in the gpio_chip).
  */
 int of_gpio_simple_xlate(struct gpio_chip *gc,
@@ -337,7 +336,7 @@ int of_mm_gpiochip_add_data(struct device_node *np,
 	int ret = -ENOMEM;
 	struct gpio_chip *gc = &mm_gc->gc;
 
-	gc->label = kstrdup(np->full_name, GFP_KERNEL);
+	gc->label = kasprintf(GFP_KERNEL, "%pOF", np);
 	if (!gc->label)
 		goto err0;
 
@@ -362,8 +361,7 @@ err2:
 err1:
 	kfree(gc->label);
 err0:
-	pr_err("%s: GPIO chip registration failed with status %d\n",
-	       np->full_name, ret);
+	pr_err("%pOF: GPIO chip registration failed with status %d\n", np, ret);
 	return ret;
 }
 EXPORT_SYMBOL(of_mm_gpiochip_add_data);
@@ -418,8 +416,8 @@ static int of_gpiochip_add_pin_range(struct gpio_chip *chip)
 						group_names_propname,
 						index, &name);
 				if (strlen(name)) {
-					pr_err("%s: Group name of numeric GPIO ranges must be the empty string.\n",
-						np->full_name);
+					pr_err("%pOF: Group name of numeric GPIO ranges must be the empty string.\n",
+						np);
 					break;
 				}
 			}
@@ -434,14 +432,14 @@ static int of_gpiochip_add_pin_range(struct gpio_chip *chip)
 		} else {
 			/* npins == 0: special range */
 			if (pinspec.args[1]) {
-				pr_err("%s: Illegal gpio-range format.\n",
-					np->full_name);
+				pr_err("%pOF: Illegal gpio-range format.\n",
+					np);
 				break;
 			}
 
 			if (!group_names) {
-				pr_err("%s: GPIO group range requested but no %s property.\n",
-					np->full_name, group_names_propname);
+				pr_err("%pOF: GPIO group range requested but no %s property.\n",
+					np, group_names_propname);
 				break;
 			}
 
@@ -452,8 +450,8 @@ static int of_gpiochip_add_pin_range(struct gpio_chip *chip)
 				break;
 
 			if (!strlen(name)) {
-				pr_err("%s: Group name of GPIO group range cannot be the empty string.\n",
-				np->full_name);
+				pr_err("%pOF: Group name of GPIO group range cannot be the empty string.\n",
+				np);
 				break;
 			}
 
diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c
index 16fe9742597b..3f454eaf2101 100644
--- a/drivers/gpio/gpiolib-sysfs.c
+++ b/drivers/gpio/gpiolib-sysfs.c
@@ -2,6 +2,7 @@
 #include <linux/mutex.h>
 #include <linux/device.h>
 #include <linux/sysfs.h>
+#include <linux/gpio.h>
 #include <linux/gpio/consumer.h>
 #include <linux/gpio/driver.h>
 #include <linux/interrupt.h>
@@ -432,6 +433,11 @@ static struct attribute *gpiochip_attrs[] = {
 };
 ATTRIBUTE_GROUPS(gpiochip);
 
+static struct gpio_desc *gpio_to_valid_desc(int gpio)
+{
+	return gpio_is_valid(gpio) ? gpio_to_desc(gpio) : NULL;
+}
+
 /*
  * /sys/class/gpio/export ... write-only
  *	integer N ... number of GPIO to export (full access)
@@ -450,7 +456,7 @@ static ssize_t export_store(struct class *class,
 	if (status < 0)
 		goto done;
 
-	desc = gpio_to_desc(gpio);
+	desc = gpio_to_valid_desc(gpio);
 	/* reject invalid GPIOs */
 	if (!desc) {
 		pr_warn("%s: invalid GPIO %ld\n", __func__, gpio);
@@ -493,7 +499,7 @@ static ssize_t unexport_store(struct class *class,
 	if (status < 0)
 		goto done;
 
-	desc = gpio_to_desc(gpio);
+	desc = gpio_to_valid_desc(gpio);
 	/* reject bogus commands (gpio_unexport ignores them) */
 	if (!desc) {
 		pr_warn("%s: invalid GPIO %ld\n", __func__, gpio);
@@ -534,8 +540,8 @@ static struct class gpio_class = {
 
 /**
  * gpiod_export - export a GPIO through sysfs
- * @gpio: gpio to make available, already requested
- * @direction_may_change: true if userspace may change gpio direction
+ * @desc: GPIO to make available, already requested
+ * @direction_may_change: true if userspace may change GPIO direction
  * Context: arch_initcall or later
  *
  * When drivers want to make a GPIO accessible to userspace after they
@@ -643,7 +649,7 @@ static int match_export(struct device *dev, const void *desc)
  * gpiod_export_link - create a sysfs link to an exported GPIO node
  * @dev: device under which to create symlink
  * @name: name of the symlink
- * @gpio: gpio to create symlink to, already exported
+ * @desc: GPIO to create symlink to, already exported
  *
  * Set up a symlink from /sys/.../dev/name to /sys/class/gpio/gpioN
  * node. Caller is responsible for unlinking.
@@ -674,7 +680,7 @@ EXPORT_SYMBOL_GPL(gpiod_export_link);
 
 /**
  * gpiod_unexport - reverse effect of gpiod_export()
- * @gpio: gpio to make unavailable
+ * @desc: GPIO to make unavailable
  *
  * This is implicit on gpiod_free().
  */
diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c
index cd003b74512f..eb80dac4e26a 100644
--- a/drivers/gpio/gpiolib.c
+++ b/drivers/gpio/gpiolib.c
@@ -84,7 +84,12 @@ static inline void desc_set_label(struct gpio_desc *d, const char *label)
 }
 
 /**
- * Convert a GPIO number to its descriptor
+ * gpio_to_desc - Convert a GPIO number to its descriptor
+ * @gpio: global GPIO number
+ *
+ * Returns:
+ * The GPIO descriptor associated with the given GPIO, or %NULL if no GPIO
+ * with the given number exists in the system.
  */
 struct gpio_desc *gpio_to_desc(unsigned gpio)
 {
@@ -111,7 +116,14 @@ struct gpio_desc *gpio_to_desc(unsigned gpio)
 EXPORT_SYMBOL_GPL(gpio_to_desc);
 
 /**
- * Get the GPIO descriptor corresponding to the given hw number for this chip.
+ * gpiochip_get_desc - get the GPIO descriptor corresponding to the given
+ *                     hardware number for this chip
+ * @chip: GPIO chip
+ * @hwnum: hardware number of the GPIO for this chip
+ *
+ * Returns:
+ * A pointer to the GPIO descriptor or %ERR_PTR(-EINVAL) if no GPIO exists
+ * in the given chip for the specified hardware number.
  */
 struct gpio_desc *gpiochip_get_desc(struct gpio_chip *chip,
 				    u16 hwnum)
@@ -125,9 +137,14 @@ struct gpio_desc *gpiochip_get_desc(struct gpio_chip *chip,
 }
 
 /**
- * Convert a GPIO descriptor to the integer namespace.
+ * desc_to_gpio - convert a GPIO descriptor to the integer namespace
+ * @desc: GPIO descriptor
+ *
  * This should disappear in the future but is needed since we still
- * use GPIO numbers for error messages and sysfs nodes
+ * use GPIO numbers for error messages and sysfs nodes.
+ *
+ * Returns:
+ * The global GPIO number for the GPIO specified by its descriptor.
  */
 int desc_to_gpio(const struct gpio_desc *desc)
 {
@@ -254,7 +271,7 @@ static int gpiodev_add_to_list(struct gpio_device *gdev)
 	return -EBUSY;
 }
 
-/**
+/*
  * Convert a GPIO name to its descriptor
  */
 static struct gpio_desc *gpio_name_to_desc(const char * const name)
@@ -878,7 +895,7 @@ out_free_le:
 	return ret;
 }
 
-/**
+/*
  * gpio_ioctl() - ioctl handler for the GPIO chardev
  */
 static long gpio_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
@@ -1077,11 +1094,9 @@ static void gpiochip_setup_devs(void)
 /**
  * gpiochip_add_data() - register a gpio_chip
  * @chip: the chip to register, with chip->base initialized
- * Context: potentially before irqs will work
+ * @data: driver-private data associated with this chip
  *
- * Returns a negative errno if the chip can't be registered, such as
- * because the chip->base is invalid or already associated with a
- * different chip.  Otherwise it returns zero as a success code.
+ * Context: potentially before irqs will work
  *
  * When gpiochip_add_data() is called very early during boot, so that GPIOs
  * can be freely used, the chip->parent device must be registered before
@@ -1093,6 +1108,11 @@ static void gpiochip_setup_devs(void)
  *
  * If chip->base is negative, this requests dynamic assignment of
  * a range of valid GPIOs.
+ *
+ * Returns:
+ * A negative errno if the chip can't be registered, such as because the
+ * chip->base is invalid or already associated with a different chip.
+ * Otherwise it returns zero as a success code.
  */
 int gpiochip_add_data(struct gpio_chip *chip, void *data)
 {
@@ -1287,6 +1307,10 @@ EXPORT_SYMBOL_GPL(gpiochip_add_data);
 
 /**
  * gpiochip_get_data() - get per-subdriver data for the chip
+ * @chip: GPIO chip
+ *
+ * Returns:
+ * The per-subdriver data for the chip.
  */
 void *gpiochip_get_data(struct gpio_chip *chip)
 {
@@ -1370,13 +1394,16 @@ static int devm_gpio_chip_match(struct device *dev, void *res, void *data)
  * devm_gpiochip_add_data() - Resource manager piochip_add_data()
  * @dev: the device pointer on which irq_chip belongs to.
  * @chip: the chip to register, with chip->base initialized
- * Context: potentially before irqs will work
+ * @data: driver-private data associated with this chip
  *
- * Returns a negative errno if the chip can't be registered, such as
- * because the chip->base is invalid or already associated with a
- * different chip.  Otherwise it returns zero as a success code.
+ * Context: potentially before irqs will work
  *
  * The gpio chip automatically be released when the device is unbound.
+ *
+ * Returns:
+ * A negative errno if the chip can't be registered, such as because the
+ * chip->base is invalid or already associated with a different chip.
+ * Otherwise it returns zero as a success code.
  */
 int devm_gpiochip_add_data(struct device *dev, struct gpio_chip *chip,
 			   void *data)
@@ -1422,7 +1449,7 @@ EXPORT_SYMBOL_GPL(devm_gpiochip_remove);
 /**
  * gpiochip_find() - iterator for locating a specific gpio_chip
  * @data: data to pass to match function
- * @callback: Callback function to check gpio_chip
+ * @match: Callback function to check gpio_chip
  *
  * Similar to bus_find_device.  It returns a reference to a gpio_chip as
  * determined by a user supplied @match callback.  The callback should return
@@ -1604,6 +1631,9 @@ static int gpiochip_irq_map(struct irq_domain *d, unsigned int irq,
 {
 	struct gpio_chip *chip = d->host_data;
 
+	if (!gpiochip_irqchip_irq_valid(chip, hwirq))
+		return -ENXIO;
+
 	irq_set_chip_data(irq, chip);
 	/*
 	 * This lock class tells lockdep that GPIO irqs are in a different
@@ -1670,7 +1700,9 @@ static void gpiochip_irq_relres(struct irq_data *d)
 
 static int gpiochip_to_irq(struct gpio_chip *chip, unsigned offset)
 {
-	return irq_find_mapping(chip->irqdomain, offset);
+	if (!gpiochip_irqchip_irq_valid(chip, offset))
+		return -ENXIO;
+	return irq_create_mapping(chip->irqdomain, offset);
 }
 
 /**
@@ -1746,9 +1778,6 @@ int gpiochip_irqchip_add_key(struct gpio_chip *gpiochip,
 			     struct lock_class_key *lock_key)
 {
 	struct device_node *of_node;
-	bool irq_base_set = false;
-	unsigned int offset;
-	unsigned irq_base = 0;
 
 	if (!gpiochip || !irqchip)
 		return -EINVAL;
@@ -1774,7 +1803,7 @@ int gpiochip_irqchip_add_key(struct gpio_chip *gpiochip,
 	 * conflicting triggers. Tell the user, and reset to NONE.
 	 */
 	if (WARN(of_node && type != IRQ_TYPE_NONE,
-		 "%s: Ignoring %d default trigger\n", of_node->full_name, type))
+		 "%pOF: Ignoring %d default trigger\n", of_node, type))
 		type = IRQ_TYPE_NONE;
 	if (has_acpi_companion(gpiochip->parent) && type != IRQ_TYPE_NONE) {
 		acpi_handle_warn(ACPI_HANDLE(gpiochip->parent),
@@ -1805,25 +1834,6 @@ int gpiochip_irqchip_add_key(struct gpio_chip *gpiochip,
 		irqchip->irq_release_resources = gpiochip_irq_relres;
 	}
 
-	/*
-	 * Prepare the mapping since the irqchip shall be orthogonal to
-	 * any gpiochip calls. If the first_irq was zero, this is
-	 * necessary to allocate descriptors for all IRQs.
-	 */
-	for (offset = 0; offset < gpiochip->ngpio; offset++) {
-		if (!gpiochip_irqchip_irq_valid(gpiochip, offset))
-			continue;
-		irq_base = irq_create_mapping(gpiochip->irqdomain, offset);
-		if (!irq_base_set) {
-			/*
-			 * Store the base into the gpiochip to be used when
-			 * unmapping the irqs.
-			 */
-			gpiochip->irq_base = irq_base;
-			irq_base_set = true;
-		}
-	}
-
 	acpi_gpiochip_request_interrupts(gpiochip);
 
 	return 0;
@@ -1930,11 +1940,14 @@ EXPORT_SYMBOL_GPL(gpiochip_add_pingroup_range);
 /**
  * gpiochip_add_pin_range() - add a range for GPIO <-> pin mapping
  * @chip: the gpiochip to add the range for
- * @pinctrl_name: the dev_name() of the pin controller to map to
+ * @pinctl_name: the dev_name() of the pin controller to map to
  * @gpio_offset: the start offset in the current gpio_chip number space
  * @pin_offset: the start offset in the pin controller number space
  * @npins: the number of pins from the offset of each pin space (GPIO and
  *	pin controller) to accumulate in this range
+ *
+ * Returns:
+ * 0 on success, or a negative error-code on failure.
  */
 int gpiochip_add_pin_range(struct gpio_chip *chip, const char *pinctl_name,
 			   unsigned int gpio_offset, unsigned int pin_offset,
@@ -2179,7 +2192,8 @@ EXPORT_SYMBOL_GPL(gpiochip_is_requested);
 
 /**
  * gpiochip_request_own_desc - Allow GPIO chip to request its own descriptor
- * @desc: GPIO descriptor to request
+ * @chip: GPIO chip
+ * @hwnum: hardware number of the GPIO for which to request the descriptor
  * @label: label for the GPIO
  *
  * Function allows GPIO chip drivers to request and use their own GPIO
@@ -2187,6 +2201,10 @@ EXPORT_SYMBOL_GPL(gpiochip_is_requested);
  * function will not increase reference count of the GPIO chip module. This
  * allows the GPIO chip module to be unloaded as needed (we assume that the
  * GPIO chip driver handles freeing the GPIOs it has requested).
+ *
+ * Returns:
+ * A pointer to the GPIO descriptor, or an ERR_PTR()-encoded negative error
+ * code on failure.
  */
 struct gpio_desc *gpiochip_request_own_desc(struct gpio_chip *chip, u16 hwnum,
 					    const char *label)
@@ -2368,12 +2386,13 @@ int gpiod_direction_output(struct gpio_desc *desc, int value)
 EXPORT_SYMBOL_GPL(gpiod_direction_output);
 
 /**
- * gpiod_set_debounce - sets @debounce time for a @gpio
- * @gpio: the gpio to set debounce time
- * @debounce: debounce time is microseconds
+ * gpiod_set_debounce - sets @debounce time for a GPIO
+ * @desc: descriptor of the GPIO for which to set debounce time
+ * @debounce: debounce time in microseconds
  *
- * returns -ENOTSUPP if the controller does not support setting
- * debounce.
+ * Returns:
+ * 0 on success, %-ENOTSUPP if the controller doesn't support setting the
+ * debounce time.
  */
 int gpiod_set_debounce(struct gpio_desc *desc, unsigned debounce)
 {
@@ -2981,6 +3000,23 @@ void gpiod_set_raw_array_value_cansleep(unsigned int array_size,
 EXPORT_SYMBOL_GPL(gpiod_set_raw_array_value_cansleep);
 
 /**
+ * gpiod_add_lookup_tables() - register GPIO device consumers
+ * @tables: list of tables of consumers to register
+ * @n: number of tables in the list
+ */
+void gpiod_add_lookup_tables(struct gpiod_lookup_table **tables, size_t n)
+{
+	unsigned int i;
+
+	mutex_lock(&gpio_lookup_lock);
+
+	for (i = 0; i < n; i++)
+		list_add_tail(&tables[i]->list, &gpio_lookup_list);
+
+	mutex_unlock(&gpio_lookup_lock);
+}
+
+/**
  * gpiod_set_array_value_cansleep() - assign values to an array of GPIOs
  * @array_size: number of elements in the descriptor / value arrays
  * @desc_array: array of GPIO descriptors whose values will be assigned
@@ -3322,6 +3358,7 @@ EXPORT_SYMBOL_GPL(gpiod_get_index);
  * @propname:	name of the firmware property representing the GPIO
  * @index:	index of the GPIO to obtain in the consumer
  * @dflags:	GPIO initialization flags
+ * @label:	label to attach to the requested GPIO
  *
  * This function can be used for drivers that get their configuration
  * from firmware.
@@ -3330,6 +3367,7 @@ EXPORT_SYMBOL_GPL(gpiod_get_index);
  * underlying firmware interface and then makes sure that the GPIO
  * descriptor is requested before it is returned to the caller.
  *
+ * Returns:
  * On successful request the GPIO pin is configured in accordance with
  * provided @dflags.
  *
diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h
index a8be286eff86..d003ccb12781 100644
--- a/drivers/gpio/gpiolib.h
+++ b/drivers/gpio/gpiolib.h
@@ -219,7 +219,7 @@ int gpiod_hog(struct gpio_desc *desc, const char *name,
 /*
  * Return the GPIO number of the passed descriptor relative to its chip
  */
-static int __maybe_unused gpio_chip_hwgpio(const struct gpio_desc *desc)
+static inline int gpio_chip_hwgpio(const struct gpio_desc *desc)
 {
 	return desc - &desc->gdev->descs[0];
 }
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index 24a066e1841c..a8acc197dec3 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -33,7 +33,7 @@ drm_kms_helper-y := drm_crtc_helper.o drm_dp_helper.o drm_probe_helper.o \
 		drm_plane_helper.o drm_dp_mst_topology.o drm_atomic_helper.o \
 		drm_kms_helper_common.o drm_dp_dual_mode_helper.o \
 		drm_simple_kms_helper.o drm_modeset_helper.o \
-		drm_scdc_helper.o
+		drm_scdc_helper.o drm_gem_framebuffer_helper.o
 
 drm_kms_helper-$(CONFIG_DRM_PANEL_BRIDGE) += bridge/panel.o
 drm_kms_helper-$(CONFIG_DRM_LOAD_EDID_FIRMWARE) += drm_edid_load.o
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index faea6349228f..658bac0cdc5e 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -25,7 +25,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
 	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
 	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
 	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o amdgpu_atomfirmware.o \
-	amdgpu_queue_mgr.o
+	amdgpu_queue_mgr.o amdgpu_vf_error.o
 
 # add asic specific block
 amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index ff7bf1a9f967..103635ab784c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -68,13 +68,16 @@
 
 #include "gpu_scheduler.h"
 #include "amdgpu_virt.h"
+#include "amdgpu_gart.h"
 
 /*
  * Modules parameters.
  */
 extern int amdgpu_modeset;
 extern int amdgpu_vram_limit;
+extern int amdgpu_vis_vram_limit;
 extern int amdgpu_gart_size;
+extern int amdgpu_gtt_size;
 extern int amdgpu_moverate;
 extern int amdgpu_benchmarking;
 extern int amdgpu_testing;
@@ -93,6 +96,7 @@ extern int amdgpu_bapm;
 extern int amdgpu_deep_color;
 extern int amdgpu_vm_size;
 extern int amdgpu_vm_block_size;
+extern int amdgpu_vm_fragment_size;
 extern int amdgpu_vm_fault_stop;
 extern int amdgpu_vm_debug;
 extern int amdgpu_vm_update_mode;
@@ -104,6 +108,7 @@ extern unsigned amdgpu_pcie_gen_cap;
 extern unsigned amdgpu_pcie_lane_cap;
 extern unsigned amdgpu_cg_mask;
 extern unsigned amdgpu_pg_mask;
+extern unsigned amdgpu_sdma_phase_quantum;
 extern char *amdgpu_disable_cu;
 extern char *amdgpu_virtual_display;
 extern unsigned amdgpu_pp_feature_mask;
@@ -369,78 +374,10 @@ struct amdgpu_clock {
 };
 
 /*
- * BO.
+ * GEM.
  */
-struct amdgpu_bo_list_entry {
-	struct amdgpu_bo		*robj;
-	struct ttm_validate_buffer	tv;
-	struct amdgpu_bo_va		*bo_va;
-	uint32_t			priority;
-	struct page			**user_pages;
-	int				user_invalidated;
-};
-
-struct amdgpu_bo_va_mapping {
-	struct list_head		list;
-	struct rb_node			rb;
-	uint64_t			start;
-	uint64_t			last;
-	uint64_t			__subtree_last;
-	uint64_t			offset;
-	uint64_t			flags;
-};
-
-/* bo virtual addresses in a specific vm */
-struct amdgpu_bo_va {
-	/* protected by bo being reserved */
-	struct list_head		bo_list;
-	struct dma_fence	        *last_pt_update;
-	unsigned			ref_count;
-
-	/* protected by vm mutex and spinlock */
-	struct list_head		vm_status;
-
-	/* mappings for this bo_va */
-	struct list_head		invalids;
-	struct list_head		valids;
-
-	/* constant after initialization */
-	struct amdgpu_vm		*vm;
-	struct amdgpu_bo		*bo;
-};
 
 #define AMDGPU_GEM_DOMAIN_MAX		0x3
-
-struct amdgpu_bo {
-	/* Protected by tbo.reserved */
-	u32				prefered_domains;
-	u32				allowed_domains;
-	struct ttm_place		placements[AMDGPU_GEM_DOMAIN_MAX + 1];
-	struct ttm_placement		placement;
-	struct ttm_buffer_object	tbo;
-	struct ttm_bo_kmap_obj		kmap;
-	u64				flags;
-	unsigned			pin_count;
-	void				*kptr;
-	u64				tiling_flags;
-	u64				metadata_flags;
-	void				*metadata;
-	u32				metadata_size;
-	unsigned			prime_shared_count;
-	/* list of all virtual address to which this bo
-	 * is associated to
-	 */
-	struct list_head		va;
-	/* Constant after initialization */
-	struct drm_gem_object		gem_base;
-	struct amdgpu_bo		*parent;
-	struct amdgpu_bo		*shadow;
-
-	struct ttm_bo_kmap_obj		dma_buf_vmap;
-	struct amdgpu_mn		*mn;
-	struct list_head		mn_list;
-	struct list_head		shadow_list;
-};
 #define gem_to_amdgpu_bo(gobj) container_of((gobj), struct amdgpu_bo, gem_base)
 
 void amdgpu_gem_object_free(struct drm_gem_object *obj);
@@ -532,49 +469,6 @@ int amdgpu_fence_slab_init(void);
 void amdgpu_fence_slab_fini(void);
 
 /*
- * GART structures, functions & helpers
- */
-struct amdgpu_mc;
-
-#define AMDGPU_GPU_PAGE_SIZE 4096
-#define AMDGPU_GPU_PAGE_MASK (AMDGPU_GPU_PAGE_SIZE - 1)
-#define AMDGPU_GPU_PAGE_SHIFT 12
-#define AMDGPU_GPU_PAGE_ALIGN(a) (((a) + AMDGPU_GPU_PAGE_MASK) & ~AMDGPU_GPU_PAGE_MASK)
-
-struct amdgpu_gart {
-	dma_addr_t			table_addr;
-	struct amdgpu_bo		*robj;
-	void				*ptr;
-	unsigned			num_gpu_pages;
-	unsigned			num_cpu_pages;
-	unsigned			table_size;
-#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
-	struct page			**pages;
-#endif
-	bool				ready;
-
-	/* Asic default pte flags */
-	uint64_t			gart_pte_flags;
-
-	const struct amdgpu_gart_funcs *gart_funcs;
-};
-
-int amdgpu_gart_table_ram_alloc(struct amdgpu_device *adev);
-void amdgpu_gart_table_ram_free(struct amdgpu_device *adev);
-int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev);
-void amdgpu_gart_table_vram_free(struct amdgpu_device *adev);
-int amdgpu_gart_table_vram_pin(struct amdgpu_device *adev);
-void amdgpu_gart_table_vram_unpin(struct amdgpu_device *adev);
-int amdgpu_gart_init(struct amdgpu_device *adev);
-void amdgpu_gart_fini(struct amdgpu_device *adev);
-int amdgpu_gart_unbind(struct amdgpu_device *adev, uint64_t offset,
-			int pages);
-int amdgpu_gart_bind(struct amdgpu_device *adev, uint64_t offset,
-		     int pages, struct page **pagelist,
-		     dma_addr_t *dma_addr, uint64_t flags);
-int amdgpu_ttm_recover_gart(struct amdgpu_device *adev);
-
-/*
  * VMHUB structures, functions & helpers
  */
 struct amdgpu_vmhub {
@@ -598,22 +492,20 @@ struct amdgpu_mc {
 	 * about vram size near mc fb location */
 	u64			mc_vram_size;
 	u64			visible_vram_size;
-	u64			gtt_size;
-	u64			gtt_start;
-	u64			gtt_end;
+	u64			gart_size;
+	u64			gart_start;
+	u64			gart_end;
 	u64			vram_start;
 	u64			vram_end;
 	unsigned		vram_width;
 	u64			real_vram_size;
 	int			vram_mtrr;
-	u64                     gtt_base_align;
 	u64                     mc_mask;
 	const struct firmware   *fw;	/* MC firmware */
 	uint32_t                fw_version;
 	struct amdgpu_irq_src	vm_fault;
 	uint32_t		vram_type;
 	uint32_t                srbm_soft_reset;
-	struct amdgpu_mode_mc_save save;
 	bool			prt_warning;
 	uint64_t		stolen_size;
 	/* apertures */
@@ -719,15 +611,15 @@ typedef enum _AMDGPU_DOORBELL64_ASSIGNMENT
 	/* overlap the doorbell assignment with VCN as they are  mutually exclusive
 	 * VCE engine's doorbell is 32 bit and two VCE ring share one QWORD
 	 */
-	AMDGPU_DOORBELL64_RING0_1                 = 0xF8,
-	AMDGPU_DOORBELL64_RING2_3                 = 0xF9,
-	AMDGPU_DOORBELL64_RING4_5                 = 0xFA,
-	AMDGPU_DOORBELL64_RING6_7                 = 0xFB,
+	AMDGPU_DOORBELL64_UVD_RING0_1             = 0xF8,
+	AMDGPU_DOORBELL64_UVD_RING2_3             = 0xF9,
+	AMDGPU_DOORBELL64_UVD_RING4_5             = 0xFA,
+	AMDGPU_DOORBELL64_UVD_RING6_7             = 0xFB,
 
-	AMDGPU_DOORBELL64_UVD_RING0_1             = 0xFC,
-	AMDGPU_DOORBELL64_UVD_RING2_3             = 0xFD,
-	AMDGPU_DOORBELL64_UVD_RING4_5             = 0xFE,
-	AMDGPU_DOORBELL64_UVD_RING6_7             = 0xFF,
+	AMDGPU_DOORBELL64_VCE_RING0_1             = 0xFC,
+	AMDGPU_DOORBELL64_VCE_RING2_3             = 0xFD,
+	AMDGPU_DOORBELL64_VCE_RING4_5             = 0xFE,
+	AMDGPU_DOORBELL64_VCE_RING6_7             = 0xFF,
 
 	AMDGPU_DOORBELL64_MAX_ASSIGNMENT          = 0xFF,
 	AMDGPU_DOORBELL64_INVALID                 = 0xFFFF
@@ -857,6 +749,7 @@ void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr);
 struct amdgpu_fpriv {
 	struct amdgpu_vm	vm;
 	struct amdgpu_bo_va	*prt_va;
+	struct amdgpu_bo_va	*csa_va;
 	struct mutex		bo_list_lock;
 	struct idr		bo_list_handles;
 	struct amdgpu_ctx_mgr	ctx_mgr;
@@ -866,6 +759,14 @@ struct amdgpu_fpriv {
 /*
  * residency list
  */
+struct amdgpu_bo_list_entry {
+	struct amdgpu_bo		*robj;
+	struct ttm_validate_buffer	tv;
+	struct amdgpu_bo_va		*bo_va;
+	uint32_t			priority;
+	struct page			**user_pages;
+	int				user_invalidated;
+};
 
 struct amdgpu_bo_list {
 	struct mutex lock;
@@ -1159,7 +1060,9 @@ struct amdgpu_cs_parser {
 	struct list_head		validated;
 	struct dma_fence		*fence;
 	uint64_t			bytes_moved_threshold;
+	uint64_t			bytes_moved_vis_threshold;
 	uint64_t			bytes_moved;
+	uint64_t			bytes_moved_vis;
 	struct amdgpu_bo_list_entry	*evictable;
 
 	/* user fence */
@@ -1230,8 +1133,6 @@ struct amdgpu_wb {
 
 int amdgpu_wb_get(struct amdgpu_device *adev, u32 *wb);
 void amdgpu_wb_free(struct amdgpu_device *adev, u32 wb);
-int amdgpu_wb_get_64bit(struct amdgpu_device *adev, u32 *wb);
-void amdgpu_wb_free_64bit(struct amdgpu_device *adev, u32 wb);
 
 void amdgpu_get_pcie_info(struct amdgpu_device *adev);
 
@@ -1525,7 +1426,7 @@ struct amdgpu_device {
 	bool				is_atom_fw;
 	uint8_t				*bios;
 	uint32_t			bios_size;
-	struct amdgpu_bo		*stollen_vga_memory;
+	struct amdgpu_bo		*stolen_vga_memory;
 	uint32_t			bios_scratch_reg_offset;
 	uint32_t			bios_scratch[AMDGPU_BIOS_NUM_SCRATCH];
 
@@ -1557,6 +1458,10 @@ struct amdgpu_device {
 	spinlock_t gc_cac_idx_lock;
 	amdgpu_rreg_t			gc_cac_rreg;
 	amdgpu_wreg_t			gc_cac_wreg;
+	/* protects concurrent se_cac register access */
+	spinlock_t se_cac_idx_lock;
+	amdgpu_rreg_t			se_cac_rreg;
+	amdgpu_wreg_t			se_cac_wreg;
 	/* protects concurrent ENDPOINT (audio) register access */
 	spinlock_t audio_endpt_idx_lock;
 	amdgpu_block_rreg_t		audio_endpt_rreg;
@@ -1579,9 +1484,6 @@ struct amdgpu_device {
 	struct amdgpu_mman		mman;
 	struct amdgpu_vram_scratch	vram_scratch;
 	struct amdgpu_wb		wb;
-	atomic64_t			vram_usage;
-	atomic64_t			vram_vis_usage;
-	atomic64_t			gtt_usage;
 	atomic64_t			num_bytes_moved;
 	atomic64_t			num_evictions;
 	atomic64_t			num_vram_cpu_page_faults;
@@ -1593,6 +1495,7 @@ struct amdgpu_device {
 		spinlock_t		lock;
 		s64			last_update_us;
 		s64			accum_us; /* accumulated microseconds */
+		s64			accum_us_vis; /* for visible VRAM */
 		u32			log2_max_MBps;
 	} mm_stats;
 
@@ -1687,6 +1590,8 @@ struct amdgpu_device {
 	bool has_hw_reset;
 	u8				reset_magic[AMDGPU_RESET_MAGIC_NUM];
 
+	/* record last mm index being written through WREG32*/
+	unsigned long last_mm_index;
 };
 
 static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev)
@@ -1742,6 +1647,8 @@ void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v);
 #define WREG32_DIDT(reg, v) adev->didt_wreg(adev, (reg), (v))
 #define RREG32_GC_CAC(reg) adev->gc_cac_rreg(adev, (reg))
 #define WREG32_GC_CAC(reg, v) adev->gc_cac_wreg(adev, (reg), (v))
+#define RREG32_SE_CAC(reg) adev->se_cac_rreg(adev, (reg))
+#define WREG32_SE_CAC(reg, v) adev->se_cac_wreg(adev, (reg), (v))
 #define RREG32_AUDIO_ENDPT(block, reg) adev->audio_endpt_rreg(adev, (block), (reg))
 #define WREG32_AUDIO_ENDPT(block, reg, v) adev->audio_endpt_wreg(adev, (block), (reg), (v))
 #define WREG32_P(reg, val, mask)				\
@@ -1792,50 +1699,6 @@ void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v);
 #define RBIOS16(i) (RBIOS8(i) | (RBIOS8((i)+1) << 8))
 #define RBIOS32(i) ((RBIOS16(i)) | (RBIOS16((i)+2) << 16))
 
-/*
- * RING helpers.
- */
-static inline void amdgpu_ring_write(struct amdgpu_ring *ring, uint32_t v)
-{
-	if (ring->count_dw <= 0)
-		DRM_ERROR("amdgpu: writing more dwords to the ring than expected!\n");
-	ring->ring[ring->wptr++ & ring->buf_mask] = v;
-	ring->wptr &= ring->ptr_mask;
-	ring->count_dw--;
-}
-
-static inline void amdgpu_ring_write_multiple(struct amdgpu_ring *ring, void *src, int count_dw)
-{
-	unsigned occupied, chunk1, chunk2;
-	void *dst;
-
-	if (unlikely(ring->count_dw < count_dw)) {
-		DRM_ERROR("amdgpu: writing more dwords to the ring than expected!\n");
-		return;
-	}
-
-	occupied = ring->wptr & ring->buf_mask;
-	dst = (void *)&ring->ring[occupied];
-	chunk1 = ring->buf_mask + 1 - occupied;
-	chunk1 = (chunk1 >= count_dw) ? count_dw: chunk1;
-	chunk2 = count_dw - chunk1;
-	chunk1 <<= 2;
-	chunk2 <<= 2;
-
-	if (chunk1)
-		memcpy(dst, src, chunk1);
-
-	if (chunk2) {
-		src += chunk1;
-		dst = (void *)ring->ring;
-		memcpy(dst, src, chunk2);
-	}
-
-	ring->wptr += count_dw;
-	ring->wptr &= ring->ptr_mask;
-	ring->count_dw -= count_dw;
-}
-
 static inline struct amdgpu_sdma_instance *
 amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 {
@@ -1898,7 +1761,6 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_ih_get_wptr(adev) (adev)->irq.ih_funcs->get_wptr((adev))
 #define amdgpu_ih_decode_iv(adev, iv) (adev)->irq.ih_funcs->decode_iv((adev), (iv))
 #define amdgpu_ih_set_rptr(adev) (adev)->irq.ih_funcs->set_rptr((adev))
-#define amdgpu_display_set_vga_render_state(adev, r) (adev)->mode_info.funcs->set_vga_render_state((adev), (r))
 #define amdgpu_display_vblank_get_counter(adev, crtc) (adev)->mode_info.funcs->vblank_get_counter((adev), (crtc))
 #define amdgpu_display_vblank_wait(adev, crtc) (adev)->mode_info.funcs->vblank_wait((adev), (crtc))
 #define amdgpu_display_backlight_set_level(adev, e, l) (adev)->mode_info.funcs->backlight_set_level((e), (l))
@@ -1911,8 +1773,6 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_display_page_flip_get_scanoutpos(adev, crtc, vbl, pos) (adev)->mode_info.funcs->page_flip_get_scanoutpos((adev), (crtc), (vbl), (pos))
 #define amdgpu_display_add_encoder(adev, e, s, c) (adev)->mode_info.funcs->add_encoder((adev), (e), (s), (c))
 #define amdgpu_display_add_connector(adev, ci, sd, ct, ib, coi, h, r) (adev)->mode_info.funcs->add_connector((adev), (ci), (sd), (ct), (ib), (coi), (h), (r))
-#define amdgpu_display_stop_mc_access(adev, s) (adev)->mode_info.funcs->stop_mc_access((adev), (s))
-#define amdgpu_display_resume_mc_access(adev, s) (adev)->mode_info.funcs->resume_mc_access((adev), (s))
 #define amdgpu_emit_copy_buffer(adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_copy_buffer((ib),  (s), (d), (b))
 #define amdgpu_emit_fill_buffer(adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_fill_buffer((ib), (s), (d), (b))
 #define amdgpu_gfx_get_gpu_clock_counter(adev) (adev)->gfx.funcs->get_gpu_clock_counter((adev))
@@ -1927,7 +1787,8 @@ void amdgpu_pci_config_reset(struct amdgpu_device *adev);
 bool amdgpu_need_post(struct amdgpu_device *adev);
 void amdgpu_update_display_priority(struct amdgpu_device *adev);
 
-void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes);
+void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes,
+				  u64 num_vis_bytes);
 void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain);
 bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo);
 int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages);
@@ -1943,7 +1804,7 @@ bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm);
 uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
 				 struct ttm_mem_reg *mem);
 void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64 base);
-void amdgpu_gtt_location(struct amdgpu_device *adev, struct amdgpu_mc *mc);
+void amdgpu_gart_location(struct amdgpu_device *adev, struct amdgpu_mc *mc);
 void amdgpu_ttm_set_active_vram_size(struct amdgpu_device *adev, u64 size);
 int amdgpu_ttm_init(struct amdgpu_device *adev);
 void amdgpu_ttm_fini(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
index 06879d1dcabd..a52795d9b458 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
@@ -285,19 +285,20 @@ static int acp_hw_init(void *handle)
 		return 0;
 	else if (r)
 		return r;
+	if (adev->asic_type != CHIP_STONEY) {
+		adev->acp.acp_genpd = kzalloc(sizeof(struct acp_pm_domain), GFP_KERNEL);
+		if (adev->acp.acp_genpd == NULL)
+			return -ENOMEM;
 
-	adev->acp.acp_genpd = kzalloc(sizeof(struct acp_pm_domain), GFP_KERNEL);
-	if (adev->acp.acp_genpd == NULL)
-		return -ENOMEM;
-
-	adev->acp.acp_genpd->gpd.name = "ACP_AUDIO";
-	adev->acp.acp_genpd->gpd.power_off = acp_poweroff;
-	adev->acp.acp_genpd->gpd.power_on = acp_poweron;
+		adev->acp.acp_genpd->gpd.name = "ACP_AUDIO";
+		adev->acp.acp_genpd->gpd.power_off = acp_poweroff;
+		adev->acp.acp_genpd->gpd.power_on = acp_poweron;
 
 
-	adev->acp.acp_genpd->cgs_dev = adev->acp.cgs_device;
+		adev->acp.acp_genpd->cgs_dev = adev->acp.cgs_device;
 
-	pm_genpd_init(&adev->acp.acp_genpd->gpd, NULL, false);
+		pm_genpd_init(&adev->acp.acp_genpd->gpd, NULL, false);
+	}
 
 	adev->acp.acp_cell = kzalloc(sizeof(struct mfd_cell) * ACP_DEVS,
 							GFP_KERNEL);
@@ -319,14 +320,29 @@ static int acp_hw_init(void *handle)
 		return -ENOMEM;
 	}
 
-	i2s_pdata[0].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET;
+	switch (adev->asic_type) {
+	case CHIP_STONEY:
+		i2s_pdata[0].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET |
+			DW_I2S_QUIRK_16BIT_IDX_OVERRIDE;
+		break;
+	default:
+		i2s_pdata[0].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET;
+	}
 	i2s_pdata[0].cap = DWC_I2S_PLAY;
 	i2s_pdata[0].snd_rates = SNDRV_PCM_RATE_8000_96000;
 	i2s_pdata[0].i2s_reg_comp1 = ACP_I2S_COMP1_PLAY_REG_OFFSET;
 	i2s_pdata[0].i2s_reg_comp2 = ACP_I2S_COMP2_PLAY_REG_OFFSET;
+	switch (adev->asic_type) {
+	case CHIP_STONEY:
+		i2s_pdata[1].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET |
+			DW_I2S_QUIRK_COMP_PARAM1 |
+			DW_I2S_QUIRK_16BIT_IDX_OVERRIDE;
+		break;
+	default:
+		i2s_pdata[1].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET |
+			DW_I2S_QUIRK_COMP_PARAM1;
+	}
 
-	i2s_pdata[1].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET |
-				DW_I2S_QUIRK_COMP_PARAM1;
 	i2s_pdata[1].cap = DWC_I2S_RECORD;
 	i2s_pdata[1].snd_rates = SNDRV_PCM_RATE_8000_96000;
 	i2s_pdata[1].i2s_reg_comp1 = ACP_I2S_COMP1_CAP_REG_OFFSET;
@@ -373,12 +389,14 @@ static int acp_hw_init(void *handle)
 	if (r)
 		return r;
 
-	for (i = 0; i < ACP_DEVS ; i++) {
-		dev = get_mfd_cell_dev(adev->acp.acp_cell[i].name, i);
-		r = pm_genpd_add_device(&adev->acp.acp_genpd->gpd, dev);
-		if (r) {
-			dev_err(dev, "Failed to add dev to genpd\n");
-			return r;
+	if (adev->asic_type != CHIP_STONEY) {
+		for (i = 0; i < ACP_DEVS ; i++) {
+			dev = get_mfd_cell_dev(adev->acp.acp_cell[i].name, i);
+			r = pm_genpd_add_device(&adev->acp.acp_genpd->gpd, dev);
+			if (r) {
+				dev_err(dev, "Failed to add dev to genpd\n");
+				return r;
+			}
 		}
 	}
 
@@ -398,20 +416,22 @@ static int acp_hw_fini(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	/* return early if no ACP */
-	if (!adev->acp.acp_genpd)
+	if (!adev->acp.acp_cell)
 		return 0;
 
-	for (i = 0; i < ACP_DEVS ; i++) {
-		dev = get_mfd_cell_dev(adev->acp.acp_cell[i].name, i);
-		ret = pm_genpd_remove_device(&adev->acp.acp_genpd->gpd, dev);
-		/* If removal fails, dont giveup and try rest */
-		if (ret)
-			dev_err(dev, "remove dev from genpd failed\n");
+	if (adev->acp.acp_genpd) {
+		for (i = 0; i < ACP_DEVS ; i++) {
+			dev = get_mfd_cell_dev(adev->acp.acp_cell[i].name, i);
+			ret = pm_genpd_remove_device(&adev->acp.acp_genpd->gpd, dev);
+			/* If removal fails, dont giveup and try rest */
+			if (ret)
+				dev_err(dev, "remove dev from genpd failed\n");
+		}
+		kfree(adev->acp.acp_genpd);
 	}
 
 	mfd_remove_devices(adev->acp.parent);
 	kfree(adev->acp.acp_res);
-	kfree(adev->acp.acp_genpd);
 	kfree(adev->acp.acp_cell);
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
index ef79551b4cb7..57afad79f55d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
@@ -30,10 +30,10 @@
 #include <drm/drmP.h>
 #include <drm/drm_crtc_helper.h>
 #include "amdgpu.h"
+#include "amdgpu_pm.h"
 #include "amd_acpi.h"
 #include "atom.h"
 
-extern void amdgpu_pm_acpi_event_handler(struct amdgpu_device *adev);
 /* Call the ATIF method
  */
 /**
@@ -289,7 +289,7 @@ out:
  * handles it.
  * Returns NOTIFY code
  */
-int amdgpu_atif_handler(struct amdgpu_device *adev,
+static int amdgpu_atif_handler(struct amdgpu_device *adev,
 			struct acpi_bus_event *event)
 {
 	struct amdgpu_atif *atif = &adev->atif;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index 37971d9402e3..5432af39a674 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -27,16 +27,15 @@
 #include "amdgpu_gfx.h"
 #include <linux/module.h>
 
-const struct kfd2kgd_calls *kfd2kgd;
 const struct kgd2kfd_calls *kgd2kfd;
-bool (*kgd2kfd_init_p)(unsigned, const struct kgd2kfd_calls**);
+bool (*kgd2kfd_init_p)(unsigned int, const struct kgd2kfd_calls**);
 
 int amdgpu_amdkfd_init(void)
 {
 	int ret;
 
 #if defined(CONFIG_HSA_AMD_MODULE)
-	int (*kgd2kfd_init_p)(unsigned, const struct kgd2kfd_calls**);
+	int (*kgd2kfd_init_p)(unsigned int, const struct kgd2kfd_calls**);
 
 	kgd2kfd_init_p = symbol_request(kgd2kfd_init);
 
@@ -61,8 +60,21 @@ int amdgpu_amdkfd_init(void)
 	return ret;
 }
 
-bool amdgpu_amdkfd_load_interface(struct amdgpu_device *adev)
+void amdgpu_amdkfd_fini(void)
+{
+	if (kgd2kfd) {
+		kgd2kfd->exit();
+		symbol_put(kgd2kfd_init);
+	}
+}
+
+void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev)
 {
+	const struct kfd2kgd_calls *kfd2kgd;
+
+	if (!kgd2kfd)
+		return;
+
 	switch (adev->asic_type) {
 #ifdef CONFIG_DRM_AMDGPU_CIK
 	case CHIP_KAVERI:
@@ -73,25 +85,12 @@ bool amdgpu_amdkfd_load_interface(struct amdgpu_device *adev)
 		kfd2kgd = amdgpu_amdkfd_gfx_8_0_get_functions();
 		break;
 	default:
-		return false;
+		dev_info(adev->dev, "kfd not supported on this ASIC\n");
+		return;
 	}
 
-	return true;
-}
-
-void amdgpu_amdkfd_fini(void)
-{
-	if (kgd2kfd) {
-		kgd2kfd->exit();
-		symbol_put(kgd2kfd_init);
-	}
-}
-
-void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev)
-{
-	if (kgd2kfd)
-		adev->kfd = kgd2kfd->probe((struct kgd_dev *)adev,
-					adev->pdev, kfd2kgd);
+	adev->kfd = kgd2kfd->probe((struct kgd_dev *)adev,
+				   adev->pdev, kfd2kgd);
 }
 
 void amdgpu_amdkfd_device_init(struct amdgpu_device *adev)
@@ -184,7 +183,8 @@ int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
 		return -ENOMEM;
 
 	r = amdgpu_bo_create(adev, size, PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_GTT,
-			     AMDGPU_GEM_CREATE_CPU_GTT_USWC, NULL, NULL, &(*mem)->bo);
+			     AMDGPU_GEM_CREATE_CPU_GTT_USWC, NULL, NULL, 0,
+			     &(*mem)->bo);
 	if (r) {
 		dev_err(adev->dev,
 			"failed to allocate BO for amdkfd (%d)\n", r);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 73f83a10ae14..8d689ab7e429 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -26,6 +26,7 @@
 #define AMDGPU_AMDKFD_H_INCLUDED
 
 #include <linux/types.h>
+#include <linux/mmu_context.h>
 #include <kgd_kfd_interface.h>
 
 struct amdgpu_device;
@@ -39,8 +40,6 @@ struct kgd_mem {
 int amdgpu_amdkfd_init(void);
 void amdgpu_amdkfd_fini(void);
 
-bool amdgpu_amdkfd_load_interface(struct amdgpu_device *adev);
-
 void amdgpu_amdkfd_suspend(struct amdgpu_device *adev);
 int amdgpu_amdkfd_resume(struct amdgpu_device *adev);
 void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev,
@@ -62,4 +61,19 @@ uint64_t get_gpu_clock_counter(struct kgd_dev *kgd);
 
 uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
 
+#define read_user_wptr(mmptr, wptr, dst)				\
+	({								\
+		bool valid = false;					\
+		if ((mmptr) && (wptr)) {				\
+			if ((mmptr) == current->mm) {			\
+				valid = !get_user((dst), (wptr));	\
+			} else if (current->mm == NULL) {		\
+				use_mm(mmptr);				\
+				valid = !get_user((dst), (wptr));	\
+				unuse_mm(mmptr);			\
+			}						\
+		}							\
+		valid;							\
+	})
+
 #endif /* AMDGPU_AMDKFD_H_INCLUDED */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
index 5254562fd0f9..b9dbbf9cb8b0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
@@ -39,6 +39,12 @@
 #include "gmc/gmc_7_1_sh_mask.h"
 #include "cik_structs.h"
 
+enum hqd_dequeue_request_type {
+	NO_ACTION = 0,
+	DRAIN_PIPE,
+	RESET_WAVES
+};
+
 enum {
 	MAX_TRAPID = 8,		/* 3 bits in the bitfield. */
 	MAX_WATCH_ADDRESSES = 4
@@ -96,12 +102,15 @@ static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
 				uint32_t hpd_size, uint64_t hpd_gpu_addr);
 static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id);
 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
-			uint32_t queue_id, uint32_t __user *wptr);
+			uint32_t queue_id, uint32_t __user *wptr,
+			uint32_t wptr_shift, uint32_t wptr_mask,
+			struct mm_struct *mm);
 static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd);
 static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
 				uint32_t pipe_id, uint32_t queue_id);
 
-static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
+static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
+				enum kfd_preempt_type reset_type,
 				unsigned int utimeout, uint32_t pipe_id,
 				uint32_t queue_id);
 static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
@@ -126,6 +135,33 @@ static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
 static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid);
 
 static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type);
+static void set_scratch_backing_va(struct kgd_dev *kgd,
+					uint64_t va, uint32_t vmid);
+
+/* Because of REG_GET_FIELD() being used, we put this function in the
+ * asic specific file.
+ */
+static int get_tile_config(struct kgd_dev *kgd,
+		struct tile_config *config)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+
+	config->gb_addr_config = adev->gfx.config.gb_addr_config;
+	config->num_banks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg,
+				MC_ARB_RAMCFG, NOOFBANK);
+	config->num_ranks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg,
+				MC_ARB_RAMCFG, NOOFRANKS);
+
+	config->tile_config_ptr = adev->gfx.config.tile_mode_array;
+	config->num_tile_configs =
+			ARRAY_SIZE(adev->gfx.config.tile_mode_array);
+	config->macro_tile_config_ptr =
+			adev->gfx.config.macrotile_mode_array;
+	config->num_macro_tile_configs =
+			ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
+
+	return 0;
+}
 
 static const struct kfd2kgd_calls kfd2kgd = {
 	.init_gtt_mem_allocation = alloc_gtt_mem,
@@ -150,7 +186,9 @@ static const struct kfd2kgd_calls kfd2kgd = {
 	.get_atc_vmid_pasid_mapping_pasid = get_atc_vmid_pasid_mapping_pasid,
 	.get_atc_vmid_pasid_mapping_valid = get_atc_vmid_pasid_mapping_valid,
 	.write_vmid_invalidate_request = write_vmid_invalidate_request,
-	.get_fw_version = get_fw_version
+	.get_fw_version = get_fw_version,
+	.set_scratch_backing_va = set_scratch_backing_va,
+	.get_tile_config = get_tile_config,
 };
 
 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void)
@@ -186,7 +224,7 @@ static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,
 {
 	struct amdgpu_device *adev = get_amdgpu_device(kgd);
 
-	uint32_t mec = (++pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
+	uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
 	uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
 
 	lock_srbm(kgd, mec, pipe, queue_id, 0);
@@ -290,20 +328,38 @@ static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
 }
 
 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
-			uint32_t queue_id, uint32_t __user *wptr)
+			uint32_t queue_id, uint32_t __user *wptr,
+			uint32_t wptr_shift, uint32_t wptr_mask,
+			struct mm_struct *mm)
 {
 	struct amdgpu_device *adev = get_amdgpu_device(kgd);
-	uint32_t wptr_shadow, is_wptr_shadow_valid;
 	struct cik_mqd *m;
+	uint32_t *mqd_hqd;
+	uint32_t reg, wptr_val, data;
 
 	m = get_mqd(mqd);
 
-	is_wptr_shadow_valid = !get_user(wptr_shadow, wptr);
-	if (is_wptr_shadow_valid)
-		m->cp_hqd_pq_wptr = wptr_shadow;
-
 	acquire_queue(kgd, pipe_id, queue_id);
-	gfx_v7_0_mqd_commit(adev, m);
+
+	/* HQD registers extend from CP_MQD_BASE_ADDR to CP_MQD_CONTROL. */
+	mqd_hqd = &m->cp_mqd_base_addr_lo;
+
+	for (reg = mmCP_MQD_BASE_ADDR; reg <= mmCP_MQD_CONTROL; reg++)
+		WREG32(reg, mqd_hqd[reg - mmCP_MQD_BASE_ADDR]);
+
+	/* Copy userspace write pointer value to register.
+	 * Activate doorbell logic to monitor subsequent changes.
+	 */
+	data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control,
+			     CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
+	WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data);
+
+	if (read_user_wptr(mm, wptr, wptr_val))
+		WREG32(mmCP_HQD_PQ_WPTR, (wptr_val << wptr_shift) & wptr_mask);
+
+	data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
+	WREG32(mmCP_HQD_ACTIVE, data);
+
 	release_queue(kgd);
 
 	return 0;
@@ -382,30 +438,99 @@ static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
 	return false;
 }
 
-static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
+static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
+				enum kfd_preempt_type reset_type,
 				unsigned int utimeout, uint32_t pipe_id,
 				uint32_t queue_id)
 {
 	struct amdgpu_device *adev = get_amdgpu_device(kgd);
 	uint32_t temp;
-	int timeout = utimeout;
+	enum hqd_dequeue_request_type type;
+	unsigned long flags, end_jiffies;
+	int retry;
 
 	acquire_queue(kgd, pipe_id, queue_id);
 	WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
 
-	WREG32(mmCP_HQD_DEQUEUE_REQUEST, reset_type);
+	switch (reset_type) {
+	case KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN:
+		type = DRAIN_PIPE;
+		break;
+	case KFD_PREEMPT_TYPE_WAVEFRONT_RESET:
+		type = RESET_WAVES;
+		break;
+	default:
+		type = DRAIN_PIPE;
+		break;
+	}
+
+	/* Workaround: If IQ timer is active and the wait time is close to or
+	 * equal to 0, dequeueing is not safe. Wait until either the wait time
+	 * is larger or timer is cleared. Also, ensure that IQ_REQ_PEND is
+	 * cleared before continuing. Also, ensure wait times are set to at
+	 * least 0x3.
+	 */
+	local_irq_save(flags);
+	preempt_disable();
+	retry = 5000; /* wait for 500 usecs at maximum */
+	while (true) {
+		temp = RREG32(mmCP_HQD_IQ_TIMER);
+		if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, PROCESSING_IQ)) {
+			pr_debug("HW is processing IQ\n");
+			goto loop;
+		}
+		if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, ACTIVE)) {
+			if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, RETRY_TYPE)
+					== 3) /* SEM-rearm is safe */
+				break;
+			/* Wait time 3 is safe for CP, but our MMIO read/write
+			 * time is close to 1 microsecond, so check for 10 to
+			 * leave more buffer room
+			 */
+			if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, WAIT_TIME)
+					>= 10)
+				break;
+			pr_debug("IQ timer is active\n");
+		} else
+			break;
+loop:
+		if (!retry) {
+			pr_err("CP HQD IQ timer status time out\n");
+			break;
+		}
+		ndelay(100);
+		--retry;
+	}
+	retry = 1000;
+	while (true) {
+		temp = RREG32(mmCP_HQD_DEQUEUE_REQUEST);
+		if (!(temp & CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_MASK))
+			break;
+		pr_debug("Dequeue request is pending\n");
 
+		if (!retry) {
+			pr_err("CP HQD dequeue request time out\n");
+			break;
+		}
+		ndelay(100);
+		--retry;
+	}
+	local_irq_restore(flags);
+	preempt_enable();
+
+	WREG32(mmCP_HQD_DEQUEUE_REQUEST, type);
+
+	end_jiffies = (utimeout * HZ / 1000) + jiffies;
 	while (true) {
 		temp = RREG32(mmCP_HQD_ACTIVE);
-		if (temp & CP_HQD_ACTIVE__ACTIVE_MASK)
+		if (!(temp & CP_HQD_ACTIVE__ACTIVE_MASK))
 			break;
-		if (timeout <= 0) {
-			pr_err("kfd: cp queue preemption time out.\n");
+		if (time_after(jiffies, end_jiffies)) {
+			pr_err("cp queue preemption time out\n");
 			release_queue(kgd);
 			return -ETIME;
 		}
-		msleep(20);
-		timeout -= 20;
+		usleep_range(500, 1000);
 	}
 
 	release_queue(kgd);
@@ -556,6 +681,16 @@ static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid)
 	WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
 }
 
+static void set_scratch_backing_va(struct kgd_dev *kgd,
+					uint64_t va, uint32_t vmid)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+
+	lock_srbm(kgd, 0, 0, 0, vmid);
+	WREG32(mmSH_HIDDEN_PRIVATE_BASE_VMID, va);
+	unlock_srbm(kgd);
+}
+
 static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
@@ -566,42 +701,42 @@ static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)
 	switch (type) {
 	case KGD_ENGINE_PFP:
 		hdr = (const union amdgpu_firmware_header *)
-							adev->gfx.pfp_fw->data;
+						adev->gfx.pfp_fw->data;
 		break;
 
 	case KGD_ENGINE_ME:
 		hdr = (const union amdgpu_firmware_header *)
-							adev->gfx.me_fw->data;
+						adev->gfx.me_fw->data;
 		break;
 
 	case KGD_ENGINE_CE:
 		hdr = (const union amdgpu_firmware_header *)
-							adev->gfx.ce_fw->data;
+						adev->gfx.ce_fw->data;
 		break;
 
 	case KGD_ENGINE_MEC1:
 		hdr = (const union amdgpu_firmware_header *)
-							adev->gfx.mec_fw->data;
+						adev->gfx.mec_fw->data;
 		break;
 
 	case KGD_ENGINE_MEC2:
 		hdr = (const union amdgpu_firmware_header *)
-							adev->gfx.mec2_fw->data;
+						adev->gfx.mec2_fw->data;
 		break;
 
 	case KGD_ENGINE_RLC:
 		hdr = (const union amdgpu_firmware_header *)
-							adev->gfx.rlc_fw->data;
+						adev->gfx.rlc_fw->data;
 		break;
 
 	case KGD_ENGINE_SDMA1:
 		hdr = (const union amdgpu_firmware_header *)
-							adev->sdma.instance[0].fw->data;
+						adev->sdma.instance[0].fw->data;
 		break;
 
 	case KGD_ENGINE_SDMA2:
 		hdr = (const union amdgpu_firmware_header *)
-							adev->sdma.instance[1].fw->data;
+						adev->sdma.instance[1].fw->data;
 		break;
 
 	default:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
index 133d06671e46..309f2419c6d8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
@@ -39,6 +39,12 @@
 #include "vi_structs.h"
 #include "vid.h"
 
+enum hqd_dequeue_request_type {
+	NO_ACTION = 0,
+	DRAIN_PIPE,
+	RESET_WAVES
+};
+
 struct cik_sdma_rlc_registers;
 
 /*
@@ -55,12 +61,15 @@ static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
 		uint32_t hpd_size, uint64_t hpd_gpu_addr);
 static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id);
 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
-		uint32_t queue_id, uint32_t __user *wptr);
+			uint32_t queue_id, uint32_t __user *wptr,
+			uint32_t wptr_shift, uint32_t wptr_mask,
+			struct mm_struct *mm);
 static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd);
 static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
 		uint32_t pipe_id, uint32_t queue_id);
 static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
-static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
+static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
+				enum kfd_preempt_type reset_type,
 				unsigned int utimeout, uint32_t pipe_id,
 				uint32_t queue_id);
 static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
@@ -85,6 +94,33 @@ static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
 		uint8_t vmid);
 static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid);
 static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type);
+static void set_scratch_backing_va(struct kgd_dev *kgd,
+					uint64_t va, uint32_t vmid);
+
+/* Because of REG_GET_FIELD() being used, we put this function in the
+ * asic specific file.
+ */
+static int get_tile_config(struct kgd_dev *kgd,
+		struct tile_config *config)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+
+	config->gb_addr_config = adev->gfx.config.gb_addr_config;
+	config->num_banks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg,
+				MC_ARB_RAMCFG, NOOFBANK);
+	config->num_ranks = REG_GET_FIELD(adev->gfx.config.mc_arb_ramcfg,
+				MC_ARB_RAMCFG, NOOFRANKS);
+
+	config->tile_config_ptr = adev->gfx.config.tile_mode_array;
+	config->num_tile_configs =
+			ARRAY_SIZE(adev->gfx.config.tile_mode_array);
+	config->macro_tile_config_ptr =
+			adev->gfx.config.macrotile_mode_array;
+	config->num_macro_tile_configs =
+			ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
+
+	return 0;
+}
 
 static const struct kfd2kgd_calls kfd2kgd = {
 	.init_gtt_mem_allocation = alloc_gtt_mem,
@@ -111,7 +147,9 @@ static const struct kfd2kgd_calls kfd2kgd = {
 	.get_atc_vmid_pasid_mapping_valid =
 			get_atc_vmid_pasid_mapping_valid,
 	.write_vmid_invalidate_request = write_vmid_invalidate_request,
-	.get_fw_version = get_fw_version
+	.get_fw_version = get_fw_version,
+	.set_scratch_backing_va = set_scratch_backing_va,
+	.get_tile_config = get_tile_config,
 };
 
 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void)
@@ -147,7 +185,7 @@ static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id,
 {
 	struct amdgpu_device *adev = get_amdgpu_device(kgd);
 
-	uint32_t mec = (++pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
+	uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
 	uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
 
 	lock_srbm(kgd, mec, pipe, queue_id, 0);
@@ -216,7 +254,7 @@ static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id)
 	uint32_t mec;
 	uint32_t pipe;
 
-	mec = (++pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
+	mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
 	pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
 
 	lock_srbm(kgd, mec, pipe, 0, 0);
@@ -244,20 +282,67 @@ static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
 }
 
 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
-			uint32_t queue_id, uint32_t __user *wptr)
+			uint32_t queue_id, uint32_t __user *wptr,
+			uint32_t wptr_shift, uint32_t wptr_mask,
+			struct mm_struct *mm)
 {
-	struct vi_mqd *m;
-	uint32_t shadow_wptr, valid_wptr;
 	struct amdgpu_device *adev = get_amdgpu_device(kgd);
+	struct vi_mqd *m;
+	uint32_t *mqd_hqd;
+	uint32_t reg, wptr_val, data;
 
 	m = get_mqd(mqd);
 
-	valid_wptr = copy_from_user(&shadow_wptr, wptr, sizeof(shadow_wptr));
-	if (valid_wptr == 0)
-		m->cp_hqd_pq_wptr = shadow_wptr;
-
 	acquire_queue(kgd, pipe_id, queue_id);
-	gfx_v8_0_mqd_commit(adev, mqd);
+
+	/* HIQ is set during driver init period with vmid set to 0*/
+	if (m->cp_hqd_vmid == 0) {
+		uint32_t value, mec, pipe;
+
+		mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1;
+		pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec);
+
+		pr_debug("kfd: set HIQ, mec:%d, pipe:%d, queue:%d.\n",
+			mec, pipe, queue_id);
+		value = RREG32(mmRLC_CP_SCHEDULERS);
+		value = REG_SET_FIELD(value, RLC_CP_SCHEDULERS, scheduler1,
+			((mec << 5) | (pipe << 3) | queue_id | 0x80));
+		WREG32(mmRLC_CP_SCHEDULERS, value);
+	}
+
+	/* HQD registers extend from CP_MQD_BASE_ADDR to CP_HQD_EOP_WPTR_MEM. */
+	mqd_hqd = &m->cp_mqd_base_addr_lo;
+
+	for (reg = mmCP_MQD_BASE_ADDR; reg <= mmCP_HQD_EOP_CONTROL; reg++)
+		WREG32(reg, mqd_hqd[reg - mmCP_MQD_BASE_ADDR]);
+
+	/* Tonga errata: EOP RPTR/WPTR should be left unmodified.
+	 * This is safe since EOP RPTR==WPTR for any inactive HQD
+	 * on ASICs that do not support context-save.
+	 * EOP writes/reads can start anywhere in the ring.
+	 */
+	if (get_amdgpu_device(kgd)->asic_type != CHIP_TONGA) {
+		WREG32(mmCP_HQD_EOP_RPTR, m->cp_hqd_eop_rptr);
+		WREG32(mmCP_HQD_EOP_WPTR, m->cp_hqd_eop_wptr);
+		WREG32(mmCP_HQD_EOP_WPTR_MEM, m->cp_hqd_eop_wptr_mem);
+	}
+
+	for (reg = mmCP_HQD_EOP_EVENTS; reg <= mmCP_HQD_ERROR; reg++)
+		WREG32(reg, mqd_hqd[reg - mmCP_MQD_BASE_ADDR]);
+
+	/* Copy userspace write pointer value to register.
+	 * Activate doorbell logic to monitor subsequent changes.
+	 */
+	data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control,
+			     CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
+	WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data);
+
+	if (read_user_wptr(mm, wptr, wptr_val))
+		WREG32(mmCP_HQD_PQ_WPTR, (wptr_val << wptr_shift) & wptr_mask);
+
+	data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1);
+	WREG32(mmCP_HQD_ACTIVE, data);
+
 	release_queue(kgd);
 
 	return 0;
@@ -308,29 +393,102 @@ static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
 	return false;
 }
 
-static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
+static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd,
+				enum kfd_preempt_type reset_type,
 				unsigned int utimeout, uint32_t pipe_id,
 				uint32_t queue_id)
 {
 	struct amdgpu_device *adev = get_amdgpu_device(kgd);
 	uint32_t temp;
-	int timeout = utimeout;
+	enum hqd_dequeue_request_type type;
+	unsigned long flags, end_jiffies;
+	int retry;
+	struct vi_mqd *m = get_mqd(mqd);
 
 	acquire_queue(kgd, pipe_id, queue_id);
 
-	WREG32(mmCP_HQD_DEQUEUE_REQUEST, reset_type);
+	if (m->cp_hqd_vmid == 0)
+		WREG32_FIELD(RLC_CP_SCHEDULERS, scheduler1, 0);
 
+	switch (reset_type) {
+	case KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN:
+		type = DRAIN_PIPE;
+		break;
+	case KFD_PREEMPT_TYPE_WAVEFRONT_RESET:
+		type = RESET_WAVES;
+		break;
+	default:
+		type = DRAIN_PIPE;
+		break;
+	}
+
+	/* Workaround: If IQ timer is active and the wait time is close to or
+	 * equal to 0, dequeueing is not safe. Wait until either the wait time
+	 * is larger or timer is cleared. Also, ensure that IQ_REQ_PEND is
+	 * cleared before continuing. Also, ensure wait times are set to at
+	 * least 0x3.
+	 */
+	local_irq_save(flags);
+	preempt_disable();
+	retry = 5000; /* wait for 500 usecs at maximum */
+	while (true) {
+		temp = RREG32(mmCP_HQD_IQ_TIMER);
+		if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, PROCESSING_IQ)) {
+			pr_debug("HW is processing IQ\n");
+			goto loop;
+		}
+		if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, ACTIVE)) {
+			if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, RETRY_TYPE)
+					== 3) /* SEM-rearm is safe */
+				break;
+			/* Wait time 3 is safe for CP, but our MMIO read/write
+			 * time is close to 1 microsecond, so check for 10 to
+			 * leave more buffer room
+			 */
+			if (REG_GET_FIELD(temp, CP_HQD_IQ_TIMER, WAIT_TIME)
+					>= 10)
+				break;
+			pr_debug("IQ timer is active\n");
+		} else
+			break;
+loop:
+		if (!retry) {
+			pr_err("CP HQD IQ timer status time out\n");
+			break;
+		}
+		ndelay(100);
+		--retry;
+	}
+	retry = 1000;
+	while (true) {
+		temp = RREG32(mmCP_HQD_DEQUEUE_REQUEST);
+		if (!(temp & CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_MASK))
+			break;
+		pr_debug("Dequeue request is pending\n");
+
+		if (!retry) {
+			pr_err("CP HQD dequeue request time out\n");
+			break;
+		}
+		ndelay(100);
+		--retry;
+	}
+	local_irq_restore(flags);
+	preempt_enable();
+
+	WREG32(mmCP_HQD_DEQUEUE_REQUEST, type);
+
+	end_jiffies = (utimeout * HZ / 1000) + jiffies;
 	while (true) {
 		temp = RREG32(mmCP_HQD_ACTIVE);
-		if (temp & CP_HQD_ACTIVE__ACTIVE_MASK)
+		if (!(temp & CP_HQD_ACTIVE__ACTIVE_MASK))
 			break;
-		if (timeout <= 0) {
-			pr_err("kfd: cp queue preemption time out.\n");
+		if (time_after(jiffies, end_jiffies)) {
+			pr_err("cp queue preemption time out.\n");
 			release_queue(kgd);
 			return -ETIME;
 		}
-		msleep(20);
-		timeout -= 20;
+		usleep_range(500, 1000);
 	}
 
 	release_queue(kgd);
@@ -444,6 +602,16 @@ static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
 	return 0;
 }
 
+static void set_scratch_backing_va(struct kgd_dev *kgd,
+					uint64_t va, uint32_t vmid)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
+
+	lock_srbm(kgd, 0, 0, 0, vmid);
+	WREG32(mmSH_HIDDEN_PRIVATE_BASE_VMID, va);
+	unlock_srbm(kgd);
+}
+
 static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
@@ -454,42 +622,42 @@ static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)
 	switch (type) {
 	case KGD_ENGINE_PFP:
 		hdr = (const union amdgpu_firmware_header *)
-							adev->gfx.pfp_fw->data;
+						adev->gfx.pfp_fw->data;
 		break;
 
 	case KGD_ENGINE_ME:
 		hdr = (const union amdgpu_firmware_header *)
-							adev->gfx.me_fw->data;
+						adev->gfx.me_fw->data;
 		break;
 
 	case KGD_ENGINE_CE:
 		hdr = (const union amdgpu_firmware_header *)
-							adev->gfx.ce_fw->data;
+						adev->gfx.ce_fw->data;
 		break;
 
 	case KGD_ENGINE_MEC1:
 		hdr = (const union amdgpu_firmware_header *)
-							adev->gfx.mec_fw->data;
+						adev->gfx.mec_fw->data;
 		break;
 
 	case KGD_ENGINE_MEC2:
 		hdr = (const union amdgpu_firmware_header *)
-							adev->gfx.mec2_fw->data;
+						adev->gfx.mec2_fw->data;
 		break;
 
 	case KGD_ENGINE_RLC:
 		hdr = (const union amdgpu_firmware_header *)
-							adev->gfx.rlc_fw->data;
+						adev->gfx.rlc_fw->data;
 		break;
 
 	case KGD_ENGINE_SDMA1:
 		hdr = (const union amdgpu_firmware_header *)
-							adev->sdma.instance[0].fw->data;
+						adev->sdma.instance[0].fw->data;
 		break;
 
 	case KGD_ENGINE_SDMA2:
 		hdr = (const union amdgpu_firmware_header *)
-							adev->sdma.instance[1].fw->data;
+						adev->sdma.instance[1].fw->data;
 		break;
 
 	default:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
index 1e8e1123ddf4..ce443586a0c7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
@@ -1686,7 +1686,7 @@ void amdgpu_atombios_scratch_regs_lock(struct amdgpu_device *adev, bool lock)
 {
 	uint32_t bios_6_scratch;
 
-	bios_6_scratch = RREG32(mmBIOS_SCRATCH_6);
+	bios_6_scratch = RREG32(adev->bios_scratch_reg_offset + 6);
 
 	if (lock) {
 		bios_6_scratch |= ATOM_S6_CRITICAL_STATE;
@@ -1696,15 +1696,17 @@ void amdgpu_atombios_scratch_regs_lock(struct amdgpu_device *adev, bool lock)
 		bios_6_scratch |= ATOM_S6_ACC_MODE;
 	}
 
-	WREG32(mmBIOS_SCRATCH_6, bios_6_scratch);
+	WREG32(adev->bios_scratch_reg_offset + 6, bios_6_scratch);
 }
 
 void amdgpu_atombios_scratch_regs_init(struct amdgpu_device *adev)
 {
 	uint32_t bios_2_scratch, bios_6_scratch;
 
-	bios_2_scratch = RREG32(mmBIOS_SCRATCH_2);
-	bios_6_scratch = RREG32(mmBIOS_SCRATCH_6);
+	adev->bios_scratch_reg_offset = mmBIOS_SCRATCH_0;
+
+	bios_2_scratch = RREG32(adev->bios_scratch_reg_offset + 2);
+	bios_6_scratch = RREG32(adev->bios_scratch_reg_offset + 6);
 
 	/* let the bios control the backlight */
 	bios_2_scratch &= ~ATOM_S2_VRI_BRIGHT_ENABLE;
@@ -1715,8 +1717,8 @@ void amdgpu_atombios_scratch_regs_init(struct amdgpu_device *adev)
 	/* clear the vbios dpms state */
 	bios_2_scratch &= ~ATOM_S2_DEVICE_DPMS_STATE;
 
-	WREG32(mmBIOS_SCRATCH_2, bios_2_scratch);
-	WREG32(mmBIOS_SCRATCH_6, bios_6_scratch);
+	WREG32(adev->bios_scratch_reg_offset + 2, bios_2_scratch);
+	WREG32(adev->bios_scratch_reg_offset + 6, bios_6_scratch);
 }
 
 void amdgpu_atombios_scratch_regs_save(struct amdgpu_device *adev)
@@ -1724,7 +1726,7 @@ void amdgpu_atombios_scratch_regs_save(struct amdgpu_device *adev)
 	int i;
 
 	for (i = 0; i < AMDGPU_BIOS_NUM_SCRATCH; i++)
-		adev->bios_scratch[i] = RREG32(mmBIOS_SCRATCH_0 + i);
+		adev->bios_scratch[i] = RREG32(adev->bios_scratch_reg_offset + i);
 }
 
 void amdgpu_atombios_scratch_regs_restore(struct amdgpu_device *adev)
@@ -1738,20 +1740,30 @@ void amdgpu_atombios_scratch_regs_restore(struct amdgpu_device *adev)
 	adev->bios_scratch[7] &= ~ATOM_S7_ASIC_INIT_COMPLETE_MASK;
 
 	for (i = 0; i < AMDGPU_BIOS_NUM_SCRATCH; i++)
-		WREG32(mmBIOS_SCRATCH_0 + i, adev->bios_scratch[i]);
+		WREG32(adev->bios_scratch_reg_offset + i, adev->bios_scratch[i]);
 }
 
 void amdgpu_atombios_scratch_regs_engine_hung(struct amdgpu_device *adev,
 					      bool hung)
 {
-	u32 tmp = RREG32(mmBIOS_SCRATCH_3);
+	u32 tmp = RREG32(adev->bios_scratch_reg_offset + 3);
 
 	if (hung)
 		tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
 	else
 		tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
 
-	WREG32(mmBIOS_SCRATCH_3, tmp);
+	WREG32(adev->bios_scratch_reg_offset + 3, tmp);
+}
+
+bool amdgpu_atombios_scratch_need_asic_init(struct amdgpu_device *adev)
+{
+	u32 tmp = RREG32(adev->bios_scratch_reg_offset + 7);
+
+	if (tmp & ATOM_S7_ASIC_INIT_COMPLETE_MASK)
+		return false;
+	else
+		return true;
 }
 
 /* Atom needs data in little endian format
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.h
index 38d0fe32e5cd..b0d5d1d7fdba 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.h
@@ -200,6 +200,7 @@ void amdgpu_atombios_scratch_regs_save(struct amdgpu_device *adev);
 void amdgpu_atombios_scratch_regs_restore(struct amdgpu_device *adev);
 void amdgpu_atombios_scratch_regs_engine_hung(struct amdgpu_device *adev,
 					      bool hung);
+bool amdgpu_atombios_scratch_need_asic_init(struct amdgpu_device *adev);
 
 void amdgpu_atombios_copy_swap(u8 *dst, u8 *src, u8 num_bytes, bool to_le);
 int amdgpu_atombios_get_max_vddc(struct amdgpu_device *adev, u8 voltage_type,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
index 4bdda56fccee..f9ffe8ef0cd6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
@@ -66,41 +66,6 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev)
 	}
 }
 
-void amdgpu_atomfirmware_scratch_regs_save(struct amdgpu_device *adev)
-{
-	int i;
-
-	for (i = 0; i < AMDGPU_BIOS_NUM_SCRATCH; i++)
-		adev->bios_scratch[i] = RREG32(adev->bios_scratch_reg_offset + i);
-}
-
-void amdgpu_atomfirmware_scratch_regs_restore(struct amdgpu_device *adev)
-{
-	int i;
-
-	/*
-	 * VBIOS will check ASIC_INIT_COMPLETE bit to decide if
-	 * execute ASIC_Init posting via driver
-	 */
-	adev->bios_scratch[7] &= ~ATOM_S7_ASIC_INIT_COMPLETE_MASK;
-
-	for (i = 0; i < AMDGPU_BIOS_NUM_SCRATCH; i++)
-		WREG32(adev->bios_scratch_reg_offset + i, adev->bios_scratch[i]);
-}
-
-void amdgpu_atomfirmware_scratch_regs_engine_hung(struct amdgpu_device *adev,
-						  bool hung)
-{
-	u32 tmp = RREG32(adev->bios_scratch_reg_offset + 3);
-
-	if (hung)
-		tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
-	else
-		tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
-
-	WREG32(adev->bios_scratch_reg_offset + 3, tmp);
-}
-
 int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev)
 {
 	struct atom_context *ctx = adev->mode_info.atom_context;
@@ -130,3 +95,129 @@ int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev)
 	ctx->scratch_size_bytes = usage_bytes;
 	return 0;
 }
+
+union igp_info {
+	struct atom_integrated_system_info_v1_11 v11;
+};
+
+/*
+ * Return vram width from integrated system info table, if available,
+ * or 0 if not.
+ */
+int amdgpu_atomfirmware_get_vram_width(struct amdgpu_device *adev)
+{
+	struct amdgpu_mode_info *mode_info = &adev->mode_info;
+	int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
+						integratedsysteminfo);
+	u16 data_offset, size;
+	union igp_info *igp_info;
+	u8 frev, crev;
+
+	/* get any igp specific overrides */
+	if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, &size,
+				   &frev, &crev, &data_offset)) {
+		igp_info = (union igp_info *)
+			(mode_info->atom_context->bios + data_offset);
+		switch (crev) {
+		case 11:
+			return igp_info->v11.umachannelnumber * 64;
+		default:
+			return 0;
+		}
+	}
+
+	return 0;
+}
+
+union firmware_info {
+	struct atom_firmware_info_v3_1 v31;
+};
+
+union smu_info {
+	struct atom_smu_info_v3_1 v31;
+};
+
+union umc_info {
+	struct atom_umc_info_v3_1 v31;
+};
+
+int amdgpu_atomfirmware_get_clock_info(struct amdgpu_device *adev)
+{
+	struct amdgpu_mode_info *mode_info = &adev->mode_info;
+	struct amdgpu_pll *spll = &adev->clock.spll;
+	struct amdgpu_pll *mpll = &adev->clock.mpll;
+	uint8_t frev, crev;
+	uint16_t data_offset;
+	int ret = -EINVAL, index;
+
+	index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
+					    firmwareinfo);
+	if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL,
+				   &frev, &crev, &data_offset)) {
+		union firmware_info *firmware_info =
+			(union firmware_info *)(mode_info->atom_context->bios +
+						data_offset);
+
+		adev->clock.default_sclk =
+			le32_to_cpu(firmware_info->v31.bootup_sclk_in10khz);
+		adev->clock.default_mclk =
+			le32_to_cpu(firmware_info->v31.bootup_mclk_in10khz);
+
+		adev->pm.current_sclk = adev->clock.default_sclk;
+		adev->pm.current_mclk = adev->clock.default_mclk;
+
+		/* not technically a clock, but... */
+		adev->mode_info.firmware_flags =
+			le32_to_cpu(firmware_info->v31.firmware_capability);
+
+		ret = 0;
+	}
+
+	index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
+					    smu_info);
+	if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL,
+				   &frev, &crev, &data_offset)) {
+		union smu_info *smu_info =
+			(union smu_info *)(mode_info->atom_context->bios +
+					   data_offset);
+
+		/* system clock */
+		spll->reference_freq = le32_to_cpu(smu_info->v31.core_refclk_10khz);
+
+		spll->reference_div = 0;
+		spll->min_post_div = 1;
+		spll->max_post_div = 1;
+		spll->min_ref_div = 2;
+		spll->max_ref_div = 0xff;
+		spll->min_feedback_div = 4;
+		spll->max_feedback_div = 0xff;
+		spll->best_vco = 0;
+
+		ret = 0;
+	}
+
+	index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
+					    umc_info);
+	if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL,
+				   &frev, &crev, &data_offset)) {
+		union umc_info *umc_info =
+			(union umc_info *)(mode_info->atom_context->bios +
+					   data_offset);
+
+		/* memory clock */
+		mpll->reference_freq = le32_to_cpu(umc_info->v31.mem_refclk_10khz);
+
+		mpll->reference_div = 0;
+		mpll->min_post_div = 1;
+		mpll->max_post_div = 1;
+		mpll->min_ref_div = 2;
+		mpll->max_ref_div = 0xff;
+		mpll->min_feedback_div = 4;
+		mpll->max_feedback_div = 0xff;
+		mpll->best_vco = 0;
+
+		ret = 0;
+	}
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.h
index a2c3ebe22c71..288b97e54347 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.h
@@ -26,10 +26,8 @@
 
 bool amdgpu_atomfirmware_gpu_supports_virtualization(struct amdgpu_device *adev);
 void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev);
-void amdgpu_atomfirmware_scratch_regs_save(struct amdgpu_device *adev);
-void amdgpu_atomfirmware_scratch_regs_restore(struct amdgpu_device *adev);
-void amdgpu_atomfirmware_scratch_regs_engine_hung(struct amdgpu_device *adev,
-						  bool hung);
 int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev);
+int amdgpu_atomfirmware_get_vram_width(struct amdgpu_device *adev);
+int amdgpu_atomfirmware_get_clock_info(struct amdgpu_device *adev);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
index 1beae5b930d0..63ec1e1bb6aa 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c
@@ -40,7 +40,7 @@ static int amdgpu_benchmark_do_move(struct amdgpu_device *adev, unsigned size,
 	for (i = 0; i < n; i++) {
 		struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
 		r = amdgpu_copy_buffer(ring, saddr, daddr, size, NULL, &fence,
-				       false);
+				       false, false);
 		if (r)
 			goto exit_do_move;
 		r = dma_fence_wait(fence, false);
@@ -81,7 +81,7 @@ static void amdgpu_benchmark_move(struct amdgpu_device *adev, unsigned size,
 
 	n = AMDGPU_BENCHMARK_ITERATIONS;
 	r = amdgpu_bo_create(adev, size, PAGE_SIZE, true, sdomain, 0, NULL,
-			     NULL, &sobj);
+			     NULL, 0, &sobj);
 	if (r) {
 		goto out_cleanup;
 	}
@@ -94,7 +94,7 @@ static void amdgpu_benchmark_move(struct amdgpu_device *adev, unsigned size,
 		goto out_cleanup;
 	}
 	r = amdgpu_bo_create(adev, size, PAGE_SIZE, true, ddomain, 0, NULL,
-			     NULL, &dobj);
+			     NULL, 0, &dobj);
 	if (r) {
 		goto out_cleanup;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
index 365e735f6647..c21adf60a7f2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
@@ -86,19 +86,6 @@ static bool check_atom_bios(uint8_t *bios, size_t size)
 	return false;
 }
 
-static bool is_atom_fw(uint8_t *bios)
-{
-	uint16_t bios_header_start = bios[0x48] | (bios[0x49] << 8);
-	uint8_t frev = bios[bios_header_start + 2];
-	uint8_t crev = bios[bios_header_start + 3];
-
-	if ((frev < 3) ||
-	    ((frev == 3) && (crev < 3)))
-		return false;
-
-	return true;
-}
-
 /* If you boot an IGP board with a discrete card as the primary,
  * the IGP rom is not accessible via the rom bar as the IGP rom is
  * part of the system bios.  On boot, the system bios puts a
@@ -117,7 +104,7 @@ static bool igp_read_bios_from_vram(struct amdgpu_device *adev)
 
 	adev->bios = NULL;
 	vram_base = pci_resource_start(adev->pdev, 0);
-	bios = ioremap(vram_base, size);
+	bios = ioremap_wc(vram_base, size);
 	if (!bios) {
 		return false;
 	}
@@ -455,6 +442,6 @@ bool amdgpu_get_bios(struct amdgpu_device *adev)
 	return false;
 
 success:
-	adev->is_atom_fw = is_atom_fw(adev->bios);
+	adev->is_atom_fw = (adev->asic_type >= CHIP_VEGA10) ? true : false;
 	return true;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
index 5e771bc11b00..59089e027f4d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
@@ -83,7 +83,7 @@ static int amdgpu_bo_list_create(struct amdgpu_device *adev,
 	r = idr_alloc(&fpriv->bo_list_handles, list, 1, 0, GFP_KERNEL);
 	mutex_unlock(&fpriv->bo_list_lock);
 	if (r < 0) {
-		kfree(list);
+		amdgpu_bo_list_free(list);
 		return r;
 	}
 	*id = r;
@@ -136,7 +136,7 @@ static int amdgpu_bo_list_set(struct amdgpu_device *adev,
 		}
 
 		bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj));
-		drm_gem_object_unreference_unlocked(gobj);
+		drm_gem_object_put_unlocked(gobj);
 
 		usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm);
 		if (usermm) {
@@ -156,11 +156,11 @@ static int amdgpu_bo_list_set(struct amdgpu_device *adev,
 		entry->tv.bo = &entry->robj->tbo;
 		entry->tv.shared = !entry->robj->prime_shared_count;
 
-		if (entry->robj->prefered_domains == AMDGPU_GEM_DOMAIN_GDS)
+		if (entry->robj->preferred_domains == AMDGPU_GEM_DOMAIN_GDS)
 			gds_obj = entry->robj;
-		if (entry->robj->prefered_domains == AMDGPU_GEM_DOMAIN_GWS)
+		if (entry->robj->preferred_domains == AMDGPU_GEM_DOMAIN_GWS)
 			gws_obj = entry->robj;
-		if (entry->robj->prefered_domains == AMDGPU_GEM_DOMAIN_OA)
+		if (entry->robj->preferred_domains == AMDGPU_GEM_DOMAIN_OA)
 			oa_obj = entry->robj;
 
 		total_size += amdgpu_bo_size(entry->robj);
@@ -270,7 +270,7 @@ int amdgpu_bo_list_ioctl(struct drm_device *dev, void *data,
 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
 	union drm_amdgpu_bo_list *args = data;
 	uint32_t handle = args->in.list_handle;
-	const void __user *uptr = (const void*)(uintptr_t)args->in.bo_info_ptr;
+	const void __user *uptr = u64_to_user_ptr(args->in.bo_info_ptr);
 
 	struct drm_amdgpu_bo_list_entry *info;
 	struct amdgpu_bo_list *list;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
index c0a806280257..fd435a96481c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
@@ -124,7 +124,7 @@ static int amdgpu_cgs_alloc_gpu_mem(struct cgs_device *cgs_device,
 	ret = amdgpu_bo_create_restricted(adev, size, PAGE_SIZE,
 					  true, domain, flags,
 					  NULL, &placement, NULL,
-					  &obj);
+					  0, &obj);
 	if (ret) {
 		DRM_ERROR("(%d) bo create failed\n", ret);
 		return ret;
@@ -166,7 +166,7 @@ static int amdgpu_cgs_gmap_gpu_mem(struct cgs_device *cgs_device, cgs_handle_t h
 	r = amdgpu_bo_reserve(obj, true);
 	if (unlikely(r != 0))
 		return r;
-	r = amdgpu_bo_pin_restricted(obj, obj->prefered_domains,
+	r = amdgpu_bo_pin_restricted(obj, obj->preferred_domains,
 				     min_offset, max_offset, mcaddr);
 	amdgpu_bo_unreserve(obj);
 	return r;
@@ -240,6 +240,8 @@ static uint32_t amdgpu_cgs_read_ind_register(struct cgs_device *cgs_device,
 		return RREG32_DIDT(index);
 	case CGS_IND_REG_GC_CAC:
 		return RREG32_GC_CAC(index);
+	case CGS_IND_REG_SE_CAC:
+		return RREG32_SE_CAC(index);
 	case CGS_IND_REG__AUDIO_ENDPT:
 		DRM_ERROR("audio endpt register access not implemented.\n");
 		return 0;
@@ -266,6 +268,8 @@ static void amdgpu_cgs_write_ind_register(struct cgs_device *cgs_device,
 		return WREG32_DIDT(index, value);
 	case CGS_IND_REG_GC_CAC:
 		return WREG32_GC_CAC(index, value);
+	case CGS_IND_REG_SE_CAC:
+		return WREG32_SE_CAC(index, value);
 	case CGS_IND_REG__AUDIO_ENDPT:
 		DRM_ERROR("audio endpt register access not implemented.\n");
 		return;
@@ -610,6 +614,17 @@ static int amdgpu_cgs_enter_safe_mode(struct cgs_device *cgs_device,
 	return 0;
 }
 
+static void amdgpu_cgs_lock_grbm_idx(struct cgs_device *cgs_device,
+					bool lock)
+{
+	CGS_FUNC_ADEV;
+
+	if (lock)
+		mutex_lock(&adev->grbm_idx_mutex);
+	else
+		mutex_unlock(&adev->grbm_idx_mutex);
+}
+
 static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device,
 					enum cgs_ucode_id type,
 					struct cgs_firmware_info *info)
@@ -644,7 +659,7 @@ static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device,
 		info->version = (uint16_t)le32_to_cpu(header->header.ucode_version);
 
 		if (CGS_UCODE_ID_CP_MEC == type)
-			info->image_size = (header->jt_offset) << 2;
+			info->image_size = le32_to_cpu(header->jt_offset) << 2;
 
 		info->fw_version = amdgpu_get_firmware_version(cgs_device, type);
 		info->feature_version = (uint16_t)le32_to_cpu(header->ucode_feature_version);
@@ -719,7 +734,13 @@ static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device,
 				strcpy(fw_name, "amdgpu/polaris12_smc.bin");
 				break;
 			case CHIP_VEGA10:
-				strcpy(fw_name, "amdgpu/vega10_smc.bin");
+				if ((adev->pdev->device == 0x687f) &&
+					((adev->pdev->revision == 0xc0) ||
+					(adev->pdev->revision == 0xc1) ||
+					(adev->pdev->revision == 0xc3)))
+					strcpy(fw_name, "amdgpu/vega10_acg_smc.bin");
+				else
+					strcpy(fw_name, "amdgpu/vega10_smc.bin");
 				break;
 			default:
 				DRM_ERROR("SMC firmware not supported\n");
@@ -1117,6 +1138,7 @@ static const struct cgs_ops amdgpu_cgs_ops = {
 	.query_system_info = amdgpu_cgs_query_system_info,
 	.is_virtualization_enabled = amdgpu_cgs_is_virtualization_enabled,
 	.enter_safe_mode = amdgpu_cgs_enter_safe_mode,
+	.lock_grbm_idx = amdgpu_cgs_lock_grbm_idx,
 };
 
 static const struct cgs_os_ops amdgpu_cgs_os_ops = {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 5599c01b265d..60d8bedb694d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -54,7 +54,7 @@ static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p,
 
 	*offset = data->offset;
 
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 
 	if (amdgpu_ttm_tt_get_usermm(p->uf_entry.robj->tbo.ttm)) {
 		amdgpu_bo_unref(&p->uf_entry.robj);
@@ -90,7 +90,7 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
 	}
 
 	/* get chunks */
-	chunk_array_user = (uint64_t __user *)(uintptr_t)(cs->in.chunks);
+	chunk_array_user = u64_to_user_ptr(cs->in.chunks);
 	if (copy_from_user(chunk_array, chunk_array_user,
 			   sizeof(uint64_t)*cs->in.num_chunks)) {
 		ret = -EFAULT;
@@ -110,7 +110,7 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
 		struct drm_amdgpu_cs_chunk user_chunk;
 		uint32_t __user *cdata;
 
-		chunk_ptr = (void __user *)(uintptr_t)chunk_array[i];
+		chunk_ptr = u64_to_user_ptr(chunk_array[i]);
 		if (copy_from_user(&user_chunk, chunk_ptr,
 				       sizeof(struct drm_amdgpu_cs_chunk))) {
 			ret = -EFAULT;
@@ -121,7 +121,7 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
 		p->chunks[i].length_dw = user_chunk.length_dw;
 
 		size = p->chunks[i].length_dw;
-		cdata = (void __user *)(uintptr_t)user_chunk.chunk_data;
+		cdata = u64_to_user_ptr(user_chunk.chunk_data);
 
 		p->chunks[i].kdata = kvmalloc_array(size, sizeof(uint32_t), GFP_KERNEL);
 		if (p->chunks[i].kdata == NULL) {
@@ -223,10 +223,11 @@ static s64 bytes_to_us(struct amdgpu_device *adev, u64 bytes)
  * ticks. The accumulated microseconds (us) are converted to bytes and
  * returned.
  */
-static u64 amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev)
+static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev,
+					      u64 *max_bytes,
+					      u64 *max_vis_bytes)
 {
 	s64 time_us, increment_us;
-	u64 max_bytes;
 	u64 free_vram, total_vram, used_vram;
 
 	/* Allow a maximum of 200 accumulated ms. This is basically per-IB
@@ -238,11 +239,14 @@ static u64 amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev)
 	 */
 	const s64 us_upper_bound = 200000;
 
-	if (!adev->mm_stats.log2_max_MBps)
-		return 0;
+	if (!adev->mm_stats.log2_max_MBps) {
+		*max_bytes = 0;
+		*max_vis_bytes = 0;
+		return;
+	}
 
 	total_vram = adev->mc.real_vram_size - adev->vram_pin_size;
-	used_vram = atomic64_read(&adev->vram_usage);
+	used_vram = amdgpu_vram_mgr_usage(&adev->mman.bdev.man[TTM_PL_VRAM]);
 	free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram;
 
 	spin_lock(&adev->mm_stats.lock);
@@ -280,23 +284,46 @@ static u64 amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev)
 		adev->mm_stats.accum_us = max(min_us, adev->mm_stats.accum_us);
 	}
 
-	/* This returns 0 if the driver is in debt to disallow (optional)
+	/* This is set to 0 if the driver is in debt to disallow (optional)
 	 * buffer moves.
 	 */
-	max_bytes = us_to_bytes(adev, adev->mm_stats.accum_us);
+	*max_bytes = us_to_bytes(adev, adev->mm_stats.accum_us);
+
+	/* Do the same for visible VRAM if half of it is free */
+	if (adev->mc.visible_vram_size < adev->mc.real_vram_size) {
+		u64 total_vis_vram = adev->mc.visible_vram_size;
+		u64 used_vis_vram =
+			amdgpu_vram_mgr_vis_usage(&adev->mman.bdev.man[TTM_PL_VRAM]);
+
+		if (used_vis_vram < total_vis_vram) {
+			u64 free_vis_vram = total_vis_vram - used_vis_vram;
+			adev->mm_stats.accum_us_vis = min(adev->mm_stats.accum_us_vis +
+							  increment_us, us_upper_bound);
+
+			if (free_vis_vram >= total_vis_vram / 2)
+				adev->mm_stats.accum_us_vis =
+					max(bytes_to_us(adev, free_vis_vram / 2),
+					    adev->mm_stats.accum_us_vis);
+		}
+
+		*max_vis_bytes = us_to_bytes(adev, adev->mm_stats.accum_us_vis);
+	} else {
+		*max_vis_bytes = 0;
+	}
 
 	spin_unlock(&adev->mm_stats.lock);
-	return max_bytes;
 }
 
 /* Report how many bytes have really been moved for the last command
  * submission. This can result in a debt that can stop buffer migrations
  * temporarily.
  */
-void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes)
+void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes,
+				  u64 num_vis_bytes)
 {
 	spin_lock(&adev->mm_stats.lock);
 	adev->mm_stats.accum_us -= bytes_to_us(adev, num_bytes);
+	adev->mm_stats.accum_us_vis -= bytes_to_us(adev, num_vis_bytes);
 	spin_unlock(&adev->mm_stats.lock);
 }
 
@@ -304,7 +331,7 @@ static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser *p,
 				 struct amdgpu_bo *bo)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
-	u64 initial_bytes_moved;
+	u64 initial_bytes_moved, bytes_moved;
 	uint32_t domain;
 	int r;
 
@@ -314,17 +341,35 @@ static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser *p,
 	/* Don't move this buffer if we have depleted our allowance
 	 * to move it. Don't move anything if the threshold is zero.
 	 */
-	if (p->bytes_moved < p->bytes_moved_threshold)
-		domain = bo->prefered_domains;
-	else
+	if (p->bytes_moved < p->bytes_moved_threshold) {
+		if (adev->mc.visible_vram_size < adev->mc.real_vram_size &&
+		    (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)) {
+			/* And don't move a CPU_ACCESS_REQUIRED BO to limited
+			 * visible VRAM if we've depleted our allowance to do
+			 * that.
+			 */
+			if (p->bytes_moved_vis < p->bytes_moved_vis_threshold)
+				domain = bo->preferred_domains;
+			else
+				domain = bo->allowed_domains;
+		} else {
+			domain = bo->preferred_domains;
+		}
+	} else {
 		domain = bo->allowed_domains;
+	}
 
 retry:
 	amdgpu_ttm_placement_from_domain(bo, domain);
 	initial_bytes_moved = atomic64_read(&adev->num_bytes_moved);
 	r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
-	p->bytes_moved += atomic64_read(&adev->num_bytes_moved) -
-		initial_bytes_moved;
+	bytes_moved = atomic64_read(&adev->num_bytes_moved) -
+		      initial_bytes_moved;
+	p->bytes_moved += bytes_moved;
+	if (adev->mc.visible_vram_size < adev->mc.real_vram_size &&
+	    bo->tbo.mem.mem_type == TTM_PL_VRAM &&
+	    bo->tbo.mem.start < adev->mc.visible_vram_size >> PAGE_SHIFT)
+		p->bytes_moved_vis += bytes_moved;
 
 	if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
 		domain = bo->allowed_domains;
@@ -350,7 +395,8 @@ static bool amdgpu_cs_try_evict(struct amdgpu_cs_parser *p,
 		struct amdgpu_bo_list_entry *candidate = p->evictable;
 		struct amdgpu_bo *bo = candidate->robj;
 		struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
-		u64 initial_bytes_moved;
+		u64 initial_bytes_moved, bytes_moved;
+		bool update_bytes_moved_vis;
 		uint32_t other;
 
 		/* If we reached our current BO we can forget it */
@@ -370,10 +416,17 @@ static bool amdgpu_cs_try_evict(struct amdgpu_cs_parser *p,
 
 		/* Good we can try to move this BO somewhere else */
 		amdgpu_ttm_placement_from_domain(bo, other);
+		update_bytes_moved_vis =
+			adev->mc.visible_vram_size < adev->mc.real_vram_size &&
+			bo->tbo.mem.mem_type == TTM_PL_VRAM &&
+			bo->tbo.mem.start < adev->mc.visible_vram_size >> PAGE_SHIFT;
 		initial_bytes_moved = atomic64_read(&adev->num_bytes_moved);
 		r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
-		p->bytes_moved += atomic64_read(&adev->num_bytes_moved) -
+		bytes_moved = atomic64_read(&adev->num_bytes_moved) -
 			initial_bytes_moved;
+		p->bytes_moved += bytes_moved;
+		if (update_bytes_moved_vis)
+			p->bytes_moved_vis += bytes_moved;
 
 		if (unlikely(r))
 			break;
@@ -554,8 +607,10 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 		list_splice(&need_pages, &p->validated);
 	}
 
-	p->bytes_moved_threshold = amdgpu_cs_get_threshold_for_moves(p->adev);
+	amdgpu_cs_get_threshold_for_moves(p->adev, &p->bytes_moved_threshold,
+					  &p->bytes_moved_vis_threshold);
 	p->bytes_moved = 0;
+	p->bytes_moved_vis = 0;
 	p->evictable = list_last_entry(&p->validated,
 				       struct amdgpu_bo_list_entry,
 				       tv.head);
@@ -579,8 +634,8 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 		goto error_validate;
 	}
 
-	amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved);
-
+	amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved,
+				     p->bytes_moved_vis);
 	fpriv->vm.last_eviction_counter =
 		atomic64_read(&p->adev->num_evictions);
 
@@ -619,10 +674,8 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 	}
 
 error_validate:
-	if (r) {
-		amdgpu_vm_move_pt_bos_in_lru(p->adev, &fpriv->vm);
+	if (r)
 		ttm_eu_backoff_reservation(&p->ticket, &p->validated);
-	}
 
 error_free_pages:
 
@@ -670,21 +723,18 @@ static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p)
  * If error is set than unvalidate buffer, otherwise just free memory
  * used by parsing context.
  **/
-static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, bool backoff)
+static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error,
+				  bool backoff)
 {
-	struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
 	unsigned i;
 
-	if (!error) {
-		amdgpu_vm_move_pt_bos_in_lru(parser->adev, &fpriv->vm);
-
+	if (!error)
 		ttm_eu_fence_buffer_objects(&parser->ticket,
 					    &parser->validated,
 					    parser->fence);
-	} else if (backoff) {
+	else if (backoff)
 		ttm_eu_backoff_reservation(&parser->ticket,
 					   &parser->validated);
-	}
 
 	for (i = 0; i < parser->num_post_dep_syncobjs; i++)
 		drm_syncobj_put(parser->post_dep_syncobjs[i]);
@@ -737,7 +787,8 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
 
 	if (amdgpu_sriov_vf(adev)) {
 		struct dma_fence *f;
-		bo_va = vm->csa_bo_va;
+
+		bo_va = fpriv->csa_va;
 		BUG_ON(!bo_va);
 		r = amdgpu_vm_bo_update(adev, bo_va, false);
 		if (r)
@@ -774,7 +825,7 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
 
 	}
 
-	r = amdgpu_vm_clear_invalids(adev, vm, &p->job->sync);
+	r = amdgpu_vm_clear_moved(adev, vm, &p->job->sync);
 
 	if (amdgpu_vm_debug && p->bo_list) {
 		/* Invalidate all BOs to test for userspace bugs */
@@ -984,7 +1035,7 @@ static int amdgpu_syncobj_lookup_and_add_to_sync(struct amdgpu_cs_parser *p,
 {
 	int r;
 	struct dma_fence *fence;
-	r = drm_syncobj_fence_get(p->filp, handle, &fence);
+	r = drm_syncobj_find_fence(p->filp, handle, &fence);
 	if (r)
 		return r;
 
@@ -1028,6 +1079,9 @@ static int amdgpu_cs_process_syncobj_out_dep(struct amdgpu_cs_parser *p,
 					     GFP_KERNEL);
 	p->num_post_dep_syncobjs = 0;
 
+	if (!p->post_dep_syncobjs)
+		return -ENOMEM;
+
 	for (i = 0; i < num_deps; ++i) {
 		p->post_dep_syncobjs[i] = drm_syncobj_find(p->filp, deps[i].handle);
 		if (!p->post_dep_syncobjs[i])
@@ -1099,7 +1153,6 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
 	cs->out.handle = amdgpu_ctx_add_fence(p->ctx, ring, p->fence);
 	job->uf_sequence = cs->out.handle;
 	amdgpu_job_free_resources(job);
-	amdgpu_cs_parser_fini(p, 0, true);
 
 	trace_amdgpu_cs_ioctl(job);
 	amd_sched_entity_push_job(&job->base);
@@ -1157,10 +1210,7 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 		goto out;
 
 	r = amdgpu_cs_submit(&parser, cs);
-	if (r)
-		goto out;
 
-	return 0;
 out:
 	amdgpu_cs_parser_fini(&parser, r, reserved_buffers);
 	return r;
@@ -1383,7 +1433,7 @@ int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data,
 	if (fences == NULL)
 		return -ENOMEM;
 
-	fences_user = (void __user *)(uintptr_t)(wait->in.fences);
+	fences_user = u64_to_user_ptr(wait->in.fences);
 	if (copy_from_user(fences, fences_user,
 		sizeof(struct drm_amdgpu_fence) * fence_count)) {
 		r = -EFAULT;
@@ -1436,7 +1486,7 @@ amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
 			    addr > mapping->last)
 				continue;
 
-			*bo = lobj->bo_va->bo;
+			*bo = lobj->bo_va->base.bo;
 			return mapping;
 		}
 
@@ -1445,7 +1495,7 @@ amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
 			    addr > mapping->last)
 				continue;
 
-			*bo = lobj->bo_va->bo;
+			*bo = lobj->bo_va->base.bo;
 			return mapping;
 		}
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 4a8fc15467cf..e630d918fefc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -53,6 +53,9 @@
 #include "bif/bif_4_1_d.h"
 #include <linux/pci.h>
 #include <linux/firmware.h>
+#include "amdgpu_vf_error.h"
+
+#include "amdgpu_amdkfd.h"
 
 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
@@ -128,6 +131,10 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
 {
 	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
 
+	if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
+		adev->last_mm_index = v;
+	}
+
 	if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) {
 		BUG_ON(in_interrupt());
 		return amdgpu_virt_kiq_wreg(adev, reg, v);
@@ -143,6 +150,10 @@ void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
 		writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
 		spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 	}
+
+	if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
+		udelay(500);
+	}
 }
 
 u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg)
@@ -157,6 +168,9 @@ u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg)
 
 void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 {
+	if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
+		adev->last_mm_index = v;
+	}
 
 	if ((reg * 4) < adev->rio_mem_size)
 		iowrite32(v, adev->rio_mem + (reg * 4));
@@ -164,6 +178,10 @@ void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 		iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
 		iowrite32(v, adev->rio_mem + (mmMM_DATA * 4));
 	}
+
+	if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
+		udelay(500);
+	}
 }
 
 /**
@@ -318,51 +336,16 @@ static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
 
 static int amdgpu_vram_scratch_init(struct amdgpu_device *adev)
 {
-	int r;
-
-	if (adev->vram_scratch.robj == NULL) {
-		r = amdgpu_bo_create(adev, AMDGPU_GPU_PAGE_SIZE,
-				     PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM,
-				     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
-				     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
-				     NULL, NULL, &adev->vram_scratch.robj);
-		if (r) {
-			return r;
-		}
-	}
-
-	r = amdgpu_bo_reserve(adev->vram_scratch.robj, false);
-	if (unlikely(r != 0))
-		return r;
-	r = amdgpu_bo_pin(adev->vram_scratch.robj,
-			  AMDGPU_GEM_DOMAIN_VRAM, &adev->vram_scratch.gpu_addr);
-	if (r) {
-		amdgpu_bo_unreserve(adev->vram_scratch.robj);
-		return r;
-	}
-	r = amdgpu_bo_kmap(adev->vram_scratch.robj,
-				(void **)&adev->vram_scratch.ptr);
-	if (r)
-		amdgpu_bo_unpin(adev->vram_scratch.robj);
-	amdgpu_bo_unreserve(adev->vram_scratch.robj);
-
-	return r;
+	return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
+				       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
+				       &adev->vram_scratch.robj,
+				       &adev->vram_scratch.gpu_addr,
+				       (void **)&adev->vram_scratch.ptr);
 }
 
 static void amdgpu_vram_scratch_fini(struct amdgpu_device *adev)
 {
-	int r;
-
-	if (adev->vram_scratch.robj == NULL) {
-		return;
-	}
-	r = amdgpu_bo_reserve(adev->vram_scratch.robj, true);
-	if (likely(r == 0)) {
-		amdgpu_bo_kunmap(adev->vram_scratch.robj);
-		amdgpu_bo_unpin(adev->vram_scratch.robj);
-		amdgpu_bo_unreserve(adev->vram_scratch.robj);
-	}
-	amdgpu_bo_unref(&adev->vram_scratch.robj);
+	amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
 }
 
 /**
@@ -521,7 +504,8 @@ static int amdgpu_wb_init(struct amdgpu_device *adev)
 	int r;
 
 	if (adev->wb.wb_obj == NULL) {
-		r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t),
+		/* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
+		r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
 					    PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
 					    &adev->wb.wb_obj, &adev->wb.gpu_addr,
 					    (void **)&adev->wb.wb);
@@ -552,32 +536,10 @@ static int amdgpu_wb_init(struct amdgpu_device *adev)
 int amdgpu_wb_get(struct amdgpu_device *adev, u32 *wb)
 {
 	unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
-	if (offset < adev->wb.num_wb) {
-		__set_bit(offset, adev->wb.used);
-		*wb = offset;
-		return 0;
-	} else {
-		return -EINVAL;
-	}
-}
 
-/**
- * amdgpu_wb_get_64bit - Allocate a wb entry
- *
- * @adev: amdgpu_device pointer
- * @wb: wb index
- *
- * Allocate a wb slot for use by the driver (all asics).
- * Returns 0 on success or -EINVAL on failure.
- */
-int amdgpu_wb_get_64bit(struct amdgpu_device *adev, u32 *wb)
-{
-	unsigned long offset = bitmap_find_next_zero_area_off(adev->wb.used,
-				adev->wb.num_wb, 0, 2, 7, 0);
-	if ((offset + 1) < adev->wb.num_wb) {
+	if (offset < adev->wb.num_wb) {
 		__set_bit(offset, adev->wb.used);
-		__set_bit(offset + 1, adev->wb.used);
-		*wb = offset;
+		*wb = offset * 8; /* convert to dw offset */
 		return 0;
 	} else {
 		return -EINVAL;
@@ -599,22 +561,6 @@ void amdgpu_wb_free(struct amdgpu_device *adev, u32 wb)
 }
 
 /**
- * amdgpu_wb_free_64bit - Free a wb entry
- *
- * @adev: amdgpu_device pointer
- * @wb: wb index
- *
- * Free a wb slot allocated for use by the driver (all asics)
- */
-void amdgpu_wb_free_64bit(struct amdgpu_device *adev, u32 wb)
-{
-	if ((wb + 1) < adev->wb.num_wb) {
-		__clear_bit(wb, adev->wb.used);
-		__clear_bit(wb + 1, adev->wb.used);
-	}
-}
-
-/**
  * amdgpu_vram_location - try to find VRAM location
  * @adev: amdgpu device structure holding all necessary informations
  * @mc: memory controller structure holding memory informations
@@ -665,7 +611,7 @@ void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64
 }
 
 /**
- * amdgpu_gtt_location - try to find GTT location
+ * amdgpu_gart_location - try to find GTT location
  * @adev: amdgpu device structure holding all necessary informations
  * @mc: memory controller structure holding memory informations
  *
@@ -676,28 +622,28 @@ void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64
  *
  * FIXME: when reducing GTT size align new size on power of 2.
  */
-void amdgpu_gtt_location(struct amdgpu_device *adev, struct amdgpu_mc *mc)
+void amdgpu_gart_location(struct amdgpu_device *adev, struct amdgpu_mc *mc)
 {
 	u64 size_af, size_bf;
 
-	size_af = ((adev->mc.mc_mask - mc->vram_end) + mc->gtt_base_align) & ~mc->gtt_base_align;
-	size_bf = mc->vram_start & ~mc->gtt_base_align;
+	size_af = adev->mc.mc_mask - mc->vram_end;
+	size_bf = mc->vram_start;
 	if (size_bf > size_af) {
-		if (mc->gtt_size > size_bf) {
+		if (mc->gart_size > size_bf) {
 			dev_warn(adev->dev, "limiting GTT\n");
-			mc->gtt_size = size_bf;
+			mc->gart_size = size_bf;
 		}
-		mc->gtt_start = 0;
+		mc->gart_start = 0;
 	} else {
-		if (mc->gtt_size > size_af) {
+		if (mc->gart_size > size_af) {
 			dev_warn(adev->dev, "limiting GTT\n");
-			mc->gtt_size = size_af;
+			mc->gart_size = size_af;
 		}
-		mc->gtt_start = (mc->vram_end + 1 + mc->gtt_base_align) & ~mc->gtt_base_align;
+		mc->gart_start = mc->vram_end + 1;
 	}
-	mc->gtt_end = mc->gtt_start + mc->gtt_size - 1;
+	mc->gart_end = mc->gart_start + mc->gart_size - 1;
 	dev_info(adev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n",
-			mc->gtt_size >> 20, mc->gtt_start, mc->gtt_end);
+			mc->gart_size >> 20, mc->gart_start, mc->gart_end);
 }
 
 /*
@@ -720,7 +666,12 @@ bool amdgpu_need_post(struct amdgpu_device *adev)
 		adev->has_hw_reset = false;
 		return true;
 	}
-	/* then check MEM_SIZE, in case the crtcs are off */
+
+	/* bios scratch used on CIK+ */
+	if (adev->asic_type >= CHIP_BONAIRE)
+		return amdgpu_atombios_scratch_need_asic_init(adev);
+
+	/* check MEM_SIZE for older asics */
 	reg = amdgpu_asic_get_config_memsize(adev);
 
 	if ((reg != 0) && (reg != 0xffffffff))
@@ -1031,19 +982,6 @@ static unsigned int amdgpu_vga_set_decode(void *cookie, bool state)
 		return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
 }
 
-/**
- * amdgpu_check_pot_argument - check that argument is a power of two
- *
- * @arg: value to check
- *
- * Validates that a certain argument is a power of two (all asics).
- * Returns true if argument is valid.
- */
-static bool amdgpu_check_pot_argument(int arg)
-{
-	return (arg & (arg - 1)) == 0;
-}
-
 static void amdgpu_check_block_size(struct amdgpu_device *adev)
 {
 	/* defines number of bits in page table versus page directory,
@@ -1077,7 +1015,7 @@ static void amdgpu_check_vm_size(struct amdgpu_device *adev)
 	if (amdgpu_vm_size == -1)
 		return;
 
-	if (!amdgpu_check_pot_argument(amdgpu_vm_size)) {
+	if (!is_power_of_2(amdgpu_vm_size)) {
 		dev_warn(adev->dev, "VM size (%d) must be a power of 2\n",
 			 amdgpu_vm_size);
 		goto def_value;
@@ -1118,19 +1056,31 @@ static void amdgpu_check_arguments(struct amdgpu_device *adev)
 		dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
 			 amdgpu_sched_jobs);
 		amdgpu_sched_jobs = 4;
-	} else if (!amdgpu_check_pot_argument(amdgpu_sched_jobs)){
+	} else if (!is_power_of_2(amdgpu_sched_jobs)){
 		dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
 			 amdgpu_sched_jobs);
 		amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
 	}
 
-	if (amdgpu_gart_size != -1) {
+	if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
+		/* gart size must be greater or equal to 32M */
+		dev_warn(adev->dev, "gart size (%d) too small\n",
+			 amdgpu_gart_size);
+		amdgpu_gart_size = -1;
+	}
+
+	if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
 		/* gtt size must be greater or equal to 32M */
-		if (amdgpu_gart_size < 32) {
-			dev_warn(adev->dev, "gart size (%d) too small\n",
-				 amdgpu_gart_size);
-			amdgpu_gart_size = -1;
-		}
+		dev_warn(adev->dev, "gtt size (%d) too small\n",
+				 amdgpu_gtt_size);
+		amdgpu_gtt_size = -1;
+	}
+
+	/* valid range is between 4 and 9 inclusive */
+	if (amdgpu_vm_fragment_size != -1 &&
+	    (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
+		dev_warn(adev->dev, "valid range is between 4 and 9\n");
+		amdgpu_vm_fragment_size = -1;
 	}
 
 	amdgpu_check_vm_size(adev);
@@ -1138,7 +1088,7 @@ static void amdgpu_check_arguments(struct amdgpu_device *adev)
 	amdgpu_check_block_size(adev);
 
 	if (amdgpu_vram_page_split != -1 && (amdgpu_vram_page_split < 16 ||
-	    !amdgpu_check_pot_argument(amdgpu_vram_page_split))) {
+	    !is_power_of_2(amdgpu_vram_page_split))) {
 		dev_warn(adev->dev, "invalid VRAM page split (%d)\n",
 			 amdgpu_vram_page_split);
 		amdgpu_vram_page_split = 1024;
@@ -1901,7 +1851,8 @@ static int amdgpu_sriov_reinit_late(struct amdgpu_device *adev)
 		AMD_IP_BLOCK_TYPE_DCE,
 		AMD_IP_BLOCK_TYPE_GFX,
 		AMD_IP_BLOCK_TYPE_SDMA,
-		AMD_IP_BLOCK_TYPE_VCE,
+		AMD_IP_BLOCK_TYPE_UVD,
+		AMD_IP_BLOCK_TYPE_VCE
 	};
 
 	for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
@@ -2019,7 +1970,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	adev->flags = flags;
 	adev->asic_type = flags & AMD_ASIC_MASK;
 	adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
-	adev->mc.gtt_size = 512 * 1024 * 1024;
+	adev->mc.gart_size = 512 * 1024 * 1024;
 	adev->accel_working = false;
 	adev->num_rings = 0;
 	adev->mman.buffer_funcs = NULL;
@@ -2068,6 +2019,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	spin_lock_init(&adev->uvd_ctx_idx_lock);
 	spin_lock_init(&adev->didt_idx_lock);
 	spin_lock_init(&adev->gc_cac_idx_lock);
+	spin_lock_init(&adev->se_cac_idx_lock);
 	spin_lock_init(&adev->audio_endpt_idx_lock);
 	spin_lock_init(&adev->mm_stats.lock);
 
@@ -2143,6 +2095,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	r = amdgpu_atombios_init(adev);
 	if (r) {
 		dev_err(adev->dev, "amdgpu_atombios_init failed\n");
+		amdgpu_vf_error_put(AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
 		goto failed;
 	}
 
@@ -2153,6 +2106,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	if (amdgpu_vpost_needed(adev)) {
 		if (!adev->bios) {
 			dev_err(adev->dev, "no vBIOS found\n");
+			amdgpu_vf_error_put(AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
 			r = -EINVAL;
 			goto failed;
 		}
@@ -2160,18 +2114,28 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 		r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
 		if (r) {
 			dev_err(adev->dev, "gpu post error!\n");
+			amdgpu_vf_error_put(AMDGIM_ERROR_VF_GPU_POST_ERROR, 0, 0);
 			goto failed;
 		}
 	} else {
 		DRM_INFO("GPU post is not needed\n");
 	}
 
-	if (!adev->is_atom_fw) {
+	if (adev->is_atom_fw) {
+		/* Initialize clocks */
+		r = amdgpu_atomfirmware_get_clock_info(adev);
+		if (r) {
+			dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
+			amdgpu_vf_error_put(AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
+			goto failed;
+		}
+	} else {
 		/* Initialize clocks */
 		r = amdgpu_atombios_get_clock_info(adev);
 		if (r) {
 			dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
-			return r;
+			amdgpu_vf_error_put(AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
+			goto failed;
 		}
 		/* init i2c buses */
 		amdgpu_atombios_i2c_init(adev);
@@ -2181,6 +2145,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	r = amdgpu_fence_driver_init(adev);
 	if (r) {
 		dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
+		amdgpu_vf_error_put(AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
 		goto failed;
 	}
 
@@ -2190,6 +2155,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	r = amdgpu_init(adev);
 	if (r) {
 		dev_err(adev->dev, "amdgpu_init failed\n");
+		amdgpu_vf_error_put(AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
 		amdgpu_fini(adev);
 		goto failed;
 	}
@@ -2209,6 +2175,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	r = amdgpu_ib_pool_init(adev);
 	if (r) {
 		dev_err(adev->dev, "IB initialization failed (%d).\n", r);
+		amdgpu_vf_error_put(AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
 		goto failed;
 	}
 
@@ -2253,12 +2220,14 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	r = amdgpu_late_init(adev);
 	if (r) {
 		dev_err(adev->dev, "amdgpu_late_init failed\n");
+		amdgpu_vf_error_put(AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
 		goto failed;
 	}
 
 	return 0;
 
 failed:
+	amdgpu_vf_error_trans_all(adev);
 	if (runtime)
 		vga_switcheroo_fini_domain_pm_ops(adev->dev);
 	return r;
@@ -2351,6 +2320,8 @@ int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
 	}
 	drm_modeset_unlock_all(dev);
 
+	amdgpu_amdkfd_suspend(adev);
+
 	/* unpin the front buffers and cursors */
 	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
 		struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
@@ -2392,10 +2363,7 @@ int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
 	 */
 	amdgpu_bo_evict_vram(adev);
 
-	if (adev->is_atom_fw)
-		amdgpu_atomfirmware_scratch_regs_save(adev);
-	else
-		amdgpu_atombios_scratch_regs_save(adev);
+	amdgpu_atombios_scratch_regs_save(adev);
 	pci_save_state(dev->pdev);
 	if (suspend) {
 		/* Shut down the device */
@@ -2444,10 +2412,7 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
 		if (r)
 			goto unlock;
 	}
-	if (adev->is_atom_fw)
-		amdgpu_atomfirmware_scratch_regs_restore(adev);
-	else
-		amdgpu_atombios_scratch_regs_restore(adev);
+	amdgpu_atombios_scratch_regs_restore(adev);
 
 	/* post card */
 	if (amdgpu_need_post(adev)) {
@@ -2490,6 +2455,9 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
 			}
 		}
 	}
+	r = amdgpu_amdkfd_resume(adev);
+	if (r)
+		return r;
 
 	/* blat the mode back in */
 	if (fbcon) {
@@ -2654,12 +2622,6 @@ static int amdgpu_recover_vram_from_shadow(struct amdgpu_device *adev,
 			goto err;
 		}
 
-		r = amdgpu_ttm_bind(&bo->shadow->tbo, &bo->shadow->tbo.mem);
-		if (r) {
-			DRM_ERROR("%p bind failed\n", bo->shadow);
-			goto err;
-		}
-
 		r = amdgpu_bo_restore_from_shadow(adev, ring, bo,
 						 NULL, fence, true);
 		if (r) {
@@ -2860,21 +2822,9 @@ int amdgpu_gpu_reset(struct amdgpu_device *adev)
 		r = amdgpu_suspend(adev);
 
 retry:
-		/* Disable fb access */
-		if (adev->mode_info.num_crtc) {
-			struct amdgpu_mode_mc_save save;
-			amdgpu_display_stop_mc_access(adev, &save);
-			amdgpu_wait_for_idle(adev, AMD_IP_BLOCK_TYPE_GMC);
-		}
-		if (adev->is_atom_fw)
-			amdgpu_atomfirmware_scratch_regs_save(adev);
-		else
-			amdgpu_atombios_scratch_regs_save(adev);
+		amdgpu_atombios_scratch_regs_save(adev);
 		r = amdgpu_asic_reset(adev);
-		if (adev->is_atom_fw)
-			amdgpu_atomfirmware_scratch_regs_restore(adev);
-		else
-			amdgpu_atombios_scratch_regs_restore(adev);
+		amdgpu_atombios_scratch_regs_restore(adev);
 		/* post card */
 		amdgpu_atom_asic_init(adev->mode_info.atom_context);
 
@@ -2952,6 +2902,7 @@ out:
 		}
 	} else {
 		dev_err(adev->dev, "asic resume failed (%d).\n", r);
+		amdgpu_vf_error_put(AMDGIM_ERROR_VF_ASIC_RESUME_FAIL, 0, r);
 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
 			if (adev->rings[i] && adev->rings[i]->sched.thread) {
 				kthread_unpark(adev->rings[i]->sched.thread);
@@ -2962,12 +2913,16 @@ out:
 	drm_helper_resume_force_mode(adev->ddev);
 
 	ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
-	if (r)
+	if (r) {
 		/* bad news, how to tell it to userspace ? */
 		dev_info(adev->dev, "GPU reset failed\n");
-	else
+		amdgpu_vf_error_put(AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
+	}
+	else {
 		dev_info(adev->dev, "GPU reset successed!\n");
+	}
 
+	amdgpu_vf_error_trans_all(adev);
 	return r;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
index cdf2ab20166a..6ad243293a78 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
@@ -482,7 +482,7 @@ static void amdgpu_user_framebuffer_destroy(struct drm_framebuffer *fb)
 {
 	struct amdgpu_framebuffer *amdgpu_fb = to_amdgpu_framebuffer(fb);
 
-	drm_gem_object_unreference_unlocked(amdgpu_fb->obj);
+	drm_gem_object_put_unlocked(amdgpu_fb->obj);
 	drm_framebuffer_cleanup(fb);
 	kfree(amdgpu_fb);
 }
@@ -542,14 +542,14 @@ amdgpu_user_framebuffer_create(struct drm_device *dev,
 
 	amdgpu_fb = kzalloc(sizeof(*amdgpu_fb), GFP_KERNEL);
 	if (amdgpu_fb == NULL) {
-		drm_gem_object_unreference_unlocked(obj);
+		drm_gem_object_put_unlocked(obj);
 		return ERR_PTR(-ENOMEM);
 	}
 
 	ret = amdgpu_framebuffer_init(dev, amdgpu_fb, mode_cmd, obj);
 	if (ret) {
 		kfree(amdgpu_fb);
-		drm_gem_object_unreference_unlocked(obj);
+		drm_gem_object_put_unlocked(obj);
 		return ERR_PTR(ret);
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index b59f37c83fa6..0f16986ec5bc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -68,13 +68,16 @@
  * - 3.16.0 - Add reserved vmid support
  * - 3.17.0 - Add AMDGPU_NUM_VRAM_CPU_PAGE_FAULTS.
  * - 3.18.0 - Export gpu always on cu bitmap
+ * - 3.19.0 - Add support for UVD MJPEG decode
  */
 #define KMS_DRIVER_MAJOR	3
-#define KMS_DRIVER_MINOR	18
+#define KMS_DRIVER_MINOR	19
 #define KMS_DRIVER_PATCHLEVEL	0
 
 int amdgpu_vram_limit = 0;
+int amdgpu_vis_vram_limit = 0;
 int amdgpu_gart_size = -1; /* auto */
+int amdgpu_gtt_size = -1; /* auto */
 int amdgpu_moverate = -1; /* auto */
 int amdgpu_benchmarking = 0;
 int amdgpu_testing = 0;
@@ -92,6 +95,7 @@ unsigned amdgpu_ip_block_mask = 0xffffffff;
 int amdgpu_bapm = -1;
 int amdgpu_deep_color = 0;
 int amdgpu_vm_size = -1;
+int amdgpu_vm_fragment_size = -1;
 int amdgpu_vm_block_size = -1;
 int amdgpu_vm_fault_stop = 0;
 int amdgpu_vm_debug = 0;
@@ -106,6 +110,7 @@ unsigned amdgpu_pcie_gen_cap = 0;
 unsigned amdgpu_pcie_lane_cap = 0;
 unsigned amdgpu_cg_mask = 0xffffffff;
 unsigned amdgpu_pg_mask = 0xffffffff;
+unsigned amdgpu_sdma_phase_quantum = 32;
 char *amdgpu_disable_cu = NULL;
 char *amdgpu_virtual_display = NULL;
 unsigned amdgpu_pp_feature_mask = 0xffffffff;
@@ -120,8 +125,14 @@ int amdgpu_lbpw = -1;
 MODULE_PARM_DESC(vramlimit, "Restrict VRAM for testing, in megabytes");
 module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);
 
-MODULE_PARM_DESC(gartsize, "Size of PCIE/IGP gart to setup in megabytes (32, 64, etc., -1 = auto)");
-module_param_named(gartsize, amdgpu_gart_size, int, 0600);
+MODULE_PARM_DESC(vis_vramlimit, "Restrict visible VRAM for testing, in megabytes");
+module_param_named(vis_vramlimit, amdgpu_vis_vram_limit, int, 0444);
+
+MODULE_PARM_DESC(gartsize, "Size of GART to setup in megabytes (32, 64, etc., -1=auto)");
+module_param_named(gartsize, amdgpu_gart_size, uint, 0600);
+
+MODULE_PARM_DESC(gttsize, "Size of the GTT domain in megabytes (-1 = auto)");
+module_param_named(gttsize, amdgpu_gtt_size, int, 0600);
 
 MODULE_PARM_DESC(moverate, "Maximum buffer migration rate in MB/s. (32, 64, etc., -1=auto, 0=1=disabled)");
 module_param_named(moverate, amdgpu_moverate, int, 0600);
@@ -174,6 +185,9 @@ module_param_named(deep_color, amdgpu_deep_color, int, 0444);
 MODULE_PARM_DESC(vm_size, "VM address space size in gigabytes (default 64GB)");
 module_param_named(vm_size, amdgpu_vm_size, int, 0444);
 
+MODULE_PARM_DESC(vm_fragment_size, "VM fragment size in bits (4, 5, etc. 4 = 64K (default), Max 9 = 2M)");
+module_param_named(vm_fragment_size, amdgpu_vm_fragment_size, int, 0444);
+
 MODULE_PARM_DESC(vm_block_size, "VM page table size in bits (default depending on vm_size)");
 module_param_named(vm_block_size, amdgpu_vm_block_size, int, 0444);
 
@@ -186,7 +200,7 @@ module_param_named(vm_debug, amdgpu_vm_debug, int, 0644);
 MODULE_PARM_DESC(vm_update_mode, "VM update using CPU (0 = never (default except for large BAR(LB)), 1 = Graphics only, 2 = Compute only (default for LB), 3 = Both");
 module_param_named(vm_update_mode, amdgpu_vm_update_mode, int, 0444);
 
-MODULE_PARM_DESC(vram_page_split, "Number of pages after we split VRAM allocations (default 1024, -1 = disable)");
+MODULE_PARM_DESC(vram_page_split, "Number of pages after we split VRAM allocations (default 512, -1 = disable)");
 module_param_named(vram_page_split, amdgpu_vram_page_split, int, 0444);
 
 MODULE_PARM_DESC(exp_hw_support, "experimental hw support (1 = enable, 0 = disable (default))");
@@ -199,7 +213,7 @@ MODULE_PARM_DESC(sched_hw_submission, "the max number of HW submissions (default
 module_param_named(sched_hw_submission, amdgpu_sched_hw_submission, int, 0444);
 
 MODULE_PARM_DESC(ppfeaturemask, "all power features enabled (default))");
-module_param_named(ppfeaturemask, amdgpu_pp_feature_mask, int, 0444);
+module_param_named(ppfeaturemask, amdgpu_pp_feature_mask, uint, 0444);
 
 MODULE_PARM_DESC(no_evict, "Support pinning request from user space (1 = enable, 0 = disable (default))");
 module_param_named(no_evict, amdgpu_no_evict, int, 0444);
@@ -219,6 +233,9 @@ module_param_named(cg_mask, amdgpu_cg_mask, uint, 0444);
 MODULE_PARM_DESC(pg_mask, "Powergating flags mask (0 = disable power gating)");
 module_param_named(pg_mask, amdgpu_pg_mask, uint, 0444);
 
+MODULE_PARM_DESC(sdma_phase_quantum, "SDMA context switch phase quantum (x 1K GPU clock cycles, 0 = no change (default 32))");
+module_param_named(sdma_phase_quantum, amdgpu_sdma_phase_quantum, uint, 0444);
+
 MODULE_PARM_DESC(disable_cu, "Disable CUs (se.sh.cu,...)");
 module_param_named(disable_cu, amdgpu_disable_cu, charp, 0444);
 
@@ -803,7 +820,6 @@ static struct drm_driver kms_driver = {
 	.open = amdgpu_driver_open_kms,
 	.postclose = amdgpu_driver_postclose_kms,
 	.lastclose = amdgpu_driver_lastclose_kms,
-	.set_busid = drm_pci_set_busid,
 	.unload = amdgpu_driver_unload_kms,
 	.get_vblank_counter = amdgpu_get_vblank_counter_kms,
 	.enable_vblank = amdgpu_enable_vblank_kms,
@@ -823,7 +839,6 @@ static struct drm_driver kms_driver = {
 	.gem_close_object = amdgpu_gem_object_close,
 	.dumb_create = amdgpu_mode_dumb_create,
 	.dumb_map_offset = amdgpu_mode_dumb_mmap,
-	.dumb_destroy = drm_gem_dumb_destroy,
 	.fops = &amdgpu_driver_kms_fops,
 
 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
index c0d8c6ff6380..9afa9c097e1f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
@@ -118,7 +118,7 @@ static void amdgpufb_destroy_pinned_object(struct drm_gem_object *gobj)
 		amdgpu_bo_unpin(abo);
 		amdgpu_bo_unreserve(abo);
 	}
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 }
 
 static int amdgpufb_create_pinned_object(struct amdgpu_fbdev *rfbdev,
@@ -245,13 +245,12 @@ static int amdgpufb_create(struct drm_fb_helper *helper,
 
 	drm_fb_helper_fill_fix(info, fb->pitches[0], fb->format->depth);
 
-	info->flags = FBINFO_DEFAULT | FBINFO_CAN_FORCE_OUTPUT;
 	info->fbops = &amdgpufb_ops;
 
 	tmp = amdgpu_bo_gpu_offset(abo) - adev->mc.vram_start;
 	info->fix.smem_start = adev->mc.aper_base + tmp;
 	info->fix.smem_len = amdgpu_bo_size(abo);
-	info->screen_base = abo->kptr;
+	info->screen_base = amdgpu_bo_kptr(abo);
 	info->screen_size = amdgpu_bo_size(abo);
 
 	drm_fb_helper_fill_var(info, &rfbdev->helper, sizes->fb_width, sizes->fb_height);
@@ -281,7 +280,7 @@ out:
 
 	}
 	if (fb && ret) {
-		drm_gem_object_unreference_unlocked(gobj);
+		drm_gem_object_put_unlocked(gobj);
 		drm_framebuffer_unregister_private(fb);
 		drm_framebuffer_cleanup(fb);
 		kfree(fb);
@@ -312,31 +311,7 @@ static int amdgpu_fbdev_destroy(struct drm_device *dev, struct amdgpu_fbdev *rfb
 	return 0;
 }
 
-/** Sets the color ramps on behalf of fbcon */
-static void amdgpu_crtc_fb_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
-				      u16 blue, int regno)
-{
-	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
-
-	amdgpu_crtc->lut_r[regno] = red >> 6;
-	amdgpu_crtc->lut_g[regno] = green >> 6;
-	amdgpu_crtc->lut_b[regno] = blue >> 6;
-}
-
-/** Gets the color ramps on behalf of fbcon */
-static void amdgpu_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green,
-				      u16 *blue, int regno)
-{
-	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
-
-	*red = amdgpu_crtc->lut_r[regno] << 6;
-	*green = amdgpu_crtc->lut_g[regno] << 6;
-	*blue = amdgpu_crtc->lut_b[regno] << 6;
-}
-
 static const struct drm_fb_helper_funcs amdgpu_fb_helper_funcs = {
-	.gamma_set = amdgpu_crtc_fb_gamma_set,
-	.gamma_get = amdgpu_crtc_fb_gamma_get,
 	.fb_probe = amdgpufb_create,
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
index a57abc1a25fb..f4370081f6e6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
@@ -55,6 +55,7 @@
 /*
  * Common GART table functions.
  */
+
 /**
  * amdgpu_gart_table_ram_alloc - allocate system ram for gart page table
  *
@@ -131,7 +132,7 @@ int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev)
 				     PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM,
 				     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
 				     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
-				     NULL, NULL, &adev->gart.robj);
+				     NULL, NULL, 0, &adev->gart.robj);
 		if (r) {
 			return r;
 		}
@@ -263,6 +264,41 @@ int amdgpu_gart_unbind(struct amdgpu_device *adev, uint64_t offset,
 }
 
 /**
+ * amdgpu_gart_map - map dma_addresses into GART entries
+ *
+ * @adev: amdgpu_device pointer
+ * @offset: offset into the GPU's gart aperture
+ * @pages: number of pages to bind
+ * @dma_addr: DMA addresses of pages
+ *
+ * Map the dma_addresses into GART entries (all asics).
+ * Returns 0 for success, -EINVAL for failure.
+ */
+int amdgpu_gart_map(struct amdgpu_device *adev, uint64_t offset,
+		    int pages, dma_addr_t *dma_addr, uint64_t flags,
+		    void *dst)
+{
+	uint64_t page_base;
+	unsigned i, j, t;
+
+	if (!adev->gart.ready) {
+		WARN(1, "trying to bind memory to uninitialized GART !\n");
+		return -EINVAL;
+	}
+
+	t = offset / AMDGPU_GPU_PAGE_SIZE;
+
+	for (i = 0; i < pages; i++) {
+		page_base = dma_addr[i];
+		for (j = 0; j < (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE); j++, t++) {
+			amdgpu_gart_set_pte_pde(adev, dst, t, page_base, flags);
+			page_base += AMDGPU_GPU_PAGE_SIZE;
+		}
+	}
+	return 0;
+}
+
+/**
  * amdgpu_gart_bind - bind pages into the gart page table
  *
  * @adev: amdgpu_device pointer
@@ -279,31 +315,30 @@ int amdgpu_gart_bind(struct amdgpu_device *adev, uint64_t offset,
 		     int pages, struct page **pagelist, dma_addr_t *dma_addr,
 		     uint64_t flags)
 {
-	unsigned t;
-	unsigned p;
-	uint64_t page_base;
-	int i, j;
+#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
+	unsigned i,t,p;
+#endif
+	int r;
 
 	if (!adev->gart.ready) {
 		WARN(1, "trying to bind memory to uninitialized GART !\n");
 		return -EINVAL;
 	}
 
+#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
 	t = offset / AMDGPU_GPU_PAGE_SIZE;
 	p = t / (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE);
-
-	for (i = 0; i < pages; i++, p++) {
-#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
+	for (i = 0; i < pages; i++, p++)
 		adev->gart.pages[p] = pagelist[i];
 #endif
-		if (adev->gart.ptr) {
-			page_base = dma_addr[i];
-			for (j = 0; j < (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE); j++, t++) {
-				amdgpu_gart_set_pte_pde(adev, adev->gart.ptr, t, page_base, flags);
-				page_base += AMDGPU_GPU_PAGE_SIZE;
-			}
-		}
+
+	if (adev->gart.ptr) {
+		r = amdgpu_gart_map(adev, offset, pages, dma_addr, flags,
+			    adev->gart.ptr);
+		if (r)
+			return r;
 	}
+
 	mb();
 	amdgpu_gart_flush_gpu_tlb(adev, 0);
 	return 0;
@@ -333,8 +368,8 @@ int amdgpu_gart_init(struct amdgpu_device *adev)
 	if (r)
 		return r;
 	/* Compute table size */
-	adev->gart.num_cpu_pages = adev->mc.gtt_size / PAGE_SIZE;
-	adev->gart.num_gpu_pages = adev->mc.gtt_size / AMDGPU_GPU_PAGE_SIZE;
+	adev->gart.num_cpu_pages = adev->mc.gart_size / PAGE_SIZE;
+	adev->gart.num_gpu_pages = adev->mc.gart_size / AMDGPU_GPU_PAGE_SIZE;
 	DRM_INFO("GART: num cpu pages %u, num gpu pages %u\n",
 		 adev->gart.num_cpu_pages, adev->gart.num_gpu_pages);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.h
new file mode 100644
index 000000000000..afbe803b1a13
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.h
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __AMDGPU_GART_H__
+#define __AMDGPU_GART_H__
+
+#include <linux/types.h>
+
+/*
+ * GART structures, functions & helpers
+ */
+struct amdgpu_device;
+struct amdgpu_bo;
+struct amdgpu_gart_funcs;
+
+#define AMDGPU_GPU_PAGE_SIZE 4096
+#define AMDGPU_GPU_PAGE_MASK (AMDGPU_GPU_PAGE_SIZE - 1)
+#define AMDGPU_GPU_PAGE_SHIFT 12
+#define AMDGPU_GPU_PAGE_ALIGN(a) (((a) + AMDGPU_GPU_PAGE_MASK) & ~AMDGPU_GPU_PAGE_MASK)
+
+struct amdgpu_gart {
+	dma_addr_t			table_addr;
+	struct amdgpu_bo		*robj;
+	void				*ptr;
+	unsigned			num_gpu_pages;
+	unsigned			num_cpu_pages;
+	unsigned			table_size;
+#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
+	struct page			**pages;
+#endif
+	bool				ready;
+
+	/* Asic default pte flags */
+	uint64_t			gart_pte_flags;
+
+	const struct amdgpu_gart_funcs *gart_funcs;
+};
+
+int amdgpu_gart_table_ram_alloc(struct amdgpu_device *adev);
+void amdgpu_gart_table_ram_free(struct amdgpu_device *adev);
+int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev);
+void amdgpu_gart_table_vram_free(struct amdgpu_device *adev);
+int amdgpu_gart_table_vram_pin(struct amdgpu_device *adev);
+void amdgpu_gart_table_vram_unpin(struct amdgpu_device *adev);
+int amdgpu_gart_init(struct amdgpu_device *adev);
+void amdgpu_gart_fini(struct amdgpu_device *adev);
+int amdgpu_gart_unbind(struct amdgpu_device *adev, uint64_t offset,
+		       int pages);
+int amdgpu_gart_map(struct amdgpu_device *adev, uint64_t offset,
+		    int pages, dma_addr_t *dma_addr, uint64_t flags,
+		    void *dst);
+int amdgpu_gart_bind(struct amdgpu_device *adev, uint64_t offset,
+		     int pages, struct page **pagelist,
+		     dma_addr_t *dma_addr, uint64_t flags);
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 621f739103a6..7171968f261e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -49,7 +49,6 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
 				struct drm_gem_object **obj)
 {
 	struct amdgpu_bo *robj;
-	unsigned long max_size;
 	int r;
 
 	*obj = NULL;
@@ -58,20 +57,9 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
 		alignment = PAGE_SIZE;
 	}
 
-	if (!(initial_domain & (AMDGPU_GEM_DOMAIN_GDS | AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA))) {
-		/* Maximum bo size is the unpinned gtt size since we use the gtt to
-		 * handle vram to system pool migrations.
-		 */
-		max_size = adev->mc.gtt_size - adev->gart_pin_size;
-		if (size > max_size) {
-			DRM_DEBUG("Allocation size %ldMb bigger than %ldMb limit\n",
-				  size >> 20, max_size >> 20);
-			return -ENOMEM;
-		}
-	}
 retry:
 	r = amdgpu_bo_create(adev, size, alignment, kernel, initial_domain,
-			     flags, NULL, NULL, &robj);
+			     flags, NULL, NULL, 0, &robj);
 	if (r) {
 		if (r != -ERESTARTSYS) {
 			if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) {
@@ -103,7 +91,7 @@ void amdgpu_gem_force_release(struct amdgpu_device *adev)
 		spin_lock(&file->table_lock);
 		idr_for_each_entry(&file->object_idr, gobj, handle) {
 			WARN_ONCE(1, "And also active allocations!\n");
-			drm_gem_object_unreference_unlocked(gobj);
+			drm_gem_object_put_unlocked(gobj);
 		}
 		idr_destroy(&file->object_idr);
 		spin_unlock(&file->table_lock);
@@ -237,9 +225,7 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
 	if (args->in.domain_flags & ~(AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
 				      AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
 				      AMDGPU_GEM_CREATE_CPU_GTT_USWC |
-				      AMDGPU_GEM_CREATE_VRAM_CLEARED|
-				      AMDGPU_GEM_CREATE_SHADOW |
-				      AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS))
+				      AMDGPU_GEM_CREATE_VRAM_CLEARED))
 		return -EINVAL;
 
 	/* reject invalid gem domains */
@@ -275,7 +261,7 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
 
 	r = drm_gem_handle_create(filp, gobj, &handle);
 	/* drop reference from allocate - handle holds it now */
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 	if (r)
 		return r;
 
@@ -318,7 +304,7 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
 		return r;
 
 	bo = gem_to_amdgpu_bo(gobj);
-	bo->prefered_domains = AMDGPU_GEM_DOMAIN_GTT;
+	bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
 	bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
 	r = amdgpu_ttm_tt_set_userptr(bo->tbo.ttm, args->addr, args->flags);
 	if (r)
@@ -353,7 +339,7 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
 
 	r = drm_gem_handle_create(filp, gobj, &handle);
 	/* drop reference from allocate - handle holds it now */
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 	if (r)
 		return r;
 
@@ -367,7 +353,7 @@ unlock_mmap_sem:
 	up_read(&current->mm->mmap_sem);
 
 release_object:
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 
 	return r;
 }
@@ -386,11 +372,11 @@ int amdgpu_mode_dumb_mmap(struct drm_file *filp,
 	robj = gem_to_amdgpu_bo(gobj);
 	if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm) ||
 	    (robj->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) {
-		drm_gem_object_unreference_unlocked(gobj);
+		drm_gem_object_put_unlocked(gobj);
 		return -EPERM;
 	}
 	*offset_p = amdgpu_bo_mmap_offset(robj);
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 	return 0;
 }
 
@@ -460,7 +446,7 @@ int amdgpu_gem_wait_idle_ioctl(struct drm_device *dev, void *data,
 	} else
 		r = ret;
 
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 	return r;
 }
 
@@ -503,7 +489,7 @@ int amdgpu_gem_metadata_ioctl(struct drm_device *dev, void *data,
 unreserve:
 	amdgpu_bo_unreserve(robj);
 out:
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 	return r;
 }
 
@@ -635,7 +621,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
 
 	switch (args->operation) {
 	case AMDGPU_VA_OP_MAP:
-		r = amdgpu_vm_alloc_pts(adev, bo_va->vm, args->va_address,
+		r = amdgpu_vm_alloc_pts(adev, bo_va->base.vm, args->va_address,
 					args->map_size);
 		if (r)
 			goto error_backoff;
@@ -655,7 +641,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
 						args->map_size);
 		break;
 	case AMDGPU_VA_OP_REPLACE:
-		r = amdgpu_vm_alloc_pts(adev, bo_va->vm, args->va_address,
+		r = amdgpu_vm_alloc_pts(adev, bo_va->base.vm, args->va_address,
 					args->map_size);
 		if (r)
 			goto error_backoff;
@@ -676,7 +662,7 @@ error_backoff:
 	ttm_eu_backoff_reservation(&ticket, &list);
 
 error_unref:
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 	return r;
 }
 
@@ -701,11 +687,11 @@ int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
 	switch (args->op) {
 	case AMDGPU_GEM_OP_GET_GEM_CREATE_INFO: {
 		struct drm_amdgpu_gem_create_in info;
-		void __user *out = (void __user *)(uintptr_t)args->value;
+		void __user *out = u64_to_user_ptr(args->value);
 
 		info.bo_size = robj->gem_base.size;
 		info.alignment = robj->tbo.mem.page_alignment << PAGE_SHIFT;
-		info.domains = robj->prefered_domains;
+		info.domains = robj->preferred_domains;
 		info.domain_flags = robj->flags;
 		amdgpu_bo_unreserve(robj);
 		if (copy_to_user(out, &info, sizeof(info)))
@@ -723,10 +709,10 @@ int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
 			amdgpu_bo_unreserve(robj);
 			break;
 		}
-		robj->prefered_domains = args->value & (AMDGPU_GEM_DOMAIN_VRAM |
+		robj->preferred_domains = args->value & (AMDGPU_GEM_DOMAIN_VRAM |
 							AMDGPU_GEM_DOMAIN_GTT |
 							AMDGPU_GEM_DOMAIN_CPU);
-		robj->allowed_domains = robj->prefered_domains;
+		robj->allowed_domains = robj->preferred_domains;
 		if (robj->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
 			robj->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
 
@@ -738,7 +724,7 @@ int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
 	}
 
 out:
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 	return r;
 }
 
@@ -766,7 +752,7 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv,
 
 	r = drm_gem_handle_create(file_priv, gobj, &handle);
 	/* drop reference from allocate - handle holds it now */
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 	if (r) {
 		return r;
 	}
@@ -784,6 +770,7 @@ static int amdgpu_debugfs_gem_bo_info(int id, void *ptr, void *data)
 	unsigned domain;
 	const char *placement;
 	unsigned pin_count;
+	uint64_t offset;
 
 	domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
 	switch (domain) {
@@ -798,9 +785,12 @@ static int amdgpu_debugfs_gem_bo_info(int id, void *ptr, void *data)
 		placement = " CPU";
 		break;
 	}
-	seq_printf(m, "\t0x%08x: %12ld byte %s @ 0x%010Lx",
-		   id, amdgpu_bo_size(bo), placement,
-		   amdgpu_bo_gpu_offset(bo));
+	seq_printf(m, "\t0x%08x: %12ld byte %s",
+		   id, amdgpu_bo_size(bo), placement);
+
+	offset = ACCESS_ONCE(bo->tbo.mem.start);
+	if (offset != AMDGPU_BO_INVALID_OFFSET)
+		seq_printf(m, " @ 0x%010Lx", offset);
 
 	pin_count = ACCESS_ONCE(bo->pin_count);
 	if (pin_count)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index e26108aad3fe..4f6c68fc1dd9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -125,7 +125,8 @@ void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev)
 		if (mec >= adev->gfx.mec.num_mec)
 			break;
 
-		if (adev->gfx.mec.num_mec > 1) {
+		/* FIXME: spreading the queues across pipes causes perf regressions */
+		if (0) {
 			/* policy: amdgpu owns the first two queues of the first MEC */
 			if (mec == 0 && queue < 2)
 				set_bit(i, adev->gfx.mec.queue_bitmap);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
index f7d22c44034d..0d15eb7d31d7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
@@ -28,7 +28,7 @@
 struct amdgpu_gtt_mgr {
 	struct drm_mm mm;
 	spinlock_t lock;
-	uint64_t available;
+	atomic64_t available;
 };
 
 /**
@@ -42,15 +42,19 @@ struct amdgpu_gtt_mgr {
 static int amdgpu_gtt_mgr_init(struct ttm_mem_type_manager *man,
 			       unsigned long p_size)
 {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
 	struct amdgpu_gtt_mgr *mgr;
+	uint64_t start, size;
 
 	mgr = kzalloc(sizeof(*mgr), GFP_KERNEL);
 	if (!mgr)
 		return -ENOMEM;
 
-	drm_mm_init(&mgr->mm, 0, p_size);
+	start = AMDGPU_GTT_MAX_TRANSFER_SIZE * AMDGPU_GTT_NUM_TRANSFER_WINDOWS;
+	size = (adev->mc.gart_size >> PAGE_SHIFT) - start;
+	drm_mm_init(&mgr->mm, start, size);
 	spin_lock_init(&mgr->lock);
-	mgr->available = p_size;
+	atomic64_set(&mgr->available, p_size);
 	man->priv = mgr;
 	return 0;
 }
@@ -81,6 +85,20 @@ static int amdgpu_gtt_mgr_fini(struct ttm_mem_type_manager *man)
 }
 
 /**
+ * amdgpu_gtt_mgr_is_allocated - Check if mem has address space
+ *
+ * @mem: the mem object to check
+ *
+ * Check if a mem object has already address space allocated.
+ */
+bool amdgpu_gtt_mgr_is_allocated(struct ttm_mem_reg *mem)
+{
+	struct drm_mm_node *node = mem->mm_node;
+
+	return (node->start != AMDGPU_BO_INVALID_OFFSET);
+}
+
+/**
  * amdgpu_gtt_mgr_alloc - allocate new ranges
  *
  * @man: TTM memory type manager
@@ -90,18 +108,19 @@ static int amdgpu_gtt_mgr_fini(struct ttm_mem_type_manager *man)
  *
  * Allocate the address space for a node.
  */
-int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,
-			 struct ttm_buffer_object *tbo,
-			 const struct ttm_place *place,
-			 struct ttm_mem_reg *mem)
+static int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,
+				struct ttm_buffer_object *tbo,
+				const struct ttm_place *place,
+				struct ttm_mem_reg *mem)
 {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
 	struct amdgpu_gtt_mgr *mgr = man->priv;
 	struct drm_mm_node *node = mem->mm_node;
 	enum drm_mm_insert_mode mode;
 	unsigned long fpfn, lpfn;
 	int r;
 
-	if (node->start != AMDGPU_BO_INVALID_OFFSET)
+	if (amdgpu_gtt_mgr_is_allocated(mem))
 		return 0;
 
 	if (place)
@@ -112,7 +131,7 @@ int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,
 	if (place && place->lpfn)
 		lpfn = place->lpfn;
 	else
-		lpfn = man->size;
+		lpfn = adev->gart.num_cpu_pages;
 
 	mode = DRM_MM_INSERT_BEST;
 	if (place && place->flags & TTM_PL_FLAG_TOPDOWN)
@@ -124,25 +143,12 @@ int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,
 					fpfn, lpfn, mode);
 	spin_unlock(&mgr->lock);
 
-	if (!r) {
+	if (!r)
 		mem->start = node->start;
-		if (&tbo->mem == mem)
-			tbo->offset = (tbo->mem.start << PAGE_SHIFT) +
-			    tbo->bdev->man[tbo->mem.mem_type].gpu_offset;
-	}
 
 	return r;
 }
 
-void amdgpu_gtt_mgr_print(struct seq_file *m, struct ttm_mem_type_manager *man)
-{
-	struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
-	struct amdgpu_gtt_mgr *mgr = man->priv;
-
-	seq_printf(m, "man size:%llu pages, gtt available:%llu pages, usage:%lluMB\n",
-		   man->size, mgr->available, (u64)atomic64_read(&adev->gtt_usage) >> 20);
-
-}
 /**
  * amdgpu_gtt_mgr_new - allocate a new node
  *
@@ -163,11 +169,11 @@ static int amdgpu_gtt_mgr_new(struct ttm_mem_type_manager *man,
 	int r;
 
 	spin_lock(&mgr->lock);
-	if (mgr->available < mem->num_pages) {
+	if (atomic64_read(&mgr->available) < mem->num_pages) {
 		spin_unlock(&mgr->lock);
 		return 0;
 	}
-	mgr->available -= mem->num_pages;
+	atomic64_sub(mem->num_pages, &mgr->available);
 	spin_unlock(&mgr->lock);
 
 	node = kzalloc(sizeof(*node), GFP_KERNEL);
@@ -194,9 +200,7 @@ static int amdgpu_gtt_mgr_new(struct ttm_mem_type_manager *man,
 
 	return 0;
 err_out:
-	spin_lock(&mgr->lock);
-	mgr->available += mem->num_pages;
-	spin_unlock(&mgr->lock);
+	atomic64_add(mem->num_pages, &mgr->available);
 
 	return r;
 }
@@ -223,30 +227,47 @@ static void amdgpu_gtt_mgr_del(struct ttm_mem_type_manager *man,
 	spin_lock(&mgr->lock);
 	if (node->start != AMDGPU_BO_INVALID_OFFSET)
 		drm_mm_remove_node(node);
-	mgr->available += mem->num_pages;
 	spin_unlock(&mgr->lock);
+	atomic64_add(mem->num_pages, &mgr->available);
 
 	kfree(node);
 	mem->mm_node = NULL;
 }
 
 /**
+ * amdgpu_gtt_mgr_usage - return usage of GTT domain
+ *
+ * @man: TTM memory type manager
+ *
+ * Return how many bytes are used in the GTT domain
+ */
+uint64_t amdgpu_gtt_mgr_usage(struct ttm_mem_type_manager *man)
+{
+	struct amdgpu_gtt_mgr *mgr = man->priv;
+
+	return (u64)(man->size - atomic64_read(&mgr->available)) * PAGE_SIZE;
+}
+
+/**
  * amdgpu_gtt_mgr_debug - dump VRAM table
  *
  * @man: TTM memory type manager
- * @prefix: text prefix
+ * @printer: DRM printer to use
  *
  * Dump the table content using printk.
  */
 static void amdgpu_gtt_mgr_debug(struct ttm_mem_type_manager *man,
-				  const char *prefix)
+				 struct drm_printer *printer)
 {
 	struct amdgpu_gtt_mgr *mgr = man->priv;
-	struct drm_printer p = drm_debug_printer(prefix);
 
 	spin_lock(&mgr->lock);
-	drm_mm_print(&mgr->mm, &p);
+	drm_mm_print(&mgr->mm, printer);
 	spin_unlock(&mgr->lock);
+
+	drm_printf(printer, "man size:%llu pages, gtt available:%llu pages, usage:%lluMB\n",
+		   man->size, (u64)atomic64_read(&mgr->available),
+		   amdgpu_gtt_mgr_usage(man) >> 20);
 }
 
 const struct ttm_mem_type_manager_func amdgpu_gtt_mgr_func = {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
index f774b3f497d2..659997bfff30 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
@@ -130,6 +130,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 
 	unsigned i;
 	int r = 0;
+	bool need_pipe_sync = false;
 
 	if (num_ibs == 0)
 		return -EINVAL;
@@ -165,15 +166,15 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 	if (ring->funcs->emit_pipeline_sync && job &&
 	    ((tmp = amdgpu_sync_get_fence(&job->sched_sync)) ||
 	     amdgpu_vm_need_pipeline_sync(ring, job))) {
-		amdgpu_ring_emit_pipeline_sync(ring);
+		need_pipe_sync = true;
 		dma_fence_put(tmp);
 	}
 
 	if (ring->funcs->insert_start)
 		ring->funcs->insert_start(ring);
 
-	if (vm) {
-		r = amdgpu_vm_flush(ring, job);
+	if (job) {
+		r = amdgpu_vm_flush(ring, job, need_pipe_sync);
 		if (r) {
 			amdgpu_ring_undo(ring);
 			return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
index 62da6c5c6095..538e5f27d120 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
@@ -220,6 +220,11 @@ int amdgpu_irq_init(struct amdgpu_device *adev)
 	int r = 0;
 
 	spin_lock_init(&adev->irq.lock);
+
+	if (!adev->enable_virtual_display)
+		/* Disable vblank irqs aggressively for power-saving */
+		adev->ddev->vblank_disable_immediate = true;
+
 	r = drm_vblank_init(adev->ddev, adev->mode_info.num_crtc);
 	if (r) {
 		return r;
@@ -263,7 +268,6 @@ void amdgpu_irq_fini(struct amdgpu_device *adev)
 {
 	unsigned i, j;
 
-	drm_vblank_cleanup(adev->ddev);
 	if (adev->irq.installed) {
 		drm_irq_uninstall(adev->ddev);
 		adev->irq.installed = false;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index 3d641e10e6b6..4510627ae83e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -81,6 +81,8 @@ int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev, unsigned size,
 	r = amdgpu_ib_get(adev, NULL, size, &(*job)->ibs[0]);
 	if (r)
 		kfree(*job);
+	else
+		(*job)->vm_pd_addr = adev->gart.table_addr;
 
 	return r;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index b0b23101d1c8..e16229000a98 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -158,7 +158,6 @@ int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags)
 				"Error during ACPI methods call\n");
 	}
 
-	amdgpu_amdkfd_load_interface(adev);
 	amdgpu_amdkfd_device_probe(adev);
 	amdgpu_amdkfd_device_init(adev);
 
@@ -456,13 +455,13 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 		ui64 = atomic64_read(&adev->num_vram_cpu_page_faults);
 		return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0;
 	case AMDGPU_INFO_VRAM_USAGE:
-		ui64 = atomic64_read(&adev->vram_usage);
+		ui64 = amdgpu_vram_mgr_usage(&adev->mman.bdev.man[TTM_PL_VRAM]);
 		return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0;
 	case AMDGPU_INFO_VIS_VRAM_USAGE:
-		ui64 = atomic64_read(&adev->vram_vis_usage);
+		ui64 = amdgpu_vram_mgr_vis_usage(&adev->mman.bdev.man[TTM_PL_VRAM]);
 		return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0;
 	case AMDGPU_INFO_GTT_USAGE:
-		ui64 = atomic64_read(&adev->gtt_usage);
+		ui64 = amdgpu_gtt_mgr_usage(&adev->mman.bdev.man[TTM_PL_TT]);
 		return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0;
 	case AMDGPU_INFO_GDS_CONFIG: {
 		struct drm_amdgpu_info_gds gds_info;
@@ -485,7 +484,8 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 		vram_gtt.vram_size -= adev->vram_pin_size;
 		vram_gtt.vram_cpu_accessible_size = adev->mc.visible_vram_size;
 		vram_gtt.vram_cpu_accessible_size -= (adev->vram_pin_size - adev->invisible_pin_size);
-		vram_gtt.gtt_size  = adev->mc.gtt_size;
+		vram_gtt.gtt_size = adev->mman.bdev.man[TTM_PL_TT].size;
+		vram_gtt.gtt_size *= PAGE_SIZE;
 		vram_gtt.gtt_size -= adev->gart_pin_size;
 		return copy_to_user(out, &vram_gtt,
 				    min((size_t)size, sizeof(vram_gtt))) ? -EFAULT : 0;
@@ -497,7 +497,8 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 		mem.vram.total_heap_size = adev->mc.real_vram_size;
 		mem.vram.usable_heap_size =
 			adev->mc.real_vram_size - adev->vram_pin_size;
-		mem.vram.heap_usage = atomic64_read(&adev->vram_usage);
+		mem.vram.heap_usage =
+			amdgpu_vram_mgr_usage(&adev->mman.bdev.man[TTM_PL_VRAM]);
 		mem.vram.max_allocation = mem.vram.usable_heap_size * 3 / 4;
 
 		mem.cpu_accessible_vram.total_heap_size =
@@ -506,14 +507,16 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 			adev->mc.visible_vram_size -
 			(adev->vram_pin_size - adev->invisible_pin_size);
 		mem.cpu_accessible_vram.heap_usage =
-			atomic64_read(&adev->vram_vis_usage);
+			amdgpu_vram_mgr_vis_usage(&adev->mman.bdev.man[TTM_PL_VRAM]);
 		mem.cpu_accessible_vram.max_allocation =
 			mem.cpu_accessible_vram.usable_heap_size * 3 / 4;
 
-		mem.gtt.total_heap_size = adev->mc.gtt_size;
-		mem.gtt.usable_heap_size =
-			adev->mc.gtt_size - adev->gart_pin_size;
-		mem.gtt.heap_usage = atomic64_read(&adev->gtt_usage);
+		mem.gtt.total_heap_size = adev->mman.bdev.man[TTM_PL_TT].size;
+		mem.gtt.total_heap_size *= PAGE_SIZE;
+		mem.gtt.usable_heap_size = mem.gtt.total_heap_size
+			- adev->gart_pin_size;
+		mem.gtt.heap_usage =
+			amdgpu_gtt_mgr_usage(&adev->mman.bdev.man[TTM_PL_TT]);
 		mem.gtt.max_allocation = mem.gtt.usable_heap_size * 3 / 4;
 
 		return copy_to_user(out, &mem,
@@ -571,8 +574,8 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 			dev_info.max_engine_clock = amdgpu_dpm_get_sclk(adev, false) * 10;
 			dev_info.max_memory_clock = amdgpu_dpm_get_mclk(adev, false) * 10;
 		} else {
-			dev_info.max_engine_clock = adev->pm.default_sclk * 10;
-			dev_info.max_memory_clock = adev->pm.default_mclk * 10;
+			dev_info.max_engine_clock = adev->clock.default_sclk * 10;
+			dev_info.max_memory_clock = adev->clock.default_mclk * 10;
 		}
 		dev_info.enabled_rb_pipes_mask = adev->gfx.config.backend_enable_mask;
 		dev_info.num_rb_pipes = adev->gfx.config.max_backends_per_se *
@@ -587,10 +590,8 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 		dev_info.virtual_address_offset = AMDGPU_VA_RESERVED_SIZE;
 		dev_info.virtual_address_max = (uint64_t)adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
 		dev_info.virtual_address_alignment = max((int)PAGE_SIZE, AMDGPU_GPU_PAGE_SIZE);
-		dev_info.pte_fragment_size = (1 << AMDGPU_LOG2_PAGES_PER_FRAG) *
-					     AMDGPU_GPU_PAGE_SIZE;
+		dev_info.pte_fragment_size = (1 << adev->vm_manager.fragment_size) * AMDGPU_GPU_PAGE_SIZE;
 		dev_info.gart_page_size = AMDGPU_GPU_PAGE_SIZE;
-
 		dev_info.cu_active_number = adev->gfx.cu_info.number;
 		dev_info.cu_ao_mask = adev->gfx.cu_info.ao_cu_mask;
 		dev_info.ce_ram_size = adev->gfx.ce_ram_size;
@@ -839,7 +840,7 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
 	}
 
 	if (amdgpu_sriov_vf(adev)) {
-		r = amdgpu_map_static_csa(adev, &fpriv->vm);
+		r = amdgpu_map_static_csa(adev, &fpriv->vm, &fpriv->csa_va);
 		if (r)
 			goto out_suspend;
 	}
@@ -892,8 +893,8 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
 	if (amdgpu_sriov_vf(adev)) {
 		/* TODO: how to handle reserve failure */
 		BUG_ON(amdgpu_bo_reserve(adev->virt.csa_obj, true));
-		amdgpu_vm_bo_rmv(adev, fpriv->vm.csa_bo_va);
-		fpriv->vm.csa_bo_va = NULL;
+		amdgpu_vm_bo_rmv(adev, fpriv->csa_va);
+		fpriv->csa_va = NULL;
 		amdgpu_bo_unreserve(adev->virt.csa_obj);
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
index 6558a3ed57a7..3b0f2ec6eec7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
@@ -51,7 +51,7 @@ struct amdgpu_mn {
 
 	/* objects protected by lock */
 	struct mutex		lock;
-	struct rb_root		objects;
+	struct rb_root_cached	objects;
 };
 
 struct amdgpu_mn_node {
@@ -76,8 +76,8 @@ static void amdgpu_mn_destroy(struct work_struct *work)
 	mutex_lock(&adev->mn_lock);
 	mutex_lock(&rmn->lock);
 	hash_del(&rmn->node);
-	rbtree_postorder_for_each_entry_safe(node, next_node, &rmn->objects,
-					     it.rb) {
+	rbtree_postorder_for_each_entry_safe(node, next_node,
+					     &rmn->objects.rb_root, it.rb) {
 		list_for_each_entry_safe(bo, next_bo, &node->bos, mn_list) {
 			bo->mn = NULL;
 			list_del_init(&bo->mn_list);
@@ -147,36 +147,6 @@ static void amdgpu_mn_invalidate_node(struct amdgpu_mn_node *node,
 }
 
 /**
- * amdgpu_mn_invalidate_page - callback to notify about mm change
- *
- * @mn: our notifier
- * @mn: the mm this callback is about
- * @address: address of invalidate page
- *
- * Invalidation of a single page. Blocks for all BOs mapping it
- * and unmap them by move them into system domain again.
- */
-static void amdgpu_mn_invalidate_page(struct mmu_notifier *mn,
-				      struct mm_struct *mm,
-				      unsigned long address)
-{
-	struct amdgpu_mn *rmn = container_of(mn, struct amdgpu_mn, mn);
-	struct interval_tree_node *it;
-
-	mutex_lock(&rmn->lock);
-
-	it = interval_tree_iter_first(&rmn->objects, address, address);
-	if (it) {
-		struct amdgpu_mn_node *node;
-
-		node = container_of(it, struct amdgpu_mn_node, it);
-		amdgpu_mn_invalidate_node(node, address, address);
-	}
-
-	mutex_unlock(&rmn->lock);
-}
-
-/**
  * amdgpu_mn_invalidate_range_start - callback to notify about mm change
  *
  * @mn: our notifier
@@ -215,7 +185,6 @@ static void amdgpu_mn_invalidate_range_start(struct mmu_notifier *mn,
 
 static const struct mmu_notifier_ops amdgpu_mn_ops = {
 	.release = amdgpu_mn_release,
-	.invalidate_page = amdgpu_mn_invalidate_page,
 	.invalidate_range_start = amdgpu_mn_invalidate_range_start,
 };
 
@@ -252,7 +221,7 @@ static struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev)
 	rmn->mm = mm;
 	rmn->mn.ops = &amdgpu_mn_ops;
 	mutex_init(&rmn->lock);
-	rmn->objects = RB_ROOT;
+	rmn->objects = RB_ROOT_CACHED;
 
 	r = __mmu_notifier_register(&rmn->mn, mm);
 	if (r)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
index 43a9d3aec6c4..2af2678ddaf6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
@@ -257,15 +257,7 @@ struct amdgpu_audio {
 	int num_pins;
 };
 
-struct amdgpu_mode_mc_save {
-	u32 vga_render_control;
-	u32 vga_hdp_control;
-	bool crtc_enabled[AMDGPU_MAX_CRTCS];
-};
-
 struct amdgpu_display_funcs {
-	/* vga render */
-	void (*set_vga_render_state)(struct amdgpu_device *adev, bool render);
 	/* display watermarks */
 	void (*bandwidth_update)(struct amdgpu_device *adev);
 	/* get frame count */
@@ -300,10 +292,6 @@ struct amdgpu_display_funcs {
 			      uint16_t connector_object_id,
 			      struct amdgpu_hpd *hpd,
 			      struct amdgpu_router *router);
-	void (*stop_mc_access)(struct amdgpu_device *adev,
-			       struct amdgpu_mode_mc_save *save);
-	void (*resume_mc_access)(struct amdgpu_device *adev,
-				 struct amdgpu_mode_mc_save *save);
 };
 
 struct amdgpu_mode_info {
@@ -369,7 +357,6 @@ struct amdgpu_atom_ss {
 struct amdgpu_crtc {
 	struct drm_crtc base;
 	int crtc_id;
-	u16 lut_r[256], lut_g[256], lut_b[256];
 	bool enabled;
 	bool can_tile;
 	uint32_t crtc_offset;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index 8ee69652be8c..9e495da0bb03 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -37,55 +37,6 @@
 #include "amdgpu.h"
 #include "amdgpu_trace.h"
 
-
-
-static u64 amdgpu_get_vis_part_size(struct amdgpu_device *adev,
-						struct ttm_mem_reg *mem)
-{
-	if (mem->start << PAGE_SHIFT >= adev->mc.visible_vram_size)
-		return 0;
-
-	return ((mem->start << PAGE_SHIFT) + mem->size) >
-		adev->mc.visible_vram_size ?
-		adev->mc.visible_vram_size - (mem->start << PAGE_SHIFT) :
-		mem->size;
-}
-
-static void amdgpu_update_memory_usage(struct amdgpu_device *adev,
-		       struct ttm_mem_reg *old_mem,
-		       struct ttm_mem_reg *new_mem)
-{
-	u64 vis_size;
-	if (!adev)
-		return;
-
-	if (new_mem) {
-		switch (new_mem->mem_type) {
-		case TTM_PL_TT:
-			atomic64_add(new_mem->size, &adev->gtt_usage);
-			break;
-		case TTM_PL_VRAM:
-			atomic64_add(new_mem->size, &adev->vram_usage);
-			vis_size = amdgpu_get_vis_part_size(adev, new_mem);
-			atomic64_add(vis_size, &adev->vram_vis_usage);
-			break;
-		}
-	}
-
-	if (old_mem) {
-		switch (old_mem->mem_type) {
-		case TTM_PL_TT:
-			atomic64_sub(old_mem->size, &adev->gtt_usage);
-			break;
-		case TTM_PL_VRAM:
-			atomic64_sub(old_mem->size, &adev->vram_usage);
-			vis_size = amdgpu_get_vis_part_size(adev, old_mem);
-			atomic64_sub(vis_size, &adev->vram_vis_usage);
-			break;
-		}
-	}
-}
-
 static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
@@ -93,7 +44,7 @@ static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
 
 	bo = container_of(tbo, struct amdgpu_bo, tbo);
 
-	amdgpu_update_memory_usage(adev, &bo->tbo.mem, NULL);
+	amdgpu_bo_kunmap(bo);
 
 	drm_gem_object_release(&bo->gem_base);
 	amdgpu_bo_unref(&bo->parent);
@@ -140,7 +91,10 @@ static void amdgpu_ttm_placement_init(struct amdgpu_device *adev,
 
 	if (domain & AMDGPU_GEM_DOMAIN_GTT) {
 		places[c].fpfn = 0;
-		places[c].lpfn = 0;
+		if (flags & AMDGPU_GEM_CREATE_SHADOW)
+			places[c].lpfn = adev->mc.gart_size >> PAGE_SHIFT;
+		else
+			places[c].lpfn = 0;
 		places[c].flags = TTM_PL_FLAG_TT;
 		if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
 			places[c].flags |= TTM_PL_FLAG_WC |
@@ -219,7 +173,7 @@ static void amdgpu_fill_placement_to_bo(struct amdgpu_bo *bo,
 }
 
 /**
- * amdgpu_bo_create_kernel - create BO for kernel use
+ * amdgpu_bo_create_reserved - create reserved BO for kernel use
  *
  * @adev: amdgpu device object
  * @size: size for the new BO
@@ -229,24 +183,30 @@ static void amdgpu_fill_placement_to_bo(struct amdgpu_bo *bo,
  * @gpu_addr: GPU addr of the pinned BO
  * @cpu_addr: optional CPU address mapping
  *
- * Allocates and pins a BO for kernel internal use.
+ * Allocates and pins a BO for kernel internal use, and returns it still
+ * reserved.
  *
  * Returns 0 on success, negative error code otherwise.
  */
-int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
-			    unsigned long size, int align,
-			    u32 domain, struct amdgpu_bo **bo_ptr,
-			    u64 *gpu_addr, void **cpu_addr)
+int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
+			      unsigned long size, int align,
+			      u32 domain, struct amdgpu_bo **bo_ptr,
+			      u64 *gpu_addr, void **cpu_addr)
 {
+	bool free = false;
 	int r;
 
-	r = amdgpu_bo_create(adev, size, align, true, domain,
-			     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
-			     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
-			     NULL, NULL, bo_ptr);
-	if (r) {
-		dev_err(adev->dev, "(%d) failed to allocate kernel bo\n", r);
-		return r;
+	if (!*bo_ptr) {
+		r = amdgpu_bo_create(adev, size, align, true, domain,
+				     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+				     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
+				     NULL, NULL, 0, bo_ptr);
+		if (r) {
+			dev_err(adev->dev, "(%d) failed to allocate kernel bo\n",
+				r);
+			return r;
+		}
+		free = true;
 	}
 
 	r = amdgpu_bo_reserve(*bo_ptr, false);
@@ -269,20 +229,52 @@ int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
 		}
 	}
 
-	amdgpu_bo_unreserve(*bo_ptr);
-
 	return 0;
 
 error_unreserve:
 	amdgpu_bo_unreserve(*bo_ptr);
 
 error_free:
-	amdgpu_bo_unref(bo_ptr);
+	if (free)
+		amdgpu_bo_unref(bo_ptr);
 
 	return r;
 }
 
 /**
+ * amdgpu_bo_create_kernel - create BO for kernel use
+ *
+ * @adev: amdgpu device object
+ * @size: size for the new BO
+ * @align: alignment for the new BO
+ * @domain: where to place it
+ * @bo_ptr: resulting BO
+ * @gpu_addr: GPU addr of the pinned BO
+ * @cpu_addr: optional CPU address mapping
+ *
+ * Allocates and pins a BO for kernel internal use.
+ *
+ * Returns 0 on success, negative error code otherwise.
+ */
+int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
+			    unsigned long size, int align,
+			    u32 domain, struct amdgpu_bo **bo_ptr,
+			    u64 *gpu_addr, void **cpu_addr)
+{
+	int r;
+
+	r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr,
+				      gpu_addr, cpu_addr);
+
+	if (r)
+		return r;
+
+	amdgpu_bo_unreserve(*bo_ptr);
+
+	return 0;
+}
+
+/**
  * amdgpu_bo_free_kernel - free BO for kernel use
  *
  * @bo: amdgpu BO to free
@@ -317,12 +309,13 @@ int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
 				struct sg_table *sg,
 				struct ttm_placement *placement,
 				struct reservation_object *resv,
+				uint64_t init_value,
 				struct amdgpu_bo **bo_ptr)
 {
 	struct amdgpu_bo *bo;
 	enum ttm_bo_type type;
 	unsigned long page_align;
-	u64 initial_bytes_moved;
+	u64 initial_bytes_moved, bytes_moved;
 	size_t acc_size;
 	int r;
 
@@ -351,13 +344,13 @@ int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
 	}
 	INIT_LIST_HEAD(&bo->shadow_list);
 	INIT_LIST_HEAD(&bo->va);
-	bo->prefered_domains = domain & (AMDGPU_GEM_DOMAIN_VRAM |
+	bo->preferred_domains = domain & (AMDGPU_GEM_DOMAIN_VRAM |
 					 AMDGPU_GEM_DOMAIN_GTT |
 					 AMDGPU_GEM_DOMAIN_CPU |
 					 AMDGPU_GEM_DOMAIN_GDS |
 					 AMDGPU_GEM_DOMAIN_GWS |
 					 AMDGPU_GEM_DOMAIN_OA);
-	bo->allowed_domains = bo->prefered_domains;
+	bo->allowed_domains = bo->preferred_domains;
 	if (!kernel && bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
 		bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
 
@@ -398,8 +391,14 @@ int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
 	r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, type,
 				 &bo->placement, page_align, !kernel, NULL,
 				 acc_size, sg, resv, &amdgpu_ttm_bo_destroy);
-	amdgpu_cs_report_moved_bytes(adev,
-		atomic64_read(&adev->num_bytes_moved) - initial_bytes_moved);
+	bytes_moved = atomic64_read(&adev->num_bytes_moved) -
+		      initial_bytes_moved;
+	if (adev->mc.visible_vram_size < adev->mc.real_vram_size &&
+	    bo->tbo.mem.mem_type == TTM_PL_VRAM &&
+	    bo->tbo.mem.start < adev->mc.visible_vram_size >> PAGE_SHIFT)
+		amdgpu_cs_report_moved_bytes(adev, bytes_moved, bytes_moved);
+	else
+		amdgpu_cs_report_moved_bytes(adev, bytes_moved, 0);
 
 	if (unlikely(r != 0))
 		return r;
@@ -411,7 +410,7 @@ int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
 	    bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) {
 		struct dma_fence *fence;
 
-		r = amdgpu_fill_buffer(bo, 0, bo->tbo.resv, &fence);
+		r = amdgpu_fill_buffer(bo, init_value, bo->tbo.resv, &fence);
 		if (unlikely(r))
 			goto fail_unreserve;
 
@@ -426,6 +425,10 @@ int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
 
 	trace_amdgpu_bo_create(bo);
 
+	/* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */
+	if (type == ttm_bo_type_device)
+		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
+
 	return 0;
 
 fail_unreserve:
@@ -446,19 +449,19 @@ static int amdgpu_bo_create_shadow(struct amdgpu_device *adev,
 	if (bo->shadow)
 		return 0;
 
-	bo->flags |= AMDGPU_GEM_CREATE_SHADOW;
-	memset(&placements, 0,
-	       (AMDGPU_GEM_DOMAIN_MAX + 1) * sizeof(struct ttm_place));
-
-	amdgpu_ttm_placement_init(adev, &placement,
-				  placements, AMDGPU_GEM_DOMAIN_GTT,
-				  AMDGPU_GEM_CREATE_CPU_GTT_USWC);
+	memset(&placements, 0, sizeof(placements));
+	amdgpu_ttm_placement_init(adev, &placement, placements,
+				  AMDGPU_GEM_DOMAIN_GTT,
+				  AMDGPU_GEM_CREATE_CPU_GTT_USWC |
+				  AMDGPU_GEM_CREATE_SHADOW);
 
 	r = amdgpu_bo_create_restricted(adev, size, byte_align, true,
 					AMDGPU_GEM_DOMAIN_GTT,
-					AMDGPU_GEM_CREATE_CPU_GTT_USWC,
+					AMDGPU_GEM_CREATE_CPU_GTT_USWC |
+					AMDGPU_GEM_CREATE_SHADOW,
 					NULL, &placement,
 					bo->tbo.resv,
+					0,
 					&bo->shadow);
 	if (!r) {
 		bo->shadow->parent = amdgpu_bo_ref(bo);
@@ -470,39 +473,41 @@ static int amdgpu_bo_create_shadow(struct amdgpu_device *adev,
 	return r;
 }
 
+/* init_value will only take effect when flags contains
+ * AMDGPU_GEM_CREATE_VRAM_CLEARED.
+ */
 int amdgpu_bo_create(struct amdgpu_device *adev,
 		     unsigned long size, int byte_align,
 		     bool kernel, u32 domain, u64 flags,
 		     struct sg_table *sg,
 		     struct reservation_object *resv,
+		     uint64_t init_value,
 		     struct amdgpu_bo **bo_ptr)
 {
 	struct ttm_placement placement = {0};
 	struct ttm_place placements[AMDGPU_GEM_DOMAIN_MAX + 1];
+	uint64_t parent_flags = flags & ~AMDGPU_GEM_CREATE_SHADOW;
 	int r;
 
-	memset(&placements, 0,
-	       (AMDGPU_GEM_DOMAIN_MAX + 1) * sizeof(struct ttm_place));
-
-	amdgpu_ttm_placement_init(adev, &placement,
-				  placements, domain, flags);
+	memset(&placements, 0, sizeof(placements));
+	amdgpu_ttm_placement_init(adev, &placement, placements,
+				  domain, parent_flags);
 
-	r = amdgpu_bo_create_restricted(adev, size, byte_align, kernel,
-					domain, flags, sg, &placement,
-					resv, bo_ptr);
+	r = amdgpu_bo_create_restricted(adev, size, byte_align, kernel, domain,
+					parent_flags, sg, &placement, resv,
+					init_value, bo_ptr);
 	if (r)
 		return r;
 
-	if (amdgpu_need_backup(adev) && (flags & AMDGPU_GEM_CREATE_SHADOW)) {
-		if (!resv) {
-			r = ww_mutex_lock(&(*bo_ptr)->tbo.resv->lock, NULL);
-			WARN_ON(r != 0);
-		}
+	if ((flags & AMDGPU_GEM_CREATE_SHADOW) && amdgpu_need_backup(adev)) {
+		if (!resv)
+			WARN_ON(reservation_object_lock((*bo_ptr)->tbo.resv,
+							NULL));
 
 		r = amdgpu_bo_create_shadow(adev, size, byte_align, (*bo_ptr));
 
 		if (!resv)
-			ww_mutex_unlock(&(*bo_ptr)->tbo.resv->lock);
+			reservation_object_unlock((*bo_ptr)->tbo.resv);
 
 		if (r)
 			amdgpu_bo_unref(bo_ptr);
@@ -535,7 +540,7 @@ int amdgpu_bo_backup_to_shadow(struct amdgpu_device *adev,
 
 	r = amdgpu_copy_buffer(ring, bo_addr, shadow_addr,
 			       amdgpu_bo_size(bo), resv, fence,
-			       direct);
+			       direct, false);
 	if (!r)
 		amdgpu_bo_fence(bo, *fence, true);
 
@@ -551,7 +556,7 @@ int amdgpu_bo_validate(struct amdgpu_bo *bo)
 	if (bo->pin_count)
 		return 0;
 
-	domain = bo->prefered_domains;
+	domain = bo->preferred_domains;
 
 retry:
 	amdgpu_ttm_placement_from_domain(bo, domain);
@@ -588,7 +593,7 @@ int amdgpu_bo_restore_from_shadow(struct amdgpu_device *adev,
 
 	r = amdgpu_copy_buffer(ring, shadow_addr, bo_addr,
 			       amdgpu_bo_size(bo), resv, fence,
-			       direct);
+			       direct, false);
 	if (!r)
 		amdgpu_bo_fence(bo, *fence, true);
 
@@ -598,16 +603,16 @@ err:
 
 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)
 {
-	bool is_iomem;
+	void *kptr;
 	long r;
 
 	if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
 		return -EPERM;
 
-	if (bo->kptr) {
-		if (ptr) {
-			*ptr = bo->kptr;
-		}
+	kptr = amdgpu_bo_kptr(bo);
+	if (kptr) {
+		if (ptr)
+			*ptr = kptr;
 		return 0;
 	}
 
@@ -620,19 +625,23 @@ int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)
 	if (r)
 		return r;
 
-	bo->kptr = ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);
 	if (ptr)
-		*ptr = bo->kptr;
+		*ptr = amdgpu_bo_kptr(bo);
 
 	return 0;
 }
 
+void *amdgpu_bo_kptr(struct amdgpu_bo *bo)
+{
+	bool is_iomem;
+
+	return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);
+}
+
 void amdgpu_bo_kunmap(struct amdgpu_bo *bo)
 {
-	if (bo->kptr == NULL)
-		return;
-	bo->kptr = NULL;
-	ttm_bo_kunmap(&bo->kmap);
+	if (bo->kmap.bo)
+		ttm_bo_kunmap(&bo->kmap);
 }
 
 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)
@@ -724,15 +733,16 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
 		dev_err(adev->dev, "%p pin failed\n", bo);
 		goto error;
 	}
-	r = amdgpu_ttm_bind(&bo->tbo, &bo->tbo.mem);
-	if (unlikely(r)) {
-		dev_err(adev->dev, "%p bind failed\n", bo);
-		goto error;
-	}
 
 	bo->pin_count = 1;
-	if (gpu_addr != NULL)
+	if (gpu_addr != NULL) {
+		r = amdgpu_ttm_bind(&bo->tbo, &bo->tbo.mem);
+		if (unlikely(r)) {
+			dev_err(adev->dev, "%p bind failed\n", bo);
+			goto error;
+		}
 		*gpu_addr = amdgpu_bo_gpu_offset(bo);
+	}
 	if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
 		adev->vram_pin_size += amdgpu_bo_size(bo);
 		if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
@@ -921,6 +931,8 @@ void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
 	abo = container_of(bo, struct amdgpu_bo, tbo);
 	amdgpu_vm_bo_invalidate(adev, abo);
 
+	amdgpu_bo_kunmap(abo);
+
 	/* remember the eviction */
 	if (evict)
 		atomic64_inc(&adev->num_evictions);
@@ -930,8 +942,6 @@ void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
 		return;
 
 	/* move_notify is called before move happens */
-	amdgpu_update_memory_usage(adev, &bo->mem, new_mem);
-
 	trace_amdgpu_ttm_bo_move(abo, new_mem->mem_type, old_mem->mem_type);
 }
 
@@ -939,19 +949,22 @@ int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
 	struct amdgpu_bo *abo;
-	unsigned long offset, size, lpfn;
-	int i, r;
+	unsigned long offset, size;
+	int r;
 
 	if (!amdgpu_ttm_bo_is_amdgpu_bo(bo))
 		return 0;
 
 	abo = container_of(bo, struct amdgpu_bo, tbo);
+
+	/* Remember that this BO was accessed by the CPU */
+	abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
+
 	if (bo->mem.mem_type != TTM_PL_VRAM)
 		return 0;
 
 	size = bo->mem.num_pages << PAGE_SHIFT;
 	offset = bo->mem.start << PAGE_SHIFT;
-	/* TODO: figure out how to map scattered VRAM to the CPU */
 	if ((offset + size) <= adev->mc.visible_vram_size)
 		return 0;
 
@@ -961,26 +974,21 @@ int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
 
 	/* hurrah the memory is not visible ! */
 	atomic64_inc(&adev->num_vram_cpu_page_faults);
-	amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM);
-	lpfn =	adev->mc.visible_vram_size >> PAGE_SHIFT;
-	for (i = 0; i < abo->placement.num_placement; i++) {
-		/* Force into visible VRAM */
-		if ((abo->placements[i].flags & TTM_PL_FLAG_VRAM) &&
-		    (!abo->placements[i].lpfn ||
-		     abo->placements[i].lpfn > lpfn))
-			abo->placements[i].lpfn = lpfn;
-	}
+	amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
+					 AMDGPU_GEM_DOMAIN_GTT);
+
+	/* Avoid costly evictions; only set GTT as a busy placement */
+	abo->placement.num_busy_placement = 1;
+	abo->placement.busy_placement = &abo->placements[1];
+
 	r = ttm_bo_validate(bo, &abo->placement, false, false);
-	if (unlikely(r == -ENOMEM)) {
-		amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT);
-		return ttm_bo_validate(bo, &abo->placement, false, false);
-	} else if (unlikely(r != 0)) {
+	if (unlikely(r != 0))
 		return r;
-	}
 
 	offset = bo->mem.start << PAGE_SHIFT;
 	/* this should never happen */
-	if ((offset + size) > adev->mc.visible_vram_size)
+	if (bo->mem.mem_type == TTM_PL_VRAM &&
+	    (offset + size) > adev->mc.visible_vram_size)
 		return -EINVAL;
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index 382485115b06..a288fa6d72c8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -33,6 +33,61 @@
 
 #define AMDGPU_BO_INVALID_OFFSET	LONG_MAX
 
+/* bo virtual addresses in a vm */
+struct amdgpu_bo_va_mapping {
+	struct list_head		list;
+	struct rb_node			rb;
+	uint64_t			start;
+	uint64_t			last;
+	uint64_t			__subtree_last;
+	uint64_t			offset;
+	uint64_t			flags;
+};
+
+/* User space allocated BO in a VM */
+struct amdgpu_bo_va {
+	struct amdgpu_vm_bo_base	base;
+
+	/* protected by bo being reserved */
+	struct dma_fence	        *last_pt_update;
+	unsigned			ref_count;
+
+	/* mappings for this bo_va */
+	struct list_head		invalids;
+	struct list_head		valids;
+};
+
+struct amdgpu_bo {
+	/* Protected by tbo.reserved */
+	u32				preferred_domains;
+	u32				allowed_domains;
+	struct ttm_place		placements[AMDGPU_GEM_DOMAIN_MAX + 1];
+	struct ttm_placement		placement;
+	struct ttm_buffer_object	tbo;
+	struct ttm_bo_kmap_obj		kmap;
+	u64				flags;
+	unsigned			pin_count;
+	u64				tiling_flags;
+	u64				metadata_flags;
+	void				*metadata;
+	u32				metadata_size;
+	unsigned			prime_shared_count;
+	/* list of all virtual address to which this bo is associated to */
+	struct list_head		va;
+	/* Constant after initialization */
+	struct drm_gem_object		gem_base;
+	struct amdgpu_bo		*parent;
+	struct amdgpu_bo		*shadow;
+
+	struct ttm_bo_kmap_obj		dma_buf_vmap;
+	struct amdgpu_mn		*mn;
+
+	union {
+		struct list_head	mn_list;
+		struct list_head	shadow_list;
+	};
+};
+
 /**
  * amdgpu_mem_type_to_domain - return domain corresponding to mem_type
  * @mem_type:	ttm memory type
@@ -120,7 +175,11 @@ static inline u64 amdgpu_bo_mmap_offset(struct amdgpu_bo *bo)
  */
 static inline bool amdgpu_bo_gpu_accessible(struct amdgpu_bo *bo)
 {
-	return bo->tbo.mem.mem_type != TTM_PL_SYSTEM;
+	switch (bo->tbo.mem.mem_type) {
+	case TTM_PL_TT: return amdgpu_ttm_is_bound(bo->tbo.ttm);
+	case TTM_PL_VRAM: return true;
+	default: return false;
+	}
 }
 
 int amdgpu_bo_create(struct amdgpu_device *adev,
@@ -128,6 +187,7 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
 			    bool kernel, u32 domain, u64 flags,
 			    struct sg_table *sg,
 			    struct reservation_object *resv,
+			    uint64_t init_value,
 			    struct amdgpu_bo **bo_ptr);
 int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
 				unsigned long size, int byte_align,
@@ -135,7 +195,12 @@ int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
 				struct sg_table *sg,
 				struct ttm_placement *placement,
 			        struct reservation_object *resv,
+				uint64_t init_value,
 				struct amdgpu_bo **bo_ptr);
+int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
+			      unsigned long size, int align,
+			      u32 domain, struct amdgpu_bo **bo_ptr,
+			      u64 *gpu_addr, void **cpu_addr);
 int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
 			    unsigned long size, int align,
 			    u32 domain, struct amdgpu_bo **bo_ptr,
@@ -143,6 +208,7 @@ int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
 			   void **cpu_addr);
 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr);
+void *amdgpu_bo_kptr(struct amdgpu_bo *bo);
 void amdgpu_bo_kunmap(struct amdgpu_bo *bo);
 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo);
 void amdgpu_bo_unref(struct amdgpu_bo **bo);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.h
index c19c4d138751..f21a7716b90e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.h
@@ -30,6 +30,7 @@ struct cg_flag_name
 	const char *name;
 };
 
+void amdgpu_pm_acpi_event_handler(struct amdgpu_device *adev);
 int amdgpu_pm_sysfs_init(struct amdgpu_device *adev);
 void amdgpu_pm_sysfs_fini(struct amdgpu_device *adev);
 void amdgpu_pm_print_power_states(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c
index 6bdc866570ab..5b3f92891f89 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c
@@ -69,7 +69,7 @@ amdgpu_gem_prime_import_sg_table(struct drm_device *dev,
 
 	ww_mutex_lock(&resv->lock, NULL);
 	ret = amdgpu_bo_create(adev, attach->dmabuf->size, PAGE_SIZE, false,
-			       AMDGPU_GEM_DOMAIN_GTT, 0, sg, resv, &bo);
+			       AMDGPU_GEM_DOMAIN_GTT, 0, sg, resv, 0, &bo);
 	ww_mutex_unlock(&resv->lock);
 	if (ret)
 		return ERR_PTR(ret);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index 4083be61b328..8c2204c7b384 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -63,8 +63,13 @@ static int psp_sw_init(void *handle)
 		psp->smu_reload_quirk = psp_v3_1_smu_reload_quirk;
 		break;
 	case CHIP_RAVEN:
+#if 0
+		psp->init_microcode = psp_v10_0_init_microcode;
+#endif
 		psp->prep_cmd_buf = psp_v10_0_prep_cmd_buf;
 		psp->ring_init = psp_v10_0_ring_init;
+		psp->ring_create = psp_v10_0_ring_create;
+		psp->ring_destroy = psp_v10_0_ring_destroy;
 		psp->cmd_submit = psp_v10_0_cmd_submit;
 		psp->compare_sram_data = psp_v10_0_compare_sram_data;
 		break;
@@ -95,9 +100,8 @@ int psp_wait_for(struct psp_context *psp, uint32_t reg_index,
 	int i;
 	struct amdgpu_device *adev = psp->adev;
 
-	val = RREG32(reg_index);
-
 	for (i = 0; i < adev->usec_timeout; i++) {
+		val = RREG32(reg_index);
 		if (check_changed) {
 			if (val != reg_val)
 				return 0;
@@ -118,33 +122,18 @@ psp_cmd_submit_buf(struct psp_context *psp,
 		   int index)
 {
 	int ret;
-	struct amdgpu_bo *cmd_buf_bo;
-	uint64_t cmd_buf_mc_addr;
-	struct psp_gfx_cmd_resp *cmd_buf_mem;
-	struct amdgpu_device *adev = psp->adev;
-
-	ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE,
-				      AMDGPU_GEM_DOMAIN_VRAM,
-				      &cmd_buf_bo, &cmd_buf_mc_addr,
-				      (void **)&cmd_buf_mem);
-	if (ret)
-		return ret;
 
-	memset(cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
+	memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
 
-	memcpy(cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp));
+	memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp));
 
-	ret = psp_cmd_submit(psp, ucode, cmd_buf_mc_addr,
+	ret = psp_cmd_submit(psp, ucode, psp->cmd_buf_mc_addr,
 			     fence_mc_addr, index);
 
 	while (*((unsigned int *)psp->fence_buf) != index) {
 		msleep(1);
 	}
 
-	amdgpu_bo_free_kernel(&cmd_buf_bo,
-			      &cmd_buf_mc_addr,
-			      (void **)&cmd_buf_mem);
-
 	return ret;
 }
 
@@ -352,13 +341,20 @@ static int psp_load_fw(struct amdgpu_device *adev)
 				      &psp->fence_buf_mc_addr,
 				      &psp->fence_buf);
 	if (ret)
+		goto failed_mem2;
+
+	ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE,
+				      AMDGPU_GEM_DOMAIN_VRAM,
+				      &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
+				      (void **)&psp->cmd_buf_mem);
+	if (ret)
 		goto failed_mem1;
 
 	memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE);
 
 	ret = psp_ring_init(psp, PSP_RING_TYPE__KM);
 	if (ret)
-		goto failed_mem1;
+		goto failed_mem;
 
 	ret = psp_tmr_init(psp);
 	if (ret)
@@ -379,9 +375,13 @@ static int psp_load_fw(struct amdgpu_device *adev)
 	return 0;
 
 failed_mem:
+	amdgpu_bo_free_kernel(&psp->cmd_buf_bo,
+			      &psp->cmd_buf_mc_addr,
+			      (void **)&psp->cmd_buf_mem);
+failed_mem1:
 	amdgpu_bo_free_kernel(&psp->fence_buf_bo,
 			      &psp->fence_buf_mc_addr, &psp->fence_buf);
-failed_mem1:
+failed_mem2:
 	amdgpu_bo_free_kernel(&psp->fw_pri_bo,
 			      &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
 failed:
@@ -435,16 +435,15 @@ static int psp_hw_fini(void *handle)
 
 	psp_ring_destroy(psp, PSP_RING_TYPE__KM);
 
-	if (psp->tmr_buf)
-		amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, &psp->tmr_buf);
-
-	if (psp->fw_pri_buf)
-		amdgpu_bo_free_kernel(&psp->fw_pri_bo,
-				      &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
-
-	if (psp->fence_buf_bo)
-		amdgpu_bo_free_kernel(&psp->fence_buf_bo,
-				      &psp->fence_buf_mc_addr, &psp->fence_buf);
+	amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, &psp->tmr_buf);
+	amdgpu_bo_free_kernel(&psp->fw_pri_bo,
+			      &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
+	amdgpu_bo_free_kernel(&psp->fence_buf_bo,
+			      &psp->fence_buf_mc_addr, &psp->fence_buf);
+	amdgpu_bo_free_kernel(&psp->asd_shared_bo, &psp->asd_shared_mc_addr,
+			      &psp->asd_shared_buf);
+	amdgpu_bo_free_kernel(&psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
+			      (void **)&psp->cmd_buf_mem);
 
 	kfree(psp->cmd);
 	psp->cmd = NULL;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
index 1a1c8b469f93..538fa9dbfb21 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
@@ -108,6 +108,11 @@ struct psp_context
 	struct amdgpu_bo 		*fence_buf_bo;
 	uint64_t 			fence_buf_mc_addr;
 	void				*fence_buf;
+
+	/* cmd buffer */
+	struct amdgpu_bo		*cmd_buf_bo;
+	uint64_t			cmd_buf_mc_addr;
+	struct psp_gfx_cmd_resp		*cmd_buf_mem;
 };
 
 struct amdgpu_psp_funcs {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index 75165e07b1cd..5ce65280b396 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -170,6 +170,16 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
 		     unsigned irq_type)
 {
 	int r;
+	int sched_hw_submission = amdgpu_sched_hw_submission;
+
+	/* Set the hw submission limit higher for KIQ because
+	 * it's used for a number of gfx/compute tasks by both
+	 * KFD and KGD which may have outstanding fences and
+	 * it doesn't really use the gpu scheduler anyway;
+	 * KIQ tasks get submitted directly to the ring.
+	 */
+	if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
+		sched_hw_submission = max(sched_hw_submission, 256);
 
 	if (ring->adev == NULL) {
 		if (adev->num_rings >= AMDGPU_MAX_RINGS)
@@ -178,38 +188,21 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
 		ring->adev = adev;
 		ring->idx = adev->num_rings++;
 		adev->rings[ring->idx] = ring;
-		r = amdgpu_fence_driver_init_ring(ring,
-			amdgpu_sched_hw_submission);
+		r = amdgpu_fence_driver_init_ring(ring, sched_hw_submission);
 		if (r)
 			return r;
 	}
 
-	if (ring->funcs->support_64bit_ptrs) {
-		r = amdgpu_wb_get_64bit(adev, &ring->rptr_offs);
-		if (r) {
-			dev_err(adev->dev, "(%d) ring rptr_offs wb alloc failed\n", r);
-			return r;
-		}
-
-		r = amdgpu_wb_get_64bit(adev, &ring->wptr_offs);
-		if (r) {
-			dev_err(adev->dev, "(%d) ring wptr_offs wb alloc failed\n", r);
-			return r;
-		}
-
-	} else {
-		r = amdgpu_wb_get(adev, &ring->rptr_offs);
-		if (r) {
-			dev_err(adev->dev, "(%d) ring rptr_offs wb alloc failed\n", r);
-			return r;
-		}
-
-		r = amdgpu_wb_get(adev, &ring->wptr_offs);
-		if (r) {
-			dev_err(adev->dev, "(%d) ring wptr_offs wb alloc failed\n", r);
-			return r;
-		}
+	r = amdgpu_wb_get(adev, &ring->rptr_offs);
+	if (r) {
+		dev_err(adev->dev, "(%d) ring rptr_offs wb alloc failed\n", r);
+		return r;
+	}
 
+	r = amdgpu_wb_get(adev, &ring->wptr_offs);
+	if (r) {
+		dev_err(adev->dev, "(%d) ring wptr_offs wb alloc failed\n", r);
+		return r;
 	}
 
 	r = amdgpu_wb_get(adev, &ring->fence_offs);
@@ -234,8 +227,7 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
 		return r;
 	}
 
-	ring->ring_size = roundup_pow_of_two(max_dw * 4 *
-					     amdgpu_sched_hw_submission);
+	ring->ring_size = roundup_pow_of_two(max_dw * 4 * sched_hw_submission);
 
 	ring->buf_mask = (ring->ring_size / 4) - 1;
 	ring->ptr_mask = ring->funcs->support_64bit_ptrs ?
@@ -277,18 +269,15 @@ void amdgpu_ring_fini(struct amdgpu_ring *ring)
 {
 	ring->ready = false;
 
-	if (ring->funcs->support_64bit_ptrs) {
-		amdgpu_wb_free_64bit(ring->adev, ring->cond_exe_offs);
-		amdgpu_wb_free_64bit(ring->adev, ring->fence_offs);
-		amdgpu_wb_free_64bit(ring->adev, ring->rptr_offs);
-		amdgpu_wb_free_64bit(ring->adev, ring->wptr_offs);
-	} else {
-		amdgpu_wb_free(ring->adev, ring->cond_exe_offs);
-		amdgpu_wb_free(ring->adev, ring->fence_offs);
-		amdgpu_wb_free(ring->adev, ring->rptr_offs);
-		amdgpu_wb_free(ring->adev, ring->wptr_offs);
-	}
+	/* Not to finish a ring which is not initialized */
+	if (!(ring->adev) || !(ring->adev->rings[ring->idx]))
+		return;
+
+	amdgpu_wb_free(ring->adev, ring->rptr_offs);
+	amdgpu_wb_free(ring->adev, ring->wptr_offs);
 
+	amdgpu_wb_free(ring->adev, ring->cond_exe_offs);
+	amdgpu_wb_free(ring->adev, ring->fence_offs);
 
 	amdgpu_bo_free_kernel(&ring->ring_obj,
 			      &ring->gpu_addr,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index bc8dec992f73..322d25299a00 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -212,4 +212,44 @@ static inline void amdgpu_ring_clear_ring(struct amdgpu_ring *ring)
 
 }
 
+static inline void amdgpu_ring_write(struct amdgpu_ring *ring, uint32_t v)
+{
+	if (ring->count_dw <= 0)
+		DRM_ERROR("amdgpu: writing more dwords to the ring than expected!\n");
+	ring->ring[ring->wptr++ & ring->buf_mask] = v;
+	ring->wptr &= ring->ptr_mask;
+	ring->count_dw--;
+}
+
+static inline void amdgpu_ring_write_multiple(struct amdgpu_ring *ring,
+					      void *src, int count_dw)
+{
+	unsigned occupied, chunk1, chunk2;
+	void *dst;
+
+	if (unlikely(ring->count_dw < count_dw))
+		DRM_ERROR("amdgpu: writing more dwords to the ring than expected!\n");
+
+	occupied = ring->wptr & ring->buf_mask;
+	dst = (void *)&ring->ring[occupied];
+	chunk1 = ring->buf_mask + 1 - occupied;
+	chunk1 = (chunk1 >= count_dw) ? count_dw: chunk1;
+	chunk2 = count_dw - chunk1;
+	chunk1 <<= 2;
+	chunk2 <<= 2;
+
+	if (chunk1)
+		memcpy(dst, src, chunk1);
+
+	if (chunk2) {
+		src += chunk1;
+		dst = (void *)ring->ring;
+		memcpy(dst, src, chunk2);
+	}
+
+	ring->wptr += count_dw;
+	ring->wptr &= ring->ptr_mask;
+	ring->count_dw -= count_dw;
+}
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
index 5ca75a456ad2..3144400435b7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
@@ -64,7 +64,7 @@ int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev,
 		INIT_LIST_HEAD(&sa_manager->flist[i]);
 
 	r = amdgpu_bo_create(adev, size, align, true, domain,
-			     0, NULL, NULL, &sa_manager->bo);
+			     0, NULL, NULL, 0, &sa_manager->bo);
 	if (r) {
 		dev_err(adev->dev, "(%d) failed to allocate bo for manager\n", r);
 		return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
index a6899180b265..c586f44312f9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
@@ -244,6 +244,12 @@ struct dma_fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync,
 		struct dma_fence *f = e->fence;
 		struct amd_sched_fence *s_fence = to_amd_sched_fence(f);
 
+		if (dma_fence_is_signaled(f)) {
+			hash_del(&e->node);
+			dma_fence_put(f);
+			kmem_cache_free(amdgpu_sync_slab, e);
+			continue;
+		}
 		if (ring && s_fence) {
 			/* For fences from the same ring it is sufficient
 			 * when they are scheduled.
@@ -256,13 +262,6 @@ struct dma_fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync,
 			}
 		}
 
-		if (dma_fence_is_signaled(f)) {
-			hash_del(&e->node);
-			dma_fence_put(f);
-			kmem_cache_free(amdgpu_sync_slab, e);
-			continue;
-		}
-
 		return f;
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c
index 15510dadde01..ed8c3739015b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c
@@ -33,7 +33,7 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev)
 	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
 	struct amdgpu_bo *vram_obj = NULL;
 	struct amdgpu_bo **gtt_obj = NULL;
-	uint64_t gtt_addr, vram_addr;
+	uint64_t gart_addr, vram_addr;
 	unsigned n, size;
 	int i, r;
 
@@ -42,7 +42,7 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev)
 	/* Number of tests =
 	 * (Total GTT - IB pool - writeback page - ring buffers) / test size
 	 */
-	n = adev->mc.gtt_size - AMDGPU_IB_POOL_SIZE*64*1024;
+	n = adev->mc.gart_size - AMDGPU_IB_POOL_SIZE*64*1024;
 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
 		if (adev->rings[i])
 			n -= adev->rings[i]->ring_size;
@@ -61,7 +61,7 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev)
 
 	r = amdgpu_bo_create(adev, size, PAGE_SIZE, true,
 			     AMDGPU_GEM_DOMAIN_VRAM, 0,
-			     NULL, NULL, &vram_obj);
+			     NULL, NULL, 0, &vram_obj);
 	if (r) {
 		DRM_ERROR("Failed to create VRAM object\n");
 		goto out_cleanup;
@@ -76,13 +76,13 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev)
 	}
 	for (i = 0; i < n; i++) {
 		void *gtt_map, *vram_map;
-		void **gtt_start, **gtt_end;
+		void **gart_start, **gart_end;
 		void **vram_start, **vram_end;
 		struct dma_fence *fence = NULL;
 
 		r = amdgpu_bo_create(adev, size, PAGE_SIZE, true,
 				     AMDGPU_GEM_DOMAIN_GTT, 0, NULL,
-				     NULL, gtt_obj + i);
+				     NULL, 0, gtt_obj + i);
 		if (r) {
 			DRM_ERROR("Failed to create GTT object %d\n", i);
 			goto out_lclean;
@@ -91,7 +91,7 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev)
 		r = amdgpu_bo_reserve(gtt_obj[i], false);
 		if (unlikely(r != 0))
 			goto out_lclean_unref;
-		r = amdgpu_bo_pin(gtt_obj[i], AMDGPU_GEM_DOMAIN_GTT, &gtt_addr);
+		r = amdgpu_bo_pin(gtt_obj[i], AMDGPU_GEM_DOMAIN_GTT, &gart_addr);
 		if (r) {
 			DRM_ERROR("Failed to pin GTT object %d\n", i);
 			goto out_lclean_unres;
@@ -103,15 +103,15 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev)
 			goto out_lclean_unpin;
 		}
 
-		for (gtt_start = gtt_map, gtt_end = gtt_map + size;
-		     gtt_start < gtt_end;
-		     gtt_start++)
-			*gtt_start = gtt_start;
+		for (gart_start = gtt_map, gart_end = gtt_map + size;
+		     gart_start < gart_end;
+		     gart_start++)
+			*gart_start = gart_start;
 
 		amdgpu_bo_kunmap(gtt_obj[i]);
 
-		r = amdgpu_copy_buffer(ring, gtt_addr, vram_addr,
-				       size, NULL, &fence, false);
+		r = amdgpu_copy_buffer(ring, gart_addr, vram_addr,
+				       size, NULL, &fence, false, false);
 
 		if (r) {
 			DRM_ERROR("Failed GTT->VRAM copy %d\n", i);
@@ -132,21 +132,21 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev)
 			goto out_lclean_unpin;
 		}
 
-		for (gtt_start = gtt_map, gtt_end = gtt_map + size,
+		for (gart_start = gtt_map, gart_end = gtt_map + size,
 		     vram_start = vram_map, vram_end = vram_map + size;
 		     vram_start < vram_end;
-		     gtt_start++, vram_start++) {
-			if (*vram_start != gtt_start) {
+		     gart_start++, vram_start++) {
+			if (*vram_start != gart_start) {
 				DRM_ERROR("Incorrect GTT->VRAM copy %d: Got 0x%p, "
 					  "expected 0x%p (GTT/VRAM offset "
 					  "0x%16llx/0x%16llx)\n",
-					  i, *vram_start, gtt_start,
+					  i, *vram_start, gart_start,
 					  (unsigned long long)
-					  (gtt_addr - adev->mc.gtt_start +
-					   (void*)gtt_start - gtt_map),
+					  (gart_addr - adev->mc.gart_start +
+					   (void*)gart_start - gtt_map),
 					  (unsigned long long)
 					  (vram_addr - adev->mc.vram_start +
-					   (void*)gtt_start - gtt_map));
+					   (void*)gart_start - gtt_map));
 				amdgpu_bo_kunmap(vram_obj);
 				goto out_lclean_unpin;
 			}
@@ -155,8 +155,8 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev)
 
 		amdgpu_bo_kunmap(vram_obj);
 
-		r = amdgpu_copy_buffer(ring, vram_addr, gtt_addr,
-				       size, NULL, &fence, false);
+		r = amdgpu_copy_buffer(ring, vram_addr, gart_addr,
+				       size, NULL, &fence, false, false);
 
 		if (r) {
 			DRM_ERROR("Failed VRAM->GTT copy %d\n", i);
@@ -177,20 +177,20 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev)
 			goto out_lclean_unpin;
 		}
 
-		for (gtt_start = gtt_map, gtt_end = gtt_map + size,
+		for (gart_start = gtt_map, gart_end = gtt_map + size,
 		     vram_start = vram_map, vram_end = vram_map + size;
-		     gtt_start < gtt_end;
-		     gtt_start++, vram_start++) {
-			if (*gtt_start != vram_start) {
+		     gart_start < gart_end;
+		     gart_start++, vram_start++) {
+			if (*gart_start != vram_start) {
 				DRM_ERROR("Incorrect VRAM->GTT copy %d: Got 0x%p, "
 					  "expected 0x%p (VRAM/GTT offset "
 					  "0x%16llx/0x%16llx)\n",
-					  i, *gtt_start, vram_start,
+					  i, *gart_start, vram_start,
 					  (unsigned long long)
 					  (vram_addr - adev->mc.vram_start +
 					   (void*)vram_start - vram_map),
 					  (unsigned long long)
-					  (gtt_addr - adev->mc.gtt_start +
+					  (gart_addr - adev->mc.gart_start +
 					   (void*)vram_start - vram_map));
 				amdgpu_bo_kunmap(gtt_obj[i]);
 				goto out_lclean_unpin;
@@ -200,7 +200,7 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev)
 		amdgpu_bo_kunmap(gtt_obj[i]);
 
 		DRM_INFO("Tested GTT->VRAM and VRAM->GTT copy for GTT offset 0x%llx\n",
-			 gtt_addr - adev->mc.gtt_start);
+			 gart_addr - adev->mc.gart_start);
 		continue;
 
 out_lclean_unpin:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
index 8601904e670a..1c88bd5e29ad 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
@@ -14,6 +14,62 @@
 #define AMDGPU_JOB_GET_TIMELINE_NAME(job) \
 	 job->base.s_fence->finished.ops->get_timeline_name(&job->base.s_fence->finished)
 
+TRACE_EVENT(amdgpu_ttm_tt_populate,
+	    TP_PROTO(struct amdgpu_device *adev, uint64_t dma_address, uint64_t phys_address),
+	    TP_ARGS(adev, dma_address, phys_address),
+	    TP_STRUCT__entry(
+				__field(uint16_t, domain)
+				__field(uint8_t, bus)
+				__field(uint8_t, slot)
+				__field(uint8_t, func)
+				__field(uint64_t, dma)
+				__field(uint64_t, phys)
+			    ),
+	    TP_fast_assign(
+			   __entry->domain = pci_domain_nr(adev->pdev->bus);
+			   __entry->bus = adev->pdev->bus->number;
+			   __entry->slot = PCI_SLOT(adev->pdev->devfn);
+			   __entry->func = PCI_FUNC(adev->pdev->devfn);
+			   __entry->dma = dma_address;
+			   __entry->phys = phys_address;
+			   ),
+	    TP_printk("%04x:%02x:%02x.%x: 0x%llx => 0x%llx",
+		      (unsigned)__entry->domain,
+		      (unsigned)__entry->bus,
+		      (unsigned)__entry->slot,
+		      (unsigned)__entry->func,
+		      (unsigned long long)__entry->dma,
+		      (unsigned long long)__entry->phys)
+);
+
+TRACE_EVENT(amdgpu_ttm_tt_unpopulate,
+	    TP_PROTO(struct amdgpu_device *adev, uint64_t dma_address, uint64_t phys_address),
+	    TP_ARGS(adev, dma_address, phys_address),
+	    TP_STRUCT__entry(
+				__field(uint16_t, domain)
+				__field(uint8_t, bus)
+				__field(uint8_t, slot)
+				__field(uint8_t, func)
+				__field(uint64_t, dma)
+				__field(uint64_t, phys)
+			    ),
+	    TP_fast_assign(
+			   __entry->domain = pci_domain_nr(adev->pdev->bus);
+			   __entry->bus = adev->pdev->bus->number;
+			   __entry->slot = PCI_SLOT(adev->pdev->devfn);
+			   __entry->func = PCI_FUNC(adev->pdev->devfn);
+			   __entry->dma = dma_address;
+			   __entry->phys = phys_address;
+			   ),
+	    TP_printk("%04x:%02x:%02x.%x: 0x%llx => 0x%llx",
+		      (unsigned)__entry->domain,
+		      (unsigned)__entry->bus,
+		      (unsigned)__entry->slot,
+		      (unsigned)__entry->func,
+		      (unsigned long long)__entry->dma,
+		      (unsigned long long)__entry->phys)
+);
+
 TRACE_EVENT(amdgpu_mm_rreg,
 	    TP_PROTO(unsigned did, uint32_t reg, uint32_t value),
 	    TP_ARGS(did, reg, value),
@@ -105,12 +161,12 @@ TRACE_EVENT(amdgpu_bo_create,
 			   __entry->bo = bo;
 			   __entry->pages = bo->tbo.num_pages;
 			   __entry->type = bo->tbo.mem.mem_type;
-			   __entry->prefer = bo->prefered_domains;
+			   __entry->prefer = bo->preferred_domains;
 			   __entry->allow = bo->allowed_domains;
 			   __entry->visible = bo->flags;
 			   ),
 
-	    TP_printk("bo=%p, pages=%u, type=%d, prefered=%d, allowed=%d, visible=%d",
+	    TP_printk("bo=%p, pages=%u, type=%d, preferred=%d, allowed=%d, visible=%d",
 		       __entry->bo, __entry->pages, __entry->type,
 		       __entry->prefer, __entry->allow, __entry->visible)
 );
@@ -224,17 +280,17 @@ TRACE_EVENT(amdgpu_vm_bo_map,
 			     __field(long, start)
 			     __field(long, last)
 			     __field(u64, offset)
-			     __field(u32, flags)
+			     __field(u64, flags)
 			     ),
 
 	    TP_fast_assign(
-			   __entry->bo = bo_va ? bo_va->bo : NULL;
+			   __entry->bo = bo_va ? bo_va->base.bo : NULL;
 			   __entry->start = mapping->start;
 			   __entry->last = mapping->last;
 			   __entry->offset = mapping->offset;
 			   __entry->flags = mapping->flags;
 			   ),
-	    TP_printk("bo=%p, start=%lx, last=%lx, offset=%010llx, flags=%08x",
+	    TP_printk("bo=%p, start=%lx, last=%lx, offset=%010llx, flags=%llx",
 		      __entry->bo, __entry->start, __entry->last,
 		      __entry->offset, __entry->flags)
 );
@@ -248,17 +304,17 @@ TRACE_EVENT(amdgpu_vm_bo_unmap,
 			     __field(long, start)
 			     __field(long, last)
 			     __field(u64, offset)
-			     __field(u32, flags)
+			     __field(u64, flags)
 			     ),
 
 	    TP_fast_assign(
-			   __entry->bo = bo_va->bo;
+			   __entry->bo = bo_va->base.bo;
 			   __entry->start = mapping->start;
 			   __entry->last = mapping->last;
 			   __entry->offset = mapping->offset;
 			   __entry->flags = mapping->flags;
 			   ),
-	    TP_printk("bo=%p, start=%lx, last=%lx, offset=%010llx, flags=%08x",
+	    TP_printk("bo=%p, start=%lx, last=%lx, offset=%010llx, flags=%llx",
 		      __entry->bo, __entry->start, __entry->last,
 		      __entry->offset, __entry->flags)
 );
@@ -269,7 +325,7 @@ DECLARE_EVENT_CLASS(amdgpu_vm_mapping,
 	    TP_STRUCT__entry(
 			     __field(u64, soffset)
 			     __field(u64, eoffset)
-			     __field(u32, flags)
+			     __field(u64, flags)
 			     ),
 
 	    TP_fast_assign(
@@ -277,7 +333,7 @@ DECLARE_EVENT_CLASS(amdgpu_vm_mapping,
 			   __entry->eoffset = mapping->last + 1;
 			   __entry->flags = mapping->flags;
 			   ),
-	    TP_printk("soffs=%010llx, eoffs=%010llx, flags=%08x",
+	    TP_printk("soffs=%010llx, eoffs=%010llx, flags=%llx",
 		      __entry->soffset, __entry->eoffset, __entry->flags)
 );
 
@@ -293,14 +349,14 @@ DEFINE_EVENT(amdgpu_vm_mapping, amdgpu_vm_bo_mapping,
 
 TRACE_EVENT(amdgpu_vm_set_ptes,
 	    TP_PROTO(uint64_t pe, uint64_t addr, unsigned count,
-		     uint32_t incr, uint32_t flags),
+		     uint32_t incr, uint64_t flags),
 	    TP_ARGS(pe, addr, count, incr, flags),
 	    TP_STRUCT__entry(
 			     __field(u64, pe)
 			     __field(u64, addr)
 			     __field(u32, count)
 			     __field(u32, incr)
-			     __field(u32, flags)
+			     __field(u64, flags)
 			     ),
 
 	    TP_fast_assign(
@@ -310,7 +366,7 @@ TRACE_EVENT(amdgpu_vm_set_ptes,
 			   __entry->incr = incr;
 			   __entry->flags = flags;
 			   ),
-	    TP_printk("pe=%010Lx, addr=%010Lx, incr=%u, flags=%08x, count=%u",
+	    TP_printk("pe=%010Lx, addr=%010Lx, incr=%u, flags=%llx, count=%u",
 		      __entry->pe, __entry->addr, __entry->incr,
 		      __entry->flags, __entry->count)
 );
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index c9b131b13ef7..7ef6c28a34d9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -43,14 +43,20 @@
 #include <linux/pagemap.h>
 #include <linux/debugfs.h>
 #include "amdgpu.h"
+#include "amdgpu_trace.h"
 #include "bif/bif_4_1_d.h"
 
 #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
 
+static int amdgpu_map_buffer(struct ttm_buffer_object *bo,
+			     struct ttm_mem_reg *mem, unsigned num_pages,
+			     uint64_t offset, unsigned window,
+			     struct amdgpu_ring *ring,
+			     uint64_t *addr);
+
 static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev);
 static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev);
 
-
 /*
  * Global memory.
  */
@@ -97,6 +103,8 @@ static int amdgpu_ttm_global_init(struct amdgpu_device *adev)
 		goto error_bo;
 	}
 
+	mutex_init(&adev->mman.gtt_window_lock);
+
 	ring = adev->mman.buffer_funcs_ring;
 	rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_KERNEL];
 	r = amd_sched_entity_init(&ring->sched, &adev->mman.entity,
@@ -123,6 +131,7 @@ static void amdgpu_ttm_global_fini(struct amdgpu_device *adev)
 	if (adev->mman.mem_global_referenced) {
 		amd_sched_entity_fini(adev->mman.entity.sched,
 				      &adev->mman.entity);
+		mutex_destroy(&adev->mman.gtt_window_lock);
 		drm_global_item_unref(&adev->mman.bo_global_ref.ref);
 		drm_global_item_unref(&adev->mman.mem_global_ref);
 		adev->mman.mem_global_referenced = false;
@@ -150,7 +159,7 @@ static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
 		break;
 	case TTM_PL_TT:
 		man->func = &amdgpu_gtt_mgr_func;
-		man->gpu_offset = adev->mc.gtt_start;
+		man->gpu_offset = adev->mc.gart_start;
 		man->available_caching = TTM_PL_MASK_CACHING;
 		man->default_caching = TTM_PL_FLAG_CACHED;
 		man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA;
@@ -186,12 +195,11 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
 	struct amdgpu_bo *abo;
-	static struct ttm_place placements = {
+	static const struct ttm_place placements = {
 		.fpfn = 0,
 		.lpfn = 0,
 		.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM
 	};
-	unsigned i;
 
 	if (!amdgpu_ttm_bo_is_amdgpu_bo(bo)) {
 		placement->placement = &placements;
@@ -207,22 +215,36 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 		    adev->mman.buffer_funcs_ring &&
 		    adev->mman.buffer_funcs_ring->ready == false) {
 			amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
+		} else if (adev->mc.visible_vram_size < adev->mc.real_vram_size &&
+			   !(abo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)) {
+			unsigned fpfn = adev->mc.visible_vram_size >> PAGE_SHIFT;
+			struct drm_mm_node *node = bo->mem.mm_node;
+			unsigned long pages_left;
+
+			for (pages_left = bo->mem.num_pages;
+			     pages_left;
+			     pages_left -= node->size, node++) {
+				if (node->start < fpfn)
+					break;
+			}
+
+			if (!pages_left)
+				goto gtt;
+
+			/* Try evicting to the CPU inaccessible part of VRAM
+			 * first, but only set GTT as busy placement, so this
+			 * BO will be evicted to GTT rather than causing other
+			 * BOs to be evicted from VRAM
+			 */
+			amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
+							 AMDGPU_GEM_DOMAIN_GTT);
+			abo->placements[0].fpfn = fpfn;
+			abo->placements[0].lpfn = 0;
+			abo->placement.busy_placement = &abo->placements[1];
+			abo->placement.num_busy_placement = 1;
 		} else {
+gtt:
 			amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT);
-			for (i = 0; i < abo->placement.num_placement; ++i) {
-				if (!(abo->placements[i].flags &
-				      TTM_PL_FLAG_TT))
-					continue;
-
-				if (abo->placements[i].lpfn)
-					continue;
-
-				/* set an upper limit to force directly
-				 * allocating address space for the BO.
-				 */
-				abo->placements[i].lpfn =
-					adev->mc.gtt_size >> PAGE_SHIFT;
-			}
 		}
 		break;
 	case TTM_PL_TT:
@@ -252,29 +274,18 @@ static void amdgpu_move_null(struct ttm_buffer_object *bo,
 	new_mem->mm_node = NULL;
 }
 
-static int amdgpu_mm_node_addr(struct ttm_buffer_object *bo,
-			       struct drm_mm_node *mm_node,
-			       struct ttm_mem_reg *mem,
-			       uint64_t *addr)
+static uint64_t amdgpu_mm_node_addr(struct ttm_buffer_object *bo,
+				    struct drm_mm_node *mm_node,
+				    struct ttm_mem_reg *mem)
 {
-	int r;
-
-	switch (mem->mem_type) {
-	case TTM_PL_TT:
-		r = amdgpu_ttm_bind(bo, mem);
-		if (r)
-			return r;
+	uint64_t addr = 0;
 
-	case TTM_PL_VRAM:
-		*addr = mm_node->start << PAGE_SHIFT;
-		*addr += bo->bdev->man[mem->mem_type].gpu_offset;
-		break;
-	default:
-		DRM_ERROR("Unknown placement %d\n", mem->mem_type);
-		return -EINVAL;
+	if (mem->mem_type != TTM_PL_TT ||
+	    amdgpu_gtt_mgr_is_allocated(mem)) {
+		addr = mm_node->start << PAGE_SHIFT;
+		addr += bo->bdev->man[mem->mem_type].gpu_offset;
 	}
-
-	return 0;
+	return addr;
 }
 
 static int amdgpu_move_blit(struct ttm_buffer_object *bo,
@@ -299,26 +310,40 @@ static int amdgpu_move_blit(struct ttm_buffer_object *bo,
 	}
 
 	old_mm = old_mem->mm_node;
-	r = amdgpu_mm_node_addr(bo, old_mm, old_mem, &old_start);
-	if (r)
-		return r;
 	old_size = old_mm->size;
-
+	old_start = amdgpu_mm_node_addr(bo, old_mm, old_mem);
 
 	new_mm = new_mem->mm_node;
-	r = amdgpu_mm_node_addr(bo, new_mm, new_mem, &new_start);
-	if (r)
-		return r;
 	new_size = new_mm->size;
+	new_start = amdgpu_mm_node_addr(bo, new_mm, new_mem);
 
 	num_pages = new_mem->num_pages;
+	mutex_lock(&adev->mman.gtt_window_lock);
 	while (num_pages) {
-		unsigned long cur_pages = min(old_size, new_size);
+		unsigned long cur_pages = min(min(old_size, new_size),
+					      (u64)AMDGPU_GTT_MAX_TRANSFER_SIZE);
+		uint64_t from = old_start, to = new_start;
 		struct dma_fence *next;
 
-		r = amdgpu_copy_buffer(ring, old_start, new_start,
+		if (old_mem->mem_type == TTM_PL_TT &&
+		    !amdgpu_gtt_mgr_is_allocated(old_mem)) {
+			r = amdgpu_map_buffer(bo, old_mem, cur_pages,
+					      old_start, 0, ring, &from);
+			if (r)
+				goto error;
+		}
+
+		if (new_mem->mem_type == TTM_PL_TT &&
+		    !amdgpu_gtt_mgr_is_allocated(new_mem)) {
+			r = amdgpu_map_buffer(bo, new_mem, cur_pages,
+					      new_start, 1, ring, &to);
+			if (r)
+				goto error;
+		}
+
+		r = amdgpu_copy_buffer(ring, from, to,
 				       cur_pages * PAGE_SIZE,
-				       bo->resv, &next, false);
+				       bo->resv, &next, false, true);
 		if (r)
 			goto error;
 
@@ -331,10 +356,7 @@ static int amdgpu_move_blit(struct ttm_buffer_object *bo,
 
 		old_size -= cur_pages;
 		if (!old_size) {
-			r = amdgpu_mm_node_addr(bo, ++old_mm, old_mem,
-						&old_start);
-			if (r)
-				goto error;
+			old_start = amdgpu_mm_node_addr(bo, ++old_mm, old_mem);
 			old_size = old_mm->size;
 		} else {
 			old_start += cur_pages * PAGE_SIZE;
@@ -342,22 +364,21 @@ static int amdgpu_move_blit(struct ttm_buffer_object *bo,
 
 		new_size -= cur_pages;
 		if (!new_size) {
-			r = amdgpu_mm_node_addr(bo, ++new_mm, new_mem,
-						&new_start);
-			if (r)
-				goto error;
-
+			new_start = amdgpu_mm_node_addr(bo, ++new_mm, new_mem);
 			new_size = new_mm->size;
 		} else {
 			new_start += cur_pages * PAGE_SIZE;
 		}
 	}
+	mutex_unlock(&adev->mman.gtt_window_lock);
 
 	r = ttm_bo_pipeline_move(bo, fence, evict, new_mem);
 	dma_fence_put(fence);
 	return r;
 
 error:
+	mutex_unlock(&adev->mman.gtt_window_lock);
+
 	if (fence)
 		dma_fence_wait(fence, false);
 	dma_fence_put(fence);
@@ -384,7 +405,7 @@ static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo,
 	placement.num_busy_placement = 1;
 	placement.busy_placement = &placements;
 	placements.fpfn = 0;
-	placements.lpfn = adev->mc.gtt_size >> PAGE_SHIFT;
+	placements.lpfn = 0;
 	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
 	r = ttm_bo_mem_space(bo, &placement, &tmp_mem,
 			     interruptible, no_wait_gpu);
@@ -431,7 +452,7 @@ static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo,
 	placement.num_busy_placement = 1;
 	placement.busy_placement = &placements;
 	placements.fpfn = 0;
-	placements.lpfn = adev->mc.gtt_size >> PAGE_SHIFT;
+	placements.lpfn = 0;
 	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
 	r = ttm_bo_mem_space(bo, &placement, &tmp_mem,
 			     interruptible, no_wait_gpu);
@@ -507,6 +528,15 @@ memcpy:
 		}
 	}
 
+	if (bo->type == ttm_bo_type_device &&
+	    new_mem->mem_type == TTM_PL_VRAM &&
+	    old_mem->mem_type != TTM_PL_VRAM) {
+		/* amdgpu_bo_fault_reserve_notify will re-set this if the CPU
+		 * accesses the BO after it's moved.
+		 */
+		abo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
+	}
+
 	/* update statistics */
 	atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &adev->num_bytes_moved);
 	return 0;
@@ -633,6 +663,38 @@ release_pages:
 	return r;
 }
 
+static void amdgpu_trace_dma_map(struct ttm_tt *ttm)
+{
+	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
+	struct amdgpu_ttm_tt *gtt = (void *)ttm;
+	unsigned i;
+
+	if (unlikely(trace_amdgpu_ttm_tt_populate_enabled())) {
+		for (i = 0; i < ttm->num_pages; i++) {
+			trace_amdgpu_ttm_tt_populate(
+				adev,
+				gtt->ttm.dma_address[i],
+				page_to_phys(ttm->pages[i]));
+		}
+	}
+}
+
+static void amdgpu_trace_dma_unmap(struct ttm_tt *ttm)
+{
+	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
+	struct amdgpu_ttm_tt *gtt = (void *)ttm;
+	unsigned i;
+
+	if (unlikely(trace_amdgpu_ttm_tt_unpopulate_enabled())) {
+		for (i = 0; i < ttm->num_pages; i++) {
+			trace_amdgpu_ttm_tt_unpopulate(
+				adev,
+				gtt->ttm.dma_address[i],
+				page_to_phys(ttm->pages[i]));
+		}
+	}
+}
+
 /* prepare the sg table with the user pages */
 static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)
 {
@@ -659,6 +721,8 @@ static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)
 	drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
 					 gtt->ttm.dma_address, ttm->num_pages);
 
+	amdgpu_trace_dma_map(ttm);
+
 	return 0;
 
 release_sg:
@@ -692,6 +756,8 @@ static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm)
 		put_page(page);
 	}
 
+	amdgpu_trace_dma_unmap(ttm);
+
 	sg_free_table(ttm->sg);
 }
 
@@ -699,7 +765,8 @@ static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,
 				   struct ttm_mem_reg *bo_mem)
 {
 	struct amdgpu_ttm_tt *gtt = (void*)ttm;
-	int r;
+	uint64_t flags;
+	int r = 0;
 
 	if (gtt->userptr) {
 		r = amdgpu_ttm_tt_pin_userptr(ttm);
@@ -718,7 +785,25 @@ static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,
 	    bo_mem->mem_type == AMDGPU_PL_OA)
 		return -EINVAL;
 
-	return 0;
+	if (!amdgpu_gtt_mgr_is_allocated(bo_mem))
+		return 0;
+
+	spin_lock(&gtt->adev->gtt_list_lock);
+	flags = amdgpu_ttm_tt_pte_flags(gtt->adev, ttm, bo_mem);
+	gtt->offset = (u64)bo_mem->start << PAGE_SHIFT;
+	r = amdgpu_gart_bind(gtt->adev, gtt->offset, ttm->num_pages,
+		ttm->pages, gtt->ttm.dma_address, flags);
+
+	if (r) {
+		DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
+			  ttm->num_pages, gtt->offset);
+		goto error_gart_bind;
+	}
+
+	list_add_tail(&gtt->list, &gtt->adev->gtt_list);
+error_gart_bind:
+	spin_unlock(&gtt->adev->gtt_list_lock);
+	return r;
 }
 
 bool amdgpu_ttm_is_bound(struct ttm_tt *ttm)
@@ -730,36 +815,38 @@ bool amdgpu_ttm_is_bound(struct ttm_tt *ttm)
 
 int amdgpu_ttm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *bo_mem)
 {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
 	struct ttm_tt *ttm = bo->ttm;
-	struct amdgpu_ttm_tt *gtt = (void *)bo->ttm;
-	uint64_t flags;
+	struct ttm_mem_reg tmp;
+
+	struct ttm_placement placement;
+	struct ttm_place placements;
 	int r;
 
 	if (!ttm || amdgpu_ttm_is_bound(ttm))
 		return 0;
 
-	r = amdgpu_gtt_mgr_alloc(&bo->bdev->man[TTM_PL_TT], bo,
-				 NULL, bo_mem);
-	if (r) {
-		DRM_ERROR("Failed to allocate GTT address space (%d)\n", r);
-		return r;
-	}
+	tmp = bo->mem;
+	tmp.mm_node = NULL;
+	placement.num_placement = 1;
+	placement.placement = &placements;
+	placement.num_busy_placement = 1;
+	placement.busy_placement = &placements;
+	placements.fpfn = 0;
+	placements.lpfn = adev->mc.gart_size >> PAGE_SHIFT;
+	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
 
-	spin_lock(&gtt->adev->gtt_list_lock);
-	flags = amdgpu_ttm_tt_pte_flags(gtt->adev, ttm, bo_mem);
-	gtt->offset = (u64)bo_mem->start << PAGE_SHIFT;
-	r = amdgpu_gart_bind(gtt->adev, gtt->offset, ttm->num_pages,
-		ttm->pages, gtt->ttm.dma_address, flags);
+	r = ttm_bo_mem_space(bo, &placement, &tmp, true, false);
+	if (unlikely(r))
+		return r;
 
-	if (r) {
-		DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
-			  ttm->num_pages, gtt->offset);
-		goto error_gart_bind;
-	}
+	r = ttm_bo_move_ttm(bo, true, false, &tmp);
+	if (unlikely(r))
+		ttm_bo_mem_put(bo, &tmp);
+	else
+		bo->offset = (bo->mem.start << PAGE_SHIFT) +
+			bo->bdev->man[bo->mem.mem_type].gpu_offset;
 
-	list_add_tail(&gtt->list, &gtt->adev->gtt_list);
-error_gart_bind:
-	spin_unlock(&gtt->adev->gtt_list_lock);
 	return r;
 }
 
@@ -852,7 +939,7 @@ static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_bo_device *bdev,
 
 static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm)
 {
-	struct amdgpu_device *adev;
+	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
 	unsigned i;
 	int r;
@@ -875,14 +962,14 @@ static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm)
 		drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
 						 gtt->ttm.dma_address, ttm->num_pages);
 		ttm->state = tt_unbound;
-		return 0;
+		r = 0;
+		goto trace_mappings;
 	}
 
-	adev = amdgpu_ttm_adev(ttm->bdev);
-
 #ifdef CONFIG_SWIOTLB
 	if (swiotlb_nr_tbl()) {
-		return ttm_dma_populate(&gtt->ttm, adev->dev);
+		r = ttm_dma_populate(&gtt->ttm, adev->dev);
+		goto trace_mappings;
 	}
 #endif
 
@@ -905,7 +992,12 @@ static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm)
 			return -EFAULT;
 		}
 	}
-	return 0;
+
+	r = 0;
+trace_mappings:
+	if (likely(!r))
+		amdgpu_trace_dma_map(ttm);
+	return r;
 }
 
 static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
@@ -926,6 +1018,8 @@ static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
 
 	adev = amdgpu_ttm_adev(ttm->bdev);
 
+	amdgpu_trace_dma_unmap(ttm);
+
 #ifdef CONFIG_SWIOTLB
 	if (swiotlb_nr_tbl()) {
 		ttm_dma_unpopulate(&gtt->ttm, adev->dev);
@@ -1075,6 +1169,67 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 	return ttm_bo_eviction_valuable(bo, place);
 }
 
+static int amdgpu_ttm_access_memory(struct ttm_buffer_object *bo,
+				    unsigned long offset,
+				    void *buf, int len, int write)
+{
+	struct amdgpu_bo *abo = container_of(bo, struct amdgpu_bo, tbo);
+	struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
+	struct drm_mm_node *nodes = abo->tbo.mem.mm_node;
+	uint32_t value = 0;
+	int ret = 0;
+	uint64_t pos;
+	unsigned long flags;
+
+	if (bo->mem.mem_type != TTM_PL_VRAM)
+		return -EIO;
+
+	while (offset >= (nodes->size << PAGE_SHIFT)) {
+		offset -= nodes->size << PAGE_SHIFT;
+		++nodes;
+	}
+	pos = (nodes->start << PAGE_SHIFT) + offset;
+
+	while (len && pos < adev->mc.mc_vram_size) {
+		uint64_t aligned_pos = pos & ~(uint64_t)3;
+		uint32_t bytes = 4 - (pos & 3);
+		uint32_t shift = (pos & 3) * 8;
+		uint32_t mask = 0xffffffff << shift;
+
+		if (len < bytes) {
+			mask &= 0xffffffff >> (bytes - len) * 8;
+			bytes = len;
+		}
+
+		spin_lock_irqsave(&adev->mmio_idx_lock, flags);
+		WREG32(mmMM_INDEX, ((uint32_t)aligned_pos) | 0x80000000);
+		WREG32(mmMM_INDEX_HI, aligned_pos >> 31);
+		if (!write || mask != 0xffffffff)
+			value = RREG32(mmMM_DATA);
+		if (write) {
+			value &= ~mask;
+			value |= (*(uint32_t *)buf << shift) & mask;
+			WREG32(mmMM_DATA, value);
+		}
+		spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
+		if (!write) {
+			value = (value & mask) >> shift;
+			memcpy(buf, &value, bytes);
+		}
+
+		ret += bytes;
+		buf = (uint8_t *)buf + bytes;
+		pos += bytes;
+		len -= bytes;
+		if (pos >= (nodes->start + nodes->size) << PAGE_SHIFT) {
+			++nodes;
+			pos = (nodes->start << PAGE_SHIFT);
+		}
+	}
+
+	return ret;
+}
+
 static struct ttm_bo_driver amdgpu_bo_driver = {
 	.ttm_tt_create = &amdgpu_ttm_tt_create,
 	.ttm_tt_populate = &amdgpu_ttm_tt_populate,
@@ -1090,11 +1245,14 @@ static struct ttm_bo_driver amdgpu_bo_driver = {
 	.io_mem_reserve = &amdgpu_ttm_io_mem_reserve,
 	.io_mem_free = &amdgpu_ttm_io_mem_free,
 	.io_mem_pfn = amdgpu_ttm_io_mem_pfn,
+	.access_memory = &amdgpu_ttm_access_memory
 };
 
 int amdgpu_ttm_init(struct amdgpu_device *adev)
 {
+	uint64_t gtt_size;
 	int r;
+	u64 vis_vram_limit;
 
 	r = amdgpu_ttm_global_init(adev);
 	if (r) {
@@ -1118,36 +1276,37 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
 		DRM_ERROR("Failed initializing VRAM heap.\n");
 		return r;
 	}
+
+	/* Reduce size of CPU-visible VRAM if requested */
+	vis_vram_limit = (u64)amdgpu_vis_vram_limit * 1024 * 1024;
+	if (amdgpu_vis_vram_limit > 0 &&
+	    vis_vram_limit <= adev->mc.visible_vram_size)
+		adev->mc.visible_vram_size = vis_vram_limit;
+
 	/* Change the size here instead of the init above so only lpfn is affected */
 	amdgpu_ttm_set_active_vram_size(adev, adev->mc.visible_vram_size);
 
-	r = amdgpu_bo_create(adev, adev->mc.stolen_size, PAGE_SIZE, true,
-			     AMDGPU_GEM_DOMAIN_VRAM,
-			     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
-			     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
-			     NULL, NULL, &adev->stollen_vga_memory);
-	if (r) {
-		return r;
-	}
-	r = amdgpu_bo_reserve(adev->stollen_vga_memory, false);
+	r = amdgpu_bo_create_kernel(adev, adev->mc.stolen_size, PAGE_SIZE,
+				    AMDGPU_GEM_DOMAIN_VRAM,
+				    &adev->stolen_vga_memory,
+				    NULL, NULL);
 	if (r)
 		return r;
-	r = amdgpu_bo_pin(adev->stollen_vga_memory, AMDGPU_GEM_DOMAIN_VRAM, NULL);
-	amdgpu_bo_unreserve(adev->stollen_vga_memory);
-	if (r) {
-		amdgpu_bo_unref(&adev->stollen_vga_memory);
-		return r;
-	}
 	DRM_INFO("amdgpu: %uM of VRAM memory ready\n",
 		 (unsigned) (adev->mc.real_vram_size / (1024 * 1024)));
-	r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_TT,
-				adev->mc.gtt_size >> PAGE_SHIFT);
+
+	if (amdgpu_gtt_size == -1)
+		gtt_size = max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
+			       adev->mc.mc_vram_size);
+	else
+		gtt_size = (uint64_t)amdgpu_gtt_size << 20;
+	r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_TT, gtt_size >> PAGE_SHIFT);
 	if (r) {
 		DRM_ERROR("Failed initializing GTT heap.\n");
 		return r;
 	}
 	DRM_INFO("amdgpu: %uM of GTT memory ready.\n",
-		 (unsigned)(adev->mc.gtt_size / (1024 * 1024)));
+		 (unsigned)(gtt_size / (1024 * 1024)));
 
 	adev->gds.mem.total_size = adev->gds.mem.total_size << AMDGPU_GDS_SHIFT;
 	adev->gds.mem.gfx_partition_size = adev->gds.mem.gfx_partition_size << AMDGPU_GDS_SHIFT;
@@ -1203,13 +1362,13 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
 	if (!adev->mman.initialized)
 		return;
 	amdgpu_ttm_debugfs_fini(adev);
-	if (adev->stollen_vga_memory) {
-		r = amdgpu_bo_reserve(adev->stollen_vga_memory, true);
+	if (adev->stolen_vga_memory) {
+		r = amdgpu_bo_reserve(adev->stolen_vga_memory, true);
 		if (r == 0) {
-			amdgpu_bo_unpin(adev->stollen_vga_memory);
-			amdgpu_bo_unreserve(adev->stollen_vga_memory);
+			amdgpu_bo_unpin(adev->stolen_vga_memory);
+			amdgpu_bo_unreserve(adev->stolen_vga_memory);
 		}
-		amdgpu_bo_unref(&adev->stollen_vga_memory);
+		amdgpu_bo_unref(&adev->stolen_vga_memory);
 	}
 	ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_VRAM);
 	ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_TT);
@@ -1256,12 +1415,77 @@ int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma)
 	return ttm_bo_mmap(filp, vma, &adev->mman.bdev);
 }
 
-int amdgpu_copy_buffer(struct amdgpu_ring *ring,
-		       uint64_t src_offset,
-		       uint64_t dst_offset,
-		       uint32_t byte_count,
+static int amdgpu_map_buffer(struct ttm_buffer_object *bo,
+			     struct ttm_mem_reg *mem, unsigned num_pages,
+			     uint64_t offset, unsigned window,
+			     struct amdgpu_ring *ring,
+			     uint64_t *addr)
+{
+	struct amdgpu_ttm_tt *gtt = (void *)bo->ttm;
+	struct amdgpu_device *adev = ring->adev;
+	struct ttm_tt *ttm = bo->ttm;
+	struct amdgpu_job *job;
+	unsigned num_dw, num_bytes;
+	dma_addr_t *dma_address;
+	struct dma_fence *fence;
+	uint64_t src_addr, dst_addr;
+	uint64_t flags;
+	int r;
+
+	BUG_ON(adev->mman.buffer_funcs->copy_max_bytes <
+	       AMDGPU_GTT_MAX_TRANSFER_SIZE * 8);
+
+	*addr = adev->mc.gart_start;
+	*addr += (u64)window * AMDGPU_GTT_MAX_TRANSFER_SIZE *
+		AMDGPU_GPU_PAGE_SIZE;
+
+	num_dw = adev->mman.buffer_funcs->copy_num_dw;
+	while (num_dw & 0x7)
+		num_dw++;
+
+	num_bytes = num_pages * 8;
+
+	r = amdgpu_job_alloc_with_ib(adev, num_dw * 4 + num_bytes, &job);
+	if (r)
+		return r;
+
+	src_addr = num_dw * 4;
+	src_addr += job->ibs[0].gpu_addr;
+
+	dst_addr = adev->gart.table_addr;
+	dst_addr += window * AMDGPU_GTT_MAX_TRANSFER_SIZE * 8;
+	amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr,
+				dst_addr, num_bytes);
+
+	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
+	WARN_ON(job->ibs[0].length_dw > num_dw);
+
+	dma_address = &gtt->ttm.dma_address[offset >> PAGE_SHIFT];
+	flags = amdgpu_ttm_tt_pte_flags(adev, ttm, mem);
+	r = amdgpu_gart_map(adev, 0, num_pages, dma_address, flags,
+			    &job->ibs[0].ptr[num_dw]);
+	if (r)
+		goto error_free;
+
+	r = amdgpu_job_submit(job, ring, &adev->mman.entity,
+			      AMDGPU_FENCE_OWNER_UNDEFINED, &fence);
+	if (r)
+		goto error_free;
+
+	dma_fence_put(fence);
+
+	return r;
+
+error_free:
+	amdgpu_job_free(job);
+	return r;
+}
+
+int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
+		       uint64_t dst_offset, uint32_t byte_count,
 		       struct reservation_object *resv,
-		       struct dma_fence **fence, bool direct_submit)
+		       struct dma_fence **fence, bool direct_submit,
+		       bool vm_needs_flush)
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct amdgpu_job *job;
@@ -1283,6 +1507,7 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring,
 	if (r)
 		return r;
 
+	job->vm_needs_flush = vm_needs_flush;
 	if (resv) {
 		r = amdgpu_sync_resv(adev, &job->sync, resv,
 				     AMDGPU_FENCE_OWNER_UNDEFINED);
@@ -1327,11 +1552,12 @@ error_free:
 }
 
 int amdgpu_fill_buffer(struct amdgpu_bo *bo,
-		       uint32_t src_data,
+		       uint64_t src_data,
 		       struct reservation_object *resv,
 		       struct dma_fence **fence)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
+	/* max_bytes applies to SDMA_OP_PTEPDE as well as SDMA_OP_CONST_FILL*/
 	uint32_t max_bytes = adev->mman.buffer_funcs->fill_max_bytes;
 	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
 
@@ -1347,6 +1573,12 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo,
 		return -EINVAL;
 	}
 
+	if (bo->tbo.mem.mem_type == TTM_PL_TT) {
+		r = amdgpu_ttm_bind(&bo->tbo, &bo->tbo.mem);
+		if (r)
+			return r;
+	}
+
 	num_pages = bo->tbo.num_pages;
 	mm_node = bo->tbo.mem.mm_node;
 	num_loops = 0;
@@ -1357,7 +1589,9 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo,
 		num_pages -= mm_node->size;
 		++mm_node;
 	}
-	num_dw = num_loops * adev->mman.buffer_funcs->fill_num_dw;
+
+	/* 10 double words for each SDMA_OP_PTEPDE cmd */
+	num_dw = num_loops * 10;
 
 	/* for IB padding */
 	num_dw += 64;
@@ -1382,16 +1616,16 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo,
 		uint32_t byte_count = mm_node->size << PAGE_SHIFT;
 		uint64_t dst_addr;
 
-		r = amdgpu_mm_node_addr(&bo->tbo, mm_node,
-					&bo->tbo.mem, &dst_addr);
-		if (r)
-			return r;
+		WARN_ONCE(byte_count & 0x7, "size should be a multiple of 8");
 
+		dst_addr = amdgpu_mm_node_addr(&bo->tbo, mm_node, &bo->tbo.mem);
 		while (byte_count) {
 			uint32_t cur_size_in_bytes = min(byte_count, max_bytes);
 
-			amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data,
-						dst_addr, cur_size_in_bytes);
+			amdgpu_vm_set_pte_pde(adev, &job->ibs[0],
+					dst_addr, 0,
+					cur_size_in_bytes >> 3, 0,
+					src_data);
 
 			dst_addr += cur_size_in_bytes;
 			byte_count -= cur_size_in_bytes;
@@ -1417,32 +1651,16 @@ error_free:
 
 #if defined(CONFIG_DEBUG_FS)
 
-extern void amdgpu_gtt_mgr_print(struct seq_file *m, struct ttm_mem_type_manager
-				 *man);
 static int amdgpu_mm_dump_table(struct seq_file *m, void *data)
 {
 	struct drm_info_node *node = (struct drm_info_node *)m->private;
 	unsigned ttm_pl = *(int *)node->info_ent->data;
 	struct drm_device *dev = node->minor->dev;
 	struct amdgpu_device *adev = dev->dev_private;
-	struct drm_mm *mm = (struct drm_mm *)adev->mman.bdev.man[ttm_pl].priv;
-	struct ttm_bo_global *glob = adev->mman.bdev.glob;
+	struct ttm_mem_type_manager *man = &adev->mman.bdev.man[ttm_pl];
 	struct drm_printer p = drm_seq_file_printer(m);
 
-	spin_lock(&glob->lru_lock);
-	drm_mm_print(mm, &p);
-	spin_unlock(&glob->lru_lock);
-	switch (ttm_pl) {
-	case TTM_PL_VRAM:
-		seq_printf(m, "man size:%llu pages, ram usage:%lluMB, vis usage:%lluMB\n",
-			   adev->mman.bdev.man[ttm_pl].size,
-			   (u64)atomic64_read(&adev->vram_usage) >> 20,
-			   (u64)atomic64_read(&adev->vram_vis_usage) >> 20);
-		break;
-	case TTM_PL_TT:
-		amdgpu_gtt_mgr_print(m, &adev->mman.bdev.man[TTM_PL_TT]);
-		break;
-	}
+	man->func->debug(man, &p);
 	return 0;
 }
 
@@ -1574,7 +1792,7 @@ static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev)
 				  adev, &amdgpu_ttm_gtt_fops);
 	if (IS_ERR(ent))
 		return PTR_ERR(ent);
-	i_size_write(ent->d_inode, adev->mc.gtt_size);
+	i_size_write(ent->d_inode, adev->mc.gart_size);
 	adev->mman.gtt = ent;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 6bdede8ff12b..43093bffa2cf 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -34,6 +34,9 @@
 #define AMDGPU_PL_FLAG_GWS		(TTM_PL_FLAG_PRIV << 1)
 #define AMDGPU_PL_FLAG_OA		(TTM_PL_FLAG_PRIV << 2)
 
+#define AMDGPU_GTT_MAX_TRANSFER_SIZE	512
+#define AMDGPU_GTT_NUM_TRANSFER_WINDOWS	2
+
 struct amdgpu_mman {
 	struct ttm_bo_global_ref        bo_global_ref;
 	struct drm_global_reference	mem_global_ref;
@@ -49,6 +52,8 @@ struct amdgpu_mman {
 	/* buffer handling */
 	const struct amdgpu_buffer_funcs	*buffer_funcs;
 	struct amdgpu_ring			*buffer_funcs_ring;
+
+	struct mutex				gtt_window_lock;
 	/* Scheduler entity for buffer moves */
 	struct amd_sched_entity			entity;
 };
@@ -56,24 +61,25 @@ struct amdgpu_mman {
 extern const struct ttm_mem_type_manager_func amdgpu_gtt_mgr_func;
 extern const struct ttm_mem_type_manager_func amdgpu_vram_mgr_func;
 
-int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,
-			 struct ttm_buffer_object *tbo,
-			 const struct ttm_place *place,
-			 struct ttm_mem_reg *mem);
+bool amdgpu_gtt_mgr_is_allocated(struct ttm_mem_reg *mem);
+uint64_t amdgpu_gtt_mgr_usage(struct ttm_mem_type_manager *man);
+
+uint64_t amdgpu_vram_mgr_usage(struct ttm_mem_type_manager *man);
+uint64_t amdgpu_vram_mgr_vis_usage(struct ttm_mem_type_manager *man);
 
-int amdgpu_copy_buffer(struct amdgpu_ring *ring,
-		       uint64_t src_offset,
-		       uint64_t dst_offset,
-		       uint32_t byte_count,
+int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
+		       uint64_t dst_offset, uint32_t byte_count,
 		       struct reservation_object *resv,
-		       struct dma_fence **fence, bool direct_submit);
+		       struct dma_fence **fence, bool direct_submit,
+		       bool vm_needs_flush);
 int amdgpu_fill_buffer(struct amdgpu_bo *bo,
-			uint32_t src_data,
+			uint64_t src_data,
 			struct reservation_object *resv,
 			struct dma_fence **fence);
 
 int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma);
 bool amdgpu_ttm_is_bound(struct ttm_tt *ttm);
 int amdgpu_ttm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *bo_mem);
+int amdgpu_ttm_recover_gart(struct amdgpu_device *adev);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
index 4f50eeb65855..36c763310df5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
@@ -275,14 +275,10 @@ amdgpu_ucode_get_load_type(struct amdgpu_device *adev, int load_type)
 		else
 			return AMDGPU_FW_LOAD_PSP;
 	case CHIP_RAVEN:
-#if 0
-		if (!load_type)
+		if (load_type != 2)
 			return AMDGPU_FW_LOAD_DIRECT;
 		else
 			return AMDGPU_FW_LOAD_PSP;
-#else
-		return AMDGPU_FW_LOAD_DIRECT;
-#endif
 	default:
 		DRM_ERROR("Unknow firmware load type\n");
 	}
@@ -362,8 +358,6 @@ static int amdgpu_ucode_patch_jt(struct amdgpu_firmware_info *ucode,
 			   (le32_to_cpu(header->jt_offset) * 4);
 	memcpy(dst_addr, src_addr, le32_to_cpu(header->jt_size) * 4);
 
-	ucode->ucode_size += le32_to_cpu(header->jt_size) * 4;
-
 	return 0;
 }
 
@@ -377,10 +371,15 @@ int amdgpu_ucode_init_bo(struct amdgpu_device *adev)
 	struct amdgpu_firmware_info *ucode = NULL;
 	const struct common_firmware_header *header = NULL;
 
+	if (!adev->firmware.fw_size) {
+		dev_warn(adev->dev, "No ip firmware need to load\n");
+		return 0;
+	}
+
 	err = amdgpu_bo_create(adev, adev->firmware.fw_size, PAGE_SIZE, true,
 				amdgpu_sriov_vf(adev) ? AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT,
 				AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
-				NULL, NULL, bo);
+				NULL, NULL, 0, bo);
 	if (err) {
 		dev_err(adev->dev, "(%d) Firmware buffer allocate failed\n", err);
 		goto failed;
@@ -459,6 +458,9 @@ int amdgpu_ucode_fini_bo(struct amdgpu_device *adev)
 	int i;
 	struct amdgpu_firmware_info *ucode = NULL;
 
+	if (!adev->firmware.fw_size)
+		return 0;
+
 	for (i = 0; i < adev->firmware.max_ucodes; i++) {
 		ucode = &adev->firmware.ucode[i];
 		if (ucode->fw) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
index 2ca09f111f08..e19928dae8e3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
@@ -588,6 +588,10 @@ static int amdgpu_uvd_cs_msg_decode(struct amdgpu_device *adev, uint32_t *msg,
 		}
 		break;
 
+	case 8: /* MJPEG */
+		min_dpb_size = 0;
+		break;
+
 	case 16: /* H265 */
 		image_size = (ALIGN(width, 16) * ALIGN(height, 16) * 3) / 2;
 		image_size = ALIGN(image_size, 256);
@@ -1051,7 +1055,7 @@ int amdgpu_uvd_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
 			     AMDGPU_GEM_DOMAIN_VRAM,
 			     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
 			     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
-			     NULL, NULL, &bo);
+			     NULL, NULL, 0, &bo);
 	if (r)
 		return r;
 
@@ -1101,7 +1105,7 @@ int amdgpu_uvd_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
 			     AMDGPU_GEM_DOMAIN_VRAM,
 			     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
 			     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
-			     NULL, NULL, &bo);
+			     NULL, NULL, 0, &bo);
 	if (r)
 		return r;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
index b692ad402252..c855366521ab 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
@@ -937,9 +937,9 @@ int amdgpu_vce_ring_test_ring(struct amdgpu_ring *ring)
 	unsigned i;
 	int r, timeout = adev->usec_timeout;
 
-	/* workaround VCE ring test slow issue for sriov*/
+	/* skip ring test for sriov*/
 	if (amdgpu_sriov_vf(adev))
-		timeout *= 10;
+		return 0;
 
 	r = amdgpu_ring_alloc(ring, 16);
 	if (r) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
index 09190fadd228..041e0121590c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
@@ -209,9 +209,9 @@ static void amdgpu_vcn_idle_work_handler(struct work_struct *work)
 
 	if (fences == 0) {
 		if (adev->pm.dpm_enabled) {
+			/* might be used when with pg/cg
 			amdgpu_dpm_enable_uvd(adev, false);
-		} else {
-			amdgpu_asic_set_uvd_clocks(adev, 0, 0);
+			*/
 		}
 	} else {
 		schedule_delayed_work(&adev->vcn.idle_work, VCN_IDLE_TIMEOUT);
@@ -223,12 +223,10 @@ void amdgpu_vcn_ring_begin_use(struct amdgpu_ring *ring)
 	struct amdgpu_device *adev = ring->adev;
 	bool set_clocks = !cancel_delayed_work_sync(&adev->vcn.idle_work);
 
-	if (set_clocks) {
-		if (adev->pm.dpm_enabled) {
-			amdgpu_dpm_enable_uvd(adev, true);
-		} else {
-			amdgpu_asic_set_uvd_clocks(adev, 53300, 40000);
-		}
+	if (set_clocks && adev->pm.dpm_enabled) {
+		/* might be used when with pg/cg
+		amdgpu_dpm_enable_uvd(adev, true);
+		*/
 	}
 }
 
@@ -361,7 +359,7 @@ static int amdgpu_vcn_dec_get_create_msg(struct amdgpu_ring *ring, uint32_t hand
 			     AMDGPU_GEM_DOMAIN_VRAM,
 			     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
 			     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
-			     NULL, NULL, &bo);
+			     NULL, NULL, 0, &bo);
 	if (r)
 		return r;
 
@@ -413,7 +411,7 @@ static int amdgpu_vcn_dec_get_destroy_msg(struct amdgpu_ring *ring, uint32_t han
 			     AMDGPU_GEM_DOMAIN_VRAM,
 			     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
 			     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
-			     NULL, NULL, &bo);
+			     NULL, NULL, 0, &bo);
 	if (r)
 		return r;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.c
new file mode 100644
index 000000000000..45ac91861965
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.c
@@ -0,0 +1,85 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "amdgpu.h"
+#include "amdgpu_vf_error.h"
+#include "mxgpu_ai.h"
+
+#define AMDGPU_VF_ERROR_ENTRY_SIZE    16 
+
+/* struct error_entry - amdgpu VF error information. */
+struct amdgpu_vf_error_buffer {
+	int read_count;
+	int write_count;
+	uint16_t code[AMDGPU_VF_ERROR_ENTRY_SIZE];
+	uint16_t flags[AMDGPU_VF_ERROR_ENTRY_SIZE];
+	uint64_t data[AMDGPU_VF_ERROR_ENTRY_SIZE];
+};
+
+struct amdgpu_vf_error_buffer admgpu_vf_errors;
+
+
+void amdgpu_vf_error_put(uint16_t sub_error_code, uint16_t error_flags, uint64_t error_data)
+{
+	int index;
+	uint16_t error_code = AMDGIM_ERROR_CODE(AMDGIM_ERROR_CATEGORY_VF, sub_error_code);
+
+	index = admgpu_vf_errors.write_count % AMDGPU_VF_ERROR_ENTRY_SIZE;
+	admgpu_vf_errors.code [index] = error_code;
+	admgpu_vf_errors.flags [index] = error_flags;
+	admgpu_vf_errors.data [index] = error_data;
+	admgpu_vf_errors.write_count ++;
+}
+
+
+void amdgpu_vf_error_trans_all(struct amdgpu_device *adev)
+{
+	/* u32 pf2vf_flags = 0; */
+	u32 data1, data2, data3;
+	int index;
+
+	if ((NULL == adev) || (!amdgpu_sriov_vf(adev)) || (!adev->virt.ops) || (!adev->virt.ops->trans_msg)) {
+		return;
+	}
+/*
+ 	TODO: Enable these code when pv2vf_info is merged
+	AMDGPU_FW_VRAM_PF2VF_READ (adev, feature_flags, &pf2vf_flags);
+	if (!(pf2vf_flags & AMDGIM_FEATURE_ERROR_LOG_COLLECT)) {
+		return;
+	}
+*/
+	/* The errors are overlay of array, correct read_count as full. */
+	if (admgpu_vf_errors.write_count - admgpu_vf_errors.read_count > AMDGPU_VF_ERROR_ENTRY_SIZE) {
+		admgpu_vf_errors.read_count = admgpu_vf_errors.write_count - AMDGPU_VF_ERROR_ENTRY_SIZE;
+	}
+
+	while (admgpu_vf_errors.read_count < admgpu_vf_errors.write_count) {
+		index =admgpu_vf_errors.read_count % AMDGPU_VF_ERROR_ENTRY_SIZE;
+		data1 = AMDGIM_ERROR_CODE_FLAGS_TO_MAILBOX (admgpu_vf_errors.code[index], admgpu_vf_errors.flags[index]);
+		data2 = admgpu_vf_errors.data[index] & 0xFFFFFFFF;
+		data3 = (admgpu_vf_errors.data[index] >> 32) & 0xFFFFFFFF;
+
+		adev->virt.ops->trans_msg(adev, IDH_LOG_VF_ERROR, data1, data2, data3);
+		admgpu_vf_errors.read_count ++;
+	}
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.h
new file mode 100644
index 000000000000..2a3278ec76ba
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vf_error.h
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __VF_ERROR_H__
+#define __VF_ERROR_H__
+
+#define AMDGIM_ERROR_CODE_FLAGS_TO_MAILBOX(c,f)    (((c & 0xFFFF) << 16) | (f & 0xFFFF))
+#define AMDGIM_ERROR_CODE(t,c)       (((t&0xF)<<12)|(c&0xFFF))
+
+/* Please keep enum same as AMD GIM driver */
+enum AMDGIM_ERROR_VF {
+	AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL = 0,
+	AMDGIM_ERROR_VF_NO_VBIOS,
+	AMDGIM_ERROR_VF_GPU_POST_ERROR,
+	AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL,
+	AMDGIM_ERROR_VF_FENCE_INIT_FAIL,
+
+	AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL,
+	AMDGIM_ERROR_VF_IB_INIT_FAIL,
+	AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL,
+	AMDGIM_ERROR_VF_ASIC_RESUME_FAIL,
+	AMDGIM_ERROR_VF_GPU_RESET_FAIL,
+
+	AMDGIM_ERROR_VF_TEST,
+	AMDGIM_ERROR_VF_MAX
+};
+
+enum AMDGIM_ERROR_CATEGORY {
+	AMDGIM_ERROR_CATEGORY_NON_USED = 0,
+	AMDGIM_ERROR_CATEGORY_GIM,
+	AMDGIM_ERROR_CATEGORY_PF,
+	AMDGIM_ERROR_CATEGORY_VF,
+	AMDGIM_ERROR_CATEGORY_VBIOS,
+	AMDGIM_ERROR_CATEGORY_MONITOR,
+
+	AMDGIM_ERROR_CATEGORY_MAX
+};
+
+void amdgpu_vf_error_put(uint16_t sub_error_code, uint16_t error_flags, uint64_t error_data);
+void amdgpu_vf_error_trans_all (struct amdgpu_device *adev);
+
+#endif /* __VF_ERROR_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index 8a081e162d13..ab05121b9272 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -46,14 +46,14 @@ int amdgpu_allocate_static_csa(struct amdgpu_device *adev)
  * address within META_DATA init package to support SRIOV gfx preemption.
  */
 
-int amdgpu_map_static_csa(struct amdgpu_device *adev, struct amdgpu_vm *vm)
+int amdgpu_map_static_csa(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+			  struct amdgpu_bo_va **bo_va)
 {
-	int r;
-	struct amdgpu_bo_va *bo_va;
 	struct ww_acquire_ctx ticket;
 	struct list_head list;
 	struct amdgpu_bo_list_entry pd;
 	struct ttm_validate_buffer csa_tv;
+	int r;
 
 	INIT_LIST_HEAD(&list);
 	INIT_LIST_HEAD(&csa_tv.head);
@@ -69,34 +69,33 @@ int amdgpu_map_static_csa(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 		return r;
 	}
 
-	bo_va = amdgpu_vm_bo_add(adev, vm, adev->virt.csa_obj);
-	if (!bo_va) {
+	*bo_va = amdgpu_vm_bo_add(adev, vm, adev->virt.csa_obj);
+	if (!*bo_va) {
 		ttm_eu_backoff_reservation(&ticket, &list);
 		DRM_ERROR("failed to create bo_va for static CSA\n");
 		return -ENOMEM;
 	}
 
-	r = amdgpu_vm_alloc_pts(adev, bo_va->vm, AMDGPU_CSA_VADDR,
-				   AMDGPU_CSA_SIZE);
+	r = amdgpu_vm_alloc_pts(adev, (*bo_va)->base.vm, AMDGPU_CSA_VADDR,
+				AMDGPU_CSA_SIZE);
 	if (r) {
 		DRM_ERROR("failed to allocate pts for static CSA, err=%d\n", r);
-		amdgpu_vm_bo_rmv(adev, bo_va);
+		amdgpu_vm_bo_rmv(adev, *bo_va);
 		ttm_eu_backoff_reservation(&ticket, &list);
 		return r;
 	}
 
-	r = amdgpu_vm_bo_map(adev, bo_va, AMDGPU_CSA_VADDR, 0,AMDGPU_CSA_SIZE,
-						AMDGPU_PTE_READABLE | AMDGPU_PTE_WRITEABLE |
-						AMDGPU_PTE_EXECUTABLE);
+	r = amdgpu_vm_bo_map(adev, *bo_va, AMDGPU_CSA_VADDR, 0, AMDGPU_CSA_SIZE,
+			     AMDGPU_PTE_READABLE | AMDGPU_PTE_WRITEABLE |
+			     AMDGPU_PTE_EXECUTABLE);
 
 	if (r) {
 		DRM_ERROR("failed to do bo_map on static CSA, err=%d\n", r);
-		amdgpu_vm_bo_rmv(adev, bo_va);
+		amdgpu_vm_bo_rmv(adev, *bo_va);
 		ttm_eu_backoff_reservation(&ticket, &list);
 		return r;
 	}
 
-	vm->csa_bo_va = bo_va;
 	ttm_eu_backoff_reservation(&ticket, &list);
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index 9e1062edb76e..afcfb8bcfb65 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -43,6 +43,7 @@ struct amdgpu_virt_ops {
 	int (*req_full_gpu)(struct amdgpu_device *adev, bool init);
 	int (*rel_full_gpu)(struct amdgpu_device *adev, bool init);
 	int (*reset_gpu)(struct amdgpu_device *adev);
+	void (*trans_msg)(struct amdgpu_device *adev, u32 req, u32 data1, u32 data2, u32 data3);
 };
 
 /* GPU virtualization */
@@ -89,7 +90,8 @@ static inline bool is_virtual_machine(void)
 
 struct amdgpu_vm;
 int amdgpu_allocate_static_csa(struct amdgpu_device *adev);
-int amdgpu_map_static_csa(struct amdgpu_device *adev, struct amdgpu_vm *vm);
+int amdgpu_map_static_csa(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+			  struct amdgpu_bo_va **bo_va);
 void amdgpu_virt_init_setting(struct amdgpu_device *adev);
 uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
 void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 5795f81369f0..bd20ff018512 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -77,8 +77,6 @@ struct amdgpu_pte_update_params {
 	void (*func)(struct amdgpu_pte_update_params *params, uint64_t pe,
 		     uint64_t addr, unsigned count, uint32_t incr,
 		     uint64_t flags);
-	/* indicate update pt or its shadow */
-	bool shadow;
 	/* The next two are used during VM update by CPU
 	 *  DMA addresses to use for mapping
 	 *  Kernel pointer of PD/PT BO that needs to be updated
@@ -161,11 +159,18 @@ void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
  */
 static int amdgpu_vm_validate_level(struct amdgpu_vm_pt *parent,
 				    int (*validate)(void *, struct amdgpu_bo *),
-				    void *param)
+				    void *param, bool use_cpu_for_update,
+				    struct ttm_bo_global *glob)
 {
 	unsigned i;
 	int r;
 
+	if (use_cpu_for_update) {
+		r = amdgpu_bo_kmap(parent->bo, NULL);
+		if (r)
+			return r;
+	}
+
 	if (!parent->entries)
 		return 0;
 
@@ -179,11 +184,18 @@ static int amdgpu_vm_validate_level(struct amdgpu_vm_pt *parent,
 		if (r)
 			return r;
 
+		spin_lock(&glob->lru_lock);
+		ttm_bo_move_to_lru_tail(&entry->bo->tbo);
+		if (entry->bo->shadow)
+			ttm_bo_move_to_lru_tail(&entry->bo->shadow->tbo);
+		spin_unlock(&glob->lru_lock);
+
 		/*
 		 * Recurse into the sub directory. This is harmless because we
 		 * have only a maximum of 5 layers.
 		 */
-		r = amdgpu_vm_validate_level(entry, validate, param);
+		r = amdgpu_vm_validate_level(entry, validate, param,
+					     use_cpu_for_update, glob);
 		if (r)
 			return r;
 	}
@@ -214,54 +226,12 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	if (num_evictions == vm->last_eviction_counter)
 		return 0;
 
-	return amdgpu_vm_validate_level(&vm->root, validate, param);
+	return amdgpu_vm_validate_level(&vm->root, validate, param,
+					vm->use_cpu_for_update,
+					adev->mman.bdev.glob);
 }
 
 /**
- * amdgpu_vm_move_level_in_lru - move one level of PT BOs to the LRU tail
- *
- * @adev: amdgpu device instance
- * @vm: vm providing the BOs
- *
- * Move the PT BOs to the tail of the LRU.
- */
-static void amdgpu_vm_move_level_in_lru(struct amdgpu_vm_pt *parent)
-{
-	unsigned i;
-
-	if (!parent->entries)
-		return;
-
-	for (i = 0; i <= parent->last_entry_used; ++i) {
-		struct amdgpu_vm_pt *entry = &parent->entries[i];
-
-		if (!entry->bo)
-			continue;
-
-		ttm_bo_move_to_lru_tail(&entry->bo->tbo);
-		amdgpu_vm_move_level_in_lru(entry);
-	}
-}
-
-/**
- * amdgpu_vm_move_pt_bos_in_lru - move the PT BOs to the LRU tail
- *
- * @adev: amdgpu device instance
- * @vm: vm providing the BOs
- *
- * Move the PT BOs to the tail of the LRU.
- */
-void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev,
-				  struct amdgpu_vm *vm)
-{
-	struct ttm_bo_global *glob = adev->mman.bdev.glob;
-
-	spin_lock(&glob->lru_lock);
-	amdgpu_vm_move_level_in_lru(&vm->root);
-	spin_unlock(&glob->lru_lock);
-}
-
- /**
  * amdgpu_vm_alloc_levels - allocate the PD/PT levels
  *
  * @adev: amdgpu_device pointer
@@ -282,6 +252,7 @@ static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
 	unsigned pt_idx, from, to;
 	int r;
 	u64 flags;
+	uint64_t init_value = 0;
 
 	if (!parent->entries) {
 		unsigned num_entries = amdgpu_vm_num_entries(adev, level);
@@ -315,6 +286,12 @@ static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
 		flags |= (AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
 				AMDGPU_GEM_CREATE_SHADOW);
 
+	if (vm->pte_support_ats) {
+		init_value = AMDGPU_PTE_SYSTEM;
+		if (level != adev->vm_manager.num_level - 1)
+			init_value |= AMDGPU_PDE_PTE;
+	}
+
 	/* walk over the address space and allocate the page tables */
 	for (pt_idx = from; pt_idx <= to; ++pt_idx) {
 		struct reservation_object *resv = vm->root.bo->tbo.resv;
@@ -327,10 +304,18 @@ static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
 					     AMDGPU_GPU_PAGE_SIZE, true,
 					     AMDGPU_GEM_DOMAIN_VRAM,
 					     flags,
-					     NULL, resv, &pt);
+					     NULL, resv, init_value, &pt);
 			if (r)
 				return r;
 
+			if (vm->use_cpu_for_update) {
+				r = amdgpu_bo_kmap(pt, NULL);
+				if (r) {
+					amdgpu_bo_unref(&pt);
+					return r;
+				}
+			}
+
 			/* Keep a reference to the root directory to avoid
 			* freeing them up in the wrong order.
 			*/
@@ -424,7 +409,7 @@ static int amdgpu_vm_grab_reserved_vmid_locked(struct amdgpu_vm *vm,
 	struct dma_fence *updates = sync->last_vm_update;
 	int r = 0;
 	struct dma_fence *flushed, *tmp;
-	bool needs_flush = false;
+	bool needs_flush = vm->use_cpu_for_update;
 
 	flushed  = id->flushed_updates;
 	if ((amdgpu_vm_had_gpu_reset(adev, id)) ||
@@ -545,11 +530,11 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
 	}
 	kfree(fences);
 
-	job->vm_needs_flush = false;
+	job->vm_needs_flush = vm->use_cpu_for_update;
 	/* Check if we can use a VMID already assigned to this VM */
 	list_for_each_entry_reverse(id, &id_mgr->ids_lru, list) {
 		struct dma_fence *flushed;
-		bool needs_flush = false;
+		bool needs_flush = vm->use_cpu_for_update;
 
 		/* Check all the prerequisites to using this VMID */
 		if (amdgpu_vm_had_gpu_reset(adev, id))
@@ -745,7 +730,7 @@ static bool amdgpu_vm_is_large_bar(struct amdgpu_device *adev)
  *
  * Emit a VM flush when it is necessary.
  */
-int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job)
+int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync)
 {
 	struct amdgpu_device *adev = ring->adev;
 	unsigned vmhub = ring->funcs->vmhub;
@@ -767,12 +752,15 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job)
 		vm_flush_needed = true;
 	}
 
-	if (!vm_flush_needed && !gds_switch_needed)
+	if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync)
 		return 0;
 
 	if (ring->funcs->init_cond_exec)
 		patch_offset = amdgpu_ring_init_cond_exec(ring);
 
+	if (need_pipe_sync)
+		amdgpu_ring_emit_pipeline_sync(ring);
+
 	if (ring->funcs->emit_vm_flush && vm_flush_needed) {
 		struct dma_fence *fence;
 
@@ -874,8 +862,8 @@ struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
 {
 	struct amdgpu_bo_va *bo_va;
 
-	list_for_each_entry(bo_va, &bo->va, bo_list) {
-		if (bo_va->vm == vm) {
+	list_for_each_entry(bo_va, &bo->va, base.bo_list) {
+		if (bo_va->base.vm == vm) {
 			return bo_va;
 		}
 	}
@@ -981,6 +969,8 @@ static void amdgpu_vm_cpu_set_ptes(struct amdgpu_pte_update_params *params,
 	unsigned int i;
 	uint64_t value;
 
+	trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
+
 	for (i = 0; i < count; i++) {
 		value = params->pages_addr ?
 			amdgpu_vm_map_gart(params->pages_addr, addr) :
@@ -989,19 +979,16 @@ static void amdgpu_vm_cpu_set_ptes(struct amdgpu_pte_update_params *params,
 					i, value, flags);
 		addr += incr;
 	}
-
-	/* Flush HDP */
-	mb();
-	amdgpu_gart_flush_gpu_tlb(params->adev, 0);
 }
 
-static int amdgpu_vm_bo_wait(struct amdgpu_device *adev, struct amdgpu_bo *bo)
+static int amdgpu_vm_wait_pd(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+			     void *owner)
 {
 	struct amdgpu_sync sync;
 	int r;
 
 	amdgpu_sync_create(&sync);
-	amdgpu_sync_resv(adev, &sync, bo->tbo.resv, AMDGPU_FENCE_OWNER_VM);
+	amdgpu_sync_resv(adev, &sync, vm->root.bo->tbo.resv, owner);
 	r = amdgpu_sync_wait(&sync, true);
 	amdgpu_sync_free(&sync);
 
@@ -1042,23 +1029,14 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
 	params.adev = adev;
 	shadow = parent->bo->shadow;
 
-	WARN_ON(vm->use_cpu_for_update && shadow);
-	if (vm->use_cpu_for_update && !shadow) {
-		r = amdgpu_bo_kmap(parent->bo, (void **)&pd_addr);
-		if (r)
-			return r;
-		r = amdgpu_vm_bo_wait(adev, parent->bo);
-		if (unlikely(r)) {
-			amdgpu_bo_kunmap(parent->bo);
+	if (vm->use_cpu_for_update) {
+		pd_addr = (unsigned long)amdgpu_bo_kptr(parent->bo);
+		r = amdgpu_vm_wait_pd(adev, vm, AMDGPU_FENCE_OWNER_VM);
+		if (unlikely(r))
 			return r;
-		}
+
 		params.func = amdgpu_vm_cpu_set_ptes;
 	} else {
-		if (shadow) {
-			r = amdgpu_ttm_bind(&shadow->tbo, &shadow->tbo.mem);
-			if (r)
-				return r;
-		}
 		ring = container_of(vm->entity.sched, struct amdgpu_ring,
 				    sched);
 
@@ -1094,21 +1072,14 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
 		if (bo == NULL)
 			continue;
 
-		if (bo->shadow) {
-			struct amdgpu_bo *pt_shadow = bo->shadow;
-
-			r = amdgpu_ttm_bind(&pt_shadow->tbo,
-					    &pt_shadow->tbo.mem);
-			if (r)
-				return r;
-		}
-
 		pt = amdgpu_bo_gpu_offset(bo);
 		pt = amdgpu_gart_get_vm_pde(adev, pt);
-		if (parent->entries[pt_idx].addr == pt)
+		/* Don't update huge pages here */
+		if ((parent->entries[pt_idx].addr & AMDGPU_PDE_PTE) ||
+		    parent->entries[pt_idx].addr == (pt | AMDGPU_PTE_VALID))
 			continue;
 
-		parent->entries[pt_idx].addr = pt;
+		parent->entries[pt_idx].addr = pt | AMDGPU_PTE_VALID;
 
 		pde = pd_addr + pt_idx * 8;
 		if (((last_pde + 8 * count) != pde) ||
@@ -1146,28 +1117,29 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
 			    count, incr, AMDGPU_PTE_VALID);
 	}
 
-	if (params.func == amdgpu_vm_cpu_set_ptes)
-		amdgpu_bo_kunmap(parent->bo);
-	else if (params.ib->length_dw == 0) {
-		amdgpu_job_free(job);
-	} else {
-		amdgpu_ring_pad_ib(ring, params.ib);
-		amdgpu_sync_resv(adev, &job->sync, parent->bo->tbo.resv,
-				 AMDGPU_FENCE_OWNER_VM);
-		if (shadow)
-			amdgpu_sync_resv(adev, &job->sync, shadow->tbo.resv,
+	if (!vm->use_cpu_for_update) {
+		if (params.ib->length_dw == 0) {
+			amdgpu_job_free(job);
+		} else {
+			amdgpu_ring_pad_ib(ring, params.ib);
+			amdgpu_sync_resv(adev, &job->sync, parent->bo->tbo.resv,
 					 AMDGPU_FENCE_OWNER_VM);
+			if (shadow)
+				amdgpu_sync_resv(adev, &job->sync,
+						 shadow->tbo.resv,
+						 AMDGPU_FENCE_OWNER_VM);
+
+			WARN_ON(params.ib->length_dw > ndw);
+			r = amdgpu_job_submit(job, ring, &vm->entity,
+					AMDGPU_FENCE_OWNER_VM, &fence);
+			if (r)
+				goto error_free;
 
-		WARN_ON(params.ib->length_dw > ndw);
-		r = amdgpu_job_submit(job, ring, &vm->entity,
-				AMDGPU_FENCE_OWNER_VM, &fence);
-		if (r)
-			goto error_free;
-
-		amdgpu_bo_fence(parent->bo, fence, true);
-		dma_fence_put(vm->last_dir_update);
-		vm->last_dir_update = dma_fence_get(fence);
-		dma_fence_put(fence);
+			amdgpu_bo_fence(parent->bo, fence, true);
+			dma_fence_put(vm->last_dir_update);
+			vm->last_dir_update = dma_fence_get(fence);
+			dma_fence_put(fence);
+		}
 	}
 	/*
 	 * Recurse into the subdirectories. This recursion is harmless because
@@ -1235,33 +1207,112 @@ int amdgpu_vm_update_directories(struct amdgpu_device *adev,
 	if (r)
 		amdgpu_vm_invalidate_level(&vm->root);
 
+	if (vm->use_cpu_for_update) {
+		/* Flush HDP */
+		mb();
+		amdgpu_gart_flush_gpu_tlb(adev, 0);
+	}
+
 	return r;
 }
 
 /**
- * amdgpu_vm_find_pt - find the page table for an address
+ * amdgpu_vm_find_entry - find the entry for an address
  *
  * @p: see amdgpu_pte_update_params definition
  * @addr: virtual address in question
+ * @entry: resulting entry or NULL
+ * @parent: parent entry
  *
- * Find the page table BO for a virtual address, return NULL when none found.
+ * Find the vm_pt entry and it's parent for the given address.
  */
-static struct amdgpu_bo *amdgpu_vm_get_pt(struct amdgpu_pte_update_params *p,
-					  uint64_t addr)
+void amdgpu_vm_get_entry(struct amdgpu_pte_update_params *p, uint64_t addr,
+			 struct amdgpu_vm_pt **entry,
+			 struct amdgpu_vm_pt **parent)
 {
-	struct amdgpu_vm_pt *entry = &p->vm->root;
 	unsigned idx, level = p->adev->vm_manager.num_level;
 
-	while (entry->entries) {
+	*parent = NULL;
+	*entry = &p->vm->root;
+	while ((*entry)->entries) {
 		idx = addr >> (p->adev->vm_manager.block_size * level--);
-		idx %= amdgpu_bo_size(entry->bo) / 8;
-		entry = &entry->entries[idx];
+		idx %= amdgpu_bo_size((*entry)->bo) / 8;
+		*parent = *entry;
+		*entry = &(*entry)->entries[idx];
 	}
 
 	if (level)
-		return NULL;
+		*entry = NULL;
+}
 
-	return entry->bo;
+/**
+ * amdgpu_vm_handle_huge_pages - handle updating the PD with huge pages
+ *
+ * @p: see amdgpu_pte_update_params definition
+ * @entry: vm_pt entry to check
+ * @parent: parent entry
+ * @nptes: number of PTEs updated with this operation
+ * @dst: destination address where the PTEs should point to
+ * @flags: access flags fro the PTEs
+ *
+ * Check if we can update the PD with a huge page.
+ */
+static void amdgpu_vm_handle_huge_pages(struct amdgpu_pte_update_params *p,
+					struct amdgpu_vm_pt *entry,
+					struct amdgpu_vm_pt *parent,
+					unsigned nptes, uint64_t dst,
+					uint64_t flags)
+{
+	bool use_cpu_update = (p->func == amdgpu_vm_cpu_set_ptes);
+	uint64_t pd_addr, pde;
+
+	/* In the case of a mixed PT the PDE must point to it*/
+	if (p->adev->asic_type < CHIP_VEGA10 ||
+	    nptes != AMDGPU_VM_PTE_COUNT(p->adev) ||
+	    p->src ||
+	    !(flags & AMDGPU_PTE_VALID)) {
+
+		dst = amdgpu_bo_gpu_offset(entry->bo);
+		dst = amdgpu_gart_get_vm_pde(p->adev, dst);
+		flags = AMDGPU_PTE_VALID;
+	} else {
+		/* Set the huge page flag to stop scanning at this PDE */
+		flags |= AMDGPU_PDE_PTE;
+	}
+
+	if (entry->addr == (dst | flags))
+		return;
+
+	entry->addr = (dst | flags);
+
+	if (use_cpu_update) {
+		/* In case a huge page is replaced with a system
+		 * memory mapping, p->pages_addr != NULL and
+		 * amdgpu_vm_cpu_set_ptes would try to translate dst
+		 * through amdgpu_vm_map_gart. But dst is already a
+		 * GPU address (of the page table). Disable
+		 * amdgpu_vm_map_gart temporarily.
+		 */
+		dma_addr_t *tmp;
+
+		tmp = p->pages_addr;
+		p->pages_addr = NULL;
+
+		pd_addr = (unsigned long)amdgpu_bo_kptr(parent->bo);
+		pde = pd_addr + (entry - parent->entries) * 8;
+		amdgpu_vm_cpu_set_ptes(p, pde, dst, 1, 0, flags);
+
+		p->pages_addr = tmp;
+	} else {
+		if (parent->bo->shadow) {
+			pd_addr = amdgpu_bo_gpu_offset(parent->bo->shadow);
+			pde = pd_addr + (entry - parent->entries) * 8;
+			amdgpu_vm_do_set_ptes(p, pde, dst, 1, 0, flags);
+		}
+		pd_addr = amdgpu_bo_gpu_offset(parent->bo);
+		pde = pd_addr + (entry - parent->entries) * 8;
+		amdgpu_vm_do_set_ptes(p, pde, dst, 1, 0, flags);
+	}
 }
 
 /**
@@ -1287,49 +1338,44 @@ static int amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
 	uint64_t addr, pe_start;
 	struct amdgpu_bo *pt;
 	unsigned nptes;
-	int r;
 	bool use_cpu_update = (params->func == amdgpu_vm_cpu_set_ptes);
 
-
 	/* walk over the address space and update the page tables */
-	for (addr = start; addr < end; addr += nptes) {
-		pt = amdgpu_vm_get_pt(params, addr);
-		if (!pt) {
-			pr_err("PT not found, aborting update_ptes\n");
-			return -EINVAL;
-		}
-
-		if (params->shadow) {
-			if (WARN_ONCE(use_cpu_update,
-				"CPU VM update doesn't suuport shadow pages"))
-				return 0;
+	for (addr = start; addr < end; addr += nptes,
+	     dst += nptes * AMDGPU_GPU_PAGE_SIZE) {
+		struct amdgpu_vm_pt *entry, *parent;
 
-			if (!pt->shadow)
-				return 0;
-			pt = pt->shadow;
-		}
+		amdgpu_vm_get_entry(params, addr, &entry, &parent);
+		if (!entry)
+			return -ENOENT;
 
 		if ((addr & ~mask) == (end & ~mask))
 			nptes = end - addr;
 		else
 			nptes = AMDGPU_VM_PTE_COUNT(adev) - (addr & mask);
 
+		amdgpu_vm_handle_huge_pages(params, entry, parent,
+					    nptes, dst, flags);
+		/* We don't need to update PTEs for huge pages */
+		if (entry->addr & AMDGPU_PDE_PTE)
+			continue;
+
+		pt = entry->bo;
 		if (use_cpu_update) {
-			r = amdgpu_bo_kmap(pt, (void *)&pe_start);
-			if (r)
-				return r;
-		} else
+			pe_start = (unsigned long)amdgpu_bo_kptr(pt);
+		} else {
+			if (pt->shadow) {
+				pe_start = amdgpu_bo_gpu_offset(pt->shadow);
+				pe_start += (addr & mask) * 8;
+				params->func(params, pe_start, dst, nptes,
+					     AMDGPU_GPU_PAGE_SIZE, flags);
+			}
 			pe_start = amdgpu_bo_gpu_offset(pt);
+		}
 
 		pe_start += (addr & mask) * 8;
-
 		params->func(params, pe_start, dst, nptes,
 			     AMDGPU_GPU_PAGE_SIZE, flags);
-
-		dst += nptes * AMDGPU_GPU_PAGE_SIZE;
-
-		if (use_cpu_update)
-			amdgpu_bo_kunmap(pt);
 	}
 
 	return 0;
@@ -1370,10 +1416,9 @@ static int amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params	*params,
 	 * Userspace can support this by aligning virtual base address and
 	 * allocation size to the fragment size.
 	 */
-
-	/* SI and newer are optimized for 64KB */
-	uint64_t frag_flags = AMDGPU_PTE_FRAG(AMDGPU_LOG2_PAGES_PER_FRAG);
-	uint64_t frag_align = 1 << AMDGPU_LOG2_PAGES_PER_FRAG;
+	unsigned pages_per_frag = params->adev->vm_manager.fragment_size;
+	uint64_t frag_flags = AMDGPU_PTE_FRAG(pages_per_frag);
+	uint64_t frag_align = 1 << pages_per_frag;
 
 	uint64_t frag_start = ALIGN(start, frag_align);
 	uint64_t frag_end = end & ~(frag_align - 1);
@@ -1445,6 +1490,10 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
 	params.vm = vm;
 	params.src = src;
 
+	/* sync to everything on unmapping */
+	if (!(flags & AMDGPU_PTE_VALID))
+		owner = AMDGPU_FENCE_OWNER_UNDEFINED;
+
 	if (vm->use_cpu_for_update) {
 		/* params.src is used as flag to indicate system Memory */
 		if (pages_addr)
@@ -1453,23 +1502,18 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
 		/* Wait for PT BOs to be free. PTs share the same resv. object
 		 * as the root PD BO
 		 */
-		r = amdgpu_vm_bo_wait(adev, vm->root.bo);
+		r = amdgpu_vm_wait_pd(adev, vm, owner);
 		if (unlikely(r))
 			return r;
 
 		params.func = amdgpu_vm_cpu_set_ptes;
 		params.pages_addr = pages_addr;
-		params.shadow = false;
 		return amdgpu_vm_frag_ptes(&params, start, last + 1,
 					   addr, flags);
 	}
 
 	ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
 
-	/* sync to everything on unmapping */
-	if (!(flags & AMDGPU_PTE_VALID))
-		owner = AMDGPU_FENCE_OWNER_UNDEFINED;
-
 	nptes = last - start + 1;
 
 	/*
@@ -1481,6 +1525,9 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
 	/* padding, etc. */
 	ndw = 64;
 
+	/* one PDE write for each huge page */
+	ndw += ((nptes >> adev->vm_manager.block_size) + 1) * 6;
+
 	if (src) {
 		/* only copy commands needed */
 		ndw += ncmds * 7;
@@ -1542,11 +1589,6 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
 	if (r)
 		goto error_free;
 
-	params.shadow = true;
-	r = amdgpu_vm_frag_ptes(&params, start, last + 1, addr, flags);
-	if (r)
-		goto error_free;
-	params.shadow = false;
 	r = amdgpu_vm_frag_ptes(&params, start, last + 1, addr, flags);
 	if (r)
 		goto error_free;
@@ -1565,6 +1607,7 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
 
 error_free:
 	amdgpu_job_free(job);
+	amdgpu_vm_invalidate_level(&vm->root);
 	return r;
 }
 
@@ -1573,7 +1616,6 @@ error_free:
  *
  * @adev: amdgpu_device pointer
  * @exclusive: fence we need to sync to
- * @gtt_flags: flags as they are used for GTT
  * @pages_addr: DMA addresses to use for mapping
  * @vm: requested vm
  * @mapping: mapped range and flags to use for the update
@@ -1587,7 +1629,6 @@ error_free:
  */
 static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
 				      struct dma_fence *exclusive,
-				      uint64_t gtt_flags,
 				      dma_addr_t *pages_addr,
 				      struct amdgpu_vm *vm,
 				      struct amdgpu_bo_va_mapping *mapping,
@@ -1642,11 +1683,7 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
 		}
 
 		if (pages_addr) {
-			if (flags == gtt_flags)
-				src = adev->gart.table_addr +
-					(addr >> AMDGPU_GPU_PAGE_SHIFT) * 8;
-			else
-				max_entries = min(max_entries, 16ull * 1024ull);
+			max_entries = min(max_entries, 16ull * 1024ull);
 			addr = 0;
 		} else if (flags & AMDGPU_PTE_VALID) {
 			addr += adev->vm_manager.vram_base_offset;
@@ -1687,50 +1724,45 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
 			struct amdgpu_bo_va *bo_va,
 			bool clear)
 {
-	struct amdgpu_vm *vm = bo_va->vm;
+	struct amdgpu_bo *bo = bo_va->base.bo;
+	struct amdgpu_vm *vm = bo_va->base.vm;
 	struct amdgpu_bo_va_mapping *mapping;
 	dma_addr_t *pages_addr = NULL;
-	uint64_t gtt_flags, flags;
 	struct ttm_mem_reg *mem;
 	struct drm_mm_node *nodes;
 	struct dma_fence *exclusive;
+	uint64_t flags;
 	int r;
 
-	if (clear || !bo_va->bo) {
+	if (clear || !bo_va->base.bo) {
 		mem = NULL;
 		nodes = NULL;
 		exclusive = NULL;
 	} else {
 		struct ttm_dma_tt *ttm;
 
-		mem = &bo_va->bo->tbo.mem;
+		mem = &bo_va->base.bo->tbo.mem;
 		nodes = mem->mm_node;
 		if (mem->mem_type == TTM_PL_TT) {
-			ttm = container_of(bo_va->bo->tbo.ttm, struct
-					   ttm_dma_tt, ttm);
+			ttm = container_of(bo_va->base.bo->tbo.ttm,
+					   struct ttm_dma_tt, ttm);
 			pages_addr = ttm->dma_address;
 		}
-		exclusive = reservation_object_get_excl(bo_va->bo->tbo.resv);
+		exclusive = reservation_object_get_excl(bo->tbo.resv);
 	}
 
-	if (bo_va->bo) {
-		flags = amdgpu_ttm_tt_pte_flags(adev, bo_va->bo->tbo.ttm, mem);
-		gtt_flags = (amdgpu_ttm_is_bound(bo_va->bo->tbo.ttm) &&
-			adev == amdgpu_ttm_adev(bo_va->bo->tbo.bdev)) ?
-			flags : 0;
-	} else {
+	if (bo)
+		flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem);
+	else
 		flags = 0x0;
-		gtt_flags = ~0x0;
-	}
 
 	spin_lock(&vm->status_lock);
-	if (!list_empty(&bo_va->vm_status))
+	if (!list_empty(&bo_va->base.vm_status))
 		list_splice_init(&bo_va->valids, &bo_va->invalids);
 	spin_unlock(&vm->status_lock);
 
 	list_for_each_entry(mapping, &bo_va->invalids, list) {
-		r = amdgpu_vm_bo_split_mapping(adev, exclusive,
-					       gtt_flags, pages_addr, vm,
+		r = amdgpu_vm_bo_split_mapping(adev, exclusive, pages_addr, vm,
 					       mapping, flags, nodes,
 					       &bo_va->last_pt_update);
 		if (r)
@@ -1747,11 +1779,17 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
 
 	spin_lock(&vm->status_lock);
 	list_splice_init(&bo_va->invalids, &bo_va->valids);
-	list_del_init(&bo_va->vm_status);
+	list_del_init(&bo_va->base.vm_status);
 	if (clear)
-		list_add(&bo_va->vm_status, &vm->cleared);
+		list_add(&bo_va->base.vm_status, &vm->cleared);
 	spin_unlock(&vm->status_lock);
 
+	if (vm->use_cpu_for_update) {
+		/* Flush HDP */
+		mb();
+		amdgpu_gart_flush_gpu_tlb(adev, 0);
+	}
+
 	return 0;
 }
 
@@ -1905,15 +1943,19 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
 	struct amdgpu_bo_va_mapping *mapping;
 	struct dma_fence *f = NULL;
 	int r;
+	uint64_t init_pte_value = 0;
 
 	while (!list_empty(&vm->freed)) {
 		mapping = list_first_entry(&vm->freed,
 			struct amdgpu_bo_va_mapping, list);
 		list_del(&mapping->list);
 
+		if (vm->pte_support_ats)
+			init_pte_value = AMDGPU_PTE_SYSTEM;
+
 		r = amdgpu_vm_bo_update_mapping(adev, NULL, 0, NULL, vm,
 						mapping->start, mapping->last,
-						0, 0, &f);
+						init_pte_value, 0, &f);
 		amdgpu_vm_free_mapping(adev, vm, mapping, f);
 		if (r) {
 			dma_fence_put(f);
@@ -1933,26 +1975,26 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
 }
 
 /**
- * amdgpu_vm_clear_invalids - clear invalidated BOs in the PT
+ * amdgpu_vm_clear_moved - clear moved BOs in the PT
  *
  * @adev: amdgpu_device pointer
  * @vm: requested vm
  *
- * Make sure all invalidated BOs are cleared in the PT.
+ * Make sure all moved BOs are cleared in the PT.
  * Returns 0 for success.
  *
  * PTs have to be reserved and mutex must be locked!
  */
-int amdgpu_vm_clear_invalids(struct amdgpu_device *adev,
-			     struct amdgpu_vm *vm, struct amdgpu_sync *sync)
+int amdgpu_vm_clear_moved(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+			    struct amdgpu_sync *sync)
 {
 	struct amdgpu_bo_va *bo_va = NULL;
 	int r = 0;
 
 	spin_lock(&vm->status_lock);
-	while (!list_empty(&vm->invalidated)) {
-		bo_va = list_first_entry(&vm->invalidated,
-			struct amdgpu_bo_va, vm_status);
+	while (!list_empty(&vm->moved)) {
+		bo_va = list_first_entry(&vm->moved,
+			struct amdgpu_bo_va, base.vm_status);
 		spin_unlock(&vm->status_lock);
 
 		r = amdgpu_vm_bo_update(adev, bo_va, true);
@@ -1992,16 +2034,17 @@ struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
 	if (bo_va == NULL) {
 		return NULL;
 	}
-	bo_va->vm = vm;
-	bo_va->bo = bo;
+	bo_va->base.vm = vm;
+	bo_va->base.bo = bo;
+	INIT_LIST_HEAD(&bo_va->base.bo_list);
+	INIT_LIST_HEAD(&bo_va->base.vm_status);
+
 	bo_va->ref_count = 1;
-	INIT_LIST_HEAD(&bo_va->bo_list);
 	INIT_LIST_HEAD(&bo_va->valids);
 	INIT_LIST_HEAD(&bo_va->invalids);
-	INIT_LIST_HEAD(&bo_va->vm_status);
 
 	if (bo)
-		list_add_tail(&bo_va->bo_list, &bo->va);
+		list_add_tail(&bo_va->base.bo_list, &bo->va);
 
 	return bo_va;
 }
@@ -2026,7 +2069,8 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
 		     uint64_t size, uint64_t flags)
 {
 	struct amdgpu_bo_va_mapping *mapping, *tmp;
-	struct amdgpu_vm *vm = bo_va->vm;
+	struct amdgpu_bo *bo = bo_va->base.bo;
+	struct amdgpu_vm *vm = bo_va->base.vm;
 	uint64_t eaddr;
 
 	/* validate the parameters */
@@ -2037,7 +2081,7 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
 	/* make sure object fit at this offset */
 	eaddr = saddr + size - 1;
 	if (saddr >= eaddr ||
-	    (bo_va->bo && offset + size > amdgpu_bo_size(bo_va->bo)))
+	    (bo && offset + size > amdgpu_bo_size(bo)))
 		return -EINVAL;
 
 	saddr /= AMDGPU_GPU_PAGE_SIZE;
@@ -2047,7 +2091,7 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev,
 	if (tmp) {
 		/* bo and tmp overlap, invalid addr */
 		dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
-			"0x%010Lx-0x%010Lx\n", bo_va->bo, saddr, eaddr,
+			"0x%010Lx-0x%010Lx\n", bo, saddr, eaddr,
 			tmp->start, tmp->last + 1);
 		return -EINVAL;
 	}
@@ -2092,7 +2136,8 @@ int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
 			     uint64_t size, uint64_t flags)
 {
 	struct amdgpu_bo_va_mapping *mapping;
-	struct amdgpu_vm *vm = bo_va->vm;
+	struct amdgpu_bo *bo = bo_va->base.bo;
+	struct amdgpu_vm *vm = bo_va->base.vm;
 	uint64_t eaddr;
 	int r;
 
@@ -2104,7 +2149,7 @@ int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
 	/* make sure object fit at this offset */
 	eaddr = saddr + size - 1;
 	if (saddr >= eaddr ||
-	    (bo_va->bo && offset + size > amdgpu_bo_size(bo_va->bo)))
+	    (bo && offset + size > amdgpu_bo_size(bo)))
 		return -EINVAL;
 
 	/* Allocate all the needed memory */
@@ -2112,7 +2157,7 @@ int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
 	if (!mapping)
 		return -ENOMEM;
 
-	r = amdgpu_vm_bo_clear_mappings(adev, bo_va->vm, saddr, size);
+	r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size);
 	if (r) {
 		kfree(mapping);
 		return r;
@@ -2152,7 +2197,7 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
 		       uint64_t saddr)
 {
 	struct amdgpu_bo_va_mapping *mapping;
-	struct amdgpu_vm *vm = bo_va->vm;
+	struct amdgpu_vm *vm = bo_va->base.vm;
 	bool valid = true;
 
 	saddr /= AMDGPU_GPU_PAGE_SIZE;
@@ -2300,12 +2345,12 @@ void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
 		      struct amdgpu_bo_va *bo_va)
 {
 	struct amdgpu_bo_va_mapping *mapping, *next;
-	struct amdgpu_vm *vm = bo_va->vm;
+	struct amdgpu_vm *vm = bo_va->base.vm;
 
-	list_del(&bo_va->bo_list);
+	list_del(&bo_va->base.bo_list);
 
 	spin_lock(&vm->status_lock);
-	list_del(&bo_va->vm_status);
+	list_del(&bo_va->base.vm_status);
 	spin_unlock(&vm->status_lock);
 
 	list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
@@ -2337,13 +2382,14 @@ void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
 			     struct amdgpu_bo *bo)
 {
-	struct amdgpu_bo_va *bo_va;
+	struct amdgpu_vm_bo_base *bo_base;
 
-	list_for_each_entry(bo_va, &bo->va, bo_list) {
-		spin_lock(&bo_va->vm->status_lock);
-		if (list_empty(&bo_va->vm_status))
-			list_add(&bo_va->vm_status, &bo_va->vm->invalidated);
-		spin_unlock(&bo_va->vm->status_lock);
+	list_for_each_entry(bo_base, &bo->va, bo_list) {
+		spin_lock(&bo_base->vm->status_lock);
+		if (list_empty(&bo_base->vm_status))
+			list_add(&bo_base->vm_status,
+				 &bo_base->vm->moved);
+		spin_unlock(&bo_base->vm->status_lock);
 	}
 }
 
@@ -2361,12 +2407,26 @@ static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
 }
 
 /**
- * amdgpu_vm_adjust_size - adjust vm size and block size
+ * amdgpu_vm_set_fragment_size - adjust fragment size in PTE
+ *
+ * @adev: amdgpu_device pointer
+ * @fragment_size_default: the default fragment size if it's set auto
+ */
+void amdgpu_vm_set_fragment_size(struct amdgpu_device *adev, uint32_t fragment_size_default)
+{
+	if (amdgpu_vm_fragment_size == -1)
+		adev->vm_manager.fragment_size = fragment_size_default;
+	else
+		adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;
+}
+
+/**
+ * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size
  *
  * @adev: amdgpu_device pointer
  * @vm_size: the default vm size if it's set auto
  */
-void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size)
+void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size, uint32_t fragment_size_default)
 {
 	/* adjust vm size firstly */
 	if (amdgpu_vm_size == -1)
@@ -2381,8 +2441,11 @@ void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size)
 	else
 		adev->vm_manager.block_size = amdgpu_vm_block_size;
 
-	DRM_INFO("vm size is %llu GB, block size is %u-bit\n",
-		adev->vm_manager.vm_size, adev->vm_manager.block_size);
+	amdgpu_vm_set_fragment_size(adev, fragment_size_default);
+
+	DRM_INFO("vm size is %llu GB, block size is %u-bit, fragment size is %u-bit\n",
+		adev->vm_manager.vm_size, adev->vm_manager.block_size,
+		adev->vm_manager.fragment_size);
 }
 
 /**
@@ -2404,13 +2467,14 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	struct amd_sched_rq *rq;
 	int r, i;
 	u64 flags;
+	uint64_t init_pde_value = 0;
 
-	vm->va = RB_ROOT;
+	vm->va = RB_ROOT_CACHED;
 	vm->client_id = atomic64_inc_return(&adev->vm_manager.client_counter);
 	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
 		vm->reserved_vmid[i] = NULL;
 	spin_lock_init(&vm->status_lock);
-	INIT_LIST_HEAD(&vm->invalidated);
+	INIT_LIST_HEAD(&vm->moved);
 	INIT_LIST_HEAD(&vm->cleared);
 	INIT_LIST_HEAD(&vm->freed);
 
@@ -2425,10 +2489,17 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	if (r)
 		return r;
 
-	if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE)
+	vm->pte_support_ats = false;
+
+	if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE) {
 		vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
 						AMDGPU_VM_USE_CPU_FOR_COMPUTE);
-	else
+
+		if (adev->asic_type == CHIP_RAVEN) {
+			vm->pte_support_ats = true;
+			init_pde_value = AMDGPU_PTE_SYSTEM | AMDGPU_PDE_PTE;
+		}
+	} else
 		vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
 						AMDGPU_VM_USE_CPU_FOR_GFX);
 	DRM_DEBUG_DRIVER("VM update mode is %s\n",
@@ -2448,7 +2519,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	r = amdgpu_bo_create(adev, amdgpu_vm_bo_size(adev, 0), align, true,
 			     AMDGPU_GEM_DOMAIN_VRAM,
 			     flags,
-			     NULL, NULL, &vm->root.bo);
+			     NULL, NULL, init_pde_value, &vm->root.bo);
 	if (r)
 		goto error_free_sched_entity;
 
@@ -2457,6 +2528,13 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		goto error_free_root;
 
 	vm->last_eviction_counter = atomic64_read(&adev->num_evictions);
+
+	if (vm->use_cpu_for_update) {
+		r = amdgpu_bo_kmap(vm->root.bo, NULL);
+		if (r)
+			goto error_free_root;
+	}
+
 	amdgpu_bo_unreserve(vm->root.bo);
 
 	return 0;
@@ -2512,10 +2590,11 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 
 	amd_sched_entity_fini(vm->entity.sched, &vm->entity);
 
-	if (!RB_EMPTY_ROOT(&vm->va)) {
+	if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
 		dev_err(adev->dev, "still active bo inside vm\n");
 	}
-	rbtree_postorder_for_each_entry_safe(mapping, tmp, &vm->va, rb) {
+	rbtree_postorder_for_each_entry_safe(mapping, tmp,
+					     &vm->va.rb_root, rb) {
 		list_del(&mapping->list);
 		amdgpu_vm_it_remove(mapping, &vm->va);
 		kfree(mapping);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 936f158bc5ec..6716355403ec 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -50,9 +50,6 @@ struct amdgpu_bo_list_entry;
 /* PTBs (Page Table Blocks) need to be aligned to 32K */
 #define AMDGPU_VM_PTB_ALIGN_SIZE   32768
 
-/* LOG2 number of continuous pages for the fragment field */
-#define AMDGPU_LOG2_PAGES_PER_FRAG 4
-
 #define AMDGPU_PTE_VALID	(1ULL << 0)
 #define AMDGPU_PTE_SYSTEM	(1ULL << 1)
 #define AMDGPU_PTE_SNOOPED	(1ULL << 2)
@@ -68,6 +65,9 @@ struct amdgpu_bo_list_entry;
 /* TILED for VEGA10, reserved for older ASICs  */
 #define AMDGPU_PTE_PRT		(1ULL << 51)
 
+/* PDE is handled as PTE for VEGA10 */
+#define AMDGPU_PDE_PTE		(1ULL << 54)
+
 /* VEGA10 only */
 #define AMDGPU_PTE_MTYPE(a)    ((uint64_t)a << 57)
 #define AMDGPU_PTE_MTYPE_MASK	AMDGPU_PTE_MTYPE(3ULL)
@@ -94,6 +94,18 @@ struct amdgpu_bo_list_entry;
 #define AMDGPU_VM_USE_CPU_FOR_GFX (1 << 0)
 #define AMDGPU_VM_USE_CPU_FOR_COMPUTE (1 << 1)
 
+/* base structure for tracking BO usage in a VM */
+struct amdgpu_vm_bo_base {
+	/* constant after initialization */
+	struct amdgpu_vm		*vm;
+	struct amdgpu_bo		*bo;
+
+	/* protected by bo being reserved */
+	struct list_head		bo_list;
+
+	/* protected by spinlock */
+	struct list_head		vm_status;
+};
 
 struct amdgpu_vm_pt {
 	struct amdgpu_bo	*bo;
@@ -106,13 +118,13 @@ struct amdgpu_vm_pt {
 
 struct amdgpu_vm {
 	/* tree of virtual addresses mapped */
-	struct rb_root		va;
+	struct rb_root_cached	va;
 
 	/* protecting invalidated */
 	spinlock_t		status_lock;
 
 	/* BOs moved, but not yet updated in the PT */
-	struct list_head	invalidated;
+	struct list_head	moved;
 
 	/* BOs cleared in the PT because of a move */
 	struct list_head	cleared;
@@ -135,11 +147,12 @@ struct amdgpu_vm {
 	u64                     client_id;
 	/* dedicated to vm */
 	struct amdgpu_vm_id	*reserved_vmid[AMDGPU_MAX_VMHUBS];
-	/* each VM will map on CSA */
-	struct amdgpu_bo_va *csa_bo_va;
 
 	/* Flag to indicate if VM tables are updated by CPU or GPU (SDMA) */
 	bool                    use_cpu_for_update;
+
+	/* Flag to indicate ATS support from PTE for GFX9 */
+	bool			pte_support_ats;
 };
 
 struct amdgpu_vm_id {
@@ -182,6 +195,7 @@ struct amdgpu_vm_manager {
 	uint32_t				num_level;
 	uint64_t				vm_size;
 	uint32_t				block_size;
+	uint32_t				fragment_size;
 	/* vram base address for page table entry  */
 	u64					vram_base_offset;
 	/* vm pte handling */
@@ -214,15 +228,13 @@ void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
 int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			      int (*callback)(void *p, struct amdgpu_bo *bo),
 			      void *param);
-void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev,
-				  struct amdgpu_vm *vm);
 int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
 			struct amdgpu_vm *vm,
 			uint64_t saddr, uint64_t size);
 int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
 		      struct amdgpu_sync *sync, struct dma_fence *fence,
 		      struct amdgpu_job *job);
-int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job);
+int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync);
 void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vmhub,
 			unsigned vmid);
 void amdgpu_vm_reset_all_ids(struct amdgpu_device *adev);
@@ -231,8 +243,8 @@ int amdgpu_vm_update_directories(struct amdgpu_device *adev,
 int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
 			  struct amdgpu_vm *vm,
 			  struct dma_fence **fence);
-int amdgpu_vm_clear_invalids(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-			     struct amdgpu_sync *sync);
+int amdgpu_vm_clear_moved(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+			  struct amdgpu_sync *sync);
 int amdgpu_vm_bo_update(struct amdgpu_device *adev,
 			struct amdgpu_bo_va *bo_va,
 			bool clear);
@@ -259,7 +271,10 @@ int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
 				uint64_t saddr, uint64_t size);
 void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
 		      struct amdgpu_bo_va *bo_va);
-void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size);
+void amdgpu_vm_set_fragment_size(struct amdgpu_device *adev,
+				uint32_t fragment_size_default);
+void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size,
+				uint32_t fragment_size_default);
 int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
 bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
 				  struct amdgpu_job *job);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
index a2c59a08b2bd..26e900627971 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
@@ -28,6 +28,8 @@
 struct amdgpu_vram_mgr {
 	struct drm_mm mm;
 	spinlock_t lock;
+	atomic64_t usage;
+	atomic64_t vis_usage;
 };
 
 /**
@@ -79,6 +81,27 @@ static int amdgpu_vram_mgr_fini(struct ttm_mem_type_manager *man)
 }
 
 /**
+ * amdgpu_vram_mgr_vis_size - Calculate visible node size
+ *
+ * @adev: amdgpu device structure
+ * @node: MM node structure
+ *
+ * Calculate how many bytes of the MM node are inside visible VRAM
+ */
+static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev,
+				    struct drm_mm_node *node)
+{
+	uint64_t start = node->start << PAGE_SHIFT;
+	uint64_t end = (node->size + node->start) << PAGE_SHIFT;
+
+	if (start >= adev->mc.visible_vram_size)
+		return 0;
+
+	return (end > adev->mc.visible_vram_size ?
+		adev->mc.visible_vram_size : end) - start;
+}
+
+/**
  * amdgpu_vram_mgr_new - allocate new ranges
  *
  * @man: TTM memory type manager
@@ -93,11 +116,13 @@ static int amdgpu_vram_mgr_new(struct ttm_mem_type_manager *man,
 			       const struct ttm_place *place,
 			       struct ttm_mem_reg *mem)
 {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
 	struct amdgpu_vram_mgr *mgr = man->priv;
 	struct drm_mm *mm = &mgr->mm;
 	struct drm_mm_node *nodes;
 	enum drm_mm_insert_mode mode;
 	unsigned long lpfn, num_nodes, pages_per_node, pages_left;
+	uint64_t usage = 0, vis_usage = 0;
 	unsigned i;
 	int r;
 
@@ -142,6 +167,9 @@ static int amdgpu_vram_mgr_new(struct ttm_mem_type_manager *man,
 		if (unlikely(r))
 			goto error;
 
+		usage += nodes[i].size << PAGE_SHIFT;
+		vis_usage += amdgpu_vram_mgr_vis_size(adev, &nodes[i]);
+
 		/* Calculate a virtual BO start address to easily check if
 		 * everything is CPU accessible.
 		 */
@@ -155,6 +183,9 @@ static int amdgpu_vram_mgr_new(struct ttm_mem_type_manager *man,
 	}
 	spin_unlock(&mgr->lock);
 
+	atomic64_add(usage, &mgr->usage);
+	atomic64_add(vis_usage, &mgr->vis_usage);
+
 	mem->mm_node = nodes;
 
 	return 0;
@@ -181,8 +212,10 @@ error:
 static void amdgpu_vram_mgr_del(struct ttm_mem_type_manager *man,
 				struct ttm_mem_reg *mem)
 {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
 	struct amdgpu_vram_mgr *mgr = man->priv;
 	struct drm_mm_node *nodes = mem->mm_node;
+	uint64_t usage = 0, vis_usage = 0;
 	unsigned pages = mem->num_pages;
 
 	if (!mem->mm_node)
@@ -192,31 +225,67 @@ static void amdgpu_vram_mgr_del(struct ttm_mem_type_manager *man,
 	while (pages) {
 		pages -= nodes->size;
 		drm_mm_remove_node(nodes);
+		usage += nodes->size << PAGE_SHIFT;
+		vis_usage += amdgpu_vram_mgr_vis_size(adev, nodes);
 		++nodes;
 	}
 	spin_unlock(&mgr->lock);
 
+	atomic64_sub(usage, &mgr->usage);
+	atomic64_sub(vis_usage, &mgr->vis_usage);
+
 	kfree(mem->mm_node);
 	mem->mm_node = NULL;
 }
 
 /**
+ * amdgpu_vram_mgr_usage - how many bytes are used in this domain
+ *
+ * @man: TTM memory type manager
+ *
+ * Returns how many bytes are used in this domain.
+ */
+uint64_t amdgpu_vram_mgr_usage(struct ttm_mem_type_manager *man)
+{
+	struct amdgpu_vram_mgr *mgr = man->priv;
+
+	return atomic64_read(&mgr->usage);
+}
+
+/**
+ * amdgpu_vram_mgr_vis_usage - how many bytes are used in the visible part
+ *
+ * @man: TTM memory type manager
+ *
+ * Returns how many bytes are used in the visible part of VRAM
+ */
+uint64_t amdgpu_vram_mgr_vis_usage(struct ttm_mem_type_manager *man)
+{
+	struct amdgpu_vram_mgr *mgr = man->priv;
+
+	return atomic64_read(&mgr->vis_usage);
+}
+
+/**
  * amdgpu_vram_mgr_debug - dump VRAM table
  *
  * @man: TTM memory type manager
- * @prefix: text prefix
+ * @printer: DRM printer to use
  *
  * Dump the table content using printk.
  */
 static void amdgpu_vram_mgr_debug(struct ttm_mem_type_manager *man,
-				  const char *prefix)
+				  struct drm_printer *printer)
 {
 	struct amdgpu_vram_mgr *mgr = man->priv;
-	struct drm_printer p = drm_debug_printer(prefix);
 
 	spin_lock(&mgr->lock);
-	drm_mm_print(&mgr->mm, &p);
+	drm_mm_print(&mgr->mm, printer);
 	spin_unlock(&mgr->lock);
+
+	drm_printf(printer, "man size:%llu pages, ram usage:%lluMB, vis usage:%lluMB\n",
+		   man->size, amdgpu_vram_mgr_usage(man) >> 20,
+		   amdgpu_vram_mgr_vis_usage(man) >> 20);
 }
 
 const struct ttm_mem_type_manager_func amdgpu_vram_mgr_func = {
diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c
index 37a499ab30eb..567c4a5cf90c 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik.c
@@ -1824,21 +1824,14 @@ static int cik_common_suspend(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-	amdgpu_amdkfd_suspend(adev);
-
 	return cik_common_hw_fini(adev);
 }
 
 static int cik_common_resume(void *handle)
 {
-	int r;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-	r = cik_common_hw_init(adev);
-	if (r)
-		return r;
-
-	return amdgpu_amdkfd_resume(adev);
+	return cik_common_hw_init(adev);
 }
 
 static bool cik_common_is_idle(void *handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
index c216e16826c9..f508f4d01e4a 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
@@ -342,6 +342,63 @@ static void cik_sdma_rlc_stop(struct amdgpu_device *adev)
 }
 
 /**
+ * cik_ctx_switch_enable - stop the async dma engines context switch
+ *
+ * @adev: amdgpu_device pointer
+ * @enable: enable/disable the DMA MEs context switch.
+ *
+ * Halt or unhalt the async dma engines context switch (VI).
+ */
+static void cik_ctx_switch_enable(struct amdgpu_device *adev, bool enable)
+{
+	u32 f32_cntl, phase_quantum = 0;
+	int i;
+
+	if (amdgpu_sdma_phase_quantum) {
+		unsigned value = amdgpu_sdma_phase_quantum;
+		unsigned unit = 0;
+
+		while (value > (SDMA0_PHASE0_QUANTUM__VALUE_MASK >>
+				SDMA0_PHASE0_QUANTUM__VALUE__SHIFT)) {
+			value = (value + 1) >> 1;
+			unit++;
+		}
+		if (unit > (SDMA0_PHASE0_QUANTUM__UNIT_MASK >>
+			    SDMA0_PHASE0_QUANTUM__UNIT__SHIFT)) {
+			value = (SDMA0_PHASE0_QUANTUM__VALUE_MASK >>
+				 SDMA0_PHASE0_QUANTUM__VALUE__SHIFT);
+			unit = (SDMA0_PHASE0_QUANTUM__UNIT_MASK >>
+				SDMA0_PHASE0_QUANTUM__UNIT__SHIFT);
+			WARN_ONCE(1,
+			"clamping sdma_phase_quantum to %uK clock cycles\n",
+				  value << unit);
+		}
+		phase_quantum =
+			value << SDMA0_PHASE0_QUANTUM__VALUE__SHIFT |
+			unit  << SDMA0_PHASE0_QUANTUM__UNIT__SHIFT;
+	}
+
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		f32_cntl = RREG32(mmSDMA0_CNTL + sdma_offsets[i]);
+		if (enable) {
+			f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL,
+					AUTO_CTXSW_ENABLE, 1);
+			if (amdgpu_sdma_phase_quantum) {
+				WREG32(mmSDMA0_PHASE0_QUANTUM + sdma_offsets[i],
+				       phase_quantum);
+				WREG32(mmSDMA0_PHASE1_QUANTUM + sdma_offsets[i],
+				       phase_quantum);
+			}
+		} else {
+			f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL,
+					AUTO_CTXSW_ENABLE, 0);
+		}
+
+		WREG32(mmSDMA0_CNTL + sdma_offsets[i], f32_cntl);
+	}
+}
+
+/**
  * cik_sdma_enable - stop the async dma engines
  *
  * @adev: amdgpu_device pointer
@@ -537,6 +594,8 @@ static int cik_sdma_start(struct amdgpu_device *adev)
 
 	/* halt the engine before programing */
 	cik_sdma_enable(adev, false);
+	/* enable sdma ring preemption */
+	cik_ctx_switch_enable(adev, true);
 
 	/* start the gfx rings and rlc compute queues */
 	r = cik_sdma_gfx_resume(adev);
@@ -984,6 +1043,7 @@ static int cik_sdma_hw_fini(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+	cik_ctx_switch_enable(adev, false);
 	cik_sdma_enable(adev, false);
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
index 9f78c03a2e31..4e519dc42916 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
@@ -484,134 +484,6 @@ static bool dce_v10_0_is_display_hung(struct amdgpu_device *adev)
 	return true;
 }
 
-static void dce_v10_0_stop_mc_access(struct amdgpu_device *adev,
-				     struct amdgpu_mode_mc_save *save)
-{
-	u32 crtc_enabled, tmp;
-	int i;
-
-	save->vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
-	save->vga_hdp_control = RREG32(mmVGA_HDP_CONTROL);
-
-	/* disable VGA render */
-	tmp = RREG32(mmVGA_RENDER_CONTROL);
-	tmp = REG_SET_FIELD(tmp, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
-	WREG32(mmVGA_RENDER_CONTROL, tmp);
-
-	/* blank the display controllers */
-	for (i = 0; i < adev->mode_info.num_crtc; i++) {
-		crtc_enabled = REG_GET_FIELD(RREG32(mmCRTC_CONTROL + crtc_offsets[i]),
-					     CRTC_CONTROL, CRTC_MASTER_EN);
-		if (crtc_enabled) {
-#if 0
-			u32 frame_count;
-			int j;
-
-			save->crtc_enabled[i] = true;
-			tmp = RREG32(mmCRTC_BLANK_CONTROL + crtc_offsets[i]);
-			if (REG_GET_FIELD(tmp, CRTC_BLANK_CONTROL, CRTC_BLANK_DATA_EN) == 0) {
-				amdgpu_display_vblank_wait(adev, i);
-				WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 1);
-				tmp = REG_SET_FIELD(tmp, CRTC_BLANK_CONTROL, CRTC_BLANK_DATA_EN, 1);
-				WREG32(mmCRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
-				WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 0);
-			}
-			/* wait for the next frame */
-			frame_count = amdgpu_display_vblank_get_counter(adev, i);
-			for (j = 0; j < adev->usec_timeout; j++) {
-				if (amdgpu_display_vblank_get_counter(adev, i) != frame_count)
-					break;
-				udelay(1);
-			}
-			tmp = RREG32(mmGRPH_UPDATE + crtc_offsets[i]);
-			if (REG_GET_FIELD(tmp, GRPH_UPDATE, GRPH_UPDATE_LOCK) == 0) {
-				tmp = REG_SET_FIELD(tmp, GRPH_UPDATE, GRPH_UPDATE_LOCK, 1);
-				WREG32(mmGRPH_UPDATE + crtc_offsets[i], tmp);
-			}
-			tmp = RREG32(mmMASTER_UPDATE_LOCK + crtc_offsets[i]);
-			if (REG_GET_FIELD(tmp, MASTER_UPDATE_LOCK, MASTER_UPDATE_LOCK) == 0) {
-				tmp = REG_SET_FIELD(tmp, MASTER_UPDATE_LOCK, MASTER_UPDATE_LOCK, 1);
-				WREG32(mmMASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
-			}
-#else
-			/* XXX this is a hack to avoid strange behavior with EFI on certain systems */
-			WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 1);
-			tmp = RREG32(mmCRTC_CONTROL + crtc_offsets[i]);
-			tmp = REG_SET_FIELD(tmp, CRTC_CONTROL, CRTC_MASTER_EN, 0);
-			WREG32(mmCRTC_CONTROL + crtc_offsets[i], tmp);
-			WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 0);
-			save->crtc_enabled[i] = false;
-			/* ***** */
-#endif
-		} else {
-			save->crtc_enabled[i] = false;
-		}
-	}
-}
-
-static void dce_v10_0_resume_mc_access(struct amdgpu_device *adev,
-				       struct amdgpu_mode_mc_save *save)
-{
-	u32 tmp, frame_count;
-	int i, j;
-
-	/* update crtc base addresses */
-	for (i = 0; i < adev->mode_info.num_crtc; i++) {
-		WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
-		       upper_32_bits(adev->mc.vram_start));
-		WREG32(mmGRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
-		       upper_32_bits(adev->mc.vram_start));
-		WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
-		       (u32)adev->mc.vram_start);
-		WREG32(mmGRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
-		       (u32)adev->mc.vram_start);
-
-		if (save->crtc_enabled[i]) {
-			tmp = RREG32(mmMASTER_UPDATE_MODE + crtc_offsets[i]);
-			if (REG_GET_FIELD(tmp, MASTER_UPDATE_MODE, MASTER_UPDATE_MODE) != 0) {
-				tmp = REG_SET_FIELD(tmp, MASTER_UPDATE_MODE, MASTER_UPDATE_MODE, 0);
-				WREG32(mmMASTER_UPDATE_MODE + crtc_offsets[i], tmp);
-			}
-			tmp = RREG32(mmGRPH_UPDATE + crtc_offsets[i]);
-			if (REG_GET_FIELD(tmp, GRPH_UPDATE, GRPH_UPDATE_LOCK)) {
-				tmp = REG_SET_FIELD(tmp, GRPH_UPDATE, GRPH_UPDATE_LOCK, 0);
-				WREG32(mmGRPH_UPDATE + crtc_offsets[i], tmp);
-			}
-			tmp = RREG32(mmMASTER_UPDATE_LOCK + crtc_offsets[i]);
-			if (REG_GET_FIELD(tmp, MASTER_UPDATE_LOCK, MASTER_UPDATE_LOCK)) {
-				tmp = REG_SET_FIELD(tmp, MASTER_UPDATE_LOCK, MASTER_UPDATE_LOCK, 0);
-				WREG32(mmMASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
-			}
-			for (j = 0; j < adev->usec_timeout; j++) {
-				tmp = RREG32(mmGRPH_UPDATE + crtc_offsets[i]);
-				if (REG_GET_FIELD(tmp, GRPH_UPDATE, GRPH_SURFACE_UPDATE_PENDING) == 0)
-					break;
-				udelay(1);
-			}
-			tmp = RREG32(mmCRTC_BLANK_CONTROL + crtc_offsets[i]);
-			tmp = REG_SET_FIELD(tmp, CRTC_BLANK_CONTROL, CRTC_BLANK_DATA_EN, 0);
-			WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 1);
-			WREG32(mmCRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
-			WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 0);
-			/* wait for the next frame */
-			frame_count = amdgpu_display_vblank_get_counter(adev, i);
-			for (j = 0; j < adev->usec_timeout; j++) {
-				if (amdgpu_display_vblank_get_counter(adev, i) != frame_count)
-					break;
-				udelay(1);
-			}
-		}
-	}
-
-	WREG32(mmVGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(adev->mc.vram_start));
-	WREG32(mmVGA_MEMORY_BASE_ADDRESS, lower_32_bits(adev->mc.vram_start));
-
-	/* Unlock vga access */
-	WREG32(mmVGA_HDP_CONTROL, save->vga_hdp_control);
-	mdelay(1);
-	WREG32(mmVGA_RENDER_CONTROL, save->vga_render_control);
-}
-
 static void dce_v10_0_set_vga_render_state(struct amdgpu_device *adev,
 					   bool render)
 {
@@ -1867,7 +1739,7 @@ static void dce_v10_0_afmt_setmode(struct drm_encoder *encoder,
 	dce_v10_0_audio_write_sad_regs(encoder);
 	dce_v10_0_audio_write_latency_fields(encoder, mode);
 
-	err = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode);
+	err = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode, false);
 	if (err < 0) {
 		DRM_ERROR("failed to setup AVI infoframe: %zd\n", err);
 		return;
@@ -2267,6 +2139,7 @@ static void dce_v10_0_crtc_load_lut(struct drm_crtc *crtc)
 	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct amdgpu_device *adev = dev->dev_private;
+	u16 *r, *g, *b;
 	int i;
 	u32 tmp;
 
@@ -2304,11 +2177,14 @@ static void dce_v10_0_crtc_load_lut(struct drm_crtc *crtc)
 	WREG32(mmDC_LUT_WRITE_EN_MASK + amdgpu_crtc->crtc_offset, 0x00000007);
 
 	WREG32(mmDC_LUT_RW_INDEX + amdgpu_crtc->crtc_offset, 0);
+	r = crtc->gamma_store;
+	g = r + crtc->gamma_size;
+	b = g + crtc->gamma_size;
 	for (i = 0; i < 256; i++) {
 		WREG32(mmDC_LUT_30_COLOR + amdgpu_crtc->crtc_offset,
-		       (amdgpu_crtc->lut_r[i] << 20) |
-		       (amdgpu_crtc->lut_g[i] << 10) |
-		       (amdgpu_crtc->lut_b[i] << 0));
+		       ((*r++ & 0xffc0) << 14) |
+		       ((*g++ & 0xffc0) << 4) |
+		       (*b++ >> 6));
 	}
 
 	tmp = RREG32(mmDEGAMMA_CONTROL + amdgpu_crtc->crtc_offset);
@@ -2555,7 +2431,7 @@ static int dce_v10_0_crtc_cursor_set2(struct drm_crtc *crtc,
 	aobj = gem_to_amdgpu_bo(obj);
 	ret = amdgpu_bo_reserve(aobj, false);
 	if (ret != 0) {
-		drm_gem_object_unreference_unlocked(obj);
+		drm_gem_object_put_unlocked(obj);
 		return ret;
 	}
 
@@ -2563,7 +2439,7 @@ static int dce_v10_0_crtc_cursor_set2(struct drm_crtc *crtc,
 	amdgpu_bo_unreserve(aobj);
 	if (ret) {
 		DRM_ERROR("Failed to pin new cursor BO (%d)\n", ret);
-		drm_gem_object_unreference_unlocked(obj);
+		drm_gem_object_put_unlocked(obj);
 		return ret;
 	}
 
@@ -2597,7 +2473,7 @@ unpin:
 			amdgpu_bo_unpin(aobj);
 			amdgpu_bo_unreserve(aobj);
 		}
-		drm_gem_object_unreference_unlocked(amdgpu_crtc->cursor_bo);
+		drm_gem_object_put_unlocked(amdgpu_crtc->cursor_bo);
 	}
 
 	amdgpu_crtc->cursor_bo = obj;
@@ -2624,15 +2500,6 @@ static int dce_v10_0_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
 				    u16 *blue, uint32_t size,
 				    struct drm_modeset_acquire_ctx *ctx)
 {
-	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
-	int i;
-
-	/* userspace palettes are always correct as is */
-	for (i = 0; i < size; i++) {
-		amdgpu_crtc->lut_r[i] = red[i] >> 6;
-		amdgpu_crtc->lut_g[i] = green[i] >> 6;
-		amdgpu_crtc->lut_b[i] = blue[i] >> 6;
-	}
 	dce_v10_0_crtc_load_lut(crtc);
 
 	return 0;
@@ -2844,14 +2711,12 @@ static const struct drm_crtc_helper_funcs dce_v10_0_crtc_helper_funcs = {
 	.mode_set_base_atomic = dce_v10_0_crtc_set_base_atomic,
 	.prepare = dce_v10_0_crtc_prepare,
 	.commit = dce_v10_0_crtc_commit,
-	.load_lut = dce_v10_0_crtc_load_lut,
 	.disable = dce_v10_0_crtc_disable,
 };
 
 static int dce_v10_0_crtc_init(struct amdgpu_device *adev, int index)
 {
 	struct amdgpu_crtc *amdgpu_crtc;
-	int i;
 
 	amdgpu_crtc = kzalloc(sizeof(struct amdgpu_crtc) +
 			      (AMDGPUFB_CONN_LIMIT * sizeof(struct drm_connector *)), GFP_KERNEL);
@@ -2869,12 +2734,6 @@ static int dce_v10_0_crtc_init(struct amdgpu_device *adev, int index)
 	adev->ddev->mode_config.cursor_width = amdgpu_crtc->max_cursor_width;
 	adev->ddev->mode_config.cursor_height = amdgpu_crtc->max_cursor_height;
 
-	for (i = 0; i < 256; i++) {
-		amdgpu_crtc->lut_r[i] = i << 2;
-		amdgpu_crtc->lut_g[i] = i << 2;
-		amdgpu_crtc->lut_b[i] = i << 2;
-	}
-
 	switch (amdgpu_crtc->crtc_id) {
 	case 0:
 	default:
@@ -3025,6 +2884,8 @@ static int dce_v10_0_hw_init(void *handle)
 
 	dce_v10_0_init_golden_registers(adev);
 
+	/* disable vga render */
+	dce_v10_0_set_vga_render_state(adev, false);
 	/* init dig PHYs, disp eng pll */
 	amdgpu_atombios_encoder_init_dig(adev);
 	amdgpu_atombios_crtc_set_disp_eng_pll(adev, adev->clock.default_dispclk);
@@ -3737,7 +3598,6 @@ static void dce_v10_0_encoder_add(struct amdgpu_device *adev,
 }
 
 static const struct amdgpu_display_funcs dce_v10_0_display_funcs = {
-	.set_vga_render_state = &dce_v10_0_set_vga_render_state,
 	.bandwidth_update = &dce_v10_0_bandwidth_update,
 	.vblank_get_counter = &dce_v10_0_vblank_get_counter,
 	.vblank_wait = &dce_v10_0_vblank_wait,
@@ -3750,8 +3610,6 @@ static const struct amdgpu_display_funcs dce_v10_0_display_funcs = {
 	.page_flip_get_scanoutpos = &dce_v10_0_crtc_get_scanoutpos,
 	.add_encoder = &dce_v10_0_encoder_add,
 	.add_connector = &amdgpu_connector_add,
-	.stop_mc_access = &dce_v10_0_stop_mc_access,
-	.resume_mc_access = &dce_v10_0_resume_mc_access,
 };
 
 static void dce_v10_0_set_display_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
index 4bcf01dc567a..11edc75edaa9 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
@@ -499,79 +499,6 @@ static bool dce_v11_0_is_display_hung(struct amdgpu_device *adev)
 	return true;
 }
 
-static void dce_v11_0_stop_mc_access(struct amdgpu_device *adev,
-				     struct amdgpu_mode_mc_save *save)
-{
-	u32 crtc_enabled, tmp;
-	int i;
-
-	save->vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
-	save->vga_hdp_control = RREG32(mmVGA_HDP_CONTROL);
-
-	/* disable VGA render */
-	tmp = RREG32(mmVGA_RENDER_CONTROL);
-	tmp = REG_SET_FIELD(tmp, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
-	WREG32(mmVGA_RENDER_CONTROL, tmp);
-
-	/* blank the display controllers */
-	for (i = 0; i < adev->mode_info.num_crtc; i++) {
-		crtc_enabled = REG_GET_FIELD(RREG32(mmCRTC_CONTROL + crtc_offsets[i]),
-					     CRTC_CONTROL, CRTC_MASTER_EN);
-		if (crtc_enabled) {
-#if 1
-			save->crtc_enabled[i] = true;
-			tmp = RREG32(mmCRTC_BLANK_CONTROL + crtc_offsets[i]);
-			if (REG_GET_FIELD(tmp, CRTC_BLANK_CONTROL, CRTC_BLANK_DATA_EN) == 0) {
-				/*it is correct only for RGB ; black is 0*/
-				WREG32(mmCRTC_BLANK_DATA_COLOR + crtc_offsets[i], 0);
-				tmp = REG_SET_FIELD(tmp, CRTC_BLANK_CONTROL, CRTC_BLANK_DATA_EN, 1);
-				WREG32(mmCRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
-			}
-#else
-			/* XXX this is a hack to avoid strange behavior with EFI on certain systems */
-			WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 1);
-			tmp = RREG32(mmCRTC_CONTROL + crtc_offsets[i]);
-			tmp = REG_SET_FIELD(tmp, CRTC_CONTROL, CRTC_MASTER_EN, 0);
-			WREG32(mmCRTC_CONTROL + crtc_offsets[i], tmp);
-			WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 0);
-			save->crtc_enabled[i] = false;
-			/* ***** */
-#endif
-		} else {
-			save->crtc_enabled[i] = false;
-		}
-	}
-}
-
-static void dce_v11_0_resume_mc_access(struct amdgpu_device *adev,
-				       struct amdgpu_mode_mc_save *save)
-{
-	u32 tmp;
-	int i;
-
-	/* update crtc base addresses */
-	for (i = 0; i < adev->mode_info.num_crtc; i++) {
-		WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
-		       upper_32_bits(adev->mc.vram_start));
-		WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
-		       (u32)adev->mc.vram_start);
-
-		if (save->crtc_enabled[i]) {
-			tmp = RREG32(mmCRTC_BLANK_CONTROL + crtc_offsets[i]);
-			tmp = REG_SET_FIELD(tmp, CRTC_BLANK_CONTROL, CRTC_BLANK_DATA_EN, 0);
-			WREG32(mmCRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
-		}
-	}
-
-	WREG32(mmVGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(adev->mc.vram_start));
-	WREG32(mmVGA_MEMORY_BASE_ADDRESS, lower_32_bits(adev->mc.vram_start));
-
-	/* Unlock vga access */
-	WREG32(mmVGA_HDP_CONTROL, save->vga_hdp_control);
-	mdelay(1);
-	WREG32(mmVGA_RENDER_CONTROL, save->vga_render_control);
-}
-
 static void dce_v11_0_set_vga_render_state(struct amdgpu_device *adev,
 					   bool render)
 {
@@ -1851,7 +1778,7 @@ static void dce_v11_0_afmt_setmode(struct drm_encoder *encoder,
 	dce_v11_0_audio_write_sad_regs(encoder);
 	dce_v11_0_audio_write_latency_fields(encoder, mode);
 
-	err = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode);
+	err = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode, false);
 	if (err < 0) {
 		DRM_ERROR("failed to setup AVI infoframe: %zd\n", err);
 		return;
@@ -2251,6 +2178,7 @@ static void dce_v11_0_crtc_load_lut(struct drm_crtc *crtc)
 	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct amdgpu_device *adev = dev->dev_private;
+	u16 *r, *g, *b;
 	int i;
 	u32 tmp;
 
@@ -2282,11 +2210,14 @@ static void dce_v11_0_crtc_load_lut(struct drm_crtc *crtc)
 	WREG32(mmDC_LUT_WRITE_EN_MASK + amdgpu_crtc->crtc_offset, 0x00000007);
 
 	WREG32(mmDC_LUT_RW_INDEX + amdgpu_crtc->crtc_offset, 0);
+	r = crtc->gamma_store;
+	g = r + crtc->gamma_size;
+	b = g + crtc->gamma_size;
 	for (i = 0; i < 256; i++) {
 		WREG32(mmDC_LUT_30_COLOR + amdgpu_crtc->crtc_offset,
-		       (amdgpu_crtc->lut_r[i] << 20) |
-		       (amdgpu_crtc->lut_g[i] << 10) |
-		       (amdgpu_crtc->lut_b[i] << 0));
+		       ((*r++ & 0xffc0) << 14) |
+		       ((*g++ & 0xffc0) << 4) |
+		       (*b++ >> 6));
 	}
 
 	tmp = RREG32(mmDEGAMMA_CONTROL + amdgpu_crtc->crtc_offset);
@@ -2575,7 +2506,7 @@ static int dce_v11_0_crtc_cursor_set2(struct drm_crtc *crtc,
 	aobj = gem_to_amdgpu_bo(obj);
 	ret = amdgpu_bo_reserve(aobj, false);
 	if (ret != 0) {
-		drm_gem_object_unreference_unlocked(obj);
+		drm_gem_object_put_unlocked(obj);
 		return ret;
 	}
 
@@ -2583,7 +2514,7 @@ static int dce_v11_0_crtc_cursor_set2(struct drm_crtc *crtc,
 	amdgpu_bo_unreserve(aobj);
 	if (ret) {
 		DRM_ERROR("Failed to pin new cursor BO (%d)\n", ret);
-		drm_gem_object_unreference_unlocked(obj);
+		drm_gem_object_put_unlocked(obj);
 		return ret;
 	}
 
@@ -2617,7 +2548,7 @@ unpin:
 			amdgpu_bo_unpin(aobj);
 			amdgpu_bo_unreserve(aobj);
 		}
-		drm_gem_object_unreference_unlocked(amdgpu_crtc->cursor_bo);
+		drm_gem_object_put_unlocked(amdgpu_crtc->cursor_bo);
 	}
 
 	amdgpu_crtc->cursor_bo = obj;
@@ -2644,15 +2575,6 @@ static int dce_v11_0_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
 				    u16 *blue, uint32_t size,
 				    struct drm_modeset_acquire_ctx *ctx)
 {
-	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
-	int i;
-
-	/* userspace palettes are always correct as is */
-	for (i = 0; i < size; i++) {
-		amdgpu_crtc->lut_r[i] = red[i] >> 6;
-		amdgpu_crtc->lut_g[i] = green[i] >> 6;
-		amdgpu_crtc->lut_b[i] = blue[i] >> 6;
-	}
 	dce_v11_0_crtc_load_lut(crtc);
 
 	return 0;
@@ -2892,14 +2814,12 @@ static const struct drm_crtc_helper_funcs dce_v11_0_crtc_helper_funcs = {
 	.mode_set_base_atomic = dce_v11_0_crtc_set_base_atomic,
 	.prepare = dce_v11_0_crtc_prepare,
 	.commit = dce_v11_0_crtc_commit,
-	.load_lut = dce_v11_0_crtc_load_lut,
 	.disable = dce_v11_0_crtc_disable,
 };
 
 static int dce_v11_0_crtc_init(struct amdgpu_device *adev, int index)
 {
 	struct amdgpu_crtc *amdgpu_crtc;
-	int i;
 
 	amdgpu_crtc = kzalloc(sizeof(struct amdgpu_crtc) +
 			      (AMDGPUFB_CONN_LIMIT * sizeof(struct drm_connector *)), GFP_KERNEL);
@@ -2917,12 +2837,6 @@ static int dce_v11_0_crtc_init(struct amdgpu_device *adev, int index)
 	adev->ddev->mode_config.cursor_width = amdgpu_crtc->max_cursor_width;
 	adev->ddev->mode_config.cursor_height = amdgpu_crtc->max_cursor_height;
 
-	for (i = 0; i < 256; i++) {
-		amdgpu_crtc->lut_r[i] = i << 2;
-		amdgpu_crtc->lut_g[i] = i << 2;
-		amdgpu_crtc->lut_b[i] = i << 2;
-	}
-
 	switch (amdgpu_crtc->crtc_id) {
 	case 0:
 	default:
@@ -3086,6 +3000,8 @@ static int dce_v11_0_hw_init(void *handle)
 
 	dce_v11_0_init_golden_registers(adev);
 
+	/* disable vga render */
+	dce_v11_0_set_vga_render_state(adev, false);
 	/* init dig PHYs, disp eng pll */
 	amdgpu_atombios_crtc_powergate_init(adev);
 	amdgpu_atombios_encoder_init_dig(adev);
@@ -3806,7 +3722,6 @@ static void dce_v11_0_encoder_add(struct amdgpu_device *adev,
 }
 
 static const struct amdgpu_display_funcs dce_v11_0_display_funcs = {
-	.set_vga_render_state = &dce_v11_0_set_vga_render_state,
 	.bandwidth_update = &dce_v11_0_bandwidth_update,
 	.vblank_get_counter = &dce_v11_0_vblank_get_counter,
 	.vblank_wait = &dce_v11_0_vblank_wait,
@@ -3819,8 +3734,6 @@ static const struct amdgpu_display_funcs dce_v11_0_display_funcs = {
 	.page_flip_get_scanoutpos = &dce_v11_0_crtc_get_scanoutpos,
 	.add_encoder = &dce_v11_0_encoder_add,
 	.add_connector = &amdgpu_connector_add,
-	.stop_mc_access = &dce_v11_0_stop_mc_access,
-	.resume_mc_access = &dce_v11_0_resume_mc_access,
 };
 
 static void dce_v11_0_set_display_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
index fd134a4629d7..a51e35f824a1 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
@@ -42,6 +42,7 @@
 #include "dce/dce_6_0_d.h"
 #include "dce/dce_6_0_sh_mask.h"
 #include "gca/gfx_7_2_enum.h"
+#include "dce_v6_0.h"
 #include "si_enums.h"
 
 static void dce_v6_0_set_display_funcs(struct amdgpu_device *adev);
@@ -392,117 +393,6 @@ static u32 dce_v6_0_hpd_get_gpio_reg(struct amdgpu_device *adev)
 	return mmDC_GPIO_HPD_A;
 }
 
-static u32 evergreen_get_vblank_counter(struct amdgpu_device* adev, int crtc)
-{
-	if (crtc >= adev->mode_info.num_crtc)
-		return 0;
-	else
-		return RREG32(mmCRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
-}
-
-static void dce_v6_0_stop_mc_access(struct amdgpu_device *adev,
-				    struct amdgpu_mode_mc_save *save)
-{
-	u32 crtc_enabled, tmp, frame_count;
-	int i, j;
-
-	save->vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
-	save->vga_hdp_control = RREG32(mmVGA_HDP_CONTROL);
-
-	/* disable VGA render */
-	WREG32(mmVGA_RENDER_CONTROL, 0);
-
-	/* blank the display controllers */
-	for (i = 0; i < adev->mode_info.num_crtc; i++) {
-		crtc_enabled = RREG32(mmCRTC_CONTROL + crtc_offsets[i]) & CRTC_CONTROL__CRTC_MASTER_EN_MASK;
-		if (crtc_enabled) {
-			save->crtc_enabled[i] = true;
-			tmp = RREG32(mmCRTC_BLANK_CONTROL + crtc_offsets[i]);
-
-			if (!(tmp & CRTC_BLANK_CONTROL__CRTC_BLANK_DATA_EN_MASK)) {
-				dce_v6_0_vblank_wait(adev, i);
-				WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 1);
-				tmp |= CRTC_BLANK_CONTROL__CRTC_BLANK_DATA_EN_MASK;
-				WREG32(mmCRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
-				WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 0);
-			}
-			/* wait for the next frame */
-			frame_count = evergreen_get_vblank_counter(adev, i);
-			for (j = 0; j < adev->usec_timeout; j++) {
-				if (evergreen_get_vblank_counter(adev, i) != frame_count)
-					break;
-				udelay(1);
-			}
-
-			/* XXX this is a hack to avoid strange behavior with EFI on certain systems */
-			WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 1);
-			tmp = RREG32(mmCRTC_CONTROL + crtc_offsets[i]);
-			tmp &= ~CRTC_CONTROL__CRTC_MASTER_EN_MASK;
-			WREG32(mmCRTC_CONTROL + crtc_offsets[i], tmp);
-			WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 0);
-			save->crtc_enabled[i] = false;
-			/* ***** */
-		} else {
-			save->crtc_enabled[i] = false;
-		}
-	}
-}
-
-static void dce_v6_0_resume_mc_access(struct amdgpu_device *adev,
-				      struct amdgpu_mode_mc_save *save)
-{
-	u32 tmp;
-	int i, j;
-
-	/* update crtc base addresses */
-	for (i = 0; i < adev->mode_info.num_crtc; i++) {
-		WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
-		       upper_32_bits(adev->mc.vram_start));
-		WREG32(mmGRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
-		       upper_32_bits(adev->mc.vram_start));
-		WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
-		       (u32)adev->mc.vram_start);
-		WREG32(mmGRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
-		       (u32)adev->mc.vram_start);
-	}
-
-	WREG32(mmVGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(adev->mc.vram_start));
-	WREG32(mmVGA_MEMORY_BASE_ADDRESS, (u32)adev->mc.vram_start);
-
-	/* unlock regs and wait for update */
-	for (i = 0; i < adev->mode_info.num_crtc; i++) {
-		if (save->crtc_enabled[i]) {
-			tmp = RREG32(mmMASTER_UPDATE_MODE + crtc_offsets[i]);
-			if ((tmp & 0x7) != 0) {
-				tmp &= ~0x7;
-				WREG32(mmMASTER_UPDATE_MODE + crtc_offsets[i], tmp);
-			}
-			tmp = RREG32(mmGRPH_UPDATE + crtc_offsets[i]);
-			if (tmp & GRPH_UPDATE__GRPH_UPDATE_LOCK_MASK) {
-				tmp &= ~GRPH_UPDATE__GRPH_UPDATE_LOCK_MASK;
-				WREG32(mmGRPH_UPDATE + crtc_offsets[i], tmp);
-			}
-			tmp = RREG32(mmMASTER_UPDATE_LOCK + crtc_offsets[i]);
-			if (tmp & 1) {
-				tmp &= ~1;
-				WREG32(mmMASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
-			}
-			for (j = 0; j < adev->usec_timeout; j++) {
-				tmp = RREG32(mmGRPH_UPDATE + crtc_offsets[i]);
-				if ((tmp & GRPH_UPDATE__GRPH_SURFACE_UPDATE_PENDING_MASK) == 0)
-					break;
-				udelay(1);
-			}
-		}
-	}
-
-	/* Unlock vga access */
-	WREG32(mmVGA_HDP_CONTROL, save->vga_hdp_control);
-	mdelay(1);
-	WREG32(mmVGA_RENDER_CONTROL, save->vga_render_control);
-
-}
-
 static void dce_v6_0_set_vga_render_state(struct amdgpu_device *adev,
 					  bool render)
 {
@@ -1597,7 +1487,7 @@ static void dce_v6_0_audio_set_avi_infoframe(struct drm_encoder *encoder,
 	ssize_t err;
 	u32 tmp;
 
-	err = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode);
+	err = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode, false);
 	if (err < 0) {
 		DRM_ERROR("failed to setup AVI infoframe: %zd\n", err);
 		return;
@@ -2182,6 +2072,7 @@ static void dce_v6_0_crtc_load_lut(struct drm_crtc *crtc)
 	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct amdgpu_device *adev = dev->dev_private;
+	u16 *r, *g, *b;
 	int i;
 
 	DRM_DEBUG_KMS("%d\n", amdgpu_crtc->crtc_id);
@@ -2211,11 +2102,14 @@ static void dce_v6_0_crtc_load_lut(struct drm_crtc *crtc)
 	WREG32(mmDC_LUT_WRITE_EN_MASK + amdgpu_crtc->crtc_offset, 0x00000007);
 
 	WREG32(mmDC_LUT_RW_INDEX + amdgpu_crtc->crtc_offset, 0);
+	r = crtc->gamma_store;
+	g = r + crtc->gamma_size;
+	b = g + crtc->gamma_size;
 	for (i = 0; i < 256; i++) {
 		WREG32(mmDC_LUT_30_COLOR + amdgpu_crtc->crtc_offset,
-		       (amdgpu_crtc->lut_r[i] << 20) |
-		       (amdgpu_crtc->lut_g[i] << 10) |
-		       (amdgpu_crtc->lut_b[i] << 0));
+		       ((*r++ & 0xffc0) << 14) |
+		       ((*g++ & 0xffc0) << 4) |
+		       (*b++ >> 6));
 	}
 
 	WREG32(mmDEGAMMA_CONTROL + amdgpu_crtc->crtc_offset,
@@ -2428,7 +2322,7 @@ static int dce_v6_0_crtc_cursor_set2(struct drm_crtc *crtc,
 	aobj = gem_to_amdgpu_bo(obj);
 	ret = amdgpu_bo_reserve(aobj, false);
 	if (ret != 0) {
-		drm_gem_object_unreference_unlocked(obj);
+		drm_gem_object_put_unlocked(obj);
 		return ret;
 	}
 
@@ -2436,7 +2330,7 @@ static int dce_v6_0_crtc_cursor_set2(struct drm_crtc *crtc,
 	amdgpu_bo_unreserve(aobj);
 	if (ret) {
 		DRM_ERROR("Failed to pin new cursor BO (%d)\n", ret);
-		drm_gem_object_unreference_unlocked(obj);
+		drm_gem_object_put_unlocked(obj);
 		return ret;
 	}
 
@@ -2470,7 +2364,7 @@ unpin:
 			amdgpu_bo_unpin(aobj);
 			amdgpu_bo_unreserve(aobj);
 		}
-		drm_gem_object_unreference_unlocked(amdgpu_crtc->cursor_bo);
+		drm_gem_object_put_unlocked(amdgpu_crtc->cursor_bo);
 	}
 
 	amdgpu_crtc->cursor_bo = obj;
@@ -2496,15 +2390,6 @@ static int dce_v6_0_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
 				   u16 *blue, uint32_t size,
 				   struct drm_modeset_acquire_ctx *ctx)
 {
-	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
-	int i;
-
-	/* userspace palettes are always correct as is */
-	for (i = 0; i < size; i++) {
-		amdgpu_crtc->lut_r[i] = red[i] >> 6;
-		amdgpu_crtc->lut_g[i] = green[i] >> 6;
-		amdgpu_crtc->lut_b[i] = blue[i] >> 6;
-	}
 	dce_v6_0_crtc_load_lut(crtc);
 
 	return 0;
@@ -2712,14 +2597,12 @@ static const struct drm_crtc_helper_funcs dce_v6_0_crtc_helper_funcs = {
 	.mode_set_base_atomic = dce_v6_0_crtc_set_base_atomic,
 	.prepare = dce_v6_0_crtc_prepare,
 	.commit = dce_v6_0_crtc_commit,
-	.load_lut = dce_v6_0_crtc_load_lut,
 	.disable = dce_v6_0_crtc_disable,
 };
 
 static int dce_v6_0_crtc_init(struct amdgpu_device *adev, int index)
 {
 	struct amdgpu_crtc *amdgpu_crtc;
-	int i;
 
 	amdgpu_crtc = kzalloc(sizeof(struct amdgpu_crtc) +
 			      (AMDGPUFB_CONN_LIMIT * sizeof(struct drm_connector *)), GFP_KERNEL);
@@ -2737,12 +2620,6 @@ static int dce_v6_0_crtc_init(struct amdgpu_device *adev, int index)
 	adev->ddev->mode_config.cursor_width = amdgpu_crtc->max_cursor_width;
 	adev->ddev->mode_config.cursor_height = amdgpu_crtc->max_cursor_height;
 
-	for (i = 0; i < 256; i++) {
-		amdgpu_crtc->lut_r[i] = i << 2;
-		amdgpu_crtc->lut_g[i] = i << 2;
-		amdgpu_crtc->lut_b[i] = i << 2;
-	}
-
 	amdgpu_crtc->crtc_offset = crtc_offsets[amdgpu_crtc->crtc_id];
 
 	amdgpu_crtc->pll_id = ATOM_PPLL_INVALID;
@@ -2873,6 +2750,8 @@ static int dce_v6_0_hw_init(void *handle)
 	int i;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+	/* disable vga render */
+	dce_v6_0_set_vga_render_state(adev, false);
 	/* init dig PHYs, disp eng pll */
 	amdgpu_atombios_encoder_init_dig(adev);
 	amdgpu_atombios_crtc_set_disp_eng_pll(adev, adev->clock.default_dispclk);
@@ -3525,7 +3404,6 @@ static void dce_v6_0_encoder_add(struct amdgpu_device *adev,
 }
 
 static const struct amdgpu_display_funcs dce_v6_0_display_funcs = {
-	.set_vga_render_state = &dce_v6_0_set_vga_render_state,
 	.bandwidth_update = &dce_v6_0_bandwidth_update,
 	.vblank_get_counter = &dce_v6_0_vblank_get_counter,
 	.vblank_wait = &dce_v6_0_vblank_wait,
@@ -3538,8 +3416,6 @@ static const struct amdgpu_display_funcs dce_v6_0_display_funcs = {
 	.page_flip_get_scanoutpos = &dce_v6_0_crtc_get_scanoutpos,
 	.add_encoder = &dce_v6_0_encoder_add,
 	.add_connector = &amdgpu_connector_add,
-	.stop_mc_access = &dce_v6_0_stop_mc_access,
-	.resume_mc_access = &dce_v6_0_resume_mc_access,
 };
 
 static void dce_v6_0_set_display_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
index a9e869554627..9cf14b8b2db9 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
@@ -419,81 +419,6 @@ static bool dce_v8_0_is_display_hung(struct amdgpu_device *adev)
 	return true;
 }
 
-static void dce_v8_0_stop_mc_access(struct amdgpu_device *adev,
-				    struct amdgpu_mode_mc_save *save)
-{
-	u32 crtc_enabled, tmp;
-	int i;
-
-	save->vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
-	save->vga_hdp_control = RREG32(mmVGA_HDP_CONTROL);
-
-	/* disable VGA render */
-	tmp = RREG32(mmVGA_RENDER_CONTROL);
-	tmp = REG_SET_FIELD(tmp, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
-	WREG32(mmVGA_RENDER_CONTROL, tmp);
-
-	/* blank the display controllers */
-	for (i = 0; i < adev->mode_info.num_crtc; i++) {
-		crtc_enabled = REG_GET_FIELD(RREG32(mmCRTC_CONTROL + crtc_offsets[i]),
-					     CRTC_CONTROL, CRTC_MASTER_EN);
-		if (crtc_enabled) {
-#if 1
-			save->crtc_enabled[i] = true;
-			tmp = RREG32(mmCRTC_BLANK_CONTROL + crtc_offsets[i]);
-			if (REG_GET_FIELD(tmp, CRTC_BLANK_CONTROL, CRTC_BLANK_DATA_EN) == 0) {
-				/*it is correct only for RGB ; black is 0*/
-				WREG32(mmCRTC_BLANK_DATA_COLOR + crtc_offsets[i], 0);
-				tmp = REG_SET_FIELD(tmp, CRTC_BLANK_CONTROL, CRTC_BLANK_DATA_EN, 1);
-				WREG32(mmCRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
-			}
-			mdelay(20);
-#else
-			/* XXX this is a hack to avoid strange behavior with EFI on certain systems */
-			WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 1);
-			tmp = RREG32(mmCRTC_CONTROL + crtc_offsets[i]);
-			tmp = REG_SET_FIELD(tmp, CRTC_CONTROL, CRTC_MASTER_EN, 0);
-			WREG32(mmCRTC_CONTROL + crtc_offsets[i], tmp);
-			WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 0);
-			save->crtc_enabled[i] = false;
-			/* ***** */
-#endif
-		} else {
-			save->crtc_enabled[i] = false;
-		}
-	}
-}
-
-static void dce_v8_0_resume_mc_access(struct amdgpu_device *adev,
-				      struct amdgpu_mode_mc_save *save)
-{
-	u32 tmp;
-	int i;
-
-	/* update crtc base addresses */
-	for (i = 0; i < adev->mode_info.num_crtc; i++) {
-		WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
-		       upper_32_bits(adev->mc.vram_start));
-		WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
-		       (u32)adev->mc.vram_start);
-
-		if (save->crtc_enabled[i]) {
-			tmp = RREG32(mmCRTC_BLANK_CONTROL + crtc_offsets[i]);
-			tmp = REG_SET_FIELD(tmp, CRTC_BLANK_CONTROL, CRTC_BLANK_DATA_EN, 0);
-			WREG32(mmCRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
-		}
-		mdelay(20);
-	}
-
-	WREG32(mmVGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(adev->mc.vram_start));
-	WREG32(mmVGA_MEMORY_BASE_ADDRESS, lower_32_bits(adev->mc.vram_start));
-
-	/* Unlock vga access */
-	WREG32(mmVGA_HDP_CONTROL, save->vga_hdp_control);
-	mdelay(1);
-	WREG32(mmVGA_RENDER_CONTROL, save->vga_render_control);
-}
-
 static void dce_v8_0_set_vga_render_state(struct amdgpu_device *adev,
 					  bool render)
 {
@@ -1750,7 +1675,7 @@ static void dce_v8_0_afmt_setmode(struct drm_encoder *encoder,
 	dce_v8_0_audio_write_sad_regs(encoder);
 	dce_v8_0_audio_write_latency_fields(encoder, mode);
 
-	err = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode);
+	err = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode, false);
 	if (err < 0) {
 		DRM_ERROR("failed to setup AVI infoframe: %zd\n", err);
 		return;
@@ -2124,6 +2049,7 @@ static void dce_v8_0_crtc_load_lut(struct drm_crtc *crtc)
 	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct amdgpu_device *adev = dev->dev_private;
+	u16 *r, *g, *b;
 	int i;
 
 	DRM_DEBUG_KMS("%d\n", amdgpu_crtc->crtc_id);
@@ -2153,11 +2079,14 @@ static void dce_v8_0_crtc_load_lut(struct drm_crtc *crtc)
 	WREG32(mmDC_LUT_WRITE_EN_MASK + amdgpu_crtc->crtc_offset, 0x00000007);
 
 	WREG32(mmDC_LUT_RW_INDEX + amdgpu_crtc->crtc_offset, 0);
+	r = crtc->gamma_store;
+	g = r + crtc->gamma_size;
+	b = g + crtc->gamma_size;
 	for (i = 0; i < 256; i++) {
 		WREG32(mmDC_LUT_30_COLOR + amdgpu_crtc->crtc_offset,
-		       (amdgpu_crtc->lut_r[i] << 20) |
-		       (amdgpu_crtc->lut_g[i] << 10) |
-		       (amdgpu_crtc->lut_b[i] << 0));
+		       ((*r++ & 0xffc0) << 14) |
+		       ((*g++ & 0xffc0) << 4) |
+		       (*b++ >> 6));
 	}
 
 	WREG32(mmDEGAMMA_CONTROL + amdgpu_crtc->crtc_offset,
@@ -2406,7 +2335,7 @@ static int dce_v8_0_crtc_cursor_set2(struct drm_crtc *crtc,
 	aobj = gem_to_amdgpu_bo(obj);
 	ret = amdgpu_bo_reserve(aobj, false);
 	if (ret != 0) {
-		drm_gem_object_unreference_unlocked(obj);
+		drm_gem_object_put_unlocked(obj);
 		return ret;
 	}
 
@@ -2414,7 +2343,7 @@ static int dce_v8_0_crtc_cursor_set2(struct drm_crtc *crtc,
 	amdgpu_bo_unreserve(aobj);
 	if (ret) {
 		DRM_ERROR("Failed to pin new cursor BO (%d)\n", ret);
-		drm_gem_object_unreference_unlocked(obj);
+		drm_gem_object_put_unlocked(obj);
 		return ret;
 	}
 
@@ -2448,7 +2377,7 @@ unpin:
 			amdgpu_bo_unpin(aobj);
 			amdgpu_bo_unreserve(aobj);
 		}
-		drm_gem_object_unreference_unlocked(amdgpu_crtc->cursor_bo);
+		drm_gem_object_put_unlocked(amdgpu_crtc->cursor_bo);
 	}
 
 	amdgpu_crtc->cursor_bo = obj;
@@ -2475,15 +2404,6 @@ static int dce_v8_0_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
 				   u16 *blue, uint32_t size,
 				   struct drm_modeset_acquire_ctx *ctx)
 {
-	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
-	int i;
-
-	/* userspace palettes are always correct as is */
-	for (i = 0; i < size; i++) {
-		amdgpu_crtc->lut_r[i] = red[i] >> 6;
-		amdgpu_crtc->lut_g[i] = green[i] >> 6;
-		amdgpu_crtc->lut_b[i] = blue[i] >> 6;
-	}
 	dce_v8_0_crtc_load_lut(crtc);
 
 	return 0;
@@ -2702,14 +2622,12 @@ static const struct drm_crtc_helper_funcs dce_v8_0_crtc_helper_funcs = {
 	.mode_set_base_atomic = dce_v8_0_crtc_set_base_atomic,
 	.prepare = dce_v8_0_crtc_prepare,
 	.commit = dce_v8_0_crtc_commit,
-	.load_lut = dce_v8_0_crtc_load_lut,
 	.disable = dce_v8_0_crtc_disable,
 };
 
 static int dce_v8_0_crtc_init(struct amdgpu_device *adev, int index)
 {
 	struct amdgpu_crtc *amdgpu_crtc;
-	int i;
 
 	amdgpu_crtc = kzalloc(sizeof(struct amdgpu_crtc) +
 			      (AMDGPUFB_CONN_LIMIT * sizeof(struct drm_connector *)), GFP_KERNEL);
@@ -2727,12 +2645,6 @@ static int dce_v8_0_crtc_init(struct amdgpu_device *adev, int index)
 	adev->ddev->mode_config.cursor_width = amdgpu_crtc->max_cursor_width;
 	adev->ddev->mode_config.cursor_height = amdgpu_crtc->max_cursor_height;
 
-	for (i = 0; i < 256; i++) {
-		amdgpu_crtc->lut_r[i] = i << 2;
-		amdgpu_crtc->lut_g[i] = i << 2;
-		amdgpu_crtc->lut_b[i] = i << 2;
-	}
-
 	amdgpu_crtc->crtc_offset = crtc_offsets[amdgpu_crtc->crtc_id];
 
 	amdgpu_crtc->pll_id = ATOM_PPLL_INVALID;
@@ -2870,6 +2782,8 @@ static int dce_v8_0_hw_init(void *handle)
 	int i;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+	/* disable vga render */
+	dce_v8_0_set_vga_render_state(adev, false);
 	/* init dig PHYs, disp eng pll */
 	amdgpu_atombios_encoder_init_dig(adev);
 	amdgpu_atombios_crtc_set_disp_eng_pll(adev, adev->clock.default_dispclk);
@@ -3574,7 +3488,6 @@ static void dce_v8_0_encoder_add(struct amdgpu_device *adev,
 }
 
 static const struct amdgpu_display_funcs dce_v8_0_display_funcs = {
-	.set_vga_render_state = &dce_v8_0_set_vga_render_state,
 	.bandwidth_update = &dce_v8_0_bandwidth_update,
 	.vblank_get_counter = &dce_v8_0_vblank_get_counter,
 	.vblank_wait = &dce_v8_0_vblank_wait,
@@ -3587,8 +3500,6 @@ static const struct amdgpu_display_funcs dce_v8_0_display_funcs = {
 	.page_flip_get_scanoutpos = &dce_v8_0_crtc_get_scanoutpos,
 	.add_encoder = &dce_v8_0_encoder_add,
 	.add_connector = &amdgpu_connector_add,
-	.stop_mc_access = &dce_v8_0_stop_mc_access,
-	.resume_mc_access = &dce_v8_0_resume_mc_access,
 };
 
 static void dce_v8_0_set_display_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
index 90bb08309a53..b9ee9073cb0d 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
@@ -95,62 +95,6 @@ static u32 dce_virtual_hpd_get_gpio_reg(struct amdgpu_device *adev)
 	return 0;
 }
 
-static void dce_virtual_stop_mc_access(struct amdgpu_device *adev,
-			      struct amdgpu_mode_mc_save *save)
-{
-	switch (adev->asic_type) {
-#ifdef CONFIG_DRM_AMDGPU_SI
-	case CHIP_TAHITI:
-	case CHIP_PITCAIRN:
-	case CHIP_VERDE:
-	case CHIP_OLAND:
-		dce_v6_0_disable_dce(adev);
-		break;
-#endif
-#ifdef CONFIG_DRM_AMDGPU_CIK
-	case CHIP_BONAIRE:
-	case CHIP_HAWAII:
-	case CHIP_KAVERI:
-	case CHIP_KABINI:
-	case CHIP_MULLINS:
-		dce_v8_0_disable_dce(adev);
-		break;
-#endif
-	case CHIP_FIJI:
-	case CHIP_TONGA:
-		dce_v10_0_disable_dce(adev);
-		break;
-	case CHIP_CARRIZO:
-	case CHIP_STONEY:
-	case CHIP_POLARIS10:
-	case CHIP_POLARIS11:
-	case CHIP_POLARIS12:
-		dce_v11_0_disable_dce(adev);
-		break;
-	case CHIP_TOPAZ:
-#ifdef CONFIG_DRM_AMDGPU_SI
-	case CHIP_HAINAN:
-#endif
-		/* no DCE */
-		return;
-	default:
-		DRM_ERROR("Virtual display unsupported ASIC type: 0x%X\n", adev->asic_type);
-	}
-
-	return;
-}
-static void dce_virtual_resume_mc_access(struct amdgpu_device *adev,
-				struct amdgpu_mode_mc_save *save)
-{
-	return;
-}
-
-static void dce_virtual_set_vga_render_state(struct amdgpu_device *adev,
-				    bool render)
-{
-	return;
-}
-
 /**
  * dce_virtual_bandwidth_update - program display watermarks
  *
@@ -168,16 +112,6 @@ static int dce_virtual_crtc_gamma_set(struct drm_crtc *crtc, u16 *red,
 				      u16 *green, u16 *blue, uint32_t size,
 				      struct drm_modeset_acquire_ctx *ctx)
 {
-	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
-	int i;
-
-	/* userspace palettes are always correct as is */
-	for (i = 0; i < size; i++) {
-		amdgpu_crtc->lut_r[i] = red[i] >> 6;
-		amdgpu_crtc->lut_g[i] = green[i] >> 6;
-		amdgpu_crtc->lut_b[i] = blue[i] >> 6;
-	}
-
 	return 0;
 }
 
@@ -289,11 +223,6 @@ static int dce_virtual_crtc_set_base(struct drm_crtc *crtc, int x, int y,
 	return 0;
 }
 
-static void dce_virtual_crtc_load_lut(struct drm_crtc *crtc)
-{
-	return;
-}
-
 static int dce_virtual_crtc_set_base_atomic(struct drm_crtc *crtc,
 					 struct drm_framebuffer *fb,
 					 int x, int y, enum mode_set_atomic state)
@@ -309,14 +238,12 @@ static const struct drm_crtc_helper_funcs dce_virtual_crtc_helper_funcs = {
 	.mode_set_base_atomic = dce_virtual_crtc_set_base_atomic,
 	.prepare = dce_virtual_crtc_prepare,
 	.commit = dce_virtual_crtc_commit,
-	.load_lut = dce_virtual_crtc_load_lut,
 	.disable = dce_virtual_crtc_disable,
 };
 
 static int dce_virtual_crtc_init(struct amdgpu_device *adev, int index)
 {
 	struct amdgpu_crtc *amdgpu_crtc;
-	int i;
 
 	amdgpu_crtc = kzalloc(sizeof(struct amdgpu_crtc) +
 			      (AMDGPUFB_CONN_LIMIT * sizeof(struct drm_connector *)), GFP_KERNEL);
@@ -329,12 +256,6 @@ static int dce_virtual_crtc_init(struct amdgpu_device *adev, int index)
 	amdgpu_crtc->crtc_id = index;
 	adev->mode_info.crtcs[index] = amdgpu_crtc;
 
-	for (i = 0; i < 256; i++) {
-		amdgpu_crtc->lut_r[i] = i << 2;
-		amdgpu_crtc->lut_g[i] = i << 2;
-		amdgpu_crtc->lut_b[i] = i << 2;
-	}
-
 	amdgpu_crtc->pll_id = ATOM_PPLL_INVALID;
 	amdgpu_crtc->encoder = NULL;
 	amdgpu_crtc->connector = NULL;
@@ -522,6 +443,47 @@ static int dce_virtual_sw_fini(void *handle)
 
 static int dce_virtual_hw_init(void *handle)
 {
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	switch (adev->asic_type) {
+#ifdef CONFIG_DRM_AMDGPU_SI
+	case CHIP_TAHITI:
+	case CHIP_PITCAIRN:
+	case CHIP_VERDE:
+	case CHIP_OLAND:
+		dce_v6_0_disable_dce(adev);
+		break;
+#endif
+#ifdef CONFIG_DRM_AMDGPU_CIK
+	case CHIP_BONAIRE:
+	case CHIP_HAWAII:
+	case CHIP_KAVERI:
+	case CHIP_KABINI:
+	case CHIP_MULLINS:
+		dce_v8_0_disable_dce(adev);
+		break;
+#endif
+	case CHIP_FIJI:
+	case CHIP_TONGA:
+		dce_v10_0_disable_dce(adev);
+		break;
+	case CHIP_CARRIZO:
+	case CHIP_STONEY:
+	case CHIP_POLARIS11:
+	case CHIP_POLARIS10:
+		dce_v11_0_disable_dce(adev);
+		break;
+	case CHIP_TOPAZ:
+#ifdef CONFIG_DRM_AMDGPU_SI
+	case CHIP_HAINAN:
+#endif
+		/* no DCE */
+		break;
+	case CHIP_VEGA10:
+		break;
+	default:
+		DRM_ERROR("Virtual display unsupported ASIC type: 0x%X\n", adev->asic_type);
+	}
 	return 0;
 }
 
@@ -677,7 +639,6 @@ static int dce_virtual_connector_encoder_init(struct amdgpu_device *adev,
 }
 
 static const struct amdgpu_display_funcs dce_virtual_display_funcs = {
-	.set_vga_render_state = &dce_virtual_set_vga_render_state,
 	.bandwidth_update = &dce_virtual_bandwidth_update,
 	.vblank_get_counter = &dce_virtual_vblank_get_counter,
 	.vblank_wait = &dce_virtual_vblank_wait,
@@ -690,8 +651,6 @@ static const struct amdgpu_display_funcs dce_virtual_display_funcs = {
 	.page_flip_get_scanoutpos = &dce_virtual_crtc_get_scanoutpos,
 	.add_encoder = NULL,
 	.add_connector = NULL,
-	.stop_mc_access = &dce_virtual_stop_mc_access,
-	.resume_mc_access = &dce_virtual_resume_mc_access,
 };
 
 static void dce_virtual_set_display_funcs(struct amdgpu_device *adev)
@@ -809,7 +768,7 @@ static const struct amdgpu_irq_src_funcs dce_virtual_crtc_irq_funcs = {
 
 static void dce_virtual_set_irq_funcs(struct amdgpu_device *adev)
 {
-	adev->crtc_irq.num_types = AMDGPU_CRTC_IRQ_LAST;
+	adev->crtc_irq.num_types = AMDGPU_CRTC_IRQ_VBLANK6 + 1;
 	adev->crtc_irq.funcs = &dce_virtual_crtc_irq_funcs;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
index 5173ca1fd159..d228f5a99044 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
@@ -1573,7 +1573,7 @@ static void gfx_v6_0_gpu_init(struct amdgpu_device *adev)
 
 static void gfx_v6_0_scratch_init(struct amdgpu_device *adev)
 {
-	adev->gfx.scratch.num_reg = 7;
+	adev->gfx.scratch.num_reg = 8;
 	adev->gfx.scratch.reg_base = mmSCRATCH_REG0;
 	adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
 }
@@ -2217,40 +2217,9 @@ static void gfx_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
 
 static void gfx_v6_0_rlc_fini(struct amdgpu_device *adev)
 {
-	int r;
-
-	if (adev->gfx.rlc.save_restore_obj) {
-		r = amdgpu_bo_reserve(adev->gfx.rlc.save_restore_obj, true);
-		if (unlikely(r != 0))
-			dev_warn(adev->dev, "(%d) reserve RLC sr bo failed\n", r);
-		amdgpu_bo_unpin(adev->gfx.rlc.save_restore_obj);
-		amdgpu_bo_unreserve(adev->gfx.rlc.save_restore_obj);
-
-		amdgpu_bo_unref(&adev->gfx.rlc.save_restore_obj);
-		adev->gfx.rlc.save_restore_obj = NULL;
-	}
-
-	if (adev->gfx.rlc.clear_state_obj) {
-		r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, true);
-		if (unlikely(r != 0))
-			dev_warn(adev->dev, "(%d) reserve RLC c bo failed\n", r);
-		amdgpu_bo_unpin(adev->gfx.rlc.clear_state_obj);
-		amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
-
-		amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj);
-		adev->gfx.rlc.clear_state_obj = NULL;
-	}
-
-	if (adev->gfx.rlc.cp_table_obj) {
-		r = amdgpu_bo_reserve(adev->gfx.rlc.cp_table_obj, true);
-		if (unlikely(r != 0))
-			dev_warn(adev->dev, "(%d) reserve RLC cp table bo failed\n", r);
-		amdgpu_bo_unpin(adev->gfx.rlc.cp_table_obj);
-		amdgpu_bo_unreserve(adev->gfx.rlc.cp_table_obj);
-
-		amdgpu_bo_unref(&adev->gfx.rlc.cp_table_obj);
-		adev->gfx.rlc.cp_table_obj = NULL;
-	}
+	amdgpu_bo_free_kernel(&adev->gfx.rlc.save_restore_obj, NULL, NULL);
+	amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj, NULL, NULL);
+	amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, NULL, NULL);
 }
 
 static int gfx_v6_0_rlc_init(struct amdgpu_device *adev)
@@ -2273,43 +2242,23 @@ static int gfx_v6_0_rlc_init(struct amdgpu_device *adev)
 
 	if (src_ptr) {
 		/* save restore block */
-		if (adev->gfx.rlc.save_restore_obj == NULL) {
-			r = amdgpu_bo_create(adev, dws * 4, PAGE_SIZE, true,
-					     AMDGPU_GEM_DOMAIN_VRAM,
-					     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
-					     NULL, NULL,
-					     &adev->gfx.rlc.save_restore_obj);
-
-			if (r) {
-				dev_warn(adev->dev, "(%d) create RLC sr bo failed\n", r);
-				return r;
-			}
-		}
-
-		r = amdgpu_bo_reserve(adev->gfx.rlc.save_restore_obj, false);
-		if (unlikely(r != 0)) {
-			gfx_v6_0_rlc_fini(adev);
-			return r;
-		}
-		r = amdgpu_bo_pin(adev->gfx.rlc.save_restore_obj, AMDGPU_GEM_DOMAIN_VRAM,
-				  &adev->gfx.rlc.save_restore_gpu_addr);
+		r = amdgpu_bo_create_reserved(adev, dws * 4, PAGE_SIZE,
+					      AMDGPU_GEM_DOMAIN_VRAM,
+					      &adev->gfx.rlc.save_restore_obj,
+					      &adev->gfx.rlc.save_restore_gpu_addr,
+					      (void **)&adev->gfx.rlc.sr_ptr);
 		if (r) {
-			amdgpu_bo_unreserve(adev->gfx.rlc.save_restore_obj);
-			dev_warn(adev->dev, "(%d) pin RLC sr bo failed\n", r);
+			dev_warn(adev->dev, "(%d) create RLC sr bo failed\n",
+				 r);
 			gfx_v6_0_rlc_fini(adev);
 			return r;
 		}
 
-		r = amdgpu_bo_kmap(adev->gfx.rlc.save_restore_obj, (void **)&adev->gfx.rlc.sr_ptr);
-		if (r) {
-			dev_warn(adev->dev, "(%d) map RLC sr bo failed\n", r);
-			gfx_v6_0_rlc_fini(adev);
-			return r;
-		}
 		/* write the sr buffer */
 		dst_ptr = adev->gfx.rlc.sr_ptr;
 		for (i = 0; i < adev->gfx.rlc.reg_list_size; i++)
 			dst_ptr[i] = cpu_to_le32(src_ptr[i]);
+
 		amdgpu_bo_kunmap(adev->gfx.rlc.save_restore_obj);
 		amdgpu_bo_unreserve(adev->gfx.rlc.save_restore_obj);
 	}
@@ -2319,39 +2268,17 @@ static int gfx_v6_0_rlc_init(struct amdgpu_device *adev)
 		adev->gfx.rlc.clear_state_size = gfx_v6_0_get_csb_size(adev);
 		dws = adev->gfx.rlc.clear_state_size + (256 / 4);
 
-		if (adev->gfx.rlc.clear_state_obj == NULL) {
-			r = amdgpu_bo_create(adev, dws * 4, PAGE_SIZE, true,
-					     AMDGPU_GEM_DOMAIN_VRAM,
-					     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
-					     NULL, NULL,
-					     &adev->gfx.rlc.clear_state_obj);
-
-			if (r) {
-				dev_warn(adev->dev, "(%d) create RLC c bo failed\n", r);
-				gfx_v6_0_rlc_fini(adev);
-				return r;
-			}
-		}
-		r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, false);
-		if (unlikely(r != 0)) {
-			gfx_v6_0_rlc_fini(adev);
-			return r;
-		}
-		r = amdgpu_bo_pin(adev->gfx.rlc.clear_state_obj, AMDGPU_GEM_DOMAIN_VRAM,
-				  &adev->gfx.rlc.clear_state_gpu_addr);
+		r = amdgpu_bo_create_reserved(adev, dws * 4, PAGE_SIZE,
+					      AMDGPU_GEM_DOMAIN_VRAM,
+					      &adev->gfx.rlc.clear_state_obj,
+					      &adev->gfx.rlc.clear_state_gpu_addr,
+					      (void **)&adev->gfx.rlc.cs_ptr);
 		if (r) {
-			amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
-			dev_warn(adev->dev, "(%d) pin RLC c bo failed\n", r);
+			dev_warn(adev->dev, "(%d) create RLC c bo failed\n", r);
 			gfx_v6_0_rlc_fini(adev);
 			return r;
 		}
 
-		r = amdgpu_bo_kmap(adev->gfx.rlc.clear_state_obj, (void **)&adev->gfx.rlc.cs_ptr);
-		if (r) {
-			dev_warn(adev->dev, "(%d) map RLC c bo failed\n", r);
-			gfx_v6_0_rlc_fini(adev);
-			return r;
-		}
 		/* set up the cs buffer */
 		dst_ptr = adev->gfx.rlc.cs_ptr;
 		reg_list_mc_addr = adev->gfx.rlc.clear_state_gpu_addr + 256;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
index 37b45e4403d1..00868764a0dd 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
@@ -1823,7 +1823,7 @@ static void gfx_v7_0_setup_rb(struct amdgpu_device *adev)
 }
 
 /**
- * gmc_v7_0_init_compute_vmid - gart enable
+ * gfx_v7_0_init_compute_vmid - gart enable
  *
  * @adev: amdgpu_device pointer
  *
@@ -1833,7 +1833,7 @@ static void gfx_v7_0_setup_rb(struct amdgpu_device *adev)
 #define DEFAULT_SH_MEM_BASES	(0x6000)
 #define FIRST_COMPUTE_VMID	(8)
 #define LAST_COMPUTE_VMID	(16)
-static void gmc_v7_0_init_compute_vmid(struct amdgpu_device *adev)
+static void gfx_v7_0_init_compute_vmid(struct amdgpu_device *adev)
 {
 	int i;
 	uint32_t sh_mem_config;
@@ -1921,6 +1921,7 @@ static void gfx_v7_0_gpu_init(struct amdgpu_device *adev)
 				   ELEMENT_SIZE, 1);
 	sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG,
 				   INDEX_STRIDE, 3);
+	WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg);
 
 	mutex_lock(&adev->srbm_mutex);
 	for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) {
@@ -1934,12 +1935,11 @@ static void gfx_v7_0_gpu_init(struct amdgpu_device *adev)
 		WREG32(mmSH_MEM_APE1_BASE, 1);
 		WREG32(mmSH_MEM_APE1_LIMIT, 0);
 		WREG32(mmSH_MEM_BASES, sh_mem_base);
-		WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg);
 	}
 	cik_srbm_select(adev, 0, 0, 0, 0);
 	mutex_unlock(&adev->srbm_mutex);
 
-	gmc_v7_0_init_compute_vmid(adev);
+	gfx_v7_0_init_compute_vmid(adev);
 
 	WREG32(mmSX_DEBUG_1, 0x20);
 
@@ -2021,7 +2021,7 @@ static void gfx_v7_0_gpu_init(struct amdgpu_device *adev)
  */
 static void gfx_v7_0_scratch_init(struct amdgpu_device *adev)
 {
-	adev->gfx.scratch.num_reg = 7;
+	adev->gfx.scratch.num_reg = 8;
 	adev->gfx.scratch.reg_base = mmSCRATCH_REG0;
 	adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
 }
@@ -2774,39 +2774,18 @@ static int gfx_v7_0_cp_compute_load_microcode(struct amdgpu_device *adev)
  */
 static void gfx_v7_0_cp_compute_fini(struct amdgpu_device *adev)
 {
-	int i, r;
+	int i;
 
 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 		struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
 
-		if (ring->mqd_obj) {
-			r = amdgpu_bo_reserve(ring->mqd_obj, true);
-			if (unlikely(r != 0))
-				dev_warn(adev->dev, "(%d) reserve MQD bo failed\n", r);
-
-			amdgpu_bo_unpin(ring->mqd_obj);
-			amdgpu_bo_unreserve(ring->mqd_obj);
-
-			amdgpu_bo_unref(&ring->mqd_obj);
-			ring->mqd_obj = NULL;
-		}
+		amdgpu_bo_free_kernel(&ring->mqd_obj, NULL, NULL);
 	}
 }
 
 static void gfx_v7_0_mec_fini(struct amdgpu_device *adev)
 {
-	int r;
-
-	if (adev->gfx.mec.hpd_eop_obj) {
-		r = amdgpu_bo_reserve(adev->gfx.mec.hpd_eop_obj, true);
-		if (unlikely(r != 0))
-			dev_warn(adev->dev, "(%d) reserve HPD EOP bo failed\n", r);
-		amdgpu_bo_unpin(adev->gfx.mec.hpd_eop_obj);
-		amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
-
-		amdgpu_bo_unref(&adev->gfx.mec.hpd_eop_obj);
-		adev->gfx.mec.hpd_eop_obj = NULL;
-	}
+	amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
 }
 
 static int gfx_v7_0_mec_init(struct amdgpu_device *adev)
@@ -2823,33 +2802,14 @@ static int gfx_v7_0_mec_init(struct amdgpu_device *adev)
 	/* allocate space for ALL pipes (even the ones we don't own) */
 	mec_hpd_size = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec
 		* GFX7_MEC_HPD_SIZE * 2;
-	if (adev->gfx.mec.hpd_eop_obj == NULL) {
-		r = amdgpu_bo_create(adev,
-				     mec_hpd_size,
-				     PAGE_SIZE, true,
-				     AMDGPU_GEM_DOMAIN_GTT, 0, NULL, NULL,
-				     &adev->gfx.mec.hpd_eop_obj);
-		if (r) {
-			dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
-			return r;
-		}
-	}
 
-	r = amdgpu_bo_reserve(adev->gfx.mec.hpd_eop_obj, false);
-	if (unlikely(r != 0)) {
-		gfx_v7_0_mec_fini(adev);
-		return r;
-	}
-	r = amdgpu_bo_pin(adev->gfx.mec.hpd_eop_obj, AMDGPU_GEM_DOMAIN_GTT,
-			  &adev->gfx.mec.hpd_eop_gpu_addr);
-	if (r) {
-		dev_warn(adev->dev, "(%d) pin HDP EOP bo failed\n", r);
-		gfx_v7_0_mec_fini(adev);
-		return r;
-	}
-	r = amdgpu_bo_kmap(adev->gfx.mec.hpd_eop_obj, (void **)&hpd);
+	r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
+				      AMDGPU_GEM_DOMAIN_GTT,
+				      &adev->gfx.mec.hpd_eop_obj,
+				      &adev->gfx.mec.hpd_eop_gpu_addr,
+				      (void **)&hpd);
 	if (r) {
-		dev_warn(adev->dev, "(%d) map HDP EOP bo failed\n", r);
+		dev_warn(adev->dev, "(%d) create, pin or map of HDP EOP bo failed\n", r);
 		gfx_v7_0_mec_fini(adev);
 		return r;
 	}
@@ -3108,32 +3068,12 @@ static int gfx_v7_0_compute_queue_init(struct amdgpu_device *adev, int ring_id)
 	struct cik_mqd *mqd;
 	struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
 
-	if (ring->mqd_obj == NULL) {
-		r = amdgpu_bo_create(adev,
-				sizeof(struct cik_mqd),
-				PAGE_SIZE, true,
-				AMDGPU_GEM_DOMAIN_GTT, 0, NULL, NULL,
-				&ring->mqd_obj);
-		if (r) {
-			dev_warn(adev->dev, "(%d) create MQD bo failed\n", r);
-			return r;
-		}
-	}
-
-	r = amdgpu_bo_reserve(ring->mqd_obj, false);
-	if (unlikely(r != 0))
-		goto out;
-
-	r = amdgpu_bo_pin(ring->mqd_obj, AMDGPU_GEM_DOMAIN_GTT,
-			&mqd_gpu_addr);
-	if (r) {
-		dev_warn(adev->dev, "(%d) pin MQD bo failed\n", r);
-		goto out_unreserve;
-	}
-	r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&mqd);
+	r = amdgpu_bo_create_reserved(adev, sizeof(struct cik_mqd), PAGE_SIZE,
+				      AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,
+				      &mqd_gpu_addr, (void **)&mqd);
 	if (r) {
-		dev_warn(adev->dev, "(%d) map MQD bo failed\n", r);
-		goto out_unreserve;
+		dev_warn(adev->dev, "(%d) create MQD bo failed\n", r);
+		return r;
 	}
 
 	mutex_lock(&adev->srbm_mutex);
@@ -3147,9 +3087,7 @@ static int gfx_v7_0_compute_queue_init(struct amdgpu_device *adev, int ring_id)
 	mutex_unlock(&adev->srbm_mutex);
 
 	amdgpu_bo_kunmap(ring->mqd_obj);
-out_unreserve:
 	amdgpu_bo_unreserve(ring->mqd_obj);
-out:
 	return 0;
 }
 
@@ -3361,43 +3299,9 @@ static void gfx_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
  */
 static void gfx_v7_0_rlc_fini(struct amdgpu_device *adev)
 {
-	int r;
-
-	/* save restore block */
-	if (adev->gfx.rlc.save_restore_obj) {
-		r = amdgpu_bo_reserve(adev->gfx.rlc.save_restore_obj, true);
-		if (unlikely(r != 0))
-			dev_warn(adev->dev, "(%d) reserve RLC sr bo failed\n", r);
-		amdgpu_bo_unpin(adev->gfx.rlc.save_restore_obj);
-		amdgpu_bo_unreserve(adev->gfx.rlc.save_restore_obj);
-
-		amdgpu_bo_unref(&adev->gfx.rlc.save_restore_obj);
-		adev->gfx.rlc.save_restore_obj = NULL;
-	}
-
-	/* clear state block */
-	if (adev->gfx.rlc.clear_state_obj) {
-		r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, true);
-		if (unlikely(r != 0))
-			dev_warn(adev->dev, "(%d) reserve RLC c bo failed\n", r);
-		amdgpu_bo_unpin(adev->gfx.rlc.clear_state_obj);
-		amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
-
-		amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj);
-		adev->gfx.rlc.clear_state_obj = NULL;
-	}
-
-	/* clear state block */
-	if (adev->gfx.rlc.cp_table_obj) {
-		r = amdgpu_bo_reserve(adev->gfx.rlc.cp_table_obj, true);
-		if (unlikely(r != 0))
-			dev_warn(adev->dev, "(%d) reserve RLC cp table bo failed\n", r);
-		amdgpu_bo_unpin(adev->gfx.rlc.cp_table_obj);
-		amdgpu_bo_unreserve(adev->gfx.rlc.cp_table_obj);
-
-		amdgpu_bo_unref(&adev->gfx.rlc.cp_table_obj);
-		adev->gfx.rlc.cp_table_obj = NULL;
-	}
+	amdgpu_bo_free_kernel(&adev->gfx.rlc.save_restore_obj, NULL, NULL);
+	amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj, NULL, NULL);
+	amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, NULL, NULL);
 }
 
 static int gfx_v7_0_rlc_init(struct amdgpu_device *adev)
@@ -3432,39 +3336,17 @@ static int gfx_v7_0_rlc_init(struct amdgpu_device *adev)
 
 	if (src_ptr) {
 		/* save restore block */
-		if (adev->gfx.rlc.save_restore_obj == NULL) {
-			r = amdgpu_bo_create(adev, dws * 4, PAGE_SIZE, true,
-					     AMDGPU_GEM_DOMAIN_VRAM,
-					     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
-					     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
-					     NULL, NULL,
-					     &adev->gfx.rlc.save_restore_obj);
-			if (r) {
-				dev_warn(adev->dev, "(%d) create RLC sr bo failed\n", r);
-				return r;
-			}
-		}
-
-		r = amdgpu_bo_reserve(adev->gfx.rlc.save_restore_obj, false);
-		if (unlikely(r != 0)) {
-			gfx_v7_0_rlc_fini(adev);
-			return r;
-		}
-		r = amdgpu_bo_pin(adev->gfx.rlc.save_restore_obj, AMDGPU_GEM_DOMAIN_VRAM,
-				  &adev->gfx.rlc.save_restore_gpu_addr);
+		r = amdgpu_bo_create_reserved(adev, dws * 4, PAGE_SIZE,
+					      AMDGPU_GEM_DOMAIN_VRAM,
+					      &adev->gfx.rlc.save_restore_obj,
+					      &adev->gfx.rlc.save_restore_gpu_addr,
+					      (void **)&adev->gfx.rlc.sr_ptr);
 		if (r) {
-			amdgpu_bo_unreserve(adev->gfx.rlc.save_restore_obj);
-			dev_warn(adev->dev, "(%d) pin RLC sr bo failed\n", r);
+			dev_warn(adev->dev, "(%d) create, pin or map of RLC sr bo failed\n", r);
 			gfx_v7_0_rlc_fini(adev);
 			return r;
 		}
 
-		r = amdgpu_bo_kmap(adev->gfx.rlc.save_restore_obj, (void **)&adev->gfx.rlc.sr_ptr);
-		if (r) {
-			dev_warn(adev->dev, "(%d) map RLC sr bo failed\n", r);
-			gfx_v7_0_rlc_fini(adev);
-			return r;
-		}
 		/* write the sr buffer */
 		dst_ptr = adev->gfx.rlc.sr_ptr;
 		for (i = 0; i < adev->gfx.rlc.reg_list_size; i++)
@@ -3477,39 +3359,17 @@ static int gfx_v7_0_rlc_init(struct amdgpu_device *adev)
 		/* clear state block */
 		adev->gfx.rlc.clear_state_size = dws = gfx_v7_0_get_csb_size(adev);
 
-		if (adev->gfx.rlc.clear_state_obj == NULL) {
-			r = amdgpu_bo_create(adev, dws * 4, PAGE_SIZE, true,
-					     AMDGPU_GEM_DOMAIN_VRAM,
-					     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
-					     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
-					     NULL, NULL,
-					     &adev->gfx.rlc.clear_state_obj);
-			if (r) {
-				dev_warn(adev->dev, "(%d) create RLC c bo failed\n", r);
-				gfx_v7_0_rlc_fini(adev);
-				return r;
-			}
-		}
-		r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, false);
-		if (unlikely(r != 0)) {
-			gfx_v7_0_rlc_fini(adev);
-			return r;
-		}
-		r = amdgpu_bo_pin(adev->gfx.rlc.clear_state_obj, AMDGPU_GEM_DOMAIN_VRAM,
-				  &adev->gfx.rlc.clear_state_gpu_addr);
+		r = amdgpu_bo_create_reserved(adev, dws * 4, PAGE_SIZE,
+					      AMDGPU_GEM_DOMAIN_VRAM,
+					      &adev->gfx.rlc.clear_state_obj,
+					      &adev->gfx.rlc.clear_state_gpu_addr,
+					      (void **)&adev->gfx.rlc.cs_ptr);
 		if (r) {
-			amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
-			dev_warn(adev->dev, "(%d) pin RLC c bo failed\n", r);
+			dev_warn(adev->dev, "(%d) create RLC c bo failed\n", r);
 			gfx_v7_0_rlc_fini(adev);
 			return r;
 		}
 
-		r = amdgpu_bo_kmap(adev->gfx.rlc.clear_state_obj, (void **)&adev->gfx.rlc.cs_ptr);
-		if (r) {
-			dev_warn(adev->dev, "(%d) map RLC c bo failed\n", r);
-			gfx_v7_0_rlc_fini(adev);
-			return r;
-		}
 		/* set up the cs buffer */
 		dst_ptr = adev->gfx.rlc.cs_ptr;
 		gfx_v7_0_get_csb_buffer(adev, dst_ptr);
@@ -3518,37 +3378,14 @@ static int gfx_v7_0_rlc_init(struct amdgpu_device *adev)
 	}
 
 	if (adev->gfx.rlc.cp_table_size) {
-		if (adev->gfx.rlc.cp_table_obj == NULL) {
-			r = amdgpu_bo_create(adev, adev->gfx.rlc.cp_table_size, PAGE_SIZE, true,
-					     AMDGPU_GEM_DOMAIN_VRAM,
-					     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
-					     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
-					     NULL, NULL,
-					     &adev->gfx.rlc.cp_table_obj);
-			if (r) {
-				dev_warn(adev->dev, "(%d) create RLC cp table bo failed\n", r);
-				gfx_v7_0_rlc_fini(adev);
-				return r;
-			}
-		}
 
-		r = amdgpu_bo_reserve(adev->gfx.rlc.cp_table_obj, false);
-		if (unlikely(r != 0)) {
-			dev_warn(adev->dev, "(%d) reserve RLC cp table bo failed\n", r);
-			gfx_v7_0_rlc_fini(adev);
-			return r;
-		}
-		r = amdgpu_bo_pin(adev->gfx.rlc.cp_table_obj, AMDGPU_GEM_DOMAIN_VRAM,
-				  &adev->gfx.rlc.cp_table_gpu_addr);
-		if (r) {
-			amdgpu_bo_unreserve(adev->gfx.rlc.cp_table_obj);
-			dev_warn(adev->dev, "(%d) pin RLC cp_table bo failed\n", r);
-			gfx_v7_0_rlc_fini(adev);
-			return r;
-		}
-		r = amdgpu_bo_kmap(adev->gfx.rlc.cp_table_obj, (void **)&adev->gfx.rlc.cp_table_ptr);
+		r = amdgpu_bo_create_reserved(adev, adev->gfx.rlc.cp_table_size,
+					      PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
+					      &adev->gfx.rlc.cp_table_obj,
+					      &adev->gfx.rlc.cp_table_gpu_addr,
+					      (void **)&adev->gfx.rlc.cp_table_ptr);
 		if (r) {
-			dev_warn(adev->dev, "(%d) map RLC cp table bo failed\n", r);
+			dev_warn(adev->dev, "(%d) create RLC cp table bo failed\n", r);
 			gfx_v7_0_rlc_fini(adev);
 			return r;
 		}
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index aa5a50f5eac8..fc260c13b1da 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -193,8 +193,8 @@ static const u32 tonga_golden_common_all[] =
 	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
 	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
 	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
-	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
-	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF
+	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
+	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF
 };
 
 static const u32 tonga_mgcg_cgcg_init[] =
@@ -303,8 +303,8 @@ static const u32 polaris11_golden_common_all[] =
 	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011002,
 	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
 	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
-	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
-	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
+	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
+	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
 };
 
 static const u32 golden_settings_polaris10_a11[] =
@@ -336,8 +336,8 @@ static const u32 polaris10_golden_common_all[] =
 	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
 	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
 	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
-	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
-	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
+	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
+	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
 };
 
 static const u32 fiji_golden_common_all[] =
@@ -348,8 +348,8 @@ static const u32 fiji_golden_common_all[] =
 	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
 	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
 	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
-	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
-	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
+	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
+	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
 	mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009,
 };
@@ -436,8 +436,8 @@ static const u32 iceland_golden_common_all[] =
 	mmGB_ADDR_CONFIG, 0xffffffff, 0x22010001,
 	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
 	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
-	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
-	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF
+	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
+	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF
 };
 
 static const u32 iceland_mgcg_cgcg_init[] =
@@ -532,8 +532,8 @@ static const u32 cz_golden_common_all[] =
 	mmGB_ADDR_CONFIG, 0xffffffff, 0x22010001,
 	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
 	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
-	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
-	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF
+	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
+	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF
 };
 
 static const u32 cz_mgcg_cgcg_init[] =
@@ -637,8 +637,8 @@ static const u32 stoney_golden_common_all[] =
 	mmGB_ADDR_CONFIG, 0xffffffff, 0x12010001,
 	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
 	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
-	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
-	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
+	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
+	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
 };
 
 static const u32 stoney_mgcg_cgcg_init[] =
@@ -750,7 +750,7 @@ static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev)
 
 static void gfx_v8_0_scratch_init(struct amdgpu_device *adev)
 {
-	adev->gfx.scratch.num_reg = 7;
+	adev->gfx.scratch.num_reg = 8;
 	adev->gfx.scratch.reg_base = mmSCRATCH_REG0;
 	adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
 }
@@ -1238,29 +1238,8 @@ static void cz_init_cp_jump_table(struct amdgpu_device *adev)
 
 static void gfx_v8_0_rlc_fini(struct amdgpu_device *adev)
 {
-	int r;
-
-	/* clear state block */
-	if (adev->gfx.rlc.clear_state_obj) {
-		r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, true);
-		if (unlikely(r != 0))
-			dev_warn(adev->dev, "(%d) reserve RLC cbs bo failed\n", r);
-		amdgpu_bo_unpin(adev->gfx.rlc.clear_state_obj);
-		amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
-		amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj);
-		adev->gfx.rlc.clear_state_obj = NULL;
-	}
-
-	/* jump table block */
-	if (adev->gfx.rlc.cp_table_obj) {
-		r = amdgpu_bo_reserve(adev->gfx.rlc.cp_table_obj, true);
-		if (unlikely(r != 0))
-			dev_warn(adev->dev, "(%d) reserve RLC cp table bo failed\n", r);
-		amdgpu_bo_unpin(adev->gfx.rlc.cp_table_obj);
-		amdgpu_bo_unreserve(adev->gfx.rlc.cp_table_obj);
-		amdgpu_bo_unref(&adev->gfx.rlc.cp_table_obj);
-		adev->gfx.rlc.cp_table_obj = NULL;
-	}
+	amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj, NULL, NULL);
+	amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, NULL, NULL);
 }
 
 static int gfx_v8_0_rlc_init(struct amdgpu_device *adev)
@@ -1278,39 +1257,17 @@ static int gfx_v8_0_rlc_init(struct amdgpu_device *adev)
 		/* clear state block */
 		adev->gfx.rlc.clear_state_size = dws = gfx_v8_0_get_csb_size(adev);
 
-		if (adev->gfx.rlc.clear_state_obj == NULL) {
-			r = amdgpu_bo_create(adev, dws * 4, PAGE_SIZE, true,
-					     AMDGPU_GEM_DOMAIN_VRAM,
-					     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
-					     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
-					     NULL, NULL,
-					     &adev->gfx.rlc.clear_state_obj);
-			if (r) {
-				dev_warn(adev->dev, "(%d) create RLC c bo failed\n", r);
-				gfx_v8_0_rlc_fini(adev);
-				return r;
-			}
-		}
-		r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, false);
-		if (unlikely(r != 0)) {
-			gfx_v8_0_rlc_fini(adev);
-			return r;
-		}
-		r = amdgpu_bo_pin(adev->gfx.rlc.clear_state_obj, AMDGPU_GEM_DOMAIN_VRAM,
-				  &adev->gfx.rlc.clear_state_gpu_addr);
+		r = amdgpu_bo_create_reserved(adev, dws * 4, PAGE_SIZE,
+					      AMDGPU_GEM_DOMAIN_VRAM,
+					      &adev->gfx.rlc.clear_state_obj,
+					      &adev->gfx.rlc.clear_state_gpu_addr,
+					      (void **)&adev->gfx.rlc.cs_ptr);
 		if (r) {
-			amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
-			dev_warn(adev->dev, "(%d) pin RLC cbs bo failed\n", r);
+			dev_warn(adev->dev, "(%d) create RLC c bo failed\n", r);
 			gfx_v8_0_rlc_fini(adev);
 			return r;
 		}
 
-		r = amdgpu_bo_kmap(adev->gfx.rlc.clear_state_obj, (void **)&adev->gfx.rlc.cs_ptr);
-		if (r) {
-			dev_warn(adev->dev, "(%d) map RLC cbs bo failed\n", r);
-			gfx_v8_0_rlc_fini(adev);
-			return r;
-		}
 		/* set up the cs buffer */
 		dst_ptr = adev->gfx.rlc.cs_ptr;
 		gfx_v8_0_get_csb_buffer(adev, dst_ptr);
@@ -1321,34 +1278,13 @@ static int gfx_v8_0_rlc_init(struct amdgpu_device *adev)
 	if ((adev->asic_type == CHIP_CARRIZO) ||
 	    (adev->asic_type == CHIP_STONEY)) {
 		adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
-		if (adev->gfx.rlc.cp_table_obj == NULL) {
-			r = amdgpu_bo_create(adev, adev->gfx.rlc.cp_table_size, PAGE_SIZE, true,
-					     AMDGPU_GEM_DOMAIN_VRAM,
-					     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
-					     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
-					     NULL, NULL,
-					     &adev->gfx.rlc.cp_table_obj);
-			if (r) {
-				dev_warn(adev->dev, "(%d) create RLC cp table bo failed\n", r);
-				return r;
-			}
-		}
-
-		r = amdgpu_bo_reserve(adev->gfx.rlc.cp_table_obj, false);
-		if (unlikely(r != 0)) {
-			dev_warn(adev->dev, "(%d) reserve RLC cp table bo failed\n", r);
-			return r;
-		}
-		r = amdgpu_bo_pin(adev->gfx.rlc.cp_table_obj, AMDGPU_GEM_DOMAIN_VRAM,
-				  &adev->gfx.rlc.cp_table_gpu_addr);
-		if (r) {
-			amdgpu_bo_unreserve(adev->gfx.rlc.cp_table_obj);
-			dev_warn(adev->dev, "(%d) pin RLC cp table bo failed\n", r);
-			return r;
-		}
-		r = amdgpu_bo_kmap(adev->gfx.rlc.cp_table_obj, (void **)&adev->gfx.rlc.cp_table_ptr);
+		r = amdgpu_bo_create_reserved(adev, adev->gfx.rlc.cp_table_size,
+					      PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
+					      &adev->gfx.rlc.cp_table_obj,
+					      &adev->gfx.rlc.cp_table_gpu_addr,
+					      (void **)&adev->gfx.rlc.cp_table_ptr);
 		if (r) {
-			dev_warn(adev->dev, "(%d) map RLC cp table bo failed\n", r);
+			dev_warn(adev->dev, "(%d) create RLC cp table bo failed\n", r);
 			return r;
 		}
 
@@ -1363,17 +1299,7 @@ static int gfx_v8_0_rlc_init(struct amdgpu_device *adev)
 
 static void gfx_v8_0_mec_fini(struct amdgpu_device *adev)
 {
-	int r;
-
-	if (adev->gfx.mec.hpd_eop_obj) {
-		r = amdgpu_bo_reserve(adev->gfx.mec.hpd_eop_obj, true);
-		if (unlikely(r != 0))
-			dev_warn(adev->dev, "(%d) reserve HPD EOP bo failed\n", r);
-		amdgpu_bo_unpin(adev->gfx.mec.hpd_eop_obj);
-		amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
-		amdgpu_bo_unref(&adev->gfx.mec.hpd_eop_obj);
-		adev->gfx.mec.hpd_eop_obj = NULL;
-	}
+	amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
 }
 
 static int gfx_v8_0_mec_init(struct amdgpu_device *adev)
@@ -1389,34 +1315,13 @@ static int gfx_v8_0_mec_init(struct amdgpu_device *adev)
 
 	mec_hpd_size = adev->gfx.num_compute_rings * GFX8_MEC_HPD_SIZE;
 
-	if (adev->gfx.mec.hpd_eop_obj == NULL) {
-		r = amdgpu_bo_create(adev,
-				     mec_hpd_size,
-				     PAGE_SIZE, true,
-				     AMDGPU_GEM_DOMAIN_GTT, 0, NULL, NULL,
-				     &adev->gfx.mec.hpd_eop_obj);
-		if (r) {
-			dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
-			return r;
-		}
-	}
-
-	r = amdgpu_bo_reserve(adev->gfx.mec.hpd_eop_obj, false);
-	if (unlikely(r != 0)) {
-		gfx_v8_0_mec_fini(adev);
-		return r;
-	}
-	r = amdgpu_bo_pin(adev->gfx.mec.hpd_eop_obj, AMDGPU_GEM_DOMAIN_GTT,
-			  &adev->gfx.mec.hpd_eop_gpu_addr);
+	r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
+				      AMDGPU_GEM_DOMAIN_GTT,
+				      &adev->gfx.mec.hpd_eop_obj,
+				      &adev->gfx.mec.hpd_eop_gpu_addr,
+				      (void **)&hpd);
 	if (r) {
-		dev_warn(adev->dev, "(%d) pin HDP EOP bo failed\n", r);
-		gfx_v8_0_mec_fini(adev);
-		return r;
-	}
-	r = amdgpu_bo_kmap(adev->gfx.mec.hpd_eop_obj, (void **)&hpd);
-	if (r) {
-		dev_warn(adev->dev, "(%d) map HDP EOP bo failed\n", r);
-		gfx_v8_0_mec_fini(adev);
+		dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
 		return r;
 	}
 
@@ -3802,6 +3707,8 @@ static void gfx_v8_0_gpu_init(struct amdgpu_device *adev)
 				   ELEMENT_SIZE, 1);
 	sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG,
 				   INDEX_STRIDE, 3);
+	WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg);
+
 	mutex_lock(&adev->srbm_mutex);
 	for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) {
 		vi_srbm_select(adev, 0, 0, 0, i);
@@ -3825,7 +3732,6 @@ static void gfx_v8_0_gpu_init(struct amdgpu_device *adev)
 
 		WREG32(mmSH_MEM_APE1_BASE, 1);
 		WREG32(mmSH_MEM_APE1_LIMIT, 0);
-		WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg);
 	}
 	vi_srbm_select(adev, 0, 0, 0, 0);
 	mutex_unlock(&adev->srbm_mutex);
@@ -4564,7 +4470,7 @@ static int gfx_v8_0_kiq_kcq_enable(struct amdgpu_device *adev)
 		/* This situation may be hit in the future if a new HW
 		 * generation exposes more than 64 queues. If so, the
 		 * definition of queue_mask needs updating */
-		if (WARN_ON(i > (sizeof(queue_mask)*8))) {
+		if (WARN_ON(i >= (sizeof(queue_mask)*8))) {
 			DRM_ERROR("Invalid KCQ enabled: %d\n", i);
 			break;
 		}
@@ -4673,9 +4579,9 @@ static int gfx_v8_0_mqd_init(struct amdgpu_ring *ring)
 	mqd->compute_misc_reserved = 0x00000003;
 	if (!(adev->flags & AMD_IS_APU)) {
 		mqd->dynamic_cu_mask_addr_lo = lower_32_bits(ring->mqd_gpu_addr
-					     + offsetof(struct vi_mqd_allocation, dyamic_cu_mask));
+					     + offsetof(struct vi_mqd_allocation, dynamic_cu_mask));
 		mqd->dynamic_cu_mask_addr_hi = upper_32_bits(ring->mqd_gpu_addr
-					     + offsetof(struct vi_mqd_allocation, dyamic_cu_mask));
+					     + offsetof(struct vi_mqd_allocation, dynamic_cu_mask));
 	}
 	eop_base_addr = ring->eop_gpu_addr >> 8;
 	mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
@@ -4862,8 +4768,8 @@ static int gfx_v8_0_kiq_init_queue(struct amdgpu_ring *ring)
 		mutex_unlock(&adev->srbm_mutex);
 	} else {
 		memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation));
-		((struct vi_mqd_allocation *)mqd)->dyamic_cu_mask = 0xFFFFFFFF;
-		((struct vi_mqd_allocation *)mqd)->dyamic_rb_mask = 0xFFFFFFFF;
+		((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
+		((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
 		mutex_lock(&adev->srbm_mutex);
 		vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
 		gfx_v8_0_mqd_init(ring);
@@ -4886,8 +4792,8 @@ static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring)
 
 	if (!adev->gfx.in_reset && !adev->gfx.in_suspend) {
 		memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation));
-		((struct vi_mqd_allocation *)mqd)->dyamic_cu_mask = 0xFFFFFFFF;
-		((struct vi_mqd_allocation *)mqd)->dyamic_rb_mask = 0xFFFFFFFF;
+		((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
+		((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
 		mutex_lock(&adev->srbm_mutex);
 		vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
 		gfx_v8_0_mqd_init(ring);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index c9b9c88231aa..69182eeca264 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -116,7 +116,9 @@ static const u32 golden_settings_gc_9_0[] =
 	SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_UTCL1_CNTL_2), 0x08000000, 0x08000080,
 	SOC15_REG_OFFSET(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL), 0x08000000, 0x08000080,
 	SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_UTCL1_CNTL), 0x08000000, 0x08000080,
+	SOC15_REG_OFFSET(GC, 0, mmSH_MEM_CONFIG), 0x00001000, 0x00001000,
 	SOC15_REG_OFFSET(GC, 0, mmSPI_CONFIG_CNTL_1), 0x0000000f, 0x01000107,
+	SOC15_REG_OFFSET(GC, 0, mmSQC_CONFIG), 0x03000000, 0x020a2000,
 	SOC15_REG_OFFSET(GC, 0, mmTA_CNTL_AUX), 0xfffffeef, 0x010b0000,
 	SOC15_REG_OFFSET(GC, 0, mmTCP_CHAN_STEER_HI), 0xffffffff, 0x4a2c0e68,
 	SOC15_REG_OFFSET(GC, 0, mmTCP_CHAN_STEER_LO), 0xffffffff, 0xb5d3f197,
@@ -211,7 +213,7 @@ static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
 
 static void gfx_v9_0_scratch_init(struct amdgpu_device *adev)
 {
-	adev->gfx.scratch.num_reg = 7;
+	adev->gfx.scratch.num_reg = 8;
 	adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
 	adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
 }
@@ -772,18 +774,16 @@ static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
 	if (cs_data) {
 		/* clear state block */
 		adev->gfx.rlc.clear_state_size = dws = gfx_v9_0_get_csb_size(adev);
-		if (adev->gfx.rlc.clear_state_obj == NULL) {
-			r = amdgpu_bo_create_kernel(adev, dws * 4, PAGE_SIZE,
-						AMDGPU_GEM_DOMAIN_VRAM,
-						&adev->gfx.rlc.clear_state_obj,
-						&adev->gfx.rlc.clear_state_gpu_addr,
-						(void **)&adev->gfx.rlc.cs_ptr);
-			if (r) {
-				dev_err(adev->dev,
-					"(%d) failed to create rlc csb bo\n", r);
-				gfx_v9_0_rlc_fini(adev);
-				return r;
-			}
+		r = amdgpu_bo_create_reserved(adev, dws * 4, PAGE_SIZE,
+					      AMDGPU_GEM_DOMAIN_VRAM,
+					      &adev->gfx.rlc.clear_state_obj,
+					      &adev->gfx.rlc.clear_state_gpu_addr,
+					      (void **)&adev->gfx.rlc.cs_ptr);
+		if (r) {
+			dev_err(adev->dev, "(%d) failed to create rlc csb bo\n",
+				r);
+			gfx_v9_0_rlc_fini(adev);
+			return r;
 		}
 		/* set up the cs buffer */
 		dst_ptr = adev->gfx.rlc.cs_ptr;
@@ -795,18 +795,16 @@ static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
 	if (adev->asic_type == CHIP_RAVEN) {
 		/* TODO: double check the cp_table_size for RV */
 		adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
-		if (adev->gfx.rlc.cp_table_obj == NULL) {
-			r = amdgpu_bo_create_kernel(adev, adev->gfx.rlc.cp_table_size,
-						PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
-						&adev->gfx.rlc.cp_table_obj,
-						&adev->gfx.rlc.cp_table_gpu_addr,
-						(void **)&adev->gfx.rlc.cp_table_ptr);
-			if (r) {
-				dev_err(adev->dev,
-					"(%d) failed to create cp table bo\n", r);
-				gfx_v9_0_rlc_fini(adev);
-				return r;
-			}
+		r = amdgpu_bo_create_reserved(adev, adev->gfx.rlc.cp_table_size,
+					      PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
+					      &adev->gfx.rlc.cp_table_obj,
+					      &adev->gfx.rlc.cp_table_gpu_addr,
+					      (void **)&adev->gfx.rlc.cp_table_ptr);
+		if (r) {
+			dev_err(adev->dev,
+				"(%d) failed to create cp table bo\n", r);
+			gfx_v9_0_rlc_fini(adev);
+			return r;
 		}
 
 		rv_init_cp_jump_table(adev);
@@ -821,28 +819,8 @@ static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
 
 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
 {
-	int r;
-
-	if (adev->gfx.mec.hpd_eop_obj) {
-		r = amdgpu_bo_reserve(adev->gfx.mec.hpd_eop_obj, true);
-		if (unlikely(r != 0))
-			dev_warn(adev->dev, "(%d) reserve HPD EOP bo failed\n", r);
-		amdgpu_bo_unpin(adev->gfx.mec.hpd_eop_obj);
-		amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
-
-		amdgpu_bo_unref(&adev->gfx.mec.hpd_eop_obj);
-		adev->gfx.mec.hpd_eop_obj = NULL;
-	}
-	if (adev->gfx.mec.mec_fw_obj) {
-		r = amdgpu_bo_reserve(adev->gfx.mec.mec_fw_obj, true);
-		if (unlikely(r != 0))
-			dev_warn(adev->dev, "(%d) reserve mec firmware bo failed\n", r);
-		amdgpu_bo_unpin(adev->gfx.mec.mec_fw_obj);
-		amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
-
-		amdgpu_bo_unref(&adev->gfx.mec.mec_fw_obj);
-		adev->gfx.mec.mec_fw_obj = NULL;
-	}
+	amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
+	amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
 }
 
 static int gfx_v9_0_mec_init(struct amdgpu_device *adev)
@@ -862,33 +840,13 @@ static int gfx_v9_0_mec_init(struct amdgpu_device *adev)
 	amdgpu_gfx_compute_queue_acquire(adev);
 	mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE;
 
-	if (adev->gfx.mec.hpd_eop_obj == NULL) {
-		r = amdgpu_bo_create(adev,
-				     mec_hpd_size,
-				     PAGE_SIZE, true,
-				     AMDGPU_GEM_DOMAIN_GTT, 0, NULL, NULL,
-				     &adev->gfx.mec.hpd_eop_obj);
-		if (r) {
-			dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
-			return r;
-		}
-	}
-
-	r = amdgpu_bo_reserve(adev->gfx.mec.hpd_eop_obj, false);
-	if (unlikely(r != 0)) {
-		gfx_v9_0_mec_fini(adev);
-		return r;
-	}
-	r = amdgpu_bo_pin(adev->gfx.mec.hpd_eop_obj, AMDGPU_GEM_DOMAIN_GTT,
-			  &adev->gfx.mec.hpd_eop_gpu_addr);
+	r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
+				      AMDGPU_GEM_DOMAIN_GTT,
+				      &adev->gfx.mec.hpd_eop_obj,
+				      &adev->gfx.mec.hpd_eop_gpu_addr,
+				      (void **)&hpd);
 	if (r) {
-		dev_warn(adev->dev, "(%d) pin HDP EOP bo failed\n", r);
-		gfx_v9_0_mec_fini(adev);
-		return r;
-	}
-	r = amdgpu_bo_kmap(adev->gfx.mec.hpd_eop_obj, (void **)&hpd);
-	if (r) {
-		dev_warn(adev->dev, "(%d) map HDP EOP bo failed\n", r);
+		dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
 		gfx_v9_0_mec_fini(adev);
 		return r;
 	}
@@ -905,42 +863,22 @@ static int gfx_v9_0_mec_init(struct amdgpu_device *adev)
 		 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
 	fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
 
-	if (adev->gfx.mec.mec_fw_obj == NULL) {
-		r = amdgpu_bo_create(adev,
-			mec_hdr->header.ucode_size_bytes,
-			PAGE_SIZE, true,
-			AMDGPU_GEM_DOMAIN_GTT, 0, NULL, NULL,
-			&adev->gfx.mec.mec_fw_obj);
-		if (r) {
-			dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);
-			return r;
-		}
-	}
-
-	r = amdgpu_bo_reserve(adev->gfx.mec.mec_fw_obj, false);
-	if (unlikely(r != 0)) {
-		gfx_v9_0_mec_fini(adev);
-		return r;
-	}
-	r = amdgpu_bo_pin(adev->gfx.mec.mec_fw_obj, AMDGPU_GEM_DOMAIN_GTT,
-			&adev->gfx.mec.mec_fw_gpu_addr);
-	if (r) {
-		dev_warn(adev->dev, "(%d) pin mec firmware bo failed\n", r);
-		gfx_v9_0_mec_fini(adev);
-		return r;
-	}
-	r = amdgpu_bo_kmap(adev->gfx.mec.mec_fw_obj, (void **)&fw);
+	r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
+				      PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
+				      &adev->gfx.mec.mec_fw_obj,
+				      &adev->gfx.mec.mec_fw_gpu_addr,
+				      (void **)&fw);
 	if (r) {
-		dev_warn(adev->dev, "(%d) map firmware bo failed\n", r);
+		dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);
 		gfx_v9_0_mec_fini(adev);
 		return r;
 	}
+
 	memcpy(fw, fw_data, fw_size);
 
 	amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
 	amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
 
-
 	return 0;
 }
 
@@ -2219,7 +2157,7 @@ static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
 	struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
 	const struct cs_section_def *sect = NULL;
 	const struct cs_extent_def *ext = NULL;
-	int r, i;
+	int r, i, tmp;
 
 	/* init the CP */
 	WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
@@ -2227,7 +2165,7 @@ static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
 
 	gfx_v9_0_cp_gfx_enable(adev, true);
 
-	r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4);
+	r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3);
 	if (r) {
 		DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
 		return r;
@@ -2265,6 +2203,12 @@ static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
 	amdgpu_ring_write(ring, 0x8000);
 	amdgpu_ring_write(ring, 0x8000);
 
+	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1));
+	tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE |
+		(SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START));
+	amdgpu_ring_write(ring, tmp);
+	amdgpu_ring_write(ring, 0);
+
 	amdgpu_ring_commit(ring);
 
 	return 0;
@@ -2427,7 +2371,7 @@ static int gfx_v9_0_kiq_kcq_enable(struct amdgpu_device *adev)
 		/* This situation may be hit in the future if a new HW
 		 * generation exposes more than 64 queues. If so, the
 		 * definition of queue_mask needs updating */
-		if (WARN_ON(i > (sizeof(queue_mask)*8))) {
+		if (WARN_ON(i >= (sizeof(queue_mask)*8))) {
 			DRM_ERROR("Invalid KCQ enabled: %d\n", i);
 			break;
 		}
@@ -4158,7 +4102,7 @@ static int gfx_v9_0_kiq_irq(struct amdgpu_device *adev,
 	return 0;
 }
 
-const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
+static const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
 	.name = "gfx_v9_0",
 	.early_init = gfx_v9_0_early_init,
 	.late_init = gfx_v9_0_late_init,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h
index 56ef652a575d..fa5a3fbaf6ab 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.h
@@ -24,7 +24,6 @@
 #ifndef __GFX_V9_0_H__
 #define __GFX_V9_0_H__
 
-extern const struct amd_ip_funcs gfx_v9_0_ip_funcs;
 extern const struct amdgpu_ip_block_version gfx_v9_0_ip_block;
 
 void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
index a42f483767e7..6c8040e616c4 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
@@ -58,14 +58,14 @@ static void gfxhub_v1_0_init_gart_aperture_regs(struct amdgpu_device *adev)
 	gfxhub_v1_0_init_gart_pt_regs(adev);
 
 	WREG32_SOC15(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
-		     (u32)(adev->mc.gtt_start >> 12));
+		     (u32)(adev->mc.gart_start >> 12));
 	WREG32_SOC15(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
-		     (u32)(adev->mc.gtt_start >> 44));
+		     (u32)(adev->mc.gart_start >> 44));
 
 	WREG32_SOC15(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
-		     (u32)(adev->mc.gtt_end >> 12));
+		     (u32)(adev->mc.gart_end >> 12));
 	WREG32_SOC15(GC, 0, mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
-		     (u32)(adev->mc.gtt_end >> 44));
+		     (u32)(adev->mc.gart_end >> 44));
 }
 
 static void gfxhub_v1_0_init_system_aperture_regs(struct amdgpu_device *adev)
@@ -129,7 +129,7 @@ static void gfxhub_v1_0_init_cache_regs(struct amdgpu_device *adev)
 	/* Setup L2 cache */
 	tmp = RREG32_SOC15(GC, 0, mmVM_L2_CNTL);
 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1);
-	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 0);
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 1);
 	/* XXX for emulation, Refer to closed source code.*/
 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, L2_PDE0_CACHE_TAG_GENERATION_MODE,
 			    0);
@@ -144,6 +144,8 @@ static void gfxhub_v1_0_init_cache_regs(struct amdgpu_device *adev)
 	WREG32_SOC15(GC, 0, mmVM_L2_CNTL2, tmp);
 
 	tmp = mmVM_L2_CNTL3_DEFAULT;
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 9);
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
 	WREG32_SOC15(GC, 0, mmVM_L2_CNTL3, tmp);
 
 	tmp = mmVM_L2_CNTL4_DEFAULT;
@@ -206,6 +208,9 @@ static void gfxhub_v1_0_setup_vmid_config(struct amdgpu_device *adev)
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 				PAGE_TABLE_BLOCK_SIZE,
 				adev->vm_manager.block_size - 9);
+		/* Send no-retry XNACK on fault to suppress VM fault storm. */
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+				    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0);
 		WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_CNTL, i, tmp);
 		WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, i*2, 0);
 		WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, i*2, 0);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.h b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.h
index d2dbb085f480..206e29cad753 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.h
@@ -30,7 +30,5 @@ void gfxhub_v1_0_set_fault_enable_default(struct amdgpu_device *adev,
 					  bool value);
 void gfxhub_v1_0_init(struct amdgpu_device *adev);
 u64 gfxhub_v1_0_get_mc_fb_offset(struct amdgpu_device *adev);
-extern const struct amd_ip_funcs gfxhub_v1_0_ip_funcs;
-extern const struct amdgpu_ip_block_version gfxhub_v1_0_ip_block;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
index d0214d942bfc..5be9c83dfcf7 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
@@ -66,14 +66,10 @@ static const u32 crtc_offsets[6] =
 	SI_CRTC5_REGISTER_OFFSET
 };
 
-static void gmc_v6_0_mc_stop(struct amdgpu_device *adev,
-			     struct amdgpu_mode_mc_save *save)
+static void gmc_v6_0_mc_stop(struct amdgpu_device *adev)
 {
 	u32 blackout;
 
-	if (adev->mode_info.num_crtc)
-		amdgpu_display_stop_mc_access(adev, save);
-
 	gmc_v6_0_wait_for_idle((void *)adev);
 
 	blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
@@ -90,8 +86,7 @@ static void gmc_v6_0_mc_stop(struct amdgpu_device *adev,
 
 }
 
-static void gmc_v6_0_mc_resume(struct amdgpu_device *adev,
-			       struct amdgpu_mode_mc_save *save)
+static void gmc_v6_0_mc_resume(struct amdgpu_device *adev)
 {
 	u32 tmp;
 
@@ -103,10 +98,6 @@ static void gmc_v6_0_mc_resume(struct amdgpu_device *adev,
 	tmp = REG_SET_FIELD(0, BIF_FB_EN, FB_READ_EN, 1);
 	tmp = REG_SET_FIELD(tmp, BIF_FB_EN, FB_WRITE_EN, 1);
 	WREG32(mmBIF_FB_EN, tmp);
-
-	if (adev->mode_info.num_crtc)
-		amdgpu_display_resume_mc_access(adev, save);
-
 }
 
 static int gmc_v6_0_init_microcode(struct amdgpu_device *adev)
@@ -228,20 +219,20 @@ static int gmc_v6_0_mc_load_microcode(struct amdgpu_device *adev)
 static void gmc_v6_0_vram_gtt_location(struct amdgpu_device *adev,
 				       struct amdgpu_mc *mc)
 {
+	u64 base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF;
+	base <<= 24;
+
 	if (mc->mc_vram_size > 0xFFC0000000ULL) {
 		dev_warn(adev->dev, "limiting VRAM\n");
 		mc->real_vram_size = 0xFFC0000000ULL;
 		mc->mc_vram_size = 0xFFC0000000ULL;
 	}
-	amdgpu_vram_location(adev, &adev->mc, 0);
-	adev->mc.gtt_base_align = 0;
-	amdgpu_gtt_location(adev, mc);
+	amdgpu_vram_location(adev, &adev->mc, base);
+	amdgpu_gart_location(adev, mc);
 }
 
 static void gmc_v6_0_mc_program(struct amdgpu_device *adev)
 {
-	struct amdgpu_mode_mc_save save;
-	u32 tmp;
 	int i, j;
 
 	/* Initialize HDP */
@@ -254,16 +245,23 @@ static void gmc_v6_0_mc_program(struct amdgpu_device *adev)
 	}
 	WREG32(mmHDP_REG_COHERENCY_FLUSH_CNTL, 0);
 
-	if (adev->mode_info.num_crtc)
-		amdgpu_display_set_vga_render_state(adev, false);
-
-	gmc_v6_0_mc_stop(adev, &save);
-
 	if (gmc_v6_0_wait_for_idle((void *)adev)) {
 		dev_warn(adev->dev, "Wait for MC idle timedout !\n");
 	}
 
-	WREG32(mmVGA_HDP_CONTROL, VGA_HDP_CONTROL__VGA_MEMORY_DISABLE_MASK);
+	if (adev->mode_info.num_crtc) {
+		u32 tmp;
+
+		/* Lockout access through VGA aperture*/
+		tmp = RREG32(mmVGA_HDP_CONTROL);
+		tmp |= VGA_HDP_CONTROL__VGA_MEMORY_DISABLE_MASK;
+		WREG32(mmVGA_HDP_CONTROL, tmp);
+
+		/* disable VGA render */
+		tmp = RREG32(mmVGA_RENDER_CONTROL);
+		tmp &= ~VGA_VSTATUS_CNTL;
+		WREG32(mmVGA_RENDER_CONTROL, tmp);
+	}
 	/* Update configuration */
 	WREG32(mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,
 	       adev->mc.vram_start >> 12);
@@ -271,13 +269,6 @@ static void gmc_v6_0_mc_program(struct amdgpu_device *adev)
 	       adev->mc.vram_end >> 12);
 	WREG32(mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
 	       adev->vram_scratch.gpu_addr >> 12);
-	tmp = ((adev->mc.vram_end >> 24) & 0xFFFF) << 16;
-	tmp |= ((adev->mc.vram_start >> 24) & 0xFFFF);
-	WREG32(mmMC_VM_FB_LOCATION, tmp);
-	/* XXX double check these! */
-	WREG32(mmHDP_NONSURFACE_BASE, (adev->mc.vram_start >> 8));
-	WREG32(mmHDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
-	WREG32(mmHDP_NONSURFACE_SIZE, 0x3FFFFFFF);
 	WREG32(mmMC_VM_AGP_BASE, 0);
 	WREG32(mmMC_VM_AGP_TOP, 0x0FFFFFFF);
 	WREG32(mmMC_VM_AGP_BOT, 0x0FFFFFFF);
@@ -285,7 +276,6 @@ static void gmc_v6_0_mc_program(struct amdgpu_device *adev)
 	if (gmc_v6_0_wait_for_idle((void *)adev)) {
 		dev_warn(adev->dev, "Wait for MC idle timedout !\n");
 	}
-	gmc_v6_0_mc_resume(adev, &save);
 }
 
 static int gmc_v6_0_mc_init(struct amdgpu_device *adev)
@@ -342,14 +332,23 @@ static int gmc_v6_0_mc_init(struct amdgpu_device *adev)
 	adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
 	adev->mc.visible_vram_size = adev->mc.aper_size;
 
-	/* unless the user had overridden it, set the gart
-	 * size equal to the 1024 or vram, whichever is larger.
-	 */
-	if (amdgpu_gart_size == -1)
-		adev->mc.gtt_size = max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
-					adev->mc.mc_vram_size);
-	else
-		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
+	/* set the gart size */
+	if (amdgpu_gart_size == -1) {
+		switch (adev->asic_type) {
+		case CHIP_HAINAN:    /* no MM engines */
+		default:
+			adev->mc.gart_size = 256ULL << 20;
+			break;
+		case CHIP_VERDE:    /* UVD, VCE do not support GPUVM */
+		case CHIP_TAHITI:   /* UVD, VCE do not support GPUVM */
+		case CHIP_PITCAIRN: /* UVD, VCE do not support GPUVM */
+		case CHIP_OLAND:    /* UVD, VCE do not support GPUVM */
+			adev->mc.gart_size = 1024ULL << 20;
+			break;
+		}
+	} else {
+		adev->mc.gart_size = (u64)amdgpu_gart_size << 20;
+	}
 
 	gmc_v6_0_vram_gtt_location(adev, &adev->mc);
 
@@ -479,6 +478,7 @@ static void gmc_v6_0_set_prt(struct amdgpu_device *adev, bool enable)
 static int gmc_v6_0_gart_enable(struct amdgpu_device *adev)
 {
 	int r, i;
+	u32 field;
 
 	if (adev->gart.robj == NULL) {
 		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
@@ -506,13 +506,15 @@ static int gmc_v6_0_gart_enable(struct amdgpu_device *adev)
 	WREG32(mmVM_L2_CNTL2,
 	       VM_L2_CNTL2__INVALIDATE_ALL_L1_TLBS_MASK |
 	       VM_L2_CNTL2__INVALIDATE_L2_CACHE_MASK);
+
+	field = adev->vm_manager.fragment_size;
 	WREG32(mmVM_L2_CNTL3,
 	       VM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY_MASK |
-	       (4UL << VM_L2_CNTL3__BANK_SELECT__SHIFT) |
-	       (4UL << VM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT));
+	       (field << VM_L2_CNTL3__BANK_SELECT__SHIFT) |
+	       (field << VM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT));
 	/* setup context0 */
-	WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->mc.gtt_start >> 12);
-	WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->mc.gtt_end >> 12);
+	WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->mc.gart_start >> 12);
+	WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->mc.gart_end >> 12);
 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, adev->gart.table_addr >> 12);
 	WREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
 			(u32)(adev->dummy_page.addr >> 12));
@@ -559,7 +561,7 @@ static int gmc_v6_0_gart_enable(struct amdgpu_device *adev)
 
 	gmc_v6_0_gart_flush_gpu_tlb(adev, 0);
 	dev_info(adev->dev, "PCIE GART of %uM enabled (table at 0x%016llX).\n",
-		 (unsigned)(adev->mc.gtt_size >> 20),
+		 (unsigned)(adev->mc.gart_size >> 20),
 		 (unsigned long long)adev->gart.table_addr);
 	adev->gart.ready = true;
 	return 0;
@@ -829,7 +831,7 @@ static int gmc_v6_0_sw_init(void *handle)
 	if (r)
 		return r;
 
-	amdgpu_vm_adjust_size(adev, 64);
+	amdgpu_vm_adjust_size(adev, 64, 4);
 	adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
 
 	adev->mc.mc_mask = 0xffffffffffULL;
@@ -987,7 +989,6 @@ static int gmc_v6_0_wait_for_idle(void *handle)
 static int gmc_v6_0_soft_reset(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-	struct amdgpu_mode_mc_save save;
 	u32 srbm_soft_reset = 0;
 	u32 tmp = RREG32(mmSRBM_STATUS);
 
@@ -1003,7 +1004,7 @@ static int gmc_v6_0_soft_reset(void *handle)
 	}
 
 	if (srbm_soft_reset) {
-		gmc_v6_0_mc_stop(adev, &save);
+		gmc_v6_0_mc_stop(adev);
 		if (gmc_v6_0_wait_for_idle(adev)) {
 			dev_warn(adev->dev, "Wait for GMC idle timed out !\n");
 		}
@@ -1023,7 +1024,7 @@ static int gmc_v6_0_soft_reset(void *handle)
 
 		udelay(50);
 
-		gmc_v6_0_mc_resume(adev, &save);
+		gmc_v6_0_mc_resume(adev);
 		udelay(50);
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
index 7e9ea53edf8b..eace9e7182c8 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
@@ -37,6 +37,9 @@
 #include "oss/oss_2_0_d.h"
 #include "oss/oss_2_0_sh_mask.h"
 
+#include "dce/dce_8_0_d.h"
+#include "dce/dce_8_0_sh_mask.h"
+
 #include "amdgpu_atombios.h"
 
 static void gmc_v7_0_set_gart_funcs(struct amdgpu_device *adev);
@@ -76,14 +79,10 @@ static void gmc_v7_0_init_golden_registers(struct amdgpu_device *adev)
 	}
 }
 
-static void gmc_v7_0_mc_stop(struct amdgpu_device *adev,
-			     struct amdgpu_mode_mc_save *save)
+static void gmc_v7_0_mc_stop(struct amdgpu_device *adev)
 {
 	u32 blackout;
 
-	if (adev->mode_info.num_crtc)
-		amdgpu_display_stop_mc_access(adev, save);
-
 	gmc_v7_0_wait_for_idle((void *)adev);
 
 	blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
@@ -99,8 +98,7 @@ static void gmc_v7_0_mc_stop(struct amdgpu_device *adev,
 	udelay(100);
 }
 
-static void gmc_v7_0_mc_resume(struct amdgpu_device *adev,
-			       struct amdgpu_mode_mc_save *save)
+static void gmc_v7_0_mc_resume(struct amdgpu_device *adev)
 {
 	u32 tmp;
 
@@ -112,9 +110,6 @@ static void gmc_v7_0_mc_resume(struct amdgpu_device *adev,
 	tmp = REG_SET_FIELD(0, BIF_FB_EN, FB_READ_EN, 1);
 	tmp = REG_SET_FIELD(tmp, BIF_FB_EN, FB_WRITE_EN, 1);
 	WREG32(mmBIF_FB_EN, tmp);
-
-	if (adev->mode_info.num_crtc)
-		amdgpu_display_resume_mc_access(adev, save);
 }
 
 /**
@@ -242,15 +237,17 @@ static int gmc_v7_0_mc_load_microcode(struct amdgpu_device *adev)
 static void gmc_v7_0_vram_gtt_location(struct amdgpu_device *adev,
 				       struct amdgpu_mc *mc)
 {
+	u64 base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF;
+	base <<= 24;
+
 	if (mc->mc_vram_size > 0xFFC0000000ULL) {
 		/* leave room for at least 1024M GTT */
 		dev_warn(adev->dev, "limiting VRAM\n");
 		mc->real_vram_size = 0xFFC0000000ULL;
 		mc->mc_vram_size = 0xFFC0000000ULL;
 	}
-	amdgpu_vram_location(adev, &adev->mc, 0);
-	adev->mc.gtt_base_align = 0;
-	amdgpu_gtt_location(adev, mc);
+	amdgpu_vram_location(adev, &adev->mc, base);
+	amdgpu_gart_location(adev, mc);
 }
 
 /**
@@ -263,7 +260,6 @@ static void gmc_v7_0_vram_gtt_location(struct amdgpu_device *adev,
  */
 static void gmc_v7_0_mc_program(struct amdgpu_device *adev)
 {
-	struct amdgpu_mode_mc_save save;
 	u32 tmp;
 	int i, j;
 
@@ -277,13 +273,20 @@ static void gmc_v7_0_mc_program(struct amdgpu_device *adev)
 	}
 	WREG32(mmHDP_REG_COHERENCY_FLUSH_CNTL, 0);
 
-	if (adev->mode_info.num_crtc)
-		amdgpu_display_set_vga_render_state(adev, false);
-
-	gmc_v7_0_mc_stop(adev, &save);
 	if (gmc_v7_0_wait_for_idle((void *)adev)) {
 		dev_warn(adev->dev, "Wait for MC idle timedout !\n");
 	}
+	if (adev->mode_info.num_crtc) {
+		/* Lockout access through VGA aperture*/
+		tmp = RREG32(mmVGA_HDP_CONTROL);
+		tmp = REG_SET_FIELD(tmp, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
+		WREG32(mmVGA_HDP_CONTROL, tmp);
+
+		/* disable VGA render */
+		tmp = RREG32(mmVGA_RENDER_CONTROL);
+		tmp = REG_SET_FIELD(tmp, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
+		WREG32(mmVGA_RENDER_CONTROL, tmp);
+	}
 	/* Update configuration */
 	WREG32(mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,
 	       adev->mc.vram_start >> 12);
@@ -291,20 +294,12 @@ static void gmc_v7_0_mc_program(struct amdgpu_device *adev)
 	       adev->mc.vram_end >> 12);
 	WREG32(mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
 	       adev->vram_scratch.gpu_addr >> 12);
-	tmp = ((adev->mc.vram_end >> 24) & 0xFFFF) << 16;
-	tmp |= ((adev->mc.vram_start >> 24) & 0xFFFF);
-	WREG32(mmMC_VM_FB_LOCATION, tmp);
-	/* XXX double check these! */
-	WREG32(mmHDP_NONSURFACE_BASE, (adev->mc.vram_start >> 8));
-	WREG32(mmHDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
-	WREG32(mmHDP_NONSURFACE_SIZE, 0x3FFFFFFF);
 	WREG32(mmMC_VM_AGP_BASE, 0);
 	WREG32(mmMC_VM_AGP_TOP, 0x0FFFFFFF);
 	WREG32(mmMC_VM_AGP_BOT, 0x0FFFFFFF);
 	if (gmc_v7_0_wait_for_idle((void *)adev)) {
 		dev_warn(adev->dev, "Wait for MC idle timedout !\n");
 	}
-	gmc_v7_0_mc_resume(adev, &save);
 
 	WREG32(mmBIF_FB_EN, BIF_FB_EN__FB_READ_EN_MASK | BIF_FB_EN__FB_WRITE_EN_MASK);
 
@@ -391,14 +386,26 @@ static int gmc_v7_0_mc_init(struct amdgpu_device *adev)
 	if (adev->mc.visible_vram_size > adev->mc.real_vram_size)
 		adev->mc.visible_vram_size = adev->mc.real_vram_size;
 
-	/* unless the user had overridden it, set the gart
-	 * size equal to the 1024 or vram, whichever is larger.
-	 */
-	if (amdgpu_gart_size == -1)
-		adev->mc.gtt_size = max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
-					adev->mc.mc_vram_size);
-	else
-		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
+	/* set the gart size */
+	if (amdgpu_gart_size == -1) {
+		switch (adev->asic_type) {
+		case CHIP_TOPAZ:     /* no MM engines */
+		default:
+			adev->mc.gart_size = 256ULL << 20;
+			break;
+#ifdef CONFIG_DRM_AMDGPU_CIK
+		case CHIP_BONAIRE: /* UVD, VCE do not support GPUVM */
+		case CHIP_HAWAII:  /* UVD, VCE do not support GPUVM */
+		case CHIP_KAVERI:  /* UVD, VCE do not support GPUVM */
+		case CHIP_KABINI:  /* UVD, VCE do not support GPUVM */
+		case CHIP_MULLINS: /* UVD, VCE do not support GPUVM */
+			adev->mc.gart_size = 1024ULL << 20;
+			break;
+#endif
+		}
+	} else {
+		adev->mc.gart_size = (u64)amdgpu_gart_size << 20;
+	}
 
 	gmc_v7_0_vram_gtt_location(adev, &adev->mc);
 
@@ -575,7 +582,7 @@ static void gmc_v7_0_set_prt(struct amdgpu_device *adev, bool enable)
 static int gmc_v7_0_gart_enable(struct amdgpu_device *adev)
 {
 	int r, i;
-	u32 tmp;
+	u32 tmp, field;
 
 	if (adev->gart.robj == NULL) {
 		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
@@ -605,14 +612,16 @@ static int gmc_v7_0_gart_enable(struct amdgpu_device *adev)
 	tmp = REG_SET_FIELD(0, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
 	WREG32(mmVM_L2_CNTL2, tmp);
+
+	field = adev->vm_manager.fragment_size;
 	tmp = RREG32(mmVM_L2_CNTL3);
 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY, 1);
-	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 4);
-	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, 4);
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, field);
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, field);
 	WREG32(mmVM_L2_CNTL3, tmp);
 	/* setup context0 */
-	WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->mc.gtt_start >> 12);
-	WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->mc.gtt_end >> 12);
+	WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->mc.gart_start >> 12);
+	WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->mc.gart_end >> 12);
 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, adev->gart.table_addr >> 12);
 	WREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
 			(u32)(adev->dummy_page.addr >> 12));
@@ -666,7 +675,7 @@ static int gmc_v7_0_gart_enable(struct amdgpu_device *adev)
 
 	gmc_v7_0_gart_flush_gpu_tlb(adev, 0);
 	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
-		 (unsigned)(adev->mc.gtt_size >> 20),
+		 (unsigned)(adev->mc.gart_size >> 20),
 		 (unsigned long long)adev->gart.table_addr);
 	adev->gart.ready = true;
 	return 0;
@@ -961,7 +970,7 @@ static int gmc_v7_0_sw_init(void *handle)
 	 * Currently set to 4GB ((1 << 20) 4k pages).
 	 * Max GPUVM size for cayman and SI is 40 bits.
 	 */
-	amdgpu_vm_adjust_size(adev, 64);
+	amdgpu_vm_adjust_size(adev, 64, 4);
 	adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
 
 	/* Set the internal MC address mask
@@ -1138,7 +1147,6 @@ static int gmc_v7_0_wait_for_idle(void *handle)
 static int gmc_v7_0_soft_reset(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-	struct amdgpu_mode_mc_save save;
 	u32 srbm_soft_reset = 0;
 	u32 tmp = RREG32(mmSRBM_STATUS);
 
@@ -1154,7 +1162,7 @@ static int gmc_v7_0_soft_reset(void *handle)
 	}
 
 	if (srbm_soft_reset) {
-		gmc_v7_0_mc_stop(adev, &save);
+		gmc_v7_0_mc_stop(adev);
 		if (gmc_v7_0_wait_for_idle((void *)adev)) {
 			dev_warn(adev->dev, "Wait for GMC idle timed out !\n");
 		}
@@ -1175,7 +1183,7 @@ static int gmc_v7_0_soft_reset(void *handle)
 		/* Wait a little for things to settle down */
 		udelay(50);
 
-		gmc_v7_0_mc_resume(adev, &save);
+		gmc_v7_0_mc_resume(adev);
 		udelay(50);
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index cc9f88057cd5..3b3326daf32b 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -35,6 +35,9 @@
 #include "oss/oss_3_0_d.h"
 #include "oss/oss_3_0_sh_mask.h"
 
+#include "dce/dce_10_0_d.h"
+#include "dce/dce_10_0_sh_mask.h"
+
 #include "vid.h"
 #include "vi.h"
 
@@ -161,14 +164,10 @@ static void gmc_v8_0_init_golden_registers(struct amdgpu_device *adev)
 	}
 }
 
-static void gmc_v8_0_mc_stop(struct amdgpu_device *adev,
-			     struct amdgpu_mode_mc_save *save)
+static void gmc_v8_0_mc_stop(struct amdgpu_device *adev)
 {
 	u32 blackout;
 
-	if (adev->mode_info.num_crtc)
-		amdgpu_display_stop_mc_access(adev, save);
-
 	gmc_v8_0_wait_for_idle(adev);
 
 	blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
@@ -184,8 +183,7 @@ static void gmc_v8_0_mc_stop(struct amdgpu_device *adev,
 	udelay(100);
 }
 
-static void gmc_v8_0_mc_resume(struct amdgpu_device *adev,
-			       struct amdgpu_mode_mc_save *save)
+static void gmc_v8_0_mc_resume(struct amdgpu_device *adev)
 {
 	u32 tmp;
 
@@ -197,9 +195,6 @@ static void gmc_v8_0_mc_resume(struct amdgpu_device *adev,
 	tmp = REG_SET_FIELD(0, BIF_FB_EN, FB_READ_EN, 1);
 	tmp = REG_SET_FIELD(tmp, BIF_FB_EN, FB_WRITE_EN, 1);
 	WREG32(mmBIF_FB_EN, tmp);
-
-	if (adev->mode_info.num_crtc)
-		amdgpu_display_resume_mc_access(adev, save);
 }
 
 /**
@@ -404,15 +399,20 @@ static int gmc_v8_0_polaris_mc_load_microcode(struct amdgpu_device *adev)
 static void gmc_v8_0_vram_gtt_location(struct amdgpu_device *adev,
 				       struct amdgpu_mc *mc)
 {
+	u64 base = 0;
+
+	if (!amdgpu_sriov_vf(adev))
+		base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF;
+	base <<= 24;
+
 	if (mc->mc_vram_size > 0xFFC0000000ULL) {
 		/* leave room for at least 1024M GTT */
 		dev_warn(adev->dev, "limiting VRAM\n");
 		mc->real_vram_size = 0xFFC0000000ULL;
 		mc->mc_vram_size = 0xFFC0000000ULL;
 	}
-	amdgpu_vram_location(adev, &adev->mc, 0);
-	adev->mc.gtt_base_align = 0;
-	amdgpu_gtt_location(adev, mc);
+	amdgpu_vram_location(adev, &adev->mc, base);
+	amdgpu_gart_location(adev, mc);
 }
 
 /**
@@ -425,7 +425,6 @@ static void gmc_v8_0_vram_gtt_location(struct amdgpu_device *adev,
  */
 static void gmc_v8_0_mc_program(struct amdgpu_device *adev)
 {
-	struct amdgpu_mode_mc_save save;
 	u32 tmp;
 	int i, j;
 
@@ -439,13 +438,20 @@ static void gmc_v8_0_mc_program(struct amdgpu_device *adev)
 	}
 	WREG32(mmHDP_REG_COHERENCY_FLUSH_CNTL, 0);
 
-	if (adev->mode_info.num_crtc)
-		amdgpu_display_set_vga_render_state(adev, false);
-
-	gmc_v8_0_mc_stop(adev, &save);
 	if (gmc_v8_0_wait_for_idle((void *)adev)) {
 		dev_warn(adev->dev, "Wait for MC idle timedout !\n");
 	}
+	if (adev->mode_info.num_crtc) {
+		/* Lockout access through VGA aperture*/
+		tmp = RREG32(mmVGA_HDP_CONTROL);
+		tmp = REG_SET_FIELD(tmp, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
+		WREG32(mmVGA_HDP_CONTROL, tmp);
+
+		/* disable VGA render */
+		tmp = RREG32(mmVGA_RENDER_CONTROL);
+		tmp = REG_SET_FIELD(tmp, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
+		WREG32(mmVGA_RENDER_CONTROL, tmp);
+	}
 	/* Update configuration */
 	WREG32(mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,
 	       adev->mc.vram_start >> 12);
@@ -453,20 +459,23 @@ static void gmc_v8_0_mc_program(struct amdgpu_device *adev)
 	       adev->mc.vram_end >> 12);
 	WREG32(mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
 	       adev->vram_scratch.gpu_addr >> 12);
-	tmp = ((adev->mc.vram_end >> 24) & 0xFFFF) << 16;
-	tmp |= ((adev->mc.vram_start >> 24) & 0xFFFF);
-	WREG32(mmMC_VM_FB_LOCATION, tmp);
-	/* XXX double check these! */
-	WREG32(mmHDP_NONSURFACE_BASE, (adev->mc.vram_start >> 8));
-	WREG32(mmHDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
-	WREG32(mmHDP_NONSURFACE_SIZE, 0x3FFFFFFF);
+
+	if (amdgpu_sriov_vf(adev)) {
+		tmp = ((adev->mc.vram_end >> 24) & 0xFFFF) << 16;
+		tmp |= ((adev->mc.vram_start >> 24) & 0xFFFF);
+		WREG32(mmMC_VM_FB_LOCATION, tmp);
+		/* XXX double check these! */
+		WREG32(mmHDP_NONSURFACE_BASE, (adev->mc.vram_start >> 8));
+		WREG32(mmHDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
+		WREG32(mmHDP_NONSURFACE_SIZE, 0x3FFFFFFF);
+	}
+
 	WREG32(mmMC_VM_AGP_BASE, 0);
 	WREG32(mmMC_VM_AGP_TOP, 0x0FFFFFFF);
 	WREG32(mmMC_VM_AGP_BOT, 0x0FFFFFFF);
 	if (gmc_v8_0_wait_for_idle((void *)adev)) {
 		dev_warn(adev->dev, "Wait for MC idle timedout !\n");
 	}
-	gmc_v8_0_mc_resume(adev, &save);
 
 	WREG32(mmBIF_FB_EN, BIF_FB_EN__FB_READ_EN_MASK | BIF_FB_EN__FB_WRITE_EN_MASK);
 
@@ -553,14 +562,25 @@ static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
 	if (adev->mc.visible_vram_size > adev->mc.real_vram_size)
 		adev->mc.visible_vram_size = adev->mc.real_vram_size;
 
-	/* unless the user had overridden it, set the gart
-	 * size equal to the 1024 or vram, whichever is larger.
-	 */
-	if (amdgpu_gart_size == -1)
-		adev->mc.gtt_size = max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
-					adev->mc.mc_vram_size);
-	else
-		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
+	/* set the gart size */
+	if (amdgpu_gart_size == -1) {
+		switch (adev->asic_type) {
+		case CHIP_POLARIS11: /* all engines support GPUVM */
+		case CHIP_POLARIS10: /* all engines support GPUVM */
+		case CHIP_POLARIS12: /* all engines support GPUVM */
+		default:
+			adev->mc.gart_size = 256ULL << 20;
+			break;
+		case CHIP_TONGA:   /* UVD, VCE do not support GPUVM */
+		case CHIP_FIJI:    /* UVD, VCE do not support GPUVM */
+		case CHIP_CARRIZO: /* UVD, VCE do not support GPUVM, DCE SG support */
+		case CHIP_STONEY:  /* UVD does not support GPUVM, DCE SG support */
+			adev->mc.gart_size = 1024ULL << 20;
+			break;
+		}
+	} else {
+		adev->mc.gart_size = (u64)amdgpu_gart_size << 20;
+	}
 
 	gmc_v8_0_vram_gtt_location(adev, &adev->mc);
 
@@ -761,7 +781,7 @@ static void gmc_v8_0_set_prt(struct amdgpu_device *adev, bool enable)
 static int gmc_v8_0_gart_enable(struct amdgpu_device *adev)
 {
 	int r, i;
-	u32 tmp;
+	u32 tmp, field;
 
 	if (adev->gart.robj == NULL) {
 		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
@@ -792,10 +812,12 @@ static int gmc_v8_0_gart_enable(struct amdgpu_device *adev)
 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
 	WREG32(mmVM_L2_CNTL2, tmp);
+
+	field = adev->vm_manager.fragment_size;
 	tmp = RREG32(mmVM_L2_CNTL3);
 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY, 1);
-	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 4);
-	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, 4);
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, field);
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, field);
 	WREG32(mmVM_L2_CNTL3, tmp);
 	/* XXX: set to enable PTE/PDE in system memory */
 	tmp = RREG32(mmVM_L2_CNTL4);
@@ -813,8 +835,8 @@ static int gmc_v8_0_gart_enable(struct amdgpu_device *adev)
 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PTE_REQUEST_SNOOP, 0);
 	WREG32(mmVM_L2_CNTL4, tmp);
 	/* setup context0 */
-	WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->mc.gtt_start >> 12);
-	WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->mc.gtt_end >> 12);
+	WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->mc.gart_start >> 12);
+	WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->mc.gart_end >> 12);
 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, adev->gart.table_addr >> 12);
 	WREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
 			(u32)(adev->dummy_page.addr >> 12));
@@ -869,7 +891,7 @@ static int gmc_v8_0_gart_enable(struct amdgpu_device *adev)
 
 	gmc_v8_0_gart_flush_gpu_tlb(adev, 0);
 	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
-		 (unsigned)(adev->mc.gtt_size >> 20),
+		 (unsigned)(adev->mc.gart_size >> 20),
 		 (unsigned long long)adev->gart.table_addr);
 	adev->gart.ready = true;
 	return 0;
@@ -1045,7 +1067,7 @@ static int gmc_v8_0_sw_init(void *handle)
 	 * Currently set to 4GB ((1 << 20) 4k pages).
 	 * Max GPUVM size for cayman and SI is 40 bits.
 	 */
-	amdgpu_vm_adjust_size(adev, 64);
+	amdgpu_vm_adjust_size(adev, 64, 4);
 	adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
 
 	/* Set the internal MC address mask
@@ -1260,7 +1282,7 @@ static int gmc_v8_0_pre_soft_reset(void *handle)
 	if (!adev->mc.srbm_soft_reset)
 		return 0;
 
-	gmc_v8_0_mc_stop(adev, &adev->mc.save);
+	gmc_v8_0_mc_stop(adev);
 	if (gmc_v8_0_wait_for_idle(adev)) {
 		dev_warn(adev->dev, "Wait for GMC idle timed out !\n");
 	}
@@ -1306,7 +1328,7 @@ static int gmc_v8_0_post_soft_reset(void *handle)
 	if (!adev->mc.srbm_soft_reset)
 		return 0;
 
-	gmc_v8_0_mc_resume(adev, &adev->mc.save);
+	gmc_v8_0_mc_resume(adev);
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index 175ba5f9691c..d04d0b123212 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -23,11 +23,14 @@
 #include <linux/firmware.h>
 #include "amdgpu.h"
 #include "gmc_v9_0.h"
+#include "amdgpu_atomfirmware.h"
 
 #include "vega10/soc15ip.h"
 #include "vega10/HDP/hdp_4_0_offset.h"
 #include "vega10/HDP/hdp_4_0_sh_mask.h"
 #include "vega10/GC/gc_9_0_sh_mask.h"
+#include "vega10/DC/dce_12_0_offset.h"
+#include "vega10/DC/dce_12_0_sh_mask.h"
 #include "vega10/vega10_enum.h"
 
 #include "soc15_common.h"
@@ -419,8 +422,7 @@ static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,
 	if (!amdgpu_sriov_vf(adev))
 		base = mmhub_v1_0_get_fb_location(adev);
 	amdgpu_vram_location(adev, &adev->mc, base);
-	adev->mc.gtt_base_align = 0;
-	amdgpu_gtt_location(adev, mc);
+	amdgpu_gart_location(adev, mc);
 	/* base offset of vram pages */
 	if (adev->flags & AMD_IS_APU)
 		adev->vm_manager.vram_base_offset = gfxhub_v1_0_get_mc_fb_offset(adev);
@@ -442,43 +444,46 @@ static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
 	u32 tmp;
 	int chansize, numchan;
 
-	/* hbm memory channel size */
-	chansize = 128;
-
-	tmp = RREG32_SOC15(DF, 0, mmDF_CS_AON0_DramBaseAddress0);
-	tmp &= DF_CS_AON0_DramBaseAddress0__IntLvNumChan_MASK;
-	tmp >>= DF_CS_AON0_DramBaseAddress0__IntLvNumChan__SHIFT;
-	switch (tmp) {
-	case 0:
-	default:
-		numchan = 1;
-		break;
-	case 1:
-		numchan = 2;
-		break;
-	case 2:
-		numchan = 0;
-		break;
-	case 3:
-		numchan = 4;
-		break;
-	case 4:
-		numchan = 0;
-		break;
-	case 5:
-		numchan = 8;
-		break;
-	case 6:
-		numchan = 0;
-		break;
-	case 7:
-		numchan = 16;
-		break;
-	case 8:
-		numchan = 2;
-		break;
+	adev->mc.vram_width = amdgpu_atomfirmware_get_vram_width(adev);
+	if (!adev->mc.vram_width) {
+		/* hbm memory channel size */
+		chansize = 128;
+
+		tmp = RREG32_SOC15(DF, 0, mmDF_CS_AON0_DramBaseAddress0);
+		tmp &= DF_CS_AON0_DramBaseAddress0__IntLvNumChan_MASK;
+		tmp >>= DF_CS_AON0_DramBaseAddress0__IntLvNumChan__SHIFT;
+		switch (tmp) {
+		case 0:
+		default:
+			numchan = 1;
+			break;
+		case 1:
+			numchan = 2;
+			break;
+		case 2:
+			numchan = 0;
+			break;
+		case 3:
+			numchan = 4;
+			break;
+		case 4:
+			numchan = 0;
+			break;
+		case 5:
+			numchan = 8;
+			break;
+		case 6:
+			numchan = 0;
+			break;
+		case 7:
+			numchan = 16;
+			break;
+		case 8:
+			numchan = 2;
+			break;
+		}
+		adev->mc.vram_width = numchan * chansize;
 	}
-	adev->mc.vram_width = numchan * chansize;
 
 	/* Could aper size report 0 ? */
 	adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
@@ -494,14 +499,20 @@ static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
 	if (adev->mc.visible_vram_size > adev->mc.real_vram_size)
 		adev->mc.visible_vram_size = adev->mc.real_vram_size;
 
-	/* unless the user had overridden it, set the gart
-	 * size equal to the 1024 or vram, whichever is larger.
-	 */
-	if (amdgpu_gart_size == -1)
-		adev->mc.gtt_size = max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
-					adev->mc.mc_vram_size);
-	else
-		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
+	/* set the gart size */
+	if (amdgpu_gart_size == -1) {
+		switch (adev->asic_type) {
+		case CHIP_VEGA10:  /* all engines support GPUVM */
+		default:
+			adev->mc.gart_size = 256ULL << 20;
+			break;
+		case CHIP_RAVEN:   /* DCE SG support */
+			adev->mc.gart_size = 1024ULL << 20;
+			break;
+		}
+	} else {
+		adev->mc.gart_size = (u64)amdgpu_gart_size << 20;
+	}
 
 	gmc_v9_0_vram_gtt_location(adev, &adev->mc);
 
@@ -537,10 +548,21 @@ static int gmc_v9_0_sw_init(void *handle)
 
 	spin_lock_init(&adev->mc.invalidate_lock);
 
-	if (adev->flags & AMD_IS_APU) {
+	switch (adev->asic_type) {
+	case CHIP_RAVEN:
 		adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;
-		amdgpu_vm_adjust_size(adev, 64);
-	} else {
+		if (adev->rev_id == 0x0 || adev->rev_id == 0x1) {
+			adev->vm_manager.vm_size = 1U << 18;
+			adev->vm_manager.block_size = 9;
+			adev->vm_manager.num_level = 3;
+			amdgpu_vm_set_fragment_size(adev, 9);
+		} else {
+			/* vm_size is 64GB for legacy 2-level page support */
+			amdgpu_vm_adjust_size(adev, 64, 9);
+			adev->vm_manager.num_level = 1;
+		}
+		break;
+	case CHIP_VEGA10:
 		/* XXX Don't know how to get VRAM type yet. */
 		adev->mc.vram_type = AMDGPU_VRAM_TYPE_HBM;
 		/*
@@ -550,11 +572,18 @@ static int gmc_v9_0_sw_init(void *handle)
 		 */
 		adev->vm_manager.vm_size = 1U << 18;
 		adev->vm_manager.block_size = 9;
-		DRM_INFO("vm size is %llu GB, block size is %u-bit\n",
-				adev->vm_manager.vm_size,
-				adev->vm_manager.block_size);
+		adev->vm_manager.num_level = 3;
+		amdgpu_vm_set_fragment_size(adev, 9);
+		break;
+	default:
+		break;
 	}
 
+	DRM_INFO("vm size is %llu GB, block size is %u-bit,fragment size is %u-bit\n",
+			adev->vm_manager.vm_size,
+			adev->vm_manager.block_size,
+			adev->vm_manager.fragment_size);
+
 	/* This interrupt is VMC page fault.*/
 	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_VMC, 0,
 				&adev->mc.vm_fault);
@@ -619,11 +648,6 @@ static int gmc_v9_0_sw_init(void *handle)
 	adev->vm_manager.id_mgr[AMDGPU_GFXHUB].num_ids = AMDGPU_NUM_OF_VMIDS;
 	adev->vm_manager.id_mgr[AMDGPU_MMHUB].num_ids = AMDGPU_NUM_OF_VMIDS;
 
-	/* TODO: fix num_level for APU when updating vm size and block size */
-	if (adev->flags & AMD_IS_APU)
-		adev->vm_manager.num_level = 1;
-	else
-		adev->vm_manager.num_level = 3;
 	amdgpu_vm_manager_init(adev);
 
 	return 0;
@@ -731,7 +755,7 @@ static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
 	gmc_v9_0_gart_flush_gpu_tlb(adev, 0);
 
 	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
-		 (unsigned)(adev->mc.gtt_size >> 20),
+		 (unsigned)(adev->mc.gart_size >> 20),
 		 (unsigned long long)adev->gart.table_addr);
 	adev->gart.ready = true;
 	return 0;
@@ -745,6 +769,20 @@ static int gmc_v9_0_hw_init(void *handle)
 	/* The sequence of these two function calls matters.*/
 	gmc_v9_0_init_golden_registers(adev);
 
+	if (adev->mode_info.num_crtc) {
+		u32 tmp;
+
+		/* Lockout access through VGA aperture*/
+		tmp = RREG32_SOC15(DCE, 0, mmVGA_HDP_CONTROL);
+		tmp = REG_SET_FIELD(tmp, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
+		WREG32_SOC15(DCE, 0, mmVGA_HDP_CONTROL, tmp);
+
+		/* disable VGA render */
+		tmp = RREG32_SOC15(DCE, 0, mmVGA_RENDER_CONTROL);
+		tmp = REG_SET_FIELD(tmp, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
+		WREG32_SOC15(DCE, 0, mmVGA_RENDER_CONTROL, tmp);
+	}
+
 	r = gmc_v9_0_gart_enable(adev);
 
 	return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
index 9804318f3488..74cb647da30e 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
@@ -69,14 +69,14 @@ static void mmhub_v1_0_init_gart_aperture_regs(struct amdgpu_device *adev)
 	mmhub_v1_0_init_gart_pt_regs(adev);
 
 	WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
-		     (u32)(adev->mc.gtt_start >> 12));
+		     (u32)(adev->mc.gart_start >> 12));
 	WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
-		     (u32)(adev->mc.gtt_start >> 44));
+		     (u32)(adev->mc.gart_start >> 44));
 
 	WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
-		     (u32)(adev->mc.gtt_end >> 12));
+		     (u32)(adev->mc.gart_end >> 12));
 	WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
-		     (u32)(adev->mc.gtt_end >> 44));
+		     (u32)(adev->mc.gart_end >> 44));
 }
 
 static void mmhub_v1_0_init_system_aperture_regs(struct amdgpu_device *adev)
@@ -143,7 +143,7 @@ static void mmhub_v1_0_init_cache_regs(struct amdgpu_device *adev)
 	/* Setup L2 cache */
 	tmp = RREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL);
 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1);
-	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 0);
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 1);
 	/* XXX for emulation, Refer to closed source code.*/
 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, L2_PDE0_CACHE_TAG_GENERATION_MODE,
 			    0);
@@ -158,6 +158,8 @@ static void mmhub_v1_0_init_cache_regs(struct amdgpu_device *adev)
 	WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL2, tmp);
 
 	tmp = mmVM_L2_CNTL3_DEFAULT;
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 9);
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
 	WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL3, tmp);
 
 	tmp = mmVM_L2_CNTL4_DEFAULT;
@@ -222,6 +224,9 @@ static void mmhub_v1_0_setup_vmid_config(struct amdgpu_device *adev)
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 				PAGE_TABLE_BLOCK_SIZE,
 				adev->vm_manager.block_size - 9);
+		/* Send no-retry XNACK on fault to suppress VM fault storm. */
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
+				    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0);
 		WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_CNTL, i, tmp);
 		WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, i*2, 0);
 		WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, i*2, 0);
@@ -245,28 +250,28 @@ static void mmhub_v1_0_program_invalidation(struct amdgpu_device *adev)
 }
 
 struct pctl_data {
-    uint32_t index;
-    uint32_t data;
+	uint32_t index;
+	uint32_t data;
 };
 
-const struct pctl_data pctl0_data[] = {
-    {0x0, 0x7a640},
-    {0x9, 0x2a64a},
-    {0xd, 0x2a680},
-    {0x11, 0x6a684},
-    {0x19, 0xea68e},
-    {0x29, 0xa69e},
-    {0x2b, 0x34a6c0},
-    {0x61, 0x83a707},
-    {0xe6, 0x8a7a4},
-    {0xf0, 0x1a7b8},
-    {0xf3, 0xfa7cc},
-    {0x104, 0x17a7dd},
-    {0x11d, 0xa7dc},
-    {0x11f, 0x12a7f5},
-    {0x133, 0xa808},
-    {0x135, 0x12a810},
-    {0x149, 0x7a82c}
+static const struct pctl_data pctl0_data[] = {
+	{0x0, 0x7a640},
+	{0x9, 0x2a64a},
+	{0xd, 0x2a680},
+	{0x11, 0x6a684},
+	{0x19, 0xea68e},
+	{0x29, 0xa69e},
+	{0x2b, 0x34a6c0},
+	{0x61, 0x83a707},
+	{0xe6, 0x8a7a4},
+	{0xf0, 0x1a7b8},
+	{0xf3, 0xfa7cc},
+	{0x104, 0x17a7dd},
+	{0x11d, 0xa7dc},
+	{0x11f, 0x12a7f5},
+	{0x133, 0xa808},
+	{0x135, 0x12a810},
+	{0x149, 0x7a82c}
 };
 #define PCTL0_DATA_LEN (sizeof(pctl0_data)/sizeof(pctl0_data[0]))
 
@@ -274,32 +279,39 @@ const struct pctl_data pctl0_data[] = {
 #define PCTL0_STCTRL_REG_SAVE_RANGE0_BASE  0xa640
 #define PCTL0_STCTRL_REG_SAVE_RANGE0_LIMIT 0xa833
 
-const struct pctl_data pctl1_data[] = {
-    {0x0, 0x39a000},
-    {0x3b, 0x44a040},
-    {0x81, 0x2a08d},
-    {0x85, 0x6ba094},
-    {0xf2, 0x18a100},
-    {0x10c, 0x4a132},
-    {0x112, 0xca141},
-    {0x120, 0x2fa158},
-    {0x151, 0x17a1d0},
-    {0x16a, 0x1a1e9},
-    {0x16d, 0x13a1ec},
-    {0x182, 0x7a201},
-    {0x18b, 0x3a20a},
-    {0x190, 0x7a580},
-    {0x199, 0xa590},
-    {0x19b, 0x4a594},
-    {0x1a1, 0x1a59c},
-    {0x1a4, 0x7a82c},
-    {0x1ad, 0xfa7cc},
-    {0x1be, 0x17a7dd},
-    {0x1d7, 0x12a810}
+static const struct pctl_data pctl1_data[] = {
+	{0x0, 0x39a000},
+	{0x3b, 0x44a040},
+	{0x81, 0x2a08d},
+	{0x85, 0x6ba094},
+	{0xf2, 0x18a100},
+	{0x10c, 0x4a132},
+	{0x112, 0xca141},
+	{0x120, 0x2fa158},
+	{0x151, 0x17a1d0},
+	{0x16a, 0x1a1e9},
+	{0x16d, 0x13a1ec},
+	{0x182, 0x7a201},
+	{0x18b, 0x3a20a},
+	{0x190, 0x7a580},
+	{0x199, 0xa590},
+	{0x19b, 0x4a594},
+	{0x1a1, 0x1a59c},
+	{0x1a4, 0x7a82c},
+	{0x1ad, 0xfa7cc},
+	{0x1be, 0x17a7dd},
+	{0x1d7, 0x12a810},
+	{0x1eb, 0x4000a7e1},
+	{0x1ec, 0x5000a7f5},
+	{0x1ed, 0x4000a7e2},
+	{0x1ee, 0x5000a7dc},
+	{0x1ef, 0x4000a7e3},
+	{0x1f0, 0x5000a7f6},
+	{0x1f1, 0x5000a7e4}
 };
 #define PCTL1_DATA_LEN (sizeof(pctl1_data)/sizeof(pctl1_data[0]))
 
-#define PCTL1_RENG_EXEC_END_PTR 0x1ea
+#define PCTL1_RENG_EXEC_END_PTR 0x1f1
 #define PCTL1_STCTRL_REG_SAVE_RANGE0_BASE  0xa000
 #define PCTL1_STCTRL_REG_SAVE_RANGE0_LIMIT 0xa20d
 #define PCTL1_STCTRL_REG_SAVE_RANGE1_BASE  0xa580
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.h b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.h
index 57bb940c0ecd..5d38229baf69 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.h
@@ -36,7 +36,4 @@ void mmhub_v1_0_initialize_power_gating(struct amdgpu_device *adev);
 void mmhub_v1_0_update_power_gating(struct amdgpu_device *adev,
                                 bool enable);
 
-extern const struct amd_ip_funcs mmhub_v1_0_ip_funcs;
-extern const struct amdgpu_ip_block_version mmhub_v1_0_ip_block;
-
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
index bde3ca3c21c1..2812d88a8bdd 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
@@ -72,21 +72,6 @@ static void xgpu_ai_mailbox_set_valid(struct amdgpu_device *adev, bool val)
 		      reg);
 }
 
-static void xgpu_ai_mailbox_trans_msg(struct amdgpu_device *adev,
-				      enum idh_request req)
-{
-	u32 reg;
-
-	reg = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0,
-					     mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0));
-	reg = REG_SET_FIELD(reg, BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0,
-			    MSGBUF_DATA, req);
-	WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0),
-		      reg);
-
-	xgpu_ai_mailbox_set_valid(adev, true);
-}
-
 static int xgpu_ai_mailbox_rcv_msg(struct amdgpu_device *adev,
 				   enum idh_event event)
 {
@@ -154,13 +139,25 @@ static int xgpu_ai_poll_msg(struct amdgpu_device *adev, enum idh_event event)
 	return r;
 }
 
-
-static int xgpu_ai_send_access_requests(struct amdgpu_device *adev,
-					enum idh_request req)
-{
+static void xgpu_ai_mailbox_trans_msg (struct amdgpu_device *adev,
+	      enum idh_request req, u32 data1, u32 data2, u32 data3) {
+	u32 reg;
 	int r;
 
-	xgpu_ai_mailbox_trans_msg(adev, req);
+	reg = RREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0,
+					     mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0));
+	reg = REG_SET_FIELD(reg, BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0,
+			    MSGBUF_DATA, req);
+	WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0),
+		      reg);
+	WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW1),
+				data1);
+	WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW2),
+				data2);
+	WREG32_NO_KIQ(SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW3),
+				data3);
+
+	xgpu_ai_mailbox_set_valid(adev, true);
 
 	/* start to poll ack */
 	r = xgpu_ai_poll_ack(adev);
@@ -168,6 +165,14 @@ static int xgpu_ai_send_access_requests(struct amdgpu_device *adev,
 		pr_err("Doesn't get ack from pf, continue\n");
 
 	xgpu_ai_mailbox_set_valid(adev, false);
+}
+
+static int xgpu_ai_send_access_requests(struct amdgpu_device *adev,
+					enum idh_request req)
+{
+	int r;
+
+	xgpu_ai_mailbox_trans_msg(adev, req, 0, 0, 0);
 
 	/* start to check msg if request is idh_req_gpu_init_access */
 	if (req == IDH_REQ_GPU_INIT_ACCESS ||
@@ -342,4 +347,5 @@ const struct amdgpu_virt_ops xgpu_ai_virt_ops = {
 	.req_full_gpu	= xgpu_ai_request_full_gpu_access,
 	.rel_full_gpu	= xgpu_ai_release_full_gpu_access,
 	.reset_gpu = xgpu_ai_request_reset,
+	.trans_msg = xgpu_ai_mailbox_trans_msg,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h
index 9aefc44d2c34..1e91b9a1c591 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h
@@ -31,7 +31,9 @@ enum idh_request {
 	IDH_REL_GPU_INIT_ACCESS,
 	IDH_REQ_GPU_FINI_ACCESS,
 	IDH_REL_GPU_FINI_ACCESS,
-	IDH_REQ_GPU_RESET_ACCESS
+	IDH_REQ_GPU_RESET_ACCESS,
+
+	IDH_LOG_VF_ERROR       = 200,
 };
 
 enum idh_event {
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
index 171a658135b5..c25a831f94ec 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
@@ -613,4 +613,5 @@ const struct amdgpu_virt_ops xgpu_vi_virt_ops = {
 	.req_full_gpu		= xgpu_vi_request_full_gpu_access,
 	.rel_full_gpu		= xgpu_vi_release_full_gpu_access,
 	.reset_gpu		= xgpu_vi_request_reset,
+	.trans_msg		= NULL, /* Does not need to trans VF errors to host. */
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
index 2db741131bc6..c791d73d2d54 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.h
@@ -32,7 +32,9 @@ enum idh_request {
 	IDH_REL_GPU_INIT_ACCESS,
 	IDH_REQ_GPU_FINI_ACCESS,
 	IDH_REL_GPU_FINI_ACCESS,
-	IDH_REQ_GPU_RESET_ACCESS
+	IDH_REQ_GPU_RESET_ACCESS,
+
+	IDH_LOG_VF_ERROR       = 200,
 };
 
 /* VI mailbox messages data */
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c
index 1e272f785def..045988b18bc3 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c
@@ -32,6 +32,7 @@
 
 #define smnCPM_CONTROL                                                                                  0x11180460
 #define smnPCIE_CNTL2                                                                                   0x11180070
+#define smnPCIE_CONFIG_CNTL                                                                             0x11180044
 
 u32 nbio_v6_1_get_rev_id(struct amdgpu_device *adev)
 {
@@ -67,7 +68,7 @@ void nbio_v6_1_mc_access_enable(struct amdgpu_device *adev, bool enable)
 
 void nbio_v6_1_hdp_flush(struct amdgpu_device *adev)
 {
-	WREG32_SOC15(NBIO, 0, mmBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
+	WREG32_SOC15_NO_KIQ(NBIO, 0, mmBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
 }
 
 u32 nbio_v6_1_get_memsize(struct amdgpu_device *adev)
@@ -256,3 +257,15 @@ void nbio_v6_1_detect_hw_virt(struct amdgpu_device *adev)
 			adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE;
 	}
 }
+
+void nbio_v6_1_init_registers(struct amdgpu_device *adev)
+{
+	uint32_t def, data;
+
+	def = data = RREG32_PCIE(smnPCIE_CONFIG_CNTL);
+	data = REG_SET_FIELD(data, PCIE_CONFIG_CNTL, CI_SWUS_MAX_READ_REQUEST_SIZE_MODE, 1);
+	data = REG_SET_FIELD(data, PCIE_CONFIG_CNTL, CI_SWUS_MAX_READ_REQUEST_SIZE_PRIV, 1);
+
+	if (def != data)
+		WREG32_PCIE(smnPCIE_CONFIG_CNTL, data);
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.h b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.h
index f6f8bc045518..686e4b4d296a 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.h
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.h
@@ -50,5 +50,6 @@ void nbio_v6_1_update_medium_grain_clock_gating(struct amdgpu_device *adev, bool
 void nbio_v6_1_update_medium_grain_light_sleep(struct amdgpu_device *adev, bool enable);
 void nbio_v6_1_get_clockgating_state(struct amdgpu_device *adev, u32 *flags);
 void nbio_v6_1_detect_hw_virt(struct amdgpu_device *adev);
+void nbio_v6_1_init_registers(struct amdgpu_device *adev);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c
index aa04632523fa..11b70d601922 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c
@@ -65,7 +65,7 @@ void nbio_v7_0_mc_access_enable(struct amdgpu_device *adev, bool enable)
 
 void nbio_v7_0_hdp_flush(struct amdgpu_device *adev)
 {
-	WREG32_SOC15(NBIO, 0, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0);
+	WREG32_SOC15_NO_KIQ(NBIO, 0, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0);
 }
 
 u32 nbio_v7_0_get_memsize(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
index 2258323a3c26..f7cf994b1da2 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
@@ -86,6 +86,52 @@ psp_v10_0_get_fw_type(struct amdgpu_firmware_info *ucode, enum psp_gfx_fw_type *
 	return 0;
 }
 
+int psp_v10_0_init_microcode(struct psp_context *psp)
+{
+	struct amdgpu_device *adev = psp->adev;
+	const char *chip_name;
+	char fw_name[30];
+	int err = 0;
+	const struct psp_firmware_header_v1_0 *hdr;
+
+	DRM_DEBUG("\n");
+
+	switch (adev->asic_type) {
+	case CHIP_RAVEN:
+		chip_name = "raven";
+		break;
+	default: BUG();
+	}
+
+	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_asd.bin", chip_name);
+	err = request_firmware(&adev->psp.asd_fw, fw_name, adev->dev);
+	if (err)
+		goto out;
+
+	err = amdgpu_ucode_validate(adev->psp.asd_fw);
+	if (err)
+		goto out;
+
+	hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.asd_fw->data;
+	adev->psp.asd_fw_version = le32_to_cpu(hdr->header.ucode_version);
+	adev->psp.asd_feature_version = le32_to_cpu(hdr->ucode_feature_version);
+	adev->psp.asd_ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes);
+	adev->psp.asd_start_addr = (uint8_t *)hdr +
+				le32_to_cpu(hdr->header.ucode_array_offset_bytes);
+
+	return 0;
+out:
+	if (err) {
+		dev_err(adev->dev,
+			"psp v10.0: Failed to load firmware \"%s\"\n",
+			fw_name);
+		release_firmware(adev->psp.asd_fw);
+		adev->psp.asd_fw = NULL;
+	}
+
+	return err;
+}
+
 int psp_v10_0_prep_cmd_buf(struct amdgpu_firmware_info *ucode, struct psp_gfx_cmd_resp *cmd)
 {
 	int ret;
@@ -110,7 +156,6 @@ int psp_v10_0_prep_cmd_buf(struct amdgpu_firmware_info *ucode, struct psp_gfx_cm
 int psp_v10_0_ring_init(struct psp_context *psp, enum psp_ring_type ring_type)
 {
 	int ret = 0;
-	unsigned int psp_ring_reg = 0;
 	struct psp_ring *ring;
 	struct amdgpu_device *adev = psp->adev;
 
@@ -130,6 +175,16 @@ int psp_v10_0_ring_init(struct psp_context *psp, enum psp_ring_type ring_type)
 		return ret;
 	}
 
+	return 0;
+}
+
+int psp_v10_0_ring_create(struct psp_context *psp, enum psp_ring_type ring_type)
+{
+	int ret = 0;
+	unsigned int psp_ring_reg = 0;
+	struct psp_ring *ring = &psp->km_ring;
+	struct amdgpu_device *adev = psp->adev;
+
 	/* Write low address of the ring to C2PMSG_69 */
 	psp_ring_reg = lower_32_bits(ring->ring_mem_mc_addr);
 	WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_69, psp_ring_reg);
@@ -143,13 +198,42 @@ int psp_v10_0_ring_init(struct psp_context *psp, enum psp_ring_type ring_type)
 	psp_ring_reg = ring_type;
 	psp_ring_reg = psp_ring_reg << 16;
 	WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_64, psp_ring_reg);
+
+	/* There might be handshake issue with hardware which needs delay */
+	mdelay(20);
+
 	/* Wait for response flag (bit 31) in C2PMSG_64 */
-	psp_ring_reg = 0;
-	while ((psp_ring_reg & 0x80000000) == 0) {
-		psp_ring_reg = RREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_64);
-	}
+	ret = psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_64),
+			   0x80000000, 0x8000FFFF, false);
 
-	return 0;
+	return ret;
+}
+
+int psp_v10_0_ring_destroy(struct psp_context *psp, enum psp_ring_type ring_type)
+{
+	int ret = 0;
+	struct psp_ring *ring;
+	unsigned int psp_ring_reg = 0;
+	struct amdgpu_device *adev = psp->adev;
+
+	ring = &psp->km_ring;
+
+	/* Write the ring destroy command to C2PMSG_64 */
+	psp_ring_reg = 3 << 16;
+	WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_64, psp_ring_reg);
+
+	/* There might be handshake issue with hardware which needs delay */
+	mdelay(20);
+
+	/* Wait for response flag (bit 31) in C2PMSG_64 */
+	ret = psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_64),
+			   0x80000000, 0x80000000, false);
+
+	amdgpu_bo_free_kernel(&adev->firmware.rbuf,
+			      &ring->ring_mem_mc_addr,
+			      (void **)&ring->ring_mem);
+
+	return ret;
 }
 
 int psp_v10_0_cmd_submit(struct psp_context *psp,
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.h b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.h
index 2022b7b7151e..e76cde2f01f9 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.h
@@ -27,10 +27,15 @@
 
 #include "amdgpu_psp.h"
 
+extern int psp_v10_0_init_microcode(struct psp_context *psp);
 extern int psp_v10_0_prep_cmd_buf(struct amdgpu_firmware_info *ucode,
 				 struct psp_gfx_cmd_resp *cmd);
 extern int psp_v10_0_ring_init(struct psp_context *psp,
 			      enum psp_ring_type ring_type);
+extern int psp_v10_0_ring_create(struct psp_context *psp,
+				 enum psp_ring_type ring_type);
+extern int psp_v10_0_ring_destroy(struct psp_context *psp,
+				  enum psp_ring_type ring_type);
 extern int psp_v10_0_cmd_submit(struct psp_context *psp,
 			       struct amdgpu_firmware_info *ucode,
 			       uint64_t cmd_buf_mc_addr, uint64_t fence_mc_addr,
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
index c98d77d0c8f8..2a535a4b8d5b 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
@@ -237,11 +237,9 @@ int psp_v3_1_bootloader_load_sos(struct psp_context *psp)
 
 	/* there might be handshake issue with hardware which needs delay */
 	mdelay(20);
-#if 0
 	ret = psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_81),
 			   RREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_81),
 			   0, true);
-#endif
 
 	return ret;
 }
@@ -341,10 +339,10 @@ int psp_v3_1_ring_destroy(struct psp_context *psp, enum psp_ring_type ring_type)
 	ret = psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_64),
 			   0x80000000, 0x80000000, false);
 
-	if (ring->ring_mem)
-		amdgpu_bo_free_kernel(&adev->firmware.rbuf,
-				      &ring->ring_mem_mc_addr,
-				      (void **)&ring->ring_mem);
+	amdgpu_bo_free_kernel(&adev->firmware.rbuf,
+			      &ring->ring_mem_mc_addr,
+			      (void **)&ring->ring_mem);
+
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index 1d766ae98dc8..b1de44f22824 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -551,17 +551,53 @@ static void sdma_v3_0_rlc_stop(struct amdgpu_device *adev)
  */
 static void sdma_v3_0_ctx_switch_enable(struct amdgpu_device *adev, bool enable)
 {
-	u32 f32_cntl;
+	u32 f32_cntl, phase_quantum = 0;
 	int i;
 
+	if (amdgpu_sdma_phase_quantum) {
+		unsigned value = amdgpu_sdma_phase_quantum;
+		unsigned unit = 0;
+
+		while (value > (SDMA0_PHASE0_QUANTUM__VALUE_MASK >>
+				SDMA0_PHASE0_QUANTUM__VALUE__SHIFT)) {
+			value = (value + 1) >> 1;
+			unit++;
+		}
+		if (unit > (SDMA0_PHASE0_QUANTUM__UNIT_MASK >>
+			    SDMA0_PHASE0_QUANTUM__UNIT__SHIFT)) {
+			value = (SDMA0_PHASE0_QUANTUM__VALUE_MASK >>
+				 SDMA0_PHASE0_QUANTUM__VALUE__SHIFT);
+			unit = (SDMA0_PHASE0_QUANTUM__UNIT_MASK >>
+				SDMA0_PHASE0_QUANTUM__UNIT__SHIFT);
+			WARN_ONCE(1,
+			"clamping sdma_phase_quantum to %uK clock cycles\n",
+				  value << unit);
+		}
+		phase_quantum =
+			value << SDMA0_PHASE0_QUANTUM__VALUE__SHIFT |
+			unit  << SDMA0_PHASE0_QUANTUM__UNIT__SHIFT;
+	}
+
 	for (i = 0; i < adev->sdma.num_instances; i++) {
 		f32_cntl = RREG32(mmSDMA0_CNTL + sdma_offsets[i]);
-		if (enable)
+		if (enable) {
 			f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL,
 					AUTO_CTXSW_ENABLE, 1);
-		else
+			f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL,
+					ATC_L1_ENABLE, 1);
+			if (amdgpu_sdma_phase_quantum) {
+				WREG32(mmSDMA0_PHASE0_QUANTUM + sdma_offsets[i],
+				       phase_quantum);
+				WREG32(mmSDMA0_PHASE1_QUANTUM + sdma_offsets[i],
+				       phase_quantum);
+			}
+		} else {
 			f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL,
 					AUTO_CTXSW_ENABLE, 0);
+			f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL,
+					ATC_L1_ENABLE, 1);
+		}
+
 		WREG32(mmSDMA0_CNTL + sdma_offsets[i], f32_cntl);
 	}
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
index 4a65697ccc94..fd7c72aaafa6 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
@@ -291,6 +291,8 @@ static void sdma_v4_0_ring_set_wptr(struct amdgpu_ring *ring)
 
 	DRM_DEBUG("Setting write pointer\n");
 	if (ring->use_doorbell) {
+		u64 *wb = (u64 *)&adev->wb.wb[ring->wptr_offs];
+
 		DRM_DEBUG("Using doorbell -- "
 				"wptr_offs == 0x%08x "
 				"lower_32_bits(ring->wptr) << 2 == 0x%08x "
@@ -299,8 +301,7 @@ static void sdma_v4_0_ring_set_wptr(struct amdgpu_ring *ring)
 				lower_32_bits(ring->wptr << 2),
 				upper_32_bits(ring->wptr << 2));
 		/* XXX check if swapping is necessary on BE */
-		adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr << 2);
-		adev->wb.wb[ring->wptr_offs + 1] = upper_32_bits(ring->wptr << 2);
+		WRITE_ONCE(*wb, (ring->wptr << 2));
 		DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n",
 				ring->doorbell_index, ring->wptr << 2);
 		WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
@@ -493,13 +494,45 @@ static void sdma_v4_0_rlc_stop(struct amdgpu_device *adev)
  */
 static void sdma_v4_0_ctx_switch_enable(struct amdgpu_device *adev, bool enable)
 {
-	u32 f32_cntl;
+	u32 f32_cntl, phase_quantum = 0;
 	int i;
 
+	if (amdgpu_sdma_phase_quantum) {
+		unsigned value = amdgpu_sdma_phase_quantum;
+		unsigned unit = 0;
+
+		while (value > (SDMA0_PHASE0_QUANTUM__VALUE_MASK >>
+				SDMA0_PHASE0_QUANTUM__VALUE__SHIFT)) {
+			value = (value + 1) >> 1;
+			unit++;
+		}
+		if (unit > (SDMA0_PHASE0_QUANTUM__UNIT_MASK >>
+			    SDMA0_PHASE0_QUANTUM__UNIT__SHIFT)) {
+			value = (SDMA0_PHASE0_QUANTUM__VALUE_MASK >>
+				 SDMA0_PHASE0_QUANTUM__VALUE__SHIFT);
+			unit = (SDMA0_PHASE0_QUANTUM__UNIT_MASK >>
+				SDMA0_PHASE0_QUANTUM__UNIT__SHIFT);
+			WARN_ONCE(1,
+			"clamping sdma_phase_quantum to %uK clock cycles\n",
+				  value << unit);
+		}
+		phase_quantum =
+			value << SDMA0_PHASE0_QUANTUM__VALUE__SHIFT |
+			unit  << SDMA0_PHASE0_QUANTUM__UNIT__SHIFT;
+	}
+
 	for (i = 0; i < adev->sdma.num_instances; i++) {
 		f32_cntl = RREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_CNTL));
 		f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL,
 				AUTO_CTXSW_ENABLE, enable ? 1 : 0);
+		if (enable && amdgpu_sdma_phase_quantum) {
+			WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_PHASE0_QUANTUM),
+			       phase_quantum);
+			WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_PHASE1_QUANTUM),
+			       phase_quantum);
+			WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_PHASE2_QUANTUM),
+			       phase_quantum);
+		}
 		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_CNTL), f32_cntl);
 	}
 
@@ -541,12 +574,13 @@ static void sdma_v4_0_enable(struct amdgpu_device *adev, bool enable)
 static int sdma_v4_0_gfx_resume(struct amdgpu_device *adev)
 {
 	struct amdgpu_ring *ring;
-	u32 rb_cntl, ib_cntl;
+	u32 rb_cntl, ib_cntl, wptr_poll_cntl;
 	u32 rb_bufsz;
 	u32 wb_offset;
 	u32 doorbell;
 	u32 doorbell_offset;
 	u32 temp;
+	u64 wptr_gpu_addr;
 	int i, r;
 
 	for (i = 0; i < adev->sdma.num_instances; i++) {
@@ -628,6 +662,19 @@ static int sdma_v4_0_gfx_resume(struct amdgpu_device *adev)
 			WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_F32_CNTL), temp);
 		}
 
+		/* setup the wptr shadow polling */
+		wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
+		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_RB_WPTR_POLL_ADDR_LO),
+		       lower_32_bits(wptr_gpu_addr));
+		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_RB_WPTR_POLL_ADDR_HI),
+		       upper_32_bits(wptr_gpu_addr));
+		wptr_poll_cntl = RREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_RB_WPTR_POLL_CNTL));
+		if (amdgpu_sriov_vf(adev))
+			wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl, SDMA0_GFX_RB_WPTR_POLL_CNTL, F32_POLL_ENABLE, 1);
+		else
+			wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl, SDMA0_GFX_RB_WPTR_POLL_CNTL, F32_POLL_ENABLE, 0);
+		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_RB_WPTR_POLL_CNTL), wptr_poll_cntl);
+
 		/* enable DMA RB */
 		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 1);
 		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_RB_CNTL), rb_cntl);
@@ -655,6 +702,7 @@ static int sdma_v4_0_gfx_resume(struct amdgpu_device *adev)
 
 		if (adev->mman.buffer_funcs_ring == ring)
 			amdgpu_ttm_set_active_vram_size(adev, adev->mc.real_vram_size);
+
 	}
 
 	return 0;
@@ -751,15 +799,12 @@ static int sdma_v4_0_load_microcode(struct amdgpu_device *adev)
 	const struct sdma_firmware_header_v1_0 *hdr;
 	const __le32 *fw_data;
 	u32 fw_size;
-	u32 digest_size = 0;
 	int i, j;
 
 	/* halt the MEs */
 	sdma_v4_0_enable(adev, false);
 
 	for (i = 0; i < adev->sdma.num_instances; i++) {
-		uint16_t version_major;
-		uint16_t version_minor;
 		if (!adev->sdma.instance[i].fw)
 			return -EINVAL;
 
@@ -767,23 +812,12 @@ static int sdma_v4_0_load_microcode(struct amdgpu_device *adev)
 		amdgpu_ucode_print_sdma_hdr(&hdr->header);
 		fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
 
-		version_major = le16_to_cpu(hdr->header.header_version_major);
-		version_minor = le16_to_cpu(hdr->header.header_version_minor);
-
-		if (version_major == 1 && version_minor >= 1) {
-			const struct sdma_firmware_header_v1_1 *sdma_v1_1_hdr = (const struct sdma_firmware_header_v1_1 *) hdr;
-			digest_size = le32_to_cpu(sdma_v1_1_hdr->digest_size);
-		}
-
-		fw_size -= digest_size;
-
 		fw_data = (const __le32 *)
 			(adev->sdma.instance[i].fw->data +
 				le32_to_cpu(hdr->header.ucode_array_offset_bytes));
 
 		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_UCODE_ADDR), 0);
 
-
 		for (j = 0; j < fw_size; j++)
 			WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_UCODE_DATA), le32_to_cpup(fw_data++));
 
diff --git a/drivers/gpu/drm/amd/amdgpu/si.c b/drivers/gpu/drm/amd/amdgpu/si.c
index 4267fa417997..8284d5dbfc30 100644
--- a/drivers/gpu/drm/amd/amdgpu/si.c
+++ b/drivers/gpu/drm/amd/amdgpu/si.c
@@ -1150,6 +1150,33 @@ static bool si_read_disabled_bios(struct amdgpu_device *adev)
 	return r;
 }
 
+#define mmROM_INDEX 0x2A
+#define mmROM_DATA  0x2B
+
+static bool si_read_bios_from_rom(struct amdgpu_device *adev,
+				  u8 *bios, u32 length_bytes)
+{
+	u32 *dw_ptr;
+	u32 i, length_dw;
+
+	if (bios == NULL)
+		return false;
+	if (length_bytes == 0)
+		return false;
+	/* APU vbios image is part of sbios image */
+	if (adev->flags & AMD_IS_APU)
+		return false;
+
+	dw_ptr = (u32 *)bios;
+	length_dw = ALIGN(length_bytes, 4) / 4;
+	/* set rom index to 0 */
+	WREG32(mmROM_INDEX, 0);
+	for (i = 0; i < length_dw; i++)
+		dw_ptr[i] = RREG32(mmROM_DATA);
+
+	return true;
+}
+
 //xxx: not implemented
 static int si_asic_reset(struct amdgpu_device *adev)
 {
@@ -1206,6 +1233,7 @@ static void si_detect_hw_virtualization(struct amdgpu_device *adev)
 static const struct amdgpu_asic_funcs si_asic_funcs =
 {
 	.read_disabled_bios = &si_read_disabled_bios,
+	.read_bios_from_rom = &si_read_bios_from_rom,
 	.read_register = &si_read_register,
 	.reset = &si_asic_reset,
 	.set_vga_state = &si_vga_set_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dpm.c b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
index a7ad8390981c..d63873f3f574 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
@@ -2055,6 +2055,7 @@ static void si_initialize_powertune_defaults(struct amdgpu_device *adev)
 		case 0x682C:
 			si_pi->cac_weights = cac_weights_cape_verde_pro;
 			si_pi->dte_data = dte_data_sun_xt;
+			update_dte_from_pl2 = true;
 			break;
 		case 0x6825:
 		case 0x6827:
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index a7341d88a320..f2c3a49f73a0 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -25,7 +25,7 @@
 #include <linux/module.h>
 #include <drm/drmP.h>
 #include "amdgpu.h"
-#include "amdgpu_atomfirmware.h"
+#include "amdgpu_atombios.h"
 #include "amdgpu_ih.h"
 #include "amdgpu_uvd.h"
 #include "amdgpu_vce.h"
@@ -62,8 +62,6 @@
 #include "dce_virtual.h"
 #include "mxgpu_ai.h"
 
-MODULE_FIRMWARE("amdgpu/vega10_smc.bin");
-
 #define mmFabricConfigAccessControl                                                                    0x0410
 #define mmFabricConfigAccessControl_BASE_IDX                                                           0
 #define mmFabricConfigAccessControl_DEFAULT                                      0x00000000
@@ -198,6 +196,50 @@ static void soc15_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 }
 
+static u32 soc15_gc_cac_rreg(struct amdgpu_device *adev, u32 reg)
+{
+	unsigned long flags;
+	u32 r;
+
+	spin_lock_irqsave(&adev->gc_cac_idx_lock, flags);
+	WREG32_SOC15(GC, 0, mmGC_CAC_IND_INDEX, (reg));
+	r = RREG32_SOC15(GC, 0, mmGC_CAC_IND_DATA);
+	spin_unlock_irqrestore(&adev->gc_cac_idx_lock, flags);
+	return r;
+}
+
+static void soc15_gc_cac_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&adev->gc_cac_idx_lock, flags);
+	WREG32_SOC15(GC, 0, mmGC_CAC_IND_INDEX, (reg));
+	WREG32_SOC15(GC, 0, mmGC_CAC_IND_DATA, (v));
+	spin_unlock_irqrestore(&adev->gc_cac_idx_lock, flags);
+}
+
+static u32 soc15_se_cac_rreg(struct amdgpu_device *adev, u32 reg)
+{
+	unsigned long flags;
+	u32 r;
+
+	spin_lock_irqsave(&adev->se_cac_idx_lock, flags);
+	WREG32_SOC15(GC, 0, mmSE_CAC_IND_INDEX, (reg));
+	r = RREG32_SOC15(GC, 0, mmSE_CAC_IND_DATA);
+	spin_unlock_irqrestore(&adev->se_cac_idx_lock, flags);
+	return r;
+}
+
+static void soc15_se_cac_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&adev->se_cac_idx_lock, flags);
+	WREG32_SOC15(GC, 0, mmSE_CAC_IND_INDEX, (reg));
+	WREG32_SOC15(GC, 0, mmSE_CAC_IND_DATA, (v));
+	spin_unlock_irqrestore(&adev->se_cac_idx_lock, flags);
+}
+
 static u32 soc15_get_config_memsize(struct amdgpu_device *adev)
 {
 	if (adev->flags & AMD_IS_APU)
@@ -392,11 +434,11 @@ static void soc15_gpu_pci_config_reset(struct amdgpu_device *adev)
 
 static int soc15_asic_reset(struct amdgpu_device *adev)
 {
-	amdgpu_atomfirmware_scratch_regs_engine_hung(adev, true);
+	amdgpu_atombios_scratch_regs_engine_hung(adev, true);
 
 	soc15_gpu_pci_config_reset(adev);
 
-	amdgpu_atomfirmware_scratch_regs_engine_hung(adev, false);
+	amdgpu_atombios_scratch_regs_engine_hung(adev, false);
 
 	return 0;
 }
@@ -524,13 +566,6 @@ static uint32_t soc15_get_rev_id(struct amdgpu_device *adev)
 		return nbio_v6_1_get_rev_id(adev);
 }
 
-
-int gmc_v9_0_mc_wait_for_idle(struct amdgpu_device *adev)
-{
-	/* to be implemented in MC IP*/
-	return 0;
-}
-
 static const struct amdgpu_asic_funcs soc15_asic_funcs =
 {
 	.read_disabled_bios = &soc15_read_disabled_bios,
@@ -557,6 +592,10 @@ static int soc15_common_early_init(void *handle)
 	adev->uvd_ctx_wreg = &soc15_uvd_ctx_wreg;
 	adev->didt_rreg = &soc15_didt_rreg;
 	adev->didt_wreg = &soc15_didt_wreg;
+	adev->gc_cac_rreg = &soc15_gc_cac_rreg;
+	adev->gc_cac_wreg = &soc15_gc_cac_wreg;
+	adev->se_cac_rreg = &soc15_se_cac_rreg;
+	adev->se_cac_wreg = &soc15_se_cac_wreg;
 
 	adev->asic_funcs = &soc15_asic_funcs;
 
@@ -681,6 +720,9 @@ static int soc15_common_hw_init(void *handle)
 	soc15_pcie_gen3_enable(adev);
 	/* enable aspm */
 	soc15_program_aspm(adev);
+	/* setup nbio registers */
+	if (!(adev->flags & AMD_IS_APU))
+		nbio_v6_1_init_registers(adev);
 	/* enable the doorbell aperture */
 	soc15_enable_doorbell_aperture(adev, true);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15_common.h b/drivers/gpu/drm/amd/amdgpu/soc15_common.h
index e2d330eed952..7a8e4e28abb2 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15_common.h
+++ b/drivers/gpu/drm/amd/amdgpu/soc15_common.h
@@ -77,6 +77,13 @@ struct nbio_pcie_index_data {
 		(3 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG3 + reg : \
 		(ip##_BASE__INST##inst##_SEG4 + reg))))), value)
 
+#define WREG32_SOC15_NO_KIQ(ip, inst, reg, value) \
+	WREG32_NO_KIQ( (0 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG0 + reg : \
+		(1 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG1 + reg : \
+		(2 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG2 + reg : \
+		(3 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG3 + reg : \
+		(ip##_BASE__INST##inst##_SEG4 + reg))))), value)
+
 #define WREG32_SOC15_OFFSET(ip, inst, reg, offset, value) \
 	WREG32( (0 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG0 + reg : \
 		(1 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG1 + reg : \
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15d.h b/drivers/gpu/drm/amd/amdgpu/soc15d.h
index e79befd80eed..7f408f85fdb6 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15d.h
+++ b/drivers/gpu/drm/amd/amdgpu/soc15d.h
@@ -250,6 +250,7 @@
 #define	PACKET3_SET_UCONFIG_REG				0x79
 #define		PACKET3_SET_UCONFIG_REG_START			0x0000c000
 #define		PACKET3_SET_UCONFIG_REG_END			0x0000c400
+#define		PACKET3_SET_UCONFIG_REG_INDEX_TYPE		(2 << 28)
 #define	PACKET3_SCRATCH_RAM_WRITE			0x7D
 #define	PACKET3_SCRATCH_RAM_READ			0x7E
 #define	PACKET3_LOAD_CONST_RAM				0x80
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
index 987b958368ac..23a85750edd6 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
@@ -165,6 +165,9 @@ static int uvd_v7_0_enc_ring_test_ring(struct amdgpu_ring *ring)
 	unsigned i;
 	int r;
 
+	if (amdgpu_sriov_vf(adev))
+		return 0;
+
 	r = amdgpu_ring_alloc(ring, 16);
 	if (r) {
 		DRM_ERROR("amdgpu: uvd enc failed to lock ring %d (%d).\n",
@@ -432,13 +435,19 @@ static int uvd_v7_0_sw_init(void *handle)
 			return r;
 	}
 
-
 	for (i = 0; i < adev->uvd.num_enc_rings; ++i) {
 		ring = &adev->uvd.ring_enc[i];
 		sprintf(ring->name, "uvd_enc%d", i);
 		if (amdgpu_sriov_vf(adev)) {
 			ring->use_doorbell = true;
-			ring->doorbell_index = AMDGPU_DOORBELL64_UVD_RING0_1 * 2;
+
+			/* currently only use the first enconding ring for
+			 * sriov, so set unused location for other unused rings.
+			 */
+			if (i == 0)
+				ring->doorbell_index = AMDGPU_DOORBELL64_UVD_RING0_1 * 2;
+			else
+				ring->doorbell_index = AMDGPU_DOORBELL64_UVD_RING2_3 * 2 + 1;
 		}
 		r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.irq, 0);
 		if (r)
@@ -685,6 +694,11 @@ static int uvd_v7_0_mmsch_start(struct amdgpu_device *adev,
 	/* 4, set resp to zero */
 	WREG32_SOC15(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_RESP, 0);
 
+	WDOORBELL32(adev->uvd.ring_enc[0].doorbell_index, 0);
+	adev->wb.wb[adev->uvd.ring_enc[0].wptr_offs] = 0;
+	adev->uvd.ring_enc[0].wptr = 0;
+	adev->uvd.ring_enc[0].wptr_old = 0;
+
 	/* 5, kick off the initialization and wait until VCE_MMSCH_VF_MAILBOX_RESP becomes non-zero */
 	WREG32_SOC15(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_HOST, 0x10000001);
 
@@ -702,7 +716,6 @@ static int uvd_v7_0_mmsch_start(struct amdgpu_device *adev,
 		dev_err(adev->dev, "failed to init MMSCH, mmVCE_MMSCH_VF_MAILBOX_RESP = %x\n", data);
 		return -EBUSY;
 	}
-	WDOORBELL32(adev->uvd.ring_enc[0].doorbell_index, 0);
 
 	return 0;
 }
@@ -736,11 +749,9 @@ static int uvd_v7_0_sriov_start(struct amdgpu_device *adev)
 		init_table += header->uvd_table_offset;
 
 		ring = &adev->uvd.ring;
+		ring->wptr = 0;
 		size = AMDGPU_GPU_PAGE_ALIGN(adev->uvd.fw->size + 4);
 
-		/* disable clock gating */
-		MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_POWER_STATUS),
-						   ~UVD_POWER_STATUS__UVD_PG_MODE_MASK, 0);
 		MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS),
 						   0xFFFFFFFF, 0x00000004);
 		/* mc resume*/
@@ -777,12 +788,6 @@ static int uvd_v7_0_sriov_start(struct amdgpu_device *adev)
 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CACHE_SIZE2),
 					    AMDGPU_UVD_STACK_SIZE + (AMDGPU_UVD_SESSION_SIZE * 40));
 
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_UDEC_ADDR_CONFIG),
-					    adev->gfx.config.gb_addr_config);
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_UDEC_DB_ADDR_CONFIG),
-					    adev->gfx.config.gb_addr_config);
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_UDEC_DBW_ADDR_CONFIG),
-					    adev->gfx.config.gb_addr_config);
 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_GP_SCRATCH4), adev->uvd.max_handles);
 		/* mc resume end*/
 
@@ -819,17 +824,6 @@ static int uvd_v7_0_sriov_start(struct amdgpu_device *adev)
 						       UVD_LMI_CTRL__REQ_MODE_MASK |
 						       0x00100000L));
 
-		/* disable byte swapping */
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_SWAP_CNTL), 0);
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_MP_SWAP_CNTL), 0);
-
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_MPC_SET_MUXA0), 0x40c2040);
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_MPC_SET_MUXA1), 0x0);
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_MPC_SET_MUXB0), 0x40c2040);
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_MPC_SET_MUXB1), 0x0);
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_MPC_SET_ALU), 0);
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_MPC_SET_MUX), 0x88);
-
 		/* take all subblocks out of reset, except VCPU */
 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
 					    UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
@@ -838,15 +832,6 @@ static int uvd_v7_0_sriov_start(struct amdgpu_device *adev)
 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CNTL),
 					    UVD_VCPU_CNTL__CLK_EN_MASK);
 
-		/* enable UMC */
-		MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2),
-						   ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK, 0);
-
-		/* boot up the VCPU */
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 0);
-
-		MMSCH_V1_0_INSERT_DIRECT_POLL(SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS), 0x02, 0x02);
-
 		/* enable master interrupt */
 		MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN),
 						   ~(UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK),
@@ -859,40 +844,31 @@ static int uvd_v7_0_sriov_start(struct amdgpu_device *adev)
 		/* force RBC into idle state */
 		size = order_base_2(ring->ring_size);
 		tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, size);
-		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
 		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
-		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_WPTR_POLL_EN, 0);
-		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
-		tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_RB_CNTL), tmp);
 
-		/* set the write pointer delay */
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_RB_WPTR_CNTL), 0);
-
-		/* set the wb address */
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_RB_RPTR_ADDR),
-					    (upper_32_bits(ring->gpu_addr) >> 2));
-
-		/* programm the RB_BASE for ring buffer */
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW),
-					    lower_32_bits(ring->gpu_addr));
-		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH),
-					    upper_32_bits(ring->gpu_addr));
-
-		ring->wptr = 0;
 		ring = &adev->uvd.ring_enc[0];
+		ring->wptr = 0;
 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_BASE_LO), ring->gpu_addr);
 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_BASE_HI), upper_32_bits(ring->gpu_addr));
 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_RB_SIZE), ring->ring_size / 4);
 
+		/* boot up the VCPU */
+		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 0);
+
+		/* enable UMC */
+		MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2),
+										   ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK, 0);
+
+		MMSCH_V1_0_INSERT_DIRECT_POLL(SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS), 0x02, 0x02);
+
 		/* add end packet */
 		memcpy((void *)init_table, &end, sizeof(struct mmsch_v1_0_cmd_end));
 		table_size += sizeof(struct mmsch_v1_0_cmd_end) / 4;
 		header->uvd_table_size = table_size;
 
-		return uvd_v7_0_mmsch_start(adev, &adev->virt.mm_table);
 	}
-	return -EINVAL; /* already initializaed ? */
+	return uvd_v7_0_mmsch_start(adev, &adev->virt.mm_table);
 }
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
index 1ecd6bb90c1f..11134d5f7443 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
@@ -173,6 +173,11 @@ static int vce_v4_0_mmsch_start(struct amdgpu_device *adev,
 	/* 4, set resp to zero */
 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_RESP), 0);
 
+	WDOORBELL32(adev->vce.ring[0].doorbell_index, 0);
+	adev->wb.wb[adev->vce.ring[0].wptr_offs] = 0;
+	adev->vce.ring[0].wptr = 0;
+	adev->vce.ring[0].wptr_old = 0;
+
 	/* 5, kick off the initialization and wait until VCE_MMSCH_VF_MAILBOX_RESP becomes non-zero */
 	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_HOST), 0x10000001);
 
@@ -190,7 +195,6 @@ static int vce_v4_0_mmsch_start(struct amdgpu_device *adev,
 		dev_err(adev->dev, "failed to init MMSCH, mmVCE_MMSCH_VF_MAILBOX_RESP = %x\n", data);
 		return -EBUSY;
 	}
-	WDOORBELL32(adev->vce.ring[0].doorbell_index, 0);
 
 	return 0;
 }
@@ -274,7 +278,8 @@ static int vce_v4_0_sriov_start(struct amdgpu_device *adev)
 
 		MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CTRL2), ~0x100, 0);
 		MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_SYS_INT_EN),
-						   0xffffffff, VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK);
+						   VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK,
+						   VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK);
 
 		/* end of MC_RESUME */
 		MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS),
@@ -296,11 +301,9 @@ static int vce_v4_0_sriov_start(struct amdgpu_device *adev)
 		memcpy((void *)init_table, &end, sizeof(struct mmsch_v1_0_cmd_end));
 		table_size += sizeof(struct mmsch_v1_0_cmd_end) / 4;
 		header->vce_table_size = table_size;
-
-		return vce_v4_0_mmsch_start(adev, &adev->virt.mm_table);
 	}
 
-	return -EINVAL; /* already initializaed ? */
+	return vce_v4_0_mmsch_start(adev, &adev->virt.mm_table);
 }
 
 /**
@@ -443,12 +446,14 @@ static int vce_v4_0_sw_init(void *handle)
 		if (amdgpu_sriov_vf(adev)) {
 			/* DOORBELL only works under SRIOV */
 			ring->use_doorbell = true;
+
+			/* currently only use the first encoding ring for sriov,
+			 * so set unused location for other unused rings.
+			 */
 			if (i == 0)
-				ring->doorbell_index = AMDGPU_DOORBELL64_RING0_1 * 2;
-			else if (i == 1)
-				ring->doorbell_index = AMDGPU_DOORBELL64_RING2_3 * 2;
+				ring->doorbell_index = AMDGPU_DOORBELL64_VCE_RING0_1 * 2;
 			else
-				ring->doorbell_index = AMDGPU_DOORBELL64_RING2_3 * 2 + 1;
+				ring->doorbell_index = AMDGPU_DOORBELL64_VCE_RING2_3 * 2 + 1;
 		}
 		r = amdgpu_ring_init(adev, ring, 512, &adev->vce.irq, 0);
 		if (r)
@@ -990,11 +995,13 @@ static int vce_v4_0_set_interrupt_state(struct amdgpu_device *adev,
 {
 	uint32_t val = 0;
 
-	if (state == AMDGPU_IRQ_STATE_ENABLE)
-		val |= VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK;
+	if (!amdgpu_sriov_vf(adev)) {
+		if (state == AMDGPU_IRQ_STATE_ENABLE)
+			val |= VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK;
 
-	WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_SYS_INT_EN), val,
-			~VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK);
+		WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_SYS_INT_EN), val,
+				~VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK);
+	}
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index 6cac291c96da..9ff69b90df36 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -1028,8 +1028,7 @@ static int vi_common_early_init(void *handle)
 		/* rev0 hardware requires workarounds to support PG */
 		adev->pg_flags = 0;
 		if (adev->rev_id != 0x00 || CZ_REV_BRISTOL(adev->pdev->revision)) {
-			adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
-				AMD_PG_SUPPORT_GFX_SMG |
+			adev->pg_flags |= AMD_PG_SUPPORT_GFX_SMG |
 				AMD_PG_SUPPORT_GFX_PIPELINE |
 				AMD_PG_SUPPORT_CP |
 				AMD_PG_SUPPORT_UVD |
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 6316aad43a73..e4a8c2e52cb2 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -142,12 +142,12 @@ static int set_queue_properties_from_user(struct queue_properties *q_properties,
 				struct kfd_ioctl_create_queue_args *args)
 {
 	if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {
-		pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
+		pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
 		return -EINVAL;
 	}
 
 	if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
-		pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
+		pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
 		return -EINVAL;
 	}
 
@@ -155,26 +155,26 @@ static int set_queue_properties_from_user(struct queue_properties *q_properties,
 		(!access_ok(VERIFY_WRITE,
 			(const void __user *) args->ring_base_address,
 			sizeof(uint64_t)))) {
-		pr_err("kfd: can't access ring base address\n");
+		pr_err("Can't access ring base address\n");
 		return -EFAULT;
 	}
 
 	if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
-		pr_err("kfd: ring size must be a power of 2 or 0\n");
+		pr_err("Ring size must be a power of 2 or 0\n");
 		return -EINVAL;
 	}
 
 	if (!access_ok(VERIFY_WRITE,
 			(const void __user *) args->read_pointer_address,
 			sizeof(uint32_t))) {
-		pr_err("kfd: can't access read pointer\n");
+		pr_err("Can't access read pointer\n");
 		return -EFAULT;
 	}
 
 	if (!access_ok(VERIFY_WRITE,
 			(const void __user *) args->write_pointer_address,
 			sizeof(uint32_t))) {
-		pr_err("kfd: can't access write pointer\n");
+		pr_err("Can't access write pointer\n");
 		return -EFAULT;
 	}
 
@@ -182,7 +182,7 @@ static int set_queue_properties_from_user(struct queue_properties *q_properties,
 		!access_ok(VERIFY_WRITE,
 			(const void __user *) args->eop_buffer_address,
 			sizeof(uint32_t))) {
-		pr_debug("kfd: can't access eop buffer");
+		pr_debug("Can't access eop buffer");
 		return -EFAULT;
 	}
 
@@ -190,7 +190,7 @@ static int set_queue_properties_from_user(struct queue_properties *q_properties,
 		!access_ok(VERIFY_WRITE,
 			(const void __user *) args->ctx_save_restore_address,
 			sizeof(uint32_t))) {
-		pr_debug("kfd: can't access ctx save restore buffer");
+		pr_debug("Can't access ctx save restore buffer");
 		return -EFAULT;
 	}
 
@@ -219,27 +219,27 @@ static int set_queue_properties_from_user(struct queue_properties *q_properties,
 	else
 		q_properties->format = KFD_QUEUE_FORMAT_PM4;
 
-	pr_debug("Queue Percentage (%d, %d)\n",
+	pr_debug("Queue Percentage: %d, %d\n",
 			q_properties->queue_percent, args->queue_percentage);
 
-	pr_debug("Queue Priority (%d, %d)\n",
+	pr_debug("Queue Priority: %d, %d\n",
 			q_properties->priority, args->queue_priority);
 
-	pr_debug("Queue Address (0x%llX, 0x%llX)\n",
+	pr_debug("Queue Address: 0x%llX, 0x%llX\n",
 			q_properties->queue_address, args->ring_base_address);
 
-	pr_debug("Queue Size (0x%llX, %u)\n",
+	pr_debug("Queue Size: 0x%llX, %u\n",
 			q_properties->queue_size, args->ring_size);
 
-	pr_debug("Queue r/w Pointers (0x%llX, 0x%llX)\n",
-			(uint64_t) q_properties->read_ptr,
-			(uint64_t) q_properties->write_ptr);
+	pr_debug("Queue r/w Pointers: %p, %p\n",
+			q_properties->read_ptr,
+			q_properties->write_ptr);
 
-	pr_debug("Queue Format (%d)\n", q_properties->format);
+	pr_debug("Queue Format: %d\n", q_properties->format);
 
-	pr_debug("Queue EOP (0x%llX)\n", q_properties->eop_ring_buffer_address);
+	pr_debug("Queue EOP: 0x%llX\n", q_properties->eop_ring_buffer_address);
 
-	pr_debug("Queue CTX save arex (0x%llX)\n",
+	pr_debug("Queue CTX save area: 0x%llX\n",
 			q_properties->ctx_save_restore_area_address);
 
 	return 0;
@@ -257,16 +257,16 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
 
 	memset(&q_properties, 0, sizeof(struct queue_properties));
 
-	pr_debug("kfd: creating queue ioctl\n");
+	pr_debug("Creating queue ioctl\n");
 
 	err = set_queue_properties_from_user(&q_properties, args);
 	if (err)
 		return err;
 
-	pr_debug("kfd: looking for gpu id 0x%x\n", args->gpu_id);
+	pr_debug("Looking for gpu id 0x%x\n", args->gpu_id);
 	dev = kfd_device_by_id(args->gpu_id);
-	if (dev == NULL) {
-		pr_debug("kfd: gpu id 0x%x was not found\n", args->gpu_id);
+	if (!dev) {
+		pr_debug("Could not find gpu id 0x%x\n", args->gpu_id);
 		return -EINVAL;
 	}
 
@@ -278,7 +278,7 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
 		goto err_bind_process;
 	}
 
-	pr_debug("kfd: creating queue for PASID %d on GPU 0x%x\n",
+	pr_debug("Creating queue for PASID %d on gpu 0x%x\n",
 			p->pasid,
 			dev->id);
 
@@ -296,15 +296,15 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
 
 	mutex_unlock(&p->mutex);
 
-	pr_debug("kfd: queue id %d was created successfully\n", args->queue_id);
+	pr_debug("Queue id %d was created successfully\n", args->queue_id);
 
-	pr_debug("ring buffer address == 0x%016llX\n",
+	pr_debug("Ring buffer address == 0x%016llX\n",
 			args->ring_base_address);
 
-	pr_debug("read ptr address    == 0x%016llX\n",
+	pr_debug("Read ptr address    == 0x%016llX\n",
 			args->read_pointer_address);
 
-	pr_debug("write ptr address   == 0x%016llX\n",
+	pr_debug("Write ptr address   == 0x%016llX\n",
 			args->write_pointer_address);
 
 	return 0;
@@ -321,7 +321,7 @@ static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p,
 	int retval;
 	struct kfd_ioctl_destroy_queue_args *args = data;
 
-	pr_debug("kfd: destroying queue id %d for PASID %d\n",
+	pr_debug("Destroying queue id %d for pasid %d\n",
 				args->queue_id,
 				p->pasid);
 
@@ -341,12 +341,12 @@ static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,
 	struct queue_properties properties;
 
 	if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {
-		pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
+		pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
 		return -EINVAL;
 	}
 
 	if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
-		pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
+		pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
 		return -EINVAL;
 	}
 
@@ -354,12 +354,12 @@ static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,
 		(!access_ok(VERIFY_WRITE,
 			(const void __user *) args->ring_base_address,
 			sizeof(uint64_t)))) {
-		pr_err("kfd: can't access ring base address\n");
+		pr_err("Can't access ring base address\n");
 		return -EFAULT;
 	}
 
 	if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
-		pr_err("kfd: ring size must be a power of 2 or 0\n");
+		pr_err("Ring size must be a power of 2 or 0\n");
 		return -EINVAL;
 	}
 
@@ -368,7 +368,7 @@ static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,
 	properties.queue_percent = args->queue_percentage;
 	properties.priority = args->queue_priority;
 
-	pr_debug("kfd: updating queue id %d for PASID %d\n",
+	pr_debug("Updating queue id %d for pasid %d\n",
 			args->queue_id, p->pasid);
 
 	mutex_lock(&p->mutex);
@@ -400,7 +400,7 @@ static int kfd_ioctl_set_memory_policy(struct file *filep,
 	}
 
 	dev = kfd_device_by_id(args->gpu_id);
-	if (dev == NULL)
+	if (!dev)
 		return -EINVAL;
 
 	mutex_lock(&p->mutex);
@@ -443,7 +443,7 @@ static int kfd_ioctl_dbg_register(struct file *filep,
 	long status = 0;
 
 	dev = kfd_device_by_id(args->gpu_id);
-	if (dev == NULL)
+	if (!dev)
 		return -EINVAL;
 
 	if (dev->device_info->asic_family == CHIP_CARRIZO) {
@@ -460,12 +460,11 @@ static int kfd_ioctl_dbg_register(struct file *filep,
 	 */
 	pdd = kfd_bind_process_to_device(dev, p);
 	if (IS_ERR(pdd)) {
-		mutex_unlock(&p->mutex);
-		mutex_unlock(kfd_get_dbgmgr_mutex());
-		return PTR_ERR(pdd);
+		status = PTR_ERR(pdd);
+		goto out;
 	}
 
-	if (dev->dbgmgr == NULL) {
+	if (!dev->dbgmgr) {
 		/* In case of a legal call, we have no dbgmgr yet */
 		create_ok = kfd_dbgmgr_create(&dbgmgr_ptr, dev);
 		if (create_ok) {
@@ -480,6 +479,7 @@ static int kfd_ioctl_dbg_register(struct file *filep,
 		status = -EINVAL;
 	}
 
+out:
 	mutex_unlock(&p->mutex);
 	mutex_unlock(kfd_get_dbgmgr_mutex());
 
@@ -494,7 +494,7 @@ static int kfd_ioctl_dbg_unregister(struct file *filep,
 	long status;
 
 	dev = kfd_device_by_id(args->gpu_id);
-	if (dev == NULL)
+	if (!dev)
 		return -EINVAL;
 
 	if (dev->device_info->asic_family == CHIP_CARRIZO) {
@@ -505,7 +505,7 @@ static int kfd_ioctl_dbg_unregister(struct file *filep,
 	mutex_lock(kfd_get_dbgmgr_mutex());
 
 	status = kfd_dbgmgr_unregister(dev->dbgmgr, p);
-	if (status == 0) {
+	if (!status) {
 		kfd_dbgmgr_destroy(dev->dbgmgr);
 		dev->dbgmgr = NULL;
 	}
@@ -539,7 +539,7 @@ static int kfd_ioctl_dbg_address_watch(struct file *filep,
 	memset((void *) &aw_info, 0, sizeof(struct dbg_address_watch_info));
 
 	dev = kfd_device_by_id(args->gpu_id);
-	if (dev == NULL)
+	if (!dev)
 		return -EINVAL;
 
 	if (dev->device_info->asic_family == CHIP_CARRIZO) {
@@ -580,8 +580,8 @@ static int kfd_ioctl_dbg_address_watch(struct file *filep,
 	args_idx += sizeof(aw_info.watch_address) * aw_info.num_watch_points;
 
 	if (args_idx >= args->buf_size_in_bytes - sizeof(*args)) {
-		kfree(args_buff);
-		return -EINVAL;
+		status = -EINVAL;
+		goto out;
 	}
 
 	watch_mask_value = (uint64_t) args_buff[args_idx];
@@ -604,8 +604,8 @@ static int kfd_ioctl_dbg_address_watch(struct file *filep,
 	}
 
 	if (args_idx >= args->buf_size_in_bytes - sizeof(args)) {
-		kfree(args_buff);
-		return -EINVAL;
+		status = -EINVAL;
+		goto out;
 	}
 
 	/* Currently HSA Event is not supported for DBG */
@@ -617,6 +617,7 @@ static int kfd_ioctl_dbg_address_watch(struct file *filep,
 
 	mutex_unlock(kfd_get_dbgmgr_mutex());
 
+out:
 	kfree(args_buff);
 
 	return status;
@@ -646,7 +647,7 @@ static int kfd_ioctl_dbg_wave_control(struct file *filep,
 				sizeof(wac_info.trapId);
 
 	dev = kfd_device_by_id(args->gpu_id);
-	if (dev == NULL)
+	if (!dev)
 		return -EINVAL;
 
 	if (dev->device_info->asic_family == CHIP_CARRIZO) {
@@ -782,8 +783,9 @@ static int kfd_ioctl_get_process_apertures(struct file *filp,
 				"scratch_limit %llX\n", pdd->scratch_limit);
 
 			args->num_of_nodes++;
-		} while ((pdd = kfd_get_next_process_device_data(p, pdd)) != NULL &&
-				(args->num_of_nodes < NUM_OF_SUPPORTED_GPUS));
+
+			pdd = kfd_get_next_process_device_data(p, pdd);
+		} while (pdd && (args->num_of_nodes < NUM_OF_SUPPORTED_GPUS));
 	}
 
 	mutex_unlock(&p->mutex);
@@ -846,9 +848,84 @@ static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p,
 
 	return err;
 }
+static int kfd_ioctl_set_scratch_backing_va(struct file *filep,
+					struct kfd_process *p, void *data)
+{
+	struct kfd_ioctl_set_scratch_backing_va_args *args = data;
+	struct kfd_process_device *pdd;
+	struct kfd_dev *dev;
+	long err;
+
+	dev = kfd_device_by_id(args->gpu_id);
+	if (!dev)
+		return -EINVAL;
+
+	mutex_lock(&p->mutex);
+
+	pdd = kfd_bind_process_to_device(dev, p);
+	if (IS_ERR(pdd)) {
+		err = PTR_ERR(pdd);
+		goto bind_process_to_device_fail;
+	}
+
+	pdd->qpd.sh_hidden_private_base = args->va_addr;
+
+	mutex_unlock(&p->mutex);
+
+	if (sched_policy == KFD_SCHED_POLICY_NO_HWS && pdd->qpd.vmid != 0)
+		dev->kfd2kgd->set_scratch_backing_va(
+			dev->kgd, args->va_addr, pdd->qpd.vmid);
+
+	return 0;
+
+bind_process_to_device_fail:
+	mutex_unlock(&p->mutex);
+	return err;
+}
+
+static int kfd_ioctl_get_tile_config(struct file *filep,
+		struct kfd_process *p, void *data)
+{
+	struct kfd_ioctl_get_tile_config_args *args = data;
+	struct kfd_dev *dev;
+	struct tile_config config;
+	int err = 0;
+
+	dev = kfd_device_by_id(args->gpu_id);
+
+	dev->kfd2kgd->get_tile_config(dev->kgd, &config);
+
+	args->gb_addr_config = config.gb_addr_config;
+	args->num_banks = config.num_banks;
+	args->num_ranks = config.num_ranks;
+
+	if (args->num_tile_configs > config.num_tile_configs)
+		args->num_tile_configs = config.num_tile_configs;
+	err = copy_to_user((void __user *)args->tile_config_ptr,
+			config.tile_config_ptr,
+			args->num_tile_configs * sizeof(uint32_t));
+	if (err) {
+		args->num_tile_configs = 0;
+		return -EFAULT;
+	}
+
+	if (args->num_macro_tile_configs > config.num_macro_tile_configs)
+		args->num_macro_tile_configs =
+				config.num_macro_tile_configs;
+	err = copy_to_user((void __user *)args->macro_tile_config_ptr,
+			config.macro_tile_config_ptr,
+			args->num_macro_tile_configs * sizeof(uint32_t));
+	if (err) {
+		args->num_macro_tile_configs = 0;
+		return -EFAULT;
+	}
+
+	return 0;
+}
 
 #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
-	[_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, .cmd_drv = 0, .name = #ioctl}
+	[_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \
+			    .cmd_drv = 0, .name = #ioctl}
 
 /** Ioctl table */
 static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
@@ -899,6 +976,12 @@ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
 
 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_WAVE_CONTROL,
 			kfd_ioctl_dbg_wave_control, 0),
+
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_SCRATCH_BACKING_VA,
+			kfd_ioctl_set_scratch_backing_va, 0),
+
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_TILE_CONFIG,
+			kfd_ioctl_get_tile_config, 0)
 };
 
 #define AMDKFD_CORE_IOCTL_COUNT	ARRAY_SIZE(amdkfd_ioctls)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c b/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c
index d5e19b5fbbfb..0aa021aa0aa1 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c
@@ -42,8 +42,6 @@
 
 static void dbgdev_address_watch_disable_nodiq(struct kfd_dev *dev)
 {
-	BUG_ON(!dev || !dev->kfd2kgd);
-
 	dev->kfd2kgd->address_watch_disable(dev->kgd);
 }
 
@@ -62,7 +60,8 @@ static int dbgdev_diq_submit_ib(struct kfd_dbgdev *dbgdev,
 	unsigned int *ib_packet_buff;
 	int status;
 
-	BUG_ON(!dbgdev || !dbgdev->kq || !packet_buff || !size_in_bytes);
+	if (WARN_ON(!size_in_bytes))
+		return -EINVAL;
 
 	kq = dbgdev->kq;
 
@@ -77,8 +76,8 @@ static int dbgdev_diq_submit_ib(struct kfd_dbgdev *dbgdev,
 	status = kq->ops.acquire_packet_buffer(kq,
 				pq_packets_size_in_bytes / sizeof(uint32_t),
 				&ib_packet_buff);
-	if (status != 0) {
-		pr_err("amdkfd: acquire_packet_buffer failed\n");
+	if (status) {
+		pr_err("acquire_packet_buffer failed\n");
 		return status;
 	}
 
@@ -115,8 +114,8 @@ static int dbgdev_diq_submit_ib(struct kfd_dbgdev *dbgdev,
 	status = kfd_gtt_sa_allocate(dbgdev->dev, sizeof(uint64_t),
 					&mem_obj);
 
-	if (status != 0) {
-		pr_err("amdkfd: Failed to allocate GART memory\n");
+	if (status) {
+		pr_err("Failed to allocate GART memory\n");
 		kq->ops.rollback_packet(kq);
 		return status;
 	}
@@ -168,8 +167,6 @@ static int dbgdev_diq_submit_ib(struct kfd_dbgdev *dbgdev,
 
 static int dbgdev_register_nodiq(struct kfd_dbgdev *dbgdev)
 {
-	BUG_ON(!dbgdev);
-
 	/*
 	 * no action is needed in this case,
 	 * just make sure diq will not be used
@@ -187,14 +184,12 @@ static int dbgdev_register_diq(struct kfd_dbgdev *dbgdev)
 	struct kernel_queue *kq = NULL;
 	int status;
 
-	BUG_ON(!dbgdev || !dbgdev->pqm || !dbgdev->dev);
-
 	status = pqm_create_queue(dbgdev->pqm, dbgdev->dev, NULL,
 				&properties, 0, KFD_QUEUE_TYPE_DIQ,
 				&qid);
 
 	if (status) {
-		pr_err("amdkfd: Failed to create DIQ\n");
+		pr_err("Failed to create DIQ\n");
 		return status;
 	}
 
@@ -202,8 +197,8 @@ static int dbgdev_register_diq(struct kfd_dbgdev *dbgdev)
 
 	kq = pqm_get_kernel_queue(dbgdev->pqm, qid);
 
-	if (kq == NULL) {
-		pr_err("amdkfd: Error getting DIQ\n");
+	if (!kq) {
+		pr_err("Error getting DIQ\n");
 		pqm_destroy_queue(dbgdev->pqm, qid);
 		return -EFAULT;
 	}
@@ -215,8 +210,6 @@ static int dbgdev_register_diq(struct kfd_dbgdev *dbgdev)
 
 static int dbgdev_unregister_nodiq(struct kfd_dbgdev *dbgdev)
 {
-	BUG_ON(!dbgdev || !dbgdev->dev);
-
 	/* disable watch address */
 	dbgdev_address_watch_disable_nodiq(dbgdev->dev);
 	return 0;
@@ -227,8 +220,6 @@ static int dbgdev_unregister_diq(struct kfd_dbgdev *dbgdev)
 	/* todo - disable address watch */
 	int status;
 
-	BUG_ON(!dbgdev || !dbgdev->pqm || !dbgdev->kq);
-
 	status = pqm_destroy_queue(dbgdev->pqm,
 			dbgdev->kq->queue->properties.queue_id);
 	dbgdev->kq = NULL;
@@ -245,14 +236,12 @@ static void dbgdev_address_watch_set_registers(
 {
 	union ULARGE_INTEGER addr;
 
-	BUG_ON(!adw_info || !addrHi || !addrLo || !cntl);
-
 	addr.quad_part = 0;
 	addrHi->u32All = 0;
 	addrLo->u32All = 0;
 	cntl->u32All = 0;
 
-	if (adw_info->watch_mask != NULL)
+	if (adw_info->watch_mask)
 		cntl->bitfields.mask =
 			(uint32_t) (adw_info->watch_mask[index] &
 					ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK);
@@ -279,7 +268,7 @@ static void dbgdev_address_watch_set_registers(
 }
 
 static int dbgdev_address_watch_nodiq(struct kfd_dbgdev *dbgdev,
-					struct dbg_address_watch_info *adw_info)
+				      struct dbg_address_watch_info *adw_info)
 {
 	union TCP_WATCH_ADDR_H_BITS addrHi;
 	union TCP_WATCH_ADDR_L_BITS addrLo;
@@ -287,13 +276,11 @@ static int dbgdev_address_watch_nodiq(struct kfd_dbgdev *dbgdev,
 	struct kfd_process_device *pdd;
 	unsigned int i;
 
-	BUG_ON(!dbgdev || !dbgdev->dev || !adw_info);
-
 	/* taking the vmid for that process on the safe way using pdd */
 	pdd = kfd_get_process_device_data(dbgdev->dev,
 					adw_info->process);
 	if (!pdd) {
-		pr_err("amdkfd: Failed to get pdd for wave control no DIQ\n");
+		pr_err("Failed to get pdd for wave control no DIQ\n");
 		return -EFAULT;
 	}
 
@@ -303,17 +290,16 @@ static int dbgdev_address_watch_nodiq(struct kfd_dbgdev *dbgdev,
 
 	if ((adw_info->num_watch_points > MAX_WATCH_ADDRESSES) ||
 			(adw_info->num_watch_points == 0)) {
-		pr_err("amdkfd: num_watch_points is invalid\n");
+		pr_err("num_watch_points is invalid\n");
 		return -EINVAL;
 	}
 
-	if ((adw_info->watch_mode == NULL) ||
-		(adw_info->watch_address == NULL)) {
-		pr_err("amdkfd: adw_info fields are not valid\n");
+	if (!adw_info->watch_mode || !adw_info->watch_address) {
+		pr_err("adw_info fields are not valid\n");
 		return -EINVAL;
 	}
 
-	for (i = 0 ; i < adw_info->num_watch_points ; i++) {
+	for (i = 0; i < adw_info->num_watch_points; i++) {
 		dbgdev_address_watch_set_registers(adw_info, &addrHi, &addrLo,
 						&cntl, i, pdd->qpd.vmid);
 
@@ -348,7 +334,7 @@ static int dbgdev_address_watch_nodiq(struct kfd_dbgdev *dbgdev,
 }
 
 static int dbgdev_address_watch_diq(struct kfd_dbgdev *dbgdev,
-					struct dbg_address_watch_info *adw_info)
+				    struct dbg_address_watch_info *adw_info)
 {
 	struct pm4__set_config_reg *packets_vec;
 	union TCP_WATCH_ADDR_H_BITS addrHi;
@@ -363,28 +349,25 @@ static int dbgdev_address_watch_diq(struct kfd_dbgdev *dbgdev,
 	/* we do not control the vmid in DIQ mode, just a place holder */
 	unsigned int vmid = 0;
 
-	BUG_ON(!dbgdev || !dbgdev->dev || !adw_info);
-
 	addrHi.u32All = 0;
 	addrLo.u32All = 0;
 	cntl.u32All = 0;
 
 	if ((adw_info->num_watch_points > MAX_WATCH_ADDRESSES) ||
 			(adw_info->num_watch_points == 0)) {
-		pr_err("amdkfd: num_watch_points is invalid\n");
+		pr_err("num_watch_points is invalid\n");
 		return -EINVAL;
 	}
 
-	if ((NULL == adw_info->watch_mode) ||
-			(NULL == adw_info->watch_address)) {
-		pr_err("amdkfd: adw_info fields are not valid\n");
+	if (!adw_info->watch_mode || !adw_info->watch_address) {
+		pr_err("adw_info fields are not valid\n");
 		return -EINVAL;
 	}
 
 	status = kfd_gtt_sa_allocate(dbgdev->dev, ib_size, &mem_obj);
 
-	if (status != 0) {
-		pr_err("amdkfd: Failed to allocate GART memory\n");
+	if (status) {
+		pr_err("Failed to allocate GART memory\n");
 		return status;
 	}
 
@@ -442,8 +425,6 @@ static int dbgdev_address_watch_diq(struct kfd_dbgdev *dbgdev,
 					i,
 					ADDRESS_WATCH_REG_CNTL);
 
-		aw_reg_add_dword /= sizeof(uint32_t);
-
 		packets_vec[0].bitfields2.reg_offset =
 					aw_reg_add_dword - AMD_CONFIG_REG_BASE;
 
@@ -455,8 +436,6 @@ static int dbgdev_address_watch_diq(struct kfd_dbgdev *dbgdev,
 					i,
 					ADDRESS_WATCH_REG_ADDR_HI);
 
-		aw_reg_add_dword /= sizeof(uint32_t);
-
 		packets_vec[1].bitfields2.reg_offset =
 					aw_reg_add_dword - AMD_CONFIG_REG_BASE;
 		packets_vec[1].reg_data[0] = addrHi.u32All;
@@ -467,8 +446,6 @@ static int dbgdev_address_watch_diq(struct kfd_dbgdev *dbgdev,
 					i,
 					ADDRESS_WATCH_REG_ADDR_LO);
 
-		aw_reg_add_dword /= sizeof(uint32_t);
-
 		packets_vec[2].bitfields2.reg_offset =
 				aw_reg_add_dword - AMD_CONFIG_REG_BASE;
 		packets_vec[2].reg_data[0] = addrLo.u32All;
@@ -485,8 +462,6 @@ static int dbgdev_address_watch_diq(struct kfd_dbgdev *dbgdev,
 					i,
 					ADDRESS_WATCH_REG_CNTL);
 
-		aw_reg_add_dword /= sizeof(uint32_t);
-
 		packets_vec[3].bitfields2.reg_offset =
 					aw_reg_add_dword - AMD_CONFIG_REG_BASE;
 		packets_vec[3].reg_data[0] = cntl.u32All;
@@ -498,8 +473,8 @@ static int dbgdev_address_watch_diq(struct kfd_dbgdev *dbgdev,
 					packet_buff_uint,
 					ib_size);
 
-		if (status != 0) {
-			pr_err("amdkfd: Failed to submit IB to DIQ\n");
+		if (status) {
+			pr_err("Failed to submit IB to DIQ\n");
 			break;
 		}
 	}
@@ -518,8 +493,6 @@ static int dbgdev_wave_control_set_registers(
 	union GRBM_GFX_INDEX_BITS reg_gfx_index;
 	struct HsaDbgWaveMsgAMDGen2 *pMsg;
 
-	BUG_ON(!wac_info || !in_reg_sq_cmd || !in_reg_gfx_index);
-
 	reg_sq_cmd.u32All = 0;
 	reg_gfx_index.u32All = 0;
 	pMsg = &wac_info->dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2;
@@ -620,18 +593,16 @@ static int dbgdev_wave_control_diq(struct kfd_dbgdev *dbgdev,
 	struct pm4__set_config_reg *packets_vec;
 	size_t ib_size = sizeof(struct pm4__set_config_reg) * 3;
 
-	BUG_ON(!dbgdev || !wac_info);
-
 	reg_sq_cmd.u32All = 0;
 
 	status = dbgdev_wave_control_set_registers(wac_info, &reg_sq_cmd,
 							&reg_gfx_index);
 	if (status) {
-		pr_err("amdkfd: Failed to set wave control registers\n");
+		pr_err("Failed to set wave control registers\n");
 		return status;
 	}
 
-	/* we do not control the VMID in DIQ,so reset it to a known value */
+	/* we do not control the VMID in DIQ, so reset it to a known value */
 	reg_sq_cmd.bits.vm_id = 0;
 
 	pr_debug("\t\t %30s\n", "* * * * * * * * * * * * * * * * * *");
@@ -667,7 +638,7 @@ static int dbgdev_wave_control_diq(struct kfd_dbgdev *dbgdev,
 	status = kfd_gtt_sa_allocate(dbgdev->dev, ib_size, &mem_obj);
 
 	if (status != 0) {
-		pr_err("amdkfd: Failed to allocate GART memory\n");
+		pr_err("Failed to allocate GART memory\n");
 		return status;
 	}
 
@@ -719,8 +690,8 @@ static int dbgdev_wave_control_diq(struct kfd_dbgdev *dbgdev,
 			packet_buff_uint,
 			ib_size);
 
-	if (status != 0)
-		pr_err("amdkfd: Failed to submit IB to DIQ\n");
+	if (status)
+		pr_err("Failed to submit IB to DIQ\n");
 
 	kfd_gtt_sa_free(dbgdev->dev, mem_obj);
 
@@ -735,21 +706,19 @@ static int dbgdev_wave_control_nodiq(struct kfd_dbgdev *dbgdev,
 	union GRBM_GFX_INDEX_BITS reg_gfx_index;
 	struct kfd_process_device *pdd;
 
-	BUG_ON(!dbgdev || !dbgdev->dev || !wac_info);
-
 	reg_sq_cmd.u32All = 0;
 
 	/* taking the VMID for that process on the safe way using PDD */
 	pdd = kfd_get_process_device_data(dbgdev->dev, wac_info->process);
 
 	if (!pdd) {
-		pr_err("amdkfd: Failed to get pdd for wave control no DIQ\n");
+		pr_err("Failed to get pdd for wave control no DIQ\n");
 		return -EFAULT;
 	}
 	status = dbgdev_wave_control_set_registers(wac_info, &reg_sq_cmd,
 							&reg_gfx_index);
 	if (status) {
-		pr_err("amdkfd: Failed to set wave control registers\n");
+		pr_err("Failed to set wave control registers\n");
 		return status;
 	}
 
@@ -818,12 +787,13 @@ int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p)
 
 	/* Scan all registers in the range ATC_VMID8_PASID_MAPPING ..
 	 * ATC_VMID15_PASID_MAPPING
-	 * to check which VMID the current process is mapped to. */
+	 * to check which VMID the current process is mapped to.
+	 */
 
 	for (vmid = first_vmid_to_scan; vmid <= last_vmid_to_scan; vmid++) {
 		if (dev->kfd2kgd->get_atc_vmid_pasid_mapping_valid
 				(dev->kgd, vmid)) {
-			if (dev->kfd2kgd->get_atc_vmid_pasid_mapping_valid
+			if (dev->kfd2kgd->get_atc_vmid_pasid_mapping_pasid
 					(dev->kgd, vmid) == p->pasid) {
 				pr_debug("Killing wave fronts of vmid %d and pasid %d\n",
 						vmid, p->pasid);
@@ -833,7 +803,7 @@ int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p)
 	}
 
 	if (vmid > last_vmid_to_scan) {
-		pr_err("amdkfd: didn't found vmid for pasid (%d)\n", p->pasid);
+		pr_err("Didn't find vmid for pasid %d\n", p->pasid);
 		return -EFAULT;
 	}
 
@@ -860,8 +830,6 @@ int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p)
 void kfd_dbgdev_init(struct kfd_dbgdev *pdbgdev, struct kfd_dev *pdev,
 			enum DBGDEV_TYPE type)
 {
-	BUG_ON(!pdbgdev || !pdev);
-
 	pdbgdev->dev = pdev;
 	pdbgdev->kq = NULL;
 	pdbgdev->type = type;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.c b/drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.c
index 56d676396342..3da25f7bda6b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.c
@@ -44,8 +44,6 @@ struct mutex *kfd_get_dbgmgr_mutex(void)
 
 static void kfd_dbgmgr_uninitialize(struct kfd_dbgmgr *pmgr)
 {
-	BUG_ON(!pmgr);
-
 	kfree(pmgr->dbgdev);
 
 	pmgr->dbgdev = NULL;
@@ -55,7 +53,7 @@ static void kfd_dbgmgr_uninitialize(struct kfd_dbgmgr *pmgr)
 
 void kfd_dbgmgr_destroy(struct kfd_dbgmgr *pmgr)
 {
-	if (pmgr != NULL) {
+	if (pmgr) {
 		kfd_dbgmgr_uninitialize(pmgr);
 		kfree(pmgr);
 	}
@@ -66,12 +64,12 @@ bool kfd_dbgmgr_create(struct kfd_dbgmgr **ppmgr, struct kfd_dev *pdev)
 	enum DBGDEV_TYPE type = DBGDEV_TYPE_DIQ;
 	struct kfd_dbgmgr *new_buff;
 
-	BUG_ON(pdev == NULL);
-	BUG_ON(!pdev->init_complete);
+	if (WARN_ON(!pdev->init_complete))
+		return false;
 
 	new_buff = kfd_alloc_struct(new_buff);
 	if (!new_buff) {
-		pr_err("amdkfd: Failed to allocate dbgmgr instance\n");
+		pr_err("Failed to allocate dbgmgr instance\n");
 		return false;
 	}
 
@@ -79,7 +77,7 @@ bool kfd_dbgmgr_create(struct kfd_dbgmgr **ppmgr, struct kfd_dev *pdev)
 	new_buff->dev = pdev;
 	new_buff->dbgdev = kfd_alloc_struct(new_buff->dbgdev);
 	if (!new_buff->dbgdev) {
-		pr_err("amdkfd: Failed to allocate dbgdev instance\n");
+		pr_err("Failed to allocate dbgdev instance\n");
 		kfree(new_buff);
 		return false;
 	}
@@ -96,8 +94,6 @@ bool kfd_dbgmgr_create(struct kfd_dbgmgr **ppmgr, struct kfd_dev *pdev)
 
 long kfd_dbgmgr_register(struct kfd_dbgmgr *pmgr, struct kfd_process *p)
 {
-	BUG_ON(!p || !pmgr || !pmgr->dbgdev);
-
 	if (pmgr->pasid != 0) {
 		pr_debug("H/W debugger is already active using pasid %d\n",
 				pmgr->pasid);
@@ -118,8 +114,6 @@ long kfd_dbgmgr_register(struct kfd_dbgmgr *pmgr, struct kfd_process *p)
 
 long kfd_dbgmgr_unregister(struct kfd_dbgmgr *pmgr, struct kfd_process *p)
 {
-	BUG_ON(!p || !pmgr || !pmgr->dbgdev);
-
 	/* Is the requests coming from the already registered process? */
 	if (pmgr->pasid != p->pasid) {
 		pr_debug("H/W debugger is not registered by calling pasid %d\n",
@@ -137,8 +131,6 @@ long kfd_dbgmgr_unregister(struct kfd_dbgmgr *pmgr, struct kfd_process *p)
 long kfd_dbgmgr_wave_control(struct kfd_dbgmgr *pmgr,
 				struct dbg_wave_control_info *wac_info)
 {
-	BUG_ON(!pmgr || !pmgr->dbgdev || !wac_info);
-
 	/* Is the requests coming from the already registered process? */
 	if (pmgr->pasid != wac_info->process->pasid) {
 		pr_debug("H/W debugger support was not registered for requester pasid %d\n",
@@ -152,9 +144,6 @@ long kfd_dbgmgr_wave_control(struct kfd_dbgmgr *pmgr,
 long kfd_dbgmgr_address_watch(struct kfd_dbgmgr *pmgr,
 				struct dbg_address_watch_info *adw_info)
 {
-	BUG_ON(!pmgr || !pmgr->dbgdev || !adw_info);
-
-
 	/* Is the requests coming from the already registered process? */
 	if (pmgr->pasid != adw_info->process->pasid) {
 		pr_debug("H/W debugger support was not registered for requester pasid %d\n",
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.h b/drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.h
index 257a745ad0b5..a04a1fe1d0d9 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_dbgmgr.h
@@ -30,13 +30,11 @@
 #pragma pack(push, 4)
 
 enum HSA_DBG_WAVEOP {
-	HSA_DBG_WAVEOP_HALT = 1,	/* Halts a wavefront		*/
-	HSA_DBG_WAVEOP_RESUME = 2,	/* Resumes a wavefront		*/
-	HSA_DBG_WAVEOP_KILL = 3,	/* Kills a wavefront		*/
-	HSA_DBG_WAVEOP_DEBUG = 4,	/* Causes wavefront to enter
-						debug mode		*/
-	HSA_DBG_WAVEOP_TRAP = 5,	/* Causes wavefront to take
-						a trap			*/
+	HSA_DBG_WAVEOP_HALT = 1,   /* Halts a wavefront */
+	HSA_DBG_WAVEOP_RESUME = 2, /* Resumes a wavefront */
+	HSA_DBG_WAVEOP_KILL = 3,   /* Kills a wavefront */
+	HSA_DBG_WAVEOP_DEBUG = 4,  /* Causes wavefront to enter dbg mode */
+	HSA_DBG_WAVEOP_TRAP = 5,   /* Causes wavefront to take a trap */
 	HSA_DBG_NUM_WAVEOP = 5,
 	HSA_DBG_MAX_WAVEOP = 0xFFFFFFFF
 };
@@ -81,15 +79,13 @@ struct HsaDbgWaveMsgAMDGen2 {
 			uint32_t UserData:8;	/* user data */
 			uint32_t ShaderArray:1;	/* Shader array */
 			uint32_t Priv:1;	/* Privileged */
-			uint32_t Reserved0:4;	/* This field is reserved,
-						   should be 0 */
+			uint32_t Reserved0:4;	/* Reserved, should be 0 */
 			uint32_t WaveId:4;	/* wave id */
 			uint32_t SIMD:2;	/* SIMD id */
 			uint32_t HSACU:4;	/* Compute unit */
 			uint32_t ShaderEngine:2;/* Shader engine */
 			uint32_t MessageType:2;	/* see HSA_DBG_WAVEMSG_TYPE */
-			uint32_t Reserved1:4;	/* This field is reserved,
-						   should be 0 */
+			uint32_t Reserved1:4;	/* Reserved, should be 0 */
 		} ui32;
 		uint32_t Value;
 	};
@@ -121,20 +117,23 @@ struct HsaDbgWaveMessage {
  * in the user mode instruction stream. The OS scheduler event is typically
  * associated and signaled by an interrupt issued by the GPU, but other HSA
  * system interrupt conditions from other HW (e.g. IOMMUv2) may be surfaced
- * by the KFD by this mechanism, too. */
+ * by the KFD by this mechanism, too.
+ */
 
 /* these are the new definitions for events */
 enum HSA_EVENTTYPE {
 	HSA_EVENTTYPE_SIGNAL = 0,	/* user-mode generated GPU signal */
 	HSA_EVENTTYPE_NODECHANGE = 1,	/* HSA node change (attach/detach) */
 	HSA_EVENTTYPE_DEVICESTATECHANGE = 2,	/* HSA device state change
-						   (start/stop) */
+						 * (start/stop)
+						 */
 	HSA_EVENTTYPE_HW_EXCEPTION = 3,	/* GPU shader exception event */
 	HSA_EVENTTYPE_SYSTEM_EVENT = 4,	/* GPU SYSCALL with parameter info */
 	HSA_EVENTTYPE_DEBUG_EVENT = 5,	/* GPU signal for debugging */
 	HSA_EVENTTYPE_PROFILE_EVENT = 6,/* GPU signal for profiling */
 	HSA_EVENTTYPE_QUEUE_EVENT = 7,	/* GPU signal queue idle state
-					   (EOP pm4) */
+					 * (EOP pm4)
+					 */
 	/* ...  */
 	HSA_EVENTTYPE_MAXID,
 	HSA_EVENTTYPE_TYPE_SIZE = 0xFFFFFFFF
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index 3f95f7cb4019..61fff25b4ce7 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -26,7 +26,7 @@
 #include <linux/slab.h>
 #include "kfd_priv.h"
 #include "kfd_device_queue_manager.h"
-#include "kfd_pm4_headers.h"
+#include "kfd_pm4_headers_vi.h"
 
 #define MQD_SIZE_ALIGNED 768
 
@@ -98,11 +98,14 @@ static const struct kfd_device_info *lookup_device_info(unsigned short did)
 
 	for (i = 0; i < ARRAY_SIZE(supported_devices); i++) {
 		if (supported_devices[i].did == did) {
-			BUG_ON(supported_devices[i].device_info == NULL);
+			WARN_ON(!supported_devices[i].device_info);
 			return supported_devices[i].device_info;
 		}
 	}
 
+	dev_warn(kfd_device, "DID %04x is missing in supported_devices\n",
+		 did);
+
 	return NULL;
 }
 
@@ -114,8 +117,10 @@ struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd,
 	const struct kfd_device_info *device_info =
 					lookup_device_info(pdev->device);
 
-	if (!device_info)
+	if (!device_info) {
+		dev_err(kfd_device, "kgd2kfd_probe failed\n");
 		return NULL;
+	}
 
 	kfd = kzalloc(sizeof(*kfd), GFP_KERNEL);
 	if (!kfd)
@@ -152,15 +157,16 @@ static bool device_iommu_pasid_init(struct kfd_dev *kfd)
 	}
 
 	if ((iommu_info.flags & required_iommu_flags) != required_iommu_flags) {
-		dev_err(kfd_device, "error required iommu flags ats(%i), pri(%i), pasid(%i)\n",
+		dev_err(kfd_device, "error required iommu flags ats %i, pri %i, pasid %i\n",
 		       (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP) != 0,
 		       (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) != 0,
-		       (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP) != 0);
+		       (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP)
+									!= 0);
 		return false;
 	}
 
 	pasid_limit = min_t(unsigned int,
-			(unsigned int)1 << kfd->device_info->max_pasid_bits,
+			(unsigned int)(1 << kfd->device_info->max_pasid_bits),
 			iommu_info.max_pasids);
 	/*
 	 * last pasid is used for kernel queues doorbells
@@ -211,9 +217,8 @@ static int iommu_invalid_ppr_cb(struct pci_dev *pdev, int pasid,
 			flags);
 
 	dev = kfd_device_by_pci_dev(pdev);
-	BUG_ON(dev == NULL);
-
-	kfd_signal_iommu_event(dev, pasid, address,
+	if (!WARN_ON(!dev))
+		kfd_signal_iommu_event(dev, pasid, address,
 			flags & PPR_FAULT_WRITE, flags & PPR_FAULT_EXEC);
 
 	return AMD_IOMMU_INV_PRI_RSP_INVALID;
@@ -234,9 +239,9 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
 	 * calculate max size of runlist packet.
 	 * There can be only 2 packets at once
 	 */
-	size += (KFD_MAX_NUM_OF_PROCESSES * sizeof(struct pm4_map_process) +
-		max_num_of_queues_per_device *
-		sizeof(struct pm4_map_queues) + sizeof(struct pm4_runlist)) * 2;
+	size += (KFD_MAX_NUM_OF_PROCESSES * sizeof(struct pm4_mes_map_process) +
+		max_num_of_queues_per_device * sizeof(struct pm4_mes_map_queues)
+		+ sizeof(struct pm4_mes_runlist)) * 2;
 
 	/* Add size of HIQ & DIQ */
 	size += KFD_KERNEL_QUEUE_SIZE * 2;
@@ -247,42 +252,37 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
 	if (kfd->kfd2kgd->init_gtt_mem_allocation(
 			kfd->kgd, size, &kfd->gtt_mem,
 			&kfd->gtt_start_gpu_addr, &kfd->gtt_start_cpu_ptr)){
-		dev_err(kfd_device,
-			"Could not allocate %d bytes for device (%x:%x)\n",
-			size, kfd->pdev->vendor, kfd->pdev->device);
+		dev_err(kfd_device, "Could not allocate %d bytes\n", size);
 		goto out;
 	}
 
-	dev_info(kfd_device,
-		"Allocated %d bytes on gart for device(%x:%x)\n",
-		size, kfd->pdev->vendor, kfd->pdev->device);
+	dev_info(kfd_device, "Allocated %d bytes on gart\n", size);
 
 	/* Initialize GTT sa with 512 byte chunk size */
 	if (kfd_gtt_sa_init(kfd, size, 512) != 0) {
-		dev_err(kfd_device,
-			"Error initializing gtt sub-allocator\n");
+		dev_err(kfd_device, "Error initializing gtt sub-allocator\n");
 		goto kfd_gtt_sa_init_error;
 	}
 
-	kfd_doorbell_init(kfd);
-
-	if (kfd_topology_add_device(kfd) != 0) {
+	if (kfd_doorbell_init(kfd)) {
 		dev_err(kfd_device,
-			"Error adding device (%x:%x) to topology\n",
-			kfd->pdev->vendor, kfd->pdev->device);
+			"Error initializing doorbell aperture\n");
+		goto kfd_doorbell_error;
+	}
+
+	if (kfd_topology_add_device(kfd)) {
+		dev_err(kfd_device, "Error adding device to topology\n");
 		goto kfd_topology_add_device_error;
 	}
 
 	if (kfd_interrupt_init(kfd)) {
-		dev_err(kfd_device,
-			"Error initializing interrupts for device (%x:%x)\n",
-			kfd->pdev->vendor, kfd->pdev->device);
+		dev_err(kfd_device, "Error initializing interrupts\n");
 		goto kfd_interrupt_error;
 	}
 
 	if (!device_iommu_pasid_init(kfd)) {
 		dev_err(kfd_device,
-			"Error initializing iommuv2 for device (%x:%x)\n",
+			"Error initializing iommuv2 for device %x:%x\n",
 			kfd->pdev->vendor, kfd->pdev->device);
 		goto device_iommu_pasid_error;
 	}
@@ -292,15 +292,13 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
 
 	kfd->dqm = device_queue_manager_init(kfd);
 	if (!kfd->dqm) {
-		dev_err(kfd_device,
-			"Error initializing queue manager for device (%x:%x)\n",
-			kfd->pdev->vendor, kfd->pdev->device);
+		dev_err(kfd_device, "Error initializing queue manager\n");
 		goto device_queue_manager_error;
 	}
 
-	if (kfd->dqm->ops.start(kfd->dqm) != 0) {
+	if (kfd->dqm->ops.start(kfd->dqm)) {
 		dev_err(kfd_device,
-			"Error starting queuen manager for device (%x:%x)\n",
+			"Error starting queue manager for device %x:%x\n",
 			kfd->pdev->vendor, kfd->pdev->device);
 		goto dqm_start_error;
 	}
@@ -308,10 +306,10 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
 	kfd->dbgmgr = NULL;
 
 	kfd->init_complete = true;
-	dev_info(kfd_device, "added device (%x:%x)\n", kfd->pdev->vendor,
+	dev_info(kfd_device, "added device %x:%x\n", kfd->pdev->vendor,
 		 kfd->pdev->device);
 
-	pr_debug("kfd: Starting kfd with the following scheduling policy %d\n",
+	pr_debug("Starting kfd with the following scheduling policy %d\n",
 		sched_policy);
 
 	goto out;
@@ -325,11 +323,13 @@ device_iommu_pasid_error:
 kfd_interrupt_error:
 	kfd_topology_remove_device(kfd);
 kfd_topology_add_device_error:
+	kfd_doorbell_fini(kfd);
+kfd_doorbell_error:
 	kfd_gtt_sa_fini(kfd);
 kfd_gtt_sa_init_error:
 	kfd->kfd2kgd->free_gtt_mem(kfd->kgd, kfd->gtt_mem);
 	dev_err(kfd_device,
-		"device (%x:%x) NOT added due to errors\n",
+		"device %x:%x NOT added due to errors\n",
 		kfd->pdev->vendor, kfd->pdev->device);
 out:
 	return kfd->init_complete;
@@ -342,6 +342,7 @@ void kgd2kfd_device_exit(struct kfd_dev *kfd)
 		amd_iommu_free_device(kfd->pdev);
 		kfd_interrupt_exit(kfd);
 		kfd_topology_remove_device(kfd);
+		kfd_doorbell_fini(kfd);
 		kfd_gtt_sa_fini(kfd);
 		kfd->kfd2kgd->free_gtt_mem(kfd->kgd, kfd->gtt_mem);
 	}
@@ -351,8 +352,6 @@ void kgd2kfd_device_exit(struct kfd_dev *kfd)
 
 void kgd2kfd_suspend(struct kfd_dev *kfd)
 {
-	BUG_ON(kfd == NULL);
-
 	if (kfd->init_complete) {
 		kfd->dqm->ops.stop(kfd->dqm);
 		amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
@@ -366,14 +365,15 @@ int kgd2kfd_resume(struct kfd_dev *kfd)
 	unsigned int pasid_limit;
 	int err;
 
-	BUG_ON(kfd == NULL);
-
 	pasid_limit = kfd_get_pasid_limit();
 
 	if (kfd->init_complete) {
 		err = amd_iommu_init_device(kfd->pdev, pasid_limit);
-		if (err < 0)
+		if (err < 0) {
+			dev_err(kfd_device, "failed to initialize iommu\n");
 			return -ENXIO;
+		}
+
 		amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
 						iommu_pasid_shutdown_callback);
 		amd_iommu_set_invalid_ppr_cb(kfd->pdev, iommu_invalid_ppr_cb);
@@ -402,26 +402,27 @@ void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)
 static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,
 				unsigned int chunk_size)
 {
-	unsigned int num_of_bits;
+	unsigned int num_of_longs;
 
-	BUG_ON(!kfd);
-	BUG_ON(!kfd->gtt_mem);
-	BUG_ON(buf_size < chunk_size);
-	BUG_ON(buf_size == 0);
-	BUG_ON(chunk_size == 0);
+	if (WARN_ON(buf_size < chunk_size))
+		return -EINVAL;
+	if (WARN_ON(buf_size == 0))
+		return -EINVAL;
+	if (WARN_ON(chunk_size == 0))
+		return -EINVAL;
 
 	kfd->gtt_sa_chunk_size = chunk_size;
 	kfd->gtt_sa_num_of_chunks = buf_size / chunk_size;
 
-	num_of_bits = kfd->gtt_sa_num_of_chunks / BITS_PER_BYTE;
-	BUG_ON(num_of_bits == 0);
+	num_of_longs = (kfd->gtt_sa_num_of_chunks + BITS_PER_LONG - 1) /
+		BITS_PER_LONG;
 
-	kfd->gtt_sa_bitmap = kzalloc(num_of_bits, GFP_KERNEL);
+	kfd->gtt_sa_bitmap = kcalloc(num_of_longs, sizeof(long), GFP_KERNEL);
 
 	if (!kfd->gtt_sa_bitmap)
 		return -ENOMEM;
 
-	pr_debug("kfd: gtt_sa_num_of_chunks = %d, gtt_sa_bitmap = %p\n",
+	pr_debug("gtt_sa_num_of_chunks = %d, gtt_sa_bitmap = %p\n",
 			kfd->gtt_sa_num_of_chunks, kfd->gtt_sa_bitmap);
 
 	mutex_init(&kfd->gtt_sa_lock);
@@ -455,8 +456,6 @@ int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size,
 {
 	unsigned int found, start_search, cur_size;
 
-	BUG_ON(!kfd);
-
 	if (size == 0)
 		return -EINVAL;
 
@@ -467,7 +466,7 @@ int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size,
 	if ((*mem_obj) == NULL)
 		return -ENOMEM;
 
-	pr_debug("kfd: allocated mem_obj = %p for size = %d\n", *mem_obj, size);
+	pr_debug("Allocated mem_obj = %p for size = %d\n", *mem_obj, size);
 
 	start_search = 0;
 
@@ -479,7 +478,7 @@ kfd_gtt_restart_search:
 					kfd->gtt_sa_num_of_chunks,
 					start_search);
 
-	pr_debug("kfd: found = %d\n", found);
+	pr_debug("Found = %d\n", found);
 
 	/* If there wasn't any free chunk, bail out */
 	if (found == kfd->gtt_sa_num_of_chunks)
@@ -497,12 +496,12 @@ kfd_gtt_restart_search:
 					found,
 					kfd->gtt_sa_chunk_size);
 
-	pr_debug("kfd: gpu_addr = %p, cpu_addr = %p\n",
+	pr_debug("gpu_addr = %p, cpu_addr = %p\n",
 			(uint64_t *) (*mem_obj)->gpu_addr, (*mem_obj)->cpu_ptr);
 
 	/* If we need only one chunk, mark it as allocated and get out */
 	if (size <= kfd->gtt_sa_chunk_size) {
-		pr_debug("kfd: single bit\n");
+		pr_debug("Single bit\n");
 		set_bit(found, kfd->gtt_sa_bitmap);
 		goto kfd_gtt_out;
 	}
@@ -537,7 +536,7 @@ kfd_gtt_restart_search:
 
 	} while (cur_size > 0);
 
-	pr_debug("kfd: range_start = %d, range_end = %d\n",
+	pr_debug("range_start = %d, range_end = %d\n",
 		(*mem_obj)->range_start, (*mem_obj)->range_end);
 
 	/* Mark the chunks as allocated */
@@ -551,7 +550,7 @@ kfd_gtt_out:
 	return 0;
 
 kfd_gtt_no_free_chunk:
-	pr_debug("kfd: allocation failed with mem_obj = %p\n", mem_obj);
+	pr_debug("Allocation failed with mem_obj = %p\n", mem_obj);
 	mutex_unlock(&kfd->gtt_sa_lock);
 	kfree(mem_obj);
 	return -ENOMEM;
@@ -561,13 +560,11 @@ int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj)
 {
 	unsigned int bit;
 
-	BUG_ON(!kfd);
-
 	/* Act like kfree when trying to free a NULL object */
 	if (!mem_obj)
 		return 0;
 
-	pr_debug("kfd: free mem_obj = %p, range_start = %d, range_end = %d\n",
+	pr_debug("Free mem_obj = %p, range_start = %d, range_end = %d\n",
 			mem_obj, mem_obj->range_start, mem_obj->range_end);
 
 	mutex_lock(&kfd->gtt_sa_lock);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index 602769ced3bd..53a66e821624 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -79,20 +79,17 @@ static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe)
 
 unsigned int get_queues_num(struct device_queue_manager *dqm)
 {
-	BUG_ON(!dqm || !dqm->dev);
 	return bitmap_weight(dqm->dev->shared_resources.queue_bitmap,
 				KGD_MAX_QUEUES);
 }
 
 unsigned int get_queues_per_pipe(struct device_queue_manager *dqm)
 {
-	BUG_ON(!dqm || !dqm->dev);
 	return dqm->dev->shared_resources.num_queue_per_pipe;
 }
 
 unsigned int get_pipes_per_mec(struct device_queue_manager *dqm)
 {
-	BUG_ON(!dqm || !dqm->dev);
 	return dqm->dev->shared_resources.num_pipe_per_mec;
 }
 
@@ -121,7 +118,7 @@ static int allocate_vmid(struct device_queue_manager *dqm,
 
 	/* Kaveri kfd vmid's starts from vmid 8 */
 	allocated_vmid = bit + KFD_VMID_START_OFFSET;
-	pr_debug("kfd: vmid allocation %d\n", allocated_vmid);
+	pr_debug("vmid allocation %d\n", allocated_vmid);
 	qpd->vmid = allocated_vmid;
 	q->properties.vmid = allocated_vmid;
 
@@ -152,42 +149,38 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
 {
 	int retval;
 
-	BUG_ON(!dqm || !q || !qpd || !allocated_vmid);
-
-	pr_debug("kfd: In func %s\n", __func__);
 	print_queue(q);
 
 	mutex_lock(&dqm->lock);
 
 	if (dqm->total_queue_count >= max_num_of_queues_per_device) {
-		pr_warn("amdkfd: Can't create new usermode queue because %d queues were already created\n",
+		pr_warn("Can't create new usermode queue because %d queues were already created\n",
 				dqm->total_queue_count);
-		mutex_unlock(&dqm->lock);
-		return -EPERM;
+		retval = -EPERM;
+		goto out_unlock;
 	}
 
 	if (list_empty(&qpd->queues_list)) {
 		retval = allocate_vmid(dqm, qpd, q);
-		if (retval != 0) {
-			mutex_unlock(&dqm->lock);
-			return retval;
-		}
+		if (retval)
+			goto out_unlock;
 	}
 	*allocated_vmid = qpd->vmid;
 	q->properties.vmid = qpd->vmid;
 
 	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
 		retval = create_compute_queue_nocpsch(dqm, q, qpd);
-	if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
+	else if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
 		retval = create_sdma_queue_nocpsch(dqm, q, qpd);
+	else
+		retval = -EINVAL;
 
-	if (retval != 0) {
+	if (retval) {
 		if (list_empty(&qpd->queues_list)) {
 			deallocate_vmid(dqm, qpd, q);
 			*allocated_vmid = 0;
 		}
-		mutex_unlock(&dqm->lock);
-		return retval;
+		goto out_unlock;
 	}
 
 	list_add(&q->list, &qpd->queues_list);
@@ -205,8 +198,9 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
 	pr_debug("Total of %d queues are accountable so far\n",
 			dqm->total_queue_count);
 
+out_unlock:
 	mutex_unlock(&dqm->lock);
-	return 0;
+	return retval;
 }
 
 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
@@ -216,7 +210,8 @@ static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
 
 	set = false;
 
-	for (pipe = dqm->next_pipe_to_allocate, i = 0; i < get_pipes_per_mec(dqm);
+	for (pipe = dqm->next_pipe_to_allocate, i = 0;
+			i < get_pipes_per_mec(dqm);
 			pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) {
 
 		if (!is_pipe_enabled(dqm, 0, pipe))
@@ -239,8 +234,7 @@ static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
 	if (!set)
 		return -EBUSY;
 
-	pr_debug("kfd: DQM %s hqd slot - pipe (%d) queue(%d)\n",
-				__func__, q->pipe, q->queue);
+	pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue);
 	/* horizontal hqd allocation */
 	dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm);
 
@@ -260,36 +254,38 @@ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
 	int retval;
 	struct mqd_manager *mqd;
 
-	BUG_ON(!dqm || !q || !qpd);
-
 	mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
-	if (mqd == NULL)
+	if (!mqd)
 		return -ENOMEM;
 
 	retval = allocate_hqd(dqm, q);
-	if (retval != 0)
+	if (retval)
 		return retval;
 
 	retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
 				&q->gart_mqd_addr, &q->properties);
-	if (retval != 0) {
-		deallocate_hqd(dqm, q);
-		return retval;
-	}
+	if (retval)
+		goto out_deallocate_hqd;
 
-	pr_debug("kfd: loading mqd to hqd on pipe (%d) queue (%d)\n",
-			q->pipe,
-			q->queue);
+	pr_debug("Loading mqd to hqd on pipe %d, queue %d\n",
+			q->pipe, q->queue);
 
-	retval = mqd->load_mqd(mqd, q->mqd, q->pipe,
-			q->queue, (uint32_t __user *) q->properties.write_ptr);
-	if (retval != 0) {
-		deallocate_hqd(dqm, q);
-		mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
-		return retval;
-	}
+	dqm->dev->kfd2kgd->set_scratch_backing_va(
+			dqm->dev->kgd, qpd->sh_hidden_private_base, qpd->vmid);
+
+	retval = mqd->load_mqd(mqd, q->mqd, q->pipe, q->queue, &q->properties,
+			       q->process->mm);
+	if (retval)
+		goto out_uninit_mqd;
 
 	return 0;
+
+out_uninit_mqd:
+	mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
+out_deallocate_hqd:
+	deallocate_hqd(dqm, q);
+
+	return retval;
 }
 
 static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
@@ -299,12 +295,8 @@ static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
 	int retval;
 	struct mqd_manager *mqd;
 
-	BUG_ON(!dqm || !q || !q->mqd || !qpd);
-
 	retval = 0;
 
-	pr_debug("kfd: In Func %s\n", __func__);
-
 	mutex_lock(&dqm->lock);
 
 	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
@@ -323,7 +315,7 @@ static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
 		dqm->sdma_queue_count--;
 		deallocate_sdma_queue(dqm, q->sdma_id);
 	} else {
-		pr_debug("q->properties.type is invalid (%d)\n",
+		pr_debug("q->properties.type %d is invalid\n",
 				q->properties.type);
 		retval = -EINVAL;
 		goto out;
@@ -334,7 +326,7 @@ static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
 				QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS,
 				q->pipe, q->queue);
 
-	if (retval != 0)
+	if (retval)
 		goto out;
 
 	mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
@@ -364,14 +356,12 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
 	struct mqd_manager *mqd;
 	bool prev_active = false;
 
-	BUG_ON(!dqm || !q || !q->mqd);
-
 	mutex_lock(&dqm->lock);
 	mqd = dqm->ops.get_mqd_manager(dqm,
 			get_mqd_type_from_queue_type(q->properties.type));
-	if (mqd == NULL) {
-		mutex_unlock(&dqm->lock);
-		return -ENOMEM;
+	if (!mqd) {
+		retval = -ENOMEM;
+		goto out_unlock;
 	}
 
 	if (q->properties.is_active)
@@ -385,12 +375,13 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
 	retval = mqd->update_mqd(mqd, q->mqd, &q->properties);
 	if ((q->properties.is_active) && (!prev_active))
 		dqm->queue_count++;
-	else if ((!q->properties.is_active) && (prev_active))
+	else if (!q->properties.is_active && prev_active)
 		dqm->queue_count--;
 
 	if (sched_policy != KFD_SCHED_POLICY_NO_HWS)
 		retval = execute_queues_cpsch(dqm, false);
 
+out_unlock:
 	mutex_unlock(&dqm->lock);
 	return retval;
 }
@@ -400,15 +391,16 @@ static struct mqd_manager *get_mqd_manager_nocpsch(
 {
 	struct mqd_manager *mqd;
 
-	BUG_ON(!dqm || type >= KFD_MQD_TYPE_MAX);
+	if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
+		return NULL;
 
-	pr_debug("kfd: In func %s mqd type %d\n", __func__, type);
+	pr_debug("mqd type %d\n", type);
 
 	mqd = dqm->mqds[type];
 	if (!mqd) {
 		mqd = mqd_manager_init(type, dqm->dev);
-		if (mqd == NULL)
-			pr_err("kfd: mqd manager is NULL");
+		if (!mqd)
+			pr_err("mqd manager is NULL");
 		dqm->mqds[type] = mqd;
 	}
 
@@ -421,11 +413,7 @@ static int register_process_nocpsch(struct device_queue_manager *dqm,
 	struct device_process_node *n;
 	int retval;
 
-	BUG_ON(!dqm || !qpd);
-
-	pr_debug("kfd: In func %s\n", __func__);
-
-	n = kzalloc(sizeof(struct device_process_node), GFP_KERNEL);
+	n = kzalloc(sizeof(*n), GFP_KERNEL);
 	if (!n)
 		return -ENOMEM;
 
@@ -449,10 +437,6 @@ static int unregister_process_nocpsch(struct device_queue_manager *dqm,
 	int retval;
 	struct device_process_node *cur, *next;
 
-	BUG_ON(!dqm || !qpd);
-
-	pr_debug("In func %s\n", __func__);
-
 	pr_debug("qpd->queues_list is %s\n",
 			list_empty(&qpd->queues_list) ? "empty" : "not empty");
 
@@ -493,51 +477,39 @@ static void init_interrupts(struct device_queue_manager *dqm)
 {
 	unsigned int i;
 
-	BUG_ON(dqm == NULL);
-
 	for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++)
 		if (is_pipe_enabled(dqm, 0, i))
 			dqm->dev->kfd2kgd->init_interrupts(dqm->dev->kgd, i);
 }
 
-static int init_scheduler(struct device_queue_manager *dqm)
-{
-	int retval = 0;
-
-	BUG_ON(!dqm);
-
-	pr_debug("kfd: In %s\n", __func__);
-
-	return retval;
-}
-
 static int initialize_nocpsch(struct device_queue_manager *dqm)
 {
-	int i;
+	int pipe, queue;
 
-	BUG_ON(!dqm);
+	pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
 
-	pr_debug("kfd: In func %s num of pipes: %d\n",
-			__func__, get_pipes_per_mec(dqm));
+	dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm),
+					sizeof(unsigned int), GFP_KERNEL);
+	if (!dqm->allocated_queues)
+		return -ENOMEM;
 
 	mutex_init(&dqm->lock);
 	INIT_LIST_HEAD(&dqm->queues);
 	dqm->queue_count = dqm->next_pipe_to_allocate = 0;
 	dqm->sdma_queue_count = 0;
-	dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm),
-					sizeof(unsigned int), GFP_KERNEL);
-	if (!dqm->allocated_queues) {
-		mutex_destroy(&dqm->lock);
-		return -ENOMEM;
-	}
 
-	for (i = 0; i < get_pipes_per_mec(dqm); i++)
-		dqm->allocated_queues[i] = (1 << get_queues_per_pipe(dqm)) - 1;
+	for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
+		int pipe_offset = pipe * get_queues_per_pipe(dqm);
+
+		for (queue = 0; queue < get_queues_per_pipe(dqm); queue++)
+			if (test_bit(pipe_offset + queue,
+				     dqm->dev->shared_resources.queue_bitmap))
+				dqm->allocated_queues[pipe] |= 1 << queue;
+	}
 
 	dqm->vmid_bitmap = (1 << VMID_PER_DEVICE) - 1;
 	dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1;
 
-	init_scheduler(dqm);
 	return 0;
 }
 
@@ -545,9 +517,7 @@ static void uninitialize_nocpsch(struct device_queue_manager *dqm)
 {
 	int i;
 
-	BUG_ON(!dqm);
-
-	BUG_ON(dqm->queue_count > 0 || dqm->processes_count > 0);
+	WARN_ON(dqm->queue_count > 0 || dqm->processes_count > 0);
 
 	kfree(dqm->allocated_queues);
 	for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
@@ -604,33 +574,34 @@ static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
 		return -ENOMEM;
 
 	retval = allocate_sdma_queue(dqm, &q->sdma_id);
-	if (retval != 0)
+	if (retval)
 		return retval;
 
 	q->properties.sdma_queue_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
 	q->properties.sdma_engine_id = q->sdma_id / CIK_SDMA_ENGINE_NUM;
 
-	pr_debug("kfd: sdma id is:    %d\n", q->sdma_id);
-	pr_debug("     sdma queue id: %d\n", q->properties.sdma_queue_id);
-	pr_debug("     sdma engine id: %d\n", q->properties.sdma_engine_id);
+	pr_debug("SDMA id is:    %d\n", q->sdma_id);
+	pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id);
+	pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
 
 	dqm->ops_asic_specific.init_sdma_vm(dqm, q, qpd);
 	retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
 				&q->gart_mqd_addr, &q->properties);
-	if (retval != 0) {
-		deallocate_sdma_queue(dqm, q->sdma_id);
-		return retval;
-	}
+	if (retval)
+		goto out_deallocate_sdma_queue;
 
-	retval = mqd->load_mqd(mqd, q->mqd, 0,
-				0, NULL);
-	if (retval != 0) {
-		deallocate_sdma_queue(dqm, q->sdma_id);
-		mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
-		return retval;
-	}
+	retval = mqd->load_mqd(mqd, q->mqd, 0, 0, &q->properties, NULL);
+	if (retval)
+		goto out_uninit_mqd;
 
 	return 0;
+
+out_uninit_mqd:
+	mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
+out_deallocate_sdma_queue:
+	deallocate_sdma_queue(dqm, q->sdma_id);
+
+	return retval;
 }
 
 /*
@@ -642,10 +613,6 @@ static int set_sched_resources(struct device_queue_manager *dqm)
 	int i, mec;
 	struct scheduling_resources res;
 
-	BUG_ON(!dqm);
-
-	pr_debug("kfd: In func %s\n", __func__);
-
 	res.vmid_mask = (1 << VMID_PER_DEVICE) - 1;
 	res.vmid_mask <<= KFD_VMID_START_OFFSET;
 
@@ -663,8 +630,9 @@ static int set_sched_resources(struct device_queue_manager *dqm)
 
 		/* This situation may be hit in the future if a new HW
 		 * generation exposes more than 64 queues. If so, the
-		 * definition of res.queue_mask needs updating */
-		if (WARN_ON(i > (sizeof(res.queue_mask)*8))) {
+		 * definition of res.queue_mask needs updating
+		 */
+		if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) {
 			pr_err("Invalid queue enabled by amdgpu: %d\n", i);
 			break;
 		}
@@ -674,9 +642,9 @@ static int set_sched_resources(struct device_queue_manager *dqm)
 	res.gws_mask = res.oac_mask = res.gds_heap_base =
 						res.gds_heap_size = 0;
 
-	pr_debug("kfd: scheduling resources:\n"
-			"      vmid mask: 0x%8X\n"
-			"      queue mask: 0x%8llX\n",
+	pr_debug("Scheduling resources:\n"
+			"vmid mask: 0x%8X\n"
+			"queue mask: 0x%8llX\n",
 			res.vmid_mask, res.queue_mask);
 
 	return pm_send_set_resources(&dqm->packets, &res);
@@ -686,10 +654,7 @@ static int initialize_cpsch(struct device_queue_manager *dqm)
 {
 	int retval;
 
-	BUG_ON(!dqm);
-
-	pr_debug("kfd: In func %s num of pipes: %d\n",
-			__func__, get_pipes_per_mec(dqm));
+	pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
 
 	mutex_init(&dqm->lock);
 	INIT_LIST_HEAD(&dqm->queues);
@@ -697,13 +662,9 @@ static int initialize_cpsch(struct device_queue_manager *dqm)
 	dqm->sdma_queue_count = 0;
 	dqm->active_runlist = false;
 	retval = dqm->ops_asic_specific.initialize(dqm);
-	if (retval != 0)
-		goto fail_init_pipelines;
-
-	return 0;
+	if (retval)
+		mutex_destroy(&dqm->lock);
 
-fail_init_pipelines:
-	mutex_destroy(&dqm->lock);
 	return retval;
 }
 
@@ -712,25 +673,23 @@ static int start_cpsch(struct device_queue_manager *dqm)
 	struct device_process_node *node;
 	int retval;
 
-	BUG_ON(!dqm);
-
 	retval = 0;
 
 	retval = pm_init(&dqm->packets, dqm);
-	if (retval != 0)
+	if (retval)
 		goto fail_packet_manager_init;
 
 	retval = set_sched_resources(dqm);
-	if (retval != 0)
+	if (retval)
 		goto fail_set_sched_resources;
 
-	pr_debug("kfd: allocating fence memory\n");
+	pr_debug("Allocating fence memory\n");
 
 	/* allocate fence memory on the gart */
 	retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr),
 					&dqm->fence_mem);
 
-	if (retval != 0)
+	if (retval)
 		goto fail_allocate_vidmem;
 
 	dqm->fence_addr = dqm->fence_mem->cpu_ptr;
@@ -758,8 +717,6 @@ static int stop_cpsch(struct device_queue_manager *dqm)
 	struct device_process_node *node;
 	struct kfd_process_device *pdd;
 
-	BUG_ON(!dqm);
-
 	destroy_queues_cpsch(dqm, true, true);
 
 	list_for_each_entry(node, &dqm->queues, list) {
@@ -776,13 +733,9 @@ static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
 					struct kernel_queue *kq,
 					struct qcm_process_device *qpd)
 {
-	BUG_ON(!dqm || !kq || !qpd);
-
-	pr_debug("kfd: In func %s\n", __func__);
-
 	mutex_lock(&dqm->lock);
 	if (dqm->total_queue_count >= max_num_of_queues_per_device) {
-		pr_warn("amdkfd: Can't create new kernel queue because %d queues were already created\n",
+		pr_warn("Can't create new kernel queue because %d queues were already created\n",
 				dqm->total_queue_count);
 		mutex_unlock(&dqm->lock);
 		return -EPERM;
@@ -809,10 +762,6 @@ static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
 					struct kernel_queue *kq,
 					struct qcm_process_device *qpd)
 {
-	BUG_ON(!dqm || !kq);
-
-	pr_debug("kfd: In %s\n", __func__);
-
 	mutex_lock(&dqm->lock);
 	/* here we actually preempt the DIQ */
 	destroy_queues_cpsch(dqm, true, false);
@@ -844,8 +793,6 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 	int retval;
 	struct mqd_manager *mqd;
 
-	BUG_ON(!dqm || !q || !qpd);
-
 	retval = 0;
 
 	if (allocate_vmid)
@@ -854,7 +801,7 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 	mutex_lock(&dqm->lock);
 
 	if (dqm->total_queue_count >= max_num_of_queues_per_device) {
-		pr_warn("amdkfd: Can't create new usermode queue because %d queues were already created\n",
+		pr_warn("Can't create new usermode queue because %d queues were already created\n",
 				dqm->total_queue_count);
 		retval = -EPERM;
 		goto out;
@@ -866,15 +813,15 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 	mqd = dqm->ops.get_mqd_manager(dqm,
 			get_mqd_type_from_queue_type(q->properties.type));
 
-	if (mqd == NULL) {
-		mutex_unlock(&dqm->lock);
-		return -ENOMEM;
+	if (!mqd) {
+		retval = -ENOMEM;
+		goto out;
 	}
 
 	dqm->ops_asic_specific.init_sdma_vm(dqm, q, qpd);
 	retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
 				&q->gart_mqd_addr, &q->properties);
-	if (retval != 0)
+	if (retval)
 		goto out;
 
 	list_add(&q->list, &qpd->queues_list);
@@ -884,7 +831,7 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 	}
 
 	if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
-			dqm->sdma_queue_count++;
+		dqm->sdma_queue_count++;
 	/*
 	 * Unconditionally increment this counter, regardless of the queue's
 	 * type or whether the queue is active.
@@ -903,12 +850,11 @@ int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
 				unsigned int fence_value,
 				unsigned long timeout)
 {
-	BUG_ON(!fence_addr);
 	timeout += jiffies;
 
 	while (*fence_addr != fence_value) {
 		if (time_after(jiffies, timeout)) {
-			pr_err("kfd: qcm fence wait loop timeout expired\n");
+			pr_err("qcm fence wait loop timeout expired\n");
 			return -ETIME;
 		}
 		schedule();
@@ -932,8 +878,6 @@ static int destroy_queues_cpsch(struct device_queue_manager *dqm,
 	enum kfd_preempt_type_filter preempt_type;
 	struct kfd_process_device *pdd;
 
-	BUG_ON(!dqm);
-
 	retval = 0;
 
 	if (lock)
@@ -941,7 +885,7 @@ static int destroy_queues_cpsch(struct device_queue_manager *dqm,
 	if (!dqm->active_runlist)
 		goto out;
 
-	pr_debug("kfd: Before destroying queues, sdma queue count is : %u\n",
+	pr_debug("Before destroying queues, sdma queue count is : %u\n",
 		dqm->sdma_queue_count);
 
 	if (dqm->sdma_queue_count > 0) {
@@ -955,7 +899,7 @@ static int destroy_queues_cpsch(struct device_queue_manager *dqm,
 
 	retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE,
 			preempt_type, 0, false, 0);
-	if (retval != 0)
+	if (retval)
 		goto out;
 
 	*dqm->fence_addr = KFD_FENCE_INIT;
@@ -964,7 +908,7 @@ static int destroy_queues_cpsch(struct device_queue_manager *dqm,
 	/* should be timed out */
 	retval = amdkfd_fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED,
 				QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS);
-	if (retval != 0) {
+	if (retval) {
 		pdd = kfd_get_process_device_data(dqm->dev,
 				kfd_get_process(current));
 		pdd->reset_wavefronts = true;
@@ -983,14 +927,12 @@ static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock)
 {
 	int retval;
 
-	BUG_ON(!dqm);
-
 	if (lock)
 		mutex_lock(&dqm->lock);
 
 	retval = destroy_queues_cpsch(dqm, false, false);
-	if (retval != 0) {
-		pr_err("kfd: the cp might be in an unrecoverable state due to an unsuccessful queues preemption");
+	if (retval) {
+		pr_err("The cp might be in an unrecoverable state due to an unsuccessful queues preemption");
 		goto out;
 	}
 
@@ -1005,8 +947,8 @@ static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock)
 	}
 
 	retval = pm_send_runlist(&dqm->packets, &dqm->queues);
-	if (retval != 0) {
-		pr_err("kfd: failed to execute runlist");
+	if (retval) {
+		pr_err("failed to execute runlist");
 		goto out;
 	}
 	dqm->active_runlist = true;
@@ -1025,8 +967,6 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,
 	struct mqd_manager *mqd;
 	bool preempt_all_queues;
 
-	BUG_ON(!dqm || !qpd || !q);
-
 	preempt_all_queues = false;
 
 	retval = 0;
@@ -1098,8 +1038,6 @@ static bool set_cache_memory_policy(struct device_queue_manager *dqm,
 {
 	bool retval;
 
-	pr_debug("kfd: In func %s\n", __func__);
-
 	mutex_lock(&dqm->lock);
 
 	if (alternate_aperture_size == 0) {
@@ -1120,14 +1058,11 @@ static bool set_cache_memory_policy(struct device_queue_manager *dqm,
 		uint64_t base = (uintptr_t)alternate_aperture_base;
 		uint64_t limit = base + alternate_aperture_size - 1;
 
-		if (limit <= base)
-			goto out;
-
-		if ((base & APE1_FIXED_BITS_MASK) != 0)
-			goto out;
-
-		if ((limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT)
+		if (limit <= base || (base & APE1_FIXED_BITS_MASK) != 0 ||
+		   (limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT) {
+			retval = false;
 			goto out;
+		}
 
 		qpd->sh_mem_ape1_base = base >> 16;
 		qpd->sh_mem_ape1_limit = limit >> 16;
@@ -1144,27 +1079,22 @@ static bool set_cache_memory_policy(struct device_queue_manager *dqm,
 	if ((sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
 		program_sh_mem_settings(dqm, qpd);
 
-	pr_debug("kfd: sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
+	pr_debug("sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
 		qpd->sh_mem_config, qpd->sh_mem_ape1_base,
 		qpd->sh_mem_ape1_limit);
 
-	mutex_unlock(&dqm->lock);
-	return retval;
-
 out:
 	mutex_unlock(&dqm->lock);
-	return false;
+	return retval;
 }
 
 struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
 {
 	struct device_queue_manager *dqm;
 
-	BUG_ON(!dev);
+	pr_debug("Loading device queue manager\n");
 
-	pr_debug("kfd: loading device queue manager\n");
-
-	dqm = kzalloc(sizeof(struct device_queue_manager), GFP_KERNEL);
+	dqm = kzalloc(sizeof(*dqm), GFP_KERNEL);
 	if (!dqm)
 		return NULL;
 
@@ -1202,8 +1132,8 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
 		dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
 		break;
 	default:
-		BUG();
-		break;
+		pr_err("Invalid scheduling policy %d\n", sched_policy);
+		goto out_free;
 	}
 
 	switch (dev->device_info->asic_family) {
@@ -1216,18 +1146,16 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
 		break;
 	}
 
-	if (dqm->ops.initialize(dqm) != 0) {
-		kfree(dqm);
-		return NULL;
-	}
+	if (!dqm->ops.initialize(dqm))
+		return dqm;
 
-	return dqm;
+out_free:
+	kfree(dqm);
+	return NULL;
 }
 
 void device_queue_manager_uninit(struct device_queue_manager *dqm)
 {
-	BUG_ON(!dqm);
-
 	dqm->ops.uninitialize(dqm);
 	kfree(dqm);
 }
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c
index 48dc0561b402..72c3cbabc0a7 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c
@@ -24,6 +24,7 @@
 #include "kfd_device_queue_manager.h"
 #include "cik_regs.h"
 #include "oss/oss_2_4_sh_mask.h"
+#include "gca/gfx_7_2_sh_mask.h"
 
 static bool set_cache_memory_policy_cik(struct device_queue_manager *dqm,
 				   struct qcm_process_device *qpd,
@@ -65,7 +66,7 @@ static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble)
 	 * for LDS/Scratch and GPUVM.
 	 */
 
-	BUG_ON((top_address_nybble & 1) || top_address_nybble > 0xE ||
+	WARN_ON((top_address_nybble & 1) || top_address_nybble > 0xE ||
 		top_address_nybble == 0);
 
 	return PRIVATE_BASE(top_address_nybble << 12) |
@@ -104,8 +105,6 @@ static int register_process_cik(struct device_queue_manager *dqm,
 	struct kfd_process_device *pdd;
 	unsigned int temp;
 
-	BUG_ON(!dqm || !qpd);
-
 	pdd = qpd_to_pdd(qpd);
 
 	/* check if sh_mem_config register already configured */
@@ -125,9 +124,10 @@ static int register_process_cik(struct device_queue_manager *dqm,
 	} else {
 		temp = get_sh_mem_bases_nybble_64(pdd);
 		qpd->sh_mem_bases = compute_sh_mem_bases_64bit(temp);
+		qpd->sh_mem_config |= 1  << SH_MEM_CONFIG__PRIVATE_ATC__SHIFT;
 	}
 
-	pr_debug("kfd: is32bit process: %d sh_mem_bases nybble: 0x%X and register 0x%X\n",
+	pr_debug("is32bit process: %d sh_mem_bases nybble: 0x%X and register 0x%X\n",
 		qpd->pqm->process->is_32bit_user_mode, temp, qpd->sh_mem_bases);
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c
index 7e9cae9d349b..40e9ddd096cd 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c
@@ -67,7 +67,7 @@ static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble)
 	 * for LDS/Scratch and GPUVM.
 	 */
 
-	BUG_ON((top_address_nybble & 1) || top_address_nybble > 0xE ||
+	WARN_ON((top_address_nybble & 1) || top_address_nybble > 0xE ||
 		top_address_nybble == 0);
 
 	return top_address_nybble << 12 |
@@ -110,8 +110,6 @@ static int register_process_vi(struct device_queue_manager *dqm,
 	struct kfd_process_device *pdd;
 	unsigned int temp;
 
-	BUG_ON(!dqm || !qpd);
-
 	pdd = qpd_to_pdd(qpd);
 
 	/* check if sh_mem_config register already configured */
@@ -137,9 +135,11 @@ static int register_process_vi(struct device_queue_manager *dqm,
 		qpd->sh_mem_bases = compute_sh_mem_bases_64bit(temp);
 		qpd->sh_mem_config |= SH_MEM_ADDRESS_MODE_HSA64 <<
 			SH_MEM_CONFIG__ADDRESS_MODE__SHIFT;
+		qpd->sh_mem_config |= 1  <<
+			SH_MEM_CONFIG__PRIVATE_ATC__SHIFT;
 	}
 
-	pr_debug("kfd: is32bit process: %d sh_mem_bases nybble: 0x%X and register 0x%X\n",
+	pr_debug("is32bit process: %d sh_mem_bases nybble: 0x%X and register 0x%X\n",
 		qpd->pqm->process->is_32bit_user_mode, temp, qpd->sh_mem_bases);
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
index 453c5d66e5c3..acf4d2a977ad 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
@@ -59,7 +59,7 @@ static inline size_t doorbell_process_allocation(void)
 }
 
 /* Doorbell calculations for device init. */
-void kfd_doorbell_init(struct kfd_dev *kfd)
+int kfd_doorbell_init(struct kfd_dev *kfd)
 {
 	size_t doorbell_start_offset;
 	size_t doorbell_aperture_size;
@@ -95,26 +95,35 @@ void kfd_doorbell_init(struct kfd_dev *kfd)
 	kfd->doorbell_kernel_ptr = ioremap(kfd->doorbell_base,
 						doorbell_process_allocation());
 
-	BUG_ON(!kfd->doorbell_kernel_ptr);
+	if (!kfd->doorbell_kernel_ptr)
+		return -ENOMEM;
 
-	pr_debug("kfd: doorbell initialization:\n");
-	pr_debug("kfd: doorbell base           == 0x%08lX\n",
+	pr_debug("Doorbell initialization:\n");
+	pr_debug("doorbell base           == 0x%08lX\n",
 			(uintptr_t)kfd->doorbell_base);
 
-	pr_debug("kfd: doorbell_id_offset      == 0x%08lX\n",
+	pr_debug("doorbell_id_offset      == 0x%08lX\n",
 			kfd->doorbell_id_offset);
 
-	pr_debug("kfd: doorbell_process_limit  == 0x%08lX\n",
+	pr_debug("doorbell_process_limit  == 0x%08lX\n",
 			doorbell_process_limit);
 
-	pr_debug("kfd: doorbell_kernel_offset  == 0x%08lX\n",
+	pr_debug("doorbell_kernel_offset  == 0x%08lX\n",
 			(uintptr_t)kfd->doorbell_base);
 
-	pr_debug("kfd: doorbell aperture size  == 0x%08lX\n",
+	pr_debug("doorbell aperture size  == 0x%08lX\n",
 			kfd->shared_resources.doorbell_aperture_size);
 
-	pr_debug("kfd: doorbell kernel address == 0x%08lX\n",
+	pr_debug("doorbell kernel address == 0x%08lX\n",
 			(uintptr_t)kfd->doorbell_kernel_ptr);
+
+	return 0;
+}
+
+void kfd_doorbell_fini(struct kfd_dev *kfd)
+{
+	if (kfd->doorbell_kernel_ptr)
+		iounmap(kfd->doorbell_kernel_ptr);
 }
 
 int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma)
@@ -131,7 +140,7 @@ int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma)
 
 	/* Find kfd device according to gpu id */
 	dev = kfd_device_by_id(vma->vm_pgoff);
-	if (dev == NULL)
+	if (!dev)
 		return -EINVAL;
 
 	/* Calculate physical address of doorbell */
@@ -142,12 +151,11 @@ int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma)
 
 	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 
-	pr_debug("kfd: mapping doorbell page in %s\n"
+	pr_debug("Mapping doorbell page\n"
 		 "     target user address == 0x%08llX\n"
 		 "     physical address    == 0x%08llX\n"
 		 "     vm_flags            == 0x%04lX\n"
 		 "     size                == 0x%04lX\n",
-		 __func__,
 		 (unsigned long long) vma->vm_start, address, vma->vm_flags,
 		 doorbell_process_allocation());
 
@@ -166,8 +174,6 @@ u32 __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
 {
 	u32 inx;
 
-	BUG_ON(!kfd || !doorbell_off);
-
 	mutex_lock(&kfd->doorbell_mutex);
 	inx = find_first_zero_bit(kfd->doorbell_available_index,
 					KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
@@ -185,7 +191,7 @@ u32 __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
 	*doorbell_off = KERNEL_DOORBELL_PASID * (doorbell_process_allocation() /
 							sizeof(u32)) + inx;
 
-	pr_debug("kfd: get kernel queue doorbell\n"
+	pr_debug("Get kernel queue doorbell\n"
 			 "     doorbell offset   == 0x%08X\n"
 			 "     kernel address    == 0x%08lX\n",
 		*doorbell_off, (uintptr_t)(kfd->doorbell_kernel_ptr + inx));
@@ -197,8 +203,6 @@ void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr)
 {
 	unsigned int inx;
 
-	BUG_ON(!kfd || !db_addr);
-
 	inx = (unsigned int)(db_addr - kfd->doorbell_kernel_ptr);
 
 	mutex_lock(&kfd->doorbell_mutex);
@@ -210,7 +214,7 @@ inline void write_kernel_doorbell(u32 __iomem *db, u32 value)
 {
 	if (db) {
 		writel(value, db);
-		pr_debug("writing %d to doorbell address 0x%p\n", value, db);
+		pr_debug("Writing %d to doorbell address 0x%p\n", value, db);
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
index d1ce83d73a87..5979158c3f7b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
@@ -110,7 +110,7 @@ static bool allocate_free_slot(struct kfd_process *process,
 			*out_page = page;
 			*out_slot_index = slot;
 
-			pr_debug("allocated event signal slot in page %p, slot %d\n",
+			pr_debug("Allocated event signal slot in page %p, slot %d\n",
 					page, slot);
 
 			return true;
@@ -155,9 +155,9 @@ static bool allocate_signal_page(struct file *devkfd, struct kfd_process *p)
 						   struct signal_page,
 						   event_pages)->page_index + 1;
 
-	pr_debug("allocated new event signal page at %p, for process %p\n",
+	pr_debug("Allocated new event signal page at %p, for process %p\n",
 			page, p);
-	pr_debug("page index is %d\n", page->page_index);
+	pr_debug("Page index is %d\n", page->page_index);
 
 	list_add(&page->event_pages, &p->signal_event_pages);
 
@@ -194,7 +194,8 @@ static void release_event_notification_slot(struct signal_page *page,
 	page->free_slots++;
 
 	/* We don't free signal pages, they are retained by the process
-	 * and reused until it exits. */
+	 * and reused until it exits.
+	 */
 }
 
 static struct signal_page *lookup_signal_page_by_index(struct kfd_process *p,
@@ -246,7 +247,7 @@ static u32 make_nonsignal_event_id(struct kfd_process *p)
 
 	for (id = p->next_nonsignal_event_id;
 		id < KFD_LAST_NONSIGNAL_EVENT_ID &&
-		lookup_event_by_id(p, id) != NULL;
+		lookup_event_by_id(p, id);
 		id++)
 		;
 
@@ -265,7 +266,7 @@ static u32 make_nonsignal_event_id(struct kfd_process *p)
 
 	for (id = KFD_FIRST_NONSIGNAL_EVENT_ID;
 		id < KFD_LAST_NONSIGNAL_EVENT_ID &&
-		lookup_event_by_id(p, id) != NULL;
+		lookup_event_by_id(p, id);
 		id++)
 		;
 
@@ -291,13 +292,13 @@ static int create_signal_event(struct file *devkfd,
 				struct kfd_event *ev)
 {
 	if (p->signal_event_count == KFD_SIGNAL_EVENT_LIMIT) {
-		pr_warn("amdkfd: Signal event wasn't created because limit was reached\n");
+		pr_warn("Signal event wasn't created because limit was reached\n");
 		return -ENOMEM;
 	}
 
 	if (!allocate_event_notification_slot(devkfd, p, &ev->signal_page,
 						&ev->signal_slot_index)) {
-		pr_warn("amdkfd: Signal event wasn't created because out of kernel memory\n");
+		pr_warn("Signal event wasn't created because out of kernel memory\n");
 		return -ENOMEM;
 	}
 
@@ -309,11 +310,7 @@ static int create_signal_event(struct file *devkfd,
 	ev->event_id = make_signal_event_id(ev->signal_page,
 						ev->signal_slot_index);
 
-	pr_debug("signal event number %zu created with id %d, address %p\n",
-			p->signal_event_count, ev->event_id,
-			ev->user_signal_address);
-
-	pr_debug("signal event number %zu created with id %d, address %p\n",
+	pr_debug("Signal event number %zu created with id %d, address %p\n",
 			p->signal_event_count, ev->event_id,
 			ev->user_signal_address);
 
@@ -345,7 +342,7 @@ void kfd_event_init_process(struct kfd_process *p)
 
 static void destroy_event(struct kfd_process *p, struct kfd_event *ev)
 {
-	if (ev->signal_page != NULL) {
+	if (ev->signal_page) {
 		release_event_notification_slot(ev->signal_page,
 						ev->signal_slot_index);
 		p->signal_event_count--;
@@ -584,7 +581,7 @@ void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id,
 		 * search faster.
 		 */
 		struct signal_page *page;
-		unsigned i;
+		unsigned int i;
 
 		list_for_each_entry(page, &p->signal_event_pages, event_pages)
 			for (i = 0; i < SLOTS_PER_PAGE; i++)
@@ -816,7 +813,7 @@ int kfd_event_mmap(struct kfd_process *p, struct vm_area_struct *vma)
 	/* check required size is logical */
 	if (get_order(KFD_SIGNAL_EVENT_LIMIT * 8) !=
 			get_order(vma->vm_end - vma->vm_start)) {
-		pr_err("amdkfd: event page mmap requested illegal size\n");
+		pr_err("Event page mmap requested illegal size\n");
 		return -EINVAL;
 	}
 
@@ -825,7 +822,7 @@ int kfd_event_mmap(struct kfd_process *p, struct vm_area_struct *vma)
 	page = lookup_signal_page_by_index(p, page_index);
 	if (!page) {
 		/* Probably KFD bug, but mmap is user-accessible. */
-		pr_debug("signal page could not be found for page_index %u\n",
+		pr_debug("Signal page could not be found for page_index %u\n",
 				page_index);
 		return -EINVAL;
 	}
@@ -836,7 +833,7 @@ int kfd_event_mmap(struct kfd_process *p, struct vm_area_struct *vma)
 	vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE
 		       | VM_DONTDUMP | VM_PFNMAP;
 
-	pr_debug("mapping signal page\n");
+	pr_debug("Mapping signal page\n");
 	pr_debug("     start user address  == 0x%08lx\n", vma->vm_start);
 	pr_debug("     end user address    == 0x%08lx\n", vma->vm_end);
 	pr_debug("     pfn                 == 0x%016lX\n", pfn);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
index 2b655103ba79..c59384bbbc5f 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
@@ -304,7 +304,7 @@ int kfd_init_apertures(struct kfd_process *process)
 		id < NUM_OF_SUPPORTED_GPUS) {
 
 		pdd = kfd_create_process_device_data(dev, process);
-		if (pdd == NULL) {
+		if (!pdd) {
 			pr_err("Failed to create process device data\n");
 			return -1;
 		}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c b/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c
index 7f134aa9bfd3..70b3a99cffc2 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c
@@ -179,7 +179,7 @@ static void interrupt_wq(struct work_struct *work)
 bool interrupt_is_wanted(struct kfd_dev *dev, const uint32_t *ih_ring_entry)
 {
 	/* integer and bitwise OR so there is no boolean short-circuiting */
-	unsigned wanted = 0;
+	unsigned int wanted = 0;
 
 	wanted |= dev->device_info->event_interrupt_class->interrupt_isr(dev,
 								ih_ring_entry);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
index d135cd002a95..0649dd43e780 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
@@ -41,11 +41,11 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
 	int retval;
 	union PM4_MES_TYPE_3_HEADER nop;
 
-	BUG_ON(!kq || !dev);
-	BUG_ON(type != KFD_QUEUE_TYPE_DIQ && type != KFD_QUEUE_TYPE_HIQ);
+	if (WARN_ON(type != KFD_QUEUE_TYPE_DIQ && type != KFD_QUEUE_TYPE_HIQ))
+		return false;
 
-	pr_debug("amdkfd: In func %s initializing queue type %d size %d\n",
-			__func__, KFD_QUEUE_TYPE_HIQ, queue_size);
+	pr_debug("Initializing queue type %d size %d\n", KFD_QUEUE_TYPE_HIQ,
+			queue_size);
 
 	memset(&prop, 0, sizeof(prop));
 	memset(&nop, 0, sizeof(nop));
@@ -63,23 +63,23 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
 						KFD_MQD_TYPE_HIQ);
 		break;
 	default:
-		BUG();
-		break;
+		pr_err("Invalid queue type %d\n", type);
+		return false;
 	}
 
-	if (kq->mqd == NULL)
+	if (!kq->mqd)
 		return false;
 
 	prop.doorbell_ptr = kfd_get_kernel_doorbell(dev, &prop.doorbell_off);
 
-	if (prop.doorbell_ptr == NULL) {
-		pr_err("amdkfd: error init doorbell");
+	if (!prop.doorbell_ptr) {
+		pr_err("Failed to initialize doorbell");
 		goto err_get_kernel_doorbell;
 	}
 
 	retval = kfd_gtt_sa_allocate(dev, queue_size, &kq->pq);
 	if (retval != 0) {
-		pr_err("amdkfd: error init pq queues size (%d)\n", queue_size);
+		pr_err("Failed to init pq queues size %d\n", queue_size);
 		goto err_pq_allocate_vidmem;
 	}
 
@@ -87,7 +87,7 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
 	kq->pq_gpu_addr = kq->pq->gpu_addr;
 
 	retval = kq->ops_asic_specific.initialize(kq, dev, type, queue_size);
-	if (retval == false)
+	if (!retval)
 		goto err_eop_allocate_vidmem;
 
 	retval = kfd_gtt_sa_allocate(dev, sizeof(*kq->rptr_kernel),
@@ -139,11 +139,12 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev,
 
 	/* assign HIQ to HQD */
 	if (type == KFD_QUEUE_TYPE_HIQ) {
-		pr_debug("assigning hiq to hqd\n");
+		pr_debug("Assigning hiq to hqd\n");
 		kq->queue->pipe = KFD_CIK_HIQ_PIPE;
 		kq->queue->queue = KFD_CIK_HIQ_QUEUE;
 		kq->mqd->load_mqd(kq->mqd, kq->queue->mqd, kq->queue->pipe,
-					kq->queue->queue, NULL);
+				  kq->queue->queue, &kq->queue->properties,
+				  NULL);
 	} else {
 		/* allocate fence for DIQ */
 
@@ -180,11 +181,9 @@ err_get_kernel_doorbell:
 
 static void uninitialize(struct kernel_queue *kq)
 {
-	BUG_ON(!kq);
-
 	if (kq->queue->properties.type == KFD_QUEUE_TYPE_HIQ)
 		kq->mqd->destroy_mqd(kq->mqd,
-					NULL,
+					kq->queue->mqd,
 					false,
 					QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS,
 					kq->queue->pipe,
@@ -211,8 +210,6 @@ static int acquire_packet_buffer(struct kernel_queue *kq,
 	uint32_t wptr, rptr;
 	unsigned int *queue_address;
 
-	BUG_ON(!kq || !buffer_ptr);
-
 	rptr = *kq->rptr_kernel;
 	wptr = *kq->wptr_kernel;
 	queue_address = (unsigned int *)kq->pq_kernel_addr;
@@ -252,11 +249,7 @@ static void submit_packet(struct kernel_queue *kq)
 {
 #ifdef DEBUG
 	int i;
-#endif
-
-	BUG_ON(!kq);
 
-#ifdef DEBUG
 	for (i = *kq->wptr_kernel; i < kq->pending_wptr; i++) {
 		pr_debug("0x%2X ", kq->pq_kernel_addr[i]);
 		if (i % 15 == 0)
@@ -272,7 +265,6 @@ static void submit_packet(struct kernel_queue *kq)
 
 static void rollback_packet(struct kernel_queue *kq)
 {
-	BUG_ON(!kq);
 	kq->pending_wptr = *kq->queue->properties.write_ptr;
 }
 
@@ -281,9 +273,7 @@ struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
 {
 	struct kernel_queue *kq;
 
-	BUG_ON(!dev);
-
-	kq = kzalloc(sizeof(struct kernel_queue), GFP_KERNEL);
+	kq = kzalloc(sizeof(*kq), GFP_KERNEL);
 	if (!kq)
 		return NULL;
 
@@ -304,7 +294,7 @@ struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
 	}
 
 	if (!kq->ops.initialize(kq, dev, type, KFD_KERNEL_QUEUE_SIZE)) {
-		pr_err("amdkfd: failed to init kernel queue\n");
+		pr_err("Failed to init kernel queue\n");
 		kfree(kq);
 		return NULL;
 	}
@@ -313,32 +303,37 @@ struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
 
 void kernel_queue_uninit(struct kernel_queue *kq)
 {
-	BUG_ON(!kq);
-
 	kq->ops.uninitialize(kq);
 	kfree(kq);
 }
 
+/* FIXME: Can this test be removed? */
 static __attribute__((unused)) void test_kq(struct kfd_dev *dev)
 {
 	struct kernel_queue *kq;
 	uint32_t *buffer, i;
 	int retval;
 
-	BUG_ON(!dev);
-
-	pr_err("amdkfd: starting kernel queue test\n");
+	pr_err("Starting kernel queue test\n");
 
 	kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_HIQ);
-	BUG_ON(!kq);
+	if (unlikely(!kq)) {
+		pr_err("  Failed to initialize HIQ\n");
+		pr_err("Kernel queue test failed\n");
+		return;
+	}
 
 	retval = kq->ops.acquire_packet_buffer(kq, 5, &buffer);
-	BUG_ON(retval != 0);
+	if (unlikely(retval != 0)) {
+		pr_err("  Failed to acquire packet buffer\n");
+		pr_err("Kernel queue test failed\n");
+		return;
+	}
 	for (i = 0; i < 5; i++)
 		buffer[i] = kq->nop_packet;
 	kq->ops.submit_packet(kq);
 
-	pr_err("amdkfd: ending kernel queue test\n");
+	pr_err("Ending kernel queue test\n");
 }
 
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_module.c b/drivers/gpu/drm/amd/amdkfd/kfd_module.c
index 850a5623661f..0d73bea22c45 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_module.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_module.c
@@ -61,7 +61,8 @@ MODULE_PARM_DESC(send_sigterm,
 
 static int amdkfd_init_completed;
 
-int kgd2kfd_init(unsigned interface_version, const struct kgd2kfd_calls **g2f)
+int kgd2kfd_init(unsigned int interface_version,
+		const struct kgd2kfd_calls **g2f)
 {
 	if (!amdkfd_init_completed)
 		return -EPROBE_DEFER;
@@ -90,7 +91,7 @@ static int __init kfd_module_init(void)
 	/* Verify module parameters */
 	if ((sched_policy < KFD_SCHED_POLICY_HWS) ||
 		(sched_policy > KFD_SCHED_POLICY_NO_HWS)) {
-		pr_err("kfd: sched_policy has invalid value\n");
+		pr_err("sched_policy has invalid value\n");
 		return -1;
 	}
 
@@ -98,13 +99,13 @@ static int __init kfd_module_init(void)
 	if ((max_num_of_queues_per_device < 1) ||
 		(max_num_of_queues_per_device >
 			KFD_MAX_NUM_OF_QUEUES_PER_DEVICE)) {
-		pr_err("kfd: max_num_of_queues_per_device must be between 1 to KFD_MAX_NUM_OF_QUEUES_PER_DEVICE\n");
+		pr_err("max_num_of_queues_per_device must be between 1 to KFD_MAX_NUM_OF_QUEUES_PER_DEVICE\n");
 		return -1;
 	}
 
 	err = kfd_pasid_init();
 	if (err < 0)
-		goto err_pasid;
+		return err;
 
 	err = kfd_chardev_init();
 	if (err < 0)
@@ -126,7 +127,6 @@ err_topology:
 	kfd_chardev_exit();
 err_ioctl:
 	kfd_pasid_exit();
-err_pasid:
 	return err;
 }
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
index 213a71e0b6c7..1f3a6ba7eed2 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
@@ -67,7 +67,8 @@ struct mqd_manager {
 
 	int	(*load_mqd)(struct mqd_manager *mm, void *mqd,
 				uint32_t pipe_id, uint32_t queue_id,
-				uint32_t __user *wptr);
+				struct queue_properties *p,
+				struct mm_struct *mms);
 
 	int	(*update_mqd)(struct mqd_manager *mm, void *mqd,
 				struct queue_properties *q);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
index 6acc4313363e..44ffd23348fc 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
@@ -44,10 +44,6 @@ static int init_mqd(struct mqd_manager *mm, void **mqd,
 	struct cik_mqd *m;
 	int retval;
 
-	BUG_ON(!mm || !q || !mqd);
-
-	pr_debug("kfd: In func %s\n", __func__);
-
 	retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct cik_mqd),
 					mqd_mem_obj);
 
@@ -101,7 +97,7 @@ static int init_mqd(struct mqd_manager *mm, void **mqd,
 		m->cp_hqd_iq_rptr = AQL_ENABLE;
 
 	*mqd = m;
-	if (gart_addr != NULL)
+	if (gart_addr)
 		*gart_addr = addr;
 	retval = mm->update_mqd(mm, m, q);
 
@@ -115,8 +111,6 @@ static int init_mqd_sdma(struct mqd_manager *mm, void **mqd,
 	int retval;
 	struct cik_sdma_rlc_registers *m;
 
-	BUG_ON(!mm || !mqd || !mqd_mem_obj);
-
 	retval = kfd_gtt_sa_allocate(mm->dev,
 					sizeof(struct cik_sdma_rlc_registers),
 					mqd_mem_obj);
@@ -129,7 +123,7 @@ static int init_mqd_sdma(struct mqd_manager *mm, void **mqd,
 	memset(m, 0, sizeof(struct cik_sdma_rlc_registers));
 
 	*mqd = m;
-	if (gart_addr != NULL)
+	if (gart_addr)
 		*gart_addr = (*mqd_mem_obj)->gpu_addr;
 
 	retval = mm->update_mqd(mm, m, q);
@@ -140,27 +134,31 @@ static int init_mqd_sdma(struct mqd_manager *mm, void **mqd,
 static void uninit_mqd(struct mqd_manager *mm, void *mqd,
 			struct kfd_mem_obj *mqd_mem_obj)
 {
-	BUG_ON(!mm || !mqd);
 	kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
 }
 
 static void uninit_mqd_sdma(struct mqd_manager *mm, void *mqd,
 				struct kfd_mem_obj *mqd_mem_obj)
 {
-	BUG_ON(!mm || !mqd);
 	kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
 }
 
 static int load_mqd(struct mqd_manager *mm, void *mqd, uint32_t pipe_id,
-			uint32_t queue_id, uint32_t __user *wptr)
+		    uint32_t queue_id, struct queue_properties *p,
+		    struct mm_struct *mms)
 {
-	return mm->dev->kfd2kgd->hqd_load
-		(mm->dev->kgd, mqd, pipe_id, queue_id, wptr);
+	/* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */
+	uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);
+	uint32_t wptr_mask = (uint32_t)((p->queue_size / sizeof(uint32_t)) - 1);
+
+	return mm->dev->kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id,
+					  (uint32_t __user *)p->write_ptr,
+					  wptr_shift, wptr_mask, mms);
 }
 
 static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
-			uint32_t pipe_id, uint32_t queue_id,
-			uint32_t __user *wptr)
+			 uint32_t pipe_id, uint32_t queue_id,
+			 struct queue_properties *p, struct mm_struct *mms)
 {
 	return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd);
 }
@@ -170,10 +168,6 @@ static int update_mqd(struct mqd_manager *mm, void *mqd,
 {
 	struct cik_mqd *m;
 
-	BUG_ON(!mm || !q || !mqd);
-
-	pr_debug("kfd: In func %s\n", __func__);
-
 	m = get_mqd(mqd);
 	m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE |
 				DEFAULT_MIN_AVAIL_SIZE | PQ_ATC_EN;
@@ -188,21 +182,17 @@ static int update_mqd(struct mqd_manager *mm, void *mqd,
 	m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
 	m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
 	m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
-	m->cp_hqd_pq_doorbell_control = DOORBELL_EN |
-					DOORBELL_OFFSET(q->doorbell_off);
+	m->cp_hqd_pq_doorbell_control = DOORBELL_OFFSET(q->doorbell_off);
 
 	m->cp_hqd_vmid = q->vmid;
 
-	if (q->format == KFD_QUEUE_FORMAT_AQL) {
+	if (q->format == KFD_QUEUE_FORMAT_AQL)
 		m->cp_hqd_pq_control |= NO_UPDATE_RPTR;
-	}
 
-	m->cp_hqd_active = 0;
 	q->is_active = false;
 	if (q->queue_size > 0 &&
 			q->queue_address != 0 &&
 			q->queue_percent > 0) {
-		m->cp_hqd_active = 1;
 		q->is_active = true;
 	}
 
@@ -214,8 +204,6 @@ static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
 {
 	struct cik_sdma_rlc_registers *m;
 
-	BUG_ON(!mm || !mqd || !q);
-
 	m = get_sdma_mqd(mqd);
 	m->sdma_rlc_rb_cntl = ffs(q->queue_size / sizeof(unsigned int)) <<
 			SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT |
@@ -254,7 +242,7 @@ static int destroy_mqd(struct mqd_manager *mm, void *mqd,
 			unsigned int timeout, uint32_t pipe_id,
 			uint32_t queue_id)
 {
-	return mm->dev->kfd2kgd->hqd_destroy(mm->dev->kgd, type, timeout,
+	return mm->dev->kfd2kgd->hqd_destroy(mm->dev->kgd, mqd, type, timeout,
 					pipe_id, queue_id);
 }
 
@@ -301,10 +289,6 @@ static int init_mqd_hiq(struct mqd_manager *mm, void **mqd,
 	struct cik_mqd *m;
 	int retval;
 
-	BUG_ON(!mm || !q || !mqd || !mqd_mem_obj);
-
-	pr_debug("kfd: In func %s\n", __func__);
-
 	retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct cik_mqd),
 					mqd_mem_obj);
 
@@ -359,10 +343,6 @@ static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
 {
 	struct cik_mqd *m;
 
-	BUG_ON(!mm || !q || !mqd);
-
-	pr_debug("kfd: In func %s\n", __func__);
-
 	m = get_mqd(mqd);
 	m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE |
 				DEFAULT_MIN_AVAIL_SIZE |
@@ -400,8 +380,6 @@ struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
 {
 	struct cik_sdma_rlc_registers *m;
 
-	BUG_ON(!mqd);
-
 	m = (struct cik_sdma_rlc_registers *)mqd;
 
 	return m;
@@ -412,12 +390,10 @@ struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
 {
 	struct mqd_manager *mqd;
 
-	BUG_ON(!dev);
-	BUG_ON(type >= KFD_MQD_TYPE_MAX);
-
-	pr_debug("kfd: In func %s\n", __func__);
+	if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
+		return NULL;
 
-	mqd = kzalloc(sizeof(struct mqd_manager), GFP_KERNEL);
+	mqd = kzalloc(sizeof(*mqd), GFP_KERNEL);
 	if (!mqd)
 		return NULL;
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
index a9b9882a9a77..73cbfe186dd2 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
@@ -85,7 +85,7 @@ static int init_mqd(struct mqd_manager *mm, void **mqd,
 		m->cp_hqd_iq_rptr = 1;
 
 	*mqd = m;
-	if (gart_addr != NULL)
+	if (gart_addr)
 		*gart_addr = addr;
 	retval = mm->update_mqd(mm, m, q);
 
@@ -94,10 +94,15 @@ static int init_mqd(struct mqd_manager *mm, void **mqd,
 
 static int load_mqd(struct mqd_manager *mm, void *mqd,
 			uint32_t pipe_id, uint32_t queue_id,
-			uint32_t __user *wptr)
+			struct queue_properties *p, struct mm_struct *mms)
 {
-	return mm->dev->kfd2kgd->hqd_load
-		(mm->dev->kgd, mqd, pipe_id, queue_id, wptr);
+	/* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */
+	uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);
+	uint32_t wptr_mask = (uint32_t)((p->queue_size / sizeof(uint32_t)) - 1);
+
+	return mm->dev->kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id,
+					  (uint32_t __user *)p->write_ptr,
+					  wptr_shift, wptr_mask, mms);
 }
 
 static int __update_mqd(struct mqd_manager *mm, void *mqd,
@@ -106,10 +111,6 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd,
 {
 	struct vi_mqd *m;
 
-	BUG_ON(!mm || !q || !mqd);
-
-	pr_debug("kfd: In func %s\n", __func__);
-
 	m = get_mqd(mqd);
 
 	m->cp_hqd_pq_control = 5 << CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE__SHIFT |
@@ -117,7 +118,7 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd,
 			mtype << CP_HQD_PQ_CONTROL__MTYPE__SHIFT;
 	m->cp_hqd_pq_control |=
 			ffs(q->queue_size / sizeof(unsigned int)) - 1 - 1;
-	pr_debug("kfd: cp_hqd_pq_control 0x%x\n", m->cp_hqd_pq_control);
+	pr_debug("cp_hqd_pq_control 0x%x\n", m->cp_hqd_pq_control);
 
 	m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
 	m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
@@ -126,10 +127,9 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd,
 	m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
 
 	m->cp_hqd_pq_doorbell_control =
-		1 << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN__SHIFT |
 		q->doorbell_off <<
 			CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
-	pr_debug("kfd: cp_hqd_pq_doorbell_control 0x%x\n",
+	pr_debug("cp_hqd_pq_doorbell_control 0x%x\n",
 			m->cp_hqd_pq_doorbell_control);
 
 	m->cp_hqd_eop_control = atc_bit << CP_HQD_EOP_CONTROL__EOP_ATC__SHIFT |
@@ -139,8 +139,15 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd,
 			3 << CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE__SHIFT |
 			mtype << CP_HQD_IB_CONTROL__MTYPE__SHIFT;
 
-	m->cp_hqd_eop_control |=
-		ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1;
+	/*
+	 * HW does not clamp this field correctly. Maximum EOP queue size
+	 * is constrained by per-SE EOP done signal count, which is 8-bit.
+	 * Limit is 0xFF EOP entries (= 0x7F8 dwords). CP will not submit
+	 * more than (EOP entry count - 1) so a queue size of 0x800 dwords
+	 * is safe, giving a maximum field value of 0xA.
+	 */
+	m->cp_hqd_eop_control |= min(0xA,
+		ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1);
 	m->cp_hqd_eop_base_addr_lo =
 			lower_32_bits(q->eop_ring_buffer_address >> 8);
 	m->cp_hqd_eop_base_addr_hi =
@@ -156,12 +163,10 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd,
 				2 << CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT;
 	}
 
-	m->cp_hqd_active = 0;
 	q->is_active = false;
 	if (q->queue_size > 0 &&
 			q->queue_address != 0 &&
 			q->queue_percent > 0) {
-		m->cp_hqd_active = 1;
 		q->is_active = true;
 	}
 
@@ -181,14 +186,13 @@ static int destroy_mqd(struct mqd_manager *mm, void *mqd,
 			uint32_t queue_id)
 {
 	return mm->dev->kfd2kgd->hqd_destroy
-		(mm->dev->kgd, type, timeout,
+		(mm->dev->kgd, mqd, type, timeout,
 		pipe_id, queue_id);
 }
 
 static void uninit_mqd(struct mqd_manager *mm, void *mqd,
 			struct kfd_mem_obj *mqd_mem_obj)
 {
-	BUG_ON(!mm || !mqd);
 	kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
 }
 
@@ -238,12 +242,10 @@ struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
 {
 	struct mqd_manager *mqd;
 
-	BUG_ON(!dev);
-	BUG_ON(type >= KFD_MQD_TYPE_MAX);
-
-	pr_debug("kfd: In func %s\n", __func__);
+	if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
+		return NULL;
 
-	mqd = kzalloc(sizeof(struct mqd_manager), GFP_KERNEL);
+	mqd = kzalloc(sizeof(*mqd), GFP_KERNEL);
 	if (!mqd)
 		return NULL;
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
index 7131998848d7..1d312603de9f 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
@@ -26,7 +26,6 @@
 #include "kfd_device_queue_manager.h"
 #include "kfd_kernel_queue.h"
 #include "kfd_priv.h"
-#include "kfd_pm4_headers.h"
 #include "kfd_pm4_headers_vi.h"
 #include "kfd_pm4_opcodes.h"
 
@@ -35,7 +34,8 @@ static inline void inc_wptr(unsigned int *wptr, unsigned int increment_bytes,
 {
 	unsigned int temp = *wptr + increment_bytes / sizeof(uint32_t);
 
-	BUG_ON((temp * sizeof(uint32_t)) > buffer_size_bytes);
+	WARN((temp * sizeof(uint32_t)) > buffer_size_bytes,
+	     "Runlist IB overflow");
 	*wptr = temp;
 }
 
@@ -43,12 +43,12 @@ static unsigned int build_pm4_header(unsigned int opcode, size_t packet_size)
 {
 	union PM4_MES_TYPE_3_HEADER header;
 
-	header.u32all = 0;
+	header.u32All = 0;
 	header.opcode = opcode;
 	header.count = packet_size/sizeof(uint32_t) - 2;
 	header.type = PM4_TYPE_3;
 
-	return header.u32all;
+	return header.u32All;
 }
 
 static void pm_calc_rlib_size(struct packet_manager *pm,
@@ -58,8 +58,6 @@ static void pm_calc_rlib_size(struct packet_manager *pm,
 	unsigned int process_count, queue_count;
 	unsigned int map_queue_size;
 
-	BUG_ON(!pm || !rlib_size || !over_subscription);
-
 	process_count = pm->dqm->processes_count;
 	queue_count = pm->dqm->queue_count;
 
@@ -67,15 +65,12 @@ static void pm_calc_rlib_size(struct packet_manager *pm,
 	*over_subscription = false;
 	if ((process_count > 1) || queue_count > get_queues_num(pm->dqm)) {
 		*over_subscription = true;
-		pr_debug("kfd: over subscribed runlist\n");
+		pr_debug("Over subscribed runlist\n");
 	}
 
-	map_queue_size =
-		(pm->dqm->dev->device_info->asic_family == CHIP_CARRIZO) ?
-		sizeof(struct pm4_mes_map_queues) :
-		sizeof(struct pm4_map_queues);
+	map_queue_size = sizeof(struct pm4_mes_map_queues);
 	/* calculate run list ib allocation size */
-	*rlib_size = process_count * sizeof(struct pm4_map_process) +
+	*rlib_size = process_count * sizeof(struct pm4_mes_map_process) +
 		     queue_count * map_queue_size;
 
 	/*
@@ -83,9 +78,9 @@ static void pm_calc_rlib_size(struct packet_manager *pm,
 	 * when over subscription
 	 */
 	if (*over_subscription)
-		*rlib_size += sizeof(struct pm4_runlist);
+		*rlib_size += sizeof(struct pm4_mes_runlist);
 
-	pr_debug("kfd: runlist ib size %d\n", *rlib_size);
+	pr_debug("runlist ib size %d\n", *rlib_size);
 }
 
 static int pm_allocate_runlist_ib(struct packet_manager *pm,
@@ -96,17 +91,16 @@ static int pm_allocate_runlist_ib(struct packet_manager *pm,
 {
 	int retval;
 
-	BUG_ON(!pm);
-	BUG_ON(pm->allocated);
-	BUG_ON(is_over_subscription == NULL);
+	if (WARN_ON(pm->allocated))
+		return -EINVAL;
 
 	pm_calc_rlib_size(pm, rl_buffer_size, is_over_subscription);
 
 	retval = kfd_gtt_sa_allocate(pm->dqm->dev, *rl_buffer_size,
 					&pm->ib_buffer_obj);
 
-	if (retval != 0) {
-		pr_err("kfd: failed to allocate runlist IB\n");
+	if (retval) {
+		pr_err("Failed to allocate runlist IB\n");
 		return retval;
 	}
 
@@ -121,15 +115,16 @@ static int pm_allocate_runlist_ib(struct packet_manager *pm,
 static int pm_create_runlist(struct packet_manager *pm, uint32_t *buffer,
 			uint64_t ib, size_t ib_size_in_dwords, bool chain)
 {
-	struct pm4_runlist *packet;
+	struct pm4_mes_runlist *packet;
 
-	BUG_ON(!pm || !buffer || !ib);
+	if (WARN_ON(!ib))
+		return -EFAULT;
 
-	packet = (struct pm4_runlist *)buffer;
+	packet = (struct pm4_mes_runlist *)buffer;
 
-	memset(buffer, 0, sizeof(struct pm4_runlist));
-	packet->header.u32all = build_pm4_header(IT_RUN_LIST,
-						sizeof(struct pm4_runlist));
+	memset(buffer, 0, sizeof(struct pm4_mes_runlist));
+	packet->header.u32All = build_pm4_header(IT_RUN_LIST,
+						sizeof(struct pm4_mes_runlist));
 
 	packet->bitfields4.ib_size = ib_size_in_dwords;
 	packet->bitfields4.chain = chain ? 1 : 0;
@@ -144,20 +139,16 @@ static int pm_create_runlist(struct packet_manager *pm, uint32_t *buffer,
 static int pm_create_map_process(struct packet_manager *pm, uint32_t *buffer,
 				struct qcm_process_device *qpd)
 {
-	struct pm4_map_process *packet;
+	struct pm4_mes_map_process *packet;
 	struct queue *cur;
 	uint32_t num_queues;
 
-	BUG_ON(!pm || !buffer || !qpd);
-
-	packet = (struct pm4_map_process *)buffer;
-
-	pr_debug("kfd: In func %s\n", __func__);
+	packet = (struct pm4_mes_map_process *)buffer;
 
-	memset(buffer, 0, sizeof(struct pm4_map_process));
+	memset(buffer, 0, sizeof(struct pm4_mes_map_process));
 
-	packet->header.u32all = build_pm4_header(IT_MAP_PROCESS,
-					sizeof(struct pm4_map_process));
+	packet->header.u32All = build_pm4_header(IT_MAP_PROCESS,
+					sizeof(struct pm4_mes_map_process));
 	packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0;
 	packet->bitfields2.process_quantum = 1;
 	packet->bitfields2.pasid = qpd->pqm->process->pasid;
@@ -175,27 +166,26 @@ static int pm_create_map_process(struct packet_manager *pm, uint32_t *buffer,
 	packet->sh_mem_ape1_base = qpd->sh_mem_ape1_base;
 	packet->sh_mem_ape1_limit = qpd->sh_mem_ape1_limit;
 
+	/* TODO: scratch support */
+	packet->sh_hidden_private_base_vmid = 0;
+
 	packet->gds_addr_lo = lower_32_bits(qpd->gds_context_area);
 	packet->gds_addr_hi = upper_32_bits(qpd->gds_context_area);
 
 	return 0;
 }
 
-static int pm_create_map_queue_vi(struct packet_manager *pm, uint32_t *buffer,
+static int pm_create_map_queue(struct packet_manager *pm, uint32_t *buffer,
 		struct queue *q, bool is_static)
 {
 	struct pm4_mes_map_queues *packet;
 	bool use_static = is_static;
 
-	BUG_ON(!pm || !buffer || !q);
-
-	pr_debug("kfd: In func %s\n", __func__);
-
 	packet = (struct pm4_mes_map_queues *)buffer;
-	memset(buffer, 0, sizeof(struct pm4_map_queues));
+	memset(buffer, 0, sizeof(struct pm4_mes_map_queues));
 
-	packet->header.u32all = build_pm4_header(IT_MAP_QUEUES,
-						sizeof(struct pm4_map_queues));
+	packet->header.u32All = build_pm4_header(IT_MAP_QUEUES,
+						sizeof(struct pm4_mes_map_queues));
 	packet->bitfields2.alloc_format =
 		alloc_format__mes_map_queues__one_per_pipe_vi;
 	packet->bitfields2.num_queues = 1;
@@ -223,10 +213,8 @@ static int pm_create_map_queue_vi(struct packet_manager *pm, uint32_t *buffer,
 		use_static = false; /* no static queues under SDMA */
 		break;
 	default:
-		pr_err("kfd: in %s queue type %d\n", __func__,
-				q->properties.type);
-		BUG();
-		break;
+		WARN(1, "queue type %d", q->properties.type);
+		return -EINVAL;
 	}
 	packet->bitfields3.doorbell_offset =
 			q->properties.doorbell_off;
@@ -246,68 +234,6 @@ static int pm_create_map_queue_vi(struct packet_manager *pm, uint32_t *buffer,
 	return 0;
 }
 
-static int pm_create_map_queue(struct packet_manager *pm, uint32_t *buffer,
-				struct queue *q, bool is_static)
-{
-	struct pm4_map_queues *packet;
-	bool use_static = is_static;
-
-	BUG_ON(!pm || !buffer || !q);
-
-	pr_debug("kfd: In func %s\n", __func__);
-
-	packet = (struct pm4_map_queues *)buffer;
-	memset(buffer, 0, sizeof(struct pm4_map_queues));
-
-	packet->header.u32all = build_pm4_header(IT_MAP_QUEUES,
-						sizeof(struct pm4_map_queues));
-	packet->bitfields2.alloc_format =
-				alloc_format__mes_map_queues__one_per_pipe;
-	packet->bitfields2.num_queues = 1;
-	packet->bitfields2.queue_sel =
-		queue_sel__mes_map_queues__map_to_hws_determined_queue_slots;
-
-	packet->bitfields2.vidmem = (q->properties.is_interop) ?
-			vidmem__mes_map_queues__uses_video_memory :
-			vidmem__mes_map_queues__uses_no_video_memory;
-
-	switch (q->properties.type) {
-	case KFD_QUEUE_TYPE_COMPUTE:
-	case KFD_QUEUE_TYPE_DIQ:
-		packet->bitfields2.engine_sel =
-				engine_sel__mes_map_queues__compute;
-		break;
-	case KFD_QUEUE_TYPE_SDMA:
-		packet->bitfields2.engine_sel =
-				engine_sel__mes_map_queues__sdma0;
-		use_static = false; /* no static queues under SDMA */
-		break;
-	default:
-		BUG();
-		break;
-	}
-
-	packet->mes_map_queues_ordinals[0].bitfields3.doorbell_offset =
-			q->properties.doorbell_off;
-
-	packet->mes_map_queues_ordinals[0].bitfields3.is_static =
-			(use_static) ? 1 : 0;
-
-	packet->mes_map_queues_ordinals[0].mqd_addr_lo =
-			lower_32_bits(q->gart_mqd_addr);
-
-	packet->mes_map_queues_ordinals[0].mqd_addr_hi =
-			upper_32_bits(q->gart_mqd_addr);
-
-	packet->mes_map_queues_ordinals[0].wptr_addr_lo =
-			lower_32_bits((uint64_t)q->properties.write_ptr);
-
-	packet->mes_map_queues_ordinals[0].wptr_addr_hi =
-			upper_32_bits((uint64_t)q->properties.write_ptr);
-
-	return 0;
-}
-
 static int pm_create_runlist_ib(struct packet_manager *pm,
 				struct list_head *queues,
 				uint64_t *rl_gpu_addr,
@@ -322,19 +248,16 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
 	struct kernel_queue *kq;
 	bool is_over_subscription;
 
-	BUG_ON(!pm || !queues || !rl_size_bytes || !rl_gpu_addr);
-
 	rl_wptr = retval = proccesses_mapped = 0;
 
 	retval = pm_allocate_runlist_ib(pm, &rl_buffer, rl_gpu_addr,
 				&alloc_size_bytes, &is_over_subscription);
-	if (retval != 0)
+	if (retval)
 		return retval;
 
 	*rl_size_bytes = alloc_size_bytes;
 
-	pr_debug("kfd: In func %s\n", __func__);
-	pr_debug("kfd: building runlist ib process count: %d queues count %d\n",
+	pr_debug("Building runlist ib process count: %d queues count %d\n",
 		pm->dqm->processes_count, pm->dqm->queue_count);
 
 	/* build the run list ib packet */
@@ -342,42 +265,35 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
 		qpd = cur->qpd;
 		/* build map process packet */
 		if (proccesses_mapped >= pm->dqm->processes_count) {
-			pr_debug("kfd: not enough space left in runlist IB\n");
+			pr_debug("Not enough space left in runlist IB\n");
 			pm_release_ib(pm);
 			return -ENOMEM;
 		}
 
 		retval = pm_create_map_process(pm, &rl_buffer[rl_wptr], qpd);
-		if (retval != 0)
+		if (retval)
 			return retval;
 
 		proccesses_mapped++;
-		inc_wptr(&rl_wptr, sizeof(struct pm4_map_process),
+		inc_wptr(&rl_wptr, sizeof(struct pm4_mes_map_process),
 				alloc_size_bytes);
 
 		list_for_each_entry(kq, &qpd->priv_queue_list, list) {
 			if (!kq->queue->properties.is_active)
 				continue;
 
-			pr_debug("kfd: static_queue, mapping kernel q %d, is debug status %d\n",
+			pr_debug("static_queue, mapping kernel q %d, is debug status %d\n",
 				kq->queue->queue, qpd->is_debug);
 
-			if (pm->dqm->dev->device_info->asic_family ==
-					CHIP_CARRIZO)
-				retval = pm_create_map_queue_vi(pm,
-						&rl_buffer[rl_wptr],
-						kq->queue,
-						qpd->is_debug);
-			else
-				retval = pm_create_map_queue(pm,
+			retval = pm_create_map_queue(pm,
 						&rl_buffer[rl_wptr],
 						kq->queue,
 						qpd->is_debug);
-			if (retval != 0)
+			if (retval)
 				return retval;
 
 			inc_wptr(&rl_wptr,
-				sizeof(struct pm4_map_queues),
+				sizeof(struct pm4_mes_map_queues),
 				alloc_size_bytes);
 		}
 
@@ -385,51 +301,44 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
 			if (!q->properties.is_active)
 				continue;
 
-			pr_debug("kfd: static_queue, mapping user queue %d, is debug status %d\n",
+			pr_debug("static_queue, mapping user queue %d, is debug status %d\n",
 				q->queue, qpd->is_debug);
 
-			if (pm->dqm->dev->device_info->asic_family ==
-					CHIP_CARRIZO)
-				retval = pm_create_map_queue_vi(pm,
-						&rl_buffer[rl_wptr],
-						q,
-						qpd->is_debug);
-			else
-				retval = pm_create_map_queue(pm,
+			retval = pm_create_map_queue(pm,
 						&rl_buffer[rl_wptr],
 						q,
 						qpd->is_debug);
 
-			if (retval != 0)
+			if (retval)
 				return retval;
 
 			inc_wptr(&rl_wptr,
-				sizeof(struct pm4_map_queues),
+				sizeof(struct pm4_mes_map_queues),
 				alloc_size_bytes);
 		}
 	}
 
-	pr_debug("kfd: finished map process and queues to runlist\n");
+	pr_debug("Finished map process and queues to runlist\n");
 
 	if (is_over_subscription)
-		pm_create_runlist(pm, &rl_buffer[rl_wptr], *rl_gpu_addr,
-				alloc_size_bytes / sizeof(uint32_t), true);
+		retval = pm_create_runlist(pm, &rl_buffer[rl_wptr],
+					*rl_gpu_addr,
+					alloc_size_bytes / sizeof(uint32_t),
+					true);
 
 	for (i = 0; i < alloc_size_bytes / sizeof(uint32_t); i++)
 		pr_debug("0x%2X ", rl_buffer[i]);
 	pr_debug("\n");
 
-	return 0;
+	return retval;
 }
 
 int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm)
 {
-	BUG_ON(!dqm);
-
 	pm->dqm = dqm;
 	mutex_init(&pm->lock);
 	pm->priv_queue = kernel_queue_init(dqm->dev, KFD_QUEUE_TYPE_HIQ);
-	if (pm->priv_queue == NULL) {
+	if (!pm->priv_queue) {
 		mutex_destroy(&pm->lock);
 		return -ENOMEM;
 	}
@@ -440,8 +349,6 @@ int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm)
 
 void pm_uninit(struct packet_manager *pm)
 {
-	BUG_ON(!pm);
-
 	mutex_destroy(&pm->lock);
 	kernel_queue_uninit(pm->priv_queue);
 }
@@ -449,25 +356,22 @@ void pm_uninit(struct packet_manager *pm)
 int pm_send_set_resources(struct packet_manager *pm,
 				struct scheduling_resources *res)
 {
-	struct pm4_set_resources *packet;
-
-	BUG_ON(!pm || !res);
-
-	pr_debug("kfd: In func %s\n", __func__);
+	struct pm4_mes_set_resources *packet;
+	int retval = 0;
 
 	mutex_lock(&pm->lock);
 	pm->priv_queue->ops.acquire_packet_buffer(pm->priv_queue,
 					sizeof(*packet) / sizeof(uint32_t),
-			(unsigned int **)&packet);
-	if (packet == NULL) {
-		mutex_unlock(&pm->lock);
-		pr_err("kfd: failed to allocate buffer on kernel queue\n");
-		return -ENOMEM;
+					(unsigned int **)&packet);
+	if (!packet) {
+		pr_err("Failed to allocate buffer on kernel queue\n");
+		retval = -ENOMEM;
+		goto out;
 	}
 
-	memset(packet, 0, sizeof(struct pm4_set_resources));
-	packet->header.u32all = build_pm4_header(IT_SET_RESOURCES,
-					sizeof(struct pm4_set_resources));
+	memset(packet, 0, sizeof(struct pm4_mes_set_resources));
+	packet->header.u32All = build_pm4_header(IT_SET_RESOURCES,
+					sizeof(struct pm4_mes_set_resources));
 
 	packet->bitfields2.queue_type =
 			queue_type__mes_set_resources__hsa_interface_queue_hiq;
@@ -485,9 +389,10 @@ int pm_send_set_resources(struct packet_manager *pm,
 
 	pm->priv_queue->ops.submit_packet(pm->priv_queue);
 
+out:
 	mutex_unlock(&pm->lock);
 
-	return 0;
+	return retval;
 }
 
 int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues)
@@ -497,26 +402,24 @@ int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues)
 	size_t rl_ib_size, packet_size_dwords;
 	int retval;
 
-	BUG_ON(!pm || !dqm_queues);
-
 	retval = pm_create_runlist_ib(pm, dqm_queues, &rl_gpu_ib_addr,
 					&rl_ib_size);
-	if (retval != 0)
+	if (retval)
 		goto fail_create_runlist_ib;
 
-	pr_debug("kfd: runlist IB address: 0x%llX\n", rl_gpu_ib_addr);
+	pr_debug("runlist IB address: 0x%llX\n", rl_gpu_ib_addr);
 
-	packet_size_dwords = sizeof(struct pm4_runlist) / sizeof(uint32_t);
+	packet_size_dwords = sizeof(struct pm4_mes_runlist) / sizeof(uint32_t);
 	mutex_lock(&pm->lock);
 
 	retval = pm->priv_queue->ops.acquire_packet_buffer(pm->priv_queue,
 					packet_size_dwords, &rl_buffer);
-	if (retval != 0)
+	if (retval)
 		goto fail_acquire_packet_buffer;
 
 	retval = pm_create_runlist(pm, rl_buffer, rl_gpu_ib_addr,
 					rl_ib_size / sizeof(uint32_t), false);
-	if (retval != 0)
+	if (retval)
 		goto fail_create_runlist;
 
 	pm->priv_queue->ops.submit_packet(pm->priv_queue);
@@ -530,8 +433,7 @@ fail_create_runlist:
 fail_acquire_packet_buffer:
 	mutex_unlock(&pm->lock);
 fail_create_runlist_ib:
-	if (pm->allocated)
-		pm_release_ib(pm);
+	pm_release_ib(pm);
 	return retval;
 }
 
@@ -539,20 +441,21 @@ int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address,
 			uint32_t fence_value)
 {
 	int retval;
-	struct pm4_query_status *packet;
+	struct pm4_mes_query_status *packet;
 
-	BUG_ON(!pm || !fence_address);
+	if (WARN_ON(!fence_address))
+		return -EFAULT;
 
 	mutex_lock(&pm->lock);
 	retval = pm->priv_queue->ops.acquire_packet_buffer(
 			pm->priv_queue,
-			sizeof(struct pm4_query_status) / sizeof(uint32_t),
+			sizeof(struct pm4_mes_query_status) / sizeof(uint32_t),
 			(unsigned int **)&packet);
-	if (retval != 0)
+	if (retval)
 		goto fail_acquire_packet_buffer;
 
-	packet->header.u32all = build_pm4_header(IT_QUERY_STATUS,
-					sizeof(struct pm4_query_status));
+	packet->header.u32All = build_pm4_header(IT_QUERY_STATUS,
+					sizeof(struct pm4_mes_query_status));
 
 	packet->bitfields2.context_id = 0;
 	packet->bitfields2.interrupt_sel =
@@ -566,9 +469,6 @@ int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address,
 	packet->data_lo = lower_32_bits((uint64_t)fence_value);
 
 	pm->priv_queue->ops.submit_packet(pm->priv_queue);
-	mutex_unlock(&pm->lock);
-
-	return 0;
 
 fail_acquire_packet_buffer:
 	mutex_unlock(&pm->lock);
@@ -582,24 +482,22 @@ int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
 {
 	int retval;
 	uint32_t *buffer;
-	struct pm4_unmap_queues *packet;
-
-	BUG_ON(!pm);
+	struct pm4_mes_unmap_queues *packet;
 
 	mutex_lock(&pm->lock);
 	retval = pm->priv_queue->ops.acquire_packet_buffer(
 			pm->priv_queue,
-			sizeof(struct pm4_unmap_queues) / sizeof(uint32_t),
+			sizeof(struct pm4_mes_unmap_queues) / sizeof(uint32_t),
 			&buffer);
-	if (retval != 0)
+	if (retval)
 		goto err_acquire_packet_buffer;
 
-	packet = (struct pm4_unmap_queues *)buffer;
-	memset(buffer, 0, sizeof(struct pm4_unmap_queues));
-	pr_debug("kfd: static_queue: unmapping queues: mode is %d , reset is %d , type is %d\n",
+	packet = (struct pm4_mes_unmap_queues *)buffer;
+	memset(buffer, 0, sizeof(struct pm4_mes_unmap_queues));
+	pr_debug("static_queue: unmapping queues: mode is %d , reset is %d , type is %d\n",
 		mode, reset, type);
-	packet->header.u32all = build_pm4_header(IT_UNMAP_QUEUES,
-					sizeof(struct pm4_unmap_queues));
+	packet->header.u32All = build_pm4_header(IT_UNMAP_QUEUES,
+					sizeof(struct pm4_mes_unmap_queues));
 	switch (type) {
 	case KFD_QUEUE_TYPE_COMPUTE:
 	case KFD_QUEUE_TYPE_DIQ:
@@ -611,8 +509,9 @@ int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
 			engine_sel__mes_unmap_queues__sdma0 + sdma_engine;
 		break;
 	default:
-		BUG();
-		break;
+		WARN(1, "queue type %d", type);
+		retval = -EINVAL;
+		goto err_invalid;
 	}
 
 	if (reset)
@@ -636,16 +535,17 @@ int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
 		break;
 	case KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES:
 		packet->bitfields2.queue_sel =
-				queue_sel__mes_unmap_queues__perform_request_on_all_active_queues;
+				queue_sel__mes_unmap_queues__unmap_all_queues;
 		break;
 	case KFD_PREEMPT_TYPE_FILTER_DYNAMIC_QUEUES:
 		/* in this case, we do not preempt static queues */
 		packet->bitfields2.queue_sel =
-				queue_sel__mes_unmap_queues__perform_request_on_dynamic_queues_only;
+				queue_sel__mes_unmap_queues__unmap_all_non_static_queues;
 		break;
 	default:
-		BUG();
-		break;
+		WARN(1, "filter %d", mode);
+		retval = -EINVAL;
+		goto err_invalid;
 	}
 
 	pm->priv_queue->ops.submit_packet(pm->priv_queue);
@@ -653,6 +553,8 @@ int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
 	mutex_unlock(&pm->lock);
 	return 0;
 
+err_invalid:
+	pm->priv_queue->ops.rollback_packet(pm->priv_queue);
 err_acquire_packet_buffer:
 	mutex_unlock(&pm->lock);
 	return retval;
@@ -660,8 +562,6 @@ err_acquire_packet_buffer:
 
 void pm_release_ib(struct packet_manager *pm)
 {
-	BUG_ON(!pm);
-
 	mutex_lock(&pm->lock);
 	if (pm->allocated) {
 		kfd_gtt_sa_free(pm->dqm->dev, pm->ib_buffer_obj);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_pasid.c b/drivers/gpu/drm/amd/amdkfd/kfd_pasid.c
index 6cfe7f1f18cf..1e06de0bc673 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_pasid.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_pasid.c
@@ -32,7 +32,8 @@ int kfd_pasid_init(void)
 {
 	pasid_limit = KFD_MAX_NUM_OF_PROCESSES;
 
-	pasid_bitmap = kcalloc(BITS_TO_LONGS(pasid_limit), sizeof(long), GFP_KERNEL);
+	pasid_bitmap = kcalloc(BITS_TO_LONGS(pasid_limit), sizeof(long),
+				GFP_KERNEL);
 	if (!pasid_bitmap)
 		return -ENOMEM;
 
@@ -91,6 +92,6 @@ unsigned int kfd_pasid_alloc(void)
 
 void kfd_pasid_free(unsigned int pasid)
 {
-	BUG_ON(pasid == 0 || pasid >= pasid_limit);
-	clear_bit(pasid, pasid_bitmap);
+	if (!WARN_ON(pasid == 0 || pasid >= pasid_limit))
+		clear_bit(pasid, pasid_bitmap);
 }
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers.h b/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers.h
index 5b393f3e34a9..e50f73d25de6 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers.h
@@ -28,112 +28,19 @@
 #define PM4_MES_HEADER_DEFINED
 union PM4_MES_TYPE_3_HEADER {
 	struct {
-		uint32_t reserved1:8;	/* < reserved */
-		uint32_t opcode:8;	/* < IT opcode */
-		uint32_t count:14;	/* < number of DWORDs - 1
-					 * in the information body.
-					 */
-		uint32_t type:2;	/* < packet identifier.
-					 * It should be 3 for type 3 packets
-					 */
+		/* reserved */
+		uint32_t reserved1:8;
+		/* IT opcode */
+		uint32_t opcode:8;
+		/* number of DWORDs - 1 in the information body */
+		uint32_t count:14;
+		/* packet identifier. It should be 3 for type 3 packets */
+		uint32_t type:2;
 	};
 	uint32_t u32all;
 };
 #endif /* PM4_MES_HEADER_DEFINED */
 
-/* --------------------MES_SET_RESOURCES-------------------- */
-
-#ifndef PM4_MES_SET_RESOURCES_DEFINED
-#define PM4_MES_SET_RESOURCES_DEFINED
-enum set_resources_queue_type_enum {
-	queue_type__mes_set_resources__kernel_interface_queue_kiq = 0,
-	queue_type__mes_set_resources__hsa_interface_queue_hiq = 1,
-	queue_type__mes_set_resources__hsa_debug_interface_queue = 4
-};
-
-struct pm4_set_resources {
-	union {
-		union PM4_MES_TYPE_3_HEADER header;	/* header */
-		uint32_t ordinal1;
-	};
-
-	union {
-		struct {
-			uint32_t vmid_mask:16;
-			uint32_t unmap_latency:8;
-			uint32_t reserved1:5;
-			enum set_resources_queue_type_enum queue_type:3;
-		} bitfields2;
-		uint32_t ordinal2;
-	};
-
-	uint32_t queue_mask_lo;
-	uint32_t queue_mask_hi;
-	uint32_t gws_mask_lo;
-	uint32_t gws_mask_hi;
-
-	union {
-		struct {
-			uint32_t oac_mask:16;
-			uint32_t reserved2:16;
-		} bitfields7;
-		uint32_t ordinal7;
-	};
-
-	union {
-		struct {
-			uint32_t gds_heap_base:6;
-			uint32_t reserved3:5;
-			uint32_t gds_heap_size:6;
-			uint32_t reserved4:15;
-		} bitfields8;
-		uint32_t ordinal8;
-	};
-
-};
-#endif
-
-/*--------------------MES_RUN_LIST-------------------- */
-
-#ifndef PM4_MES_RUN_LIST_DEFINED
-#define PM4_MES_RUN_LIST_DEFINED
-
-struct pm4_runlist {
-	union {
-		union PM4_MES_TYPE_3_HEADER header;	/* header */
-		uint32_t ordinal1;
-	};
-
-	union {
-		struct {
-			uint32_t reserved1:2;
-			uint32_t ib_base_lo:30;
-		} bitfields2;
-		uint32_t ordinal2;
-	};
-
-	union {
-		struct {
-			uint32_t ib_base_hi:16;
-			uint32_t reserved2:16;
-		} bitfields3;
-		uint32_t ordinal3;
-	};
-
-	union {
-		struct {
-			uint32_t ib_size:20;
-			uint32_t chain:1;
-			uint32_t offload_polling:1;
-			uint32_t reserved3:1;
-			uint32_t valid:1;
-			uint32_t reserved4:8;
-		} bitfields4;
-		uint32_t ordinal4;
-	};
-
-};
-#endif
 
 /*--------------------MES_MAP_PROCESS-------------------- */
 
@@ -186,217 +93,58 @@ struct pm4_map_process {
 };
 #endif
 
-/*--------------------MES_MAP_QUEUES--------------------*/
-
-#ifndef PM4_MES_MAP_QUEUES_DEFINED
-#define PM4_MES_MAP_QUEUES_DEFINED
-enum map_queues_queue_sel_enum {
-	queue_sel__mes_map_queues__map_to_specified_queue_slots = 0,
-	queue_sel__mes_map_queues__map_to_hws_determined_queue_slots = 1,
-	queue_sel__mes_map_queues__enable_process_queues = 2
-};
+#ifndef PM4_MES_MAP_PROCESS_DEFINED_KV_SCRATCH
+#define PM4_MES_MAP_PROCESS_DEFINED_KV_SCRATCH
 
-enum map_queues_vidmem_enum {
-	vidmem__mes_map_queues__uses_no_video_memory = 0,
-	vidmem__mes_map_queues__uses_video_memory = 1
-};
-
-enum map_queues_alloc_format_enum {
-	alloc_format__mes_map_queues__one_per_pipe = 0,
-	alloc_format__mes_map_queues__all_on_one_pipe = 1
-};
-
-enum map_queues_engine_sel_enum {
-	engine_sel__mes_map_queues__compute = 0,
-	engine_sel__mes_map_queues__sdma0 = 2,
-	engine_sel__mes_map_queues__sdma1 = 3
-};
-
-struct pm4_map_queues {
+struct pm4_map_process_scratch_kv {
 	union {
-		union PM4_MES_TYPE_3_HEADER header;	/* header */
-		uint32_t ordinal1;
-	};
-
-	union {
-		struct {
-			uint32_t reserved1:4;
-			enum map_queues_queue_sel_enum queue_sel:2;
-			uint32_t reserved2:2;
-			uint32_t vmid:4;
-			uint32_t reserved3:4;
-			enum map_queues_vidmem_enum vidmem:2;
-			uint32_t reserved4:6;
-			enum map_queues_alloc_format_enum alloc_format:2;
-			enum map_queues_engine_sel_enum engine_sel:3;
-			uint32_t num_queues:3;
-		} bitfields2;
-		uint32_t ordinal2;
-	};
-
-	struct {
-		union {
-			struct {
-				uint32_t is_static:1;
-				uint32_t reserved5:1;
-				uint32_t doorbell_offset:21;
-				uint32_t reserved6:3;
-				uint32_t queue:6;
-			} bitfields3;
-			uint32_t ordinal3;
-		};
-
-		uint32_t mqd_addr_lo;
-		uint32_t mqd_addr_hi;
-		uint32_t wptr_addr_lo;
-		uint32_t wptr_addr_hi;
-
-	} mes_map_queues_ordinals[1];	/* 1..N of these ordinal groups */
-
-};
-#endif
-
-/*--------------------MES_QUERY_STATUS--------------------*/
-
-#ifndef PM4_MES_QUERY_STATUS_DEFINED
-#define PM4_MES_QUERY_STATUS_DEFINED
-enum query_status_interrupt_sel_enum {
-	interrupt_sel__mes_query_status__completion_status = 0,
-	interrupt_sel__mes_query_status__process_status = 1,
-	interrupt_sel__mes_query_status__queue_status = 2
-};
-
-enum query_status_command_enum {
-	command__mes_query_status__interrupt_only = 0,
-	command__mes_query_status__fence_only_immediate = 1,
-	command__mes_query_status__fence_only_after_write_ack = 2,
-	command__mes_query_status__fence_wait_for_write_ack_send_interrupt = 3
-};
-
-enum query_status_engine_sel_enum {
-	engine_sel__mes_query_status__compute = 0,
-	engine_sel__mes_query_status__sdma0_queue = 2,
-	engine_sel__mes_query_status__sdma1_queue = 3
-};
-
-struct pm4_query_status {
-	union {
-		union PM4_MES_TYPE_3_HEADER header;	/* header */
-		uint32_t ordinal1;
-	};
-
-	union {
-		struct {
-			uint32_t context_id:28;
-			enum query_status_interrupt_sel_enum interrupt_sel:2;
-			enum query_status_command_enum command:2;
-		} bitfields2;
-		uint32_t ordinal2;
+		union PM4_MES_TYPE_3_HEADER   header; /* header */
+		uint32_t            ordinal1;
 	};
 
 	union {
 		struct {
 			uint32_t pasid:16;
-			uint32_t reserved1:16;
-		} bitfields3a;
-		struct {
-			uint32_t reserved2:2;
-			uint32_t doorbell_offset:21;
-			uint32_t reserved3:3;
-			enum query_status_engine_sel_enum engine_sel:3;
-			uint32_t reserved4:3;
-		} bitfields3b;
-		uint32_t ordinal3;
-	};
-
-	uint32_t addr_lo;
-	uint32_t addr_hi;
-	uint32_t data_lo;
-	uint32_t data_hi;
-};
-#endif
-
-/*--------------------MES_UNMAP_QUEUES--------------------*/
-
-#ifndef PM4_MES_UNMAP_QUEUES_DEFINED
-#define PM4_MES_UNMAP_QUEUES_DEFINED
-enum unmap_queues_action_enum {
-	action__mes_unmap_queues__preempt_queues = 0,
-	action__mes_unmap_queues__reset_queues = 1,
-	action__mes_unmap_queues__disable_process_queues = 2
-};
-
-enum unmap_queues_queue_sel_enum {
-	queue_sel__mes_unmap_queues__perform_request_on_specified_queues = 0,
-	queue_sel__mes_unmap_queues__perform_request_on_pasid_queues = 1,
-	queue_sel__mes_unmap_queues__perform_request_on_all_active_queues = 2,
-	queue_sel__mes_unmap_queues__perform_request_on_dynamic_queues_only = 3
-};
-
-enum unmap_queues_engine_sel_enum {
-	engine_sel__mes_unmap_queues__compute = 0,
-	engine_sel__mes_unmap_queues__sdma0 = 2,
-	engine_sel__mes_unmap_queues__sdma1 = 3
-};
-
-struct pm4_unmap_queues {
-	union {
-		union PM4_MES_TYPE_3_HEADER header;	/* header */
-		uint32_t ordinal1;
-	};
-
-	union {
-		struct {
-			enum unmap_queues_action_enum action:2;
-			uint32_t reserved1:2;
-			enum unmap_queues_queue_sel_enum queue_sel:2;
-			uint32_t reserved2:20;
-			enum unmap_queues_engine_sel_enum engine_sel:3;
-			uint32_t num_queues:3;
+			uint32_t reserved1:8;
+			uint32_t diq_enable:1;
+			uint32_t process_quantum:7;
 		} bitfields2;
 		uint32_t ordinal2;
 	};
 
 	union {
 		struct {
-			uint32_t pasid:16;
-			uint32_t reserved3:16;
-		} bitfields3a;
-		struct {
-			uint32_t reserved4:2;
-			uint32_t doorbell_offset0:21;
-			uint32_t reserved5:9;
-		} bitfields3b;
+			uint32_t page_table_base:28;
+			uint32_t reserved2:4;
+		} bitfields3;
 		uint32_t ordinal3;
 	};
 
-	union {
-		struct {
-			uint32_t reserved6:2;
-			uint32_t doorbell_offset1:21;
-			uint32_t reserved7:9;
-		} bitfields4;
-		uint32_t ordinal4;
-	};
-
-	union {
-		struct {
-			uint32_t reserved8:2;
-			uint32_t doorbell_offset2:21;
-			uint32_t reserved9:9;
-		} bitfields5;
-		uint32_t ordinal5;
-	};
+	uint32_t reserved3;
+	uint32_t sh_mem_bases;
+	uint32_t sh_mem_config;
+	uint32_t sh_mem_ape1_base;
+	uint32_t sh_mem_ape1_limit;
+	uint32_t sh_hidden_private_base_vmid;
+	uint32_t reserved4;
+	uint32_t reserved5;
+	uint32_t gds_addr_lo;
+	uint32_t gds_addr_hi;
 
 	union {
 		struct {
-			uint32_t reserved10:2;
-			uint32_t doorbell_offset3:21;
-			uint32_t reserved11:9;
-		} bitfields6;
-		uint32_t ordinal6;
+			uint32_t num_gws:6;
+			uint32_t reserved6:2;
+			uint32_t num_oac:4;
+			uint32_t reserved7:4;
+			uint32_t gds_size:6;
+			uint32_t num_queues:10;
+		} bitfields14;
+		uint32_t ordinal14;
 	};
 
+	uint32_t completion_signal_lo32;
+uint32_t completion_signal_hi32;
 };
 #endif
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_vi.h b/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_vi.h
index 08c721922812..7c8d9b357749 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_vi.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_vi.h
@@ -30,10 +30,12 @@ union PM4_MES_TYPE_3_HEADER {
 	struct {
 		uint32_t reserved1 : 8; /* < reserved */
 		uint32_t opcode    : 8; /* < IT opcode */
-		uint32_t count     : 14;/* < number of DWORDs - 1 in the
-		information body. */
-		uint32_t type      : 2; /* < packet identifier.
-					It should be 3 for type 3 packets */
+		uint32_t count     : 14;/* < Number of DWORDS - 1 in the
+					 *   information body
+					 */
+		uint32_t type      : 2; /* < packet identifier
+					 *   It should be 3 for type 3 packets
+					 */
 	};
 	uint32_t u32All;
 };
@@ -124,9 +126,10 @@ struct pm4_mes_runlist {
 			uint32_t ib_size:20;
 			uint32_t chain:1;
 			uint32_t offload_polling:1;
-			uint32_t reserved3:1;
+			uint32_t reserved2:1;
 			uint32_t valid:1;
-			uint32_t reserved4:8;
+			uint32_t process_cnt:4;
+			uint32_t reserved3:4;
 		} bitfields4;
 		uint32_t ordinal4;
 	};
@@ -141,8 +144,8 @@ struct pm4_mes_runlist {
 
 struct pm4_mes_map_process {
 	union {
-		union PM4_MES_TYPE_3_HEADER   header;            /* header */
-		uint32_t            ordinal1;
+		union PM4_MES_TYPE_3_HEADER header;	/* header */
+		uint32_t ordinal1;
 	};
 
 	union {
@@ -153,36 +156,48 @@ struct pm4_mes_map_process {
 			uint32_t process_quantum:7;
 		} bitfields2;
 		uint32_t ordinal2;
-};
+	};
 
 	union {
 		struct {
 			uint32_t page_table_base:28;
-			uint32_t reserved2:4;
+			uint32_t reserved3:4;
 		} bitfields3;
 		uint32_t ordinal3;
 	};
 
+	uint32_t reserved;
+
 	uint32_t sh_mem_bases;
+	uint32_t sh_mem_config;
 	uint32_t sh_mem_ape1_base;
 	uint32_t sh_mem_ape1_limit;
-	uint32_t sh_mem_config;
+
+	uint32_t sh_hidden_private_base_vmid;
+
+	uint32_t reserved2;
+	uint32_t reserved3;
+
 	uint32_t gds_addr_lo;
 	uint32_t gds_addr_hi;
 
 	union {
 		struct {
 			uint32_t num_gws:6;
-			uint32_t reserved3:2;
+			uint32_t reserved4:2;
 			uint32_t num_oac:4;
-			uint32_t reserved4:4;
+			uint32_t reserved5:4;
 			uint32_t gds_size:6;
 			uint32_t num_queues:10;
 		} bitfields10;
 		uint32_t ordinal10;
 	};
 
+	uint32_t completion_signal_lo;
+	uint32_t completion_signal_hi;
+
 };
+
 #endif
 
 /*--------------------MES_MAP_QUEUES--------------------*/
@@ -335,7 +350,7 @@ enum mes_unmap_queues_engine_sel_enum {
 	engine_sel__mes_unmap_queues__sdmal = 3
 };
 
-struct PM4_MES_UNMAP_QUEUES {
+struct pm4_mes_unmap_queues {
 	union {
 		union PM4_MES_TYPE_3_HEADER   header;            /* header */
 		uint32_t            ordinal1;
@@ -395,4 +410,101 @@ struct PM4_MES_UNMAP_QUEUES {
 };
 #endif
 
+#ifndef PM4_MEC_RELEASE_MEM_DEFINED
+#define PM4_MEC_RELEASE_MEM_DEFINED
+enum RELEASE_MEM_event_index_enum {
+	event_index___release_mem__end_of_pipe = 5,
+	event_index___release_mem__shader_done = 6
+};
+
+enum RELEASE_MEM_cache_policy_enum {
+	cache_policy___release_mem__lru = 0,
+	cache_policy___release_mem__stream = 1,
+	cache_policy___release_mem__bypass = 2
+};
+
+enum RELEASE_MEM_dst_sel_enum {
+	dst_sel___release_mem__memory_controller = 0,
+	dst_sel___release_mem__tc_l2 = 1,
+	dst_sel___release_mem__queue_write_pointer_register = 2,
+	dst_sel___release_mem__queue_write_pointer_poll_mask_bit = 3
+};
+
+enum RELEASE_MEM_int_sel_enum {
+	int_sel___release_mem__none = 0,
+	int_sel___release_mem__send_interrupt_only = 1,
+	int_sel___release_mem__send_interrupt_after_write_confirm = 2,
+	int_sel___release_mem__send_data_after_write_confirm = 3
+};
+
+enum RELEASE_MEM_data_sel_enum {
+	data_sel___release_mem__none = 0,
+	data_sel___release_mem__send_32_bit_low = 1,
+	data_sel___release_mem__send_64_bit_data = 2,
+	data_sel___release_mem__send_gpu_clock_counter = 3,
+	data_sel___release_mem__send_cp_perfcounter_hi_lo = 4,
+	data_sel___release_mem__store_gds_data_to_memory = 5
+};
+
+struct pm4_mec_release_mem {
+	union {
+		union PM4_MES_TYPE_3_HEADER header;     /*header */
+		unsigned int ordinal1;
+	};
+
+	union {
+		struct {
+			unsigned int event_type:6;
+			unsigned int reserved1:2;
+			enum RELEASE_MEM_event_index_enum event_index:4;
+			unsigned int tcl1_vol_action_ena:1;
+			unsigned int tc_vol_action_ena:1;
+			unsigned int reserved2:1;
+			unsigned int tc_wb_action_ena:1;
+			unsigned int tcl1_action_ena:1;
+			unsigned int tc_action_ena:1;
+			unsigned int reserved3:6;
+			unsigned int atc:1;
+			enum RELEASE_MEM_cache_policy_enum cache_policy:2;
+			unsigned int reserved4:5;
+		} bitfields2;
+		unsigned int ordinal2;
+	};
+
+	union {
+		struct {
+			unsigned int reserved5:16;
+			enum RELEASE_MEM_dst_sel_enum dst_sel:2;
+			unsigned int reserved6:6;
+			enum RELEASE_MEM_int_sel_enum int_sel:3;
+			unsigned int reserved7:2;
+			enum RELEASE_MEM_data_sel_enum data_sel:3;
+		} bitfields3;
+		unsigned int ordinal3;
+	};
+
+	union {
+		struct {
+			unsigned int reserved8:2;
+			unsigned int address_lo_32b:30;
+		} bitfields4;
+		struct {
+			unsigned int reserved9:3;
+			unsigned int address_lo_64b:29;
+		} bitfields5;
+		unsigned int ordinal4;
+	};
+
+	unsigned int address_hi;
+
+	unsigned int data_lo;
+
+	unsigned int data_hi;
+};
+#endif
+
+enum {
+	CACHE_FLUSH_AND_INV_TS_EVENT = 0x00000014
+};
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 4750cabe4252..b397ec726400 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -239,11 +239,6 @@ enum kfd_preempt_type_filter {
 	KFD_PREEMPT_TYPE_FILTER_BY_PASID
 };
 
-enum kfd_preempt_type {
-	KFD_PREEMPT_TYPE_WAVEFRONT,
-	KFD_PREEMPT_TYPE_WAVEFRONT_RESET
-};
-
 /**
  * enum kfd_queue_type
  *
@@ -294,13 +289,13 @@ enum kfd_queue_format {
  * @write_ptr: Defines the number of dwords written to the ring buffer.
  *
  * @doorbell_ptr: This field aim is to notify the H/W of new packet written to
- * the queue ring buffer. This field should be similar to write_ptr and the user
- * should update this field after he updated the write_ptr.
+ * the queue ring buffer. This field should be similar to write_ptr and the
+ * user should update this field after he updated the write_ptr.
  *
  * @doorbell_off: The doorbell offset in the doorbell pci-bar.
  *
- * @is_interop: Defines if this is a interop queue. Interop queue means that the
- * queue can access both graphics and compute resources.
+ * @is_interop: Defines if this is a interop queue. Interop queue means that
+ * the queue can access both graphics and compute resources.
  *
  * @is_active: Defines if the queue is active or not.
  *
@@ -352,9 +347,10 @@ struct queue_properties {
  * @properties: The queue properties.
  *
  * @mec: Used only in no cp scheduling mode and identifies to micro engine id
- * that the queue should be execute on.
+ *	 that the queue should be execute on.
  *
- * @pipe: Used only in no cp scheduling mode and identifies the queue's pipe id.
+ * @pipe: Used only in no cp scheduling mode and identifies the queue's pipe
+ *	  id.
  *
  * @queue: Used only in no cp scheduliong mode and identifies the queue's slot.
  *
@@ -436,6 +432,7 @@ struct qcm_process_device {
 	uint32_t gds_size;
 	uint32_t num_gws;
 	uint32_t num_oac;
+	uint32_t sh_hidden_private_base;
 };
 
 /* Data that is per-process-per device. */
@@ -520,8 +517,8 @@ struct kfd_process {
 	struct mutex event_mutex;
 	/* All events in process hashed by ID, linked on kfd_event.events. */
 	DECLARE_HASHTABLE(events, 4);
-	struct list_head signal_event_pages;	/* struct slot_page_header.
-								event_pages */
+	/* struct slot_page_header.event_pages */
+	struct list_head signal_event_pages;
 	u32 next_nonsignal_event_id;
 	size_t signal_event_count;
 };
@@ -559,8 +556,10 @@ struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
 							struct kfd_process *p);
 
 /* Process device data iterator */
-struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p);
-struct kfd_process_device *kfd_get_next_process_device_data(struct kfd_process *p,
+struct kfd_process_device *kfd_get_first_process_device_data(
+							struct kfd_process *p);
+struct kfd_process_device *kfd_get_next_process_device_data(
+						struct kfd_process *p,
 						struct kfd_process_device *pdd);
 bool kfd_has_process_device_data(struct kfd_process *p);
 
@@ -573,7 +572,8 @@ unsigned int kfd_pasid_alloc(void);
 void kfd_pasid_free(unsigned int pasid);
 
 /* Doorbells */
-void kfd_doorbell_init(struct kfd_dev *kfd);
+int kfd_doorbell_init(struct kfd_dev *kfd);
+void kfd_doorbell_fini(struct kfd_dev *kfd);
 int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma);
 u32 __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
 					unsigned int *doorbell_off);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 035bbc98a63d..c74cf22a1ed9 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -79,9 +79,7 @@ struct kfd_process *kfd_create_process(const struct task_struct *thread)
 {
 	struct kfd_process *process;
 
-	BUG_ON(!kfd_process_wq);
-
-	if (thread->mm == NULL)
+	if (!thread->mm)
 		return ERR_PTR(-EINVAL);
 
 	/* Only the pthreads threading model is supported. */
@@ -101,7 +99,7 @@ struct kfd_process *kfd_create_process(const struct task_struct *thread)
 	/* A prior open of /dev/kfd could have already created the process. */
 	process = find_process(thread);
 	if (process)
-		pr_debug("kfd: process already found\n");
+		pr_debug("Process already found\n");
 
 	if (!process)
 		process = create_process(thread);
@@ -117,7 +115,7 @@ struct kfd_process *kfd_get_process(const struct task_struct *thread)
 {
 	struct kfd_process *process;
 
-	if (thread->mm == NULL)
+	if (!thread->mm)
 		return ERR_PTR(-EINVAL);
 
 	/* Only the pthreads threading model is supported. */
@@ -202,10 +200,8 @@ static void kfd_process_destroy_delayed(struct rcu_head *rcu)
 	struct kfd_process_release_work *work;
 	struct kfd_process *p;
 
-	BUG_ON(!kfd_process_wq);
-
 	p = container_of(rcu, struct kfd_process, rcu);
-	BUG_ON(atomic_read(&p->mm->mm_count) <= 0);
+	WARN_ON(atomic_read(&p->mm->mm_count) <= 0);
 
 	mmdrop(p->mm);
 
@@ -229,7 +225,8 @@ static void kfd_process_notifier_release(struct mmu_notifier *mn,
 	 * mmu_notifier srcu is read locked
 	 */
 	p = container_of(mn, struct kfd_process, mmu_notifier);
-	BUG_ON(p->mm != mm);
+	if (WARN_ON(p->mm != mm))
+		return;
 
 	mutex_lock(&kfd_processes_mutex);
 	hash_del_rcu(&p->kfd_processes);
@@ -250,7 +247,7 @@ static void kfd_process_notifier_release(struct mmu_notifier *mn,
 			kfd_dbgmgr_destroy(pdd->dev->dbgmgr);
 
 		if (pdd->reset_wavefronts) {
-			pr_warn("amdkfd: Resetting all wave fronts\n");
+			pr_warn("Resetting all wave fronts\n");
 			dbgdev_wave_reset_wavefronts(pdd->dev, p);
 			pdd->reset_wavefronts = false;
 		}
@@ -407,8 +404,6 @@ void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid)
 	struct kfd_process *p;
 	struct kfd_process_device *pdd;
 
-	BUG_ON(dev == NULL);
-
 	/*
 	 * Look for the process that matches the pasid. If there is no such
 	 * process, we either released it in amdkfd's own notifier, or there
@@ -449,14 +444,16 @@ void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid)
 	mutex_unlock(&p->mutex);
 }
 
-struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p)
+struct kfd_process_device *kfd_get_first_process_device_data(
+						struct kfd_process *p)
 {
 	return list_first_entry(&p->per_device_data,
 				struct kfd_process_device,
 				per_device_list);
 }
 
-struct kfd_process_device *kfd_get_next_process_device_data(struct kfd_process *p,
+struct kfd_process_device *kfd_get_next_process_device_data(
+						struct kfd_process *p,
 						struct kfd_process_device *pdd)
 {
 	if (list_is_last(&pdd->per_device_list, &p->per_device_data))
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
index 32cdf2b483db..03bec765b03d 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
@@ -32,12 +32,9 @@ static inline struct process_queue_node *get_queue_by_qid(
 {
 	struct process_queue_node *pqn;
 
-	BUG_ON(!pqm);
-
 	list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
-		if (pqn->q && pqn->q->properties.queue_id == qid)
-			return pqn;
-		if (pqn->kq && pqn->kq->queue->properties.queue_id == qid)
+		if ((pqn->q && pqn->q->properties.queue_id == qid) ||
+		    (pqn->kq && pqn->kq->queue->properties.queue_id == qid))
 			return pqn;
 	}
 
@@ -49,17 +46,13 @@ static int find_available_queue_slot(struct process_queue_manager *pqm,
 {
 	unsigned long found;
 
-	BUG_ON(!pqm || !qid);
-
-	pr_debug("kfd: in %s\n", __func__);
-
 	found = find_first_zero_bit(pqm->queue_slot_bitmap,
 			KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
 
-	pr_debug("kfd: the new slot id %lu\n", found);
+	pr_debug("The new slot id %lu\n", found);
 
 	if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {
-		pr_info("amdkfd: Can not open more queues for process with pasid %d\n",
+		pr_info("Cannot open more queues for process with pasid %d\n",
 				pqm->process->pasid);
 		return -ENOMEM;
 	}
@@ -72,13 +65,11 @@ static int find_available_queue_slot(struct process_queue_manager *pqm,
 
 int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p)
 {
-	BUG_ON(!pqm);
-
 	INIT_LIST_HEAD(&pqm->queues);
 	pqm->queue_slot_bitmap =
 			kzalloc(DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
 					BITS_PER_BYTE), GFP_KERNEL);
-	if (pqm->queue_slot_bitmap == NULL)
+	if (!pqm->queue_slot_bitmap)
 		return -ENOMEM;
 	pqm->process = p;
 
@@ -90,10 +81,6 @@ void pqm_uninit(struct process_queue_manager *pqm)
 	int retval;
 	struct process_queue_node *pqn, *next;
 
-	BUG_ON(!pqm);
-
-	pr_debug("In func %s\n", __func__);
-
 	list_for_each_entry_safe(pqn, next, &pqm->queues, process_queue_list) {
 		retval = pqm_destroy_queue(
 				pqm,
@@ -102,7 +89,7 @@ void pqm_uninit(struct process_queue_manager *pqm)
 					pqn->kq->queue->properties.queue_id);
 
 		if (retval != 0) {
-			pr_err("kfd: failed to destroy queue\n");
+			pr_err("failed to destroy queue\n");
 			return;
 		}
 	}
@@ -117,8 +104,6 @@ static int create_cp_queue(struct process_queue_manager *pqm,
 {
 	int retval;
 
-	retval = 0;
-
 	/* Doorbell initialized in user space*/
 	q_properties->doorbell_ptr = NULL;
 
@@ -131,16 +116,13 @@ static int create_cp_queue(struct process_queue_manager *pqm,
 
 	retval = init_queue(q, q_properties);
 	if (retval != 0)
-		goto err_init_queue;
+		return retval;
 
 	(*q)->device = dev;
 	(*q)->process = pqm->process;
 
-	pr_debug("kfd: PQM After init queue");
-
-	return retval;
+	pr_debug("PQM After init queue");
 
-err_init_queue:
 	return retval;
 }
 
@@ -161,9 +143,6 @@ int pqm_create_queue(struct process_queue_manager *pqm,
 	int num_queues = 0;
 	struct queue *cur;
 
-	BUG_ON(!pqm || !dev || !properties || !qid);
-
-	memset(&q_properties, 0, sizeof(struct queue_properties));
 	memcpy(&q_properties, properties, sizeof(struct queue_properties));
 	q = NULL;
 	kq = NULL;
@@ -185,7 +164,7 @@ int pqm_create_queue(struct process_queue_manager *pqm,
 		list_for_each_entry(cur, &pdd->qpd.queues_list, list)
 			num_queues++;
 		if (num_queues >= dev->device_info->max_no_of_hqd/2)
-			return (-ENOSPC);
+			return -ENOSPC;
 	}
 
 	retval = find_available_queue_slot(pqm, qid);
@@ -197,7 +176,7 @@ int pqm_create_queue(struct process_queue_manager *pqm,
 		dev->dqm->ops.register_process(dev->dqm, &pdd->qpd);
 	}
 
-	pqn = kzalloc(sizeof(struct process_queue_node), GFP_KERNEL);
+	pqn = kzalloc(sizeof(*pqn), GFP_KERNEL);
 	if (!pqn) {
 		retval = -ENOMEM;
 		goto err_allocate_pqn;
@@ -210,7 +189,7 @@ int pqm_create_queue(struct process_queue_manager *pqm,
 		if ((sched_policy == KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION) &&
 		((dev->dqm->processes_count >= VMID_PER_DEVICE) ||
 		(dev->dqm->queue_count >= get_queues_num(dev->dqm)))) {
-			pr_err("kfd: over-subscription is not allowed in radeon_kfd.sched_policy == 1\n");
+			pr_err("Over-subscription is not allowed in radeon_kfd.sched_policy == 1\n");
 			retval = -EPERM;
 			goto err_create_queue;
 		}
@@ -227,7 +206,7 @@ int pqm_create_queue(struct process_queue_manager *pqm,
 		break;
 	case KFD_QUEUE_TYPE_DIQ:
 		kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_DIQ);
-		if (kq == NULL) {
+		if (!kq) {
 			retval = -ENOMEM;
 			goto err_create_queue;
 		}
@@ -238,22 +217,22 @@ int pqm_create_queue(struct process_queue_manager *pqm,
 							kq, &pdd->qpd);
 		break;
 	default:
-		BUG();
-		break;
+		WARN(1, "Invalid queue type %d", type);
+		retval = -EINVAL;
 	}
 
 	if (retval != 0) {
-		pr_debug("Error dqm create queue\n");
+		pr_err("DQM create queue failed\n");
 		goto err_create_queue;
 	}
 
-	pr_debug("kfd: PQM After DQM create queue\n");
+	pr_debug("PQM After DQM create queue\n");
 
 	list_add(&pqn->process_queue_list, &pqm->queues);
 
 	if (q) {
 		*properties = q->properties;
-		pr_debug("kfd: PQM done creating queue\n");
+		pr_debug("PQM done creating queue\n");
 		print_queue_properties(properties);
 	}
 
@@ -279,14 +258,11 @@ int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)
 
 	dqm = NULL;
 
-	BUG_ON(!pqm);
 	retval = 0;
 
-	pr_debug("kfd: In Func %s\n", __func__);
-
 	pqn = get_queue_by_qid(pqm, qid);
-	if (pqn == NULL) {
-		pr_err("kfd: queue id does not match any known queue\n");
+	if (!pqn) {
+		pr_err("Queue id does not match any known queue\n");
 		return -EINVAL;
 	}
 
@@ -295,7 +271,8 @@ int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)
 		dev = pqn->kq->dev;
 	if (pqn->q)
 		dev = pqn->q->device;
-	BUG_ON(!dev);
+	if (WARN_ON(!dev))
+		return -ENODEV;
 
 	pdd = kfd_get_process_device_data(dev, pqm->process);
 	if (!pdd) {
@@ -335,12 +312,9 @@ int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
 	int retval;
 	struct process_queue_node *pqn;
 
-	BUG_ON(!pqm);
-
 	pqn = get_queue_by_qid(pqm, qid);
 	if (!pqn) {
-		pr_debug("amdkfd: No queue %d exists for update operation\n",
-				qid);
+		pr_debug("No queue %d exists for update operation\n", qid);
 		return -EFAULT;
 	}
 
@@ -363,8 +337,6 @@ struct kernel_queue *pqm_get_kernel_queue(
 {
 	struct process_queue_node *pqn;
 
-	BUG_ON(!pqm);
-
 	pqn = get_queue_by_qid(pqm, qid);
 	if (pqn && pqn->kq)
 		return pqn->kq;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
index 0ab197077f2d..a5315d4f1c95 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
@@ -65,17 +65,15 @@ void print_queue(struct queue *q)
 
 int init_queue(struct queue **q, const struct queue_properties *properties)
 {
-	struct queue *tmp;
+	struct queue *tmp_q;
 
-	BUG_ON(!q);
-
-	tmp = kzalloc(sizeof(struct queue), GFP_KERNEL);
-	if (!tmp)
+	tmp_q = kzalloc(sizeof(*tmp_q), GFP_KERNEL);
+	if (!tmp_q)
 		return -ENOMEM;
 
-	memcpy(&tmp->properties, properties, sizeof(struct queue_properties));
+	memcpy(&tmp_q->properties, properties, sizeof(*properties));
 
-	*q = tmp;
+	*q = tmp_q;
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index 1e5064749959..19ce59028d6b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -108,9 +108,6 @@ static int kfd_topology_get_crat_acpi(void *crat_image, size_t *size)
 static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev,
 		struct crat_subtype_computeunit *cu)
 {
-	BUG_ON(!dev);
-	BUG_ON(!cu);
-
 	dev->node_props.cpu_cores_count = cu->num_cpu_cores;
 	dev->node_props.cpu_core_id_base = cu->processor_id_low;
 	if (cu->hsa_capability & CRAT_CU_FLAGS_IOMMU_PRESENT)
@@ -123,9 +120,6 @@ static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev,
 static void kfd_populated_cu_info_gpu(struct kfd_topology_device *dev,
 		struct crat_subtype_computeunit *cu)
 {
-	BUG_ON(!dev);
-	BUG_ON(!cu);
-
 	dev->node_props.simd_id_base = cu->processor_id_low;
 	dev->node_props.simd_count = cu->num_simd_cores;
 	dev->node_props.lds_size_in_kb = cu->lds_size_in_kb;
@@ -148,8 +142,6 @@ static int kfd_parse_subtype_cu(struct crat_subtype_computeunit *cu)
 	struct kfd_topology_device *dev;
 	int i = 0;
 
-	BUG_ON(!cu);
-
 	pr_info("Found CU entry in CRAT table with proximity_domain=%d caps=%x\n",
 			cu->proximity_domain, cu->hsa_capability);
 	list_for_each_entry(dev, &topology_device_list, list) {
@@ -177,8 +169,6 @@ static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem)
 	struct kfd_topology_device *dev;
 	int i = 0;
 
-	BUG_ON(!mem);
-
 	pr_info("Found memory entry in CRAT table with proximity_domain=%d\n",
 			mem->promixity_domain);
 	list_for_each_entry(dev, &topology_device_list, list) {
@@ -223,8 +213,6 @@ static int kfd_parse_subtype_cache(struct crat_subtype_cache *cache)
 	struct kfd_topology_device *dev;
 	uint32_t id;
 
-	BUG_ON(!cache);
-
 	id = cache->processor_id_low;
 
 	pr_info("Found cache entry in CRAT table with processor_id=%d\n", id);
@@ -274,8 +262,6 @@ static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink)
 	uint32_t id_from;
 	uint32_t id_to;
 
-	BUG_ON(!iolink);
-
 	id_from = iolink->proximity_domain_from;
 	id_to = iolink->proximity_domain_to;
 
@@ -323,8 +309,6 @@ static int kfd_parse_subtype(struct crat_subtype_generic *sub_type_hdr)
 	struct crat_subtype_iolink *iolink;
 	int ret = 0;
 
-	BUG_ON(!sub_type_hdr);
-
 	switch (sub_type_hdr->type) {
 	case CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY:
 		cu = (struct crat_subtype_computeunit *)sub_type_hdr;
@@ -368,8 +352,6 @@ static void kfd_release_topology_device(struct kfd_topology_device *dev)
 	struct kfd_cache_properties *cache;
 	struct kfd_iolink_properties *iolink;
 
-	BUG_ON(!dev);
-
 	list_del(&dev->list);
 
 	while (dev->mem_props.next != &dev->mem_props) {
@@ -416,7 +398,7 @@ static struct kfd_topology_device *kfd_create_topology_device(void)
 	struct kfd_topology_device *dev;
 
 	dev = kfd_alloc_struct(dev);
-	if (dev == NULL) {
+	if (!dev) {
 		pr_err("No memory to allocate a topology device");
 		return NULL;
 	}
@@ -666,7 +648,7 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
 			dev->node_props.simd_count);
 
 	if (dev->mem_bank_count < dev->node_props.mem_banks_count) {
-		pr_info_once("kfd: mem_banks_count truncated from %d to %d\n",
+		pr_info_once("mem_banks_count truncated from %d to %d\n",
 				dev->node_props.mem_banks_count,
 				dev->mem_bank_count);
 		sysfs_show_32bit_prop(buffer, "mem_banks_count",
@@ -763,8 +745,6 @@ static void kfd_remove_sysfs_node_entry(struct kfd_topology_device *dev)
 	struct kfd_cache_properties *cache;
 	struct kfd_mem_properties *mem;
 
-	BUG_ON(!dev);
-
 	if (dev->kobj_iolink) {
 		list_for_each_entry(iolink, &dev->io_link_props, list)
 			if (iolink->kobj) {
@@ -819,12 +799,12 @@ static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev,
 	int ret;
 	uint32_t i;
 
-	BUG_ON(!dev);
+	if (WARN_ON(dev->kobj_node))
+		return -EEXIST;
 
 	/*
 	 * Creating the sysfs folders
 	 */
-	BUG_ON(dev->kobj_node);
 	dev->kobj_node = kfd_alloc_struct(dev->kobj_node);
 	if (!dev->kobj_node)
 		return -ENOMEM;
@@ -957,7 +937,7 @@ static int kfd_topology_update_sysfs(void)
 	int ret;
 
 	pr_info("Creating topology SYSFS entries\n");
-	if (sys_props.kobj_topology == NULL) {
+	if (!sys_props.kobj_topology) {
 		sys_props.kobj_topology =
 				kfd_alloc_struct(sys_props.kobj_topology);
 		if (!sys_props.kobj_topology)
@@ -1117,10 +1097,8 @@ static struct kfd_topology_device *kfd_assign_gpu(struct kfd_dev *gpu)
 	struct kfd_topology_device *dev;
 	struct kfd_topology_device *out_dev = NULL;
 
-	BUG_ON(!gpu);
-
 	list_for_each_entry(dev, &topology_device_list, list)
-		if (dev->gpu == NULL && dev->node_props.simd_count > 0) {
+		if (!dev->gpu && (dev->node_props.simd_count > 0)) {
 			dev->gpu = gpu;
 			out_dev = dev;
 			break;
@@ -1143,11 +1121,9 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
 	struct kfd_topology_device *dev;
 	int res;
 
-	BUG_ON(!gpu);
-
 	gpu_id = kfd_generate_gpu_id(gpu);
 
-	pr_debug("kfd: Adding new GPU (ID: 0x%x) to topology\n", gpu_id);
+	pr_debug("Adding new GPU (ID: 0x%x) to topology\n", gpu_id);
 
 	down_write(&topology_lock);
 	/*
@@ -1170,8 +1146,8 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
 		 * GPU vBIOS
 		 */
 
-		/*
-		 * Update the SYSFS tree, since we added another topology device
+		/* Update the SYSFS tree, since we added another topology
+		 * device
 		 */
 		if (kfd_topology_update_sysfs() < 0)
 			kfd_topology_release_sysfs();
@@ -1190,7 +1166,7 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
 
 	if (dev->gpu->device_info->asic_family == CHIP_CARRIZO) {
 		dev->node_props.capability |= HSA_CAP_DOORBELL_PACKET_TYPE;
-		pr_info("amdkfd: adding doorbell packet type capability\n");
+		pr_info("Adding doorbell packet type capability\n");
 	}
 
 	res = 0;
@@ -1210,8 +1186,6 @@ int kfd_topology_remove_device(struct kfd_dev *gpu)
 	uint32_t gpu_id;
 	int res = -ENODEV;
 
-	BUG_ON(!gpu);
-
 	down_write(&topology_lock);
 
 	list_for_each_entry(dev, &topology_device_list, list)
diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h
index 0021a1c63356..837296db9628 100644
--- a/drivers/gpu/drm/amd/include/atomfirmware.h
+++ b/drivers/gpu/drm/amd/include/atomfirmware.h
@@ -1233,6 +1233,69 @@ struct  atom_asic_profiling_info_v4_1
   uint32_t  phyclk2gfxclk_c;
 };
 
+struct  atom_asic_profiling_info_v4_2 {
+	struct  atom_common_table_header  table_header;
+	uint32_t  maxvddc;
+	uint32_t  minvddc;
+	uint32_t  avfs_meannsigma_acontant0;
+	uint32_t  avfs_meannsigma_acontant1;
+	uint32_t  avfs_meannsigma_acontant2;
+	uint16_t  avfs_meannsigma_dc_tol_sigma;
+	uint16_t  avfs_meannsigma_platform_mean;
+	uint16_t  avfs_meannsigma_platform_sigma;
+	uint32_t  gb_vdroop_table_cksoff_a0;
+	uint32_t  gb_vdroop_table_cksoff_a1;
+	uint32_t  gb_vdroop_table_cksoff_a2;
+	uint32_t  gb_vdroop_table_ckson_a0;
+	uint32_t  gb_vdroop_table_ckson_a1;
+	uint32_t  gb_vdroop_table_ckson_a2;
+	uint32_t  avfsgb_fuse_table_cksoff_m1;
+	uint32_t  avfsgb_fuse_table_cksoff_m2;
+	uint32_t  avfsgb_fuse_table_cksoff_b;
+	uint32_t  avfsgb_fuse_table_ckson_m1;
+	uint32_t  avfsgb_fuse_table_ckson_m2;
+	uint32_t  avfsgb_fuse_table_ckson_b;
+	uint16_t  max_voltage_0_25mv;
+	uint8_t   enable_gb_vdroop_table_cksoff;
+	uint8_t   enable_gb_vdroop_table_ckson;
+	uint8_t   enable_gb_fuse_table_cksoff;
+	uint8_t   enable_gb_fuse_table_ckson;
+	uint16_t  psm_age_comfactor;
+	uint8_t   enable_apply_avfs_cksoff_voltage;
+	uint8_t   reserved;
+	uint32_t  dispclk2gfxclk_a;
+	uint32_t  dispclk2gfxclk_b;
+	uint32_t  dispclk2gfxclk_c;
+	uint32_t  pixclk2gfxclk_a;
+	uint32_t  pixclk2gfxclk_b;
+	uint32_t  pixclk2gfxclk_c;
+	uint32_t  dcefclk2gfxclk_a;
+	uint32_t  dcefclk2gfxclk_b;
+	uint32_t  dcefclk2gfxclk_c;
+	uint32_t  phyclk2gfxclk_a;
+	uint32_t  phyclk2gfxclk_b;
+	uint32_t  phyclk2gfxclk_c;
+	uint32_t  acg_gb_vdroop_table_a0;
+	uint32_t  acg_gb_vdroop_table_a1;
+	uint32_t  acg_gb_vdroop_table_a2;
+	uint32_t  acg_avfsgb_fuse_table_m1;
+	uint32_t  acg_avfsgb_fuse_table_m2;
+	uint32_t  acg_avfsgb_fuse_table_b;
+	uint8_t   enable_acg_gb_vdroop_table;
+	uint8_t   enable_acg_gb_fuse_table;
+	uint32_t  acg_dispclk2gfxclk_a;
+	uint32_t  acg_dispclk2gfxclk_b;
+	uint32_t  acg_dispclk2gfxclk_c;
+	uint32_t  acg_pixclk2gfxclk_a;
+	uint32_t  acg_pixclk2gfxclk_b;
+	uint32_t  acg_pixclk2gfxclk_c;
+	uint32_t  acg_dcefclk2gfxclk_a;
+	uint32_t  acg_dcefclk2gfxclk_b;
+	uint32_t  acg_dcefclk2gfxclk_c;
+	uint32_t  acg_phyclk2gfxclk_a;
+	uint32_t  acg_phyclk2gfxclk_b;
+	uint32_t  acg_phyclk2gfxclk_c;
+};
 
 /* 
   ***************************************************************************
diff --git a/drivers/gpu/drm/amd/include/cgs_common.h b/drivers/gpu/drm/amd/include/cgs_common.h
index 0a94f749e3c0..0214f63f52fc 100644
--- a/drivers/gpu/drm/amd/include/cgs_common.h
+++ b/drivers/gpu/drm/amd/include/cgs_common.h
@@ -50,6 +50,7 @@ enum cgs_ind_reg {
 	CGS_IND_REG__UVD_CTX,
 	CGS_IND_REG__DIDT,
 	CGS_IND_REG_GC_CAC,
+	CGS_IND_REG_SE_CAC,
 	CGS_IND_REG__AUDIO_ENDPT
 };
 
@@ -406,6 +407,8 @@ typedef int (*cgs_is_virtualization_enabled_t)(void *cgs_device);
 
 typedef int (*cgs_enter_safe_mode)(struct cgs_device *cgs_device, bool en);
 
+typedef void (*cgs_lock_grbm_idx)(struct cgs_device *cgs_device, bool lock);
+
 struct cgs_ops {
 	/* memory management calls (similar to KFD interface) */
 	cgs_alloc_gpu_mem_t alloc_gpu_mem;
@@ -441,6 +444,7 @@ struct cgs_ops {
 	cgs_query_system_info query_system_info;
 	cgs_is_virtualization_enabled_t is_virtualization_enabled;
 	cgs_enter_safe_mode enter_safe_mode;
+	cgs_lock_grbm_idx lock_grbm_idx;
 };
 
 struct cgs_os_ops; /* To be define in OS-specific CGS header */
@@ -517,4 +521,6 @@ struct cgs_device
 #define cgs_enter_safe_mode(cgs_device, en) \
 		CGS_CALL(enter_safe_mode, cgs_device, en)
 
+#define cgs_lock_grbm_idx(cgs_device, lock) \
+		CGS_CALL(lock_grbm_idx, cgs_device, lock)
 #endif /* _CGS_COMMON_H */
diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
index 36f376677a53..94277cb734d2 100644
--- a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
+++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
@@ -41,6 +41,11 @@ struct kgd_dev;
 
 struct kgd_mem;
 
+enum kfd_preempt_type {
+	KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN = 0,
+	KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
+};
+
 enum kgd_memory_pool {
 	KGD_POOL_SYSTEM_CACHEABLE = 1,
 	KGD_POOL_SYSTEM_WRITECOMBINE = 2,
@@ -82,6 +87,17 @@ struct kgd2kfd_shared_resources {
 	size_t doorbell_start_offset;
 };
 
+struct tile_config {
+	uint32_t *tile_config_ptr;
+	uint32_t *macro_tile_config_ptr;
+	uint32_t num_tile_configs;
+	uint32_t num_macro_tile_configs;
+
+	uint32_t gb_addr_config;
+	uint32_t num_banks;
+	uint32_t num_ranks;
+};
+
 /**
  * struct kfd2kgd_calls
  *
@@ -123,6 +139,11 @@ struct kgd2kfd_shared_resources {
  *
  * @get_fw_version: Returns FW versions from the header
  *
+ * @set_scratch_backing_va: Sets VA for scratch backing memory of a VMID.
+ * Only used for no cp scheduling mode
+ *
+ * @get_tile_config: Returns GPU-specific tiling mode information
+ *
  * This structure contains function pointers to services that the kgd driver
  * provides to amdkfd driver.
  *
@@ -153,14 +174,16 @@ struct kfd2kgd_calls {
 	int (*init_interrupts)(struct kgd_dev *kgd, uint32_t pipe_id);
 
 	int (*hqd_load)(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
-			uint32_t queue_id, uint32_t __user *wptr);
+			uint32_t queue_id, uint32_t __user *wptr,
+			uint32_t wptr_shift, uint32_t wptr_mask,
+			struct mm_struct *mm);
 
 	int (*hqd_sdma_load)(struct kgd_dev *kgd, void *mqd);
 
 	bool (*hqd_is_occupied)(struct kgd_dev *kgd, uint64_t queue_address,
 				uint32_t pipe_id, uint32_t queue_id);
 
-	int (*hqd_destroy)(struct kgd_dev *kgd, uint32_t reset_type,
+	int (*hqd_destroy)(struct kgd_dev *kgd, void *mqd, uint32_t reset_type,
 				unsigned int timeout, uint32_t pipe_id,
 				uint32_t queue_id);
 
@@ -192,6 +215,9 @@ struct kfd2kgd_calls {
 
 	uint16_t (*get_fw_version)(struct kgd_dev *kgd,
 				enum kgd_engine_type type);
+	void (*set_scratch_backing_va)(struct kgd_dev *kgd,
+				uint64_t va, uint32_t vmid);
+	int (*get_tile_config)(struct kgd_dev *kgd, struct tile_config *config);
 };
 
 /**
diff --git a/drivers/gpu/drm/amd/include/vi_structs.h b/drivers/gpu/drm/amd/include/vi_structs.h
index ca93b5160ba6..3e606a761d0e 100644
--- a/drivers/gpu/drm/amd/include/vi_structs.h
+++ b/drivers/gpu/drm/amd/include/vi_structs.h
@@ -419,8 +419,8 @@ struct vi_mqd_allocation {
 	struct vi_mqd mqd;
 	uint32_t wptr_poll_mem;
 	uint32_t rptr_report_mem;
-	uint32_t dyamic_cu_mask;
-	uint32_t dyamic_rb_mask;
+	uint32_t dynamic_cu_mask;
+	uint32_t dynamic_rb_mask;
 };
 
 struct cz_mqd {
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
index 0b74da3dca8b..bc839ff0bdd0 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
@@ -1240,13 +1240,18 @@ static int cz_phm_force_dpm_highest(struct pp_hwmgr *hwmgr)
 {
 	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
 
-	if (cz_hwmgr->sclk_dpm.soft_min_clk !=
-				cz_hwmgr->sclk_dpm.soft_max_clk)
-		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
-						PPSMC_MSG_SetSclkSoftMin,
-						cz_get_sclk_level(hwmgr,
-						cz_hwmgr->sclk_dpm.soft_max_clk,
-						PPSMC_MSG_SetSclkSoftMin));
+	smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+					PPSMC_MSG_SetSclkSoftMin,
+					cz_get_sclk_level(hwmgr,
+					cz_hwmgr->sclk_dpm.soft_max_clk,
+					PPSMC_MSG_SetSclkSoftMin));
+
+	smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_SetSclkSoftMax,
+				cz_get_sclk_level(hwmgr,
+				cz_hwmgr->sclk_dpm.soft_max_clk,
+				PPSMC_MSG_SetSclkSoftMax));
+
 	return 0;
 }
 
@@ -1292,17 +1297,55 @@ static int cz_phm_force_dpm_lowest(struct pp_hwmgr *hwmgr)
 {
 	struct cz_hwmgr *cz_hwmgr = (struct cz_hwmgr *)(hwmgr->backend);
 
-	if (cz_hwmgr->sclk_dpm.soft_min_clk !=
-				cz_hwmgr->sclk_dpm.soft_max_clk) {
-		cz_hwmgr->sclk_dpm.soft_max_clk =
-			cz_hwmgr->sclk_dpm.soft_min_clk;
+	smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+			PPSMC_MSG_SetSclkSoftMax,
+			cz_get_sclk_level(hwmgr,
+			cz_hwmgr->sclk_dpm.soft_min_clk,
+			PPSMC_MSG_SetSclkSoftMax));
 
-		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+	smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_SetSclkSoftMin,
+				cz_get_sclk_level(hwmgr,
+				cz_hwmgr->sclk_dpm.soft_min_clk,
+				PPSMC_MSG_SetSclkSoftMin));
+
+	return 0;
+}
+
+static int cz_phm_force_dpm_sclk(struct pp_hwmgr *hwmgr, uint32_t sclk)
+{
+	smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+				PPSMC_MSG_SetSclkSoftMin,
+				cz_get_sclk_level(hwmgr,
+				sclk,
+				PPSMC_MSG_SetSclkSoftMin));
+
+	smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
 				PPSMC_MSG_SetSclkSoftMax,
 				cz_get_sclk_level(hwmgr,
-				cz_hwmgr->sclk_dpm.soft_max_clk,
+				sclk,
 				PPSMC_MSG_SetSclkSoftMax));
+	return 0;
+}
+
+static int cz_get_profiling_clk(struct pp_hwmgr *hwmgr, uint32_t *sclk)
+{
+	struct phm_clock_voltage_dependency_table *table =
+		hwmgr->dyn_state.vddc_dependency_on_sclk;
+	int32_t tmp_sclk;
+	int32_t count;
+
+	tmp_sclk = table->entries[table->count-1].clk * 70 / 100;
+
+	for (count = table->count-1; count >= 0; count--) {
+		if (tmp_sclk >= table->entries[count].clk) {
+			tmp_sclk = table->entries[count].clk;
+			*sclk = tmp_sclk;
+			break;
+		}
 	}
+	if (count < 0)
+		*sclk = table->entries[0].clk;
 
 	return 0;
 }
@@ -1310,30 +1353,70 @@ static int cz_phm_force_dpm_lowest(struct pp_hwmgr *hwmgr)
 static int cz_dpm_force_dpm_level(struct pp_hwmgr *hwmgr,
 				enum amd_dpm_forced_level level)
 {
+	uint32_t sclk = 0;
 	int ret = 0;
+	uint32_t profile_mode_mask = AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD |
+					AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK |
+					AMD_DPM_FORCED_LEVEL_PROFILE_PEAK;
+
+	if (level == hwmgr->dpm_level)
+		return ret;
+
+	if (!(hwmgr->dpm_level & profile_mode_mask)) {
+		/* enter profile mode, save current level, disable gfx cg*/
+		if (level & profile_mode_mask) {
+			hwmgr->saved_dpm_level = hwmgr->dpm_level;
+			cgs_set_clockgating_state(hwmgr->device,
+						AMD_IP_BLOCK_TYPE_GFX,
+						AMD_CG_STATE_UNGATE);
+		}
+	} else {
+		/* exit profile mode, restore level, enable gfx cg*/
+		if (!(level & profile_mode_mask)) {
+			if (level == AMD_DPM_FORCED_LEVEL_PROFILE_EXIT)
+				level = hwmgr->saved_dpm_level;
+			cgs_set_clockgating_state(hwmgr->device,
+					AMD_IP_BLOCK_TYPE_GFX,
+					AMD_CG_STATE_GATE);
+		}
+	}
 
 	switch (level) {
 	case AMD_DPM_FORCED_LEVEL_HIGH:
+	case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK:
 		ret = cz_phm_force_dpm_highest(hwmgr);
 		if (ret)
 			return ret;
+		hwmgr->dpm_level = level;
 		break;
 	case AMD_DPM_FORCED_LEVEL_LOW:
+	case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK:
 		ret = cz_phm_force_dpm_lowest(hwmgr);
 		if (ret)
 			return ret;
+		hwmgr->dpm_level = level;
 		break;
 	case AMD_DPM_FORCED_LEVEL_AUTO:
 		ret = cz_phm_unforce_dpm_levels(hwmgr);
 		if (ret)
 			return ret;
+		hwmgr->dpm_level = level;
+		break;
+	case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD:
+		ret = cz_get_profiling_clk(hwmgr, &sclk);
+		if (ret)
+			return ret;
+		hwmgr->dpm_level = level;
+		cz_phm_force_dpm_sclk(hwmgr, sclk);
+		break;
+	case AMD_DPM_FORCED_LEVEL_MANUAL:
+		hwmgr->dpm_level = level;
 		break;
+	case AMD_DPM_FORCED_LEVEL_PROFILE_EXIT:
 	default:
 		break;
 	}
 
-	hwmgr->dpm_level = level;
-
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
index d025653c7823..9547f265a8bb 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
@@ -557,9 +557,8 @@ uint16_t phm_find_closest_vddci(struct pp_atomctrl_voltage_table *vddci_table, u
 			return vddci_table->entries[i].value;
 	}
 
-	PP_ASSERT_WITH_CODE(false,
-			"VDDCI is larger than max VDDCI in VDDCI Voltage Table!",
-			return vddci_table->entries[i-1].value);
+	pr_debug("vddci is larger than max value in vddci_table\n");
+	return vddci_table->entries[i-1].value;
 }
 
 int phm_find_boot_level(void *table,
@@ -583,26 +582,26 @@ int phm_get_sclk_for_voltage_evv(struct pp_hwmgr *hwmgr,
 	phm_ppt_v1_voltage_lookup_table *lookup_table,
 	uint16_t virtual_voltage_id, int32_t *sclk)
 {
-	uint8_t entryId;
-	uint8_t voltageId;
+	uint8_t entry_id;
+	uint8_t voltage_id;
 	struct phm_ppt_v1_information *table_info =
 			(struct phm_ppt_v1_information *)(hwmgr->pptable);
 
 	PP_ASSERT_WITH_CODE(lookup_table->count != 0, "Lookup table is empty", return -EINVAL);
 
 	/* search for leakage voltage ID 0xff01 ~ 0xff08 and sckl */
-	for (entryId = 0; entryId < table_info->vdd_dep_on_sclk->count; entryId++) {
-		voltageId = table_info->vdd_dep_on_sclk->entries[entryId].vddInd;
-		if (lookup_table->entries[voltageId].us_vdd == virtual_voltage_id)
+	for (entry_id = 0; entry_id < table_info->vdd_dep_on_sclk->count; entry_id++) {
+		voltage_id = table_info->vdd_dep_on_sclk->entries[entry_id].vddInd;
+		if (lookup_table->entries[voltage_id].us_vdd == virtual_voltage_id)
 			break;
 	}
 
-	PP_ASSERT_WITH_CODE(entryId < table_info->vdd_dep_on_sclk->count,
-			"Can't find requested voltage id in vdd_dep_on_sclk table!",
-			return -EINVAL;
-			);
+	if (entry_id >= table_info->vdd_dep_on_sclk->count) {
+		pr_debug("Can't find requested voltage id in vdd_dep_on_sclk table\n");
+		return -EINVAL;
+	}
 
-	*sclk = table_info->vdd_dep_on_sclk->entries[entryId].clk;
+	*sclk = table_info->vdd_dep_on_sclk->entries[entry_id].clk;
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c
index 720d5006ff62..c062844b15f3 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c
@@ -142,7 +142,7 @@ int pp_atomfwctrl_get_voltage_table_v4(struct pp_hwmgr *hwmgr,
 		}
 	} else if (voltage_mode == VOLTAGE_OBJ_SVID2) {
 		voltage_table->psi1_enable =
-			voltage_object->svid2_voltage_obj.loadline_psi1 & 0x1;
+			(voltage_object->svid2_voltage_obj.loadline_psi1 & 0x20) >> 5;
 		voltage_table->psi0_enable =
 			voltage_object->svid2_voltage_obj.psi0_enable & 0x1;
 		voltage_table->max_vid_step =
@@ -276,7 +276,10 @@ int pp_atomfwctrl_get_avfs_information(struct pp_hwmgr *hwmgr,
 		struct pp_atomfwctrl_avfs_parameters *param)
 {
 	uint16_t idx;
+	uint8_t format_revision, content_revision;
+
 	struct atom_asic_profiling_info_v4_1 *profile;
+	struct atom_asic_profiling_info_v4_2 *profile_v4_2;
 
 	idx = GetIndexIntoMasterDataTable(asic_profiling_info);
 	profile = (struct atom_asic_profiling_info_v4_1 *)
@@ -286,76 +289,172 @@ int pp_atomfwctrl_get_avfs_information(struct pp_hwmgr *hwmgr,
 	if (!profile)
 		return -1;
 
-	param->ulMaxVddc = le32_to_cpu(profile->maxvddc);
-	param->ulMinVddc = le32_to_cpu(profile->minvddc);
-	param->ulMeanNsigmaAcontant0 =
-			le32_to_cpu(profile->avfs_meannsigma_acontant0);
-	param->ulMeanNsigmaAcontant1 =
-			le32_to_cpu(profile->avfs_meannsigma_acontant1);
-	param->ulMeanNsigmaAcontant2 =
-			le32_to_cpu(profile->avfs_meannsigma_acontant2);
-	param->usMeanNsigmaDcTolSigma =
-			le16_to_cpu(profile->avfs_meannsigma_dc_tol_sigma);
-	param->usMeanNsigmaPlatformMean =
-			le16_to_cpu(profile->avfs_meannsigma_platform_mean);
-	param->usMeanNsigmaPlatformSigma =
-			le16_to_cpu(profile->avfs_meannsigma_platform_sigma);
-	param->ulGbVdroopTableCksoffA0 =
-			le32_to_cpu(profile->gb_vdroop_table_cksoff_a0);
-	param->ulGbVdroopTableCksoffA1 =
-			le32_to_cpu(profile->gb_vdroop_table_cksoff_a1);
-	param->ulGbVdroopTableCksoffA2 =
-			le32_to_cpu(profile->gb_vdroop_table_cksoff_a2);
-	param->ulGbVdroopTableCksonA0 =
-			le32_to_cpu(profile->gb_vdroop_table_ckson_a0);
-	param->ulGbVdroopTableCksonA1 =
-			le32_to_cpu(profile->gb_vdroop_table_ckson_a1);
-	param->ulGbVdroopTableCksonA2 =
-			le32_to_cpu(profile->gb_vdroop_table_ckson_a2);
-	param->ulGbFuseTableCksoffM1 =
-			le32_to_cpu(profile->avfsgb_fuse_table_cksoff_m1);
-	param->ulGbFuseTableCksoffM2 =
-			le32_to_cpu(profile->avfsgb_fuse_table_cksoff_m2);
-	param->ulGbFuseTableCksoffB =
-			le32_to_cpu(profile->avfsgb_fuse_table_cksoff_b);
-	param->ulGbFuseTableCksonM1 =
-			le32_to_cpu(profile->avfsgb_fuse_table_ckson_m1);
-	param->ulGbFuseTableCksonM2 =
-			le32_to_cpu(profile->avfsgb_fuse_table_ckson_m2);
-	param->ulGbFuseTableCksonB =
-			le32_to_cpu(profile->avfsgb_fuse_table_ckson_b);
-
-	param->ucEnableGbVdroopTableCkson =
-			profile->enable_gb_vdroop_table_ckson;
-	param->ucEnableGbFuseTableCkson =
-			profile->enable_gb_fuse_table_ckson;
-	param->usPsmAgeComfactor =
-			le16_to_cpu(profile->psm_age_comfactor);
-
-	param->ulDispclk2GfxclkM1 =
-			le32_to_cpu(profile->dispclk2gfxclk_a);
-	param->ulDispclk2GfxclkM2 =
-			le32_to_cpu(profile->dispclk2gfxclk_b);
-	param->ulDispclk2GfxclkB =
-			le32_to_cpu(profile->dispclk2gfxclk_c);
-	param->ulDcefclk2GfxclkM1 =
-			le32_to_cpu(profile->dcefclk2gfxclk_a);
-	param->ulDcefclk2GfxclkM2 =
-			le32_to_cpu(profile->dcefclk2gfxclk_b);
-	param->ulDcefclk2GfxclkB =
-			le32_to_cpu(profile->dcefclk2gfxclk_c);
-	param->ulPixelclk2GfxclkM1 =
-			le32_to_cpu(profile->pixclk2gfxclk_a);
-	param->ulPixelclk2GfxclkM2 =
-			le32_to_cpu(profile->pixclk2gfxclk_b);
-	param->ulPixelclk2GfxclkB =
-			le32_to_cpu(profile->pixclk2gfxclk_c);
-	param->ulPhyclk2GfxclkM1 =
-			le32_to_cpu(profile->phyclk2gfxclk_a);
-	param->ulPhyclk2GfxclkM2 =
-			le32_to_cpu(profile->phyclk2gfxclk_b);
-	param->ulPhyclk2GfxclkB =
-			le32_to_cpu(profile->phyclk2gfxclk_c);
+	format_revision = ((struct atom_common_table_header *)profile)->format_revision;
+	content_revision = ((struct atom_common_table_header *)profile)->content_revision;
+
+	if (format_revision == 4 && content_revision == 1) {
+		param->ulMaxVddc = le32_to_cpu(profile->maxvddc);
+		param->ulMinVddc = le32_to_cpu(profile->minvddc);
+		param->ulMeanNsigmaAcontant0 =
+				le32_to_cpu(profile->avfs_meannsigma_acontant0);
+		param->ulMeanNsigmaAcontant1 =
+				le32_to_cpu(profile->avfs_meannsigma_acontant1);
+		param->ulMeanNsigmaAcontant2 =
+				le32_to_cpu(profile->avfs_meannsigma_acontant2);
+		param->usMeanNsigmaDcTolSigma =
+				le16_to_cpu(profile->avfs_meannsigma_dc_tol_sigma);
+		param->usMeanNsigmaPlatformMean =
+				le16_to_cpu(profile->avfs_meannsigma_platform_mean);
+		param->usMeanNsigmaPlatformSigma =
+				le16_to_cpu(profile->avfs_meannsigma_platform_sigma);
+		param->ulGbVdroopTableCksoffA0 =
+				le32_to_cpu(profile->gb_vdroop_table_cksoff_a0);
+		param->ulGbVdroopTableCksoffA1 =
+				le32_to_cpu(profile->gb_vdroop_table_cksoff_a1);
+		param->ulGbVdroopTableCksoffA2 =
+				le32_to_cpu(profile->gb_vdroop_table_cksoff_a2);
+		param->ulGbVdroopTableCksonA0 =
+				le32_to_cpu(profile->gb_vdroop_table_ckson_a0);
+		param->ulGbVdroopTableCksonA1 =
+				le32_to_cpu(profile->gb_vdroop_table_ckson_a1);
+		param->ulGbVdroopTableCksonA2 =
+				le32_to_cpu(profile->gb_vdroop_table_ckson_a2);
+		param->ulGbFuseTableCksoffM1 =
+				le32_to_cpu(profile->avfsgb_fuse_table_cksoff_m1);
+		param->ulGbFuseTableCksoffM2 =
+				le32_to_cpu(profile->avfsgb_fuse_table_cksoff_m2);
+		param->ulGbFuseTableCksoffB =
+				le32_to_cpu(profile->avfsgb_fuse_table_cksoff_b);
+		param->ulGbFuseTableCksonM1 =
+				le32_to_cpu(profile->avfsgb_fuse_table_ckson_m1);
+		param->ulGbFuseTableCksonM2 =
+				le32_to_cpu(profile->avfsgb_fuse_table_ckson_m2);
+		param->ulGbFuseTableCksonB =
+				le32_to_cpu(profile->avfsgb_fuse_table_ckson_b);
+
+		param->ucEnableGbVdroopTableCkson =
+				profile->enable_gb_vdroop_table_ckson;
+		param->ucEnableGbFuseTableCkson =
+				profile->enable_gb_fuse_table_ckson;
+		param->usPsmAgeComfactor =
+				le16_to_cpu(profile->psm_age_comfactor);
+
+		param->ulDispclk2GfxclkM1 =
+				le32_to_cpu(profile->dispclk2gfxclk_a);
+		param->ulDispclk2GfxclkM2 =
+				le32_to_cpu(profile->dispclk2gfxclk_b);
+		param->ulDispclk2GfxclkB =
+				le32_to_cpu(profile->dispclk2gfxclk_c);
+		param->ulDcefclk2GfxclkM1 =
+				le32_to_cpu(profile->dcefclk2gfxclk_a);
+		param->ulDcefclk2GfxclkM2 =
+				le32_to_cpu(profile->dcefclk2gfxclk_b);
+		param->ulDcefclk2GfxclkB =
+				le32_to_cpu(profile->dcefclk2gfxclk_c);
+		param->ulPixelclk2GfxclkM1 =
+				le32_to_cpu(profile->pixclk2gfxclk_a);
+		param->ulPixelclk2GfxclkM2 =
+				le32_to_cpu(profile->pixclk2gfxclk_b);
+		param->ulPixelclk2GfxclkB =
+				le32_to_cpu(profile->pixclk2gfxclk_c);
+		param->ulPhyclk2GfxclkM1 =
+				le32_to_cpu(profile->phyclk2gfxclk_a);
+		param->ulPhyclk2GfxclkM2 =
+				le32_to_cpu(profile->phyclk2gfxclk_b);
+		param->ulPhyclk2GfxclkB =
+				le32_to_cpu(profile->phyclk2gfxclk_c);
+		param->ulAcgGbVdroopTableA0           = 0;
+		param->ulAcgGbVdroopTableA1           = 0;
+		param->ulAcgGbVdroopTableA2           = 0;
+		param->ulAcgGbFuseTableM1             = 0;
+		param->ulAcgGbFuseTableM2             = 0;
+		param->ulAcgGbFuseTableB              = 0;
+		param->ucAcgEnableGbVdroopTable       = 0;
+		param->ucAcgEnableGbFuseTable         = 0;
+	} else if (format_revision == 4 && content_revision == 2) {
+		profile_v4_2 = (struct atom_asic_profiling_info_v4_2 *)profile;
+		param->ulMaxVddc = le32_to_cpu(profile_v4_2->maxvddc);
+		param->ulMinVddc = le32_to_cpu(profile_v4_2->minvddc);
+		param->ulMeanNsigmaAcontant0 =
+				le32_to_cpu(profile_v4_2->avfs_meannsigma_acontant0);
+		param->ulMeanNsigmaAcontant1 =
+				le32_to_cpu(profile_v4_2->avfs_meannsigma_acontant1);
+		param->ulMeanNsigmaAcontant2 =
+				le32_to_cpu(profile_v4_2->avfs_meannsigma_acontant2);
+		param->usMeanNsigmaDcTolSigma =
+				le16_to_cpu(profile_v4_2->avfs_meannsigma_dc_tol_sigma);
+		param->usMeanNsigmaPlatformMean =
+				le16_to_cpu(profile_v4_2->avfs_meannsigma_platform_mean);
+		param->usMeanNsigmaPlatformSigma =
+				le16_to_cpu(profile_v4_2->avfs_meannsigma_platform_sigma);
+		param->ulGbVdroopTableCksoffA0 =
+				le32_to_cpu(profile_v4_2->gb_vdroop_table_cksoff_a0);
+		param->ulGbVdroopTableCksoffA1 =
+				le32_to_cpu(profile_v4_2->gb_vdroop_table_cksoff_a1);
+		param->ulGbVdroopTableCksoffA2 =
+				le32_to_cpu(profile_v4_2->gb_vdroop_table_cksoff_a2);
+		param->ulGbVdroopTableCksonA0 =
+				le32_to_cpu(profile_v4_2->gb_vdroop_table_ckson_a0);
+		param->ulGbVdroopTableCksonA1 =
+				le32_to_cpu(profile_v4_2->gb_vdroop_table_ckson_a1);
+		param->ulGbVdroopTableCksonA2 =
+				le32_to_cpu(profile_v4_2->gb_vdroop_table_ckson_a2);
+		param->ulGbFuseTableCksoffM1 =
+				le32_to_cpu(profile_v4_2->avfsgb_fuse_table_cksoff_m1);
+		param->ulGbFuseTableCksoffM2 =
+				le32_to_cpu(profile_v4_2->avfsgb_fuse_table_cksoff_m2);
+		param->ulGbFuseTableCksoffB =
+				le32_to_cpu(profile_v4_2->avfsgb_fuse_table_cksoff_b);
+		param->ulGbFuseTableCksonM1 =
+				le32_to_cpu(profile_v4_2->avfsgb_fuse_table_ckson_m1);
+		param->ulGbFuseTableCksonM2 =
+				le32_to_cpu(profile_v4_2->avfsgb_fuse_table_ckson_m2);
+		param->ulGbFuseTableCksonB =
+				le32_to_cpu(profile_v4_2->avfsgb_fuse_table_ckson_b);
+
+		param->ucEnableGbVdroopTableCkson =
+				profile_v4_2->enable_gb_vdroop_table_ckson;
+		param->ucEnableGbFuseTableCkson =
+				profile_v4_2->enable_gb_fuse_table_ckson;
+		param->usPsmAgeComfactor =
+				le16_to_cpu(profile_v4_2->psm_age_comfactor);
+
+		param->ulDispclk2GfxclkM1 =
+				le32_to_cpu(profile_v4_2->dispclk2gfxclk_a);
+		param->ulDispclk2GfxclkM2 =
+				le32_to_cpu(profile_v4_2->dispclk2gfxclk_b);
+		param->ulDispclk2GfxclkB =
+				le32_to_cpu(profile_v4_2->dispclk2gfxclk_c);
+		param->ulDcefclk2GfxclkM1 =
+				le32_to_cpu(profile_v4_2->dcefclk2gfxclk_a);
+		param->ulDcefclk2GfxclkM2 =
+				le32_to_cpu(profile_v4_2->dcefclk2gfxclk_b);
+		param->ulDcefclk2GfxclkB =
+				le32_to_cpu(profile_v4_2->dcefclk2gfxclk_c);
+		param->ulPixelclk2GfxclkM1 =
+				le32_to_cpu(profile_v4_2->pixclk2gfxclk_a);
+		param->ulPixelclk2GfxclkM2 =
+				le32_to_cpu(profile_v4_2->pixclk2gfxclk_b);
+		param->ulPixelclk2GfxclkB =
+				le32_to_cpu(profile_v4_2->pixclk2gfxclk_c);
+		param->ulPhyclk2GfxclkM1 =
+				le32_to_cpu(profile->phyclk2gfxclk_a);
+		param->ulPhyclk2GfxclkM2 =
+				le32_to_cpu(profile_v4_2->phyclk2gfxclk_b);
+		param->ulPhyclk2GfxclkB =
+				le32_to_cpu(profile_v4_2->phyclk2gfxclk_c);
+		param->ulAcgGbVdroopTableA0 = le32_to_cpu(profile_v4_2->acg_gb_vdroop_table_a0);
+		param->ulAcgGbVdroopTableA1 = le32_to_cpu(profile_v4_2->acg_gb_vdroop_table_a1);
+		param->ulAcgGbVdroopTableA2 = le32_to_cpu(profile_v4_2->acg_gb_vdroop_table_a2);
+		param->ulAcgGbFuseTableM1 = le32_to_cpu(profile_v4_2->acg_avfsgb_fuse_table_m1);
+		param->ulAcgGbFuseTableM2 = le32_to_cpu(profile_v4_2->acg_avfsgb_fuse_table_m2);
+		param->ulAcgGbFuseTableB = le32_to_cpu(profile_v4_2->acg_avfsgb_fuse_table_b);
+		param->ucAcgEnableGbVdroopTable = le32_to_cpu(profile_v4_2->enable_acg_gb_vdroop_table);
+		param->ucAcgEnableGbFuseTable = le32_to_cpu(profile_v4_2->enable_acg_gb_fuse_table);
+	} else {
+		pr_info("Invalid VBIOS AVFS ProfilingInfo Revision!\n");
+		return -EINVAL;
+	}
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.h b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.h
index 81908b5cfd5f..8e6b1f0ddebc 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.h
@@ -109,6 +109,14 @@ struct pp_atomfwctrl_avfs_parameters {
 	uint32_t   ulPhyclk2GfxclkM1;
 	uint32_t   ulPhyclk2GfxclkM2;
 	uint32_t   ulPhyclk2GfxclkB;
+	uint32_t   ulAcgGbVdroopTableA0;
+	uint32_t   ulAcgGbVdroopTableA1;
+	uint32_t   ulAcgGbVdroopTableA2;
+	uint32_t   ulAcgGbFuseTableM1;
+	uint32_t   ulAcgGbFuseTableM2;
+	uint32_t   ulAcgGbFuseTableB;
+	uint32_t   ucAcgEnableGbVdroopTable;
+	uint32_t   ucAcgEnableGbFuseTable;
 };
 
 struct pp_atomfwctrl_gpio_parameters {
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/rv_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/rv_hwmgr.c
index 4c7f430b36eb..2c3e6baf2524 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/rv_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/rv_hwmgr.c
@@ -265,6 +265,15 @@ static int rv_tf_set_clock_limit(struct pp_hwmgr *hwmgr, void *input,
 		}
 	} */
 
+	if (((hwmgr->uvd_arbiter.vclk_soft_min / 100) != rv_data->vclk_soft_min) ||
+	    ((hwmgr->uvd_arbiter.dclk_soft_min / 100) != rv_data->dclk_soft_min)) {
+		rv_data->vclk_soft_min = hwmgr->uvd_arbiter.vclk_soft_min / 100;
+		rv_data->dclk_soft_min = hwmgr->uvd_arbiter.dclk_soft_min / 100;
+		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+			PPSMC_MSG_SetSoftMinVcn,
+			(rv_data->vclk_soft_min << 16) | rv_data->vclk_soft_min);
+	}
+
 	if((hwmgr->gfx_arbiter.sclk_hard_min != 0) &&
 		((hwmgr->gfx_arbiter.sclk_hard_min / 100) != rv_data->soc_actual_hard_min_freq)) {
 		smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
@@ -308,8 +317,8 @@ static int rv_tf_set_num_active_display(struct pp_hwmgr *hwmgr, void *input,
 }
 
 static const struct phm_master_table_item rv_set_power_state_list[] = {
-	{ NULL, rv_tf_set_clock_limit },
-	{ NULL, rv_tf_set_num_active_display },
+	{ .tableFunction = rv_tf_set_clock_limit },
+	{ .tableFunction = rv_tf_set_num_active_display },
 	{ }
 };
 
@@ -382,7 +391,7 @@ static int rv_tf_disable_gfx_off(struct pp_hwmgr *hwmgr,
 }
 
 static const struct phm_master_table_item rv_disable_dpm_list[] = {
-	{NULL, rv_tf_disable_gfx_off},
+	{ .tableFunction = rv_tf_disable_gfx_off },
 	{ },
 };
 
@@ -407,7 +416,7 @@ static int rv_tf_enable_gfx_off(struct pp_hwmgr *hwmgr,
 }
 
 static const struct phm_master_table_item rv_enable_dpm_list[] = {
-	{NULL, rv_tf_enable_gfx_off},
+	{ .tableFunction = rv_tf_enable_gfx_off },
 	{ },
 };
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/rv_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/rv_hwmgr.h
index afb852295a15..2472b50e54cf 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/rv_hwmgr.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/rv_hwmgr.h
@@ -280,6 +280,8 @@ struct rv_hwmgr {
 
 	uint32_t                        f_actual_hard_min_freq;
 	uint32_t                        fabric_actual_soft_min_freq;
+	uint32_t                        vclk_soft_min;
+	uint32_t                        dclk_soft_min;
 	uint32_t                        gfx_actual_soft_min_freq;
 
 	bool                           vcn_power_gated;
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
index 1f01020ce3a9..c2743233ba10 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
@@ -1962,9 +1962,6 @@ static int smu7_thermal_parameter_init(struct pp_hwmgr *hwmgr)
 			temp_reg = PHM_SET_FIELD(temp_reg, CNB_PWRMGT_CNTL, DPM_ENABLED, 0x1);
 			break;
 		default:
-			PP_ASSERT_WITH_CODE(0,
-			"Failed to setup PCC HW register! Wrong GPIO assigned for VDDC_PCC_GPIO_PINID!",
-			);
 			break;
 		}
 		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, ixCNB_PWRMGT_CNTL, temp_reg);
@@ -4630,6 +4627,15 @@ static int smu7_set_power_profile_state(struct pp_hwmgr *hwmgr,
 
 static int smu7_avfs_control(struct pp_hwmgr *hwmgr, bool enable)
 {
+	struct pp_smumgr *smumgr = (struct pp_smumgr *)(hwmgr->smumgr);
+	struct smu7_smumgr *smu_data = (struct smu7_smumgr *)(smumgr->backend);
+
+	if (smu_data == NULL)
+		return -EINVAL;
+
+	if (smu_data->avfs.avfs_btc_status == AVFS_BTC_NOTSUPPORTED)
+		return 0;
+
 	if (enable) {
 		if (!PHM_READ_VFPF_INDIRECT_FIELD(hwmgr->device,
 				CGS_IND_REG__SMC, FEATURE_STATUS, AVS_ON))
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c
index 197174e562d2..f8f02e70b8bc 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c
@@ -78,6 +78,8 @@ uint32_t channel_number[] = {1, 2, 0, 4, 0, 8, 0, 16, 2};
 #define DF_CS_AON0_DramBaseAddress0__IntLvNumChan_MASK                                                        0x000000F0L
 #define DF_CS_AON0_DramBaseAddress0__IntLvAddrSel_MASK                                                        0x00000700L
 #define DF_CS_AON0_DramBaseAddress0__DramBaseAddr_MASK                                                        0xFFFFF000L
+static int vega10_force_clock_level(struct pp_hwmgr *hwmgr,
+		enum pp_clock_type type, uint32_t mask);
 
 const ULONG PhwVega10_Magic = (ULONG)(PHM_VIslands_Magic);
 
@@ -146,6 +148,19 @@ static void vega10_set_default_registry_data(struct pp_hwmgr *hwmgr)
 	data->registry_data.vr1hot_enabled = 1;
 	data->registry_data.regulator_hot_gpio_support = 1;
 
+	data->registry_data.didt_support = 1;
+	if (data->registry_data.didt_support) {
+		data->registry_data.didt_mode = 6;
+		data->registry_data.sq_ramping_support = 1;
+		data->registry_data.db_ramping_support = 0;
+		data->registry_data.td_ramping_support = 0;
+		data->registry_data.tcp_ramping_support = 0;
+		data->registry_data.dbr_ramping_support = 0;
+		data->registry_data.edc_didt_support = 1;
+		data->registry_data.gc_didt_support = 0;
+		data->registry_data.psm_didt_support = 0;
+	}
+
 	data->display_voltage_mode = PPVEGA10_VEGA10DISPLAYVOLTAGEMODE_DFLT;
 	data->dcef_clk_quad_eqn_a = PPREGKEY_VEGA10QUADRATICEQUATION_DFLT;
 	data->dcef_clk_quad_eqn_b = PPREGKEY_VEGA10QUADRATICEQUATION_DFLT;
@@ -223,6 +238,8 @@ static int vega10_set_features_platform_caps(struct pp_hwmgr *hwmgr)
 	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_PowerContainment);
 	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_DiDtSupport);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_SQRamping);
 	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_DBRamping);
@@ -230,6 +247,34 @@ static int vega10_set_features_platform_caps(struct pp_hwmgr *hwmgr)
 			PHM_PlatformCaps_TDRamping);
 	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_TCPRamping);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_DBRRamping);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_DiDtEDCEnable);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_GCEDC);
+	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+			PHM_PlatformCaps_PSM);
+
+	if (data->registry_data.didt_support) {
+		phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DiDtSupport);
+		if (data->registry_data.sq_ramping_support)
+			phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SQRamping);
+		if (data->registry_data.db_ramping_support)
+			phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DBRamping);
+		if (data->registry_data.td_ramping_support)
+			phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TDRamping);
+		if (data->registry_data.tcp_ramping_support)
+			phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TCPRamping);
+		if (data->registry_data.dbr_ramping_support)
+			phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DBRRamping);
+		if (data->registry_data.edc_didt_support)
+			phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DiDtEDCEnable);
+		if (data->registry_data.gc_didt_support)
+			phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_GCEDC);
+		if (data->registry_data.psm_didt_support)
+			phm_cap_set(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_PSM);
+	}
 
 	if (data->registry_data.power_containment_support)
 		phm_cap_set(hwmgr->platform_descriptor.platformCaps,
@@ -321,8 +366,8 @@ static void vega10_init_dpm_defaults(struct pp_hwmgr *hwmgr)
 			FEATURE_LED_DISPLAY_BIT;
 	data->smu_features[GNLD_FAN_CONTROL].smu_feature_id =
 			FEATURE_FAN_CONTROL_BIT;
-	data->smu_features[GNLD_VOLTAGE_CONTROLLER].smu_feature_id =
-			FEATURE_VOLTAGE_CONTROLLER_BIT;
+	data->smu_features[GNLD_ACG].smu_feature_id = FEATURE_ACG_BIT;
+	data->smu_features[GNLD_DIDT].smu_feature_id = FEATURE_GFX_EDC_BIT;
 
 	if (!data->registry_data.prefetcher_dpm_key_disabled)
 		data->smu_features[GNLD_DPM_PREFETCHER].supported = true;
@@ -386,6 +431,15 @@ static void vega10_init_dpm_defaults(struct pp_hwmgr *hwmgr)
 	if (data->registry_data.vr0hot_enabled)
 		data->smu_features[GNLD_VR0HOT].supported = true;
 
+	smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_GetSmuVersion);
+	vega10_read_arg_from_smc(hwmgr->smumgr, &(data->smu_version));
+		/* ACG firmware has major version 5 */
+	if ((data->smu_version & 0xff000000) == 0x5000000)
+		data->smu_features[GNLD_ACG].supported = true;
+
+	if (data->registry_data.didt_support)
+		data->smu_features[GNLD_DIDT].supported = true;
+
 }
 
 #ifdef PPLIB_VEGA10_EVV_SUPPORT
@@ -1504,7 +1558,8 @@ static int vega10_populate_smc_link_levels(struct pp_hwmgr *hwmgr)
 */
 
 static int vega10_populate_single_gfx_level(struct pp_hwmgr *hwmgr,
-		uint32_t gfx_clock, PllSetting_t *current_gfxclk_level)
+		uint32_t gfx_clock, PllSetting_t *current_gfxclk_level,
+		uint32_t *acg_freq)
 {
 	struct phm_ppt_v2_information *table_info =
 			(struct phm_ppt_v2_information *)(hwmgr->pptable);
@@ -1555,6 +1610,8 @@ static int vega10_populate_single_gfx_level(struct pp_hwmgr *hwmgr,
 			cpu_to_le16(dividers.usPll_ss_slew_frac);
 	current_gfxclk_level->Did = (uint8_t)(dividers.ulDid);
 
+	*acg_freq = gfx_clock / 100; /* 100 Khz to Mhz conversion */
+
 	return 0;
 }
 
@@ -1635,7 +1692,8 @@ static int vega10_populate_all_graphic_levels(struct pp_hwmgr *hwmgr)
 	for (i = 0; i < dpm_table->count; i++) {
 		result = vega10_populate_single_gfx_level(hwmgr,
 				dpm_table->dpm_levels[i].value,
-				&(pp_table->GfxclkLevel[i]));
+				&(pp_table->GfxclkLevel[i]),
+				&(pp_table->AcgFreqTable[i]));
 		if (result)
 			return result;
 	}
@@ -1644,7 +1702,8 @@ static int vega10_populate_all_graphic_levels(struct pp_hwmgr *hwmgr)
 	while (i < NUM_GFXCLK_DPM_LEVELS) {
 		result = vega10_populate_single_gfx_level(hwmgr,
 				dpm_table->dpm_levels[j].value,
-				&(pp_table->GfxclkLevel[i]));
+				&(pp_table->GfxclkLevel[i]),
+				&(pp_table->AcgFreqTable[i]));
 		if (result)
 			return result;
 		i++;
@@ -2222,6 +2281,21 @@ static int vega10_populate_avfs_parameters(struct pp_hwmgr *hwmgr)
 			pp_table->DisplayClock2Gfxclk[DSPCLK_PHYCLK].m1_shift = 24;
 			pp_table->DisplayClock2Gfxclk[DSPCLK_PHYCLK].m2_shift = 12;
 			pp_table->DisplayClock2Gfxclk[DSPCLK_PHYCLK].b_shift = 12;
+
+			pp_table->AcgBtcGbVdroopTable.a0       = avfs_params.ulAcgGbVdroopTableA0;
+			pp_table->AcgBtcGbVdroopTable.a0_shift = 20;
+			pp_table->AcgBtcGbVdroopTable.a1       = avfs_params.ulAcgGbVdroopTableA1;
+			pp_table->AcgBtcGbVdroopTable.a1_shift = 20;
+			pp_table->AcgBtcGbVdroopTable.a2       = avfs_params.ulAcgGbVdroopTableA2;
+			pp_table->AcgBtcGbVdroopTable.a2_shift = 20;
+
+			pp_table->AcgAvfsGb.m1                   = avfs_params.ulAcgGbFuseTableM1;
+			pp_table->AcgAvfsGb.m2                   = avfs_params.ulAcgGbFuseTableM2;
+			pp_table->AcgAvfsGb.b                    = avfs_params.ulAcgGbFuseTableB;
+			pp_table->AcgAvfsGb.m1_shift             = 0;
+			pp_table->AcgAvfsGb.m2_shift             = 0;
+			pp_table->AcgAvfsGb.b_shift              = 0;
+
 		} else {
 			data->smu_features[GNLD_AVFS].supported = false;
 		}
@@ -2230,6 +2304,55 @@ static int vega10_populate_avfs_parameters(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
+static int vega10_acg_enable(struct pp_hwmgr *hwmgr)
+{
+	struct vega10_hwmgr *data =
+			(struct vega10_hwmgr *)(hwmgr->backend);
+	uint32_t agc_btc_response;
+
+	if (data->smu_features[GNLD_ACG].supported) {
+		if (0 == vega10_enable_smc_features(hwmgr->smumgr, true,
+					data->smu_features[GNLD_DPM_PREFETCHER].smu_feature_bitmap))
+			data->smu_features[GNLD_DPM_PREFETCHER].enabled = true;
+
+		smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_InitializeAcg);
+
+		smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_RunAcgBtc);
+		vega10_read_arg_from_smc(hwmgr->smumgr, &agc_btc_response);
+
+		if (1 == agc_btc_response) {
+			if (1 == data->acg_loop_state)
+				smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_RunAcgInClosedLoop);
+			else if (2 == data->acg_loop_state)
+				smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_RunAcgInOpenLoop);
+			if (0 == vega10_enable_smc_features(hwmgr->smumgr, true,
+				data->smu_features[GNLD_ACG].smu_feature_bitmap))
+					data->smu_features[GNLD_ACG].enabled = true;
+		} else {
+			pr_info("[ACG_Enable] ACG BTC Returned Failed Status!\n");
+			data->smu_features[GNLD_ACG].enabled = false;
+		}
+	}
+
+	return 0;
+}
+
+static int vega10_acg_disable(struct pp_hwmgr *hwmgr)
+{
+	struct vega10_hwmgr *data =
+			(struct vega10_hwmgr *)(hwmgr->backend);
+
+	if (data->smu_features[GNLD_ACG].supported) {
+		if (data->smu_features[GNLD_ACG].enabled) {
+		if (0 == vega10_enable_smc_features(hwmgr->smumgr, false,
+				data->smu_features[GNLD_ACG].smu_feature_bitmap))
+			data->smu_features[GNLD_ACG].enabled = false;
+		}
+	}
+
+	return 0;
+}
+
 static int vega10_populate_gpio_parameters(struct pp_hwmgr *hwmgr)
 {
 	struct vega10_hwmgr *data =
@@ -2404,6 +2527,9 @@ static int vega10_init_smc_table(struct pp_hwmgr *hwmgr)
 	pp_table->DisplayDpmVoltageMode =
 			(uint8_t)(table_info->uc_dcef_dpm_voltage_mode);
 
+	data->vddc_voltage_table.psi0_enable = voltage_table.psi0_enable;
+	data->vddc_voltage_table.psi1_enable = voltage_table.psi1_enable;
+
 	if (data->registry_data.ulv_support &&
 			table_info->us_ulv_voltage_offset) {
 		result = vega10_populate_ulv_state(hwmgr);
@@ -2500,7 +2626,7 @@ static int vega10_init_smc_table(struct pp_hwmgr *hwmgr)
 	result = vega10_avfs_enable(hwmgr, true);
 	PP_ASSERT_WITH_CODE(!result, "Attempt to enable AVFS feature Failed!",
 					return result);
-
+	vega10_acg_enable(hwmgr);
 	vega10_save_default_power_profile(hwmgr);
 
 	return 0;
@@ -2832,6 +2958,11 @@ static int vega10_enable_dpm_tasks(struct pp_hwmgr *hwmgr)
 	PP_ASSERT_WITH_CODE(!tmp_result,
 			"Failed to start DPM!", result = tmp_result);
 
+	/* enable didt, do not abort if failed didt */
+	tmp_result = vega10_enable_didt_config(hwmgr);
+	PP_ASSERT(!tmp_result,
+			"Failed to enable didt config!");
+
 	tmp_result = vega10_enable_power_containment(hwmgr);
 	PP_ASSERT_WITH_CODE(!tmp_result,
 			"Failed to enable power containment!",
@@ -3578,10 +3709,22 @@ static void vega10_apply_dal_minimum_voltage_request(
 	return;
 }
 
+static int vega10_get_soc_index_for_max_uclk(struct pp_hwmgr *hwmgr)
+{
+	struct phm_ppt_v1_clock_voltage_dependency_table *vdd_dep_table_on_mclk;
+	struct phm_ppt_v2_information *table_info =
+			(struct phm_ppt_v2_information *)(hwmgr->pptable);
+
+	vdd_dep_table_on_mclk  = table_info->vdd_dep_on_mclk;
+
+	return vdd_dep_table_on_mclk->entries[NUM_UCLK_DPM_LEVELS - 1].vddInd + 1;
+}
+
 static int vega10_upload_dpm_bootup_level(struct pp_hwmgr *hwmgr)
 {
 	struct vega10_hwmgr *data =
 			(struct vega10_hwmgr *)(hwmgr->backend);
+	uint32_t socclk_idx;
 
 	vega10_apply_dal_minimum_voltage_request(hwmgr);
 
@@ -3602,13 +3745,22 @@ static int vega10_upload_dpm_bootup_level(struct pp_hwmgr *hwmgr)
 	if (!data->registry_data.mclk_dpm_key_disabled) {
 		if (data->smc_state_table.mem_boot_level !=
 				data->dpm_table.mem_table.dpm_state.soft_min_level) {
+			if (data->smc_state_table.mem_boot_level == NUM_UCLK_DPM_LEVELS - 1) {
+				socclk_idx = vega10_get_soc_index_for_max_uclk(hwmgr);
 				PP_ASSERT_WITH_CODE(!smum_send_msg_to_smc_with_parameter(
-				hwmgr->smumgr,
-				 PPSMC_MSG_SetSoftMinUclkByIndex,
-				data->smc_state_table.mem_boot_level),
-				"Failed to set soft min mclk index!",
-				return -EINVAL);
-
+							hwmgr->smumgr,
+						PPSMC_MSG_SetSoftMinSocclkByIndex,
+						socclk_idx),
+						"Failed to set soft min uclk index!",
+						return -EINVAL);
+			} else {
+				PP_ASSERT_WITH_CODE(!smum_send_msg_to_smc_with_parameter(
+						hwmgr->smumgr,
+						PPSMC_MSG_SetSoftMinUclkByIndex,
+						data->smc_state_table.mem_boot_level),
+						"Failed to set soft min uclk index!",
+						return -EINVAL);
+			}
 			data->dpm_table.mem_table.dpm_state.soft_min_level =
 					data->smc_state_table.mem_boot_level;
 		}
@@ -4015,7 +4167,7 @@ static int vega10_notify_smc_display_config_after_ps_adjustment(
 			pr_info("Attempt to set Hard Min for DCEFCLK Failed!");
 		}
 	} else {
-		pr_info("Cannot find requested DCEFCLK!");
+		pr_debug("Cannot find requested DCEFCLK!");
 	}
 
 	if (min_clocks.memoryClock != 0) {
@@ -4097,34 +4249,30 @@ static int vega10_unforce_dpm_levels(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
-static int vega10_dpm_force_dpm_level(struct pp_hwmgr *hwmgr,
-				enum amd_dpm_forced_level level)
+static int vega10_get_profiling_clk_mask(struct pp_hwmgr *hwmgr, enum amd_dpm_forced_level level,
+				uint32_t *sclk_mask, uint32_t *mclk_mask, uint32_t *soc_mask)
 {
-	int ret = 0;
+	struct phm_ppt_v2_information *table_info =
+			(struct phm_ppt_v2_information *)(hwmgr->pptable);
 
-	switch (level) {
-	case AMD_DPM_FORCED_LEVEL_HIGH:
-		ret = vega10_force_dpm_highest(hwmgr);
-		if (ret)
-			return ret;
-		break;
-	case AMD_DPM_FORCED_LEVEL_LOW:
-		ret = vega10_force_dpm_lowest(hwmgr);
-		if (ret)
-			return ret;
-		break;
-	case AMD_DPM_FORCED_LEVEL_AUTO:
-		ret = vega10_unforce_dpm_levels(hwmgr);
-		if (ret)
-			return ret;
-		break;
-	default:
-		break;
+	if (table_info->vdd_dep_on_sclk->count > VEGA10_UMD_PSTATE_GFXCLK_LEVEL &&
+		table_info->vdd_dep_on_socclk->count > VEGA10_UMD_PSTATE_SOCCLK_LEVEL &&
+		table_info->vdd_dep_on_mclk->count > VEGA10_UMD_PSTATE_MCLK_LEVEL) {
+		*sclk_mask = VEGA10_UMD_PSTATE_GFXCLK_LEVEL;
+		*soc_mask = VEGA10_UMD_PSTATE_SOCCLK_LEVEL;
+		*mclk_mask = VEGA10_UMD_PSTATE_MCLK_LEVEL;
 	}
 
-	hwmgr->dpm_level = level;
-
-	return ret;
+	if (level == AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK) {
+		*sclk_mask = 0;
+	} else if (level == AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK) {
+		*mclk_mask = 0;
+	} else if (level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK) {
+		*sclk_mask = table_info->vdd_dep_on_sclk->count - 1;
+		*soc_mask = table_info->vdd_dep_on_socclk->count - 1;
+		*mclk_mask = table_info->vdd_dep_on_mclk->count - 1;
+	}
+	return 0;
 }
 
 static int vega10_set_fan_control_mode(struct pp_hwmgr *hwmgr, uint32_t mode)
@@ -4151,6 +4299,86 @@ static int vega10_set_fan_control_mode(struct pp_hwmgr *hwmgr, uint32_t mode)
 	return result;
 }
 
+static int vega10_dpm_force_dpm_level(struct pp_hwmgr *hwmgr,
+				enum amd_dpm_forced_level level)
+{
+	int ret = 0;
+	uint32_t sclk_mask = 0;
+	uint32_t mclk_mask = 0;
+	uint32_t soc_mask = 0;
+	uint32_t profile_mode_mask = AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD |
+					AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK |
+					AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK |
+					AMD_DPM_FORCED_LEVEL_PROFILE_PEAK;
+
+	if (level == hwmgr->dpm_level)
+		return ret;
+
+	if (!(hwmgr->dpm_level & profile_mode_mask)) {
+		/* enter profile mode, save current level, disable gfx cg*/
+		if (level & profile_mode_mask) {
+			hwmgr->saved_dpm_level = hwmgr->dpm_level;
+			cgs_set_clockgating_state(hwmgr->device,
+						AMD_IP_BLOCK_TYPE_GFX,
+						AMD_CG_STATE_UNGATE);
+		}
+	} else {
+		/* exit profile mode, restore level, enable gfx cg*/
+		if (!(level & profile_mode_mask)) {
+			if (level == AMD_DPM_FORCED_LEVEL_PROFILE_EXIT)
+				level = hwmgr->saved_dpm_level;
+			cgs_set_clockgating_state(hwmgr->device,
+					AMD_IP_BLOCK_TYPE_GFX,
+					AMD_CG_STATE_GATE);
+		}
+	}
+
+	switch (level) {
+	case AMD_DPM_FORCED_LEVEL_HIGH:
+		ret = vega10_force_dpm_highest(hwmgr);
+		if (ret)
+			return ret;
+		hwmgr->dpm_level = level;
+		break;
+	case AMD_DPM_FORCED_LEVEL_LOW:
+		ret = vega10_force_dpm_lowest(hwmgr);
+		if (ret)
+			return ret;
+		hwmgr->dpm_level = level;
+		break;
+	case AMD_DPM_FORCED_LEVEL_AUTO:
+		ret = vega10_unforce_dpm_levels(hwmgr);
+		if (ret)
+			return ret;
+		hwmgr->dpm_level = level;
+		break;
+	case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD:
+	case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK:
+	case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK:
+	case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK:
+		ret = vega10_get_profiling_clk_mask(hwmgr, level, &sclk_mask, &mclk_mask, &soc_mask);
+		if (ret)
+			return ret;
+		hwmgr->dpm_level = level;
+		vega10_force_clock_level(hwmgr, PP_SCLK, 1<<sclk_mask);
+		vega10_force_clock_level(hwmgr, PP_MCLK, 1<<mclk_mask);
+		break;
+	case AMD_DPM_FORCED_LEVEL_MANUAL:
+		hwmgr->dpm_level = level;
+		break;
+	case AMD_DPM_FORCED_LEVEL_PROFILE_EXIT:
+	default:
+		break;
+	}
+
+	if (level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK && hwmgr->saved_dpm_level != AMD_DPM_FORCED_LEVEL_PROFILE_PEAK)
+		vega10_set_fan_control_mode(hwmgr, AMD_FAN_CTRL_NONE);
+	else if (level != AMD_DPM_FORCED_LEVEL_PROFILE_PEAK && hwmgr->saved_dpm_level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK)
+		vega10_set_fan_control_mode(hwmgr, AMD_FAN_CTRL_AUTO);
+
+	return 0;
+}
+
 static int vega10_get_fan_control_mode(struct pp_hwmgr *hwmgr)
 {
 	struct vega10_hwmgr *data = (struct vega10_hwmgr *)(hwmgr->backend);
@@ -4396,7 +4624,9 @@ static int vega10_force_clock_level(struct pp_hwmgr *hwmgr,
 	struct vega10_hwmgr *data = (struct vega10_hwmgr *)(hwmgr->backend);
 	int i;
 
-	if (hwmgr->dpm_level != AMD_DPM_FORCED_LEVEL_MANUAL)
+	if (hwmgr->dpm_level & (AMD_DPM_FORCED_LEVEL_AUTO |
+				AMD_DPM_FORCED_LEVEL_LOW |
+				AMD_DPM_FORCED_LEVEL_HIGH))
 		return -EINVAL;
 
 	switch (type) {
@@ -4661,6 +4891,10 @@ static int vega10_disable_dpm_tasks(struct pp_hwmgr *hwmgr)
 	PP_ASSERT_WITH_CODE((tmp_result == 0),
 			"Failed to disable power containment!", result = tmp_result);
 
+	tmp_result = vega10_disable_didt_config(hwmgr);
+	PP_ASSERT_WITH_CODE((tmp_result == 0),
+			"Failed to disable didt config!", result = tmp_result);
+
 	tmp_result = vega10_avfs_enable(hwmgr, false);
 	PP_ASSERT_WITH_CODE((tmp_result == 0),
 			"Failed to disable AVFS!", result = tmp_result);
@@ -4677,6 +4911,9 @@ static int vega10_disable_dpm_tasks(struct pp_hwmgr *hwmgr)
 	PP_ASSERT_WITH_CODE((tmp_result == 0),
 			"Failed to disable ulv!", result = tmp_result);
 
+	tmp_result =  vega10_acg_disable(hwmgr);
+	PP_ASSERT_WITH_CODE((tmp_result == 0),
+			"Failed to disable acg!", result = tmp_result);
 	return result;
 }
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.h
index 6e5c5b99593b..676cd7735883 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.h
@@ -64,7 +64,9 @@ enum {
 	GNLD_FW_CTF,
 	GNLD_LED_DISPLAY,
 	GNLD_FAN_CONTROL,
-	GNLD_VOLTAGE_CONTROLLER,
+	GNLD_FEATURE_FAST_PPT_BIT,
+	GNLD_DIDT,
+	GNLD_ACG,
 	GNLD_FEATURES_MAX
 };
 
@@ -230,7 +232,9 @@ struct vega10_registry_data {
 	uint8_t   cac_support;
 	uint8_t   clock_stretcher_support;
 	uint8_t   db_ramping_support;
+	uint8_t   didt_mode;
 	uint8_t   didt_support;
+	uint8_t   edc_didt_support;
 	uint8_t   dynamic_state_patching_support;
 	uint8_t   enable_pkg_pwr_tracking_feature;
 	uint8_t   enable_tdc_limit_feature;
@@ -263,6 +267,9 @@ struct vega10_registry_data {
 	uint8_t   tcp_ramping_support;
 	uint8_t   tdc_support;
 	uint8_t   td_ramping_support;
+	uint8_t   dbr_ramping_support;
+	uint8_t   gc_didt_support;
+	uint8_t   psm_didt_support;
 	uint8_t   thermal_out_gpio_support;
 	uint8_t   thermal_support;
 	uint8_t   fw_ctf_enabled;
@@ -381,6 +388,8 @@ struct vega10_hwmgr {
 	struct vega10_smc_state_table  smc_state_table;
 
 	uint32_t                       config_telemetry;
+	uint32_t                       smu_version;
+	uint32_t                       acg_loop_state;
 };
 
 #define VEGA10_DPM2_NEAR_TDP_DEC                      10
@@ -425,6 +434,10 @@ struct vega10_hwmgr {
 #define PPVEGA10_VEGA10UCLKCLKAVERAGEALPHA_DFLT      25 /* 10% * 255 = 25 */
 #define PPVEGA10_VEGA10GFXACTIVITYAVERAGEALPHA_DFLT  25 /* 10% * 255 = 25 */
 
+#define VEGA10_UMD_PSTATE_GFXCLK_LEVEL         0x3
+#define VEGA10_UMD_PSTATE_SOCCLK_LEVEL         0x3
+#define VEGA10_UMD_PSTATE_MCLK_LEVEL           0x2
+
 extern int tonga_initializa_dynamic_state_adjustment_rule_settings(struct pp_hwmgr *hwmgr);
 extern int tonga_hwmgr_backend_fini(struct pp_hwmgr *hwmgr);
 extern int tonga_get_mc_microcode_version (struct pp_hwmgr *hwmgr);
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_powertune.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_powertune.c
index 3f72268e99bb..e7fa67063cdc 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_powertune.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_powertune.c
@@ -26,7 +26,1298 @@
 #include "vega10_powertune.h"
 #include "vega10_smumgr.h"
 #include "vega10_ppsmc.h"
+#include "vega10_inc.h"
 #include "pp_debug.h"
+#include "pp_soc15.h"
+
+static const struct vega10_didt_config_reg SEDiDtTuningCtrlConfig_Vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                 Shift                                                  Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	/* DIDT_SQ */
+	{   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,        DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,        0x3853 },
+	{   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,        DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,        0x3153 },
+
+	/* DIDT_TD */
+	{   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,        DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,        0x0dde },
+	{   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,        DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,        0x0dde },
+
+	/* DIDT_TCP */
+	{   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,       DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,       0x3dde },
+	{   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,       DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,       0x3dde },
+
+	/* DIDT_DB */
+	{   ixDIDT_DB_TUNING_CTRL,             DIDT_DB_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,        DIDT_DB_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,        0x3dde },
+	{   ixDIDT_DB_TUNING_CTRL,             DIDT_DB_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,        DIDT_DB_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,        0x3dde },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+static const struct vega10_didt_config_reg SEDiDtCtrl3Config_vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset               Mask                                                     Shift                                                            Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	/*DIDT_SQ_CTRL3 */
+	{   ixDIDT_SQ_CTRL3,     DIDT_SQ_CTRL3__GC_DIDT_ENABLE_MASK,       DIDT_SQ_CTRL3__GC_DIDT_ENABLE__SHIFT,             0x0000 },
+	{   ixDIDT_SQ_CTRL3,     DIDT_SQ_CTRL3__GC_DIDT_CLK_EN_OVERRIDE_MASK,       DIDT_SQ_CTRL3__GC_DIDT_CLK_EN_OVERRIDE__SHIFT,             0x0000 },
+	{   ixDIDT_SQ_CTRL3,     DIDT_SQ_CTRL3__THROTTLE_POLICY_MASK,       DIDT_SQ_CTRL3__THROTTLE_POLICY__SHIFT,             0x0003 },
+	{   ixDIDT_SQ_CTRL3,     DIDT_SQ_CTRL3__DIDT_TRIGGER_THROTTLE_LOWBIT_MASK,       DIDT_SQ_CTRL3__DIDT_TRIGGER_THROTTLE_LOWBIT__SHIFT,             0x0000 },
+	{   ixDIDT_SQ_CTRL3,     DIDT_SQ_CTRL3__DIDT_POWER_LEVEL_LOWBIT_MASK,       DIDT_SQ_CTRL3__DIDT_POWER_LEVEL_LOWBIT__SHIFT,             0x0000 },
+	{   ixDIDT_SQ_CTRL3,     DIDT_SQ_CTRL3__DIDT_STALL_PATTERN_BIT_NUMS_MASK,       DIDT_SQ_CTRL3__DIDT_STALL_PATTERN_BIT_NUMS__SHIFT,             0x0003 },
+	{   ixDIDT_SQ_CTRL3,     DIDT_SQ_CTRL3__GC_DIDT_LEVEL_COMB_EN_MASK,       DIDT_SQ_CTRL3__GC_DIDT_LEVEL_COMB_EN__SHIFT,             0x0000 },
+	{   ixDIDT_SQ_CTRL3,     DIDT_SQ_CTRL3__SE_DIDT_LEVEL_COMB_EN_MASK,       DIDT_SQ_CTRL3__SE_DIDT_LEVEL_COMB_EN__SHIFT,             0x0000 },
+	{   ixDIDT_SQ_CTRL3,     DIDT_SQ_CTRL3__QUALIFY_STALL_EN_MASK,       DIDT_SQ_CTRL3__QUALIFY_STALL_EN__SHIFT,             0x0000 },
+	{   ixDIDT_SQ_CTRL3,     DIDT_SQ_CTRL3__DIDT_STALL_SEL_MASK,       DIDT_SQ_CTRL3__DIDT_STALL_SEL__SHIFT,             0x0000 },
+	{   ixDIDT_SQ_CTRL3,     DIDT_SQ_CTRL3__DIDT_FORCE_STALL_MASK,       DIDT_SQ_CTRL3__DIDT_FORCE_STALL__SHIFT,             0x0000 },
+	{   ixDIDT_SQ_CTRL3,     DIDT_SQ_CTRL3__DIDT_STALL_DELAY_EN_MASK,       DIDT_SQ_CTRL3__DIDT_STALL_DELAY_EN__SHIFT,             0x0000 },
+
+	/*DIDT_TCP_CTRL3 */
+	{   ixDIDT_TCP_CTRL3,    DIDT_TCP_CTRL3__GC_DIDT_ENABLE_MASK,      DIDT_TCP_CTRL3__GC_DIDT_ENABLE__SHIFT,            0x0000 },
+	{   ixDIDT_TCP_CTRL3,    DIDT_TCP_CTRL3__GC_DIDT_CLK_EN_OVERRIDE_MASK,      DIDT_TCP_CTRL3__GC_DIDT_CLK_EN_OVERRIDE__SHIFT,            0x0000 },
+	{   ixDIDT_TCP_CTRL3,    DIDT_TCP_CTRL3__THROTTLE_POLICY_MASK,      DIDT_TCP_CTRL3__THROTTLE_POLICY__SHIFT,            0x0003 },
+	{   ixDIDT_TCP_CTRL3,    DIDT_TCP_CTRL3__DIDT_TRIGGER_THROTTLE_LOWBIT_MASK,      DIDT_TCP_CTRL3__DIDT_TRIGGER_THROTTLE_LOWBIT__SHIFT,            0x0000 },
+	{   ixDIDT_TCP_CTRL3,    DIDT_TCP_CTRL3__DIDT_POWER_LEVEL_LOWBIT_MASK,      DIDT_TCP_CTRL3__DIDT_POWER_LEVEL_LOWBIT__SHIFT,            0x0000 },
+	{   ixDIDT_TCP_CTRL3,    DIDT_TCP_CTRL3__DIDT_STALL_PATTERN_BIT_NUMS_MASK,      DIDT_TCP_CTRL3__DIDT_STALL_PATTERN_BIT_NUMS__SHIFT,            0x0003 },
+	{   ixDIDT_TCP_CTRL3,    DIDT_TCP_CTRL3__GC_DIDT_LEVEL_COMB_EN_MASK,      DIDT_TCP_CTRL3__GC_DIDT_LEVEL_COMB_EN__SHIFT,            0x0000 },
+	{   ixDIDT_TCP_CTRL3,    DIDT_TCP_CTRL3__SE_DIDT_LEVEL_COMB_EN_MASK,      DIDT_TCP_CTRL3__SE_DIDT_LEVEL_COMB_EN__SHIFT,            0x0000 },
+	{   ixDIDT_TCP_CTRL3,    DIDT_TCP_CTRL3__QUALIFY_STALL_EN_MASK,      DIDT_TCP_CTRL3__QUALIFY_STALL_EN__SHIFT,            0x0000 },
+	{   ixDIDT_TCP_CTRL3,    DIDT_TCP_CTRL3__DIDT_STALL_SEL_MASK,      DIDT_TCP_CTRL3__DIDT_STALL_SEL__SHIFT,            0x0000 },
+	{   ixDIDT_TCP_CTRL3,    DIDT_TCP_CTRL3__DIDT_FORCE_STALL_MASK,      DIDT_TCP_CTRL3__DIDT_FORCE_STALL__SHIFT,            0x0000 },
+	{   ixDIDT_TCP_CTRL3,    DIDT_TCP_CTRL3__DIDT_STALL_DELAY_EN_MASK,      DIDT_TCP_CTRL3__DIDT_STALL_DELAY_EN__SHIFT,            0x0000 },
+
+	/*DIDT_TD_CTRL3 */
+	{   ixDIDT_TD_CTRL3,     DIDT_TD_CTRL3__GC_DIDT_ENABLE_MASK,       DIDT_TD_CTRL3__GC_DIDT_ENABLE__SHIFT,             0x0000 },
+	{   ixDIDT_TD_CTRL3,     DIDT_TD_CTRL3__GC_DIDT_CLK_EN_OVERRIDE_MASK,       DIDT_TD_CTRL3__GC_DIDT_CLK_EN_OVERRIDE__SHIFT,             0x0000 },
+	{   ixDIDT_TD_CTRL3,     DIDT_TD_CTRL3__THROTTLE_POLICY_MASK,       DIDT_TD_CTRL3__THROTTLE_POLICY__SHIFT,             0x0003 },
+	{   ixDIDT_TD_CTRL3,     DIDT_TD_CTRL3__DIDT_TRIGGER_THROTTLE_LOWBIT_MASK,       DIDT_TD_CTRL3__DIDT_TRIGGER_THROTTLE_LOWBIT__SHIFT,             0x0000 },
+	{   ixDIDT_TD_CTRL3,     DIDT_TD_CTRL3__DIDT_POWER_LEVEL_LOWBIT_MASK,       DIDT_TD_CTRL3__DIDT_POWER_LEVEL_LOWBIT__SHIFT,             0x0000 },
+	{   ixDIDT_TD_CTRL3,     DIDT_TD_CTRL3__DIDT_STALL_PATTERN_BIT_NUMS_MASK,       DIDT_TD_CTRL3__DIDT_STALL_PATTERN_BIT_NUMS__SHIFT,             0x0003 },
+	{   ixDIDT_TD_CTRL3,     DIDT_TD_CTRL3__GC_DIDT_LEVEL_COMB_EN_MASK,       DIDT_TD_CTRL3__GC_DIDT_LEVEL_COMB_EN__SHIFT,             0x0000 },
+	{   ixDIDT_TD_CTRL3,     DIDT_TD_CTRL3__SE_DIDT_LEVEL_COMB_EN_MASK,       DIDT_TD_CTRL3__SE_DIDT_LEVEL_COMB_EN__SHIFT,             0x0000 },
+	{   ixDIDT_TD_CTRL3,     DIDT_TD_CTRL3__QUALIFY_STALL_EN_MASK,       DIDT_TD_CTRL3__QUALIFY_STALL_EN__SHIFT,             0x0000 },
+	{   ixDIDT_TD_CTRL3,     DIDT_TD_CTRL3__DIDT_STALL_SEL_MASK,       DIDT_TD_CTRL3__DIDT_STALL_SEL__SHIFT,             0x0000 },
+	{   ixDIDT_TD_CTRL3,     DIDT_TD_CTRL3__DIDT_FORCE_STALL_MASK,       DIDT_TD_CTRL3__DIDT_FORCE_STALL__SHIFT,             0x0000 },
+	{   ixDIDT_TD_CTRL3,     DIDT_TD_CTRL3__DIDT_STALL_DELAY_EN_MASK,       DIDT_TD_CTRL3__DIDT_STALL_DELAY_EN__SHIFT,             0x0000 },
+
+	/*DIDT_DB_CTRL3 */
+	{   ixDIDT_DB_CTRL3,     DIDT_DB_CTRL3__GC_DIDT_ENABLE_MASK,       DIDT_DB_CTRL3__GC_DIDT_ENABLE__SHIFT,             0x0000 },
+	{   ixDIDT_DB_CTRL3,     DIDT_DB_CTRL3__GC_DIDT_CLK_EN_OVERRIDE_MASK,       DIDT_DB_CTRL3__GC_DIDT_CLK_EN_OVERRIDE__SHIFT,             0x0000 },
+	{   ixDIDT_DB_CTRL3,     DIDT_DB_CTRL3__THROTTLE_POLICY_MASK,       DIDT_DB_CTRL3__THROTTLE_POLICY__SHIFT,             0x0003 },
+	{   ixDIDT_DB_CTRL3,     DIDT_DB_CTRL3__DIDT_TRIGGER_THROTTLE_LOWBIT_MASK,       DIDT_DB_CTRL3__DIDT_TRIGGER_THROTTLE_LOWBIT__SHIFT,             0x0000 },
+	{   ixDIDT_DB_CTRL3,     DIDT_DB_CTRL3__DIDT_POWER_LEVEL_LOWBIT_MASK,       DIDT_DB_CTRL3__DIDT_POWER_LEVEL_LOWBIT__SHIFT,             0x0000 },
+	{   ixDIDT_DB_CTRL3,     DIDT_DB_CTRL3__DIDT_STALL_PATTERN_BIT_NUMS_MASK,       DIDT_DB_CTRL3__DIDT_STALL_PATTERN_BIT_NUMS__SHIFT,             0x0003 },
+	{   ixDIDT_DB_CTRL3,     DIDT_DB_CTRL3__GC_DIDT_LEVEL_COMB_EN_MASK,       DIDT_DB_CTRL3__GC_DIDT_LEVEL_COMB_EN__SHIFT,             0x0000 },
+	{   ixDIDT_DB_CTRL3,     DIDT_DB_CTRL3__SE_DIDT_LEVEL_COMB_EN_MASK,       DIDT_DB_CTRL3__SE_DIDT_LEVEL_COMB_EN__SHIFT,             0x0000 },
+	{   ixDIDT_DB_CTRL3,     DIDT_DB_CTRL3__QUALIFY_STALL_EN_MASK,       DIDT_DB_CTRL3__QUALIFY_STALL_EN__SHIFT,             0x0000 },
+	{   ixDIDT_DB_CTRL3,     DIDT_DB_CTRL3__DIDT_STALL_SEL_MASK,       DIDT_DB_CTRL3__DIDT_STALL_SEL__SHIFT,             0x0000 },
+	{   ixDIDT_DB_CTRL3,     DIDT_DB_CTRL3__DIDT_FORCE_STALL_MASK,       DIDT_DB_CTRL3__DIDT_FORCE_STALL__SHIFT,             0x0000 },
+	{   ixDIDT_DB_CTRL3,     DIDT_DB_CTRL3__DIDT_STALL_DELAY_EN_MASK,       DIDT_DB_CTRL3__DIDT_STALL_DELAY_EN__SHIFT,             0x0000 },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+static const struct vega10_didt_config_reg SEDiDtCtrl2Config_Vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                            Mask                                                 Shift                                                  Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	/* DIDT_SQ */
+	{   ixDIDT_SQ_CTRL2,                  DIDT_SQ_CTRL2__MAX_POWER_DELTA_MASK,                 DIDT_SQ_CTRL2__MAX_POWER_DELTA__SHIFT,                 0x3853 },
+	{   ixDIDT_SQ_CTRL2,                  DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,        DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,        0x00c0 },
+	{   ixDIDT_SQ_CTRL2,                  DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,        DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,        0x0000 },
+
+	/* DIDT_TD */
+	{   ixDIDT_TD_CTRL2,                  DIDT_TD_CTRL2__MAX_POWER_DELTA_MASK,                 DIDT_TD_CTRL2__MAX_POWER_DELTA__SHIFT,                 0x3fff },
+	{   ixDIDT_TD_CTRL2,                  DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,        DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,        0x00c0 },
+	{   ixDIDT_TD_CTRL2,                  DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,        DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,        0x0001 },
+
+	/* DIDT_TCP */
+	{   ixDIDT_TCP_CTRL2,                 DIDT_TCP_CTRL2__MAX_POWER_DELTA_MASK,                DIDT_TCP_CTRL2__MAX_POWER_DELTA__SHIFT,                0x3dde },
+	{   ixDIDT_TCP_CTRL2,                 DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,       DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,       0x00c0 },
+	{   ixDIDT_TCP_CTRL2,                 DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,       DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,       0x0001 },
+
+	/* DIDT_DB */
+	{   ixDIDT_DB_CTRL2,                  DIDT_DB_CTRL2__MAX_POWER_DELTA_MASK,                 DIDT_DB_CTRL2__MAX_POWER_DELTA__SHIFT,                 0x3dde },
+	{   ixDIDT_DB_CTRL2,                  DIDT_DB_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,        DIDT_DB_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,        0x00c0 },
+	{   ixDIDT_DB_CTRL2,                  DIDT_DB_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,        DIDT_DB_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,        0x0001 },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+static const struct vega10_didt_config_reg SEDiDtCtrl1Config_Vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                 Shift                                                  Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	/* DIDT_SQ */
+	{   ixDIDT_SQ_CTRL1,                   DIDT_SQ_CTRL1__MIN_POWER_MASK,                       DIDT_SQ_CTRL1__MIN_POWER__SHIFT,                       0x0000 },
+	{   ixDIDT_SQ_CTRL1,                   DIDT_SQ_CTRL1__MAX_POWER_MASK,                       DIDT_SQ_CTRL1__MAX_POWER__SHIFT,                       0xffff },
+	/* DIDT_TD */
+	{   ixDIDT_TD_CTRL1,                   DIDT_TD_CTRL1__MIN_POWER_MASK,                       DIDT_TD_CTRL1__MIN_POWER__SHIFT,                       0x0000 },
+	{   ixDIDT_TD_CTRL1,                   DIDT_TD_CTRL1__MAX_POWER_MASK,                       DIDT_TD_CTRL1__MAX_POWER__SHIFT,                       0xffff },
+	/* DIDT_TCP */
+	{   ixDIDT_TCP_CTRL1,                  DIDT_TCP_CTRL1__MIN_POWER_MASK,                      DIDT_TCP_CTRL1__MIN_POWER__SHIFT,                      0x0000 },
+	{   ixDIDT_TCP_CTRL1,                  DIDT_TCP_CTRL1__MAX_POWER_MASK,                      DIDT_TCP_CTRL1__MAX_POWER__SHIFT,                      0xffff },
+	/* DIDT_DB */
+	{   ixDIDT_DB_CTRL1,                   DIDT_DB_CTRL1__MIN_POWER_MASK,                       DIDT_DB_CTRL1__MIN_POWER__SHIFT,                       0x0000 },
+	{   ixDIDT_DB_CTRL1,                   DIDT_DB_CTRL1__MAX_POWER_MASK,                       DIDT_DB_CTRL1__MAX_POWER__SHIFT,                       0xffff },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+
+static const struct vega10_didt_config_reg SEDiDtWeightConfig_Vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                  Shift                                                 Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	/* DIDT_SQ */
+	{   ixDIDT_SQ_WEIGHT0_3,               0xFFFFFFFF,                                           0,                                                    0x2B363B1A },
+	{   ixDIDT_SQ_WEIGHT4_7,               0xFFFFFFFF,                                           0,                                                    0x270B2432 },
+	{   ixDIDT_SQ_WEIGHT8_11,              0xFFFFFFFF,                                           0,                                                    0x00000018 },
+
+	/* DIDT_TD */
+	{   ixDIDT_TD_WEIGHT0_3,               0xFFFFFFFF,                                           0,                                                    0x2B1D220F },
+	{   ixDIDT_TD_WEIGHT4_7,               0xFFFFFFFF,                                           0,                                                    0x00007558 },
+	{   ixDIDT_TD_WEIGHT8_11,              0xFFFFFFFF,                                           0,                                                    0x00000000 },
+
+	/* DIDT_TCP */
+	{   ixDIDT_TCP_WEIGHT0_3,               0xFFFFFFFF,                                          0,                                                    0x5ACE160D },
+	{   ixDIDT_TCP_WEIGHT4_7,               0xFFFFFFFF,                                          0,                                                    0x00000000 },
+	{   ixDIDT_TCP_WEIGHT8_11,              0xFFFFFFFF,                                          0,                                                    0x00000000 },
+
+	/* DIDT_DB */
+	{   ixDIDT_DB_WEIGHT0_3,                0xFFFFFFFF,                                          0,                                                    0x0E152A0F },
+	{   ixDIDT_DB_WEIGHT4_7,                0xFFFFFFFF,                                          0,                                                    0x09061813 },
+	{   ixDIDT_DB_WEIGHT8_11,               0xFFFFFFFF,                                          0,                                                    0x00000013 },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+static const struct vega10_didt_config_reg SEDiDtCtrl0Config_Vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                 Shift                                                  Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	/* DIDT_SQ */
+	{  ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK,   DIDT_SQ_CTRL0__DIDT_CTRL_EN__SHIFT,  0x0000 },
+	{  ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__PHASE_OFFSET_MASK,   DIDT_SQ_CTRL0__PHASE_OFFSET__SHIFT,  0x0000 },
+	{  ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CTRL_RST_MASK,   DIDT_SQ_CTRL0__DIDT_CTRL_RST__SHIFT,  0x0000 },
+	{  ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,   DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,  0x0000 },
+	{  ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_STALL_CTRL_EN_MASK,   DIDT_SQ_CTRL0__DIDT_STALL_CTRL_EN__SHIFT,  0x0001 },
+	{  ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_TUNING_CTRL_EN_MASK,   DIDT_SQ_CTRL0__DIDT_TUNING_CTRL_EN__SHIFT,  0x0001 },
+	{  ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_STALL_AUTO_RELEASE_EN_MASK,   DIDT_SQ_CTRL0__DIDT_STALL_AUTO_RELEASE_EN__SHIFT,  0x0001 },
+	{  ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_HI_POWER_THRESHOLD_MASK,   DIDT_SQ_CTRL0__DIDT_HI_POWER_THRESHOLD__SHIFT,  0xffff },
+	{  ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_AUTO_MPD_EN_MASK,   DIDT_SQ_CTRL0__DIDT_AUTO_MPD_EN__SHIFT,  0x0000 },
+	{  ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_STALL_EVENT_EN_MASK,   DIDT_SQ_CTRL0__DIDT_STALL_EVENT_EN__SHIFT,  0x0000 },
+	{  ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_STALL_EVENT_COUNTER_CLEAR_MASK,   DIDT_SQ_CTRL0__DIDT_STALL_EVENT_COUNTER_CLEAR__SHIFT,  0x0000 },
+	/* DIDT_TD */
+	{  ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK,   DIDT_TD_CTRL0__DIDT_CTRL_EN__SHIFT,  0x0000 },
+	{  ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__PHASE_OFFSET_MASK,   DIDT_TD_CTRL0__PHASE_OFFSET__SHIFT,  0x0000 },
+	{  ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CTRL_RST_MASK,   DIDT_TD_CTRL0__DIDT_CTRL_RST__SHIFT,  0x0000 },
+	{  ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,   DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,  0x0000 },
+	{  ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_STALL_CTRL_EN_MASK,   DIDT_TD_CTRL0__DIDT_STALL_CTRL_EN__SHIFT,  0x0001 },
+	{  ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_TUNING_CTRL_EN_MASK,   DIDT_TD_CTRL0__DIDT_TUNING_CTRL_EN__SHIFT,  0x0001 },
+	{  ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_STALL_AUTO_RELEASE_EN_MASK,   DIDT_TD_CTRL0__DIDT_STALL_AUTO_RELEASE_EN__SHIFT,  0x0001 },
+	{  ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_HI_POWER_THRESHOLD_MASK,   DIDT_TD_CTRL0__DIDT_HI_POWER_THRESHOLD__SHIFT,  0xffff },
+	{  ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_AUTO_MPD_EN_MASK,   DIDT_TD_CTRL0__DIDT_AUTO_MPD_EN__SHIFT,  0x0000 },
+	{  ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_STALL_EVENT_EN_MASK,   DIDT_TD_CTRL0__DIDT_STALL_EVENT_EN__SHIFT,  0x0000 },
+	{  ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_STALL_EVENT_COUNTER_CLEAR_MASK,   DIDT_TD_CTRL0__DIDT_STALL_EVENT_COUNTER_CLEAR__SHIFT,  0x0000 },
+	/* DIDT_TCP */
+	{  ixDIDT_TCP_CTRL0,                  DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK,  DIDT_TCP_CTRL0__DIDT_CTRL_EN__SHIFT, 0x0000 },
+	{  ixDIDT_TCP_CTRL0,                  DIDT_TCP_CTRL0__PHASE_OFFSET_MASK,  DIDT_TCP_CTRL0__PHASE_OFFSET__SHIFT, 0x0000 },
+	{  ixDIDT_TCP_CTRL0,                  DIDT_TCP_CTRL0__DIDT_CTRL_RST_MASK,  DIDT_TCP_CTRL0__DIDT_CTRL_RST__SHIFT, 0x0000 },
+	{  ixDIDT_TCP_CTRL0,                  DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,  DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT, 0x0000 },
+	{  ixDIDT_TCP_CTRL0,                  DIDT_TCP_CTRL0__DIDT_STALL_CTRL_EN_MASK,  DIDT_TCP_CTRL0__DIDT_STALL_CTRL_EN__SHIFT, 0x0001 },
+	{  ixDIDT_TCP_CTRL0,                  DIDT_TCP_CTRL0__DIDT_TUNING_CTRL_EN_MASK,  DIDT_TCP_CTRL0__DIDT_TUNING_CTRL_EN__SHIFT, 0x0001 },
+	{  ixDIDT_TCP_CTRL0,                  DIDT_TCP_CTRL0__DIDT_STALL_AUTO_RELEASE_EN_MASK,  DIDT_TCP_CTRL0__DIDT_STALL_AUTO_RELEASE_EN__SHIFT, 0x0001 },
+	{  ixDIDT_TCP_CTRL0,                  DIDT_TCP_CTRL0__DIDT_HI_POWER_THRESHOLD_MASK,  DIDT_TCP_CTRL0__DIDT_HI_POWER_THRESHOLD__SHIFT, 0xffff },
+	{  ixDIDT_TCP_CTRL0,                  DIDT_TCP_CTRL0__DIDT_AUTO_MPD_EN_MASK,  DIDT_TCP_CTRL0__DIDT_AUTO_MPD_EN__SHIFT, 0x0000 },
+	{  ixDIDT_TCP_CTRL0,                  DIDT_TCP_CTRL0__DIDT_STALL_EVENT_EN_MASK,  DIDT_TCP_CTRL0__DIDT_STALL_EVENT_EN__SHIFT, 0x0000 },
+	{  ixDIDT_TCP_CTRL0,                  DIDT_TCP_CTRL0__DIDT_STALL_EVENT_COUNTER_CLEAR_MASK,  DIDT_TCP_CTRL0__DIDT_STALL_EVENT_COUNTER_CLEAR__SHIFT, 0x0000 },
+	/* DIDT_DB */
+	{  ixDIDT_DB_CTRL0,                   DIDT_DB_CTRL0__DIDT_CTRL_EN_MASK,   DIDT_DB_CTRL0__DIDT_CTRL_EN__SHIFT,  0x0000 },
+	{  ixDIDT_DB_CTRL0,                   DIDT_DB_CTRL0__PHASE_OFFSET_MASK,   DIDT_DB_CTRL0__PHASE_OFFSET__SHIFT,  0x0000 },
+	{  ixDIDT_DB_CTRL0,                   DIDT_DB_CTRL0__DIDT_CTRL_RST_MASK,   DIDT_DB_CTRL0__DIDT_CTRL_RST__SHIFT,  0x0000 },
+	{  ixDIDT_DB_CTRL0,                   DIDT_DB_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,   DIDT_DB_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,  0x0000 },
+	{  ixDIDT_DB_CTRL0,                   DIDT_DB_CTRL0__DIDT_STALL_CTRL_EN_MASK,   DIDT_DB_CTRL0__DIDT_STALL_CTRL_EN__SHIFT,  0x0001 },
+	{  ixDIDT_DB_CTRL0,                   DIDT_DB_CTRL0__DIDT_TUNING_CTRL_EN_MASK,   DIDT_DB_CTRL0__DIDT_TUNING_CTRL_EN__SHIFT,  0x0001 },
+	{  ixDIDT_DB_CTRL0,                   DIDT_DB_CTRL0__DIDT_STALL_AUTO_RELEASE_EN_MASK,   DIDT_DB_CTRL0__DIDT_STALL_AUTO_RELEASE_EN__SHIFT,  0x0001 },
+	{  ixDIDT_DB_CTRL0,                   DIDT_DB_CTRL0__DIDT_HI_POWER_THRESHOLD_MASK,   DIDT_DB_CTRL0__DIDT_HI_POWER_THRESHOLD__SHIFT,  0xffff },
+	{  ixDIDT_DB_CTRL0,                   DIDT_DB_CTRL0__DIDT_AUTO_MPD_EN_MASK,   DIDT_DB_CTRL0__DIDT_AUTO_MPD_EN__SHIFT,  0x0000 },
+	{  ixDIDT_DB_CTRL0,                   DIDT_DB_CTRL0__DIDT_STALL_EVENT_EN_MASK,   DIDT_DB_CTRL0__DIDT_STALL_EVENT_EN__SHIFT,  0x0000 },
+	{  ixDIDT_DB_CTRL0,                   DIDT_DB_CTRL0__DIDT_STALL_EVENT_COUNTER_CLEAR_MASK,   DIDT_DB_CTRL0__DIDT_STALL_EVENT_COUNTER_CLEAR__SHIFT,  0x0000 },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+
+static const struct vega10_didt_config_reg SEDiDtStallCtrlConfig_vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                   Mask                                                     Shift                                                      Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	/* DIDT_SQ */
+	{   ixDIDT_SQ_STALL_CTRL,    DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,    DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,     0x0004 },
+	{   ixDIDT_SQ_STALL_CTRL,    DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,    DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,     0x0004 },
+	{   ixDIDT_SQ_STALL_CTRL,    DIDT_SQ_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_HI_MASK,    DIDT_SQ_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,     0x000a },
+	{   ixDIDT_SQ_STALL_CTRL,    DIDT_SQ_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_LO_MASK,    DIDT_SQ_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,     0x000a },
+
+	/* DIDT_TD */
+	{   ixDIDT_TD_STALL_CTRL,    DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,    DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,     0x0001 },
+	{   ixDIDT_TD_STALL_CTRL,    DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,    DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,     0x0001 },
+	{   ixDIDT_TD_STALL_CTRL,    DIDT_TD_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_HI_MASK,    DIDT_TD_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,     0x000a },
+	{   ixDIDT_TD_STALL_CTRL,    DIDT_TD_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_LO_MASK,    DIDT_TD_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,     0x000a },
+
+	/* DIDT_TCP */
+	{   ixDIDT_TCP_STALL_CTRL,   DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,   DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,    0x0001 },
+	{   ixDIDT_TCP_STALL_CTRL,   DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,   DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,    0x0001 },
+	{   ixDIDT_TCP_STALL_CTRL,   DIDT_TCP_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_HI_MASK,   DIDT_TCP_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,    0x000a },
+	{   ixDIDT_TCP_STALL_CTRL,   DIDT_TCP_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_LO_MASK,   DIDT_TCP_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,    0x000a },
+
+	/* DIDT_DB */
+	{   ixDIDT_DB_STALL_CTRL,    DIDT_DB_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,    DIDT_DB_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,     0x0004 },
+	{   ixDIDT_DB_STALL_CTRL,    DIDT_DB_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,    DIDT_DB_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,     0x0004 },
+	{   ixDIDT_DB_STALL_CTRL,    DIDT_DB_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_HI_MASK,    DIDT_DB_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,     0x000a },
+	{   ixDIDT_DB_STALL_CTRL,    DIDT_DB_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_LO_MASK,    DIDT_DB_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,     0x000a },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+static const struct vega10_didt_config_reg SEDiDtStallPatternConfig_vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                        Mask                                                      Shift                                                    Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	/* DIDT_SQ_STALL_PATTERN_1_2 */
+	{   ixDIDT_SQ_STALL_PATTERN_1_2,  DIDT_SQ_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_1_MASK,    DIDT_SQ_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_1__SHIFT,  0x0001 },
+	{   ixDIDT_SQ_STALL_PATTERN_1_2,  DIDT_SQ_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_2_MASK,    DIDT_SQ_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_2__SHIFT,  0x0001 },
+
+	/* DIDT_SQ_STALL_PATTERN_3_4 */
+	{   ixDIDT_SQ_STALL_PATTERN_3_4,  DIDT_SQ_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_3_MASK,    DIDT_SQ_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_3__SHIFT,  0x0001 },
+	{   ixDIDT_SQ_STALL_PATTERN_3_4,  DIDT_SQ_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_4_MASK,    DIDT_SQ_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_4__SHIFT,  0x0001 },
+
+	/* DIDT_SQ_STALL_PATTERN_5_6 */
+	{   ixDIDT_SQ_STALL_PATTERN_5_6,  DIDT_SQ_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_5_MASK,    DIDT_SQ_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_5__SHIFT,  0x0000 },
+	{   ixDIDT_SQ_STALL_PATTERN_5_6,  DIDT_SQ_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_6_MASK,    DIDT_SQ_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_6__SHIFT,  0x0000 },
+
+	/* DIDT_SQ_STALL_PATTERN_7 */
+	{   ixDIDT_SQ_STALL_PATTERN_7,    DIDT_SQ_STALL_PATTERN_7__DIDT_STALL_PATTERN_7_MASK,      DIDT_SQ_STALL_PATTERN_7__DIDT_STALL_PATTERN_7__SHIFT,    0x0000 },
+
+	/* DIDT_TCP_STALL_PATTERN_1_2 */
+	{   ixDIDT_TCP_STALL_PATTERN_1_2, DIDT_TCP_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_1_MASK,   DIDT_TCP_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_1__SHIFT, 0x0001 },
+	{   ixDIDT_TCP_STALL_PATTERN_1_2, DIDT_TCP_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_2_MASK,   DIDT_TCP_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_2__SHIFT, 0x0001 },
+
+	/* DIDT_TCP_STALL_PATTERN_3_4 */
+	{   ixDIDT_TCP_STALL_PATTERN_3_4, DIDT_TCP_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_3_MASK,   DIDT_TCP_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_3__SHIFT, 0x0001 },
+	{   ixDIDT_TCP_STALL_PATTERN_3_4, DIDT_TCP_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_4_MASK,   DIDT_TCP_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_4__SHIFT, 0x0001 },
+
+	/* DIDT_TCP_STALL_PATTERN_5_6 */
+	{   ixDIDT_TCP_STALL_PATTERN_5_6, DIDT_TCP_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_5_MASK,   DIDT_TCP_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_5__SHIFT, 0x0000 },
+	{   ixDIDT_TCP_STALL_PATTERN_5_6, DIDT_TCP_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_6_MASK,   DIDT_TCP_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_6__SHIFT, 0x0000 },
+
+	/* DIDT_TCP_STALL_PATTERN_7 */
+	{   ixDIDT_TCP_STALL_PATTERN_7,   DIDT_TCP_STALL_PATTERN_7__DIDT_STALL_PATTERN_7_MASK,     DIDT_TCP_STALL_PATTERN_7__DIDT_STALL_PATTERN_7__SHIFT,   0x0000 },
+
+	/* DIDT_TD_STALL_PATTERN_1_2 */
+	{   ixDIDT_TD_STALL_PATTERN_1_2,  DIDT_TD_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_1_MASK,    DIDT_TD_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_1__SHIFT,  0x0001 },
+	{   ixDIDT_TD_STALL_PATTERN_1_2,  DIDT_TD_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_2_MASK,    DIDT_TD_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_2__SHIFT,  0x0001 },
+
+	/* DIDT_TD_STALL_PATTERN_3_4 */
+	{   ixDIDT_TD_STALL_PATTERN_3_4,  DIDT_TD_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_3_MASK,    DIDT_TD_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_3__SHIFT,  0x0001 },
+	{   ixDIDT_TD_STALL_PATTERN_3_4,  DIDT_TD_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_4_MASK,    DIDT_TD_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_4__SHIFT,  0x0001 },
+
+	/* DIDT_TD_STALL_PATTERN_5_6 */
+	{   ixDIDT_TD_STALL_PATTERN_5_6,  DIDT_TD_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_5_MASK,    DIDT_TD_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_5__SHIFT,  0x0000 },
+	{   ixDIDT_TD_STALL_PATTERN_5_6,  DIDT_TD_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_6_MASK,    DIDT_TD_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_6__SHIFT,  0x0000 },
+
+	/* DIDT_TD_STALL_PATTERN_7 */
+	{   ixDIDT_TD_STALL_PATTERN_7,    DIDT_TD_STALL_PATTERN_7__DIDT_STALL_PATTERN_7_MASK,      DIDT_TD_STALL_PATTERN_7__DIDT_STALL_PATTERN_7__SHIFT,    0x0000 },
+
+	/* DIDT_DB_STALL_PATTERN_1_2 */
+	{   ixDIDT_DB_STALL_PATTERN_1_2,  DIDT_DB_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_1_MASK,    DIDT_DB_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_1__SHIFT,  0x0001 },
+	{   ixDIDT_DB_STALL_PATTERN_1_2,  DIDT_DB_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_2_MASK,    DIDT_DB_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_2__SHIFT,  0x0001 },
+
+	/* DIDT_DB_STALL_PATTERN_3_4 */
+	{   ixDIDT_DB_STALL_PATTERN_3_4,  DIDT_DB_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_3_MASK,    DIDT_DB_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_3__SHIFT,  0x0001 },
+	{   ixDIDT_DB_STALL_PATTERN_3_4,  DIDT_DB_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_4_MASK,    DIDT_DB_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_4__SHIFT,  0x0001 },
+
+	/* DIDT_DB_STALL_PATTERN_5_6 */
+	{   ixDIDT_DB_STALL_PATTERN_5_6,  DIDT_DB_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_5_MASK,    DIDT_DB_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_5__SHIFT,  0x0000 },
+	{   ixDIDT_DB_STALL_PATTERN_5_6,  DIDT_DB_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_6_MASK,    DIDT_DB_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_6__SHIFT,  0x0000 },
+
+	/* DIDT_DB_STALL_PATTERN_7 */
+	{   ixDIDT_DB_STALL_PATTERN_7,    DIDT_DB_STALL_PATTERN_7__DIDT_STALL_PATTERN_7_MASK,      DIDT_DB_STALL_PATTERN_7__DIDT_STALL_PATTERN_7__SHIFT,    0x0000 },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+static const struct vega10_didt_config_reg SELCacConfig_Vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                 Shift                                                  Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	/* SQ */
+	{   ixSE_CAC_CNTL,                     0xFFFFFFFF,                                          0,                                                     0x00060021 },
+	{   ixSE_CAC_CNTL,                     0xFFFFFFFF,                                          0,                                                     0x00860021 },
+	{   ixSE_CAC_CNTL,                     0xFFFFFFFF,                                          0,                                                     0x01060021 },
+	{   ixSE_CAC_CNTL,                     0xFFFFFFFF,                                          0,                                                     0x01860021 },
+	{   ixSE_CAC_CNTL,                     0xFFFFFFFF,                                          0,                                                     0x02060021 },
+	{   ixSE_CAC_CNTL,                     0xFFFFFFFF,                                          0,                                                     0x02860021 },
+	{   ixSE_CAC_CNTL,                     0xFFFFFFFF,                                          0,                                                     0x03060021 },
+	{   ixSE_CAC_CNTL,                     0xFFFFFFFF,                                          0,                                                     0x03860021 },
+	{   ixSE_CAC_CNTL,                     0xFFFFFFFF,                                          0,                                                     0x04060021 },
+	/* TD */
+	{   ixSE_CAC_CNTL,                     0xFFFFFFFF,                                          0,                                                     0x000E0020 },
+	{   ixSE_CAC_CNTL,                     0xFFFFFFFF,                                          0,                                                     0x008E0020 },
+	{   ixSE_CAC_CNTL,                     0xFFFFFFFF,                                          0,                                                     0x010E0020 },
+	{   ixSE_CAC_CNTL,                     0xFFFFFFFF,                                          0,                                                     0x018E0020 },
+	{   ixSE_CAC_CNTL,                     0xFFFFFFFF,                                          0,                                                     0x020E0020 },
+	{   ixSE_CAC_CNTL,                     0xFFFFFFFF,                                          0,                                                     0x028E0020 },
+	/* TCP */
+	{   ixSE_CAC_CNTL,                     0xFFFFFFFF,                                          0,                                                     0x001c0020 },
+	{   ixSE_CAC_CNTL,                     0xFFFFFFFF,                                          0,                                                     0x009c0020 },
+	{   ixSE_CAC_CNTL,                     0xFFFFFFFF,                                          0,                                                     0x011c0020 },
+	{   ixSE_CAC_CNTL,                     0xFFFFFFFF,                                          0,                                                     0x019c0020 },
+	{   ixSE_CAC_CNTL,                     0xFFFFFFFF,                                          0,                                                     0x021c0020 },
+	/* DB */
+	{   ixSE_CAC_CNTL,                     0xFFFFFFFF,                                          0,                                                     0x00200008 },
+	{   ixSE_CAC_CNTL,                     0xFFFFFFFF,                                          0,                                                     0x00820008 },
+	{   ixSE_CAC_CNTL,                     0xFFFFFFFF,                                          0,                                                     0x01020008 },
+	{   ixSE_CAC_CNTL,                     0xFFFFFFFF,                                          0,                                                     0x01820008 },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+
+static const struct vega10_didt_config_reg SEEDCStallPatternConfig_Vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                 Shift                                                  Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	/* SQ */
+	{   ixDIDT_SQ_EDC_STALL_PATTERN_1_2,   0xFFFFFFFF,                                          0,                                                     0x00030001 },
+	{   ixDIDT_SQ_EDC_STALL_PATTERN_3_4,   0xFFFFFFFF,                                          0,                                                     0x000F0007 },
+	{   ixDIDT_SQ_EDC_STALL_PATTERN_5_6,   0xFFFFFFFF,                                          0,                                                     0x003F001F },
+	{   ixDIDT_SQ_EDC_STALL_PATTERN_7,     0xFFFFFFFF,                                          0,                                                     0x0000007F },
+	/* TD */
+	{   ixDIDT_TD_EDC_STALL_PATTERN_1_2,   0xFFFFFFFF,                                          0,                                                     0x00000000 },
+	{   ixDIDT_TD_EDC_STALL_PATTERN_3_4,   0xFFFFFFFF,                                          0,                                                     0x00000000 },
+	{   ixDIDT_TD_EDC_STALL_PATTERN_5_6,   0xFFFFFFFF,                                          0,                                                     0x00000000 },
+	{   ixDIDT_TD_EDC_STALL_PATTERN_7,     0xFFFFFFFF,                                          0,                                                     0x00000000 },
+	/* TCP */
+	{   ixDIDT_TCP_EDC_STALL_PATTERN_1_2,   0xFFFFFFFF,                                         0,                                                     0x00000000 },
+	{   ixDIDT_TCP_EDC_STALL_PATTERN_3_4,   0xFFFFFFFF,                                         0,                                                     0x00000000 },
+	{   ixDIDT_TCP_EDC_STALL_PATTERN_5_6,   0xFFFFFFFF,                                         0,                                                     0x00000000 },
+	{   ixDIDT_TCP_EDC_STALL_PATTERN_7,     0xFFFFFFFF,                                         0,                                                     0x00000000 },
+	/* DB */
+	{   ixDIDT_DB_EDC_STALL_PATTERN_1_2,   0xFFFFFFFF,                                          0,                                                     0x00000000 },
+	{   ixDIDT_DB_EDC_STALL_PATTERN_3_4,   0xFFFFFFFF,                                          0,                                                     0x00000000 },
+	{   ixDIDT_DB_EDC_STALL_PATTERN_5_6,   0xFFFFFFFF,                                          0,                                                     0x00000000 },
+	{   ixDIDT_DB_EDC_STALL_PATTERN_7,     0xFFFFFFFF,                                          0,                                                     0x00000000 },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+static const struct vega10_didt_config_reg SEEDCForceStallPatternConfig_Vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                 Shift                                                  Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	/* SQ */
+	{   ixDIDT_SQ_EDC_STALL_PATTERN_1_2,   0xFFFFFFFF,                                          0,                                                     0x00000015 },
+	{   ixDIDT_SQ_EDC_STALL_PATTERN_3_4,   0xFFFFFFFF,                                          0,                                                     0x00000000 },
+	{   ixDIDT_SQ_EDC_STALL_PATTERN_5_6,   0xFFFFFFFF,                                          0,                                                     0x00000000 },
+	{   ixDIDT_SQ_EDC_STALL_PATTERN_7,     0xFFFFFFFF,                                          0,                                                     0x00000000 },
+	/* TD */
+	{   ixDIDT_TD_EDC_STALL_PATTERN_1_2,   0xFFFFFFFF,                                          0,                                                     0x00000015 },
+	{   ixDIDT_TD_EDC_STALL_PATTERN_3_4,   0xFFFFFFFF,                                          0,                                                     0x00000000 },
+	{   ixDIDT_TD_EDC_STALL_PATTERN_5_6,   0xFFFFFFFF,                                          0,                                                     0x00000000 },
+	{   ixDIDT_TD_EDC_STALL_PATTERN_7,     0xFFFFFFFF,                                          0,                                                     0x00000000 },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+static const struct vega10_didt_config_reg SEEDCStallDelayConfig_Vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                 Shift                                                  Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	/* SQ */
+	{   ixDIDT_SQ_EDC_STALL_DELAY_1,       0xFFFFFFFF,                                          0,                                                     0x00000000 },
+	{   ixDIDT_SQ_EDC_STALL_DELAY_2,       0xFFFFFFFF,                                          0,                                                     0x00000000 },
+	{   ixDIDT_SQ_EDC_STALL_DELAY_3,       0xFFFFFFFF,                                          0,                                                     0x00000000 },
+	{   ixDIDT_SQ_EDC_STALL_DELAY_4,       0xFFFFFFFF,                                          0,                                                     0x00000000 },
+	/* TD */
+	{   ixDIDT_TD_EDC_STALL_DELAY_1,       0xFFFFFFFF,                                          0,                                                     0x00000000 },
+	{   ixDIDT_TD_EDC_STALL_DELAY_2,       0xFFFFFFFF,                                          0,                                                     0x00000000 },
+	{   ixDIDT_TD_EDC_STALL_DELAY_3,       0xFFFFFFFF,                                          0,                                                     0x00000000 },
+	{   ixDIDT_TD_EDC_STALL_DELAY_4,       0xFFFFFFFF,                                          0,                                                     0x00000000 },
+	/* TCP */
+	{   ixDIDT_TCP_EDC_STALL_DELAY_1,      0xFFFFFFFF,                                          0,                                                     0x00000000 },
+	{   ixDIDT_TCP_EDC_STALL_DELAY_2,      0xFFFFFFFF,                                          0,                                                     0x00000000 },
+	{   ixDIDT_TCP_EDC_STALL_DELAY_3,      0xFFFFFFFF,                                          0,                                                     0x00000000 },
+	{   ixDIDT_TCP_EDC_STALL_DELAY_4,      0xFFFFFFFF,                                          0,                                                     0x00000000 },
+	/* DB */
+	{   ixDIDT_DB_EDC_STALL_DELAY_1,       0xFFFFFFFF,                                          0,                                                     0x00000000 },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+static const struct vega10_didt_config_reg SEEDCThresholdConfig_Vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                 Shift                                                  Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	{   ixDIDT_SQ_EDC_THRESHOLD,           0xFFFFFFFF,                                          0,                                                     0x0000010E },
+	{   ixDIDT_TD_EDC_THRESHOLD,           0xFFFFFFFF,                                          0,                                                     0xFFFFFFFF },
+	{   ixDIDT_TCP_EDC_THRESHOLD,          0xFFFFFFFF,                                          0,                                                     0xFFFFFFFF },
+	{   ixDIDT_DB_EDC_THRESHOLD,           0xFFFFFFFF,                                          0,                                                     0xFFFFFFFF },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+static const struct vega10_didt_config_reg SEEDCCtrlResetConfig_Vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                 Shift                                                  Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	/* SQ */
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_EN_MASK,                       DIDT_SQ_EDC_CTRL__EDC_EN__SHIFT,                        0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_SW_RST_MASK,                   DIDT_SQ_EDC_CTRL__EDC_SW_RST__SHIFT,                    0x0001 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_CLK_EN_OVERRIDE_MASK,          DIDT_SQ_EDC_CTRL__EDC_CLK_EN_OVERRIDE__SHIFT,           0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_FORCE_STALL_MASK,              DIDT_SQ_EDC_CTRL__EDC_FORCE_STALL__SHIFT,               0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT_MASK,  DIDT_SQ_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT__SHIFT,   0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS_MASK,   DIDT_SQ_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS__SHIFT,    0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA_MASK,     DIDT_SQ_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA__SHIFT,      0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__GC_EDC_EN_MASK,                    DIDT_SQ_EDC_CTRL__GC_EDC_EN__SHIFT,                     0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__GC_EDC_STALL_POLICY_MASK,          DIDT_SQ_EDC_CTRL__GC_EDC_STALL_POLICY__SHIFT,           0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__GC_EDC_LEVEL_COMB_EN_MASK,         DIDT_SQ_EDC_CTRL__GC_EDC_LEVEL_COMB_EN__SHIFT,          0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__SE_EDC_LEVEL_COMB_EN_MASK,         DIDT_SQ_EDC_CTRL__SE_EDC_LEVEL_COMB_EN__SHIFT,          0x0000 },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+static const struct vega10_didt_config_reg SEEDCCtrlConfig_Vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                 Shift                                                  Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	/* SQ */
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_EN_MASK,                       DIDT_SQ_EDC_CTRL__EDC_EN__SHIFT,                        0x0001 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_SW_RST_MASK,                   DIDT_SQ_EDC_CTRL__EDC_SW_RST__SHIFT,                    0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_CLK_EN_OVERRIDE_MASK,          DIDT_SQ_EDC_CTRL__EDC_CLK_EN_OVERRIDE__SHIFT,           0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_FORCE_STALL_MASK,              DIDT_SQ_EDC_CTRL__EDC_FORCE_STALL__SHIFT,               0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT_MASK,  DIDT_SQ_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT__SHIFT,   0x0004 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS_MASK,   DIDT_SQ_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS__SHIFT,    0x0006 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA_MASK,     DIDT_SQ_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA__SHIFT,      0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__GC_EDC_EN_MASK,                    DIDT_SQ_EDC_CTRL__GC_EDC_EN__SHIFT,                     0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__GC_EDC_STALL_POLICY_MASK,          DIDT_SQ_EDC_CTRL__GC_EDC_STALL_POLICY__SHIFT,           0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__GC_EDC_LEVEL_COMB_EN_MASK,         DIDT_SQ_EDC_CTRL__GC_EDC_LEVEL_COMB_EN__SHIFT,          0x0001 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__SE_EDC_LEVEL_COMB_EN_MASK,         DIDT_SQ_EDC_CTRL__SE_EDC_LEVEL_COMB_EN__SHIFT,          0x0000 },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+static const struct vega10_didt_config_reg SEEDCCtrlForceStallConfig_Vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                 Shift                                                  Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	/* SQ */
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_EN_MASK,                       DIDT_SQ_EDC_CTRL__EDC_EN__SHIFT,                        0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_SW_RST_MASK,                   DIDT_SQ_EDC_CTRL__EDC_SW_RST__SHIFT,                    0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_CLK_EN_OVERRIDE_MASK,          DIDT_SQ_EDC_CTRL__EDC_CLK_EN_OVERRIDE__SHIFT,           0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_FORCE_STALL_MASK,              DIDT_SQ_EDC_CTRL__EDC_FORCE_STALL__SHIFT,               0x0001 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT_MASK,  DIDT_SQ_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT__SHIFT,   0x0001 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS_MASK,   DIDT_SQ_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS__SHIFT,    0x000C },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA_MASK,     DIDT_SQ_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA__SHIFT,      0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__GC_EDC_EN_MASK,                    DIDT_SQ_EDC_CTRL__GC_EDC_EN__SHIFT,                     0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__GC_EDC_STALL_POLICY_MASK,          DIDT_SQ_EDC_CTRL__GC_EDC_STALL_POLICY__SHIFT,           0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__GC_EDC_LEVEL_COMB_EN_MASK,         DIDT_SQ_EDC_CTRL__GC_EDC_LEVEL_COMB_EN__SHIFT,          0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__SE_EDC_LEVEL_COMB_EN_MASK,         DIDT_SQ_EDC_CTRL__SE_EDC_LEVEL_COMB_EN__SHIFT,          0x0001 },
+
+	/* TD */
+	{   ixDIDT_TD_EDC_CTRL,                DIDT_TD_EDC_CTRL__EDC_EN_MASK,                       DIDT_TD_EDC_CTRL__EDC_EN__SHIFT,                        0x0000 },
+	{   ixDIDT_TD_EDC_CTRL,                DIDT_TD_EDC_CTRL__EDC_SW_RST_MASK,                   DIDT_TD_EDC_CTRL__EDC_SW_RST__SHIFT,                    0x0000 },
+	{   ixDIDT_TD_EDC_CTRL,                DIDT_TD_EDC_CTRL__EDC_CLK_EN_OVERRIDE_MASK,          DIDT_TD_EDC_CTRL__EDC_CLK_EN_OVERRIDE__SHIFT,           0x0000 },
+	{   ixDIDT_TD_EDC_CTRL,                DIDT_TD_EDC_CTRL__EDC_FORCE_STALL_MASK,              DIDT_TD_EDC_CTRL__EDC_FORCE_STALL__SHIFT,               0x0001 },
+	{   ixDIDT_TD_EDC_CTRL,                DIDT_TD_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT_MASK,  DIDT_TD_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT__SHIFT,   0x0001 },
+	{   ixDIDT_TD_EDC_CTRL,                DIDT_TD_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS_MASK,   DIDT_TD_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS__SHIFT,    0x000E },
+	{   ixDIDT_TD_EDC_CTRL,                DIDT_TD_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA_MASK,     DIDT_TD_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA__SHIFT,      0x0000 },
+	{   ixDIDT_TD_EDC_CTRL,                DIDT_TD_EDC_CTRL__GC_EDC_EN_MASK,                    DIDT_TD_EDC_CTRL__GC_EDC_EN__SHIFT,                     0x0000 },
+	{   ixDIDT_TD_EDC_CTRL,                DIDT_TD_EDC_CTRL__GC_EDC_STALL_POLICY_MASK,          DIDT_TD_EDC_CTRL__GC_EDC_STALL_POLICY__SHIFT,           0x0000 },
+	{   ixDIDT_TD_EDC_CTRL,                DIDT_TD_EDC_CTRL__GC_EDC_LEVEL_COMB_EN_MASK,         DIDT_TD_EDC_CTRL__GC_EDC_LEVEL_COMB_EN__SHIFT,          0x0000 },
+	{   ixDIDT_TD_EDC_CTRL,                DIDT_TD_EDC_CTRL__SE_EDC_LEVEL_COMB_EN_MASK,         DIDT_TD_EDC_CTRL__SE_EDC_LEVEL_COMB_EN__SHIFT,          0x0001 },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+static const struct vega10_didt_config_reg    GCDiDtDroopCtrlConfig_vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                 Shift                                                  Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	{   mmGC_DIDT_DROOP_CTRL,             GC_DIDT_DROOP_CTRL__DIDT_DROOP_LEVEL_EN_MASK,   GC_DIDT_DROOP_CTRL__DIDT_DROOP_LEVEL_EN__SHIFT,  0x0000 },
+	{   mmGC_DIDT_DROOP_CTRL,             GC_DIDT_DROOP_CTRL__DIDT_DROOP_THRESHOLD_MASK,   GC_DIDT_DROOP_CTRL__DIDT_DROOP_THRESHOLD__SHIFT,  0x0000 },
+	{   mmGC_DIDT_DROOP_CTRL,             GC_DIDT_DROOP_CTRL__DIDT_DROOP_LEVEL_INDEX_MASK,   GC_DIDT_DROOP_CTRL__DIDT_DROOP_LEVEL_INDEX__SHIFT,  0x0000 },
+	{   mmGC_DIDT_DROOP_CTRL,             GC_DIDT_DROOP_CTRL__DIDT_LEVEL_SEL_MASK,   GC_DIDT_DROOP_CTRL__DIDT_LEVEL_SEL__SHIFT,  0x0000 },
+	{   mmGC_DIDT_DROOP_CTRL,             GC_DIDT_DROOP_CTRL__DIDT_DROOP_LEVEL_OVERFLOW_MASK,   GC_DIDT_DROOP_CTRL__DIDT_DROOP_LEVEL_OVERFLOW__SHIFT,  0x0000 },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+static const struct vega10_didt_config_reg    GCDiDtCtrl0Config_vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                 Shift                                                  Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	{   mmGC_DIDT_CTRL0,                  GC_DIDT_CTRL0__DIDT_CTRL_EN_MASK,   GC_DIDT_CTRL0__DIDT_CTRL_EN__SHIFT,  0x0000 },
+	{   mmGC_DIDT_CTRL0,                  GC_DIDT_CTRL0__PHASE_OFFSET_MASK,   GC_DIDT_CTRL0__PHASE_OFFSET__SHIFT,  0x0000 },
+	{   mmGC_DIDT_CTRL0,                  GC_DIDT_CTRL0__DIDT_SW_RST_MASK,   GC_DIDT_CTRL0__DIDT_SW_RST__SHIFT,  0x0000 },
+	{   mmGC_DIDT_CTRL0,                  GC_DIDT_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,   GC_DIDT_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,  0x0000 },
+	{   mmGC_DIDT_CTRL0,                  GC_DIDT_CTRL0__DIDT_TRIGGER_THROTTLE_LOWBIT_MASK,   GC_DIDT_CTRL0__DIDT_TRIGGER_THROTTLE_LOWBIT__SHIFT,  0x0000 },
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+
+static const struct vega10_didt_config_reg   PSMSEEDCStallPatternConfig_Vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                 Shift                                                  Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	/* SQ EDC STALL PATTERNs */
+	{   ixDIDT_SQ_EDC_STALL_PATTERN_1_2,  DIDT_SQ_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_1_MASK,   DIDT_SQ_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_1__SHIFT,   0x0101 },
+	{   ixDIDT_SQ_EDC_STALL_PATTERN_1_2,  DIDT_SQ_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_2_MASK,   DIDT_SQ_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_2__SHIFT,   0x0101 },
+	{   ixDIDT_SQ_EDC_STALL_PATTERN_3_4,  DIDT_SQ_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_3_MASK,   DIDT_SQ_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_3__SHIFT,   0x1111 },
+	{   ixDIDT_SQ_EDC_STALL_PATTERN_3_4,  DIDT_SQ_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_4_MASK,   DIDT_SQ_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_4__SHIFT,   0x1111 },
+
+	{   ixDIDT_SQ_EDC_STALL_PATTERN_5_6,  DIDT_SQ_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_5_MASK,   DIDT_SQ_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_5__SHIFT,   0x1515 },
+	{   ixDIDT_SQ_EDC_STALL_PATTERN_5_6,  DIDT_SQ_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_6_MASK,   DIDT_SQ_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_6__SHIFT,   0x1515 },
+
+	{   ixDIDT_SQ_EDC_STALL_PATTERN_7,  DIDT_SQ_EDC_STALL_PATTERN_7__EDC_STALL_PATTERN_7_MASK,   DIDT_SQ_EDC_STALL_PATTERN_7__EDC_STALL_PATTERN_7__SHIFT,     0x5555 },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+static const struct vega10_didt_config_reg   PSMSEEDCStallDelayConfig_Vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                 Shift                                                  Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	/* SQ EDC STALL DELAYs */
+	{   ixDIDT_SQ_EDC_STALL_DELAY_1,      DIDT_SQ_EDC_STALL_DELAY_1__EDC_STALL_DELAY_SQ0_MASK,  DIDT_SQ_EDC_STALL_DELAY_1__EDC_STALL_DELAY_SQ0__SHIFT,  0x0000 },
+	{   ixDIDT_SQ_EDC_STALL_DELAY_1,      DIDT_SQ_EDC_STALL_DELAY_1__EDC_STALL_DELAY_SQ1_MASK,  DIDT_SQ_EDC_STALL_DELAY_1__EDC_STALL_DELAY_SQ1__SHIFT,  0x0000 },
+	{   ixDIDT_SQ_EDC_STALL_DELAY_1,      DIDT_SQ_EDC_STALL_DELAY_1__EDC_STALL_DELAY_SQ2_MASK,  DIDT_SQ_EDC_STALL_DELAY_1__EDC_STALL_DELAY_SQ2__SHIFT,  0x0000 },
+	{   ixDIDT_SQ_EDC_STALL_DELAY_1,      DIDT_SQ_EDC_STALL_DELAY_1__EDC_STALL_DELAY_SQ3_MASK,  DIDT_SQ_EDC_STALL_DELAY_1__EDC_STALL_DELAY_SQ3__SHIFT,  0x0000 },
+
+	{   ixDIDT_SQ_EDC_STALL_DELAY_2,      DIDT_SQ_EDC_STALL_DELAY_2__EDC_STALL_DELAY_SQ4_MASK,  DIDT_SQ_EDC_STALL_DELAY_2__EDC_STALL_DELAY_SQ4__SHIFT,  0x0000 },
+	{   ixDIDT_SQ_EDC_STALL_DELAY_2,      DIDT_SQ_EDC_STALL_DELAY_2__EDC_STALL_DELAY_SQ5_MASK,  DIDT_SQ_EDC_STALL_DELAY_2__EDC_STALL_DELAY_SQ5__SHIFT,  0x0000 },
+	{   ixDIDT_SQ_EDC_STALL_DELAY_2,      DIDT_SQ_EDC_STALL_DELAY_2__EDC_STALL_DELAY_SQ6_MASK,  DIDT_SQ_EDC_STALL_DELAY_2__EDC_STALL_DELAY_SQ6__SHIFT,  0x0000 },
+	{   ixDIDT_SQ_EDC_STALL_DELAY_2,      DIDT_SQ_EDC_STALL_DELAY_2__EDC_STALL_DELAY_SQ7_MASK,  DIDT_SQ_EDC_STALL_DELAY_2__EDC_STALL_DELAY_SQ7__SHIFT,  0x0000 },
+
+	{   ixDIDT_SQ_EDC_STALL_DELAY_3,      DIDT_SQ_EDC_STALL_DELAY_3__EDC_STALL_DELAY_SQ8_MASK,  DIDT_SQ_EDC_STALL_DELAY_3__EDC_STALL_DELAY_SQ8__SHIFT,  0x0000 },
+	{   ixDIDT_SQ_EDC_STALL_DELAY_3,      DIDT_SQ_EDC_STALL_DELAY_3__EDC_STALL_DELAY_SQ9_MASK,  DIDT_SQ_EDC_STALL_DELAY_3__EDC_STALL_DELAY_SQ9__SHIFT,  0x0000 },
+	{   ixDIDT_SQ_EDC_STALL_DELAY_3,      DIDT_SQ_EDC_STALL_DELAY_3__EDC_STALL_DELAY_SQ10_MASK, DIDT_SQ_EDC_STALL_DELAY_3__EDC_STALL_DELAY_SQ10__SHIFT, 0x0000 },
+	{   ixDIDT_SQ_EDC_STALL_DELAY_3,      DIDT_SQ_EDC_STALL_DELAY_3__EDC_STALL_DELAY_SQ11_MASK, DIDT_SQ_EDC_STALL_DELAY_3__EDC_STALL_DELAY_SQ11__SHIFT, 0x0000 },
+
+	{   ixDIDT_SQ_EDC_STALL_DELAY_4,      DIDT_SQ_EDC_STALL_DELAY_4__EDC_STALL_DELAY_SQ12_MASK, DIDT_SQ_EDC_STALL_DELAY_4__EDC_STALL_DELAY_SQ12__SHIFT, 0x0000 },
+	{   ixDIDT_SQ_EDC_STALL_DELAY_4,      DIDT_SQ_EDC_STALL_DELAY_4__EDC_STALL_DELAY_SQ12_MASK, DIDT_SQ_EDC_STALL_DELAY_4__EDC_STALL_DELAY_SQ13__SHIFT, 0x0000 },
+	{   ixDIDT_SQ_EDC_STALL_DELAY_4,      DIDT_SQ_EDC_STALL_DELAY_4__EDC_STALL_DELAY_SQ14_MASK, DIDT_SQ_EDC_STALL_DELAY_4__EDC_STALL_DELAY_SQ14__SHIFT, 0x0000 },
+	{   ixDIDT_SQ_EDC_STALL_DELAY_4,      DIDT_SQ_EDC_STALL_DELAY_4__EDC_STALL_DELAY_SQ15_MASK, DIDT_SQ_EDC_STALL_DELAY_4__EDC_STALL_DELAY_SQ15__SHIFT, 0x0000 },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+static const struct vega10_didt_config_reg   PSMSEEDCThresholdConfig_Vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                 Shift                                                  Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	/* SQ EDC THRESHOLD */
+	{   ixDIDT_SQ_EDC_THRESHOLD,           DIDT_SQ_EDC_THRESHOLD__EDC_THRESHOLD_MASK,           DIDT_SQ_EDC_THRESHOLD__EDC_THRESHOLD__SHIFT,            0x0000 },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+static const struct vega10_didt_config_reg   PSMSEEDCCtrlResetConfig_Vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                 Shift                                                  Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	/* SQ EDC CTRL */
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_EN_MASK,                       DIDT_SQ_EDC_CTRL__EDC_EN__SHIFT,                        0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_SW_RST_MASK,                   DIDT_SQ_EDC_CTRL__EDC_SW_RST__SHIFT,                    0x0001 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_CLK_EN_OVERRIDE_MASK,          DIDT_SQ_EDC_CTRL__EDC_CLK_EN_OVERRIDE__SHIFT,           0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_FORCE_STALL_MASK,              DIDT_SQ_EDC_CTRL__EDC_FORCE_STALL__SHIFT,               0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT_MASK,  DIDT_SQ_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT__SHIFT,   0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS_MASK,   DIDT_SQ_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS__SHIFT,    0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA_MASK,     DIDT_SQ_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA__SHIFT,      0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__GC_EDC_EN_MASK,                    DIDT_SQ_EDC_CTRL__GC_EDC_EN__SHIFT,                     0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__GC_EDC_STALL_POLICY_MASK,          DIDT_SQ_EDC_CTRL__GC_EDC_STALL_POLICY__SHIFT,           0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__GC_EDC_LEVEL_COMB_EN_MASK,         DIDT_SQ_EDC_CTRL__GC_EDC_LEVEL_COMB_EN__SHIFT,          0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__SE_EDC_LEVEL_COMB_EN_MASK,         DIDT_SQ_EDC_CTRL__SE_EDC_LEVEL_COMB_EN__SHIFT,          0x0000 },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+static const struct vega10_didt_config_reg   PSMSEEDCCtrlConfig_Vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                 Shift                                                  Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	/* SQ EDC CTRL */
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_EN_MASK,                       DIDT_SQ_EDC_CTRL__EDC_EN__SHIFT,                        0x0001 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_SW_RST_MASK,                   DIDT_SQ_EDC_CTRL__EDC_SW_RST__SHIFT,                    0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_CLK_EN_OVERRIDE_MASK,          DIDT_SQ_EDC_CTRL__EDC_CLK_EN_OVERRIDE__SHIFT,           0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_FORCE_STALL_MASK,              DIDT_SQ_EDC_CTRL__EDC_FORCE_STALL__SHIFT,               0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT_MASK,  DIDT_SQ_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT__SHIFT,   0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS_MASK,   DIDT_SQ_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS__SHIFT,    0x000E },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA_MASK,     DIDT_SQ_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA__SHIFT,      0x0000 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__GC_EDC_EN_MASK,                    DIDT_SQ_EDC_CTRL__GC_EDC_EN__SHIFT,                     0x0001 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__GC_EDC_STALL_POLICY_MASK,          DIDT_SQ_EDC_CTRL__GC_EDC_STALL_POLICY__SHIFT,           0x0003 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__GC_EDC_LEVEL_COMB_EN_MASK,         DIDT_SQ_EDC_CTRL__GC_EDC_LEVEL_COMB_EN__SHIFT,          0x0001 },
+	{   ixDIDT_SQ_EDC_CTRL,                DIDT_SQ_EDC_CTRL__SE_EDC_LEVEL_COMB_EN_MASK,         DIDT_SQ_EDC_CTRL__SE_EDC_LEVEL_COMB_EN__SHIFT,          0x0000 },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+static const struct vega10_didt_config_reg   PSMGCEDCThresholdConfig_vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                 Shift                                                  Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	{   mmGC_EDC_THRESHOLD,                GC_EDC_THRESHOLD__EDC_THRESHOLD_MASK,                GC_EDC_THRESHOLD__EDC_THRESHOLD__SHIFT,                 0x0000000 },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+static const struct vega10_didt_config_reg   PSMGCEDCDroopCtrlConfig_vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                 Shift                                                  Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	{   mmGC_EDC_DROOP_CTRL,               GC_EDC_DROOP_CTRL__EDC_DROOP_LEVEL_EN_MASK,          GC_EDC_DROOP_CTRL__EDC_DROOP_LEVEL_EN__SHIFT,           0x0001 },
+	{   mmGC_EDC_DROOP_CTRL,               GC_EDC_DROOP_CTRL__EDC_DROOP_THRESHOLD_MASK,         GC_EDC_DROOP_CTRL__EDC_DROOP_THRESHOLD__SHIFT,          0x0384 },
+	{   mmGC_EDC_DROOP_CTRL,               GC_EDC_DROOP_CTRL__EDC_DROOP_LEVEL_INDEX_MASK,       GC_EDC_DROOP_CTRL__EDC_DROOP_LEVEL_INDEX__SHIFT,        0x0001 },
+	{   mmGC_EDC_DROOP_CTRL,               GC_EDC_DROOP_CTRL__AVG_PSM_SEL_MASK,                 GC_EDC_DROOP_CTRL__AVG_PSM_SEL__SHIFT,                  0x0001 },
+	{   mmGC_EDC_DROOP_CTRL,               GC_EDC_DROOP_CTRL__EDC_LEVEL_SEL_MASK,               GC_EDC_DROOP_CTRL__EDC_LEVEL_SEL__SHIFT,                0x0001 },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+static const struct vega10_didt_config_reg   PSMGCEDCCtrlResetConfig_vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                 Shift                                                  Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	{   mmGC_EDC_CTRL,                     GC_EDC_CTRL__EDC_EN_MASK,                            GC_EDC_CTRL__EDC_EN__SHIFT,                             0x0000 },
+	{   mmGC_EDC_CTRL,                     GC_EDC_CTRL__EDC_SW_RST_MASK,                        GC_EDC_CTRL__EDC_SW_RST__SHIFT,                         0x0001 },
+	{   mmGC_EDC_CTRL,                     GC_EDC_CTRL__EDC_CLK_EN_OVERRIDE_MASK,               GC_EDC_CTRL__EDC_CLK_EN_OVERRIDE__SHIFT,                0x0000 },
+	{   mmGC_EDC_CTRL,                     GC_EDC_CTRL__EDC_FORCE_STALL_MASK,                   GC_EDC_CTRL__EDC_FORCE_STALL__SHIFT,                    0x0000 },
+	{   mmGC_EDC_CTRL,                     GC_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT_MASK,       GC_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT__SHIFT,        0x0000 },
+	{   mmGC_EDC_CTRL,                     GC_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA_MASK,          GC_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA__SHIFT,           0x0000 },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+static const struct vega10_didt_config_reg   PSMGCEDCCtrlConfig_vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                 Shift                                                  Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	{   mmGC_EDC_CTRL,                     GC_EDC_CTRL__EDC_EN_MASK,                            GC_EDC_CTRL__EDC_EN__SHIFT,                             0x0001 },
+	{   mmGC_EDC_CTRL,                     GC_EDC_CTRL__EDC_SW_RST_MASK,                        GC_EDC_CTRL__EDC_SW_RST__SHIFT,                         0x0000 },
+	{   mmGC_EDC_CTRL,                     GC_EDC_CTRL__EDC_CLK_EN_OVERRIDE_MASK,               GC_EDC_CTRL__EDC_CLK_EN_OVERRIDE__SHIFT,                0x0000 },
+	{   mmGC_EDC_CTRL,                     GC_EDC_CTRL__EDC_FORCE_STALL_MASK,                   GC_EDC_CTRL__EDC_FORCE_STALL__SHIFT,                    0x0000 },
+	{   mmGC_EDC_CTRL,                     GC_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT_MASK,       GC_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT__SHIFT,        0x0000 },
+	{   mmGC_EDC_CTRL,                     GC_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA_MASK,          GC_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA__SHIFT,           0x0000 },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+static const struct vega10_didt_config_reg    AvfsPSMResetConfig_vega10[]=
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                 Shift                                                  Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	{   0x16A02,                         0xFFFFFFFF,                                            0x0,                                                    0x0000005F },
+	{   0x16A05,                         0xFFFFFFFF,                                            0x0,                                                    0x00000001 },
+	{   0x16A06,                         0x00000001,                                            0x0,                                                    0x02000000 },
+	{   0x16A01,                         0xFFFFFFFF,                                            0x0,                                                    0x00003027 },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+static const struct vega10_didt_config_reg    AvfsPSMInitConfig_vega10[] =
+{
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                 Shift                                                  Value
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+	{   0x16A05,                         0xFFFFFFFF,                                            0x18,                                                    0x00000001 },
+	{   0x16A05,                         0xFFFFFFFF,                                            0x8,                                                     0x00000003 },
+	{   0x16A05,                         0xFFFFFFFF,                                            0xa,                                                     0x00000006 },
+	{   0x16A05,                         0xFFFFFFFF,                                            0x7,                                                     0x00000000 },
+	{   0x16A06,                         0xFFFFFFFF,                                            0x18,                                                    0x00000001 },
+	{   0x16A06,                         0xFFFFFFFF,                                            0x19,                                                    0x00000001 },
+	{   0x16A01,                         0xFFFFFFFF,                                            0x0,                                                     0x00003027 },
+
+	{   0xFFFFFFFF  }  /* End of list */
+};
+
+static int vega10_program_didt_config_registers(struct pp_hwmgr *hwmgr, const struct vega10_didt_config_reg *config_regs, enum vega10_didt_config_reg_type reg_type)
+{
+	uint32_t data;
+
+	PP_ASSERT_WITH_CODE((config_regs != NULL), "[vega10_program_didt_config_registers] Invalid config register table!", return -EINVAL);
+
+	while (config_regs->offset != 0xFFFFFFFF) {
+		switch (reg_type) {
+		case VEGA10_CONFIGREG_DIDT:
+			data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, config_regs->offset);
+			data &= ~config_regs->mask;
+			data |= ((config_regs->value << config_regs->shift) & config_regs->mask);
+			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, config_regs->offset, data);
+			break;
+		case VEGA10_CONFIGREG_GCCAC:
+			data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG_GC_CAC, config_regs->offset);
+			data &= ~config_regs->mask;
+			data |= ((config_regs->value << config_regs->shift) & config_regs->mask);
+			cgs_write_ind_register(hwmgr->device, CGS_IND_REG_GC_CAC, config_regs->offset, data);
+			break;
+		case VEGA10_CONFIGREG_SECAC:
+			data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG_SE_CAC, config_regs->offset);
+			data &= ~config_regs->mask;
+			data |= ((config_regs->value << config_regs->shift) & config_regs->mask);
+			cgs_write_ind_register(hwmgr->device, CGS_IND_REG_SE_CAC, config_regs->offset, data);
+			break;
+		default:
+			return -EINVAL;
+		}
+
+		config_regs++;
+	}
+
+	return 0;
+}
+
+static int vega10_program_gc_didt_config_registers(struct pp_hwmgr *hwmgr, const struct vega10_didt_config_reg *config_regs)
+{
+	uint32_t data;
+
+	while (config_regs->offset != 0xFFFFFFFF) {
+		data = cgs_read_register(hwmgr->device, config_regs->offset);
+		data &= ~config_regs->mask;
+		data |= ((config_regs->value << config_regs->shift) & config_regs->mask);
+		cgs_write_register(hwmgr->device, config_regs->offset, data);
+		config_regs++;
+	}
+
+	return 0;
+}
+
+static void vega10_didt_set_mask(struct pp_hwmgr *hwmgr, const bool enable)
+{
+	uint32_t data;
+	int result;
+	uint32_t en = (enable ? 1 : 0);
+	uint32_t didt_block_info = SQ_IR_MASK | TCP_IR_MASK | TD_PCC_MASK;
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SQRamping)) {
+		data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_SQ_CTRL0);
+		data &= ~DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK;
+		data |= ((en << DIDT_SQ_CTRL0__DIDT_CTRL_EN__SHIFT) & DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK);
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_SQ_CTRL0, data);
+		didt_block_info &= ~SQ_Enable_MASK;
+		didt_block_info |= en << SQ_Enable_SHIFT;
+	}
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DBRamping)) {
+		data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_DB_CTRL0);
+		data &= ~DIDT_DB_CTRL0__DIDT_CTRL_EN_MASK;
+		data |= ((en << DIDT_DB_CTRL0__DIDT_CTRL_EN__SHIFT) & DIDT_DB_CTRL0__DIDT_CTRL_EN_MASK);
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_DB_CTRL0, data);
+		didt_block_info &= ~DB_Enable_MASK;
+		didt_block_info |= en << DB_Enable_SHIFT;
+	}
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TDRamping)) {
+		data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TD_CTRL0);
+		data &= ~DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK;
+		data |= ((en << DIDT_TD_CTRL0__DIDT_CTRL_EN__SHIFT) & DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK);
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TD_CTRL0, data);
+		didt_block_info &= ~TD_Enable_MASK;
+		didt_block_info |= en << TD_Enable_SHIFT;
+	}
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TCPRamping)) {
+		data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TCP_CTRL0);
+		data &= ~DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK;
+		data |= ((en << DIDT_TCP_CTRL0__DIDT_CTRL_EN__SHIFT) & DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK);
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TCP_CTRL0, data);
+		didt_block_info &= ~TCP_Enable_MASK;
+		didt_block_info |= en << TCP_Enable_SHIFT;
+	}
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DBRRamping)) {
+		data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_DBR_CTRL0);
+		data &= ~DIDT_DBR_CTRL0__DIDT_CTRL_EN_MASK;
+		data |= ((en << DIDT_DBR_CTRL0__DIDT_CTRL_EN__SHIFT) & DIDT_DBR_CTRL0__DIDT_CTRL_EN_MASK);
+		cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_DBR_CTRL0, data);
+	}
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DiDtEDCEnable)) {
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SQRamping)) {
+			data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_SQ_EDC_CTRL);
+			data &= ~DIDT_SQ_EDC_CTRL__EDC_EN_MASK;
+			data |= ((en << DIDT_SQ_EDC_CTRL__EDC_EN__SHIFT) & DIDT_SQ_EDC_CTRL__EDC_EN_MASK);
+			data &= ~DIDT_SQ_EDC_CTRL__EDC_SW_RST_MASK;
+			data |= ((~en << DIDT_SQ_EDC_CTRL__EDC_SW_RST__SHIFT) & DIDT_SQ_EDC_CTRL__EDC_SW_RST_MASK);
+			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_SQ_EDC_CTRL, data);
+		}
+
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DBRamping)) {
+			data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_DB_EDC_CTRL);
+			data &= ~DIDT_DB_EDC_CTRL__EDC_EN_MASK;
+			data |= ((en << DIDT_DB_EDC_CTRL__EDC_EN__SHIFT) & DIDT_DB_EDC_CTRL__EDC_EN_MASK);
+			data &= ~DIDT_DB_EDC_CTRL__EDC_SW_RST_MASK;
+			data |= ((~en << DIDT_DB_EDC_CTRL__EDC_SW_RST__SHIFT) & DIDT_DB_EDC_CTRL__EDC_SW_RST_MASK);
+			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_DB_EDC_CTRL, data);
+		}
+
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TDRamping)) {
+			data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TD_EDC_CTRL);
+			data &= ~DIDT_TD_EDC_CTRL__EDC_EN_MASK;
+			data |= ((en << DIDT_TD_EDC_CTRL__EDC_EN__SHIFT) & DIDT_TD_EDC_CTRL__EDC_EN_MASK);
+			data &= ~DIDT_TD_EDC_CTRL__EDC_SW_RST_MASK;
+			data |= ((~en << DIDT_TD_EDC_CTRL__EDC_SW_RST__SHIFT) & DIDT_TD_EDC_CTRL__EDC_SW_RST_MASK);
+			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TD_EDC_CTRL, data);
+		}
+
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TCPRamping)) {
+			data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TCP_EDC_CTRL);
+			data &= ~DIDT_TCP_EDC_CTRL__EDC_EN_MASK;
+			data |= ((en << DIDT_TCP_EDC_CTRL__EDC_EN__SHIFT) & DIDT_TCP_EDC_CTRL__EDC_EN_MASK);
+			data &= ~DIDT_TCP_EDC_CTRL__EDC_SW_RST_MASK;
+			data |= ((~en << DIDT_TCP_EDC_CTRL__EDC_SW_RST__SHIFT) & DIDT_TCP_EDC_CTRL__EDC_SW_RST_MASK);
+			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TCP_EDC_CTRL, data);
+		}
+
+		if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DBRRamping)) {
+			data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_DBR_EDC_CTRL);
+			data &= ~DIDT_DBR_EDC_CTRL__EDC_EN_MASK;
+			data |= ((en << DIDT_DBR_EDC_CTRL__EDC_EN__SHIFT) & DIDT_DBR_EDC_CTRL__EDC_EN_MASK);
+			data &= ~DIDT_DBR_EDC_CTRL__EDC_SW_RST_MASK;
+			data |= ((~en << DIDT_DBR_EDC_CTRL__EDC_SW_RST__SHIFT) & DIDT_DBR_EDC_CTRL__EDC_SW_RST_MASK);
+			cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_DBR_EDC_CTRL, data);
+		}
+	}
+
+	if (enable) {
+		/* For Vega10, SMC does not support any mask yet. */
+		result = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, PPSMC_MSG_ConfigureGfxDidt, didt_block_info);
+		PP_ASSERT((0 == result), "[EnableDiDtConfig] SMC Configure Gfx Didt Failed!");
+	}
+}
+
+static int vega10_enable_cac_driving_se_didt_config(struct pp_hwmgr *hwmgr)
+{
+	int result;
+	uint32_t num_se = 0, count, data;
+	struct cgs_system_info sys_info = {0};
+	uint32_t reg;
+
+	sys_info.size = sizeof(struct cgs_system_info);
+	sys_info.info_id = CGS_SYSTEM_INFO_GFX_SE_INFO;
+	if (cgs_query_system_info(hwmgr->device, &sys_info) == 0)
+		num_se = sys_info.value;
+
+	cgs_enter_safe_mode(hwmgr->device, true);
+
+	cgs_lock_grbm_idx(hwmgr->device, true);
+	reg = soc15_get_register_offset(GC_HWID, 0, mmGRBM_GFX_INDEX_BASE_IDX, mmGRBM_GFX_INDEX);
+	for (count = 0; count < num_se; count++) {
+		data = GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES_MASK | GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK | ( count << GRBM_GFX_INDEX__SE_INDEX__SHIFT);
+		cgs_write_register(hwmgr->device, reg, data);
+
+		result =  vega10_program_didt_config_registers(hwmgr, SEDiDtStallCtrlConfig_vega10, VEGA10_CONFIGREG_DIDT);
+		result |= vega10_program_didt_config_registers(hwmgr, SEDiDtStallPatternConfig_vega10, VEGA10_CONFIGREG_DIDT);
+		result |= vega10_program_didt_config_registers(hwmgr, SEDiDtWeightConfig_Vega10, VEGA10_CONFIGREG_DIDT);
+		result |= vega10_program_didt_config_registers(hwmgr, SEDiDtCtrl1Config_Vega10, VEGA10_CONFIGREG_DIDT);
+		result |= vega10_program_didt_config_registers(hwmgr, SEDiDtCtrl2Config_Vega10, VEGA10_CONFIGREG_DIDT);
+		result |= vega10_program_didt_config_registers(hwmgr, SEDiDtCtrl3Config_vega10, VEGA10_CONFIGREG_DIDT);
+		result |= vega10_program_didt_config_registers(hwmgr, SEDiDtTuningCtrlConfig_Vega10, VEGA10_CONFIGREG_DIDT);
+		result |= vega10_program_didt_config_registers(hwmgr, SELCacConfig_Vega10, VEGA10_CONFIGREG_SECAC);
+		result |= vega10_program_didt_config_registers(hwmgr, SEDiDtCtrl0Config_Vega10, VEGA10_CONFIGREG_DIDT);
+
+		if (0 != result)
+			break;
+	}
+	cgs_write_register(hwmgr->device, reg, 0xE0000000);
+	cgs_lock_grbm_idx(hwmgr->device, false);
+
+	vega10_didt_set_mask(hwmgr, true);
+
+	cgs_enter_safe_mode(hwmgr->device, false);
+
+	return 0;
+}
+
+static int vega10_disable_cac_driving_se_didt_config(struct pp_hwmgr *hwmgr)
+{
+	cgs_enter_safe_mode(hwmgr->device, true);
+
+	vega10_didt_set_mask(hwmgr, false);
+
+	cgs_enter_safe_mode(hwmgr->device, false);
+
+	return 0;
+}
+
+static int vega10_enable_psm_gc_didt_config(struct pp_hwmgr *hwmgr)
+{
+	int result;
+	uint32_t num_se = 0, count, data;
+	struct cgs_system_info sys_info = {0};
+	uint32_t reg;
+
+	sys_info.size = sizeof(struct cgs_system_info);
+	sys_info.info_id = CGS_SYSTEM_INFO_GFX_SE_INFO;
+	if (cgs_query_system_info(hwmgr->device, &sys_info) == 0)
+		num_se = sys_info.value;
+
+	cgs_enter_safe_mode(hwmgr->device, true);
+
+	cgs_lock_grbm_idx(hwmgr->device, true);
+	reg = soc15_get_register_offset(GC_HWID, 0, mmGRBM_GFX_INDEX_BASE_IDX, mmGRBM_GFX_INDEX);
+	for (count = 0; count < num_se; count++) {
+		data = GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES_MASK | GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK | ( count << GRBM_GFX_INDEX__SE_INDEX__SHIFT);
+		cgs_write_register(hwmgr->device, reg, data);
+
+		result = vega10_program_didt_config_registers(hwmgr, SEDiDtStallCtrlConfig_vega10, VEGA10_CONFIGREG_DIDT);
+		result |= vega10_program_didt_config_registers(hwmgr, SEDiDtStallPatternConfig_vega10, VEGA10_CONFIGREG_DIDT);
+		result |= vega10_program_didt_config_registers(hwmgr, SEDiDtCtrl3Config_vega10, VEGA10_CONFIGREG_DIDT);
+		result |= vega10_program_didt_config_registers(hwmgr, SEDiDtCtrl0Config_Vega10, VEGA10_CONFIGREG_DIDT);
+		if (0 != result)
+			break;
+	}
+	cgs_write_register(hwmgr->device, reg, 0xE0000000);
+	cgs_lock_grbm_idx(hwmgr->device, false);
+
+	vega10_didt_set_mask(hwmgr, true);
+
+	cgs_enter_safe_mode(hwmgr->device, false);
+
+	vega10_program_gc_didt_config_registers(hwmgr, GCDiDtDroopCtrlConfig_vega10);
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_GCEDC))
+		vega10_program_gc_didt_config_registers(hwmgr, GCDiDtCtrl0Config_vega10);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_PSM))
+		vega10_program_gc_didt_config_registers(hwmgr,  AvfsPSMInitConfig_vega10);
+
+	return 0;
+}
+
+static int vega10_disable_psm_gc_didt_config(struct pp_hwmgr *hwmgr)
+{
+	uint32_t data;
+
+	cgs_enter_safe_mode(hwmgr->device, true);
+
+	vega10_didt_set_mask(hwmgr, false);
+
+	cgs_enter_safe_mode(hwmgr->device, false);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_GCEDC)) {
+		data = 0x00000000;
+		cgs_write_register(hwmgr->device, mmGC_DIDT_CTRL0, data);
+	}
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_PSM))
+		vega10_program_gc_didt_config_registers(hwmgr,  AvfsPSMResetConfig_vega10);
+
+	return 0;
+}
+
+static int vega10_enable_se_edc_config(struct pp_hwmgr *hwmgr)
+{
+	int result;
+	uint32_t num_se = 0, count, data;
+	struct cgs_system_info sys_info = {0};
+	uint32_t reg;
+
+	sys_info.size = sizeof(struct cgs_system_info);
+	sys_info.info_id = CGS_SYSTEM_INFO_GFX_SE_INFO;
+	if (cgs_query_system_info(hwmgr->device, &sys_info) == 0)
+		num_se = sys_info.value;
+
+	cgs_enter_safe_mode(hwmgr->device, true);
+
+	cgs_lock_grbm_idx(hwmgr->device, true);
+	reg = soc15_get_register_offset(GC_HWID, 0, mmGRBM_GFX_INDEX_BASE_IDX, mmGRBM_GFX_INDEX);
+	for (count = 0; count < num_se; count++) {
+		data = GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES_MASK | GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK | ( count << GRBM_GFX_INDEX__SE_INDEX__SHIFT);
+		cgs_write_register(hwmgr->device, reg, data);
+		result = vega10_program_didt_config_registers(hwmgr, SEDiDtWeightConfig_Vega10, VEGA10_CONFIGREG_DIDT);
+		result |= vega10_program_didt_config_registers(hwmgr, SEEDCStallPatternConfig_Vega10, VEGA10_CONFIGREG_DIDT);
+		result |= vega10_program_didt_config_registers(hwmgr, SEEDCStallDelayConfig_Vega10, VEGA10_CONFIGREG_DIDT);
+		result |= vega10_program_didt_config_registers(hwmgr, SEEDCThresholdConfig_Vega10, VEGA10_CONFIGREG_DIDT);
+		result |= vega10_program_didt_config_registers(hwmgr, SEEDCCtrlResetConfig_Vega10, VEGA10_CONFIGREG_DIDT);
+		result |= vega10_program_didt_config_registers(hwmgr, SEEDCCtrlConfig_Vega10, VEGA10_CONFIGREG_DIDT);
+
+		if (0 != result)
+			break;
+	}
+	cgs_write_register(hwmgr->device, reg, 0xE0000000);
+	cgs_lock_grbm_idx(hwmgr->device, false);
+
+	vega10_didt_set_mask(hwmgr, true);
+
+	cgs_enter_safe_mode(hwmgr->device, false);
+
+	return 0;
+}
+
+static int vega10_disable_se_edc_config(struct pp_hwmgr *hwmgr)
+{
+	cgs_enter_safe_mode(hwmgr->device, true);
+
+	vega10_didt_set_mask(hwmgr, false);
+
+	cgs_enter_safe_mode(hwmgr->device, false);
+
+	return 0;
+}
+
+static int vega10_enable_psm_gc_edc_config(struct pp_hwmgr *hwmgr)
+{
+	int result;
+	uint32_t num_se = 0;
+	uint32_t count, data;
+	struct cgs_system_info sys_info = {0};
+	uint32_t reg;
+
+	sys_info.size = sizeof(struct cgs_system_info);
+	sys_info.info_id = CGS_SYSTEM_INFO_GFX_SE_INFO;
+	if (cgs_query_system_info(hwmgr->device, &sys_info) == 0)
+		num_se = sys_info.value;
+
+	cgs_enter_safe_mode(hwmgr->device, true);
+
+	vega10_program_gc_didt_config_registers(hwmgr, AvfsPSMResetConfig_vega10);
+
+	cgs_lock_grbm_idx(hwmgr->device, true);
+	reg = soc15_get_register_offset(GC_HWID, 0, mmGRBM_GFX_INDEX_BASE_IDX, mmGRBM_GFX_INDEX);
+	for (count = 0; count < num_se; count++) {
+		data = GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES_MASK | GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK | ( count << GRBM_GFX_INDEX__SE_INDEX__SHIFT);
+		cgs_write_register(hwmgr->device, reg, data);
+		result |= vega10_program_didt_config_registers(hwmgr, PSMSEEDCStallPatternConfig_Vega10, VEGA10_CONFIGREG_DIDT);
+		result |= vega10_program_didt_config_registers(hwmgr, PSMSEEDCStallDelayConfig_Vega10, VEGA10_CONFIGREG_DIDT);
+		result |= vega10_program_didt_config_registers(hwmgr, PSMSEEDCCtrlResetConfig_Vega10, VEGA10_CONFIGREG_DIDT);
+		result |= vega10_program_didt_config_registers(hwmgr, PSMSEEDCCtrlConfig_Vega10, VEGA10_CONFIGREG_DIDT);
+
+		if (0 != result)
+			break;
+	}
+	cgs_write_register(hwmgr->device, reg, 0xE0000000);
+	cgs_lock_grbm_idx(hwmgr->device, false);
+
+	vega10_didt_set_mask(hwmgr, true);
+
+	cgs_enter_safe_mode(hwmgr->device, false);
+
+	vega10_program_gc_didt_config_registers(hwmgr, PSMGCEDCDroopCtrlConfig_vega10);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_GCEDC)) {
+		vega10_program_gc_didt_config_registers(hwmgr, PSMGCEDCCtrlResetConfig_vega10);
+		vega10_program_gc_didt_config_registers(hwmgr, PSMGCEDCCtrlConfig_vega10);
+	}
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_PSM))
+		vega10_program_gc_didt_config_registers(hwmgr,  AvfsPSMInitConfig_vega10);
+
+	return 0;
+}
+
+static int vega10_disable_psm_gc_edc_config(struct pp_hwmgr *hwmgr)
+{
+	uint32_t data;
+
+	cgs_enter_safe_mode(hwmgr->device, true);
+
+	vega10_didt_set_mask(hwmgr, false);
+
+	cgs_enter_safe_mode(hwmgr->device, false);
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_GCEDC)) {
+		data = 0x00000000;
+		cgs_write_register(hwmgr->device, mmGC_EDC_CTRL, data);
+	}
+
+	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_PSM))
+		vega10_program_gc_didt_config_registers(hwmgr,  AvfsPSMResetConfig_vega10);
+
+	return 0;
+}
+
+static int vega10_enable_se_edc_force_stall_config(struct pp_hwmgr *hwmgr)
+{
+	uint32_t reg;
+	int result;
+
+	cgs_enter_safe_mode(hwmgr->device, true);
+
+	cgs_lock_grbm_idx(hwmgr->device, true);
+	reg = soc15_get_register_offset(GC_HWID, 0, mmGRBM_GFX_INDEX_BASE_IDX, mmGRBM_GFX_INDEX);
+	cgs_write_register(hwmgr->device, reg, 0xE0000000);
+	cgs_lock_grbm_idx(hwmgr->device, false);
+
+	result = vega10_program_didt_config_registers(hwmgr, SEEDCForceStallPatternConfig_Vega10, VEGA10_CONFIGREG_DIDT);
+	result |= vega10_program_didt_config_registers(hwmgr, SEEDCCtrlForceStallConfig_Vega10, VEGA10_CONFIGREG_DIDT);
+	if (0 != result)
+		return result;
+
+	vega10_didt_set_mask(hwmgr, false);
+
+	cgs_enter_safe_mode(hwmgr->device, false);
+
+	return 0;
+}
+
+static int vega10_disable_se_edc_force_stall_config(struct pp_hwmgr *hwmgr)
+{
+	int result;
+
+	result = vega10_disable_se_edc_config(hwmgr);
+	PP_ASSERT_WITH_CODE((0 == result), "[DisableDiDtConfig] Pre DIDT disable clock gating failed!", return result);
+
+	return 0;
+}
+
+int vega10_enable_didt_config(struct pp_hwmgr *hwmgr)
+{
+	int result = 0;
+	struct vega10_hwmgr *data = (struct vega10_hwmgr *)(hwmgr->backend);
+
+	if (data->smu_features[GNLD_DIDT].supported) {
+		if (data->smu_features[GNLD_DIDT].enabled)
+			PP_DBG_LOG("[EnableDiDtConfig] Feature DiDt Already enabled!\n");
+
+		switch (data->registry_data.didt_mode) {
+		case 0:
+			result = vega10_enable_cac_driving_se_didt_config(hwmgr);
+			PP_ASSERT_WITH_CODE((0 == result), "[EnableDiDt] Attempt to enable DiDt Mode 0 Failed!", return result);
+			break;
+		case 2:
+			result = vega10_enable_psm_gc_didt_config(hwmgr);
+			PP_ASSERT_WITH_CODE((0 == result), "[EnableDiDt] Attempt to enable DiDt Mode 2 Failed!", return result);
+			break;
+		case 3:
+			result = vega10_enable_se_edc_config(hwmgr);
+			PP_ASSERT_WITH_CODE((0 == result), "[EnableDiDt] Attempt to enable DiDt Mode 3 Failed!", return result);
+			break;
+		case 1:
+		case 4:
+		case 5:
+			result = vega10_enable_psm_gc_edc_config(hwmgr);
+			PP_ASSERT_WITH_CODE((0 == result), "[EnableDiDt] Attempt to enable DiDt Mode 5 Failed!", return result);
+			break;
+		case 6:
+			result = vega10_enable_se_edc_force_stall_config(hwmgr);
+			PP_ASSERT_WITH_CODE((0 == result), "[EnableDiDt] Attempt to enable DiDt Mode 6 Failed!", return result);
+			break;
+		default:
+			result = -EINVAL;
+			break;
+		}
+
+		if (0 == result) {
+			PP_ASSERT_WITH_CODE((!vega10_enable_smc_features(hwmgr->smumgr, true, data->smu_features[GNLD_DIDT].smu_feature_bitmap)),
+				"[EnableDiDtConfig] Attempt to Enable DiDt feature Failed!", return result);
+			data->smu_features[GNLD_DIDT].enabled = true;
+		}
+	}
+
+	return result;
+}
+
+int vega10_disable_didt_config(struct pp_hwmgr *hwmgr)
+{
+	int result = 0;
+	struct vega10_hwmgr *data = (struct vega10_hwmgr *)(hwmgr->backend);
+
+	if (data->smu_features[GNLD_DIDT].supported) {
+		if (!data->smu_features[GNLD_DIDT].enabled)
+			PP_DBG_LOG("[DisableDiDtConfig] Feature DiDt Already Disabled!\n");
+
+		switch (data->registry_data.didt_mode) {
+		case 0:
+			result = vega10_disable_cac_driving_se_didt_config(hwmgr);
+			PP_ASSERT_WITH_CODE((0 == result), "[DisableDiDt] Attempt to disable DiDt Mode 0 Failed!", return result);
+			break;
+		case 2:
+			result = vega10_disable_psm_gc_didt_config(hwmgr);
+			PP_ASSERT_WITH_CODE((0 == result), "[DisableDiDt] Attempt to disable DiDt Mode 2 Failed!", return result);
+			break;
+		case 3:
+			result = vega10_disable_se_edc_config(hwmgr);
+			PP_ASSERT_WITH_CODE((0 == result), "[DisableDiDt] Attempt to disable DiDt Mode 3 Failed!", return result);
+			break;
+		case 1:
+		case 4:
+		case 5:
+			result = vega10_disable_psm_gc_edc_config(hwmgr);
+			PP_ASSERT_WITH_CODE((0 == result), "[DisableDiDt] Attempt to disable DiDt Mode 5 Failed!", return result);
+			break;
+		case 6:
+			result = vega10_disable_se_edc_force_stall_config(hwmgr);
+			PP_ASSERT_WITH_CODE((0 == result), "[DisableDiDt] Attempt to disable DiDt Mode 6 Failed!", return result);
+			break;
+		default:
+			result = -EINVAL;
+			break;
+		}
+
+		if (0 == result) {
+			PP_ASSERT_WITH_CODE((0 != vega10_enable_smc_features(hwmgr->smumgr, false, data->smu_features[GNLD_DIDT].smu_feature_bitmap)),
+					"[DisableDiDtConfig] Attempt to Disable DiDt feature Failed!", return result);
+			data->smu_features[GNLD_DIDT].enabled = false;
+		}
+	}
+
+	return result;
+}
 
 void vega10_initialize_power_tune_defaults(struct pp_hwmgr *hwmgr)
 {
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_powertune.h b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_powertune.h
index 9ecaa27c0bb5..b95771ab89cd 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_powertune.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_powertune.h
@@ -31,6 +31,12 @@ enum vega10_pt_config_reg_type {
 	VEGA10_CONFIGREG_MAX
 };
 
+enum vega10_didt_config_reg_type {
+	VEGA10_CONFIGREG_DIDT = 0,
+	VEGA10_CONFIGREG_GCCAC,
+	VEGA10_CONFIGREG_SECAC
+};
+
 /* PowerContainment Features */
 #define POWERCONTAINMENT_FEATURE_DTE             0x00000001
 #define POWERCONTAINMENT_FEATURE_TDCLimit        0x00000002
@@ -44,6 +50,13 @@ struct vega10_pt_config_reg {
 	enum vega10_pt_config_reg_type       type;
 };
 
+struct vega10_didt_config_reg {
+	uint32_t		offset;
+	uint32_t		mask;
+	uint32_t		shift;
+	uint32_t		value;
+};
+
 struct vega10_pt_defaults {
     uint8_t   SviLoadLineEn;
     uint8_t   SviLoadLineVddC;
@@ -62,5 +75,8 @@ int vega10_set_power_limit(struct pp_hwmgr *hwmgr, uint32_t n);
 int vega10_power_control_set_level(struct pp_hwmgr *hwmgr);
 int vega10_disable_power_containment(struct pp_hwmgr *hwmgr);
 
+int vega10_enable_didt_config(struct pp_hwmgr *hwmgr);
+int vega10_disable_didt_config(struct pp_hwmgr *hwmgr);
+
 #endif  /* _VEGA10_POWERTUNE_H_ */
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_processpptables.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_processpptables.c
index 1623644ea49a..e343df190375 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_processpptables.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_processpptables.c
@@ -31,6 +31,8 @@
 #include "cgs_common.h"
 #include "vega10_pptable.h"
 
+#define NUM_DSPCLK_LEVELS 8
+
 static void set_hw_cap(struct pp_hwmgr *hwmgr, bool enable,
 		enum phm_platform_caps cap)
 {
@@ -644,11 +646,11 @@ static int get_gfxclk_voltage_dependency_table(
 	return 0;
 }
 
-static int get_dcefclk_voltage_dependency_table(
+static int get_pix_clk_voltage_dependency_table(
 		struct pp_hwmgr *hwmgr,
 		struct phm_ppt_v1_clock_voltage_dependency_table
 			**pp_vega10_clk_dep_table,
-		const ATOM_Vega10_DCEFCLK_Dependency_Table *clk_dep_table)
+		const  ATOM_Vega10_PIXCLK_Dependency_Table *clk_dep_table)
 {
 	uint32_t table_size, i;
 	struct phm_ppt_v1_clock_voltage_dependency_table
@@ -681,6 +683,76 @@ static int get_dcefclk_voltage_dependency_table(
 	return 0;
 }
 
+static int get_dcefclk_voltage_dependency_table(
+		struct pp_hwmgr *hwmgr,
+		struct phm_ppt_v1_clock_voltage_dependency_table
+			**pp_vega10_clk_dep_table,
+		const ATOM_Vega10_DCEFCLK_Dependency_Table *clk_dep_table)
+{
+	uint32_t table_size, i;
+	uint8_t num_entries;
+	struct phm_ppt_v1_clock_voltage_dependency_table
+				*clk_table;
+	struct cgs_system_info sys_info = {0};
+	uint32_t dev_id;
+	uint32_t rev_id;
+
+	PP_ASSERT_WITH_CODE((clk_dep_table->ucNumEntries != 0),
+			"Invalid PowerPlay Table!", return -1);
+
+/*
+ * workaround needed to add another DPM level for pioneer cards
+ * as VBIOS is locked down.
+ * This DPM level was added to support 3DPM monitors @ 4K120Hz
+ *
+ */
+	sys_info.size = sizeof(struct cgs_system_info);
+	sys_info.info_id = CGS_SYSTEM_INFO_PCIE_DEV;
+	cgs_query_system_info(hwmgr->device, &sys_info);
+	dev_id = (uint32_t)sys_info.value;
+
+	sys_info.size = sizeof(struct cgs_system_info);
+	sys_info.info_id = CGS_SYSTEM_INFO_PCIE_REV;
+	cgs_query_system_info(hwmgr->device, &sys_info);
+	rev_id = (uint32_t)sys_info.value;
+
+	if (dev_id == 0x6863 && rev_id == 0 &&
+		clk_dep_table->entries[clk_dep_table->ucNumEntries - 1].ulClk < 90000)
+		num_entries = clk_dep_table->ucNumEntries + 1 > NUM_DSPCLK_LEVELS ?
+				NUM_DSPCLK_LEVELS : clk_dep_table->ucNumEntries + 1;
+	else
+		num_entries = clk_dep_table->ucNumEntries;
+
+
+	table_size = sizeof(uint32_t) +
+			sizeof(phm_ppt_v1_clock_voltage_dependency_record) *
+			num_entries;
+
+	clk_table = (struct phm_ppt_v1_clock_voltage_dependency_table *)
+			kzalloc(table_size, GFP_KERNEL);
+
+	if (!clk_table)
+		return -ENOMEM;
+
+	clk_table->count = (uint32_t)num_entries;
+
+	for (i = 0; i < clk_dep_table->ucNumEntries; i++) {
+		clk_table->entries[i].vddInd =
+				clk_dep_table->entries[i].ucVddInd;
+		clk_table->entries[i].clk =
+				le32_to_cpu(clk_dep_table->entries[i].ulClk);
+	}
+
+	if (i < num_entries) {
+		clk_table->entries[i].vddInd = clk_dep_table->entries[i-1].ucVddInd;
+		clk_table->entries[i].clk = 90000;
+	}
+
+	*pp_vega10_clk_dep_table = clk_table;
+
+	return 0;
+}
+
 static int get_pcie_table(struct pp_hwmgr *hwmgr,
 		struct phm_ppt_v1_pcie_table **vega10_pcie_table,
 		const Vega10_PPTable_Generic_SubTable_Header *table)
@@ -862,21 +934,21 @@ static int init_powerplay_extended_tables(
 				gfxclk_dep_table);
 
 	if (!result && powerplay_table->usPixclkDependencyTableOffset)
-		result = get_dcefclk_voltage_dependency_table(hwmgr,
+		result = get_pix_clk_voltage_dependency_table(hwmgr,
 				&pp_table_info->vdd_dep_on_pixclk,
-				(const ATOM_Vega10_DCEFCLK_Dependency_Table*)
+				(const ATOM_Vega10_PIXCLK_Dependency_Table*)
 				pixclk_dep_table);
 
 	if (!result && powerplay_table->usPhyClkDependencyTableOffset)
-		result = get_dcefclk_voltage_dependency_table(hwmgr,
+		result = get_pix_clk_voltage_dependency_table(hwmgr,
 				&pp_table_info->vdd_dep_on_phyclk,
-				(const ATOM_Vega10_DCEFCLK_Dependency_Table *)
+				(const ATOM_Vega10_PIXCLK_Dependency_Table *)
 				phyclk_dep_table);
 
 	if (!result && powerplay_table->usDispClkDependencyTableOffset)
-		result = get_dcefclk_voltage_dependency_table(hwmgr,
+		result = get_pix_clk_voltage_dependency_table(hwmgr,
 				&pp_table_info->vdd_dep_on_dispclk,
-				(const ATOM_Vega10_DCEFCLK_Dependency_Table *)
+				(const ATOM_Vega10_PIXCLK_Dependency_Table *)
 				dispclk_dep_table);
 
 	if (!result && powerplay_table->usDcefclkDependencyTableOffset)
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_thermal.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_thermal.c
index e7ab8eb8a0cf..d44243441d28 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_thermal.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_thermal.c
@@ -321,10 +321,7 @@ int vega10_fan_ctrl_reset_fan_speed_to_default(struct pp_hwmgr *hwmgr)
 
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_MicrocodeFanControl)) {
-		result = vega10_fan_ctrl_set_static_mode(hwmgr,
-				FDO_PWM_MODE_STATIC);
-		if (!result)
-			result = vega10_fan_ctrl_start_smc_fan_control(hwmgr);
+		result = vega10_fan_ctrl_start_smc_fan_control(hwmgr);
 	} else
 		result = vega10_fan_ctrl_set_default_mode(hwmgr);
 
@@ -633,7 +630,6 @@ int tf_vega10_thermal_start_smc_fan_control(struct pp_hwmgr *hwmgr,
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_MicrocodeFanControl)) {
 		vega10_fan_ctrl_start_smc_fan_control(hwmgr);
-		vega10_fan_ctrl_set_static_mode(hwmgr, FDO_PWM_MODE_STATIC);
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h b/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h
index a1ebe1014492..a4c8b09b6f14 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h
@@ -164,9 +164,14 @@ enum phm_platform_caps {
 	PHM_PlatformCaps_EnablePlatformPowerManagement,         /* indicates that Platform Power Management feature is supported */
 	PHM_PlatformCaps_SurpriseRemoval,                       /* indicates that surprise removal feature is requested */
 	PHM_PlatformCaps_NewCACVoltage,                         /* indicates new CAC voltage table support */
+	PHM_PlatformCaps_DiDtSupport,                           /* for dI/dT feature */
 	PHM_PlatformCaps_DBRamping,                             /* for dI/dT feature */
 	PHM_PlatformCaps_TDRamping,                             /* for dI/dT feature */
 	PHM_PlatformCaps_TCPRamping,                            /* for dI/dT feature */
+	PHM_PlatformCaps_DBRRamping,                            /* for dI/dT feature */
+	PHM_PlatformCaps_DiDtEDCEnable,                         /* for dI/dT feature */
+	PHM_PlatformCaps_GCEDC,                                 /* for dI/dT feature */
+	PHM_PlatformCaps_PSM,                                   /* for dI/dT feature */
 	PHM_PlatformCaps_EnableSMU7ThermalManagement,           /* SMC will manage thermal events */
 	PHM_PlatformCaps_FPS,                                   /* FPS support */
 	PHM_PlatformCaps_ACP,                                   /* ACP support */
diff --git a/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h b/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
index 47e57bd2c36f..91b0105e8240 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
@@ -128,6 +128,8 @@ struct phm_uvd_arbiter {
 	uint32_t dclk;
 	uint32_t vclk_ceiling;
 	uint32_t dclk_ceiling;
+	uint32_t vclk_soft_min;
+	uint32_t dclk_soft_min;
 };
 
 struct phm_vce_arbiter {
diff --git a/drivers/gpu/drm/amd/powerplay/inc/pp_debug.h b/drivers/gpu/drm/amd/powerplay/inc/pp_debug.h
index f3f9ebb631a5..822cd8b5bf90 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/pp_debug.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/pp_debug.h
@@ -42,6 +42,12 @@
 		}				\
 	} while (0)
 
+#define PP_ASSERT(cond, msg)	\
+	do {					\
+		if (!(cond)) {			\
+			pr_warn("%s\n", msg);	\
+		}				\
+	} while (0)
 
 #define PP_DBG_LOG(fmt, ...) \
 	do { \
diff --git a/drivers/gpu/drm/amd/powerplay/inc/pp_soc15.h b/drivers/gpu/drm/amd/powerplay/inc/pp_soc15.h
index 227d999b6bd1..a511611ec7e0 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/pp_soc15.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/pp_soc15.h
@@ -41,6 +41,8 @@ inline static uint32_t soc15_get_register_offset(
 		reg = MP1_BASE.instance[inst].segment[segment] + offset;
 	else if (hw_id == DF_HWID)
 		reg = DF_BASE.instance[inst].segment[segment] + offset;
+	else if (hw_id == GC_HWID)
+		reg = GC_BASE.instance[inst].segment[segment] + offset;
 
 	return reg;
 }
diff --git a/drivers/gpu/drm/amd/powerplay/inc/rv_ppsmc.h b/drivers/gpu/drm/amd/powerplay/inc/rv_ppsmc.h
index e0e106f1b23a..901c960cfe21 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/rv_ppsmc.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/rv_ppsmc.h
@@ -66,7 +66,12 @@
 #define PPSMC_MSG_SetMinVddcrSocVoltage         0x22
 #define PPSMC_MSG_SetMinVideoFclkFreq           0x23
 #define PPSMC_MSG_SetMinDeepSleepDcefclk        0x24
-#define PPSMC_Message_Count                     0x25
+#define PPSMC_MSG_ForcePowerDownGfx             0x25
+#define PPSMC_MSG_SetPhyclkVoltageByFreq        0x26
+#define PPSMC_MSG_SetDppclkVoltageByFreq        0x27
+#define PPSMC_MSG_SetSoftMinVcn                 0x28
+#define PPSMC_Message_Count                     0x29
+
 
 typedef uint16_t PPSMC_Result;
 typedef int      PPSMC_Msg;
diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu9.h b/drivers/gpu/drm/amd/powerplay/inc/smu9.h
index 9ef2490c7c2e..550ed675027a 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/smu9.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/smu9.h
@@ -55,9 +55,9 @@
 #define FEATURE_FW_CTF_BIT              23
 #define FEATURE_LED_DISPLAY_BIT         24
 #define FEATURE_FAN_CONTROL_BIT         25
-#define FEATURE_VOLTAGE_CONTROLLER_BIT  26
-#define FEATURE_SPARE_27_BIT            27
-#define FEATURE_SPARE_28_BIT            28
+#define FEATURE_FAST_PPT_BIT            26
+#define FEATURE_GFX_EDC_BIT             27
+#define FEATURE_ACG_BIT                 28
 #define FEATURE_SPARE_29_BIT            29
 #define FEATURE_SPARE_30_BIT            30
 #define FEATURE_SPARE_31_BIT            31
@@ -90,9 +90,10 @@
 #define FFEATURE_FW_CTF_MASK             (1 << FEATURE_FW_CTF_BIT             )
 #define FFEATURE_LED_DISPLAY_MASK        (1 << FEATURE_LED_DISPLAY_BIT        )
 #define FFEATURE_FAN_CONTROL_MASK        (1 << FEATURE_FAN_CONTROL_BIT        )
-#define FFEATURE_VOLTAGE_CONTROLLER_MASK (1 << FEATURE_VOLTAGE_CONTROLLER_BIT )
-#define FFEATURE_SPARE_27_MASK           (1 << FEATURE_SPARE_27_BIT           )
-#define FFEATURE_SPARE_28_MASK           (1 << FEATURE_SPARE_28_BIT           )
+
+#define FEATURE_FAST_PPT_MASK            (1 << FAST_PPT_BIT                   )
+#define FEATURE_GFX_EDC_MASK             (1 << FEATURE_GFX_EDC_BIT            )
+#define FEATURE_ACG_MASK                 (1 << FEATURE_ACG_BIT                )
 #define FFEATURE_SPARE_29_MASK           (1 << FEATURE_SPARE_29_BIT           )
 #define FFEATURE_SPARE_30_MASK           (1 << FEATURE_SPARE_30_BIT           )
 #define FFEATURE_SPARE_31_MASK           (1 << FEATURE_SPARE_31_BIT           )
diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu9_driver_if.h b/drivers/gpu/drm/amd/powerplay/inc/smu9_driver_if.h
index 532186b6f941..2818c98ff5ca 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/smu9_driver_if.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/smu9_driver_if.h
@@ -312,10 +312,15 @@ typedef struct {
 
   PllSetting_t GfxBoostState;
 
-  uint32_t     Reserved[14];
+  uint8_t      AcgEnable[NUM_GFXCLK_DPM_LEVELS];
+  GbVdroopTable_t AcgBtcGbVdroopTable;
+  QuadraticInt_t  AcgAvfsGb;
+
+  /* ACG Frequency Table, in Mhz */
+  uint32_t     AcgFreqTable[NUM_GFXCLK_DPM_LEVELS];
 
   /* Padding - ignore */
-  uint32_t     MmHubPadding[7]; /* SMU internal use */
+  uint32_t     MmHubPadding[3]; /* SMU internal use */
 
 } PPTable_t;
 
diff --git a/drivers/gpu/drm/amd/powerplay/inc/smumgr.h b/drivers/gpu/drm/amd/powerplay/inc/smumgr.h
index 976e942ec694..5d61cc9d4554 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/smumgr.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/smumgr.h
@@ -131,6 +131,7 @@ struct pp_smumgr_func {
 	bool (*is_dpm_running)(struct pp_hwmgr *hwmgr);
 	int (*populate_requested_graphic_levels)(struct pp_hwmgr *hwmgr,
 			struct amd_pp_profile *request);
+	bool (*is_hw_avfs_present)(struct pp_smumgr *smumgr);
 };
 
 struct pp_smumgr {
@@ -202,6 +203,8 @@ extern bool smum_is_dpm_running(struct pp_hwmgr *hwmgr);
 extern int smum_populate_requested_graphic_levels(struct pp_hwmgr *hwmgr,
 		struct amd_pp_profile *request);
 
+extern bool smum_is_hw_avfs_present(struct pp_smumgr *smumgr);
+
 #define SMUM_FIELD_SHIFT(reg, field) reg##__##field##__SHIFT
 
 #define SMUM_FIELD_MASK(reg, field) reg##__##field##_MASK
diff --git a/drivers/gpu/drm/amd/powerplay/inc/vega10_ppsmc.h b/drivers/gpu/drm/amd/powerplay/inc/vega10_ppsmc.h
index b4af9e85dfa5..cb070ebc7de1 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/vega10_ppsmc.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/vega10_ppsmc.h
@@ -124,6 +124,10 @@ typedef uint16_t PPSMC_Result;
 #define PPSMC_MSG_NumOfDisplays                  0x56
 #define PPSMC_MSG_ReadSerialNumTop32             0x58
 #define PPSMC_MSG_ReadSerialNumBottom32          0x59
+#define PPSMC_MSG_RunAcgBtc                      0x5C
+#define PPSMC_MSG_RunAcgInClosedLoop             0x5D
+#define PPSMC_MSG_RunAcgInOpenLoop               0x5E
+#define PPSMC_MSG_InitializeAcg                  0x5F
 #define PPSMC_MSG_GetCurrPkgPwr                  0x61
 #define PPSMC_Message_Count                      0x62
 
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.c b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.c
index 6a320b27aefd..8712f093d6d9 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.c
@@ -2129,6 +2129,25 @@ int fiji_thermal_setup_fan_table(struct pp_hwmgr *hwmgr)
 	return 0;
 }
 
+
+int fiji_thermal_avfs_enable(struct pp_hwmgr *hwmgr)
+{
+	int ret;
+	struct pp_smumgr *smumgr = (struct pp_smumgr *)(hwmgr->smumgr);
+	struct smu7_smumgr *smu_data = (struct smu7_smumgr *)(smumgr->backend);
+
+	if (smu_data->avfs.avfs_btc_status != AVFS_BTC_ENABLEAVFS)
+		return 0;
+
+	ret = smum_send_msg_to_smc(smumgr, PPSMC_MSG_EnableAvfs);
+
+	if (!ret)
+		/* If this param is not changed, this function could fire unnecessarily */
+		smu_data->avfs.avfs_btc_status = AVFS_BTC_COMPLETED_PREVIOUSLY;
+
+	return ret;
+}
+
 static int fiji_program_mem_timing_parameters(struct pp_hwmgr *hwmgr)
 {
 	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.h b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.h
index 0e9e1f2d7238..d9c72d992e30 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.h
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smc.h
@@ -48,5 +48,6 @@ int fiji_initialize_mc_reg_table(struct pp_hwmgr *hwmgr);
 bool fiji_is_dpm_running(struct pp_hwmgr *hwmgr);
 int fiji_populate_requested_graphic_levels(struct pp_hwmgr *hwmgr,
 		struct amd_pp_profile *request);
+int fiji_thermal_avfs_enable(struct pp_hwmgr *hwmgr);
 #endif
 
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c
index a1cb78552cf6..6ae948fc524f 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c
@@ -161,56 +161,47 @@ static int fiji_start_smu_in_non_protection_mode(struct pp_smumgr *smumgr)
 
 static int fiji_setup_pwr_virus(struct pp_smumgr *smumgr)
 {
-	int i, result = -1;
+	int i;
+	int result = -EINVAL;
 	uint32_t reg, data;
-	const PWR_Command_Table *virus = PwrVirusTable;
-	struct fiji_smumgr *priv = (struct fiji_smumgr *)(smumgr->backend);
 
-	priv->avfs.AvfsBtcStatus = AVFS_LOAD_VIRUS;
-	for (i = 0; (i < PWR_VIRUS_TABLE_SIZE); i++) {
-		switch (virus->command) {
+	const PWR_Command_Table *pvirus = PwrVirusTable;
+	struct smu7_smumgr *smu_data = (struct smu7_smumgr *)(smumgr->backend);
+
+	for (i = 0; i < PWR_VIRUS_TABLE_SIZE; i++) {
+		switch (pvirus->command) {
 		case PwrCmdWrite:
-			reg  = virus->reg;
-			data = virus->data;
+			reg  = pvirus->reg;
+			data = pvirus->data;
 			cgs_write_register(smumgr->device, reg, data);
 			break;
+
 		case PwrCmdEnd:
-			priv->avfs.AvfsBtcStatus = AVFS_BTC_VIRUS_LOADED;
 			result = 0;
 			break;
+
 		default:
-			pr_err("Table Exit with Invalid Command!");
-			priv->avfs.AvfsBtcStatus = AVFS_BTC_VIRUS_FAIL;
-			result = -1;
+			pr_info("Table Exit with Invalid Command!");
+			smu_data->avfs.avfs_btc_status = AVFS_BTC_VIRUS_FAIL;
+			result = -EINVAL;
 			break;
 		}
-		virus++;
+		pvirus++;
 	}
+
 	return result;
 }
 
 static int fiji_start_avfs_btc(struct pp_smumgr *smumgr)
 {
 	int result = 0;
-	struct fiji_smumgr *priv = (struct fiji_smumgr *)(smumgr->backend);
+	struct smu7_smumgr *smu_data = (struct smu7_smumgr *)(smumgr->backend);
 
-	priv->avfs.AvfsBtcStatus = AVFS_BTC_STARTED;
-	if (priv->avfs.AvfsBtcParam) {
-		if (!smum_send_msg_to_smc_with_parameter(smumgr,
-				PPSMC_MSG_PerformBtc, priv->avfs.AvfsBtcParam)) {
-			if (!smum_send_msg_to_smc(smumgr, PPSMC_MSG_EnableAvfs)) {
-				priv->avfs.AvfsBtcStatus = AVFS_BTC_COMPLETED_UNSAVED;
-				result = 0;
-			} else {
-				pr_err("[AVFS][fiji_start_avfs_btc] Attempt"
-						" to Enable AVFS Failed!");
-				smum_send_msg_to_smc(smumgr, PPSMC_MSG_DisableAvfs);
-				result = -1;
-			}
-		} else {
-			pr_err("[AVFS][fiji_start_avfs_btc] "
-					"PerformBTC SMU msg failed");
-			result = -1;
+	if (0 != smu_data->avfs.avfs_btc_param) {
+		if (0 != smu7_send_msg_to_smc_with_parameter(smumgr,
+				PPSMC_MSG_PerformBtc, smu_data->avfs.avfs_btc_param)) {
+			pr_info("[AVFS][Fiji_PerformBtc] PerformBTC SMU msg failed");
+			result = -EINVAL;
 		}
 	}
 	/* Soft-Reset to reset the engine before loading uCode */
@@ -224,42 +215,6 @@ static int fiji_start_avfs_btc(struct pp_smumgr *smumgr)
 	return result;
 }
 
-static int fiji_setup_pm_fuse_for_avfs(struct pp_smumgr *smumgr)
-{
-	int result = 0;
-	uint32_t table_start;
-	uint32_t charz_freq_addr, inversion_voltage_addr, charz_freq;
-	uint16_t inversion_voltage;
-
-	charz_freq = 0x30750000; /* In 10KHz units 0x00007530 Actual value */
-	inversion_voltage = 0x1A04; /* mV Q14.2 0x41A Actual value */
-
-	PP_ASSERT_WITH_CODE(0 == smu7_read_smc_sram_dword(smumgr,
-			SMU7_FIRMWARE_HEADER_LOCATION + offsetof(SMU73_Firmware_Header,
-					PmFuseTable), &table_start, 0x40000),
-			"[AVFS][Fiji_SetupGfxLvlStruct] SMU could not communicate "
-			"starting address of PmFuse structure",
-			return -1;);
-
-	charz_freq_addr = table_start +
-			offsetof(struct SMU73_Discrete_PmFuses, PsmCharzFreq);
-	inversion_voltage_addr = table_start +
-			offsetof(struct SMU73_Discrete_PmFuses, InversionVoltage);
-
-	result = smu7_copy_bytes_to_smc(smumgr, charz_freq_addr,
-			(uint8_t *)(&charz_freq), sizeof(charz_freq), 0x40000);
-	PP_ASSERT_WITH_CODE(0 == result,
-			"[AVFS][fiji_setup_pm_fuse_for_avfs] charz_freq could not "
-			"be populated.", return -1;);
-
-	result = smu7_copy_bytes_to_smc(smumgr, inversion_voltage_addr,
-			(uint8_t *)(&inversion_voltage), sizeof(inversion_voltage), 0x40000);
-	PP_ASSERT_WITH_CODE(0 == result, "[AVFS][fiji_setup_pm_fuse_for_avfs] "
-			"charz_freq could not be populated.", return -1;);
-
-	return result;
-}
-
 static int fiji_setup_graphics_level_structure(struct pp_smumgr *smumgr)
 {
 	int32_t vr_config;
@@ -298,93 +253,41 @@ static int fiji_setup_graphics_level_structure(struct pp_smumgr *smumgr)
 	return 0;
 }
 
-/* Work in Progress */
-static int fiji_restore_vft_table(struct pp_smumgr *smumgr)
-{
-	struct fiji_smumgr *priv = (struct fiji_smumgr *)(smumgr->backend);
-
-	if (AVFS_BTC_COMPLETED_SAVED == priv->avfs.AvfsBtcStatus) {
-		priv->avfs.AvfsBtcStatus = AVFS_BTC_COMPLETED_RESTORED;
-		return 0;
-	} else
-		return -EINVAL;
-}
-
-/* Work in Progress */
-static int fiji_save_vft_table(struct pp_smumgr *smumgr)
-{
-	struct fiji_smumgr *priv = (struct fiji_smumgr *)(smumgr->backend);
-
-	if (AVFS_BTC_COMPLETED_SAVED == priv->avfs.AvfsBtcStatus) {
-		priv->avfs.AvfsBtcStatus = AVFS_BTC_COMPLETED_RESTORED;
-		return 0;
-	} else
-		return -EINVAL;
-}
-
 static int fiji_avfs_event_mgr(struct pp_smumgr *smumgr, bool smu_started)
 {
-	struct fiji_smumgr *priv = (struct fiji_smumgr *)(smumgr->backend);
+	struct smu7_smumgr *smu_data = (struct smu7_smumgr *)(smumgr->backend);
 
-	switch (priv->avfs.AvfsBtcStatus) {
-	case AVFS_BTC_COMPLETED_SAVED: /*S3 State - Pre SMU Start */
-		priv->avfs.AvfsBtcStatus = AVFS_BTC_RESTOREVFT_FAILED;
-		PP_ASSERT_WITH_CODE(0 == fiji_restore_vft_table(smumgr),
-				"[AVFS][fiji_avfs_event_mgr] Could not Copy Graphics "
-				"Level table over to SMU",
-				return -1;);
-		priv->avfs.AvfsBtcStatus = AVFS_BTC_COMPLETED_RESTORED;
-		break;
-	case AVFS_BTC_COMPLETED_RESTORED: /*S3 State - Post SMU Start*/
-		priv->avfs.AvfsBtcStatus = AVFS_BTC_SMUMSG_ERROR;
-		PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(smumgr,
-				0x666),
-				"[AVFS][fiji_avfs_event_mgr] SMU did not respond "
-				"correctly to VftTableIsValid Msg",
-				return -1;);
-		priv->avfs.AvfsBtcStatus = AVFS_BTC_SMUMSG_ERROR;
-		PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(smumgr,
-				PPSMC_MSG_EnableAvfs),
-				"[AVFS][fiji_avfs_event_mgr] SMU did not respond "
-				"correctly to EnableAvfs Message Msg",
-				return -1;);
-		priv->avfs.AvfsBtcStatus = AVFS_BTC_COMPLETED_SAVED;
+	switch (smu_data->avfs.avfs_btc_status) {
+	case AVFS_BTC_COMPLETED_PREVIOUSLY:
 		break;
+
 	case AVFS_BTC_BOOT: /*Cold Boot State - Post SMU Start*/
 		if (!smu_started)
 			break;
-		priv->avfs.AvfsBtcStatus = AVFS_BTC_FAILED;
-		PP_ASSERT_WITH_CODE(0 == fiji_setup_pm_fuse_for_avfs(smumgr),
-				"[AVFS][fiji_avfs_event_mgr] Failure at "
-				"fiji_setup_pm_fuse_for_avfs",
-				return -1;);
-		priv->avfs.AvfsBtcStatus = AVFS_BTC_DPMTABLESETUP_FAILED;
+		smu_data->avfs.avfs_btc_status = AVFS_BTC_FAILED;
 		PP_ASSERT_WITH_CODE(0 == fiji_setup_graphics_level_structure(smumgr),
 				"[AVFS][fiji_avfs_event_mgr] Could not Copy Graphics Level"
 				" table over to SMU",
-				return -1;);
-		priv->avfs.AvfsBtcStatus = AVFS_BTC_VIRUS_FAIL;
+				return -EINVAL;);
+		smu_data->avfs.avfs_btc_status = AVFS_BTC_VIRUS_FAIL;
 		PP_ASSERT_WITH_CODE(0 == fiji_setup_pwr_virus(smumgr),
 				"[AVFS][fiji_avfs_event_mgr] Could not setup "
 				"Pwr Virus for AVFS ",
-				return -1;);
-		priv->avfs.AvfsBtcStatus = AVFS_BTC_FAILED;
+				return -EINVAL;);
+		smu_data->avfs.avfs_btc_status = AVFS_BTC_FAILED;
 		PP_ASSERT_WITH_CODE(0 == fiji_start_avfs_btc(smumgr),
 				"[AVFS][fiji_avfs_event_mgr] Failure at "
 				"fiji_start_avfs_btc. AVFS Disabled",
-				return -1;);
-		priv->avfs.AvfsBtcStatus = AVFS_BTC_SAVEVFT_FAILED;
-		PP_ASSERT_WITH_CODE(0 == fiji_save_vft_table(smumgr),
-				"[AVFS][fiji_avfs_event_mgr] Could not save VFT Table",
-				return -1;);
-		priv->avfs.AvfsBtcStatus = AVFS_BTC_COMPLETED_SAVED;
+				return -EINVAL;);
+
+		smu_data->avfs.avfs_btc_status = AVFS_BTC_ENABLEAVFS;
 		break;
 	case AVFS_BTC_DISABLED: /* Do nothing */
-		break;
 	case AVFS_BTC_NOTSUPPORTED: /* Do nothing */
+	case AVFS_BTC_ENABLEAVFS:
 		break;
 	default:
-		pr_err("[AVFS] Something is broken. See log!");
+		pr_err("AVFS failed status is %x !\n", smu_data->avfs.avfs_btc_status);
 		break;
 	}
 	return 0;
@@ -477,19 +380,6 @@ static int fiji_smu_init(struct pp_smumgr *smumgr)
 	if (smu7_init(smumgr))
 		return -EINVAL;
 
-	fiji_priv->avfs.AvfsBtcStatus = AVFS_BTC_BOOT;
-	if (fiji_is_hw_avfs_present(smumgr))
-		/* AVFS Parameter
-		 * 0 - BTC DC disabled, BTC AC disabled
-		 * 1 - BTC DC enabled,  BTC AC disabled
-		 * 2 - BTC DC disabled, BTC AC enabled
-		 * 3 - BTC DC enabled,  BTC AC enabled
-		 * Default is 0 - BTC DC disabled, BTC AC disabled
-		 */
-		fiji_priv->avfs.AvfsBtcParam = 0;
-	else
-		fiji_priv->avfs.AvfsBtcStatus = AVFS_BTC_NOTSUPPORTED;
-
 	for (i = 0; i < SMU73_MAX_LEVELS_GRAPHICS; i++)
 		fiji_priv->activity_target[i] = 30;
 
@@ -514,10 +404,12 @@ const struct pp_smumgr_func fiji_smu_funcs = {
 	.init_smc_table = fiji_init_smc_table,
 	.update_sclk_threshold = fiji_update_sclk_threshold,
 	.thermal_setup_fan_table = fiji_thermal_setup_fan_table,
+	.thermal_avfs_enable = fiji_thermal_avfs_enable,
 	.populate_all_graphic_levels = fiji_populate_all_graphic_levels,
 	.populate_all_memory_levels = fiji_populate_all_memory_levels,
 	.get_mac_definition = fiji_get_mac_definition,
 	.initialize_mc_reg_table = fiji_initialize_mc_reg_table,
 	.is_dpm_running = fiji_is_dpm_running,
 	.populate_requested_graphic_levels = fiji_populate_requested_graphic_levels,
+	.is_hw_avfs_present = fiji_is_hw_avfs_present,
 };
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.h b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.h
index adcbdfb209be..175bf9f8ef9c 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.h
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.h
@@ -28,17 +28,8 @@
 #include "smu7_smumgr.h"
 
 
-
-struct fiji_smu_avfs {
-	enum AVFS_BTC_STATUS AvfsBtcStatus;
-	uint32_t           AvfsBtcParam;
-};
-
-
 struct fiji_smumgr {
 	struct smu7_smumgr                   smu7_data;
-
-	struct fiji_smu_avfs avfs;
 	struct SMU73_Discrete_DpmTable       smc_state_table;
 	struct SMU73_Discrete_Ulv            ulv_setting;
 	struct SMU73_Discrete_PmFuses  power_tune_table;
@@ -47,7 +38,5 @@ struct fiji_smumgr {
 
 };
 
-
-
 #endif
 
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smc.c b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smc.c
index f68e759e8be2..99a00bd39256 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smc.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smc.c
@@ -1498,7 +1498,7 @@ static int polaris10_populate_avfs_parameters(struct pp_hwmgr *hwmgr)
 			table_info->vdd_dep_on_sclk;
 
 
-	if (smu_data->avfs.avfs_btc_status == AVFS_BTC_NOTSUPPORTED)
+	if (((struct smu7_smumgr *)smu_data)->avfs.avfs_btc_status == AVFS_BTC_NOTSUPPORTED)
 		return result;
 
 	result = atomctrl_get_avfs_information(hwmgr, &avfs_params);
@@ -1889,7 +1889,7 @@ int polaris10_thermal_avfs_enable(struct pp_hwmgr *hwmgr)
 {
 	int ret;
 	struct pp_smumgr *smumgr = (struct pp_smumgr *)(hwmgr->smumgr);
-	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend);
+	struct smu7_smumgr *smu_data = (struct smu7_smumgr *)(smumgr->backend);
 	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
 
 	if (smu_data->avfs.avfs_btc_status == AVFS_BTC_NOTSUPPORTED)
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c
index 9616cedc139c..75f43dadc56b 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c
@@ -60,16 +60,14 @@ static const SMU74_Discrete_GraphicsLevel avfs_graphics_level_polaris10[8] = {
 static const SMU74_Discrete_MemoryLevel avfs_memory_level_polaris10 = {
 	0x100ea446, 0, 0x30750000, 0x01, 0x01, 0x01, 0x00, 0x00, 0x64, 0x00, 0x00, 0x1f00, 0x00, 0x00};
 
-
 static int polaris10_setup_pwr_virus(struct pp_smumgr *smumgr)
 {
 	int i;
-	int result = -1;
+	int result = -EINVAL;
 	uint32_t reg, data;
 
 	const PWR_Command_Table *pvirus = pwr_virus_table;
-	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend);
-
+	struct smu7_smumgr *smu_data = (struct smu7_smumgr *)(smumgr->backend);
 
 	for (i = 0; i < PWR_VIRUS_TABLE_SIZE; i++) {
 		switch (pvirus->command) {
@@ -86,7 +84,7 @@ static int polaris10_setup_pwr_virus(struct pp_smumgr *smumgr)
 		default:
 			pr_info("Table Exit with Invalid Command!");
 			smu_data->avfs.avfs_btc_status = AVFS_BTC_VIRUS_FAIL;
-			result = -1;
+			result = -EINVAL;
 			break;
 		}
 		pvirus++;
@@ -98,7 +96,7 @@ static int polaris10_setup_pwr_virus(struct pp_smumgr *smumgr)
 static int polaris10_perform_btc(struct pp_smumgr *smumgr)
 {
 	int result = 0;
-	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend);
+	struct smu7_smumgr *smu_data = (struct smu7_smumgr *)(smumgr->backend);
 
 	if (0 != smu_data->avfs.avfs_btc_param) {
 		if (0 != smu7_send_msg_to_smc_with_parameter(smumgr, PPSMC_MSG_PerformBtc, smu_data->avfs.avfs_btc_param)) {
@@ -172,10 +170,11 @@ static int polaris10_setup_graphics_level_structure(struct pp_smumgr *smumgr)
 	return 0;
 }
 
+
 static int
 polaris10_avfs_event_mgr(struct pp_smumgr *smumgr, bool SMU_VFT_INTACT)
 {
-	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(smumgr->backend);
+	struct smu7_smumgr *smu_data = (struct smu7_smumgr *)(smumgr->backend);
 
 	switch (smu_data->avfs.avfs_btc_status) {
 	case AVFS_BTC_COMPLETED_PREVIOUSLY:
@@ -185,30 +184,31 @@ polaris10_avfs_event_mgr(struct pp_smumgr *smumgr, bool SMU_VFT_INTACT)
 
 		smu_data->avfs.avfs_btc_status = AVFS_BTC_DPMTABLESETUP_FAILED;
 		PP_ASSERT_WITH_CODE(0 == polaris10_setup_graphics_level_structure(smumgr),
-		"[AVFS][Polaris10_AVFSEventMgr] Could not Copy Graphics Level table over to SMU",
-		return -1);
+			"[AVFS][Polaris10_AVFSEventMgr] Could not Copy Graphics Level table over to SMU",
+			return -EINVAL);
 
 		if (smu_data->avfs.avfs_btc_param > 1) {
 			pr_info("[AVFS][Polaris10_AVFSEventMgr] AC BTC has not been successfully verified on Fiji. There may be in this setting.");
 			smu_data->avfs.avfs_btc_status = AVFS_BTC_VIRUS_FAIL;
-			PP_ASSERT_WITH_CODE(-1 == polaris10_setup_pwr_virus(smumgr),
+			PP_ASSERT_WITH_CODE(0 == polaris10_setup_pwr_virus(smumgr),
 			"[AVFS][Polaris10_AVFSEventMgr] Could not setup Pwr Virus for AVFS ",
-			return -1);
+			return -EINVAL);
 		}
 
 		smu_data->avfs.avfs_btc_status = AVFS_BTC_FAILED;
 		PP_ASSERT_WITH_CODE(0 == polaris10_perform_btc(smumgr),
 					"[AVFS][Polaris10_AVFSEventMgr] Failure at SmuPolaris10_PerformBTC. AVFS Disabled",
-				 return -1);
-
+				 return -EINVAL);
+		smu_data->avfs.avfs_btc_status = AVFS_BTC_ENABLEAVFS;
 		break;
 
 	case AVFS_BTC_DISABLED:
+	case AVFS_BTC_ENABLEAVFS:
 	case AVFS_BTC_NOTSUPPORTED:
 		break;
 
 	default:
-		pr_info("[AVFS] Something is broken. See log!");
+		pr_err("AVFS failed status is %x!\n", smu_data->avfs.avfs_btc_status);
 		break;
 	}
 
@@ -376,11 +376,6 @@ static int polaris10_smu_init(struct pp_smumgr *smumgr)
 	if (smu7_init(smumgr))
 		return -EINVAL;
 
-	if (polaris10_is_hw_avfs_present(smumgr))
-		smu_data->avfs.avfs_btc_status = AVFS_BTC_BOOT;
-	else
-		smu_data->avfs.avfs_btc_status = AVFS_BTC_NOTSUPPORTED;
-
 	for (i = 0; i < SMU74_MAX_LEVELS_GRAPHICS; i++)
 		smu_data->activity_target[i] = PPPOLARIS10_TARGETACTIVITY_DFLT;
 
@@ -410,4 +405,5 @@ const struct pp_smumgr_func polaris10_smu_funcs = {
 	.get_mac_definition = polaris10_get_mac_definition,
 	.is_dpm_running = polaris10_is_dpm_running,
 	.populate_requested_graphic_levels = polaris10_populate_requested_graphic_levels,
+	.is_hw_avfs_present = polaris10_is_hw_avfs_present,
 };
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.h b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.h
index 49ebf1d5a53c..5e19c24b0561 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.h
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.h
@@ -32,11 +32,6 @@
 
 #define SMC_RAM_END 0x40000
 
-struct polaris10_avfs {
-	enum AVFS_BTC_STATUS avfs_btc_status;
-	uint32_t           avfs_btc_param;
-};
-
 struct polaris10_pt_defaults {
 	uint8_t   SviLoadLineEn;
 	uint8_t   SviLoadLineVddC;
@@ -51,8 +46,6 @@ struct polaris10_pt_defaults {
 	uint16_t  BAPMTI_RC[SMU74_DTE_ITERATIONS * SMU74_DTE_SOURCES * SMU74_DTE_SINKS];
 };
 
-
-
 struct polaris10_range_table {
 	uint32_t trans_lower_frequency; /* in 10khz */
 	uint32_t trans_upper_frequency;
@@ -61,14 +54,13 @@ struct polaris10_range_table {
 struct polaris10_smumgr {
 	struct smu7_smumgr smu7_data;
 	uint8_t protected_mode;
-	struct polaris10_avfs  avfs;
 	SMU74_Discrete_DpmTable              smc_state_table;
 	struct SMU74_Discrete_Ulv            ulv_setting;
 	struct SMU74_Discrete_PmFuses  power_tune_table;
 	struct polaris10_range_table                range_table[NUM_SCLK_RANGE];
 	const struct polaris10_pt_defaults       *power_tune_defaults;
-	uint32_t                   activity_target[SMU74_MAX_LEVELS_GRAPHICS];
-	uint32_t                   bif_sclk_table[SMU74_MAX_LEVELS_LINK];
+	uint32_t               activity_target[SMU74_MAX_LEVELS_GRAPHICS];
+	uint32_t               bif_sclk_table[SMU74_MAX_LEVELS_LINK];
 };
 
 
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c
index 35ac27681415..c49a6f22002f 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c
@@ -380,7 +380,8 @@ static int smu7_populate_single_firmware_entry(struct pp_smumgr *smumgr,
 		entry->num_register_entries = 0;
 	}
 
-	if (fw_type == UCODE_ID_RLC_G)
+	if ((fw_type == UCODE_ID_RLC_G)
+		|| (fw_type == UCODE_ID_CP_MEC))
 		entry->flags = 1;
 	else
 		entry->flags = 0;
@@ -540,7 +541,6 @@ int smu7_upload_smu_firmware_image(struct pp_smumgr *smumgr)
 	return result;
 }
 
-
 int smu7_init(struct pp_smumgr *smumgr)
 {
 	struct smu7_smumgr *smu_data;
@@ -596,6 +596,11 @@ int smu7_init(struct pp_smumgr *smumgr)
 		(cgs_handle_t)smu_data->smu_buffer.handle);
 		return -EINVAL);
 
+	if (smum_is_hw_avfs_present(smumgr))
+		smu_data->avfs.avfs_btc_status = AVFS_BTC_BOOT;
+	else
+		smu_data->avfs.avfs_btc_status = AVFS_BTC_NOTSUPPORTED;
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.h b/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.h
index 919be435b49c..ee5e32d2921e 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.h
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.h
@@ -37,6 +37,11 @@ struct smu7_buffer_entry {
 	unsigned long  handle;
 };
 
+struct smu7_avfs {
+	enum AVFS_BTC_STATUS avfs_btc_status;
+	uint32_t           avfs_btc_param;
+};
+
 struct smu7_smumgr {
 	uint8_t *header;
 	uint8_t *mec_image;
@@ -50,7 +55,8 @@ struct smu7_smumgr {
 	uint32_t                             arb_table_start;
 	uint32_t                             ulv_setting_starts;
 	uint8_t                              security_hard_key;
-	uint32_t acpi_optimization;
+	uint32_t                             acpi_optimization;
+	struct smu7_avfs                     avfs;
 };
 
 
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c
index bcc61ffd13cb..3bdf6478de7f 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c
@@ -43,7 +43,8 @@ MODULE_FIRMWARE("amdgpu/polaris11_smc.bin");
 MODULE_FIRMWARE("amdgpu/polaris11_smc_sk.bin");
 MODULE_FIRMWARE("amdgpu/polaris11_k_smc.bin");
 MODULE_FIRMWARE("amdgpu/polaris12_smc.bin");
-
+MODULE_FIRMWARE("amdgpu/vega10_smc.bin");
+MODULE_FIRMWARE("amdgpu/vega10_acg_smc.bin");
 
 int smum_early_init(struct pp_instance *handle)
 {
@@ -403,3 +404,11 @@ int smum_populate_requested_graphic_levels(struct pp_hwmgr *hwmgr,
 
 	return 0;
 }
+
+bool smum_is_hw_avfs_present(struct pp_smumgr *smumgr)
+{
+	if (smumgr->smumgr_funcs->is_hw_avfs_present)
+		return smumgr->smumgr_funcs->is_hw_avfs_present(smumgr);
+
+	return false;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/vega10_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/vega10_smumgr.c
index 269678443862..408514c965a0 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/vega10_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/vega10_smumgr.c
@@ -356,6 +356,9 @@ int vega10_set_tools_address(struct pp_smumgr *smumgr)
 static int vega10_verify_smc_interface(struct pp_smumgr *smumgr)
 {
 	uint32_t smc_driver_if_version;
+	struct cgs_system_info sys_info = {0};
+	uint32_t dev_id;
+	uint32_t rev_id;
 
 	PP_ASSERT_WITH_CODE(!vega10_send_msg_to_smc(smumgr,
 			PPSMC_MSG_GetDriverIfVersion),
@@ -363,12 +366,27 @@ static int vega10_verify_smc_interface(struct pp_smumgr *smumgr)
 			return -EINVAL);
 	vega10_read_arg_from_smc(smumgr, &smc_driver_if_version);
 
-	if (smc_driver_if_version != SMU9_DRIVER_IF_VERSION) {
-		pr_err("Your firmware(0x%x) doesn't match \
-			SMU9_DRIVER_IF_VERSION(0x%x). \
-			Please update your firmware!\n",
-			smc_driver_if_version, SMU9_DRIVER_IF_VERSION);
-		return -EINVAL;
+	sys_info.size = sizeof(struct cgs_system_info);
+	sys_info.info_id = CGS_SYSTEM_INFO_PCIE_DEV;
+	cgs_query_system_info(smumgr->device, &sys_info);
+	dev_id = (uint32_t)sys_info.value;
+
+	sys_info.size = sizeof(struct cgs_system_info);
+	sys_info.info_id = CGS_SYSTEM_INFO_PCIE_REV;
+	cgs_query_system_info(smumgr->device, &sys_info);
+	rev_id = (uint32_t)sys_info.value;
+
+	if (!((dev_id == 0x687f) &&
+		((rev_id == 0xc0) ||
+		(rev_id == 0xc1) ||
+		(rev_id == 0xc3)))) {
+		if (smc_driver_if_version != SMU9_DRIVER_IF_VERSION) {
+			pr_err("Your firmware(0x%x) doesn't match \
+				SMU9_DRIVER_IF_VERSION(0x%x). \
+				Please update your firmware!\n",
+				smc_driver_if_version, SMU9_DRIVER_IF_VERSION);
+			return -EINVAL;
+		}
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/scheduler/gpu_sched_trace.h b/drivers/gpu/drm/amd/scheduler/gpu_sched_trace.h
index dbd4fd3a810b..8bd38102b58e 100644
--- a/drivers/gpu/drm/amd/scheduler/gpu_sched_trace.h
+++ b/drivers/gpu/drm/amd/scheduler/gpu_sched_trace.h
@@ -16,16 +16,16 @@ TRACE_EVENT(amd_sched_job,
 	    TP_ARGS(sched_job),
 	    TP_STRUCT__entry(
 			     __field(struct amd_sched_entity *, entity)
-			     __field(struct amd_sched_job *, sched_job)
 			     __field(struct dma_fence *, fence)
 			     __field(const char *, name)
+			     __field(uint64_t, id)
 			     __field(u32, job_count)
 			     __field(int, hw_job_count)
 			     ),
 
 	    TP_fast_assign(
 			   __entry->entity = sched_job->s_entity;
-			   __entry->sched_job = sched_job;
+			   __entry->id = sched_job->id;
 			   __entry->fence = &sched_job->s_fence->finished;
 			   __entry->name = sched_job->sched->name;
 			   __entry->job_count = kfifo_len(
@@ -33,8 +33,9 @@ TRACE_EVENT(amd_sched_job,
 			   __entry->hw_job_count = atomic_read(
 				   &sched_job->sched->hw_rq_count);
 			   ),
-	    TP_printk("entity=%p, sched job=%p, fence=%p, ring=%s, job count:%u, hw job count:%d",
-		      __entry->entity, __entry->sched_job, __entry->fence, __entry->name,
+	    TP_printk("entity=%p, id=%llu, fence=%p, ring=%s, job count:%u, hw job count:%d",
+		      __entry->entity, __entry->id,
+		      __entry->fence, __entry->name,
 		      __entry->job_count, __entry->hw_job_count)
 );
 
diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
index 38cea6fb25a8..97c94f9683fa 100644
--- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
+++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
@@ -205,17 +205,32 @@ void amd_sched_entity_fini(struct amd_gpu_scheduler *sched,
 			   struct amd_sched_entity *entity)
 {
 	struct amd_sched_rq *rq = entity->rq;
+	int r;
 
 	if (!amd_sched_entity_is_initialized(sched, entity))
 		return;
-
 	/**
 	 * The client will not queue more IBs during this fini, consume existing
-	 * queued IBs
+	 * queued IBs or discard them on SIGKILL
 	*/
-	wait_event(sched->job_scheduled, amd_sched_entity_is_idle(entity));
-
+	if ((current->flags & PF_SIGNALED) && current->exit_code == SIGKILL)
+		r = -ERESTARTSYS;
+	else
+		r = wait_event_killable(sched->job_scheduled,
+					amd_sched_entity_is_idle(entity));
 	amd_sched_rq_remove_entity(rq, entity);
+	if (r) {
+		struct amd_sched_job *job;
+
+		/* Park the kernel for a moment to make sure it isn't processing
+		 * our enity.
+		 */
+		kthread_park(sched->thread);
+		kthread_unpark(sched->thread);
+		while (kfifo_out(&entity->job_queue, &job, sizeof(job)))
+			sched->ops->free_job(job);
+
+	}
 	kfifo_free(&entity->job_queue);
 }
 
diff --git a/drivers/gpu/drm/arc/arcpgu_crtc.c b/drivers/gpu/drm/arc/arcpgu_crtc.c
index ad9a95916f1f..16903dc7fe0d 100644
--- a/drivers/gpu/drm/arc/arcpgu_crtc.c
+++ b/drivers/gpu/drm/arc/arcpgu_crtc.c
@@ -64,6 +64,20 @@ static const struct drm_crtc_funcs arc_pgu_crtc_funcs = {
 	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
 };
 
+static enum drm_mode_status arc_pgu_crtc_mode_valid(struct drm_crtc *crtc,
+						    const struct drm_display_mode *mode)
+{
+	struct arcpgu_drm_private *arcpgu = crtc_to_arcpgu_priv(crtc);
+	long rate, clk_rate = mode->clock * 1000;
+	long diff = clk_rate / 200; /* +-0.5% allowed by HDMI spec */
+
+	rate = clk_round_rate(arcpgu->clk, clk_rate);
+	if ((max(rate, clk_rate) - min(rate, clk_rate) < diff) && (rate > 0))
+		return MODE_OK;
+
+	return MODE_NOCLOCK;
+}
+
 static void arc_pgu_crtc_mode_set_nofb(struct drm_crtc *crtc)
 {
 	struct arcpgu_drm_private *arcpgu = crtc_to_arcpgu_priv(crtc);
@@ -106,7 +120,8 @@ static void arc_pgu_crtc_mode_set_nofb(struct drm_crtc *crtc)
 	clk_set_rate(arcpgu->clk, m->crtc_clock * 1000);
 }
 
-static void arc_pgu_crtc_enable(struct drm_crtc *crtc)
+static void arc_pgu_crtc_atomic_enable(struct drm_crtc *crtc,
+				       struct drm_crtc_state *old_state)
 {
 	struct arcpgu_drm_private *arcpgu = crtc_to_arcpgu_priv(crtc);
 
@@ -116,7 +131,8 @@ static void arc_pgu_crtc_enable(struct drm_crtc *crtc)
 		      ARCPGU_CTRL_ENABLE_MASK);
 }
 
-static void arc_pgu_crtc_disable(struct drm_crtc *crtc)
+static void arc_pgu_crtc_atomic_disable(struct drm_crtc *crtc,
+					struct drm_crtc_state *old_state)
 {
 	struct arcpgu_drm_private *arcpgu = crtc_to_arcpgu_priv(crtc);
 
@@ -129,20 +145,6 @@ static void arc_pgu_crtc_disable(struct drm_crtc *crtc)
 			      ~ARCPGU_CTRL_ENABLE_MASK);
 }
 
-static int arc_pgu_crtc_atomic_check(struct drm_crtc *crtc,
-				     struct drm_crtc_state *state)
-{
-	struct arcpgu_drm_private *arcpgu = crtc_to_arcpgu_priv(crtc);
-	struct drm_display_mode *mode = &state->adjusted_mode;
-	long rate, clk_rate = mode->clock * 1000;
-
-	rate = clk_round_rate(arcpgu->clk, clk_rate);
-	if (rate != clk_rate)
-		return -EINVAL;
-
-	return 0;
-}
-
 static void arc_pgu_crtc_atomic_begin(struct drm_crtc *crtc,
 				      struct drm_crtc_state *state)
 {
@@ -158,15 +160,13 @@ static void arc_pgu_crtc_atomic_begin(struct drm_crtc *crtc,
 }
 
 static const struct drm_crtc_helper_funcs arc_pgu_crtc_helper_funcs = {
+	.mode_valid	= arc_pgu_crtc_mode_valid,
 	.mode_set	= drm_helper_crtc_mode_set,
 	.mode_set_base	= drm_helper_crtc_mode_set_base,
 	.mode_set_nofb	= arc_pgu_crtc_mode_set_nofb,
-	.enable		= arc_pgu_crtc_enable,
-	.disable	= arc_pgu_crtc_disable,
-	.prepare	= arc_pgu_crtc_disable,
-	.commit		= arc_pgu_crtc_enable,
-	.atomic_check	= arc_pgu_crtc_atomic_check,
 	.atomic_begin	= arc_pgu_crtc_atomic_begin,
+	.atomic_enable	= arc_pgu_crtc_atomic_enable,
+	.atomic_disable	= arc_pgu_crtc_atomic_disable,
 };
 
 static void arc_pgu_plane_atomic_update(struct drm_plane *plane,
@@ -218,6 +218,7 @@ static struct drm_plane *arc_pgu_plane_init(struct drm_device *drm)
 
 	ret = drm_universal_plane_init(drm, plane, 0xff, &arc_pgu_plane_funcs,
 				       formats, ARRAY_SIZE(formats),
+				       NULL,
 				       DRM_PLANE_TYPE_PRIMARY, NULL);
 	if (ret)
 		return ERR_PTR(ret);
diff --git a/drivers/gpu/drm/arc/arcpgu_drv.c b/drivers/gpu/drm/arc/arcpgu_drv.c
index 3e43a5d4fb09..289eda54e5aa 100644
--- a/drivers/gpu/drm/arc/arcpgu_drv.c
+++ b/drivers/gpu/drm/arc/arcpgu_drv.c
@@ -31,7 +31,7 @@ static void arcpgu_fb_output_poll_changed(struct drm_device *dev)
 	drm_fbdev_cma_hotplug_event(arcpgu->fbdev);
 }
 
-static struct drm_mode_config_funcs arcpgu_drm_modecfg_funcs = {
+static const struct drm_mode_config_funcs arcpgu_drm_modecfg_funcs = {
 	.fb_create  = drm_fb_cma_create,
 	.output_poll_changed = arcpgu_fb_output_poll_changed,
 	.atomic_check = drm_atomic_helper_check,
@@ -48,29 +48,7 @@ static void arcpgu_setup_mode_config(struct drm_device *drm)
 	drm->mode_config.funcs = &arcpgu_drm_modecfg_funcs;
 }
 
-static int arcpgu_gem_mmap(struct file *filp, struct vm_area_struct *vma)
-{
-	int ret;
-
-	ret = drm_gem_mmap(filp, vma);
-	if (ret)
-		return ret;
-
-	vma->vm_page_prot = pgprot_noncached(vm_get_page_prot(vma->vm_flags));
-	return 0;
-}
-
-static const struct file_operations arcpgu_drm_ops = {
-	.owner = THIS_MODULE,
-	.open = drm_open,
-	.release = drm_release,
-	.unlocked_ioctl = drm_ioctl,
-	.compat_ioctl = drm_compat_ioctl,
-	.poll = drm_poll,
-	.read = drm_read,
-	.llseek = no_llseek,
-	.mmap = arcpgu_gem_mmap,
-};
+DEFINE_DRM_GEM_CMA_FOPS(arcpgu_drm_ops);
 
 static void arcpgu_lastclose(struct drm_device *drm)
 {
@@ -142,7 +120,7 @@ static int arcpgu_load(struct drm_device *drm)
 		return -ENODEV;
 	}
 
-	platform_set_drvdata(pdev, arcpgu);
+	platform_set_drvdata(pdev, drm);
 	return 0;
 }
 
@@ -160,11 +138,37 @@ static int arcpgu_unload(struct drm_device *drm)
 	return 0;
 }
 
+#ifdef CONFIG_DEBUG_FS
+static int arcpgu_show_pxlclock(struct seq_file *m, void *arg)
+{
+	struct drm_info_node *node = (struct drm_info_node *)m->private;
+	struct drm_device *drm = node->minor->dev;
+	struct arcpgu_drm_private *arcpgu = drm->dev_private;
+	unsigned long clkrate = clk_get_rate(arcpgu->clk);
+	unsigned long mode_clock = arcpgu->crtc.mode.crtc_clock * 1000;
+
+	seq_printf(m, "hw  : %lu\n", clkrate);
+	seq_printf(m, "mode: %lu\n", mode_clock);
+	return 0;
+}
+
+static struct drm_info_list arcpgu_debugfs_list[] = {
+	{ "clocks", arcpgu_show_pxlclock, 0 },
+	{ "fb", drm_fb_cma_debugfs_show, 0 },
+};
+
+static int arcpgu_debugfs_init(struct drm_minor *minor)
+{
+	return drm_debugfs_create_files(arcpgu_debugfs_list,
+		ARRAY_SIZE(arcpgu_debugfs_list), minor->debugfs_root, minor);
+}
+#endif
+
 static struct drm_driver arcpgu_drm_driver = {
 	.driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME |
 			   DRIVER_ATOMIC,
 	.lastclose = arcpgu_lastclose,
-	.name = "drm-arcpgu",
+	.name = "arcpgu",
 	.desc = "ARC PGU Controller",
 	.date = "20160219",
 	.major = 1,
@@ -172,8 +176,6 @@ static struct drm_driver arcpgu_drm_driver = {
 	.patchlevel = 0,
 	.fops = &arcpgu_drm_ops,
 	.dumb_create = drm_gem_cma_dumb_create,
-	.dumb_map_offset = drm_gem_cma_dumb_map_offset,
-	.dumb_destroy = drm_gem_dumb_destroy,
 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
 	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
 	.gem_free_object_unlocked = drm_gem_cma_free_object,
@@ -185,6 +187,9 @@ static struct drm_driver arcpgu_drm_driver = {
 	.gem_prime_vmap = drm_gem_cma_prime_vmap,
 	.gem_prime_vunmap = drm_gem_cma_prime_vunmap,
 	.gem_prime_mmap = drm_gem_cma_prime_mmap,
+#ifdef CONFIG_DEBUG_FS
+	.debugfs_init = arcpgu_debugfs_init,
+#endif
 };
 
 static int arcpgu_probe(struct platform_device *pdev)
diff --git a/drivers/gpu/drm/arm/hdlcd_crtc.c b/drivers/gpu/drm/arm/hdlcd_crtc.c
index d67b6f15e8b8..72b22b805412 100644
--- a/drivers/gpu/drm/arm/hdlcd_crtc.c
+++ b/drivers/gpu/drm/arm/hdlcd_crtc.c
@@ -165,7 +165,8 @@ static void hdlcd_crtc_mode_set_nofb(struct drm_crtc *crtc)
 	clk_set_rate(hdlcd->clk, m->crtc_clock * 1000);
 }
 
-static void hdlcd_crtc_enable(struct drm_crtc *crtc)
+static void hdlcd_crtc_atomic_enable(struct drm_crtc *crtc,
+				     struct drm_crtc_state *old_state)
 {
 	struct hdlcd_drm_private *hdlcd = crtc_to_hdlcd_priv(crtc);
 
@@ -175,7 +176,8 @@ static void hdlcd_crtc_enable(struct drm_crtc *crtc)
 	drm_crtc_vblank_on(crtc);
 }
 
-static void hdlcd_crtc_disable(struct drm_crtc *crtc)
+static void hdlcd_crtc_atomic_disable(struct drm_crtc *crtc,
+				      struct drm_crtc_state *old_state)
 {
 	struct hdlcd_drm_private *hdlcd = crtc_to_hdlcd_priv(crtc);
 
@@ -218,10 +220,10 @@ static void hdlcd_crtc_atomic_begin(struct drm_crtc *crtc,
 }
 
 static const struct drm_crtc_helper_funcs hdlcd_crtc_helper_funcs = {
-	.enable		= hdlcd_crtc_enable,
-	.disable	= hdlcd_crtc_disable,
 	.atomic_check	= hdlcd_crtc_atomic_check,
 	.atomic_begin	= hdlcd_crtc_atomic_begin,
+	.atomic_enable	= hdlcd_crtc_atomic_enable,
+	.atomic_disable	= hdlcd_crtc_atomic_disable,
 };
 
 static int hdlcd_plane_atomic_check(struct drm_plane *plane,
@@ -313,6 +315,7 @@ static struct drm_plane *hdlcd_plane_init(struct drm_device *drm)
 
 	ret = drm_universal_plane_init(drm, plane, 0xff, &hdlcd_plane_funcs,
 				       formats, ARRAY_SIZE(formats),
+				       NULL,
 				       DRM_PLANE_TYPE_PRIMARY, NULL);
 	if (ret) {
 		return ERR_PTR(ret);
diff --git a/drivers/gpu/drm/arm/hdlcd_drv.c b/drivers/gpu/drm/arm/hdlcd_drv.c
index d3da87fbd85a..f9bda7b0d2ec 100644
--- a/drivers/gpu/drm/arm/hdlcd_drv.c
+++ b/drivers/gpu/drm/arm/hdlcd_drv.c
@@ -253,8 +253,6 @@ static struct drm_driver hdlcd_driver = {
 	.gem_free_object_unlocked = drm_gem_cma_free_object,
 	.gem_vm_ops = &drm_gem_cma_vm_ops,
 	.dumb_create = drm_gem_cma_dumb_create,
-	.dumb_map_offset = drm_gem_cma_dumb_map_offset,
-	.dumb_destroy = drm_gem_dumb_destroy,
 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
 	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
 	.gem_prime_export = drm_gem_prime_export,
@@ -343,7 +341,6 @@ err_register:
 	}
 err_fbdev:
 	drm_kms_helper_poll_fini(drm);
-	drm_vblank_cleanup(drm);
 err_vblank:
 	pm_runtime_disable(drm->dev);
 err_pm_active:
@@ -375,7 +372,6 @@ static void hdlcd_drm_unbind(struct device *dev)
 	component_unbind_all(dev, drm);
 	of_node_put(hdlcd->crtc.port);
 	hdlcd->crtc.port = NULL;
-	drm_vblank_cleanup(drm);
 	pm_runtime_get_sync(drm->dev);
 	drm_irq_uninstall(drm);
 	pm_runtime_put_sync(drm->dev);
diff --git a/drivers/gpu/drm/arm/malidp_crtc.c b/drivers/gpu/drm/arm/malidp_crtc.c
index 4bb38a21efec..3615d18a7ddf 100644
--- a/drivers/gpu/drm/arm/malidp_crtc.c
+++ b/drivers/gpu/drm/arm/malidp_crtc.c
@@ -46,7 +46,8 @@ static enum drm_mode_status malidp_crtc_mode_valid(struct drm_crtc *crtc,
 	return MODE_OK;
 }
 
-static void malidp_crtc_enable(struct drm_crtc *crtc)
+static void malidp_crtc_atomic_enable(struct drm_crtc *crtc,
+				      struct drm_crtc_state *old_state)
 {
 	struct malidp_drm *malidp = crtc_to_malidp_device(crtc);
 	struct malidp_hw_device *hwdev = malidp->dev;
@@ -69,7 +70,8 @@ static void malidp_crtc_enable(struct drm_crtc *crtc)
 	drm_crtc_vblank_on(crtc);
 }
 
-static void malidp_crtc_disable(struct drm_crtc *crtc)
+static void malidp_crtc_atomic_disable(struct drm_crtc *crtc,
+				       struct drm_crtc_state *old_state)
 {
 	struct malidp_drm *malidp = crtc_to_malidp_device(crtc);
 	struct malidp_hw_device *hwdev = malidp->dev;
@@ -408,9 +410,9 @@ static int malidp_crtc_atomic_check(struct drm_crtc *crtc,
 
 static const struct drm_crtc_helper_funcs malidp_crtc_helper_funcs = {
 	.mode_valid = malidp_crtc_mode_valid,
-	.enable = malidp_crtc_enable,
-	.disable = malidp_crtc_disable,
 	.atomic_check = malidp_crtc_atomic_check,
+	.atomic_enable = malidp_crtc_atomic_enable,
+	.atomic_disable = malidp_crtc_atomic_disable,
 };
 
 static struct drm_crtc_state *malidp_crtc_duplicate_state(struct drm_crtc *crtc)
diff --git a/drivers/gpu/drm/arm/malidp_drv.c b/drivers/gpu/drm/arm/malidp_drv.c
index 01b13d219917..1a57cc28955e 100644
--- a/drivers/gpu/drm/arm/malidp_drv.c
+++ b/drivers/gpu/drm/arm/malidp_drv.c
@@ -225,7 +225,7 @@ static void malidp_atomic_commit_tail(struct drm_atomic_state *state)
 
 	drm_atomic_helper_commit_modeset_disables(drm, state);
 
-	for_each_crtc_in_state(state, crtc, old_crtc_state, i) {
+	for_each_old_crtc_in_state(state, crtc, old_crtc_state, i) {
 		malidp_atomic_commit_update_gamma(crtc, old_crtc_state);
 		malidp_atomic_commit_update_coloradj(crtc, old_crtc_state);
 		malidp_atomic_commit_se_config(crtc, old_crtc_state);
@@ -331,8 +331,6 @@ static struct drm_driver malidp_driver = {
 	.gem_free_object_unlocked = drm_gem_cma_free_object,
 	.gem_vm_ops = &drm_gem_cma_vm_ops,
 	.dumb_create = drm_gem_cma_dumb_create,
-	.dumb_map_offset = drm_gem_cma_dumb_map_offset,
-	.dumb_destroy = drm_gem_dumb_destroy,
 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
 	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
 	.gem_prime_export = drm_gem_prime_export,
diff --git a/drivers/gpu/drm/arm/malidp_planes.c b/drivers/gpu/drm/arm/malidp_planes.c
index 600fa7bd7f52..94e7e3fa3408 100644
--- a/drivers/gpu/drm/arm/malidp_planes.c
+++ b/drivers/gpu/drm/arm/malidp_planes.c
@@ -128,7 +128,6 @@ static void malidp_plane_atomic_print_state(struct drm_printer *p,
 static const struct drm_plane_funcs malidp_de_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
-	.set_property = drm_atomic_helper_plane_set_property,
 	.destroy = malidp_de_plane_destroy,
 	.reset = malidp_plane_reset,
 	.atomic_duplicate_state = malidp_duplicate_plane_state,
@@ -398,7 +397,7 @@ int malidp_de_planes_init(struct drm_device *drm)
 					DRM_PLANE_TYPE_OVERLAY;
 		ret = drm_universal_plane_init(drm, &plane->base, crtcs,
 					       &malidp_de_plane_funcs, formats,
-					       n, plane_type, NULL);
+					       n, NULL, plane_type, NULL);
 		if (ret < 0)
 			goto cleanup;
 
diff --git a/drivers/gpu/drm/armada/armada_crtc.c b/drivers/gpu/drm/armada/armada_crtc.c
index 4fe19fde84f9..2a4d163ac76f 100644
--- a/drivers/gpu/drm/armada/armada_crtc.c
+++ b/drivers/gpu/drm/armada/armada_crtc.c
@@ -334,16 +334,6 @@ static void armada_drm_vblank_off(struct armada_crtc *dcrtc)
 	armada_drm_plane_work_run(dcrtc, dcrtc->crtc.primary);
 }
 
-void armada_drm_crtc_gamma_set(struct drm_crtc *crtc, u16 r, u16 g, u16 b,
-	int idx)
-{
-}
-
-void armada_drm_crtc_gamma_get(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b,
-	int idx)
-{
-}
-
 /* The mode_config.mutex will be held for this call */
 static void armada_drm_crtc_dpms(struct drm_crtc *crtc, int dpms)
 {
@@ -1150,13 +1140,13 @@ int armada_drm_plane_init(struct armada_plane *plane)
 	return 0;
 }
 
-static struct drm_prop_enum_list armada_drm_csc_yuv_enum_list[] = {
+static const struct drm_prop_enum_list armada_drm_csc_yuv_enum_list[] = {
 	{ CSC_AUTO,        "Auto" },
 	{ CSC_YUV_CCIR601, "CCIR601" },
 	{ CSC_YUV_CCIR709, "CCIR709" },
 };
 
-static struct drm_prop_enum_list armada_drm_csc_rgb_enum_list[] = {
+static const struct drm_prop_enum_list armada_drm_csc_rgb_enum_list[] = {
 	{ CSC_AUTO,         "Auto" },
 	{ CSC_RGB_COMPUTER, "Computer system" },
 	{ CSC_RGB_STUDIO,   "Studio" },
@@ -1269,6 +1259,7 @@ static int armada_drm_crtc_create(struct drm_device *drm, struct device *dev,
 				       &armada_primary_plane_funcs,
 				       armada_primary_formats,
 				       ARRAY_SIZE(armada_primary_formats),
+				       NULL,
 				       DRM_PLANE_TYPE_PRIMARY, NULL);
 	if (ret) {
 		kfree(primary);
@@ -1329,8 +1320,7 @@ armada_lcd_bind(struct device *dev, struct device *master, void *data)
 		port = of_get_child_by_name(parent, "port");
 		of_node_put(np);
 		if (!port) {
-			dev_err(dev, "no port node found in %s\n",
-				parent->full_name);
+			dev_err(dev, "no port node found in %pOF\n", parent);
 			return -ENXIO;
 		}
 
@@ -1364,7 +1354,7 @@ static int armada_lcd_remove(struct platform_device *pdev)
 	return 0;
 }
 
-static struct of_device_id armada_lcd_of_match[] = {
+static const struct of_device_id armada_lcd_of_match[] = {
 	{
 		.compatible	= "marvell,dove-lcd",
 		.data		= &armada510_ops,
diff --git a/drivers/gpu/drm/armada/armada_crtc.h b/drivers/gpu/drm/armada/armada_crtc.h
index 7e8906d3ae26..bab11f483575 100644
--- a/drivers/gpu/drm/armada/armada_crtc.h
+++ b/drivers/gpu/drm/armada/armada_crtc.h
@@ -102,8 +102,6 @@ struct armada_crtc {
 };
 #define drm_to_armada_crtc(c) container_of(c, struct armada_crtc, crtc)
 
-void armada_drm_crtc_gamma_set(struct drm_crtc *, u16, u16, u16, int);
-void armada_drm_crtc_gamma_get(struct drm_crtc *, u16 *, u16 *, u16 *, int);
 void armada_drm_crtc_update_regs(struct armada_crtc *, struct armada_regs *);
 
 void armada_drm_crtc_plane_disable(struct armada_crtc *dcrtc,
diff --git a/drivers/gpu/drm/armada/armada_drv.c b/drivers/gpu/drm/armada/armada_drv.c
index e618fab7f519..0b3227c039d7 100644
--- a/drivers/gpu/drm/armada/armada_drv.c
+++ b/drivers/gpu/drm/armada/armada_drv.c
@@ -232,8 +232,8 @@ static void armada_add_endpoints(struct device *dev,
 			of_node_put(remote);
 			continue;
 		} else if (!of_device_is_available(remote->parent)) {
-			dev_warn(dev, "parent device of %s is not available\n",
-				 remote->full_name);
+			dev_warn(dev, "parent device of %pOF is not available\n",
+				 remote);
 			of_node_put(remote);
 			continue;
 		}
diff --git a/drivers/gpu/drm/armada/armada_fbdev.c b/drivers/gpu/drm/armada/armada_fbdev.c
index 602dfead8eef..29c7d047b152 100644
--- a/drivers/gpu/drm/armada/armada_fbdev.c
+++ b/drivers/gpu/drm/armada/armada_fbdev.c
@@ -81,7 +81,6 @@ static int armada_fb_create(struct drm_fb_helper *fbh,
 
 	strlcpy(info->fix.id, "armada-drmfb", sizeof(info->fix.id));
 	info->par = fbh;
-	info->flags = FBINFO_DEFAULT | FBINFO_CAN_FORCE_OUTPUT;
 	info->fbops = &armada_fb_ops;
 	info->fix.smem_start = obj->phys_addr;
 	info->fix.smem_len = obj->obj.size;
@@ -118,8 +117,6 @@ static int armada_fb_probe(struct drm_fb_helper *fbh,
 }
 
 static const struct drm_fb_helper_funcs armada_fb_helper_funcs = {
-	.gamma_set	= armada_drm_crtc_gamma_set,
-	.gamma_get	= armada_drm_crtc_gamma_get,
 	.fb_probe	= armada_fb_probe,
 };
 
diff --git a/drivers/gpu/drm/armada/armada_overlay.c b/drivers/gpu/drm/armada/armada_overlay.c
index e9a29df4b443..edc44910d79f 100644
--- a/drivers/gpu/drm/armada/armada_overlay.c
+++ b/drivers/gpu/drm/armada/armada_overlay.c
@@ -388,7 +388,7 @@ static const uint32_t armada_ovl_formats[] = {
 	DRM_FORMAT_BGR565,
 };
 
-static struct drm_prop_enum_list armada_drm_colorkey_enum_list[] = {
+static const struct drm_prop_enum_list armada_drm_colorkey_enum_list[] = {
 	{ CKMODE_DISABLE, "disabled" },
 	{ CKMODE_Y,       "Y component" },
 	{ CKMODE_U,       "U component" },
@@ -460,6 +460,7 @@ int armada_overlay_plane_create(struct drm_device *dev, unsigned long crtcs)
 				       &armada_ovl_plane_funcs,
 				       armada_ovl_formats,
 				       ARRAY_SIZE(armada_ovl_formats),
+				       NULL,
 				       DRM_PLANE_TYPE_OVERLAY, NULL);
 	if (ret) {
 		kfree(dplane);
diff --git a/drivers/gpu/drm/ast/ast_dp501.c b/drivers/gpu/drm/ast/ast_dp501.c
index 76f07f38b941..749646ae365f 100644
--- a/drivers/gpu/drm/ast/ast_dp501.c
+++ b/drivers/gpu/drm/ast/ast_dp501.c
@@ -4,16 +4,11 @@
 #include "ast_drv.h"
 MODULE_FIRMWARE("ast_dp501_fw.bin");
 
-int ast_load_dp501_microcode(struct drm_device *dev)
+static int ast_load_dp501_microcode(struct drm_device *dev)
 {
 	struct ast_private *ast = dev->dev_private;
-	static char *fw_name = "ast_dp501_fw.bin";
-	int err;
-	err = request_firmware(&ast->dp501_fw, fw_name, dev->dev);
-	if (err)
-		return err;
 
-	return 0;
+	return request_firmware(&ast->dp501_fw, "ast_dp501_fw.bin", dev->dev);
 }
 
 static void send_ack(struct ast_private *ast)
@@ -187,7 +182,7 @@ bool ast_backup_fw(struct drm_device *dev, u8 *addr, u32 size)
 	return false;
 }
 
-bool ast_launch_m68k(struct drm_device *dev)
+static bool ast_launch_m68k(struct drm_device *dev)
 {
 	struct ast_private *ast = dev->dev_private;
 	u32 i, data, len = 0;
@@ -201,7 +196,11 @@ bool ast_launch_m68k(struct drm_device *dev)
 		if (ast->dp501_fw_addr) {
 			fw_addr = ast->dp501_fw_addr;
 			len = 32*1024;
-		} else if (ast->dp501_fw) {
+		} else {
+			if (!ast->dp501_fw &&
+			    ast_load_dp501_microcode(dev) < 0)
+				return false;
+
 			fw_addr = (u8 *)ast->dp501_fw->data;
 			len = ast->dp501_fw->size;
 		}
@@ -432,3 +431,11 @@ void ast_init_3rdtx(struct drm_device *dev)
 		}
 	}
 }
+
+void ast_release_firmware(struct drm_device *dev)
+{
+	struct ast_private *ast = dev->dev_private;
+
+	release_firmware(ast->dp501_fw);
+	ast->dp501_fw = NULL;
+}
diff --git a/drivers/gpu/drm/ast/ast_drv.c b/drivers/gpu/drm/ast/ast_drv.c
index fd7c9eec92e4..69dab82a3771 100644
--- a/drivers/gpu/drm/ast/ast_drv.c
+++ b/drivers/gpu/drm/ast/ast_drv.c
@@ -197,7 +197,6 @@ static struct drm_driver driver = {
 
 	.load = ast_driver_load,
 	.unload = ast_driver_unload,
-	.set_busid = drm_pci_set_busid,
 
 	.fops = &ast_fops,
 	.name = DRIVER_NAME,
@@ -210,7 +209,6 @@ static struct drm_driver driver = {
 	.gem_free_object_unlocked = ast_gem_free_object,
 	.dumb_create = ast_dumb_create,
 	.dumb_map_offset = ast_dumb_mmap_offset,
-	.dumb_destroy = drm_gem_dumb_destroy,
 
 };
 
@@ -221,11 +219,11 @@ static int __init ast_init(void)
 
 	if (ast_modeset == 0)
 		return -EINVAL;
-	return drm_pci_init(&driver, &ast_pci_driver);
+	return pci_register_driver(&ast_pci_driver);
 }
 static void __exit ast_exit(void)
 {
-	drm_pci_exit(&driver, &ast_pci_driver);
+	pci_unregister_driver(&ast_pci_driver);
 }
 
 module_init(ast_init);
diff --git a/drivers/gpu/drm/ast/ast_drv.h b/drivers/gpu/drm/ast/ast_drv.h
index 8880f0b62e9c..e6c4cd3dc50e 100644
--- a/drivers/gpu/drm/ast/ast_drv.h
+++ b/drivers/gpu/drm/ast/ast_drv.h
@@ -245,7 +245,6 @@ struct ast_connector {
 
 struct ast_crtc {
 	struct drm_crtc base;
-	u8 lut_r[256], lut_g[256], lut_b[256];
 	struct drm_gem_object *cursor_bo;
 	uint64_t cursor_addr;
 	int cursor_width, cursor_height;
@@ -401,11 +400,10 @@ void ast_post_gpu(struct drm_device *dev);
 u32 ast_mindwm(struct ast_private *ast, u32 r);
 void ast_moutdwm(struct ast_private *ast, u32 r, u32 v);
 /* ast dp501 */
-int ast_load_dp501_microcode(struct drm_device *dev);
 void ast_set_dp501_video_output(struct drm_device *dev, u8 mode);
-bool ast_launch_m68k(struct drm_device *dev);
 bool ast_backup_fw(struct drm_device *dev, u8 *addr, u32 size);
 bool ast_dp501_read_edid(struct drm_device *dev, u8 *ediddata);
 u8 ast_get_dp501_max_clk(struct drm_device *dev);
 void ast_init_3rdtx(struct drm_device *dev);
+void ast_release_firmware(struct drm_device *dev);
 #endif
diff --git a/drivers/gpu/drm/ast/ast_fb.c b/drivers/gpu/drm/ast/ast_fb.c
index 4ad4acd0ccab..0cd827e11fa2 100644
--- a/drivers/gpu/drm/ast/ast_fb.c
+++ b/drivers/gpu/drm/ast/ast_fb.c
@@ -231,7 +231,6 @@ static int astfb_create(struct drm_fb_helper *helper,
 
 	strcpy(info->fix.id, "astdrmfb");
 
-	info->flags = FBINFO_DEFAULT | FBINFO_CAN_FORCE_OUTPUT;
 	info->fbops = &astfb_ops;
 
 	info->apertures->ranges[0].base = pci_resource_start(dev->pdev, 0);
@@ -255,27 +254,7 @@ out:
 	return ret;
 }
 
-static void ast_fb_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
-			       u16 blue, int regno)
-{
-	struct ast_crtc *ast_crtc = to_ast_crtc(crtc);
-	ast_crtc->lut_r[regno] = red >> 8;
-	ast_crtc->lut_g[regno] = green >> 8;
-	ast_crtc->lut_b[regno] = blue >> 8;
-}
-
-static void ast_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green,
-			       u16 *blue, int regno)
-{
-	struct ast_crtc *ast_crtc = to_ast_crtc(crtc);
-	*red = ast_crtc->lut_r[regno] << 8;
-	*green = ast_crtc->lut_g[regno] << 8;
-	*blue = ast_crtc->lut_b[regno] << 8;
-}
-
 static const struct drm_fb_helper_funcs ast_fb_helper_funcs = {
-	.gamma_set = ast_fb_gamma_set,
-	.gamma_get = ast_fb_gamma_get,
 	.fb_probe = astfb_create,
 };
 
@@ -287,7 +266,7 @@ static void ast_fbdev_destroy(struct drm_device *dev,
 	drm_fb_helper_unregister_fbi(&afbdev->helper);
 
 	if (afb->obj) {
-		drm_gem_object_unreference_unlocked(afb->obj);
+		drm_gem_object_put_unlocked(afb->obj);
 		afb->obj = NULL;
 	}
 	drm_fb_helper_fini(&afbdev->helper);
diff --git a/drivers/gpu/drm/ast/ast_main.c b/drivers/gpu/drm/ast/ast_main.c
index 262c2c0e43b4..dac355812adc 100644
--- a/drivers/gpu/drm/ast/ast_main.c
+++ b/drivers/gpu/drm/ast/ast_main.c
@@ -387,9 +387,9 @@ static void ast_user_framebuffer_destroy(struct drm_framebuffer *fb)
 {
 	struct ast_framebuffer *ast_fb = to_ast_framebuffer(fb);
 
-	drm_gem_object_unreference_unlocked(ast_fb->obj);
+	drm_gem_object_put_unlocked(ast_fb->obj);
 	drm_framebuffer_cleanup(fb);
-	kfree(fb);
+	kfree(ast_fb);
 }
 
 static const struct drm_framebuffer_funcs ast_fb_funcs = {
@@ -429,13 +429,13 @@ ast_user_framebuffer_create(struct drm_device *dev,
 
 	ast_fb = kzalloc(sizeof(*ast_fb), GFP_KERNEL);
 	if (!ast_fb) {
-		drm_gem_object_unreference_unlocked(obj);
+		drm_gem_object_put_unlocked(obj);
 		return ERR_PTR(-ENOMEM);
 	}
 
 	ret = ast_framebuffer_init(dev, ast_fb, mode_cmd, obj);
 	if (ret) {
-		drm_gem_object_unreference_unlocked(obj);
+		drm_gem_object_put_unlocked(obj);
 		kfree(ast_fb);
 		return ERR_PTR(ret);
 	}
@@ -576,6 +576,7 @@ void ast_driver_unload(struct drm_device *dev)
 {
 	struct ast_private *ast = dev->dev_private;
 
+	ast_release_firmware(dev);
 	kfree(ast->dp501_fw_addr);
 	ast_mode_fini(dev);
 	ast_fbdev_fini(dev);
@@ -627,7 +628,7 @@ int ast_dumb_create(struct drm_file *file,
 		return ret;
 
 	ret = drm_gem_handle_create(file, gobj, &handle);
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 	if (ret)
 		return ret;
 
@@ -675,7 +676,7 @@ ast_dumb_mmap_offset(struct drm_file *file,
 	bo = gem_to_ast_bo(obj);
 	*offset = ast_bo_mmap_offset(bo);
 
-	drm_gem_object_unreference_unlocked(obj);
+	drm_gem_object_put_unlocked(obj);
 
 	return 0;
 
diff --git a/drivers/gpu/drm/ast/ast_mode.c b/drivers/gpu/drm/ast/ast_mode.c
index aaef0a652f10..6f3849ec0c1d 100644
--- a/drivers/gpu/drm/ast/ast_mode.c
+++ b/drivers/gpu/drm/ast/ast_mode.c
@@ -63,15 +63,18 @@ static inline void ast_load_palette_index(struct ast_private *ast,
 static void ast_crtc_load_lut(struct drm_crtc *crtc)
 {
 	struct ast_private *ast = crtc->dev->dev_private;
-	struct ast_crtc *ast_crtc = to_ast_crtc(crtc);
+	u16 *r, *g, *b;
 	int i;
 
 	if (!crtc->enabled)
 		return;
 
+	r = crtc->gamma_store;
+	g = r + crtc->gamma_size;
+	b = g + crtc->gamma_size;
+
 	for (i = 0; i < 256; i++)
-		ast_load_palette_index(ast, i, ast_crtc->lut_r[i],
-				       ast_crtc->lut_g[i], ast_crtc->lut_b[i]);
+		ast_load_palette_index(ast, i, *r++ >> 8, *g++ >> 8, *b++ >> 8);
 }
 
 static bool ast_get_vbios_mode_info(struct drm_crtc *crtc, struct drm_display_mode *mode,
@@ -613,7 +616,23 @@ static int ast_crtc_mode_set(struct drm_crtc *crtc,
 
 static void ast_crtc_disable(struct drm_crtc *crtc)
 {
+	int ret;
 
+	DRM_DEBUG_KMS("\n");
+	ast_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
+	if (crtc->primary->fb) {
+		struct ast_framebuffer *ast_fb = to_ast_framebuffer(crtc->primary->fb);
+		struct drm_gem_object *obj = ast_fb->obj;
+		struct ast_bo *bo = gem_to_ast_bo(obj);
+
+		ret = ast_bo_reserve(bo, false);
+		if (ret)
+			return;
+
+		ast_bo_push_sysram(bo);
+		ast_bo_unreserve(bo);
+	}
+	crtc->primary->fb = NULL;
 }
 
 static void ast_crtc_prepare(struct drm_crtc *crtc)
@@ -633,7 +652,6 @@ static const struct drm_crtc_helper_funcs ast_crtc_helper_funcs = {
 	.mode_set = ast_crtc_mode_set,
 	.mode_set_base = ast_crtc_mode_set_base,
 	.disable = ast_crtc_disable,
-	.load_lut = ast_crtc_load_lut,
 	.prepare = ast_crtc_prepare,
 	.commit = ast_crtc_commit,
 
@@ -648,15 +666,6 @@ static int ast_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
 			      u16 *blue, uint32_t size,
 			      struct drm_modeset_acquire_ctx *ctx)
 {
-	struct ast_crtc *ast_crtc = to_ast_crtc(crtc);
-	int i;
-
-	/* userspace palettes are always correct as is */
-	for (i = 0; i < size; i++) {
-		ast_crtc->lut_r[i] = red[i] >> 8;
-		ast_crtc->lut_g[i] = green[i] >> 8;
-		ast_crtc->lut_b[i] = blue[i] >> 8;
-	}
 	ast_crtc_load_lut(crtc);
 
 	return 0;
@@ -681,7 +690,6 @@ static const struct drm_crtc_funcs ast_crtc_funcs = {
 static int ast_crtc_init(struct drm_device *dev)
 {
 	struct ast_crtc *crtc;
-	int i;
 
 	crtc = kzalloc(sizeof(struct ast_crtc), GFP_KERNEL);
 	if (!crtc)
@@ -690,12 +698,6 @@ static int ast_crtc_init(struct drm_device *dev)
 	drm_crtc_init(dev, &crtc->base, &ast_crtc_funcs);
 	drm_mode_crtc_set_gamma_size(&crtc->base, 256);
 	drm_crtc_helper_add(&crtc->base, &ast_crtc_helper_funcs);
-
-	for (i = 0; i < 256; i++) {
-		crtc->lut_r[i] = i;
-		crtc->lut_g[i] = i;
-		crtc->lut_b[i] = i;
-	}
 	return 0;
 }
 
@@ -948,7 +950,7 @@ static void ast_cursor_fini(struct drm_device *dev)
 {
 	struct ast_private *ast = dev->dev_private;
 	ttm_bo_kunmap(&ast->cache_kmap);
-	drm_gem_object_unreference_unlocked(ast->cursor_cache);
+	drm_gem_object_put_unlocked(ast->cursor_cache);
 }
 
 int ast_mode_init(struct drm_device *dev)
@@ -1213,10 +1215,10 @@ static int ast_cursor_set(struct drm_crtc *crtc,
 
 	ast_show_cursor(crtc);
 
-	drm_gem_object_unreference_unlocked(obj);
+	drm_gem_object_put_unlocked(obj);
 	return 0;
 fail:
-	drm_gem_object_unreference_unlocked(obj);
+	drm_gem_object_put_unlocked(obj);
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/ast/ast_ttm.c b/drivers/gpu/drm/ast/ast_ttm.c
index 58084985e6cf..696a15dc2f3f 100644
--- a/drivers/gpu/drm/ast/ast_ttm.c
+++ b/drivers/gpu/drm/ast/ast_ttm.c
@@ -323,10 +323,8 @@ int ast_bo_create(struct drm_device *dev, int size, int align,
 		return -ENOMEM;
 
 	ret = drm_gem_object_init(dev, &astbo->gem, size);
-	if (ret) {
-		kfree(astbo);
-		return ret;
-	}
+	if (ret)
+		goto error;
 
 	astbo->bo.bdev = &ast->ttm.bdev;
 
@@ -340,10 +338,13 @@ int ast_bo_create(struct drm_device *dev, int size, int align,
 			  align >> PAGE_SHIFT, false, NULL, acc_size,
 			  NULL, NULL, ast_bo_ttm_destroy);
 	if (ret)
-		return ret;
+		goto error;
 
 	*pastbo = astbo;
 	return 0;
+error:
+	kfree(astbo);
+	return ret;
 }
 
 static inline u64 ast_bo_gpu_offset(struct ast_bo *bo)
@@ -376,7 +377,7 @@ int ast_bo_pin(struct ast_bo *bo, u32 pl_flag, u64 *gpu_addr)
 
 int ast_bo_unpin(struct ast_bo *bo)
 {
-	int i, ret;
+	int i;
 	if (!bo->pin_count) {
 		DRM_ERROR("unpin bad %p\n", bo);
 		return 0;
@@ -387,11 +388,7 @@ int ast_bo_unpin(struct ast_bo *bo)
 
 	for (i = 0; i < bo->placement.num_placement ; i++)
 		bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;
-	ret = ttm_bo_validate(&bo->bo, &bo->placement, false, false);
-	if (ret)
-		return ret;
-
-	return 0;
+	return ttm_bo_validate(&bo->bo, &bo->placement, false, false);
 }
 
 int ast_bo_push_sysram(struct ast_bo *bo)
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c
index 53489859997b..d73281095fac 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c
@@ -149,7 +149,8 @@ atmel_hlcdc_crtc_mode_valid(struct drm_crtc *c,
 	return atmel_hlcdc_dc_mode_valid(crtc->dc, mode);
 }
 
-static void atmel_hlcdc_crtc_disable(struct drm_crtc *c)
+static void atmel_hlcdc_crtc_atomic_disable(struct drm_crtc *c,
+					    struct drm_crtc_state *old_state)
 {
 	struct drm_device *dev = c->dev;
 	struct atmel_hlcdc_crtc *crtc = drm_crtc_to_atmel_hlcdc_crtc(c);
@@ -183,7 +184,8 @@ static void atmel_hlcdc_crtc_disable(struct drm_crtc *c)
 	pm_runtime_put_sync(dev->dev);
 }
 
-static void atmel_hlcdc_crtc_enable(struct drm_crtc *c)
+static void atmel_hlcdc_crtc_atomic_enable(struct drm_crtc *c,
+					   struct drm_crtc_state *old_state)
 {
 	struct drm_device *dev = c->dev;
 	struct atmel_hlcdc_crtc *crtc = drm_crtc_to_atmel_hlcdc_crtc(c);
@@ -235,7 +237,7 @@ static int atmel_hlcdc_crtc_select_output_mode(struct drm_crtc_state *state)
 
 	crtc = drm_crtc_to_atmel_hlcdc_crtc(state->crtc);
 
-	for_each_connector_in_state(state->state, connector, cstate, i) {
+	for_each_new_connector_in_state(state->state, connector, cstate, i) {
 		struct drm_display_info *info = &connector->display_info;
 		unsigned int supported_fmts = 0;
 		int j;
@@ -319,11 +321,11 @@ static const struct drm_crtc_helper_funcs lcdc_crtc_helper_funcs = {
 	.mode_set = drm_helper_crtc_mode_set,
 	.mode_set_nofb = atmel_hlcdc_crtc_mode_set_nofb,
 	.mode_set_base = drm_helper_crtc_mode_set_base,
-	.disable = atmel_hlcdc_crtc_disable,
-	.enable = atmel_hlcdc_crtc_enable,
 	.atomic_check = atmel_hlcdc_crtc_atomic_check,
 	.atomic_begin = atmel_hlcdc_crtc_atomic_begin,
 	.atomic_flush = atmel_hlcdc_crtc_atomic_flush,
+	.atomic_enable = atmel_hlcdc_crtc_atomic_enable,
+	.atomic_disable = atmel_hlcdc_crtc_atomic_disable,
 };
 
 static void atmel_hlcdc_crtc_destroy(struct drm_crtc *c)
@@ -429,6 +431,7 @@ static const struct drm_crtc_funcs atmel_hlcdc_crtc_funcs = {
 	.atomic_destroy_state = atmel_hlcdc_crtc_destroy_state,
 	.enable_vblank = atmel_hlcdc_crtc_enable_vblank,
 	.disable_vblank = atmel_hlcdc_crtc_disable_vblank,
+	.gamma_set = drm_atomic_helper_legacy_gamma_set,
 };
 
 int atmel_hlcdc_crtc_create(struct drm_device *dev)
@@ -484,6 +487,10 @@ int atmel_hlcdc_crtc_create(struct drm_device *dev)
 	drm_crtc_helper_add(&crtc->base, &lcdc_crtc_helper_funcs);
 	drm_crtc_vblank_reset(&crtc->base);
 
+	drm_mode_crtc_set_gamma_size(&crtc->base, ATMEL_HLCDC_CLUT_SIZE);
+	drm_crtc_enable_color_mgmt(&crtc->base, 0, false,
+				   ATMEL_HLCDC_CLUT_SIZE);
+
 	dc->crtc = &crtc->base;
 
 	return 0;
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
index 30dbffdb45a3..74d66e11f688 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
@@ -42,6 +42,7 @@ static const struct atmel_hlcdc_layer_desc atmel_hlcdc_at91sam9n12_layers[] = {
 			.default_color = 3,
 			.general_config = 4,
 		},
+		.clut_offset = 0x400,
 	},
 };
 
@@ -73,6 +74,7 @@ static const struct atmel_hlcdc_layer_desc atmel_hlcdc_at91sam9x5_layers[] = {
 			.disc_pos = 5,
 			.disc_size = 6,
 		},
+		.clut_offset = 0x400,
 	},
 	{
 		.name = "overlay1",
@@ -91,6 +93,7 @@ static const struct atmel_hlcdc_layer_desc atmel_hlcdc_at91sam9x5_layers[] = {
 			.chroma_key_mask = 8,
 			.general_config = 9,
 		},
+		.clut_offset = 0x800,
 	},
 	{
 		.name = "high-end-overlay",
@@ -112,6 +115,7 @@ static const struct atmel_hlcdc_layer_desc atmel_hlcdc_at91sam9x5_layers[] = {
 			.scaler_config = 13,
 			.csc = 14,
 		},
+		.clut_offset = 0x1000,
 	},
 	{
 		.name = "cursor",
@@ -131,6 +135,7 @@ static const struct atmel_hlcdc_layer_desc atmel_hlcdc_at91sam9x5_layers[] = {
 			.chroma_key_mask = 8,
 			.general_config = 9,
 		},
+		.clut_offset = 0x1400,
 	},
 };
 
@@ -162,6 +167,7 @@ static const struct atmel_hlcdc_layer_desc atmel_hlcdc_sama5d3_layers[] = {
 			.disc_pos = 5,
 			.disc_size = 6,
 		},
+		.clut_offset = 0x600,
 	},
 	{
 		.name = "overlay1",
@@ -180,6 +186,7 @@ static const struct atmel_hlcdc_layer_desc atmel_hlcdc_sama5d3_layers[] = {
 			.chroma_key_mask = 8,
 			.general_config = 9,
 		},
+		.clut_offset = 0xa00,
 	},
 	{
 		.name = "overlay2",
@@ -198,6 +205,7 @@ static const struct atmel_hlcdc_layer_desc atmel_hlcdc_sama5d3_layers[] = {
 			.chroma_key_mask = 8,
 			.general_config = 9,
 		},
+		.clut_offset = 0xe00,
 	},
 	{
 		.name = "high-end-overlay",
@@ -223,6 +231,7 @@ static const struct atmel_hlcdc_layer_desc atmel_hlcdc_sama5d3_layers[] = {
 			},
 			.csc = 14,
 		},
+		.clut_offset = 0x1200,
 	},
 	{
 		.name = "cursor",
@@ -244,6 +253,7 @@ static const struct atmel_hlcdc_layer_desc atmel_hlcdc_sama5d3_layers[] = {
 			.general_config = 9,
 			.scaler_config = 13,
 		},
+		.clut_offset = 0x1600,
 	},
 };
 
@@ -275,6 +285,7 @@ static const struct atmel_hlcdc_layer_desc atmel_hlcdc_sama5d4_layers[] = {
 			.disc_pos = 5,
 			.disc_size = 6,
 		},
+		.clut_offset = 0x600,
 	},
 	{
 		.name = "overlay1",
@@ -293,6 +304,7 @@ static const struct atmel_hlcdc_layer_desc atmel_hlcdc_sama5d4_layers[] = {
 			.chroma_key_mask = 8,
 			.general_config = 9,
 		},
+		.clut_offset = 0xa00,
 	},
 	{
 		.name = "overlay2",
@@ -311,6 +323,7 @@ static const struct atmel_hlcdc_layer_desc atmel_hlcdc_sama5d4_layers[] = {
 			.chroma_key_mask = 8,
 			.general_config = 9,
 		},
+		.clut_offset = 0xe00,
 	},
 	{
 		.name = "high-end-overlay",
@@ -336,6 +349,7 @@ static const struct atmel_hlcdc_layer_desc atmel_hlcdc_sama5d4_layers[] = {
 			},
 			.csc = 14,
 		},
+		.clut_offset = 0x1200,
 	},
 };
 
@@ -451,8 +465,7 @@ static void atmel_hlcdc_fb_output_poll_changed(struct drm_device *dev)
 {
 	struct atmel_hlcdc_dc *dc = dev->dev_private;
 
-	if (dc->fbdev)
-		drm_fbdev_cma_hotplug_event(dc->fbdev);
+	drm_fbdev_cma_hotplug_event(dc->fbdev);
 }
 
 struct atmel_hlcdc_dc_commit {
@@ -526,14 +539,13 @@ static int atmel_hlcdc_dc_atomic_commit(struct drm_device *dev,
 		dc->commit.pending = true;
 	spin_unlock(&dc->commit.wait.lock);
 
-	if (ret) {
-		kfree(commit);
-		goto error;
-	}
+	if (ret)
+		goto err_free;
 
-	/* Swap the state, this is the point of no return. */
-	drm_atomic_helper_swap_state(state, true);
+	/* We have our own synchronization through the commit lock. */
+	BUG_ON(drm_atomic_helper_swap_state(state, false) < 0);
 
+	/* Swap state succeeded, this is the point of no return. */
 	drm_atomic_state_get(state);
 	if (async)
 		queue_work(dc->wq, &commit->work);
@@ -542,6 +554,8 @@ static int atmel_hlcdc_dc_atomic_commit(struct drm_device *dev,
 
 	return 0;
 
+err_free:
+	kfree(commit);
 error:
 	drm_atomic_helper_cleanup_planes(dev, state);
 	return ret;
@@ -747,8 +761,6 @@ static struct drm_driver atmel_hlcdc_dc_driver = {
 	.gem_prime_vunmap = drm_gem_cma_prime_vunmap,
 	.gem_prime_mmap = drm_gem_cma_prime_mmap,
 	.dumb_create = drm_gem_cma_dumb_create,
-	.dumb_map_offset = drm_gem_cma_dumb_map_offset,
-	.dumb_destroy = drm_gem_dumb_destroy,
 	.fops = &fops,
 	.name = "atmel-hlcdc",
 	.desc = "Atmel HLCD Controller DRM",
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.h b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.h
index b0596a84c1b8..4237b0446721 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.h
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.h
@@ -88,6 +88,11 @@
 #define ATMEL_HLCDC_YUV422SWP			BIT(17)
 #define ATMEL_HLCDC_DSCALEOPT			BIT(20)
 
+#define ATMEL_HLCDC_C1_MODE			ATMEL_HLCDC_CLUT_MODE(0)
+#define ATMEL_HLCDC_C2_MODE			ATMEL_HLCDC_CLUT_MODE(1)
+#define ATMEL_HLCDC_C4_MODE			ATMEL_HLCDC_CLUT_MODE(2)
+#define ATMEL_HLCDC_C8_MODE			ATMEL_HLCDC_CLUT_MODE(3)
+
 #define ATMEL_HLCDC_XRGB4444_MODE		ATMEL_HLCDC_RGB_MODE(0)
 #define ATMEL_HLCDC_ARGB4444_MODE		ATMEL_HLCDC_RGB_MODE(1)
 #define ATMEL_HLCDC_RGBA4444_MODE		ATMEL_HLCDC_RGB_MODE(2)
@@ -142,6 +147,8 @@
 #define ATMEL_HLCDC_DMA_CHANNEL_DSCR_DONE	BIT(2)
 #define ATMEL_HLCDC_DMA_CHANNEL_DSCR_OVERRUN	BIT(3)
 
+#define ATMEL_HLCDC_CLUT_SIZE			256
+
 #define ATMEL_HLCDC_MAX_LAYERS			6
 
 /**
@@ -259,6 +266,7 @@ struct atmel_hlcdc_layer_desc {
 	int id;
 	int regs_offset;
 	int cfgs_offset;
+	int clut_offset;
 	struct atmel_hlcdc_formats *formats;
 	struct atmel_hlcdc_layer_cfg_layout layout;
 	int max_width;
@@ -414,6 +422,14 @@ static inline u32 atmel_hlcdc_layer_read_cfg(struct atmel_hlcdc_layer *layer,
 					  (cfgid * sizeof(u32)));
 }
 
+static inline void atmel_hlcdc_layer_write_clut(struct atmel_hlcdc_layer *layer,
+						unsigned int c, u32 val)
+{
+	regmap_write(layer->regmap,
+		     layer->desc->clut_offset + c * sizeof(u32),
+		     val);
+}
+
 static inline void atmel_hlcdc_layer_init(struct atmel_hlcdc_layer *layer,
 				const struct atmel_hlcdc_layer_desc *desc,
 				struct regmap *regmap)
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c
index 1124200bb280..703c2d13603f 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c
@@ -83,6 +83,7 @@ drm_plane_state_to_atmel_hlcdc_plane_state(struct drm_plane_state *s)
 #define SUBPIXEL_MASK			0xffff
 
 static uint32_t rgb_formats[] = {
+	DRM_FORMAT_C8,
 	DRM_FORMAT_XRGB4444,
 	DRM_FORMAT_ARGB4444,
 	DRM_FORMAT_RGBA4444,
@@ -100,6 +101,7 @@ struct atmel_hlcdc_formats atmel_hlcdc_plane_rgb_formats = {
 };
 
 static uint32_t rgb_and_yuv_formats[] = {
+	DRM_FORMAT_C8,
 	DRM_FORMAT_XRGB4444,
 	DRM_FORMAT_ARGB4444,
 	DRM_FORMAT_RGBA4444,
@@ -128,6 +130,9 @@ struct atmel_hlcdc_formats atmel_hlcdc_plane_rgb_and_yuv_formats = {
 static int atmel_hlcdc_format_to_plane_mode(u32 format, u32 *mode)
 {
 	switch (format) {
+	case DRM_FORMAT_C8:
+		*mode = ATMEL_HLCDC_C8_MODE;
+		break;
 	case DRM_FORMAT_XRGB4444:
 		*mode = ATMEL_HLCDC_XRGB4444_MODE;
 		break;
@@ -424,6 +429,29 @@ static void atmel_hlcdc_plane_update_format(struct atmel_hlcdc_plane *plane,
 				    ATMEL_HLCDC_LAYER_FORMAT_CFG, cfg);
 }
 
+static void atmel_hlcdc_plane_update_clut(struct atmel_hlcdc_plane *plane)
+{
+	struct drm_crtc *crtc = plane->base.crtc;
+	struct drm_color_lut *lut;
+	int idx;
+
+	if (!crtc || !crtc->state)
+		return;
+
+	if (!crtc->state->color_mgmt_changed || !crtc->state->gamma_lut)
+		return;
+
+	lut = (struct drm_color_lut *)crtc->state->gamma_lut->data;
+
+	for (idx = 0; idx < ATMEL_HLCDC_CLUT_SIZE; idx++, lut++) {
+		u32 val = ((lut->red << 8) & 0xff0000) |
+			(lut->green & 0xff00) |
+			(lut->blue >> 8);
+
+		atmel_hlcdc_layer_write_clut(&plane->layer, idx, val);
+	}
+}
+
 static void atmel_hlcdc_plane_update_buffers(struct atmel_hlcdc_plane *plane,
 					struct atmel_hlcdc_plane_state *state)
 {
@@ -768,6 +796,7 @@ static void atmel_hlcdc_plane_atomic_update(struct drm_plane *p,
 	atmel_hlcdc_plane_update_pos_and_size(plane, state);
 	atmel_hlcdc_plane_update_general_settings(plane, state);
 	atmel_hlcdc_plane_update_format(plane, state);
+	atmel_hlcdc_plane_update_clut(plane);
 	atmel_hlcdc_plane_update_buffers(plane, state);
 	atmel_hlcdc_plane_update_disc_area(plane, state);
 
@@ -809,7 +838,7 @@ static void atmel_hlcdc_plane_destroy(struct drm_plane *p)
 	struct atmel_hlcdc_plane *plane = drm_plane_to_atmel_hlcdc_plane(p);
 
 	if (plane->base.fb)
-		drm_framebuffer_unreference(plane->base.fb);
+		drm_framebuffer_put(plane->base.fb);
 
 	drm_plane_cleanup(p);
 }
@@ -911,7 +940,7 @@ void atmel_hlcdc_plane_irq(struct atmel_hlcdc_plane *plane)
 			desc->name);
 }
 
-static struct drm_plane_helper_funcs atmel_hlcdc_layer_plane_helper_funcs = {
+static const struct drm_plane_helper_funcs atmel_hlcdc_layer_plane_helper_funcs = {
 	.atomic_check = atmel_hlcdc_plane_atomic_check,
 	.atomic_update = atmel_hlcdc_plane_atomic_update,
 	.atomic_disable = atmel_hlcdc_plane_atomic_disable,
@@ -958,7 +987,7 @@ static void atmel_hlcdc_plane_reset(struct drm_plane *p)
 		state = drm_plane_state_to_atmel_hlcdc_plane_state(p->state);
 
 		if (state->base.fb)
-			drm_framebuffer_unreference(state->base.fb);
+			drm_framebuffer_put(state->base.fb);
 
 		kfree(state);
 		p->state = NULL;
@@ -996,7 +1025,7 @@ atmel_hlcdc_plane_atomic_duplicate_state(struct drm_plane *p)
 	}
 
 	if (copy->base.fb)
-		drm_framebuffer_reference(copy->base.fb);
+		drm_framebuffer_get(copy->base.fb);
 
 	return &copy->base;
 }
@@ -1015,15 +1044,14 @@ static void atmel_hlcdc_plane_atomic_destroy_state(struct drm_plane *p,
 	}
 
 	if (s->fb)
-		drm_framebuffer_unreference(s->fb);
+		drm_framebuffer_put(s->fb);
 
 	kfree(state);
 }
 
-static struct drm_plane_funcs layer_plane_funcs = {
+static const struct drm_plane_funcs layer_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
-	.set_property = drm_atomic_helper_plane_set_property,
 	.destroy = atmel_hlcdc_plane_destroy,
 	.reset = atmel_hlcdc_plane_reset,
 	.atomic_duplicate_state = atmel_hlcdc_plane_atomic_duplicate_state,
@@ -1058,7 +1086,8 @@ static int atmel_hlcdc_plane_create(struct drm_device *dev,
 	ret = drm_universal_plane_init(dev, &plane->base, 0,
 				       &layer_plane_funcs,
 				       desc->formats->formats,
-				       desc->formats->nformats, type, NULL);
+				       desc->formats->nformats,
+				       NULL, type, NULL);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/bochs/bochs_drv.c b/drivers/gpu/drm/bochs/bochs_drv.c
index aa342515ddf4..7b20318483e4 100644
--- a/drivers/gpu/drm/bochs/bochs_drv.c
+++ b/drivers/gpu/drm/bochs/bochs_drv.c
@@ -84,7 +84,6 @@ static struct drm_driver bochs_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET,
 	.load			= bochs_load,
 	.unload			= bochs_unload,
-	.set_busid		= drm_pci_set_busid,
 	.fops			= &bochs_fops,
 	.name			= "bochs-drm",
 	.desc			= "bochs dispi vga interface (qemu stdvga)",
@@ -94,7 +93,6 @@ static struct drm_driver bochs_driver = {
 	.gem_free_object_unlocked = bochs_gem_free_object,
 	.dumb_create            = bochs_dumb_create,
 	.dumb_map_offset        = bochs_dumb_mmap_offset,
-	.dumb_destroy           = drm_gem_dumb_destroy,
 };
 
 /* ---------------------------------------------------------------------- */
@@ -224,12 +222,12 @@ static int __init bochs_init(void)
 	if (bochs_modeset == 0)
 		return -EINVAL;
 
-	return drm_pci_init(&bochs_driver, &bochs_pci_driver);
+	return pci_register_driver(&bochs_pci_driver);
 }
 
 static void __exit bochs_exit(void)
 {
-	drm_pci_exit(&bochs_driver, &bochs_pci_driver);
+	pci_unregister_driver(&bochs_pci_driver);
 }
 
 module_init(bochs_init);
diff --git a/drivers/gpu/drm/bochs/bochs_fbdev.c b/drivers/gpu/drm/bochs/bochs_fbdev.c
index c38deffa14de..14eb8d0d5a00 100644
--- a/drivers/gpu/drm/bochs/bochs_fbdev.c
+++ b/drivers/gpu/drm/bochs/bochs_fbdev.c
@@ -23,9 +23,9 @@ static int bochsfb_mmap(struct fb_info *info,
 static struct fb_ops bochsfb_ops = {
 	.owner = THIS_MODULE,
 	DRM_FB_HELPER_DEFAULT_OPS,
-	.fb_fillrect = drm_fb_helper_sys_fillrect,
-	.fb_copyarea = drm_fb_helper_sys_copyarea,
-	.fb_imageblit = drm_fb_helper_sys_imageblit,
+	.fb_fillrect = drm_fb_helper_cfb_fillrect,
+	.fb_copyarea = drm_fb_helper_cfb_copyarea,
+	.fb_imageblit = drm_fb_helper_cfb_imageblit,
 	.fb_mmap = bochsfb_mmap,
 };
 
@@ -118,7 +118,6 @@ static int bochsfb_create(struct drm_fb_helper *helper,
 
 	strcpy(info->fix.id, "bochsdrmfb");
 
-	info->flags = FBINFO_DEFAULT;
 	info->fbops = &bochsfb_ops;
 
 	drm_fb_helper_fill_fix(info, fb->pitches[0], fb->format->depth);
diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
index f75ab6278113..b2431aee7887 100644
--- a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
+++ b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
@@ -785,8 +785,7 @@ adv7511_connector_detect(struct drm_connector *connector, bool force)
 	return adv7511_detect(adv, connector);
 }
 
-static struct drm_connector_funcs adv7511_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
+static const struct drm_connector_funcs adv7511_connector_funcs = {
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = adv7511_connector_detect,
 	.destroy = drm_connector_cleanup,
@@ -857,7 +856,7 @@ static int adv7511_bridge_attach(struct drm_bridge *bridge)
 	return ret;
 }
 
-static struct drm_bridge_funcs adv7511_bridge_funcs = {
+static const struct drm_bridge_funcs adv7511_bridge_funcs = {
 	.enable = adv7511_bridge_enable,
 	.disable = adv7511_bridge_disable,
 	.mode_set = adv7511_bridge_mode_set,
@@ -1126,11 +1125,7 @@ static int adv7511_probe(struct i2c_client *i2c, const struct i2c_device_id *id)
 	adv7511->bridge.funcs = &adv7511_bridge_funcs;
 	adv7511->bridge.of_node = dev->of_node;
 
-	ret = drm_bridge_add(&adv7511->bridge);
-	if (ret) {
-		dev_err(dev, "failed to add adv7511 bridge\n");
-		goto err_unregister_cec;
-	}
+	drm_bridge_add(&adv7511->bridge);
 
 	adv7511_audio_init(dev, adv7511);
 
diff --git a/drivers/gpu/drm/bridge/analogix-anx78xx.c b/drivers/gpu/drm/bridge/analogix-anx78xx.c
index 9006578b9789..9385eb0b1ee4 100644
--- a/drivers/gpu/drm/bridge/analogix-anx78xx.c
+++ b/drivers/gpu/drm/bridge/analogix-anx78xx.c
@@ -1002,7 +1002,6 @@ static enum drm_connector_status anx78xx_detect(struct drm_connector *connector,
 }
 
 static const struct drm_connector_funcs anx78xx_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = anx78xx_detect,
 	.destroy = drm_connector_cleanup,
@@ -1097,7 +1096,8 @@ static void anx78xx_bridge_mode_set(struct drm_bridge *bridge,
 
 	mutex_lock(&anx78xx->lock);
 
-	err = drm_hdmi_avi_infoframe_from_display_mode(&frame, adjusted_mode);
+	err = drm_hdmi_avi_infoframe_from_display_mode(&frame, adjusted_mode,
+						       false);
 	if (err) {
 		DRM_ERROR("Failed to setup AVI infoframe: %d\n", err);
 		goto unlock;
@@ -1438,11 +1438,7 @@ static int anx78xx_i2c_probe(struct i2c_client *client,
 
 	anx78xx->bridge.funcs = &anx78xx_bridge_funcs;
 
-	err = drm_bridge_add(&anx78xx->bridge);
-	if (err < 0) {
-		DRM_ERROR("Failed to add drm bridge: %d\n", err);
-		goto err_poweroff;
-	}
+	drm_bridge_add(&anx78xx->bridge);
 
 	/* If cable is pulled out, just poweroff and wait for HPD event */
 	if (!gpiod_get_value(anx78xx->pdata.gpiod_hpd))
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
index 4c758ed51939..5dd3f1cd074a 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
@@ -1005,7 +1005,6 @@ analogix_dp_detect(struct drm_connector *connector, bool force)
 }
 
 static const struct drm_connector_funcs analogix_dp_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = analogix_dp_detect,
 	.destroy = drm_connector_cleanup,
diff --git a/drivers/gpu/drm/bridge/dumb-vga-dac.c b/drivers/gpu/drm/bridge/dumb-vga-dac.c
index 831a606c4706..de5e7dee7ad6 100644
--- a/drivers/gpu/drm/bridge/dumb-vga-dac.c
+++ b/drivers/gpu/drm/bridge/dumb-vga-dac.c
@@ -92,7 +92,6 @@ dumb_vga_connector_detect(struct drm_connector *connector, bool force)
 }
 
 static const struct drm_connector_funcs dumb_vga_con_funcs = {
-	.dpms			= drm_atomic_helper_connector_dpms,
 	.detect			= dumb_vga_connector_detect,
 	.fill_modes		= drm_helper_probe_single_connector_modes,
 	.destroy		= drm_connector_cleanup,
@@ -177,7 +176,6 @@ static struct i2c_adapter *dumb_vga_retrieve_ddc(struct device *dev)
 static int dumb_vga_probe(struct platform_device *pdev)
 {
 	struct dumb_vga *vga;
-	int ret;
 
 	vga = devm_kzalloc(&pdev->dev, sizeof(*vga), GFP_KERNEL);
 	if (!vga)
@@ -186,7 +184,7 @@ static int dumb_vga_probe(struct platform_device *pdev)
 
 	vga->vdd = devm_regulator_get_optional(&pdev->dev, "vdd");
 	if (IS_ERR(vga->vdd)) {
-		ret = PTR_ERR(vga->vdd);
+		int ret = PTR_ERR(vga->vdd);
 		if (ret == -EPROBE_DEFER)
 			return -EPROBE_DEFER;
 		vga->vdd = NULL;
@@ -207,11 +205,9 @@ static int dumb_vga_probe(struct platform_device *pdev)
 	vga->bridge.funcs = &dumb_vga_bridge_funcs;
 	vga->bridge.of_node = pdev->dev.of_node;
 
-	ret = drm_bridge_add(&vga->bridge);
-	if (ret && !IS_ERR(vga->ddc))
-		i2c_put_adapter(vga->ddc);
+	drm_bridge_add(&vga->bridge);
 
-	return ret;
+	return 0;
 }
 
 static int dumb_vga_remove(struct platform_device *pdev)
diff --git a/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c b/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c
index 11f11086a68f..7ccadba7c98c 100644
--- a/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c
+++ b/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c
@@ -193,7 +193,6 @@ static enum drm_connector_status ge_b850v3_lvds_detect(
 }
 
 static const struct drm_connector_funcs ge_b850v3_lvds_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = ge_b850v3_lvds_detect,
 	.destroy = drm_connector_cleanup,
diff --git a/drivers/gpu/drm/bridge/nxp-ptn3460.c b/drivers/gpu/drm/bridge/nxp-ptn3460.c
index 4f64e717e01b..d64a3283822a 100644
--- a/drivers/gpu/drm/bridge/nxp-ptn3460.c
+++ b/drivers/gpu/drm/bridge/nxp-ptn3460.c
@@ -238,7 +238,6 @@ static const struct drm_connector_helper_funcs ptn3460_connector_helper_funcs =
 };
 
 static const struct drm_connector_funcs ptn3460_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = drm_connector_cleanup,
 	.reset = drm_atomic_helper_connector_reset,
@@ -332,11 +331,7 @@ static int ptn3460_probe(struct i2c_client *client,
 
 	ptn_bridge->bridge.funcs = &ptn3460_bridge_funcs;
 	ptn_bridge->bridge.of_node = dev->of_node;
-	ret = drm_bridge_add(&ptn_bridge->bridge);
-	if (ret) {
-		DRM_ERROR("Failed to add bridge\n");
-		return ret;
-	}
+	drm_bridge_add(&ptn_bridge->bridge);
 
 	i2c_set_clientdata(client, ptn_bridge);
 
diff --git a/drivers/gpu/drm/bridge/panel.c b/drivers/gpu/drm/bridge/panel.c
index 67fe19e5a9c6..e0cca19b4044 100644
--- a/drivers/gpu/drm/bridge/panel.c
+++ b/drivers/gpu/drm/bridge/panel.c
@@ -50,7 +50,6 @@ panel_bridge_connector_helper_funcs = {
 };
 
 static const struct drm_connector_funcs panel_bridge_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.reset = drm_atomic_helper_connector_reset,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = drm_connector_cleanup,
@@ -158,7 +157,6 @@ struct drm_bridge *drm_panel_bridge_add(struct drm_panel *panel,
 					u32 connector_type)
 {
 	struct panel_bridge *panel_bridge;
-	int ret;
 
 	if (!panel)
 		return ERR_PTR(-EINVAL);
@@ -176,9 +174,7 @@ struct drm_bridge *drm_panel_bridge_add(struct drm_panel *panel,
 	panel_bridge->bridge.of_node = panel->dev->of_node;
 #endif
 
-	ret = drm_bridge_add(&panel_bridge->bridge);
-	if (ret)
-		return ERR_PTR(ret);
+	drm_bridge_add(&panel_bridge->bridge);
 
 	return &panel_bridge->bridge;
 }
@@ -198,3 +194,33 @@ void drm_panel_bridge_remove(struct drm_bridge *bridge)
 	devm_kfree(panel_bridge->panel->dev, bridge);
 }
 EXPORT_SYMBOL(drm_panel_bridge_remove);
+
+static void devm_drm_panel_bridge_release(struct device *dev, void *res)
+{
+	struct drm_bridge **bridge = res;
+
+	drm_panel_bridge_remove(*bridge);
+}
+
+struct drm_bridge *devm_drm_panel_bridge_add(struct device *dev,
+					     struct drm_panel *panel,
+					     u32 connector_type)
+{
+	struct drm_bridge **ptr, *bridge;
+
+	ptr = devres_alloc(devm_drm_panel_bridge_release, sizeof(*ptr),
+			   GFP_KERNEL);
+	if (!ptr)
+		return ERR_PTR(-ENOMEM);
+
+	bridge = drm_panel_bridge_add(panel, connector_type);
+	if (!IS_ERR(bridge)) {
+		*ptr = bridge;
+		devres_add(dev, ptr);
+	} else {
+		devres_free(ptr);
+	}
+
+	return bridge;
+}
+EXPORT_SYMBOL(devm_drm_panel_bridge_add);
diff --git a/drivers/gpu/drm/bridge/parade-ps8622.c b/drivers/gpu/drm/bridge/parade-ps8622.c
index 6f22f9fec9bf..81198f5e9afa 100644
--- a/drivers/gpu/drm/bridge/parade-ps8622.c
+++ b/drivers/gpu/drm/bridge/parade-ps8622.c
@@ -476,7 +476,6 @@ static const struct drm_connector_helper_funcs ps8622_connector_helper_funcs = {
 };
 
 static const struct drm_connector_funcs ps8622_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = drm_connector_cleanup,
 	.reset = drm_atomic_helper_connector_reset,
@@ -598,11 +597,7 @@ static int ps8622_probe(struct i2c_client *client,
 
 	ps8622->bridge.funcs = &ps8622_bridge_funcs;
 	ps8622->bridge.of_node = dev->of_node;
-	ret = drm_bridge_add(&ps8622->bridge);
-	if (ret) {
-		DRM_ERROR("Failed to add bridge\n");
-		return ret;
-	}
+	drm_bridge_add(&ps8622->bridge);
 
 	i2c_set_clientdata(client, ps8622);
 
diff --git a/drivers/gpu/drm/bridge/sii902x.c b/drivers/gpu/drm/bridge/sii902x.c
index 9b87067c022c..b1ab4ab09532 100644
--- a/drivers/gpu/drm/bridge/sii902x.c
+++ b/drivers/gpu/drm/bridge/sii902x.c
@@ -124,7 +124,6 @@ sii902x_connector_detect(struct drm_connector *connector, bool force)
 }
 
 static const struct drm_connector_funcs sii902x_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.detect = sii902x_connector_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = drm_connector_cleanup,
@@ -269,7 +268,7 @@ static void sii902x_bridge_mode_set(struct drm_bridge *bridge,
 	if (ret)
 		return;
 
-	ret = drm_hdmi_avi_infoframe_from_display_mode(&frame, adj);
+	ret = drm_hdmi_avi_infoframe_from_display_mode(&frame, adj, false);
 	if (ret < 0) {
 		DRM_ERROR("couldn't fill AVI infoframe\n");
 		return;
@@ -418,11 +417,7 @@ static int sii902x_probe(struct i2c_client *client,
 
 	sii902x->bridge.funcs = &sii902x_bridge_funcs;
 	sii902x->bridge.of_node = dev->of_node;
-	ret = drm_bridge_add(&sii902x->bridge);
-	if (ret) {
-		dev_err(dev, "Failed to add drm_bridge\n");
-		return ret;
-	}
+	drm_bridge_add(&sii902x->bridge);
 
 	i2c_set_clientdata(client, sii902x);
 
diff --git a/drivers/gpu/drm/bridge/synopsys/Kconfig b/drivers/gpu/drm/bridge/synopsys/Kconfig
index 53e78d092d18..3cc53b44186e 100644
--- a/drivers/gpu/drm/bridge/synopsys/Kconfig
+++ b/drivers/gpu/drm/bridge/synopsys/Kconfig
@@ -2,6 +2,7 @@ config DRM_DW_HDMI
 	tristate
 	select DRM_KMS_HELPER
 	select REGMAP_MMIO
+	select CEC_CORE if CEC_NOTIFIER
 
 config DRM_DW_HDMI_AHB_AUDIO
 	tristate "Synopsys Designware AHB Audio interface"
@@ -22,3 +23,18 @@ config DRM_DW_HDMI_I2S_AUDIO
 	help
 	  Support the I2S Audio interface which is part of the Synopsys
 	  Designware HDMI block.
+
+config DRM_DW_HDMI_CEC
+	tristate "Synopsis Designware CEC interface"
+	depends on DRM_DW_HDMI
+	select CEC_CORE
+	select CEC_NOTIFIER
+	help
+	  Support the CE interface which is part of the Synopsys
+	  Designware HDMI block.
+
+config DRM_DW_MIPI_DSI
+	tristate
+	select DRM_KMS_HELPER
+	select DRM_MIPI_DSI
+	select DRM_PANEL_BRIDGE
diff --git a/drivers/gpu/drm/bridge/synopsys/Makefile b/drivers/gpu/drm/bridge/synopsys/Makefile
index 17aa7a65b57e..5dad97d920be 100644
--- a/drivers/gpu/drm/bridge/synopsys/Makefile
+++ b/drivers/gpu/drm/bridge/synopsys/Makefile
@@ -3,3 +3,6 @@
 obj-$(CONFIG_DRM_DW_HDMI) += dw-hdmi.o
 obj-$(CONFIG_DRM_DW_HDMI_AHB_AUDIO) += dw-hdmi-ahb-audio.o
 obj-$(CONFIG_DRM_DW_HDMI_I2S_AUDIO) += dw-hdmi-i2s-audio.o
+obj-$(CONFIG_DRM_DW_HDMI_CEC) += dw-hdmi-cec.o
+
+obj-$(CONFIG_DRM_DW_MIPI_DSI) += dw-mipi-dsi.o
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-ahb-audio.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-ahb-audio.c
index 8f2d1379c880..cf3f0caf9c63 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-ahb-audio.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-ahb-audio.c
@@ -517,7 +517,7 @@ static snd_pcm_uframes_t dw_hdmi_pointer(struct snd_pcm_substream *substream)
 	return bytes_to_frames(runtime, dw->buf_offset);
 }
 
-static struct snd_pcm_ops snd_dw_hdmi_ops = {
+static const struct snd_pcm_ops snd_dw_hdmi_ops = {
 	.open = dw_hdmi_open,
 	.close = dw_hdmi_close,
 	.ioctl = snd_pcm_lib_ioctl,
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-cec.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-cec.c
new file mode 100644
index 000000000000..6c323510f128
--- /dev/null
+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-cec.c
@@ -0,0 +1,327 @@
+/*
+ * Designware HDMI CEC driver
+ *
+ * Copyright (C) 2015-2017 Russell King.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+
+#include <drm/drm_edid.h>
+
+#include <media/cec.h>
+#include <media/cec-notifier.h>
+
+#include "dw-hdmi-cec.h"
+
+enum {
+	HDMI_IH_CEC_STAT0	= 0x0106,
+	HDMI_IH_MUTE_CEC_STAT0	= 0x0186,
+
+	HDMI_CEC_CTRL		= 0x7d00,
+	CEC_CTRL_START		= BIT(0),
+	CEC_CTRL_FRAME_TYP	= 3 << 1,
+	CEC_CTRL_RETRY		= 0 << 1,
+	CEC_CTRL_NORMAL		= 1 << 1,
+	CEC_CTRL_IMMED		= 2 << 1,
+
+	HDMI_CEC_STAT		= 0x7d01,
+	CEC_STAT_DONE		= BIT(0),
+	CEC_STAT_EOM		= BIT(1),
+	CEC_STAT_NACK		= BIT(2),
+	CEC_STAT_ARBLOST	= BIT(3),
+	CEC_STAT_ERROR_INIT	= BIT(4),
+	CEC_STAT_ERROR_FOLL	= BIT(5),
+	CEC_STAT_WAKEUP		= BIT(6),
+
+	HDMI_CEC_MASK		= 0x7d02,
+	HDMI_CEC_POLARITY	= 0x7d03,
+	HDMI_CEC_INT		= 0x7d04,
+	HDMI_CEC_ADDR_L		= 0x7d05,
+	HDMI_CEC_ADDR_H		= 0x7d06,
+	HDMI_CEC_TX_CNT		= 0x7d07,
+	HDMI_CEC_RX_CNT		= 0x7d08,
+	HDMI_CEC_TX_DATA0	= 0x7d10,
+	HDMI_CEC_RX_DATA0	= 0x7d20,
+	HDMI_CEC_LOCK		= 0x7d30,
+	HDMI_CEC_WKUPCTRL	= 0x7d31,
+};
+
+struct dw_hdmi_cec {
+	struct dw_hdmi *hdmi;
+	const struct dw_hdmi_cec_ops *ops;
+	u32 addresses;
+	struct cec_adapter *adap;
+	struct cec_msg rx_msg;
+	unsigned int tx_status;
+	bool tx_done;
+	bool rx_done;
+	struct cec_notifier *notify;
+	int irq;
+};
+
+static void dw_hdmi_write(struct dw_hdmi_cec *cec, u8 val, int offset)
+{
+	cec->ops->write(cec->hdmi, val, offset);
+}
+
+static u8 dw_hdmi_read(struct dw_hdmi_cec *cec, int offset)
+{
+	return cec->ops->read(cec->hdmi, offset);
+}
+
+static int dw_hdmi_cec_log_addr(struct cec_adapter *adap, u8 logical_addr)
+{
+	struct dw_hdmi_cec *cec = cec_get_drvdata(adap);
+
+	if (logical_addr == CEC_LOG_ADDR_INVALID)
+		cec->addresses = 0;
+	else
+		cec->addresses |= BIT(logical_addr) | BIT(15);
+
+	dw_hdmi_write(cec, cec->addresses & 255, HDMI_CEC_ADDR_L);
+	dw_hdmi_write(cec, cec->addresses >> 8, HDMI_CEC_ADDR_H);
+
+	return 0;
+}
+
+static int dw_hdmi_cec_transmit(struct cec_adapter *adap, u8 attempts,
+				u32 signal_free_time, struct cec_msg *msg)
+{
+	struct dw_hdmi_cec *cec = cec_get_drvdata(adap);
+	unsigned int i, ctrl;
+
+	switch (signal_free_time) {
+	case CEC_SIGNAL_FREE_TIME_RETRY:
+		ctrl = CEC_CTRL_RETRY;
+		break;
+	case CEC_SIGNAL_FREE_TIME_NEW_INITIATOR:
+	default:
+		ctrl = CEC_CTRL_NORMAL;
+		break;
+	case CEC_SIGNAL_FREE_TIME_NEXT_XFER:
+		ctrl = CEC_CTRL_IMMED;
+		break;
+	}
+
+	for (i = 0; i < msg->len; i++)
+		dw_hdmi_write(cec, msg->msg[i], HDMI_CEC_TX_DATA0 + i);
+
+	dw_hdmi_write(cec, msg->len, HDMI_CEC_TX_CNT);
+	dw_hdmi_write(cec, ctrl | CEC_CTRL_START, HDMI_CEC_CTRL);
+
+	return 0;
+}
+
+static irqreturn_t dw_hdmi_cec_hardirq(int irq, void *data)
+{
+	struct cec_adapter *adap = data;
+	struct dw_hdmi_cec *cec = cec_get_drvdata(adap);
+	unsigned int stat = dw_hdmi_read(cec, HDMI_IH_CEC_STAT0);
+	irqreturn_t ret = IRQ_HANDLED;
+
+	if (stat == 0)
+		return IRQ_NONE;
+
+	dw_hdmi_write(cec, stat, HDMI_IH_CEC_STAT0);
+
+	if (stat & CEC_STAT_ERROR_INIT) {
+		cec->tx_status = CEC_TX_STATUS_ERROR;
+		cec->tx_done = true;
+		ret = IRQ_WAKE_THREAD;
+	} else if (stat & CEC_STAT_DONE) {
+		cec->tx_status = CEC_TX_STATUS_OK;
+		cec->tx_done = true;
+		ret = IRQ_WAKE_THREAD;
+	} else if (stat & CEC_STAT_NACK) {
+		cec->tx_status = CEC_TX_STATUS_NACK;
+		cec->tx_done = true;
+		ret = IRQ_WAKE_THREAD;
+	}
+
+	if (stat & CEC_STAT_EOM) {
+		unsigned int len, i;
+
+		len = dw_hdmi_read(cec, HDMI_CEC_RX_CNT);
+		if (len > sizeof(cec->rx_msg.msg))
+			len = sizeof(cec->rx_msg.msg);
+
+		for (i = 0; i < len; i++)
+			cec->rx_msg.msg[i] =
+				dw_hdmi_read(cec, HDMI_CEC_RX_DATA0 + i);
+
+		dw_hdmi_write(cec, 0, HDMI_CEC_LOCK);
+
+		cec->rx_msg.len = len;
+		smp_wmb();
+		cec->rx_done = true;
+
+		ret = IRQ_WAKE_THREAD;
+	}
+
+	return ret;
+}
+
+static irqreturn_t dw_hdmi_cec_thread(int irq, void *data)
+{
+	struct cec_adapter *adap = data;
+	struct dw_hdmi_cec *cec = cec_get_drvdata(adap);
+
+	if (cec->tx_done) {
+		cec->tx_done = false;
+		cec_transmit_attempt_done(adap, cec->tx_status);
+	}
+	if (cec->rx_done) {
+		cec->rx_done = false;
+		smp_rmb();
+		cec_received_msg(adap, &cec->rx_msg);
+	}
+	return IRQ_HANDLED;
+}
+
+static int dw_hdmi_cec_enable(struct cec_adapter *adap, bool enable)
+{
+	struct dw_hdmi_cec *cec = cec_get_drvdata(adap);
+
+	if (!enable) {
+		dw_hdmi_write(cec, ~0, HDMI_CEC_MASK);
+		dw_hdmi_write(cec, ~0, HDMI_IH_MUTE_CEC_STAT0);
+		dw_hdmi_write(cec, 0, HDMI_CEC_POLARITY);
+
+		cec->ops->disable(cec->hdmi);
+	} else {
+		unsigned int irqs;
+
+		dw_hdmi_write(cec, 0, HDMI_CEC_CTRL);
+		dw_hdmi_write(cec, ~0, HDMI_IH_CEC_STAT0);
+		dw_hdmi_write(cec, 0, HDMI_CEC_LOCK);
+
+		dw_hdmi_cec_log_addr(cec->adap, CEC_LOG_ADDR_INVALID);
+
+		cec->ops->enable(cec->hdmi);
+
+		irqs = CEC_STAT_ERROR_INIT | CEC_STAT_NACK | CEC_STAT_EOM |
+		       CEC_STAT_DONE;
+		dw_hdmi_write(cec, irqs, HDMI_CEC_POLARITY);
+		dw_hdmi_write(cec, ~irqs, HDMI_CEC_MASK);
+		dw_hdmi_write(cec, ~irqs, HDMI_IH_MUTE_CEC_STAT0);
+	}
+	return 0;
+}
+
+static const struct cec_adap_ops dw_hdmi_cec_ops = {
+	.adap_enable = dw_hdmi_cec_enable,
+	.adap_log_addr = dw_hdmi_cec_log_addr,
+	.adap_transmit = dw_hdmi_cec_transmit,
+};
+
+static void dw_hdmi_cec_del(void *data)
+{
+	struct dw_hdmi_cec *cec = data;
+
+	cec_delete_adapter(cec->adap);
+}
+
+static int dw_hdmi_cec_probe(struct platform_device *pdev)
+{
+	struct dw_hdmi_cec_data *data = dev_get_platdata(&pdev->dev);
+	struct dw_hdmi_cec *cec;
+	int ret;
+
+	if (!data)
+		return -ENXIO;
+
+	/*
+	 * Our device is just a convenience - we want to link to the real
+	 * hardware device here, so that userspace can see the association
+	 * between the HDMI hardware and its associated CEC chardev.
+	 */
+	cec = devm_kzalloc(&pdev->dev, sizeof(*cec), GFP_KERNEL);
+	if (!cec)
+		return -ENOMEM;
+
+	cec->irq = data->irq;
+	cec->ops = data->ops;
+	cec->hdmi = data->hdmi;
+
+	platform_set_drvdata(pdev, cec);
+
+	dw_hdmi_write(cec, 0, HDMI_CEC_TX_CNT);
+	dw_hdmi_write(cec, ~0, HDMI_CEC_MASK);
+	dw_hdmi_write(cec, ~0, HDMI_IH_MUTE_CEC_STAT0);
+	dw_hdmi_write(cec, 0, HDMI_CEC_POLARITY);
+
+	cec->adap = cec_allocate_adapter(&dw_hdmi_cec_ops, cec, "dw_hdmi",
+					 CEC_CAP_LOG_ADDRS | CEC_CAP_TRANSMIT |
+					 CEC_CAP_RC | CEC_CAP_PASSTHROUGH,
+					 CEC_MAX_LOG_ADDRS);
+	if (IS_ERR(cec->adap))
+		return PTR_ERR(cec->adap);
+
+	/* override the module pointer */
+	cec->adap->owner = THIS_MODULE;
+
+	ret = devm_add_action(&pdev->dev, dw_hdmi_cec_del, cec);
+	if (ret) {
+		cec_delete_adapter(cec->adap);
+		return ret;
+	}
+
+	ret = devm_request_threaded_irq(&pdev->dev, cec->irq,
+					dw_hdmi_cec_hardirq,
+					dw_hdmi_cec_thread, IRQF_SHARED,
+					"dw-hdmi-cec", cec->adap);
+	if (ret < 0)
+		return ret;
+
+	cec->notify = cec_notifier_get(pdev->dev.parent);
+	if (!cec->notify)
+		return -ENOMEM;
+
+	ret = cec_register_adapter(cec->adap, pdev->dev.parent);
+	if (ret < 0) {
+		cec_notifier_put(cec->notify);
+		return ret;
+	}
+
+	/*
+	 * CEC documentation says we must not call cec_delete_adapter
+	 * after a successful call to cec_register_adapter().
+	 */
+	devm_remove_action(&pdev->dev, dw_hdmi_cec_del, cec);
+
+	cec_register_cec_notifier(cec->adap, cec->notify);
+
+	return 0;
+}
+
+static int dw_hdmi_cec_remove(struct platform_device *pdev)
+{
+	struct dw_hdmi_cec *cec = platform_get_drvdata(pdev);
+
+	cec_unregister_adapter(cec->adap);
+	cec_notifier_put(cec->notify);
+
+	return 0;
+}
+
+static struct platform_driver dw_hdmi_cec_driver = {
+	.probe	= dw_hdmi_cec_probe,
+	.remove	= dw_hdmi_cec_remove,
+	.driver = {
+		.name = "dw-hdmi-cec",
+	},
+};
+module_platform_driver(dw_hdmi_cec_driver);
+
+MODULE_AUTHOR("Russell King <rmk+kernel@armlinux.org.uk>");
+MODULE_DESCRIPTION("Synopsys Designware HDMI CEC driver for i.MX");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS(PLATFORM_MODULE_PREFIX "dw-hdmi-cec");
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-cec.h b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-cec.h
new file mode 100644
index 000000000000..cf4dc121a2c4
--- /dev/null
+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-cec.h
@@ -0,0 +1,19 @@
+#ifndef DW_HDMI_CEC_H
+#define DW_HDMI_CEC_H
+
+struct dw_hdmi;
+
+struct dw_hdmi_cec_ops {
+	void (*write)(struct dw_hdmi *hdmi, u8 val, int offset);
+	u8 (*read)(struct dw_hdmi *hdmi, int offset);
+	void (*enable)(struct dw_hdmi *hdmi);
+	void (*disable)(struct dw_hdmi *hdmi);
+};
+
+struct dw_hdmi_cec_data {
+	struct dw_hdmi *hdmi;
+	const struct dw_hdmi_cec_ops *ops;
+	int irq;
+};
+
+#endif
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-i2s-audio.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-i2s-audio.c
index b2cf59f54c88..3b7e5c59a5e9 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-i2s-audio.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-i2s-audio.c
@@ -1,7 +1,8 @@
 /*
  * dw-hdmi-i2s-audio.c
  *
- * Copyright (c) 2016 Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>
+ * Copyright (c) 2017 Renesas Solutions Corp.
+ * Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
index ead11242c4b9..bf14214fa464 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
@@ -35,8 +35,12 @@
 
 #include "dw-hdmi.h"
 #include "dw-hdmi-audio.h"
+#include "dw-hdmi-cec.h"
+
+#include <media/cec-notifier.h>
 
 #define DDC_SEGMENT_ADDR	0x30
+
 #define HDMI_EDID_LEN		512
 
 enum hdmi_datamap {
@@ -130,6 +134,7 @@ struct dw_hdmi {
 	unsigned int version;
 
 	struct platform_device *audio;
+	struct platform_device *cec;
 	struct device *dev;
 	struct clk *isfr_clk;
 	struct clk *iahb_clk;
@@ -163,6 +168,7 @@ struct dw_hdmi {
 	bool bridge_is_on;		/* indicates the bridge is on */
 	bool rxsense;			/* rxsense state */
 	u8 phy_mask;			/* desired phy int mask settings */
+	u8 mc_clkdis;			/* clock disable register */
 
 	spinlock_t audio_lock;
 	struct mutex audio_mutex;
@@ -175,6 +181,8 @@ struct dw_hdmi {
 	struct regmap *regm;
 	void (*enable_audio)(struct dw_hdmi *hdmi);
 	void (*disable_audio)(struct dw_hdmi *hdmi);
+
+	struct cec_notifier *cec_notifier;
 };
 
 #define HDMI_IH_PHY_STAT0_RX_SENSE \
@@ -546,8 +554,11 @@ EXPORT_SYMBOL_GPL(dw_hdmi_set_sample_rate);
 
 static void hdmi_enable_audio_clk(struct dw_hdmi *hdmi, bool enable)
 {
-	hdmi_modb(hdmi, enable ? 0 : HDMI_MC_CLKDIS_AUDCLK_DISABLE,
-		  HDMI_MC_CLKDIS_AUDCLK_DISABLE, HDMI_MC_CLKDIS);
+	if (enable)
+		hdmi->mc_clkdis &= ~HDMI_MC_CLKDIS_AUDCLK_DISABLE;
+	else
+		hdmi->mc_clkdis |= HDMI_MC_CLKDIS_AUDCLK_DISABLE;
+	hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS);
 }
 
 static void dw_hdmi_ahb_audio_enable(struct dw_hdmi *hdmi)
@@ -1317,7 +1328,7 @@ static void hdmi_config_AVI(struct dw_hdmi *hdmi, struct drm_display_mode *mode)
 	u8 val;
 
 	/* Initialise info frame from DRM mode */
-	drm_hdmi_avi_infoframe_from_display_mode(&frame, mode);
+	drm_hdmi_avi_infoframe_from_display_mode(&frame, mode, false);
 
 	if (hdmi_bus_fmt_is_yuv444(hdmi->hdmi_data.enc_out_bus_format))
 		frame.colorspace = HDMI_COLORSPACE_YUV444;
@@ -1569,8 +1580,6 @@ static void hdmi_av_composer(struct dw_hdmi *hdmi,
 /* HDMI Initialization Step B.4 */
 static void dw_hdmi_enable_video_path(struct dw_hdmi *hdmi)
 {
-	u8 clkdis;
-
 	/* control period minimum duration */
 	hdmi_writeb(hdmi, 12, HDMI_FC_CTRLDUR);
 	hdmi_writeb(hdmi, 32, HDMI_FC_EXCTRLDUR);
@@ -1582,17 +1591,21 @@ static void dw_hdmi_enable_video_path(struct dw_hdmi *hdmi)
 	hdmi_writeb(hdmi, 0x21, HDMI_FC_CH2PREAM);
 
 	/* Enable pixel clock and tmds data path */
-	clkdis = 0x7F;
-	clkdis &= ~HDMI_MC_CLKDIS_PIXELCLK_DISABLE;
-	hdmi_writeb(hdmi, clkdis, HDMI_MC_CLKDIS);
+	hdmi->mc_clkdis |= HDMI_MC_CLKDIS_HDCPCLK_DISABLE |
+			   HDMI_MC_CLKDIS_CSCCLK_DISABLE |
+			   HDMI_MC_CLKDIS_AUDCLK_DISABLE |
+			   HDMI_MC_CLKDIS_PREPCLK_DISABLE |
+			   HDMI_MC_CLKDIS_TMDSCLK_DISABLE;
+	hdmi->mc_clkdis &= ~HDMI_MC_CLKDIS_PIXELCLK_DISABLE;
+	hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS);
 
-	clkdis &= ~HDMI_MC_CLKDIS_TMDSCLK_DISABLE;
-	hdmi_writeb(hdmi, clkdis, HDMI_MC_CLKDIS);
+	hdmi->mc_clkdis &= ~HDMI_MC_CLKDIS_TMDSCLK_DISABLE;
+	hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS);
 
 	/* Enable csc path */
 	if (is_color_space_conversion(hdmi)) {
-		clkdis &= ~HDMI_MC_CLKDIS_CSCCLK_DISABLE;
-		hdmi_writeb(hdmi, clkdis, HDMI_MC_CLKDIS);
+		hdmi->mc_clkdis &= ~HDMI_MC_CLKDIS_CSCCLK_DISABLE;
+		hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS);
 	}
 
 	/* Enable color space conversion if needed */
@@ -1783,7 +1796,6 @@ static void initialize_hdmi_ih_mutes(struct dw_hdmi *hdmi)
 	hdmi_writeb(hdmi, 0xff, HDMI_AUD_HBR_MASK);
 	hdmi_writeb(hdmi, 0xff, HDMI_GP_MASK);
 	hdmi_writeb(hdmi, 0xff, HDMI_A_APIINTMSK);
-	hdmi_writeb(hdmi, 0xff, HDMI_CEC_MASK);
 	hdmi_writeb(hdmi, 0xff, HDMI_I2CM_INT);
 	hdmi_writeb(hdmi, 0xff, HDMI_I2CM_CTLINT);
 
@@ -1896,6 +1908,7 @@ static int dw_hdmi_connector_get_modes(struct drm_connector *connector)
 		hdmi->sink_is_hdmi = drm_detect_hdmi_monitor(edid);
 		hdmi->sink_has_audio = drm_detect_monitor_audio(edid);
 		drm_mode_connector_update_edid_property(connector, edid);
+		cec_notifier_set_phys_addr_from_edid(hdmi->cec_notifier, edid);
 		ret = drm_add_edid_modes(connector, edid);
 		/* Store the ELD */
 		drm_edid_to_eld(connector, edid);
@@ -1920,7 +1933,6 @@ static void dw_hdmi_connector_force(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs dw_hdmi_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = dw_hdmi_connector_detect,
 	.destroy = drm_connector_cleanup,
@@ -2119,11 +2131,16 @@ static irqreturn_t dw_hdmi_irq(int irq, void *dev_id)
 	 * ask the source to re-read the EDID.
 	 */
 	if (intr_stat &
-	    (HDMI_IH_PHY_STAT0_RX_SENSE | HDMI_IH_PHY_STAT0_HPD))
+	    (HDMI_IH_PHY_STAT0_RX_SENSE | HDMI_IH_PHY_STAT0_HPD)) {
 		__dw_hdmi_setup_rx_sense(hdmi,
 					 phy_stat & HDMI_PHY_HPD,
 					 phy_stat & HDMI_PHY_RX_SENSE);
 
+		if ((phy_stat & (HDMI_PHY_RX_SENSE | HDMI_PHY_HPD)) == 0)
+			cec_notifier_set_phys_addr(hdmi->cec_notifier,
+						   CEC_PHYS_ADDR_INVALID);
+	}
+
 	if (intr_stat & HDMI_IH_PHY_STAT0_HPD) {
 		dev_dbg(hdmi->dev, "EVENT=%s\n",
 			phy_int_pol & HDMI_PHY_HPD ? "plugin" : "plugout");
@@ -2170,6 +2187,7 @@ static const struct dw_hdmi_phy_data dw_hdmi_phys[] = {
 		.name = "DWC HDMI 2.0 TX PHY",
 		.gen = 2,
 		.has_svsret = true,
+		.configure = hdmi_phy_configure_dwc_hdmi_3d_tx,
 	}, {
 		.type = DW_HDMI_PHY_VENDOR_PHY,
 		.name = "Vendor PHY",
@@ -2219,6 +2237,29 @@ static int dw_hdmi_detect_phy(struct dw_hdmi *hdmi)
 	return -ENODEV;
 }
 
+static void dw_hdmi_cec_enable(struct dw_hdmi *hdmi)
+{
+	mutex_lock(&hdmi->mutex);
+	hdmi->mc_clkdis &= ~HDMI_MC_CLKDIS_CECCLK_DISABLE;
+	hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS);
+	mutex_unlock(&hdmi->mutex);
+}
+
+static void dw_hdmi_cec_disable(struct dw_hdmi *hdmi)
+{
+	mutex_lock(&hdmi->mutex);
+	hdmi->mc_clkdis |= HDMI_MC_CLKDIS_CECCLK_DISABLE;
+	hdmi_writeb(hdmi, hdmi->mc_clkdis, HDMI_MC_CLKDIS);
+	mutex_unlock(&hdmi->mutex);
+}
+
+static const struct dw_hdmi_cec_ops dw_hdmi_cec_ops = {
+	.write = hdmi_writeb,
+	.read = hdmi_readb,
+	.enable = dw_hdmi_cec_enable,
+	.disable = dw_hdmi_cec_disable,
+};
+
 static const struct regmap_config hdmi_regmap_8bit_config = {
 	.reg_bits	= 32,
 	.val_bits	= 8,
@@ -2241,6 +2282,7 @@ __dw_hdmi_probe(struct platform_device *pdev,
 	struct device_node *np = dev->of_node;
 	struct platform_device_info pdevinfo;
 	struct device_node *ddc_node;
+	struct dw_hdmi_cec_data cec;
 	struct dw_hdmi *hdmi;
 	struct resource *iores = NULL;
 	int irq;
@@ -2261,6 +2303,7 @@ __dw_hdmi_probe(struct platform_device *pdev,
 	hdmi->disabled = true;
 	hdmi->rxsense = true;
 	hdmi->phy_mask = (u8)~(HDMI_PHY_HPD | HDMI_PHY_RX_SENSE);
+	hdmi->mc_clkdis = 0x7f;
 
 	mutex_init(&hdmi->mutex);
 	mutex_init(&hdmi->audio_mutex);
@@ -2376,6 +2419,12 @@ __dw_hdmi_probe(struct platform_device *pdev,
 	if (ret)
 		goto err_iahb;
 
+	hdmi->cec_notifier = cec_notifier_get(dev);
+	if (!hdmi->cec_notifier) {
+		ret = -ENOMEM;
+		goto err_iahb;
+	}
+
 	/*
 	 * To prevent overflows in HDMI_IH_FC_STAT2, set the clk regenerator
 	 * N and cts values before enabling phy
@@ -2438,6 +2487,19 @@ __dw_hdmi_probe(struct platform_device *pdev,
 		hdmi->audio = platform_device_register_full(&pdevinfo);
 	}
 
+	if (config0 & HDMI_CONFIG0_CEC) {
+		cec.hdmi = hdmi;
+		cec.ops = &dw_hdmi_cec_ops;
+		cec.irq = irq;
+
+		pdevinfo.name = "dw-hdmi-cec";
+		pdevinfo.data = &cec;
+		pdevinfo.size_data = sizeof(cec);
+		pdevinfo.dma_mask = 0;
+
+		hdmi->cec = platform_device_register_full(&pdevinfo);
+	}
+
 	/* Reset HDMI DDC I2C master controller and mute I2CM interrupts */
 	if (hdmi->i2c)
 		dw_hdmi_i2c_init(hdmi);
@@ -2452,6 +2514,9 @@ err_iahb:
 		hdmi->ddc = NULL;
 	}
 
+	if (hdmi->cec_notifier)
+		cec_notifier_put(hdmi->cec_notifier);
+
 	clk_disable_unprepare(hdmi->iahb_clk);
 err_isfr:
 	clk_disable_unprepare(hdmi->isfr_clk);
@@ -2465,10 +2530,15 @@ static void __dw_hdmi_remove(struct dw_hdmi *hdmi)
 {
 	if (hdmi->audio && !IS_ERR(hdmi->audio))
 		platform_device_unregister(hdmi->audio);
+	if (!IS_ERR(hdmi->cec))
+		platform_device_unregister(hdmi->cec);
 
 	/* Disable all interrupts */
 	hdmi_writeb(hdmi, ~0, HDMI_IH_MUTE_PHY_STAT0);
 
+	if (hdmi->cec_notifier)
+		cec_notifier_put(hdmi->cec_notifier);
+
 	clk_disable_unprepare(hdmi->iahb_clk);
 	clk_disable_unprepare(hdmi->isfr_clk);
 
@@ -2485,17 +2555,12 @@ int dw_hdmi_probe(struct platform_device *pdev,
 		  const struct dw_hdmi_plat_data *plat_data)
 {
 	struct dw_hdmi *hdmi;
-	int ret;
 
 	hdmi = __dw_hdmi_probe(pdev, plat_data);
 	if (IS_ERR(hdmi))
 		return PTR_ERR(hdmi);
 
-	ret = drm_bridge_add(&hdmi->bridge);
-	if (ret < 0) {
-		__dw_hdmi_remove(hdmi);
-		return ret;
-	}
+	drm_bridge_add(&hdmi->bridge);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.h b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.h
index c59f87e1483e..9d90eb9c46e5 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.h
+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.h
@@ -478,51 +478,6 @@
 #define HDMI_A_PRESETUP                         0x501A
 #define HDMI_A_SRM_BASE                         0x5020
 
-/* CEC Engine Registers */
-#define HDMI_CEC_CTRL                           0x7D00
-#define HDMI_CEC_STAT                           0x7D01
-#define HDMI_CEC_MASK                           0x7D02
-#define HDMI_CEC_POLARITY                       0x7D03
-#define HDMI_CEC_INT                            0x7D04
-#define HDMI_CEC_ADDR_L                         0x7D05
-#define HDMI_CEC_ADDR_H                         0x7D06
-#define HDMI_CEC_TX_CNT                         0x7D07
-#define HDMI_CEC_RX_CNT                         0x7D08
-#define HDMI_CEC_TX_DATA0                       0x7D10
-#define HDMI_CEC_TX_DATA1                       0x7D11
-#define HDMI_CEC_TX_DATA2                       0x7D12
-#define HDMI_CEC_TX_DATA3                       0x7D13
-#define HDMI_CEC_TX_DATA4                       0x7D14
-#define HDMI_CEC_TX_DATA5                       0x7D15
-#define HDMI_CEC_TX_DATA6                       0x7D16
-#define HDMI_CEC_TX_DATA7                       0x7D17
-#define HDMI_CEC_TX_DATA8                       0x7D18
-#define HDMI_CEC_TX_DATA9                       0x7D19
-#define HDMI_CEC_TX_DATA10                      0x7D1a
-#define HDMI_CEC_TX_DATA11                      0x7D1b
-#define HDMI_CEC_TX_DATA12                      0x7D1c
-#define HDMI_CEC_TX_DATA13                      0x7D1d
-#define HDMI_CEC_TX_DATA14                      0x7D1e
-#define HDMI_CEC_TX_DATA15                      0x7D1f
-#define HDMI_CEC_RX_DATA0                       0x7D20
-#define HDMI_CEC_RX_DATA1                       0x7D21
-#define HDMI_CEC_RX_DATA2                       0x7D22
-#define HDMI_CEC_RX_DATA3                       0x7D23
-#define HDMI_CEC_RX_DATA4                       0x7D24
-#define HDMI_CEC_RX_DATA5                       0x7D25
-#define HDMI_CEC_RX_DATA6                       0x7D26
-#define HDMI_CEC_RX_DATA7                       0x7D27
-#define HDMI_CEC_RX_DATA8                       0x7D28
-#define HDMI_CEC_RX_DATA9                       0x7D29
-#define HDMI_CEC_RX_DATA10                      0x7D2a
-#define HDMI_CEC_RX_DATA11                      0x7D2b
-#define HDMI_CEC_RX_DATA12                      0x7D2c
-#define HDMI_CEC_RX_DATA13                      0x7D2d
-#define HDMI_CEC_RX_DATA14                      0x7D2e
-#define HDMI_CEC_RX_DATA15                      0x7D2f
-#define HDMI_CEC_LOCK                           0x7D30
-#define HDMI_CEC_WKUPCTRL                       0x7D31
-
 /* I2C Master Registers (E-DDC) */
 #define HDMI_I2CM_SLAVE                         0x7E00
 #define HDMI_I2CM_ADDRESS                       0x7E01
@@ -555,6 +510,7 @@ enum {
 
 /* CONFIG0_ID field values */
 	HDMI_CONFIG0_I2S = 0x10,
+	HDMI_CONFIG0_CEC = 0x02,
 
 /* CONFIG1_ID field values */
 	HDMI_CONFIG1_AHB = 0x01,
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c b/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c
new file mode 100644
index 000000000000..63c7a01b7053
--- /dev/null
+++ b/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c
@@ -0,0 +1,981 @@
+/*
+ * Copyright (c) 2016, Fuzhou Rockchip Electronics Co., Ltd
+ * Copyright (C) STMicroelectronics SA 2017
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * Modified by Philippe Cornu <philippe.cornu@st.com>
+ * This generic Synopsys DesignWare MIPI DSI host driver is based on the
+ * Rockchip version from rockchip/dw-mipi-dsi.c with phy & bridge APIs.
+ */
+
+#include <linux/clk.h>
+#include <linux/component.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/reset.h>
+#include <drm/drmP.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_bridge.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_of.h>
+#include <drm/bridge/dw_mipi_dsi.h>
+#include <video/mipi_display.h>
+
+#define DSI_VERSION			0x00
+#define DSI_PWR_UP			0x04
+#define RESET				0
+#define POWERUP				BIT(0)
+
+#define DSI_CLKMGR_CFG			0x08
+#define TO_CLK_DIVIDSION(div)		(((div) & 0xff) << 8)
+#define TX_ESC_CLK_DIVIDSION(div)	(((div) & 0xff) << 0)
+
+#define DSI_DPI_VCID			0x0c
+#define DPI_VID(vid)			(((vid) & 0x3) << 0)
+
+#define DSI_DPI_COLOR_CODING		0x10
+#define EN18_LOOSELY			BIT(8)
+#define DPI_COLOR_CODING_16BIT_1	0x0
+#define DPI_COLOR_CODING_16BIT_2	0x1
+#define DPI_COLOR_CODING_16BIT_3	0x2
+#define DPI_COLOR_CODING_18BIT_1	0x3
+#define DPI_COLOR_CODING_18BIT_2	0x4
+#define DPI_COLOR_CODING_24BIT		0x5
+
+#define DSI_DPI_CFG_POL			0x14
+#define COLORM_ACTIVE_LOW		BIT(4)
+#define SHUTD_ACTIVE_LOW		BIT(3)
+#define HSYNC_ACTIVE_LOW		BIT(2)
+#define VSYNC_ACTIVE_LOW		BIT(1)
+#define DATAEN_ACTIVE_LOW		BIT(0)
+
+#define DSI_DPI_LP_CMD_TIM		0x18
+#define OUTVACT_LPCMD_TIME(p)		(((p) & 0xff) << 16)
+#define INVACT_LPCMD_TIME(p)		((p) & 0xff)
+
+#define DSI_DBI_CFG			0x20
+#define DSI_DBI_CMDSIZE			0x28
+
+#define DSI_PCKHDL_CFG			0x2c
+#define EN_CRC_RX			BIT(4)
+#define EN_ECC_RX			BIT(3)
+#define EN_BTA				BIT(2)
+#define EN_EOTP_RX			BIT(1)
+#define EN_EOTP_TX			BIT(0)
+
+#define DSI_MODE_CFG			0x34
+#define ENABLE_VIDEO_MODE		0
+#define ENABLE_CMD_MODE			BIT(0)
+
+#define DSI_VID_MODE_CFG		0x38
+#define FRAME_BTA_ACK			BIT(14)
+#define ENABLE_LOW_POWER		(0x3f << 8)
+#define ENABLE_LOW_POWER_MASK		(0x3f << 8)
+#define VID_MODE_TYPE_NON_BURST_SYNC_PULSES	0x0
+#define VID_MODE_TYPE_NON_BURST_SYNC_EVENTS	0x1
+#define VID_MODE_TYPE_BURST			0x2
+#define VID_MODE_TYPE_MASK			0x3
+
+#define DSI_VID_PKT_SIZE		0x3c
+#define VID_PKT_SIZE(p)			(((p) & 0x3fff) << 0)
+#define VID_PKT_MAX_SIZE		0x3fff
+
+#define DSI_VID_HSA_TIME		0x48
+#define DSI_VID_HBP_TIME		0x4c
+#define DSI_VID_HLINE_TIME		0x50
+#define DSI_VID_VSA_LINES		0x54
+#define DSI_VID_VBP_LINES		0x58
+#define DSI_VID_VFP_LINES		0x5c
+#define DSI_VID_VACTIVE_LINES		0x60
+#define DSI_CMD_MODE_CFG		0x68
+#define MAX_RD_PKT_SIZE_LP		BIT(24)
+#define DCS_LW_TX_LP			BIT(19)
+#define DCS_SR_0P_TX_LP			BIT(18)
+#define DCS_SW_1P_TX_LP			BIT(17)
+#define DCS_SW_0P_TX_LP			BIT(16)
+#define GEN_LW_TX_LP			BIT(14)
+#define GEN_SR_2P_TX_LP			BIT(13)
+#define GEN_SR_1P_TX_LP			BIT(12)
+#define GEN_SR_0P_TX_LP			BIT(11)
+#define GEN_SW_2P_TX_LP			BIT(10)
+#define GEN_SW_1P_TX_LP			BIT(9)
+#define GEN_SW_0P_TX_LP			BIT(8)
+#define EN_ACK_RQST			BIT(1)
+#define EN_TEAR_FX			BIT(0)
+
+#define CMD_MODE_ALL_LP			(MAX_RD_PKT_SIZE_LP | \
+					 DCS_LW_TX_LP | \
+					 DCS_SR_0P_TX_LP | \
+					 DCS_SW_1P_TX_LP | \
+					 DCS_SW_0P_TX_LP | \
+					 GEN_LW_TX_LP | \
+					 GEN_SR_2P_TX_LP | \
+					 GEN_SR_1P_TX_LP | \
+					 GEN_SR_0P_TX_LP | \
+					 GEN_SW_2P_TX_LP | \
+					 GEN_SW_1P_TX_LP | \
+					 GEN_SW_0P_TX_LP)
+
+#define DSI_GEN_HDR			0x6c
+#define GEN_HDATA(data)			(((data) & 0xffff) << 8)
+#define GEN_HDATA_MASK			(0xffff << 8)
+#define GEN_HTYPE(type)			(((type) & 0xff) << 0)
+#define GEN_HTYPE_MASK			0xff
+
+#define DSI_GEN_PLD_DATA		0x70
+
+#define DSI_CMD_PKT_STATUS		0x74
+#define GEN_CMD_EMPTY			BIT(0)
+#define GEN_CMD_FULL			BIT(1)
+#define GEN_PLD_W_EMPTY			BIT(2)
+#define GEN_PLD_W_FULL			BIT(3)
+#define GEN_PLD_R_EMPTY			BIT(4)
+#define GEN_PLD_R_FULL			BIT(5)
+#define GEN_RD_CMD_BUSY			BIT(6)
+
+#define DSI_TO_CNT_CFG			0x78
+#define HSTX_TO_CNT(p)			(((p) & 0xffff) << 16)
+#define LPRX_TO_CNT(p)			((p) & 0xffff)
+
+#define DSI_BTA_TO_CNT			0x8c
+#define DSI_LPCLK_CTRL			0x94
+#define AUTO_CLKLANE_CTRL		BIT(1)
+#define PHY_TXREQUESTCLKHS		BIT(0)
+
+#define DSI_PHY_TMR_LPCLK_CFG		0x98
+#define PHY_CLKHS2LP_TIME(lbcc)		(((lbcc) & 0x3ff) << 16)
+#define PHY_CLKLP2HS_TIME(lbcc)		((lbcc) & 0x3ff)
+
+#define DSI_PHY_TMR_CFG			0x9c
+#define PHY_HS2LP_TIME(lbcc)		(((lbcc) & 0xff) << 24)
+#define PHY_LP2HS_TIME(lbcc)		(((lbcc) & 0xff) << 16)
+#define MAX_RD_TIME(lbcc)		((lbcc) & 0x7fff)
+
+#define DSI_PHY_RSTZ			0xa0
+#define PHY_DISFORCEPLL			0
+#define PHY_ENFORCEPLL			BIT(3)
+#define PHY_DISABLECLK			0
+#define PHY_ENABLECLK			BIT(2)
+#define PHY_RSTZ			0
+#define PHY_UNRSTZ			BIT(1)
+#define PHY_SHUTDOWNZ			0
+#define PHY_UNSHUTDOWNZ			BIT(0)
+
+#define DSI_PHY_IF_CFG			0xa4
+#define N_LANES(n)			((((n) - 1) & 0x3) << 0)
+#define PHY_STOP_WAIT_TIME(cycle)	(((cycle) & 0xff) << 8)
+
+#define DSI_PHY_STATUS			0xb0
+#define LOCK				BIT(0)
+#define STOP_STATE_CLK_LANE		BIT(2)
+
+#define DSI_PHY_TST_CTRL0		0xb4
+#define PHY_TESTCLK			BIT(1)
+#define PHY_UNTESTCLK			0
+#define PHY_TESTCLR			BIT(0)
+#define PHY_UNTESTCLR			0
+
+#define DSI_PHY_TST_CTRL1		0xb8
+#define PHY_TESTEN			BIT(16)
+#define PHY_UNTESTEN			0
+#define PHY_TESTDOUT(n)			(((n) & 0xff) << 8)
+#define PHY_TESTDIN(n)			(((n) & 0xff) << 0)
+
+#define DSI_INT_ST0			0xbc
+#define DSI_INT_ST1			0xc0
+#define DSI_INT_MSK0			0xc4
+#define DSI_INT_MSK1			0xc8
+
+#define PHY_STATUS_TIMEOUT_US		10000
+#define CMD_PKT_STATUS_TIMEOUT_US	20000
+
+struct dw_mipi_dsi {
+	struct drm_bridge bridge;
+	struct mipi_dsi_host dsi_host;
+	struct drm_bridge *panel_bridge;
+	bool is_panel_bridge;
+	struct device *dev;
+	void __iomem *base;
+
+	struct clk *pclk;
+
+	unsigned int lane_mbps; /* per lane */
+	u32 channel;
+	u32 lanes;
+	u32 format;
+	unsigned long mode_flags;
+
+	const struct dw_mipi_dsi_plat_data *plat_data;
+};
+
+/*
+ * The controller should generate 2 frames before
+ * preparing the peripheral.
+ */
+static void dw_mipi_dsi_wait_for_two_frames(struct drm_display_mode *mode)
+{
+	int refresh, two_frames;
+
+	refresh = drm_mode_vrefresh(mode);
+	two_frames = DIV_ROUND_UP(MSEC_PER_SEC, refresh) * 2;
+	msleep(two_frames);
+}
+
+static inline struct dw_mipi_dsi *host_to_dsi(struct mipi_dsi_host *host)
+{
+	return container_of(host, struct dw_mipi_dsi, dsi_host);
+}
+
+static inline struct dw_mipi_dsi *bridge_to_dsi(struct drm_bridge *bridge)
+{
+	return container_of(bridge, struct dw_mipi_dsi, bridge);
+}
+
+static inline void dsi_write(struct dw_mipi_dsi *dsi, u32 reg, u32 val)
+{
+	writel(val, dsi->base + reg);
+}
+
+static inline u32 dsi_read(struct dw_mipi_dsi *dsi, u32 reg)
+{
+	return readl(dsi->base + reg);
+}
+
+static int dw_mipi_dsi_host_attach(struct mipi_dsi_host *host,
+				   struct mipi_dsi_device *device)
+{
+	struct dw_mipi_dsi *dsi = host_to_dsi(host);
+	struct drm_bridge *bridge;
+	struct drm_panel *panel;
+	int ret;
+
+	if (device->lanes > dsi->plat_data->max_data_lanes) {
+		dev_err(dsi->dev, "the number of data lanes(%u) is too many\n",
+			device->lanes);
+		return -EINVAL;
+	}
+
+	dsi->lanes = device->lanes;
+	dsi->channel = device->channel;
+	dsi->format = device->format;
+	dsi->mode_flags = device->mode_flags;
+
+	ret = drm_of_find_panel_or_bridge(host->dev->of_node, 1, 0,
+					  &panel, &bridge);
+	if (ret)
+		return ret;
+
+	if (panel) {
+		bridge = drm_panel_bridge_add(panel, DRM_MODE_CONNECTOR_DSI);
+		if (IS_ERR(bridge))
+			return PTR_ERR(bridge);
+		dsi->is_panel_bridge = true;
+	}
+
+	dsi->panel_bridge = bridge;
+
+	drm_bridge_add(&dsi->bridge);
+
+	return 0;
+}
+
+static int dw_mipi_dsi_host_detach(struct mipi_dsi_host *host,
+				   struct mipi_dsi_device *device)
+{
+	struct dw_mipi_dsi *dsi = host_to_dsi(host);
+
+	if (dsi->is_panel_bridge)
+		drm_panel_bridge_remove(dsi->panel_bridge);
+
+	drm_bridge_remove(&dsi->bridge);
+
+	return 0;
+}
+
+static void dw_mipi_message_config(struct dw_mipi_dsi *dsi,
+				   const struct mipi_dsi_msg *msg)
+{
+	bool lpm = msg->flags & MIPI_DSI_MSG_USE_LPM;
+	u32 val = 0;
+
+	if (msg->flags & MIPI_DSI_MSG_REQ_ACK)
+		val |= EN_ACK_RQST;
+	if (lpm)
+		val |= CMD_MODE_ALL_LP;
+
+	dsi_write(dsi, DSI_LPCLK_CTRL, lpm ? 0 : PHY_TXREQUESTCLKHS);
+	dsi_write(dsi, DSI_CMD_MODE_CFG, val);
+}
+
+static int dw_mipi_dsi_gen_pkt_hdr_write(struct dw_mipi_dsi *dsi, u32 hdr_val)
+{
+	int ret;
+	u32 val, mask;
+
+	ret = readl_poll_timeout(dsi->base + DSI_CMD_PKT_STATUS,
+				 val, !(val & GEN_CMD_FULL), 1000,
+				 CMD_PKT_STATUS_TIMEOUT_US);
+	if (ret < 0) {
+		dev_err(dsi->dev, "failed to get available command FIFO\n");
+		return ret;
+	}
+
+	dsi_write(dsi, DSI_GEN_HDR, hdr_val);
+
+	mask = GEN_CMD_EMPTY | GEN_PLD_W_EMPTY;
+	ret = readl_poll_timeout(dsi->base + DSI_CMD_PKT_STATUS,
+				 val, (val & mask) == mask,
+				 1000, CMD_PKT_STATUS_TIMEOUT_US);
+	if (ret < 0) {
+		dev_err(dsi->dev, "failed to write command FIFO\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int dw_mipi_dsi_dcs_short_write(struct dw_mipi_dsi *dsi,
+				       const struct mipi_dsi_msg *msg)
+{
+	const u8 *tx_buf = msg->tx_buf;
+	u16 data = 0;
+	u32 val;
+
+	if (msg->tx_len > 0)
+		data |= tx_buf[0];
+	if (msg->tx_len > 1)
+		data |= tx_buf[1] << 8;
+
+	if (msg->tx_len > 2) {
+		dev_err(dsi->dev, "too long tx buf length %zu for short write\n",
+			msg->tx_len);
+		return -EINVAL;
+	}
+
+	val = GEN_HDATA(data) | GEN_HTYPE(msg->type);
+	return dw_mipi_dsi_gen_pkt_hdr_write(dsi, val);
+}
+
+static int dw_mipi_dsi_dcs_long_write(struct dw_mipi_dsi *dsi,
+				      const struct mipi_dsi_msg *msg)
+{
+	const u8 *tx_buf = msg->tx_buf;
+	int len = msg->tx_len, pld_data_bytes = sizeof(u32), ret;
+	u32 hdr_val = GEN_HDATA(msg->tx_len) | GEN_HTYPE(msg->type);
+	u32 remainder;
+	u32 val;
+
+	if (msg->tx_len < 3) {
+		dev_err(dsi->dev, "wrong tx buf length %zu for long write\n",
+			msg->tx_len);
+		return -EINVAL;
+	}
+
+	while (DIV_ROUND_UP(len, pld_data_bytes)) {
+		if (len < pld_data_bytes) {
+			remainder = 0;
+			memcpy(&remainder, tx_buf, len);
+			dsi_write(dsi, DSI_GEN_PLD_DATA, remainder);
+			len = 0;
+		} else {
+			memcpy(&remainder, tx_buf, pld_data_bytes);
+			dsi_write(dsi, DSI_GEN_PLD_DATA, remainder);
+			tx_buf += pld_data_bytes;
+			len -= pld_data_bytes;
+		}
+
+		ret = readl_poll_timeout(dsi->base + DSI_CMD_PKT_STATUS,
+					 val, !(val & GEN_PLD_W_FULL), 1000,
+					 CMD_PKT_STATUS_TIMEOUT_US);
+		if (ret < 0) {
+			dev_err(dsi->dev,
+				"failed to get available write payload FIFO\n");
+			return ret;
+		}
+	}
+
+	return dw_mipi_dsi_gen_pkt_hdr_write(dsi, hdr_val);
+}
+
+static ssize_t dw_mipi_dsi_host_transfer(struct mipi_dsi_host *host,
+					 const struct mipi_dsi_msg *msg)
+{
+	struct dw_mipi_dsi *dsi = host_to_dsi(host);
+	int ret;
+
+	/*
+	 * TODO dw drv improvements
+	 * use mipi_dsi_create_packet() instead of all following
+	 * functions and code (no switch cases, no
+	 * dw_mipi_dsi_dcs_short_write(), only the loop in long_write...)
+	 * and use packet.header...
+	 */
+	dw_mipi_message_config(dsi, msg);
+
+	switch (msg->type) {
+	case MIPI_DSI_DCS_SHORT_WRITE:
+	case MIPI_DSI_DCS_SHORT_WRITE_PARAM:
+	case MIPI_DSI_SET_MAXIMUM_RETURN_PACKET_SIZE:
+		ret = dw_mipi_dsi_dcs_short_write(dsi, msg);
+		break;
+	case MIPI_DSI_DCS_LONG_WRITE:
+		ret = dw_mipi_dsi_dcs_long_write(dsi, msg);
+		break;
+	default:
+		dev_err(dsi->dev, "unsupported message type 0x%02x\n",
+			msg->type);
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+static const struct mipi_dsi_host_ops dw_mipi_dsi_host_ops = {
+	.attach = dw_mipi_dsi_host_attach,
+	.detach = dw_mipi_dsi_host_detach,
+	.transfer = dw_mipi_dsi_host_transfer,
+};
+
+static void dw_mipi_dsi_video_mode_config(struct dw_mipi_dsi *dsi)
+{
+	u32 val;
+
+	/*
+	 * TODO dw drv improvements
+	 * enabling low power is panel-dependent, we should use the
+	 * panel configuration here...
+	 */
+	val = ENABLE_LOW_POWER;
+
+	if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO_BURST)
+		val |= VID_MODE_TYPE_BURST;
+	else if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO_SYNC_PULSE)
+		val |= VID_MODE_TYPE_NON_BURST_SYNC_PULSES;
+	else
+		val |= VID_MODE_TYPE_NON_BURST_SYNC_EVENTS;
+
+	dsi_write(dsi, DSI_VID_MODE_CFG, val);
+}
+
+static void dw_mipi_dsi_set_mode(struct dw_mipi_dsi *dsi,
+				 unsigned long mode_flags)
+{
+	dsi_write(dsi, DSI_PWR_UP, RESET);
+
+	if (mode_flags & MIPI_DSI_MODE_VIDEO) {
+		dsi_write(dsi, DSI_MODE_CFG, ENABLE_VIDEO_MODE);
+		dw_mipi_dsi_video_mode_config(dsi);
+		dsi_write(dsi, DSI_LPCLK_CTRL, PHY_TXREQUESTCLKHS);
+	} else {
+		dsi_write(dsi, DSI_MODE_CFG, ENABLE_CMD_MODE);
+	}
+
+	dsi_write(dsi, DSI_PWR_UP, POWERUP);
+}
+
+static void dw_mipi_dsi_disable(struct dw_mipi_dsi *dsi)
+{
+	dsi_write(dsi, DSI_PWR_UP, RESET);
+	dsi_write(dsi, DSI_PHY_RSTZ, PHY_RSTZ);
+}
+
+static void dw_mipi_dsi_init(struct dw_mipi_dsi *dsi)
+{
+	/*
+	 * The maximum permitted escape clock is 20MHz and it is derived from
+	 * lanebyteclk, which is running at "lane_mbps / 8".  Thus we want:
+	 *
+	 *     (lane_mbps >> 3) / esc_clk_division < 20
+	 * which is:
+	 *     (lane_mbps >> 3) / 20 > esc_clk_division
+	 */
+	u32 esc_clk_division = (dsi->lane_mbps >> 3) / 20 + 1;
+
+	dsi_write(dsi, DSI_PWR_UP, RESET);
+
+	/*
+	 * TODO dw drv improvements
+	 * timeout clock division should be computed with the
+	 * high speed transmission counter timeout and byte lane...
+	 */
+	dsi_write(dsi, DSI_CLKMGR_CFG, TO_CLK_DIVIDSION(10) |
+		  TX_ESC_CLK_DIVIDSION(esc_clk_division));
+}
+
+static void dw_mipi_dsi_dpi_config(struct dw_mipi_dsi *dsi,
+				   struct drm_display_mode *mode)
+{
+	u32 val = 0, color = 0;
+
+	switch (dsi->format) {
+	case MIPI_DSI_FMT_RGB888:
+		color = DPI_COLOR_CODING_24BIT;
+		break;
+	case MIPI_DSI_FMT_RGB666:
+		color = DPI_COLOR_CODING_18BIT_2 | EN18_LOOSELY;
+		break;
+	case MIPI_DSI_FMT_RGB666_PACKED:
+		color = DPI_COLOR_CODING_18BIT_1;
+		break;
+	case MIPI_DSI_FMT_RGB565:
+		color = DPI_COLOR_CODING_16BIT_1;
+		break;
+	}
+
+	if (mode->flags & DRM_MODE_FLAG_NVSYNC)
+		val |= VSYNC_ACTIVE_LOW;
+	if (mode->flags & DRM_MODE_FLAG_NHSYNC)
+		val |= HSYNC_ACTIVE_LOW;
+
+	dsi_write(dsi, DSI_DPI_VCID, DPI_VID(dsi->channel));
+	dsi_write(dsi, DSI_DPI_COLOR_CODING, color);
+	dsi_write(dsi, DSI_DPI_CFG_POL, val);
+	/*
+	 * TODO dw drv improvements
+	 * largest packet sizes during hfp or during vsa/vpb/vfp
+	 * should be computed according to byte lane, lane number and only
+	 * if sending lp cmds in high speed is enable (PHY_TXREQUESTCLKHS)
+	 */
+	dsi_write(dsi, DSI_DPI_LP_CMD_TIM, OUTVACT_LPCMD_TIME(4)
+		  | INVACT_LPCMD_TIME(4));
+}
+
+static void dw_mipi_dsi_packet_handler_config(struct dw_mipi_dsi *dsi)
+{
+	dsi_write(dsi, DSI_PCKHDL_CFG, EN_CRC_RX | EN_ECC_RX | EN_BTA);
+}
+
+static void dw_mipi_dsi_video_packet_config(struct dw_mipi_dsi *dsi,
+					    struct drm_display_mode *mode)
+{
+	/*
+	 * TODO dw drv improvements
+	 * only burst mode is supported here. For non-burst video modes,
+	 * we should compute DSI_VID_PKT_SIZE, DSI_VCCR.NUMC &
+	 * DSI_VNPCR.NPSIZE... especially because this driver supports
+	 * non-burst video modes, see dw_mipi_dsi_video_mode_config()...
+	 */
+	dsi_write(dsi, DSI_VID_PKT_SIZE, VID_PKT_SIZE(mode->hdisplay));
+}
+
+static void dw_mipi_dsi_command_mode_config(struct dw_mipi_dsi *dsi)
+{
+	/*
+	 * TODO dw drv improvements
+	 * compute high speed transmission counter timeout according
+	 * to the timeout clock division (TO_CLK_DIVIDSION) and byte lane...
+	 */
+	dsi_write(dsi, DSI_TO_CNT_CFG, HSTX_TO_CNT(1000) | LPRX_TO_CNT(1000));
+	/*
+	 * TODO dw drv improvements
+	 * the Bus-Turn-Around Timeout Counter should be computed
+	 * according to byte lane...
+	 */
+	dsi_write(dsi, DSI_BTA_TO_CNT, 0xd00);
+	dsi_write(dsi, DSI_MODE_CFG, ENABLE_CMD_MODE);
+}
+
+/* Get lane byte clock cycles. */
+static u32 dw_mipi_dsi_get_hcomponent_lbcc(struct dw_mipi_dsi *dsi,
+					   struct drm_display_mode *mode,
+					   u32 hcomponent)
+{
+	u32 frac, lbcc;
+
+	lbcc = hcomponent * dsi->lane_mbps * MSEC_PER_SEC / 8;
+
+	frac = lbcc % mode->clock;
+	lbcc = lbcc / mode->clock;
+	if (frac)
+		lbcc++;
+
+	return lbcc;
+}
+
+static void dw_mipi_dsi_line_timer_config(struct dw_mipi_dsi *dsi,
+					  struct drm_display_mode *mode)
+{
+	u32 htotal, hsa, hbp, lbcc;
+
+	htotal = mode->htotal;
+	hsa = mode->hsync_end - mode->hsync_start;
+	hbp = mode->htotal - mode->hsync_end;
+
+	/*
+	 * TODO dw drv improvements
+	 * computations below may be improved...
+	 */
+	lbcc = dw_mipi_dsi_get_hcomponent_lbcc(dsi, mode, htotal);
+	dsi_write(dsi, DSI_VID_HLINE_TIME, lbcc);
+
+	lbcc = dw_mipi_dsi_get_hcomponent_lbcc(dsi, mode, hsa);
+	dsi_write(dsi, DSI_VID_HSA_TIME, lbcc);
+
+	lbcc = dw_mipi_dsi_get_hcomponent_lbcc(dsi, mode, hbp);
+	dsi_write(dsi, DSI_VID_HBP_TIME, lbcc);
+}
+
+static void dw_mipi_dsi_vertical_timing_config(struct dw_mipi_dsi *dsi,
+					       struct drm_display_mode *mode)
+{
+	u32 vactive, vsa, vfp, vbp;
+
+	vactive = mode->vdisplay;
+	vsa = mode->vsync_end - mode->vsync_start;
+	vfp = mode->vsync_start - mode->vdisplay;
+	vbp = mode->vtotal - mode->vsync_end;
+
+	dsi_write(dsi, DSI_VID_VACTIVE_LINES, vactive);
+	dsi_write(dsi, DSI_VID_VSA_LINES, vsa);
+	dsi_write(dsi, DSI_VID_VFP_LINES, vfp);
+	dsi_write(dsi, DSI_VID_VBP_LINES, vbp);
+}
+
+static void dw_mipi_dsi_dphy_timing_config(struct dw_mipi_dsi *dsi)
+{
+	/*
+	 * TODO dw drv improvements
+	 * data & clock lane timers should be computed according to panel
+	 * blankings and to the automatic clock lane control mode...
+	 * note: DSI_PHY_TMR_CFG.MAX_RD_TIME should be in line with
+	 * DSI_CMD_MODE_CFG.MAX_RD_PKT_SIZE_LP (see CMD_MODE_ALL_LP)
+	 */
+	dsi_write(dsi, DSI_PHY_TMR_CFG, PHY_HS2LP_TIME(0x40)
+		  | PHY_LP2HS_TIME(0x40) | MAX_RD_TIME(10000));
+
+	dsi_write(dsi, DSI_PHY_TMR_LPCLK_CFG, PHY_CLKHS2LP_TIME(0x40)
+		  | PHY_CLKLP2HS_TIME(0x40));
+}
+
+static void dw_mipi_dsi_dphy_interface_config(struct dw_mipi_dsi *dsi)
+{
+	/*
+	 * TODO dw drv improvements
+	 * stop wait time should be the maximum between host dsi
+	 * and panel stop wait times
+	 */
+	dsi_write(dsi, DSI_PHY_IF_CFG, PHY_STOP_WAIT_TIME(0x20) |
+		  N_LANES(dsi->lanes));
+}
+
+static void dw_mipi_dsi_dphy_init(struct dw_mipi_dsi *dsi)
+{
+	/* Clear PHY state */
+	dsi_write(dsi, DSI_PHY_RSTZ, PHY_DISFORCEPLL | PHY_DISABLECLK
+		  | PHY_RSTZ | PHY_SHUTDOWNZ);
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_UNTESTCLR);
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_TESTCLR);
+	dsi_write(dsi, DSI_PHY_TST_CTRL0, PHY_UNTESTCLR);
+}
+
+static void dw_mipi_dsi_dphy_enable(struct dw_mipi_dsi *dsi)
+{
+	u32 val;
+	int ret;
+
+	dsi_write(dsi, DSI_PHY_RSTZ, PHY_ENFORCEPLL | PHY_ENABLECLK |
+		  PHY_UNRSTZ | PHY_UNSHUTDOWNZ);
+
+	ret = readl_poll_timeout(dsi->base + DSI_PHY_STATUS,
+				 val, val & LOCK, 1000, PHY_STATUS_TIMEOUT_US);
+	if (ret < 0)
+		DRM_DEBUG_DRIVER("failed to wait phy lock state\n");
+
+	ret = readl_poll_timeout(dsi->base + DSI_PHY_STATUS,
+				 val, val & STOP_STATE_CLK_LANE, 1000,
+				 PHY_STATUS_TIMEOUT_US);
+	if (ret < 0)
+		DRM_DEBUG_DRIVER("failed to wait phy clk lane stop state\n");
+}
+
+static void dw_mipi_dsi_clear_err(struct dw_mipi_dsi *dsi)
+{
+	dsi_read(dsi, DSI_INT_ST0);
+	dsi_read(dsi, DSI_INT_ST1);
+	dsi_write(dsi, DSI_INT_MSK0, 0);
+	dsi_write(dsi, DSI_INT_MSK1, 0);
+}
+
+static void dw_mipi_dsi_bridge_post_disable(struct drm_bridge *bridge)
+{
+	struct dw_mipi_dsi *dsi = bridge_to_dsi(bridge);
+
+	/*
+	 * Switch to command mode before panel-bridge post_disable &
+	 * panel unprepare.
+	 * Note: panel-bridge disable & panel disable has been called
+	 * before by the drm framework.
+	 */
+	dw_mipi_dsi_set_mode(dsi, 0);
+
+	/*
+	 * TODO Only way found to call panel-bridge post_disable &
+	 * panel unprepare before the dsi "final" disable...
+	 * This needs to be fixed in the drm_bridge framework and the API
+	 * needs to be updated to manage our own call chains...
+	 */
+	dsi->panel_bridge->funcs->post_disable(dsi->panel_bridge);
+
+	dw_mipi_dsi_disable(dsi);
+	clk_disable_unprepare(dsi->pclk);
+	pm_runtime_put(dsi->dev);
+}
+
+void dw_mipi_dsi_bridge_mode_set(struct drm_bridge *bridge,
+				 struct drm_display_mode *mode,
+				 struct drm_display_mode *adjusted_mode)
+{
+	struct dw_mipi_dsi *dsi = bridge_to_dsi(bridge);
+	const struct dw_mipi_dsi_phy_ops *phy_ops = dsi->plat_data->phy_ops;
+	void *priv_data = dsi->plat_data->priv_data;
+	int ret;
+
+	clk_prepare_enable(dsi->pclk);
+
+	ret = phy_ops->get_lane_mbps(priv_data, mode, dsi->mode_flags,
+				     dsi->lanes, dsi->format, &dsi->lane_mbps);
+	if (ret)
+		DRM_DEBUG_DRIVER("Phy get_lane_mbps() failed\n");
+
+	pm_runtime_get_sync(dsi->dev);
+	dw_mipi_dsi_init(dsi);
+	dw_mipi_dsi_dpi_config(dsi, mode);
+	dw_mipi_dsi_packet_handler_config(dsi);
+	dw_mipi_dsi_video_mode_config(dsi);
+	dw_mipi_dsi_video_packet_config(dsi, mode);
+	dw_mipi_dsi_command_mode_config(dsi);
+	dw_mipi_dsi_line_timer_config(dsi, mode);
+	dw_mipi_dsi_vertical_timing_config(dsi, mode);
+
+	dw_mipi_dsi_dphy_init(dsi);
+	dw_mipi_dsi_dphy_timing_config(dsi);
+	dw_mipi_dsi_dphy_interface_config(dsi);
+
+	dw_mipi_dsi_clear_err(dsi);
+
+	ret = phy_ops->init(priv_data);
+	if (ret)
+		DRM_DEBUG_DRIVER("Phy init() failed\n");
+
+	dw_mipi_dsi_dphy_enable(dsi);
+
+	dw_mipi_dsi_wait_for_two_frames(mode);
+
+	/* Switch to cmd mode for panel-bridge pre_enable & panel prepare */
+	dw_mipi_dsi_set_mode(dsi, 0);
+}
+
+static void dw_mipi_dsi_bridge_enable(struct drm_bridge *bridge)
+{
+	struct dw_mipi_dsi *dsi = bridge_to_dsi(bridge);
+
+	/* Switch to video mode for panel-bridge enable & panel enable */
+	dw_mipi_dsi_set_mode(dsi, MIPI_DSI_MODE_VIDEO);
+}
+
+static enum drm_mode_status
+dw_mipi_dsi_bridge_mode_valid(struct drm_bridge *bridge,
+			      const struct drm_display_mode *mode)
+{
+	struct dw_mipi_dsi *dsi = bridge_to_dsi(bridge);
+	const struct dw_mipi_dsi_plat_data *pdata = dsi->plat_data;
+	enum drm_mode_status mode_status = MODE_OK;
+
+	if (pdata->mode_valid)
+		mode_status = pdata->mode_valid(pdata->priv_data, mode);
+
+	return mode_status;
+}
+
+static int dw_mipi_dsi_bridge_attach(struct drm_bridge *bridge)
+{
+	struct dw_mipi_dsi *dsi = bridge_to_dsi(bridge);
+
+	if (!bridge->encoder) {
+		DRM_ERROR("Parent encoder object not found\n");
+		return -ENODEV;
+	}
+
+	/* Set the encoder type as caller does not know it */
+	bridge->encoder->encoder_type = DRM_MODE_ENCODER_DSI;
+
+	/* Attach the panel-bridge to the dsi bridge */
+	return drm_bridge_attach(bridge->encoder, dsi->panel_bridge, bridge);
+}
+
+static const struct drm_bridge_funcs dw_mipi_dsi_bridge_funcs = {
+	.mode_set     = dw_mipi_dsi_bridge_mode_set,
+	.enable	      = dw_mipi_dsi_bridge_enable,
+	.post_disable = dw_mipi_dsi_bridge_post_disable,
+	.mode_valid   = dw_mipi_dsi_bridge_mode_valid,
+	.attach	      = dw_mipi_dsi_bridge_attach,
+};
+
+static struct dw_mipi_dsi *
+__dw_mipi_dsi_probe(struct platform_device *pdev,
+		    const struct dw_mipi_dsi_plat_data *plat_data)
+{
+	struct device *dev = &pdev->dev;
+	struct reset_control *apb_rst;
+	struct dw_mipi_dsi *dsi;
+	struct resource *res;
+	int ret;
+
+	dsi = devm_kzalloc(dev, sizeof(*dsi), GFP_KERNEL);
+	if (!dsi)
+		return ERR_PTR(-ENOMEM);
+
+	dsi->dev = dev;
+	dsi->plat_data = plat_data;
+
+	if (!plat_data->phy_ops->init || !plat_data->phy_ops->get_lane_mbps) {
+		DRM_ERROR("Phy not properly configured\n");
+		return ERR_PTR(-ENODEV);
+	}
+
+	if (!plat_data->base) {
+		res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+		if (!res)
+			return ERR_PTR(-ENODEV);
+
+		dsi->base = devm_ioremap_resource(dev, res);
+		if (IS_ERR(dsi->base))
+			return ERR_PTR(-ENODEV);
+
+	} else {
+		dsi->base = plat_data->base;
+	}
+
+	dsi->pclk = devm_clk_get(dev, "pclk");
+	if (IS_ERR(dsi->pclk)) {
+		ret = PTR_ERR(dsi->pclk);
+		dev_err(dev, "Unable to get pclk: %d\n", ret);
+		return ERR_PTR(ret);
+	}
+
+	/*
+	 * Note that the reset was not defined in the initial device tree, so
+	 * we have to be prepared for it not being found.
+	 */
+	apb_rst = devm_reset_control_get(dev, "apb");
+	if (IS_ERR(apb_rst)) {
+		ret = PTR_ERR(apb_rst);
+		if (ret == -ENOENT) {
+			apb_rst = NULL;
+		} else {
+			dev_err(dev, "Unable to get reset control: %d\n", ret);
+			return ERR_PTR(ret);
+		}
+	}
+
+	if (apb_rst) {
+		ret = clk_prepare_enable(dsi->pclk);
+		if (ret) {
+			dev_err(dev, "%s: Failed to enable pclk\n", __func__);
+			return ERR_PTR(ret);
+		}
+
+		reset_control_assert(apb_rst);
+		usleep_range(10, 20);
+		reset_control_deassert(apb_rst);
+
+		clk_disable_unprepare(dsi->pclk);
+	}
+
+	pm_runtime_enable(dev);
+
+	dsi->dsi_host.ops = &dw_mipi_dsi_host_ops;
+	dsi->dsi_host.dev = dev;
+	ret = mipi_dsi_host_register(&dsi->dsi_host);
+	if (ret) {
+		dev_err(dev, "Failed to register MIPI host: %d\n", ret);
+		return ERR_PTR(ret);
+	}
+
+	dsi->bridge.driver_private = dsi;
+	dsi->bridge.funcs = &dw_mipi_dsi_bridge_funcs;
+#ifdef CONFIG_OF
+	dsi->bridge.of_node = pdev->dev.of_node;
+#endif
+
+	dev_set_drvdata(dev, dsi);
+
+	return dsi;
+}
+
+static void __dw_mipi_dsi_remove(struct dw_mipi_dsi *dsi)
+{
+	pm_runtime_disable(dsi->dev);
+}
+
+/*
+ * Probe/remove API, used from platforms based on the DRM bridge API.
+ */
+int dw_mipi_dsi_probe(struct platform_device *pdev,
+		      const struct dw_mipi_dsi_plat_data *plat_data)
+{
+	struct dw_mipi_dsi *dsi;
+
+	dsi = __dw_mipi_dsi_probe(pdev, plat_data);
+	if (IS_ERR(dsi))
+		return PTR_ERR(dsi);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(dw_mipi_dsi_probe);
+
+void dw_mipi_dsi_remove(struct platform_device *pdev)
+{
+	struct dw_mipi_dsi *dsi = platform_get_drvdata(pdev);
+
+	mipi_dsi_host_unregister(&dsi->dsi_host);
+
+	__dw_mipi_dsi_remove(dsi);
+}
+EXPORT_SYMBOL_GPL(dw_mipi_dsi_remove);
+
+/*
+ * Bind/unbind API, used from platforms based on the component framework.
+ */
+int dw_mipi_dsi_bind(struct platform_device *pdev, struct drm_encoder *encoder,
+		     const struct dw_mipi_dsi_plat_data *plat_data)
+{
+	struct dw_mipi_dsi *dsi;
+	int ret;
+
+	dsi = __dw_mipi_dsi_probe(pdev, plat_data);
+	if (IS_ERR(dsi))
+		return PTR_ERR(dsi);
+
+	ret = drm_bridge_attach(encoder, &dsi->bridge, NULL);
+	if (ret) {
+		dw_mipi_dsi_remove(pdev);
+		DRM_ERROR("Failed to initialize bridge with drm\n");
+		return ret;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(dw_mipi_dsi_bind);
+
+void dw_mipi_dsi_unbind(struct device *dev)
+{
+	struct dw_mipi_dsi *dsi = dev_get_drvdata(dev);
+
+	__dw_mipi_dsi_remove(dsi);
+}
+EXPORT_SYMBOL_GPL(dw_mipi_dsi_unbind);
+
+MODULE_AUTHOR("Chris Zhong <zyw@rock-chips.com>");
+MODULE_AUTHOR("Philippe Cornu <philippe.cornu@st.com>");
+MODULE_DESCRIPTION("DW MIPI DSI host controller driver");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:dw-mipi-dsi");
diff --git a/drivers/gpu/drm/bridge/tc358767.c b/drivers/gpu/drm/bridge/tc358767.c
index 0529e500c534..8571cfd877c5 100644
--- a/drivers/gpu/drm/bridge/tc358767.c
+++ b/drivers/gpu/drm/bridge/tc358767.c
@@ -1160,7 +1160,6 @@ static const struct drm_connector_helper_funcs tc_connector_helper_funcs = {
 };
 
 static const struct drm_connector_funcs tc_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = drm_connector_cleanup,
 	.reset = drm_atomic_helper_connector_reset,
@@ -1325,11 +1324,7 @@ static int tc_probe(struct i2c_client *client, const struct i2c_device_id *id)
 
 	tc->bridge.funcs = &tc_bridge_funcs;
 	tc->bridge.of_node = dev->of_node;
-	ret = drm_bridge_add(&tc->bridge);
-	if (ret) {
-		dev_err(dev, "Failed to add drm_bridge: %d\n", ret);
-		goto err_unregister_aux;
-	}
+	drm_bridge_add(&tc->bridge);
 
 	i2c_set_clientdata(client, tc);
 
diff --git a/drivers/gpu/drm/bridge/ti-tfp410.c b/drivers/gpu/drm/bridge/ti-tfp410.c
index eee4efda829e..acb857030951 100644
--- a/drivers/gpu/drm/bridge/ti-tfp410.c
+++ b/drivers/gpu/drm/bridge/ti-tfp410.c
@@ -102,7 +102,6 @@ tfp410_connector_detect(struct drm_connector *connector, bool force)
 }
 
 static const struct drm_connector_funcs tfp410_con_funcs = {
-	.dpms			= drm_atomic_helper_connector_dpms,
 	.detect			= tfp410_connector_detect,
 	.fill_modes		= drm_helper_probe_single_connector_modes,
 	.destroy		= drm_connector_cleanup,
@@ -237,11 +236,7 @@ static int tfp410_init(struct device *dev)
 		}
 	}
 
-	ret = drm_bridge_add(&dvi->bridge);
-	if (ret) {
-		dev_err(dev, "drm_bridge_add() failed: %d\n", ret);
-		goto fail;
-	}
+	drm_bridge_add(&dvi->bridge);
 
 	return 0;
 fail:
diff --git a/drivers/gpu/drm/cirrus/cirrus_drv.c b/drivers/gpu/drm/cirrus/cirrus_drv.c
index d893ea21a359..69c4e352dd78 100644
--- a/drivers/gpu/drm/cirrus/cirrus_drv.c
+++ b/drivers/gpu/drm/cirrus/cirrus_drv.c
@@ -132,7 +132,6 @@ static struct drm_driver driver = {
 	.driver_features = DRIVER_MODESET | DRIVER_GEM,
 	.load = cirrus_driver_load,
 	.unload = cirrus_driver_unload,
-	.set_busid = drm_pci_set_busid,
 	.fops = &cirrus_driver_fops,
 	.name = DRIVER_NAME,
 	.desc = DRIVER_DESC,
@@ -143,7 +142,6 @@ static struct drm_driver driver = {
 	.gem_free_object_unlocked = cirrus_gem_free_object,
 	.dumb_create = cirrus_dumb_create,
 	.dumb_map_offset = cirrus_dumb_mmap_offset,
-	.dumb_destroy = drm_gem_dumb_destroy,
 };
 
 static const struct dev_pm_ops cirrus_pm_ops = {
@@ -166,12 +164,12 @@ static int __init cirrus_init(void)
 
 	if (cirrus_modeset == 0)
 		return -EINVAL;
-	return drm_pci_init(&driver, &cirrus_pci_driver);
+	return pci_register_driver(&cirrus_pci_driver);
 }
 
 static void __exit cirrus_exit(void)
 {
-	drm_pci_exit(&driver, &cirrus_pci_driver);
+	pci_unregister_driver(&cirrus_pci_driver);
 }
 
 module_init(cirrus_init);
diff --git a/drivers/gpu/drm/cirrus/cirrus_drv.h b/drivers/gpu/drm/cirrus/cirrus_drv.h
index 8690352d96f7..be2d7e488062 100644
--- a/drivers/gpu/drm/cirrus/cirrus_drv.h
+++ b/drivers/gpu/drm/cirrus/cirrus_drv.h
@@ -96,7 +96,6 @@
 
 struct cirrus_crtc {
 	struct drm_crtc			base;
-	u8				lut_r[256], lut_g[256], lut_b[256];
 	int				last_dpms;
 	bool				enabled;
 };
@@ -180,13 +179,6 @@ cirrus_bo(struct ttm_buffer_object *bo)
 #define to_cirrus_obj(x) container_of(x, struct cirrus_gem_object, base)
 #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
 
-				/* cirrus_mode.c */
-void cirrus_crtc_fb_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
-			     u16 blue, int regno);
-void cirrus_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green,
-			     u16 *blue, int regno);
-
-
 				/* cirrus_main.c */
 int cirrus_device_init(struct cirrus_device *cdev,
 		      struct drm_device *ddev,
diff --git a/drivers/gpu/drm/cirrus/cirrus_fbdev.c b/drivers/gpu/drm/cirrus/cirrus_fbdev.c
index 7fa58eeadc9d..32fbfba2c623 100644
--- a/drivers/gpu/drm/cirrus/cirrus_fbdev.c
+++ b/drivers/gpu/drm/cirrus/cirrus_fbdev.c
@@ -215,7 +215,6 @@ static int cirrusfb_create(struct drm_fb_helper *helper,
 
 	strcpy(info->fix.id, "cirrusdrmfb");
 
-	info->flags = FBINFO_DEFAULT;
 	info->fbops = &cirrusfb_ops;
 
 	drm_fb_helper_fill_fix(info, fb->pitches[0], fb->format->depth);
@@ -252,7 +251,7 @@ static int cirrus_fbdev_destroy(struct drm_device *dev,
 	drm_fb_helper_unregister_fbi(&gfbdev->helper);
 
 	if (gfb->obj) {
-		drm_gem_object_unreference_unlocked(gfb->obj);
+		drm_gem_object_put_unlocked(gfb->obj);
 		gfb->obj = NULL;
 	}
 
@@ -265,8 +264,6 @@ static int cirrus_fbdev_destroy(struct drm_device *dev,
 }
 
 static const struct drm_fb_helper_funcs cirrus_fb_helper_funcs = {
-	.gamma_set = cirrus_crtc_fb_gamma_set,
-	.gamma_get = cirrus_crtc_fb_gamma_get,
 	.fb_probe = cirrusfb_create,
 };
 
diff --git a/drivers/gpu/drm/cirrus/cirrus_main.c b/drivers/gpu/drm/cirrus/cirrus_main.c
index e7fc95f63dca..b5f528543956 100644
--- a/drivers/gpu/drm/cirrus/cirrus_main.c
+++ b/drivers/gpu/drm/cirrus/cirrus_main.c
@@ -18,7 +18,7 @@ static void cirrus_user_framebuffer_destroy(struct drm_framebuffer *fb)
 {
 	struct cirrus_framebuffer *cirrus_fb = to_cirrus_framebuffer(fb);
 
-	drm_gem_object_unreference_unlocked(cirrus_fb->obj);
+	drm_gem_object_put_unlocked(cirrus_fb->obj);
 	drm_framebuffer_cleanup(fb);
 	kfree(fb);
 }
@@ -67,13 +67,13 @@ cirrus_user_framebuffer_create(struct drm_device *dev,
 
 	cirrus_fb = kzalloc(sizeof(*cirrus_fb), GFP_KERNEL);
 	if (!cirrus_fb) {
-		drm_gem_object_unreference_unlocked(obj);
+		drm_gem_object_put_unlocked(obj);
 		return ERR_PTR(-ENOMEM);
 	}
 
 	ret = cirrus_framebuffer_init(dev, cirrus_fb, mode_cmd, obj);
 	if (ret) {
-		drm_gem_object_unreference_unlocked(obj);
+		drm_gem_object_put_unlocked(obj);
 		kfree(cirrus_fb);
 		return ERR_PTR(ret);
 	}
@@ -261,7 +261,7 @@ int cirrus_dumb_create(struct drm_file *file,
 		return ret;
 
 	ret = drm_gem_handle_create(file, gobj, &handle);
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 	if (ret)
 		return ret;
 
@@ -310,7 +310,7 @@ cirrus_dumb_mmap_offset(struct drm_file *file,
 	bo = gem_to_cirrus_bo(obj);
 	*offset = cirrus_bo_mmap_offset(bo);
 
-	drm_gem_object_unreference_unlocked(obj);
+	drm_gem_object_put_unlocked(obj);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/cirrus/cirrus_mode.c b/drivers/gpu/drm/cirrus/cirrus_mode.c
index 53f6f0f84206..a4c4a465b385 100644
--- a/drivers/gpu/drm/cirrus/cirrus_mode.c
+++ b/drivers/gpu/drm/cirrus/cirrus_mode.c
@@ -31,25 +31,6 @@
  * This file contains setup code for the CRTC.
  */
 
-static void cirrus_crtc_load_lut(struct drm_crtc *crtc)
-{
-	struct cirrus_crtc *cirrus_crtc = to_cirrus_crtc(crtc);
-	struct drm_device *dev = crtc->dev;
-	struct cirrus_device *cdev = dev->dev_private;
-	int i;
-
-	if (!crtc->enabled)
-		return;
-
-	for (i = 0; i < CIRRUS_LUT_SIZE; i++) {
-		/* VGA registers */
-		WREG8(PALETTE_INDEX, i);
-		WREG8(PALETTE_DATA, cirrus_crtc->lut_r[i]);
-		WREG8(PALETTE_DATA, cirrus_crtc->lut_g[i]);
-		WREG8(PALETTE_DATA, cirrus_crtc->lut_b[i]);
-	}
-}
-
 /*
  * The DRM core requires DPMS functions, but they make little sense in our
  * case and so are just stubs
@@ -330,15 +311,25 @@ static int cirrus_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
 				 u16 *blue, uint32_t size,
 				 struct drm_modeset_acquire_ctx *ctx)
 {
-	struct cirrus_crtc *cirrus_crtc = to_cirrus_crtc(crtc);
+	struct drm_device *dev = crtc->dev;
+	struct cirrus_device *cdev = dev->dev_private;
+	u16 *r, *g, *b;
 	int i;
 
-	for (i = 0; i < size; i++) {
-		cirrus_crtc->lut_r[i] = red[i];
-		cirrus_crtc->lut_g[i] = green[i];
-		cirrus_crtc->lut_b[i] = blue[i];
+	if (!crtc->enabled)
+		return 0;
+
+	r = crtc->gamma_store;
+	g = r + crtc->gamma_size;
+	b = g + crtc->gamma_size;
+
+	for (i = 0; i < CIRRUS_LUT_SIZE; i++) {
+		/* VGA registers */
+		WREG8(PALETTE_INDEX, i);
+		WREG8(PALETTE_DATA, *r++ >> 8);
+		WREG8(PALETTE_DATA, *g++ >> 8);
+		WREG8(PALETTE_DATA, *b++ >> 8);
 	}
-	cirrus_crtc_load_lut(crtc);
 
 	return 0;
 }
@@ -365,7 +356,6 @@ static const struct drm_crtc_helper_funcs cirrus_helper_funcs = {
 	.mode_set_base = cirrus_crtc_mode_set_base,
 	.prepare = cirrus_crtc_prepare,
 	.commit = cirrus_crtc_commit,
-	.load_lut = cirrus_crtc_load_lut,
 };
 
 /* CRTC setup */
@@ -373,7 +363,6 @@ static void cirrus_crtc_init(struct drm_device *dev)
 {
 	struct cirrus_device *cdev = dev->dev_private;
 	struct cirrus_crtc *cirrus_crtc;
-	int i;
 
 	cirrus_crtc = kzalloc(sizeof(struct cirrus_crtc) +
 			      (CIRRUSFB_CONN_LIMIT * sizeof(struct drm_connector *)),
@@ -387,37 +376,9 @@ static void cirrus_crtc_init(struct drm_device *dev)
 	drm_mode_crtc_set_gamma_size(&cirrus_crtc->base, CIRRUS_LUT_SIZE);
 	cdev->mode_info.crtc = cirrus_crtc;
 
-	for (i = 0; i < CIRRUS_LUT_SIZE; i++) {
-		cirrus_crtc->lut_r[i] = i;
-		cirrus_crtc->lut_g[i] = i;
-		cirrus_crtc->lut_b[i] = i;
-	}
-
 	drm_crtc_helper_add(&cirrus_crtc->base, &cirrus_helper_funcs);
 }
 
-/** Sets the color ramps on behalf of fbcon */
-void cirrus_crtc_fb_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
-			      u16 blue, int regno)
-{
-	struct cirrus_crtc *cirrus_crtc = to_cirrus_crtc(crtc);
-
-	cirrus_crtc->lut_r[regno] = red;
-	cirrus_crtc->lut_g[regno] = green;
-	cirrus_crtc->lut_b[regno] = blue;
-}
-
-/** Gets the color ramps on behalf of fbcon */
-void cirrus_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green,
-			      u16 *blue, int regno)
-{
-	struct cirrus_crtc *cirrus_crtc = to_cirrus_crtc(crtc);
-
-	*red = cirrus_crtc->lut_r[regno];
-	*green = cirrus_crtc->lut_g[regno];
-	*blue = cirrus_crtc->lut_b[regno];
-}
-
 static void cirrus_encoder_mode_set(struct drm_encoder *encoder,
 				struct drm_display_mode *mode,
 				struct drm_display_mode *adjusted_mode)
diff --git a/drivers/gpu/drm/drm_atomic.c b/drivers/gpu/drm/drm_atomic.c
index aed25c4183bb..2fd383d7253a 100644
--- a/drivers/gpu/drm/drm_atomic.c
+++ b/drivers/gpu/drm/drm_atomic.c
@@ -29,7 +29,6 @@
 #include <drm/drmP.h>
 #include <drm/drm_atomic.h>
 #include <drm/drm_mode.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_print.h>
 #include <linux/sync_file.h>
 
@@ -188,12 +187,15 @@ void drm_atomic_state_default_clear(struct drm_atomic_state *state)
 	}
 
 	for (i = 0; i < state->num_private_objs; i++) {
-		void *obj_state = state->private_objs[i].obj_state;
+		struct drm_private_obj *obj = state->private_objs[i].ptr;
 
-		state->private_objs[i].funcs->destroy_state(obj_state);
-		state->private_objs[i].obj = NULL;
-		state->private_objs[i].obj_state = NULL;
-		state->private_objs[i].funcs = NULL;
+		if (!obj)
+			continue;
+
+		obj->funcs->atomic_destroy_state(obj,
+						 state->private_objs[i].state);
+		state->private_objs[i].ptr = NULL;
+		state->private_objs[i].state = NULL;
 	}
 	state->num_private_objs = 0;
 
@@ -409,34 +411,6 @@ int drm_atomic_set_mode_prop_for_crtc(struct drm_crtc_state *state,
 }
 EXPORT_SYMBOL(drm_atomic_set_mode_prop_for_crtc);
 
-/**
- * drm_atomic_replace_property_blob - replace a blob property
- * @blob: a pointer to the member blob to be replaced
- * @new_blob: the new blob to replace with
- * @replaced: whether the blob has been replaced
- *
- * RETURNS:
- * Zero on success, error code on failure
- */
-static void
-drm_atomic_replace_property_blob(struct drm_property_blob **blob,
-				 struct drm_property_blob *new_blob,
-				 bool *replaced)
-{
-	struct drm_property_blob *old_blob = *blob;
-
-	if (old_blob == new_blob)
-		return;
-
-	drm_property_blob_put(old_blob);
-	if (new_blob)
-		drm_property_blob_get(new_blob);
-	*blob = new_blob;
-	*replaced = true;
-
-	return;
-}
-
 static int
 drm_atomic_replace_property_blob_from_id(struct drm_device *dev,
 					 struct drm_property_blob **blob,
@@ -457,7 +431,7 @@ drm_atomic_replace_property_blob_from_id(struct drm_device *dev,
 		}
 	}
 
-	drm_atomic_replace_property_blob(blob, new_blob, replaced);
+	*replaced |= drm_property_replace_blob(blob, new_blob);
 	drm_property_blob_put(new_blob);
 
 	return 0;
@@ -739,7 +713,7 @@ EXPORT_SYMBOL(drm_atomic_get_plane_state);
  * RETURNS:
  * Zero on success, error code on failure
  */
-int drm_atomic_plane_set_property(struct drm_plane *plane,
+static int drm_atomic_plane_set_property(struct drm_plane *plane,
 		struct drm_plane_state *state, struct drm_property *property,
 		uint64_t val)
 {
@@ -796,7 +770,6 @@ int drm_atomic_plane_set_property(struct drm_plane *plane,
 
 	return 0;
 }
-EXPORT_SYMBOL(drm_atomic_plane_set_property);
 
 /**
  * drm_atomic_plane_get_property - get property value from plane state
@@ -991,11 +964,44 @@ static void drm_atomic_plane_print_state(struct drm_printer *p,
 }
 
 /**
+ * drm_atomic_private_obj_init - initialize private object
+ * @obj: private object
+ * @state: initial private object state
+ * @funcs: pointer to the struct of function pointers that identify the object
+ * type
+ *
+ * Initialize the private object, which can be embedded into any
+ * driver private object that needs its own atomic state.
+ */
+void
+drm_atomic_private_obj_init(struct drm_private_obj *obj,
+			    struct drm_private_state *state,
+			    const struct drm_private_state_funcs *funcs)
+{
+	memset(obj, 0, sizeof(*obj));
+
+	obj->state = state;
+	obj->funcs = funcs;
+}
+EXPORT_SYMBOL(drm_atomic_private_obj_init);
+
+/**
+ * drm_atomic_private_obj_fini - finalize private object
+ * @obj: private object
+ *
+ * Finalize the private object.
+ */
+void
+drm_atomic_private_obj_fini(struct drm_private_obj *obj)
+{
+	obj->funcs->atomic_destroy_state(obj, obj->state);
+}
+EXPORT_SYMBOL(drm_atomic_private_obj_fini);
+
+/**
  * drm_atomic_get_private_obj_state - get private object state
  * @state: global atomic state
  * @obj: private object to get the state for
- * @funcs: pointer to the struct of function pointers that identify the object
- * type
  *
  * This function returns the private object state for the given private object,
  * allocating the state if needed. It does not grab any locks as the caller is
@@ -1005,18 +1011,18 @@ static void drm_atomic_plane_print_state(struct drm_printer *p,
  *
  * Either the allocated state or the error code encoded into a pointer.
  */
-void *
-drm_atomic_get_private_obj_state(struct drm_atomic_state *state, void *obj,
-			      const struct drm_private_state_funcs *funcs)
+struct drm_private_state *
+drm_atomic_get_private_obj_state(struct drm_atomic_state *state,
+				 struct drm_private_obj *obj)
 {
 	int index, num_objs, i;
 	size_t size;
 	struct __drm_private_objs_state *arr;
+	struct drm_private_state *obj_state;
 
 	for (i = 0; i < state->num_private_objs; i++)
-		if (obj == state->private_objs[i].obj &&
-		    state->private_objs[i].obj_state)
-			return state->private_objs[i].obj_state;
+		if (obj == state->private_objs[i].ptr)
+			return state->private_objs[i].state;
 
 	num_objs = state->num_private_objs + 1;
 	size = sizeof(*state->private_objs) * num_objs;
@@ -1028,18 +1034,21 @@ drm_atomic_get_private_obj_state(struct drm_atomic_state *state, void *obj,
 	index = state->num_private_objs;
 	memset(&state->private_objs[index], 0, sizeof(*state->private_objs));
 
-	state->private_objs[index].obj_state = funcs->duplicate_state(state, obj);
-	if (!state->private_objs[index].obj_state)
+	obj_state = obj->funcs->atomic_duplicate_state(obj);
+	if (!obj_state)
 		return ERR_PTR(-ENOMEM);
 
-	state->private_objs[index].obj = obj;
-	state->private_objs[index].funcs = funcs;
+	state->private_objs[index].state = obj_state;
+	state->private_objs[index].old_state = obj->state;
+	state->private_objs[index].new_state = obj_state;
+	state->private_objs[index].ptr = obj;
+
 	state->num_private_objs = num_objs;
 
-	DRM_DEBUG_ATOMIC("Added new private object state %p to %p\n",
-			 state->private_objs[index].obj_state, state);
+	DRM_DEBUG_ATOMIC("Added new private object %p state %p to %p\n",
+			 obj, obj_state, state);
 
-	return state->private_objs[index].obj_state;
+	return obj_state;
 }
 EXPORT_SYMBOL(drm_atomic_get_private_obj_state);
 
@@ -1135,7 +1144,7 @@ EXPORT_SYMBOL(drm_atomic_get_connector_state);
  * RETURNS:
  * Zero on success, error code on failure
  */
-int drm_atomic_connector_set_property(struct drm_connector *connector,
+static int drm_atomic_connector_set_property(struct drm_connector *connector,
 		struct drm_connector_state *state, struct drm_property *property,
 		uint64_t val)
 {
@@ -1202,7 +1211,6 @@ int drm_atomic_connector_set_property(struct drm_connector *connector,
 
 	return 0;
 }
-EXPORT_SYMBOL(drm_atomic_connector_set_property);
 
 static void drm_atomic_connector_print_state(struct drm_printer *p,
 		const struct drm_connector_state *state)
@@ -1580,38 +1588,6 @@ drm_atomic_add_affected_planes(struct drm_atomic_state *state,
 EXPORT_SYMBOL(drm_atomic_add_affected_planes);
 
 /**
- * drm_atomic_legacy_backoff - locking backoff for legacy ioctls
- * @state: atomic state
- *
- * This function should be used by legacy entry points which don't understand
- * -EDEADLK semantics. For simplicity this one will grab all modeset locks after
- * the slowpath completed.
- */
-void drm_atomic_legacy_backoff(struct drm_atomic_state *state)
-{
-	struct drm_device *dev = state->dev;
-	int ret;
-	bool global = false;
-
-	if (WARN_ON(dev->mode_config.acquire_ctx == state->acquire_ctx)) {
-		global = true;
-
-		dev->mode_config.acquire_ctx = NULL;
-	}
-
-retry:
-	drm_modeset_backoff(state->acquire_ctx);
-
-	ret = drm_modeset_lock_all_ctx(dev, state->acquire_ctx);
-	if (ret)
-		goto retry;
-
-	if (global)
-		dev->mode_config.acquire_ctx = state->acquire_ctx;
-}
-EXPORT_SYMBOL(drm_atomic_legacy_backoff);
-
-/**
  * drm_atomic_check_only - check whether a given config would work
  * @state: atomic configuration to check
  *
@@ -1857,9 +1833,60 @@ static struct drm_pending_vblank_event *create_vblank_event(
 	return e;
 }
 
-static int atomic_set_prop(struct drm_atomic_state *state,
-		struct drm_mode_object *obj, struct drm_property *prop,
-		uint64_t prop_value)
+int drm_atomic_connector_commit_dpms(struct drm_atomic_state *state,
+				     struct drm_connector *connector,
+				     int mode)
+{
+	struct drm_connector *tmp_connector;
+	struct drm_connector_state *new_conn_state;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *crtc_state;
+	int i, ret, old_mode = connector->dpms;
+	bool active = false;
+
+	ret = drm_modeset_lock(&state->dev->mode_config.connection_mutex,
+			       state->acquire_ctx);
+	if (ret)
+		return ret;
+
+	if (mode != DRM_MODE_DPMS_ON)
+		mode = DRM_MODE_DPMS_OFF;
+	connector->dpms = mode;
+
+	crtc = connector->state->crtc;
+	if (!crtc)
+		goto out;
+	ret = drm_atomic_add_affected_connectors(state, crtc);
+	if (ret)
+		goto out;
+
+	crtc_state = drm_atomic_get_crtc_state(state, crtc);
+	if (IS_ERR(crtc_state)) {
+		ret = PTR_ERR(crtc_state);
+		goto out;
+	}
+
+	for_each_new_connector_in_state(state, tmp_connector, new_conn_state, i) {
+		if (new_conn_state->crtc != crtc)
+			continue;
+		if (tmp_connector->dpms == DRM_MODE_DPMS_ON) {
+			active = true;
+			break;
+		}
+	}
+
+	crtc_state->active = active;
+	ret = drm_atomic_commit(state);
+out:
+	if (ret != 0)
+		connector->dpms = old_mode;
+	return ret;
+}
+
+int drm_atomic_set_property(struct drm_atomic_state *state,
+			    struct drm_mode_object *obj,
+			    struct drm_property *prop,
+			    uint64_t prop_value)
 {
 	struct drm_mode_object *ref;
 	int ret;
@@ -2042,7 +2069,7 @@ static int prepare_crtc_signaling(struct drm_device *dev,
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *crtc_state;
-	int i, ret;
+	int i, c = 0, ret;
 
 	if (arg->flags & DRM_MODE_ATOMIC_TEST_ONLY)
 		return 0;
@@ -2103,8 +2130,17 @@ static int prepare_crtc_signaling(struct drm_device *dev,
 
 			crtc_state->event->base.fence = fence;
 		}
+
+		c++;
 	}
 
+	/*
+	 * Having this flag means user mode pends on event which will never
+	 * reach due to lack of at least one CRTC for signaling
+	 */
+	if (c == 0 && (arg->flags & DRM_MODE_PAGE_FLIP_EVENT))
+		return -EINVAL;
+
 	return 0;
 }
 
@@ -2272,7 +2308,8 @@ retry:
 				goto out;
 			}
 
-			ret = atomic_set_prop(state, obj, prop, prop_value);
+			ret = drm_atomic_set_property(state, obj, prop,
+						      prop_value);
 			if (ret) {
 				drm_mode_object_put(obj);
 				goto out;
diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c
index 86d3093c6c9b..4e53aae9a1fb 100644
--- a/drivers/gpu/drm/drm_atomic_helper.c
+++ b/drivers/gpu/drm/drm_atomic_helper.c
@@ -795,6 +795,9 @@ int drm_atomic_helper_check(struct drm_device *dev,
 	if (ret)
 		return ret;
 
+	if (state->legacy_cursor_update)
+		state->async_update = !drm_atomic_helper_async_check(dev, state);
+
 	return ret;
 }
 EXPORT_SYMBOL(drm_atomic_helper_check);
@@ -918,16 +921,12 @@ drm_atomic_helper_update_legacy_modeset_state(struct drm_device *dev,
 		crtc = new_conn_state->crtc;
 		if ((!crtc && old_conn_state->crtc) ||
 		    (crtc && drm_atomic_crtc_needs_modeset(crtc->state))) {
-			struct drm_property *dpms_prop =
-				dev->mode_config.dpms_property;
 			int mode = DRM_MODE_DPMS_OFF;
 
 			if (crtc && crtc->state->active)
 				mode = DRM_MODE_DPMS_ON;
 
 			connector->dpms = mode;
-			drm_object_property_set_value(&connector->base,
-						      dpms_prop, mode);
 		}
 	}
 
@@ -1069,12 +1068,13 @@ void drm_atomic_helper_commit_modeset_enables(struct drm_device *dev,
 					      struct drm_atomic_state *old_state)
 {
 	struct drm_crtc *crtc;
+	struct drm_crtc_state *old_crtc_state;
 	struct drm_crtc_state *new_crtc_state;
 	struct drm_connector *connector;
 	struct drm_connector_state *new_conn_state;
 	int i;
 
-	for_each_new_crtc_in_state(old_state, crtc, new_crtc_state, i) {
+	for_each_oldnew_crtc_in_state(old_state, crtc, old_crtc_state, new_crtc_state, i) {
 		const struct drm_crtc_helper_funcs *funcs;
 
 		/* Need to filter out CRTCs where only planes change. */
@@ -1090,8 +1090,8 @@ void drm_atomic_helper_commit_modeset_enables(struct drm_device *dev,
 			DRM_DEBUG_ATOMIC("enabling [CRTC:%d:%s]\n",
 					 crtc->base.id, crtc->name);
 
-			if (funcs->enable)
-				funcs->enable(crtc);
+			if (funcs->atomic_enable)
+				funcs->atomic_enable(crtc, old_crtc_state);
 			else
 				funcs->commit(crtc);
 		}
@@ -1191,9 +1191,13 @@ EXPORT_SYMBOL(drm_atomic_helper_wait_for_fences);
  *
  * Helper to, after atomic commit, wait for vblanks on all effected
  * crtcs (ie. before cleaning up old framebuffers using
- * drm_atomic_helper_cleanup_planes()). It will only wait on crtcs where the
+ * drm_atomic_helper_cleanup_planes()). It will only wait on CRTCs where the
  * framebuffers have actually changed to optimize for the legacy cursor and
  * plane update use-case.
+ *
+ * Drivers using the nonblocking commit tracking support initialized by calling
+ * drm_atomic_helper_setup_commit() should look at
+ * drm_atomic_helper_wait_for_flip_done() as an alternative.
  */
 void
 drm_atomic_helper_wait_for_vblanks(struct drm_device *dev,
@@ -1241,27 +1245,54 @@ drm_atomic_helper_wait_for_vblanks(struct drm_device *dev,
 EXPORT_SYMBOL(drm_atomic_helper_wait_for_vblanks);
 
 /**
+ * drm_atomic_helper_wait_for_flip_done - wait for all page flips to be done
+ * @dev: DRM device
+ * @old_state: atomic state object with old state structures
+ *
+ * Helper to, after atomic commit, wait for page flips on all effected
+ * crtcs (ie. before cleaning up old framebuffers using
+ * drm_atomic_helper_cleanup_planes()). Compared to
+ * drm_atomic_helper_wait_for_vblanks() this waits for the completion of on all
+ * CRTCs, assuming that cursors-only updates are signalling their completion
+ * immediately (or using a different path).
+ *
+ * This requires that drivers use the nonblocking commit tracking support
+ * initialized using drm_atomic_helper_setup_commit().
+ */
+void drm_atomic_helper_wait_for_flip_done(struct drm_device *dev,
+					  struct drm_atomic_state *old_state)
+{
+	struct drm_crtc_state *unused;
+	struct drm_crtc *crtc;
+	int i;
+
+	for_each_new_crtc_in_state(old_state, crtc, unused, i) {
+		struct drm_crtc_commit *commit = old_state->crtcs[i].commit;
+		int ret;
+
+		if (!commit)
+			continue;
+
+		ret = wait_for_completion_timeout(&commit->flip_done, 10 * HZ);
+		if (ret == 0)
+			DRM_ERROR("[CRTC:%d:%s] flip_done timed out\n",
+				  crtc->base.id, crtc->name);
+	}
+}
+EXPORT_SYMBOL(drm_atomic_helper_wait_for_flip_done);
+
+/**
  * drm_atomic_helper_commit_tail - commit atomic update to hardware
  * @old_state: atomic state object with old state structures
  *
  * This is the default implementation for the
- * &drm_mode_config_helper_funcs.atomic_commit_tail hook.
+ * &drm_mode_config_helper_funcs.atomic_commit_tail hook, for drivers
+ * that do not support runtime_pm or do not need the CRTC to be
+ * enabled to perform a commit. Otherwise, see
+ * drm_atomic_helper_commit_tail_rpm().
  *
  * Note that the default ordering of how the various stages are called is to
- * match the legacy modeset helper library closest. One peculiarity of that is
- * that it doesn't mesh well with runtime PM at all.
- *
- * For drivers supporting runtime PM the recommended sequence is instead ::
- *
- *     drm_atomic_helper_commit_modeset_disables(dev, old_state);
- *
- *     drm_atomic_helper_commit_modeset_enables(dev, old_state);
- *
- *     drm_atomic_helper_commit_planes(dev, old_state,
- *                                     DRM_PLANE_COMMIT_ACTIVE_ONLY);
- *
- * for committing the atomic update to hardware.  See the kerneldoc entries for
- * these three functions for more details.
+ * match the legacy modeset helper library closest.
  */
 void drm_atomic_helper_commit_tail(struct drm_atomic_state *old_state)
 {
@@ -1281,6 +1312,35 @@ void drm_atomic_helper_commit_tail(struct drm_atomic_state *old_state)
 }
 EXPORT_SYMBOL(drm_atomic_helper_commit_tail);
 
+/**
+ * drm_atomic_helper_commit_tail_rpm - commit atomic update to hardware
+ * @old_state: new modeset state to be committed
+ *
+ * This is an alternative implementation for the
+ * &drm_mode_config_helper_funcs.atomic_commit_tail hook, for drivers
+ * that support runtime_pm or need the CRTC to be enabled to perform a
+ * commit. Otherwise, one should use the default implementation
+ * drm_atomic_helper_commit_tail().
+ */
+void drm_atomic_helper_commit_tail_rpm(struct drm_atomic_state *old_state)
+{
+	struct drm_device *dev = old_state->dev;
+
+	drm_atomic_helper_commit_modeset_disables(dev, old_state);
+
+	drm_atomic_helper_commit_modeset_enables(dev, old_state);
+
+	drm_atomic_helper_commit_planes(dev, old_state,
+					DRM_PLANE_COMMIT_ACTIVE_ONLY);
+
+	drm_atomic_helper_commit_hw_done(old_state);
+
+	drm_atomic_helper_wait_for_vblanks(dev, old_state);
+
+	drm_atomic_helper_cleanup_planes(dev, old_state);
+}
+EXPORT_SYMBOL(drm_atomic_helper_commit_tail_rpm);
+
 static void commit_tail(struct drm_atomic_state *old_state)
 {
 	struct drm_device *dev = old_state->dev;
@@ -1311,6 +1371,114 @@ static void commit_work(struct work_struct *work)
 }
 
 /**
+ * drm_atomic_helper_async_check - check if state can be commited asynchronously
+ * @dev: DRM device
+ * @state: the driver state object
+ *
+ * This helper will check if it is possible to commit the state asynchronously.
+ * Async commits are not supposed to swap the states like normal sync commits
+ * but just do in-place changes on the current state.
+ *
+ * It will return 0 if the commit can happen in an asynchronous fashion or error
+ * if not. Note that error just mean it can't be commited asynchronously, if it
+ * fails the commit should be treated like a normal synchronous commit.
+ */
+int drm_atomic_helper_async_check(struct drm_device *dev,
+				   struct drm_atomic_state *state)
+{
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *crtc_state;
+	struct drm_crtc_commit *commit;
+	struct drm_plane *__plane, *plane = NULL;
+	struct drm_plane_state *__plane_state, *plane_state = NULL;
+	const struct drm_plane_helper_funcs *funcs;
+	int i, j, n_planes = 0;
+
+	for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
+		if (drm_atomic_crtc_needs_modeset(crtc_state))
+			return -EINVAL;
+	}
+
+	for_each_new_plane_in_state(state, __plane, __plane_state, i) {
+		n_planes++;
+		plane = __plane;
+		plane_state = __plane_state;
+	}
+
+	/* FIXME: we support only single plane updates for now */
+	if (!plane || n_planes != 1)
+		return -EINVAL;
+
+	if (!plane_state->crtc)
+		return -EINVAL;
+
+	funcs = plane->helper_private;
+	if (!funcs->atomic_async_update)
+		return -EINVAL;
+
+	if (plane_state->fence)
+		return -EINVAL;
+
+	/*
+	 * Don't do an async update if there is an outstanding commit modifying
+	 * the plane.  This prevents our async update's changes from getting
+	 * overridden by a previous synchronous update's state.
+	 */
+	for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
+		if (plane->crtc != crtc)
+			continue;
+
+		spin_lock(&crtc->commit_lock);
+		commit = list_first_entry_or_null(&crtc->commit_list,
+						  struct drm_crtc_commit,
+						  commit_entry);
+		if (!commit) {
+			spin_unlock(&crtc->commit_lock);
+			continue;
+		}
+		spin_unlock(&crtc->commit_lock);
+
+		if (!crtc->state->state)
+			continue;
+
+		for_each_plane_in_state(crtc->state->state, __plane,
+					__plane_state, j) {
+			if (__plane == plane)
+				return -EINVAL;
+		}
+	}
+
+	return funcs->atomic_async_check(plane, plane_state);
+}
+EXPORT_SYMBOL(drm_atomic_helper_async_check);
+
+/**
+ * drm_atomic_helper_async_commit - commit state asynchronously
+ * @dev: DRM device
+ * @state: the driver state object
+ *
+ * This function commits a state asynchronously, i.e., not vblank
+ * synchronized. It should be used on a state only when
+ * drm_atomic_async_check() succeeds. Async commits are not supposed to swap
+ * the states like normal sync commits, but just do in-place changes on the
+ * current state.
+ */
+void drm_atomic_helper_async_commit(struct drm_device *dev,
+				    struct drm_atomic_state *state)
+{
+	struct drm_plane *plane;
+	struct drm_plane_state *plane_state;
+	const struct drm_plane_helper_funcs *funcs;
+	int i;
+
+	for_each_new_plane_in_state(state, plane, plane_state, i) {
+		funcs = plane->helper_private;
+		funcs->atomic_async_update(plane, plane_state);
+	}
+}
+EXPORT_SYMBOL(drm_atomic_helper_async_commit);
+
+/**
  * drm_atomic_helper_commit - commit validated state object
  * @dev: DRM device
  * @state: the driver state object
@@ -1334,6 +1502,17 @@ int drm_atomic_helper_commit(struct drm_device *dev,
 {
 	int ret;
 
+	if (state->async_update) {
+		ret = drm_atomic_helper_prepare_planes(dev, state);
+		if (ret)
+			return ret;
+
+		drm_atomic_helper_async_commit(dev, state);
+		drm_atomic_helper_cleanup_planes(dev, state);
+
+		return 0;
+	}
+
 	ret = drm_atomic_helper_setup_commit(state, nonblock);
 	if (ret)
 		return ret;
@@ -1346,10 +1525,8 @@ int drm_atomic_helper_commit(struct drm_device *dev,
 
 	if (!nonblock) {
 		ret = drm_atomic_helper_wait_for_fences(dev, state, true);
-		if (ret) {
-			drm_atomic_helper_cleanup_planes(dev, state);
-			return ret;
-		}
+		if (ret)
+			goto err;
 	}
 
 	/*
@@ -1358,7 +1535,9 @@ int drm_atomic_helper_commit(struct drm_device *dev,
 	 * the software side now.
 	 */
 
-	drm_atomic_helper_swap_state(state, true);
+	ret = drm_atomic_helper_swap_state(state, true);
+	if (ret)
+		goto err;
 
 	/*
 	 * Everything below can be run asynchronously without the need to grab
@@ -1387,6 +1566,10 @@ int drm_atomic_helper_commit(struct drm_device *dev,
 		commit_tail(state);
 
 	return 0;
+
+err:
+	drm_atomic_helper_cleanup_planes(dev, state);
+	return ret;
 }
 EXPORT_SYMBOL(drm_atomic_helper_commit);
 
@@ -1680,9 +1863,7 @@ void drm_atomic_helper_commit_hw_done(struct drm_atomic_state *old_state)
 
 		/* backend must have consumed any event by now */
 		WARN_ON(new_crtc_state->event);
-		spin_lock(&crtc->commit_lock);
 		complete_all(&commit->hw_done);
-		spin_unlock(&crtc->commit_lock);
 	}
 }
 EXPORT_SYMBOL(drm_atomic_helper_commit_hw_done);
@@ -1711,7 +1892,6 @@ void drm_atomic_helper_commit_cleanup_done(struct drm_atomic_state *old_state)
 		if (WARN_ON(!commit))
 			continue;
 
-		spin_lock(&crtc->commit_lock);
 		complete_all(&commit->cleanup_done);
 		WARN_ON(!try_wait_for_completion(&commit->hw_done));
 
@@ -1721,8 +1901,6 @@ void drm_atomic_helper_commit_cleanup_done(struct drm_atomic_state *old_state)
 		if (try_wait_for_completion(&commit->flip_done))
 			goto del_commit;
 
-		spin_unlock(&crtc->commit_lock);
-
 		/* We must wait for the vblank event to signal our completion
 		 * before releasing our reference, since the vblank work does
 		 * not hold a reference of its own. */
@@ -1732,8 +1910,8 @@ void drm_atomic_helper_commit_cleanup_done(struct drm_atomic_state *old_state)
 			DRM_ERROR("[CRTC:%d:%s] flip_done timed out\n",
 				  crtc->base.id, crtc->name);
 
-		spin_lock(&crtc->commit_lock);
 del_commit:
+		spin_lock(&crtc->commit_lock);
 		list_del(&commit->commit_entry);
 		spin_unlock(&crtc->commit_lock);
 	}
@@ -2069,14 +2247,14 @@ EXPORT_SYMBOL(drm_atomic_helper_cleanup_planes);
 /**
  * drm_atomic_helper_swap_state - store atomic state into current sw state
  * @state: atomic state
- * @stall: stall for proceeding commits
+ * @stall: stall for preceeding commits
  *
  * This function stores the atomic state into the current state pointers in all
  * driver objects. It should be called after all failing steps have been done
  * and succeeded, but before the actual hardware state is committed.
  *
  * For cleanup and error recovery the current state for all changed objects will
- * be swaped into @state.
+ * be swapped into @state.
  *
  * With that sequence it fits perfectly into the plane prepare/cleanup sequence:
  *
@@ -2095,12 +2273,16 @@ EXPORT_SYMBOL(drm_atomic_helper_cleanup_planes);
  * the &drm_plane.state, &drm_crtc.state or &drm_connector.state pointer. With
  * the current atomic helpers this is almost always the case, since the helpers
  * don't pass the right state structures to the callbacks.
+ *
+ * Returns:
+ *
+ * Returns 0 on success. Can return -ERESTARTSYS when @stall is true and the
+ * waiting for the previous commits has been interrupted.
  */
-void drm_atomic_helper_swap_state(struct drm_atomic_state *state,
+int drm_atomic_helper_swap_state(struct drm_atomic_state *state,
 				  bool stall)
 {
-	int i;
-	long ret;
+	int i, ret;
 	struct drm_connector *connector;
 	struct drm_connector_state *old_conn_state, *new_conn_state;
 	struct drm_crtc *crtc;
@@ -2108,8 +2290,8 @@ void drm_atomic_helper_swap_state(struct drm_atomic_state *state,
 	struct drm_plane *plane;
 	struct drm_plane_state *old_plane_state, *new_plane_state;
 	struct drm_crtc_commit *commit;
-	void *obj, *obj_state;
-	const struct drm_private_state_funcs *funcs;
+	struct drm_private_obj *obj;
+	struct drm_private_state *old_obj_state, *new_obj_state;
 
 	if (stall) {
 		for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
@@ -2123,12 +2305,11 @@ void drm_atomic_helper_swap_state(struct drm_atomic_state *state,
 			if (!commit)
 				continue;
 
-			ret = wait_for_completion_timeout(&commit->hw_done,
-							  10*HZ);
-			if (ret == 0)
-				DRM_ERROR("[CRTC:%d:%s] hw_done timed out\n",
-					  crtc->base.id, crtc->name);
+			ret = wait_for_completion_interruptible(&commit->hw_done);
 			drm_crtc_commit_put(commit);
+
+			if (ret)
+				return ret;
 		}
 	}
 
@@ -2171,8 +2352,17 @@ void drm_atomic_helper_swap_state(struct drm_atomic_state *state,
 		plane->state = new_plane_state;
 	}
 
-	__for_each_private_obj(state, obj, obj_state, i, funcs)
-		funcs->swap_state(obj, &state->private_objs[i].obj_state);
+	for_each_oldnew_private_obj_in_state(state, obj, old_obj_state, new_obj_state, i) {
+		WARN_ON(obj->state != old_obj_state);
+
+		old_obj_state->state = state;
+		new_obj_state->state = NULL;
+
+		state->private_objs[i].state = old_obj_state;
+		obj->state = new_obj_state;
+	}
+
+	return 0;
 }
 EXPORT_SYMBOL(drm_atomic_helper_swap_state);
 
@@ -2526,6 +2716,7 @@ int drm_atomic_helper_disable_all(struct drm_device *dev,
 	struct drm_plane *plane;
 	struct drm_crtc_state *crtc_state;
 	struct drm_crtc *crtc;
+	unsigned plane_mask = 0;
 	int ret, i;
 
 	state = drm_atomic_state_alloc(dev);
@@ -2556,22 +2747,26 @@ int drm_atomic_helper_disable_all(struct drm_device *dev,
 			goto free;
 	}
 
-	for_each_connector_in_state(state, conn, conn_state, i) {
+	for_each_new_connector_in_state(state, conn, conn_state, i) {
 		ret = drm_atomic_set_crtc_for_connector(conn_state, NULL);
 		if (ret < 0)
 			goto free;
 	}
 
-	for_each_plane_in_state(state, plane, plane_state, i) {
+	for_each_new_plane_in_state(state, plane, plane_state, i) {
 		ret = drm_atomic_set_crtc_for_plane(plane_state, NULL);
 		if (ret < 0)
 			goto free;
 
 		drm_atomic_set_fb_for_plane(plane_state, NULL);
+		plane_mask |= BIT(drm_plane_index(plane));
+		plane->old_fb = plane->fb;
 	}
 
 	ret = drm_atomic_commit(state);
 free:
+	if (plane_mask)
+		drm_atomic_clean_old_fb(dev, plane_mask, ret);
 	drm_atomic_state_put(state);
 	return ret;
 }
@@ -2702,11 +2897,16 @@ int drm_atomic_helper_commit_duplicated_state(struct drm_atomic_state *state,
 	struct drm_connector_state *new_conn_state;
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *new_crtc_state;
+	unsigned plane_mask = 0;
+	struct drm_device *dev = state->dev;
+	int ret;
 
 	state->acquire_ctx = ctx;
 
-	for_each_new_plane_in_state(state, plane, new_plane_state, i)
+	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
+		plane_mask |= BIT(drm_plane_index(plane));
 		state->planes[i].old_state = plane->state;
+	}
 
 	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i)
 		state->crtcs[i].old_state = crtc->state;
@@ -2714,7 +2914,11 @@ int drm_atomic_helper_commit_duplicated_state(struct drm_atomic_state *state,
 	for_each_new_connector_in_state(state, connector, new_conn_state, i)
 		state->connectors[i].old_state = connector->state;
 
-	return drm_atomic_commit(state);
+	ret = drm_atomic_commit(state);
+	if (plane_mask)
+		drm_atomic_clean_old_fb(dev, plane_mask, ret);
+
+	return ret;
 }
 EXPORT_SYMBOL(drm_atomic_helper_commit_duplicated_state);
 
@@ -2763,177 +2967,11 @@ out:
 }
 EXPORT_SYMBOL(drm_atomic_helper_resume);
 
-/**
- * drm_atomic_helper_crtc_set_property - helper for crtc properties
- * @crtc: DRM crtc
- * @property: DRM property
- * @val: value of property
- *
- * Provides a default crtc set_property handler using the atomic driver
- * interface.
- *
- * RETURNS:
- * Zero on success, error code on failure
- */
-int
-drm_atomic_helper_crtc_set_property(struct drm_crtc *crtc,
-				    struct drm_property *property,
-				    uint64_t val)
-{
-	struct drm_atomic_state *state;
-	struct drm_crtc_state *crtc_state;
-	int ret = 0;
-
-	state = drm_atomic_state_alloc(crtc->dev);
-	if (!state)
-		return -ENOMEM;
-
-	/* ->set_property is always called with all locks held. */
-	state->acquire_ctx = crtc->dev->mode_config.acquire_ctx;
-retry:
-	crtc_state = drm_atomic_get_crtc_state(state, crtc);
-	if (IS_ERR(crtc_state)) {
-		ret = PTR_ERR(crtc_state);
-		goto fail;
-	}
-
-	ret = drm_atomic_crtc_set_property(crtc, crtc_state,
-			property, val);
-	if (ret)
-		goto fail;
-
-	ret = drm_atomic_commit(state);
-fail:
-	if (ret == -EDEADLK)
-		goto backoff;
-
-	drm_atomic_state_put(state);
-	return ret;
-
-backoff:
-	drm_atomic_state_clear(state);
-	drm_atomic_legacy_backoff(state);
-
-	goto retry;
-}
-EXPORT_SYMBOL(drm_atomic_helper_crtc_set_property);
-
-/**
- * drm_atomic_helper_plane_set_property - helper for plane properties
- * @plane: DRM plane
- * @property: DRM property
- * @val: value of property
- *
- * Provides a default plane set_property handler using the atomic driver
- * interface.
- *
- * RETURNS:
- * Zero on success, error code on failure
- */
-int
-drm_atomic_helper_plane_set_property(struct drm_plane *plane,
-				    struct drm_property *property,
-				    uint64_t val)
-{
-	struct drm_atomic_state *state;
-	struct drm_plane_state *plane_state;
-	int ret = 0;
-
-	state = drm_atomic_state_alloc(plane->dev);
-	if (!state)
-		return -ENOMEM;
-
-	/* ->set_property is always called with all locks held. */
-	state->acquire_ctx = plane->dev->mode_config.acquire_ctx;
-retry:
-	plane_state = drm_atomic_get_plane_state(state, plane);
-	if (IS_ERR(plane_state)) {
-		ret = PTR_ERR(plane_state);
-		goto fail;
-	}
-
-	ret = drm_atomic_plane_set_property(plane, plane_state,
-			property, val);
-	if (ret)
-		goto fail;
-
-	ret = drm_atomic_commit(state);
-fail:
-	if (ret == -EDEADLK)
-		goto backoff;
-
-	drm_atomic_state_put(state);
-	return ret;
-
-backoff:
-	drm_atomic_state_clear(state);
-	drm_atomic_legacy_backoff(state);
-
-	goto retry;
-}
-EXPORT_SYMBOL(drm_atomic_helper_plane_set_property);
-
-/**
- * drm_atomic_helper_connector_set_property - helper for connector properties
- * @connector: DRM connector
- * @property: DRM property
- * @val: value of property
- *
- * Provides a default connector set_property handler using the atomic driver
- * interface.
- *
- * RETURNS:
- * Zero on success, error code on failure
- */
-int
-drm_atomic_helper_connector_set_property(struct drm_connector *connector,
-				    struct drm_property *property,
-				    uint64_t val)
-{
-	struct drm_atomic_state *state;
-	struct drm_connector_state *connector_state;
-	int ret = 0;
-
-	state = drm_atomic_state_alloc(connector->dev);
-	if (!state)
-		return -ENOMEM;
-
-	/* ->set_property is always called with all locks held. */
-	state->acquire_ctx = connector->dev->mode_config.acquire_ctx;
-retry:
-	connector_state = drm_atomic_get_connector_state(state, connector);
-	if (IS_ERR(connector_state)) {
-		ret = PTR_ERR(connector_state);
-		goto fail;
-	}
-
-	ret = drm_atomic_connector_set_property(connector, connector_state,
-			property, val);
-	if (ret)
-		goto fail;
-
-	ret = drm_atomic_commit(state);
-fail:
-	if (ret == -EDEADLK)
-		goto backoff;
-
-	drm_atomic_state_put(state);
-	return ret;
-
-backoff:
-	drm_atomic_state_clear(state);
-	drm_atomic_legacy_backoff(state);
-
-	goto retry;
-}
-EXPORT_SYMBOL(drm_atomic_helper_connector_set_property);
-
-static int page_flip_common(
-				struct drm_atomic_state *state,
-				struct drm_crtc *crtc,
-				struct drm_framebuffer *fb,
-				struct drm_pending_vblank_event *event,
-				uint32_t flags)
+static int page_flip_common(struct drm_atomic_state *state,
+			    struct drm_crtc *crtc,
+			    struct drm_framebuffer *fb,
+			    struct drm_pending_vblank_event *event,
+			    uint32_t flags)
 {
 	struct drm_plane *plane = crtc->primary;
 	struct drm_plane_state *plane_state;
@@ -3027,13 +3065,12 @@ EXPORT_SYMBOL(drm_atomic_helper_page_flip);
  * Returns:
  * Returns 0 on success, negative errno numbers on failure.
  */
-int drm_atomic_helper_page_flip_target(
-				struct drm_crtc *crtc,
-				struct drm_framebuffer *fb,
-				struct drm_pending_vblank_event *event,
-				uint32_t flags,
-				uint32_t target,
-				struct drm_modeset_acquire_ctx *ctx)
+int drm_atomic_helper_page_flip_target(struct drm_crtc *crtc,
+				       struct drm_framebuffer *fb,
+				       struct drm_pending_vblank_event *event,
+				       uint32_t flags,
+				       uint32_t target,
+				       struct drm_modeset_acquire_ctx *ctx)
 {
 	struct drm_plane *plane = crtc->primary;
 	struct drm_atomic_state *state;
@@ -3065,85 +3102,6 @@ fail:
 EXPORT_SYMBOL(drm_atomic_helper_page_flip_target);
 
 /**
- * drm_atomic_helper_connector_dpms() - connector dpms helper implementation
- * @connector: affected connector
- * @mode: DPMS mode
- *
- * This is the main helper function provided by the atomic helper framework for
- * implementing the legacy DPMS connector interface. It computes the new desired
- * &drm_crtc_state.active state for the corresponding CRTC (if the connector is
- * enabled) and updates it.
- *
- * Returns:
- * Returns 0 on success, negative errno numbers on failure.
- */
-int drm_atomic_helper_connector_dpms(struct drm_connector *connector,
-				     int mode)
-{
-	struct drm_mode_config *config = &connector->dev->mode_config;
-	struct drm_atomic_state *state;
-	struct drm_crtc_state *crtc_state;
-	struct drm_crtc *crtc;
-	struct drm_connector *tmp_connector;
-	struct drm_connector_list_iter conn_iter;
-	int ret;
-	bool active = false;
-	int old_mode = connector->dpms;
-
-	if (mode != DRM_MODE_DPMS_ON)
-		mode = DRM_MODE_DPMS_OFF;
-
-	connector->dpms = mode;
-	crtc = connector->state->crtc;
-
-	if (!crtc)
-		return 0;
-
-	state = drm_atomic_state_alloc(connector->dev);
-	if (!state)
-		return -ENOMEM;
-
-	state->acquire_ctx = crtc->dev->mode_config.acquire_ctx;
-retry:
-	crtc_state = drm_atomic_get_crtc_state(state, crtc);
-	if (IS_ERR(crtc_state)) {
-		ret = PTR_ERR(crtc_state);
-		goto fail;
-	}
-
-	WARN_ON(!drm_modeset_is_locked(&config->connection_mutex));
-
-	drm_connector_list_iter_begin(connector->dev, &conn_iter);
-	drm_for_each_connector_iter(tmp_connector, &conn_iter) {
-		if (tmp_connector->state->crtc != crtc)
-			continue;
-
-		if (tmp_connector->dpms == DRM_MODE_DPMS_ON) {
-			active = true;
-			break;
-		}
-	}
-	drm_connector_list_iter_end(&conn_iter);
-	crtc_state->active = active;
-
-	ret = drm_atomic_commit(state);
-fail:
-	if (ret == -EDEADLK)
-		goto backoff;
-	if (ret != 0)
-		connector->dpms = old_mode;
-	drm_atomic_state_put(state);
-	return ret;
-
-backoff:
-	drm_atomic_state_clear(state);
-	drm_atomic_legacy_backoff(state);
-
-	goto retry;
-}
-EXPORT_SYMBOL(drm_atomic_helper_connector_dpms);
-
-/**
  * drm_atomic_helper_best_encoder - Helper for
  * 	&drm_connector_helper_funcs.best_encoder callback
  * @connector: Connector control structure
@@ -3612,12 +3570,12 @@ int drm_atomic_helper_legacy_gamma_set(struct drm_crtc *crtc,
 				       struct drm_modeset_acquire_ctx *ctx)
 {
 	struct drm_device *dev = crtc->dev;
-	struct drm_mode_config *config = &dev->mode_config;
 	struct drm_atomic_state *state;
 	struct drm_crtc_state *crtc_state;
 	struct drm_property_blob *blob = NULL;
 	struct drm_color_lut *blob_data;
 	int i, ret = 0;
+	bool replaced;
 
 	state = drm_atomic_state_alloc(crtc->dev);
 	if (!state)
@@ -3648,20 +3606,10 @@ int drm_atomic_helper_legacy_gamma_set(struct drm_crtc *crtc,
 	}
 
 	/* Reset DEGAMMA_LUT and CTM properties. */
-	ret = drm_atomic_crtc_set_property(crtc, crtc_state,
-			config->degamma_lut_property, 0);
-	if (ret)
-		goto fail;
-
-	ret = drm_atomic_crtc_set_property(crtc, crtc_state,
-			config->ctm_property, 0);
-	if (ret)
-		goto fail;
-
-	ret = drm_atomic_crtc_set_property(crtc, crtc_state,
-			config->gamma_lut_property, blob->base.id);
-	if (ret)
-		goto fail;
+	replaced  = drm_property_replace_blob(&crtc_state->degamma_lut, NULL);
+	replaced |= drm_property_replace_blob(&crtc_state->ctm, NULL);
+	replaced |= drm_property_replace_blob(&crtc_state->gamma_lut, blob);
+	crtc_state->color_mgmt_changed |= replaced;
 
 	ret = drm_atomic_commit(state);
 
@@ -3671,3 +3619,18 @@ fail:
 	return ret;
 }
 EXPORT_SYMBOL(drm_atomic_helper_legacy_gamma_set);
+
+/**
+ * __drm_atomic_helper_private_duplicate_state - copy atomic private state
+ * @obj: CRTC object
+ * @state: new private object state
+ *
+ * Copies atomic state from a private objects's current state and resets inferred values.
+ * This is useful for drivers that subclass the private state.
+ */
+void __drm_atomic_helper_private_obj_duplicate_state(struct drm_private_obj *obj,
+						     struct drm_private_state *state)
+{
+	memcpy(state, obj->state, sizeof(*state));
+}
+EXPORT_SYMBOL(__drm_atomic_helper_private_obj_duplicate_state);
diff --git a/drivers/gpu/drm/drm_blend.c b/drivers/gpu/drm/drm_blend.c
index db6aeec50b82..2e5e089dd912 100644
--- a/drivers/gpu/drm/drm_blend.c
+++ b/drivers/gpu/drm/drm_blend.c
@@ -319,7 +319,7 @@ static int drm_atomic_helper_crtc_normalize_zpos(struct drm_crtc *crtc,
 	DRM_DEBUG_ATOMIC("[CRTC:%d:%s] calculating normalized zpos values\n",
 			 crtc->base.id, crtc->name);
 
-	states = kmalloc_array(total_planes, sizeof(*states), GFP_TEMPORARY);
+	states = kmalloc_array(total_planes, sizeof(*states), GFP_KERNEL);
 	if (!states)
 		return -ENOMEM;
 
diff --git a/drivers/gpu/drm/drm_color_mgmt.c b/drivers/gpu/drm/drm_color_mgmt.c
index 3eda500fc005..fe0982708e95 100644
--- a/drivers/gpu/drm/drm_color_mgmt.c
+++ b/drivers/gpu/drm/drm_color_mgmt.c
@@ -128,6 +128,9 @@ EXPORT_SYMBOL(drm_color_lut_extract);
  * optional. The gamma and degamma properties are only attached if
  * their size is not 0 and ctm_property is only attached if has_ctm is
  * true.
+ *
+ * Drivers should use drm_atomic_helper_legacy_gamma_set() to implement the
+ * legacy &drm_crtc_funcs.gamma_set callback.
  */
 void drm_crtc_enable_color_mgmt(struct drm_crtc *crtc,
 				uint degamma_lut_size,
diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c
index 8072e6e4c62c..ba9f36cef68c 100644
--- a/drivers/gpu/drm/drm_connector.c
+++ b/drivers/gpu/drm/drm_connector.c
@@ -717,9 +717,9 @@ DRM_ENUM_NAME_FN(drm_get_tv_subconnector_name,
  * 	drivers, it remaps to controlling the "ACTIVE" property on the CRTC the
  * 	connector is linked to. Drivers should never set this property directly,
  * 	it is handled by the DRM core by calling the &drm_connector_funcs.dpms
- * 	callback. Atomic drivers should implement this hook using
- * 	drm_atomic_helper_connector_dpms(). This is the only property standard
- * 	connector property that userspace can change.
+ * 	callback. For atomic drivers the remapping to the "ACTIVE" property is
+ * 	implemented in the DRM core.  This is the only standard connector
+ * 	property that userspace can change.
  * PATH:
  * 	Connector path property to identify how this sink is physically
  * 	connected. Used by DP MST. This should be set by calling
@@ -1225,7 +1225,6 @@ int drm_mode_connector_set_obj_prop(struct drm_mode_object *obj,
 	} else if (connector->funcs->set_property)
 		ret = connector->funcs->set_property(connector, property, value);
 
-	/* store the property value if successful */
 	if (!ret)
 		drm_object_property_set_value(&connector->base, property, value);
 	return ret;
diff --git a/drivers/gpu/drm/drm_crtc_helper.c b/drivers/gpu/drm/drm_crtc_helper.c
index 4afdf7902eda..eab36a460638 100644
--- a/drivers/gpu/drm/drm_crtc_helper.c
+++ b/drivers/gpu/drm/drm_crtc_helper.c
@@ -863,8 +863,7 @@ static int drm_helper_choose_crtc_dpms(struct drm_crtc *crtc)
  * provided by the driver.
  *
  * This function is deprecated.  New drivers must implement atomic modeset
- * support, for which this function is unsuitable. Instead drivers should use
- * drm_atomic_helper_connector_dpms().
+ * support, where DPMS is handled in the DRM core.
  *
  * Returns:
  * Always returns 0.
diff --git a/drivers/gpu/drm/drm_crtc_internal.h b/drivers/gpu/drm/drm_crtc_internal.h
index d077c5490041..a43582076b20 100644
--- a/drivers/gpu/drm/drm_crtc_internal.h
+++ b/drivers/gpu/drm/drm_crtc_internal.h
@@ -178,6 +178,13 @@ struct drm_minor;
 int drm_atomic_debugfs_init(struct drm_minor *minor);
 #endif
 
+int drm_atomic_connector_commit_dpms(struct drm_atomic_state *state,
+				     struct drm_connector *connector,
+				     int mode);
+int drm_atomic_set_property(struct drm_atomic_state *state,
+			    struct drm_mode_object *obj,
+			    struct drm_property *prop,
+			    uint64_t prop_value);
 int drm_atomic_get_property(struct drm_mode_object *obj,
 			    struct drm_property *property, uint64_t *val);
 int drm_mode_atomic_ioctl(struct drm_device *dev,
diff --git a/drivers/gpu/drm/drm_debugfs_crc.c b/drivers/gpu/drm/drm_debugfs_crc.c
index 1722d8f21449..f9e26dda56d6 100644
--- a/drivers/gpu/drm/drm_debugfs_crc.c
+++ b/drivers/gpu/drm/drm_debugfs_crc.c
@@ -136,21 +136,51 @@ static int crtc_crc_data_count(struct drm_crtc_crc *crc)
 	return CIRC_CNT(crc->head, crc->tail, DRM_CRC_ENTRIES_NR);
 }
 
+static void crtc_crc_cleanup(struct drm_crtc_crc *crc)
+{
+	kfree(crc->entries);
+	crc->entries = NULL;
+	crc->head = 0;
+	crc->tail = 0;
+	crc->values_cnt = 0;
+	crc->opened = false;
+}
+
 static int crtc_crc_open(struct inode *inode, struct file *filep)
 {
 	struct drm_crtc *crtc = inode->i_private;
 	struct drm_crtc_crc *crc = &crtc->crc;
 	struct drm_crtc_crc_entry *entries = NULL;
 	size_t values_cnt;
-	int ret;
+	int ret = 0;
 
-	if (crc->opened)
-		return -EBUSY;
+	if (drm_drv_uses_atomic_modeset(crtc->dev)) {
+		ret = drm_modeset_lock_interruptible(&crtc->mutex, NULL);
+		if (ret)
+			return ret;
+
+		if (!crtc->state->active)
+			ret = -EIO;
+		drm_modeset_unlock(&crtc->mutex);
+
+		if (ret)
+			return ret;
+	}
+
+	spin_lock_irq(&crc->lock);
+	if (!crc->opened)
+		crc->opened = true;
+	else
+		ret = -EBUSY;
+	spin_unlock_irq(&crc->lock);
 
-	ret = crtc->funcs->set_crc_source(crtc, crc->source, &values_cnt);
 	if (ret)
 		return ret;
 
+	ret = crtc->funcs->set_crc_source(crtc, crc->source, &values_cnt);
+	if (ret)
+		goto err;
+
 	if (WARN_ON(values_cnt > DRM_MAX_CRC_NR)) {
 		ret = -EINVAL;
 		goto err_disable;
@@ -170,7 +200,6 @@ static int crtc_crc_open(struct inode *inode, struct file *filep)
 	spin_lock_irq(&crc->lock);
 	crc->entries = entries;
 	crc->values_cnt = values_cnt;
-	crc->opened = true;
 
 	/*
 	 * Only return once we got a first frame, so userspace doesn't have to
@@ -182,12 +211,17 @@ static int crtc_crc_open(struct inode *inode, struct file *filep)
 						crc->lock);
 	spin_unlock_irq(&crc->lock);
 
-	WARN_ON(ret);
+	if (ret)
+		goto err_disable;
 
 	return 0;
 
 err_disable:
 	crtc->funcs->set_crc_source(crtc, NULL, &values_cnt);
+err:
+	spin_lock_irq(&crc->lock);
+	crtc_crc_cleanup(crc);
+	spin_unlock_irq(&crc->lock);
 	return ret;
 }
 
@@ -197,17 +231,12 @@ static int crtc_crc_release(struct inode *inode, struct file *filep)
 	struct drm_crtc_crc *crc = &crtc->crc;
 	size_t values_cnt;
 
+	crtc->funcs->set_crc_source(crtc, NULL, &values_cnt);
+
 	spin_lock_irq(&crc->lock);
-	kfree(crc->entries);
-	crc->entries = NULL;
-	crc->head = 0;
-	crc->tail = 0;
-	crc->values_cnt = 0;
-	crc->opened = false;
+	crtc_crc_cleanup(crc);
 	spin_unlock_irq(&crc->lock);
 
-	crtc->funcs->set_crc_source(crtc, NULL, &values_cnt);
-
 	return 0;
 }
 
@@ -334,7 +363,7 @@ int drm_crtc_add_crc_entry(struct drm_crtc *crtc, bool has_frame,
 	spin_lock(&crc->lock);
 
 	/* Caller may not have noticed yet that userspace has stopped reading */
-	if (!crc->opened) {
+	if (!crc->entries) {
 		spin_unlock(&crc->lock);
 		return -EINVAL;
 	}
diff --git a/drivers/gpu/drm/drm_dp_dual_mode_helper.c b/drivers/gpu/drm/drm_dp_dual_mode_helper.c
index 80e62f669321..0ef9011a1856 100644
--- a/drivers/gpu/drm/drm_dp_dual_mode_helper.c
+++ b/drivers/gpu/drm/drm_dp_dual_mode_helper.c
@@ -111,7 +111,7 @@ ssize_t drm_dp_dual_mode_write(struct i2c_adapter *adapter,
 	void *data;
 	int ret;
 
-	data = kmalloc(msg.len, GFP_TEMPORARY);
+	data = kmalloc(msg.len, GFP_KERNEL);
 	if (!data)
 		return -ENOMEM;
 
diff --git a/drivers/gpu/drm/drm_dp_mst_topology.c b/drivers/gpu/drm/drm_dp_mst_topology.c
index ae5f06895562..41b492f99955 100644
--- a/drivers/gpu/drm/drm_dp_mst_topology.c
+++ b/drivers/gpu/drm/drm_dp_mst_topology.c
@@ -31,6 +31,8 @@
 #include <drm/drmP.h>
 
 #include <drm/drm_fixed.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
 
 /**
  * DOC: dp mst helper
@@ -1342,15 +1344,17 @@ static void drm_dp_mst_link_probe_work(struct work_struct *work)
 static bool drm_dp_validate_guid(struct drm_dp_mst_topology_mgr *mgr,
 				 u8 *guid)
 {
-	static u8 zero_guid[16];
+	u64 salt;
 
-	if (!memcmp(guid, zero_guid, 16)) {
-		u64 salt = get_jiffies_64();
-		memcpy(&guid[0], &salt, sizeof(u64));
-		memcpy(&guid[8], &salt, sizeof(u64));
-		return false;
-	}
-	return true;
+	if (memchr_inv(guid, 0, 16))
+		return true;
+
+	salt = get_jiffies_64();
+
+	memcpy(&guid[0], &salt, sizeof(u64));
+	memcpy(&guid[8], &salt, sizeof(u64));
+
+	return false;
 }
 
 #if 0
@@ -2540,8 +2544,8 @@ int drm_dp_atomic_find_vcpi_slots(struct drm_atomic_state *state,
 	int req_slots;
 
 	topology_state = drm_atomic_get_mst_topology_state(state, mgr);
-	if (topology_state == NULL)
-		return -ENOMEM;
+	if (IS_ERR(topology_state))
+		return PTR_ERR(topology_state);
 
 	port = drm_dp_get_validated_port_ref(mgr, port);
 	if (port == NULL)
@@ -2580,8 +2584,8 @@ int drm_dp_atomic_release_vcpi_slots(struct drm_atomic_state *state,
 	struct drm_dp_mst_topology_state *topology_state;
 
 	topology_state = drm_atomic_get_mst_topology_state(state, mgr);
-	if (topology_state == NULL)
-		return -ENOMEM;
+	if (IS_ERR(topology_state))
+		return PTR_ERR(topology_state);
 
 	/* We cannot rely on port->vcpi.num_slots to update
 	 * topology_state->avail_slots as the port may not exist if the parent
@@ -3017,41 +3021,32 @@ static void drm_dp_destroy_connector_work(struct work_struct *work)
 		(*mgr->cbs->hotplug)(mgr);
 }
 
-void *drm_dp_mst_duplicate_state(struct drm_atomic_state *state, void *obj)
+static struct drm_private_state *
+drm_dp_mst_duplicate_state(struct drm_private_obj *obj)
 {
-	struct drm_dp_mst_topology_mgr *mgr = obj;
-	struct drm_dp_mst_topology_state *new_mst_state;
+	struct drm_dp_mst_topology_state *state;
 
-	if (WARN_ON(!mgr->state))
+	state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
+	if (!state)
 		return NULL;
 
-	new_mst_state = kmemdup(mgr->state, sizeof(*new_mst_state), GFP_KERNEL);
-	if (new_mst_state)
-		new_mst_state->state = state;
-	return new_mst_state;
-}
-
-void drm_dp_mst_swap_state(void *obj, void **obj_state_ptr)
-{
-	struct drm_dp_mst_topology_mgr *mgr = obj;
-	struct drm_dp_mst_topology_state **topology_state_ptr;
-
-	topology_state_ptr = (struct drm_dp_mst_topology_state **)obj_state_ptr;
+	__drm_atomic_helper_private_obj_duplicate_state(obj, &state->base);
 
-	mgr->state->state = (*topology_state_ptr)->state;
-	swap(*topology_state_ptr, mgr->state);
-	mgr->state->state = NULL;
+	return &state->base;
 }
 
-void drm_dp_mst_destroy_state(void *obj_state)
+static void drm_dp_mst_destroy_state(struct drm_private_obj *obj,
+				     struct drm_private_state *state)
 {
-	kfree(obj_state);
+	struct drm_dp_mst_topology_state *mst_state =
+		to_dp_mst_topology_state(state);
+
+	kfree(mst_state);
 }
 
 static const struct drm_private_state_funcs mst_state_funcs = {
-	.duplicate_state = drm_dp_mst_duplicate_state,
-	.swap_state = drm_dp_mst_swap_state,
-	.destroy_state = drm_dp_mst_destroy_state,
+	.atomic_duplicate_state = drm_dp_mst_duplicate_state,
+	.atomic_destroy_state = drm_dp_mst_destroy_state,
 };
 
 /**
@@ -3075,8 +3070,7 @@ struct drm_dp_mst_topology_state *drm_atomic_get_mst_topology_state(struct drm_a
 	struct drm_device *dev = mgr->dev;
 
 	WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex));
-	return drm_atomic_get_private_obj_state(state, mgr,
-						&mst_state_funcs);
+	return to_dp_mst_topology_state(drm_atomic_get_private_obj_state(state, &mgr->base));
 }
 EXPORT_SYMBOL(drm_atomic_get_mst_topology_state);
 
@@ -3096,6 +3090,8 @@ int drm_dp_mst_topology_mgr_init(struct drm_dp_mst_topology_mgr *mgr,
 				 int max_dpcd_transaction_bytes,
 				 int max_payloads, int conn_base_id)
 {
+	struct drm_dp_mst_topology_state *mst_state;
+
 	mutex_init(&mgr->lock);
 	mutex_init(&mgr->qlock);
 	mutex_init(&mgr->payload_lock);
@@ -3124,14 +3120,18 @@ int drm_dp_mst_topology_mgr_init(struct drm_dp_mst_topology_mgr *mgr,
 	if (test_calc_pbn_mode() < 0)
 		DRM_ERROR("MST PBN self-test failed\n");
 
-	mgr->state = kzalloc(sizeof(*mgr->state), GFP_KERNEL);
-	if (mgr->state == NULL)
+	mst_state = kzalloc(sizeof(*mst_state), GFP_KERNEL);
+	if (mst_state == NULL)
 		return -ENOMEM;
-	mgr->state->mgr = mgr;
+
+	mst_state->mgr = mgr;
 
 	/* max. time slots - one slot for MTP header */
-	mgr->state->avail_slots = 63;
-	mgr->funcs = &mst_state_funcs;
+	mst_state->avail_slots = 63;
+
+	drm_atomic_private_obj_init(&mgr->base,
+				    &mst_state->base,
+				    &mst_state_funcs);
 
 	return 0;
 }
@@ -3153,8 +3153,7 @@ void drm_dp_mst_topology_mgr_destroy(struct drm_dp_mst_topology_mgr *mgr)
 	mutex_unlock(&mgr->payload_lock);
 	mgr->dev = NULL;
 	mgr->aux = NULL;
-	kfree(mgr->state);
-	mgr->state = NULL;
+	drm_atomic_private_obj_fini(&mgr->base);
 	mgr->funcs = NULL;
 }
 EXPORT_SYMBOL(drm_dp_mst_topology_mgr_destroy);
diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index 37b8ad3e30d8..be38ac7050d4 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -63,6 +63,15 @@ module_param_named(debug, drm_debug, int, 0600);
 static DEFINE_SPINLOCK(drm_minor_lock);
 static struct idr drm_minors_idr;
 
+/*
+ * If the drm core fails to init for whatever reason,
+ * we should prevent any drivers from registering with it.
+ * It's best to check this at drm_dev_init(), as some drivers
+ * prefer to embed struct drm_device into their own device
+ * structure and call drm_dev_init() themselves.
+ */
+static bool drm_core_init_complete = false;
+
 static struct dentry *drm_debugfs_root;
 
 #define DRM_PRINTK_FMT "[" DRM_NAME ":%s]%s %pV"
@@ -282,7 +291,7 @@ struct drm_minor *drm_minor_acquire(unsigned int minor_id)
 
 	if (!minor) {
 		return ERR_PTR(-ENODEV);
-	} else if (drm_device_is_unplugged(minor->dev)) {
+	} else if (drm_dev_is_unplugged(minor->dev)) {
 		drm_dev_unref(minor->dev);
 		return ERR_PTR(-ENODEV);
 	}
@@ -355,26 +364,32 @@ void drm_put_dev(struct drm_device *dev)
 }
 EXPORT_SYMBOL(drm_put_dev);
 
-void drm_unplug_dev(struct drm_device *dev)
+static void drm_device_set_unplugged(struct drm_device *dev)
 {
-	/* for a USB device */
-	if (drm_core_check_feature(dev, DRIVER_MODESET))
-		drm_modeset_unregister_all(dev);
+	smp_wmb();
+	atomic_set(&dev->unplugged, 1);
+}
 
-	drm_minor_unregister(dev, DRM_MINOR_PRIMARY);
-	drm_minor_unregister(dev, DRM_MINOR_RENDER);
-	drm_minor_unregister(dev, DRM_MINOR_CONTROL);
+/**
+ * drm_dev_unplug - unplug a DRM device
+ * @dev: DRM device
+ *
+ * This unplugs a hotpluggable DRM device, which makes it inaccessible to
+ * userspace operations. Entry-points can use drm_dev_is_unplugged(). This
+ * essentially unregisters the device like drm_dev_unregister(), but can be
+ * called while there are still open users of @dev.
+ */
+void drm_dev_unplug(struct drm_device *dev)
+{
+	drm_dev_unregister(dev);
 
 	mutex_lock(&drm_global_mutex);
-
 	drm_device_set_unplugged(dev);
-
-	if (dev->open_count == 0) {
-		drm_put_dev(dev);
-	}
+	if (dev->open_count == 0)
+		drm_dev_unref(dev);
 	mutex_unlock(&drm_global_mutex);
 }
-EXPORT_SYMBOL(drm_unplug_dev);
+EXPORT_SYMBOL(drm_dev_unplug);
 
 /*
  * DRM internal mount
@@ -484,6 +499,11 @@ int drm_dev_init(struct drm_device *dev,
 {
 	int ret;
 
+	if (!drm_core_init_complete) {
+		DRM_ERROR("DRM core is not initialized\n");
+		return -ENODEV;
+	}
+
 	kref_init(&dev->ref);
 	dev->dev = parent;
 	dev->driver = driver;
@@ -821,6 +841,9 @@ EXPORT_SYMBOL(drm_dev_register);
  * drm_dev_register() but does not deallocate the device. The caller must call
  * drm_dev_unref() to drop their final reference.
  *
+ * A special form of unregistering for hotpluggable devices is drm_dev_unplug(),
+ * which can be called while there are still open users of @dev.
+ *
  * This should be called first in the device teardown code to make sure
  * userspace can't access the device instance any more.
  */
@@ -828,7 +851,8 @@ void drm_dev_unregister(struct drm_device *dev)
 {
 	struct drm_map_list *r_list, *list_temp;
 
-	drm_lastclose(dev);
+	if (drm_core_check_feature(dev, DRIVER_LEGACY))
+		drm_lastclose(dev);
 
 	dev->registered = false;
 
@@ -966,6 +990,8 @@ static int __init drm_core_init(void)
 	if (ret < 0)
 		goto error;
 
+	drm_core_init_complete = true;
+
 	DRM_DEBUG("Initialized\n");
 	return 0;
 
diff --git a/drivers/gpu/drm/drm_dumb_buffers.c b/drivers/gpu/drm/drm_dumb_buffers.c
index 10307cc16d75..39ac15ce4702 100644
--- a/drivers/gpu/drm/drm_dumb_buffers.c
+++ b/drivers/gpu/drm/drm_dumb_buffers.c
@@ -24,6 +24,7 @@
  */
 
 #include <drm/drmP.h>
+#include <drm/drm_gem.h>
 
 #include "drm_crtc_internal.h"
 
@@ -42,9 +43,10 @@
  * create dumb buffers suitable for scanout, which can then be used to create
  * KMS frame buffers.
  *
- * To support dumb objects drivers must implement the &drm_driver.dumb_create,
- * &drm_driver.dumb_destroy and &drm_driver.dumb_map_offset operations. See
- * there for further details.
+ * To support dumb objects drivers must implement the &drm_driver.dumb_create
+ * operation. &drm_driver.dumb_destroy defaults to drm_gem_dumb_destroy() if
+ * not set and &drm_driver.dumb_map_offset defaults to
+ * drm_gem_dumb_map_offset(). See the callbacks for further details.
  *
  * Note that dumb objects may not be used for gpu acceleration, as has been
  * attempted on some ARM embedded platforms. Such drivers really must have
@@ -108,11 +110,16 @@ int drm_mode_mmap_dumb_ioctl(struct drm_device *dev,
 {
 	struct drm_mode_map_dumb *args = data;
 
-	/* call driver ioctl to get mmap offset */
-	if (!dev->driver->dumb_map_offset)
+	if (!dev->driver->dumb_create)
 		return -ENOSYS;
 
-	return dev->driver->dumb_map_offset(file_priv, dev, args->handle, &args->offset);
+	if (dev->driver->dumb_map_offset)
+		return dev->driver->dumb_map_offset(file_priv, dev,
+						    args->handle,
+						    &args->offset);
+	else
+		return drm_gem_dumb_map_offset(file_priv, dev, args->handle,
+					       &args->offset);
 }
 
 int drm_mode_destroy_dumb_ioctl(struct drm_device *dev,
@@ -120,9 +127,12 @@ int drm_mode_destroy_dumb_ioctl(struct drm_device *dev,
 {
 	struct drm_mode_destroy_dumb *args = data;
 
-	if (!dev->driver->dumb_destroy)
+	if (!dev->driver->dumb_create)
 		return -ENOSYS;
 
-	return dev->driver->dumb_destroy(file_priv, dev, args->handle);
+	if (dev->driver->dumb_destroy)
+		return dev->driver->dumb_destroy(file_priv, dev, args->handle);
+	else
+		return drm_gem_dumb_destroy(file_priv, dev, args->handle);
 }
 
diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c
index 2e55599816aa..6bb6337be920 100644
--- a/drivers/gpu/drm/drm_edid.c
+++ b/drivers/gpu/drm/drm_edid.c
@@ -1006,6 +1006,221 @@ static const struct drm_display_mode edid_cea_modes[] = {
 		   2492, 2640, 0, 1080, 1084, 1089, 1125, 0,
 		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
 	 .vrefresh = 100, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, },
+	/* 65 - 1280x720@24Hz */
+	{ DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 59400, 1280, 3040,
+		   3080, 3300, 0, 720, 725, 730, 750, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 24, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 66 - 1280x720@25Hz */
+	{ DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 3700,
+		   3740, 3960, 0, 720, 725, 730, 750, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 25, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 67 - 1280x720@30Hz */
+	{ DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 3040,
+		   3080, 3300, 0, 720, 725, 730, 750, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 30, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 68 - 1280x720@50Hz */
+	{ DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 1720,
+		   1760, 1980, 0, 720, 725, 730, 750, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 50, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 69 - 1280x720@60Hz */
+	{ DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 1390,
+		   1430, 1650, 0, 720, 725, 730, 750, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 60, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 70 - 1280x720@100Hz */
+	{ DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 148500, 1280, 1720,
+		   1760, 1980, 0, 720, 725, 730, 750, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 100, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 71 - 1280x720@120Hz */
+	{ DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 148500, 1280, 1390,
+		   1430, 1650, 0, 720, 725, 730, 750, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 120, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 72 - 1920x1080@24Hz */
+	{ DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2558,
+		   2602, 2750, 0, 1080, 1084, 1089, 1125, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 24, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 73 - 1920x1080@25Hz */
+	{ DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2448,
+		   2492, 2640, 0, 1080, 1084, 1089, 1125, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 25, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 74 - 1920x1080@30Hz */
+	{ DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2008,
+		   2052, 2200, 0, 1080, 1084, 1089, 1125, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 30, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 75 - 1920x1080@50Hz */
+	{ DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2448,
+		   2492, 2640, 0, 1080, 1084, 1089, 1125, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 50, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 76 - 1920x1080@60Hz */
+	{ DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2008,
+		   2052, 2200, 0, 1080, 1084, 1089, 1125, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 60, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 77 - 1920x1080@100Hz */
+	{ DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 297000, 1920, 2448,
+		   2492, 2640, 0, 1080, 1084, 1089, 1125, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 100, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 78 - 1920x1080@120Hz */
+	{ DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 297000, 1920, 2008,
+		   2052, 2200, 0, 1080, 1084, 1089, 1125, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 120, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 79 - 1680x720@24Hz */
+	{ DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 59400, 1680, 3040,
+		   3080, 3300, 0, 720, 725, 730, 750, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 24, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 80 - 1680x720@25Hz */
+	{ DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 59400, 1680, 2908,
+		   2948, 3168, 0, 720, 725, 730, 750, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 25, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 81 - 1680x720@30Hz */
+	{ DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 59400, 1680, 2380,
+		   2420, 2640, 0, 720, 725, 730, 750, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 30, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 82 - 1680x720@50Hz */
+	{ DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 82500, 1680, 1940,
+		   1980, 2200, 0, 720, 725, 730, 750, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 50, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 83 - 1680x720@60Hz */
+	{ DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 99000, 1680, 1940,
+		   1980, 2200, 0, 720, 725, 730, 750, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 60, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 84 - 1680x720@100Hz */
+	{ DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 165000, 1680, 1740,
+		   1780, 2000, 0, 720, 725, 730, 825, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 100, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 85 - 1680x720@120Hz */
+	{ DRM_MODE("1680x720", DRM_MODE_TYPE_DRIVER, 198000, 1680, 1740,
+		   1780, 2000, 0, 720, 725, 730, 825, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 120, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 86 - 2560x1080@24Hz */
+	{ DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 99000, 2560, 3558,
+		   3602, 3750, 0, 1080, 1084, 1089, 1100, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 24, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 87 - 2560x1080@25Hz */
+	{ DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 90000, 2560, 3008,
+		   3052, 3200, 0, 1080, 1084, 1089, 1125, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 25, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 88 - 2560x1080@30Hz */
+	{ DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 118800, 2560, 3328,
+		   3372, 3520, 0, 1080, 1084, 1089, 1125, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 30, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 89 - 2560x1080@50Hz */
+	{ DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 185625, 2560, 3108,
+		   3152, 3300, 0, 1080, 1084, 1089, 1125, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 50, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 90 - 2560x1080@60Hz */
+	{ DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 198000, 2560, 2808,
+		   2852, 3000, 0, 1080, 1084, 1089, 1100, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 60, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 91 - 2560x1080@100Hz */
+	{ DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 371250, 2560, 2778,
+		   2822, 2970, 0, 1080, 1084, 1089, 1250, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 100, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 92 - 2560x1080@120Hz */
+	{ DRM_MODE("2560x1080", DRM_MODE_TYPE_DRIVER, 495000, 2560, 3108,
+		   3152, 3300, 0, 1080, 1084, 1089, 1250, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 120, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 93 - 3840x2160p@24Hz 16:9 */
+	{ DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 297000, 3840, 5116,
+		   5204, 5500, 0, 2160, 2168, 2178, 2250, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 24, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, },
+	/* 94 - 3840x2160p@25Hz 16:9 */
+	{ DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 297000, 3840, 4896,
+		   4984, 5280, 0, 2160, 2168, 2178, 2250, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 25, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, },
+	/* 95 - 3840x2160p@30Hz 16:9 */
+	{ DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 297000, 3840, 4016,
+		   4104, 4400, 0, 2160, 2168, 2178, 2250, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 30, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, },
+	/* 96 - 3840x2160p@50Hz 16:9 */
+	{ DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 594000, 3840, 4896,
+		   4984, 5280, 0, 2160, 2168, 2178, 2250, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 50, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, },
+	/* 97 - 3840x2160p@60Hz 16:9 */
+	{ DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 594000, 3840, 4016,
+		   4104, 4400, 0, 2160, 2168, 2178, 2250, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 60, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9, },
+	/* 98 - 4096x2160p@24Hz 256:135 */
+	{ DRM_MODE("4096x2160", DRM_MODE_TYPE_DRIVER, 297000, 4096, 5116,
+		   5204, 5500, 0, 2160, 2168, 2178, 2250, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 24, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_256_135, },
+	/* 99 - 4096x2160p@25Hz 256:135 */
+	{ DRM_MODE("4096x2160", DRM_MODE_TYPE_DRIVER, 297000, 4096, 5064,
+		   5152, 5280, 0, 2160, 2168, 2178, 2250, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 25, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_256_135, },
+	/* 100 - 4096x2160p@30Hz 256:135 */
+	{ DRM_MODE("4096x2160", DRM_MODE_TYPE_DRIVER, 297000, 4096, 4184,
+		   4272, 4400, 0, 2160, 2168, 2178, 2250, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 30, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_256_135, },
+	/* 101 - 4096x2160p@50Hz 256:135 */
+	{ DRM_MODE("4096x2160", DRM_MODE_TYPE_DRIVER, 594000, 4096, 5064,
+		   5152, 5280, 0, 2160, 2168, 2178, 2250, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 50, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_256_135, },
+	/* 102 - 4096x2160p@60Hz 256:135 */
+	{ DRM_MODE("4096x2160", DRM_MODE_TYPE_DRIVER, 594000, 4096, 4184,
+		   4272, 4400, 0, 2160, 2168, 2178, 2250, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 60, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_256_135, },
+	/* 103 - 3840x2160p@24Hz 64:27 */
+	{ DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 297000, 3840, 5116,
+		   5204, 5500, 0, 2160, 2168, 2178, 2250, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 24, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 104 - 3840x2160p@25Hz 64:27 */
+	{ DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 297000, 3840, 4896,
+		   4984, 5280, 0, 2160, 2168, 2178, 2250, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 25, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 105 - 3840x2160p@30Hz 64:27 */
+	{ DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 297000, 3840, 4016,
+		   4104, 4400, 0, 2160, 2168, 2178, 2250, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 30, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 106 - 3840x2160p@50Hz 64:27 */
+	{ DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 594000, 3840, 4896,
+		   4984, 5280, 0, 2160, 2168, 2178, 2250, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 50, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
+	/* 107 - 3840x2160p@60Hz 64:27 */
+	{ DRM_MODE("3840x2160", DRM_MODE_TYPE_DRIVER, 594000, 3840, 4016,
+		   4104, 4400, 0, 2160, 2168, 2178, 2250, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC),
+	  .vrefresh = 60, .picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27, },
 };
 
 /*
@@ -2566,7 +2781,10 @@ add_detailed_modes(struct drm_connector *connector, struct edid *edid,
 #define VIDEO_BLOCK     0x02
 #define VENDOR_BLOCK    0x03
 #define SPEAKER_BLOCK	0x04
-#define VIDEO_CAPABILITY_BLOCK	0x07
+#define USE_EXTENDED_TAG 0x07
+#define EXT_VIDEO_CAPABILITY_BLOCK 0x00
+#define EXT_VIDEO_DATA_BLOCK_420	0x0E
+#define EXT_VIDEO_CAP_BLOCK_Y420CMDB 0x0F
 #define EDID_BASIC_AUDIO	(1 << 6)
 #define EDID_CEA_YCRCB444	(1 << 5)
 #define EDID_CEA_YCRCB422	(1 << 4)
@@ -2902,6 +3120,15 @@ add_alternate_cea_modes(struct drm_connector *connector, struct edid *edid)
 	return modes;
 }
 
+static u8 svd_to_vic(u8 svd)
+{
+	/* 0-6 bit vic, 7th bit native mode indicator */
+	if ((svd >= 1 &&  svd <= 64) || (svd >= 129 && svd <= 192))
+		return svd & 127;
+
+	return svd;
+}
+
 static struct drm_display_mode *
 drm_display_mode_from_vic_index(struct drm_connector *connector,
 				const u8 *video_db, u8 video_len,
@@ -2915,7 +3142,7 @@ drm_display_mode_from_vic_index(struct drm_connector *connector,
 		return NULL;
 
 	/* CEA modes are numbered 1..127 */
-	vic = (video_db[video_index] & 127);
+	vic = svd_to_vic(video_db[video_index]);
 	if (!drm_valid_cea_vic(vic))
 		return NULL;
 
@@ -2928,15 +3155,85 @@ drm_display_mode_from_vic_index(struct drm_connector *connector,
 	return newmode;
 }
 
+/*
+ * do_y420vdb_modes - Parse YCBCR 420 only modes
+ * @connector: connector corresponding to the HDMI sink
+ * @svds: start of the data block of CEA YCBCR 420 VDB
+ * @len: length of the CEA YCBCR 420 VDB
+ *
+ * Parse the CEA-861-F YCBCR 420 Video Data Block (Y420VDB)
+ * which contains modes which can be supported in YCBCR 420
+ * output format only.
+ */
+static int do_y420vdb_modes(struct drm_connector *connector,
+			    const u8 *svds, u8 svds_len)
+{
+	int modes = 0, i;
+	struct drm_device *dev = connector->dev;
+	struct drm_display_info *info = &connector->display_info;
+	struct drm_hdmi_info *hdmi = &info->hdmi;
+
+	for (i = 0; i < svds_len; i++) {
+		u8 vic = svd_to_vic(svds[i]);
+		struct drm_display_mode *newmode;
+
+		if (!drm_valid_cea_vic(vic))
+			continue;
+
+		newmode = drm_mode_duplicate(dev, &edid_cea_modes[vic]);
+		if (!newmode)
+			break;
+		bitmap_set(hdmi->y420_vdb_modes, vic, 1);
+		drm_mode_probed_add(connector, newmode);
+		modes++;
+	}
+
+	if (modes > 0)
+		info->color_formats |= DRM_COLOR_FORMAT_YCRCB420;
+	return modes;
+}
+
+/*
+ * drm_add_cmdb_modes - Add a YCBCR 420 mode into bitmap
+ * @connector: connector corresponding to the HDMI sink
+ * @vic: CEA vic for the video mode to be added in the map
+ *
+ * Makes an entry for a videomode in the YCBCR 420 bitmap
+ */
+static void
+drm_add_cmdb_modes(struct drm_connector *connector, u8 svd)
+{
+	u8 vic = svd_to_vic(svd);
+	struct drm_hdmi_info *hdmi = &connector->display_info.hdmi;
+
+	if (!drm_valid_cea_vic(vic))
+		return;
+
+	bitmap_set(hdmi->y420_cmdb_modes, vic, 1);
+}
+
 static int
 do_cea_modes(struct drm_connector *connector, const u8 *db, u8 len)
 {
 	int i, modes = 0;
+	struct drm_hdmi_info *hdmi = &connector->display_info.hdmi;
 
 	for (i = 0; i < len; i++) {
 		struct drm_display_mode *mode;
 		mode = drm_display_mode_from_vic_index(connector, db, len, i);
 		if (mode) {
+			/*
+			 * YCBCR420 capability block contains a bitmap which
+			 * gives the index of CEA modes from CEA VDB, which
+			 * can support YCBCR 420 sampling output also (apart
+			 * from RGB/YCBCR444 etc).
+			 * For example, if the bit 0 in bitmap is set,
+			 * first mode in VDB can support YCBCR420 output too.
+			 * Add YCBCR420 modes only if sink is HDMI 2.0 capable.
+			 */
+			if (i < 64 && hdmi->y420_cmdb_map & (1ULL << i))
+				drm_add_cmdb_modes(connector, db[i]);
+
 			drm_mode_probed_add(connector, mode);
 			modes++;
 		}
@@ -3218,6 +3515,12 @@ cea_db_payload_len(const u8 *db)
 }
 
 static int
+cea_db_extended_tag(const u8 *db)
+{
+	return db[1];
+}
+
+static int
 cea_db_tag(const u8 *db)
 {
 	return db[0] >> 5;
@@ -3272,9 +3575,77 @@ static bool cea_db_is_hdmi_forum_vsdb(const u8 *db)
 	return oui == HDMI_FORUM_IEEE_OUI;
 }
 
+static bool cea_db_is_y420cmdb(const u8 *db)
+{
+	if (cea_db_tag(db) != USE_EXTENDED_TAG)
+		return false;
+
+	if (!cea_db_payload_len(db))
+		return false;
+
+	if (cea_db_extended_tag(db) != EXT_VIDEO_CAP_BLOCK_Y420CMDB)
+		return false;
+
+	return true;
+}
+
+static bool cea_db_is_y420vdb(const u8 *db)
+{
+	if (cea_db_tag(db) != USE_EXTENDED_TAG)
+		return false;
+
+	if (!cea_db_payload_len(db))
+		return false;
+
+	if (cea_db_extended_tag(db) != EXT_VIDEO_DATA_BLOCK_420)
+		return false;
+
+	return true;
+}
+
 #define for_each_cea_db(cea, i, start, end) \
 	for ((i) = (start); (i) < (end) && (i) + cea_db_payload_len(&(cea)[(i)]) < (end); (i) += cea_db_payload_len(&(cea)[(i)]) + 1)
 
+static void drm_parse_y420cmdb_bitmap(struct drm_connector *connector,
+				      const u8 *db)
+{
+	struct drm_display_info *info = &connector->display_info;
+	struct drm_hdmi_info *hdmi = &info->hdmi;
+	u8 map_len = cea_db_payload_len(db) - 1;
+	u8 count;
+	u64 map = 0;
+
+	if (map_len == 0) {
+		/* All CEA modes support ycbcr420 sampling also.*/
+		hdmi->y420_cmdb_map = U64_MAX;
+		info->color_formats |= DRM_COLOR_FORMAT_YCRCB420;
+		return;
+	}
+
+	/*
+	 * This map indicates which of the existing CEA block modes
+	 * from VDB can support YCBCR420 output too. So if bit=0 is
+	 * set, first mode from VDB can support YCBCR420 output too.
+	 * We will parse and keep this map, before parsing VDB itself
+	 * to avoid going through the same block again and again.
+	 *
+	 * Spec is not clear about max possible size of this block.
+	 * Clamping max bitmap block size at 8 bytes. Every byte can
+	 * address 8 CEA modes, in this way this map can address
+	 * 8*8 = first 64 SVDs.
+	 */
+	if (WARN_ON_ONCE(map_len > 8))
+		map_len = 8;
+
+	for (count = 0; count < map_len; count++)
+		map |= (u64)db[2 + count] << (8 * count);
+
+	if (map)
+		info->color_formats |= DRM_COLOR_FORMAT_YCRCB420;
+
+	hdmi->y420_cmdb_map = map;
+}
+
 static int
 add_cea_modes(struct drm_connector *connector, struct edid *edid)
 {
@@ -3297,10 +3668,16 @@ add_cea_modes(struct drm_connector *connector, struct edid *edid)
 				video = db + 1;
 				video_len = dbl;
 				modes += do_cea_modes(connector, video, dbl);
-			}
-			else if (cea_db_is_hdmi_vsdb(db)) {
+			} else if (cea_db_is_hdmi_vsdb(db)) {
 				hdmi = db;
 				hdmi_len = dbl;
+			} else if (cea_db_is_y420vdb(db)) {
+				const u8 *vdb420 = &db[2];
+
+				/* Add 4:2:0(only) modes present in EDID */
+				modes += do_y420vdb_modes(connector,
+							  vdb420,
+							  dbl - 1);
 			}
 		}
 	}
@@ -3793,8 +4170,10 @@ bool drm_rgb_quant_range_selectable(struct edid *edid)
 		return false;
 
 	for_each_cea_db(edid_ext, i, start, end) {
-		if (cea_db_tag(&edid_ext[i]) == VIDEO_CAPABILITY_BLOCK &&
-		    cea_db_payload_len(&edid_ext[i]) == 2) {
+		if (cea_db_tag(&edid_ext[i]) == USE_EXTENDED_TAG &&
+		    cea_db_payload_len(&edid_ext[i]) == 2 &&
+		    cea_db_extended_tag(&edid_ext[i]) ==
+			EXT_VIDEO_CAPABILITY_BLOCK) {
 			DRM_DEBUG_KMS("CEA VCDB 0x%02x\n", edid_ext[i + 2]);
 			return edid_ext[i + 2] & EDID_CEA_VCDB_QS;
 		}
@@ -3823,6 +4202,16 @@ drm_default_rgb_quant_range(const struct drm_display_mode *mode)
 }
 EXPORT_SYMBOL(drm_default_rgb_quant_range);
 
+static void drm_parse_ycbcr420_deep_color_info(struct drm_connector *connector,
+					       const u8 *db)
+{
+	u8 dc_mask;
+	struct drm_hdmi_info *hdmi = &connector->display_info.hdmi;
+
+	dc_mask = db[7] & DRM_EDID_YCBCR420_DC_MASK;
+	hdmi->y420_dc_modes |= dc_mask;
+}
+
 static void drm_parse_hdmi_forum_vsdb(struct drm_connector *connector,
 				 const u8 *hf_vsdb)
 {
@@ -3863,6 +4252,8 @@ static void drm_parse_hdmi_forum_vsdb(struct drm_connector *connector,
 				scdc->scrambling.low_rates = true;
 		}
 	}
+
+	drm_parse_ycbcr420_deep_color_info(connector, hf_vsdb);
 }
 
 static void drm_parse_hdmi_deep_color_info(struct drm_connector *connector,
@@ -3981,6 +4372,8 @@ static void drm_parse_cea_ext(struct drm_connector *connector,
 			drm_parse_hdmi_vsdb_video(connector, db);
 		if (cea_db_is_hdmi_forum_vsdb(db))
 			drm_parse_hdmi_forum_vsdb(connector, db);
+		if (cea_db_is_y420cmdb(db))
+			drm_parse_y420cmdb_bitmap(connector, db);
 	}
 }
 
@@ -4215,6 +4608,13 @@ int drm_add_edid_modes(struct drm_connector *connector, struct edid *edid)
 	quirks = edid_get_quirks(edid);
 
 	/*
+	 * CEA-861-F adds ycbcr capability map block, for HDMI 2.0 sinks.
+	 * To avoid multiple parsing of same block, lets parse that map
+	 * from sink info, before parsing CEA modes.
+	 */
+	drm_add_display_info(connector, edid);
+
+	/*
 	 * EDID spec says modes should be preferred in this order:
 	 * - preferred detailed mode
 	 * - other detailed modes from base block
@@ -4241,8 +4641,6 @@ int drm_add_edid_modes(struct drm_connector *connector, struct edid *edid)
 	if (quirks & (EDID_QUIRK_PREFER_LARGE_60 | EDID_QUIRK_PREFER_LARGE_75))
 		edid_fixup_preferred(connector, quirks);
 
-	drm_add_display_info(connector, edid);
-
 	if (quirks & EDID_QUIRK_FORCE_6BPC)
 		connector->display_info.bpc = 6;
 
@@ -4334,12 +4732,14 @@ EXPORT_SYMBOL(drm_set_preferred_mode);
  *                                              data from a DRM display mode
  * @frame: HDMI AVI infoframe
  * @mode: DRM display mode
+ * @is_hdmi2_sink: Sink is HDMI 2.0 compliant
  *
  * Return: 0 on success or a negative error code on failure.
  */
 int
 drm_hdmi_avi_infoframe_from_display_mode(struct hdmi_avi_infoframe *frame,
-					 const struct drm_display_mode *mode)
+					 const struct drm_display_mode *mode,
+					 bool is_hdmi2_sink)
 {
 	int err;
 
@@ -4355,6 +4755,28 @@ drm_hdmi_avi_infoframe_from_display_mode(struct hdmi_avi_infoframe *frame,
 
 	frame->video_code = drm_match_cea_mode(mode);
 
+	/*
+	 * HDMI 1.4 VIC range: 1 <= VIC <= 64 (CEA-861-D) but
+	 * HDMI 2.0 VIC range: 1 <= VIC <= 107 (CEA-861-F). So we
+	 * have to make sure we dont break HDMI 1.4 sinks.
+	 */
+	if (!is_hdmi2_sink && frame->video_code > 64)
+		frame->video_code = 0;
+
+	/*
+	 * HDMI spec says if a mode is found in HDMI 1.4b 4K modes
+	 * we should send its VIC in vendor infoframes, else send the
+	 * VIC in AVI infoframes. Lets check if this mode is present in
+	 * HDMI 1.4b 4K modes
+	 */
+	if (frame->video_code) {
+		u8 vendor_if_vic = drm_match_hdmi_mode(mode);
+		bool is_s3d = mode->flags & DRM_MODE_FLAG_3D_MASK;
+
+		if (drm_valid_hdmi_vic(vendor_if_vic) && !is_s3d)
+			frame->video_code = 0;
+	}
+
 	frame->picture_aspect = HDMI_PICTURE_ASPECT_NONE;
 
 	/*
diff --git a/drivers/gpu/drm/drm_fb_cma_helper.c b/drivers/gpu/drm/drm_fb_cma_helper.c
index 53f9bdf470d7..f2ee88363015 100644
--- a/drivers/gpu/drm/drm_fb_cma_helper.c
+++ b/drivers/gpu/drm/drm_fb_cma_helper.c
@@ -18,27 +18,17 @@
  */
 
 #include <drm/drmP.h>
-#include <drm/drm_atomic.h>
-#include <drm/drm_crtc.h>
 #include <drm/drm_fb_helper.h>
-#include <drm/drm_crtc_helper.h>
+#include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_fb_cma_helper.h>
-#include <linux/dma-buf.h>
-#include <linux/dma-mapping.h>
 #include <linux/module.h>
-#include <linux/reservation.h>
 
 #define DEFAULT_FBDEFIO_DELAY_MS 50
 
-struct drm_fb_cma {
-	struct drm_framebuffer		fb;
-	struct drm_gem_cma_object	*obj[4];
-};
-
 struct drm_fbdev_cma {
 	struct drm_fb_helper	fb_helper;
-	struct drm_fb_cma	*fb;
 	const struct drm_framebuffer_funcs *fb_funcs;
 };
 
@@ -90,69 +80,19 @@ static inline struct drm_fbdev_cma *to_fbdev_cma(struct drm_fb_helper *helper)
 	return container_of(helper, struct drm_fbdev_cma, fb_helper);
 }
 
-static inline struct drm_fb_cma *to_fb_cma(struct drm_framebuffer *fb)
-{
-	return container_of(fb, struct drm_fb_cma, fb);
-}
-
 void drm_fb_cma_destroy(struct drm_framebuffer *fb)
 {
-	struct drm_fb_cma *fb_cma = to_fb_cma(fb);
-	int i;
-
-	for (i = 0; i < 4; i++) {
-		if (fb_cma->obj[i])
-			drm_gem_object_put_unlocked(&fb_cma->obj[i]->base);
-	}
-
-	drm_framebuffer_cleanup(fb);
-	kfree(fb_cma);
+	drm_gem_fb_destroy(fb);
 }
 EXPORT_SYMBOL(drm_fb_cma_destroy);
 
 int drm_fb_cma_create_handle(struct drm_framebuffer *fb,
 	struct drm_file *file_priv, unsigned int *handle)
 {
-	struct drm_fb_cma *fb_cma = to_fb_cma(fb);
-
-	return drm_gem_handle_create(file_priv,
-			&fb_cma->obj[0]->base, handle);
+	return drm_gem_fb_create_handle(fb, file_priv, handle);
 }
 EXPORT_SYMBOL(drm_fb_cma_create_handle);
 
-static struct drm_framebuffer_funcs drm_fb_cma_funcs = {
-	.destroy	= drm_fb_cma_destroy,
-	.create_handle	= drm_fb_cma_create_handle,
-};
-
-static struct drm_fb_cma *drm_fb_cma_alloc(struct drm_device *dev,
-	const struct drm_mode_fb_cmd2 *mode_cmd,
-	struct drm_gem_cma_object **obj,
-	unsigned int num_planes, const struct drm_framebuffer_funcs *funcs)
-{
-	struct drm_fb_cma *fb_cma;
-	int ret;
-	int i;
-
-	fb_cma = kzalloc(sizeof(*fb_cma), GFP_KERNEL);
-	if (!fb_cma)
-		return ERR_PTR(-ENOMEM);
-
-	drm_helper_mode_fill_fb_struct(dev, &fb_cma->fb, mode_cmd);
-
-	for (i = 0; i < num_planes; i++)
-		fb_cma->obj[i] = obj[i];
-
-	ret = drm_framebuffer_init(dev, &fb_cma->fb, funcs);
-	if (ret) {
-		dev_err(dev->dev, "Failed to initialize framebuffer: %d\n", ret);
-		kfree(fb_cma);
-		return ERR_PTR(ret);
-	}
-
-	return fb_cma;
-}
-
 /**
  * drm_fb_cma_create_with_funcs() - helper function for the
  *                                  &drm_mode_config_funcs.fb_create
@@ -170,53 +110,7 @@ struct drm_framebuffer *drm_fb_cma_create_with_funcs(struct drm_device *dev,
 	struct drm_file *file_priv, const struct drm_mode_fb_cmd2 *mode_cmd,
 	const struct drm_framebuffer_funcs *funcs)
 {
-	const struct drm_format_info *info;
-	struct drm_fb_cma *fb_cma;
-	struct drm_gem_cma_object *objs[4];
-	struct drm_gem_object *obj;
-	int ret;
-	int i;
-
-	info = drm_get_format_info(dev, mode_cmd);
-	if (!info)
-		return ERR_PTR(-EINVAL);
-
-	for (i = 0; i < info->num_planes; i++) {
-		unsigned int width = mode_cmd->width / (i ? info->hsub : 1);
-		unsigned int height = mode_cmd->height / (i ? info->vsub : 1);
-		unsigned int min_size;
-
-		obj = drm_gem_object_lookup(file_priv, mode_cmd->handles[i]);
-		if (!obj) {
-			dev_err(dev->dev, "Failed to lookup GEM object\n");
-			ret = -ENOENT;
-			goto err_gem_object_put;
-		}
-
-		min_size = (height - 1) * mode_cmd->pitches[i]
-			 + width * info->cpp[i]
-			 + mode_cmd->offsets[i];
-
-		if (obj->size < min_size) {
-			drm_gem_object_put_unlocked(obj);
-			ret = -EINVAL;
-			goto err_gem_object_put;
-		}
-		objs[i] = to_drm_gem_cma_obj(obj);
-	}
-
-	fb_cma = drm_fb_cma_alloc(dev, mode_cmd, objs, i, funcs);
-	if (IS_ERR(fb_cma)) {
-		ret = PTR_ERR(fb_cma);
-		goto err_gem_object_put;
-	}
-
-	return &fb_cma->fb;
-
-err_gem_object_put:
-	for (i--; i >= 0; i--)
-		drm_gem_object_put_unlocked(&objs[i]->base);
-	return ERR_PTR(ret);
+	return drm_gem_fb_create_with_funcs(dev, file_priv, mode_cmd, funcs);
 }
 EXPORT_SYMBOL_GPL(drm_fb_cma_create_with_funcs);
 
@@ -233,8 +127,7 @@ EXPORT_SYMBOL_GPL(drm_fb_cma_create_with_funcs);
 struct drm_framebuffer *drm_fb_cma_create(struct drm_device *dev,
 	struct drm_file *file_priv, const struct drm_mode_fb_cmd2 *mode_cmd)
 {
-	return drm_fb_cma_create_with_funcs(dev, file_priv, mode_cmd,
-					    &drm_fb_cma_funcs);
+	return drm_gem_fb_create(dev, file_priv, mode_cmd);
 }
 EXPORT_SYMBOL_GPL(drm_fb_cma_create);
 
@@ -250,12 +143,13 @@ EXPORT_SYMBOL_GPL(drm_fb_cma_create);
 struct drm_gem_cma_object *drm_fb_cma_get_gem_obj(struct drm_framebuffer *fb,
 						  unsigned int plane)
 {
-	struct drm_fb_cma *fb_cma = to_fb_cma(fb);
+	struct drm_gem_object *gem;
 
-	if (plane >= 4)
+	gem = drm_gem_fb_get_obj(fb, plane);
+	if (!gem)
 		return NULL;
 
-	return fb_cma->obj[plane];
+	return to_drm_gem_cma_obj(gem);
 }
 EXPORT_SYMBOL_GPL(drm_fb_cma_get_gem_obj);
 
@@ -272,13 +166,14 @@ dma_addr_t drm_fb_cma_get_gem_addr(struct drm_framebuffer *fb,
 				   struct drm_plane_state *state,
 				   unsigned int plane)
 {
-	struct drm_fb_cma *fb_cma = to_fb_cma(fb);
+	struct drm_gem_cma_object *obj;
 	dma_addr_t paddr;
 
-	if (plane >= 4)
+	obj = drm_fb_cma_get_gem_obj(fb, plane);
+	if (!obj)
 		return 0;
 
-	paddr = fb_cma->obj[plane]->paddr + fb->offsets[plane];
+	paddr = obj->paddr + fb->offsets[plane];
 	paddr += fb->format->cpp[plane] * (state->src_x >> 16);
 	paddr += fb->pitches[plane] * (state->src_y >> 16);
 
@@ -302,26 +197,13 @@ EXPORT_SYMBOL_GPL(drm_fb_cma_get_gem_addr);
 int drm_fb_cma_prepare_fb(struct drm_plane *plane,
 			  struct drm_plane_state *state)
 {
-	struct dma_buf *dma_buf;
-	struct dma_fence *fence;
-
-	if ((plane->state->fb == state->fb) || !state->fb)
-		return 0;
-
-	dma_buf = drm_fb_cma_get_gem_obj(state->fb, 0)->base.dma_buf;
-	if (dma_buf) {
-		fence = reservation_object_get_excl_rcu(dma_buf->resv);
-		drm_atomic_set_fence_for_plane(state, fence);
-	}
-
-	return 0;
+	return drm_gem_fb_prepare_fb(plane, state);
 }
 EXPORT_SYMBOL_GPL(drm_fb_cma_prepare_fb);
 
 #ifdef CONFIG_DEBUG_FS
 static void drm_fb_cma_describe(struct drm_framebuffer *fb, struct seq_file *m)
 {
-	struct drm_fb_cma *fb_cma = to_fb_cma(fb);
 	int i;
 
 	seq_printf(m, "fb: %dx%d@%4.4s\n", fb->width, fb->height,
@@ -330,7 +212,7 @@ static void drm_fb_cma_describe(struct drm_framebuffer *fb, struct seq_file *m)
 	for (i = 0; i < fb->format->num_planes; i++) {
 		seq_printf(m, "   %d: offset=%d pitch=%d, obj: ",
 				i, fb->offsets[i], fb->pitches[i]);
-		drm_gem_cma_describe(fb_cma->obj[i], m);
+		drm_gem_cma_describe(drm_fb_cma_get_gem_obj(fb, i), m);
 	}
 }
 
@@ -431,7 +313,6 @@ drm_fbdev_cma_create(struct drm_fb_helper *helper,
 	struct drm_fb_helper_surface_size *sizes)
 {
 	struct drm_fbdev_cma *fbdev_cma = to_fbdev_cma(helper);
-	struct drm_mode_fb_cmd2 mode_cmd = { 0 };
 	struct drm_device *dev = helper->dev;
 	struct drm_gem_cma_object *obj;
 	struct drm_framebuffer *fb;
@@ -446,14 +327,7 @@ drm_fbdev_cma_create(struct drm_fb_helper *helper,
 			sizes->surface_bpp);
 
 	bytes_per_pixel = DIV_ROUND_UP(sizes->surface_bpp, 8);
-
-	mode_cmd.width = sizes->surface_width;
-	mode_cmd.height = sizes->surface_height;
-	mode_cmd.pitches[0] = sizes->surface_width * bytes_per_pixel;
-	mode_cmd.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp,
-		sizes->surface_depth);
-
-	size = mode_cmd.pitches[0] * mode_cmd.height;
+	size = sizes->surface_width * sizes->surface_height * bytes_per_pixel;
 	obj = drm_gem_cma_create(dev, size);
 	if (IS_ERR(obj))
 		return -ENOMEM;
@@ -464,15 +338,14 @@ drm_fbdev_cma_create(struct drm_fb_helper *helper,
 		goto err_gem_free_object;
 	}
 
-	fbdev_cma->fb = drm_fb_cma_alloc(dev, &mode_cmd, &obj, 1,
-					 fbdev_cma->fb_funcs);
-	if (IS_ERR(fbdev_cma->fb)) {
+	fb = drm_gem_fbdev_fb_create(dev, sizes, 0, &obj->base,
+				     fbdev_cma->fb_funcs);
+	if (IS_ERR(fb)) {
 		dev_err(dev->dev, "Failed to allocate DRM framebuffer.\n");
-		ret = PTR_ERR(fbdev_cma->fb);
+		ret = PTR_ERR(fb);
 		goto err_fb_info_destroy;
 	}
 
-	fb = &fbdev_cma->fb->fb;
 	helper->fb = fb;
 
 	fbi->par = helper;
@@ -500,7 +373,7 @@ drm_fbdev_cma_create(struct drm_fb_helper *helper,
 	return 0;
 
 err_cma_destroy:
-	drm_framebuffer_remove(&fbdev_cma->fb->fb);
+	drm_framebuffer_remove(fb);
 err_fb_info_destroy:
 	drm_fb_helper_fini(helper);
 err_gem_free_object:
@@ -570,6 +443,11 @@ err_free:
 }
 EXPORT_SYMBOL_GPL(drm_fbdev_cma_init_with_funcs);
 
+static const struct drm_framebuffer_funcs drm_fb_cma_funcs = {
+	.destroy	= drm_gem_fb_destroy,
+	.create_handle	= drm_gem_fb_create_handle,
+};
+
 /**
  * drm_fbdev_cma_init() - Allocate and initializes a drm_fbdev_cma struct
  * @dev: DRM device
@@ -597,8 +475,8 @@ void drm_fbdev_cma_fini(struct drm_fbdev_cma *fbdev_cma)
 	if (fbdev_cma->fb_helper.fbdev)
 		drm_fbdev_cma_defio_fini(fbdev_cma->fb_helper.fbdev);
 
-	if (fbdev_cma->fb)
-		drm_framebuffer_remove(&fbdev_cma->fb->fb);
+	if (fbdev_cma->fb_helper.fb)
+		drm_framebuffer_remove(fbdev_cma->fb_helper.fb);
 
 	drm_fb_helper_fini(&fbdev_cma->fb_helper);
 	kfree(fbdev_cma);
@@ -640,7 +518,7 @@ EXPORT_SYMBOL_GPL(drm_fbdev_cma_hotplug_event);
  * Calls drm_fb_helper_set_suspend, which is a wrapper around
  * fb_set_suspend implemented by fbdev core.
  */
-void drm_fbdev_cma_set_suspend(struct drm_fbdev_cma *fbdev_cma, int state)
+void drm_fbdev_cma_set_suspend(struct drm_fbdev_cma *fbdev_cma, bool state)
 {
 	if (fbdev_cma)
 		drm_fb_helper_set_suspend(&fbdev_cma->fb_helper, state);
@@ -657,7 +535,7 @@ EXPORT_SYMBOL(drm_fbdev_cma_set_suspend);
  * fb_set_suspend implemented by fbdev core.
  */
 void drm_fbdev_cma_set_suspend_unlocked(struct drm_fbdev_cma *fbdev_cma,
-					int state)
+					bool state)
 {
 	if (fbdev_cma)
 		drm_fb_helper_set_suspend_unlocked(&fbdev_cma->fb_helper,
diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c
index 574af01d3ce9..1b8f013ffa65 100644
--- a/drivers/gpu/drm/drm_fb_helper.c
+++ b/drivers/gpu/drm/drm_fb_helper.c
@@ -106,11 +106,11 @@ static DEFINE_MUTEX(kernel_fb_helper_lock);
  */
 
 #define drm_fb_helper_for_each_connector(fbh, i__) \
-	for (({ lockdep_assert_held(&(fbh)->dev->mode_config.mutex); }), \
+	for (({ lockdep_assert_held(&(fbh)->lock); }), \
 	     i__ = 0; i__ < (fbh)->connector_count; i__++)
 
-int drm_fb_helper_add_one_connector(struct drm_fb_helper *fb_helper,
-				    struct drm_connector *connector)
+static int __drm_fb_helper_add_one_connector(struct drm_fb_helper *fb_helper,
+					     struct drm_connector *connector)
 {
 	struct drm_fb_helper_connector *fb_conn;
 	struct drm_fb_helper_connector **temp;
@@ -119,7 +119,7 @@ int drm_fb_helper_add_one_connector(struct drm_fb_helper *fb_helper,
 	if (!drm_fbdev_emulation)
 		return 0;
 
-	WARN_ON(!mutex_is_locked(&fb_helper->dev->mode_config.mutex));
+	lockdep_assert_held(&fb_helper->lock);
 
 	count = fb_helper->connector_count + 1;
 
@@ -141,8 +141,21 @@ int drm_fb_helper_add_one_connector(struct drm_fb_helper *fb_helper,
 	drm_connector_get(connector);
 	fb_conn->connector = connector;
 	fb_helper->connector_info[fb_helper->connector_count++] = fb_conn;
+
 	return 0;
 }
+
+int drm_fb_helper_add_one_connector(struct drm_fb_helper *fb_helper,
+				    struct drm_connector *connector)
+{
+	int err;
+
+	mutex_lock(&fb_helper->lock);
+	err = __drm_fb_helper_add_one_connector(fb_helper, connector);
+	mutex_unlock(&fb_helper->lock);
+
+	return err;
+}
 EXPORT_SYMBOL(drm_fb_helper_add_one_connector);
 
 /**
@@ -169,11 +182,10 @@ int drm_fb_helper_single_add_all_connectors(struct drm_fb_helper *fb_helper)
 	if (!drm_fbdev_emulation)
 		return 0;
 
-	mutex_lock(&dev->mode_config.mutex);
+	mutex_lock(&fb_helper->lock);
 	drm_connector_list_iter_begin(dev, &conn_iter);
 	drm_for_each_connector_iter(connector, &conn_iter) {
-		ret = drm_fb_helper_add_one_connector(fb_helper, connector);
-
+		ret = __drm_fb_helper_add_one_connector(fb_helper, connector);
 		if (ret)
 			goto fail;
 	}
@@ -192,14 +204,14 @@ fail:
 	fb_helper->connector_count = 0;
 out:
 	drm_connector_list_iter_end(&conn_iter);
-	mutex_unlock(&dev->mode_config.mutex);
+	mutex_unlock(&fb_helper->lock);
 
 	return ret;
 }
 EXPORT_SYMBOL(drm_fb_helper_single_add_all_connectors);
 
-int drm_fb_helper_remove_one_connector(struct drm_fb_helper *fb_helper,
-				       struct drm_connector *connector)
+static int __drm_fb_helper_remove_one_connector(struct drm_fb_helper *fb_helper,
+						struct drm_connector *connector)
 {
 	struct drm_fb_helper_connector *fb_helper_connector;
 	int i, j;
@@ -207,9 +219,9 @@ int drm_fb_helper_remove_one_connector(struct drm_fb_helper *fb_helper,
 	if (!drm_fbdev_emulation)
 		return 0;
 
-	WARN_ON(!mutex_is_locked(&fb_helper->dev->mode_config.mutex));
+	lockdep_assert_held(&fb_helper->lock);
 
-	for (i = 0; i < fb_helper->connector_count; i++) {
+	drm_fb_helper_for_each_connector(fb_helper, i) {
 		if (fb_helper->connector_info[i]->connector == connector)
 			break;
 	}
@@ -227,23 +239,19 @@ int drm_fb_helper_remove_one_connector(struct drm_fb_helper *fb_helper,
 
 	return 0;
 }
-EXPORT_SYMBOL(drm_fb_helper_remove_one_connector);
 
-static void drm_fb_helper_save_lut_atomic(struct drm_crtc *crtc, struct drm_fb_helper *helper)
+int drm_fb_helper_remove_one_connector(struct drm_fb_helper *fb_helper,
+				       struct drm_connector *connector)
 {
-	uint16_t *r_base, *g_base, *b_base;
-	int i;
-
-	if (helper->funcs->gamma_get == NULL)
-		return;
+	int err;
 
-	r_base = crtc->gamma_store;
-	g_base = r_base + crtc->gamma_size;
-	b_base = g_base + crtc->gamma_size;
+	mutex_lock(&fb_helper->lock);
+	err = __drm_fb_helper_remove_one_connector(fb_helper, connector);
+	mutex_unlock(&fb_helper->lock);
 
-	for (i = 0; i < crtc->gamma_size; i++)
-		helper->funcs->gamma_get(crtc, &r_base[i], &g_base[i], &b_base[i], i);
+	return err;
 }
+EXPORT_SYMBOL(drm_fb_helper_remove_one_connector);
 
 static void drm_fb_helper_restore_lut_atomic(struct drm_crtc *crtc)
 {
@@ -285,7 +293,6 @@ int drm_fb_helper_debug_enter(struct fb_info *info)
 			if (drm_drv_uses_atomic_modeset(mode_set->crtc->dev))
 				continue;
 
-			drm_fb_helper_save_lut_atomic(mode_set->crtc, helper);
 			funcs->mode_set_base_atomic(mode_set->crtc,
 						    mode_set->fb,
 						    mode_set->x,
@@ -298,20 +305,6 @@ int drm_fb_helper_debug_enter(struct fb_info *info)
 }
 EXPORT_SYMBOL(drm_fb_helper_debug_enter);
 
-/* Find the real fb for a given fb helper CRTC */
-static struct drm_framebuffer *drm_mode_config_fb(struct drm_crtc *crtc)
-{
-	struct drm_device *dev = crtc->dev;
-	struct drm_crtc *c;
-
-	drm_for_each_crtc(c, dev) {
-		if (crtc->base.id == c->base.id)
-			return c->primary->fb;
-	}
-
-	return NULL;
-}
-
 /**
  * drm_fb_helper_debug_leave - implementation for &fb_ops.fb_debug_leave
  * @info: fbdev registered by the helper
@@ -328,8 +321,11 @@ int drm_fb_helper_debug_leave(struct fb_info *info)
 		struct drm_mode_set *mode_set = &helper->crtc_info[i].mode_set;
 
 		crtc = mode_set->crtc;
+		if (drm_drv_uses_atomic_modeset(crtc->dev))
+			continue;
+
 		funcs = crtc->helper_private;
-		fb = drm_mode_config_fb(crtc);
+		fb = crtc->primary->fb;
 
 		if (!crtc->enabled)
 			continue;
@@ -342,9 +338,6 @@ int drm_fb_helper_debug_leave(struct fb_info *info)
 		if (funcs->mode_set_base_atomic == NULL)
 			continue;
 
-		if (drm_drv_uses_atomic_modeset(crtc->dev))
-			continue;
-
 		drm_fb_helper_restore_lut_atomic(mode_set->crtc);
 		funcs->mode_set_base_atomic(mode_set->crtc, fb, crtc->x,
 					    crtc->y, LEAVE_ATOMIC_MODE_SET);
@@ -354,19 +347,24 @@ int drm_fb_helper_debug_leave(struct fb_info *info)
 }
 EXPORT_SYMBOL(drm_fb_helper_debug_leave);
 
-static int restore_fbdev_mode_atomic(struct drm_fb_helper *fb_helper)
+static int restore_fbdev_mode_atomic(struct drm_fb_helper *fb_helper, bool active)
 {
 	struct drm_device *dev = fb_helper->dev;
 	struct drm_plane *plane;
 	struct drm_atomic_state *state;
 	int i, ret;
 	unsigned int plane_mask;
+	struct drm_modeset_acquire_ctx ctx;
+
+	drm_modeset_acquire_init(&ctx, 0);
 
 	state = drm_atomic_state_alloc(dev);
-	if (!state)
-		return -ENOMEM;
+	if (!state) {
+		ret = -ENOMEM;
+		goto out_ctx;
+	}
 
-	state->acquire_ctx = dev->mode_config.acquire_ctx;
+	state->acquire_ctx = &ctx;
 retry:
 	plane_mask = 0;
 	drm_for_each_plane(plane, dev) {
@@ -375,7 +373,7 @@ retry:
 		plane_state = drm_atomic_get_plane_state(state, plane);
 		if (IS_ERR(plane_state)) {
 			ret = PTR_ERR(plane_state);
-			goto fail;
+			goto out_state;
 		}
 
 		plane_state->rotation = DRM_MODE_ROTATE_0;
@@ -389,7 +387,7 @@ retry:
 
 		ret = __drm_atomic_helper_disable_plane(plane, plane_state);
 		if (ret != 0)
-			goto fail;
+			goto out_state;
 	}
 
 	for (i = 0; i < fb_helper->crtc_count; i++) {
@@ -397,23 +395,38 @@ retry:
 
 		ret = __drm_atomic_helper_set_config(mode_set, state);
 		if (ret != 0)
-			goto fail;
+			goto out_state;
+
+		/*
+		 * __drm_atomic_helper_set_config() sets active when a
+		 * mode is set, unconditionally clear it if we force DPMS off
+		 */
+		if (!active) {
+			struct drm_crtc *crtc = mode_set->crtc;
+			struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
+
+			crtc_state->active = false;
+		}
 	}
 
 	ret = drm_atomic_commit(state);
 
-fail:
+out_state:
 	drm_atomic_clean_old_fb(dev, plane_mask, ret);
 
 	if (ret == -EDEADLK)
 		goto backoff;
 
 	drm_atomic_state_put(state);
+out_ctx:
+	drm_modeset_drop_locks(&ctx);
+	drm_modeset_acquire_fini(&ctx);
+
 	return ret;
 
 backoff:
 	drm_atomic_state_clear(state);
-	drm_atomic_legacy_backoff(state);
+	drm_modeset_backoff(&ctx);
 
 	goto retry;
 }
@@ -422,8 +435,9 @@ static int restore_fbdev_mode_legacy(struct drm_fb_helper *fb_helper)
 {
 	struct drm_device *dev = fb_helper->dev;
 	struct drm_plane *plane;
-	int i;
+	int i, ret = 0;
 
+	drm_modeset_lock_all(fb_helper->dev);
 	drm_for_each_plane(plane, dev) {
 		if (plane->type != DRM_PLANE_TYPE_PRIMARY)
 			drm_plane_force_disable(plane);
@@ -437,34 +451,33 @@ static int restore_fbdev_mode_legacy(struct drm_fb_helper *fb_helper)
 	for (i = 0; i < fb_helper->crtc_count; i++) {
 		struct drm_mode_set *mode_set = &fb_helper->crtc_info[i].mode_set;
 		struct drm_crtc *crtc = mode_set->crtc;
-		int ret;
 
 		if (crtc->funcs->cursor_set2) {
 			ret = crtc->funcs->cursor_set2(crtc, NULL, 0, 0, 0, 0, 0);
 			if (ret)
-				return ret;
+				goto out;
 		} else if (crtc->funcs->cursor_set) {
 			ret = crtc->funcs->cursor_set(crtc, NULL, 0, 0, 0);
 			if (ret)
-				return ret;
+				goto out;
 		}
 
 		ret = drm_mode_set_config_internal(mode_set);
 		if (ret)
-			return ret;
+			goto out;
 	}
+out:
+	drm_modeset_unlock_all(fb_helper->dev);
 
-	return 0;
+	return ret;
 }
 
 static int restore_fbdev_mode(struct drm_fb_helper *fb_helper)
 {
 	struct drm_device *dev = fb_helper->dev;
 
-	drm_warn_on_modeset_not_all_locked(dev);
-
 	if (drm_drv_uses_atomic_modeset(dev))
-		return restore_fbdev_mode_atomic(fb_helper);
+		return restore_fbdev_mode_atomic(fb_helper, true);
 	else
 		return restore_fbdev_mode_legacy(fb_helper);
 }
@@ -482,23 +495,26 @@ static int restore_fbdev_mode(struct drm_fb_helper *fb_helper)
  */
 int drm_fb_helper_restore_fbdev_mode_unlocked(struct drm_fb_helper *fb_helper)
 {
-	struct drm_device *dev = fb_helper->dev;
 	bool do_delayed;
 	int ret;
 
 	if (!drm_fbdev_emulation)
 		return -ENODEV;
 
-	drm_modeset_lock_all(dev);
+	if (READ_ONCE(fb_helper->deferred_setup))
+		return 0;
+
+	mutex_lock(&fb_helper->lock);
 	ret = restore_fbdev_mode(fb_helper);
 
 	do_delayed = fb_helper->delayed_hotplug;
 	if (do_delayed)
 		fb_helper->delayed_hotplug = false;
-	drm_modeset_unlock_all(dev);
+	mutex_unlock(&fb_helper->lock);
 
 	if (do_delayed)
 		drm_fb_helper_hotplug_event(fb_helper);
+
 	return ret;
 }
 EXPORT_SYMBOL(drm_fb_helper_restore_fbdev_mode_unlocked);
@@ -517,10 +533,12 @@ static bool drm_fb_helper_is_bound(struct drm_fb_helper *fb_helper)
 		return false;
 
 	drm_for_each_crtc(crtc, dev) {
+		drm_modeset_lock(&crtc->mutex, NULL);
 		if (crtc->primary->fb)
 			crtcs_bound++;
 		if (crtc->primary->fb == fb_helper->fb)
 			bound++;
+		drm_modeset_unlock(&crtc->mutex);
 	}
 
 	if (bound < crtcs_bound)
@@ -548,11 +566,11 @@ static bool drm_fb_helper_force_kernel_mode(void)
 		if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
 			continue;
 
-		drm_modeset_lock_all(dev);
+		mutex_lock(&helper->lock);
 		ret = restore_fbdev_mode(helper);
 		if (ret)
 			error = true;
-		drm_modeset_unlock_all(dev);
+		mutex_unlock(&helper->lock);
 	}
 	return error;
 }
@@ -581,23 +599,14 @@ static struct sysrq_key_op sysrq_drm_fb_helper_restore_op = {
 static struct sysrq_key_op sysrq_drm_fb_helper_restore_op = { };
 #endif
 
-static void drm_fb_helper_dpms(struct fb_info *info, int dpms_mode)
+static void dpms_legacy(struct drm_fb_helper *fb_helper, int dpms_mode)
 {
-	struct drm_fb_helper *fb_helper = info->par;
 	struct drm_device *dev = fb_helper->dev;
 	struct drm_crtc *crtc;
 	struct drm_connector *connector;
 	int i, j;
 
-	/*
-	 * For each CRTC in this fb, turn the connectors on/off.
-	 */
 	drm_modeset_lock_all(dev);
-	if (!drm_fb_helper_is_bound(fb_helper)) {
-		drm_modeset_unlock_all(dev);
-		return;
-	}
-
 	for (i = 0; i < fb_helper->crtc_count; i++) {
 		crtc = fb_helper->crtc_info[i].mode_set.crtc;
 
@@ -615,6 +624,26 @@ static void drm_fb_helper_dpms(struct fb_info *info, int dpms_mode)
 	drm_modeset_unlock_all(dev);
 }
 
+static void drm_fb_helper_dpms(struct fb_info *info, int dpms_mode)
+{
+	struct drm_fb_helper *fb_helper = info->par;
+
+	/*
+	 * For each CRTC in this fb, turn the connectors on/off.
+	 */
+	mutex_lock(&fb_helper->lock);
+	if (!drm_fb_helper_is_bound(fb_helper)) {
+		mutex_unlock(&fb_helper->lock);
+		return;
+	}
+
+	if (drm_drv_uses_atomic_modeset(fb_helper->dev))
+		restore_fbdev_mode_atomic(fb_helper, dpms_mode == DRM_MODE_DPMS_ON);
+	else
+		dpms_legacy(fb_helper, dpms_mode);
+	mutex_unlock(&fb_helper->lock);
+}
+
 /**
  * drm_fb_helper_blank - implementation for &fb_ops.fb_blank
  * @blank: desired blanking state
@@ -734,6 +763,7 @@ void drm_fb_helper_prepare(struct drm_device *dev, struct drm_fb_helper *helper,
 	INIT_WORK(&helper->resume_work, drm_fb_helper_resume_worker);
 	INIT_WORK(&helper->dirty_work, drm_fb_helper_dirty_work);
 	helper->dirty_clip.x1 = helper->dirty_clip.y1 = ~0;
+	mutex_init(&helper->lock);
 	helper->funcs = funcs;
 	helper->dev = dev;
 }
@@ -899,6 +929,7 @@ void drm_fb_helper_fini(struct drm_fb_helper *fb_helper)
 	}
 	mutex_unlock(&kernel_fb_helper_lock);
 
+	mutex_destroy(&fb_helper->lock);
 	drm_fb_helper_crtc_free(fb_helper);
 
 }
@@ -1167,22 +1198,23 @@ void drm_fb_helper_set_suspend_unlocked(struct drm_fb_helper *fb_helper,
 }
 EXPORT_SYMBOL(drm_fb_helper_set_suspend_unlocked);
 
-static int setcolreg(struct drm_crtc *crtc, u16 red, u16 green,
-		     u16 blue, u16 regno, struct fb_info *info)
+static int setcmap_pseudo_palette(struct fb_cmap *cmap, struct fb_info *info)
 {
-	struct drm_fb_helper *fb_helper = info->par;
-	struct drm_framebuffer *fb = fb_helper->fb;
+	u32 *palette = (u32 *)info->pseudo_palette;
+	int i;
 
-	if (info->fix.visual == FB_VISUAL_TRUECOLOR) {
-		u32 *palette;
+	if (cmap->start + cmap->len > 16)
+		return -EINVAL;
+
+	for (i = 0; i < cmap->len; ++i) {
+		u16 red = cmap->red[i];
+		u16 green = cmap->green[i];
+		u16 blue = cmap->blue[i];
 		u32 value;
-		/* place color in psuedopalette */
-		if (regno > 16)
-			return -EINVAL;
-		palette = (u32 *)info->pseudo_palette;
-		red >>= (16 - info->var.red.length);
-		green >>= (16 - info->var.green.length);
-		blue >>= (16 - info->var.blue.length);
+
+		red >>= 16 - info->var.red.length;
+		green >>= 16 - info->var.green.length;
+		blue >>= 16 - info->var.blue.length;
 		value = (red << info->var.red.offset) |
 			(green << info->var.green.offset) |
 			(blue << info->var.blue.offset);
@@ -1192,23 +1224,169 @@ static int setcolreg(struct drm_crtc *crtc, u16 red, u16 green,
 			mask <<= info->var.transp.offset;
 			value |= mask;
 		}
-		palette[regno] = value;
-		return 0;
+		palette[cmap->start + i] = value;
 	}
 
-	/*
-	 * The driver really shouldn't advertise pseudo/directcolor
-	 * visuals if it can't deal with the palette.
-	 */
-	if (WARN_ON(!fb_helper->funcs->gamma_set ||
-		    !fb_helper->funcs->gamma_get))
-		return -EINVAL;
+	return 0;
+}
+
+static int setcmap_legacy(struct fb_cmap *cmap, struct fb_info *info)
+{
+	struct drm_fb_helper *fb_helper = info->par;
+	struct drm_crtc *crtc;
+	u16 *r, *g, *b;
+	int i, ret = 0;
 
-	WARN_ON(fb->format->cpp[0] != 1);
+	drm_modeset_lock_all(fb_helper->dev);
+	for (i = 0; i < fb_helper->crtc_count; i++) {
+		crtc = fb_helper->crtc_info[i].mode_set.crtc;
+		if (!crtc->funcs->gamma_set || !crtc->gamma_size)
+			return -EINVAL;
 
-	fb_helper->funcs->gamma_set(crtc, red, green, blue, regno);
+		if (cmap->start + cmap->len > crtc->gamma_size)
+			return -EINVAL;
 
-	return 0;
+		r = crtc->gamma_store;
+		g = r + crtc->gamma_size;
+		b = g + crtc->gamma_size;
+
+		memcpy(r + cmap->start, cmap->red, cmap->len * sizeof(*r));
+		memcpy(g + cmap->start, cmap->green, cmap->len * sizeof(*g));
+		memcpy(b + cmap->start, cmap->blue, cmap->len * sizeof(*b));
+
+		ret = crtc->funcs->gamma_set(crtc, r, g, b,
+					     crtc->gamma_size, NULL);
+		if (ret)
+			return ret;
+	}
+	drm_modeset_unlock_all(fb_helper->dev);
+
+	return ret;
+}
+
+static struct drm_property_blob *setcmap_new_gamma_lut(struct drm_crtc *crtc,
+						       struct fb_cmap *cmap)
+{
+	struct drm_device *dev = crtc->dev;
+	struct drm_property_blob *gamma_lut;
+	struct drm_color_lut *lut;
+	int size = crtc->gamma_size;
+	int i;
+
+	if (!size || cmap->start + cmap->len > size)
+		return ERR_PTR(-EINVAL);
+
+	gamma_lut = drm_property_create_blob(dev, sizeof(*lut) * size, NULL);
+	if (IS_ERR(gamma_lut))
+		return gamma_lut;
+
+	lut = (struct drm_color_lut *)gamma_lut->data;
+	if (cmap->start || cmap->len != size) {
+		u16 *r = crtc->gamma_store;
+		u16 *g = r + crtc->gamma_size;
+		u16 *b = g + crtc->gamma_size;
+
+		for (i = 0; i < cmap->start; i++) {
+			lut[i].red = r[i];
+			lut[i].green = g[i];
+			lut[i].blue = b[i];
+		}
+		for (i = cmap->start + cmap->len; i < size; i++) {
+			lut[i].red = r[i];
+			lut[i].green = g[i];
+			lut[i].blue = b[i];
+		}
+	}
+
+	for (i = 0; i < cmap->len; i++) {
+		lut[cmap->start + i].red = cmap->red[i];
+		lut[cmap->start + i].green = cmap->green[i];
+		lut[cmap->start + i].blue = cmap->blue[i];
+	}
+
+	return gamma_lut;
+}
+
+static int setcmap_atomic(struct fb_cmap *cmap, struct fb_info *info)
+{
+	struct drm_fb_helper *fb_helper = info->par;
+	struct drm_device *dev = fb_helper->dev;
+	struct drm_property_blob *gamma_lut = NULL;
+	struct drm_modeset_acquire_ctx ctx;
+	struct drm_crtc_state *crtc_state;
+	struct drm_atomic_state *state;
+	struct drm_crtc *crtc;
+	u16 *r, *g, *b;
+	int i, ret = 0;
+	bool replaced;
+
+	drm_modeset_acquire_init(&ctx, 0);
+
+	state = drm_atomic_state_alloc(dev);
+	if (!state) {
+		ret = -ENOMEM;
+		goto out_ctx;
+	}
+
+	state->acquire_ctx = &ctx;
+retry:
+	for (i = 0; i < fb_helper->crtc_count; i++) {
+		crtc = fb_helper->crtc_info[i].mode_set.crtc;
+
+		if (!gamma_lut)
+			gamma_lut = setcmap_new_gamma_lut(crtc, cmap);
+		if (IS_ERR(gamma_lut)) {
+			ret = PTR_ERR(gamma_lut);
+			gamma_lut = NULL;
+			goto out_state;
+		}
+
+		crtc_state = drm_atomic_get_crtc_state(state, crtc);
+		if (IS_ERR(crtc_state)) {
+			ret = PTR_ERR(crtc_state);
+			goto out_state;
+		}
+
+		replaced  = drm_property_replace_blob(&crtc_state->degamma_lut,
+						      NULL);
+		replaced |= drm_property_replace_blob(&crtc_state->ctm, NULL);
+		replaced |= drm_property_replace_blob(&crtc_state->gamma_lut,
+						      gamma_lut);
+		crtc_state->color_mgmt_changed |= replaced;
+	}
+
+	ret = drm_atomic_commit(state);
+	if (ret)
+		goto out_state;
+
+	for (i = 0; i < fb_helper->crtc_count; i++) {
+		crtc = fb_helper->crtc_info[i].mode_set.crtc;
+
+		r = crtc->gamma_store;
+		g = r + crtc->gamma_size;
+		b = g + crtc->gamma_size;
+
+		memcpy(r + cmap->start, cmap->red, cmap->len * sizeof(*r));
+		memcpy(g + cmap->start, cmap->green, cmap->len * sizeof(*g));
+		memcpy(b + cmap->start, cmap->blue, cmap->len * sizeof(*b));
+	}
+
+out_state:
+	if (ret == -EDEADLK)
+		goto backoff;
+
+	drm_property_blob_put(gamma_lut);
+	drm_atomic_state_put(state);
+out_ctx:
+	drm_modeset_drop_locks(&ctx);
+	drm_modeset_acquire_fini(&ctx);
+
+	return ret;
+
+backoff:
+	drm_atomic_state_clear(state);
+	drm_modeset_backoff(&ctx);
+	goto retry;
 }
 
 /**
@@ -1219,52 +1397,29 @@ static int setcolreg(struct drm_crtc *crtc, u16 red, u16 green,
 int drm_fb_helper_setcmap(struct fb_cmap *cmap, struct fb_info *info)
 {
 	struct drm_fb_helper *fb_helper = info->par;
-	struct drm_device *dev = fb_helper->dev;
-	const struct drm_crtc_helper_funcs *crtc_funcs;
-	u16 *red, *green, *blue, *transp;
-	struct drm_crtc *crtc;
-	int i, j, rc = 0;
-	int start;
+	int ret;
 
 	if (oops_in_progress)
 		return -EBUSY;
 
-	drm_modeset_lock_all(dev);
+	mutex_lock(&fb_helper->lock);
+
 	if (!drm_fb_helper_is_bound(fb_helper)) {
-		drm_modeset_unlock_all(dev);
-		return -EBUSY;
+		ret = -EBUSY;
+		goto out;
 	}
 
-	for (i = 0; i < fb_helper->crtc_count; i++) {
-		crtc = fb_helper->crtc_info[i].mode_set.crtc;
-		crtc_funcs = crtc->helper_private;
-
-		red = cmap->red;
-		green = cmap->green;
-		blue = cmap->blue;
-		transp = cmap->transp;
-		start = cmap->start;
-
-		for (j = 0; j < cmap->len; j++) {
-			u16 hred, hgreen, hblue, htransp = 0xffff;
-
-			hred = *red++;
-			hgreen = *green++;
-			hblue = *blue++;
+	if (info->fix.visual == FB_VISUAL_TRUECOLOR)
+		ret = setcmap_pseudo_palette(cmap, info);
+	else if (drm_drv_uses_atomic_modeset(fb_helper->dev))
+		ret = setcmap_atomic(cmap, info);
+	else
+		ret = setcmap_legacy(cmap, info);
 
-			if (transp)
-				htransp = *transp++;
+out:
+	mutex_unlock(&fb_helper->lock);
 
-			rc = setcolreg(crtc, hred, hgreen, hblue, start++, info);
-			if (rc)
-				goto out;
-		}
-		if (crtc_funcs->load_lut)
-			crtc_funcs->load_lut(crtc);
-	}
- out:
-	drm_modeset_unlock_all(dev);
-	return rc;
+	return ret;
 }
 EXPORT_SYMBOL(drm_fb_helper_setcmap);
 
@@ -1281,12 +1436,11 @@ int drm_fb_helper_ioctl(struct fb_info *info, unsigned int cmd,
 			unsigned long arg)
 {
 	struct drm_fb_helper *fb_helper = info->par;
-	struct drm_device *dev = fb_helper->dev;
 	struct drm_mode_set *mode_set;
 	struct drm_crtc *crtc;
 	int ret = 0;
 
-	mutex_lock(&dev->mode_config.mutex);
+	mutex_lock(&fb_helper->lock);
 	if (!drm_fb_helper_is_bound(fb_helper)) {
 		ret = -EBUSY;
 		goto unlock;
@@ -1331,7 +1485,7 @@ int drm_fb_helper_ioctl(struct fb_info *info, unsigned int cmd,
 	}
 
 unlock:
-	mutex_unlock(&dev->mode_config.mutex);
+	mutex_unlock(&fb_helper->lock);
 	return ret;
 }
 EXPORT_SYMBOL(drm_fb_helper_ioctl);
@@ -1463,61 +1617,36 @@ int drm_fb_helper_set_par(struct fb_info *info)
 }
 EXPORT_SYMBOL(drm_fb_helper_set_par);
 
-static int pan_display_atomic(struct fb_var_screeninfo *var,
-			      struct fb_info *info)
+static void pan_set(struct drm_fb_helper *fb_helper, int x, int y)
 {
-	struct drm_fb_helper *fb_helper = info->par;
-	struct drm_device *dev = fb_helper->dev;
-	struct drm_atomic_state *state;
-	struct drm_plane *plane;
-	int i, ret;
-	unsigned int plane_mask;
-
-	state = drm_atomic_state_alloc(dev);
-	if (!state)
-		return -ENOMEM;
+	int i;
 
-	state->acquire_ctx = dev->mode_config.acquire_ctx;
-retry:
-	plane_mask = 0;
 	for (i = 0; i < fb_helper->crtc_count; i++) {
 		struct drm_mode_set *mode_set;
 
 		mode_set = &fb_helper->crtc_info[i].mode_set;
 
-		mode_set->x = var->xoffset;
-		mode_set->y = var->yoffset;
-
-		ret = __drm_atomic_helper_set_config(mode_set, state);
-		if (ret != 0)
-			goto fail;
-
-		plane = mode_set->crtc->primary;
-		plane_mask |= (1 << drm_plane_index(plane));
-		plane->old_fb = plane->fb;
+		mode_set->x = x;
+		mode_set->y = y;
 	}
+}
 
-	ret = drm_atomic_commit(state);
-	if (ret != 0)
-		goto fail;
+static int pan_display_atomic(struct fb_var_screeninfo *var,
+			      struct fb_info *info)
+{
+	struct drm_fb_helper *fb_helper = info->par;
+	int ret;
 
-	info->var.xoffset = var->xoffset;
-	info->var.yoffset = var->yoffset;
+	pan_set(fb_helper, var->xoffset, var->yoffset);
 
-fail:
-	drm_atomic_clean_old_fb(dev, plane_mask, ret);
+	ret = restore_fbdev_mode_atomic(fb_helper, true);
+	if (!ret) {
+		info->var.xoffset = var->xoffset;
+		info->var.yoffset = var->yoffset;
+	} else
+		pan_set(fb_helper, info->var.xoffset, info->var.yoffset);
 
-	if (ret == -EDEADLK)
-		goto backoff;
-
-	drm_atomic_state_put(state);
 	return ret;
-
-backoff:
-	drm_atomic_state_clear(state);
-	drm_atomic_legacy_backoff(state);
-
-	goto retry;
 }
 
 static int pan_display_legacy(struct fb_var_screeninfo *var,
@@ -1528,6 +1657,7 @@ static int pan_display_legacy(struct fb_var_screeninfo *var,
 	int ret = 0;
 	int i;
 
+	drm_modeset_lock_all(fb_helper->dev);
 	for (i = 0; i < fb_helper->crtc_count; i++) {
 		modeset = &fb_helper->crtc_info[i].mode_set;
 
@@ -1542,6 +1672,7 @@ static int pan_display_legacy(struct fb_var_screeninfo *var,
 			}
 		}
 	}
+	drm_modeset_unlock_all(fb_helper->dev);
 
 	return ret;
 }
@@ -1561,9 +1692,9 @@ int drm_fb_helper_pan_display(struct fb_var_screeninfo *var,
 	if (oops_in_progress)
 		return -EBUSY;
 
-	drm_modeset_lock_all(dev);
+	mutex_lock(&fb_helper->lock);
 	if (!drm_fb_helper_is_bound(fb_helper)) {
-		drm_modeset_unlock_all(dev);
+		mutex_unlock(&fb_helper->lock);
 		return -EBUSY;
 	}
 
@@ -1571,7 +1702,7 @@ int drm_fb_helper_pan_display(struct fb_var_screeninfo *var,
 		ret = pan_display_atomic(var, info);
 	else
 		ret = pan_display_legacy(var, info);
-	drm_modeset_unlock_all(dev);
+	mutex_unlock(&fb_helper->lock);
 
 	return ret;
 }
@@ -1579,8 +1710,7 @@ EXPORT_SYMBOL(drm_fb_helper_pan_display);
 
 /*
  * Allocates the backing storage and sets up the fbdev info structure through
- * the ->fb_probe callback and then registers the fbdev and sets up the panic
- * notifier.
+ * the ->fb_probe callback.
  */
 static int drm_fb_helper_single_fb_probe(struct drm_fb_helper *fb_helper,
 					 int preferred_bpp)
@@ -1678,13 +1808,8 @@ static int drm_fb_helper_single_fb_probe(struct drm_fb_helper *fb_helper,
 	}
 
 	if (crtc_count == 0 || sizes.fb_width == -1 || sizes.fb_height == -1) {
-		/*
-		 * hmm everyone went away - assume VGA cable just fell out
-		 * and will come back later.
-		 */
-		DRM_INFO("Cannot find any crtc or sizes - going 1024x768\n");
-		sizes.fb_width = sizes.surface_width = 1024;
-		sizes.fb_height = sizes.surface_height = 768;
+		DRM_INFO("Cannot find any crtc or sizes\n");
+		return -EAGAIN;
 	}
 
 	/* Handle our overallocation */
@@ -1696,17 +1821,6 @@ static int drm_fb_helper_single_fb_probe(struct drm_fb_helper *fb_helper,
 	if (ret < 0)
 		return ret;
 
-	/*
-	 * Set the fb pointer - usually drm_setup_crtcs does this for hotplug
-	 * events, but at init time drm_setup_crtcs needs to be called before
-	 * the fb is allocated (since we need to figure out the desired size of
-	 * the fb before we can allocate it ...). Hence we need to fix things up
-	 * here again.
-	 */
-	for (i = 0; i < fb_helper->crtc_count; i++)
-		if (fb_helper->crtc_info[i].mode_set.num_connectors)
-			fb_helper->crtc_info[i].mode_set.fb = fb_helper->fb;
-
 	return 0;
 }
 
@@ -1768,8 +1882,6 @@ void drm_fb_helper_fill_var(struct fb_info *info, struct drm_fb_helper *fb_helpe
 	info->var.xoffset = 0;
 	info->var.yoffset = 0;
 	info->var.activate = FB_ACTIVATE_NOW;
-	info->var.height = -1;
-	info->var.width = -1;
 
 	switch (fb->format->depth) {
 	case 8:
@@ -1831,12 +1943,11 @@ void drm_fb_helper_fill_var(struct fb_info *info, struct drm_fb_helper *fb_helpe
 EXPORT_SYMBOL(drm_fb_helper_fill_var);
 
 static int drm_fb_helper_probe_connector_modes(struct drm_fb_helper *fb_helper,
-					       uint32_t maxX,
-					       uint32_t maxY)
+						uint32_t maxX,
+						uint32_t maxY)
 {
 	struct drm_connector *connector;
-	int count = 0;
-	int i;
+	int i, count = 0;
 
 	drm_fb_helper_for_each_connector(fb_helper, i) {
 		connector = fb_helper->connector_info[i]->connector;
@@ -2234,11 +2345,8 @@ static void drm_setup_crtcs(struct drm_fb_helper *fb_helper,
 	int i;
 
 	DRM_DEBUG_KMS("\n");
-	if (drm_fb_helper_probe_connector_modes(fb_helper, width, height) == 0)
-		DRM_DEBUG_KMS("No connectors reported connected with modes\n");
-
 	/* prevent concurrent modification of connector_count by hotplug */
-	lockdep_assert_held(&fb_helper->dev->mode_config.mutex);
+	lockdep_assert_held(&fb_helper->lock);
 
 	crtcs = kcalloc(fb_helper->connector_count,
 			sizeof(struct drm_fb_helper_crtc *), GFP_KERNEL);
@@ -2253,6 +2361,9 @@ static void drm_setup_crtcs(struct drm_fb_helper *fb_helper,
 		goto out;
 	}
 
+	mutex_lock(&fb_helper->dev->mode_config.mutex);
+	if (drm_fb_helper_probe_connector_modes(fb_helper, width, height) == 0)
+		DRM_DEBUG_KMS("No connectors reported connected with modes\n");
 	drm_enable_connectors(fb_helper, enabled);
 
 	if (!(fb_helper->funcs->initial_config &&
@@ -2274,6 +2385,7 @@ static void drm_setup_crtcs(struct drm_fb_helper *fb_helper,
 
 		drm_pick_crtcs(fb_helper, crtcs, modes, 0, width, height);
 	}
+	mutex_unlock(&fb_helper->dev->mode_config.mutex);
 
 	/* need to set the modesets up here for use later */
 	/* fill out the connector<->crtc mappings into the modesets */
@@ -2285,9 +2397,9 @@ static void drm_setup_crtcs(struct drm_fb_helper *fb_helper,
 		struct drm_display_mode *mode = modes[i];
 		struct drm_fb_helper_crtc *fb_crtc = crtcs[i];
 		struct drm_fb_offset *offset = &offsets[i];
-		struct drm_mode_set *modeset = &fb_crtc->mode_set;
 
 		if (mode && fb_crtc) {
+			struct drm_mode_set *modeset = &fb_crtc->mode_set;
 			struct drm_connector *connector =
 				fb_helper->connector_info[i]->connector;
 
@@ -2301,7 +2413,6 @@ static void drm_setup_crtcs(struct drm_fb_helper *fb_helper,
 							   fb_crtc->desired_mode);
 			drm_connector_get(connector);
 			modeset->connectors[modeset->num_connectors++] = connector;
-			modeset->fb = fb_helper->fb;
 			modeset->x = offset->x;
 			modeset->y = offset->y;
 		}
@@ -2313,6 +2424,91 @@ out:
 	kfree(enabled);
 }
 
+/*
+ * This is a continuation of drm_setup_crtcs() that sets up anything related
+ * to the framebuffer. During initialization, drm_setup_crtcs() is called before
+ * the framebuffer has been allocated (fb_helper->fb and fb_helper->fbdev).
+ * So, any setup that touches those fields needs to be done here instead of in
+ * drm_setup_crtcs().
+ */
+static void drm_setup_crtcs_fb(struct drm_fb_helper *fb_helper)
+{
+	struct fb_info *info = fb_helper->fbdev;
+	int i;
+
+	for (i = 0; i < fb_helper->crtc_count; i++)
+		if (fb_helper->crtc_info[i].mode_set.num_connectors)
+			fb_helper->crtc_info[i].mode_set.fb = fb_helper->fb;
+
+	mutex_lock(&fb_helper->dev->mode_config.mutex);
+	drm_fb_helper_for_each_connector(fb_helper, i) {
+		struct drm_connector *connector =
+					fb_helper->connector_info[i]->connector;
+
+		/* use first connected connector for the physical dimensions */
+		if (connector->status == connector_status_connected) {
+			info->var.width = connector->display_info.width_mm;
+			info->var.height = connector->display_info.height_mm;
+			break;
+		}
+	}
+	mutex_unlock(&fb_helper->dev->mode_config.mutex);
+}
+
+/* Note: Drops fb_helper->lock before returning. */
+static int
+__drm_fb_helper_initial_config_and_unlock(struct drm_fb_helper *fb_helper,
+					  int bpp_sel)
+{
+	struct drm_device *dev = fb_helper->dev;
+	struct fb_info *info;
+	unsigned int width, height;
+	int ret;
+
+	width = dev->mode_config.max_width;
+	height = dev->mode_config.max_height;
+
+	drm_setup_crtcs(fb_helper, width, height);
+	ret = drm_fb_helper_single_fb_probe(fb_helper, bpp_sel);
+	if (ret < 0) {
+		if (ret == -EAGAIN) {
+			fb_helper->preferred_bpp = bpp_sel;
+			fb_helper->deferred_setup = true;
+			ret = 0;
+		}
+		mutex_unlock(&fb_helper->lock);
+
+		return ret;
+	}
+	drm_setup_crtcs_fb(fb_helper);
+
+	fb_helper->deferred_setup = false;
+
+	info = fb_helper->fbdev;
+	info->var.pixclock = 0;
+
+	/* Need to drop locks to avoid recursive deadlock in
+	 * register_framebuffer. This is ok because the only thing left to do is
+	 * register the fbdev emulation instance in kernel_fb_helper_list. */
+	mutex_unlock(&fb_helper->lock);
+
+	ret = register_framebuffer(info);
+	if (ret < 0)
+		return ret;
+
+	dev_info(dev->dev, "fb%d: %s frame buffer device\n",
+		 info->node, info->fix.id);
+
+	mutex_lock(&kernel_fb_helper_lock);
+	if (list_empty(&kernel_fb_helper_list))
+		register_sysrq_key('v', &sysrq_drm_fb_helper_restore_op);
+
+	list_add(&fb_helper->kernel_fb_list, &kernel_fb_helper_list);
+	mutex_unlock(&kernel_fb_helper_lock);
+
+	return 0;
+}
+
 /**
  * drm_fb_helper_initial_config - setup a sane initial connector configuration
  * @fb_helper: fb_helper device struct
@@ -2357,39 +2553,15 @@ out:
  */
 int drm_fb_helper_initial_config(struct drm_fb_helper *fb_helper, int bpp_sel)
 {
-	struct drm_device *dev = fb_helper->dev;
-	struct fb_info *info;
 	int ret;
 
 	if (!drm_fbdev_emulation)
 		return 0;
 
-	mutex_lock(&dev->mode_config.mutex);
-	drm_setup_crtcs(fb_helper,
-			dev->mode_config.max_width,
-			dev->mode_config.max_height);
-	ret = drm_fb_helper_single_fb_probe(fb_helper, bpp_sel);
-	mutex_unlock(&dev->mode_config.mutex);
-	if (ret)
-		return ret;
+	mutex_lock(&fb_helper->lock);
+	ret = __drm_fb_helper_initial_config_and_unlock(fb_helper, bpp_sel);
 
-	info = fb_helper->fbdev;
-	info->var.pixclock = 0;
-	ret = register_framebuffer(info);
-	if (ret < 0)
-		return ret;
-
-	dev_info(dev->dev, "fb%d: %s frame buffer device\n",
-		 info->node, info->fix.id);
-
-	mutex_lock(&kernel_fb_helper_lock);
-	if (list_empty(&kernel_fb_helper_list))
-		register_sysrq_key('v', &sysrq_drm_fb_helper_restore_op);
-
-	list_add(&fb_helper->kernel_fb_list, &kernel_fb_helper_list);
-	mutex_unlock(&kernel_fb_helper_lock);
-
-	return 0;
+	return ret;
 }
 EXPORT_SYMBOL(drm_fb_helper_initial_config);
 
@@ -2416,22 +2588,29 @@ EXPORT_SYMBOL(drm_fb_helper_initial_config);
  */
 int drm_fb_helper_hotplug_event(struct drm_fb_helper *fb_helper)
 {
-	struct drm_device *dev = fb_helper->dev;
+	int err = 0;
 
 	if (!drm_fbdev_emulation)
 		return 0;
 
-	mutex_lock(&dev->mode_config.mutex);
+	mutex_lock(&fb_helper->lock);
+	if (fb_helper->deferred_setup) {
+		err = __drm_fb_helper_initial_config_and_unlock(fb_helper,
+				fb_helper->preferred_bpp);
+		return err;
+	}
+
 	if (!fb_helper->fb || !drm_fb_helper_is_bound(fb_helper)) {
 		fb_helper->delayed_hotplug = true;
-		mutex_unlock(&dev->mode_config.mutex);
-		return 0;
+		mutex_unlock(&fb_helper->lock);
+		return err;
 	}
+
 	DRM_DEBUG_KMS("\n");
 
 	drm_setup_crtcs(fb_helper, fb_helper->fb->width, fb_helper->fb->height);
-
-	mutex_unlock(&dev->mode_config.mutex);
+	drm_setup_crtcs_fb(fb_helper);
+	mutex_unlock(&fb_helper->lock);
 
 	drm_fb_helper_set_par(fb_helper->fbdev);
 
diff --git a/drivers/gpu/drm/drm_file.c b/drivers/gpu/drm/drm_file.c
index 84f3a242cc39..b3c6e997ccdb 100644
--- a/drivers/gpu/drm/drm_file.c
+++ b/drivers/gpu/drm/drm_file.c
@@ -75,7 +75,7 @@ DEFINE_MUTEX(drm_global_mutex);
  * for drivers which use the CMA GEM helpers it's drm_gem_cma_mmap().
  *
  * No other file operations are supported by the DRM userspace API. Overall the
- * following is an example #file_operations structure::
+ * following is an example &file_operations structure::
  *
  *     static const example_drm_fops = {
  *             .owner = THIS_MODULE,
@@ -92,6 +92,11 @@ DEFINE_MUTEX(drm_global_mutex);
  * For plain GEM based drivers there is the DEFINE_DRM_GEM_FOPS() macro, and for
  * CMA based drivers there is the DEFINE_DRM_GEM_CMA_FOPS() macro to make this
  * simpler.
+ *
+ * The driver's &file_operations must be stored in &drm_driver.fops.
+ *
+ * For driver-private IOCTL handling see the more detailed discussion in
+ * :ref:`IOCTL support in the userland interfaces chapter<drm_driver_ioctl>`.
  */
 
 static int drm_open_helper(struct file *filp, struct drm_minor *minor);
@@ -431,7 +436,7 @@ int drm_release(struct inode *inode, struct file *filp)
 
 	if (!--dev->open_count) {
 		drm_lastclose(dev);
-		if (drm_device_is_unplugged(dev))
+		if (drm_dev_is_unplugged(dev))
 			drm_put_dev(dev);
 	}
 	mutex_unlock(&drm_global_mutex);
diff --git a/drivers/gpu/drm/drm_framebuffer.c b/drivers/gpu/drm/drm_framebuffer.c
index b3ef4f1c2630..af279844d7ce 100644
--- a/drivers/gpu/drm/drm_framebuffer.c
+++ b/drivers/gpu/drm/drm_framebuffer.c
@@ -817,7 +817,7 @@ retry:
 		plane->old_fb = plane->fb;
 	}
 
-	for_each_connector_in_state(state, conn, conn_state, i) {
+	for_each_new_connector_in_state(state, conn, conn_state, i) {
 		ret = drm_atomic_set_crtc_for_connector(conn_state, NULL);
 
 		if (ret)
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index cdaac37907b1..c55f338e380b 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -36,6 +36,7 @@
 #include <linux/pagemap.h>
 #include <linux/shmem_fs.h>
 #include <linux/dma-buf.h>
+#include <linux/mem_encrypt.h>
 #include <drm/drmP.h>
 #include <drm/drm_vma_manager.h>
 #include <drm/drm_gem.h>
@@ -311,6 +312,41 @@ drm_gem_handle_delete(struct drm_file *filp, u32 handle)
 EXPORT_SYMBOL(drm_gem_handle_delete);
 
 /**
+ * drm_gem_dumb_map_offset - return the fake mmap offset for a gem object
+ * @file: drm file-private structure containing the gem object
+ * @dev: corresponding drm_device
+ * @handle: gem object handle
+ * @offset: return location for the fake mmap offset
+ *
+ * This implements the &drm_driver.dumb_map_offset kms driver callback for
+ * drivers which use gem to manage their backing storage.
+ *
+ * Returns:
+ * 0 on success or a negative error code on failure.
+ */
+int drm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
+			    u32 handle, u64 *offset)
+{
+	struct drm_gem_object *obj;
+	int ret;
+
+	obj = drm_gem_object_lookup(file, handle);
+	if (!obj)
+		return -ENOENT;
+
+	ret = drm_gem_create_mmap_offset(obj);
+	if (ret)
+		goto out;
+
+	*offset = drm_vma_node_offset_addr(&obj->vma_node);
+out:
+	drm_gem_object_put_unlocked(obj);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(drm_gem_dumb_map_offset);
+
+/**
  * drm_gem_dumb_destroy - dumb fb callback helper for gem based drivers
  * @file: drm file-private structure to remove the dumb handle from
  * @dev: corresponding drm_device
@@ -826,13 +862,15 @@ drm_gem_object_put_unlocked(struct drm_gem_object *obj)
 		return;
 
 	dev = obj->dev;
-	might_lock(&dev->struct_mutex);
 
-	if (dev->driver->gem_free_object_unlocked)
+	if (dev->driver->gem_free_object_unlocked) {
 		kref_put(&obj->refcount, drm_gem_object_free);
-	else if (kref_put_mutex(&obj->refcount, drm_gem_object_free,
+	} else {
+		might_lock(&dev->struct_mutex);
+		if (kref_put_mutex(&obj->refcount, drm_gem_object_free,
 				&dev->struct_mutex))
-		mutex_unlock(&dev->struct_mutex);
+			mutex_unlock(&dev->struct_mutex);
+	}
 }
 EXPORT_SYMBOL(drm_gem_object_put_unlocked);
 
@@ -928,6 +966,7 @@ int drm_gem_mmap_obj(struct drm_gem_object *obj, unsigned long obj_size,
 	vma->vm_ops = dev->driver->gem_vm_ops;
 	vma->vm_private_data = obj;
 	vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
+	vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
 
 	/* Take a ref for this mapping of the object, so that the fault
 	 * handler can dereference the mmap offset's pointer to the object.
@@ -964,7 +1003,7 @@ int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
 	struct drm_vma_offset_node *node;
 	int ret;
 
-	if (drm_device_is_unplugged(dev))
+	if (drm_dev_is_unplugged(dev))
 		return -ENODEV;
 
 	drm_vma_offset_lock_lookup(dev->vma_offset_manager);
diff --git a/drivers/gpu/drm/drm_gem_cma_helper.c b/drivers/gpu/drm/drm_gem_cma_helper.c
index bc28e7575254..373e33f22be4 100644
--- a/drivers/gpu/drm/drm_gem_cma_helper.c
+++ b/drivers/gpu/drm/drm_gem_cma_helper.c
@@ -177,7 +177,7 @@ drm_gem_cma_create_with_handle(struct drm_file *file_priv,
  * This function frees the backing memory of the CMA GEM object, cleans up the
  * GEM object state and frees the memory used to store the object itself.
  * Drivers using the CMA helpers should set this as their
- * &drm_driver.gem_free_object callback.
+ * &drm_driver.gem_free_object_unlocked callback.
  */
 void drm_gem_cma_free_object(struct drm_gem_object *gem_obj)
 {
@@ -264,41 +264,6 @@ int drm_gem_cma_dumb_create(struct drm_file *file_priv,
 }
 EXPORT_SYMBOL_GPL(drm_gem_cma_dumb_create);
 
-/**
- * drm_gem_cma_dumb_map_offset - return the fake mmap offset for a CMA GEM
- *     object
- * @file_priv: DRM file-private structure containing the GEM object
- * @drm: DRM device
- * @handle: GEM object handle
- * @offset: return location for the fake mmap offset
- *
- * This function look up an object by its handle and returns the fake mmap
- * offset associated with it. Drivers using the CMA helpers should set this
- * as their &drm_driver.dumb_map_offset callback.
- *
- * Returns:
- * 0 on success or a negative error code on failure.
- */
-int drm_gem_cma_dumb_map_offset(struct drm_file *file_priv,
-				struct drm_device *drm, u32 handle,
-				u64 *offset)
-{
-	struct drm_gem_object *gem_obj;
-
-	gem_obj = drm_gem_object_lookup(file_priv, handle);
-	if (!gem_obj) {
-		dev_err(drm->dev, "failed to lookup GEM object\n");
-		return -EINVAL;
-	}
-
-	*offset = drm_vma_node_offset_addr(&gem_obj->vma_node);
-
-	drm_gem_object_put_unlocked(gem_obj);
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(drm_gem_cma_dumb_map_offset);
-
 const struct vm_operations_struct drm_gem_cma_vm_ops = {
 	.open = drm_gem_vm_open,
 	.close = drm_gem_vm_close,
@@ -390,7 +355,7 @@ unsigned long drm_gem_cma_get_unmapped_area(struct file *filp,
 	struct drm_device *dev = priv->minor->dev;
 	struct drm_vma_offset_node *node;
 
-	if (drm_device_is_unplugged(dev))
+	if (drm_dev_is_unplugged(dev))
 		return -ENODEV;
 
 	drm_vma_offset_lock_lookup(dev->vma_offset_manager);
diff --git a/drivers/gpu/drm/drm_gem_framebuffer_helper.c b/drivers/gpu/drm/drm_gem_framebuffer_helper.c
new file mode 100644
index 000000000000..d54a083dc5dd
--- /dev/null
+++ b/drivers/gpu/drm/drm_gem_framebuffer_helper.c
@@ -0,0 +1,283 @@
+/*
+ * drm gem framebuffer helper functions
+ *
+ * Copyright (C) 2017 Noralf Trønnes
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/dma-buf.h>
+#include <linux/dma-fence.h>
+#include <linux/reservation.h>
+#include <linux/slab.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_fourcc.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_gem.h>
+#include <drm/drm_gem_framebuffer_helper.h>
+#include <drm/drm_modeset_helper.h>
+
+/**
+ * DOC: overview
+ *
+ * This library provides helpers for drivers that don't subclass
+ * &drm_framebuffer and and use &drm_gem_object for their backing storage.
+ *
+ * Drivers without additional needs to validate framebuffers can simply use
+ * drm_gem_fb_create() and everything is wired up automatically. But all
+ * parts can be used individually.
+ */
+
+/**
+ * drm_gem_fb_get_obj() - Get GEM object for framebuffer
+ * @fb: The framebuffer
+ * @plane: Which plane
+ *
+ * Returns the GEM object for given framebuffer.
+ */
+struct drm_gem_object *drm_gem_fb_get_obj(struct drm_framebuffer *fb,
+					  unsigned int plane)
+{
+	if (plane >= 4)
+		return NULL;
+
+	return fb->obj[plane];
+}
+EXPORT_SYMBOL_GPL(drm_gem_fb_get_obj);
+
+static struct drm_framebuffer *
+drm_gem_fb_alloc(struct drm_device *dev,
+		 const struct drm_mode_fb_cmd2 *mode_cmd,
+		 struct drm_gem_object **obj, unsigned int num_planes,
+		 const struct drm_framebuffer_funcs *funcs)
+{
+	struct drm_framebuffer *fb;
+	int ret, i;
+
+	fb = kzalloc(sizeof(*fb), GFP_KERNEL);
+	if (!fb)
+		return ERR_PTR(-ENOMEM);
+
+	drm_helper_mode_fill_fb_struct(dev, fb, mode_cmd);
+
+	for (i = 0; i < num_planes; i++)
+		fb->obj[i] = obj[i];
+
+	ret = drm_framebuffer_init(dev, fb, funcs);
+	if (ret) {
+		DRM_DEV_ERROR(dev->dev, "Failed to init framebuffer: %d\n",
+			      ret);
+		kfree(fb);
+		return ERR_PTR(ret);
+	}
+
+	return fb;
+}
+
+/**
+ * drm_gem_fb_destroy - Free GEM backed framebuffer
+ * @fb: DRM framebuffer
+ *
+ * Frees a GEM backed framebuffer with its backing buffer(s) and the structure
+ * itself. Drivers can use this as their &drm_framebuffer_funcs->destroy
+ * callback.
+ */
+void drm_gem_fb_destroy(struct drm_framebuffer *fb)
+{
+	int i;
+
+	for (i = 0; i < 4; i++)
+		drm_gem_object_put_unlocked(fb->obj[i]);
+
+	drm_framebuffer_cleanup(fb);
+	kfree(fb);
+}
+EXPORT_SYMBOL(drm_gem_fb_destroy);
+
+/**
+ * drm_gem_fb_create_handle - Create handle for GEM backed framebuffer
+ * @fb: DRM framebuffer
+ * @file: drm file
+ * @handle: handle created
+ *
+ * Drivers can use this as their &drm_framebuffer_funcs->create_handle
+ * callback.
+ *
+ * Returns:
+ * 0 on success or a negative error code on failure.
+ */
+int drm_gem_fb_create_handle(struct drm_framebuffer *fb, struct drm_file *file,
+			     unsigned int *handle)
+{
+	return drm_gem_handle_create(file, fb->obj[0], handle);
+}
+EXPORT_SYMBOL(drm_gem_fb_create_handle);
+
+/**
+ * drm_gem_fb_create_with_funcs() - helper function for the
+ *                                  &drm_mode_config_funcs.fb_create
+ *                                  callback
+ * @dev: DRM device
+ * @file: drm file for the ioctl call
+ * @mode_cmd: metadata from the userspace fb creation request
+ * @funcs: vtable to be used for the new framebuffer object
+ *
+ * This can be used to set &drm_framebuffer_funcs for drivers that need the
+ * &drm_framebuffer_funcs.dirty callback. Use drm_gem_fb_create() if you don't
+ * need to change &drm_framebuffer_funcs.
+ * The function does buffer size validation.
+ */
+struct drm_framebuffer *
+drm_gem_fb_create_with_funcs(struct drm_device *dev, struct drm_file *file,
+			     const struct drm_mode_fb_cmd2 *mode_cmd,
+			     const struct drm_framebuffer_funcs *funcs)
+{
+	const struct drm_format_info *info;
+	struct drm_gem_object *objs[4];
+	struct drm_framebuffer *fb;
+	int ret, i;
+
+	info = drm_get_format_info(dev, mode_cmd);
+	if (!info)
+		return ERR_PTR(-EINVAL);
+
+	for (i = 0; i < info->num_planes; i++) {
+		unsigned int width = mode_cmd->width / (i ? info->hsub : 1);
+		unsigned int height = mode_cmd->height / (i ? info->vsub : 1);
+		unsigned int min_size;
+
+		objs[i] = drm_gem_object_lookup(file, mode_cmd->handles[i]);
+		if (!objs[i]) {
+			DRM_DEV_ERROR(dev->dev, "Failed to lookup GEM\n");
+			ret = -ENOENT;
+			goto err_gem_object_put;
+		}
+
+		min_size = (height - 1) * mode_cmd->pitches[i]
+			 + width * info->cpp[i]
+			 + mode_cmd->offsets[i];
+
+		if (objs[i]->size < min_size) {
+			drm_gem_object_put_unlocked(objs[i]);
+			ret = -EINVAL;
+			goto err_gem_object_put;
+		}
+	}
+
+	fb = drm_gem_fb_alloc(dev, mode_cmd, objs, i, funcs);
+	if (IS_ERR(fb)) {
+		ret = PTR_ERR(fb);
+		goto err_gem_object_put;
+	}
+
+	return fb;
+
+err_gem_object_put:
+	for (i--; i >= 0; i--)
+		drm_gem_object_put_unlocked(objs[i]);
+
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(drm_gem_fb_create_with_funcs);
+
+static const struct drm_framebuffer_funcs drm_gem_fb_funcs = {
+	.destroy	= drm_gem_fb_destroy,
+	.create_handle	= drm_gem_fb_create_handle,
+};
+
+/**
+ * drm_gem_fb_create() - &drm_mode_config_funcs.fb_create callback function
+ * @dev: DRM device
+ * @file: drm file for the ioctl call
+ * @mode_cmd: metadata from the userspace fb creation request
+ *
+ * If your hardware has special alignment or pitch requirements these should be
+ * checked before calling this function. The function does buffer size
+ * validation. Use drm_gem_fb_create_with_funcs() if you need to set
+ * &drm_framebuffer_funcs.dirty.
+ */
+struct drm_framebuffer *
+drm_gem_fb_create(struct drm_device *dev, struct drm_file *file,
+		  const struct drm_mode_fb_cmd2 *mode_cmd)
+{
+	return drm_gem_fb_create_with_funcs(dev, file, mode_cmd,
+					    &drm_gem_fb_funcs);
+}
+EXPORT_SYMBOL_GPL(drm_gem_fb_create);
+
+/**
+ * drm_gem_fb_prepare_fb() - Prepare gem framebuffer
+ * @plane: Which plane
+ * @state: Plane state attach fence to
+ *
+ * This can be used as the &drm_plane_helper_funcs.prepare_fb hook.
+ *
+ * This function checks if the plane FB has an dma-buf attached, extracts
+ * the exclusive fence and attaches it to plane state for the atomic helper
+ * to wait on.
+ *
+ * There is no need for &drm_plane_helper_funcs.cleanup_fb hook for simple
+ * gem based framebuffer drivers which have their buffers always pinned in
+ * memory.
+ */
+int drm_gem_fb_prepare_fb(struct drm_plane *plane,
+			  struct drm_plane_state *state)
+{
+	struct dma_buf *dma_buf;
+	struct dma_fence *fence;
+
+	if ((plane->state->fb == state->fb) || !state->fb)
+		return 0;
+
+	dma_buf = drm_gem_fb_get_obj(state->fb, 0)->dma_buf;
+	if (dma_buf) {
+		fence = reservation_object_get_excl_rcu(dma_buf->resv);
+		drm_atomic_set_fence_for_plane(state, fence);
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(drm_gem_fb_prepare_fb);
+
+/**
+ * drm_gem_fbdev_fb_create - Create a drm_framebuffer for fbdev emulation
+ * @dev: DRM device
+ * @sizes: fbdev size description
+ * @pitch_align: optional pitch alignment
+ * @obj: GEM object backing the framebuffer
+ * @funcs: vtable to be used for the new framebuffer object
+ *
+ * This function creates a framebuffer for use with fbdev emulation.
+ *
+ * Returns:
+ * Pointer to a drm_framebuffer on success or an error pointer on failure.
+ */
+struct drm_framebuffer *
+drm_gem_fbdev_fb_create(struct drm_device *dev,
+			struct drm_fb_helper_surface_size *sizes,
+			unsigned int pitch_align, struct drm_gem_object *obj,
+			const struct drm_framebuffer_funcs *funcs)
+{
+	struct drm_mode_fb_cmd2 mode_cmd = { 0 };
+
+	mode_cmd.width = sizes->surface_width;
+	mode_cmd.height = sizes->surface_height;
+	mode_cmd.pitches[0] = sizes->surface_width *
+			      DIV_ROUND_UP(sizes->surface_bpp, 8);
+	if (pitch_align)
+		mode_cmd.pitches[0] = roundup(mode_cmd.pitches[0],
+					      pitch_align);
+	mode_cmd.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp,
+							sizes->surface_depth);
+	if (obj->size < mode_cmd.pitches[0] * mode_cmd.height)
+		return ERR_PTR(-EINVAL);
+
+	return drm_gem_fb_alloc(dev, &mode_cmd, &obj, 1, funcs);
+}
+EXPORT_SYMBOL(drm_gem_fbdev_fb_create);
diff --git a/drivers/gpu/drm/drm_internal.h b/drivers/gpu/drm/drm_internal.h
index 5edc24bd10fa..fbc3f308fa19 100644
--- a/drivers/gpu/drm/drm_internal.h
+++ b/drivers/gpu/drm/drm_internal.h
@@ -32,6 +32,7 @@ void drm_lastclose(struct drm_device *dev);
 int drm_irq_by_busid(struct drm_device *dev, void *data,
 		     struct drm_file *file_priv);
 void drm_pci_agp_destroy(struct drm_device *dev);
+int drm_pci_set_busid(struct drm_device *dev, struct drm_master *master);
 
 /* drm_prime.c */
 int drm_prime_handle_to_fd_ioctl(struct drm_device *dev, void *data,
@@ -56,14 +57,19 @@ int drm_gem_name_info(struct seq_file *m, void *data);
 /* drm_vblank.c */
 extern unsigned int drm_timestamp_monotonic;
 void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe);
+void drm_vblank_cleanup(struct drm_device *dev);
+
+/* IOCTLS */
+int drm_wait_vblank_ioctl(struct drm_device *dev, void *data,
+			  struct drm_file *filp);
+int drm_legacy_modeset_ctl_ioctl(struct drm_device *dev, void *data,
+				 struct drm_file *file_priv);
+
+/* drm_irq.c */
 
 /* IOCTLS */
-int drm_wait_vblank(struct drm_device *dev, void *data,
-		    struct drm_file *filp);
 int drm_legacy_irq_control(struct drm_device *dev, void *data,
 			   struct drm_file *file_priv);
-int drm_legacy_modeset_ctl(struct drm_device *dev, void *data,
-			   struct drm_file *file_priv);
 
 /* drm_auth.c */
 int drm_getmagic(struct drm_device *dev, void *data,
@@ -161,3 +167,9 @@ int drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
 				   struct drm_file *file_private);
 int drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
 				   struct drm_file *file_private);
+int drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
+			   struct drm_file *file_private);
+int drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
+			    struct drm_file *file_private);
+int drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
+			     struct drm_file *file_private);
diff --git a/drivers/gpu/drm/drm_ioc32.c b/drivers/gpu/drm/drm_ioc32.c
index d1f202852028..f8e96e648acf 100644
--- a/drivers/gpu/drm/drm_ioc32.c
+++ b/drivers/gpu/drm/drm_ioc32.c
@@ -842,7 +842,7 @@ static int compat_drm_wait_vblank(struct file *file, unsigned int cmd,
 	req.request.type = req32.request.type;
 	req.request.sequence = req32.request.sequence;
 	req.request.signal = req32.request.signal;
-	err = drm_ioctl_kernel(file, drm_wait_vblank, &req, DRM_UNLOCKED);
+	err = drm_ioctl_kernel(file, drm_wait_vblank_ioctl, &req, DRM_UNLOCKED);
 	if (err)
 		return err;
 
diff --git a/drivers/gpu/drm/drm_ioctl.c b/drivers/gpu/drm/drm_ioctl.c
index f1eb326524cf..a9ae6dd2d593 100644
--- a/drivers/gpu/drm/drm_ioctl.c
+++ b/drivers/gpu/drm/drm_ioctl.c
@@ -143,8 +143,8 @@ static int drm_set_busid(struct drm_device *dev, struct drm_file *file_priv)
 	if (master->unique != NULL)
 		drm_unset_busid(dev, master);
 
-	if (dev->driver->set_busid) {
-		ret = dev->driver->set_busid(dev, master);
+	if (dev->dev && dev_is_pci(dev->dev)) {
+		ret = drm_pci_set_busid(dev, master);
 		if (ret) {
 			drm_unset_busid(dev, master);
 			return ret;
@@ -603,9 +603,9 @@ static const struct drm_ioctl_desc drm_ioctls[] = {
 	DRM_IOCTL_DEF(DRM_IOCTL_SG_ALLOC, drm_legacy_sg_alloc, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
 	DRM_IOCTL_DEF(DRM_IOCTL_SG_FREE, drm_legacy_sg_free, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
 
-	DRM_IOCTL_DEF(DRM_IOCTL_WAIT_VBLANK, drm_wait_vblank, DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_WAIT_VBLANK, drm_wait_vblank_ioctl, DRM_UNLOCKED),
 
-	DRM_IOCTL_DEF(DRM_IOCTL_MODESET_CTL, drm_legacy_modeset_ctl, 0),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODESET_CTL, drm_legacy_modeset_ctl_ioctl, 0),
 
 	DRM_IOCTL_DEF(DRM_IOCTL_UPDATE_DRAW, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
 
@@ -657,6 +657,12 @@ static const struct drm_ioctl_desc drm_ioctls[] = {
 		      DRM_UNLOCKED|DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF(DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE, drm_syncobj_fd_to_handle_ioctl,
 		      DRM_UNLOCKED|DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF(DRM_IOCTL_SYNCOBJ_WAIT, drm_syncobj_wait_ioctl,
+		      DRM_UNLOCKED|DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF(DRM_IOCTL_SYNCOBJ_RESET, drm_syncobj_reset_ioctl,
+		      DRM_UNLOCKED|DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF(DRM_IOCTL_SYNCOBJ_SIGNAL, drm_syncobj_signal_ioctl,
+		      DRM_UNLOCKED|DRM_RENDER_ALLOW),
 };
 
 #define DRM_CORE_IOCTL_COUNT	ARRAY_SIZE( drm_ioctls )
@@ -695,7 +701,7 @@ static const struct drm_ioctl_desc drm_ioctls[] = {
  * 
  * DRM driver private IOCTL must be in the range from DRM_COMMAND_BASE to
  * DRM_COMMAND_END. Finally you need an array of &struct drm_ioctl_desc to wire
- * up the handlers and set the access rights:
+ * up the handlers and set the access rights::
  *
  *     static const struct drm_ioctl_desc my_driver_ioctls[] = {
  *         DRM_IOCTL_DEF_DRV(MY_DRIVER_OPERATION, my_driver_operation,
@@ -704,6 +710,9 @@ static const struct drm_ioctl_desc drm_ioctls[] = {
  *
  * And then assign this to the &drm_driver.ioctls field in your driver
  * structure.
+ *
+ * See the separate chapter on :ref:`file operations<drm_driver_fops>` for how
+ * the driver-specific IOCTLs are wired up.
  */
 
 long drm_ioctl_kernel(struct file *file, drm_ioctl_t *func, void *kdata,
@@ -713,7 +722,7 @@ long drm_ioctl_kernel(struct file *file, drm_ioctl_t *func, void *kdata,
 	struct drm_device *dev = file_priv->minor->dev;
 	int retcode;
 
-	if (drm_device_is_unplugged(dev))
+	if (drm_dev_is_unplugged(dev))
 		return -ENODEV;
 
 	retcode = drm_ioctl_permit(flags, file_priv);
@@ -762,7 +771,7 @@ long drm_ioctl(struct file *filp,
 
 	dev = file_priv->minor->dev;
 
-	if (drm_device_is_unplugged(dev))
+	if (drm_dev_is_unplugged(dev))
 		return -ENODEV;
 
 	is_driver_ioctl = nr >= DRM_COMMAND_BASE && nr < DRM_COMMAND_END;
diff --git a/drivers/gpu/drm/drm_mipi_dsi.c b/drivers/gpu/drm/drm_mipi_dsi.c
index 1160a579e0dc..4b47226b90d4 100644
--- a/drivers/gpu/drm/drm_mipi_dsi.c
+++ b/drivers/gpu/drm/drm_mipi_dsi.c
@@ -165,14 +165,14 @@ of_mipi_dsi_device_add(struct mipi_dsi_host *host, struct device_node *node)
 	u32 reg;
 
 	if (of_modalias_node(node, info.type, sizeof(info.type)) < 0) {
-		dev_err(dev, "modalias failure on %s\n", node->full_name);
+		dev_err(dev, "modalias failure on %pOF\n", node);
 		return ERR_PTR(-EINVAL);
 	}
 
 	ret = of_property_read_u32(node, "reg", &reg);
 	if (ret) {
-		dev_err(dev, "device node %s has no valid reg property: %d\n",
-			node->full_name, ret);
+		dev_err(dev, "device node %pOF has no valid reg property: %d\n",
+			node, ret);
 		return ERR_PTR(-EINVAL);
 	}
 
diff --git a/drivers/gpu/drm/drm_mm.c b/drivers/gpu/drm/drm_mm.c
index f794089d30ac..61a1c8ea74bc 100644
--- a/drivers/gpu/drm/drm_mm.c
+++ b/drivers/gpu/drm/drm_mm.c
@@ -169,7 +169,7 @@ INTERVAL_TREE_DEFINE(struct drm_mm_node, rb,
 struct drm_mm_node *
 __drm_mm_interval_first(const struct drm_mm *mm, u64 start, u64 last)
 {
-	return drm_mm_interval_tree_iter_first((struct rb_root *)&mm->interval_tree,
+	return drm_mm_interval_tree_iter_first((struct rb_root_cached *)&mm->interval_tree,
 					       start, last) ?: (struct drm_mm_node *)&mm->head_node;
 }
 EXPORT_SYMBOL(__drm_mm_interval_first);
@@ -180,6 +180,7 @@ static void drm_mm_interval_tree_add_node(struct drm_mm_node *hole_node,
 	struct drm_mm *mm = hole_node->mm;
 	struct rb_node **link, *rb;
 	struct drm_mm_node *parent;
+	bool leftmost = true;
 
 	node->__subtree_last = LAST(node);
 
@@ -196,9 +197,10 @@ static void drm_mm_interval_tree_add_node(struct drm_mm_node *hole_node,
 
 		rb = &hole_node->rb;
 		link = &hole_node->rb.rb_right;
+		leftmost = false;
 	} else {
 		rb = NULL;
-		link = &mm->interval_tree.rb_node;
+		link = &mm->interval_tree.rb_root.rb_node;
 	}
 
 	while (*link) {
@@ -208,14 +210,15 @@ static void drm_mm_interval_tree_add_node(struct drm_mm_node *hole_node,
 			parent->__subtree_last = node->__subtree_last;
 		if (node->start < parent->start)
 			link = &parent->rb.rb_left;
-		else
+		else {
 			link = &parent->rb.rb_right;
+			leftmost = true;
+		}
 	}
 
 	rb_link_node(&node->rb, rb, link);
-	rb_insert_augmented(&node->rb,
-			    &mm->interval_tree,
-			    &drm_mm_interval_tree_augment);
+	rb_insert_augmented_cached(&node->rb, &mm->interval_tree, leftmost,
+				   &drm_mm_interval_tree_augment);
 }
 
 #define RB_INSERT(root, member, expr) do { \
@@ -577,7 +580,7 @@ void drm_mm_replace_node(struct drm_mm_node *old, struct drm_mm_node *new)
 	*new = *old;
 
 	list_replace(&old->node_list, &new->node_list);
-	rb_replace_node(&old->rb, &new->rb, &old->mm->interval_tree);
+	rb_replace_node(&old->rb, &new->rb, &old->mm->interval_tree.rb_root);
 
 	if (drm_mm_hole_follows(old)) {
 		list_replace(&old->hole_stack, &new->hole_stack);
@@ -863,7 +866,7 @@ void drm_mm_init(struct drm_mm *mm, u64 start, u64 size)
 	mm->color_adjust = NULL;
 
 	INIT_LIST_HEAD(&mm->hole_stack);
-	mm->interval_tree = RB_ROOT;
+	mm->interval_tree = RB_ROOT_CACHED;
 	mm->holes_size = RB_ROOT;
 	mm->holes_addr = RB_ROOT;
 
diff --git a/drivers/gpu/drm/drm_mode_config.c b/drivers/gpu/drm/drm_mode_config.c
index d9862259a2a7..74f6ff5df656 100644
--- a/drivers/gpu/drm/drm_mode_config.c
+++ b/drivers/gpu/drm/drm_mode_config.c
@@ -337,6 +337,13 @@ static int drm_mode_create_standard_properties(struct drm_device *dev)
 		return -ENOMEM;
 	dev->mode_config.gamma_lut_size_property = prop;
 
+	prop = drm_property_create(dev,
+				   DRM_MODE_PROP_IMMUTABLE | DRM_MODE_PROP_BLOB,
+				   "IN_FORMATS", 0);
+	if (!prop)
+		return -ENOMEM;
+	dev->mode_config.modifiers_property = prop;
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/drm_mode_object.c b/drivers/gpu/drm/drm_mode_object.c
index da9a9adbcc98..1055533792f3 100644
--- a/drivers/gpu/drm/drm_mode_object.c
+++ b/drivers/gpu/drm/drm_mode_object.c
@@ -233,6 +233,9 @@ int drm_object_property_set_value(struct drm_mode_object *obj,
 {
 	int i;
 
+	WARN_ON(drm_drv_uses_atomic_modeset(property->dev) &&
+		!(property->flags & DRM_MODE_PROP_IMMUTABLE));
+
 	for (i = 0; i < obj->properties->count; i++) {
 		if (obj->properties->properties[i] == property) {
 			obj->properties->values[i] = val;
@@ -244,24 +247,7 @@ int drm_object_property_set_value(struct drm_mode_object *obj,
 }
 EXPORT_SYMBOL(drm_object_property_set_value);
 
-/**
- * drm_object_property_get_value - retrieve the value of a property
- * @obj: drm mode object to get property value from
- * @property: property to retrieve
- * @val: storage for the property value
- *
- * This function retrieves the softare state of the given property for the given
- * property. Since there is no driver callback to retrieve the current property
- * value this might be out of sync with the hardware, depending upon the driver
- * and property.
- *
- * Atomic drivers should never call this function directly, the core will read
- * out property values through the various ->atomic_get_property callbacks.
- *
- * Returns:
- * Zero on success, error code on failure.
- */
-int drm_object_property_get_value(struct drm_mode_object *obj,
+int __drm_object_property_get_value(struct drm_mode_object *obj,
 				  struct drm_property *property, uint64_t *val)
 {
 	int i;
@@ -284,6 +270,31 @@ int drm_object_property_get_value(struct drm_mode_object *obj,
 
 	return -EINVAL;
 }
+
+/**
+ * drm_object_property_get_value - retrieve the value of a property
+ * @obj: drm mode object to get property value from
+ * @property: property to retrieve
+ * @val: storage for the property value
+ *
+ * This function retrieves the softare state of the given property for the given
+ * property. Since there is no driver callback to retrieve the current property
+ * value this might be out of sync with the hardware, depending upon the driver
+ * and property.
+ *
+ * Atomic drivers should never call this function directly, the core will read
+ * out property values through the various ->atomic_get_property callbacks.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
+int drm_object_property_get_value(struct drm_mode_object *obj,
+				  struct drm_property *property, uint64_t *val)
+{
+	WARN_ON(drm_drv_uses_atomic_modeset(property->dev));
+
+	return __drm_object_property_get_value(obj, property, val);
+}
 EXPORT_SYMBOL(drm_object_property_get_value);
 
 /* helper for getconnector and getproperties ioctls */
@@ -302,7 +313,7 @@ int drm_mode_object_get_properties(struct drm_mode_object *obj, bool atomic,
 			continue;
 
 		if (*arg_count_props > count) {
-			ret = drm_object_property_get_value(obj, prop, &val);
+			ret = __drm_object_property_get_value(obj, prop, &val);
 			if (ret)
 				return ret;
 
@@ -381,6 +392,83 @@ struct drm_property *drm_mode_obj_find_prop_id(struct drm_mode_object *obj,
 	return NULL;
 }
 
+static int set_property_legacy(struct drm_mode_object *obj,
+			       struct drm_property *prop,
+			       uint64_t prop_value)
+{
+	struct drm_device *dev = prop->dev;
+	struct drm_mode_object *ref;
+	int ret = -EINVAL;
+
+	if (!drm_property_change_valid_get(prop, prop_value, &ref))
+		return -EINVAL;
+
+	drm_modeset_lock_all(dev);
+	switch (obj->type) {
+	case DRM_MODE_OBJECT_CONNECTOR:
+		ret = drm_mode_connector_set_obj_prop(obj, prop,
+						      prop_value);
+		break;
+	case DRM_MODE_OBJECT_CRTC:
+		ret = drm_mode_crtc_set_obj_prop(obj, prop, prop_value);
+		break;
+	case DRM_MODE_OBJECT_PLANE:
+		ret = drm_mode_plane_set_obj_prop(obj_to_plane(obj),
+						  prop, prop_value);
+		break;
+	}
+	drm_property_change_valid_put(prop, ref);
+	drm_modeset_unlock_all(dev);
+
+	return ret;
+}
+
+static int set_property_atomic(struct drm_mode_object *obj,
+			       struct drm_property *prop,
+			       uint64_t prop_value)
+{
+	struct drm_device *dev = prop->dev;
+	struct drm_atomic_state *state;
+	struct drm_modeset_acquire_ctx ctx;
+	int ret;
+
+	drm_modeset_acquire_init(&ctx, 0);
+
+	state = drm_atomic_state_alloc(dev);
+	if (!state)
+		return -ENOMEM;
+	state->acquire_ctx = &ctx;
+retry:
+	if (prop == state->dev->mode_config.dpms_property) {
+		if (obj->type != DRM_MODE_OBJECT_CONNECTOR) {
+			ret = -EINVAL;
+			goto out;
+		}
+
+		ret = drm_atomic_connector_commit_dpms(state,
+						       obj_to_connector(obj),
+						       prop_value);
+	} else {
+		ret = drm_atomic_set_property(state, obj, prop, prop_value);
+		if (ret)
+			goto out;
+		ret = drm_atomic_commit(state);
+	}
+out:
+	if (ret == -EDEADLK) {
+		drm_atomic_state_clear(state);
+		drm_modeset_backoff(&ctx);
+		goto retry;
+	}
+
+	drm_atomic_state_put(state);
+
+	drm_modeset_drop_locks(&ctx);
+	drm_modeset_acquire_fini(&ctx);
+
+	return ret;
+}
+
 int drm_mode_obj_set_property_ioctl(struct drm_device *dev, void *data,
 				    struct drm_file *file_priv)
 {
@@ -388,18 +476,13 @@ int drm_mode_obj_set_property_ioctl(struct drm_device *dev, void *data,
 	struct drm_mode_object *arg_obj;
 	struct drm_property *property;
 	int ret = -EINVAL;
-	struct drm_mode_object *ref;
 
 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
 		return -EINVAL;
 
-	drm_modeset_lock_all(dev);
-
 	arg_obj = drm_mode_object_find(dev, arg->obj_id, arg->obj_type);
-	if (!arg_obj) {
-		ret = -ENOENT;
-		goto out;
-	}
+	if (!arg_obj)
+		return -ENOENT;
 
 	if (!arg_obj->properties)
 		goto out_unref;
@@ -408,28 +491,12 @@ int drm_mode_obj_set_property_ioctl(struct drm_device *dev, void *data,
 	if (!property)
 		goto out_unref;
 
-	if (!drm_property_change_valid_get(property, arg->value, &ref))
-		goto out_unref;
-
-	switch (arg_obj->type) {
-	case DRM_MODE_OBJECT_CONNECTOR:
-		ret = drm_mode_connector_set_obj_prop(arg_obj, property,
-						      arg->value);
-		break;
-	case DRM_MODE_OBJECT_CRTC:
-		ret = drm_mode_crtc_set_obj_prop(arg_obj, property, arg->value);
-		break;
-	case DRM_MODE_OBJECT_PLANE:
-		ret = drm_mode_plane_set_obj_prop(obj_to_plane(arg_obj),
-						  property, arg->value);
-		break;
-	}
-
-	drm_property_change_valid_put(property, ref);
+	if (drm_drv_uses_atomic_modeset(property->dev))
+		ret = set_property_atomic(arg_obj, property, arg->value);
+	else
+		ret = set_property_legacy(arg_obj, property, arg->value);
 
 out_unref:
 	drm_mode_object_put(arg_obj);
-out:
-	drm_modeset_unlock_all(dev);
 	return ret;
 }
diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c
index f2493b9b82e6..4a3f68a33844 100644
--- a/drivers/gpu/drm/drm_modes.c
+++ b/drivers/gpu/drm/drm_modes.c
@@ -709,8 +709,8 @@ int of_get_drm_display_mode(struct device_node *np,
 	if (bus_flags)
 		drm_bus_flags_from_videomode(&vm, bus_flags);
 
-	pr_debug("%s: got %dx%d display mode from %s\n",
-		of_node_full_name(np), vm.hactive, vm.vactive, np->name);
+	pr_debug("%pOF: got %dx%d display mode from %s\n",
+		np, vm.hactive, vm.vactive, np->name);
 	drm_mode_debug_printmodeline(dmode);
 
 	return 0;
@@ -1083,6 +1083,34 @@ drm_mode_validate_size(const struct drm_display_mode *mode,
 }
 EXPORT_SYMBOL(drm_mode_validate_size);
 
+/**
+ * drm_mode_validate_ycbcr420 - add 'ycbcr420-only' modes only when allowed
+ * @mode: mode to check
+ * @connector: drm connector under action
+ *
+ * This function is a helper which can be used to filter out any YCBCR420
+ * only mode, when the source doesn't support it.
+ *
+ * Returns:
+ * The mode status
+ */
+enum drm_mode_status
+drm_mode_validate_ycbcr420(const struct drm_display_mode *mode,
+			   struct drm_connector *connector)
+{
+	u8 vic = drm_match_cea_mode(mode);
+	enum drm_mode_status status = MODE_OK;
+	struct drm_hdmi_info *hdmi = &connector->display_info.hdmi;
+
+	if (test_bit(vic, hdmi->y420_vdb_modes)) {
+		if (!connector->ycbcr_420_allowed)
+			status = MODE_NO_420;
+	}
+
+	return status;
+}
+EXPORT_SYMBOL(drm_mode_validate_ycbcr420);
+
 #define MODE_STATUS(status) [MODE_ ## status + 3] = #status
 
 static const char * const drm_mode_status_names[] = {
@@ -1122,6 +1150,7 @@ static const char * const drm_mode_status_names[] = {
 	MODE_STATUS(ONE_SIZE),
 	MODE_STATUS(NO_REDUCED),
 	MODE_STATUS(NO_STEREO),
+	MODE_STATUS(NO_420),
 	MODE_STATUS(STALE),
 	MODE_STATUS(BAD),
 	MODE_STATUS(ERROR),
@@ -1576,3 +1605,61 @@ int drm_mode_convert_umode(struct drm_display_mode *out,
 out:
 	return ret;
 }
+
+/**
+ * drm_mode_is_420_only - if a given videomode can be only supported in YCBCR420
+ * output format
+ *
+ * @display: display under action
+ * @mode: video mode to be tested.
+ *
+ * Returns:
+ * true if the mode can be supported in YCBCR420 format
+ * false if not.
+ */
+bool drm_mode_is_420_only(const struct drm_display_info *display,
+			  const struct drm_display_mode *mode)
+{
+	u8 vic = drm_match_cea_mode(mode);
+
+	return test_bit(vic, display->hdmi.y420_vdb_modes);
+}
+EXPORT_SYMBOL(drm_mode_is_420_only);
+
+/**
+ * drm_mode_is_420_also - if a given videomode can be supported in YCBCR420
+ * output format also (along with RGB/YCBCR444/422)
+ *
+ * @display: display under action.
+ * @mode: video mode to be tested.
+ *
+ * Returns:
+ * true if the mode can be support YCBCR420 format
+ * false if not.
+ */
+bool drm_mode_is_420_also(const struct drm_display_info *display,
+			  const struct drm_display_mode *mode)
+{
+	u8 vic = drm_match_cea_mode(mode);
+
+	return test_bit(vic, display->hdmi.y420_cmdb_modes);
+}
+EXPORT_SYMBOL(drm_mode_is_420_also);
+/**
+ * drm_mode_is_420 - if a given videomode can be supported in YCBCR420
+ * output format
+ *
+ * @display: display under action.
+ * @mode: video mode to be tested.
+ *
+ * Returns:
+ * true if the mode can be supported in YCBCR420 format
+ * false if not.
+ */
+bool drm_mode_is_420(const struct drm_display_info *display,
+		     const struct drm_display_mode *mode)
+{
+	return drm_mode_is_420_only(display, mode) ||
+		drm_mode_is_420_also(display, mode);
+}
+EXPORT_SYMBOL(drm_mode_is_420);
diff --git a/drivers/gpu/drm/drm_modeset_helper.c b/drivers/gpu/drm/drm_modeset_helper.c
index 2b33825f2f93..9cb1eede0b4d 100644
--- a/drivers/gpu/drm/drm_modeset_helper.c
+++ b/drivers/gpu/drm/drm_modeset_helper.c
@@ -124,6 +124,7 @@ static struct drm_plane *create_primary_plane(struct drm_device *dev)
 				       &drm_primary_helper_funcs,
 				       safe_modeset_formats,
 				       ARRAY_SIZE(safe_modeset_formats),
+				       NULL,
 				       DRM_PLANE_TYPE_PRIMARY, NULL);
 	if (ret) {
 		kfree(primary);
diff --git a/drivers/gpu/drm/drm_modeset_lock.c b/drivers/gpu/drm/drm_modeset_lock.c
index 64ef09a6cccb..af4e906c630d 100644
--- a/drivers/gpu/drm/drm_modeset_lock.c
+++ b/drivers/gpu/drm/drm_modeset_lock.c
@@ -52,7 +52,12 @@
  *     drm_modeset_drop_locks(&ctx);
  *     drm_modeset_acquire_fini(&ctx);
  *
- * On top of of these per-object locks using &ww_mutex there's also an overall
+ * If all that is needed is a single modeset lock, then the &struct
+ * drm_modeset_acquire_ctx is not needed and the locking can be simplified
+ * by passing a NULL instead of ctx in the drm_modeset_lock()
+ * call and, when done, by calling drm_modeset_unlock().
+ *
+ * On top of these per-object locks using &ww_mutex there's also an overall
  * &drm_mode_config.mutex, for protecting everything else. Mostly this means
  * probe state of connectors, and preventing hotplug add/removal of connectors.
  *
@@ -313,11 +318,14 @@ EXPORT_SYMBOL(drm_modeset_lock_init);
  * @lock: lock to take
  * @ctx: acquire ctx
  *
- * If ctx is not NULL, then its ww acquire context is used and the
+ * If @ctx is not NULL, then its ww acquire context is used and the
  * lock will be tracked by the context and can be released by calling
  * drm_modeset_drop_locks().  If -EDEADLK is returned, this means a
  * deadlock scenario has been detected and it is an error to attempt
  * to take any more locks without first calling drm_modeset_backoff().
+ *
+ * If @ctx is NULL then the function call behaves like a normal,
+ * non-nesting mutex_lock() call.
  */
 int drm_modeset_lock(struct drm_modeset_lock *lock,
 		struct drm_modeset_acquire_ctx *ctx)
diff --git a/drivers/gpu/drm/drm_of.c b/drivers/gpu/drm/drm_of.c
index 2120f33bdf4a..8dafbdfcd2ea 100644
--- a/drivers/gpu/drm/drm_of.c
+++ b/drivers/gpu/drm/drm_of.c
@@ -160,8 +160,8 @@ int drm_of_component_probe(struct device *dev,
 				of_node_put(remote);
 				continue;
 			} else if (!of_device_is_available(remote->parent)) {
-				dev_warn(dev, "parent device of %s is not available\n",
-					 remote->full_name);
+				dev_warn(dev, "parent device of %pOF is not available\n",
+					 remote);
 				of_node_put(remote);
 				continue;
 			}
diff --git a/drivers/gpu/drm/drm_pci.c b/drivers/gpu/drm/drm_pci.c
index 1eb4fc3eee20..1235c9877d6f 100644
--- a/drivers/gpu/drm/drm_pci.c
+++ b/drivers/gpu/drm/drm_pci.c
@@ -149,7 +149,6 @@ int drm_pci_set_busid(struct drm_device *dev, struct drm_master *master)
 	master->unique_len = strlen(master->unique);
 	return 0;
 }
-EXPORT_SYMBOL(drm_pci_set_busid);
 
 static int drm_pci_irq_by_busid(struct drm_device *dev, struct drm_irq_busid *p)
 {
@@ -281,20 +280,15 @@ err_free:
 EXPORT_SYMBOL(drm_get_pci_dev);
 
 /**
- * drm_pci_init - Register matching PCI devices with the DRM subsystem
+ * drm_legacy_pci_init - shadow-attach a legacy DRM PCI driver
  * @driver: DRM device driver
  * @pdriver: PCI device driver
  *
- * Initializes a drm_device structures, registering the stubs and initializing
- * the AGP device.
- *
- * NOTE: This function is deprecated. Modern modesetting drm drivers should use
- * pci_register_driver() directly, this function only provides shadow-binding
- * support for old legacy drivers on top of that core pci function.
+ * This is only used by legacy dri1 drivers and deprecated.
  *
  * Return: 0 on success or a negative error code on failure.
  */
-int drm_pci_init(struct drm_driver *driver, struct pci_driver *pdriver)
+int drm_legacy_pci_init(struct drm_driver *driver, struct pci_driver *pdriver)
 {
 	struct pci_dev *pdev = NULL;
 	const struct pci_device_id *pid;
@@ -302,8 +296,8 @@ int drm_pci_init(struct drm_driver *driver, struct pci_driver *pdriver)
 
 	DRM_DEBUG("\n");
 
-	if (!(driver->driver_features & DRIVER_LEGACY))
-		return pci_register_driver(pdriver);
+	if (WARN_ON(!(driver->driver_features & DRIVER_LEGACY)))
+		return -EINVAL;
 
 	/* If not using KMS, fall back to stealth mode manual scanning. */
 	INIT_LIST_HEAD(&driver->legacy_dev_list);
@@ -330,6 +324,7 @@ int drm_pci_init(struct drm_driver *driver, struct pci_driver *pdriver)
 	}
 	return 0;
 }
+EXPORT_SYMBOL(drm_legacy_pci_init);
 
 int drm_pcie_get_speed_cap_mask(struct drm_device *dev, u32 *mask)
 {
@@ -391,11 +386,6 @@ EXPORT_SYMBOL(drm_pcie_get_max_link_width);
 
 #else
 
-int drm_pci_init(struct drm_driver *driver, struct pci_driver *pdriver)
-{
-	return -1;
-}
-
 void drm_pci_agp_destroy(struct drm_device *dev) {}
 
 int drm_irq_by_busid(struct drm_device *dev, void *data,
@@ -405,27 +395,21 @@ int drm_irq_by_busid(struct drm_device *dev, void *data,
 }
 #endif
 
-EXPORT_SYMBOL(drm_pci_init);
-
 /**
- * drm_pci_exit - Unregister matching PCI devices from the DRM subsystem
+ * drm_legacy_pci_exit - unregister shadow-attach legacy DRM driver
  * @driver: DRM device driver
  * @pdriver: PCI device driver
  *
- * Unregisters one or more devices matched by a PCI driver from the DRM
- * subsystem.
- *
- * NOTE: This function is deprecated. Modern modesetting drm drivers should use
- * pci_unregister_driver() directly, this function only provides shadow-binding
- * support for old legacy drivers on top of that core pci function.
+ * Unregister a DRM driver shadow-attached through drm_legacy_pci_init(). This
+ * is deprecated and only used by dri1 drivers.
  */
-void drm_pci_exit(struct drm_driver *driver, struct pci_driver *pdriver)
+void drm_legacy_pci_exit(struct drm_driver *driver, struct pci_driver *pdriver)
 {
 	struct drm_device *dev, *tmp;
 	DRM_DEBUG("\n");
 
 	if (!(driver->driver_features & DRIVER_LEGACY)) {
-		pci_unregister_driver(pdriver);
+		WARN_ON(1);
 	} else {
 		list_for_each_entry_safe(dev, tmp, &driver->legacy_dev_list,
 					 legacy_dev_list) {
@@ -435,4 +419,4 @@ void drm_pci_exit(struct drm_driver *driver, struct pci_driver *pdriver)
 	}
 	DRM_INFO("Module unloaded\n");
 }
-EXPORT_SYMBOL(drm_pci_exit);
+EXPORT_SYMBOL(drm_legacy_pci_exit);
diff --git a/drivers/gpu/drm/drm_plane.c b/drivers/gpu/drm/drm_plane.c
index e40c12fabbde..7a00351d5b5d 100644
--- a/drivers/gpu/drm/drm_plane.c
+++ b/drivers/gpu/drm/drm_plane.c
@@ -62,6 +62,87 @@ static unsigned int drm_num_planes(struct drm_device *dev)
 	return num;
 }
 
+static inline u32 *
+formats_ptr(struct drm_format_modifier_blob *blob)
+{
+	return (u32 *)(((char *)blob) + blob->formats_offset);
+}
+
+static inline struct drm_format_modifier *
+modifiers_ptr(struct drm_format_modifier_blob *blob)
+{
+	return (struct drm_format_modifier *)(((char *)blob) + blob->modifiers_offset);
+}
+
+static int create_in_format_blob(struct drm_device *dev, struct drm_plane *plane)
+{
+	const struct drm_mode_config *config = &dev->mode_config;
+	struct drm_property_blob *blob;
+	struct drm_format_modifier *mod;
+	size_t blob_size, formats_size, modifiers_size;
+	struct drm_format_modifier_blob *blob_data;
+	unsigned int i, j;
+
+	formats_size = sizeof(__u32) * plane->format_count;
+	if (WARN_ON(!formats_size)) {
+		/* 0 formats are never expected */
+		return 0;
+	}
+
+	modifiers_size =
+		sizeof(struct drm_format_modifier) * plane->modifier_count;
+
+	blob_size = sizeof(struct drm_format_modifier_blob);
+	/* Modifiers offset is a pointer to a struct with a 64 bit field so it
+	 * should be naturally aligned to 8B.
+	 */
+	BUILD_BUG_ON(sizeof(struct drm_format_modifier_blob) % 8);
+	blob_size += ALIGN(formats_size, 8);
+	blob_size += modifiers_size;
+
+	blob = drm_property_create_blob(dev, blob_size, NULL);
+	if (IS_ERR(blob))
+		return -1;
+
+	blob_data = (struct drm_format_modifier_blob *)blob->data;
+	blob_data->version = FORMAT_BLOB_CURRENT;
+	blob_data->count_formats = plane->format_count;
+	blob_data->formats_offset = sizeof(struct drm_format_modifier_blob);
+	blob_data->count_modifiers = plane->modifier_count;
+
+	blob_data->modifiers_offset =
+		ALIGN(blob_data->formats_offset + formats_size, 8);
+
+	memcpy(formats_ptr(blob_data), plane->format_types, formats_size);
+
+	/* If we can't determine support, just bail */
+	if (!plane->funcs->format_mod_supported)
+		goto done;
+
+	mod = modifiers_ptr(blob_data);
+	for (i = 0; i < plane->modifier_count; i++) {
+		for (j = 0; j < plane->format_count; j++) {
+			if (plane->funcs->format_mod_supported(plane,
+							       plane->format_types[j],
+							       plane->modifiers[i])) {
+
+				mod->formats |= 1ULL << j;
+			}
+		}
+
+		mod->modifier = plane->modifiers[i];
+		mod->offset = 0;
+		mod->pad = 0;
+		mod++;
+	}
+
+done:
+	drm_object_attach_property(&plane->base, config->modifiers_property,
+				   blob->base.id);
+
+	return 0;
+}
+
 /**
  * drm_universal_plane_init - Initialize a new universal plane object
  * @dev: DRM device
@@ -70,6 +151,8 @@ static unsigned int drm_num_planes(struct drm_device *dev)
  * @funcs: callbacks for the new plane
  * @formats: array of supported formats (DRM_FORMAT\_\*)
  * @format_count: number of elements in @formats
+ * @format_modifiers: array of struct drm_format modifiers terminated by
+ *                    DRM_FORMAT_MOD_INVALID
  * @type: type of plane (overlay, primary, cursor)
  * @name: printf style format string for the plane name, or NULL for default name
  *
@@ -82,10 +165,12 @@ int drm_universal_plane_init(struct drm_device *dev, struct drm_plane *plane,
 			     uint32_t possible_crtcs,
 			     const struct drm_plane_funcs *funcs,
 			     const uint32_t *formats, unsigned int format_count,
+			     const uint64_t *format_modifiers,
 			     enum drm_plane_type type,
 			     const char *name, ...)
 {
 	struct drm_mode_config *config = &dev->mode_config;
+	unsigned int format_modifier_count = 0;
 	int ret;
 
 	ret = drm_mode_object_add(dev, &plane->base, DRM_MODE_OBJECT_PLANE);
@@ -105,6 +190,31 @@ int drm_universal_plane_init(struct drm_device *dev, struct drm_plane *plane,
 		return -ENOMEM;
 	}
 
+	/*
+	 * First driver to need more than 64 formats needs to fix this. Each
+	 * format is encoded as a bit and the current code only supports a u64.
+	 */
+	if (WARN_ON(format_count > 64))
+		return -EINVAL;
+
+	if (format_modifiers) {
+		const uint64_t *temp_modifiers = format_modifiers;
+		while (*temp_modifiers++ != DRM_FORMAT_MOD_INVALID)
+			format_modifier_count++;
+	}
+
+	plane->modifier_count = format_modifier_count;
+	plane->modifiers = kmalloc_array(format_modifier_count,
+					 sizeof(format_modifiers[0]),
+					 GFP_KERNEL);
+
+	if (format_modifier_count && !plane->modifiers) {
+		DRM_DEBUG_KMS("out of memory when allocating plane\n");
+		kfree(plane->format_types);
+		drm_mode_object_unregister(dev, &plane->base);
+		return -ENOMEM;
+	}
+
 	if (name) {
 		va_list ap;
 
@@ -117,12 +227,15 @@ int drm_universal_plane_init(struct drm_device *dev, struct drm_plane *plane,
 	}
 	if (!plane->name) {
 		kfree(plane->format_types);
+		kfree(plane->modifiers);
 		drm_mode_object_unregister(dev, &plane->base);
 		return -ENOMEM;
 	}
 
 	memcpy(plane->format_types, formats, format_count * sizeof(uint32_t));
 	plane->format_count = format_count;
+	memcpy(plane->modifiers, format_modifiers,
+	       format_modifier_count * sizeof(format_modifiers[0]));
 	plane->possible_crtcs = possible_crtcs;
 	plane->type = type;
 
@@ -149,6 +262,9 @@ int drm_universal_plane_init(struct drm_device *dev, struct drm_plane *plane,
 		drm_object_attach_property(&plane->base, config->prop_src_h, 0);
 	}
 
+	if (config->allow_fb_modifiers)
+		create_in_format_blob(dev, plane);
+
 	return 0;
 }
 EXPORT_SYMBOL(drm_universal_plane_init);
@@ -205,7 +321,8 @@ int drm_plane_init(struct drm_device *dev, struct drm_plane *plane,
 
 	type = is_primary ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY;
 	return drm_universal_plane_init(dev, plane, possible_crtcs, funcs,
-					formats, format_count, type, NULL);
+					formats, format_count,
+					NULL, type, NULL);
 }
 EXPORT_SYMBOL(drm_plane_init);
 
@@ -224,6 +341,7 @@ void drm_plane_cleanup(struct drm_plane *plane)
 	drm_modeset_lock_fini(&plane->mutex);
 
 	kfree(plane->format_types);
+	kfree(plane->modifiers);
 	drm_mode_object_unregister(dev, &plane->base);
 
 	BUG_ON(list_empty(&plane->head));
diff --git a/drivers/gpu/drm/drm_probe_helper.c b/drivers/gpu/drm/drm_probe_helper.c
index 00e6832a8c1a..904966cde32b 100644
--- a/drivers/gpu/drm/drm_probe_helper.c
+++ b/drivers/gpu/drm/drm_probe_helper.c
@@ -528,6 +528,10 @@ retry:
 		if (mode->status == MODE_OK)
 			mode->status = drm_mode_validate_pipeline(mode,
 								  connector);
+
+		if (mode->status == MODE_OK)
+			mode->status = drm_mode_validate_ycbcr420(mode,
+								  connector);
 	}
 
 prune:
diff --git a/drivers/gpu/drm/drm_property.c b/drivers/gpu/drm/drm_property.c
index 3e88fa24eab3..bc5128203056 100644
--- a/drivers/gpu/drm/drm_property.c
+++ b/drivers/gpu/drm/drm_property.c
@@ -709,6 +709,29 @@ err_created:
 }
 EXPORT_SYMBOL(drm_property_replace_global_blob);
 
+/**
+ * drm_property_replace_blob - replace a blob property
+ * @blob: a pointer to the member blob to be replaced
+ * @new_blob: the new blob to replace with
+ *
+ * Return: true if the blob was in fact replaced.
+ */
+bool drm_property_replace_blob(struct drm_property_blob **blob,
+			       struct drm_property_blob *new_blob)
+{
+	struct drm_property_blob *old_blob = *blob;
+
+	if (old_blob == new_blob)
+		return false;
+
+	drm_property_blob_put(old_blob);
+	if (new_blob)
+		drm_property_blob_get(new_blob);
+	*blob = new_blob;
+	return true;
+}
+EXPORT_SYMBOL(drm_property_replace_blob);
+
 int drm_mode_getblob_ioctl(struct drm_device *dev,
 			   void *data, struct drm_file *file_priv)
 {
diff --git a/drivers/gpu/drm/drm_scdc_helper.c b/drivers/gpu/drm/drm_scdc_helper.c
index 3cd96a95736d..935653eb3616 100644
--- a/drivers/gpu/drm/drm_scdc_helper.c
+++ b/drivers/gpu/drm/drm_scdc_helper.c
@@ -102,7 +102,7 @@ ssize_t drm_scdc_write(struct i2c_adapter *adapter, u8 offset,
 	void *data;
 	int err;
 
-	data = kmalloc(1 + size, GFP_TEMPORARY);
+	data = kmalloc(1 + size, GFP_KERNEL);
 	if (!data)
 		return -ENOMEM;
 
@@ -194,19 +194,26 @@ EXPORT_SYMBOL(drm_scdc_set_scrambling);
  * @adapter: I2C adapter for DDC channel
  * @set: ret or reset the high clock ratio
  *
- * TMDS clock ratio calculations go like this:
- * TMDS character = 10 bit TMDS encoded value
- * TMDS character rate = The rate at which TMDS characters are transmitted(Mcsc)
- * TMDS bit rate = 10x TMDS character rate
- * As per the spec:
- * TMDS clock rate for pixel clock < 340 MHz = 1x the character rate
- *	= 1/10 pixel clock rate
- * TMDS clock rate for pixel clock > 340 MHz = 0.25x the character rate
- *	= 1/40 pixel clock rate
- *
- * Writes to the TMDS config register over SCDC channel, and:
- * sets TMDS clock ratio to 1/40 when set = 1
- * sets TMDS clock ratio to 1/10 when set = 0
+ *
+ *	TMDS clock ratio calculations go like this:
+ *		TMDS character = 10 bit TMDS encoded value
+ *
+ *		TMDS character rate = The rate at which TMDS characters are
+ *		transmitted (Mcsc)
+ *
+ *		TMDS bit rate = 10x TMDS character rate
+ *
+ *	As per the spec:
+ *		TMDS clock rate for pixel clock < 340 MHz = 1x the character
+ *		rate = 1/10 pixel clock rate
+ *
+ *		TMDS clock rate for pixel clock > 340 MHz = 0.25x the character
+ *		rate = 1/40 pixel clock rate
+ *
+ *	Writes to the TMDS config register over SCDC channel, and:
+ *		sets TMDS clock ratio to 1/40 when set = 1
+ *
+ *		sets TMDS clock ratio to 1/10 when set = 0
  *
  * Returns:
  * True if write is successful, false otherwise.
diff --git a/drivers/gpu/drm/drm_simple_kms_helper.c b/drivers/gpu/drm/drm_simple_kms_helper.c
index e084f9f8ca66..dc9fd109de14 100644
--- a/drivers/gpu/drm/drm_simple_kms_helper.c
+++ b/drivers/gpu/drm/drm_simple_kms_helper.c
@@ -37,10 +37,18 @@ static const struct drm_encoder_funcs drm_simple_kms_encoder_funcs = {
 static int drm_simple_kms_crtc_check(struct drm_crtc *crtc,
 				     struct drm_crtc_state *state)
 {
+	bool has_primary = state->plane_mask &
+			   BIT(drm_plane_index(crtc->primary));
+
+	/* We always want to have an active plane with an active CRTC */
+	if (has_primary != state->enable)
+		return -EINVAL;
+
 	return drm_atomic_add_affected_planes(state->state, crtc);
 }
 
-static void drm_simple_kms_crtc_enable(struct drm_crtc *crtc)
+static void drm_simple_kms_crtc_enable(struct drm_crtc *crtc,
+				       struct drm_crtc_state *old_state)
 {
 	struct drm_simple_display_pipe *pipe;
 
@@ -51,7 +59,8 @@ static void drm_simple_kms_crtc_enable(struct drm_crtc *crtc)
 	pipe->funcs->enable(pipe, crtc->state);
 }
 
-static void drm_simple_kms_crtc_disable(struct drm_crtc *crtc)
+static void drm_simple_kms_crtc_disable(struct drm_crtc *crtc,
+					struct drm_crtc_state *old_state)
 {
 	struct drm_simple_display_pipe *pipe;
 
@@ -64,8 +73,8 @@ static void drm_simple_kms_crtc_disable(struct drm_crtc *crtc)
 
 static const struct drm_crtc_helper_funcs drm_simple_kms_crtc_helper_funcs = {
 	.atomic_check = drm_simple_kms_crtc_check,
-	.disable = drm_simple_kms_crtc_disable,
-	.enable = drm_simple_kms_crtc_enable,
+	.atomic_enable = drm_simple_kms_crtc_enable,
+	.atomic_disable = drm_simple_kms_crtc_disable,
 };
 
 static const struct drm_crtc_funcs drm_simple_kms_crtc_funcs = {
@@ -88,9 +97,6 @@ static int drm_simple_kms_plane_atomic_check(struct drm_plane *plane,
 	pipe = container_of(plane, struct drm_simple_display_pipe, plane);
 	crtc_state = drm_atomic_get_new_crtc_state(plane_state->state,
 						   &pipe->crtc);
-	if (crtc_state->enable != !!plane_state->crtc)
-		return -EINVAL; /* plane must match crtc enable state */
-
 	if (!crtc_state->enable)
 		return 0; /* nothing to check when disabling or disabled */
 
@@ -193,6 +199,7 @@ EXPORT_SYMBOL(drm_simple_display_pipe_attach_bridge);
  * @funcs: callbacks for the display pipe (optional)
  * @formats: array of supported formats (DRM_FORMAT\_\*)
  * @format_count: number of elements in @formats
+ * @format_modifiers: array of formats modifiers
  * @connector: connector to attach and register (optional)
  *
  * Sets up a display pipeline which consist of a really simple
@@ -213,6 +220,7 @@ int drm_simple_display_pipe_init(struct drm_device *dev,
 			struct drm_simple_display_pipe *pipe,
 			const struct drm_simple_display_pipe_funcs *funcs,
 			const uint32_t *formats, unsigned int format_count,
+			const uint64_t *format_modifiers,
 			struct drm_connector *connector)
 {
 	struct drm_encoder *encoder = &pipe->encoder;
@@ -227,6 +235,7 @@ int drm_simple_display_pipe_init(struct drm_device *dev,
 	ret = drm_universal_plane_init(dev, plane, 0,
 				       &drm_simple_kms_plane_funcs,
 				       formats, format_count,
+				       format_modifiers,
 				       DRM_PLANE_TYPE_PRIMARY, NULL);
 	if (ret)
 		return ret;
diff --git a/drivers/gpu/drm/drm_syncobj.c b/drivers/gpu/drm/drm_syncobj.c
index 789ba0b37f7b..0422b8c2c2e7 100644
--- a/drivers/gpu/drm/drm_syncobj.c
+++ b/drivers/gpu/drm/drm_syncobj.c
@@ -1,5 +1,7 @@
 /*
  * Copyright 2017 Red Hat
+ * Parts ported from amdgpu (fence wait code).
+ * Copyright 2016 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -31,6 +33,9 @@
  * that contain an optional fence. The fence can be updated with a new
  * fence, or be NULL.
  *
+ * syncobj's can be waited upon, where it will wait for the underlying
+ * fence.
+ *
  * syncobj's can be export to fd's and back, these fd's are opaque and
  * have no other use case, except passing the syncobj between processes.
  *
@@ -46,6 +51,7 @@
 #include <linux/fs.h>
 #include <linux/anon_inodes.h>
 #include <linux/sync_file.h>
+#include <linux/sched/signal.h>
 
 #include "drm_internal.h"
 #include <drm/drm_syncobj.h>
@@ -75,6 +81,75 @@ struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
 }
 EXPORT_SYMBOL(drm_syncobj_find);
 
+static void drm_syncobj_add_callback_locked(struct drm_syncobj *syncobj,
+					    struct drm_syncobj_cb *cb,
+					    drm_syncobj_func_t func)
+{
+	cb->func = func;
+	list_add_tail(&cb->node, &syncobj->cb_list);
+}
+
+static int drm_syncobj_fence_get_or_add_callback(struct drm_syncobj *syncobj,
+						 struct dma_fence **fence,
+						 struct drm_syncobj_cb *cb,
+						 drm_syncobj_func_t func)
+{
+	int ret;
+
+	*fence = drm_syncobj_fence_get(syncobj);
+	if (*fence)
+		return 1;
+
+	spin_lock(&syncobj->lock);
+	/* We've already tried once to get a fence and failed.  Now that we
+	 * have the lock, try one more time just to be sure we don't add a
+	 * callback when a fence has already been set.
+	 */
+	if (syncobj->fence) {
+		*fence = dma_fence_get(syncobj->fence);
+		ret = 1;
+	} else {
+		*fence = NULL;
+		drm_syncobj_add_callback_locked(syncobj, cb, func);
+		ret = 0;
+	}
+	spin_unlock(&syncobj->lock);
+
+	return ret;
+}
+
+/**
+ * drm_syncobj_add_callback - adds a callback to syncobj::cb_list
+ * @syncobj: Sync object to which to add the callback
+ * @cb: Callback to add
+ * @func: Func to use when initializing the drm_syncobj_cb struct
+ *
+ * This adds a callback to be called next time the fence is replaced
+ */
+void drm_syncobj_add_callback(struct drm_syncobj *syncobj,
+			      struct drm_syncobj_cb *cb,
+			      drm_syncobj_func_t func)
+{
+	spin_lock(&syncobj->lock);
+	drm_syncobj_add_callback_locked(syncobj, cb, func);
+	spin_unlock(&syncobj->lock);
+}
+EXPORT_SYMBOL(drm_syncobj_add_callback);
+
+/**
+ * drm_syncobj_add_callback - removes a callback to syncobj::cb_list
+ * @syncobj: Sync object from which to remove the callback
+ * @cb: Callback to remove
+ */
+void drm_syncobj_remove_callback(struct drm_syncobj *syncobj,
+				 struct drm_syncobj_cb *cb)
+{
+	spin_lock(&syncobj->lock);
+	list_del_init(&cb->node);
+	spin_unlock(&syncobj->lock);
+}
+EXPORT_SYMBOL(drm_syncobj_remove_callback);
+
 /**
  * drm_syncobj_replace_fence - replace fence in a sync object.
  * @syncobj: Sync object to replace fence in
@@ -86,18 +161,75 @@ void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
 			       struct dma_fence *fence)
 {
 	struct dma_fence *old_fence;
+	struct drm_syncobj_cb *cur, *tmp;
 
 	if (fence)
 		dma_fence_get(fence);
-	old_fence = xchg(&syncobj->fence, fence);
+
+	spin_lock(&syncobj->lock);
+
+	old_fence = syncobj->fence;
+	syncobj->fence = fence;
+
+	if (fence != old_fence) {
+		list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node) {
+			list_del_init(&cur->node);
+			cur->func(syncobj, cur);
+		}
+	}
+
+	spin_unlock(&syncobj->lock);
 
 	dma_fence_put(old_fence);
 }
 EXPORT_SYMBOL(drm_syncobj_replace_fence);
 
-int drm_syncobj_fence_get(struct drm_file *file_private,
-			  u32 handle,
-			  struct dma_fence **fence)
+struct drm_syncobj_null_fence {
+	struct dma_fence base;
+	spinlock_t lock;
+};
+
+static const char *drm_syncobj_null_fence_get_name(struct dma_fence *fence)
+{
+        return "syncobjnull";
+}
+
+static bool drm_syncobj_null_fence_enable_signaling(struct dma_fence *fence)
+{
+    dma_fence_enable_sw_signaling(fence);
+    return !dma_fence_is_signaled(fence);
+}
+
+static const struct dma_fence_ops drm_syncobj_null_fence_ops = {
+	.get_driver_name = drm_syncobj_null_fence_get_name,
+	.get_timeline_name = drm_syncobj_null_fence_get_name,
+	.enable_signaling = drm_syncobj_null_fence_enable_signaling,
+	.wait = dma_fence_default_wait,
+	.release = NULL,
+};
+
+static int drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
+{
+	struct drm_syncobj_null_fence *fence;
+	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
+	if (fence == NULL)
+		return -ENOMEM;
+
+	spin_lock_init(&fence->lock);
+	dma_fence_init(&fence->base, &drm_syncobj_null_fence_ops,
+		       &fence->lock, 0, 0);
+	dma_fence_signal(&fence->base);
+
+	drm_syncobj_replace_fence(syncobj, &fence->base);
+
+	dma_fence_put(&fence->base);
+
+	return 0;
+}
+
+int drm_syncobj_find_fence(struct drm_file *file_private,
+			   u32 handle,
+			   struct dma_fence **fence)
 {
 	struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 	int ret = 0;
@@ -105,14 +237,14 @@ int drm_syncobj_fence_get(struct drm_file *file_private,
 	if (!syncobj)
 		return -ENOENT;
 
-	*fence = dma_fence_get(syncobj->fence);
+	*fence = drm_syncobj_fence_get(syncobj);
 	if (!*fence) {
 		ret = -EINVAL;
 	}
 	drm_syncobj_put(syncobj);
 	return ret;
 }
-EXPORT_SYMBOL(drm_syncobj_fence_get);
+EXPORT_SYMBOL(drm_syncobj_find_fence);
 
 /**
  * drm_syncobj_free - free a sync object.
@@ -125,13 +257,13 @@ void drm_syncobj_free(struct kref *kref)
 	struct drm_syncobj *syncobj = container_of(kref,
 						   struct drm_syncobj,
 						   refcount);
-	dma_fence_put(syncobj->fence);
+	drm_syncobj_replace_fence(syncobj, NULL);
 	kfree(syncobj);
 }
 EXPORT_SYMBOL(drm_syncobj_free);
 
 static int drm_syncobj_create(struct drm_file *file_private,
-			      u32 *handle)
+			      u32 *handle, uint32_t flags)
 {
 	int ret;
 	struct drm_syncobj *syncobj;
@@ -141,6 +273,16 @@ static int drm_syncobj_create(struct drm_file *file_private,
 		return -ENOMEM;
 
 	kref_init(&syncobj->refcount);
+	INIT_LIST_HEAD(&syncobj->cb_list);
+	spin_lock_init(&syncobj->lock);
+
+	if (flags & DRM_SYNCOBJ_CREATE_SIGNALED) {
+		ret = drm_syncobj_assign_null_handle(syncobj);
+		if (ret < 0) {
+			drm_syncobj_put(syncobj);
+			return ret;
+		}
+	}
 
 	idr_preload(GFP_KERNEL);
 	spin_lock(&file_private->syncobj_table_lock);
@@ -307,7 +449,7 @@ int drm_syncobj_export_sync_file(struct drm_file *file_private,
 	if (fd < 0)
 		return fd;
 
-	ret = drm_syncobj_fence_get(file_private, handle, &fence);
+	ret = drm_syncobj_find_fence(file_private, handle, &fence);
 	if (ret)
 		goto err_put_fd;
 
@@ -330,7 +472,6 @@ err_put_fd:
 }
 /**
  * drm_syncobj_open - initalizes syncobj file-private structures at devnode open time
- * @dev: drm_device which is being opened by userspace
  * @file_private: drm file-private structure to set up
  *
  * Called at device open time, sets up the structure for handling refcounting
@@ -354,7 +495,6 @@ drm_syncobj_release_handle(int id, void *ptr, void *data)
 
 /**
  * drm_syncobj_release - release file-private sync object resources
- * @dev: drm_device which is being closed by userspace
  * @file_private: drm file-private structure to clean up
  *
  * Called at close time when the filp is going away.
@@ -379,11 +519,11 @@ drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
 		return -ENODEV;
 
 	/* no valid flags yet */
-	if (args->flags)
+	if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
 		return -EINVAL;
 
 	return drm_syncobj_create(file_private,
-				  &args->handle);
+				  &args->handle, args->flags);
 }
 
 int
@@ -449,3 +589,368 @@ drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
 	return drm_syncobj_fd_to_handle(file_private, args->fd,
 					&args->handle);
 }
+
+struct syncobj_wait_entry {
+	struct task_struct *task;
+	struct dma_fence *fence;
+	struct dma_fence_cb fence_cb;
+	struct drm_syncobj_cb syncobj_cb;
+};
+
+static void syncobj_wait_fence_func(struct dma_fence *fence,
+				    struct dma_fence_cb *cb)
+{
+	struct syncobj_wait_entry *wait =
+		container_of(cb, struct syncobj_wait_entry, fence_cb);
+
+	wake_up_process(wait->task);
+}
+
+static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
+				      struct drm_syncobj_cb *cb)
+{
+	struct syncobj_wait_entry *wait =
+		container_of(cb, struct syncobj_wait_entry, syncobj_cb);
+
+	/* This happens inside the syncobj lock */
+	wait->fence = dma_fence_get(syncobj->fence);
+	wake_up_process(wait->task);
+}
+
+static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
+						  uint32_t count,
+						  uint32_t flags,
+						  signed long timeout,
+						  uint32_t *idx)
+{
+	struct syncobj_wait_entry *entries;
+	struct dma_fence *fence;
+	signed long ret;
+	uint32_t signaled_count, i;
+
+	entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
+	if (!entries)
+		return -ENOMEM;
+
+	/* Walk the list of sync objects and initialize entries.  We do
+	 * this up-front so that we can properly return -EINVAL if there is
+	 * a syncobj with a missing fence and then never have the chance of
+	 * returning -EINVAL again.
+	 */
+	signaled_count = 0;
+	for (i = 0; i < count; ++i) {
+		entries[i].task = current;
+		entries[i].fence = drm_syncobj_fence_get(syncobjs[i]);
+		if (!entries[i].fence) {
+			if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
+				continue;
+			} else {
+				ret = -EINVAL;
+				goto cleanup_entries;
+			}
+		}
+
+		if (dma_fence_is_signaled(entries[i].fence)) {
+			if (signaled_count == 0 && idx)
+				*idx = i;
+			signaled_count++;
+		}
+	}
+
+	/* Initialize ret to the max of timeout and 1.  That way, the
+	 * default return value indicates a successful wait and not a
+	 * timeout.
+	 */
+	ret = max_t(signed long, timeout, 1);
+
+	if (signaled_count == count ||
+	    (signaled_count > 0 &&
+	     !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
+		goto cleanup_entries;
+
+	/* There's a very annoying laxness in the dma_fence API here, in
+	 * that backends are not required to automatically report when a
+	 * fence is signaled prior to fence->ops->enable_signaling() being
+	 * called.  So here if we fail to match signaled_count, we need to
+	 * fallthough and try a 0 timeout wait!
+	 */
+
+	if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
+		for (i = 0; i < count; ++i) {
+			drm_syncobj_fence_get_or_add_callback(syncobjs[i],
+							      &entries[i].fence,
+							      &entries[i].syncobj_cb,
+							      syncobj_wait_syncobj_func);
+		}
+	}
+
+	do {
+		set_current_state(TASK_INTERRUPTIBLE);
+
+		signaled_count = 0;
+		for (i = 0; i < count; ++i) {
+			fence = entries[i].fence;
+			if (!fence)
+				continue;
+
+			if (dma_fence_is_signaled(fence) ||
+			    (!entries[i].fence_cb.func &&
+			     dma_fence_add_callback(fence,
+						    &entries[i].fence_cb,
+						    syncobj_wait_fence_func))) {
+				/* The fence has been signaled */
+				if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
+					signaled_count++;
+				} else {
+					if (idx)
+						*idx = i;
+					goto done_waiting;
+				}
+			}
+		}
+
+		if (signaled_count == count)
+			goto done_waiting;
+
+		if (timeout == 0) {
+			/* If we are doing a 0 timeout wait and we got
+			 * here, then we just timed out.
+			 */
+			ret = 0;
+			goto done_waiting;
+		}
+
+		ret = schedule_timeout(ret);
+
+		if (ret > 0 && signal_pending(current))
+			ret = -ERESTARTSYS;
+	} while (ret > 0);
+
+done_waiting:
+	__set_current_state(TASK_RUNNING);
+
+cleanup_entries:
+	for (i = 0; i < count; ++i) {
+		if (entries[i].syncobj_cb.func)
+			drm_syncobj_remove_callback(syncobjs[i],
+						    &entries[i].syncobj_cb);
+		if (entries[i].fence_cb.func)
+			dma_fence_remove_callback(entries[i].fence,
+						  &entries[i].fence_cb);
+		dma_fence_put(entries[i].fence);
+	}
+	kfree(entries);
+
+	return ret;
+}
+
+/**
+ * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value
+ *
+ * @timeout_nsec: timeout nsec component in ns, 0 for poll
+ *
+ * Calculate the timeout in jiffies from an absolute time in sec/nsec.
+ */
+static signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
+{
+	ktime_t abs_timeout, now;
+	u64 timeout_ns, timeout_jiffies64;
+
+	/* make 0 timeout means poll - absolute 0 doesn't seem valid */
+	if (timeout_nsec == 0)
+		return 0;
+
+	abs_timeout = ns_to_ktime(timeout_nsec);
+	now = ktime_get();
+
+	if (!ktime_after(abs_timeout, now))
+		return 0;
+
+	timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
+
+	timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns);
+	/*  clamp timeout to avoid infinite timeout */
+	if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
+		return MAX_SCHEDULE_TIMEOUT - 1;
+
+	return timeout_jiffies64 + 1;
+}
+
+static int drm_syncobj_array_wait(struct drm_device *dev,
+				  struct drm_file *file_private,
+				  struct drm_syncobj_wait *wait,
+				  struct drm_syncobj **syncobjs)
+{
+	signed long timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
+	signed long ret = 0;
+	uint32_t first = ~0;
+
+	ret = drm_syncobj_array_wait_timeout(syncobjs,
+					     wait->count_handles,
+					     wait->flags,
+					     timeout, &first);
+	if (ret < 0)
+		return ret;
+
+	wait->first_signaled = first;
+	if (ret == 0)
+		return -ETIME;
+	return 0;
+}
+
+static int drm_syncobj_array_find(struct drm_file *file_private,
+				  void *user_handles, uint32_t count_handles,
+				  struct drm_syncobj ***syncobjs_out)
+{
+	uint32_t i, *handles;
+	struct drm_syncobj **syncobjs;
+	int ret;
+
+	handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);
+	if (handles == NULL)
+		return -ENOMEM;
+
+	if (copy_from_user(handles, user_handles,
+			   sizeof(uint32_t) * count_handles)) {
+		ret = -EFAULT;
+		goto err_free_handles;
+	}
+
+	syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);
+	if (syncobjs == NULL) {
+		ret = -ENOMEM;
+		goto err_free_handles;
+	}
+
+	for (i = 0; i < count_handles; i++) {
+		syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
+		if (!syncobjs[i]) {
+			ret = -ENOENT;
+			goto err_put_syncobjs;
+		}
+	}
+
+	kfree(handles);
+	*syncobjs_out = syncobjs;
+	return 0;
+
+err_put_syncobjs:
+	while (i-- > 0)
+		drm_syncobj_put(syncobjs[i]);
+	kfree(syncobjs);
+err_free_handles:
+	kfree(handles);
+
+	return ret;
+}
+
+static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
+				   uint32_t count)
+{
+	uint32_t i;
+	for (i = 0; i < count; i++)
+		drm_syncobj_put(syncobjs[i]);
+	kfree(syncobjs);
+}
+
+int
+drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
+		       struct drm_file *file_private)
+{
+	struct drm_syncobj_wait *args = data;
+	struct drm_syncobj **syncobjs;
+	int ret = 0;
+
+	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
+		return -ENODEV;
+
+	if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
+			    DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
+		return -EINVAL;
+
+	if (args->count_handles == 0)
+		return -EINVAL;
+
+	ret = drm_syncobj_array_find(file_private,
+				     u64_to_user_ptr(args->handles),
+				     args->count_handles,
+				     &syncobjs);
+	if (ret < 0)
+		return ret;
+
+	ret = drm_syncobj_array_wait(dev, file_private,
+				     args, syncobjs);
+
+	drm_syncobj_array_free(syncobjs, args->count_handles);
+
+	return ret;
+}
+
+int
+drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
+			struct drm_file *file_private)
+{
+	struct drm_syncobj_array *args = data;
+	struct drm_syncobj **syncobjs;
+	uint32_t i;
+	int ret;
+
+	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
+		return -ENODEV;
+
+	if (args->pad != 0)
+		return -EINVAL;
+
+	if (args->count_handles == 0)
+		return -EINVAL;
+
+	ret = drm_syncobj_array_find(file_private,
+				     u64_to_user_ptr(args->handles),
+				     args->count_handles,
+				     &syncobjs);
+	if (ret < 0)
+		return ret;
+
+	for (i = 0; i < args->count_handles; i++)
+		drm_syncobj_replace_fence(syncobjs[i], NULL);
+
+	drm_syncobj_array_free(syncobjs, args->count_handles);
+
+	return 0;
+}
+
+int
+drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
+			 struct drm_file *file_private)
+{
+	struct drm_syncobj_array *args = data;
+	struct drm_syncobj **syncobjs;
+	uint32_t i;
+	int ret;
+
+	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
+		return -ENODEV;
+
+	if (args->pad != 0)
+		return -EINVAL;
+
+	if (args->count_handles == 0)
+		return -EINVAL;
+
+	ret = drm_syncobj_array_find(file_private,
+				     u64_to_user_ptr(args->handles),
+				     args->count_handles,
+				     &syncobjs);
+	if (ret < 0)
+		return ret;
+
+	for (i = 0; i < args->count_handles; i++) {
+		ret = drm_syncobj_assign_null_handle(syncobjs[i]);
+		if (ret < 0)
+			break;
+	}
+
+	drm_syncobj_array_free(syncobjs, args->count_handles);
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/drm_vblank.c b/drivers/gpu/drm/drm_vblank.c
index e9f33cd805dd..70f2b9593edc 100644
--- a/drivers/gpu/drm/drm_vblank.c
+++ b/drivers/gpu/drm/drm_vblank.c
@@ -31,6 +31,41 @@
 #include "drm_trace.h"
 #include "drm_internal.h"
 
+/**
+ * DOC: vblank handling
+ *
+ * Vertical blanking plays a major role in graphics rendering. To achieve
+ * tear-free display, users must synchronize page flips and/or rendering to
+ * vertical blanking. The DRM API offers ioctls to perform page flips
+ * synchronized to vertical blanking and wait for vertical blanking.
+ *
+ * The DRM core handles most of the vertical blanking management logic, which
+ * involves filtering out spurious interrupts, keeping race-free blanking
+ * counters, coping with counter wrap-around and resets and keeping use counts.
+ * It relies on the driver to generate vertical blanking interrupts and
+ * optionally provide a hardware vertical blanking counter.
+ *
+ * Drivers must initialize the vertical blanking handling core with a call to
+ * drm_vblank_init(). Minimally, a driver needs to implement
+ * &drm_crtc_funcs.enable_vblank and &drm_crtc_funcs.disable_vblank plus call
+ * drm_crtc_handle_vblank() in it's vblank interrupt handler for working vblank
+ * support.
+ *
+ * Vertical blanking interrupts can be enabled by the DRM core or by drivers
+ * themselves (for instance to handle page flipping operations).  The DRM core
+ * maintains a vertical blanking use count to ensure that the interrupts are not
+ * disabled while a user still needs them. To increment the use count, drivers
+ * call drm_crtc_vblank_get() and release the vblank reference again with
+ * drm_crtc_vblank_put(). In between these two calls vblank interrupts are
+ * guaranteed to be enabled.
+ *
+ * On many hardware disabling the vblank interrupt cannot be done in a race-free
+ * manner, see &drm_driver.vblank_disable_immediate and
+ * &drm_driver.max_vblank_count. In that case the vblank core only disables the
+ * vblanks after a timer has expired, which can be configured through the
+ * ``vblankoffdelay`` module parameter.
+ */
+
 /* Retry timestamp calculation up to 3 times to satisfy
  * drm_timestamp_precision before giving up.
  */
@@ -259,16 +294,17 @@ static u32 drm_vblank_count(struct drm_device *dev, unsigned int pipe)
 }
 
 /**
- * drm_accurate_vblank_count - retrieve the master vblank counter
+ * drm_crtc_accurate_vblank_count - retrieve the master vblank counter
  * @crtc: which counter to retrieve
  *
- * This function is similar to @drm_crtc_vblank_count but this
- * function interpolates to handle a race with vblank irq's.
+ * This function is similar to drm_crtc_vblank_count() but this function
+ * interpolates to handle a race with vblank interrupts using the high precision
+ * timestamping support.
  *
- * This is mostly useful for hardware that can obtain the scanout
- * position, but doesn't have a frame counter.
+ * This is mostly useful for hardware that can obtain the scanout position, but
+ * doesn't have a hardware frame counter.
  */
-u32 drm_accurate_vblank_count(struct drm_crtc *crtc)
+u32 drm_crtc_accurate_vblank_count(struct drm_crtc *crtc)
 {
 	struct drm_device *dev = crtc->dev;
 	unsigned int pipe = drm_crtc_index(crtc);
@@ -287,7 +323,7 @@ u32 drm_accurate_vblank_count(struct drm_crtc *crtc)
 
 	return vblank;
 }
-EXPORT_SYMBOL(drm_accurate_vblank_count);
+EXPORT_SYMBOL(drm_crtc_accurate_vblank_count);
 
 static void __disable_vblank(struct drm_device *dev, unsigned int pipe)
 {
@@ -358,15 +394,6 @@ static void vblank_disable_fn(unsigned long arg)
 	spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
 }
 
-/**
- * drm_vblank_cleanup - cleanup vblank support
- * @dev: DRM device
- *
- * This function cleans up any resources allocated in drm_vblank_init.
- *
- * Drivers which don't use drm_irq_install() need to set &drm_device.irq_enabled
- * themselves, to signal to the DRM core that vblank interrupts are enabled.
- */
 void drm_vblank_cleanup(struct drm_device *dev)
 {
 	unsigned int pipe;
@@ -388,7 +415,6 @@ void drm_vblank_cleanup(struct drm_device *dev)
 
 	dev->num_crtcs = 0;
 }
-EXPORT_SYMBOL(drm_vblank_cleanup);
 
 /**
  * drm_vblank_init - initialize vblank support
@@ -396,6 +422,8 @@ EXPORT_SYMBOL(drm_vblank_cleanup);
  * @num_crtcs: number of CRTCs supported by @dev
  *
  * This function initializes vblank support for @num_crtcs display pipelines.
+ * Cleanup is handled by the DRM core, or through calling drm_dev_fini() for
+ * drivers with a &drm_driver.release callback.
  *
  * Returns:
  * Zero on success or a negative error code on failure.
@@ -468,11 +496,11 @@ EXPORT_SYMBOL(drm_crtc_vblank_waitqueue);
  * @crtc: drm_crtc whose timestamp constants should be updated.
  * @mode: display mode containing the scanout timings
  *
- * Calculate and store various constants which are later
- * needed by vblank and swap-completion timestamping, e.g,
- * by drm_calc_vbltimestamp_from_scanoutpos(). They are
- * derived from CRTC's true scanout timing, so they take
- * things like panel scaling or other adjustments into account.
+ * Calculate and store various constants which are later needed by vblank and
+ * swap-completion timestamping, e.g, by
+ * drm_calc_vbltimestamp_from_scanoutpos(). They are derived from CRTC's true
+ * scanout timing, so they take things like panel scaling or other adjustments
+ * into account.
  */
 void drm_calc_timestamping_constants(struct drm_crtc *crtc,
 				     const struct drm_display_mode *mode)
@@ -535,25 +563,14 @@ EXPORT_SYMBOL(drm_calc_timestamping_constants);
  *     if flag is set.
  *
  * Implements calculation of exact vblank timestamps from given drm_display_mode
- * timings and current video scanout position of a CRTC. This can be called from
- * within get_vblank_timestamp() implementation of a kms driver to implement the
- * actual timestamping.
- *
- * Should return timestamps conforming to the OML_sync_control OpenML
- * extension specification. The timestamp corresponds to the end of
- * the vblank interval, aka start of scanout of topmost-leftmost display
- * pixel in the following video frame.
+ * timings and current video scanout position of a CRTC. This can be directly
+ * used as the &drm_driver.get_vblank_timestamp implementation of a kms driver
+ * if &drm_driver.get_scanout_position is implemented.
  *
- * Requires support for optional dev->driver->get_scanout_position()
- * in kms driver, plus a bit of setup code to provide a drm_display_mode
- * that corresponds to the true scanout timing.
- *
- * The current implementation only handles standard video modes. It
- * returns as no operation if a doublescan or interlaced video mode is
- * active. Higher level code is expected to handle this.
- *
- * This function can be used to implement the &drm_driver.get_vblank_timestamp
- * directly, if the driver implements the &drm_driver.get_scanout_position hook.
+ * The current implementation only handles standard video modes. For double scan
+ * and interlaced modes the driver is supposed to adjust the hardware mode
+ * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
+ * match the scanout position reported.
  *
  * Note that atomic drivers must call drm_calc_timestamping_constants() before
  * enabling a CRTC. The atomic helpers already take care of that in
@@ -738,7 +755,9 @@ drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
  *
  * Fetches the "cooked" vblank count value that represents the number of
  * vblank events since the system was booted, including lost events due to
- * modesetting activity.
+ * modesetting activity. Note that this timer isn't correct against a racing
+ * vblank interrupt (since it only reports the software vblank counter), see
+ * drm_crtc_accurate_vblank_count() for such use-cases.
  *
  * Returns:
  * The software vblank counter.
@@ -749,20 +768,6 @@ u32 drm_crtc_vblank_count(struct drm_crtc *crtc)
 }
 EXPORT_SYMBOL(drm_crtc_vblank_count);
 
-/**
- * drm_vblank_count_and_time - retrieve "cooked" vblank counter value and the
- *     system timestamp corresponding to that vblank counter value.
- * @dev: DRM device
- * @pipe: index of CRTC whose counter to retrieve
- * @vblanktime: Pointer to struct timeval to receive the vblank timestamp.
- *
- * Fetches the "cooked" vblank count value that represents the number of
- * vblank events since the system was booted, including lost events due to
- * modesetting activity. Returns corresponding system timestamp of the time
- * of the vblank interval that corresponds to the current vblank counter value.
- *
- * This is the legacy version of drm_crtc_vblank_count_and_time().
- */
 static u32 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
 				     struct timeval *vblanktime)
 {
@@ -831,7 +836,7 @@ static void send_vblank_event(struct drm_device *dev,
  * NOTE: Drivers using this to send out the &drm_crtc_state.event as part of an
  * atomic commit must ensure that the next vblank happens at exactly the same
  * time as the atomic commit is committed to the hardware. This function itself
- * does **not** protect again the next vblank interrupt racing with either this
+ * does **not** protect against the next vblank interrupt racing with either this
  * function call or the atomic commit operation. A possible sequence could be:
  *
  * 1. Driver commits new hardware state into vblank-synchronized registers.
@@ -852,8 +857,8 @@ static void send_vblank_event(struct drm_device *dev,
  * handler by calling drm_crtc_send_vblank_event() and make sure that there's no
  * possible race with the hardware committing the atomic update.
  *
- * Caller must hold event lock. Caller must also hold a vblank reference for
- * the event @e, which will be dropped when the next vblank arrives.
+ * Caller must hold a vblank reference for the event @e, which will be dropped
+ * when the next vblank arrives.
  */
 void drm_crtc_arm_vblank_event(struct drm_crtc *crtc,
 			       struct drm_pending_vblank_event *e)
@@ -913,14 +918,6 @@ static int __enable_vblank(struct drm_device *dev, unsigned int pipe)
 	return dev->driver->enable_vblank(dev, pipe);
 }
 
-/**
- * drm_vblank_enable - enable the vblank interrupt on a CRTC
- * @dev: DRM device
- * @pipe: CRTC index
- *
- * Returns:
- * Zero on success or a negative error code on failure.
- */
 static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)
 {
 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
@@ -958,19 +955,6 @@ static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)
 	return ret;
 }
 
-/**
- * drm_vblank_get - get a reference count on vblank events
- * @dev: DRM device
- * @pipe: index of CRTC to own
- *
- * Acquire a reference count on vblank events to avoid having them disabled
- * while in use.
- *
- * This is the legacy version of drm_crtc_vblank_get().
- *
- * Returns:
- * Zero on success or a negative error code on failure.
- */
 static int drm_vblank_get(struct drm_device *dev, unsigned int pipe)
 {
 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
@@ -1014,16 +998,6 @@ int drm_crtc_vblank_get(struct drm_crtc *crtc)
 }
 EXPORT_SYMBOL(drm_crtc_vblank_get);
 
-/**
- * drm_vblank_put - release ownership of vblank events
- * @dev: DRM device
- * @pipe: index of CRTC to release
- *
- * Release ownership of a given vblank counter, turning off interrupts
- * if possible. Disable interrupts after drm_vblank_offdelay milliseconds.
- *
- * This is the legacy version of drm_crtc_vblank_put().
- */
 static void drm_vblank_put(struct drm_device *dev, unsigned int pipe)
 {
 	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
@@ -1067,6 +1041,8 @@ EXPORT_SYMBOL(drm_crtc_vblank_put);
  * This waits for one vblank to pass on @pipe, using the irq driver interfaces.
  * It is a failure to call this when the vblank irq for @pipe is disabled, e.g.
  * due to lack of driver support or because the crtc is off.
+ *
+ * This is the legacy version of drm_crtc_wait_one_vblank().
  */
 void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)
 {
@@ -1116,7 +1092,7 @@ EXPORT_SYMBOL(drm_crtc_wait_one_vblank);
  * stored so that drm_vblank_on can restore it again.
  *
  * Drivers must use this function when the hardware vblank counter can get
- * reset, e.g. when suspending.
+ * reset, e.g. when suspending or disabling the @crtc in general.
  */
 void drm_crtc_vblank_off(struct drm_crtc *crtc)
 {
@@ -1184,6 +1160,8 @@ EXPORT_SYMBOL(drm_crtc_vblank_off);
  * drm_crtc_vblank_on() functions. The difference compared to
  * drm_crtc_vblank_off() is that this function doesn't save the vblank counter
  * and hence doesn't need to call any driver hooks.
+ *
+ * This is useful for recovering driver state e.g. on driver load, or on resume.
  */
 void drm_crtc_vblank_reset(struct drm_crtc *crtc)
 {
@@ -1212,9 +1190,10 @@ EXPORT_SYMBOL(drm_crtc_vblank_reset);
  * @crtc: CRTC in question
  *
  * This functions restores the vblank interrupt state captured with
- * drm_crtc_vblank_off() again. Note that calls to drm_crtc_vblank_on() and
- * drm_crtc_vblank_off() can be unbalanced and so can also be unconditionally called
- * in driver load code to reflect the current hardware state of the crtc.
+ * drm_crtc_vblank_off() again and is generally called when enabling @crtc. Note
+ * that calls to drm_crtc_vblank_on() and drm_crtc_vblank_off() can be
+ * unbalanced and so can also be unconditionally called in driver load code to
+ * reflect the current hardware state of the crtc.
  */
 void drm_crtc_vblank_on(struct drm_crtc *crtc)
 {
@@ -1299,8 +1278,8 @@ static void drm_legacy_vblank_post_modeset(struct drm_device *dev,
 	}
 }
 
-int drm_legacy_modeset_ctl(struct drm_device *dev, void *data,
-			   struct drm_file *file_priv)
+int drm_legacy_modeset_ctl_ioctl(struct drm_device *dev, void *data,
+				 struct drm_file *file_priv)
 {
 	struct drm_modeset_ctl *modeset = data;
 	unsigned int pipe;
@@ -1419,22 +1398,8 @@ static bool drm_wait_vblank_is_query(union drm_wait_vblank *vblwait)
 					  _DRM_VBLANK_NEXTONMISS));
 }
 
-/*
- * Wait for VBLANK.
- *
- * \param inode device inode.
- * \param file_priv DRM file private.
- * \param cmd command.
- * \param data user argument, pointing to a drm_wait_vblank structure.
- * \return zero on success or a negative number on failure.
- *
- * This function enables the vblank interrupt on the pipe requested, then
- * sleeps waiting for the requested sequence number to occur, and drops
- * the vblank interrupt refcount afterwards. (vblank IRQ disable follows that
- * after a timeout with no further vblank waits scheduled).
- */
-int drm_wait_vblank(struct drm_device *dev, void *data,
-		    struct drm_file *file_priv)
+int drm_wait_vblank_ioctl(struct drm_device *dev, void *data,
+			  struct drm_file *file_priv)
 {
 	struct drm_vblank_crtc *vblank;
 	union drm_wait_vblank *vblwait = data;
diff --git a/drivers/gpu/drm/drm_vm.c b/drivers/gpu/drm/drm_vm.c
index 1170b3209a12..2660543ad86a 100644
--- a/drivers/gpu/drm/drm_vm.c
+++ b/drivers/gpu/drm/drm_vm.c
@@ -40,6 +40,7 @@
 #include <linux/efi.h>
 #include <linux/slab.h>
 #endif
+#include <linux/mem_encrypt.h>
 #include <asm/pgtable.h>
 #include "drm_internal.h"
 #include "drm_legacy.h"
@@ -58,6 +59,9 @@ static pgprot_t drm_io_prot(struct drm_local_map *map,
 {
 	pgprot_t tmp = vm_get_page_prot(vma->vm_flags);
 
+	/* We don't want graphics memory to be mapped encrypted */
+	tmp = pgprot_decrypted(tmp);
+
 #if defined(__i386__) || defined(__x86_64__) || defined(__powerpc__)
 	if (map->type == _DRM_REGISTERS && !(map->flags & _DRM_WRITE_COMBINING))
 		tmp = pgprot_noncached(tmp);
@@ -631,7 +635,7 @@ int drm_legacy_mmap(struct file *filp, struct vm_area_struct *vma)
 	struct drm_device *dev = priv->minor->dev;
 	int ret;
 
-	if (drm_device_is_unplugged(dev))
+	if (drm_dev_is_unplugged(dev))
 		return -ENODEV;
 
 	mutex_lock(&dev->struct_mutex);
diff --git a/drivers/gpu/drm/drm_vma_manager.c b/drivers/gpu/drm/drm_vma_manager.c
index d9100b565198..28f1226576f8 100644
--- a/drivers/gpu/drm/drm_vma_manager.c
+++ b/drivers/gpu/drm/drm_vma_manager.c
@@ -147,7 +147,7 @@ struct drm_vma_offset_node *drm_vma_offset_lookup_locked(struct drm_vma_offset_m
 	struct rb_node *iter;
 	unsigned long offset;
 
-	iter = mgr->vm_addr_space_mm.interval_tree.rb_node;
+	iter = mgr->vm_addr_space_mm.interval_tree.rb_root.rb_node;
 	best = NULL;
 
 	while (likely(iter)) {
diff --git a/drivers/gpu/drm/etnaviv/Kconfig b/drivers/gpu/drm/etnaviv/Kconfig
index 71cee4e9fefb..38b477b5fbf9 100644
--- a/drivers/gpu/drm/etnaviv/Kconfig
+++ b/drivers/gpu/drm/etnaviv/Kconfig
@@ -10,6 +10,8 @@ config DRM_ETNAVIV
 	select IOMMU_API
 	select IOMMU_SUPPORT
 	select WANT_DEV_COREDUMP
+	select CMA if HAVE_DMA_CONTIGUOUS
+	select DMA_CMA if HAVE_DMA_CONTIGUOUS
 	help
 	  DRM driver for Vivante GPUs.
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
index 91e17aeee1da..2cb4773823c2 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
@@ -316,7 +316,7 @@ static int etnaviv_ioctl_gem_cpu_prep(struct drm_device *dev, void *data,
 
 	ret = etnaviv_gem_cpu_prep(obj, args->op, &TS(args->timeout));
 
-	drm_gem_object_unreference_unlocked(obj);
+	drm_gem_object_put_unlocked(obj);
 
 	return ret;
 }
@@ -337,7 +337,7 @@ static int etnaviv_ioctl_gem_cpu_fini(struct drm_device *dev, void *data,
 
 	ret = etnaviv_gem_cpu_fini(obj);
 
-	drm_gem_object_unreference_unlocked(obj);
+	drm_gem_object_put_unlocked(obj);
 
 	return ret;
 }
@@ -357,7 +357,7 @@ static int etnaviv_ioctl_gem_info(struct drm_device *dev, void *data,
 		return -ENOENT;
 
 	ret = etnaviv_gem_mmap_offset(obj, &args->offset);
-	drm_gem_object_unreference_unlocked(obj);
+	drm_gem_object_put_unlocked(obj);
 
 	return ret;
 }
@@ -446,7 +446,7 @@ static int etnaviv_ioctl_gem_wait(struct drm_device *dev, void *data,
 
 	ret = etnaviv_gem_wait_bo(gpu, obj, timeout);
 
-	drm_gem_object_unreference_unlocked(obj);
+	drm_gem_object_put_unlocked(obj);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
index 9a3bea738330..5a634594a6ce 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
@@ -68,7 +68,7 @@ static int etnaviv_gem_shmem_get_pages(struct etnaviv_gem_object *etnaviv_obj)
 	struct page **p = drm_gem_get_pages(&etnaviv_obj->base);
 
 	if (IS_ERR(p)) {
-		dev_err(dev->dev, "could not get pages: %ld\n", PTR_ERR(p));
+		dev_dbg(dev->dev, "could not get pages: %ld\n", PTR_ERR(p));
 		return PTR_ERR(p);
 	}
 
@@ -265,7 +265,7 @@ void etnaviv_gem_mapping_reference(struct etnaviv_vram_mapping *mapping)
 {
 	struct etnaviv_gem_object *etnaviv_obj = mapping->object;
 
-	drm_gem_object_reference(&etnaviv_obj->base);
+	drm_gem_object_get(&etnaviv_obj->base);
 
 	mutex_lock(&etnaviv_obj->lock);
 	WARN_ON(mapping->use == 0);
@@ -282,7 +282,7 @@ void etnaviv_gem_mapping_unreference(struct etnaviv_vram_mapping *mapping)
 	mapping->use -= 1;
 	mutex_unlock(&etnaviv_obj->lock);
 
-	drm_gem_object_unreference_unlocked(&etnaviv_obj->base);
+	drm_gem_object_put_unlocked(&etnaviv_obj->base);
 }
 
 struct etnaviv_vram_mapping *etnaviv_gem_mapping_get(
@@ -358,7 +358,7 @@ out:
 		return ERR_PTR(ret);
 
 	/* Take a reference on the object */
-	drm_gem_object_reference(obj);
+	drm_gem_object_get(obj);
 	return mapping;
 }
 
@@ -413,6 +413,16 @@ int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op,
 	bool write = !!(op & ETNA_PREP_WRITE);
 	int ret;
 
+	if (!etnaviv_obj->sgt) {
+		void *ret;
+
+		mutex_lock(&etnaviv_obj->lock);
+		ret = etnaviv_gem_get_pages(etnaviv_obj);
+		mutex_unlock(&etnaviv_obj->lock);
+		if (IS_ERR(ret))
+			return PTR_ERR(ret);
+	}
+
 	if (op & ETNA_PREP_NOSYNC) {
 		if (!reservation_object_test_signaled_rcu(etnaviv_obj->resv,
 							  write))
@@ -427,16 +437,6 @@ int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op,
 	}
 
 	if (etnaviv_obj->flags & ETNA_BO_CACHED) {
-		if (!etnaviv_obj->sgt) {
-			void *ret;
-
-			mutex_lock(&etnaviv_obj->lock);
-			ret = etnaviv_gem_get_pages(etnaviv_obj);
-			mutex_unlock(&etnaviv_obj->lock);
-			if (IS_ERR(ret))
-				return PTR_ERR(ret);
-		}
-
 		dma_sync_sg_for_cpu(dev->dev, etnaviv_obj->sgt->sgl,
 				    etnaviv_obj->sgt->nents,
 				    etnaviv_op_to_dma_dir(op));
@@ -662,7 +662,8 @@ static struct drm_gem_object *__etnaviv_gem_new(struct drm_device *dev,
 		 * going to pin these pages.
 		 */
 		mapping = obj->filp->f_mapping;
-		mapping_set_gfp_mask(mapping, GFP_HIGHUSER);
+		mapping_set_gfp_mask(mapping, GFP_HIGHUSER |
+				     __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
 	}
 
 	if (ret)
@@ -671,7 +672,7 @@ static struct drm_gem_object *__etnaviv_gem_new(struct drm_device *dev,
 	return obj;
 
 fail:
-	drm_gem_object_unreference_unlocked(obj);
+	drm_gem_object_put_unlocked(obj);
 	return ERR_PTR(ret);
 }
 
@@ -688,14 +689,14 @@ int etnaviv_gem_new_handle(struct drm_device *dev, struct drm_file *file,
 
 	ret = etnaviv_gem_obj_add(dev, obj);
 	if (ret < 0) {
-		drm_gem_object_unreference_unlocked(obj);
+		drm_gem_object_put_unlocked(obj);
 		return ret;
 	}
 
 	ret = drm_gem_handle_create(file, obj, handle);
 
 	/* drop reference from allocate - handle holds it now */
-	drm_gem_object_unreference_unlocked(obj);
+	drm_gem_object_put_unlocked(obj);
 
 	return ret;
 }
@@ -712,7 +713,7 @@ struct drm_gem_object *etnaviv_gem_new(struct drm_device *dev,
 
 	ret = etnaviv_gem_obj_add(dev, obj);
 	if (ret < 0) {
-		drm_gem_object_unreference_unlocked(obj);
+		drm_gem_object_put_unlocked(obj);
 		return ERR_PTR(ret);
 	}
 
@@ -800,7 +801,7 @@ static void __etnaviv_gem_userptr_get_pages(struct work_struct *_work)
 	}
 
 	mutex_unlock(&etnaviv_obj->lock);
-	drm_gem_object_unreference_unlocked(&etnaviv_obj->base);
+	drm_gem_object_put_unlocked(&etnaviv_obj->base);
 
 	mmput(work->mm);
 	put_task_struct(work->task);
@@ -858,7 +859,7 @@ static int etnaviv_gem_userptr_get_pages(struct etnaviv_gem_object *etnaviv_obj)
 	}
 
 	get_task_struct(current);
-	drm_gem_object_reference(&etnaviv_obj->base);
+	drm_gem_object_get(&etnaviv_obj->base);
 
 	work->mm = mm;
 	work->task = current;
@@ -924,6 +925,6 @@ int etnaviv_gem_new_userptr(struct drm_device *dev, struct drm_file *file,
 	ret = drm_gem_handle_create(file, &etnaviv_obj->base, handle);
 unreference:
 	/* drop reference from allocate - handle holds it now */
-	drm_gem_object_unreference_unlocked(&etnaviv_obj->base);
+	drm_gem_object_put_unlocked(&etnaviv_obj->base);
 	return ret;
 }
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
index e5da4f2300ba..ae884723e9b1 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
@@ -146,7 +146,7 @@ struct drm_gem_object *etnaviv_gem_prime_import_sg_table(struct drm_device *dev,
 	return &etnaviv_obj->base;
 
 fail:
-	drm_gem_object_unreference_unlocked(&etnaviv_obj->base);
+	drm_gem_object_put_unlocked(&etnaviv_obj->base);
 
 	return ERR_PTR(ret);
 }
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
index 6463fc2c736f..026ef4e02f85 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
@@ -37,7 +37,7 @@ static struct etnaviv_gem_submit *submit_create(struct drm_device *dev,
 	struct etnaviv_gem_submit *submit;
 	size_t sz = size_vstruct(nr, sizeof(submit->bos[0]), sizeof(*submit));
 
-	submit = kmalloc(sz, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
+	submit = kmalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
 	if (submit) {
 		submit->dev = dev;
 		submit->gpu = gpu;
@@ -88,7 +88,7 @@ static int submit_lookup_objects(struct etnaviv_gem_submit *submit,
 		 * Take a refcount on the object. The file table lock
 		 * prevents the object_idr's refcount on this being dropped.
 		 */
-		drm_gem_object_reference(obj);
+		drm_gem_object_get(obj);
 
 		submit->bos[i].obj = to_etnaviv_bo(obj);
 	}
@@ -291,7 +291,7 @@ static void submit_cleanup(struct etnaviv_gem_submit *submit)
 		struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
 
 		submit_unlock_object(submit, i);
-		drm_gem_object_unreference_unlocked(&etnaviv_obj->base);
+		drm_gem_object_put_unlocked(&etnaviv_obj->base);
 	}
 
 	ww_acquire_fini(&submit->ticket);
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
index ada45fdd0eae..fc9a6a83dfc7 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
@@ -1622,10 +1622,12 @@ static int etnaviv_gpu_bind(struct device *dev, struct device *master,
 	struct etnaviv_gpu *gpu = dev_get_drvdata(dev);
 	int ret;
 
-	gpu->cooling = thermal_of_cooling_device_register(dev->of_node,
+	if (IS_ENABLED(CONFIG_THERMAL)) {
+		gpu->cooling = thermal_of_cooling_device_register(dev->of_node,
 				(char *)dev_name(dev), gpu, &cooling_ops);
-	if (IS_ERR(gpu->cooling))
-		return PTR_ERR(gpu->cooling);
+		if (IS_ERR(gpu->cooling))
+			return PTR_ERR(gpu->cooling);
+	}
 
 #ifdef CONFIG_PM
 	ret = pm_runtime_get_sync(gpu->dev);
diff --git a/drivers/gpu/drm/exynos/exynos5433_drm_decon.c b/drivers/gpu/drm/exynos/exynos5433_drm_decon.c
index 5792ca88ab7a..6be5b53c3b27 100644
--- a/drivers/gpu/drm/exynos/exynos5433_drm_decon.c
+++ b/drivers/gpu/drm/exynos/exynos5433_drm_decon.c
@@ -13,6 +13,8 @@
 #include <linux/platform_device.h>
 #include <linux/clk.h>
 #include <linux/component.h>
+#include <linux/iopoll.h>
+#include <linux/irq.h>
 #include <linux/mfd/syscon.h>
 #include <linux/of_device.h>
 #include <linux/of_gpio.h>
@@ -33,9 +35,8 @@
 #define WINDOWS_NR	3
 #define MIN_FB_WIDTH_FOR_16WORD_BURST	128
 
-#define IFTYPE_I80	(1 << 0)
-#define I80_HW_TRG	(1 << 1)
-#define IFTYPE_HDMI	(1 << 2)
+#define I80_HW_TRG	(1 << 0)
+#define IFTYPE_HDMI	(1 << 1)
 
 static const char * const decon_clks_name[] = {
 	"pclk",
@@ -57,6 +58,8 @@ struct decon_context {
 	struct regmap			*sysreg;
 	struct clk			*clks[ARRAY_SIZE(decon_clks_name)];
 	unsigned int			irq;
+	unsigned int			irq_vsync;
+	unsigned int			irq_lcd_sys;
 	unsigned int			te_irq;
 	unsigned long			out_type;
 	int				first_win;
@@ -90,7 +93,7 @@ static int decon_enable_vblank(struct exynos_drm_crtc *crtc)
 	u32 val;
 
 	val = VIDINTCON0_INTEN;
-	if (ctx->out_type & IFTYPE_I80)
+	if (crtc->i80_mode)
 		val |= VIDINTCON0_FRAMEDONE;
 	else
 		val |= VIDINTCON0_INTFRMEN | VIDINTCON0_FRAMESEL_FP;
@@ -139,7 +142,7 @@ static u32 decon_get_frame_count(struct decon_context *ctx, bool end)
 
 	switch (status & (VIDCON1_VSTATUS_MASK | VIDCON1_I80_ACTIVE)) {
 	case VIDCON1_VSTATUS_VS:
-		if (!(ctx->out_type & IFTYPE_I80))
+		if (!(ctx->crtc->i80_mode))
 			--frm;
 		break;
 	case VIDCON1_VSTATUS_BP:
@@ -166,7 +169,7 @@ static u32 decon_get_vblank_counter(struct exynos_drm_crtc *crtc)
 
 static void decon_setup_trigger(struct decon_context *ctx)
 {
-	if (!(ctx->out_type & (IFTYPE_I80 | I80_HW_TRG)))
+	if (!ctx->crtc->i80_mode && !(ctx->out_type & I80_HW_TRG))
 		return;
 
 	if (!(ctx->out_type & I80_HW_TRG)) {
@@ -206,7 +209,7 @@ static void decon_commit(struct exynos_drm_crtc *crtc)
 	val = VIDOUT_LCD_ON;
 	if (interlaced)
 		val |= VIDOUT_INTERLACE_EN_F;
-	if (ctx->out_type & IFTYPE_I80) {
+	if (crtc->i80_mode) {
 		val |= VIDOUT_COMMAND_IF;
 	} else {
 		val |= VIDOUT_RGB_IF;
@@ -222,7 +225,7 @@ static void decon_commit(struct exynos_drm_crtc *crtc)
 			VIDTCON2_HOZVAL(m->hdisplay - 1);
 	writel(val, ctx->addr + DECON_VIDTCON2);
 
-	if (!(ctx->out_type & IFTYPE_I80)) {
+	if (!crtc->i80_mode) {
 		int vbp = m->crtc_vtotal - m->crtc_vsync_end;
 		int vfp = m->crtc_vsync_start - m->crtc_vdisplay;
 
@@ -277,16 +280,14 @@ static void decon_win_set_pixfmt(struct decon_context *ctx, unsigned int win,
 		val |= WINCONx_BURSTLEN_16WORD;
 		break;
 	case DRM_FORMAT_ARGB8888:
+	default:
 		val |= WINCONx_BPPMODE_32BPP_A8888;
 		val |= WINCONx_WSWP_F | WINCONx_BLD_PIX_F | WINCONx_ALPHA_SEL_F;
 		val |= WINCONx_BURSTLEN_16WORD;
 		break;
-	default:
-		DRM_ERROR("Proper pixel format is not set\n");
-		return;
 	}
 
-	DRM_DEBUG_KMS("bpp = %u\n", fb->format->cpp[0] * 8);
+	DRM_DEBUG_KMS("cpp = %u\n", fb->format->cpp[0]);
 
 	/*
 	 * In case of exynos, setting dma-burst to 16Word causes permanent
@@ -329,7 +330,7 @@ static void decon_update_plane(struct exynos_drm_crtc *crtc,
 	struct decon_context *ctx = crtc->ctx;
 	struct drm_framebuffer *fb = state->base.fb;
 	unsigned int win = plane->index;
-	unsigned int bpp = fb->format->cpp[0];
+	unsigned int cpp = fb->format->cpp[0];
 	unsigned int pitch = fb->pitches[0];
 	dma_addr_t dma_addr = exynos_drm_fb_dma_addr(fb, 0);
 	u32 val;
@@ -365,11 +366,11 @@ static void decon_update_plane(struct exynos_drm_crtc *crtc,
 	writel(val, ctx->addr + DECON_VIDW0xADD1B0(win));
 
 	if (!(ctx->out_type & IFTYPE_HDMI))
-		val = BIT_VAL(pitch - state->crtc.w * bpp, 27, 14)
-			| BIT_VAL(state->crtc.w * bpp, 13, 0);
+		val = BIT_VAL(pitch - state->crtc.w * cpp, 27, 14)
+			| BIT_VAL(state->crtc.w * cpp, 13, 0);
 	else
-		val = BIT_VAL(pitch - state->crtc.w * bpp, 29, 15)
-			| BIT_VAL(state->crtc.w * bpp, 14, 0);
+		val = BIT_VAL(pitch - state->crtc.w * cpp, 29, 15)
+			| BIT_VAL(state->crtc.w * cpp, 14, 0);
 	writel(val, ctx->addr + DECON_VIDW0xADD2(win));
 
 	decon_win_set_pixfmt(ctx, win, fb);
@@ -407,24 +408,19 @@ static void decon_atomic_flush(struct exynos_drm_crtc *crtc)
 
 static void decon_swreset(struct decon_context *ctx)
 {
-	unsigned int tries;
 	unsigned long flags;
+	u32 val;
+	int ret;
 
 	writel(0, ctx->addr + DECON_VIDCON0);
-	for (tries = 2000; tries; --tries) {
-		if (~readl(ctx->addr + DECON_VIDCON0) & VIDCON0_STOP_STATUS)
-			break;
-		udelay(10);
-	}
+	readl_poll_timeout(ctx->addr + DECON_VIDCON0, val,
+			   ~val & VIDCON0_STOP_STATUS, 12, 20000);
 
 	writel(VIDCON0_SWRESET, ctx->addr + DECON_VIDCON0);
-	for (tries = 2000; tries; --tries) {
-		if (~readl(ctx->addr + DECON_VIDCON0) & VIDCON0_SWRESET)
-			break;
-		udelay(10);
-	}
+	ret = readl_poll_timeout(ctx->addr + DECON_VIDCON0, val,
+				 ~val & VIDCON0_SWRESET, 12, 20000);
 
-	WARN(tries == 0, "failed to software reset DECON\n");
+	WARN(ret < 0, "failed to software reset DECON\n");
 
 	spin_lock_irqsave(&ctx->vblank_lock, flags);
 	ctx->frame_id = 0;
@@ -515,6 +511,22 @@ err:
 		clk_disable_unprepare(ctx->clks[i]);
 }
 
+static enum drm_mode_status decon_mode_valid(struct exynos_drm_crtc *crtc,
+		const struct drm_display_mode *mode)
+{
+	struct decon_context *ctx = crtc->ctx;
+
+	ctx->irq = crtc->i80_mode ? ctx->irq_lcd_sys : ctx->irq_vsync;
+
+	if (ctx->irq)
+		return MODE_OK;
+
+	dev_info(ctx->dev, "Sink requires %s mode, but appropriate interrupt is not provided.\n",
+			crtc->i80_mode ? "command" : "video");
+
+	return MODE_BAD;
+}
+
 static const struct exynos_drm_crtc_ops decon_crtc_ops = {
 	.enable			= decon_enable,
 	.disable		= decon_disable,
@@ -524,6 +536,7 @@ static const struct exynos_drm_crtc_ops decon_crtc_ops = {
 	.atomic_begin		= decon_atomic_begin,
 	.update_plane		= decon_update_plane,
 	.disable_plane		= decon_disable_plane,
+	.mode_valid		= decon_mode_valid,
 	.atomic_flush		= decon_atomic_flush,
 };
 
@@ -674,19 +687,22 @@ static const struct of_device_id exynos5433_decon_driver_dt_match[] = {
 MODULE_DEVICE_TABLE(of, exynos5433_decon_driver_dt_match);
 
 static int decon_conf_irq(struct decon_context *ctx, const char *name,
-		irq_handler_t handler, unsigned long int flags, bool required)
+		irq_handler_t handler, unsigned long int flags)
 {
 	struct platform_device *pdev = to_platform_device(ctx->dev);
 	int ret, irq = platform_get_irq_byname(pdev, name);
 
 	if (irq < 0) {
-		if (irq == -EPROBE_DEFER)
+		switch (irq) {
+		case -EPROBE_DEFER:
 			return irq;
-		if (required)
-			dev_err(ctx->dev, "cannot get %s IRQ\n", name);
-		else
-			irq = 0;
-		return irq;
+		case -ENODATA:
+		case -ENXIO:
+			return 0;
+		default:
+			dev_err(ctx->dev, "IRQ %s get failed, %d\n", name, irq);
+			return irq;
+		}
 	}
 	irq_set_status_flags(irq, IRQ_NOAUTOEN);
 	ret = devm_request_irq(ctx->dev, irq, handler, flags, "drm_decon", ctx);
@@ -714,11 +730,8 @@ static int exynos5433_decon_probe(struct platform_device *pdev)
 	ctx->out_type = (unsigned long)of_device_get_match_data(dev);
 	spin_lock_init(&ctx->vblank_lock);
 
-	if (ctx->out_type & IFTYPE_HDMI) {
+	if (ctx->out_type & IFTYPE_HDMI)
 		ctx->first_win = 1;
-	} else if (of_get_child_by_name(dev->of_node, "i80-if-timings")) {
-		ctx->out_type |= IFTYPE_I80;
-	}
 
 	for (i = 0; i < ARRAY_SIZE(decon_clks_name); i++) {
 		struct clk *clk;
@@ -742,25 +755,23 @@ static int exynos5433_decon_probe(struct platform_device *pdev)
 		return PTR_ERR(ctx->addr);
 	}
 
-	if (ctx->out_type & IFTYPE_I80) {
-		ret = decon_conf_irq(ctx, "lcd_sys", decon_irq_handler, 0, true);
-		if (ret < 0)
-			return ret;
-		ctx->irq = ret;
+	ret = decon_conf_irq(ctx, "vsync", decon_irq_handler, 0);
+	if (ret < 0)
+		return ret;
+	ctx->irq_vsync = ret;
 
-		ret = decon_conf_irq(ctx, "te", decon_te_irq_handler,
-				     IRQF_TRIGGER_RISING, false);
-		if (ret < 0)
-			return ret;
-		if (ret) {
-			ctx->te_irq = ret;
-			ctx->out_type &= ~I80_HW_TRG;
-		}
-	} else {
-		ret = decon_conf_irq(ctx, "vsync", decon_irq_handler, 0, true);
-		if (ret < 0)
+	ret = decon_conf_irq(ctx, "lcd_sys", decon_irq_handler, 0);
+	if (ret < 0)
+		return ret;
+	ctx->irq_lcd_sys = ret;
+
+	ret = decon_conf_irq(ctx, "te", decon_te_irq_handler,
+			IRQF_TRIGGER_RISING);
+	if (ret < 0)
 			return ret;
-		ctx->irq = ret;
+	if (ret) {
+		ctx->te_irq = ret;
+		ctx->out_type &= ~I80_HW_TRG;
 	}
 
 	if (ctx->out_type & I80_HW_TRG) {
diff --git a/drivers/gpu/drm/exynos/exynos7_drm_decon.c b/drivers/gpu/drm/exynos/exynos7_drm_decon.c
index 3e88269fdc2e..615efcf7782a 100644
--- a/drivers/gpu/drm/exynos/exynos7_drm_decon.c
+++ b/drivers/gpu/drm/exynos/exynos7_drm_decon.c
@@ -309,19 +309,14 @@ static void decon_win_set_pixfmt(struct decon_context *ctx, unsigned int win,
 		val |= WINCONx_BURSTLEN_16WORD;
 		break;
 	case DRM_FORMAT_BGRA8888:
+	default:
 		val |= WINCONx_BPPMODE_32BPP_BGRA | WINCONx_BLD_PIX |
 			WINCONx_ALPHA_SEL;
 		val |= WINCONx_BURSTLEN_16WORD;
 		break;
-	default:
-		DRM_DEBUG_KMS("invalid pixel size so using unpacked 24bpp.\n");
-
-		val |= WINCONx_BPPMODE_24BPP_xRGB;
-		val |= WINCONx_BURSTLEN_16WORD;
-		break;
 	}
 
-	DRM_DEBUG_KMS("bpp = %d\n", fb->format->cpp[0] * 8);
+	DRM_DEBUG_KMS("cpp = %d\n", fb->format->cpp[0]);
 
 	/*
 	 * In case of exynos, setting dma-burst to 16Word causes permanent
@@ -398,7 +393,7 @@ static void decon_update_plane(struct exynos_drm_crtc *crtc,
 	unsigned int last_x;
 	unsigned int last_y;
 	unsigned int win = plane->index;
-	unsigned int bpp = fb->format->cpp[0];
+	unsigned int cpp = fb->format->cpp[0];
 	unsigned int pitch = fb->pitches[0];
 
 	if (ctx->suspended)
@@ -418,7 +413,7 @@ static void decon_update_plane(struct exynos_drm_crtc *crtc,
 	val = (unsigned long)exynos_drm_fb_dma_addr(fb, 0);
 	writel(val, ctx->regs + VIDW_BUF_START(win));
 
-	padding = (pitch / bpp) - fb->width;
+	padding = (pitch / cpp) - fb->width;
 
 	/* buffer size */
 	writel(fb->width + padding, ctx->regs + VIDW_WHOLE_X(win));
diff --git a/drivers/gpu/drm/exynos/exynos_dp.c b/drivers/gpu/drm/exynos/exynos_dp.c
index 385537b726a6..39629e7a80b9 100644
--- a/drivers/gpu/drm/exynos/exynos_dp.c
+++ b/drivers/gpu/drm/exynos/exynos_dp.c
@@ -155,7 +155,7 @@ static int exynos_dp_bind(struct device *dev, struct device *master, void *data)
 	struct exynos_dp_device *dp = dev_get_drvdata(dev);
 	struct drm_encoder *encoder = &dp->encoder;
 	struct drm_device *drm_dev = data;
-	int pipe, ret;
+	int ret;
 
 	/*
 	 * Just like the probe function said, we don't need the
@@ -179,20 +179,15 @@ static int exynos_dp_bind(struct device *dev, struct device *master, void *data)
 			return ret;
 	}
 
-	pipe = exynos_drm_crtc_get_pipe_from_type(drm_dev,
-						  EXYNOS_DISPLAY_TYPE_LCD);
-	if (pipe < 0)
-		return pipe;
-
-	encoder->possible_crtcs = 1 << pipe;
-
-	DRM_DEBUG_KMS("possible_crtcs = 0x%x\n", encoder->possible_crtcs);
-
 	drm_encoder_init(drm_dev, encoder, &exynos_dp_encoder_funcs,
 			 DRM_MODE_ENCODER_TMDS, NULL);
 
 	drm_encoder_helper_add(encoder, &exynos_dp_encoder_helper_funcs);
 
+	ret = exynos_drm_set_possible_crtcs(encoder, EXYNOS_DISPLAY_TYPE_LCD);
+	if (ret < 0)
+		return ret;
+
 	dp->plat_data.encoder = encoder;
 
 	return analogix_dp_bind(dev, dp->drm_dev, &dp->plat_data);
diff --git a/drivers/gpu/drm/exynos/exynos_drm_core.c b/drivers/gpu/drm/exynos/exynos_drm_core.c
index edbd98ff293e..b0c0621fcdf7 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_core.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_core.c
@@ -13,6 +13,7 @@
  */
 
 #include <drm/drmP.h>
+
 #include "exynos_drm_drv.h"
 #include "exynos_drm_crtc.h"
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_crtc.c b/drivers/gpu/drm/exynos/exynos_drm_crtc.c
index d72777f6411a..6ce0821590df 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_crtc.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_crtc.c
@@ -16,12 +16,14 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
+#include <drm/drm_encoder.h>
 
 #include "exynos_drm_crtc.h"
 #include "exynos_drm_drv.h"
 #include "exynos_drm_plane.h"
 
-static void exynos_drm_crtc_enable(struct drm_crtc *crtc)
+static void exynos_drm_crtc_atomic_enable(struct drm_crtc *crtc,
+					  struct drm_crtc_state *old_state)
 {
 	struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
 
@@ -31,7 +33,8 @@ static void exynos_drm_crtc_enable(struct drm_crtc *crtc)
 	drm_crtc_vblank_on(crtc);
 }
 
-static void exynos_drm_crtc_disable(struct drm_crtc *crtc)
+static void exynos_drm_crtc_atomic_disable(struct drm_crtc *crtc,
+					   struct drm_crtc_state *old_state)
 {
 	struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
 
@@ -81,12 +84,24 @@ static void exynos_crtc_atomic_flush(struct drm_crtc *crtc,
 		exynos_crtc->ops->atomic_flush(exynos_crtc);
 }
 
+static enum drm_mode_status exynos_crtc_mode_valid(struct drm_crtc *crtc,
+	const struct drm_display_mode *mode)
+{
+	struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
+
+	if (exynos_crtc->ops->mode_valid)
+		return exynos_crtc->ops->mode_valid(exynos_crtc, mode);
+
+	return MODE_OK;
+}
+
 static const struct drm_crtc_helper_funcs exynos_crtc_helper_funcs = {
-	.enable		= exynos_drm_crtc_enable,
-	.disable	= exynos_drm_crtc_disable,
+	.mode_valid	= exynos_crtc_mode_valid,
 	.atomic_check	= exynos_crtc_atomic_check,
 	.atomic_begin	= exynos_crtc_atomic_begin,
 	.atomic_flush	= exynos_crtc_atomic_flush,
+	.atomic_enable	= exynos_drm_crtc_atomic_enable,
+	.atomic_disable	= exynos_drm_crtc_atomic_disable,
 };
 
 void exynos_crtc_handle_event(struct exynos_drm_crtc *exynos_crtc)
@@ -189,16 +204,30 @@ err_crtc:
 	return ERR_PTR(ret);
 }
 
-int exynos_drm_crtc_get_pipe_from_type(struct drm_device *drm_dev,
+struct exynos_drm_crtc *exynos_drm_crtc_get_by_type(struct drm_device *drm_dev,
 				       enum exynos_drm_output_type out_type)
 {
 	struct drm_crtc *crtc;
 
 	drm_for_each_crtc(crtc, drm_dev)
 		if (to_exynos_crtc(crtc)->type == out_type)
-			return drm_crtc_index(crtc);
+			return to_exynos_crtc(crtc);
 
-	return -EPERM;
+	return ERR_PTR(-EPERM);
+}
+
+int exynos_drm_set_possible_crtcs(struct drm_encoder *encoder,
+		enum exynos_drm_output_type out_type)
+{
+	struct exynos_drm_crtc *crtc = exynos_drm_crtc_get_by_type(encoder->dev,
+						out_type);
+
+	if (IS_ERR(crtc))
+		return PTR_ERR(crtc);
+
+	encoder->possible_crtcs = drm_crtc_mask(&crtc->base);
+
+	return 0;
 }
 
 void exynos_drm_crtc_te_handler(struct drm_crtc *crtc)
diff --git a/drivers/gpu/drm/exynos/exynos_drm_crtc.h b/drivers/gpu/drm/exynos/exynos_drm_crtc.h
index ef58b64e3d2d..dec446109e6c 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_crtc.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_crtc.h
@@ -15,21 +15,25 @@
 #ifndef _EXYNOS_DRM_CRTC_H_
 #define _EXYNOS_DRM_CRTC_H_
 
+
 #include "exynos_drm_drv.h"
 
 struct exynos_drm_crtc *exynos_drm_crtc_create(struct drm_device *drm_dev,
 					struct drm_plane *plane,
-					enum exynos_drm_output_type type,
+					enum exynos_drm_output_type out_type,
 					const struct exynos_drm_crtc_ops *ops,
 					void *context);
 void exynos_drm_crtc_wait_pending_update(struct exynos_drm_crtc *exynos_crtc);
 void exynos_drm_crtc_finish_update(struct exynos_drm_crtc *exynos_crtc,
 				   struct exynos_drm_plane *exynos_plane);
 
-/* This function gets pipe value to crtc device matched with out_type. */
-int exynos_drm_crtc_get_pipe_from_type(struct drm_device *drm_dev,
+/* This function gets crtc device matched with out_type. */
+struct exynos_drm_crtc *exynos_drm_crtc_get_by_type(struct drm_device *drm_dev,
 				       enum exynos_drm_output_type out_type);
 
+int exynos_drm_set_possible_crtcs(struct drm_encoder *encoder,
+		enum exynos_drm_output_type out_type);
+
 /*
  * This function calls the crtc device(manager)'s te_handler() callback
  * to trigger to transfer video image at the tearing effect synchronization
diff --git a/drivers/gpu/drm/exynos/exynos_drm_dpi.c b/drivers/gpu/drm/exynos/exynos_drm_dpi.c
index 63abcd280fa0..66945e0dc57f 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_dpi.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_dpi.c
@@ -59,7 +59,6 @@ static void exynos_dpi_connector_destroy(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs exynos_dpi_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.detect = exynos_dpi_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = exynos_dpi_connector_destroy,
@@ -203,19 +202,15 @@ int exynos_dpi_bind(struct drm_device *dev, struct drm_encoder *encoder)
 {
 	int ret;
 
-	ret = exynos_drm_crtc_get_pipe_from_type(dev, EXYNOS_DISPLAY_TYPE_LCD);
-	if (ret < 0)
-		return ret;
-
-	encoder->possible_crtcs = 1 << ret;
-
-	DRM_DEBUG_KMS("possible_crtcs = 0x%x\n", encoder->possible_crtcs);
-
 	drm_encoder_init(dev, encoder, &exynos_dpi_encoder_funcs,
 			 DRM_MODE_ENCODER_TMDS, NULL);
 
 	drm_encoder_helper_add(encoder, &exynos_dpi_encoder_helper_funcs);
 
+	ret = exynos_drm_set_possible_crtcs(encoder, EXYNOS_DISPLAY_TYPE_LCD);
+	if (ret < 0)
+		return ret;
+
 	ret = exynos_dpi_create_connector(encoder);
 	if (ret) {
 		DRM_ERROR("failed to create connector ret = %d\n", ret);
diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.c b/drivers/gpu/drm/exynos/exynos_drm_drv.c
index 242bd50faa26..e651a58c18cf 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_drv.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_drv.c
@@ -145,8 +145,6 @@ static struct drm_driver exynos_drm_driver = {
 	.gem_free_object_unlocked = exynos_drm_gem_free_object,
 	.gem_vm_ops		= &exynos_drm_gem_vm_ops,
 	.dumb_create		= exynos_drm_gem_dumb_create,
-	.dumb_map_offset	= exynos_drm_gem_dumb_map_offset,
-	.dumb_destroy		= drm_gem_dumb_destroy,
 	.prime_handle_to_fd	= drm_gem_prime_handle_to_fd,
 	.prime_fd_to_handle	= drm_gem_prime_fd_to_handle,
 	.gem_prime_export	= drm_gem_prime_export,
@@ -170,23 +168,19 @@ static struct drm_driver exynos_drm_driver = {
 static int exynos_drm_suspend(struct device *dev)
 {
 	struct drm_device *drm_dev = dev_get_drvdata(dev);
-	struct drm_connector *connector;
-	struct drm_connector_list_iter conn_iter;
+	struct exynos_drm_private *private = drm_dev->dev_private;
 
 	if (pm_runtime_suspended(dev) || !drm_dev)
 		return 0;
 
-	drm_connector_list_iter_begin(drm_dev, &conn_iter);
-	drm_for_each_connector_iter(connector, &conn_iter) {
-		int old_dpms = connector->dpms;
-
-		if (connector->funcs->dpms)
-			connector->funcs->dpms(connector, DRM_MODE_DPMS_OFF);
-
-		/* Set the old mode back to the connector for resume */
-		connector->dpms = old_dpms;
+	drm_kms_helper_poll_disable(drm_dev);
+	exynos_drm_fbdev_suspend(drm_dev);
+	private->suspend_state = drm_atomic_helper_suspend(drm_dev);
+	if (IS_ERR(private->suspend_state)) {
+		exynos_drm_fbdev_resume(drm_dev);
+		drm_kms_helper_poll_enable(drm_dev);
+		return PTR_ERR(private->suspend_state);
 	}
-	drm_connector_list_iter_end(&conn_iter);
 
 	return 0;
 }
@@ -194,22 +188,14 @@ static int exynos_drm_suspend(struct device *dev)
 static int exynos_drm_resume(struct device *dev)
 {
 	struct drm_device *drm_dev = dev_get_drvdata(dev);
-	struct drm_connector *connector;
-	struct drm_connector_list_iter conn_iter;
+	struct exynos_drm_private *private = drm_dev->dev_private;
 
 	if (pm_runtime_suspended(dev) || !drm_dev)
 		return 0;
 
-	drm_connector_list_iter_begin(drm_dev, &conn_iter);
-	drm_for_each_connector_iter(connector, &conn_iter) {
-		if (connector->funcs->dpms) {
-			int dpms = connector->dpms;
-
-			connector->dpms = DRM_MODE_DPMS_OFF;
-			connector->funcs->dpms(connector, dpms);
-		}
-	}
-	drm_connector_list_iter_end(&conn_iter);
+	drm_atomic_helper_resume(drm_dev, private->suspend_state);
+	exynos_drm_fbdev_resume(drm_dev);
+	drm_kms_helper_poll_enable(drm_dev);
 
 	return 0;
 }
@@ -395,8 +381,9 @@ static int exynos_drm_bind(struct device *dev)
 	/* init kms poll for handling hpd */
 	drm_kms_helper_poll_init(drm);
 
-	/* force connectors detection */
-	drm_helper_hpd_irq_event(drm);
+	ret = exynos_drm_fbdev_init(drm);
+	if (ret)
+		goto err_cleanup_poll;
 
 	/* register the DRM device */
 	ret = drm_dev_register(drm, 0);
@@ -407,6 +394,7 @@ static int exynos_drm_bind(struct device *dev)
 
 err_cleanup_fbdev:
 	exynos_drm_fbdev_fini(drm);
+err_cleanup_poll:
 	drm_kms_helper_poll_fini(drm);
 	exynos_drm_device_subdrv_remove(drm);
 err_unbind_all:
diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.h b/drivers/gpu/drm/exynos/exynos_drm_drv.h
index a93de321706b..f8bae4cb4823 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_drv.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_drv.h
@@ -91,6 +91,7 @@ struct exynos_drm_plane {
 #define EXYNOS_DRM_PLANE_CAP_DOUBLE	(1 << 0)
 #define EXYNOS_DRM_PLANE_CAP_SCALE	(1 << 1)
 #define EXYNOS_DRM_PLANE_CAP_ZPOS	(1 << 2)
+#define EXYNOS_DRM_PLANE_CAP_TILE	(1 << 3)
 
 /*
  * Exynos DRM plane configuration structure.
@@ -117,6 +118,7 @@ struct exynos_drm_plane_config {
  * @disable: disable the device
  * @enable_vblank: specific driver callback for enabling vblank interrupt.
  * @disable_vblank: specific driver callback for disabling vblank interrupt.
+ * @mode_valid: specific driver callback for mode validation
  * @atomic_check: validate state
  * @atomic_begin: prepare device to receive an update
  * @atomic_flush: mark the end of device update
@@ -132,6 +134,8 @@ struct exynos_drm_crtc_ops {
 	int (*enable_vblank)(struct exynos_drm_crtc *crtc);
 	void (*disable_vblank)(struct exynos_drm_crtc *crtc);
 	u32 (*get_vblank_counter)(struct exynos_drm_crtc *crtc);
+	enum drm_mode_status (*mode_valid)(struct exynos_drm_crtc *crtc,
+		const struct drm_display_mode *mode);
 	int (*atomic_check)(struct exynos_drm_crtc *crtc,
 			    struct drm_crtc_state *state);
 	void (*atomic_begin)(struct exynos_drm_crtc *crtc);
@@ -162,6 +166,7 @@ struct exynos_drm_crtc {
 	const struct exynos_drm_crtc_ops	*ops;
 	void				*ctx;
 	struct exynos_drm_clk		*pipe_clk;
+	bool				i80_mode : 1;
 };
 
 static inline void exynos_drm_pipe_clk_enable(struct exynos_drm_crtc *crtc,
@@ -197,6 +202,7 @@ struct drm_exynos_file_private {
  */
 struct exynos_drm_private {
 	struct drm_fb_helper *fb_helper;
+	struct drm_atomic_state *suspend_state;
 
 	struct device *dma_dev;
 	void *mapping;
diff --git a/drivers/gpu/drm/exynos/exynos_drm_dsi.c b/drivers/gpu/drm/exynos/exynos_drm_dsi.c
index b6a46d9a016e..7904ffa9abfb 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_dsi.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_dsi.c
@@ -254,7 +254,6 @@ struct exynos_dsi {
 	struct drm_encoder encoder;
 	struct mipi_dsi_host dsi_host;
 	struct drm_connector connector;
-	struct device_node *panel_node;
 	struct drm_panel *panel;
 	struct device *dev;
 
@@ -1329,12 +1328,13 @@ static int exynos_dsi_init(struct exynos_dsi *dsi)
 	return 0;
 }
 
-static int exynos_dsi_register_te_irq(struct exynos_dsi *dsi)
+static int exynos_dsi_register_te_irq(struct exynos_dsi *dsi,
+				      struct device *panel)
 {
 	int ret;
 	int te_gpio_irq;
 
-	dsi->te_gpio = of_get_named_gpio(dsi->panel_node, "te-gpios", 0);
+	dsi->te_gpio = of_get_named_gpio(panel->of_node, "te-gpios", 0);
 	if (dsi->te_gpio == -ENOENT)
 		return 0;
 
@@ -1374,85 +1374,6 @@ static void exynos_dsi_unregister_te_irq(struct exynos_dsi *dsi)
 	}
 }
 
-static int exynos_dsi_host_attach(struct mipi_dsi_host *host,
-				  struct mipi_dsi_device *device)
-{
-	struct exynos_dsi *dsi = host_to_dsi(host);
-
-	dsi->lanes = device->lanes;
-	dsi->format = device->format;
-	dsi->mode_flags = device->mode_flags;
-	dsi->panel_node = device->dev.of_node;
-
-	/*
-	 * This is a temporary solution and should be made by more generic way.
-	 *
-	 * If attached panel device is for command mode one, dsi should register
-	 * TE interrupt handler.
-	 */
-	if (!(dsi->mode_flags & MIPI_DSI_MODE_VIDEO)) {
-		int ret = exynos_dsi_register_te_irq(dsi);
-
-		if (ret)
-			return ret;
-	}
-
-	if (dsi->connector.dev)
-		drm_helper_hpd_irq_event(dsi->connector.dev);
-
-	return 0;
-}
-
-static int exynos_dsi_host_detach(struct mipi_dsi_host *host,
-				  struct mipi_dsi_device *device)
-{
-	struct exynos_dsi *dsi = host_to_dsi(host);
-
-	exynos_dsi_unregister_te_irq(dsi);
-
-	dsi->panel_node = NULL;
-
-	if (dsi->connector.dev)
-		drm_helper_hpd_irq_event(dsi->connector.dev);
-
-	return 0;
-}
-
-static ssize_t exynos_dsi_host_transfer(struct mipi_dsi_host *host,
-				        const struct mipi_dsi_msg *msg)
-{
-	struct exynos_dsi *dsi = host_to_dsi(host);
-	struct exynos_dsi_transfer xfer;
-	int ret;
-
-	if (!(dsi->state & DSIM_STATE_ENABLED))
-		return -EINVAL;
-
-	if (!(dsi->state & DSIM_STATE_INITIALIZED)) {
-		ret = exynos_dsi_init(dsi);
-		if (ret)
-			return ret;
-		dsi->state |= DSIM_STATE_INITIALIZED;
-	}
-
-	ret = mipi_dsi_create_packet(&xfer.packet, msg);
-	if (ret < 0)
-		return ret;
-
-	xfer.rx_len = msg->rx_len;
-	xfer.rx_payload = msg->rx_buf;
-	xfer.flags = msg->flags;
-
-	ret = exynos_dsi_transfer(dsi, &xfer);
-	return (ret < 0) ? ret : xfer.rx_done;
-}
-
-static const struct mipi_dsi_host_ops exynos_dsi_ops = {
-	.attach = exynos_dsi_host_attach,
-	.detach = exynos_dsi_host_detach,
-	.transfer = exynos_dsi_host_transfer,
-};
-
 static void exynos_dsi_enable(struct drm_encoder *encoder)
 {
 	struct exynos_dsi *dsi = encoder_to_dsi(encoder);
@@ -1508,25 +1429,7 @@ static void exynos_dsi_disable(struct drm_encoder *encoder)
 static enum drm_connector_status
 exynos_dsi_detect(struct drm_connector *connector, bool force)
 {
-	struct exynos_dsi *dsi = connector_to_dsi(connector);
-
-	if (!dsi->panel) {
-		dsi->panel = of_drm_find_panel(dsi->panel_node);
-		if (dsi->panel)
-			drm_panel_attach(dsi->panel, &dsi->connector);
-	} else if (!dsi->panel_node) {
-		struct drm_encoder *encoder;
-
-		encoder = platform_get_drvdata(to_platform_device(dsi->dev));
-		exynos_dsi_disable(encoder);
-		drm_panel_detach(dsi->panel);
-		dsi->panel = NULL;
-	}
-
-	if (dsi->panel)
-		return connector_status_connected;
-
-	return connector_status_disconnected;
+	return connector->status;
 }
 
 static void exynos_dsi_connector_destroy(struct drm_connector *connector)
@@ -1537,7 +1440,6 @@ static void exynos_dsi_connector_destroy(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs exynos_dsi_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.detect = exynos_dsi_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = exynos_dsi_connector_destroy,
@@ -1576,6 +1478,7 @@ static int exynos_dsi_create_connector(struct drm_encoder *encoder)
 		return ret;
 	}
 
+	connector->status = connector_status_disconnected;
 	drm_connector_helper_add(connector, &exynos_dsi_connector_helper_funcs);
 	drm_mode_connector_attach_encoder(connector, encoder);
 
@@ -1612,14 +1515,112 @@ static const struct drm_encoder_funcs exynos_dsi_encoder_funcs = {
 
 MODULE_DEVICE_TABLE(of, exynos_dsi_of_match);
 
+static int exynos_dsi_host_attach(struct mipi_dsi_host *host,
+				  struct mipi_dsi_device *device)
+{
+	struct exynos_dsi *dsi = host_to_dsi(host);
+	struct drm_device *drm = dsi->connector.dev;
+
+	/*
+	 * This is a temporary solution and should be made by more generic way.
+	 *
+	 * If attached panel device is for command mode one, dsi should register
+	 * TE interrupt handler.
+	 */
+	if (!(device->mode_flags & MIPI_DSI_MODE_VIDEO)) {
+		int ret = exynos_dsi_register_te_irq(dsi, &device->dev);
+		if (ret)
+			return ret;
+	}
+
+	mutex_lock(&drm->mode_config.mutex);
+
+	dsi->lanes = device->lanes;
+	dsi->format = device->format;
+	dsi->mode_flags = device->mode_flags;
+	dsi->panel = of_drm_find_panel(device->dev.of_node);
+	if (dsi->panel) {
+		drm_panel_attach(dsi->panel, &dsi->connector);
+		dsi->connector.status = connector_status_connected;
+	}
+	exynos_drm_crtc_get_by_type(drm, EXYNOS_DISPLAY_TYPE_LCD)->i80_mode =
+			!(dsi->mode_flags & MIPI_DSI_MODE_VIDEO);
+
+	mutex_unlock(&drm->mode_config.mutex);
+
+	if (drm->mode_config.poll_enabled)
+		drm_kms_helper_hotplug_event(drm);
+
+	return 0;
+}
+
+static int exynos_dsi_host_detach(struct mipi_dsi_host *host,
+				  struct mipi_dsi_device *device)
+{
+	struct exynos_dsi *dsi = host_to_dsi(host);
+	struct drm_device *drm = dsi->connector.dev;
+
+	mutex_lock(&drm->mode_config.mutex);
+
+	if (dsi->panel) {
+		exynos_dsi_disable(&dsi->encoder);
+		drm_panel_detach(dsi->panel);
+		dsi->panel = NULL;
+		dsi->connector.status = connector_status_disconnected;
+	}
+
+	mutex_unlock(&drm->mode_config.mutex);
+
+	if (drm->mode_config.poll_enabled)
+		drm_kms_helper_hotplug_event(drm);
+
+	exynos_dsi_unregister_te_irq(dsi);
+
+	return 0;
+}
+
+static ssize_t exynos_dsi_host_transfer(struct mipi_dsi_host *host,
+					 const struct mipi_dsi_msg *msg)
+{
+	struct exynos_dsi *dsi = host_to_dsi(host);
+	struct exynos_dsi_transfer xfer;
+	int ret;
+
+	if (!(dsi->state & DSIM_STATE_ENABLED))
+		return -EINVAL;
+
+	if (!(dsi->state & DSIM_STATE_INITIALIZED)) {
+		ret = exynos_dsi_init(dsi);
+		if (ret)
+			return ret;
+		dsi->state |= DSIM_STATE_INITIALIZED;
+	}
+
+	ret = mipi_dsi_create_packet(&xfer.packet, msg);
+	if (ret < 0)
+		return ret;
+
+	xfer.rx_len = msg->rx_len;
+	xfer.rx_payload = msg->rx_buf;
+	xfer.flags = msg->flags;
+
+	ret = exynos_dsi_transfer(dsi, &xfer);
+	return (ret < 0) ? ret : xfer.rx_done;
+}
+
+static const struct mipi_dsi_host_ops exynos_dsi_ops = {
+	.attach = exynos_dsi_host_attach,
+	.detach = exynos_dsi_host_detach,
+	.transfer = exynos_dsi_host_transfer,
+};
+
 static int exynos_dsi_of_read_u32(const struct device_node *np,
 				  const char *propname, u32 *out_value)
 {
 	int ret = of_property_read_u32(np, propname, out_value);
 
 	if (ret < 0)
-		pr_err("%s: failed to get '%s' property\n", np->full_name,
-		       propname);
+		pr_err("%pOF: failed to get '%s' property\n", np, propname);
 
 	return ret;
 }
@@ -1664,20 +1665,15 @@ static int exynos_dsi_bind(struct device *dev, struct device *master,
 	struct drm_bridge *bridge;
 	int ret;
 
-	ret = exynos_drm_crtc_get_pipe_from_type(drm_dev,
-						  EXYNOS_DISPLAY_TYPE_LCD);
-	if (ret < 0)
-		return ret;
-
-	encoder->possible_crtcs = 1 << ret;
-
-	DRM_DEBUG_KMS("possible_crtcs = 0x%x\n", encoder->possible_crtcs);
-
 	drm_encoder_init(drm_dev, encoder, &exynos_dsi_encoder_funcs,
 			 DRM_MODE_ENCODER_TMDS, NULL);
 
 	drm_encoder_helper_add(encoder, &exynos_dsi_encoder_helper_funcs);
 
+	ret = exynos_drm_set_possible_crtcs(encoder, EXYNOS_DISPLAY_TYPE_LCD);
+	if (ret < 0)
+		return ret;
+
 	ret = exynos_dsi_create_connector(encoder);
 	if (ret) {
 		DRM_ERROR("failed to create connector ret = %d\n", ret);
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fb.c b/drivers/gpu/drm/exynos/exynos_drm_fb.c
index 73217c281c9a..8208df56a88f 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fb.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fb.c
@@ -199,33 +199,8 @@ dma_addr_t exynos_drm_fb_dma_addr(struct drm_framebuffer *fb, int index)
 	return exynos_fb->dma_addr[index];
 }
 
-static void exynos_drm_atomic_commit_tail(struct drm_atomic_state *state)
-{
-	struct drm_device *dev = state->dev;
-
-	drm_atomic_helper_commit_modeset_disables(dev, state);
-
-	drm_atomic_helper_commit_modeset_enables(dev, state);
-
-	/*
-	 * Exynos can't update planes with CRTCs and encoders disabled,
-	 * its updates routines, specially for FIMD, requires the clocks
-	 * to be enabled. So it is necessary to handle the modeset operations
-	 * *before* the commit_planes() step, this way it will always
-	 * have the relevant clocks enabled to perform the update.
-	 */
-	drm_atomic_helper_commit_planes(dev, state,
-					DRM_PLANE_COMMIT_ACTIVE_ONLY);
-
-	drm_atomic_helper_commit_hw_done(state);
-
-	drm_atomic_helper_wait_for_vblanks(dev, state);
-
-	drm_atomic_helper_cleanup_planes(dev, state);
-}
-
 static struct drm_mode_config_helper_funcs exynos_drm_mode_config_helpers = {
-	.atomic_commit_tail = exynos_drm_atomic_commit_tail,
+	.atomic_commit_tail = drm_atomic_helper_commit_tail_rpm,
 };
 
 static const struct drm_mode_config_funcs exynos_drm_mode_config_funcs = {
@@ -250,4 +225,6 @@ void exynos_drm_mode_config_init(struct drm_device *dev)
 
 	dev->mode_config.funcs = &exynos_drm_mode_config_funcs;
 	dev->mode_config.helper_private = &exynos_drm_mode_config_helpers;
+
+	dev->mode_config.allow_fb_modifiers = true;
 }
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fbdev.c b/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
index 641531243e04..dfb66ecf417b 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
@@ -18,6 +18,8 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/exynos_drm.h>
 
+#include <linux/console.h>
+
 #include "exynos_drm_drv.h"
 #include "exynos_drm_fb.h"
 #include "exynos_drm_fbdev.h"
@@ -183,24 +185,6 @@ static const struct drm_fb_helper_funcs exynos_drm_fb_helper_funcs = {
 	.fb_probe =	exynos_drm_fbdev_create,
 };
 
-static bool exynos_drm_fbdev_is_anything_connected(struct drm_device *dev)
-{
-	struct drm_connector *connector;
-	bool ret = false;
-
-	mutex_lock(&dev->mode_config.mutex);
-	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-		if (connector->status != connector_status_connected)
-			continue;
-
-		ret = true;
-		break;
-	}
-	mutex_unlock(&dev->mode_config.mutex);
-
-	return ret;
-}
-
 int exynos_drm_fbdev_init(struct drm_device *dev)
 {
 	struct exynos_drm_fbdev *fbdev;
@@ -211,9 +195,6 @@ int exynos_drm_fbdev_init(struct drm_device *dev)
 	if (!dev->mode_config.num_crtc || !dev->mode_config.num_connector)
 		return 0;
 
-	if (!exynos_drm_fbdev_is_anything_connected(dev))
-		return 0;
-
 	fbdev = kzalloc(sizeof(*fbdev), GFP_KERNEL);
 	if (!fbdev)
 		return -ENOMEM;
@@ -304,8 +285,23 @@ void exynos_drm_output_poll_changed(struct drm_device *dev)
 	struct exynos_drm_private *private = dev->dev_private;
 	struct drm_fb_helper *fb_helper = private->fb_helper;
 
-	if (fb_helper)
-		drm_fb_helper_hotplug_event(fb_helper);
-	else
-		exynos_drm_fbdev_init(dev);
+	drm_fb_helper_hotplug_event(fb_helper);
+}
+
+void exynos_drm_fbdev_suspend(struct drm_device *dev)
+{
+	struct exynos_drm_private *private = dev->dev_private;
+
+	console_lock();
+	drm_fb_helper_set_suspend(private->fb_helper, 1);
+	console_unlock();
+}
+
+void exynos_drm_fbdev_resume(struct drm_device *dev)
+{
+	struct exynos_drm_private *private = dev->dev_private;
+
+	console_lock();
+	drm_fb_helper_set_suspend(private->fb_helper, 0);
+	console_unlock();
 }
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fbdev.h b/drivers/gpu/drm/exynos/exynos_drm_fbdev.h
index 330eef87f718..645d1bb7f665 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fbdev.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_fbdev.h
@@ -21,6 +21,8 @@ int exynos_drm_fbdev_init(struct drm_device *dev);
 void exynos_drm_fbdev_fini(struct drm_device *dev);
 void exynos_drm_fbdev_restore_mode(struct drm_device *dev);
 void exynos_drm_output_poll_changed(struct drm_device *dev);
+void exynos_drm_fbdev_suspend(struct drm_device *drm);
+void exynos_drm_fbdev_resume(struct drm_device *drm);
 
 #else
 
@@ -39,6 +41,14 @@ static inline void exynos_drm_fbdev_restore_mode(struct drm_device *dev)
 
 #define exynos_drm_output_poll_changed (NULL)
 
+static inline void exynos_drm_fbdev_suspend(struct drm_device *drm)
+{
+}
+
+static inline void exynos_drm_fbdev_resume(struct drm_device *drm)
+{
+}
+
 #endif
 
 #endif
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fimd.c b/drivers/gpu/drm/exynos/exynos_drm_fimd.c
index 60f93cad6643..d42ae2bc3e56 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fimd.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fimd.c
@@ -583,18 +583,12 @@ static void fimd_win_set_pixfmt(struct fimd_context *ctx, unsigned int win,
 		val |= WINCONx_BURSTLEN_16WORD;
 		break;
 	case DRM_FORMAT_ARGB8888:
+	default:
 		val |= WINCON1_BPPMODE_25BPP_A1888
 			| WINCON1_BLD_PIX | WINCON1_ALPHA_SEL;
 		val |= WINCONx_WSWP;
 		val |= WINCONx_BURSTLEN_16WORD;
 		break;
-	default:
-		DRM_DEBUG_KMS("invalid pixel size so using unpacked 24bpp.\n");
-
-		val |= WINCON0_BPPMODE_24BPP_888;
-		val |= WINCONx_WSWP;
-		val |= WINCONx_BURSTLEN_16WORD;
-		break;
 	}
 
 	/*
@@ -718,13 +712,13 @@ static void fimd_update_plane(struct exynos_drm_crtc *crtc,
 	unsigned long val, size, offset;
 	unsigned int last_x, last_y, buf_offsize, line_size;
 	unsigned int win = plane->index;
-	unsigned int bpp = fb->format->cpp[0];
+	unsigned int cpp = fb->format->cpp[0];
 	unsigned int pitch = fb->pitches[0];
 
 	if (ctx->suspended)
 		return;
 
-	offset = state->src.x * bpp;
+	offset = state->src.x * cpp;
 	offset += state->src.y * pitch;
 
 	/* buffer start address */
@@ -743,8 +737,8 @@ static void fimd_update_plane(struct exynos_drm_crtc *crtc,
 			state->crtc.w, state->crtc.h);
 
 	/* buffer size */
-	buf_offsize = pitch - (state->crtc.w * bpp);
-	line_size = state->crtc.w * bpp;
+	buf_offsize = pitch - (state->crtc.w * cpp);
+	line_size = state->crtc.w * cpp;
 	val = VIDW_BUF_SIZE_OFFSET(buf_offsize) |
 		VIDW_BUF_SIZE_PAGEWIDTH(line_size) |
 		VIDW_BUF_SIZE_OFFSET_E(buf_offsize) |
diff --git a/drivers/gpu/drm/exynos/exynos_drm_gem.c b/drivers/gpu/drm/exynos/exynos_drm_gem.c
index c23479be4850..077de014d610 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_gem.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_gem.c
@@ -286,8 +286,8 @@ int exynos_drm_gem_map_ioctl(struct drm_device *dev, void *data,
 {
 	struct drm_exynos_gem_map *args = data;
 
-	return exynos_drm_gem_dumb_map_offset(file_priv, dev, args->handle,
-					      &args->offset);
+	return drm_gem_dumb_map_offset(file_priv, dev, args->handle,
+				       &args->offset);
 }
 
 dma_addr_t *exynos_drm_gem_get_dma_addr(struct drm_device *dev,
@@ -422,32 +422,6 @@ int exynos_drm_gem_dumb_create(struct drm_file *file_priv,
 	return 0;
 }
 
-int exynos_drm_gem_dumb_map_offset(struct drm_file *file_priv,
-				   struct drm_device *dev, uint32_t handle,
-				   uint64_t *offset)
-{
-	struct drm_gem_object *obj;
-	int ret = 0;
-
-	/*
-	 * get offset of memory allocated for drm framebuffer.
-	 * - this callback would be called by user application
-	 *	with DRM_IOCTL_MODE_MAP_DUMB command.
-	 */
-
-	obj = drm_gem_object_lookup(file_priv, handle);
-	if (!obj) {
-		DRM_ERROR("failed to lookup gem object.\n");
-		return -EINVAL;
-	}
-
-	*offset = drm_vma_node_offset_addr(&obj->vma_node);
-	DRM_DEBUG_KMS("offset = 0x%lx\n", (unsigned long)*offset);
-
-	drm_gem_object_unreference_unlocked(obj);
-	return ret;
-}
-
 int exynos_drm_gem_fault(struct vm_fault *vmf)
 {
 	struct vm_area_struct *vma = vmf->vma;
diff --git a/drivers/gpu/drm/exynos/exynos_drm_gem.h b/drivers/gpu/drm/exynos/exynos_drm_gem.h
index 85457255fcd1..e86d1a9518c3 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_gem.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_gem.h
@@ -110,11 +110,6 @@ int exynos_drm_gem_dumb_create(struct drm_file *file_priv,
 			       struct drm_device *dev,
 			       struct drm_mode_create_dumb *args);
 
-/* map memory region for drm framebuffer to user space. */
-int exynos_drm_gem_dumb_map_offset(struct drm_file *file_priv,
-				   struct drm_device *dev, uint32_t handle,
-				   uint64_t *offset);
-
 /* page fault handler and mmap fault address(virtual) to physical memory. */
 int exynos_drm_gem_fault(struct vm_fault *vmf);
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_mic.c b/drivers/gpu/drm/exynos/exynos_drm_mic.c
index 16bbee897e0d..ba4a32b132ba 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_mic.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_mic.c
@@ -21,9 +21,12 @@
 #include <linux/component.h>
 #include <linux/pm_runtime.h>
 #include <drm/drmP.h>
+#include <drm/drm_encoder.h>
 #include <linux/mfd/syscon.h>
 #include <linux/regmap.h>
 
+#include "exynos_drm_drv.h"
+
 /* Sysreg registers for MIC */
 #define DSD_CFG_MUX	0x1004
 #define MIC0_RGB_MUX	(1 << 0)
@@ -85,12 +88,6 @@
 
 #define MIC_BS_SIZE_2D(x)	((x) & 0x3fff)
 
-enum {
-	ENDPOINT_DECON_NODE,
-	ENDPOINT_DSI_NODE,
-	NUM_ENDPOINTS
-};
-
 static char *clk_names[] = { "pclk_mic0", "sclk_rgb_vclk_to_mic0" };
 #define NUM_CLKS		ARRAY_SIZE(clk_names)
 static DEFINE_MUTEX(mic_mutex);
@@ -229,36 +226,6 @@ static void mic_set_reg_on(struct exynos_mic *mic, bool enable)
 	writel(reg, mic->reg + MIC_OP);
 }
 
-static int parse_dt(struct exynos_mic *mic)
-{
-	int ret = 0, i, j;
-	struct device_node *remote_node;
-	struct device_node *nodes[3];
-
-	/*
-	 * The order of endpoints does matter.
-	 * The first node must be for decon and the second one must be for dsi.
-	 */
-	for (i = 0, j = 0; i < NUM_ENDPOINTS; i++) {
-		remote_node = of_graph_get_remote_node(mic->dev->of_node, i, 0);
-		if (!remote_node) {
-			ret = -EPIPE;
-			goto exit;
-		}
-		nodes[j++] = remote_node;
-
-		if (i == ENDPOINT_DECON_NODE &&
-			of_get_child_by_name(remote_node, "i80-if-timings"))
-			mic->i80_mode = 1;
-	}
-
-exit:
-	while (--j > -1)
-		of_node_put(nodes[j]);
-
-	return ret;
-}
-
 static void mic_disable(struct drm_bridge *bridge) { }
 
 static void mic_post_disable(struct drm_bridge *bridge)
@@ -286,6 +253,7 @@ static void mic_mode_set(struct drm_bridge *bridge,
 
 	mutex_lock(&mic_mutex);
 	drm_display_mode_to_videomode(mode, &mic->vm);
+	mic->i80_mode = to_exynos_crtc(bridge->encoder->crtc)->i80_mode;
 	mutex_unlock(&mic_mutex);
 }
 
@@ -417,10 +385,6 @@ static int exynos_mic_probe(struct platform_device *pdev)
 
 	mic->dev = dev;
 
-	ret = parse_dt(mic);
-	if (ret)
-		goto err;
-
 	ret = of_address_to_resource(dev->of_node, 0, &res);
 	if (ret) {
 		DRM_ERROR("mic: Failed to get mem region for MIC\n");
diff --git a/drivers/gpu/drm/exynos/exynos_drm_plane.c b/drivers/gpu/drm/exynos/exynos_drm_plane.c
index 611b6fd65433..d2a90dae5c71 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_plane.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_plane.c
@@ -173,13 +173,35 @@ static struct drm_plane_funcs exynos_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
 	.destroy	= drm_plane_cleanup,
-	.set_property	= drm_atomic_helper_plane_set_property,
 	.reset		= exynos_drm_plane_reset,
 	.atomic_duplicate_state = exynos_drm_plane_duplicate_state,
 	.atomic_destroy_state = exynos_drm_plane_destroy_state,
 };
 
 static int
+exynos_drm_plane_check_format(const struct exynos_drm_plane_config *config,
+			      struct exynos_drm_plane_state *state)
+{
+	struct drm_framebuffer *fb = state->base.fb;
+
+	switch (fb->modifier) {
+	case DRM_FORMAT_MOD_SAMSUNG_64_32_TILE:
+		if (!(config->capabilities & EXYNOS_DRM_PLANE_CAP_TILE))
+			return -ENOTSUPP;
+		break;
+
+	case DRM_FORMAT_MOD_LINEAR:
+		break;
+
+	default:
+		DRM_ERROR("unsupported pixel format modifier");
+		return -ENOTSUPP;
+	}
+
+	return 0;
+}
+
+static int
 exynos_drm_plane_check_size(const struct exynos_drm_plane_config *config,
 			    struct exynos_drm_plane_state *state)
 {
@@ -223,6 +245,10 @@ static int exynos_plane_atomic_check(struct drm_plane *plane,
 	/* translate state into exynos_state */
 	exynos_plane_mode_set(exynos_state);
 
+	ret = exynos_drm_plane_check_format(exynos_plane->config, exynos_state);
+	if (ret)
+		return ret;
+
 	ret = exynos_drm_plane_check_size(exynos_plane->config, exynos_state);
 	return ret;
 }
@@ -283,7 +309,7 @@ int exynos_plane_init(struct drm_device *dev,
 				       &exynos_plane_funcs,
 				       config->pixel_formats,
 				       config->num_pixel_formats,
-				       config->type, NULL);
+				       NULL, config->type, NULL);
 	if (err) {
 		DRM_ERROR("failed to initialize plane\n");
 		return err;
diff --git a/drivers/gpu/drm/exynos/exynos_drm_vidi.c b/drivers/gpu/drm/exynos/exynos_drm_vidi.c
index cb8a72842537..53e03f8af3d5 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_vidi.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_vidi.c
@@ -289,7 +289,6 @@ static void vidi_connector_destroy(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs vidi_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = vidi_detect,
 	.destroy = vidi_connector_destroy,
@@ -382,7 +381,7 @@ static int vidi_bind(struct device *dev, struct device *master, void *data)
 	struct exynos_drm_plane *exynos_plane;
 	struct exynos_drm_plane_config plane_config = { 0 };
 	unsigned int i;
-	int pipe, ret;
+	int ret;
 
 	ctx->drm_dev = drm_dev;
 
@@ -407,20 +406,15 @@ static int vidi_bind(struct device *dev, struct device *master, void *data)
 		return PTR_ERR(ctx->crtc);
 	}
 
-	pipe = exynos_drm_crtc_get_pipe_from_type(drm_dev,
-						  EXYNOS_DISPLAY_TYPE_VIDI);
-	if (pipe < 0)
-		return pipe;
-
-	encoder->possible_crtcs = 1 << pipe;
-
-	DRM_DEBUG_KMS("possible_crtcs = 0x%x\n", encoder->possible_crtcs);
-
 	drm_encoder_init(drm_dev, encoder, &exynos_vidi_encoder_funcs,
 			 DRM_MODE_ENCODER_TMDS, NULL);
 
 	drm_encoder_helper_add(encoder, &exynos_vidi_encoder_helper_funcs);
 
+	ret = exynos_drm_set_possible_crtcs(encoder, EXYNOS_DISPLAY_TYPE_VIDI);
+	if (ret < 0)
+		return ret;
+
 	ret = vidi_create_connector(encoder);
 	if (ret) {
 		DRM_ERROR("failed to create connector ret = %d\n", ret);
diff --git a/drivers/gpu/drm/exynos/exynos_hdmi.c b/drivers/gpu/drm/exynos/exynos_hdmi.c
index d3b69d66736f..0109ff40b1db 100644
--- a/drivers/gpu/drm/exynos/exynos_hdmi.c
+++ b/drivers/gpu/drm/exynos/exynos_hdmi.c
@@ -784,7 +784,7 @@ static void hdmi_reg_infoframes(struct hdmi_context *hdata)
 	}
 
 	ret = drm_hdmi_avi_infoframe_from_display_mode(&frm.avi,
-			&hdata->current_mode);
+			&hdata->current_mode, false);
 	if (!ret)
 		ret = hdmi_avi_infoframe_pack(&frm.avi, buf, sizeof(buf));
 	if (ret > 0) {
@@ -835,7 +835,6 @@ static void hdmi_connector_destroy(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs hdmi_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = hdmi_detect,
 	.destroy = hdmi_connector_destroy,
@@ -945,22 +944,27 @@ static bool hdmi_mode_fixup(struct drm_encoder *encoder,
 	struct drm_device *dev = encoder->dev;
 	struct drm_connector *connector;
 	struct drm_display_mode *m;
+	struct drm_connector_list_iter conn_iter;
 	int mode_ok;
 
 	drm_mode_set_crtcinfo(adjusted_mode, 0);
 
-	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+	drm_connector_list_iter_begin(dev, &conn_iter);
+	drm_for_each_connector_iter(connector, &conn_iter) {
 		if (connector->encoder == encoder)
 			break;
 	}
+	if (connector)
+		drm_connector_get(connector);
+	drm_connector_list_iter_end(&conn_iter);
 
-	if (connector->encoder != encoder)
+	if (!connector)
 		return true;
 
 	mode_ok = hdmi_mode_valid(connector, adjusted_mode);
 
 	if (mode_ok == MODE_OK)
-		return true;
+		goto cleanup;
 
 	/*
 	 * Find the most suitable mode and copy it to adjusted_mode.
@@ -980,6 +984,9 @@ static bool hdmi_mode_fixup(struct drm_encoder *encoder,
 		}
 	}
 
+cleanup:
+	drm_connector_put(connector);
+
 	return true;
 }
 
@@ -1698,32 +1705,25 @@ static int hdmi_bind(struct device *dev, struct device *master, void *data)
 	struct drm_device *drm_dev = data;
 	struct hdmi_context *hdata = dev_get_drvdata(dev);
 	struct drm_encoder *encoder = &hdata->encoder;
-	struct exynos_drm_crtc *exynos_crtc;
-	struct drm_crtc *crtc;
-	int ret, pipe;
+	struct exynos_drm_crtc *crtc;
+	int ret;
 
 	hdata->drm_dev = drm_dev;
 
-	pipe = exynos_drm_crtc_get_pipe_from_type(drm_dev,
-						  EXYNOS_DISPLAY_TYPE_HDMI);
-	if (pipe < 0)
-		return pipe;
-
 	hdata->phy_clk.enable = hdmiphy_clk_enable;
 
-	crtc = drm_crtc_from_index(drm_dev, pipe);
-	exynos_crtc = to_exynos_crtc(crtc);
-	exynos_crtc->pipe_clk = &hdata->phy_clk;
-
-	encoder->possible_crtcs = 1 << pipe;
-
-	DRM_DEBUG_KMS("possible_crtcs = 0x%x\n", encoder->possible_crtcs);
-
 	drm_encoder_init(drm_dev, encoder, &exynos_hdmi_encoder_funcs,
 			 DRM_MODE_ENCODER_TMDS, NULL);
 
 	drm_encoder_helper_add(encoder, &exynos_hdmi_encoder_helper_funcs);
 
+	ret = exynos_drm_set_possible_crtcs(encoder, EXYNOS_DISPLAY_TYPE_HDMI);
+	if (ret < 0)
+		return ret;
+
+	crtc = exynos_drm_crtc_get_by_type(drm_dev, EXYNOS_DISPLAY_TYPE_HDMI);
+	crtc->pipe_clk = &hdata->phy_clk;
+
 	ret = hdmi_create_connector(encoder);
 	if (ret) {
 		DRM_ERROR("failed to create connector ret = %d\n", ret);
diff --git a/drivers/gpu/drm/exynos/exynos_mixer.c b/drivers/gpu/drm/exynos/exynos_mixer.c
index a998a8dd783c..002755415e00 100644
--- a/drivers/gpu/drm/exynos/exynos_mixer.c
+++ b/drivers/gpu/drm/exynos/exynos_mixer.c
@@ -148,7 +148,8 @@ static const struct exynos_drm_plane_config plane_configs[MIXER_WIN_NR] = {
 		.pixel_formats = vp_formats,
 		.num_pixel_formats = ARRAY_SIZE(vp_formats),
 		.capabilities = EXYNOS_DRM_PLANE_CAP_SCALE |
-				EXYNOS_DRM_PLANE_CAP_ZPOS,
+				EXYNOS_DRM_PLANE_CAP_ZPOS |
+				EXYNOS_DRM_PLANE_CAP_TILE,
 	},
 };
 
@@ -483,29 +484,18 @@ static void vp_video_buffer(struct mixer_context *ctx,
 	unsigned int priority = state->base.normalized_zpos + 1;
 	unsigned long flags;
 	dma_addr_t luma_addr[2], chroma_addr[2];
-	bool tiled_mode = false;
-	bool crcb_mode = false;
+	bool is_tiled, is_nv21;
 	u32 val;
 
-	switch (fb->format->format) {
-	case DRM_FORMAT_NV12:
-		crcb_mode = false;
-		break;
-	case DRM_FORMAT_NV21:
-		crcb_mode = true;
-		break;
-	default:
-		DRM_ERROR("pixel format for vp is wrong [%d].\n",
-				fb->format->format);
-		return;
-	}
+	is_nv21 = (fb->format->format == DRM_FORMAT_NV21);
+	is_tiled = (fb->modifier == DRM_FORMAT_MOD_SAMSUNG_64_32_TILE);
 
 	luma_addr[0] = exynos_drm_fb_dma_addr(fb, 0);
 	chroma_addr[0] = exynos_drm_fb_dma_addr(fb, 1);
 
 	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
 		__set_bit(MXR_BIT_INTERLACE, &ctx->flags);
-		if (tiled_mode) {
+		if (is_tiled) {
 			luma_addr[1] = luma_addr[0] + 0x40;
 			chroma_addr[1] = chroma_addr[0] + 0x40;
 		} else {
@@ -525,14 +515,14 @@ static void vp_video_buffer(struct mixer_context *ctx,
 	vp_reg_writemask(res, VP_MODE, val, VP_MODE_LINE_SKIP);
 
 	/* setup format */
-	val = (crcb_mode ? VP_MODE_NV21 : VP_MODE_NV12);
-	val |= (tiled_mode ? VP_MODE_MEM_TILED : VP_MODE_MEM_LINEAR);
+	val = (is_nv21 ? VP_MODE_NV21 : VP_MODE_NV12);
+	val |= (is_tiled ? VP_MODE_MEM_TILED : VP_MODE_MEM_LINEAR);
 	vp_reg_writemask(res, VP_MODE, val, VP_MODE_FMT_MASK);
 
 	/* setting size of input image */
 	vp_reg_write(res, VP_IMG_SIZE_Y, VP_IMG_HSIZE(fb->pitches[0]) |
 		VP_IMG_VSIZE(fb->height));
-	/* chroma height has to reduced by 2 to avoid chroma distorions */
+	/* chroma plane for NV12/NV21 is half the height of the luma plane */
 	vp_reg_write(res, VP_IMG_SIZE_C, VP_IMG_HSIZE(fb->pitches[0]) |
 		VP_IMG_VSIZE(fb->height / 2));
 
@@ -594,7 +584,7 @@ static void mixer_graph_buffer(struct mixer_context *ctx,
 	unsigned long flags;
 	unsigned int win = plane->index;
 	unsigned int x_ratio = 0, y_ratio = 0;
-	unsigned int src_x_offset, src_y_offset, dst_x_offset, dst_y_offset;
+	unsigned int dst_x_offset, dst_y_offset;
 	dma_addr_t dma_addr;
 	unsigned int fmt;
 	u32 val;
@@ -616,12 +606,9 @@ static void mixer_graph_buffer(struct mixer_context *ctx,
 
 	case DRM_FORMAT_XRGB8888:
 	case DRM_FORMAT_ARGB8888:
+	default:
 		fmt = MXR_FORMAT_ARGB8888;
 		break;
-
-	default:
-		DRM_DEBUG_KMS("pixelformat unsupported by mixer\n");
-		return;
 	}
 
 	/* ratio is already checked by common plane code */
@@ -631,12 +618,10 @@ static void mixer_graph_buffer(struct mixer_context *ctx,
 	dst_x_offset = state->crtc.x;
 	dst_y_offset = state->crtc.y;
 
-	/* converting dma address base and source offset */
+	/* translate dma address base s.t. the source image offset is zero */
 	dma_addr = exynos_drm_fb_dma_addr(fb, 0)
 		+ (state->src.x * fb->format->cpp[0])
 		+ (state->src.y * fb->pitches[0]);
-	src_x_offset = 0;
-	src_y_offset = 0;
 
 	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
 		__set_bit(MXR_BIT_INTERLACE, &ctx->flags);
@@ -667,11 +652,6 @@ static void mixer_graph_buffer(struct mixer_context *ctx,
 	val |= MXR_GRP_WH_V_SCALE(y_ratio);
 	mixer_reg_write(res, MXR_GRAPHIC_WH(win), val);
 
-	/* setup offsets in source image */
-	val  = MXR_GRP_SXY_SX(src_x_offset);
-	val |= MXR_GRP_SXY_SY(src_y_offset);
-	mixer_reg_write(res, MXR_GRAPHIC_SXY(win), val);
-
 	/* setup offsets in display image */
 	val  = MXR_GRP_DXY_DX(dst_x_offset);
 	val |= MXR_GRP_DXY_DY(dst_y_offset);
@@ -748,6 +728,10 @@ static void mixer_win_reset(struct mixer_context *ctx)
 	if (test_bit(MXR_BIT_VP_ENABLED, &ctx->flags))
 		mixer_reg_writemask(res, MXR_CFG, 0, MXR_CFG_VP_ENABLE);
 
+	/* set all source image offsets to zero */
+	mixer_reg_write(res, MXR_GRAPHIC_SXY(0), 0);
+	mixer_reg_write(res, MXR_GRAPHIC_SXY(1), 0);
+
 	spin_unlock_irqrestore(&res->reg_slock, flags);
 }
 
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c
index cc4e944a1d3c..0e3752437e44 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c
@@ -63,7 +63,8 @@ static void fsl_dcu_drm_crtc_atomic_disable(struct drm_crtc *crtc,
 	clk_disable_unprepare(fsl_dev->pix_clk);
 }
 
-static void fsl_dcu_drm_crtc_enable(struct drm_crtc *crtc)
+static void fsl_dcu_drm_crtc_atomic_enable(struct drm_crtc *crtc,
+					   struct drm_crtc_state *old_state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct fsl_dcu_drm_device *fsl_dev = dev->dev_private;
@@ -133,7 +134,7 @@ static void fsl_dcu_drm_crtc_mode_set_nofb(struct drm_crtc *crtc)
 static const struct drm_crtc_helper_funcs fsl_dcu_drm_crtc_helper_funcs = {
 	.atomic_disable = fsl_dcu_drm_crtc_atomic_disable,
 	.atomic_flush = fsl_dcu_drm_crtc_atomic_flush,
-	.enable = fsl_dcu_drm_crtc_enable,
+	.atomic_enable = fsl_dcu_drm_crtc_atomic_enable,
 	.mode_set_nofb = fsl_dcu_drm_crtc_mode_set_nofb,
 };
 
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
index 5cbde196895a..58e9e0601a61 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
@@ -176,8 +176,6 @@ static struct drm_driver fsl_dcu_drm_driver = {
 	.gem_prime_vunmap	= drm_gem_cma_prime_vunmap,
 	.gem_prime_mmap		= drm_gem_cma_prime_mmap,
 	.dumb_create		= drm_gem_cma_dumb_create,
-	.dumb_map_offset	= drm_gem_cma_dumb_map_offset,
-	.dumb_destroy		= drm_gem_dumb_destroy,
 	.fops			= &fsl_dcu_drm_fops,
 	.name			= "fsl-dcu-drm",
 	.desc			= "Freescale DCU DRM",
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c
index 0a20723aa6e1..9554b245746e 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c
@@ -224,7 +224,7 @@ struct drm_plane *fsl_dcu_drm_primary_create_plane(struct drm_device *dev)
 				       &fsl_dcu_drm_plane_funcs,
 				       fsl_dcu_drm_plane_formats,
 				       ARRAY_SIZE(fsl_dcu_drm_plane_formats),
-				       DRM_PLANE_TYPE_PRIMARY, NULL);
+				       NULL, DRM_PLANE_TYPE_PRIMARY, NULL);
 	if (ret) {
 		kfree(primary);
 		primary = NULL;
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c
index dcbf3c06e1d8..edd7d8127d19 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c
@@ -63,7 +63,6 @@ static const struct drm_connector_funcs fsl_dcu_drm_connector_funcs = {
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 	.destroy = fsl_dcu_drm_connector_destroy,
-	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.reset = drm_atomic_helper_connector_reset,
 };
diff --git a/drivers/gpu/drm/gma500/framebuffer.c b/drivers/gpu/drm/gma500/framebuffer.c
index 7da70b6c83f0..2570c7f647a6 100644
--- a/drivers/gpu/drm/gma500/framebuffer.c
+++ b/drivers/gpu/drm/gma500/framebuffer.c
@@ -479,26 +479,6 @@ static struct drm_framebuffer *psb_user_framebuffer_create
 	return psb_framebuffer_create(dev, cmd, r);
 }
 
-static void psbfb_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
-							u16 blue, int regno)
-{
-	struct gma_crtc *gma_crtc = to_gma_crtc(crtc);
-
-	gma_crtc->lut_r[regno] = red >> 8;
-	gma_crtc->lut_g[regno] = green >> 8;
-	gma_crtc->lut_b[regno] = blue >> 8;
-}
-
-static void psbfb_gamma_get(struct drm_crtc *crtc, u16 *red,
-					u16 *green, u16 *blue, int regno)
-{
-	struct gma_crtc *gma_crtc = to_gma_crtc(crtc);
-
-	*red = gma_crtc->lut_r[regno] << 8;
-	*green = gma_crtc->lut_g[regno] << 8;
-	*blue = gma_crtc->lut_b[regno] << 8;
-}
-
 static int psbfb_probe(struct drm_fb_helper *helper,
 				struct drm_fb_helper_surface_size *sizes)
 {
@@ -525,8 +505,6 @@ static int psbfb_probe(struct drm_fb_helper *helper,
 }
 
 static const struct drm_fb_helper_funcs psb_fb_helper_funcs = {
-	.gamma_set = psbfb_gamma_set,
-	.gamma_get = psbfb_gamma_get,
 	.fb_probe = psbfb_probe,
 };
 
diff --git a/drivers/gpu/drm/gma500/gem.c b/drivers/gpu/drm/gma500/gem.c
index 7da061aab729..131239759a75 100644
--- a/drivers/gpu/drm/gma500/gem.c
+++ b/drivers/gpu/drm/gma500/gem.c
@@ -48,36 +48,6 @@ int psb_gem_get_aperture(struct drm_device *dev, void *data,
 }
 
 /**
- *	psb_gem_dumb_map_gtt	-	buffer mapping for dumb interface
- *	@file: our drm client file
- *	@dev: drm device
- *	@handle: GEM handle to the object (from dumb_create)
- *
- *	Do the necessary setup to allow the mapping of the frame buffer
- *	into user memory. We don't have to do much here at the moment.
- */
-int psb_gem_dumb_map_gtt(struct drm_file *file, struct drm_device *dev,
-			 uint32_t handle, uint64_t *offset)
-{
-	int ret = 0;
-	struct drm_gem_object *obj;
-
-	/* GEM does all our handle to object mapping */
-	obj = drm_gem_object_lookup(file, handle);
-	if (obj == NULL)
-		return -ENOENT;
-
-	/* Make it mmapable */
-	ret = drm_gem_create_mmap_offset(obj);
-	if (ret)
-		goto out;
-	*offset = drm_vma_node_offset_addr(&obj->vma_node);
-out:
-	drm_gem_object_unreference_unlocked(obj);
-	return ret;
-}
-
-/**
  *	psb_gem_create		-	create a mappable object
  *	@file: the DRM file of the client
  *	@dev: our device
diff --git a/drivers/gpu/drm/gma500/gma_display.c b/drivers/gpu/drm/gma500/gma_display.c
index e7fd356acf2e..f3c48a2be71b 100644
--- a/drivers/gpu/drm/gma500/gma_display.c
+++ b/drivers/gpu/drm/gma500/gma_display.c
@@ -144,33 +144,32 @@ void gma_crtc_load_lut(struct drm_crtc *crtc)
 	struct gma_crtc *gma_crtc = to_gma_crtc(crtc);
 	const struct psb_offset *map = &dev_priv->regmap[gma_crtc->pipe];
 	int palreg = map->palette;
+	u16 *r, *g, *b;
 	int i;
 
 	/* The clocks have to be on to load the palette. */
 	if (!crtc->enabled)
 		return;
 
+	r = crtc->gamma_store;
+	g = r + crtc->gamma_size;
+	b = g + crtc->gamma_size;
+
 	if (gma_power_begin(dev, false)) {
 		for (i = 0; i < 256; i++) {
 			REG_WRITE(palreg + 4 * i,
-				  ((gma_crtc->lut_r[i] +
-				  gma_crtc->lut_adj[i]) << 16) |
-				  ((gma_crtc->lut_g[i] +
-				  gma_crtc->lut_adj[i]) << 8) |
-				  (gma_crtc->lut_b[i] +
-				  gma_crtc->lut_adj[i]));
+				  (((*r++ >> 8) + gma_crtc->lut_adj[i]) << 16) |
+				  (((*g++ >> 8) + gma_crtc->lut_adj[i]) << 8) |
+				  ((*b++ >> 8) + gma_crtc->lut_adj[i]));
 		}
 		gma_power_end(dev);
 	} else {
 		for (i = 0; i < 256; i++) {
 			/* FIXME: Why pipe[0] and not pipe[..._crtc->pipe]? */
 			dev_priv->regs.pipe[0].palette[i] =
-				  ((gma_crtc->lut_r[i] +
-				  gma_crtc->lut_adj[i]) << 16) |
-				  ((gma_crtc->lut_g[i] +
-				  gma_crtc->lut_adj[i]) << 8) |
-				  (gma_crtc->lut_b[i] +
-				  gma_crtc->lut_adj[i]);
+				(((*r++ >> 8) + gma_crtc->lut_adj[i]) << 16) |
+				(((*g++ >> 8) + gma_crtc->lut_adj[i]) << 8) |
+				((*b++ >> 8) + gma_crtc->lut_adj[i]);
 		}
 
 	}
@@ -180,15 +179,6 @@ int gma_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green, u16 *blue,
 		       u32 size,
 		       struct drm_modeset_acquire_ctx *ctx)
 {
-	struct gma_crtc *gma_crtc = to_gma_crtc(crtc);
-	int i;
-
-	for (i = 0; i < size; i++) {
-		gma_crtc->lut_r[i] = red[i] >> 8;
-		gma_crtc->lut_g[i] = green[i] >> 8;
-		gma_crtc->lut_b[i] = blue[i] >> 8;
-	}
-
 	gma_crtc_load_lut(crtc);
 
 	return 0;
diff --git a/drivers/gpu/drm/gma500/mdfld_dsi_pkg_sender.c b/drivers/gpu/drm/gma500/mdfld_dsi_pkg_sender.c
index 1616af209bfc..c50534c923df 100644
--- a/drivers/gpu/drm/gma500/mdfld_dsi_pkg_sender.c
+++ b/drivers/gpu/drm/gma500/mdfld_dsi_pkg_sender.c
@@ -520,7 +520,7 @@ static int __read_panel_data(struct mdfld_dsi_pkg_sender *sender, u8 data_type,
 			u8 *data, u16 len, u32 *data_out, u16 len_out, bool hs)
 {
 	unsigned long flags;
-	struct drm_device *dev = sender->dev;
+	struct drm_device *dev;
 	int i;
 	u32 gen_data_reg;
 	int retry = MDFLD_DSI_READ_MAX_COUNT;
@@ -530,6 +530,8 @@ static int __read_panel_data(struct mdfld_dsi_pkg_sender *sender, u8 data_type,
 		return -EINVAL;
 	}
 
+	dev = sender->dev;
+
 	/**
 	 * do reading.
 	 * 0) send out generic read request
diff --git a/drivers/gpu/drm/gma500/mdfld_intel_display.c b/drivers/gpu/drm/gma500/mdfld_intel_display.c
index 63c6e08600ae..531e4450c000 100644
--- a/drivers/gpu/drm/gma500/mdfld_intel_display.c
+++ b/drivers/gpu/drm/gma500/mdfld_intel_display.c
@@ -737,11 +737,7 @@ static int mdfld_crtc_mode_set(struct drm_crtc *crtc,
 					sizeof(struct drm_display_mode));
 
 	list_for_each_entry(connector, &mode_config->connector_list, head) {
-		if (!connector)
-			continue;
-
 		encoder = connector->encoder;
-
 		if (!encoder)
 			continue;
 
diff --git a/drivers/gpu/drm/gma500/psb_drv.c b/drivers/gpu/drm/gma500/psb_drv.c
index 1f9b35afefee..37a3be71acd9 100644
--- a/drivers/gpu/drm/gma500/psb_drv.c
+++ b/drivers/gpu/drm/gma500/psb_drv.c
@@ -480,7 +480,6 @@ static struct drm_driver driver = {
 	.load = psb_driver_load,
 	.unload = psb_driver_unload,
 	.lastclose = psb_driver_lastclose,
-	.set_busid = drm_pci_set_busid,
 
 	.num_ioctls = ARRAY_SIZE(psb_ioctls),
 	.irq_preinstall = psb_irq_preinstall,
@@ -495,8 +494,6 @@ static struct drm_driver driver = {
 	.gem_vm_ops = &psb_gem_vm_ops,
 
 	.dumb_create = psb_gem_dumb_create,
-	.dumb_map_offset = psb_gem_dumb_map_gtt,
-	.dumb_destroy = drm_gem_dumb_destroy,
 	.ioctls = psb_ioctls,
 	.fops = &psb_gem_fops,
 	.name = DRIVER_NAME,
@@ -517,12 +514,12 @@ static struct pci_driver psb_pci_driver = {
 
 static int __init psb_init(void)
 {
-	return drm_pci_init(&driver, &psb_pci_driver);
+	return pci_register_driver(&psb_pci_driver);
 }
 
 static void __exit psb_exit(void)
 {
-	drm_pci_exit(&driver, &psb_pci_driver);
+	pci_unregister_driver(&psb_pci_driver);
 }
 
 late_initcall(psb_init);
diff --git a/drivers/gpu/drm/gma500/psb_drv.h b/drivers/gpu/drm/gma500/psb_drv.h
index 83667087d6e5..821497dbd3fc 100644
--- a/drivers/gpu/drm/gma500/psb_drv.h
+++ b/drivers/gpu/drm/gma500/psb_drv.h
@@ -750,8 +750,6 @@ extern int psb_gem_get_aperture(struct drm_device *dev, void *data,
 			struct drm_file *file);
 extern int psb_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
 			struct drm_mode_create_dumb *args);
-extern int psb_gem_dumb_map_gtt(struct drm_file *file, struct drm_device *dev,
-			uint32_t handle, uint64_t *offset);
 extern int psb_gem_fault(struct vm_fault *vmf);
 
 /* psb_device.c */
diff --git a/drivers/gpu/drm/gma500/psb_intel_display.c b/drivers/gpu/drm/gma500/psb_intel_display.c
index 7b6c84925098..8762efaef283 100644
--- a/drivers/gpu/drm/gma500/psb_intel_display.c
+++ b/drivers/gpu/drm/gma500/psb_intel_display.c
@@ -518,13 +518,8 @@ void psb_intel_crtc_init(struct drm_device *dev, int pipe,
 	gma_crtc->pipe = pipe;
 	gma_crtc->plane = pipe;
 
-	for (i = 0; i < 256; i++) {
-		gma_crtc->lut_r[i] = i;
-		gma_crtc->lut_g[i] = i;
-		gma_crtc->lut_b[i] = i;
-
+	for (i = 0; i < 256; i++)
 		gma_crtc->lut_adj[i] = 0;
-	}
 
 	gma_crtc->mode_dev = mode_dev;
 	gma_crtc->cursor_addr = 0;
diff --git a/drivers/gpu/drm/gma500/psb_intel_drv.h b/drivers/gpu/drm/gma500/psb_intel_drv.h
index 6a10215fc42d..e8e4ea14b12b 100644
--- a/drivers/gpu/drm/gma500/psb_intel_drv.h
+++ b/drivers/gpu/drm/gma500/psb_intel_drv.h
@@ -172,7 +172,6 @@ struct gma_crtc {
 	int plane;
 	uint32_t cursor_addr;
 	struct gtt_range *cursor_gt;
-	u8 lut_r[256], lut_g[256], lut_b[256];
 	u8 lut_adj[256];
 	struct psb_intel_framebuffer *fbdev_fb;
 	/* a mode_set for fbdev users on this crtc */
diff --git a/drivers/gpu/drm/gma500/tc35876x-dsi-lvds.c b/drivers/gpu/drm/gma500/tc35876x-dsi-lvds.c
index 771ff66711af..37c997e24b9e 100644
--- a/drivers/gpu/drm/gma500/tc35876x-dsi-lvds.c
+++ b/drivers/gpu/drm/gma500/tc35876x-dsi-lvds.c
@@ -26,7 +26,7 @@
 #include "mdfld_output.h"
 #include "mdfld_dsi_pkg_sender.h"
 #include "tc35876x-dsi-lvds.h"
-#include <linux/i2c/tc35876x.h>
+#include <linux/platform_data/tc35876x.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <asm/intel_scu_ipc.h>
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c
index 59542bddc980..a956545774a3 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c
@@ -150,7 +150,6 @@ static const u32 channel_formats1[] = {
 static struct drm_plane_funcs hibmc_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
-	.set_property = drm_atomic_helper_plane_set_property,
 	.destroy = drm_plane_cleanup,
 	.reset = drm_atomic_helper_plane_reset,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
@@ -181,6 +180,7 @@ static struct drm_plane *hibmc_plane_init(struct hibmc_drm_private *priv)
 	ret = drm_universal_plane_init(dev, plane, 1, &hibmc_plane_funcs,
 				       channel_formats1,
 				       ARRAY_SIZE(channel_formats1),
+				       NULL,
 				       DRM_PLANE_TYPE_PRIMARY,
 				       NULL);
 	if (ret) {
@@ -192,7 +192,8 @@ static struct drm_plane *hibmc_plane_init(struct hibmc_drm_private *priv)
 	return plane;
 }
 
-static void hibmc_crtc_enable(struct drm_crtc *crtc)
+static void hibmc_crtc_atomic_enable(struct drm_crtc *crtc,
+				     struct drm_crtc_state *old_state)
 {
 	unsigned int reg;
 	struct hibmc_drm_private *priv = crtc->dev->dev_private;
@@ -209,7 +210,8 @@ static void hibmc_crtc_enable(struct drm_crtc *crtc)
 	drm_crtc_vblank_on(crtc);
 }
 
-static void hibmc_crtc_disable(struct drm_crtc *crtc)
+static void hibmc_crtc_atomic_disable(struct drm_crtc *crtc,
+				      struct drm_crtc_state *old_state)
 {
 	unsigned int reg;
 	struct hibmc_drm_private *priv = crtc->dev->dev_private;
@@ -453,11 +455,11 @@ static const struct drm_crtc_funcs hibmc_crtc_funcs = {
 };
 
 static const struct drm_crtc_helper_funcs hibmc_crtc_helper_funcs = {
-	.enable		= hibmc_crtc_enable,
-	.disable	= hibmc_crtc_disable,
 	.mode_set_nofb	= hibmc_crtc_mode_set_nofb,
 	.atomic_begin	= hibmc_crtc_atomic_begin,
 	.atomic_flush	= hibmc_crtc_atomic_flush,
+	.atomic_enable	= hibmc_crtc_atomic_enable,
+	.atomic_disable	= hibmc_crtc_atomic_disable,
 };
 
 int hibmc_de_init(struct hibmc_drm_private *priv)
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c
index 2ffdbf9801bd..d4f6f1f9df5b 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c
@@ -67,7 +67,6 @@ static struct drm_driver hibmc_driver = {
 	.gem_free_object_unlocked = hibmc_gem_free_object,
 	.dumb_create            = hibmc_dumb_create,
 	.dumb_map_offset        = hibmc_dumb_mmap_offset,
-	.dumb_destroy           = drm_gem_dumb_destroy,
 	.irq_handler		= hibmc_drm_interrupt,
 };
 
@@ -276,11 +275,12 @@ static int hibmc_unload(struct drm_device *dev)
 
 	hibmc_fbdev_fini(priv);
 
+	drm_atomic_helper_shutdown(dev);
+
 	if (dev->irq_enabled)
 		drm_irq_uninstall(dev);
 	if (priv->msi_enabled)
 		pci_disable_msi(dev->pdev);
-	drm_vblank_cleanup(dev);
 
 	hibmc_kms_fini(priv);
 	hibmc_mm_fini(priv);
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_fbdev.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_fbdev.c
index f5ac80daeef2..b92595c477ef 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_fbdev.c
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_fbdev.c
@@ -131,7 +131,6 @@ static int hibmc_drm_fb_create(struct drm_fb_helper *helper,
 
 	strcpy(info->fix.id, "hibmcdrmfb");
 
-	info->flags = FBINFO_DEFAULT;
 	info->fbops = &hibmc_drm_fb_ops;
 
 	drm_fb_helper_fill_fix(info, hi_fbdev->fb->fb.pitches[0],
@@ -158,7 +157,7 @@ out_unpin_bo:
 out_unreserve_ttm_bo:
 	ttm_bo_unreserve(&bo->bo);
 out_unref_gem:
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 
 	return ret;
 }
@@ -173,7 +172,7 @@ static void hibmc_fbdev_destroy(struct hibmc_fbdev *fbdev)
 	drm_fb_helper_fini(fbh);
 
 	if (gfb)
-		drm_framebuffer_unreference(&gfb->fb);
+		drm_framebuffer_put(&gfb->fb);
 }
 
 static const struct drm_fb_helper_funcs hibmc_fbdev_helper_funcs = {
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c
index 12a18557c5fd..ec4dd9df9150 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c
@@ -47,7 +47,6 @@ static const struct drm_connector_helper_funcs
 };
 
 static const struct drm_connector_funcs hibmc_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = drm_connector_cleanup,
 	.reset = drm_atomic_helper_connector_reset,
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_ttm.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_ttm.c
index ac457c779caa..3518167a7dc4 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_ttm.c
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_ttm.c
@@ -444,7 +444,7 @@ int hibmc_dumb_create(struct drm_file *file, struct drm_device *dev,
 	}
 
 	ret = drm_gem_handle_create(file, gobj, &handle);
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 	if (ret) {
 		DRM_ERROR("failed to unreference GEM object: %d\n", ret);
 		return ret;
@@ -479,7 +479,7 @@ int hibmc_dumb_mmap_offset(struct drm_file *file, struct drm_device *dev,
 	bo = gem_to_hibmc_bo(obj);
 	*offset = hibmc_bo_mmap_offset(bo);
 
-	drm_gem_object_unreference_unlocked(obj);
+	drm_gem_object_put_unlocked(obj);
 	return 0;
 }
 
@@ -487,7 +487,7 @@ static void hibmc_user_framebuffer_destroy(struct drm_framebuffer *fb)
 {
 	struct hibmc_framebuffer *hibmc_fb = to_hibmc_framebuffer(fb);
 
-	drm_gem_object_unreference_unlocked(hibmc_fb->obj);
+	drm_gem_object_put_unlocked(hibmc_fb->obj);
 	drm_framebuffer_cleanup(fb);
 	kfree(hibmc_fb);
 }
@@ -543,7 +543,7 @@ hibmc_user_framebuffer_create(struct drm_device *dev,
 
 	hibmc_fb = hibmc_framebuffer_init(dev, mode_cmd, obj);
 	if (IS_ERR(hibmc_fb)) {
-		drm_gem_object_unreference_unlocked(obj);
+		drm_gem_object_put_unlocked(obj);
 		return ERR_PTR((long)hibmc_fb);
 	}
 	return &hibmc_fb->fb;
diff --git a/drivers/gpu/drm/hisilicon/kirin/dw_drm_dsi.c b/drivers/gpu/drm/hisilicon/kirin/dw_drm_dsi.c
index f77dcfaade6c..b4c7af3ab6ae 100644
--- a/drivers/gpu/drm/hisilicon/kirin/dw_drm_dsi.c
+++ b/drivers/gpu/drm/hisilicon/kirin/dw_drm_dsi.c
@@ -603,6 +603,72 @@ static void dsi_encoder_enable(struct drm_encoder *encoder)
 	dsi->enable = true;
 }
 
+static enum drm_mode_status dsi_encoder_phy_mode_valid(
+					struct drm_encoder *encoder,
+					const struct drm_display_mode *mode)
+{
+	struct dw_dsi *dsi = encoder_to_dsi(encoder);
+	struct mipi_phy_params phy;
+	u32 bpp = mipi_dsi_pixel_format_to_bpp(dsi->format);
+	u32 req_kHz, act_kHz, lane_byte_clk_kHz;
+
+	/* Calculate the lane byte clk using the adjusted mode clk */
+	memset(&phy, 0, sizeof(phy));
+	req_kHz = mode->clock * bpp / dsi->lanes;
+	act_kHz = dsi_calc_phy_rate(req_kHz, &phy);
+	lane_byte_clk_kHz = act_kHz / 8;
+
+	DRM_DEBUG_DRIVER("Checking mode %ix%i-%i@%i clock: %i...",
+			mode->hdisplay, mode->vdisplay, bpp,
+			drm_mode_vrefresh(mode), mode->clock);
+
+	/*
+	 * Make sure the adjusted mode clock and the lane byte clk
+	 * have a common denominator base frequency
+	 */
+	if (mode->clock/dsi->lanes == lane_byte_clk_kHz/3) {
+		DRM_DEBUG_DRIVER("OK!\n");
+		return MODE_OK;
+	}
+
+	DRM_DEBUG_DRIVER("BAD!\n");
+	return MODE_BAD;
+}
+
+static enum drm_mode_status dsi_encoder_mode_valid(struct drm_encoder *encoder,
+					const struct drm_display_mode *mode)
+
+{
+	const struct drm_crtc_helper_funcs *crtc_funcs = NULL;
+	struct drm_crtc *crtc = NULL;
+	struct drm_display_mode adj_mode;
+	enum drm_mode_status ret;
+
+	/*
+	 * The crtc might adjust the mode, so go through the
+	 * possible crtcs (technically just one) and call
+	 * mode_fixup to figure out the adjusted mode before we
+	 * validate it.
+	 */
+	drm_for_each_crtc(crtc, encoder->dev) {
+		/*
+		 * reset adj_mode to the mode value each time,
+		 * so we don't adjust the mode twice
+		 */
+		drm_mode_copy(&adj_mode, mode);
+
+		crtc_funcs = crtc->helper_private;
+		if (crtc_funcs && crtc_funcs->mode_fixup)
+			if (!crtc_funcs->mode_fixup(crtc, mode, &adj_mode))
+				return MODE_BAD;
+
+		ret = dsi_encoder_phy_mode_valid(encoder, &adj_mode);
+		if (ret != MODE_OK)
+			return ret;
+	}
+	return MODE_OK;
+}
+
 static void dsi_encoder_mode_set(struct drm_encoder *encoder,
 				 struct drm_display_mode *mode,
 				 struct drm_display_mode *adj_mode)
@@ -622,6 +688,7 @@ static int dsi_encoder_atomic_check(struct drm_encoder *encoder,
 
 static const struct drm_encoder_helper_funcs dw_encoder_helper_funcs = {
 	.atomic_check	= dsi_encoder_atomic_check,
+	.mode_valid	= dsi_encoder_mode_valid,
 	.mode_set	= dsi_encoder_mode_set,
 	.enable		= dsi_encoder_enable,
 	.disable	= dsi_encoder_disable
diff --git a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c
index c96c228a9898..9823477b1855 100644
--- a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c
+++ b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c
@@ -178,6 +178,19 @@ static void ade_init(struct ade_hw_ctx *ctx)
 			FRM_END_START_MASK, REG_EFFECTIVE_IN_ADEEN_FRMEND);
 }
 
+static bool ade_crtc_mode_fixup(struct drm_crtc *crtc,
+				const struct drm_display_mode *mode,
+				struct drm_display_mode *adjusted_mode)
+{
+	struct ade_crtc *acrtc = to_ade_crtc(crtc);
+	struct ade_hw_ctx *ctx = acrtc->ctx;
+
+	adjusted_mode->clock =
+		clk_round_rate(ctx->ade_pix_clk, mode->clock * 1000) / 1000;
+	return true;
+}
+
+
 static void ade_set_pix_clk(struct ade_hw_ctx *ctx,
 			    struct drm_display_mode *mode,
 			    struct drm_display_mode *adj_mode)
@@ -467,7 +480,8 @@ static void ade_dump_regs(void __iomem *base)
 static void ade_dump_regs(void __iomem *base) { }
 #endif
 
-static void ade_crtc_enable(struct drm_crtc *crtc)
+static void ade_crtc_atomic_enable(struct drm_crtc *crtc,
+				   struct drm_crtc_state *old_state)
 {
 	struct ade_crtc *acrtc = to_ade_crtc(crtc);
 	struct ade_hw_ctx *ctx = acrtc->ctx;
@@ -489,7 +503,8 @@ static void ade_crtc_enable(struct drm_crtc *crtc)
 	acrtc->enable = true;
 }
 
-static void ade_crtc_disable(struct drm_crtc *crtc)
+static void ade_crtc_atomic_disable(struct drm_crtc *crtc,
+				    struct drm_crtc_state *old_state)
 {
 	struct ade_crtc *acrtc = to_ade_crtc(crtc);
 	struct ade_hw_ctx *ctx = acrtc->ctx;
@@ -553,11 +568,12 @@ static void ade_crtc_atomic_flush(struct drm_crtc *crtc,
 }
 
 static const struct drm_crtc_helper_funcs ade_crtc_helper_funcs = {
-	.enable		= ade_crtc_enable,
-	.disable	= ade_crtc_disable,
+	.mode_fixup	= ade_crtc_mode_fixup,
 	.mode_set_nofb	= ade_crtc_mode_set_nofb,
 	.atomic_begin	= ade_crtc_atomic_begin,
 	.atomic_flush	= ade_crtc_atomic_flush,
+	.atomic_enable	= ade_crtc_atomic_enable,
+	.atomic_disable	= ade_crtc_atomic_disable,
 };
 
 static const struct drm_crtc_funcs ade_crtc_funcs = {
@@ -565,7 +581,6 @@ static const struct drm_crtc_funcs ade_crtc_funcs = {
 	.set_config	= drm_atomic_helper_set_config,
 	.page_flip	= drm_atomic_helper_page_flip,
 	.reset		= drm_atomic_helper_crtc_reset,
-	.set_property = drm_atomic_helper_crtc_set_property,
 	.atomic_duplicate_state	= drm_atomic_helper_crtc_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_crtc_destroy_state,
 	.enable_vblank	= ade_crtc_enable_vblank,
@@ -583,8 +598,7 @@ static int ade_crtc_init(struct drm_device *dev, struct drm_crtc *crtc,
 	 */
 	port = of_get_child_by_name(dev->dev->of_node, "port");
 	if (!port) {
-		DRM_ERROR("no port node found in %s\n",
-			  dev->dev->of_node->full_name);
+		DRM_ERROR("no port node found in %pOF\n", dev->dev->of_node);
 		return -EINVAL;
 	}
 	of_node_put(port);
@@ -889,7 +903,6 @@ static const struct drm_plane_helper_funcs ade_plane_helper_funcs = {
 static struct drm_plane_funcs ade_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
-	.set_property = drm_atomic_helper_plane_set_property,
 	.destroy = drm_plane_cleanup,
 	.reset = drm_atomic_helper_plane_reset,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
@@ -909,7 +922,7 @@ static int ade_plane_init(struct drm_device *dev, struct ade_plane *aplane,
 		return ret;
 
 	ret = drm_universal_plane_init(dev, &aplane->base, 1, &ade_plane_funcs,
-				       fmts, fmts_cnt, type, NULL);
+				       fmts, fmts_cnt, NULL, type, NULL);
 	if (ret) {
 		DRM_ERROR("fail to init plane, ch=%d\n", aplane->ch);
 		return ret;
diff --git a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.c b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.c
index 9c903672f582..e27352ca26c4 100644
--- a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.c
+++ b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.c
@@ -34,14 +34,12 @@ static int kirin_drm_kms_cleanup(struct drm_device *dev)
 {
 	struct kirin_drm_private *priv = dev->dev_private;
 
-#ifdef CONFIG_DRM_FBDEV_EMULATION
 	if (priv->fbdev) {
 		drm_fbdev_cma_fini(priv->fbdev);
 		priv->fbdev = NULL;
 	}
-#endif
+
 	drm_kms_helper_poll_fini(dev);
-	drm_vblank_cleanup(dev);
 	dc_ops->cleanup(to_platform_device(dev->dev));
 	drm_mode_config_cleanup(dev);
 	devm_kfree(dev->dev, priv);
@@ -50,27 +48,16 @@ static int kirin_drm_kms_cleanup(struct drm_device *dev)
 	return 0;
 }
 
-#ifdef CONFIG_DRM_FBDEV_EMULATION
 static void kirin_fbdev_output_poll_changed(struct drm_device *dev)
 {
 	struct kirin_drm_private *priv = dev->dev_private;
 
-	if (priv->fbdev) {
-		drm_fbdev_cma_hotplug_event(priv->fbdev);
-	} else {
-		priv->fbdev = drm_fbdev_cma_init(dev, 32,
-						 dev->mode_config.num_connector);
-		if (IS_ERR(priv->fbdev))
-			priv->fbdev = NULL;
-	}
+	drm_fbdev_cma_hotplug_event(priv->fbdev);
 }
-#endif
 
 static const struct drm_mode_config_funcs kirin_drm_mode_config_funcs = {
 	.fb_create = drm_fb_cma_create,
-#ifdef CONFIG_DRM_FBDEV_EMULATION
 	.output_poll_changed = kirin_fbdev_output_poll_changed,
-#endif
 	.atomic_check = drm_atomic_helper_check,
 	.atomic_commit = drm_atomic_helper_commit,
 };
@@ -129,11 +116,18 @@ static int kirin_drm_kms_init(struct drm_device *dev)
 	/* init kms poll for handling hpd */
 	drm_kms_helper_poll_init(dev);
 
-	/* force detection after connectors init */
-	(void)drm_helper_hpd_irq_event(dev);
+	priv->fbdev = drm_fbdev_cma_init(dev, 32,
+					 dev->mode_config.num_connector);
 
+	if (IS_ERR(priv->fbdev)) {
+		DRM_ERROR("failed to initialize fbdev.\n");
+		ret = PTR_ERR(priv->fbdev);
+		goto err_cleanup_poll;
+	}
 	return 0;
 
+err_cleanup_poll:
+	drm_kms_helper_poll_fini(dev);
 err_unbind_all:
 	component_unbind_all(dev->dev, dev);
 err_dc_cleanup:
@@ -163,8 +157,6 @@ static struct drm_driver kirin_drm_driver = {
 	.gem_free_object_unlocked = drm_gem_cma_free_object,
 	.gem_vm_ops		= &drm_gem_cma_vm_ops,
 	.dumb_create		= kirin_gem_cma_dumb_create,
-	.dumb_map_offset	= drm_gem_cma_dumb_map_offset,
-	.dumb_destroy		= drm_gem_dumb_destroy,
 
 	.prime_handle_to_fd	= drm_gem_prime_handle_to_fd,
 	.prime_fd_to_handle	= drm_gem_prime_fd_to_handle,
diff --git a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.h b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.h
index 7f60c64915d9..56cb62df065c 100644
--- a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.h
+++ b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.h
@@ -20,9 +20,7 @@ struct kirin_dc_ops {
 };
 
 struct kirin_drm_private {
-#ifdef CONFIG_DRM_FBDEV_EMULATION
 	struct drm_fbdev_cma *fbdev;
-#endif
 };
 
 extern const struct kirin_dc_ops ade_dc_ops;
diff --git a/drivers/gpu/drm/i2c/tda998x_drv.c b/drivers/gpu/drm/i2c/tda998x_drv.c
index 86f47e190309..54e3255dde13 100644
--- a/drivers/gpu/drm/i2c/tda998x_drv.c
+++ b/drivers/gpu/drm/i2c/tda998x_drv.c
@@ -712,7 +712,7 @@ tda998x_write_avi(struct tda998x_priv *priv, struct drm_display_mode *mode)
 {
 	union hdmi_infoframe frame;
 
-	drm_hdmi_avi_infoframe_from_display_mode(&frame.avi, mode);
+	drm_hdmi_avi_infoframe_from_display_mode(&frame.avi, mode, false);
 	frame.avi.quantization_range = HDMI_QUANTIZATION_RANGE_FULL;
 
 	tda998x_write_if(priv, DIP_IF_FLAGS_IF2, REG_IF2_HB0, &frame);
@@ -969,14 +969,6 @@ static int tda998x_audio_codec_init(struct tda998x_priv *priv,
 
 /* DRM connector functions */
 
-static int tda998x_connector_dpms(struct drm_connector *connector, int mode)
-{
-	if (drm_core_check_feature(connector->dev, DRIVER_ATOMIC))
-		return drm_atomic_helper_connector_dpms(connector, mode);
-	else
-		return drm_helper_connector_dpms(connector, mode);
-}
-
 static int tda998x_connector_fill_modes(struct drm_connector *connector,
 					uint32_t maxX, uint32_t maxY)
 {
@@ -1014,7 +1006,7 @@ static void tda998x_connector_destroy(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs tda998x_connector_funcs = {
-	.dpms = tda998x_connector_dpms,
+	.dpms = drm_helper_connector_dpms,
 	.reset = drm_atomic_helper_connector_reset,
 	.fill_modes = tda998x_connector_fill_modes,
 	.detect = tda998x_connector_detect,
diff --git a/drivers/gpu/drm/i810/i810_drv.c b/drivers/gpu/drm/i810/i810_drv.c
index 37fd0906f807..c69d5c487f51 100644
--- a/drivers/gpu/drm/i810/i810_drv.c
+++ b/drivers/gpu/drm/i810/i810_drv.c
@@ -59,7 +59,6 @@ static struct drm_driver driver = {
 	.load = i810_driver_load,
 	.lastclose = i810_driver_lastclose,
 	.preclose = i810_driver_preclose,
-	.set_busid = drm_pci_set_busid,
 	.dma_quiescent = i810_driver_dma_quiescent,
 	.ioctls = i810_ioctls,
 	.fops = &i810_driver_fops,
@@ -83,12 +82,12 @@ static int __init i810_init(void)
 		return -EINVAL;
 	}
 	driver.num_ioctls = i810_max_ioctl;
-	return drm_pci_init(&driver, &i810_pci_driver);
+	return drm_legacy_pci_init(&driver, &i810_pci_driver);
 }
 
 static void __exit i810_exit(void)
 {
-	drm_pci_exit(&driver, &i810_pci_driver);
+	drm_legacy_pci_exit(&driver, &i810_pci_driver);
 }
 
 module_init(i810_init);
diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig
index a5cd5dacf055..e9e64e8e9765 100644
--- a/drivers/gpu/drm/i915/Kconfig
+++ b/drivers/gpu/drm/i915/Kconfig
@@ -21,6 +21,7 @@ config DRM_I915
 	select ACPI_BUTTON if ACPI
 	select SYNC_FILE
 	select IOSF_MBI
+	select CRC32
 	help
 	  Choose this option if you have a system that has "Intel Graphics
 	  Media Accelerator" or "HD Graphics" integrated graphics,
diff --git a/drivers/gpu/drm/i915/Kconfig.debug b/drivers/gpu/drm/i915/Kconfig.debug
index 78c5c049a347..aed7d207ea84 100644
--- a/drivers/gpu/drm/i915/Kconfig.debug
+++ b/drivers/gpu/drm/i915/Kconfig.debug
@@ -25,6 +25,7 @@ config DRM_I915_DEBUG
         select DRM_VGEM # used by igt/prime_vgem (dmabuf interop checks)
         select DRM_DEBUG_MM if DRM=y
 	select DRM_DEBUG_MM_SELFTEST
+	select SW_SYNC # signaling validation framework (igt/syncobj*)
 	select DRM_I915_SW_FENCE_DEBUG_OBJECTS
 	select DRM_I915_SELFTEST
         default n
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index f8227318dcaf..892f52b53060 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -39,6 +39,7 @@ i915-y += i915_cmd_parser.o \
 	  i915_gem_gtt.o \
 	  i915_gem_internal.o \
 	  i915_gem.o \
+	  i915_gem_object.o \
 	  i915_gem_render_state.o \
 	  i915_gem_request.o \
 	  i915_gem_shrinker.o \
diff --git a/drivers/gpu/drm/i915/gvt/aperture_gm.c b/drivers/gpu/drm/i915/gvt/aperture_gm.c
index 325618d969fe..ca3d1925beda 100644
--- a/drivers/gpu/drm/i915/gvt/aperture_gm.c
+++ b/drivers/gpu/drm/i915/gvt/aperture_gm.c
@@ -285,8 +285,8 @@ static int alloc_resource(struct intel_vgpu *vgpu,
 	return 0;
 
 no_enough_resource:
-	gvt_vgpu_err("fail to allocate resource %s\n", item);
-	gvt_vgpu_err("request %luMB avail %luMB max %luMB taken %luMB\n",
+	gvt_err("fail to allocate resource %s\n", item);
+	gvt_err("request %luMB avail %luMB max %luMB taken %luMB\n",
 		BYTES_TO_MB(request), BYTES_TO_MB(avail),
 		BYTES_TO_MB(max), BYTES_TO_MB(taken));
 	return -ENOSPC;
diff --git a/drivers/gpu/drm/i915/gvt/cfg_space.c b/drivers/gpu/drm/i915/gvt/cfg_space.c
index 40af17ec6312..ff3154fe6588 100644
--- a/drivers/gpu/drm/i915/gvt/cfg_space.c
+++ b/drivers/gpu/drm/i915/gvt/cfg_space.c
@@ -197,78 +197,65 @@ static int emulate_pci_command_write(struct intel_vgpu *vgpu,
 static int emulate_pci_bar_write(struct intel_vgpu *vgpu, unsigned int offset,
 	void *p_data, unsigned int bytes)
 {
-	unsigned int bar_index =
-		(rounddown(offset, 8) % PCI_BASE_ADDRESS_0) / 8;
 	u32 new = *(u32 *)(p_data);
 	bool lo = IS_ALIGNED(offset, 8);
 	u64 size;
 	int ret = 0;
 	bool mmio_enabled =
 		vgpu_cfg_space(vgpu)[PCI_COMMAND] & PCI_COMMAND_MEMORY;
+	struct intel_vgpu_pci_bar *bars = vgpu->cfg_space.bar;
 
-	if (WARN_ON(bar_index >= INTEL_GVT_PCI_BAR_MAX))
-		return -EINVAL;
-
+	/*
+	 * Power-up software can determine how much address
+	 * space the device requires by writing a value of
+	 * all 1's to the register and then reading the value
+	 * back. The device will return 0's in all don't-care
+	 * address bits.
+	 */
 	if (new == 0xffffffff) {
-		/*
-		 * Power-up software can determine how much address
-		 * space the device requires by writing a value of
-		 * all 1's to the register and then reading the value
-		 * back. The device will return 0's in all don't-care
-		 * address bits.
-		 */
-		size = vgpu->cfg_space.bar[bar_index].size;
-		if (lo) {
-			new = rounddown(new, size);
-		} else {
-			u32 val = vgpu_cfg_space(vgpu)[rounddown(offset, 8)];
-			/* for 32bit mode bar it returns all-0 in upper 32
-			 * bit, for 64bit mode bar it will calculate the
-			 * size with lower 32bit and return the corresponding
-			 * value
+		switch (offset) {
+		case PCI_BASE_ADDRESS_0:
+		case PCI_BASE_ADDRESS_1:
+			size = ~(bars[INTEL_GVT_PCI_BAR_GTTMMIO].size -1);
+			intel_vgpu_write_pci_bar(vgpu, offset,
+						size >> (lo ? 0 : 32), lo);
+			/*
+			 * Untrap the BAR, since guest hasn't configured a
+			 * valid GPA
 			 */
-			if (val & PCI_BASE_ADDRESS_MEM_TYPE_64)
-				new &= (~(size-1)) >> 32;
-			else
-				new = 0;
-		}
-		/*
-		 * Unmapp & untrap the BAR, since guest hasn't configured a
-		 * valid GPA
-		 */
-		switch (bar_index) {
-		case INTEL_GVT_PCI_BAR_GTTMMIO:
 			ret = trap_gttmmio(vgpu, false);
 			break;
-		case INTEL_GVT_PCI_BAR_APERTURE:
+		case PCI_BASE_ADDRESS_2:
+		case PCI_BASE_ADDRESS_3:
+			size = ~(bars[INTEL_GVT_PCI_BAR_APERTURE].size -1);
+			intel_vgpu_write_pci_bar(vgpu, offset,
+						size >> (lo ? 0 : 32), lo);
 			ret = map_aperture(vgpu, false);
 			break;
+		default:
+			/* Unimplemented BARs */
+			intel_vgpu_write_pci_bar(vgpu, offset, 0x0, false);
 		}
-		intel_vgpu_write_pci_bar(vgpu, offset, new, lo);
 	} else {
-		/*
-		 * Unmapp & untrap the old BAR first, since guest has
-		 * re-configured the BAR
-		 */
-		switch (bar_index) {
-		case INTEL_GVT_PCI_BAR_GTTMMIO:
-			ret = trap_gttmmio(vgpu, false);
+		switch (offset) {
+		case PCI_BASE_ADDRESS_0:
+		case PCI_BASE_ADDRESS_1:
+			/*
+			 * Untrap the old BAR first, since guest has
+			 * re-configured the BAR
+			 */
+			trap_gttmmio(vgpu, false);
+			intel_vgpu_write_pci_bar(vgpu, offset, new, lo);
+			ret = trap_gttmmio(vgpu, mmio_enabled);
 			break;
-		case INTEL_GVT_PCI_BAR_APERTURE:
-			ret = map_aperture(vgpu, false);
+		case PCI_BASE_ADDRESS_2:
+		case PCI_BASE_ADDRESS_3:
+			map_aperture(vgpu, false);
+			intel_vgpu_write_pci_bar(vgpu, offset, new, lo);
+			ret = map_aperture(vgpu, mmio_enabled);
 			break;
-		}
-		intel_vgpu_write_pci_bar(vgpu, offset, new, lo);
-		/* Track the new BAR */
-		if (mmio_enabled) {
-			switch (bar_index) {
-			case INTEL_GVT_PCI_BAR_GTTMMIO:
-				ret = trap_gttmmio(vgpu, true);
-				break;
-			case INTEL_GVT_PCI_BAR_APERTURE:
-				ret = map_aperture(vgpu, true);
-				break;
-			}
+		default:
+			intel_vgpu_write_pci_bar(vgpu, offset, new, lo);
 		}
 	}
 	return ret;
@@ -299,10 +286,7 @@ int intel_vgpu_emulate_cfg_write(struct intel_vgpu *vgpu, unsigned int offset,
 	}
 
 	switch (rounddown(offset, 4)) {
-	case PCI_BASE_ADDRESS_0:
-	case PCI_BASE_ADDRESS_1:
-	case PCI_BASE_ADDRESS_2:
-	case PCI_BASE_ADDRESS_3:
+	case PCI_BASE_ADDRESS_0 ... PCI_BASE_ADDRESS_5:
 		if (WARN_ON(!IS_ALIGNED(offset, 4)))
 			return -EINVAL;
 		return emulate_pci_bar_write(vgpu, offset, p_data, bytes);
@@ -344,7 +328,6 @@ void intel_vgpu_init_cfg_space(struct intel_vgpu *vgpu,
 	struct intel_gvt *gvt = vgpu->gvt;
 	const struct intel_gvt_device_info *info = &gvt->device_info;
 	u16 *gmch_ctl;
-	int i;
 
 	memcpy(vgpu_cfg_space(vgpu), gvt->firmware.cfg_space,
 	       info->cfg_space_size);
@@ -371,13 +354,13 @@ void intel_vgpu_init_cfg_space(struct intel_vgpu *vgpu,
 	 */
 	memset(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_1, 0, 4);
 	memset(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_3, 0, 4);
+	memset(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_4, 0, 8);
 	memset(vgpu_cfg_space(vgpu) + INTEL_GVT_PCI_OPREGION, 0, 4);
 
-	for (i = 0; i < INTEL_GVT_MAX_BAR_NUM; i++) {
-		vgpu->cfg_space.bar[i].size = pci_resource_len(
-					      gvt->dev_priv->drm.pdev, i * 2);
-		vgpu->cfg_space.bar[i].tracked = false;
-	}
+	vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_GTTMMIO].size =
+				pci_resource_len(gvt->dev_priv->drm.pdev, 0);
+	vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_APERTURE].size =
+				pci_resource_len(gvt->dev_priv->drm.pdev, 2);
 }
 
 /**
diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c
index 713848c36349..21c36e256884 100644
--- a/drivers/gpu/drm/i915/gvt/cmd_parser.c
+++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c
@@ -1382,13 +1382,13 @@ static inline int cmd_address_audit(struct parser_exec_state *s,
 			ret = -EINVAL;
 			goto err;
 		}
-	} else if ((!vgpu_gmadr_is_valid(s->vgpu, guest_gma)) ||
-			(!vgpu_gmadr_is_valid(s->vgpu,
-					      guest_gma + op_size - 1))) {
+	} else if (!intel_gvt_ggtt_validate_range(vgpu, guest_gma, op_size)) {
 		ret = -EINVAL;
 		goto err;
 	}
+
 	return 0;
+
 err:
 	gvt_vgpu_err("cmd_parser: Malicious %s detected, addr=0x%lx, len=%d!\n",
 			s->info->name, guest_gma, op_size);
@@ -2647,7 +2647,7 @@ static int shadow_workload_ring_buffer(struct intel_vgpu_workload *workload)
 	return 0;
 }
 
-int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload)
+int intel_gvt_scan_and_shadow_ringbuffer(struct intel_vgpu_workload *workload)
 {
 	int ret;
 	struct intel_vgpu *vgpu = workload->vgpu;
@@ -2714,7 +2714,7 @@ static int shadow_indirect_ctx(struct intel_shadow_wa_ctx *wa_ctx)
 unmap_src:
 	i915_gem_object_unpin_map(obj);
 put_obj:
-	i915_gem_object_put(wa_ctx->indirect_ctx.obj);
+	i915_gem_object_put(obj);
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.h b/drivers/gpu/drm/i915/gvt/cmd_parser.h
index bed33514103c..286703643002 100644
--- a/drivers/gpu/drm/i915/gvt/cmd_parser.h
+++ b/drivers/gpu/drm/i915/gvt/cmd_parser.h
@@ -42,7 +42,7 @@ void intel_gvt_clean_cmd_parser(struct intel_gvt *gvt);
 
 int intel_gvt_init_cmd_parser(struct intel_gvt *gvt);
 
-int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload);
+int intel_gvt_scan_and_shadow_ringbuffer(struct intel_vgpu_workload *workload);
 
 int intel_gvt_scan_and_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx);
 
diff --git a/drivers/gpu/drm/i915/gvt/display.c b/drivers/gpu/drm/i915/gvt/display.c
index 7cb0818a13de..3c318439a659 100644
--- a/drivers/gpu/drm/i915/gvt/display.c
+++ b/drivers/gpu/drm/i915/gvt/display.c
@@ -178,9 +178,9 @@ static void emulate_monitor_status_change(struct intel_vgpu *vgpu)
 				SDE_PORTE_HOTPLUG_SPT);
 		vgpu_vreg(vgpu, SKL_FUSE_STATUS) |=
 				SKL_FUSE_DOWNLOAD_STATUS |
-				SKL_FUSE_PG0_DIST_STATUS |
-				SKL_FUSE_PG1_DIST_STATUS |
-				SKL_FUSE_PG2_DIST_STATUS;
+				SKL_FUSE_PG_DIST_STATUS(SKL_PG0) |
+				SKL_FUSE_PG_DIST_STATUS(SKL_PG1) |
+				SKL_FUSE_PG_DIST_STATUS(SKL_PG2);
 		vgpu_vreg(vgpu, LCPLL1_CTL) |=
 				LCPLL_PLL_ENABLE |
 				LCPLL_PLL_LOCK;
diff --git a/drivers/gpu/drm/i915/gvt/execlist.c b/drivers/gpu/drm/i915/gvt/execlist.c
index 1648887d3f55..91b4300f3b39 100644
--- a/drivers/gpu/drm/i915/gvt/execlist.c
+++ b/drivers/gpu/drm/i915/gvt/execlist.c
@@ -622,6 +622,7 @@ static int submit_context(struct intel_vgpu *vgpu, int ring_id,
 	struct list_head *q = workload_q_head(vgpu, ring_id);
 	struct intel_vgpu_workload *last_workload = get_last_workload(q);
 	struct intel_vgpu_workload *workload = NULL;
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
 	u64 ring_context_gpa;
 	u32 head, tail, start, ctl, ctx_ctl, per_ctx, indirect_ctx;
 	int ret;
@@ -685,6 +686,7 @@ static int submit_context(struct intel_vgpu *vgpu, int ring_id,
 	workload->complete = complete_execlist_workload;
 	workload->status = -EINPROGRESS;
 	workload->emulate_schedule_in = emulate_schedule_in;
+	workload->shadowed = false;
 
 	if (ring_id == RCS) {
 		intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
@@ -718,6 +720,17 @@ static int submit_context(struct intel_vgpu *vgpu, int ring_id,
 		return ret;
 	}
 
+	/* Only scan and shadow the first workload in the queue
+	 * as there is only one pre-allocated buf-obj for shadow.
+	 */
+	if (list_empty(workload_q_head(vgpu, ring_id))) {
+		intel_runtime_pm_get(dev_priv);
+		mutex_lock(&dev_priv->drm.struct_mutex);
+		intel_gvt_scan_and_shadow_workload(workload);
+		mutex_unlock(&dev_priv->drm.struct_mutex);
+		intel_runtime_pm_put(dev_priv);
+	}
+
 	queue_workload(workload);
 	return 0;
 }
@@ -800,6 +813,8 @@ static void clean_workloads(struct intel_vgpu *vgpu, unsigned long engine_mask)
 			list_del_init(&pos->list);
 			free_workload(pos);
 		}
+
+		clear_bit(engine->id, vgpu->shadow_ctx_desc_updated);
 	}
 }
 
diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c
index 6166e34d892b..e6dfc3331f4b 100644
--- a/drivers/gpu/drm/i915/gvt/gtt.c
+++ b/drivers/gpu/drm/i915/gvt/gtt.c
@@ -259,7 +259,7 @@ static void write_pte64(struct drm_i915_private *dev_priv,
 	writeq(pte, addr);
 }
 
-static inline struct intel_gvt_gtt_entry *gtt_get_entry64(void *pt,
+static inline int gtt_get_entry64(void *pt,
 		struct intel_gvt_gtt_entry *e,
 		unsigned long index, bool hypervisor_access, unsigned long gpa,
 		struct intel_vgpu *vgpu)
@@ -268,22 +268,23 @@ static inline struct intel_gvt_gtt_entry *gtt_get_entry64(void *pt,
 	int ret;
 
 	if (WARN_ON(info->gtt_entry_size != 8))
-		return e;
+		return -EINVAL;
 
 	if (hypervisor_access) {
 		ret = intel_gvt_hypervisor_read_gpa(vgpu, gpa +
 				(index << info->gtt_entry_size_shift),
 				&e->val64, 8);
-		WARN_ON(ret);
+		if (WARN_ON(ret))
+			return ret;
 	} else if (!pt) {
 		e->val64 = read_pte64(vgpu->gvt->dev_priv, index);
 	} else {
 		e->val64 = *((u64 *)pt + index);
 	}
-	return e;
+	return 0;
 }
 
-static inline struct intel_gvt_gtt_entry *gtt_set_entry64(void *pt,
+static inline int gtt_set_entry64(void *pt,
 		struct intel_gvt_gtt_entry *e,
 		unsigned long index, bool hypervisor_access, unsigned long gpa,
 		struct intel_vgpu *vgpu)
@@ -292,19 +293,20 @@ static inline struct intel_gvt_gtt_entry *gtt_set_entry64(void *pt,
 	int ret;
 
 	if (WARN_ON(info->gtt_entry_size != 8))
-		return e;
+		return -EINVAL;
 
 	if (hypervisor_access) {
 		ret = intel_gvt_hypervisor_write_gpa(vgpu, gpa +
 				(index << info->gtt_entry_size_shift),
 				&e->val64, 8);
-		WARN_ON(ret);
+		if (WARN_ON(ret))
+			return ret;
 	} else if (!pt) {
 		write_pte64(vgpu->gvt->dev_priv, index, e->val64);
 	} else {
 		*((u64 *)pt + index) = e->val64;
 	}
-	return e;
+	return 0;
 }
 
 #define GTT_HAW 46
@@ -445,21 +447,25 @@ static int gtt_entry_p2m(struct intel_vgpu *vgpu, struct intel_gvt_gtt_entry *p,
 /*
  * MM helpers.
  */
-struct intel_gvt_gtt_entry *intel_vgpu_mm_get_entry(struct intel_vgpu_mm *mm,
+int intel_vgpu_mm_get_entry(struct intel_vgpu_mm *mm,
 		void *page_table, struct intel_gvt_gtt_entry *e,
 		unsigned long index)
 {
 	struct intel_gvt *gvt = mm->vgpu->gvt;
 	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
+	int ret;
 
 	e->type = mm->page_table_entry_type;
 
-	ops->get_entry(page_table, e, index, false, 0, mm->vgpu);
+	ret = ops->get_entry(page_table, e, index, false, 0, mm->vgpu);
+	if (ret)
+		return ret;
+
 	ops->test_pse(e);
-	return e;
+	return 0;
 }
 
-struct intel_gvt_gtt_entry *intel_vgpu_mm_set_entry(struct intel_vgpu_mm *mm,
+int intel_vgpu_mm_set_entry(struct intel_vgpu_mm *mm,
 		void *page_table, struct intel_gvt_gtt_entry *e,
 		unsigned long index)
 {
@@ -472,7 +478,7 @@ struct intel_gvt_gtt_entry *intel_vgpu_mm_set_entry(struct intel_vgpu_mm *mm,
 /*
  * PPGTT shadow page table helpers.
  */
-static inline struct intel_gvt_gtt_entry *ppgtt_spt_get_entry(
+static inline int ppgtt_spt_get_entry(
 		struct intel_vgpu_ppgtt_spt *spt,
 		void *page_table, int type,
 		struct intel_gvt_gtt_entry *e, unsigned long index,
@@ -480,20 +486,24 @@ static inline struct intel_gvt_gtt_entry *ppgtt_spt_get_entry(
 {
 	struct intel_gvt *gvt = spt->vgpu->gvt;
 	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
+	int ret;
 
 	e->type = get_entry_type(type);
 
 	if (WARN(!gtt_type_is_entry(e->type), "invalid entry type\n"))
-		return e;
+		return -EINVAL;
 
-	ops->get_entry(page_table, e, index, guest,
+	ret = ops->get_entry(page_table, e, index, guest,
 			spt->guest_page.gfn << GTT_PAGE_SHIFT,
 			spt->vgpu);
+	if (ret)
+		return ret;
+
 	ops->test_pse(e);
-	return e;
+	return 0;
 }
 
-static inline struct intel_gvt_gtt_entry *ppgtt_spt_set_entry(
+static inline int ppgtt_spt_set_entry(
 		struct intel_vgpu_ppgtt_spt *spt,
 		void *page_table, int type,
 		struct intel_gvt_gtt_entry *e, unsigned long index,
@@ -503,7 +513,7 @@ static inline struct intel_gvt_gtt_entry *ppgtt_spt_set_entry(
 	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
 
 	if (WARN(!gtt_type_is_entry(e->type), "invalid entry type\n"))
-		return e;
+		return -EINVAL;
 
 	return ops->set_entry(page_table, e, index, guest,
 			spt->guest_page.gfn << GTT_PAGE_SHIFT,
@@ -792,13 +802,13 @@ static struct intel_vgpu_ppgtt_spt *ppgtt_find_shadow_page(
 
 #define for_each_present_guest_entry(spt, e, i) \
 	for (i = 0; i < pt_entries(spt); i++) \
-	if (spt->vgpu->gvt->gtt.pte_ops->test_present( \
-		ppgtt_get_guest_entry(spt, e, i)))
+		if (!ppgtt_get_guest_entry(spt, e, i) && \
+		    spt->vgpu->gvt->gtt.pte_ops->test_present(e))
 
 #define for_each_present_shadow_entry(spt, e, i) \
 	for (i = 0; i < pt_entries(spt); i++) \
-	if (spt->vgpu->gvt->gtt.pte_ops->test_present( \
-		ppgtt_get_shadow_entry(spt, e, i)))
+		if (!ppgtt_get_shadow_entry(spt, e, i) && \
+		    spt->vgpu->gvt->gtt.pte_ops->test_present(e))
 
 static void ppgtt_get_shadow_page(struct intel_vgpu_ppgtt_spt *spt)
 {
@@ -979,29 +989,26 @@ fail:
 }
 
 static int ppgtt_handle_guest_entry_removal(struct intel_vgpu_guest_page *gpt,
-		unsigned long index)
+		struct intel_gvt_gtt_entry *se, unsigned long index)
 {
 	struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt);
 	struct intel_vgpu_shadow_page *sp = &spt->shadow_page;
 	struct intel_vgpu *vgpu = spt->vgpu;
 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
-	struct intel_gvt_gtt_entry e;
 	int ret;
 
-	ppgtt_get_shadow_entry(spt, &e, index);
-
-	trace_gpt_change(spt->vgpu->id, "remove", spt, sp->type, e.val64,
+	trace_gpt_change(spt->vgpu->id, "remove", spt, sp->type, se->val64,
 			 index);
 
-	if (!ops->test_present(&e))
+	if (!ops->test_present(se))
 		return 0;
 
-	if (ops->get_pfn(&e) == vgpu->gtt.scratch_pt[sp->type].page_mfn)
+	if (ops->get_pfn(se) == vgpu->gtt.scratch_pt[sp->type].page_mfn)
 		return 0;
 
-	if (gtt_type_is_pt(get_next_pt_type(e.type))) {
+	if (gtt_type_is_pt(get_next_pt_type(se->type))) {
 		struct intel_vgpu_ppgtt_spt *s =
-			ppgtt_find_shadow_page(vgpu, ops->get_pfn(&e));
+			ppgtt_find_shadow_page(vgpu, ops->get_pfn(se));
 		if (!s) {
 			gvt_vgpu_err("fail to find guest page\n");
 			ret = -ENXIO;
@@ -1011,12 +1018,10 @@ static int ppgtt_handle_guest_entry_removal(struct intel_vgpu_guest_page *gpt,
 		if (ret)
 			goto fail;
 	}
-	ops->set_pfn(&e, vgpu->gtt.scratch_pt[sp->type].page_mfn);
-	ppgtt_set_shadow_entry(spt, &e, index);
 	return 0;
 fail:
 	gvt_vgpu_err("fail: shadow page %p guest entry 0x%llx type %d\n",
-			spt, e.val64, e.type);
+			spt, se->val64, se->type);
 	return ret;
 }
 
@@ -1236,22 +1241,37 @@ static int ppgtt_handle_guest_write_page_table(
 {
 	struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt);
 	struct intel_vgpu *vgpu = spt->vgpu;
+	int type = spt->shadow_page.type;
 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+	struct intel_gvt_gtt_entry se;
 
 	int ret;
 	int new_present;
 
 	new_present = ops->test_present(we);
 
-	ret = ppgtt_handle_guest_entry_removal(gpt, index);
-	if (ret)
-		goto fail;
+	/*
+	 * Adding the new entry first and then removing the old one, that can
+	 * guarantee the ppgtt table is validated during the window between
+	 * adding and removal.
+	 */
+	ppgtt_get_shadow_entry(spt, &se, index);
 
 	if (new_present) {
 		ret = ppgtt_handle_guest_entry_add(gpt, we, index);
 		if (ret)
 			goto fail;
 	}
+
+	ret = ppgtt_handle_guest_entry_removal(gpt, &se, index);
+	if (ret)
+		goto fail;
+
+	if (!new_present) {
+		ops->set_pfn(&se, vgpu->gtt.scratch_pt[type].page_mfn);
+		ppgtt_set_shadow_entry(spt, &se, index);
+	}
+
 	return 0;
 fail:
 	gvt_vgpu_err("fail: shadow page %p guest entry 0x%llx type %d.\n",
@@ -1323,7 +1343,7 @@ static int ppgtt_handle_guest_write_page_table_bytes(void *gp,
 	struct intel_vgpu *vgpu = spt->vgpu;
 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
 	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
-	struct intel_gvt_gtt_entry we;
+	struct intel_gvt_gtt_entry we, se;
 	unsigned long index;
 	int ret;
 
@@ -1339,7 +1359,8 @@ static int ppgtt_handle_guest_write_page_table_bytes(void *gp,
 			return ret;
 	} else {
 		if (!test_bit(index, spt->post_shadow_bitmap)) {
-			ret = ppgtt_handle_guest_entry_removal(gpt, index);
+			ppgtt_get_shadow_entry(spt, &se, index);
+			ret = ppgtt_handle_guest_entry_removal(gpt, &se, index);
 			if (ret)
 				return ret;
 		}
@@ -1713,8 +1734,10 @@ unsigned long intel_vgpu_gma_to_gpa(struct intel_vgpu_mm *mm, unsigned long gma)
 		if (!vgpu_gmadr_is_valid(vgpu, gma))
 			goto err;
 
-		ggtt_get_guest_entry(mm, &e,
-			gma_ops->gma_to_ggtt_pte_index(gma));
+		ret = ggtt_get_guest_entry(mm, &e,
+				gma_ops->gma_to_ggtt_pte_index(gma));
+		if (ret)
+			goto err;
 		gpa = (pte_ops->get_pfn(&e) << GTT_PAGE_SHIFT)
 			+ (gma & ~GTT_PAGE_MASK);
 
@@ -1724,7 +1747,9 @@ unsigned long intel_vgpu_gma_to_gpa(struct intel_vgpu_mm *mm, unsigned long gma)
 
 	switch (mm->page_table_level) {
 	case 4:
-		ppgtt_get_shadow_root_entry(mm, &e, 0);
+		ret = ppgtt_get_shadow_root_entry(mm, &e, 0);
+		if (ret)
+			goto err;
 		gma_index[0] = gma_ops->gma_to_pml4_index(gma);
 		gma_index[1] = gma_ops->gma_to_l4_pdp_index(gma);
 		gma_index[2] = gma_ops->gma_to_pde_index(gma);
@@ -1732,15 +1757,19 @@ unsigned long intel_vgpu_gma_to_gpa(struct intel_vgpu_mm *mm, unsigned long gma)
 		index = 4;
 		break;
 	case 3:
-		ppgtt_get_shadow_root_entry(mm, &e,
+		ret = ppgtt_get_shadow_root_entry(mm, &e,
 				gma_ops->gma_to_l3_pdp_index(gma));
+		if (ret)
+			goto err;
 		gma_index[0] = gma_ops->gma_to_pde_index(gma);
 		gma_index[1] = gma_ops->gma_to_pte_index(gma);
 		index = 2;
 		break;
 	case 2:
-		ppgtt_get_shadow_root_entry(mm, &e,
+		ret = ppgtt_get_shadow_root_entry(mm, &e,
 				gma_ops->gma_to_pde_index(gma));
+		if (ret)
+			goto err;
 		gma_index[0] = gma_ops->gma_to_pte_index(gma);
 		index = 1;
 		break;
@@ -1755,6 +1784,11 @@ unsigned long intel_vgpu_gma_to_gpa(struct intel_vgpu_mm *mm, unsigned long gma)
 			(i == index - 1));
 		if (ret)
 			goto err;
+
+		if (!pte_ops->test_present(&e)) {
+			gvt_dbg_core("GMA 0x%lx is not present\n", gma);
+			goto err;
+		}
 	}
 
 	gpa = (pte_ops->get_pfn(&e) << GTT_PAGE_SHIFT)
@@ -2329,13 +2363,12 @@ void intel_vgpu_reset_ggtt(struct intel_vgpu *vgpu)
 /**
  * intel_vgpu_reset_gtt - reset the all GTT related status
  * @vgpu: a vGPU
- * @dmlr: true for vGPU Device Model Level Reset, false for GT Reset
  *
  * This function is called from vfio core to reset reset all
  * GTT related status, including GGTT, PPGTT, scratch page.
  *
  */
-void intel_vgpu_reset_gtt(struct intel_vgpu *vgpu, bool dmlr)
+void intel_vgpu_reset_gtt(struct intel_vgpu *vgpu)
 {
 	int i;
 
@@ -2347,9 +2380,6 @@ void intel_vgpu_reset_gtt(struct intel_vgpu *vgpu, bool dmlr)
 	 */
 	intel_vgpu_free_mm(vgpu, INTEL_GVT_MM_PPGTT);
 
-	if (!dmlr)
-		return;
-
 	intel_vgpu_reset_ggtt(vgpu);
 
 	/* clear scratch page for security */
diff --git a/drivers/gpu/drm/i915/gvt/gtt.h b/drivers/gpu/drm/i915/gvt/gtt.h
index f88eb5e89bea..30a4c8d16026 100644
--- a/drivers/gpu/drm/i915/gvt/gtt.h
+++ b/drivers/gpu/drm/i915/gvt/gtt.h
@@ -49,14 +49,18 @@ struct intel_gvt_gtt_entry {
 };
 
 struct intel_gvt_gtt_pte_ops {
-	struct intel_gvt_gtt_entry *(*get_entry)(void *pt,
-		struct intel_gvt_gtt_entry *e,
-		unsigned long index, bool hypervisor_access, unsigned long gpa,
-		struct intel_vgpu *vgpu);
-	struct intel_gvt_gtt_entry *(*set_entry)(void *pt,
-		struct intel_gvt_gtt_entry *e,
-		unsigned long index, bool hypervisor_access, unsigned long gpa,
-		struct intel_vgpu *vgpu);
+	int (*get_entry)(void *pt,
+			 struct intel_gvt_gtt_entry *e,
+			 unsigned long index,
+			 bool hypervisor_access,
+			 unsigned long gpa,
+			 struct intel_vgpu *vgpu);
+	int (*set_entry)(void *pt,
+			 struct intel_gvt_gtt_entry *e,
+			 unsigned long index,
+			 bool hypervisor_access,
+			 unsigned long gpa,
+			 struct intel_vgpu *vgpu);
 	bool (*test_present)(struct intel_gvt_gtt_entry *e);
 	void (*clear_present)(struct intel_gvt_gtt_entry *e);
 	bool (*test_pse)(struct intel_gvt_gtt_entry *e);
@@ -143,12 +147,12 @@ struct intel_vgpu_mm {
 	struct intel_vgpu *vgpu;
 };
 
-extern struct intel_gvt_gtt_entry *intel_vgpu_mm_get_entry(
+extern int intel_vgpu_mm_get_entry(
 		struct intel_vgpu_mm *mm,
 		void *page_table, struct intel_gvt_gtt_entry *e,
 		unsigned long index);
 
-extern struct intel_gvt_gtt_entry *intel_vgpu_mm_set_entry(
+extern int intel_vgpu_mm_set_entry(
 		struct intel_vgpu_mm *mm,
 		void *page_table, struct intel_gvt_gtt_entry *e,
 		unsigned long index);
@@ -208,7 +212,7 @@ extern void intel_vgpu_clean_gtt(struct intel_vgpu *vgpu);
 void intel_vgpu_reset_ggtt(struct intel_vgpu *vgpu);
 
 extern int intel_gvt_init_gtt(struct intel_gvt *gvt);
-extern void intel_vgpu_reset_gtt(struct intel_vgpu *vgpu, bool dmlr);
+void intel_vgpu_reset_gtt(struct intel_vgpu *vgpu);
 extern void intel_gvt_clean_gtt(struct intel_gvt *gvt);
 
 extern struct intel_vgpu_mm *intel_gvt_find_ppgtt_mm(struct intel_vgpu *vgpu,
diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h
index 2964a4d01a66..44b719eda8c4 100644
--- a/drivers/gpu/drm/i915/gvt/gvt.h
+++ b/drivers/gpu/drm/i915/gvt/gvt.h
@@ -167,6 +167,7 @@ struct intel_vgpu {
 	atomic_t running_workload_num;
 	DECLARE_BITMAP(tlb_handle_pending, I915_NUM_ENGINES);
 	struct i915_gem_context *shadow_ctx;
+	DECLARE_BITMAP(shadow_ctx_desc_updated, I915_NUM_ENGINES);
 
 #if IS_ENABLED(CONFIG_DRM_I915_GVT_KVMGT)
 	struct {
@@ -482,6 +483,8 @@ int intel_vgpu_init_opregion(struct intel_vgpu *vgpu, u32 gpa);
 int intel_vgpu_emulate_opregion_request(struct intel_vgpu *vgpu, u32 swsci);
 void populate_pvinfo_page(struct intel_vgpu *vgpu);
 
+int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload);
+
 struct intel_gvt_ops {
 	int (*emulate_cfg_read)(struct intel_vgpu *, unsigned int, void *,
 				unsigned int);
diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c
index feed9921b3b3..2294466dd415 100644
--- a/drivers/gpu/drm/i915/gvt/handlers.c
+++ b/drivers/gpu/drm/i915/gvt/handlers.c
@@ -113,9 +113,17 @@ static int new_mmio_info(struct intel_gvt *gvt,
 
 		info->offset = i;
 		p = find_mmio_info(gvt, info->offset);
-		if (p)
-			gvt_err("dup mmio definition offset %x\n",
+		if (p) {
+			WARN(1, "dup mmio definition offset %x\n",
 				info->offset);
+			kfree(info);
+
+			/* We return -EEXIST here to make GVT-g load fail.
+			 * So duplicated MMIO can be found as soon as
+			 * possible.
+			 */
+			return -EEXIST;
+		}
 
 		info->ro_mask = ro_mask;
 		info->device = device;
@@ -1222,10 +1230,12 @@ static int power_well_ctl_mmio_write(struct intel_vgpu *vgpu,
 {
 	write_vreg(vgpu, offset, p_data, bytes);
 
-	if (vgpu_vreg(vgpu, offset) & HSW_PWR_WELL_ENABLE_REQUEST)
-		vgpu_vreg(vgpu, offset) |= HSW_PWR_WELL_STATE_ENABLED;
+	if (vgpu_vreg(vgpu, offset) & HSW_PWR_WELL_CTL_REQ(HSW_DISP_PW_GLOBAL))
+		vgpu_vreg(vgpu, offset) |=
+			HSW_PWR_WELL_CTL_STATE(HSW_DISP_PW_GLOBAL);
 	else
-		vgpu_vreg(vgpu, offset) &= ~HSW_PWR_WELL_STATE_ENABLED;
+		vgpu_vreg(vgpu, offset) &=
+			~HSW_PWR_WELL_CTL_STATE(HSW_DISP_PW_GLOBAL);
 	return 0;
 }
 
@@ -2242,10 +2252,17 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_D(GEN6_RC6p_THRESHOLD, D_ALL);
 	MMIO_D(GEN6_RC6pp_THRESHOLD, D_ALL);
 	MMIO_D(GEN6_PMINTRMSK, D_ALL);
-	MMIO_DH(HSW_PWR_WELL_BIOS, D_BDW, NULL, power_well_ctl_mmio_write);
-	MMIO_DH(HSW_PWR_WELL_DRIVER, D_BDW, NULL, power_well_ctl_mmio_write);
-	MMIO_DH(HSW_PWR_WELL_KVMR, D_BDW, NULL, power_well_ctl_mmio_write);
-	MMIO_DH(HSW_PWR_WELL_DEBUG, D_BDW, NULL, power_well_ctl_mmio_write);
+	/*
+	 * Use an arbitrary power well controlled by the PWR_WELL_CTL
+	 * register.
+	 */
+	MMIO_DH(HSW_PWR_WELL_CTL_BIOS(HSW_DISP_PW_GLOBAL), D_BDW, NULL,
+		power_well_ctl_mmio_write);
+	MMIO_DH(HSW_PWR_WELL_CTL_DRIVER(HSW_DISP_PW_GLOBAL), D_BDW, NULL,
+		power_well_ctl_mmio_write);
+	MMIO_DH(HSW_PWR_WELL_CTL_KVMR, D_BDW, NULL, power_well_ctl_mmio_write);
+	MMIO_DH(HSW_PWR_WELL_CTL_DEBUG(HSW_DISP_PW_GLOBAL), D_BDW, NULL,
+		power_well_ctl_mmio_write);
 	MMIO_DH(HSW_PWR_WELL_CTL5, D_BDW, NULL, power_well_ctl_mmio_write);
 	MMIO_DH(HSW_PWR_WELL_CTL6, D_BDW, NULL, power_well_ctl_mmio_write);
 
@@ -2581,7 +2598,6 @@ static int init_broadwell_mmio_info(struct intel_gvt *gvt)
 	MMIO_F(0x24d0, 48, F_CMD_ACCESS, 0, 0, D_BDW_PLUS,
 		NULL, force_nonpriv_write);
 
-	MMIO_D(0x22040, D_BDW_PLUS);
 	MMIO_D(0x44484, D_BDW_PLUS);
 	MMIO_D(0x4448c, D_BDW_PLUS);
 
@@ -2636,10 +2652,13 @@ static int init_skl_mmio_info(struct intel_gvt *gvt)
 	MMIO_F(_DPD_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_SKL_PLUS, NULL,
 						dp_aux_ch_ctl_mmio_write);
 
-	MMIO_D(HSW_PWR_WELL_BIOS, D_SKL_PLUS);
-	MMIO_DH(HSW_PWR_WELL_DRIVER, D_SKL_PLUS, NULL,
-						skl_power_well_ctl_write);
-	MMIO_DH(GEN6_PCODE_MAILBOX, D_SKL_PLUS, NULL, mailbox_write);
+	/*
+	 * Use an arbitrary power well controlled by the PWR_WELL_CTL
+	 * register.
+	 */
+	MMIO_D(HSW_PWR_WELL_CTL_BIOS(SKL_DISP_PW_MISC_IO), D_SKL_PLUS);
+	MMIO_DH(HSW_PWR_WELL_CTL_DRIVER(SKL_DISP_PW_MISC_IO), D_SKL_PLUS, NULL,
+		skl_power_well_ctl_write);
 
 	MMIO_D(0xa210, D_SKL_PLUS);
 	MMIO_D(GEN9_MEDIA_PG_IDLE_HYSTERESIS, D_SKL_PLUS);
@@ -2831,7 +2850,6 @@ static int init_skl_mmio_info(struct intel_gvt *gvt)
 	MMIO_D(0x320f0, D_SKL | D_KBL);
 
 	MMIO_DFH(_REG_VCS2_EXCC, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(_REG_VECS_EXCC, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL);
 	MMIO_D(0x70034, D_SKL_PLUS);
 	MMIO_D(0x71034, D_SKL_PLUS);
 	MMIO_D(0x72034, D_SKL_PLUS);
@@ -2849,10 +2867,7 @@ static int init_skl_mmio_info(struct intel_gvt *gvt)
 		NULL, NULL);
 
 	MMIO_D(0x4ab8, D_KBL);
-	MMIO_D(0x940c, D_SKL_PLUS);
 	MMIO_D(0x2248, D_SKL_PLUS | D_KBL);
-	MMIO_D(0x4ab0, D_SKL | D_KBL);
-	MMIO_D(0x20d4, D_SKL | D_KBL);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c
index fd0c85f9ef3c..83e88c70272a 100644
--- a/drivers/gpu/drm/i915/gvt/kvmgt.c
+++ b/drivers/gpu/drm/i915/gvt/kvmgt.c
@@ -1170,10 +1170,27 @@ vgpu_id_show(struct device *dev, struct device_attribute *attr,
 	return sprintf(buf, "\n");
 }
 
+static ssize_t
+hw_id_show(struct device *dev, struct device_attribute *attr,
+	   char *buf)
+{
+	struct mdev_device *mdev = mdev_from_dev(dev);
+
+	if (mdev) {
+		struct intel_vgpu *vgpu = (struct intel_vgpu *)
+			mdev_get_drvdata(mdev);
+		return sprintf(buf, "%u\n",
+			       vgpu->shadow_ctx->hw_id);
+	}
+	return sprintf(buf, "\n");
+}
+
 static DEVICE_ATTR_RO(vgpu_id);
+static DEVICE_ATTR_RO(hw_id);
 
 static struct attribute *intel_vgpu_attrs[] = {
 	&dev_attr_vgpu_id.attr,
+	&dev_attr_hw_id.attr,
 	NULL
 };
 
diff --git a/drivers/gpu/drm/i915/gvt/render.c b/drivers/gpu/drm/i915/gvt/render.c
index 504e57c3bc23..2ea542257f03 100644
--- a/drivers/gpu/drm/i915/gvt/render.c
+++ b/drivers/gpu/drm/i915/gvt/render.c
@@ -207,18 +207,16 @@ static void load_mocs(struct intel_vgpu *vgpu, int ring_id)
 
 	offset.reg = regs[ring_id];
 	for (i = 0; i < 64; i++) {
-		gen9_render_mocs[ring_id][i] = I915_READ(offset);
+		gen9_render_mocs[ring_id][i] = I915_READ_FW(offset);
 		I915_WRITE(offset, vgpu_vreg(vgpu, offset));
-		POSTING_READ(offset);
 		offset.reg += 4;
 	}
 
 	if (ring_id == RCS) {
 		l3_offset.reg = 0xb020;
 		for (i = 0; i < 32; i++) {
-			gen9_render_mocs_L3[i] = I915_READ(l3_offset);
-			I915_WRITE(l3_offset, vgpu_vreg(vgpu, l3_offset));
-			POSTING_READ(l3_offset);
+			gen9_render_mocs_L3[i] = I915_READ_FW(l3_offset);
+			I915_WRITE_FW(l3_offset, vgpu_vreg(vgpu, l3_offset));
 			l3_offset.reg += 4;
 		}
 	}
@@ -242,18 +240,16 @@ static void restore_mocs(struct intel_vgpu *vgpu, int ring_id)
 
 	offset.reg = regs[ring_id];
 	for (i = 0; i < 64; i++) {
-		vgpu_vreg(vgpu, offset) = I915_READ(offset);
-		I915_WRITE(offset, gen9_render_mocs[ring_id][i]);
-		POSTING_READ(offset);
+		vgpu_vreg(vgpu, offset) = I915_READ_FW(offset);
+		I915_WRITE_FW(offset, gen9_render_mocs[ring_id][i]);
 		offset.reg += 4;
 	}
 
 	if (ring_id == RCS) {
 		l3_offset.reg = 0xb020;
 		for (i = 0; i < 32; i++) {
-			vgpu_vreg(vgpu, l3_offset) = I915_READ(l3_offset);
-			I915_WRITE(l3_offset, gen9_render_mocs_L3[i]);
-			POSTING_READ(l3_offset);
+			vgpu_vreg(vgpu, l3_offset) = I915_READ_FW(l3_offset);
+			I915_WRITE_FW(l3_offset, gen9_render_mocs_L3[i]);
 			l3_offset.reg += 4;
 		}
 	}
@@ -272,6 +268,7 @@ static void switch_mmio_to_vgpu(struct intel_vgpu *vgpu, int ring_id)
 	u32 ctx_ctrl = reg_state[CTX_CONTEXT_CONTROL_VAL];
 	u32 inhibit_mask =
 		_MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
+	i915_reg_t last_reg = _MMIO(0);
 
 	if (IS_SKYLAKE(vgpu->gvt->dev_priv)
 		|| IS_KABYLAKE(vgpu->gvt->dev_priv)) {
@@ -287,7 +284,7 @@ static void switch_mmio_to_vgpu(struct intel_vgpu *vgpu, int ring_id)
 		if (mmio->ring_id != ring_id)
 			continue;
 
-		mmio->value = I915_READ(mmio->reg);
+		mmio->value = I915_READ_FW(mmio->reg);
 
 		/*
 		 * if it is an inhibit context, load in_context mmio
@@ -304,13 +301,18 @@ static void switch_mmio_to_vgpu(struct intel_vgpu *vgpu, int ring_id)
 		else
 			v = vgpu_vreg(vgpu, mmio->reg);
 
-		I915_WRITE(mmio->reg, v);
-		POSTING_READ(mmio->reg);
+		I915_WRITE_FW(mmio->reg, v);
+		last_reg = mmio->reg;
 
 		trace_render_mmio(vgpu->id, "load",
 				  i915_mmio_reg_offset(mmio->reg),
 				  mmio->value, v);
 	}
+
+	/* Make sure the swiched MMIOs has taken effect. */
+	if (likely(INTEL_GVT_MMIO_OFFSET(last_reg)))
+		I915_READ_FW(last_reg);
+
 	handle_tlb_pending_event(vgpu, ring_id);
 }
 
@@ -319,6 +321,7 @@ static void switch_mmio_to_host(struct intel_vgpu *vgpu, int ring_id)
 {
 	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
 	struct render_mmio *mmio;
+	i915_reg_t last_reg = _MMIO(0);
 	u32 v;
 	int i, array_size;
 
@@ -335,7 +338,7 @@ static void switch_mmio_to_host(struct intel_vgpu *vgpu, int ring_id)
 		if (mmio->ring_id != ring_id)
 			continue;
 
-		vgpu_vreg(vgpu, mmio->reg) = I915_READ(mmio->reg);
+		vgpu_vreg(vgpu, mmio->reg) = I915_READ_FW(mmio->reg);
 
 		if (mmio->mask) {
 			vgpu_vreg(vgpu, mmio->reg) &= ~(mmio->mask << 16);
@@ -346,13 +349,17 @@ static void switch_mmio_to_host(struct intel_vgpu *vgpu, int ring_id)
 		if (mmio->in_context)
 			continue;
 
-		I915_WRITE(mmio->reg, v);
-		POSTING_READ(mmio->reg);
+		I915_WRITE_FW(mmio->reg, v);
+		last_reg = mmio->reg;
 
 		trace_render_mmio(vgpu->id, "restore",
 				  i915_mmio_reg_offset(mmio->reg),
 				  mmio->value, v);
 	}
+
+	/* Make sure the swiched MMIOs has taken effect. */
+	if (likely(INTEL_GVT_MMIO_OFFSET(last_reg)))
+		I915_READ_FW(last_reg);
 }
 
 /**
@@ -367,12 +374,23 @@ static void switch_mmio_to_host(struct intel_vgpu *vgpu, int ring_id)
 void intel_gvt_switch_mmio(struct intel_vgpu *pre,
 			   struct intel_vgpu *next, int ring_id)
 {
+	struct drm_i915_private *dev_priv;
+
 	if (WARN_ON(!pre && !next))
 		return;
 
 	gvt_dbg_render("switch ring %d from %s to %s\n", ring_id,
 		       pre ? "vGPU" : "host", next ? "vGPU" : "HOST");
 
+	dev_priv = pre ? pre->gvt->dev_priv : next->gvt->dev_priv;
+
+	/**
+	 * We are using raw mmio access wrapper to improve the
+	 * performace for batch mmio read/write, so we need
+	 * handle forcewake mannually.
+	 */
+	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+
 	/**
 	 * TODO: Optimize for vGPU to vGPU switch by merging
 	 * switch_mmio_to_host() and switch_mmio_to_vgpu().
@@ -382,4 +400,6 @@ void intel_gvt_switch_mmio(struct intel_vgpu *pre,
 
 	if (next)
 		switch_mmio_to_vgpu(next, ring_id);
+
+	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 }
diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c
index 22e08eb2d0b7..391800d2067b 100644
--- a/drivers/gpu/drm/i915/gvt/scheduler.c
+++ b/drivers/gpu/drm/i915/gvt/scheduler.c
@@ -184,41 +184,52 @@ static int shadow_context_status_change(struct notifier_block *nb,
 	return NOTIFY_OK;
 }
 
-static int dispatch_workload(struct intel_vgpu_workload *workload)
+static void shadow_context_descriptor_update(struct i915_gem_context *ctx,
+		struct intel_engine_cs *engine)
+{
+	struct intel_context *ce = &ctx->engine[engine->id];
+	u64 desc = 0;
+
+	desc = ce->lrc_desc;
+
+	/* Update bits 0-11 of the context descriptor which includes flags
+	 * like GEN8_CTX_* cached in desc_template
+	 */
+	desc &= U64_MAX << 12;
+	desc |= ctx->desc_template & ((1ULL << 12) - 1);
+
+	ce->lrc_desc = desc;
+}
+
+/**
+ * intel_gvt_scan_and_shadow_workload - audit the workload by scanning and
+ * shadow it as well, include ringbuffer,wa_ctx and ctx.
+ * @workload: an abstract entity for each execlist submission.
+ *
+ * This function is called before the workload submitting to i915, to make
+ * sure the content of the workload is valid.
+ */
+int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload)
 {
 	int ring_id = workload->ring_id;
 	struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx;
 	struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv;
-	struct intel_engine_cs *engine = dev_priv->engine[ring_id];
 	struct drm_i915_gem_request *rq;
 	struct intel_vgpu *vgpu = workload->vgpu;
-	struct intel_ring *ring;
 	int ret;
 
-	gvt_dbg_sched("ring id %d prepare to dispatch workload %p\n",
-		ring_id, workload);
+	lockdep_assert_held(&dev_priv->drm.struct_mutex);
+
+	if (workload->shadowed)
+		return 0;
 
 	shadow_ctx->desc_template &= ~(0x3 << GEN8_CTX_ADDRESSING_MODE_SHIFT);
 	shadow_ctx->desc_template |= workload->ctx_desc.addressing_mode <<
 				    GEN8_CTX_ADDRESSING_MODE_SHIFT;
 
-	mutex_lock(&dev_priv->drm.struct_mutex);
-
-	/* pin shadow context by gvt even the shadow context will be pinned
-	 * when i915 alloc request. That is because gvt will update the guest
-	 * context from shadow context when workload is completed, and at that
-	 * moment, i915 may already unpined the shadow context to make the
-	 * shadow_ctx pages invalid. So gvt need to pin itself. After update
-	 * the guest context, gvt can unpin the shadow_ctx safely.
-	 */
-	ring = engine->context_pin(engine, shadow_ctx);
-	if (IS_ERR(ring)) {
-		ret = PTR_ERR(ring);
-		gvt_vgpu_err("fail to pin shadow context\n");
-		workload->status = ret;
-		mutex_unlock(&dev_priv->drm.struct_mutex);
-		return ret;
-	}
+	if (!test_and_set_bit(ring_id, vgpu->shadow_ctx_desc_updated))
+		shadow_context_descriptor_update(shadow_ctx,
+					dev_priv->engine[ring_id]);
 
 	rq = i915_gem_request_alloc(dev_priv->engine[ring_id], shadow_ctx);
 	if (IS_ERR(rq)) {
@@ -231,7 +242,7 @@ static int dispatch_workload(struct intel_vgpu_workload *workload)
 
 	workload->req = i915_gem_request_get(rq);
 
-	ret = intel_gvt_scan_and_shadow_workload(workload);
+	ret = intel_gvt_scan_and_shadow_ringbuffer(workload);
 	if (ret)
 		goto out;
 
@@ -246,25 +257,61 @@ static int dispatch_workload(struct intel_vgpu_workload *workload)
 	if (ret)
 		goto out;
 
+	workload->shadowed = true;
+
+out:
+	return ret;
+}
+
+static int dispatch_workload(struct intel_vgpu_workload *workload)
+{
+	int ring_id = workload->ring_id;
+	struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx;
+	struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv;
+	struct intel_engine_cs *engine = dev_priv->engine[ring_id];
+	struct intel_vgpu *vgpu = workload->vgpu;
+	struct intel_ring *ring;
+	int ret = 0;
+
+	gvt_dbg_sched("ring id %d prepare to dispatch workload %p\n",
+		ring_id, workload);
+
+	mutex_lock(&dev_priv->drm.struct_mutex);
+
+	ret = intel_gvt_scan_and_shadow_workload(workload);
+	if (ret)
+		goto out;
+
 	if (workload->prepare) {
 		ret = workload->prepare(workload);
 		if (ret)
 			goto out;
 	}
 
-	gvt_dbg_sched("ring id %d submit workload to i915 %p\n",
-			ring_id, workload->req);
+	/* pin shadow context by gvt even the shadow context will be pinned
+	 * when i915 alloc request. That is because gvt will update the guest
+	 * context from shadow context when workload is completed, and at that
+	 * moment, i915 may already unpined the shadow context to make the
+	 * shadow_ctx pages invalid. So gvt need to pin itself. After update
+	 * the guest context, gvt can unpin the shadow_ctx safely.
+	 */
+	ring = engine->context_pin(engine, shadow_ctx);
+	if (IS_ERR(ring)) {
+		ret = PTR_ERR(ring);
+		gvt_vgpu_err("fail to pin shadow context\n");
+		goto out;
+	}
 
-	ret = 0;
-	workload->dispatched = true;
 out:
 	if (ret)
 		workload->status = ret;
 
-	if (!IS_ERR_OR_NULL(rq))
-		i915_add_request(rq);
-	else
-		engine->context_unpin(engine, shadow_ctx);
+	if (!IS_ERR_OR_NULL(workload->req)) {
+		gvt_dbg_sched("ring id %d submit workload to i915 %p\n",
+				ring_id, workload->req);
+		i915_add_request(workload->req);
+		workload->dispatched = true;
+	}
 
 	mutex_unlock(&dev_priv->drm.struct_mutex);
 	return ret;
@@ -617,7 +664,7 @@ err:
 
 void intel_vgpu_clean_gvt_context(struct intel_vgpu *vgpu)
 {
-	i915_gem_context_put_unlocked(vgpu->shadow_ctx);
+	i915_gem_context_put(vgpu->shadow_ctx);
 }
 
 int intel_vgpu_init_gvt_context(struct intel_vgpu *vgpu)
@@ -631,5 +678,7 @@ int intel_vgpu_init_gvt_context(struct intel_vgpu *vgpu)
 
 	vgpu->shadow_ctx->engine[RCS].initialised = true;
 
+	bitmap_zero(vgpu->shadow_ctx_desc_updated, I915_NUM_ENGINES);
+
 	return 0;
 }
diff --git a/drivers/gpu/drm/i915/gvt/scheduler.h b/drivers/gpu/drm/i915/gvt/scheduler.h
index 9b6bf51e9b9b..0d431a968a32 100644
--- a/drivers/gpu/drm/i915/gvt/scheduler.h
+++ b/drivers/gpu/drm/i915/gvt/scheduler.h
@@ -82,6 +82,7 @@ struct intel_vgpu_workload {
 	struct drm_i915_gem_request *req;
 	/* if this workload has been dispatched to i915? */
 	bool dispatched;
+	bool shadowed;
 	int status;
 
 	struct intel_vgpu_mm *shadow_mm;
diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c
index 3deadcbd5a24..02c61a1ad56a 100644
--- a/drivers/gpu/drm/i915/gvt/vgpu.c
+++ b/drivers/gpu/drm/i915/gvt/vgpu.c
@@ -43,6 +43,7 @@ void populate_pvinfo_page(struct intel_vgpu *vgpu)
 	vgpu_vreg(vgpu, vgtif_reg(version_minor)) = 0;
 	vgpu_vreg(vgpu, vgtif_reg(display_ready)) = 0;
 	vgpu_vreg(vgpu, vgtif_reg(vgt_id)) = vgpu->id;
+	vgpu_vreg(vgpu, vgtif_reg(vgt_caps)) = VGT_CAPS_FULL_48BIT_PPGTT;
 	vgpu_vreg(vgpu, vgtif_reg(avail_rs.mappable_gmadr.base)) =
 		vgpu_aperture_gmadr_base(vgpu);
 	vgpu_vreg(vgpu, vgtif_reg(avail_rs.mappable_gmadr.size)) =
@@ -504,11 +505,11 @@ void intel_gvt_reset_vgpu_locked(struct intel_vgpu *vgpu, bool dmlr,
 	/* full GPU reset or device model level reset */
 	if (engine_mask == ALL_ENGINES || dmlr) {
 
-		intel_vgpu_reset_gtt(vgpu, dmlr);
-
 		/*fence will not be reset during virtual reset */
-		if (dmlr)
+		if (dmlr) {
+			intel_vgpu_reset_gtt(vgpu);
 			intel_vgpu_reset_resource(vgpu);
+		}
 
 		intel_vgpu_reset_mmio(vgpu, dmlr);
 		populate_pvinfo_page(vgpu);
diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c
index f0cb22cc0dd6..8ba932b22f7c 100644
--- a/drivers/gpu/drm/i915/i915_cmd_parser.c
+++ b/drivers/gpu/drm/i915/i915_cmd_parser.c
@@ -1073,7 +1073,7 @@ static u32 *copy_batch(struct drm_i915_gem_object *dst_obj,
 		goto unpin_src;
 	}
 
-	dst = i915_gem_object_pin_map(dst_obj, I915_MAP_WB);
+	dst = i915_gem_object_pin_map(dst_obj, I915_MAP_FORCE_WB);
 	if (IS_ERR(dst))
 		goto unpin_dst;
 
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 00d8967c8512..e4d4b6b41e26 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -28,6 +28,7 @@
 
 #include <linux/debugfs.h>
 #include <linux/sort.h>
+#include <linux/sched/mm.h>
 #include "intel_drv.h"
 
 static inline struct drm_i915_private *node_to_i915(struct drm_info_node *node)
@@ -543,75 +544,6 @@ static int i915_gem_gtt_info(struct seq_file *m, void *data)
 	return 0;
 }
 
-static int i915_gem_pageflip_info(struct seq_file *m, void *data)
-{
-	struct drm_i915_private *dev_priv = node_to_i915(m->private);
-	struct drm_device *dev = &dev_priv->drm;
-	struct intel_crtc *crtc;
-	int ret;
-
-	ret = mutex_lock_interruptible(&dev->struct_mutex);
-	if (ret)
-		return ret;
-
-	for_each_intel_crtc(dev, crtc) {
-		const char pipe = pipe_name(crtc->pipe);
-		const char plane = plane_name(crtc->plane);
-		struct intel_flip_work *work;
-
-		spin_lock_irq(&dev->event_lock);
-		work = crtc->flip_work;
-		if (work == NULL) {
-			seq_printf(m, "No flip due on pipe %c (plane %c)\n",
-				   pipe, plane);
-		} else {
-			u32 pending;
-			u32 addr;
-
-			pending = atomic_read(&work->pending);
-			if (pending) {
-				seq_printf(m, "Flip ioctl preparing on pipe %c (plane %c)\n",
-					   pipe, plane);
-			} else {
-				seq_printf(m, "Flip pending (waiting for vsync) on pipe %c (plane %c)\n",
-					   pipe, plane);
-			}
-			if (work->flip_queued_req) {
-				struct intel_engine_cs *engine = work->flip_queued_req->engine;
-
-				seq_printf(m, "Flip queued on %s at seqno %x, last submitted seqno %x [current breadcrumb %x], completed? %d\n",
-					   engine->name,
-					   work->flip_queued_req->global_seqno,
-					   intel_engine_last_submit(engine),
-					   intel_engine_get_seqno(engine),
-					   i915_gem_request_completed(work->flip_queued_req));
-			} else
-				seq_printf(m, "Flip not associated with any ring\n");
-			seq_printf(m, "Flip queued on frame %d, (was ready on frame %d), now %d\n",
-				   work->flip_queued_vblank,
-				   work->flip_ready_vblank,
-				   intel_crtc_get_vblank_counter(crtc));
-			seq_printf(m, "%d prepares\n", atomic_read(&work->pending));
-
-			if (INTEL_GEN(dev_priv) >= 4)
-				addr = I915_HI_DISPBASE(I915_READ(DSPSURF(crtc->plane)));
-			else
-				addr = I915_READ(DSPADDR(crtc->plane));
-			seq_printf(m, "Current scanout address 0x%08x\n", addr);
-
-			if (work->pending_flip_obj) {
-				seq_printf(m, "New framebuffer address 0x%08lx\n", (long)work->gtt_offset);
-				seq_printf(m, "MMIO update completed? %d\n",  addr == work->gtt_offset);
-			}
-		}
-		spin_unlock_irq(&dev->event_lock);
-	}
-
-	mutex_unlock(&dev->struct_mutex);
-
-	return 0;
-}
-
 static int i915_gem_batch_pool_info(struct seq_file *m, void *data)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
@@ -1159,7 +1091,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
 		intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
 
 		reqf = I915_READ(GEN6_RPNSWREQ);
-		if (IS_GEN9(dev_priv))
+		if (INTEL_GEN(dev_priv) >= 9)
 			reqf >>= 23;
 		else {
 			reqf &= ~GEN6_TURBO_DISABLE;
@@ -1181,7 +1113,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
 		rpdownei = I915_READ(GEN6_RP_CUR_DOWN_EI) & GEN6_CURIAVG_MASK;
 		rpcurdown = I915_READ(GEN6_RP_CUR_DOWN) & GEN6_CURBSYTAVG_MASK;
 		rpprevdown = I915_READ(GEN6_RP_PREV_DOWN) & GEN6_CURBSYTAVG_MASK;
-		if (IS_GEN9(dev_priv))
+		if (INTEL_GEN(dev_priv) >= 9)
 			cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
 		else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 			cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
@@ -1210,7 +1142,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
 			   dev_priv->rps.pm_intrmsk_mbz);
 		seq_printf(m, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status);
 		seq_printf(m, "Render p-state ratio: %d\n",
-			   (gt_perf_status & (IS_GEN9(dev_priv) ? 0x1ff00 : 0xff00)) >> 8);
+			   (gt_perf_status & (INTEL_GEN(dev_priv) >= 9 ? 0x1ff00 : 0xff00)) >> 8);
 		seq_printf(m, "Render p-state VID: %d\n",
 			   gt_perf_status & 0xff);
 		seq_printf(m, "Render p-state limit: %d\n",
@@ -1241,18 +1173,21 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
 
 		max_freq = (IS_GEN9_LP(dev_priv) ? rp_state_cap >> 0 :
 			    rp_state_cap >> 16) & 0xff;
-		max_freq *= (IS_GEN9_BC(dev_priv) ? GEN9_FREQ_SCALER : 1);
+		max_freq *= (IS_GEN9_BC(dev_priv) ||
+			     IS_CANNONLAKE(dev_priv) ? GEN9_FREQ_SCALER : 1);
 		seq_printf(m, "Lowest (RPN) frequency: %dMHz\n",
 			   intel_gpu_freq(dev_priv, max_freq));
 
 		max_freq = (rp_state_cap & 0xff00) >> 8;
-		max_freq *= (IS_GEN9_BC(dev_priv) ? GEN9_FREQ_SCALER : 1);
+		max_freq *= (IS_GEN9_BC(dev_priv) ||
+			     IS_CANNONLAKE(dev_priv) ? GEN9_FREQ_SCALER : 1);
 		seq_printf(m, "Nominal (RP1) frequency: %dMHz\n",
 			   intel_gpu_freq(dev_priv, max_freq));
 
 		max_freq = (IS_GEN9_LP(dev_priv) ? rp_state_cap >> 16 :
 			    rp_state_cap >> 0) & 0xff;
-		max_freq *= (IS_GEN9_BC(dev_priv) ? GEN9_FREQ_SCALER : 1);
+		max_freq *= (IS_GEN9_BC(dev_priv) ||
+			     IS_CANNONLAKE(dev_priv) ? GEN9_FREQ_SCALER : 1);
 		seq_printf(m, "Max non-overclocked (RP0) frequency: %dMHz\n",
 			   intel_gpu_freq(dev_priv, max_freq));
 		seq_printf(m, "Max overclocked frequency: %dMHz\n",
@@ -1407,6 +1342,23 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
 	return 0;
 }
 
+static int i915_reset_info(struct seq_file *m, void *unused)
+{
+	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct i915_gpu_error *error = &dev_priv->gpu_error;
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+
+	seq_printf(m, "full gpu reset = %u\n", i915_reset_count(error));
+
+	for_each_engine(engine, dev_priv, id) {
+		seq_printf(m, "%s = %u\n", engine->name,
+			   i915_reset_engine_count(error, engine));
+	}
+
+	return 0;
+}
+
 static int ironlake_drpc_info(struct seq_file *m)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
@@ -1838,7 +1790,7 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused)
 	if (ret)
 		goto out;
 
-	if (IS_GEN9_BC(dev_priv)) {
+	if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) {
 		/* Convert GT frequency to 50 HZ units */
 		min_gpu_freq =
 			dev_priv->rps.min_freq_softlimit / GEN9_FREQ_SCALER;
@@ -1858,7 +1810,8 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused)
 				       &ia_freq);
 		seq_printf(m, "%d\t\t%d\t\t\t\t%d\n",
 			   intel_gpu_freq(dev_priv, (gpu_freq *
-						     (IS_GEN9_BC(dev_priv) ?
+						     (IS_GEN9_BC(dev_priv) ||
+						      IS_CANNONLAKE(dev_priv) ?
 						      GEN9_FREQ_SCALER : 1))),
 			   ((ia_freq >> 0) & 0xff) * 100,
 			   ((ia_freq >> 8) & 0xff) * 100);
@@ -1914,7 +1867,7 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
 		return ret;
 
 #ifdef CONFIG_DRM_FBDEV_EMULATION
-	if (dev_priv->fbdev) {
+	if (dev_priv->fbdev && dev_priv->fbdev->helper.fb) {
 		fbdev_fb = to_intel_framebuffer(dev_priv->fbdev->helper.fb);
 
 		seq_printf(m, "fbcon size: %d x %d, depth %d, %d bpp, modifier 0x%llx, refcount %d, obj ",
@@ -1970,7 +1923,7 @@ static int i915_context_status(struct seq_file *m, void *unused)
 	if (ret)
 		return ret;
 
-	list_for_each_entry(ctx, &dev_priv->context_list, link) {
+	list_for_each_entry(ctx, &dev_priv->contexts.list, link) {
 		seq_printf(m, "HW context %u ", ctx->hw_id);
 		if (ctx->pid) {
 			struct task_struct *task;
@@ -2002,12 +1955,6 @@ static int i915_context_status(struct seq_file *m, void *unused)
 			seq_putc(m, '\n');
 		}
 
-		seq_printf(m,
-			   "\tvma hashtable size=%u (actual %lu), count=%u\n",
-			   ctx->vma_lut.ht_size,
-			   BIT(ctx->vma_lut.ht_bits),
-			   ctx->vma_lut.ht_count);
-
 		seq_putc(m, '\n');
 	}
 
@@ -2076,7 +2023,7 @@ static int i915_dump_lrc(struct seq_file *m, void *unused)
 	if (ret)
 		return ret;
 
-	list_for_each_entry(ctx, &dev_priv->context_list, link)
+	list_for_each_entry(ctx, &dev_priv->contexts.list, link)
 		for_each_engine(engine, dev_priv, id)
 			i915_dump_lrc_obj(m, ctx, engine);
 
@@ -2310,6 +2257,8 @@ static int i915_rps_boost_info(struct seq_file *m, void *data)
 	seq_printf(m, "GPU busy? %s [%d requests]\n",
 		   yesno(dev_priv->gt.awake), dev_priv->gt.active_requests);
 	seq_printf(m, "CPU waiting? %d\n", count_irq_waiters(dev_priv));
+	seq_printf(m, "Boosts outstanding? %d\n",
+		   atomic_read(&dev_priv->rps.num_waiters));
 	seq_printf(m, "Frequency requested %d\n",
 		   intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq));
 	seq_printf(m, "  min hard:%d, soft:%d; max soft:%d, hard:%d\n",
@@ -2323,22 +2272,20 @@ static int i915_rps_boost_info(struct seq_file *m, void *data)
 		   intel_gpu_freq(dev_priv, dev_priv->rps.boost_freq));
 
 	mutex_lock(&dev->filelist_mutex);
-	spin_lock(&dev_priv->rps.client_lock);
 	list_for_each_entry_reverse(file, &dev->filelist, lhead) {
 		struct drm_i915_file_private *file_priv = file->driver_priv;
 		struct task_struct *task;
 
 		rcu_read_lock();
 		task = pid_task(file->pid, PIDTYPE_PID);
-		seq_printf(m, "%s [%d]: %d boosts%s\n",
+		seq_printf(m, "%s [%d]: %d boosts\n",
 			   task ? task->comm : "<unknown>",
 			   task ? task->pid : -1,
-			   file_priv->rps.boosts,
-			   list_empty(&file_priv->rps.link) ? "" : ", active");
+			   atomic_read(&file_priv->rps.boosts));
 		rcu_read_unlock();
 	}
-	seq_printf(m, "Kernel (anonymous) boosts: %d\n", dev_priv->rps.boosts);
-	spin_unlock(&dev_priv->rps.client_lock);
+	seq_printf(m, "Kernel (anonymous) boosts: %d\n",
+		   atomic_read(&dev_priv->rps.boosts));
 	mutex_unlock(&dev->filelist_mutex);
 
 	if (INTEL_GEN(dev_priv) >= 6 &&
@@ -2831,7 +2778,7 @@ out:
 static int i915_energy_uJ(struct seq_file *m, void *data)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
-	u64 power;
+	unsigned long long power;
 	u32 units;
 
 	if (INTEL_GEN(dev_priv) < 6)
@@ -2839,15 +2786,18 @@ static int i915_energy_uJ(struct seq_file *m, void *data)
 
 	intel_runtime_pm_get(dev_priv);
 
-	rdmsrl(MSR_RAPL_POWER_UNIT, power);
-	power = (power & 0x1f00) >> 8;
-	units = 1000000 / (1 << power); /* convert to uJ */
+	if (rdmsrl_safe(MSR_RAPL_POWER_UNIT, &power)) {
+		intel_runtime_pm_put(dev_priv);
+		return -ENODEV;
+	}
+
+	units = (power & 0x1f00) >> 8;
 	power = I915_READ(MCH_SECP_NRG_STTS);
-	power *= units;
+	power = (1000000 * power) >> units; /* convert to uJ */
 
 	intel_runtime_pm_put(dev_priv);
 
-	seq_printf(m, "%llu", (long long unsigned)power);
+	seq_printf(m, "%llu", power);
 
 	return 0;
 }
@@ -3289,6 +3239,7 @@ static int i915_display_info(struct seq_file *m, void *unused)
 static int i915_engine_info(struct seq_file *m, void *unused)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct i915_gpu_error *error = &dev_priv->gpu_error;
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
 
@@ -3312,6 +3263,8 @@ static int i915_engine_info(struct seq_file *m, void *unused)
 			   engine->hangcheck.seqno,
 			   jiffies_to_msecs(jiffies - engine->hangcheck.action_timestamp),
 			   engine->timeline->inflight_seqnos);
+		seq_printf(m, "\tReset count: %d\n",
+			   i915_reset_engine_count(error, engine));
 
 		rcu_read_lock();
 
@@ -3370,8 +3323,10 @@ static int i915_engine_info(struct seq_file *m, void *unused)
 			ptr = I915_READ(RING_CONTEXT_STATUS_PTR(engine));
 			read = GEN8_CSB_READ_PTR(ptr);
 			write = GEN8_CSB_WRITE_PTR(ptr);
-			seq_printf(m, "\tExeclist CSB read %d, write %d\n",
-				   read, write);
+			seq_printf(m, "\tExeclist CSB read %d, write %d, interrupt posted? %s\n",
+				   read, write,
+				   yesno(test_bit(ENGINE_IRQ_EXECLIST,
+						  &engine->irq_posted)));
 			if (read >= GEN8_CSB_ENTRIES)
 				read = 0;
 			if (write >= GEN8_CSB_ENTRIES)
@@ -3758,13 +3713,18 @@ static ssize_t i915_displayport_test_active_write(struct file *file,
 
 	drm_connector_list_iter_begin(dev, &conn_iter);
 	drm_for_each_connector_iter(connector, &conn_iter) {
+		struct intel_encoder *encoder;
+
 		if (connector->connector_type !=
 		    DRM_MODE_CONNECTOR_DisplayPort)
 			continue;
 
-		if (connector->status == connector_status_connected &&
-		    connector->encoder != NULL) {
-			intel_dp = enc_to_intel_dp(connector->encoder);
+		encoder = to_intel_encoder(connector->encoder);
+		if (encoder && encoder->type == INTEL_OUTPUT_DP_MST)
+			continue;
+
+		if (encoder && connector->status == connector_status_connected) {
+			intel_dp = enc_to_intel_dp(&encoder->base);
 			status = kstrtoint(input_buffer, 10, &val);
 			if (status < 0)
 				break;
@@ -3796,13 +3756,18 @@ static int i915_displayport_test_active_show(struct seq_file *m, void *data)
 
 	drm_connector_list_iter_begin(dev, &conn_iter);
 	drm_for_each_connector_iter(connector, &conn_iter) {
+		struct intel_encoder *encoder;
+
 		if (connector->connector_type !=
 		    DRM_MODE_CONNECTOR_DisplayPort)
 			continue;
 
-		if (connector->status == connector_status_connected &&
-		    connector->encoder != NULL) {
-			intel_dp = enc_to_intel_dp(connector->encoder);
+		encoder = to_intel_encoder(connector->encoder);
+		if (encoder && encoder->type == INTEL_OUTPUT_DP_MST)
+			continue;
+
+		if (encoder && connector->status == connector_status_connected) {
+			intel_dp = enc_to_intel_dp(&encoder->base);
 			if (intel_dp->compliance.test_active)
 				seq_puts(m, "1");
 			else
@@ -3842,13 +3807,18 @@ static int i915_displayport_test_data_show(struct seq_file *m, void *data)
 
 	drm_connector_list_iter_begin(dev, &conn_iter);
 	drm_for_each_connector_iter(connector, &conn_iter) {
+		struct intel_encoder *encoder;
+
 		if (connector->connector_type !=
 		    DRM_MODE_CONNECTOR_DisplayPort)
 			continue;
 
-		if (connector->status == connector_status_connected &&
-		    connector->encoder != NULL) {
-			intel_dp = enc_to_intel_dp(connector->encoder);
+		encoder = to_intel_encoder(connector->encoder);
+		if (encoder && encoder->type == INTEL_OUTPUT_DP_MST)
+			continue;
+
+		if (encoder && connector->status == connector_status_connected) {
+			intel_dp = enc_to_intel_dp(&encoder->base);
 			if (intel_dp->compliance.test_type ==
 			    DP_TEST_LINK_EDID_READ)
 				seq_printf(m, "%lx",
@@ -3895,13 +3865,18 @@ static int i915_displayport_test_type_show(struct seq_file *m, void *data)
 
 	drm_connector_list_iter_begin(dev, &conn_iter);
 	drm_for_each_connector_iter(connector, &conn_iter) {
+		struct intel_encoder *encoder;
+
 		if (connector->connector_type !=
 		    DRM_MODE_CONNECTOR_DisplayPort)
 			continue;
 
-		if (connector->status == connector_status_connected &&
-		    connector->encoder != NULL) {
-			intel_dp = enc_to_intel_dp(connector->encoder);
+		encoder = to_intel_encoder(connector->encoder);
+		if (encoder && encoder->type == INTEL_OUTPUT_DP_MST)
+			continue;
+
+		if (encoder && connector->status == connector_status_connected) {
+			intel_dp = enc_to_intel_dp(&encoder->base);
 			seq_printf(m, "%02lx", intel_dp->compliance.test_type);
 		} else
 			seq_puts(m, "0");
@@ -4331,16 +4306,16 @@ i915_drop_caches_set(void *data, u64 val)
 		mutex_unlock(&dev->struct_mutex);
 	}
 
-	lockdep_set_current_reclaim_state(GFP_KERNEL);
+	fs_reclaim_acquire(GFP_KERNEL);
 	if (val & DROP_BOUND)
-		i915_gem_shrink(dev_priv, LONG_MAX, I915_SHRINK_BOUND);
+		i915_gem_shrink(dev_priv, LONG_MAX, NULL, I915_SHRINK_BOUND);
 
 	if (val & DROP_UNBOUND)
-		i915_gem_shrink(dev_priv, LONG_MAX, I915_SHRINK_UNBOUND);
+		i915_gem_shrink(dev_priv, LONG_MAX, NULL, I915_SHRINK_UNBOUND);
 
 	if (val & DROP_SHRINK_ALL)
 		i915_gem_shrink_all(dev_priv);
-	lockdep_clear_current_reclaim_state();
+	fs_reclaim_release(GFP_KERNEL);
 
 	if (val & DROP_FREED) {
 		synchronize_rcu();
@@ -4580,7 +4555,7 @@ static void gen9_sseu_device_status(struct drm_i915_private *dev_priv,
 
 		sseu->slice_mask |= BIT(s);
 
-		if (IS_GEN9_BC(dev_priv))
+		if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv))
 			sseu->subslice_mask =
 				INTEL_INFO(dev_priv)->sseu.subslice_mask;
 
@@ -4810,7 +4785,6 @@ static const struct drm_info_list i915_debugfs_list[] = {
 	{"i915_gem_gtt", i915_gem_gtt_info, 0},
 	{"i915_gem_pin_display", i915_gem_gtt_info, 0, (void *)1},
 	{"i915_gem_stolen", i915_gem_stolen_list_info },
-	{"i915_gem_pageflip", i915_gem_pageflip_info, 0},
 	{"i915_gem_request", i915_gem_request_info, 0},
 	{"i915_gem_seqno", i915_gem_seqno_info, 0},
 	{"i915_gem_fence_regs", i915_gem_fence_regs_info, 0},
@@ -4824,6 +4798,7 @@ static const struct drm_info_list i915_debugfs_list[] = {
 	{"i915_huc_load_status", i915_huc_load_status_info, 0},
 	{"i915_frequency_info", i915_frequency_info, 0},
 	{"i915_hangcheck_info", i915_hangcheck_info, 0},
+	{"i915_reset_info", i915_reset_info, 0},
 	{"i915_drpc_info", i915_drpc_info, 0},
 	{"i915_emon_status", i915_emon_status, 0},
 	{"i915_ring_freq_table", i915_ring_freq_table, 0},
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index fc307e03943c..9f45cfeae775 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -132,9 +132,13 @@ static enum intel_pch intel_virt_detect_pch(struct drm_i915_private *dev_priv)
 		DRM_DEBUG_KMS("Assuming Ibex Peak PCH\n");
 	} else if (IS_GEN6(dev_priv) || IS_IVYBRIDGE(dev_priv)) {
 		ret = PCH_CPT;
-		DRM_DEBUG_KMS("Assuming CouarPoint PCH\n");
+		DRM_DEBUG_KMS("Assuming CougarPoint PCH\n");
 	} else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
 		ret = PCH_LPT;
+		if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv))
+			dev_priv->pch_id = INTEL_PCH_LPT_LP_DEVICE_ID_TYPE;
+		else
+			dev_priv->pch_id = INTEL_PCH_LPT_DEVICE_ID_TYPE;
 		DRM_DEBUG_KMS("Assuming LynxPoint PCH\n");
 	} else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
 		ret = PCH_SPT;
@@ -173,29 +177,25 @@ static void intel_detect_pch(struct drm_i915_private *dev_priv)
 	while ((pch = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, pch))) {
 		if (pch->vendor == PCI_VENDOR_ID_INTEL) {
 			unsigned short id = pch->device & INTEL_PCH_DEVICE_ID_MASK;
-			unsigned short id_ext = pch->device &
-				INTEL_PCH_DEVICE_ID_MASK_EXT;
+
+			dev_priv->pch_id = id;
 
 			if (id == INTEL_PCH_IBX_DEVICE_ID_TYPE) {
-				dev_priv->pch_id = id;
 				dev_priv->pch_type = PCH_IBX;
 				DRM_DEBUG_KMS("Found Ibex Peak PCH\n");
 				WARN_ON(!IS_GEN5(dev_priv));
 			} else if (id == INTEL_PCH_CPT_DEVICE_ID_TYPE) {
-				dev_priv->pch_id = id;
 				dev_priv->pch_type = PCH_CPT;
 				DRM_DEBUG_KMS("Found CougarPoint PCH\n");
-				WARN_ON(!(IS_GEN6(dev_priv) ||
-					IS_IVYBRIDGE(dev_priv)));
+				WARN_ON(!IS_GEN6(dev_priv) &&
+					!IS_IVYBRIDGE(dev_priv));
 			} else if (id == INTEL_PCH_PPT_DEVICE_ID_TYPE) {
 				/* PantherPoint is CPT compatible */
-				dev_priv->pch_id = id;
 				dev_priv->pch_type = PCH_CPT;
 				DRM_DEBUG_KMS("Found PantherPoint PCH\n");
-				WARN_ON(!(IS_GEN6(dev_priv) ||
-					IS_IVYBRIDGE(dev_priv)));
+				WARN_ON(!IS_GEN6(dev_priv) &&
+					!IS_IVYBRIDGE(dev_priv));
 			} else if (id == INTEL_PCH_LPT_DEVICE_ID_TYPE) {
-				dev_priv->pch_id = id;
 				dev_priv->pch_type = PCH_LPT;
 				DRM_DEBUG_KMS("Found LynxPoint PCH\n");
 				WARN_ON(!IS_HASWELL(dev_priv) &&
@@ -203,51 +203,60 @@ static void intel_detect_pch(struct drm_i915_private *dev_priv)
 				WARN_ON(IS_HSW_ULT(dev_priv) ||
 					IS_BDW_ULT(dev_priv));
 			} else if (id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
-				dev_priv->pch_id = id;
 				dev_priv->pch_type = PCH_LPT;
 				DRM_DEBUG_KMS("Found LynxPoint LP PCH\n");
 				WARN_ON(!IS_HASWELL(dev_priv) &&
 					!IS_BROADWELL(dev_priv));
 				WARN_ON(!IS_HSW_ULT(dev_priv) &&
 					!IS_BDW_ULT(dev_priv));
+			} else if (id == INTEL_PCH_WPT_DEVICE_ID_TYPE) {
+				/* WildcatPoint is LPT compatible */
+				dev_priv->pch_type = PCH_LPT;
+				DRM_DEBUG_KMS("Found WildcatPoint PCH\n");
+				WARN_ON(!IS_HASWELL(dev_priv) &&
+					!IS_BROADWELL(dev_priv));
+				WARN_ON(IS_HSW_ULT(dev_priv) ||
+					IS_BDW_ULT(dev_priv));
+			} else if (id == INTEL_PCH_WPT_LP_DEVICE_ID_TYPE) {
+				/* WildcatPoint is LPT compatible */
+				dev_priv->pch_type = PCH_LPT;
+				DRM_DEBUG_KMS("Found WildcatPoint LP PCH\n");
+				WARN_ON(!IS_HASWELL(dev_priv) &&
+					!IS_BROADWELL(dev_priv));
+				WARN_ON(!IS_HSW_ULT(dev_priv) &&
+					!IS_BDW_ULT(dev_priv));
 			} else if (id == INTEL_PCH_SPT_DEVICE_ID_TYPE) {
-				dev_priv->pch_id = id;
 				dev_priv->pch_type = PCH_SPT;
 				DRM_DEBUG_KMS("Found SunrisePoint PCH\n");
 				WARN_ON(!IS_SKYLAKE(dev_priv) &&
 					!IS_KABYLAKE(dev_priv));
-			} else if (id_ext == INTEL_PCH_SPT_LP_DEVICE_ID_TYPE) {
-				dev_priv->pch_id = id_ext;
+			} else if (id == INTEL_PCH_SPT_LP_DEVICE_ID_TYPE) {
 				dev_priv->pch_type = PCH_SPT;
 				DRM_DEBUG_KMS("Found SunrisePoint LP PCH\n");
 				WARN_ON(!IS_SKYLAKE(dev_priv) &&
 					!IS_KABYLAKE(dev_priv));
 			} else if (id == INTEL_PCH_KBP_DEVICE_ID_TYPE) {
-				dev_priv->pch_id = id;
 				dev_priv->pch_type = PCH_KBP;
-				DRM_DEBUG_KMS("Found KabyPoint PCH\n");
+				DRM_DEBUG_KMS("Found Kaby Lake PCH (KBP)\n");
 				WARN_ON(!IS_SKYLAKE(dev_priv) &&
 					!IS_KABYLAKE(dev_priv));
 			} else if (id == INTEL_PCH_CNP_DEVICE_ID_TYPE) {
-				dev_priv->pch_id = id;
 				dev_priv->pch_type = PCH_CNP;
-				DRM_DEBUG_KMS("Found CannonPoint PCH\n");
+				DRM_DEBUG_KMS("Found Cannon Lake PCH (CNP)\n");
 				WARN_ON(!IS_CANNONLAKE(dev_priv) &&
 					!IS_COFFEELAKE(dev_priv));
-			} else if (id_ext == INTEL_PCH_CNP_LP_DEVICE_ID_TYPE) {
-				dev_priv->pch_id = id_ext;
+			} else if (id == INTEL_PCH_CNP_LP_DEVICE_ID_TYPE) {
 				dev_priv->pch_type = PCH_CNP;
-				DRM_DEBUG_KMS("Found CannonPoint LP PCH\n");
+				DRM_DEBUG_KMS("Found Cannon Lake LP PCH (CNP-LP)\n");
 				WARN_ON(!IS_CANNONLAKE(dev_priv) &&
 					!IS_COFFEELAKE(dev_priv));
-			} else if ((id == INTEL_PCH_P2X_DEVICE_ID_TYPE) ||
-				   (id == INTEL_PCH_P3X_DEVICE_ID_TYPE) ||
-				   ((id == INTEL_PCH_QEMU_DEVICE_ID_TYPE) &&
+			} else if (id == INTEL_PCH_P2X_DEVICE_ID_TYPE ||
+				   id == INTEL_PCH_P3X_DEVICE_ID_TYPE ||
+				   (id == INTEL_PCH_QEMU_DEVICE_ID_TYPE &&
 				    pch->subsystem_vendor ==
 					    PCI_SUBVENDOR_ID_REDHAT_QUMRANET &&
 				    pch->subsystem_device ==
 					    PCI_SUBDEVICE_ID_QEMU)) {
-				dev_priv->pch_id = id;
 				dev_priv->pch_type =
 					intel_virt_detect_pch(dev_priv);
 			} else
@@ -331,6 +340,8 @@ static int i915_getparam(struct drm_device *dev, void *data,
 		break;
 	case I915_PARAM_HAS_GPU_RESET:
 		value = i915.enable_hangcheck && intel_has_gpu_reset(dev_priv);
+		if (value && intel_has_reset_engine(dev_priv))
+			value = 2;
 		break;
 	case I915_PARAM_HAS_RESOURCE_STREAMER:
 		value = HAS_RESOURCE_STREAMER(dev_priv);
@@ -377,6 +388,7 @@ static int i915_getparam(struct drm_device *dev, void *data,
 	case I915_PARAM_HAS_EXEC_FENCE:
 	case I915_PARAM_HAS_EXEC_CAPTURE:
 	case I915_PARAM_HAS_EXEC_BATCH_FIRST:
+	case I915_PARAM_HAS_EXEC_FENCE_ARRAY:
 		/* For the time being all of these are always true;
 		 * if some supported hardware does not have one of these
 		 * features this value needs to be provided from
@@ -585,16 +597,19 @@ static const struct vga_switcheroo_client_ops i915_switcheroo_ops = {
 
 static void i915_gem_fini(struct drm_i915_private *dev_priv)
 {
+	/* Flush any outstanding unpin_work. */
+	i915_gem_drain_workqueue(dev_priv);
+
 	mutex_lock(&dev_priv->drm.struct_mutex);
 	intel_uc_fini_hw(dev_priv);
 	i915_gem_cleanup_engines(dev_priv);
-	i915_gem_context_fini(dev_priv);
+	i915_gem_contexts_fini(dev_priv);
 	i915_gem_cleanup_userptr(dev_priv);
 	mutex_unlock(&dev_priv->drm.struct_mutex);
 
 	i915_gem_drain_freed_objects(dev_priv);
 
-	WARN_ON(!list_empty(&dev_priv->context_list));
+	WARN_ON(!list_empty(&dev_priv->contexts.list));
 }
 
 static int i915_load_modeset_init(struct drm_device *dev)
@@ -862,7 +877,6 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv,
 	spin_lock_init(&dev_priv->uncore.lock);
 
 	spin_lock_init(&dev_priv->mm.object_stat_lock);
-	spin_lock_init(&dev_priv->mmio_flip_lock);
 	mutex_init(&dev_priv->sb_lock);
 	mutex_init(&dev_priv->modeset_restore_lock);
 	mutex_init(&dev_priv->av_mutex);
@@ -1227,6 +1241,7 @@ static void i915_driver_register(struct drm_i915_private *dev_priv)
  */
 static void i915_driver_unregister(struct drm_i915_private *dev_priv)
 {
+	intel_fbdev_unregister(dev_priv);
 	intel_audio_deinit(dev_priv);
 
 	intel_gpu_ips_teardown();
@@ -1319,7 +1334,7 @@ int i915_driver_load(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	ret = i915_load_modeset_init(&dev_priv->drm);
 	if (ret < 0)
-		goto out_cleanup_vblank;
+		goto out_cleanup_hw;
 
 	i915_driver_register(dev_priv);
 
@@ -1336,8 +1351,6 @@ int i915_driver_load(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	return 0;
 
-out_cleanup_vblank:
-	drm_vblank_cleanup(&dev_priv->drm);
 out_cleanup_hw:
 	i915_driver_cleanup_hw(dev_priv);
 out_cleanup_mmio:
@@ -1360,7 +1373,7 @@ void i915_driver_unload(struct drm_device *dev)
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct pci_dev *pdev = dev_priv->drm.pdev;
 
-	intel_fbdev_fini(dev);
+	i915_driver_unregister(dev_priv);
 
 	if (i915_gem_suspend(dev_priv))
 		DRM_ERROR("failed to idle hardware; continuing to unload!\n");
@@ -1371,10 +1384,6 @@ void i915_driver_unload(struct drm_device *dev)
 
 	intel_gvt_cleanup(dev_priv);
 
-	i915_driver_unregister(dev_priv);
-
-	drm_vblank_cleanup(dev);
-
 	intel_modeset_cleanup(dev);
 
 	/*
@@ -1400,9 +1409,6 @@ void i915_driver_unload(struct drm_device *dev)
 	cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
 	i915_reset_error_state(dev_priv);
 
-	/* Flush any outstanding unpin_work. */
-	drain_workqueue(dev_priv->wq);
-
 	i915_gem_fini(dev_priv);
 	intel_uc_fini_fw(dev_priv);
 	intel_fbc_cleanup_cfb(dev_priv);
@@ -1427,9 +1433,10 @@ static void i915_driver_release(struct drm_device *dev)
 
 static int i915_driver_open(struct drm_device *dev, struct drm_file *file)
 {
+	struct drm_i915_private *i915 = to_i915(dev);
 	int ret;
 
-	ret = i915_gem_open(dev, file);
+	ret = i915_gem_open(i915, file);
 	if (ret)
 		return ret;
 
@@ -1459,7 +1466,7 @@ static void i915_driver_postclose(struct drm_device *dev, struct drm_file *file)
 	struct drm_i915_file_private *file_priv = file->driver_priv;
 
 	mutex_lock(&dev->struct_mutex);
-	i915_gem_context_close(dev, file);
+	i915_gem_context_close(file);
 	i915_gem_release(dev, file);
 	mutex_unlock(&dev->struct_mutex);
 
@@ -1825,7 +1832,8 @@ static int i915_resume_switcheroo(struct drm_device *dev)
 
 /**
  * i915_reset - reset chip after a hang
- * @dev_priv: device private to reset
+ * @i915: #drm_i915_private to reset
+ * @flags: Instructions
  *
  * Reset the chip.  Useful if a hang is detected. Marks the device as wedged
  * on failure.
@@ -1840,33 +1848,34 @@ static int i915_resume_switcheroo(struct drm_device *dev)
  *   - re-init interrupt state
  *   - re-init display
  */
-void i915_reset(struct drm_i915_private *dev_priv)
+void i915_reset(struct drm_i915_private *i915, unsigned int flags)
 {
-	struct i915_gpu_error *error = &dev_priv->gpu_error;
+	struct i915_gpu_error *error = &i915->gpu_error;
 	int ret;
 
-	lockdep_assert_held(&dev_priv->drm.struct_mutex);
+	lockdep_assert_held(&i915->drm.struct_mutex);
 	GEM_BUG_ON(!test_bit(I915_RESET_BACKOFF, &error->flags));
 
 	if (!test_bit(I915_RESET_HANDOFF, &error->flags))
 		return;
 
 	/* Clear any previous failed attempts at recovery. Time to try again. */
-	if (!i915_gem_unset_wedged(dev_priv))
+	if (!i915_gem_unset_wedged(i915))
 		goto wakeup;
 
+	if (!(flags & I915_RESET_QUIET))
+		dev_notice(i915->drm.dev, "Resetting chip after gpu hang\n");
 	error->reset_count++;
 
-	pr_notice("drm/i915: Resetting chip after gpu hang\n");
-	disable_irq(dev_priv->drm.irq);
-	ret = i915_gem_reset_prepare(dev_priv);
+	disable_irq(i915->drm.irq);
+	ret = i915_gem_reset_prepare(i915);
 	if (ret) {
 		DRM_ERROR("GPU recovery failed\n");
-		intel_gpu_reset(dev_priv, ALL_ENGINES);
+		intel_gpu_reset(i915, ALL_ENGINES);
 		goto error;
 	}
 
-	ret = intel_gpu_reset(dev_priv, ALL_ENGINES);
+	ret = intel_gpu_reset(i915, ALL_ENGINES);
 	if (ret) {
 		if (ret != -ENODEV)
 			DRM_ERROR("Failed to reset chip: %i\n", ret);
@@ -1875,16 +1884,22 @@ void i915_reset(struct drm_i915_private *dev_priv)
 		goto error;
 	}
 
-	i915_gem_reset(dev_priv);
-	intel_overlay_reset(dev_priv);
+	i915_gem_reset(i915);
+	intel_overlay_reset(i915);
 
 	/* Ok, now get things going again... */
 
 	/*
 	 * Everything depends on having the GTT running, so we need to start
-	 * there.  Fortunately we don't need to do this unless we reset the
-	 * chip at a PCI level.
-	 *
+	 * there.
+	 */
+	ret = i915_ggtt_enable_hw(i915);
+	if (ret) {
+		DRM_ERROR("Failed to re-enable GGTT following reset %d\n", ret);
+		goto error;
+	}
+
+	/*
 	 * Next we need to restore the context, but we don't use those
 	 * yet either...
 	 *
@@ -1892,17 +1907,17 @@ void i915_reset(struct drm_i915_private *dev_priv)
 	 * was running at the time of the reset (i.e. we weren't VT
 	 * switched away).
 	 */
-	ret = i915_gem_init_hw(dev_priv);
+	ret = i915_gem_init_hw(i915);
 	if (ret) {
 		DRM_ERROR("Failed hw init on reset %d\n", ret);
 		goto error;
 	}
 
-	i915_queue_hangcheck(dev_priv);
+	i915_queue_hangcheck(i915);
 
 finish:
-	i915_gem_reset_finish(dev_priv);
-	enable_irq(dev_priv->drm.irq);
+	i915_gem_reset_finish(i915);
+	enable_irq(i915->drm.irq);
 
 wakeup:
 	clear_bit(I915_RESET_HANDOFF, &error->flags);
@@ -1910,10 +1925,74 @@ wakeup:
 	return;
 
 error:
-	i915_gem_set_wedged(dev_priv);
+	i915_gem_set_wedged(i915);
+	i915_gem_retire_requests(i915);
 	goto finish;
 }
 
+/**
+ * i915_reset_engine - reset GPU engine to recover from a hang
+ * @engine: engine to reset
+ * @flags: options
+ *
+ * Reset a specific GPU engine. Useful if a hang is detected.
+ * Returns zero on successful reset or otherwise an error code.
+ *
+ * Procedure is:
+ *  - identifies the request that caused the hang and it is dropped
+ *  - reset engine (which will force the engine to idle)
+ *  - re-init/configure engine
+ */
+int i915_reset_engine(struct intel_engine_cs *engine, unsigned int flags)
+{
+	struct i915_gpu_error *error = &engine->i915->gpu_error;
+	struct drm_i915_gem_request *active_request;
+	int ret;
+
+	GEM_BUG_ON(!test_bit(I915_RESET_ENGINE + engine->id, &error->flags));
+
+	if (!(flags & I915_RESET_QUIET)) {
+		dev_notice(engine->i915->drm.dev,
+			   "Resetting %s after gpu hang\n", engine->name);
+	}
+	error->reset_engine_count[engine->id]++;
+
+	active_request = i915_gem_reset_prepare_engine(engine);
+	if (IS_ERR(active_request)) {
+		DRM_DEBUG_DRIVER("Previous reset failed, promote to full reset\n");
+		ret = PTR_ERR(active_request);
+		goto out;
+	}
+
+	ret = intel_gpu_reset(engine->i915, intel_engine_flag(engine));
+	if (ret) {
+		/* If we fail here, we expect to fallback to a global reset */
+		DRM_DEBUG_DRIVER("Failed to reset %s, ret=%d\n",
+				 engine->name, ret);
+		goto out;
+	}
+
+	/*
+	 * The request that caused the hang is stuck on elsp, we know the
+	 * active request and can drop it, adjust head to skip the offending
+	 * request to resume executing remaining requests in the queue.
+	 */
+	i915_gem_reset_engine(engine, active_request);
+
+	/*
+	 * The engine and its registers (and workarounds in case of render)
+	 * have been reset to their default values. Follow the init_ring
+	 * process to program RING_MODE, HWSP and re-enable submission.
+	 */
+	ret = engine->init_hw(engine);
+	if (ret)
+		goto out;
+
+out:
+	i915_gem_reset_finish_engine(engine);
+	return ret;
+}
+
 static int i915_pm_suspend(struct device *kdev)
 {
 	struct pci_dev *pdev = to_pci_dev(kdev);
@@ -2657,6 +2736,8 @@ static const struct drm_ioctl_desc i915_ioctls[] = {
 	DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_GETPARAM, i915_gem_context_getparam_ioctl, DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_SETPARAM, i915_gem_context_setparam_ioctl, DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(I915_PERF_OPEN, i915_perf_open_ioctl, DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(I915_PERF_ADD_CONFIG, i915_perf_add_config_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(I915_PERF_REMOVE_CONFIG, i915_perf_remove_config_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
 };
 
 static struct drm_driver driver = {
@@ -2665,12 +2746,11 @@ static struct drm_driver driver = {
 	 */
 	.driver_features =
 	    DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM | DRIVER_PRIME |
-	    DRIVER_RENDER | DRIVER_MODESET | DRIVER_ATOMIC,
+	    DRIVER_RENDER | DRIVER_MODESET | DRIVER_ATOMIC | DRIVER_SYNCOBJ,
 	.release = i915_driver_release,
 	.open = i915_driver_open,
 	.lastclose = i915_driver_lastclose,
 	.postclose = i915_driver_postclose,
-	.set_busid = drm_pci_set_busid,
 
 	.gem_close_object = i915_gem_close_object,
 	.gem_free_object_unlocked = i915_gem_free_object,
@@ -2683,7 +2763,6 @@ static struct drm_driver driver = {
 
 	.dumb_create = i915_gem_dumb_create,
 	.dumb_map_offset = i915_gem_mmap_gtt,
-	.dumb_destroy = drm_gem_dumb_destroy,
 	.ioctls = i915_ioctls,
 	.num_ioctls = ARRAY_SIZE(i915_ioctls),
 	.fops = &i915_driver_fops,
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index e1f7c97a338a..18d9da53282b 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -80,8 +80,8 @@
 
 #define DRIVER_NAME		"i915"
 #define DRIVER_DESC		"Intel Graphics"
-#define DRIVER_DATE		"20170619"
-#define DRIVER_TIMESTAMP	1497857498
+#define DRIVER_DATE		"20170818"
+#define DRIVER_TIMESTAMP	1503088845
 
 /* Use I915_STATE_WARN(x) and I915_STATE_WARN_ON() (rather than WARN() and
  * WARN_ON()) for hw state sanity checks to check for unexpected conditions
@@ -122,7 +122,7 @@ static inline bool is_fixed16_zero(uint_fixed_16_16_t val)
 	return false;
 }
 
-static inline uint_fixed_16_16_t u32_to_fixed_16_16(uint32_t val)
+static inline uint_fixed_16_16_t u32_to_fixed16(uint32_t val)
 {
 	uint_fixed_16_16_t fp;
 
@@ -132,17 +132,17 @@ static inline uint_fixed_16_16_t u32_to_fixed_16_16(uint32_t val)
 	return fp;
 }
 
-static inline uint32_t fixed_16_16_to_u32_round_up(uint_fixed_16_16_t fp)
+static inline uint32_t fixed16_to_u32_round_up(uint_fixed_16_16_t fp)
 {
 	return DIV_ROUND_UP(fp.val, 1 << 16);
 }
 
-static inline uint32_t fixed_16_16_to_u32(uint_fixed_16_16_t fp)
+static inline uint32_t fixed16_to_u32(uint_fixed_16_16_t fp)
 {
 	return fp.val >> 16;
 }
 
-static inline uint_fixed_16_16_t min_fixed_16_16(uint_fixed_16_16_t min1,
+static inline uint_fixed_16_16_t min_fixed16(uint_fixed_16_16_t min1,
 						 uint_fixed_16_16_t min2)
 {
 	uint_fixed_16_16_t min;
@@ -151,7 +151,7 @@ static inline uint_fixed_16_16_t min_fixed_16_16(uint_fixed_16_16_t min1,
 	return min;
 }
 
-static inline uint_fixed_16_16_t max_fixed_16_16(uint_fixed_16_16_t max1,
+static inline uint_fixed_16_16_t max_fixed16(uint_fixed_16_16_t max1,
 						 uint_fixed_16_16_t max2)
 {
 	uint_fixed_16_16_t max;
@@ -160,6 +160,14 @@ static inline uint_fixed_16_16_t max_fixed_16_16(uint_fixed_16_16_t max1,
 	return max;
 }
 
+static inline uint_fixed_16_16_t clamp_u64_to_fixed16(uint64_t val)
+{
+	uint_fixed_16_16_t fp;
+	WARN_ON(val >> 32);
+	fp.val = clamp_t(uint32_t, val, 0, ~0);
+	return fp;
+}
+
 static inline uint32_t div_round_up_fixed16(uint_fixed_16_16_t val,
 					    uint_fixed_16_16_t d)
 {
@@ -170,48 +178,30 @@ static inline uint32_t mul_round_up_u32_fixed16(uint32_t val,
 						uint_fixed_16_16_t mul)
 {
 	uint64_t intermediate_val;
-	uint32_t result;
 
 	intermediate_val = (uint64_t) val * mul.val;
 	intermediate_val = DIV_ROUND_UP_ULL(intermediate_val, 1 << 16);
 	WARN_ON(intermediate_val >> 32);
-	result = clamp_t(uint32_t, intermediate_val, 0, ~0);
-	return result;
+	return clamp_t(uint32_t, intermediate_val, 0, ~0);
 }
 
 static inline uint_fixed_16_16_t mul_fixed16(uint_fixed_16_16_t val,
 					     uint_fixed_16_16_t mul)
 {
 	uint64_t intermediate_val;
-	uint_fixed_16_16_t fp;
 
 	intermediate_val = (uint64_t) val.val * mul.val;
 	intermediate_val = intermediate_val >> 16;
-	WARN_ON(intermediate_val >> 32);
-	fp.val = clamp_t(uint32_t, intermediate_val, 0, ~0);
-	return fp;
+	return clamp_u64_to_fixed16(intermediate_val);
 }
 
-static inline uint_fixed_16_16_t fixed_16_16_div(uint32_t val, uint32_t d)
+static inline uint_fixed_16_16_t div_fixed16(uint32_t val, uint32_t d)
 {
-	uint_fixed_16_16_t fp, res;
-
-	fp = u32_to_fixed_16_16(val);
-	res.val = DIV_ROUND_UP(fp.val, d);
-	return res;
-}
-
-static inline uint_fixed_16_16_t fixed_16_16_div_u64(uint32_t val, uint32_t d)
-{
-	uint_fixed_16_16_t res;
 	uint64_t interm_val;
 
 	interm_val = (uint64_t)val << 16;
 	interm_val = DIV_ROUND_UP_ULL(interm_val, d);
-	WARN_ON(interm_val >> 32);
-	res.val = (uint32_t) interm_val;
-
-	return res;
+	return clamp_u64_to_fixed16(interm_val);
 }
 
 static inline uint32_t div_round_up_u32_fixed16(uint32_t val,
@@ -225,16 +215,32 @@ static inline uint32_t div_round_up_u32_fixed16(uint32_t val,
 	return clamp_t(uint32_t, interm_val, 0, ~0);
 }
 
-static inline uint_fixed_16_16_t mul_u32_fixed_16_16(uint32_t val,
+static inline uint_fixed_16_16_t mul_u32_fixed16(uint32_t val,
 						     uint_fixed_16_16_t mul)
 {
 	uint64_t intermediate_val;
-	uint_fixed_16_16_t fp;
 
 	intermediate_val = (uint64_t) val * mul.val;
-	WARN_ON(intermediate_val >> 32);
-	fp.val = (uint32_t) intermediate_val;
-	return fp;
+	return clamp_u64_to_fixed16(intermediate_val);
+}
+
+static inline uint_fixed_16_16_t add_fixed16(uint_fixed_16_16_t add1,
+					     uint_fixed_16_16_t add2)
+{
+	uint64_t interm_sum;
+
+	interm_sum = (uint64_t) add1.val + add2.val;
+	return clamp_u64_to_fixed16(interm_sum);
+}
+
+static inline uint_fixed_16_16_t add_fixed16_u32(uint_fixed_16_16_t add1,
+						 uint32_t add2)
+{
+	uint64_t interm_sum;
+	uint_fixed_16_16_t interm_add2 = u32_to_fixed16(add2);
+
+	interm_sum = (uint64_t) add1.val + interm_add2.val;
+	return clamp_u64_to_fixed16(interm_sum);
 }
 
 static inline const char *yesno(bool v)
@@ -584,8 +590,7 @@ struct drm_i915_file_private {
 	struct idr context_idr;
 
 	struct intel_rps_client {
-		struct list_head link;
-		unsigned boosts;
+		atomic_t boosts;
 	} rps;
 
 	unsigned int bsd_engine;
@@ -597,7 +602,7 @@ struct drm_i915_file_private {
  * to limit the badly behaving clients access to gpu.
  */
 #define I915_MAX_CLIENT_CONTEXT_BANS 3
-	int context_bans;
+	atomic_t context_bans;
 };
 
 /* Used by dp and fdi links */
@@ -641,6 +646,7 @@ struct intel_opregion {
 	u32 swsci_sbcb_sub_functions;
 	struct opregion_asle *asle;
 	void *rvda;
+	void *vbt_firmware;
 	const void *vbt;
 	u32 vbt_size;
 	u32 *lid_state;
@@ -710,11 +716,6 @@ struct drm_i915_display_funcs {
 	void (*fdi_link_train)(struct intel_crtc *crtc,
 			       const struct intel_crtc_state *crtc_state);
 	void (*init_clock_gating)(struct drm_i915_private *dev_priv);
-	int (*queue_flip)(struct drm_device *dev, struct drm_crtc *crtc,
-			  struct drm_framebuffer *fb,
-			  struct drm_i915_gem_object *obj,
-			  struct drm_i915_gem_request *req,
-			  uint32_t flags);
 	void (*hpd_irq_setup)(struct drm_i915_private *dev_priv);
 	/* clock updates for mode set */
 	/* cursor updates */
@@ -753,6 +754,7 @@ struct intel_csr {
 	func(has_csr); \
 	func(has_ddi); \
 	func(has_dp_mst); \
+	func(has_reset_engine); \
 	func(has_fbc); \
 	func(has_fpga_dbg); \
 	func(has_full_ppgtt); \
@@ -917,6 +919,7 @@ struct i915_gpu_state {
 		enum intel_engine_hangcheck_action hangcheck_action;
 		struct i915_address_space *vm;
 		int num_requests;
+		u32 reset_count;
 
 		/* position of active request inside the ring */
 		u32 rq_head, rq_post, rq_tail;
@@ -1056,6 +1059,11 @@ struct intel_fbc {
 	bool underrun_detected;
 	struct work_struct underrun_work;
 
+	/*
+	 * Due to the atomic rules we can't access some structures without the
+	 * appropriate locking, so we cache information here in order to avoid
+	 * these problems.
+	 */
 	struct intel_fbc_state_cache {
 		struct i915_vma *vma;
 
@@ -1077,6 +1085,13 @@ struct intel_fbc {
 		} fb;
 	} state_cache;
 
+	/*
+	 * This structure contains everything that's relevant to program the
+	 * hardware registers. When we want to figure out if we need to disable
+	 * and re-enable FBC for a new configuration we just check if there's
+	 * something different in the struct. The genx_fbc_activate functions
+	 * are supposed to read from it in order to program the registers.
+	 */
 	struct intel_fbc_reg_params {
 		struct i915_vma *vma;
 
@@ -1149,11 +1164,11 @@ struct i915_psr {
 enum intel_pch {
 	PCH_NONE = 0,	/* No PCH present */
 	PCH_IBX,	/* Ibexpeak PCH */
-	PCH_CPT,	/* Cougarpoint PCH */
-	PCH_LPT,	/* Lynxpoint PCH */
+	PCH_CPT,	/* Cougarpoint/Pantherpoint PCH */
+	PCH_LPT,	/* Lynxpoint/Wildcatpoint PCH */
 	PCH_SPT,        /* Sunrisepoint PCH */
-	PCH_KBP,        /* Kabypoint PCH */
-	PCH_CNP,        /* Cannonpoint PCH */
+	PCH_KBP,        /* Kaby Lake PCH */
+	PCH_CNP,        /* Cannon Lake PCH */
 	PCH_NOP,
 };
 
@@ -1166,6 +1181,7 @@ enum intel_sbi_destination {
 #define QUIRK_INVERT_BRIGHTNESS (1<<2)
 #define QUIRK_BACKLIGHT_PRESENT (1<<3)
 #define QUIRK_PIN_SWIZZLED_PAGES (1<<5)
+#define QUIRK_INCREASE_T12_DELAY (1<<6)
 
 struct intel_fbdev;
 struct intel_fbc_work;
@@ -1301,13 +1317,10 @@ struct intel_gen6_power_mgmt {
 	int last_adj;
 	enum { LOW_POWER, BETWEEN, HIGH_POWER } power;
 
-	spinlock_t client_lock;
-	struct list_head clients;
-	bool client_boost;
-
 	bool enabled;
 	struct delayed_work autoenable_work;
-	unsigned boosts;
+	atomic_t num_waiters;
+	atomic_t boosts;
 
 	/* manual wa residency calculations */
 	struct intel_rps_ei ei;
@@ -1383,12 +1396,23 @@ struct i915_power_well {
 	bool hw_enabled;
 	u64 domains;
 	/* unique identifier for this power well */
-	unsigned long id;
+	enum i915_power_well_id id;
 	/*
 	 * Arbitraty data associated with this power well. Platform and power
 	 * well specific.
 	 */
-	unsigned long data;
+	union {
+		struct {
+			enum dpio_phy phy;
+		} bxt;
+		struct {
+			/* Mask of pipes whose IRQ logic is backed by the pw */
+			u8 irq_pipe_mask;
+			/* The pw is backing the VGA functionality */
+			bool has_vga:1;
+			bool has_fuses:1;
+		} hsw;
+	};
 	const struct i915_power_well_ops *ops;
 };
 
@@ -1505,6 +1529,8 @@ struct i915_gpu_error {
 	/* Protected by the above dev->gpu_error.lock. */
 	struct i915_gpu_state *first_error;
 
+	atomic_t pending_fb_pin;
+
 	unsigned long missed_irq_rings;
 
 	/**
@@ -1550,6 +1576,12 @@ struct i915_gpu_error {
 	 * inspect the bit and do the reset directly, otherwise the worker
 	 * waits for the struct_mutex.
 	 *
+	 * #I915_RESET_ENGINE[num_engines] - Since the driver doesn't need to
+	 * acquire the struct_mutex to reset an engine, we need an explicit
+	 * flag to prevent two concurrent reset attempts in the same engine.
+	 * As the number of engines continues to grow, allocate the flags from
+	 * the most significant bits.
+	 *
 	 * #I915_WEDGED - If reset fails and we can no longer use the GPU,
 	 * we set the #I915_WEDGED bit. Prior to command submission, e.g.
 	 * i915_gem_request_alloc(), this bit is checked and the sequence
@@ -1558,7 +1590,12 @@ struct i915_gpu_error {
 	unsigned long flags;
 #define I915_RESET_BACKOFF	0
 #define I915_RESET_HANDOFF	1
+#define I915_RESET_MODESET	2
 #define I915_WEDGED		(BITS_PER_LONG - 1)
+#define I915_RESET_ENGINE	(I915_WEDGED - I915_NUM_ENGINES)
+
+	/** Number of times an engine has been reset */
+	u32 reset_engine_count[I915_NUM_ENGINES];
 
 	/**
 	 * Waitqueue to signal when a hang is detected. Used to for waiters
@@ -1869,6 +1906,7 @@ struct i915_workarounds {
 
 struct i915_virtual_gpu {
 	bool active;
+	u32 caps;
 };
 
 /* used in computing the new watermarks state */
@@ -1888,6 +1926,24 @@ struct i915_oa_reg {
 	u32 value;
 };
 
+struct i915_oa_config {
+	char uuid[UUID_STRING_LEN + 1];
+	int id;
+
+	const struct i915_oa_reg *mux_regs;
+	u32 mux_regs_len;
+	const struct i915_oa_reg *b_counter_regs;
+	u32 b_counter_regs_len;
+	const struct i915_oa_reg *flex_regs;
+	u32 flex_regs_len;
+
+	struct attribute_group sysfs_metric;
+	struct attribute *attrs[2];
+	struct device_attribute sysfs_metric_id;
+
+	atomic_t ref_count;
+};
+
 struct i915_perf_stream;
 
 /**
@@ -2000,6 +2056,11 @@ struct i915_perf_stream {
 	 * type of configured stream.
 	 */
 	const struct i915_perf_stream_ops *ops;
+
+	/**
+	 * @oa_config: The OA configuration used by the stream.
+	 */
+	struct i915_oa_config *oa_config;
 };
 
 /**
@@ -2007,6 +2068,25 @@ struct i915_perf_stream {
  */
 struct i915_oa_ops {
 	/**
+	 * @is_valid_b_counter_reg: Validates register's address for
+	 * programming boolean counters for a particular platform.
+	 */
+	bool (*is_valid_b_counter_reg)(struct drm_i915_private *dev_priv,
+				       u32 addr);
+
+	/**
+	 * @is_valid_mux_reg: Validates register's address for programming mux
+	 * for a particular platform.
+	 */
+	bool (*is_valid_mux_reg)(struct drm_i915_private *dev_priv, u32 addr);
+
+	/**
+	 * @is_valid_flex_reg: Validates register's address for programming
+	 * flex EU filtering for a particular platform.
+	 */
+	bool (*is_valid_flex_reg)(struct drm_i915_private *dev_priv, u32 addr);
+
+	/**
 	 * @init_oa_buffer: Resets the head and tail pointers of the
 	 * circular buffer for periodic OA reports.
 	 *
@@ -2024,20 +2104,13 @@ struct i915_oa_ops {
 	void (*init_oa_buffer)(struct drm_i915_private *dev_priv);
 
 	/**
-	 * @select_metric_set: The auto generated code that checks whether a
-	 * requested OA config is applicable to the system and if so sets up
-	 * the mux, oa and flex eu register config pointers according to the
-	 * current dev_priv->perf.oa.metrics_set.
-	 */
-	int (*select_metric_set)(struct drm_i915_private *dev_priv);
-
-	/**
 	 * @enable_metric_set: Selects and applies any MUX configuration to set
 	 * up the Boolean and Custom (B/C) counters that are part of the
 	 * counter reports being sampled. May apply system constraints such as
 	 * disabling EU clock gating as required.
 	 */
-	int (*enable_metric_set)(struct drm_i915_private *dev_priv);
+	int (*enable_metric_set)(struct drm_i915_private *dev_priv,
+				 const struct i915_oa_config *oa_config);
 
 	/**
 	 * @disable_metric_set: Remove system constraints associated with using
@@ -2083,6 +2156,7 @@ struct drm_i915_private {
 
 	struct kmem_cache *objects;
 	struct kmem_cache *vmas;
+	struct kmem_cache *luts;
 	struct kmem_cache *requests;
 	struct kmem_cache *dependencies;
 	struct kmem_cache *priorities;
@@ -2133,9 +2207,6 @@ struct drm_i915_private {
 	/* protects the irq masks */
 	spinlock_t irq_lock;
 
-	/* protects the mmio flip data */
-	spinlock_t mmio_flip_lock;
-
 	bool display_irqs_enabled;
 
 	/* To control wakeup latency, e.g. for irq-driven dp aux transfers. */
@@ -2236,18 +2307,10 @@ struct drm_i915_private {
 	DECLARE_HASHTABLE(mm_structs, 7);
 	struct mutex mm_lock;
 
-	/* The hw wants to have a stable context identifier for the lifetime
-	 * of the context (for OA, PASID, faults, etc). This is limited
-	 * in execlists to 21 bits.
-	 */
-	struct ida context_hw_ida;
-#define MAX_CONTEXT_HW_ID (1<<21) /* exclusive */
-
 	/* Kernel Modesetting */
 
 	struct intel_crtc *plane_to_crtc_mapping[I915_MAX_PIPES];
 	struct intel_crtc *pipe_to_crtc_mapping[I915_MAX_PIPES];
-	wait_queue_head_t pending_flip_queue;
 
 #ifdef CONFIG_DEBUG_FS
 	struct intel_pipe_crc pipe_crc[I915_MAX_PIPES];
@@ -2303,11 +2366,9 @@ struct drm_i915_private {
 
 	struct drm_i915_gem_object *vlv_pctx;
 
-#ifdef CONFIG_DRM_FBDEV_EMULATION
 	/* list of fbdev register on this device */
 	struct intel_fbdev *fbdev;
 	struct work_struct fbdev_suspend_work;
-#endif
 
 	struct drm_property *broadcast_rgb_property;
 	struct drm_property *force_audio_property;
@@ -2321,7 +2382,18 @@ struct drm_i915_private {
 	 */
 	struct mutex av_mutex;
 
-	struct list_head context_list;
+	struct {
+		struct list_head list;
+		struct llist_head free_list;
+		struct work_struct free_work;
+
+		/* The hw wants to have a stable context identifier for the
+		 * lifetime of the context (for OA, PASID, faults, etc).
+		 * This is limited in execlists to 21 bits.
+		 */
+		struct ida hw_ida;
+#define MAX_CONTEXT_HW_ID (1<<21) /* exclusive */
+	} contexts;
 
 	u32 fdi_rx_config;
 
@@ -2399,10 +2471,32 @@ struct drm_i915_private {
 		struct kobject *metrics_kobj;
 		struct ctl_table_header *sysctl_header;
 
+		/*
+		 * Lock associated with adding/modifying/removing OA configs
+		 * in dev_priv->perf.metrics_idr.
+		 */
+		struct mutex metrics_lock;
+
+		/*
+		 * List of dynamic configurations, you need to hold
+		 * dev_priv->perf.metrics_lock to access it.
+		 */
+		struct idr metrics_idr;
+
+		/*
+		 * Lock associated with anything below within this structure
+		 * except exclusive_stream.
+		 */
 		struct mutex lock;
 		struct list_head streams;
 
 		struct {
+			/*
+			 * The stream currently using the OA unit. If accessed
+			 * outside a syscall associated to its file
+			 * descriptor, you need to hold
+			 * dev_priv->drm.struct_mutex.
+			 */
 			struct i915_perf_stream *exclusive_stream;
 
 			u32 specific_ctx_id;
@@ -2421,16 +2515,7 @@ struct drm_i915_private {
 			int period_exponent;
 			int timestamp_frequency;
 
-			int metrics_set;
-
-			const struct i915_oa_reg *mux_regs[6];
-			int mux_regs_lens[6];
-			int n_mux_configs;
-
-			const struct i915_oa_reg *b_counter_regs;
-			int b_counter_regs_len;
-			const struct i915_oa_reg *flex_regs;
-			int flex_regs_len;
+			struct i915_oa_config test_config;
 
 			struct {
 				struct i915_vma *vma;
@@ -2517,7 +2602,6 @@ struct drm_i915_private {
 
 			struct i915_oa_ops ops;
 			const struct i915_oa_format *oa_formats;
-			int n_builtin_sets;
 		} oa;
 	} perf;
 
@@ -2996,16 +3080,17 @@ intel_info(const struct drm_i915_private *dev_priv)
 
 #define HAS_POOLED_EU(dev_priv)	((dev_priv)->info.has_pooled_eu)
 
-#define INTEL_PCH_DEVICE_ID_MASK		0xff00
-#define INTEL_PCH_DEVICE_ID_MASK_EXT		0xff80
+#define INTEL_PCH_DEVICE_ID_MASK		0xff80
 #define INTEL_PCH_IBX_DEVICE_ID_TYPE		0x3b00
 #define INTEL_PCH_CPT_DEVICE_ID_TYPE		0x1c00
 #define INTEL_PCH_PPT_DEVICE_ID_TYPE		0x1e00
 #define INTEL_PCH_LPT_DEVICE_ID_TYPE		0x8c00
 #define INTEL_PCH_LPT_LP_DEVICE_ID_TYPE		0x9c00
+#define INTEL_PCH_WPT_DEVICE_ID_TYPE		0x8c80
+#define INTEL_PCH_WPT_LP_DEVICE_ID_TYPE		0x9c80
 #define INTEL_PCH_SPT_DEVICE_ID_TYPE		0xA100
 #define INTEL_PCH_SPT_LP_DEVICE_ID_TYPE		0x9D00
-#define INTEL_PCH_KBP_DEVICE_ID_TYPE		0xA200
+#define INTEL_PCH_KBP_DEVICE_ID_TYPE		0xA280
 #define INTEL_PCH_CNP_DEVICE_ID_TYPE		0xA300
 #define INTEL_PCH_CNP_LP_DEVICE_ID_TYPE		0x9D80
 #define INTEL_PCH_P2X_DEVICE_ID_TYPE		0x7100
@@ -3020,9 +3105,11 @@ intel_info(const struct drm_i915_private *dev_priv)
 #define HAS_PCH_SPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_SPT)
 #define HAS_PCH_LPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_LPT)
 #define HAS_PCH_LPT_LP(dev_priv) \
-	((dev_priv)->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE)
+	((dev_priv)->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE || \
+	 (dev_priv)->pch_id == INTEL_PCH_WPT_LP_DEVICE_ID_TYPE)
 #define HAS_PCH_LPT_H(dev_priv) \
-	((dev_priv)->pch_id == INTEL_PCH_LPT_DEVICE_ID_TYPE)
+	((dev_priv)->pch_id == INTEL_PCH_LPT_DEVICE_ID_TYPE || \
+	 (dev_priv)->pch_id == INTEL_PCH_WPT_DEVICE_ID_TYPE)
 #define HAS_PCH_CPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_CPT)
 #define HAS_PCH_IBX(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_IBX)
 #define HAS_PCH_NOP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_NOP)
@@ -3088,7 +3175,13 @@ extern int i915_driver_load(struct pci_dev *pdev,
 extern void i915_driver_unload(struct drm_device *dev);
 extern int intel_gpu_reset(struct drm_i915_private *dev_priv, u32 engine_mask);
 extern bool intel_has_gpu_reset(struct drm_i915_private *dev_priv);
-extern void i915_reset(struct drm_i915_private *dev_priv);
+
+#define I915_RESET_QUIET BIT(0)
+extern void i915_reset(struct drm_i915_private *i915, unsigned int flags);
+extern int i915_reset_engine(struct intel_engine_cs *engine,
+			     unsigned int flags);
+
+extern bool intel_has_reset_engine(struct drm_i915_private *dev_priv);
 extern int intel_guc_reset(struct drm_i915_private *dev_priv);
 extern void intel_engine_init_hangcheck(struct intel_engine_cs *engine);
 extern void intel_hangcheck_init(struct drm_i915_private *dev_priv);
@@ -3107,7 +3200,8 @@ void intel_hpd_irq_handler(struct drm_i915_private *dev_priv,
 void intel_hpd_init(struct drm_i915_private *dev_priv);
 void intel_hpd_init_work(struct drm_i915_private *dev_priv);
 void intel_hpd_cancel_work(struct drm_i915_private *dev_priv);
-bool intel_hpd_pin_to_port(enum hpd_pin pin, enum port *port);
+enum port intel_hpd_pin_to_port(enum hpd_pin pin);
+enum hpd_pin intel_hpd_pin(enum port port);
 bool intel_hpd_disable(struct drm_i915_private *dev_priv, enum hpd_pin pin);
 void intel_hpd_enable(struct drm_i915_private *dev_priv, enum hpd_pin pin);
 
@@ -3276,6 +3370,26 @@ static inline void i915_gem_drain_freed_objects(struct drm_i915_private *i915)
 	} while (flush_work(&i915->mm.free_work));
 }
 
+static inline void i915_gem_drain_workqueue(struct drm_i915_private *i915)
+{
+	/*
+	 * Similar to objects above (see i915_gem_drain_freed-objects), in
+	 * general we have workers that are armed by RCU and then rearm
+	 * themselves in their callbacks. To be paranoid, we need to
+	 * drain the workqueue a second time after waiting for the RCU
+	 * grace period so that we catch work queued via RCU from the first
+	 * pass. As neither drain_workqueue() nor flush_workqueue() report
+	 * a result, we make an assumption that we only don't require more
+	 * than 2 passes to catch all recursive RCU delayed work.
+	 *
+	 */
+	int pass = 2;
+	do {
+		rcu_barrier();
+		drain_workqueue(i915->wq);
+	} while (--pass);
+}
+
 struct i915_vma * __must_check
 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
 			 const struct i915_ggtt_view *view,
@@ -3365,6 +3479,9 @@ void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj);
 enum i915_map_type {
 	I915_MAP_WB = 0,
 	I915_MAP_WC,
+#define I915_MAP_OVERRIDE BIT(31)
+	I915_MAP_FORCE_WB = I915_MAP_WB | I915_MAP_OVERRIDE,
+	I915_MAP_FORCE_WC = I915_MAP_WC | I915_MAP_OVERRIDE,
 };
 
 /**
@@ -3461,11 +3578,22 @@ static inline u32 i915_reset_count(struct i915_gpu_error *error)
 	return READ_ONCE(error->reset_count);
 }
 
+static inline u32 i915_reset_engine_count(struct i915_gpu_error *error,
+					  struct intel_engine_cs *engine)
+{
+	return READ_ONCE(error->reset_engine_count[engine->id]);
+}
+
+struct drm_i915_gem_request *
+i915_gem_reset_prepare_engine(struct intel_engine_cs *engine);
 int i915_gem_reset_prepare(struct drm_i915_private *dev_priv);
 void i915_gem_reset(struct drm_i915_private *dev_priv);
+void i915_gem_reset_finish_engine(struct intel_engine_cs *engine);
 void i915_gem_reset_finish(struct drm_i915_private *dev_priv);
 void i915_gem_set_wedged(struct drm_i915_private *dev_priv);
 bool i915_gem_unset_wedged(struct drm_i915_private *dev_priv);
+void i915_gem_reset_engine(struct intel_engine_cs *engine,
+			   struct drm_i915_gem_request *request);
 
 void i915_gem_init_mmio(struct drm_i915_private *i915);
 int __must_check i915_gem_init(struct drm_i915_private *dev_priv);
@@ -3499,7 +3627,7 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 void i915_gem_object_unpin_from_display_plane(struct i915_vma *vma);
 int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
 				int align);
-int i915_gem_open(struct drm_device *dev, struct drm_file *file);
+int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file);
 void i915_gem_release(struct drm_device *dev, struct drm_file *file);
 
 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
@@ -3531,38 +3659,23 @@ void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj,
 					 struct sg_table *pages);
 
 static inline struct i915_gem_context *
-i915_gem_context_lookup(struct drm_i915_file_private *file_priv, u32 id)
+__i915_gem_context_lookup_rcu(struct drm_i915_file_private *file_priv, u32 id)
 {
-	struct i915_gem_context *ctx;
-
-	lockdep_assert_held(&file_priv->dev_priv->drm.struct_mutex);
-
-	ctx = idr_find(&file_priv->context_idr, id);
-	if (!ctx)
-		return ERR_PTR(-ENOENT);
-
-	return ctx;
+	return idr_find(&file_priv->context_idr, id);
 }
 
 static inline struct i915_gem_context *
-i915_gem_context_get(struct i915_gem_context *ctx)
-{
-	kref_get(&ctx->ref);
-	return ctx;
-}
-
-static inline void i915_gem_context_put(struct i915_gem_context *ctx)
+i915_gem_context_lookup(struct drm_i915_file_private *file_priv, u32 id)
 {
-	lockdep_assert_held(&ctx->i915->drm.struct_mutex);
-	kref_put(&ctx->ref, i915_gem_context_free);
-}
+	struct i915_gem_context *ctx;
 
-static inline void i915_gem_context_put_unlocked(struct i915_gem_context *ctx)
-{
-	struct mutex *lock = &ctx->i915->drm.struct_mutex;
+	rcu_read_lock();
+	ctx = __i915_gem_context_lookup_rcu(file_priv, id);
+	if (ctx && !kref_get_unless_zero(&ctx->ref))
+		ctx = NULL;
+	rcu_read_unlock();
 
-	if (kref_put_mutex(&ctx->ref, i915_gem_context_free, lock))
-		mutex_unlock(lock);
+	return ctx;
 }
 
 static inline struct intel_timeline *
@@ -3577,6 +3690,10 @@ i915_gem_context_lookup_timeline(struct i915_gem_context *ctx,
 
 int i915_perf_open_ioctl(struct drm_device *dev, void *data,
 			 struct drm_file *file);
+int i915_perf_add_config_ioctl(struct drm_device *dev, void *data,
+			       struct drm_file *file);
+int i915_perf_remove_config_ioctl(struct drm_device *dev, void *data,
+				  struct drm_file *file);
 void i915_oa_init_reg_state(struct intel_engine_cs *engine,
 			    struct i915_gem_context *ctx,
 			    uint32_t *reg_state);
@@ -3628,6 +3745,7 @@ i915_gem_object_create_internal(struct drm_i915_private *dev_priv,
 /* i915_gem_shrinker.c */
 unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv,
 			      unsigned long target,
+			      unsigned long *nr_scanned,
 			      unsigned flags);
 #define I915_SHRINK_PURGEABLE 0x1
 #define I915_SHRINK_UNBOUND 0x2
@@ -4064,6 +4182,11 @@ static inline unsigned long msecs_to_jiffies_timeout(const unsigned int m)
 
 static inline unsigned long nsecs_to_jiffies_timeout(const u64 n)
 {
+	/* nsecs_to_jiffies64() does not guard against overflow */
+	if (NSEC_PER_SEC % HZ &&
+	    div_u64(n, NSEC_PER_SEC) >= MAX_JIFFY_OFFSET / HZ)
+		return MAX_JIFFY_OFFSET;
+
         return min_t(u64, MAX_JIFFY_OFFSET, nsecs_to_jiffies64(n) + 1);
 }
 
@@ -4210,10 +4333,11 @@ int remap_io_mapping(struct vm_area_struct *vma,
 		     unsigned long addr, unsigned long pfn, unsigned long size,
 		     struct io_mapping *iomap);
 
-static inline bool i915_gem_object_is_coherent(struct drm_i915_gem_object *obj)
+static inline bool
+intel_engine_can_store_dword(struct intel_engine_cs *engine)
 {
-	return (obj->cache_level != I915_CACHE_NONE ||
-		HAS_LLC(to_i915(obj->base.dev)));
+	return __intel_engine_can_store_dword(INTEL_GEN(engine->i915),
+					      engine->class);
 }
 
 #endif
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 969bac8404f1..19404c96eeb1 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -52,7 +52,7 @@ static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
 	if (obj->cache_dirty)
 		return false;
 
-	if (!obj->cache_coherent)
+	if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE))
 		return true;
 
 	return obj->pin_display;
@@ -253,7 +253,7 @@ __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
 
 	if (needs_clflush &&
 	    (obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0 &&
-	    !obj->cache_coherent)
+	    !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))
 		drm_clflush_sg(pages);
 
 	__start_cpu_write(obj);
@@ -388,7 +388,7 @@ i915_gem_object_wait_fence(struct dma_fence *fence,
 	 */
 	if (rps) {
 		if (INTEL_GEN(rq->i915) >= 6)
-			gen6_rps_boost(rq->i915, rps, rq->emitted_jiffies);
+			gen6_rps_boost(rq, rps);
 		else
 			rps = NULL;
 	}
@@ -399,22 +399,6 @@ out:
 	if (flags & I915_WAIT_LOCKED && i915_gem_request_completed(rq))
 		i915_gem_request_retire_upto(rq);
 
-	if (rps && i915_gem_request_global_seqno(rq) == intel_engine_last_submit(rq->engine)) {
-		/* The GPU is now idle and this client has stalled.
-		 * Since no other client has submitted a request in the
-		 * meantime, assume that this client is the only one
-		 * supplying work to the GPU but is unable to keep that
-		 * work supplied because it is waiting. Since the GPU is
-		 * then never kept fully busy, RPS autoclocking will
-		 * keep the clocks relatively low, causing further delays.
-		 * Compensate by giving the synchronous client credit for
-		 * a waitboost next time.
-		 */
-		spin_lock(&rq->i915->rps.client_lock);
-		list_del_init(&rps->link);
-		spin_unlock(&rq->i915->rps.client_lock);
-	}
-
 	return timeout;
 }
 
@@ -577,46 +561,6 @@ static struct intel_rps_client *to_rps_client(struct drm_file *file)
 	return &fpriv->rps;
 }
 
-int
-i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
-			    int align)
-{
-	int ret;
-
-	if (align > obj->base.size)
-		return -EINVAL;
-
-	if (obj->ops == &i915_gem_phys_ops)
-		return 0;
-
-	if (obj->mm.madv != I915_MADV_WILLNEED)
-		return -EFAULT;
-
-	if (obj->base.filp == NULL)
-		return -EINVAL;
-
-	ret = i915_gem_object_unbind(obj);
-	if (ret)
-		return ret;
-
-	__i915_gem_object_put_pages(obj, I915_MM_NORMAL);
-	if (obj->mm.pages)
-		return -EBUSY;
-
-	GEM_BUG_ON(obj->ops != &i915_gem_object_ops);
-	obj->ops = &i915_gem_phys_ops;
-
-	ret = i915_gem_object_pin_pages(obj);
-	if (ret)
-		goto err_xfer;
-
-	return 0;
-
-err_xfer:
-	obj->ops = &i915_gem_object_ops;
-	return ret;
-}
-
 static int
 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
 		     struct drm_i915_gem_pwrite *args,
@@ -751,12 +695,11 @@ flush_write_domain(struct drm_i915_gem_object *obj, unsigned int flush_domains)
 	switch (obj->base.write_domain) {
 	case I915_GEM_DOMAIN_GTT:
 		if (INTEL_GEN(dev_priv) >= 6 && !HAS_LLC(dev_priv)) {
-			if (intel_runtime_pm_get_if_in_use(dev_priv)) {
-				spin_lock_irq(&dev_priv->uncore.lock);
-				POSTING_READ_FW(RING_ACTHD(dev_priv->engine[RCS]->mmio_base));
-				spin_unlock_irq(&dev_priv->uncore.lock);
-				intel_runtime_pm_put(dev_priv);
-			}
+			intel_runtime_pm_get(dev_priv);
+			spin_lock_irq(&dev_priv->uncore.lock);
+			POSTING_READ_FW(RING_ACTHD(dev_priv->engine[RCS]->mmio_base));
+			spin_unlock_irq(&dev_priv->uncore.lock);
+			intel_runtime_pm_put(dev_priv);
 		}
 
 		intel_fb_obj_flush(obj,
@@ -856,7 +799,8 @@ int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
 	if (ret)
 		return ret;
 
-	if (obj->cache_coherent || !static_cpu_has(X86_FEATURE_CLFLUSH)) {
+	if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ ||
+	    !static_cpu_has(X86_FEATURE_CLFLUSH)) {
 		ret = i915_gem_object_set_to_cpu_domain(obj, false);
 		if (ret)
 			goto err_unpin;
@@ -908,7 +852,8 @@ int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
 	if (ret)
 		return ret;
 
-	if (obj->cache_coherent || !static_cpu_has(X86_FEATURE_CLFLUSH)) {
+	if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE ||
+	    !static_cpu_has(X86_FEATURE_CLFLUSH)) {
 		ret = i915_gem_object_set_to_cpu_domain(obj, true);
 		if (ret)
 			goto err_unpin;
@@ -2267,7 +2212,7 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj,
 static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
 {
 	struct radix_tree_iter iter;
-	void **slot;
+	void __rcu **slot;
 
 	radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0)
 		radix_tree_delete(&obj->mm.get_page.radix, iter.index);
@@ -2408,7 +2353,7 @@ rebuild_st:
 				goto err_sg;
 			}
 
-			i915_gem_shrink(dev_priv, 2 * page_count, *s++);
+			i915_gem_shrink(dev_priv, 2 * page_count, NULL, *s++);
 			cond_resched();
 
 			/* We've tried hard to allocate the memory by reaping
@@ -2595,7 +2540,7 @@ static void *i915_gem_object_map(const struct drm_i915_gem_object *obj,
 
 	if (n_pages > ARRAY_SIZE(stack_pages)) {
 		/* Too big for stack -- allocate temporary array instead */
-		pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_TEMPORARY);
+		pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL);
 		if (!pages)
 			return NULL;
 	}
@@ -2607,6 +2552,9 @@ static void *i915_gem_object_map(const struct drm_i915_gem_object *obj,
 	GEM_BUG_ON(i != n_pages);
 
 	switch (type) {
+	default:
+		MISSING_CASE(type);
+		/* fallthrough to use PAGE_KERNEL anyway */
 	case I915_MAP_WB:
 		pgprot = PAGE_KERNEL;
 		break;
@@ -2637,7 +2585,9 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
 	if (ret)
 		return ERR_PTR(ret);
 
-	pinned = true;
+	pinned = !(type & I915_MAP_OVERRIDE);
+	type &= ~I915_MAP_OVERRIDE;
+
 	if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) {
 		if (unlikely(IS_ERR_OR_NULL(obj->mm.pages))) {
 			ret = ____i915_gem_object_get_pages(obj);
@@ -2756,34 +2706,38 @@ i915_gem_object_pwrite_gtt(struct drm_i915_gem_object *obj,
 	return 0;
 }
 
-static bool ban_context(const struct i915_gem_context *ctx)
+static bool ban_context(const struct i915_gem_context *ctx,
+			unsigned int score)
 {
 	return (i915_gem_context_is_bannable(ctx) &&
-		ctx->ban_score >= CONTEXT_SCORE_BAN_THRESHOLD);
+		score >= CONTEXT_SCORE_BAN_THRESHOLD);
 }
 
 static void i915_gem_context_mark_guilty(struct i915_gem_context *ctx)
 {
-	ctx->guilty_count++;
-	ctx->ban_score += CONTEXT_SCORE_GUILTY;
-	if (ban_context(ctx))
-		i915_gem_context_set_banned(ctx);
+	unsigned int score;
+	bool banned;
 
-	DRM_DEBUG_DRIVER("context %s marked guilty (score %d) banned? %s\n",
-			 ctx->name, ctx->ban_score,
-			 yesno(i915_gem_context_is_banned(ctx)));
+	atomic_inc(&ctx->guilty_count);
 
-	if (!i915_gem_context_is_banned(ctx) || IS_ERR_OR_NULL(ctx->file_priv))
+	score = atomic_add_return(CONTEXT_SCORE_GUILTY, &ctx->ban_score);
+	banned = ban_context(ctx, score);
+	DRM_DEBUG_DRIVER("context %s marked guilty (score %d) banned? %s\n",
+			 ctx->name, score, yesno(banned));
+	if (!banned)
 		return;
 
-	ctx->file_priv->context_bans++;
-	DRM_DEBUG_DRIVER("client %s has had %d context banned\n",
-			 ctx->name, ctx->file_priv->context_bans);
+	i915_gem_context_set_banned(ctx);
+	if (!IS_ERR_OR_NULL(ctx->file_priv)) {
+		atomic_inc(&ctx->file_priv->context_bans);
+		DRM_DEBUG_DRIVER("client %s has had %d context banned\n",
+				 ctx->name, atomic_read(&ctx->file_priv->context_bans));
+	}
 }
 
 static void i915_gem_context_mark_innocent(struct i915_gem_context *ctx)
 {
-	ctx->active_count++;
+	atomic_inc(&ctx->active_count);
 }
 
 struct drm_i915_gem_request *
@@ -2832,46 +2786,62 @@ static bool engine_stalled(struct intel_engine_cs *engine)
 	return true;
 }
 
+/*
+ * Ensure irq handler finishes, and not run again.
+ * Also return the active request so that we only search for it once.
+ */
+struct drm_i915_gem_request *
+i915_gem_reset_prepare_engine(struct intel_engine_cs *engine)
+{
+	struct drm_i915_gem_request *request = NULL;
+
+	/* Prevent the signaler thread from updating the request
+	 * state (by calling dma_fence_signal) as we are processing
+	 * the reset. The write from the GPU of the seqno is
+	 * asynchronous and the signaler thread may see a different
+	 * value to us and declare the request complete, even though
+	 * the reset routine have picked that request as the active
+	 * (incomplete) request. This conflict is not handled
+	 * gracefully!
+	 */
+	kthread_park(engine->breadcrumbs.signaler);
+
+	/* Prevent request submission to the hardware until we have
+	 * completed the reset in i915_gem_reset_finish(). If a request
+	 * is completed by one engine, it may then queue a request
+	 * to a second via its engine->irq_tasklet *just* as we are
+	 * calling engine->init_hw() and also writing the ELSP.
+	 * Turning off the engine->irq_tasklet until the reset is over
+	 * prevents the race.
+	 */
+	tasklet_kill(&engine->irq_tasklet);
+	tasklet_disable(&engine->irq_tasklet);
+
+	if (engine->irq_seqno_barrier)
+		engine->irq_seqno_barrier(engine);
+
+	request = i915_gem_find_active_request(engine);
+	if (request && request->fence.error == -EIO)
+		request = ERR_PTR(-EIO); /* Previous reset failed! */
+
+	return request;
+}
+
 int i915_gem_reset_prepare(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
+	struct drm_i915_gem_request *request;
 	enum intel_engine_id id;
 	int err = 0;
 
-	/* Ensure irq handler finishes, and not run again. */
 	for_each_engine(engine, dev_priv, id) {
-		struct drm_i915_gem_request *request;
-
-		/* Prevent the signaler thread from updating the request
-		 * state (by calling dma_fence_signal) as we are processing
-		 * the reset. The write from the GPU of the seqno is
-		 * asynchronous and the signaler thread may see a different
-		 * value to us and declare the request complete, even though
-		 * the reset routine have picked that request as the active
-		 * (incomplete) request. This conflict is not handled
-		 * gracefully!
-		 */
-		kthread_park(engine->breadcrumbs.signaler);
-
-		/* Prevent request submission to the hardware until we have
-		 * completed the reset in i915_gem_reset_finish(). If a request
-		 * is completed by one engine, it may then queue a request
-		 * to a second via its engine->irq_tasklet *just* as we are
-		 * calling engine->init_hw() and also writing the ELSP.
-		 * Turning off the engine->irq_tasklet until the reset is over
-		 * prevents the race.
-		 */
-		tasklet_kill(&engine->irq_tasklet);
-		tasklet_disable(&engine->irq_tasklet);
-
-		if (engine->irq_seqno_barrier)
-			engine->irq_seqno_barrier(engine);
-
-		if (engine_stalled(engine)) {
-			request = i915_gem_find_active_request(engine);
-			if (request && request->fence.error == -EIO)
-				err = -EIO; /* Previous reset failed! */
+		request = i915_gem_reset_prepare_engine(engine);
+		if (IS_ERR(request)) {
+			err = PTR_ERR(request);
+			continue;
 		}
+
+		engine->hangcheck.active_request = request;
 	}
 
 	i915_gem_revoke_fences(dev_priv);
@@ -2921,12 +2891,11 @@ static void engine_skip_context(struct drm_i915_gem_request *request)
 	spin_unlock_irqrestore(&engine->timeline->lock, flags);
 }
 
-/* Returns true if the request was guilty of hang */
-static bool i915_gem_reset_request(struct drm_i915_gem_request *request)
+/* Returns the request if it was guilty of the hang */
+static struct drm_i915_gem_request *
+i915_gem_reset_request(struct intel_engine_cs *engine,
+		       struct drm_i915_gem_request *request)
 {
-	/* Read once and return the resolution */
-	const bool guilty = engine_stalled(request->engine);
-
 	/* The guilty request will get skipped on a hung engine.
 	 *
 	 * Users of client default contexts do not rely on logical
@@ -2948,29 +2917,47 @@ static bool i915_gem_reset_request(struct drm_i915_gem_request *request)
 	 * subsequent hangs.
 	 */
 
-	if (guilty) {
+	if (engine_stalled(engine)) {
 		i915_gem_context_mark_guilty(request->ctx);
 		skip_request(request);
+
+		/* If this context is now banned, skip all pending requests. */
+		if (i915_gem_context_is_banned(request->ctx))
+			engine_skip_context(request);
 	} else {
-		i915_gem_context_mark_innocent(request->ctx);
-		dma_fence_set_error(&request->fence, -EAGAIN);
+		/*
+		 * Since this is not the hung engine, it may have advanced
+		 * since the hang declaration. Double check by refinding
+		 * the active request at the time of the reset.
+		 */
+		request = i915_gem_find_active_request(engine);
+		if (request) {
+			i915_gem_context_mark_innocent(request->ctx);
+			dma_fence_set_error(&request->fence, -EAGAIN);
+
+			/* Rewind the engine to replay the incomplete rq */
+			spin_lock_irq(&engine->timeline->lock);
+			request = list_prev_entry(request, link);
+			if (&request->link == &engine->timeline->requests)
+				request = NULL;
+			spin_unlock_irq(&engine->timeline->lock);
+		}
 	}
 
-	return guilty;
+	return request;
 }
 
-static void i915_gem_reset_engine(struct intel_engine_cs *engine)
+void i915_gem_reset_engine(struct intel_engine_cs *engine,
+			   struct drm_i915_gem_request *request)
 {
-	struct drm_i915_gem_request *request;
+	engine->irq_posted = 0;
 
-	request = i915_gem_find_active_request(engine);
-	if (request && i915_gem_reset_request(request)) {
+	if (request)
+		request = i915_gem_reset_request(engine, request);
+
+	if (request) {
 		DRM_DEBUG_DRIVER("resetting %s to restart from tail of request 0x%x\n",
 				 engine->name, request->global_seqno);
-
-		/* If this context is now banned, skip all pending requests. */
-		if (i915_gem_context_is_banned(request->ctx))
-			engine_skip_context(request);
 	}
 
 	/* Setup the CS to resume from the breadcrumb of the hung request */
@@ -2989,7 +2976,7 @@ void i915_gem_reset(struct drm_i915_private *dev_priv)
 	for_each_engine(engine, dev_priv, id) {
 		struct i915_gem_context *ctx;
 
-		i915_gem_reset_engine(engine);
+		i915_gem_reset_engine(engine, engine->hangcheck.active_request);
 		ctx = fetch_and_zero(&engine->last_retired_context);
 		if (ctx)
 			engine->context_unpin(engine, ctx);
@@ -3005,6 +2992,12 @@ void i915_gem_reset(struct drm_i915_private *dev_priv)
 	}
 }
 
+void i915_gem_reset_finish_engine(struct intel_engine_cs *engine)
+{
+	tasklet_enable(&engine->irq_tasklet);
+	kthread_unpark(engine->breadcrumbs.signaler);
+}
+
 void i915_gem_reset_finish(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
@@ -3013,13 +3006,14 @@ void i915_gem_reset_finish(struct drm_i915_private *dev_priv)
 	lockdep_assert_held(&dev_priv->drm.struct_mutex);
 
 	for_each_engine(engine, dev_priv, id) {
-		tasklet_enable(&engine->irq_tasklet);
-		kthread_unpark(engine->breadcrumbs.signaler);
+		engine->hangcheck.active_request = NULL;
+		i915_gem_reset_finish_engine(engine);
 	}
 }
 
 static void nop_submit_request(struct drm_i915_gem_request *request)
 {
+	GEM_BUG_ON(!i915_terminally_wedged(&request->i915->gpu_error));
 	dma_fence_set_error(&request->fence, -EIO);
 	i915_gem_request_submit(request);
 	intel_engine_init_global_seqno(request->engine, request->global_seqno);
@@ -3041,16 +3035,10 @@ static void engine_set_wedged(struct intel_engine_cs *engine)
 	/* Mark all executing requests as skipped */
 	spin_lock_irqsave(&engine->timeline->lock, flags);
 	list_for_each_entry(request, &engine->timeline->requests, link)
-		dma_fence_set_error(&request->fence, -EIO);
+		if (!i915_gem_request_completed(request))
+			dma_fence_set_error(&request->fence, -EIO);
 	spin_unlock_irqrestore(&engine->timeline->lock, flags);
 
-	/* Mark all pending requests as complete so that any concurrent
-	 * (lockless) lookup doesn't try and wait upon the request as we
-	 * reset it.
-	 */
-	intel_engine_init_global_seqno(engine,
-				       intel_engine_last_submit(engine));
-
 	/*
 	 * Clear the execlists queue up before freeing the requests, as those
 	 * are the ones that keep the context and ringbuffer backing objects
@@ -3071,7 +3059,21 @@ static void engine_set_wedged(struct intel_engine_cs *engine)
 		engine->execlist_first = NULL;
 
 		spin_unlock_irqrestore(&engine->timeline->lock, flags);
+
+		/* The port is checked prior to scheduling a tasklet, but
+		 * just in case we have suspended the tasklet to do the
+		 * wedging make sure that when it wakes, it decides there
+		 * is no work to do by clearing the irq_posted bit.
+		 */
+		clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
 	}
+
+	/* Mark all pending requests as complete so that any concurrent
+	 * (lockless) lookup doesn't try and wait upon the request as we
+	 * reset it.
+	 */
+	intel_engine_init_global_seqno(engine,
+				       intel_engine_last_submit(engine));
 }
 
 static int __i915_gem_set_wedged_BKL(void *data)
@@ -3083,25 +3085,15 @@ static int __i915_gem_set_wedged_BKL(void *data)
 	for_each_engine(engine, i915, id)
 		engine_set_wedged(engine);
 
+	set_bit(I915_WEDGED, &i915->gpu_error.flags);
+	wake_up_all(&i915->gpu_error.reset_queue);
+
 	return 0;
 }
 
 void i915_gem_set_wedged(struct drm_i915_private *dev_priv)
 {
-	lockdep_assert_held(&dev_priv->drm.struct_mutex);
-	set_bit(I915_WEDGED, &dev_priv->gpu_error.flags);
-
-	/* Retire completed requests first so the list of inflight/incomplete
-	 * requests is accurate and we don't try and mark successful requests
-	 * as in error during __i915_gem_set_wedged_BKL().
-	 */
-	i915_gem_retire_requests(dev_priv);
-
 	stop_machine(__i915_gem_set_wedged_BKL, dev_priv, NULL);
-
-	i915_gem_context_lost(dev_priv);
-
-	mod_delayed_work(dev_priv->wq, &dev_priv->gt.idle_work, 0);
 }
 
 bool i915_gem_unset_wedged(struct drm_i915_private *i915)
@@ -3156,6 +3148,7 @@ bool i915_gem_unset_wedged(struct drm_i915_private *i915)
 	 * context and do not require stop_machine().
 	 */
 	intel_engines_reset_default_submission(i915);
+	i915_gem_contexts_lost(i915);
 
 	smp_mb__before_atomic(); /* complete takeover before enabling execbuf */
 	clear_bit(I915_WEDGED, &i915->gpu_error.flags);
@@ -3253,25 +3246,39 @@ out_rearm:
 
 void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
 {
+	struct drm_i915_private *i915 = to_i915(gem->dev);
 	struct drm_i915_gem_object *obj = to_intel_bo(gem);
 	struct drm_i915_file_private *fpriv = file->driver_priv;
-	struct i915_vma *vma, *vn;
+	struct i915_lut_handle *lut, *ln;
 
-	mutex_lock(&obj->base.dev->struct_mutex);
-	list_for_each_entry_safe(vma, vn, &obj->vma_list, obj_link)
-		if (vma->vm->file == fpriv)
+	mutex_lock(&i915->drm.struct_mutex);
+
+	list_for_each_entry_safe(lut, ln, &obj->lut_list, obj_link) {
+		struct i915_gem_context *ctx = lut->ctx;
+		struct i915_vma *vma;
+
+		if (ctx->file_priv != fpriv)
+			continue;
+
+		vma = radix_tree_delete(&ctx->handles_vma, lut->handle);
+
+		GEM_BUG_ON(vma->obj != obj);
+
+		/* We allow the process to have multiple handles to the same
+		 * vma, in the same fd namespace, by virtue of flink/open.
+		 */
+		GEM_BUG_ON(!vma->open_count);
+		if (!--vma->open_count && !i915_vma_is_ggtt(vma))
 			i915_vma_close(vma);
 
-	vma = obj->vma_hashed;
-	if (vma && vma->ctx->file_priv == fpriv)
-		i915_vma_unlink_ctx(vma);
+		list_del(&lut->obj_link);
+		list_del(&lut->ctx_link);
 
-	if (i915_gem_object_is_active(obj) &&
-	    !i915_gem_object_has_active_reference(obj)) {
-		i915_gem_object_set_active_reference(obj);
-		i915_gem_object_get(obj);
+		kmem_cache_free(i915->luts, lut);
+		__i915_gem_object_release_unless_active(obj);
 	}
-	mutex_unlock(&obj->base.dev->struct_mutex);
+
+	mutex_unlock(&i915->drm.struct_mutex);
 }
 
 static unsigned long to_wait_timeout(s64 timeout_ns)
@@ -3297,7 +3304,7 @@ static unsigned long to_wait_timeout(s64 timeout_ns)
  *  -ERESTARTSYS: signal interrupted the wait
  *  -ENONENT: object doesn't exist
  * Also possible, but rare:
- *  -EAGAIN: GPU wedged
+ *  -EAGAIN: incomplete, restart syscall
  *  -ENOMEM: damn
  *  -ENODEV: Internal IRQ fail
  *  -E?: The add request failed
@@ -3345,6 +3352,10 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 		 */
 		if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns))
 			args->timeout_ns = 0;
+
+		/* Asked to wait beyond the jiffie/scheduler precision? */
+		if (ret == -ETIME && args->timeout_ns)
+			ret = -EAGAIN;
 	}
 
 	i915_gem_object_put(obj);
@@ -3686,8 +3697,7 @@ restart:
 
 	list_for_each_entry(vma, &obj->vma_list, obj_link)
 		vma->node.color = cache_level;
-	obj->cache_level = cache_level;
-	obj->cache_coherent = i915_gem_object_is_coherent(obj);
+	i915_gem_object_set_cache_coherency(obj, cache_level);
 	obj->cache_dirty = true; /* Always invalidate stale cachelines */
 
 	return 0;
@@ -4260,6 +4270,7 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj,
 	INIT_LIST_HEAD(&obj->global_link);
 	INIT_LIST_HEAD(&obj->userfault_link);
 	INIT_LIST_HEAD(&obj->vma_list);
+	INIT_LIST_HEAD(&obj->lut_list);
 	INIT_LIST_HEAD(&obj->batch_pool_link);
 
 	obj->ops = ops;
@@ -4292,6 +4303,7 @@ i915_gem_object_create(struct drm_i915_private *dev_priv, u64 size)
 {
 	struct drm_i915_gem_object *obj;
 	struct address_space *mapping;
+	unsigned int cache_level;
 	gfp_t mask;
 	int ret;
 
@@ -4330,7 +4342,7 @@ i915_gem_object_create(struct drm_i915_private *dev_priv, u64 size)
 	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
 
-	if (HAS_LLC(dev_priv)) {
+	if (HAS_LLC(dev_priv))
 		/* On some devices, we can have the GPU use the LLC (the CPU
 		 * cache) for about a 10% performance improvement
 		 * compared to uncached.  Graphics requests other than
@@ -4343,12 +4355,11 @@ i915_gem_object_create(struct drm_i915_private *dev_priv, u64 size)
 		 * However, we maintain the display planes as UC, and so
 		 * need to rebind when first used as such.
 		 */
-		obj->cache_level = I915_CACHE_LLC;
-	} else
-		obj->cache_level = I915_CACHE_NONE;
+		cache_level = I915_CACHE_LLC;
+	else
+		cache_level = I915_CACHE_NONE;
 
-	obj->cache_coherent = i915_gem_object_is_coherent(obj);
-	obj->cache_dirty = !obj->cache_coherent;
+	i915_gem_object_set_cache_coherency(obj, cache_level);
 
 	trace_i915_gem_object_create(obj);
 
@@ -4503,8 +4514,8 @@ void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj)
 {
 	lockdep_assert_held(&obj->base.dev->struct_mutex);
 
-	GEM_BUG_ON(i915_gem_object_has_active_reference(obj));
-	if (i915_gem_object_is_active(obj))
+	if (!i915_gem_object_has_active_reference(obj) &&
+	    i915_gem_object_is_active(obj))
 		i915_gem_object_set_active_reference(obj);
 	else
 		i915_gem_object_put(obj);
@@ -4565,7 +4576,7 @@ int i915_gem_suspend(struct drm_i915_private *dev_priv)
 		goto err_unlock;
 
 	assert_kernel_context_is_current(dev_priv);
-	i915_gem_context_lost(dev_priv);
+	i915_gem_contexts_lost(dev_priv);
 	mutex_unlock(&dev->struct_mutex);
 
 	intel_guc_suspend(dev_priv);
@@ -4579,8 +4590,6 @@ int i915_gem_suspend(struct drm_i915_private *dev_priv)
 	while (flush_delayed_work(&dev_priv->gt.idle_work))
 		;
 
-	i915_gem_drain_freed_objects(dev_priv);
-
 	/* Assert that we sucessfully flushed all the work and
 	 * reset the GPU back to its idle, low power state.
 	 */
@@ -4812,7 +4821,7 @@ int i915_gem_init(struct drm_i915_private *dev_priv)
 	if (ret)
 		goto out_unlock;
 
-	ret = i915_gem_context_init(dev_priv);
+	ret = i915_gem_contexts_init(dev_priv);
 	if (ret)
 		goto out_unlock;
 
@@ -4898,12 +4907,16 @@ i915_gem_load_init(struct drm_i915_private *dev_priv)
 	if (!dev_priv->vmas)
 		goto err_objects;
 
+	dev_priv->luts = KMEM_CACHE(i915_lut_handle, 0);
+	if (!dev_priv->luts)
+		goto err_vmas;
+
 	dev_priv->requests = KMEM_CACHE(drm_i915_gem_request,
 					SLAB_HWCACHE_ALIGN |
 					SLAB_RECLAIM_ACCOUNT |
 					SLAB_TYPESAFE_BY_RCU);
 	if (!dev_priv->requests)
-		goto err_vmas;
+		goto err_luts;
 
 	dev_priv->dependencies = KMEM_CACHE(i915_dependency,
 					    SLAB_HWCACHE_ALIGN |
@@ -4922,7 +4935,6 @@ i915_gem_load_init(struct drm_i915_private *dev_priv)
 	if (err)
 		goto err_priorities;
 
-	INIT_LIST_HEAD(&dev_priv->context_list);
 	INIT_WORK(&dev_priv->mm.free_work, __i915_gem_free_work);
 	init_llist_head(&dev_priv->mm.free_list);
 	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
@@ -4936,8 +4948,6 @@ i915_gem_load_init(struct drm_i915_private *dev_priv)
 	init_waitqueue_head(&dev_priv->gpu_error.wait_queue);
 	init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
 
-	init_waitqueue_head(&dev_priv->pending_flip_queue);
-
 	atomic_set(&dev_priv->mm.bsd_engine_dispatch_index, 0);
 
 	spin_lock_init(&dev_priv->fb_tracking.lock);
@@ -4950,6 +4960,8 @@ err_dependencies:
 	kmem_cache_destroy(dev_priv->dependencies);
 err_requests:
 	kmem_cache_destroy(dev_priv->requests);
+err_luts:
+	kmem_cache_destroy(dev_priv->luts);
 err_vmas:
 	kmem_cache_destroy(dev_priv->vmas);
 err_objects:
@@ -4972,6 +4984,7 @@ void i915_gem_load_cleanup(struct drm_i915_private *dev_priv)
 	kmem_cache_destroy(dev_priv->priorities);
 	kmem_cache_destroy(dev_priv->dependencies);
 	kmem_cache_destroy(dev_priv->requests);
+	kmem_cache_destroy(dev_priv->luts);
 	kmem_cache_destroy(dev_priv->vmas);
 	kmem_cache_destroy(dev_priv->objects);
 
@@ -5012,7 +5025,7 @@ int i915_gem_freeze_late(struct drm_i915_private *dev_priv)
 	 * the objects as well, see i915_gem_freeze()
 	 */
 
-	i915_gem_shrink(dev_priv, -1UL, I915_SHRINK_UNBOUND);
+	i915_gem_shrink(dev_priv, -1UL, NULL, I915_SHRINK_UNBOUND);
 	i915_gem_drain_freed_objects(dev_priv);
 
 	mutex_lock(&dev_priv->drm.struct_mutex);
@@ -5038,15 +5051,9 @@ void i915_gem_release(struct drm_device *dev, struct drm_file *file)
 	list_for_each_entry(request, &file_priv->mm.request_list, client_link)
 		request->file_priv = NULL;
 	spin_unlock(&file_priv->mm.lock);
-
-	if (!list_empty(&file_priv->rps.link)) {
-		spin_lock(&to_i915(dev)->rps.client_lock);
-		list_del(&file_priv->rps.link);
-		spin_unlock(&to_i915(dev)->rps.client_lock);
-	}
 }
 
-int i915_gem_open(struct drm_device *dev, struct drm_file *file)
+int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file)
 {
 	struct drm_i915_file_private *file_priv;
 	int ret;
@@ -5058,16 +5065,15 @@ int i915_gem_open(struct drm_device *dev, struct drm_file *file)
 		return -ENOMEM;
 
 	file->driver_priv = file_priv;
-	file_priv->dev_priv = to_i915(dev);
+	file_priv->dev_priv = i915;
 	file_priv->file = file;
-	INIT_LIST_HEAD(&file_priv->rps.link);
 
 	spin_lock_init(&file_priv->mm.lock);
 	INIT_LIST_HEAD(&file_priv->mm.request_list);
 
 	file_priv->bsd_engine = -1;
 
-	ret = i915_gem_context_open(dev, file);
+	ret = i915_gem_context_open(i915, file);
 	if (ret)
 		kfree(file_priv);
 
@@ -5311,6 +5317,64 @@ i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
 	return sg_dma_address(sg) + (offset << PAGE_SHIFT);
 }
 
+int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align)
+{
+	struct sg_table *pages;
+	int err;
+
+	if (align > obj->base.size)
+		return -EINVAL;
+
+	if (obj->ops == &i915_gem_phys_ops)
+		return 0;
+
+	if (obj->ops != &i915_gem_object_ops)
+		return -EINVAL;
+
+	err = i915_gem_object_unbind(obj);
+	if (err)
+		return err;
+
+	mutex_lock(&obj->mm.lock);
+
+	if (obj->mm.madv != I915_MADV_WILLNEED) {
+		err = -EFAULT;
+		goto err_unlock;
+	}
+
+	if (obj->mm.quirked) {
+		err = -EFAULT;
+		goto err_unlock;
+	}
+
+	if (obj->mm.mapping) {
+		err = -EBUSY;
+		goto err_unlock;
+	}
+
+	pages = obj->mm.pages;
+	obj->ops = &i915_gem_phys_ops;
+
+	err = ____i915_gem_object_get_pages(obj);
+	if (err)
+		goto err_xfer;
+
+	/* Perma-pin (until release) the physical set of pages */
+	__i915_gem_object_pin_pages(obj);
+
+	if (!IS_ERR_OR_NULL(pages))
+		i915_gem_object_ops.put_pages(obj, pages);
+	mutex_unlock(&obj->mm.lock);
+	return 0;
+
+err_xfer:
+	obj->ops = &i915_gem_object_ops;
+	obj->mm.pages = pages;
+err_unlock:
+	mutex_unlock(&obj->mm.lock);
+	return err;
+}
+
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 #include "selftests/scatterlist.c"
 #include "selftests/mock_gem_device.c"
diff --git a/drivers/gpu/drm/i915/i915_gem_clflush.c b/drivers/gpu/drm/i915/i915_gem_clflush.c
index 348b29a845c9..8a04d33055be 100644
--- a/drivers/gpu/drm/i915/i915_gem_clflush.c
+++ b/drivers/gpu/drm/i915/i915_gem_clflush.c
@@ -139,7 +139,8 @@ bool i915_gem_clflush_object(struct drm_i915_gem_object *obj,
 	 * snooping behaviour occurs naturally as the result of our domain
 	 * tracking.
 	 */
-	if (!(flags & I915_CLFLUSH_FORCE) && obj->cache_coherent)
+	if (!(flags & I915_CLFLUSH_FORCE) &&
+	    obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)
 		return false;
 
 	trace_i915_gem_object_clflush(obj);
diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c
index 39ed58a21fc1..58a2a44f88bd 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/i915_gem_context.c
@@ -93,81 +93,37 @@
 
 #define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1
 
-/* Initial size (as log2) to preallocate the handle->object hashtable */
-#define VMA_HT_BITS 2u /* 4 x 2 pointers, 64 bytes minimum */
-
-static void resize_vma_ht(struct work_struct *work)
+static void lut_close(struct i915_gem_context *ctx)
 {
-	struct i915_gem_context_vma_lut *lut =
-		container_of(work, typeof(*lut), resize);
-	unsigned int bits, new_bits, size, i;
-	struct hlist_head *new_ht;
-
-	GEM_BUG_ON(!(lut->ht_size & I915_CTX_RESIZE_IN_PROGRESS));
-
-	bits = 1 + ilog2(4*lut->ht_count/3 + 1);
-	new_bits = min_t(unsigned int,
-			 max(bits, VMA_HT_BITS),
-			 sizeof(unsigned int) * BITS_PER_BYTE - 1);
-	if (new_bits == lut->ht_bits)
-		goto out;
-
-	new_ht = kzalloc(sizeof(*new_ht)<<new_bits, GFP_KERNEL | __GFP_NOWARN);
-	if (!new_ht)
-		new_ht = vzalloc(sizeof(*new_ht)<<new_bits);
-	if (!new_ht)
-		/* Pretend resize succeeded and stop calling us for a bit! */
-		goto out;
+	struct i915_lut_handle *lut, *ln;
+	struct radix_tree_iter iter;
+	void __rcu **slot;
 
-	size = BIT(lut->ht_bits);
-	for (i = 0; i < size; i++) {
-		struct i915_vma *vma;
-		struct hlist_node *tmp;
-
-		hlist_for_each_entry_safe(vma, tmp, &lut->ht[i], ctx_node)
-			hlist_add_head(&vma->ctx_node,
-				       &new_ht[hash_32(vma->ctx_handle,
-						       new_bits)]);
+	list_for_each_entry_safe(lut, ln, &ctx->handles_list, ctx_link) {
+		list_del(&lut->obj_link);
+		kmem_cache_free(ctx->i915->luts, lut);
 	}
-	kvfree(lut->ht);
-	lut->ht = new_ht;
-	lut->ht_bits = new_bits;
-out:
-	smp_store_release(&lut->ht_size, BIT(bits));
-	GEM_BUG_ON(lut->ht_size & I915_CTX_RESIZE_IN_PROGRESS);
-}
 
-static void vma_lut_free(struct i915_gem_context *ctx)
-{
-	struct i915_gem_context_vma_lut *lut = &ctx->vma_lut;
-	unsigned int i, size;
+	radix_tree_for_each_slot(slot, &ctx->handles_vma, &iter, 0) {
+		struct i915_vma *vma = rcu_dereference_raw(*slot);
+		struct drm_i915_gem_object *obj = vma->obj;
 
-	if (lut->ht_size & I915_CTX_RESIZE_IN_PROGRESS)
-		cancel_work_sync(&lut->resize);
+		radix_tree_iter_delete(&ctx->handles_vma, &iter, slot);
 
-	size = BIT(lut->ht_bits);
-	for (i = 0; i < size; i++) {
-		struct i915_vma *vma;
+		if (!i915_vma_is_ggtt(vma))
+			i915_vma_close(vma);
 
-		hlist_for_each_entry(vma, &lut->ht[i], ctx_node) {
-			vma->obj->vma_hashed = NULL;
-			vma->ctx = NULL;
-			i915_vma_put(vma);
-		}
+		__i915_gem_object_release_unless_active(obj);
 	}
-	kvfree(lut->ht);
 }
 
-void i915_gem_context_free(struct kref *ctx_ref)
+static void i915_gem_context_free(struct i915_gem_context *ctx)
 {
-	struct i915_gem_context *ctx = container_of(ctx_ref, typeof(*ctx), ref);
 	int i;
 
 	lockdep_assert_held(&ctx->i915->drm.struct_mutex);
-	trace_i915_context_free(ctx);
 	GEM_BUG_ON(!i915_gem_context_is_closed(ctx));
 
-	vma_lut_free(ctx);
 	i915_ppgtt_put(ctx->ppgtt);
 
 	for (i = 0; i < I915_NUM_ENGINES; i++) {
@@ -188,15 +144,64 @@ void i915_gem_context_free(struct kref *ctx_ref)
 
 	list_del(&ctx->link);
 
-	ida_simple_remove(&ctx->i915->context_hw_ida, ctx->hw_id);
-	kfree(ctx);
+	ida_simple_remove(&ctx->i915->contexts.hw_ida, ctx->hw_id);
+	kfree_rcu(ctx, rcu);
+}
+
+static void contexts_free(struct drm_i915_private *i915)
+{
+	struct llist_node *freed = llist_del_all(&i915->contexts.free_list);
+	struct i915_gem_context *ctx, *cn;
+
+	lockdep_assert_held(&i915->drm.struct_mutex);
+
+	llist_for_each_entry_safe(ctx, cn, freed, free_link)
+		i915_gem_context_free(ctx);
+}
+
+static void contexts_free_first(struct drm_i915_private *i915)
+{
+	struct i915_gem_context *ctx;
+	struct llist_node *freed;
+
+	lockdep_assert_held(&i915->drm.struct_mutex);
+
+	freed = llist_del_first(&i915->contexts.free_list);
+	if (!freed)
+		return;
+
+	ctx = container_of(freed, typeof(*ctx), free_link);
+	i915_gem_context_free(ctx);
+}
+
+static void contexts_free_worker(struct work_struct *work)
+{
+	struct drm_i915_private *i915 =
+		container_of(work, typeof(*i915), contexts.free_work);
+
+	mutex_lock(&i915->drm.struct_mutex);
+	contexts_free(i915);
+	mutex_unlock(&i915->drm.struct_mutex);
+}
+
+void i915_gem_context_release(struct kref *ref)
+{
+	struct i915_gem_context *ctx = container_of(ref, typeof(*ctx), ref);
+	struct drm_i915_private *i915 = ctx->i915;
+
+	trace_i915_context_free(ctx);
+	if (llist_add(&ctx->free_link, &i915->contexts.free_list))
+		queue_work(i915->wq, &i915->contexts.free_work);
 }
 
 static void context_close(struct i915_gem_context *ctx)
 {
 	i915_gem_context_set_closed(ctx);
+
+	lut_close(ctx);
 	if (ctx->ppgtt)
 		i915_ppgtt_close(&ctx->ppgtt->base);
+
 	ctx->file_priv = ERR_PTR(-EBADF);
 	i915_gem_context_put(ctx);
 }
@@ -205,7 +210,7 @@ static int assign_hw_id(struct drm_i915_private *dev_priv, unsigned *out)
 {
 	int ret;
 
-	ret = ida_simple_get(&dev_priv->context_hw_ida,
+	ret = ida_simple_get(&dev_priv->contexts.hw_ida,
 			     0, MAX_CONTEXT_HW_ID, GFP_KERNEL);
 	if (ret < 0) {
 		/* Contexts are only released when no longer active.
@@ -213,7 +218,7 @@ static int assign_hw_id(struct drm_i915_private *dev_priv, unsigned *out)
 		 * stale contexts and try again.
 		 */
 		i915_gem_retire_requests(dev_priv);
-		ret = ida_simple_get(&dev_priv->context_hw_ida,
+		ret = ida_simple_get(&dev_priv->contexts.hw_ida,
 				     0, MAX_CONTEXT_HW_ID, GFP_KERNEL);
 		if (ret < 0)
 			return ret;
@@ -265,20 +270,12 @@ __create_hw_context(struct drm_i915_private *dev_priv,
 	}
 
 	kref_init(&ctx->ref);
-	list_add_tail(&ctx->link, &dev_priv->context_list);
+	list_add_tail(&ctx->link, &dev_priv->contexts.list);
 	ctx->i915 = dev_priv;
 	ctx->priority = I915_PRIORITY_NORMAL;
 
-	ctx->vma_lut.ht_bits = VMA_HT_BITS;
-	ctx->vma_lut.ht_size = BIT(VMA_HT_BITS);
-	BUILD_BUG_ON(BIT(VMA_HT_BITS) == I915_CTX_RESIZE_IN_PROGRESS);
-	ctx->vma_lut.ht = kcalloc(ctx->vma_lut.ht_size,
-				  sizeof(*ctx->vma_lut.ht),
-				  GFP_KERNEL);
-	if (!ctx->vma_lut.ht)
-		goto err_out;
-
-	INIT_WORK(&ctx->vma_lut.resize, resize_vma_ht);
+	INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL);
+	INIT_LIST_HEAD(&ctx->handles_list);
 
 	/* Default context will never have a file_priv */
 	ret = DEFAULT_CONTEXT_HANDLE;
@@ -328,8 +325,6 @@ err_pid:
 	put_pid(ctx->pid);
 	idr_remove(&file_priv->context_idr, ctx->user_handle);
 err_lut:
-	kvfree(ctx->vma_lut.ht);
-err_out:
 	context_close(ctx);
 	return ERR_PTR(ret);
 }
@@ -354,6 +349,9 @@ i915_gem_create_context(struct drm_i915_private *dev_priv,
 
 	lockdep_assert_held(&dev_priv->drm.struct_mutex);
 
+	/* Reap the most stale context */
+	contexts_free_first(dev_priv);
+
 	ctx = __create_hw_context(dev_priv, file_priv);
 	if (IS_ERR(ctx))
 		return ctx;
@@ -418,7 +416,7 @@ out:
 	return ctx;
 }
 
-int i915_gem_context_init(struct drm_i915_private *dev_priv)
+int i915_gem_contexts_init(struct drm_i915_private *dev_priv)
 {
 	struct i915_gem_context *ctx;
 
@@ -427,6 +425,10 @@ int i915_gem_context_init(struct drm_i915_private *dev_priv)
 	if (WARN_ON(dev_priv->kernel_context))
 		return 0;
 
+	INIT_LIST_HEAD(&dev_priv->contexts.list);
+	INIT_WORK(&dev_priv->contexts.free_work, contexts_free_worker);
+	init_llist_head(&dev_priv->contexts.free_list);
+
 	if (intel_vgpu_active(dev_priv) &&
 	    HAS_LOGICAL_RING_CONTEXTS(dev_priv)) {
 		if (!i915.enable_execlists) {
@@ -437,7 +439,7 @@ int i915_gem_context_init(struct drm_i915_private *dev_priv)
 
 	/* Using the simple ida interface, the max is limited by sizeof(int) */
 	BUILD_BUG_ON(MAX_CONTEXT_HW_ID > INT_MAX);
-	ida_init(&dev_priv->context_hw_ida);
+	ida_init(&dev_priv->contexts.hw_ida);
 
 	ctx = i915_gem_create_context(dev_priv, NULL);
 	if (IS_ERR(ctx)) {
@@ -463,7 +465,7 @@ int i915_gem_context_init(struct drm_i915_private *dev_priv)
 	return 0;
 }
 
-void i915_gem_context_lost(struct drm_i915_private *dev_priv)
+void i915_gem_contexts_lost(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
@@ -484,7 +486,7 @@ void i915_gem_context_lost(struct drm_i915_private *dev_priv)
 	if (!i915.enable_execlists) {
 		struct i915_gem_context *ctx;
 
-		list_for_each_entry(ctx, &dev_priv->context_list, link) {
+		list_for_each_entry(ctx, &dev_priv->contexts.list, link) {
 			if (!i915_gem_context_is_default(ctx))
 				continue;
 
@@ -503,18 +505,20 @@ void i915_gem_context_lost(struct drm_i915_private *dev_priv)
 	}
 }
 
-void i915_gem_context_fini(struct drm_i915_private *dev_priv)
+void i915_gem_contexts_fini(struct drm_i915_private *i915)
 {
-	struct i915_gem_context *dctx = dev_priv->kernel_context;
-
-	lockdep_assert_held(&dev_priv->drm.struct_mutex);
+	struct i915_gem_context *ctx;
 
-	GEM_BUG_ON(!i915_gem_context_is_kernel(dctx));
+	lockdep_assert_held(&i915->drm.struct_mutex);
 
-	context_close(dctx);
-	dev_priv->kernel_context = NULL;
+	/* Keep the context so that we can free it immediately ourselves */
+	ctx = i915_gem_context_get(fetch_and_zero(&i915->kernel_context));
+	GEM_BUG_ON(!i915_gem_context_is_kernel(ctx));
+	context_close(ctx);
+	i915_gem_context_free(ctx);
 
-	ida_destroy(&dev_priv->context_hw_ida);
+	/* Must free all deferred contexts (via flush_workqueue) first */
+	ida_destroy(&i915->contexts.hw_ida);
 }
 
 static int context_idr_cleanup(int id, void *p, void *data)
@@ -525,32 +529,32 @@ static int context_idr_cleanup(int id, void *p, void *data)
 	return 0;
 }
 
-int i915_gem_context_open(struct drm_device *dev, struct drm_file *file)
+int i915_gem_context_open(struct drm_i915_private *i915,
+			  struct drm_file *file)
 {
 	struct drm_i915_file_private *file_priv = file->driver_priv;
 	struct i915_gem_context *ctx;
 
 	idr_init(&file_priv->context_idr);
 
-	mutex_lock(&dev->struct_mutex);
-	ctx = i915_gem_create_context(to_i915(dev), file_priv);
-	mutex_unlock(&dev->struct_mutex);
-
-	GEM_BUG_ON(i915_gem_context_is_kernel(ctx));
-
+	mutex_lock(&i915->drm.struct_mutex);
+	ctx = i915_gem_create_context(i915, file_priv);
+	mutex_unlock(&i915->drm.struct_mutex);
 	if (IS_ERR(ctx)) {
 		idr_destroy(&file_priv->context_idr);
 		return PTR_ERR(ctx);
 	}
 
+	GEM_BUG_ON(i915_gem_context_is_kernel(ctx));
+
 	return 0;
 }
 
-void i915_gem_context_close(struct drm_device *dev, struct drm_file *file)
+void i915_gem_context_close(struct drm_file *file)
 {
 	struct drm_i915_file_private *file_priv = file->driver_priv;
 
-	lockdep_assert_held(&dev->struct_mutex);
+	lockdep_assert_held(&file_priv->dev_priv->drm.struct_mutex);
 
 	idr_for_each(&file_priv->context_idr, context_idr_cleanup, NULL);
 	idr_destroy(&file_priv->context_idr);
@@ -688,19 +692,19 @@ static inline bool skip_rcs_switch(struct i915_hw_ppgtt *ppgtt,
 }
 
 static bool
-needs_pd_load_pre(struct i915_hw_ppgtt *ppgtt,
-		  struct intel_engine_cs *engine,
-		  struct i915_gem_context *to)
+needs_pd_load_pre(struct i915_hw_ppgtt *ppgtt, struct intel_engine_cs *engine)
 {
+	struct i915_gem_context *from = engine->legacy_active_context;
+
 	if (!ppgtt)
 		return false;
 
 	/* Always load the ppgtt on first use */
-	if (!engine->legacy_active_context)
+	if (!from)
 		return true;
 
 	/* Same context without new entries, skip */
-	if (engine->legacy_active_context == to &&
+	if ((!from->ppgtt || from->ppgtt == ppgtt) &&
 	    !(intel_engine_flag(engine) & ppgtt->pd_dirty_rings))
 		return false;
 
@@ -744,7 +748,7 @@ static int do_rcs_switch(struct drm_i915_gem_request *req)
 	if (skip_rcs_switch(ppgtt, engine, to))
 		return 0;
 
-	if (needs_pd_load_pre(ppgtt, engine, to)) {
+	if (needs_pd_load_pre(ppgtt, engine)) {
 		/* Older GENs and non render rings still want the load first,
 		 * "PP_DCLV followed by PP_DIR_BASE register through Load
 		 * Register Immediate commands in Ring Buffer before submitting
@@ -841,7 +845,7 @@ int i915_switch_context(struct drm_i915_gem_request *req)
 		struct i915_hw_ppgtt *ppgtt =
 			to->ppgtt ?: req->i915->mm.aliasing_ppgtt;
 
-		if (needs_pd_load_pre(ppgtt, engine, to)) {
+		if (needs_pd_load_pre(ppgtt, engine)) {
 			int ret;
 
 			trace_switch_mm(engine, to);
@@ -852,6 +856,7 @@ int i915_switch_context(struct drm_i915_gem_request *req)
 			ppgtt->pd_dirty_rings &= ~intel_engine_flag(engine);
 		}
 
+		engine->legacy_active_context = to;
 		return 0;
 	}
 
@@ -924,7 +929,7 @@ int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv)
 
 static bool client_is_banned(struct drm_i915_file_private *file_priv)
 {
-	return file_priv->context_bans > I915_MAX_CLIENT_CONTEXT_BANS;
+	return atomic_read(&file_priv->context_bans) > I915_MAX_CLIENT_CONTEXT_BANS;
 }
 
 int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
@@ -981,20 +986,19 @@ int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data,
 	if (args->ctx_id == DEFAULT_CONTEXT_HANDLE)
 		return -ENOENT;
 
-	ret = i915_mutex_lock_interruptible(dev);
-	if (ret)
-		return ret;
-
 	ctx = i915_gem_context_lookup(file_priv, args->ctx_id);
-	if (IS_ERR(ctx)) {
-		mutex_unlock(&dev->struct_mutex);
-		return PTR_ERR(ctx);
-	}
+	if (!ctx)
+		return -ENOENT;
+
+	ret = mutex_lock_interruptible(&dev->struct_mutex);
+	if (ret)
+		goto out;
 
 	__destroy_hw_context(ctx, file_priv);
 	mutex_unlock(&dev->struct_mutex);
 
-	DRM_DEBUG("HW context %d destroyed\n", args->ctx_id);
+out:
+	i915_gem_context_put(ctx);
 	return 0;
 }
 
@@ -1004,17 +1008,11 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data,
 	struct drm_i915_file_private *file_priv = file->driver_priv;
 	struct drm_i915_gem_context_param *args = data;
 	struct i915_gem_context *ctx;
-	int ret;
-
-	ret = i915_mutex_lock_interruptible(dev);
-	if (ret)
-		return ret;
+	int ret = 0;
 
 	ctx = i915_gem_context_lookup(file_priv, args->ctx_id);
-	if (IS_ERR(ctx)) {
-		mutex_unlock(&dev->struct_mutex);
-		return PTR_ERR(ctx);
-	}
+	if (!ctx)
+		return -ENOENT;
 
 	args->size = 0;
 	switch (args->param) {
@@ -1042,8 +1040,8 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data,
 		ret = -EINVAL;
 		break;
 	}
-	mutex_unlock(&dev->struct_mutex);
 
+	i915_gem_context_put(ctx);
 	return ret;
 }
 
@@ -1055,15 +1053,13 @@ int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data,
 	struct i915_gem_context *ctx;
 	int ret;
 
+	ctx = i915_gem_context_lookup(file_priv, args->ctx_id);
+	if (!ctx)
+		return -ENOENT;
+
 	ret = i915_mutex_lock_interruptible(dev);
 	if (ret)
-		return ret;
-
-	ctx = i915_gem_context_lookup(file_priv, args->ctx_id);
-	if (IS_ERR(ctx)) {
-		mutex_unlock(&dev->struct_mutex);
-		return PTR_ERR(ctx);
-	}
+		goto out;
 
 	switch (args->param) {
 	case I915_CONTEXT_PARAM_BAN_PERIOD:
@@ -1101,6 +1097,8 @@ int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data,
 	}
 	mutex_unlock(&dev->struct_mutex);
 
+out:
+	i915_gem_context_put(ctx);
 	return ret;
 }
 
@@ -1115,27 +1113,31 @@ int i915_gem_context_reset_stats_ioctl(struct drm_device *dev,
 	if (args->flags || args->pad)
 		return -EINVAL;
 
-	ret = i915_mutex_lock_interruptible(dev);
-	if (ret)
-		return ret;
+	ret = -ENOENT;
+	rcu_read_lock();
+	ctx = __i915_gem_context_lookup_rcu(file->driver_priv, args->ctx_id);
+	if (!ctx)
+		goto out;
 
-	ctx = i915_gem_context_lookup(file->driver_priv, args->ctx_id);
-	if (IS_ERR(ctx)) {
-		mutex_unlock(&dev->struct_mutex);
-		return PTR_ERR(ctx);
-	}
+	/*
+	 * We opt for unserialised reads here. This may result in tearing
+	 * in the extremely unlikely event of a GPU hang on this context
+	 * as we are querying them. If we need that extra layer of protection,
+	 * we should wrap the hangstats with a seqlock.
+	 */
 
 	if (capable(CAP_SYS_ADMIN))
 		args->reset_count = i915_reset_count(&dev_priv->gpu_error);
 	else
 		args->reset_count = 0;
 
-	args->batch_active = ctx->guilty_count;
-	args->batch_pending = ctx->active_count;
-
-	mutex_unlock(&dev->struct_mutex);
+	args->batch_active = atomic_read(&ctx->guilty_count);
+	args->batch_pending = atomic_read(&ctx->active_count);
 
-	return 0;
+	ret = 0;
+out:
+	rcu_read_unlock();
+	return ret;
 }
 
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
diff --git a/drivers/gpu/drm/i915/i915_gem_context.h b/drivers/gpu/drm/i915/i915_gem_context.h
index 82c99ba92ad3..44688e22a5c2 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.h
+++ b/drivers/gpu/drm/i915/i915_gem_context.h
@@ -27,6 +27,7 @@
 
 #include <linux/bitops.h>
 #include <linux/list.h>
+#include <linux/radix-tree.h>
 
 struct pid;
 
@@ -86,6 +87,7 @@ struct i915_gem_context {
 
 	/** link: place with &drm_i915_private.context_list */
 	struct list_head link;
+	struct llist_node free_link;
 
 	/**
 	 * @ref: reference count
@@ -99,6 +101,11 @@ struct i915_gem_context {
 	struct kref ref;
 
 	/**
+	 * @rcu: rcu_head for deferred freeing.
+	 */
+	struct rcu_head rcu;
+
+	/**
 	 * @flags: small set of booleans
 	 */
 	unsigned long flags;
@@ -143,32 +150,6 @@ struct i915_gem_context {
 	/** ggtt_offset_bias: placement restriction for context objects */
 	u32 ggtt_offset_bias;
 
-	struct i915_gem_context_vma_lut {
-		/** ht_size: last request size to allocate the hashtable for. */
-		unsigned int ht_size;
-#define I915_CTX_RESIZE_IN_PROGRESS BIT(0)
-		/** ht_bits: real log2(size) of hashtable. */
-		unsigned int ht_bits;
-		/** ht_count: current number of entries inside the hashtable */
-		unsigned int ht_count;
-
-		/** ht: the array of buckets comprising the simple hashtable */
-		struct hlist_head *ht;
-
-		/**
-		 * resize: After an execbuf completes, we check the load factor
-		 * of the hashtable. If the hashtable is too full, or too empty,
-		 * we schedule a task to resize the hashtable. During the
-		 * resize, the entries are moved between different buckets and
-		 * so we cannot simultaneously read the hashtable as it is
-		 * being resized (unlike rhashtable). Therefore we treat the
-		 * active work as a strong barrier, pausing a subsequent
-		 * execbuf to wait for the resize worker to complete, if
-		 * required.
-		 */
-		struct work_struct resize;
-	} vma_lut;
-
 	/** engine: per-engine logical HW state */
 	struct intel_context {
 		struct i915_vma *state;
@@ -185,20 +166,32 @@ struct i915_gem_context {
 	u32 desc_template;
 
 	/** guilty_count: How many times this context has caused a GPU hang. */
-	unsigned int guilty_count;
+	atomic_t guilty_count;
 	/**
 	 * @active_count: How many times this context was active during a GPU
 	 * hang, but did not cause it.
 	 */
-	unsigned int active_count;
+	atomic_t active_count;
 
 #define CONTEXT_SCORE_GUILTY		10
 #define CONTEXT_SCORE_BAN_THRESHOLD	40
 	/** ban_score: Accumulated score of all hangs caused by this context. */
-	int ban_score;
+	atomic_t ban_score;
 
 	/** remap_slice: Bitmask of cache lines that need remapping */
 	u8 remap_slice;
+
+	/** handles_vma: rbtree to look up our context specific obj/vma for
+	 * the user handle. (user handles are per fd, but the binding is
+	 * per vm, which may be one per context or shared with the global GTT)
+	 */
+	struct radix_tree_root handles_vma;
+
+	/** handles_list: reverse list of all the rbtree entries in use for
+	 * this context, which allows us to free all the allocations on
+	 * context close.
+	 */
+	struct list_head handles_list;
 };
 
 static inline bool i915_gem_context_is_closed(const struct i915_gem_context *ctx)
@@ -273,14 +266,18 @@ static inline bool i915_gem_context_is_kernel(struct i915_gem_context *ctx)
 }
 
 /* i915_gem_context.c */
-int __must_check i915_gem_context_init(struct drm_i915_private *dev_priv);
-void i915_gem_context_lost(struct drm_i915_private *dev_priv);
-void i915_gem_context_fini(struct drm_i915_private *dev_priv);
-int i915_gem_context_open(struct drm_device *dev, struct drm_file *file);
-void i915_gem_context_close(struct drm_device *dev, struct drm_file *file);
+int __must_check i915_gem_contexts_init(struct drm_i915_private *dev_priv);
+void i915_gem_contexts_lost(struct drm_i915_private *dev_priv);
+void i915_gem_contexts_fini(struct drm_i915_private *dev_priv);
+
+int i915_gem_context_open(struct drm_i915_private *i915,
+			  struct drm_file *file);
+void i915_gem_context_close(struct drm_file *file);
+
 int i915_switch_context(struct drm_i915_gem_request *req);
 int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv);
-void i915_gem_context_free(struct kref *ctx_ref);
+
+void i915_gem_context_release(struct kref *ctx_ref);
 struct i915_gem_context *
 i915_gem_context_create_gvt(struct drm_device *dev);
 
@@ -295,4 +292,16 @@ int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data,
 int i915_gem_context_reset_stats_ioctl(struct drm_device *dev, void *data,
 				       struct drm_file *file);
 
+static inline struct i915_gem_context *
+i915_gem_context_get(struct i915_gem_context *ctx)
+{
+	kref_get(&ctx->ref);
+	return ctx;
+}
+
+static inline void i915_gem_context_put(struct i915_gem_context *ctx)
+{
+	kref_put(&ctx->ref, i915_gem_context_release);
+}
+
 #endif /* !__I915_GEM_CONTEXT_H__ */
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index a193f1b36c67..4df039ef2ce3 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -318,8 +318,8 @@ int i915_gem_evict_for_node(struct i915_address_space *vm,
 		/* Overlap of objects in the same batch? */
 		if (i915_vma_is_pinned(vma)) {
 			ret = -ENOSPC;
-			if (vma->exec_entry &&
-			    vma->exec_entry->flags & EXEC_OBJECT_PINNED)
+			if (vma->exec_flags &&
+			    *vma->exec_flags & EXEC_OBJECT_PINNED)
 				ret = -EINVAL;
 			break;
 		}
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index e9503f6d1100..92437f455b43 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -32,6 +32,7 @@
 #include <linux/uaccess.h>
 
 #include <drm/drmP.h>
+#include <drm/drm_syncobj.h>
 #include <drm/i915_drm.h>
 
 #include "i915_drv.h"
@@ -191,6 +192,8 @@ struct i915_execbuffer {
 	struct drm_file *file; /** per-file lookup tables and limits */
 	struct drm_i915_gem_execbuffer2 *args; /** ioctl parameters */
 	struct drm_i915_gem_exec_object2 *exec; /** ioctl execobj[] */
+	struct i915_vma **vma;
+	unsigned int *flags;
 
 	struct intel_engine_cs *engine; /** engine to queue the request to */
 	struct i915_gem_context *ctx; /** context for building the request */
@@ -244,13 +247,7 @@ struct i915_execbuffer {
 	struct hlist_head *buckets; /** ht for relocation handles */
 };
 
-/*
- * As an alternative to creating a hashtable of handle-to-vma for a batch,
- * we used the last available reserved field in the execobject[] and stash
- * a link from the execobj to its vma.
- */
-#define __exec_to_vma(ee) (ee)->rsvd2
-#define exec_to_vma(ee) u64_to_ptr(struct i915_vma, __exec_to_vma(ee))
+#define exec_entry(EB, VMA) (&(EB)->exec[(VMA)->exec_flags - (EB)->flags])
 
 /*
  * Used to convert any address to canonical form.
@@ -288,7 +285,7 @@ static int eb_create(struct i915_execbuffer *eb)
 		 * direct lookup.
 		 */
 		do {
-			unsigned int flags;
+			gfp_t flags;
 
 			/* While we can still reduce the allocation size, don't
 			 * raise a warning and allow the allocation to fail.
@@ -296,7 +293,7 @@ static int eb_create(struct i915_execbuffer *eb)
 			 * as possible to perform the allocation and warn
 			 * if it fails.
 			 */
-			flags = GFP_TEMPORARY;
+			flags = GFP_KERNEL;
 			if (size > 1)
 				flags |= __GFP_NORETRY | __GFP_NOWARN;
 
@@ -319,85 +316,82 @@ static int eb_create(struct i915_execbuffer *eb)
 
 static bool
 eb_vma_misplaced(const struct drm_i915_gem_exec_object2 *entry,
-		 const struct i915_vma *vma)
+		 const struct i915_vma *vma,
+		 unsigned int flags)
 {
-	if (!(entry->flags & __EXEC_OBJECT_HAS_PIN))
-		return true;
-
 	if (vma->node.size < entry->pad_to_size)
 		return true;
 
 	if (entry->alignment && !IS_ALIGNED(vma->node.start, entry->alignment))
 		return true;
 
-	if (entry->flags & EXEC_OBJECT_PINNED &&
+	if (flags & EXEC_OBJECT_PINNED &&
 	    vma->node.start != entry->offset)
 		return true;
 
-	if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS &&
+	if (flags & __EXEC_OBJECT_NEEDS_BIAS &&
 	    vma->node.start < BATCH_OFFSET_BIAS)
 		return true;
 
-	if (!(entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) &&
+	if (!(flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) &&
 	    (vma->node.start + vma->node.size - 1) >> 32)
 		return true;
 
 	return false;
 }
 
-static inline void
+static inline bool
 eb_pin_vma(struct i915_execbuffer *eb,
-	   struct drm_i915_gem_exec_object2 *entry,
+	   const struct drm_i915_gem_exec_object2 *entry,
 	   struct i915_vma *vma)
 {
-	u64 flags;
+	unsigned int exec_flags = *vma->exec_flags;
+	u64 pin_flags;
 
 	if (vma->node.size)
-		flags = vma->node.start;
+		pin_flags = vma->node.start;
 	else
-		flags = entry->offset & PIN_OFFSET_MASK;
+		pin_flags = entry->offset & PIN_OFFSET_MASK;
 
-	flags |= PIN_USER | PIN_NOEVICT | PIN_OFFSET_FIXED;
-	if (unlikely(entry->flags & EXEC_OBJECT_NEEDS_GTT))
-		flags |= PIN_GLOBAL;
+	pin_flags |= PIN_USER | PIN_NOEVICT | PIN_OFFSET_FIXED;
+	if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_GTT))
+		pin_flags |= PIN_GLOBAL;
 
-	if (unlikely(i915_vma_pin(vma, 0, 0, flags)))
-		return;
+	if (unlikely(i915_vma_pin(vma, 0, 0, pin_flags)))
+		return false;
 
-	if (unlikely(entry->flags & EXEC_OBJECT_NEEDS_FENCE)) {
+	if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_FENCE)) {
 		if (unlikely(i915_vma_get_fence(vma))) {
 			i915_vma_unpin(vma);
-			return;
+			return false;
 		}
 
 		if (i915_vma_pin_fence(vma))
-			entry->flags |= __EXEC_OBJECT_HAS_FENCE;
+			exec_flags |= __EXEC_OBJECT_HAS_FENCE;
 	}
 
-	entry->flags |= __EXEC_OBJECT_HAS_PIN;
+	*vma->exec_flags = exec_flags | __EXEC_OBJECT_HAS_PIN;
+	return !eb_vma_misplaced(entry, vma, exec_flags);
 }
 
-static inline void
-__eb_unreserve_vma(struct i915_vma *vma,
-		   const struct drm_i915_gem_exec_object2 *entry)
+static inline void __eb_unreserve_vma(struct i915_vma *vma, unsigned int flags)
 {
-	GEM_BUG_ON(!(entry->flags & __EXEC_OBJECT_HAS_PIN));
+	GEM_BUG_ON(!(flags & __EXEC_OBJECT_HAS_PIN));
 
-	if (unlikely(entry->flags & __EXEC_OBJECT_HAS_FENCE))
+	if (unlikely(flags & __EXEC_OBJECT_HAS_FENCE))
 		i915_vma_unpin_fence(vma);
 
 	__i915_vma_unpin(vma);
 }
 
 static inline void
-eb_unreserve_vma(struct i915_vma *vma,
-		 struct drm_i915_gem_exec_object2 *entry)
+eb_unreserve_vma(struct i915_vma *vma, unsigned int *flags)
 {
-	if (!(entry->flags & __EXEC_OBJECT_HAS_PIN))
+	if (!(*flags & __EXEC_OBJECT_HAS_PIN))
 		return;
 
-	__eb_unreserve_vma(vma, entry);
-	entry->flags &= ~__EXEC_OBJECT_RESERVED;
+	__eb_unreserve_vma(vma, *flags);
+	*flags &= ~__EXEC_OBJECT_RESERVED;
 }
 
 static int
@@ -427,7 +421,7 @@ eb_validate_vma(struct i915_execbuffer *eb,
 		entry->pad_to_size = 0;
 	}
 
-	if (unlikely(vma->exec_entry)) {
+	if (unlikely(vma->exec_flags)) {
 		DRM_DEBUG("Object [handle %d, index %d] appears more than once in object list\n",
 			  entry->handle, (int)(entry - eb->exec));
 		return -EINVAL;
@@ -440,14 +434,25 @@ eb_validate_vma(struct i915_execbuffer *eb,
 	 */
 	entry->offset = gen8_noncanonical_addr(entry->offset);
 
+	if (!eb->reloc_cache.has_fence) {
+		entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
+	} else {
+		if ((entry->flags & EXEC_OBJECT_NEEDS_FENCE ||
+		     eb->reloc_cache.needs_unfenced) &&
+		    i915_gem_object_is_tiled(vma->obj))
+			entry->flags |= EXEC_OBJECT_NEEDS_GTT | __EXEC_OBJECT_NEEDS_MAP;
+	}
+
+	if (!(entry->flags & EXEC_OBJECT_PINNED))
+		entry->flags |= eb->context_flags;
+
 	return 0;
 }
 
 static int
-eb_add_vma(struct i915_execbuffer *eb,
-	   struct drm_i915_gem_exec_object2 *entry,
-	   struct i915_vma *vma)
+eb_add_vma(struct i915_execbuffer *eb, unsigned int i, struct i915_vma *vma)
 {
+	struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
 	int err;
 
 	GEM_BUG_ON(i915_vma_is_closed(vma));
@@ -468,40 +473,28 @@ eb_add_vma(struct i915_execbuffer *eb,
 	if (entry->relocation_count)
 		list_add_tail(&vma->reloc_link, &eb->relocs);
 
-	if (!eb->reloc_cache.has_fence) {
-		entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
-	} else {
-		if ((entry->flags & EXEC_OBJECT_NEEDS_FENCE ||
-		     eb->reloc_cache.needs_unfenced) &&
-		    i915_gem_object_is_tiled(vma->obj))
-			entry->flags |= EXEC_OBJECT_NEEDS_GTT | __EXEC_OBJECT_NEEDS_MAP;
-	}
-
-	if (!(entry->flags & EXEC_OBJECT_PINNED))
-		entry->flags |= eb->context_flags;
-
 	/*
 	 * Stash a pointer from the vma to execobj, so we can query its flags,
 	 * size, alignment etc as provided by the user. Also we stash a pointer
 	 * to the vma inside the execobj so that we can use a direct lookup
 	 * to find the right target VMA when doing relocations.
 	 */
-	vma->exec_entry = entry;
-	__exec_to_vma(entry) = (uintptr_t)vma;
+	eb->vma[i] = vma;
+	eb->flags[i] = entry->flags;
+	vma->exec_flags = &eb->flags[i];
 
 	err = 0;
-	eb_pin_vma(eb, entry, vma);
-	if (eb_vma_misplaced(entry, vma)) {
-		eb_unreserve_vma(vma, entry);
-
-		list_add_tail(&vma->exec_link, &eb->unbound);
-		if (drm_mm_node_allocated(&vma->node))
-			err = i915_vma_unbind(vma);
-	} else {
+	if (eb_pin_vma(eb, entry, vma)) {
 		if (entry->offset != vma->node.start) {
 			entry->offset = vma->node.start | UPDATE;
 			eb->args->flags |= __EXEC_HAS_RELOC;
 		}
+	} else {
+		eb_unreserve_vma(vma, vma->exec_flags);
+
+		list_add_tail(&vma->exec_link, &eb->unbound);
+		if (drm_mm_node_allocated(&vma->node))
+			err = i915_vma_unbind(vma);
 	}
 	return err;
 }
@@ -526,32 +519,35 @@ static inline int use_cpu_reloc(const struct reloc_cache *cache,
 static int eb_reserve_vma(const struct i915_execbuffer *eb,
 			  struct i915_vma *vma)
 {
-	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
-	u64 flags;
+	struct drm_i915_gem_exec_object2 *entry = exec_entry(eb, vma);
+	unsigned int exec_flags = *vma->exec_flags;
+	u64 pin_flags;
 	int err;
 
-	flags = PIN_USER | PIN_NONBLOCK;
-	if (entry->flags & EXEC_OBJECT_NEEDS_GTT)
-		flags |= PIN_GLOBAL;
+	pin_flags = PIN_USER | PIN_NONBLOCK;
+	if (exec_flags & EXEC_OBJECT_NEEDS_GTT)
+		pin_flags |= PIN_GLOBAL;
 
 	/*
 	 * Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset,
 	 * limit address to the first 4GBs for unflagged objects.
 	 */
-	if (!(entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS))
-		flags |= PIN_ZONE_4G;
+	if (!(exec_flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS))
+		pin_flags |= PIN_ZONE_4G;
 
-	if (entry->flags & __EXEC_OBJECT_NEEDS_MAP)
-		flags |= PIN_MAPPABLE;
+	if (exec_flags & __EXEC_OBJECT_NEEDS_MAP)
+		pin_flags |= PIN_MAPPABLE;
 
-	if (entry->flags & EXEC_OBJECT_PINNED) {
-		flags |= entry->offset | PIN_OFFSET_FIXED;
-		flags &= ~PIN_NONBLOCK; /* force overlapping PINNED checks */
-	} else if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS) {
-		flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
+	if (exec_flags & EXEC_OBJECT_PINNED) {
+		pin_flags |= entry->offset | PIN_OFFSET_FIXED;
+		pin_flags &= ~PIN_NONBLOCK; /* force overlapping checks */
+	} else if (exec_flags & __EXEC_OBJECT_NEEDS_BIAS) {
+		pin_flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
 	}
 
-	err = i915_vma_pin(vma, entry->pad_to_size, entry->alignment, flags);
+	err = i915_vma_pin(vma,
+			   entry->pad_to_size, entry->alignment,
+			   pin_flags);
 	if (err)
 		return err;
 
@@ -560,7 +556,7 @@ static int eb_reserve_vma(const struct i915_execbuffer *eb,
 		eb->args->flags |= __EXEC_HAS_RELOC;
 	}
 
-	if (unlikely(entry->flags & EXEC_OBJECT_NEEDS_FENCE)) {
+	if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_FENCE)) {
 		err = i915_vma_get_fence(vma);
 		if (unlikely(err)) {
 			i915_vma_unpin(vma);
@@ -568,11 +564,11 @@ static int eb_reserve_vma(const struct i915_execbuffer *eb,
 		}
 
 		if (i915_vma_pin_fence(vma))
-			entry->flags |= __EXEC_OBJECT_HAS_FENCE;
+			exec_flags |= __EXEC_OBJECT_HAS_FENCE;
 	}
 
-	entry->flags |= __EXEC_OBJECT_HAS_PIN;
-	GEM_BUG_ON(eb_vma_misplaced(entry, vma));
+	*vma->exec_flags = exec_flags | __EXEC_OBJECT_HAS_PIN;
+	GEM_BUG_ON(eb_vma_misplaced(entry, vma, exec_flags));
 
 	return 0;
 }
@@ -614,18 +610,18 @@ static int eb_reserve(struct i915_execbuffer *eb)
 		INIT_LIST_HEAD(&eb->unbound);
 		INIT_LIST_HEAD(&last);
 		for (i = 0; i < count; i++) {
-			struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
+			unsigned int flags = eb->flags[i];
+			struct i915_vma *vma = eb->vma[i];
 
-			if (entry->flags & EXEC_OBJECT_PINNED &&
-			    entry->flags & __EXEC_OBJECT_HAS_PIN)
+			if (flags & EXEC_OBJECT_PINNED &&
+			    flags & __EXEC_OBJECT_HAS_PIN)
 				continue;
 
-			vma = exec_to_vma(entry);
-			eb_unreserve_vma(vma, entry);
+			eb_unreserve_vma(vma, &eb->flags[i]);
 
-			if (entry->flags & EXEC_OBJECT_PINNED)
+			if (flags & EXEC_OBJECT_PINNED)
 				list_add(&vma->exec_link, &eb->unbound);
-			else if (entry->flags & __EXEC_OBJECT_NEEDS_MAP)
+			else if (flags & __EXEC_OBJECT_NEEDS_MAP)
 				list_add_tail(&vma->exec_link, &eb->unbound);
 			else
 				list_add_tail(&vma->exec_link, &last);
@@ -649,19 +645,6 @@ static int eb_reserve(struct i915_execbuffer *eb)
 	} while (1);
 }
 
-static inline struct hlist_head *
-ht_head(const  struct i915_gem_context_vma_lut *lut, u32 handle)
-{
-	return &lut->ht[hash_32(handle, lut->ht_bits)];
-}
-
-static inline bool
-ht_needs_resize(const struct i915_gem_context_vma_lut *lut)
-{
-	return (4*lut->ht_count > 3*lut->ht_size ||
-		4*lut->ht_count + 1 < lut->ht_size);
-}
-
 static unsigned int eb_batch_index(const struct i915_execbuffer *eb)
 {
 	if (eb->args->flags & I915_EXEC_BATCH_FIRST)
@@ -675,16 +658,10 @@ static int eb_select_context(struct i915_execbuffer *eb)
 	struct i915_gem_context *ctx;
 
 	ctx = i915_gem_context_lookup(eb->file->driver_priv, eb->args->rsvd1);
-	if (unlikely(IS_ERR(ctx)))
-		return PTR_ERR(ctx);
-
-	if (unlikely(i915_gem_context_is_banned(ctx))) {
-		DRM_DEBUG("Context %u tried to submit while banned\n",
-			  ctx->user_handle);
-		return -EIO;
-	}
+	if (unlikely(!ctx))
+		return -ENOENT;
 
-	eb->ctx = i915_gem_context_get(ctx);
+	eb->ctx = ctx;
 	eb->vm = ctx->ppgtt ? &ctx->ppgtt->base : &eb->i915->ggtt.base;
 
 	eb->context_flags = 0;
@@ -696,132 +673,75 @@ static int eb_select_context(struct i915_execbuffer *eb)
 
 static int eb_lookup_vmas(struct i915_execbuffer *eb)
 {
-#define INTERMEDIATE BIT(0)
-	const unsigned int count = eb->buffer_count;
-	struct i915_gem_context_vma_lut *lut = &eb->ctx->vma_lut;
-	struct i915_vma *vma;
-	struct idr *idr;
+	struct radix_tree_root *handles_vma = &eb->ctx->handles_vma;
+	struct drm_i915_gem_object *uninitialized_var(obj);
 	unsigned int i;
-	int slow_pass = -1;
 	int err;
 
-	INIT_LIST_HEAD(&eb->relocs);
-	INIT_LIST_HEAD(&eb->unbound);
-
-	if (unlikely(lut->ht_size & I915_CTX_RESIZE_IN_PROGRESS))
-		flush_work(&lut->resize);
-	GEM_BUG_ON(lut->ht_size & I915_CTX_RESIZE_IN_PROGRESS);
-
-	for (i = 0; i < count; i++) {
-		__exec_to_vma(&eb->exec[i]) = 0;
-
-		hlist_for_each_entry(vma,
-				     ht_head(lut, eb->exec[i].handle),
-				     ctx_node) {
-			if (vma->ctx_handle != eb->exec[i].handle)
-				continue;
-
-			err = eb_add_vma(eb, &eb->exec[i], vma);
-			if (unlikely(err))
-				return err;
-
-			goto next_vma;
-		}
+	if (unlikely(i915_gem_context_is_closed(eb->ctx)))
+		return -ENOENT;
 
-		if (slow_pass < 0)
-			slow_pass = i;
-next_vma: ;
-	}
+	if (unlikely(i915_gem_context_is_banned(eb->ctx)))
+		return -EIO;
 
-	if (slow_pass < 0)
-		goto out;
+	INIT_LIST_HEAD(&eb->relocs);
+	INIT_LIST_HEAD(&eb->unbound);
 
-	spin_lock(&eb->file->table_lock);
-	/*
-	 * Grab a reference to the object and release the lock so we can lookup
-	 * or create the VMA without using GFP_ATOMIC
-	 */
-	idr = &eb->file->object_idr;
-	for (i = slow_pass; i < count; i++) {
-		struct drm_i915_gem_object *obj;
+	for (i = 0; i < eb->buffer_count; i++) {
+		u32 handle = eb->exec[i].handle;
+		struct i915_lut_handle *lut;
+		struct i915_vma *vma;
 
-		if (__exec_to_vma(&eb->exec[i]))
-			continue;
+		vma = radix_tree_lookup(handles_vma, handle);
+		if (likely(vma))
+			goto add_vma;
 
-		obj = to_intel_bo(idr_find(idr, eb->exec[i].handle));
+		obj = i915_gem_object_lookup(eb->file, handle);
 		if (unlikely(!obj)) {
-			spin_unlock(&eb->file->table_lock);
-			DRM_DEBUG("Invalid object handle %d at index %d\n",
-				  eb->exec[i].handle, i);
 			err = -ENOENT;
-			goto err;
+			goto err_vma;
 		}
 
-		__exec_to_vma(&eb->exec[i]) = INTERMEDIATE | (uintptr_t)obj;
-	}
-	spin_unlock(&eb->file->table_lock);
-
-	for (i = slow_pass; i < count; i++) {
-		struct drm_i915_gem_object *obj;
-
-		if (!(__exec_to_vma(&eb->exec[i]) & INTERMEDIATE))
-			continue;
-
-		/*
-		 * NOTE: We can leak any vmas created here when something fails
-		 * later on. But that's no issue since vma_unbind can deal with
-		 * vmas which are not actually bound. And since only
-		 * lookup_or_create exists as an interface to get at the vma
-		 * from the (obj, vm) we don't run the risk of creating
-		 * duplicated vmas for the same vm.
-		 */
-		obj = u64_to_ptr(typeof(*obj),
-				 __exec_to_vma(&eb->exec[i]) & ~INTERMEDIATE);
 		vma = i915_vma_instance(obj, eb->vm, NULL);
 		if (unlikely(IS_ERR(vma))) {
-			DRM_DEBUG("Failed to lookup VMA\n");
 			err = PTR_ERR(vma);
-			goto err;
+			goto err_obj;
 		}
 
-		/* First come, first served */
-		if (!vma->ctx) {
-			vma->ctx = eb->ctx;
-			vma->ctx_handle = eb->exec[i].handle;
-			hlist_add_head(&vma->ctx_node,
-				       ht_head(lut, eb->exec[i].handle));
-			lut->ht_count++;
-			lut->ht_size |= I915_CTX_RESIZE_IN_PROGRESS;
-			if (i915_vma_is_ggtt(vma)) {
-				GEM_BUG_ON(obj->vma_hashed);
-				obj->vma_hashed = vma;
-			}
+		lut = kmem_cache_alloc(eb->i915->luts, GFP_KERNEL);
+		if (unlikely(!lut)) {
+			err = -ENOMEM;
+			goto err_obj;
+		}
 
-			i915_vma_get(vma);
+		err = radix_tree_insert(handles_vma, handle, vma);
+		if (unlikely(err)) {
+			kfree(lut);
+			goto err_obj;
 		}
 
-		err = eb_add_vma(eb, &eb->exec[i], vma);
-		if (unlikely(err))
-			goto err;
+		vma->open_count++;
+		list_add(&lut->obj_link, &obj->lut_list);
+		list_add(&lut->ctx_link, &eb->ctx->handles_list);
+		lut->ctx = eb->ctx;
+		lut->handle = handle;
 
-		/* Only after we validated the user didn't use our bits */
-		if (vma->ctx != eb->ctx) {
-			i915_vma_get(vma);
-			eb->exec[i].flags |= __EXEC_OBJECT_HAS_REF;
-		}
-	}
+		/* transfer ref to ctx */
+		obj = NULL;
 
-	if (lut->ht_size & I915_CTX_RESIZE_IN_PROGRESS) {
-		if (ht_needs_resize(lut))
-			queue_work(system_highpri_wq, &lut->resize);
-		else
-			lut->ht_size &= ~I915_CTX_RESIZE_IN_PROGRESS;
+add_vma:
+		err = eb_add_vma(eb, i, vma);
+		if (unlikely(err))
+			goto err_obj;
+
+		GEM_BUG_ON(vma != eb->vma[i]);
+		GEM_BUG_ON(vma->exec_flags != &eb->flags[i]);
 	}
 
-out:
 	/* take note of the batch buffer before we might reorder the lists */
 	i = eb_batch_index(eb);
-	eb->batch = exec_to_vma(&eb->exec[i]);
+	eb->batch = eb->vma[i];
+	GEM_BUG_ON(eb->batch->exec_flags != &eb->flags[i]);
 
 	/*
 	 * SNA is doing fancy tricks with compressing batch buffers, which leads
@@ -832,22 +752,20 @@ out:
 	 * Note that actual hangs have only been observed on gen7, but for
 	 * paranoia do it everywhere.
 	 */
-	if (!(eb->exec[i].flags & EXEC_OBJECT_PINNED))
-		eb->exec[i].flags |= __EXEC_OBJECT_NEEDS_BIAS;
+	if (!(eb->flags[i] & EXEC_OBJECT_PINNED))
+		eb->flags[i] |= __EXEC_OBJECT_NEEDS_BIAS;
 	if (eb->reloc_cache.has_fence)
-		eb->exec[i].flags |= EXEC_OBJECT_NEEDS_FENCE;
+		eb->flags[i] |= EXEC_OBJECT_NEEDS_FENCE;
 
 	eb->args->flags |= __EXEC_VALIDATED;
 	return eb_reserve(eb);
 
-err:
-	for (i = slow_pass; i < count; i++) {
-		if (__exec_to_vma(&eb->exec[i]) & INTERMEDIATE)
-			__exec_to_vma(&eb->exec[i]) = 0;
-	}
-	lut->ht_size &= ~I915_CTX_RESIZE_IN_PROGRESS;
+err_obj:
+	if (obj)
+		i915_gem_object_put(obj);
+err_vma:
+	eb->vma[i] = NULL;
 	return err;
-#undef INTERMEDIATE
 }
 
 static struct i915_vma *
@@ -856,7 +774,7 @@ eb_get_vma(const struct i915_execbuffer *eb, unsigned long handle)
 	if (eb->lut_size < 0) {
 		if (handle >= -eb->lut_size)
 			return NULL;
-		return exec_to_vma(&eb->exec[handle]);
+		return eb->vma[handle];
 	} else {
 		struct hlist_head *head;
 		struct i915_vma *vma;
@@ -876,24 +794,21 @@ static void eb_release_vmas(const struct i915_execbuffer *eb)
 	unsigned int i;
 
 	for (i = 0; i < count; i++) {
-		struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
-		struct i915_vma *vma = exec_to_vma(entry);
+		struct i915_vma *vma = eb->vma[i];
+		unsigned int flags = eb->flags[i];
 
 		if (!vma)
-			continue;
+			break;
 
-		GEM_BUG_ON(vma->exec_entry != entry);
-		vma->exec_entry = NULL;
-		__exec_to_vma(entry) = 0;
+		GEM_BUG_ON(vma->exec_flags != &eb->flags[i]);
+		vma->exec_flags = NULL;
+		eb->vma[i] = NULL;
 
-		if (entry->flags & __EXEC_OBJECT_HAS_PIN)
-			__eb_unreserve_vma(vma, entry);
+		if (flags & __EXEC_OBJECT_HAS_PIN)
+			__eb_unreserve_vma(vma, flags);
 
-		if (entry->flags & __EXEC_OBJECT_HAS_REF)
+		if (flags & __EXEC_OBJECT_HAS_REF)
 			i915_vma_put(vma);
-
-		entry->flags &=
-			~(__EXEC_OBJECT_RESERVED | __EXEC_OBJECT_HAS_REF);
 	}
 }
 
@@ -1156,7 +1071,9 @@ static int __reloc_gpu_alloc(struct i915_execbuffer *eb,
 		return PTR_ERR(obj);
 
 	cmd = i915_gem_object_pin_map(obj,
-				      cache->has_llc ? I915_MAP_WB : I915_MAP_WC);
+				      cache->has_llc ?
+				      I915_MAP_FORCE_WB :
+				      I915_MAP_FORCE_WC);
 	i915_gem_object_unpin_pages(obj);
 	if (IS_ERR(cmd))
 		return PTR_ERR(cmd);
@@ -1266,7 +1183,9 @@ relocate_entry(struct i915_vma *vma,
 
 	if (!eb->reloc_cache.vaddr &&
 	    (DBG_FORCE_RELOC == FORCE_GPU_RELOC ||
-	     !reservation_object_test_signaled_rcu(vma->resv, true))) {
+	     !reservation_object_test_signaled_rcu(vma->resv, true)) &&
+	    __intel_engine_can_store_dword(eb->reloc_cache.gen,
+					   eb->engine->class)) {
 		const unsigned int gen = eb->reloc_cache.gen;
 		unsigned int len;
 		u32 *batch;
@@ -1276,10 +1195,8 @@ relocate_entry(struct i915_vma *vma,
 			len = offset & 7 ? 8 : 5;
 		else if (gen >= 4)
 			len = 4;
-		else if (gen >= 3)
+		else
 			len = 3;
-		else /* On gen2 MI_STORE_DWORD_IMM uses a physical address */
-			goto repeat;
 
 		batch = reloc_gpu(eb, vma, len);
 		if (IS_ERR(batch))
@@ -1382,7 +1299,7 @@ eb_relocate_entry(struct i915_execbuffer *eb,
 	}
 
 	if (reloc->write_domain) {
-		target->exec_entry->flags |= EXEC_OBJECT_WRITE;
+		*target->exec_flags |= EXEC_OBJECT_WRITE;
 
 		/*
 		 * Sandybridge PPGTT errata: We need a global gtt mapping
@@ -1432,9 +1349,9 @@ eb_relocate_entry(struct i915_execbuffer *eb,
 	 * patching using the GPU (though that should be serialised by the
 	 * timeline). To be completely sure, and since we are required to
 	 * do relocations we are already stalling, disable the user's opt
-	 * of our synchronisation.
+	 * out of our synchronisation.
 	 */
-	vma->exec_entry->flags &= ~EXEC_OBJECT_ASYNC;
+	*vma->exec_flags &= ~EXEC_OBJECT_ASYNC;
 
 	/* and update the user's relocation entry */
 	return relocate_entry(vma, reloc, eb, target);
@@ -1445,7 +1362,7 @@ static int eb_relocate_vma(struct i915_execbuffer *eb, struct i915_vma *vma)
 #define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
 	struct drm_i915_gem_relocation_entry stack[N_RELOC(512)];
 	struct drm_i915_gem_relocation_entry __user *urelocs;
-	const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
+	const struct drm_i915_gem_exec_object2 *entry = exec_entry(eb, vma);
 	unsigned int remain;
 
 	urelocs = u64_to_user_ptr(entry->relocs_ptr);
@@ -1528,7 +1445,7 @@ out:
 static int
 eb_relocate_vma_slow(struct i915_execbuffer *eb, struct i915_vma *vma)
 {
-	const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
+	const struct drm_i915_gem_exec_object2 *entry = exec_entry(eb, vma);
 	struct drm_i915_gem_relocation_entry *relocs =
 		u64_to_ptr(typeof(*relocs), entry->relocs_ptr);
 	unsigned int i;
@@ -1598,7 +1515,7 @@ static int eb_copy_relocations(const struct i915_execbuffer *eb)
 		urelocs = u64_to_user_ptr(eb->exec[i].relocs_ptr);
 		size = nreloc * sizeof(*relocs);
 
-		relocs = kvmalloc_array(size, 1, GFP_TEMPORARY);
+		relocs = kvmalloc_array(size, 1, GFP_KERNEL);
 		if (!relocs) {
 			kvfree(relocs);
 			err = -ENOMEM;
@@ -1612,7 +1529,7 @@ static int eb_copy_relocations(const struct i915_execbuffer *eb)
 				min_t(u64, BIT_ULL(31), size - copied);
 
 			if (__copy_from_user((char *)relocs + copied,
-					     (char *)urelocs + copied,
+					     (char __user *)urelocs + copied,
 					     len)) {
 				kvfree(relocs);
 				err = -EFAULT;
@@ -1732,6 +1649,8 @@ repeat:
 	if (err)
 		goto err;
 
+	GEM_BUG_ON(!eb->batch);
+
 	list_for_each_entry(vma, &eb->relocs, reloc_link) {
 		if (!have_copy) {
 			pagefault_disable();
@@ -1825,11 +1744,11 @@ static int eb_move_to_gpu(struct i915_execbuffer *eb)
 	int err;
 
 	for (i = 0; i < count; i++) {
-		struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
-		struct i915_vma *vma = exec_to_vma(entry);
+		unsigned int flags = eb->flags[i];
+		struct i915_vma *vma = eb->vma[i];
 		struct drm_i915_gem_object *obj = vma->obj;
 
-		if (entry->flags & EXEC_OBJECT_CAPTURE) {
+		if (flags & EXEC_OBJECT_CAPTURE) {
 			struct i915_gem_capture_list *capture;
 
 			capture = kmalloc(sizeof(*capture), GFP_KERNEL);
@@ -1837,35 +1756,47 @@ static int eb_move_to_gpu(struct i915_execbuffer *eb)
 				return -ENOMEM;
 
 			capture->next = eb->request->capture_list;
-			capture->vma = vma;
+			capture->vma = eb->vma[i];
 			eb->request->capture_list = capture;
 		}
 
-		if (unlikely(obj->cache_dirty && !obj->cache_coherent)) {
+		/*
+		 * If the GPU is not _reading_ through the CPU cache, we need
+		 * to make sure that any writes (both previous GPU writes from
+		 * before a change in snooping levels and normal CPU writes)
+		 * caught in that cache are flushed to main memory.
+		 *
+		 * We want to say
+		 *   obj->cache_dirty &&
+		 *   !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)
+		 * but gcc's optimiser doesn't handle that as well and emits
+		 * two jumps instead of one. Maybe one day...
+		 */
+		if (unlikely(obj->cache_dirty & ~obj->cache_coherent)) {
 			if (i915_gem_clflush_object(obj, 0))
-				entry->flags &= ~EXEC_OBJECT_ASYNC;
+				flags &= ~EXEC_OBJECT_ASYNC;
 		}
 
-		if (entry->flags & EXEC_OBJECT_ASYNC)
-			goto skip_flushes;
+		if (flags & EXEC_OBJECT_ASYNC)
+			continue;
 
 		err = i915_gem_request_await_object
-			(eb->request, obj, entry->flags & EXEC_OBJECT_WRITE);
+			(eb->request, obj, flags & EXEC_OBJECT_WRITE);
 		if (err)
 			return err;
-
-skip_flushes:
-		i915_vma_move_to_active(vma, eb->request, entry->flags);
-		__eb_unreserve_vma(vma, entry);
-		vma->exec_entry = NULL;
 	}
 
 	for (i = 0; i < count; i++) {
-		const struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
-		struct i915_vma *vma = exec_to_vma(entry);
+		unsigned int flags = eb->flags[i];
+		struct i915_vma *vma = eb->vma[i];
 
-		eb_export_fence(vma, eb->request, entry->flags);
-		if (unlikely(entry->flags & __EXEC_OBJECT_HAS_REF))
+		i915_vma_move_to_active(vma, eb->request, flags);
+		eb_export_fence(vma, eb->request, flags);
+
+		__eb_unreserve_vma(vma, flags);
+		vma->exec_flags = NULL;
+
+		if (unlikely(flags & __EXEC_OBJECT_HAS_REF))
 			i915_vma_put(vma);
 	}
 	eb->exec = NULL;
@@ -1883,8 +1814,10 @@ static bool i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
 		return false;
 
 	/* Kernel clipping was a DRI1 misfeature */
-	if (exec->num_cliprects || exec->cliprects_ptr)
-		return false;
+	if (!(exec->flags & I915_EXEC_FENCE_ARRAY)) {
+		if (exec->num_cliprects || exec->cliprects_ptr)
+			return false;
+	}
 
 	if (exec->DR4 == 0xffffffff) {
 		DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
@@ -1992,11 +1925,11 @@ static struct i915_vma *eb_parse(struct i915_execbuffer *eb, bool is_master)
 	if (IS_ERR(vma))
 		goto out;
 
-	vma->exec_entry =
-		memset(&eb->exec[eb->buffer_count++],
-		       0, sizeof(*vma->exec_entry));
-	vma->exec_entry->flags = __EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_REF;
-	__exec_to_vma(vma->exec_entry) = (uintptr_t)i915_vma_get(vma);
+	eb->vma[eb->buffer_count] = i915_vma_get(vma);
+	eb->flags[eb->buffer_count] =
+		__EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_REF;
+	vma->exec_flags = &eb->flags[eb->buffer_count];
+	eb->buffer_count++;
 
 out:
 	i915_gem_object_unpin_pages(shadow_batch_obj);
@@ -2115,11 +2048,129 @@ eb_select_engine(struct drm_i915_private *dev_priv,
 	return engine;
 }
 
+static void
+__free_fence_array(struct drm_syncobj **fences, unsigned int n)
+{
+	while (n--)
+		drm_syncobj_put(ptr_mask_bits(fences[n], 2));
+	kvfree(fences);
+}
+
+static struct drm_syncobj **
+get_fence_array(struct drm_i915_gem_execbuffer2 *args,
+		struct drm_file *file)
+{
+	const unsigned int nfences = args->num_cliprects;
+	struct drm_i915_gem_exec_fence __user *user;
+	struct drm_syncobj **fences;
+	unsigned int n;
+	int err;
+
+	if (!(args->flags & I915_EXEC_FENCE_ARRAY))
+		return NULL;
+
+	if (nfences > SIZE_MAX / sizeof(*fences))
+		return ERR_PTR(-EINVAL);
+
+	user = u64_to_user_ptr(args->cliprects_ptr);
+	if (!access_ok(VERIFY_READ, user, nfences * 2 * sizeof(u32)))
+		return ERR_PTR(-EFAULT);
+
+	fences = kvmalloc_array(args->num_cliprects, sizeof(*fences),
+				__GFP_NOWARN | GFP_KERNEL);
+	if (!fences)
+		return ERR_PTR(-ENOMEM);
+
+	for (n = 0; n < nfences; n++) {
+		struct drm_i915_gem_exec_fence fence;
+		struct drm_syncobj *syncobj;
+
+		if (__copy_from_user(&fence, user++, sizeof(fence))) {
+			err = -EFAULT;
+			goto err;
+		}
+
+		syncobj = drm_syncobj_find(file, fence.handle);
+		if (!syncobj) {
+			DRM_DEBUG("Invalid syncobj handle provided\n");
+			err = -ENOENT;
+			goto err;
+		}
+
+		fences[n] = ptr_pack_bits(syncobj, fence.flags, 2);
+	}
+
+	return fences;
+
+err:
+	__free_fence_array(fences, n);
+	return ERR_PTR(err);
+}
+
+static void
+put_fence_array(struct drm_i915_gem_execbuffer2 *args,
+		struct drm_syncobj **fences)
+{
+	if (fences)
+		__free_fence_array(fences, args->num_cliprects);
+}
+
+static int
+await_fence_array(struct i915_execbuffer *eb,
+		  struct drm_syncobj **fences)
+{
+	const unsigned int nfences = eb->args->num_cliprects;
+	unsigned int n;
+	int err;
+
+	for (n = 0; n < nfences; n++) {
+		struct drm_syncobj *syncobj;
+		struct dma_fence *fence;
+		unsigned int flags;
+
+		syncobj = ptr_unpack_bits(fences[n], &flags, 2);
+		if (!(flags & I915_EXEC_FENCE_WAIT))
+			continue;
+
+		fence = drm_syncobj_fence_get(syncobj);
+		if (!fence)
+			return -EINVAL;
+
+		err = i915_gem_request_await_dma_fence(eb->request, fence);
+		dma_fence_put(fence);
+		if (err < 0)
+			return err;
+	}
+
+	return 0;
+}
+
+static void
+signal_fence_array(struct i915_execbuffer *eb,
+		   struct drm_syncobj **fences)
+{
+	const unsigned int nfences = eb->args->num_cliprects;
+	struct dma_fence * const fence = &eb->request->fence;
+	unsigned int n;
+
+	for (n = 0; n < nfences; n++) {
+		struct drm_syncobj *syncobj;
+		unsigned int flags;
+
+		syncobj = ptr_unpack_bits(fences[n], &flags, 2);
+		if (!(flags & I915_EXEC_FENCE_SIGNAL))
+			continue;
+
+		drm_syncobj_replace_fence(syncobj, fence);
+	}
+}
+
 static int
 i915_gem_do_execbuffer(struct drm_device *dev,
 		       struct drm_file *file,
 		       struct drm_i915_gem_execbuffer2 *args,
-		       struct drm_i915_gem_exec_object2 *exec)
+		       struct drm_i915_gem_exec_object2 *exec,
+		       struct drm_syncobj **fences)
 {
 	struct i915_execbuffer eb;
 	struct dma_fence *in_fence = NULL;
@@ -2135,8 +2186,12 @@ i915_gem_do_execbuffer(struct drm_device *dev,
 	eb.args = args;
 	if (DBG_FORCE_RELOC || !(args->flags & I915_EXEC_NO_RELOC))
 		args->flags |= __EXEC_HAS_RELOC;
+
 	eb.exec = exec;
-	eb.ctx = NULL;
+	eb.vma = (struct i915_vma **)(exec + args->buffer_count + 1);
+	eb.vma[0] = NULL;
+	eb.flags = (unsigned int *)(eb.vma + args->buffer_count + 1);
+
 	eb.invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
 	if (USES_FULL_PPGTT(eb.i915))
 		eb.invalid_flags |= EXEC_OBJECT_NEEDS_GTT;
@@ -2194,6 +2249,10 @@ i915_gem_do_execbuffer(struct drm_device *dev,
 
 	GEM_BUG_ON(!eb.lut_size);
 
+	err = eb_select_context(&eb);
+	if (unlikely(err))
+		goto err_destroy;
+
 	/*
 	 * Take a local wakeref for preparing to dispatch the execbuf as
 	 * we expect to access the hardware fairly frequently in the
@@ -2202,14 +2261,11 @@ i915_gem_do_execbuffer(struct drm_device *dev,
 	 * 100ms.
 	 */
 	intel_runtime_pm_get(eb.i915);
+
 	err = i915_mutex_lock_interruptible(dev);
 	if (err)
 		goto err_rpm;
 
-	err = eb_select_context(&eb);
-	if (unlikely(err))
-		goto err_unlock;
-
 	err = eb_relocate(&eb);
 	if (err) {
 		/*
@@ -2223,7 +2279,7 @@ i915_gem_do_execbuffer(struct drm_device *dev,
 		goto err_vma;
 	}
 
-	if (unlikely(eb.batch->exec_entry->flags & EXEC_OBJECT_WRITE)) {
+	if (unlikely(*eb.batch->exec_flags & EXEC_OBJECT_WRITE)) {
 		DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
 		err = -EINVAL;
 		goto err_vma;
@@ -2305,6 +2361,12 @@ i915_gem_do_execbuffer(struct drm_device *dev,
 			goto err_request;
 	}
 
+	if (fences) {
+		err = await_fence_array(&eb, fences);
+		if (err)
+			goto err_request;
+	}
+
 	if (out_fence_fd != -1) {
 		out_fence = sync_file_create(&eb.request->fence);
 		if (!out_fence) {
@@ -2328,6 +2390,9 @@ err_request:
 	__i915_add_request(eb.request, err == 0);
 	add_to_client(eb.request, file);
 
+	if (fences)
+		signal_fence_array(&eb, fences);
+
 	if (out_fence) {
 		if (err == 0) {
 			fd_install(out_fence_fd, out_fence->file);
@@ -2345,11 +2410,11 @@ err_batch_unpin:
 err_vma:
 	if (eb.exec)
 		eb_release_vmas(&eb);
-	i915_gem_context_put(eb.ctx);
-err_unlock:
 	mutex_unlock(&dev->struct_mutex);
 err_rpm:
 	intel_runtime_pm_put(eb.i915);
+	i915_gem_context_put(eb.ctx);
+err_destroy:
 	eb_destroy(&eb);
 err_out_fence:
 	if (out_fence_fd != -1)
@@ -2367,7 +2432,9 @@ int
 i915_gem_execbuffer(struct drm_device *dev, void *data,
 		    struct drm_file *file)
 {
-	const size_t sz = sizeof(struct drm_i915_gem_exec_object2);
+	const size_t sz = (sizeof(struct drm_i915_gem_exec_object2) +
+			   sizeof(struct i915_vma *) +
+			   sizeof(unsigned int));
 	struct drm_i915_gem_execbuffer *args = data;
 	struct drm_i915_gem_execbuffer2 exec2;
 	struct drm_i915_gem_exec_object *exec_list = NULL;
@@ -2396,9 +2463,9 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
 
 	/* Copy in the exec list from userland */
 	exec_list = kvmalloc_array(args->buffer_count, sizeof(*exec_list),
-				   __GFP_NOWARN | GFP_TEMPORARY);
+				   __GFP_NOWARN | GFP_KERNEL);
 	exec2_list = kvmalloc_array(args->buffer_count + 1, sz,
-				    __GFP_NOWARN | GFP_TEMPORARY);
+				    __GFP_NOWARN | GFP_KERNEL);
 	if (exec_list == NULL || exec2_list == NULL) {
 		DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
 			  args->buffer_count);
@@ -2429,7 +2496,7 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
 			exec2_list[i].flags = 0;
 	}
 
-	err = i915_gem_do_execbuffer(dev, file, &exec2, exec2_list);
+	err = i915_gem_do_execbuffer(dev, file, &exec2, exec2_list, NULL);
 	if (exec2.flags & __EXEC_HAS_RELOC) {
 		struct drm_i915_gem_exec_object __user *user_exec_list =
 			u64_to_user_ptr(args->buffers_ptr);
@@ -2458,9 +2525,12 @@ int
 i915_gem_execbuffer2(struct drm_device *dev, void *data,
 		     struct drm_file *file)
 {
-	const size_t sz = sizeof(struct drm_i915_gem_exec_object2);
+	const size_t sz = (sizeof(struct drm_i915_gem_exec_object2) +
+			   sizeof(struct i915_vma *) +
+			   sizeof(unsigned int));
 	struct drm_i915_gem_execbuffer2 *args = data;
 	struct drm_i915_gem_exec_object2 *exec2_list;
+	struct drm_syncobj **fences = NULL;
 	int err;
 
 	if (args->buffer_count < 1 || args->buffer_count > SIZE_MAX / sz - 1) {
@@ -2473,7 +2543,7 @@ i915_gem_execbuffer2(struct drm_device *dev, void *data,
 
 	/* Allocate an extra slot for use by the command parser */
 	exec2_list = kvmalloc_array(args->buffer_count + 1, sz,
-				    __GFP_NOWARN | GFP_TEMPORARY);
+				    __GFP_NOWARN | GFP_KERNEL);
 	if (exec2_list == NULL) {
 		DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
 			  args->buffer_count);
@@ -2487,7 +2557,15 @@ i915_gem_execbuffer2(struct drm_device *dev, void *data,
 		return -EFAULT;
 	}
 
-	err = i915_gem_do_execbuffer(dev, file, args, exec2_list);
+	if (args->flags & I915_EXEC_FENCE_ARRAY) {
+		fences = get_fence_array(args, file);
+		if (IS_ERR(fences)) {
+			kvfree(exec2_list);
+			return PTR_ERR(fences);
+		}
+	}
+
+	err = i915_gem_do_execbuffer(dev, file, args, exec2_list, fences);
 
 	/*
 	 * Now that we have begun execution of the batchbuffer, we ignore
@@ -2517,6 +2595,7 @@ end_user:
 	}
 
 	args->flags &= ~__I915_EXEC_UNKNOWN_FLAGS;
+	put_fence_array(args, fences);
 	kvfree(exec2_list);
 	return err;
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 61fc7e90a7da..e2410eb5d96e 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -144,9 +144,9 @@ int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv,
 	has_full_48bit_ppgtt = dev_priv->info.has_full_48bit_ppgtt;
 
 	if (intel_vgpu_active(dev_priv)) {
-		/* emulation is too hard */
+		/* GVT-g has no support for 32bit ppgtt */
 		has_full_ppgtt = false;
-		has_full_48bit_ppgtt = false;
+		has_full_48bit_ppgtt = intel_vgpu_has_full_48bit_ppgtt(dev_priv);
 	}
 
 	if (!has_aliasing_ppgtt)
@@ -180,10 +180,15 @@ int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv,
 		return 0;
 	}
 
-	if (INTEL_GEN(dev_priv) >= 8 && i915.enable_execlists && has_full_ppgtt)
-		return has_full_48bit_ppgtt ? 3 : 2;
-	else
-		return has_aliasing_ppgtt ? 1 : 0;
+	if (INTEL_GEN(dev_priv) >= 8 && i915.enable_execlists) {
+		if (has_full_48bit_ppgtt)
+			return 3;
+
+		if (has_full_ppgtt)
+			return 2;
+	}
+
+	return has_aliasing_ppgtt ? 1 : 0;
 }
 
 static int ppgtt_bind_vma(struct i915_vma *vma,
@@ -207,8 +212,7 @@ static int ppgtt_bind_vma(struct i915_vma *vma,
 	if (vma->obj->gt_ro)
 		pte_flags |= PTE_READ_ONLY;
 
-	vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start,
-				cache_level, pte_flags);
+	vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
 
 	return 0;
 }
@@ -907,37 +911,35 @@ gen8_ppgtt_insert_pte_entries(struct i915_hw_ppgtt *ppgtt,
 }
 
 static void gen8_ppgtt_insert_3lvl(struct i915_address_space *vm,
-				   struct sg_table *pages,
-				   u64 start,
+				   struct i915_vma *vma,
 				   enum i915_cache_level cache_level,
 				   u32 unused)
 {
 	struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 	struct sgt_dma iter = {
-		.sg = pages->sgl,
+		.sg = vma->pages->sgl,
 		.dma = sg_dma_address(iter.sg),
 		.max = iter.dma + iter.sg->length,
 	};
-	struct gen8_insert_pte idx = gen8_insert_pte(start);
+	struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start);
 
 	gen8_ppgtt_insert_pte_entries(ppgtt, &ppgtt->pdp, &iter, &idx,
 				      cache_level);
 }
 
 static void gen8_ppgtt_insert_4lvl(struct i915_address_space *vm,
-				   struct sg_table *pages,
-				   u64 start,
+				   struct i915_vma *vma,
 				   enum i915_cache_level cache_level,
 				   u32 unused)
 {
 	struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 	struct sgt_dma iter = {
-		.sg = pages->sgl,
+		.sg = vma->pages->sgl,
 		.dma = sg_dma_address(iter.sg),
 		.max = iter.dma + iter.sg->length,
 	};
 	struct i915_page_directory_pointer **pdps = ppgtt->pml4.pdps;
-	struct gen8_insert_pte idx = gen8_insert_pte(start);
+	struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start);
 
 	while (gen8_ppgtt_insert_pte_entries(ppgtt, pdps[idx.pml4e++], &iter,
 					     &idx, cache_level))
@@ -1621,13 +1623,12 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
 }
 
 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
-				      struct sg_table *pages,
-				      u64 start,
+				      struct i915_vma *vma,
 				      enum i915_cache_level cache_level,
 				      u32 flags)
 {
 	struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
-	unsigned first_entry = start >> PAGE_SHIFT;
+	unsigned first_entry = vma->node.start >> PAGE_SHIFT;
 	unsigned act_pt = first_entry / GEN6_PTES;
 	unsigned act_pte = first_entry % GEN6_PTES;
 	const u32 pte_encode = vm->pte_encode(0, cache_level, flags);
@@ -1635,7 +1636,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
 	gen6_pte_t *vaddr;
 
 	vaddr = kmap_atomic_px(ppgtt->pd.page_table[act_pt]);
-	iter.sg = pages->sgl;
+	iter.sg = vma->pages->sgl;
 	iter.dma = sg_dma_address(iter.sg);
 	iter.max = iter.dma + iter.sg->length;
 	do {
@@ -2061,7 +2062,7 @@ int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj,
 		 */
 		GEM_BUG_ON(obj->mm.pages == pages);
 	} while (i915_gem_shrink(to_i915(obj->base.dev),
-				 obj->base.size >> PAGE_SHIFT,
+				 obj->base.size >> PAGE_SHIFT, NULL,
 				 I915_SHRINK_BOUND |
 				 I915_SHRINK_UNBOUND |
 				 I915_SHRINK_ACTIVE));
@@ -2090,8 +2091,7 @@ static void gen8_ggtt_insert_page(struct i915_address_space *vm,
 }
 
 static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
-				     struct sg_table *st,
-				     u64 start,
+				     struct i915_vma *vma,
 				     enum i915_cache_level level,
 				     u32 unused)
 {
@@ -2102,8 +2102,8 @@ static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
 	dma_addr_t addr;
 
 	gtt_entries = (gen8_pte_t __iomem *)ggtt->gsm;
-	gtt_entries += start >> PAGE_SHIFT;
-	for_each_sgt_dma(addr, sgt_iter, st)
+	gtt_entries += vma->node.start >> PAGE_SHIFT;
+	for_each_sgt_dma(addr, sgt_iter, vma->pages)
 		gen8_set_pte(gtt_entries++, pte_encode | addr);
 
 	wmb();
@@ -2137,17 +2137,16 @@ static void gen6_ggtt_insert_page(struct i915_address_space *vm,
  * mapped BAR (dev_priv->mm.gtt->gtt).
  */
 static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
-				     struct sg_table *st,
-				     u64 start,
+				     struct i915_vma *vma,
 				     enum i915_cache_level level,
 				     u32 flags)
 {
 	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
 	gen6_pte_t __iomem *entries = (gen6_pte_t __iomem *)ggtt->gsm;
-	unsigned int i = start >> PAGE_SHIFT;
+	unsigned int i = vma->node.start >> PAGE_SHIFT;
 	struct sgt_iter iter;
 	dma_addr_t addr;
-	for_each_sgt_dma(addr, iter, st)
+	for_each_sgt_dma(addr, iter, vma->pages)
 		iowrite32(vm->pte_encode(addr, level, flags), &entries[i++]);
 	wmb();
 
@@ -2229,8 +2228,7 @@ static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm,
 
 struct insert_entries {
 	struct i915_address_space *vm;
-	struct sg_table *st;
-	u64 start;
+	struct i915_vma *vma;
 	enum i915_cache_level level;
 };
 
@@ -2238,19 +2236,18 @@ static int bxt_vtd_ggtt_insert_entries__cb(void *_arg)
 {
 	struct insert_entries *arg = _arg;
 
-	gen8_ggtt_insert_entries(arg->vm, arg->st, arg->start, arg->level, 0);
+	gen8_ggtt_insert_entries(arg->vm, arg->vma, arg->level, 0);
 	bxt_vtd_ggtt_wa(arg->vm);
 
 	return 0;
 }
 
 static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm,
-					     struct sg_table *st,
-					     u64 start,
+					     struct i915_vma *vma,
 					     enum i915_cache_level level,
 					     u32 unused)
 {
-	struct insert_entries arg = { vm, st, start, level };
+	struct insert_entries arg = { vm, vma, level };
 
 	stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL);
 }
@@ -2316,15 +2313,15 @@ static void i915_ggtt_insert_page(struct i915_address_space *vm,
 }
 
 static void i915_ggtt_insert_entries(struct i915_address_space *vm,
-				     struct sg_table *pages,
-				     u64 start,
+				     struct i915_vma *vma,
 				     enum i915_cache_level cache_level,
 				     u32 unused)
 {
 	unsigned int flags = (cache_level == I915_CACHE_NONE) ?
 		AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
 
-	intel_gtt_insert_sg_entries(pages, start >> PAGE_SHIFT, flags);
+	intel_gtt_insert_sg_entries(vma->pages, vma->node.start >> PAGE_SHIFT,
+				    flags);
 }
 
 static void i915_ggtt_clear_range(struct i915_address_space *vm,
@@ -2353,8 +2350,7 @@ static int ggtt_bind_vma(struct i915_vma *vma,
 		pte_flags |= PTE_READ_ONLY;
 
 	intel_runtime_pm_get(i915);
-	vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start,
-				cache_level, pte_flags);
+	vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
 	intel_runtime_pm_put(i915);
 
 	/*
@@ -2407,16 +2403,13 @@ static int aliasing_gtt_bind_vma(struct i915_vma *vma,
 				goto err_pages;
 		}
 
-		appgtt->base.insert_entries(&appgtt->base,
-					    vma->pages, vma->node.start,
-					    cache_level, pte_flags);
+		appgtt->base.insert_entries(&appgtt->base, vma, cache_level,
+					    pte_flags);
 	}
 
 	if (flags & I915_VMA_GLOBAL_BIND) {
 		intel_runtime_pm_get(i915);
-		vma->vm->insert_entries(vma->vm,
-					vma->pages, vma->node.start,
-					cache_level, pte_flags);
+		vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
 		intel_runtime_pm_put(i915);
 	}
 
@@ -2749,6 +2742,24 @@ static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
 	return 0;
 }
 
+static void cnl_setup_private_ppat(struct drm_i915_private *dev_priv)
+{
+	/* XXX: spec is unclear if this is still needed for CNL+ */
+	if (!USES_PPGTT(dev_priv)) {
+		I915_WRITE(GEN10_PAT_INDEX(0), GEN8_PPAT_UC);
+		return;
+	}
+
+	I915_WRITE(GEN10_PAT_INDEX(0), GEN8_PPAT_WB | GEN8_PPAT_LLC);
+	I915_WRITE(GEN10_PAT_INDEX(1), GEN8_PPAT_WC | GEN8_PPAT_LLCELLC);
+	I915_WRITE(GEN10_PAT_INDEX(2), GEN8_PPAT_WT | GEN8_PPAT_LLCELLC);
+	I915_WRITE(GEN10_PAT_INDEX(3), GEN8_PPAT_UC);
+	I915_WRITE(GEN10_PAT_INDEX(4), GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0));
+	I915_WRITE(GEN10_PAT_INDEX(5), GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1));
+	I915_WRITE(GEN10_PAT_INDEX(6), GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2));
+	I915_WRITE(GEN10_PAT_INDEX(7), GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
+}
+
 /* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
  * bits. When using advanced contexts each context stores its own PAT, but
  * writing this data shouldn't be harmful even in those cases. */
@@ -2863,7 +2874,9 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt)
 
 	ggtt->base.total = (size / sizeof(gen8_pte_t)) << PAGE_SHIFT;
 
-	if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv))
+	if (INTEL_GEN(dev_priv) >= 10)
+		cnl_setup_private_ppat(dev_priv);
+	else if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv))
 		chv_setup_private_ppat(dev_priv);
 	else
 		bdw_setup_private_ppat(dev_priv);
@@ -3145,7 +3158,9 @@ void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv)
 	ggtt->base.closed = false;
 
 	if (INTEL_GEN(dev_priv) >= 8) {
-		if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv))
+		if (INTEL_GEN(dev_priv) >= 10)
+			cnl_setup_private_ppat(dev_priv);
+		else if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv))
 			chv_setup_private_ppat(dev_priv);
 		else
 			bdw_setup_private_ppat(dev_priv);
@@ -3216,7 +3231,7 @@ intel_rotate_pages(struct intel_rotation_info *rot_info,
 	/* Allocate a temporary list of source pages for random access. */
 	page_addr_list = kvmalloc_array(n_pages,
 					sizeof(dma_addr_t),
-					GFP_TEMPORARY);
+					GFP_KERNEL);
 	if (!page_addr_list)
 		return ERR_PTR(ret);
 
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index 1b2a56c3e5d3..b4e3aa7c0ce1 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -313,8 +313,7 @@ struct i915_address_space {
 			    enum i915_cache_level cache_level,
 			    u32 flags);
 	void (*insert_entries)(struct i915_address_space *vm,
-			       struct sg_table *st,
-			       u64 start,
+			       struct i915_vma *vma,
 			       enum i915_cache_level cache_level,
 			       u32 flags);
 	void (*cleanup)(struct i915_address_space *vm);
diff --git a/drivers/gpu/drm/i915/i915_gem_internal.c b/drivers/gpu/drm/i915/i915_gem_internal.c
index 568bf83af1f5..c1f64ddaf8aa 100644
--- a/drivers/gpu/drm/i915/i915_gem_internal.c
+++ b/drivers/gpu/drm/i915/i915_gem_internal.c
@@ -174,6 +174,7 @@ i915_gem_object_create_internal(struct drm_i915_private *i915,
 				phys_addr_t size)
 {
 	struct drm_i915_gem_object *obj;
+	unsigned int cache_level;
 
 	GEM_BUG_ON(!size);
 	GEM_BUG_ON(!IS_ALIGNED(size, PAGE_SIZE));
@@ -190,9 +191,9 @@ i915_gem_object_create_internal(struct drm_i915_private *i915,
 
 	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
 	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
-	obj->cache_level = HAS_LLC(i915) ? I915_CACHE_LLC : I915_CACHE_NONE;
-	obj->cache_coherent = i915_gem_object_is_coherent(obj);
-	obj->cache_dirty = !obj->cache_coherent;
+
+	cache_level = HAS_LLC(i915) ? I915_CACHE_LLC : I915_CACHE_NONE;
+	i915_gem_object_set_cache_coherency(obj, cache_level);
 
 	return obj;
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_object.c b/drivers/gpu/drm/i915/i915_gem_object.c
new file mode 100644
index 000000000000..aab8cdd80e6d
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem_object.c
@@ -0,0 +1,48 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include "i915_drv.h"
+#include "i915_gem_object.h"
+
+/**
+ * Mark up the object's coherency levels for a given cache_level
+ * @obj: #drm_i915_gem_object
+ * @cache_level: cache level
+ */
+void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj,
+					 unsigned int cache_level)
+{
+	obj->cache_level = cache_level;
+
+	if (cache_level != I915_CACHE_NONE)
+		obj->cache_coherent = (I915_BO_CACHE_COHERENT_FOR_READ |
+				       I915_BO_CACHE_COHERENT_FOR_WRITE);
+	else if (HAS_LLC(to_i915(obj->base.dev)))
+		obj->cache_coherent = I915_BO_CACHE_COHERENT_FOR_READ;
+	else
+		obj->cache_coherent = 0;
+
+	obj->cache_dirty =
+		!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE);
+}
diff --git a/drivers/gpu/drm/i915/i915_gem_object.h b/drivers/gpu/drm/i915/i915_gem_object.h
index 5b19a4916a4d..c30d8f808185 100644
--- a/drivers/gpu/drm/i915/i915_gem_object.h
+++ b/drivers/gpu/drm/i915/i915_gem_object.h
@@ -33,8 +33,24 @@
 
 #include <drm/i915_drm.h>
 
+#include "i915_gem_request.h"
 #include "i915_selftest.h"
 
+struct drm_i915_gem_object;
+
+/*
+ * struct i915_lut_handle tracks the fast lookups from handle to vma used
+ * for execbuf. Although we use a radixtree for that mapping, in order to
+ * remove them as the object or context is closed, we need a secondary list
+ * and a translation entry (i915_lut_handle).
+ */
+struct i915_lut_handle {
+	struct list_head obj_link;
+	struct list_head ctx_link;
+	struct i915_gem_context *ctx;
+	u32 handle;
+};
+
 struct drm_i915_gem_object_ops {
 	unsigned int flags;
 #define I915_GEM_OBJECT_HAS_STRUCT_PAGE BIT(0)
@@ -86,7 +102,15 @@ struct drm_i915_gem_object {
 	 * They are also added to @vma_list for easy iteration.
 	 */
 	struct rb_root vma_tree;
-	struct i915_vma *vma_hashed;
+
+	/**
+	 * @lut_list: List of vma lookup entries in use for this object.
+	 *
+	 * If this object is closed, we need to remove all of its VMA from
+	 * the fast lookup index in associated contexts; @lut_list provides
+	 * this translation from object to context->handles_vma.
+	 */
+	struct list_head lut_list;
 
 	/** Stolen memory for this object, instead of being backed by shmem. */
 	struct drm_mm_node *stolen;
@@ -118,8 +142,10 @@ struct drm_i915_gem_object {
 	 */
 	unsigned long gt_ro:1;
 	unsigned int cache_level:3;
+	unsigned int cache_coherent:2;
+#define I915_BO_CACHE_COHERENT_FOR_READ BIT(0)
+#define I915_BO_CACHE_COHERENT_FOR_WRITE BIT(1)
 	unsigned int cache_dirty:1;
-	unsigned int cache_coherent:1;
 
 	atomic_t frontbuffer_bits;
 	unsigned int frontbuffer_ggtt_origin; /* write once */
@@ -391,6 +417,8 @@ i915_gem_object_last_write_engine(struct drm_i915_gem_object *obj)
 	return engine;
 }
 
+void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj,
+					 unsigned int cache_level);
 void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj);
 
 #endif
diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.c b/drivers/gpu/drm/i915/i915_gem_render_state.c
index 7032c542a9b1..4dd4c2159a92 100644
--- a/drivers/gpu/drm/i915/i915_gem_render_state.c
+++ b/drivers/gpu/drm/i915/i915_gem_render_state.c
@@ -242,6 +242,10 @@ int i915_gem_render_state_emit(struct drm_i915_gem_request *req)
 			goto err_unpin;
 	}
 
+	ret = req->engine->emit_flush(req, EMIT_INVALIDATE);
+	if (ret)
+		goto err_unpin;
+
 	ret = req->engine->emit_bb_start(req,
 					 so->batch_offset, so->batch_size,
 					 I915_DISPATCH_SECURE);
diff --git a/drivers/gpu/drm/i915/i915_gem_request.c b/drivers/gpu/drm/i915/i915_gem_request.c
index 8c59c79cbd8b..813a3b546d6e 100644
--- a/drivers/gpu/drm/i915/i915_gem_request.c
+++ b/drivers/gpu/drm/i915/i915_gem_request.c
@@ -213,6 +213,10 @@ static int reset_all_global_seqno(struct drm_i915_private *i915, u32 seqno)
 				cond_resched();
 		}
 
+		/* Check we are idle before we fiddle with hw state! */
+		GEM_BUG_ON(!intel_engine_is_idle(engine));
+		GEM_BUG_ON(i915_gem_active_isset(&engine->timeline->last_request));
+
 		/* Finally reset hw state */
 		intel_engine_init_global_seqno(engine, seqno);
 		tl->seqno = seqno;
@@ -240,27 +244,60 @@ int i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno)
 	return reset_all_global_seqno(dev_priv, seqno - 1);
 }
 
-static int reserve_seqno(struct intel_engine_cs *engine)
+static void mark_busy(struct drm_i915_private *i915)
 {
+	if (i915->gt.awake)
+		return;
+
+	GEM_BUG_ON(!i915->gt.active_requests);
+
+	intel_runtime_pm_get_noresume(i915);
+	i915->gt.awake = true;
+
+	intel_enable_gt_powersave(i915);
+	i915_update_gfx_val(i915);
+	if (INTEL_GEN(i915) >= 6)
+		gen6_rps_busy(i915);
+
+	queue_delayed_work(i915->wq,
+			   &i915->gt.retire_work,
+			   round_jiffies_up_relative(HZ));
+}
+
+static int reserve_engine(struct intel_engine_cs *engine)
+{
+	struct drm_i915_private *i915 = engine->i915;
 	u32 active = ++engine->timeline->inflight_seqnos;
 	u32 seqno = engine->timeline->seqno;
 	int ret;
 
 	/* Reservation is fine until we need to wrap around */
-	if (likely(!add_overflows(seqno, active)))
-		return 0;
-
-	ret = reset_all_global_seqno(engine->i915, 0);
-	if (ret) {
-		engine->timeline->inflight_seqnos--;
-		return ret;
+	if (unlikely(add_overflows(seqno, active))) {
+		ret = reset_all_global_seqno(i915, 0);
+		if (ret) {
+			engine->timeline->inflight_seqnos--;
+			return ret;
+		}
 	}
 
+	if (!i915->gt.active_requests++)
+		mark_busy(i915);
+
 	return 0;
 }
 
-static void unreserve_seqno(struct intel_engine_cs *engine)
+static void unreserve_engine(struct intel_engine_cs *engine)
 {
+	struct drm_i915_private *i915 = engine->i915;
+
+	if (!--i915->gt.active_requests) {
+		/* Cancel the mark_busy() from our reserve_engine() */
+		GEM_BUG_ON(!i915->gt.awake);
+		mod_delayed_work(i915->wq,
+				 &i915->gt.idle_work,
+				 msecs_to_jiffies(100));
+	}
+
 	GEM_BUG_ON(!engine->timeline->inflight_seqnos);
 	engine->timeline->inflight_seqnos--;
 }
@@ -329,13 +366,7 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request)
 	list_del_init(&request->link);
 	spin_unlock_irq(&engine->timeline->lock);
 
-	if (!--request->i915->gt.active_requests) {
-		GEM_BUG_ON(!request->i915->gt.awake);
-		mod_delayed_work(request->i915->wq,
-				 &request->i915->gt.idle_work,
-				 msecs_to_jiffies(100));
-	}
-	unreserve_seqno(request->engine);
+	unreserve_engine(request->engine);
 	advance_ring(request);
 
 	free_capture_list(request);
@@ -370,8 +401,7 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request)
 	i915_gem_request_remove_from_client(request);
 
 	/* Retirement decays the ban score as it is a sign of ctx progress */
-	if (request->ctx->ban_score > 0)
-		request->ctx->ban_score--;
+	atomic_dec_if_positive(&request->ctx->ban_score);
 
 	/* The backing object for the context is done after switching to the
 	 * *next* context. Therefore we cannot retire the previous context until
@@ -384,7 +414,11 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request)
 		engine->context_unpin(engine, engine->last_retired_context);
 	engine->last_retired_context = request->ctx;
 
-	dma_fence_signal(&request->fence);
+	spin_lock_irq(&request->lock);
+	if (request->waitboost)
+		atomic_dec(&request->i915->rps.num_waiters);
+	dma_fence_signal_locked(&request->fence);
+	spin_unlock_irq(&request->lock);
 
 	i915_priotree_fini(request->i915, &request->priotree);
 	i915_gem_request_put(request);
@@ -568,7 +602,7 @@ i915_gem_request_alloc(struct intel_engine_cs *engine,
 		return ERR_CAST(ring);
 	GEM_BUG_ON(!ring);
 
-	ret = reserve_seqno(engine);
+	ret = reserve_engine(engine);
 	if (ret)
 		goto err_unpin;
 
@@ -639,6 +673,7 @@ i915_gem_request_alloc(struct intel_engine_cs *engine,
 	req->file_priv = NULL;
 	req->batch = NULL;
 	req->capture_list = NULL;
+	req->waitboost = false;
 
 	/*
 	 * Reserve space in the ring buffer for all the commands required to
@@ -673,7 +708,7 @@ err_ctx:
 
 	kmem_cache_free(dev_priv->requests, req);
 err_unreserve:
-	unreserve_seqno(engine);
+	unreserve_engine(engine);
 err_unpin:
 	engine->context_unpin(engine, ctx);
 	return ERR_PTR(ret);
@@ -855,28 +890,6 @@ i915_gem_request_await_object(struct drm_i915_gem_request *to,
 	return ret;
 }
 
-static void i915_gem_mark_busy(const struct intel_engine_cs *engine)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-
-	if (dev_priv->gt.awake)
-		return;
-
-	GEM_BUG_ON(!dev_priv->gt.active_requests);
-
-	intel_runtime_pm_get_noresume(dev_priv);
-	dev_priv->gt.awake = true;
-
-	intel_enable_gt_powersave(dev_priv);
-	i915_update_gfx_val(dev_priv);
-	if (INTEL_GEN(dev_priv) >= 6)
-		gen6_rps_busy(dev_priv);
-
-	queue_delayed_work(dev_priv->wq,
-			   &dev_priv->gt.retire_work,
-			   round_jiffies_up_relative(HZ));
-}
-
 /*
  * NB: This function is not allowed to fail. Doing so would mean the the
  * request is not being tracked for completion but the work itself is
@@ -958,9 +971,6 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches)
 	list_add_tail(&request->ring_link, &ring->request_list);
 	request->emitted_jiffies = jiffies;
 
-	if (!request->i915->gt.active_requests++)
-		i915_gem_mark_busy(engine);
-
 	/* Let the backend know a new request has arrived that may need
 	 * to adjust the existing execution schedule due to a high priority
 	 * request - i.e. we may want to preempt the current request in order
@@ -1063,7 +1073,7 @@ static bool __i915_wait_request_check_and_reset(struct drm_i915_gem_request *req
 		return false;
 
 	__set_current_state(TASK_RUNNING);
-	i915_reset(request->i915);
+	i915_reset(request->i915, 0);
 	return true;
 }
 
diff --git a/drivers/gpu/drm/i915/i915_gem_request.h b/drivers/gpu/drm/i915/i915_gem_request.h
index 7579b9702c22..49a4c8994ff0 100644
--- a/drivers/gpu/drm/i915/i915_gem_request.h
+++ b/drivers/gpu/drm/i915/i915_gem_request.h
@@ -184,6 +184,8 @@ struct drm_i915_gem_request {
 	/** Time at which this request was emitted, in jiffies. */
 	unsigned long emitted_jiffies;
 
+	bool waitboost;
+
 	/** engine->request_list entry for this request */
 	struct list_head link;
 
diff --git a/drivers/gpu/drm/i915/i915_gem_shrinker.c b/drivers/gpu/drm/i915/i915_gem_shrinker.c
index 77fb39808131..74002b2d1b6f 100644
--- a/drivers/gpu/drm/i915/i915_gem_shrinker.c
+++ b/drivers/gpu/drm/i915/i915_gem_shrinker.c
@@ -136,6 +136,7 @@ static bool unsafe_drop_pages(struct drm_i915_gem_object *obj)
  * i915_gem_shrink - Shrink buffer object caches
  * @dev_priv: i915 device
  * @target: amount of memory to make available, in pages
+ * @nr_scanned: optional output for number of pages scanned (incremental)
  * @flags: control flags for selecting cache types
  *
  * This function is the main interface to the shrinker. It will try to release
@@ -158,7 +159,9 @@ static bool unsafe_drop_pages(struct drm_i915_gem_object *obj)
  */
 unsigned long
 i915_gem_shrink(struct drm_i915_private *dev_priv,
-		unsigned long target, unsigned flags)
+		unsigned long target,
+		unsigned long *nr_scanned,
+		unsigned flags)
 {
 	const struct {
 		struct list_head *list;
@@ -169,6 +172,7 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 		{ NULL, 0 },
 	}, *phase;
 	unsigned long count = 0;
+	unsigned long scanned = 0;
 	bool unlock;
 
 	if (!shrinker_lock(dev_priv, &unlock))
@@ -249,6 +253,7 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 					count += obj->base.size >> PAGE_SHIFT;
 				}
 				mutex_unlock(&obj->mm.lock);
+				scanned += obj->base.size >> PAGE_SHIFT;
 			}
 		}
 		list_splice_tail(&still_in_list, phase->list);
@@ -261,6 +266,8 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 
 	shrinker_unlock(dev_priv, unlock);
 
+	if (nr_scanned)
+		*nr_scanned += scanned;
 	return count;
 }
 
@@ -283,7 +290,7 @@ unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv)
 	unsigned long freed;
 
 	intel_runtime_pm_get(dev_priv);
-	freed = i915_gem_shrink(dev_priv, -1UL,
+	freed = i915_gem_shrink(dev_priv, -1UL, NULL,
 				I915_SHRINK_BOUND |
 				I915_SHRINK_UNBOUND |
 				I915_SHRINK_ACTIVE);
@@ -329,23 +336,28 @@ i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
 	unsigned long freed;
 	bool unlock;
 
+	sc->nr_scanned = 0;
+
 	if (!shrinker_lock(dev_priv, &unlock))
 		return SHRINK_STOP;
 
 	freed = i915_gem_shrink(dev_priv,
 				sc->nr_to_scan,
+				&sc->nr_scanned,
 				I915_SHRINK_BOUND |
 				I915_SHRINK_UNBOUND |
 				I915_SHRINK_PURGEABLE);
 	if (freed < sc->nr_to_scan)
 		freed += i915_gem_shrink(dev_priv,
-					 sc->nr_to_scan - freed,
+					 sc->nr_to_scan - sc->nr_scanned,
+					 &sc->nr_scanned,
 					 I915_SHRINK_BOUND |
 					 I915_SHRINK_UNBOUND);
 	if (freed < sc->nr_to_scan && current_is_kswapd()) {
 		intel_runtime_pm_get(dev_priv);
 		freed += i915_gem_shrink(dev_priv,
-					 sc->nr_to_scan - freed,
+					 sc->nr_to_scan - sc->nr_scanned,
+					 &sc->nr_scanned,
 					 I915_SHRINK_ACTIVE |
 					 I915_SHRINK_BOUND |
 					 I915_SHRINK_UNBOUND);
@@ -354,7 +366,7 @@ i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
 
 	shrinker_unlock(dev_priv, unlock);
 
-	return freed;
+	return sc->nr_scanned ? freed : SHRINK_STOP;
 }
 
 static bool
@@ -453,7 +465,7 @@ i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr
 		goto out;
 
 	intel_runtime_pm_get(dev_priv);
-	freed_pages += i915_gem_shrink(dev_priv, -1UL,
+	freed_pages += i915_gem_shrink(dev_priv, -1UL, NULL,
 				       I915_SHRINK_BOUND |
 				       I915_SHRINK_UNBOUND |
 				       I915_SHRINK_ACTIVE |
diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c
index a817b3e0b17e..507c9f0d8df1 100644
--- a/drivers/gpu/drm/i915/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/i915_gem_stolen.c
@@ -254,9 +254,10 @@ static dma_addr_t i915_stolen_to_dma(struct drm_i915_private *dev_priv)
 		 * This is a BIOS w/a: Some BIOS wrap stolen in the root
 		 * PCI bus, but have an off-by-one error. Hence retry the
 		 * reservation starting from 1 instead of 0.
+		 * There's also BIOS with off-by-one on the other end.
 		 */
 		r = devm_request_mem_region(dev_priv->drm.dev, base + 1,
-					    ggtt->stolen_size - 1,
+					    ggtt->stolen_size - 2,
 					    "Graphics Stolen Memory");
 		/*
 		 * GEN3 firmware likes to smash pci bridges into the stolen
@@ -579,6 +580,7 @@ _i915_gem_object_create_stolen(struct drm_i915_private *dev_priv,
 			       struct drm_mm_node *stolen)
 {
 	struct drm_i915_gem_object *obj;
+	unsigned int cache_level;
 
 	obj = i915_gem_object_alloc(dev_priv);
 	if (obj == NULL)
@@ -589,8 +591,8 @@ _i915_gem_object_create_stolen(struct drm_i915_private *dev_priv,
 
 	obj->stolen = stolen;
 	obj->base.read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT;
-	obj->cache_level = HAS_LLC(dev_priv) ? I915_CACHE_LLC : I915_CACHE_NONE;
-	obj->cache_coherent = true; /* assumptions! more like cache_oblivious */
+	cache_level = HAS_LLC(dev_priv) ? I915_CACHE_LLC : I915_CACHE_NONE;
+	i915_gem_object_set_cache_coherency(obj, cache_level);
 
 	if (i915_gem_object_pin_pages(obj))
 		goto cleanup;
diff --git a/drivers/gpu/drm/i915/i915_gem_userptr.c b/drivers/gpu/drm/i915/i915_gem_userptr.c
index ccd09e8419f5..709efe2357ea 100644
--- a/drivers/gpu/drm/i915/i915_gem_userptr.c
+++ b/drivers/gpu/drm/i915/i915_gem_userptr.c
@@ -49,7 +49,7 @@ struct i915_mmu_notifier {
 	spinlock_t lock;
 	struct hlist_node node;
 	struct mmu_notifier mn;
-	struct rb_root objects;
+	struct rb_root_cached objects;
 	struct workqueue_struct *wq;
 };
 
@@ -123,7 +123,7 @@ static void i915_gem_userptr_mn_invalidate_range_start(struct mmu_notifier *_mn,
 	struct interval_tree_node *it;
 	LIST_HEAD(cancelled);
 
-	if (RB_EMPTY_ROOT(&mn->objects))
+	if (RB_EMPTY_ROOT(&mn->objects.rb_root))
 		return;
 
 	/* interval ranges are inclusive, but invalidate range is exclusive */
@@ -172,7 +172,7 @@ i915_mmu_notifier_create(struct mm_struct *mm)
 
 	spin_lock_init(&mn->lock);
 	mn->mn.ops = &i915_gem_userptr_notifier;
-	mn->objects = RB_ROOT;
+	mn->objects = RB_ROOT_CACHED;
 	mn->wq = alloc_workqueue("i915-userptr-release", WQ_UNBOUND, 0);
 	if (mn->wq == NULL) {
 		kfree(mn);
@@ -507,7 +507,7 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
 	ret = -ENOMEM;
 	pinned = 0;
 
-	pvec = kvmalloc_array(npages, sizeof(struct page *), GFP_TEMPORARY);
+	pvec = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
 	if (pvec != NULL) {
 		struct mm_struct *mm = obj->userptr.mm->mm;
 		unsigned int flags = 0;
@@ -643,7 +643,7 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
 
 	if (mm == current->mm) {
 		pvec = kvmalloc_array(num_pages, sizeof(struct page *),
-				      GFP_TEMPORARY |
+				      GFP_KERNEL |
 				      __GFP_NORETRY |
 				      __GFP_NOWARN);
 		if (pvec) /* defer to worker if malloc fails */
@@ -804,9 +804,7 @@ i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file
 	i915_gem_object_init(obj, &i915_gem_userptr_ops);
 	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
 	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
-	obj->cache_level = I915_CACHE_LLC;
-	obj->cache_coherent = i915_gem_object_is_coherent(obj);
-	obj->cache_dirty = !obj->cache_coherent;
+	i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
 
 	obj->userptr.ptr = args->user_ptr;
 	obj->userptr.read_only = !!(args->flags & I915_USERPTR_READ_ONLY);
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index e18f350bc364..0c779671fe2d 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -463,6 +463,7 @@ static void error_print_engine(struct drm_i915_error_state_buf *m,
 	err_printf(m, "  hangcheck action timestamp: %lu, %u ms ago\n",
 		   ee->hangcheck_timestamp,
 		   jiffies_to_msecs(jiffies - ee->hangcheck_timestamp));
+	err_printf(m, "  engine reset count: %u\n", ee->reset_count);
 
 	error_print_request(m, "  ELSP[0]: ", &ee->execlist[0]);
 	error_print_request(m, "  ELSP[1]: ", &ee->execlist[1]);
@@ -786,16 +787,16 @@ int i915_error_state_buf_init(struct drm_i915_error_state_buf *ebuf,
 	 */
 	ebuf->size = count + 1 > PAGE_SIZE ? count + 1 : PAGE_SIZE;
 	ebuf->buf = kmalloc(ebuf->size,
-				GFP_TEMPORARY | __GFP_NORETRY | __GFP_NOWARN);
+				GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN);
 
 	if (ebuf->buf == NULL) {
 		ebuf->size = PAGE_SIZE;
-		ebuf->buf = kmalloc(ebuf->size, GFP_TEMPORARY);
+		ebuf->buf = kmalloc(ebuf->size, GFP_KERNEL);
 	}
 
 	if (ebuf->buf == NULL) {
 		ebuf->size = 128;
-		ebuf->buf = kmalloc(ebuf->size, GFP_TEMPORARY);
+		ebuf->buf = kmalloc(ebuf->size, GFP_KERNEL);
 	}
 
 	if (ebuf->buf == NULL)
@@ -1236,6 +1237,8 @@ static void error_record_engine_registers(struct i915_gpu_state *error,
 	ee->hangcheck_timestamp = engine->hangcheck.action_timestamp;
 	ee->hangcheck_action = engine->hangcheck.action;
 	ee->hangcheck_stalled = engine->hangcheck.stalled;
+	ee->reset_count = i915_reset_engine_count(&dev_priv->gpu_error,
+						  engine);
 
 	if (USES_PPGTT(dev_priv)) {
 		int i;
@@ -1263,7 +1266,7 @@ static void record_request(struct drm_i915_gem_request *request,
 			   struct drm_i915_error_request *erq)
 {
 	erq->context = request->ctx->hw_id;
-	erq->ban_score = request->ctx->ban_score;
+	erq->ban_score = atomic_read(&request->ctx->ban_score);
 	erq->seqno = request->global_seqno;
 	erq->jiffies = request->emitted_jiffies;
 	erq->head = request->head;
@@ -1354,9 +1357,9 @@ static void record_context(struct drm_i915_error_context *e,
 
 	e->handle = ctx->user_handle;
 	e->hw_id = ctx->hw_id;
-	e->ban_score = ctx->ban_score;
-	e->guilty = ctx->guilty_count;
-	e->active = ctx->active_count;
+	e->ban_score = atomic_read(&ctx->ban_score);
+	e->guilty = atomic_read(&ctx->guilty_count);
+	e->active = atomic_read(&ctx->active_count);
 }
 
 static void request_record_user_bo(struct drm_i915_gem_request *request,
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 4cd9ee1ba332..b63893eeca73 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -275,17 +275,17 @@ void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 
 static i915_reg_t gen6_pm_iir(struct drm_i915_private *dev_priv)
 {
-	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
+	return INTEL_GEN(dev_priv) >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
 }
 
 static i915_reg_t gen6_pm_imr(struct drm_i915_private *dev_priv)
 {
-	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
+	return INTEL_GEN(dev_priv) >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
 }
 
 static i915_reg_t gen6_pm_ier(struct drm_i915_private *dev_priv)
 {
-	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
+	return INTEL_GEN(dev_priv) >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
 }
 
 /**
@@ -839,7 +839,6 @@ static bool i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
 								pipe);
 	int position;
 	int vbl_start, vbl_end, hsync_start, htotal, vtotal;
-	bool in_vbl = true;
 	unsigned long irqflags;
 
 	if (WARN_ON(!mode->crtc_clock)) {
@@ -922,8 +921,6 @@ static bool i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
 
 	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 
-	in_vbl = position >= vbl_start && position < vbl_end;
-
 	/*
 	 * While in vblank, position will be negative
 	 * counting up towards 0 at vbl_end. And outside
@@ -1091,18 +1088,6 @@ static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
 	return events;
 }
 
-static bool any_waiters(struct drm_i915_private *dev_priv)
-{
-	struct intel_engine_cs *engine;
-	enum intel_engine_id id;
-
-	for_each_engine(engine, dev_priv, id)
-		if (intel_engine_has_waiter(engine))
-			return true;
-
-	return false;
-}
-
 static void gen6_pm_rps_work(struct work_struct *work)
 {
 	struct drm_i915_private *dev_priv =
@@ -1114,7 +1099,7 @@ static void gen6_pm_rps_work(struct work_struct *work)
 	spin_lock_irq(&dev_priv->irq_lock);
 	if (dev_priv->rps.interrupts_enabled) {
 		pm_iir = fetch_and_zero(&dev_priv->rps.pm_iir);
-		client_boost = fetch_and_zero(&dev_priv->rps.client_boost);
+		client_boost = atomic_read(&dev_priv->rps.num_waiters);
 	}
 	spin_unlock_irq(&dev_priv->irq_lock);
 
@@ -1131,7 +1116,7 @@ static void gen6_pm_rps_work(struct work_struct *work)
 	new_delay = dev_priv->rps.cur_freq;
 	min = dev_priv->rps.min_freq_softlimit;
 	max = dev_priv->rps.max_freq_softlimit;
-	if (client_boost || any_waiters(dev_priv))
+	if (client_boost)
 		max = dev_priv->rps.max_freq;
 	if (client_boost && new_delay < dev_priv->rps.boost_freq) {
 		new_delay = dev_priv->rps.boost_freq;
@@ -1144,7 +1129,7 @@ static void gen6_pm_rps_work(struct work_struct *work)
 
 		if (new_delay >= dev_priv->rps.max_freq_softlimit)
 			adj = 0;
-	} else if (client_boost || any_waiters(dev_priv)) {
+	} else if (client_boost) {
 		adj = 0;
 	} else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
 		if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
@@ -1513,7 +1498,8 @@ static void intel_get_hpd_pins(u32 *pin_mask, u32 *long_mask,
 
 		*pin_mask |= BIT(i);
 
-		if (!intel_hpd_pin_to_port(i, &port))
+		port = intel_hpd_pin_to_port(i);
+		if (port == PORT_NONE)
 			continue;
 
 		if (long_pulse_detect(port, dig_hotplug_reg))
@@ -1603,7 +1589,7 @@ static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
 		crcs[3] = crc3;
 		crcs[4] = crc4;
 		drm_crtc_add_crc_entry(&crtc->base, true,
-				       drm_accurate_vblank_count(&crtc->base),
+				       drm_crtc_accurate_vblank_count(&crtc->base),
 				       crcs);
 	}
 }
@@ -1673,7 +1659,7 @@ static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
 		spin_unlock(&dev_priv->irq_lock);
 	}
 
-	if (INTEL_INFO(dev_priv)->gen >= 8)
+	if (INTEL_GEN(dev_priv) >= 8)
 		return;
 
 	if (HAS_VEBOX(dev_priv)) {
@@ -1720,18 +1706,6 @@ static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 gt_iir)
 	}
 }
 
-static bool intel_pipe_handle_vblank(struct drm_i915_private *dev_priv,
-				     enum pipe pipe)
-{
-	bool ret;
-
-	ret = drm_handle_vblank(&dev_priv->drm, pipe);
-	if (ret)
-		intel_finish_page_flip_mmio(dev_priv, pipe);
-
-	return ret;
-}
-
 static void valleyview_pipestat_irq_ack(struct drm_i915_private *dev_priv,
 					u32 iir, u32 pipe_stats[I915_MAX_PIPES])
 {
@@ -1796,12 +1770,8 @@ static void valleyview_pipestat_irq_handler(struct drm_i915_private *dev_priv,
 	enum pipe pipe;
 
 	for_each_pipe(dev_priv, pipe) {
-		if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
-		    intel_pipe_handle_vblank(dev_priv, pipe))
-			intel_check_page_flip(dev_priv, pipe);
-
-		if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV)
-			intel_finish_page_flip_cs(dev_priv, pipe);
+		if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
+			drm_handle_vblank(&dev_priv->drm, pipe);
 
 		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
 			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
@@ -2098,10 +2068,10 @@ static void ibx_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
 		DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
 
 	if (pch_iir & SDE_TRANSA_FIFO_UNDER)
-		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
+		intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_A);
 
 	if (pch_iir & SDE_TRANSB_FIFO_UNDER)
-		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
+		intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_B);
 }
 
 static void ivb_err_int_handler(struct drm_i915_private *dev_priv)
@@ -2135,13 +2105,13 @@ static void cpt_serr_int_handler(struct drm_i915_private *dev_priv)
 		DRM_ERROR("PCH poison interrupt\n");
 
 	if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
-		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
+		intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_A);
 
 	if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
-		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
+		intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_B);
 
 	if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
-		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C);
+		intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_C);
 
 	I915_WRITE(SERR_INT, serr_int);
 }
@@ -2253,19 +2223,14 @@ static void ilk_display_irq_handler(struct drm_i915_private *dev_priv,
 		DRM_ERROR("Poison interrupt\n");
 
 	for_each_pipe(dev_priv, pipe) {
-		if (de_iir & DE_PIPE_VBLANK(pipe) &&
-		    intel_pipe_handle_vblank(dev_priv, pipe))
-			intel_check_page_flip(dev_priv, pipe);
+		if (de_iir & DE_PIPE_VBLANK(pipe))
+			drm_handle_vblank(&dev_priv->drm, pipe);
 
 		if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
 			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
 
 		if (de_iir & DE_PIPE_CRC_DONE(pipe))
 			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
-
-		/* plane/pipes map 1:1 on ilk+ */
-		if (de_iir & DE_PLANE_FLIP_DONE(pipe))
-			intel_finish_page_flip_cs(dev_priv, pipe);
 	}
 
 	/* check event from PCH */
@@ -2304,13 +2269,8 @@ static void ivb_display_irq_handler(struct drm_i915_private *dev_priv,
 		intel_opregion_asle_intr(dev_priv);
 
 	for_each_pipe(dev_priv, pipe) {
-		if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
-		    intel_pipe_handle_vblank(dev_priv, pipe))
-			intel_check_page_flip(dev_priv, pipe);
-
-		/* plane/pipes map 1:1 on ilk+ */
-		if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe))
-			intel_finish_page_flip_cs(dev_priv, pipe);
+		if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)))
+			drm_handle_vblank(&dev_priv->drm, pipe);
 	}
 
 	/* check event from PCH */
@@ -2452,7 +2412,7 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
 			ret = IRQ_HANDLED;
 
 			tmp_mask = GEN8_AUX_CHANNEL_A;
-			if (INTEL_INFO(dev_priv)->gen >= 9)
+			if (INTEL_GEN(dev_priv) >= 9)
 				tmp_mask |= GEN9_AUX_CHANNEL_B |
 					    GEN9_AUX_CHANNEL_C |
 					    GEN9_AUX_CHANNEL_D;
@@ -2491,7 +2451,7 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
 	}
 
 	for_each_pipe(dev_priv, pipe) {
-		u32 flip_done, fault_errors;
+		u32 fault_errors;
 
 		if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
 			continue;
@@ -2505,18 +2465,8 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
 		ret = IRQ_HANDLED;
 		I915_WRITE(GEN8_DE_PIPE_IIR(pipe), iir);
 
-		if (iir & GEN8_PIPE_VBLANK &&
-		    intel_pipe_handle_vblank(dev_priv, pipe))
-			intel_check_page_flip(dev_priv, pipe);
-
-		flip_done = iir;
-		if (INTEL_INFO(dev_priv)->gen >= 9)
-			flip_done &= GEN9_PIPE_PLANE1_FLIP_DONE;
-		else
-			flip_done &= GEN8_PIPE_PRIMARY_FLIP_DONE;
-
-		if (flip_done)
-			intel_finish_page_flip_cs(dev_priv, pipe);
+		if (iir & GEN8_PIPE_VBLANK)
+			drm_handle_vblank(&dev_priv->drm, pipe);
 
 		if (iir & GEN8_PIPE_CDCLK_CRC_DONE)
 			hsw_pipe_crc_irq_handler(dev_priv, pipe);
@@ -2525,7 +2475,7 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
 			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
 
 		fault_errors = iir;
-		if (INTEL_INFO(dev_priv)->gen >= 9)
+		if (INTEL_GEN(dev_priv) >= 9)
 			fault_errors &= GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
 		else
 			fault_errors &= GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
@@ -2599,86 +2549,93 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg)
 	return ret;
 }
 
+struct wedge_me {
+	struct delayed_work work;
+	struct drm_i915_private *i915;
+	const char *name;
+};
+
+static void wedge_me(struct work_struct *work)
+{
+	struct wedge_me *w = container_of(work, typeof(*w), work.work);
+
+	dev_err(w->i915->drm.dev,
+		"%s timed out, cancelling all in-flight rendering.\n",
+		w->name);
+	i915_gem_set_wedged(w->i915);
+}
+
+static void __init_wedge(struct wedge_me *w,
+			 struct drm_i915_private *i915,
+			 long timeout,
+			 const char *name)
+{
+	w->i915 = i915;
+	w->name = name;
+
+	INIT_DELAYED_WORK_ONSTACK(&w->work, wedge_me);
+	schedule_delayed_work(&w->work, timeout);
+}
+
+static void __fini_wedge(struct wedge_me *w)
+{
+	cancel_delayed_work_sync(&w->work);
+	destroy_delayed_work_on_stack(&w->work);
+	w->i915 = NULL;
+}
+
+#define i915_wedge_on_timeout(W, DEV, TIMEOUT)				\
+	for (__init_wedge((W), (DEV), (TIMEOUT), __func__);		\
+	     (W)->i915;							\
+	     __fini_wedge((W)))
+
 /**
- * i915_reset_and_wakeup - do process context error handling work
+ * i915_reset_device - do process context error handling work
  * @dev_priv: i915 device private
  *
  * Fire an error uevent so userspace can see that a hang or error
  * was detected.
  */
-static void i915_reset_and_wakeup(struct drm_i915_private *dev_priv)
+static void i915_reset_device(struct drm_i915_private *dev_priv)
 {
 	struct kobject *kobj = &dev_priv->drm.primary->kdev->kobj;
 	char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
 	char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
 	char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
+	struct wedge_me w;
 
 	kobject_uevent_env(kobj, KOBJ_CHANGE, error_event);
 
 	DRM_DEBUG_DRIVER("resetting chip\n");
 	kobject_uevent_env(kobj, KOBJ_CHANGE, reset_event);
 
-	intel_prepare_reset(dev_priv);
+	/* Use a watchdog to ensure that our reset completes */
+	i915_wedge_on_timeout(&w, dev_priv, 5*HZ) {
+		intel_prepare_reset(dev_priv);
 
-	set_bit(I915_RESET_HANDOFF, &dev_priv->gpu_error.flags);
-	wake_up_all(&dev_priv->gpu_error.wait_queue);
+		/* Signal that locked waiters should reset the GPU */
+		set_bit(I915_RESET_HANDOFF, &dev_priv->gpu_error.flags);
+		wake_up_all(&dev_priv->gpu_error.wait_queue);
 
-	do {
-		/*
-		 * All state reset _must_ be completed before we update the
-		 * reset counter, for otherwise waiters might miss the reset
-		 * pending state and not properly drop locks, resulting in
-		 * deadlocks with the reset work.
+		/* Wait for anyone holding the lock to wakeup, without
+		 * blocking indefinitely on struct_mutex.
 		 */
-		if (mutex_trylock(&dev_priv->drm.struct_mutex)) {
-			i915_reset(dev_priv);
-			mutex_unlock(&dev_priv->drm.struct_mutex);
-		}
-
-		/* We need to wait for anyone holding the lock to wakeup */
-	} while (wait_on_bit_timeout(&dev_priv->gpu_error.flags,
-				     I915_RESET_HANDOFF,
-				     TASK_UNINTERRUPTIBLE,
-				     HZ));
+		do {
+			if (mutex_trylock(&dev_priv->drm.struct_mutex)) {
+				i915_reset(dev_priv, 0);
+				mutex_unlock(&dev_priv->drm.struct_mutex);
+			}
+		} while (wait_on_bit_timeout(&dev_priv->gpu_error.flags,
+					     I915_RESET_HANDOFF,
+					     TASK_UNINTERRUPTIBLE,
+					     1));
 
-	intel_finish_reset(dev_priv);
+		intel_finish_reset(dev_priv);
+	}
 
 	if (!test_bit(I915_WEDGED, &dev_priv->gpu_error.flags))
 		kobject_uevent_env(kobj,
 				   KOBJ_CHANGE, reset_done_event);
-
-	/*
-	 * Note: The wake_up also serves as a memory barrier so that
-	 * waiters see the updated value of the dev_priv->gpu_error.
-	 */
-	clear_bit(I915_RESET_BACKOFF, &dev_priv->gpu_error.flags);
-	wake_up_all(&dev_priv->gpu_error.reset_queue);
-}
-
-static inline void
-i915_err_print_instdone(struct drm_i915_private *dev_priv,
-			struct intel_instdone *instdone)
-{
-	int slice;
-	int subslice;
-
-	pr_err("  INSTDONE: 0x%08x\n", instdone->instdone);
-
-	if (INTEL_GEN(dev_priv) <= 3)
-		return;
-
-	pr_err("  SC_INSTDONE: 0x%08x\n", instdone->slice_common);
-
-	if (INTEL_GEN(dev_priv) <= 6)
-		return;
-
-	for_each_instdone_slice_subslice(dev_priv, slice, subslice)
-		pr_err("  SAMPLER_INSTDONE[%d][%d]: 0x%08x\n",
-		       slice, subslice, instdone->sampler[slice][subslice]);
-
-	for_each_instdone_slice_subslice(dev_priv, slice, subslice)
-		pr_err("  ROW_INSTDONE[%d][%d]: 0x%08x\n",
-		       slice, subslice, instdone->row[slice][subslice]);
 }
 
 static void i915_clear_error_registers(struct drm_i915_private *dev_priv)
@@ -2722,6 +2679,8 @@ void i915_handle_error(struct drm_i915_private *dev_priv,
 		       u32 engine_mask,
 		       const char *fmt, ...)
 {
+	struct intel_engine_cs *engine;
+	unsigned int tmp;
 	va_list args;
 	char error_msg[80];
 
@@ -2741,14 +2700,56 @@ void i915_handle_error(struct drm_i915_private *dev_priv,
 	i915_capture_error_state(dev_priv, engine_mask, error_msg);
 	i915_clear_error_registers(dev_priv);
 
+	/*
+	 * Try engine reset when available. We fall back to full reset if
+	 * single reset fails.
+	 */
+	if (intel_has_reset_engine(dev_priv)) {
+		for_each_engine_masked(engine, dev_priv, engine_mask, tmp) {
+			BUILD_BUG_ON(I915_RESET_MODESET >= I915_RESET_ENGINE);
+			if (test_and_set_bit(I915_RESET_ENGINE + engine->id,
+					     &dev_priv->gpu_error.flags))
+				continue;
+
+			if (i915_reset_engine(engine, 0) == 0)
+				engine_mask &= ~intel_engine_flag(engine);
+
+			clear_bit(I915_RESET_ENGINE + engine->id,
+				  &dev_priv->gpu_error.flags);
+			wake_up_bit(&dev_priv->gpu_error.flags,
+				    I915_RESET_ENGINE + engine->id);
+		}
+	}
+
 	if (!engine_mask)
 		goto out;
 
-	if (test_and_set_bit(I915_RESET_BACKOFF,
-			     &dev_priv->gpu_error.flags))
+	/* Full reset needs the mutex, stop any other user trying to do so. */
+	if (test_and_set_bit(I915_RESET_BACKOFF, &dev_priv->gpu_error.flags)) {
+		wait_event(dev_priv->gpu_error.reset_queue,
+			   !test_bit(I915_RESET_BACKOFF,
+				     &dev_priv->gpu_error.flags));
 		goto out;
+	}
+
+	/* Prevent any other reset-engine attempt. */
+	for_each_engine(engine, dev_priv, tmp) {
+		while (test_and_set_bit(I915_RESET_ENGINE + engine->id,
+					&dev_priv->gpu_error.flags))
+			wait_on_bit(&dev_priv->gpu_error.flags,
+				    I915_RESET_ENGINE + engine->id,
+				    TASK_UNINTERRUPTIBLE);
+	}
 
-	i915_reset_and_wakeup(dev_priv);
+	i915_reset_device(dev_priv);
+
+	for_each_engine(engine, dev_priv, tmp) {
+		clear_bit(I915_RESET_ENGINE + engine->id,
+			  &dev_priv->gpu_error.flags);
+	}
+
+	clear_bit(I915_RESET_BACKOFF, &dev_priv->gpu_error.flags);
+	wake_up_all(&dev_priv->gpu_error.reset_queue);
 
 out:
 	intel_runtime_pm_put(dev_priv);
@@ -3009,7 +3010,7 @@ static void gen8_irq_reset(struct drm_device *dev)
 }
 
 void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
-				     unsigned int pipe_mask)
+				     u8 pipe_mask)
 {
 	uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
 	enum pipe pipe;
@@ -3023,7 +3024,7 @@ void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
 }
 
 void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv,
-				     unsigned int pipe_mask)
+				     u8 pipe_mask)
 {
 	enum pipe pipe;
 
@@ -3427,7 +3428,7 @@ static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
 	u32 de_misc_masked = GEN8_DE_MISC_GSE;
 	enum pipe pipe;
 
-	if (INTEL_INFO(dev_priv)->gen >= 9) {
+	if (INTEL_GEN(dev_priv) >= 9) {
 		de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE |
 				  GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
 		de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
@@ -3610,34 +3611,6 @@ static int i8xx_irq_postinstall(struct drm_device *dev)
 /*
  * Returns true when a page flip has completed.
  */
-static bool i8xx_handle_vblank(struct drm_i915_private *dev_priv,
-			       int plane, int pipe, u32 iir)
-{
-	u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
-
-	if (!intel_pipe_handle_vblank(dev_priv, pipe))
-		return false;
-
-	if ((iir & flip_pending) == 0)
-		goto check_page_flip;
-
-	/* We detect FlipDone by looking for the change in PendingFlip from '1'
-	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
-	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
-	 * the flip is completed (no longer pending). Since this doesn't raise
-	 * an interrupt per se, we watch for the change at vblank.
-	 */
-	if (I915_READ16(ISR) & flip_pending)
-		goto check_page_flip;
-
-	intel_finish_page_flip_cs(dev_priv, pipe);
-	return true;
-
-check_page_flip:
-	intel_check_page_flip(dev_priv, pipe);
-	return false;
-}
-
 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
 {
 	struct drm_device *dev = arg;
@@ -3645,9 +3618,6 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
 	u16 iir, new_iir;
 	u32 pipe_stats[2];
 	int pipe;
-	u16 flip_mask =
-		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
-		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
 	irqreturn_t ret;
 
 	if (!intel_irqs_enabled(dev_priv))
@@ -3661,7 +3631,7 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
 	if (iir == 0)
 		goto out;
 
-	while (iir & ~flip_mask) {
+	while (iir) {
 		/* Can't rely on pipestat interrupt bit in iir as it might
 		 * have been cleared after the pipestat interrupt was received.
 		 * It doesn't set the bit in iir again, but it still produces
@@ -3683,7 +3653,7 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
 		}
 		spin_unlock(&dev_priv->irq_lock);
 
-		I915_WRITE16(IIR, iir & ~flip_mask);
+		I915_WRITE16(IIR, iir);
 		new_iir = I915_READ16(IIR); /* Flush posted writes */
 
 		if (iir & I915_USER_INTERRUPT)
@@ -3694,9 +3664,8 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
 			if (HAS_FBC(dev_priv))
 				plane = !plane;
 
-			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
-			    i8xx_handle_vblank(dev_priv, plane, pipe, iir))
-				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
+			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
+				drm_handle_vblank(&dev_priv->drm, pipe);
 
 			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
 				i9xx_pipe_crc_irq_handler(dev_priv, pipe);
@@ -3796,45 +3765,11 @@ static int i915_irq_postinstall(struct drm_device *dev)
 	return 0;
 }
 
-/*
- * Returns true when a page flip has completed.
- */
-static bool i915_handle_vblank(struct drm_i915_private *dev_priv,
-			       int plane, int pipe, u32 iir)
-{
-	u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
-
-	if (!intel_pipe_handle_vblank(dev_priv, pipe))
-		return false;
-
-	if ((iir & flip_pending) == 0)
-		goto check_page_flip;
-
-	/* We detect FlipDone by looking for the change in PendingFlip from '1'
-	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
-	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
-	 * the flip is completed (no longer pending). Since this doesn't raise
-	 * an interrupt per se, we watch for the change at vblank.
-	 */
-	if (I915_READ(ISR) & flip_pending)
-		goto check_page_flip;
-
-	intel_finish_page_flip_cs(dev_priv, pipe);
-	return true;
-
-check_page_flip:
-	intel_check_page_flip(dev_priv, pipe);
-	return false;
-}
-
 static irqreturn_t i915_irq_handler(int irq, void *arg)
 {
 	struct drm_device *dev = arg;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
-	u32 flip_mask =
-		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
-		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
 	int pipe, ret = IRQ_NONE;
 
 	if (!intel_irqs_enabled(dev_priv))
@@ -3845,7 +3780,7 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
 
 	iir = I915_READ(IIR);
 	do {
-		bool irq_received = (iir & ~flip_mask) != 0;
+		bool irq_received = (iir) != 0;
 		bool blc_event = false;
 
 		/* Can't rely on pipestat interrupt bit in iir as it might
@@ -3880,7 +3815,7 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
 				i9xx_hpd_irq_handler(dev_priv, hotplug_status);
 		}
 
-		I915_WRITE(IIR, iir & ~flip_mask);
+		I915_WRITE(IIR, iir);
 		new_iir = I915_READ(IIR); /* Flush posted writes */
 
 		if (iir & I915_USER_INTERRUPT)
@@ -3891,9 +3826,8 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
 			if (HAS_FBC(dev_priv))
 				plane = !plane;
 
-			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
-			    i915_handle_vblank(dev_priv, plane, pipe, iir))
-				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
+			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
+				drm_handle_vblank(&dev_priv->drm, pipe);
 
 			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
 				blc_event = true;
@@ -3926,7 +3860,7 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
 		 */
 		ret = IRQ_HANDLED;
 		iir = new_iir;
-	} while (iir & ~flip_mask);
+	} while (iir);
 
 	enable_rpm_wakeref_asserts(dev_priv);
 
@@ -4061,9 +3995,6 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
 	u32 iir, new_iir;
 	u32 pipe_stats[I915_MAX_PIPES];
 	int ret = IRQ_NONE, pipe;
-	u32 flip_mask =
-		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
-		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
 
 	if (!intel_irqs_enabled(dev_priv))
 		return IRQ_NONE;
@@ -4074,7 +4005,7 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
 	iir = I915_READ(IIR);
 
 	for (;;) {
-		bool irq_received = (iir & ~flip_mask) != 0;
+		bool irq_received = (iir) != 0;
 		bool blc_event = false;
 
 		/* Can't rely on pipestat interrupt bit in iir as it might
@@ -4112,7 +4043,7 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
 				i9xx_hpd_irq_handler(dev_priv, hotplug_status);
 		}
 
-		I915_WRITE(IIR, iir & ~flip_mask);
+		I915_WRITE(IIR, iir);
 		new_iir = I915_READ(IIR); /* Flush posted writes */
 
 		if (iir & I915_USER_INTERRUPT)
@@ -4121,9 +4052,8 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
 			notify_ring(dev_priv->engine[VCS]);
 
 		for_each_pipe(dev_priv, pipe) {
-			if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
-			    i915_handle_vblank(dev_priv, pipe, pipe, iir))
-				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
+			if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
+				drm_handle_vblank(&dev_priv->drm, pipe);
 
 			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
 				blc_event = true;
@@ -4225,16 +4155,16 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
 	 *
 	 * TODO: verify if this can be reproduced on VLV,CHV.
 	 */
-	if (INTEL_INFO(dev_priv)->gen <= 7)
+	if (INTEL_GEN(dev_priv) <= 7)
 		dev_priv->rps.pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
 
-	if (INTEL_INFO(dev_priv)->gen >= 8)
+	if (INTEL_GEN(dev_priv) >= 8)
 		dev_priv->rps.pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
 
 	if (IS_GEN2(dev_priv)) {
 		/* Gen2 doesn't have a hardware frame counter */
 		dev->max_vblank_count = 0;
-	} else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
+	} else if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) {
 		dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
 		dev->driver->get_vblank_counter = g4x_get_vblank_counter;
 	} else {
@@ -4281,7 +4211,7 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
 		dev->driver->enable_vblank = i965_enable_vblank;
 		dev->driver->disable_vblank = i965_disable_vblank;
 		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
-	} else if (INTEL_INFO(dev_priv)->gen >= 8) {
+	} else if (INTEL_GEN(dev_priv) >= 8) {
 		dev->driver->irq_handler = gen8_irq_handler;
 		dev->driver->irq_preinstall = gen8_irq_reset;
 		dev->driver->irq_postinstall = gen8_irq_postinstall;
diff --git a/drivers/gpu/drm/i915/i915_oa_bdw.c b/drivers/gpu/drm/i915/i915_oa_bdw.c
index d4462c2aaaee..abdf4d0abcce 100644
--- a/drivers/gpu/drm/i915/i915_oa_bdw.c
+++ b/drivers/gpu/drm/i915/i915_oa_bdw.c
@@ -31,3981 +31,6 @@
 #include "i915_drv.h"
 #include "i915_oa_bdw.h"
 
-enum metric_set_id {
-	METRIC_SET_ID_RENDER_BASIC = 1,
-	METRIC_SET_ID_COMPUTE_BASIC,
-	METRIC_SET_ID_RENDER_PIPE_PROFILE,
-	METRIC_SET_ID_MEMORY_READS,
-	METRIC_SET_ID_MEMORY_WRITES,
-	METRIC_SET_ID_COMPUTE_EXTENDED,
-	METRIC_SET_ID_COMPUTE_L3_CACHE,
-	METRIC_SET_ID_DATA_PORT_READS_COALESCING,
-	METRIC_SET_ID_DATA_PORT_WRITES_COALESCING,
-	METRIC_SET_ID_HDC_AND_SF,
-	METRIC_SET_ID_L3_1,
-	METRIC_SET_ID_L3_2,
-	METRIC_SET_ID_L3_3,
-	METRIC_SET_ID_L3_4,
-	METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND,
-	METRIC_SET_ID_SAMPLER_1,
-	METRIC_SET_ID_SAMPLER_2,
-	METRIC_SET_ID_TDL_1,
-	METRIC_SET_ID_TDL_2,
-	METRIC_SET_ID_COMPUTE_EXTRA,
-	METRIC_SET_ID_VME_PIPE,
-	METRIC_SET_ID_TEST_OA,
-};
-
-int i915_oa_n_builtin_metric_sets_bdw = 22;
-
-static const struct i915_oa_reg b_counter_config_render_basic[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2740), 0x00000000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_render_basic[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_render_basic_0_slices_0x01[] = {
-	{ _MMIO(0x9888), 0x143f000f },
-	{ _MMIO(0x9888), 0x14110014 },
-	{ _MMIO(0x9888), 0x14310014 },
-	{ _MMIO(0x9888), 0x14bf000f },
-	{ _MMIO(0x9888), 0x118a0317 },
-	{ _MMIO(0x9888), 0x13837be0 },
-	{ _MMIO(0x9888), 0x3b800060 },
-	{ _MMIO(0x9888), 0x3d800005 },
-	{ _MMIO(0x9888), 0x005c4000 },
-	{ _MMIO(0x9888), 0x065c8000 },
-	{ _MMIO(0x9888), 0x085cc000 },
-	{ _MMIO(0x9888), 0x003d8000 },
-	{ _MMIO(0x9888), 0x183d0800 },
-	{ _MMIO(0x9888), 0x0a3f0023 },
-	{ _MMIO(0x9888), 0x103f0000 },
-	{ _MMIO(0x9888), 0x00584000 },
-	{ _MMIO(0x9888), 0x08584000 },
-	{ _MMIO(0x9888), 0x0a5a4000 },
-	{ _MMIO(0x9888), 0x005b4000 },
-	{ _MMIO(0x9888), 0x0e5b8000 },
-	{ _MMIO(0x9888), 0x185b2400 },
-	{ _MMIO(0x9888), 0x0a1d4000 },
-	{ _MMIO(0x9888), 0x0c1f0800 },
-	{ _MMIO(0x9888), 0x0e1faa00 },
-	{ _MMIO(0x9888), 0x00384000 },
-	{ _MMIO(0x9888), 0x0e384000 },
-	{ _MMIO(0x9888), 0x16384000 },
-	{ _MMIO(0x9888), 0x18380001 },
-	{ _MMIO(0x9888), 0x00392000 },
-	{ _MMIO(0x9888), 0x06398000 },
-	{ _MMIO(0x9888), 0x0839a000 },
-	{ _MMIO(0x9888), 0x0a391000 },
-	{ _MMIO(0x9888), 0x00104000 },
-	{ _MMIO(0x9888), 0x08104000 },
-	{ _MMIO(0x9888), 0x00110030 },
-	{ _MMIO(0x9888), 0x08110031 },
-	{ _MMIO(0x9888), 0x10110000 },
-	{ _MMIO(0x9888), 0x00134000 },
-	{ _MMIO(0x9888), 0x16130020 },
-	{ _MMIO(0x9888), 0x06308000 },
-	{ _MMIO(0x9888), 0x08308000 },
-	{ _MMIO(0x9888), 0x06311800 },
-	{ _MMIO(0x9888), 0x08311880 },
-	{ _MMIO(0x9888), 0x10310000 },
-	{ _MMIO(0x9888), 0x0e334000 },
-	{ _MMIO(0x9888), 0x16330080 },
-	{ _MMIO(0x9888), 0x0abf1180 },
-	{ _MMIO(0x9888), 0x10bf0000 },
-	{ _MMIO(0x9888), 0x0ada8000 },
-	{ _MMIO(0x9888), 0x0a9d8000 },
-	{ _MMIO(0x9888), 0x109f0002 },
-	{ _MMIO(0x9888), 0x0ab94000 },
-	{ _MMIO(0x9888), 0x0d888000 },
-	{ _MMIO(0x9888), 0x038a0380 },
-	{ _MMIO(0x9888), 0x058a000e },
-	{ _MMIO(0x9888), 0x018a8000 },
-	{ _MMIO(0x9888), 0x0f8a8000 },
-	{ _MMIO(0x9888), 0x198a8000 },
-	{ _MMIO(0x9888), 0x1b8a00a0 },
-	{ _MMIO(0x9888), 0x078a0000 },
-	{ _MMIO(0x9888), 0x098a0000 },
-	{ _MMIO(0x9888), 0x238b2820 },
-	{ _MMIO(0x9888), 0x258b2550 },
-	{ _MMIO(0x9888), 0x198c1000 },
-	{ _MMIO(0x9888), 0x0b8d8000 },
-	{ _MMIO(0x9888), 0x1f85aa80 },
-	{ _MMIO(0x9888), 0x2185aaa0 },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x0d831021 },
-	{ _MMIO(0x9888), 0x0f83572f },
-	{ _MMIO(0x9888), 0x01835680 },
-	{ _MMIO(0x9888), 0x0383002c },
-	{ _MMIO(0x9888), 0x11830000 },
-	{ _MMIO(0x9888), 0x19835400 },
-	{ _MMIO(0x9888), 0x1b830001 },
-	{ _MMIO(0x9888), 0x05830000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0184c000 },
-	{ _MMIO(0x9888), 0x07848000 },
-	{ _MMIO(0x9888), 0x0984c000 },
-	{ _MMIO(0x9888), 0x0b84c000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x05844000 },
-	{ _MMIO(0x9888), 0x1b80c137 },
-	{ _MMIO(0x9888), 0x1d80c147 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x1180c000 },
-	{ _MMIO(0x9888), 0x17808000 },
-	{ _MMIO(0x9888), 0x1980c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x15804000 },
-	{ _MMIO(0x9888), 0x4d801110 },
-	{ _MMIO(0x9888), 0x4f800331 },
-	{ _MMIO(0x9888), 0x43800802 },
-	{ _MMIO(0x9888), 0x51800000 },
-	{ _MMIO(0x9888), 0x45801465 },
-	{ _MMIO(0x9888), 0x53801111 },
-	{ _MMIO(0x9888), 0x478014a5 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x3f800ca5 },
-	{ _MMIO(0x9888), 0x41800003 },
-};
-
-static const struct i915_oa_reg mux_config_render_basic_1_slices_0x02[] = {
-	{ _MMIO(0x9888), 0x143f000f },
-	{ _MMIO(0x9888), 0x14bf000f },
-	{ _MMIO(0x9888), 0x14910014 },
-	{ _MMIO(0x9888), 0x14b10014 },
-	{ _MMIO(0x9888), 0x118a0317 },
-	{ _MMIO(0x9888), 0x13837be0 },
-	{ _MMIO(0x9888), 0x3b800060 },
-	{ _MMIO(0x9888), 0x3d800005 },
-	{ _MMIO(0x9888), 0x0a3f0023 },
-	{ _MMIO(0x9888), 0x103f0000 },
-	{ _MMIO(0x9888), 0x0a5a4000 },
-	{ _MMIO(0x9888), 0x0a1d4000 },
-	{ _MMIO(0x9888), 0x0e1f8000 },
-	{ _MMIO(0x9888), 0x0a391000 },
-	{ _MMIO(0x9888), 0x00dc4000 },
-	{ _MMIO(0x9888), 0x06dc8000 },
-	{ _MMIO(0x9888), 0x08dcc000 },
-	{ _MMIO(0x9888), 0x00bd8000 },
-	{ _MMIO(0x9888), 0x18bd0800 },
-	{ _MMIO(0x9888), 0x0abf1180 },
-	{ _MMIO(0x9888), 0x10bf0000 },
-	{ _MMIO(0x9888), 0x00d84000 },
-	{ _MMIO(0x9888), 0x08d84000 },
-	{ _MMIO(0x9888), 0x0ada8000 },
-	{ _MMIO(0x9888), 0x00db4000 },
-	{ _MMIO(0x9888), 0x0edb8000 },
-	{ _MMIO(0x9888), 0x18db2400 },
-	{ _MMIO(0x9888), 0x0a9d8000 },
-	{ _MMIO(0x9888), 0x0c9f0800 },
-	{ _MMIO(0x9888), 0x0e9f2a00 },
-	{ _MMIO(0x9888), 0x109f0002 },
-	{ _MMIO(0x9888), 0x00b84000 },
-	{ _MMIO(0x9888), 0x0eb84000 },
-	{ _MMIO(0x9888), 0x16b84000 },
-	{ _MMIO(0x9888), 0x18b80001 },
-	{ _MMIO(0x9888), 0x00b92000 },
-	{ _MMIO(0x9888), 0x06b98000 },
-	{ _MMIO(0x9888), 0x08b9a000 },
-	{ _MMIO(0x9888), 0x0ab94000 },
-	{ _MMIO(0x9888), 0x00904000 },
-	{ _MMIO(0x9888), 0x08904000 },
-	{ _MMIO(0x9888), 0x00910030 },
-	{ _MMIO(0x9888), 0x08910031 },
-	{ _MMIO(0x9888), 0x10910000 },
-	{ _MMIO(0x9888), 0x00934000 },
-	{ _MMIO(0x9888), 0x16930020 },
-	{ _MMIO(0x9888), 0x06b08000 },
-	{ _MMIO(0x9888), 0x08b08000 },
-	{ _MMIO(0x9888), 0x06b11800 },
-	{ _MMIO(0x9888), 0x08b11880 },
-	{ _MMIO(0x9888), 0x10b10000 },
-	{ _MMIO(0x9888), 0x0eb34000 },
-	{ _MMIO(0x9888), 0x16b30080 },
-	{ _MMIO(0x9888), 0x01888000 },
-	{ _MMIO(0x9888), 0x0d88b800 },
-	{ _MMIO(0x9888), 0x038a0380 },
-	{ _MMIO(0x9888), 0x058a000e },
-	{ _MMIO(0x9888), 0x1b8a0080 },
-	{ _MMIO(0x9888), 0x078a0000 },
-	{ _MMIO(0x9888), 0x098a0000 },
-	{ _MMIO(0x9888), 0x238b2840 },
-	{ _MMIO(0x9888), 0x258b26a0 },
-	{ _MMIO(0x9888), 0x018c4000 },
-	{ _MMIO(0x9888), 0x0f8c4000 },
-	{ _MMIO(0x9888), 0x178c2000 },
-	{ _MMIO(0x9888), 0x198c1100 },
-	{ _MMIO(0x9888), 0x018d2000 },
-	{ _MMIO(0x9888), 0x078d8000 },
-	{ _MMIO(0x9888), 0x098da000 },
-	{ _MMIO(0x9888), 0x0b8d8000 },
-	{ _MMIO(0x9888), 0x1f85aa80 },
-	{ _MMIO(0x9888), 0x2185aaa0 },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x0d831021 },
-	{ _MMIO(0x9888), 0x0f83572f },
-	{ _MMIO(0x9888), 0x01835680 },
-	{ _MMIO(0x9888), 0x0383002c },
-	{ _MMIO(0x9888), 0x11830000 },
-	{ _MMIO(0x9888), 0x19835400 },
-	{ _MMIO(0x9888), 0x1b830001 },
-	{ _MMIO(0x9888), 0x05830000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0184c000 },
-	{ _MMIO(0x9888), 0x07848000 },
-	{ _MMIO(0x9888), 0x0984c000 },
-	{ _MMIO(0x9888), 0x0b84c000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x05844000 },
-	{ _MMIO(0x9888), 0x1b80c137 },
-	{ _MMIO(0x9888), 0x1d80c147 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x1180c000 },
-	{ _MMIO(0x9888), 0x17808000 },
-	{ _MMIO(0x9888), 0x1980c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x15804000 },
-	{ _MMIO(0x9888), 0x4d801550 },
-	{ _MMIO(0x9888), 0x4f800331 },
-	{ _MMIO(0x9888), 0x43800802 },
-	{ _MMIO(0x9888), 0x51800400 },
-	{ _MMIO(0x9888), 0x458004a1 },
-	{ _MMIO(0x9888), 0x53805555 },
-	{ _MMIO(0x9888), 0x47800421 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x3f801421 },
-	{ _MMIO(0x9888), 0x41800845 },
-};
-
-static int
-get_render_basic_mux_config(struct drm_i915_private *dev_priv,
-			    const struct i915_oa_reg **regs,
-			    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 2);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 2);
-
-	if (INTEL_INFO(dev_priv)->sseu.slice_mask & 0x01) {
-		regs[n] = mux_config_render_basic_0_slices_0x01;
-		lens[n] = ARRAY_SIZE(mux_config_render_basic_0_slices_0x01);
-		n++;
-	}
-	if (INTEL_INFO(dev_priv)->sseu.slice_mask & 0x02) {
-		regs[n] = mux_config_render_basic_1_slices_0x02;
-		lens[n] = ARRAY_SIZE(mux_config_render_basic_1_slices_0x02);
-		n++;
-	}
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_basic[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2740), 0x00000000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_basic[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00778008 },
-	{ _MMIO(0xe45c), 0x00088078 },
-	{ _MMIO(0xe55c), 0x00808708 },
-	{ _MMIO(0xe65c), 0x00a08908 },
-};
-
-static const struct i915_oa_reg mux_config_compute_basic_0_slices_0x01[] = {
-	{ _MMIO(0x9888), 0x105c00e0 },
-	{ _MMIO(0x9888), 0x105800e0 },
-	{ _MMIO(0x9888), 0x103800e0 },
-	{ _MMIO(0x9888), 0x3580001a },
-	{ _MMIO(0x9888), 0x3b800060 },
-	{ _MMIO(0x9888), 0x3d800005 },
-	{ _MMIO(0x9888), 0x065c2100 },
-	{ _MMIO(0x9888), 0x0a5c0041 },
-	{ _MMIO(0x9888), 0x0c5c6600 },
-	{ _MMIO(0x9888), 0x005c6580 },
-	{ _MMIO(0x9888), 0x085c8000 },
-	{ _MMIO(0x9888), 0x0e5c8000 },
-	{ _MMIO(0x9888), 0x00580042 },
-	{ _MMIO(0x9888), 0x08582080 },
-	{ _MMIO(0x9888), 0x0c58004c },
-	{ _MMIO(0x9888), 0x0e582580 },
-	{ _MMIO(0x9888), 0x005b4000 },
-	{ _MMIO(0x9888), 0x185b1000 },
-	{ _MMIO(0x9888), 0x1a5b0104 },
-	{ _MMIO(0x9888), 0x0c1fa800 },
-	{ _MMIO(0x9888), 0x0e1faa00 },
-	{ _MMIO(0x9888), 0x101f02aa },
-	{ _MMIO(0x9888), 0x08380042 },
-	{ _MMIO(0x9888), 0x0a382080 },
-	{ _MMIO(0x9888), 0x0e38404c },
-	{ _MMIO(0x9888), 0x0238404b },
-	{ _MMIO(0x9888), 0x00384000 },
-	{ _MMIO(0x9888), 0x16380000 },
-	{ _MMIO(0x9888), 0x18381145 },
-	{ _MMIO(0x9888), 0x04380000 },
-	{ _MMIO(0x9888), 0x0039a000 },
-	{ _MMIO(0x9888), 0x06398000 },
-	{ _MMIO(0x9888), 0x0839a000 },
-	{ _MMIO(0x9888), 0x0a39a000 },
-	{ _MMIO(0x9888), 0x0c39a000 },
-	{ _MMIO(0x9888), 0x0e39a000 },
-	{ _MMIO(0x9888), 0x02392000 },
-	{ _MMIO(0x9888), 0x018a8000 },
-	{ _MMIO(0x9888), 0x0f8a8000 },
-	{ _MMIO(0x9888), 0x198a8000 },
-	{ _MMIO(0x9888), 0x1b8aaaa0 },
-	{ _MMIO(0x9888), 0x1d8a0002 },
-	{ _MMIO(0x9888), 0x038a8000 },
-	{ _MMIO(0x9888), 0x058a8000 },
-	{ _MMIO(0x9888), 0x238b02a0 },
-	{ _MMIO(0x9888), 0x258b5550 },
-	{ _MMIO(0x9888), 0x278b0015 },
-	{ _MMIO(0x9888), 0x1f850a80 },
-	{ _MMIO(0x9888), 0x2185aaa0 },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x01834000 },
-	{ _MMIO(0x9888), 0x0f834000 },
-	{ _MMIO(0x9888), 0x19835400 },
-	{ _MMIO(0x9888), 0x1b830155 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x0184c000 },
-	{ _MMIO(0x9888), 0x07848000 },
-	{ _MMIO(0x9888), 0x0984c000 },
-	{ _MMIO(0x9888), 0x0b84c000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x03844000 },
-	{ _MMIO(0x9888), 0x17808137 },
-	{ _MMIO(0x9888), 0x1980c147 },
-	{ _MMIO(0x9888), 0x1b80c0e5 },
-	{ _MMIO(0x9888), 0x1d80c0e3 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x1180c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x13804000 },
-	{ _MMIO(0x9888), 0x15800000 },
-	{ _MMIO(0xd24), 0x00000000 },
-	{ _MMIO(0x9888), 0x4d801000 },
-	{ _MMIO(0x9888), 0x4f800111 },
-	{ _MMIO(0x9888), 0x43800062 },
-	{ _MMIO(0x9888), 0x51800000 },
-	{ _MMIO(0x9888), 0x45800062 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x47800062 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x3f801062 },
-	{ _MMIO(0x9888), 0x41801084 },
-};
-
-static const struct i915_oa_reg mux_config_compute_basic_2_slices_0x02[] = {
-	{ _MMIO(0x9888), 0x10dc00e0 },
-	{ _MMIO(0x9888), 0x10d800e0 },
-	{ _MMIO(0x9888), 0x10b800e0 },
-	{ _MMIO(0x9888), 0x3580001a },
-	{ _MMIO(0x9888), 0x3b800060 },
-	{ _MMIO(0x9888), 0x3d800005 },
-	{ _MMIO(0x9888), 0x06dc2100 },
-	{ _MMIO(0x9888), 0x0adc0041 },
-	{ _MMIO(0x9888), 0x0cdc6600 },
-	{ _MMIO(0x9888), 0x00dc6580 },
-	{ _MMIO(0x9888), 0x08dc8000 },
-	{ _MMIO(0x9888), 0x0edc8000 },
-	{ _MMIO(0x9888), 0x00d80042 },
-	{ _MMIO(0x9888), 0x08d82080 },
-	{ _MMIO(0x9888), 0x0cd8004c },
-	{ _MMIO(0x9888), 0x0ed82580 },
-	{ _MMIO(0x9888), 0x00db4000 },
-	{ _MMIO(0x9888), 0x18db1000 },
-	{ _MMIO(0x9888), 0x1adb0104 },
-	{ _MMIO(0x9888), 0x0c9fa800 },
-	{ _MMIO(0x9888), 0x0e9faa00 },
-	{ _MMIO(0x9888), 0x109f02aa },
-	{ _MMIO(0x9888), 0x08b80042 },
-	{ _MMIO(0x9888), 0x0ab82080 },
-	{ _MMIO(0x9888), 0x0eb8404c },
-	{ _MMIO(0x9888), 0x02b8404b },
-	{ _MMIO(0x9888), 0x00b84000 },
-	{ _MMIO(0x9888), 0x16b80000 },
-	{ _MMIO(0x9888), 0x18b81145 },
-	{ _MMIO(0x9888), 0x04b80000 },
-	{ _MMIO(0x9888), 0x00b9a000 },
-	{ _MMIO(0x9888), 0x06b98000 },
-	{ _MMIO(0x9888), 0x08b9a000 },
-	{ _MMIO(0x9888), 0x0ab9a000 },
-	{ _MMIO(0x9888), 0x0cb9a000 },
-	{ _MMIO(0x9888), 0x0eb9a000 },
-	{ _MMIO(0x9888), 0x02b92000 },
-	{ _MMIO(0x9888), 0x01888000 },
-	{ _MMIO(0x9888), 0x0d88f800 },
-	{ _MMIO(0x9888), 0x0f88000f },
-	{ _MMIO(0x9888), 0x03888000 },
-	{ _MMIO(0x9888), 0x05888000 },
-	{ _MMIO(0x9888), 0x238b0540 },
-	{ _MMIO(0x9888), 0x258baaa0 },
-	{ _MMIO(0x9888), 0x278b002a },
-	{ _MMIO(0x9888), 0x018c4000 },
-	{ _MMIO(0x9888), 0x0f8c4000 },
-	{ _MMIO(0x9888), 0x178c2000 },
-	{ _MMIO(0x9888), 0x198c5500 },
-	{ _MMIO(0x9888), 0x1b8c0015 },
-	{ _MMIO(0x9888), 0x038c4000 },
-	{ _MMIO(0x9888), 0x058c4000 },
-	{ _MMIO(0x9888), 0x018da000 },
-	{ _MMIO(0x9888), 0x078d8000 },
-	{ _MMIO(0x9888), 0x098da000 },
-	{ _MMIO(0x9888), 0x0b8da000 },
-	{ _MMIO(0x9888), 0x0d8da000 },
-	{ _MMIO(0x9888), 0x0f8da000 },
-	{ _MMIO(0x9888), 0x038d2000 },
-	{ _MMIO(0x9888), 0x1f850a80 },
-	{ _MMIO(0x9888), 0x2185aaa0 },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x01834000 },
-	{ _MMIO(0x9888), 0x0f834000 },
-	{ _MMIO(0x9888), 0x19835400 },
-	{ _MMIO(0x9888), 0x1b830155 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x0184c000 },
-	{ _MMIO(0x9888), 0x07848000 },
-	{ _MMIO(0x9888), 0x0984c000 },
-	{ _MMIO(0x9888), 0x0b84c000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x03844000 },
-	{ _MMIO(0x9888), 0x17808137 },
-	{ _MMIO(0x9888), 0x1980c147 },
-	{ _MMIO(0x9888), 0x1b80c0e5 },
-	{ _MMIO(0x9888), 0x1d80c0e3 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x1180c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x13804000 },
-	{ _MMIO(0x9888), 0x15800000 },
-	{ _MMIO(0xd24), 0x00000000 },
-	{ _MMIO(0x9888), 0x4d805000 },
-	{ _MMIO(0x9888), 0x4f800555 },
-	{ _MMIO(0x9888), 0x43800062 },
-	{ _MMIO(0x9888), 0x51800000 },
-	{ _MMIO(0x9888), 0x45800062 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x47800062 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x3f800062 },
-	{ _MMIO(0x9888), 0x41800000 },
-};
-
-static int
-get_compute_basic_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 2);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 2);
-
-	if (INTEL_INFO(dev_priv)->sseu.slice_mask & 0x01) {
-		regs[n] = mux_config_compute_basic_0_slices_0x01;
-		lens[n] = ARRAY_SIZE(mux_config_compute_basic_0_slices_0x01);
-		n++;
-	}
-	if (INTEL_INFO(dev_priv)->sseu.slice_mask & 0x02) {
-		regs[n] = mux_config_compute_basic_2_slices_0x02;
-		lens[n] = ARRAY_SIZE(mux_config_compute_basic_2_slices_0x02);
-		n++;
-	}
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_render_pipe_profile[] = {
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007ffea },
-	{ _MMIO(0x2774), 0x00007ffc },
-	{ _MMIO(0x2778), 0x0007affa },
-	{ _MMIO(0x277c), 0x0000f5fd },
-	{ _MMIO(0x2780), 0x00079ffa },
-	{ _MMIO(0x2784), 0x0000f3fb },
-	{ _MMIO(0x2788), 0x0007bf7a },
-	{ _MMIO(0x278c), 0x0000f7e7 },
-	{ _MMIO(0x2790), 0x0007fefa },
-	{ _MMIO(0x2794), 0x0000f7cf },
-	{ _MMIO(0x2798), 0x00077ffa },
-	{ _MMIO(0x279c), 0x0000efdf },
-	{ _MMIO(0x27a0), 0x0006fffa },
-	{ _MMIO(0x27a4), 0x0000cfbf },
-	{ _MMIO(0x27a8), 0x0003fffa },
-	{ _MMIO(0x27ac), 0x00005f7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_render_pipe_profile[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00015014 },
-	{ _MMIO(0xe658), 0x00025024 },
-	{ _MMIO(0xe758), 0x00035034 },
-	{ _MMIO(0xe45c), 0x00045044 },
-	{ _MMIO(0xe55c), 0x00055054 },
-	{ _MMIO(0xe65c), 0x00065064 },
-};
-
-static const struct i915_oa_reg mux_config_render_pipe_profile[] = {
-	{ _MMIO(0x9888), 0x0a1e0000 },
-	{ _MMIO(0x9888), 0x0c1f000f },
-	{ _MMIO(0x9888), 0x10176800 },
-	{ _MMIO(0x9888), 0x1191001f },
-	{ _MMIO(0x9888), 0x0b880320 },
-	{ _MMIO(0x9888), 0x01890c40 },
-	{ _MMIO(0x9888), 0x118a1c00 },
-	{ _MMIO(0x9888), 0x118d7c00 },
-	{ _MMIO(0x9888), 0x118e0020 },
-	{ _MMIO(0x9888), 0x118f4c00 },
-	{ _MMIO(0x9888), 0x11900000 },
-	{ _MMIO(0x9888), 0x13900001 },
-	{ _MMIO(0x9888), 0x065c4000 },
-	{ _MMIO(0x9888), 0x0c3d8000 },
-	{ _MMIO(0x9888), 0x06584000 },
-	{ _MMIO(0x9888), 0x0c5b4000 },
-	{ _MMIO(0x9888), 0x081e0040 },
-	{ _MMIO(0x9888), 0x0e1e0000 },
-	{ _MMIO(0x9888), 0x021f5400 },
-	{ _MMIO(0x9888), 0x001f0000 },
-	{ _MMIO(0x9888), 0x101f0010 },
-	{ _MMIO(0x9888), 0x0e1f0080 },
-	{ _MMIO(0x9888), 0x0c384000 },
-	{ _MMIO(0x9888), 0x06392000 },
-	{ _MMIO(0x9888), 0x0c13c000 },
-	{ _MMIO(0x9888), 0x06164000 },
-	{ _MMIO(0x9888), 0x06170012 },
-	{ _MMIO(0x9888), 0x00170000 },
-	{ _MMIO(0x9888), 0x01910005 },
-	{ _MMIO(0x9888), 0x07880002 },
-	{ _MMIO(0x9888), 0x01880c00 },
-	{ _MMIO(0x9888), 0x0f880000 },
-	{ _MMIO(0x9888), 0x0d880000 },
-	{ _MMIO(0x9888), 0x05880000 },
-	{ _MMIO(0x9888), 0x09890032 },
-	{ _MMIO(0x9888), 0x078a0800 },
-	{ _MMIO(0x9888), 0x0f8a0a00 },
-	{ _MMIO(0x9888), 0x198a4000 },
-	{ _MMIO(0x9888), 0x1b8a2000 },
-	{ _MMIO(0x9888), 0x1d8a0000 },
-	{ _MMIO(0x9888), 0x038a4000 },
-	{ _MMIO(0x9888), 0x0b8a8000 },
-	{ _MMIO(0x9888), 0x0d8a8000 },
-	{ _MMIO(0x9888), 0x238b54c0 },
-	{ _MMIO(0x9888), 0x258baa55 },
-	{ _MMIO(0x9888), 0x278b0019 },
-	{ _MMIO(0x9888), 0x198c0100 },
-	{ _MMIO(0x9888), 0x058c4000 },
-	{ _MMIO(0x9888), 0x0f8d0015 },
-	{ _MMIO(0x9888), 0x018d1000 },
-	{ _MMIO(0x9888), 0x098d8000 },
-	{ _MMIO(0x9888), 0x0b8df000 },
-	{ _MMIO(0x9888), 0x0d8d3000 },
-	{ _MMIO(0x9888), 0x038de000 },
-	{ _MMIO(0x9888), 0x058d3000 },
-	{ _MMIO(0x9888), 0x0d8e0004 },
-	{ _MMIO(0x9888), 0x058e000c },
-	{ _MMIO(0x9888), 0x098e0000 },
-	{ _MMIO(0x9888), 0x078e0000 },
-	{ _MMIO(0x9888), 0x038e0000 },
-	{ _MMIO(0x9888), 0x0b8f0020 },
-	{ _MMIO(0x9888), 0x198f0c00 },
-	{ _MMIO(0x9888), 0x078f8000 },
-	{ _MMIO(0x9888), 0x098f4000 },
-	{ _MMIO(0x9888), 0x0b900980 },
-	{ _MMIO(0x9888), 0x03900d80 },
-	{ _MMIO(0x9888), 0x01900000 },
-	{ _MMIO(0x9888), 0x1f85aa80 },
-	{ _MMIO(0x9888), 0x2185aaaa },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x01834000 },
-	{ _MMIO(0x9888), 0x0f834000 },
-	{ _MMIO(0x9888), 0x19835400 },
-	{ _MMIO(0x9888), 0x1b830155 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x0184c000 },
-	{ _MMIO(0x9888), 0x0784c000 },
-	{ _MMIO(0x9888), 0x0984c000 },
-	{ _MMIO(0x9888), 0x0b84c000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x1180c000 },
-	{ _MMIO(0x9888), 0x1780c000 },
-	{ _MMIO(0x9888), 0x1980c000 },
-	{ _MMIO(0x9888), 0x1b80c000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0xd24), 0x00000000 },
-	{ _MMIO(0x9888), 0x4d801111 },
-	{ _MMIO(0x9888), 0x3d800800 },
-	{ _MMIO(0x9888), 0x4f801011 },
-	{ _MMIO(0x9888), 0x43800443 },
-	{ _MMIO(0x9888), 0x51801111 },
-	{ _MMIO(0x9888), 0x45800422 },
-	{ _MMIO(0x9888), 0x53801111 },
-	{ _MMIO(0x9888), 0x47800c60 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x3f800422 },
-	{ _MMIO(0x9888), 0x41800021 },
-};
-
-static int
-get_render_pipe_profile_mux_config(struct drm_i915_private *dev_priv,
-				   const struct i915_oa_reg **regs,
-				   int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_render_pipe_profile;
-	lens[n] = ARRAY_SIZE(mux_config_render_pipe_profile);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_memory_reads[] = {
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x274c), 0x86543210 },
-	{ _MMIO(0x2748), 0x86543210 },
-	{ _MMIO(0x2744), 0x00006667 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x275c), 0x86543210 },
-	{ _MMIO(0x2758), 0x86543210 },
-	{ _MMIO(0x2754), 0x00006465 },
-	{ _MMIO(0x2750), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007f81a },
-	{ _MMIO(0x2774), 0x0000fe00 },
-	{ _MMIO(0x2778), 0x0007f82a },
-	{ _MMIO(0x277c), 0x0000fe00 },
-	{ _MMIO(0x2780), 0x0007f872 },
-	{ _MMIO(0x2784), 0x0000fe00 },
-	{ _MMIO(0x2788), 0x0007f8ba },
-	{ _MMIO(0x278c), 0x0000fe00 },
-	{ _MMIO(0x2790), 0x0007f87a },
-	{ _MMIO(0x2794), 0x0000fe00 },
-	{ _MMIO(0x2798), 0x0007f8ea },
-	{ _MMIO(0x279c), 0x0000fe00 },
-	{ _MMIO(0x27a0), 0x0007f8e2 },
-	{ _MMIO(0x27a4), 0x0000fe00 },
-	{ _MMIO(0x27a8), 0x0007f8f2 },
-	{ _MMIO(0x27ac), 0x0000fe00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_memory_reads[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00015014 },
-	{ _MMIO(0xe658), 0x00025024 },
-	{ _MMIO(0xe758), 0x00035034 },
-	{ _MMIO(0xe45c), 0x00045044 },
-	{ _MMIO(0xe55c), 0x00055054 },
-	{ _MMIO(0xe65c), 0x00065064 },
-};
-
-static const struct i915_oa_reg mux_config_memory_reads[] = {
-	{ _MMIO(0x9888), 0x198b0343 },
-	{ _MMIO(0x9888), 0x13845800 },
-	{ _MMIO(0x9888), 0x15840018 },
-	{ _MMIO(0x9888), 0x3580001a },
-	{ _MMIO(0x9888), 0x038b6300 },
-	{ _MMIO(0x9888), 0x058b6b62 },
-	{ _MMIO(0x9888), 0x078b006a },
-	{ _MMIO(0x9888), 0x118b0000 },
-	{ _MMIO(0x9888), 0x238b0000 },
-	{ _MMIO(0x9888), 0x258b0000 },
-	{ _MMIO(0x9888), 0x1f85a080 },
-	{ _MMIO(0x9888), 0x2185aaaa },
-	{ _MMIO(0x9888), 0x2385000a },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x01840018 },
-	{ _MMIO(0x9888), 0x07844c80 },
-	{ _MMIO(0x9888), 0x09840d9a },
-	{ _MMIO(0x9888), 0x0b840e9c },
-	{ _MMIO(0x9888), 0x0d840f9e },
-	{ _MMIO(0x9888), 0x0f840010 },
-	{ _MMIO(0x9888), 0x11840000 },
-	{ _MMIO(0x9888), 0x03848000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x2f8000e5 },
-	{ _MMIO(0x9888), 0x138080e3 },
-	{ _MMIO(0x9888), 0x1580c0e1 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x11804000 },
-	{ _MMIO(0x9888), 0x1780c000 },
-	{ _MMIO(0x9888), 0x1980c000 },
-	{ _MMIO(0x9888), 0x1b80c000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f804000 },
-	{ _MMIO(0xd24), 0x00000000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3d800800 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x43800842 },
-	{ _MMIO(0x9888), 0x51800000 },
-	{ _MMIO(0x9888), 0x45800842 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x47801042 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x3f800084 },
-	{ _MMIO(0x9888), 0x41800000 },
-};
-
-static int
-get_memory_reads_mux_config(struct drm_i915_private *dev_priv,
-			    const struct i915_oa_reg **regs,
-			    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_memory_reads;
-	lens[n] = ARRAY_SIZE(mux_config_memory_reads);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_memory_writes[] = {
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x274c), 0x86543210 },
-	{ _MMIO(0x2748), 0x86543210 },
-	{ _MMIO(0x2744), 0x00006667 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x275c), 0x86543210 },
-	{ _MMIO(0x2758), 0x86543210 },
-	{ _MMIO(0x2754), 0x00006465 },
-	{ _MMIO(0x2750), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007f81a },
-	{ _MMIO(0x2774), 0x0000fe00 },
-	{ _MMIO(0x2778), 0x0007f82a },
-	{ _MMIO(0x277c), 0x0000fe00 },
-	{ _MMIO(0x2780), 0x0007f822 },
-	{ _MMIO(0x2784), 0x0000fe00 },
-	{ _MMIO(0x2788), 0x0007f8ba },
-	{ _MMIO(0x278c), 0x0000fe00 },
-	{ _MMIO(0x2790), 0x0007f87a },
-	{ _MMIO(0x2794), 0x0000fe00 },
-	{ _MMIO(0x2798), 0x0007f8ea },
-	{ _MMIO(0x279c), 0x0000fe00 },
-	{ _MMIO(0x27a0), 0x0007f8e2 },
-	{ _MMIO(0x27a4), 0x0000fe00 },
-	{ _MMIO(0x27a8), 0x0007f8f2 },
-	{ _MMIO(0x27ac), 0x0000fe00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_memory_writes[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00015014 },
-	{ _MMIO(0xe658), 0x00025024 },
-	{ _MMIO(0xe758), 0x00035034 },
-	{ _MMIO(0xe45c), 0x00045044 },
-	{ _MMIO(0xe55c), 0x00055054 },
-	{ _MMIO(0xe65c), 0x00065064 },
-};
-
-static const struct i915_oa_reg mux_config_memory_writes[] = {
-	{ _MMIO(0x9888), 0x198b0343 },
-	{ _MMIO(0x9888), 0x13845400 },
-	{ _MMIO(0x9888), 0x3580001a },
-	{ _MMIO(0x9888), 0x3d800805 },
-	{ _MMIO(0x9888), 0x038b6300 },
-	{ _MMIO(0x9888), 0x058b6b62 },
-	{ _MMIO(0x9888), 0x078b006a },
-	{ _MMIO(0x9888), 0x118b0000 },
-	{ _MMIO(0x9888), 0x238b0000 },
-	{ _MMIO(0x9888), 0x258b0000 },
-	{ _MMIO(0x9888), 0x1f85a080 },
-	{ _MMIO(0x9888), 0x2185aaaa },
-	{ _MMIO(0x9888), 0x23850002 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x01840010 },
-	{ _MMIO(0x9888), 0x07844880 },
-	{ _MMIO(0x9888), 0x09840992 },
-	{ _MMIO(0x9888), 0x0b840a94 },
-	{ _MMIO(0x9888), 0x0d840b96 },
-	{ _MMIO(0x9888), 0x11840000 },
-	{ _MMIO(0x9888), 0x03848000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x2d800147 },
-	{ _MMIO(0x9888), 0x2f8000e5 },
-	{ _MMIO(0x9888), 0x138080e3 },
-	{ _MMIO(0x9888), 0x1580c0e1 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x11804000 },
-	{ _MMIO(0x9888), 0x1780c000 },
-	{ _MMIO(0x9888), 0x1980c000 },
-	{ _MMIO(0x9888), 0x1b80c000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f800000 },
-	{ _MMIO(0xd24), 0x00000000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x43800842 },
-	{ _MMIO(0x9888), 0x51800000 },
-	{ _MMIO(0x9888), 0x45800842 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x47801082 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x3f800084 },
-	{ _MMIO(0x9888), 0x41800000 },
-};
-
-static int
-get_memory_writes_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_memory_writes;
-	lens[n] = ARRAY_SIZE(mux_config_memory_writes);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_extended[] = {
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007fc2a },
-	{ _MMIO(0x2774), 0x0000bf00 },
-	{ _MMIO(0x2778), 0x0007fc6a },
-	{ _MMIO(0x277c), 0x0000bf00 },
-	{ _MMIO(0x2780), 0x0007fc92 },
-	{ _MMIO(0x2784), 0x0000bf00 },
-	{ _MMIO(0x2788), 0x0007fca2 },
-	{ _MMIO(0x278c), 0x0000bf00 },
-	{ _MMIO(0x2790), 0x0007fc32 },
-	{ _MMIO(0x2794), 0x0000bf00 },
-	{ _MMIO(0x2798), 0x0007fc9a },
-	{ _MMIO(0x279c), 0x0000bf00 },
-	{ _MMIO(0x27a0), 0x0007fe6a },
-	{ _MMIO(0x27a4), 0x0000bf00 },
-	{ _MMIO(0x27a8), 0x0007fe7a },
-	{ _MMIO(0x27ac), 0x0000bf00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_extended[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00778008 },
-	{ _MMIO(0xe45c), 0x00088078 },
-	{ _MMIO(0xe55c), 0x00808708 },
-	{ _MMIO(0xe65c), 0x00a08908 },
-};
-
-static const struct i915_oa_reg mux_config_compute_extended_0_subslices_0x01[] = {
-	{ _MMIO(0x9888), 0x143d0160 },
-	{ _MMIO(0x9888), 0x163d2800 },
-	{ _MMIO(0x9888), 0x183d0120 },
-	{ _MMIO(0x9888), 0x105800e0 },
-	{ _MMIO(0x9888), 0x005cc000 },
-	{ _MMIO(0x9888), 0x065c8000 },
-	{ _MMIO(0x9888), 0x085cc000 },
-	{ _MMIO(0x9888), 0x0a5cc000 },
-	{ _MMIO(0x9888), 0x0c5cc000 },
-	{ _MMIO(0x9888), 0x0e5cc000 },
-	{ _MMIO(0x9888), 0x025cc000 },
-	{ _MMIO(0x9888), 0x045cc000 },
-	{ _MMIO(0x9888), 0x003d0011 },
-	{ _MMIO(0x9888), 0x063d0900 },
-	{ _MMIO(0x9888), 0x083d0a13 },
-	{ _MMIO(0x9888), 0x0a3d0b15 },
-	{ _MMIO(0x9888), 0x0c3d2317 },
-	{ _MMIO(0x9888), 0x043d21b7 },
-	{ _MMIO(0x9888), 0x103d0000 },
-	{ _MMIO(0x9888), 0x0e3d0000 },
-	{ _MMIO(0x9888), 0x1a3d0000 },
-	{ _MMIO(0x9888), 0x0e5825c1 },
-	{ _MMIO(0x9888), 0x00586100 },
-	{ _MMIO(0x9888), 0x0258204c },
-	{ _MMIO(0x9888), 0x06588000 },
-	{ _MMIO(0x9888), 0x0858c000 },
-	{ _MMIO(0x9888), 0x0a58c000 },
-	{ _MMIO(0x9888), 0x0c58c000 },
-	{ _MMIO(0x9888), 0x0458c000 },
-	{ _MMIO(0x9888), 0x005b4000 },
-	{ _MMIO(0x9888), 0x0e5b4000 },
-	{ _MMIO(0x9888), 0x185b5400 },
-	{ _MMIO(0x9888), 0x1a5b0155 },
-	{ _MMIO(0x9888), 0x025b4000 },
-	{ _MMIO(0x9888), 0x045b4000 },
-	{ _MMIO(0x9888), 0x065b4000 },
-	{ _MMIO(0x9888), 0x085b4000 },
-	{ _MMIO(0x9888), 0x0a5b4000 },
-	{ _MMIO(0x9888), 0x0c1fa800 },
-	{ _MMIO(0x9888), 0x0e1faa2a },
-	{ _MMIO(0x9888), 0x101f02aa },
-	{ _MMIO(0x9888), 0x00384000 },
-	{ _MMIO(0x9888), 0x0e384000 },
-	{ _MMIO(0x9888), 0x16384000 },
-	{ _MMIO(0x9888), 0x18381555 },
-	{ _MMIO(0x9888), 0x02384000 },
-	{ _MMIO(0x9888), 0x04384000 },
-	{ _MMIO(0x9888), 0x06384000 },
-	{ _MMIO(0x9888), 0x08384000 },
-	{ _MMIO(0x9888), 0x0a384000 },
-	{ _MMIO(0x9888), 0x0039a000 },
-	{ _MMIO(0x9888), 0x06398000 },
-	{ _MMIO(0x9888), 0x0839a000 },
-	{ _MMIO(0x9888), 0x0a39a000 },
-	{ _MMIO(0x9888), 0x0c39a000 },
-	{ _MMIO(0x9888), 0x0e39a000 },
-	{ _MMIO(0x9888), 0x0239a000 },
-	{ _MMIO(0x9888), 0x0439a000 },
-	{ _MMIO(0x9888), 0x018a8000 },
-	{ _MMIO(0x9888), 0x0f8a8000 },
-	{ _MMIO(0x9888), 0x198a8000 },
-	{ _MMIO(0x9888), 0x1b8aaaa0 },
-	{ _MMIO(0x9888), 0x1d8a0002 },
-	{ _MMIO(0x9888), 0x038a8000 },
-	{ _MMIO(0x9888), 0x058a8000 },
-	{ _MMIO(0x9888), 0x078a8000 },
-	{ _MMIO(0x9888), 0x098a8000 },
-	{ _MMIO(0x9888), 0x0b8a8000 },
-	{ _MMIO(0x9888), 0x238b2aa0 },
-	{ _MMIO(0x9888), 0x258b5551 },
-	{ _MMIO(0x9888), 0x278b0015 },
-	{ _MMIO(0x9888), 0x1f85aa80 },
-	{ _MMIO(0x9888), 0x2185aaa2 },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x01834000 },
-	{ _MMIO(0x9888), 0x0f834000 },
-	{ _MMIO(0x9888), 0x19835400 },
-	{ _MMIO(0x9888), 0x1b830155 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0184c000 },
-	{ _MMIO(0x9888), 0x07848000 },
-	{ _MMIO(0x9888), 0x0984c000 },
-	{ _MMIO(0x9888), 0x0b84c000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x1180c000 },
-	{ _MMIO(0x9888), 0x17808000 },
-	{ _MMIO(0x9888), 0x1980c000 },
-	{ _MMIO(0x9888), 0x1b80c000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0xd24), 0x00000000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3d800000 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x43800000 },
-	{ _MMIO(0x9888), 0x51800000 },
-	{ _MMIO(0x9888), 0x45800000 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x47800420 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x3f800421 },
-	{ _MMIO(0x9888), 0x41800000 },
-};
-
-static const struct i915_oa_reg mux_config_compute_extended_2_subslices_0x02[] = {
-	{ _MMIO(0x9888), 0x105c00e0 },
-	{ _MMIO(0x9888), 0x145b0160 },
-	{ _MMIO(0x9888), 0x165b2800 },
-	{ _MMIO(0x9888), 0x185b0120 },
-	{ _MMIO(0x9888), 0x0e5c25c1 },
-	{ _MMIO(0x9888), 0x005c6100 },
-	{ _MMIO(0x9888), 0x025c204c },
-	{ _MMIO(0x9888), 0x065c8000 },
-	{ _MMIO(0x9888), 0x085cc000 },
-	{ _MMIO(0x9888), 0x0a5cc000 },
-	{ _MMIO(0x9888), 0x0c5cc000 },
-	{ _MMIO(0x9888), 0x045cc000 },
-	{ _MMIO(0x9888), 0x005b0011 },
-	{ _MMIO(0x9888), 0x065b0900 },
-	{ _MMIO(0x9888), 0x085b0a13 },
-	{ _MMIO(0x9888), 0x0a5b0b15 },
-	{ _MMIO(0x9888), 0x0c5b2317 },
-	{ _MMIO(0x9888), 0x045b21b7 },
-	{ _MMIO(0x9888), 0x105b0000 },
-	{ _MMIO(0x9888), 0x0e5b0000 },
-	{ _MMIO(0x9888), 0x1a5b0000 },
-	{ _MMIO(0x9888), 0x0c1fa800 },
-	{ _MMIO(0x9888), 0x0e1faa2a },
-	{ _MMIO(0x9888), 0x101f02aa },
-	{ _MMIO(0x9888), 0x00384000 },
-	{ _MMIO(0x9888), 0x0e384000 },
-	{ _MMIO(0x9888), 0x16384000 },
-	{ _MMIO(0x9888), 0x18381555 },
-	{ _MMIO(0x9888), 0x02384000 },
-	{ _MMIO(0x9888), 0x04384000 },
-	{ _MMIO(0x9888), 0x06384000 },
-	{ _MMIO(0x9888), 0x08384000 },
-	{ _MMIO(0x9888), 0x0a384000 },
-	{ _MMIO(0x9888), 0x0039a000 },
-	{ _MMIO(0x9888), 0x06398000 },
-	{ _MMIO(0x9888), 0x0839a000 },
-	{ _MMIO(0x9888), 0x0a39a000 },
-	{ _MMIO(0x9888), 0x0c39a000 },
-	{ _MMIO(0x9888), 0x0e39a000 },
-	{ _MMIO(0x9888), 0x0239a000 },
-	{ _MMIO(0x9888), 0x0439a000 },
-	{ _MMIO(0x9888), 0x018a8000 },
-	{ _MMIO(0x9888), 0x0f8a8000 },
-	{ _MMIO(0x9888), 0x198a8000 },
-	{ _MMIO(0x9888), 0x1b8aaaa0 },
-	{ _MMIO(0x9888), 0x1d8a0002 },
-	{ _MMIO(0x9888), 0x038a8000 },
-	{ _MMIO(0x9888), 0x058a8000 },
-	{ _MMIO(0x9888), 0x078a8000 },
-	{ _MMIO(0x9888), 0x098a8000 },
-	{ _MMIO(0x9888), 0x0b8a8000 },
-	{ _MMIO(0x9888), 0x238b2aa0 },
-	{ _MMIO(0x9888), 0x258b5551 },
-	{ _MMIO(0x9888), 0x278b0015 },
-	{ _MMIO(0x9888), 0x1f85aa80 },
-	{ _MMIO(0x9888), 0x2185aaa2 },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x01834000 },
-	{ _MMIO(0x9888), 0x0f834000 },
-	{ _MMIO(0x9888), 0x19835400 },
-	{ _MMIO(0x9888), 0x1b830155 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0184c000 },
-	{ _MMIO(0x9888), 0x07848000 },
-	{ _MMIO(0x9888), 0x0984c000 },
-	{ _MMIO(0x9888), 0x0b84c000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x1180c000 },
-	{ _MMIO(0x9888), 0x17808000 },
-	{ _MMIO(0x9888), 0x1980c000 },
-	{ _MMIO(0x9888), 0x1b80c000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0xd24), 0x00000000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3d800000 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x43800000 },
-	{ _MMIO(0x9888), 0x51800000 },
-	{ _MMIO(0x9888), 0x45800000 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x47800420 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x3f800421 },
-	{ _MMIO(0x9888), 0x41800000 },
-};
-
-static const struct i915_oa_reg mux_config_compute_extended_4_subslices_0x04[] = {
-	{ _MMIO(0x9888), 0x103800e0 },
-	{ _MMIO(0x9888), 0x143a0160 },
-	{ _MMIO(0x9888), 0x163a2800 },
-	{ _MMIO(0x9888), 0x183a0120 },
-	{ _MMIO(0x9888), 0x0c1fa800 },
-	{ _MMIO(0x9888), 0x0e1faa2a },
-	{ _MMIO(0x9888), 0x101f02aa },
-	{ _MMIO(0x9888), 0x0e38a5c1 },
-	{ _MMIO(0x9888), 0x0038a100 },
-	{ _MMIO(0x9888), 0x0238204c },
-	{ _MMIO(0x9888), 0x16388000 },
-	{ _MMIO(0x9888), 0x183802aa },
-	{ _MMIO(0x9888), 0x04380000 },
-	{ _MMIO(0x9888), 0x06380000 },
-	{ _MMIO(0x9888), 0x08388000 },
-	{ _MMIO(0x9888), 0x0a388000 },
-	{ _MMIO(0x9888), 0x0039a000 },
-	{ _MMIO(0x9888), 0x06398000 },
-	{ _MMIO(0x9888), 0x0839a000 },
-	{ _MMIO(0x9888), 0x0a39a000 },
-	{ _MMIO(0x9888), 0x0c39a000 },
-	{ _MMIO(0x9888), 0x0e39a000 },
-	{ _MMIO(0x9888), 0x0239a000 },
-	{ _MMIO(0x9888), 0x0439a000 },
-	{ _MMIO(0x9888), 0x003a0011 },
-	{ _MMIO(0x9888), 0x063a0900 },
-	{ _MMIO(0x9888), 0x083a0a13 },
-	{ _MMIO(0x9888), 0x0a3a0b15 },
-	{ _MMIO(0x9888), 0x0c3a2317 },
-	{ _MMIO(0x9888), 0x043a21b7 },
-	{ _MMIO(0x9888), 0x103a0000 },
-	{ _MMIO(0x9888), 0x0e3a0000 },
-	{ _MMIO(0x9888), 0x1a3a0000 },
-	{ _MMIO(0x9888), 0x018a8000 },
-	{ _MMIO(0x9888), 0x0f8a8000 },
-	{ _MMIO(0x9888), 0x198a8000 },
-	{ _MMIO(0x9888), 0x1b8aaaa0 },
-	{ _MMIO(0x9888), 0x1d8a0002 },
-	{ _MMIO(0x9888), 0x038a8000 },
-	{ _MMIO(0x9888), 0x058a8000 },
-	{ _MMIO(0x9888), 0x078a8000 },
-	{ _MMIO(0x9888), 0x098a8000 },
-	{ _MMIO(0x9888), 0x0b8a8000 },
-	{ _MMIO(0x9888), 0x238b2aa0 },
-	{ _MMIO(0x9888), 0x258b5551 },
-	{ _MMIO(0x9888), 0x278b0015 },
-	{ _MMIO(0x9888), 0x1f85aa80 },
-	{ _MMIO(0x9888), 0x2185aaa2 },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x01834000 },
-	{ _MMIO(0x9888), 0x0f834000 },
-	{ _MMIO(0x9888), 0x19835400 },
-	{ _MMIO(0x9888), 0x1b830155 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0184c000 },
-	{ _MMIO(0x9888), 0x07848000 },
-	{ _MMIO(0x9888), 0x0984c000 },
-	{ _MMIO(0x9888), 0x0b84c000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x1180c000 },
-	{ _MMIO(0x9888), 0x17808000 },
-	{ _MMIO(0x9888), 0x1980c000 },
-	{ _MMIO(0x9888), 0x1b80c000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0xd24), 0x00000000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3d800000 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x43800000 },
-	{ _MMIO(0x9888), 0x51800000 },
-	{ _MMIO(0x9888), 0x45800000 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x47800420 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x3f800421 },
-	{ _MMIO(0x9888), 0x41800000 },
-};
-
-static const struct i915_oa_reg mux_config_compute_extended_1_subslices_0x08[] = {
-	{ _MMIO(0x9888), 0x14bd0160 },
-	{ _MMIO(0x9888), 0x16bd2800 },
-	{ _MMIO(0x9888), 0x18bd0120 },
-	{ _MMIO(0x9888), 0x10d800e0 },
-	{ _MMIO(0x9888), 0x00dcc000 },
-	{ _MMIO(0x9888), 0x06dc8000 },
-	{ _MMIO(0x9888), 0x08dcc000 },
-	{ _MMIO(0x9888), 0x0adcc000 },
-	{ _MMIO(0x9888), 0x0cdcc000 },
-	{ _MMIO(0x9888), 0x0edcc000 },
-	{ _MMIO(0x9888), 0x02dcc000 },
-	{ _MMIO(0x9888), 0x04dcc000 },
-	{ _MMIO(0x9888), 0x00bd0011 },
-	{ _MMIO(0x9888), 0x06bd0900 },
-	{ _MMIO(0x9888), 0x08bd0a13 },
-	{ _MMIO(0x9888), 0x0abd0b15 },
-	{ _MMIO(0x9888), 0x0cbd2317 },
-	{ _MMIO(0x9888), 0x04bd21b7 },
-	{ _MMIO(0x9888), 0x10bd0000 },
-	{ _MMIO(0x9888), 0x0ebd0000 },
-	{ _MMIO(0x9888), 0x1abd0000 },
-	{ _MMIO(0x9888), 0x0ed825c1 },
-	{ _MMIO(0x9888), 0x00d86100 },
-	{ _MMIO(0x9888), 0x02d8204c },
-	{ _MMIO(0x9888), 0x06d88000 },
-	{ _MMIO(0x9888), 0x08d8c000 },
-	{ _MMIO(0x9888), 0x0ad8c000 },
-	{ _MMIO(0x9888), 0x0cd8c000 },
-	{ _MMIO(0x9888), 0x04d8c000 },
-	{ _MMIO(0x9888), 0x00db4000 },
-	{ _MMIO(0x9888), 0x0edb4000 },
-	{ _MMIO(0x9888), 0x18db5400 },
-	{ _MMIO(0x9888), 0x1adb0155 },
-	{ _MMIO(0x9888), 0x02db4000 },
-	{ _MMIO(0x9888), 0x04db4000 },
-	{ _MMIO(0x9888), 0x06db4000 },
-	{ _MMIO(0x9888), 0x08db4000 },
-	{ _MMIO(0x9888), 0x0adb4000 },
-	{ _MMIO(0x9888), 0x0c9fa800 },
-	{ _MMIO(0x9888), 0x0e9faa2a },
-	{ _MMIO(0x9888), 0x109f02aa },
-	{ _MMIO(0x9888), 0x00b84000 },
-	{ _MMIO(0x9888), 0x0eb84000 },
-	{ _MMIO(0x9888), 0x16b84000 },
-	{ _MMIO(0x9888), 0x18b81555 },
-	{ _MMIO(0x9888), 0x02b84000 },
-	{ _MMIO(0x9888), 0x04b84000 },
-	{ _MMIO(0x9888), 0x06b84000 },
-	{ _MMIO(0x9888), 0x08b84000 },
-	{ _MMIO(0x9888), 0x0ab84000 },
-	{ _MMIO(0x9888), 0x00b9a000 },
-	{ _MMIO(0x9888), 0x06b98000 },
-	{ _MMIO(0x9888), 0x08b9a000 },
-	{ _MMIO(0x9888), 0x0ab9a000 },
-	{ _MMIO(0x9888), 0x0cb9a000 },
-	{ _MMIO(0x9888), 0x0eb9a000 },
-	{ _MMIO(0x9888), 0x02b9a000 },
-	{ _MMIO(0x9888), 0x04b9a000 },
-	{ _MMIO(0x9888), 0x01888000 },
-	{ _MMIO(0x9888), 0x0d88f800 },
-	{ _MMIO(0x9888), 0x0f88000f },
-	{ _MMIO(0x9888), 0x03888000 },
-	{ _MMIO(0x9888), 0x05888000 },
-	{ _MMIO(0x9888), 0x07888000 },
-	{ _MMIO(0x9888), 0x09888000 },
-	{ _MMIO(0x9888), 0x0b888000 },
-	{ _MMIO(0x9888), 0x238b5540 },
-	{ _MMIO(0x9888), 0x258baaa2 },
-	{ _MMIO(0x9888), 0x278b002a },
-	{ _MMIO(0x9888), 0x018c4000 },
-	{ _MMIO(0x9888), 0x0f8c4000 },
-	{ _MMIO(0x9888), 0x178c2000 },
-	{ _MMIO(0x9888), 0x198c5500 },
-	{ _MMIO(0x9888), 0x1b8c0015 },
-	{ _MMIO(0x9888), 0x038c4000 },
-	{ _MMIO(0x9888), 0x058c4000 },
-	{ _MMIO(0x9888), 0x078c4000 },
-	{ _MMIO(0x9888), 0x098c4000 },
-	{ _MMIO(0x9888), 0x0b8c4000 },
-	{ _MMIO(0x9888), 0x018da000 },
-	{ _MMIO(0x9888), 0x078d8000 },
-	{ _MMIO(0x9888), 0x098da000 },
-	{ _MMIO(0x9888), 0x0b8da000 },
-	{ _MMIO(0x9888), 0x0d8da000 },
-	{ _MMIO(0x9888), 0x0f8da000 },
-	{ _MMIO(0x9888), 0x038da000 },
-	{ _MMIO(0x9888), 0x058da000 },
-	{ _MMIO(0x9888), 0x1f85aa80 },
-	{ _MMIO(0x9888), 0x2185aaa2 },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x01834000 },
-	{ _MMIO(0x9888), 0x0f834000 },
-	{ _MMIO(0x9888), 0x19835400 },
-	{ _MMIO(0x9888), 0x1b830155 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0184c000 },
-	{ _MMIO(0x9888), 0x07848000 },
-	{ _MMIO(0x9888), 0x0984c000 },
-	{ _MMIO(0x9888), 0x0b84c000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x1180c000 },
-	{ _MMIO(0x9888), 0x17808000 },
-	{ _MMIO(0x9888), 0x1980c000 },
-	{ _MMIO(0x9888), 0x1b80c000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0xd24), 0x00000000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3d800000 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x43800000 },
-	{ _MMIO(0x9888), 0x51800000 },
-	{ _MMIO(0x9888), 0x45800000 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x47800420 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x3f800421 },
-	{ _MMIO(0x9888), 0x41800000 },
-};
-
-static const struct i915_oa_reg mux_config_compute_extended_3_subslices_0x10[] = {
-	{ _MMIO(0x9888), 0x10dc00e0 },
-	{ _MMIO(0x9888), 0x14db0160 },
-	{ _MMIO(0x9888), 0x16db2800 },
-	{ _MMIO(0x9888), 0x18db0120 },
-	{ _MMIO(0x9888), 0x0edc25c1 },
-	{ _MMIO(0x9888), 0x00dc6100 },
-	{ _MMIO(0x9888), 0x02dc204c },
-	{ _MMIO(0x9888), 0x06dc8000 },
-	{ _MMIO(0x9888), 0x08dcc000 },
-	{ _MMIO(0x9888), 0x0adcc000 },
-	{ _MMIO(0x9888), 0x0cdcc000 },
-	{ _MMIO(0x9888), 0x04dcc000 },
-	{ _MMIO(0x9888), 0x00db0011 },
-	{ _MMIO(0x9888), 0x06db0900 },
-	{ _MMIO(0x9888), 0x08db0a13 },
-	{ _MMIO(0x9888), 0x0adb0b15 },
-	{ _MMIO(0x9888), 0x0cdb2317 },
-	{ _MMIO(0x9888), 0x04db21b7 },
-	{ _MMIO(0x9888), 0x10db0000 },
-	{ _MMIO(0x9888), 0x0edb0000 },
-	{ _MMIO(0x9888), 0x1adb0000 },
-	{ _MMIO(0x9888), 0x0c9fa800 },
-	{ _MMIO(0x9888), 0x0e9faa2a },
-	{ _MMIO(0x9888), 0x109f02aa },
-	{ _MMIO(0x9888), 0x00b84000 },
-	{ _MMIO(0x9888), 0x0eb84000 },
-	{ _MMIO(0x9888), 0x16b84000 },
-	{ _MMIO(0x9888), 0x18b81555 },
-	{ _MMIO(0x9888), 0x02b84000 },
-	{ _MMIO(0x9888), 0x04b84000 },
-	{ _MMIO(0x9888), 0x06b84000 },
-	{ _MMIO(0x9888), 0x08b84000 },
-	{ _MMIO(0x9888), 0x0ab84000 },
-	{ _MMIO(0x9888), 0x00b9a000 },
-	{ _MMIO(0x9888), 0x06b98000 },
-	{ _MMIO(0x9888), 0x08b9a000 },
-	{ _MMIO(0x9888), 0x0ab9a000 },
-	{ _MMIO(0x9888), 0x0cb9a000 },
-	{ _MMIO(0x9888), 0x0eb9a000 },
-	{ _MMIO(0x9888), 0x02b9a000 },
-	{ _MMIO(0x9888), 0x04b9a000 },
-	{ _MMIO(0x9888), 0x01888000 },
-	{ _MMIO(0x9888), 0x0d88f800 },
-	{ _MMIO(0x9888), 0x0f88000f },
-	{ _MMIO(0x9888), 0x03888000 },
-	{ _MMIO(0x9888), 0x05888000 },
-	{ _MMIO(0x9888), 0x07888000 },
-	{ _MMIO(0x9888), 0x09888000 },
-	{ _MMIO(0x9888), 0x0b888000 },
-	{ _MMIO(0x9888), 0x238b5540 },
-	{ _MMIO(0x9888), 0x258baaa2 },
-	{ _MMIO(0x9888), 0x278b002a },
-	{ _MMIO(0x9888), 0x018c4000 },
-	{ _MMIO(0x9888), 0x0f8c4000 },
-	{ _MMIO(0x9888), 0x178c2000 },
-	{ _MMIO(0x9888), 0x198c5500 },
-	{ _MMIO(0x9888), 0x1b8c0015 },
-	{ _MMIO(0x9888), 0x038c4000 },
-	{ _MMIO(0x9888), 0x058c4000 },
-	{ _MMIO(0x9888), 0x078c4000 },
-	{ _MMIO(0x9888), 0x098c4000 },
-	{ _MMIO(0x9888), 0x0b8c4000 },
-	{ _MMIO(0x9888), 0x018da000 },
-	{ _MMIO(0x9888), 0x078d8000 },
-	{ _MMIO(0x9888), 0x098da000 },
-	{ _MMIO(0x9888), 0x0b8da000 },
-	{ _MMIO(0x9888), 0x0d8da000 },
-	{ _MMIO(0x9888), 0x0f8da000 },
-	{ _MMIO(0x9888), 0x038da000 },
-	{ _MMIO(0x9888), 0x058da000 },
-	{ _MMIO(0x9888), 0x1f85aa80 },
-	{ _MMIO(0x9888), 0x2185aaa2 },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x01834000 },
-	{ _MMIO(0x9888), 0x0f834000 },
-	{ _MMIO(0x9888), 0x19835400 },
-	{ _MMIO(0x9888), 0x1b830155 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0184c000 },
-	{ _MMIO(0x9888), 0x07848000 },
-	{ _MMIO(0x9888), 0x0984c000 },
-	{ _MMIO(0x9888), 0x0b84c000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x1180c000 },
-	{ _MMIO(0x9888), 0x17808000 },
-	{ _MMIO(0x9888), 0x1980c000 },
-	{ _MMIO(0x9888), 0x1b80c000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0xd24), 0x00000000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3d800000 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x43800000 },
-	{ _MMIO(0x9888), 0x51800000 },
-	{ _MMIO(0x9888), 0x45800000 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x47800420 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x3f800421 },
-	{ _MMIO(0x9888), 0x41800000 },
-};
-
-static const struct i915_oa_reg mux_config_compute_extended_5_subslices_0x20[] = {
-	{ _MMIO(0x9888), 0x10b800e0 },
-	{ _MMIO(0x9888), 0x14ba0160 },
-	{ _MMIO(0x9888), 0x16ba2800 },
-	{ _MMIO(0x9888), 0x18ba0120 },
-	{ _MMIO(0x9888), 0x0c9fa800 },
-	{ _MMIO(0x9888), 0x0e9faa2a },
-	{ _MMIO(0x9888), 0x109f02aa },
-	{ _MMIO(0x9888), 0x0eb8a5c1 },
-	{ _MMIO(0x9888), 0x00b8a100 },
-	{ _MMIO(0x9888), 0x02b8204c },
-	{ _MMIO(0x9888), 0x16b88000 },
-	{ _MMIO(0x9888), 0x18b802aa },
-	{ _MMIO(0x9888), 0x04b80000 },
-	{ _MMIO(0x9888), 0x06b80000 },
-	{ _MMIO(0x9888), 0x08b88000 },
-	{ _MMIO(0x9888), 0x0ab88000 },
-	{ _MMIO(0x9888), 0x00b9a000 },
-	{ _MMIO(0x9888), 0x06b98000 },
-	{ _MMIO(0x9888), 0x08b9a000 },
-	{ _MMIO(0x9888), 0x0ab9a000 },
-	{ _MMIO(0x9888), 0x0cb9a000 },
-	{ _MMIO(0x9888), 0x0eb9a000 },
-	{ _MMIO(0x9888), 0x02b9a000 },
-	{ _MMIO(0x9888), 0x04b9a000 },
-	{ _MMIO(0x9888), 0x00ba0011 },
-	{ _MMIO(0x9888), 0x06ba0900 },
-	{ _MMIO(0x9888), 0x08ba0a13 },
-	{ _MMIO(0x9888), 0x0aba0b15 },
-	{ _MMIO(0x9888), 0x0cba2317 },
-	{ _MMIO(0x9888), 0x04ba21b7 },
-	{ _MMIO(0x9888), 0x10ba0000 },
-	{ _MMIO(0x9888), 0x0eba0000 },
-	{ _MMIO(0x9888), 0x1aba0000 },
-	{ _MMIO(0x9888), 0x01888000 },
-	{ _MMIO(0x9888), 0x0d88f800 },
-	{ _MMIO(0x9888), 0x0f88000f },
-	{ _MMIO(0x9888), 0x03888000 },
-	{ _MMIO(0x9888), 0x05888000 },
-	{ _MMIO(0x9888), 0x07888000 },
-	{ _MMIO(0x9888), 0x09888000 },
-	{ _MMIO(0x9888), 0x0b888000 },
-	{ _MMIO(0x9888), 0x238b5540 },
-	{ _MMIO(0x9888), 0x258baaa2 },
-	{ _MMIO(0x9888), 0x278b002a },
-	{ _MMIO(0x9888), 0x018c4000 },
-	{ _MMIO(0x9888), 0x0f8c4000 },
-	{ _MMIO(0x9888), 0x178c2000 },
-	{ _MMIO(0x9888), 0x198c5500 },
-	{ _MMIO(0x9888), 0x1b8c0015 },
-	{ _MMIO(0x9888), 0x038c4000 },
-	{ _MMIO(0x9888), 0x058c4000 },
-	{ _MMIO(0x9888), 0x078c4000 },
-	{ _MMIO(0x9888), 0x098c4000 },
-	{ _MMIO(0x9888), 0x0b8c4000 },
-	{ _MMIO(0x9888), 0x018da000 },
-	{ _MMIO(0x9888), 0x078d8000 },
-	{ _MMIO(0x9888), 0x098da000 },
-	{ _MMIO(0x9888), 0x0b8da000 },
-	{ _MMIO(0x9888), 0x0d8da000 },
-	{ _MMIO(0x9888), 0x0f8da000 },
-	{ _MMIO(0x9888), 0x038da000 },
-	{ _MMIO(0x9888), 0x058da000 },
-	{ _MMIO(0x9888), 0x1f85aa80 },
-	{ _MMIO(0x9888), 0x2185aaa2 },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x01834000 },
-	{ _MMIO(0x9888), 0x0f834000 },
-	{ _MMIO(0x9888), 0x19835400 },
-	{ _MMIO(0x9888), 0x1b830155 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0184c000 },
-	{ _MMIO(0x9888), 0x07848000 },
-	{ _MMIO(0x9888), 0x0984c000 },
-	{ _MMIO(0x9888), 0x0b84c000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x1180c000 },
-	{ _MMIO(0x9888), 0x17808000 },
-	{ _MMIO(0x9888), 0x1980c000 },
-	{ _MMIO(0x9888), 0x1b80c000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0xd24), 0x00000000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3d800000 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x43800000 },
-	{ _MMIO(0x9888), 0x51800000 },
-	{ _MMIO(0x9888), 0x45800000 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x47800420 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x3f800421 },
-	{ _MMIO(0x9888), 0x41800000 },
-};
-
-static int
-get_compute_extended_mux_config(struct drm_i915_private *dev_priv,
-				const struct i915_oa_reg **regs,
-				int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 6);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 6);
-
-	if (INTEL_INFO(dev_priv)->sseu.subslice_mask & 0x01) {
-		regs[n] = mux_config_compute_extended_0_subslices_0x01;
-		lens[n] = ARRAY_SIZE(mux_config_compute_extended_0_subslices_0x01);
-		n++;
-	}
-	if (INTEL_INFO(dev_priv)->sseu.subslice_mask & 0x08) {
-		regs[n] = mux_config_compute_extended_1_subslices_0x08;
-		lens[n] = ARRAY_SIZE(mux_config_compute_extended_1_subslices_0x08);
-		n++;
-	}
-	if (INTEL_INFO(dev_priv)->sseu.subslice_mask & 0x02) {
-		regs[n] = mux_config_compute_extended_2_subslices_0x02;
-		lens[n] = ARRAY_SIZE(mux_config_compute_extended_2_subslices_0x02);
-		n++;
-	}
-	if (INTEL_INFO(dev_priv)->sseu.subslice_mask & 0x10) {
-		regs[n] = mux_config_compute_extended_3_subslices_0x10;
-		lens[n] = ARRAY_SIZE(mux_config_compute_extended_3_subslices_0x10);
-		n++;
-	}
-	if (INTEL_INFO(dev_priv)->sseu.subslice_mask & 0x04) {
-		regs[n] = mux_config_compute_extended_4_subslices_0x04;
-		lens[n] = ARRAY_SIZE(mux_config_compute_extended_4_subslices_0x04);
-		n++;
-	}
-	if (INTEL_INFO(dev_priv)->sseu.subslice_mask & 0x20) {
-		regs[n] = mux_config_compute_extended_5_subslices_0x20;
-		lens[n] = ARRAY_SIZE(mux_config_compute_extended_5_subslices_0x20);
-		n++;
-	}
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_l3_cache[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x30800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007fffa },
-	{ _MMIO(0x2774), 0x0000fefe },
-	{ _MMIO(0x2778), 0x0007fffa },
-	{ _MMIO(0x277c), 0x0000fefd },
-	{ _MMIO(0x2790), 0x0007fffa },
-	{ _MMIO(0x2794), 0x0000fbef },
-	{ _MMIO(0x2798), 0x0007fffa },
-	{ _MMIO(0x279c), 0x0000fbdf },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_l3_cache[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00101100 },
-	{ _MMIO(0xe45c), 0x00201200 },
-	{ _MMIO(0xe55c), 0x00301300 },
-	{ _MMIO(0xe65c), 0x00401400 },
-};
-
-static const struct i915_oa_reg mux_config_compute_l3_cache[] = {
-	{ _MMIO(0x9888), 0x143f00b3 },
-	{ _MMIO(0x9888), 0x14bf00b3 },
-	{ _MMIO(0x9888), 0x138303c0 },
-	{ _MMIO(0x9888), 0x3b800060 },
-	{ _MMIO(0x9888), 0x3d800805 },
-	{ _MMIO(0x9888), 0x003f0029 },
-	{ _MMIO(0x9888), 0x063f1400 },
-	{ _MMIO(0x9888), 0x083f1225 },
-	{ _MMIO(0x9888), 0x0e3f1327 },
-	{ _MMIO(0x9888), 0x103f0000 },
-	{ _MMIO(0x9888), 0x005a4000 },
-	{ _MMIO(0x9888), 0x065a8000 },
-	{ _MMIO(0x9888), 0x085ac000 },
-	{ _MMIO(0x9888), 0x0e5ac000 },
-	{ _MMIO(0x9888), 0x001d4000 },
-	{ _MMIO(0x9888), 0x061d8000 },
-	{ _MMIO(0x9888), 0x081dc000 },
-	{ _MMIO(0x9888), 0x0e1dc000 },
-	{ _MMIO(0x9888), 0x0c1f0800 },
-	{ _MMIO(0x9888), 0x0e1f2a00 },
-	{ _MMIO(0x9888), 0x101f0280 },
-	{ _MMIO(0x9888), 0x00391000 },
-	{ _MMIO(0x9888), 0x06394000 },
-	{ _MMIO(0x9888), 0x08395000 },
-	{ _MMIO(0x9888), 0x0e395000 },
-	{ _MMIO(0x9888), 0x0abf1429 },
-	{ _MMIO(0x9888), 0x0cbf1225 },
-	{ _MMIO(0x9888), 0x00bf1380 },
-	{ _MMIO(0x9888), 0x02bf0026 },
-	{ _MMIO(0x9888), 0x10bf0000 },
-	{ _MMIO(0x9888), 0x0adac000 },
-	{ _MMIO(0x9888), 0x0cdac000 },
-	{ _MMIO(0x9888), 0x00da8000 },
-	{ _MMIO(0x9888), 0x02da4000 },
-	{ _MMIO(0x9888), 0x0a9dc000 },
-	{ _MMIO(0x9888), 0x0c9dc000 },
-	{ _MMIO(0x9888), 0x009d8000 },
-	{ _MMIO(0x9888), 0x029d4000 },
-	{ _MMIO(0x9888), 0x0e9f8000 },
-	{ _MMIO(0x9888), 0x109f002a },
-	{ _MMIO(0x9888), 0x0c9fa000 },
-	{ _MMIO(0x9888), 0x0ab95000 },
-	{ _MMIO(0x9888), 0x0cb95000 },
-	{ _MMIO(0x9888), 0x00b94000 },
-	{ _MMIO(0x9888), 0x02b91000 },
-	{ _MMIO(0x9888), 0x0d88c000 },
-	{ _MMIO(0x9888), 0x0f880003 },
-	{ _MMIO(0x9888), 0x03888000 },
-	{ _MMIO(0x9888), 0x05888000 },
-	{ _MMIO(0x9888), 0x018a8000 },
-	{ _MMIO(0x9888), 0x0f8a8000 },
-	{ _MMIO(0x9888), 0x198a8000 },
-	{ _MMIO(0x9888), 0x1b8a8020 },
-	{ _MMIO(0x9888), 0x1d8a0002 },
-	{ _MMIO(0x9888), 0x238b0520 },
-	{ _MMIO(0x9888), 0x258ba950 },
-	{ _MMIO(0x9888), 0x278b0016 },
-	{ _MMIO(0x9888), 0x198c5400 },
-	{ _MMIO(0x9888), 0x1b8c0001 },
-	{ _MMIO(0x9888), 0x038c4000 },
-	{ _MMIO(0x9888), 0x058c4000 },
-	{ _MMIO(0x9888), 0x0b8da000 },
-	{ _MMIO(0x9888), 0x0d8da000 },
-	{ _MMIO(0x9888), 0x018d8000 },
-	{ _MMIO(0x9888), 0x038d2000 },
-	{ _MMIO(0x9888), 0x1f85aa80 },
-	{ _MMIO(0x9888), 0x2185aaa0 },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x03835180 },
-	{ _MMIO(0x9888), 0x05834022 },
-	{ _MMIO(0x9888), 0x11830000 },
-	{ _MMIO(0x9888), 0x01834000 },
-	{ _MMIO(0x9888), 0x0f834000 },
-	{ _MMIO(0x9888), 0x19835400 },
-	{ _MMIO(0x9888), 0x1b830155 },
-	{ _MMIO(0x9888), 0x07830000 },
-	{ _MMIO(0x9888), 0x09830000 },
-	{ _MMIO(0x9888), 0x0184c000 },
-	{ _MMIO(0x9888), 0x07848000 },
-	{ _MMIO(0x9888), 0x0984c000 },
-	{ _MMIO(0x9888), 0x0b84c000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x05844000 },
-	{ _MMIO(0x9888), 0x1b80c137 },
-	{ _MMIO(0x9888), 0x1d80c147 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x1180c000 },
-	{ _MMIO(0x9888), 0x17808000 },
-	{ _MMIO(0x9888), 0x1980c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x15804000 },
-	{ _MMIO(0xd24), 0x00000000 },
-	{ _MMIO(0x9888), 0x4d801000 },
-	{ _MMIO(0x9888), 0x4f800111 },
-	{ _MMIO(0x9888), 0x43800842 },
-	{ _MMIO(0x9888), 0x51800000 },
-	{ _MMIO(0x9888), 0x45800000 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x47800840 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x3f800800 },
-	{ _MMIO(0x9888), 0x418014a2 },
-};
-
-static int
-get_compute_l3_cache_mux_config(struct drm_i915_private *dev_priv,
-				const struct i915_oa_reg **regs,
-				int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_l3_cache;
-	lens[n] = ARRAY_SIZE(mux_config_compute_l3_cache);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_data_port_reads_coalescing[] = {
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x274c), 0xba98ba98 },
-	{ _MMIO(0x2748), 0xba98ba98 },
-	{ _MMIO(0x2744), 0x00003377 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007fff2 },
-	{ _MMIO(0x2774), 0x00007ff0 },
-	{ _MMIO(0x2778), 0x0007ffe2 },
-	{ _MMIO(0x277c), 0x00007ff0 },
-	{ _MMIO(0x2780), 0x0007ffc2 },
-	{ _MMIO(0x2784), 0x00007ff0 },
-	{ _MMIO(0x2788), 0x0007ff82 },
-	{ _MMIO(0x278c), 0x00007ff0 },
-	{ _MMIO(0x2790), 0x0007fffa },
-	{ _MMIO(0x2794), 0x0000bfef },
-	{ _MMIO(0x2798), 0x0007fffa },
-	{ _MMIO(0x279c), 0x0000bfdf },
-	{ _MMIO(0x27a0), 0x0007fffa },
-	{ _MMIO(0x27a4), 0x0000bfbf },
-	{ _MMIO(0x27a8), 0x0007fffa },
-	{ _MMIO(0x27ac), 0x0000bf7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_data_port_reads_coalescing[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00778008 },
-	{ _MMIO(0xe45c), 0x00088078 },
-	{ _MMIO(0xe55c), 0x00808708 },
-	{ _MMIO(0xe65c), 0x00a08908 },
-};
-
-static const struct i915_oa_reg mux_config_data_port_reads_coalescing_0_subslices_0x01[] = {
-	{ _MMIO(0x9888), 0x103d0005 },
-	{ _MMIO(0x9888), 0x163d240b },
-	{ _MMIO(0x9888), 0x1058022f },
-	{ _MMIO(0x9888), 0x185b5520 },
-	{ _MMIO(0x9888), 0x198b0003 },
-	{ _MMIO(0x9888), 0x005cc000 },
-	{ _MMIO(0x9888), 0x065cc000 },
-	{ _MMIO(0x9888), 0x085cc000 },
-	{ _MMIO(0x9888), 0x0a5cc000 },
-	{ _MMIO(0x9888), 0x0c5cc000 },
-	{ _MMIO(0x9888), 0x0e5cc000 },
-	{ _MMIO(0x9888), 0x025c4000 },
-	{ _MMIO(0x9888), 0x045c8000 },
-	{ _MMIO(0x9888), 0x003d0000 },
-	{ _MMIO(0x9888), 0x063d00b0 },
-	{ _MMIO(0x9888), 0x083d0182 },
-	{ _MMIO(0x9888), 0x0a3d10a0 },
-	{ _MMIO(0x9888), 0x0c3d11a2 },
-	{ _MMIO(0x9888), 0x0e3d0000 },
-	{ _MMIO(0x9888), 0x183d0000 },
-	{ _MMIO(0x9888), 0x1a3d0000 },
-	{ _MMIO(0x9888), 0x0e582242 },
-	{ _MMIO(0x9888), 0x00586700 },
-	{ _MMIO(0x9888), 0x0258004f },
-	{ _MMIO(0x9888), 0x0658c000 },
-	{ _MMIO(0x9888), 0x0858c000 },
-	{ _MMIO(0x9888), 0x0a58c000 },
-	{ _MMIO(0x9888), 0x0c58c000 },
-	{ _MMIO(0x9888), 0x045b6300 },
-	{ _MMIO(0x9888), 0x105b0000 },
-	{ _MMIO(0x9888), 0x005b4000 },
-	{ _MMIO(0x9888), 0x0e5b4000 },
-	{ _MMIO(0x9888), 0x1a5b0155 },
-	{ _MMIO(0x9888), 0x025b4000 },
-	{ _MMIO(0x9888), 0x0a5b0000 },
-	{ _MMIO(0x9888), 0x0c5b4000 },
-	{ _MMIO(0x9888), 0x0c1fa800 },
-	{ _MMIO(0x9888), 0x0e1faaa0 },
-	{ _MMIO(0x9888), 0x101f02aa },
-	{ _MMIO(0x9888), 0x00384000 },
-	{ _MMIO(0x9888), 0x0e384000 },
-	{ _MMIO(0x9888), 0x16384000 },
-	{ _MMIO(0x9888), 0x18381555 },
-	{ _MMIO(0x9888), 0x02384000 },
-	{ _MMIO(0x9888), 0x04384000 },
-	{ _MMIO(0x9888), 0x0a384000 },
-	{ _MMIO(0x9888), 0x0c384000 },
-	{ _MMIO(0x9888), 0x0039a000 },
-	{ _MMIO(0x9888), 0x0639a000 },
-	{ _MMIO(0x9888), 0x0839a000 },
-	{ _MMIO(0x9888), 0x0a39a000 },
-	{ _MMIO(0x9888), 0x0c39a000 },
-	{ _MMIO(0x9888), 0x0e39a000 },
-	{ _MMIO(0x9888), 0x02392000 },
-	{ _MMIO(0x9888), 0x04398000 },
-	{ _MMIO(0x9888), 0x018a8000 },
-	{ _MMIO(0x9888), 0x0f8a8000 },
-	{ _MMIO(0x9888), 0x198a8000 },
-	{ _MMIO(0x9888), 0x1b8aaaa0 },
-	{ _MMIO(0x9888), 0x1d8a0002 },
-	{ _MMIO(0x9888), 0x038a8000 },
-	{ _MMIO(0x9888), 0x058a8000 },
-	{ _MMIO(0x9888), 0x0b8a8000 },
-	{ _MMIO(0x9888), 0x0d8a8000 },
-	{ _MMIO(0x9888), 0x038b6300 },
-	{ _MMIO(0x9888), 0x058b0062 },
-	{ _MMIO(0x9888), 0x118b0000 },
-	{ _MMIO(0x9888), 0x238b02a0 },
-	{ _MMIO(0x9888), 0x258b5555 },
-	{ _MMIO(0x9888), 0x278b0015 },
-	{ _MMIO(0x9888), 0x1f85aa80 },
-	{ _MMIO(0x9888), 0x2185aaaa },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x01834000 },
-	{ _MMIO(0x9888), 0x0f834000 },
-	{ _MMIO(0x9888), 0x19835400 },
-	{ _MMIO(0x9888), 0x1b830155 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x0184c000 },
-	{ _MMIO(0x9888), 0x0784c000 },
-	{ _MMIO(0x9888), 0x0984c000 },
-	{ _MMIO(0x9888), 0x0b84c000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x1180c000 },
-	{ _MMIO(0x9888), 0x1780c000 },
-	{ _MMIO(0x9888), 0x1980c000 },
-	{ _MMIO(0x9888), 0x1b80c000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0xd24), 0x00000000 },
-	{ _MMIO(0x9888), 0x4d801000 },
-	{ _MMIO(0x9888), 0x3d800000 },
-	{ _MMIO(0x9888), 0x4f800001 },
-	{ _MMIO(0x9888), 0x43800000 },
-	{ _MMIO(0x9888), 0x51800000 },
-	{ _MMIO(0x9888), 0x45800000 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x47800420 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x3f800421 },
-	{ _MMIO(0x9888), 0x41800041 },
-};
-
-static int
-get_data_port_reads_coalescing_mux_config(struct drm_i915_private *dev_priv,
-					  const struct i915_oa_reg **regs,
-					  int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	if (INTEL_INFO(dev_priv)->sseu.subslice_mask & 0x01) {
-		regs[n] = mux_config_data_port_reads_coalescing_0_subslices_0x01;
-		lens[n] = ARRAY_SIZE(mux_config_data_port_reads_coalescing_0_subslices_0x01);
-		n++;
-	}
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_data_port_writes_coalescing[] = {
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x274c), 0xba98ba98 },
-	{ _MMIO(0x2748), 0xba98ba98 },
-	{ _MMIO(0x2744), 0x00003377 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007ff72 },
-	{ _MMIO(0x2774), 0x0000bfd0 },
-	{ _MMIO(0x2778), 0x0007ff62 },
-	{ _MMIO(0x277c), 0x0000bfd0 },
-	{ _MMIO(0x2780), 0x0007ff42 },
-	{ _MMIO(0x2784), 0x0000bfd0 },
-	{ _MMIO(0x2788), 0x0007ff02 },
-	{ _MMIO(0x278c), 0x0000bfd0 },
-	{ _MMIO(0x2790), 0x0005fff2 },
-	{ _MMIO(0x2794), 0x0000bfd0 },
-	{ _MMIO(0x2798), 0x0005ffe2 },
-	{ _MMIO(0x279c), 0x0000bfd0 },
-	{ _MMIO(0x27a0), 0x0005ffc2 },
-	{ _MMIO(0x27a4), 0x0000bfd0 },
-	{ _MMIO(0x27a8), 0x0005ff82 },
-	{ _MMIO(0x27ac), 0x0000bfd0 },
-};
-
-static const struct i915_oa_reg flex_eu_config_data_port_writes_coalescing[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00778008 },
-	{ _MMIO(0xe45c), 0x00088078 },
-	{ _MMIO(0xe55c), 0x00808708 },
-	{ _MMIO(0xe65c), 0x00a08908 },
-};
-
-static const struct i915_oa_reg mux_config_data_port_writes_coalescing_0_subslices_0x01[] = {
-	{ _MMIO(0x9888), 0x103d0005 },
-	{ _MMIO(0x9888), 0x143d0120 },
-	{ _MMIO(0x9888), 0x163d2400 },
-	{ _MMIO(0x9888), 0x1058022f },
-	{ _MMIO(0x9888), 0x105b0000 },
-	{ _MMIO(0x9888), 0x198b0003 },
-	{ _MMIO(0x9888), 0x005cc000 },
-	{ _MMIO(0x9888), 0x065cc000 },
-	{ _MMIO(0x9888), 0x085cc000 },
-	{ _MMIO(0x9888), 0x0a5cc000 },
-	{ _MMIO(0x9888), 0x0e5cc000 },
-	{ _MMIO(0x9888), 0x025c4000 },
-	{ _MMIO(0x9888), 0x045c8000 },
-	{ _MMIO(0x9888), 0x003d0000 },
-	{ _MMIO(0x9888), 0x063d0094 },
-	{ _MMIO(0x9888), 0x083d0182 },
-	{ _MMIO(0x9888), 0x0a3d1814 },
-	{ _MMIO(0x9888), 0x0e3d0000 },
-	{ _MMIO(0x9888), 0x183d0000 },
-	{ _MMIO(0x9888), 0x1a3d0000 },
-	{ _MMIO(0x9888), 0x0c3d0000 },
-	{ _MMIO(0x9888), 0x0e582242 },
-	{ _MMIO(0x9888), 0x00586700 },
-	{ _MMIO(0x9888), 0x0258004f },
-	{ _MMIO(0x9888), 0x0658c000 },
-	{ _MMIO(0x9888), 0x0858c000 },
-	{ _MMIO(0x9888), 0x0a58c000 },
-	{ _MMIO(0x9888), 0x045b6a80 },
-	{ _MMIO(0x9888), 0x005b4000 },
-	{ _MMIO(0x9888), 0x0e5b4000 },
-	{ _MMIO(0x9888), 0x185b5400 },
-	{ _MMIO(0x9888), 0x1a5b0141 },
-	{ _MMIO(0x9888), 0x025b4000 },
-	{ _MMIO(0x9888), 0x0a5b0000 },
-	{ _MMIO(0x9888), 0x0c5b4000 },
-	{ _MMIO(0x9888), 0x0c1fa800 },
-	{ _MMIO(0x9888), 0x0e1faaa0 },
-	{ _MMIO(0x9888), 0x101f0282 },
-	{ _MMIO(0x9888), 0x00384000 },
-	{ _MMIO(0x9888), 0x0e384000 },
-	{ _MMIO(0x9888), 0x16384000 },
-	{ _MMIO(0x9888), 0x18381415 },
-	{ _MMIO(0x9888), 0x02384000 },
-	{ _MMIO(0x9888), 0x04384000 },
-	{ _MMIO(0x9888), 0x0a384000 },
-	{ _MMIO(0x9888), 0x0c384000 },
-	{ _MMIO(0x9888), 0x0039a000 },
-	{ _MMIO(0x9888), 0x0639a000 },
-	{ _MMIO(0x9888), 0x0839a000 },
-	{ _MMIO(0x9888), 0x0a39a000 },
-	{ _MMIO(0x9888), 0x0e39a000 },
-	{ _MMIO(0x9888), 0x02392000 },
-	{ _MMIO(0x9888), 0x04398000 },
-	{ _MMIO(0x9888), 0x018a8000 },
-	{ _MMIO(0x9888), 0x0f8a8000 },
-	{ _MMIO(0x9888), 0x198a8000 },
-	{ _MMIO(0x9888), 0x1b8a82a0 },
-	{ _MMIO(0x9888), 0x1d8a0002 },
-	{ _MMIO(0x9888), 0x038a8000 },
-	{ _MMIO(0x9888), 0x058a8000 },
-	{ _MMIO(0x9888), 0x0b8a8000 },
-	{ _MMIO(0x9888), 0x0d8a8000 },
-	{ _MMIO(0x9888), 0x038b6300 },
-	{ _MMIO(0x9888), 0x058b0062 },
-	{ _MMIO(0x9888), 0x118b0000 },
-	{ _MMIO(0x9888), 0x238b02a0 },
-	{ _MMIO(0x9888), 0x258b1555 },
-	{ _MMIO(0x9888), 0x278b0014 },
-	{ _MMIO(0x9888), 0x1f85aa80 },
-	{ _MMIO(0x9888), 0x21852aaa },
-	{ _MMIO(0x9888), 0x23850028 },
-	{ _MMIO(0x9888), 0x01834000 },
-	{ _MMIO(0x9888), 0x0f834000 },
-	{ _MMIO(0x9888), 0x19835400 },
-	{ _MMIO(0x9888), 0x1b830141 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x0184c000 },
-	{ _MMIO(0x9888), 0x0784c000 },
-	{ _MMIO(0x9888), 0x0984c000 },
-	{ _MMIO(0x9888), 0x0b84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x1180c000 },
-	{ _MMIO(0x9888), 0x1780c000 },
-	{ _MMIO(0x9888), 0x1980c000 },
-	{ _MMIO(0x9888), 0x1b80c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0xd24), 0x00000000 },
-	{ _MMIO(0x9888), 0x4d801000 },
-	{ _MMIO(0x9888), 0x3d800000 },
-	{ _MMIO(0x9888), 0x4f800001 },
-	{ _MMIO(0x9888), 0x43800000 },
-	{ _MMIO(0x9888), 0x51800000 },
-	{ _MMIO(0x9888), 0x45800000 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x47800420 },
-	{ _MMIO(0x9888), 0x3f800421 },
-	{ _MMIO(0x9888), 0x41800041 },
-};
-
-static int
-get_data_port_writes_coalescing_mux_config(struct drm_i915_private *dev_priv,
-					   const struct i915_oa_reg **regs,
-					   int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	if (INTEL_INFO(dev_priv)->sseu.subslice_mask & 0x01) {
-		regs[n] = mux_config_data_port_writes_coalescing_0_subslices_0x01;
-		lens[n] = ARRAY_SIZE(mux_config_data_port_writes_coalescing_0_subslices_0x01);
-		n++;
-	}
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_hdc_and_sf[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x10800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x0000fff7 },
-};
-
-static const struct i915_oa_reg flex_eu_config_hdc_and_sf[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_hdc_and_sf[] = {
-	{ _MMIO(0x9888), 0x105c0232 },
-	{ _MMIO(0x9888), 0x10580232 },
-	{ _MMIO(0x9888), 0x10380232 },
-	{ _MMIO(0x9888), 0x10dc0232 },
-	{ _MMIO(0x9888), 0x10d80232 },
-	{ _MMIO(0x9888), 0x10b80232 },
-	{ _MMIO(0x9888), 0x118e4400 },
-	{ _MMIO(0x9888), 0x025c6080 },
-	{ _MMIO(0x9888), 0x045c004b },
-	{ _MMIO(0x9888), 0x005c8000 },
-	{ _MMIO(0x9888), 0x00582080 },
-	{ _MMIO(0x9888), 0x0258004b },
-	{ _MMIO(0x9888), 0x025b4000 },
-	{ _MMIO(0x9888), 0x045b4000 },
-	{ _MMIO(0x9888), 0x0c1fa000 },
-	{ _MMIO(0x9888), 0x0e1f00aa },
-	{ _MMIO(0x9888), 0x04386080 },
-	{ _MMIO(0x9888), 0x0638404b },
-	{ _MMIO(0x9888), 0x02384000 },
-	{ _MMIO(0x9888), 0x08384000 },
-	{ _MMIO(0x9888), 0x0a380000 },
-	{ _MMIO(0x9888), 0x0c380000 },
-	{ _MMIO(0x9888), 0x00398000 },
-	{ _MMIO(0x9888), 0x0239a000 },
-	{ _MMIO(0x9888), 0x0439a000 },
-	{ _MMIO(0x9888), 0x06392000 },
-	{ _MMIO(0x9888), 0x0cdc25c1 },
-	{ _MMIO(0x9888), 0x0adcc000 },
-	{ _MMIO(0x9888), 0x0ad825c1 },
-	{ _MMIO(0x9888), 0x18db4000 },
-	{ _MMIO(0x9888), 0x1adb0001 },
-	{ _MMIO(0x9888), 0x0e9f8000 },
-	{ _MMIO(0x9888), 0x109f02aa },
-	{ _MMIO(0x9888), 0x0eb825c1 },
-	{ _MMIO(0x9888), 0x18b80154 },
-	{ _MMIO(0x9888), 0x0ab9a000 },
-	{ _MMIO(0x9888), 0x0cb9a000 },
-	{ _MMIO(0x9888), 0x0eb9a000 },
-	{ _MMIO(0x9888), 0x0d88c000 },
-	{ _MMIO(0x9888), 0x0f88000f },
-	{ _MMIO(0x9888), 0x038a8000 },
-	{ _MMIO(0x9888), 0x058a8000 },
-	{ _MMIO(0x9888), 0x078a8000 },
-	{ _MMIO(0x9888), 0x098a8000 },
-	{ _MMIO(0x9888), 0x0b8a8000 },
-	{ _MMIO(0x9888), 0x0d8a8000 },
-	{ _MMIO(0x9888), 0x258baa05 },
-	{ _MMIO(0x9888), 0x278b002a },
-	{ _MMIO(0x9888), 0x238b2a80 },
-	{ _MMIO(0x9888), 0x198c5400 },
-	{ _MMIO(0x9888), 0x1b8c0015 },
-	{ _MMIO(0x9888), 0x098dc000 },
-	{ _MMIO(0x9888), 0x0b8da000 },
-	{ _MMIO(0x9888), 0x0d8da000 },
-	{ _MMIO(0x9888), 0x0f8da000 },
-	{ _MMIO(0x9888), 0x098e05c0 },
-	{ _MMIO(0x9888), 0x058e0000 },
-	{ _MMIO(0x9888), 0x198f0020 },
-	{ _MMIO(0x9888), 0x2185aa0a },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x1f85aa00 },
-	{ _MMIO(0x9888), 0x19835000 },
-	{ _MMIO(0x9888), 0x1b830155 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x09848000 },
-	{ _MMIO(0x9888), 0x0b84c000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x01848000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x07844000 },
-	{ _MMIO(0x9888), 0x19808000 },
-	{ _MMIO(0x9888), 0x1b80c000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x11808000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0x9888), 0x17804000 },
-	{ _MMIO(0x9888), 0x51800040 },
-	{ _MMIO(0x9888), 0x43800400 },
-	{ _MMIO(0x9888), 0x45800800 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x47800c62 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3f801042 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x418014a4 },
-};
-
-static int
-get_hdc_and_sf_mux_config(struct drm_i915_private *dev_priv,
-			  const struct i915_oa_reg **regs,
-			  int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_hdc_and_sf;
-	lens[n] = ARRAY_SIZE(mux_config_hdc_and_sf);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_l3_1[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2770), 0x00100070 },
-	{ _MMIO(0x2774), 0x0000fff1 },
-	{ _MMIO(0x2778), 0x00014002 },
-	{ _MMIO(0x277c), 0x0000c3ff },
-	{ _MMIO(0x2780), 0x00010002 },
-	{ _MMIO(0x2784), 0x0000c7ff },
-	{ _MMIO(0x2788), 0x00004002 },
-	{ _MMIO(0x278c), 0x0000d3ff },
-	{ _MMIO(0x2790), 0x00100700 },
-	{ _MMIO(0x2794), 0x0000ff1f },
-	{ _MMIO(0x2798), 0x00001402 },
-	{ _MMIO(0x279c), 0x0000fc3f },
-	{ _MMIO(0x27a0), 0x00001002 },
-	{ _MMIO(0x27a4), 0x0000fc7f },
-	{ _MMIO(0x27a8), 0x00000402 },
-	{ _MMIO(0x27ac), 0x0000fd3f },
-};
-
-static const struct i915_oa_reg flex_eu_config_l3_1[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_l3_1[] = {
-	{ _MMIO(0x9888), 0x10bf03da },
-	{ _MMIO(0x9888), 0x14bf0001 },
-	{ _MMIO(0x9888), 0x12980340 },
-	{ _MMIO(0x9888), 0x12990340 },
-	{ _MMIO(0x9888), 0x0cbf1187 },
-	{ _MMIO(0x9888), 0x0ebf1205 },
-	{ _MMIO(0x9888), 0x00bf0500 },
-	{ _MMIO(0x9888), 0x02bf042b },
-	{ _MMIO(0x9888), 0x04bf002c },
-	{ _MMIO(0x9888), 0x0cdac000 },
-	{ _MMIO(0x9888), 0x0edac000 },
-	{ _MMIO(0x9888), 0x00da8000 },
-	{ _MMIO(0x9888), 0x02dac000 },
-	{ _MMIO(0x9888), 0x04da4000 },
-	{ _MMIO(0x9888), 0x04983400 },
-	{ _MMIO(0x9888), 0x10980000 },
-	{ _MMIO(0x9888), 0x06990034 },
-	{ _MMIO(0x9888), 0x10990000 },
-	{ _MMIO(0x9888), 0x0c9dc000 },
-	{ _MMIO(0x9888), 0x0e9dc000 },
-	{ _MMIO(0x9888), 0x009d8000 },
-	{ _MMIO(0x9888), 0x029dc000 },
-	{ _MMIO(0x9888), 0x049d4000 },
-	{ _MMIO(0x9888), 0x109f02a8 },
-	{ _MMIO(0x9888), 0x0c9fa000 },
-	{ _MMIO(0x9888), 0x0e9f00ba },
-	{ _MMIO(0x9888), 0x0cb88000 },
-	{ _MMIO(0x9888), 0x0cb95000 },
-	{ _MMIO(0x9888), 0x0eb95000 },
-	{ _MMIO(0x9888), 0x00b94000 },
-	{ _MMIO(0x9888), 0x02b95000 },
-	{ _MMIO(0x9888), 0x04b91000 },
-	{ _MMIO(0x9888), 0x06b92000 },
-	{ _MMIO(0x9888), 0x0cba4000 },
-	{ _MMIO(0x9888), 0x0f88000f },
-	{ _MMIO(0x9888), 0x03888000 },
-	{ _MMIO(0x9888), 0x05888000 },
-	{ _MMIO(0x9888), 0x07888000 },
-	{ _MMIO(0x9888), 0x09888000 },
-	{ _MMIO(0x9888), 0x0b888000 },
-	{ _MMIO(0x9888), 0x0d880400 },
-	{ _MMIO(0x9888), 0x258b800a },
-	{ _MMIO(0x9888), 0x278b002a },
-	{ _MMIO(0x9888), 0x238b5500 },
-	{ _MMIO(0x9888), 0x198c4000 },
-	{ _MMIO(0x9888), 0x1b8c0015 },
-	{ _MMIO(0x9888), 0x038c4000 },
-	{ _MMIO(0x9888), 0x058c4000 },
-	{ _MMIO(0x9888), 0x078c4000 },
-	{ _MMIO(0x9888), 0x098c4000 },
-	{ _MMIO(0x9888), 0x0b8c4000 },
-	{ _MMIO(0x9888), 0x0d8c4000 },
-	{ _MMIO(0x9888), 0x0d8da000 },
-	{ _MMIO(0x9888), 0x0f8da000 },
-	{ _MMIO(0x9888), 0x018d8000 },
-	{ _MMIO(0x9888), 0x038da000 },
-	{ _MMIO(0x9888), 0x058da000 },
-	{ _MMIO(0x9888), 0x078d2000 },
-	{ _MMIO(0x9888), 0x2185800a },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x1f85aa00 },
-	{ _MMIO(0x9888), 0x1b830154 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x01848000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x07844000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x11808000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0x9888), 0x17804000 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x45800000 },
-	{ _MMIO(0x9888), 0x47800000 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3f800000 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x41800060 },
-};
-
-static int
-get_l3_1_mux_config(struct drm_i915_private *dev_priv,
-		    const struct i915_oa_reg **regs,
-		    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_l3_1;
-	lens[n] = ARRAY_SIZE(mux_config_l3_1);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_l3_2[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2770), 0x00100070 },
-	{ _MMIO(0x2774), 0x0000fff1 },
-	{ _MMIO(0x2778), 0x00014002 },
-	{ _MMIO(0x277c), 0x0000c3ff },
-	{ _MMIO(0x2780), 0x00010002 },
-	{ _MMIO(0x2784), 0x0000c7ff },
-	{ _MMIO(0x2788), 0x00004002 },
-	{ _MMIO(0x278c), 0x0000d3ff },
-	{ _MMIO(0x2790), 0x00100700 },
-	{ _MMIO(0x2794), 0x0000ff1f },
-	{ _MMIO(0x2798), 0x00001402 },
-	{ _MMIO(0x279c), 0x0000fc3f },
-	{ _MMIO(0x27a0), 0x00001002 },
-	{ _MMIO(0x27a4), 0x0000fc7f },
-	{ _MMIO(0x27a8), 0x00000402 },
-	{ _MMIO(0x27ac), 0x0000fd3f },
-};
-
-static const struct i915_oa_reg flex_eu_config_l3_2[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_l3_2[] = {
-	{ _MMIO(0x9888), 0x103f03da },
-	{ _MMIO(0x9888), 0x143f0001 },
-	{ _MMIO(0x9888), 0x12180340 },
-	{ _MMIO(0x9888), 0x12190340 },
-	{ _MMIO(0x9888), 0x0c3f1187 },
-	{ _MMIO(0x9888), 0x0e3f1205 },
-	{ _MMIO(0x9888), 0x003f0500 },
-	{ _MMIO(0x9888), 0x023f042b },
-	{ _MMIO(0x9888), 0x043f002c },
-	{ _MMIO(0x9888), 0x0c5ac000 },
-	{ _MMIO(0x9888), 0x0e5ac000 },
-	{ _MMIO(0x9888), 0x005a8000 },
-	{ _MMIO(0x9888), 0x025ac000 },
-	{ _MMIO(0x9888), 0x045a4000 },
-	{ _MMIO(0x9888), 0x04183400 },
-	{ _MMIO(0x9888), 0x10180000 },
-	{ _MMIO(0x9888), 0x06190034 },
-	{ _MMIO(0x9888), 0x10190000 },
-	{ _MMIO(0x9888), 0x0c1dc000 },
-	{ _MMIO(0x9888), 0x0e1dc000 },
-	{ _MMIO(0x9888), 0x001d8000 },
-	{ _MMIO(0x9888), 0x021dc000 },
-	{ _MMIO(0x9888), 0x041d4000 },
-	{ _MMIO(0x9888), 0x101f02a8 },
-	{ _MMIO(0x9888), 0x0c1fa000 },
-	{ _MMIO(0x9888), 0x0e1f00ba },
-	{ _MMIO(0x9888), 0x0c388000 },
-	{ _MMIO(0x9888), 0x0c395000 },
-	{ _MMIO(0x9888), 0x0e395000 },
-	{ _MMIO(0x9888), 0x00394000 },
-	{ _MMIO(0x9888), 0x02395000 },
-	{ _MMIO(0x9888), 0x04391000 },
-	{ _MMIO(0x9888), 0x06392000 },
-	{ _MMIO(0x9888), 0x0c3a4000 },
-	{ _MMIO(0x9888), 0x1b8aa800 },
-	{ _MMIO(0x9888), 0x1d8a0002 },
-	{ _MMIO(0x9888), 0x038a8000 },
-	{ _MMIO(0x9888), 0x058a8000 },
-	{ _MMIO(0x9888), 0x078a8000 },
-	{ _MMIO(0x9888), 0x098a8000 },
-	{ _MMIO(0x9888), 0x0b8a8000 },
-	{ _MMIO(0x9888), 0x0d8a8000 },
-	{ _MMIO(0x9888), 0x258b4005 },
-	{ _MMIO(0x9888), 0x278b0015 },
-	{ _MMIO(0x9888), 0x238b2a80 },
-	{ _MMIO(0x9888), 0x2185800a },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x1f85aa00 },
-	{ _MMIO(0x9888), 0x1b830154 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x01848000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x07844000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x11808000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0x9888), 0x17804000 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x45800000 },
-	{ _MMIO(0x9888), 0x47800000 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3f800000 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x41800060 },
-};
-
-static int
-get_l3_2_mux_config(struct drm_i915_private *dev_priv,
-		    const struct i915_oa_reg **regs,
-		    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_l3_2;
-	lens[n] = ARRAY_SIZE(mux_config_l3_2);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_l3_3[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2770), 0x00100070 },
-	{ _MMIO(0x2774), 0x0000fff1 },
-	{ _MMIO(0x2778), 0x00014002 },
-	{ _MMIO(0x277c), 0x0000c3ff },
-	{ _MMIO(0x2780), 0x00010002 },
-	{ _MMIO(0x2784), 0x0000c7ff },
-	{ _MMIO(0x2788), 0x00004002 },
-	{ _MMIO(0x278c), 0x0000d3ff },
-	{ _MMIO(0x2790), 0x00100700 },
-	{ _MMIO(0x2794), 0x0000ff1f },
-	{ _MMIO(0x2798), 0x00001402 },
-	{ _MMIO(0x279c), 0x0000fc3f },
-	{ _MMIO(0x27a0), 0x00001002 },
-	{ _MMIO(0x27a4), 0x0000fc7f },
-	{ _MMIO(0x27a8), 0x00000402 },
-	{ _MMIO(0x27ac), 0x0000fd3f },
-};
-
-static const struct i915_oa_reg flex_eu_config_l3_3[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_l3_3[] = {
-	{ _MMIO(0x9888), 0x121b0340 },
-	{ _MMIO(0x9888), 0x103f0274 },
-	{ _MMIO(0x9888), 0x123f0000 },
-	{ _MMIO(0x9888), 0x129b0340 },
-	{ _MMIO(0x9888), 0x10bf0274 },
-	{ _MMIO(0x9888), 0x12bf0000 },
-	{ _MMIO(0x9888), 0x041b3400 },
-	{ _MMIO(0x9888), 0x101b0000 },
-	{ _MMIO(0x9888), 0x045c8000 },
-	{ _MMIO(0x9888), 0x0a3d4000 },
-	{ _MMIO(0x9888), 0x003f0080 },
-	{ _MMIO(0x9888), 0x023f0793 },
-	{ _MMIO(0x9888), 0x043f0014 },
-	{ _MMIO(0x9888), 0x04588000 },
-	{ _MMIO(0x9888), 0x005a8000 },
-	{ _MMIO(0x9888), 0x025ac000 },
-	{ _MMIO(0x9888), 0x045a4000 },
-	{ _MMIO(0x9888), 0x0a5b4000 },
-	{ _MMIO(0x9888), 0x001d8000 },
-	{ _MMIO(0x9888), 0x021dc000 },
-	{ _MMIO(0x9888), 0x041d4000 },
-	{ _MMIO(0x9888), 0x0c1fa000 },
-	{ _MMIO(0x9888), 0x0e1f002a },
-	{ _MMIO(0x9888), 0x0a384000 },
-	{ _MMIO(0x9888), 0x00394000 },
-	{ _MMIO(0x9888), 0x02395000 },
-	{ _MMIO(0x9888), 0x04399000 },
-	{ _MMIO(0x9888), 0x069b0034 },
-	{ _MMIO(0x9888), 0x109b0000 },
-	{ _MMIO(0x9888), 0x06dc4000 },
-	{ _MMIO(0x9888), 0x0cbd4000 },
-	{ _MMIO(0x9888), 0x0cbf0981 },
-	{ _MMIO(0x9888), 0x0ebf0a0f },
-	{ _MMIO(0x9888), 0x06d84000 },
-	{ _MMIO(0x9888), 0x0cdac000 },
-	{ _MMIO(0x9888), 0x0edac000 },
-	{ _MMIO(0x9888), 0x0cdb4000 },
-	{ _MMIO(0x9888), 0x0c9dc000 },
-	{ _MMIO(0x9888), 0x0e9dc000 },
-	{ _MMIO(0x9888), 0x109f02a8 },
-	{ _MMIO(0x9888), 0x0e9f0080 },
-	{ _MMIO(0x9888), 0x0cb84000 },
-	{ _MMIO(0x9888), 0x0cb95000 },
-	{ _MMIO(0x9888), 0x0eb95000 },
-	{ _MMIO(0x9888), 0x06b92000 },
-	{ _MMIO(0x9888), 0x0f88000f },
-	{ _MMIO(0x9888), 0x0d880400 },
-	{ _MMIO(0x9888), 0x038a8000 },
-	{ _MMIO(0x9888), 0x058a8000 },
-	{ _MMIO(0x9888), 0x078a8000 },
-	{ _MMIO(0x9888), 0x098a8000 },
-	{ _MMIO(0x9888), 0x0b8a8000 },
-	{ _MMIO(0x9888), 0x258b8009 },
-	{ _MMIO(0x9888), 0x278b002a },
-	{ _MMIO(0x9888), 0x238b2a80 },
-	{ _MMIO(0x9888), 0x198c4000 },
-	{ _MMIO(0x9888), 0x1b8c0015 },
-	{ _MMIO(0x9888), 0x0d8c4000 },
-	{ _MMIO(0x9888), 0x0d8da000 },
-	{ _MMIO(0x9888), 0x0f8da000 },
-	{ _MMIO(0x9888), 0x078d2000 },
-	{ _MMIO(0x9888), 0x2185800a },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x1f85aa00 },
-	{ _MMIO(0x9888), 0x1b830154 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x01848000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x07844000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x11808000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0x9888), 0x17804000 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x45800c00 },
-	{ _MMIO(0x9888), 0x47800c63 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3f8014a5 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x41800045 },
-};
-
-static int
-get_l3_3_mux_config(struct drm_i915_private *dev_priv,
-		    const struct i915_oa_reg **regs,
-		    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_l3_3;
-	lens[n] = ARRAY_SIZE(mux_config_l3_3);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_l3_4[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2770), 0x00100070 },
-	{ _MMIO(0x2774), 0x0000fff1 },
-	{ _MMIO(0x2778), 0x00014002 },
-	{ _MMIO(0x277c), 0x0000c3ff },
-	{ _MMIO(0x2780), 0x00010002 },
-	{ _MMIO(0x2784), 0x0000c7ff },
-	{ _MMIO(0x2788), 0x00004002 },
-	{ _MMIO(0x278c), 0x0000d3ff },
-	{ _MMIO(0x2790), 0x00100700 },
-	{ _MMIO(0x2794), 0x0000ff1f },
-	{ _MMIO(0x2798), 0x00001402 },
-	{ _MMIO(0x279c), 0x0000fc3f },
-	{ _MMIO(0x27a0), 0x00001002 },
-	{ _MMIO(0x27a4), 0x0000fc7f },
-	{ _MMIO(0x27a8), 0x00000402 },
-	{ _MMIO(0x27ac), 0x0000fd3f },
-};
-
-static const struct i915_oa_reg flex_eu_config_l3_4[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_l3_4[] = {
-	{ _MMIO(0x9888), 0x121a0340 },
-	{ _MMIO(0x9888), 0x103f0017 },
-	{ _MMIO(0x9888), 0x123f0020 },
-	{ _MMIO(0x9888), 0x129a0340 },
-	{ _MMIO(0x9888), 0x10bf0017 },
-	{ _MMIO(0x9888), 0x12bf0020 },
-	{ _MMIO(0x9888), 0x041a3400 },
-	{ _MMIO(0x9888), 0x101a0000 },
-	{ _MMIO(0x9888), 0x043b8000 },
-	{ _MMIO(0x9888), 0x0a3e0010 },
-	{ _MMIO(0x9888), 0x003f0200 },
-	{ _MMIO(0x9888), 0x023f0113 },
-	{ _MMIO(0x9888), 0x043f0014 },
-	{ _MMIO(0x9888), 0x02592000 },
-	{ _MMIO(0x9888), 0x005a8000 },
-	{ _MMIO(0x9888), 0x025ac000 },
-	{ _MMIO(0x9888), 0x045a4000 },
-	{ _MMIO(0x9888), 0x0a1c8000 },
-	{ _MMIO(0x9888), 0x001d8000 },
-	{ _MMIO(0x9888), 0x021dc000 },
-	{ _MMIO(0x9888), 0x041d4000 },
-	{ _MMIO(0x9888), 0x0a1e8000 },
-	{ _MMIO(0x9888), 0x0c1fa000 },
-	{ _MMIO(0x9888), 0x0e1f001a },
-	{ _MMIO(0x9888), 0x00394000 },
-	{ _MMIO(0x9888), 0x02395000 },
-	{ _MMIO(0x9888), 0x04391000 },
-	{ _MMIO(0x9888), 0x069a0034 },
-	{ _MMIO(0x9888), 0x109a0000 },
-	{ _MMIO(0x9888), 0x06bb4000 },
-	{ _MMIO(0x9888), 0x0abe0040 },
-	{ _MMIO(0x9888), 0x0cbf0984 },
-	{ _MMIO(0x9888), 0x0ebf0a02 },
-	{ _MMIO(0x9888), 0x02d94000 },
-	{ _MMIO(0x9888), 0x0cdac000 },
-	{ _MMIO(0x9888), 0x0edac000 },
-	{ _MMIO(0x9888), 0x0c9c0400 },
-	{ _MMIO(0x9888), 0x0c9dc000 },
-	{ _MMIO(0x9888), 0x0e9dc000 },
-	{ _MMIO(0x9888), 0x0c9e0400 },
-	{ _MMIO(0x9888), 0x109f02a8 },
-	{ _MMIO(0x9888), 0x0e9f0040 },
-	{ _MMIO(0x9888), 0x0cb95000 },
-	{ _MMIO(0x9888), 0x0eb95000 },
-	{ _MMIO(0x9888), 0x0f88000f },
-	{ _MMIO(0x9888), 0x0d880400 },
-	{ _MMIO(0x9888), 0x038a8000 },
-	{ _MMIO(0x9888), 0x058a8000 },
-	{ _MMIO(0x9888), 0x078a8000 },
-	{ _MMIO(0x9888), 0x098a8000 },
-	{ _MMIO(0x9888), 0x0b8a8000 },
-	{ _MMIO(0x9888), 0x258b8009 },
-	{ _MMIO(0x9888), 0x278b002a },
-	{ _MMIO(0x9888), 0x238b2a80 },
-	{ _MMIO(0x9888), 0x198c4000 },
-	{ _MMIO(0x9888), 0x1b8c0015 },
-	{ _MMIO(0x9888), 0x0d8c4000 },
-	{ _MMIO(0x9888), 0x0d8da000 },
-	{ _MMIO(0x9888), 0x0f8da000 },
-	{ _MMIO(0x9888), 0x078d2000 },
-	{ _MMIO(0x9888), 0x2185800a },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x1f85aa00 },
-	{ _MMIO(0x9888), 0x1b830154 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x01848000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x07844000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x11808000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0x9888), 0x17804000 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x45800800 },
-	{ _MMIO(0x9888), 0x47800842 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3f801084 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x41800044 },
-};
-
-static int
-get_l3_4_mux_config(struct drm_i915_private *dev_priv,
-		    const struct i915_oa_reg **regs,
-		    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_l3_4;
-	lens[n] = ARRAY_SIZE(mux_config_l3_4);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2770), 0x00006000 },
-	{ _MMIO(0x2774), 0x0000f3ff },
-	{ _MMIO(0x2778), 0x00001800 },
-	{ _MMIO(0x277c), 0x0000fcff },
-	{ _MMIO(0x2780), 0x00000600 },
-	{ _MMIO(0x2784), 0x0000ff3f },
-	{ _MMIO(0x2788), 0x00000180 },
-	{ _MMIO(0x278c), 0x0000ffcf },
-	{ _MMIO(0x2790), 0x00000060 },
-	{ _MMIO(0x2794), 0x0000fff3 },
-	{ _MMIO(0x2798), 0x00000018 },
-	{ _MMIO(0x279c), 0x0000fffc },
-};
-
-static const struct i915_oa_reg flex_eu_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0x9888), 0x143b000e },
-	{ _MMIO(0x9888), 0x043c55c0 },
-	{ _MMIO(0x9888), 0x0a1e0280 },
-	{ _MMIO(0x9888), 0x0c1e0408 },
-	{ _MMIO(0x9888), 0x10390000 },
-	{ _MMIO(0x9888), 0x12397a1f },
-	{ _MMIO(0x9888), 0x14bb000e },
-	{ _MMIO(0x9888), 0x04bc5000 },
-	{ _MMIO(0x9888), 0x0a9e0296 },
-	{ _MMIO(0x9888), 0x0c9e0008 },
-	{ _MMIO(0x9888), 0x10b90000 },
-	{ _MMIO(0x9888), 0x12b97a1f },
-	{ _MMIO(0x9888), 0x063b0042 },
-	{ _MMIO(0x9888), 0x103b0000 },
-	{ _MMIO(0x9888), 0x083c0000 },
-	{ _MMIO(0x9888), 0x0a3e0040 },
-	{ _MMIO(0x9888), 0x043f8000 },
-	{ _MMIO(0x9888), 0x02594000 },
-	{ _MMIO(0x9888), 0x045a8000 },
-	{ _MMIO(0x9888), 0x0c1c0400 },
-	{ _MMIO(0x9888), 0x041d8000 },
-	{ _MMIO(0x9888), 0x081e02c0 },
-	{ _MMIO(0x9888), 0x0e1e0000 },
-	{ _MMIO(0x9888), 0x0c1fa800 },
-	{ _MMIO(0x9888), 0x0e1f0260 },
-	{ _MMIO(0x9888), 0x101f0014 },
-	{ _MMIO(0x9888), 0x003905e0 },
-	{ _MMIO(0x9888), 0x06390bc0 },
-	{ _MMIO(0x9888), 0x02390018 },
-	{ _MMIO(0x9888), 0x04394000 },
-	{ _MMIO(0x9888), 0x04bb0042 },
-	{ _MMIO(0x9888), 0x10bb0000 },
-	{ _MMIO(0x9888), 0x02bc05c0 },
-	{ _MMIO(0x9888), 0x08bc0000 },
-	{ _MMIO(0x9888), 0x0abe0004 },
-	{ _MMIO(0x9888), 0x02bf8000 },
-	{ _MMIO(0x9888), 0x02d91000 },
-	{ _MMIO(0x9888), 0x02da8000 },
-	{ _MMIO(0x9888), 0x089c8000 },
-	{ _MMIO(0x9888), 0x029d8000 },
-	{ _MMIO(0x9888), 0x089e8000 },
-	{ _MMIO(0x9888), 0x0e9e0000 },
-	{ _MMIO(0x9888), 0x0e9fa806 },
-	{ _MMIO(0x9888), 0x109f0142 },
-	{ _MMIO(0x9888), 0x08b90617 },
-	{ _MMIO(0x9888), 0x0ab90be0 },
-	{ _MMIO(0x9888), 0x02b94000 },
-	{ _MMIO(0x9888), 0x0d88f000 },
-	{ _MMIO(0x9888), 0x0f88000c },
-	{ _MMIO(0x9888), 0x07888000 },
-	{ _MMIO(0x9888), 0x09888000 },
-	{ _MMIO(0x9888), 0x018a8000 },
-	{ _MMIO(0x9888), 0x0f8a8000 },
-	{ _MMIO(0x9888), 0x1b8a2800 },
-	{ _MMIO(0x9888), 0x038a8000 },
-	{ _MMIO(0x9888), 0x058a8000 },
-	{ _MMIO(0x9888), 0x0b8a8000 },
-	{ _MMIO(0x9888), 0x0d8a8000 },
-	{ _MMIO(0x9888), 0x238b52a0 },
-	{ _MMIO(0x9888), 0x258b6a95 },
-	{ _MMIO(0x9888), 0x278b0029 },
-	{ _MMIO(0x9888), 0x178c2000 },
-	{ _MMIO(0x9888), 0x198c1500 },
-	{ _MMIO(0x9888), 0x1b8c0014 },
-	{ _MMIO(0x9888), 0x078c4000 },
-	{ _MMIO(0x9888), 0x098c4000 },
-	{ _MMIO(0x9888), 0x098da000 },
-	{ _MMIO(0x9888), 0x0b8da000 },
-	{ _MMIO(0x9888), 0x0f8da000 },
-	{ _MMIO(0x9888), 0x038d8000 },
-	{ _MMIO(0x9888), 0x058d2000 },
-	{ _MMIO(0x9888), 0x1f85aa80 },
-	{ _MMIO(0x9888), 0x2185aaaa },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x01834000 },
-	{ _MMIO(0x9888), 0x0f834000 },
-	{ _MMIO(0x9888), 0x19835400 },
-	{ _MMIO(0x9888), 0x1b830155 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x0184c000 },
-	{ _MMIO(0x9888), 0x0784c000 },
-	{ _MMIO(0x9888), 0x0984c000 },
-	{ _MMIO(0x9888), 0x0b84c000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x1180c000 },
-	{ _MMIO(0x9888), 0x1780c000 },
-	{ _MMIO(0x9888), 0x1980c000 },
-	{ _MMIO(0x9888), 0x1b80c000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0x9888), 0x4d800444 },
-	{ _MMIO(0x9888), 0x3d800000 },
-	{ _MMIO(0x9888), 0x4f804000 },
-	{ _MMIO(0x9888), 0x43801080 },
-	{ _MMIO(0x9888), 0x51800000 },
-	{ _MMIO(0x9888), 0x45800084 },
-	{ _MMIO(0x9888), 0x53800044 },
-	{ _MMIO(0x9888), 0x47801080 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x3f800000 },
-	{ _MMIO(0x9888), 0x41800840 },
-};
-
-static int
-get_rasterizer_and_pixel_backend_mux_config(struct drm_i915_private *dev_priv,
-					    const struct i915_oa_reg **regs,
-					    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_rasterizer_and_pixel_backend;
-	lens[n] = ARRAY_SIZE(mux_config_rasterizer_and_pixel_backend);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_sampler_1[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x70800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x0000c000 },
-	{ _MMIO(0x2774), 0x0000e7ff },
-	{ _MMIO(0x2778), 0x00003000 },
-	{ _MMIO(0x277c), 0x0000f9ff },
-	{ _MMIO(0x2780), 0x00000c00 },
-	{ _MMIO(0x2784), 0x0000fe7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_sampler_1[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_sampler_1[] = {
-	{ _MMIO(0x9888), 0x18921400 },
-	{ _MMIO(0x9888), 0x149500ab },
-	{ _MMIO(0x9888), 0x18b21400 },
-	{ _MMIO(0x9888), 0x14b500ab },
-	{ _MMIO(0x9888), 0x18d21400 },
-	{ _MMIO(0x9888), 0x14d500ab },
-	{ _MMIO(0x9888), 0x0cdc8000 },
-	{ _MMIO(0x9888), 0x0edc4000 },
-	{ _MMIO(0x9888), 0x02dcc000 },
-	{ _MMIO(0x9888), 0x04dcc000 },
-	{ _MMIO(0x9888), 0x1abd00a0 },
-	{ _MMIO(0x9888), 0x0abd8000 },
-	{ _MMIO(0x9888), 0x0cd88000 },
-	{ _MMIO(0x9888), 0x0ed84000 },
-	{ _MMIO(0x9888), 0x04d88000 },
-	{ _MMIO(0x9888), 0x1adb0050 },
-	{ _MMIO(0x9888), 0x04db8000 },
-	{ _MMIO(0x9888), 0x06db8000 },
-	{ _MMIO(0x9888), 0x08db8000 },
-	{ _MMIO(0x9888), 0x0adb4000 },
-	{ _MMIO(0x9888), 0x109f02a0 },
-	{ _MMIO(0x9888), 0x0c9fa000 },
-	{ _MMIO(0x9888), 0x0e9f00aa },
-	{ _MMIO(0x9888), 0x18b82500 },
-	{ _MMIO(0x9888), 0x02b88000 },
-	{ _MMIO(0x9888), 0x04b84000 },
-	{ _MMIO(0x9888), 0x06b84000 },
-	{ _MMIO(0x9888), 0x08b84000 },
-	{ _MMIO(0x9888), 0x0ab84000 },
-	{ _MMIO(0x9888), 0x0cb88000 },
-	{ _MMIO(0x9888), 0x0cb98000 },
-	{ _MMIO(0x9888), 0x0eb9a000 },
-	{ _MMIO(0x9888), 0x00b98000 },
-	{ _MMIO(0x9888), 0x02b9a000 },
-	{ _MMIO(0x9888), 0x04b9a000 },
-	{ _MMIO(0x9888), 0x06b92000 },
-	{ _MMIO(0x9888), 0x1aba0200 },
-	{ _MMIO(0x9888), 0x02ba8000 },
-	{ _MMIO(0x9888), 0x0cba8000 },
-	{ _MMIO(0x9888), 0x04908000 },
-	{ _MMIO(0x9888), 0x04918000 },
-	{ _MMIO(0x9888), 0x04927300 },
-	{ _MMIO(0x9888), 0x10920000 },
-	{ _MMIO(0x9888), 0x1893000a },
-	{ _MMIO(0x9888), 0x0a934000 },
-	{ _MMIO(0x9888), 0x0a946000 },
-	{ _MMIO(0x9888), 0x0c959000 },
-	{ _MMIO(0x9888), 0x0e950098 },
-	{ _MMIO(0x9888), 0x10950000 },
-	{ _MMIO(0x9888), 0x04b04000 },
-	{ _MMIO(0x9888), 0x04b14000 },
-	{ _MMIO(0x9888), 0x04b20073 },
-	{ _MMIO(0x9888), 0x10b20000 },
-	{ _MMIO(0x9888), 0x04b38000 },
-	{ _MMIO(0x9888), 0x06b38000 },
-	{ _MMIO(0x9888), 0x08b34000 },
-	{ _MMIO(0x9888), 0x04b4c000 },
-	{ _MMIO(0x9888), 0x02b59890 },
-	{ _MMIO(0x9888), 0x10b50000 },
-	{ _MMIO(0x9888), 0x06d04000 },
-	{ _MMIO(0x9888), 0x06d14000 },
-	{ _MMIO(0x9888), 0x06d20073 },
-	{ _MMIO(0x9888), 0x10d20000 },
-	{ _MMIO(0x9888), 0x18d30020 },
-	{ _MMIO(0x9888), 0x02d38000 },
-	{ _MMIO(0x9888), 0x0cd34000 },
-	{ _MMIO(0x9888), 0x0ad48000 },
-	{ _MMIO(0x9888), 0x04d42000 },
-	{ _MMIO(0x9888), 0x0ed59000 },
-	{ _MMIO(0x9888), 0x00d59800 },
-	{ _MMIO(0x9888), 0x10d50000 },
-	{ _MMIO(0x9888), 0x0f88000e },
-	{ _MMIO(0x9888), 0x03888000 },
-	{ _MMIO(0x9888), 0x05888000 },
-	{ _MMIO(0x9888), 0x07888000 },
-	{ _MMIO(0x9888), 0x09888000 },
-	{ _MMIO(0x9888), 0x0b888000 },
-	{ _MMIO(0x9888), 0x0d880400 },
-	{ _MMIO(0x9888), 0x278b002a },
-	{ _MMIO(0x9888), 0x238b5500 },
-	{ _MMIO(0x9888), 0x258b000a },
-	{ _MMIO(0x9888), 0x1b8c0015 },
-	{ _MMIO(0x9888), 0x038c4000 },
-	{ _MMIO(0x9888), 0x058c4000 },
-	{ _MMIO(0x9888), 0x078c4000 },
-	{ _MMIO(0x9888), 0x098c4000 },
-	{ _MMIO(0x9888), 0x0b8c4000 },
-	{ _MMIO(0x9888), 0x0d8c4000 },
-	{ _MMIO(0x9888), 0x0d8d8000 },
-	{ _MMIO(0x9888), 0x0f8da000 },
-	{ _MMIO(0x9888), 0x018d8000 },
-	{ _MMIO(0x9888), 0x038da000 },
-	{ _MMIO(0x9888), 0x058da000 },
-	{ _MMIO(0x9888), 0x078d2000 },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x1f85aa00 },
-	{ _MMIO(0x9888), 0x2185000a },
-	{ _MMIO(0x9888), 0x1b830150 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x0d848000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x01848000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x07844000 },
-	{ _MMIO(0x9888), 0x1d808000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x11808000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0x9888), 0x17804000 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x47801021 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3f800c64 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x41800c02 },
-};
-
-static int
-get_sampler_1_mux_config(struct drm_i915_private *dev_priv,
-			 const struct i915_oa_reg **regs,
-			 int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_sampler_1;
-	lens[n] = ARRAY_SIZE(mux_config_sampler_1);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_sampler_2[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x70800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x0000c000 },
-	{ _MMIO(0x2774), 0x0000e7ff },
-	{ _MMIO(0x2778), 0x00003000 },
-	{ _MMIO(0x277c), 0x0000f9ff },
-	{ _MMIO(0x2780), 0x00000c00 },
-	{ _MMIO(0x2784), 0x0000fe7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_sampler_2[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_sampler_2[] = {
-	{ _MMIO(0x9888), 0x18121400 },
-	{ _MMIO(0x9888), 0x141500ab },
-	{ _MMIO(0x9888), 0x18321400 },
-	{ _MMIO(0x9888), 0x143500ab },
-	{ _MMIO(0x9888), 0x18521400 },
-	{ _MMIO(0x9888), 0x145500ab },
-	{ _MMIO(0x9888), 0x0c5c8000 },
-	{ _MMIO(0x9888), 0x0e5c4000 },
-	{ _MMIO(0x9888), 0x025cc000 },
-	{ _MMIO(0x9888), 0x045cc000 },
-	{ _MMIO(0x9888), 0x1a3d00a0 },
-	{ _MMIO(0x9888), 0x0a3d8000 },
-	{ _MMIO(0x9888), 0x0c588000 },
-	{ _MMIO(0x9888), 0x0e584000 },
-	{ _MMIO(0x9888), 0x04588000 },
-	{ _MMIO(0x9888), 0x1a5b0050 },
-	{ _MMIO(0x9888), 0x045b8000 },
-	{ _MMIO(0x9888), 0x065b8000 },
-	{ _MMIO(0x9888), 0x085b8000 },
-	{ _MMIO(0x9888), 0x0a5b4000 },
-	{ _MMIO(0x9888), 0x101f02a0 },
-	{ _MMIO(0x9888), 0x0c1fa000 },
-	{ _MMIO(0x9888), 0x0e1f00aa },
-	{ _MMIO(0x9888), 0x18382500 },
-	{ _MMIO(0x9888), 0x02388000 },
-	{ _MMIO(0x9888), 0x04384000 },
-	{ _MMIO(0x9888), 0x06384000 },
-	{ _MMIO(0x9888), 0x08384000 },
-	{ _MMIO(0x9888), 0x0a384000 },
-	{ _MMIO(0x9888), 0x0c388000 },
-	{ _MMIO(0x9888), 0x0c398000 },
-	{ _MMIO(0x9888), 0x0e39a000 },
-	{ _MMIO(0x9888), 0x00398000 },
-	{ _MMIO(0x9888), 0x0239a000 },
-	{ _MMIO(0x9888), 0x0439a000 },
-	{ _MMIO(0x9888), 0x06392000 },
-	{ _MMIO(0x9888), 0x1a3a0200 },
-	{ _MMIO(0x9888), 0x023a8000 },
-	{ _MMIO(0x9888), 0x0c3a8000 },
-	{ _MMIO(0x9888), 0x04108000 },
-	{ _MMIO(0x9888), 0x04118000 },
-	{ _MMIO(0x9888), 0x04127300 },
-	{ _MMIO(0x9888), 0x10120000 },
-	{ _MMIO(0x9888), 0x1813000a },
-	{ _MMIO(0x9888), 0x0a134000 },
-	{ _MMIO(0x9888), 0x0a146000 },
-	{ _MMIO(0x9888), 0x0c159000 },
-	{ _MMIO(0x9888), 0x0e150098 },
-	{ _MMIO(0x9888), 0x10150000 },
-	{ _MMIO(0x9888), 0x04304000 },
-	{ _MMIO(0x9888), 0x04314000 },
-	{ _MMIO(0x9888), 0x04320073 },
-	{ _MMIO(0x9888), 0x10320000 },
-	{ _MMIO(0x9888), 0x04338000 },
-	{ _MMIO(0x9888), 0x06338000 },
-	{ _MMIO(0x9888), 0x08334000 },
-	{ _MMIO(0x9888), 0x0434c000 },
-	{ _MMIO(0x9888), 0x02359890 },
-	{ _MMIO(0x9888), 0x10350000 },
-	{ _MMIO(0x9888), 0x06504000 },
-	{ _MMIO(0x9888), 0x06514000 },
-	{ _MMIO(0x9888), 0x06520073 },
-	{ _MMIO(0x9888), 0x10520000 },
-	{ _MMIO(0x9888), 0x18530020 },
-	{ _MMIO(0x9888), 0x02538000 },
-	{ _MMIO(0x9888), 0x0c534000 },
-	{ _MMIO(0x9888), 0x0a548000 },
-	{ _MMIO(0x9888), 0x04542000 },
-	{ _MMIO(0x9888), 0x0e559000 },
-	{ _MMIO(0x9888), 0x00559800 },
-	{ _MMIO(0x9888), 0x10550000 },
-	{ _MMIO(0x9888), 0x1b8aa000 },
-	{ _MMIO(0x9888), 0x1d8a0002 },
-	{ _MMIO(0x9888), 0x038a8000 },
-	{ _MMIO(0x9888), 0x058a8000 },
-	{ _MMIO(0x9888), 0x078a8000 },
-	{ _MMIO(0x9888), 0x098a8000 },
-	{ _MMIO(0x9888), 0x0b8a8000 },
-	{ _MMIO(0x9888), 0x0d8a8000 },
-	{ _MMIO(0x9888), 0x278b0015 },
-	{ _MMIO(0x9888), 0x238b2a80 },
-	{ _MMIO(0x9888), 0x258b0005 },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x1f85aa00 },
-	{ _MMIO(0x9888), 0x2185000a },
-	{ _MMIO(0x9888), 0x1b830150 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x0d848000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x01848000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x07844000 },
-	{ _MMIO(0x9888), 0x1d808000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x11808000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0x9888), 0x17804000 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x47801021 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3f800c64 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x41800c02 },
-};
-
-static int
-get_sampler_2_mux_config(struct drm_i915_private *dev_priv,
-			 const struct i915_oa_reg **regs,
-			 int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_sampler_2;
-	lens[n] = ARRAY_SIZE(mux_config_sampler_2);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_tdl_1[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x0000fdff },
-	{ _MMIO(0x2778), 0x00000000 },
-	{ _MMIO(0x277c), 0x0000fe7f },
-	{ _MMIO(0x2780), 0x00000002 },
-	{ _MMIO(0x2784), 0x0000ffbf },
-	{ _MMIO(0x2788), 0x00000000 },
-	{ _MMIO(0x278c), 0x0000ffcf },
-	{ _MMIO(0x2790), 0x00000002 },
-	{ _MMIO(0x2794), 0x0000fff7 },
-	{ _MMIO(0x2798), 0x00000000 },
-	{ _MMIO(0x279c), 0x0000fff9 },
-};
-
-static const struct i915_oa_reg flex_eu_config_tdl_1[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_tdl_1[] = {
-	{ _MMIO(0x9888), 0x16154d60 },
-	{ _MMIO(0x9888), 0x16352e60 },
-	{ _MMIO(0x9888), 0x16554d60 },
-	{ _MMIO(0x9888), 0x16950000 },
-	{ _MMIO(0x9888), 0x16b50000 },
-	{ _MMIO(0x9888), 0x16d50000 },
-	{ _MMIO(0x9888), 0x005c8000 },
-	{ _MMIO(0x9888), 0x045cc000 },
-	{ _MMIO(0x9888), 0x065c4000 },
-	{ _MMIO(0x9888), 0x083d8000 },
-	{ _MMIO(0x9888), 0x0a3d8000 },
-	{ _MMIO(0x9888), 0x0458c000 },
-	{ _MMIO(0x9888), 0x025b8000 },
-	{ _MMIO(0x9888), 0x085b4000 },
-	{ _MMIO(0x9888), 0x0a5b4000 },
-	{ _MMIO(0x9888), 0x0c5b8000 },
-	{ _MMIO(0x9888), 0x0c1fa000 },
-	{ _MMIO(0x9888), 0x0e1f00aa },
-	{ _MMIO(0x9888), 0x02384000 },
-	{ _MMIO(0x9888), 0x04388000 },
-	{ _MMIO(0x9888), 0x06388000 },
-	{ _MMIO(0x9888), 0x08384000 },
-	{ _MMIO(0x9888), 0x0a384000 },
-	{ _MMIO(0x9888), 0x0c384000 },
-	{ _MMIO(0x9888), 0x00398000 },
-	{ _MMIO(0x9888), 0x0239a000 },
-	{ _MMIO(0x9888), 0x0439a000 },
-	{ _MMIO(0x9888), 0x06392000 },
-	{ _MMIO(0x9888), 0x043a8000 },
-	{ _MMIO(0x9888), 0x063a8000 },
-	{ _MMIO(0x9888), 0x08138000 },
-	{ _MMIO(0x9888), 0x0a138000 },
-	{ _MMIO(0x9888), 0x06143000 },
-	{ _MMIO(0x9888), 0x0415cfc7 },
-	{ _MMIO(0x9888), 0x10150000 },
-	{ _MMIO(0x9888), 0x02338000 },
-	{ _MMIO(0x9888), 0x0c338000 },
-	{ _MMIO(0x9888), 0x04342000 },
-	{ _MMIO(0x9888), 0x06344000 },
-	{ _MMIO(0x9888), 0x0035c700 },
-	{ _MMIO(0x9888), 0x063500cf },
-	{ _MMIO(0x9888), 0x10350000 },
-	{ _MMIO(0x9888), 0x04538000 },
-	{ _MMIO(0x9888), 0x06538000 },
-	{ _MMIO(0x9888), 0x0454c000 },
-	{ _MMIO(0x9888), 0x0255cfc7 },
-	{ _MMIO(0x9888), 0x10550000 },
-	{ _MMIO(0x9888), 0x06dc8000 },
-	{ _MMIO(0x9888), 0x08dc4000 },
-	{ _MMIO(0x9888), 0x0cdcc000 },
-	{ _MMIO(0x9888), 0x0edcc000 },
-	{ _MMIO(0x9888), 0x1abd00a8 },
-	{ _MMIO(0x9888), 0x0cd8c000 },
-	{ _MMIO(0x9888), 0x0ed84000 },
-	{ _MMIO(0x9888), 0x0edb8000 },
-	{ _MMIO(0x9888), 0x18db0800 },
-	{ _MMIO(0x9888), 0x1adb0254 },
-	{ _MMIO(0x9888), 0x0e9faa00 },
-	{ _MMIO(0x9888), 0x109f02aa },
-	{ _MMIO(0x9888), 0x0eb84000 },
-	{ _MMIO(0x9888), 0x16b84000 },
-	{ _MMIO(0x9888), 0x18b8156a },
-	{ _MMIO(0x9888), 0x06b98000 },
-	{ _MMIO(0x9888), 0x08b9a000 },
-	{ _MMIO(0x9888), 0x0ab9a000 },
-	{ _MMIO(0x9888), 0x0cb9a000 },
-	{ _MMIO(0x9888), 0x0eb9a000 },
-	{ _MMIO(0x9888), 0x18baa000 },
-	{ _MMIO(0x9888), 0x1aba0002 },
-	{ _MMIO(0x9888), 0x16934000 },
-	{ _MMIO(0x9888), 0x1893000a },
-	{ _MMIO(0x9888), 0x0a947000 },
-	{ _MMIO(0x9888), 0x0c95c5c1 },
-	{ _MMIO(0x9888), 0x0e9500c3 },
-	{ _MMIO(0x9888), 0x10950000 },
-	{ _MMIO(0x9888), 0x0eb38000 },
-	{ _MMIO(0x9888), 0x16b30040 },
-	{ _MMIO(0x9888), 0x18b30020 },
-	{ _MMIO(0x9888), 0x06b48000 },
-	{ _MMIO(0x9888), 0x08b41000 },
-	{ _MMIO(0x9888), 0x0ab48000 },
-	{ _MMIO(0x9888), 0x06b5c500 },
-	{ _MMIO(0x9888), 0x08b500c3 },
-	{ _MMIO(0x9888), 0x0eb5c100 },
-	{ _MMIO(0x9888), 0x10b50000 },
-	{ _MMIO(0x9888), 0x16d31500 },
-	{ _MMIO(0x9888), 0x08d4e000 },
-	{ _MMIO(0x9888), 0x08d5c100 },
-	{ _MMIO(0x9888), 0x0ad5c3c5 },
-	{ _MMIO(0x9888), 0x10d50000 },
-	{ _MMIO(0x9888), 0x0d88f800 },
-	{ _MMIO(0x9888), 0x0f88000f },
-	{ _MMIO(0x9888), 0x038a8000 },
-	{ _MMIO(0x9888), 0x058a8000 },
-	{ _MMIO(0x9888), 0x078a8000 },
-	{ _MMIO(0x9888), 0x098a8000 },
-	{ _MMIO(0x9888), 0x0b8a8000 },
-	{ _MMIO(0x9888), 0x0d8a8000 },
-	{ _MMIO(0x9888), 0x258baaa5 },
-	{ _MMIO(0x9888), 0x278b002a },
-	{ _MMIO(0x9888), 0x238b2a80 },
-	{ _MMIO(0x9888), 0x0f8c4000 },
-	{ _MMIO(0x9888), 0x178c2000 },
-	{ _MMIO(0x9888), 0x198c5500 },
-	{ _MMIO(0x9888), 0x1b8c0015 },
-	{ _MMIO(0x9888), 0x078d8000 },
-	{ _MMIO(0x9888), 0x098da000 },
-	{ _MMIO(0x9888), 0x0b8da000 },
-	{ _MMIO(0x9888), 0x0d8da000 },
-	{ _MMIO(0x9888), 0x0f8da000 },
-	{ _MMIO(0x9888), 0x2185aaaa },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x1f85aa00 },
-	{ _MMIO(0x9888), 0x0f834000 },
-	{ _MMIO(0x9888), 0x19835400 },
-	{ _MMIO(0x9888), 0x1b830155 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x0784c000 },
-	{ _MMIO(0x9888), 0x0984c000 },
-	{ _MMIO(0x9888), 0x0b84c000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x01848000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x1780c000 },
-	{ _MMIO(0x9888), 0x1980c000 },
-	{ _MMIO(0x9888), 0x1b80c000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x11808000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x43800c42 },
-	{ _MMIO(0x9888), 0x51800000 },
-	{ _MMIO(0x9888), 0x45800063 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x47800800 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3f8014a4 },
-	{ _MMIO(0x9888), 0x41801042 },
-};
-
-static int
-get_tdl_1_mux_config(struct drm_i915_private *dev_priv,
-		     const struct i915_oa_reg **regs,
-		     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_tdl_1;
-	lens[n] = ARRAY_SIZE(mux_config_tdl_1);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_tdl_2[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x0000fdff },
-	{ _MMIO(0x2778), 0x00000000 },
-	{ _MMIO(0x277c), 0x0000fe7f },
-	{ _MMIO(0x2780), 0x00000000 },
-	{ _MMIO(0x2784), 0x0000ff9f },
-	{ _MMIO(0x2788), 0x00000000 },
-	{ _MMIO(0x278c), 0x0000ffe7 },
-	{ _MMIO(0x2790), 0x00000002 },
-	{ _MMIO(0x2794), 0x0000fffb },
-	{ _MMIO(0x2798), 0x00000002 },
-	{ _MMIO(0x279c), 0x0000fffd },
-};
-
-static const struct i915_oa_reg flex_eu_config_tdl_2[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_tdl_2[] = {
-	{ _MMIO(0x9888), 0x16150000 },
-	{ _MMIO(0x9888), 0x16350000 },
-	{ _MMIO(0x9888), 0x16550000 },
-	{ _MMIO(0x9888), 0x16952e60 },
-	{ _MMIO(0x9888), 0x16b54d60 },
-	{ _MMIO(0x9888), 0x16d52e60 },
-	{ _MMIO(0x9888), 0x065c8000 },
-	{ _MMIO(0x9888), 0x085cc000 },
-	{ _MMIO(0x9888), 0x0a5cc000 },
-	{ _MMIO(0x9888), 0x0c5c4000 },
-	{ _MMIO(0x9888), 0x0e3d8000 },
-	{ _MMIO(0x9888), 0x183da000 },
-	{ _MMIO(0x9888), 0x06588000 },
-	{ _MMIO(0x9888), 0x08588000 },
-	{ _MMIO(0x9888), 0x0a584000 },
-	{ _MMIO(0x9888), 0x0e5b4000 },
-	{ _MMIO(0x9888), 0x185b5800 },
-	{ _MMIO(0x9888), 0x1a5b000a },
-	{ _MMIO(0x9888), 0x0e1faa00 },
-	{ _MMIO(0x9888), 0x101f02aa },
-	{ _MMIO(0x9888), 0x0e384000 },
-	{ _MMIO(0x9888), 0x16384000 },
-	{ _MMIO(0x9888), 0x18382a55 },
-	{ _MMIO(0x9888), 0x06398000 },
-	{ _MMIO(0x9888), 0x0839a000 },
-	{ _MMIO(0x9888), 0x0a39a000 },
-	{ _MMIO(0x9888), 0x0c39a000 },
-	{ _MMIO(0x9888), 0x0e39a000 },
-	{ _MMIO(0x9888), 0x1a3a02a0 },
-	{ _MMIO(0x9888), 0x0e138000 },
-	{ _MMIO(0x9888), 0x16130500 },
-	{ _MMIO(0x9888), 0x06148000 },
-	{ _MMIO(0x9888), 0x08146000 },
-	{ _MMIO(0x9888), 0x0615c100 },
-	{ _MMIO(0x9888), 0x0815c500 },
-	{ _MMIO(0x9888), 0x0a1500c3 },
-	{ _MMIO(0x9888), 0x10150000 },
-	{ _MMIO(0x9888), 0x16335040 },
-	{ _MMIO(0x9888), 0x08349000 },
-	{ _MMIO(0x9888), 0x0a341000 },
-	{ _MMIO(0x9888), 0x083500c1 },
-	{ _MMIO(0x9888), 0x0a35c500 },
-	{ _MMIO(0x9888), 0x0c3500c3 },
-	{ _MMIO(0x9888), 0x10350000 },
-	{ _MMIO(0x9888), 0x1853002a },
-	{ _MMIO(0x9888), 0x0a54e000 },
-	{ _MMIO(0x9888), 0x0c55c500 },
-	{ _MMIO(0x9888), 0x0e55c1c3 },
-	{ _MMIO(0x9888), 0x10550000 },
-	{ _MMIO(0x9888), 0x00dc8000 },
-	{ _MMIO(0x9888), 0x02dcc000 },
-	{ _MMIO(0x9888), 0x04dc4000 },
-	{ _MMIO(0x9888), 0x04bd8000 },
-	{ _MMIO(0x9888), 0x06bd8000 },
-	{ _MMIO(0x9888), 0x02d8c000 },
-	{ _MMIO(0x9888), 0x02db8000 },
-	{ _MMIO(0x9888), 0x04db4000 },
-	{ _MMIO(0x9888), 0x06db4000 },
-	{ _MMIO(0x9888), 0x08db8000 },
-	{ _MMIO(0x9888), 0x0c9fa000 },
-	{ _MMIO(0x9888), 0x0e9f00aa },
-	{ _MMIO(0x9888), 0x02b84000 },
-	{ _MMIO(0x9888), 0x04b84000 },
-	{ _MMIO(0x9888), 0x06b84000 },
-	{ _MMIO(0x9888), 0x08b84000 },
-	{ _MMIO(0x9888), 0x0ab88000 },
-	{ _MMIO(0x9888), 0x0cb88000 },
-	{ _MMIO(0x9888), 0x00b98000 },
-	{ _MMIO(0x9888), 0x02b9a000 },
-	{ _MMIO(0x9888), 0x04b9a000 },
-	{ _MMIO(0x9888), 0x06b92000 },
-	{ _MMIO(0x9888), 0x0aba8000 },
-	{ _MMIO(0x9888), 0x0cba8000 },
-	{ _MMIO(0x9888), 0x04938000 },
-	{ _MMIO(0x9888), 0x06938000 },
-	{ _MMIO(0x9888), 0x0494c000 },
-	{ _MMIO(0x9888), 0x0295cfc7 },
-	{ _MMIO(0x9888), 0x10950000 },
-	{ _MMIO(0x9888), 0x02b38000 },
-	{ _MMIO(0x9888), 0x08b38000 },
-	{ _MMIO(0x9888), 0x04b42000 },
-	{ _MMIO(0x9888), 0x06b41000 },
-	{ _MMIO(0x9888), 0x00b5c700 },
-	{ _MMIO(0x9888), 0x04b500cf },
-	{ _MMIO(0x9888), 0x10b50000 },
-	{ _MMIO(0x9888), 0x0ad38000 },
-	{ _MMIO(0x9888), 0x0cd38000 },
-	{ _MMIO(0x9888), 0x06d46000 },
-	{ _MMIO(0x9888), 0x04d5c700 },
-	{ _MMIO(0x9888), 0x06d500cf },
-	{ _MMIO(0x9888), 0x10d50000 },
-	{ _MMIO(0x9888), 0x03888000 },
-	{ _MMIO(0x9888), 0x05888000 },
-	{ _MMIO(0x9888), 0x07888000 },
-	{ _MMIO(0x9888), 0x09888000 },
-	{ _MMIO(0x9888), 0x0b888000 },
-	{ _MMIO(0x9888), 0x0d880400 },
-	{ _MMIO(0x9888), 0x0f8a8000 },
-	{ _MMIO(0x9888), 0x198a8000 },
-	{ _MMIO(0x9888), 0x1b8aaaa0 },
-	{ _MMIO(0x9888), 0x1d8a0002 },
-	{ _MMIO(0x9888), 0x258b555a },
-	{ _MMIO(0x9888), 0x278b0015 },
-	{ _MMIO(0x9888), 0x238b5500 },
-	{ _MMIO(0x9888), 0x038c4000 },
-	{ _MMIO(0x9888), 0x058c4000 },
-	{ _MMIO(0x9888), 0x078c4000 },
-	{ _MMIO(0x9888), 0x098c4000 },
-	{ _MMIO(0x9888), 0x0b8c4000 },
-	{ _MMIO(0x9888), 0x0d8c4000 },
-	{ _MMIO(0x9888), 0x018d8000 },
-	{ _MMIO(0x9888), 0x038da000 },
-	{ _MMIO(0x9888), 0x058da000 },
-	{ _MMIO(0x9888), 0x078d2000 },
-	{ _MMIO(0x9888), 0x2185aaaa },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x1f85aa00 },
-	{ _MMIO(0x9888), 0x0f834000 },
-	{ _MMIO(0x9888), 0x19835400 },
-	{ _MMIO(0x9888), 0x1b830155 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x0784c000 },
-	{ _MMIO(0x9888), 0x0984c000 },
-	{ _MMIO(0x9888), 0x0b84c000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x01848000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x1780c000 },
-	{ _MMIO(0x9888), 0x1980c000 },
-	{ _MMIO(0x9888), 0x1b80c000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x11808000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x43800882 },
-	{ _MMIO(0x9888), 0x51800000 },
-	{ _MMIO(0x9888), 0x45801082 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x478014a5 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3f800002 },
-	{ _MMIO(0x9888), 0x41800c62 },
-};
-
-static int
-get_tdl_2_mux_config(struct drm_i915_private *dev_priv,
-		     const struct i915_oa_reg **regs,
-		     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_tdl_2;
-	lens[n] = ARRAY_SIZE(mux_config_tdl_2);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_extra[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_extra[] = {
-	{ _MMIO(0xe458), 0x00001000 },
-	{ _MMIO(0xe558), 0x00003002 },
-	{ _MMIO(0xe658), 0x00005004 },
-	{ _MMIO(0xe758), 0x00011010 },
-	{ _MMIO(0xe45c), 0x00050012 },
-	{ _MMIO(0xe55c), 0x00052051 },
-	{ _MMIO(0xe65c), 0x00000008 },
-};
-
-static const struct i915_oa_reg mux_config_compute_extra[] = {
-	{ _MMIO(0x9888), 0x161503e0 },
-	{ _MMIO(0x9888), 0x163503e0 },
-	{ _MMIO(0x9888), 0x165503e0 },
-	{ _MMIO(0x9888), 0x169503e0 },
-	{ _MMIO(0x9888), 0x16b503e0 },
-	{ _MMIO(0x9888), 0x16d503e0 },
-	{ _MMIO(0x9888), 0x045cc000 },
-	{ _MMIO(0x9888), 0x083d8000 },
-	{ _MMIO(0x9888), 0x04584000 },
-	{ _MMIO(0x9888), 0x085b4000 },
-	{ _MMIO(0x9888), 0x0a5b8000 },
-	{ _MMIO(0x9888), 0x0e1f00a8 },
-	{ _MMIO(0x9888), 0x08384000 },
-	{ _MMIO(0x9888), 0x0a384000 },
-	{ _MMIO(0x9888), 0x0c388000 },
-	{ _MMIO(0x9888), 0x0439a000 },
-	{ _MMIO(0x9888), 0x06392000 },
-	{ _MMIO(0x9888), 0x0c3a8000 },
-	{ _MMIO(0x9888), 0x08138000 },
-	{ _MMIO(0x9888), 0x06141000 },
-	{ _MMIO(0x9888), 0x041500c3 },
-	{ _MMIO(0x9888), 0x10150000 },
-	{ _MMIO(0x9888), 0x0a338000 },
-	{ _MMIO(0x9888), 0x06342000 },
-	{ _MMIO(0x9888), 0x0435c300 },
-	{ _MMIO(0x9888), 0x10350000 },
-	{ _MMIO(0x9888), 0x0c538000 },
-	{ _MMIO(0x9888), 0x06544000 },
-	{ _MMIO(0x9888), 0x065500c3 },
-	{ _MMIO(0x9888), 0x10550000 },
-	{ _MMIO(0x9888), 0x00dc8000 },
-	{ _MMIO(0x9888), 0x02dc4000 },
-	{ _MMIO(0x9888), 0x02bd8000 },
-	{ _MMIO(0x9888), 0x00d88000 },
-	{ _MMIO(0x9888), 0x02db4000 },
-	{ _MMIO(0x9888), 0x04db8000 },
-	{ _MMIO(0x9888), 0x0c9fa000 },
-	{ _MMIO(0x9888), 0x0e9f0002 },
-	{ _MMIO(0x9888), 0x02b84000 },
-	{ _MMIO(0x9888), 0x04b84000 },
-	{ _MMIO(0x9888), 0x06b88000 },
-	{ _MMIO(0x9888), 0x00b98000 },
-	{ _MMIO(0x9888), 0x02b9a000 },
-	{ _MMIO(0x9888), 0x06ba8000 },
-	{ _MMIO(0x9888), 0x02938000 },
-	{ _MMIO(0x9888), 0x04942000 },
-	{ _MMIO(0x9888), 0x0095c300 },
-	{ _MMIO(0x9888), 0x10950000 },
-	{ _MMIO(0x9888), 0x04b38000 },
-	{ _MMIO(0x9888), 0x04b44000 },
-	{ _MMIO(0x9888), 0x02b500c3 },
-	{ _MMIO(0x9888), 0x10b50000 },
-	{ _MMIO(0x9888), 0x06d38000 },
-	{ _MMIO(0x9888), 0x04d48000 },
-	{ _MMIO(0x9888), 0x02d5c300 },
-	{ _MMIO(0x9888), 0x10d50000 },
-	{ _MMIO(0x9888), 0x03888000 },
-	{ _MMIO(0x9888), 0x05888000 },
-	{ _MMIO(0x9888), 0x07888000 },
-	{ _MMIO(0x9888), 0x098a8000 },
-	{ _MMIO(0x9888), 0x0b8a8000 },
-	{ _MMIO(0x9888), 0x0d8a8000 },
-	{ _MMIO(0x9888), 0x238b3500 },
-	{ _MMIO(0x9888), 0x258b0005 },
-	{ _MMIO(0x9888), 0x038c4000 },
-	{ _MMIO(0x9888), 0x058c4000 },
-	{ _MMIO(0x9888), 0x078c4000 },
-	{ _MMIO(0x9888), 0x018d8000 },
-	{ _MMIO(0x9888), 0x038da000 },
-	{ _MMIO(0x9888), 0x1f85aa00 },
-	{ _MMIO(0x9888), 0x2185000a },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x01848000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x07844000 },
-	{ _MMIO(0x9888), 0x11808000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0x9888), 0x17804000 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3f800c40 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x41801482 },
-	{ _MMIO(0x9888), 0x31800000 },
-};
-
-static int
-get_compute_extra_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_extra;
-	lens[n] = ARRAY_SIZE(mux_config_compute_extra);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_vme_pipe[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2770), 0x00100030 },
-	{ _MMIO(0x2774), 0x0000fff9 },
-	{ _MMIO(0x2778), 0x00000002 },
-	{ _MMIO(0x277c), 0x0000fffc },
-	{ _MMIO(0x2780), 0x00000002 },
-	{ _MMIO(0x2784), 0x0000fff3 },
-	{ _MMIO(0x2788), 0x00100180 },
-	{ _MMIO(0x278c), 0x0000ffcf },
-	{ _MMIO(0x2790), 0x00000002 },
-	{ _MMIO(0x2794), 0x0000ffcf },
-	{ _MMIO(0x2798), 0x00000002 },
-	{ _MMIO(0x279c), 0x0000ff3f },
-};
-
-static const struct i915_oa_reg flex_eu_config_vme_pipe[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00008003 },
-};
-
-static const struct i915_oa_reg mux_config_vme_pipe[] = {
-	{ _MMIO(0x9888), 0x14100812 },
-	{ _MMIO(0x9888), 0x14125800 },
-	{ _MMIO(0x9888), 0x161200c0 },
-	{ _MMIO(0x9888), 0x14300812 },
-	{ _MMIO(0x9888), 0x14325800 },
-	{ _MMIO(0x9888), 0x163200c0 },
-	{ _MMIO(0x9888), 0x005c4000 },
-	{ _MMIO(0x9888), 0x065c8000 },
-	{ _MMIO(0x9888), 0x085cc000 },
-	{ _MMIO(0x9888), 0x0a5cc000 },
-	{ _MMIO(0x9888), 0x0c5cc000 },
-	{ _MMIO(0x9888), 0x003d8000 },
-	{ _MMIO(0x9888), 0x0e3d8000 },
-	{ _MMIO(0x9888), 0x183d2800 },
-	{ _MMIO(0x9888), 0x00584000 },
-	{ _MMIO(0x9888), 0x06588000 },
-	{ _MMIO(0x9888), 0x0858c000 },
-	{ _MMIO(0x9888), 0x005b4000 },
-	{ _MMIO(0x9888), 0x0e5b4000 },
-	{ _MMIO(0x9888), 0x185b9400 },
-	{ _MMIO(0x9888), 0x1a5b002a },
-	{ _MMIO(0x9888), 0x0c1f0800 },
-	{ _MMIO(0x9888), 0x0e1faa00 },
-	{ _MMIO(0x9888), 0x101f002a },
-	{ _MMIO(0x9888), 0x00384000 },
-	{ _MMIO(0x9888), 0x0e384000 },
-	{ _MMIO(0x9888), 0x16384000 },
-	{ _MMIO(0x9888), 0x18380155 },
-	{ _MMIO(0x9888), 0x00392000 },
-	{ _MMIO(0x9888), 0x06398000 },
-	{ _MMIO(0x9888), 0x0839a000 },
-	{ _MMIO(0x9888), 0x0a39a000 },
-	{ _MMIO(0x9888), 0x0c39a000 },
-	{ _MMIO(0x9888), 0x00100047 },
-	{ _MMIO(0x9888), 0x06101a80 },
-	{ _MMIO(0x9888), 0x10100000 },
-	{ _MMIO(0x9888), 0x0810c000 },
-	{ _MMIO(0x9888), 0x0811c000 },
-	{ _MMIO(0x9888), 0x08126151 },
-	{ _MMIO(0x9888), 0x10120000 },
-	{ _MMIO(0x9888), 0x00134000 },
-	{ _MMIO(0x9888), 0x0e134000 },
-	{ _MMIO(0x9888), 0x161300a0 },
-	{ _MMIO(0x9888), 0x0a301ac7 },
-	{ _MMIO(0x9888), 0x10300000 },
-	{ _MMIO(0x9888), 0x0c30c000 },
-	{ _MMIO(0x9888), 0x0c31c000 },
-	{ _MMIO(0x9888), 0x0c326151 },
-	{ _MMIO(0x9888), 0x10320000 },
-	{ _MMIO(0x9888), 0x16332a00 },
-	{ _MMIO(0x9888), 0x18330001 },
-	{ _MMIO(0x9888), 0x018a8000 },
-	{ _MMIO(0x9888), 0x0f8a8000 },
-	{ _MMIO(0x9888), 0x198a8000 },
-	{ _MMIO(0x9888), 0x1b8a2aa0 },
-	{ _MMIO(0x9888), 0x238b0020 },
-	{ _MMIO(0x9888), 0x258b5550 },
-	{ _MMIO(0x9888), 0x278b0001 },
-	{ _MMIO(0x9888), 0x1f850080 },
-	{ _MMIO(0x9888), 0x2185aaa0 },
-	{ _MMIO(0x9888), 0x23850002 },
-	{ _MMIO(0x9888), 0x01834000 },
-	{ _MMIO(0x9888), 0x0f834000 },
-	{ _MMIO(0x9888), 0x19835400 },
-	{ _MMIO(0x9888), 0x1b830015 },
-	{ _MMIO(0x9888), 0x01844000 },
-	{ _MMIO(0x9888), 0x07848000 },
-	{ _MMIO(0x9888), 0x0984c000 },
-	{ _MMIO(0x9888), 0x0b84c000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x11804000 },
-	{ _MMIO(0x9888), 0x17808000 },
-	{ _MMIO(0x9888), 0x1980c000 },
-	{ _MMIO(0x9888), 0x1b80c000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3d800800 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x43800002 },
-	{ _MMIO(0x9888), 0x51800000 },
-	{ _MMIO(0x9888), 0x45800884 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x47800002 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-};
-
-static int
-get_vme_pipe_mux_config(struct drm_i915_private *dev_priv,
-			const struct i915_oa_reg **regs,
-			int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_vme_pipe;
-	lens[n] = ARRAY_SIZE(mux_config_vme_pipe);
-	n++;
-
-	return n;
-}
-
 static const struct i915_oa_reg b_counter_config_test_oa[] = {
 	{ _MMIO(0x2740), 0x00000000 },
 	{ _MMIO(0x2744), 0x00800000 },
@@ -4035,6 +60,7 @@ static const struct i915_oa_reg flex_eu_config_test_oa[] = {
 };
 
 static const struct i915_oa_reg mux_config_test_oa[] = {
+	{ _MMIO(0x9840), 0x000000a0 },
 	{ _MMIO(0x9888), 0x198b0000 },
 	{ _MMIO(0x9888), 0x078b0066 },
 	{ _MMIO(0x9888), 0x118b0000 },
@@ -4047,1330 +73,38 @@ static const struct i915_oa_reg mux_config_test_oa[] = {
 	{ _MMIO(0x9888), 0x4f800000 },
 	{ _MMIO(0x9888), 0x41800000 },
 	{ _MMIO(0x9888), 0x31800000 },
-};
-
-static int
-get_test_oa_mux_config(struct drm_i915_private *dev_priv,
-		       const struct i915_oa_reg **regs,
-		       int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_test_oa;
-	lens[n] = ARRAY_SIZE(mux_config_test_oa);
-	n++;
-
-	return n;
-}
-
-int i915_oa_select_metric_set_bdw(struct drm_i915_private *dev_priv)
-{
-	dev_priv->perf.oa.n_mux_configs = 0;
-	dev_priv->perf.oa.b_counter_regs = NULL;
-	dev_priv->perf.oa.b_counter_regs_len = 0;
-	dev_priv->perf.oa.flex_regs = NULL;
-	dev_priv->perf.oa.flex_regs_len = 0;
-
-	switch (dev_priv->perf.oa.metrics_set) {
-	case METRIC_SET_ID_RENDER_BASIC:
-		dev_priv->perf.oa.n_mux_configs =
-			get_render_basic_mux_config(dev_priv,
-						    dev_priv->perf.oa.mux_regs,
-						    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_BASIC\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_render_basic;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_render_basic);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_render_basic;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_render_basic);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_BASIC:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_basic_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_BASIC\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_basic;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_basic);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_basic;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_basic);
-
-		return 0;
-	case METRIC_SET_ID_RENDER_PIPE_PROFILE:
-		dev_priv->perf.oa.n_mux_configs =
-			get_render_pipe_profile_mux_config(dev_priv,
-							   dev_priv->perf.oa.mux_regs,
-							   dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_PIPE_PROFILE\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_render_pipe_profile;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_render_pipe_profile);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_render_pipe_profile;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_render_pipe_profile);
-
-		return 0;
-	case METRIC_SET_ID_MEMORY_READS:
-		dev_priv->perf.oa.n_mux_configs =
-			get_memory_reads_mux_config(dev_priv,
-						    dev_priv->perf.oa.mux_regs,
-						    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_READS\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_memory_reads;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_memory_reads);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_memory_reads;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_memory_reads);
-
-		return 0;
-	case METRIC_SET_ID_MEMORY_WRITES:
-		dev_priv->perf.oa.n_mux_configs =
-			get_memory_writes_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_WRITES\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_memory_writes;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_memory_writes);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_memory_writes;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_memory_writes);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_EXTENDED:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_extended_mux_config(dev_priv,
-							dev_priv->perf.oa.mux_regs,
-							dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTENDED\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_extended;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_extended);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_extended;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_extended);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_L3_CACHE:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_l3_cache_mux_config(dev_priv,
-							dev_priv->perf.oa.mux_regs,
-							dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_L3_CACHE\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_l3_cache;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_l3_cache);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_l3_cache;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_l3_cache);
-
-		return 0;
-	case METRIC_SET_ID_DATA_PORT_READS_COALESCING:
-		dev_priv->perf.oa.n_mux_configs =
-			get_data_port_reads_coalescing_mux_config(dev_priv,
-								  dev_priv->perf.oa.mux_regs,
-								  dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"DATA_PORT_READS_COALESCING\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_data_port_reads_coalescing;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_data_port_reads_coalescing);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_data_port_reads_coalescing;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_data_port_reads_coalescing);
-
-		return 0;
-	case METRIC_SET_ID_DATA_PORT_WRITES_COALESCING:
-		dev_priv->perf.oa.n_mux_configs =
-			get_data_port_writes_coalescing_mux_config(dev_priv,
-								   dev_priv->perf.oa.mux_regs,
-								   dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"DATA_PORT_WRITES_COALESCING\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_data_port_writes_coalescing;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_data_port_writes_coalescing);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_data_port_writes_coalescing;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_data_port_writes_coalescing);
-
-		return 0;
-	case METRIC_SET_ID_HDC_AND_SF:
-		dev_priv->perf.oa.n_mux_configs =
-			get_hdc_and_sf_mux_config(dev_priv,
-						  dev_priv->perf.oa.mux_regs,
-						  dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"HDC_AND_SF\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_hdc_and_sf;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_hdc_and_sf);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_hdc_and_sf;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_hdc_and_sf);
-
-		return 0;
-	case METRIC_SET_ID_L3_1:
-		dev_priv->perf.oa.n_mux_configs =
-			get_l3_1_mux_config(dev_priv,
-					    dev_priv->perf.oa.mux_regs,
-					    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_1\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_l3_1;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_l3_1);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_l3_1;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_l3_1);
-
-		return 0;
-	case METRIC_SET_ID_L3_2:
-		dev_priv->perf.oa.n_mux_configs =
-			get_l3_2_mux_config(dev_priv,
-					    dev_priv->perf.oa.mux_regs,
-					    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_2\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_l3_2;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_l3_2);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_l3_2;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_l3_2);
-
-		return 0;
-	case METRIC_SET_ID_L3_3:
-		dev_priv->perf.oa.n_mux_configs =
-			get_l3_3_mux_config(dev_priv,
-					    dev_priv->perf.oa.mux_regs,
-					    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_3\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_l3_3;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_l3_3);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_l3_3;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_l3_3);
-
-		return 0;
-	case METRIC_SET_ID_L3_4:
-		dev_priv->perf.oa.n_mux_configs =
-			get_l3_4_mux_config(dev_priv,
-					    dev_priv->perf.oa.mux_regs,
-					    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_4\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_l3_4;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_l3_4);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_l3_4;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_l3_4);
-
-		return 0;
-	case METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND:
-		dev_priv->perf.oa.n_mux_configs =
-			get_rasterizer_and_pixel_backend_mux_config(dev_priv,
-								    dev_priv->perf.oa.mux_regs,
-								    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RASTERIZER_AND_PIXEL_BACKEND\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_rasterizer_and_pixel_backend;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_rasterizer_and_pixel_backend);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_rasterizer_and_pixel_backend;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_rasterizer_and_pixel_backend);
-
-		return 0;
-	case METRIC_SET_ID_SAMPLER_1:
-		dev_priv->perf.oa.n_mux_configs =
-			get_sampler_1_mux_config(dev_priv,
-						 dev_priv->perf.oa.mux_regs,
-						 dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"SAMPLER_1\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_sampler_1;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_sampler_1);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_sampler_1;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_sampler_1);
-
-		return 0;
-	case METRIC_SET_ID_SAMPLER_2:
-		dev_priv->perf.oa.n_mux_configs =
-			get_sampler_2_mux_config(dev_priv,
-						 dev_priv->perf.oa.mux_regs,
-						 dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"SAMPLER_2\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_sampler_2;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_sampler_2);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_sampler_2;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_sampler_2);
-
-		return 0;
-	case METRIC_SET_ID_TDL_1:
-		dev_priv->perf.oa.n_mux_configs =
-			get_tdl_1_mux_config(dev_priv,
-					     dev_priv->perf.oa.mux_regs,
-					     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_1\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_tdl_1;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_tdl_1);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_tdl_1;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_tdl_1);
-
-		return 0;
-	case METRIC_SET_ID_TDL_2:
-		dev_priv->perf.oa.n_mux_configs =
-			get_tdl_2_mux_config(dev_priv,
-					     dev_priv->perf.oa.mux_regs,
-					     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_2\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_tdl_2;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_tdl_2);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_tdl_2;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_tdl_2);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_EXTRA:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_extra_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTRA\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_extra;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_extra);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_extra;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_extra);
-
-		return 0;
-	case METRIC_SET_ID_VME_PIPE:
-		dev_priv->perf.oa.n_mux_configs =
-			get_vme_pipe_mux_config(dev_priv,
-						dev_priv->perf.oa.mux_regs,
-						dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"VME_PIPE\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_vme_pipe;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_vme_pipe);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_vme_pipe;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_vme_pipe);
-
-		return 0;
-	case METRIC_SET_ID_TEST_OA:
-		dev_priv->perf.oa.n_mux_configs =
-			get_test_oa_mux_config(dev_priv,
-					       dev_priv->perf.oa.mux_regs,
-					       dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TEST_OA\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_test_oa;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_test_oa);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_test_oa;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_test_oa);
-
-		return 0;
-	default:
-		return -ENODEV;
-	}
-}
-
-static ssize_t
-show_render_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_BASIC);
-}
-
-static struct device_attribute dev_attr_render_basic_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_render_basic_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_render_basic[] = {
-	&dev_attr_render_basic_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_render_basic = {
-	.name = "b541bd57-0e0f-4154-b4c0-5858010a2bf7",
-	.attrs =  attrs_render_basic,
-};
-
-static ssize_t
-show_compute_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_BASIC);
-}
-
-static struct device_attribute dev_attr_compute_basic_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_basic_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_basic[] = {
-	&dev_attr_compute_basic_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_basic = {
-	.name = "35fbc9b2-a891-40a6-a38d-022bb7057552",
-	.attrs =  attrs_compute_basic,
-};
-
-static ssize_t
-show_render_pipe_profile_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_PIPE_PROFILE);
-}
-
-static struct device_attribute dev_attr_render_pipe_profile_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_render_pipe_profile_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_render_pipe_profile[] = {
-	&dev_attr_render_pipe_profile_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_render_pipe_profile = {
-	.name = "233d0544-fff7-4281-8291-e02f222aff72",
-	.attrs =  attrs_render_pipe_profile,
-};
-
-static ssize_t
-show_memory_reads_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_READS);
-}
-
-static struct device_attribute dev_attr_memory_reads_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_memory_reads_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_memory_reads[] = {
-	&dev_attr_memory_reads_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_memory_reads = {
-	.name = "2b255d48-2117-4fef-a8f7-f151e1d25a2c",
-	.attrs =  attrs_memory_reads,
-};
-
-static ssize_t
-show_memory_writes_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_WRITES);
-}
-
-static struct device_attribute dev_attr_memory_writes_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_memory_writes_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_memory_writes[] = {
-	&dev_attr_memory_writes_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_memory_writes = {
-	.name = "f7fd3220-b466-4a4d-9f98-b0caf3f2394c",
-	.attrs =  attrs_memory_writes,
-};
-
-static ssize_t
-show_compute_extended_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTENDED);
-}
-
-static struct device_attribute dev_attr_compute_extended_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_extended_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_extended[] = {
-	&dev_attr_compute_extended_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_extended = {
-	.name = "e99ccaca-821c-4df9-97a7-96bdb7204e43",
-	.attrs =  attrs_compute_extended,
-};
-
-static ssize_t
-show_compute_l3_cache_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_L3_CACHE);
-}
-
-static struct device_attribute dev_attr_compute_l3_cache_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_l3_cache_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_l3_cache[] = {
-	&dev_attr_compute_l3_cache_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_l3_cache = {
-	.name = "27a364dc-8225-4ecb-b607-d6f1925598d9",
-	.attrs =  attrs_compute_l3_cache,
-};
-
-static ssize_t
-show_data_port_reads_coalescing_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_DATA_PORT_READS_COALESCING);
-}
-
-static struct device_attribute dev_attr_data_port_reads_coalescing_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_data_port_reads_coalescing_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_data_port_reads_coalescing[] = {
-	&dev_attr_data_port_reads_coalescing_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_data_port_reads_coalescing = {
-	.name = "857fc630-2f09-4804-85f1-084adfadd5ab",
-	.attrs =  attrs_data_port_reads_coalescing,
-};
-
-static ssize_t
-show_data_port_writes_coalescing_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_DATA_PORT_WRITES_COALESCING);
-}
-
-static struct device_attribute dev_attr_data_port_writes_coalescing_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_data_port_writes_coalescing_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_data_port_writes_coalescing[] = {
-	&dev_attr_data_port_writes_coalescing_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_data_port_writes_coalescing = {
-	.name = "343ebc99-4a55-414c-8c17-d8e259cf5e20",
-	.attrs =  attrs_data_port_writes_coalescing,
-};
-
-static ssize_t
-show_hdc_and_sf_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_HDC_AND_SF);
-}
-
-static struct device_attribute dev_attr_hdc_and_sf_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_hdc_and_sf_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_hdc_and_sf[] = {
-	&dev_attr_hdc_and_sf_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_hdc_and_sf = {
-	.name = "7bdafd88-a4fa-4ed5-bc09-1a977aa5be3e",
-	.attrs =  attrs_hdc_and_sf,
-};
-
-static ssize_t
-show_l3_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_1);
-}
-
-static struct device_attribute dev_attr_l3_1_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_l3_1_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_l3_1[] = {
-	&dev_attr_l3_1_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_l3_1 = {
-	.name = "9385ebb2-f34f-4aa5-aec5-7e9cbbea0f0b",
-	.attrs =  attrs_l3_1,
-};
-
-static ssize_t
-show_l3_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_2);
-}
-
-static struct device_attribute dev_attr_l3_2_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_l3_2_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_l3_2[] = {
-	&dev_attr_l3_2_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_l3_2 = {
-	.name = "446ae59b-ff2e-41c9-b49e-0184a54bf00a",
-	.attrs =  attrs_l3_2,
-};
-
-static ssize_t
-show_l3_3_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_3);
-}
-
-static struct device_attribute dev_attr_l3_3_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_l3_3_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_l3_3[] = {
-	&dev_attr_l3_3_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_l3_3 = {
-	.name = "84a7956f-1ea4-4d0d-837f-e39a0376e38c",
-	.attrs =  attrs_l3_3,
-};
-
-static ssize_t
-show_l3_4_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_4);
-}
-
-static struct device_attribute dev_attr_l3_4_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_l3_4_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_l3_4[] = {
-	&dev_attr_l3_4_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_l3_4 = {
-	.name = "92b493d9-df18-4bed-be06-5cac6f2a6f5f",
-	.attrs =  attrs_l3_4,
-};
-
-static ssize_t
-show_rasterizer_and_pixel_backend_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND);
-}
-
-static struct device_attribute dev_attr_rasterizer_and_pixel_backend_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_rasterizer_and_pixel_backend_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_rasterizer_and_pixel_backend[] = {
-	&dev_attr_rasterizer_and_pixel_backend_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_rasterizer_and_pixel_backend = {
-	.name = "14345c35-cc46-40d0-bb04-6ed1fbb43679",
-	.attrs =  attrs_rasterizer_and_pixel_backend,
-};
-
-static ssize_t
-show_sampler_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_SAMPLER_1);
-}
-
-static struct device_attribute dev_attr_sampler_1_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_sampler_1_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_sampler_1[] = {
-	&dev_attr_sampler_1_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_sampler_1 = {
-	.name = "f0c6ba37-d3d3-4211-91b5-226730312a54",
-	.attrs =  attrs_sampler_1,
-};
-
-static ssize_t
-show_sampler_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_SAMPLER_2);
-}
-
-static struct device_attribute dev_attr_sampler_2_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_sampler_2_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_sampler_2[] = {
-	&dev_attr_sampler_2_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_sampler_2 = {
-	.name = "30bf3702-48cf-4bca-b412-7cf50bb2f564",
-	.attrs =  attrs_sampler_2,
-};
-
-static ssize_t
-show_tdl_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_1);
-}
-
-static struct device_attribute dev_attr_tdl_1_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_tdl_1_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_tdl_1[] = {
-	&dev_attr_tdl_1_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_tdl_1 = {
-	.name = "238bec85-df05-44f3-b905-d166712f2451",
-	.attrs =  attrs_tdl_1,
-};
-
-static ssize_t
-show_tdl_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_2);
-}
-
-static struct device_attribute dev_attr_tdl_2_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_tdl_2_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_tdl_2[] = {
-	&dev_attr_tdl_2_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_tdl_2 = {
-	.name = "24bf02cd-8693-4583-981c-c4165b33da01",
-	.attrs =  attrs_tdl_2,
-};
-
-static ssize_t
-show_compute_extra_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTRA);
-}
-
-static struct device_attribute dev_attr_compute_extra_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_extra_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_extra[] = {
-	&dev_attr_compute_extra_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_extra = {
-	.name = "8fb61ba2-2fbb-454c-a136-2dec5a8a595e",
-	.attrs =  attrs_compute_extra,
-};
-
-static ssize_t
-show_vme_pipe_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_VME_PIPE);
-}
-
-static struct device_attribute dev_attr_vme_pipe_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_vme_pipe_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_vme_pipe[] = {
-	&dev_attr_vme_pipe_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_vme_pipe = {
-	.name = "e1743ca0-7fc8-410b-a066-de7bbb9280b7",
-	.attrs =  attrs_vme_pipe,
+	{ _MMIO(0x9840), 0x00000080 },
 };
 
 static ssize_t
 show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TEST_OA);
+	return sprintf(buf, "1\n");
 }
 
-static struct device_attribute dev_attr_test_oa_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_test_oa_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_test_oa[] = {
-	&dev_attr_test_oa_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_test_oa = {
-	.name = "d6de6f55-e526-4f79-a6a6-d7315c09044e",
-	.attrs =  attrs_test_oa,
-};
-
-int
-i915_perf_register_sysfs_bdw(struct drm_i915_private *dev_priv)
+void
+i915_perf_load_test_config_bdw(struct drm_i915_private *dev_priv)
 {
-	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
-	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
-	int ret = 0;
+	strncpy(dev_priv->perf.oa.test_config.uuid,
+		"d6de6f55-e526-4f79-a6a6-d7315c09044e",
+		UUID_STRING_LEN);
+	dev_priv->perf.oa.test_config.id = 1;
 
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-		if (ret)
-			goto error_render_basic;
-	}
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-		if (ret)
-			goto error_compute_basic;
-	}
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-		if (ret)
-			goto error_render_pipe_profile;
-	}
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-		if (ret)
-			goto error_memory_reads;
-	}
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-		if (ret)
-			goto error_memory_writes;
-	}
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-		if (ret)
-			goto error_compute_extended;
-	}
-	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
-		if (ret)
-			goto error_compute_l3_cache;
-	}
-	if (get_data_port_reads_coalescing_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_data_port_reads_coalescing);
-		if (ret)
-			goto error_data_port_reads_coalescing;
-	}
-	if (get_data_port_writes_coalescing_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_data_port_writes_coalescing);
-		if (ret)
-			goto error_data_port_writes_coalescing;
-	}
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-		if (ret)
-			goto error_hdc_and_sf;
-	}
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-		if (ret)
-			goto error_l3_1;
-	}
-	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_2);
-		if (ret)
-			goto error_l3_2;
-	}
-	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_3);
-		if (ret)
-			goto error_l3_3;
-	}
-	if (get_l3_4_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_4);
-		if (ret)
-			goto error_l3_4;
-	}
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-		if (ret)
-			goto error_rasterizer_and_pixel_backend;
-	}
-	if (get_sampler_1_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_sampler_1);
-		if (ret)
-			goto error_sampler_1;
-	}
-	if (get_sampler_2_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_sampler_2);
-		if (ret)
-			goto error_sampler_2;
-	}
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-		if (ret)
-			goto error_tdl_1;
-	}
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-		if (ret)
-			goto error_tdl_2;
-	}
-	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
-		if (ret)
-			goto error_compute_extra;
-	}
-	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
-		if (ret)
-			goto error_vme_pipe;
-	}
-	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_test_oa);
-		if (ret)
-			goto error_test_oa;
-	}
+	dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
+	dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa);
 
-	return 0;
+	dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa;
+	dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa);
 
-error_test_oa:
-	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
-error_vme_pipe:
-	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
-error_compute_extra:
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-error_tdl_2:
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-error_tdl_1:
-	if (get_sampler_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler_2);
-error_sampler_2:
-	if (get_sampler_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler_1);
-error_sampler_1:
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-error_rasterizer_and_pixel_backend:
-	if (get_l3_4_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_4);
-error_l3_4:
-	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
-error_l3_3:
-	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
-error_l3_2:
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-error_l3_1:
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-error_hdc_and_sf:
-	if (get_data_port_writes_coalescing_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_data_port_writes_coalescing);
-error_data_port_writes_coalescing:
-	if (get_data_port_reads_coalescing_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_data_port_reads_coalescing);
-error_data_port_reads_coalescing:
-	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
-error_compute_l3_cache:
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-error_compute_extended:
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-error_memory_writes:
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-error_memory_reads:
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-error_render_pipe_profile:
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-error_compute_basic:
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-error_render_basic:
-	return ret;
-}
+	dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa;
+	dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa);
 
-void
-i915_perf_unregister_sysfs_bdw(struct drm_i915_private *dev_priv)
-{
-	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
-	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+	dev_priv->perf.oa.test_config.sysfs_metric.name = "d6de6f55-e526-4f79-a6a6-d7315c09044e";
+	dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs;
+
+	dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr;
 
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
-	if (get_data_port_reads_coalescing_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_data_port_reads_coalescing);
-	if (get_data_port_writes_coalescing_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_data_port_writes_coalescing);
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
-	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
-	if (get_l3_4_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_4);
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-	if (get_sampler_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler_1);
-	if (get_sampler_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler_2);
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
-	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
-	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id";
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444;
+	dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id;
 }
diff --git a/drivers/gpu/drm/i915/i915_oa_bdw.h b/drivers/gpu/drm/i915/i915_oa_bdw.h
index 6363ff9f64c0..b812d16162ac 100644
--- a/drivers/gpu/drm/i915/i915_oa_bdw.h
+++ b/drivers/gpu/drm/i915/i915_oa_bdw.h
@@ -29,12 +29,6 @@
 #ifndef __I915_OA_BDW_H__
 #define __I915_OA_BDW_H__
 
-extern int i915_oa_n_builtin_metric_sets_bdw;
-
-extern int i915_oa_select_metric_set_bdw(struct drm_i915_private *dev_priv);
-
-extern int i915_perf_register_sysfs_bdw(struct drm_i915_private *dev_priv);
-
-extern void i915_perf_unregister_sysfs_bdw(struct drm_i915_private *dev_priv);
+extern void i915_perf_load_test_config_bdw(struct drm_i915_private *dev_priv);
 
 #endif
diff --git a/drivers/gpu/drm/i915/i915_oa_bxt.c b/drivers/gpu/drm/i915/i915_oa_bxt.c
index 93864d8f32dd..b69b900de0fe 100644
--- a/drivers/gpu/drm/i915/i915_oa_bxt.c
+++ b/drivers/gpu/drm/i915/i915_oa_bxt.c
@@ -31,1702 +31,6 @@
 #include "i915_drv.h"
 #include "i915_oa_bxt.h"
 
-enum metric_set_id {
-	METRIC_SET_ID_RENDER_BASIC = 1,
-	METRIC_SET_ID_COMPUTE_BASIC,
-	METRIC_SET_ID_RENDER_PIPE_PROFILE,
-	METRIC_SET_ID_MEMORY_READS,
-	METRIC_SET_ID_MEMORY_WRITES,
-	METRIC_SET_ID_COMPUTE_EXTENDED,
-	METRIC_SET_ID_COMPUTE_L3_CACHE,
-	METRIC_SET_ID_HDC_AND_SF,
-	METRIC_SET_ID_L3_1,
-	METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND,
-	METRIC_SET_ID_SAMPLER,
-	METRIC_SET_ID_TDL_1,
-	METRIC_SET_ID_TDL_2,
-	METRIC_SET_ID_COMPUTE_EXTRA,
-	METRIC_SET_ID_TEST_OA,
-};
-
-int i915_oa_n_builtin_metric_sets_bxt = 15;
-
-static const struct i915_oa_reg b_counter_config_render_basic[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2740), 0x00000000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_render_basic[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_render_basic_0_sku_gte_0x03[] = {
-	{ _MMIO(0x9888), 0x166c00f0 },
-	{ _MMIO(0x9888), 0x12120280 },
-	{ _MMIO(0x9888), 0x12320280 },
-	{ _MMIO(0x9888), 0x11930317 },
-	{ _MMIO(0x9888), 0x159303df },
-	{ _MMIO(0x9888), 0x3f900c00 },
-	{ _MMIO(0x9888), 0x419000a0 },
-	{ _MMIO(0x9888), 0x002d1000 },
-	{ _MMIO(0x9888), 0x062d4000 },
-	{ _MMIO(0x9888), 0x082d5000 },
-	{ _MMIO(0x9888), 0x0a2d1000 },
-	{ _MMIO(0x9888), 0x0c2e0800 },
-	{ _MMIO(0x9888), 0x0e2e5900 },
-	{ _MMIO(0x9888), 0x0a4c8000 },
-	{ _MMIO(0x9888), 0x0c4c8000 },
-	{ _MMIO(0x9888), 0x0e4c4000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084e8000 },
-	{ _MMIO(0x9888), 0x0a4e2000 },
-	{ _MMIO(0x9888), 0x1c4f0010 },
-	{ _MMIO(0x9888), 0x0a6c0053 },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1a0fcc00 },
-	{ _MMIO(0x9888), 0x1c0f0002 },
-	{ _MMIO(0x9888), 0x1c2c0040 },
-	{ _MMIO(0x9888), 0x00101000 },
-	{ _MMIO(0x9888), 0x04101000 },
-	{ _MMIO(0x9888), 0x00114000 },
-	{ _MMIO(0x9888), 0x08114000 },
-	{ _MMIO(0x9888), 0x00120020 },
-	{ _MMIO(0x9888), 0x08120021 },
-	{ _MMIO(0x9888), 0x00141000 },
-	{ _MMIO(0x9888), 0x08141000 },
-	{ _MMIO(0x9888), 0x02308000 },
-	{ _MMIO(0x9888), 0x04302000 },
-	{ _MMIO(0x9888), 0x06318000 },
-	{ _MMIO(0x9888), 0x08318000 },
-	{ _MMIO(0x9888), 0x06320800 },
-	{ _MMIO(0x9888), 0x08320840 },
-	{ _MMIO(0x9888), 0x00320000 },
-	{ _MMIO(0x9888), 0x06344000 },
-	{ _MMIO(0x9888), 0x08344000 },
-	{ _MMIO(0x9888), 0x0d931831 },
-	{ _MMIO(0x9888), 0x0f939f3f },
-	{ _MMIO(0x9888), 0x01939e80 },
-	{ _MMIO(0x9888), 0x039303bc },
-	{ _MMIO(0x9888), 0x0593000e },
-	{ _MMIO(0x9888), 0x1993002a },
-	{ _MMIO(0x9888), 0x07930000 },
-	{ _MMIO(0x9888), 0x09930000 },
-	{ _MMIO(0x9888), 0x1d900177 },
-	{ _MMIO(0x9888), 0x1f900187 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x13904000 },
-	{ _MMIO(0x9888), 0x21904000 },
-	{ _MMIO(0x9888), 0x23904000 },
-	{ _MMIO(0x9888), 0x25904000 },
-	{ _MMIO(0x9888), 0x27904000 },
-	{ _MMIO(0x9888), 0x2b904000 },
-	{ _MMIO(0x9888), 0x2d904000 },
-	{ _MMIO(0x9888), 0x2f904000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x15904000 },
-	{ _MMIO(0x9888), 0x17904000 },
-	{ _MMIO(0x9888), 0x19904000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x53901110 },
-	{ _MMIO(0x9888), 0x43900423 },
-	{ _MMIO(0x9888), 0x55900111 },
-	{ _MMIO(0x9888), 0x47900c02 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900020 },
-	{ _MMIO(0x9888), 0x59901111 },
-	{ _MMIO(0x9888), 0x4b900421 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4d900001 },
-	{ _MMIO(0x9888), 0x45900821 },
-};
-
-static int
-get_render_basic_mux_config(struct drm_i915_private *dev_priv,
-			    const struct i915_oa_reg **regs,
-			    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	if (dev_priv->drm.pdev->revision >= 0x03) {
-		regs[n] = mux_config_render_basic_0_sku_gte_0x03;
-		lens[n] = ARRAY_SIZE(mux_config_render_basic_0_sku_gte_0x03);
-		n++;
-	}
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_basic[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2740), 0x00000000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_basic[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00778008 },
-	{ _MMIO(0xe45c), 0x00088078 },
-	{ _MMIO(0xe55c), 0x00808708 },
-	{ _MMIO(0xe65c), 0x00a08908 },
-};
-
-static const struct i915_oa_reg mux_config_compute_basic[] = {
-	{ _MMIO(0x9888), 0x104f00e0 },
-	{ _MMIO(0x9888), 0x124f1c00 },
-	{ _MMIO(0x9888), 0x39900340 },
-	{ _MMIO(0x9888), 0x3f900c00 },
-	{ _MMIO(0x9888), 0x41900000 },
-	{ _MMIO(0x9888), 0x002d5000 },
-	{ _MMIO(0x9888), 0x062d4000 },
-	{ _MMIO(0x9888), 0x082d4000 },
-	{ _MMIO(0x9888), 0x0a2d1000 },
-	{ _MMIO(0x9888), 0x0c2d5000 },
-	{ _MMIO(0x9888), 0x0e2d4000 },
-	{ _MMIO(0x9888), 0x0c2e1400 },
-	{ _MMIO(0x9888), 0x0e2e5100 },
-	{ _MMIO(0x9888), 0x102e0114 },
-	{ _MMIO(0x9888), 0x044cc000 },
-	{ _MMIO(0x9888), 0x0a4c8000 },
-	{ _MMIO(0x9888), 0x0c4c8000 },
-	{ _MMIO(0x9888), 0x0e4c4000 },
-	{ _MMIO(0x9888), 0x104c8000 },
-	{ _MMIO(0x9888), 0x124c8000 },
-	{ _MMIO(0x9888), 0x164c2000 },
-	{ _MMIO(0x9888), 0x004ea000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084e8000 },
-	{ _MMIO(0x9888), 0x0a4e2000 },
-	{ _MMIO(0x9888), 0x0c4ea000 },
-	{ _MMIO(0x9888), 0x0e4e8000 },
-	{ _MMIO(0x9888), 0x004f6b42 },
-	{ _MMIO(0x9888), 0x064f6200 },
-	{ _MMIO(0x9888), 0x084f4100 },
-	{ _MMIO(0x9888), 0x0a4f0061 },
-	{ _MMIO(0x9888), 0x0c4f6c4c },
-	{ _MMIO(0x9888), 0x0e4f4b00 },
-	{ _MMIO(0x9888), 0x1a4f0000 },
-	{ _MMIO(0x9888), 0x1c4f0000 },
-	{ _MMIO(0x9888), 0x180f5000 },
-	{ _MMIO(0x9888), 0x1a0f8800 },
-	{ _MMIO(0x9888), 0x1c0f08a2 },
-	{ _MMIO(0x9888), 0x182c4000 },
-	{ _MMIO(0x9888), 0x1c2c1451 },
-	{ _MMIO(0x9888), 0x1e2c0001 },
-	{ _MMIO(0x9888), 0x1a2c0010 },
-	{ _MMIO(0x9888), 0x01938000 },
-	{ _MMIO(0x9888), 0x0f938000 },
-	{ _MMIO(0x9888), 0x19938a28 },
-	{ _MMIO(0x9888), 0x03938000 },
-	{ _MMIO(0x9888), 0x19900177 },
-	{ _MMIO(0x9888), 0x1b900178 },
-	{ _MMIO(0x9888), 0x1d900125 },
-	{ _MMIO(0x9888), 0x1f900123 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x13904000 },
-	{ _MMIO(0x9888), 0x21904000 },
-	{ _MMIO(0x9888), 0x25904000 },
-	{ _MMIO(0x9888), 0x27904000 },
-	{ _MMIO(0x9888), 0x2b904000 },
-	{ _MMIO(0x9888), 0x2d904000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x15904000 },
-	{ _MMIO(0x9888), 0x53901000 },
-	{ _MMIO(0x9888), 0x43900000 },
-	{ _MMIO(0x9888), 0x55900111 },
-	{ _MMIO(0x9888), 0x47900000 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x4b900000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4d900000 },
-	{ _MMIO(0x9888), 0x45900000 },
-};
-
-static int
-get_compute_basic_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_basic;
-	lens[n] = ARRAY_SIZE(mux_config_compute_basic);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_render_pipe_profile[] = {
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007ffea },
-	{ _MMIO(0x2774), 0x00007ffc },
-	{ _MMIO(0x2778), 0x0007affa },
-	{ _MMIO(0x277c), 0x0000f5fd },
-	{ _MMIO(0x2780), 0x00079ffa },
-	{ _MMIO(0x2784), 0x0000f3fb },
-	{ _MMIO(0x2788), 0x0007bf7a },
-	{ _MMIO(0x278c), 0x0000f7e7 },
-	{ _MMIO(0x2790), 0x0007fefa },
-	{ _MMIO(0x2794), 0x0000f7cf },
-	{ _MMIO(0x2798), 0x00077ffa },
-	{ _MMIO(0x279c), 0x0000efdf },
-	{ _MMIO(0x27a0), 0x0006fffa },
-	{ _MMIO(0x27a4), 0x0000cfbf },
-	{ _MMIO(0x27a8), 0x0003fffa },
-	{ _MMIO(0x27ac), 0x00005f7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_render_pipe_profile[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00015014 },
-	{ _MMIO(0xe658), 0x00025024 },
-	{ _MMIO(0xe758), 0x00035034 },
-	{ _MMIO(0xe45c), 0x00045044 },
-	{ _MMIO(0xe55c), 0x00055054 },
-	{ _MMIO(0xe65c), 0x00065064 },
-};
-
-static const struct i915_oa_reg mux_config_render_pipe_profile[] = {
-	{ _MMIO(0x9888), 0x0c2e001f },
-	{ _MMIO(0x9888), 0x0a2f0000 },
-	{ _MMIO(0x9888), 0x10186800 },
-	{ _MMIO(0x9888), 0x11810019 },
-	{ _MMIO(0x9888), 0x15810013 },
-	{ _MMIO(0x9888), 0x13820020 },
-	{ _MMIO(0x9888), 0x11830020 },
-	{ _MMIO(0x9888), 0x17840000 },
-	{ _MMIO(0x9888), 0x11860007 },
-	{ _MMIO(0x9888), 0x21860000 },
-	{ _MMIO(0x9888), 0x178703e0 },
-	{ _MMIO(0x9888), 0x0c2d8000 },
-	{ _MMIO(0x9888), 0x042d4000 },
-	{ _MMIO(0x9888), 0x062d1000 },
-	{ _MMIO(0x9888), 0x022e5400 },
-	{ _MMIO(0x9888), 0x002e0000 },
-	{ _MMIO(0x9888), 0x0e2e0080 },
-	{ _MMIO(0x9888), 0x082f0040 },
-	{ _MMIO(0x9888), 0x002f0000 },
-	{ _MMIO(0x9888), 0x06143000 },
-	{ _MMIO(0x9888), 0x06174000 },
-	{ _MMIO(0x9888), 0x06180012 },
-	{ _MMIO(0x9888), 0x00180000 },
-	{ _MMIO(0x9888), 0x0d804000 },
-	{ _MMIO(0x9888), 0x0f804000 },
-	{ _MMIO(0x9888), 0x05804000 },
-	{ _MMIO(0x9888), 0x09810200 },
-	{ _MMIO(0x9888), 0x0b810030 },
-	{ _MMIO(0x9888), 0x03810003 },
-	{ _MMIO(0x9888), 0x21819140 },
-	{ _MMIO(0x9888), 0x23819050 },
-	{ _MMIO(0x9888), 0x25810018 },
-	{ _MMIO(0x9888), 0x0b820980 },
-	{ _MMIO(0x9888), 0x03820d80 },
-	{ _MMIO(0x9888), 0x11820000 },
-	{ _MMIO(0x9888), 0x0182c000 },
-	{ _MMIO(0x9888), 0x07828000 },
-	{ _MMIO(0x9888), 0x09824000 },
-	{ _MMIO(0x9888), 0x0f828000 },
-	{ _MMIO(0x9888), 0x0d830004 },
-	{ _MMIO(0x9888), 0x0583000c },
-	{ _MMIO(0x9888), 0x0f831000 },
-	{ _MMIO(0x9888), 0x01848072 },
-	{ _MMIO(0x9888), 0x11840000 },
-	{ _MMIO(0x9888), 0x07848000 },
-	{ _MMIO(0x9888), 0x09844000 },
-	{ _MMIO(0x9888), 0x0f848000 },
-	{ _MMIO(0x9888), 0x07860000 },
-	{ _MMIO(0x9888), 0x09860092 },
-	{ _MMIO(0x9888), 0x0f860400 },
-	{ _MMIO(0x9888), 0x01869100 },
-	{ _MMIO(0x9888), 0x0f870065 },
-	{ _MMIO(0x9888), 0x01870000 },
-	{ _MMIO(0x9888), 0x19930800 },
-	{ _MMIO(0x9888), 0x0b938000 },
-	{ _MMIO(0x9888), 0x0d938000 },
-	{ _MMIO(0x9888), 0x1b952000 },
-	{ _MMIO(0x9888), 0x1d955055 },
-	{ _MMIO(0x9888), 0x1f951455 },
-	{ _MMIO(0x9888), 0x0992a000 },
-	{ _MMIO(0x9888), 0x0f928000 },
-	{ _MMIO(0x9888), 0x1192a800 },
-	{ _MMIO(0x9888), 0x1392028a },
-	{ _MMIO(0x9888), 0x0b92a000 },
-	{ _MMIO(0x9888), 0x0d922000 },
-	{ _MMIO(0x9888), 0x13908000 },
-	{ _MMIO(0x9888), 0x21908000 },
-	{ _MMIO(0x9888), 0x23908000 },
-	{ _MMIO(0x9888), 0x25908000 },
-	{ _MMIO(0x9888), 0x27908000 },
-	{ _MMIO(0x9888), 0x29908000 },
-	{ _MMIO(0x9888), 0x2b908000 },
-	{ _MMIO(0x9888), 0x2d904000 },
-	{ _MMIO(0x9888), 0x2f908000 },
-	{ _MMIO(0x9888), 0x31908000 },
-	{ _MMIO(0x9888), 0x15908000 },
-	{ _MMIO(0x9888), 0x17908000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1d904000 },
-	{ _MMIO(0x9888), 0x1f904000 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x43900c01 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x47900000 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900863 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x4b900061 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4d900000 },
-	{ _MMIO(0x9888), 0x45900c22 },
-};
-
-static int
-get_render_pipe_profile_mux_config(struct drm_i915_private *dev_priv,
-				   const struct i915_oa_reg **regs,
-				   int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_render_pipe_profile;
-	lens[n] = ARRAY_SIZE(mux_config_render_pipe_profile);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_memory_reads[] = {
-	{ _MMIO(0x272c), 0xffffffff },
-	{ _MMIO(0x2728), 0xffffffff },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x271c), 0xffffffff },
-	{ _MMIO(0x2718), 0xffffffff },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x274c), 0x86543210 },
-	{ _MMIO(0x2748), 0x86543210 },
-	{ _MMIO(0x2744), 0x00006667 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x275c), 0x86543210 },
-	{ _MMIO(0x2758), 0x86543210 },
-	{ _MMIO(0x2754), 0x00006465 },
-	{ _MMIO(0x2750), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007f81a },
-	{ _MMIO(0x2774), 0x0000fe00 },
-	{ _MMIO(0x2778), 0x0007f82a },
-	{ _MMIO(0x277c), 0x0000fe00 },
-	{ _MMIO(0x2780), 0x0007f872 },
-	{ _MMIO(0x2784), 0x0000fe00 },
-	{ _MMIO(0x2788), 0x0007f8ba },
-	{ _MMIO(0x278c), 0x0000fe00 },
-	{ _MMIO(0x2790), 0x0007f87a },
-	{ _MMIO(0x2794), 0x0000fe00 },
-	{ _MMIO(0x2798), 0x0007f8ea },
-	{ _MMIO(0x279c), 0x0000fe00 },
-	{ _MMIO(0x27a0), 0x0007f8e2 },
-	{ _MMIO(0x27a4), 0x0000fe00 },
-	{ _MMIO(0x27a8), 0x0007f8f2 },
-	{ _MMIO(0x27ac), 0x0000fe00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_memory_reads[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00015014 },
-	{ _MMIO(0xe658), 0x00025024 },
-	{ _MMIO(0xe758), 0x00035034 },
-	{ _MMIO(0xe45c), 0x00045044 },
-	{ _MMIO(0xe55c), 0x00055054 },
-	{ _MMIO(0xe65c), 0x00065064 },
-};
-
-static const struct i915_oa_reg mux_config_memory_reads[] = {
-	{ _MMIO(0x9888), 0x19800343 },
-	{ _MMIO(0x9888), 0x39900340 },
-	{ _MMIO(0x9888), 0x3f901000 },
-	{ _MMIO(0x9888), 0x41900003 },
-	{ _MMIO(0x9888), 0x03803180 },
-	{ _MMIO(0x9888), 0x058035e2 },
-	{ _MMIO(0x9888), 0x0780006a },
-	{ _MMIO(0x9888), 0x11800000 },
-	{ _MMIO(0x9888), 0x2181a000 },
-	{ _MMIO(0x9888), 0x2381000a },
-	{ _MMIO(0x9888), 0x1d950550 },
-	{ _MMIO(0x9888), 0x0b928000 },
-	{ _MMIO(0x9888), 0x0d92a000 },
-	{ _MMIO(0x9888), 0x0f922000 },
-	{ _MMIO(0x9888), 0x13900170 },
-	{ _MMIO(0x9888), 0x21900171 },
-	{ _MMIO(0x9888), 0x23900172 },
-	{ _MMIO(0x9888), 0x25900173 },
-	{ _MMIO(0x9888), 0x27900174 },
-	{ _MMIO(0x9888), 0x29900175 },
-	{ _MMIO(0x9888), 0x2b900176 },
-	{ _MMIO(0x9888), 0x2d900177 },
-	{ _MMIO(0x9888), 0x2f90017f },
-	{ _MMIO(0x9888), 0x31900125 },
-	{ _MMIO(0x9888), 0x15900123 },
-	{ _MMIO(0x9888), 0x17900121 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1d908000 },
-	{ _MMIO(0x9888), 0x1f908000 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x43901084 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x47901080 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49901084 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x4b901084 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4d900004 },
-	{ _MMIO(0x9888), 0x45900000 },
-};
-
-static int
-get_memory_reads_mux_config(struct drm_i915_private *dev_priv,
-			    const struct i915_oa_reg **regs,
-			    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_memory_reads;
-	lens[n] = ARRAY_SIZE(mux_config_memory_reads);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_memory_writes[] = {
-	{ _MMIO(0x272c), 0xffffffff },
-	{ _MMIO(0x2728), 0xffffffff },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x271c), 0xffffffff },
-	{ _MMIO(0x2718), 0xffffffff },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x274c), 0x86543210 },
-	{ _MMIO(0x2748), 0x86543210 },
-	{ _MMIO(0x2744), 0x00006667 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x275c), 0x86543210 },
-	{ _MMIO(0x2758), 0x86543210 },
-	{ _MMIO(0x2754), 0x00006465 },
-	{ _MMIO(0x2750), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007f81a },
-	{ _MMIO(0x2774), 0x0000fe00 },
-	{ _MMIO(0x2778), 0x0007f82a },
-	{ _MMIO(0x277c), 0x0000fe00 },
-	{ _MMIO(0x2780), 0x0007f822 },
-	{ _MMIO(0x2784), 0x0000fe00 },
-	{ _MMIO(0x2788), 0x0007f8ba },
-	{ _MMIO(0x278c), 0x0000fe00 },
-	{ _MMIO(0x2790), 0x0007f87a },
-	{ _MMIO(0x2794), 0x0000fe00 },
-	{ _MMIO(0x2798), 0x0007f8ea },
-	{ _MMIO(0x279c), 0x0000fe00 },
-	{ _MMIO(0x27a0), 0x0007f8e2 },
-	{ _MMIO(0x27a4), 0x0000fe00 },
-	{ _MMIO(0x27a8), 0x0007f8f2 },
-	{ _MMIO(0x27ac), 0x0000fe00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_memory_writes[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00015014 },
-	{ _MMIO(0xe658), 0x00025024 },
-	{ _MMIO(0xe758), 0x00035034 },
-	{ _MMIO(0xe45c), 0x00045044 },
-	{ _MMIO(0xe55c), 0x00055054 },
-	{ _MMIO(0xe65c), 0x00065064 },
-};
-
-static const struct i915_oa_reg mux_config_memory_writes[] = {
-	{ _MMIO(0x9888), 0x19800343 },
-	{ _MMIO(0x9888), 0x39900340 },
-	{ _MMIO(0x9888), 0x3f900000 },
-	{ _MMIO(0x9888), 0x41900080 },
-	{ _MMIO(0x9888), 0x03803180 },
-	{ _MMIO(0x9888), 0x058035e2 },
-	{ _MMIO(0x9888), 0x0780006a },
-	{ _MMIO(0x9888), 0x11800000 },
-	{ _MMIO(0x9888), 0x2181a000 },
-	{ _MMIO(0x9888), 0x2381000a },
-	{ _MMIO(0x9888), 0x1d950550 },
-	{ _MMIO(0x9888), 0x0b928000 },
-	{ _MMIO(0x9888), 0x0d92a000 },
-	{ _MMIO(0x9888), 0x0f922000 },
-	{ _MMIO(0x9888), 0x13900180 },
-	{ _MMIO(0x9888), 0x21900181 },
-	{ _MMIO(0x9888), 0x23900182 },
-	{ _MMIO(0x9888), 0x25900183 },
-	{ _MMIO(0x9888), 0x27900184 },
-	{ _MMIO(0x9888), 0x29900185 },
-	{ _MMIO(0x9888), 0x2b900186 },
-	{ _MMIO(0x9888), 0x2d900187 },
-	{ _MMIO(0x9888), 0x2f900170 },
-	{ _MMIO(0x9888), 0x31900125 },
-	{ _MMIO(0x9888), 0x15900123 },
-	{ _MMIO(0x9888), 0x17900121 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1d908000 },
-	{ _MMIO(0x9888), 0x1f908000 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x43901084 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x47901080 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49901084 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x4b901084 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4d900004 },
-	{ _MMIO(0x9888), 0x45900000 },
-};
-
-static int
-get_memory_writes_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_memory_writes;
-	lens[n] = ARRAY_SIZE(mux_config_memory_writes);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_extended[] = {
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007fc2a },
-	{ _MMIO(0x2774), 0x0000bf00 },
-	{ _MMIO(0x2778), 0x0007fc6a },
-	{ _MMIO(0x277c), 0x0000bf00 },
-	{ _MMIO(0x2780), 0x0007fc92 },
-	{ _MMIO(0x2784), 0x0000bf00 },
-	{ _MMIO(0x2788), 0x0007fca2 },
-	{ _MMIO(0x278c), 0x0000bf00 },
-	{ _MMIO(0x2790), 0x0007fc32 },
-	{ _MMIO(0x2794), 0x0000bf00 },
-	{ _MMIO(0x2798), 0x0007fc9a },
-	{ _MMIO(0x279c), 0x0000bf00 },
-	{ _MMIO(0x27a0), 0x0007fe6a },
-	{ _MMIO(0x27a4), 0x0000bf00 },
-	{ _MMIO(0x27a8), 0x0007fe7a },
-	{ _MMIO(0x27ac), 0x0000bf00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_extended[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00778008 },
-	{ _MMIO(0xe45c), 0x00088078 },
-	{ _MMIO(0xe55c), 0x00808708 },
-	{ _MMIO(0xe65c), 0x00a08908 },
-};
-
-static const struct i915_oa_reg mux_config_compute_extended[] = {
-	{ _MMIO(0x9888), 0x104f00e0 },
-	{ _MMIO(0x9888), 0x141c0160 },
-	{ _MMIO(0x9888), 0x161c0015 },
-	{ _MMIO(0x9888), 0x181c0120 },
-	{ _MMIO(0x9888), 0x002d5000 },
-	{ _MMIO(0x9888), 0x062d4000 },
-	{ _MMIO(0x9888), 0x082d5000 },
-	{ _MMIO(0x9888), 0x0a2d5000 },
-	{ _MMIO(0x9888), 0x0c2d5000 },
-	{ _MMIO(0x9888), 0x0e2d5000 },
-	{ _MMIO(0x9888), 0x022d5000 },
-	{ _MMIO(0x9888), 0x042d5000 },
-	{ _MMIO(0x9888), 0x0c2e5400 },
-	{ _MMIO(0x9888), 0x0e2e5515 },
-	{ _MMIO(0x9888), 0x102e0155 },
-	{ _MMIO(0x9888), 0x044cc000 },
-	{ _MMIO(0x9888), 0x0a4c8000 },
-	{ _MMIO(0x9888), 0x0c4cc000 },
-	{ _MMIO(0x9888), 0x0e4cc000 },
-	{ _MMIO(0x9888), 0x104c8000 },
-	{ _MMIO(0x9888), 0x124c8000 },
-	{ _MMIO(0x9888), 0x144c8000 },
-	{ _MMIO(0x9888), 0x164c2000 },
-	{ _MMIO(0x9888), 0x064cc000 },
-	{ _MMIO(0x9888), 0x084cc000 },
-	{ _MMIO(0x9888), 0x004ea000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084ea000 },
-	{ _MMIO(0x9888), 0x0a4ea000 },
-	{ _MMIO(0x9888), 0x0c4ea000 },
-	{ _MMIO(0x9888), 0x0e4ea000 },
-	{ _MMIO(0x9888), 0x024ea000 },
-	{ _MMIO(0x9888), 0x044ea000 },
-	{ _MMIO(0x9888), 0x0e4f4b41 },
-	{ _MMIO(0x9888), 0x004f4200 },
-	{ _MMIO(0x9888), 0x024f404c },
-	{ _MMIO(0x9888), 0x1c4f0000 },
-	{ _MMIO(0x9888), 0x1a4f0000 },
-	{ _MMIO(0x9888), 0x001b4000 },
-	{ _MMIO(0x9888), 0x061b8000 },
-	{ _MMIO(0x9888), 0x081bc000 },
-	{ _MMIO(0x9888), 0x0a1bc000 },
-	{ _MMIO(0x9888), 0x0c1bc000 },
-	{ _MMIO(0x9888), 0x041bc000 },
-	{ _MMIO(0x9888), 0x001c0031 },
-	{ _MMIO(0x9888), 0x061c1900 },
-	{ _MMIO(0x9888), 0x081c1a33 },
-	{ _MMIO(0x9888), 0x0a1c1b35 },
-	{ _MMIO(0x9888), 0x0c1c3337 },
-	{ _MMIO(0x9888), 0x041c31c7 },
-	{ _MMIO(0x9888), 0x180f5000 },
-	{ _MMIO(0x9888), 0x1a0fa8aa },
-	{ _MMIO(0x9888), 0x1c0f0aaa },
-	{ _MMIO(0x9888), 0x182c8000 },
-	{ _MMIO(0x9888), 0x1c2c6aaa },
-	{ _MMIO(0x9888), 0x1e2c0001 },
-	{ _MMIO(0x9888), 0x1a2c2950 },
-	{ _MMIO(0x9888), 0x01938000 },
-	{ _MMIO(0x9888), 0x0f938000 },
-	{ _MMIO(0x9888), 0x1993aaaa },
-	{ _MMIO(0x9888), 0x03938000 },
-	{ _MMIO(0x9888), 0x05938000 },
-	{ _MMIO(0x9888), 0x07938000 },
-	{ _MMIO(0x9888), 0x09938000 },
-	{ _MMIO(0x9888), 0x0b938000 },
-	{ _MMIO(0x9888), 0x13904000 },
-	{ _MMIO(0x9888), 0x21904000 },
-	{ _MMIO(0x9888), 0x23904000 },
-	{ _MMIO(0x9888), 0x25904000 },
-	{ _MMIO(0x9888), 0x27904000 },
-	{ _MMIO(0x9888), 0x29904000 },
-	{ _MMIO(0x9888), 0x2b904000 },
-	{ _MMIO(0x9888), 0x2d904000 },
-	{ _MMIO(0x9888), 0x2f904000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x15904000 },
-	{ _MMIO(0x9888), 0x17904000 },
-	{ _MMIO(0x9888), 0x19904000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x1d904000 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x43900420 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x47900000 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x4b900400 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4d900001 },
-	{ _MMIO(0x9888), 0x45900001 },
-};
-
-static int
-get_compute_extended_mux_config(struct drm_i915_private *dev_priv,
-				const struct i915_oa_reg **regs,
-				int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_extended;
-	lens[n] = ARRAY_SIZE(mux_config_compute_extended);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_l3_cache[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x30800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007fffa },
-	{ _MMIO(0x2774), 0x0000fefe },
-	{ _MMIO(0x2778), 0x0007fffa },
-	{ _MMIO(0x277c), 0x0000fefd },
-	{ _MMIO(0x2790), 0x0007fffa },
-	{ _MMIO(0x2794), 0x0000fbef },
-	{ _MMIO(0x2798), 0x0007fffa },
-	{ _MMIO(0x279c), 0x0000fbdf },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_l3_cache[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00101100 },
-	{ _MMIO(0xe45c), 0x00201200 },
-	{ _MMIO(0xe55c), 0x00301300 },
-	{ _MMIO(0xe65c), 0x00401400 },
-};
-
-static const struct i915_oa_reg mux_config_compute_l3_cache[] = {
-	{ _MMIO(0x9888), 0x166c03b0 },
-	{ _MMIO(0x9888), 0x1593001e },
-	{ _MMIO(0x9888), 0x3f900c00 },
-	{ _MMIO(0x9888), 0x41900000 },
-	{ _MMIO(0x9888), 0x002d1000 },
-	{ _MMIO(0x9888), 0x062d4000 },
-	{ _MMIO(0x9888), 0x082d5000 },
-	{ _MMIO(0x9888), 0x0e2d5000 },
-	{ _MMIO(0x9888), 0x0c2e0400 },
-	{ _MMIO(0x9888), 0x0e2e1500 },
-	{ _MMIO(0x9888), 0x102e0140 },
-	{ _MMIO(0x9888), 0x044c4000 },
-	{ _MMIO(0x9888), 0x0a4c8000 },
-	{ _MMIO(0x9888), 0x0c4cc000 },
-	{ _MMIO(0x9888), 0x144c8000 },
-	{ _MMIO(0x9888), 0x164c2000 },
-	{ _MMIO(0x9888), 0x004e2000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084ea000 },
-	{ _MMIO(0x9888), 0x0e4ea000 },
-	{ _MMIO(0x9888), 0x1a4f4001 },
-	{ _MMIO(0x9888), 0x1c4f5005 },
-	{ _MMIO(0x9888), 0x006c0051 },
-	{ _MMIO(0x9888), 0x066c5000 },
-	{ _MMIO(0x9888), 0x086c5c5d },
-	{ _MMIO(0x9888), 0x0e6c5e5f },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x146c0000 },
-	{ _MMIO(0x9888), 0x1a6c0000 },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x180f1000 },
-	{ _MMIO(0x9888), 0x1a0fa800 },
-	{ _MMIO(0x9888), 0x1c0f0a00 },
-	{ _MMIO(0x9888), 0x182c4000 },
-	{ _MMIO(0x9888), 0x1c2c4015 },
-	{ _MMIO(0x9888), 0x1e2c0001 },
-	{ _MMIO(0x9888), 0x03931980 },
-	{ _MMIO(0x9888), 0x05930032 },
-	{ _MMIO(0x9888), 0x11930000 },
-	{ _MMIO(0x9888), 0x01938000 },
-	{ _MMIO(0x9888), 0x0f938000 },
-	{ _MMIO(0x9888), 0x1993a00a },
-	{ _MMIO(0x9888), 0x07930000 },
-	{ _MMIO(0x9888), 0x09930000 },
-	{ _MMIO(0x9888), 0x1d900177 },
-	{ _MMIO(0x9888), 0x1f900178 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x13904000 },
-	{ _MMIO(0x9888), 0x21904000 },
-	{ _MMIO(0x9888), 0x23904000 },
-	{ _MMIO(0x9888), 0x25904000 },
-	{ _MMIO(0x9888), 0x2f904000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x19904000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x53901000 },
-	{ _MMIO(0x9888), 0x43900000 },
-	{ _MMIO(0x9888), 0x55900111 },
-	{ _MMIO(0x9888), 0x47900001 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x4b900000 },
-	{ _MMIO(0x9888), 0x4d900000 },
-	{ _MMIO(0x9888), 0x45900400 },
-};
-
-static int
-get_compute_l3_cache_mux_config(struct drm_i915_private *dev_priv,
-				const struct i915_oa_reg **regs,
-				int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_l3_cache;
-	lens[n] = ARRAY_SIZE(mux_config_compute_l3_cache);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_hdc_and_sf[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x10800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x0000fdff },
-};
-
-static const struct i915_oa_reg flex_eu_config_hdc_and_sf[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_hdc_and_sf[] = {
-	{ _MMIO(0x9888), 0x104f0232 },
-	{ _MMIO(0x9888), 0x124f4640 },
-	{ _MMIO(0x9888), 0x11834400 },
-	{ _MMIO(0x9888), 0x022d4000 },
-	{ _MMIO(0x9888), 0x042d5000 },
-	{ _MMIO(0x9888), 0x062d1000 },
-	{ _MMIO(0x9888), 0x0e2e0055 },
-	{ _MMIO(0x9888), 0x064c8000 },
-	{ _MMIO(0x9888), 0x084cc000 },
-	{ _MMIO(0x9888), 0x0a4c4000 },
-	{ _MMIO(0x9888), 0x024e8000 },
-	{ _MMIO(0x9888), 0x044ea000 },
-	{ _MMIO(0x9888), 0x064e2000 },
-	{ _MMIO(0x9888), 0x024f6100 },
-	{ _MMIO(0x9888), 0x044f416b },
-	{ _MMIO(0x9888), 0x064f004b },
-	{ _MMIO(0x9888), 0x1a4f0000 },
-	{ _MMIO(0x9888), 0x1a0f02a8 },
-	{ _MMIO(0x9888), 0x1a2c5500 },
-	{ _MMIO(0x9888), 0x0f808000 },
-	{ _MMIO(0x9888), 0x25810020 },
-	{ _MMIO(0x9888), 0x0f8305c0 },
-	{ _MMIO(0x9888), 0x07938000 },
-	{ _MMIO(0x9888), 0x09938000 },
-	{ _MMIO(0x9888), 0x0b938000 },
-	{ _MMIO(0x9888), 0x0d938000 },
-	{ _MMIO(0x9888), 0x1f951000 },
-	{ _MMIO(0x9888), 0x13920200 },
-	{ _MMIO(0x9888), 0x31908000 },
-	{ _MMIO(0x9888), 0x19904000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x1d904000 },
-	{ _MMIO(0x9888), 0x1f904000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x4d900003 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900000 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x47900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_hdc_and_sf_mux_config(struct drm_i915_private *dev_priv,
-			  const struct i915_oa_reg **regs,
-			  int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_hdc_and_sf;
-	lens[n] = ARRAY_SIZE(mux_config_hdc_and_sf);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_l3_1[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2770), 0x00100070 },
-	{ _MMIO(0x2774), 0x0000fff1 },
-	{ _MMIO(0x2778), 0x00014002 },
-	{ _MMIO(0x277c), 0x0000c3ff },
-	{ _MMIO(0x2780), 0x00010002 },
-	{ _MMIO(0x2784), 0x0000c7ff },
-	{ _MMIO(0x2788), 0x00004002 },
-	{ _MMIO(0x278c), 0x0000d3ff },
-	{ _MMIO(0x2790), 0x00100700 },
-	{ _MMIO(0x2794), 0x0000ff1f },
-	{ _MMIO(0x2798), 0x00001402 },
-	{ _MMIO(0x279c), 0x0000fc3f },
-	{ _MMIO(0x27a0), 0x00001002 },
-	{ _MMIO(0x27a4), 0x0000fc7f },
-	{ _MMIO(0x27a8), 0x00000402 },
-	{ _MMIO(0x27ac), 0x0000fd3f },
-};
-
-static const struct i915_oa_reg flex_eu_config_l3_1[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_l3_1_0_sku_gte_0x03[] = {
-	{ _MMIO(0x9888), 0x12643400 },
-	{ _MMIO(0x9888), 0x12653400 },
-	{ _MMIO(0x9888), 0x106c6800 },
-	{ _MMIO(0x9888), 0x126c001e },
-	{ _MMIO(0x9888), 0x166c0010 },
-	{ _MMIO(0x9888), 0x0c2d5000 },
-	{ _MMIO(0x9888), 0x0e2d5000 },
-	{ _MMIO(0x9888), 0x002d4000 },
-	{ _MMIO(0x9888), 0x022d5000 },
-	{ _MMIO(0x9888), 0x042d5000 },
-	{ _MMIO(0x9888), 0x062d1000 },
-	{ _MMIO(0x9888), 0x102e0154 },
-	{ _MMIO(0x9888), 0x0c2e5000 },
-	{ _MMIO(0x9888), 0x0e2e0055 },
-	{ _MMIO(0x9888), 0x104c8000 },
-	{ _MMIO(0x9888), 0x124c8000 },
-	{ _MMIO(0x9888), 0x144c8000 },
-	{ _MMIO(0x9888), 0x164c2000 },
-	{ _MMIO(0x9888), 0x044c8000 },
-	{ _MMIO(0x9888), 0x064cc000 },
-	{ _MMIO(0x9888), 0x084cc000 },
-	{ _MMIO(0x9888), 0x0a4c4000 },
-	{ _MMIO(0x9888), 0x0c4ea000 },
-	{ _MMIO(0x9888), 0x0e4ea000 },
-	{ _MMIO(0x9888), 0x004e8000 },
-	{ _MMIO(0x9888), 0x024ea000 },
-	{ _MMIO(0x9888), 0x044ea000 },
-	{ _MMIO(0x9888), 0x064e2000 },
-	{ _MMIO(0x9888), 0x1c4f5500 },
-	{ _MMIO(0x9888), 0x1a4f1554 },
-	{ _MMIO(0x9888), 0x0a640024 },
-	{ _MMIO(0x9888), 0x10640000 },
-	{ _MMIO(0x9888), 0x04640000 },
-	{ _MMIO(0x9888), 0x0c650024 },
-	{ _MMIO(0x9888), 0x10650000 },
-	{ _MMIO(0x9888), 0x06650000 },
-	{ _MMIO(0x9888), 0x0c6c5327 },
-	{ _MMIO(0x9888), 0x0e6c5425 },
-	{ _MMIO(0x9888), 0x006c2a00 },
-	{ _MMIO(0x9888), 0x026c285b },
-	{ _MMIO(0x9888), 0x046c005c },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1a6c0900 },
-	{ _MMIO(0x9888), 0x1c0f0aa0 },
-	{ _MMIO(0x9888), 0x180f4000 },
-	{ _MMIO(0x9888), 0x1a0f02aa },
-	{ _MMIO(0x9888), 0x1c2c5400 },
-	{ _MMIO(0x9888), 0x1e2c0001 },
-	{ _MMIO(0x9888), 0x1a2c5550 },
-	{ _MMIO(0x9888), 0x1993aa00 },
-	{ _MMIO(0x9888), 0x03938000 },
-	{ _MMIO(0x9888), 0x05938000 },
-	{ _MMIO(0x9888), 0x07938000 },
-	{ _MMIO(0x9888), 0x09938000 },
-	{ _MMIO(0x9888), 0x0b938000 },
-	{ _MMIO(0x9888), 0x0d938000 },
-	{ _MMIO(0x9888), 0x2b904000 },
-	{ _MMIO(0x9888), 0x2d904000 },
-	{ _MMIO(0x9888), 0x2f904000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x15904000 },
-	{ _MMIO(0x9888), 0x17904000 },
-	{ _MMIO(0x9888), 0x19904000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x1d904000 },
-	{ _MMIO(0x9888), 0x1f904000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x4b900421 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4d900001 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x43900420 },
-	{ _MMIO(0x9888), 0x45900021 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x47900000 },
-};
-
-static const struct i915_oa_reg mux_config_l3_1_0_sku_lt_0x03[] = {
-	{ _MMIO(0x9888), 0x14640340 },
-	{ _MMIO(0x9888), 0x14650340 },
-	{ _MMIO(0x9888), 0x106c6800 },
-	{ _MMIO(0x9888), 0x126c001e },
-	{ _MMIO(0x9888), 0x166c0010 },
-	{ _MMIO(0x9888), 0x0c2d5000 },
-	{ _MMIO(0x9888), 0x0e2d5000 },
-	{ _MMIO(0x9888), 0x002d4000 },
-	{ _MMIO(0x9888), 0x022d5000 },
-	{ _MMIO(0x9888), 0x042d5000 },
-	{ _MMIO(0x9888), 0x062d1000 },
-	{ _MMIO(0x9888), 0x102e0154 },
-	{ _MMIO(0x9888), 0x0c2e5000 },
-	{ _MMIO(0x9888), 0x0e2e0055 },
-	{ _MMIO(0x9888), 0x104c8000 },
-	{ _MMIO(0x9888), 0x124c8000 },
-	{ _MMIO(0x9888), 0x144c8000 },
-	{ _MMIO(0x9888), 0x164c2000 },
-	{ _MMIO(0x9888), 0x044c8000 },
-	{ _MMIO(0x9888), 0x064cc000 },
-	{ _MMIO(0x9888), 0x084cc000 },
-	{ _MMIO(0x9888), 0x0a4c4000 },
-	{ _MMIO(0x9888), 0x0c4ea000 },
-	{ _MMIO(0x9888), 0x0e4ea000 },
-	{ _MMIO(0x9888), 0x004e8000 },
-	{ _MMIO(0x9888), 0x024ea000 },
-	{ _MMIO(0x9888), 0x044ea000 },
-	{ _MMIO(0x9888), 0x064e2000 },
-	{ _MMIO(0x9888), 0x1c4f5500 },
-	{ _MMIO(0x9888), 0x1a4f1554 },
-	{ _MMIO(0x9888), 0x04642400 },
-	{ _MMIO(0x9888), 0x22640000 },
-	{ _MMIO(0x9888), 0x1a640000 },
-	{ _MMIO(0x9888), 0x06650024 },
-	{ _MMIO(0x9888), 0x22650000 },
-	{ _MMIO(0x9888), 0x1c650000 },
-	{ _MMIO(0x9888), 0x0c6c5327 },
-	{ _MMIO(0x9888), 0x0e6c5425 },
-	{ _MMIO(0x9888), 0x006c2a00 },
-	{ _MMIO(0x9888), 0x026c285b },
-	{ _MMIO(0x9888), 0x046c005c },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1a6c0900 },
-	{ _MMIO(0x9888), 0x1c0f0aa0 },
-	{ _MMIO(0x9888), 0x180f4000 },
-	{ _MMIO(0x9888), 0x1a0f02aa },
-	{ _MMIO(0x9888), 0x1c2c5400 },
-	{ _MMIO(0x9888), 0x1e2c0001 },
-	{ _MMIO(0x9888), 0x1a2c5550 },
-	{ _MMIO(0x9888), 0x1993aa00 },
-	{ _MMIO(0x9888), 0x03938000 },
-	{ _MMIO(0x9888), 0x05938000 },
-	{ _MMIO(0x9888), 0x07938000 },
-	{ _MMIO(0x9888), 0x09938000 },
-	{ _MMIO(0x9888), 0x0b938000 },
-	{ _MMIO(0x9888), 0x0d938000 },
-	{ _MMIO(0x9888), 0x2b904000 },
-	{ _MMIO(0x9888), 0x2d904000 },
-	{ _MMIO(0x9888), 0x2f904000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x15904000 },
-	{ _MMIO(0x9888), 0x17904000 },
-	{ _MMIO(0x9888), 0x19904000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x1d904000 },
-	{ _MMIO(0x9888), 0x1f904000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x4b900421 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4d900001 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x43900420 },
-	{ _MMIO(0x9888), 0x45900021 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x47900000 },
-};
-
-static int
-get_l3_1_mux_config(struct drm_i915_private *dev_priv,
-		    const struct i915_oa_reg **regs,
-		    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 2);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 2);
-
-	if (dev_priv->drm.pdev->revision >= 0x03) {
-		regs[n] = mux_config_l3_1_0_sku_gte_0x03;
-		lens[n] = ARRAY_SIZE(mux_config_l3_1_0_sku_gte_0x03);
-		n++;
-	}
-	if (dev_priv->drm.pdev->revision < 0x03) {
-		regs[n] = mux_config_l3_1_0_sku_lt_0x03;
-		lens[n] = ARRAY_SIZE(mux_config_l3_1_0_sku_lt_0x03);
-		n++;
-	}
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x30800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x0000efff },
-	{ _MMIO(0x2778), 0x00006000 },
-	{ _MMIO(0x277c), 0x0000f3ff },
-};
-
-static const struct i915_oa_reg flex_eu_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0x9888), 0x102d7800 },
-	{ _MMIO(0x9888), 0x122d79e0 },
-	{ _MMIO(0x9888), 0x0c2f0004 },
-	{ _MMIO(0x9888), 0x100e3800 },
-	{ _MMIO(0x9888), 0x180f0005 },
-	{ _MMIO(0x9888), 0x002d0940 },
-	{ _MMIO(0x9888), 0x022d802f },
-	{ _MMIO(0x9888), 0x042d4013 },
-	{ _MMIO(0x9888), 0x062d1000 },
-	{ _MMIO(0x9888), 0x0e2e0050 },
-	{ _MMIO(0x9888), 0x022f0010 },
-	{ _MMIO(0x9888), 0x002f0000 },
-	{ _MMIO(0x9888), 0x084c8000 },
-	{ _MMIO(0x9888), 0x0a4c4000 },
-	{ _MMIO(0x9888), 0x044e8000 },
-	{ _MMIO(0x9888), 0x064e2000 },
-	{ _MMIO(0x9888), 0x040e0480 },
-	{ _MMIO(0x9888), 0x000e0000 },
-	{ _MMIO(0x9888), 0x060f0027 },
-	{ _MMIO(0x9888), 0x100f0000 },
-	{ _MMIO(0x9888), 0x1a0f0040 },
-	{ _MMIO(0x9888), 0x03938000 },
-	{ _MMIO(0x9888), 0x05938000 },
-	{ _MMIO(0x9888), 0x07938000 },
-	{ _MMIO(0x9888), 0x09938000 },
-	{ _MMIO(0x9888), 0x0b938000 },
-	{ _MMIO(0x9888), 0x0d938000 },
-	{ _MMIO(0x9888), 0x15904000 },
-	{ _MMIO(0x9888), 0x17904000 },
-	{ _MMIO(0x9888), 0x19904000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x1d904000 },
-	{ _MMIO(0x9888), 0x1f904000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x439014a0 },
-	{ _MMIO(0x9888), 0x459000a4 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x47900001 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_rasterizer_and_pixel_backend_mux_config(struct drm_i915_private *dev_priv,
-					    const struct i915_oa_reg **regs,
-					    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_rasterizer_and_pixel_backend;
-	lens[n] = ARRAY_SIZE(mux_config_rasterizer_and_pixel_backend);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_sampler[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x70800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x0000c000 },
-	{ _MMIO(0x2774), 0x0000e7ff },
-	{ _MMIO(0x2778), 0x00003000 },
-	{ _MMIO(0x277c), 0x0000f9ff },
-	{ _MMIO(0x2780), 0x00000c00 },
-	{ _MMIO(0x2784), 0x0000fe7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_sampler[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_sampler[] = {
-	{ _MMIO(0x9888), 0x121300a0 },
-	{ _MMIO(0x9888), 0x141600ab },
-	{ _MMIO(0x9888), 0x123300a0 },
-	{ _MMIO(0x9888), 0x143600ab },
-	{ _MMIO(0x9888), 0x125300a0 },
-	{ _MMIO(0x9888), 0x145600ab },
-	{ _MMIO(0x9888), 0x0c2d4000 },
-	{ _MMIO(0x9888), 0x0e2d5000 },
-	{ _MMIO(0x9888), 0x002d4000 },
-	{ _MMIO(0x9888), 0x022d5000 },
-	{ _MMIO(0x9888), 0x042d5000 },
-	{ _MMIO(0x9888), 0x062d1000 },
-	{ _MMIO(0x9888), 0x102e01a0 },
-	{ _MMIO(0x9888), 0x0c2e5000 },
-	{ _MMIO(0x9888), 0x0e2e0065 },
-	{ _MMIO(0x9888), 0x164c2000 },
-	{ _MMIO(0x9888), 0x044c8000 },
-	{ _MMIO(0x9888), 0x064cc000 },
-	{ _MMIO(0x9888), 0x084c4000 },
-	{ _MMIO(0x9888), 0x0a4c4000 },
-	{ _MMIO(0x9888), 0x0e4e8000 },
-	{ _MMIO(0x9888), 0x004e8000 },
-	{ _MMIO(0x9888), 0x024ea000 },
-	{ _MMIO(0x9888), 0x044e2000 },
-	{ _MMIO(0x9888), 0x064e2000 },
-	{ _MMIO(0x9888), 0x1c0f0800 },
-	{ _MMIO(0x9888), 0x180f4000 },
-	{ _MMIO(0x9888), 0x1a0f023f },
-	{ _MMIO(0x9888), 0x1e2c0003 },
-	{ _MMIO(0x9888), 0x1a2cc030 },
-	{ _MMIO(0x9888), 0x04132180 },
-	{ _MMIO(0x9888), 0x02130000 },
-	{ _MMIO(0x9888), 0x0c148000 },
-	{ _MMIO(0x9888), 0x0e142000 },
-	{ _MMIO(0x9888), 0x04148000 },
-	{ _MMIO(0x9888), 0x1e150140 },
-	{ _MMIO(0x9888), 0x1c150040 },
-	{ _MMIO(0x9888), 0x0c163000 },
-	{ _MMIO(0x9888), 0x0e160068 },
-	{ _MMIO(0x9888), 0x10160000 },
-	{ _MMIO(0x9888), 0x18160000 },
-	{ _MMIO(0x9888), 0x0a164000 },
-	{ _MMIO(0x9888), 0x04330043 },
-	{ _MMIO(0x9888), 0x02330000 },
-	{ _MMIO(0x9888), 0x0234a000 },
-	{ _MMIO(0x9888), 0x04342000 },
-	{ _MMIO(0x9888), 0x1c350015 },
-	{ _MMIO(0x9888), 0x02363460 },
-	{ _MMIO(0x9888), 0x10360000 },
-	{ _MMIO(0x9888), 0x04360000 },
-	{ _MMIO(0x9888), 0x06360000 },
-	{ _MMIO(0x9888), 0x08364000 },
-	{ _MMIO(0x9888), 0x06530043 },
-	{ _MMIO(0x9888), 0x02530000 },
-	{ _MMIO(0x9888), 0x0e548000 },
-	{ _MMIO(0x9888), 0x00548000 },
-	{ _MMIO(0x9888), 0x06542000 },
-	{ _MMIO(0x9888), 0x1e550400 },
-	{ _MMIO(0x9888), 0x1a552000 },
-	{ _MMIO(0x9888), 0x1c550100 },
-	{ _MMIO(0x9888), 0x0e563000 },
-	{ _MMIO(0x9888), 0x00563400 },
-	{ _MMIO(0x9888), 0x10560000 },
-	{ _MMIO(0x9888), 0x18560000 },
-	{ _MMIO(0x9888), 0x02560000 },
-	{ _MMIO(0x9888), 0x0c564000 },
-	{ _MMIO(0x9888), 0x1993a800 },
-	{ _MMIO(0x9888), 0x03938000 },
-	{ _MMIO(0x9888), 0x05938000 },
-	{ _MMIO(0x9888), 0x07938000 },
-	{ _MMIO(0x9888), 0x09938000 },
-	{ _MMIO(0x9888), 0x0b938000 },
-	{ _MMIO(0x9888), 0x0d938000 },
-	{ _MMIO(0x9888), 0x2d904000 },
-	{ _MMIO(0x9888), 0x2f904000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x15904000 },
-	{ _MMIO(0x9888), 0x17904000 },
-	{ _MMIO(0x9888), 0x19904000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x1d904000 },
-	{ _MMIO(0x9888), 0x1f904000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x4b9014a0 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4d900001 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x43900820 },
-	{ _MMIO(0x9888), 0x45901022 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x47900000 },
-};
-
-static int
-get_sampler_mux_config(struct drm_i915_private *dev_priv,
-		       const struct i915_oa_reg **regs,
-		       int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_sampler;
-	lens[n] = ARRAY_SIZE(mux_config_sampler);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_tdl_1[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x00007fff },
-	{ _MMIO(0x2778), 0x00000000 },
-	{ _MMIO(0x277c), 0x00009fff },
-	{ _MMIO(0x2780), 0x00000002 },
-	{ _MMIO(0x2784), 0x0000efff },
-	{ _MMIO(0x2788), 0x00000000 },
-	{ _MMIO(0x278c), 0x0000f3ff },
-	{ _MMIO(0x2790), 0x00000002 },
-	{ _MMIO(0x2794), 0x0000fdff },
-	{ _MMIO(0x2798), 0x00000000 },
-	{ _MMIO(0x279c), 0x0000fe7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_tdl_1[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_tdl_1[] = {
-	{ _MMIO(0x9888), 0x141a0000 },
-	{ _MMIO(0x9888), 0x143a0000 },
-	{ _MMIO(0x9888), 0x145a0000 },
-	{ _MMIO(0x9888), 0x0c2d4000 },
-	{ _MMIO(0x9888), 0x0e2d5000 },
-	{ _MMIO(0x9888), 0x002d4000 },
-	{ _MMIO(0x9888), 0x022d5000 },
-	{ _MMIO(0x9888), 0x042d5000 },
-	{ _MMIO(0x9888), 0x062d1000 },
-	{ _MMIO(0x9888), 0x102e0150 },
-	{ _MMIO(0x9888), 0x0c2e5000 },
-	{ _MMIO(0x9888), 0x0e2e006a },
-	{ _MMIO(0x9888), 0x124c8000 },
-	{ _MMIO(0x9888), 0x144c8000 },
-	{ _MMIO(0x9888), 0x164c2000 },
-	{ _MMIO(0x9888), 0x044c8000 },
-	{ _MMIO(0x9888), 0x064c4000 },
-	{ _MMIO(0x9888), 0x0a4c4000 },
-	{ _MMIO(0x9888), 0x0c4e8000 },
-	{ _MMIO(0x9888), 0x0e4ea000 },
-	{ _MMIO(0x9888), 0x004e8000 },
-	{ _MMIO(0x9888), 0x024e2000 },
-	{ _MMIO(0x9888), 0x064e2000 },
-	{ _MMIO(0x9888), 0x1c0f0bc0 },
-	{ _MMIO(0x9888), 0x180f4000 },
-	{ _MMIO(0x9888), 0x1a0f0302 },
-	{ _MMIO(0x9888), 0x1e2c0003 },
-	{ _MMIO(0x9888), 0x1a2c00f0 },
-	{ _MMIO(0x9888), 0x021a3080 },
-	{ _MMIO(0x9888), 0x041a31e5 },
-	{ _MMIO(0x9888), 0x02148000 },
-	{ _MMIO(0x9888), 0x0414a000 },
-	{ _MMIO(0x9888), 0x1c150054 },
-	{ _MMIO(0x9888), 0x06168000 },
-	{ _MMIO(0x9888), 0x08168000 },
-	{ _MMIO(0x9888), 0x0a168000 },
-	{ _MMIO(0x9888), 0x0c3a3280 },
-	{ _MMIO(0x9888), 0x0e3a0063 },
-	{ _MMIO(0x9888), 0x063a0061 },
-	{ _MMIO(0x9888), 0x023a0000 },
-	{ _MMIO(0x9888), 0x0c348000 },
-	{ _MMIO(0x9888), 0x0e342000 },
-	{ _MMIO(0x9888), 0x06342000 },
-	{ _MMIO(0x9888), 0x1e350140 },
-	{ _MMIO(0x9888), 0x1c350100 },
-	{ _MMIO(0x9888), 0x18360028 },
-	{ _MMIO(0x9888), 0x0c368000 },
-	{ _MMIO(0x9888), 0x0e5a3080 },
-	{ _MMIO(0x9888), 0x005a3280 },
-	{ _MMIO(0x9888), 0x025a0063 },
-	{ _MMIO(0x9888), 0x0e548000 },
-	{ _MMIO(0x9888), 0x00548000 },
-	{ _MMIO(0x9888), 0x02542000 },
-	{ _MMIO(0x9888), 0x1e550400 },
-	{ _MMIO(0x9888), 0x1a552000 },
-	{ _MMIO(0x9888), 0x1c550001 },
-	{ _MMIO(0x9888), 0x18560080 },
-	{ _MMIO(0x9888), 0x02568000 },
-	{ _MMIO(0x9888), 0x04568000 },
-	{ _MMIO(0x9888), 0x1993a800 },
-	{ _MMIO(0x9888), 0x03938000 },
-	{ _MMIO(0x9888), 0x05938000 },
-	{ _MMIO(0x9888), 0x07938000 },
-	{ _MMIO(0x9888), 0x09938000 },
-	{ _MMIO(0x9888), 0x0b938000 },
-	{ _MMIO(0x9888), 0x0d938000 },
-	{ _MMIO(0x9888), 0x2d904000 },
-	{ _MMIO(0x9888), 0x2f904000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x15904000 },
-	{ _MMIO(0x9888), 0x17904000 },
-	{ _MMIO(0x9888), 0x19904000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x1d904000 },
-	{ _MMIO(0x9888), 0x1f904000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x4b900420 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4d900000 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x43900000 },
-	{ _MMIO(0x9888), 0x45901084 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x47900001 },
-};
-
-static int
-get_tdl_1_mux_config(struct drm_i915_private *dev_priv,
-		     const struct i915_oa_reg **regs,
-		     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_tdl_1;
-	lens[n] = ARRAY_SIZE(mux_config_tdl_1);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_tdl_2[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_tdl_2[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_tdl_2[] = {
-	{ _MMIO(0x9888), 0x141a026b },
-	{ _MMIO(0x9888), 0x143a0173 },
-	{ _MMIO(0x9888), 0x145a026b },
-	{ _MMIO(0x9888), 0x002d4000 },
-	{ _MMIO(0x9888), 0x022d5000 },
-	{ _MMIO(0x9888), 0x042d5000 },
-	{ _MMIO(0x9888), 0x062d1000 },
-	{ _MMIO(0x9888), 0x0c2e5000 },
-	{ _MMIO(0x9888), 0x0e2e0069 },
-	{ _MMIO(0x9888), 0x044c8000 },
-	{ _MMIO(0x9888), 0x064cc000 },
-	{ _MMIO(0x9888), 0x0a4c4000 },
-	{ _MMIO(0x9888), 0x004e8000 },
-	{ _MMIO(0x9888), 0x024ea000 },
-	{ _MMIO(0x9888), 0x064e2000 },
-	{ _MMIO(0x9888), 0x180f6000 },
-	{ _MMIO(0x9888), 0x1a0f030a },
-	{ _MMIO(0x9888), 0x1a2c03c0 },
-	{ _MMIO(0x9888), 0x041a37e7 },
-	{ _MMIO(0x9888), 0x021a0000 },
-	{ _MMIO(0x9888), 0x0414a000 },
-	{ _MMIO(0x9888), 0x1c150050 },
-	{ _MMIO(0x9888), 0x08168000 },
-	{ _MMIO(0x9888), 0x0a168000 },
-	{ _MMIO(0x9888), 0x003a3380 },
-	{ _MMIO(0x9888), 0x063a006f },
-	{ _MMIO(0x9888), 0x023a0000 },
-	{ _MMIO(0x9888), 0x00348000 },
-	{ _MMIO(0x9888), 0x06342000 },
-	{ _MMIO(0x9888), 0x1a352000 },
-	{ _MMIO(0x9888), 0x1c350100 },
-	{ _MMIO(0x9888), 0x02368000 },
-	{ _MMIO(0x9888), 0x0c368000 },
-	{ _MMIO(0x9888), 0x025a37e7 },
-	{ _MMIO(0x9888), 0x0254a000 },
-	{ _MMIO(0x9888), 0x1c550005 },
-	{ _MMIO(0x9888), 0x04568000 },
-	{ _MMIO(0x9888), 0x06568000 },
-	{ _MMIO(0x9888), 0x03938000 },
-	{ _MMIO(0x9888), 0x05938000 },
-	{ _MMIO(0x9888), 0x07938000 },
-	{ _MMIO(0x9888), 0x09938000 },
-	{ _MMIO(0x9888), 0x0b938000 },
-	{ _MMIO(0x9888), 0x0d938000 },
-	{ _MMIO(0x9888), 0x15904000 },
-	{ _MMIO(0x9888), 0x17904000 },
-	{ _MMIO(0x9888), 0x19904000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x1d904000 },
-	{ _MMIO(0x9888), 0x1f904000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x43900020 },
-	{ _MMIO(0x9888), 0x45901080 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x47900001 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_tdl_2_mux_config(struct drm_i915_private *dev_priv,
-		     const struct i915_oa_reg **regs,
-		     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_tdl_2;
-	lens[n] = ARRAY_SIZE(mux_config_tdl_2);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_extra[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_extra[] = {
-	{ _MMIO(0xe458), 0x00001000 },
-	{ _MMIO(0xe558), 0x00003002 },
-	{ _MMIO(0xe658), 0x00005004 },
-	{ _MMIO(0xe758), 0x00011010 },
-	{ _MMIO(0xe45c), 0x00050012 },
-	{ _MMIO(0xe55c), 0x00052051 },
-	{ _MMIO(0xe65c), 0x00000008 },
-};
-
-static const struct i915_oa_reg mux_config_compute_extra[] = {
-	{ _MMIO(0x9888), 0x141a001f },
-	{ _MMIO(0x9888), 0x143a001f },
-	{ _MMIO(0x9888), 0x145a001f },
-	{ _MMIO(0x9888), 0x042d5000 },
-	{ _MMIO(0x9888), 0x062d1000 },
-	{ _MMIO(0x9888), 0x0e2e0094 },
-	{ _MMIO(0x9888), 0x084cc000 },
-	{ _MMIO(0x9888), 0x044ea000 },
-	{ _MMIO(0x9888), 0x1a0f00e0 },
-	{ _MMIO(0x9888), 0x1a2c0c00 },
-	{ _MMIO(0x9888), 0x061a0063 },
-	{ _MMIO(0x9888), 0x021a0000 },
-	{ _MMIO(0x9888), 0x06142000 },
-	{ _MMIO(0x9888), 0x1c150100 },
-	{ _MMIO(0x9888), 0x0c168000 },
-	{ _MMIO(0x9888), 0x043a3180 },
-	{ _MMIO(0x9888), 0x023a0000 },
-	{ _MMIO(0x9888), 0x04348000 },
-	{ _MMIO(0x9888), 0x1c350040 },
-	{ _MMIO(0x9888), 0x0a368000 },
-	{ _MMIO(0x9888), 0x045a0063 },
-	{ _MMIO(0x9888), 0x025a0000 },
-	{ _MMIO(0x9888), 0x04542000 },
-	{ _MMIO(0x9888), 0x1c550010 },
-	{ _MMIO(0x9888), 0x08568000 },
-	{ _MMIO(0x9888), 0x09938000 },
-	{ _MMIO(0x9888), 0x0b938000 },
-	{ _MMIO(0x9888), 0x0d938000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x1d904000 },
-	{ _MMIO(0x9888), 0x1f904000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900400 },
-	{ _MMIO(0x9888), 0x47900004 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_compute_extra_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_extra;
-	lens[n] = ARRAY_SIZE(mux_config_compute_extra);
-	n++;
-
-	return n;
-}
-
 static const struct i915_oa_reg b_counter_config_test_oa[] = {
 	{ _MMIO(0x2740), 0x00000000 },
 	{ _MMIO(0x2744), 0x00800000 },
@@ -1756,6 +60,7 @@ static const struct i915_oa_reg flex_eu_config_test_oa[] = {
 };
 
 static const struct i915_oa_reg mux_config_test_oa[] = {
+	{ _MMIO(0x9840), 0x00000080 },
 	{ _MMIO(0x9888), 0x19800000 },
 	{ _MMIO(0x9888), 0x07800063 },
 	{ _MMIO(0x9888), 0x11800000 },
@@ -1769,922 +74,35 @@ static const struct i915_oa_reg mux_config_test_oa[] = {
 	{ _MMIO(0x9888), 0x33900000 },
 };
 
-static int
-get_test_oa_mux_config(struct drm_i915_private *dev_priv,
-		       const struct i915_oa_reg **regs,
-		       int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_test_oa;
-	lens[n] = ARRAY_SIZE(mux_config_test_oa);
-	n++;
-
-	return n;
-}
-
-int i915_oa_select_metric_set_bxt(struct drm_i915_private *dev_priv)
-{
-	dev_priv->perf.oa.n_mux_configs = 0;
-	dev_priv->perf.oa.b_counter_regs = NULL;
-	dev_priv->perf.oa.b_counter_regs_len = 0;
-	dev_priv->perf.oa.flex_regs = NULL;
-	dev_priv->perf.oa.flex_regs_len = 0;
-
-	switch (dev_priv->perf.oa.metrics_set) {
-	case METRIC_SET_ID_RENDER_BASIC:
-		dev_priv->perf.oa.n_mux_configs =
-			get_render_basic_mux_config(dev_priv,
-						    dev_priv->perf.oa.mux_regs,
-						    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_BASIC\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_render_basic;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_render_basic);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_render_basic;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_render_basic);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_BASIC:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_basic_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_BASIC\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_basic;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_basic);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_basic;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_basic);
-
-		return 0;
-	case METRIC_SET_ID_RENDER_PIPE_PROFILE:
-		dev_priv->perf.oa.n_mux_configs =
-			get_render_pipe_profile_mux_config(dev_priv,
-							   dev_priv->perf.oa.mux_regs,
-							   dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_PIPE_PROFILE\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_render_pipe_profile;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_render_pipe_profile);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_render_pipe_profile;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_render_pipe_profile);
-
-		return 0;
-	case METRIC_SET_ID_MEMORY_READS:
-		dev_priv->perf.oa.n_mux_configs =
-			get_memory_reads_mux_config(dev_priv,
-						    dev_priv->perf.oa.mux_regs,
-						    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_READS\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_memory_reads;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_memory_reads);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_memory_reads;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_memory_reads);
-
-		return 0;
-	case METRIC_SET_ID_MEMORY_WRITES:
-		dev_priv->perf.oa.n_mux_configs =
-			get_memory_writes_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_WRITES\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_memory_writes;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_memory_writes);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_memory_writes;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_memory_writes);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_EXTENDED:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_extended_mux_config(dev_priv,
-							dev_priv->perf.oa.mux_regs,
-							dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTENDED\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_extended;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_extended);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_extended;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_extended);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_L3_CACHE:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_l3_cache_mux_config(dev_priv,
-							dev_priv->perf.oa.mux_regs,
-							dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_L3_CACHE\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_l3_cache;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_l3_cache);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_l3_cache;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_l3_cache);
-
-		return 0;
-	case METRIC_SET_ID_HDC_AND_SF:
-		dev_priv->perf.oa.n_mux_configs =
-			get_hdc_and_sf_mux_config(dev_priv,
-						  dev_priv->perf.oa.mux_regs,
-						  dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"HDC_AND_SF\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_hdc_and_sf;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_hdc_and_sf);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_hdc_and_sf;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_hdc_and_sf);
-
-		return 0;
-	case METRIC_SET_ID_L3_1:
-		dev_priv->perf.oa.n_mux_configs =
-			get_l3_1_mux_config(dev_priv,
-					    dev_priv->perf.oa.mux_regs,
-					    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_1\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_l3_1;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_l3_1);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_l3_1;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_l3_1);
-
-		return 0;
-	case METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND:
-		dev_priv->perf.oa.n_mux_configs =
-			get_rasterizer_and_pixel_backend_mux_config(dev_priv,
-								    dev_priv->perf.oa.mux_regs,
-								    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RASTERIZER_AND_PIXEL_BACKEND\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_rasterizer_and_pixel_backend;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_rasterizer_and_pixel_backend);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_rasterizer_and_pixel_backend;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_rasterizer_and_pixel_backend);
-
-		return 0;
-	case METRIC_SET_ID_SAMPLER:
-		dev_priv->perf.oa.n_mux_configs =
-			get_sampler_mux_config(dev_priv,
-					       dev_priv->perf.oa.mux_regs,
-					       dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"SAMPLER\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_sampler;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_sampler);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_sampler;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_sampler);
-
-		return 0;
-	case METRIC_SET_ID_TDL_1:
-		dev_priv->perf.oa.n_mux_configs =
-			get_tdl_1_mux_config(dev_priv,
-					     dev_priv->perf.oa.mux_regs,
-					     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_1\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_tdl_1;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_tdl_1);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_tdl_1;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_tdl_1);
-
-		return 0;
-	case METRIC_SET_ID_TDL_2:
-		dev_priv->perf.oa.n_mux_configs =
-			get_tdl_2_mux_config(dev_priv,
-					     dev_priv->perf.oa.mux_regs,
-					     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_2\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_tdl_2;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_tdl_2);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_tdl_2;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_tdl_2);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_EXTRA:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_extra_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTRA\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_extra;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_extra);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_extra;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_extra);
-
-		return 0;
-	case METRIC_SET_ID_TEST_OA:
-		dev_priv->perf.oa.n_mux_configs =
-			get_test_oa_mux_config(dev_priv,
-					       dev_priv->perf.oa.mux_regs,
-					       dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TEST_OA\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_test_oa;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_test_oa);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_test_oa;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_test_oa);
-
-		return 0;
-	default:
-		return -ENODEV;
-	}
-}
-
-static ssize_t
-show_render_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_BASIC);
-}
-
-static struct device_attribute dev_attr_render_basic_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_render_basic_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_render_basic[] = {
-	&dev_attr_render_basic_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_render_basic = {
-	.name = "22b9519a-e9ba-4c41-8b54-f4f8ca14fa0a",
-	.attrs =  attrs_render_basic,
-};
-
-static ssize_t
-show_compute_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_BASIC);
-}
-
-static struct device_attribute dev_attr_compute_basic_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_basic_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_basic[] = {
-	&dev_attr_compute_basic_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_basic = {
-	.name = "012d72cf-82a9-4d25-8ddf-74076fd30797",
-	.attrs =  attrs_compute_basic,
-};
-
-static ssize_t
-show_render_pipe_profile_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_PIPE_PROFILE);
-}
-
-static struct device_attribute dev_attr_render_pipe_profile_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_render_pipe_profile_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_render_pipe_profile[] = {
-	&dev_attr_render_pipe_profile_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_render_pipe_profile = {
-	.name = "ce416533-e49e-4211-80af-ec513590a914",
-	.attrs =  attrs_render_pipe_profile,
-};
-
-static ssize_t
-show_memory_reads_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_READS);
-}
-
-static struct device_attribute dev_attr_memory_reads_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_memory_reads_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_memory_reads[] = {
-	&dev_attr_memory_reads_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_memory_reads = {
-	.name = "398e2452-18d7-42d0-b241-e4d0a9148ada",
-	.attrs =  attrs_memory_reads,
-};
-
-static ssize_t
-show_memory_writes_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_WRITES);
-}
-
-static struct device_attribute dev_attr_memory_writes_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_memory_writes_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_memory_writes[] = {
-	&dev_attr_memory_writes_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_memory_writes = {
-	.name = "d324a0d6-7269-4847-a5c2-6f71ddc7fed5",
-	.attrs =  attrs_memory_writes,
-};
-
-static ssize_t
-show_compute_extended_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTENDED);
-}
-
-static struct device_attribute dev_attr_compute_extended_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_extended_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_extended[] = {
-	&dev_attr_compute_extended_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_extended = {
-	.name = "caf3596a-7bb1-4dec-b3b3-2a080d283b49",
-	.attrs =  attrs_compute_extended,
-};
-
-static ssize_t
-show_compute_l3_cache_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_L3_CACHE);
-}
-
-static struct device_attribute dev_attr_compute_l3_cache_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_l3_cache_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_l3_cache[] = {
-	&dev_attr_compute_l3_cache_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_l3_cache = {
-	.name = "49b956e2-d5b9-47e0-9d8a-cee5e8cec527",
-	.attrs =  attrs_compute_l3_cache,
-};
-
-static ssize_t
-show_hdc_and_sf_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_HDC_AND_SF);
-}
-
-static struct device_attribute dev_attr_hdc_and_sf_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_hdc_and_sf_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_hdc_and_sf[] = {
-	&dev_attr_hdc_and_sf_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_hdc_and_sf = {
-	.name = "f64ef50a-bdba-4b35-8f09-203c13d8ee5a",
-	.attrs =  attrs_hdc_and_sf,
-};
-
-static ssize_t
-show_l3_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_1);
-}
-
-static struct device_attribute dev_attr_l3_1_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_l3_1_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_l3_1[] = {
-	&dev_attr_l3_1_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_l3_1 = {
-	.name = "00ad5a41-7eab-4f7a-9103-49d411c67219",
-	.attrs =  attrs_l3_1,
-};
-
-static ssize_t
-show_rasterizer_and_pixel_backend_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND);
-}
-
-static struct device_attribute dev_attr_rasterizer_and_pixel_backend_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_rasterizer_and_pixel_backend_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_rasterizer_and_pixel_backend[] = {
-	&dev_attr_rasterizer_and_pixel_backend_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_rasterizer_and_pixel_backend = {
-	.name = "46dc44ca-491c-4cc1-a951-e7b3e62bf02b",
-	.attrs =  attrs_rasterizer_and_pixel_backend,
-};
-
-static ssize_t
-show_sampler_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_SAMPLER);
-}
-
-static struct device_attribute dev_attr_sampler_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_sampler_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_sampler[] = {
-	&dev_attr_sampler_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_sampler = {
-	.name = "8364e2a8-af63-40af-b0d5-42969a255654",
-	.attrs =  attrs_sampler,
-};
-
-static ssize_t
-show_tdl_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_1);
-}
-
-static struct device_attribute dev_attr_tdl_1_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_tdl_1_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_tdl_1[] = {
-	&dev_attr_tdl_1_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_tdl_1 = {
-	.name = "175c8092-cb25-4d1e-8dc7-b4fdd39e2d92",
-	.attrs =  attrs_tdl_1,
-};
-
-static ssize_t
-show_tdl_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_2);
-}
-
-static struct device_attribute dev_attr_tdl_2_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_tdl_2_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_tdl_2[] = {
-	&dev_attr_tdl_2_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_tdl_2 = {
-	.name = "d260f03f-b34d-4b49-a44e-436819117332",
-	.attrs =  attrs_tdl_2,
-};
-
-static ssize_t
-show_compute_extra_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTRA);
-}
-
-static struct device_attribute dev_attr_compute_extra_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_extra_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_extra[] = {
-	&dev_attr_compute_extra_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_extra = {
-	.name = "fa6ecf21-2cb8-4d0b-9308-6e4a7b4ca87a",
-	.attrs =  attrs_compute_extra,
-};
-
 static ssize_t
 show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TEST_OA);
+	return sprintf(buf, "1\n");
 }
 
-static struct device_attribute dev_attr_test_oa_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_test_oa_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_test_oa[] = {
-	&dev_attr_test_oa_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_test_oa = {
-	.name = "5ee72f5c-092f-421e-8b70-225f7c3e9612",
-	.attrs =  attrs_test_oa,
-};
-
-int
-i915_perf_register_sysfs_bxt(struct drm_i915_private *dev_priv)
+void
+i915_perf_load_test_config_bxt(struct drm_i915_private *dev_priv)
 {
-	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
-	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
-	int ret = 0;
+	strncpy(dev_priv->perf.oa.test_config.uuid,
+		"5ee72f5c-092f-421e-8b70-225f7c3e9612",
+		UUID_STRING_LEN);
+	dev_priv->perf.oa.test_config.id = 1;
 
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-		if (ret)
-			goto error_render_basic;
-	}
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-		if (ret)
-			goto error_compute_basic;
-	}
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-		if (ret)
-			goto error_render_pipe_profile;
-	}
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-		if (ret)
-			goto error_memory_reads;
-	}
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-		if (ret)
-			goto error_memory_writes;
-	}
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-		if (ret)
-			goto error_compute_extended;
-	}
-	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
-		if (ret)
-			goto error_compute_l3_cache;
-	}
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-		if (ret)
-			goto error_hdc_and_sf;
-	}
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-		if (ret)
-			goto error_l3_1;
-	}
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-		if (ret)
-			goto error_rasterizer_and_pixel_backend;
-	}
-	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_sampler);
-		if (ret)
-			goto error_sampler;
-	}
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-		if (ret)
-			goto error_tdl_1;
-	}
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-		if (ret)
-			goto error_tdl_2;
-	}
-	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
-		if (ret)
-			goto error_compute_extra;
-	}
-	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_test_oa);
-		if (ret)
-			goto error_test_oa;
-	}
+	dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
+	dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa);
 
-	return 0;
+	dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa;
+	dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa);
 
-error_test_oa:
-	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
-error_compute_extra:
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-error_tdl_2:
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-error_tdl_1:
-	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
-error_sampler:
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-error_rasterizer_and_pixel_backend:
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-error_l3_1:
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-error_hdc_and_sf:
-	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
-error_compute_l3_cache:
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-error_compute_extended:
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-error_memory_writes:
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-error_memory_reads:
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-error_render_pipe_profile:
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-error_compute_basic:
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-error_render_basic:
-	return ret;
-}
+	dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa;
+	dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa);
 
-void
-i915_perf_unregister_sysfs_bxt(struct drm_i915_private *dev_priv)
-{
-	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
-	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+	dev_priv->perf.oa.test_config.sysfs_metric.name = "5ee72f5c-092f-421e-8b70-225f7c3e9612";
+	dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs;
+
+	dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr;
 
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
-	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id";
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444;
+	dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id;
 }
diff --git a/drivers/gpu/drm/i915/i915_oa_bxt.h b/drivers/gpu/drm/i915/i915_oa_bxt.h
index 6cf7ba746e7e..690b963a2383 100644
--- a/drivers/gpu/drm/i915/i915_oa_bxt.h
+++ b/drivers/gpu/drm/i915/i915_oa_bxt.h
@@ -29,12 +29,6 @@
 #ifndef __I915_OA_BXT_H__
 #define __I915_OA_BXT_H__
 
-extern int i915_oa_n_builtin_metric_sets_bxt;
-
-extern int i915_oa_select_metric_set_bxt(struct drm_i915_private *dev_priv);
-
-extern int i915_perf_register_sysfs_bxt(struct drm_i915_private *dev_priv);
-
-extern void i915_perf_unregister_sysfs_bxt(struct drm_i915_private *dev_priv);
+extern void i915_perf_load_test_config_bxt(struct drm_i915_private *dev_priv);
 
 #endif
diff --git a/drivers/gpu/drm/i915/i915_oa_chv.c b/drivers/gpu/drm/i915/i915_oa_chv.c
index aa6bece7e75f..322a3f94cd16 100644
--- a/drivers/gpu/drm/i915/i915_oa_chv.c
+++ b/drivers/gpu/drm/i915/i915_oa_chv.c
@@ -31,1943 +31,6 @@
 #include "i915_drv.h"
 #include "i915_oa_chv.h"
 
-enum metric_set_id {
-	METRIC_SET_ID_RENDER_BASIC = 1,
-	METRIC_SET_ID_COMPUTE_BASIC,
-	METRIC_SET_ID_RENDER_PIPE_PROFILE,
-	METRIC_SET_ID_HDC_AND_SF,
-	METRIC_SET_ID_L3_1,
-	METRIC_SET_ID_L3_2,
-	METRIC_SET_ID_L3_3,
-	METRIC_SET_ID_L3_4,
-	METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND,
-	METRIC_SET_ID_SAMPLER_1,
-	METRIC_SET_ID_SAMPLER_2,
-	METRIC_SET_ID_TDL_1,
-	METRIC_SET_ID_TDL_2,
-	METRIC_SET_ID_TEST_OA,
-};
-
-int i915_oa_n_builtin_metric_sets_chv = 14;
-
-static const struct i915_oa_reg b_counter_config_render_basic[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_render_basic[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_render_basic[] = {
-	{ _MMIO(0x9888), 0x59800000 },
-	{ _MMIO(0x9888), 0x59800001 },
-	{ _MMIO(0x9888), 0x285a0006 },
-	{ _MMIO(0x9888), 0x2c110014 },
-	{ _MMIO(0x9888), 0x2e110000 },
-	{ _MMIO(0x9888), 0x2c310014 },
-	{ _MMIO(0x9888), 0x2e310000 },
-	{ _MMIO(0x9888), 0x2b8303df },
-	{ _MMIO(0x9888), 0x3580024f },
-	{ _MMIO(0x9888), 0x00580888 },
-	{ _MMIO(0x9888), 0x1e5a0015 },
-	{ _MMIO(0x9888), 0x205a0014 },
-	{ _MMIO(0x9888), 0x045a0000 },
-	{ _MMIO(0x9888), 0x025a0000 },
-	{ _MMIO(0x9888), 0x02180500 },
-	{ _MMIO(0x9888), 0x00190555 },
-	{ _MMIO(0x9888), 0x021d0500 },
-	{ _MMIO(0x9888), 0x021f0a00 },
-	{ _MMIO(0x9888), 0x00380444 },
-	{ _MMIO(0x9888), 0x02390500 },
-	{ _MMIO(0x9888), 0x003a0666 },
-	{ _MMIO(0x9888), 0x00100111 },
-	{ _MMIO(0x9888), 0x06110030 },
-	{ _MMIO(0x9888), 0x0a110031 },
-	{ _MMIO(0x9888), 0x0e110046 },
-	{ _MMIO(0x9888), 0x04110000 },
-	{ _MMIO(0x9888), 0x00110000 },
-	{ _MMIO(0x9888), 0x00130111 },
-	{ _MMIO(0x9888), 0x00300444 },
-	{ _MMIO(0x9888), 0x08310030 },
-	{ _MMIO(0x9888), 0x0c310031 },
-	{ _MMIO(0x9888), 0x10310046 },
-	{ _MMIO(0x9888), 0x04310000 },
-	{ _MMIO(0x9888), 0x00310000 },
-	{ _MMIO(0x9888), 0x00330444 },
-	{ _MMIO(0x9888), 0x038a0a00 },
-	{ _MMIO(0x9888), 0x018b0fff },
-	{ _MMIO(0x9888), 0x038b0a00 },
-	{ _MMIO(0x9888), 0x01855000 },
-	{ _MMIO(0x9888), 0x03850055 },
-	{ _MMIO(0x9888), 0x13830021 },
-	{ _MMIO(0x9888), 0x15830020 },
-	{ _MMIO(0x9888), 0x1783002f },
-	{ _MMIO(0x9888), 0x1983002e },
-	{ _MMIO(0x9888), 0x1b83002d },
-	{ _MMIO(0x9888), 0x1d83002c },
-	{ _MMIO(0x9888), 0x05830000 },
-	{ _MMIO(0x9888), 0x01840555 },
-	{ _MMIO(0x9888), 0x03840500 },
-	{ _MMIO(0x9888), 0x23800074 },
-	{ _MMIO(0x9888), 0x2580007d },
-	{ _MMIO(0x9888), 0x05800000 },
-	{ _MMIO(0x9888), 0x01805000 },
-	{ _MMIO(0x9888), 0x03800055 },
-	{ _MMIO(0x9888), 0x01865000 },
-	{ _MMIO(0x9888), 0x03860055 },
-	{ _MMIO(0x9888), 0x01875000 },
-	{ _MMIO(0x9888), 0x03870055 },
-	{ _MMIO(0x9888), 0x418000aa },
-	{ _MMIO(0x9888), 0x4380000a },
-	{ _MMIO(0x9888), 0x45800000 },
-	{ _MMIO(0x9888), 0x4780000a },
-	{ _MMIO(0x9888), 0x49800000 },
-	{ _MMIO(0x9888), 0x4b800000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x51800000 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x55800000 },
-	{ _MMIO(0x9888), 0x57800000 },
-	{ _MMIO(0x9888), 0x59800000 },
-};
-
-static int
-get_render_basic_mux_config(struct drm_i915_private *dev_priv,
-			    const struct i915_oa_reg **regs,
-			    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_render_basic;
-	lens[n] = ARRAY_SIZE(mux_config_render_basic);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_basic[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_basic[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00778008 },
-	{ _MMIO(0xe45c), 0x00088078 },
-	{ _MMIO(0xe55c), 0x00808708 },
-	{ _MMIO(0xe65c), 0x00a08908 },
-};
-
-static const struct i915_oa_reg mux_config_compute_basic[] = {
-	{ _MMIO(0x9888), 0x59800000 },
-	{ _MMIO(0x9888), 0x59800001 },
-	{ _MMIO(0x9888), 0x2e5800e0 },
-	{ _MMIO(0x9888), 0x2e3800e0 },
-	{ _MMIO(0x9888), 0x3580024f },
-	{ _MMIO(0x9888), 0x3d800140 },
-	{ _MMIO(0x9888), 0x08580042 },
-	{ _MMIO(0x9888), 0x0c580040 },
-	{ _MMIO(0x9888), 0x1058004c },
-	{ _MMIO(0x9888), 0x1458004b },
-	{ _MMIO(0x9888), 0x04580000 },
-	{ _MMIO(0x9888), 0x00580000 },
-	{ _MMIO(0x9888), 0x00195555 },
-	{ _MMIO(0x9888), 0x06380042 },
-	{ _MMIO(0x9888), 0x0a380040 },
-	{ _MMIO(0x9888), 0x0e38004c },
-	{ _MMIO(0x9888), 0x1238004b },
-	{ _MMIO(0x9888), 0x04380000 },
-	{ _MMIO(0x9888), 0x00384444 },
-	{ _MMIO(0x9888), 0x003a5555 },
-	{ _MMIO(0x9888), 0x018bffff },
-	{ _MMIO(0x9888), 0x01845555 },
-	{ _MMIO(0x9888), 0x17800074 },
-	{ _MMIO(0x9888), 0x1980007d },
-	{ _MMIO(0x9888), 0x1b80007c },
-	{ _MMIO(0x9888), 0x1d8000b6 },
-	{ _MMIO(0x9888), 0x1f8000b7 },
-	{ _MMIO(0x9888), 0x05800000 },
-	{ _MMIO(0x9888), 0x03800000 },
-	{ _MMIO(0x9888), 0x418000aa },
-	{ _MMIO(0x9888), 0x438000aa },
-	{ _MMIO(0x9888), 0x45800000 },
-	{ _MMIO(0x9888), 0x47800000 },
-	{ _MMIO(0x9888), 0x4980012a },
-	{ _MMIO(0x9888), 0x4b80012a },
-	{ _MMIO(0x9888), 0x4d80012a },
-	{ _MMIO(0x9888), 0x4f80012a },
-	{ _MMIO(0x9888), 0x518001ce },
-	{ _MMIO(0x9888), 0x538001ce },
-	{ _MMIO(0x9888), 0x5580000e },
-	{ _MMIO(0x9888), 0x59800000 },
-};
-
-static int
-get_compute_basic_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_basic;
-	lens[n] = ARRAY_SIZE(mux_config_compute_basic);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_render_pipe_profile[] = {
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007ffea },
-	{ _MMIO(0x2774), 0x00007ffc },
-	{ _MMIO(0x2778), 0x0007affa },
-	{ _MMIO(0x277c), 0x0000f5fd },
-	{ _MMIO(0x2780), 0x00079ffa },
-	{ _MMIO(0x2784), 0x0000f3fb },
-	{ _MMIO(0x2788), 0x0007bf7a },
-	{ _MMIO(0x278c), 0x0000f7e7 },
-	{ _MMIO(0x2790), 0x0007fefa },
-	{ _MMIO(0x2794), 0x0000f7cf },
-	{ _MMIO(0x2798), 0x00077ffa },
-	{ _MMIO(0x279c), 0x0000efdf },
-	{ _MMIO(0x27a0), 0x0006fffa },
-	{ _MMIO(0x27a4), 0x0000cfbf },
-	{ _MMIO(0x27a8), 0x0003fffa },
-	{ _MMIO(0x27ac), 0x00005f7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_render_pipe_profile[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00015014 },
-	{ _MMIO(0xe658), 0x00025024 },
-	{ _MMIO(0xe758), 0x00035034 },
-	{ _MMIO(0xe45c), 0x00045044 },
-	{ _MMIO(0xe55c), 0x00055054 },
-	{ _MMIO(0xe65c), 0x00065064 },
-};
-
-static const struct i915_oa_reg mux_config_render_pipe_profile[] = {
-	{ _MMIO(0x9888), 0x59800000 },
-	{ _MMIO(0x9888), 0x59800001 },
-	{ _MMIO(0x9888), 0x261e0000 },
-	{ _MMIO(0x9888), 0x281f000f },
-	{ _MMIO(0x9888), 0x2817001a },
-	{ _MMIO(0x9888), 0x2791001f },
-	{ _MMIO(0x9888), 0x27880019 },
-	{ _MMIO(0x9888), 0x2d890000 },
-	{ _MMIO(0x9888), 0x278a0007 },
-	{ _MMIO(0x9888), 0x298d001f },
-	{ _MMIO(0x9888), 0x278e0020 },
-	{ _MMIO(0x9888), 0x2b8f0012 },
-	{ _MMIO(0x9888), 0x29900000 },
-	{ _MMIO(0x9888), 0x00184000 },
-	{ _MMIO(0x9888), 0x02181000 },
-	{ _MMIO(0x9888), 0x02194000 },
-	{ _MMIO(0x9888), 0x141e0002 },
-	{ _MMIO(0x9888), 0x041e0000 },
-	{ _MMIO(0x9888), 0x001e0000 },
-	{ _MMIO(0x9888), 0x221f0015 },
-	{ _MMIO(0x9888), 0x041f0000 },
-	{ _MMIO(0x9888), 0x001f4000 },
-	{ _MMIO(0x9888), 0x021f0000 },
-	{ _MMIO(0x9888), 0x023a8000 },
-	{ _MMIO(0x9888), 0x0213c000 },
-	{ _MMIO(0x9888), 0x02164000 },
-	{ _MMIO(0x9888), 0x24170012 },
-	{ _MMIO(0x9888), 0x04170000 },
-	{ _MMIO(0x9888), 0x07910005 },
-	{ _MMIO(0x9888), 0x05910000 },
-	{ _MMIO(0x9888), 0x01911500 },
-	{ _MMIO(0x9888), 0x03910501 },
-	{ _MMIO(0x9888), 0x0d880002 },
-	{ _MMIO(0x9888), 0x1d880003 },
-	{ _MMIO(0x9888), 0x05880000 },
-	{ _MMIO(0x9888), 0x0b890032 },
-	{ _MMIO(0x9888), 0x1b890031 },
-	{ _MMIO(0x9888), 0x05890000 },
-	{ _MMIO(0x9888), 0x01890040 },
-	{ _MMIO(0x9888), 0x03890040 },
-	{ _MMIO(0x9888), 0x098a0000 },
-	{ _MMIO(0x9888), 0x198a0004 },
-	{ _MMIO(0x9888), 0x058a0000 },
-	{ _MMIO(0x9888), 0x018a8050 },
-	{ _MMIO(0x9888), 0x038a2050 },
-	{ _MMIO(0x9888), 0x018b95a9 },
-	{ _MMIO(0x9888), 0x038be5a9 },
-	{ _MMIO(0x9888), 0x018c1500 },
-	{ _MMIO(0x9888), 0x038c0501 },
-	{ _MMIO(0x9888), 0x178d0015 },
-	{ _MMIO(0x9888), 0x058d0000 },
-	{ _MMIO(0x9888), 0x138e0004 },
-	{ _MMIO(0x9888), 0x218e000c },
-	{ _MMIO(0x9888), 0x058e0000 },
-	{ _MMIO(0x9888), 0x018e0500 },
-	{ _MMIO(0x9888), 0x038e0101 },
-	{ _MMIO(0x9888), 0x0f8f0027 },
-	{ _MMIO(0x9888), 0x058f0000 },
-	{ _MMIO(0x9888), 0x018f0000 },
-	{ _MMIO(0x9888), 0x038f0001 },
-	{ _MMIO(0x9888), 0x11900013 },
-	{ _MMIO(0x9888), 0x1f900017 },
-	{ _MMIO(0x9888), 0x05900000 },
-	{ _MMIO(0x9888), 0x01900100 },
-	{ _MMIO(0x9888), 0x03900001 },
-	{ _MMIO(0x9888), 0x01845555 },
-	{ _MMIO(0x9888), 0x03845555 },
-	{ _MMIO(0x9888), 0x418000aa },
-	{ _MMIO(0x9888), 0x438000aa },
-	{ _MMIO(0x9888), 0x458000aa },
-	{ _MMIO(0x9888), 0x478000aa },
-	{ _MMIO(0x9888), 0x4980018c },
-	{ _MMIO(0x9888), 0x4b80014b },
-	{ _MMIO(0x9888), 0x4d800128 },
-	{ _MMIO(0x9888), 0x4f80012a },
-	{ _MMIO(0x9888), 0x51800187 },
-	{ _MMIO(0x9888), 0x5380014b },
-	{ _MMIO(0x9888), 0x55800149 },
-	{ _MMIO(0x9888), 0x5780010a },
-	{ _MMIO(0x9888), 0x59800000 },
-};
-
-static int
-get_render_pipe_profile_mux_config(struct drm_i915_private *dev_priv,
-				   const struct i915_oa_reg **regs,
-				   int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_render_pipe_profile;
-	lens[n] = ARRAY_SIZE(mux_config_render_pipe_profile);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_hdc_and_sf[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x10800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x0000fff7 },
-};
-
-static const struct i915_oa_reg flex_eu_config_hdc_and_sf[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_hdc_and_sf[] = {
-	{ _MMIO(0x9888), 0x105c0232 },
-	{ _MMIO(0x9888), 0x10580232 },
-	{ _MMIO(0x9888), 0x10380232 },
-	{ _MMIO(0x9888), 0x10dc0232 },
-	{ _MMIO(0x9888), 0x10d80232 },
-	{ _MMIO(0x9888), 0x10b80232 },
-	{ _MMIO(0x9888), 0x118e4400 },
-	{ _MMIO(0x9888), 0x025c6080 },
-	{ _MMIO(0x9888), 0x045c004b },
-	{ _MMIO(0x9888), 0x005c8000 },
-	{ _MMIO(0x9888), 0x00582080 },
-	{ _MMIO(0x9888), 0x0258004b },
-	{ _MMIO(0x9888), 0x025b4000 },
-	{ _MMIO(0x9888), 0x045b4000 },
-	{ _MMIO(0x9888), 0x0c1fa000 },
-	{ _MMIO(0x9888), 0x0e1f00aa },
-	{ _MMIO(0x9888), 0x04386080 },
-	{ _MMIO(0x9888), 0x0638404b },
-	{ _MMIO(0x9888), 0x02384000 },
-	{ _MMIO(0x9888), 0x08384000 },
-	{ _MMIO(0x9888), 0x0a380000 },
-	{ _MMIO(0x9888), 0x0c380000 },
-	{ _MMIO(0x9888), 0x00398000 },
-	{ _MMIO(0x9888), 0x0239a000 },
-	{ _MMIO(0x9888), 0x0439a000 },
-	{ _MMIO(0x9888), 0x06392000 },
-	{ _MMIO(0x9888), 0x0cdc25c1 },
-	{ _MMIO(0x9888), 0x0adcc000 },
-	{ _MMIO(0x9888), 0x0ad825c1 },
-	{ _MMIO(0x9888), 0x18db4000 },
-	{ _MMIO(0x9888), 0x1adb0001 },
-	{ _MMIO(0x9888), 0x0e9f8000 },
-	{ _MMIO(0x9888), 0x109f02aa },
-	{ _MMIO(0x9888), 0x0eb825c1 },
-	{ _MMIO(0x9888), 0x18b80154 },
-	{ _MMIO(0x9888), 0x0ab9a000 },
-	{ _MMIO(0x9888), 0x0cb9a000 },
-	{ _MMIO(0x9888), 0x0eb9a000 },
-	{ _MMIO(0x9888), 0x0d88c000 },
-	{ _MMIO(0x9888), 0x0f88000f },
-	{ _MMIO(0x9888), 0x038a8000 },
-	{ _MMIO(0x9888), 0x058a8000 },
-	{ _MMIO(0x9888), 0x078a8000 },
-	{ _MMIO(0x9888), 0x098a8000 },
-	{ _MMIO(0x9888), 0x0b8a8000 },
-	{ _MMIO(0x9888), 0x0d8a8000 },
-	{ _MMIO(0x9888), 0x258baa05 },
-	{ _MMIO(0x9888), 0x278b002a },
-	{ _MMIO(0x9888), 0x238b2a80 },
-	{ _MMIO(0x9888), 0x198c5400 },
-	{ _MMIO(0x9888), 0x1b8c0015 },
-	{ _MMIO(0x9888), 0x098dc000 },
-	{ _MMIO(0x9888), 0x0b8da000 },
-	{ _MMIO(0x9888), 0x0d8da000 },
-	{ _MMIO(0x9888), 0x0f8da000 },
-	{ _MMIO(0x9888), 0x098e05c0 },
-	{ _MMIO(0x9888), 0x058e0000 },
-	{ _MMIO(0x9888), 0x198f0020 },
-	{ _MMIO(0x9888), 0x2185aa0a },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x1f85aa00 },
-	{ _MMIO(0x9888), 0x19835000 },
-	{ _MMIO(0x9888), 0x1b830155 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x09848000 },
-	{ _MMIO(0x9888), 0x0b84c000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x01848000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x07844000 },
-	{ _MMIO(0x9888), 0x19808000 },
-	{ _MMIO(0x9888), 0x1b80c000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x11808000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0x9888), 0x17804000 },
-	{ _MMIO(0x9888), 0x51800040 },
-	{ _MMIO(0x9888), 0x43800400 },
-	{ _MMIO(0x9888), 0x45800800 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x47800c62 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3f801042 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x418014a4 },
-};
-
-static int
-get_hdc_and_sf_mux_config(struct drm_i915_private *dev_priv,
-			  const struct i915_oa_reg **regs,
-			  int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_hdc_and_sf;
-	lens[n] = ARRAY_SIZE(mux_config_hdc_and_sf);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_l3_1[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2770), 0x00100070 },
-	{ _MMIO(0x2774), 0x0000fff1 },
-	{ _MMIO(0x2778), 0x00014002 },
-	{ _MMIO(0x277c), 0x0000c3ff },
-	{ _MMIO(0x2780), 0x00010002 },
-	{ _MMIO(0x2784), 0x0000c7ff },
-	{ _MMIO(0x2788), 0x00004002 },
-	{ _MMIO(0x278c), 0x0000d3ff },
-	{ _MMIO(0x2790), 0x00100700 },
-	{ _MMIO(0x2794), 0x0000ff1f },
-	{ _MMIO(0x2798), 0x00001402 },
-	{ _MMIO(0x279c), 0x0000fc3f },
-	{ _MMIO(0x27a0), 0x00001002 },
-	{ _MMIO(0x27a4), 0x0000fc7f },
-	{ _MMIO(0x27a8), 0x00000402 },
-	{ _MMIO(0x27ac), 0x0000fd3f },
-};
-
-static const struct i915_oa_reg flex_eu_config_l3_1[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_l3_1[] = {
-	{ _MMIO(0x9888), 0x10bf03da },
-	{ _MMIO(0x9888), 0x14bf0001 },
-	{ _MMIO(0x9888), 0x12980340 },
-	{ _MMIO(0x9888), 0x12990340 },
-	{ _MMIO(0x9888), 0x0cbf1187 },
-	{ _MMIO(0x9888), 0x0ebf1205 },
-	{ _MMIO(0x9888), 0x00bf0500 },
-	{ _MMIO(0x9888), 0x02bf042b },
-	{ _MMIO(0x9888), 0x04bf002c },
-	{ _MMIO(0x9888), 0x0cdac000 },
-	{ _MMIO(0x9888), 0x0edac000 },
-	{ _MMIO(0x9888), 0x00da8000 },
-	{ _MMIO(0x9888), 0x02dac000 },
-	{ _MMIO(0x9888), 0x04da4000 },
-	{ _MMIO(0x9888), 0x04983400 },
-	{ _MMIO(0x9888), 0x10980000 },
-	{ _MMIO(0x9888), 0x06990034 },
-	{ _MMIO(0x9888), 0x10990000 },
-	{ _MMIO(0x9888), 0x0c9dc000 },
-	{ _MMIO(0x9888), 0x0e9dc000 },
-	{ _MMIO(0x9888), 0x009d8000 },
-	{ _MMIO(0x9888), 0x029dc000 },
-	{ _MMIO(0x9888), 0x049d4000 },
-	{ _MMIO(0x9888), 0x109f02a8 },
-	{ _MMIO(0x9888), 0x0c9fa000 },
-	{ _MMIO(0x9888), 0x0e9f00ba },
-	{ _MMIO(0x9888), 0x0cb88000 },
-	{ _MMIO(0x9888), 0x0cb95000 },
-	{ _MMIO(0x9888), 0x0eb95000 },
-	{ _MMIO(0x9888), 0x00b94000 },
-	{ _MMIO(0x9888), 0x02b95000 },
-	{ _MMIO(0x9888), 0x04b91000 },
-	{ _MMIO(0x9888), 0x06b92000 },
-	{ _MMIO(0x9888), 0x0cba4000 },
-	{ _MMIO(0x9888), 0x0f88000f },
-	{ _MMIO(0x9888), 0x03888000 },
-	{ _MMIO(0x9888), 0x05888000 },
-	{ _MMIO(0x9888), 0x07888000 },
-	{ _MMIO(0x9888), 0x09888000 },
-	{ _MMIO(0x9888), 0x0b888000 },
-	{ _MMIO(0x9888), 0x0d880400 },
-	{ _MMIO(0x9888), 0x258b800a },
-	{ _MMIO(0x9888), 0x278b002a },
-	{ _MMIO(0x9888), 0x238b5500 },
-	{ _MMIO(0x9888), 0x198c4000 },
-	{ _MMIO(0x9888), 0x1b8c0015 },
-	{ _MMIO(0x9888), 0x038c4000 },
-	{ _MMIO(0x9888), 0x058c4000 },
-	{ _MMIO(0x9888), 0x078c4000 },
-	{ _MMIO(0x9888), 0x098c4000 },
-	{ _MMIO(0x9888), 0x0b8c4000 },
-	{ _MMIO(0x9888), 0x0d8c4000 },
-	{ _MMIO(0x9888), 0x0d8da000 },
-	{ _MMIO(0x9888), 0x0f8da000 },
-	{ _MMIO(0x9888), 0x018d8000 },
-	{ _MMIO(0x9888), 0x038da000 },
-	{ _MMIO(0x9888), 0x058da000 },
-	{ _MMIO(0x9888), 0x078d2000 },
-	{ _MMIO(0x9888), 0x2185800a },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x1f85aa00 },
-	{ _MMIO(0x9888), 0x1b830154 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x01848000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x07844000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x11808000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0x9888), 0x17804000 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x45800000 },
-	{ _MMIO(0x9888), 0x47800000 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3f800000 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x41800060 },
-};
-
-static int
-get_l3_1_mux_config(struct drm_i915_private *dev_priv,
-		    const struct i915_oa_reg **regs,
-		    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_l3_1;
-	lens[n] = ARRAY_SIZE(mux_config_l3_1);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_l3_2[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2770), 0x00100070 },
-	{ _MMIO(0x2774), 0x0000fff1 },
-	{ _MMIO(0x2778), 0x00014002 },
-	{ _MMIO(0x277c), 0x0000c3ff },
-	{ _MMIO(0x2780), 0x00010002 },
-	{ _MMIO(0x2784), 0x0000c7ff },
-	{ _MMIO(0x2788), 0x00004002 },
-	{ _MMIO(0x278c), 0x0000d3ff },
-	{ _MMIO(0x2790), 0x00100700 },
-	{ _MMIO(0x2794), 0x0000ff1f },
-	{ _MMIO(0x2798), 0x00001402 },
-	{ _MMIO(0x279c), 0x0000fc3f },
-	{ _MMIO(0x27a0), 0x00001002 },
-	{ _MMIO(0x27a4), 0x0000fc7f },
-	{ _MMIO(0x27a8), 0x00000402 },
-	{ _MMIO(0x27ac), 0x0000fd3f },
-};
-
-static const struct i915_oa_reg flex_eu_config_l3_2[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_l3_2[] = {
-	{ _MMIO(0x9888), 0x103f03da },
-	{ _MMIO(0x9888), 0x143f0001 },
-	{ _MMIO(0x9888), 0x12180340 },
-	{ _MMIO(0x9888), 0x12190340 },
-	{ _MMIO(0x9888), 0x0c3f1187 },
-	{ _MMIO(0x9888), 0x0e3f1205 },
-	{ _MMIO(0x9888), 0x003f0500 },
-	{ _MMIO(0x9888), 0x023f042b },
-	{ _MMIO(0x9888), 0x043f002c },
-	{ _MMIO(0x9888), 0x0c5ac000 },
-	{ _MMIO(0x9888), 0x0e5ac000 },
-	{ _MMIO(0x9888), 0x005a8000 },
-	{ _MMIO(0x9888), 0x025ac000 },
-	{ _MMIO(0x9888), 0x045a4000 },
-	{ _MMIO(0x9888), 0x04183400 },
-	{ _MMIO(0x9888), 0x10180000 },
-	{ _MMIO(0x9888), 0x06190034 },
-	{ _MMIO(0x9888), 0x10190000 },
-	{ _MMIO(0x9888), 0x0c1dc000 },
-	{ _MMIO(0x9888), 0x0e1dc000 },
-	{ _MMIO(0x9888), 0x001d8000 },
-	{ _MMIO(0x9888), 0x021dc000 },
-	{ _MMIO(0x9888), 0x041d4000 },
-	{ _MMIO(0x9888), 0x101f02a8 },
-	{ _MMIO(0x9888), 0x0c1fa000 },
-	{ _MMIO(0x9888), 0x0e1f00ba },
-	{ _MMIO(0x9888), 0x0c388000 },
-	{ _MMIO(0x9888), 0x0c395000 },
-	{ _MMIO(0x9888), 0x0e395000 },
-	{ _MMIO(0x9888), 0x00394000 },
-	{ _MMIO(0x9888), 0x02395000 },
-	{ _MMIO(0x9888), 0x04391000 },
-	{ _MMIO(0x9888), 0x06392000 },
-	{ _MMIO(0x9888), 0x0c3a4000 },
-	{ _MMIO(0x9888), 0x1b8aa800 },
-	{ _MMIO(0x9888), 0x1d8a0002 },
-	{ _MMIO(0x9888), 0x038a8000 },
-	{ _MMIO(0x9888), 0x058a8000 },
-	{ _MMIO(0x9888), 0x078a8000 },
-	{ _MMIO(0x9888), 0x098a8000 },
-	{ _MMIO(0x9888), 0x0b8a8000 },
-	{ _MMIO(0x9888), 0x0d8a8000 },
-	{ _MMIO(0x9888), 0x258b4005 },
-	{ _MMIO(0x9888), 0x278b0015 },
-	{ _MMIO(0x9888), 0x238b2a80 },
-	{ _MMIO(0x9888), 0x2185800a },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x1f85aa00 },
-	{ _MMIO(0x9888), 0x1b830154 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x01848000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x07844000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x11808000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0x9888), 0x17804000 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x45800000 },
-	{ _MMIO(0x9888), 0x47800000 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3f800000 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x41800060 },
-};
-
-static int
-get_l3_2_mux_config(struct drm_i915_private *dev_priv,
-		    const struct i915_oa_reg **regs,
-		    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_l3_2;
-	lens[n] = ARRAY_SIZE(mux_config_l3_2);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_l3_3[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2770), 0x00100070 },
-	{ _MMIO(0x2774), 0x0000fff1 },
-	{ _MMIO(0x2778), 0x00014002 },
-	{ _MMIO(0x277c), 0x0000c3ff },
-	{ _MMIO(0x2780), 0x00010002 },
-	{ _MMIO(0x2784), 0x0000c7ff },
-	{ _MMIO(0x2788), 0x00004002 },
-	{ _MMIO(0x278c), 0x0000d3ff },
-	{ _MMIO(0x2790), 0x00100700 },
-	{ _MMIO(0x2794), 0x0000ff1f },
-	{ _MMIO(0x2798), 0x00001402 },
-	{ _MMIO(0x279c), 0x0000fc3f },
-	{ _MMIO(0x27a0), 0x00001002 },
-	{ _MMIO(0x27a4), 0x0000fc7f },
-	{ _MMIO(0x27a8), 0x00000402 },
-	{ _MMIO(0x27ac), 0x0000fd3f },
-};
-
-static const struct i915_oa_reg flex_eu_config_l3_3[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_l3_3[] = {
-	{ _MMIO(0x9888), 0x121b0340 },
-	{ _MMIO(0x9888), 0x103f0274 },
-	{ _MMIO(0x9888), 0x123f0000 },
-	{ _MMIO(0x9888), 0x129b0340 },
-	{ _MMIO(0x9888), 0x10bf0274 },
-	{ _MMIO(0x9888), 0x12bf0000 },
-	{ _MMIO(0x9888), 0x041b3400 },
-	{ _MMIO(0x9888), 0x101b0000 },
-	{ _MMIO(0x9888), 0x045c8000 },
-	{ _MMIO(0x9888), 0x0a3d4000 },
-	{ _MMIO(0x9888), 0x003f0080 },
-	{ _MMIO(0x9888), 0x023f0793 },
-	{ _MMIO(0x9888), 0x043f0014 },
-	{ _MMIO(0x9888), 0x04588000 },
-	{ _MMIO(0x9888), 0x005a8000 },
-	{ _MMIO(0x9888), 0x025ac000 },
-	{ _MMIO(0x9888), 0x045a4000 },
-	{ _MMIO(0x9888), 0x0a5b4000 },
-	{ _MMIO(0x9888), 0x001d8000 },
-	{ _MMIO(0x9888), 0x021dc000 },
-	{ _MMIO(0x9888), 0x041d4000 },
-	{ _MMIO(0x9888), 0x0c1fa000 },
-	{ _MMIO(0x9888), 0x0e1f002a },
-	{ _MMIO(0x9888), 0x0a384000 },
-	{ _MMIO(0x9888), 0x00394000 },
-	{ _MMIO(0x9888), 0x02395000 },
-	{ _MMIO(0x9888), 0x04399000 },
-	{ _MMIO(0x9888), 0x069b0034 },
-	{ _MMIO(0x9888), 0x109b0000 },
-	{ _MMIO(0x9888), 0x06dc4000 },
-	{ _MMIO(0x9888), 0x0cbd4000 },
-	{ _MMIO(0x9888), 0x0cbf0981 },
-	{ _MMIO(0x9888), 0x0ebf0a0f },
-	{ _MMIO(0x9888), 0x06d84000 },
-	{ _MMIO(0x9888), 0x0cdac000 },
-	{ _MMIO(0x9888), 0x0edac000 },
-	{ _MMIO(0x9888), 0x0cdb4000 },
-	{ _MMIO(0x9888), 0x0c9dc000 },
-	{ _MMIO(0x9888), 0x0e9dc000 },
-	{ _MMIO(0x9888), 0x109f02a8 },
-	{ _MMIO(0x9888), 0x0e9f0080 },
-	{ _MMIO(0x9888), 0x0cb84000 },
-	{ _MMIO(0x9888), 0x0cb95000 },
-	{ _MMIO(0x9888), 0x0eb95000 },
-	{ _MMIO(0x9888), 0x06b92000 },
-	{ _MMIO(0x9888), 0x0f88000f },
-	{ _MMIO(0x9888), 0x0d880400 },
-	{ _MMIO(0x9888), 0x038a8000 },
-	{ _MMIO(0x9888), 0x058a8000 },
-	{ _MMIO(0x9888), 0x078a8000 },
-	{ _MMIO(0x9888), 0x098a8000 },
-	{ _MMIO(0x9888), 0x0b8a8000 },
-	{ _MMIO(0x9888), 0x258b8009 },
-	{ _MMIO(0x9888), 0x278b002a },
-	{ _MMIO(0x9888), 0x238b2a80 },
-	{ _MMIO(0x9888), 0x198c4000 },
-	{ _MMIO(0x9888), 0x1b8c0015 },
-	{ _MMIO(0x9888), 0x0d8c4000 },
-	{ _MMIO(0x9888), 0x0d8da000 },
-	{ _MMIO(0x9888), 0x0f8da000 },
-	{ _MMIO(0x9888), 0x078d2000 },
-	{ _MMIO(0x9888), 0x2185800a },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x1f85aa00 },
-	{ _MMIO(0x9888), 0x1b830154 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x01848000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x07844000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x11808000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0x9888), 0x17804000 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x45800c00 },
-	{ _MMIO(0x9888), 0x47800c63 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3f8014a5 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x41800045 },
-};
-
-static int
-get_l3_3_mux_config(struct drm_i915_private *dev_priv,
-		    const struct i915_oa_reg **regs,
-		    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_l3_3;
-	lens[n] = ARRAY_SIZE(mux_config_l3_3);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_l3_4[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2770), 0x00100070 },
-	{ _MMIO(0x2774), 0x0000fff1 },
-	{ _MMIO(0x2778), 0x00014002 },
-	{ _MMIO(0x277c), 0x0000c3ff },
-	{ _MMIO(0x2780), 0x00010002 },
-	{ _MMIO(0x2784), 0x0000c7ff },
-	{ _MMIO(0x2788), 0x00004002 },
-	{ _MMIO(0x278c), 0x0000d3ff },
-	{ _MMIO(0x2790), 0x00100700 },
-	{ _MMIO(0x2794), 0x0000ff1f },
-	{ _MMIO(0x2798), 0x00001402 },
-	{ _MMIO(0x279c), 0x0000fc3f },
-	{ _MMIO(0x27a0), 0x00001002 },
-	{ _MMIO(0x27a4), 0x0000fc7f },
-	{ _MMIO(0x27a8), 0x00000402 },
-	{ _MMIO(0x27ac), 0x0000fd3f },
-};
-
-static const struct i915_oa_reg flex_eu_config_l3_4[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_l3_4[] = {
-	{ _MMIO(0x9888), 0x121a0340 },
-	{ _MMIO(0x9888), 0x103f0017 },
-	{ _MMIO(0x9888), 0x123f0020 },
-	{ _MMIO(0x9888), 0x129a0340 },
-	{ _MMIO(0x9888), 0x10bf0017 },
-	{ _MMIO(0x9888), 0x12bf0020 },
-	{ _MMIO(0x9888), 0x041a3400 },
-	{ _MMIO(0x9888), 0x101a0000 },
-	{ _MMIO(0x9888), 0x043b8000 },
-	{ _MMIO(0x9888), 0x0a3e0010 },
-	{ _MMIO(0x9888), 0x003f0200 },
-	{ _MMIO(0x9888), 0x023f0113 },
-	{ _MMIO(0x9888), 0x043f0014 },
-	{ _MMIO(0x9888), 0x02592000 },
-	{ _MMIO(0x9888), 0x005a8000 },
-	{ _MMIO(0x9888), 0x025ac000 },
-	{ _MMIO(0x9888), 0x045a4000 },
-	{ _MMIO(0x9888), 0x0a1c8000 },
-	{ _MMIO(0x9888), 0x001d8000 },
-	{ _MMIO(0x9888), 0x021dc000 },
-	{ _MMIO(0x9888), 0x041d4000 },
-	{ _MMIO(0x9888), 0x0a1e8000 },
-	{ _MMIO(0x9888), 0x0c1fa000 },
-	{ _MMIO(0x9888), 0x0e1f001a },
-	{ _MMIO(0x9888), 0x00394000 },
-	{ _MMIO(0x9888), 0x02395000 },
-	{ _MMIO(0x9888), 0x04391000 },
-	{ _MMIO(0x9888), 0x069a0034 },
-	{ _MMIO(0x9888), 0x109a0000 },
-	{ _MMIO(0x9888), 0x06bb4000 },
-	{ _MMIO(0x9888), 0x0abe0040 },
-	{ _MMIO(0x9888), 0x0cbf0984 },
-	{ _MMIO(0x9888), 0x0ebf0a02 },
-	{ _MMIO(0x9888), 0x02d94000 },
-	{ _MMIO(0x9888), 0x0cdac000 },
-	{ _MMIO(0x9888), 0x0edac000 },
-	{ _MMIO(0x9888), 0x0c9c0400 },
-	{ _MMIO(0x9888), 0x0c9dc000 },
-	{ _MMIO(0x9888), 0x0e9dc000 },
-	{ _MMIO(0x9888), 0x0c9e0400 },
-	{ _MMIO(0x9888), 0x109f02a8 },
-	{ _MMIO(0x9888), 0x0e9f0040 },
-	{ _MMIO(0x9888), 0x0cb95000 },
-	{ _MMIO(0x9888), 0x0eb95000 },
-	{ _MMIO(0x9888), 0x0f88000f },
-	{ _MMIO(0x9888), 0x0d880400 },
-	{ _MMIO(0x9888), 0x038a8000 },
-	{ _MMIO(0x9888), 0x058a8000 },
-	{ _MMIO(0x9888), 0x078a8000 },
-	{ _MMIO(0x9888), 0x098a8000 },
-	{ _MMIO(0x9888), 0x0b8a8000 },
-	{ _MMIO(0x9888), 0x258b8009 },
-	{ _MMIO(0x9888), 0x278b002a },
-	{ _MMIO(0x9888), 0x238b2a80 },
-	{ _MMIO(0x9888), 0x198c4000 },
-	{ _MMIO(0x9888), 0x1b8c0015 },
-	{ _MMIO(0x9888), 0x0d8c4000 },
-	{ _MMIO(0x9888), 0x0d8da000 },
-	{ _MMIO(0x9888), 0x0f8da000 },
-	{ _MMIO(0x9888), 0x078d2000 },
-	{ _MMIO(0x9888), 0x2185800a },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x1f85aa00 },
-	{ _MMIO(0x9888), 0x1b830154 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x01848000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x07844000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x11808000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0x9888), 0x17804000 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x45800800 },
-	{ _MMIO(0x9888), 0x47800842 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3f801084 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x41800044 },
-};
-
-static int
-get_l3_4_mux_config(struct drm_i915_private *dev_priv,
-		    const struct i915_oa_reg **regs,
-		    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_l3_4;
-	lens[n] = ARRAY_SIZE(mux_config_l3_4);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2770), 0x00006000 },
-	{ _MMIO(0x2774), 0x0000f3ff },
-	{ _MMIO(0x2778), 0x00001800 },
-	{ _MMIO(0x277c), 0x0000fcff },
-	{ _MMIO(0x2780), 0x00000600 },
-	{ _MMIO(0x2784), 0x0000ff3f },
-	{ _MMIO(0x2788), 0x00000180 },
-	{ _MMIO(0x278c), 0x0000ffcf },
-	{ _MMIO(0x2790), 0x00000060 },
-	{ _MMIO(0x2794), 0x0000fff3 },
-	{ _MMIO(0x2798), 0x00000018 },
-	{ _MMIO(0x279c), 0x0000fffc },
-};
-
-static const struct i915_oa_reg flex_eu_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0x9888), 0x143b000e },
-	{ _MMIO(0x9888), 0x043c55c0 },
-	{ _MMIO(0x9888), 0x0a1e0280 },
-	{ _MMIO(0x9888), 0x0c1e0408 },
-	{ _MMIO(0x9888), 0x10390000 },
-	{ _MMIO(0x9888), 0x12397a1f },
-	{ _MMIO(0x9888), 0x14bb000e },
-	{ _MMIO(0x9888), 0x04bc5000 },
-	{ _MMIO(0x9888), 0x0a9e0296 },
-	{ _MMIO(0x9888), 0x0c9e0008 },
-	{ _MMIO(0x9888), 0x10b90000 },
-	{ _MMIO(0x9888), 0x12b97a1f },
-	{ _MMIO(0x9888), 0x063b0042 },
-	{ _MMIO(0x9888), 0x103b0000 },
-	{ _MMIO(0x9888), 0x083c0000 },
-	{ _MMIO(0x9888), 0x0a3e0040 },
-	{ _MMIO(0x9888), 0x043f8000 },
-	{ _MMIO(0x9888), 0x02594000 },
-	{ _MMIO(0x9888), 0x045a8000 },
-	{ _MMIO(0x9888), 0x0c1c0400 },
-	{ _MMIO(0x9888), 0x041d8000 },
-	{ _MMIO(0x9888), 0x081e02c0 },
-	{ _MMIO(0x9888), 0x0e1e0000 },
-	{ _MMIO(0x9888), 0x0c1fa800 },
-	{ _MMIO(0x9888), 0x0e1f0260 },
-	{ _MMIO(0x9888), 0x101f0014 },
-	{ _MMIO(0x9888), 0x003905e0 },
-	{ _MMIO(0x9888), 0x06390bc0 },
-	{ _MMIO(0x9888), 0x02390018 },
-	{ _MMIO(0x9888), 0x04394000 },
-	{ _MMIO(0x9888), 0x04bb0042 },
-	{ _MMIO(0x9888), 0x10bb0000 },
-	{ _MMIO(0x9888), 0x02bc05c0 },
-	{ _MMIO(0x9888), 0x08bc0000 },
-	{ _MMIO(0x9888), 0x0abe0004 },
-	{ _MMIO(0x9888), 0x02bf8000 },
-	{ _MMIO(0x9888), 0x02d91000 },
-	{ _MMIO(0x9888), 0x02da8000 },
-	{ _MMIO(0x9888), 0x089c8000 },
-	{ _MMIO(0x9888), 0x029d8000 },
-	{ _MMIO(0x9888), 0x089e8000 },
-	{ _MMIO(0x9888), 0x0e9e0000 },
-	{ _MMIO(0x9888), 0x0e9fa806 },
-	{ _MMIO(0x9888), 0x109f0142 },
-	{ _MMIO(0x9888), 0x08b90617 },
-	{ _MMIO(0x9888), 0x0ab90be0 },
-	{ _MMIO(0x9888), 0x02b94000 },
-	{ _MMIO(0x9888), 0x0d88f000 },
-	{ _MMIO(0x9888), 0x0f88000c },
-	{ _MMIO(0x9888), 0x07888000 },
-	{ _MMIO(0x9888), 0x09888000 },
-	{ _MMIO(0x9888), 0x018a8000 },
-	{ _MMIO(0x9888), 0x0f8a8000 },
-	{ _MMIO(0x9888), 0x1b8a2800 },
-	{ _MMIO(0x9888), 0x038a8000 },
-	{ _MMIO(0x9888), 0x058a8000 },
-	{ _MMIO(0x9888), 0x0b8a8000 },
-	{ _MMIO(0x9888), 0x0d8a8000 },
-	{ _MMIO(0x9888), 0x238b52a0 },
-	{ _MMIO(0x9888), 0x258b6a95 },
-	{ _MMIO(0x9888), 0x278b0029 },
-	{ _MMIO(0x9888), 0x178c2000 },
-	{ _MMIO(0x9888), 0x198c1500 },
-	{ _MMIO(0x9888), 0x1b8c0014 },
-	{ _MMIO(0x9888), 0x078c4000 },
-	{ _MMIO(0x9888), 0x098c4000 },
-	{ _MMIO(0x9888), 0x098da000 },
-	{ _MMIO(0x9888), 0x0b8da000 },
-	{ _MMIO(0x9888), 0x0f8da000 },
-	{ _MMIO(0x9888), 0x038d8000 },
-	{ _MMIO(0x9888), 0x058d2000 },
-	{ _MMIO(0x9888), 0x1f85aa80 },
-	{ _MMIO(0x9888), 0x2185aaaa },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x01834000 },
-	{ _MMIO(0x9888), 0x0f834000 },
-	{ _MMIO(0x9888), 0x19835400 },
-	{ _MMIO(0x9888), 0x1b830155 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x0184c000 },
-	{ _MMIO(0x9888), 0x0784c000 },
-	{ _MMIO(0x9888), 0x0984c000 },
-	{ _MMIO(0x9888), 0x0b84c000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x1180c000 },
-	{ _MMIO(0x9888), 0x1780c000 },
-	{ _MMIO(0x9888), 0x1980c000 },
-	{ _MMIO(0x9888), 0x1b80c000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0x9888), 0x4d800444 },
-	{ _MMIO(0x9888), 0x3d800000 },
-	{ _MMIO(0x9888), 0x4f804000 },
-	{ _MMIO(0x9888), 0x43801080 },
-	{ _MMIO(0x9888), 0x51800000 },
-	{ _MMIO(0x9888), 0x45800084 },
-	{ _MMIO(0x9888), 0x53800044 },
-	{ _MMIO(0x9888), 0x47801080 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x3f800000 },
-	{ _MMIO(0x9888), 0x41800840 },
-};
-
-static int
-get_rasterizer_and_pixel_backend_mux_config(struct drm_i915_private *dev_priv,
-					    const struct i915_oa_reg **regs,
-					    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_rasterizer_and_pixel_backend;
-	lens[n] = ARRAY_SIZE(mux_config_rasterizer_and_pixel_backend);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_sampler_1[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x70800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x0000c000 },
-	{ _MMIO(0x2774), 0x0000e7ff },
-	{ _MMIO(0x2778), 0x00003000 },
-	{ _MMIO(0x277c), 0x0000f9ff },
-	{ _MMIO(0x2780), 0x00000c00 },
-	{ _MMIO(0x2784), 0x0000fe7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_sampler_1[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_sampler_1[] = {
-	{ _MMIO(0x9888), 0x18921400 },
-	{ _MMIO(0x9888), 0x149500ab },
-	{ _MMIO(0x9888), 0x18b21400 },
-	{ _MMIO(0x9888), 0x14b500ab },
-	{ _MMIO(0x9888), 0x18d21400 },
-	{ _MMIO(0x9888), 0x14d500ab },
-	{ _MMIO(0x9888), 0x0cdc8000 },
-	{ _MMIO(0x9888), 0x0edc4000 },
-	{ _MMIO(0x9888), 0x02dcc000 },
-	{ _MMIO(0x9888), 0x04dcc000 },
-	{ _MMIO(0x9888), 0x1abd00a0 },
-	{ _MMIO(0x9888), 0x0abd8000 },
-	{ _MMIO(0x9888), 0x0cd88000 },
-	{ _MMIO(0x9888), 0x0ed84000 },
-	{ _MMIO(0x9888), 0x04d88000 },
-	{ _MMIO(0x9888), 0x1adb0050 },
-	{ _MMIO(0x9888), 0x04db8000 },
-	{ _MMIO(0x9888), 0x06db8000 },
-	{ _MMIO(0x9888), 0x08db8000 },
-	{ _MMIO(0x9888), 0x0adb4000 },
-	{ _MMIO(0x9888), 0x109f02a0 },
-	{ _MMIO(0x9888), 0x0c9fa000 },
-	{ _MMIO(0x9888), 0x0e9f00aa },
-	{ _MMIO(0x9888), 0x18b82500 },
-	{ _MMIO(0x9888), 0x02b88000 },
-	{ _MMIO(0x9888), 0x04b84000 },
-	{ _MMIO(0x9888), 0x06b84000 },
-	{ _MMIO(0x9888), 0x08b84000 },
-	{ _MMIO(0x9888), 0x0ab84000 },
-	{ _MMIO(0x9888), 0x0cb88000 },
-	{ _MMIO(0x9888), 0x0cb98000 },
-	{ _MMIO(0x9888), 0x0eb9a000 },
-	{ _MMIO(0x9888), 0x00b98000 },
-	{ _MMIO(0x9888), 0x02b9a000 },
-	{ _MMIO(0x9888), 0x04b9a000 },
-	{ _MMIO(0x9888), 0x06b92000 },
-	{ _MMIO(0x9888), 0x1aba0200 },
-	{ _MMIO(0x9888), 0x02ba8000 },
-	{ _MMIO(0x9888), 0x0cba8000 },
-	{ _MMIO(0x9888), 0x04908000 },
-	{ _MMIO(0x9888), 0x04918000 },
-	{ _MMIO(0x9888), 0x04927300 },
-	{ _MMIO(0x9888), 0x10920000 },
-	{ _MMIO(0x9888), 0x1893000a },
-	{ _MMIO(0x9888), 0x0a934000 },
-	{ _MMIO(0x9888), 0x0a946000 },
-	{ _MMIO(0x9888), 0x0c959000 },
-	{ _MMIO(0x9888), 0x0e950098 },
-	{ _MMIO(0x9888), 0x10950000 },
-	{ _MMIO(0x9888), 0x04b04000 },
-	{ _MMIO(0x9888), 0x04b14000 },
-	{ _MMIO(0x9888), 0x04b20073 },
-	{ _MMIO(0x9888), 0x10b20000 },
-	{ _MMIO(0x9888), 0x04b38000 },
-	{ _MMIO(0x9888), 0x06b38000 },
-	{ _MMIO(0x9888), 0x08b34000 },
-	{ _MMIO(0x9888), 0x04b4c000 },
-	{ _MMIO(0x9888), 0x02b59890 },
-	{ _MMIO(0x9888), 0x10b50000 },
-	{ _MMIO(0x9888), 0x06d04000 },
-	{ _MMIO(0x9888), 0x06d14000 },
-	{ _MMIO(0x9888), 0x06d20073 },
-	{ _MMIO(0x9888), 0x10d20000 },
-	{ _MMIO(0x9888), 0x18d30020 },
-	{ _MMIO(0x9888), 0x02d38000 },
-	{ _MMIO(0x9888), 0x0cd34000 },
-	{ _MMIO(0x9888), 0x0ad48000 },
-	{ _MMIO(0x9888), 0x04d42000 },
-	{ _MMIO(0x9888), 0x0ed59000 },
-	{ _MMIO(0x9888), 0x00d59800 },
-	{ _MMIO(0x9888), 0x10d50000 },
-	{ _MMIO(0x9888), 0x0f88000e },
-	{ _MMIO(0x9888), 0x03888000 },
-	{ _MMIO(0x9888), 0x05888000 },
-	{ _MMIO(0x9888), 0x07888000 },
-	{ _MMIO(0x9888), 0x09888000 },
-	{ _MMIO(0x9888), 0x0b888000 },
-	{ _MMIO(0x9888), 0x0d880400 },
-	{ _MMIO(0x9888), 0x278b002a },
-	{ _MMIO(0x9888), 0x238b5500 },
-	{ _MMIO(0x9888), 0x258b000a },
-	{ _MMIO(0x9888), 0x1b8c0015 },
-	{ _MMIO(0x9888), 0x038c4000 },
-	{ _MMIO(0x9888), 0x058c4000 },
-	{ _MMIO(0x9888), 0x078c4000 },
-	{ _MMIO(0x9888), 0x098c4000 },
-	{ _MMIO(0x9888), 0x0b8c4000 },
-	{ _MMIO(0x9888), 0x0d8c4000 },
-	{ _MMIO(0x9888), 0x0d8d8000 },
-	{ _MMIO(0x9888), 0x0f8da000 },
-	{ _MMIO(0x9888), 0x018d8000 },
-	{ _MMIO(0x9888), 0x038da000 },
-	{ _MMIO(0x9888), 0x058da000 },
-	{ _MMIO(0x9888), 0x078d2000 },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x1f85aa00 },
-	{ _MMIO(0x9888), 0x2185000a },
-	{ _MMIO(0x9888), 0x1b830150 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x0d848000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x01848000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x07844000 },
-	{ _MMIO(0x9888), 0x1d808000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x11808000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0x9888), 0x17804000 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x47801021 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3f800c64 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x41800c02 },
-};
-
-static int
-get_sampler_1_mux_config(struct drm_i915_private *dev_priv,
-			 const struct i915_oa_reg **regs,
-			 int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_sampler_1;
-	lens[n] = ARRAY_SIZE(mux_config_sampler_1);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_sampler_2[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x70800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x0000c000 },
-	{ _MMIO(0x2774), 0x0000e7ff },
-	{ _MMIO(0x2778), 0x00003000 },
-	{ _MMIO(0x277c), 0x0000f9ff },
-	{ _MMIO(0x2780), 0x00000c00 },
-	{ _MMIO(0x2784), 0x0000fe7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_sampler_2[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_sampler_2[] = {
-	{ _MMIO(0x9888), 0x18121400 },
-	{ _MMIO(0x9888), 0x141500ab },
-	{ _MMIO(0x9888), 0x18321400 },
-	{ _MMIO(0x9888), 0x143500ab },
-	{ _MMIO(0x9888), 0x18521400 },
-	{ _MMIO(0x9888), 0x145500ab },
-	{ _MMIO(0x9888), 0x0c5c8000 },
-	{ _MMIO(0x9888), 0x0e5c4000 },
-	{ _MMIO(0x9888), 0x025cc000 },
-	{ _MMIO(0x9888), 0x045cc000 },
-	{ _MMIO(0x9888), 0x1a3d00a0 },
-	{ _MMIO(0x9888), 0x0a3d8000 },
-	{ _MMIO(0x9888), 0x0c588000 },
-	{ _MMIO(0x9888), 0x0e584000 },
-	{ _MMIO(0x9888), 0x04588000 },
-	{ _MMIO(0x9888), 0x1a5b0050 },
-	{ _MMIO(0x9888), 0x045b8000 },
-	{ _MMIO(0x9888), 0x065b8000 },
-	{ _MMIO(0x9888), 0x085b8000 },
-	{ _MMIO(0x9888), 0x0a5b4000 },
-	{ _MMIO(0x9888), 0x101f02a0 },
-	{ _MMIO(0x9888), 0x0c1fa000 },
-	{ _MMIO(0x9888), 0x0e1f00aa },
-	{ _MMIO(0x9888), 0x18382500 },
-	{ _MMIO(0x9888), 0x02388000 },
-	{ _MMIO(0x9888), 0x04384000 },
-	{ _MMIO(0x9888), 0x06384000 },
-	{ _MMIO(0x9888), 0x08384000 },
-	{ _MMIO(0x9888), 0x0a384000 },
-	{ _MMIO(0x9888), 0x0c388000 },
-	{ _MMIO(0x9888), 0x0c398000 },
-	{ _MMIO(0x9888), 0x0e39a000 },
-	{ _MMIO(0x9888), 0x00398000 },
-	{ _MMIO(0x9888), 0x0239a000 },
-	{ _MMIO(0x9888), 0x0439a000 },
-	{ _MMIO(0x9888), 0x06392000 },
-	{ _MMIO(0x9888), 0x1a3a0200 },
-	{ _MMIO(0x9888), 0x023a8000 },
-	{ _MMIO(0x9888), 0x0c3a8000 },
-	{ _MMIO(0x9888), 0x04108000 },
-	{ _MMIO(0x9888), 0x04118000 },
-	{ _MMIO(0x9888), 0x04127300 },
-	{ _MMIO(0x9888), 0x10120000 },
-	{ _MMIO(0x9888), 0x1813000a },
-	{ _MMIO(0x9888), 0x0a134000 },
-	{ _MMIO(0x9888), 0x0a146000 },
-	{ _MMIO(0x9888), 0x0c159000 },
-	{ _MMIO(0x9888), 0x0e150098 },
-	{ _MMIO(0x9888), 0x10150000 },
-	{ _MMIO(0x9888), 0x04304000 },
-	{ _MMIO(0x9888), 0x04314000 },
-	{ _MMIO(0x9888), 0x04320073 },
-	{ _MMIO(0x9888), 0x10320000 },
-	{ _MMIO(0x9888), 0x04338000 },
-	{ _MMIO(0x9888), 0x06338000 },
-	{ _MMIO(0x9888), 0x08334000 },
-	{ _MMIO(0x9888), 0x0434c000 },
-	{ _MMIO(0x9888), 0x02359890 },
-	{ _MMIO(0x9888), 0x10350000 },
-	{ _MMIO(0x9888), 0x06504000 },
-	{ _MMIO(0x9888), 0x06514000 },
-	{ _MMIO(0x9888), 0x06520073 },
-	{ _MMIO(0x9888), 0x10520000 },
-	{ _MMIO(0x9888), 0x18530020 },
-	{ _MMIO(0x9888), 0x02538000 },
-	{ _MMIO(0x9888), 0x0c534000 },
-	{ _MMIO(0x9888), 0x0a548000 },
-	{ _MMIO(0x9888), 0x04542000 },
-	{ _MMIO(0x9888), 0x0e559000 },
-	{ _MMIO(0x9888), 0x00559800 },
-	{ _MMIO(0x9888), 0x10550000 },
-	{ _MMIO(0x9888), 0x1b8aa000 },
-	{ _MMIO(0x9888), 0x1d8a0002 },
-	{ _MMIO(0x9888), 0x038a8000 },
-	{ _MMIO(0x9888), 0x058a8000 },
-	{ _MMIO(0x9888), 0x078a8000 },
-	{ _MMIO(0x9888), 0x098a8000 },
-	{ _MMIO(0x9888), 0x0b8a8000 },
-	{ _MMIO(0x9888), 0x0d8a8000 },
-	{ _MMIO(0x9888), 0x278b0015 },
-	{ _MMIO(0x9888), 0x238b2a80 },
-	{ _MMIO(0x9888), 0x258b0005 },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x1f85aa00 },
-	{ _MMIO(0x9888), 0x2185000a },
-	{ _MMIO(0x9888), 0x1b830150 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x0d848000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x01848000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x07844000 },
-	{ _MMIO(0x9888), 0x1d808000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x11808000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0x9888), 0x17804000 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x47801021 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3f800c64 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x41800c02 },
-};
-
-static int
-get_sampler_2_mux_config(struct drm_i915_private *dev_priv,
-			 const struct i915_oa_reg **regs,
-			 int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_sampler_2;
-	lens[n] = ARRAY_SIZE(mux_config_sampler_2);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_tdl_1[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x0000fdff },
-	{ _MMIO(0x2778), 0x00000000 },
-	{ _MMIO(0x277c), 0x0000fe7f },
-	{ _MMIO(0x2780), 0x00000002 },
-	{ _MMIO(0x2784), 0x0000ffbf },
-	{ _MMIO(0x2788), 0x00000000 },
-	{ _MMIO(0x278c), 0x0000ffcf },
-	{ _MMIO(0x2790), 0x00000002 },
-	{ _MMIO(0x2794), 0x0000fff7 },
-	{ _MMIO(0x2798), 0x00000000 },
-	{ _MMIO(0x279c), 0x0000fff9 },
-};
-
-static const struct i915_oa_reg flex_eu_config_tdl_1[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_tdl_1[] = {
-	{ _MMIO(0x9888), 0x16154d60 },
-	{ _MMIO(0x9888), 0x16352e60 },
-	{ _MMIO(0x9888), 0x16554d60 },
-	{ _MMIO(0x9888), 0x16950000 },
-	{ _MMIO(0x9888), 0x16b50000 },
-	{ _MMIO(0x9888), 0x16d50000 },
-	{ _MMIO(0x9888), 0x005c8000 },
-	{ _MMIO(0x9888), 0x045cc000 },
-	{ _MMIO(0x9888), 0x065c4000 },
-	{ _MMIO(0x9888), 0x083d8000 },
-	{ _MMIO(0x9888), 0x0a3d8000 },
-	{ _MMIO(0x9888), 0x0458c000 },
-	{ _MMIO(0x9888), 0x025b8000 },
-	{ _MMIO(0x9888), 0x085b4000 },
-	{ _MMIO(0x9888), 0x0a5b4000 },
-	{ _MMIO(0x9888), 0x0c5b8000 },
-	{ _MMIO(0x9888), 0x0c1fa000 },
-	{ _MMIO(0x9888), 0x0e1f00aa },
-	{ _MMIO(0x9888), 0x02384000 },
-	{ _MMIO(0x9888), 0x04388000 },
-	{ _MMIO(0x9888), 0x06388000 },
-	{ _MMIO(0x9888), 0x08384000 },
-	{ _MMIO(0x9888), 0x0a384000 },
-	{ _MMIO(0x9888), 0x0c384000 },
-	{ _MMIO(0x9888), 0x00398000 },
-	{ _MMIO(0x9888), 0x0239a000 },
-	{ _MMIO(0x9888), 0x0439a000 },
-	{ _MMIO(0x9888), 0x06392000 },
-	{ _MMIO(0x9888), 0x043a8000 },
-	{ _MMIO(0x9888), 0x063a8000 },
-	{ _MMIO(0x9888), 0x08138000 },
-	{ _MMIO(0x9888), 0x0a138000 },
-	{ _MMIO(0x9888), 0x06143000 },
-	{ _MMIO(0x9888), 0x0415cfc7 },
-	{ _MMIO(0x9888), 0x10150000 },
-	{ _MMIO(0x9888), 0x02338000 },
-	{ _MMIO(0x9888), 0x0c338000 },
-	{ _MMIO(0x9888), 0x04342000 },
-	{ _MMIO(0x9888), 0x06344000 },
-	{ _MMIO(0x9888), 0x0035c700 },
-	{ _MMIO(0x9888), 0x063500cf },
-	{ _MMIO(0x9888), 0x10350000 },
-	{ _MMIO(0x9888), 0x04538000 },
-	{ _MMIO(0x9888), 0x06538000 },
-	{ _MMIO(0x9888), 0x0454c000 },
-	{ _MMIO(0x9888), 0x0255cfc7 },
-	{ _MMIO(0x9888), 0x10550000 },
-	{ _MMIO(0x9888), 0x06dc8000 },
-	{ _MMIO(0x9888), 0x08dc4000 },
-	{ _MMIO(0x9888), 0x0cdcc000 },
-	{ _MMIO(0x9888), 0x0edcc000 },
-	{ _MMIO(0x9888), 0x1abd00a8 },
-	{ _MMIO(0x9888), 0x0cd8c000 },
-	{ _MMIO(0x9888), 0x0ed84000 },
-	{ _MMIO(0x9888), 0x0edb8000 },
-	{ _MMIO(0x9888), 0x18db0800 },
-	{ _MMIO(0x9888), 0x1adb0254 },
-	{ _MMIO(0x9888), 0x0e9faa00 },
-	{ _MMIO(0x9888), 0x109f02aa },
-	{ _MMIO(0x9888), 0x0eb84000 },
-	{ _MMIO(0x9888), 0x16b84000 },
-	{ _MMIO(0x9888), 0x18b8156a },
-	{ _MMIO(0x9888), 0x06b98000 },
-	{ _MMIO(0x9888), 0x08b9a000 },
-	{ _MMIO(0x9888), 0x0ab9a000 },
-	{ _MMIO(0x9888), 0x0cb9a000 },
-	{ _MMIO(0x9888), 0x0eb9a000 },
-	{ _MMIO(0x9888), 0x18baa000 },
-	{ _MMIO(0x9888), 0x1aba0002 },
-	{ _MMIO(0x9888), 0x16934000 },
-	{ _MMIO(0x9888), 0x1893000a },
-	{ _MMIO(0x9888), 0x0a947000 },
-	{ _MMIO(0x9888), 0x0c95c5c1 },
-	{ _MMIO(0x9888), 0x0e9500c3 },
-	{ _MMIO(0x9888), 0x10950000 },
-	{ _MMIO(0x9888), 0x0eb38000 },
-	{ _MMIO(0x9888), 0x16b30040 },
-	{ _MMIO(0x9888), 0x18b30020 },
-	{ _MMIO(0x9888), 0x06b48000 },
-	{ _MMIO(0x9888), 0x08b41000 },
-	{ _MMIO(0x9888), 0x0ab48000 },
-	{ _MMIO(0x9888), 0x06b5c500 },
-	{ _MMIO(0x9888), 0x08b500c3 },
-	{ _MMIO(0x9888), 0x0eb5c100 },
-	{ _MMIO(0x9888), 0x10b50000 },
-	{ _MMIO(0x9888), 0x16d31500 },
-	{ _MMIO(0x9888), 0x08d4e000 },
-	{ _MMIO(0x9888), 0x08d5c100 },
-	{ _MMIO(0x9888), 0x0ad5c3c5 },
-	{ _MMIO(0x9888), 0x10d50000 },
-	{ _MMIO(0x9888), 0x0d88f800 },
-	{ _MMIO(0x9888), 0x0f88000f },
-	{ _MMIO(0x9888), 0x038a8000 },
-	{ _MMIO(0x9888), 0x058a8000 },
-	{ _MMIO(0x9888), 0x078a8000 },
-	{ _MMIO(0x9888), 0x098a8000 },
-	{ _MMIO(0x9888), 0x0b8a8000 },
-	{ _MMIO(0x9888), 0x0d8a8000 },
-	{ _MMIO(0x9888), 0x258baaa5 },
-	{ _MMIO(0x9888), 0x278b002a },
-	{ _MMIO(0x9888), 0x238b2a80 },
-	{ _MMIO(0x9888), 0x0f8c4000 },
-	{ _MMIO(0x9888), 0x178c2000 },
-	{ _MMIO(0x9888), 0x198c5500 },
-	{ _MMIO(0x9888), 0x1b8c0015 },
-	{ _MMIO(0x9888), 0x078d8000 },
-	{ _MMIO(0x9888), 0x098da000 },
-	{ _MMIO(0x9888), 0x0b8da000 },
-	{ _MMIO(0x9888), 0x0d8da000 },
-	{ _MMIO(0x9888), 0x0f8da000 },
-	{ _MMIO(0x9888), 0x2185aaaa },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x1f85aa00 },
-	{ _MMIO(0x9888), 0x0f834000 },
-	{ _MMIO(0x9888), 0x19835400 },
-	{ _MMIO(0x9888), 0x1b830155 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x0784c000 },
-	{ _MMIO(0x9888), 0x0984c000 },
-	{ _MMIO(0x9888), 0x0b84c000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x01848000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x1780c000 },
-	{ _MMIO(0x9888), 0x1980c000 },
-	{ _MMIO(0x9888), 0x1b80c000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x11808000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x43800c42 },
-	{ _MMIO(0x9888), 0x51800000 },
-	{ _MMIO(0x9888), 0x45800063 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x47800800 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3f8014a4 },
-	{ _MMIO(0x9888), 0x41801042 },
-};
-
-static int
-get_tdl_1_mux_config(struct drm_i915_private *dev_priv,
-		     const struct i915_oa_reg **regs,
-		     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_tdl_1;
-	lens[n] = ARRAY_SIZE(mux_config_tdl_1);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_tdl_2[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x0000fdff },
-	{ _MMIO(0x2778), 0x00000000 },
-	{ _MMIO(0x277c), 0x0000fe7f },
-	{ _MMIO(0x2780), 0x00000000 },
-	{ _MMIO(0x2784), 0x0000ff9f },
-	{ _MMIO(0x2788), 0x00000000 },
-	{ _MMIO(0x278c), 0x0000ffe7 },
-	{ _MMIO(0x2790), 0x00000002 },
-	{ _MMIO(0x2794), 0x0000fffb },
-	{ _MMIO(0x2798), 0x00000002 },
-	{ _MMIO(0x279c), 0x0000fffd },
-};
-
-static const struct i915_oa_reg flex_eu_config_tdl_2[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_tdl_2[] = {
-	{ _MMIO(0x9888), 0x16150000 },
-	{ _MMIO(0x9888), 0x16350000 },
-	{ _MMIO(0x9888), 0x16550000 },
-	{ _MMIO(0x9888), 0x16952e60 },
-	{ _MMIO(0x9888), 0x16b54d60 },
-	{ _MMIO(0x9888), 0x16d52e60 },
-	{ _MMIO(0x9888), 0x065c8000 },
-	{ _MMIO(0x9888), 0x085cc000 },
-	{ _MMIO(0x9888), 0x0a5cc000 },
-	{ _MMIO(0x9888), 0x0c5c4000 },
-	{ _MMIO(0x9888), 0x0e3d8000 },
-	{ _MMIO(0x9888), 0x183da000 },
-	{ _MMIO(0x9888), 0x06588000 },
-	{ _MMIO(0x9888), 0x08588000 },
-	{ _MMIO(0x9888), 0x0a584000 },
-	{ _MMIO(0x9888), 0x0e5b4000 },
-	{ _MMIO(0x9888), 0x185b5800 },
-	{ _MMIO(0x9888), 0x1a5b000a },
-	{ _MMIO(0x9888), 0x0e1faa00 },
-	{ _MMIO(0x9888), 0x101f02aa },
-	{ _MMIO(0x9888), 0x0e384000 },
-	{ _MMIO(0x9888), 0x16384000 },
-	{ _MMIO(0x9888), 0x18382a55 },
-	{ _MMIO(0x9888), 0x06398000 },
-	{ _MMIO(0x9888), 0x0839a000 },
-	{ _MMIO(0x9888), 0x0a39a000 },
-	{ _MMIO(0x9888), 0x0c39a000 },
-	{ _MMIO(0x9888), 0x0e39a000 },
-	{ _MMIO(0x9888), 0x1a3a02a0 },
-	{ _MMIO(0x9888), 0x0e138000 },
-	{ _MMIO(0x9888), 0x16130500 },
-	{ _MMIO(0x9888), 0x06148000 },
-	{ _MMIO(0x9888), 0x08146000 },
-	{ _MMIO(0x9888), 0x0615c100 },
-	{ _MMIO(0x9888), 0x0815c500 },
-	{ _MMIO(0x9888), 0x0a1500c3 },
-	{ _MMIO(0x9888), 0x10150000 },
-	{ _MMIO(0x9888), 0x16335040 },
-	{ _MMIO(0x9888), 0x08349000 },
-	{ _MMIO(0x9888), 0x0a341000 },
-	{ _MMIO(0x9888), 0x083500c1 },
-	{ _MMIO(0x9888), 0x0a35c500 },
-	{ _MMIO(0x9888), 0x0c3500c3 },
-	{ _MMIO(0x9888), 0x10350000 },
-	{ _MMIO(0x9888), 0x1853002a },
-	{ _MMIO(0x9888), 0x0a54e000 },
-	{ _MMIO(0x9888), 0x0c55c500 },
-	{ _MMIO(0x9888), 0x0e55c1c3 },
-	{ _MMIO(0x9888), 0x10550000 },
-	{ _MMIO(0x9888), 0x00dc8000 },
-	{ _MMIO(0x9888), 0x02dcc000 },
-	{ _MMIO(0x9888), 0x04dc4000 },
-	{ _MMIO(0x9888), 0x04bd8000 },
-	{ _MMIO(0x9888), 0x06bd8000 },
-	{ _MMIO(0x9888), 0x02d8c000 },
-	{ _MMIO(0x9888), 0x02db8000 },
-	{ _MMIO(0x9888), 0x04db4000 },
-	{ _MMIO(0x9888), 0x06db4000 },
-	{ _MMIO(0x9888), 0x08db8000 },
-	{ _MMIO(0x9888), 0x0c9fa000 },
-	{ _MMIO(0x9888), 0x0e9f00aa },
-	{ _MMIO(0x9888), 0x02b84000 },
-	{ _MMIO(0x9888), 0x04b84000 },
-	{ _MMIO(0x9888), 0x06b84000 },
-	{ _MMIO(0x9888), 0x08b84000 },
-	{ _MMIO(0x9888), 0x0ab88000 },
-	{ _MMIO(0x9888), 0x0cb88000 },
-	{ _MMIO(0x9888), 0x00b98000 },
-	{ _MMIO(0x9888), 0x02b9a000 },
-	{ _MMIO(0x9888), 0x04b9a000 },
-	{ _MMIO(0x9888), 0x06b92000 },
-	{ _MMIO(0x9888), 0x0aba8000 },
-	{ _MMIO(0x9888), 0x0cba8000 },
-	{ _MMIO(0x9888), 0x04938000 },
-	{ _MMIO(0x9888), 0x06938000 },
-	{ _MMIO(0x9888), 0x0494c000 },
-	{ _MMIO(0x9888), 0x0295cfc7 },
-	{ _MMIO(0x9888), 0x10950000 },
-	{ _MMIO(0x9888), 0x02b38000 },
-	{ _MMIO(0x9888), 0x08b38000 },
-	{ _MMIO(0x9888), 0x04b42000 },
-	{ _MMIO(0x9888), 0x06b41000 },
-	{ _MMIO(0x9888), 0x00b5c700 },
-	{ _MMIO(0x9888), 0x04b500cf },
-	{ _MMIO(0x9888), 0x10b50000 },
-	{ _MMIO(0x9888), 0x0ad38000 },
-	{ _MMIO(0x9888), 0x0cd38000 },
-	{ _MMIO(0x9888), 0x06d46000 },
-	{ _MMIO(0x9888), 0x04d5c700 },
-	{ _MMIO(0x9888), 0x06d500cf },
-	{ _MMIO(0x9888), 0x10d50000 },
-	{ _MMIO(0x9888), 0x03888000 },
-	{ _MMIO(0x9888), 0x05888000 },
-	{ _MMIO(0x9888), 0x07888000 },
-	{ _MMIO(0x9888), 0x09888000 },
-	{ _MMIO(0x9888), 0x0b888000 },
-	{ _MMIO(0x9888), 0x0d880400 },
-	{ _MMIO(0x9888), 0x0f8a8000 },
-	{ _MMIO(0x9888), 0x198a8000 },
-	{ _MMIO(0x9888), 0x1b8aaaa0 },
-	{ _MMIO(0x9888), 0x1d8a0002 },
-	{ _MMIO(0x9888), 0x258b555a },
-	{ _MMIO(0x9888), 0x278b0015 },
-	{ _MMIO(0x9888), 0x238b5500 },
-	{ _MMIO(0x9888), 0x038c4000 },
-	{ _MMIO(0x9888), 0x058c4000 },
-	{ _MMIO(0x9888), 0x078c4000 },
-	{ _MMIO(0x9888), 0x098c4000 },
-	{ _MMIO(0x9888), 0x0b8c4000 },
-	{ _MMIO(0x9888), 0x0d8c4000 },
-	{ _MMIO(0x9888), 0x018d8000 },
-	{ _MMIO(0x9888), 0x038da000 },
-	{ _MMIO(0x9888), 0x058da000 },
-	{ _MMIO(0x9888), 0x078d2000 },
-	{ _MMIO(0x9888), 0x2185aaaa },
-	{ _MMIO(0x9888), 0x2385002a },
-	{ _MMIO(0x9888), 0x1f85aa00 },
-	{ _MMIO(0x9888), 0x0f834000 },
-	{ _MMIO(0x9888), 0x19835400 },
-	{ _MMIO(0x9888), 0x1b830155 },
-	{ _MMIO(0x9888), 0x03834000 },
-	{ _MMIO(0x9888), 0x05834000 },
-	{ _MMIO(0x9888), 0x07834000 },
-	{ _MMIO(0x9888), 0x09834000 },
-	{ _MMIO(0x9888), 0x0b834000 },
-	{ _MMIO(0x9888), 0x0d834000 },
-	{ _MMIO(0x9888), 0x0784c000 },
-	{ _MMIO(0x9888), 0x0984c000 },
-	{ _MMIO(0x9888), 0x0b84c000 },
-	{ _MMIO(0x9888), 0x0d84c000 },
-	{ _MMIO(0x9888), 0x0f84c000 },
-	{ _MMIO(0x9888), 0x01848000 },
-	{ _MMIO(0x9888), 0x0384c000 },
-	{ _MMIO(0x9888), 0x0584c000 },
-	{ _MMIO(0x9888), 0x1780c000 },
-	{ _MMIO(0x9888), 0x1980c000 },
-	{ _MMIO(0x9888), 0x1b80c000 },
-	{ _MMIO(0x9888), 0x1d80c000 },
-	{ _MMIO(0x9888), 0x1f80c000 },
-	{ _MMIO(0x9888), 0x11808000 },
-	{ _MMIO(0x9888), 0x1380c000 },
-	{ _MMIO(0x9888), 0x1580c000 },
-	{ _MMIO(0x9888), 0x4f800000 },
-	{ _MMIO(0x9888), 0x43800882 },
-	{ _MMIO(0x9888), 0x51800000 },
-	{ _MMIO(0x9888), 0x45801082 },
-	{ _MMIO(0x9888), 0x53800000 },
-	{ _MMIO(0x9888), 0x478014a5 },
-	{ _MMIO(0x9888), 0x21800000 },
-	{ _MMIO(0x9888), 0x31800000 },
-	{ _MMIO(0x9888), 0x4d800000 },
-	{ _MMIO(0x9888), 0x3f800002 },
-	{ _MMIO(0x9888), 0x41800c62 },
-};
-
-static int
-get_tdl_2_mux_config(struct drm_i915_private *dev_priv,
-		     const struct i915_oa_reg **regs,
-		     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_tdl_2;
-	lens[n] = ARRAY_SIZE(mux_config_tdl_2);
-	n++;
-
-	return n;
-}
-
 static const struct i915_oa_reg b_counter_config_test_oa[] = {
 	{ _MMIO(0x2740), 0x00000000 },
 	{ _MMIO(0x2744), 0x00800000 },
@@ -1997,6 +60,7 @@ static const struct i915_oa_reg flex_eu_config_test_oa[] = {
 };
 
 static const struct i915_oa_reg mux_config_test_oa[] = {
+	{ _MMIO(0x9840), 0x000000a0 },
 	{ _MMIO(0x9888), 0x59800000 },
 	{ _MMIO(0x9888), 0x59800001 },
 	{ _MMIO(0x9888), 0x338b0000 },
@@ -2008,866 +72,38 @@ static const struct i915_oa_reg mux_config_test_oa[] = {
 	{ _MMIO(0x9888), 0x57800000 },
 	{ _MMIO(0x1823a4), 0x00000000 },
 	{ _MMIO(0x9888), 0x59800000 },
-};
-
-static int
-get_test_oa_mux_config(struct drm_i915_private *dev_priv,
-		       const struct i915_oa_reg **regs,
-		       int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_test_oa;
-	lens[n] = ARRAY_SIZE(mux_config_test_oa);
-	n++;
-
-	return n;
-}
-
-int i915_oa_select_metric_set_chv(struct drm_i915_private *dev_priv)
-{
-	dev_priv->perf.oa.n_mux_configs = 0;
-	dev_priv->perf.oa.b_counter_regs = NULL;
-	dev_priv->perf.oa.b_counter_regs_len = 0;
-	dev_priv->perf.oa.flex_regs = NULL;
-	dev_priv->perf.oa.flex_regs_len = 0;
-
-	switch (dev_priv->perf.oa.metrics_set) {
-	case METRIC_SET_ID_RENDER_BASIC:
-		dev_priv->perf.oa.n_mux_configs =
-			get_render_basic_mux_config(dev_priv,
-						    dev_priv->perf.oa.mux_regs,
-						    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_BASIC\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_render_basic;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_render_basic);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_render_basic;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_render_basic);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_BASIC:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_basic_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_BASIC\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_basic;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_basic);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_basic;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_basic);
-
-		return 0;
-	case METRIC_SET_ID_RENDER_PIPE_PROFILE:
-		dev_priv->perf.oa.n_mux_configs =
-			get_render_pipe_profile_mux_config(dev_priv,
-							   dev_priv->perf.oa.mux_regs,
-							   dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_PIPE_PROFILE\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_render_pipe_profile;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_render_pipe_profile);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_render_pipe_profile;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_render_pipe_profile);
-
-		return 0;
-	case METRIC_SET_ID_HDC_AND_SF:
-		dev_priv->perf.oa.n_mux_configs =
-			get_hdc_and_sf_mux_config(dev_priv,
-						  dev_priv->perf.oa.mux_regs,
-						  dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"HDC_AND_SF\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_hdc_and_sf;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_hdc_and_sf);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_hdc_and_sf;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_hdc_and_sf);
-
-		return 0;
-	case METRIC_SET_ID_L3_1:
-		dev_priv->perf.oa.n_mux_configs =
-			get_l3_1_mux_config(dev_priv,
-					    dev_priv->perf.oa.mux_regs,
-					    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_1\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_l3_1;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_l3_1);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_l3_1;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_l3_1);
-
-		return 0;
-	case METRIC_SET_ID_L3_2:
-		dev_priv->perf.oa.n_mux_configs =
-			get_l3_2_mux_config(dev_priv,
-					    dev_priv->perf.oa.mux_regs,
-					    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_2\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_l3_2;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_l3_2);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_l3_2;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_l3_2);
-
-		return 0;
-	case METRIC_SET_ID_L3_3:
-		dev_priv->perf.oa.n_mux_configs =
-			get_l3_3_mux_config(dev_priv,
-					    dev_priv->perf.oa.mux_regs,
-					    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_3\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_l3_3;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_l3_3);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_l3_3;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_l3_3);
-
-		return 0;
-	case METRIC_SET_ID_L3_4:
-		dev_priv->perf.oa.n_mux_configs =
-			get_l3_4_mux_config(dev_priv,
-					    dev_priv->perf.oa.mux_regs,
-					    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_4\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_l3_4;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_l3_4);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_l3_4;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_l3_4);
-
-		return 0;
-	case METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND:
-		dev_priv->perf.oa.n_mux_configs =
-			get_rasterizer_and_pixel_backend_mux_config(dev_priv,
-								    dev_priv->perf.oa.mux_regs,
-								    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RASTERIZER_AND_PIXEL_BACKEND\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_rasterizer_and_pixel_backend;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_rasterizer_and_pixel_backend);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_rasterizer_and_pixel_backend;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_rasterizer_and_pixel_backend);
-
-		return 0;
-	case METRIC_SET_ID_SAMPLER_1:
-		dev_priv->perf.oa.n_mux_configs =
-			get_sampler_1_mux_config(dev_priv,
-						 dev_priv->perf.oa.mux_regs,
-						 dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"SAMPLER_1\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_sampler_1;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_sampler_1);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_sampler_1;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_sampler_1);
-
-		return 0;
-	case METRIC_SET_ID_SAMPLER_2:
-		dev_priv->perf.oa.n_mux_configs =
-			get_sampler_2_mux_config(dev_priv,
-						 dev_priv->perf.oa.mux_regs,
-						 dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"SAMPLER_2\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_sampler_2;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_sampler_2);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_sampler_2;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_sampler_2);
-
-		return 0;
-	case METRIC_SET_ID_TDL_1:
-		dev_priv->perf.oa.n_mux_configs =
-			get_tdl_1_mux_config(dev_priv,
-					     dev_priv->perf.oa.mux_regs,
-					     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_1\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_tdl_1;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_tdl_1);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_tdl_1;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_tdl_1);
-
-		return 0;
-	case METRIC_SET_ID_TDL_2:
-		dev_priv->perf.oa.n_mux_configs =
-			get_tdl_2_mux_config(dev_priv,
-					     dev_priv->perf.oa.mux_regs,
-					     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_2\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_tdl_2;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_tdl_2);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_tdl_2;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_tdl_2);
-
-		return 0;
-	case METRIC_SET_ID_TEST_OA:
-		dev_priv->perf.oa.n_mux_configs =
-			get_test_oa_mux_config(dev_priv,
-					       dev_priv->perf.oa.mux_regs,
-					       dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TEST_OA\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_test_oa;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_test_oa);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_test_oa;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_test_oa);
-
-		return 0;
-	default:
-		return -ENODEV;
-	}
-}
-
-static ssize_t
-show_render_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_BASIC);
-}
-
-static struct device_attribute dev_attr_render_basic_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_render_basic_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_render_basic[] = {
-	&dev_attr_render_basic_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_render_basic = {
-	.name = "9d8a3af5-c02c-4a4a-b947-f1672469e0fb",
-	.attrs =  attrs_render_basic,
-};
-
-static ssize_t
-show_compute_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_BASIC);
-}
-
-static struct device_attribute dev_attr_compute_basic_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_basic_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_basic[] = {
-	&dev_attr_compute_basic_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_basic = {
-	.name = "f522a89c-ecd1-4522-8331-3383c54af5f5",
-	.attrs =  attrs_compute_basic,
-};
-
-static ssize_t
-show_render_pipe_profile_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_PIPE_PROFILE);
-}
-
-static struct device_attribute dev_attr_render_pipe_profile_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_render_pipe_profile_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_render_pipe_profile[] = {
-	&dev_attr_render_pipe_profile_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_render_pipe_profile = {
-	.name = "a9ccc03d-a943-4e6b-9cd6-13e063075927",
-	.attrs =  attrs_render_pipe_profile,
-};
-
-static ssize_t
-show_hdc_and_sf_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_HDC_AND_SF);
-}
-
-static struct device_attribute dev_attr_hdc_and_sf_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_hdc_and_sf_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_hdc_and_sf[] = {
-	&dev_attr_hdc_and_sf_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_hdc_and_sf = {
-	.name = "2cf0c064-68df-4fac-9b3f-57f51ca8a069",
-	.attrs =  attrs_hdc_and_sf,
-};
-
-static ssize_t
-show_l3_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_1);
-}
-
-static struct device_attribute dev_attr_l3_1_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_l3_1_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_l3_1[] = {
-	&dev_attr_l3_1_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_l3_1 = {
-	.name = "78a87ff9-543a-49ce-95ea-26d86071ea93",
-	.attrs =  attrs_l3_1,
-};
-
-static ssize_t
-show_l3_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_2);
-}
-
-static struct device_attribute dev_attr_l3_2_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_l3_2_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_l3_2[] = {
-	&dev_attr_l3_2_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_l3_2 = {
-	.name = "9f2cece5-7bfe-4320-ad66-8c7cc526bec5",
-	.attrs =  attrs_l3_2,
-};
-
-static ssize_t
-show_l3_3_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_3);
-}
-
-static struct device_attribute dev_attr_l3_3_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_l3_3_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_l3_3[] = {
-	&dev_attr_l3_3_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_l3_3 = {
-	.name = "d890ef38-d309-47e4-b8b5-aa779bb19ab0",
-	.attrs =  attrs_l3_3,
-};
-
-static ssize_t
-show_l3_4_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_4);
-}
-
-static struct device_attribute dev_attr_l3_4_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_l3_4_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_l3_4[] = {
-	&dev_attr_l3_4_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_l3_4 = {
-	.name = "5fdff4a6-9dc8-45e1-bfda-ef54869fbdd4",
-	.attrs =  attrs_l3_4,
-};
-
-static ssize_t
-show_rasterizer_and_pixel_backend_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND);
-}
-
-static struct device_attribute dev_attr_rasterizer_and_pixel_backend_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_rasterizer_and_pixel_backend_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_rasterizer_and_pixel_backend[] = {
-	&dev_attr_rasterizer_and_pixel_backend_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_rasterizer_and_pixel_backend = {
-	.name = "2c0e45e1-7e2c-4a14-ae00-0b7ec868b8aa",
-	.attrs =  attrs_rasterizer_and_pixel_backend,
-};
-
-static ssize_t
-show_sampler_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_SAMPLER_1);
-}
-
-static struct device_attribute dev_attr_sampler_1_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_sampler_1_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_sampler_1[] = {
-	&dev_attr_sampler_1_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_sampler_1 = {
-	.name = "71148d78-baf5-474f-878a-e23158d0265d",
-	.attrs =  attrs_sampler_1,
-};
-
-static ssize_t
-show_sampler_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_SAMPLER_2);
-}
-
-static struct device_attribute dev_attr_sampler_2_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_sampler_2_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_sampler_2[] = {
-	&dev_attr_sampler_2_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_sampler_2 = {
-	.name = "b996a2b7-c59c-492d-877a-8cd54fd6df84",
-	.attrs =  attrs_sampler_2,
-};
-
-static ssize_t
-show_tdl_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_1);
-}
-
-static struct device_attribute dev_attr_tdl_1_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_tdl_1_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_tdl_1[] = {
-	&dev_attr_tdl_1_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_tdl_1 = {
-	.name = "eb2fecba-b431-42e7-8261-fe9429a6e67a",
-	.attrs =  attrs_tdl_1,
-};
-
-static ssize_t
-show_tdl_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_2);
-}
-
-static struct device_attribute dev_attr_tdl_2_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_tdl_2_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_tdl_2[] = {
-	&dev_attr_tdl_2_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_tdl_2 = {
-	.name = "60749470-a648-4a4b-9f10-dbfe1e36e44d",
-	.attrs =  attrs_tdl_2,
+	{ _MMIO(0x9840), 0x00000080 },
 };
 
 static ssize_t
 show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TEST_OA);
+	return sprintf(buf, "1\n");
 }
 
-static struct device_attribute dev_attr_test_oa_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_test_oa_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_test_oa[] = {
-	&dev_attr_test_oa_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_test_oa = {
-	.name = "4a534b07-cba3-414d-8d60-874830e883aa",
-	.attrs =  attrs_test_oa,
-};
-
-int
-i915_perf_register_sysfs_chv(struct drm_i915_private *dev_priv)
+void
+i915_perf_load_test_config_chv(struct drm_i915_private *dev_priv)
 {
-	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
-	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
-	int ret = 0;
+	strncpy(dev_priv->perf.oa.test_config.uuid,
+		"4a534b07-cba3-414d-8d60-874830e883aa",
+		UUID_STRING_LEN);
+	dev_priv->perf.oa.test_config.id = 1;
 
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-		if (ret)
-			goto error_render_basic;
-	}
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-		if (ret)
-			goto error_compute_basic;
-	}
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-		if (ret)
-			goto error_render_pipe_profile;
-	}
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-		if (ret)
-			goto error_hdc_and_sf;
-	}
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-		if (ret)
-			goto error_l3_1;
-	}
-	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_2);
-		if (ret)
-			goto error_l3_2;
-	}
-	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_3);
-		if (ret)
-			goto error_l3_3;
-	}
-	if (get_l3_4_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_4);
-		if (ret)
-			goto error_l3_4;
-	}
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-		if (ret)
-			goto error_rasterizer_and_pixel_backend;
-	}
-	if (get_sampler_1_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_sampler_1);
-		if (ret)
-			goto error_sampler_1;
-	}
-	if (get_sampler_2_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_sampler_2);
-		if (ret)
-			goto error_sampler_2;
-	}
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-		if (ret)
-			goto error_tdl_1;
-	}
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-		if (ret)
-			goto error_tdl_2;
-	}
-	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_test_oa);
-		if (ret)
-			goto error_test_oa;
-	}
+	dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
+	dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa);
 
-	return 0;
+	dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa;
+	dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa);
 
-error_test_oa:
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-error_tdl_2:
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-error_tdl_1:
-	if (get_sampler_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler_2);
-error_sampler_2:
-	if (get_sampler_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler_1);
-error_sampler_1:
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-error_rasterizer_and_pixel_backend:
-	if (get_l3_4_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_4);
-error_l3_4:
-	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
-error_l3_3:
-	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
-error_l3_2:
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-error_l3_1:
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-error_hdc_and_sf:
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-error_render_pipe_profile:
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-error_compute_basic:
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-error_render_basic:
-	return ret;
-}
+	dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa;
+	dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa);
 
-void
-i915_perf_unregister_sysfs_chv(struct drm_i915_private *dev_priv)
-{
-	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
-	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+	dev_priv->perf.oa.test_config.sysfs_metric.name = "4a534b07-cba3-414d-8d60-874830e883aa";
+	dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs;
+
+	dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr;
 
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
-	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
-	if (get_l3_4_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_4);
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-	if (get_sampler_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler_1);
-	if (get_sampler_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler_2);
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id";
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444;
+	dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id;
 }
diff --git a/drivers/gpu/drm/i915/i915_oa_chv.h b/drivers/gpu/drm/i915/i915_oa_chv.h
index 8b8bdc26d726..b9622496979e 100644
--- a/drivers/gpu/drm/i915/i915_oa_chv.h
+++ b/drivers/gpu/drm/i915/i915_oa_chv.h
@@ -29,12 +29,6 @@
 #ifndef __I915_OA_CHV_H__
 #define __I915_OA_CHV_H__
 
-extern int i915_oa_n_builtin_metric_sets_chv;
-
-extern int i915_oa_select_metric_set_chv(struct drm_i915_private *dev_priv);
-
-extern int i915_perf_register_sysfs_chv(struct drm_i915_private *dev_priv);
-
-extern void i915_perf_unregister_sysfs_chv(struct drm_i915_private *dev_priv);
+extern void i915_perf_load_test_config_chv(struct drm_i915_private *dev_priv);
 
 #endif
diff --git a/drivers/gpu/drm/i915/i915_oa_glk.c b/drivers/gpu/drm/i915/i915_oa_glk.c
index 2f356d51bff8..4ee527e4c926 100644
--- a/drivers/gpu/drm/i915/i915_oa_glk.c
+++ b/drivers/gpu/drm/i915/i915_oa_glk.c
@@ -31,1614 +31,6 @@
 #include "i915_drv.h"
 #include "i915_oa_glk.h"
 
-enum metric_set_id {
-	METRIC_SET_ID_RENDER_BASIC = 1,
-	METRIC_SET_ID_COMPUTE_BASIC,
-	METRIC_SET_ID_RENDER_PIPE_PROFILE,
-	METRIC_SET_ID_MEMORY_READS,
-	METRIC_SET_ID_MEMORY_WRITES,
-	METRIC_SET_ID_COMPUTE_EXTENDED,
-	METRIC_SET_ID_COMPUTE_L3_CACHE,
-	METRIC_SET_ID_HDC_AND_SF,
-	METRIC_SET_ID_L3_1,
-	METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND,
-	METRIC_SET_ID_SAMPLER,
-	METRIC_SET_ID_TDL_1,
-	METRIC_SET_ID_TDL_2,
-	METRIC_SET_ID_COMPUTE_EXTRA,
-	METRIC_SET_ID_TEST_OA,
-};
-
-int i915_oa_n_builtin_metric_sets_glk = 15;
-
-static const struct i915_oa_reg b_counter_config_render_basic[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2740), 0x00000000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_render_basic[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_render_basic[] = {
-	{ _MMIO(0x9888), 0x166c00f0 },
-	{ _MMIO(0x9888), 0x12120280 },
-	{ _MMIO(0x9888), 0x12320280 },
-	{ _MMIO(0x9888), 0x11930317 },
-	{ _MMIO(0x9888), 0x159303df },
-	{ _MMIO(0x9888), 0x3f900c00 },
-	{ _MMIO(0x9888), 0x419000a0 },
-	{ _MMIO(0x9888), 0x002d1000 },
-	{ _MMIO(0x9888), 0x062d4000 },
-	{ _MMIO(0x9888), 0x082d5000 },
-	{ _MMIO(0x9888), 0x0a2d1000 },
-	{ _MMIO(0x9888), 0x0c2e0800 },
-	{ _MMIO(0x9888), 0x0e2e5900 },
-	{ _MMIO(0x9888), 0x0a4c8000 },
-	{ _MMIO(0x9888), 0x0c4c8000 },
-	{ _MMIO(0x9888), 0x0e4c4000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084e8000 },
-	{ _MMIO(0x9888), 0x0a4e2000 },
-	{ _MMIO(0x9888), 0x1c4f0010 },
-	{ _MMIO(0x9888), 0x0a6c0053 },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1a0fcc00 },
-	{ _MMIO(0x9888), 0x1c0f0002 },
-	{ _MMIO(0x9888), 0x1c2c0040 },
-	{ _MMIO(0x9888), 0x00101000 },
-	{ _MMIO(0x9888), 0x04101000 },
-	{ _MMIO(0x9888), 0x00114000 },
-	{ _MMIO(0x9888), 0x08114000 },
-	{ _MMIO(0x9888), 0x00120020 },
-	{ _MMIO(0x9888), 0x08120021 },
-	{ _MMIO(0x9888), 0x00141000 },
-	{ _MMIO(0x9888), 0x08141000 },
-	{ _MMIO(0x9888), 0x02308000 },
-	{ _MMIO(0x9888), 0x04302000 },
-	{ _MMIO(0x9888), 0x06318000 },
-	{ _MMIO(0x9888), 0x08318000 },
-	{ _MMIO(0x9888), 0x06320800 },
-	{ _MMIO(0x9888), 0x08320840 },
-	{ _MMIO(0x9888), 0x00320000 },
-	{ _MMIO(0x9888), 0x06344000 },
-	{ _MMIO(0x9888), 0x08344000 },
-	{ _MMIO(0x9888), 0x0d931831 },
-	{ _MMIO(0x9888), 0x0f939f3f },
-	{ _MMIO(0x9888), 0x01939e80 },
-	{ _MMIO(0x9888), 0x039303bc },
-	{ _MMIO(0x9888), 0x0593000e },
-	{ _MMIO(0x9888), 0x1993002a },
-	{ _MMIO(0x9888), 0x07930000 },
-	{ _MMIO(0x9888), 0x09930000 },
-	{ _MMIO(0x9888), 0x1d900177 },
-	{ _MMIO(0x9888), 0x1f900187 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x13904000 },
-	{ _MMIO(0x9888), 0x21904000 },
-	{ _MMIO(0x9888), 0x23904000 },
-	{ _MMIO(0x9888), 0x25904000 },
-	{ _MMIO(0x9888), 0x27904000 },
-	{ _MMIO(0x9888), 0x2b904000 },
-	{ _MMIO(0x9888), 0x2d904000 },
-	{ _MMIO(0x9888), 0x2f904000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x15904000 },
-	{ _MMIO(0x9888), 0x17904000 },
-	{ _MMIO(0x9888), 0x19904000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x53901110 },
-	{ _MMIO(0x9888), 0x43900423 },
-	{ _MMIO(0x9888), 0x55900111 },
-	{ _MMIO(0x9888), 0x47900c02 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900020 },
-	{ _MMIO(0x9888), 0x59901111 },
-	{ _MMIO(0x9888), 0x4b900421 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4d900001 },
-	{ _MMIO(0x9888), 0x45900821 },
-};
-
-static int
-get_render_basic_mux_config(struct drm_i915_private *dev_priv,
-			    const struct i915_oa_reg **regs,
-			    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_render_basic;
-	lens[n] = ARRAY_SIZE(mux_config_render_basic);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_basic[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2740), 0x00000000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_basic[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00778008 },
-	{ _MMIO(0xe45c), 0x00088078 },
-	{ _MMIO(0xe55c), 0x00808708 },
-	{ _MMIO(0xe65c), 0x00a08908 },
-};
-
-static const struct i915_oa_reg mux_config_compute_basic[] = {
-	{ _MMIO(0x9888), 0x104f00e0 },
-	{ _MMIO(0x9888), 0x124f1c00 },
-	{ _MMIO(0x9888), 0x39900340 },
-	{ _MMIO(0x9888), 0x3f900c00 },
-	{ _MMIO(0x9888), 0x41900000 },
-	{ _MMIO(0x9888), 0x002d5000 },
-	{ _MMIO(0x9888), 0x062d4000 },
-	{ _MMIO(0x9888), 0x082d4000 },
-	{ _MMIO(0x9888), 0x0a2d1000 },
-	{ _MMIO(0x9888), 0x0c2d5000 },
-	{ _MMIO(0x9888), 0x0e2d4000 },
-	{ _MMIO(0x9888), 0x0c2e1400 },
-	{ _MMIO(0x9888), 0x0e2e5100 },
-	{ _MMIO(0x9888), 0x102e0114 },
-	{ _MMIO(0x9888), 0x044cc000 },
-	{ _MMIO(0x9888), 0x0a4c8000 },
-	{ _MMIO(0x9888), 0x0c4c8000 },
-	{ _MMIO(0x9888), 0x0e4c4000 },
-	{ _MMIO(0x9888), 0x104c8000 },
-	{ _MMIO(0x9888), 0x124c8000 },
-	{ _MMIO(0x9888), 0x164c2000 },
-	{ _MMIO(0x9888), 0x004ea000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084e8000 },
-	{ _MMIO(0x9888), 0x0a4e2000 },
-	{ _MMIO(0x9888), 0x0c4ea000 },
-	{ _MMIO(0x9888), 0x0e4e8000 },
-	{ _MMIO(0x9888), 0x004f6b42 },
-	{ _MMIO(0x9888), 0x064f6200 },
-	{ _MMIO(0x9888), 0x084f4100 },
-	{ _MMIO(0x9888), 0x0a4f0061 },
-	{ _MMIO(0x9888), 0x0c4f6c4c },
-	{ _MMIO(0x9888), 0x0e4f4b00 },
-	{ _MMIO(0x9888), 0x1a4f0000 },
-	{ _MMIO(0x9888), 0x1c4f0000 },
-	{ _MMIO(0x9888), 0x180f5000 },
-	{ _MMIO(0x9888), 0x1a0f8800 },
-	{ _MMIO(0x9888), 0x1c0f08a2 },
-	{ _MMIO(0x9888), 0x182c4000 },
-	{ _MMIO(0x9888), 0x1c2c1451 },
-	{ _MMIO(0x9888), 0x1e2c0001 },
-	{ _MMIO(0x9888), 0x1a2c0010 },
-	{ _MMIO(0x9888), 0x01938000 },
-	{ _MMIO(0x9888), 0x0f938000 },
-	{ _MMIO(0x9888), 0x19938a28 },
-	{ _MMIO(0x9888), 0x03938000 },
-	{ _MMIO(0x9888), 0x19900177 },
-	{ _MMIO(0x9888), 0x1b900178 },
-	{ _MMIO(0x9888), 0x1d900125 },
-	{ _MMIO(0x9888), 0x1f900123 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x13904000 },
-	{ _MMIO(0x9888), 0x21904000 },
-	{ _MMIO(0x9888), 0x25904000 },
-	{ _MMIO(0x9888), 0x27904000 },
-	{ _MMIO(0x9888), 0x2b904000 },
-	{ _MMIO(0x9888), 0x2d904000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x15904000 },
-	{ _MMIO(0x9888), 0x53901000 },
-	{ _MMIO(0x9888), 0x43900000 },
-	{ _MMIO(0x9888), 0x55900111 },
-	{ _MMIO(0x9888), 0x47900000 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x4b900000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4d900000 },
-	{ _MMIO(0x9888), 0x45900000 },
-};
-
-static int
-get_compute_basic_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_basic;
-	lens[n] = ARRAY_SIZE(mux_config_compute_basic);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_render_pipe_profile[] = {
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007ffea },
-	{ _MMIO(0x2774), 0x00007ffc },
-	{ _MMIO(0x2778), 0x0007affa },
-	{ _MMIO(0x277c), 0x0000f5fd },
-	{ _MMIO(0x2780), 0x00079ffa },
-	{ _MMIO(0x2784), 0x0000f3fb },
-	{ _MMIO(0x2788), 0x0007bf7a },
-	{ _MMIO(0x278c), 0x0000f7e7 },
-	{ _MMIO(0x2790), 0x0007fefa },
-	{ _MMIO(0x2794), 0x0000f7cf },
-	{ _MMIO(0x2798), 0x00077ffa },
-	{ _MMIO(0x279c), 0x0000efdf },
-	{ _MMIO(0x27a0), 0x0006fffa },
-	{ _MMIO(0x27a4), 0x0000cfbf },
-	{ _MMIO(0x27a8), 0x0003fffa },
-	{ _MMIO(0x27ac), 0x00005f7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_render_pipe_profile[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00015014 },
-	{ _MMIO(0xe658), 0x00025024 },
-	{ _MMIO(0xe758), 0x00035034 },
-	{ _MMIO(0xe45c), 0x00045044 },
-	{ _MMIO(0xe55c), 0x00055054 },
-	{ _MMIO(0xe65c), 0x00065064 },
-};
-
-static const struct i915_oa_reg mux_config_render_pipe_profile[] = {
-	{ _MMIO(0x9888), 0x0c2e001f },
-	{ _MMIO(0x9888), 0x0a2f0000 },
-	{ _MMIO(0x9888), 0x10186800 },
-	{ _MMIO(0x9888), 0x11810019 },
-	{ _MMIO(0x9888), 0x15810013 },
-	{ _MMIO(0x9888), 0x13820020 },
-	{ _MMIO(0x9888), 0x11830020 },
-	{ _MMIO(0x9888), 0x17840000 },
-	{ _MMIO(0x9888), 0x11860007 },
-	{ _MMIO(0x9888), 0x21860000 },
-	{ _MMIO(0x9888), 0x178703e0 },
-	{ _MMIO(0x9888), 0x0c2d8000 },
-	{ _MMIO(0x9888), 0x042d4000 },
-	{ _MMIO(0x9888), 0x062d1000 },
-	{ _MMIO(0x9888), 0x022e5400 },
-	{ _MMIO(0x9888), 0x002e0000 },
-	{ _MMIO(0x9888), 0x0e2e0080 },
-	{ _MMIO(0x9888), 0x082f0040 },
-	{ _MMIO(0x9888), 0x002f0000 },
-	{ _MMIO(0x9888), 0x06143000 },
-	{ _MMIO(0x9888), 0x06174000 },
-	{ _MMIO(0x9888), 0x06180012 },
-	{ _MMIO(0x9888), 0x00180000 },
-	{ _MMIO(0x9888), 0x0d804000 },
-	{ _MMIO(0x9888), 0x0f804000 },
-	{ _MMIO(0x9888), 0x05804000 },
-	{ _MMIO(0x9888), 0x09810200 },
-	{ _MMIO(0x9888), 0x0b810030 },
-	{ _MMIO(0x9888), 0x03810003 },
-	{ _MMIO(0x9888), 0x21819140 },
-	{ _MMIO(0x9888), 0x23819050 },
-	{ _MMIO(0x9888), 0x25810018 },
-	{ _MMIO(0x9888), 0x0b820980 },
-	{ _MMIO(0x9888), 0x03820d80 },
-	{ _MMIO(0x9888), 0x11820000 },
-	{ _MMIO(0x9888), 0x0182c000 },
-	{ _MMIO(0x9888), 0x07828000 },
-	{ _MMIO(0x9888), 0x09824000 },
-	{ _MMIO(0x9888), 0x0f828000 },
-	{ _MMIO(0x9888), 0x0d830004 },
-	{ _MMIO(0x9888), 0x0583000c },
-	{ _MMIO(0x9888), 0x0f831000 },
-	{ _MMIO(0x9888), 0x01848072 },
-	{ _MMIO(0x9888), 0x11840000 },
-	{ _MMIO(0x9888), 0x07848000 },
-	{ _MMIO(0x9888), 0x09844000 },
-	{ _MMIO(0x9888), 0x0f848000 },
-	{ _MMIO(0x9888), 0x07860000 },
-	{ _MMIO(0x9888), 0x09860092 },
-	{ _MMIO(0x9888), 0x0f860400 },
-	{ _MMIO(0x9888), 0x01869100 },
-	{ _MMIO(0x9888), 0x0f870065 },
-	{ _MMIO(0x9888), 0x01870000 },
-	{ _MMIO(0x9888), 0x19930800 },
-	{ _MMIO(0x9888), 0x0b938000 },
-	{ _MMIO(0x9888), 0x0d938000 },
-	{ _MMIO(0x9888), 0x1b952000 },
-	{ _MMIO(0x9888), 0x1d955055 },
-	{ _MMIO(0x9888), 0x1f951455 },
-	{ _MMIO(0x9888), 0x0992a000 },
-	{ _MMIO(0x9888), 0x0f928000 },
-	{ _MMIO(0x9888), 0x1192a800 },
-	{ _MMIO(0x9888), 0x1392028a },
-	{ _MMIO(0x9888), 0x0b92a000 },
-	{ _MMIO(0x9888), 0x0d922000 },
-	{ _MMIO(0x9888), 0x13908000 },
-	{ _MMIO(0x9888), 0x21908000 },
-	{ _MMIO(0x9888), 0x23908000 },
-	{ _MMIO(0x9888), 0x25908000 },
-	{ _MMIO(0x9888), 0x27908000 },
-	{ _MMIO(0x9888), 0x29908000 },
-	{ _MMIO(0x9888), 0x2b908000 },
-	{ _MMIO(0x9888), 0x2d904000 },
-	{ _MMIO(0x9888), 0x2f908000 },
-	{ _MMIO(0x9888), 0x31908000 },
-	{ _MMIO(0x9888), 0x15908000 },
-	{ _MMIO(0x9888), 0x17908000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1d904000 },
-	{ _MMIO(0x9888), 0x1f904000 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x43900c01 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x47900000 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900863 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x4b900061 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4d900000 },
-	{ _MMIO(0x9888), 0x45900c22 },
-};
-
-static int
-get_render_pipe_profile_mux_config(struct drm_i915_private *dev_priv,
-				   const struct i915_oa_reg **regs,
-				   int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_render_pipe_profile;
-	lens[n] = ARRAY_SIZE(mux_config_render_pipe_profile);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_memory_reads[] = {
-	{ _MMIO(0x272c), 0xffffffff },
-	{ _MMIO(0x2728), 0xffffffff },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x271c), 0xffffffff },
-	{ _MMIO(0x2718), 0xffffffff },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x274c), 0x86543210 },
-	{ _MMIO(0x2748), 0x86543210 },
-	{ _MMIO(0x2744), 0x00006667 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x275c), 0x86543210 },
-	{ _MMIO(0x2758), 0x86543210 },
-	{ _MMIO(0x2754), 0x00006465 },
-	{ _MMIO(0x2750), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007f81a },
-	{ _MMIO(0x2774), 0x0000fe00 },
-	{ _MMIO(0x2778), 0x0007f82a },
-	{ _MMIO(0x277c), 0x0000fe00 },
-	{ _MMIO(0x2780), 0x0007f872 },
-	{ _MMIO(0x2784), 0x0000fe00 },
-	{ _MMIO(0x2788), 0x0007f8ba },
-	{ _MMIO(0x278c), 0x0000fe00 },
-	{ _MMIO(0x2790), 0x0007f87a },
-	{ _MMIO(0x2794), 0x0000fe00 },
-	{ _MMIO(0x2798), 0x0007f8ea },
-	{ _MMIO(0x279c), 0x0000fe00 },
-	{ _MMIO(0x27a0), 0x0007f8e2 },
-	{ _MMIO(0x27a4), 0x0000fe00 },
-	{ _MMIO(0x27a8), 0x0007f8f2 },
-	{ _MMIO(0x27ac), 0x0000fe00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_memory_reads[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00015014 },
-	{ _MMIO(0xe658), 0x00025024 },
-	{ _MMIO(0xe758), 0x00035034 },
-	{ _MMIO(0xe45c), 0x00045044 },
-	{ _MMIO(0xe55c), 0x00055054 },
-	{ _MMIO(0xe65c), 0x00065064 },
-};
-
-static const struct i915_oa_reg mux_config_memory_reads[] = {
-	{ _MMIO(0x9888), 0x19800343 },
-	{ _MMIO(0x9888), 0x39900340 },
-	{ _MMIO(0x9888), 0x3f901000 },
-	{ _MMIO(0x9888), 0x41900003 },
-	{ _MMIO(0x9888), 0x03803180 },
-	{ _MMIO(0x9888), 0x058035e2 },
-	{ _MMIO(0x9888), 0x0780006a },
-	{ _MMIO(0x9888), 0x11800000 },
-	{ _MMIO(0x9888), 0x2181a000 },
-	{ _MMIO(0x9888), 0x2381000a },
-	{ _MMIO(0x9888), 0x1d950550 },
-	{ _MMIO(0x9888), 0x0b928000 },
-	{ _MMIO(0x9888), 0x0d92a000 },
-	{ _MMIO(0x9888), 0x0f922000 },
-	{ _MMIO(0x9888), 0x13900170 },
-	{ _MMIO(0x9888), 0x21900171 },
-	{ _MMIO(0x9888), 0x23900172 },
-	{ _MMIO(0x9888), 0x25900173 },
-	{ _MMIO(0x9888), 0x27900174 },
-	{ _MMIO(0x9888), 0x29900175 },
-	{ _MMIO(0x9888), 0x2b900176 },
-	{ _MMIO(0x9888), 0x2d900177 },
-	{ _MMIO(0x9888), 0x2f90017f },
-	{ _MMIO(0x9888), 0x31900125 },
-	{ _MMIO(0x9888), 0x15900123 },
-	{ _MMIO(0x9888), 0x17900121 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1d908000 },
-	{ _MMIO(0x9888), 0x1f908000 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x43901084 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x47901080 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49901084 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x4b901084 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4d900004 },
-	{ _MMIO(0x9888), 0x45900000 },
-};
-
-static int
-get_memory_reads_mux_config(struct drm_i915_private *dev_priv,
-			    const struct i915_oa_reg **regs,
-			    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_memory_reads;
-	lens[n] = ARRAY_SIZE(mux_config_memory_reads);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_memory_writes[] = {
-	{ _MMIO(0x272c), 0xffffffff },
-	{ _MMIO(0x2728), 0xffffffff },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x271c), 0xffffffff },
-	{ _MMIO(0x2718), 0xffffffff },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x274c), 0x86543210 },
-	{ _MMIO(0x2748), 0x86543210 },
-	{ _MMIO(0x2744), 0x00006667 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x275c), 0x86543210 },
-	{ _MMIO(0x2758), 0x86543210 },
-	{ _MMIO(0x2754), 0x00006465 },
-	{ _MMIO(0x2750), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007f81a },
-	{ _MMIO(0x2774), 0x0000fe00 },
-	{ _MMIO(0x2778), 0x0007f82a },
-	{ _MMIO(0x277c), 0x0000fe00 },
-	{ _MMIO(0x2780), 0x0007f822 },
-	{ _MMIO(0x2784), 0x0000fe00 },
-	{ _MMIO(0x2788), 0x0007f8ba },
-	{ _MMIO(0x278c), 0x0000fe00 },
-	{ _MMIO(0x2790), 0x0007f87a },
-	{ _MMIO(0x2794), 0x0000fe00 },
-	{ _MMIO(0x2798), 0x0007f8ea },
-	{ _MMIO(0x279c), 0x0000fe00 },
-	{ _MMIO(0x27a0), 0x0007f8e2 },
-	{ _MMIO(0x27a4), 0x0000fe00 },
-	{ _MMIO(0x27a8), 0x0007f8f2 },
-	{ _MMIO(0x27ac), 0x0000fe00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_memory_writes[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00015014 },
-	{ _MMIO(0xe658), 0x00025024 },
-	{ _MMIO(0xe758), 0x00035034 },
-	{ _MMIO(0xe45c), 0x00045044 },
-	{ _MMIO(0xe55c), 0x00055054 },
-	{ _MMIO(0xe65c), 0x00065064 },
-};
-
-static const struct i915_oa_reg mux_config_memory_writes[] = {
-	{ _MMIO(0x9888), 0x19800343 },
-	{ _MMIO(0x9888), 0x39900340 },
-	{ _MMIO(0x9888), 0x3f900000 },
-	{ _MMIO(0x9888), 0x41900080 },
-	{ _MMIO(0x9888), 0x03803180 },
-	{ _MMIO(0x9888), 0x058035e2 },
-	{ _MMIO(0x9888), 0x0780006a },
-	{ _MMIO(0x9888), 0x11800000 },
-	{ _MMIO(0x9888), 0x2181a000 },
-	{ _MMIO(0x9888), 0x2381000a },
-	{ _MMIO(0x9888), 0x1d950550 },
-	{ _MMIO(0x9888), 0x0b928000 },
-	{ _MMIO(0x9888), 0x0d92a000 },
-	{ _MMIO(0x9888), 0x0f922000 },
-	{ _MMIO(0x9888), 0x13900180 },
-	{ _MMIO(0x9888), 0x21900181 },
-	{ _MMIO(0x9888), 0x23900182 },
-	{ _MMIO(0x9888), 0x25900183 },
-	{ _MMIO(0x9888), 0x27900184 },
-	{ _MMIO(0x9888), 0x29900185 },
-	{ _MMIO(0x9888), 0x2b900186 },
-	{ _MMIO(0x9888), 0x2d900187 },
-	{ _MMIO(0x9888), 0x2f900170 },
-	{ _MMIO(0x9888), 0x31900125 },
-	{ _MMIO(0x9888), 0x15900123 },
-	{ _MMIO(0x9888), 0x17900121 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1d908000 },
-	{ _MMIO(0x9888), 0x1f908000 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x43901084 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x47901080 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49901084 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x4b901084 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4d900004 },
-	{ _MMIO(0x9888), 0x45900000 },
-};
-
-static int
-get_memory_writes_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_memory_writes;
-	lens[n] = ARRAY_SIZE(mux_config_memory_writes);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_extended[] = {
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007fc2a },
-	{ _MMIO(0x2774), 0x0000bf00 },
-	{ _MMIO(0x2778), 0x0007fc6a },
-	{ _MMIO(0x277c), 0x0000bf00 },
-	{ _MMIO(0x2780), 0x0007fc92 },
-	{ _MMIO(0x2784), 0x0000bf00 },
-	{ _MMIO(0x2788), 0x0007fca2 },
-	{ _MMIO(0x278c), 0x0000bf00 },
-	{ _MMIO(0x2790), 0x0007fc32 },
-	{ _MMIO(0x2794), 0x0000bf00 },
-	{ _MMIO(0x2798), 0x0007fc9a },
-	{ _MMIO(0x279c), 0x0000bf00 },
-	{ _MMIO(0x27a0), 0x0007fe6a },
-	{ _MMIO(0x27a4), 0x0000bf00 },
-	{ _MMIO(0x27a8), 0x0007fe7a },
-	{ _MMIO(0x27ac), 0x0000bf00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_extended[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00778008 },
-	{ _MMIO(0xe45c), 0x00088078 },
-	{ _MMIO(0xe55c), 0x00808708 },
-	{ _MMIO(0xe65c), 0x00a08908 },
-};
-
-static const struct i915_oa_reg mux_config_compute_extended[] = {
-	{ _MMIO(0x9888), 0x104f00e0 },
-	{ _MMIO(0x9888), 0x141c0160 },
-	{ _MMIO(0x9888), 0x161c0015 },
-	{ _MMIO(0x9888), 0x181c0120 },
-	{ _MMIO(0x9888), 0x002d5000 },
-	{ _MMIO(0x9888), 0x062d4000 },
-	{ _MMIO(0x9888), 0x082d5000 },
-	{ _MMIO(0x9888), 0x0a2d5000 },
-	{ _MMIO(0x9888), 0x0c2d5000 },
-	{ _MMIO(0x9888), 0x0e2d5000 },
-	{ _MMIO(0x9888), 0x022d5000 },
-	{ _MMIO(0x9888), 0x042d5000 },
-	{ _MMIO(0x9888), 0x0c2e5400 },
-	{ _MMIO(0x9888), 0x0e2e5515 },
-	{ _MMIO(0x9888), 0x102e0155 },
-	{ _MMIO(0x9888), 0x044cc000 },
-	{ _MMIO(0x9888), 0x0a4c8000 },
-	{ _MMIO(0x9888), 0x0c4cc000 },
-	{ _MMIO(0x9888), 0x0e4cc000 },
-	{ _MMIO(0x9888), 0x104c8000 },
-	{ _MMIO(0x9888), 0x124c8000 },
-	{ _MMIO(0x9888), 0x144c8000 },
-	{ _MMIO(0x9888), 0x164c2000 },
-	{ _MMIO(0x9888), 0x064cc000 },
-	{ _MMIO(0x9888), 0x084cc000 },
-	{ _MMIO(0x9888), 0x004ea000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084ea000 },
-	{ _MMIO(0x9888), 0x0a4ea000 },
-	{ _MMIO(0x9888), 0x0c4ea000 },
-	{ _MMIO(0x9888), 0x0e4ea000 },
-	{ _MMIO(0x9888), 0x024ea000 },
-	{ _MMIO(0x9888), 0x044ea000 },
-	{ _MMIO(0x9888), 0x0e4f4b41 },
-	{ _MMIO(0x9888), 0x004f4200 },
-	{ _MMIO(0x9888), 0x024f404c },
-	{ _MMIO(0x9888), 0x1c4f0000 },
-	{ _MMIO(0x9888), 0x1a4f0000 },
-	{ _MMIO(0x9888), 0x001b4000 },
-	{ _MMIO(0x9888), 0x061b8000 },
-	{ _MMIO(0x9888), 0x081bc000 },
-	{ _MMIO(0x9888), 0x0a1bc000 },
-	{ _MMIO(0x9888), 0x0c1bc000 },
-	{ _MMIO(0x9888), 0x041bc000 },
-	{ _MMIO(0x9888), 0x001c0031 },
-	{ _MMIO(0x9888), 0x061c1900 },
-	{ _MMIO(0x9888), 0x081c1a33 },
-	{ _MMIO(0x9888), 0x0a1c1b35 },
-	{ _MMIO(0x9888), 0x0c1c3337 },
-	{ _MMIO(0x9888), 0x041c31c7 },
-	{ _MMIO(0x9888), 0x180f5000 },
-	{ _MMIO(0x9888), 0x1a0fa8aa },
-	{ _MMIO(0x9888), 0x1c0f0aaa },
-	{ _MMIO(0x9888), 0x182c8000 },
-	{ _MMIO(0x9888), 0x1c2c6aaa },
-	{ _MMIO(0x9888), 0x1e2c0001 },
-	{ _MMIO(0x9888), 0x1a2c2950 },
-	{ _MMIO(0x9888), 0x01938000 },
-	{ _MMIO(0x9888), 0x0f938000 },
-	{ _MMIO(0x9888), 0x1993aaaa },
-	{ _MMIO(0x9888), 0x03938000 },
-	{ _MMIO(0x9888), 0x05938000 },
-	{ _MMIO(0x9888), 0x07938000 },
-	{ _MMIO(0x9888), 0x09938000 },
-	{ _MMIO(0x9888), 0x0b938000 },
-	{ _MMIO(0x9888), 0x13904000 },
-	{ _MMIO(0x9888), 0x21904000 },
-	{ _MMIO(0x9888), 0x23904000 },
-	{ _MMIO(0x9888), 0x25904000 },
-	{ _MMIO(0x9888), 0x27904000 },
-	{ _MMIO(0x9888), 0x29904000 },
-	{ _MMIO(0x9888), 0x2b904000 },
-	{ _MMIO(0x9888), 0x2d904000 },
-	{ _MMIO(0x9888), 0x2f904000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x15904000 },
-	{ _MMIO(0x9888), 0x17904000 },
-	{ _MMIO(0x9888), 0x19904000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x1d904000 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x43900420 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x47900000 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x4b900400 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4d900001 },
-	{ _MMIO(0x9888), 0x45900001 },
-};
-
-static int
-get_compute_extended_mux_config(struct drm_i915_private *dev_priv,
-				const struct i915_oa_reg **regs,
-				int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_extended;
-	lens[n] = ARRAY_SIZE(mux_config_compute_extended);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_l3_cache[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x30800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007fffa },
-	{ _MMIO(0x2774), 0x0000fefe },
-	{ _MMIO(0x2778), 0x0007fffa },
-	{ _MMIO(0x277c), 0x0000fefd },
-	{ _MMIO(0x2790), 0x0007fffa },
-	{ _MMIO(0x2794), 0x0000fbef },
-	{ _MMIO(0x2798), 0x0007fffa },
-	{ _MMIO(0x279c), 0x0000fbdf },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_l3_cache[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00101100 },
-	{ _MMIO(0xe45c), 0x00201200 },
-	{ _MMIO(0xe55c), 0x00301300 },
-	{ _MMIO(0xe65c), 0x00401400 },
-};
-
-static const struct i915_oa_reg mux_config_compute_l3_cache[] = {
-	{ _MMIO(0x9888), 0x166c03b0 },
-	{ _MMIO(0x9888), 0x1593001e },
-	{ _MMIO(0x9888), 0x3f900c00 },
-	{ _MMIO(0x9888), 0x41900000 },
-	{ _MMIO(0x9888), 0x002d1000 },
-	{ _MMIO(0x9888), 0x062d4000 },
-	{ _MMIO(0x9888), 0x082d5000 },
-	{ _MMIO(0x9888), 0x0e2d5000 },
-	{ _MMIO(0x9888), 0x0c2e0400 },
-	{ _MMIO(0x9888), 0x0e2e1500 },
-	{ _MMIO(0x9888), 0x102e0140 },
-	{ _MMIO(0x9888), 0x044c4000 },
-	{ _MMIO(0x9888), 0x0a4c8000 },
-	{ _MMIO(0x9888), 0x0c4cc000 },
-	{ _MMIO(0x9888), 0x144c8000 },
-	{ _MMIO(0x9888), 0x164c2000 },
-	{ _MMIO(0x9888), 0x004e2000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084ea000 },
-	{ _MMIO(0x9888), 0x0e4ea000 },
-	{ _MMIO(0x9888), 0x1a4f4001 },
-	{ _MMIO(0x9888), 0x1c4f5005 },
-	{ _MMIO(0x9888), 0x006c0051 },
-	{ _MMIO(0x9888), 0x066c5000 },
-	{ _MMIO(0x9888), 0x086c5c5d },
-	{ _MMIO(0x9888), 0x0e6c5e5f },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x146c0000 },
-	{ _MMIO(0x9888), 0x1a6c0000 },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x180f1000 },
-	{ _MMIO(0x9888), 0x1a0fa800 },
-	{ _MMIO(0x9888), 0x1c0f0a00 },
-	{ _MMIO(0x9888), 0x182c4000 },
-	{ _MMIO(0x9888), 0x1c2c4015 },
-	{ _MMIO(0x9888), 0x1e2c0001 },
-	{ _MMIO(0x9888), 0x03931980 },
-	{ _MMIO(0x9888), 0x05930032 },
-	{ _MMIO(0x9888), 0x11930000 },
-	{ _MMIO(0x9888), 0x01938000 },
-	{ _MMIO(0x9888), 0x0f938000 },
-	{ _MMIO(0x9888), 0x1993a00a },
-	{ _MMIO(0x9888), 0x07930000 },
-	{ _MMIO(0x9888), 0x09930000 },
-	{ _MMIO(0x9888), 0x1d900177 },
-	{ _MMIO(0x9888), 0x1f900178 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x13904000 },
-	{ _MMIO(0x9888), 0x21904000 },
-	{ _MMIO(0x9888), 0x23904000 },
-	{ _MMIO(0x9888), 0x25904000 },
-	{ _MMIO(0x9888), 0x2f904000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x19904000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x53901000 },
-	{ _MMIO(0x9888), 0x43900000 },
-	{ _MMIO(0x9888), 0x55900111 },
-	{ _MMIO(0x9888), 0x47900001 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x4b900000 },
-	{ _MMIO(0x9888), 0x4d900000 },
-	{ _MMIO(0x9888), 0x45900400 },
-};
-
-static int
-get_compute_l3_cache_mux_config(struct drm_i915_private *dev_priv,
-				const struct i915_oa_reg **regs,
-				int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_l3_cache;
-	lens[n] = ARRAY_SIZE(mux_config_compute_l3_cache);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_hdc_and_sf[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x10800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x0000fdff },
-};
-
-static const struct i915_oa_reg flex_eu_config_hdc_and_sf[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_hdc_and_sf[] = {
-	{ _MMIO(0x9888), 0x104f0232 },
-	{ _MMIO(0x9888), 0x124f4640 },
-	{ _MMIO(0x9888), 0x11834400 },
-	{ _MMIO(0x9888), 0x022d4000 },
-	{ _MMIO(0x9888), 0x042d5000 },
-	{ _MMIO(0x9888), 0x062d1000 },
-	{ _MMIO(0x9888), 0x0e2e0055 },
-	{ _MMIO(0x9888), 0x064c8000 },
-	{ _MMIO(0x9888), 0x084cc000 },
-	{ _MMIO(0x9888), 0x0a4c4000 },
-	{ _MMIO(0x9888), 0x024e8000 },
-	{ _MMIO(0x9888), 0x044ea000 },
-	{ _MMIO(0x9888), 0x064e2000 },
-	{ _MMIO(0x9888), 0x024f6100 },
-	{ _MMIO(0x9888), 0x044f416b },
-	{ _MMIO(0x9888), 0x064f004b },
-	{ _MMIO(0x9888), 0x1a4f0000 },
-	{ _MMIO(0x9888), 0x1a0f02a8 },
-	{ _MMIO(0x9888), 0x1a2c5500 },
-	{ _MMIO(0x9888), 0x0f808000 },
-	{ _MMIO(0x9888), 0x25810020 },
-	{ _MMIO(0x9888), 0x0f8305c0 },
-	{ _MMIO(0x9888), 0x07938000 },
-	{ _MMIO(0x9888), 0x09938000 },
-	{ _MMIO(0x9888), 0x0b938000 },
-	{ _MMIO(0x9888), 0x0d938000 },
-	{ _MMIO(0x9888), 0x1f951000 },
-	{ _MMIO(0x9888), 0x13920200 },
-	{ _MMIO(0x9888), 0x31908000 },
-	{ _MMIO(0x9888), 0x19904000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x1d904000 },
-	{ _MMIO(0x9888), 0x1f904000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x4d900003 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900000 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x47900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_hdc_and_sf_mux_config(struct drm_i915_private *dev_priv,
-			  const struct i915_oa_reg **regs,
-			  int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_hdc_and_sf;
-	lens[n] = ARRAY_SIZE(mux_config_hdc_and_sf);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_l3_1[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2770), 0x00100070 },
-	{ _MMIO(0x2774), 0x0000fff1 },
-	{ _MMIO(0x2778), 0x00014002 },
-	{ _MMIO(0x277c), 0x0000c3ff },
-	{ _MMIO(0x2780), 0x00010002 },
-	{ _MMIO(0x2784), 0x0000c7ff },
-	{ _MMIO(0x2788), 0x00004002 },
-	{ _MMIO(0x278c), 0x0000d3ff },
-	{ _MMIO(0x2790), 0x00100700 },
-	{ _MMIO(0x2794), 0x0000ff1f },
-	{ _MMIO(0x2798), 0x00001402 },
-	{ _MMIO(0x279c), 0x0000fc3f },
-	{ _MMIO(0x27a0), 0x00001002 },
-	{ _MMIO(0x27a4), 0x0000fc7f },
-	{ _MMIO(0x27a8), 0x00000402 },
-	{ _MMIO(0x27ac), 0x0000fd3f },
-};
-
-static const struct i915_oa_reg flex_eu_config_l3_1[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_l3_1[] = {
-	{ _MMIO(0x9888), 0x12643400 },
-	{ _MMIO(0x9888), 0x12653400 },
-	{ _MMIO(0x9888), 0x106c6800 },
-	{ _MMIO(0x9888), 0x126c001e },
-	{ _MMIO(0x9888), 0x166c0010 },
-	{ _MMIO(0x9888), 0x0c2d5000 },
-	{ _MMIO(0x9888), 0x0e2d5000 },
-	{ _MMIO(0x9888), 0x002d4000 },
-	{ _MMIO(0x9888), 0x022d5000 },
-	{ _MMIO(0x9888), 0x042d5000 },
-	{ _MMIO(0x9888), 0x062d1000 },
-	{ _MMIO(0x9888), 0x102e0154 },
-	{ _MMIO(0x9888), 0x0c2e5000 },
-	{ _MMIO(0x9888), 0x0e2e0055 },
-	{ _MMIO(0x9888), 0x104c8000 },
-	{ _MMIO(0x9888), 0x124c8000 },
-	{ _MMIO(0x9888), 0x144c8000 },
-	{ _MMIO(0x9888), 0x164c2000 },
-	{ _MMIO(0x9888), 0x044c8000 },
-	{ _MMIO(0x9888), 0x064cc000 },
-	{ _MMIO(0x9888), 0x084cc000 },
-	{ _MMIO(0x9888), 0x0a4c4000 },
-	{ _MMIO(0x9888), 0x0c4ea000 },
-	{ _MMIO(0x9888), 0x0e4ea000 },
-	{ _MMIO(0x9888), 0x004e8000 },
-	{ _MMIO(0x9888), 0x024ea000 },
-	{ _MMIO(0x9888), 0x044ea000 },
-	{ _MMIO(0x9888), 0x064e2000 },
-	{ _MMIO(0x9888), 0x1c4f5500 },
-	{ _MMIO(0x9888), 0x1a4f1554 },
-	{ _MMIO(0x9888), 0x0a640024 },
-	{ _MMIO(0x9888), 0x10640000 },
-	{ _MMIO(0x9888), 0x04640000 },
-	{ _MMIO(0x9888), 0x0c650024 },
-	{ _MMIO(0x9888), 0x10650000 },
-	{ _MMIO(0x9888), 0x06650000 },
-	{ _MMIO(0x9888), 0x0c6c5327 },
-	{ _MMIO(0x9888), 0x0e6c5425 },
-	{ _MMIO(0x9888), 0x006c2a00 },
-	{ _MMIO(0x9888), 0x026c285b },
-	{ _MMIO(0x9888), 0x046c005c },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1a6c0900 },
-	{ _MMIO(0x9888), 0x1c0f0aa0 },
-	{ _MMIO(0x9888), 0x180f4000 },
-	{ _MMIO(0x9888), 0x1a0f02aa },
-	{ _MMIO(0x9888), 0x1c2c5400 },
-	{ _MMIO(0x9888), 0x1e2c0001 },
-	{ _MMIO(0x9888), 0x1a2c5550 },
-	{ _MMIO(0x9888), 0x1993aa00 },
-	{ _MMIO(0x9888), 0x03938000 },
-	{ _MMIO(0x9888), 0x05938000 },
-	{ _MMIO(0x9888), 0x07938000 },
-	{ _MMIO(0x9888), 0x09938000 },
-	{ _MMIO(0x9888), 0x0b938000 },
-	{ _MMIO(0x9888), 0x0d938000 },
-	{ _MMIO(0x9888), 0x2b904000 },
-	{ _MMIO(0x9888), 0x2d904000 },
-	{ _MMIO(0x9888), 0x2f904000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x15904000 },
-	{ _MMIO(0x9888), 0x17904000 },
-	{ _MMIO(0x9888), 0x19904000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x1d904000 },
-	{ _MMIO(0x9888), 0x1f904000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x4b900421 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4d900001 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x43900420 },
-	{ _MMIO(0x9888), 0x45900021 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x47900000 },
-};
-
-static int
-get_l3_1_mux_config(struct drm_i915_private *dev_priv,
-		    const struct i915_oa_reg **regs,
-		    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_l3_1;
-	lens[n] = ARRAY_SIZE(mux_config_l3_1);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x30800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x0000efff },
-	{ _MMIO(0x2778), 0x00006000 },
-	{ _MMIO(0x277c), 0x0000f3ff },
-};
-
-static const struct i915_oa_reg flex_eu_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0x9888), 0x102d7800 },
-	{ _MMIO(0x9888), 0x122d79e0 },
-	{ _MMIO(0x9888), 0x0c2f0004 },
-	{ _MMIO(0x9888), 0x100e3800 },
-	{ _MMIO(0x9888), 0x180f0005 },
-	{ _MMIO(0x9888), 0x002d0940 },
-	{ _MMIO(0x9888), 0x022d802f },
-	{ _MMIO(0x9888), 0x042d4013 },
-	{ _MMIO(0x9888), 0x062d1000 },
-	{ _MMIO(0x9888), 0x0e2e0050 },
-	{ _MMIO(0x9888), 0x022f0010 },
-	{ _MMIO(0x9888), 0x002f0000 },
-	{ _MMIO(0x9888), 0x084c8000 },
-	{ _MMIO(0x9888), 0x0a4c4000 },
-	{ _MMIO(0x9888), 0x044e8000 },
-	{ _MMIO(0x9888), 0x064e2000 },
-	{ _MMIO(0x9888), 0x040e0480 },
-	{ _MMIO(0x9888), 0x000e0000 },
-	{ _MMIO(0x9888), 0x060f0027 },
-	{ _MMIO(0x9888), 0x100f0000 },
-	{ _MMIO(0x9888), 0x1a0f0040 },
-	{ _MMIO(0x9888), 0x03938000 },
-	{ _MMIO(0x9888), 0x05938000 },
-	{ _MMIO(0x9888), 0x07938000 },
-	{ _MMIO(0x9888), 0x09938000 },
-	{ _MMIO(0x9888), 0x0b938000 },
-	{ _MMIO(0x9888), 0x0d938000 },
-	{ _MMIO(0x9888), 0x15904000 },
-	{ _MMIO(0x9888), 0x17904000 },
-	{ _MMIO(0x9888), 0x19904000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x1d904000 },
-	{ _MMIO(0x9888), 0x1f904000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x439014a0 },
-	{ _MMIO(0x9888), 0x459000a4 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x47900001 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_rasterizer_and_pixel_backend_mux_config(struct drm_i915_private *dev_priv,
-					    const struct i915_oa_reg **regs,
-					    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_rasterizer_and_pixel_backend;
-	lens[n] = ARRAY_SIZE(mux_config_rasterizer_and_pixel_backend);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_sampler[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x70800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x0000c000 },
-	{ _MMIO(0x2774), 0x0000e7ff },
-	{ _MMIO(0x2778), 0x00003000 },
-	{ _MMIO(0x277c), 0x0000f9ff },
-	{ _MMIO(0x2780), 0x00000c00 },
-	{ _MMIO(0x2784), 0x0000fe7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_sampler[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_sampler[] = {
-	{ _MMIO(0x9888), 0x121300a0 },
-	{ _MMIO(0x9888), 0x141600ab },
-	{ _MMIO(0x9888), 0x123300a0 },
-	{ _MMIO(0x9888), 0x143600ab },
-	{ _MMIO(0x9888), 0x125300a0 },
-	{ _MMIO(0x9888), 0x145600ab },
-	{ _MMIO(0x9888), 0x0c2d4000 },
-	{ _MMIO(0x9888), 0x0e2d5000 },
-	{ _MMIO(0x9888), 0x002d4000 },
-	{ _MMIO(0x9888), 0x022d5000 },
-	{ _MMIO(0x9888), 0x042d5000 },
-	{ _MMIO(0x9888), 0x062d1000 },
-	{ _MMIO(0x9888), 0x102e01a0 },
-	{ _MMIO(0x9888), 0x0c2e5000 },
-	{ _MMIO(0x9888), 0x0e2e0065 },
-	{ _MMIO(0x9888), 0x164c2000 },
-	{ _MMIO(0x9888), 0x044c8000 },
-	{ _MMIO(0x9888), 0x064cc000 },
-	{ _MMIO(0x9888), 0x084c4000 },
-	{ _MMIO(0x9888), 0x0a4c4000 },
-	{ _MMIO(0x9888), 0x0e4e8000 },
-	{ _MMIO(0x9888), 0x004e8000 },
-	{ _MMIO(0x9888), 0x024ea000 },
-	{ _MMIO(0x9888), 0x044e2000 },
-	{ _MMIO(0x9888), 0x064e2000 },
-	{ _MMIO(0x9888), 0x1c0f0800 },
-	{ _MMIO(0x9888), 0x180f4000 },
-	{ _MMIO(0x9888), 0x1a0f023f },
-	{ _MMIO(0x9888), 0x1e2c0003 },
-	{ _MMIO(0x9888), 0x1a2cc030 },
-	{ _MMIO(0x9888), 0x04132180 },
-	{ _MMIO(0x9888), 0x02130000 },
-	{ _MMIO(0x9888), 0x0c148000 },
-	{ _MMIO(0x9888), 0x0e142000 },
-	{ _MMIO(0x9888), 0x04148000 },
-	{ _MMIO(0x9888), 0x1e150140 },
-	{ _MMIO(0x9888), 0x1c150040 },
-	{ _MMIO(0x9888), 0x0c163000 },
-	{ _MMIO(0x9888), 0x0e160068 },
-	{ _MMIO(0x9888), 0x10160000 },
-	{ _MMIO(0x9888), 0x18160000 },
-	{ _MMIO(0x9888), 0x0a164000 },
-	{ _MMIO(0x9888), 0x04330043 },
-	{ _MMIO(0x9888), 0x02330000 },
-	{ _MMIO(0x9888), 0x0234a000 },
-	{ _MMIO(0x9888), 0x04342000 },
-	{ _MMIO(0x9888), 0x1c350015 },
-	{ _MMIO(0x9888), 0x02363460 },
-	{ _MMIO(0x9888), 0x10360000 },
-	{ _MMIO(0x9888), 0x04360000 },
-	{ _MMIO(0x9888), 0x06360000 },
-	{ _MMIO(0x9888), 0x08364000 },
-	{ _MMIO(0x9888), 0x06530043 },
-	{ _MMIO(0x9888), 0x02530000 },
-	{ _MMIO(0x9888), 0x0e548000 },
-	{ _MMIO(0x9888), 0x00548000 },
-	{ _MMIO(0x9888), 0x06542000 },
-	{ _MMIO(0x9888), 0x1e550400 },
-	{ _MMIO(0x9888), 0x1a552000 },
-	{ _MMIO(0x9888), 0x1c550100 },
-	{ _MMIO(0x9888), 0x0e563000 },
-	{ _MMIO(0x9888), 0x00563400 },
-	{ _MMIO(0x9888), 0x10560000 },
-	{ _MMIO(0x9888), 0x18560000 },
-	{ _MMIO(0x9888), 0x02560000 },
-	{ _MMIO(0x9888), 0x0c564000 },
-	{ _MMIO(0x9888), 0x1993a800 },
-	{ _MMIO(0x9888), 0x03938000 },
-	{ _MMIO(0x9888), 0x05938000 },
-	{ _MMIO(0x9888), 0x07938000 },
-	{ _MMIO(0x9888), 0x09938000 },
-	{ _MMIO(0x9888), 0x0b938000 },
-	{ _MMIO(0x9888), 0x0d938000 },
-	{ _MMIO(0x9888), 0x2d904000 },
-	{ _MMIO(0x9888), 0x2f904000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x15904000 },
-	{ _MMIO(0x9888), 0x17904000 },
-	{ _MMIO(0x9888), 0x19904000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x1d904000 },
-	{ _MMIO(0x9888), 0x1f904000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x4b9014a0 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4d900001 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x43900820 },
-	{ _MMIO(0x9888), 0x45901022 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x47900000 },
-};
-
-static int
-get_sampler_mux_config(struct drm_i915_private *dev_priv,
-		       const struct i915_oa_reg **regs,
-		       int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_sampler;
-	lens[n] = ARRAY_SIZE(mux_config_sampler);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_tdl_1[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x00007fff },
-	{ _MMIO(0x2778), 0x00000000 },
-	{ _MMIO(0x277c), 0x00009fff },
-	{ _MMIO(0x2780), 0x00000002 },
-	{ _MMIO(0x2784), 0x0000efff },
-	{ _MMIO(0x2788), 0x00000000 },
-	{ _MMIO(0x278c), 0x0000f3ff },
-	{ _MMIO(0x2790), 0x00000002 },
-	{ _MMIO(0x2794), 0x0000fdff },
-	{ _MMIO(0x2798), 0x00000000 },
-	{ _MMIO(0x279c), 0x0000fe7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_tdl_1[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_tdl_1[] = {
-	{ _MMIO(0x9888), 0x141a0000 },
-	{ _MMIO(0x9888), 0x143a0000 },
-	{ _MMIO(0x9888), 0x145a0000 },
-	{ _MMIO(0x9888), 0x0c2d4000 },
-	{ _MMIO(0x9888), 0x0e2d5000 },
-	{ _MMIO(0x9888), 0x002d4000 },
-	{ _MMIO(0x9888), 0x022d5000 },
-	{ _MMIO(0x9888), 0x042d5000 },
-	{ _MMIO(0x9888), 0x062d1000 },
-	{ _MMIO(0x9888), 0x102e0150 },
-	{ _MMIO(0x9888), 0x0c2e5000 },
-	{ _MMIO(0x9888), 0x0e2e006a },
-	{ _MMIO(0x9888), 0x124c8000 },
-	{ _MMIO(0x9888), 0x144c8000 },
-	{ _MMIO(0x9888), 0x164c2000 },
-	{ _MMIO(0x9888), 0x044c8000 },
-	{ _MMIO(0x9888), 0x064c4000 },
-	{ _MMIO(0x9888), 0x0a4c4000 },
-	{ _MMIO(0x9888), 0x0c4e8000 },
-	{ _MMIO(0x9888), 0x0e4ea000 },
-	{ _MMIO(0x9888), 0x004e8000 },
-	{ _MMIO(0x9888), 0x024e2000 },
-	{ _MMIO(0x9888), 0x064e2000 },
-	{ _MMIO(0x9888), 0x1c0f0bc0 },
-	{ _MMIO(0x9888), 0x180f4000 },
-	{ _MMIO(0x9888), 0x1a0f0302 },
-	{ _MMIO(0x9888), 0x1e2c0003 },
-	{ _MMIO(0x9888), 0x1a2c00f0 },
-	{ _MMIO(0x9888), 0x021a3080 },
-	{ _MMIO(0x9888), 0x041a31e5 },
-	{ _MMIO(0x9888), 0x02148000 },
-	{ _MMIO(0x9888), 0x0414a000 },
-	{ _MMIO(0x9888), 0x1c150054 },
-	{ _MMIO(0x9888), 0x06168000 },
-	{ _MMIO(0x9888), 0x08168000 },
-	{ _MMIO(0x9888), 0x0a168000 },
-	{ _MMIO(0x9888), 0x0c3a3280 },
-	{ _MMIO(0x9888), 0x0e3a0063 },
-	{ _MMIO(0x9888), 0x063a0061 },
-	{ _MMIO(0x9888), 0x023a0000 },
-	{ _MMIO(0x9888), 0x0c348000 },
-	{ _MMIO(0x9888), 0x0e342000 },
-	{ _MMIO(0x9888), 0x06342000 },
-	{ _MMIO(0x9888), 0x1e350140 },
-	{ _MMIO(0x9888), 0x1c350100 },
-	{ _MMIO(0x9888), 0x18360028 },
-	{ _MMIO(0x9888), 0x0c368000 },
-	{ _MMIO(0x9888), 0x0e5a3080 },
-	{ _MMIO(0x9888), 0x005a3280 },
-	{ _MMIO(0x9888), 0x025a0063 },
-	{ _MMIO(0x9888), 0x0e548000 },
-	{ _MMIO(0x9888), 0x00548000 },
-	{ _MMIO(0x9888), 0x02542000 },
-	{ _MMIO(0x9888), 0x1e550400 },
-	{ _MMIO(0x9888), 0x1a552000 },
-	{ _MMIO(0x9888), 0x1c550001 },
-	{ _MMIO(0x9888), 0x18560080 },
-	{ _MMIO(0x9888), 0x02568000 },
-	{ _MMIO(0x9888), 0x04568000 },
-	{ _MMIO(0x9888), 0x1993a800 },
-	{ _MMIO(0x9888), 0x03938000 },
-	{ _MMIO(0x9888), 0x05938000 },
-	{ _MMIO(0x9888), 0x07938000 },
-	{ _MMIO(0x9888), 0x09938000 },
-	{ _MMIO(0x9888), 0x0b938000 },
-	{ _MMIO(0x9888), 0x0d938000 },
-	{ _MMIO(0x9888), 0x2d904000 },
-	{ _MMIO(0x9888), 0x2f904000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x15904000 },
-	{ _MMIO(0x9888), 0x17904000 },
-	{ _MMIO(0x9888), 0x19904000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x1d904000 },
-	{ _MMIO(0x9888), 0x1f904000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x4b900420 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4d900000 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x43900000 },
-	{ _MMIO(0x9888), 0x45901084 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x47900001 },
-};
-
-static int
-get_tdl_1_mux_config(struct drm_i915_private *dev_priv,
-		     const struct i915_oa_reg **regs,
-		     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_tdl_1;
-	lens[n] = ARRAY_SIZE(mux_config_tdl_1);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_tdl_2[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_tdl_2[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_tdl_2[] = {
-	{ _MMIO(0x9888), 0x141a026b },
-	{ _MMIO(0x9888), 0x143a0173 },
-	{ _MMIO(0x9888), 0x145a026b },
-	{ _MMIO(0x9888), 0x002d4000 },
-	{ _MMIO(0x9888), 0x022d5000 },
-	{ _MMIO(0x9888), 0x042d5000 },
-	{ _MMIO(0x9888), 0x062d1000 },
-	{ _MMIO(0x9888), 0x0c2e5000 },
-	{ _MMIO(0x9888), 0x0e2e0069 },
-	{ _MMIO(0x9888), 0x044c8000 },
-	{ _MMIO(0x9888), 0x064cc000 },
-	{ _MMIO(0x9888), 0x0a4c4000 },
-	{ _MMIO(0x9888), 0x004e8000 },
-	{ _MMIO(0x9888), 0x024ea000 },
-	{ _MMIO(0x9888), 0x064e2000 },
-	{ _MMIO(0x9888), 0x180f6000 },
-	{ _MMIO(0x9888), 0x1a0f030a },
-	{ _MMIO(0x9888), 0x1a2c03c0 },
-	{ _MMIO(0x9888), 0x041a37e7 },
-	{ _MMIO(0x9888), 0x021a0000 },
-	{ _MMIO(0x9888), 0x0414a000 },
-	{ _MMIO(0x9888), 0x1c150050 },
-	{ _MMIO(0x9888), 0x08168000 },
-	{ _MMIO(0x9888), 0x0a168000 },
-	{ _MMIO(0x9888), 0x003a3380 },
-	{ _MMIO(0x9888), 0x063a006f },
-	{ _MMIO(0x9888), 0x023a0000 },
-	{ _MMIO(0x9888), 0x00348000 },
-	{ _MMIO(0x9888), 0x06342000 },
-	{ _MMIO(0x9888), 0x1a352000 },
-	{ _MMIO(0x9888), 0x1c350100 },
-	{ _MMIO(0x9888), 0x02368000 },
-	{ _MMIO(0x9888), 0x0c368000 },
-	{ _MMIO(0x9888), 0x025a37e7 },
-	{ _MMIO(0x9888), 0x0254a000 },
-	{ _MMIO(0x9888), 0x1c550005 },
-	{ _MMIO(0x9888), 0x04568000 },
-	{ _MMIO(0x9888), 0x06568000 },
-	{ _MMIO(0x9888), 0x03938000 },
-	{ _MMIO(0x9888), 0x05938000 },
-	{ _MMIO(0x9888), 0x07938000 },
-	{ _MMIO(0x9888), 0x09938000 },
-	{ _MMIO(0x9888), 0x0b938000 },
-	{ _MMIO(0x9888), 0x0d938000 },
-	{ _MMIO(0x9888), 0x15904000 },
-	{ _MMIO(0x9888), 0x17904000 },
-	{ _MMIO(0x9888), 0x19904000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x1d904000 },
-	{ _MMIO(0x9888), 0x1f904000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x43900020 },
-	{ _MMIO(0x9888), 0x45901080 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x47900001 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_tdl_2_mux_config(struct drm_i915_private *dev_priv,
-		     const struct i915_oa_reg **regs,
-		     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_tdl_2;
-	lens[n] = ARRAY_SIZE(mux_config_tdl_2);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_extra[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_extra[] = {
-	{ _MMIO(0xe458), 0x00001000 },
-	{ _MMIO(0xe558), 0x00003002 },
-	{ _MMIO(0xe658), 0x00005004 },
-	{ _MMIO(0xe758), 0x00011010 },
-	{ _MMIO(0xe45c), 0x00050012 },
-	{ _MMIO(0xe55c), 0x00052051 },
-	{ _MMIO(0xe65c), 0x00000008 },
-};
-
-static const struct i915_oa_reg mux_config_compute_extra[] = {
-	{ _MMIO(0x9888), 0x141a001f },
-	{ _MMIO(0x9888), 0x143a001f },
-	{ _MMIO(0x9888), 0x145a001f },
-	{ _MMIO(0x9888), 0x042d5000 },
-	{ _MMIO(0x9888), 0x062d1000 },
-	{ _MMIO(0x9888), 0x0e2e0094 },
-	{ _MMIO(0x9888), 0x084cc000 },
-	{ _MMIO(0x9888), 0x044ea000 },
-	{ _MMIO(0x9888), 0x1a0f00e0 },
-	{ _MMIO(0x9888), 0x1a2c0c00 },
-	{ _MMIO(0x9888), 0x061a0063 },
-	{ _MMIO(0x9888), 0x021a0000 },
-	{ _MMIO(0x9888), 0x06142000 },
-	{ _MMIO(0x9888), 0x1c150100 },
-	{ _MMIO(0x9888), 0x0c168000 },
-	{ _MMIO(0x9888), 0x043a3180 },
-	{ _MMIO(0x9888), 0x023a0000 },
-	{ _MMIO(0x9888), 0x04348000 },
-	{ _MMIO(0x9888), 0x1c350040 },
-	{ _MMIO(0x9888), 0x0a368000 },
-	{ _MMIO(0x9888), 0x045a0063 },
-	{ _MMIO(0x9888), 0x025a0000 },
-	{ _MMIO(0x9888), 0x04542000 },
-	{ _MMIO(0x9888), 0x1c550010 },
-	{ _MMIO(0x9888), 0x08568000 },
-	{ _MMIO(0x9888), 0x09938000 },
-	{ _MMIO(0x9888), 0x0b938000 },
-	{ _MMIO(0x9888), 0x0d938000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x1d904000 },
-	{ _MMIO(0x9888), 0x1f904000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900400 },
-	{ _MMIO(0x9888), 0x47900004 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_compute_extra_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_extra;
-	lens[n] = ARRAY_SIZE(mux_config_compute_extra);
-	n++;
-
-	return n;
-}
-
 static const struct i915_oa_reg b_counter_config_test_oa[] = {
 	{ _MMIO(0x2740), 0x00000000 },
 	{ _MMIO(0x2744), 0x00800000 },
@@ -1668,6 +60,7 @@ static const struct i915_oa_reg flex_eu_config_test_oa[] = {
 };
 
 static const struct i915_oa_reg mux_config_test_oa[] = {
+	{ _MMIO(0x9840), 0x00000080 },
 	{ _MMIO(0x9888), 0x19800000 },
 	{ _MMIO(0x9888), 0x07800063 },
 	{ _MMIO(0x9888), 0x11800000 },
@@ -1681,922 +74,35 @@ static const struct i915_oa_reg mux_config_test_oa[] = {
 	{ _MMIO(0x9888), 0x33900000 },
 };
 
-static int
-get_test_oa_mux_config(struct drm_i915_private *dev_priv,
-		       const struct i915_oa_reg **regs,
-		       int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_test_oa;
-	lens[n] = ARRAY_SIZE(mux_config_test_oa);
-	n++;
-
-	return n;
-}
-
-int i915_oa_select_metric_set_glk(struct drm_i915_private *dev_priv)
-{
-	dev_priv->perf.oa.n_mux_configs = 0;
-	dev_priv->perf.oa.b_counter_regs = NULL;
-	dev_priv->perf.oa.b_counter_regs_len = 0;
-	dev_priv->perf.oa.flex_regs = NULL;
-	dev_priv->perf.oa.flex_regs_len = 0;
-
-	switch (dev_priv->perf.oa.metrics_set) {
-	case METRIC_SET_ID_RENDER_BASIC:
-		dev_priv->perf.oa.n_mux_configs =
-			get_render_basic_mux_config(dev_priv,
-						    dev_priv->perf.oa.mux_regs,
-						    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_BASIC\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_render_basic;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_render_basic);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_render_basic;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_render_basic);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_BASIC:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_basic_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_BASIC\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_basic;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_basic);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_basic;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_basic);
-
-		return 0;
-	case METRIC_SET_ID_RENDER_PIPE_PROFILE:
-		dev_priv->perf.oa.n_mux_configs =
-			get_render_pipe_profile_mux_config(dev_priv,
-							   dev_priv->perf.oa.mux_regs,
-							   dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_PIPE_PROFILE\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_render_pipe_profile;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_render_pipe_profile);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_render_pipe_profile;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_render_pipe_profile);
-
-		return 0;
-	case METRIC_SET_ID_MEMORY_READS:
-		dev_priv->perf.oa.n_mux_configs =
-			get_memory_reads_mux_config(dev_priv,
-						    dev_priv->perf.oa.mux_regs,
-						    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_READS\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_memory_reads;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_memory_reads);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_memory_reads;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_memory_reads);
-
-		return 0;
-	case METRIC_SET_ID_MEMORY_WRITES:
-		dev_priv->perf.oa.n_mux_configs =
-			get_memory_writes_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_WRITES\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_memory_writes;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_memory_writes);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_memory_writes;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_memory_writes);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_EXTENDED:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_extended_mux_config(dev_priv,
-							dev_priv->perf.oa.mux_regs,
-							dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTENDED\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_extended;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_extended);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_extended;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_extended);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_L3_CACHE:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_l3_cache_mux_config(dev_priv,
-							dev_priv->perf.oa.mux_regs,
-							dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_L3_CACHE\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_l3_cache;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_l3_cache);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_l3_cache;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_l3_cache);
-
-		return 0;
-	case METRIC_SET_ID_HDC_AND_SF:
-		dev_priv->perf.oa.n_mux_configs =
-			get_hdc_and_sf_mux_config(dev_priv,
-						  dev_priv->perf.oa.mux_regs,
-						  dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"HDC_AND_SF\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_hdc_and_sf;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_hdc_and_sf);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_hdc_and_sf;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_hdc_and_sf);
-
-		return 0;
-	case METRIC_SET_ID_L3_1:
-		dev_priv->perf.oa.n_mux_configs =
-			get_l3_1_mux_config(dev_priv,
-					    dev_priv->perf.oa.mux_regs,
-					    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_1\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_l3_1;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_l3_1);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_l3_1;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_l3_1);
-
-		return 0;
-	case METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND:
-		dev_priv->perf.oa.n_mux_configs =
-			get_rasterizer_and_pixel_backend_mux_config(dev_priv,
-								    dev_priv->perf.oa.mux_regs,
-								    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RASTERIZER_AND_PIXEL_BACKEND\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_rasterizer_and_pixel_backend;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_rasterizer_and_pixel_backend);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_rasterizer_and_pixel_backend;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_rasterizer_and_pixel_backend);
-
-		return 0;
-	case METRIC_SET_ID_SAMPLER:
-		dev_priv->perf.oa.n_mux_configs =
-			get_sampler_mux_config(dev_priv,
-					       dev_priv->perf.oa.mux_regs,
-					       dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"SAMPLER\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_sampler;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_sampler);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_sampler;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_sampler);
-
-		return 0;
-	case METRIC_SET_ID_TDL_1:
-		dev_priv->perf.oa.n_mux_configs =
-			get_tdl_1_mux_config(dev_priv,
-					     dev_priv->perf.oa.mux_regs,
-					     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_1\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_tdl_1;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_tdl_1);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_tdl_1;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_tdl_1);
-
-		return 0;
-	case METRIC_SET_ID_TDL_2:
-		dev_priv->perf.oa.n_mux_configs =
-			get_tdl_2_mux_config(dev_priv,
-					     dev_priv->perf.oa.mux_regs,
-					     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_2\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_tdl_2;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_tdl_2);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_tdl_2;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_tdl_2);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_EXTRA:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_extra_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTRA\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_extra;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_extra);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_extra;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_extra);
-
-		return 0;
-	case METRIC_SET_ID_TEST_OA:
-		dev_priv->perf.oa.n_mux_configs =
-			get_test_oa_mux_config(dev_priv,
-					       dev_priv->perf.oa.mux_regs,
-					       dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TEST_OA\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_test_oa;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_test_oa);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_test_oa;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_test_oa);
-
-		return 0;
-	default:
-		return -ENODEV;
-	}
-}
-
-static ssize_t
-show_render_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_BASIC);
-}
-
-static struct device_attribute dev_attr_render_basic_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_render_basic_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_render_basic[] = {
-	&dev_attr_render_basic_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_render_basic = {
-	.name = "d72df5c7-5b4a-4274-a43f-00b0fd51fc68",
-	.attrs =  attrs_render_basic,
-};
-
-static ssize_t
-show_compute_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_BASIC);
-}
-
-static struct device_attribute dev_attr_compute_basic_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_basic_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_basic[] = {
-	&dev_attr_compute_basic_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_basic = {
-	.name = "814285f6-354d-41d2-ba49-e24e622714a0",
-	.attrs =  attrs_compute_basic,
-};
-
-static ssize_t
-show_render_pipe_profile_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_PIPE_PROFILE);
-}
-
-static struct device_attribute dev_attr_render_pipe_profile_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_render_pipe_profile_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_render_pipe_profile[] = {
-	&dev_attr_render_pipe_profile_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_render_pipe_profile = {
-	.name = "07d397a6-b3e6-49f6-9433-a4f293d55978",
-	.attrs =  attrs_render_pipe_profile,
-};
-
-static ssize_t
-show_memory_reads_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_READS);
-}
-
-static struct device_attribute dev_attr_memory_reads_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_memory_reads_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_memory_reads[] = {
-	&dev_attr_memory_reads_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_memory_reads = {
-	.name = "1a356946-5428-450b-a2f0-89f8783a302d",
-	.attrs =  attrs_memory_reads,
-};
-
-static ssize_t
-show_memory_writes_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_WRITES);
-}
-
-static struct device_attribute dev_attr_memory_writes_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_memory_writes_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_memory_writes[] = {
-	&dev_attr_memory_writes_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_memory_writes = {
-	.name = "5299be9d-7a61-4c99-9f81-f87e6c5aaca9",
-	.attrs =  attrs_memory_writes,
-};
-
-static ssize_t
-show_compute_extended_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTENDED);
-}
-
-static struct device_attribute dev_attr_compute_extended_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_extended_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_extended[] = {
-	&dev_attr_compute_extended_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_extended = {
-	.name = "bc9bcff2-459a-4cbc-986d-a84b077153f3",
-	.attrs =  attrs_compute_extended,
-};
-
-static ssize_t
-show_compute_l3_cache_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_L3_CACHE);
-}
-
-static struct device_attribute dev_attr_compute_l3_cache_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_l3_cache_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_l3_cache[] = {
-	&dev_attr_compute_l3_cache_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_l3_cache = {
-	.name = "88ec931f-5b4a-453a-9db6-a61232b6143d",
-	.attrs =  attrs_compute_l3_cache,
-};
-
-static ssize_t
-show_hdc_and_sf_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_HDC_AND_SF);
-}
-
-static struct device_attribute dev_attr_hdc_and_sf_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_hdc_and_sf_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_hdc_and_sf[] = {
-	&dev_attr_hdc_and_sf_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_hdc_and_sf = {
-	.name = "530d176d-2a18-4014-adf8-1500c6c60835",
-	.attrs =  attrs_hdc_and_sf,
-};
-
-static ssize_t
-show_l3_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_1);
-}
-
-static struct device_attribute dev_attr_l3_1_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_l3_1_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_l3_1[] = {
-	&dev_attr_l3_1_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_l3_1 = {
-	.name = "fdee5a5a-f23c-43d1-aa73-f6257c71671d",
-	.attrs =  attrs_l3_1,
-};
-
-static ssize_t
-show_rasterizer_and_pixel_backend_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND);
-}
-
-static struct device_attribute dev_attr_rasterizer_and_pixel_backend_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_rasterizer_and_pixel_backend_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_rasterizer_and_pixel_backend[] = {
-	&dev_attr_rasterizer_and_pixel_backend_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_rasterizer_and_pixel_backend = {
-	.name = "6617623e-ca73-4791-b2b7-ddedd0846a0c",
-	.attrs =  attrs_rasterizer_and_pixel_backend,
-};
-
-static ssize_t
-show_sampler_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_SAMPLER);
-}
-
-static struct device_attribute dev_attr_sampler_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_sampler_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_sampler[] = {
-	&dev_attr_sampler_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_sampler = {
-	.name = "f3b2ea63-e82e-4234-b418-44dd20dd34d0",
-	.attrs =  attrs_sampler,
-};
-
-static ssize_t
-show_tdl_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_1);
-}
-
-static struct device_attribute dev_attr_tdl_1_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_tdl_1_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_tdl_1[] = {
-	&dev_attr_tdl_1_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_tdl_1 = {
-	.name = "14411d35-cbf6-4f5e-b68b-190faf9a1a83",
-	.attrs =  attrs_tdl_1,
-};
-
-static ssize_t
-show_tdl_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_2);
-}
-
-static struct device_attribute dev_attr_tdl_2_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_tdl_2_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_tdl_2[] = {
-	&dev_attr_tdl_2_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_tdl_2 = {
-	.name = "ffa3f263-0478-4724-8c9f-c911c5ec0f1d",
-	.attrs =  attrs_tdl_2,
-};
-
-static ssize_t
-show_compute_extra_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTRA);
-}
-
-static struct device_attribute dev_attr_compute_extra_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_extra_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_extra[] = {
-	&dev_attr_compute_extra_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_extra = {
-	.name = "15274c82-27d2-4819-876a-7cb1a2c59ba4",
-	.attrs =  attrs_compute_extra,
-};
-
 static ssize_t
 show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TEST_OA);
+	return sprintf(buf, "1\n");
 }
 
-static struct device_attribute dev_attr_test_oa_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_test_oa_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_test_oa[] = {
-	&dev_attr_test_oa_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_test_oa = {
-	.name = "dd3fd789-e783-4204-8cd0-b671bbccb0cf",
-	.attrs =  attrs_test_oa,
-};
-
-int
-i915_perf_register_sysfs_glk(struct drm_i915_private *dev_priv)
+void
+i915_perf_load_test_config_glk(struct drm_i915_private *dev_priv)
 {
-	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
-	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
-	int ret = 0;
+	strncpy(dev_priv->perf.oa.test_config.uuid,
+		"dd3fd789-e783-4204-8cd0-b671bbccb0cf",
+		UUID_STRING_LEN);
+	dev_priv->perf.oa.test_config.id = 1;
 
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-		if (ret)
-			goto error_render_basic;
-	}
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-		if (ret)
-			goto error_compute_basic;
-	}
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-		if (ret)
-			goto error_render_pipe_profile;
-	}
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-		if (ret)
-			goto error_memory_reads;
-	}
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-		if (ret)
-			goto error_memory_writes;
-	}
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-		if (ret)
-			goto error_compute_extended;
-	}
-	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
-		if (ret)
-			goto error_compute_l3_cache;
-	}
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-		if (ret)
-			goto error_hdc_and_sf;
-	}
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-		if (ret)
-			goto error_l3_1;
-	}
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-		if (ret)
-			goto error_rasterizer_and_pixel_backend;
-	}
-	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_sampler);
-		if (ret)
-			goto error_sampler;
-	}
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-		if (ret)
-			goto error_tdl_1;
-	}
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-		if (ret)
-			goto error_tdl_2;
-	}
-	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
-		if (ret)
-			goto error_compute_extra;
-	}
-	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_test_oa);
-		if (ret)
-			goto error_test_oa;
-	}
+	dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
+	dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa);
 
-	return 0;
+	dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa;
+	dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa);
 
-error_test_oa:
-	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
-error_compute_extra:
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-error_tdl_2:
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-error_tdl_1:
-	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
-error_sampler:
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-error_rasterizer_and_pixel_backend:
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-error_l3_1:
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-error_hdc_and_sf:
-	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
-error_compute_l3_cache:
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-error_compute_extended:
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-error_memory_writes:
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-error_memory_reads:
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-error_render_pipe_profile:
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-error_compute_basic:
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-error_render_basic:
-	return ret;
-}
+	dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa;
+	dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa);
 
-void
-i915_perf_unregister_sysfs_glk(struct drm_i915_private *dev_priv)
-{
-	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
-	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+	dev_priv->perf.oa.test_config.sysfs_metric.name = "dd3fd789-e783-4204-8cd0-b671bbccb0cf";
+	dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs;
+
+	dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr;
 
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
-	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id";
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444;
+	dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id;
 }
diff --git a/drivers/gpu/drm/i915/i915_oa_glk.h b/drivers/gpu/drm/i915/i915_oa_glk.h
index 5511bb1cecf7..63bd113f4bc9 100644
--- a/drivers/gpu/drm/i915/i915_oa_glk.h
+++ b/drivers/gpu/drm/i915/i915_oa_glk.h
@@ -29,12 +29,6 @@
 #ifndef __I915_OA_GLK_H__
 #define __I915_OA_GLK_H__
 
-extern int i915_oa_n_builtin_metric_sets_glk;
-
-extern int i915_oa_select_metric_set_glk(struct drm_i915_private *dev_priv);
-
-extern int i915_perf_register_sysfs_glk(struct drm_i915_private *dev_priv);
-
-extern void i915_perf_unregister_sysfs_glk(struct drm_i915_private *dev_priv);
+extern void i915_perf_load_test_config_glk(struct drm_i915_private *dev_priv);
 
 #endif
diff --git a/drivers/gpu/drm/i915/i915_oa_hsw.c b/drivers/gpu/drm/i915/i915_oa_hsw.c
index 10f169f683b7..56b03773bb9d 100644
--- a/drivers/gpu/drm/i915/i915_oa_hsw.c
+++ b/drivers/gpu/drm/i915/i915_oa_hsw.c
@@ -31,17 +31,6 @@
 #include "i915_drv.h"
 #include "i915_oa_hsw.h"
 
-enum metric_set_id {
-	METRIC_SET_ID_RENDER_BASIC = 1,
-	METRIC_SET_ID_COMPUTE_BASIC,
-	METRIC_SET_ID_COMPUTE_EXTENDED,
-	METRIC_SET_ID_MEMORY_READS,
-	METRIC_SET_ID_MEMORY_WRITES,
-	METRIC_SET_ID_SAMPLER_BALANCE,
-};
-
-int i915_oa_n_builtin_metric_sets_hsw = 6;
-
 static const struct i915_oa_reg b_counter_config_render_basic[] = {
 	{ _MMIO(0x2724), 0x00800000 },
 	{ _MMIO(0x2720), 0x00000000 },
@@ -53,6 +42,7 @@ static const struct i915_oa_reg flex_eu_config_render_basic[] = {
 };
 
 static const struct i915_oa_reg mux_config_render_basic[] = {
+	{ _MMIO(0x9840), 0x00000080 },
 	{ _MMIO(0x253a4), 0x01600000 },
 	{ _MMIO(0x25440), 0x00100000 },
 	{ _MMIO(0x25128), 0x00000000 },
@@ -114,750 +104,35 @@ static const struct i915_oa_reg mux_config_render_basic[] = {
 	{ _MMIO(0x25428), 0x00042049 },
 };
 
-static int
-get_render_basic_mux_config(struct drm_i915_private *dev_priv,
-			    const struct i915_oa_reg **regs,
-			    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_render_basic;
-	lens[n] = ARRAY_SIZE(mux_config_render_basic);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_basic[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2718), 0xaaaaaaaa },
-	{ _MMIO(0x271c), 0xaaaaaaaa },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2728), 0xaaaaaaaa },
-	{ _MMIO(0x272c), 0xaaaaaaaa },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00000000 },
-	{ _MMIO(0x2748), 0x00000000 },
-	{ _MMIO(0x274c), 0x00000000 },
-	{ _MMIO(0x2750), 0x00000000 },
-	{ _MMIO(0x2754), 0x00000000 },
-	{ _MMIO(0x2758), 0x00000000 },
-	{ _MMIO(0x275c), 0x00000000 },
-	{ _MMIO(0x236c), 0x00000000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_basic[] = {
-};
-
-static const struct i915_oa_reg mux_config_compute_basic[] = {
-	{ _MMIO(0x253a4), 0x00000000 },
-	{ _MMIO(0x2681c), 0x01f00800 },
-	{ _MMIO(0x26820), 0x00001000 },
-	{ _MMIO(0x2781c), 0x01f00800 },
-	{ _MMIO(0x26520), 0x00000007 },
-	{ _MMIO(0x265a0), 0x00000007 },
-	{ _MMIO(0x25380), 0x00000010 },
-	{ _MMIO(0x2538c), 0x00300000 },
-	{ _MMIO(0x25384), 0xaa8aaaaa },
-	{ _MMIO(0x25404), 0xffffffff },
-	{ _MMIO(0x26800), 0x00004202 },
-	{ _MMIO(0x26808), 0x00605817 },
-	{ _MMIO(0x2680c), 0x10001005 },
-	{ _MMIO(0x26804), 0x00000000 },
-	{ _MMIO(0x27800), 0x00000102 },
-	{ _MMIO(0x27808), 0x0c0701e0 },
-	{ _MMIO(0x2780c), 0x000200a0 },
-	{ _MMIO(0x27804), 0x00000000 },
-	{ _MMIO(0x26484), 0x44000000 },
-	{ _MMIO(0x26704), 0x44000000 },
-	{ _MMIO(0x26500), 0x00000006 },
-	{ _MMIO(0x26510), 0x00000001 },
-	{ _MMIO(0x26504), 0x88000000 },
-	{ _MMIO(0x26580), 0x00000006 },
-	{ _MMIO(0x26590), 0x00000020 },
-	{ _MMIO(0x26584), 0x00000000 },
-	{ _MMIO(0x26104), 0x55822222 },
-	{ _MMIO(0x26184), 0xaa866666 },
-	{ _MMIO(0x25420), 0x08320c83 },
-	{ _MMIO(0x25424), 0x06820c83 },
-	{ _MMIO(0x2541c), 0x00000000 },
-	{ _MMIO(0x25428), 0x00000c03 },
-};
-
-static int
-get_compute_basic_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_basic;
-	lens[n] = ARRAY_SIZE(mux_config_compute_basic);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_extended[] = {
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007fe2a },
-	{ _MMIO(0x2774), 0x0000ff00 },
-	{ _MMIO(0x2778), 0x0007fe6a },
-	{ _MMIO(0x277c), 0x0000ff00 },
-	{ _MMIO(0x2780), 0x0007fe92 },
-	{ _MMIO(0x2784), 0x0000ff00 },
-	{ _MMIO(0x2788), 0x0007fea2 },
-	{ _MMIO(0x278c), 0x0000ff00 },
-	{ _MMIO(0x2790), 0x0007fe32 },
-	{ _MMIO(0x2794), 0x0000ff00 },
-	{ _MMIO(0x2798), 0x0007fe9a },
-	{ _MMIO(0x279c), 0x0000ff00 },
-	{ _MMIO(0x27a0), 0x0007ff23 },
-	{ _MMIO(0x27a4), 0x0000ff00 },
-	{ _MMIO(0x27a8), 0x0007fff3 },
-	{ _MMIO(0x27ac), 0x0000fffe },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_extended[] = {
-};
-
-static const struct i915_oa_reg mux_config_compute_extended[] = {
-	{ _MMIO(0x2681c), 0x3eb00800 },
-	{ _MMIO(0x26820), 0x00900000 },
-	{ _MMIO(0x25384), 0x02aaaaaa },
-	{ _MMIO(0x25404), 0x03ffffff },
-	{ _MMIO(0x26800), 0x00142284 },
-	{ _MMIO(0x26808), 0x0e629062 },
-	{ _MMIO(0x2680c), 0x3f6f55cb },
-	{ _MMIO(0x26810), 0x00000014 },
-	{ _MMIO(0x26804), 0x00000000 },
-	{ _MMIO(0x26104), 0x02aaaaaa },
-	{ _MMIO(0x26184), 0x02aaaaaa },
-	{ _MMIO(0x25420), 0x00000000 },
-	{ _MMIO(0x25424), 0x00000000 },
-	{ _MMIO(0x2541c), 0x00000000 },
-	{ _MMIO(0x25428), 0x00000000 },
-};
-
-static int
-get_compute_extended_mux_config(struct drm_i915_private *dev_priv,
-				const struct i915_oa_reg **regs,
-				int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_extended;
-	lens[n] = ARRAY_SIZE(mux_config_compute_extended);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_memory_reads[] = {
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x274c), 0x76543298 },
-	{ _MMIO(0x2748), 0x98989898 },
-	{ _MMIO(0x2744), 0x000000e4 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x275c), 0x98a98a98 },
-	{ _MMIO(0x2758), 0x88888888 },
-	{ _MMIO(0x2754), 0x000c5500 },
-	{ _MMIO(0x2750), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007f81a },
-	{ _MMIO(0x2774), 0x0000fc00 },
-	{ _MMIO(0x2778), 0x0007f82a },
-	{ _MMIO(0x277c), 0x0000fc00 },
-	{ _MMIO(0x2780), 0x0007f872 },
-	{ _MMIO(0x2784), 0x0000fc00 },
-	{ _MMIO(0x2788), 0x0007f8ba },
-	{ _MMIO(0x278c), 0x0000fc00 },
-	{ _MMIO(0x2790), 0x0007f87a },
-	{ _MMIO(0x2794), 0x0000fc00 },
-	{ _MMIO(0x2798), 0x0007f8ea },
-	{ _MMIO(0x279c), 0x0000fc00 },
-	{ _MMIO(0x27a0), 0x0007f8e2 },
-	{ _MMIO(0x27a4), 0x0000fc00 },
-	{ _MMIO(0x27a8), 0x0007f8f2 },
-	{ _MMIO(0x27ac), 0x0000fc00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_memory_reads[] = {
-};
-
-static const struct i915_oa_reg mux_config_memory_reads[] = {
-	{ _MMIO(0x253a4), 0x34300000 },
-	{ _MMIO(0x25440), 0x2d800000 },
-	{ _MMIO(0x25444), 0x00000008 },
-	{ _MMIO(0x25128), 0x0e600000 },
-	{ _MMIO(0x25380), 0x00000450 },
-	{ _MMIO(0x25390), 0x00052c43 },
-	{ _MMIO(0x25384), 0x00000000 },
-	{ _MMIO(0x25400), 0x00006144 },
-	{ _MMIO(0x25408), 0x0a418820 },
-	{ _MMIO(0x2540c), 0x000820e6 },
-	{ _MMIO(0x25404), 0xff500000 },
-	{ _MMIO(0x25100), 0x000005d6 },
-	{ _MMIO(0x2510c), 0x0ef00000 },
-	{ _MMIO(0x25104), 0x00000000 },
-	{ _MMIO(0x25420), 0x02108421 },
-	{ _MMIO(0x25424), 0x00008421 },
-	{ _MMIO(0x2541c), 0x00000000 },
-	{ _MMIO(0x25428), 0x00000000 },
-};
-
-static int
-get_memory_reads_mux_config(struct drm_i915_private *dev_priv,
-			    const struct i915_oa_reg **regs,
-			    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_memory_reads;
-	lens[n] = ARRAY_SIZE(mux_config_memory_reads);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_memory_writes[] = {
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x274c), 0x76543298 },
-	{ _MMIO(0x2748), 0x98989898 },
-	{ _MMIO(0x2744), 0x000000e4 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x275c), 0xbabababa },
-	{ _MMIO(0x2758), 0x88888888 },
-	{ _MMIO(0x2754), 0x000c5500 },
-	{ _MMIO(0x2750), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007f81a },
-	{ _MMIO(0x2774), 0x0000fc00 },
-	{ _MMIO(0x2778), 0x0007f82a },
-	{ _MMIO(0x277c), 0x0000fc00 },
-	{ _MMIO(0x2780), 0x0007f822 },
-	{ _MMIO(0x2784), 0x0000fc00 },
-	{ _MMIO(0x2788), 0x0007f8ba },
-	{ _MMIO(0x278c), 0x0000fc00 },
-	{ _MMIO(0x2790), 0x0007f87a },
-	{ _MMIO(0x2794), 0x0000fc00 },
-	{ _MMIO(0x2798), 0x0007f8ea },
-	{ _MMIO(0x279c), 0x0000fc00 },
-	{ _MMIO(0x27a0), 0x0007f8e2 },
-	{ _MMIO(0x27a4), 0x0000fc00 },
-	{ _MMIO(0x27a8), 0x0007f8f2 },
-	{ _MMIO(0x27ac), 0x0000fc00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_memory_writes[] = {
-};
-
-static const struct i915_oa_reg mux_config_memory_writes[] = {
-	{ _MMIO(0x253a4), 0x34300000 },
-	{ _MMIO(0x25440), 0x01500000 },
-	{ _MMIO(0x25444), 0x00000120 },
-	{ _MMIO(0x25128), 0x0c200000 },
-	{ _MMIO(0x25380), 0x00000450 },
-	{ _MMIO(0x25390), 0x00052c43 },
-	{ _MMIO(0x25384), 0x00000000 },
-	{ _MMIO(0x25400), 0x00007184 },
-	{ _MMIO(0x25408), 0x0a418820 },
-	{ _MMIO(0x2540c), 0x000820e6 },
-	{ _MMIO(0x25404), 0xff500000 },
-	{ _MMIO(0x25100), 0x000005d6 },
-	{ _MMIO(0x2510c), 0x1e700000 },
-	{ _MMIO(0x25104), 0x00000000 },
-	{ _MMIO(0x25420), 0x02108421 },
-	{ _MMIO(0x25424), 0x00008421 },
-	{ _MMIO(0x2541c), 0x00000000 },
-	{ _MMIO(0x25428), 0x00000000 },
-};
-
-static int
-get_memory_writes_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_memory_writes;
-	lens[n] = ARRAY_SIZE(mux_config_memory_writes);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_sampler_balance[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_sampler_balance[] = {
-};
-
-static const struct i915_oa_reg mux_config_sampler_balance[] = {
-	{ _MMIO(0x2eb9c), 0x01906400 },
-	{ _MMIO(0x2fb9c), 0x01906400 },
-	{ _MMIO(0x253a4), 0x00000000 },
-	{ _MMIO(0x26b9c), 0x01906400 },
-	{ _MMIO(0x27b9c), 0x01906400 },
-	{ _MMIO(0x27104), 0x00a00000 },
-	{ _MMIO(0x27184), 0x00a50000 },
-	{ _MMIO(0x2e804), 0x00500000 },
-	{ _MMIO(0x2e984), 0x00500000 },
-	{ _MMIO(0x2eb04), 0x00500000 },
-	{ _MMIO(0x2eb80), 0x00000084 },
-	{ _MMIO(0x2eb8c), 0x14200000 },
-	{ _MMIO(0x2eb84), 0x00000000 },
-	{ _MMIO(0x2f804), 0x00050000 },
-	{ _MMIO(0x2f984), 0x00050000 },
-	{ _MMIO(0x2fb04), 0x00050000 },
-	{ _MMIO(0x2fb80), 0x00000084 },
-	{ _MMIO(0x2fb8c), 0x00050800 },
-	{ _MMIO(0x2fb84), 0x00000000 },
-	{ _MMIO(0x25380), 0x00000010 },
-	{ _MMIO(0x2538c), 0x000000c0 },
-	{ _MMIO(0x25384), 0xaa550000 },
-	{ _MMIO(0x25404), 0xffffc000 },
-	{ _MMIO(0x26804), 0x50000000 },
-	{ _MMIO(0x26984), 0x50000000 },
-	{ _MMIO(0x26b04), 0x50000000 },
-	{ _MMIO(0x26b80), 0x00000084 },
-	{ _MMIO(0x26b90), 0x00050800 },
-	{ _MMIO(0x26b84), 0x00000000 },
-	{ _MMIO(0x27804), 0x05000000 },
-	{ _MMIO(0x27984), 0x05000000 },
-	{ _MMIO(0x27b04), 0x05000000 },
-	{ _MMIO(0x27b80), 0x00000084 },
-	{ _MMIO(0x27b90), 0x00000142 },
-	{ _MMIO(0x27b84), 0x00000000 },
-	{ _MMIO(0x26104), 0xa0000000 },
-	{ _MMIO(0x26184), 0xa5000000 },
-	{ _MMIO(0x25424), 0x00008620 },
-	{ _MMIO(0x2541c), 0x00000000 },
-	{ _MMIO(0x25428), 0x0004a54a },
-};
-
-static int
-get_sampler_balance_mux_config(struct drm_i915_private *dev_priv,
-			       const struct i915_oa_reg **regs,
-			       int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_sampler_balance;
-	lens[n] = ARRAY_SIZE(mux_config_sampler_balance);
-	n++;
-
-	return n;
-}
-
-int i915_oa_select_metric_set_hsw(struct drm_i915_private *dev_priv)
-{
-	dev_priv->perf.oa.n_mux_configs = 0;
-	dev_priv->perf.oa.b_counter_regs = NULL;
-	dev_priv->perf.oa.b_counter_regs_len = 0;
-
-	switch (dev_priv->perf.oa.metrics_set) {
-	case METRIC_SET_ID_RENDER_BASIC:
-		dev_priv->perf.oa.n_mux_configs =
-			get_render_basic_mux_config(dev_priv,
-						    dev_priv->perf.oa.mux_regs,
-						    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_BASIC\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_render_basic;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_render_basic);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_render_basic;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_render_basic);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_BASIC:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_basic_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_BASIC\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_basic;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_basic);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_basic;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_basic);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_EXTENDED:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_extended_mux_config(dev_priv,
-							dev_priv->perf.oa.mux_regs,
-							dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTENDED\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_extended;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_extended);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_extended;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_extended);
-
-		return 0;
-	case METRIC_SET_ID_MEMORY_READS:
-		dev_priv->perf.oa.n_mux_configs =
-			get_memory_reads_mux_config(dev_priv,
-						    dev_priv->perf.oa.mux_regs,
-						    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_READS\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_memory_reads;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_memory_reads);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_memory_reads;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_memory_reads);
-
-		return 0;
-	case METRIC_SET_ID_MEMORY_WRITES:
-		dev_priv->perf.oa.n_mux_configs =
-			get_memory_writes_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_WRITES\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_memory_writes;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_memory_writes);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_memory_writes;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_memory_writes);
-
-		return 0;
-	case METRIC_SET_ID_SAMPLER_BALANCE:
-		dev_priv->perf.oa.n_mux_configs =
-			get_sampler_balance_mux_config(dev_priv,
-						       dev_priv->perf.oa.mux_regs,
-						       dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"SAMPLER_BALANCE\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_sampler_balance;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_sampler_balance);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_sampler_balance;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_sampler_balance);
-
-		return 0;
-	default:
-		return -ENODEV;
-	}
-}
-
 static ssize_t
 show_render_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_BASIC);
-}
-
-static struct device_attribute dev_attr_render_basic_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_render_basic_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_render_basic[] = {
-	&dev_attr_render_basic_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_render_basic = {
-	.name = "403d8832-1a27-4aa6-a64e-f5389ce7b212",
-	.attrs =  attrs_render_basic,
-};
-
-static ssize_t
-show_compute_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_BASIC);
-}
-
-static struct device_attribute dev_attr_compute_basic_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_basic_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_basic[] = {
-	&dev_attr_compute_basic_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_basic = {
-	.name = "39ad14bc-2380-45c4-91eb-fbcb3aa7ae7b",
-	.attrs =  attrs_compute_basic,
-};
-
-static ssize_t
-show_compute_extended_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTENDED);
+	return sprintf(buf, "1\n");
 }
 
-static struct device_attribute dev_attr_compute_extended_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_extended_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_extended[] = {
-	&dev_attr_compute_extended_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_extended = {
-	.name = "3865be28-6982-49fe-9494-e4d1b4795413",
-	.attrs =  attrs_compute_extended,
-};
-
-static ssize_t
-show_memory_reads_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_READS);
-}
-
-static struct device_attribute dev_attr_memory_reads_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_memory_reads_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_memory_reads[] = {
-	&dev_attr_memory_reads_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_memory_reads = {
-	.name = "bb5ed49b-2497-4095-94f6-26ba294db88a",
-	.attrs =  attrs_memory_reads,
-};
-
-static ssize_t
-show_memory_writes_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_WRITES);
-}
-
-static struct device_attribute dev_attr_memory_writes_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_memory_writes_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_memory_writes[] = {
-	&dev_attr_memory_writes_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_memory_writes = {
-	.name = "3358d639-9b5f-45ab-976d-9b08cbfc6240",
-	.attrs =  attrs_memory_writes,
-};
-
-static ssize_t
-show_sampler_balance_id(struct device *kdev, struct device_attribute *attr, char *buf)
+void
+i915_perf_load_test_config_hsw(struct drm_i915_private *dev_priv)
 {
-	return sprintf(buf, "%d\n", METRIC_SET_ID_SAMPLER_BALANCE);
-}
-
-static struct device_attribute dev_attr_sampler_balance_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_sampler_balance_id,
-	.store = NULL,
-};
+	strncpy(dev_priv->perf.oa.test_config.uuid,
+		"403d8832-1a27-4aa6-a64e-f5389ce7b212",
+		UUID_STRING_LEN);
+	dev_priv->perf.oa.test_config.id = 1;
 
-static struct attribute *attrs_sampler_balance[] = {
-	&dev_attr_sampler_balance_id.attr,
-	NULL,
-};
+	dev_priv->perf.oa.test_config.mux_regs = mux_config_render_basic;
+	dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_render_basic);
 
-static struct attribute_group group_sampler_balance = {
-	.name = "bc274488-b4b6-40c7-90da-b77d7ad16189",
-	.attrs =  attrs_sampler_balance,
-};
+	dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_render_basic;
+	dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_render_basic);
 
-int
-i915_perf_register_sysfs_hsw(struct drm_i915_private *dev_priv)
-{
-	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
-	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
-	int ret = 0;
+	dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_render_basic;
+	dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_render_basic);
 
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-		if (ret)
-			goto error_render_basic;
-	}
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-		if (ret)
-			goto error_compute_basic;
-	}
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-		if (ret)
-			goto error_compute_extended;
-	}
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-		if (ret)
-			goto error_memory_reads;
-	}
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-		if (ret)
-			goto error_memory_writes;
-	}
-	if (get_sampler_balance_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_sampler_balance);
-		if (ret)
-			goto error_sampler_balance;
-	}
+	dev_priv->perf.oa.test_config.sysfs_metric.name = "403d8832-1a27-4aa6-a64e-f5389ce7b212";
+	dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs;
 
-	return 0;
-
-error_sampler_balance:
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-error_memory_writes:
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-error_memory_reads:
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-error_compute_extended:
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-error_compute_basic:
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-error_render_basic:
-	return ret;
-}
-
-void
-i915_perf_unregister_sysfs_hsw(struct drm_i915_private *dev_priv)
-{
-	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
-	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+	dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr;
 
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-	if (get_sampler_balance_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler_balance);
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id";
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444;
+	dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_render_basic_id;
 }
diff --git a/drivers/gpu/drm/i915/i915_oa_hsw.h b/drivers/gpu/drm/i915/i915_oa_hsw.h
index 6fe7e0690ef3..74d03439c157 100644
--- a/drivers/gpu/drm/i915/i915_oa_hsw.h
+++ b/drivers/gpu/drm/i915/i915_oa_hsw.h
@@ -29,12 +29,6 @@
 #ifndef __I915_OA_HSW_H__
 #define __I915_OA_HSW_H__
 
-extern int i915_oa_n_builtin_metric_sets_hsw;
-
-extern int i915_oa_select_metric_set_hsw(struct drm_i915_private *dev_priv);
-
-extern int i915_perf_register_sysfs_hsw(struct drm_i915_private *dev_priv);
-
-extern void i915_perf_unregister_sysfs_hsw(struct drm_i915_private *dev_priv);
+extern void i915_perf_load_test_config_hsw(struct drm_i915_private *dev_priv);
 
 #endif
diff --git a/drivers/gpu/drm/i915/i915_oa_kblgt2.c b/drivers/gpu/drm/i915/i915_oa_kblgt2.c
index 87dbd0a0b076..b6e7cc774136 100644
--- a/drivers/gpu/drm/i915/i915_oa_kblgt2.c
+++ b/drivers/gpu/drm/i915/i915_oa_kblgt2.c
@@ -31,1828 +31,6 @@
 #include "i915_drv.h"
 #include "i915_oa_kblgt2.h"
 
-enum metric_set_id {
-	METRIC_SET_ID_RENDER_BASIC = 1,
-	METRIC_SET_ID_COMPUTE_BASIC,
-	METRIC_SET_ID_RENDER_PIPE_PROFILE,
-	METRIC_SET_ID_MEMORY_READS,
-	METRIC_SET_ID_MEMORY_WRITES,
-	METRIC_SET_ID_COMPUTE_EXTENDED,
-	METRIC_SET_ID_COMPUTE_L3_CACHE,
-	METRIC_SET_ID_HDC_AND_SF,
-	METRIC_SET_ID_L3_1,
-	METRIC_SET_ID_L3_2,
-	METRIC_SET_ID_L3_3,
-	METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND,
-	METRIC_SET_ID_SAMPLER,
-	METRIC_SET_ID_TDL_1,
-	METRIC_SET_ID_TDL_2,
-	METRIC_SET_ID_COMPUTE_EXTRA,
-	METRIC_SET_ID_VME_PIPE,
-	METRIC_SET_ID_TEST_OA,
-};
-
-int i915_oa_n_builtin_metric_sets_kblgt2 = 18;
-
-static const struct i915_oa_reg b_counter_config_render_basic[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2740), 0x00000000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_render_basic[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_render_basic[] = {
-	{ _MMIO(0x9888), 0x166c01e0 },
-	{ _MMIO(0x9888), 0x12170280 },
-	{ _MMIO(0x9888), 0x12370280 },
-	{ _MMIO(0x9888), 0x11930317 },
-	{ _MMIO(0x9888), 0x159303df },
-	{ _MMIO(0x9888), 0x3f900003 },
-	{ _MMIO(0x9888), 0x1a4e0080 },
-	{ _MMIO(0x9888), 0x0a6c0053 },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x0a1b4000 },
-	{ _MMIO(0x9888), 0x1c1c0001 },
-	{ _MMIO(0x9888), 0x002f1000 },
-	{ _MMIO(0x9888), 0x042f1000 },
-	{ _MMIO(0x9888), 0x004c4000 },
-	{ _MMIO(0x9888), 0x0a4c8400 },
-	{ _MMIO(0x9888), 0x000d2000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0a0d2000 },
-	{ _MMIO(0x9888), 0x0c0f0400 },
-	{ _MMIO(0x9888), 0x0e0f6600 },
-	{ _MMIO(0x9888), 0x002c8000 },
-	{ _MMIO(0x9888), 0x162c2200 },
-	{ _MMIO(0x9888), 0x062d8000 },
-	{ _MMIO(0x9888), 0x082d8000 },
-	{ _MMIO(0x9888), 0x00133000 },
-	{ _MMIO(0x9888), 0x08133000 },
-	{ _MMIO(0x9888), 0x00170020 },
-	{ _MMIO(0x9888), 0x08170021 },
-	{ _MMIO(0x9888), 0x10170000 },
-	{ _MMIO(0x9888), 0x0633c000 },
-	{ _MMIO(0x9888), 0x0833c000 },
-	{ _MMIO(0x9888), 0x06370800 },
-	{ _MMIO(0x9888), 0x08370840 },
-	{ _MMIO(0x9888), 0x10370000 },
-	{ _MMIO(0x9888), 0x0d933031 },
-	{ _MMIO(0x9888), 0x0f933e3f },
-	{ _MMIO(0x9888), 0x01933d00 },
-	{ _MMIO(0x9888), 0x0393073c },
-	{ _MMIO(0x9888), 0x0593000e },
-	{ _MMIO(0x9888), 0x1d930000 },
-	{ _MMIO(0x9888), 0x19930000 },
-	{ _MMIO(0x9888), 0x1b930000 },
-	{ _MMIO(0x9888), 0x1d900157 },
-	{ _MMIO(0x9888), 0x1f900158 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x2b908000 },
-	{ _MMIO(0x9888), 0x2d908000 },
-	{ _MMIO(0x9888), 0x2f908000 },
-	{ _MMIO(0x9888), 0x31908000 },
-	{ _MMIO(0x9888), 0x15908000 },
-	{ _MMIO(0x9888), 0x17908000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1190001f },
-	{ _MMIO(0x9888), 0x51904400 },
-	{ _MMIO(0x9888), 0x41900020 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900c21 },
-	{ _MMIO(0x9888), 0x47900061 },
-	{ _MMIO(0x9888), 0x57904440 },
-	{ _MMIO(0x9888), 0x49900000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900000 },
-	{ _MMIO(0x9888), 0x59900004 },
-	{ _MMIO(0x9888), 0x43900000 },
-	{ _MMIO(0x9888), 0x53904444 },
-};
-
-static int
-get_render_basic_mux_config(struct drm_i915_private *dev_priv,
-			    const struct i915_oa_reg **regs,
-			    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_render_basic;
-	lens[n] = ARRAY_SIZE(mux_config_render_basic);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_basic[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2740), 0x00000000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_basic[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00778008 },
-	{ _MMIO(0xe45c), 0x00088078 },
-	{ _MMIO(0xe55c), 0x00808708 },
-	{ _MMIO(0xe65c), 0x00a08908 },
-};
-
-static const struct i915_oa_reg mux_config_compute_basic[] = {
-	{ _MMIO(0x9888), 0x104f00e0 },
-	{ _MMIO(0x9888), 0x124f1c00 },
-	{ _MMIO(0x9888), 0x106c00e0 },
-	{ _MMIO(0x9888), 0x37906800 },
-	{ _MMIO(0x9888), 0x3f900003 },
-	{ _MMIO(0x9888), 0x004e8000 },
-	{ _MMIO(0x9888), 0x1a4e0820 },
-	{ _MMIO(0x9888), 0x1c4e0002 },
-	{ _MMIO(0x9888), 0x064f0900 },
-	{ _MMIO(0x9888), 0x084f0032 },
-	{ _MMIO(0x9888), 0x0a4f1891 },
-	{ _MMIO(0x9888), 0x0c4f0e00 },
-	{ _MMIO(0x9888), 0x0e4f003c },
-	{ _MMIO(0x9888), 0x004f0d80 },
-	{ _MMIO(0x9888), 0x024f003b },
-	{ _MMIO(0x9888), 0x006c0002 },
-	{ _MMIO(0x9888), 0x086c0100 },
-	{ _MMIO(0x9888), 0x0c6c000c },
-	{ _MMIO(0x9888), 0x0e6c0b00 },
-	{ _MMIO(0x9888), 0x186c0000 },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1e6c0000 },
-	{ _MMIO(0x9888), 0x001b4000 },
-	{ _MMIO(0x9888), 0x081b8000 },
-	{ _MMIO(0x9888), 0x0c1b4000 },
-	{ _MMIO(0x9888), 0x0e1b8000 },
-	{ _MMIO(0x9888), 0x101c8000 },
-	{ _MMIO(0x9888), 0x1a1c8000 },
-	{ _MMIO(0x9888), 0x1c1c0024 },
-	{ _MMIO(0x9888), 0x065b8000 },
-	{ _MMIO(0x9888), 0x085b4000 },
-	{ _MMIO(0x9888), 0x0a5bc000 },
-	{ _MMIO(0x9888), 0x0c5b8000 },
-	{ _MMIO(0x9888), 0x0e5b4000 },
-	{ _MMIO(0x9888), 0x005b8000 },
-	{ _MMIO(0x9888), 0x025b4000 },
-	{ _MMIO(0x9888), 0x1a5c6000 },
-	{ _MMIO(0x9888), 0x1c5c001b },
-	{ _MMIO(0x9888), 0x125c8000 },
-	{ _MMIO(0x9888), 0x145c8000 },
-	{ _MMIO(0x9888), 0x004c8000 },
-	{ _MMIO(0x9888), 0x0a4c2000 },
-	{ _MMIO(0x9888), 0x0c4c0208 },
-	{ _MMIO(0x9888), 0x000da000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0a0da000 },
-	{ _MMIO(0x9888), 0x0c0da000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x020d2000 },
-	{ _MMIO(0x9888), 0x0c0f5400 },
-	{ _MMIO(0x9888), 0x0e0f5500 },
-	{ _MMIO(0x9888), 0x100f0155 },
-	{ _MMIO(0x9888), 0x002c8000 },
-	{ _MMIO(0x9888), 0x0e2cc000 },
-	{ _MMIO(0x9888), 0x162cfb00 },
-	{ _MMIO(0x9888), 0x182c00be },
-	{ _MMIO(0x9888), 0x022cc000 },
-	{ _MMIO(0x9888), 0x042cc000 },
-	{ _MMIO(0x9888), 0x19900157 },
-	{ _MMIO(0x9888), 0x1b900158 },
-	{ _MMIO(0x9888), 0x1d900105 },
-	{ _MMIO(0x9888), 0x1f900103 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x11900fff },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900800 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900821 },
-	{ _MMIO(0x9888), 0x47900802 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900802 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900002 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900422 },
-	{ _MMIO(0x9888), 0x53904444 },
-};
-
-static int
-get_compute_basic_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_basic;
-	lens[n] = ARRAY_SIZE(mux_config_compute_basic);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_render_pipe_profile[] = {
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007ffea },
-	{ _MMIO(0x2774), 0x00007ffc },
-	{ _MMIO(0x2778), 0x0007affa },
-	{ _MMIO(0x277c), 0x0000f5fd },
-	{ _MMIO(0x2780), 0x00079ffa },
-	{ _MMIO(0x2784), 0x0000f3fb },
-	{ _MMIO(0x2788), 0x0007bf7a },
-	{ _MMIO(0x278c), 0x0000f7e7 },
-	{ _MMIO(0x2790), 0x0007fefa },
-	{ _MMIO(0x2794), 0x0000f7cf },
-	{ _MMIO(0x2798), 0x00077ffa },
-	{ _MMIO(0x279c), 0x0000efdf },
-	{ _MMIO(0x27a0), 0x0006fffa },
-	{ _MMIO(0x27a4), 0x0000cfbf },
-	{ _MMIO(0x27a8), 0x0003fffa },
-	{ _MMIO(0x27ac), 0x00005f7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_render_pipe_profile[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00015014 },
-	{ _MMIO(0xe658), 0x00025024 },
-	{ _MMIO(0xe758), 0x00035034 },
-	{ _MMIO(0xe45c), 0x00045044 },
-	{ _MMIO(0xe55c), 0x00055054 },
-	{ _MMIO(0xe65c), 0x00065064 },
-};
-
-static const struct i915_oa_reg mux_config_render_pipe_profile[] = {
-	{ _MMIO(0x9888), 0x0c0e001f },
-	{ _MMIO(0x9888), 0x0a0f0000 },
-	{ _MMIO(0x9888), 0x10116800 },
-	{ _MMIO(0x9888), 0x178a03e0 },
-	{ _MMIO(0x9888), 0x11824c00 },
-	{ _MMIO(0x9888), 0x11830020 },
-	{ _MMIO(0x9888), 0x13840020 },
-	{ _MMIO(0x9888), 0x11850019 },
-	{ _MMIO(0x9888), 0x11860007 },
-	{ _MMIO(0x9888), 0x01870c40 },
-	{ _MMIO(0x9888), 0x17880000 },
-	{ _MMIO(0x9888), 0x022f4000 },
-	{ _MMIO(0x9888), 0x0a4c0040 },
-	{ _MMIO(0x9888), 0x0c0d8000 },
-	{ _MMIO(0x9888), 0x040d4000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x020e5400 },
-	{ _MMIO(0x9888), 0x000e0000 },
-	{ _MMIO(0x9888), 0x080f0040 },
-	{ _MMIO(0x9888), 0x000f0000 },
-	{ _MMIO(0x9888), 0x100f0000 },
-	{ _MMIO(0x9888), 0x0e0f0040 },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x06104000 },
-	{ _MMIO(0x9888), 0x06110012 },
-	{ _MMIO(0x9888), 0x06131000 },
-	{ _MMIO(0x9888), 0x01898000 },
-	{ _MMIO(0x9888), 0x0d890100 },
-	{ _MMIO(0x9888), 0x03898000 },
-	{ _MMIO(0x9888), 0x09808000 },
-	{ _MMIO(0x9888), 0x0b808000 },
-	{ _MMIO(0x9888), 0x0380c000 },
-	{ _MMIO(0x9888), 0x0f8a0075 },
-	{ _MMIO(0x9888), 0x1d8a0000 },
-	{ _MMIO(0x9888), 0x118a8000 },
-	{ _MMIO(0x9888), 0x1b8a4000 },
-	{ _MMIO(0x9888), 0x138a8000 },
-	{ _MMIO(0x9888), 0x1d81a000 },
-	{ _MMIO(0x9888), 0x15818000 },
-	{ _MMIO(0x9888), 0x17818000 },
-	{ _MMIO(0x9888), 0x0b820030 },
-	{ _MMIO(0x9888), 0x07828000 },
-	{ _MMIO(0x9888), 0x0d824000 },
-	{ _MMIO(0x9888), 0x0f828000 },
-	{ _MMIO(0x9888), 0x05824000 },
-	{ _MMIO(0x9888), 0x0d830003 },
-	{ _MMIO(0x9888), 0x0583000c },
-	{ _MMIO(0x9888), 0x09830000 },
-	{ _MMIO(0x9888), 0x03838000 },
-	{ _MMIO(0x9888), 0x07838000 },
-	{ _MMIO(0x9888), 0x0b840980 },
-	{ _MMIO(0x9888), 0x03844d80 },
-	{ _MMIO(0x9888), 0x11840000 },
-	{ _MMIO(0x9888), 0x09848000 },
-	{ _MMIO(0x9888), 0x09850080 },
-	{ _MMIO(0x9888), 0x03850003 },
-	{ _MMIO(0x9888), 0x01850000 },
-	{ _MMIO(0x9888), 0x07860000 },
-	{ _MMIO(0x9888), 0x0f860400 },
-	{ _MMIO(0x9888), 0x09870032 },
-	{ _MMIO(0x9888), 0x01888052 },
-	{ _MMIO(0x9888), 0x11880000 },
-	{ _MMIO(0x9888), 0x09884000 },
-	{ _MMIO(0x9888), 0x1b931001 },
-	{ _MMIO(0x9888), 0x1d930001 },
-	{ _MMIO(0x9888), 0x19934000 },
-	{ _MMIO(0x9888), 0x1b958000 },
-	{ _MMIO(0x9888), 0x1d950094 },
-	{ _MMIO(0x9888), 0x19958000 },
-	{ _MMIO(0x9888), 0x09e58000 },
-	{ _MMIO(0x9888), 0x0be58000 },
-	{ _MMIO(0x9888), 0x03e5c000 },
-	{ _MMIO(0x9888), 0x0592c000 },
-	{ _MMIO(0x9888), 0x0b928000 },
-	{ _MMIO(0x9888), 0x0d924000 },
-	{ _MMIO(0x9888), 0x0f924000 },
-	{ _MMIO(0x9888), 0x11928000 },
-	{ _MMIO(0x9888), 0x1392c000 },
-	{ _MMIO(0x9888), 0x09924000 },
-	{ _MMIO(0x9888), 0x01985000 },
-	{ _MMIO(0x9888), 0x07988000 },
-	{ _MMIO(0x9888), 0x09981000 },
-	{ _MMIO(0x9888), 0x0b982000 },
-	{ _MMIO(0x9888), 0x0d982000 },
-	{ _MMIO(0x9888), 0x0f989000 },
-	{ _MMIO(0x9888), 0x05982000 },
-	{ _MMIO(0x9888), 0x13904000 },
-	{ _MMIO(0x9888), 0x21904000 },
-	{ _MMIO(0x9888), 0x23904000 },
-	{ _MMIO(0x9888), 0x25908000 },
-	{ _MMIO(0x9888), 0x27904000 },
-	{ _MMIO(0x9888), 0x29908000 },
-	{ _MMIO(0x9888), 0x2b904000 },
-	{ _MMIO(0x9888), 0x2f904000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x15904000 },
-	{ _MMIO(0x9888), 0x17908000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x1190c080 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900440 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900400 },
-	{ _MMIO(0x9888), 0x47900c21 },
-	{ _MMIO(0x9888), 0x57900400 },
-	{ _MMIO(0x9888), 0x49900042 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900024 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900841 },
-	{ _MMIO(0x9888), 0x53900400 },
-};
-
-static int
-get_render_pipe_profile_mux_config(struct drm_i915_private *dev_priv,
-				   const struct i915_oa_reg **regs,
-				   int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_render_pipe_profile;
-	lens[n] = ARRAY_SIZE(mux_config_render_pipe_profile);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_memory_reads[] = {
-	{ _MMIO(0x272c), 0xffffffff },
-	{ _MMIO(0x2728), 0xffffffff },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x271c), 0xffffffff },
-	{ _MMIO(0x2718), 0xffffffff },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x274c), 0x86543210 },
-	{ _MMIO(0x2748), 0x86543210 },
-	{ _MMIO(0x2744), 0x00006667 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x275c), 0x86543210 },
-	{ _MMIO(0x2758), 0x86543210 },
-	{ _MMIO(0x2754), 0x00006465 },
-	{ _MMIO(0x2750), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007f81a },
-	{ _MMIO(0x2774), 0x0000fe00 },
-	{ _MMIO(0x2778), 0x0007f82a },
-	{ _MMIO(0x277c), 0x0000fe00 },
-	{ _MMIO(0x2780), 0x0007f872 },
-	{ _MMIO(0x2784), 0x0000fe00 },
-	{ _MMIO(0x2788), 0x0007f8ba },
-	{ _MMIO(0x278c), 0x0000fe00 },
-	{ _MMIO(0x2790), 0x0007f87a },
-	{ _MMIO(0x2794), 0x0000fe00 },
-	{ _MMIO(0x2798), 0x0007f8ea },
-	{ _MMIO(0x279c), 0x0000fe00 },
-	{ _MMIO(0x27a0), 0x0007f8e2 },
-	{ _MMIO(0x27a4), 0x0000fe00 },
-	{ _MMIO(0x27a8), 0x0007f8f2 },
-	{ _MMIO(0x27ac), 0x0000fe00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_memory_reads[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00015014 },
-	{ _MMIO(0xe658), 0x00025024 },
-	{ _MMIO(0xe758), 0x00035034 },
-	{ _MMIO(0xe45c), 0x00045044 },
-	{ _MMIO(0xe55c), 0x00055054 },
-	{ _MMIO(0xe65c), 0x00065064 },
-};
-
-static const struct i915_oa_reg mux_config_memory_reads[] = {
-	{ _MMIO(0x9888), 0x11810c00 },
-	{ _MMIO(0x9888), 0x1381001a },
-	{ _MMIO(0x9888), 0x37906800 },
-	{ _MMIO(0x9888), 0x3f900064 },
-	{ _MMIO(0x9888), 0x03811300 },
-	{ _MMIO(0x9888), 0x05811b12 },
-	{ _MMIO(0x9888), 0x0781001a },
-	{ _MMIO(0x9888), 0x1f810000 },
-	{ _MMIO(0x9888), 0x17810000 },
-	{ _MMIO(0x9888), 0x19810000 },
-	{ _MMIO(0x9888), 0x1b810000 },
-	{ _MMIO(0x9888), 0x1d810000 },
-	{ _MMIO(0x9888), 0x1b930055 },
-	{ _MMIO(0x9888), 0x03e58000 },
-	{ _MMIO(0x9888), 0x05e5c000 },
-	{ _MMIO(0x9888), 0x07e54000 },
-	{ _MMIO(0x9888), 0x13900150 },
-	{ _MMIO(0x9888), 0x21900151 },
-	{ _MMIO(0x9888), 0x23900152 },
-	{ _MMIO(0x9888), 0x25900153 },
-	{ _MMIO(0x9888), 0x27900154 },
-	{ _MMIO(0x9888), 0x29900155 },
-	{ _MMIO(0x9888), 0x2b900156 },
-	{ _MMIO(0x9888), 0x2d900157 },
-	{ _MMIO(0x9888), 0x2f90015f },
-	{ _MMIO(0x9888), 0x31900105 },
-	{ _MMIO(0x9888), 0x15900103 },
-	{ _MMIO(0x9888), 0x17900101 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1d908000 },
-	{ _MMIO(0x9888), 0x1f908000 },
-	{ _MMIO(0x9888), 0x11900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c60 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900c00 },
-	{ _MMIO(0x9888), 0x47900c63 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900c63 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900063 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900003 },
-	{ _MMIO(0x9888), 0x53900000 },
-};
-
-static int
-get_memory_reads_mux_config(struct drm_i915_private *dev_priv,
-			    const struct i915_oa_reg **regs,
-			    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_memory_reads;
-	lens[n] = ARRAY_SIZE(mux_config_memory_reads);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_memory_writes[] = {
-	{ _MMIO(0x272c), 0xffffffff },
-	{ _MMIO(0x2728), 0xffffffff },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x271c), 0xffffffff },
-	{ _MMIO(0x2718), 0xffffffff },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x274c), 0x86543210 },
-	{ _MMIO(0x2748), 0x86543210 },
-	{ _MMIO(0x2744), 0x00006667 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x275c), 0x86543210 },
-	{ _MMIO(0x2758), 0x86543210 },
-	{ _MMIO(0x2754), 0x00006465 },
-	{ _MMIO(0x2750), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007f81a },
-	{ _MMIO(0x2774), 0x0000fe00 },
-	{ _MMIO(0x2778), 0x0007f82a },
-	{ _MMIO(0x277c), 0x0000fe00 },
-	{ _MMIO(0x2780), 0x0007f822 },
-	{ _MMIO(0x2784), 0x0000fe00 },
-	{ _MMIO(0x2788), 0x0007f8ba },
-	{ _MMIO(0x278c), 0x0000fe00 },
-	{ _MMIO(0x2790), 0x0007f87a },
-	{ _MMIO(0x2794), 0x0000fe00 },
-	{ _MMIO(0x2798), 0x0007f8ea },
-	{ _MMIO(0x279c), 0x0000fe00 },
-	{ _MMIO(0x27a0), 0x0007f8e2 },
-	{ _MMIO(0x27a4), 0x0000fe00 },
-	{ _MMIO(0x27a8), 0x0007f8f2 },
-	{ _MMIO(0x27ac), 0x0000fe00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_memory_writes[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00015014 },
-	{ _MMIO(0xe658), 0x00025024 },
-	{ _MMIO(0xe758), 0x00035034 },
-	{ _MMIO(0xe45c), 0x00045044 },
-	{ _MMIO(0xe55c), 0x00055054 },
-	{ _MMIO(0xe65c), 0x00065064 },
-};
-
-static const struct i915_oa_reg mux_config_memory_writes[] = {
-	{ _MMIO(0x9888), 0x11810c00 },
-	{ _MMIO(0x9888), 0x1381001a },
-	{ _MMIO(0x9888), 0x37906800 },
-	{ _MMIO(0x9888), 0x3f901000 },
-	{ _MMIO(0x9888), 0x03811300 },
-	{ _MMIO(0x9888), 0x05811b12 },
-	{ _MMIO(0x9888), 0x0781001a },
-	{ _MMIO(0x9888), 0x1f810000 },
-	{ _MMIO(0x9888), 0x17810000 },
-	{ _MMIO(0x9888), 0x19810000 },
-	{ _MMIO(0x9888), 0x1b810000 },
-	{ _MMIO(0x9888), 0x1d810000 },
-	{ _MMIO(0x9888), 0x1b930055 },
-	{ _MMIO(0x9888), 0x03e58000 },
-	{ _MMIO(0x9888), 0x05e5c000 },
-	{ _MMIO(0x9888), 0x07e54000 },
-	{ _MMIO(0x9888), 0x13900160 },
-	{ _MMIO(0x9888), 0x21900161 },
-	{ _MMIO(0x9888), 0x23900162 },
-	{ _MMIO(0x9888), 0x25900163 },
-	{ _MMIO(0x9888), 0x27900164 },
-	{ _MMIO(0x9888), 0x29900165 },
-	{ _MMIO(0x9888), 0x2b900166 },
-	{ _MMIO(0x9888), 0x2d900167 },
-	{ _MMIO(0x9888), 0x2f900150 },
-	{ _MMIO(0x9888), 0x31900105 },
-	{ _MMIO(0x9888), 0x15900103 },
-	{ _MMIO(0x9888), 0x17900101 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1d908000 },
-	{ _MMIO(0x9888), 0x1f908000 },
-	{ _MMIO(0x9888), 0x11900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c60 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900c00 },
-	{ _MMIO(0x9888), 0x47900c63 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900c63 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900063 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900003 },
-	{ _MMIO(0x9888), 0x53900000 },
-};
-
-static int
-get_memory_writes_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_memory_writes;
-	lens[n] = ARRAY_SIZE(mux_config_memory_writes);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_extended[] = {
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007fc2a },
-	{ _MMIO(0x2774), 0x0000bf00 },
-	{ _MMIO(0x2778), 0x0007fc6a },
-	{ _MMIO(0x277c), 0x0000bf00 },
-	{ _MMIO(0x2780), 0x0007fc92 },
-	{ _MMIO(0x2784), 0x0000bf00 },
-	{ _MMIO(0x2788), 0x0007fca2 },
-	{ _MMIO(0x278c), 0x0000bf00 },
-	{ _MMIO(0x2790), 0x0007fc32 },
-	{ _MMIO(0x2794), 0x0000bf00 },
-	{ _MMIO(0x2798), 0x0007fc9a },
-	{ _MMIO(0x279c), 0x0000bf00 },
-	{ _MMIO(0x27a0), 0x0007fe6a },
-	{ _MMIO(0x27a4), 0x0000bf00 },
-	{ _MMIO(0x27a8), 0x0007fe7a },
-	{ _MMIO(0x27ac), 0x0000bf00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_extended[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00778008 },
-	{ _MMIO(0xe45c), 0x00088078 },
-	{ _MMIO(0xe55c), 0x00808708 },
-	{ _MMIO(0xe65c), 0x00a08908 },
-};
-
-static const struct i915_oa_reg mux_config_compute_extended[] = {
-	{ _MMIO(0x9888), 0x106c00e0 },
-	{ _MMIO(0x9888), 0x141c8160 },
-	{ _MMIO(0x9888), 0x161c8015 },
-	{ _MMIO(0x9888), 0x181c0120 },
-	{ _MMIO(0x9888), 0x004e8000 },
-	{ _MMIO(0x9888), 0x0e4e8000 },
-	{ _MMIO(0x9888), 0x184e8000 },
-	{ _MMIO(0x9888), 0x1a4eaaa0 },
-	{ _MMIO(0x9888), 0x1c4e0002 },
-	{ _MMIO(0x9888), 0x024e8000 },
-	{ _MMIO(0x9888), 0x044e8000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084e8000 },
-	{ _MMIO(0x9888), 0x0a4e8000 },
-	{ _MMIO(0x9888), 0x0e6c0b01 },
-	{ _MMIO(0x9888), 0x006c0200 },
-	{ _MMIO(0x9888), 0x026c000c },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1e6c0000 },
-	{ _MMIO(0x9888), 0x1a6c0000 },
-	{ _MMIO(0x9888), 0x0e1bc000 },
-	{ _MMIO(0x9888), 0x001b8000 },
-	{ _MMIO(0x9888), 0x021bc000 },
-	{ _MMIO(0x9888), 0x001c0041 },
-	{ _MMIO(0x9888), 0x061c4200 },
-	{ _MMIO(0x9888), 0x081c4443 },
-	{ _MMIO(0x9888), 0x0a1c4645 },
-	{ _MMIO(0x9888), 0x0c1c7647 },
-	{ _MMIO(0x9888), 0x041c7357 },
-	{ _MMIO(0x9888), 0x1c1c0030 },
-	{ _MMIO(0x9888), 0x101c0000 },
-	{ _MMIO(0x9888), 0x1a1c0000 },
-	{ _MMIO(0x9888), 0x121c8000 },
-	{ _MMIO(0x9888), 0x004c8000 },
-	{ _MMIO(0x9888), 0x0a4caa2a },
-	{ _MMIO(0x9888), 0x0c4c02aa },
-	{ _MMIO(0x9888), 0x084ca000 },
-	{ _MMIO(0x9888), 0x000da000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0a0da000 },
-	{ _MMIO(0x9888), 0x0c0da000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x0c0f5400 },
-	{ _MMIO(0x9888), 0x0e0f5515 },
-	{ _MMIO(0x9888), 0x100f0155 },
-	{ _MMIO(0x9888), 0x002c8000 },
-	{ _MMIO(0x9888), 0x0e2c8000 },
-	{ _MMIO(0x9888), 0x162caa00 },
-	{ _MMIO(0x9888), 0x182c00aa },
-	{ _MMIO(0x9888), 0x022c8000 },
-	{ _MMIO(0x9888), 0x042c8000 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x11907fff },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900040 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900802 },
-	{ _MMIO(0x9888), 0x47900842 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900842 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900800 },
-	{ _MMIO(0x9888), 0x53900000 },
-};
-
-static int
-get_compute_extended_mux_config(struct drm_i915_private *dev_priv,
-				const struct i915_oa_reg **regs,
-				int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_extended;
-	lens[n] = ARRAY_SIZE(mux_config_compute_extended);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_l3_cache[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x30800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007fffa },
-	{ _MMIO(0x2774), 0x0000fefe },
-	{ _MMIO(0x2778), 0x0007fffa },
-	{ _MMIO(0x277c), 0x0000fefd },
-	{ _MMIO(0x2790), 0x0007fffa },
-	{ _MMIO(0x2794), 0x0000fbef },
-	{ _MMIO(0x2798), 0x0007fffa },
-	{ _MMIO(0x279c), 0x0000fbdf },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_l3_cache[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00101100 },
-	{ _MMIO(0xe45c), 0x00201200 },
-	{ _MMIO(0xe55c), 0x00301300 },
-	{ _MMIO(0xe65c), 0x00401400 },
-};
-
-static const struct i915_oa_reg mux_config_compute_l3_cache[] = {
-	{ _MMIO(0x9888), 0x166c0760 },
-	{ _MMIO(0x9888), 0x1593001e },
-	{ _MMIO(0x9888), 0x3f900003 },
-	{ _MMIO(0x9888), 0x004e8000 },
-	{ _MMIO(0x9888), 0x0e4e8000 },
-	{ _MMIO(0x9888), 0x184e8000 },
-	{ _MMIO(0x9888), 0x1a4e8020 },
-	{ _MMIO(0x9888), 0x1c4e0002 },
-	{ _MMIO(0x9888), 0x006c0051 },
-	{ _MMIO(0x9888), 0x066c5000 },
-	{ _MMIO(0x9888), 0x086c5c5d },
-	{ _MMIO(0x9888), 0x0e6c5e5f },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x186c0000 },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1e6c0000 },
-	{ _MMIO(0x9888), 0x001b4000 },
-	{ _MMIO(0x9888), 0x061b8000 },
-	{ _MMIO(0x9888), 0x081bc000 },
-	{ _MMIO(0x9888), 0x0e1bc000 },
-	{ _MMIO(0x9888), 0x101c8000 },
-	{ _MMIO(0x9888), 0x1a1ce000 },
-	{ _MMIO(0x9888), 0x1c1c0030 },
-	{ _MMIO(0x9888), 0x004c8000 },
-	{ _MMIO(0x9888), 0x0a4c2a00 },
-	{ _MMIO(0x9888), 0x0c4c0280 },
-	{ _MMIO(0x9888), 0x000d2000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x0c0f0400 },
-	{ _MMIO(0x9888), 0x0e0f1500 },
-	{ _MMIO(0x9888), 0x100f0140 },
-	{ _MMIO(0x9888), 0x002c8000 },
-	{ _MMIO(0x9888), 0x0e2c8000 },
-	{ _MMIO(0x9888), 0x162c0a00 },
-	{ _MMIO(0x9888), 0x182c00a0 },
-	{ _MMIO(0x9888), 0x03933300 },
-	{ _MMIO(0x9888), 0x05930032 },
-	{ _MMIO(0x9888), 0x11930000 },
-	{ _MMIO(0x9888), 0x1b930000 },
-	{ _MMIO(0x9888), 0x1d900157 },
-	{ _MMIO(0x9888), 0x1f900158 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1190030f },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900000 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900021 },
-	{ _MMIO(0x9888), 0x47900000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x4b900000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x53904444 },
-	{ _MMIO(0x9888), 0x43900000 },
-};
-
-static int
-get_compute_l3_cache_mux_config(struct drm_i915_private *dev_priv,
-				const struct i915_oa_reg **regs,
-				int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_l3_cache;
-	lens[n] = ARRAY_SIZE(mux_config_compute_l3_cache);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_hdc_and_sf[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x10800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x0000fdff },
-};
-
-static const struct i915_oa_reg flex_eu_config_hdc_and_sf[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_hdc_and_sf[] = {
-	{ _MMIO(0x9888), 0x104f0232 },
-	{ _MMIO(0x9888), 0x124f4640 },
-	{ _MMIO(0x9888), 0x106c0232 },
-	{ _MMIO(0x9888), 0x11834400 },
-	{ _MMIO(0x9888), 0x0a4e8000 },
-	{ _MMIO(0x9888), 0x0c4e8000 },
-	{ _MMIO(0x9888), 0x004f1880 },
-	{ _MMIO(0x9888), 0x024f08bb },
-	{ _MMIO(0x9888), 0x044f001b },
-	{ _MMIO(0x9888), 0x046c0100 },
-	{ _MMIO(0x9888), 0x066c000b },
-	{ _MMIO(0x9888), 0x1a6c0000 },
-	{ _MMIO(0x9888), 0x041b8000 },
-	{ _MMIO(0x9888), 0x061b4000 },
-	{ _MMIO(0x9888), 0x1a1c1800 },
-	{ _MMIO(0x9888), 0x005b8000 },
-	{ _MMIO(0x9888), 0x025bc000 },
-	{ _MMIO(0x9888), 0x045b4000 },
-	{ _MMIO(0x9888), 0x125c8000 },
-	{ _MMIO(0x9888), 0x145c8000 },
-	{ _MMIO(0x9888), 0x165c8000 },
-	{ _MMIO(0x9888), 0x185c8000 },
-	{ _MMIO(0x9888), 0x0a4c00a0 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x0c0f5000 },
-	{ _MMIO(0x9888), 0x0e0f0055 },
-	{ _MMIO(0x9888), 0x022cc000 },
-	{ _MMIO(0x9888), 0x042cc000 },
-	{ _MMIO(0x9888), 0x062cc000 },
-	{ _MMIO(0x9888), 0x082cc000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x0f828000 },
-	{ _MMIO(0x9888), 0x0f8305c0 },
-	{ _MMIO(0x9888), 0x09830000 },
-	{ _MMIO(0x9888), 0x07830000 },
-	{ _MMIO(0x9888), 0x1d950080 },
-	{ _MMIO(0x9888), 0x13928000 },
-	{ _MMIO(0x9888), 0x0f988000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x1190fc00 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x4b900040 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900800 },
-	{ _MMIO(0x9888), 0x43900842 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_hdc_and_sf_mux_config(struct drm_i915_private *dev_priv,
-			  const struct i915_oa_reg **regs,
-			  int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_hdc_and_sf;
-	lens[n] = ARRAY_SIZE(mux_config_hdc_and_sf);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_l3_1[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2770), 0x00100070 },
-	{ _MMIO(0x2774), 0x0000fff1 },
-	{ _MMIO(0x2778), 0x00014002 },
-	{ _MMIO(0x277c), 0x0000c3ff },
-	{ _MMIO(0x2780), 0x00010002 },
-	{ _MMIO(0x2784), 0x0000c7ff },
-	{ _MMIO(0x2788), 0x00004002 },
-	{ _MMIO(0x278c), 0x0000d3ff },
-	{ _MMIO(0x2790), 0x00100700 },
-	{ _MMIO(0x2794), 0x0000ff1f },
-	{ _MMIO(0x2798), 0x00001402 },
-	{ _MMIO(0x279c), 0x0000fc3f },
-	{ _MMIO(0x27a0), 0x00001002 },
-	{ _MMIO(0x27a4), 0x0000fc7f },
-	{ _MMIO(0x27a8), 0x00000402 },
-	{ _MMIO(0x27ac), 0x0000fd3f },
-};
-
-static const struct i915_oa_reg flex_eu_config_l3_1[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_l3_1[] = {
-	{ _MMIO(0x9888), 0x126c7b40 },
-	{ _MMIO(0x9888), 0x166c0020 },
-	{ _MMIO(0x9888), 0x0a603444 },
-	{ _MMIO(0x9888), 0x0a613400 },
-	{ _MMIO(0x9888), 0x1a4ea800 },
-	{ _MMIO(0x9888), 0x1c4e0002 },
-	{ _MMIO(0x9888), 0x024e8000 },
-	{ _MMIO(0x9888), 0x044e8000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084e8000 },
-	{ _MMIO(0x9888), 0x0a4e8000 },
-	{ _MMIO(0x9888), 0x064f4000 },
-	{ _MMIO(0x9888), 0x0c6c5327 },
-	{ _MMIO(0x9888), 0x0e6c5425 },
-	{ _MMIO(0x9888), 0x006c2a00 },
-	{ _MMIO(0x9888), 0x026c285b },
-	{ _MMIO(0x9888), 0x046c005c },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1e6c0000 },
-	{ _MMIO(0x9888), 0x1a6c0800 },
-	{ _MMIO(0x9888), 0x0c1bc000 },
-	{ _MMIO(0x9888), 0x0e1bc000 },
-	{ _MMIO(0x9888), 0x001b8000 },
-	{ _MMIO(0x9888), 0x021bc000 },
-	{ _MMIO(0x9888), 0x041bc000 },
-	{ _MMIO(0x9888), 0x1c1c003c },
-	{ _MMIO(0x9888), 0x121c8000 },
-	{ _MMIO(0x9888), 0x141c8000 },
-	{ _MMIO(0x9888), 0x161c8000 },
-	{ _MMIO(0x9888), 0x181c8000 },
-	{ _MMIO(0x9888), 0x1a1c0800 },
-	{ _MMIO(0x9888), 0x065b4000 },
-	{ _MMIO(0x9888), 0x1a5c1000 },
-	{ _MMIO(0x9888), 0x10600000 },
-	{ _MMIO(0x9888), 0x04600000 },
-	{ _MMIO(0x9888), 0x0c610044 },
-	{ _MMIO(0x9888), 0x10610000 },
-	{ _MMIO(0x9888), 0x06610000 },
-	{ _MMIO(0x9888), 0x0c4c02a8 },
-	{ _MMIO(0x9888), 0x084ca000 },
-	{ _MMIO(0x9888), 0x0a4c002a },
-	{ _MMIO(0x9888), 0x0c0da000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x100f0154 },
-	{ _MMIO(0x9888), 0x0c0f5000 },
-	{ _MMIO(0x9888), 0x0e0f0055 },
-	{ _MMIO(0x9888), 0x182c00aa },
-	{ _MMIO(0x9888), 0x022c8000 },
-	{ _MMIO(0x9888), 0x042c8000 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2cc000 },
-	{ _MMIO(0x9888), 0x1190ffc0 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900420 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900021 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900400 },
-	{ _MMIO(0x9888), 0x43900421 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900040 },
-};
-
-static int
-get_l3_1_mux_config(struct drm_i915_private *dev_priv,
-		    const struct i915_oa_reg **regs,
-		    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_l3_1;
-	lens[n] = ARRAY_SIZE(mux_config_l3_1);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_l3_2[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00100070 },
-	{ _MMIO(0x2774), 0x0000fff1 },
-	{ _MMIO(0x2778), 0x00028002 },
-	{ _MMIO(0x277c), 0x000087ff },
-	{ _MMIO(0x2780), 0x00020002 },
-	{ _MMIO(0x2784), 0x00008fff },
-	{ _MMIO(0x2788), 0x00008002 },
-	{ _MMIO(0x278c), 0x0000a7ff },
-};
-
-static const struct i915_oa_reg flex_eu_config_l3_2[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_l3_2[] = {
-	{ _MMIO(0x9888), 0x126c02e0 },
-	{ _MMIO(0x9888), 0x146c0001 },
-	{ _MMIO(0x9888), 0x0a623400 },
-	{ _MMIO(0x9888), 0x044e8000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084e8000 },
-	{ _MMIO(0x9888), 0x0a4e8000 },
-	{ _MMIO(0x9888), 0x064f4000 },
-	{ _MMIO(0x9888), 0x026c3324 },
-	{ _MMIO(0x9888), 0x046c3422 },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x1a6c0000 },
-	{ _MMIO(0x9888), 0x021bc000 },
-	{ _MMIO(0x9888), 0x041bc000 },
-	{ _MMIO(0x9888), 0x141c8000 },
-	{ _MMIO(0x9888), 0x161c8000 },
-	{ _MMIO(0x9888), 0x181c8000 },
-	{ _MMIO(0x9888), 0x1a1c0800 },
-	{ _MMIO(0x9888), 0x065b4000 },
-	{ _MMIO(0x9888), 0x1a5c1000 },
-	{ _MMIO(0x9888), 0x06614000 },
-	{ _MMIO(0x9888), 0x0c620044 },
-	{ _MMIO(0x9888), 0x10620000 },
-	{ _MMIO(0x9888), 0x06620000 },
-	{ _MMIO(0x9888), 0x084c8000 },
-	{ _MMIO(0x9888), 0x0a4c002a },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x0c0f4000 },
-	{ _MMIO(0x9888), 0x0e0f0055 },
-	{ _MMIO(0x9888), 0x042c8000 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2cc000 },
-	{ _MMIO(0x9888), 0x1190f800 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x43900000 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_l3_2_mux_config(struct drm_i915_private *dev_priv,
-		    const struct i915_oa_reg **regs,
-		    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_l3_2;
-	lens[n] = ARRAY_SIZE(mux_config_l3_2);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_l3_3[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00100070 },
-	{ _MMIO(0x2774), 0x0000fff1 },
-	{ _MMIO(0x2778), 0x00028002 },
-	{ _MMIO(0x277c), 0x000087ff },
-	{ _MMIO(0x2780), 0x00020002 },
-	{ _MMIO(0x2784), 0x00008fff },
-	{ _MMIO(0x2788), 0x00008002 },
-	{ _MMIO(0x278c), 0x0000a7ff },
-};
-
-static const struct i915_oa_reg flex_eu_config_l3_3[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_l3_3[] = {
-	{ _MMIO(0x9888), 0x126c4e80 },
-	{ _MMIO(0x9888), 0x146c0000 },
-	{ _MMIO(0x9888), 0x0a633400 },
-	{ _MMIO(0x9888), 0x044e8000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084e8000 },
-	{ _MMIO(0x9888), 0x0a4e8000 },
-	{ _MMIO(0x9888), 0x0c4e8000 },
-	{ _MMIO(0x9888), 0x026c3321 },
-	{ _MMIO(0x9888), 0x046c342f },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x1a6c2000 },
-	{ _MMIO(0x9888), 0x021bc000 },
-	{ _MMIO(0x9888), 0x041bc000 },
-	{ _MMIO(0x9888), 0x061b4000 },
-	{ _MMIO(0x9888), 0x141c8000 },
-	{ _MMIO(0x9888), 0x161c8000 },
-	{ _MMIO(0x9888), 0x181c8000 },
-	{ _MMIO(0x9888), 0x1a1c1800 },
-	{ _MMIO(0x9888), 0x06604000 },
-	{ _MMIO(0x9888), 0x0c630044 },
-	{ _MMIO(0x9888), 0x10630000 },
-	{ _MMIO(0x9888), 0x06630000 },
-	{ _MMIO(0x9888), 0x084c8000 },
-	{ _MMIO(0x9888), 0x0a4c00aa },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x0c0f4000 },
-	{ _MMIO(0x9888), 0x0e0f0055 },
-	{ _MMIO(0x9888), 0x042c8000 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x1190f800 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x43900842 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900002 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_l3_3_mux_config(struct drm_i915_private *dev_priv,
-		    const struct i915_oa_reg **regs,
-		    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_l3_3;
-	lens[n] = ARRAY_SIZE(mux_config_l3_3);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x30800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x0000efff },
-	{ _MMIO(0x2778), 0x00006000 },
-	{ _MMIO(0x277c), 0x0000f3ff },
-};
-
-static const struct i915_oa_reg flex_eu_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0x9888), 0x102f3800 },
-	{ _MMIO(0x9888), 0x144d0500 },
-	{ _MMIO(0x9888), 0x120d03c0 },
-	{ _MMIO(0x9888), 0x140d03cf },
-	{ _MMIO(0x9888), 0x0c0f0004 },
-	{ _MMIO(0x9888), 0x0c4e4000 },
-	{ _MMIO(0x9888), 0x042f0480 },
-	{ _MMIO(0x9888), 0x082f0000 },
-	{ _MMIO(0x9888), 0x022f0000 },
-	{ _MMIO(0x9888), 0x0a4c0090 },
-	{ _MMIO(0x9888), 0x064d0027 },
-	{ _MMIO(0x9888), 0x004d0000 },
-	{ _MMIO(0x9888), 0x000d0d40 },
-	{ _MMIO(0x9888), 0x020d803f },
-	{ _MMIO(0x9888), 0x040d8023 },
-	{ _MMIO(0x9888), 0x100d0000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x020f0010 },
-	{ _MMIO(0x9888), 0x000f0000 },
-	{ _MMIO(0x9888), 0x0e0f0050 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x1190fc00 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41901400 },
-	{ _MMIO(0x9888), 0x43901485 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900001 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_rasterizer_and_pixel_backend_mux_config(struct drm_i915_private *dev_priv,
-					    const struct i915_oa_reg **regs,
-					    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_rasterizer_and_pixel_backend;
-	lens[n] = ARRAY_SIZE(mux_config_rasterizer_and_pixel_backend);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_sampler[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x70800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x0000c000 },
-	{ _MMIO(0x2774), 0x0000e7ff },
-	{ _MMIO(0x2778), 0x00003000 },
-	{ _MMIO(0x277c), 0x0000f9ff },
-	{ _MMIO(0x2780), 0x00000c00 },
-	{ _MMIO(0x2784), 0x0000fe7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_sampler[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_sampler[] = {
-	{ _MMIO(0x9888), 0x14152c00 },
-	{ _MMIO(0x9888), 0x16150005 },
-	{ _MMIO(0x9888), 0x121600a0 },
-	{ _MMIO(0x9888), 0x14352c00 },
-	{ _MMIO(0x9888), 0x16350005 },
-	{ _MMIO(0x9888), 0x123600a0 },
-	{ _MMIO(0x9888), 0x14552c00 },
-	{ _MMIO(0x9888), 0x16550005 },
-	{ _MMIO(0x9888), 0x125600a0 },
-	{ _MMIO(0x9888), 0x062f6000 },
-	{ _MMIO(0x9888), 0x022f2000 },
-	{ _MMIO(0x9888), 0x0c4c0050 },
-	{ _MMIO(0x9888), 0x0a4c0010 },
-	{ _MMIO(0x9888), 0x0c0d8000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x100f0350 },
-	{ _MMIO(0x9888), 0x0c0fb000 },
-	{ _MMIO(0x9888), 0x0e0f00da },
-	{ _MMIO(0x9888), 0x182c0028 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x022dc000 },
-	{ _MMIO(0x9888), 0x042d4000 },
-	{ _MMIO(0x9888), 0x0c138000 },
-	{ _MMIO(0x9888), 0x0e132000 },
-	{ _MMIO(0x9888), 0x0413c000 },
-	{ _MMIO(0x9888), 0x1c140018 },
-	{ _MMIO(0x9888), 0x0c157000 },
-	{ _MMIO(0x9888), 0x0e150078 },
-	{ _MMIO(0x9888), 0x10150000 },
-	{ _MMIO(0x9888), 0x04162180 },
-	{ _MMIO(0x9888), 0x02160000 },
-	{ _MMIO(0x9888), 0x04174000 },
-	{ _MMIO(0x9888), 0x0233a000 },
-	{ _MMIO(0x9888), 0x04333000 },
-	{ _MMIO(0x9888), 0x14348000 },
-	{ _MMIO(0x9888), 0x16348000 },
-	{ _MMIO(0x9888), 0x02357870 },
-	{ _MMIO(0x9888), 0x10350000 },
-	{ _MMIO(0x9888), 0x04360043 },
-	{ _MMIO(0x9888), 0x02360000 },
-	{ _MMIO(0x9888), 0x04371000 },
-	{ _MMIO(0x9888), 0x0e538000 },
-	{ _MMIO(0x9888), 0x00538000 },
-	{ _MMIO(0x9888), 0x06533000 },
-	{ _MMIO(0x9888), 0x1c540020 },
-	{ _MMIO(0x9888), 0x12548000 },
-	{ _MMIO(0x9888), 0x0e557000 },
-	{ _MMIO(0x9888), 0x00557800 },
-	{ _MMIO(0x9888), 0x10550000 },
-	{ _MMIO(0x9888), 0x06560043 },
-	{ _MMIO(0x9888), 0x02560000 },
-	{ _MMIO(0x9888), 0x06571000 },
-	{ _MMIO(0x9888), 0x1190ff80 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900060 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c00 },
-	{ _MMIO(0x9888), 0x43900842 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900060 },
-};
-
-static int
-get_sampler_mux_config(struct drm_i915_private *dev_priv,
-		       const struct i915_oa_reg **regs,
-		       int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_sampler;
-	lens[n] = ARRAY_SIZE(mux_config_sampler);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_tdl_1[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x00007fff },
-	{ _MMIO(0x2778), 0x00000000 },
-	{ _MMIO(0x277c), 0x00009fff },
-	{ _MMIO(0x2780), 0x00000002 },
-	{ _MMIO(0x2784), 0x0000efff },
-	{ _MMIO(0x2788), 0x00000000 },
-	{ _MMIO(0x278c), 0x0000f3ff },
-	{ _MMIO(0x2790), 0x00000002 },
-	{ _MMIO(0x2794), 0x0000fdff },
-	{ _MMIO(0x2798), 0x00000000 },
-	{ _MMIO(0x279c), 0x0000fe7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_tdl_1[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_tdl_1[] = {
-	{ _MMIO(0x9888), 0x12120000 },
-	{ _MMIO(0x9888), 0x12320000 },
-	{ _MMIO(0x9888), 0x12520000 },
-	{ _MMIO(0x9888), 0x002f8000 },
-	{ _MMIO(0x9888), 0x022f3000 },
-	{ _MMIO(0x9888), 0x0a4c0015 },
-	{ _MMIO(0x9888), 0x0c0d8000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x100f03a0 },
-	{ _MMIO(0x9888), 0x0c0ff000 },
-	{ _MMIO(0x9888), 0x0e0f0095 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2d8000 },
-	{ _MMIO(0x9888), 0x0e2d4000 },
-	{ _MMIO(0x9888), 0x062d4000 },
-	{ _MMIO(0x9888), 0x02108000 },
-	{ _MMIO(0x9888), 0x0410c000 },
-	{ _MMIO(0x9888), 0x02118000 },
-	{ _MMIO(0x9888), 0x0411c000 },
-	{ _MMIO(0x9888), 0x02121880 },
-	{ _MMIO(0x9888), 0x041219b5 },
-	{ _MMIO(0x9888), 0x00120000 },
-	{ _MMIO(0x9888), 0x02134000 },
-	{ _MMIO(0x9888), 0x04135000 },
-	{ _MMIO(0x9888), 0x0c308000 },
-	{ _MMIO(0x9888), 0x0e304000 },
-	{ _MMIO(0x9888), 0x06304000 },
-	{ _MMIO(0x9888), 0x0c318000 },
-	{ _MMIO(0x9888), 0x0e314000 },
-	{ _MMIO(0x9888), 0x06314000 },
-	{ _MMIO(0x9888), 0x0c321a80 },
-	{ _MMIO(0x9888), 0x0e320033 },
-	{ _MMIO(0x9888), 0x06320031 },
-	{ _MMIO(0x9888), 0x00320000 },
-	{ _MMIO(0x9888), 0x0c334000 },
-	{ _MMIO(0x9888), 0x0e331000 },
-	{ _MMIO(0x9888), 0x06331000 },
-	{ _MMIO(0x9888), 0x0e508000 },
-	{ _MMIO(0x9888), 0x00508000 },
-	{ _MMIO(0x9888), 0x02504000 },
-	{ _MMIO(0x9888), 0x0e518000 },
-	{ _MMIO(0x9888), 0x00518000 },
-	{ _MMIO(0x9888), 0x02514000 },
-	{ _MMIO(0x9888), 0x0e521880 },
-	{ _MMIO(0x9888), 0x00521a80 },
-	{ _MMIO(0x9888), 0x02520033 },
-	{ _MMIO(0x9888), 0x0e534000 },
-	{ _MMIO(0x9888), 0x00534000 },
-	{ _MMIO(0x9888), 0x02531000 },
-	{ _MMIO(0x9888), 0x1190ff80 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900800 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900062 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c00 },
-	{ _MMIO(0x9888), 0x43900003 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900040 },
-};
-
-static int
-get_tdl_1_mux_config(struct drm_i915_private *dev_priv,
-		     const struct i915_oa_reg **regs,
-		     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_tdl_1;
-	lens[n] = ARRAY_SIZE(mux_config_tdl_1);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_tdl_2[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_tdl_2[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_tdl_2[] = {
-	{ _MMIO(0x9888), 0x12124d60 },
-	{ _MMIO(0x9888), 0x12322e60 },
-	{ _MMIO(0x9888), 0x12524d60 },
-	{ _MMIO(0x9888), 0x022f3000 },
-	{ _MMIO(0x9888), 0x0a4c0014 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x0c0fe000 },
-	{ _MMIO(0x9888), 0x0e0f0097 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x002d8000 },
-	{ _MMIO(0x9888), 0x062d4000 },
-	{ _MMIO(0x9888), 0x0410c000 },
-	{ _MMIO(0x9888), 0x0411c000 },
-	{ _MMIO(0x9888), 0x04121fb7 },
-	{ _MMIO(0x9888), 0x00120000 },
-	{ _MMIO(0x9888), 0x04135000 },
-	{ _MMIO(0x9888), 0x00308000 },
-	{ _MMIO(0x9888), 0x06304000 },
-	{ _MMIO(0x9888), 0x00318000 },
-	{ _MMIO(0x9888), 0x06314000 },
-	{ _MMIO(0x9888), 0x00321b80 },
-	{ _MMIO(0x9888), 0x0632003f },
-	{ _MMIO(0x9888), 0x00334000 },
-	{ _MMIO(0x9888), 0x06331000 },
-	{ _MMIO(0x9888), 0x0250c000 },
-	{ _MMIO(0x9888), 0x0251c000 },
-	{ _MMIO(0x9888), 0x02521fb7 },
-	{ _MMIO(0x9888), 0x00520000 },
-	{ _MMIO(0x9888), 0x02535000 },
-	{ _MMIO(0x9888), 0x1190fc00 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900800 },
-	{ _MMIO(0x9888), 0x43900063 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900040 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_tdl_2_mux_config(struct drm_i915_private *dev_priv,
-		     const struct i915_oa_reg **regs,
-		     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_tdl_2;
-	lens[n] = ARRAY_SIZE(mux_config_tdl_2);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_extra[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_extra[] = {
-	{ _MMIO(0xe458), 0x00001000 },
-	{ _MMIO(0xe558), 0x00003002 },
-	{ _MMIO(0xe658), 0x00005004 },
-	{ _MMIO(0xe758), 0x00011010 },
-	{ _MMIO(0xe45c), 0x00050012 },
-	{ _MMIO(0xe55c), 0x00052051 },
-	{ _MMIO(0xe65c), 0x00000008 },
-};
-
-static const struct i915_oa_reg mux_config_compute_extra[] = {
-	{ _MMIO(0x9888), 0x121203e0 },
-	{ _MMIO(0x9888), 0x123203e0 },
-	{ _MMIO(0x9888), 0x125203e0 },
-	{ _MMIO(0x9888), 0x022f4000 },
-	{ _MMIO(0x9888), 0x0a4c0040 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x0e0f006c },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x042d8000 },
-	{ _MMIO(0x9888), 0x06104000 },
-	{ _MMIO(0x9888), 0x06114000 },
-	{ _MMIO(0x9888), 0x06120033 },
-	{ _MMIO(0x9888), 0x00120000 },
-	{ _MMIO(0x9888), 0x06131000 },
-	{ _MMIO(0x9888), 0x04308000 },
-	{ _MMIO(0x9888), 0x04318000 },
-	{ _MMIO(0x9888), 0x04321980 },
-	{ _MMIO(0x9888), 0x00320000 },
-	{ _MMIO(0x9888), 0x04334000 },
-	{ _MMIO(0x9888), 0x04504000 },
-	{ _MMIO(0x9888), 0x04514000 },
-	{ _MMIO(0x9888), 0x04520033 },
-	{ _MMIO(0x9888), 0x00520000 },
-	{ _MMIO(0x9888), 0x04531000 },
-	{ _MMIO(0x9888), 0x1190e000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x43900c00 },
-	{ _MMIO(0x9888), 0x45900002 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_compute_extra_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_extra;
-	lens[n] = ARRAY_SIZE(mux_config_compute_extra);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_vme_pipe[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2770), 0x00100030 },
-	{ _MMIO(0x2774), 0x0000fff9 },
-	{ _MMIO(0x2778), 0x00000002 },
-	{ _MMIO(0x277c), 0x0000fffc },
-	{ _MMIO(0x2780), 0x00000002 },
-	{ _MMIO(0x2784), 0x0000fff3 },
-	{ _MMIO(0x2788), 0x00100180 },
-	{ _MMIO(0x278c), 0x0000ffcf },
-	{ _MMIO(0x2790), 0x00000002 },
-	{ _MMIO(0x2794), 0x0000ffcf },
-	{ _MMIO(0x2798), 0x00000002 },
-	{ _MMIO(0x279c), 0x0000ff3f },
-};
-
-static const struct i915_oa_reg flex_eu_config_vme_pipe[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00008003 },
-};
-
-static const struct i915_oa_reg mux_config_vme_pipe[] = {
-	{ _MMIO(0x9888), 0x141a5800 },
-	{ _MMIO(0x9888), 0x161a00c0 },
-	{ _MMIO(0x9888), 0x12180240 },
-	{ _MMIO(0x9888), 0x14180002 },
-	{ _MMIO(0x9888), 0x143a5800 },
-	{ _MMIO(0x9888), 0x163a00c0 },
-	{ _MMIO(0x9888), 0x12380240 },
-	{ _MMIO(0x9888), 0x14380002 },
-	{ _MMIO(0x9888), 0x002f1000 },
-	{ _MMIO(0x9888), 0x022f8000 },
-	{ _MMIO(0x9888), 0x042f3000 },
-	{ _MMIO(0x9888), 0x004c4000 },
-	{ _MMIO(0x9888), 0x0a4c1500 },
-	{ _MMIO(0x9888), 0x000d2000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0a0da000 },
-	{ _MMIO(0x9888), 0x0c0da000 },
-	{ _MMIO(0x9888), 0x0c0f0400 },
-	{ _MMIO(0x9888), 0x0e0f9500 },
-	{ _MMIO(0x9888), 0x100f002a },
-	{ _MMIO(0x9888), 0x002c8000 },
-	{ _MMIO(0x9888), 0x0e2c8000 },
-	{ _MMIO(0x9888), 0x162c0a00 },
-	{ _MMIO(0x9888), 0x0a2dc000 },
-	{ _MMIO(0x9888), 0x0c2dc000 },
-	{ _MMIO(0x9888), 0x04193000 },
-	{ _MMIO(0x9888), 0x081a28c1 },
-	{ _MMIO(0x9888), 0x001a0000 },
-	{ _MMIO(0x9888), 0x00133000 },
-	{ _MMIO(0x9888), 0x0613c000 },
-	{ _MMIO(0x9888), 0x0813f000 },
-	{ _MMIO(0x9888), 0x00172000 },
-	{ _MMIO(0x9888), 0x06178000 },
-	{ _MMIO(0x9888), 0x0817a000 },
-	{ _MMIO(0x9888), 0x00180037 },
-	{ _MMIO(0x9888), 0x06180940 },
-	{ _MMIO(0x9888), 0x08180000 },
-	{ _MMIO(0x9888), 0x02180000 },
-	{ _MMIO(0x9888), 0x04183000 },
-	{ _MMIO(0x9888), 0x06393000 },
-	{ _MMIO(0x9888), 0x0c3a28c1 },
-	{ _MMIO(0x9888), 0x003a0000 },
-	{ _MMIO(0x9888), 0x0a33f000 },
-	{ _MMIO(0x9888), 0x0c33f000 },
-	{ _MMIO(0x9888), 0x0a37a000 },
-	{ _MMIO(0x9888), 0x0c37a000 },
-	{ _MMIO(0x9888), 0x0a380977 },
-	{ _MMIO(0x9888), 0x08380000 },
-	{ _MMIO(0x9888), 0x04380000 },
-	{ _MMIO(0x9888), 0x06383000 },
-	{ _MMIO(0x9888), 0x119000ff },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900040 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900800 },
-	{ _MMIO(0x9888), 0x47901000 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900844 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_vme_pipe_mux_config(struct drm_i915_private *dev_priv,
-			const struct i915_oa_reg **regs,
-			int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_vme_pipe;
-	lens[n] = ARRAY_SIZE(mux_config_vme_pipe);
-	n++;
-
-	return n;
-}
-
 static const struct i915_oa_reg b_counter_config_test_oa[] = {
 	{ _MMIO(0x2740), 0x00000000 },
 	{ _MMIO(0x2744), 0x00800000 },
@@ -1882,6 +60,7 @@ static const struct i915_oa_reg flex_eu_config_test_oa[] = {
 };
 
 static const struct i915_oa_reg mux_config_test_oa[] = {
+	{ _MMIO(0x9840), 0x00000080 },
 	{ _MMIO(0x9888), 0x11810000 },
 	{ _MMIO(0x9888), 0x07810013 },
 	{ _MMIO(0x9888), 0x1f810000 },
@@ -1896,1096 +75,35 @@ static const struct i915_oa_reg mux_config_test_oa[] = {
 	{ _MMIO(0x9888), 0x33900000 },
 };
 
-static int
-get_test_oa_mux_config(struct drm_i915_private *dev_priv,
-		       const struct i915_oa_reg **regs,
-		       int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_test_oa;
-	lens[n] = ARRAY_SIZE(mux_config_test_oa);
-	n++;
-
-	return n;
-}
-
-int i915_oa_select_metric_set_kblgt2(struct drm_i915_private *dev_priv)
-{
-	dev_priv->perf.oa.n_mux_configs = 0;
-	dev_priv->perf.oa.b_counter_regs = NULL;
-	dev_priv->perf.oa.b_counter_regs_len = 0;
-	dev_priv->perf.oa.flex_regs = NULL;
-	dev_priv->perf.oa.flex_regs_len = 0;
-
-	switch (dev_priv->perf.oa.metrics_set) {
-	case METRIC_SET_ID_RENDER_BASIC:
-		dev_priv->perf.oa.n_mux_configs =
-			get_render_basic_mux_config(dev_priv,
-						    dev_priv->perf.oa.mux_regs,
-						    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_BASIC\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_render_basic;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_render_basic);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_render_basic;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_render_basic);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_BASIC:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_basic_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_BASIC\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_basic;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_basic);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_basic;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_basic);
-
-		return 0;
-	case METRIC_SET_ID_RENDER_PIPE_PROFILE:
-		dev_priv->perf.oa.n_mux_configs =
-			get_render_pipe_profile_mux_config(dev_priv,
-							   dev_priv->perf.oa.mux_regs,
-							   dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_PIPE_PROFILE\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_render_pipe_profile;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_render_pipe_profile);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_render_pipe_profile;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_render_pipe_profile);
-
-		return 0;
-	case METRIC_SET_ID_MEMORY_READS:
-		dev_priv->perf.oa.n_mux_configs =
-			get_memory_reads_mux_config(dev_priv,
-						    dev_priv->perf.oa.mux_regs,
-						    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_READS\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_memory_reads;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_memory_reads);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_memory_reads;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_memory_reads);
-
-		return 0;
-	case METRIC_SET_ID_MEMORY_WRITES:
-		dev_priv->perf.oa.n_mux_configs =
-			get_memory_writes_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_WRITES\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_memory_writes;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_memory_writes);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_memory_writes;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_memory_writes);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_EXTENDED:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_extended_mux_config(dev_priv,
-							dev_priv->perf.oa.mux_regs,
-							dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTENDED\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_extended;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_extended);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_extended;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_extended);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_L3_CACHE:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_l3_cache_mux_config(dev_priv,
-							dev_priv->perf.oa.mux_regs,
-							dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_L3_CACHE\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_l3_cache;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_l3_cache);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_l3_cache;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_l3_cache);
-
-		return 0;
-	case METRIC_SET_ID_HDC_AND_SF:
-		dev_priv->perf.oa.n_mux_configs =
-			get_hdc_and_sf_mux_config(dev_priv,
-						  dev_priv->perf.oa.mux_regs,
-						  dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"HDC_AND_SF\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_hdc_and_sf;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_hdc_and_sf);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_hdc_and_sf;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_hdc_and_sf);
-
-		return 0;
-	case METRIC_SET_ID_L3_1:
-		dev_priv->perf.oa.n_mux_configs =
-			get_l3_1_mux_config(dev_priv,
-					    dev_priv->perf.oa.mux_regs,
-					    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_1\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_l3_1;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_l3_1);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_l3_1;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_l3_1);
-
-		return 0;
-	case METRIC_SET_ID_L3_2:
-		dev_priv->perf.oa.n_mux_configs =
-			get_l3_2_mux_config(dev_priv,
-					    dev_priv->perf.oa.mux_regs,
-					    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_2\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_l3_2;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_l3_2);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_l3_2;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_l3_2);
-
-		return 0;
-	case METRIC_SET_ID_L3_3:
-		dev_priv->perf.oa.n_mux_configs =
-			get_l3_3_mux_config(dev_priv,
-					    dev_priv->perf.oa.mux_regs,
-					    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_3\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_l3_3;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_l3_3);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_l3_3;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_l3_3);
-
-		return 0;
-	case METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND:
-		dev_priv->perf.oa.n_mux_configs =
-			get_rasterizer_and_pixel_backend_mux_config(dev_priv,
-								    dev_priv->perf.oa.mux_regs,
-								    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RASTERIZER_AND_PIXEL_BACKEND\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_rasterizer_and_pixel_backend;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_rasterizer_and_pixel_backend);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_rasterizer_and_pixel_backend;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_rasterizer_and_pixel_backend);
-
-		return 0;
-	case METRIC_SET_ID_SAMPLER:
-		dev_priv->perf.oa.n_mux_configs =
-			get_sampler_mux_config(dev_priv,
-					       dev_priv->perf.oa.mux_regs,
-					       dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"SAMPLER\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_sampler;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_sampler);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_sampler;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_sampler);
-
-		return 0;
-	case METRIC_SET_ID_TDL_1:
-		dev_priv->perf.oa.n_mux_configs =
-			get_tdl_1_mux_config(dev_priv,
-					     dev_priv->perf.oa.mux_regs,
-					     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_1\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_tdl_1;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_tdl_1);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_tdl_1;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_tdl_1);
-
-		return 0;
-	case METRIC_SET_ID_TDL_2:
-		dev_priv->perf.oa.n_mux_configs =
-			get_tdl_2_mux_config(dev_priv,
-					     dev_priv->perf.oa.mux_regs,
-					     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_2\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_tdl_2;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_tdl_2);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_tdl_2;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_tdl_2);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_EXTRA:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_extra_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTRA\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_extra;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_extra);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_extra;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_extra);
-
-		return 0;
-	case METRIC_SET_ID_VME_PIPE:
-		dev_priv->perf.oa.n_mux_configs =
-			get_vme_pipe_mux_config(dev_priv,
-						dev_priv->perf.oa.mux_regs,
-						dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"VME_PIPE\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_vme_pipe;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_vme_pipe);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_vme_pipe;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_vme_pipe);
-
-		return 0;
-	case METRIC_SET_ID_TEST_OA:
-		dev_priv->perf.oa.n_mux_configs =
-			get_test_oa_mux_config(dev_priv,
-					       dev_priv->perf.oa.mux_regs,
-					       dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TEST_OA\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_test_oa;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_test_oa);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_test_oa;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_test_oa);
-
-		return 0;
-	default:
-		return -ENODEV;
-	}
-}
-
-static ssize_t
-show_render_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_BASIC);
-}
-
-static struct device_attribute dev_attr_render_basic_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_render_basic_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_render_basic[] = {
-	&dev_attr_render_basic_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_render_basic = {
-	.name = "f8d677e9-ff6f-4df1-9310-0334c6efacce",
-	.attrs =  attrs_render_basic,
-};
-
-static ssize_t
-show_compute_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_BASIC);
-}
-
-static struct device_attribute dev_attr_compute_basic_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_basic_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_basic[] = {
-	&dev_attr_compute_basic_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_basic = {
-	.name = "e17fc42a-e614-41b6-90c4-1074841a6c77",
-	.attrs =  attrs_compute_basic,
-};
-
-static ssize_t
-show_render_pipe_profile_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_PIPE_PROFILE);
-}
-
-static struct device_attribute dev_attr_render_pipe_profile_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_render_pipe_profile_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_render_pipe_profile[] = {
-	&dev_attr_render_pipe_profile_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_render_pipe_profile = {
-	.name = "d7a17a3a-ca71-40d2-a919-ace80d50633f",
-	.attrs =  attrs_render_pipe_profile,
-};
-
-static ssize_t
-show_memory_reads_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_READS);
-}
-
-static struct device_attribute dev_attr_memory_reads_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_memory_reads_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_memory_reads[] = {
-	&dev_attr_memory_reads_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_memory_reads = {
-	.name = "57b59202-172b-477a-87de-33f85572c589",
-	.attrs =  attrs_memory_reads,
-};
-
-static ssize_t
-show_memory_writes_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_WRITES);
-}
-
-static struct device_attribute dev_attr_memory_writes_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_memory_writes_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_memory_writes[] = {
-	&dev_attr_memory_writes_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_memory_writes = {
-	.name = "3addf8ef-8e9b-40f5-a448-3dbb5d5128b0",
-	.attrs =  attrs_memory_writes,
-};
-
-static ssize_t
-show_compute_extended_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTENDED);
-}
-
-static struct device_attribute dev_attr_compute_extended_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_extended_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_extended[] = {
-	&dev_attr_compute_extended_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_extended = {
-	.name = "4af0400a-81c3-47db-a6b6-deddbd75680e",
-	.attrs =  attrs_compute_extended,
-};
-
-static ssize_t
-show_compute_l3_cache_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_L3_CACHE);
-}
-
-static struct device_attribute dev_attr_compute_l3_cache_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_l3_cache_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_l3_cache[] = {
-	&dev_attr_compute_l3_cache_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_l3_cache = {
-	.name = "0e22f995-79ca-4f67-83ab-e9d9772488d8",
-	.attrs =  attrs_compute_l3_cache,
-};
-
-static ssize_t
-show_hdc_and_sf_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_HDC_AND_SF);
-}
-
-static struct device_attribute dev_attr_hdc_and_sf_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_hdc_and_sf_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_hdc_and_sf[] = {
-	&dev_attr_hdc_and_sf_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_hdc_and_sf = {
-	.name = "bc2a00f7-cb8a-4ff2-8ad0-e241dad16937",
-	.attrs =  attrs_hdc_and_sf,
-};
-
-static ssize_t
-show_l3_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_1);
-}
-
-static struct device_attribute dev_attr_l3_1_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_l3_1_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_l3_1[] = {
-	&dev_attr_l3_1_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_l3_1 = {
-	.name = "d2bbe790-f058-42d9-81c6-cdedcf655bc2",
-	.attrs =  attrs_l3_1,
-};
-
-static ssize_t
-show_l3_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_2);
-}
-
-static struct device_attribute dev_attr_l3_2_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_l3_2_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_l3_2[] = {
-	&dev_attr_l3_2_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_l3_2 = {
-	.name = "2f8e32e4-5956-46e2-af31-c8ea95887332",
-	.attrs =  attrs_l3_2,
-};
-
-static ssize_t
-show_l3_3_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_3);
-}
-
-static struct device_attribute dev_attr_l3_3_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_l3_3_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_l3_3[] = {
-	&dev_attr_l3_3_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_l3_3 = {
-	.name = "ca046aad-b5fb-4101-adce-6473ee6e5b14",
-	.attrs =  attrs_l3_3,
-};
-
-static ssize_t
-show_rasterizer_and_pixel_backend_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND);
-}
-
-static struct device_attribute dev_attr_rasterizer_and_pixel_backend_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_rasterizer_and_pixel_backend_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_rasterizer_and_pixel_backend[] = {
-	&dev_attr_rasterizer_and_pixel_backend_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_rasterizer_and_pixel_backend = {
-	.name = "605f388f-24bb-455c-88e3-8d57ae0d7e9f",
-	.attrs =  attrs_rasterizer_and_pixel_backend,
-};
-
-static ssize_t
-show_sampler_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_SAMPLER);
-}
-
-static struct device_attribute dev_attr_sampler_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_sampler_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_sampler[] = {
-	&dev_attr_sampler_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_sampler = {
-	.name = "31dd157c-bf4e-4bab-bf2b-f5c8174af1af",
-	.attrs =  attrs_sampler,
-};
-
-static ssize_t
-show_tdl_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_1);
-}
-
-static struct device_attribute dev_attr_tdl_1_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_tdl_1_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_tdl_1[] = {
-	&dev_attr_tdl_1_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_tdl_1 = {
-	.name = "105db928-5542-466b-9128-e1f3c91426cb",
-	.attrs =  attrs_tdl_1,
-};
-
-static ssize_t
-show_tdl_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_2);
-}
-
-static struct device_attribute dev_attr_tdl_2_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_tdl_2_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_tdl_2[] = {
-	&dev_attr_tdl_2_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_tdl_2 = {
-	.name = "03db94d2-b37f-4c58-a791-0d2067b013bb",
-	.attrs =  attrs_tdl_2,
-};
-
-static ssize_t
-show_compute_extra_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTRA);
-}
-
-static struct device_attribute dev_attr_compute_extra_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_extra_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_extra[] = {
-	&dev_attr_compute_extra_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_extra = {
-	.name = "aa7a3fb9-22fb-43ff-a32d-0ab6c13bbd16",
-	.attrs =  attrs_compute_extra,
-};
-
-static ssize_t
-show_vme_pipe_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_VME_PIPE);
-}
-
-static struct device_attribute dev_attr_vme_pipe_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_vme_pipe_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_vme_pipe[] = {
-	&dev_attr_vme_pipe_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_vme_pipe = {
-	.name = "398a4268-ef6f-4ffc-b55f-3c7b5363ce61",
-	.attrs =  attrs_vme_pipe,
-};
-
 static ssize_t
 show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TEST_OA);
+	return sprintf(buf, "1\n");
 }
 
-static struct device_attribute dev_attr_test_oa_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_test_oa_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_test_oa[] = {
-	&dev_attr_test_oa_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_test_oa = {
-	.name = "baa3c7e4-52b6-4b85-801e-465a94b746dd",
-	.attrs =  attrs_test_oa,
-};
-
-int
-i915_perf_register_sysfs_kblgt2(struct drm_i915_private *dev_priv)
+void
+i915_perf_load_test_config_kblgt2(struct drm_i915_private *dev_priv)
 {
-	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
-	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
-	int ret = 0;
+	strncpy(dev_priv->perf.oa.test_config.uuid,
+		"baa3c7e4-52b6-4b85-801e-465a94b746dd",
+		UUID_STRING_LEN);
+	dev_priv->perf.oa.test_config.id = 1;
 
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-		if (ret)
-			goto error_render_basic;
-	}
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-		if (ret)
-			goto error_compute_basic;
-	}
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-		if (ret)
-			goto error_render_pipe_profile;
-	}
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-		if (ret)
-			goto error_memory_reads;
-	}
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-		if (ret)
-			goto error_memory_writes;
-	}
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-		if (ret)
-			goto error_compute_extended;
-	}
-	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
-		if (ret)
-			goto error_compute_l3_cache;
-	}
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-		if (ret)
-			goto error_hdc_and_sf;
-	}
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-		if (ret)
-			goto error_l3_1;
-	}
-	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_2);
-		if (ret)
-			goto error_l3_2;
-	}
-	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_3);
-		if (ret)
-			goto error_l3_3;
-	}
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-		if (ret)
-			goto error_rasterizer_and_pixel_backend;
-	}
-	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_sampler);
-		if (ret)
-			goto error_sampler;
-	}
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-		if (ret)
-			goto error_tdl_1;
-	}
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-		if (ret)
-			goto error_tdl_2;
-	}
-	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
-		if (ret)
-			goto error_compute_extra;
-	}
-	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
-		if (ret)
-			goto error_vme_pipe;
-	}
-	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_test_oa);
-		if (ret)
-			goto error_test_oa;
-	}
+	dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
+	dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa);
 
-	return 0;
+	dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa;
+	dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa);
 
-error_test_oa:
-	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
-error_vme_pipe:
-	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
-error_compute_extra:
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-error_tdl_2:
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-error_tdl_1:
-	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
-error_sampler:
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-error_rasterizer_and_pixel_backend:
-	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
-error_l3_3:
-	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
-error_l3_2:
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-error_l3_1:
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-error_hdc_and_sf:
-	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
-error_compute_l3_cache:
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-error_compute_extended:
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-error_memory_writes:
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-error_memory_reads:
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-error_render_pipe_profile:
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-error_compute_basic:
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-error_render_basic:
-	return ret;
-}
+	dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa;
+	dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa);
 
-void
-i915_perf_unregister_sysfs_kblgt2(struct drm_i915_private *dev_priv)
-{
-	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
-	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+	dev_priv->perf.oa.test_config.sysfs_metric.name = "baa3c7e4-52b6-4b85-801e-465a94b746dd";
+	dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs;
+
+	dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr;
 
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
-	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
-	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
-	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id";
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444;
+	dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id;
 }
diff --git a/drivers/gpu/drm/i915/i915_oa_kblgt2.h b/drivers/gpu/drm/i915/i915_oa_kblgt2.h
index 7e61bfc4f9f5..25b803546dc1 100644
--- a/drivers/gpu/drm/i915/i915_oa_kblgt2.h
+++ b/drivers/gpu/drm/i915/i915_oa_kblgt2.h
@@ -29,12 +29,6 @@
 #ifndef __I915_OA_KBLGT2_H__
 #define __I915_OA_KBLGT2_H__
 
-extern int i915_oa_n_builtin_metric_sets_kblgt2;
-
-extern int i915_oa_select_metric_set_kblgt2(struct drm_i915_private *dev_priv);
-
-extern int i915_perf_register_sysfs_kblgt2(struct drm_i915_private *dev_priv);
-
-extern void i915_perf_unregister_sysfs_kblgt2(struct drm_i915_private *dev_priv);
+extern void i915_perf_load_test_config_kblgt2(struct drm_i915_private *dev_priv);
 
 #endif
diff --git a/drivers/gpu/drm/i915/i915_oa_kblgt3.c b/drivers/gpu/drm/i915/i915_oa_kblgt3.c
index 6ed092566a32..5576afdd9a7e 100644
--- a/drivers/gpu/drm/i915/i915_oa_kblgt3.c
+++ b/drivers/gpu/drm/i915/i915_oa_kblgt3.c
@@ -31,1877 +31,6 @@
 #include "i915_drv.h"
 #include "i915_oa_kblgt3.h"
 
-enum metric_set_id {
-	METRIC_SET_ID_RENDER_BASIC = 1,
-	METRIC_SET_ID_COMPUTE_BASIC,
-	METRIC_SET_ID_RENDER_PIPE_PROFILE,
-	METRIC_SET_ID_MEMORY_READS,
-	METRIC_SET_ID_MEMORY_WRITES,
-	METRIC_SET_ID_COMPUTE_EXTENDED,
-	METRIC_SET_ID_COMPUTE_L3_CACHE,
-	METRIC_SET_ID_HDC_AND_SF,
-	METRIC_SET_ID_L3_1,
-	METRIC_SET_ID_L3_2,
-	METRIC_SET_ID_L3_3,
-	METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND,
-	METRIC_SET_ID_SAMPLER,
-	METRIC_SET_ID_TDL_1,
-	METRIC_SET_ID_TDL_2,
-	METRIC_SET_ID_COMPUTE_EXTRA,
-	METRIC_SET_ID_VME_PIPE,
-	METRIC_SET_ID_TEST_OA,
-};
-
-int i915_oa_n_builtin_metric_sets_kblgt3 = 18;
-
-static const struct i915_oa_reg b_counter_config_render_basic[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2740), 0x00000000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_render_basic[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_render_basic[] = {
-	{ _MMIO(0x9888), 0x166c01e0 },
-	{ _MMIO(0x9888), 0x12170280 },
-	{ _MMIO(0x9888), 0x12370280 },
-	{ _MMIO(0x9888), 0x16ec01e0 },
-	{ _MMIO(0x9888), 0x11930317 },
-	{ _MMIO(0x9888), 0x159303df },
-	{ _MMIO(0x9888), 0x3f900003 },
-	{ _MMIO(0x9888), 0x1a4e0380 },
-	{ _MMIO(0x9888), 0x0a6c0053 },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x0a1b4000 },
-	{ _MMIO(0x9888), 0x1c1c0001 },
-	{ _MMIO(0x9888), 0x002f1000 },
-	{ _MMIO(0x9888), 0x042f1000 },
-	{ _MMIO(0x9888), 0x004c4000 },
-	{ _MMIO(0x9888), 0x0a4c8400 },
-	{ _MMIO(0x9888), 0x0c4c0002 },
-	{ _MMIO(0x9888), 0x000d2000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0a0da000 },
-	{ _MMIO(0x9888), 0x0c0f0400 },
-	{ _MMIO(0x9888), 0x0e0f6600 },
-	{ _MMIO(0x9888), 0x100f0001 },
-	{ _MMIO(0x9888), 0x002c8000 },
-	{ _MMIO(0x9888), 0x162ca200 },
-	{ _MMIO(0x9888), 0x062d8000 },
-	{ _MMIO(0x9888), 0x082d8000 },
-	{ _MMIO(0x9888), 0x00133000 },
-	{ _MMIO(0x9888), 0x08133000 },
-	{ _MMIO(0x9888), 0x00170020 },
-	{ _MMIO(0x9888), 0x08170021 },
-	{ _MMIO(0x9888), 0x10170000 },
-	{ _MMIO(0x9888), 0x0633c000 },
-	{ _MMIO(0x9888), 0x0833c000 },
-	{ _MMIO(0x9888), 0x06370800 },
-	{ _MMIO(0x9888), 0x08370840 },
-	{ _MMIO(0x9888), 0x10370000 },
-	{ _MMIO(0x9888), 0x1ace0200 },
-	{ _MMIO(0x9888), 0x0aec5300 },
-	{ _MMIO(0x9888), 0x10ec0000 },
-	{ _MMIO(0x9888), 0x1cec0000 },
-	{ _MMIO(0x9888), 0x0a9b8000 },
-	{ _MMIO(0x9888), 0x1c9c0002 },
-	{ _MMIO(0x9888), 0x0ccc0002 },
-	{ _MMIO(0x9888), 0x0a8d8000 },
-	{ _MMIO(0x9888), 0x108f0001 },
-	{ _MMIO(0x9888), 0x16ac8000 },
-	{ _MMIO(0x9888), 0x0d933031 },
-	{ _MMIO(0x9888), 0x0f933e3f },
-	{ _MMIO(0x9888), 0x01933d00 },
-	{ _MMIO(0x9888), 0x0393073c },
-	{ _MMIO(0x9888), 0x0593000e },
-	{ _MMIO(0x9888), 0x1d930000 },
-	{ _MMIO(0x9888), 0x19930000 },
-	{ _MMIO(0x9888), 0x1b930000 },
-	{ _MMIO(0x9888), 0x1d900157 },
-	{ _MMIO(0x9888), 0x1f900158 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x2b908000 },
-	{ _MMIO(0x9888), 0x2d908000 },
-	{ _MMIO(0x9888), 0x2f908000 },
-	{ _MMIO(0x9888), 0x31908000 },
-	{ _MMIO(0x9888), 0x15908000 },
-	{ _MMIO(0x9888), 0x17908000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1190003f },
-	{ _MMIO(0x9888), 0x51902240 },
-	{ _MMIO(0x9888), 0x41900c00 },
-	{ _MMIO(0x9888), 0x55900242 },
-	{ _MMIO(0x9888), 0x45900084 },
-	{ _MMIO(0x9888), 0x47901400 },
-	{ _MMIO(0x9888), 0x57902220 },
-	{ _MMIO(0x9888), 0x49900c60 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900063 },
-	{ _MMIO(0x9888), 0x59900002 },
-	{ _MMIO(0x9888), 0x43900c63 },
-	{ _MMIO(0x9888), 0x53902222 },
-};
-
-static int
-get_render_basic_mux_config(struct drm_i915_private *dev_priv,
-			    const struct i915_oa_reg **regs,
-			    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_render_basic;
-	lens[n] = ARRAY_SIZE(mux_config_render_basic);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_basic[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2740), 0x00000000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_basic[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00778008 },
-	{ _MMIO(0xe45c), 0x00088078 },
-	{ _MMIO(0xe55c), 0x00808708 },
-	{ _MMIO(0xe65c), 0x00a08908 },
-};
-
-static const struct i915_oa_reg mux_config_compute_basic[] = {
-	{ _MMIO(0x9888), 0x104f00e0 },
-	{ _MMIO(0x9888), 0x124f1c00 },
-	{ _MMIO(0x9888), 0x106c00e0 },
-	{ _MMIO(0x9888), 0x37906800 },
-	{ _MMIO(0x9888), 0x3f900003 },
-	{ _MMIO(0x9888), 0x004e8000 },
-	{ _MMIO(0x9888), 0x1a4e0820 },
-	{ _MMIO(0x9888), 0x1c4e0002 },
-	{ _MMIO(0x9888), 0x064f0900 },
-	{ _MMIO(0x9888), 0x084f0032 },
-	{ _MMIO(0x9888), 0x0a4f1891 },
-	{ _MMIO(0x9888), 0x0c4f0e00 },
-	{ _MMIO(0x9888), 0x0e4f003c },
-	{ _MMIO(0x9888), 0x004f0d80 },
-	{ _MMIO(0x9888), 0x024f003b },
-	{ _MMIO(0x9888), 0x006c0002 },
-	{ _MMIO(0x9888), 0x086c0100 },
-	{ _MMIO(0x9888), 0x0c6c000c },
-	{ _MMIO(0x9888), 0x0e6c0b00 },
-	{ _MMIO(0x9888), 0x186c0000 },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1e6c0000 },
-	{ _MMIO(0x9888), 0x001b4000 },
-	{ _MMIO(0x9888), 0x081b8000 },
-	{ _MMIO(0x9888), 0x0c1b4000 },
-	{ _MMIO(0x9888), 0x0e1b8000 },
-	{ _MMIO(0x9888), 0x101c8000 },
-	{ _MMIO(0x9888), 0x1a1c8000 },
-	{ _MMIO(0x9888), 0x1c1c0024 },
-	{ _MMIO(0x9888), 0x065b8000 },
-	{ _MMIO(0x9888), 0x085b4000 },
-	{ _MMIO(0x9888), 0x0a5bc000 },
-	{ _MMIO(0x9888), 0x0c5b8000 },
-	{ _MMIO(0x9888), 0x0e5b4000 },
-	{ _MMIO(0x9888), 0x005b8000 },
-	{ _MMIO(0x9888), 0x025b4000 },
-	{ _MMIO(0x9888), 0x1a5c6000 },
-	{ _MMIO(0x9888), 0x1c5c001b },
-	{ _MMIO(0x9888), 0x125c8000 },
-	{ _MMIO(0x9888), 0x145c8000 },
-	{ _MMIO(0x9888), 0x004c8000 },
-	{ _MMIO(0x9888), 0x0a4c2000 },
-	{ _MMIO(0x9888), 0x0c4c0208 },
-	{ _MMIO(0x9888), 0x000da000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0a0da000 },
-	{ _MMIO(0x9888), 0x0c0da000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x020d2000 },
-	{ _MMIO(0x9888), 0x0c0f5400 },
-	{ _MMIO(0x9888), 0x0e0f5500 },
-	{ _MMIO(0x9888), 0x100f0155 },
-	{ _MMIO(0x9888), 0x002c8000 },
-	{ _MMIO(0x9888), 0x0e2cc000 },
-	{ _MMIO(0x9888), 0x162cfb00 },
-	{ _MMIO(0x9888), 0x182c00be },
-	{ _MMIO(0x9888), 0x022cc000 },
-	{ _MMIO(0x9888), 0x042cc000 },
-	{ _MMIO(0x9888), 0x19900157 },
-	{ _MMIO(0x9888), 0x1b900158 },
-	{ _MMIO(0x9888), 0x1d900105 },
-	{ _MMIO(0x9888), 0x1f900103 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x11900fff },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900800 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900821 },
-	{ _MMIO(0x9888), 0x47900802 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900802 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900002 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900422 },
-	{ _MMIO(0x9888), 0x53904444 },
-};
-
-static int
-get_compute_basic_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_basic;
-	lens[n] = ARRAY_SIZE(mux_config_compute_basic);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_render_pipe_profile[] = {
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007ffea },
-	{ _MMIO(0x2774), 0x00007ffc },
-	{ _MMIO(0x2778), 0x0007affa },
-	{ _MMIO(0x277c), 0x0000f5fd },
-	{ _MMIO(0x2780), 0x00079ffa },
-	{ _MMIO(0x2784), 0x0000f3fb },
-	{ _MMIO(0x2788), 0x0007bf7a },
-	{ _MMIO(0x278c), 0x0000f7e7 },
-	{ _MMIO(0x2790), 0x0007fefa },
-	{ _MMIO(0x2794), 0x0000f7cf },
-	{ _MMIO(0x2798), 0x00077ffa },
-	{ _MMIO(0x279c), 0x0000efdf },
-	{ _MMIO(0x27a0), 0x0006fffa },
-	{ _MMIO(0x27a4), 0x0000cfbf },
-	{ _MMIO(0x27a8), 0x0003fffa },
-	{ _MMIO(0x27ac), 0x00005f7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_render_pipe_profile[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00015014 },
-	{ _MMIO(0xe658), 0x00025024 },
-	{ _MMIO(0xe758), 0x00035034 },
-	{ _MMIO(0xe45c), 0x00045044 },
-	{ _MMIO(0xe55c), 0x00055054 },
-	{ _MMIO(0xe65c), 0x00065064 },
-};
-
-static const struct i915_oa_reg mux_config_render_pipe_profile[] = {
-	{ _MMIO(0x9888), 0x0c0e001f },
-	{ _MMIO(0x9888), 0x0a0f0000 },
-	{ _MMIO(0x9888), 0x10116800 },
-	{ _MMIO(0x9888), 0x178a03e0 },
-	{ _MMIO(0x9888), 0x11824c00 },
-	{ _MMIO(0x9888), 0x11830020 },
-	{ _MMIO(0x9888), 0x13840020 },
-	{ _MMIO(0x9888), 0x11850019 },
-	{ _MMIO(0x9888), 0x11860007 },
-	{ _MMIO(0x9888), 0x01870c40 },
-	{ _MMIO(0x9888), 0x17880000 },
-	{ _MMIO(0x9888), 0x022f4000 },
-	{ _MMIO(0x9888), 0x0a4c0040 },
-	{ _MMIO(0x9888), 0x0c0d8000 },
-	{ _MMIO(0x9888), 0x040d4000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x020e5400 },
-	{ _MMIO(0x9888), 0x000e0000 },
-	{ _MMIO(0x9888), 0x080f0040 },
-	{ _MMIO(0x9888), 0x000f0000 },
-	{ _MMIO(0x9888), 0x100f0000 },
-	{ _MMIO(0x9888), 0x0e0f0040 },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x06104000 },
-	{ _MMIO(0x9888), 0x06110012 },
-	{ _MMIO(0x9888), 0x06131000 },
-	{ _MMIO(0x9888), 0x01898000 },
-	{ _MMIO(0x9888), 0x0d890100 },
-	{ _MMIO(0x9888), 0x03898000 },
-	{ _MMIO(0x9888), 0x09808000 },
-	{ _MMIO(0x9888), 0x0b808000 },
-	{ _MMIO(0x9888), 0x0380c000 },
-	{ _MMIO(0x9888), 0x0f8a0075 },
-	{ _MMIO(0x9888), 0x1d8a0000 },
-	{ _MMIO(0x9888), 0x118a8000 },
-	{ _MMIO(0x9888), 0x1b8a4000 },
-	{ _MMIO(0x9888), 0x138a8000 },
-	{ _MMIO(0x9888), 0x1d81a000 },
-	{ _MMIO(0x9888), 0x15818000 },
-	{ _MMIO(0x9888), 0x17818000 },
-	{ _MMIO(0x9888), 0x0b820030 },
-	{ _MMIO(0x9888), 0x07828000 },
-	{ _MMIO(0x9888), 0x0d824000 },
-	{ _MMIO(0x9888), 0x0f828000 },
-	{ _MMIO(0x9888), 0x05824000 },
-	{ _MMIO(0x9888), 0x0d830003 },
-	{ _MMIO(0x9888), 0x0583000c },
-	{ _MMIO(0x9888), 0x09830000 },
-	{ _MMIO(0x9888), 0x03838000 },
-	{ _MMIO(0x9888), 0x07838000 },
-	{ _MMIO(0x9888), 0x0b840980 },
-	{ _MMIO(0x9888), 0x03844d80 },
-	{ _MMIO(0x9888), 0x11840000 },
-	{ _MMIO(0x9888), 0x09848000 },
-	{ _MMIO(0x9888), 0x09850080 },
-	{ _MMIO(0x9888), 0x03850003 },
-	{ _MMIO(0x9888), 0x01850000 },
-	{ _MMIO(0x9888), 0x07860000 },
-	{ _MMIO(0x9888), 0x0f860400 },
-	{ _MMIO(0x9888), 0x09870032 },
-	{ _MMIO(0x9888), 0x01888052 },
-	{ _MMIO(0x9888), 0x11880000 },
-	{ _MMIO(0x9888), 0x09884000 },
-	{ _MMIO(0x9888), 0x1b931001 },
-	{ _MMIO(0x9888), 0x1d930001 },
-	{ _MMIO(0x9888), 0x19934000 },
-	{ _MMIO(0x9888), 0x1b958000 },
-	{ _MMIO(0x9888), 0x1d950094 },
-	{ _MMIO(0x9888), 0x19958000 },
-	{ _MMIO(0x9888), 0x09e58000 },
-	{ _MMIO(0x9888), 0x0be58000 },
-	{ _MMIO(0x9888), 0x03e5c000 },
-	{ _MMIO(0x9888), 0x0592c000 },
-	{ _MMIO(0x9888), 0x0b928000 },
-	{ _MMIO(0x9888), 0x0d924000 },
-	{ _MMIO(0x9888), 0x0f924000 },
-	{ _MMIO(0x9888), 0x11928000 },
-	{ _MMIO(0x9888), 0x1392c000 },
-	{ _MMIO(0x9888), 0x09924000 },
-	{ _MMIO(0x9888), 0x01985000 },
-	{ _MMIO(0x9888), 0x07988000 },
-	{ _MMIO(0x9888), 0x09981000 },
-	{ _MMIO(0x9888), 0x0b982000 },
-	{ _MMIO(0x9888), 0x0d982000 },
-	{ _MMIO(0x9888), 0x0f989000 },
-	{ _MMIO(0x9888), 0x05982000 },
-	{ _MMIO(0x9888), 0x13904000 },
-	{ _MMIO(0x9888), 0x21904000 },
-	{ _MMIO(0x9888), 0x23904000 },
-	{ _MMIO(0x9888), 0x25908000 },
-	{ _MMIO(0x9888), 0x27904000 },
-	{ _MMIO(0x9888), 0x29908000 },
-	{ _MMIO(0x9888), 0x2b904000 },
-	{ _MMIO(0x9888), 0x2f904000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x15904000 },
-	{ _MMIO(0x9888), 0x17908000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x1190c080 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900440 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900400 },
-	{ _MMIO(0x9888), 0x47900c21 },
-	{ _MMIO(0x9888), 0x57900400 },
-	{ _MMIO(0x9888), 0x49900042 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900024 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900841 },
-	{ _MMIO(0x9888), 0x53900400 },
-};
-
-static int
-get_render_pipe_profile_mux_config(struct drm_i915_private *dev_priv,
-				   const struct i915_oa_reg **regs,
-				   int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_render_pipe_profile;
-	lens[n] = ARRAY_SIZE(mux_config_render_pipe_profile);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_memory_reads[] = {
-	{ _MMIO(0x272c), 0xffffffff },
-	{ _MMIO(0x2728), 0xffffffff },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x271c), 0xffffffff },
-	{ _MMIO(0x2718), 0xffffffff },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x274c), 0x86543210 },
-	{ _MMIO(0x2748), 0x86543210 },
-	{ _MMIO(0x2744), 0x00006667 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x275c), 0x86543210 },
-	{ _MMIO(0x2758), 0x86543210 },
-	{ _MMIO(0x2754), 0x00006465 },
-	{ _MMIO(0x2750), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007f81a },
-	{ _MMIO(0x2774), 0x0000fe00 },
-	{ _MMIO(0x2778), 0x0007f82a },
-	{ _MMIO(0x277c), 0x0000fe00 },
-	{ _MMIO(0x2780), 0x0007f872 },
-	{ _MMIO(0x2784), 0x0000fe00 },
-	{ _MMIO(0x2788), 0x0007f8ba },
-	{ _MMIO(0x278c), 0x0000fe00 },
-	{ _MMIO(0x2790), 0x0007f87a },
-	{ _MMIO(0x2794), 0x0000fe00 },
-	{ _MMIO(0x2798), 0x0007f8ea },
-	{ _MMIO(0x279c), 0x0000fe00 },
-	{ _MMIO(0x27a0), 0x0007f8e2 },
-	{ _MMIO(0x27a4), 0x0000fe00 },
-	{ _MMIO(0x27a8), 0x0007f8f2 },
-	{ _MMIO(0x27ac), 0x0000fe00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_memory_reads[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00015014 },
-	{ _MMIO(0xe658), 0x00025024 },
-	{ _MMIO(0xe758), 0x00035034 },
-	{ _MMIO(0xe45c), 0x00045044 },
-	{ _MMIO(0xe55c), 0x00055054 },
-	{ _MMIO(0xe65c), 0x00065064 },
-};
-
-static const struct i915_oa_reg mux_config_memory_reads[] = {
-	{ _MMIO(0x9888), 0x11810c00 },
-	{ _MMIO(0x9888), 0x1381001a },
-	{ _MMIO(0x9888), 0x37906800 },
-	{ _MMIO(0x9888), 0x3f900064 },
-	{ _MMIO(0x9888), 0x03811300 },
-	{ _MMIO(0x9888), 0x05811b12 },
-	{ _MMIO(0x9888), 0x0781001a },
-	{ _MMIO(0x9888), 0x1f810000 },
-	{ _MMIO(0x9888), 0x17810000 },
-	{ _MMIO(0x9888), 0x19810000 },
-	{ _MMIO(0x9888), 0x1b810000 },
-	{ _MMIO(0x9888), 0x1d810000 },
-	{ _MMIO(0x9888), 0x1b930055 },
-	{ _MMIO(0x9888), 0x03e58000 },
-	{ _MMIO(0x9888), 0x05e5c000 },
-	{ _MMIO(0x9888), 0x07e54000 },
-	{ _MMIO(0x9888), 0x13900150 },
-	{ _MMIO(0x9888), 0x21900151 },
-	{ _MMIO(0x9888), 0x23900152 },
-	{ _MMIO(0x9888), 0x25900153 },
-	{ _MMIO(0x9888), 0x27900154 },
-	{ _MMIO(0x9888), 0x29900155 },
-	{ _MMIO(0x9888), 0x2b900156 },
-	{ _MMIO(0x9888), 0x2d900157 },
-	{ _MMIO(0x9888), 0x2f90015f },
-	{ _MMIO(0x9888), 0x31900105 },
-	{ _MMIO(0x9888), 0x15900103 },
-	{ _MMIO(0x9888), 0x17900101 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1d908000 },
-	{ _MMIO(0x9888), 0x1f908000 },
-	{ _MMIO(0x9888), 0x11900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c60 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900c00 },
-	{ _MMIO(0x9888), 0x47900c63 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900c63 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900063 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900003 },
-	{ _MMIO(0x9888), 0x53900000 },
-};
-
-static int
-get_memory_reads_mux_config(struct drm_i915_private *dev_priv,
-			    const struct i915_oa_reg **regs,
-			    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_memory_reads;
-	lens[n] = ARRAY_SIZE(mux_config_memory_reads);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_memory_writes[] = {
-	{ _MMIO(0x272c), 0xffffffff },
-	{ _MMIO(0x2728), 0xffffffff },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x271c), 0xffffffff },
-	{ _MMIO(0x2718), 0xffffffff },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x274c), 0x86543210 },
-	{ _MMIO(0x2748), 0x86543210 },
-	{ _MMIO(0x2744), 0x00006667 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x275c), 0x86543210 },
-	{ _MMIO(0x2758), 0x86543210 },
-	{ _MMIO(0x2754), 0x00006465 },
-	{ _MMIO(0x2750), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007f81a },
-	{ _MMIO(0x2774), 0x0000fe00 },
-	{ _MMIO(0x2778), 0x0007f82a },
-	{ _MMIO(0x277c), 0x0000fe00 },
-	{ _MMIO(0x2780), 0x0007f822 },
-	{ _MMIO(0x2784), 0x0000fe00 },
-	{ _MMIO(0x2788), 0x0007f8ba },
-	{ _MMIO(0x278c), 0x0000fe00 },
-	{ _MMIO(0x2790), 0x0007f87a },
-	{ _MMIO(0x2794), 0x0000fe00 },
-	{ _MMIO(0x2798), 0x0007f8ea },
-	{ _MMIO(0x279c), 0x0000fe00 },
-	{ _MMIO(0x27a0), 0x0007f8e2 },
-	{ _MMIO(0x27a4), 0x0000fe00 },
-	{ _MMIO(0x27a8), 0x0007f8f2 },
-	{ _MMIO(0x27ac), 0x0000fe00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_memory_writes[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00015014 },
-	{ _MMIO(0xe658), 0x00025024 },
-	{ _MMIO(0xe758), 0x00035034 },
-	{ _MMIO(0xe45c), 0x00045044 },
-	{ _MMIO(0xe55c), 0x00055054 },
-	{ _MMIO(0xe65c), 0x00065064 },
-};
-
-static const struct i915_oa_reg mux_config_memory_writes[] = {
-	{ _MMIO(0x9888), 0x11810c00 },
-	{ _MMIO(0x9888), 0x1381001a },
-	{ _MMIO(0x9888), 0x37906800 },
-	{ _MMIO(0x9888), 0x3f901000 },
-	{ _MMIO(0x9888), 0x03811300 },
-	{ _MMIO(0x9888), 0x05811b12 },
-	{ _MMIO(0x9888), 0x0781001a },
-	{ _MMIO(0x9888), 0x1f810000 },
-	{ _MMIO(0x9888), 0x17810000 },
-	{ _MMIO(0x9888), 0x19810000 },
-	{ _MMIO(0x9888), 0x1b810000 },
-	{ _MMIO(0x9888), 0x1d810000 },
-	{ _MMIO(0x9888), 0x1b930055 },
-	{ _MMIO(0x9888), 0x03e58000 },
-	{ _MMIO(0x9888), 0x05e5c000 },
-	{ _MMIO(0x9888), 0x07e54000 },
-	{ _MMIO(0x9888), 0x13900160 },
-	{ _MMIO(0x9888), 0x21900161 },
-	{ _MMIO(0x9888), 0x23900162 },
-	{ _MMIO(0x9888), 0x25900163 },
-	{ _MMIO(0x9888), 0x27900164 },
-	{ _MMIO(0x9888), 0x29900165 },
-	{ _MMIO(0x9888), 0x2b900166 },
-	{ _MMIO(0x9888), 0x2d900167 },
-	{ _MMIO(0x9888), 0x2f900150 },
-	{ _MMIO(0x9888), 0x31900105 },
-	{ _MMIO(0x9888), 0x15900103 },
-	{ _MMIO(0x9888), 0x17900101 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1d908000 },
-	{ _MMIO(0x9888), 0x1f908000 },
-	{ _MMIO(0x9888), 0x11900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c60 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900c00 },
-	{ _MMIO(0x9888), 0x47900c63 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900c63 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900063 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900003 },
-	{ _MMIO(0x9888), 0x53900000 },
-};
-
-static int
-get_memory_writes_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_memory_writes;
-	lens[n] = ARRAY_SIZE(mux_config_memory_writes);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_extended[] = {
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007fc2a },
-	{ _MMIO(0x2774), 0x0000bf00 },
-	{ _MMIO(0x2778), 0x0007fc6a },
-	{ _MMIO(0x277c), 0x0000bf00 },
-	{ _MMIO(0x2780), 0x0007fc92 },
-	{ _MMIO(0x2784), 0x0000bf00 },
-	{ _MMIO(0x2788), 0x0007fca2 },
-	{ _MMIO(0x278c), 0x0000bf00 },
-	{ _MMIO(0x2790), 0x0007fc32 },
-	{ _MMIO(0x2794), 0x0000bf00 },
-	{ _MMIO(0x2798), 0x0007fc9a },
-	{ _MMIO(0x279c), 0x0000bf00 },
-	{ _MMIO(0x27a0), 0x0007fe6a },
-	{ _MMIO(0x27a4), 0x0000bf00 },
-	{ _MMIO(0x27a8), 0x0007fe7a },
-	{ _MMIO(0x27ac), 0x0000bf00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_extended[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00778008 },
-	{ _MMIO(0xe45c), 0x00088078 },
-	{ _MMIO(0xe55c), 0x00808708 },
-	{ _MMIO(0xe65c), 0x00a08908 },
-};
-
-static const struct i915_oa_reg mux_config_compute_extended[] = {
-	{ _MMIO(0x9888), 0x106c00e0 },
-	{ _MMIO(0x9888), 0x141c8160 },
-	{ _MMIO(0x9888), 0x161c8015 },
-	{ _MMIO(0x9888), 0x181c0120 },
-	{ _MMIO(0x9888), 0x004e8000 },
-	{ _MMIO(0x9888), 0x0e4e8000 },
-	{ _MMIO(0x9888), 0x184e8000 },
-	{ _MMIO(0x9888), 0x1a4eaaa0 },
-	{ _MMIO(0x9888), 0x1c4e0002 },
-	{ _MMIO(0x9888), 0x024e8000 },
-	{ _MMIO(0x9888), 0x044e8000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084e8000 },
-	{ _MMIO(0x9888), 0x0a4e8000 },
-	{ _MMIO(0x9888), 0x0e6c0b01 },
-	{ _MMIO(0x9888), 0x006c0200 },
-	{ _MMIO(0x9888), 0x026c000c },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1e6c0000 },
-	{ _MMIO(0x9888), 0x1a6c0000 },
-	{ _MMIO(0x9888), 0x0e1bc000 },
-	{ _MMIO(0x9888), 0x001b8000 },
-	{ _MMIO(0x9888), 0x021bc000 },
-	{ _MMIO(0x9888), 0x001c0041 },
-	{ _MMIO(0x9888), 0x061c4200 },
-	{ _MMIO(0x9888), 0x081c4443 },
-	{ _MMIO(0x9888), 0x0a1c4645 },
-	{ _MMIO(0x9888), 0x0c1c7647 },
-	{ _MMIO(0x9888), 0x041c7357 },
-	{ _MMIO(0x9888), 0x1c1c0030 },
-	{ _MMIO(0x9888), 0x101c0000 },
-	{ _MMIO(0x9888), 0x1a1c0000 },
-	{ _MMIO(0x9888), 0x121c8000 },
-	{ _MMIO(0x9888), 0x004c8000 },
-	{ _MMIO(0x9888), 0x0a4caa2a },
-	{ _MMIO(0x9888), 0x0c4c02aa },
-	{ _MMIO(0x9888), 0x084ca000 },
-	{ _MMIO(0x9888), 0x000da000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0a0da000 },
-	{ _MMIO(0x9888), 0x0c0da000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x0c0f5400 },
-	{ _MMIO(0x9888), 0x0e0f5515 },
-	{ _MMIO(0x9888), 0x100f0155 },
-	{ _MMIO(0x9888), 0x002c8000 },
-	{ _MMIO(0x9888), 0x0e2c8000 },
-	{ _MMIO(0x9888), 0x162caa00 },
-	{ _MMIO(0x9888), 0x182c00aa },
-	{ _MMIO(0x9888), 0x022c8000 },
-	{ _MMIO(0x9888), 0x042c8000 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x11907fff },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900040 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900802 },
-	{ _MMIO(0x9888), 0x47900842 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900842 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900800 },
-	{ _MMIO(0x9888), 0x53900000 },
-};
-
-static int
-get_compute_extended_mux_config(struct drm_i915_private *dev_priv,
-				const struct i915_oa_reg **regs,
-				int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_extended;
-	lens[n] = ARRAY_SIZE(mux_config_compute_extended);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_l3_cache[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x30800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007fffa },
-	{ _MMIO(0x2774), 0x0000fefe },
-	{ _MMIO(0x2778), 0x0007fffa },
-	{ _MMIO(0x277c), 0x0000fefd },
-	{ _MMIO(0x2790), 0x0007fffa },
-	{ _MMIO(0x2794), 0x0000fbef },
-	{ _MMIO(0x2798), 0x0007fffa },
-	{ _MMIO(0x279c), 0x0000fbdf },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_l3_cache[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00101100 },
-	{ _MMIO(0xe45c), 0x00201200 },
-	{ _MMIO(0xe55c), 0x00301300 },
-	{ _MMIO(0xe65c), 0x00401400 },
-};
-
-static const struct i915_oa_reg mux_config_compute_l3_cache[] = {
-	{ _MMIO(0x9888), 0x166c0760 },
-	{ _MMIO(0x9888), 0x1593001e },
-	{ _MMIO(0x9888), 0x3f900003 },
-	{ _MMIO(0x9888), 0x004e8000 },
-	{ _MMIO(0x9888), 0x0e4e8000 },
-	{ _MMIO(0x9888), 0x184e8000 },
-	{ _MMIO(0x9888), 0x1a4e8020 },
-	{ _MMIO(0x9888), 0x1c4e0002 },
-	{ _MMIO(0x9888), 0x006c0051 },
-	{ _MMIO(0x9888), 0x066c5000 },
-	{ _MMIO(0x9888), 0x086c5c5d },
-	{ _MMIO(0x9888), 0x0e6c5e5f },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x186c0000 },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1e6c0000 },
-	{ _MMIO(0x9888), 0x001b4000 },
-	{ _MMIO(0x9888), 0x061b8000 },
-	{ _MMIO(0x9888), 0x081bc000 },
-	{ _MMIO(0x9888), 0x0e1bc000 },
-	{ _MMIO(0x9888), 0x101c8000 },
-	{ _MMIO(0x9888), 0x1a1ce000 },
-	{ _MMIO(0x9888), 0x1c1c0030 },
-	{ _MMIO(0x9888), 0x004c8000 },
-	{ _MMIO(0x9888), 0x0a4c2a00 },
-	{ _MMIO(0x9888), 0x0c4c0280 },
-	{ _MMIO(0x9888), 0x000d2000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x0c0f0400 },
-	{ _MMIO(0x9888), 0x0e0f1500 },
-	{ _MMIO(0x9888), 0x100f0140 },
-	{ _MMIO(0x9888), 0x002c8000 },
-	{ _MMIO(0x9888), 0x0e2c8000 },
-	{ _MMIO(0x9888), 0x162c0a00 },
-	{ _MMIO(0x9888), 0x182c00a0 },
-	{ _MMIO(0x9888), 0x03933300 },
-	{ _MMIO(0x9888), 0x05930032 },
-	{ _MMIO(0x9888), 0x11930000 },
-	{ _MMIO(0x9888), 0x1b930000 },
-	{ _MMIO(0x9888), 0x1d900157 },
-	{ _MMIO(0x9888), 0x1f900158 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1190030f },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900000 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900021 },
-	{ _MMIO(0x9888), 0x47900000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x4b900000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x53904444 },
-	{ _MMIO(0x9888), 0x43900000 },
-};
-
-static int
-get_compute_l3_cache_mux_config(struct drm_i915_private *dev_priv,
-				const struct i915_oa_reg **regs,
-				int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_l3_cache;
-	lens[n] = ARRAY_SIZE(mux_config_compute_l3_cache);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_hdc_and_sf[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x10800000 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x0000fdff },
-};
-
-static const struct i915_oa_reg flex_eu_config_hdc_and_sf[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_hdc_and_sf[] = {
-	{ _MMIO(0x9888), 0x104f0232 },
-	{ _MMIO(0x9888), 0x124f4640 },
-	{ _MMIO(0x9888), 0x106c0232 },
-	{ _MMIO(0x9888), 0x11834400 },
-	{ _MMIO(0x9888), 0x0a4e8000 },
-	{ _MMIO(0x9888), 0x0c4e8000 },
-	{ _MMIO(0x9888), 0x004f1880 },
-	{ _MMIO(0x9888), 0x024f08bb },
-	{ _MMIO(0x9888), 0x044f001b },
-	{ _MMIO(0x9888), 0x046c0100 },
-	{ _MMIO(0x9888), 0x066c000b },
-	{ _MMIO(0x9888), 0x1a6c0000 },
-	{ _MMIO(0x9888), 0x041b8000 },
-	{ _MMIO(0x9888), 0x061b4000 },
-	{ _MMIO(0x9888), 0x1a1c1800 },
-	{ _MMIO(0x9888), 0x005b8000 },
-	{ _MMIO(0x9888), 0x025bc000 },
-	{ _MMIO(0x9888), 0x045b4000 },
-	{ _MMIO(0x9888), 0x125c8000 },
-	{ _MMIO(0x9888), 0x145c8000 },
-	{ _MMIO(0x9888), 0x165c8000 },
-	{ _MMIO(0x9888), 0x185c8000 },
-	{ _MMIO(0x9888), 0x0a4c00a0 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x0c0f5000 },
-	{ _MMIO(0x9888), 0x0e0f0055 },
-	{ _MMIO(0x9888), 0x022cc000 },
-	{ _MMIO(0x9888), 0x042cc000 },
-	{ _MMIO(0x9888), 0x062cc000 },
-	{ _MMIO(0x9888), 0x082cc000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x0f828000 },
-	{ _MMIO(0x9888), 0x0f8305c0 },
-	{ _MMIO(0x9888), 0x09830000 },
-	{ _MMIO(0x9888), 0x07830000 },
-	{ _MMIO(0x9888), 0x1d950080 },
-	{ _MMIO(0x9888), 0x13928000 },
-	{ _MMIO(0x9888), 0x0f988000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x1190fc00 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x4b900040 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900800 },
-	{ _MMIO(0x9888), 0x43900842 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_hdc_and_sf_mux_config(struct drm_i915_private *dev_priv,
-			  const struct i915_oa_reg **regs,
-			  int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_hdc_and_sf;
-	lens[n] = ARRAY_SIZE(mux_config_hdc_and_sf);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_l3_1[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2770), 0x00100070 },
-	{ _MMIO(0x2774), 0x0000fff1 },
-	{ _MMIO(0x2778), 0x00014002 },
-	{ _MMIO(0x277c), 0x0000c3ff },
-	{ _MMIO(0x2780), 0x00010002 },
-	{ _MMIO(0x2784), 0x0000c7ff },
-	{ _MMIO(0x2788), 0x00004002 },
-	{ _MMIO(0x278c), 0x0000d3ff },
-	{ _MMIO(0x2790), 0x00100700 },
-	{ _MMIO(0x2794), 0x0000ff1f },
-	{ _MMIO(0x2798), 0x00001402 },
-	{ _MMIO(0x279c), 0x0000fc3f },
-	{ _MMIO(0x27a0), 0x00001002 },
-	{ _MMIO(0x27a4), 0x0000fc7f },
-	{ _MMIO(0x27a8), 0x00000402 },
-	{ _MMIO(0x27ac), 0x0000fd3f },
-};
-
-static const struct i915_oa_reg flex_eu_config_l3_1[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_l3_1[] = {
-	{ _MMIO(0x9888), 0x126c7b40 },
-	{ _MMIO(0x9888), 0x166c0020 },
-	{ _MMIO(0x9888), 0x0a603444 },
-	{ _MMIO(0x9888), 0x0a613400 },
-	{ _MMIO(0x9888), 0x1a4ea800 },
-	{ _MMIO(0x9888), 0x1c4e0002 },
-	{ _MMIO(0x9888), 0x024e8000 },
-	{ _MMIO(0x9888), 0x044e8000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084e8000 },
-	{ _MMIO(0x9888), 0x0a4e8000 },
-	{ _MMIO(0x9888), 0x064f4000 },
-	{ _MMIO(0x9888), 0x0c6c5327 },
-	{ _MMIO(0x9888), 0x0e6c5425 },
-	{ _MMIO(0x9888), 0x006c2a00 },
-	{ _MMIO(0x9888), 0x026c285b },
-	{ _MMIO(0x9888), 0x046c005c },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1e6c0000 },
-	{ _MMIO(0x9888), 0x1a6c0800 },
-	{ _MMIO(0x9888), 0x0c1bc000 },
-	{ _MMIO(0x9888), 0x0e1bc000 },
-	{ _MMIO(0x9888), 0x001b8000 },
-	{ _MMIO(0x9888), 0x021bc000 },
-	{ _MMIO(0x9888), 0x041bc000 },
-	{ _MMIO(0x9888), 0x1c1c003c },
-	{ _MMIO(0x9888), 0x121c8000 },
-	{ _MMIO(0x9888), 0x141c8000 },
-	{ _MMIO(0x9888), 0x161c8000 },
-	{ _MMIO(0x9888), 0x181c8000 },
-	{ _MMIO(0x9888), 0x1a1c0800 },
-	{ _MMIO(0x9888), 0x065b4000 },
-	{ _MMIO(0x9888), 0x1a5c1000 },
-	{ _MMIO(0x9888), 0x10600000 },
-	{ _MMIO(0x9888), 0x04600000 },
-	{ _MMIO(0x9888), 0x0c610044 },
-	{ _MMIO(0x9888), 0x10610000 },
-	{ _MMIO(0x9888), 0x06610000 },
-	{ _MMIO(0x9888), 0x0c4c02a8 },
-	{ _MMIO(0x9888), 0x084ca000 },
-	{ _MMIO(0x9888), 0x0a4c002a },
-	{ _MMIO(0x9888), 0x0c0da000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x100f0154 },
-	{ _MMIO(0x9888), 0x0c0f5000 },
-	{ _MMIO(0x9888), 0x0e0f0055 },
-	{ _MMIO(0x9888), 0x182c00aa },
-	{ _MMIO(0x9888), 0x022c8000 },
-	{ _MMIO(0x9888), 0x042c8000 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2cc000 },
-	{ _MMIO(0x9888), 0x1190ffc0 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900420 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900021 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900400 },
-	{ _MMIO(0x9888), 0x43900421 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900040 },
-};
-
-static int
-get_l3_1_mux_config(struct drm_i915_private *dev_priv,
-		    const struct i915_oa_reg **regs,
-		    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_l3_1;
-	lens[n] = ARRAY_SIZE(mux_config_l3_1);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_l3_2[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00100070 },
-	{ _MMIO(0x2774), 0x0000fff1 },
-	{ _MMIO(0x2778), 0x00028002 },
-	{ _MMIO(0x277c), 0x000087ff },
-	{ _MMIO(0x2780), 0x00020002 },
-	{ _MMIO(0x2784), 0x00008fff },
-	{ _MMIO(0x2788), 0x00008002 },
-	{ _MMIO(0x278c), 0x0000a7ff },
-};
-
-static const struct i915_oa_reg flex_eu_config_l3_2[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_l3_2[] = {
-	{ _MMIO(0x9888), 0x126c02e0 },
-	{ _MMIO(0x9888), 0x146c0001 },
-	{ _MMIO(0x9888), 0x0a623400 },
-	{ _MMIO(0x9888), 0x044e8000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084e8000 },
-	{ _MMIO(0x9888), 0x0a4e8000 },
-	{ _MMIO(0x9888), 0x064f4000 },
-	{ _MMIO(0x9888), 0x026c3324 },
-	{ _MMIO(0x9888), 0x046c3422 },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x1a6c0000 },
-	{ _MMIO(0x9888), 0x021bc000 },
-	{ _MMIO(0x9888), 0x041bc000 },
-	{ _MMIO(0x9888), 0x141c8000 },
-	{ _MMIO(0x9888), 0x161c8000 },
-	{ _MMIO(0x9888), 0x181c8000 },
-	{ _MMIO(0x9888), 0x1a1c0800 },
-	{ _MMIO(0x9888), 0x065b4000 },
-	{ _MMIO(0x9888), 0x1a5c1000 },
-	{ _MMIO(0x9888), 0x06614000 },
-	{ _MMIO(0x9888), 0x0c620044 },
-	{ _MMIO(0x9888), 0x10620000 },
-	{ _MMIO(0x9888), 0x06620000 },
-	{ _MMIO(0x9888), 0x084c8000 },
-	{ _MMIO(0x9888), 0x0a4c002a },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x0c0f4000 },
-	{ _MMIO(0x9888), 0x0e0f0055 },
-	{ _MMIO(0x9888), 0x042c8000 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2cc000 },
-	{ _MMIO(0x9888), 0x1190f800 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x43900000 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_l3_2_mux_config(struct drm_i915_private *dev_priv,
-		    const struct i915_oa_reg **regs,
-		    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_l3_2;
-	lens[n] = ARRAY_SIZE(mux_config_l3_2);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_l3_3[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00100070 },
-	{ _MMIO(0x2774), 0x0000fff1 },
-	{ _MMIO(0x2778), 0x00028002 },
-	{ _MMIO(0x277c), 0x000087ff },
-	{ _MMIO(0x2780), 0x00020002 },
-	{ _MMIO(0x2784), 0x00008fff },
-	{ _MMIO(0x2788), 0x00008002 },
-	{ _MMIO(0x278c), 0x0000a7ff },
-};
-
-static const struct i915_oa_reg flex_eu_config_l3_3[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_l3_3[] = {
-	{ _MMIO(0x9888), 0x126c4e80 },
-	{ _MMIO(0x9888), 0x146c0000 },
-	{ _MMIO(0x9888), 0x0a633400 },
-	{ _MMIO(0x9888), 0x044e8000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084e8000 },
-	{ _MMIO(0x9888), 0x0a4e8000 },
-	{ _MMIO(0x9888), 0x0c4e8000 },
-	{ _MMIO(0x9888), 0x026c3321 },
-	{ _MMIO(0x9888), 0x046c342f },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x1a6c2000 },
-	{ _MMIO(0x9888), 0x021bc000 },
-	{ _MMIO(0x9888), 0x041bc000 },
-	{ _MMIO(0x9888), 0x061b4000 },
-	{ _MMIO(0x9888), 0x141c8000 },
-	{ _MMIO(0x9888), 0x161c8000 },
-	{ _MMIO(0x9888), 0x181c8000 },
-	{ _MMIO(0x9888), 0x1a1c1800 },
-	{ _MMIO(0x9888), 0x06604000 },
-	{ _MMIO(0x9888), 0x0c630044 },
-	{ _MMIO(0x9888), 0x10630000 },
-	{ _MMIO(0x9888), 0x06630000 },
-	{ _MMIO(0x9888), 0x084c8000 },
-	{ _MMIO(0x9888), 0x0a4c00aa },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x0c0f4000 },
-	{ _MMIO(0x9888), 0x0e0f0055 },
-	{ _MMIO(0x9888), 0x042c8000 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x1190f800 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x43900842 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900002 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_l3_3_mux_config(struct drm_i915_private *dev_priv,
-		    const struct i915_oa_reg **regs,
-		    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_l3_3;
-	lens[n] = ARRAY_SIZE(mux_config_l3_3);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x30800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x0000efff },
-	{ _MMIO(0x2778), 0x00006000 },
-	{ _MMIO(0x277c), 0x0000f3ff },
-};
-
-static const struct i915_oa_reg flex_eu_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0x9888), 0x102f3800 },
-	{ _MMIO(0x9888), 0x144d0500 },
-	{ _MMIO(0x9888), 0x120d03c0 },
-	{ _MMIO(0x9888), 0x140d03cf },
-	{ _MMIO(0x9888), 0x0c0f0004 },
-	{ _MMIO(0x9888), 0x0c4e4000 },
-	{ _MMIO(0x9888), 0x042f0480 },
-	{ _MMIO(0x9888), 0x082f0000 },
-	{ _MMIO(0x9888), 0x022f0000 },
-	{ _MMIO(0x9888), 0x0a4c0090 },
-	{ _MMIO(0x9888), 0x064d0027 },
-	{ _MMIO(0x9888), 0x004d0000 },
-	{ _MMIO(0x9888), 0x000d0d40 },
-	{ _MMIO(0x9888), 0x020d803f },
-	{ _MMIO(0x9888), 0x040d8023 },
-	{ _MMIO(0x9888), 0x100d0000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x020f0010 },
-	{ _MMIO(0x9888), 0x000f0000 },
-	{ _MMIO(0x9888), 0x0e0f0050 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x1190fc00 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41901400 },
-	{ _MMIO(0x9888), 0x43901485 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900001 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_rasterizer_and_pixel_backend_mux_config(struct drm_i915_private *dev_priv,
-					    const struct i915_oa_reg **regs,
-					    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_rasterizer_and_pixel_backend;
-	lens[n] = ARRAY_SIZE(mux_config_rasterizer_and_pixel_backend);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_sampler[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x70800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x0000c000 },
-	{ _MMIO(0x2774), 0x0000e7ff },
-	{ _MMIO(0x2778), 0x00003000 },
-	{ _MMIO(0x277c), 0x0000f9ff },
-	{ _MMIO(0x2780), 0x00000c00 },
-	{ _MMIO(0x2784), 0x0000fe7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_sampler[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_sampler[] = {
-	{ _MMIO(0x9888), 0x14152c00 },
-	{ _MMIO(0x9888), 0x16150005 },
-	{ _MMIO(0x9888), 0x121600a0 },
-	{ _MMIO(0x9888), 0x14352c00 },
-	{ _MMIO(0x9888), 0x16350005 },
-	{ _MMIO(0x9888), 0x123600a0 },
-	{ _MMIO(0x9888), 0x14552c00 },
-	{ _MMIO(0x9888), 0x16550005 },
-	{ _MMIO(0x9888), 0x125600a0 },
-	{ _MMIO(0x9888), 0x062f6000 },
-	{ _MMIO(0x9888), 0x022f2000 },
-	{ _MMIO(0x9888), 0x0c4c0050 },
-	{ _MMIO(0x9888), 0x0a4c0010 },
-	{ _MMIO(0x9888), 0x0c0d8000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x100f0350 },
-	{ _MMIO(0x9888), 0x0c0fb000 },
-	{ _MMIO(0x9888), 0x0e0f00da },
-	{ _MMIO(0x9888), 0x182c0028 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x022dc000 },
-	{ _MMIO(0x9888), 0x042d4000 },
-	{ _MMIO(0x9888), 0x0c138000 },
-	{ _MMIO(0x9888), 0x0e132000 },
-	{ _MMIO(0x9888), 0x0413c000 },
-	{ _MMIO(0x9888), 0x1c140018 },
-	{ _MMIO(0x9888), 0x0c157000 },
-	{ _MMIO(0x9888), 0x0e150078 },
-	{ _MMIO(0x9888), 0x10150000 },
-	{ _MMIO(0x9888), 0x04162180 },
-	{ _MMIO(0x9888), 0x02160000 },
-	{ _MMIO(0x9888), 0x04174000 },
-	{ _MMIO(0x9888), 0x0233a000 },
-	{ _MMIO(0x9888), 0x04333000 },
-	{ _MMIO(0x9888), 0x14348000 },
-	{ _MMIO(0x9888), 0x16348000 },
-	{ _MMIO(0x9888), 0x02357870 },
-	{ _MMIO(0x9888), 0x10350000 },
-	{ _MMIO(0x9888), 0x04360043 },
-	{ _MMIO(0x9888), 0x02360000 },
-	{ _MMIO(0x9888), 0x04371000 },
-	{ _MMIO(0x9888), 0x0e538000 },
-	{ _MMIO(0x9888), 0x00538000 },
-	{ _MMIO(0x9888), 0x06533000 },
-	{ _MMIO(0x9888), 0x1c540020 },
-	{ _MMIO(0x9888), 0x12548000 },
-	{ _MMIO(0x9888), 0x0e557000 },
-	{ _MMIO(0x9888), 0x00557800 },
-	{ _MMIO(0x9888), 0x10550000 },
-	{ _MMIO(0x9888), 0x06560043 },
-	{ _MMIO(0x9888), 0x02560000 },
-	{ _MMIO(0x9888), 0x06571000 },
-	{ _MMIO(0x9888), 0x1190ff80 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900060 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c00 },
-	{ _MMIO(0x9888), 0x43900842 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900060 },
-};
-
-static int
-get_sampler_mux_config(struct drm_i915_private *dev_priv,
-		       const struct i915_oa_reg **regs,
-		       int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_sampler;
-	lens[n] = ARRAY_SIZE(mux_config_sampler);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_tdl_1[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x00007fff },
-	{ _MMIO(0x2778), 0x00000000 },
-	{ _MMIO(0x277c), 0x00009fff },
-	{ _MMIO(0x2780), 0x00000002 },
-	{ _MMIO(0x2784), 0x0000efff },
-	{ _MMIO(0x2788), 0x00000000 },
-	{ _MMIO(0x278c), 0x0000f3ff },
-	{ _MMIO(0x2790), 0x00000002 },
-	{ _MMIO(0x2794), 0x0000fdff },
-	{ _MMIO(0x2798), 0x00000000 },
-	{ _MMIO(0x279c), 0x0000fe7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_tdl_1[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_tdl_1[] = {
-	{ _MMIO(0x9888), 0x12120000 },
-	{ _MMIO(0x9888), 0x12320000 },
-	{ _MMIO(0x9888), 0x12520000 },
-	{ _MMIO(0x9888), 0x002f8000 },
-	{ _MMIO(0x9888), 0x022f3000 },
-	{ _MMIO(0x9888), 0x0a4c0015 },
-	{ _MMIO(0x9888), 0x0c0d8000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x100f03a0 },
-	{ _MMIO(0x9888), 0x0c0ff000 },
-	{ _MMIO(0x9888), 0x0e0f0095 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2d8000 },
-	{ _MMIO(0x9888), 0x0e2d4000 },
-	{ _MMIO(0x9888), 0x062d4000 },
-	{ _MMIO(0x9888), 0x02108000 },
-	{ _MMIO(0x9888), 0x0410c000 },
-	{ _MMIO(0x9888), 0x02118000 },
-	{ _MMIO(0x9888), 0x0411c000 },
-	{ _MMIO(0x9888), 0x02121880 },
-	{ _MMIO(0x9888), 0x041219b5 },
-	{ _MMIO(0x9888), 0x00120000 },
-	{ _MMIO(0x9888), 0x02134000 },
-	{ _MMIO(0x9888), 0x04135000 },
-	{ _MMIO(0x9888), 0x0c308000 },
-	{ _MMIO(0x9888), 0x0e304000 },
-	{ _MMIO(0x9888), 0x06304000 },
-	{ _MMIO(0x9888), 0x0c318000 },
-	{ _MMIO(0x9888), 0x0e314000 },
-	{ _MMIO(0x9888), 0x06314000 },
-	{ _MMIO(0x9888), 0x0c321a80 },
-	{ _MMIO(0x9888), 0x0e320033 },
-	{ _MMIO(0x9888), 0x06320031 },
-	{ _MMIO(0x9888), 0x00320000 },
-	{ _MMIO(0x9888), 0x0c334000 },
-	{ _MMIO(0x9888), 0x0e331000 },
-	{ _MMIO(0x9888), 0x06331000 },
-	{ _MMIO(0x9888), 0x0e508000 },
-	{ _MMIO(0x9888), 0x00508000 },
-	{ _MMIO(0x9888), 0x02504000 },
-	{ _MMIO(0x9888), 0x0e518000 },
-	{ _MMIO(0x9888), 0x00518000 },
-	{ _MMIO(0x9888), 0x02514000 },
-	{ _MMIO(0x9888), 0x0e521880 },
-	{ _MMIO(0x9888), 0x00521a80 },
-	{ _MMIO(0x9888), 0x02520033 },
-	{ _MMIO(0x9888), 0x0e534000 },
-	{ _MMIO(0x9888), 0x00534000 },
-	{ _MMIO(0x9888), 0x02531000 },
-	{ _MMIO(0x9888), 0x1190ff80 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900800 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900062 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c00 },
-	{ _MMIO(0x9888), 0x43900003 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900040 },
-};
-
-static int
-get_tdl_1_mux_config(struct drm_i915_private *dev_priv,
-		     const struct i915_oa_reg **regs,
-		     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_tdl_1;
-	lens[n] = ARRAY_SIZE(mux_config_tdl_1);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_tdl_2[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_tdl_2[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_tdl_2[] = {
-	{ _MMIO(0x9888), 0x12124d60 },
-	{ _MMIO(0x9888), 0x12322e60 },
-	{ _MMIO(0x9888), 0x12524d60 },
-	{ _MMIO(0x9888), 0x022f3000 },
-	{ _MMIO(0x9888), 0x0a4c0014 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x0c0fe000 },
-	{ _MMIO(0x9888), 0x0e0f0097 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x002d8000 },
-	{ _MMIO(0x9888), 0x062d4000 },
-	{ _MMIO(0x9888), 0x0410c000 },
-	{ _MMIO(0x9888), 0x0411c000 },
-	{ _MMIO(0x9888), 0x04121fb7 },
-	{ _MMIO(0x9888), 0x00120000 },
-	{ _MMIO(0x9888), 0x04135000 },
-	{ _MMIO(0x9888), 0x00308000 },
-	{ _MMIO(0x9888), 0x06304000 },
-	{ _MMIO(0x9888), 0x00318000 },
-	{ _MMIO(0x9888), 0x06314000 },
-	{ _MMIO(0x9888), 0x00321b80 },
-	{ _MMIO(0x9888), 0x0632003f },
-	{ _MMIO(0x9888), 0x00334000 },
-	{ _MMIO(0x9888), 0x06331000 },
-	{ _MMIO(0x9888), 0x0250c000 },
-	{ _MMIO(0x9888), 0x0251c000 },
-	{ _MMIO(0x9888), 0x02521fb7 },
-	{ _MMIO(0x9888), 0x00520000 },
-	{ _MMIO(0x9888), 0x02535000 },
-	{ _MMIO(0x9888), 0x1190fc00 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900800 },
-	{ _MMIO(0x9888), 0x43900063 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900040 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_tdl_2_mux_config(struct drm_i915_private *dev_priv,
-		     const struct i915_oa_reg **regs,
-		     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_tdl_2;
-	lens[n] = ARRAY_SIZE(mux_config_tdl_2);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_extra[] = {
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_extra[] = {
-};
-
-static const struct i915_oa_reg mux_config_compute_extra[] = {
-	{ _MMIO(0x9888), 0x121203e0 },
-	{ _MMIO(0x9888), 0x123203e0 },
-	{ _MMIO(0x9888), 0x125203e0 },
-	{ _MMIO(0x9888), 0x129203e0 },
-	{ _MMIO(0x9888), 0x12b203e0 },
-	{ _MMIO(0x9888), 0x12d203e0 },
-	{ _MMIO(0x9888), 0x024ec000 },
-	{ _MMIO(0x9888), 0x044ec000 },
-	{ _MMIO(0x9888), 0x064ec000 },
-	{ _MMIO(0x9888), 0x022f4000 },
-	{ _MMIO(0x9888), 0x084ca000 },
-	{ _MMIO(0x9888), 0x0a4c0042 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x0c0f5000 },
-	{ _MMIO(0x9888), 0x0e0f006d },
-	{ _MMIO(0x9888), 0x022c8000 },
-	{ _MMIO(0x9888), 0x042c8000 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x042d8000 },
-	{ _MMIO(0x9888), 0x06104000 },
-	{ _MMIO(0x9888), 0x06114000 },
-	{ _MMIO(0x9888), 0x06120033 },
-	{ _MMIO(0x9888), 0x00120000 },
-	{ _MMIO(0x9888), 0x06131000 },
-	{ _MMIO(0x9888), 0x04308000 },
-	{ _MMIO(0x9888), 0x04318000 },
-	{ _MMIO(0x9888), 0x04321980 },
-	{ _MMIO(0x9888), 0x00320000 },
-	{ _MMIO(0x9888), 0x04334000 },
-	{ _MMIO(0x9888), 0x04504000 },
-	{ _MMIO(0x9888), 0x04514000 },
-	{ _MMIO(0x9888), 0x04520033 },
-	{ _MMIO(0x9888), 0x00520000 },
-	{ _MMIO(0x9888), 0x04531000 },
-	{ _MMIO(0x9888), 0x00af8000 },
-	{ _MMIO(0x9888), 0x0acc0001 },
-	{ _MMIO(0x9888), 0x008d8000 },
-	{ _MMIO(0x9888), 0x028da000 },
-	{ _MMIO(0x9888), 0x0c8fb000 },
-	{ _MMIO(0x9888), 0x0e8f0001 },
-	{ _MMIO(0x9888), 0x06ac8000 },
-	{ _MMIO(0x9888), 0x02ad4000 },
-	{ _MMIO(0x9888), 0x02908000 },
-	{ _MMIO(0x9888), 0x02918000 },
-	{ _MMIO(0x9888), 0x02921980 },
-	{ _MMIO(0x9888), 0x00920000 },
-	{ _MMIO(0x9888), 0x02934000 },
-	{ _MMIO(0x9888), 0x02b04000 },
-	{ _MMIO(0x9888), 0x02b14000 },
-	{ _MMIO(0x9888), 0x02b20033 },
-	{ _MMIO(0x9888), 0x00b20000 },
-	{ _MMIO(0x9888), 0x02b31000 },
-	{ _MMIO(0x9888), 0x00d08000 },
-	{ _MMIO(0x9888), 0x00d18000 },
-	{ _MMIO(0x9888), 0x00d21980 },
-	{ _MMIO(0x9888), 0x00d34000 },
-	{ _MMIO(0x9888), 0x1190fc00 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c00 },
-	{ _MMIO(0x9888), 0x43900002 },
-	{ _MMIO(0x9888), 0x53900420 },
-	{ _MMIO(0x9888), 0x459000a1 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_compute_extra_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_extra;
-	lens[n] = ARRAY_SIZE(mux_config_compute_extra);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_vme_pipe[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2770), 0x00100030 },
-	{ _MMIO(0x2774), 0x0000fff9 },
-	{ _MMIO(0x2778), 0x00000002 },
-	{ _MMIO(0x277c), 0x0000fffc },
-	{ _MMIO(0x2780), 0x00000002 },
-	{ _MMIO(0x2784), 0x0000fff3 },
-	{ _MMIO(0x2788), 0x00100180 },
-	{ _MMIO(0x278c), 0x0000ffcf },
-	{ _MMIO(0x2790), 0x00000002 },
-	{ _MMIO(0x2794), 0x0000ffcf },
-	{ _MMIO(0x2798), 0x00000002 },
-	{ _MMIO(0x279c), 0x0000ff3f },
-};
-
-static const struct i915_oa_reg flex_eu_config_vme_pipe[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00008003 },
-};
-
-static const struct i915_oa_reg mux_config_vme_pipe[] = {
-	{ _MMIO(0x9888), 0x141a5800 },
-	{ _MMIO(0x9888), 0x161a00c0 },
-	{ _MMIO(0x9888), 0x12180240 },
-	{ _MMIO(0x9888), 0x14180002 },
-	{ _MMIO(0x9888), 0x149a5800 },
-	{ _MMIO(0x9888), 0x169a00c0 },
-	{ _MMIO(0x9888), 0x12980240 },
-	{ _MMIO(0x9888), 0x14980002 },
-	{ _MMIO(0x9888), 0x1a4e3fc0 },
-	{ _MMIO(0x9888), 0x002f1000 },
-	{ _MMIO(0x9888), 0x022f8000 },
-	{ _MMIO(0x9888), 0x042f3000 },
-	{ _MMIO(0x9888), 0x004c4000 },
-	{ _MMIO(0x9888), 0x0a4c9500 },
-	{ _MMIO(0x9888), 0x0c4c002a },
-	{ _MMIO(0x9888), 0x000d2000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0a0da000 },
-	{ _MMIO(0x9888), 0x0c0da000 },
-	{ _MMIO(0x9888), 0x0c0f0400 },
-	{ _MMIO(0x9888), 0x0e0f5500 },
-	{ _MMIO(0x9888), 0x100f0015 },
-	{ _MMIO(0x9888), 0x002c8000 },
-	{ _MMIO(0x9888), 0x0e2c8000 },
-	{ _MMIO(0x9888), 0x162caa00 },
-	{ _MMIO(0x9888), 0x182c000a },
-	{ _MMIO(0x9888), 0x04193000 },
-	{ _MMIO(0x9888), 0x081a28c1 },
-	{ _MMIO(0x9888), 0x001a0000 },
-	{ _MMIO(0x9888), 0x00133000 },
-	{ _MMIO(0x9888), 0x0613c000 },
-	{ _MMIO(0x9888), 0x0813f000 },
-	{ _MMIO(0x9888), 0x00172000 },
-	{ _MMIO(0x9888), 0x06178000 },
-	{ _MMIO(0x9888), 0x0817a000 },
-	{ _MMIO(0x9888), 0x00180037 },
-	{ _MMIO(0x9888), 0x06180940 },
-	{ _MMIO(0x9888), 0x08180000 },
-	{ _MMIO(0x9888), 0x02180000 },
-	{ _MMIO(0x9888), 0x04183000 },
-	{ _MMIO(0x9888), 0x04afc000 },
-	{ _MMIO(0x9888), 0x06af3000 },
-	{ _MMIO(0x9888), 0x0acc4000 },
-	{ _MMIO(0x9888), 0x0ccc0015 },
-	{ _MMIO(0x9888), 0x0a8da000 },
-	{ _MMIO(0x9888), 0x0c8da000 },
-	{ _MMIO(0x9888), 0x0e8f4000 },
-	{ _MMIO(0x9888), 0x108f0015 },
-	{ _MMIO(0x9888), 0x16aca000 },
-	{ _MMIO(0x9888), 0x18ac000a },
-	{ _MMIO(0x9888), 0x06993000 },
-	{ _MMIO(0x9888), 0x0c9a28c1 },
-	{ _MMIO(0x9888), 0x009a0000 },
-	{ _MMIO(0x9888), 0x0a93f000 },
-	{ _MMIO(0x9888), 0x0c93f000 },
-	{ _MMIO(0x9888), 0x0a97a000 },
-	{ _MMIO(0x9888), 0x0c97a000 },
-	{ _MMIO(0x9888), 0x0a980977 },
-	{ _MMIO(0x9888), 0x08980000 },
-	{ _MMIO(0x9888), 0x04980000 },
-	{ _MMIO(0x9888), 0x06983000 },
-	{ _MMIO(0x9888), 0x119000ff },
-	{ _MMIO(0x9888), 0x51900040 },
-	{ _MMIO(0x9888), 0x41900020 },
-	{ _MMIO(0x9888), 0x55900004 },
-	{ _MMIO(0x9888), 0x45900400 },
-	{ _MMIO(0x9888), 0x479008a5 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900002 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_vme_pipe_mux_config(struct drm_i915_private *dev_priv,
-			const struct i915_oa_reg **regs,
-			int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_vme_pipe;
-	lens[n] = ARRAY_SIZE(mux_config_vme_pipe);
-	n++;
-
-	return n;
-}
-
 static const struct i915_oa_reg b_counter_config_test_oa[] = {
 	{ _MMIO(0x2740), 0x00000000 },
 	{ _MMIO(0x2744), 0x00800000 },
@@ -1931,6 +60,7 @@ static const struct i915_oa_reg flex_eu_config_test_oa[] = {
 };
 
 static const struct i915_oa_reg mux_config_test_oa[] = {
+	{ _MMIO(0x9840), 0x00000080 },
 	{ _MMIO(0x9888), 0x11810000 },
 	{ _MMIO(0x9888), 0x07810013 },
 	{ _MMIO(0x9888), 0x1f810000 },
@@ -1945,1096 +75,35 @@ static const struct i915_oa_reg mux_config_test_oa[] = {
 	{ _MMIO(0x9888), 0x33900000 },
 };
 
-static int
-get_test_oa_mux_config(struct drm_i915_private *dev_priv,
-		       const struct i915_oa_reg **regs,
-		       int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_test_oa;
-	lens[n] = ARRAY_SIZE(mux_config_test_oa);
-	n++;
-
-	return n;
-}
-
-int i915_oa_select_metric_set_kblgt3(struct drm_i915_private *dev_priv)
-{
-	dev_priv->perf.oa.n_mux_configs = 0;
-	dev_priv->perf.oa.b_counter_regs = NULL;
-	dev_priv->perf.oa.b_counter_regs_len = 0;
-	dev_priv->perf.oa.flex_regs = NULL;
-	dev_priv->perf.oa.flex_regs_len = 0;
-
-	switch (dev_priv->perf.oa.metrics_set) {
-	case METRIC_SET_ID_RENDER_BASIC:
-		dev_priv->perf.oa.n_mux_configs =
-			get_render_basic_mux_config(dev_priv,
-						    dev_priv->perf.oa.mux_regs,
-						    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_BASIC\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_render_basic;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_render_basic);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_render_basic;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_render_basic);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_BASIC:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_basic_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_BASIC\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_basic;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_basic);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_basic;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_basic);
-
-		return 0;
-	case METRIC_SET_ID_RENDER_PIPE_PROFILE:
-		dev_priv->perf.oa.n_mux_configs =
-			get_render_pipe_profile_mux_config(dev_priv,
-							   dev_priv->perf.oa.mux_regs,
-							   dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_PIPE_PROFILE\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_render_pipe_profile;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_render_pipe_profile);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_render_pipe_profile;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_render_pipe_profile);
-
-		return 0;
-	case METRIC_SET_ID_MEMORY_READS:
-		dev_priv->perf.oa.n_mux_configs =
-			get_memory_reads_mux_config(dev_priv,
-						    dev_priv->perf.oa.mux_regs,
-						    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_READS\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_memory_reads;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_memory_reads);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_memory_reads;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_memory_reads);
-
-		return 0;
-	case METRIC_SET_ID_MEMORY_WRITES:
-		dev_priv->perf.oa.n_mux_configs =
-			get_memory_writes_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_WRITES\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_memory_writes;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_memory_writes);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_memory_writes;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_memory_writes);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_EXTENDED:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_extended_mux_config(dev_priv,
-							dev_priv->perf.oa.mux_regs,
-							dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTENDED\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_extended;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_extended);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_extended;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_extended);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_L3_CACHE:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_l3_cache_mux_config(dev_priv,
-							dev_priv->perf.oa.mux_regs,
-							dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_L3_CACHE\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_l3_cache;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_l3_cache);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_l3_cache;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_l3_cache);
-
-		return 0;
-	case METRIC_SET_ID_HDC_AND_SF:
-		dev_priv->perf.oa.n_mux_configs =
-			get_hdc_and_sf_mux_config(dev_priv,
-						  dev_priv->perf.oa.mux_regs,
-						  dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"HDC_AND_SF\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_hdc_and_sf;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_hdc_and_sf);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_hdc_and_sf;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_hdc_and_sf);
-
-		return 0;
-	case METRIC_SET_ID_L3_1:
-		dev_priv->perf.oa.n_mux_configs =
-			get_l3_1_mux_config(dev_priv,
-					    dev_priv->perf.oa.mux_regs,
-					    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_1\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_l3_1;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_l3_1);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_l3_1;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_l3_1);
-
-		return 0;
-	case METRIC_SET_ID_L3_2:
-		dev_priv->perf.oa.n_mux_configs =
-			get_l3_2_mux_config(dev_priv,
-					    dev_priv->perf.oa.mux_regs,
-					    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_2\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_l3_2;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_l3_2);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_l3_2;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_l3_2);
-
-		return 0;
-	case METRIC_SET_ID_L3_3:
-		dev_priv->perf.oa.n_mux_configs =
-			get_l3_3_mux_config(dev_priv,
-					    dev_priv->perf.oa.mux_regs,
-					    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_3\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_l3_3;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_l3_3);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_l3_3;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_l3_3);
-
-		return 0;
-	case METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND:
-		dev_priv->perf.oa.n_mux_configs =
-			get_rasterizer_and_pixel_backend_mux_config(dev_priv,
-								    dev_priv->perf.oa.mux_regs,
-								    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RASTERIZER_AND_PIXEL_BACKEND\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_rasterizer_and_pixel_backend;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_rasterizer_and_pixel_backend);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_rasterizer_and_pixel_backend;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_rasterizer_and_pixel_backend);
-
-		return 0;
-	case METRIC_SET_ID_SAMPLER:
-		dev_priv->perf.oa.n_mux_configs =
-			get_sampler_mux_config(dev_priv,
-					       dev_priv->perf.oa.mux_regs,
-					       dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"SAMPLER\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_sampler;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_sampler);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_sampler;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_sampler);
-
-		return 0;
-	case METRIC_SET_ID_TDL_1:
-		dev_priv->perf.oa.n_mux_configs =
-			get_tdl_1_mux_config(dev_priv,
-					     dev_priv->perf.oa.mux_regs,
-					     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_1\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_tdl_1;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_tdl_1);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_tdl_1;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_tdl_1);
-
-		return 0;
-	case METRIC_SET_ID_TDL_2:
-		dev_priv->perf.oa.n_mux_configs =
-			get_tdl_2_mux_config(dev_priv,
-					     dev_priv->perf.oa.mux_regs,
-					     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_2\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_tdl_2;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_tdl_2);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_tdl_2;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_tdl_2);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_EXTRA:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_extra_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTRA\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_extra;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_extra);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_extra;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_extra);
-
-		return 0;
-	case METRIC_SET_ID_VME_PIPE:
-		dev_priv->perf.oa.n_mux_configs =
-			get_vme_pipe_mux_config(dev_priv,
-						dev_priv->perf.oa.mux_regs,
-						dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"VME_PIPE\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_vme_pipe;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_vme_pipe);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_vme_pipe;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_vme_pipe);
-
-		return 0;
-	case METRIC_SET_ID_TEST_OA:
-		dev_priv->perf.oa.n_mux_configs =
-			get_test_oa_mux_config(dev_priv,
-					       dev_priv->perf.oa.mux_regs,
-					       dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TEST_OA\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_test_oa;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_test_oa);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_test_oa;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_test_oa);
-
-		return 0;
-	default:
-		return -ENODEV;
-	}
-}
-
-static ssize_t
-show_render_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_BASIC);
-}
-
-static struct device_attribute dev_attr_render_basic_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_render_basic_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_render_basic[] = {
-	&dev_attr_render_basic_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_render_basic = {
-	.name = "0286c920-2f6d-493b-b22d-7a5280df43de",
-	.attrs =  attrs_render_basic,
-};
-
-static ssize_t
-show_compute_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_BASIC);
-}
-
-static struct device_attribute dev_attr_compute_basic_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_basic_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_basic[] = {
-	&dev_attr_compute_basic_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_basic = {
-	.name = "9823aaa1-b06f-40ce-884b-cd798c79f0c2",
-	.attrs =  attrs_compute_basic,
-};
-
-static ssize_t
-show_render_pipe_profile_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_PIPE_PROFILE);
-}
-
-static struct device_attribute dev_attr_render_pipe_profile_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_render_pipe_profile_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_render_pipe_profile[] = {
-	&dev_attr_render_pipe_profile_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_render_pipe_profile = {
-	.name = "c7c735f3-ce58-45cf-aa04-30b183f1faff",
-	.attrs =  attrs_render_pipe_profile,
-};
-
-static ssize_t
-show_memory_reads_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_READS);
-}
-
-static struct device_attribute dev_attr_memory_reads_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_memory_reads_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_memory_reads[] = {
-	&dev_attr_memory_reads_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_memory_reads = {
-	.name = "96ec2219-040b-428a-856a-6bc03363a057",
-	.attrs =  attrs_memory_reads,
-};
-
-static ssize_t
-show_memory_writes_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_WRITES);
-}
-
-static struct device_attribute dev_attr_memory_writes_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_memory_writes_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_memory_writes[] = {
-	&dev_attr_memory_writes_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_memory_writes = {
-	.name = "03372b64-4996-4d3b-aa18-790e75eeb9c2",
-	.attrs =  attrs_memory_writes,
-};
-
-static ssize_t
-show_compute_extended_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTENDED);
-}
-
-static struct device_attribute dev_attr_compute_extended_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_extended_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_extended[] = {
-	&dev_attr_compute_extended_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_extended = {
-	.name = "31b4ce5a-bd61-4c1f-bb5d-f2e731412150",
-	.attrs =  attrs_compute_extended,
-};
-
-static ssize_t
-show_compute_l3_cache_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_L3_CACHE);
-}
-
-static struct device_attribute dev_attr_compute_l3_cache_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_l3_cache_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_l3_cache[] = {
-	&dev_attr_compute_l3_cache_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_l3_cache = {
-	.name = "2ce0911a-27fc-4887-96f0-11084fa807c3",
-	.attrs =  attrs_compute_l3_cache,
-};
-
-static ssize_t
-show_hdc_and_sf_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_HDC_AND_SF);
-}
-
-static struct device_attribute dev_attr_hdc_and_sf_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_hdc_and_sf_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_hdc_and_sf[] = {
-	&dev_attr_hdc_and_sf_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_hdc_and_sf = {
-	.name = "546c4c1d-99b8-42fb-a107-5aaabb5314a8",
-	.attrs =  attrs_hdc_and_sf,
-};
-
-static ssize_t
-show_l3_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_1);
-}
-
-static struct device_attribute dev_attr_l3_1_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_l3_1_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_l3_1[] = {
-	&dev_attr_l3_1_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_l3_1 = {
-	.name = "4e93d156-9b39-4268-8544-a8e0480806d7",
-	.attrs =  attrs_l3_1,
-};
-
-static ssize_t
-show_l3_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_2);
-}
-
-static struct device_attribute dev_attr_l3_2_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_l3_2_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_l3_2[] = {
-	&dev_attr_l3_2_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_l3_2 = {
-	.name = "de1bec86-ca92-4b43-89fa-147653221cc0",
-	.attrs =  attrs_l3_2,
-};
-
-static ssize_t
-show_l3_3_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_3);
-}
-
-static struct device_attribute dev_attr_l3_3_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_l3_3_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_l3_3[] = {
-	&dev_attr_l3_3_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_l3_3 = {
-	.name = "e63537bb-10be-4d4a-92c4-c6b0c65e02ef",
-	.attrs =  attrs_l3_3,
-};
-
-static ssize_t
-show_rasterizer_and_pixel_backend_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND);
-}
-
-static struct device_attribute dev_attr_rasterizer_and_pixel_backend_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_rasterizer_and_pixel_backend_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_rasterizer_and_pixel_backend[] = {
-	&dev_attr_rasterizer_and_pixel_backend_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_rasterizer_and_pixel_backend = {
-	.name = "7a03a9f8-ec5e-46bb-8b67-1f0ff1476281",
-	.attrs =  attrs_rasterizer_and_pixel_backend,
-};
-
-static ssize_t
-show_sampler_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_SAMPLER);
-}
-
-static struct device_attribute dev_attr_sampler_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_sampler_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_sampler[] = {
-	&dev_attr_sampler_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_sampler = {
-	.name = "b25d2ebf-a6e0-4b29-96be-a9b010edeeda",
-	.attrs =  attrs_sampler,
-};
-
-static ssize_t
-show_tdl_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_1);
-}
-
-static struct device_attribute dev_attr_tdl_1_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_tdl_1_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_tdl_1[] = {
-	&dev_attr_tdl_1_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_tdl_1 = {
-	.name = "469a05e5-e299-46f7-9598-7b05f3c34991",
-	.attrs =  attrs_tdl_1,
-};
-
-static ssize_t
-show_tdl_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_2);
-}
-
-static struct device_attribute dev_attr_tdl_2_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_tdl_2_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_tdl_2[] = {
-	&dev_attr_tdl_2_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_tdl_2 = {
-	.name = "52f925c6-786a-4ec6-86ce-cba85c83453a",
-	.attrs =  attrs_tdl_2,
-};
-
-static ssize_t
-show_compute_extra_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTRA);
-}
-
-static struct device_attribute dev_attr_compute_extra_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_extra_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_extra[] = {
-	&dev_attr_compute_extra_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_extra = {
-	.name = "efc497ac-884e-4ee4-a4a8-15fba22aaf21",
-	.attrs =  attrs_compute_extra,
-};
-
-static ssize_t
-show_vme_pipe_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_VME_PIPE);
-}
-
-static struct device_attribute dev_attr_vme_pipe_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_vme_pipe_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_vme_pipe[] = {
-	&dev_attr_vme_pipe_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_vme_pipe = {
-	.name = "bfd9764d-2c5b-4c16-bfc1-89de3ca10917",
-	.attrs =  attrs_vme_pipe,
-};
-
 static ssize_t
 show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TEST_OA);
+	return sprintf(buf, "1\n");
 }
 
-static struct device_attribute dev_attr_test_oa_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_test_oa_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_test_oa[] = {
-	&dev_attr_test_oa_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_test_oa = {
-	.name = "f1792f32-6db2-4b50-b4b2-557128f1688d",
-	.attrs =  attrs_test_oa,
-};
-
-int
-i915_perf_register_sysfs_kblgt3(struct drm_i915_private *dev_priv)
+void
+i915_perf_load_test_config_kblgt3(struct drm_i915_private *dev_priv)
 {
-	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
-	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
-	int ret = 0;
+	strncpy(dev_priv->perf.oa.test_config.uuid,
+		"f1792f32-6db2-4b50-b4b2-557128f1688d",
+		UUID_STRING_LEN);
+	dev_priv->perf.oa.test_config.id = 1;
 
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-		if (ret)
-			goto error_render_basic;
-	}
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-		if (ret)
-			goto error_compute_basic;
-	}
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-		if (ret)
-			goto error_render_pipe_profile;
-	}
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-		if (ret)
-			goto error_memory_reads;
-	}
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-		if (ret)
-			goto error_memory_writes;
-	}
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-		if (ret)
-			goto error_compute_extended;
-	}
-	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
-		if (ret)
-			goto error_compute_l3_cache;
-	}
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-		if (ret)
-			goto error_hdc_and_sf;
-	}
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-		if (ret)
-			goto error_l3_1;
-	}
-	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_2);
-		if (ret)
-			goto error_l3_2;
-	}
-	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_3);
-		if (ret)
-			goto error_l3_3;
-	}
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-		if (ret)
-			goto error_rasterizer_and_pixel_backend;
-	}
-	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_sampler);
-		if (ret)
-			goto error_sampler;
-	}
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-		if (ret)
-			goto error_tdl_1;
-	}
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-		if (ret)
-			goto error_tdl_2;
-	}
-	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
-		if (ret)
-			goto error_compute_extra;
-	}
-	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
-		if (ret)
-			goto error_vme_pipe;
-	}
-	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_test_oa);
-		if (ret)
-			goto error_test_oa;
-	}
+	dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
+	dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa);
 
-	return 0;
+	dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa;
+	dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa);
 
-error_test_oa:
-	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
-error_vme_pipe:
-	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
-error_compute_extra:
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-error_tdl_2:
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-error_tdl_1:
-	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
-error_sampler:
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-error_rasterizer_and_pixel_backend:
-	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
-error_l3_3:
-	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
-error_l3_2:
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-error_l3_1:
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-error_hdc_and_sf:
-	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
-error_compute_l3_cache:
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-error_compute_extended:
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-error_memory_writes:
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-error_memory_reads:
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-error_render_pipe_profile:
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-error_compute_basic:
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-error_render_basic:
-	return ret;
-}
+	dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa;
+	dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa);
 
-void
-i915_perf_unregister_sysfs_kblgt3(struct drm_i915_private *dev_priv)
-{
-	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
-	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+	dev_priv->perf.oa.test_config.sysfs_metric.name = "f1792f32-6db2-4b50-b4b2-557128f1688d";
+	dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs;
+
+	dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr;
 
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
-	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
-	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
-	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id";
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444;
+	dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id;
 }
diff --git a/drivers/gpu/drm/i915/i915_oa_kblgt3.h b/drivers/gpu/drm/i915/i915_oa_kblgt3.h
index b0ca7f3114d3..d5b5b5c1923e 100644
--- a/drivers/gpu/drm/i915/i915_oa_kblgt3.h
+++ b/drivers/gpu/drm/i915/i915_oa_kblgt3.h
@@ -29,12 +29,6 @@
 #ifndef __I915_OA_KBLGT3_H__
 #define __I915_OA_KBLGT3_H__
 
-extern int i915_oa_n_builtin_metric_sets_kblgt3;
-
-extern int i915_oa_select_metric_set_kblgt3(struct drm_i915_private *dev_priv);
-
-extern int i915_perf_register_sysfs_kblgt3(struct drm_i915_private *dev_priv);
-
-extern void i915_perf_unregister_sysfs_kblgt3(struct drm_i915_private *dev_priv);
+extern void i915_perf_load_test_config_kblgt3(struct drm_i915_private *dev_priv);
 
 #endif
diff --git a/drivers/gpu/drm/i915/i915_oa_sklgt2.c b/drivers/gpu/drm/i915/i915_oa_sklgt2.c
index 1268beda212c..890d55879946 100644
--- a/drivers/gpu/drm/i915/i915_oa_sklgt2.c
+++ b/drivers/gpu/drm/i915/i915_oa_sklgt2.c
@@ -31,2317 +31,6 @@
 #include "i915_drv.h"
 #include "i915_oa_sklgt2.h"
 
-enum metric_set_id {
-	METRIC_SET_ID_RENDER_BASIC = 1,
-	METRIC_SET_ID_COMPUTE_BASIC,
-	METRIC_SET_ID_RENDER_PIPE_PROFILE,
-	METRIC_SET_ID_MEMORY_READS,
-	METRIC_SET_ID_MEMORY_WRITES,
-	METRIC_SET_ID_COMPUTE_EXTENDED,
-	METRIC_SET_ID_COMPUTE_L3_CACHE,
-	METRIC_SET_ID_HDC_AND_SF,
-	METRIC_SET_ID_L3_1,
-	METRIC_SET_ID_L3_2,
-	METRIC_SET_ID_L3_3,
-	METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND,
-	METRIC_SET_ID_SAMPLER,
-	METRIC_SET_ID_TDL_1,
-	METRIC_SET_ID_TDL_2,
-	METRIC_SET_ID_COMPUTE_EXTRA,
-	METRIC_SET_ID_VME_PIPE,
-	METRIC_SET_ID_TEST_OA,
-};
-
-int i915_oa_n_builtin_metric_sets_sklgt2 = 18;
-
-static const struct i915_oa_reg b_counter_config_render_basic[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2740), 0x00000000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_render_basic[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_render_basic_1_sku_gte_0x02[] = {
-	{ _MMIO(0x9888), 0x166c01e0 },
-	{ _MMIO(0x9888), 0x12170280 },
-	{ _MMIO(0x9888), 0x12370280 },
-	{ _MMIO(0x9888), 0x11930317 },
-	{ _MMIO(0x9888), 0x159303df },
-	{ _MMIO(0x9888), 0x3f900003 },
-	{ _MMIO(0x9888), 0x1a4e0080 },
-	{ _MMIO(0x9888), 0x0a6c0053 },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x0a1b4000 },
-	{ _MMIO(0x9888), 0x1c1c0001 },
-	{ _MMIO(0x9888), 0x002f1000 },
-	{ _MMIO(0x9888), 0x042f1000 },
-	{ _MMIO(0x9888), 0x004c4000 },
-	{ _MMIO(0x9888), 0x0a4c8400 },
-	{ _MMIO(0x9888), 0x000d2000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0a0d2000 },
-	{ _MMIO(0x9888), 0x0c0f0400 },
-	{ _MMIO(0x9888), 0x0e0f6600 },
-	{ _MMIO(0x9888), 0x002c8000 },
-	{ _MMIO(0x9888), 0x162c2200 },
-	{ _MMIO(0x9888), 0x062d8000 },
-	{ _MMIO(0x9888), 0x082d8000 },
-	{ _MMIO(0x9888), 0x00133000 },
-	{ _MMIO(0x9888), 0x08133000 },
-	{ _MMIO(0x9888), 0x00170020 },
-	{ _MMIO(0x9888), 0x08170021 },
-	{ _MMIO(0x9888), 0x10170000 },
-	{ _MMIO(0x9888), 0x0633c000 },
-	{ _MMIO(0x9888), 0x0833c000 },
-	{ _MMIO(0x9888), 0x06370800 },
-	{ _MMIO(0x9888), 0x08370840 },
-	{ _MMIO(0x9888), 0x10370000 },
-	{ _MMIO(0x9888), 0x0d933031 },
-	{ _MMIO(0x9888), 0x0f933e3f },
-	{ _MMIO(0x9888), 0x01933d00 },
-	{ _MMIO(0x9888), 0x0393073c },
-	{ _MMIO(0x9888), 0x0593000e },
-	{ _MMIO(0x9888), 0x1d930000 },
-	{ _MMIO(0x9888), 0x19930000 },
-	{ _MMIO(0x9888), 0x1b930000 },
-	{ _MMIO(0x9888), 0x1d900157 },
-	{ _MMIO(0x9888), 0x1f900158 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x2b908000 },
-	{ _MMIO(0x9888), 0x2d908000 },
-	{ _MMIO(0x9888), 0x2f908000 },
-	{ _MMIO(0x9888), 0x31908000 },
-	{ _MMIO(0x9888), 0x15908000 },
-	{ _MMIO(0x9888), 0x17908000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1190001f },
-	{ _MMIO(0x9888), 0x51904400 },
-	{ _MMIO(0x9888), 0x41900020 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900c21 },
-	{ _MMIO(0x9888), 0x47900061 },
-	{ _MMIO(0x9888), 0x57904440 },
-	{ _MMIO(0x9888), 0x49900000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900000 },
-	{ _MMIO(0x9888), 0x59900004 },
-	{ _MMIO(0x9888), 0x43900000 },
-	{ _MMIO(0x9888), 0x53904444 },
-};
-
-static int
-get_render_basic_mux_config(struct drm_i915_private *dev_priv,
-			    const struct i915_oa_reg **regs,
-			    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	if (dev_priv->drm.pdev->revision >= 0x02) {
-		regs[n] = mux_config_render_basic_1_sku_gte_0x02;
-		lens[n] = ARRAY_SIZE(mux_config_render_basic_1_sku_gte_0x02);
-		n++;
-	}
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_basic[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2740), 0x00000000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_basic[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00778008 },
-	{ _MMIO(0xe45c), 0x00088078 },
-	{ _MMIO(0xe55c), 0x00808708 },
-	{ _MMIO(0xe65c), 0x00a08908 },
-};
-
-static const struct i915_oa_reg mux_config_compute_basic_0_slices_0x01_and_sku_lt_0x02[] = {
-	{ _MMIO(0x9888), 0x104f00e0 },
-	{ _MMIO(0x9888), 0x124f1c00 },
-	{ _MMIO(0x9888), 0x106c00e0 },
-	{ _MMIO(0x9888), 0x37906800 },
-	{ _MMIO(0x9888), 0x3f901403 },
-	{ _MMIO(0x9888), 0x184e8000 },
-	{ _MMIO(0x9888), 0x1a4e8200 },
-	{ _MMIO(0x9888), 0x044e8000 },
-	{ _MMIO(0x9888), 0x004f0db2 },
-	{ _MMIO(0x9888), 0x064f0900 },
-	{ _MMIO(0x9888), 0x084f1880 },
-	{ _MMIO(0x9888), 0x0a4f0011 },
-	{ _MMIO(0x9888), 0x0c4f0e3c },
-	{ _MMIO(0x9888), 0x0e4f1d80 },
-	{ _MMIO(0x9888), 0x086c0002 },
-	{ _MMIO(0x9888), 0x0a6c0100 },
-	{ _MMIO(0x9888), 0x0e6c000c },
-	{ _MMIO(0x9888), 0x026c000b },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1a6c0000 },
-	{ _MMIO(0x9888), 0x081b4000 },
-	{ _MMIO(0x9888), 0x0a1b8000 },
-	{ _MMIO(0x9888), 0x0e1b4000 },
-	{ _MMIO(0x9888), 0x021b4000 },
-	{ _MMIO(0x9888), 0x1a1c4000 },
-	{ _MMIO(0x9888), 0x1c1c0012 },
-	{ _MMIO(0x9888), 0x141c8000 },
-	{ _MMIO(0x9888), 0x005bc000 },
-	{ _MMIO(0x9888), 0x065b8000 },
-	{ _MMIO(0x9888), 0x085b8000 },
-	{ _MMIO(0x9888), 0x0a5b4000 },
-	{ _MMIO(0x9888), 0x0c5bc000 },
-	{ _MMIO(0x9888), 0x0e5b8000 },
-	{ _MMIO(0x9888), 0x105c8000 },
-	{ _MMIO(0x9888), 0x1a5ca000 },
-	{ _MMIO(0x9888), 0x1c5c002d },
-	{ _MMIO(0x9888), 0x125c8000 },
-	{ _MMIO(0x9888), 0x0a4c0800 },
-	{ _MMIO(0x9888), 0x0c4c0082 },
-	{ _MMIO(0x9888), 0x084c8000 },
-	{ _MMIO(0x9888), 0x000da000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0a0da000 },
-	{ _MMIO(0x9888), 0x0c0da000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x020d2000 },
-	{ _MMIO(0x9888), 0x0c0f5400 },
-	{ _MMIO(0x9888), 0x0e0f5500 },
-	{ _MMIO(0x9888), 0x100f0155 },
-	{ _MMIO(0x9888), 0x002cc000 },
-	{ _MMIO(0x9888), 0x0e2cc000 },
-	{ _MMIO(0x9888), 0x162cbe00 },
-	{ _MMIO(0x9888), 0x182c00ef },
-	{ _MMIO(0x9888), 0x022cc000 },
-	{ _MMIO(0x9888), 0x042c8000 },
-	{ _MMIO(0x9888), 0x19900157 },
-	{ _MMIO(0x9888), 0x1b900167 },
-	{ _MMIO(0x9888), 0x1d900105 },
-	{ _MMIO(0x9888), 0x1f900103 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0xd28), 0x00000000 },
-	{ _MMIO(0x9888), 0x11900fff },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900840 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900842 },
-	{ _MMIO(0x9888), 0x47900840 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900840 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900040 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900840 },
-	{ _MMIO(0x9888), 0x53901111 },
-};
-
-static const struct i915_oa_reg mux_config_compute_basic_0_slices_0x01_and_sku_gte_0x02[] = {
-	{ _MMIO(0x9888), 0x104f00e0 },
-	{ _MMIO(0x9888), 0x124f1c00 },
-	{ _MMIO(0x9888), 0x106c00e0 },
-	{ _MMIO(0x9888), 0x37906800 },
-	{ _MMIO(0x9888), 0x3f901403 },
-	{ _MMIO(0x9888), 0x004e8000 },
-	{ _MMIO(0x9888), 0x1a4e0820 },
-	{ _MMIO(0x9888), 0x1c4e0002 },
-	{ _MMIO(0x9888), 0x064f0900 },
-	{ _MMIO(0x9888), 0x084f0032 },
-	{ _MMIO(0x9888), 0x0a4f1810 },
-	{ _MMIO(0x9888), 0x0c4f0e00 },
-	{ _MMIO(0x9888), 0x0e4f003c },
-	{ _MMIO(0x9888), 0x004f0d80 },
-	{ _MMIO(0x9888), 0x024f003b },
-	{ _MMIO(0x9888), 0x006c0002 },
-	{ _MMIO(0x9888), 0x086c0000 },
-	{ _MMIO(0x9888), 0x0c6c000c },
-	{ _MMIO(0x9888), 0x0e6c0b00 },
-	{ _MMIO(0x9888), 0x186c0000 },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1e6c0000 },
-	{ _MMIO(0x9888), 0x001b4000 },
-	{ _MMIO(0x9888), 0x081b8000 },
-	{ _MMIO(0x9888), 0x0c1b4000 },
-	{ _MMIO(0x9888), 0x0e1b8000 },
-	{ _MMIO(0x9888), 0x101c8000 },
-	{ _MMIO(0x9888), 0x1a1c8000 },
-	{ _MMIO(0x9888), 0x1c1c0024 },
-	{ _MMIO(0x9888), 0x065b8000 },
-	{ _MMIO(0x9888), 0x085b4000 },
-	{ _MMIO(0x9888), 0x0a5bc000 },
-	{ _MMIO(0x9888), 0x0c5b8000 },
-	{ _MMIO(0x9888), 0x0e5b4000 },
-	{ _MMIO(0x9888), 0x005b8000 },
-	{ _MMIO(0x9888), 0x025b4000 },
-	{ _MMIO(0x9888), 0x1a5c6000 },
-	{ _MMIO(0x9888), 0x1c5c001b },
-	{ _MMIO(0x9888), 0x125c8000 },
-	{ _MMIO(0x9888), 0x145c8000 },
-	{ _MMIO(0x9888), 0x004c8000 },
-	{ _MMIO(0x9888), 0x0a4c2000 },
-	{ _MMIO(0x9888), 0x0c4c0208 },
-	{ _MMIO(0x9888), 0x000da000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0a0da000 },
-	{ _MMIO(0x9888), 0x0c0da000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x020d2000 },
-	{ _MMIO(0x9888), 0x0c0f5400 },
-	{ _MMIO(0x9888), 0x0e0f5500 },
-	{ _MMIO(0x9888), 0x100f0155 },
-	{ _MMIO(0x9888), 0x002c8000 },
-	{ _MMIO(0x9888), 0x0e2cc000 },
-	{ _MMIO(0x9888), 0x162cfb00 },
-	{ _MMIO(0x9888), 0x182c00be },
-	{ _MMIO(0x9888), 0x022cc000 },
-	{ _MMIO(0x9888), 0x042cc000 },
-	{ _MMIO(0x9888), 0x19900157 },
-	{ _MMIO(0x9888), 0x1b900167 },
-	{ _MMIO(0x9888), 0x1d900105 },
-	{ _MMIO(0x9888), 0x1f900103 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x11900fff },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900800 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900842 },
-	{ _MMIO(0x9888), 0x47900802 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900802 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900002 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900842 },
-	{ _MMIO(0x9888), 0x53901111 },
-};
-
-static int
-get_compute_basic_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 2);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 2);
-
-	if ((INTEL_INFO(dev_priv)->sseu.slice_mask & 0x01) &&
-	    (dev_priv->drm.pdev->revision < 0x02)) {
-		regs[n] = mux_config_compute_basic_0_slices_0x01_and_sku_lt_0x02;
-		lens[n] = ARRAY_SIZE(mux_config_compute_basic_0_slices_0x01_and_sku_lt_0x02);
-		n++;
-	}
-	if ((INTEL_INFO(dev_priv)->sseu.slice_mask & 0x01) &&
-		   (dev_priv->drm.pdev->revision >= 0x02)) {
-		regs[n] = mux_config_compute_basic_0_slices_0x01_and_sku_gte_0x02;
-		lens[n] = ARRAY_SIZE(mux_config_compute_basic_0_slices_0x01_and_sku_gte_0x02);
-		n++;
-	}
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_render_pipe_profile[] = {
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007ffea },
-	{ _MMIO(0x2774), 0x00007ffc },
-	{ _MMIO(0x2778), 0x0007affa },
-	{ _MMIO(0x277c), 0x0000f5fd },
-	{ _MMIO(0x2780), 0x00079ffa },
-	{ _MMIO(0x2784), 0x0000f3fb },
-	{ _MMIO(0x2788), 0x0007bf7a },
-	{ _MMIO(0x278c), 0x0000f7e7 },
-	{ _MMIO(0x2790), 0x0007fefa },
-	{ _MMIO(0x2794), 0x0000f7cf },
-	{ _MMIO(0x2798), 0x00077ffa },
-	{ _MMIO(0x279c), 0x0000efdf },
-	{ _MMIO(0x27a0), 0x0006fffa },
-	{ _MMIO(0x27a4), 0x0000cfbf },
-	{ _MMIO(0x27a8), 0x0003fffa },
-	{ _MMIO(0x27ac), 0x00005f7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_render_pipe_profile[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00015014 },
-	{ _MMIO(0xe658), 0x00025024 },
-	{ _MMIO(0xe758), 0x00035034 },
-	{ _MMIO(0xe45c), 0x00045044 },
-	{ _MMIO(0xe55c), 0x00055054 },
-	{ _MMIO(0xe65c), 0x00065064 },
-};
-
-static const struct i915_oa_reg mux_config_render_pipe_profile_0_sku_lt_0x02[] = {
-	{ _MMIO(0x9888), 0x0c0e001f },
-	{ _MMIO(0x9888), 0x0a0f0000 },
-	{ _MMIO(0x9888), 0x10116800 },
-	{ _MMIO(0x9888), 0x178a03e0 },
-	{ _MMIO(0x9888), 0x11824c00 },
-	{ _MMIO(0x9888), 0x11830020 },
-	{ _MMIO(0x9888), 0x13840020 },
-	{ _MMIO(0x9888), 0x11850019 },
-	{ _MMIO(0x9888), 0x11860007 },
-	{ _MMIO(0x9888), 0x01870c40 },
-	{ _MMIO(0x9888), 0x17880000 },
-	{ _MMIO(0x9888), 0x022f4000 },
-	{ _MMIO(0x9888), 0x0a4c0040 },
-	{ _MMIO(0x9888), 0x0c0d8000 },
-	{ _MMIO(0x9888), 0x040d4000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x020e5400 },
-	{ _MMIO(0x9888), 0x000e0000 },
-	{ _MMIO(0x9888), 0x080f0040 },
-	{ _MMIO(0x9888), 0x000f0000 },
-	{ _MMIO(0x9888), 0x100f0000 },
-	{ _MMIO(0x9888), 0x0e0f0040 },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x06104000 },
-	{ _MMIO(0x9888), 0x06110012 },
-	{ _MMIO(0x9888), 0x06131000 },
-	{ _MMIO(0x9888), 0x01898000 },
-	{ _MMIO(0x9888), 0x0d890100 },
-	{ _MMIO(0x9888), 0x03898000 },
-	{ _MMIO(0x9888), 0x09808000 },
-	{ _MMIO(0x9888), 0x0b808000 },
-	{ _MMIO(0x9888), 0x0380c000 },
-	{ _MMIO(0x9888), 0x0f8a0075 },
-	{ _MMIO(0x9888), 0x1d8a0000 },
-	{ _MMIO(0x9888), 0x118a8000 },
-	{ _MMIO(0x9888), 0x1b8a4000 },
-	{ _MMIO(0x9888), 0x138a8000 },
-	{ _MMIO(0x9888), 0x1d81a000 },
-	{ _MMIO(0x9888), 0x15818000 },
-	{ _MMIO(0x9888), 0x17818000 },
-	{ _MMIO(0x9888), 0x0b820030 },
-	{ _MMIO(0x9888), 0x07828000 },
-	{ _MMIO(0x9888), 0x0d824000 },
-	{ _MMIO(0x9888), 0x0f828000 },
-	{ _MMIO(0x9888), 0x05824000 },
-	{ _MMIO(0x9888), 0x0d830003 },
-	{ _MMIO(0x9888), 0x0583000c },
-	{ _MMIO(0x9888), 0x09830000 },
-	{ _MMIO(0x9888), 0x03838000 },
-	{ _MMIO(0x9888), 0x07838000 },
-	{ _MMIO(0x9888), 0x0b840980 },
-	{ _MMIO(0x9888), 0x03844d80 },
-	{ _MMIO(0x9888), 0x11840000 },
-	{ _MMIO(0x9888), 0x09848000 },
-	{ _MMIO(0x9888), 0x09850080 },
-	{ _MMIO(0x9888), 0x03850003 },
-	{ _MMIO(0x9888), 0x01850000 },
-	{ _MMIO(0x9888), 0x07860000 },
-	{ _MMIO(0x9888), 0x0f860400 },
-	{ _MMIO(0x9888), 0x09870032 },
-	{ _MMIO(0x9888), 0x01888052 },
-	{ _MMIO(0x9888), 0x11880000 },
-	{ _MMIO(0x9888), 0x09884000 },
-	{ _MMIO(0x9888), 0x15968000 },
-	{ _MMIO(0x9888), 0x17968000 },
-	{ _MMIO(0x9888), 0x0f96c000 },
-	{ _MMIO(0x9888), 0x1f950011 },
-	{ _MMIO(0x9888), 0x1d950014 },
-	{ _MMIO(0x9888), 0x0592c000 },
-	{ _MMIO(0x9888), 0x0b928000 },
-	{ _MMIO(0x9888), 0x0d924000 },
-	{ _MMIO(0x9888), 0x0f924000 },
-	{ _MMIO(0x9888), 0x11928000 },
-	{ _MMIO(0x9888), 0x1392c000 },
-	{ _MMIO(0x9888), 0x09924000 },
-	{ _MMIO(0x9888), 0x01985000 },
-	{ _MMIO(0x9888), 0x07988000 },
-	{ _MMIO(0x9888), 0x09981000 },
-	{ _MMIO(0x9888), 0x0b982000 },
-	{ _MMIO(0x9888), 0x0d982000 },
-	{ _MMIO(0x9888), 0x0f989000 },
-	{ _MMIO(0x9888), 0x05982000 },
-	{ _MMIO(0x9888), 0x13904000 },
-	{ _MMIO(0x9888), 0x21904000 },
-	{ _MMIO(0x9888), 0x23904000 },
-	{ _MMIO(0x9888), 0x25908000 },
-	{ _MMIO(0x9888), 0x27904000 },
-	{ _MMIO(0x9888), 0x29908000 },
-	{ _MMIO(0x9888), 0x2b904000 },
-	{ _MMIO(0x9888), 0x2f904000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x15904000 },
-	{ _MMIO(0x9888), 0x17908000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x0b978000 },
-	{ _MMIO(0x9888), 0x0f974000 },
-	{ _MMIO(0x9888), 0x11974000 },
-	{ _MMIO(0x9888), 0x13978000 },
-	{ _MMIO(0x9888), 0x09974000 },
-	{ _MMIO(0xd28), 0x00000000 },
-	{ _MMIO(0x9888), 0x1190c080 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x419010a0 },
-	{ _MMIO(0x9888), 0x55904000 },
-	{ _MMIO(0x9888), 0x45901000 },
-	{ _MMIO(0x9888), 0x47900084 },
-	{ _MMIO(0x9888), 0x57904400 },
-	{ _MMIO(0x9888), 0x499000a5 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900081 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x439014a4 },
-	{ _MMIO(0x9888), 0x53900400 },
-};
-
-static const struct i915_oa_reg mux_config_render_pipe_profile_0_sku_gte_0x02[] = {
-	{ _MMIO(0x9888), 0x0c0e001f },
-	{ _MMIO(0x9888), 0x0a0f0000 },
-	{ _MMIO(0x9888), 0x10116800 },
-	{ _MMIO(0x9888), 0x178a03e0 },
-	{ _MMIO(0x9888), 0x11824c00 },
-	{ _MMIO(0x9888), 0x11830020 },
-	{ _MMIO(0x9888), 0x13840020 },
-	{ _MMIO(0x9888), 0x11850019 },
-	{ _MMIO(0x9888), 0x11860007 },
-	{ _MMIO(0x9888), 0x01870c40 },
-	{ _MMIO(0x9888), 0x17880000 },
-	{ _MMIO(0x9888), 0x022f4000 },
-	{ _MMIO(0x9888), 0x0a4c0040 },
-	{ _MMIO(0x9888), 0x0c0d8000 },
-	{ _MMIO(0x9888), 0x040d4000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x020e5400 },
-	{ _MMIO(0x9888), 0x000e0000 },
-	{ _MMIO(0x9888), 0x080f0040 },
-	{ _MMIO(0x9888), 0x000f0000 },
-	{ _MMIO(0x9888), 0x100f0000 },
-	{ _MMIO(0x9888), 0x0e0f0040 },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x06104000 },
-	{ _MMIO(0x9888), 0x06110012 },
-	{ _MMIO(0x9888), 0x06131000 },
-	{ _MMIO(0x9888), 0x01898000 },
-	{ _MMIO(0x9888), 0x0d890100 },
-	{ _MMIO(0x9888), 0x03898000 },
-	{ _MMIO(0x9888), 0x09808000 },
-	{ _MMIO(0x9888), 0x0b808000 },
-	{ _MMIO(0x9888), 0x0380c000 },
-	{ _MMIO(0x9888), 0x0f8a0075 },
-	{ _MMIO(0x9888), 0x1d8a0000 },
-	{ _MMIO(0x9888), 0x118a8000 },
-	{ _MMIO(0x9888), 0x1b8a4000 },
-	{ _MMIO(0x9888), 0x138a8000 },
-	{ _MMIO(0x9888), 0x1d81a000 },
-	{ _MMIO(0x9888), 0x15818000 },
-	{ _MMIO(0x9888), 0x17818000 },
-	{ _MMIO(0x9888), 0x0b820030 },
-	{ _MMIO(0x9888), 0x07828000 },
-	{ _MMIO(0x9888), 0x0d824000 },
-	{ _MMIO(0x9888), 0x0f828000 },
-	{ _MMIO(0x9888), 0x05824000 },
-	{ _MMIO(0x9888), 0x0d830003 },
-	{ _MMIO(0x9888), 0x0583000c },
-	{ _MMIO(0x9888), 0x09830000 },
-	{ _MMIO(0x9888), 0x03838000 },
-	{ _MMIO(0x9888), 0x07838000 },
-	{ _MMIO(0x9888), 0x0b840980 },
-	{ _MMIO(0x9888), 0x03844d80 },
-	{ _MMIO(0x9888), 0x11840000 },
-	{ _MMIO(0x9888), 0x09848000 },
-	{ _MMIO(0x9888), 0x09850080 },
-	{ _MMIO(0x9888), 0x03850003 },
-	{ _MMIO(0x9888), 0x01850000 },
-	{ _MMIO(0x9888), 0x07860000 },
-	{ _MMIO(0x9888), 0x0f860400 },
-	{ _MMIO(0x9888), 0x09870032 },
-	{ _MMIO(0x9888), 0x01888052 },
-	{ _MMIO(0x9888), 0x11880000 },
-	{ _MMIO(0x9888), 0x09884000 },
-	{ _MMIO(0x9888), 0x1b931001 },
-	{ _MMIO(0x9888), 0x1d930001 },
-	{ _MMIO(0x9888), 0x19934000 },
-	{ _MMIO(0x9888), 0x1b958000 },
-	{ _MMIO(0x9888), 0x1d950094 },
-	{ _MMIO(0x9888), 0x19958000 },
-	{ _MMIO(0x9888), 0x05e5a000 },
-	{ _MMIO(0x9888), 0x01e5c000 },
-	{ _MMIO(0x9888), 0x0592c000 },
-	{ _MMIO(0x9888), 0x0b928000 },
-	{ _MMIO(0x9888), 0x0d924000 },
-	{ _MMIO(0x9888), 0x0f924000 },
-	{ _MMIO(0x9888), 0x11928000 },
-	{ _MMIO(0x9888), 0x1392c000 },
-	{ _MMIO(0x9888), 0x09924000 },
-	{ _MMIO(0x9888), 0x01985000 },
-	{ _MMIO(0x9888), 0x07988000 },
-	{ _MMIO(0x9888), 0x09981000 },
-	{ _MMIO(0x9888), 0x0b982000 },
-	{ _MMIO(0x9888), 0x0d982000 },
-	{ _MMIO(0x9888), 0x0f989000 },
-	{ _MMIO(0x9888), 0x05982000 },
-	{ _MMIO(0x9888), 0x13904000 },
-	{ _MMIO(0x9888), 0x21904000 },
-	{ _MMIO(0x9888), 0x23904000 },
-	{ _MMIO(0x9888), 0x25908000 },
-	{ _MMIO(0x9888), 0x27904000 },
-	{ _MMIO(0x9888), 0x29908000 },
-	{ _MMIO(0x9888), 0x2b904000 },
-	{ _MMIO(0x9888), 0x2f904000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x15904000 },
-	{ _MMIO(0x9888), 0x17908000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x1190c080 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x419010a0 },
-	{ _MMIO(0x9888), 0x55904000 },
-	{ _MMIO(0x9888), 0x45901000 },
-	{ _MMIO(0x9888), 0x47900084 },
-	{ _MMIO(0x9888), 0x57904400 },
-	{ _MMIO(0x9888), 0x499000a5 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900081 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x439014a4 },
-	{ _MMIO(0x9888), 0x53900400 },
-};
-
-static int
-get_render_pipe_profile_mux_config(struct drm_i915_private *dev_priv,
-				   const struct i915_oa_reg **regs,
-				   int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 2);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 2);
-
-	if (dev_priv->drm.pdev->revision < 0x02) {
-		regs[n] = mux_config_render_pipe_profile_0_sku_lt_0x02;
-		lens[n] = ARRAY_SIZE(mux_config_render_pipe_profile_0_sku_lt_0x02);
-		n++;
-	}
-	if (dev_priv->drm.pdev->revision >= 0x02) {
-		regs[n] = mux_config_render_pipe_profile_0_sku_gte_0x02;
-		lens[n] = ARRAY_SIZE(mux_config_render_pipe_profile_0_sku_gte_0x02);
-		n++;
-	}
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_memory_reads[] = {
-	{ _MMIO(0x272c), 0xffffffff },
-	{ _MMIO(0x2728), 0xffffffff },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x271c), 0xffffffff },
-	{ _MMIO(0x2718), 0xffffffff },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x274c), 0x86543210 },
-	{ _MMIO(0x2748), 0x86543210 },
-	{ _MMIO(0x2744), 0x00006667 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x275c), 0x86543210 },
-	{ _MMIO(0x2758), 0x86543210 },
-	{ _MMIO(0x2754), 0x00006465 },
-	{ _MMIO(0x2750), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007f81a },
-	{ _MMIO(0x2774), 0x0000fe00 },
-	{ _MMIO(0x2778), 0x0007f82a },
-	{ _MMIO(0x277c), 0x0000fe00 },
-	{ _MMIO(0x2780), 0x0007f872 },
-	{ _MMIO(0x2784), 0x0000fe00 },
-	{ _MMIO(0x2788), 0x0007f8ba },
-	{ _MMIO(0x278c), 0x0000fe00 },
-	{ _MMIO(0x2790), 0x0007f87a },
-	{ _MMIO(0x2794), 0x0000fe00 },
-	{ _MMIO(0x2798), 0x0007f8ea },
-	{ _MMIO(0x279c), 0x0000fe00 },
-	{ _MMIO(0x27a0), 0x0007f8e2 },
-	{ _MMIO(0x27a4), 0x0000fe00 },
-	{ _MMIO(0x27a8), 0x0007f8f2 },
-	{ _MMIO(0x27ac), 0x0000fe00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_memory_reads[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00015014 },
-	{ _MMIO(0xe658), 0x00025024 },
-	{ _MMIO(0xe758), 0x00035034 },
-	{ _MMIO(0xe45c), 0x00045044 },
-	{ _MMIO(0xe55c), 0x00055054 },
-	{ _MMIO(0xe65c), 0x00065064 },
-};
-
-static const struct i915_oa_reg mux_config_memory_reads_0_slices_0x01_and_sku_lt_0x02[] = {
-	{ _MMIO(0x9888), 0x11810c00 },
-	{ _MMIO(0x9888), 0x1381001a },
-	{ _MMIO(0x9888), 0x13946000 },
-	{ _MMIO(0x9888), 0x37906800 },
-	{ _MMIO(0x9888), 0x3f900003 },
-	{ _MMIO(0x9888), 0x03811300 },
-	{ _MMIO(0x9888), 0x05811b12 },
-	{ _MMIO(0x9888), 0x0781001a },
-	{ _MMIO(0x9888), 0x1f810000 },
-	{ _MMIO(0x9888), 0x17810000 },
-	{ _MMIO(0x9888), 0x19810000 },
-	{ _MMIO(0x9888), 0x1b810000 },
-	{ _MMIO(0x9888), 0x1d810000 },
-	{ _MMIO(0x9888), 0x0f968000 },
-	{ _MMIO(0x9888), 0x1196c000 },
-	{ _MMIO(0x9888), 0x13964000 },
-	{ _MMIO(0x9888), 0x11938000 },
-	{ _MMIO(0x9888), 0x1b93fe00 },
-	{ _MMIO(0x9888), 0x01940010 },
-	{ _MMIO(0x9888), 0x07941100 },
-	{ _MMIO(0x9888), 0x09941312 },
-	{ _MMIO(0x9888), 0x0b941514 },
-	{ _MMIO(0x9888), 0x0d941716 },
-	{ _MMIO(0x9888), 0x11940000 },
-	{ _MMIO(0x9888), 0x19940000 },
-	{ _MMIO(0x9888), 0x1b940000 },
-	{ _MMIO(0x9888), 0x1d940000 },
-	{ _MMIO(0x9888), 0x1b954000 },
-	{ _MMIO(0x9888), 0x1d95a550 },
-	{ _MMIO(0x9888), 0x1f9502aa },
-	{ _MMIO(0x9888), 0x2f900157 },
-	{ _MMIO(0x9888), 0x31900105 },
-	{ _MMIO(0x9888), 0x15900103 },
-	{ _MMIO(0x9888), 0x17900101 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x13908000 },
-	{ _MMIO(0x9888), 0x21908000 },
-	{ _MMIO(0x9888), 0x23908000 },
-	{ _MMIO(0x9888), 0x25908000 },
-	{ _MMIO(0x9888), 0x27908000 },
-	{ _MMIO(0x9888), 0x29908000 },
-	{ _MMIO(0x9888), 0x2b908000 },
-	{ _MMIO(0x9888), 0x2d908000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1d908000 },
-	{ _MMIO(0x9888), 0x1f908000 },
-	{ _MMIO(0xd28), 0x00000000 },
-	{ _MMIO(0x9888), 0x11900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c00 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900000 },
-	{ _MMIO(0x9888), 0x47900000 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900063 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900003 },
-	{ _MMIO(0x9888), 0x53900000 },
-};
-
-static const struct i915_oa_reg mux_config_memory_reads_0_sku_lt_0x05_and_sku_gte_0x02[] = {
-	{ _MMIO(0x9888), 0x11810c00 },
-	{ _MMIO(0x9888), 0x1381001a },
-	{ _MMIO(0x9888), 0x13946000 },
-	{ _MMIO(0x9888), 0x15940016 },
-	{ _MMIO(0x9888), 0x37906800 },
-	{ _MMIO(0x9888), 0x03811300 },
-	{ _MMIO(0x9888), 0x05811b12 },
-	{ _MMIO(0x9888), 0x0781001a },
-	{ _MMIO(0x9888), 0x1f810000 },
-	{ _MMIO(0x9888), 0x17810000 },
-	{ _MMIO(0x9888), 0x19810000 },
-	{ _MMIO(0x9888), 0x1b810000 },
-	{ _MMIO(0x9888), 0x1d810000 },
-	{ _MMIO(0x9888), 0x19930800 },
-	{ _MMIO(0x9888), 0x1b93aa55 },
-	{ _MMIO(0x9888), 0x1d9300aa },
-	{ _MMIO(0x9888), 0x01940010 },
-	{ _MMIO(0x9888), 0x07941100 },
-	{ _MMIO(0x9888), 0x09941312 },
-	{ _MMIO(0x9888), 0x0b941514 },
-	{ _MMIO(0x9888), 0x0d941716 },
-	{ _MMIO(0x9888), 0x0f940018 },
-	{ _MMIO(0x9888), 0x1b940000 },
-	{ _MMIO(0x9888), 0x11940000 },
-	{ _MMIO(0x9888), 0x01e58000 },
-	{ _MMIO(0x9888), 0x03e57000 },
-	{ _MMIO(0x9888), 0x31900105 },
-	{ _MMIO(0x9888), 0x15900103 },
-	{ _MMIO(0x9888), 0x17900101 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x13908000 },
-	{ _MMIO(0x9888), 0x21908000 },
-	{ _MMIO(0x9888), 0x23908000 },
-	{ _MMIO(0x9888), 0x25908000 },
-	{ _MMIO(0x9888), 0x27908000 },
-	{ _MMIO(0x9888), 0x29908000 },
-	{ _MMIO(0x9888), 0x2b908000 },
-	{ _MMIO(0x9888), 0x2d908000 },
-	{ _MMIO(0x9888), 0x2f908000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1d908000 },
-	{ _MMIO(0x9888), 0x1f908000 },
-	{ _MMIO(0x9888), 0x11900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c20 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900400 },
-	{ _MMIO(0x9888), 0x47900421 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900421 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900061 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900003 },
-	{ _MMIO(0x9888), 0x53900000 },
-};
-
-static const struct i915_oa_reg mux_config_memory_reads_0_sku_gte_0x05[] = {
-	{ _MMIO(0x9888), 0x11810c00 },
-	{ _MMIO(0x9888), 0x1381001a },
-	{ _MMIO(0x9888), 0x37906800 },
-	{ _MMIO(0x9888), 0x3f900064 },
-	{ _MMIO(0x9888), 0x03811300 },
-	{ _MMIO(0x9888), 0x05811b12 },
-	{ _MMIO(0x9888), 0x0781001a },
-	{ _MMIO(0x9888), 0x1f810000 },
-	{ _MMIO(0x9888), 0x17810000 },
-	{ _MMIO(0x9888), 0x19810000 },
-	{ _MMIO(0x9888), 0x1b810000 },
-	{ _MMIO(0x9888), 0x1d810000 },
-	{ _MMIO(0x9888), 0x1b930055 },
-	{ _MMIO(0x9888), 0x03e58000 },
-	{ _MMIO(0x9888), 0x05e5c000 },
-	{ _MMIO(0x9888), 0x07e54000 },
-	{ _MMIO(0x9888), 0x13900150 },
-	{ _MMIO(0x9888), 0x21900151 },
-	{ _MMIO(0x9888), 0x23900152 },
-	{ _MMIO(0x9888), 0x25900153 },
-	{ _MMIO(0x9888), 0x27900154 },
-	{ _MMIO(0x9888), 0x29900155 },
-	{ _MMIO(0x9888), 0x2b900156 },
-	{ _MMIO(0x9888), 0x2d900157 },
-	{ _MMIO(0x9888), 0x2f90015f },
-	{ _MMIO(0x9888), 0x31900105 },
-	{ _MMIO(0x9888), 0x15900103 },
-	{ _MMIO(0x9888), 0x17900101 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1d908000 },
-	{ _MMIO(0x9888), 0x1f908000 },
-	{ _MMIO(0x9888), 0x11900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c60 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900c00 },
-	{ _MMIO(0x9888), 0x47900c63 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900c63 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900063 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900003 },
-	{ _MMIO(0x9888), 0x53900000 },
-};
-
-static int
-get_memory_reads_mux_config(struct drm_i915_private *dev_priv,
-			    const struct i915_oa_reg **regs,
-			    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 3);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 3);
-
-	if ((INTEL_INFO(dev_priv)->sseu.slice_mask & 0x01) &&
-	    (dev_priv->drm.pdev->revision < 0x02)) {
-		regs[n] = mux_config_memory_reads_0_slices_0x01_and_sku_lt_0x02;
-		lens[n] = ARRAY_SIZE(mux_config_memory_reads_0_slices_0x01_and_sku_lt_0x02);
-		n++;
-	}
-	if ((dev_priv->drm.pdev->revision < 0x05) &&
-		   (dev_priv->drm.pdev->revision >= 0x02)) {
-		regs[n] = mux_config_memory_reads_0_sku_lt_0x05_and_sku_gte_0x02;
-		lens[n] = ARRAY_SIZE(mux_config_memory_reads_0_sku_lt_0x05_and_sku_gte_0x02);
-		n++;
-	}
-	if (dev_priv->drm.pdev->revision >= 0x05) {
-		regs[n] = mux_config_memory_reads_0_sku_gte_0x05;
-		lens[n] = ARRAY_SIZE(mux_config_memory_reads_0_sku_gte_0x05);
-		n++;
-	}
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_memory_writes[] = {
-	{ _MMIO(0x272c), 0xffffffff },
-	{ _MMIO(0x2728), 0xffffffff },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x271c), 0xffffffff },
-	{ _MMIO(0x2718), 0xffffffff },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x274c), 0x86543210 },
-	{ _MMIO(0x2748), 0x86543210 },
-	{ _MMIO(0x2744), 0x00006667 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x275c), 0x86543210 },
-	{ _MMIO(0x2758), 0x86543210 },
-	{ _MMIO(0x2754), 0x00006465 },
-	{ _MMIO(0x2750), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007f81a },
-	{ _MMIO(0x2774), 0x0000fe00 },
-	{ _MMIO(0x2778), 0x0007f82a },
-	{ _MMIO(0x277c), 0x0000fe00 },
-	{ _MMIO(0x2780), 0x0007f822 },
-	{ _MMIO(0x2784), 0x0000fe00 },
-	{ _MMIO(0x2788), 0x0007f8ba },
-	{ _MMIO(0x278c), 0x0000fe00 },
-	{ _MMIO(0x2790), 0x0007f87a },
-	{ _MMIO(0x2794), 0x0000fe00 },
-	{ _MMIO(0x2798), 0x0007f8ea },
-	{ _MMIO(0x279c), 0x0000fe00 },
-	{ _MMIO(0x27a0), 0x0007f8e2 },
-	{ _MMIO(0x27a4), 0x0000fe00 },
-	{ _MMIO(0x27a8), 0x0007f8f2 },
-	{ _MMIO(0x27ac), 0x0000fe00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_memory_writes[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00015014 },
-	{ _MMIO(0xe658), 0x00025024 },
-	{ _MMIO(0xe758), 0x00035034 },
-	{ _MMIO(0xe45c), 0x00045044 },
-	{ _MMIO(0xe55c), 0x00055054 },
-	{ _MMIO(0xe65c), 0x00065064 },
-};
-
-static const struct i915_oa_reg mux_config_memory_writes_0_slices_0x01_and_sku_lt_0x02[] = {
-	{ _MMIO(0x9888), 0x11810c00 },
-	{ _MMIO(0x9888), 0x1381001a },
-	{ _MMIO(0x9888), 0x13945400 },
-	{ _MMIO(0x9888), 0x37906800 },
-	{ _MMIO(0x9888), 0x3f901400 },
-	{ _MMIO(0x9888), 0x03811300 },
-	{ _MMIO(0x9888), 0x05811b12 },
-	{ _MMIO(0x9888), 0x0781001a },
-	{ _MMIO(0x9888), 0x1f810000 },
-	{ _MMIO(0x9888), 0x17810000 },
-	{ _MMIO(0x9888), 0x19810000 },
-	{ _MMIO(0x9888), 0x1b810000 },
-	{ _MMIO(0x9888), 0x1d810000 },
-	{ _MMIO(0x9888), 0x0f968000 },
-	{ _MMIO(0x9888), 0x1196c000 },
-	{ _MMIO(0x9888), 0x13964000 },
-	{ _MMIO(0x9888), 0x11938000 },
-	{ _MMIO(0x9888), 0x1b93fe00 },
-	{ _MMIO(0x9888), 0x01940010 },
-	{ _MMIO(0x9888), 0x07941100 },
-	{ _MMIO(0x9888), 0x09941312 },
-	{ _MMIO(0x9888), 0x0b941514 },
-	{ _MMIO(0x9888), 0x0d941716 },
-	{ _MMIO(0x9888), 0x11940000 },
-	{ _MMIO(0x9888), 0x19940000 },
-	{ _MMIO(0x9888), 0x1b940000 },
-	{ _MMIO(0x9888), 0x1d940000 },
-	{ _MMIO(0x9888), 0x1b954000 },
-	{ _MMIO(0x9888), 0x1d95a550 },
-	{ _MMIO(0x9888), 0x1f9502aa },
-	{ _MMIO(0x9888), 0x2f900167 },
-	{ _MMIO(0x9888), 0x31900105 },
-	{ _MMIO(0x9888), 0x15900103 },
-	{ _MMIO(0x9888), 0x17900101 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x13908000 },
-	{ _MMIO(0x9888), 0x21908000 },
-	{ _MMIO(0x9888), 0x23908000 },
-	{ _MMIO(0x9888), 0x25908000 },
-	{ _MMIO(0x9888), 0x27908000 },
-	{ _MMIO(0x9888), 0x29908000 },
-	{ _MMIO(0x9888), 0x2b908000 },
-	{ _MMIO(0x9888), 0x2d908000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1d908000 },
-	{ _MMIO(0x9888), 0x1f908000 },
-	{ _MMIO(0xd28), 0x00000000 },
-	{ _MMIO(0x9888), 0x11900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c00 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900000 },
-	{ _MMIO(0x9888), 0x47900000 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900063 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900003 },
-	{ _MMIO(0x9888), 0x53900000 },
-};
-
-static const struct i915_oa_reg mux_config_memory_writes_0_sku_lt_0x05_and_sku_gte_0x02[] = {
-	{ _MMIO(0x9888), 0x11810c00 },
-	{ _MMIO(0x9888), 0x1381001a },
-	{ _MMIO(0x9888), 0x13945400 },
-	{ _MMIO(0x9888), 0x37906800 },
-	{ _MMIO(0x9888), 0x3f901400 },
-	{ _MMIO(0x9888), 0x03811300 },
-	{ _MMIO(0x9888), 0x05811b12 },
-	{ _MMIO(0x9888), 0x0781001a },
-	{ _MMIO(0x9888), 0x1f810000 },
-	{ _MMIO(0x9888), 0x17810000 },
-	{ _MMIO(0x9888), 0x19810000 },
-	{ _MMIO(0x9888), 0x1b810000 },
-	{ _MMIO(0x9888), 0x1d810000 },
-	{ _MMIO(0x9888), 0x19930800 },
-	{ _MMIO(0x9888), 0x1b93aa55 },
-	{ _MMIO(0x9888), 0x1d93002a },
-	{ _MMIO(0x9888), 0x01940010 },
-	{ _MMIO(0x9888), 0x07941100 },
-	{ _MMIO(0x9888), 0x09941312 },
-	{ _MMIO(0x9888), 0x0b941514 },
-	{ _MMIO(0x9888), 0x0d941716 },
-	{ _MMIO(0x9888), 0x1b940000 },
-	{ _MMIO(0x9888), 0x11940000 },
-	{ _MMIO(0x9888), 0x01e58000 },
-	{ _MMIO(0x9888), 0x03e57000 },
-	{ _MMIO(0x9888), 0x2f900167 },
-	{ _MMIO(0x9888), 0x31900105 },
-	{ _MMIO(0x9888), 0x15900103 },
-	{ _MMIO(0x9888), 0x17900101 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x13908000 },
-	{ _MMIO(0x9888), 0x21908000 },
-	{ _MMIO(0x9888), 0x23908000 },
-	{ _MMIO(0x9888), 0x25908000 },
-	{ _MMIO(0x9888), 0x27908000 },
-	{ _MMIO(0x9888), 0x29908000 },
-	{ _MMIO(0x9888), 0x2b908000 },
-	{ _MMIO(0x9888), 0x2d908000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1d908000 },
-	{ _MMIO(0x9888), 0x1f908000 },
-	{ _MMIO(0x9888), 0x11900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c20 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900400 },
-	{ _MMIO(0x9888), 0x47900421 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900421 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900063 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900003 },
-	{ _MMIO(0x9888), 0x53900000 },
-};
-
-static const struct i915_oa_reg mux_config_memory_writes_0_sku_gte_0x05[] = {
-	{ _MMIO(0x9888), 0x11810c00 },
-	{ _MMIO(0x9888), 0x1381001a },
-	{ _MMIO(0x9888), 0x37906800 },
-	{ _MMIO(0x9888), 0x3f901000 },
-	{ _MMIO(0x9888), 0x03811300 },
-	{ _MMIO(0x9888), 0x05811b12 },
-	{ _MMIO(0x9888), 0x0781001a },
-	{ _MMIO(0x9888), 0x1f810000 },
-	{ _MMIO(0x9888), 0x17810000 },
-	{ _MMIO(0x9888), 0x19810000 },
-	{ _MMIO(0x9888), 0x1b810000 },
-	{ _MMIO(0x9888), 0x1d810000 },
-	{ _MMIO(0x9888), 0x1b930055 },
-	{ _MMIO(0x9888), 0x03e58000 },
-	{ _MMIO(0x9888), 0x05e5c000 },
-	{ _MMIO(0x9888), 0x07e54000 },
-	{ _MMIO(0x9888), 0x13900160 },
-	{ _MMIO(0x9888), 0x21900161 },
-	{ _MMIO(0x9888), 0x23900162 },
-	{ _MMIO(0x9888), 0x25900163 },
-	{ _MMIO(0x9888), 0x27900164 },
-	{ _MMIO(0x9888), 0x29900165 },
-	{ _MMIO(0x9888), 0x2b900166 },
-	{ _MMIO(0x9888), 0x2d900167 },
-	{ _MMIO(0x9888), 0x2f900150 },
-	{ _MMIO(0x9888), 0x31900105 },
-	{ _MMIO(0x9888), 0x15900103 },
-	{ _MMIO(0x9888), 0x17900101 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1d908000 },
-	{ _MMIO(0x9888), 0x1f908000 },
-	{ _MMIO(0x9888), 0x11900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c60 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900c00 },
-	{ _MMIO(0x9888), 0x47900c63 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900c63 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900063 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900003 },
-	{ _MMIO(0x9888), 0x53900000 },
-};
-
-static int
-get_memory_writes_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 3);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 3);
-
-	if ((INTEL_INFO(dev_priv)->sseu.slice_mask & 0x01) &&
-	    (dev_priv->drm.pdev->revision < 0x02)) {
-		regs[n] = mux_config_memory_writes_0_slices_0x01_and_sku_lt_0x02;
-		lens[n] = ARRAY_SIZE(mux_config_memory_writes_0_slices_0x01_and_sku_lt_0x02);
-		n++;
-	}
-	if ((dev_priv->drm.pdev->revision < 0x05) &&
-		   (dev_priv->drm.pdev->revision >= 0x02)) {
-		regs[n] = mux_config_memory_writes_0_sku_lt_0x05_and_sku_gte_0x02;
-		lens[n] = ARRAY_SIZE(mux_config_memory_writes_0_sku_lt_0x05_and_sku_gte_0x02);
-		n++;
-	}
-	if (dev_priv->drm.pdev->revision >= 0x05) {
-		regs[n] = mux_config_memory_writes_0_sku_gte_0x05;
-		lens[n] = ARRAY_SIZE(mux_config_memory_writes_0_sku_gte_0x05);
-		n++;
-	}
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_extended[] = {
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007fc2a },
-	{ _MMIO(0x2774), 0x0000bf00 },
-	{ _MMIO(0x2778), 0x0007fc6a },
-	{ _MMIO(0x277c), 0x0000bf00 },
-	{ _MMIO(0x2780), 0x0007fc92 },
-	{ _MMIO(0x2784), 0x0000bf00 },
-	{ _MMIO(0x2788), 0x0007fca2 },
-	{ _MMIO(0x278c), 0x0000bf00 },
-	{ _MMIO(0x2790), 0x0007fc32 },
-	{ _MMIO(0x2794), 0x0000bf00 },
-	{ _MMIO(0x2798), 0x0007fc9a },
-	{ _MMIO(0x279c), 0x0000bf00 },
-	{ _MMIO(0x27a0), 0x0007fe6a },
-	{ _MMIO(0x27a4), 0x0000bf00 },
-	{ _MMIO(0x27a8), 0x0007fe7a },
-	{ _MMIO(0x27ac), 0x0000bf00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_extended[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00778008 },
-	{ _MMIO(0xe45c), 0x00088078 },
-	{ _MMIO(0xe55c), 0x00808708 },
-	{ _MMIO(0xe65c), 0x00a08908 },
-};
-
-static const struct i915_oa_reg mux_config_compute_extended_0_subslices_0x01[] = {
-	{ _MMIO(0x9888), 0x106c00e0 },
-	{ _MMIO(0x9888), 0x141c8160 },
-	{ _MMIO(0x9888), 0x161c8015 },
-	{ _MMIO(0x9888), 0x181c0120 },
-	{ _MMIO(0x9888), 0x004e8000 },
-	{ _MMIO(0x9888), 0x0e4e8000 },
-	{ _MMIO(0x9888), 0x184e8000 },
-	{ _MMIO(0x9888), 0x1a4eaaa0 },
-	{ _MMIO(0x9888), 0x1c4e0002 },
-	{ _MMIO(0x9888), 0x024e8000 },
-	{ _MMIO(0x9888), 0x044e8000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084e8000 },
-	{ _MMIO(0x9888), 0x0a4e8000 },
-	{ _MMIO(0x9888), 0x0e6c0b01 },
-	{ _MMIO(0x9888), 0x006c0200 },
-	{ _MMIO(0x9888), 0x026c000c },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1e6c0000 },
-	{ _MMIO(0x9888), 0x1a6c0000 },
-	{ _MMIO(0x9888), 0x0e1bc000 },
-	{ _MMIO(0x9888), 0x001b8000 },
-	{ _MMIO(0x9888), 0x021bc000 },
-	{ _MMIO(0x9888), 0x001c0041 },
-	{ _MMIO(0x9888), 0x061c4200 },
-	{ _MMIO(0x9888), 0x081c4443 },
-	{ _MMIO(0x9888), 0x0a1c4645 },
-	{ _MMIO(0x9888), 0x0c1c7647 },
-	{ _MMIO(0x9888), 0x041c7357 },
-	{ _MMIO(0x9888), 0x1c1c0030 },
-	{ _MMIO(0x9888), 0x101c0000 },
-	{ _MMIO(0x9888), 0x1a1c0000 },
-	{ _MMIO(0x9888), 0x121c8000 },
-	{ _MMIO(0x9888), 0x004c8000 },
-	{ _MMIO(0x9888), 0x0a4caa2a },
-	{ _MMIO(0x9888), 0x0c4c02aa },
-	{ _MMIO(0x9888), 0x084ca000 },
-	{ _MMIO(0x9888), 0x000da000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0a0da000 },
-	{ _MMIO(0x9888), 0x0c0da000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x0c0f5400 },
-	{ _MMIO(0x9888), 0x0e0f5515 },
-	{ _MMIO(0x9888), 0x100f0155 },
-	{ _MMIO(0x9888), 0x002c8000 },
-	{ _MMIO(0x9888), 0x0e2c8000 },
-	{ _MMIO(0x9888), 0x162caa00 },
-	{ _MMIO(0x9888), 0x182c00aa },
-	{ _MMIO(0x9888), 0x022c8000 },
-	{ _MMIO(0x9888), 0x042c8000 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0xd28), 0x00000000 },
-	{ _MMIO(0x9888), 0x11907fff },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900040 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900802 },
-	{ _MMIO(0x9888), 0x47900842 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900842 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900800 },
-	{ _MMIO(0x9888), 0x53900000 },
-};
-
-static int
-get_compute_extended_mux_config(struct drm_i915_private *dev_priv,
-				const struct i915_oa_reg **regs,
-				int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	if (INTEL_INFO(dev_priv)->sseu.subslice_mask & 0x01) {
-		regs[n] = mux_config_compute_extended_0_subslices_0x01;
-		lens[n] = ARRAY_SIZE(mux_config_compute_extended_0_subslices_0x01);
-		n++;
-	}
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_l3_cache[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x30800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007fffa },
-	{ _MMIO(0x2774), 0x0000fefe },
-	{ _MMIO(0x2778), 0x0007fffa },
-	{ _MMIO(0x277c), 0x0000fefd },
-	{ _MMIO(0x2790), 0x0007fffa },
-	{ _MMIO(0x2794), 0x0000fbef },
-	{ _MMIO(0x2798), 0x0007fffa },
-	{ _MMIO(0x279c), 0x0000fbdf },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_l3_cache[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00101100 },
-	{ _MMIO(0xe45c), 0x00201200 },
-	{ _MMIO(0xe55c), 0x00301300 },
-	{ _MMIO(0xe65c), 0x00401400 },
-};
-
-static const struct i915_oa_reg mux_config_compute_l3_cache[] = {
-	{ _MMIO(0x9888), 0x166c0760 },
-	{ _MMIO(0x9888), 0x1593001e },
-	{ _MMIO(0x9888), 0x3f901403 },
-	{ _MMIO(0x9888), 0x004e8000 },
-	{ _MMIO(0x9888), 0x0e4e8000 },
-	{ _MMIO(0x9888), 0x184e8000 },
-	{ _MMIO(0x9888), 0x1a4e8020 },
-	{ _MMIO(0x9888), 0x1c4e0002 },
-	{ _MMIO(0x9888), 0x006c0051 },
-	{ _MMIO(0x9888), 0x066c5000 },
-	{ _MMIO(0x9888), 0x086c5c5d },
-	{ _MMIO(0x9888), 0x0e6c5e5f },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x186c0000 },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1e6c0000 },
-	{ _MMIO(0x9888), 0x001b4000 },
-	{ _MMIO(0x9888), 0x061b8000 },
-	{ _MMIO(0x9888), 0x081bc000 },
-	{ _MMIO(0x9888), 0x0e1bc000 },
-	{ _MMIO(0x9888), 0x101c8000 },
-	{ _MMIO(0x9888), 0x1a1ce000 },
-	{ _MMIO(0x9888), 0x1c1c0030 },
-	{ _MMIO(0x9888), 0x004c8000 },
-	{ _MMIO(0x9888), 0x0a4c2a00 },
-	{ _MMIO(0x9888), 0x0c4c0280 },
-	{ _MMIO(0x9888), 0x000d2000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x0c0f0400 },
-	{ _MMIO(0x9888), 0x0e0f1500 },
-	{ _MMIO(0x9888), 0x100f0140 },
-	{ _MMIO(0x9888), 0x002c8000 },
-	{ _MMIO(0x9888), 0x0e2c8000 },
-	{ _MMIO(0x9888), 0x162c0a00 },
-	{ _MMIO(0x9888), 0x182c00a0 },
-	{ _MMIO(0x9888), 0x03933300 },
-	{ _MMIO(0x9888), 0x05930032 },
-	{ _MMIO(0x9888), 0x11930000 },
-	{ _MMIO(0x9888), 0x1b930000 },
-	{ _MMIO(0x9888), 0x1d900157 },
-	{ _MMIO(0x9888), 0x1f900167 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1190030f },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900000 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900042 },
-	{ _MMIO(0x9888), 0x47900000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x4b900000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x53901111 },
-	{ _MMIO(0x9888), 0x43900420 },
-};
-
-static int
-get_compute_l3_cache_mux_config(struct drm_i915_private *dev_priv,
-				const struct i915_oa_reg **regs,
-				int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_l3_cache;
-	lens[n] = ARRAY_SIZE(mux_config_compute_l3_cache);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_hdc_and_sf[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x10800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x0000fdff },
-};
-
-static const struct i915_oa_reg flex_eu_config_hdc_and_sf[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_hdc_and_sf[] = {
-	{ _MMIO(0x9888), 0x104f0232 },
-	{ _MMIO(0x9888), 0x124f4640 },
-	{ _MMIO(0x9888), 0x106c0232 },
-	{ _MMIO(0x9888), 0x11834400 },
-	{ _MMIO(0x9888), 0x0a4e8000 },
-	{ _MMIO(0x9888), 0x0c4e8000 },
-	{ _MMIO(0x9888), 0x004f1880 },
-	{ _MMIO(0x9888), 0x024f08bb },
-	{ _MMIO(0x9888), 0x044f001b },
-	{ _MMIO(0x9888), 0x046c0100 },
-	{ _MMIO(0x9888), 0x066c000b },
-	{ _MMIO(0x9888), 0x1a6c0000 },
-	{ _MMIO(0x9888), 0x041b8000 },
-	{ _MMIO(0x9888), 0x061b4000 },
-	{ _MMIO(0x9888), 0x1a1c1800 },
-	{ _MMIO(0x9888), 0x005b8000 },
-	{ _MMIO(0x9888), 0x025bc000 },
-	{ _MMIO(0x9888), 0x045b4000 },
-	{ _MMIO(0x9888), 0x125c8000 },
-	{ _MMIO(0x9888), 0x145c8000 },
-	{ _MMIO(0x9888), 0x165c8000 },
-	{ _MMIO(0x9888), 0x185c8000 },
-	{ _MMIO(0x9888), 0x0a4c00a0 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x0c0f5000 },
-	{ _MMIO(0x9888), 0x0e0f0055 },
-	{ _MMIO(0x9888), 0x022cc000 },
-	{ _MMIO(0x9888), 0x042cc000 },
-	{ _MMIO(0x9888), 0x062cc000 },
-	{ _MMIO(0x9888), 0x082cc000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x0f828000 },
-	{ _MMIO(0x9888), 0x0f8305c0 },
-	{ _MMIO(0x9888), 0x09830000 },
-	{ _MMIO(0x9888), 0x07830000 },
-	{ _MMIO(0x9888), 0x1d950080 },
-	{ _MMIO(0x9888), 0x13928000 },
-	{ _MMIO(0x9888), 0x0f988000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x1190fc00 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x4b9000a0 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900800 },
-	{ _MMIO(0x9888), 0x43900842 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_hdc_and_sf_mux_config(struct drm_i915_private *dev_priv,
-			  const struct i915_oa_reg **regs,
-			  int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_hdc_and_sf;
-	lens[n] = ARRAY_SIZE(mux_config_hdc_and_sf);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_l3_1[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2770), 0x00100070 },
-	{ _MMIO(0x2774), 0x0000fff1 },
-	{ _MMIO(0x2778), 0x00014002 },
-	{ _MMIO(0x277c), 0x0000c3ff },
-	{ _MMIO(0x2780), 0x00010002 },
-	{ _MMIO(0x2784), 0x0000c7ff },
-	{ _MMIO(0x2788), 0x00004002 },
-	{ _MMIO(0x278c), 0x0000d3ff },
-	{ _MMIO(0x2790), 0x00100700 },
-	{ _MMIO(0x2794), 0x0000ff1f },
-	{ _MMIO(0x2798), 0x00001402 },
-	{ _MMIO(0x279c), 0x0000fc3f },
-	{ _MMIO(0x27a0), 0x00001002 },
-	{ _MMIO(0x27a4), 0x0000fc7f },
-	{ _MMIO(0x27a8), 0x00000402 },
-	{ _MMIO(0x27ac), 0x0000fd3f },
-};
-
-static const struct i915_oa_reg flex_eu_config_l3_1[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_l3_1[] = {
-	{ _MMIO(0x9888), 0x126c7b40 },
-	{ _MMIO(0x9888), 0x166c0020 },
-	{ _MMIO(0x9888), 0x0a603444 },
-	{ _MMIO(0x9888), 0x0a613400 },
-	{ _MMIO(0x9888), 0x1a4ea800 },
-	{ _MMIO(0x9888), 0x1c4e0002 },
-	{ _MMIO(0x9888), 0x024e8000 },
-	{ _MMIO(0x9888), 0x044e8000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084e8000 },
-	{ _MMIO(0x9888), 0x0a4e8000 },
-	{ _MMIO(0x9888), 0x064f4000 },
-	{ _MMIO(0x9888), 0x0c6c5327 },
-	{ _MMIO(0x9888), 0x0e6c5425 },
-	{ _MMIO(0x9888), 0x006c2a00 },
-	{ _MMIO(0x9888), 0x026c285b },
-	{ _MMIO(0x9888), 0x046c005c },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1e6c0000 },
-	{ _MMIO(0x9888), 0x1a6c0800 },
-	{ _MMIO(0x9888), 0x0c1bc000 },
-	{ _MMIO(0x9888), 0x0e1bc000 },
-	{ _MMIO(0x9888), 0x001b8000 },
-	{ _MMIO(0x9888), 0x021bc000 },
-	{ _MMIO(0x9888), 0x041bc000 },
-	{ _MMIO(0x9888), 0x1c1c003c },
-	{ _MMIO(0x9888), 0x121c8000 },
-	{ _MMIO(0x9888), 0x141c8000 },
-	{ _MMIO(0x9888), 0x161c8000 },
-	{ _MMIO(0x9888), 0x181c8000 },
-	{ _MMIO(0x9888), 0x1a1c0800 },
-	{ _MMIO(0x9888), 0x065b4000 },
-	{ _MMIO(0x9888), 0x1a5c1000 },
-	{ _MMIO(0x9888), 0x10600000 },
-	{ _MMIO(0x9888), 0x04600000 },
-	{ _MMIO(0x9888), 0x0c610044 },
-	{ _MMIO(0x9888), 0x10610000 },
-	{ _MMIO(0x9888), 0x06610000 },
-	{ _MMIO(0x9888), 0x0c4c02a8 },
-	{ _MMIO(0x9888), 0x084ca000 },
-	{ _MMIO(0x9888), 0x0a4c002a },
-	{ _MMIO(0x9888), 0x0c0da000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x100f0154 },
-	{ _MMIO(0x9888), 0x0c0f5000 },
-	{ _MMIO(0x9888), 0x0e0f0055 },
-	{ _MMIO(0x9888), 0x182c00aa },
-	{ _MMIO(0x9888), 0x022c8000 },
-	{ _MMIO(0x9888), 0x042c8000 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2cc000 },
-	{ _MMIO(0x9888), 0x1190ffc0 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900420 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900021 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900400 },
-	{ _MMIO(0x9888), 0x43900421 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900040 },
-};
-
-static int
-get_l3_1_mux_config(struct drm_i915_private *dev_priv,
-		    const struct i915_oa_reg **regs,
-		    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_l3_1;
-	lens[n] = ARRAY_SIZE(mux_config_l3_1);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_l3_2[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00100070 },
-	{ _MMIO(0x2774), 0x0000fff1 },
-	{ _MMIO(0x2778), 0x00028002 },
-	{ _MMIO(0x277c), 0x000087ff },
-	{ _MMIO(0x2780), 0x00020002 },
-	{ _MMIO(0x2784), 0x00008fff },
-	{ _MMIO(0x2788), 0x00008002 },
-	{ _MMIO(0x278c), 0x0000a7ff },
-};
-
-static const struct i915_oa_reg flex_eu_config_l3_2[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_l3_2[] = {
-	{ _MMIO(0x9888), 0x126c02e0 },
-	{ _MMIO(0x9888), 0x146c0001 },
-	{ _MMIO(0x9888), 0x0a623400 },
-	{ _MMIO(0x9888), 0x044e8000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084e8000 },
-	{ _MMIO(0x9888), 0x0a4e8000 },
-	{ _MMIO(0x9888), 0x064f4000 },
-	{ _MMIO(0x9888), 0x026c3324 },
-	{ _MMIO(0x9888), 0x046c3422 },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x1a6c0000 },
-	{ _MMIO(0x9888), 0x021bc000 },
-	{ _MMIO(0x9888), 0x041bc000 },
-	{ _MMIO(0x9888), 0x141c8000 },
-	{ _MMIO(0x9888), 0x161c8000 },
-	{ _MMIO(0x9888), 0x181c8000 },
-	{ _MMIO(0x9888), 0x1a1c0800 },
-	{ _MMIO(0x9888), 0x065b4000 },
-	{ _MMIO(0x9888), 0x1a5c1000 },
-	{ _MMIO(0x9888), 0x06614000 },
-	{ _MMIO(0x9888), 0x0c620044 },
-	{ _MMIO(0x9888), 0x10620000 },
-	{ _MMIO(0x9888), 0x06620000 },
-	{ _MMIO(0x9888), 0x084c8000 },
-	{ _MMIO(0x9888), 0x0a4c002a },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x0c0f4000 },
-	{ _MMIO(0x9888), 0x0e0f0055 },
-	{ _MMIO(0x9888), 0x042c8000 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2cc000 },
-	{ _MMIO(0x9888), 0x1190f800 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x43900000 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_l3_2_mux_config(struct drm_i915_private *dev_priv,
-		    const struct i915_oa_reg **regs,
-		    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_l3_2;
-	lens[n] = ARRAY_SIZE(mux_config_l3_2);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_l3_3[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00100070 },
-	{ _MMIO(0x2774), 0x0000fff1 },
-	{ _MMIO(0x2778), 0x00028002 },
-	{ _MMIO(0x277c), 0x000087ff },
-	{ _MMIO(0x2780), 0x00020002 },
-	{ _MMIO(0x2784), 0x00008fff },
-	{ _MMIO(0x2788), 0x00008002 },
-	{ _MMIO(0x278c), 0x0000a7ff },
-};
-
-static const struct i915_oa_reg flex_eu_config_l3_3[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_l3_3[] = {
-	{ _MMIO(0x9888), 0x126c4e80 },
-	{ _MMIO(0x9888), 0x146c0000 },
-	{ _MMIO(0x9888), 0x0a633400 },
-	{ _MMIO(0x9888), 0x044e8000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084e8000 },
-	{ _MMIO(0x9888), 0x0a4e8000 },
-	{ _MMIO(0x9888), 0x0c4e8000 },
-	{ _MMIO(0x9888), 0x026c3321 },
-	{ _MMIO(0x9888), 0x046c342f },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x1a6c2000 },
-	{ _MMIO(0x9888), 0x021bc000 },
-	{ _MMIO(0x9888), 0x041bc000 },
-	{ _MMIO(0x9888), 0x061b4000 },
-	{ _MMIO(0x9888), 0x141c8000 },
-	{ _MMIO(0x9888), 0x161c8000 },
-	{ _MMIO(0x9888), 0x181c8000 },
-	{ _MMIO(0x9888), 0x1a1c1800 },
-	{ _MMIO(0x9888), 0x06604000 },
-	{ _MMIO(0x9888), 0x0c630044 },
-	{ _MMIO(0x9888), 0x10630000 },
-	{ _MMIO(0x9888), 0x06630000 },
-	{ _MMIO(0x9888), 0x084c8000 },
-	{ _MMIO(0x9888), 0x0a4c00aa },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x0c0f4000 },
-	{ _MMIO(0x9888), 0x0e0f0055 },
-	{ _MMIO(0x9888), 0x042c8000 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x1190f800 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x43900842 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900002 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_l3_3_mux_config(struct drm_i915_private *dev_priv,
-		    const struct i915_oa_reg **regs,
-		    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_l3_3;
-	lens[n] = ARRAY_SIZE(mux_config_l3_3);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x30800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x0000efff },
-	{ _MMIO(0x2778), 0x00006000 },
-	{ _MMIO(0x277c), 0x0000f3ff },
-};
-
-static const struct i915_oa_reg flex_eu_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0x9888), 0x102f3800 },
-	{ _MMIO(0x9888), 0x144d0500 },
-	{ _MMIO(0x9888), 0x120d03c0 },
-	{ _MMIO(0x9888), 0x140d03cf },
-	{ _MMIO(0x9888), 0x0c0f0004 },
-	{ _MMIO(0x9888), 0x0c4e4000 },
-	{ _MMIO(0x9888), 0x042f0480 },
-	{ _MMIO(0x9888), 0x082f0000 },
-	{ _MMIO(0x9888), 0x022f0000 },
-	{ _MMIO(0x9888), 0x0a4c0090 },
-	{ _MMIO(0x9888), 0x064d0027 },
-	{ _MMIO(0x9888), 0x004d0000 },
-	{ _MMIO(0x9888), 0x000d0d40 },
-	{ _MMIO(0x9888), 0x020d803f },
-	{ _MMIO(0x9888), 0x040d8023 },
-	{ _MMIO(0x9888), 0x100d0000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x020f0010 },
-	{ _MMIO(0x9888), 0x000f0000 },
-	{ _MMIO(0x9888), 0x0e0f0050 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x1190fc00 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41901400 },
-	{ _MMIO(0x9888), 0x43901485 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900001 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_rasterizer_and_pixel_backend_mux_config(struct drm_i915_private *dev_priv,
-					    const struct i915_oa_reg **regs,
-					    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_rasterizer_and_pixel_backend;
-	lens[n] = ARRAY_SIZE(mux_config_rasterizer_and_pixel_backend);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_sampler[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x70800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x0000c000 },
-	{ _MMIO(0x2774), 0x0000e7ff },
-	{ _MMIO(0x2778), 0x00003000 },
-	{ _MMIO(0x277c), 0x0000f9ff },
-	{ _MMIO(0x2780), 0x00000c00 },
-	{ _MMIO(0x2784), 0x0000fe7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_sampler[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_sampler[] = {
-	{ _MMIO(0x9888), 0x14152c00 },
-	{ _MMIO(0x9888), 0x16150005 },
-	{ _MMIO(0x9888), 0x121600a0 },
-	{ _MMIO(0x9888), 0x14352c00 },
-	{ _MMIO(0x9888), 0x16350005 },
-	{ _MMIO(0x9888), 0x123600a0 },
-	{ _MMIO(0x9888), 0x14552c00 },
-	{ _MMIO(0x9888), 0x16550005 },
-	{ _MMIO(0x9888), 0x125600a0 },
-	{ _MMIO(0x9888), 0x062f6000 },
-	{ _MMIO(0x9888), 0x022f2000 },
-	{ _MMIO(0x9888), 0x0c4c0050 },
-	{ _MMIO(0x9888), 0x0a4c0010 },
-	{ _MMIO(0x9888), 0x0c0d8000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x100f0350 },
-	{ _MMIO(0x9888), 0x0c0fb000 },
-	{ _MMIO(0x9888), 0x0e0f00da },
-	{ _MMIO(0x9888), 0x182c0028 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x022dc000 },
-	{ _MMIO(0x9888), 0x042d4000 },
-	{ _MMIO(0x9888), 0x0c138000 },
-	{ _MMIO(0x9888), 0x0e132000 },
-	{ _MMIO(0x9888), 0x0413c000 },
-	{ _MMIO(0x9888), 0x1c140018 },
-	{ _MMIO(0x9888), 0x0c157000 },
-	{ _MMIO(0x9888), 0x0e150078 },
-	{ _MMIO(0x9888), 0x10150000 },
-	{ _MMIO(0x9888), 0x04162180 },
-	{ _MMIO(0x9888), 0x02160000 },
-	{ _MMIO(0x9888), 0x04174000 },
-	{ _MMIO(0x9888), 0x0233a000 },
-	{ _MMIO(0x9888), 0x04333000 },
-	{ _MMIO(0x9888), 0x14348000 },
-	{ _MMIO(0x9888), 0x16348000 },
-	{ _MMIO(0x9888), 0x02357870 },
-	{ _MMIO(0x9888), 0x10350000 },
-	{ _MMIO(0x9888), 0x04360043 },
-	{ _MMIO(0x9888), 0x02360000 },
-	{ _MMIO(0x9888), 0x04371000 },
-	{ _MMIO(0x9888), 0x0e538000 },
-	{ _MMIO(0x9888), 0x00538000 },
-	{ _MMIO(0x9888), 0x06533000 },
-	{ _MMIO(0x9888), 0x1c540020 },
-	{ _MMIO(0x9888), 0x12548000 },
-	{ _MMIO(0x9888), 0x0e557000 },
-	{ _MMIO(0x9888), 0x00557800 },
-	{ _MMIO(0x9888), 0x10550000 },
-	{ _MMIO(0x9888), 0x06560043 },
-	{ _MMIO(0x9888), 0x02560000 },
-	{ _MMIO(0x9888), 0x06571000 },
-	{ _MMIO(0x9888), 0x1190ff80 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900060 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c00 },
-	{ _MMIO(0x9888), 0x43900842 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900060 },
-};
-
-static int
-get_sampler_mux_config(struct drm_i915_private *dev_priv,
-		       const struct i915_oa_reg **regs,
-		       int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_sampler;
-	lens[n] = ARRAY_SIZE(mux_config_sampler);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_tdl_1[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x00007fff },
-	{ _MMIO(0x2778), 0x00000000 },
-	{ _MMIO(0x277c), 0x00009fff },
-	{ _MMIO(0x2780), 0x00000002 },
-	{ _MMIO(0x2784), 0x0000efff },
-	{ _MMIO(0x2788), 0x00000000 },
-	{ _MMIO(0x278c), 0x0000f3ff },
-	{ _MMIO(0x2790), 0x00000002 },
-	{ _MMIO(0x2794), 0x0000fdff },
-	{ _MMIO(0x2798), 0x00000000 },
-	{ _MMIO(0x279c), 0x0000fe7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_tdl_1[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_tdl_1[] = {
-	{ _MMIO(0x9888), 0x12120000 },
-	{ _MMIO(0x9888), 0x12320000 },
-	{ _MMIO(0x9888), 0x12520000 },
-	{ _MMIO(0x9888), 0x002f8000 },
-	{ _MMIO(0x9888), 0x022f3000 },
-	{ _MMIO(0x9888), 0x0a4c0015 },
-	{ _MMIO(0x9888), 0x0c0d8000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x100f03a0 },
-	{ _MMIO(0x9888), 0x0c0ff000 },
-	{ _MMIO(0x9888), 0x0e0f0095 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2d8000 },
-	{ _MMIO(0x9888), 0x0e2d4000 },
-	{ _MMIO(0x9888), 0x062d4000 },
-	{ _MMIO(0x9888), 0x02108000 },
-	{ _MMIO(0x9888), 0x0410c000 },
-	{ _MMIO(0x9888), 0x02118000 },
-	{ _MMIO(0x9888), 0x0411c000 },
-	{ _MMIO(0x9888), 0x02121880 },
-	{ _MMIO(0x9888), 0x041219b5 },
-	{ _MMIO(0x9888), 0x00120000 },
-	{ _MMIO(0x9888), 0x02134000 },
-	{ _MMIO(0x9888), 0x04135000 },
-	{ _MMIO(0x9888), 0x0c308000 },
-	{ _MMIO(0x9888), 0x0e304000 },
-	{ _MMIO(0x9888), 0x06304000 },
-	{ _MMIO(0x9888), 0x0c318000 },
-	{ _MMIO(0x9888), 0x0e314000 },
-	{ _MMIO(0x9888), 0x06314000 },
-	{ _MMIO(0x9888), 0x0c321a80 },
-	{ _MMIO(0x9888), 0x0e320033 },
-	{ _MMIO(0x9888), 0x06320031 },
-	{ _MMIO(0x9888), 0x00320000 },
-	{ _MMIO(0x9888), 0x0c334000 },
-	{ _MMIO(0x9888), 0x0e331000 },
-	{ _MMIO(0x9888), 0x06331000 },
-	{ _MMIO(0x9888), 0x0e508000 },
-	{ _MMIO(0x9888), 0x00508000 },
-	{ _MMIO(0x9888), 0x02504000 },
-	{ _MMIO(0x9888), 0x0e518000 },
-	{ _MMIO(0x9888), 0x00518000 },
-	{ _MMIO(0x9888), 0x02514000 },
-	{ _MMIO(0x9888), 0x0e521880 },
-	{ _MMIO(0x9888), 0x00521a80 },
-	{ _MMIO(0x9888), 0x02520033 },
-	{ _MMIO(0x9888), 0x0e534000 },
-	{ _MMIO(0x9888), 0x00534000 },
-	{ _MMIO(0x9888), 0x02531000 },
-	{ _MMIO(0x9888), 0x1190ff80 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900800 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900062 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c00 },
-	{ _MMIO(0x9888), 0x43900003 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900040 },
-};
-
-static int
-get_tdl_1_mux_config(struct drm_i915_private *dev_priv,
-		     const struct i915_oa_reg **regs,
-		     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_tdl_1;
-	lens[n] = ARRAY_SIZE(mux_config_tdl_1);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_tdl_2[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_tdl_2[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_tdl_2[] = {
-	{ _MMIO(0x9888), 0x12124d60 },
-	{ _MMIO(0x9888), 0x12322e60 },
-	{ _MMIO(0x9888), 0x12524d60 },
-	{ _MMIO(0x9888), 0x022f3000 },
-	{ _MMIO(0x9888), 0x0a4c0014 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x0c0fe000 },
-	{ _MMIO(0x9888), 0x0e0f0097 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x002d8000 },
-	{ _MMIO(0x9888), 0x062d4000 },
-	{ _MMIO(0x9888), 0x0410c000 },
-	{ _MMIO(0x9888), 0x0411c000 },
-	{ _MMIO(0x9888), 0x04121fb7 },
-	{ _MMIO(0x9888), 0x00120000 },
-	{ _MMIO(0x9888), 0x04135000 },
-	{ _MMIO(0x9888), 0x00308000 },
-	{ _MMIO(0x9888), 0x06304000 },
-	{ _MMIO(0x9888), 0x00318000 },
-	{ _MMIO(0x9888), 0x06314000 },
-	{ _MMIO(0x9888), 0x00321b80 },
-	{ _MMIO(0x9888), 0x0632003f },
-	{ _MMIO(0x9888), 0x00334000 },
-	{ _MMIO(0x9888), 0x06331000 },
-	{ _MMIO(0x9888), 0x0250c000 },
-	{ _MMIO(0x9888), 0x0251c000 },
-	{ _MMIO(0x9888), 0x02521fb7 },
-	{ _MMIO(0x9888), 0x00520000 },
-	{ _MMIO(0x9888), 0x02535000 },
-	{ _MMIO(0x9888), 0x1190fc00 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900800 },
-	{ _MMIO(0x9888), 0x43900063 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900040 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_tdl_2_mux_config(struct drm_i915_private *dev_priv,
-		     const struct i915_oa_reg **regs,
-		     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_tdl_2;
-	lens[n] = ARRAY_SIZE(mux_config_tdl_2);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_extra[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_extra[] = {
-	{ _MMIO(0xe458), 0x00001000 },
-	{ _MMIO(0xe558), 0x00003002 },
-	{ _MMIO(0xe658), 0x00005004 },
-	{ _MMIO(0xe758), 0x00011010 },
-	{ _MMIO(0xe45c), 0x00050012 },
-	{ _MMIO(0xe55c), 0x00052051 },
-	{ _MMIO(0xe65c), 0x00000008 },
-};
-
-static const struct i915_oa_reg mux_config_compute_extra[] = {
-	{ _MMIO(0x9888), 0x121203e0 },
-	{ _MMIO(0x9888), 0x123203e0 },
-	{ _MMIO(0x9888), 0x125203e0 },
-	{ _MMIO(0x9888), 0x022f4000 },
-	{ _MMIO(0x9888), 0x0a4c0040 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x0e0f006c },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x042d8000 },
-	{ _MMIO(0x9888), 0x06104000 },
-	{ _MMIO(0x9888), 0x06114000 },
-	{ _MMIO(0x9888), 0x06120033 },
-	{ _MMIO(0x9888), 0x00120000 },
-	{ _MMIO(0x9888), 0x06131000 },
-	{ _MMIO(0x9888), 0x04308000 },
-	{ _MMIO(0x9888), 0x04318000 },
-	{ _MMIO(0x9888), 0x04321980 },
-	{ _MMIO(0x9888), 0x00320000 },
-	{ _MMIO(0x9888), 0x04334000 },
-	{ _MMIO(0x9888), 0x04504000 },
-	{ _MMIO(0x9888), 0x04514000 },
-	{ _MMIO(0x9888), 0x04520033 },
-	{ _MMIO(0x9888), 0x00520000 },
-	{ _MMIO(0x9888), 0x04531000 },
-	{ _MMIO(0x9888), 0x1190e000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x43900c00 },
-	{ _MMIO(0x9888), 0x45900002 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_compute_extra_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_extra;
-	lens[n] = ARRAY_SIZE(mux_config_compute_extra);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_vme_pipe[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2770), 0x00100030 },
-	{ _MMIO(0x2774), 0x0000fff9 },
-	{ _MMIO(0x2778), 0x00000002 },
-	{ _MMIO(0x277c), 0x0000fffc },
-	{ _MMIO(0x2780), 0x00000002 },
-	{ _MMIO(0x2784), 0x0000fff3 },
-	{ _MMIO(0x2788), 0x00100180 },
-	{ _MMIO(0x278c), 0x0000ffcf },
-	{ _MMIO(0x2790), 0x00000002 },
-	{ _MMIO(0x2794), 0x0000ffcf },
-	{ _MMIO(0x2798), 0x00000002 },
-	{ _MMIO(0x279c), 0x0000ff3f },
-};
-
-static const struct i915_oa_reg flex_eu_config_vme_pipe[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00008003 },
-};
-
-static const struct i915_oa_reg mux_config_vme_pipe[] = {
-	{ _MMIO(0x9888), 0x141a5800 },
-	{ _MMIO(0x9888), 0x161a00c0 },
-	{ _MMIO(0x9888), 0x12180240 },
-	{ _MMIO(0x9888), 0x14180002 },
-	{ _MMIO(0x9888), 0x143a5800 },
-	{ _MMIO(0x9888), 0x163a00c0 },
-	{ _MMIO(0x9888), 0x12380240 },
-	{ _MMIO(0x9888), 0x14380002 },
-	{ _MMIO(0x9888), 0x002f1000 },
-	{ _MMIO(0x9888), 0x022f8000 },
-	{ _MMIO(0x9888), 0x042f3000 },
-	{ _MMIO(0x9888), 0x004c4000 },
-	{ _MMIO(0x9888), 0x0a4c1500 },
-	{ _MMIO(0x9888), 0x000d2000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0a0da000 },
-	{ _MMIO(0x9888), 0x0c0da000 },
-	{ _MMIO(0x9888), 0x0c0f0400 },
-	{ _MMIO(0x9888), 0x0e0f9500 },
-	{ _MMIO(0x9888), 0x100f002a },
-	{ _MMIO(0x9888), 0x002c8000 },
-	{ _MMIO(0x9888), 0x0e2c8000 },
-	{ _MMIO(0x9888), 0x162c0a00 },
-	{ _MMIO(0x9888), 0x0a2dc000 },
-	{ _MMIO(0x9888), 0x0c2dc000 },
-	{ _MMIO(0x9888), 0x04193000 },
-	{ _MMIO(0x9888), 0x081a28c1 },
-	{ _MMIO(0x9888), 0x001a0000 },
-	{ _MMIO(0x9888), 0x00133000 },
-	{ _MMIO(0x9888), 0x0613c000 },
-	{ _MMIO(0x9888), 0x0813f000 },
-	{ _MMIO(0x9888), 0x00172000 },
-	{ _MMIO(0x9888), 0x06178000 },
-	{ _MMIO(0x9888), 0x0817a000 },
-	{ _MMIO(0x9888), 0x00180037 },
-	{ _MMIO(0x9888), 0x06180940 },
-	{ _MMIO(0x9888), 0x08180000 },
-	{ _MMIO(0x9888), 0x02180000 },
-	{ _MMIO(0x9888), 0x04183000 },
-	{ _MMIO(0x9888), 0x06393000 },
-	{ _MMIO(0x9888), 0x0c3a28c1 },
-	{ _MMIO(0x9888), 0x003a0000 },
-	{ _MMIO(0x9888), 0x0a33f000 },
-	{ _MMIO(0x9888), 0x0c33f000 },
-	{ _MMIO(0x9888), 0x0a37a000 },
-	{ _MMIO(0x9888), 0x0c37a000 },
-	{ _MMIO(0x9888), 0x0a380977 },
-	{ _MMIO(0x9888), 0x08380000 },
-	{ _MMIO(0x9888), 0x04380000 },
-	{ _MMIO(0x9888), 0x06383000 },
-	{ _MMIO(0x9888), 0x119000ff },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900040 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900800 },
-	{ _MMIO(0x9888), 0x47901000 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900844 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_vme_pipe_mux_config(struct drm_i915_private *dev_priv,
-			const struct i915_oa_reg **regs,
-			int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_vme_pipe;
-	lens[n] = ARRAY_SIZE(mux_config_vme_pipe);
-	n++;
-
-	return n;
-}
-
 static const struct i915_oa_reg b_counter_config_test_oa[] = {
 	{ _MMIO(0x2740), 0x00000000 },
 	{ _MMIO(0x2714), 0xf0800000 },
@@ -2370,6 +59,7 @@ static const struct i915_oa_reg flex_eu_config_test_oa[] = {
 };
 
 static const struct i915_oa_reg mux_config_test_oa[] = {
+	{ _MMIO(0x9840), 0x00000080 },
 	{ _MMIO(0x9888), 0x11810000 },
 	{ _MMIO(0x9888), 0x07810016 },
 	{ _MMIO(0x9888), 0x1f810000 },
@@ -2384,1096 +74,35 @@ static const struct i915_oa_reg mux_config_test_oa[] = {
 	{ _MMIO(0x9888), 0x33900000 },
 };
 
-static int
-get_test_oa_mux_config(struct drm_i915_private *dev_priv,
-		       const struct i915_oa_reg **regs,
-		       int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_test_oa;
-	lens[n] = ARRAY_SIZE(mux_config_test_oa);
-	n++;
-
-	return n;
-}
-
-int i915_oa_select_metric_set_sklgt2(struct drm_i915_private *dev_priv)
-{
-	dev_priv->perf.oa.n_mux_configs = 0;
-	dev_priv->perf.oa.b_counter_regs = NULL;
-	dev_priv->perf.oa.b_counter_regs_len = 0;
-	dev_priv->perf.oa.flex_regs = NULL;
-	dev_priv->perf.oa.flex_regs_len = 0;
-
-	switch (dev_priv->perf.oa.metrics_set) {
-	case METRIC_SET_ID_RENDER_BASIC:
-		dev_priv->perf.oa.n_mux_configs =
-			get_render_basic_mux_config(dev_priv,
-						    dev_priv->perf.oa.mux_regs,
-						    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_BASIC\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_render_basic;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_render_basic);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_render_basic;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_render_basic);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_BASIC:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_basic_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_BASIC\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_basic;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_basic);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_basic;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_basic);
-
-		return 0;
-	case METRIC_SET_ID_RENDER_PIPE_PROFILE:
-		dev_priv->perf.oa.n_mux_configs =
-			get_render_pipe_profile_mux_config(dev_priv,
-							   dev_priv->perf.oa.mux_regs,
-							   dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_PIPE_PROFILE\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_render_pipe_profile;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_render_pipe_profile);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_render_pipe_profile;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_render_pipe_profile);
-
-		return 0;
-	case METRIC_SET_ID_MEMORY_READS:
-		dev_priv->perf.oa.n_mux_configs =
-			get_memory_reads_mux_config(dev_priv,
-						    dev_priv->perf.oa.mux_regs,
-						    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_READS\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_memory_reads;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_memory_reads);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_memory_reads;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_memory_reads);
-
-		return 0;
-	case METRIC_SET_ID_MEMORY_WRITES:
-		dev_priv->perf.oa.n_mux_configs =
-			get_memory_writes_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_WRITES\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_memory_writes;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_memory_writes);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_memory_writes;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_memory_writes);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_EXTENDED:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_extended_mux_config(dev_priv,
-							dev_priv->perf.oa.mux_regs,
-							dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTENDED\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_extended;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_extended);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_extended;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_extended);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_L3_CACHE:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_l3_cache_mux_config(dev_priv,
-							dev_priv->perf.oa.mux_regs,
-							dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_L3_CACHE\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_l3_cache;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_l3_cache);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_l3_cache;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_l3_cache);
-
-		return 0;
-	case METRIC_SET_ID_HDC_AND_SF:
-		dev_priv->perf.oa.n_mux_configs =
-			get_hdc_and_sf_mux_config(dev_priv,
-						  dev_priv->perf.oa.mux_regs,
-						  dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"HDC_AND_SF\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_hdc_and_sf;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_hdc_and_sf);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_hdc_and_sf;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_hdc_and_sf);
-
-		return 0;
-	case METRIC_SET_ID_L3_1:
-		dev_priv->perf.oa.n_mux_configs =
-			get_l3_1_mux_config(dev_priv,
-					    dev_priv->perf.oa.mux_regs,
-					    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_1\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_l3_1;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_l3_1);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_l3_1;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_l3_1);
-
-		return 0;
-	case METRIC_SET_ID_L3_2:
-		dev_priv->perf.oa.n_mux_configs =
-			get_l3_2_mux_config(dev_priv,
-					    dev_priv->perf.oa.mux_regs,
-					    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_2\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_l3_2;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_l3_2);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_l3_2;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_l3_2);
-
-		return 0;
-	case METRIC_SET_ID_L3_3:
-		dev_priv->perf.oa.n_mux_configs =
-			get_l3_3_mux_config(dev_priv,
-					    dev_priv->perf.oa.mux_regs,
-					    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_3\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_l3_3;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_l3_3);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_l3_3;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_l3_3);
-
-		return 0;
-	case METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND:
-		dev_priv->perf.oa.n_mux_configs =
-			get_rasterizer_and_pixel_backend_mux_config(dev_priv,
-								    dev_priv->perf.oa.mux_regs,
-								    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RASTERIZER_AND_PIXEL_BACKEND\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_rasterizer_and_pixel_backend;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_rasterizer_and_pixel_backend);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_rasterizer_and_pixel_backend;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_rasterizer_and_pixel_backend);
-
-		return 0;
-	case METRIC_SET_ID_SAMPLER:
-		dev_priv->perf.oa.n_mux_configs =
-			get_sampler_mux_config(dev_priv,
-					       dev_priv->perf.oa.mux_regs,
-					       dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"SAMPLER\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_sampler;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_sampler);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_sampler;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_sampler);
-
-		return 0;
-	case METRIC_SET_ID_TDL_1:
-		dev_priv->perf.oa.n_mux_configs =
-			get_tdl_1_mux_config(dev_priv,
-					     dev_priv->perf.oa.mux_regs,
-					     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_1\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_tdl_1;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_tdl_1);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_tdl_1;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_tdl_1);
-
-		return 0;
-	case METRIC_SET_ID_TDL_2:
-		dev_priv->perf.oa.n_mux_configs =
-			get_tdl_2_mux_config(dev_priv,
-					     dev_priv->perf.oa.mux_regs,
-					     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_2\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_tdl_2;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_tdl_2);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_tdl_2;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_tdl_2);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_EXTRA:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_extra_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTRA\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_extra;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_extra);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_extra;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_extra);
-
-		return 0;
-	case METRIC_SET_ID_VME_PIPE:
-		dev_priv->perf.oa.n_mux_configs =
-			get_vme_pipe_mux_config(dev_priv,
-						dev_priv->perf.oa.mux_regs,
-						dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"VME_PIPE\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_vme_pipe;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_vme_pipe);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_vme_pipe;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_vme_pipe);
-
-		return 0;
-	case METRIC_SET_ID_TEST_OA:
-		dev_priv->perf.oa.n_mux_configs =
-			get_test_oa_mux_config(dev_priv,
-					       dev_priv->perf.oa.mux_regs,
-					       dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TEST_OA\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_test_oa;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_test_oa);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_test_oa;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_test_oa);
-
-		return 0;
-	default:
-		return -ENODEV;
-	}
-}
-
-static ssize_t
-show_render_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_BASIC);
-}
-
-static struct device_attribute dev_attr_render_basic_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_render_basic_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_render_basic[] = {
-	&dev_attr_render_basic_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_render_basic = {
-	.name = "f519e481-24d2-4d42-87c9-3fdd12c00202",
-	.attrs =  attrs_render_basic,
-};
-
-static ssize_t
-show_compute_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_BASIC);
-}
-
-static struct device_attribute dev_attr_compute_basic_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_basic_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_basic[] = {
-	&dev_attr_compute_basic_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_basic = {
-	.name = "fe47b29d-ae51-423e-bff4-27d965a95b60",
-	.attrs =  attrs_compute_basic,
-};
-
-static ssize_t
-show_render_pipe_profile_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_PIPE_PROFILE);
-}
-
-static struct device_attribute dev_attr_render_pipe_profile_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_render_pipe_profile_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_render_pipe_profile[] = {
-	&dev_attr_render_pipe_profile_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_render_pipe_profile = {
-	.name = "e0ad5ae0-84ba-4f29-a723-1906c12cb774",
-	.attrs =  attrs_render_pipe_profile,
-};
-
-static ssize_t
-show_memory_reads_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_READS);
-}
-
-static struct device_attribute dev_attr_memory_reads_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_memory_reads_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_memory_reads[] = {
-	&dev_attr_memory_reads_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_memory_reads = {
-	.name = "9bc436dd-6130-4add-affc-283eb6eaa864",
-	.attrs =  attrs_memory_reads,
-};
-
-static ssize_t
-show_memory_writes_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_WRITES);
-}
-
-static struct device_attribute dev_attr_memory_writes_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_memory_writes_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_memory_writes[] = {
-	&dev_attr_memory_writes_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_memory_writes = {
-	.name = "2ea0da8f-3527-4669-9d9d-13099a7435bf",
-	.attrs =  attrs_memory_writes,
-};
-
-static ssize_t
-show_compute_extended_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTENDED);
-}
-
-static struct device_attribute dev_attr_compute_extended_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_extended_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_extended[] = {
-	&dev_attr_compute_extended_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_extended = {
-	.name = "d97d16af-028b-4cd1-a672-6210cb5513dd",
-	.attrs =  attrs_compute_extended,
-};
-
-static ssize_t
-show_compute_l3_cache_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_L3_CACHE);
-}
-
-static struct device_attribute dev_attr_compute_l3_cache_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_l3_cache_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_l3_cache[] = {
-	&dev_attr_compute_l3_cache_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_l3_cache = {
-	.name = "9fb22842-e708-43f7-9752-e0e41670c39e",
-	.attrs =  attrs_compute_l3_cache,
-};
-
-static ssize_t
-show_hdc_and_sf_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_HDC_AND_SF);
-}
-
-static struct device_attribute dev_attr_hdc_and_sf_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_hdc_and_sf_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_hdc_and_sf[] = {
-	&dev_attr_hdc_and_sf_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_hdc_and_sf = {
-	.name = "5378e2a1-4248-4188-a4ae-da25a794c603",
-	.attrs =  attrs_hdc_and_sf,
-};
-
-static ssize_t
-show_l3_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_1);
-}
-
-static struct device_attribute dev_attr_l3_1_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_l3_1_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_l3_1[] = {
-	&dev_attr_l3_1_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_l3_1 = {
-	.name = "f42cdd6a-b000-42cb-870f-5eb423a7f514",
-	.attrs =  attrs_l3_1,
-};
-
-static ssize_t
-show_l3_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_2);
-}
-
-static struct device_attribute dev_attr_l3_2_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_l3_2_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_l3_2[] = {
-	&dev_attr_l3_2_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_l3_2 = {
-	.name = "b9bf2423-d88c-4a7b-a051-627611d00dcc",
-	.attrs =  attrs_l3_2,
-};
-
-static ssize_t
-show_l3_3_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_3);
-}
-
-static struct device_attribute dev_attr_l3_3_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_l3_3_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_l3_3[] = {
-	&dev_attr_l3_3_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_l3_3 = {
-	.name = "2414a93d-d84f-406e-99c0-472161194b40",
-	.attrs =  attrs_l3_3,
-};
-
-static ssize_t
-show_rasterizer_and_pixel_backend_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND);
-}
-
-static struct device_attribute dev_attr_rasterizer_and_pixel_backend_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_rasterizer_and_pixel_backend_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_rasterizer_and_pixel_backend[] = {
-	&dev_attr_rasterizer_and_pixel_backend_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_rasterizer_and_pixel_backend = {
-	.name = "53a45d2d-170b-4cf5-b7bb-585120c8e2f5",
-	.attrs =  attrs_rasterizer_and_pixel_backend,
-};
-
-static ssize_t
-show_sampler_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_SAMPLER);
-}
-
-static struct device_attribute dev_attr_sampler_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_sampler_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_sampler[] = {
-	&dev_attr_sampler_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_sampler = {
-	.name = "b4cff514-a91e-4798-a0b3-426ca13fc9c1",
-	.attrs =  attrs_sampler,
-};
-
-static ssize_t
-show_tdl_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_1);
-}
-
-static struct device_attribute dev_attr_tdl_1_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_tdl_1_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_tdl_1[] = {
-	&dev_attr_tdl_1_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_tdl_1 = {
-	.name = "7821d13b-9b8b-4405-9618-78cd56b62cce",
-	.attrs =  attrs_tdl_1,
-};
-
-static ssize_t
-show_tdl_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_2);
-}
-
-static struct device_attribute dev_attr_tdl_2_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_tdl_2_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_tdl_2[] = {
-	&dev_attr_tdl_2_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_tdl_2 = {
-	.name = "893f1a4d-919d-4388-8cb7-746d73ea7259",
-	.attrs =  attrs_tdl_2,
-};
-
-static ssize_t
-show_compute_extra_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTRA);
-}
-
-static struct device_attribute dev_attr_compute_extra_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_extra_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_extra[] = {
-	&dev_attr_compute_extra_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_extra = {
-	.name = "41a24047-7484-4ead-ae37-de907e5ff2b2",
-	.attrs =  attrs_compute_extra,
-};
-
-static ssize_t
-show_vme_pipe_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_VME_PIPE);
-}
-
-static struct device_attribute dev_attr_vme_pipe_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_vme_pipe_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_vme_pipe[] = {
-	&dev_attr_vme_pipe_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_vme_pipe = {
-	.name = "95910492-943f-44bd-9461-390240f243fd",
-	.attrs =  attrs_vme_pipe,
-};
-
 static ssize_t
 show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TEST_OA);
+	return sprintf(buf, "1\n");
 }
 
-static struct device_attribute dev_attr_test_oa_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_test_oa_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_test_oa[] = {
-	&dev_attr_test_oa_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_test_oa = {
-	.name = "1651949f-0ac0-4cb1-a06f-dafd74a407d1",
-	.attrs =  attrs_test_oa,
-};
-
-int
-i915_perf_register_sysfs_sklgt2(struct drm_i915_private *dev_priv)
+void
+i915_perf_load_test_config_sklgt2(struct drm_i915_private *dev_priv)
 {
-	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
-	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
-	int ret = 0;
+	strncpy(dev_priv->perf.oa.test_config.uuid,
+		"1651949f-0ac0-4cb1-a06f-dafd74a407d1",
+		UUID_STRING_LEN);
+	dev_priv->perf.oa.test_config.id = 1;
 
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-		if (ret)
-			goto error_render_basic;
-	}
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-		if (ret)
-			goto error_compute_basic;
-	}
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-		if (ret)
-			goto error_render_pipe_profile;
-	}
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-		if (ret)
-			goto error_memory_reads;
-	}
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-		if (ret)
-			goto error_memory_writes;
-	}
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-		if (ret)
-			goto error_compute_extended;
-	}
-	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
-		if (ret)
-			goto error_compute_l3_cache;
-	}
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-		if (ret)
-			goto error_hdc_and_sf;
-	}
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-		if (ret)
-			goto error_l3_1;
-	}
-	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_2);
-		if (ret)
-			goto error_l3_2;
-	}
-	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_3);
-		if (ret)
-			goto error_l3_3;
-	}
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-		if (ret)
-			goto error_rasterizer_and_pixel_backend;
-	}
-	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_sampler);
-		if (ret)
-			goto error_sampler;
-	}
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-		if (ret)
-			goto error_tdl_1;
-	}
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-		if (ret)
-			goto error_tdl_2;
-	}
-	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
-		if (ret)
-			goto error_compute_extra;
-	}
-	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
-		if (ret)
-			goto error_vme_pipe;
-	}
-	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_test_oa);
-		if (ret)
-			goto error_test_oa;
-	}
+	dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
+	dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa);
 
-	return 0;
+	dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa;
+	dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa);
 
-error_test_oa:
-	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
-error_vme_pipe:
-	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
-error_compute_extra:
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-error_tdl_2:
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-error_tdl_1:
-	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
-error_sampler:
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-error_rasterizer_and_pixel_backend:
-	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
-error_l3_3:
-	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
-error_l3_2:
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-error_l3_1:
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-error_hdc_and_sf:
-	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
-error_compute_l3_cache:
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-error_compute_extended:
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-error_memory_writes:
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-error_memory_reads:
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-error_render_pipe_profile:
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-error_compute_basic:
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-error_render_basic:
-	return ret;
-}
+	dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa;
+	dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa);
 
-void
-i915_perf_unregister_sysfs_sklgt2(struct drm_i915_private *dev_priv)
-{
-	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
-	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+	dev_priv->perf.oa.test_config.sysfs_metric.name = "1651949f-0ac0-4cb1-a06f-dafd74a407d1";
+	dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs;
+
+	dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr;
 
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
-	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
-	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
-	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id";
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444;
+	dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id;
 }
diff --git a/drivers/gpu/drm/i915/i915_oa_sklgt2.h b/drivers/gpu/drm/i915/i915_oa_sklgt2.h
index f4397baf3328..fe1aa2c03958 100644
--- a/drivers/gpu/drm/i915/i915_oa_sklgt2.h
+++ b/drivers/gpu/drm/i915/i915_oa_sklgt2.h
@@ -29,12 +29,6 @@
 #ifndef __I915_OA_SKLGT2_H__
 #define __I915_OA_SKLGT2_H__
 
-extern int i915_oa_n_builtin_metric_sets_sklgt2;
-
-extern int i915_oa_select_metric_set_sklgt2(struct drm_i915_private *dev_priv);
-
-extern int i915_perf_register_sysfs_sklgt2(struct drm_i915_private *dev_priv);
-
-extern void i915_perf_unregister_sysfs_sklgt2(struct drm_i915_private *dev_priv);
+extern void i915_perf_load_test_config_sklgt2(struct drm_i915_private *dev_priv);
 
 #endif
diff --git a/drivers/gpu/drm/i915/i915_oa_sklgt3.c b/drivers/gpu/drm/i915/i915_oa_sklgt3.c
index 7765e22dfa17..85e51addf86a 100644
--- a/drivers/gpu/drm/i915/i915_oa_sklgt3.c
+++ b/drivers/gpu/drm/i915/i915_oa_sklgt3.c
@@ -31,1876 +31,6 @@
 #include "i915_drv.h"
 #include "i915_oa_sklgt3.h"
 
-enum metric_set_id {
-	METRIC_SET_ID_RENDER_BASIC = 1,
-	METRIC_SET_ID_COMPUTE_BASIC,
-	METRIC_SET_ID_RENDER_PIPE_PROFILE,
-	METRIC_SET_ID_MEMORY_READS,
-	METRIC_SET_ID_MEMORY_WRITES,
-	METRIC_SET_ID_COMPUTE_EXTENDED,
-	METRIC_SET_ID_COMPUTE_L3_CACHE,
-	METRIC_SET_ID_HDC_AND_SF,
-	METRIC_SET_ID_L3_1,
-	METRIC_SET_ID_L3_2,
-	METRIC_SET_ID_L3_3,
-	METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND,
-	METRIC_SET_ID_SAMPLER,
-	METRIC_SET_ID_TDL_1,
-	METRIC_SET_ID_TDL_2,
-	METRIC_SET_ID_COMPUTE_EXTRA,
-	METRIC_SET_ID_VME_PIPE,
-	METRIC_SET_ID_TEST_OA,
-};
-
-int i915_oa_n_builtin_metric_sets_sklgt3 = 18;
-
-static const struct i915_oa_reg b_counter_config_render_basic[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2740), 0x00000000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_render_basic[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_render_basic[] = {
-	{ _MMIO(0x9888), 0x166c01e0 },
-	{ _MMIO(0x9888), 0x12170280 },
-	{ _MMIO(0x9888), 0x12370280 },
-	{ _MMIO(0x9888), 0x16ec01e0 },
-	{ _MMIO(0x9888), 0x11930317 },
-	{ _MMIO(0x9888), 0x159303df },
-	{ _MMIO(0x9888), 0x3f900003 },
-	{ _MMIO(0x9888), 0x1a4e0380 },
-	{ _MMIO(0x9888), 0x0a6c0053 },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x0a1b4000 },
-	{ _MMIO(0x9888), 0x1c1c0001 },
-	{ _MMIO(0x9888), 0x002f1000 },
-	{ _MMIO(0x9888), 0x042f1000 },
-	{ _MMIO(0x9888), 0x004c4000 },
-	{ _MMIO(0x9888), 0x0a4c8400 },
-	{ _MMIO(0x9888), 0x0c4c0002 },
-	{ _MMIO(0x9888), 0x000d2000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0a0da000 },
-	{ _MMIO(0x9888), 0x0c0f0400 },
-	{ _MMIO(0x9888), 0x0e0f6600 },
-	{ _MMIO(0x9888), 0x100f0001 },
-	{ _MMIO(0x9888), 0x002c8000 },
-	{ _MMIO(0x9888), 0x162ca200 },
-	{ _MMIO(0x9888), 0x062d8000 },
-	{ _MMIO(0x9888), 0x082d8000 },
-	{ _MMIO(0x9888), 0x00133000 },
-	{ _MMIO(0x9888), 0x08133000 },
-	{ _MMIO(0x9888), 0x00170020 },
-	{ _MMIO(0x9888), 0x08170021 },
-	{ _MMIO(0x9888), 0x10170000 },
-	{ _MMIO(0x9888), 0x0633c000 },
-	{ _MMIO(0x9888), 0x0833c000 },
-	{ _MMIO(0x9888), 0x06370800 },
-	{ _MMIO(0x9888), 0x08370840 },
-	{ _MMIO(0x9888), 0x10370000 },
-	{ _MMIO(0x9888), 0x1ace0200 },
-	{ _MMIO(0x9888), 0x0aec5300 },
-	{ _MMIO(0x9888), 0x10ec0000 },
-	{ _MMIO(0x9888), 0x1cec0000 },
-	{ _MMIO(0x9888), 0x0a9b8000 },
-	{ _MMIO(0x9888), 0x1c9c0002 },
-	{ _MMIO(0x9888), 0x0ccc0002 },
-	{ _MMIO(0x9888), 0x0a8d8000 },
-	{ _MMIO(0x9888), 0x108f0001 },
-	{ _MMIO(0x9888), 0x16ac8000 },
-	{ _MMIO(0x9888), 0x0d933031 },
-	{ _MMIO(0x9888), 0x0f933e3f },
-	{ _MMIO(0x9888), 0x01933d00 },
-	{ _MMIO(0x9888), 0x0393073c },
-	{ _MMIO(0x9888), 0x0593000e },
-	{ _MMIO(0x9888), 0x1d930000 },
-	{ _MMIO(0x9888), 0x19930000 },
-	{ _MMIO(0x9888), 0x1b930000 },
-	{ _MMIO(0x9888), 0x1d900157 },
-	{ _MMIO(0x9888), 0x1f900158 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x2b908000 },
-	{ _MMIO(0x9888), 0x2d908000 },
-	{ _MMIO(0x9888), 0x2f908000 },
-	{ _MMIO(0x9888), 0x31908000 },
-	{ _MMIO(0x9888), 0x15908000 },
-	{ _MMIO(0x9888), 0x17908000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1190003f },
-	{ _MMIO(0x9888), 0x51907710 },
-	{ _MMIO(0x9888), 0x419020a0 },
-	{ _MMIO(0x9888), 0x55901515 },
-	{ _MMIO(0x9888), 0x45900529 },
-	{ _MMIO(0x9888), 0x47901025 },
-	{ _MMIO(0x9888), 0x57907770 },
-	{ _MMIO(0x9888), 0x49902100 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900108 },
-	{ _MMIO(0x9888), 0x59900007 },
-	{ _MMIO(0x9888), 0x43902108 },
-	{ _MMIO(0x9888), 0x53907777 },
-};
-
-static int
-get_render_basic_mux_config(struct drm_i915_private *dev_priv,
-			    const struct i915_oa_reg **regs,
-			    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_render_basic;
-	lens[n] = ARRAY_SIZE(mux_config_render_basic);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_basic[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2740), 0x00000000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_basic[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00778008 },
-	{ _MMIO(0xe45c), 0x00088078 },
-	{ _MMIO(0xe55c), 0x00808708 },
-	{ _MMIO(0xe65c), 0x00a08908 },
-};
-
-static const struct i915_oa_reg mux_config_compute_basic[] = {
-	{ _MMIO(0x9888), 0x104f00e0 },
-	{ _MMIO(0x9888), 0x124f1c00 },
-	{ _MMIO(0x9888), 0x106c00e0 },
-	{ _MMIO(0x9888), 0x37906800 },
-	{ _MMIO(0x9888), 0x3f900003 },
-	{ _MMIO(0x9888), 0x004e8000 },
-	{ _MMIO(0x9888), 0x1a4e0820 },
-	{ _MMIO(0x9888), 0x1c4e0002 },
-	{ _MMIO(0x9888), 0x064f0900 },
-	{ _MMIO(0x9888), 0x084f0032 },
-	{ _MMIO(0x9888), 0x0a4f1891 },
-	{ _MMIO(0x9888), 0x0c4f0e00 },
-	{ _MMIO(0x9888), 0x0e4f003c },
-	{ _MMIO(0x9888), 0x004f0d80 },
-	{ _MMIO(0x9888), 0x024f003b },
-	{ _MMIO(0x9888), 0x006c0002 },
-	{ _MMIO(0x9888), 0x086c0100 },
-	{ _MMIO(0x9888), 0x0c6c000c },
-	{ _MMIO(0x9888), 0x0e6c0b00 },
-	{ _MMIO(0x9888), 0x186c0000 },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1e6c0000 },
-	{ _MMIO(0x9888), 0x001b4000 },
-	{ _MMIO(0x9888), 0x081b8000 },
-	{ _MMIO(0x9888), 0x0c1b4000 },
-	{ _MMIO(0x9888), 0x0e1b8000 },
-	{ _MMIO(0x9888), 0x101c8000 },
-	{ _MMIO(0x9888), 0x1a1c8000 },
-	{ _MMIO(0x9888), 0x1c1c0024 },
-	{ _MMIO(0x9888), 0x065b8000 },
-	{ _MMIO(0x9888), 0x085b4000 },
-	{ _MMIO(0x9888), 0x0a5bc000 },
-	{ _MMIO(0x9888), 0x0c5b8000 },
-	{ _MMIO(0x9888), 0x0e5b4000 },
-	{ _MMIO(0x9888), 0x005b8000 },
-	{ _MMIO(0x9888), 0x025b4000 },
-	{ _MMIO(0x9888), 0x1a5c6000 },
-	{ _MMIO(0x9888), 0x1c5c001b },
-	{ _MMIO(0x9888), 0x125c8000 },
-	{ _MMIO(0x9888), 0x145c8000 },
-	{ _MMIO(0x9888), 0x004c8000 },
-	{ _MMIO(0x9888), 0x0a4c2000 },
-	{ _MMIO(0x9888), 0x0c4c0208 },
-	{ _MMIO(0x9888), 0x000da000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0a0da000 },
-	{ _MMIO(0x9888), 0x0c0da000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x020d2000 },
-	{ _MMIO(0x9888), 0x0c0f5400 },
-	{ _MMIO(0x9888), 0x0e0f5500 },
-	{ _MMIO(0x9888), 0x100f0155 },
-	{ _MMIO(0x9888), 0x002c8000 },
-	{ _MMIO(0x9888), 0x0e2cc000 },
-	{ _MMIO(0x9888), 0x162cfb00 },
-	{ _MMIO(0x9888), 0x182c00be },
-	{ _MMIO(0x9888), 0x022cc000 },
-	{ _MMIO(0x9888), 0x042cc000 },
-	{ _MMIO(0x9888), 0x19900157 },
-	{ _MMIO(0x9888), 0x1b900158 },
-	{ _MMIO(0x9888), 0x1d900105 },
-	{ _MMIO(0x9888), 0x1f900103 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x11900fff },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900800 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900863 },
-	{ _MMIO(0x9888), 0x47900802 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900802 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900002 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900c62 },
-	{ _MMIO(0x9888), 0x53903333 },
-};
-
-static int
-get_compute_basic_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_basic;
-	lens[n] = ARRAY_SIZE(mux_config_compute_basic);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_render_pipe_profile[] = {
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007ffea },
-	{ _MMIO(0x2774), 0x00007ffc },
-	{ _MMIO(0x2778), 0x0007affa },
-	{ _MMIO(0x277c), 0x0000f5fd },
-	{ _MMIO(0x2780), 0x00079ffa },
-	{ _MMIO(0x2784), 0x0000f3fb },
-	{ _MMIO(0x2788), 0x0007bf7a },
-	{ _MMIO(0x278c), 0x0000f7e7 },
-	{ _MMIO(0x2790), 0x0007fefa },
-	{ _MMIO(0x2794), 0x0000f7cf },
-	{ _MMIO(0x2798), 0x00077ffa },
-	{ _MMIO(0x279c), 0x0000efdf },
-	{ _MMIO(0x27a0), 0x0006fffa },
-	{ _MMIO(0x27a4), 0x0000cfbf },
-	{ _MMIO(0x27a8), 0x0003fffa },
-	{ _MMIO(0x27ac), 0x00005f7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_render_pipe_profile[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00015014 },
-	{ _MMIO(0xe658), 0x00025024 },
-	{ _MMIO(0xe758), 0x00035034 },
-	{ _MMIO(0xe45c), 0x00045044 },
-	{ _MMIO(0xe55c), 0x00055054 },
-	{ _MMIO(0xe65c), 0x00065064 },
-};
-
-static const struct i915_oa_reg mux_config_render_pipe_profile[] = {
-	{ _MMIO(0x9888), 0x0c0e001f },
-	{ _MMIO(0x9888), 0x0a0f0000 },
-	{ _MMIO(0x9888), 0x10116800 },
-	{ _MMIO(0x9888), 0x178a03e0 },
-	{ _MMIO(0x9888), 0x11824c00 },
-	{ _MMIO(0x9888), 0x11830020 },
-	{ _MMIO(0x9888), 0x13840020 },
-	{ _MMIO(0x9888), 0x11850019 },
-	{ _MMIO(0x9888), 0x11860007 },
-	{ _MMIO(0x9888), 0x01870c40 },
-	{ _MMIO(0x9888), 0x17880000 },
-	{ _MMIO(0x9888), 0x022f4000 },
-	{ _MMIO(0x9888), 0x0a4c0040 },
-	{ _MMIO(0x9888), 0x0c0d8000 },
-	{ _MMIO(0x9888), 0x040d4000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x020e5400 },
-	{ _MMIO(0x9888), 0x000e0000 },
-	{ _MMIO(0x9888), 0x080f0040 },
-	{ _MMIO(0x9888), 0x000f0000 },
-	{ _MMIO(0x9888), 0x100f0000 },
-	{ _MMIO(0x9888), 0x0e0f0040 },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x06104000 },
-	{ _MMIO(0x9888), 0x06110012 },
-	{ _MMIO(0x9888), 0x06131000 },
-	{ _MMIO(0x9888), 0x01898000 },
-	{ _MMIO(0x9888), 0x0d890100 },
-	{ _MMIO(0x9888), 0x03898000 },
-	{ _MMIO(0x9888), 0x09808000 },
-	{ _MMIO(0x9888), 0x0b808000 },
-	{ _MMIO(0x9888), 0x0380c000 },
-	{ _MMIO(0x9888), 0x0f8a0075 },
-	{ _MMIO(0x9888), 0x1d8a0000 },
-	{ _MMIO(0x9888), 0x118a8000 },
-	{ _MMIO(0x9888), 0x1b8a4000 },
-	{ _MMIO(0x9888), 0x138a8000 },
-	{ _MMIO(0x9888), 0x1d81a000 },
-	{ _MMIO(0x9888), 0x15818000 },
-	{ _MMIO(0x9888), 0x17818000 },
-	{ _MMIO(0x9888), 0x0b820030 },
-	{ _MMIO(0x9888), 0x07828000 },
-	{ _MMIO(0x9888), 0x0d824000 },
-	{ _MMIO(0x9888), 0x0f828000 },
-	{ _MMIO(0x9888), 0x05824000 },
-	{ _MMIO(0x9888), 0x0d830003 },
-	{ _MMIO(0x9888), 0x0583000c },
-	{ _MMIO(0x9888), 0x09830000 },
-	{ _MMIO(0x9888), 0x03838000 },
-	{ _MMIO(0x9888), 0x07838000 },
-	{ _MMIO(0x9888), 0x0b840980 },
-	{ _MMIO(0x9888), 0x03844d80 },
-	{ _MMIO(0x9888), 0x11840000 },
-	{ _MMIO(0x9888), 0x09848000 },
-	{ _MMIO(0x9888), 0x09850080 },
-	{ _MMIO(0x9888), 0x03850003 },
-	{ _MMIO(0x9888), 0x01850000 },
-	{ _MMIO(0x9888), 0x07860000 },
-	{ _MMIO(0x9888), 0x0f860400 },
-	{ _MMIO(0x9888), 0x09870032 },
-	{ _MMIO(0x9888), 0x01888052 },
-	{ _MMIO(0x9888), 0x11880000 },
-	{ _MMIO(0x9888), 0x09884000 },
-	{ _MMIO(0x9888), 0x1b931001 },
-	{ _MMIO(0x9888), 0x1d930001 },
-	{ _MMIO(0x9888), 0x19934000 },
-	{ _MMIO(0x9888), 0x1b958000 },
-	{ _MMIO(0x9888), 0x1d950094 },
-	{ _MMIO(0x9888), 0x19958000 },
-	{ _MMIO(0x9888), 0x09e58000 },
-	{ _MMIO(0x9888), 0x0be58000 },
-	{ _MMIO(0x9888), 0x03e5c000 },
-	{ _MMIO(0x9888), 0x0592c000 },
-	{ _MMIO(0x9888), 0x0b928000 },
-	{ _MMIO(0x9888), 0x0d924000 },
-	{ _MMIO(0x9888), 0x0f924000 },
-	{ _MMIO(0x9888), 0x11928000 },
-	{ _MMIO(0x9888), 0x1392c000 },
-	{ _MMIO(0x9888), 0x09924000 },
-	{ _MMIO(0x9888), 0x01985000 },
-	{ _MMIO(0x9888), 0x07988000 },
-	{ _MMIO(0x9888), 0x09981000 },
-	{ _MMIO(0x9888), 0x0b982000 },
-	{ _MMIO(0x9888), 0x0d982000 },
-	{ _MMIO(0x9888), 0x0f989000 },
-	{ _MMIO(0x9888), 0x05982000 },
-	{ _MMIO(0x9888), 0x13904000 },
-	{ _MMIO(0x9888), 0x21904000 },
-	{ _MMIO(0x9888), 0x23904000 },
-	{ _MMIO(0x9888), 0x25908000 },
-	{ _MMIO(0x9888), 0x27904000 },
-	{ _MMIO(0x9888), 0x29908000 },
-	{ _MMIO(0x9888), 0x2b904000 },
-	{ _MMIO(0x9888), 0x2f904000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x15904000 },
-	{ _MMIO(0x9888), 0x17908000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x1190c080 },
-	{ _MMIO(0x9888), 0x51901150 },
-	{ _MMIO(0x9888), 0x41901400 },
-	{ _MMIO(0x9888), 0x55905111 },
-	{ _MMIO(0x9888), 0x45901400 },
-	{ _MMIO(0x9888), 0x479004a5 },
-	{ _MMIO(0x9888), 0x57903455 },
-	{ _MMIO(0x9888), 0x49900000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b9000a0 },
-	{ _MMIO(0x9888), 0x59900001 },
-	{ _MMIO(0x9888), 0x43900005 },
-	{ _MMIO(0x9888), 0x53900455 },
-};
-
-static int
-get_render_pipe_profile_mux_config(struct drm_i915_private *dev_priv,
-				   const struct i915_oa_reg **regs,
-				   int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_render_pipe_profile;
-	lens[n] = ARRAY_SIZE(mux_config_render_pipe_profile);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_memory_reads[] = {
-	{ _MMIO(0x272c), 0xffffffff },
-	{ _MMIO(0x2728), 0xffffffff },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x271c), 0xffffffff },
-	{ _MMIO(0x2718), 0xffffffff },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x274c), 0x86543210 },
-	{ _MMIO(0x2748), 0x86543210 },
-	{ _MMIO(0x2744), 0x00006667 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x275c), 0x86543210 },
-	{ _MMIO(0x2758), 0x86543210 },
-	{ _MMIO(0x2754), 0x00006465 },
-	{ _MMIO(0x2750), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007f81a },
-	{ _MMIO(0x2774), 0x0000fe00 },
-	{ _MMIO(0x2778), 0x0007f82a },
-	{ _MMIO(0x277c), 0x0000fe00 },
-	{ _MMIO(0x2780), 0x0007f872 },
-	{ _MMIO(0x2784), 0x0000fe00 },
-	{ _MMIO(0x2788), 0x0007f8ba },
-	{ _MMIO(0x278c), 0x0000fe00 },
-	{ _MMIO(0x2790), 0x0007f87a },
-	{ _MMIO(0x2794), 0x0000fe00 },
-	{ _MMIO(0x2798), 0x0007f8ea },
-	{ _MMIO(0x279c), 0x0000fe00 },
-	{ _MMIO(0x27a0), 0x0007f8e2 },
-	{ _MMIO(0x27a4), 0x0000fe00 },
-	{ _MMIO(0x27a8), 0x0007f8f2 },
-	{ _MMIO(0x27ac), 0x0000fe00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_memory_reads[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00015014 },
-	{ _MMIO(0xe658), 0x00025024 },
-	{ _MMIO(0xe758), 0x00035034 },
-	{ _MMIO(0xe45c), 0x00045044 },
-	{ _MMIO(0xe55c), 0x00055054 },
-	{ _MMIO(0xe65c), 0x00065064 },
-};
-
-static const struct i915_oa_reg mux_config_memory_reads[] = {
-	{ _MMIO(0x9888), 0x11810c00 },
-	{ _MMIO(0x9888), 0x1381001a },
-	{ _MMIO(0x9888), 0x37906800 },
-	{ _MMIO(0x9888), 0x3f900064 },
-	{ _MMIO(0x9888), 0x03811300 },
-	{ _MMIO(0x9888), 0x05811b12 },
-	{ _MMIO(0x9888), 0x0781001a },
-	{ _MMIO(0x9888), 0x1f810000 },
-	{ _MMIO(0x9888), 0x17810000 },
-	{ _MMIO(0x9888), 0x19810000 },
-	{ _MMIO(0x9888), 0x1b810000 },
-	{ _MMIO(0x9888), 0x1d810000 },
-	{ _MMIO(0x9888), 0x1b930055 },
-	{ _MMIO(0x9888), 0x03e58000 },
-	{ _MMIO(0x9888), 0x05e5c000 },
-	{ _MMIO(0x9888), 0x07e54000 },
-	{ _MMIO(0x9888), 0x13900150 },
-	{ _MMIO(0x9888), 0x21900151 },
-	{ _MMIO(0x9888), 0x23900152 },
-	{ _MMIO(0x9888), 0x25900153 },
-	{ _MMIO(0x9888), 0x27900154 },
-	{ _MMIO(0x9888), 0x29900155 },
-	{ _MMIO(0x9888), 0x2b900156 },
-	{ _MMIO(0x9888), 0x2d900157 },
-	{ _MMIO(0x9888), 0x2f90015f },
-	{ _MMIO(0x9888), 0x31900105 },
-	{ _MMIO(0x9888), 0x15900103 },
-	{ _MMIO(0x9888), 0x17900101 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1d908000 },
-	{ _MMIO(0x9888), 0x1f908000 },
-	{ _MMIO(0x9888), 0x11900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c60 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900c00 },
-	{ _MMIO(0x9888), 0x47900c63 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900c63 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900063 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900003 },
-	{ _MMIO(0x9888), 0x53900000 },
-};
-
-static int
-get_memory_reads_mux_config(struct drm_i915_private *dev_priv,
-			    const struct i915_oa_reg **regs,
-			    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_memory_reads;
-	lens[n] = ARRAY_SIZE(mux_config_memory_reads);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_memory_writes[] = {
-	{ _MMIO(0x272c), 0xffffffff },
-	{ _MMIO(0x2728), 0xffffffff },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x271c), 0xffffffff },
-	{ _MMIO(0x2718), 0xffffffff },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x274c), 0x86543210 },
-	{ _MMIO(0x2748), 0x86543210 },
-	{ _MMIO(0x2744), 0x00006667 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x275c), 0x86543210 },
-	{ _MMIO(0x2758), 0x86543210 },
-	{ _MMIO(0x2754), 0x00006465 },
-	{ _MMIO(0x2750), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007f81a },
-	{ _MMIO(0x2774), 0x0000fe00 },
-	{ _MMIO(0x2778), 0x0007f82a },
-	{ _MMIO(0x277c), 0x0000fe00 },
-	{ _MMIO(0x2780), 0x0007f822 },
-	{ _MMIO(0x2784), 0x0000fe00 },
-	{ _MMIO(0x2788), 0x0007f8ba },
-	{ _MMIO(0x278c), 0x0000fe00 },
-	{ _MMIO(0x2790), 0x0007f87a },
-	{ _MMIO(0x2794), 0x0000fe00 },
-	{ _MMIO(0x2798), 0x0007f8ea },
-	{ _MMIO(0x279c), 0x0000fe00 },
-	{ _MMIO(0x27a0), 0x0007f8e2 },
-	{ _MMIO(0x27a4), 0x0000fe00 },
-	{ _MMIO(0x27a8), 0x0007f8f2 },
-	{ _MMIO(0x27ac), 0x0000fe00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_memory_writes[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00015014 },
-	{ _MMIO(0xe658), 0x00025024 },
-	{ _MMIO(0xe758), 0x00035034 },
-	{ _MMIO(0xe45c), 0x00045044 },
-	{ _MMIO(0xe55c), 0x00055054 },
-	{ _MMIO(0xe65c), 0x00065064 },
-};
-
-static const struct i915_oa_reg mux_config_memory_writes[] = {
-	{ _MMIO(0x9888), 0x11810c00 },
-	{ _MMIO(0x9888), 0x1381001a },
-	{ _MMIO(0x9888), 0x37906800 },
-	{ _MMIO(0x9888), 0x3f901000 },
-	{ _MMIO(0x9888), 0x03811300 },
-	{ _MMIO(0x9888), 0x05811b12 },
-	{ _MMIO(0x9888), 0x0781001a },
-	{ _MMIO(0x9888), 0x1f810000 },
-	{ _MMIO(0x9888), 0x17810000 },
-	{ _MMIO(0x9888), 0x19810000 },
-	{ _MMIO(0x9888), 0x1b810000 },
-	{ _MMIO(0x9888), 0x1d810000 },
-	{ _MMIO(0x9888), 0x1b930055 },
-	{ _MMIO(0x9888), 0x03e58000 },
-	{ _MMIO(0x9888), 0x05e5c000 },
-	{ _MMIO(0x9888), 0x07e54000 },
-	{ _MMIO(0x9888), 0x13900160 },
-	{ _MMIO(0x9888), 0x21900161 },
-	{ _MMIO(0x9888), 0x23900162 },
-	{ _MMIO(0x9888), 0x25900163 },
-	{ _MMIO(0x9888), 0x27900164 },
-	{ _MMIO(0x9888), 0x29900165 },
-	{ _MMIO(0x9888), 0x2b900166 },
-	{ _MMIO(0x9888), 0x2d900167 },
-	{ _MMIO(0x9888), 0x2f900150 },
-	{ _MMIO(0x9888), 0x31900105 },
-	{ _MMIO(0x9888), 0x15900103 },
-	{ _MMIO(0x9888), 0x17900101 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1d908000 },
-	{ _MMIO(0x9888), 0x1f908000 },
-	{ _MMIO(0x9888), 0x11900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c60 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900c00 },
-	{ _MMIO(0x9888), 0x47900c63 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900c63 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900063 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900003 },
-	{ _MMIO(0x9888), 0x53900000 },
-};
-
-static int
-get_memory_writes_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_memory_writes;
-	lens[n] = ARRAY_SIZE(mux_config_memory_writes);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_extended[] = {
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007fc2a },
-	{ _MMIO(0x2774), 0x0000bf00 },
-	{ _MMIO(0x2778), 0x0007fc6a },
-	{ _MMIO(0x277c), 0x0000bf00 },
-	{ _MMIO(0x2780), 0x0007fc92 },
-	{ _MMIO(0x2784), 0x0000bf00 },
-	{ _MMIO(0x2788), 0x0007fca2 },
-	{ _MMIO(0x278c), 0x0000bf00 },
-	{ _MMIO(0x2790), 0x0007fc32 },
-	{ _MMIO(0x2794), 0x0000bf00 },
-	{ _MMIO(0x2798), 0x0007fc9a },
-	{ _MMIO(0x279c), 0x0000bf00 },
-	{ _MMIO(0x27a0), 0x0007fe6a },
-	{ _MMIO(0x27a4), 0x0000bf00 },
-	{ _MMIO(0x27a8), 0x0007fe7a },
-	{ _MMIO(0x27ac), 0x0000bf00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_extended[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00778008 },
-	{ _MMIO(0xe45c), 0x00088078 },
-	{ _MMIO(0xe55c), 0x00808708 },
-	{ _MMIO(0xe65c), 0x00a08908 },
-};
-
-static const struct i915_oa_reg mux_config_compute_extended[] = {
-	{ _MMIO(0x9888), 0x106c00e0 },
-	{ _MMIO(0x9888), 0x141c8160 },
-	{ _MMIO(0x9888), 0x161c8015 },
-	{ _MMIO(0x9888), 0x181c0120 },
-	{ _MMIO(0x9888), 0x004e8000 },
-	{ _MMIO(0x9888), 0x0e4e8000 },
-	{ _MMIO(0x9888), 0x184e8000 },
-	{ _MMIO(0x9888), 0x1a4eaaa0 },
-	{ _MMIO(0x9888), 0x1c4e0002 },
-	{ _MMIO(0x9888), 0x024e8000 },
-	{ _MMIO(0x9888), 0x044e8000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084e8000 },
-	{ _MMIO(0x9888), 0x0a4e8000 },
-	{ _MMIO(0x9888), 0x0e6c0b01 },
-	{ _MMIO(0x9888), 0x006c0200 },
-	{ _MMIO(0x9888), 0x026c000c },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1e6c0000 },
-	{ _MMIO(0x9888), 0x1a6c0000 },
-	{ _MMIO(0x9888), 0x0e1bc000 },
-	{ _MMIO(0x9888), 0x001b8000 },
-	{ _MMIO(0x9888), 0x021bc000 },
-	{ _MMIO(0x9888), 0x001c0041 },
-	{ _MMIO(0x9888), 0x061c4200 },
-	{ _MMIO(0x9888), 0x081c4443 },
-	{ _MMIO(0x9888), 0x0a1c4645 },
-	{ _MMIO(0x9888), 0x0c1c7647 },
-	{ _MMIO(0x9888), 0x041c7357 },
-	{ _MMIO(0x9888), 0x1c1c0030 },
-	{ _MMIO(0x9888), 0x101c0000 },
-	{ _MMIO(0x9888), 0x1a1c0000 },
-	{ _MMIO(0x9888), 0x121c8000 },
-	{ _MMIO(0x9888), 0x004c8000 },
-	{ _MMIO(0x9888), 0x0a4caa2a },
-	{ _MMIO(0x9888), 0x0c4c02aa },
-	{ _MMIO(0x9888), 0x084ca000 },
-	{ _MMIO(0x9888), 0x000da000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0a0da000 },
-	{ _MMIO(0x9888), 0x0c0da000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x0c0f5400 },
-	{ _MMIO(0x9888), 0x0e0f5515 },
-	{ _MMIO(0x9888), 0x100f0155 },
-	{ _MMIO(0x9888), 0x002c8000 },
-	{ _MMIO(0x9888), 0x0e2c8000 },
-	{ _MMIO(0x9888), 0x162caa00 },
-	{ _MMIO(0x9888), 0x182c00aa },
-	{ _MMIO(0x9888), 0x022c8000 },
-	{ _MMIO(0x9888), 0x042c8000 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x11907fff },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900040 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900802 },
-	{ _MMIO(0x9888), 0x47900842 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900842 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900800 },
-	{ _MMIO(0x9888), 0x53900000 },
-};
-
-static int
-get_compute_extended_mux_config(struct drm_i915_private *dev_priv,
-				const struct i915_oa_reg **regs,
-				int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_extended;
-	lens[n] = ARRAY_SIZE(mux_config_compute_extended);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_l3_cache[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x30800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007fffa },
-	{ _MMIO(0x2774), 0x0000fefe },
-	{ _MMIO(0x2778), 0x0007fffa },
-	{ _MMIO(0x277c), 0x0000fefd },
-	{ _MMIO(0x2790), 0x0007fffa },
-	{ _MMIO(0x2794), 0x0000fbef },
-	{ _MMIO(0x2798), 0x0007fffa },
-	{ _MMIO(0x279c), 0x0000fbdf },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_l3_cache[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00101100 },
-	{ _MMIO(0xe45c), 0x00201200 },
-	{ _MMIO(0xe55c), 0x00301300 },
-	{ _MMIO(0xe65c), 0x00401400 },
-};
-
-static const struct i915_oa_reg mux_config_compute_l3_cache[] = {
-	{ _MMIO(0x9888), 0x166c0760 },
-	{ _MMIO(0x9888), 0x1593001e },
-	{ _MMIO(0x9888), 0x3f900003 },
-	{ _MMIO(0x9888), 0x004e8000 },
-	{ _MMIO(0x9888), 0x0e4e8000 },
-	{ _MMIO(0x9888), 0x184e8000 },
-	{ _MMIO(0x9888), 0x1a4e8020 },
-	{ _MMIO(0x9888), 0x1c4e0002 },
-	{ _MMIO(0x9888), 0x006c0051 },
-	{ _MMIO(0x9888), 0x066c5000 },
-	{ _MMIO(0x9888), 0x086c5c5d },
-	{ _MMIO(0x9888), 0x0e6c5e5f },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x186c0000 },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1e6c0000 },
-	{ _MMIO(0x9888), 0x001b4000 },
-	{ _MMIO(0x9888), 0x061b8000 },
-	{ _MMIO(0x9888), 0x081bc000 },
-	{ _MMIO(0x9888), 0x0e1bc000 },
-	{ _MMIO(0x9888), 0x101c8000 },
-	{ _MMIO(0x9888), 0x1a1ce000 },
-	{ _MMIO(0x9888), 0x1c1c0030 },
-	{ _MMIO(0x9888), 0x004c8000 },
-	{ _MMIO(0x9888), 0x0a4c2a00 },
-	{ _MMIO(0x9888), 0x0c4c0280 },
-	{ _MMIO(0x9888), 0x000d2000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x0c0f0400 },
-	{ _MMIO(0x9888), 0x0e0f1500 },
-	{ _MMIO(0x9888), 0x100f0140 },
-	{ _MMIO(0x9888), 0x002c8000 },
-	{ _MMIO(0x9888), 0x0e2c8000 },
-	{ _MMIO(0x9888), 0x162c0a00 },
-	{ _MMIO(0x9888), 0x182c00a0 },
-	{ _MMIO(0x9888), 0x03933300 },
-	{ _MMIO(0x9888), 0x05930032 },
-	{ _MMIO(0x9888), 0x11930000 },
-	{ _MMIO(0x9888), 0x1b930000 },
-	{ _MMIO(0x9888), 0x1d900157 },
-	{ _MMIO(0x9888), 0x1f900158 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1190030f },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900000 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900063 },
-	{ _MMIO(0x9888), 0x47900000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x4b900000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x53903333 },
-	{ _MMIO(0x9888), 0x43900840 },
-};
-
-static int
-get_compute_l3_cache_mux_config(struct drm_i915_private *dev_priv,
-				const struct i915_oa_reg **regs,
-				int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_l3_cache;
-	lens[n] = ARRAY_SIZE(mux_config_compute_l3_cache);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_hdc_and_sf[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x10800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x0000fdff },
-};
-
-static const struct i915_oa_reg flex_eu_config_hdc_and_sf[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_hdc_and_sf[] = {
-	{ _MMIO(0x9888), 0x104f0232 },
-	{ _MMIO(0x9888), 0x124f4640 },
-	{ _MMIO(0x9888), 0x106c0232 },
-	{ _MMIO(0x9888), 0x11834400 },
-	{ _MMIO(0x9888), 0x0a4e8000 },
-	{ _MMIO(0x9888), 0x0c4e8000 },
-	{ _MMIO(0x9888), 0x004f1880 },
-	{ _MMIO(0x9888), 0x024f08bb },
-	{ _MMIO(0x9888), 0x044f001b },
-	{ _MMIO(0x9888), 0x046c0100 },
-	{ _MMIO(0x9888), 0x066c000b },
-	{ _MMIO(0x9888), 0x1a6c0000 },
-	{ _MMIO(0x9888), 0x041b8000 },
-	{ _MMIO(0x9888), 0x061b4000 },
-	{ _MMIO(0x9888), 0x1a1c1800 },
-	{ _MMIO(0x9888), 0x005b8000 },
-	{ _MMIO(0x9888), 0x025bc000 },
-	{ _MMIO(0x9888), 0x045b4000 },
-	{ _MMIO(0x9888), 0x125c8000 },
-	{ _MMIO(0x9888), 0x145c8000 },
-	{ _MMIO(0x9888), 0x165c8000 },
-	{ _MMIO(0x9888), 0x185c8000 },
-	{ _MMIO(0x9888), 0x0a4c00a0 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x0c0f5000 },
-	{ _MMIO(0x9888), 0x0e0f0055 },
-	{ _MMIO(0x9888), 0x022cc000 },
-	{ _MMIO(0x9888), 0x042cc000 },
-	{ _MMIO(0x9888), 0x062cc000 },
-	{ _MMIO(0x9888), 0x082cc000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x0f828000 },
-	{ _MMIO(0x9888), 0x0f8305c0 },
-	{ _MMIO(0x9888), 0x09830000 },
-	{ _MMIO(0x9888), 0x07830000 },
-	{ _MMIO(0x9888), 0x1d950080 },
-	{ _MMIO(0x9888), 0x13928000 },
-	{ _MMIO(0x9888), 0x0f988000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x1190fc00 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x59900005 },
-	{ _MMIO(0x9888), 0x4b900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900800 },
-	{ _MMIO(0x9888), 0x43900842 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_hdc_and_sf_mux_config(struct drm_i915_private *dev_priv,
-			  const struct i915_oa_reg **regs,
-			  int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_hdc_and_sf;
-	lens[n] = ARRAY_SIZE(mux_config_hdc_and_sf);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_l3_1[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2770), 0x00100070 },
-	{ _MMIO(0x2774), 0x0000fff1 },
-	{ _MMIO(0x2778), 0x00014002 },
-	{ _MMIO(0x277c), 0x0000c3ff },
-	{ _MMIO(0x2780), 0x00010002 },
-	{ _MMIO(0x2784), 0x0000c7ff },
-	{ _MMIO(0x2788), 0x00004002 },
-	{ _MMIO(0x278c), 0x0000d3ff },
-	{ _MMIO(0x2790), 0x00100700 },
-	{ _MMIO(0x2794), 0x0000ff1f },
-	{ _MMIO(0x2798), 0x00001402 },
-	{ _MMIO(0x279c), 0x0000fc3f },
-	{ _MMIO(0x27a0), 0x00001002 },
-	{ _MMIO(0x27a4), 0x0000fc7f },
-	{ _MMIO(0x27a8), 0x00000402 },
-	{ _MMIO(0x27ac), 0x0000fd3f },
-};
-
-static const struct i915_oa_reg flex_eu_config_l3_1[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_l3_1[] = {
-	{ _MMIO(0x9888), 0x126c7b40 },
-	{ _MMIO(0x9888), 0x166c0020 },
-	{ _MMIO(0x9888), 0x0a603444 },
-	{ _MMIO(0x9888), 0x0a613400 },
-	{ _MMIO(0x9888), 0x1a4ea800 },
-	{ _MMIO(0x9888), 0x1c4e0002 },
-	{ _MMIO(0x9888), 0x024e8000 },
-	{ _MMIO(0x9888), 0x044e8000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084e8000 },
-	{ _MMIO(0x9888), 0x0a4e8000 },
-	{ _MMIO(0x9888), 0x064f4000 },
-	{ _MMIO(0x9888), 0x0c6c5327 },
-	{ _MMIO(0x9888), 0x0e6c5425 },
-	{ _MMIO(0x9888), 0x006c2a00 },
-	{ _MMIO(0x9888), 0x026c285b },
-	{ _MMIO(0x9888), 0x046c005c },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1e6c0000 },
-	{ _MMIO(0x9888), 0x1a6c0800 },
-	{ _MMIO(0x9888), 0x0c1bc000 },
-	{ _MMIO(0x9888), 0x0e1bc000 },
-	{ _MMIO(0x9888), 0x001b8000 },
-	{ _MMIO(0x9888), 0x021bc000 },
-	{ _MMIO(0x9888), 0x041bc000 },
-	{ _MMIO(0x9888), 0x1c1c003c },
-	{ _MMIO(0x9888), 0x121c8000 },
-	{ _MMIO(0x9888), 0x141c8000 },
-	{ _MMIO(0x9888), 0x161c8000 },
-	{ _MMIO(0x9888), 0x181c8000 },
-	{ _MMIO(0x9888), 0x1a1c0800 },
-	{ _MMIO(0x9888), 0x065b4000 },
-	{ _MMIO(0x9888), 0x1a5c1000 },
-	{ _MMIO(0x9888), 0x10600000 },
-	{ _MMIO(0x9888), 0x04600000 },
-	{ _MMIO(0x9888), 0x0c610044 },
-	{ _MMIO(0x9888), 0x10610000 },
-	{ _MMIO(0x9888), 0x06610000 },
-	{ _MMIO(0x9888), 0x0c4c02a8 },
-	{ _MMIO(0x9888), 0x084ca000 },
-	{ _MMIO(0x9888), 0x0a4c002a },
-	{ _MMIO(0x9888), 0x0c0da000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x100f0154 },
-	{ _MMIO(0x9888), 0x0c0f5000 },
-	{ _MMIO(0x9888), 0x0e0f0055 },
-	{ _MMIO(0x9888), 0x182c00aa },
-	{ _MMIO(0x9888), 0x022c8000 },
-	{ _MMIO(0x9888), 0x042c8000 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2cc000 },
-	{ _MMIO(0x9888), 0x1190ffc0 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900420 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900021 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900400 },
-	{ _MMIO(0x9888), 0x43900421 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900040 },
-};
-
-static int
-get_l3_1_mux_config(struct drm_i915_private *dev_priv,
-		    const struct i915_oa_reg **regs,
-		    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_l3_1;
-	lens[n] = ARRAY_SIZE(mux_config_l3_1);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_l3_2[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00100070 },
-	{ _MMIO(0x2774), 0x0000fff1 },
-	{ _MMIO(0x2778), 0x00028002 },
-	{ _MMIO(0x277c), 0x000087ff },
-	{ _MMIO(0x2780), 0x00020002 },
-	{ _MMIO(0x2784), 0x00008fff },
-	{ _MMIO(0x2788), 0x00008002 },
-	{ _MMIO(0x278c), 0x0000a7ff },
-};
-
-static const struct i915_oa_reg flex_eu_config_l3_2[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_l3_2[] = {
-	{ _MMIO(0x9888), 0x126c02e0 },
-	{ _MMIO(0x9888), 0x146c0001 },
-	{ _MMIO(0x9888), 0x0a623400 },
-	{ _MMIO(0x9888), 0x044e8000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084e8000 },
-	{ _MMIO(0x9888), 0x0a4e8000 },
-	{ _MMIO(0x9888), 0x064f4000 },
-	{ _MMIO(0x9888), 0x026c3324 },
-	{ _MMIO(0x9888), 0x046c3422 },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x1a6c0000 },
-	{ _MMIO(0x9888), 0x021bc000 },
-	{ _MMIO(0x9888), 0x041bc000 },
-	{ _MMIO(0x9888), 0x141c8000 },
-	{ _MMIO(0x9888), 0x161c8000 },
-	{ _MMIO(0x9888), 0x181c8000 },
-	{ _MMIO(0x9888), 0x1a1c0800 },
-	{ _MMIO(0x9888), 0x065b4000 },
-	{ _MMIO(0x9888), 0x1a5c1000 },
-	{ _MMIO(0x9888), 0x06614000 },
-	{ _MMIO(0x9888), 0x0c620044 },
-	{ _MMIO(0x9888), 0x10620000 },
-	{ _MMIO(0x9888), 0x06620000 },
-	{ _MMIO(0x9888), 0x084c8000 },
-	{ _MMIO(0x9888), 0x0a4c002a },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x0c0f4000 },
-	{ _MMIO(0x9888), 0x0e0f0055 },
-	{ _MMIO(0x9888), 0x042c8000 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2cc000 },
-	{ _MMIO(0x9888), 0x1190f800 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x43900000 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_l3_2_mux_config(struct drm_i915_private *dev_priv,
-		    const struct i915_oa_reg **regs,
-		    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_l3_2;
-	lens[n] = ARRAY_SIZE(mux_config_l3_2);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_l3_3[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00100070 },
-	{ _MMIO(0x2774), 0x0000fff1 },
-	{ _MMIO(0x2778), 0x00028002 },
-	{ _MMIO(0x277c), 0x000087ff },
-	{ _MMIO(0x2780), 0x00020002 },
-	{ _MMIO(0x2784), 0x00008fff },
-	{ _MMIO(0x2788), 0x00008002 },
-	{ _MMIO(0x278c), 0x0000a7ff },
-};
-
-static const struct i915_oa_reg flex_eu_config_l3_3[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_l3_3[] = {
-	{ _MMIO(0x9888), 0x126c4e80 },
-	{ _MMIO(0x9888), 0x146c0000 },
-	{ _MMIO(0x9888), 0x0a633400 },
-	{ _MMIO(0x9888), 0x044e8000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084e8000 },
-	{ _MMIO(0x9888), 0x0a4e8000 },
-	{ _MMIO(0x9888), 0x0c4e8000 },
-	{ _MMIO(0x9888), 0x026c3321 },
-	{ _MMIO(0x9888), 0x046c342f },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x1a6c2000 },
-	{ _MMIO(0x9888), 0x021bc000 },
-	{ _MMIO(0x9888), 0x041bc000 },
-	{ _MMIO(0x9888), 0x061b4000 },
-	{ _MMIO(0x9888), 0x141c8000 },
-	{ _MMIO(0x9888), 0x161c8000 },
-	{ _MMIO(0x9888), 0x181c8000 },
-	{ _MMIO(0x9888), 0x1a1c1800 },
-	{ _MMIO(0x9888), 0x06604000 },
-	{ _MMIO(0x9888), 0x0c630044 },
-	{ _MMIO(0x9888), 0x10630000 },
-	{ _MMIO(0x9888), 0x06630000 },
-	{ _MMIO(0x9888), 0x084c8000 },
-	{ _MMIO(0x9888), 0x0a4c00aa },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x0c0f4000 },
-	{ _MMIO(0x9888), 0x0e0f0055 },
-	{ _MMIO(0x9888), 0x042c8000 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x1190f800 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x43900842 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900002 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_l3_3_mux_config(struct drm_i915_private *dev_priv,
-		    const struct i915_oa_reg **regs,
-		    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_l3_3;
-	lens[n] = ARRAY_SIZE(mux_config_l3_3);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x30800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x0000efff },
-	{ _MMIO(0x2778), 0x00006000 },
-	{ _MMIO(0x277c), 0x0000f3ff },
-};
-
-static const struct i915_oa_reg flex_eu_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0x9888), 0x102f3800 },
-	{ _MMIO(0x9888), 0x144d0500 },
-	{ _MMIO(0x9888), 0x120d03c0 },
-	{ _MMIO(0x9888), 0x140d03cf },
-	{ _MMIO(0x9888), 0x0c0f0004 },
-	{ _MMIO(0x9888), 0x0c4e4000 },
-	{ _MMIO(0x9888), 0x042f0480 },
-	{ _MMIO(0x9888), 0x082f0000 },
-	{ _MMIO(0x9888), 0x022f0000 },
-	{ _MMIO(0x9888), 0x0a4c0090 },
-	{ _MMIO(0x9888), 0x064d0027 },
-	{ _MMIO(0x9888), 0x004d0000 },
-	{ _MMIO(0x9888), 0x000d0d40 },
-	{ _MMIO(0x9888), 0x020d803f },
-	{ _MMIO(0x9888), 0x040d8023 },
-	{ _MMIO(0x9888), 0x100d0000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x020f0010 },
-	{ _MMIO(0x9888), 0x000f0000 },
-	{ _MMIO(0x9888), 0x0e0f0050 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x1190fc00 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41901400 },
-	{ _MMIO(0x9888), 0x43901485 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900001 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_rasterizer_and_pixel_backend_mux_config(struct drm_i915_private *dev_priv,
-					    const struct i915_oa_reg **regs,
-					    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_rasterizer_and_pixel_backend;
-	lens[n] = ARRAY_SIZE(mux_config_rasterizer_and_pixel_backend);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_sampler[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x70800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x0000c000 },
-	{ _MMIO(0x2774), 0x0000e7ff },
-	{ _MMIO(0x2778), 0x00003000 },
-	{ _MMIO(0x277c), 0x0000f9ff },
-	{ _MMIO(0x2780), 0x00000c00 },
-	{ _MMIO(0x2784), 0x0000fe7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_sampler[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_sampler[] = {
-	{ _MMIO(0x9888), 0x14152c00 },
-	{ _MMIO(0x9888), 0x16150005 },
-	{ _MMIO(0x9888), 0x121600a0 },
-	{ _MMIO(0x9888), 0x14352c00 },
-	{ _MMIO(0x9888), 0x16350005 },
-	{ _MMIO(0x9888), 0x123600a0 },
-	{ _MMIO(0x9888), 0x14552c00 },
-	{ _MMIO(0x9888), 0x16550005 },
-	{ _MMIO(0x9888), 0x125600a0 },
-	{ _MMIO(0x9888), 0x062f6000 },
-	{ _MMIO(0x9888), 0x022f2000 },
-	{ _MMIO(0x9888), 0x0c4c0050 },
-	{ _MMIO(0x9888), 0x0a4c0010 },
-	{ _MMIO(0x9888), 0x0c0d8000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x100f0350 },
-	{ _MMIO(0x9888), 0x0c0fb000 },
-	{ _MMIO(0x9888), 0x0e0f00da },
-	{ _MMIO(0x9888), 0x182c0028 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x022dc000 },
-	{ _MMIO(0x9888), 0x042d4000 },
-	{ _MMIO(0x9888), 0x0c138000 },
-	{ _MMIO(0x9888), 0x0e132000 },
-	{ _MMIO(0x9888), 0x0413c000 },
-	{ _MMIO(0x9888), 0x1c140018 },
-	{ _MMIO(0x9888), 0x0c157000 },
-	{ _MMIO(0x9888), 0x0e150078 },
-	{ _MMIO(0x9888), 0x10150000 },
-	{ _MMIO(0x9888), 0x04162180 },
-	{ _MMIO(0x9888), 0x02160000 },
-	{ _MMIO(0x9888), 0x04174000 },
-	{ _MMIO(0x9888), 0x0233a000 },
-	{ _MMIO(0x9888), 0x04333000 },
-	{ _MMIO(0x9888), 0x14348000 },
-	{ _MMIO(0x9888), 0x16348000 },
-	{ _MMIO(0x9888), 0x02357870 },
-	{ _MMIO(0x9888), 0x10350000 },
-	{ _MMIO(0x9888), 0x04360043 },
-	{ _MMIO(0x9888), 0x02360000 },
-	{ _MMIO(0x9888), 0x04371000 },
-	{ _MMIO(0x9888), 0x0e538000 },
-	{ _MMIO(0x9888), 0x00538000 },
-	{ _MMIO(0x9888), 0x06533000 },
-	{ _MMIO(0x9888), 0x1c540020 },
-	{ _MMIO(0x9888), 0x12548000 },
-	{ _MMIO(0x9888), 0x0e557000 },
-	{ _MMIO(0x9888), 0x00557800 },
-	{ _MMIO(0x9888), 0x10550000 },
-	{ _MMIO(0x9888), 0x06560043 },
-	{ _MMIO(0x9888), 0x02560000 },
-	{ _MMIO(0x9888), 0x06571000 },
-	{ _MMIO(0x9888), 0x1190ff80 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900060 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c00 },
-	{ _MMIO(0x9888), 0x43900842 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900060 },
-};
-
-static int
-get_sampler_mux_config(struct drm_i915_private *dev_priv,
-		       const struct i915_oa_reg **regs,
-		       int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_sampler;
-	lens[n] = ARRAY_SIZE(mux_config_sampler);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_tdl_1[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x00007fff },
-	{ _MMIO(0x2778), 0x00000000 },
-	{ _MMIO(0x277c), 0x00009fff },
-	{ _MMIO(0x2780), 0x00000002 },
-	{ _MMIO(0x2784), 0x0000efff },
-	{ _MMIO(0x2788), 0x00000000 },
-	{ _MMIO(0x278c), 0x0000f3ff },
-	{ _MMIO(0x2790), 0x00000002 },
-	{ _MMIO(0x2794), 0x0000fdff },
-	{ _MMIO(0x2798), 0x00000000 },
-	{ _MMIO(0x279c), 0x0000fe7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_tdl_1[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_tdl_1[] = {
-	{ _MMIO(0x9888), 0x12120000 },
-	{ _MMIO(0x9888), 0x12320000 },
-	{ _MMIO(0x9888), 0x12520000 },
-	{ _MMIO(0x9888), 0x002f8000 },
-	{ _MMIO(0x9888), 0x022f3000 },
-	{ _MMIO(0x9888), 0x0a4c0015 },
-	{ _MMIO(0x9888), 0x0c0d8000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x100f03a0 },
-	{ _MMIO(0x9888), 0x0c0ff000 },
-	{ _MMIO(0x9888), 0x0e0f0095 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2d8000 },
-	{ _MMIO(0x9888), 0x0e2d4000 },
-	{ _MMIO(0x9888), 0x062d4000 },
-	{ _MMIO(0x9888), 0x02108000 },
-	{ _MMIO(0x9888), 0x0410c000 },
-	{ _MMIO(0x9888), 0x02118000 },
-	{ _MMIO(0x9888), 0x0411c000 },
-	{ _MMIO(0x9888), 0x02121880 },
-	{ _MMIO(0x9888), 0x041219b5 },
-	{ _MMIO(0x9888), 0x00120000 },
-	{ _MMIO(0x9888), 0x02134000 },
-	{ _MMIO(0x9888), 0x04135000 },
-	{ _MMIO(0x9888), 0x0c308000 },
-	{ _MMIO(0x9888), 0x0e304000 },
-	{ _MMIO(0x9888), 0x06304000 },
-	{ _MMIO(0x9888), 0x0c318000 },
-	{ _MMIO(0x9888), 0x0e314000 },
-	{ _MMIO(0x9888), 0x06314000 },
-	{ _MMIO(0x9888), 0x0c321a80 },
-	{ _MMIO(0x9888), 0x0e320033 },
-	{ _MMIO(0x9888), 0x06320031 },
-	{ _MMIO(0x9888), 0x00320000 },
-	{ _MMIO(0x9888), 0x0c334000 },
-	{ _MMIO(0x9888), 0x0e331000 },
-	{ _MMIO(0x9888), 0x06331000 },
-	{ _MMIO(0x9888), 0x0e508000 },
-	{ _MMIO(0x9888), 0x00508000 },
-	{ _MMIO(0x9888), 0x02504000 },
-	{ _MMIO(0x9888), 0x0e518000 },
-	{ _MMIO(0x9888), 0x00518000 },
-	{ _MMIO(0x9888), 0x02514000 },
-	{ _MMIO(0x9888), 0x0e521880 },
-	{ _MMIO(0x9888), 0x00521a80 },
-	{ _MMIO(0x9888), 0x02520033 },
-	{ _MMIO(0x9888), 0x0e534000 },
-	{ _MMIO(0x9888), 0x00534000 },
-	{ _MMIO(0x9888), 0x02531000 },
-	{ _MMIO(0x9888), 0x1190ff80 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900800 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900062 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c00 },
-	{ _MMIO(0x9888), 0x43900003 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900040 },
-};
-
-static int
-get_tdl_1_mux_config(struct drm_i915_private *dev_priv,
-		     const struct i915_oa_reg **regs,
-		     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_tdl_1;
-	lens[n] = ARRAY_SIZE(mux_config_tdl_1);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_tdl_2[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_tdl_2[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_tdl_2[] = {
-	{ _MMIO(0x9888), 0x12124d60 },
-	{ _MMIO(0x9888), 0x12322e60 },
-	{ _MMIO(0x9888), 0x12524d60 },
-	{ _MMIO(0x9888), 0x022f3000 },
-	{ _MMIO(0x9888), 0x0a4c0014 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x0c0fe000 },
-	{ _MMIO(0x9888), 0x0e0f0097 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x002d8000 },
-	{ _MMIO(0x9888), 0x062d4000 },
-	{ _MMIO(0x9888), 0x0410c000 },
-	{ _MMIO(0x9888), 0x0411c000 },
-	{ _MMIO(0x9888), 0x04121fb7 },
-	{ _MMIO(0x9888), 0x00120000 },
-	{ _MMIO(0x9888), 0x04135000 },
-	{ _MMIO(0x9888), 0x00308000 },
-	{ _MMIO(0x9888), 0x06304000 },
-	{ _MMIO(0x9888), 0x00318000 },
-	{ _MMIO(0x9888), 0x06314000 },
-	{ _MMIO(0x9888), 0x00321b80 },
-	{ _MMIO(0x9888), 0x0632003f },
-	{ _MMIO(0x9888), 0x00334000 },
-	{ _MMIO(0x9888), 0x06331000 },
-	{ _MMIO(0x9888), 0x0250c000 },
-	{ _MMIO(0x9888), 0x0251c000 },
-	{ _MMIO(0x9888), 0x02521fb7 },
-	{ _MMIO(0x9888), 0x00520000 },
-	{ _MMIO(0x9888), 0x02535000 },
-	{ _MMIO(0x9888), 0x1190fc00 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900800 },
-	{ _MMIO(0x9888), 0x43900063 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900040 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_tdl_2_mux_config(struct drm_i915_private *dev_priv,
-		     const struct i915_oa_reg **regs,
-		     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_tdl_2;
-	lens[n] = ARRAY_SIZE(mux_config_tdl_2);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_extra[] = {
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_extra[] = {
-};
-
-static const struct i915_oa_reg mux_config_compute_extra[] = {
-	{ _MMIO(0x9888), 0x121203e0 },
-	{ _MMIO(0x9888), 0x123203e0 },
-	{ _MMIO(0x9888), 0x125203e0 },
-	{ _MMIO(0x9888), 0x129203e0 },
-	{ _MMIO(0x9888), 0x12b203e0 },
-	{ _MMIO(0x9888), 0x12d203e0 },
-	{ _MMIO(0x9888), 0x024ec000 },
-	{ _MMIO(0x9888), 0x044ec000 },
-	{ _MMIO(0x9888), 0x064ec000 },
-	{ _MMIO(0x9888), 0x022f4000 },
-	{ _MMIO(0x9888), 0x084ca000 },
-	{ _MMIO(0x9888), 0x0a4c0042 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x0c0f5000 },
-	{ _MMIO(0x9888), 0x0e0f006d },
-	{ _MMIO(0x9888), 0x022c8000 },
-	{ _MMIO(0x9888), 0x042c8000 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x042d8000 },
-	{ _MMIO(0x9888), 0x06104000 },
-	{ _MMIO(0x9888), 0x06114000 },
-	{ _MMIO(0x9888), 0x06120033 },
-	{ _MMIO(0x9888), 0x00120000 },
-	{ _MMIO(0x9888), 0x06131000 },
-	{ _MMIO(0x9888), 0x04308000 },
-	{ _MMIO(0x9888), 0x04318000 },
-	{ _MMIO(0x9888), 0x04321980 },
-	{ _MMIO(0x9888), 0x00320000 },
-	{ _MMIO(0x9888), 0x04334000 },
-	{ _MMIO(0x9888), 0x04504000 },
-	{ _MMIO(0x9888), 0x04514000 },
-	{ _MMIO(0x9888), 0x04520033 },
-	{ _MMIO(0x9888), 0x00520000 },
-	{ _MMIO(0x9888), 0x04531000 },
-	{ _MMIO(0x9888), 0x00af8000 },
-	{ _MMIO(0x9888), 0x0acc0001 },
-	{ _MMIO(0x9888), 0x008d8000 },
-	{ _MMIO(0x9888), 0x028da000 },
-	{ _MMIO(0x9888), 0x0c8fb000 },
-	{ _MMIO(0x9888), 0x0e8f0001 },
-	{ _MMIO(0x9888), 0x06ac8000 },
-	{ _MMIO(0x9888), 0x02ad4000 },
-	{ _MMIO(0x9888), 0x02908000 },
-	{ _MMIO(0x9888), 0x02918000 },
-	{ _MMIO(0x9888), 0x02921980 },
-	{ _MMIO(0x9888), 0x00920000 },
-	{ _MMIO(0x9888), 0x02934000 },
-	{ _MMIO(0x9888), 0x02b04000 },
-	{ _MMIO(0x9888), 0x02b14000 },
-	{ _MMIO(0x9888), 0x02b20033 },
-	{ _MMIO(0x9888), 0x00b20000 },
-	{ _MMIO(0x9888), 0x02b31000 },
-	{ _MMIO(0x9888), 0x00d08000 },
-	{ _MMIO(0x9888), 0x00d18000 },
-	{ _MMIO(0x9888), 0x00d21980 },
-	{ _MMIO(0x9888), 0x00d34000 },
-	{ _MMIO(0x9888), 0x1190fc00 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c00 },
-	{ _MMIO(0x9888), 0x43900402 },
-	{ _MMIO(0x9888), 0x53901550 },
-	{ _MMIO(0x9888), 0x45900080 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_compute_extra_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_extra;
-	lens[n] = ARRAY_SIZE(mux_config_compute_extra);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_vme_pipe[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2770), 0x00100030 },
-	{ _MMIO(0x2774), 0x0000fff9 },
-	{ _MMIO(0x2778), 0x00000002 },
-	{ _MMIO(0x277c), 0x0000fffc },
-	{ _MMIO(0x2780), 0x00000002 },
-	{ _MMIO(0x2784), 0x0000fff3 },
-	{ _MMIO(0x2788), 0x00100180 },
-	{ _MMIO(0x278c), 0x0000ffcf },
-	{ _MMIO(0x2790), 0x00000002 },
-	{ _MMIO(0x2794), 0x0000ffcf },
-	{ _MMIO(0x2798), 0x00000002 },
-	{ _MMIO(0x279c), 0x0000ff3f },
-};
-
-static const struct i915_oa_reg flex_eu_config_vme_pipe[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00008003 },
-};
-
-static const struct i915_oa_reg mux_config_vme_pipe[] = {
-	{ _MMIO(0x9888), 0x141a5800 },
-	{ _MMIO(0x9888), 0x161a00c0 },
-	{ _MMIO(0x9888), 0x12180240 },
-	{ _MMIO(0x9888), 0x14180002 },
-	{ _MMIO(0x9888), 0x149a5800 },
-	{ _MMIO(0x9888), 0x169a00c0 },
-	{ _MMIO(0x9888), 0x12980240 },
-	{ _MMIO(0x9888), 0x14980002 },
-	{ _MMIO(0x9888), 0x1a4e3fc0 },
-	{ _MMIO(0x9888), 0x002f1000 },
-	{ _MMIO(0x9888), 0x022f8000 },
-	{ _MMIO(0x9888), 0x042f3000 },
-	{ _MMIO(0x9888), 0x004c4000 },
-	{ _MMIO(0x9888), 0x0a4c9500 },
-	{ _MMIO(0x9888), 0x0c4c002a },
-	{ _MMIO(0x9888), 0x000d2000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0a0da000 },
-	{ _MMIO(0x9888), 0x0c0da000 },
-	{ _MMIO(0x9888), 0x0c0f0400 },
-	{ _MMIO(0x9888), 0x0e0f5500 },
-	{ _MMIO(0x9888), 0x100f0015 },
-	{ _MMIO(0x9888), 0x002c8000 },
-	{ _MMIO(0x9888), 0x0e2c8000 },
-	{ _MMIO(0x9888), 0x162caa00 },
-	{ _MMIO(0x9888), 0x182c000a },
-	{ _MMIO(0x9888), 0x04193000 },
-	{ _MMIO(0x9888), 0x081a28c1 },
-	{ _MMIO(0x9888), 0x001a0000 },
-	{ _MMIO(0x9888), 0x00133000 },
-	{ _MMIO(0x9888), 0x0613c000 },
-	{ _MMIO(0x9888), 0x0813f000 },
-	{ _MMIO(0x9888), 0x00172000 },
-	{ _MMIO(0x9888), 0x06178000 },
-	{ _MMIO(0x9888), 0x0817a000 },
-	{ _MMIO(0x9888), 0x00180037 },
-	{ _MMIO(0x9888), 0x06180940 },
-	{ _MMIO(0x9888), 0x08180000 },
-	{ _MMIO(0x9888), 0x02180000 },
-	{ _MMIO(0x9888), 0x04183000 },
-	{ _MMIO(0x9888), 0x04afc000 },
-	{ _MMIO(0x9888), 0x06af3000 },
-	{ _MMIO(0x9888), 0x0acc4000 },
-	{ _MMIO(0x9888), 0x0ccc0015 },
-	{ _MMIO(0x9888), 0x0a8da000 },
-	{ _MMIO(0x9888), 0x0c8da000 },
-	{ _MMIO(0x9888), 0x0e8f4000 },
-	{ _MMIO(0x9888), 0x108f0015 },
-	{ _MMIO(0x9888), 0x16aca000 },
-	{ _MMIO(0x9888), 0x18ac000a },
-	{ _MMIO(0x9888), 0x06993000 },
-	{ _MMIO(0x9888), 0x0c9a28c1 },
-	{ _MMIO(0x9888), 0x009a0000 },
-	{ _MMIO(0x9888), 0x0a93f000 },
-	{ _MMIO(0x9888), 0x0c93f000 },
-	{ _MMIO(0x9888), 0x0a97a000 },
-	{ _MMIO(0x9888), 0x0c97a000 },
-	{ _MMIO(0x9888), 0x0a980977 },
-	{ _MMIO(0x9888), 0x08980000 },
-	{ _MMIO(0x9888), 0x04980000 },
-	{ _MMIO(0x9888), 0x06983000 },
-	{ _MMIO(0x9888), 0x119000ff },
-	{ _MMIO(0x9888), 0x51900050 },
-	{ _MMIO(0x9888), 0x41900000 },
-	{ _MMIO(0x9888), 0x55900115 },
-	{ _MMIO(0x9888), 0x45900000 },
-	{ _MMIO(0x9888), 0x47900884 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900002 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_vme_pipe_mux_config(struct drm_i915_private *dev_priv,
-			const struct i915_oa_reg **regs,
-			int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_vme_pipe;
-	lens[n] = ARRAY_SIZE(mux_config_vme_pipe);
-	n++;
-
-	return n;
-}
-
 static const struct i915_oa_reg b_counter_config_test_oa[] = {
 	{ _MMIO(0x2740), 0x00000000 },
 	{ _MMIO(0x2744), 0x00800000 },
@@ -1930,6 +60,7 @@ static const struct i915_oa_reg flex_eu_config_test_oa[] = {
 };
 
 static const struct i915_oa_reg mux_config_test_oa[] = {
+	{ _MMIO(0x9840), 0x00000080 },
 	{ _MMIO(0x9888), 0x11810000 },
 	{ _MMIO(0x9888), 0x07810013 },
 	{ _MMIO(0x9888), 0x1f810000 },
@@ -1944,1096 +75,35 @@ static const struct i915_oa_reg mux_config_test_oa[] = {
 	{ _MMIO(0x9888), 0x33900000 },
 };
 
-static int
-get_test_oa_mux_config(struct drm_i915_private *dev_priv,
-		       const struct i915_oa_reg **regs,
-		       int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_test_oa;
-	lens[n] = ARRAY_SIZE(mux_config_test_oa);
-	n++;
-
-	return n;
-}
-
-int i915_oa_select_metric_set_sklgt3(struct drm_i915_private *dev_priv)
-{
-	dev_priv->perf.oa.n_mux_configs = 0;
-	dev_priv->perf.oa.b_counter_regs = NULL;
-	dev_priv->perf.oa.b_counter_regs_len = 0;
-	dev_priv->perf.oa.flex_regs = NULL;
-	dev_priv->perf.oa.flex_regs_len = 0;
-
-	switch (dev_priv->perf.oa.metrics_set) {
-	case METRIC_SET_ID_RENDER_BASIC:
-		dev_priv->perf.oa.n_mux_configs =
-			get_render_basic_mux_config(dev_priv,
-						    dev_priv->perf.oa.mux_regs,
-						    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_BASIC\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_render_basic;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_render_basic);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_render_basic;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_render_basic);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_BASIC:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_basic_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_BASIC\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_basic;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_basic);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_basic;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_basic);
-
-		return 0;
-	case METRIC_SET_ID_RENDER_PIPE_PROFILE:
-		dev_priv->perf.oa.n_mux_configs =
-			get_render_pipe_profile_mux_config(dev_priv,
-							   dev_priv->perf.oa.mux_regs,
-							   dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_PIPE_PROFILE\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_render_pipe_profile;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_render_pipe_profile);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_render_pipe_profile;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_render_pipe_profile);
-
-		return 0;
-	case METRIC_SET_ID_MEMORY_READS:
-		dev_priv->perf.oa.n_mux_configs =
-			get_memory_reads_mux_config(dev_priv,
-						    dev_priv->perf.oa.mux_regs,
-						    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_READS\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_memory_reads;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_memory_reads);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_memory_reads;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_memory_reads);
-
-		return 0;
-	case METRIC_SET_ID_MEMORY_WRITES:
-		dev_priv->perf.oa.n_mux_configs =
-			get_memory_writes_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_WRITES\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_memory_writes;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_memory_writes);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_memory_writes;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_memory_writes);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_EXTENDED:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_extended_mux_config(dev_priv,
-							dev_priv->perf.oa.mux_regs,
-							dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTENDED\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_extended;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_extended);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_extended;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_extended);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_L3_CACHE:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_l3_cache_mux_config(dev_priv,
-							dev_priv->perf.oa.mux_regs,
-							dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_L3_CACHE\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_l3_cache;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_l3_cache);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_l3_cache;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_l3_cache);
-
-		return 0;
-	case METRIC_SET_ID_HDC_AND_SF:
-		dev_priv->perf.oa.n_mux_configs =
-			get_hdc_and_sf_mux_config(dev_priv,
-						  dev_priv->perf.oa.mux_regs,
-						  dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"HDC_AND_SF\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_hdc_and_sf;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_hdc_and_sf);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_hdc_and_sf;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_hdc_and_sf);
-
-		return 0;
-	case METRIC_SET_ID_L3_1:
-		dev_priv->perf.oa.n_mux_configs =
-			get_l3_1_mux_config(dev_priv,
-					    dev_priv->perf.oa.mux_regs,
-					    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_1\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_l3_1;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_l3_1);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_l3_1;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_l3_1);
-
-		return 0;
-	case METRIC_SET_ID_L3_2:
-		dev_priv->perf.oa.n_mux_configs =
-			get_l3_2_mux_config(dev_priv,
-					    dev_priv->perf.oa.mux_regs,
-					    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_2\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_l3_2;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_l3_2);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_l3_2;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_l3_2);
-
-		return 0;
-	case METRIC_SET_ID_L3_3:
-		dev_priv->perf.oa.n_mux_configs =
-			get_l3_3_mux_config(dev_priv,
-					    dev_priv->perf.oa.mux_regs,
-					    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_3\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_l3_3;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_l3_3);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_l3_3;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_l3_3);
-
-		return 0;
-	case METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND:
-		dev_priv->perf.oa.n_mux_configs =
-			get_rasterizer_and_pixel_backend_mux_config(dev_priv,
-								    dev_priv->perf.oa.mux_regs,
-								    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RASTERIZER_AND_PIXEL_BACKEND\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_rasterizer_and_pixel_backend;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_rasterizer_and_pixel_backend);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_rasterizer_and_pixel_backend;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_rasterizer_and_pixel_backend);
-
-		return 0;
-	case METRIC_SET_ID_SAMPLER:
-		dev_priv->perf.oa.n_mux_configs =
-			get_sampler_mux_config(dev_priv,
-					       dev_priv->perf.oa.mux_regs,
-					       dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"SAMPLER\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_sampler;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_sampler);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_sampler;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_sampler);
-
-		return 0;
-	case METRIC_SET_ID_TDL_1:
-		dev_priv->perf.oa.n_mux_configs =
-			get_tdl_1_mux_config(dev_priv,
-					     dev_priv->perf.oa.mux_regs,
-					     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_1\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_tdl_1;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_tdl_1);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_tdl_1;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_tdl_1);
-
-		return 0;
-	case METRIC_SET_ID_TDL_2:
-		dev_priv->perf.oa.n_mux_configs =
-			get_tdl_2_mux_config(dev_priv,
-					     dev_priv->perf.oa.mux_regs,
-					     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_2\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_tdl_2;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_tdl_2);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_tdl_2;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_tdl_2);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_EXTRA:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_extra_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTRA\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_extra;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_extra);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_extra;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_extra);
-
-		return 0;
-	case METRIC_SET_ID_VME_PIPE:
-		dev_priv->perf.oa.n_mux_configs =
-			get_vme_pipe_mux_config(dev_priv,
-						dev_priv->perf.oa.mux_regs,
-						dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"VME_PIPE\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_vme_pipe;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_vme_pipe);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_vme_pipe;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_vme_pipe);
-
-		return 0;
-	case METRIC_SET_ID_TEST_OA:
-		dev_priv->perf.oa.n_mux_configs =
-			get_test_oa_mux_config(dev_priv,
-					       dev_priv->perf.oa.mux_regs,
-					       dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TEST_OA\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_test_oa;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_test_oa);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_test_oa;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_test_oa);
-
-		return 0;
-	default:
-		return -ENODEV;
-	}
-}
-
-static ssize_t
-show_render_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_BASIC);
-}
-
-static struct device_attribute dev_attr_render_basic_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_render_basic_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_render_basic[] = {
-	&dev_attr_render_basic_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_render_basic = {
-	.name = "4616d450-2393-4836-8146-53c5ed84d359",
-	.attrs =  attrs_render_basic,
-};
-
-static ssize_t
-show_compute_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_BASIC);
-}
-
-static struct device_attribute dev_attr_compute_basic_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_basic_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_basic[] = {
-	&dev_attr_compute_basic_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_basic = {
-	.name = "4320492b-fd03-42ac-922f-dbe1ef3b7b58",
-	.attrs =  attrs_compute_basic,
-};
-
-static ssize_t
-show_render_pipe_profile_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_PIPE_PROFILE);
-}
-
-static struct device_attribute dev_attr_render_pipe_profile_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_render_pipe_profile_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_render_pipe_profile[] = {
-	&dev_attr_render_pipe_profile_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_render_pipe_profile = {
-	.name = "bd2d9cae-b9ec-4f5b-9d2f-934bed398a2d",
-	.attrs =  attrs_render_pipe_profile,
-};
-
-static ssize_t
-show_memory_reads_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_READS);
-}
-
-static struct device_attribute dev_attr_memory_reads_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_memory_reads_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_memory_reads[] = {
-	&dev_attr_memory_reads_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_memory_reads = {
-	.name = "4ca0f3fe-7fd3-4924-98cb-1807d9879767",
-	.attrs =  attrs_memory_reads,
-};
-
-static ssize_t
-show_memory_writes_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_WRITES);
-}
-
-static struct device_attribute dev_attr_memory_writes_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_memory_writes_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_memory_writes[] = {
-	&dev_attr_memory_writes_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_memory_writes = {
-	.name = "a0c0172c-ee13-403d-99ff-2bdf6936cf14",
-	.attrs =  attrs_memory_writes,
-};
-
-static ssize_t
-show_compute_extended_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTENDED);
-}
-
-static struct device_attribute dev_attr_compute_extended_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_extended_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_extended[] = {
-	&dev_attr_compute_extended_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_extended = {
-	.name = "52435e0b-f188-42ea-8680-21a56ee20dee",
-	.attrs =  attrs_compute_extended,
-};
-
-static ssize_t
-show_compute_l3_cache_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_L3_CACHE);
-}
-
-static struct device_attribute dev_attr_compute_l3_cache_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_l3_cache_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_l3_cache[] = {
-	&dev_attr_compute_l3_cache_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_l3_cache = {
-	.name = "27076eeb-49f3-4fed-8423-c66506005c63",
-	.attrs =  attrs_compute_l3_cache,
-};
-
-static ssize_t
-show_hdc_and_sf_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_HDC_AND_SF);
-}
-
-static struct device_attribute dev_attr_hdc_and_sf_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_hdc_and_sf_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_hdc_and_sf[] = {
-	&dev_attr_hdc_and_sf_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_hdc_and_sf = {
-	.name = "8071b409-c39a-4674-94d7-32962ecfb512",
-	.attrs =  attrs_hdc_and_sf,
-};
-
-static ssize_t
-show_l3_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_1);
-}
-
-static struct device_attribute dev_attr_l3_1_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_l3_1_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_l3_1[] = {
-	&dev_attr_l3_1_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_l3_1 = {
-	.name = "5e0b391e-9ea8-4901-b2ff-b64ff616c7ed",
-	.attrs =  attrs_l3_1,
-};
-
-static ssize_t
-show_l3_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_2);
-}
-
-static struct device_attribute dev_attr_l3_2_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_l3_2_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_l3_2[] = {
-	&dev_attr_l3_2_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_l3_2 = {
-	.name = "25dc828e-1d2d-426e-9546-a1d4233cdf16",
-	.attrs =  attrs_l3_2,
-};
-
-static ssize_t
-show_l3_3_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_3);
-}
-
-static struct device_attribute dev_attr_l3_3_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_l3_3_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_l3_3[] = {
-	&dev_attr_l3_3_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_l3_3 = {
-	.name = "3dba9405-2d7e-4d70-8199-e734e82fd6bf",
-	.attrs =  attrs_l3_3,
-};
-
-static ssize_t
-show_rasterizer_and_pixel_backend_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND);
-}
-
-static struct device_attribute dev_attr_rasterizer_and_pixel_backend_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_rasterizer_and_pixel_backend_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_rasterizer_and_pixel_backend[] = {
-	&dev_attr_rasterizer_and_pixel_backend_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_rasterizer_and_pixel_backend = {
-	.name = "76935d7b-09c9-46bf-87f1-c18b4a86ebe5",
-	.attrs =  attrs_rasterizer_and_pixel_backend,
-};
-
-static ssize_t
-show_sampler_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_SAMPLER);
-}
-
-static struct device_attribute dev_attr_sampler_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_sampler_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_sampler[] = {
-	&dev_attr_sampler_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_sampler = {
-	.name = "1b34c0d6-4f4c-4d7b-833f-4aaf236d87a6",
-	.attrs =  attrs_sampler,
-};
-
-static ssize_t
-show_tdl_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_1);
-}
-
-static struct device_attribute dev_attr_tdl_1_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_tdl_1_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_tdl_1[] = {
-	&dev_attr_tdl_1_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_tdl_1 = {
-	.name = "b375c985-9953-455b-bda2-b03f7594e9db",
-	.attrs =  attrs_tdl_1,
-};
-
-static ssize_t
-show_tdl_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_2);
-}
-
-static struct device_attribute dev_attr_tdl_2_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_tdl_2_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_tdl_2[] = {
-	&dev_attr_tdl_2_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_tdl_2 = {
-	.name = "3e2be2bb-884a-49bb-82c5-2358e6bd5f2d",
-	.attrs =  attrs_tdl_2,
-};
-
-static ssize_t
-show_compute_extra_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTRA);
-}
-
-static struct device_attribute dev_attr_compute_extra_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_extra_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_extra[] = {
-	&dev_attr_compute_extra_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_extra = {
-	.name = "2d80a648-7b5a-4e92-bbe7-3b5c76f2e221",
-	.attrs =  attrs_compute_extra,
-};
-
-static ssize_t
-show_vme_pipe_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_VME_PIPE);
-}
-
-static struct device_attribute dev_attr_vme_pipe_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_vme_pipe_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_vme_pipe[] = {
-	&dev_attr_vme_pipe_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_vme_pipe = {
-	.name = "cfae9232-6ffc-42cc-a703-9790016925f0",
-	.attrs =  attrs_vme_pipe,
-};
-
 static ssize_t
 show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TEST_OA);
+	return sprintf(buf, "1\n");
 }
 
-static struct device_attribute dev_attr_test_oa_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_test_oa_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_test_oa[] = {
-	&dev_attr_test_oa_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_test_oa = {
-	.name = "2b985803-d3c9-4629-8a4f-634bfecba0e8",
-	.attrs =  attrs_test_oa,
-};
-
-int
-i915_perf_register_sysfs_sklgt3(struct drm_i915_private *dev_priv)
+void
+i915_perf_load_test_config_sklgt3(struct drm_i915_private *dev_priv)
 {
-	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
-	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
-	int ret = 0;
+	strncpy(dev_priv->perf.oa.test_config.uuid,
+		"2b985803-d3c9-4629-8a4f-634bfecba0e8",
+		UUID_STRING_LEN);
+	dev_priv->perf.oa.test_config.id = 1;
 
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-		if (ret)
-			goto error_render_basic;
-	}
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-		if (ret)
-			goto error_compute_basic;
-	}
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-		if (ret)
-			goto error_render_pipe_profile;
-	}
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-		if (ret)
-			goto error_memory_reads;
-	}
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-		if (ret)
-			goto error_memory_writes;
-	}
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-		if (ret)
-			goto error_compute_extended;
-	}
-	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
-		if (ret)
-			goto error_compute_l3_cache;
-	}
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-		if (ret)
-			goto error_hdc_and_sf;
-	}
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-		if (ret)
-			goto error_l3_1;
-	}
-	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_2);
-		if (ret)
-			goto error_l3_2;
-	}
-	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_3);
-		if (ret)
-			goto error_l3_3;
-	}
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-		if (ret)
-			goto error_rasterizer_and_pixel_backend;
-	}
-	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_sampler);
-		if (ret)
-			goto error_sampler;
-	}
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-		if (ret)
-			goto error_tdl_1;
-	}
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-		if (ret)
-			goto error_tdl_2;
-	}
-	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
-		if (ret)
-			goto error_compute_extra;
-	}
-	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
-		if (ret)
-			goto error_vme_pipe;
-	}
-	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_test_oa);
-		if (ret)
-			goto error_test_oa;
-	}
+	dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
+	dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa);
 
-	return 0;
+	dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa;
+	dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa);
 
-error_test_oa:
-	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
-error_vme_pipe:
-	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
-error_compute_extra:
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-error_tdl_2:
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-error_tdl_1:
-	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
-error_sampler:
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-error_rasterizer_and_pixel_backend:
-	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
-error_l3_3:
-	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
-error_l3_2:
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-error_l3_1:
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-error_hdc_and_sf:
-	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
-error_compute_l3_cache:
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-error_compute_extended:
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-error_memory_writes:
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-error_memory_reads:
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-error_render_pipe_profile:
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-error_compute_basic:
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-error_render_basic:
-	return ret;
-}
+	dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa;
+	dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa);
 
-void
-i915_perf_unregister_sysfs_sklgt3(struct drm_i915_private *dev_priv)
-{
-	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
-	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+	dev_priv->perf.oa.test_config.sysfs_metric.name = "2b985803-d3c9-4629-8a4f-634bfecba0e8";
+	dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs;
+
+	dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr;
 
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
-	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
-	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
-	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id";
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444;
+	dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id;
 }
diff --git a/drivers/gpu/drm/i915/i915_oa_sklgt3.h b/drivers/gpu/drm/i915/i915_oa_sklgt3.h
index c0accb1f9b74..06746b2616c8 100644
--- a/drivers/gpu/drm/i915/i915_oa_sklgt3.h
+++ b/drivers/gpu/drm/i915/i915_oa_sklgt3.h
@@ -29,12 +29,6 @@
 #ifndef __I915_OA_SKLGT3_H__
 #define __I915_OA_SKLGT3_H__
 
-extern int i915_oa_n_builtin_metric_sets_sklgt3;
-
-extern int i915_oa_select_metric_set_sklgt3(struct drm_i915_private *dev_priv);
-
-extern int i915_perf_register_sysfs_sklgt3(struct drm_i915_private *dev_priv);
-
-extern void i915_perf_unregister_sysfs_sklgt3(struct drm_i915_private *dev_priv);
+extern void i915_perf_load_test_config_sklgt3(struct drm_i915_private *dev_priv);
 
 #endif
diff --git a/drivers/gpu/drm/i915/i915_oa_sklgt4.c b/drivers/gpu/drm/i915/i915_oa_sklgt4.c
index 9ddab43a2176..bce031ee4445 100644
--- a/drivers/gpu/drm/i915/i915_oa_sklgt4.c
+++ b/drivers/gpu/drm/i915/i915_oa_sklgt4.c
@@ -31,1930 +31,6 @@
 #include "i915_drv.h"
 #include "i915_oa_sklgt4.h"
 
-enum metric_set_id {
-	METRIC_SET_ID_RENDER_BASIC = 1,
-	METRIC_SET_ID_COMPUTE_BASIC,
-	METRIC_SET_ID_RENDER_PIPE_PROFILE,
-	METRIC_SET_ID_MEMORY_READS,
-	METRIC_SET_ID_MEMORY_WRITES,
-	METRIC_SET_ID_COMPUTE_EXTENDED,
-	METRIC_SET_ID_COMPUTE_L3_CACHE,
-	METRIC_SET_ID_HDC_AND_SF,
-	METRIC_SET_ID_L3_1,
-	METRIC_SET_ID_L3_2,
-	METRIC_SET_ID_L3_3,
-	METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND,
-	METRIC_SET_ID_SAMPLER,
-	METRIC_SET_ID_TDL_1,
-	METRIC_SET_ID_TDL_2,
-	METRIC_SET_ID_COMPUTE_EXTRA,
-	METRIC_SET_ID_VME_PIPE,
-	METRIC_SET_ID_TEST_OA,
-};
-
-int i915_oa_n_builtin_metric_sets_sklgt4 = 18;
-
-static const struct i915_oa_reg b_counter_config_render_basic[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2740), 0x00000000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_render_basic[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_render_basic[] = {
-	{ _MMIO(0x9888), 0x166c01e0 },
-	{ _MMIO(0x9888), 0x12170280 },
-	{ _MMIO(0x9888), 0x12370280 },
-	{ _MMIO(0x9888), 0x16ec01e0 },
-	{ _MMIO(0x9888), 0x176c01e0 },
-	{ _MMIO(0x9888), 0x11930317 },
-	{ _MMIO(0x9888), 0x159303df },
-	{ _MMIO(0x9888), 0x3f900003 },
-	{ _MMIO(0x9888), 0x1a4e03b0 },
-	{ _MMIO(0x9888), 0x0a6c0053 },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x0a1b4000 },
-	{ _MMIO(0x9888), 0x1c1c0001 },
-	{ _MMIO(0x9888), 0x002f1000 },
-	{ _MMIO(0x9888), 0x042f1000 },
-	{ _MMIO(0x9888), 0x004c4000 },
-	{ _MMIO(0x9888), 0x0a4ca400 },
-	{ _MMIO(0x9888), 0x0c4c0002 },
-	{ _MMIO(0x9888), 0x000d2000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0a0da000 },
-	{ _MMIO(0x9888), 0x0c0f0400 },
-	{ _MMIO(0x9888), 0x0e0f5600 },
-	{ _MMIO(0x9888), 0x100f0001 },
-	{ _MMIO(0x9888), 0x002c8000 },
-	{ _MMIO(0x9888), 0x162caa00 },
-	{ _MMIO(0x9888), 0x062d8000 },
-	{ _MMIO(0x9888), 0x00133000 },
-	{ _MMIO(0x9888), 0x08133000 },
-	{ _MMIO(0x9888), 0x00170020 },
-	{ _MMIO(0x9888), 0x08170021 },
-	{ _MMIO(0x9888), 0x10170000 },
-	{ _MMIO(0x9888), 0x0633c000 },
-	{ _MMIO(0x9888), 0x06370800 },
-	{ _MMIO(0x9888), 0x10370000 },
-	{ _MMIO(0x9888), 0x1ace0230 },
-	{ _MMIO(0x9888), 0x0aec5300 },
-	{ _MMIO(0x9888), 0x10ec0000 },
-	{ _MMIO(0x9888), 0x1cec0000 },
-	{ _MMIO(0x9888), 0x0a9b8000 },
-	{ _MMIO(0x9888), 0x1c9c0002 },
-	{ _MMIO(0x9888), 0x0acc2000 },
-	{ _MMIO(0x9888), 0x0ccc0002 },
-	{ _MMIO(0x9888), 0x088d8000 },
-	{ _MMIO(0x9888), 0x0a8d8000 },
-	{ _MMIO(0x9888), 0x0e8f1000 },
-	{ _MMIO(0x9888), 0x108f0001 },
-	{ _MMIO(0x9888), 0x16ac8800 },
-	{ _MMIO(0x9888), 0x1b4e0020 },
-	{ _MMIO(0x9888), 0x096c5300 },
-	{ _MMIO(0x9888), 0x116c0000 },
-	{ _MMIO(0x9888), 0x1d6c0000 },
-	{ _MMIO(0x9888), 0x091b8000 },
-	{ _MMIO(0x9888), 0x1b1c8000 },
-	{ _MMIO(0x9888), 0x0b4c2000 },
-	{ _MMIO(0x9888), 0x090d8000 },
-	{ _MMIO(0x9888), 0x0f0f1000 },
-	{ _MMIO(0x9888), 0x172c0800 },
-	{ _MMIO(0x9888), 0x0d933031 },
-	{ _MMIO(0x9888), 0x0f933e3f },
-	{ _MMIO(0x9888), 0x01933d00 },
-	{ _MMIO(0x9888), 0x0393073c },
-	{ _MMIO(0x9888), 0x0593000e },
-	{ _MMIO(0x9888), 0x1d930000 },
-	{ _MMIO(0x9888), 0x19930000 },
-	{ _MMIO(0x9888), 0x1b930000 },
-	{ _MMIO(0x9888), 0x1d900157 },
-	{ _MMIO(0x9888), 0x1f900158 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x2b908000 },
-	{ _MMIO(0x9888), 0x2d908000 },
-	{ _MMIO(0x9888), 0x2f908000 },
-	{ _MMIO(0x9888), 0x31908000 },
-	{ _MMIO(0x9888), 0x15908000 },
-	{ _MMIO(0x9888), 0x17908000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1190003f },
-	{ _MMIO(0x9888), 0x5190ff30 },
-	{ _MMIO(0x9888), 0x41900060 },
-	{ _MMIO(0x9888), 0x55903033 },
-	{ _MMIO(0x9888), 0x45901421 },
-	{ _MMIO(0x9888), 0x47900803 },
-	{ _MMIO(0x9888), 0x5790fff1 },
-	{ _MMIO(0x9888), 0x49900001 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900000 },
-	{ _MMIO(0x9888), 0x5990000f },
-	{ _MMIO(0x9888), 0x43900000 },
-	{ _MMIO(0x9888), 0x5390ffff },
-};
-
-static int
-get_render_basic_mux_config(struct drm_i915_private *dev_priv,
-			    const struct i915_oa_reg **regs,
-			    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_render_basic;
-	lens[n] = ARRAY_SIZE(mux_config_render_basic);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_basic[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2740), 0x00000000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_basic[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00778008 },
-	{ _MMIO(0xe45c), 0x00088078 },
-	{ _MMIO(0xe55c), 0x00808708 },
-	{ _MMIO(0xe65c), 0x00a08908 },
-};
-
-static const struct i915_oa_reg mux_config_compute_basic[] = {
-	{ _MMIO(0x9888), 0x104f00e0 },
-	{ _MMIO(0x9888), 0x124f1c00 },
-	{ _MMIO(0x9888), 0x106c00e0 },
-	{ _MMIO(0x9888), 0x37906800 },
-	{ _MMIO(0x9888), 0x3f900003 },
-	{ _MMIO(0x9888), 0x004e8000 },
-	{ _MMIO(0x9888), 0x1a4e0820 },
-	{ _MMIO(0x9888), 0x1c4e0002 },
-	{ _MMIO(0x9888), 0x064f0900 },
-	{ _MMIO(0x9888), 0x084f0032 },
-	{ _MMIO(0x9888), 0x0a4f1891 },
-	{ _MMIO(0x9888), 0x0c4f0e00 },
-	{ _MMIO(0x9888), 0x0e4f003c },
-	{ _MMIO(0x9888), 0x004f0d80 },
-	{ _MMIO(0x9888), 0x024f003b },
-	{ _MMIO(0x9888), 0x006c0002 },
-	{ _MMIO(0x9888), 0x086c0100 },
-	{ _MMIO(0x9888), 0x0c6c000c },
-	{ _MMIO(0x9888), 0x0e6c0b00 },
-	{ _MMIO(0x9888), 0x186c0000 },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1e6c0000 },
-	{ _MMIO(0x9888), 0x001b4000 },
-	{ _MMIO(0x9888), 0x081b8000 },
-	{ _MMIO(0x9888), 0x0c1b4000 },
-	{ _MMIO(0x9888), 0x0e1b8000 },
-	{ _MMIO(0x9888), 0x101c8000 },
-	{ _MMIO(0x9888), 0x1a1c8000 },
-	{ _MMIO(0x9888), 0x1c1c0024 },
-	{ _MMIO(0x9888), 0x065b8000 },
-	{ _MMIO(0x9888), 0x085b4000 },
-	{ _MMIO(0x9888), 0x0a5bc000 },
-	{ _MMIO(0x9888), 0x0c5b8000 },
-	{ _MMIO(0x9888), 0x0e5b4000 },
-	{ _MMIO(0x9888), 0x005b8000 },
-	{ _MMIO(0x9888), 0x025b4000 },
-	{ _MMIO(0x9888), 0x1a5c6000 },
-	{ _MMIO(0x9888), 0x1c5c001b },
-	{ _MMIO(0x9888), 0x125c8000 },
-	{ _MMIO(0x9888), 0x145c8000 },
-	{ _MMIO(0x9888), 0x004c8000 },
-	{ _MMIO(0x9888), 0x0a4c2000 },
-	{ _MMIO(0x9888), 0x0c4c0208 },
-	{ _MMIO(0x9888), 0x000da000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0a0da000 },
-	{ _MMIO(0x9888), 0x0c0da000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x020d2000 },
-	{ _MMIO(0x9888), 0x0c0f5400 },
-	{ _MMIO(0x9888), 0x0e0f5500 },
-	{ _MMIO(0x9888), 0x100f0155 },
-	{ _MMIO(0x9888), 0x002c8000 },
-	{ _MMIO(0x9888), 0x0e2cc000 },
-	{ _MMIO(0x9888), 0x162cfb00 },
-	{ _MMIO(0x9888), 0x182c00be },
-	{ _MMIO(0x9888), 0x022cc000 },
-	{ _MMIO(0x9888), 0x042cc000 },
-	{ _MMIO(0x9888), 0x19900157 },
-	{ _MMIO(0x9888), 0x1b900158 },
-	{ _MMIO(0x9888), 0x1d900105 },
-	{ _MMIO(0x9888), 0x1f900103 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x11900fff },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900800 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900821 },
-	{ _MMIO(0x9888), 0x47900802 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900802 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900002 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900422 },
-	{ _MMIO(0x9888), 0x53905555 },
-};
-
-static int
-get_compute_basic_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_basic;
-	lens[n] = ARRAY_SIZE(mux_config_compute_basic);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_render_pipe_profile[] = {
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007ffea },
-	{ _MMIO(0x2774), 0x00007ffc },
-	{ _MMIO(0x2778), 0x0007affa },
-	{ _MMIO(0x277c), 0x0000f5fd },
-	{ _MMIO(0x2780), 0x00079ffa },
-	{ _MMIO(0x2784), 0x0000f3fb },
-	{ _MMIO(0x2788), 0x0007bf7a },
-	{ _MMIO(0x278c), 0x0000f7e7 },
-	{ _MMIO(0x2790), 0x0007fefa },
-	{ _MMIO(0x2794), 0x0000f7cf },
-	{ _MMIO(0x2798), 0x00077ffa },
-	{ _MMIO(0x279c), 0x0000efdf },
-	{ _MMIO(0x27a0), 0x0006fffa },
-	{ _MMIO(0x27a4), 0x0000cfbf },
-	{ _MMIO(0x27a8), 0x0003fffa },
-	{ _MMIO(0x27ac), 0x00005f7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_render_pipe_profile[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00015014 },
-	{ _MMIO(0xe658), 0x00025024 },
-	{ _MMIO(0xe758), 0x00035034 },
-	{ _MMIO(0xe45c), 0x00045044 },
-	{ _MMIO(0xe55c), 0x00055054 },
-	{ _MMIO(0xe65c), 0x00065064 },
-};
-
-static const struct i915_oa_reg mux_config_render_pipe_profile[] = {
-	{ _MMIO(0x9888), 0x0c0e001f },
-	{ _MMIO(0x9888), 0x0a0f0000 },
-	{ _MMIO(0x9888), 0x10116800 },
-	{ _MMIO(0x9888), 0x178a03e0 },
-	{ _MMIO(0x9888), 0x11824c00 },
-	{ _MMIO(0x9888), 0x11830020 },
-	{ _MMIO(0x9888), 0x13840020 },
-	{ _MMIO(0x9888), 0x11850019 },
-	{ _MMIO(0x9888), 0x11860007 },
-	{ _MMIO(0x9888), 0x01870c40 },
-	{ _MMIO(0x9888), 0x17880000 },
-	{ _MMIO(0x9888), 0x022f4000 },
-	{ _MMIO(0x9888), 0x0a4c0040 },
-	{ _MMIO(0x9888), 0x0c0d8000 },
-	{ _MMIO(0x9888), 0x040d4000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x020e5400 },
-	{ _MMIO(0x9888), 0x000e0000 },
-	{ _MMIO(0x9888), 0x080f0040 },
-	{ _MMIO(0x9888), 0x000f0000 },
-	{ _MMIO(0x9888), 0x100f0000 },
-	{ _MMIO(0x9888), 0x0e0f0040 },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x06104000 },
-	{ _MMIO(0x9888), 0x06110012 },
-	{ _MMIO(0x9888), 0x06131000 },
-	{ _MMIO(0x9888), 0x01898000 },
-	{ _MMIO(0x9888), 0x0d890100 },
-	{ _MMIO(0x9888), 0x03898000 },
-	{ _MMIO(0x9888), 0x09808000 },
-	{ _MMIO(0x9888), 0x0b808000 },
-	{ _MMIO(0x9888), 0x0380c000 },
-	{ _MMIO(0x9888), 0x0f8a0075 },
-	{ _MMIO(0x9888), 0x1d8a0000 },
-	{ _MMIO(0x9888), 0x118a8000 },
-	{ _MMIO(0x9888), 0x1b8a4000 },
-	{ _MMIO(0x9888), 0x138a8000 },
-	{ _MMIO(0x9888), 0x1d81a000 },
-	{ _MMIO(0x9888), 0x15818000 },
-	{ _MMIO(0x9888), 0x17818000 },
-	{ _MMIO(0x9888), 0x0b820030 },
-	{ _MMIO(0x9888), 0x07828000 },
-	{ _MMIO(0x9888), 0x0d824000 },
-	{ _MMIO(0x9888), 0x0f828000 },
-	{ _MMIO(0x9888), 0x05824000 },
-	{ _MMIO(0x9888), 0x0d830003 },
-	{ _MMIO(0x9888), 0x0583000c },
-	{ _MMIO(0x9888), 0x09830000 },
-	{ _MMIO(0x9888), 0x03838000 },
-	{ _MMIO(0x9888), 0x07838000 },
-	{ _MMIO(0x9888), 0x0b840980 },
-	{ _MMIO(0x9888), 0x03844d80 },
-	{ _MMIO(0x9888), 0x11840000 },
-	{ _MMIO(0x9888), 0x09848000 },
-	{ _MMIO(0x9888), 0x09850080 },
-	{ _MMIO(0x9888), 0x03850003 },
-	{ _MMIO(0x9888), 0x01850000 },
-	{ _MMIO(0x9888), 0x07860000 },
-	{ _MMIO(0x9888), 0x0f860400 },
-	{ _MMIO(0x9888), 0x09870032 },
-	{ _MMIO(0x9888), 0x01888052 },
-	{ _MMIO(0x9888), 0x11880000 },
-	{ _MMIO(0x9888), 0x09884000 },
-	{ _MMIO(0x9888), 0x1b931001 },
-	{ _MMIO(0x9888), 0x1d930001 },
-	{ _MMIO(0x9888), 0x19934000 },
-	{ _MMIO(0x9888), 0x1b958000 },
-	{ _MMIO(0x9888), 0x1d950094 },
-	{ _MMIO(0x9888), 0x19958000 },
-	{ _MMIO(0x9888), 0x09e58000 },
-	{ _MMIO(0x9888), 0x0be58000 },
-	{ _MMIO(0x9888), 0x03e5c000 },
-	{ _MMIO(0x9888), 0x0592c000 },
-	{ _MMIO(0x9888), 0x0b928000 },
-	{ _MMIO(0x9888), 0x0d924000 },
-	{ _MMIO(0x9888), 0x0f924000 },
-	{ _MMIO(0x9888), 0x11928000 },
-	{ _MMIO(0x9888), 0x1392c000 },
-	{ _MMIO(0x9888), 0x09924000 },
-	{ _MMIO(0x9888), 0x01985000 },
-	{ _MMIO(0x9888), 0x07988000 },
-	{ _MMIO(0x9888), 0x09981000 },
-	{ _MMIO(0x9888), 0x0b982000 },
-	{ _MMIO(0x9888), 0x0d982000 },
-	{ _MMIO(0x9888), 0x0f989000 },
-	{ _MMIO(0x9888), 0x05982000 },
-	{ _MMIO(0x9888), 0x13904000 },
-	{ _MMIO(0x9888), 0x21904000 },
-	{ _MMIO(0x9888), 0x23904000 },
-	{ _MMIO(0x9888), 0x25908000 },
-	{ _MMIO(0x9888), 0x27904000 },
-	{ _MMIO(0x9888), 0x29908000 },
-	{ _MMIO(0x9888), 0x2b904000 },
-	{ _MMIO(0x9888), 0x2f904000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x15904000 },
-	{ _MMIO(0x9888), 0x17908000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b904000 },
-	{ _MMIO(0x9888), 0x1190c080 },
-	{ _MMIO(0x9888), 0x51901110 },
-	{ _MMIO(0x9888), 0x41900440 },
-	{ _MMIO(0x9888), 0x55901111 },
-	{ _MMIO(0x9888), 0x45900400 },
-	{ _MMIO(0x9888), 0x47900c21 },
-	{ _MMIO(0x9888), 0x57901411 },
-	{ _MMIO(0x9888), 0x49900042 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900024 },
-	{ _MMIO(0x9888), 0x59900001 },
-	{ _MMIO(0x9888), 0x43900841 },
-	{ _MMIO(0x9888), 0x53900411 },
-};
-
-static int
-get_render_pipe_profile_mux_config(struct drm_i915_private *dev_priv,
-				   const struct i915_oa_reg **regs,
-				   int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_render_pipe_profile;
-	lens[n] = ARRAY_SIZE(mux_config_render_pipe_profile);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_memory_reads[] = {
-	{ _MMIO(0x272c), 0xffffffff },
-	{ _MMIO(0x2728), 0xffffffff },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x271c), 0xffffffff },
-	{ _MMIO(0x2718), 0xffffffff },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x274c), 0x86543210 },
-	{ _MMIO(0x2748), 0x86543210 },
-	{ _MMIO(0x2744), 0x00006667 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x275c), 0x86543210 },
-	{ _MMIO(0x2758), 0x86543210 },
-	{ _MMIO(0x2754), 0x00006465 },
-	{ _MMIO(0x2750), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007f81a },
-	{ _MMIO(0x2774), 0x0000fe00 },
-	{ _MMIO(0x2778), 0x0007f82a },
-	{ _MMIO(0x277c), 0x0000fe00 },
-	{ _MMIO(0x2780), 0x0007f872 },
-	{ _MMIO(0x2784), 0x0000fe00 },
-	{ _MMIO(0x2788), 0x0007f8ba },
-	{ _MMIO(0x278c), 0x0000fe00 },
-	{ _MMIO(0x2790), 0x0007f87a },
-	{ _MMIO(0x2794), 0x0000fe00 },
-	{ _MMIO(0x2798), 0x0007f8ea },
-	{ _MMIO(0x279c), 0x0000fe00 },
-	{ _MMIO(0x27a0), 0x0007f8e2 },
-	{ _MMIO(0x27a4), 0x0000fe00 },
-	{ _MMIO(0x27a8), 0x0007f8f2 },
-	{ _MMIO(0x27ac), 0x0000fe00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_memory_reads[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00015014 },
-	{ _MMIO(0xe658), 0x00025024 },
-	{ _MMIO(0xe758), 0x00035034 },
-	{ _MMIO(0xe45c), 0x00045044 },
-	{ _MMIO(0xe55c), 0x00055054 },
-	{ _MMIO(0xe65c), 0x00065064 },
-};
-
-static const struct i915_oa_reg mux_config_memory_reads[] = {
-	{ _MMIO(0x9888), 0x11810c00 },
-	{ _MMIO(0x9888), 0x1381001a },
-	{ _MMIO(0x9888), 0x37906800 },
-	{ _MMIO(0x9888), 0x3f900064 },
-	{ _MMIO(0x9888), 0x03811300 },
-	{ _MMIO(0x9888), 0x05811b12 },
-	{ _MMIO(0x9888), 0x0781001a },
-	{ _MMIO(0x9888), 0x1f810000 },
-	{ _MMIO(0x9888), 0x17810000 },
-	{ _MMIO(0x9888), 0x19810000 },
-	{ _MMIO(0x9888), 0x1b810000 },
-	{ _MMIO(0x9888), 0x1d810000 },
-	{ _MMIO(0x9888), 0x1b930055 },
-	{ _MMIO(0x9888), 0x03e58000 },
-	{ _MMIO(0x9888), 0x05e5c000 },
-	{ _MMIO(0x9888), 0x07e54000 },
-	{ _MMIO(0x9888), 0x13900150 },
-	{ _MMIO(0x9888), 0x21900151 },
-	{ _MMIO(0x9888), 0x23900152 },
-	{ _MMIO(0x9888), 0x25900153 },
-	{ _MMIO(0x9888), 0x27900154 },
-	{ _MMIO(0x9888), 0x29900155 },
-	{ _MMIO(0x9888), 0x2b900156 },
-	{ _MMIO(0x9888), 0x2d900157 },
-	{ _MMIO(0x9888), 0x2f90015f },
-	{ _MMIO(0x9888), 0x31900105 },
-	{ _MMIO(0x9888), 0x15900103 },
-	{ _MMIO(0x9888), 0x17900101 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1d908000 },
-	{ _MMIO(0x9888), 0x1f908000 },
-	{ _MMIO(0x9888), 0x11900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c60 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900c00 },
-	{ _MMIO(0x9888), 0x47900c63 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900c63 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900063 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900003 },
-	{ _MMIO(0x9888), 0x53900000 },
-};
-
-static int
-get_memory_reads_mux_config(struct drm_i915_private *dev_priv,
-			    const struct i915_oa_reg **regs,
-			    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_memory_reads;
-	lens[n] = ARRAY_SIZE(mux_config_memory_reads);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_memory_writes[] = {
-	{ _MMIO(0x272c), 0xffffffff },
-	{ _MMIO(0x2728), 0xffffffff },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x271c), 0xffffffff },
-	{ _MMIO(0x2718), 0xffffffff },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x274c), 0x86543210 },
-	{ _MMIO(0x2748), 0x86543210 },
-	{ _MMIO(0x2744), 0x00006667 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x275c), 0x86543210 },
-	{ _MMIO(0x2758), 0x86543210 },
-	{ _MMIO(0x2754), 0x00006465 },
-	{ _MMIO(0x2750), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007f81a },
-	{ _MMIO(0x2774), 0x0000fe00 },
-	{ _MMIO(0x2778), 0x0007f82a },
-	{ _MMIO(0x277c), 0x0000fe00 },
-	{ _MMIO(0x2780), 0x0007f822 },
-	{ _MMIO(0x2784), 0x0000fe00 },
-	{ _MMIO(0x2788), 0x0007f8ba },
-	{ _MMIO(0x278c), 0x0000fe00 },
-	{ _MMIO(0x2790), 0x0007f87a },
-	{ _MMIO(0x2794), 0x0000fe00 },
-	{ _MMIO(0x2798), 0x0007f8ea },
-	{ _MMIO(0x279c), 0x0000fe00 },
-	{ _MMIO(0x27a0), 0x0007f8e2 },
-	{ _MMIO(0x27a4), 0x0000fe00 },
-	{ _MMIO(0x27a8), 0x0007f8f2 },
-	{ _MMIO(0x27ac), 0x0000fe00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_memory_writes[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00015014 },
-	{ _MMIO(0xe658), 0x00025024 },
-	{ _MMIO(0xe758), 0x00035034 },
-	{ _MMIO(0xe45c), 0x00045044 },
-	{ _MMIO(0xe55c), 0x00055054 },
-	{ _MMIO(0xe65c), 0x00065064 },
-};
-
-static const struct i915_oa_reg mux_config_memory_writes[] = {
-	{ _MMIO(0x9888), 0x11810c00 },
-	{ _MMIO(0x9888), 0x1381001a },
-	{ _MMIO(0x9888), 0x37906800 },
-	{ _MMIO(0x9888), 0x3f901000 },
-	{ _MMIO(0x9888), 0x03811300 },
-	{ _MMIO(0x9888), 0x05811b12 },
-	{ _MMIO(0x9888), 0x0781001a },
-	{ _MMIO(0x9888), 0x1f810000 },
-	{ _MMIO(0x9888), 0x17810000 },
-	{ _MMIO(0x9888), 0x19810000 },
-	{ _MMIO(0x9888), 0x1b810000 },
-	{ _MMIO(0x9888), 0x1d810000 },
-	{ _MMIO(0x9888), 0x1b930055 },
-	{ _MMIO(0x9888), 0x03e58000 },
-	{ _MMIO(0x9888), 0x05e5c000 },
-	{ _MMIO(0x9888), 0x07e54000 },
-	{ _MMIO(0x9888), 0x13900160 },
-	{ _MMIO(0x9888), 0x21900161 },
-	{ _MMIO(0x9888), 0x23900162 },
-	{ _MMIO(0x9888), 0x25900163 },
-	{ _MMIO(0x9888), 0x27900164 },
-	{ _MMIO(0x9888), 0x29900165 },
-	{ _MMIO(0x9888), 0x2b900166 },
-	{ _MMIO(0x9888), 0x2d900167 },
-	{ _MMIO(0x9888), 0x2f900150 },
-	{ _MMIO(0x9888), 0x31900105 },
-	{ _MMIO(0x9888), 0x15900103 },
-	{ _MMIO(0x9888), 0x17900101 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1d908000 },
-	{ _MMIO(0x9888), 0x1f908000 },
-	{ _MMIO(0x9888), 0x11900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c60 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900c00 },
-	{ _MMIO(0x9888), 0x47900c63 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900c63 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900063 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900003 },
-	{ _MMIO(0x9888), 0x53900000 },
-};
-
-static int
-get_memory_writes_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_memory_writes;
-	lens[n] = ARRAY_SIZE(mux_config_memory_writes);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_extended[] = {
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007fc2a },
-	{ _MMIO(0x2774), 0x0000bf00 },
-	{ _MMIO(0x2778), 0x0007fc6a },
-	{ _MMIO(0x277c), 0x0000bf00 },
-	{ _MMIO(0x2780), 0x0007fc92 },
-	{ _MMIO(0x2784), 0x0000bf00 },
-	{ _MMIO(0x2788), 0x0007fca2 },
-	{ _MMIO(0x278c), 0x0000bf00 },
-	{ _MMIO(0x2790), 0x0007fc32 },
-	{ _MMIO(0x2794), 0x0000bf00 },
-	{ _MMIO(0x2798), 0x0007fc9a },
-	{ _MMIO(0x279c), 0x0000bf00 },
-	{ _MMIO(0x27a0), 0x0007fe6a },
-	{ _MMIO(0x27a4), 0x0000bf00 },
-	{ _MMIO(0x27a8), 0x0007fe7a },
-	{ _MMIO(0x27ac), 0x0000bf00 },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_extended[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00778008 },
-	{ _MMIO(0xe45c), 0x00088078 },
-	{ _MMIO(0xe55c), 0x00808708 },
-	{ _MMIO(0xe65c), 0x00a08908 },
-};
-
-static const struct i915_oa_reg mux_config_compute_extended[] = {
-	{ _MMIO(0x9888), 0x106c00e0 },
-	{ _MMIO(0x9888), 0x141c8160 },
-	{ _MMIO(0x9888), 0x161c8015 },
-	{ _MMIO(0x9888), 0x181c0120 },
-	{ _MMIO(0x9888), 0x004e8000 },
-	{ _MMIO(0x9888), 0x0e4e8000 },
-	{ _MMIO(0x9888), 0x184e8000 },
-	{ _MMIO(0x9888), 0x1a4eaaa0 },
-	{ _MMIO(0x9888), 0x1c4e0002 },
-	{ _MMIO(0x9888), 0x024e8000 },
-	{ _MMIO(0x9888), 0x044e8000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084e8000 },
-	{ _MMIO(0x9888), 0x0a4e8000 },
-	{ _MMIO(0x9888), 0x0e6c0b01 },
-	{ _MMIO(0x9888), 0x006c0200 },
-	{ _MMIO(0x9888), 0x026c000c },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1e6c0000 },
-	{ _MMIO(0x9888), 0x1a6c0000 },
-	{ _MMIO(0x9888), 0x0e1bc000 },
-	{ _MMIO(0x9888), 0x001b8000 },
-	{ _MMIO(0x9888), 0x021bc000 },
-	{ _MMIO(0x9888), 0x001c0041 },
-	{ _MMIO(0x9888), 0x061c4200 },
-	{ _MMIO(0x9888), 0x081c4443 },
-	{ _MMIO(0x9888), 0x0a1c4645 },
-	{ _MMIO(0x9888), 0x0c1c7647 },
-	{ _MMIO(0x9888), 0x041c7357 },
-	{ _MMIO(0x9888), 0x1c1c0030 },
-	{ _MMIO(0x9888), 0x101c0000 },
-	{ _MMIO(0x9888), 0x1a1c0000 },
-	{ _MMIO(0x9888), 0x121c8000 },
-	{ _MMIO(0x9888), 0x004c8000 },
-	{ _MMIO(0x9888), 0x0a4caa2a },
-	{ _MMIO(0x9888), 0x0c4c02aa },
-	{ _MMIO(0x9888), 0x084ca000 },
-	{ _MMIO(0x9888), 0x000da000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0a0da000 },
-	{ _MMIO(0x9888), 0x0c0da000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x0c0f5400 },
-	{ _MMIO(0x9888), 0x0e0f5515 },
-	{ _MMIO(0x9888), 0x100f0155 },
-	{ _MMIO(0x9888), 0x002c8000 },
-	{ _MMIO(0x9888), 0x0e2c8000 },
-	{ _MMIO(0x9888), 0x162caa00 },
-	{ _MMIO(0x9888), 0x182c00aa },
-	{ _MMIO(0x9888), 0x022c8000 },
-	{ _MMIO(0x9888), 0x042c8000 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x11907fff },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900040 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900802 },
-	{ _MMIO(0x9888), 0x47900842 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900842 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x43900800 },
-	{ _MMIO(0x9888), 0x53900000 },
-};
-
-static int
-get_compute_extended_mux_config(struct drm_i915_private *dev_priv,
-				const struct i915_oa_reg **regs,
-				int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_extended;
-	lens[n] = ARRAY_SIZE(mux_config_compute_extended);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_l3_cache[] = {
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x30800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2770), 0x0007fffa },
-	{ _MMIO(0x2774), 0x0000fefe },
-	{ _MMIO(0x2778), 0x0007fffa },
-	{ _MMIO(0x277c), 0x0000fefd },
-	{ _MMIO(0x2790), 0x0007fffa },
-	{ _MMIO(0x2794), 0x0000fbef },
-	{ _MMIO(0x2798), 0x0007fffa },
-	{ _MMIO(0x279c), 0x0000fbdf },
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_l3_cache[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00000003 },
-	{ _MMIO(0xe658), 0x00002001 },
-	{ _MMIO(0xe758), 0x00101100 },
-	{ _MMIO(0xe45c), 0x00201200 },
-	{ _MMIO(0xe55c), 0x00301300 },
-	{ _MMIO(0xe65c), 0x00401400 },
-};
-
-static const struct i915_oa_reg mux_config_compute_l3_cache[] = {
-	{ _MMIO(0x9888), 0x166c0760 },
-	{ _MMIO(0x9888), 0x1593001e },
-	{ _MMIO(0x9888), 0x3f900003 },
-	{ _MMIO(0x9888), 0x004e8000 },
-	{ _MMIO(0x9888), 0x0e4e8000 },
-	{ _MMIO(0x9888), 0x184e8000 },
-	{ _MMIO(0x9888), 0x1a4e8020 },
-	{ _MMIO(0x9888), 0x1c4e0002 },
-	{ _MMIO(0x9888), 0x006c0051 },
-	{ _MMIO(0x9888), 0x066c5000 },
-	{ _MMIO(0x9888), 0x086c5c5d },
-	{ _MMIO(0x9888), 0x0e6c5e5f },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x186c0000 },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1e6c0000 },
-	{ _MMIO(0x9888), 0x001b4000 },
-	{ _MMIO(0x9888), 0x061b8000 },
-	{ _MMIO(0x9888), 0x081bc000 },
-	{ _MMIO(0x9888), 0x0e1bc000 },
-	{ _MMIO(0x9888), 0x101c8000 },
-	{ _MMIO(0x9888), 0x1a1ce000 },
-	{ _MMIO(0x9888), 0x1c1c0030 },
-	{ _MMIO(0x9888), 0x004c8000 },
-	{ _MMIO(0x9888), 0x0a4c2a00 },
-	{ _MMIO(0x9888), 0x0c4c0280 },
-	{ _MMIO(0x9888), 0x000d2000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x0c0f0400 },
-	{ _MMIO(0x9888), 0x0e0f1500 },
-	{ _MMIO(0x9888), 0x100f0140 },
-	{ _MMIO(0x9888), 0x002c8000 },
-	{ _MMIO(0x9888), 0x0e2c8000 },
-	{ _MMIO(0x9888), 0x162c0a00 },
-	{ _MMIO(0x9888), 0x182c00a0 },
-	{ _MMIO(0x9888), 0x03933300 },
-	{ _MMIO(0x9888), 0x05930032 },
-	{ _MMIO(0x9888), 0x11930000 },
-	{ _MMIO(0x9888), 0x1b930000 },
-	{ _MMIO(0x9888), 0x1d900157 },
-	{ _MMIO(0x9888), 0x1f900158 },
-	{ _MMIO(0x9888), 0x35900000 },
-	{ _MMIO(0x9888), 0x19908000 },
-	{ _MMIO(0x9888), 0x1b908000 },
-	{ _MMIO(0x9888), 0x1190030f },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900000 },
-	{ _MMIO(0x9888), 0x55900000 },
-	{ _MMIO(0x9888), 0x45900021 },
-	{ _MMIO(0x9888), 0x47900000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x4b900000 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x53905555 },
-	{ _MMIO(0x9888), 0x43900000 },
-};
-
-static int
-get_compute_l3_cache_mux_config(struct drm_i915_private *dev_priv,
-				const struct i915_oa_reg **regs,
-				int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_l3_cache;
-	lens[n] = ARRAY_SIZE(mux_config_compute_l3_cache);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_hdc_and_sf[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x10800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x0000fdff },
-};
-
-static const struct i915_oa_reg flex_eu_config_hdc_and_sf[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_hdc_and_sf[] = {
-	{ _MMIO(0x9888), 0x104f0232 },
-	{ _MMIO(0x9888), 0x124f4640 },
-	{ _MMIO(0x9888), 0x106c0232 },
-	{ _MMIO(0x9888), 0x11834400 },
-	{ _MMIO(0x9888), 0x0a4e8000 },
-	{ _MMIO(0x9888), 0x0c4e8000 },
-	{ _MMIO(0x9888), 0x004f1880 },
-	{ _MMIO(0x9888), 0x024f08bb },
-	{ _MMIO(0x9888), 0x044f001b },
-	{ _MMIO(0x9888), 0x046c0100 },
-	{ _MMIO(0x9888), 0x066c000b },
-	{ _MMIO(0x9888), 0x1a6c0000 },
-	{ _MMIO(0x9888), 0x041b8000 },
-	{ _MMIO(0x9888), 0x061b4000 },
-	{ _MMIO(0x9888), 0x1a1c1800 },
-	{ _MMIO(0x9888), 0x005b8000 },
-	{ _MMIO(0x9888), 0x025bc000 },
-	{ _MMIO(0x9888), 0x045b4000 },
-	{ _MMIO(0x9888), 0x125c8000 },
-	{ _MMIO(0x9888), 0x145c8000 },
-	{ _MMIO(0x9888), 0x165c8000 },
-	{ _MMIO(0x9888), 0x185c8000 },
-	{ _MMIO(0x9888), 0x0a4c00a0 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x0c0f5000 },
-	{ _MMIO(0x9888), 0x0e0f0055 },
-	{ _MMIO(0x9888), 0x022cc000 },
-	{ _MMIO(0x9888), 0x042cc000 },
-	{ _MMIO(0x9888), 0x062cc000 },
-	{ _MMIO(0x9888), 0x082cc000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x0f828000 },
-	{ _MMIO(0x9888), 0x0f8305c0 },
-	{ _MMIO(0x9888), 0x09830000 },
-	{ _MMIO(0x9888), 0x07830000 },
-	{ _MMIO(0x9888), 0x1d950080 },
-	{ _MMIO(0x9888), 0x13928000 },
-	{ _MMIO(0x9888), 0x0f988000 },
-	{ _MMIO(0x9888), 0x31904000 },
-	{ _MMIO(0x9888), 0x1190fc00 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x59900001 },
-	{ _MMIO(0x9888), 0x4b900040 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900800 },
-	{ _MMIO(0x9888), 0x43900842 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_hdc_and_sf_mux_config(struct drm_i915_private *dev_priv,
-			  const struct i915_oa_reg **regs,
-			  int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_hdc_and_sf;
-	lens[n] = ARRAY_SIZE(mux_config_hdc_and_sf);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_l3_1[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0xf0800000 },
-	{ _MMIO(0x2770), 0x00100070 },
-	{ _MMIO(0x2774), 0x0000fff1 },
-	{ _MMIO(0x2778), 0x00014002 },
-	{ _MMIO(0x277c), 0x0000c3ff },
-	{ _MMIO(0x2780), 0x00010002 },
-	{ _MMIO(0x2784), 0x0000c7ff },
-	{ _MMIO(0x2788), 0x00004002 },
-	{ _MMIO(0x278c), 0x0000d3ff },
-	{ _MMIO(0x2790), 0x00100700 },
-	{ _MMIO(0x2794), 0x0000ff1f },
-	{ _MMIO(0x2798), 0x00001402 },
-	{ _MMIO(0x279c), 0x0000fc3f },
-	{ _MMIO(0x27a0), 0x00001002 },
-	{ _MMIO(0x27a4), 0x0000fc7f },
-	{ _MMIO(0x27a8), 0x00000402 },
-	{ _MMIO(0x27ac), 0x0000fd3f },
-};
-
-static const struct i915_oa_reg flex_eu_config_l3_1[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_l3_1[] = {
-	{ _MMIO(0x9888), 0x126c7b40 },
-	{ _MMIO(0x9888), 0x166c0020 },
-	{ _MMIO(0x9888), 0x0a603444 },
-	{ _MMIO(0x9888), 0x0a613400 },
-	{ _MMIO(0x9888), 0x1a4ea800 },
-	{ _MMIO(0x9888), 0x1c4e0002 },
-	{ _MMIO(0x9888), 0x024e8000 },
-	{ _MMIO(0x9888), 0x044e8000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084e8000 },
-	{ _MMIO(0x9888), 0x0a4e8000 },
-	{ _MMIO(0x9888), 0x064f4000 },
-	{ _MMIO(0x9888), 0x0c6c5327 },
-	{ _MMIO(0x9888), 0x0e6c5425 },
-	{ _MMIO(0x9888), 0x006c2a00 },
-	{ _MMIO(0x9888), 0x026c285b },
-	{ _MMIO(0x9888), 0x046c005c },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x1c6c0000 },
-	{ _MMIO(0x9888), 0x1e6c0000 },
-	{ _MMIO(0x9888), 0x1a6c0800 },
-	{ _MMIO(0x9888), 0x0c1bc000 },
-	{ _MMIO(0x9888), 0x0e1bc000 },
-	{ _MMIO(0x9888), 0x001b8000 },
-	{ _MMIO(0x9888), 0x021bc000 },
-	{ _MMIO(0x9888), 0x041bc000 },
-	{ _MMIO(0x9888), 0x1c1c003c },
-	{ _MMIO(0x9888), 0x121c8000 },
-	{ _MMIO(0x9888), 0x141c8000 },
-	{ _MMIO(0x9888), 0x161c8000 },
-	{ _MMIO(0x9888), 0x181c8000 },
-	{ _MMIO(0x9888), 0x1a1c0800 },
-	{ _MMIO(0x9888), 0x065b4000 },
-	{ _MMIO(0x9888), 0x1a5c1000 },
-	{ _MMIO(0x9888), 0x10600000 },
-	{ _MMIO(0x9888), 0x04600000 },
-	{ _MMIO(0x9888), 0x0c610044 },
-	{ _MMIO(0x9888), 0x10610000 },
-	{ _MMIO(0x9888), 0x06610000 },
-	{ _MMIO(0x9888), 0x0c4c02a8 },
-	{ _MMIO(0x9888), 0x084ca000 },
-	{ _MMIO(0x9888), 0x0a4c002a },
-	{ _MMIO(0x9888), 0x0c0da000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x100f0154 },
-	{ _MMIO(0x9888), 0x0c0f5000 },
-	{ _MMIO(0x9888), 0x0e0f0055 },
-	{ _MMIO(0x9888), 0x182c00aa },
-	{ _MMIO(0x9888), 0x022c8000 },
-	{ _MMIO(0x9888), 0x042c8000 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2cc000 },
-	{ _MMIO(0x9888), 0x1190ffc0 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900420 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900021 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900400 },
-	{ _MMIO(0x9888), 0x43900421 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900040 },
-};
-
-static int
-get_l3_1_mux_config(struct drm_i915_private *dev_priv,
-		    const struct i915_oa_reg **regs,
-		    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_l3_1;
-	lens[n] = ARRAY_SIZE(mux_config_l3_1);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_l3_2[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00100070 },
-	{ _MMIO(0x2774), 0x0000fff1 },
-	{ _MMIO(0x2778), 0x00028002 },
-	{ _MMIO(0x277c), 0x000087ff },
-	{ _MMIO(0x2780), 0x00020002 },
-	{ _MMIO(0x2784), 0x00008fff },
-	{ _MMIO(0x2788), 0x00008002 },
-	{ _MMIO(0x278c), 0x0000a7ff },
-};
-
-static const struct i915_oa_reg flex_eu_config_l3_2[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_l3_2[] = {
-	{ _MMIO(0x9888), 0x126c02e0 },
-	{ _MMIO(0x9888), 0x146c0001 },
-	{ _MMIO(0x9888), 0x0a623400 },
-	{ _MMIO(0x9888), 0x044e8000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084e8000 },
-	{ _MMIO(0x9888), 0x0a4e8000 },
-	{ _MMIO(0x9888), 0x064f4000 },
-	{ _MMIO(0x9888), 0x026c3324 },
-	{ _MMIO(0x9888), 0x046c3422 },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x1a6c0000 },
-	{ _MMIO(0x9888), 0x021bc000 },
-	{ _MMIO(0x9888), 0x041bc000 },
-	{ _MMIO(0x9888), 0x141c8000 },
-	{ _MMIO(0x9888), 0x161c8000 },
-	{ _MMIO(0x9888), 0x181c8000 },
-	{ _MMIO(0x9888), 0x1a1c0800 },
-	{ _MMIO(0x9888), 0x065b4000 },
-	{ _MMIO(0x9888), 0x1a5c1000 },
-	{ _MMIO(0x9888), 0x06614000 },
-	{ _MMIO(0x9888), 0x0c620044 },
-	{ _MMIO(0x9888), 0x10620000 },
-	{ _MMIO(0x9888), 0x06620000 },
-	{ _MMIO(0x9888), 0x084c8000 },
-	{ _MMIO(0x9888), 0x0a4c002a },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x0c0f4000 },
-	{ _MMIO(0x9888), 0x0e0f0055 },
-	{ _MMIO(0x9888), 0x042c8000 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2cc000 },
-	{ _MMIO(0x9888), 0x1190f800 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x43900000 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_l3_2_mux_config(struct drm_i915_private *dev_priv,
-		    const struct i915_oa_reg **regs,
-		    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_l3_2;
-	lens[n] = ARRAY_SIZE(mux_config_l3_2);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_l3_3[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00100070 },
-	{ _MMIO(0x2774), 0x0000fff1 },
-	{ _MMIO(0x2778), 0x00028002 },
-	{ _MMIO(0x277c), 0x000087ff },
-	{ _MMIO(0x2780), 0x00020002 },
-	{ _MMIO(0x2784), 0x00008fff },
-	{ _MMIO(0x2788), 0x00008002 },
-	{ _MMIO(0x278c), 0x0000a7ff },
-};
-
-static const struct i915_oa_reg flex_eu_config_l3_3[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_l3_3[] = {
-	{ _MMIO(0x9888), 0x126c4e80 },
-	{ _MMIO(0x9888), 0x146c0000 },
-	{ _MMIO(0x9888), 0x0a633400 },
-	{ _MMIO(0x9888), 0x044e8000 },
-	{ _MMIO(0x9888), 0x064e8000 },
-	{ _MMIO(0x9888), 0x084e8000 },
-	{ _MMIO(0x9888), 0x0a4e8000 },
-	{ _MMIO(0x9888), 0x0c4e8000 },
-	{ _MMIO(0x9888), 0x026c3321 },
-	{ _MMIO(0x9888), 0x046c342f },
-	{ _MMIO(0x9888), 0x106c0000 },
-	{ _MMIO(0x9888), 0x1a6c2000 },
-	{ _MMIO(0x9888), 0x021bc000 },
-	{ _MMIO(0x9888), 0x041bc000 },
-	{ _MMIO(0x9888), 0x061b4000 },
-	{ _MMIO(0x9888), 0x141c8000 },
-	{ _MMIO(0x9888), 0x161c8000 },
-	{ _MMIO(0x9888), 0x181c8000 },
-	{ _MMIO(0x9888), 0x1a1c1800 },
-	{ _MMIO(0x9888), 0x06604000 },
-	{ _MMIO(0x9888), 0x0c630044 },
-	{ _MMIO(0x9888), 0x10630000 },
-	{ _MMIO(0x9888), 0x06630000 },
-	{ _MMIO(0x9888), 0x084c8000 },
-	{ _MMIO(0x9888), 0x0a4c00aa },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x0c0f4000 },
-	{ _MMIO(0x9888), 0x0e0f0055 },
-	{ _MMIO(0x9888), 0x042c8000 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x1190f800 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x43900842 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900002 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_l3_3_mux_config(struct drm_i915_private *dev_priv,
-		    const struct i915_oa_reg **regs,
-		    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_l3_3;
-	lens[n] = ARRAY_SIZE(mux_config_l3_3);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x30800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x0000efff },
-	{ _MMIO(0x2778), 0x00006000 },
-	{ _MMIO(0x277c), 0x0000f3ff },
-};
-
-static const struct i915_oa_reg flex_eu_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_rasterizer_and_pixel_backend[] = {
-	{ _MMIO(0x9888), 0x102f3800 },
-	{ _MMIO(0x9888), 0x144d0500 },
-	{ _MMIO(0x9888), 0x120d03c0 },
-	{ _MMIO(0x9888), 0x140d03cf },
-	{ _MMIO(0x9888), 0x0c0f0004 },
-	{ _MMIO(0x9888), 0x0c4e4000 },
-	{ _MMIO(0x9888), 0x042f0480 },
-	{ _MMIO(0x9888), 0x082f0000 },
-	{ _MMIO(0x9888), 0x022f0000 },
-	{ _MMIO(0x9888), 0x0a4c0090 },
-	{ _MMIO(0x9888), 0x064d0027 },
-	{ _MMIO(0x9888), 0x004d0000 },
-	{ _MMIO(0x9888), 0x000d0d40 },
-	{ _MMIO(0x9888), 0x020d803f },
-	{ _MMIO(0x9888), 0x040d8023 },
-	{ _MMIO(0x9888), 0x100d0000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x020f0010 },
-	{ _MMIO(0x9888), 0x000f0000 },
-	{ _MMIO(0x9888), 0x0e0f0050 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x1190fc00 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41901400 },
-	{ _MMIO(0x9888), 0x43901485 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900001 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_rasterizer_and_pixel_backend_mux_config(struct drm_i915_private *dev_priv,
-					    const struct i915_oa_reg **regs,
-					    int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_rasterizer_and_pixel_backend;
-	lens[n] = ARRAY_SIZE(mux_config_rasterizer_and_pixel_backend);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_sampler[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x70800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-	{ _MMIO(0x2770), 0x0000c000 },
-	{ _MMIO(0x2774), 0x0000e7ff },
-	{ _MMIO(0x2778), 0x00003000 },
-	{ _MMIO(0x277c), 0x0000f9ff },
-	{ _MMIO(0x2780), 0x00000c00 },
-	{ _MMIO(0x2784), 0x0000fe7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_sampler[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_sampler[] = {
-	{ _MMIO(0x9888), 0x14152c00 },
-	{ _MMIO(0x9888), 0x16150005 },
-	{ _MMIO(0x9888), 0x121600a0 },
-	{ _MMIO(0x9888), 0x14352c00 },
-	{ _MMIO(0x9888), 0x16350005 },
-	{ _MMIO(0x9888), 0x123600a0 },
-	{ _MMIO(0x9888), 0x14552c00 },
-	{ _MMIO(0x9888), 0x16550005 },
-	{ _MMIO(0x9888), 0x125600a0 },
-	{ _MMIO(0x9888), 0x062f6000 },
-	{ _MMIO(0x9888), 0x022f2000 },
-	{ _MMIO(0x9888), 0x0c4c0050 },
-	{ _MMIO(0x9888), 0x0a4c0010 },
-	{ _MMIO(0x9888), 0x0c0d8000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x100f0350 },
-	{ _MMIO(0x9888), 0x0c0fb000 },
-	{ _MMIO(0x9888), 0x0e0f00da },
-	{ _MMIO(0x9888), 0x182c0028 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x022dc000 },
-	{ _MMIO(0x9888), 0x042d4000 },
-	{ _MMIO(0x9888), 0x0c138000 },
-	{ _MMIO(0x9888), 0x0e132000 },
-	{ _MMIO(0x9888), 0x0413c000 },
-	{ _MMIO(0x9888), 0x1c140018 },
-	{ _MMIO(0x9888), 0x0c157000 },
-	{ _MMIO(0x9888), 0x0e150078 },
-	{ _MMIO(0x9888), 0x10150000 },
-	{ _MMIO(0x9888), 0x04162180 },
-	{ _MMIO(0x9888), 0x02160000 },
-	{ _MMIO(0x9888), 0x04174000 },
-	{ _MMIO(0x9888), 0x0233a000 },
-	{ _MMIO(0x9888), 0x04333000 },
-	{ _MMIO(0x9888), 0x14348000 },
-	{ _MMIO(0x9888), 0x16348000 },
-	{ _MMIO(0x9888), 0x02357870 },
-	{ _MMIO(0x9888), 0x10350000 },
-	{ _MMIO(0x9888), 0x04360043 },
-	{ _MMIO(0x9888), 0x02360000 },
-	{ _MMIO(0x9888), 0x04371000 },
-	{ _MMIO(0x9888), 0x0e538000 },
-	{ _MMIO(0x9888), 0x00538000 },
-	{ _MMIO(0x9888), 0x06533000 },
-	{ _MMIO(0x9888), 0x1c540020 },
-	{ _MMIO(0x9888), 0x12548000 },
-	{ _MMIO(0x9888), 0x0e557000 },
-	{ _MMIO(0x9888), 0x00557800 },
-	{ _MMIO(0x9888), 0x10550000 },
-	{ _MMIO(0x9888), 0x06560043 },
-	{ _MMIO(0x9888), 0x02560000 },
-	{ _MMIO(0x9888), 0x06571000 },
-	{ _MMIO(0x9888), 0x1190ff80 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900000 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900060 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c00 },
-	{ _MMIO(0x9888), 0x43900842 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900060 },
-};
-
-static int
-get_sampler_mux_config(struct drm_i915_private *dev_priv,
-		       const struct i915_oa_reg **regs,
-		       int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_sampler;
-	lens[n] = ARRAY_SIZE(mux_config_sampler);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_tdl_1[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2770), 0x00000002 },
-	{ _MMIO(0x2774), 0x00007fff },
-	{ _MMIO(0x2778), 0x00000000 },
-	{ _MMIO(0x277c), 0x00009fff },
-	{ _MMIO(0x2780), 0x00000002 },
-	{ _MMIO(0x2784), 0x0000efff },
-	{ _MMIO(0x2788), 0x00000000 },
-	{ _MMIO(0x278c), 0x0000f3ff },
-	{ _MMIO(0x2790), 0x00000002 },
-	{ _MMIO(0x2794), 0x0000fdff },
-	{ _MMIO(0x2798), 0x00000000 },
-	{ _MMIO(0x279c), 0x0000fe7f },
-};
-
-static const struct i915_oa_reg flex_eu_config_tdl_1[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_tdl_1[] = {
-	{ _MMIO(0x9888), 0x12120000 },
-	{ _MMIO(0x9888), 0x12320000 },
-	{ _MMIO(0x9888), 0x12520000 },
-	{ _MMIO(0x9888), 0x002f8000 },
-	{ _MMIO(0x9888), 0x022f3000 },
-	{ _MMIO(0x9888), 0x0a4c0015 },
-	{ _MMIO(0x9888), 0x0c0d8000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x100f03a0 },
-	{ _MMIO(0x9888), 0x0c0ff000 },
-	{ _MMIO(0x9888), 0x0e0f0095 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x0c2d8000 },
-	{ _MMIO(0x9888), 0x0e2d4000 },
-	{ _MMIO(0x9888), 0x062d4000 },
-	{ _MMIO(0x9888), 0x02108000 },
-	{ _MMIO(0x9888), 0x0410c000 },
-	{ _MMIO(0x9888), 0x02118000 },
-	{ _MMIO(0x9888), 0x0411c000 },
-	{ _MMIO(0x9888), 0x02121880 },
-	{ _MMIO(0x9888), 0x041219b5 },
-	{ _MMIO(0x9888), 0x00120000 },
-	{ _MMIO(0x9888), 0x02134000 },
-	{ _MMIO(0x9888), 0x04135000 },
-	{ _MMIO(0x9888), 0x0c308000 },
-	{ _MMIO(0x9888), 0x0e304000 },
-	{ _MMIO(0x9888), 0x06304000 },
-	{ _MMIO(0x9888), 0x0c318000 },
-	{ _MMIO(0x9888), 0x0e314000 },
-	{ _MMIO(0x9888), 0x06314000 },
-	{ _MMIO(0x9888), 0x0c321a80 },
-	{ _MMIO(0x9888), 0x0e320033 },
-	{ _MMIO(0x9888), 0x06320031 },
-	{ _MMIO(0x9888), 0x00320000 },
-	{ _MMIO(0x9888), 0x0c334000 },
-	{ _MMIO(0x9888), 0x0e331000 },
-	{ _MMIO(0x9888), 0x06331000 },
-	{ _MMIO(0x9888), 0x0e508000 },
-	{ _MMIO(0x9888), 0x00508000 },
-	{ _MMIO(0x9888), 0x02504000 },
-	{ _MMIO(0x9888), 0x0e518000 },
-	{ _MMIO(0x9888), 0x00518000 },
-	{ _MMIO(0x9888), 0x02514000 },
-	{ _MMIO(0x9888), 0x0e521880 },
-	{ _MMIO(0x9888), 0x00521a80 },
-	{ _MMIO(0x9888), 0x02520033 },
-	{ _MMIO(0x9888), 0x0e534000 },
-	{ _MMIO(0x9888), 0x00534000 },
-	{ _MMIO(0x9888), 0x02531000 },
-	{ _MMIO(0x9888), 0x1190ff80 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900800 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900062 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900c00 },
-	{ _MMIO(0x9888), 0x43900003 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900040 },
-};
-
-static int
-get_tdl_1_mux_config(struct drm_i915_private *dev_priv,
-		     const struct i915_oa_reg **regs,
-		     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_tdl_1;
-	lens[n] = ARRAY_SIZE(mux_config_tdl_1);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_tdl_2[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2744), 0x00800000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0x00800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x00800000 },
-};
-
-static const struct i915_oa_reg flex_eu_config_tdl_2[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00010003 },
-	{ _MMIO(0xe658), 0x00012011 },
-	{ _MMIO(0xe758), 0x00015014 },
-	{ _MMIO(0xe45c), 0x00051050 },
-	{ _MMIO(0xe55c), 0x00053052 },
-	{ _MMIO(0xe65c), 0x00055054 },
-};
-
-static const struct i915_oa_reg mux_config_tdl_2[] = {
-	{ _MMIO(0x9888), 0x12124d60 },
-	{ _MMIO(0x9888), 0x12322e60 },
-	{ _MMIO(0x9888), 0x12524d60 },
-	{ _MMIO(0x9888), 0x022f3000 },
-	{ _MMIO(0x9888), 0x0a4c0014 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x0c0fe000 },
-	{ _MMIO(0x9888), 0x0e0f0097 },
-	{ _MMIO(0x9888), 0x082c8000 },
-	{ _MMIO(0x9888), 0x0a2c8000 },
-	{ _MMIO(0x9888), 0x002d8000 },
-	{ _MMIO(0x9888), 0x062d4000 },
-	{ _MMIO(0x9888), 0x0410c000 },
-	{ _MMIO(0x9888), 0x0411c000 },
-	{ _MMIO(0x9888), 0x04121fb7 },
-	{ _MMIO(0x9888), 0x00120000 },
-	{ _MMIO(0x9888), 0x04135000 },
-	{ _MMIO(0x9888), 0x00308000 },
-	{ _MMIO(0x9888), 0x06304000 },
-	{ _MMIO(0x9888), 0x00318000 },
-	{ _MMIO(0x9888), 0x06314000 },
-	{ _MMIO(0x9888), 0x00321b80 },
-	{ _MMIO(0x9888), 0x0632003f },
-	{ _MMIO(0x9888), 0x00334000 },
-	{ _MMIO(0x9888), 0x06331000 },
-	{ _MMIO(0x9888), 0x0250c000 },
-	{ _MMIO(0x9888), 0x0251c000 },
-	{ _MMIO(0x9888), 0x02521fb7 },
-	{ _MMIO(0x9888), 0x00520000 },
-	{ _MMIO(0x9888), 0x02535000 },
-	{ _MMIO(0x9888), 0x1190fc00 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x51900000 },
-	{ _MMIO(0x9888), 0x41900800 },
-	{ _MMIO(0x9888), 0x43900063 },
-	{ _MMIO(0x9888), 0x53900000 },
-	{ _MMIO(0x9888), 0x45900040 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_tdl_2_mux_config(struct drm_i915_private *dev_priv,
-		     const struct i915_oa_reg **regs,
-		     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_tdl_2;
-	lens[n] = ARRAY_SIZE(mux_config_tdl_2);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_compute_extra[] = {
-};
-
-static const struct i915_oa_reg flex_eu_config_compute_extra[] = {
-};
-
-static const struct i915_oa_reg mux_config_compute_extra[] = {
-	{ _MMIO(0x9888), 0x121203e0 },
-	{ _MMIO(0x9888), 0x123203e0 },
-	{ _MMIO(0x9888), 0x125203e0 },
-	{ _MMIO(0x9888), 0x129203e0 },
-	{ _MMIO(0x9888), 0x12b203e0 },
-	{ _MMIO(0x9888), 0x12d203e0 },
-	{ _MMIO(0x9888), 0x131203e0 },
-	{ _MMIO(0x9888), 0x133203e0 },
-	{ _MMIO(0x9888), 0x135203e0 },
-	{ _MMIO(0x9888), 0x1a4ef000 },
-	{ _MMIO(0x9888), 0x1c4e0003 },
-	{ _MMIO(0x9888), 0x024ec000 },
-	{ _MMIO(0x9888), 0x044ec000 },
-	{ _MMIO(0x9888), 0x064ec000 },
-	{ _MMIO(0x9888), 0x022f4000 },
-	{ _MMIO(0x9888), 0x0c4c02a0 },
-	{ _MMIO(0x9888), 0x084ca000 },
-	{ _MMIO(0x9888), 0x0a4c0042 },
-	{ _MMIO(0x9888), 0x0c0d8000 },
-	{ _MMIO(0x9888), 0x0e0da000 },
-	{ _MMIO(0x9888), 0x000d8000 },
-	{ _MMIO(0x9888), 0x020da000 },
-	{ _MMIO(0x9888), 0x040da000 },
-	{ _MMIO(0x9888), 0x060d2000 },
-	{ _MMIO(0x9888), 0x100f0150 },
-	{ _MMIO(0x9888), 0x0c0f5000 },
-	{ _MMIO(0x9888), 0x0e0f006d },
-	{ _MMIO(0x9888), 0x182c00a8 },
-	{ _MMIO(0x9888), 0x022c8000 },
-	{ _MMIO(0x9888), 0x042c8000 },
-	{ _MMIO(0x9888), 0x062c8000 },
-	{ _MMIO(0x9888), 0x0c2c8000 },
-	{ _MMIO(0x9888), 0x042d8000 },
-	{ _MMIO(0x9888), 0x06104000 },
-	{ _MMIO(0x9888), 0x06114000 },
-	{ _MMIO(0x9888), 0x06120033 },
-	{ _MMIO(0x9888), 0x00120000 },
-	{ _MMIO(0x9888), 0x06131000 },
-	{ _MMIO(0x9888), 0x04308000 },
-	{ _MMIO(0x9888), 0x04318000 },
-	{ _MMIO(0x9888), 0x04321980 },
-	{ _MMIO(0x9888), 0x00320000 },
-	{ _MMIO(0x9888), 0x04334000 },
-	{ _MMIO(0x9888), 0x04504000 },
-	{ _MMIO(0x9888), 0x04514000 },
-	{ _MMIO(0x9888), 0x04520033 },
-	{ _MMIO(0x9888), 0x00520000 },
-	{ _MMIO(0x9888), 0x04531000 },
-	{ _MMIO(0x9888), 0x1acef000 },
-	{ _MMIO(0x9888), 0x1cce0003 },
-	{ _MMIO(0x9888), 0x00af8000 },
-	{ _MMIO(0x9888), 0x0ccc02a0 },
-	{ _MMIO(0x9888), 0x0acc0001 },
-	{ _MMIO(0x9888), 0x0c8d8000 },
-	{ _MMIO(0x9888), 0x0e8da000 },
-	{ _MMIO(0x9888), 0x008d8000 },
-	{ _MMIO(0x9888), 0x028da000 },
-	{ _MMIO(0x9888), 0x108f0150 },
-	{ _MMIO(0x9888), 0x0c8fb000 },
-	{ _MMIO(0x9888), 0x0e8f0001 },
-	{ _MMIO(0x9888), 0x18ac00a8 },
-	{ _MMIO(0x9888), 0x06ac8000 },
-	{ _MMIO(0x9888), 0x02ad4000 },
-	{ _MMIO(0x9888), 0x02908000 },
-	{ _MMIO(0x9888), 0x02918000 },
-	{ _MMIO(0x9888), 0x02921980 },
-	{ _MMIO(0x9888), 0x00920000 },
-	{ _MMIO(0x9888), 0x02934000 },
-	{ _MMIO(0x9888), 0x02b04000 },
-	{ _MMIO(0x9888), 0x02b14000 },
-	{ _MMIO(0x9888), 0x02b20033 },
-	{ _MMIO(0x9888), 0x00b20000 },
-	{ _MMIO(0x9888), 0x02b31000 },
-	{ _MMIO(0x9888), 0x00d08000 },
-	{ _MMIO(0x9888), 0x00d18000 },
-	{ _MMIO(0x9888), 0x00d21980 },
-	{ _MMIO(0x9888), 0x00d34000 },
-	{ _MMIO(0x9888), 0x072f8000 },
-	{ _MMIO(0x9888), 0x0d4c0100 },
-	{ _MMIO(0x9888), 0x0d0d8000 },
-	{ _MMIO(0x9888), 0x0f0da000 },
-	{ _MMIO(0x9888), 0x110f01b0 },
-	{ _MMIO(0x9888), 0x192c0080 },
-	{ _MMIO(0x9888), 0x0f2d4000 },
-	{ _MMIO(0x9888), 0x0f108000 },
-	{ _MMIO(0x9888), 0x0f118000 },
-	{ _MMIO(0x9888), 0x0f121980 },
-	{ _MMIO(0x9888), 0x01120000 },
-	{ _MMIO(0x9888), 0x0f134000 },
-	{ _MMIO(0x9888), 0x0f304000 },
-	{ _MMIO(0x9888), 0x0f314000 },
-	{ _MMIO(0x9888), 0x0f320033 },
-	{ _MMIO(0x9888), 0x01320000 },
-	{ _MMIO(0x9888), 0x0f331000 },
-	{ _MMIO(0x9888), 0x0d508000 },
-	{ _MMIO(0x9888), 0x0d518000 },
-	{ _MMIO(0x9888), 0x0d521980 },
-	{ _MMIO(0x9888), 0x01520000 },
-	{ _MMIO(0x9888), 0x0d534000 },
-	{ _MMIO(0x9888), 0x1190ff80 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900c00 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-	{ _MMIO(0x9888), 0x4b900002 },
-	{ _MMIO(0x9888), 0x59900000 },
-	{ _MMIO(0x9888), 0x51901100 },
-	{ _MMIO(0x9888), 0x41901000 },
-	{ _MMIO(0x9888), 0x43901423 },
-	{ _MMIO(0x9888), 0x53903331 },
-	{ _MMIO(0x9888), 0x45900044 },
-};
-
-static int
-get_compute_extra_mux_config(struct drm_i915_private *dev_priv,
-			     const struct i915_oa_reg **regs,
-			     int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_compute_extra;
-	lens[n] = ARRAY_SIZE(mux_config_compute_extra);
-	n++;
-
-	return n;
-}
-
-static const struct i915_oa_reg b_counter_config_vme_pipe[] = {
-	{ _MMIO(0x2740), 0x00000000 },
-	{ _MMIO(0x2710), 0x00000000 },
-	{ _MMIO(0x2714), 0xf0800000 },
-	{ _MMIO(0x2720), 0x00000000 },
-	{ _MMIO(0x2724), 0x30800000 },
-	{ _MMIO(0x2770), 0x00100030 },
-	{ _MMIO(0x2774), 0x0000fff9 },
-	{ _MMIO(0x2778), 0x00000002 },
-	{ _MMIO(0x277c), 0x0000fffc },
-	{ _MMIO(0x2780), 0x00000002 },
-	{ _MMIO(0x2784), 0x0000fff3 },
-	{ _MMIO(0x2788), 0x00100180 },
-	{ _MMIO(0x278c), 0x0000ffcf },
-	{ _MMIO(0x2790), 0x00000002 },
-	{ _MMIO(0x2794), 0x0000ffcf },
-	{ _MMIO(0x2798), 0x00000002 },
-	{ _MMIO(0x279c), 0x0000ff3f },
-};
-
-static const struct i915_oa_reg flex_eu_config_vme_pipe[] = {
-	{ _MMIO(0xe458), 0x00005004 },
-	{ _MMIO(0xe558), 0x00008003 },
-};
-
-static const struct i915_oa_reg mux_config_vme_pipe[] = {
-	{ _MMIO(0x9888), 0x141a5800 },
-	{ _MMIO(0x9888), 0x161a00c0 },
-	{ _MMIO(0x9888), 0x12180240 },
-	{ _MMIO(0x9888), 0x14180002 },
-	{ _MMIO(0x9888), 0x149a5800 },
-	{ _MMIO(0x9888), 0x169a00c0 },
-	{ _MMIO(0x9888), 0x12980240 },
-	{ _MMIO(0x9888), 0x14980002 },
-	{ _MMIO(0x9888), 0x1a4e3fc0 },
-	{ _MMIO(0x9888), 0x002f1000 },
-	{ _MMIO(0x9888), 0x022f8000 },
-	{ _MMIO(0x9888), 0x042f3000 },
-	{ _MMIO(0x9888), 0x004c4000 },
-	{ _MMIO(0x9888), 0x0a4c9500 },
-	{ _MMIO(0x9888), 0x0c4c002a },
-	{ _MMIO(0x9888), 0x000d2000 },
-	{ _MMIO(0x9888), 0x060d8000 },
-	{ _MMIO(0x9888), 0x080da000 },
-	{ _MMIO(0x9888), 0x0a0da000 },
-	{ _MMIO(0x9888), 0x0c0da000 },
-	{ _MMIO(0x9888), 0x0c0f0400 },
-	{ _MMIO(0x9888), 0x0e0f5500 },
-	{ _MMIO(0x9888), 0x100f0015 },
-	{ _MMIO(0x9888), 0x002c8000 },
-	{ _MMIO(0x9888), 0x0e2c8000 },
-	{ _MMIO(0x9888), 0x162caa00 },
-	{ _MMIO(0x9888), 0x182c000a },
-	{ _MMIO(0x9888), 0x04193000 },
-	{ _MMIO(0x9888), 0x081a28c1 },
-	{ _MMIO(0x9888), 0x001a0000 },
-	{ _MMIO(0x9888), 0x00133000 },
-	{ _MMIO(0x9888), 0x0613c000 },
-	{ _MMIO(0x9888), 0x0813f000 },
-	{ _MMIO(0x9888), 0x00172000 },
-	{ _MMIO(0x9888), 0x06178000 },
-	{ _MMIO(0x9888), 0x0817a000 },
-	{ _MMIO(0x9888), 0x00180037 },
-	{ _MMIO(0x9888), 0x06180940 },
-	{ _MMIO(0x9888), 0x08180000 },
-	{ _MMIO(0x9888), 0x02180000 },
-	{ _MMIO(0x9888), 0x04183000 },
-	{ _MMIO(0x9888), 0x04afc000 },
-	{ _MMIO(0x9888), 0x06af3000 },
-	{ _MMIO(0x9888), 0x0acc4000 },
-	{ _MMIO(0x9888), 0x0ccc0015 },
-	{ _MMIO(0x9888), 0x0a8da000 },
-	{ _MMIO(0x9888), 0x0c8da000 },
-	{ _MMIO(0x9888), 0x0e8f4000 },
-	{ _MMIO(0x9888), 0x108f0015 },
-	{ _MMIO(0x9888), 0x16aca000 },
-	{ _MMIO(0x9888), 0x18ac000a },
-	{ _MMIO(0x9888), 0x06993000 },
-	{ _MMIO(0x9888), 0x0c9a28c1 },
-	{ _MMIO(0x9888), 0x009a0000 },
-	{ _MMIO(0x9888), 0x0a93f000 },
-	{ _MMIO(0x9888), 0x0c93f000 },
-	{ _MMIO(0x9888), 0x0a97a000 },
-	{ _MMIO(0x9888), 0x0c97a000 },
-	{ _MMIO(0x9888), 0x0a980977 },
-	{ _MMIO(0x9888), 0x08980000 },
-	{ _MMIO(0x9888), 0x04980000 },
-	{ _MMIO(0x9888), 0x06983000 },
-	{ _MMIO(0x9888), 0x119000ff },
-	{ _MMIO(0x9888), 0x51900010 },
-	{ _MMIO(0x9888), 0x41900060 },
-	{ _MMIO(0x9888), 0x55900111 },
-	{ _MMIO(0x9888), 0x45900c00 },
-	{ _MMIO(0x9888), 0x47900821 },
-	{ _MMIO(0x9888), 0x57900000 },
-	{ _MMIO(0x9888), 0x49900002 },
-	{ _MMIO(0x9888), 0x37900000 },
-	{ _MMIO(0x9888), 0x33900000 },
-};
-
-static int
-get_vme_pipe_mux_config(struct drm_i915_private *dev_priv,
-			const struct i915_oa_reg **regs,
-			int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_vme_pipe;
-	lens[n] = ARRAY_SIZE(mux_config_vme_pipe);
-	n++;
-
-	return n;
-}
-
 static const struct i915_oa_reg b_counter_config_test_oa[] = {
 	{ _MMIO(0x2740), 0x00000000 },
 	{ _MMIO(0x2744), 0x00800000 },
@@ -1984,6 +60,7 @@ static const struct i915_oa_reg flex_eu_config_test_oa[] = {
 };
 
 static const struct i915_oa_reg mux_config_test_oa[] = {
+	{ _MMIO(0x9840), 0x00000080 },
 	{ _MMIO(0x9888), 0x11810000 },
 	{ _MMIO(0x9888), 0x07810013 },
 	{ _MMIO(0x9888), 0x1f810000 },
@@ -1998,1096 +75,35 @@ static const struct i915_oa_reg mux_config_test_oa[] = {
 	{ _MMIO(0x9888), 0x33900000 },
 };
 
-static int
-get_test_oa_mux_config(struct drm_i915_private *dev_priv,
-		       const struct i915_oa_reg **regs,
-		       int *lens)
-{
-	int n = 0;
-
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs) < 1);
-	BUILD_BUG_ON(ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens) < 1);
-
-	regs[n] = mux_config_test_oa;
-	lens[n] = ARRAY_SIZE(mux_config_test_oa);
-	n++;
-
-	return n;
-}
-
-int i915_oa_select_metric_set_sklgt4(struct drm_i915_private *dev_priv)
-{
-	dev_priv->perf.oa.n_mux_configs = 0;
-	dev_priv->perf.oa.b_counter_regs = NULL;
-	dev_priv->perf.oa.b_counter_regs_len = 0;
-	dev_priv->perf.oa.flex_regs = NULL;
-	dev_priv->perf.oa.flex_regs_len = 0;
-
-	switch (dev_priv->perf.oa.metrics_set) {
-	case METRIC_SET_ID_RENDER_BASIC:
-		dev_priv->perf.oa.n_mux_configs =
-			get_render_basic_mux_config(dev_priv,
-						    dev_priv->perf.oa.mux_regs,
-						    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_BASIC\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_render_basic;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_render_basic);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_render_basic;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_render_basic);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_BASIC:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_basic_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_BASIC\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_basic;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_basic);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_basic;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_basic);
-
-		return 0;
-	case METRIC_SET_ID_RENDER_PIPE_PROFILE:
-		dev_priv->perf.oa.n_mux_configs =
-			get_render_pipe_profile_mux_config(dev_priv,
-							   dev_priv->perf.oa.mux_regs,
-							   dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RENDER_PIPE_PROFILE\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_render_pipe_profile;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_render_pipe_profile);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_render_pipe_profile;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_render_pipe_profile);
-
-		return 0;
-	case METRIC_SET_ID_MEMORY_READS:
-		dev_priv->perf.oa.n_mux_configs =
-			get_memory_reads_mux_config(dev_priv,
-						    dev_priv->perf.oa.mux_regs,
-						    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_READS\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_memory_reads;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_memory_reads);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_memory_reads;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_memory_reads);
-
-		return 0;
-	case METRIC_SET_ID_MEMORY_WRITES:
-		dev_priv->perf.oa.n_mux_configs =
-			get_memory_writes_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"MEMORY_WRITES\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_memory_writes;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_memory_writes);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_memory_writes;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_memory_writes);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_EXTENDED:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_extended_mux_config(dev_priv,
-							dev_priv->perf.oa.mux_regs,
-							dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTENDED\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_extended;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_extended);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_extended;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_extended);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_L3_CACHE:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_l3_cache_mux_config(dev_priv,
-							dev_priv->perf.oa.mux_regs,
-							dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_L3_CACHE\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_l3_cache;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_l3_cache);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_l3_cache;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_l3_cache);
-
-		return 0;
-	case METRIC_SET_ID_HDC_AND_SF:
-		dev_priv->perf.oa.n_mux_configs =
-			get_hdc_and_sf_mux_config(dev_priv,
-						  dev_priv->perf.oa.mux_regs,
-						  dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"HDC_AND_SF\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_hdc_and_sf;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_hdc_and_sf);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_hdc_and_sf;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_hdc_and_sf);
-
-		return 0;
-	case METRIC_SET_ID_L3_1:
-		dev_priv->perf.oa.n_mux_configs =
-			get_l3_1_mux_config(dev_priv,
-					    dev_priv->perf.oa.mux_regs,
-					    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_1\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_l3_1;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_l3_1);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_l3_1;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_l3_1);
-
-		return 0;
-	case METRIC_SET_ID_L3_2:
-		dev_priv->perf.oa.n_mux_configs =
-			get_l3_2_mux_config(dev_priv,
-					    dev_priv->perf.oa.mux_regs,
-					    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_2\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_l3_2;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_l3_2);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_l3_2;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_l3_2);
-
-		return 0;
-	case METRIC_SET_ID_L3_3:
-		dev_priv->perf.oa.n_mux_configs =
-			get_l3_3_mux_config(dev_priv,
-					    dev_priv->perf.oa.mux_regs,
-					    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"L3_3\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_l3_3;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_l3_3);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_l3_3;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_l3_3);
-
-		return 0;
-	case METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND:
-		dev_priv->perf.oa.n_mux_configs =
-			get_rasterizer_and_pixel_backend_mux_config(dev_priv,
-								    dev_priv->perf.oa.mux_regs,
-								    dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"RASTERIZER_AND_PIXEL_BACKEND\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_rasterizer_and_pixel_backend;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_rasterizer_and_pixel_backend);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_rasterizer_and_pixel_backend;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_rasterizer_and_pixel_backend);
-
-		return 0;
-	case METRIC_SET_ID_SAMPLER:
-		dev_priv->perf.oa.n_mux_configs =
-			get_sampler_mux_config(dev_priv,
-					       dev_priv->perf.oa.mux_regs,
-					       dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"SAMPLER\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_sampler;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_sampler);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_sampler;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_sampler);
-
-		return 0;
-	case METRIC_SET_ID_TDL_1:
-		dev_priv->perf.oa.n_mux_configs =
-			get_tdl_1_mux_config(dev_priv,
-					     dev_priv->perf.oa.mux_regs,
-					     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_1\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_tdl_1;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_tdl_1);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_tdl_1;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_tdl_1);
-
-		return 0;
-	case METRIC_SET_ID_TDL_2:
-		dev_priv->perf.oa.n_mux_configs =
-			get_tdl_2_mux_config(dev_priv,
-					     dev_priv->perf.oa.mux_regs,
-					     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TDL_2\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_tdl_2;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_tdl_2);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_tdl_2;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_tdl_2);
-
-		return 0;
-	case METRIC_SET_ID_COMPUTE_EXTRA:
-		dev_priv->perf.oa.n_mux_configs =
-			get_compute_extra_mux_config(dev_priv,
-						     dev_priv->perf.oa.mux_regs,
-						     dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"COMPUTE_EXTRA\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_compute_extra;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_compute_extra);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_compute_extra;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_compute_extra);
-
-		return 0;
-	case METRIC_SET_ID_VME_PIPE:
-		dev_priv->perf.oa.n_mux_configs =
-			get_vme_pipe_mux_config(dev_priv,
-						dev_priv->perf.oa.mux_regs,
-						dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"VME_PIPE\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_vme_pipe;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_vme_pipe);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_vme_pipe;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_vme_pipe);
-
-		return 0;
-	case METRIC_SET_ID_TEST_OA:
-		dev_priv->perf.oa.n_mux_configs =
-			get_test_oa_mux_config(dev_priv,
-					       dev_priv->perf.oa.mux_regs,
-					       dev_priv->perf.oa.mux_regs_lens);
-		if (dev_priv->perf.oa.n_mux_configs == 0) {
-			DRM_DEBUG_DRIVER("No suitable MUX config for \"TEST_OA\" metric set\n");
-
-			/* EINVAL because *_register_sysfs already checked this
-			 * and so it wouldn't have been advertised to userspace and
-			 * so shouldn't have been requested
-			 */
-			return -EINVAL;
-		}
-
-		dev_priv->perf.oa.b_counter_regs =
-			b_counter_config_test_oa;
-		dev_priv->perf.oa.b_counter_regs_len =
-			ARRAY_SIZE(b_counter_config_test_oa);
-
-		dev_priv->perf.oa.flex_regs =
-			flex_eu_config_test_oa;
-		dev_priv->perf.oa.flex_regs_len =
-			ARRAY_SIZE(flex_eu_config_test_oa);
-
-		return 0;
-	default:
-		return -ENODEV;
-	}
-}
-
-static ssize_t
-show_render_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_BASIC);
-}
-
-static struct device_attribute dev_attr_render_basic_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_render_basic_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_render_basic[] = {
-	&dev_attr_render_basic_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_render_basic = {
-	.name = "bad77c24-cc64-480d-99bf-e7b740713800",
-	.attrs =  attrs_render_basic,
-};
-
-static ssize_t
-show_compute_basic_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_BASIC);
-}
-
-static struct device_attribute dev_attr_compute_basic_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_basic_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_basic[] = {
-	&dev_attr_compute_basic_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_basic = {
-	.name = "7277228f-e7f3-4743-945a-6a2049d11377",
-	.attrs =  attrs_compute_basic,
-};
-
-static ssize_t
-show_render_pipe_profile_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RENDER_PIPE_PROFILE);
-}
-
-static struct device_attribute dev_attr_render_pipe_profile_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_render_pipe_profile_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_render_pipe_profile[] = {
-	&dev_attr_render_pipe_profile_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_render_pipe_profile = {
-	.name = "463c668c-3f60-49b6-8f85-d995b635b3b2",
-	.attrs =  attrs_render_pipe_profile,
-};
-
-static ssize_t
-show_memory_reads_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_READS);
-}
-
-static struct device_attribute dev_attr_memory_reads_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_memory_reads_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_memory_reads[] = {
-	&dev_attr_memory_reads_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_memory_reads = {
-	.name = "3ae6e74c-72c3-4040-9bd0-7961430b8cc8",
-	.attrs =  attrs_memory_reads,
-};
-
-static ssize_t
-show_memory_writes_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_MEMORY_WRITES);
-}
-
-static struct device_attribute dev_attr_memory_writes_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_memory_writes_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_memory_writes[] = {
-	&dev_attr_memory_writes_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_memory_writes = {
-	.name = "055f256d-4052-467c-8dec-6064a4806433",
-	.attrs =  attrs_memory_writes,
-};
-
-static ssize_t
-show_compute_extended_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTENDED);
-}
-
-static struct device_attribute dev_attr_compute_extended_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_extended_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_extended[] = {
-	&dev_attr_compute_extended_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_extended = {
-	.name = "753972d4-87cd-4460-824d-754463ac5054",
-	.attrs =  attrs_compute_extended,
-};
-
-static ssize_t
-show_compute_l3_cache_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_L3_CACHE);
-}
-
-static struct device_attribute dev_attr_compute_l3_cache_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_l3_cache_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_l3_cache[] = {
-	&dev_attr_compute_l3_cache_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_l3_cache = {
-	.name = "4e4392e9-8f73-457b-ab44-b49f7a0c733b",
-	.attrs =  attrs_compute_l3_cache,
-};
-
-static ssize_t
-show_hdc_and_sf_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_HDC_AND_SF);
-}
-
-static struct device_attribute dev_attr_hdc_and_sf_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_hdc_and_sf_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_hdc_and_sf[] = {
-	&dev_attr_hdc_and_sf_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_hdc_and_sf = {
-	.name = "730d95dd-7da8-4e1c-ab8d-c0eb1e4c1805",
-	.attrs =  attrs_hdc_and_sf,
-};
-
-static ssize_t
-show_l3_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_1);
-}
-
-static struct device_attribute dev_attr_l3_1_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_l3_1_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_l3_1[] = {
-	&dev_attr_l3_1_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_l3_1 = {
-	.name = "d9e86d70-462b-462a-851e-fd63e8c13d63",
-	.attrs =  attrs_l3_1,
-};
-
-static ssize_t
-show_l3_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_2);
-}
-
-static struct device_attribute dev_attr_l3_2_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_l3_2_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_l3_2[] = {
-	&dev_attr_l3_2_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_l3_2 = {
-	.name = "52200424-6ee9-48b3-b7fa-0afcf1975e4d",
-	.attrs =  attrs_l3_2,
-};
-
-static ssize_t
-show_l3_3_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_L3_3);
-}
-
-static struct device_attribute dev_attr_l3_3_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_l3_3_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_l3_3[] = {
-	&dev_attr_l3_3_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_l3_3 = {
-	.name = "1988315f-0a26-44df-acb0-df7ec86b1456",
-	.attrs =  attrs_l3_3,
-};
-
-static ssize_t
-show_rasterizer_and_pixel_backend_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_RASTERIZER_AND_PIXEL_BACKEND);
-}
-
-static struct device_attribute dev_attr_rasterizer_and_pixel_backend_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_rasterizer_and_pixel_backend_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_rasterizer_and_pixel_backend[] = {
-	&dev_attr_rasterizer_and_pixel_backend_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_rasterizer_and_pixel_backend = {
-	.name = "f1f17ca7-286e-4ae5-9d15-9fccad6c665d",
-	.attrs =  attrs_rasterizer_and_pixel_backend,
-};
-
-static ssize_t
-show_sampler_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_SAMPLER);
-}
-
-static struct device_attribute dev_attr_sampler_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_sampler_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_sampler[] = {
-	&dev_attr_sampler_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_sampler = {
-	.name = "00a9e0fb-3d2e-4405-852c-dce6334ffb3b",
-	.attrs =  attrs_sampler,
-};
-
-static ssize_t
-show_tdl_1_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_1);
-}
-
-static struct device_attribute dev_attr_tdl_1_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_tdl_1_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_tdl_1[] = {
-	&dev_attr_tdl_1_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_tdl_1 = {
-	.name = "13dcc50a-7ec0-409b-99d6-a3f932cedcb3",
-	.attrs =  attrs_tdl_1,
-};
-
-static ssize_t
-show_tdl_2_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TDL_2);
-}
-
-static struct device_attribute dev_attr_tdl_2_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_tdl_2_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_tdl_2[] = {
-	&dev_attr_tdl_2_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_tdl_2 = {
-	.name = "97875e21-6624-4aee-9191-682feb3eae21",
-	.attrs =  attrs_tdl_2,
-};
-
-static ssize_t
-show_compute_extra_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_COMPUTE_EXTRA);
-}
-
-static struct device_attribute dev_attr_compute_extra_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_compute_extra_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_compute_extra[] = {
-	&dev_attr_compute_extra_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_compute_extra = {
-	.name = "a5aa857d-e8f0-4dfa-8981-ce340fa748fd",
-	.attrs =  attrs_compute_extra,
-};
-
-static ssize_t
-show_vme_pipe_id(struct device *kdev, struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", METRIC_SET_ID_VME_PIPE);
-}
-
-static struct device_attribute dev_attr_vme_pipe_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_vme_pipe_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_vme_pipe[] = {
-	&dev_attr_vme_pipe_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_vme_pipe = {
-	.name = "0e8d8b86-4ee7-4cdd-aaaa-58adc92cb29e",
-	.attrs =  attrs_vme_pipe,
-};
-
 static ssize_t
 show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-	return sprintf(buf, "%d\n", METRIC_SET_ID_TEST_OA);
+	return sprintf(buf, "1\n");
 }
 
-static struct device_attribute dev_attr_test_oa_id = {
-	.attr = { .name = "id", .mode = 0444 },
-	.show = show_test_oa_id,
-	.store = NULL,
-};
-
-static struct attribute *attrs_test_oa[] = {
-	&dev_attr_test_oa_id.attr,
-	NULL,
-};
-
-static struct attribute_group group_test_oa = {
-	.name = "882fa433-1f4a-4a67-a962-c741888fe5f5",
-	.attrs =  attrs_test_oa,
-};
-
-int
-i915_perf_register_sysfs_sklgt4(struct drm_i915_private *dev_priv)
+void
+i915_perf_load_test_config_sklgt4(struct drm_i915_private *dev_priv)
 {
-	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
-	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
-	int ret = 0;
+	strncpy(dev_priv->perf.oa.test_config.uuid,
+		"882fa433-1f4a-4a67-a962-c741888fe5f5",
+		UUID_STRING_LEN);
+	dev_priv->perf.oa.test_config.id = 1;
 
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-		if (ret)
-			goto error_render_basic;
-	}
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-		if (ret)
-			goto error_compute_basic;
-	}
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-		if (ret)
-			goto error_render_pipe_profile;
-	}
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-		if (ret)
-			goto error_memory_reads;
-	}
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-		if (ret)
-			goto error_memory_writes;
-	}
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-		if (ret)
-			goto error_compute_extended;
-	}
-	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
-		if (ret)
-			goto error_compute_l3_cache;
-	}
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-		if (ret)
-			goto error_hdc_and_sf;
-	}
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-		if (ret)
-			goto error_l3_1;
-	}
-	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_2);
-		if (ret)
-			goto error_l3_2;
-	}
-	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_l3_3);
-		if (ret)
-			goto error_l3_3;
-	}
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-		if (ret)
-			goto error_rasterizer_and_pixel_backend;
-	}
-	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_sampler);
-		if (ret)
-			goto error_sampler;
-	}
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-		if (ret)
-			goto error_tdl_1;
-	}
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-		if (ret)
-			goto error_tdl_2;
-	}
-	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
-		if (ret)
-			goto error_compute_extra;
-	}
-	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
-		if (ret)
-			goto error_vme_pipe;
-	}
-	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens)) {
-		ret = sysfs_create_group(dev_priv->perf.metrics_kobj, &group_test_oa);
-		if (ret)
-			goto error_test_oa;
-	}
+	dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
+	dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa);
 
-	return 0;
+	dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa;
+	dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa);
 
-error_test_oa:
-	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
-error_vme_pipe:
-	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
-error_compute_extra:
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-error_tdl_2:
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-error_tdl_1:
-	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
-error_sampler:
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-error_rasterizer_and_pixel_backend:
-	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
-error_l3_3:
-	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
-error_l3_2:
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-error_l3_1:
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-error_hdc_and_sf:
-	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
-error_compute_l3_cache:
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-error_compute_extended:
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-error_memory_writes:
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-error_memory_reads:
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-error_render_pipe_profile:
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-error_compute_basic:
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-error_render_basic:
-	return ret;
-}
+	dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa;
+	dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa);
 
-void
-i915_perf_unregister_sysfs_sklgt4(struct drm_i915_private *dev_priv)
-{
-	const struct i915_oa_reg *mux_regs[ARRAY_SIZE(dev_priv->perf.oa.mux_regs)];
-	int mux_lens[ARRAY_SIZE(dev_priv->perf.oa.mux_regs_lens)];
+	dev_priv->perf.oa.test_config.sysfs_metric.name = "882fa433-1f4a-4a67-a962-c741888fe5f5";
+	dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs;
+
+	dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr;
 
-	if (get_render_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_basic);
-	if (get_compute_basic_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_basic);
-	if (get_render_pipe_profile_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_render_pipe_profile);
-	if (get_memory_reads_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_reads);
-	if (get_memory_writes_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_memory_writes);
-	if (get_compute_extended_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extended);
-	if (get_compute_l3_cache_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_l3_cache);
-	if (get_hdc_and_sf_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_hdc_and_sf);
-	if (get_l3_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_1);
-	if (get_l3_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_2);
-	if (get_l3_3_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_l3_3);
-	if (get_rasterizer_and_pixel_backend_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_rasterizer_and_pixel_backend);
-	if (get_sampler_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_sampler);
-	if (get_tdl_1_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_1);
-	if (get_tdl_2_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_tdl_2);
-	if (get_compute_extra_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_compute_extra);
-	if (get_vme_pipe_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_vme_pipe);
-	if (get_test_oa_mux_config(dev_priv, mux_regs, mux_lens))
-		sysfs_remove_group(dev_priv->perf.metrics_kobj, &group_test_oa);
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id";
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444;
+	dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id;
 }
diff --git a/drivers/gpu/drm/i915/i915_oa_sklgt4.h b/drivers/gpu/drm/i915/i915_oa_sklgt4.h
index 1b718f15f62e..944fd525c8b1 100644
--- a/drivers/gpu/drm/i915/i915_oa_sklgt4.h
+++ b/drivers/gpu/drm/i915/i915_oa_sklgt4.h
@@ -29,12 +29,6 @@
 #ifndef __I915_OA_SKLGT4_H__
 #define __I915_OA_SKLGT4_H__
 
-extern int i915_oa_n_builtin_metric_sets_sklgt4;
-
-extern int i915_oa_select_metric_set_sklgt4(struct drm_i915_private *dev_priv);
-
-extern int i915_perf_register_sysfs_sklgt4(struct drm_i915_private *dev_priv);
-
-extern void i915_perf_unregister_sysfs_sklgt4(struct drm_i915_private *dev_priv);
+extern void i915_perf_load_test_config_sklgt4(struct drm_i915_private *dev_priv);
 
 #endif
diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c
index b6a7e363d076..8ab003dca113 100644
--- a/drivers/gpu/drm/i915/i915_params.c
+++ b/drivers/gpu/drm/i915/i915_params.c
@@ -46,7 +46,7 @@ struct i915_params i915 __read_mostly = {
 	.prefault_disable = 0,
 	.load_detect_test = 0,
 	.force_reset_modeset_test = 0,
-	.reset = true,
+	.reset = 2,
 	.error_capture = true,
 	.invert_brightness = 0,
 	.disable_display = 0,
@@ -115,8 +115,12 @@ MODULE_PARM_DESC(vbt_sdvo_panel_type,
 	"Override/Ignore selection of SDVO panel mode in the VBT "
 	"(-2=ignore, -1=auto [default], index in VBT BIOS table)");
 
-module_param_named_unsafe(reset, i915.reset, bool, 0600);
-MODULE_PARM_DESC(reset, "Attempt GPU resets (default: true)");
+module_param_named_unsafe(reset, i915.reset, int, 0600);
+MODULE_PARM_DESC(reset, "Attempt GPU resets (0=disabled, 1=full gpu reset, 2=engine reset [default])");
+
+module_param_named_unsafe(vbt_firmware, i915.vbt_firmware, charp, 0400);
+MODULE_PARM_DESC(vbt_firmware,
+		 "Load VBT from specified file under /lib/firmware");
 
 #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
 module_param_named(error_capture, i915.error_capture, bool, 0600);
diff --git a/drivers/gpu/drm/i915/i915_params.h b/drivers/gpu/drm/i915/i915_params.h
index 34148cc8637c..ac844709c97e 100644
--- a/drivers/gpu/drm/i915/i915_params.h
+++ b/drivers/gpu/drm/i915/i915_params.h
@@ -28,6 +28,7 @@
 #include <linux/cache.h> /* for __read_mostly */
 
 #define I915_PARAMS_FOR_EACH(func) \
+	func(char *, vbt_firmware); \
 	func(int, modeset); \
 	func(int, panel_ignore_lid); \
 	func(int, semaphores); \
@@ -51,6 +52,7 @@
 	func(int, use_mmio_flip); \
 	func(int, mmio_debug); \
 	func(int, edp_vswing); \
+	func(int, reset); \
 	func(unsigned int, inject_load_failure); \
 	/* leave bools at the end to not create holes */ \
 	func(bool, alpha_support); \
@@ -60,7 +62,6 @@
 	func(bool, prefault_disable); \
 	func(bool, load_detect_test); \
 	func(bool, force_reset_modeset_test); \
-	func(bool, reset); \
 	func(bool, error_capture); \
 	func(bool, disable_display); \
 	func(bool, verbose_state_checks); \
diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c
index 506ec32b9e53..09d97e0990b7 100644
--- a/drivers/gpu/drm/i915/i915_pci.c
+++ b/drivers/gpu/drm/i915/i915_pci.c
@@ -310,7 +310,8 @@ static const struct intel_device_info intel_haswell_info = {
 	BDW_COLORS, \
 	.has_logical_ring_contexts = 1, \
 	.has_full_48bit_ppgtt = 1, \
-	.has_64bit_reloc = 1
+	.has_64bit_reloc = 1, \
+	.has_reset_engine = 1
 
 #define BDW_PLATFORM \
 	BDW_FEATURES, \
@@ -342,6 +343,7 @@ static const struct intel_device_info intel_cherryview_info = {
 	.has_gmch_display = 1,
 	.has_aliasing_ppgtt = 1,
 	.has_full_ppgtt = 1,
+	.has_reset_engine = 1,
 	.display_mmio_offset = VLV_DISPLAY_BASE,
 	GEN_CHV_PIPEOFFSETS,
 	CURSOR_OFFSETS,
@@ -387,6 +389,7 @@ static const struct intel_device_info intel_skylake_gt3_info = {
 	.has_aliasing_ppgtt = 1, \
 	.has_full_ppgtt = 1, \
 	.has_full_48bit_ppgtt = 1, \
+	.has_reset_engine = 1, \
 	GEN_DEFAULT_PIPEOFFSETS, \
 	IVB_CURSOR_OFFSETS, \
 	BDW_COLORS
@@ -395,6 +398,7 @@ static const struct intel_device_info intel_broxton_info = {
 	GEN9_LP_FEATURES,
 	.platform = INTEL_BROXTON,
 	.ddb_size = 512,
+	.has_reset_engine = false,
 };
 
 static const struct intel_device_info intel_geminilake_info = {
@@ -446,6 +450,7 @@ static const struct intel_device_info intel_cannonlake_info = {
 	.gen = 10,
 	.ddb_size = 1024,
 	.has_csr = 1,
+	.color = { .degamma_lut_size = 0, .gamma_lut_size = 1024 }
 };
 
 /*
diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c
index f33d90226704..94185d610673 100644
--- a/drivers/gpu/drm/i915/i915_perf.c
+++ b/drivers/gpu/drm/i915/i915_perf.c
@@ -193,6 +193,7 @@
 
 #include <linux/anon_inodes.h>
 #include <linux/sizes.h>
+#include <linux/uuid.h>
 
 #include "i915_drv.h"
 #include "i915_oa_hsw.h"
@@ -357,6 +358,54 @@ struct perf_open_properties {
 	int oa_period_exponent;
 };
 
+static void free_oa_config(struct drm_i915_private *dev_priv,
+			   struct i915_oa_config *oa_config)
+{
+	if (!PTR_ERR(oa_config->flex_regs))
+		kfree(oa_config->flex_regs);
+	if (!PTR_ERR(oa_config->b_counter_regs))
+		kfree(oa_config->b_counter_regs);
+	if (!PTR_ERR(oa_config->mux_regs))
+		kfree(oa_config->mux_regs);
+	kfree(oa_config);
+}
+
+static void put_oa_config(struct drm_i915_private *dev_priv,
+			  struct i915_oa_config *oa_config)
+{
+	if (!atomic_dec_and_test(&oa_config->ref_count))
+		return;
+
+	free_oa_config(dev_priv, oa_config);
+}
+
+static int get_oa_config(struct drm_i915_private *dev_priv,
+			 int metrics_set,
+			 struct i915_oa_config **out_config)
+{
+	int ret;
+
+	if (metrics_set == 1) {
+		*out_config = &dev_priv->perf.oa.test_config;
+		atomic_inc(&dev_priv->perf.oa.test_config.ref_count);
+		return 0;
+	}
+
+	ret = mutex_lock_interruptible(&dev_priv->perf.metrics_lock);
+	if (ret)
+		return ret;
+
+	*out_config = idr_find(&dev_priv->perf.metrics_idr, metrics_set);
+	if (!*out_config)
+		ret = -EINVAL;
+	else
+		atomic_inc(&(*out_config)->ref_count);
+
+	mutex_unlock(&dev_priv->perf.metrics_lock);
+
+	return ret;
+}
+
 static u32 gen8_oa_hw_tail_read(struct drm_i915_private *dev_priv)
 {
 	return I915_READ(GEN8_OATAILPTR) & GEN8_OATAILPTR_MASK;
@@ -1246,10 +1295,12 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream)
 	BUG_ON(stream != dev_priv->perf.oa.exclusive_stream);
 
 	/*
-	 * Unset exclusive_stream first, it might be checked while
-	 * disabling the metric set on gen8+.
+	 * Unset exclusive_stream first, it will be checked while disabling
+	 * the metric set on gen8+.
 	 */
+	mutex_lock(&dev_priv->drm.struct_mutex);
 	dev_priv->perf.oa.exclusive_stream = NULL;
+	mutex_unlock(&dev_priv->drm.struct_mutex);
 
 	dev_priv->perf.oa.ops.disable_metric_set(dev_priv);
 
@@ -1261,6 +1312,8 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream)
 	if (stream->ctx)
 		oa_put_render_ctx_id(stream);
 
+	put_oa_config(dev_priv, stream->oa_config);
+
 	if (dev_priv->perf.oa.spurious_report_rs.missed) {
 		DRM_NOTE("%d spurious OA report notices suppressed due to ratelimiting\n",
 			 dev_priv->perf.oa.spurious_report_rs.missed);
@@ -1440,9 +1493,9 @@ unlock:
 
 static void config_oa_regs(struct drm_i915_private *dev_priv,
 			   const struct i915_oa_reg *regs,
-			   int n_regs)
+			   u32 n_regs)
 {
-	int i;
+	u32 i;
 
 	for (i = 0; i < n_regs; i++) {
 		const struct i915_oa_reg *reg = regs + i;
@@ -1451,17 +1504,9 @@ static void config_oa_regs(struct drm_i915_private *dev_priv,
 	}
 }
 
-static int hsw_enable_metric_set(struct drm_i915_private *dev_priv)
+static int hsw_enable_metric_set(struct drm_i915_private *dev_priv,
+				 const struct i915_oa_config *oa_config)
 {
-	int ret = i915_oa_select_metric_set_hsw(dev_priv);
-	int i;
-
-	if (ret)
-		return ret;
-
-	I915_WRITE(GDT_CHICKEN_BITS, (I915_READ(GDT_CHICKEN_BITS) |
-				      GT_NOA_ENABLE));
-
 	/* PRM:
 	 *
 	 * OA unit is using “crclk� for its functionality. When trunk
@@ -1476,10 +1521,7 @@ static int hsw_enable_metric_set(struct drm_i915_private *dev_priv)
 	I915_WRITE(GEN6_UCGCTL1, (I915_READ(GEN6_UCGCTL1) |
 				  GEN6_CSUNIT_CLOCK_GATE_DISABLE));
 
-	for (i = 0; i < dev_priv->perf.oa.n_mux_configs; i++) {
-		config_oa_regs(dev_priv, dev_priv->perf.oa.mux_regs[i],
-			       dev_priv->perf.oa.mux_regs_lens[i]);
-	}
+	config_oa_regs(dev_priv, oa_config->mux_regs, oa_config->mux_regs_len);
 
 	/* It apparently takes a fairly long time for a new MUX
 	 * configuration to be be applied after these register writes.
@@ -1504,8 +1546,8 @@ static int hsw_enable_metric_set(struct drm_i915_private *dev_priv)
 	 */
 	usleep_range(15000, 20000);
 
-	config_oa_regs(dev_priv, dev_priv->perf.oa.b_counter_regs,
-		       dev_priv->perf.oa.b_counter_regs_len);
+	config_oa_regs(dev_priv, oa_config->b_counter_regs,
+		       oa_config->b_counter_regs_len);
 
 	return 0;
 }
@@ -1529,11 +1571,10 @@ static void hsw_disable_metric_set(struct drm_i915_private *dev_priv)
  * in the case that the OA unit has been disabled.
  */
 static void gen8_update_reg_state_unlocked(struct i915_gem_context *ctx,
-					   u32 *reg_state)
+					   u32 *reg_state,
+					   const struct i915_oa_config *oa_config)
 {
 	struct drm_i915_private *dev_priv = ctx->i915;
-	const struct i915_oa_reg *flex_regs = dev_priv->perf.oa.flex_regs;
-	int n_flex_regs = dev_priv->perf.oa.flex_regs_len;
 	u32 ctx_oactxctrl = dev_priv->perf.oa.ctx_oactxctrl_offset;
 	u32 ctx_flexeu0 = dev_priv->perf.oa.ctx_flexeu0_offset;
 	/* The MMIO offsets for Flex EU registers aren't contiguous */
@@ -1565,12 +1606,15 @@ static void gen8_update_reg_state_unlocked(struct i915_gem_context *ctx,
 		 * will be an explicit 'No Event' we can select, but not yet...
 		 */
 		u32 value = 0;
-		int j;
 
-		for (j = 0; j < n_flex_regs; j++) {
-			if (i915_mmio_reg_offset(flex_regs[j].addr) == mmio) {
-				value = flex_regs[j].value;
-				break;
+		if (oa_config) {
+			u32 j;
+
+			for (j = 0; j < oa_config->flex_regs_len; j++) {
+				if (i915_mmio_reg_offset(oa_config->flex_regs[j].addr) == mmio) {
+					value = oa_config->flex_regs[j].value;
+					break;
+				}
 			}
 		}
 
@@ -1583,11 +1627,10 @@ static void gen8_update_reg_state_unlocked(struct i915_gem_context *ctx,
  * Same as gen8_update_reg_state_unlocked only through the batchbuffer. This
  * is only used by the kernel context.
  */
-static int gen8_emit_oa_config(struct drm_i915_gem_request *req)
+static int gen8_emit_oa_config(struct drm_i915_gem_request *req,
+			       const struct i915_oa_config *oa_config)
 {
 	struct drm_i915_private *dev_priv = req->i915;
-	const struct i915_oa_reg *flex_regs = dev_priv->perf.oa.flex_regs;
-	int n_flex_regs = dev_priv->perf.oa.flex_regs_len;
 	/* The MMIO offsets for Flex EU registers aren't contiguous */
 	u32 flex_mmio[] = {
 		i915_mmio_reg_offset(EU_PERF_CNTL0),
@@ -1622,12 +1665,15 @@ static int gen8_emit_oa_config(struct drm_i915_gem_request *req)
 		 * yet...
 		 */
 		u32 value = 0;
-		int j;
 
-		for (j = 0; j < n_flex_regs; j++) {
-			if (i915_mmio_reg_offset(flex_regs[j].addr) == mmio) {
-				value = flex_regs[j].value;
-				break;
+		if (oa_config) {
+			u32 j;
+
+			for (j = 0; j < oa_config->flex_regs_len; j++) {
+				if (i915_mmio_reg_offset(oa_config->flex_regs[j].addr) == mmio) {
+					value = oa_config->flex_regs[j].value;
+					break;
+				}
 			}
 		}
 
@@ -1641,7 +1687,8 @@ static int gen8_emit_oa_config(struct drm_i915_gem_request *req)
 	return 0;
 }
 
-static int gen8_switch_to_updated_kernel_context(struct drm_i915_private *dev_priv)
+static int gen8_switch_to_updated_kernel_context(struct drm_i915_private *dev_priv,
+						 const struct i915_oa_config *oa_config)
 {
 	struct intel_engine_cs *engine = dev_priv->engine[RCS];
 	struct i915_gem_timeline *timeline;
@@ -1656,7 +1703,7 @@ static int gen8_switch_to_updated_kernel_context(struct drm_i915_private *dev_pr
 	if (IS_ERR(req))
 		return PTR_ERR(req);
 
-	ret = gen8_emit_oa_config(req);
+	ret = gen8_emit_oa_config(req, oa_config);
 	if (ret) {
 		i915_add_request(req);
 		return ret;
@@ -1707,6 +1754,7 @@ static int gen8_switch_to_updated_kernel_context(struct drm_i915_private *dev_pr
  * Note: it's only the RCS/Render context that has any OA state.
  */
 static int gen8_configure_all_contexts(struct drm_i915_private *dev_priv,
+				       const struct i915_oa_config *oa_config,
 				       bool interruptible)
 {
 	struct i915_gem_context *ctx;
@@ -1724,7 +1772,7 @@ static int gen8_configure_all_contexts(struct drm_i915_private *dev_priv,
 	}
 
 	/* Switch away from any user context. */
-	ret = gen8_switch_to_updated_kernel_context(dev_priv);
+	ret = gen8_switch_to_updated_kernel_context(dev_priv, oa_config);
 	if (ret)
 		goto out;
 
@@ -1746,7 +1794,7 @@ static int gen8_configure_all_contexts(struct drm_i915_private *dev_priv,
 		goto out;
 
 	/* Update all contexts now that we've stalled the submission. */
-	list_for_each_entry(ctx, &dev_priv->context_list, link) {
+	list_for_each_entry(ctx, &dev_priv->contexts.list, link) {
 		struct intel_context *ce = &ctx->engine[RCS];
 		u32 *regs;
 
@@ -1763,7 +1811,7 @@ static int gen8_configure_all_contexts(struct drm_i915_private *dev_priv,
 		ce->state->obj->mm.dirty = true;
 		regs += LRC_STATE_PN * PAGE_SIZE / sizeof(*regs);
 
-		gen8_update_reg_state_unlocked(ctx, regs);
+		gen8_update_reg_state_unlocked(ctx, regs, oa_config);
 
 		i915_gem_object_unpin_map(ce->state->obj);
 	}
@@ -1774,13 +1822,10 @@ static int gen8_configure_all_contexts(struct drm_i915_private *dev_priv,
 	return ret;
 }
 
-static int gen8_enable_metric_set(struct drm_i915_private *dev_priv)
+static int gen8_enable_metric_set(struct drm_i915_private *dev_priv,
+				  const struct i915_oa_config *oa_config)
 {
-	int ret = dev_priv->perf.oa.ops.select_metric_set(dev_priv);
-	int i;
-
-	if (ret)
-		return ret;
+	int ret;
 
 	/*
 	 * We disable slice/unslice clock ratio change reports on SKL since
@@ -1817,19 +1862,14 @@ static int gen8_enable_metric_set(struct drm_i915_private *dev_priv)
 	 * to make sure all slices/subslices are ON before writing to NOA
 	 * registers.
 	 */
-	ret = gen8_configure_all_contexts(dev_priv, true);
+	ret = gen8_configure_all_contexts(dev_priv, oa_config, true);
 	if (ret)
 		return ret;
 
-	I915_WRITE(GDT_CHICKEN_BITS, 0xA0);
-	for (i = 0; i < dev_priv->perf.oa.n_mux_configs; i++) {
-		config_oa_regs(dev_priv, dev_priv->perf.oa.mux_regs[i],
-			       dev_priv->perf.oa.mux_regs_lens[i]);
-	}
-	I915_WRITE(GDT_CHICKEN_BITS, 0x80);
+	config_oa_regs(dev_priv, oa_config->mux_regs, oa_config->mux_regs_len);
 
-	config_oa_regs(dev_priv, dev_priv->perf.oa.b_counter_regs,
-		       dev_priv->perf.oa.b_counter_regs_len);
+	config_oa_regs(dev_priv, oa_config->b_counter_regs,
+		       oa_config->b_counter_regs_len);
 
 	return 0;
 }
@@ -1837,7 +1877,11 @@ static int gen8_enable_metric_set(struct drm_i915_private *dev_priv)
 static void gen8_disable_metric_set(struct drm_i915_private *dev_priv)
 {
 	/* Reset all contexts' slices/subslices configurations. */
-	gen8_configure_all_contexts(dev_priv, false);
+	gen8_configure_all_contexts(dev_priv, NULL, false);
+
+	I915_WRITE(GDT_CHICKEN_BITS, (I915_READ(GDT_CHICKEN_BITS) &
+				      ~GT_NOA_ENABLE));
+
 }
 
 static void gen7_oa_enable(struct drm_i915_private *dev_priv)
@@ -2011,11 +2055,6 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream,
 		return -EBUSY;
 	}
 
-	if (!props->metrics_set) {
-		DRM_DEBUG("OA metric set not specified\n");
-		return -EINVAL;
-	}
-
 	if (!props->oa_format) {
 		DRM_DEBUG("OA report format not specified\n");
 		return -EINVAL;
@@ -2055,8 +2094,6 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream,
 	dev_priv->perf.oa.oa_buffer.format =
 		dev_priv->perf.oa.oa_formats[props->oa_format].format;
 
-	dev_priv->perf.oa.metrics_set = props->metrics_set;
-
 	dev_priv->perf.oa.periodic = props->oa_periodic;
 	if (dev_priv->perf.oa.periodic)
 		dev_priv->perf.oa.period_exponent = props->oa_period_exponent;
@@ -2067,6 +2104,10 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream,
 			return ret;
 	}
 
+	ret = get_oa_config(dev_priv, props->metrics_set, &stream->oa_config);
+	if (ret)
+		goto err_config;
+
 	/* PRM - observability performance counters:
 	 *
 	 *   OACONTROL, performance counter enable, note:
@@ -2086,22 +2127,39 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream,
 	if (ret)
 		goto err_oa_buf_alloc;
 
-	ret = dev_priv->perf.oa.ops.enable_metric_set(dev_priv);
+	ret = dev_priv->perf.oa.ops.enable_metric_set(dev_priv,
+						      stream->oa_config);
 	if (ret)
 		goto err_enable;
 
 	stream->ops = &i915_oa_stream_ops;
 
+	/* Lock device for exclusive_stream access late because
+	 * enable_metric_set() might lock as well on gen8+.
+	 */
+	ret = i915_mutex_lock_interruptible(&dev_priv->drm);
+	if (ret)
+		goto err_lock;
+
 	dev_priv->perf.oa.exclusive_stream = stream;
 
+	mutex_unlock(&dev_priv->drm.struct_mutex);
+
 	return 0;
 
+err_lock:
+	dev_priv->perf.oa.ops.disable_metric_set(dev_priv);
+
 err_enable:
 	free_oa_buffer(dev_priv);
 
 err_oa_buf_alloc:
+	put_oa_config(dev_priv, stream->oa_config);
+
 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 	intel_runtime_pm_put(dev_priv);
+
+err_config:
 	if (stream->ctx)
 		oa_put_render_ctx_id(stream);
 
@@ -2112,15 +2170,14 @@ void i915_oa_init_reg_state(struct intel_engine_cs *engine,
 			    struct i915_gem_context *ctx,
 			    u32 *reg_state)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
+	struct i915_perf_stream *stream;
 
 	if (engine->id != RCS)
 		return;
 
-	if (!dev_priv->perf.initialized)
-		return;
-
-	gen8_update_reg_state_unlocked(ctx, reg_state);
+	stream = engine->i915->perf.oa.exclusive_stream;
+	if (stream)
+		gen8_update_reg_state_unlocked(ctx, reg_state, stream->oa_config);
 }
 
 /**
@@ -2444,7 +2501,7 @@ static void i915_perf_destroy_locked(struct i915_perf_stream *stream)
 	list_del(&stream->link);
 
 	if (stream->ctx)
-		i915_gem_context_put_unlocked(stream->ctx);
+		i915_gem_context_put(stream->ctx);
 
 	kfree(stream);
 }
@@ -2483,27 +2540,6 @@ static const struct file_operations fops = {
 };
 
 
-static struct i915_gem_context *
-lookup_context(struct drm_i915_private *dev_priv,
-	       struct drm_i915_file_private *file_priv,
-	       u32 ctx_user_handle)
-{
-	struct i915_gem_context *ctx;
-	int ret;
-
-	ret = i915_mutex_lock_interruptible(&dev_priv->drm);
-	if (ret)
-		return ERR_PTR(ret);
-
-	ctx = i915_gem_context_lookup(file_priv, ctx_user_handle);
-	if (!IS_ERR(ctx))
-		i915_gem_context_get(ctx);
-
-	mutex_unlock(&dev_priv->drm.struct_mutex);
-
-	return ctx;
-}
-
 /**
  * i915_perf_open_ioctl_locked - DRM ioctl() for userspace to open a stream FD
  * @dev_priv: i915 device instance
@@ -2545,12 +2581,11 @@ i915_perf_open_ioctl_locked(struct drm_i915_private *dev_priv,
 		u32 ctx_handle = props->ctx_handle;
 		struct drm_i915_file_private *file_priv = file->driver_priv;
 
-		specific_ctx = lookup_context(dev_priv, file_priv, ctx_handle);
-		if (IS_ERR(specific_ctx)) {
-			ret = PTR_ERR(specific_ctx);
-			if (ret != -EINTR)
-				DRM_DEBUG("Failed to look up context with ID %u for opening perf stream\n",
-					  ctx_handle);
+		specific_ctx = i915_gem_context_lookup(file_priv, ctx_handle);
+		if (!specific_ctx) {
+			DRM_DEBUG("Failed to look up context with ID %u for opening perf stream\n",
+				  ctx_handle);
+			ret = -ENOENT;
 			goto err;
 		}
 	}
@@ -2633,7 +2668,7 @@ err_alloc:
 	kfree(stream);
 err_ctx:
 	if (specific_ctx)
-		i915_gem_context_put_unlocked(specific_ctx);
+		i915_gem_context_put(specific_ctx);
 err:
 	return ret;
 }
@@ -2665,7 +2700,7 @@ static int read_properties_unlocked(struct drm_i915_private *dev_priv,
 				    struct perf_open_properties *props)
 {
 	u64 __user *uprop = uprops;
-	int i;
+	u32 i;
 
 	memset(props, 0, sizeof(struct perf_open_properties));
 
@@ -2712,8 +2747,7 @@ static int read_properties_unlocked(struct drm_i915_private *dev_priv,
 			props->sample_flags |= SAMPLE_OA_REPORT;
 			break;
 		case DRM_I915_PERF_PROP_OA_METRICS_SET:
-			if (value == 0 ||
-			    value > dev_priv->perf.oa.n_builtin_sets) {
+			if (value == 0) {
 				DRM_DEBUG("Unknown OA metric set ID\n");
 				return -EINVAL;
 			}
@@ -2852,6 +2886,8 @@ int i915_perf_open_ioctl(struct drm_device *dev, void *data,
  */
 void i915_perf_register(struct drm_i915_private *dev_priv)
 {
+	int ret;
+
 	if (!dev_priv->perf.initialized)
 		return;
 
@@ -2867,44 +2903,42 @@ void i915_perf_register(struct drm_i915_private *dev_priv)
 	if (!dev_priv->perf.metrics_kobj)
 		goto exit;
 
+	sysfs_attr_init(&dev_priv->perf.oa.test_config.sysfs_metric_id.attr);
+
 	if (IS_HASWELL(dev_priv)) {
-		if (i915_perf_register_sysfs_hsw(dev_priv))
-			goto sysfs_error;
+		i915_perf_load_test_config_hsw(dev_priv);
 	} else if (IS_BROADWELL(dev_priv)) {
-		if (i915_perf_register_sysfs_bdw(dev_priv))
-			goto sysfs_error;
+		i915_perf_load_test_config_bdw(dev_priv);
 	} else if (IS_CHERRYVIEW(dev_priv)) {
-		if (i915_perf_register_sysfs_chv(dev_priv))
-			goto sysfs_error;
+		i915_perf_load_test_config_chv(dev_priv);
 	} else if (IS_SKYLAKE(dev_priv)) {
-		if (IS_SKL_GT2(dev_priv)) {
-			if (i915_perf_register_sysfs_sklgt2(dev_priv))
-				goto sysfs_error;
-		} else if (IS_SKL_GT3(dev_priv)) {
-			if (i915_perf_register_sysfs_sklgt3(dev_priv))
-				goto sysfs_error;
-		} else if (IS_SKL_GT4(dev_priv)) {
-			if (i915_perf_register_sysfs_sklgt4(dev_priv))
-				goto sysfs_error;
-		} else
-			goto sysfs_error;
+		if (IS_SKL_GT2(dev_priv))
+			i915_perf_load_test_config_sklgt2(dev_priv);
+		else if (IS_SKL_GT3(dev_priv))
+			i915_perf_load_test_config_sklgt3(dev_priv);
+		else if (IS_SKL_GT4(dev_priv))
+			i915_perf_load_test_config_sklgt4(dev_priv);
 	} else if (IS_BROXTON(dev_priv)) {
-		if (i915_perf_register_sysfs_bxt(dev_priv))
-			goto sysfs_error;
+		i915_perf_load_test_config_bxt(dev_priv);
 	} else if (IS_KABYLAKE(dev_priv)) {
-		if (IS_KBL_GT2(dev_priv)) {
-			if (i915_perf_register_sysfs_kblgt2(dev_priv))
-				goto sysfs_error;
-		} else if (IS_KBL_GT3(dev_priv)) {
-			if (i915_perf_register_sysfs_kblgt3(dev_priv))
-				goto sysfs_error;
-		} else
-			goto sysfs_error;
+		if (IS_KBL_GT2(dev_priv))
+			i915_perf_load_test_config_kblgt2(dev_priv);
+		else if (IS_KBL_GT3(dev_priv))
+			i915_perf_load_test_config_kblgt3(dev_priv);
 	} else if (IS_GEMINILAKE(dev_priv)) {
-		if (i915_perf_register_sysfs_glk(dev_priv))
-			goto sysfs_error;
+		i915_perf_load_test_config_glk(dev_priv);
 	}
 
+	if (dev_priv->perf.oa.test_config.id == 0)
+		goto sysfs_error;
+
+	ret = sysfs_create_group(dev_priv->perf.metrics_kobj,
+				 &dev_priv->perf.oa.test_config.sysfs_metric);
+	if (ret)
+		goto sysfs_error;
+
+	atomic_set(&dev_priv->perf.oa.test_config.ref_count, 1);
+
 	goto exit;
 
 sysfs_error:
@@ -2929,34 +2963,375 @@ void i915_perf_unregister(struct drm_i915_private *dev_priv)
 	if (!dev_priv->perf.metrics_kobj)
 		return;
 
-	if (IS_HASWELL(dev_priv))
-		i915_perf_unregister_sysfs_hsw(dev_priv);
-	else if (IS_BROADWELL(dev_priv))
-		i915_perf_unregister_sysfs_bdw(dev_priv);
-	else if (IS_CHERRYVIEW(dev_priv))
-		i915_perf_unregister_sysfs_chv(dev_priv);
-	else if (IS_SKYLAKE(dev_priv)) {
-		if (IS_SKL_GT2(dev_priv))
-			i915_perf_unregister_sysfs_sklgt2(dev_priv);
-		else if (IS_SKL_GT3(dev_priv))
-			i915_perf_unregister_sysfs_sklgt3(dev_priv);
-		else if (IS_SKL_GT4(dev_priv))
-			i915_perf_unregister_sysfs_sklgt4(dev_priv);
-	} else if (IS_BROXTON(dev_priv))
-		i915_perf_unregister_sysfs_bxt(dev_priv);
-	else if (IS_KABYLAKE(dev_priv)) {
-		if (IS_KBL_GT2(dev_priv))
-			i915_perf_unregister_sysfs_kblgt2(dev_priv);
-		else if (IS_KBL_GT3(dev_priv))
-			i915_perf_unregister_sysfs_kblgt3(dev_priv);
-	} else if (IS_GEMINILAKE(dev_priv))
-		i915_perf_unregister_sysfs_glk(dev_priv);
-
+	sysfs_remove_group(dev_priv->perf.metrics_kobj,
+			   &dev_priv->perf.oa.test_config.sysfs_metric);
 
 	kobject_put(dev_priv->perf.metrics_kobj);
 	dev_priv->perf.metrics_kobj = NULL;
 }
 
+static bool gen8_is_valid_flex_addr(struct drm_i915_private *dev_priv, u32 addr)
+{
+	static const i915_reg_t flex_eu_regs[] = {
+		EU_PERF_CNTL0,
+		EU_PERF_CNTL1,
+		EU_PERF_CNTL2,
+		EU_PERF_CNTL3,
+		EU_PERF_CNTL4,
+		EU_PERF_CNTL5,
+		EU_PERF_CNTL6,
+	};
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(flex_eu_regs); i++) {
+		if (flex_eu_regs[i].reg == addr)
+			return true;
+	}
+	return false;
+}
+
+static bool gen7_is_valid_b_counter_addr(struct drm_i915_private *dev_priv, u32 addr)
+{
+	return (addr >= OASTARTTRIG1.reg && addr <= OASTARTTRIG8.reg) ||
+		(addr >= OAREPORTTRIG1.reg && addr <= OAREPORTTRIG8.reg) ||
+		(addr >= OACEC0_0.reg && addr <= OACEC7_1.reg);
+}
+
+static bool gen7_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr)
+{
+	return addr == HALF_SLICE_CHICKEN2.reg ||
+		(addr >= MICRO_BP0_0.reg && addr <= NOA_WRITE.reg) ||
+		(addr >= OA_PERFCNT1_LO.reg && addr <= OA_PERFCNT2_HI.reg) ||
+		(addr >= OA_PERFMATRIX_LO.reg && addr <= OA_PERFMATRIX_HI.reg);
+}
+
+static bool gen8_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr)
+{
+	return gen7_is_valid_mux_addr(dev_priv, addr) ||
+		addr == WAIT_FOR_RC6_EXIT.reg ||
+		(addr >= RPM_CONFIG0.reg && addr <= NOA_CONFIG(8).reg);
+}
+
+static bool hsw_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr)
+{
+	return gen7_is_valid_mux_addr(dev_priv, addr) ||
+		(addr >= 0x25100 && addr <= 0x2FF90) ||
+		addr == 0x9ec0;
+}
+
+static bool chv_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr)
+{
+	return gen7_is_valid_mux_addr(dev_priv, addr) ||
+		(addr >= 0x182300 && addr <= 0x1823A4);
+}
+
+static uint32_t mask_reg_value(u32 reg, u32 val)
+{
+	/* HALF_SLICE_CHICKEN2 is programmed with a the
+	 * WaDisableSTUnitPowerOptimization workaround. Make sure the value
+	 * programmed by userspace doesn't change this.
+	 */
+	if (HALF_SLICE_CHICKEN2.reg == reg)
+		val = val & ~_MASKED_BIT_ENABLE(GEN8_ST_PO_DISABLE);
+
+	/* WAIT_FOR_RC6_EXIT has only one bit fullfilling the function
+	 * indicated by its name and a bunch of selection fields used by OA
+	 * configs.
+	 */
+	if (WAIT_FOR_RC6_EXIT.reg == reg)
+		val = val & ~_MASKED_BIT_ENABLE(HSW_WAIT_FOR_RC6_EXIT_ENABLE);
+
+	return val;
+}
+
+static struct i915_oa_reg *alloc_oa_regs(struct drm_i915_private *dev_priv,
+					 bool (*is_valid)(struct drm_i915_private *dev_priv, u32 addr),
+					 u32 __user *regs,
+					 u32 n_regs)
+{
+	struct i915_oa_reg *oa_regs;
+	int err;
+	u32 i;
+
+	if (!n_regs)
+		return NULL;
+
+	if (!access_ok(VERIFY_READ, regs, n_regs * sizeof(u32) * 2))
+		return ERR_PTR(-EFAULT);
+
+	/* No is_valid function means we're not allowing any register to be programmed. */
+	GEM_BUG_ON(!is_valid);
+	if (!is_valid)
+		return ERR_PTR(-EINVAL);
+
+	oa_regs = kmalloc_array(n_regs, sizeof(*oa_regs), GFP_KERNEL);
+	if (!oa_regs)
+		return ERR_PTR(-ENOMEM);
+
+	for (i = 0; i < n_regs; i++) {
+		u32 addr, value;
+
+		err = get_user(addr, regs);
+		if (err)
+			goto addr_err;
+
+		if (!is_valid(dev_priv, addr)) {
+			DRM_DEBUG("Invalid oa_reg address: %X\n", addr);
+			err = -EINVAL;
+			goto addr_err;
+		}
+
+		err = get_user(value, regs + 1);
+		if (err)
+			goto addr_err;
+
+		oa_regs[i].addr = _MMIO(addr);
+		oa_regs[i].value = mask_reg_value(addr, value);
+
+		regs += 2;
+	}
+
+	return oa_regs;
+
+addr_err:
+	kfree(oa_regs);
+	return ERR_PTR(err);
+}
+
+static ssize_t show_dynamic_id(struct device *dev,
+			       struct device_attribute *attr,
+			       char *buf)
+{
+	struct i915_oa_config *oa_config =
+		container_of(attr, typeof(*oa_config), sysfs_metric_id);
+
+	return sprintf(buf, "%d\n", oa_config->id);
+}
+
+static int create_dynamic_oa_sysfs_entry(struct drm_i915_private *dev_priv,
+					 struct i915_oa_config *oa_config)
+{
+	sysfs_attr_init(&oa_config->sysfs_metric_id.attr);
+	oa_config->sysfs_metric_id.attr.name = "id";
+	oa_config->sysfs_metric_id.attr.mode = S_IRUGO;
+	oa_config->sysfs_metric_id.show = show_dynamic_id;
+	oa_config->sysfs_metric_id.store = NULL;
+
+	oa_config->attrs[0] = &oa_config->sysfs_metric_id.attr;
+	oa_config->attrs[1] = NULL;
+
+	oa_config->sysfs_metric.name = oa_config->uuid;
+	oa_config->sysfs_metric.attrs = oa_config->attrs;
+
+	return sysfs_create_group(dev_priv->perf.metrics_kobj,
+				  &oa_config->sysfs_metric);
+}
+
+/**
+ * i915_perf_add_config_ioctl - DRM ioctl() for userspace to add a new OA config
+ * @dev: drm device
+ * @data: ioctl data (pointer to struct drm_i915_perf_oa_config) copied from
+ *        userspace (unvalidated)
+ * @file: drm file
+ *
+ * Validates the submitted OA register to be saved into a new OA config that
+ * can then be used for programming the OA unit and its NOA network.
+ *
+ * Returns: A new allocated config number to be used with the perf open ioctl
+ * or a negative error code on failure.
+ */
+int i915_perf_add_config_ioctl(struct drm_device *dev, void *data,
+			       struct drm_file *file)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_perf_oa_config *args = data;
+	struct i915_oa_config *oa_config, *tmp;
+	int err, id;
+
+	if (!dev_priv->perf.initialized) {
+		DRM_DEBUG("i915 perf interface not available for this system\n");
+		return -ENOTSUPP;
+	}
+
+	if (!dev_priv->perf.metrics_kobj) {
+		DRM_DEBUG("OA metrics weren't advertised via sysfs\n");
+		return -EINVAL;
+	}
+
+	if (i915_perf_stream_paranoid && !capable(CAP_SYS_ADMIN)) {
+		DRM_DEBUG("Insufficient privileges to add i915 OA config\n");
+		return -EACCES;
+	}
+
+	if ((!args->mux_regs_ptr || !args->n_mux_regs) &&
+	    (!args->boolean_regs_ptr || !args->n_boolean_regs) &&
+	    (!args->flex_regs_ptr || !args->n_flex_regs)) {
+		DRM_DEBUG("No OA registers given\n");
+		return -EINVAL;
+	}
+
+	oa_config = kzalloc(sizeof(*oa_config), GFP_KERNEL);
+	if (!oa_config) {
+		DRM_DEBUG("Failed to allocate memory for the OA config\n");
+		return -ENOMEM;
+	}
+
+	atomic_set(&oa_config->ref_count, 1);
+
+	if (!uuid_is_valid(args->uuid)) {
+		DRM_DEBUG("Invalid uuid format for OA config\n");
+		err = -EINVAL;
+		goto reg_err;
+	}
+
+	/* Last character in oa_config->uuid will be 0 because oa_config is
+	 * kzalloc.
+	 */
+	memcpy(oa_config->uuid, args->uuid, sizeof(args->uuid));
+
+	oa_config->mux_regs_len = args->n_mux_regs;
+	oa_config->mux_regs =
+		alloc_oa_regs(dev_priv,
+			      dev_priv->perf.oa.ops.is_valid_mux_reg,
+			      u64_to_user_ptr(args->mux_regs_ptr),
+			      args->n_mux_regs);
+
+	if (IS_ERR(oa_config->mux_regs)) {
+		DRM_DEBUG("Failed to create OA config for mux_regs\n");
+		err = PTR_ERR(oa_config->mux_regs);
+		goto reg_err;
+	}
+
+	oa_config->b_counter_regs_len = args->n_boolean_regs;
+	oa_config->b_counter_regs =
+		alloc_oa_regs(dev_priv,
+			      dev_priv->perf.oa.ops.is_valid_b_counter_reg,
+			      u64_to_user_ptr(args->boolean_regs_ptr),
+			      args->n_boolean_regs);
+
+	if (IS_ERR(oa_config->b_counter_regs)) {
+		DRM_DEBUG("Failed to create OA config for b_counter_regs\n");
+		err = PTR_ERR(oa_config->b_counter_regs);
+		goto reg_err;
+	}
+
+	if (INTEL_GEN(dev_priv) < 8) {
+		if (args->n_flex_regs != 0) {
+			err = -EINVAL;
+			goto reg_err;
+		}
+	} else {
+		oa_config->flex_regs_len = args->n_flex_regs;
+		oa_config->flex_regs =
+			alloc_oa_regs(dev_priv,
+				      dev_priv->perf.oa.ops.is_valid_flex_reg,
+				      u64_to_user_ptr(args->flex_regs_ptr),
+				      args->n_flex_regs);
+
+		if (IS_ERR(oa_config->flex_regs)) {
+			DRM_DEBUG("Failed to create OA config for flex_regs\n");
+			err = PTR_ERR(oa_config->flex_regs);
+			goto reg_err;
+		}
+	}
+
+	err = mutex_lock_interruptible(&dev_priv->perf.metrics_lock);
+	if (err)
+		goto reg_err;
+
+	/* We shouldn't have too many configs, so this iteration shouldn't be
+	 * too costly.
+	 */
+	idr_for_each_entry(&dev_priv->perf.metrics_idr, tmp, id) {
+		if (!strcmp(tmp->uuid, oa_config->uuid)) {
+			DRM_DEBUG("OA config already exists with this uuid\n");
+			err = -EADDRINUSE;
+			goto sysfs_err;
+		}
+	}
+
+	err = create_dynamic_oa_sysfs_entry(dev_priv, oa_config);
+	if (err) {
+		DRM_DEBUG("Failed to create sysfs entry for OA config\n");
+		goto sysfs_err;
+	}
+
+	/* Config id 0 is invalid, id 1 for kernel stored test config. */
+	oa_config->id = idr_alloc(&dev_priv->perf.metrics_idr,
+				  oa_config, 2,
+				  0, GFP_KERNEL);
+	if (oa_config->id < 0) {
+		DRM_DEBUG("Failed to create sysfs entry for OA config\n");
+		err = oa_config->id;
+		goto sysfs_err;
+	}
+
+	mutex_unlock(&dev_priv->perf.metrics_lock);
+
+	return oa_config->id;
+
+sysfs_err:
+	mutex_unlock(&dev_priv->perf.metrics_lock);
+reg_err:
+	put_oa_config(dev_priv, oa_config);
+	DRM_DEBUG("Failed to add new OA config\n");
+	return err;
+}
+
+/**
+ * i915_perf_remove_config_ioctl - DRM ioctl() for userspace to remove an OA config
+ * @dev: drm device
+ * @data: ioctl data (pointer to u64 integer) copied from userspace
+ * @file: drm file
+ *
+ * Configs can be removed while being used, the will stop appearing in sysfs
+ * and their content will be freed when the stream using the config is closed.
+ *
+ * Returns: 0 on success or a negative error code on failure.
+ */
+int i915_perf_remove_config_ioctl(struct drm_device *dev, void *data,
+				  struct drm_file *file)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u64 *arg = data;
+	struct i915_oa_config *oa_config;
+	int ret;
+
+	if (!dev_priv->perf.initialized) {
+		DRM_DEBUG("i915 perf interface not available for this system\n");
+		return -ENOTSUPP;
+	}
+
+	if (i915_perf_stream_paranoid && !capable(CAP_SYS_ADMIN)) {
+		DRM_DEBUG("Insufficient privileges to remove i915 OA config\n");
+		return -EACCES;
+	}
+
+	ret = mutex_lock_interruptible(&dev_priv->perf.metrics_lock);
+	if (ret)
+		goto lock_err;
+
+	oa_config = idr_find(&dev_priv->perf.metrics_idr, *arg);
+	if (!oa_config) {
+		DRM_DEBUG("Failed to remove unknown OA config\n");
+		ret = -ENOENT;
+		goto config_err;
+	}
+
+	GEM_BUG_ON(*arg != oa_config->id);
+
+	sysfs_remove_group(dev_priv->perf.metrics_kobj,
+			   &oa_config->sysfs_metric);
+
+	idr_remove(&dev_priv->perf.metrics_idr, *arg);
+	put_oa_config(dev_priv, oa_config);
+
+config_err:
+	mutex_unlock(&dev_priv->perf.metrics_lock);
+lock_err:
+	return ret;
+}
+
 static struct ctl_table oa_table[] = {
 	{
 	 .procname = "perf_stream_paranoid",
@@ -3010,9 +3385,14 @@ static struct ctl_table dev_root[] = {
  */
 void i915_perf_init(struct drm_i915_private *dev_priv)
 {
-	dev_priv->perf.oa.n_builtin_sets = 0;
+	dev_priv->perf.oa.timestamp_frequency = 0;
 
 	if (IS_HASWELL(dev_priv)) {
+		dev_priv->perf.oa.ops.is_valid_b_counter_reg =
+			gen7_is_valid_b_counter_addr;
+		dev_priv->perf.oa.ops.is_valid_mux_reg =
+			hsw_is_valid_mux_addr;
+		dev_priv->perf.oa.ops.is_valid_flex_reg = NULL;
 		dev_priv->perf.oa.ops.init_oa_buffer = gen7_init_oa_buffer;
 		dev_priv->perf.oa.ops.enable_metric_set = hsw_enable_metric_set;
 		dev_priv->perf.oa.ops.disable_metric_set = hsw_disable_metric_set;
@@ -3025,9 +3405,6 @@ void i915_perf_init(struct drm_i915_private *dev_priv)
 		dev_priv->perf.oa.timestamp_frequency = 12500000;
 
 		dev_priv->perf.oa.oa_formats = hsw_oa_formats;
-
-		dev_priv->perf.oa.n_builtin_sets =
-			i915_oa_n_builtin_metric_sets_hsw;
 	} else if (i915.enable_execlists) {
 		/* Note: that although we could theoretically also support the
 		 * legacy ringbuffer mode on BDW (and earlier iterations of
@@ -3035,6 +3412,22 @@ void i915_perf_init(struct drm_i915_private *dev_priv)
 		 * worth the complexity to maintain now that BDW+ enable
 		 * execlist mode by default.
 		 */
+		dev_priv->perf.oa.ops.is_valid_b_counter_reg =
+			gen7_is_valid_b_counter_addr;
+		dev_priv->perf.oa.ops.is_valid_mux_reg =
+			gen8_is_valid_mux_addr;
+		dev_priv->perf.oa.ops.is_valid_flex_reg =
+			gen8_is_valid_flex_addr;
+
+		dev_priv->perf.oa.ops.init_oa_buffer = gen8_init_oa_buffer;
+		dev_priv->perf.oa.ops.enable_metric_set = gen8_enable_metric_set;
+		dev_priv->perf.oa.ops.disable_metric_set = gen8_disable_metric_set;
+		dev_priv->perf.oa.ops.oa_enable = gen8_oa_enable;
+		dev_priv->perf.oa.ops.oa_disable = gen8_oa_disable;
+		dev_priv->perf.oa.ops.read = gen8_oa_read;
+		dev_priv->perf.oa.ops.oa_hw_tail_read = gen8_oa_hw_tail_read;
+
+		dev_priv->perf.oa.oa_formats = gen8_plus_oa_formats;
 
 		if (IS_GEN8(dev_priv)) {
 			dev_priv->perf.oa.ctx_oactxctrl_offset = 0x120;
@@ -3043,85 +3436,35 @@ void i915_perf_init(struct drm_i915_private *dev_priv)
 			dev_priv->perf.oa.timestamp_frequency = 12500000;
 
 			dev_priv->perf.oa.gen8_valid_ctx_bit = (1<<25);
-
-			if (IS_BROADWELL(dev_priv)) {
-				dev_priv->perf.oa.n_builtin_sets =
-					i915_oa_n_builtin_metric_sets_bdw;
-				dev_priv->perf.oa.ops.select_metric_set =
-					i915_oa_select_metric_set_bdw;
-			} else if (IS_CHERRYVIEW(dev_priv)) {
-				dev_priv->perf.oa.n_builtin_sets =
-					i915_oa_n_builtin_metric_sets_chv;
-				dev_priv->perf.oa.ops.select_metric_set =
-					i915_oa_select_metric_set_chv;
+			if (IS_CHERRYVIEW(dev_priv)) {
+				dev_priv->perf.oa.ops.is_valid_mux_reg =
+					chv_is_valid_mux_addr;
 			}
 		} else if (IS_GEN9(dev_priv)) {
 			dev_priv->perf.oa.ctx_oactxctrl_offset = 0x128;
 			dev_priv->perf.oa.ctx_flexeu0_offset = 0x3de;
 
-			dev_priv->perf.oa.timestamp_frequency = 12000000;
-
 			dev_priv->perf.oa.gen8_valid_ctx_bit = (1<<16);
 
-			if (IS_SKL_GT2(dev_priv)) {
-				dev_priv->perf.oa.n_builtin_sets =
-					i915_oa_n_builtin_metric_sets_sklgt2;
-				dev_priv->perf.oa.ops.select_metric_set =
-					i915_oa_select_metric_set_sklgt2;
-			} else if (IS_SKL_GT3(dev_priv)) {
-				dev_priv->perf.oa.n_builtin_sets =
-					i915_oa_n_builtin_metric_sets_sklgt3;
-				dev_priv->perf.oa.ops.select_metric_set =
-					i915_oa_select_metric_set_sklgt3;
-			} else if (IS_SKL_GT4(dev_priv)) {
-				dev_priv->perf.oa.n_builtin_sets =
-					i915_oa_n_builtin_metric_sets_sklgt4;
-				dev_priv->perf.oa.ops.select_metric_set =
-					i915_oa_select_metric_set_sklgt4;
-			} else if (IS_BROXTON(dev_priv)) {
+			switch (dev_priv->info.platform) {
+			case INTEL_BROXTON:
+			case INTEL_GEMINILAKE:
 				dev_priv->perf.oa.timestamp_frequency = 19200000;
-
-				dev_priv->perf.oa.n_builtin_sets =
-					i915_oa_n_builtin_metric_sets_bxt;
-				dev_priv->perf.oa.ops.select_metric_set =
-					i915_oa_select_metric_set_bxt;
-			} else if (IS_KBL_GT2(dev_priv)) {
-				dev_priv->perf.oa.n_builtin_sets =
-					i915_oa_n_builtin_metric_sets_kblgt2;
-				dev_priv->perf.oa.ops.select_metric_set =
-					i915_oa_select_metric_set_kblgt2;
-			} else if (IS_KBL_GT3(dev_priv)) {
-				dev_priv->perf.oa.n_builtin_sets =
-					i915_oa_n_builtin_metric_sets_kblgt3;
-				dev_priv->perf.oa.ops.select_metric_set =
-					i915_oa_select_metric_set_kblgt3;
-			} else if (IS_GEMINILAKE(dev_priv)) {
-				dev_priv->perf.oa.timestamp_frequency = 19200000;
-
-				dev_priv->perf.oa.n_builtin_sets =
-					i915_oa_n_builtin_metric_sets_glk;
-				dev_priv->perf.oa.ops.select_metric_set =
-					i915_oa_select_metric_set_glk;
+				break;
+			case INTEL_SKYLAKE:
+			case INTEL_KABYLAKE:
+				dev_priv->perf.oa.timestamp_frequency = 12000000;
+				break;
+			default:
+				/* Leave timestamp_frequency to 0 so we can
+				 * detect unsupported platforms.
+				 */
+				break;
 			}
 		}
-
-		if (dev_priv->perf.oa.n_builtin_sets) {
-			dev_priv->perf.oa.ops.init_oa_buffer = gen8_init_oa_buffer;
-			dev_priv->perf.oa.ops.enable_metric_set =
-				gen8_enable_metric_set;
-			dev_priv->perf.oa.ops.disable_metric_set =
-				gen8_disable_metric_set;
-			dev_priv->perf.oa.ops.oa_enable = gen8_oa_enable;
-			dev_priv->perf.oa.ops.oa_disable = gen8_oa_disable;
-			dev_priv->perf.oa.ops.read = gen8_oa_read;
-			dev_priv->perf.oa.ops.oa_hw_tail_read =
-				gen8_oa_hw_tail_read;
-
-			dev_priv->perf.oa.oa_formats = gen8_plus_oa_formats;
-		}
 	}
 
-	if (dev_priv->perf.oa.n_builtin_sets) {
+	if (dev_priv->perf.oa.timestamp_frequency) {
 		hrtimer_init(&dev_priv->perf.oa.poll_check_timer,
 				CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 		dev_priv->perf.oa.poll_check_timer.function = oa_poll_check_timer_cb;
@@ -3135,10 +3478,23 @@ void i915_perf_init(struct drm_i915_private *dev_priv)
 			dev_priv->perf.oa.timestamp_frequency / 2;
 		dev_priv->perf.sysctl_header = register_sysctl_table(dev_root);
 
+		mutex_init(&dev_priv->perf.metrics_lock);
+		idr_init(&dev_priv->perf.metrics_idr);
+
 		dev_priv->perf.initialized = true;
 	}
 }
 
+static int destroy_config(int id, void *p, void *data)
+{
+	struct drm_i915_private *dev_priv = data;
+	struct i915_oa_config *oa_config = p;
+
+	put_oa_config(dev_priv, oa_config);
+
+	return 0;
+}
+
 /**
  * i915_perf_fini - Counter part to i915_perf_init()
  * @dev_priv: i915 device instance
@@ -3148,6 +3504,9 @@ void i915_perf_fini(struct drm_i915_private *dev_priv)
 	if (!dev_priv->perf.initialized)
 		return;
 
+	idr_for_each(&dev_priv->perf.metrics_idr, destroy_config, dev_priv);
+	idr_destroy(&dev_priv->perf.metrics_idr);
+
 	unregister_sysctl_table(dev_priv->perf.sysctl_header);
 
 	memset(&dev_priv->perf.oa.ops, 0, sizeof(dev_priv->perf.oa.ops));
diff --git a/drivers/gpu/drm/i915/i915_pvinfo.h b/drivers/gpu/drm/i915/i915_pvinfo.h
index 2cfe96d3e5d1..0679a58cdbae 100644
--- a/drivers/gpu/drm/i915/i915_pvinfo.h
+++ b/drivers/gpu/drm/i915/i915_pvinfo.h
@@ -49,12 +49,18 @@ enum vgt_g2v_type {
 	VGT_G2V_MAX,
 };
 
+/*
+ * VGT capabilities type
+ */
+#define VGT_CAPS_FULL_48BIT_PPGTT	BIT(2)
+
 struct vgt_if {
 	u64 magic;		/* VGT_MAGIC */
 	u16 version_major;
 	u16 version_minor;
 	u32 vgt_id;		/* ID of vGT instance */
-	u32 rsv1[12];		/* pad to offset 0x40 */
+	u32 vgt_caps;		/* VGT capabilities */
+	u32 rsv1[11];		/* pad to offset 0x40 */
 	/*
 	 *  Data structure to describe the balooning info of resources.
 	 *  Each VM can only have one portion of continuous area for now.
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 64cc674b652a..ed7cd9ee2c2a 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -25,6 +25,97 @@
 #ifndef _I915_REG_H_
 #define _I915_REG_H_
 
+/**
+ * DOC: The i915 register macro definition style guide
+ *
+ * Follow the style described here for new macros, and while changing existing
+ * macros. Do **not** mass change existing definitions just to update the style.
+ *
+ * Layout
+ * ''''''
+ *
+ * Keep helper macros near the top. For example, _PIPE() and friends.
+ *
+ * Prefix macros that generally should not be used outside of this file with
+ * underscore '_'. For example, _PIPE() and friends, single instances of
+ * registers that are defined solely for the use by function-like macros.
+ *
+ * Avoid using the underscore prefixed macros outside of this file. There are
+ * exceptions, but keep them to a minimum.
+ *
+ * There are two basic types of register definitions: Single registers and
+ * register groups. Register groups are registers which have two or more
+ * instances, for example one per pipe, port, transcoder, etc. Register groups
+ * should be defined using function-like macros.
+ *
+ * For single registers, define the register offset first, followed by register
+ * contents.
+ *
+ * For register groups, define the register instance offsets first, prefixed
+ * with underscore, followed by a function-like macro choosing the right
+ * instance based on the parameter, followed by register contents.
+ *
+ * Define the register contents (i.e. bit and bit field macros) from most
+ * significant to least significant bit. Indent the register content macros
+ * using two extra spaces between ``#define`` and the macro name.
+ *
+ * For bit fields, define a ``_MASK`` and a ``_SHIFT`` macro. Define bit field
+ * contents so that they are already shifted in place, and can be directly
+ * OR'd. For convenience, function-like macros may be used to define bit fields,
+ * but do note that the macros may be needed to read as well as write the
+ * register contents.
+ *
+ * Define bits using ``(1 << N)`` instead of ``BIT(N)``. We may change this in
+ * the future, but this is the prevailing style. Do **not** add ``_BIT`` suffix
+ * to the name.
+ *
+ * Group the register and its contents together without blank lines, separate
+ * from other registers and their contents with one blank line.
+ *
+ * Indent macro values from macro names using TABs. Align values vertically. Use
+ * braces in macro values as needed to avoid unintended precedence after macro
+ * substitution. Use spaces in macro values according to kernel coding
+ * style. Use lower case in hexadecimal values.
+ *
+ * Naming
+ * ''''''
+ *
+ * Try to name registers according to the specs. If the register name changes in
+ * the specs from platform to another, stick to the original name.
+ *
+ * Try to re-use existing register macro definitions. Only add new macros for
+ * new register offsets, or when the register contents have changed enough to
+ * warrant a full redefinition.
+ *
+ * When a register macro changes for a new platform, prefix the new macro using
+ * the platform acronym or generation. For example, ``SKL_`` or ``GEN8_``. The
+ * prefix signifies the start platform/generation using the register.
+ *
+ * When a bit (field) macro changes or gets added for a new platform, while
+ * retaining the existing register macro, add a platform acronym or generation
+ * suffix to the name. For example, ``_SKL`` or ``_GEN8``.
+ *
+ * Examples
+ * ''''''''
+ *
+ * (Note that the values in the example are indented using spaces instead of
+ * TABs to avoid misalignment in generated documentation. Use TABs in the
+ * definitions.)::
+ *
+ *  #define _FOO_A                      0xf000
+ *  #define _FOO_B                      0xf001
+ *  #define FOO(pipe)                   _MMIO_PIPE(pipe, _FOO_A, _FOO_B)
+ *  #define   FOO_ENABLE                (1 << 31)
+ *  #define   FOO_MODE_MASK             (0xf << 16)
+ *  #define   FOO_MODE_SHIFT            16
+ *  #define   FOO_MODE_BAR              (0 << 16)
+ *  #define   FOO_MODE_BAZ              (1 << 16)
+ *  #define   FOO_MODE_QUX_SNB          (2 << 16)
+ *
+ *  #define BAR                         _MMIO(0xb000)
+ *  #define GEN8_BAR                    _MMIO(0xb888)
+ */
+
 typedef struct {
 	uint32_t reg;
 } i915_reg_t;
@@ -229,6 +320,28 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define   GEN8_RPCS_EU_MIN_SHIFT	0
 #define   GEN8_RPCS_EU_MIN_MASK		(0xf << GEN8_RPCS_EU_MIN_SHIFT)
 
+#define WAIT_FOR_RC6_EXIT		_MMIO(0x20CC)
+/* HSW only */
+#define   HSW_SELECTIVE_READ_ADDRESSING_SHIFT		2
+#define   HSW_SELECTIVE_READ_ADDRESSING_MASK		(0x3 << HSW_SLECTIVE_READ_ADDRESSING_SHIFT)
+#define   HSW_SELECTIVE_WRITE_ADDRESS_SHIFT		4
+#define   HSW_SELECTIVE_WRITE_ADDRESS_MASK		(0x7 << HSW_SELECTIVE_WRITE_ADDRESS_SHIFT)
+/* HSW+ */
+#define   HSW_WAIT_FOR_RC6_EXIT_ENABLE			(1 << 0)
+#define   HSW_RCS_CONTEXT_ENABLE			(1 << 7)
+#define   HSW_RCS_INHIBIT				(1 << 8)
+/* Gen8 */
+#define   GEN8_SELECTIVE_WRITE_ADDRESS_SHIFT		4
+#define   GEN8_SELECTIVE_WRITE_ADDRESS_MASK		(0x3 << GEN8_SELECTIVE_WRITE_ADDRESS_SHIFT)
+#define   GEN8_SELECTIVE_WRITE_ADDRESS_SHIFT		4
+#define   GEN8_SELECTIVE_WRITE_ADDRESS_MASK		(0x3 << GEN8_SELECTIVE_WRITE_ADDRESS_SHIFT)
+#define   GEN8_SELECTIVE_WRITE_ADDRESSING_ENABLE	(1 << 6)
+#define   GEN8_SELECTIVE_READ_SUBSLICE_SELECT_SHIFT	9
+#define   GEN8_SELECTIVE_READ_SUBSLICE_SELECT_MASK	(0x3 << GEN8_SELECTIVE_READ_SUBSLICE_SELECT_SHIFT)
+#define   GEN8_SELECTIVE_READ_SLICE_SELECT_SHIFT	11
+#define   GEN8_SELECTIVE_READ_SLICE_SELECT_MASK		(0x3 << GEN8_SELECTIVE_READ_SLICE_SELECT_SHIFT)
+#define   GEN8_SELECTIVE_READ_ADDRESSING_ENABLE         (1 << 13)
+
 #define GAM_ECOCHK			_MMIO(0x4090)
 #define   BDW_DISABLE_HDC_INVALIDATION	(1<<25)
 #define   ECOCHK_SNB_BIT		(1<<10)
@@ -729,119 +842,10 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define EU_PERF_CNTL5	    _MMIO(0xe55c)
 #define EU_PERF_CNTL6	    _MMIO(0xe65c)
 
-#define GDT_CHICKEN_BITS    _MMIO(0x9840)
-#define GT_NOA_ENABLE	    0x00000080
-
 /*
  * OA Boolean state
  */
 
-#define OAREPORTTRIG1 _MMIO(0x2740)
-#define OAREPORTTRIG1_THRESHOLD_MASK 0xffff
-#define OAREPORTTRIG1_EDGE_LEVEL_TRIGER_SELECT_MASK 0xffff0000 /* 0=level */
-
-#define OAREPORTTRIG2 _MMIO(0x2744)
-#define OAREPORTTRIG2_INVERT_A_0  (1<<0)
-#define OAREPORTTRIG2_INVERT_A_1  (1<<1)
-#define OAREPORTTRIG2_INVERT_A_2  (1<<2)
-#define OAREPORTTRIG2_INVERT_A_3  (1<<3)
-#define OAREPORTTRIG2_INVERT_A_4  (1<<4)
-#define OAREPORTTRIG2_INVERT_A_5  (1<<5)
-#define OAREPORTTRIG2_INVERT_A_6  (1<<6)
-#define OAREPORTTRIG2_INVERT_A_7  (1<<7)
-#define OAREPORTTRIG2_INVERT_A_8  (1<<8)
-#define OAREPORTTRIG2_INVERT_A_9  (1<<9)
-#define OAREPORTTRIG2_INVERT_A_10 (1<<10)
-#define OAREPORTTRIG2_INVERT_A_11 (1<<11)
-#define OAREPORTTRIG2_INVERT_A_12 (1<<12)
-#define OAREPORTTRIG2_INVERT_A_13 (1<<13)
-#define OAREPORTTRIG2_INVERT_A_14 (1<<14)
-#define OAREPORTTRIG2_INVERT_A_15 (1<<15)
-#define OAREPORTTRIG2_INVERT_B_0  (1<<16)
-#define OAREPORTTRIG2_INVERT_B_1  (1<<17)
-#define OAREPORTTRIG2_INVERT_B_2  (1<<18)
-#define OAREPORTTRIG2_INVERT_B_3  (1<<19)
-#define OAREPORTTRIG2_INVERT_C_0  (1<<20)
-#define OAREPORTTRIG2_INVERT_C_1  (1<<21)
-#define OAREPORTTRIG2_INVERT_D_0  (1<<22)
-#define OAREPORTTRIG2_THRESHOLD_ENABLE	    (1<<23)
-#define OAREPORTTRIG2_REPORT_TRIGGER_ENABLE (1<<31)
-
-#define OAREPORTTRIG3 _MMIO(0x2748)
-#define OAREPORTTRIG3_NOA_SELECT_MASK	    0xf
-#define OAREPORTTRIG3_NOA_SELECT_8_SHIFT    0
-#define OAREPORTTRIG3_NOA_SELECT_9_SHIFT    4
-#define OAREPORTTRIG3_NOA_SELECT_10_SHIFT   8
-#define OAREPORTTRIG3_NOA_SELECT_11_SHIFT   12
-#define OAREPORTTRIG3_NOA_SELECT_12_SHIFT   16
-#define OAREPORTTRIG3_NOA_SELECT_13_SHIFT   20
-#define OAREPORTTRIG3_NOA_SELECT_14_SHIFT   24
-#define OAREPORTTRIG3_NOA_SELECT_15_SHIFT   28
-
-#define OAREPORTTRIG4 _MMIO(0x274c)
-#define OAREPORTTRIG4_NOA_SELECT_MASK	    0xf
-#define OAREPORTTRIG4_NOA_SELECT_0_SHIFT    0
-#define OAREPORTTRIG4_NOA_SELECT_1_SHIFT    4
-#define OAREPORTTRIG4_NOA_SELECT_2_SHIFT    8
-#define OAREPORTTRIG4_NOA_SELECT_3_SHIFT    12
-#define OAREPORTTRIG4_NOA_SELECT_4_SHIFT    16
-#define OAREPORTTRIG4_NOA_SELECT_5_SHIFT    20
-#define OAREPORTTRIG4_NOA_SELECT_6_SHIFT    24
-#define OAREPORTTRIG4_NOA_SELECT_7_SHIFT    28
-
-#define OAREPORTTRIG5 _MMIO(0x2750)
-#define OAREPORTTRIG5_THRESHOLD_MASK 0xffff
-#define OAREPORTTRIG5_EDGE_LEVEL_TRIGER_SELECT_MASK 0xffff0000 /* 0=level */
-
-#define OAREPORTTRIG6 _MMIO(0x2754)
-#define OAREPORTTRIG6_INVERT_A_0  (1<<0)
-#define OAREPORTTRIG6_INVERT_A_1  (1<<1)
-#define OAREPORTTRIG6_INVERT_A_2  (1<<2)
-#define OAREPORTTRIG6_INVERT_A_3  (1<<3)
-#define OAREPORTTRIG6_INVERT_A_4  (1<<4)
-#define OAREPORTTRIG6_INVERT_A_5  (1<<5)
-#define OAREPORTTRIG6_INVERT_A_6  (1<<6)
-#define OAREPORTTRIG6_INVERT_A_7  (1<<7)
-#define OAREPORTTRIG6_INVERT_A_8  (1<<8)
-#define OAREPORTTRIG6_INVERT_A_9  (1<<9)
-#define OAREPORTTRIG6_INVERT_A_10 (1<<10)
-#define OAREPORTTRIG6_INVERT_A_11 (1<<11)
-#define OAREPORTTRIG6_INVERT_A_12 (1<<12)
-#define OAREPORTTRIG6_INVERT_A_13 (1<<13)
-#define OAREPORTTRIG6_INVERT_A_14 (1<<14)
-#define OAREPORTTRIG6_INVERT_A_15 (1<<15)
-#define OAREPORTTRIG6_INVERT_B_0  (1<<16)
-#define OAREPORTTRIG6_INVERT_B_1  (1<<17)
-#define OAREPORTTRIG6_INVERT_B_2  (1<<18)
-#define OAREPORTTRIG6_INVERT_B_3  (1<<19)
-#define OAREPORTTRIG6_INVERT_C_0  (1<<20)
-#define OAREPORTTRIG6_INVERT_C_1  (1<<21)
-#define OAREPORTTRIG6_INVERT_D_0  (1<<22)
-#define OAREPORTTRIG6_THRESHOLD_ENABLE	    (1<<23)
-#define OAREPORTTRIG6_REPORT_TRIGGER_ENABLE (1<<31)
-
-#define OAREPORTTRIG7 _MMIO(0x2758)
-#define OAREPORTTRIG7_NOA_SELECT_MASK	    0xf
-#define OAREPORTTRIG7_NOA_SELECT_8_SHIFT    0
-#define OAREPORTTRIG7_NOA_SELECT_9_SHIFT    4
-#define OAREPORTTRIG7_NOA_SELECT_10_SHIFT   8
-#define OAREPORTTRIG7_NOA_SELECT_11_SHIFT   12
-#define OAREPORTTRIG7_NOA_SELECT_12_SHIFT   16
-#define OAREPORTTRIG7_NOA_SELECT_13_SHIFT   20
-#define OAREPORTTRIG7_NOA_SELECT_14_SHIFT   24
-#define OAREPORTTRIG7_NOA_SELECT_15_SHIFT   28
-
-#define OAREPORTTRIG8 _MMIO(0x275c)
-#define OAREPORTTRIG8_NOA_SELECT_MASK	    0xf
-#define OAREPORTTRIG8_NOA_SELECT_0_SHIFT    0
-#define OAREPORTTRIG8_NOA_SELECT_1_SHIFT    4
-#define OAREPORTTRIG8_NOA_SELECT_2_SHIFT    8
-#define OAREPORTTRIG8_NOA_SELECT_3_SHIFT    12
-#define OAREPORTTRIG8_NOA_SELECT_4_SHIFT    16
-#define OAREPORTTRIG8_NOA_SELECT_5_SHIFT    20
-#define OAREPORTTRIG8_NOA_SELECT_6_SHIFT    24
-#define OAREPORTTRIG8_NOA_SELECT_7_SHIFT    28
-
 #define OASTARTTRIG1 _MMIO(0x2710)
 #define OASTARTTRIG1_THRESHOLD_COUNT_MASK_MBZ 0xffff0000
 #define OASTARTTRIG1_THRESHOLD_MASK	      0xffff
@@ -956,6 +960,112 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define OASTARTTRIG8_NOA_SELECT_6_SHIFT    24
 #define OASTARTTRIG8_NOA_SELECT_7_SHIFT    28
 
+#define OAREPORTTRIG1 _MMIO(0x2740)
+#define OAREPORTTRIG1_THRESHOLD_MASK 0xffff
+#define OAREPORTTRIG1_EDGE_LEVEL_TRIGER_SELECT_MASK 0xffff0000 /* 0=level */
+
+#define OAREPORTTRIG2 _MMIO(0x2744)
+#define OAREPORTTRIG2_INVERT_A_0  (1<<0)
+#define OAREPORTTRIG2_INVERT_A_1  (1<<1)
+#define OAREPORTTRIG2_INVERT_A_2  (1<<2)
+#define OAREPORTTRIG2_INVERT_A_3  (1<<3)
+#define OAREPORTTRIG2_INVERT_A_4  (1<<4)
+#define OAREPORTTRIG2_INVERT_A_5  (1<<5)
+#define OAREPORTTRIG2_INVERT_A_6  (1<<6)
+#define OAREPORTTRIG2_INVERT_A_7  (1<<7)
+#define OAREPORTTRIG2_INVERT_A_8  (1<<8)
+#define OAREPORTTRIG2_INVERT_A_9  (1<<9)
+#define OAREPORTTRIG2_INVERT_A_10 (1<<10)
+#define OAREPORTTRIG2_INVERT_A_11 (1<<11)
+#define OAREPORTTRIG2_INVERT_A_12 (1<<12)
+#define OAREPORTTRIG2_INVERT_A_13 (1<<13)
+#define OAREPORTTRIG2_INVERT_A_14 (1<<14)
+#define OAREPORTTRIG2_INVERT_A_15 (1<<15)
+#define OAREPORTTRIG2_INVERT_B_0  (1<<16)
+#define OAREPORTTRIG2_INVERT_B_1  (1<<17)
+#define OAREPORTTRIG2_INVERT_B_2  (1<<18)
+#define OAREPORTTRIG2_INVERT_B_3  (1<<19)
+#define OAREPORTTRIG2_INVERT_C_0  (1<<20)
+#define OAREPORTTRIG2_INVERT_C_1  (1<<21)
+#define OAREPORTTRIG2_INVERT_D_0  (1<<22)
+#define OAREPORTTRIG2_THRESHOLD_ENABLE	    (1<<23)
+#define OAREPORTTRIG2_REPORT_TRIGGER_ENABLE (1<<31)
+
+#define OAREPORTTRIG3 _MMIO(0x2748)
+#define OAREPORTTRIG3_NOA_SELECT_MASK	    0xf
+#define OAREPORTTRIG3_NOA_SELECT_8_SHIFT    0
+#define OAREPORTTRIG3_NOA_SELECT_9_SHIFT    4
+#define OAREPORTTRIG3_NOA_SELECT_10_SHIFT   8
+#define OAREPORTTRIG3_NOA_SELECT_11_SHIFT   12
+#define OAREPORTTRIG3_NOA_SELECT_12_SHIFT   16
+#define OAREPORTTRIG3_NOA_SELECT_13_SHIFT   20
+#define OAREPORTTRIG3_NOA_SELECT_14_SHIFT   24
+#define OAREPORTTRIG3_NOA_SELECT_15_SHIFT   28
+
+#define OAREPORTTRIG4 _MMIO(0x274c)
+#define OAREPORTTRIG4_NOA_SELECT_MASK	    0xf
+#define OAREPORTTRIG4_NOA_SELECT_0_SHIFT    0
+#define OAREPORTTRIG4_NOA_SELECT_1_SHIFT    4
+#define OAREPORTTRIG4_NOA_SELECT_2_SHIFT    8
+#define OAREPORTTRIG4_NOA_SELECT_3_SHIFT    12
+#define OAREPORTTRIG4_NOA_SELECT_4_SHIFT    16
+#define OAREPORTTRIG4_NOA_SELECT_5_SHIFT    20
+#define OAREPORTTRIG4_NOA_SELECT_6_SHIFT    24
+#define OAREPORTTRIG4_NOA_SELECT_7_SHIFT    28
+
+#define OAREPORTTRIG5 _MMIO(0x2750)
+#define OAREPORTTRIG5_THRESHOLD_MASK 0xffff
+#define OAREPORTTRIG5_EDGE_LEVEL_TRIGER_SELECT_MASK 0xffff0000 /* 0=level */
+
+#define OAREPORTTRIG6 _MMIO(0x2754)
+#define OAREPORTTRIG6_INVERT_A_0  (1<<0)
+#define OAREPORTTRIG6_INVERT_A_1  (1<<1)
+#define OAREPORTTRIG6_INVERT_A_2  (1<<2)
+#define OAREPORTTRIG6_INVERT_A_3  (1<<3)
+#define OAREPORTTRIG6_INVERT_A_4  (1<<4)
+#define OAREPORTTRIG6_INVERT_A_5  (1<<5)
+#define OAREPORTTRIG6_INVERT_A_6  (1<<6)
+#define OAREPORTTRIG6_INVERT_A_7  (1<<7)
+#define OAREPORTTRIG6_INVERT_A_8  (1<<8)
+#define OAREPORTTRIG6_INVERT_A_9  (1<<9)
+#define OAREPORTTRIG6_INVERT_A_10 (1<<10)
+#define OAREPORTTRIG6_INVERT_A_11 (1<<11)
+#define OAREPORTTRIG6_INVERT_A_12 (1<<12)
+#define OAREPORTTRIG6_INVERT_A_13 (1<<13)
+#define OAREPORTTRIG6_INVERT_A_14 (1<<14)
+#define OAREPORTTRIG6_INVERT_A_15 (1<<15)
+#define OAREPORTTRIG6_INVERT_B_0  (1<<16)
+#define OAREPORTTRIG6_INVERT_B_1  (1<<17)
+#define OAREPORTTRIG6_INVERT_B_2  (1<<18)
+#define OAREPORTTRIG6_INVERT_B_3  (1<<19)
+#define OAREPORTTRIG6_INVERT_C_0  (1<<20)
+#define OAREPORTTRIG6_INVERT_C_1  (1<<21)
+#define OAREPORTTRIG6_INVERT_D_0  (1<<22)
+#define OAREPORTTRIG6_THRESHOLD_ENABLE	    (1<<23)
+#define OAREPORTTRIG6_REPORT_TRIGGER_ENABLE (1<<31)
+
+#define OAREPORTTRIG7 _MMIO(0x2758)
+#define OAREPORTTRIG7_NOA_SELECT_MASK	    0xf
+#define OAREPORTTRIG7_NOA_SELECT_8_SHIFT    0
+#define OAREPORTTRIG7_NOA_SELECT_9_SHIFT    4
+#define OAREPORTTRIG7_NOA_SELECT_10_SHIFT   8
+#define OAREPORTTRIG7_NOA_SELECT_11_SHIFT   12
+#define OAREPORTTRIG7_NOA_SELECT_12_SHIFT   16
+#define OAREPORTTRIG7_NOA_SELECT_13_SHIFT   20
+#define OAREPORTTRIG7_NOA_SELECT_14_SHIFT   24
+#define OAREPORTTRIG7_NOA_SELECT_15_SHIFT   28
+
+#define OAREPORTTRIG8 _MMIO(0x275c)
+#define OAREPORTTRIG8_NOA_SELECT_MASK	    0xf
+#define OAREPORTTRIG8_NOA_SELECT_0_SHIFT    0
+#define OAREPORTTRIG8_NOA_SELECT_1_SHIFT    4
+#define OAREPORTTRIG8_NOA_SELECT_2_SHIFT    8
+#define OAREPORTTRIG8_NOA_SELECT_3_SHIFT    12
+#define OAREPORTTRIG8_NOA_SELECT_4_SHIFT    16
+#define OAREPORTTRIG8_NOA_SELECT_5_SHIFT    20
+#define OAREPORTTRIG8_NOA_SELECT_6_SHIFT    24
+#define OAREPORTTRIG8_NOA_SELECT_7_SHIFT    28
+
 /* CECX_0 */
 #define OACEC_COMPARE_LESS_OR_EQUAL	6
 #define OACEC_COMPARE_NOT_EQUAL		5
@@ -994,6 +1104,51 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define OACEC7_0 _MMIO(0x27a8)
 #define OACEC7_1 _MMIO(0x27ac)
 
+/* OA perf counters */
+#define OA_PERFCNT1_LO      _MMIO(0x91B8)
+#define OA_PERFCNT1_HI      _MMIO(0x91BC)
+#define OA_PERFCNT2_LO      _MMIO(0x91C0)
+#define OA_PERFCNT2_HI      _MMIO(0x91C4)
+
+#define OA_PERFMATRIX_LO    _MMIO(0x91C8)
+#define OA_PERFMATRIX_HI    _MMIO(0x91CC)
+
+/* RPM unit config (Gen8+) */
+#define RPM_CONFIG0	    _MMIO(0x0D00)
+#define RPM_CONFIG1	    _MMIO(0x0D04)
+
+/* RPC unit config (Gen8+) */
+#define RPM_CONFIG	    _MMIO(0x0D08)
+
+/* NOA (Gen8+) */
+#define NOA_CONFIG(i)	    _MMIO(0x0D0C + (i) * 4)
+
+#define MICRO_BP0_0	    _MMIO(0x9800)
+#define MICRO_BP0_2	    _MMIO(0x9804)
+#define MICRO_BP0_1	    _MMIO(0x9808)
+
+#define MICRO_BP1_0	    _MMIO(0x980C)
+#define MICRO_BP1_2	    _MMIO(0x9810)
+#define MICRO_BP1_1	    _MMIO(0x9814)
+
+#define MICRO_BP2_0	    _MMIO(0x9818)
+#define MICRO_BP2_2	    _MMIO(0x981C)
+#define MICRO_BP2_1	    _MMIO(0x9820)
+
+#define MICRO_BP3_0	    _MMIO(0x9824)
+#define MICRO_BP3_2	    _MMIO(0x9828)
+#define MICRO_BP3_1	    _MMIO(0x982C)
+
+#define MICRO_BP_TRIGGER		_MMIO(0x9830)
+#define MICRO_BP3_COUNT_STATUS01	_MMIO(0x9834)
+#define MICRO_BP3_COUNT_STATUS23	_MMIO(0x9838)
+#define MICRO_BP_FIRED_ARMED		_MMIO(0x983C)
+
+#define GDT_CHICKEN_BITS    _MMIO(0x9840)
+#define   GT_NOA_ENABLE	    0x00000080
+
+#define NOA_DATA	    _MMIO(0x986C)
+#define NOA_WRITE	    _MMIO(0x9888)
 
 #define _GEN7_PIPEA_DE_LOAD_SL	0x70068
 #define _GEN7_PIPEB_DE_LOAD_SL	0x71068
@@ -1063,9 +1218,26 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define   DP_SSS_RESET(pipe)			_DP_SSS(0x2, (pipe))
 #define   DP_SSS_PWR_GATE(pipe)			_DP_SSS(0x3, (pipe))
 
-/* See the PUNIT HAS v0.8 for the below bits */
-enum punit_power_well {
-	/* These numbers are fixed and must match the position of the pw bits */
+/*
+ * i915_power_well_id:
+ *
+ * Platform specific IDs used to look up power wells and - except for custom
+ * power wells - to define request/status register flag bit positions. As such
+ * the set of IDs on a given platform must be unique and except for custom
+ * power wells their value must stay fixed.
+ */
+enum i915_power_well_id {
+	/*
+	 * I830
+	 *  - custom power well
+	 */
+	I830_DISP_PW_PIPES = 0,
+
+	/*
+	 * VLV/CHV
+	 *  - PUNIT_REG_PWRGT_CTRL (bit: id*2),
+	 *    PUNIT_REG_PWRGT_STATUS (bit: id*2) (PUNIT HAS v0.8)
+	 */
 	PUNIT_POWER_WELL_RENDER			= 0,
 	PUNIT_POWER_WELL_MEDIA			= 1,
 	PUNIT_POWER_WELL_DISP2D			= 3,
@@ -1077,14 +1249,20 @@ enum punit_power_well {
 	PUNIT_POWER_WELL_DPIO_RX0		= 10,
 	PUNIT_POWER_WELL_DPIO_RX1		= 11,
 	PUNIT_POWER_WELL_DPIO_CMN_D		= 12,
-
-	/* Not actual bit groups. Used as IDs for lookup_power_well() */
-	PUNIT_POWER_WELL_ALWAYS_ON,
-};
-
-enum skl_disp_power_wells {
-	/* These numbers are fixed and must match the position of the pw bits */
-	SKL_DISP_PW_MISC_IO,
+	/*  - custom power well */
+	CHV_DISP_PW_PIPE_A,			/* 13 */
+
+	/*
+	 * HSW/BDW
+	 *  - HSW_PWR_WELL_CTL_DRIVER(0) (status bit: id*2, req bit: id*2+1)
+	 */
+	HSW_DISP_PW_GLOBAL = 15,
+
+	/*
+	 * GEN9+
+	 *  - HSW_PWR_WELL_CTL_DRIVER(0) (status bit: id*2, req bit: id*2+1)
+	 */
+	SKL_DISP_PW_MISC_IO = 0,
 	SKL_DISP_PW_DDI_A_E,
 	GLK_DISP_PW_DDI_A = SKL_DISP_PW_DDI_A_E,
 	CNL_DISP_PW_DDI_A = SKL_DISP_PW_DDI_A_E,
@@ -1103,18 +1281,20 @@ enum skl_disp_power_wells {
 	SKL_DISP_PW_1 = 14,
 	SKL_DISP_PW_2,
 
-	/* Not actual bit groups. Used as IDs for lookup_power_well() */
-	SKL_DISP_PW_ALWAYS_ON,
+	/* - custom power wells */
 	SKL_DISP_PW_DC_OFF,
-
 	BXT_DPIO_CMN_A,
 	BXT_DPIO_CMN_BC,
-	GLK_DPIO_CMN_C,
+	GLK_DPIO_CMN_C,			/* 19 */
+
+	/*
+	 * Multiple platforms.
+	 * Must start following the highest ID of any platform.
+	 * - custom power wells
+	 */
+	I915_DISP_PW_ALWAYS_ON = 20,
 };
 
-#define SKL_POWER_WELL_STATE(pw) (1 << ((pw) * 2))
-#define SKL_POWER_WELL_REQ(pw) (1 << (((pw) * 2) + 1))
-
 #define PUNIT_REG_PWRGT_CTRL			0x60
 #define PUNIT_REG_PWRGT_STATUS			0x61
 #define   PUNIT_PWRGT_MASK(power_well)		(3 << ((power_well) * 2))
@@ -2156,6 +2336,7 @@ enum skl_disp_power_wells {
 #define DONE_REG		_MMIO(0x40b0)
 #define GEN8_PRIVATE_PAT_LO	_MMIO(0x40e0)
 #define GEN8_PRIVATE_PAT_HI	_MMIO(0x40e0 + 4)
+#define GEN10_PAT_INDEX(index)	_MMIO(0x40e0 + index*4)
 #define BSD_HWS_PGA_GEN7	_MMIO(0x04180)
 #define BLT_HWS_PGA_GEN7	_MMIO(0x04280)
 #define VEBOX_HWS_PGA_GEN7	_MMIO(0x04380)
@@ -3522,7 +3703,7 @@ enum skl_disp_power_wells {
 #define INTERVAL_1_28_US(us)	roundup(((us) * 100) >> 7, 25)
 #define INTERVAL_1_33_US(us)	(((us) * 3)   >> 2)
 #define INTERVAL_0_833_US(us)	(((us) * 6) / 5)
-#define GT_INTERVAL_FROM_US(dev_priv, us) (IS_GEN9(dev_priv) ? \
+#define GT_INTERVAL_FROM_US(dev_priv, us) (INTEL_GEN(dev_priv) >= 9 ? \
 				(IS_GEN9_LP(dev_priv) ? \
 				INTERVAL_0_833_US(us) : \
 				INTERVAL_1_33_US(us)) : \
@@ -3531,7 +3712,7 @@ enum skl_disp_power_wells {
 #define INTERVAL_1_28_TO_US(interval)  (((interval) << 7) / 100)
 #define INTERVAL_1_33_TO_US(interval)  (((interval) << 2) / 3)
 #define INTERVAL_0_833_TO_US(interval) (((interval) * 5)  / 6)
-#define GT_PM_INTERVAL_TO_US(dev_priv, interval) (IS_GEN9(dev_priv) ? \
+#define GT_PM_INTERVAL_TO_US(dev_priv, interval) (INTEL_GEN(dev_priv) >= 9 ? \
                            (IS_GEN9_LP(dev_priv) ? \
                            INTERVAL_0_833_TO_US(interval) : \
                            INTERVAL_1_33_TO_US(interval)) : \
@@ -3783,6 +3964,7 @@ enum {
 #define EDP_PSR_CTL				_MMIO(dev_priv->psr_mmio_base + 0)
 #define   EDP_PSR_ENABLE			(1<<31)
 #define   BDW_PSR_SINGLE_FRAME			(1<<30)
+#define   EDP_PSR_RESTORE_PSR_ACTIVE_CTX_MASK	(1<<29) /* SW can't modify */
 #define   EDP_PSR_LINK_STANDBY			(1<<27)
 #define   EDP_PSR_MIN_LINK_ENTRY_TIME_MASK	(3<<25)
 #define   EDP_PSR_MIN_LINK_ENTRY_TIME_8_LINES	(0<<25)
@@ -5227,6 +5409,9 @@ enum {
 
 #define _PIPE_MISC_A			0x70030
 #define _PIPE_MISC_B			0x71030
+#define   PIPEMISC_YUV420_ENABLE	(1<<27)
+#define   PIPEMISC_YUV420_MODE_FULL_BLEND (1<<26)
+#define   PIPEMISC_OUTPUT_COLORSPACE_YUV  (1<<11)
 #define   PIPEMISC_DITHER_BPC_MASK	(7<<5)
 #define   PIPEMISC_DITHER_8_BPC		(0<<5)
 #define   PIPEMISC_DITHER_10_BPC	(1<<5)
@@ -6106,6 +6291,10 @@ enum {
 #define _PLANE_KEYMSK_2_A			0x70298
 #define _PLANE_KEYMAX_1_A			0x701a0
 #define _PLANE_KEYMAX_2_A			0x702a0
+#define _PLANE_AUX_DIST_1_A			0x701c0
+#define _PLANE_AUX_DIST_2_A			0x702c0
+#define _PLANE_AUX_OFFSET_1_A			0x701c4
+#define _PLANE_AUX_OFFSET_2_A			0x702c4
 #define _PLANE_COLOR_CTL_1_A			0x701CC /* GLK+ */
 #define _PLANE_COLOR_CTL_2_A			0x702CC /* GLK+ */
 #define _PLANE_COLOR_CTL_3_A			0x703CC /* GLK+ */
@@ -6212,6 +6401,24 @@ enum {
 #define PLANE_NV12_BUF_CFG(pipe, plane)	\
 	_MMIO_PLANE(plane, _PLANE_NV12_BUF_CFG_1(pipe), _PLANE_NV12_BUF_CFG_2(pipe))
 
+#define _PLANE_AUX_DIST_1_B		0x711c0
+#define _PLANE_AUX_DIST_2_B		0x712c0
+#define _PLANE_AUX_DIST_1(pipe) \
+			_PIPE(pipe, _PLANE_AUX_DIST_1_A, _PLANE_AUX_DIST_1_B)
+#define _PLANE_AUX_DIST_2(pipe) \
+			_PIPE(pipe, _PLANE_AUX_DIST_2_A, _PLANE_AUX_DIST_2_B)
+#define PLANE_AUX_DIST(pipe, plane)     \
+	_MMIO_PLANE(plane, _PLANE_AUX_DIST_1(pipe), _PLANE_AUX_DIST_2(pipe))
+
+#define _PLANE_AUX_OFFSET_1_B		0x711c4
+#define _PLANE_AUX_OFFSET_2_B		0x712c4
+#define _PLANE_AUX_OFFSET_1(pipe)       \
+		_PIPE(pipe, _PLANE_AUX_OFFSET_1_A, _PLANE_AUX_OFFSET_1_B)
+#define _PLANE_AUX_OFFSET_2(pipe)       \
+		_PIPE(pipe, _PLANE_AUX_OFFSET_2_A, _PLANE_AUX_OFFSET_2_B)
+#define PLANE_AUX_OFFSET(pipe, plane)   \
+	_MMIO_PLANE(plane, _PLANE_AUX_OFFSET_1(pipe), _PLANE_AUX_OFFSET_2(pipe))
+
 #define _PLANE_COLOR_CTL_1_B			0x711CC
 #define _PLANE_COLOR_CTL_2_B			0x712CC
 #define _PLANE_COLOR_CTL_3_B			0x713CC
@@ -6695,6 +6902,7 @@ enum {
 # define CHICKEN3_DGMG_DONE_FIX_DISABLE		(1 << 2)
 
 #define CHICKEN_PAR1_1		_MMIO(0x42080)
+#define  SKL_RC_HASH_OUTSIDE	(1 << 15)
 #define  DPA_MASK_VBLANK_SRD	(1 << 15)
 #define  FORCE_ARB_IDLE_PLANES	(1 << 14)
 #define  SKL_EDP_PSR_FIX_RDWRAP	(1 << 3)
@@ -6703,12 +6911,10 @@ enum {
 #define  KVM_CONFIG_CHANGE_NOTIFICATION_SELECT	(1 << 14)
 
 #define CHICKEN_MISC_2		_MMIO(0x42084)
-#define  GLK_CL0_PWR_DOWN	(1 << 10)
-#define  GLK_CL1_PWR_DOWN	(1 << 11)
+#define  CNL_COMP_PWR_DOWN	(1 << 23)
 #define  GLK_CL2_PWR_DOWN	(1 << 12)
-
-#define CHICKEN_MISC_2		_MMIO(0x42084)
-#define  COMP_PWR_DOWN		(1 << 23)
+#define  GLK_CL1_PWR_DOWN	(1 << 11)
+#define  GLK_CL0_PWR_DOWN	(1 << 10)
 
 #define _CHICKEN_PIPESL_1_A	0x420b0
 #define _CHICKEN_PIPESL_1_B	0x420b4
@@ -7984,12 +8190,31 @@ enum {
 #define   SKL_AUD_CODEC_WAKE_SIGNAL		(1 << 15)
 
 /* HSW Power Wells */
-#define HSW_PWR_WELL_BIOS			_MMIO(0x45400) /* CTL1 */
-#define HSW_PWR_WELL_DRIVER			_MMIO(0x45404) /* CTL2 */
-#define HSW_PWR_WELL_KVMR			_MMIO(0x45408) /* CTL3 */
-#define HSW_PWR_WELL_DEBUG			_MMIO(0x4540C) /* CTL4 */
-#define   HSW_PWR_WELL_ENABLE_REQUEST		(1<<31)
-#define   HSW_PWR_WELL_STATE_ENABLED		(1<<30)
+#define _HSW_PWR_WELL_CTL1			0x45400
+#define _HSW_PWR_WELL_CTL2			0x45404
+#define _HSW_PWR_WELL_CTL3			0x45408
+#define _HSW_PWR_WELL_CTL4			0x4540C
+
+/*
+ * Each power well control register contains up to 16 (request, status) HW
+ * flag tuples. The register index and HW flag shift is determined by the
+ * power well ID (see i915_power_well_id). There are 4 possible sources of
+ * power well requests each source having its own set of control registers:
+ * BIOS, DRIVER, KVMR, DEBUG.
+ */
+#define _HSW_PW_REG_IDX(pw)			((pw) >> 4)
+#define _HSW_PW_SHIFT(pw)			(((pw) & 0xf) * 2)
+/* TODO: Add all PWR_WELL_CTL registers below for new platforms */
+#define HSW_PWR_WELL_CTL_BIOS(pw)	_MMIO(_PICK(_HSW_PW_REG_IDX(pw),       \
+						    _HSW_PWR_WELL_CTL1))
+#define HSW_PWR_WELL_CTL_DRIVER(pw)	_MMIO(_PICK(_HSW_PW_REG_IDX(pw),       \
+						    _HSW_PWR_WELL_CTL2))
+#define HSW_PWR_WELL_CTL_KVMR		_MMIO(_HSW_PWR_WELL_CTL3)
+#define HSW_PWR_WELL_CTL_DEBUG(pw)	_MMIO(_PICK(_HSW_PW_REG_IDX(pw),       \
+						    _HSW_PWR_WELL_CTL4))
+
+#define   HSW_PWR_WELL_CTL_REQ(pw)		(1 << (_HSW_PW_SHIFT(pw) + 1))
+#define   HSW_PWR_WELL_CTL_STATE(pw)		(1 << _HSW_PW_SHIFT(pw))
 #define HSW_PWR_WELL_CTL5			_MMIO(0x45410)
 #define   HSW_PWR_WELL_ENABLE_SINGLE_STEP	(1<<31)
 #define   HSW_PWR_WELL_PWR_GATE_OVERRIDE	(1<<20)
@@ -7997,11 +8222,17 @@ enum {
 #define HSW_PWR_WELL_CTL6			_MMIO(0x45414)
 
 /* SKL Fuse Status */
+enum skl_power_gate {
+	SKL_PG0,
+	SKL_PG1,
+	SKL_PG2,
+};
+
 #define SKL_FUSE_STATUS				_MMIO(0x42000)
-#define  SKL_FUSE_DOWNLOAD_STATUS              (1<<31)
-#define  SKL_FUSE_PG0_DIST_STATUS              (1<<27)
-#define  SKL_FUSE_PG1_DIST_STATUS              (1<<26)
-#define  SKL_FUSE_PG2_DIST_STATUS              (1<<25)
+#define  SKL_FUSE_DOWNLOAD_STATUS		(1<<31)
+/* PG0 (HW control->no power well ID), PG1..PG2 (SKL_DISP_PW1..SKL_DISP_PW2) */
+#define  SKL_PW_TO_PG(pw)			((pw) - SKL_DISP_PW_1 + SKL_PG1)
+#define  SKL_FUSE_PG_DIST_STATUS(pg)		(1 << (27 - (pg)))
 
 /* Per-pipe DDI Function Control */
 #define _TRANS_DDI_FUNC_CTL_A		0x60400
@@ -8343,6 +8574,7 @@ enum {
 #define  DPLL_CFGCR0_LINK_RATE_3240	(6 << 25)
 #define  DPLL_CFGCR0_LINK_RATE_4050	(7 << 25)
 #define  DPLL_CFGCR0_DCO_FRACTION_MASK	(0x7fff << 10)
+#define  DPLL_CFGCR0_DCO_FRAC_SHIFT	(10)
 #define  DPLL_CFGCR0_DCO_FRACTION(x)	((x) << 10)
 #define  DPLL_CFGCR0_DCO_INTEGER_MASK	(0x3ff)
 #define CNL_DPLL_CFGCR0(pll)		_MMIO_PLL(pll, _CNL_DPLL0_CFGCR0, _CNL_DPLL1_CFGCR0)
@@ -8350,6 +8582,7 @@ enum {
 #define _CNL_DPLL0_CFGCR1		0x6C004
 #define _CNL_DPLL1_CFGCR1		0x6C084
 #define  DPLL_CFGCR1_QDIV_RATIO_MASK	(0xff << 10)
+#define  DPLL_CFGCR1_QDIV_RATIO_SHIFT	(10)
 #define  DPLL_CFGCR1_QDIV_RATIO(x)	((x) << 10)
 #define  DPLL_CFGCR1_QDIV_MODE(x)	((x) << 9)
 #define  DPLL_CFGCR1_KDIV_MASK		(7 << 6)
diff --git a/drivers/gpu/drm/i915/i915_selftest.h b/drivers/gpu/drm/i915/i915_selftest.h
index 9d7d86f1733d..78e1a1b168ff 100644
--- a/drivers/gpu/drm/i915/i915_selftest.h
+++ b/drivers/gpu/drm/i915/i915_selftest.h
@@ -101,6 +101,4 @@ bool __igt_timeout(unsigned long timeout, const char *fmt, ...);
 #define igt_timeout(t, fmt, ...) \
 	__igt_timeout((t), KERN_WARNING pr_fmt(fmt), ##__VA_ARGS__)
 
-#define igt_can_mi_store_dword_imm(D) (INTEL_GEN(D) > 2)
-
 #endif /* !__I915_SELFTEST_H__ */
diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c
index 1eef3fae4db3..d61c8727f756 100644
--- a/drivers/gpu/drm/i915/i915_sysfs.c
+++ b/drivers/gpu/drm/i915/i915_sysfs.c
@@ -96,7 +96,7 @@ static struct attribute *rc6_attrs[] = {
 	NULL
 };
 
-static struct attribute_group rc6_attr_group = {
+static const struct attribute_group rc6_attr_group = {
 	.name = power_group_name,
 	.attrs =  rc6_attrs
 };
@@ -107,7 +107,7 @@ static struct attribute *rc6p_attrs[] = {
 	NULL
 };
 
-static struct attribute_group rc6p_attr_group = {
+static const struct attribute_group rc6p_attr_group = {
 	.name = power_group_name,
 	.attrs =  rc6p_attrs
 };
@@ -117,7 +117,7 @@ static struct attribute *media_rc6_attrs[] = {
 	NULL
 };
 
-static struct attribute_group media_rc6_attr_group = {
+static const struct attribute_group media_rc6_attr_group = {
 	.name = power_group_name,
 	.attrs =  media_rc6_attrs
 };
@@ -209,7 +209,7 @@ i915_l3_write(struct file *filp, struct kobject *kobj,
 	memcpy(*remap_info + (offset/4), buf, count);
 
 	/* NB: We defer the remapping until we switch to the context */
-	list_for_each_entry(ctx, &dev_priv->context_list, link)
+	list_for_each_entry(ctx, &dev_priv->contexts.list, link)
 		ctx->remap_slice |= (1<<slice);
 
 	ret = count;
@@ -220,7 +220,7 @@ out:
 	return ret;
 }
 
-static struct bin_attribute dpf_attrs = {
+static const struct bin_attribute dpf_attrs = {
 	.attr = {.name = "l3_parity", .mode = (S_IRUSR | S_IWUSR)},
 	.size = GEN7_L3LOG_SIZE,
 	.read = i915_l3_read,
@@ -229,7 +229,7 @@ static struct bin_attribute dpf_attrs = {
 	.private = (void *)0
 };
 
-static struct bin_attribute dpf_attrs_1 = {
+static const struct bin_attribute dpf_attrs_1 = {
 	.attr = {.name = "l3_parity_slice_1", .mode = (S_IRUSR | S_IWUSR)},
 	.size = GEN7_L3LOG_SIZE,
 	.read = i915_l3_read,
@@ -253,7 +253,7 @@ static ssize_t gt_act_freq_mhz_show(struct device *kdev,
 		ret = intel_gpu_freq(dev_priv, (freq >> 8) & 0xff);
 	} else {
 		u32 rpstat = I915_READ(GEN6_RPSTAT1);
-		if (IS_GEN9(dev_priv))
+		if (INTEL_GEN(dev_priv) >= 9)
 			ret = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
 		else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 			ret = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
@@ -532,7 +532,7 @@ static ssize_t error_state_write(struct file *file, struct kobject *kobj,
 	return count;
 }
 
-static struct bin_attribute error_state_attr = {
+static const struct bin_attribute error_state_attr = {
 	.attr.name = "error",
 	.attr.mode = S_IRUSR | S_IWUSR,
 	.size = 0,
diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h
index b24a83d43559..6fd5c57e21f6 100644
--- a/drivers/gpu/drm/i915/i915_trace.h
+++ b/drivers/gpu/drm/i915/i915_trace.h
@@ -38,7 +38,7 @@ TRACE_EVENT(intel_cpu_fifo_underrun,
 );
 
 TRACE_EVENT(intel_pch_fifo_underrun,
-	    TP_PROTO(struct drm_i915_private *dev_priv, enum transcoder pch_transcoder),
+	    TP_PROTO(struct drm_i915_private *dev_priv, enum pipe pch_transcoder),
 	    TP_ARGS(dev_priv, pch_transcoder),
 
 	    TP_STRUCT__entry(
@@ -48,7 +48,7 @@ TRACE_EVENT(intel_pch_fifo_underrun,
 			     ),
 
 	    TP_fast_assign(
-			   enum pipe pipe = (enum pipe)pch_transcoder;
+			   enum pipe pipe = pch_transcoder;
 			   __entry->pipe = pipe;
 			   __entry->frame = dev_priv->drm.driver->get_vblank_counter(&dev_priv->drm, pipe);
 			   __entry->scanline = intel_get_crtc_scanline(intel_get_crtc_for_pipe(dev_priv, pipe));
diff --git a/drivers/gpu/drm/i915/i915_vgpu.c b/drivers/gpu/drm/i915/i915_vgpu.c
index cf7a958e4d3c..5fe9f3f39467 100644
--- a/drivers/gpu/drm/i915/i915_vgpu.c
+++ b/drivers/gpu/drm/i915/i915_vgpu.c
@@ -75,10 +75,17 @@ void i915_check_vgpu(struct drm_i915_private *dev_priv)
 		return;
 	}
 
+	dev_priv->vgpu.caps = __raw_i915_read32(dev_priv, vgtif_reg(vgt_caps));
+
 	dev_priv->vgpu.active = true;
 	DRM_INFO("Virtual GPU for Intel GVT-g detected.\n");
 }
 
+bool intel_vgpu_has_full_48bit_ppgtt(struct drm_i915_private *dev_priv)
+{
+	return dev_priv->vgpu.caps & VGT_CAPS_FULL_48BIT_PPGTT;
+}
+
 struct _balloon_info_ {
 	/*
 	 * There are up to 2 regions per mappable/unmappable graphic
diff --git a/drivers/gpu/drm/i915/i915_vgpu.h b/drivers/gpu/drm/i915/i915_vgpu.h
index 3c3b2d24e830..b72bd2956b70 100644
--- a/drivers/gpu/drm/i915/i915_vgpu.h
+++ b/drivers/gpu/drm/i915/i915_vgpu.h
@@ -27,6 +27,9 @@
 #include "i915_pvinfo.h"
 
 void i915_check_vgpu(struct drm_i915_private *dev_priv);
+
+bool intel_vgpu_has_full_48bit_ppgtt(struct drm_i915_private *dev_priv);
+
 int intel_vgt_balloon(struct drm_i915_private *dev_priv);
 void intel_vgt_deballoon(struct drm_i915_private *dev_priv);
 
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index 1cfe137cdc32..02d1a5eacb00 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -579,11 +579,17 @@ err_unpin:
 
 static void i915_vma_destroy(struct i915_vma *vma)
 {
+	int i;
+
 	GEM_BUG_ON(vma->node.allocated);
 	GEM_BUG_ON(i915_vma_is_active(vma));
 	GEM_BUG_ON(!i915_vma_is_closed(vma));
 	GEM_BUG_ON(vma->fence);
 
+	for (i = 0; i < ARRAY_SIZE(vma->last_read); i++)
+		GEM_BUG_ON(i915_gem_active_isset(&vma->last_read[i]));
+	GEM_BUG_ON(i915_gem_active_isset(&vma->last_fence));
+
 	list_del(&vma->vm_link);
 	if (!i915_vma_is_ggtt(vma))
 		i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm));
@@ -591,33 +597,11 @@ static void i915_vma_destroy(struct i915_vma *vma)
 	kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma);
 }
 
-void i915_vma_unlink_ctx(struct i915_vma *vma)
-{
-	struct i915_gem_context *ctx = vma->ctx;
-
-	if (ctx->vma_lut.ht_size & I915_CTX_RESIZE_IN_PROGRESS) {
-		cancel_work_sync(&ctx->vma_lut.resize);
-		ctx->vma_lut.ht_size &= ~I915_CTX_RESIZE_IN_PROGRESS;
-	}
-
-	__hlist_del(&vma->ctx_node);
-	ctx->vma_lut.ht_count--;
-
-	if (i915_vma_is_ggtt(vma))
-		vma->obj->vma_hashed = NULL;
-	vma->ctx = NULL;
-
-	i915_vma_put(vma);
-}
-
 void i915_vma_close(struct i915_vma *vma)
 {
 	GEM_BUG_ON(i915_vma_is_closed(vma));
 	vma->flags |= I915_VMA_CLOSED;
 
-	if (vma->ctx)
-		i915_vma_unlink_ctx(vma);
-
 	list_del(&vma->obj_link);
 	rb_erase(&vma->obj_node, &vma->obj->vma_tree);
 
@@ -680,9 +664,8 @@ int i915_vma_unbind(struct i915_vma *vma)
 		__i915_vma_unpin(vma);
 		if (ret)
 			return ret;
-
-		GEM_BUG_ON(i915_vma_is_active(vma));
 	}
+	GEM_BUG_ON(i915_vma_is_active(vma));
 
 	if (i915_vma_is_pinned(vma))
 		return -EBUSY;
diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h
index 20cf272c97b1..e811067c7724 100644
--- a/drivers/gpu/drm/i915/i915_vma.h
+++ b/drivers/gpu/drm/i915/i915_vma.h
@@ -59,6 +59,12 @@ struct i915_vma {
 	u32 fence_size;
 	u32 fence_alignment;
 
+	/**
+	 * Count of the number of times this vma has been opened by different
+	 * handles (but same file) for execbuf, i.e. the number of aliases
+	 * that exist in the ctx->handle_vmas LUT for this vma.
+	 */
+	unsigned int open_count;
 	unsigned int flags;
 	/**
 	 * How many users have pinned this object in GTT space. The following
@@ -112,13 +118,9 @@ struct i915_vma {
 	/**
 	 * Used for performing relocations during execbuffer insertion.
 	 */
-	struct drm_i915_gem_exec_object2 *exec_entry;
+	unsigned int *exec_flags;
 	struct hlist_node exec_node;
 	u32 exec_handle;
-
-	struct i915_gem_context *ctx;
-	struct hlist_node ctx_node;
-	u32 ctx_handle;
 };
 
 struct i915_vma *
diff --git a/drivers/gpu/drm/i915/intel_atomic_plane.c b/drivers/gpu/drm/i915/intel_atomic_plane.c
index 4325cb0a04f5..ee76fab7bb6f 100644
--- a/drivers/gpu/drm/i915/intel_atomic_plane.c
+++ b/drivers/gpu/drm/i915/intel_atomic_plane.c
@@ -114,6 +114,8 @@ int intel_plane_atomic_check_with_state(struct intel_crtc_state *crtc_state,
 	struct drm_i915_private *dev_priv = to_i915(plane->dev);
 	struct drm_plane_state *state = &intel_state->base;
 	struct intel_plane *intel_plane = to_intel_plane(plane);
+	const struct drm_display_mode *adjusted_mode =
+		&crtc_state->base.adjusted_mode;
 	int ret;
 
 	/*
@@ -173,6 +175,19 @@ int intel_plane_atomic_check_with_state(struct intel_crtc_state *crtc_state,
 	if (ret)
 		return ret;
 
+	/*
+	 * Y-tiling is not supported in IF-ID Interlace mode in
+	 * GEN9 and above.
+	 */
+	if (state->fb && INTEL_GEN(dev_priv) >= 9 && crtc_state->base.enable &&
+	    adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
+		if (state->fb->modifier == I915_FORMAT_MOD_Y_TILED ||
+		    state->fb->modifier == I915_FORMAT_MOD_Yf_TILED) {
+			DRM_DEBUG_KMS("Y/Yf tiling not supported in IF-ID mode\n");
+			return -EINVAL;
+		}
+	}
+
 	/* FIXME pre-g4x don't work like this */
 	if (intel_state->base.visible)
 		crtc_state->active_planes |= BIT(intel_plane->id);
diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c
index 639d45c1dd2e..183e87e8ea31 100644
--- a/drivers/gpu/drm/i915/intel_bios.c
+++ b/drivers/gpu/drm/i915/intel_bios.c
@@ -1120,8 +1120,8 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port,
 	bool is_dvi, is_hdmi, is_dp, is_edp, is_crt;
 	uint8_t aux_channel, ddc_pin;
 	/* Each DDI port can have more than one value on the "DVO Port" field,
-	 * so look for all the possible values for each port and abort if more
-	 * than one is found. */
+	 * so look for all the possible values for each port.
+	 */
 	int dvo_ports[][3] = {
 		{DVO_PORT_HDMIA, DVO_PORT_DPA, -1},
 		{DVO_PORT_HDMIB, DVO_PORT_DPB, -1},
@@ -1130,7 +1130,10 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port,
 		{DVO_PORT_CRT, DVO_PORT_HDMIE, DVO_PORT_DPE},
 	};
 
-	/* Find the child device to use, abort if more than one found. */
+	/*
+	 * Find the first child device to reference the port, report if more
+	 * than one found.
+	 */
 	for (i = 0; i < dev_priv->vbt.child_dev_num; i++) {
 		it = dev_priv->vbt.child_dev + i;
 
@@ -1140,11 +1143,11 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port,
 
 			if (it->common.dvo_port == dvo_ports[port][j]) {
 				if (child) {
-					DRM_DEBUG_KMS("More than one child device for port %c in VBT.\n",
+					DRM_DEBUG_KMS("More than one child device for port %c in VBT, using the first.\n",
 						      port_name(port));
-					return;
+				} else {
+					child = it;
 				}
-				child = it;
 			}
 		}
 	}
@@ -1187,6 +1190,15 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port,
 	if (is_dvi) {
 		info->alternate_ddc_pin = ddc_pin;
 
+		/*
+		 * All VBTs that we got so far for B Stepping has this
+		 * information wrong for Port D. So, let's just ignore for now.
+		 */
+		if (IS_CNL_REVID(dev_priv, CNL_REVID_B0, CNL_REVID_B0) &&
+		    port == PORT_D) {
+			info->alternate_ddc_pin = 0;
+		}
+
 		sanitize_ddc_pin(dev_priv, port);
 	}
 
diff --git a/drivers/gpu/drm/i915/intel_color.c b/drivers/gpu/drm/i915/intel_color.c
index 17c4ae7e4e7c..ff9ecd211abb 100644
--- a/drivers/gpu/drm/i915/intel_color.c
+++ b/drivers/gpu/drm/i915/intel_color.c
@@ -41,6 +41,22 @@
 
 #define LEGACY_LUT_LENGTH		(sizeof(struct drm_color_lut) * 256)
 
+/* Post offset values for RGB->YCBCR conversion */
+#define POSTOFF_RGB_TO_YUV_HI 0x800
+#define POSTOFF_RGB_TO_YUV_ME 0x100
+#define POSTOFF_RGB_TO_YUV_LO 0x800
+
+/*
+ * These values are direct register values specified in the Bspec,
+ * for RGB->YUV conversion matrix (colorspace BT709)
+ */
+#define CSC_RGB_TO_YUV_RU_GU 0x2ba809d8
+#define CSC_RGB_TO_YUV_BU 0x37e80000
+#define CSC_RGB_TO_YUV_RY_GY 0x1e089cc0
+#define CSC_RGB_TO_YUV_BY 0xb5280000
+#define CSC_RGB_TO_YUV_RV_GV 0xbce89ad8
+#define CSC_RGB_TO_YUV_BV 0x1e080000
+
 /*
  * Extract the CSC coefficient from a CTM coefficient (in U32.32 fixed point
  * format). This macro takes the coefficient we want transformed and the
@@ -91,6 +107,30 @@ static void ctm_mult_by_limited(uint64_t *result, int64_t *input)
 	}
 }
 
+static void i9xx_load_ycbcr_conversion_matrix(struct intel_crtc *intel_crtc)
+{
+	int pipe = intel_crtc->pipe;
+	struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
+
+	I915_WRITE(PIPE_CSC_PREOFF_HI(pipe), 0);
+	I915_WRITE(PIPE_CSC_PREOFF_ME(pipe), 0);
+	I915_WRITE(PIPE_CSC_PREOFF_LO(pipe), 0);
+
+	I915_WRITE(PIPE_CSC_COEFF_RU_GU(pipe), CSC_RGB_TO_YUV_RU_GU);
+	I915_WRITE(PIPE_CSC_COEFF_BU(pipe), CSC_RGB_TO_YUV_BU);
+
+	I915_WRITE(PIPE_CSC_COEFF_RY_GY(pipe), CSC_RGB_TO_YUV_RY_GY);
+	I915_WRITE(PIPE_CSC_COEFF_BY(pipe), CSC_RGB_TO_YUV_BY);
+
+	I915_WRITE(PIPE_CSC_COEFF_RV_GV(pipe), CSC_RGB_TO_YUV_RV_GV);
+	I915_WRITE(PIPE_CSC_COEFF_BV(pipe), CSC_RGB_TO_YUV_BV);
+
+	I915_WRITE(PIPE_CSC_POSTOFF_HI(pipe), POSTOFF_RGB_TO_YUV_HI);
+	I915_WRITE(PIPE_CSC_POSTOFF_ME(pipe), POSTOFF_RGB_TO_YUV_ME);
+	I915_WRITE(PIPE_CSC_POSTOFF_LO(pipe), POSTOFF_RGB_TO_YUV_LO);
+	I915_WRITE(PIPE_CSC_MODE(pipe), 0);
+}
+
 /* Set up the pipe CSC unit. */
 static void i9xx_load_csc_matrix(struct drm_crtc_state *crtc_state)
 {
@@ -101,7 +141,10 @@ static void i9xx_load_csc_matrix(struct drm_crtc_state *crtc_state)
 	uint16_t coeffs[9] = { 0, };
 	struct intel_crtc_state *intel_crtc_state = to_intel_crtc_state(crtc_state);
 
-	if (crtc_state->ctm) {
+	if (intel_crtc_state->ycbcr420) {
+		i9xx_load_ycbcr_conversion_matrix(intel_crtc);
+		return;
+	} else if (crtc_state->ctm) {
 		struct drm_color_ctm *ctm =
 			(struct drm_color_ctm *)crtc_state->ctm->data;
 		uint64_t input[9] = { 0, };
@@ -616,7 +659,7 @@ void intel_color_init(struct drm_crtc *crtc)
 		   IS_BROXTON(dev_priv)) {
 		dev_priv->display.load_csc_matrix = i9xx_load_csc_matrix;
 		dev_priv->display.load_luts = broadwell_load_luts;
-	} else if (IS_GEMINILAKE(dev_priv)) {
+	} else if (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) {
 		dev_priv->display.load_csc_matrix = i9xx_load_csc_matrix;
 		dev_priv->display.load_luts = glk_load_luts;
 	} else {
diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c
index 84a1f5e85153..70e0ff41070c 100644
--- a/drivers/gpu/drm/i915/intel_crt.c
+++ b/drivers/gpu/drm/i915/intel_crt.c
@@ -802,12 +802,10 @@ void intel_crt_reset(struct drm_encoder *encoder)
  */
 
 static const struct drm_connector_funcs intel_crt_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.late_register = intel_connector_register,
 	.early_unregister = intel_connector_unregister,
 	.destroy = intel_crt_destroy,
-	.set_property = drm_atomic_helper_connector_set_property,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
 };
diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c
index 9edeaaef77ad..4b4fd1f8110b 100644
--- a/drivers/gpu/drm/i915/intel_ddi.c
+++ b/drivers/gpu/drm/i915/intel_ddi.c
@@ -1103,6 +1103,62 @@ static int skl_calc_wrpll_link(struct drm_i915_private *dev_priv,
 	return dco_freq / (p0 * p1 * p2 * 5);
 }
 
+static int cnl_calc_wrpll_link(struct drm_i915_private *dev_priv,
+			       uint32_t pll_id)
+{
+	uint32_t cfgcr0, cfgcr1;
+	uint32_t p0, p1, p2, dco_freq, ref_clock;
+
+	cfgcr0 = I915_READ(CNL_DPLL_CFGCR0(pll_id));
+	cfgcr1 = I915_READ(CNL_DPLL_CFGCR1(pll_id));
+
+	p0 = cfgcr1 & DPLL_CFGCR1_PDIV_MASK;
+	p2 = cfgcr1 & DPLL_CFGCR1_KDIV_MASK;
+
+	if (cfgcr1 & DPLL_CFGCR1_QDIV_MODE(1))
+		p1 = (cfgcr1 & DPLL_CFGCR1_QDIV_RATIO_MASK) >>
+			DPLL_CFGCR1_QDIV_RATIO_SHIFT;
+	else
+		p1 = 1;
+
+
+	switch (p0) {
+	case DPLL_CFGCR1_PDIV_2:
+		p0 = 2;
+		break;
+	case DPLL_CFGCR1_PDIV_3:
+		p0 = 3;
+		break;
+	case DPLL_CFGCR1_PDIV_5:
+		p0 = 5;
+		break;
+	case DPLL_CFGCR1_PDIV_7:
+		p0 = 7;
+		break;
+	}
+
+	switch (p2) {
+	case DPLL_CFGCR1_KDIV_1:
+		p2 = 1;
+		break;
+	case DPLL_CFGCR1_KDIV_2:
+		p2 = 2;
+		break;
+	case DPLL_CFGCR1_KDIV_4:
+		p2 = 4;
+		break;
+	}
+
+	ref_clock = dev_priv->cdclk.hw.ref;
+
+	dco_freq = (cfgcr0 & DPLL_CFGCR0_DCO_INTEGER_MASK) * ref_clock;
+
+	dco_freq += (((cfgcr0 & DPLL_CFGCR0_DCO_FRACTION_MASK) >>
+		      DPLL_CFGCR0_DCO_FRAC_SHIFT) * ref_clock) / 0x8000;
+
+	return dco_freq / (p0 * p1 * p2 * 5);
+}
+
 static void ddi_dotclock_get(struct intel_crtc_state *pipe_config)
 {
 	int dotclock;
@@ -1118,12 +1174,68 @@ static void ddi_dotclock_get(struct intel_crtc_state *pipe_config)
 	else
 		dotclock = pipe_config->port_clock;
 
+	if (pipe_config->ycbcr420)
+		dotclock *= 2;
+
 	if (pipe_config->pixel_multiplier)
 		dotclock /= pipe_config->pixel_multiplier;
 
 	pipe_config->base.adjusted_mode.crtc_clock = dotclock;
 }
 
+static void cnl_ddi_clock_get(struct intel_encoder *encoder,
+			      struct intel_crtc_state *pipe_config)
+{
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	int link_clock = 0;
+	uint32_t cfgcr0, pll_id;
+
+	pll_id = intel_get_shared_dpll_id(dev_priv, pipe_config->shared_dpll);
+
+	cfgcr0 = I915_READ(CNL_DPLL_CFGCR0(pll_id));
+
+	if (cfgcr0 & DPLL_CFGCR0_HDMI_MODE) {
+		link_clock = cnl_calc_wrpll_link(dev_priv, pll_id);
+	} else {
+		link_clock = cfgcr0 & DPLL_CFGCR0_LINK_RATE_MASK;
+
+		switch (link_clock) {
+		case DPLL_CFGCR0_LINK_RATE_810:
+			link_clock = 81000;
+			break;
+		case DPLL_CFGCR0_LINK_RATE_1080:
+			link_clock = 108000;
+			break;
+		case DPLL_CFGCR0_LINK_RATE_1350:
+			link_clock = 135000;
+			break;
+		case DPLL_CFGCR0_LINK_RATE_1620:
+			link_clock = 162000;
+			break;
+		case DPLL_CFGCR0_LINK_RATE_2160:
+			link_clock = 216000;
+			break;
+		case DPLL_CFGCR0_LINK_RATE_2700:
+			link_clock = 270000;
+			break;
+		case DPLL_CFGCR0_LINK_RATE_3240:
+			link_clock = 324000;
+			break;
+		case DPLL_CFGCR0_LINK_RATE_4050:
+			link_clock = 405000;
+			break;
+		default:
+			WARN(1, "Unsupported link rate\n");
+			break;
+		}
+		link_clock *= 2;
+	}
+
+	pipe_config->port_clock = link_clock;
+
+	ddi_dotclock_get(pipe_config);
+}
+
 static void skl_ddi_clock_get(struct intel_encoder *encoder,
 				struct intel_crtc_state *pipe_config)
 {
@@ -1267,6 +1379,8 @@ void intel_ddi_clock_get(struct intel_encoder *encoder,
 		skl_ddi_clock_get(encoder, pipe_config);
 	else if (IS_GEN9_LP(dev_priv))
 		bxt_ddi_clock_get(encoder, pipe_config);
+	else if (IS_CANNONLAKE(dev_priv))
+		cnl_ddi_clock_get(encoder, pipe_config);
 }
 
 void intel_ddi_set_pipe_settings(const struct intel_crtc_state *crtc_state)
@@ -1762,7 +1876,7 @@ cnl_get_buf_trans_edp(struct drm_i915_private *dev_priv,
 	if (dev_priv->vbt.edp.low_vswing) {
 		if (voltage == VOLTAGE_INFO_0_85V) {
 			*n_entries = ARRAY_SIZE(cnl_ddi_translations_edp_0_85V);
-			return cnl_ddi_translations_dp_0_85V;
+			return cnl_ddi_translations_edp_0_85V;
 		} else if (voltage == VOLTAGE_INFO_0_95V) {
 			*n_entries = ARRAY_SIZE(cnl_ddi_translations_edp_0_95V);
 			return cnl_ddi_translations_edp_0_95V;
@@ -1868,9 +1982,12 @@ static void cnl_ddi_vswing_sequence(struct intel_encoder *encoder, u32 level)
 	if ((intel_dp) && (type == INTEL_OUTPUT_EDP || type == INTEL_OUTPUT_DP)) {
 		width = intel_dp->lane_count;
 		rate = intel_dp->link_rate;
-	} else {
+	} else if (type == INTEL_OUTPUT_HDMI) {
 		width = 4;
 		/* Rate is always < than 6GHz for HDMI */
+	} else {
+		MISSING_CASE(type);
+		return;
 	}
 
 	/*
diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c
index 77d3214e1a77..5f91ddc78c7a 100644
--- a/drivers/gpu/drm/i915/intel_device_info.c
+++ b/drivers/gpu/drm/i915/intel_device_info.c
@@ -363,7 +363,7 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv)
 		 */
 		if (fuse_strap & ILK_INTERNAL_DISPLAY_DISABLE ||
 		    sfuse_strap & SFUSE_STRAP_DISPLAY_DISABLED ||
-		    (dev_priv->pch_type == PCH_CPT &&
+		    (HAS_PCH_CPT(dev_priv) &&
 		     !(sfuse_strap & SFUSE_STRAP_FUSE_LOCK))) {
 			DRM_INFO("Display fused off, disabling\n");
 			info->num_pipes = 0;
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index 9471c88d449e..00cd17c76fdc 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -49,11 +49,6 @@
 #include <linux/dma_remapping.h>
 #include <linux/reservation.h>
 
-static bool is_mmio_work(struct intel_flip_work *work)
-{
-	return work->mmio_work.func;
-}
-
 /* Primary plane formats for gen <= 3 */
 static const uint32_t i8xx_primary_formats[] = {
 	DRM_FORMAT_C8,
@@ -72,6 +67,12 @@ static const uint32_t i965_primary_formats[] = {
 	DRM_FORMAT_XBGR2101010,
 };
 
+static const uint64_t i9xx_format_modifiers[] = {
+	I915_FORMAT_MOD_X_TILED,
+	DRM_FORMAT_MOD_LINEAR,
+	DRM_FORMAT_MOD_INVALID
+};
+
 static const uint32_t skl_primary_formats[] = {
 	DRM_FORMAT_C8,
 	DRM_FORMAT_RGB565,
@@ -87,11 +88,34 @@ static const uint32_t skl_primary_formats[] = {
 	DRM_FORMAT_VYUY,
 };
 
+static const uint64_t skl_format_modifiers_noccs[] = {
+	I915_FORMAT_MOD_Yf_TILED,
+	I915_FORMAT_MOD_Y_TILED,
+	I915_FORMAT_MOD_X_TILED,
+	DRM_FORMAT_MOD_LINEAR,
+	DRM_FORMAT_MOD_INVALID
+};
+
+static const uint64_t skl_format_modifiers_ccs[] = {
+	I915_FORMAT_MOD_Yf_TILED_CCS,
+	I915_FORMAT_MOD_Y_TILED_CCS,
+	I915_FORMAT_MOD_Yf_TILED,
+	I915_FORMAT_MOD_Y_TILED,
+	I915_FORMAT_MOD_X_TILED,
+	DRM_FORMAT_MOD_LINEAR,
+	DRM_FORMAT_MOD_INVALID
+};
+
 /* Cursor formats */
 static const uint32_t intel_cursor_formats[] = {
 	DRM_FORMAT_ARGB8888,
 };
 
+static const uint64_t cursor_format_modifiers[] = {
+	DRM_FORMAT_MOD_LINEAR,
+	DRM_FORMAT_MOD_INVALID
+};
+
 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
 				struct intel_crtc_state *pipe_config);
 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
@@ -1777,7 +1801,7 @@ static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
 
 	/* FDI must be feeding us bits for PCH ports */
 	assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder);
-	assert_fdi_rx_enabled(dev_priv, TRANSCODER_A);
+	assert_fdi_rx_enabled(dev_priv, PIPE_A);
 
 	/* Workaround: set timing override bit. */
 	val = I915_READ(TRANS_CHICKEN2(PIPE_A));
@@ -1853,16 +1877,16 @@ void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
 	I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
 }
 
-enum transcoder intel_crtc_pch_transcoder(struct intel_crtc *crtc)
+enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 
 	WARN_ON(!crtc->config->has_pch_encoder);
 
 	if (HAS_PCH_LPT(dev_priv))
-		return TRANSCODER_A;
+		return PIPE_A;
 	else
-		return (enum transcoder) crtc->pipe;
+		return crtc->pipe;
 }
 
 /**
@@ -1901,7 +1925,7 @@ static void intel_enable_pipe(struct intel_crtc *crtc)
 		if (crtc->config->has_pch_encoder) {
 			/* if driving the PCH, we need FDI enabled */
 			assert_fdi_rx_pll_enabled(dev_priv,
-						  (enum pipe) intel_crtc_pch_transcoder(crtc));
+						  intel_crtc_pch_transcoder(crtc));
 			assert_fdi_tx_pll_enabled(dev_priv,
 						  (enum pipe) cpu_transcoder);
 		}
@@ -1999,11 +2023,19 @@ intel_tile_width_bytes(const struct drm_framebuffer *fb, int plane)
 			return 128;
 		else
 			return 512;
+	case I915_FORMAT_MOD_Y_TILED_CCS:
+		if (plane == 1)
+			return 128;
+		/* fall through */
 	case I915_FORMAT_MOD_Y_TILED:
 		if (IS_GEN2(dev_priv) || HAS_128_BYTE_Y_TILING(dev_priv))
 			return 128;
 		else
 			return 512;
+	case I915_FORMAT_MOD_Yf_TILED_CCS:
+		if (plane == 1)
+			return 128;
+		/* fall through */
 	case I915_FORMAT_MOD_Yf_TILED:
 		switch (cpp) {
 		case 1:
@@ -2110,7 +2142,7 @@ static unsigned int intel_surf_alignment(const struct drm_framebuffer *fb,
 	struct drm_i915_private *dev_priv = to_i915(fb->dev);
 
 	/* AUX_DIST needs only 4K alignment */
-	if (fb->format->format == DRM_FORMAT_NV12 && plane == 1)
+	if (plane == 1)
 		return 4096;
 
 	switch (fb->modifier) {
@@ -2120,6 +2152,8 @@ static unsigned int intel_surf_alignment(const struct drm_framebuffer *fb,
 		if (INTEL_GEN(dev_priv) >= 9)
 			return 256 * 1024;
 		return 0;
+	case I915_FORMAT_MOD_Y_TILED_CCS:
+	case I915_FORMAT_MOD_Yf_TILED_CCS:
 	case I915_FORMAT_MOD_Y_TILED:
 	case I915_FORMAT_MOD_Yf_TILED:
 		return 1 * 1024 * 1024;
@@ -2162,6 +2196,8 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, unsigned int rotation)
 	 */
 	intel_runtime_pm_get(dev_priv);
 
+	atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
+
 	vma = i915_gem_object_pin_to_display_plane(obj, alignment, &view);
 	if (IS_ERR(vma))
 		goto err;
@@ -2189,6 +2225,8 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, unsigned int rotation)
 
 	i915_vma_get(vma);
 err:
+	atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
+
 	intel_runtime_pm_put(dev_priv);
 	return vma;
 }
@@ -2250,17 +2288,13 @@ void intel_add_fb_offsets(int *x, int *y,
 	}
 }
 
-/*
- * Input tile dimensions and pitch must already be
- * rotated to match x and y, and in pixel units.
- */
-static u32 _intel_adjust_tile_offset(int *x, int *y,
-				     unsigned int tile_width,
-				     unsigned int tile_height,
-				     unsigned int tile_size,
-				     unsigned int pitch_tiles,
-				     u32 old_offset,
-				     u32 new_offset)
+static u32 __intel_adjust_tile_offset(int *x, int *y,
+				      unsigned int tile_width,
+				      unsigned int tile_height,
+				      unsigned int tile_size,
+				      unsigned int pitch_tiles,
+				      u32 old_offset,
+				      u32 new_offset)
 {
 	unsigned int pitch_pixels = pitch_tiles * tile_width;
 	unsigned int tiles;
@@ -2281,18 +2315,13 @@ static u32 _intel_adjust_tile_offset(int *x, int *y,
 	return new_offset;
 }
 
-/*
- * Adjust the tile offset by moving the difference into
- * the x/y offsets.
- */
-static u32 intel_adjust_tile_offset(int *x, int *y,
-				    const struct intel_plane_state *state, int plane,
-				    u32 old_offset, u32 new_offset)
+static u32 _intel_adjust_tile_offset(int *x, int *y,
+				     const struct drm_framebuffer *fb, int plane,
+				     unsigned int rotation,
+				     u32 old_offset, u32 new_offset)
 {
-	const struct drm_i915_private *dev_priv = to_i915(state->base.plane->dev);
-	const struct drm_framebuffer *fb = state->base.fb;
+	const struct drm_i915_private *dev_priv = to_i915(fb->dev);
 	unsigned int cpp = fb->format->cpp[plane];
-	unsigned int rotation = state->base.rotation;
 	unsigned int pitch = intel_fb_pitch(fb, plane, rotation);
 
 	WARN_ON(new_offset > old_offset);
@@ -2311,9 +2340,9 @@ static u32 intel_adjust_tile_offset(int *x, int *y,
 			pitch_tiles = pitch / (tile_width * cpp);
 		}
 
-		_intel_adjust_tile_offset(x, y, tile_width, tile_height,
-					  tile_size, pitch_tiles,
-					  old_offset, new_offset);
+		__intel_adjust_tile_offset(x, y, tile_width, tile_height,
+					   tile_size, pitch_tiles,
+					   old_offset, new_offset);
 	} else {
 		old_offset += *y * pitch + *x * cpp;
 
@@ -2325,6 +2354,19 @@ static u32 intel_adjust_tile_offset(int *x, int *y,
 }
 
 /*
+ * Adjust the tile offset by moving the difference into
+ * the x/y offsets.
+ */
+static u32 intel_adjust_tile_offset(int *x, int *y,
+				    const struct intel_plane_state *state, int plane,
+				    u32 old_offset, u32 new_offset)
+{
+	return _intel_adjust_tile_offset(x, y, state->base.fb, plane,
+					 state->base.rotation,
+					 old_offset, new_offset);
+}
+
+/*
  * Computes the linear offset to the base tile and adjusts
  * x, y. bytes per pixel is assumed to be a power-of-two.
  *
@@ -2375,9 +2417,9 @@ static u32 _intel_compute_tile_offset(const struct drm_i915_private *dev_priv,
 		offset = (tile_rows * pitch_tiles + tiles) * tile_size;
 		offset_aligned = offset & ~alignment;
 
-		_intel_adjust_tile_offset(x, y, tile_width, tile_height,
-					  tile_size, pitch_tiles,
-					  offset, offset_aligned);
+		__intel_adjust_tile_offset(x, y, tile_width, tile_height,
+					   tile_size, pitch_tiles,
+					   offset, offset_aligned);
 	} else {
 		offset = *y * pitch + *x * cpp;
 		offset_aligned = offset & ~alignment;
@@ -2409,16 +2451,24 @@ u32 intel_compute_tile_offset(int *x, int *y,
 					  rotation, alignment);
 }
 
-/* Convert the fb->offset[] linear offset into x/y offsets */
-static void intel_fb_offset_to_xy(int *x, int *y,
-				  const struct drm_framebuffer *fb, int plane)
+/* Convert the fb->offset[] into x/y offsets */
+static int intel_fb_offset_to_xy(int *x, int *y,
+				 const struct drm_framebuffer *fb, int plane)
 {
-	unsigned int cpp = fb->format->cpp[plane];
-	unsigned int pitch = fb->pitches[plane];
-	u32 linear_offset = fb->offsets[plane];
+	struct drm_i915_private *dev_priv = to_i915(fb->dev);
+
+	if (fb->modifier != DRM_FORMAT_MOD_LINEAR &&
+	    fb->offsets[plane] % intel_tile_size(dev_priv))
+		return -EINVAL;
 
-	*y = linear_offset / pitch;
-	*x = linear_offset % pitch / cpp;
+	*x = 0;
+	*y = 0;
+
+	_intel_adjust_tile_offset(x, y,
+				  fb, plane, DRM_MODE_ROTATE_0,
+				  fb->offsets[plane], 0);
+
+	return 0;
 }
 
 static unsigned int intel_fb_modifier_to_tiling(uint64_t fb_modifier)
@@ -2427,12 +2477,48 @@ static unsigned int intel_fb_modifier_to_tiling(uint64_t fb_modifier)
 	case I915_FORMAT_MOD_X_TILED:
 		return I915_TILING_X;
 	case I915_FORMAT_MOD_Y_TILED:
+	case I915_FORMAT_MOD_Y_TILED_CCS:
 		return I915_TILING_Y;
 	default:
 		return I915_TILING_NONE;
 	}
 }
 
+static const struct drm_format_info ccs_formats[] = {
+	{ .format = DRM_FORMAT_XRGB8888, .depth = 24, .num_planes = 2, .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
+	{ .format = DRM_FORMAT_XBGR8888, .depth = 24, .num_planes = 2, .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
+	{ .format = DRM_FORMAT_ARGB8888, .depth = 32, .num_planes = 2, .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
+	{ .format = DRM_FORMAT_ABGR8888, .depth = 32, .num_planes = 2, .cpp = { 4, 1, }, .hsub = 8, .vsub = 16, },
+};
+
+static const struct drm_format_info *
+lookup_format_info(const struct drm_format_info formats[],
+		   int num_formats, u32 format)
+{
+	int i;
+
+	for (i = 0; i < num_formats; i++) {
+		if (formats[i].format == format)
+			return &formats[i];
+	}
+
+	return NULL;
+}
+
+static const struct drm_format_info *
+intel_get_format_info(const struct drm_mode_fb_cmd2 *cmd)
+{
+	switch (cmd->modifier[0]) {
+	case I915_FORMAT_MOD_Y_TILED_CCS:
+	case I915_FORMAT_MOD_Yf_TILED_CCS:
+		return lookup_format_info(ccs_formats,
+					  ARRAY_SIZE(ccs_formats),
+					  cmd->pixel_format);
+	default:
+		return NULL;
+	}
+}
+
 static int
 intel_fill_fb_info(struct drm_i915_private *dev_priv,
 		   struct drm_framebuffer *fb)
@@ -2449,12 +2535,50 @@ intel_fill_fb_info(struct drm_i915_private *dev_priv,
 		unsigned int cpp, size;
 		u32 offset;
 		int x, y;
+		int ret;
 
 		cpp = fb->format->cpp[i];
 		width = drm_framebuffer_plane_width(fb->width, fb, i);
 		height = drm_framebuffer_plane_height(fb->height, fb, i);
 
-		intel_fb_offset_to_xy(&x, &y, fb, i);
+		ret = intel_fb_offset_to_xy(&x, &y, fb, i);
+		if (ret) {
+			DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
+				      i, fb->offsets[i]);
+			return ret;
+		}
+
+		if ((fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
+		     fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS) && i == 1) {
+			int hsub = fb->format->hsub;
+			int vsub = fb->format->vsub;
+			int tile_width, tile_height;
+			int main_x, main_y;
+			int ccs_x, ccs_y;
+
+			intel_tile_dims(fb, i, &tile_width, &tile_height);
+			tile_width *= hsub;
+			tile_height *= vsub;
+
+			ccs_x = (x * hsub) % tile_width;
+			ccs_y = (y * vsub) % tile_height;
+			main_x = intel_fb->normal[0].x % tile_width;
+			main_y = intel_fb->normal[0].y % tile_height;
+
+			/*
+			 * CCS doesn't have its own x/y offset register, so the intra CCS tile
+			 * x/y offsets must match between CCS and the main surface.
+			 */
+			if (main_x != ccs_x || main_y != ccs_y) {
+				DRM_DEBUG_KMS("Bad CCS x/y (main %d,%d ccs %d,%d) full (main %d,%d ccs %d,%d)\n",
+					      main_x, main_y,
+					      ccs_x, ccs_y,
+					      intel_fb->normal[0].x,
+					      intel_fb->normal[0].y,
+					      x, y);
+				return -EINVAL;
+			}
+		}
 
 		/*
 		 * The fence (if used) is aligned to the start of the object
@@ -2465,7 +2589,7 @@ intel_fill_fb_info(struct drm_i915_private *dev_priv,
 		 * fb layout agrees with the fence layout. We already check that the
 		 * fb stride matches the fence stride elsewhere.
 		 */
-		if (i915_gem_object_is_tiled(intel_fb->obj) &&
+		if (i == 0 && i915_gem_object_is_tiled(intel_fb->obj) &&
 		    (x + width) * cpp > fb->pitches[i]) {
 			DRM_DEBUG_KMS("bad fb plane %d offset: 0x%x\n",
 				      i, fb->offsets[i]);
@@ -2528,10 +2652,10 @@ intel_fill_fb_info(struct drm_i915_private *dev_priv,
 			 * We only keep the x/y offsets, so push all of the
 			 * gtt offset into the x/y offsets.
 			 */
-			_intel_adjust_tile_offset(&x, &y,
-						  tile_width, tile_height,
-						  tile_size, pitch_tiles,
-						  gtt_offset_rotated * tile_size, 0);
+			__intel_adjust_tile_offset(&x, &y,
+						   tile_width, tile_height,
+						   tile_size, pitch_tiles,
+						   gtt_offset_rotated * tile_size, 0);
 
 			gtt_offset_rotated += rot_info->plane[i].width * rot_info->plane[i].height;
 
@@ -2664,20 +2788,6 @@ out_unref_obj:
 	return false;
 }
 
-/* Update plane->state->fb to match plane->fb after driver-internal updates */
-static void
-update_state_fb(struct drm_plane *plane)
-{
-	if (plane->fb == plane->state->fb)
-		return;
-
-	if (plane->state->fb)
-		drm_framebuffer_unreference(plane->state->fb);
-	plane->state->fb = plane->fb;
-	if (plane->state->fb)
-		drm_framebuffer_reference(plane->state->fb);
-}
-
 static void
 intel_set_plane_visible(struct intel_crtc_state *crtc_state,
 			struct intel_plane_state *plane_state,
@@ -2830,6 +2940,9 @@ static int skl_max_plane_width(const struct drm_framebuffer *fb, int plane,
 			break;
 		}
 		break;
+	case I915_FORMAT_MOD_Y_TILED_CCS:
+	case I915_FORMAT_MOD_Yf_TILED_CCS:
+		/* FIXME AUX plane? */
 	case I915_FORMAT_MOD_Y_TILED:
 	case I915_FORMAT_MOD_Yf_TILED:
 		switch (cpp) {
@@ -2852,6 +2965,44 @@ static int skl_max_plane_width(const struct drm_framebuffer *fb, int plane,
 	return 2048;
 }
 
+static bool skl_check_main_ccs_coordinates(struct intel_plane_state *plane_state,
+					   int main_x, int main_y, u32 main_offset)
+{
+	const struct drm_framebuffer *fb = plane_state->base.fb;
+	int hsub = fb->format->hsub;
+	int vsub = fb->format->vsub;
+	int aux_x = plane_state->aux.x;
+	int aux_y = plane_state->aux.y;
+	u32 aux_offset = plane_state->aux.offset;
+	u32 alignment = intel_surf_alignment(fb, 1);
+
+	while (aux_offset >= main_offset && aux_y <= main_y) {
+		int x, y;
+
+		if (aux_x == main_x && aux_y == main_y)
+			break;
+
+		if (aux_offset == 0)
+			break;
+
+		x = aux_x / hsub;
+		y = aux_y / vsub;
+		aux_offset = intel_adjust_tile_offset(&x, &y, plane_state, 1,
+						      aux_offset, aux_offset - alignment);
+		aux_x = x * hsub + aux_x % hsub;
+		aux_y = y * vsub + aux_y % vsub;
+	}
+
+	if (aux_x != main_x || aux_y != main_y)
+		return false;
+
+	plane_state->aux.offset = aux_offset;
+	plane_state->aux.x = aux_x;
+	plane_state->aux.y = aux_y;
+
+	return true;
+}
+
 static int skl_check_main_surface(struct intel_plane_state *plane_state)
 {
 	const struct drm_framebuffer *fb = plane_state->base.fb;
@@ -2894,7 +3045,7 @@ static int skl_check_main_surface(struct intel_plane_state *plane_state)
 
 		while ((x + w) * cpp > fb->pitches[0]) {
 			if (offset == 0) {
-				DRM_DEBUG_KMS("Unable to find suitable display surface offset\n");
+				DRM_DEBUG_KMS("Unable to find suitable display surface offset due to X-tiling\n");
 				return -EINVAL;
 			}
 
@@ -2903,6 +3054,26 @@ static int skl_check_main_surface(struct intel_plane_state *plane_state)
 		}
 	}
 
+	/*
+	 * CCS AUX surface doesn't have its own x/y offsets, we must make sure
+	 * they match with the main surface x/y offsets.
+	 */
+	if (fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
+	    fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS) {
+		while (!skl_check_main_ccs_coordinates(plane_state, x, y, offset)) {
+			if (offset == 0)
+				break;
+
+			offset = intel_adjust_tile_offset(&x, &y, plane_state, 0,
+							  offset, offset - alignment);
+		}
+
+		if (x != plane_state->aux.x || y != plane_state->aux.y) {
+			DRM_DEBUG_KMS("Unable to find suitable display surface offset due to CCS\n");
+			return -EINVAL;
+		}
+	}
+
 	plane_state->main.offset = offset;
 	plane_state->main.x = x;
 	plane_state->main.y = y;
@@ -2939,6 +3110,49 @@ static int skl_check_nv12_aux_surface(struct intel_plane_state *plane_state)
 	return 0;
 }
 
+static int skl_check_ccs_aux_surface(struct intel_plane_state *plane_state)
+{
+	struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
+	struct intel_crtc *crtc = to_intel_crtc(plane_state->base.crtc);
+	const struct drm_framebuffer *fb = plane_state->base.fb;
+	int src_x = plane_state->base.src.x1 >> 16;
+	int src_y = plane_state->base.src.y1 >> 16;
+	int hsub = fb->format->hsub;
+	int vsub = fb->format->vsub;
+	int x = src_x / hsub;
+	int y = src_y / vsub;
+	u32 offset;
+
+	switch (plane->id) {
+	case PLANE_PRIMARY:
+	case PLANE_SPRITE0:
+		break;
+	default:
+		DRM_DEBUG_KMS("RC support only on plane 1 and 2\n");
+		return -EINVAL;
+	}
+
+	if (crtc->pipe == PIPE_C) {
+		DRM_DEBUG_KMS("No RC support on pipe C\n");
+		return -EINVAL;
+	}
+
+	if (plane_state->base.rotation & ~(DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_180)) {
+		DRM_DEBUG_KMS("RC support only with 0/180 degree rotation %x\n",
+			      plane_state->base.rotation);
+		return -EINVAL;
+	}
+
+	intel_add_fb_offsets(&x, &y, plane_state, 1);
+	offset = intel_compute_tile_offset(&x, &y, plane_state, 1);
+
+	plane_state->aux.offset = offset;
+	plane_state->aux.x = x * hsub + src_x % hsub;
+	plane_state->aux.y = y * vsub + src_y % vsub;
+
+	return 0;
+}
+
 int skl_check_plane_surface(struct intel_plane_state *plane_state)
 {
 	const struct drm_framebuffer *fb = plane_state->base.fb;
@@ -2962,6 +3176,11 @@ int skl_check_plane_surface(struct intel_plane_state *plane_state)
 		ret = skl_check_nv12_aux_surface(plane_state);
 		if (ret)
 			return ret;
+	} else if (fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
+		   fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS) {
+		ret = skl_check_ccs_aux_surface(plane_state);
+		if (ret)
+			return ret;
 	} else {
 		plane_state->aux.offset = ~0xfff;
 		plane_state->aux.x = 0;
@@ -3268,8 +3487,12 @@ static u32 skl_plane_ctl_tiling(uint64_t fb_modifier)
 		return PLANE_CTL_TILED_X;
 	case I915_FORMAT_MOD_Y_TILED:
 		return PLANE_CTL_TILED_Y;
+	case I915_FORMAT_MOD_Y_TILED_CCS:
+		return PLANE_CTL_TILED_Y | PLANE_CTL_DECOMPRESSION_ENABLE;
 	case I915_FORMAT_MOD_Yf_TILED:
 		return PLANE_CTL_TILED_YF;
+	case I915_FORMAT_MOD_Yf_TILED_CCS:
+		return PLANE_CTL_TILED_YF | PLANE_CTL_DECOMPRESSION_ENABLE;
 	default:
 		MISSING_CASE(fb_modifier);
 	}
@@ -3311,7 +3534,7 @@ u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state,
 
 	plane_ctl = PLANE_CTL_ENABLE;
 
-	if (!IS_GEMINILAKE(dev_priv)) {
+	if (!IS_GEMINILAKE(dev_priv) && !IS_CANNONLAKE(dev_priv)) {
 		plane_ctl |=
 			PLANE_CTL_PIPE_GAMMA_ENABLE |
 			PLANE_CTL_PIPE_CSC_ENABLE |
@@ -3342,6 +3565,7 @@ static void skylake_update_primary_plane(struct intel_plane *plane,
 	u32 plane_ctl = plane_state->ctl;
 	unsigned int rotation = plane_state->base.rotation;
 	u32 stride = skl_plane_stride(fb, 0, rotation);
+	u32 aux_stride = skl_plane_stride(fb, 1, rotation);
 	u32 surf_addr = plane_state->main.offset;
 	int scaler_id = plane_state->scaler_id;
 	int src_x = plane_state->main.x;
@@ -3367,7 +3591,7 @@ static void skylake_update_primary_plane(struct intel_plane *plane,
 
 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 
-	if (IS_GEMINILAKE(dev_priv)) {
+	if (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) {
 		I915_WRITE_FW(PLANE_COLOR_CTL(pipe, plane_id),
 			      PLANE_COLOR_PIPE_GAMMA_ENABLE |
 			      PLANE_COLOR_PIPE_CSC_ENABLE |
@@ -3378,6 +3602,10 @@ static void skylake_update_primary_plane(struct intel_plane *plane,
 	I915_WRITE_FW(PLANE_OFFSET(pipe, plane_id), (src_y << 16) | src_x);
 	I915_WRITE_FW(PLANE_STRIDE(pipe, plane_id), stride);
 	I915_WRITE_FW(PLANE_SIZE(pipe, plane_id), (src_h << 16) | src_w);
+	I915_WRITE_FW(PLANE_AUX_DIST(pipe, plane_id),
+		      (plane_state->aux.offset - surf_addr) | aux_stride);
+	I915_WRITE_FW(PLANE_AUX_OFFSET(pipe, plane_id),
+		      (plane_state->aux.y << 16) | plane_state->aux.x);
 
 	if (scaler_id >= 0) {
 		uint32_t ps_ctrl = 0;
@@ -3419,14 +3647,6 @@ static void skylake_disable_primary_plane(struct intel_plane *primary,
 	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 }
 
-static void intel_complete_page_flips(struct drm_i915_private *dev_priv)
-{
-	struct intel_crtc *crtc;
-
-	for_each_intel_crtc(&dev_priv->drm, crtc)
-		intel_finish_page_flip_cs(dev_priv, crtc->pipe);
-}
-
 static int
 __intel_display_resume(struct drm_device *dev,
 		       struct drm_atomic_state *state,
@@ -3485,6 +3705,15 @@ void intel_prepare_reset(struct drm_i915_private *dev_priv)
 	    !gpu_reset_clobbers_display(dev_priv))
 		return;
 
+	/* We have a modeset vs reset deadlock, defensively unbreak it. */
+	set_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags);
+	wake_up_all(&dev_priv->gpu_error.wait_queue);
+
+	if (atomic_read(&dev_priv->gpu_error.pending_fb_pin)) {
+		DRM_DEBUG_KMS("Modeset potentially stuck, unbreaking through wedging\n");
+		i915_gem_set_wedged(dev_priv);
+	}
+
 	/*
 	 * Need mode_config.mutex so that we don't
 	 * trample ongoing ->detect() and whatnot.
@@ -3535,13 +3764,6 @@ void intel_finish_reset(struct drm_i915_private *dev_priv)
 	if (!state)
 		goto unlock;
 
-	/*
-	 * Flips in the rings will be nuked by the reset,
-	 * so complete all pending flips so that user space
-	 * will get its events and not get stuck.
-	 */
-	intel_complete_page_flips(dev_priv);
-
 	dev_priv->modeset_restore_state = NULL;
 
 	/* reset doesn't touch the display */
@@ -3578,35 +3800,8 @@ unlock:
 	drm_modeset_drop_locks(ctx);
 	drm_modeset_acquire_fini(ctx);
 	mutex_unlock(&dev->mode_config.mutex);
-}
 
-static bool abort_flip_on_reset(struct intel_crtc *crtc)
-{
-	struct i915_gpu_error *error = &to_i915(crtc->base.dev)->gpu_error;
-
-	if (i915_reset_backoff(error))
-		return true;
-
-	if (crtc->reset_count != i915_reset_count(error))
-		return true;
-
-	return false;
-}
-
-static bool intel_crtc_has_pending_flip(struct drm_crtc *crtc)
-{
-	struct drm_device *dev = crtc->dev;
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	bool pending;
-
-	if (abort_flip_on_reset(intel_crtc))
-		return false;
-
-	spin_lock_irq(&dev->event_lock);
-	pending = to_intel_crtc(crtc)->flip_work != NULL;
-	spin_unlock_irq(&dev->event_lock);
-
-	return pending;
+	clear_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags);
 }
 
 static void intel_update_pipe_config(struct intel_crtc *crtc,
@@ -4163,21 +4358,22 @@ static void ironlake_fdi_disable(struct drm_crtc *crtc)
 
 bool intel_has_pending_fb_unpin(struct drm_i915_private *dev_priv)
 {
-	struct intel_crtc *crtc;
-
-	/* Note that we don't need to be called with mode_config.lock here
-	 * as our list of CRTC objects is static for the lifetime of the
-	 * device and so cannot disappear as we iterate. Similarly, we can
-	 * happily treat the predicates as racy, atomic checks as userspace
-	 * cannot claim and pin a new fb without at least acquring the
-	 * struct_mutex and so serialising with us.
-	 */
-	for_each_intel_crtc(&dev_priv->drm, crtc) {
-		if (atomic_read(&crtc->unpin_work_count) == 0)
+	struct drm_crtc *crtc;
+	bool cleanup_done;
+
+	drm_for_each_crtc(crtc, &dev_priv->drm) {
+		struct drm_crtc_commit *commit;
+		spin_lock(&crtc->commit_lock);
+		commit = list_first_entry_or_null(&crtc->commit_list,
+						  struct drm_crtc_commit, commit_entry);
+		cleanup_done = commit ?
+			try_wait_for_completion(&commit->cleanup_done) : true;
+		spin_unlock(&crtc->commit_lock);
+
+		if (cleanup_done)
 			continue;
 
-		if (crtc->flip_work)
-			intel_wait_for_vblank(dev_priv, crtc->pipe);
+		drm_crtc_wait_one_vblank(crtc);
 
 		return true;
 	}
@@ -4185,57 +4381,6 @@ bool intel_has_pending_fb_unpin(struct drm_i915_private *dev_priv)
 	return false;
 }
 
-static void page_flip_completed(struct intel_crtc *intel_crtc)
-{
-	struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
-	struct intel_flip_work *work = intel_crtc->flip_work;
-
-	intel_crtc->flip_work = NULL;
-
-	if (work->event)
-		drm_crtc_send_vblank_event(&intel_crtc->base, work->event);
-
-	drm_crtc_vblank_put(&intel_crtc->base);
-
-	wake_up_all(&dev_priv->pending_flip_queue);
-	trace_i915_flip_complete(intel_crtc->plane,
-				 work->pending_flip_obj);
-
-	queue_work(dev_priv->wq, &work->unpin_work);
-}
-
-static int intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc)
-{
-	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	long ret;
-
-	WARN_ON(waitqueue_active(&dev_priv->pending_flip_queue));
-
-	ret = wait_event_interruptible_timeout(
-					dev_priv->pending_flip_queue,
-					!intel_crtc_has_pending_flip(crtc),
-					60*HZ);
-
-	if (ret < 0)
-		return ret;
-
-	if (ret == 0) {
-		struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-		struct intel_flip_work *work;
-
-		spin_lock_irq(&dev->event_lock);
-		work = intel_crtc->flip_work;
-		if (work && !is_mmio_work(work)) {
-			WARN_ONCE(1, "Removing stuck page flip\n");
-			page_flip_completed(intel_crtc);
-		}
-		spin_unlock_irq(&dev->event_lock);
-	}
-
-	return 0;
-}
-
 void lpt_disable_iclkip(struct drm_i915_private *dev_priv)
 {
 	u32 temp;
@@ -4555,7 +4700,7 @@ static void lpt_pch_enable(const struct intel_crtc_state *crtc_state)
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
 
-	assert_pch_transcoder_disabled(dev_priv, TRANSCODER_A);
+	assert_pch_transcoder_disabled(dev_priv, PIPE_A);
 
 	lpt_program_iclkip(crtc);
 
@@ -4588,6 +4733,9 @@ skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
 		&crtc_state->scaler_state;
 	struct intel_crtc *intel_crtc =
 		to_intel_crtc(crtc_state->base.crtc);
+	struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
+	const struct drm_display_mode *adjusted_mode =
+		&crtc_state->base.adjusted_mode;
 	int need_scaling;
 
 	/*
@@ -4597,6 +4745,21 @@ skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
 	 */
 	need_scaling = src_w != dst_w || src_h != dst_h;
 
+	if (crtc_state->ycbcr420 && scaler_user == SKL_CRTC_INDEX)
+		need_scaling = true;
+
+	/*
+	 * Scaling/fitting not supported in IF-ID mode in GEN9+
+	 * TODO: Interlace fetch mode doesn't support YUV420 planar formats.
+	 * Once NV12 is enabled, handle it here while allocating scaler
+	 * for NV12.
+	 */
+	if (INTEL_GEN(dev_priv) >= 9 && crtc_state->base.enable &&
+	    need_scaling && adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
+		DRM_DEBUG_KMS("Pipe/Plane scaling not supported with IF-ID mode\n");
+		return -EINVAL;
+	}
+
 	/*
 	 * if plane is being disabled or scaler is no more required or force detach
 	 *  - free scaler binded to this plane/crtc
@@ -5308,8 +5471,7 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
 		return;
 
 	if (intel_crtc->config->has_pch_encoder)
-		intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
-						      false);
+		intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, false);
 
 	intel_encoders_pre_pll_enable(crtc, pipe_config, old_state);
 
@@ -5394,8 +5556,7 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
 		intel_wait_for_vblank(dev_priv, pipe);
 		intel_wait_for_vblank(dev_priv, pipe);
 		intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
-		intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
-						      true);
+		intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true);
 	}
 
 	/* If we change the relative order between pipe/planes enabling, we need
@@ -5492,8 +5653,7 @@ static void haswell_crtc_disable(struct intel_crtc_state *old_crtc_state,
 	enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
 
 	if (intel_crtc->config->has_pch_encoder)
-		intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
-						      false);
+		intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, false);
 
 	intel_encoders_disable(crtc, old_crtc_state, old_state);
 
@@ -5521,8 +5681,7 @@ static void haswell_crtc_disable(struct intel_crtc_state *old_crtc_state,
 	intel_encoders_post_disable(crtc, old_crtc_state, old_state);
 
 	if (old_crtc_state->has_pch_encoder)
-		intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
-						      true);
+		intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true);
 }
 
 static void i9xx_pfit_enable(struct intel_crtc *crtc)
@@ -5831,8 +5990,6 @@ static void intel_crtc_disable_noatomic(struct drm_crtc *crtc,
 		return;
 
 	if (crtc->primary->state->visible) {
-		WARN_ON(intel_crtc->flip_work);
-
 		intel_pre_disable_primary_noatomic(crtc);
 
 		intel_crtc_disable_planes(crtc, 1 << drm_plane_index(crtc->primary));
@@ -6241,6 +6398,16 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc,
 		return -EINVAL;
 	}
 
+	if (pipe_config->ycbcr420 && pipe_config->base.ctm) {
+		/*
+		 * There is only one pipe CSC unit per pipe, and we need that
+		 * for output conversion from RGB->YCBCR. So if CTM is already
+		 * applied we can't support YCBCR420 output.
+		 */
+		DRM_DEBUG_KMS("YCBCR420 and CTM together are not possible\n");
+		return -EINVAL;
+	}
+
 	/*
 	 * Pipe horizontal size must be even in:
 	 * - DVO ganged mode
@@ -8034,6 +8201,7 @@ static void haswell_set_pipemisc(struct drm_crtc *crtc)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct intel_crtc_state *config = intel_crtc->config;
 
 	if (IS_BROADWELL(dev_priv) || INTEL_INFO(dev_priv)->gen >= 9) {
 		u32 val = 0;
@@ -8059,6 +8227,12 @@ static void haswell_set_pipemisc(struct drm_crtc *crtc)
 		if (intel_crtc->config->dither)
 			val |= PIPEMISC_DITHER_ENABLE | PIPEMISC_DITHER_TYPE_SP;
 
+		if (config->ycbcr420) {
+			val |= PIPEMISC_OUTPUT_COLORSPACE_YUV |
+				PIPEMISC_YUV420_ENABLE |
+				PIPEMISC_YUV420_MODE_FULL_BLEND;
+		}
+
 		I915_WRITE(PIPEMISC(intel_crtc->pipe), val);
 	}
 }
@@ -8386,10 +8560,16 @@ skylake_get_initial_plane_config(struct intel_crtc *crtc,
 		fb->modifier = I915_FORMAT_MOD_X_TILED;
 		break;
 	case PLANE_CTL_TILED_Y:
-		fb->modifier = I915_FORMAT_MOD_Y_TILED;
+		if (val & PLANE_CTL_DECOMPRESSION_ENABLE)
+			fb->modifier = I915_FORMAT_MOD_Y_TILED_CCS;
+		else
+			fb->modifier = I915_FORMAT_MOD_Y_TILED;
 		break;
 	case PLANE_CTL_TILED_YF:
-		fb->modifier = I915_FORMAT_MOD_Yf_TILED;
+		if (val & PLANE_CTL_DECOMPRESSION_ENABLE)
+			fb->modifier = I915_FORMAT_MOD_Yf_TILED_CCS;
+		else
+			fb->modifier = I915_FORMAT_MOD_Yf_TILED;
 		break;
 	default:
 		MISSING_CASE(tiling);
@@ -8623,7 +8803,8 @@ static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
 		I915_STATE_WARN(crtc->active, "CRTC for pipe %c enabled\n",
 		     pipe_name(crtc->pipe));
 
-	I915_STATE_WARN(I915_READ(HSW_PWR_WELL_DRIVER), "Power well on\n");
+	I915_STATE_WARN(I915_READ(HSW_PWR_WELL_CTL_DRIVER(HSW_DISP_PW_GLOBAL)),
+			"Display power well on\n");
 	I915_STATE_WARN(I915_READ(SPLL_CTL) & SPLL_PLL_ENABLE, "SPLL enabled\n");
 	I915_STATE_WARN(I915_READ(WRPLL_CTL(0)) & WRPLL_PLL_ENABLE, "WRPLL1 enabled\n");
 	I915_STATE_WARN(I915_READ(WRPLL_CTL(1)) & WRPLL_PLL_ENABLE, "WRPLL2 enabled\n");
@@ -9093,12 +9274,7 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc,
 	u64 power_domain_mask;
 	bool active;
 
-	if (INTEL_GEN(dev_priv) >= 9) {
-		intel_crtc_init_scalers(crtc, pipe_config);
-
-		pipe_config->scaler_state.scaler_id = -1;
-		pipe_config->scaler_state.scaler_users &= ~(1 << SKL_CRTC_INDEX);
-	}
+	intel_crtc_init_scalers(crtc, pipe_config);
 
 	power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
 	if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
@@ -9128,6 +9304,23 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc,
 	pipe_config->gamma_mode =
 		I915_READ(GAMMA_MODE(crtc->pipe)) & GAMMA_MODE_MODE_MASK;
 
+	if (IS_BROADWELL(dev_priv) || dev_priv->info.gen >= 9) {
+		u32 tmp = I915_READ(PIPEMISC(crtc->pipe));
+		bool clrspace_yuv = tmp & PIPEMISC_OUTPUT_COLORSPACE_YUV;
+
+		if (IS_GEMINILAKE(dev_priv) || dev_priv->info.gen >= 10) {
+			bool blend_mode_420 = tmp &
+					      PIPEMISC_YUV420_MODE_FULL_BLEND;
+
+			pipe_config->ycbcr420 = tmp & PIPEMISC_YUV420_ENABLE;
+			if (pipe_config->ycbcr420 != clrspace_yuv ||
+			    pipe_config->ycbcr420 != blend_mode_420)
+				DRM_DEBUG_KMS("Bad 4:2:0 mode (%08x)\n", tmp);
+		} else if (clrspace_yuv) {
+			DRM_DEBUG_KMS("YCbCr 4:2:0 Unsupported\n");
+		}
+	}
+
 	power_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe);
 	if (intel_display_power_get_if_enabled(dev_priv, power_domain)) {
 		power_domain_mask |= BIT_ULL(power_domain);
@@ -10105,849 +10298,11 @@ struct drm_display_mode *intel_crtc_mode_get(struct drm_device *dev,
 static void intel_crtc_destroy(struct drm_crtc *crtc)
 {
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	struct drm_device *dev = crtc->dev;
-	struct intel_flip_work *work;
-
-	spin_lock_irq(&dev->event_lock);
-	work = intel_crtc->flip_work;
-	intel_crtc->flip_work = NULL;
-	spin_unlock_irq(&dev->event_lock);
-
-	if (work) {
-		cancel_work_sync(&work->mmio_work);
-		cancel_work_sync(&work->unpin_work);
-		kfree(work);
-	}
 
 	drm_crtc_cleanup(crtc);
-
 	kfree(intel_crtc);
 }
 
-static void intel_unpin_work_fn(struct work_struct *__work)
-{
-	struct intel_flip_work *work =
-		container_of(__work, struct intel_flip_work, unpin_work);
-	struct intel_crtc *crtc = to_intel_crtc(work->crtc);
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_plane *primary = crtc->base.primary;
-
-	if (is_mmio_work(work))
-		flush_work(&work->mmio_work);
-
-	mutex_lock(&dev->struct_mutex);
-	intel_unpin_fb_vma(work->old_vma);
-	i915_gem_object_put(work->pending_flip_obj);
-	mutex_unlock(&dev->struct_mutex);
-
-	i915_gem_request_put(work->flip_queued_req);
-
-	intel_frontbuffer_flip_complete(to_i915(dev),
-					to_intel_plane(primary)->frontbuffer_bit);
-	intel_fbc_post_update(crtc);
-	drm_framebuffer_unreference(work->old_fb);
-
-	BUG_ON(atomic_read(&crtc->unpin_work_count) == 0);
-	atomic_dec(&crtc->unpin_work_count);
-
-	kfree(work);
-}
-
-/* Is 'a' after or equal to 'b'? */
-static bool g4x_flip_count_after_eq(u32 a, u32 b)
-{
-	return !((a - b) & 0x80000000);
-}
-
-static bool __pageflip_finished_cs(struct intel_crtc *crtc,
-				   struct intel_flip_work *work)
-{
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
-	if (abort_flip_on_reset(crtc))
-		return true;
-
-	/*
-	 * The relevant registers doen't exist on pre-ctg.
-	 * As the flip done interrupt doesn't trigger for mmio
-	 * flips on gmch platforms, a flip count check isn't
-	 * really needed there. But since ctg has the registers,
-	 * include it in the check anyway.
-	 */
-	if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
-		return true;
-
-	/*
-	 * BDW signals flip done immediately if the plane
-	 * is disabled, even if the plane enable is already
-	 * armed to occur at the next vblank :(
-	 */
-
-	/*
-	 * A DSPSURFLIVE check isn't enough in case the mmio and CS flips
-	 * used the same base address. In that case the mmio flip might
-	 * have completed, but the CS hasn't even executed the flip yet.
-	 *
-	 * A flip count check isn't enough as the CS might have updated
-	 * the base address just after start of vblank, but before we
-	 * managed to process the interrupt. This means we'd complete the
-	 * CS flip too soon.
-	 *
-	 * Combining both checks should get us a good enough result. It may
-	 * still happen that the CS flip has been executed, but has not
-	 * yet actually completed. But in case the base address is the same
-	 * anyway, we don't really care.
-	 */
-	return (I915_READ(DSPSURFLIVE(crtc->plane)) & ~0xfff) ==
-		crtc->flip_work->gtt_offset &&
-		g4x_flip_count_after_eq(I915_READ(PIPE_FLIPCOUNT_G4X(crtc->pipe)),
-				    crtc->flip_work->flip_count);
-}
-
-static bool
-__pageflip_finished_mmio(struct intel_crtc *crtc,
-			       struct intel_flip_work *work)
-{
-	/*
-	 * MMIO work completes when vblank is different from
-	 * flip_queued_vblank.
-	 *
-	 * Reset counter value doesn't matter, this is handled by
-	 * i915_wait_request finishing early, so no need to handle
-	 * reset here.
-	 */
-	return intel_crtc_get_vblank_counter(crtc) != work->flip_queued_vblank;
-}
-
-
-static bool pageflip_finished(struct intel_crtc *crtc,
-			      struct intel_flip_work *work)
-{
-	if (!atomic_read(&work->pending))
-		return false;
-
-	smp_rmb();
-
-	if (is_mmio_work(work))
-		return __pageflip_finished_mmio(crtc, work);
-	else
-		return __pageflip_finished_cs(crtc, work);
-}
-
-void intel_finish_page_flip_cs(struct drm_i915_private *dev_priv, int pipe)
-{
-	struct drm_device *dev = &dev_priv->drm;
-	struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
-	struct intel_flip_work *work;
-	unsigned long flags;
-
-	/* Ignore early vblank irqs */
-	if (!crtc)
-		return;
-
-	/*
-	 * This is called both by irq handlers and the reset code (to complete
-	 * lost pageflips) so needs the full irqsave spinlocks.
-	 */
-	spin_lock_irqsave(&dev->event_lock, flags);
-	work = crtc->flip_work;
-
-	if (work != NULL &&
-	    !is_mmio_work(work) &&
-	    pageflip_finished(crtc, work))
-		page_flip_completed(crtc);
-
-	spin_unlock_irqrestore(&dev->event_lock, flags);
-}
-
-void intel_finish_page_flip_mmio(struct drm_i915_private *dev_priv, int pipe)
-{
-	struct drm_device *dev = &dev_priv->drm;
-	struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
-	struct intel_flip_work *work;
-	unsigned long flags;
-
-	/* Ignore early vblank irqs */
-	if (!crtc)
-		return;
-
-	/*
-	 * This is called both by irq handlers and the reset code (to complete
-	 * lost pageflips) so needs the full irqsave spinlocks.
-	 */
-	spin_lock_irqsave(&dev->event_lock, flags);
-	work = crtc->flip_work;
-
-	if (work != NULL &&
-	    is_mmio_work(work) &&
-	    pageflip_finished(crtc, work))
-		page_flip_completed(crtc);
-
-	spin_unlock_irqrestore(&dev->event_lock, flags);
-}
-
-static inline void intel_mark_page_flip_active(struct intel_crtc *crtc,
-					       struct intel_flip_work *work)
-{
-	work->flip_queued_vblank = intel_crtc_get_vblank_counter(crtc);
-
-	/* Ensure that the work item is consistent when activating it ... */
-	smp_mb__before_atomic();
-	atomic_set(&work->pending, 1);
-}
-
-static int intel_gen2_queue_flip(struct drm_device *dev,
-				 struct drm_crtc *crtc,
-				 struct drm_framebuffer *fb,
-				 struct drm_i915_gem_object *obj,
-				 struct drm_i915_gem_request *req,
-				 uint32_t flags)
-{
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	u32 flip_mask, *cs;
-
-	cs = intel_ring_begin(req, 6);
-	if (IS_ERR(cs))
-		return PTR_ERR(cs);
-
-	/* Can't queue multiple flips, so wait for the previous
-	 * one to finish before executing the next.
-	 */
-	if (intel_crtc->plane)
-		flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
-	else
-		flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
-	*cs++ = MI_WAIT_FOR_EVENT | flip_mask;
-	*cs++ = MI_NOOP;
-	*cs++ = MI_DISPLAY_FLIP | MI_DISPLAY_FLIP_PLANE(intel_crtc->plane);
-	*cs++ = fb->pitches[0];
-	*cs++ = intel_crtc->flip_work->gtt_offset;
-	*cs++ = 0; /* aux display base address, unused */
-
-	return 0;
-}
-
-static int intel_gen3_queue_flip(struct drm_device *dev,
-				 struct drm_crtc *crtc,
-				 struct drm_framebuffer *fb,
-				 struct drm_i915_gem_object *obj,
-				 struct drm_i915_gem_request *req,
-				 uint32_t flags)
-{
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	u32 flip_mask, *cs;
-
-	cs = intel_ring_begin(req, 6);
-	if (IS_ERR(cs))
-		return PTR_ERR(cs);
-
-	if (intel_crtc->plane)
-		flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
-	else
-		flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
-	*cs++ = MI_WAIT_FOR_EVENT | flip_mask;
-	*cs++ = MI_NOOP;
-	*cs++ = MI_DISPLAY_FLIP_I915 | MI_DISPLAY_FLIP_PLANE(intel_crtc->plane);
-	*cs++ = fb->pitches[0];
-	*cs++ = intel_crtc->flip_work->gtt_offset;
-	*cs++ = MI_NOOP;
-
-	return 0;
-}
-
-static int intel_gen4_queue_flip(struct drm_device *dev,
-				 struct drm_crtc *crtc,
-				 struct drm_framebuffer *fb,
-				 struct drm_i915_gem_object *obj,
-				 struct drm_i915_gem_request *req,
-				 uint32_t flags)
-{
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	u32 pf, pipesrc, *cs;
-
-	cs = intel_ring_begin(req, 4);
-	if (IS_ERR(cs))
-		return PTR_ERR(cs);
-
-	/* i965+ uses the linear or tiled offsets from the
-	 * Display Registers (which do not change across a page-flip)
-	 * so we need only reprogram the base address.
-	 */
-	*cs++ = MI_DISPLAY_FLIP | MI_DISPLAY_FLIP_PLANE(intel_crtc->plane);
-	*cs++ = fb->pitches[0];
-	*cs++ = intel_crtc->flip_work->gtt_offset |
-		intel_fb_modifier_to_tiling(fb->modifier);
-
-	/* XXX Enabling the panel-fitter across page-flip is so far
-	 * untested on non-native modes, so ignore it for now.
-	 * pf = I915_READ(pipe == 0 ? PFA_CTL_1 : PFB_CTL_1) & PF_ENABLE;
-	 */
-	pf = 0;
-	pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
-	*cs++ = pf | pipesrc;
-
-	return 0;
-}
-
-static int intel_gen6_queue_flip(struct drm_device *dev,
-				 struct drm_crtc *crtc,
-				 struct drm_framebuffer *fb,
-				 struct drm_i915_gem_object *obj,
-				 struct drm_i915_gem_request *req,
-				 uint32_t flags)
-{
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	u32 pf, pipesrc, *cs;
-
-	cs = intel_ring_begin(req, 4);
-	if (IS_ERR(cs))
-		return PTR_ERR(cs);
-
-	*cs++ = MI_DISPLAY_FLIP | MI_DISPLAY_FLIP_PLANE(intel_crtc->plane);
-	*cs++ = fb->pitches[0] | intel_fb_modifier_to_tiling(fb->modifier);
-	*cs++ = intel_crtc->flip_work->gtt_offset;
-
-	/* Contrary to the suggestions in the documentation,
-	 * "Enable Panel Fitter" does not seem to be required when page
-	 * flipping with a non-native mode, and worse causes a normal
-	 * modeset to fail.
-	 * pf = I915_READ(PF_CTL(intel_crtc->pipe)) & PF_ENABLE;
-	 */
-	pf = 0;
-	pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
-	*cs++ = pf | pipesrc;
-
-	return 0;
-}
-
-static int intel_gen7_queue_flip(struct drm_device *dev,
-				 struct drm_crtc *crtc,
-				 struct drm_framebuffer *fb,
-				 struct drm_i915_gem_object *obj,
-				 struct drm_i915_gem_request *req,
-				 uint32_t flags)
-{
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	u32 *cs, plane_bit = 0;
-	int len, ret;
-
-	switch (intel_crtc->plane) {
-	case PLANE_A:
-		plane_bit = MI_DISPLAY_FLIP_IVB_PLANE_A;
-		break;
-	case PLANE_B:
-		plane_bit = MI_DISPLAY_FLIP_IVB_PLANE_B;
-		break;
-	case PLANE_C:
-		plane_bit = MI_DISPLAY_FLIP_IVB_PLANE_C;
-		break;
-	default:
-		WARN_ONCE(1, "unknown plane in flip command\n");
-		return -ENODEV;
-	}
-
-	len = 4;
-	if (req->engine->id == RCS) {
-		len += 6;
-		/*
-		 * On Gen 8, SRM is now taking an extra dword to accommodate
-		 * 48bits addresses, and we need a NOOP for the batch size to
-		 * stay even.
-		 */
-		if (IS_GEN8(dev_priv))
-			len += 2;
-	}
-
-	/*
-	 * BSpec MI_DISPLAY_FLIP for IVB:
-	 * "The full packet must be contained within the same cache line."
-	 *
-	 * Currently the LRI+SRM+MI_DISPLAY_FLIP all fit within the same
-	 * cacheline, if we ever start emitting more commands before
-	 * the MI_DISPLAY_FLIP we may need to first emit everything else,
-	 * then do the cacheline alignment, and finally emit the
-	 * MI_DISPLAY_FLIP.
-	 */
-	ret = intel_ring_cacheline_align(req);
-	if (ret)
-		return ret;
-
-	cs = intel_ring_begin(req, len);
-	if (IS_ERR(cs))
-		return PTR_ERR(cs);
-
-	/* Unmask the flip-done completion message. Note that the bspec says that
-	 * we should do this for both the BCS and RCS, and that we must not unmask
-	 * more than one flip event at any time (or ensure that one flip message
-	 * can be sent by waiting for flip-done prior to queueing new flips).
-	 * Experimentation says that BCS works despite DERRMR masking all
-	 * flip-done completion events and that unmasking all planes at once
-	 * for the RCS also doesn't appear to drop events. Setting the DERRMR
-	 * to zero does lead to lockups within MI_DISPLAY_FLIP.
-	 */
-	if (req->engine->id == RCS) {
-		*cs++ = MI_LOAD_REGISTER_IMM(1);
-		*cs++ = i915_mmio_reg_offset(DERRMR);
-		*cs++ = ~(DERRMR_PIPEA_PRI_FLIP_DONE |
-			  DERRMR_PIPEB_PRI_FLIP_DONE |
-			  DERRMR_PIPEC_PRI_FLIP_DONE);
-		if (IS_GEN8(dev_priv))
-			*cs++ = MI_STORE_REGISTER_MEM_GEN8 |
-				MI_SRM_LRM_GLOBAL_GTT;
-		else
-			*cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
-		*cs++ = i915_mmio_reg_offset(DERRMR);
-		*cs++ = i915_ggtt_offset(req->engine->scratch) + 256;
-		if (IS_GEN8(dev_priv)) {
-			*cs++ = 0;
-			*cs++ = MI_NOOP;
-		}
-	}
-
-	*cs++ = MI_DISPLAY_FLIP_I915 | plane_bit;
-	*cs++ = fb->pitches[0] | intel_fb_modifier_to_tiling(fb->modifier);
-	*cs++ = intel_crtc->flip_work->gtt_offset;
-	*cs++ = MI_NOOP;
-
-	return 0;
-}
-
-static bool use_mmio_flip(struct intel_engine_cs *engine,
-			  struct drm_i915_gem_object *obj)
-{
-	/*
-	 * This is not being used for older platforms, because
-	 * non-availability of flip done interrupt forces us to use
-	 * CS flips. Older platforms derive flip done using some clever
-	 * tricks involving the flip_pending status bits and vblank irqs.
-	 * So using MMIO flips there would disrupt this mechanism.
-	 */
-
-	if (engine == NULL)
-		return true;
-
-	if (INTEL_GEN(engine->i915) < 5)
-		return false;
-
-	if (i915.use_mmio_flip < 0)
-		return false;
-	else if (i915.use_mmio_flip > 0)
-		return true;
-	else if (i915.enable_execlists)
-		return true;
-
-	return engine != i915_gem_object_last_write_engine(obj);
-}
-
-static void skl_do_mmio_flip(struct intel_crtc *intel_crtc,
-			     unsigned int rotation,
-			     struct intel_flip_work *work)
-{
-	struct drm_device *dev = intel_crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct drm_framebuffer *fb = intel_crtc->base.primary->fb;
-	const enum pipe pipe = intel_crtc->pipe;
-	u32 ctl, stride = skl_plane_stride(fb, 0, rotation);
-
-	ctl = I915_READ(PLANE_CTL(pipe, 0));
-	ctl &= ~PLANE_CTL_TILED_MASK;
-	switch (fb->modifier) {
-	case DRM_FORMAT_MOD_LINEAR:
-		break;
-	case I915_FORMAT_MOD_X_TILED:
-		ctl |= PLANE_CTL_TILED_X;
-		break;
-	case I915_FORMAT_MOD_Y_TILED:
-		ctl |= PLANE_CTL_TILED_Y;
-		break;
-	case I915_FORMAT_MOD_Yf_TILED:
-		ctl |= PLANE_CTL_TILED_YF;
-		break;
-	default:
-		MISSING_CASE(fb->modifier);
-	}
-
-	/*
-	 * Both PLANE_CTL and PLANE_STRIDE are not updated on vblank but on
-	 * PLANE_SURF updates, the update is then guaranteed to be atomic.
-	 */
-	I915_WRITE(PLANE_CTL(pipe, 0), ctl);
-	I915_WRITE(PLANE_STRIDE(pipe, 0), stride);
-
-	I915_WRITE(PLANE_SURF(pipe, 0), work->gtt_offset);
-	POSTING_READ(PLANE_SURF(pipe, 0));
-}
-
-static void ilk_do_mmio_flip(struct intel_crtc *intel_crtc,
-			     struct intel_flip_work *work)
-{
-	struct drm_device *dev = intel_crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct drm_framebuffer *fb = intel_crtc->base.primary->fb;
-	i915_reg_t reg = DSPCNTR(intel_crtc->plane);
-	u32 dspcntr;
-
-	dspcntr = I915_READ(reg);
-
-	if (fb->modifier == I915_FORMAT_MOD_X_TILED)
-		dspcntr |= DISPPLANE_TILED;
-	else
-		dspcntr &= ~DISPPLANE_TILED;
-
-	I915_WRITE(reg, dspcntr);
-
-	I915_WRITE(DSPSURF(intel_crtc->plane), work->gtt_offset);
-	POSTING_READ(DSPSURF(intel_crtc->plane));
-}
-
-static void intel_mmio_flip_work_func(struct work_struct *w)
-{
-	struct intel_flip_work *work =
-		container_of(w, struct intel_flip_work, mmio_work);
-	struct intel_crtc *crtc = to_intel_crtc(work->crtc);
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	struct intel_framebuffer *intel_fb =
-		to_intel_framebuffer(crtc->base.primary->fb);
-	struct drm_i915_gem_object *obj = intel_fb->obj;
-
-	WARN_ON(i915_gem_object_wait(obj, 0, MAX_SCHEDULE_TIMEOUT, NULL) < 0);
-
-	intel_pipe_update_start(crtc);
-
-	if (INTEL_GEN(dev_priv) >= 9)
-		skl_do_mmio_flip(crtc, work->rotation, work);
-	else
-		/* use_mmio_flip() retricts MMIO flips to ilk+ */
-		ilk_do_mmio_flip(crtc, work);
-
-	intel_pipe_update_end(crtc, work);
-}
-
-static int intel_default_queue_flip(struct drm_device *dev,
-				    struct drm_crtc *crtc,
-				    struct drm_framebuffer *fb,
-				    struct drm_i915_gem_object *obj,
-				    struct drm_i915_gem_request *req,
-				    uint32_t flags)
-{
-	return -ENODEV;
-}
-
-static bool __pageflip_stall_check_cs(struct drm_i915_private *dev_priv,
-				      struct intel_crtc *intel_crtc,
-				      struct intel_flip_work *work)
-{
-	u32 addr, vblank;
-
-	if (!atomic_read(&work->pending))
-		return false;
-
-	smp_rmb();
-
-	vblank = intel_crtc_get_vblank_counter(intel_crtc);
-	if (work->flip_ready_vblank == 0) {
-		if (work->flip_queued_req &&
-		    !i915_gem_request_completed(work->flip_queued_req))
-			return false;
-
-		work->flip_ready_vblank = vblank;
-	}
-
-	if (vblank - work->flip_ready_vblank < 3)
-		return false;
-
-	/* Potential stall - if we see that the flip has happened,
-	 * assume a missed interrupt. */
-	if (INTEL_GEN(dev_priv) >= 4)
-		addr = I915_HI_DISPBASE(I915_READ(DSPSURF(intel_crtc->plane)));
-	else
-		addr = I915_READ(DSPADDR(intel_crtc->plane));
-
-	/* There is a potential issue here with a false positive after a flip
-	 * to the same address. We could address this by checking for a
-	 * non-incrementing frame counter.
-	 */
-	return addr == work->gtt_offset;
-}
-
-void intel_check_page_flip(struct drm_i915_private *dev_priv, int pipe)
-{
-	struct drm_device *dev = &dev_priv->drm;
-	struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
-	struct intel_flip_work *work;
-
-	WARN_ON(!in_interrupt());
-
-	if (crtc == NULL)
-		return;
-
-	spin_lock(&dev->event_lock);
-	work = crtc->flip_work;
-
-	if (work != NULL && !is_mmio_work(work) &&
-	    __pageflip_stall_check_cs(dev_priv, crtc, work)) {
-		WARN_ONCE(1,
-			  "Kicking stuck page flip: queued at %d, now %d\n",
-			work->flip_queued_vblank, intel_crtc_get_vblank_counter(crtc));
-		page_flip_completed(crtc);
-		work = NULL;
-	}
-
-	if (work != NULL && !is_mmio_work(work) &&
-	    intel_crtc_get_vblank_counter(crtc) - work->flip_queued_vblank > 1)
-		intel_queue_rps_boost_for_request(work->flip_queued_req);
-	spin_unlock(&dev->event_lock);
-}
-
-__maybe_unused
-static int intel_crtc_page_flip(struct drm_crtc *crtc,
-				struct drm_framebuffer *fb,
-				struct drm_pending_vblank_event *event,
-				uint32_t page_flip_flags)
-{
-	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct drm_framebuffer *old_fb = crtc->primary->fb;
-	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	struct drm_plane *primary = crtc->primary;
-	enum pipe pipe = intel_crtc->pipe;
-	struct intel_flip_work *work;
-	struct intel_engine_cs *engine;
-	bool mmio_flip;
-	struct drm_i915_gem_request *request;
-	struct i915_vma *vma;
-	int ret;
-
-	/*
-	 * drm_mode_page_flip_ioctl() should already catch this, but double
-	 * check to be safe.  In the future we may enable pageflipping from
-	 * a disabled primary plane.
-	 */
-	if (WARN_ON(intel_fb_obj(old_fb) == NULL))
-		return -EBUSY;
-
-	/* Can't change pixel format via MI display flips. */
-	if (fb->format != crtc->primary->fb->format)
-		return -EINVAL;
-
-	/*
-	 * TILEOFF/LINOFF registers can't be changed via MI display flips.
-	 * Note that pitch changes could also affect these register.
-	 */
-	if (INTEL_GEN(dev_priv) > 3 &&
-	    (fb->offsets[0] != crtc->primary->fb->offsets[0] ||
-	     fb->pitches[0] != crtc->primary->fb->pitches[0]))
-		return -EINVAL;
-
-	if (i915_terminally_wedged(&dev_priv->gpu_error))
-		goto out_hang;
-
-	work = kzalloc(sizeof(*work), GFP_KERNEL);
-	if (work == NULL)
-		return -ENOMEM;
-
-	work->event = event;
-	work->crtc = crtc;
-	work->old_fb = old_fb;
-	INIT_WORK(&work->unpin_work, intel_unpin_work_fn);
-
-	ret = drm_crtc_vblank_get(crtc);
-	if (ret)
-		goto free_work;
-
-	/* We borrow the event spin lock for protecting flip_work */
-	spin_lock_irq(&dev->event_lock);
-	if (intel_crtc->flip_work) {
-		/* Before declaring the flip queue wedged, check if
-		 * the hardware completed the operation behind our backs.
-		 */
-		if (pageflip_finished(intel_crtc, intel_crtc->flip_work)) {
-			DRM_DEBUG_DRIVER("flip queue: previous flip completed, continuing\n");
-			page_flip_completed(intel_crtc);
-		} else {
-			DRM_DEBUG_DRIVER("flip queue: crtc already busy\n");
-			spin_unlock_irq(&dev->event_lock);
-
-			drm_crtc_vblank_put(crtc);
-			kfree(work);
-			return -EBUSY;
-		}
-	}
-	intel_crtc->flip_work = work;
-	spin_unlock_irq(&dev->event_lock);
-
-	if (atomic_read(&intel_crtc->unpin_work_count) >= 2)
-		flush_workqueue(dev_priv->wq);
-
-	/* Reference the objects for the scheduled work. */
-	drm_framebuffer_reference(work->old_fb);
-
-	crtc->primary->fb = fb;
-	update_state_fb(crtc->primary);
-
-	work->pending_flip_obj = i915_gem_object_get(obj);
-
-	ret = i915_mutex_lock_interruptible(dev);
-	if (ret)
-		goto cleanup;
-
-	intel_crtc->reset_count = i915_reset_count(&dev_priv->gpu_error);
-	if (i915_reset_backoff_or_wedged(&dev_priv->gpu_error)) {
-		ret = -EIO;
-		goto unlock;
-	}
-
-	atomic_inc(&intel_crtc->unpin_work_count);
-
-	if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
-		work->flip_count = I915_READ(PIPE_FLIPCOUNT_G4X(pipe)) + 1;
-
-	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
-		engine = dev_priv->engine[BCS];
-		if (fb->modifier != old_fb->modifier)
-			/* vlv: DISPLAY_FLIP fails to change tiling */
-			engine = NULL;
-	} else if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv)) {
-		engine = dev_priv->engine[BCS];
-	} else if (INTEL_GEN(dev_priv) >= 7) {
-		engine = i915_gem_object_last_write_engine(obj);
-		if (engine == NULL || engine->id != RCS)
-			engine = dev_priv->engine[BCS];
-	} else {
-		engine = dev_priv->engine[RCS];
-	}
-
-	mmio_flip = use_mmio_flip(engine, obj);
-
-	vma = intel_pin_and_fence_fb_obj(fb, primary->state->rotation);
-	if (IS_ERR(vma)) {
-		ret = PTR_ERR(vma);
-		goto cleanup_pending;
-	}
-
-	work->old_vma = to_intel_plane_state(primary->state)->vma;
-	to_intel_plane_state(primary->state)->vma = vma;
-
-	work->gtt_offset = i915_ggtt_offset(vma) + intel_crtc->dspaddr_offset;
-	work->rotation = crtc->primary->state->rotation;
-
-	/*
-	 * There's the potential that the next frame will not be compatible with
-	 * FBC, so we want to call pre_update() before the actual page flip.
-	 * The problem is that pre_update() caches some information about the fb
-	 * object, so we want to do this only after the object is pinned. Let's
-	 * be on the safe side and do this immediately before scheduling the
-	 * flip.
-	 */
-	intel_fbc_pre_update(intel_crtc, intel_crtc->config,
-			     to_intel_plane_state(primary->state));
-
-	if (mmio_flip) {
-		INIT_WORK(&work->mmio_work, intel_mmio_flip_work_func);
-		queue_work(system_unbound_wq, &work->mmio_work);
-	} else {
-		request = i915_gem_request_alloc(engine,
-						 dev_priv->kernel_context);
-		if (IS_ERR(request)) {
-			ret = PTR_ERR(request);
-			goto cleanup_unpin;
-		}
-
-		ret = i915_gem_request_await_object(request, obj, false);
-		if (ret)
-			goto cleanup_request;
-
-		ret = dev_priv->display.queue_flip(dev, crtc, fb, obj, request,
-						   page_flip_flags);
-		if (ret)
-			goto cleanup_request;
-
-		intel_mark_page_flip_active(intel_crtc, work);
-
-		work->flip_queued_req = i915_gem_request_get(request);
-		i915_add_request(request);
-	}
-
-	i915_gem_object_wait_priority(obj, 0, I915_PRIORITY_DISPLAY);
-	i915_gem_track_fb(intel_fb_obj(old_fb), obj,
-			  to_intel_plane(primary)->frontbuffer_bit);
-	mutex_unlock(&dev->struct_mutex);
-
-	intel_frontbuffer_flip_prepare(to_i915(dev),
-				       to_intel_plane(primary)->frontbuffer_bit);
-
-	trace_i915_flip_request(intel_crtc->plane, obj);
-
-	return 0;
-
-cleanup_request:
-	i915_add_request(request);
-cleanup_unpin:
-	to_intel_plane_state(primary->state)->vma = work->old_vma;
-	intel_unpin_fb_vma(vma);
-cleanup_pending:
-	atomic_dec(&intel_crtc->unpin_work_count);
-unlock:
-	mutex_unlock(&dev->struct_mutex);
-cleanup:
-	crtc->primary->fb = old_fb;
-	update_state_fb(crtc->primary);
-
-	i915_gem_object_put(obj);
-	drm_framebuffer_unreference(work->old_fb);
-
-	spin_lock_irq(&dev->event_lock);
-	intel_crtc->flip_work = NULL;
-	spin_unlock_irq(&dev->event_lock);
-
-	drm_crtc_vblank_put(crtc);
-free_work:
-	kfree(work);
-
-	if (ret == -EIO) {
-		struct drm_atomic_state *state;
-		struct drm_plane_state *plane_state;
-
-out_hang:
-		state = drm_atomic_state_alloc(dev);
-		if (!state)
-			return -ENOMEM;
-		state->acquire_ctx = dev->mode_config.acquire_ctx;
-
-retry:
-		plane_state = drm_atomic_get_plane_state(state, primary);
-		ret = PTR_ERR_OR_ZERO(plane_state);
-		if (!ret) {
-			drm_atomic_set_fb_for_plane(plane_state, fb);
-
-			ret = drm_atomic_set_crtc_for_plane(plane_state, crtc);
-			if (!ret)
-				ret = drm_atomic_commit(state);
-		}
-
-		if (ret == -EDEADLK) {
-			drm_modeset_backoff(state->acquire_ctx);
-			drm_atomic_state_clear(state);
-			goto retry;
-		}
-
-		drm_atomic_state_put(state);
-
-		if (ret == 0 && event) {
-			spin_lock_irq(&dev->event_lock);
-			drm_crtc_send_vblank_event(crtc, event);
-			spin_unlock_irq(&dev->event_lock);
-		}
-	}
-	return ret;
-}
-
-
 /**
  * intel_wm_need_update - Check whether watermarks need updating
  * @plane: drm plane
@@ -11345,6 +10700,9 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc,
 				      pipe_config->fdi_lanes,
 				      &pipe_config->fdi_m_n);
 
+	if (pipe_config->ycbcr420)
+		DRM_DEBUG_KMS("YCbCr 4:2:0 output enabled\n");
+
 	if (intel_crtc_has_dp_encoder(pipe_config)) {
 		intel_dump_m_n_config(pipe_config, "dp m_n",
 				pipe_config->lane_count, &pipe_config->dp_m_n);
@@ -11916,6 +11274,7 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv,
 	PIPE_CONF_CHECK_I(hdmi_scrambling);
 	PIPE_CONF_CHECK_I(hdmi_high_tmds_clock_ratio);
 	PIPE_CONF_CHECK_I(has_infoframe);
+	PIPE_CONF_CHECK_I(ycbcr420);
 
 	PIPE_CONF_CHECK_I(has_audio);
 
@@ -12757,31 +12116,7 @@ static int intel_atomic_check(struct drm_device *dev,
 static int intel_atomic_prepare_commit(struct drm_device *dev,
 				       struct drm_atomic_state *state)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct drm_crtc_state *crtc_state;
-	struct drm_crtc *crtc;
-	int i, ret;
-
-	for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
-		if (state->legacy_cursor_update)
-			continue;
-
-		ret = intel_crtc_wait_for_pending_flips(crtc);
-		if (ret)
-			return ret;
-
-		if (atomic_read(&to_intel_crtc(crtc)->unpin_work_count) >= 2)
-			flush_workqueue(dev_priv->wq);
-	}
-
-	ret = mutex_lock_interruptible(&dev->struct_mutex);
-	if (ret)
-		return ret;
-
-	ret = drm_atomic_helper_prepare_planes(dev, state);
-	mutex_unlock(&dev->struct_mutex);
-
-	return ret;
+	return drm_atomic_helper_prepare_planes(dev, state);
 }
 
 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
@@ -12789,7 +12124,7 @@ u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
 	struct drm_device *dev = crtc->base.dev;
 
 	if (!dev->max_vblank_count)
-		return drm_accurate_vblank_count(&crtc->base);
+		return drm_crtc_accurate_vblank_count(&crtc->base);
 
 	return dev->driver->get_vblank_counter(dev, crtc->pipe);
 }
@@ -12992,6 +12327,30 @@ static void intel_atomic_helper_free_state_worker(struct work_struct *work)
 	intel_atomic_helper_free_state(dev_priv);
 }
 
+static void intel_atomic_commit_fence_wait(struct intel_atomic_state *intel_state)
+{
+	struct wait_queue_entry wait_fence, wait_reset;
+	struct drm_i915_private *dev_priv = to_i915(intel_state->base.dev);
+
+	init_wait_entry(&wait_fence, 0);
+	init_wait_entry(&wait_reset, 0);
+	for (;;) {
+		prepare_to_wait(&intel_state->commit_ready.wait,
+				&wait_fence, TASK_UNINTERRUPTIBLE);
+		prepare_to_wait(&dev_priv->gpu_error.wait_queue,
+				&wait_reset, TASK_UNINTERRUPTIBLE);
+
+
+		if (i915_sw_fence_done(&intel_state->commit_ready)
+		    || test_bit(I915_RESET_MODESET, &dev_priv->gpu_error.flags))
+			break;
+
+		schedule();
+	}
+	finish_wait(&intel_state->commit_ready.wait, &wait_fence);
+	finish_wait(&dev_priv->gpu_error.wait_queue, &wait_reset);
+}
+
 static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 {
 	struct drm_device *dev = state->dev;
@@ -13005,6 +12364,8 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 	unsigned crtc_vblank_mask = 0;
 	int i;
 
+	intel_atomic_commit_fence_wait(intel_state);
+
 	drm_atomic_helper_wait_for_dependencies(state);
 
 	if (intel_state->modeset)
@@ -13144,9 +12505,7 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 		intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET);
 	}
 
-	mutex_lock(&dev->struct_mutex);
 	drm_atomic_helper_cleanup_planes(dev, state);
-	mutex_unlock(&dev->struct_mutex);
 
 	drm_atomic_helper_commit_cleanup_done(state);
 
@@ -13172,10 +12531,8 @@ intel_atomic_commit_ready(struct i915_sw_fence *fence,
 
 	switch (notify) {
 	case FENCE_COMPLETE:
-		if (state->base.commit_work.func)
-			queue_work(system_unbound_wq, &state->base.commit_work);
+		/* we do blocking waits in the worker, nothing to do here */
 		break;
-
 	case FENCE_FREE:
 		{
 			struct intel_atomic_helper *helper =
@@ -13257,7 +12614,13 @@ static int intel_atomic_commit(struct drm_device *dev,
 	if (INTEL_GEN(dev_priv) < 9)
 		state->legacy_cursor_update = false;
 
-	drm_atomic_helper_swap_state(state, true);
+	ret = drm_atomic_helper_swap_state(state, true);
+	if (ret) {
+		i915_sw_fence_commit(&intel_state->commit_ready);
+
+		drm_atomic_helper_cleanup_planes(dev, state);
+		return ret;
+	}
 	dev_priv->wm.distrust_bios_wm = false;
 	intel_shared_dpll_swap_state(state);
 	intel_atomic_track_fbs(state);
@@ -13271,14 +12634,14 @@ static int intel_atomic_commit(struct drm_device *dev,
 	}
 
 	drm_atomic_state_get(state);
-	INIT_WORK(&state->commit_work,
-		  nonblock ? intel_atomic_commit_work : NULL);
+	INIT_WORK(&state->commit_work, intel_atomic_commit_work);
 
 	i915_sw_fence_commit(&intel_state->commit_ready);
-	if (!nonblock) {
-		i915_sw_fence_wait(&intel_state->commit_ready);
+	if (nonblock)
+		queue_work(system_unbound_wq, &state->commit_work);
+	else
 		intel_atomic_commit_tail(state);
-	}
+
 
 	return 0;
 }
@@ -13286,7 +12649,6 @@ static int intel_atomic_commit(struct drm_device *dev,
 static const struct drm_crtc_funcs intel_crtc_funcs = {
 	.gamma_set = drm_atomic_helper_legacy_gamma_set,
 	.set_config = drm_atomic_helper_set_config,
-	.set_property = drm_atomic_helper_crtc_set_property,
 	.destroy = intel_crtc_destroy,
 	.page_flip = drm_atomic_helper_page_flip,
 	.atomic_duplicate_state = intel_crtc_duplicate_state,
@@ -13320,32 +12682,6 @@ intel_prepare_plane_fb(struct drm_plane *plane,
 	struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb);
 	int ret;
 
-	if (obj) {
-		if (plane->type == DRM_PLANE_TYPE_CURSOR &&
-		    INTEL_INFO(dev_priv)->cursor_needs_physical) {
-			const int align = intel_cursor_alignment(dev_priv);
-
-			ret = i915_gem_object_attach_phys(obj, align);
-			if (ret) {
-				DRM_DEBUG_KMS("failed to attach phys object\n");
-				return ret;
-			}
-		} else {
-			struct i915_vma *vma;
-
-			vma = intel_pin_and_fence_fb_obj(fb, new_state->rotation);
-			if (IS_ERR(vma)) {
-				DRM_DEBUG_KMS("failed to pin object\n");
-				return PTR_ERR(vma);
-			}
-
-			to_intel_plane_state(new_state)->vma = vma;
-		}
-	}
-
-	if (!obj && !old_obj)
-		return 0;
-
 	if (old_obj) {
 		struct drm_crtc_state *crtc_state =
 			drm_atomic_get_existing_crtc_state(new_state->state,
@@ -13384,6 +12720,38 @@ intel_prepare_plane_fb(struct drm_plane *plane,
 	if (!obj)
 		return 0;
 
+	ret = i915_gem_object_pin_pages(obj);
+	if (ret)
+		return ret;
+
+	ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex);
+	if (ret) {
+		i915_gem_object_unpin_pages(obj);
+		return ret;
+	}
+
+	if (plane->type == DRM_PLANE_TYPE_CURSOR &&
+	    INTEL_INFO(dev_priv)->cursor_needs_physical) {
+		const int align = intel_cursor_alignment(dev_priv);
+
+		ret = i915_gem_object_attach_phys(obj, align);
+	} else {
+		struct i915_vma *vma;
+
+		vma = intel_pin_and_fence_fb_obj(fb, new_state->rotation);
+		if (!IS_ERR(vma))
+			to_intel_plane_state(new_state)->vma = vma;
+		else
+			ret =  PTR_ERR(vma);
+	}
+
+	i915_gem_object_wait_priority(obj, 0, I915_PRIORITY_DISPLAY);
+
+	mutex_unlock(&dev_priv->drm.struct_mutex);
+	i915_gem_object_unpin_pages(obj);
+	if (ret)
+		return ret;
+
 	if (!new_state->fence) { /* implicit fencing */
 		ret = i915_sw_fence_await_reservation(&intel_state->commit_ready,
 						      obj->resv, NULL,
@@ -13391,8 +12759,6 @@ intel_prepare_plane_fb(struct drm_plane *plane,
 						      GFP_KERNEL);
 		if (ret < 0)
 			return ret;
-
-		i915_gem_object_wait_priority(obj, 0, I915_PRIORITY_DISPLAY);
 	}
 
 	return 0;
@@ -13415,8 +12781,11 @@ intel_cleanup_plane_fb(struct drm_plane *plane,
 
 	/* Should only be called after a successful intel_prepare_plane_fb()! */
 	vma = fetch_and_zero(&to_intel_plane_state(old_state)->vma);
-	if (vma)
+	if (vma) {
+		mutex_lock(&plane->dev->struct_mutex);
 		intel_unpin_fb_vma(vma);
+		mutex_unlock(&plane->dev->struct_mutex);
+	}
 }
 
 int
@@ -13543,7 +12912,7 @@ static void intel_finish_crtc_commit(struct drm_crtc *crtc,
 {
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 
-	intel_pipe_update_end(intel_crtc, NULL);
+	intel_pipe_update_end(intel_crtc);
 }
 
 /**
@@ -13559,15 +12928,110 @@ void intel_plane_destroy(struct drm_plane *plane)
 	kfree(to_intel_plane(plane));
 }
 
-const struct drm_plane_funcs intel_plane_funcs = {
+static bool i8xx_mod_supported(uint32_t format, uint64_t modifier)
+{
+	switch (format) {
+	case DRM_FORMAT_C8:
+	case DRM_FORMAT_RGB565:
+	case DRM_FORMAT_XRGB1555:
+	case DRM_FORMAT_XRGB8888:
+		return modifier == DRM_FORMAT_MOD_LINEAR ||
+			modifier == I915_FORMAT_MOD_X_TILED;
+	default:
+		return false;
+	}
+}
+
+static bool i965_mod_supported(uint32_t format, uint64_t modifier)
+{
+	switch (format) {
+	case DRM_FORMAT_C8:
+	case DRM_FORMAT_RGB565:
+	case DRM_FORMAT_XRGB8888:
+	case DRM_FORMAT_XBGR8888:
+	case DRM_FORMAT_XRGB2101010:
+	case DRM_FORMAT_XBGR2101010:
+		return modifier == DRM_FORMAT_MOD_LINEAR ||
+			modifier == I915_FORMAT_MOD_X_TILED;
+	default:
+		return false;
+	}
+}
+
+static bool skl_mod_supported(uint32_t format, uint64_t modifier)
+{
+	switch (format) {
+	case DRM_FORMAT_XRGB8888:
+	case DRM_FORMAT_XBGR8888:
+	case DRM_FORMAT_ARGB8888:
+	case DRM_FORMAT_ABGR8888:
+		if (modifier == I915_FORMAT_MOD_Yf_TILED_CCS ||
+		    modifier == I915_FORMAT_MOD_Y_TILED_CCS)
+			return true;
+		/* fall through */
+	case DRM_FORMAT_RGB565:
+	case DRM_FORMAT_XRGB2101010:
+	case DRM_FORMAT_XBGR2101010:
+	case DRM_FORMAT_YUYV:
+	case DRM_FORMAT_YVYU:
+	case DRM_FORMAT_UYVY:
+	case DRM_FORMAT_VYUY:
+		if (modifier == I915_FORMAT_MOD_Yf_TILED)
+			return true;
+		/* fall through */
+	case DRM_FORMAT_C8:
+		if (modifier == DRM_FORMAT_MOD_LINEAR ||
+		    modifier == I915_FORMAT_MOD_X_TILED ||
+		    modifier == I915_FORMAT_MOD_Y_TILED)
+			return true;
+		/* fall through */
+	default:
+		return false;
+	}
+}
+
+static bool intel_primary_plane_format_mod_supported(struct drm_plane *plane,
+						     uint32_t format,
+						     uint64_t modifier)
+{
+	struct drm_i915_private *dev_priv = to_i915(plane->dev);
+
+	if (WARN_ON(modifier == DRM_FORMAT_MOD_INVALID))
+		return false;
+
+	if ((modifier >> 56) != DRM_FORMAT_MOD_VENDOR_INTEL &&
+	    modifier != DRM_FORMAT_MOD_LINEAR)
+		return false;
+
+	if (INTEL_GEN(dev_priv) >= 9)
+		return skl_mod_supported(format, modifier);
+	else if (INTEL_GEN(dev_priv) >= 4)
+		return i965_mod_supported(format, modifier);
+	else
+		return i8xx_mod_supported(format, modifier);
+
+	unreachable();
+}
+
+static bool intel_cursor_plane_format_mod_supported(struct drm_plane *plane,
+						    uint32_t format,
+						    uint64_t modifier)
+{
+	if (WARN_ON(modifier == DRM_FORMAT_MOD_INVALID))
+		return false;
+
+	return modifier == DRM_FORMAT_MOD_LINEAR && format == DRM_FORMAT_ARGB8888;
+}
+
+static struct drm_plane_funcs intel_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = intel_plane_destroy,
-	.set_property = drm_atomic_helper_plane_set_property,
 	.atomic_get_property = intel_plane_atomic_get_property,
 	.atomic_set_property = intel_plane_atomic_set_property,
 	.atomic_duplicate_state = intel_plane_duplicate_state,
 	.atomic_destroy_state = intel_plane_destroy_state,
+	.format_mod_supported = intel_primary_plane_format_mod_supported,
 };
 
 static int
@@ -13586,7 +13050,7 @@ intel_legacy_cursor_update(struct drm_plane *plane,
 	struct intel_plane *intel_plane = to_intel_plane(plane);
 	struct drm_framebuffer *old_fb;
 	struct drm_crtc_state *crtc_state = crtc->state;
-	struct i915_vma *old_vma;
+	struct i915_vma *old_vma, *vma;
 
 	/*
 	 * When crtc is inactive or there is a modeset pending,
@@ -13644,8 +13108,6 @@ intel_legacy_cursor_update(struct drm_plane *plane,
 			goto out_unlock;
 		}
 	} else {
-		struct i915_vma *vma;
-
 		vma = intel_pin_and_fence_fb_obj(fb, new_plane_state->rotation);
 		if (IS_ERR(vma)) {
 			DRM_DEBUG_KMS("failed to pin object\n");
@@ -13668,7 +13130,7 @@ intel_legacy_cursor_update(struct drm_plane *plane,
 	*to_intel_plane_state(old_plane_state) = *to_intel_plane_state(new_plane_state);
 	new_plane_state->fence = NULL;
 	new_plane_state->fb = old_fb;
-	to_intel_plane_state(new_plane_state)->vma = old_vma;
+	to_intel_plane_state(new_plane_state)->vma = NULL;
 
 	if (plane->state->visible) {
 		trace_intel_update_plane(plane, to_intel_crtc(crtc));
@@ -13680,7 +13142,8 @@ intel_legacy_cursor_update(struct drm_plane *plane,
 		intel_plane->disable_plane(intel_plane, to_intel_crtc(crtc));
 	}
 
-	intel_cleanup_plane_fb(plane, new_plane_state);
+	if (old_vma)
+		intel_unpin_fb_vma(old_vma);
 
 out_unlock:
 	mutex_unlock(&dev_priv->drm.struct_mutex);
@@ -13698,11 +13161,11 @@ static const struct drm_plane_funcs intel_cursor_plane_funcs = {
 	.update_plane = intel_legacy_cursor_update,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = intel_plane_destroy,
-	.set_property = drm_atomic_helper_plane_set_property,
 	.atomic_get_property = intel_plane_atomic_get_property,
 	.atomic_set_property = intel_plane_atomic_set_property,
 	.atomic_duplicate_state = intel_plane_duplicate_state,
 	.atomic_destroy_state = intel_plane_destroy_state,
+	.format_mod_supported = intel_cursor_plane_format_mod_supported,
 };
 
 static struct intel_plane *
@@ -13713,6 +13176,7 @@ intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
 	const uint32_t *intel_primary_formats;
 	unsigned int supported_rotations;
 	unsigned int num_formats;
+	const uint64_t *modifiers;
 	int ret;
 
 	primary = kzalloc(sizeof(*primary), GFP_KERNEL);
@@ -13748,21 +13212,34 @@ intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
 	primary->frontbuffer_bit = INTEL_FRONTBUFFER_PRIMARY(pipe);
 	primary->check_plane = intel_check_primary_plane;
 
-	if (INTEL_GEN(dev_priv) >= 9) {
+	if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) {
+		intel_primary_formats = skl_primary_formats;
+		num_formats = ARRAY_SIZE(skl_primary_formats);
+		modifiers = skl_format_modifiers_ccs;
+
+		primary->update_plane = skylake_update_primary_plane;
+		primary->disable_plane = skylake_disable_primary_plane;
+	} else if (INTEL_GEN(dev_priv) >= 9) {
 		intel_primary_formats = skl_primary_formats;
 		num_formats = ARRAY_SIZE(skl_primary_formats);
+		if (pipe < PIPE_C)
+			modifiers = skl_format_modifiers_ccs;
+		else
+			modifiers = skl_format_modifiers_noccs;
 
 		primary->update_plane = skylake_update_primary_plane;
 		primary->disable_plane = skylake_disable_primary_plane;
 	} else if (INTEL_GEN(dev_priv) >= 4) {
 		intel_primary_formats = i965_primary_formats;
 		num_formats = ARRAY_SIZE(i965_primary_formats);
+		modifiers = i9xx_format_modifiers;
 
 		primary->update_plane = i9xx_update_primary_plane;
 		primary->disable_plane = i9xx_disable_primary_plane;
 	} else {
 		intel_primary_formats = i8xx_primary_formats;
 		num_formats = ARRAY_SIZE(i8xx_primary_formats);
+		modifiers = i9xx_format_modifiers;
 
 		primary->update_plane = i9xx_update_primary_plane;
 		primary->disable_plane = i9xx_disable_primary_plane;
@@ -13772,18 +13249,21 @@ intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
 		ret = drm_universal_plane_init(&dev_priv->drm, &primary->base,
 					       0, &intel_plane_funcs,
 					       intel_primary_formats, num_formats,
+					       modifiers,
 					       DRM_PLANE_TYPE_PRIMARY,
 					       "plane 1%c", pipe_name(pipe));
 	else if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
 		ret = drm_universal_plane_init(&dev_priv->drm, &primary->base,
 					       0, &intel_plane_funcs,
 					       intel_primary_formats, num_formats,
+					       modifiers,
 					       DRM_PLANE_TYPE_PRIMARY,
 					       "primary %c", pipe_name(pipe));
 	else
 		ret = drm_universal_plane_init(&dev_priv->drm, &primary->base,
 					       0, &intel_plane_funcs,
 					       intel_primary_formats, num_formats,
+					       modifiers,
 					       DRM_PLANE_TYPE_PRIMARY,
 					       "plane %c", plane_name(primary->plane));
 	if (ret)
@@ -13869,6 +13349,7 @@ intel_cursor_plane_create(struct drm_i915_private *dev_priv,
 				       0, &intel_cursor_plane_funcs,
 				       intel_cursor_formats,
 				       ARRAY_SIZE(intel_cursor_formats),
+				       cursor_format_modifiers,
 				       DRM_PLANE_TYPE_CURSOR,
 				       "cursor %c", pipe_name(pipe));
 	if (ret)
@@ -14391,10 +13872,12 @@ static int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
 				  struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
+	struct drm_framebuffer *fb = &intel_fb->base;
 	struct drm_format_name_buf format_name;
-	u32 pitch_limit, stride_alignment;
+	u32 pitch_limit;
 	unsigned int tiling, stride;
 	int ret = -EINVAL;
+	int i;
 
 	i915_gem_object_lock(obj);
 	obj->framebuffer_references++;
@@ -14423,6 +13906,19 @@ static int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
 
 	/* Passed in modifier sanity checking. */
 	switch (mode_cmd->modifier[0]) {
+	case I915_FORMAT_MOD_Y_TILED_CCS:
+	case I915_FORMAT_MOD_Yf_TILED_CCS:
+		switch (mode_cmd->pixel_format) {
+		case DRM_FORMAT_XBGR8888:
+		case DRM_FORMAT_ABGR8888:
+		case DRM_FORMAT_XRGB8888:
+		case DRM_FORMAT_ARGB8888:
+			break;
+		default:
+			DRM_DEBUG_KMS("RC supported only with RGB8888 formats\n");
+			goto err;
+		}
+		/* fall through */
 	case I915_FORMAT_MOD_Y_TILED:
 	case I915_FORMAT_MOD_Yf_TILED:
 		if (INTEL_GEN(dev_priv) < 9) {
@@ -14527,25 +14023,46 @@ static int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
 	if (mode_cmd->offsets[0] != 0)
 		goto err;
 
-	drm_helper_mode_fill_fb_struct(&dev_priv->drm,
-				       &intel_fb->base, mode_cmd);
+	drm_helper_mode_fill_fb_struct(&dev_priv->drm, fb, mode_cmd);
 
-	stride_alignment = intel_fb_stride_alignment(&intel_fb->base, 0);
-	if (mode_cmd->pitches[0] & (stride_alignment - 1)) {
-		DRM_DEBUG_KMS("pitch (%d) must be at least %u byte aligned\n",
-			      mode_cmd->pitches[0], stride_alignment);
-		goto err;
+	for (i = 0; i < fb->format->num_planes; i++) {
+		u32 stride_alignment;
+
+		if (mode_cmd->handles[i] != mode_cmd->handles[0]) {
+			DRM_DEBUG_KMS("bad plane %d handle\n", i);
+			goto err;
+		}
+
+		stride_alignment = intel_fb_stride_alignment(fb, i);
+
+		/*
+		 * Display WA #0531: skl,bxt,kbl,glk
+		 *
+		 * Render decompression and plane width > 3840
+		 * combined with horizontal panning requires the
+		 * plane stride to be a multiple of 4. We'll just
+		 * require the entire fb to accommodate that to avoid
+		 * potential runtime errors at plane configuration time.
+		 */
+		if (IS_GEN9(dev_priv) && i == 0 && fb->width > 3840 &&
+		    (fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
+		     fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS))
+			stride_alignment *= 4;
+
+		if (fb->pitches[i] & (stride_alignment - 1)) {
+			DRM_DEBUG_KMS("plane %d pitch (%d) must be at least %u byte aligned\n",
+				      i, fb->pitches[i], stride_alignment);
+			goto err;
+		}
 	}
 
 	intel_fb->obj = obj;
 
-	ret = intel_fill_fb_info(dev_priv, &intel_fb->base);
+	ret = intel_fill_fb_info(dev_priv, fb);
 	if (ret)
 		goto err;
 
-	ret = drm_framebuffer_init(obj->base.dev,
-				   &intel_fb->base,
-				   &intel_fb_funcs);
+	ret = drm_framebuffer_init(&dev_priv->drm, fb, &intel_fb_funcs);
 	if (ret) {
 		DRM_ERROR("framebuffer init failed %d\n", ret);
 		goto err;
@@ -14593,6 +14110,7 @@ static void intel_atomic_state_free(struct drm_atomic_state *state)
 
 static const struct drm_mode_config_funcs intel_mode_funcs = {
 	.fb_create = intel_user_framebuffer_create,
+	.get_format_info = intel_get_format_info,
 	.output_poll_changed = intel_fbdev_output_poll_changed,
 	.atomic_check = intel_atomic_check,
 	.atomic_commit = intel_atomic_commit,
@@ -14692,34 +14210,6 @@ void intel_init_display_hooks(struct drm_i915_private *dev_priv)
 		dev_priv->display.update_crtcs = skl_update_crtcs;
 	else
 		dev_priv->display.update_crtcs = intel_update_crtcs;
-
-	switch (INTEL_INFO(dev_priv)->gen) {
-	case 2:
-		dev_priv->display.queue_flip = intel_gen2_queue_flip;
-		break;
-
-	case 3:
-		dev_priv->display.queue_flip = intel_gen3_queue_flip;
-		break;
-
-	case 4:
-	case 5:
-		dev_priv->display.queue_flip = intel_gen4_queue_flip;
-		break;
-
-	case 6:
-		dev_priv->display.queue_flip = intel_gen6_queue_flip;
-		break;
-	case 7:
-	case 8: /* FIXME(BDW): Check that the gen8 RCS flip works. */
-		dev_priv->display.queue_flip = intel_gen7_queue_flip;
-		break;
-	case 9:
-		/* Drop through - unsupported since execlist only. */
-	default:
-		/* Default just returns -ENODEV to indicate unsupported */
-		dev_priv->display.queue_flip = intel_default_queue_flip;
-	}
 }
 
 /*
@@ -14751,6 +14241,17 @@ static void quirk_backlight_present(struct drm_device *dev)
 	DRM_INFO("applying backlight present quirk\n");
 }
 
+/* Toshiba Satellite P50-C-18C requires T12 delay to be min 800ms
+ * which is 300 ms greater than eDP spec T12 min.
+ */
+static void quirk_increase_t12_delay(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = to_i915(dev);
+
+	dev_priv->quirks |= QUIRK_INCREASE_T12_DELAY;
+	DRM_INFO("Applying T12 delay quirk\n");
+}
+
 struct intel_quirk {
 	int device;
 	int subsystem_vendor;
@@ -14834,6 +14335,9 @@ static struct intel_quirk intel_quirks[] = {
 
 	/* Dell Chromebook 11 (2015 version) */
 	{ 0x0a16, 0x1028, 0x0a35, quirk_backlight_present },
+
+	/* Toshiba Satellite P50-C-18C */
+	{ 0x191B, 0x1179, 0xF840, quirk_increase_t12_delay },
 };
 
 static void intel_init_quirks(struct drm_device *dev)
@@ -15636,7 +15140,7 @@ intel_modeset_setup_hw_state(struct drm_device *dev,
 	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		vlv_wm_get_hw_state(dev);
 		vlv_wm_sanitize(dev_priv);
-	} else if (IS_GEN9(dev_priv)) {
+	} else if (INTEL_GEN(dev_priv) >= 9) {
 		skl_wm_get_hw_state(dev);
 	} else if (HAS_PCH_SPLIT(dev_priv)) {
 		ilk_wm_get_hw_state(dev);
@@ -15743,6 +15247,9 @@ void intel_modeset_cleanup(struct drm_device *dev)
 	 */
 	drm_kms_helper_poll_fini(dev);
 
+	/* poll work can call into fbdev, hence clean that up afterwards */
+	intel_fbdev_fini(dev_priv);
+
 	intel_unregister_dsm_handler();
 
 	intel_fbc_global_disable(dev_priv);
@@ -15862,7 +15369,8 @@ intel_display_capture_error_state(struct drm_i915_private *dev_priv)
 		return NULL;
 
 	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
-		error->power_well_driver = I915_READ(HSW_PWR_WELL_DRIVER);
+		error->power_well_driver =
+			I915_READ(HSW_PWR_WELL_CTL_DRIVER(HSW_DISP_PW_GLOBAL));
 
 	for_each_pipe(dev_priv, i) {
 		error->pipe[i].power_domain_on =
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index 64fa774c855b..64134947c0aa 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -97,6 +97,9 @@ static const int bxt_rates[] = { 162000, 216000, 243000, 270000,
 				  324000, 432000, 540000 };
 static const int skl_rates[] = { 162000, 216000, 270000,
 				  324000, 432000, 540000 };
+static const int cnl_rates[] = { 162000, 216000, 270000,
+				 324000, 432000, 540000,
+				 648000, 810000 };
 static const int default_rates[] = { 162000, 270000, 540000 };
 
 /**
@@ -229,8 +232,10 @@ intel_dp_set_source_rates(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
+	enum port port = dig_port->port;
 	const int *source_rates;
 	int size;
+	u32 voltage;
 
 	/* This should only be done once */
 	WARN_ON(intel_dp->source_rates || intel_dp->num_source_rates);
@@ -238,6 +243,13 @@ intel_dp_set_source_rates(struct intel_dp *intel_dp)
 	if (IS_GEN9_LP(dev_priv)) {
 		source_rates = bxt_rates;
 		size = ARRAY_SIZE(bxt_rates);
+	} else if (IS_CANNONLAKE(dev_priv)) {
+		source_rates = cnl_rates;
+		size = ARRAY_SIZE(cnl_rates);
+		voltage = I915_READ(CNL_PORT_COMP_DW3) & VOLTAGE_INFO_MASK;
+		if (port == PORT_A || port == PORT_D ||
+		    voltage == VOLTAGE_INFO_0_85V)
+			size -= 2;
 	} else if (IS_GEN9_BC(dev_priv)) {
 		source_rates = skl_rates;
 		size = ARRAY_SIZE(skl_rates);
@@ -322,19 +334,20 @@ static int intel_dp_common_len_rate_limit(struct intel_dp *intel_dp,
 	return 0;
 }
 
-static bool intel_dp_link_params_valid(struct intel_dp *intel_dp)
+static bool intel_dp_link_params_valid(struct intel_dp *intel_dp, int link_rate,
+				       uint8_t lane_count)
 {
 	/*
 	 * FIXME: we need to synchronize the current link parameters with
 	 * hardware readout. Currently fast link training doesn't work on
 	 * boot-up.
 	 */
-	if (intel_dp->link_rate == 0 ||
-	    intel_dp->link_rate > intel_dp->max_link_rate)
+	if (link_rate == 0 ||
+	    link_rate > intel_dp->max_link_rate)
 		return false;
 
-	if (intel_dp->lane_count == 0 ||
-	    intel_dp->lane_count > intel_dp_max_lane_count(intel_dp))
+	if (lane_count == 0 ||
+	    lane_count > intel_dp_max_lane_count(intel_dp))
 		return false;
 
 	return true;
@@ -1606,6 +1619,23 @@ static int intel_dp_compute_bpp(struct intel_dp *intel_dp,
 	return bpp;
 }
 
+static bool intel_edp_compare_alt_mode(struct drm_display_mode *m1,
+				       struct drm_display_mode *m2)
+{
+	bool bres = false;
+
+	if (m1 && m2)
+		bres = (m1->hdisplay == m2->hdisplay &&
+			m1->hsync_start == m2->hsync_start &&
+			m1->hsync_end == m2->hsync_end &&
+			m1->htotal == m2->htotal &&
+			m1->vdisplay == m2->vdisplay &&
+			m1->vsync_start == m2->vsync_start &&
+			m1->vsync_end == m2->vsync_end &&
+			m1->vtotal == m2->vtotal);
+	return bres;
+}
+
 bool
 intel_dp_compute_config(struct intel_encoder *encoder,
 			struct intel_crtc_state *pipe_config,
@@ -1652,8 +1682,16 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 		pipe_config->has_audio = intel_conn_state->force_audio == HDMI_AUDIO_ON;
 
 	if (is_edp(intel_dp) && intel_connector->panel.fixed_mode) {
-		intel_fixed_panel_mode(intel_connector->panel.fixed_mode,
-				       adjusted_mode);
+		struct drm_display_mode *panel_mode =
+			intel_connector->panel.alt_fixed_mode;
+		struct drm_display_mode *req_mode = &pipe_config->base.mode;
+
+		if (!intel_edp_compare_alt_mode(req_mode, panel_mode))
+			panel_mode = intel_connector->panel.fixed_mode;
+
+		drm_mode_debug_printmodeline(panel_mode);
+
+		intel_fixed_panel_mode(panel_mode, adjusted_mode);
 
 		if (INTEL_GEN(dev_priv) >= 9) {
 			int ret;
@@ -1677,12 +1715,18 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 	if (intel_dp->compliance.test_type == DP_TEST_LINK_TRAINING) {
 		int index;
 
-		index = intel_dp_rate_index(intel_dp->common_rates,
-					    intel_dp->num_common_rates,
-					    intel_dp->compliance.test_link_rate);
-		if (index >= 0)
-			min_clock = max_clock = index;
-		min_lane_count = max_lane_count = intel_dp->compliance.test_lane_count;
+		/* Validate the compliance test data since max values
+		 * might have changed due to link train fallback.
+		 */
+		if (intel_dp_link_params_valid(intel_dp, intel_dp->compliance.test_link_rate,
+					       intel_dp->compliance.test_lane_count)) {
+			index = intel_dp_rate_index(intel_dp->common_rates,
+						    intel_dp->num_common_rates,
+						    intel_dp->compliance.test_link_rate);
+			if (index >= 0)
+				min_clock = max_clock = index;
+			min_lane_count = max_lane_count = intel_dp->compliance.test_lane_count;
+		}
 	}
 	DRM_DEBUG_KMS("DP link computation with max lane count %i "
 		      "max bw %d pixel clock %iKHz\n",
@@ -3963,8 +4007,7 @@ intel_dp_get_sink_irq_esi(struct intel_dp *intel_dp, u8 *sink_irq_vector)
 static uint8_t intel_dp_autotest_link_training(struct intel_dp *intel_dp)
 {
 	int status = 0;
-	int min_lane_count = 1;
-	int link_rate_index, test_link_rate;
+	int test_link_rate;
 	uint8_t test_lane_count, test_link_bw;
 	/* (DP CTS 1.2)
 	 * 4.3.1.11
@@ -3978,10 +4021,6 @@ static uint8_t intel_dp_autotest_link_training(struct intel_dp *intel_dp)
 		return DP_TEST_NAK;
 	}
 	test_lane_count &= DP_MAX_LANE_COUNT_MASK;
-	/* Validate the requested lane count */
-	if (test_lane_count < min_lane_count ||
-	    test_lane_count > intel_dp->max_link_lane_count)
-		return DP_TEST_NAK;
 
 	status = drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_LINK_RATE,
 				   &test_link_bw);
@@ -3989,12 +4028,11 @@ static uint8_t intel_dp_autotest_link_training(struct intel_dp *intel_dp)
 		DRM_DEBUG_KMS("Link Rate read failed\n");
 		return DP_TEST_NAK;
 	}
-	/* Validate the requested link rate */
 	test_link_rate = drm_dp_bw_code_to_link_rate(test_link_bw);
-	link_rate_index = intel_dp_rate_index(intel_dp->common_rates,
-					      intel_dp->num_common_rates,
-					      test_link_rate);
-	if (link_rate_index < 0)
+
+	/* Validate the requested link rate and lane count */
+	if (!intel_dp_link_params_valid(intel_dp, test_link_rate,
+					test_lane_count))
 		return DP_TEST_NAK;
 
 	intel_dp->compliance.test_lane_count = test_lane_count;
@@ -4263,7 +4301,8 @@ intel_dp_check_link_status(struct intel_dp *intel_dp)
 	 * Validate the cached values of intel_dp->link_rate and
 	 * intel_dp->lane_count before attempting to retrain.
 	 */
-	if (!intel_dp_link_params_valid(intel_dp))
+	if (!intel_dp_link_params_valid(intel_dp, intel_dp->link_rate,
+					intel_dp->lane_count))
 		return;
 
 	/* Retrain if Channel EQ or CR not ok */
@@ -4418,8 +4457,6 @@ static bool ibx_digital_port_connected(struct drm_i915_private *dev_priv,
 	u32 bit;
 
 	switch (port->port) {
-	case PORT_A:
-		return true;
 	case PORT_B:
 		bit = SDE_PORTB_HOTPLUG;
 		break;
@@ -4443,8 +4480,6 @@ static bool cpt_digital_port_connected(struct drm_i915_private *dev_priv,
 	u32 bit;
 
 	switch (port->port) {
-	case PORT_A:
-		return true;
 	case PORT_B:
 		bit = SDE_PORTB_HOTPLUG_CPT;
 		break;
@@ -4454,12 +4489,28 @@ static bool cpt_digital_port_connected(struct drm_i915_private *dev_priv,
 	case PORT_D:
 		bit = SDE_PORTD_HOTPLUG_CPT;
 		break;
+	default:
+		MISSING_CASE(port->port);
+		return false;
+	}
+
+	return I915_READ(SDEISR) & bit;
+}
+
+static bool spt_digital_port_connected(struct drm_i915_private *dev_priv,
+				       struct intel_digital_port *port)
+{
+	u32 bit;
+
+	switch (port->port) {
+	case PORT_A:
+		bit = SDE_PORTA_HOTPLUG_SPT;
+		break;
 	case PORT_E:
 		bit = SDE_PORTE_HOTPLUG_SPT;
 		break;
 	default:
-		MISSING_CASE(port->port);
-		return false;
+		return cpt_digital_port_connected(dev_priv, port);
 	}
 
 	return I915_READ(SDEISR) & bit;
@@ -4511,6 +4562,42 @@ static bool gm45_digital_port_connected(struct drm_i915_private *dev_priv,
 	return I915_READ(PORT_HOTPLUG_STAT) & bit;
 }
 
+static bool ilk_digital_port_connected(struct drm_i915_private *dev_priv,
+				       struct intel_digital_port *port)
+{
+	if (port->port == PORT_A)
+		return I915_READ(DEISR) & DE_DP_A_HOTPLUG;
+	else
+		return ibx_digital_port_connected(dev_priv, port);
+}
+
+static bool snb_digital_port_connected(struct drm_i915_private *dev_priv,
+				       struct intel_digital_port *port)
+{
+	if (port->port == PORT_A)
+		return I915_READ(DEISR) & DE_DP_A_HOTPLUG;
+	else
+		return cpt_digital_port_connected(dev_priv, port);
+}
+
+static bool ivb_digital_port_connected(struct drm_i915_private *dev_priv,
+				       struct intel_digital_port *port)
+{
+	if (port->port == PORT_A)
+		return I915_READ(DEISR) & DE_DP_A_HOTPLUG_IVB;
+	else
+		return cpt_digital_port_connected(dev_priv, port);
+}
+
+static bool bdw_digital_port_connected(struct drm_i915_private *dev_priv,
+				       struct intel_digital_port *port)
+{
+	if (port->port == PORT_A)
+		return I915_READ(GEN8_DE_PORT_ISR) & GEN8_PORT_DP_A_HOTPLUG;
+	else
+		return cpt_digital_port_connected(dev_priv, port);
+}
+
 static bool bxt_digital_port_connected(struct drm_i915_private *dev_priv,
 				       struct intel_digital_port *intel_dig_port)
 {
@@ -4518,7 +4605,7 @@ static bool bxt_digital_port_connected(struct drm_i915_private *dev_priv,
 	enum port port;
 	u32 bit;
 
-	intel_hpd_pin_to_port(intel_encoder->hpd_pin, &port);
+	port = intel_hpd_pin_to_port(intel_encoder->hpd_pin);
 	switch (port) {
 	case PORT_A:
 		bit = BXT_DE_PORT_HP_DDIA;
@@ -4547,16 +4634,25 @@ static bool bxt_digital_port_connected(struct drm_i915_private *dev_priv,
 bool intel_digital_port_connected(struct drm_i915_private *dev_priv,
 				  struct intel_digital_port *port)
 {
-	if (HAS_PCH_IBX(dev_priv))
-		return ibx_digital_port_connected(dev_priv, port);
-	else if (HAS_PCH_SPLIT(dev_priv))
-		return cpt_digital_port_connected(dev_priv, port);
+	if (HAS_GMCH_DISPLAY(dev_priv)) {
+		if (IS_GM45(dev_priv))
+			return gm45_digital_port_connected(dev_priv, port);
+		else
+			return g4x_digital_port_connected(dev_priv, port);
+	}
+
+	if (IS_GEN5(dev_priv))
+		return ilk_digital_port_connected(dev_priv, port);
+	else if (IS_GEN6(dev_priv))
+		return snb_digital_port_connected(dev_priv, port);
+	else if (IS_GEN7(dev_priv))
+		return ivb_digital_port_connected(dev_priv, port);
+	else if (IS_GEN8(dev_priv))
+		return bdw_digital_port_connected(dev_priv, port);
 	else if (IS_GEN9_LP(dev_priv))
 		return bxt_digital_port_connected(dev_priv, port);
-	else if (IS_GM45(dev_priv))
-		return gm45_digital_port_connected(dev_priv, port);
 	else
-		return g4x_digital_port_connected(dev_priv, port);
+		return spt_digital_port_connected(dev_priv, port);
 }
 
 static struct edid *
@@ -4950,10 +5046,8 @@ void intel_dp_encoder_reset(struct drm_encoder *encoder)
 }
 
 static const struct drm_connector_funcs intel_dp_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.force = intel_dp_force,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.set_property = drm_atomic_helper_connector_set_property,
 	.atomic_get_property = intel_digital_connector_atomic_get_property,
 	.atomic_set_property = intel_digital_connector_atomic_set_property,
 	.late_register = intel_dp_connector_register,
@@ -5121,12 +5215,8 @@ intel_pps_readout_hw_state(struct drm_i915_private *dev_priv,
 		   PANEL_POWER_DOWN_DELAY_SHIFT;
 
 	if (IS_GEN9_LP(dev_priv) || HAS_PCH_CNP(dev_priv)) {
-		u16 tmp = (pp_ctl & BXT_POWER_CYCLE_DELAY_MASK) >>
-			BXT_POWER_CYCLE_DELAY_SHIFT;
-		if (tmp > 0)
-			seq->t11_t12 = (tmp - 1) * 1000;
-		else
-			seq->t11_t12 = 0;
+		seq->t11_t12 = ((pp_ctl & BXT_POWER_CYCLE_DELAY_MASK) >>
+				BXT_POWER_CYCLE_DELAY_SHIFT) * 1000;
 	} else {
 		seq->t11_t12 = ((pp_div & PANEL_POWER_CYCLE_DELAY_MASK) >>
 		       PANEL_POWER_CYCLE_DELAY_SHIFT) * 1000;
@@ -5177,6 +5267,21 @@ intel_dp_init_panel_power_sequencer(struct drm_device *dev,
 	intel_pps_dump_state("cur", &cur);
 
 	vbt = dev_priv->vbt.edp.pps;
+	/* On Toshiba Satellite P50-C-18C system the VBT T12 delay
+	 * of 500ms appears to be too short. Ocassionally the panel
+	 * just fails to power back on. Increasing the delay to 800ms
+	 * seems sufficient to avoid this problem.
+	 */
+	if (dev_priv->quirks & QUIRK_INCREASE_T12_DELAY) {
+		vbt.t11_t12 = max_t(u16, vbt.t11_t12, 900 * 10);
+		DRM_DEBUG_KMS("Increasing T12 panel delay as per the quirk to %d\n",
+			      vbt.t11_t12);
+	}
+	/* T11_T12 delay is special and actually in units of 100ms, but zero
+	 * based in the hw (so we need to add 100 ms). But the sw vbt
+	 * table multiplies it with 1000 to make it in units of 100usec,
+	 * too. */
+	vbt.t11_t12 += 100 * 10;
 
 	/* Upper limits from eDP 1.3 spec. Note that we use the clunky units of
 	 * our hw here, which are all in 100usec. */
@@ -5280,7 +5385,7 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
 	if (IS_GEN9_LP(dev_priv) || HAS_PCH_CNP(dev_priv)) {
 		pp_div = I915_READ(regs.pp_ctrl);
 		pp_div &= ~BXT_POWER_CYCLE_DELAY_MASK;
-		pp_div |= (DIV_ROUND_UP((seq->t11_t12 + 1), 1000)
+		pp_div |= (DIV_ROUND_UP(seq->t11_t12, 1000)
 				<< BXT_POWER_CYCLE_DELAY_SHIFT);
 	} else {
 		pp_div = ((100 * div)/2 - 1) << PP_REFERENCE_DIVIDER_SHIFT;
@@ -5714,6 +5819,7 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
 	struct drm_device *dev = intel_encoder->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_display_mode *fixed_mode = NULL;
+	struct drm_display_mode *alt_fixed_mode = NULL;
 	struct drm_display_mode *downclock_mode = NULL;
 	bool has_dpcd;
 	struct drm_display_mode *scan;
@@ -5769,13 +5875,14 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
 	}
 	intel_connector->edid = edid;
 
-	/* prefer fixed mode from EDID if available */
+	/* prefer fixed mode from EDID if available, save an alt mode also */
 	list_for_each_entry(scan, &connector->probed_modes, head) {
 		if ((scan->type & DRM_MODE_TYPE_PREFERRED)) {
 			fixed_mode = drm_mode_duplicate(dev, scan);
 			downclock_mode = intel_dp_drrs_init(
 						intel_connector, fixed_mode);
-			break;
+		} else if (!alt_fixed_mode) {
+			alt_fixed_mode = drm_mode_duplicate(dev, scan);
 		}
 	}
 
@@ -5812,7 +5919,8 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
 			      pipe_name(pipe));
 	}
 
-	intel_panel_init(&intel_connector->panel, fixed_mode, downclock_mode);
+	intel_panel_init(&intel_connector->panel, fixed_mode, alt_fixed_mode,
+			 downclock_mode);
 	intel_connector->panel.backlight.power = intel_edp_backlight_power;
 	intel_panel_setup_backlight(connector, pipe);
 
@@ -5838,26 +5946,22 @@ intel_dp_init_connector_port_info(struct intel_digital_port *intel_dig_port)
 	struct intel_encoder *encoder = &intel_dig_port->base;
 	struct intel_dp *intel_dp = &intel_dig_port->dp;
 
+	encoder->hpd_pin = intel_hpd_pin(intel_dig_port->port);
+
 	switch (intel_dig_port->port) {
 	case PORT_A:
-		encoder->hpd_pin = HPD_PORT_A;
 		intel_dp->aux_power_domain = POWER_DOMAIN_AUX_A;
 		break;
 	case PORT_B:
-		encoder->hpd_pin = HPD_PORT_B;
 		intel_dp->aux_power_domain = POWER_DOMAIN_AUX_B;
 		break;
 	case PORT_C:
-		encoder->hpd_pin = HPD_PORT_C;
 		intel_dp->aux_power_domain = POWER_DOMAIN_AUX_C;
 		break;
 	case PORT_D:
-		encoder->hpd_pin = HPD_PORT_D;
 		intel_dp->aux_power_domain = POWER_DOMAIN_AUX_D;
 		break;
 	case PORT_E:
-		encoder->hpd_pin = HPD_PORT_E;
-
 		/* FIXME: Check VBT for actual wiring of PORT E */
 		intel_dp->aux_power_domain = POWER_DOMAIN_AUX_D;
 		break;
diff --git a/drivers/gpu/drm/i915/intel_dp_aux_backlight.c b/drivers/gpu/drm/i915/intel_dp_aux_backlight.c
index 228ca06d9f0b..d2830ba3162e 100644
--- a/drivers/gpu/drm/i915/intel_dp_aux_backlight.c
+++ b/drivers/gpu/drm/i915/intel_dp_aux_backlight.c
@@ -98,13 +98,87 @@ intel_dp_aux_set_backlight(const struct drm_connector_state *conn_state, u32 lev
 	}
 }
 
+/*
+ * Set PWM Frequency divider to match desired frequency in vbt.
+ * The PWM Frequency is calculated as 27Mhz / (F x P).
+ * - Where F = PWM Frequency Pre-Divider value programmed by field 7:0 of the
+ *             EDP_BACKLIGHT_FREQ_SET register (DPCD Address 00728h)
+ * - Where P = 2^Pn, where Pn is the value programmed by field 4:0 of the
+ *             EDP_PWMGEN_BIT_COUNT register (DPCD Address 00724h)
+ */
+static bool intel_dp_aux_set_pwm_freq(struct intel_connector *connector)
+{
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct intel_dp *intel_dp = enc_to_intel_dp(&connector->encoder->base);
+	int freq, fxp, fxp_min, fxp_max, fxp_actual, f = 1;
+	u8 pn, pn_min, pn_max;
+
+	/* Find desired value of (F x P)
+	 * Note that, if F x P is out of supported range, the maximum value or
+	 * minimum value will applied automatically. So no need to check that.
+	 */
+	freq = dev_priv->vbt.backlight.pwm_freq_hz;
+	DRM_DEBUG_KMS("VBT defined backlight frequency %u Hz\n", freq);
+	if (!freq) {
+		DRM_DEBUG_KMS("Use panel default backlight frequency\n");
+		return false;
+	}
+
+	fxp = DIV_ROUND_CLOSEST(KHz(DP_EDP_BACKLIGHT_FREQ_BASE_KHZ), freq);
+
+	/* Use highest possible value of Pn for more granularity of brightness
+	 * adjustment while satifying the conditions below.
+	 * - Pn is in the range of Pn_min and Pn_max
+	 * - F is in the range of 1 and 255
+	 * - FxP is within 25% of desired value.
+	 *   Note: 25% is arbitrary value and may need some tweak.
+	 */
+	if (drm_dp_dpcd_readb(&intel_dp->aux,
+			       DP_EDP_PWMGEN_BIT_COUNT_CAP_MIN, &pn_min) != 1) {
+		DRM_DEBUG_KMS("Failed to read pwmgen bit count cap min\n");
+		return false;
+	}
+	if (drm_dp_dpcd_readb(&intel_dp->aux,
+			       DP_EDP_PWMGEN_BIT_COUNT_CAP_MAX, &pn_max) != 1) {
+		DRM_DEBUG_KMS("Failed to read pwmgen bit count cap max\n");
+		return false;
+	}
+	pn_min &= DP_EDP_PWMGEN_BIT_COUNT_MASK;
+	pn_max &= DP_EDP_PWMGEN_BIT_COUNT_MASK;
+
+	fxp_min = DIV_ROUND_CLOSEST(fxp * 3, 4);
+	fxp_max = DIV_ROUND_CLOSEST(fxp * 5, 4);
+	if (fxp_min < (1 << pn_min) || (255 << pn_max) < fxp_max) {
+		DRM_DEBUG_KMS("VBT defined backlight frequency out of range\n");
+		return false;
+	}
+
+	for (pn = pn_max; pn >= pn_min; pn--) {
+		f = clamp(DIV_ROUND_CLOSEST(fxp, 1 << pn), 1, 255);
+		fxp_actual = f << pn;
+		if (fxp_min <= fxp_actual && fxp_actual <= fxp_max)
+			break;
+	}
+
+	if (drm_dp_dpcd_writeb(&intel_dp->aux,
+			       DP_EDP_PWMGEN_BIT_COUNT, pn) < 0) {
+		DRM_DEBUG_KMS("Failed to write aux pwmgen bit count\n");
+		return false;
+	}
+	if (drm_dp_dpcd_writeb(&intel_dp->aux,
+			       DP_EDP_BACKLIGHT_FREQ_SET, (u8) f) < 0) {
+		DRM_DEBUG_KMS("Failed to write aux backlight freq\n");
+		return false;
+	}
+	return true;
+}
+
 static void intel_dp_aux_enable_backlight(const struct intel_crtc_state *crtc_state,
 					  const struct drm_connector_state *conn_state)
 {
 	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	struct intel_dp *intel_dp = enc_to_intel_dp(&connector->encoder->base);
-	uint8_t dpcd_buf = 0;
-	uint8_t edp_backlight_mode = 0;
+	uint8_t dpcd_buf, new_dpcd_buf, edp_backlight_mode;
 
 	if (drm_dp_dpcd_readb(&intel_dp->aux,
 			DP_EDP_BACKLIGHT_MODE_SET_REGISTER, &dpcd_buf) != 1) {
@@ -113,18 +187,15 @@ static void intel_dp_aux_enable_backlight(const struct intel_crtc_state *crtc_st
 		return;
 	}
 
+	new_dpcd_buf = dpcd_buf;
 	edp_backlight_mode = dpcd_buf & DP_EDP_BACKLIGHT_CONTROL_MODE_MASK;
 
 	switch (edp_backlight_mode) {
 	case DP_EDP_BACKLIGHT_CONTROL_MODE_PWM:
 	case DP_EDP_BACKLIGHT_CONTROL_MODE_PRESET:
 	case DP_EDP_BACKLIGHT_CONTROL_MODE_PRODUCT:
-		dpcd_buf &= ~DP_EDP_BACKLIGHT_CONTROL_MODE_MASK;
-		dpcd_buf |= DP_EDP_BACKLIGHT_CONTROL_MODE_DPCD;
-		if (drm_dp_dpcd_writeb(&intel_dp->aux,
-			DP_EDP_BACKLIGHT_MODE_SET_REGISTER, dpcd_buf) < 0) {
-			DRM_DEBUG_KMS("Failed to write aux backlight mode\n");
-		}
+		new_dpcd_buf &= ~DP_EDP_BACKLIGHT_CONTROL_MODE_MASK;
+		new_dpcd_buf |= DP_EDP_BACKLIGHT_CONTROL_MODE_DPCD;
 		break;
 
 	/* Do nothing when it is already DPCD mode */
@@ -133,6 +204,17 @@ static void intel_dp_aux_enable_backlight(const struct intel_crtc_state *crtc_st
 		break;
 	}
 
+	if (intel_dp->edp_dpcd[2] & DP_EDP_BACKLIGHT_FREQ_AUX_SET_CAP)
+		if (intel_dp_aux_set_pwm_freq(connector))
+			new_dpcd_buf |= DP_EDP_BACKLIGHT_FREQ_AUX_SET_ENABLE;
+
+	if (new_dpcd_buf != dpcd_buf) {
+		if (drm_dp_dpcd_writeb(&intel_dp->aux,
+			DP_EDP_BACKLIGHT_MODE_SET_REGISTER, new_dpcd_buf) < 0) {
+			DRM_DEBUG_KMS("Failed to write aux backlight mode\n");
+		}
+	}
+
 	set_aux_backlight_enable(intel_dp, true);
 	intel_dp_aux_set_backlight(conn_state, connector->panel.backlight.level);
 }
diff --git a/drivers/gpu/drm/i915/intel_dp_link_training.c b/drivers/gpu/drm/i915/intel_dp_link_training.c
index b79c1c0e404c..05907fa8a553 100644
--- a/drivers/gpu/drm/i915/intel_dp_link_training.c
+++ b/drivers/gpu/drm/i915/intel_dp_link_training.c
@@ -321,12 +321,16 @@ intel_dp_start_link_train(struct intel_dp *intel_dp)
 	if (!intel_dp_link_training_channel_equalization(intel_dp))
 		goto failure_handling;
 
-	DRM_DEBUG_KMS("Link Training Passed at Link Rate = %d, Lane count = %d",
+	DRM_DEBUG_KMS("[CONNECTOR:%d:%s] Link Training Passed at Link Rate = %d, Lane count = %d",
+		      intel_connector->base.base.id,
+		      intel_connector->base.name,
 		      intel_dp->link_rate, intel_dp->lane_count);
 	return;
 
  failure_handling:
-	DRM_DEBUG_KMS("Link Training failed at link rate = %d, lane count = %d",
+	DRM_DEBUG_KMS("[CONNECTOR:%d:%s] Link Training failed at link rate = %d, lane count = %d",
+		      intel_connector->base.base.id,
+		      intel_connector->base.name,
 		      intel_dp->link_rate, intel_dp->lane_count);
 	if (!intel_dp_get_link_train_fallback_values(intel_dp,
 						     intel_dp->link_rate,
diff --git a/drivers/gpu/drm/i915/intel_dp_mst.c b/drivers/gpu/drm/i915/intel_dp_mst.c
index 2cf046beae0f..93fc8ab9bb31 100644
--- a/drivers/gpu/drm/i915/intel_dp_mst.c
+++ b/drivers/gpu/drm/i915/intel_dp_mst.c
@@ -346,10 +346,8 @@ intel_dp_mst_connector_destroy(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs intel_dp_mst_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.detect = intel_dp_mst_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.set_property = drm_atomic_helper_connector_set_property,
 	.late_register = intel_connector_register,
 	.early_unregister = intel_connector_unregister,
 	.destroy = intel_dp_mst_connector_destroy,
@@ -372,6 +370,9 @@ intel_dp_mst_mode_valid(struct drm_connector *connector,
 	int bpp = 24; /* MST uses fixed bpp */
 	int max_rate, mode_rate, max_lanes, max_link_clock;
 
+	if (!intel_dp)
+		return MODE_ERROR;
+
 	max_link_clock = intel_dp_max_link_rate(intel_dp);
 	max_lanes = intel_dp_max_lane_count(intel_dp);
 
@@ -443,28 +444,6 @@ static bool intel_dp_mst_get_hw_state(struct intel_connector *connector)
 	return false;
 }
 
-static void intel_connector_add_to_fbdev(struct intel_connector *connector)
-{
-#ifdef CONFIG_DRM_FBDEV_EMULATION
-	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
-
-	if (dev_priv->fbdev)
-		drm_fb_helper_add_one_connector(&dev_priv->fbdev->helper,
-						&connector->base);
-#endif
-}
-
-static void intel_connector_remove_from_fbdev(struct intel_connector *connector)
-{
-#ifdef CONFIG_DRM_FBDEV_EMULATION
-	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
-
-	if (dev_priv->fbdev)
-		drm_fb_helper_remove_one_connector(&dev_priv->fbdev->helper,
-						   &connector->base);
-#endif
-}
-
 static struct drm_connector *intel_dp_add_mst_connector(struct drm_dp_mst_topology_mgr *mgr, struct drm_dp_mst_port *port, const char *pathprop)
 {
 	struct intel_dp *intel_dp = container_of(mgr, struct intel_dp, mst_mgr);
@@ -500,31 +479,32 @@ static struct drm_connector *intel_dp_add_mst_connector(struct drm_dp_mst_topolo
 
 static void intel_dp_register_mst_connector(struct drm_connector *connector)
 {
-	struct intel_connector *intel_connector = to_intel_connector(connector);
-	struct drm_device *dev = connector->dev;
+	struct drm_i915_private *dev_priv = to_i915(connector->dev);
 
-	drm_modeset_lock_all(dev);
-	intel_connector_add_to_fbdev(intel_connector);
-	drm_modeset_unlock_all(dev);
+	if (dev_priv->fbdev)
+		drm_fb_helper_add_one_connector(&dev_priv->fbdev->helper,
+						connector);
 
-	drm_connector_register(&intel_connector->base);
+	drm_connector_register(connector);
 }
 
 static void intel_dp_destroy_mst_connector(struct drm_dp_mst_topology_mgr *mgr,
 					   struct drm_connector *connector)
 {
 	struct intel_connector *intel_connector = to_intel_connector(connector);
-	struct drm_device *dev = connector->dev;
+	struct drm_i915_private *dev_priv = to_i915(connector->dev);
 
 	drm_connector_unregister(connector);
 
-	/* need to nuke the connector */
-	drm_modeset_lock_all(dev);
-	intel_connector_remove_from_fbdev(intel_connector);
+	if (dev_priv->fbdev)
+		drm_fb_helper_remove_one_connector(&dev_priv->fbdev->helper,
+						   connector);
+	/* prevent race with the check in ->detect */
+	drm_modeset_lock(&connector->dev->mode_config.connection_mutex, NULL);
 	intel_connector->mst_port = NULL;
-	drm_modeset_unlock_all(dev);
+	drm_modeset_unlock(&connector->dev->mode_config.connection_mutex);
 
-	drm_connector_unreference(&intel_connector->base);
+	drm_connector_unreference(connector);
 	DRM_DEBUG_KMS("\n");
 }
 
diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.c b/drivers/gpu/drm/i915/intel_dpll_mgr.c
index 2f7b0e64f628..a2a3d93d67bd 100644
--- a/drivers/gpu/drm/i915/intel_dpll_mgr.c
+++ b/drivers/gpu/drm/i915/intel_dpll_mgr.c
@@ -2379,6 +2379,15 @@ cnl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
 	return pll;
 }
 
+static void cnl_dump_hw_state(struct drm_i915_private *dev_priv,
+			      struct intel_dpll_hw_state *hw_state)
+{
+	DRM_DEBUG_KMS("dpll_hw_state: "
+		      "cfgcr0: 0x%x, cfgcr1: 0x%x\n",
+		      hw_state->cfgcr0,
+		      hw_state->cfgcr1);
+}
+
 static const struct intel_shared_dpll_funcs cnl_ddi_pll_funcs = {
 	.enable = cnl_ddi_pll_enable,
 	.disable = cnl_ddi_pll_disable,
@@ -2395,7 +2404,7 @@ static const struct dpll_info cnl_plls[] = {
 static const struct intel_dpll_mgr cnl_pll_mgr = {
 	.dpll_info = cnl_plls,
 	.get_dpll = cnl_get_dpll,
-	.dump_hw_state = skl_dump_hw_state,
+	.dump_hw_state = cnl_dump_hw_state,
 };
 
 /**
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index d93efb49a2e2..fa47285918f4 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -265,6 +265,7 @@ struct intel_encoder {
 
 struct intel_panel {
 	struct drm_display_mode *fixed_mode;
+	struct drm_display_mode *alt_fixed_mode;
 	struct drm_display_mode *downclock_mode;
 
 	/* backlight */
@@ -780,13 +781,15 @@ struct intel_crtc_state {
 
 	/* HDMI High TMDS char rate ratio */
 	bool hdmi_high_tmds_clock_ratio;
+
+	/* output format is YCBCR 4:2:0 */
+	bool ycbcr420;
 };
 
 struct intel_crtc {
 	struct drm_crtc base;
 	enum pipe pipe;
 	enum plane plane;
-	u8 lut_r[256], lut_g[256], lut_b[256];
 	/*
 	 * Whether the crtc and the connected output pipeline is active. Implies
 	 * that crtc->enabled is set, i.e. the current mode configuration has
@@ -797,9 +800,6 @@ struct intel_crtc {
 	u8 plane_ids_mask;
 	unsigned long long enabled_power_domains;
 	struct intel_overlay *overlay;
-	struct intel_flip_work *flip_work;
-
-	atomic_t unpin_work_count;
 
 	/* Display surface base address adjustement for pageflips. Note that on
 	 * gen4+ this only adjusts up to a tile, offsets within a tile are
@@ -1132,24 +1132,6 @@ intel_get_crtc_for_plane(struct drm_i915_private *dev_priv, enum plane plane)
 	return dev_priv->plane_to_crtc_mapping[plane];
 }
 
-struct intel_flip_work {
-	struct work_struct unpin_work;
-	struct work_struct mmio_work;
-
-	struct drm_crtc *crtc;
-	struct i915_vma *old_vma;
-	struct drm_framebuffer *old_fb;
-	struct drm_i915_gem_object *pending_flip_obj;
-	struct drm_pending_vblank_event *event;
-	atomic_t pending;
-	u32 flip_count;
-	u32 gtt_offset;
-	struct drm_i915_gem_request *flip_queued_req;
-	u32 flip_queued_vblank;
-	u32 flip_ready_vblank;
-	unsigned int rotation;
-};
-
 struct intel_load_detect_pipe {
 	struct drm_atomic_state *restore_state;
 };
@@ -1211,12 +1193,12 @@ hdmi_to_dig_port(struct intel_hdmi *intel_hdmi)
 bool intel_set_cpu_fifo_underrun_reporting(struct drm_i915_private *dev_priv,
 					   enum pipe pipe, bool enable);
 bool intel_set_pch_fifo_underrun_reporting(struct drm_i915_private *dev_priv,
-					   enum transcoder pch_transcoder,
+					   enum pipe pch_transcoder,
 					   bool enable);
 void intel_cpu_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv,
 					 enum pipe pipe);
 void intel_pch_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv,
-					 enum transcoder pch_transcoder);
+					 enum pipe pch_transcoder);
 void intel_check_cpu_fifo_underruns(struct drm_i915_private *dev_priv);
 void intel_check_pch_fifo_underruns(struct drm_i915_private *dev_priv);
 
@@ -1251,9 +1233,9 @@ static inline bool intel_irqs_enabled(struct drm_i915_private *dev_priv)
 
 int intel_get_crtc_scanline(struct intel_crtc *crtc);
 void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
-				     unsigned int pipe_mask);
+				     u8 pipe_mask);
 void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv,
-				     unsigned int pipe_mask);
+				     u8 pipe_mask);
 void gen9_reset_guc_interrupts(struct drm_i915_private *dev_priv);
 void gen9_enable_guc_interrupts(struct drm_i915_private *dev_priv);
 void gen9_disable_guc_interrupts(struct drm_i915_private *dev_priv);
@@ -1326,7 +1308,7 @@ void intel_set_cdclk(struct drm_i915_private *dev_priv,
 /* intel_display.c */
 void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe);
 void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe);
-enum transcoder intel_crtc_pch_transcoder(struct intel_crtc *crtc);
+enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc);
 void intel_update_rawclk(struct drm_i915_private *dev_priv);
 int vlv_get_hpll_vco(struct drm_i915_private *dev_priv);
 int vlv_get_cck_clock(struct drm_i915_private *dev_priv,
@@ -1335,7 +1317,6 @@ int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv,
 			   const char *name, u32 reg);
 void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv);
 void lpt_disable_iclkip(struct drm_i915_private *dev_priv);
-extern const struct drm_plane_funcs intel_plane_funcs;
 void intel_init_display_hooks(struct drm_i915_private *dev_priv);
 unsigned int intel_fb_xy_to_linear(int x, int y,
 				   const struct intel_plane_state *state,
@@ -1408,9 +1389,6 @@ void intel_unpin_fb_vma(struct i915_vma *vma);
 struct drm_framebuffer *
 intel_framebuffer_create(struct drm_i915_gem_object *obj,
 			 struct drm_mode_fb_cmd2 *mode_cmd);
-void intel_finish_page_flip_cs(struct drm_i915_private *dev_priv, int pipe);
-void intel_finish_page_flip_mmio(struct drm_i915_private *dev_priv, int pipe);
-void intel_check_page_flip(struct drm_i915_private *dev_priv, int pipe);
 int intel_prepare_plane_fb(struct drm_plane *plane,
 			   struct drm_plane_state *new_state);
 void intel_cleanup_plane_fb(struct drm_plane *plane,
@@ -1597,7 +1575,8 @@ void intel_hpd_poll_init(struct drm_i915_private *dev_priv);
 #ifdef CONFIG_DRM_FBDEV_EMULATION
 extern int intel_fbdev_init(struct drm_device *dev);
 extern void intel_fbdev_initial_config_async(struct drm_device *dev);
-extern void intel_fbdev_fini(struct drm_device *dev);
+extern void intel_fbdev_unregister(struct drm_i915_private *dev_priv);
+extern void intel_fbdev_fini(struct drm_i915_private *dev_priv);
 extern void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous);
 extern void intel_fbdev_output_poll_changed(struct drm_device *dev);
 extern void intel_fbdev_restore_mode(struct drm_device *dev);
@@ -1611,7 +1590,11 @@ static inline void intel_fbdev_initial_config_async(struct drm_device *dev)
 {
 }
 
-static inline void intel_fbdev_fini(struct drm_device *dev)
+static inline void intel_fbdev_unregister(struct drm_i915_private *dev_priv)
+{
+}
+
+static inline void intel_fbdev_fini(struct drm_i915_private *dev_priv)
 {
 }
 
@@ -1696,6 +1679,7 @@ void intel_overlay_reset(struct drm_i915_private *dev_priv);
 /* intel_panel.c */
 int intel_panel_init(struct intel_panel *panel,
 		     struct drm_display_mode *fixed_mode,
+		     struct drm_display_mode *alt_fixed_mode,
 		     struct drm_display_mode *downclock_mode);
 void intel_panel_fini(struct intel_panel *panel);
 void intel_fixed_panel_mode(const struct drm_display_mode *fixed_mode,
@@ -1858,9 +1842,8 @@ void intel_suspend_gt_powersave(struct drm_i915_private *dev_priv);
 void gen6_rps_busy(struct drm_i915_private *dev_priv);
 void gen6_rps_reset_ei(struct drm_i915_private *dev_priv);
 void gen6_rps_idle(struct drm_i915_private *dev_priv);
-void gen6_rps_boost(struct drm_i915_private *dev_priv,
-		    struct intel_rps_client *rps,
-		    unsigned long submitted);
+void gen6_rps_boost(struct drm_i915_gem_request *rq,
+		    struct intel_rps_client *rps);
 void intel_queue_rps_boost_for_request(struct drm_i915_gem_request *req);
 void g4x_wm_get_hw_state(struct drm_device *dev);
 void vlv_wm_get_hw_state(struct drm_device *dev);
@@ -1902,7 +1885,7 @@ struct intel_plane *intel_sprite_plane_create(struct drm_i915_private *dev_priv,
 int intel_sprite_set_colorkey(struct drm_device *dev, void *data,
 			      struct drm_file *file_priv);
 void intel_pipe_update_start(struct intel_crtc *crtc);
-void intel_pipe_update_end(struct intel_crtc *crtc, struct intel_flip_work *work);
+void intel_pipe_update_end(struct intel_crtc *crtc);
 
 /* intel_tv.c */
 void intel_tv_init(struct drm_i915_private *dev_priv);
diff --git a/drivers/gpu/drm/i915/intel_dsi.c b/drivers/gpu/drm/i915/intel_dsi.c
index 50ec836da8b1..7442891762be 100644
--- a/drivers/gpu/drm/i915/intel_dsi.c
+++ b/drivers/gpu/drm/i915/intel_dsi.c
@@ -892,8 +892,6 @@ static void intel_dsi_disable(struct intel_encoder *encoder,
 			      struct intel_crtc_state *old_crtc_state,
 			      struct drm_connector_state *old_conn_state)
 {
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
 	enum port port;
 
@@ -903,15 +901,6 @@ static void intel_dsi_disable(struct intel_encoder *encoder,
 	intel_panel_disable_backlight(old_conn_state);
 
 	/*
-	 * Disable Device ready before the port shutdown in order
-	 * to avoid split screen
-	 */
-	if (IS_BROXTON(dev_priv)) {
-		for_each_dsi_port(port, intel_dsi->ports)
-			I915_WRITE(MIPI_DEVICE_READY(port), 0);
-	}
-
-	/*
 	 * According to the spec we should send SHUTDOWN before
 	 * MIPI_SEQ_DISPLAY_OFF only for v3+ VBTs, but field testing
 	 * has shown that the v3 sequence works for v2 VBTs too
@@ -1653,12 +1642,10 @@ static const struct drm_connector_helper_funcs intel_dsi_connector_helper_funcs
 };
 
 static const struct drm_connector_funcs intel_dsi_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.late_register = intel_connector_register,
 	.early_unregister = intel_connector_unregister,
 	.destroy = intel_dsi_connector_destroy,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.set_property = drm_atomic_helper_connector_set_property,
 	.atomic_get_property = intel_digital_connector_atomic_get_property,
 	.atomic_set_property = intel_digital_connector_atomic_set_property,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
@@ -1851,7 +1838,7 @@ void intel_dsi_init(struct drm_i915_private *dev_priv)
 	connector->display_info.width_mm = fixed_mode->width_mm;
 	connector->display_info.height_mm = fixed_mode->height_mm;
 
-	intel_panel_init(&intel_connector->panel, fixed_mode, NULL);
+	intel_panel_init(&intel_connector->panel, fixed_mode, NULL, NULL);
 	intel_panel_setup_backlight(connector, INVALID_PIPE);
 
 	intel_dsi_add_properties(intel_connector);
diff --git a/drivers/gpu/drm/i915/intel_dsi_dcs_backlight.c b/drivers/gpu/drm/i915/intel_dsi_dcs_backlight.c
index 6e09ceb71500..150a156f3b1e 100644
--- a/drivers/gpu/drm/i915/intel_dsi_dcs_backlight.c
+++ b/drivers/gpu/drm/i915/intel_dsi_dcs_backlight.c
@@ -46,7 +46,7 @@ static u32 dcs_get_backlight(struct intel_connector *connector)
 	struct intel_encoder *encoder = connector->encoder;
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
 	struct mipi_dsi_device *dsi_device;
-	u8 data;
+	u8 data = 0;
 	enum port port;
 
 	/* FIXME: Need to take care of 16 bit brightness level */
diff --git a/drivers/gpu/drm/i915/intel_dsi_vbt.c b/drivers/gpu/drm/i915/intel_dsi_vbt.c
index 7158c7ce9c09..91c07b0c8db9 100644
--- a/drivers/gpu/drm/i915/intel_dsi_vbt.c
+++ b/drivers/gpu/drm/i915/intel_dsi_vbt.c
@@ -306,7 +306,7 @@ static void bxt_exec_gpio(struct drm_i915_private *dev_priv,
 
 	if (!gpio_desc) {
 		gpio_desc = devm_gpiod_get_index(dev_priv->drm.dev,
-						 "panel", gpio_index,
+						 NULL, gpio_index,
 						 value ? GPIOD_OUT_LOW :
 						 GPIOD_OUT_HIGH);
 
diff --git a/drivers/gpu/drm/i915/intel_dvo.c b/drivers/gpu/drm/i915/intel_dvo.c
index c1544a53095d..c0a027274c06 100644
--- a/drivers/gpu/drm/i915/intel_dvo.c
+++ b/drivers/gpu/drm/i915/intel_dvo.c
@@ -344,13 +344,11 @@ static void intel_dvo_destroy(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs intel_dvo_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.detect = intel_dvo_detect,
 	.late_register = intel_connector_register,
 	.early_unregister = intel_connector_unregister,
 	.destroy = intel_dvo_destroy,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.set_property = drm_atomic_helper_connector_set_property,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
 };
@@ -554,7 +552,7 @@ void intel_dvo_init(struct drm_i915_private *dev_priv)
 			 */
 			intel_panel_init(&intel_connector->panel,
 					 intel_dvo_get_current_mode(connector),
-					 NULL);
+					 NULL, NULL);
 			intel_dvo->panel_wants_dither = true;
 		}
 
diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c
index 5b4de719bec3..9ab596941372 100644
--- a/drivers/gpu/drm/i915/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/intel_engine_cs.c
@@ -149,6 +149,7 @@ __intel_engine_context_size(struct drm_i915_private *dev_priv, u8 class)
 		switch (INTEL_GEN(dev_priv)) {
 		default:
 			MISSING_CASE(INTEL_GEN(dev_priv));
+		case 10:
 		case 9:
 			return GEN9_LR_CONTEXT_RENDER_SIZE;
 		case 8:
@@ -291,11 +292,9 @@ cleanup:
  */
 int intel_engines_init(struct drm_i915_private *dev_priv)
 {
-	struct intel_device_info *device_info = mkwrite_device_info(dev_priv);
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id, err_id;
-	unsigned int mask = 0;
-	int err = 0;
+	int err;
 
 	for_each_engine(engine, dev_priv, id) {
 		const struct engine_class_info *class_info =
@@ -306,40 +305,30 @@ int intel_engines_init(struct drm_i915_private *dev_priv)
 			init = class_info->init_execlists;
 		else
 			init = class_info->init_legacy;
-		if (!init) {
-			kfree(engine);
-			dev_priv->engine[id] = NULL;
-			continue;
-		}
+
+		err = -EINVAL;
+		err_id = id;
+
+		if (GEM_WARN_ON(!init))
+			goto cleanup;
 
 		err = init(engine);
-		if (err) {
-			err_id = id;
+		if (err)
 			goto cleanup;
-		}
 
 		GEM_BUG_ON(!engine->submit_request);
-		mask |= ENGINE_MASK(id);
 	}
 
-	/*
-	 * Catch failures to update intel_engines table when the new engines
-	 * are added to the driver by a warning and disabling the forgotten
-	 * engines.
-	 */
-	if (WARN_ON(mask != INTEL_INFO(dev_priv)->ring_mask))
-		device_info->ring_mask = mask;
-
-	device_info->num_rings = hweight32(mask);
-
 	return 0;
 
 cleanup:
 	for_each_engine(engine, dev_priv, id) {
-		if (id >= err_id)
+		if (id >= err_id) {
 			kfree(engine);
-		else
+			dev_priv->engine[id] = NULL;
+		} else {
 			dev_priv->gt.cleanup_engine(engine);
+		}
 	}
 	return err;
 }
@@ -348,9 +337,6 @@ void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
 
-	GEM_BUG_ON(!intel_engine_is_idle(engine));
-	GEM_BUG_ON(i915_gem_active_isset(&engine->timeline->last_request));
-
 	/* Our semaphore implementation is strictly monotonic (i.e. we proceed
 	 * so long as the semaphore value in the register/page is greater
 	 * than the sync value), so whenever we reset the seqno,
@@ -1294,6 +1280,10 @@ bool intel_engine_is_idle(struct intel_engine_cs *engine)
 	if (port_request(&engine->execlist_port[0]))
 		return false;
 
+	/* ELSP is empty, but there are ready requests? */
+	if (READ_ONCE(engine->execlist_first))
+		return false;
+
 	/* Ring stopped? */
 	if (!ring_is_idle(engine))
 		return false;
@@ -1340,6 +1330,7 @@ void intel_engines_mark_idle(struct drm_i915_private *i915)
 	for_each_engine(engine, i915, id) {
 		intel_engine_disarm_breadcrumbs(engine);
 		i915_gem_batch_pool_fini(&engine->batch_pool);
+		tasklet_kill(&engine->irq_tasklet);
 		engine->no_priolist = false;
 	}
 }
diff --git a/drivers/gpu/drm/i915/intel_fbc.c b/drivers/gpu/drm/i915/intel_fbc.c
index 860b8c26d29b..8c8ead2276e0 100644
--- a/drivers/gpu/drm/i915/intel_fbc.c
+++ b/drivers/gpu/drm/i915/intel_fbc.c
@@ -406,9 +406,7 @@ static void intel_fbc_work_fn(struct work_struct *__work)
 	struct drm_vblank_crtc *vblank = &dev_priv->drm.vblank[crtc->pipe];
 
 	if (drm_crtc_vblank_get(&crtc->base)) {
-		DRM_ERROR("vblank not available for FBC on pipe %c\n",
-			  pipe_name(crtc->pipe));
-
+		/* CRTC is now off, leave FBC deactivated */
 		mutex_lock(&fbc->lock);
 		work->scheduled = false;
 		mutex_unlock(&fbc->lock);
@@ -461,6 +459,8 @@ static void intel_fbc_schedule_activation(struct intel_crtc *crtc)
 	struct intel_fbc_work *work = &fbc->work;
 
 	WARN_ON(!mutex_is_locked(&fbc->lock));
+	if (WARN_ON(!fbc->enabled))
+		return;
 
 	if (drm_crtc_vblank_get(&crtc->base)) {
 		DRM_ERROR("vblank not available for FBC on pipe %c\n",
@@ -1216,7 +1216,7 @@ static void intel_fbc_underrun_work_fn(struct work_struct *work)
 	mutex_lock(&fbc->lock);
 
 	/* Maybe we were scheduled twice. */
-	if (fbc->underrun_detected)
+	if (fbc->underrun_detected || !fbc->enabled)
 		goto out;
 
 	DRM_DEBUG_KMS("Disabling FBC due to FIFO underrun.\n");
diff --git a/drivers/gpu/drm/i915/intel_fbdev.c b/drivers/gpu/drm/i915/intel_fbdev.c
index 0c4cde6b2e6f..262e75c00dd2 100644
--- a/drivers/gpu/drm/i915/intel_fbdev.c
+++ b/drivers/gpu/drm/i915/intel_fbdev.c
@@ -232,7 +232,6 @@ static int intelfb_create(struct drm_fb_helper *helper,
 
 	strcpy(info->fix.id, "inteldrmfb");
 
-	info->flags = FBINFO_DEFAULT | FBINFO_CAN_FORCE_OUTPUT;
 	info->fbops = &intelfb_ops;
 
 	/* setup aperture base/size for vesafb takeover */
@@ -281,27 +280,6 @@ out_unlock:
 	return ret;
 }
 
-/** Sets the color ramps on behalf of RandR */
-static void intel_crtc_fb_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
-				    u16 blue, int regno)
-{
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-
-	intel_crtc->lut_r[regno] = red >> 8;
-	intel_crtc->lut_g[regno] = green >> 8;
-	intel_crtc->lut_b[regno] = blue >> 8;
-}
-
-static void intel_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green,
-				    u16 *blue, int regno)
-{
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-
-	*red = intel_crtc->lut_r[regno] << 8;
-	*green = intel_crtc->lut_g[regno] << 8;
-	*blue = intel_crtc->lut_b[regno] << 8;
-}
-
 static struct drm_fb_helper_crtc *
 intel_fb_helper_crtc(struct drm_fb_helper *fb_helper, struct drm_crtc *crtc)
 {
@@ -352,14 +330,20 @@ static bool intel_fb_initial_config(struct drm_fb_helper *fb_helper,
 	unsigned int count = min(fb_helper->connector_count, BITS_PER_LONG);
 	int i, j;
 	bool *save_enabled;
-	bool fallback = true;
+	bool fallback = true, ret = true;
 	int num_connectors_enabled = 0;
 	int num_connectors_detected = 0;
+	struct drm_modeset_acquire_ctx ctx;
 
 	save_enabled = kcalloc(count, sizeof(bool), GFP_KERNEL);
 	if (!save_enabled)
 		return false;
 
+	drm_modeset_acquire_init(&ctx, 0);
+
+	while (drm_modeset_lock_all_ctx(fb_helper->dev, &ctx) != 0)
+		drm_modeset_backoff(&ctx);
+
 	memcpy(save_enabled, enabled, count);
 	mask = GENMASK(count - 1, 0);
 	conn_configured = 0;
@@ -370,7 +354,6 @@ retry:
 		struct drm_connector *connector;
 		struct drm_encoder *encoder;
 		struct drm_fb_helper_crtc *new_crtc;
-		struct intel_crtc *intel_crtc;
 
 		fb_conn = fb_helper->connector_info[i];
 		connector = fb_conn->connector;
@@ -412,13 +395,6 @@ retry:
 
 		num_connectors_enabled++;
 
-		intel_crtc = to_intel_crtc(connector->state->crtc);
-		for (j = 0; j < 256; j++) {
-			intel_crtc->lut_r[j] = j;
-			intel_crtc->lut_g[j] = j;
-			intel_crtc->lut_b[j] = j;
-		}
-
 		new_crtc = intel_fb_helper_crtc(fb_helper,
 						connector->state->crtc);
 
@@ -509,18 +485,18 @@ retry:
 bail:
 		DRM_DEBUG_KMS("Not using firmware configuration\n");
 		memcpy(enabled, save_enabled, count);
-		kfree(save_enabled);
-		return false;
+		ret = false;
 	}
 
+	drm_modeset_drop_locks(&ctx);
+	drm_modeset_acquire_fini(&ctx);
+
 	kfree(save_enabled);
-	return true;
+	return ret;
 }
 
 static const struct drm_fb_helper_funcs intel_fb_helper_funcs = {
 	.initial_config = intel_fb_initial_config,
-	.gamma_set = intel_crtc_fb_gamma_set,
-	.gamma_get = intel_crtc_fb_gamma_get,
 	.fb_probe = intelfb_create,
 };
 
@@ -531,8 +507,6 @@ static void intel_fbdev_destroy(struct intel_fbdev *ifbdev)
 	 * trying to rectify all the possible error paths leading here.
 	 */
 
-	drm_fb_helper_unregister_fbi(&ifbdev->helper);
-
 	drm_fb_helper_fini(&ifbdev->helper);
 
 	if (ifbdev->vma) {
@@ -720,8 +694,10 @@ static void intel_fbdev_initial_config(void *data, async_cookie_t cookie)
 
 	/* Due to peculiar init order wrt to hpd handling this is separate. */
 	if (drm_fb_helper_initial_config(&ifbdev->helper,
-					 ifbdev->preferred_bpp))
-		intel_fbdev_fini(ifbdev->helper.dev);
+					 ifbdev->preferred_bpp)) {
+		intel_fbdev_unregister(to_i915(ifbdev->helper.dev));
+		intel_fbdev_fini(to_i915(ifbdev->helper.dev));
+	}
 }
 
 void intel_fbdev_initial_config_async(struct drm_device *dev)
@@ -744,9 +720,8 @@ static void intel_fbdev_sync(struct intel_fbdev *ifbdev)
 	ifbdev->cookie = 0;
 }
 
-void intel_fbdev_fini(struct drm_device *dev)
+void intel_fbdev_unregister(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_fbdev *ifbdev = dev_priv->fbdev;
 
 	if (!ifbdev)
@@ -756,8 +731,17 @@ void intel_fbdev_fini(struct drm_device *dev)
 	if (!current_is_async())
 		intel_fbdev_sync(ifbdev);
 
+	drm_fb_helper_unregister_fbi(&ifbdev->helper);
+}
+
+void intel_fbdev_fini(struct drm_i915_private *dev_priv)
+{
+	struct intel_fbdev *ifbdev = fetch_and_zero(&dev_priv->fbdev);
+
+	if (!ifbdev)
+		return;
+
 	intel_fbdev_destroy(ifbdev);
-	dev_priv->fbdev = NULL;
 }
 
 void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous)
@@ -813,7 +797,7 @@ void intel_fbdev_output_poll_changed(struct drm_device *dev)
 {
 	struct intel_fbdev *ifbdev = to_i915(dev)->fbdev;
 
-	if (ifbdev && ifbdev->vma)
+	if (ifbdev)
 		drm_fb_helper_hotplug_event(&ifbdev->helper);
 }
 
diff --git a/drivers/gpu/drm/i915/intel_fifo_underrun.c b/drivers/gpu/drm/i915/intel_fifo_underrun.c
index d484862cc7df..04689600e337 100644
--- a/drivers/gpu/drm/i915/intel_fifo_underrun.c
+++ b/drivers/gpu/drm/i915/intel_fifo_underrun.c
@@ -187,11 +187,11 @@ static void broadwell_set_fifo_underrun_reporting(struct drm_device *dev,
 }
 
 static void ibx_set_fifo_underrun_reporting(struct drm_device *dev,
-					    enum transcoder pch_transcoder,
+					    enum pipe pch_transcoder,
 					    bool enable)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	uint32_t bit = (pch_transcoder == TRANSCODER_A) ?
+	uint32_t bit = (pch_transcoder == PIPE_A) ?
 		       SDE_TRANSA_FIFO_UNDER : SDE_TRANSB_FIFO_UNDER;
 
 	if (enable)
@@ -203,7 +203,7 @@ static void ibx_set_fifo_underrun_reporting(struct drm_device *dev,
 static void cpt_check_pch_fifo_underruns(struct intel_crtc *crtc)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	enum transcoder pch_transcoder = (enum transcoder) crtc->pipe;
+	enum pipe pch_transcoder = crtc->pipe;
 	uint32_t serr_int = I915_READ(SERR_INT);
 
 	lockdep_assert_held(&dev_priv->irq_lock);
@@ -215,12 +215,12 @@ static void cpt_check_pch_fifo_underruns(struct intel_crtc *crtc)
 	POSTING_READ(SERR_INT);
 
 	trace_intel_pch_fifo_underrun(dev_priv, pch_transcoder);
-	DRM_ERROR("pch fifo underrun on pch transcoder %s\n",
-		  transcoder_name(pch_transcoder));
+	DRM_ERROR("pch fifo underrun on pch transcoder %c\n",
+		  pipe_name(pch_transcoder));
 }
 
 static void cpt_set_fifo_underrun_reporting(struct drm_device *dev,
-					    enum transcoder pch_transcoder,
+					    enum pipe pch_transcoder,
 					    bool enable, bool old)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
@@ -238,8 +238,8 @@ static void cpt_set_fifo_underrun_reporting(struct drm_device *dev,
 
 		if (old && I915_READ(SERR_INT) &
 		    SERR_INT_TRANS_FIFO_UNDERRUN(pch_transcoder)) {
-			DRM_ERROR("uncleared pch fifo underrun on pch transcoder %s\n",
-				  transcoder_name(pch_transcoder));
+			DRM_ERROR("uncleared pch fifo underrun on pch transcoder %c\n",
+				  pipe_name(pch_transcoder));
 		}
 	}
 }
@@ -313,11 +313,11 @@ bool intel_set_cpu_fifo_underrun_reporting(struct drm_i915_private *dev_priv,
  * Returns the previous state of underrun reporting.
  */
 bool intel_set_pch_fifo_underrun_reporting(struct drm_i915_private *dev_priv,
-					   enum transcoder pch_transcoder,
+					   enum pipe pch_transcoder,
 					   bool enable)
 {
 	struct intel_crtc *crtc =
-		intel_get_crtc_for_pipe(dev_priv, (enum pipe) pch_transcoder);
+		intel_get_crtc_for_pipe(dev_priv, pch_transcoder);
 	unsigned long flags;
 	bool old;
 
@@ -390,13 +390,13 @@ void intel_cpu_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv,
  * interrupt to avoid an irq storm.
  */
 void intel_pch_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv,
-					 enum transcoder pch_transcoder)
+					 enum pipe pch_transcoder)
 {
 	if (intel_set_pch_fifo_underrun_reporting(dev_priv, pch_transcoder,
 						  false)) {
 		trace_intel_pch_fifo_underrun(dev_priv, pch_transcoder);
-		DRM_ERROR("PCH transcoder %s FIFO underrun\n",
-			  transcoder_name(pch_transcoder));
+		DRM_ERROR("PCH transcoder %c FIFO underrun\n",
+			  pipe_name(pch_transcoder));
 	}
 }
 
diff --git a/drivers/gpu/drm/i915/intel_hangcheck.c b/drivers/gpu/drm/i915/intel_hangcheck.c
index 9b0ece427bdc..d9d87d96fb69 100644
--- a/drivers/gpu/drm/i915/intel_hangcheck.c
+++ b/drivers/gpu/drm/i915/intel_hangcheck.c
@@ -324,7 +324,7 @@ hangcheck_get_action(struct intel_engine_cs *engine,
 	if (engine->hangcheck.seqno != hc->seqno)
 		return ENGINE_ACTIVE_SEQNO;
 
-	if (i915_seqno_passed(hc->seqno, intel_engine_last_submit(engine)))
+	if (intel_engine_is_idle(engine))
 		return ENGINE_IDLE;
 
 	return engine_stuck(engine, hc->acthd);
diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c
index ec0779a52d53..e8abea7594ec 100644
--- a/drivers/gpu/drm/i915/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/intel_hdmi.c
@@ -459,22 +459,31 @@ static void intel_hdmi_set_avi_infoframe(struct drm_encoder *encoder,
 	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
 	const struct drm_display_mode *adjusted_mode =
 		&crtc_state->base.adjusted_mode;
+	struct drm_connector *connector = &intel_hdmi->attached_connector->base;
+	bool is_hdmi2_sink = connector->display_info.hdmi.scdc.supported;
 	union hdmi_infoframe frame;
 	int ret;
 
 	ret = drm_hdmi_avi_infoframe_from_display_mode(&frame.avi,
-						       adjusted_mode);
+						       adjusted_mode,
+						       is_hdmi2_sink);
 	if (ret < 0) {
 		DRM_ERROR("couldn't fill AVI infoframe\n");
 		return;
 	}
 
+	if (crtc_state->ycbcr420)
+		frame.avi.colorspace = HDMI_COLORSPACE_YUV420;
+	else
+		frame.avi.colorspace = HDMI_COLORSPACE_RGB;
+
 	drm_hdmi_avi_infoframe_quant_range(&frame.avi, adjusted_mode,
 					   crtc_state->limited_color_range ?
 					   HDMI_QUANTIZATION_RANGE_LIMITED :
 					   HDMI_QUANTIZATION_RANGE_FULL,
 					   intel_hdmi->rgb_quant_range_selectable);
 
+	/* TODO: handle pixel repetition for YCBCR420 outputs */
 	intel_write_infoframe(encoder, crtc_state, &frame);
 }
 
@@ -1292,6 +1301,9 @@ intel_hdmi_mode_valid(struct drm_connector *connector,
 	if (mode->flags & DRM_MODE_FLAG_DBLCLK)
 		clock *= 2;
 
+	if (drm_mode_is_420_only(&connector->display_info, mode))
+		clock /= 2;
+
 	/* check if we can do 8bpc */
 	status = hdmi_port_clock_valid(hdmi, clock, true, force_dvi);
 
@@ -1321,14 +1333,21 @@ static bool hdmi_12bpc_possible(struct intel_crtc_state *crtc_state)
 	if (crtc_state->output_types != 1 << INTEL_OUTPUT_HDMI)
 		return false;
 
-	for_each_connector_in_state(state, connector, connector_state, i) {
+	for_each_new_connector_in_state(state, connector, connector_state, i) {
 		const struct drm_display_info *info = &connector->display_info;
 
 		if (connector_state->crtc != crtc_state->base.crtc)
 			continue;
 
-		if ((info->edid_hdmi_dc_modes & DRM_EDID_HDMI_DC_36) == 0)
-			return false;
+		if (crtc_state->ycbcr420) {
+			const struct drm_hdmi_info *hdmi = &info->hdmi;
+
+			if (!(hdmi->y420_dc_modes & DRM_EDID_YCBCR420_DC_36))
+				return false;
+		} else {
+			if (!(info->edid_hdmi_dc_modes & DRM_EDID_HDMI_DC_36))
+				return false;
+		}
 	}
 
 	/* Display Wa #1139 */
@@ -1339,6 +1358,36 @@ static bool hdmi_12bpc_possible(struct intel_crtc_state *crtc_state)
 	return true;
 }
 
+static bool
+intel_hdmi_ycbcr420_config(struct drm_connector *connector,
+			   struct intel_crtc_state *config,
+			   int *clock_12bpc, int *clock_8bpc)
+{
+	struct intel_crtc *intel_crtc = to_intel_crtc(config->base.crtc);
+
+	if (!connector->ycbcr_420_allowed) {
+		DRM_ERROR("Platform doesn't support YCBCR420 output\n");
+		return false;
+	}
+
+	/* YCBCR420 TMDS rate requirement is half the pixel clock */
+	config->port_clock /= 2;
+	*clock_12bpc /= 2;
+	*clock_8bpc /= 2;
+	config->ycbcr420 = true;
+
+	/* YCBCR 420 output conversion needs a scaler */
+	if (skl_update_scaler_crtc(config)) {
+		DRM_DEBUG_KMS("Scaler allocation for output failed\n");
+		return false;
+	}
+
+	intel_pch_panel_fitting(intel_crtc, config,
+				DRM_MODE_SCALE_FULLSCREEN);
+
+	return true;
+}
+
 bool intel_hdmi_compute_config(struct intel_encoder *encoder,
 			       struct intel_crtc_state *pipe_config,
 			       struct drm_connector_state *conn_state)
@@ -1346,7 +1395,8 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder,
 	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
-	struct drm_scdc *scdc = &conn_state->connector->display_info.hdmi.scdc;
+	struct drm_connector *connector = conn_state->connector;
+	struct drm_scdc *scdc = &connector->display_info.hdmi.scdc;
 	struct intel_digital_connector_state *intel_conn_state =
 		to_intel_digital_connector_state(conn_state);
 	int clock_8bpc = pipe_config->base.adjusted_mode.crtc_clock;
@@ -1376,6 +1426,14 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder,
 		clock_12bpc *= 2;
 	}
 
+	if (drm_mode_is_420_only(&connector->display_info, adjusted_mode)) {
+		if (!intel_hdmi_ycbcr420_config(connector, pipe_config,
+						&clock_12bpc, &clock_8bpc)) {
+			DRM_ERROR("Can't support YCBCR420 output\n");
+			return false;
+		}
+	}
+
 	if (HAS_PCH_SPLIT(dev_priv) && !HAS_DDI(dev_priv))
 		pipe_config->has_pch_encoder = true;
 
@@ -1703,11 +1761,9 @@ static void intel_hdmi_destroy(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs intel_hdmi_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.detect = intel_hdmi_detect,
 	.force = intel_hdmi_force,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.set_property = drm_atomic_helper_connector_set_property,
 	.atomic_get_property = intel_digital_connector_atomic_get_property,
 	.atomic_set_property = intel_digital_connector_atomic_set_property,
 	.late_register = intel_connector_register,
@@ -1787,45 +1843,114 @@ void intel_hdmi_handle_sink_scrambling(struct intel_encoder *encoder,
 	DRM_DEBUG_KMS("sink scrambling handled\n");
 }
 
-static u8 intel_hdmi_ddc_pin(struct drm_i915_private *dev_priv,
-			     enum port port)
+static u8 chv_port_to_ddc_pin(struct drm_i915_private *dev_priv, enum port port)
 {
-	const struct ddi_vbt_port_info *info =
-		&dev_priv->vbt.ddi_port_info[port];
 	u8 ddc_pin;
 
-	if (info->alternate_ddc_pin) {
-		DRM_DEBUG_KMS("Using DDC pin 0x%x for port %c (VBT)\n",
-			      info->alternate_ddc_pin, port_name(port));
-		return info->alternate_ddc_pin;
+	switch (port) {
+	case PORT_B:
+		ddc_pin = GMBUS_PIN_DPB;
+		break;
+	case PORT_C:
+		ddc_pin = GMBUS_PIN_DPC;
+		break;
+	case PORT_D:
+		ddc_pin = GMBUS_PIN_DPD_CHV;
+		break;
+	default:
+		MISSING_CASE(port);
+		ddc_pin = GMBUS_PIN_DPB;
+		break;
 	}
+	return ddc_pin;
+}
+
+static u8 bxt_port_to_ddc_pin(struct drm_i915_private *dev_priv, enum port port)
+{
+	u8 ddc_pin;
 
 	switch (port) {
 	case PORT_B:
-		if (IS_GEN9_LP(dev_priv) || HAS_PCH_CNP(dev_priv))
-			ddc_pin = GMBUS_PIN_1_BXT;
-		else
-			ddc_pin = GMBUS_PIN_DPB;
+		ddc_pin = GMBUS_PIN_1_BXT;
 		break;
 	case PORT_C:
-		if (IS_GEN9_LP(dev_priv) || HAS_PCH_CNP(dev_priv))
-			ddc_pin = GMBUS_PIN_2_BXT;
-		else
-			ddc_pin = GMBUS_PIN_DPC;
+		ddc_pin = GMBUS_PIN_2_BXT;
+		break;
+	default:
+		MISSING_CASE(port);
+		ddc_pin = GMBUS_PIN_1_BXT;
+		break;
+	}
+	return ddc_pin;
+}
+
+static u8 cnp_port_to_ddc_pin(struct drm_i915_private *dev_priv,
+			      enum port port)
+{
+	u8 ddc_pin;
+
+	switch (port) {
+	case PORT_B:
+		ddc_pin = GMBUS_PIN_1_BXT;
+		break;
+	case PORT_C:
+		ddc_pin = GMBUS_PIN_2_BXT;
 		break;
 	case PORT_D:
-		if (HAS_PCH_CNP(dev_priv))
-			ddc_pin = GMBUS_PIN_4_CNP;
-		else if (IS_CHERRYVIEW(dev_priv))
-			ddc_pin = GMBUS_PIN_DPD_CHV;
-		else
-			ddc_pin = GMBUS_PIN_DPD;
+		ddc_pin = GMBUS_PIN_4_CNP;
+		break;
+	default:
+		MISSING_CASE(port);
+		ddc_pin = GMBUS_PIN_1_BXT;
+		break;
+	}
+	return ddc_pin;
+}
+
+static u8 g4x_port_to_ddc_pin(struct drm_i915_private *dev_priv,
+			      enum port port)
+{
+	u8 ddc_pin;
+
+	switch (port) {
+	case PORT_B:
+		ddc_pin = GMBUS_PIN_DPB;
+		break;
+	case PORT_C:
+		ddc_pin = GMBUS_PIN_DPC;
+		break;
+	case PORT_D:
+		ddc_pin = GMBUS_PIN_DPD;
 		break;
 	default:
 		MISSING_CASE(port);
 		ddc_pin = GMBUS_PIN_DPB;
 		break;
 	}
+	return ddc_pin;
+}
+
+static u8 intel_hdmi_ddc_pin(struct drm_i915_private *dev_priv,
+			     enum port port)
+{
+	const struct ddi_vbt_port_info *info =
+		&dev_priv->vbt.ddi_port_info[port];
+	u8 ddc_pin;
+
+	if (info->alternate_ddc_pin) {
+		DRM_DEBUG_KMS("Using DDC pin 0x%x for port %c (VBT)\n",
+			      info->alternate_ddc_pin, port_name(port));
+		return info->alternate_ddc_pin;
+	}
+
+	if (IS_CHERRYVIEW(dev_priv))
+		ddc_pin = chv_port_to_ddc_pin(dev_priv, port);
+	else if (IS_GEN9_LP(dev_priv))
+		ddc_pin = bxt_port_to_ddc_pin(dev_priv, port);
+	else if (HAS_PCH_CNP(dev_priv))
+		ddc_pin = cnp_port_to_ddc_pin(dev_priv, port);
+	else
+		ddc_pin = g4x_port_to_ddc_pin(dev_priv, port);
 
 	DRM_DEBUG_KMS("Using DDC pin 0x%x for port %c (platform default)\n",
 		      ddc_pin, port_name(port));
@@ -1859,25 +1984,14 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
 	connector->doublescan_allowed = 0;
 	connector->stereo_allowed = 1;
 
+	if (IS_GEMINILAKE(dev_priv))
+		connector->ycbcr_420_allowed = true;
+
 	intel_hdmi->ddc_bus = intel_hdmi_ddc_pin(dev_priv, port);
 
-	switch (port) {
-	case PORT_B:
-		intel_encoder->hpd_pin = HPD_PORT_B;
-		break;
-	case PORT_C:
-		intel_encoder->hpd_pin = HPD_PORT_C;
-		break;
-	case PORT_D:
-		intel_encoder->hpd_pin = HPD_PORT_D;
-		break;
-	case PORT_E:
-		intel_encoder->hpd_pin = HPD_PORT_E;
-		break;
-	default:
-		MISSING_CASE(port);
+	if (WARN_ON(port == PORT_A))
 		return;
-	}
+	intel_encoder->hpd_pin = intel_hpd_pin(port);
 
 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		intel_hdmi->write_infoframe = vlv_write_infoframe;
diff --git a/drivers/gpu/drm/i915/intel_hotplug.c b/drivers/gpu/drm/i915/intel_hotplug.c
index f1200272a699..875d5d218d5c 100644
--- a/drivers/gpu/drm/i915/intel_hotplug.c
+++ b/drivers/gpu/drm/i915/intel_hotplug.c
@@ -76,26 +76,54 @@
  * it will use i915_hotplug_work_func where this logic is handled.
  */
 
-bool intel_hpd_pin_to_port(enum hpd_pin pin, enum port *port)
+/**
+ * intel_hpd_port - return port hard associated with certain pin.
+ * @pin: the hpd pin to get associated port
+ *
+ * Return port that is associatade with @pin and PORT_NONE if no port is
+ * hard associated with that @pin.
+ */
+enum port intel_hpd_pin_to_port(enum hpd_pin pin)
 {
 	switch (pin) {
 	case HPD_PORT_A:
-		*port = PORT_A;
-		return true;
+		return PORT_A;
 	case HPD_PORT_B:
-		*port = PORT_B;
-		return true;
+		return PORT_B;
 	case HPD_PORT_C:
-		*port = PORT_C;
-		return true;
+		return PORT_C;
 	case HPD_PORT_D:
-		*port = PORT_D;
-		return true;
+		return PORT_D;
 	case HPD_PORT_E:
-		*port = PORT_E;
-		return true;
+		return PORT_E;
+	default:
+		return PORT_NONE; /* no port for this pin */
+	}
+}
+
+/**
+ * intel_hpd_pin - return pin hard associated with certain port.
+ * @port: the hpd port to get associated pin
+ *
+ * Return pin that is associatade with @port and HDP_NONE if no pin is
+ * hard associated with that @port.
+ */
+enum hpd_pin intel_hpd_pin(enum port port)
+{
+	switch (port) {
+	case PORT_A:
+		return HPD_PORT_A;
+	case PORT_B:
+		return HPD_PORT_B;
+	case PORT_C:
+		return HPD_PORT_C;
+	case PORT_D:
+		return HPD_PORT_D;
+	case PORT_E:
+		return HPD_PORT_E;
 	default:
-		return false;	/* no hpd */
+		MISSING_CASE(port);
+		return HPD_NONE;
 	}
 }
 
@@ -389,8 +417,9 @@ void intel_hpd_irq_handler(struct drm_i915_private *dev_priv,
 		if (!(BIT(i) & pin_mask))
 			continue;
 
-		is_dig_port = intel_hpd_pin_to_port(i, &port) &&
-			      dev_priv->hotplug.irq_port[port];
+		port = intel_hpd_pin_to_port(i);
+		is_dig_port = port != PORT_NONE &&
+			dev_priv->hotplug.irq_port[port];
 
 		if (is_dig_port) {
 			bool long_hpd = long_mask & BIT(i);
diff --git a/drivers/gpu/drm/i915/intel_i2c.c b/drivers/gpu/drm/i915/intel_i2c.c
index 3c9e00d4ba5a..eb5827110d8f 100644
--- a/drivers/gpu/drm/i915/intel_i2c.c
+++ b/drivers/gpu/drm/i915/intel_i2c.c
@@ -592,7 +592,6 @@ gmbus_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num)
 	int ret;
 
 	intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS);
-	mutex_lock(&dev_priv->gmbus_mutex);
 
 	if (bus->force_bit) {
 		ret = i2c_bit_algo.master_xfer(adapter, msgs, num);
@@ -604,7 +603,6 @@ gmbus_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num)
 			bus->force_bit |= GMBUS_FORCE_BIT_RETRY;
 	}
 
-	mutex_unlock(&dev_priv->gmbus_mutex);
 	intel_display_power_put(dev_priv, POWER_DOMAIN_GMBUS);
 
 	return ret;
@@ -624,6 +622,39 @@ static const struct i2c_algorithm gmbus_algorithm = {
 	.functionality	= gmbus_func
 };
 
+static void gmbus_lock_bus(struct i2c_adapter *adapter,
+			   unsigned int flags)
+{
+	struct intel_gmbus *bus = to_intel_gmbus(adapter);
+	struct drm_i915_private *dev_priv = bus->dev_priv;
+
+	mutex_lock(&dev_priv->gmbus_mutex);
+}
+
+static int gmbus_trylock_bus(struct i2c_adapter *adapter,
+			     unsigned int flags)
+{
+	struct intel_gmbus *bus = to_intel_gmbus(adapter);
+	struct drm_i915_private *dev_priv = bus->dev_priv;
+
+	return mutex_trylock(&dev_priv->gmbus_mutex);
+}
+
+static void gmbus_unlock_bus(struct i2c_adapter *adapter,
+			     unsigned int flags)
+{
+	struct intel_gmbus *bus = to_intel_gmbus(adapter);
+	struct drm_i915_private *dev_priv = bus->dev_priv;
+
+	mutex_unlock(&dev_priv->gmbus_mutex);
+}
+
+static const struct i2c_lock_operations gmbus_lock_ops = {
+	.lock_bus =    gmbus_lock_bus,
+	.trylock_bus = gmbus_trylock_bus,
+	.unlock_bus =  gmbus_unlock_bus,
+};
+
 /**
  * intel_gmbus_setup - instantiate all Intel i2c GMBuses
  * @dev_priv: i915 device private
@@ -665,6 +696,7 @@ int intel_setup_gmbus(struct drm_i915_private *dev_priv)
 		bus->dev_priv = dev_priv;
 
 		bus->adapter.algo = &gmbus_algorithm;
+		bus->adapter.lock_ops = &gmbus_lock_ops;
 
 		/*
 		 * We wish to retry with bit banging
diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c
index 7404cf2aac28..6f972e6ec663 100644
--- a/drivers/gpu/drm/i915/intel_lrc.c
+++ b/drivers/gpu/drm/i915/intel_lrc.c
@@ -1221,6 +1221,14 @@ static int intel_init_workaround_bb(struct intel_engine_cs *engine)
 	return ret;
 }
 
+static u8 gtiir[] = {
+	[RCS] = 0,
+	[BCS] = 0,
+	[VCS] = 1,
+	[VCS2] = 1,
+	[VECS] = 3,
+};
+
 static int gen8_init_common_ring(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
@@ -1245,9 +1253,22 @@ static int gen8_init_common_ring(struct intel_engine_cs *engine)
 
 	DRM_DEBUG_DRIVER("Execlists enabled for %s\n", engine->name);
 
-	/* After a GPU reset, we may have requests to replay */
+	GEM_BUG_ON(engine->id >= ARRAY_SIZE(gtiir));
+
+	/*
+	 * Clear any pending interrupt state.
+	 *
+	 * We do it twice out of paranoia that some of the IIR are double
+	 * buffered, and if we only reset it once there may still be
+	 * an interrupt pending.
+	 */
+	I915_WRITE(GEN8_GT_IIR(gtiir[engine->id]),
+		   GT_CONTEXT_SWITCH_INTERRUPT << engine->irq_shift);
+	I915_WRITE(GEN8_GT_IIR(gtiir[engine->id]),
+		   GT_CONTEXT_SWITCH_INTERRUPT << engine->irq_shift);
 	clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
 
+	/* After a GPU reset, we may have requests to replay */
 	submit = false;
 	for (n = 0; n < ARRAY_SIZE(engine->execlist_port); n++) {
 		if (!port_isset(&port[n]))
@@ -1306,6 +1327,31 @@ static void reset_common_ring(struct intel_engine_cs *engine,
 {
 	struct execlist_port *port = engine->execlist_port;
 	struct intel_context *ce;
+	unsigned int n;
+
+	/*
+	 * Catch up with any missed context-switch interrupts.
+	 *
+	 * Ideally we would just read the remaining CSB entries now that we
+	 * know the gpu is idle. However, the CSB registers are sometimes^W
+	 * often trashed across a GPU reset! Instead we have to rely on
+	 * guessing the missed context-switch events by looking at what
+	 * requests were completed.
+	 */
+	if (!request) {
+		for (n = 0; n < ARRAY_SIZE(engine->execlist_port); n++)
+			i915_gem_request_put(port_request(&port[n]));
+		memset(engine->execlist_port, 0, sizeof(engine->execlist_port));
+		return;
+	}
+
+	if (request->ctx != port_request(port)->ctx) {
+		i915_gem_request_put(port_request(port));
+		port[0] = port[1];
+		memset(&port[1], 0, sizeof(port[1]));
+	}
+
+	GEM_BUG_ON(request->ctx != port_request(port)->ctx);
 
 	/* If the request was innocent, we leave the request in the ELSP
 	 * and will try to replay it on restarting. The context image may
@@ -1317,7 +1363,7 @@ static void reset_common_ring(struct intel_engine_cs *engine,
 	 * and have to at least restore the RING register in the context
 	 * image back to the expected values to skip over the guilty request.
 	 */
-	if (!request || request->fence.error != -EIO)
+	if (request->fence.error != -EIO)
 		return;
 
 	/* We want a simple context + ring to execute the breadcrumb update.
@@ -1339,15 +1385,6 @@ static void reset_common_ring(struct intel_engine_cs *engine,
 	request->ring->head = request->postfix;
 	intel_ring_update_space(request->ring);
 
-	/* Catch up with any missed context-switch interrupts */
-	if (request->ctx != port_request(port)->ctx) {
-		i915_gem_request_put(port_request(port));
-		port[0] = port[1];
-		memset(&port[1], 0, sizeof(port[1]));
-	}
-
-	GEM_BUG_ON(request->ctx != port_request(port)->ctx);
-
 	/* Reset WaIdleLiteRestore:bdw,skl as well */
 	request->tail =
 		intel_ring_wrap(request->ring,
@@ -2071,7 +2108,7 @@ void intel_lr_context_resume(struct drm_i915_private *dev_priv)
 	 * So to avoid that we reset the context images upon resume. For
 	 * simplicity, we just zero everything out.
 	 */
-	list_for_each_entry(ctx, &dev_priv->context_list, link) {
+	list_for_each_entry(ctx, &dev_priv->contexts.list, link) {
 		for_each_engine(engine, dev_priv, id) {
 			struct intel_context *ce = &ctx->engine[engine->id];
 			u32 *reg;
diff --git a/drivers/gpu/drm/i915/intel_lrc.h b/drivers/gpu/drm/i915/intel_lrc.h
index 52b3a1fd4059..57ef5833c427 100644
--- a/drivers/gpu/drm/i915/intel_lrc.h
+++ b/drivers/gpu/drm/i915/intel_lrc.h
@@ -63,7 +63,6 @@ enum {
 };
 
 /* Logical Rings */
-void intel_logical_ring_stop(struct intel_engine_cs *engine);
 void intel_logical_ring_cleanup(struct intel_engine_cs *engine);
 int logical_render_ring_init(struct intel_engine_cs *engine);
 int logical_xcs_ring_init(struct intel_engine_cs *engine);
diff --git a/drivers/gpu/drm/i915/intel_lspcon.c b/drivers/gpu/drm/i915/intel_lspcon.c
index 5abef482eacf..beb9baaf2f2e 100644
--- a/drivers/gpu/drm/i915/intel_lspcon.c
+++ b/drivers/gpu/drm/i915/intel_lspcon.c
@@ -210,8 +210,8 @@ bool lspcon_init(struct intel_digital_port *intel_dig_port)
 	struct drm_device *dev = intel_dig_port->base.base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 
-	if (!IS_GEN9(dev_priv)) {
-		DRM_ERROR("LSPCON is supported on GEN9 only\n");
+	if (!HAS_LSPCON(dev_priv)) {
+		DRM_ERROR("LSPCON is not supported on this platform\n");
 		return false;
 	}
 
diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c
index 6fe5d7c3bc23..8e215777c7f4 100644
--- a/drivers/gpu/drm/i915/intel_lvds.c
+++ b/drivers/gpu/drm/i915/intel_lvds.c
@@ -595,10 +595,8 @@ static const struct drm_connector_helper_funcs intel_lvds_connector_helper_funcs
 };
 
 static const struct drm_connector_funcs intel_lvds_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.detect = intel_lvds_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.set_property = drm_atomic_helper_connector_set_property,
 	.atomic_get_property = intel_digital_connector_atomic_get_property,
 	.atomic_set_property = intel_digital_connector_atomic_set_property,
 	.late_register = intel_connector_register,
@@ -1140,7 +1138,8 @@ void intel_lvds_init(struct drm_i915_private *dev_priv)
 out:
 	mutex_unlock(&dev->mode_config.mutex);
 
-	intel_panel_init(&intel_connector->panel, fixed_mode, downclock_mode);
+	intel_panel_init(&intel_connector->panel, fixed_mode, NULL,
+			 downclock_mode);
 	intel_panel_setup_backlight(connector, INVALID_PIPE);
 
 	lvds_encoder->is_dual_link = compute_is_dual_link_lvds(lvds_encoder);
diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c
index 2bd03001cc70..98154efcb2f4 100644
--- a/drivers/gpu/drm/i915/intel_opregion.c
+++ b/drivers/gpu/drm/i915/intel_opregion.c
@@ -27,6 +27,7 @@
 
 #include <linux/acpi.h>
 #include <linux/dmi.h>
+#include <linux/firmware.h>
 #include <acpi/video.h>
 
 #include <drm/drmP.h>
@@ -829,6 +830,10 @@ void intel_opregion_unregister(struct drm_i915_private *dev_priv)
 		memunmap(opregion->rvda);
 		opregion->rvda = NULL;
 	}
+	if (opregion->vbt_firmware) {
+		kfree(opregion->vbt_firmware);
+		opregion->vbt_firmware = NULL;
+	}
 	opregion->header = NULL;
 	opregion->acpi = NULL;
 	opregion->swsci = NULL;
@@ -912,6 +917,43 @@ static const struct dmi_system_id intel_no_opregion_vbt[] = {
 	{ }
 };
 
+static int intel_load_vbt_firmware(struct drm_i915_private *dev_priv)
+{
+	struct intel_opregion *opregion = &dev_priv->opregion;
+	const struct firmware *fw = NULL;
+	const char *name = i915.vbt_firmware;
+	int ret;
+
+	if (!name || !*name)
+		return -ENOENT;
+
+	ret = request_firmware(&fw, name, &dev_priv->drm.pdev->dev);
+	if (ret) {
+		DRM_ERROR("Requesting VBT firmware \"%s\" failed (%d)\n",
+			  name, ret);
+		return ret;
+	}
+
+	if (intel_bios_is_valid_vbt(fw->data, fw->size)) {
+		opregion->vbt_firmware = kmemdup(fw->data, fw->size, GFP_KERNEL);
+		if (opregion->vbt_firmware) {
+			DRM_DEBUG_KMS("Found valid VBT firmware \"%s\"\n", name);
+			opregion->vbt = opregion->vbt_firmware;
+			opregion->vbt_size = fw->size;
+			ret = 0;
+		} else {
+			ret = -ENOMEM;
+		}
+	} else {
+		DRM_DEBUG_KMS("Invalid VBT firmware \"%s\"\n", name);
+		ret = -EINVAL;
+	}
+
+	release_firmware(fw);
+
+	return ret;
+}
+
 int intel_opregion_setup(struct drm_i915_private *dev_priv)
 {
 	struct intel_opregion *opregion = &dev_priv->opregion;
@@ -974,6 +1016,9 @@ int intel_opregion_setup(struct drm_i915_private *dev_priv)
 	if (mboxes & MBOX_ASLE_EXT)
 		DRM_DEBUG_DRIVER("ASLE extension supported\n");
 
+	if (intel_load_vbt_firmware(dev_priv) == 0)
+		goto out;
+
 	if (dmi_check_system(intel_no_opregion_vbt))
 		goto out;
 
diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c
index b96aed941b97..aace22e7ccac 100644
--- a/drivers/gpu/drm/i915/intel_overlay.c
+++ b/drivers/gpu/drm/i915/intel_overlay.c
@@ -799,9 +799,13 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
 	if (ret != 0)
 		return ret;
 
+	atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
+
 	vma = i915_gem_object_pin_to_display_plane(new_bo, 0, NULL);
-	if (IS_ERR(vma))
-		return PTR_ERR(vma);
+	if (IS_ERR(vma)) {
+		ret = PTR_ERR(vma);
+		goto out_pin_section;
+	}
 
 	ret = i915_vma_put_fence(vma);
 	if (ret)
@@ -886,6 +890,9 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
 
 out_unpin:
 	i915_gem_object_unpin_from_display_plane(vma);
+out_pin_section:
+	atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
+
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c
index 593349be8b9d..3b1c5d783ee7 100644
--- a/drivers/gpu/drm/i915/intel_panel.c
+++ b/drivers/gpu/drm/i915/intel_panel.c
@@ -110,7 +110,8 @@ intel_pch_panel_fitting(struct intel_crtc *intel_crtc,
 
 	/* Native modes don't need fitting */
 	if (adjusted_mode->crtc_hdisplay == pipe_config->pipe_src_w &&
-	    adjusted_mode->crtc_vdisplay == pipe_config->pipe_src_h)
+	    adjusted_mode->crtc_vdisplay == pipe_config->pipe_src_h &&
+	    !pipe_config->ycbcr420)
 		goto done;
 
 	switch (fitting_mode) {
@@ -1698,6 +1699,8 @@ bxt_setup_backlight(struct intel_connector *connector, enum pipe unused)
 	if (!panel->backlight.max)
 		return -ENODEV;
 
+	panel->backlight.min = get_backlight_min_vbt(connector);
+
 	val = bxt_get_backlight(connector);
 	val = intel_panel_compute_brightness(connector, val);
 	panel->backlight.level = clamp(val, panel->backlight.min,
@@ -1734,6 +1737,8 @@ cnp_setup_backlight(struct intel_connector *connector, enum pipe unused)
 	if (!panel->backlight.max)
 		return -ENODEV;
 
+	panel->backlight.min = get_backlight_min_vbt(connector);
+
 	val = bxt_get_backlight(connector);
 	val = intel_panel_compute_brightness(connector, val);
 	panel->backlight.level = clamp(val, panel->backlight.min,
@@ -1919,11 +1924,13 @@ intel_panel_init_backlight_funcs(struct intel_panel *panel)
 
 int intel_panel_init(struct intel_panel *panel,
 		     struct drm_display_mode *fixed_mode,
+		     struct drm_display_mode *alt_fixed_mode,
 		     struct drm_display_mode *downclock_mode)
 {
 	intel_panel_init_backlight_funcs(panel);
 
 	panel->fixed_mode = fixed_mode;
+	panel->alt_fixed_mode = alt_fixed_mode;
 	panel->downclock_mode = downclock_mode;
 
 	return 0;
@@ -1937,6 +1944,10 @@ void intel_panel_fini(struct intel_panel *panel)
 	if (panel->fixed_mode)
 		drm_mode_destroy(intel_connector->base.dev, panel->fixed_mode);
 
+	if (panel->alt_fixed_mode)
+		drm_mode_destroy(intel_connector->base.dev,
+				panel->alt_fixed_mode);
+
 	if (panel->downclock_mode)
 		drm_mode_destroy(intel_connector->base.dev,
 				panel->downclock_mode);
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index 40b224b44d1b..ed662937ec3c 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -62,6 +62,20 @@ static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
 	I915_WRITE(CHICKEN_PAR1_1,
 		   I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
 
+	/*
+	 * Display WA#0390: skl,bxt,kbl,glk
+	 *
+	 * Must match Sampler, Pixel Back End, and Media
+	 * (0xE194 bit 8, 0x7014 bit 13, 0x4DDC bits 27 and 31).
+	 *
+	 * Including bits outside the page in the hash would
+	 * require 2 (or 4?) MiB alignment of resources. Just
+	 * assume the defaul hashing mode which only uses bits
+	 * within the page.
+	 */
+	I915_WRITE(CHICKEN_PAR1_1,
+		   I915_READ(CHICKEN_PAR1_1) & ~SKL_RC_HASH_OUTSIDE);
+
 	I915_WRITE(GEN8_CONFIG0,
 		   I915_READ(GEN8_CONFIG0) | GEN9_DEFAULT_FIXES);
 
@@ -78,6 +92,12 @@ static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
 	/* WaFbcHighMemBwCorruptionAvoidance:skl,bxt,kbl,cfl */
 	I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
 		   ILK_DPFC_DISABLE_DUMMY0);
+
+	if (IS_SKYLAKE(dev_priv)) {
+		/* WaDisableDopClockGating */
+		I915_WRITE(GEN7_MISCCPCTL, I915_READ(GEN7_MISCCPCTL)
+			   & ~GEN7_DOP_CLOCK_GATE_ENABLE);
+	}
 }
 
 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
@@ -2758,7 +2778,7 @@ hsw_compute_linetime_wm(const struct intel_crtc_state *cstate)
 static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
 				  uint16_t wm[8])
 {
-	if (IS_GEN9(dev_priv)) {
+	if (INTEL_GEN(dev_priv) >= 9) {
 		uint32_t val;
 		int ret, i;
 		int level, max_level = ilk_wm_max_level(dev_priv);
@@ -2818,7 +2838,7 @@ static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
 		}
 
 		/*
-		 * WaWmMemoryReadLatency:skl,glk
+		 * WaWmMemoryReadLatency:skl+,glk
 		 *
 		 * punit doesn't take into account the read latency so we need
 		 * to add 2us to the various latency levels we retrieve from the
@@ -2857,6 +2877,8 @@ static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
 		wm[0] = 7;
 		wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
 		wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
+	} else {
+		MISSING_CASE(INTEL_DEVID(dev_priv));
 	}
 }
 
@@ -2912,7 +2934,7 @@ static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
 		 * - latencies are in us on gen9.
 		 * - before then, WM1+ latency values are in 0.5us units
 		 */
-		if (IS_GEN9(dev_priv))
+		if (INTEL_GEN(dev_priv) >= 9)
 			latency *= 10;
 		else if (level > 0)
 			latency *= 5;
@@ -3530,8 +3552,6 @@ bool ilk_disable_lp_wm(struct drm_device *dev)
 	return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
 }
 
-#define SKL_SAGV_BLOCK_TIME	30 /* µs */
-
 /*
  * FIXME: We still don't have the proper code detect if we need to apply the WA,
  * so assume we'll always need it in order to avoid underruns.
@@ -3549,7 +3569,8 @@ static bool skl_needs_memory_bw_wa(struct intel_atomic_state *state)
 static bool
 intel_has_sagv(struct drm_i915_private *dev_priv)
 {
-	if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv))
+	if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv) ||
+	    IS_CANNONLAKE(dev_priv))
 		return true;
 
 	if (IS_SKYLAKE(dev_priv) &&
@@ -3655,12 +3676,13 @@ bool intel_can_enable_sagv(struct drm_atomic_state *state)
 	struct intel_crtc_state *cstate;
 	enum pipe pipe;
 	int level, latency;
+	int sagv_block_time_us = IS_GEN9(dev_priv) ? 30 : 20;
 
 	if (!intel_has_sagv(dev_priv))
 		return false;
 
 	/*
-	 * SKL workaround: bspec recommends we disable the SAGV when we have
+	 * SKL+ workaround: bspec recommends we disable the SAGV when we have
 	 * more then one pipe enabled
 	 *
 	 * If there are no active CRTCs, no additional checks need be performed
@@ -3699,11 +3721,11 @@ bool intel_can_enable_sagv(struct drm_atomic_state *state)
 			latency += 15;
 
 		/*
-		 * If any of the planes on this pipe don't enable wm levels
-		 * that incur memory latencies higher then 30µs we can't enable
-		 * the SAGV
+		 * If any of the planes on this pipe don't enable wm levels that
+		 * incur memory latencies higher than sagv_block_time_us we
+		 * can't enable the SAGV.
 		 */
-		if (latency < SKL_SAGV_BLOCK_TIME)
+		if (latency < sagv_block_time_us)
 			return false;
 	}
 
@@ -3837,7 +3859,7 @@ skl_plane_downscale_amount(const struct intel_crtc_state *cstate,
 	uint_fixed_16_16_t downscale_h, downscale_w;
 
 	if (WARN_ON(!intel_wm_plane_visible(cstate, pstate)))
-		return u32_to_fixed_16_16(0);
+		return u32_to_fixed16(0);
 
 	/* n.b., src is 16.16 fixed point, dst is whole integer */
 	if (plane->id == PLANE_CURSOR) {
@@ -3861,10 +3883,10 @@ skl_plane_downscale_amount(const struct intel_crtc_state *cstate,
 		dst_h = drm_rect_height(&pstate->base.dst);
 	}
 
-	fp_w_ratio = fixed_16_16_div(src_w, dst_w);
-	fp_h_ratio = fixed_16_16_div(src_h, dst_h);
-	downscale_w = max_fixed_16_16(fp_w_ratio, u32_to_fixed_16_16(1));
-	downscale_h = max_fixed_16_16(fp_h_ratio, u32_to_fixed_16_16(1));
+	fp_w_ratio = div_fixed16(src_w, dst_w);
+	fp_h_ratio = div_fixed16(src_h, dst_h);
+	downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
+	downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
 
 	return mul_fixed16(downscale_w, downscale_h);
 }
@@ -3872,7 +3894,7 @@ skl_plane_downscale_amount(const struct intel_crtc_state *cstate,
 static uint_fixed_16_16_t
 skl_pipe_downscale_amount(const struct intel_crtc_state *crtc_state)
 {
-	uint_fixed_16_16_t pipe_downscale = u32_to_fixed_16_16(1);
+	uint_fixed_16_16_t pipe_downscale = u32_to_fixed16(1);
 
 	if (!crtc_state->base.enable)
 		return pipe_downscale;
@@ -3891,10 +3913,10 @@ skl_pipe_downscale_amount(const struct intel_crtc_state *crtc_state)
 		if (!dst_w || !dst_h)
 			return pipe_downscale;
 
-		fp_w_ratio = fixed_16_16_div(src_w, dst_w);
-		fp_h_ratio = fixed_16_16_div(src_h, dst_h);
-		downscale_w = max_fixed_16_16(fp_w_ratio, u32_to_fixed_16_16(1));
-		downscale_h = max_fixed_16_16(fp_h_ratio, u32_to_fixed_16_16(1));
+		fp_w_ratio = div_fixed16(src_w, dst_w);
+		fp_h_ratio = div_fixed16(src_h, dst_h);
+		downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
+		downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
 
 		pipe_downscale = mul_fixed16(downscale_w, downscale_h);
 	}
@@ -3913,14 +3935,14 @@ int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc,
 	int crtc_clock, dotclk;
 	uint32_t pipe_max_pixel_rate;
 	uint_fixed_16_16_t pipe_downscale;
-	uint_fixed_16_16_t max_downscale = u32_to_fixed_16_16(1);
+	uint_fixed_16_16_t max_downscale = u32_to_fixed16(1);
 
 	if (!cstate->base.enable)
 		return 0;
 
 	drm_atomic_crtc_state_for_each_plane_state(plane, pstate, crtc_state) {
 		uint_fixed_16_16_t plane_downscale;
-		uint_fixed_16_16_t fp_9_div_8 = fixed_16_16_div(9, 8);
+		uint_fixed_16_16_t fp_9_div_8 = div_fixed16(9, 8);
 		int bpp;
 
 		if (!intel_wm_plane_visible(cstate,
@@ -3938,7 +3960,7 @@ int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc,
 			plane_downscale = mul_fixed16(plane_downscale,
 						      fp_9_div_8);
 
-		max_downscale = max_fixed_16_16(plane_downscale, max_downscale);
+		max_downscale = max_fixed16(plane_downscale, max_downscale);
 	}
 	pipe_downscale = skl_pipe_downscale_amount(cstate);
 
@@ -4071,7 +4093,9 @@ skl_ddb_min_alloc(const struct drm_plane_state *pstate,
 
 	/* For Non Y-tile return 8-blocks */
 	if (fb->modifier != I915_FORMAT_MOD_Y_TILED &&
-	    fb->modifier != I915_FORMAT_MOD_Yf_TILED)
+	    fb->modifier != I915_FORMAT_MOD_Yf_TILED &&
+	    fb->modifier != I915_FORMAT_MOD_Y_TILED_CCS &&
+	    fb->modifier != I915_FORMAT_MOD_Yf_TILED_CCS)
 		return 8;
 
 	/*
@@ -4266,8 +4290,9 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
  * should allow pixel_rate up to ~2 GHz which seems sufficient since max
  * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
 */
-static uint_fixed_16_16_t skl_wm_method1(uint32_t pixel_rate, uint8_t cpp,
-					 uint32_t latency)
+static uint_fixed_16_16_t
+skl_wm_method1(const struct drm_i915_private *dev_priv, uint32_t pixel_rate,
+	       uint8_t cpp, uint32_t latency)
 {
 	uint32_t wm_intermediate_val;
 	uint_fixed_16_16_t ret;
@@ -4276,7 +4301,11 @@ static uint_fixed_16_16_t skl_wm_method1(uint32_t pixel_rate, uint8_t cpp,
 		return FP_16_16_MAX;
 
 	wm_intermediate_val = latency * pixel_rate * cpp;
-	ret = fixed_16_16_div_u64(wm_intermediate_val, 1000 * 512);
+	ret = div_fixed16(wm_intermediate_val, 1000 * 512);
+
+	if (INTEL_GEN(dev_priv) >= 10)
+		ret = add_fixed16_u32(ret, 1);
+
 	return ret;
 }
 
@@ -4294,7 +4323,7 @@ static uint_fixed_16_16_t skl_wm_method2(uint32_t pixel_rate,
 	wm_intermediate_val = latency * pixel_rate;
 	wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
 					   pipe_htotal * 1000);
-	ret = mul_u32_fixed_16_16(wm_intermediate_val, plane_blocks_per_line);
+	ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
 	return ret;
 }
 
@@ -4306,15 +4335,15 @@ intel_get_linetime_us(struct intel_crtc_state *cstate)
 	uint_fixed_16_16_t linetime_us;
 
 	if (!cstate->base.active)
-		return u32_to_fixed_16_16(0);
+		return u32_to_fixed16(0);
 
 	pixel_rate = cstate->pixel_rate;
 
 	if (WARN_ON(pixel_rate == 0))
-		return u32_to_fixed_16_16(0);
+		return u32_to_fixed16(0);
 
 	crtc_htotal = cstate->base.adjusted_mode.crtc_htotal;
-	linetime_us = fixed_16_16_div_u64(crtc_htotal * 1000, pixel_rate);
+	linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
 
 	return linetime_us;
 }
@@ -4361,7 +4390,7 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 	uint32_t plane_bytes_per_line;
 	uint32_t res_blocks, res_lines;
 	uint8_t cpp;
-	uint32_t width = 0, height = 0;
+	uint32_t width = 0;
 	uint32_t plane_pixel_rate;
 	uint_fixed_16_16_t y_tile_minimum;
 	uint32_t y_min_scanlines;
@@ -4377,7 +4406,9 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 	}
 
 	y_tiled = fb->modifier == I915_FORMAT_MOD_Y_TILED ||
-		  fb->modifier == I915_FORMAT_MOD_Yf_TILED;
+		  fb->modifier == I915_FORMAT_MOD_Yf_TILED ||
+		  fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
+		  fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
 	x_tiled = fb->modifier == I915_FORMAT_MOD_X_TILED;
 
 	/* Display WA #1141: kbl,cfl */
@@ -4390,7 +4421,6 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 
 	if (plane->id == PLANE_CURSOR) {
 		width = intel_pstate->base.crtc_w;
-		height = intel_pstate->base.crtc_h;
 	} else {
 		/*
 		 * Src coordinates are already rotated by 270 degrees for
@@ -4398,16 +4428,13 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 		 * GTT mapping), hence no need to account for rotation here.
 		 */
 		width = drm_rect_width(&intel_pstate->base.src) >> 16;
-		height = drm_rect_height(&intel_pstate->base.src) >> 16;
 	}
 
-	cpp = fb->format->cpp[0];
+	cpp = (fb->format->format == DRM_FORMAT_NV12) ? fb->format->cpp[1] :
+							fb->format->cpp[0];
 	plane_pixel_rate = skl_adjusted_plane_pixel_rate(cstate, intel_pstate);
 
 	if (drm_rotation_90_or_270(pstate->rotation)) {
-		int cpp = (fb->format->format == DRM_FORMAT_NV12) ?
-			fb->format->cpp[1] :
-			fb->format->cpp[0];
 
 		switch (cpp) {
 		case 1:
@@ -4434,51 +4461,62 @@ static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
 	if (y_tiled) {
 		interm_pbpl = DIV_ROUND_UP(plane_bytes_per_line *
 					   y_min_scanlines, 512);
-		plane_blocks_per_line = fixed_16_16_div(interm_pbpl,
+
+		if (INTEL_GEN(dev_priv) >= 10)
+			interm_pbpl++;
+
+		plane_blocks_per_line = div_fixed16(interm_pbpl,
 							y_min_scanlines);
-	} else if (x_tiled) {
+	} else if (x_tiled && INTEL_GEN(dev_priv) == 9) {
 		interm_pbpl = DIV_ROUND_UP(plane_bytes_per_line, 512);
-		plane_blocks_per_line = u32_to_fixed_16_16(interm_pbpl);
+		plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
 	} else {
 		interm_pbpl = DIV_ROUND_UP(plane_bytes_per_line, 512) + 1;
-		plane_blocks_per_line = u32_to_fixed_16_16(interm_pbpl);
+		plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
 	}
 
-	method1 = skl_wm_method1(plane_pixel_rate, cpp, latency);
+	method1 = skl_wm_method1(dev_priv, plane_pixel_rate, cpp, latency);
 	method2 = skl_wm_method2(plane_pixel_rate,
 				 cstate->base.adjusted_mode.crtc_htotal,
 				 latency,
 				 plane_blocks_per_line);
 
-	y_tile_minimum = mul_u32_fixed_16_16(y_min_scanlines,
-					     plane_blocks_per_line);
+	y_tile_minimum = mul_u32_fixed16(y_min_scanlines,
+					 plane_blocks_per_line);
 
 	if (y_tiled) {
-		selected_result = max_fixed_16_16(method2, y_tile_minimum);
+		selected_result = max_fixed16(method2, y_tile_minimum);
 	} else {
 		uint32_t linetime_us;
 
-		linetime_us = fixed_16_16_to_u32_round_up(
+		linetime_us = fixed16_to_u32_round_up(
 				intel_get_linetime_us(cstate));
 		if ((cpp * cstate->base.adjusted_mode.crtc_htotal / 512 < 1) &&
 		    (plane_bytes_per_line / 512 < 1))
 			selected_result = method2;
 		else if (ddb_allocation >=
-			 fixed_16_16_to_u32_round_up(plane_blocks_per_line))
-			selected_result = min_fixed_16_16(method1, method2);
+			 fixed16_to_u32_round_up(plane_blocks_per_line))
+			selected_result = min_fixed16(method1, method2);
 		else if (latency >= linetime_us)
-			selected_result = min_fixed_16_16(method1, method2);
+			selected_result = min_fixed16(method1, method2);
 		else
 			selected_result = method1;
 	}
 
-	res_blocks = fixed_16_16_to_u32_round_up(selected_result) + 1;
+	res_blocks = fixed16_to_u32_round_up(selected_result) + 1;
 	res_lines = div_round_up_fixed16(selected_result,
 					 plane_blocks_per_line);
 
+	/* Display WA #1125: skl,bxt,kbl,glk */
+	if (level == 0 &&
+	    (fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
+	     fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS))
+		res_blocks += fixed16_to_u32_round_up(y_tile_minimum);
+
+	/* Display WA #1126: skl,bxt,kbl,glk */
 	if (level >= 1 && level <= 7) {
 		if (y_tiled) {
-			res_blocks += fixed_16_16_to_u32_round_up(y_tile_minimum);
+			res_blocks += fixed16_to_u32_round_up(y_tile_minimum);
 			res_lines += y_min_scanlines;
 		} else {
 			res_blocks++;
@@ -4563,8 +4601,7 @@ skl_compute_linetime_wm(struct intel_crtc_state *cstate)
 	if (is_fixed16_zero(linetime_us))
 		return 0;
 
-	linetime_wm = fixed_16_16_to_u32_round_up(mul_u32_fixed_16_16(8,
-				linetime_us));
+	linetime_wm = fixed16_to_u32_round_up(mul_u32_fixed16(8, linetime_us));
 
 	/* Display WA #1135: bxt. */
 	if (IS_BROXTON(dev_priv) && dev_priv->ipc_enabled)
@@ -5852,7 +5889,7 @@ static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val)
 	 * the hw runs at the minimal clock before selecting the desired
 	 * frequency, if the down threshold expires in that window we will not
 	 * receive a down interrupt. */
-	if (IS_GEN9(dev_priv)) {
+	if (INTEL_GEN(dev_priv) >= 9) {
 		limits = (dev_priv->rps.max_freq_softlimit) << 23;
 		if (val <= dev_priv->rps.min_freq_softlimit)
 			limits |= (dev_priv->rps.min_freq_softlimit) << 14;
@@ -5994,7 +6031,7 @@ static int gen6_set_rps(struct drm_i915_private *dev_priv, u8 val)
 	if (val != dev_priv->rps.cur_freq) {
 		gen6_set_rps_thresholds(dev_priv, val);
 
-		if (IS_GEN9(dev_priv))
+		if (INTEL_GEN(dev_priv) >= 9)
 			I915_WRITE(GEN6_RPNSWREQ,
 				   GEN9_FREQUENCY(val));
 		else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
@@ -6126,47 +6163,35 @@ void gen6_rps_idle(struct drm_i915_private *dev_priv)
 			   gen6_sanitize_rps_pm_mask(dev_priv, ~0));
 	}
 	mutex_unlock(&dev_priv->rps.hw_lock);
-
-	spin_lock(&dev_priv->rps.client_lock);
-	while (!list_empty(&dev_priv->rps.clients))
-		list_del_init(dev_priv->rps.clients.next);
-	spin_unlock(&dev_priv->rps.client_lock);
 }
 
-void gen6_rps_boost(struct drm_i915_private *dev_priv,
-		    struct intel_rps_client *rps,
-		    unsigned long submitted)
+void gen6_rps_boost(struct drm_i915_gem_request *rq,
+		    struct intel_rps_client *rps)
 {
+	struct drm_i915_private *i915 = rq->i915;
+	bool boost;
+
 	/* This is intentionally racy! We peek at the state here, then
 	 * validate inside the RPS worker.
 	 */
-	if (!(dev_priv->gt.awake &&
-	      dev_priv->rps.enabled &&
-	      dev_priv->rps.cur_freq < dev_priv->rps.boost_freq))
+	if (!i915->rps.enabled)
 		return;
 
-	/* Force a RPS boost (and don't count it against the client) if
-	 * the GPU is severely congested.
-	 */
-	if (rps && time_after(jiffies, submitted + DRM_I915_THROTTLE_JIFFIES))
-		rps = NULL;
-
-	spin_lock(&dev_priv->rps.client_lock);
-	if (rps == NULL || list_empty(&rps->link)) {
-		spin_lock_irq(&dev_priv->irq_lock);
-		if (dev_priv->rps.interrupts_enabled) {
-			dev_priv->rps.client_boost = true;
-			schedule_work(&dev_priv->rps.work);
-		}
-		spin_unlock_irq(&dev_priv->irq_lock);
-
-		if (rps != NULL) {
-			list_add(&rps->link, &dev_priv->rps.clients);
-			rps->boosts++;
-		} else
-			dev_priv->rps.boosts++;
+	boost = false;
+	spin_lock_irq(&rq->lock);
+	if (!rq->waitboost && !i915_gem_request_completed(rq)) {
+		atomic_inc(&i915->rps.num_waiters);
+		rq->waitboost = true;
+		boost = true;
 	}
-	spin_unlock(&dev_priv->rps.client_lock);
+	spin_unlock_irq(&rq->lock);
+	if (!boost)
+		return;
+
+	if (READ_ONCE(i915->rps.cur_freq) < i915->rps.boost_freq)
+		schedule_work(&i915->rps.work);
+
+	atomic_inc(rps ? &rps->boosts : &i915->rps.boosts);
 }
 
 int intel_set_rps(struct drm_i915_private *dev_priv, u8 val)
@@ -6365,7 +6390,7 @@ static void gen6_init_rps_frequencies(struct drm_i915_private *dev_priv)
 
 	dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq;
 	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv) ||
-	    IS_GEN9_BC(dev_priv)) {
+	    IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) {
 		u32 ddcc_status = 0;
 
 		if (sandybridge_pcode_read(dev_priv,
@@ -6378,7 +6403,7 @@ static void gen6_init_rps_frequencies(struct drm_i915_private *dev_priv)
 					dev_priv->rps.max_freq);
 	}
 
-	if (IS_GEN9_BC(dev_priv)) {
+	if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) {
 		/* Store the frequency values in 16.66 MHZ units, which is
 		 * the natural hardware unit for SKL
 		 */
@@ -6684,7 +6709,7 @@ static void gen6_update_ring_freq(struct drm_i915_private *dev_priv)
 	/* convert DDR frequency from units of 266.6MHz to bandwidth */
 	min_ring_freq = mult_frac(min_ring_freq, 8, 3);
 
-	if (IS_GEN9_BC(dev_priv)) {
+	if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) {
 		/* Convert GT frequency to 50 HZ units */
 		min_gpu_freq = dev_priv->rps.min_freq / GEN9_FREQ_SCALER;
 		max_gpu_freq = dev_priv->rps.max_freq / GEN9_FREQ_SCALER;
@@ -6702,7 +6727,7 @@ static void gen6_update_ring_freq(struct drm_i915_private *dev_priv)
 		int diff = max_gpu_freq - gpu_freq;
 		unsigned int ia_freq = 0, ring_freq = 0;
 
-		if (IS_GEN9_BC(dev_priv)) {
+		if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) {
 			/*
 			 * ring_freq = 2 * GT. ring_freq is in 100MHz units
 			 * No floor required for ring frequency on SKL.
@@ -7833,7 +7858,7 @@ void intel_enable_gt_powersave(struct drm_i915_private *dev_priv)
 	} else if (INTEL_GEN(dev_priv) >= 9) {
 		gen9_enable_rc6(dev_priv);
 		gen9_enable_rps(dev_priv);
-		if (IS_GEN9_BC(dev_priv))
+		if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv))
 			gen6_update_ring_freq(dev_priv);
 	} else if (IS_BROADWELL(dev_priv)) {
 		gen8_enable_rps(dev_priv);
@@ -8848,6 +8873,7 @@ static inline int gen6_check_mailbox_status(struct drm_i915_private *dev_priv)
 	case GEN6_PCODE_SUCCESS:
 		return 0;
 	case GEN6_PCODE_UNIMPLEMENTED_CMD:
+		return -ENODEV;
 	case GEN6_PCODE_ILLEGAL_CMD:
 		return -ENXIO;
 	case GEN6_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE:
@@ -8895,7 +8921,8 @@ int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val
 	 */
 
 	if (I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
-		DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
+		DRM_DEBUG_DRIVER("warning: pcode (read from mbox %x) mailbox access failed for %ps\n",
+				 mbox, __builtin_return_address(0));
 		return -EAGAIN;
 	}
 
@@ -8906,7 +8933,8 @@ int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val
 	if (__intel_wait_for_register_fw(dev_priv,
 					 GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0,
 					 500, 0, NULL)) {
-		DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
+		DRM_ERROR("timeout waiting for pcode read (from mbox %x) to finish for %ps\n",
+			  mbox, __builtin_return_address(0));
 		return -ETIMEDOUT;
 	}
 
@@ -8919,8 +8947,8 @@ int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val
 		status = gen6_check_mailbox_status(dev_priv);
 
 	if (status) {
-		DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed: %d\n",
-				 status);
+		DRM_DEBUG_DRIVER("warning: pcode (read from mbox %x) mailbox access failed for %ps: %d\n",
+				 mbox, __builtin_return_address(0), status);
 		return status;
 	}
 
@@ -8940,7 +8968,8 @@ int sandybridge_pcode_write(struct drm_i915_private *dev_priv,
 	 */
 
 	if (I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
-		DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
+		DRM_DEBUG_DRIVER("warning: pcode (write of 0x%08x to mbox %x) mailbox access failed for %ps\n",
+				 val, mbox, __builtin_return_address(0));
 		return -EAGAIN;
 	}
 
@@ -8951,7 +8980,8 @@ int sandybridge_pcode_write(struct drm_i915_private *dev_priv,
 	if (__intel_wait_for_register_fw(dev_priv,
 					 GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0,
 					 500, 0, NULL)) {
-		DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
+		DRM_ERROR("timeout waiting for pcode write of 0x%08x to mbox %x to finish for %ps\n",
+			  val, mbox, __builtin_return_address(0));
 		return -ETIMEDOUT;
 	}
 
@@ -8963,8 +8993,8 @@ int sandybridge_pcode_write(struct drm_i915_private *dev_priv,
 		status = gen6_check_mailbox_status(dev_priv);
 
 	if (status) {
-		DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed: %d\n",
-				 status);
+		DRM_DEBUG_DRIVER("warning: pcode (write of 0x%08x to mbox %x) mailbox access failed for %ps: %d\n",
+				 val, mbox, __builtin_return_address(0), status);
 		return status;
 	}
 
@@ -9078,7 +9108,7 @@ static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
 
 int intel_gpu_freq(struct drm_i915_private *dev_priv, int val)
 {
-	if (IS_GEN9(dev_priv))
+	if (INTEL_GEN(dev_priv) >= 9)
 		return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
 					 GEN9_FREQ_SCALER);
 	else if (IS_CHERRYVIEW(dev_priv))
@@ -9091,7 +9121,7 @@ int intel_gpu_freq(struct drm_i915_private *dev_priv, int val)
 
 int intel_freq_opcode(struct drm_i915_private *dev_priv, int val)
 {
-	if (IS_GEN9(dev_priv))
+	if (INTEL_GEN(dev_priv) >= 9)
 		return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
 					 GT_FREQUENCY_MULTIPLIER);
 	else if (IS_CHERRYVIEW(dev_priv))
@@ -9113,7 +9143,7 @@ static void __intel_rps_boost_work(struct work_struct *work)
 	struct drm_i915_gem_request *req = boost->req;
 
 	if (!i915_gem_request_completed(req))
-		gen6_rps_boost(req->i915, NULL, req->emitted_jiffies);
+		gen6_rps_boost(req, NULL);
 
 	i915_gem_request_put(req);
 	kfree(boost);
@@ -9142,11 +9172,10 @@ void intel_queue_rps_boost_for_request(struct drm_i915_gem_request *req)
 void intel_pm_setup(struct drm_i915_private *dev_priv)
 {
 	mutex_init(&dev_priv->rps.hw_lock);
-	spin_lock_init(&dev_priv->rps.client_lock);
 
 	INIT_DELAYED_WORK(&dev_priv->rps.autoenable_work,
 			  __intel_autoenable_gt_powersave);
-	INIT_LIST_HEAD(&dev_priv->rps.clients);
+	atomic_set(&dev_priv->rps.num_waiters, 0);
 
 	dev_priv->pm.suspended = false;
 	atomic_set(&dev_priv->pm.wakeref_count, 0);
diff --git a/drivers/gpu/drm/i915/intel_psr.c b/drivers/gpu/drm/i915/intel_psr.c
index 559f1ab42bfc..1b31ab002dae 100644
--- a/drivers/gpu/drm/i915/intel_psr.c
+++ b/drivers/gpu/drm/i915/intel_psr.c
@@ -315,6 +315,7 @@ static void intel_enable_source_psr1(struct intel_dp *intel_dp)
 	else
 		val |= EDP_PSR_TP1_TP2_SEL;
 
+	val |= I915_READ(EDP_PSR_CTL) & EDP_PSR_RESTORE_PSR_ACTIVE_CTX_MASK;
 	I915_WRITE(EDP_PSR_CTL, val);
 }
 
diff --git a/drivers/gpu/drm/i915/intel_renderstate_gen9.c b/drivers/gpu/drm/i915/intel_renderstate_gen9.c
index 16a7ec273bd9..7d3ac02f0177 100644
--- a/drivers/gpu/drm/i915/intel_renderstate_gen9.c
+++ b/drivers/gpu/drm/i915/intel_renderstate_gen9.c
@@ -20,7 +20,7 @@
  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  * DEALINGS IN THE SOFTWARE.
  *
- * Generated by: intel-gpu-tools-1.8-220-g01153e7
+ * Generated by: intel-gpu-tools-1.19-177-g68e2eab2
  */
 
 #include "intel_renderstate.h"
@@ -873,7 +873,7 @@ static const u32 gen9_null_state_batch[] = {
 	0x00000000,
 	0x00000000,
 	0x78550003,
-	0x00000000,
+	0x0000000f,
 	0x00000000,
 	0x00000000,
 	0x00000000,
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index acd1da9b62a3..cdf084ef5aae 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -1712,6 +1712,9 @@ u32 *intel_ring_begin(struct drm_i915_gem_request *req,
 	unsigned int total_bytes;
 	u32 *cs;
 
+	/* Packets must be qword aligned. */
+	GEM_BUG_ON(num_dwords & 1);
+
 	total_bytes = bytes + req->reserved_space;
 	GEM_BUG_ON(total_bytes > ring->effective_size);
 
@@ -2140,7 +2143,7 @@ static void intel_ring_default_vfuncs(struct drm_i915_private *dev_priv,
 
 		engine->emit_breadcrumb = gen6_sema_emit_breadcrumb;
 
-		num_rings = hweight32(INTEL_INFO(dev_priv)->ring_mask) - 1;
+		num_rings = INTEL_INFO(dev_priv)->num_rings - 1;
 		if (INTEL_GEN(dev_priv) >= 8) {
 			engine->emit_breadcrumb_sz += num_rings * 6;
 		} else {
@@ -2184,8 +2187,7 @@ int intel_init_render_ring_buffer(struct intel_engine_cs *engine)
 
 			engine->semaphore.signal = gen8_rcs_signal;
 
-			num_rings =
-				hweight32(INTEL_INFO(dev_priv)->ring_mask) - 1;
+			num_rings = INTEL_INFO(dev_priv)->num_rings - 1;
 			engine->emit_breadcrumb_sz += num_rings * 8;
 		}
 	} else if (INTEL_GEN(dev_priv) >= 6) {
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index 6aa20ac8cde3..02d8974bf9ab 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -121,6 +121,7 @@ struct intel_engine_hangcheck {
 	unsigned long action_timestamp;
 	int deadlock;
 	struct intel_instdone instdone;
+	struct drm_i915_gem_request *active_request;
 	bool stalled;
 };
 
@@ -734,4 +735,16 @@ bool intel_engines_are_idle(struct drm_i915_private *dev_priv);
 void intel_engines_mark_idle(struct drm_i915_private *i915);
 void intel_engines_reset_default_submission(struct drm_i915_private *i915);
 
+static inline bool
+__intel_engine_can_store_dword(unsigned int gen, unsigned int class)
+{
+	if (gen <= 2)
+		return false; /* uses physical not virtual addresses */
+
+	if (gen == 6 && class == VIDEO_DECODE_CLASS)
+		return false; /* b0rked */
+
+	return true;
+}
+
 #endif /* _INTEL_RINGBUFFER_H_ */
diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c
index efe80ed5fd4d..b66d8e136aa3 100644
--- a/drivers/gpu/drm/i915/intel_runtime_pm.c
+++ b/drivers/gpu/drm/i915/intel_runtime_pm.c
@@ -50,10 +50,11 @@
  */
 
 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
-				    int power_well_id);
+					 enum i915_power_well_id power_well_id);
 
 static struct i915_power_well *
-lookup_power_well(struct drm_i915_private *dev_priv, int power_well_id);
+lookup_power_well(struct drm_i915_private *dev_priv,
+		  enum i915_power_well_id power_well_id);
 
 const char *
 intel_display_power_domain_str(enum intel_display_power_domain domain)
@@ -168,18 +169,6 @@ static void intel_power_well_put(struct drm_i915_private *dev_priv,
 		intel_power_well_disable(dev_priv, power_well);
 }
 
-/*
- * We should only use the power well if we explicitly asked the hardware to
- * enable it, so check if it's enabled and also check if we've requested it to
- * be enabled.
- */
-static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv,
-				   struct i915_power_well *power_well)
-{
-	return I915_READ(HSW_PWR_WELL_DRIVER) ==
-		     (HSW_PWR_WELL_ENABLE_REQUEST | HSW_PWR_WELL_STATE_ENABLED);
-}
-
 /**
  * __intel_display_power_is_enabled - unlocked check for a power domain
  * @dev_priv: i915 device instance
@@ -278,7 +267,8 @@ void intel_display_set_init_power(struct drm_i915_private *dev_priv,
  * to be enabled, and it will only be disabled if none of the registers is
  * requesting it to be enabled.
  */
-static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv)
+static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv,
+				       u8 irq_pipe_mask, bool has_vga)
 {
 	struct pci_dev *pdev = dev_priv->drm.pdev;
 
@@ -292,264 +282,158 @@ static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv)
 	 * sure vgacon can keep working normally without triggering interrupts
 	 * and error messages.
 	 */
-	vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
-	outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
-	vga_put(pdev, VGA_RSRC_LEGACY_IO);
+	if (has_vga) {
+		vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
+		outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
+		vga_put(pdev, VGA_RSRC_LEGACY_IO);
+	}
 
-	if (IS_BROADWELL(dev_priv))
-		gen8_irq_power_well_post_enable(dev_priv,
-						1 << PIPE_C | 1 << PIPE_B);
+	if (irq_pipe_mask)
+		gen8_irq_power_well_post_enable(dev_priv, irq_pipe_mask);
 }
 
-static void hsw_power_well_pre_disable(struct drm_i915_private *dev_priv)
+static void hsw_power_well_pre_disable(struct drm_i915_private *dev_priv,
+				       u8 irq_pipe_mask)
 {
-	if (IS_BROADWELL(dev_priv))
-		gen8_irq_power_well_pre_disable(dev_priv,
-						1 << PIPE_C | 1 << PIPE_B);
+	if (irq_pipe_mask)
+		gen8_irq_power_well_pre_disable(dev_priv, irq_pipe_mask);
 }
 
-static void skl_power_well_post_enable(struct drm_i915_private *dev_priv,
-				       struct i915_power_well *power_well)
+
+static void hsw_wait_for_power_well_enable(struct drm_i915_private *dev_priv,
+					   struct i915_power_well *power_well)
 {
-	struct pci_dev *pdev = dev_priv->drm.pdev;
+	enum i915_power_well_id id = power_well->id;
+
+	/* Timeout for PW1:10 us, AUX:not specified, other PWs:20 us. */
+	WARN_ON(intel_wait_for_register(dev_priv,
+					HSW_PWR_WELL_CTL_DRIVER(id),
+					HSW_PWR_WELL_CTL_STATE(id),
+					HSW_PWR_WELL_CTL_STATE(id),
+					1));
+}
+
+static u32 hsw_power_well_requesters(struct drm_i915_private *dev_priv,
+				     enum i915_power_well_id id)
+{
+	u32 req_mask = HSW_PWR_WELL_CTL_REQ(id);
+	u32 ret;
+
+	ret = I915_READ(HSW_PWR_WELL_CTL_BIOS(id)) & req_mask ? 1 : 0;
+	ret |= I915_READ(HSW_PWR_WELL_CTL_DRIVER(id)) & req_mask ? 2 : 0;
+	ret |= I915_READ(HSW_PWR_WELL_CTL_KVMR) & req_mask ? 4 : 0;
+	ret |= I915_READ(HSW_PWR_WELL_CTL_DEBUG(id)) & req_mask ? 8 : 0;
+
+	return ret;
+}
+
+static void hsw_wait_for_power_well_disable(struct drm_i915_private *dev_priv,
+					    struct i915_power_well *power_well)
+{
+	enum i915_power_well_id id = power_well->id;
+	bool disabled;
+	u32 reqs;
 
 	/*
-	 * After we re-enable the power well, if we touch VGA register 0x3d5
-	 * we'll get unclaimed register interrupts. This stops after we write
-	 * anything to the VGA MSR register. The vgacon module uses this
-	 * register all the time, so if we unbind our driver and, as a
-	 * consequence, bind vgacon, we'll get stuck in an infinite loop at
-	 * console_unlock(). So make here we touch the VGA MSR register, making
-	 * sure vgacon can keep working normally without triggering interrupts
-	 * and error messages.
+	 * Bspec doesn't require waiting for PWs to get disabled, but still do
+	 * this for paranoia. The known cases where a PW will be forced on:
+	 * - a KVMR request on any power well via the KVMR request register
+	 * - a DMC request on PW1 and MISC_IO power wells via the BIOS and
+	 *   DEBUG request registers
+	 * Skip the wait in case any of the request bits are set and print a
+	 * diagnostic message.
 	 */
-	if (power_well->id == SKL_DISP_PW_2) {
-		vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
-		outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
-		vga_put(pdev, VGA_RSRC_LEGACY_IO);
+	wait_for((disabled = !(I915_READ(HSW_PWR_WELL_CTL_DRIVER(id)) &
+			       HSW_PWR_WELL_CTL_STATE(id))) ||
+		 (reqs = hsw_power_well_requesters(dev_priv, id)), 1);
+	if (disabled)
+		return;
 
-		gen8_irq_power_well_post_enable(dev_priv,
-						1 << PIPE_C | 1 << PIPE_B);
-	}
+	DRM_DEBUG_KMS("%s forced on (bios:%d driver:%d kvmr:%d debug:%d)\n",
+		      power_well->name,
+		      !!(reqs & 1), !!(reqs & 2), !!(reqs & 4), !!(reqs & 8));
 }
 
-static void skl_power_well_pre_disable(struct drm_i915_private *dev_priv,
-				       struct i915_power_well *power_well)
+static void gen9_wait_for_power_well_fuses(struct drm_i915_private *dev_priv,
+					   enum skl_power_gate pg)
 {
-	if (power_well->id == SKL_DISP_PW_2)
-		gen8_irq_power_well_pre_disable(dev_priv,
-						1 << PIPE_C | 1 << PIPE_B);
+	/* Timeout 5us for PG#0, for other PGs 1us */
+	WARN_ON(intel_wait_for_register(dev_priv, SKL_FUSE_STATUS,
+					SKL_FUSE_PG_DIST_STATUS(pg),
+					SKL_FUSE_PG_DIST_STATUS(pg), 1));
 }
 
-static void hsw_set_power_well(struct drm_i915_private *dev_priv,
-			       struct i915_power_well *power_well, bool enable)
+static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
+				  struct i915_power_well *power_well)
 {
-	bool is_enabled, enable_requested;
-	uint32_t tmp;
-
-	tmp = I915_READ(HSW_PWR_WELL_DRIVER);
-	is_enabled = tmp & HSW_PWR_WELL_STATE_ENABLED;
-	enable_requested = tmp & HSW_PWR_WELL_ENABLE_REQUEST;
-
-	if (enable) {
-		if (!enable_requested)
-			I915_WRITE(HSW_PWR_WELL_DRIVER,
-				   HSW_PWR_WELL_ENABLE_REQUEST);
-
-		if (!is_enabled) {
-			DRM_DEBUG_KMS("Enabling power well\n");
-			if (intel_wait_for_register(dev_priv,
-						    HSW_PWR_WELL_DRIVER,
-						    HSW_PWR_WELL_STATE_ENABLED,
-						    HSW_PWR_WELL_STATE_ENABLED,
-						    20))
-				DRM_ERROR("Timeout enabling power well\n");
-			hsw_power_well_post_enable(dev_priv);
-		}
+	enum i915_power_well_id id = power_well->id;
+	bool wait_fuses = power_well->hsw.has_fuses;
+	enum skl_power_gate pg;
+	u32 val;
 
-	} else {
-		if (enable_requested) {
-			hsw_power_well_pre_disable(dev_priv);
-			I915_WRITE(HSW_PWR_WELL_DRIVER, 0);
-			POSTING_READ(HSW_PWR_WELL_DRIVER);
-			DRM_DEBUG_KMS("Requesting to disable the power well\n");
-		}
+	if (wait_fuses) {
+		pg = SKL_PW_TO_PG(id);
+		/*
+		 * For PW1 we have to wait both for the PW0/PG0 fuse state
+		 * before enabling the power well and PW1/PG1's own fuse
+		 * state after the enabling. For all other power wells with
+		 * fuses we only have to wait for that PW/PG's fuse state
+		 * after the enabling.
+		 */
+		if (pg == SKL_PG1)
+			gen9_wait_for_power_well_fuses(dev_priv, SKL_PG0);
 	}
+
+	val = I915_READ(HSW_PWR_WELL_CTL_DRIVER(id));
+	I915_WRITE(HSW_PWR_WELL_CTL_DRIVER(id), val | HSW_PWR_WELL_CTL_REQ(id));
+	hsw_wait_for_power_well_enable(dev_priv, power_well);
+
+	if (wait_fuses)
+		gen9_wait_for_power_well_fuses(dev_priv, pg);
+
+	hsw_power_well_post_enable(dev_priv, power_well->hsw.irq_pipe_mask,
+				   power_well->hsw.has_vga);
 }
 
-#define SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS (		\
-	BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |		\
-	BIT_ULL(POWER_DOMAIN_PIPE_B) |			\
-	BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |		\
-	BIT_ULL(POWER_DOMAIN_PIPE_C) |			\
-	BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |		\
-	BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |		\
-	BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_E_LANES) |		\
-	BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
-	BIT_ULL(POWER_DOMAIN_AUX_C) |			\
-	BIT_ULL(POWER_DOMAIN_AUX_D) |			\
-	BIT_ULL(POWER_DOMAIN_AUDIO) |			\
-	BIT_ULL(POWER_DOMAIN_VGA) |				\
-	BIT_ULL(POWER_DOMAIN_INIT))
-#define SKL_DISPLAY_DDI_IO_A_E_POWER_DOMAINS (		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO) |		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_E_IO) |		\
-	BIT_ULL(POWER_DOMAIN_INIT))
-#define SKL_DISPLAY_DDI_IO_B_POWER_DOMAINS (		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |		\
-	BIT_ULL(POWER_DOMAIN_INIT))
-#define SKL_DISPLAY_DDI_IO_C_POWER_DOMAINS (		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |		\
-	BIT_ULL(POWER_DOMAIN_INIT))
-#define SKL_DISPLAY_DDI_IO_D_POWER_DOMAINS (		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |		\
-	BIT_ULL(POWER_DOMAIN_INIT))
-#define SKL_DISPLAY_DC_OFF_POWER_DOMAINS (		\
-	SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS |		\
-	BIT_ULL(POWER_DOMAIN_MODESET) |			\
-	BIT_ULL(POWER_DOMAIN_AUX_A) |			\
-	BIT_ULL(POWER_DOMAIN_INIT))
+static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
+				   struct i915_power_well *power_well)
+{
+	enum i915_power_well_id id = power_well->id;
+	u32 val;
 
-#define BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS (		\
-	BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |		\
-	BIT_ULL(POWER_DOMAIN_PIPE_B) |			\
-	BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |		\
-	BIT_ULL(POWER_DOMAIN_PIPE_C) |			\
-	BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |		\
-	BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |		\
-	BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
-	BIT_ULL(POWER_DOMAIN_AUX_B) |			\
-	BIT_ULL(POWER_DOMAIN_AUX_C) |			\
-	BIT_ULL(POWER_DOMAIN_AUDIO) |			\
-	BIT_ULL(POWER_DOMAIN_VGA) |				\
-	BIT_ULL(POWER_DOMAIN_GMBUS) |			\
-	BIT_ULL(POWER_DOMAIN_INIT))
-#define BXT_DISPLAY_DC_OFF_POWER_DOMAINS (		\
-	BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS |		\
-	BIT_ULL(POWER_DOMAIN_MODESET) |			\
-	BIT_ULL(POWER_DOMAIN_AUX_A) |			\
-	BIT_ULL(POWER_DOMAIN_INIT))
-#define BXT_DPIO_CMN_A_POWER_DOMAINS (			\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |		\
-	BIT_ULL(POWER_DOMAIN_AUX_A) |			\
-	BIT_ULL(POWER_DOMAIN_INIT))
-#define BXT_DPIO_CMN_BC_POWER_DOMAINS (			\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
-	BIT_ULL(POWER_DOMAIN_AUX_B) |			\
-	BIT_ULL(POWER_DOMAIN_AUX_C) |			\
-	BIT_ULL(POWER_DOMAIN_INIT))
+	hsw_power_well_pre_disable(dev_priv, power_well->hsw.irq_pipe_mask);
 
-#define GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS (		\
-	BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |		\
-	BIT_ULL(POWER_DOMAIN_PIPE_B) |			\
-	BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |		\
-	BIT_ULL(POWER_DOMAIN_PIPE_C) |			\
-	BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |		\
-	BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |		\
-	BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
-	BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
-	BIT_ULL(POWER_DOMAIN_AUX_C) |			\
-	BIT_ULL(POWER_DOMAIN_AUDIO) |			\
-	BIT_ULL(POWER_DOMAIN_VGA) |				\
-	BIT_ULL(POWER_DOMAIN_INIT))
-#define GLK_DISPLAY_DDI_IO_A_POWER_DOMAINS (		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO))
-#define GLK_DISPLAY_DDI_IO_B_POWER_DOMAINS (		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO))
-#define GLK_DISPLAY_DDI_IO_C_POWER_DOMAINS (		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO))
-#define GLK_DPIO_CMN_A_POWER_DOMAINS (			\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |		\
-	BIT_ULL(POWER_DOMAIN_AUX_A) |			\
-	BIT_ULL(POWER_DOMAIN_INIT))
-#define GLK_DPIO_CMN_B_POWER_DOMAINS (			\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
-	BIT_ULL(POWER_DOMAIN_AUX_B) |			\
-	BIT_ULL(POWER_DOMAIN_INIT))
-#define GLK_DPIO_CMN_C_POWER_DOMAINS (			\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
-	BIT_ULL(POWER_DOMAIN_AUX_C) |			\
-	BIT_ULL(POWER_DOMAIN_INIT))
-#define GLK_DISPLAY_AUX_A_POWER_DOMAINS (		\
-	BIT_ULL(POWER_DOMAIN_AUX_A) |		\
-	BIT_ULL(POWER_DOMAIN_INIT))
-#define GLK_DISPLAY_AUX_B_POWER_DOMAINS (		\
-	BIT_ULL(POWER_DOMAIN_AUX_B) |		\
-	BIT_ULL(POWER_DOMAIN_INIT))
-#define GLK_DISPLAY_AUX_C_POWER_DOMAINS (		\
-	BIT_ULL(POWER_DOMAIN_AUX_C) |		\
-	BIT_ULL(POWER_DOMAIN_INIT))
-#define GLK_DISPLAY_DC_OFF_POWER_DOMAINS (		\
-	GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS |		\
-	BIT_ULL(POWER_DOMAIN_MODESET) |			\
-	BIT_ULL(POWER_DOMAIN_AUX_A) |			\
-	BIT_ULL(POWER_DOMAIN_INIT))
+	val = I915_READ(HSW_PWR_WELL_CTL_DRIVER(id));
+	I915_WRITE(HSW_PWR_WELL_CTL_DRIVER(id),
+		   val & ~HSW_PWR_WELL_CTL_REQ(id));
+	hsw_wait_for_power_well_disable(dev_priv, power_well);
+}
 
-#define CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS (		\
-	BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |		\
-	BIT_ULL(POWER_DOMAIN_PIPE_B) |			\
-	BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |		\
-	BIT_ULL(POWER_DOMAIN_PIPE_C) |			\
-	BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |		\
-	BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |		\
-	BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_E_LANES) |		\
-	BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
-	BIT_ULL(POWER_DOMAIN_AUX_C) |			\
-	BIT_ULL(POWER_DOMAIN_AUX_D) |			\
-	BIT_ULL(POWER_DOMAIN_AUDIO) |			\
-	BIT_ULL(POWER_DOMAIN_VGA) |				\
-	BIT_ULL(POWER_DOMAIN_INIT))
-#define CNL_DISPLAY_DDI_A_IO_POWER_DOMAINS (		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO) |		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_E_IO) |		\
-	BIT_ULL(POWER_DOMAIN_INIT))
-#define CNL_DISPLAY_DDI_B_IO_POWER_DOMAINS (		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |		\
-	BIT_ULL(POWER_DOMAIN_INIT))
-#define CNL_DISPLAY_DDI_C_IO_POWER_DOMAINS (		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |		\
-	BIT_ULL(POWER_DOMAIN_INIT))
-#define CNL_DISPLAY_DDI_D_IO_POWER_DOMAINS (		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |		\
-	BIT_ULL(POWER_DOMAIN_INIT))
-#define CNL_DISPLAY_AUX_A_POWER_DOMAINS (		\
-	BIT_ULL(POWER_DOMAIN_AUX_A) |			\
-	BIT_ULL(POWER_DOMAIN_INIT))
-#define CNL_DISPLAY_AUX_B_POWER_DOMAINS (		\
-	BIT_ULL(POWER_DOMAIN_AUX_B) |			\
-	BIT_ULL(POWER_DOMAIN_INIT))
-#define CNL_DISPLAY_AUX_C_POWER_DOMAINS (		\
-	BIT_ULL(POWER_DOMAIN_AUX_C) |			\
-	BIT_ULL(POWER_DOMAIN_INIT))
-#define CNL_DISPLAY_AUX_D_POWER_DOMAINS (		\
-	BIT_ULL(POWER_DOMAIN_AUX_D) |			\
-	BIT_ULL(POWER_DOMAIN_INIT))
-#define CNL_DISPLAY_DC_OFF_POWER_DOMAINS (		\
-	CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS |		\
-	BIT_ULL(POWER_DOMAIN_MODESET) |			\
-	BIT_ULL(POWER_DOMAIN_AUX_A) |			\
-	BIT_ULL(POWER_DOMAIN_INIT))
+/*
+ * We should only use the power well if we explicitly asked the hardware to
+ * enable it, so check if it's enabled and also check if we've requested it to
+ * be enabled.
+ */
+static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv,
+				   struct i915_power_well *power_well)
+{
+	enum i915_power_well_id id = power_well->id;
+	u32 mask = HSW_PWR_WELL_CTL_REQ(id) | HSW_PWR_WELL_CTL_STATE(id);
+
+	return (I915_READ(HSW_PWR_WELL_CTL_DRIVER(id)) & mask) == mask;
+}
 
 static void assert_can_enable_dc9(struct drm_i915_private *dev_priv)
 {
+	enum i915_power_well_id id = SKL_DISP_PW_2;
+
 	WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_DC9),
 		  "DC9 already programmed to be enabled.\n");
 	WARN_ONCE(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5,
 		  "DC5 still not disabled to enable DC9.\n");
-	WARN_ONCE(I915_READ(HSW_PWR_WELL_DRIVER), "Power well on.\n");
+	WARN_ONCE(I915_READ(HSW_PWR_WELL_CTL_DRIVER(id)) &
+		  HSW_PWR_WELL_CTL_REQ(id),
+		  "Power well 2 on.\n");
 	WARN_ONCE(intel_irqs_enabled(dev_priv),
 		  "Interrupts not disabled yet.\n");
 
@@ -744,223 +628,39 @@ void skl_disable_dc6(struct drm_i915_private *dev_priv)
 	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 }
 
-static void
-gen9_sanitize_power_well_requests(struct drm_i915_private *dev_priv,
-				  struct i915_power_well *power_well)
-{
-	enum skl_disp_power_wells power_well_id = power_well->id;
-	u32 val;
-	u32 mask;
-
-	mask = SKL_POWER_WELL_REQ(power_well_id);
-
-	val = I915_READ(HSW_PWR_WELL_KVMR);
-	if (WARN_ONCE(val & mask, "Clearing unexpected KVMR request for %s\n",
-		      power_well->name))
-		I915_WRITE(HSW_PWR_WELL_KVMR, val & ~mask);
-
-	val = I915_READ(HSW_PWR_WELL_BIOS);
-	val |= I915_READ(HSW_PWR_WELL_DEBUG);
-
-	if (!(val & mask))
-		return;
-
-	/*
-	 * DMC is known to force on the request bits for power well 1 on SKL
-	 * and BXT and the misc IO power well on SKL but we don't expect any
-	 * other request bits to be set, so WARN for those.
-	 */
-	if (power_well_id == SKL_DISP_PW_1 ||
-	    (IS_GEN9_BC(dev_priv) &&
-	     power_well_id == SKL_DISP_PW_MISC_IO))
-		DRM_DEBUG_DRIVER("Clearing auxiliary requests for %s forced on "
-				 "by DMC\n", power_well->name);
-	else
-		WARN_ONCE(1, "Clearing unexpected auxiliary requests for %s\n",
-			  power_well->name);
-
-	I915_WRITE(HSW_PWR_WELL_BIOS, val & ~mask);
-	I915_WRITE(HSW_PWR_WELL_DEBUG, val & ~mask);
-}
-
-static void skl_set_power_well(struct drm_i915_private *dev_priv,
-			       struct i915_power_well *power_well, bool enable)
-{
-	uint32_t tmp, fuse_status;
-	uint32_t req_mask, state_mask;
-	bool is_enabled, enable_requested, check_fuse_status = false;
-
-	tmp = I915_READ(HSW_PWR_WELL_DRIVER);
-	fuse_status = I915_READ(SKL_FUSE_STATUS);
-
-	switch (power_well->id) {
-	case SKL_DISP_PW_1:
-		if (intel_wait_for_register(dev_priv,
-					    SKL_FUSE_STATUS,
-					    SKL_FUSE_PG0_DIST_STATUS,
-					    SKL_FUSE_PG0_DIST_STATUS,
-					    1)) {
-			DRM_ERROR("PG0 not enabled\n");
-			return;
-		}
-		break;
-	case SKL_DISP_PW_2:
-		if (!(fuse_status & SKL_FUSE_PG1_DIST_STATUS)) {
-			DRM_ERROR("PG1 in disabled state\n");
-			return;
-		}
-		break;
-	case SKL_DISP_PW_MISC_IO:
-	case SKL_DISP_PW_DDI_A_E: /* GLK_DISP_PW_DDI_A, CNL_DISP_PW_DDI_A */
-	case SKL_DISP_PW_DDI_B:
-	case SKL_DISP_PW_DDI_C:
-	case SKL_DISP_PW_DDI_D:
-	case GLK_DISP_PW_AUX_A: /* CNL_DISP_PW_AUX_A */
-	case GLK_DISP_PW_AUX_B: /* CNL_DISP_PW_AUX_B */
-	case GLK_DISP_PW_AUX_C: /* CNL_DISP_PW_AUX_C */
-	case CNL_DISP_PW_AUX_D:
-		break;
-	default:
-		WARN(1, "Unknown power well %lu\n", power_well->id);
-		return;
-	}
-
-	req_mask = SKL_POWER_WELL_REQ(power_well->id);
-	enable_requested = tmp & req_mask;
-	state_mask = SKL_POWER_WELL_STATE(power_well->id);
-	is_enabled = tmp & state_mask;
-
-	if (!enable && enable_requested)
-		skl_power_well_pre_disable(dev_priv, power_well);
-
-	if (enable) {
-		if (!enable_requested) {
-			WARN((tmp & state_mask) &&
-				!I915_READ(HSW_PWR_WELL_BIOS),
-				"Invalid for power well status to be enabled, unless done by the BIOS, \
-				when request is to disable!\n");
-			I915_WRITE(HSW_PWR_WELL_DRIVER, tmp | req_mask);
-		}
-
-		if (!is_enabled) {
-			DRM_DEBUG_KMS("Enabling %s\n", power_well->name);
-			check_fuse_status = true;
-		}
-	} else {
-		if (enable_requested) {
-			I915_WRITE(HSW_PWR_WELL_DRIVER,	tmp & ~req_mask);
-			POSTING_READ(HSW_PWR_WELL_DRIVER);
-			DRM_DEBUG_KMS("Disabling %s\n", power_well->name);
-		}
-
-		gen9_sanitize_power_well_requests(dev_priv, power_well);
-	}
-
-	if (wait_for(!!(I915_READ(HSW_PWR_WELL_DRIVER) & state_mask) == enable,
-		     1))
-		DRM_ERROR("%s %s timeout\n",
-			  power_well->name, enable ? "enable" : "disable");
-
-	if (check_fuse_status) {
-		if (power_well->id == SKL_DISP_PW_1) {
-			if (intel_wait_for_register(dev_priv,
-						    SKL_FUSE_STATUS,
-						    SKL_FUSE_PG1_DIST_STATUS,
-						    SKL_FUSE_PG1_DIST_STATUS,
-						    1))
-				DRM_ERROR("PG1 distributing status timeout\n");
-		} else if (power_well->id == SKL_DISP_PW_2) {
-			if (intel_wait_for_register(dev_priv,
-						    SKL_FUSE_STATUS,
-						    SKL_FUSE_PG2_DIST_STATUS,
-						    SKL_FUSE_PG2_DIST_STATUS,
-						    1))
-				DRM_ERROR("PG2 distributing status timeout\n");
-		}
-	}
-
-	if (enable && !is_enabled)
-		skl_power_well_post_enable(dev_priv, power_well);
-}
-
 static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv,
 				   struct i915_power_well *power_well)
 {
-	/* Take over the request bit if set by BIOS. */
-	if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE_REQUEST) {
-		if (!(I915_READ(HSW_PWR_WELL_DRIVER) &
-		      HSW_PWR_WELL_ENABLE_REQUEST))
-			I915_WRITE(HSW_PWR_WELL_DRIVER,
-				   HSW_PWR_WELL_ENABLE_REQUEST);
-		I915_WRITE(HSW_PWR_WELL_BIOS, 0);
-	}
-}
-
-static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
-				  struct i915_power_well *power_well)
-{
-	hsw_set_power_well(dev_priv, power_well, true);
-}
-
-static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
-				   struct i915_power_well *power_well)
-{
-	hsw_set_power_well(dev_priv, power_well, false);
-}
-
-static bool skl_power_well_enabled(struct drm_i915_private *dev_priv,
-					struct i915_power_well *power_well)
-{
-	uint32_t mask = SKL_POWER_WELL_REQ(power_well->id) |
-		SKL_POWER_WELL_STATE(power_well->id);
-
-	return (I915_READ(HSW_PWR_WELL_DRIVER) & mask) == mask;
-}
-
-static void skl_power_well_sync_hw(struct drm_i915_private *dev_priv,
-				struct i915_power_well *power_well)
-{
-	uint32_t mask = SKL_POWER_WELL_REQ(power_well->id);
-	uint32_t bios_req = I915_READ(HSW_PWR_WELL_BIOS);
+	enum i915_power_well_id id = power_well->id;
+	u32 mask = HSW_PWR_WELL_CTL_REQ(id);
+	u32 bios_req = I915_READ(HSW_PWR_WELL_CTL_BIOS(id));
 
 	/* Take over the request bit if set by BIOS. */
 	if (bios_req & mask) {
-		uint32_t drv_req = I915_READ(HSW_PWR_WELL_DRIVER);
+		u32 drv_req = I915_READ(HSW_PWR_WELL_CTL_DRIVER(id));
 
 		if (!(drv_req & mask))
-			I915_WRITE(HSW_PWR_WELL_DRIVER, drv_req | mask);
-		I915_WRITE(HSW_PWR_WELL_BIOS, bios_req & ~mask);
+			I915_WRITE(HSW_PWR_WELL_CTL_DRIVER(id), drv_req | mask);
+		I915_WRITE(HSW_PWR_WELL_CTL_BIOS(id), bios_req & ~mask);
 	}
 }
 
-static void skl_power_well_enable(struct drm_i915_private *dev_priv,
-				struct i915_power_well *power_well)
-{
-	skl_set_power_well(dev_priv, power_well, true);
-}
-
-static void skl_power_well_disable(struct drm_i915_private *dev_priv,
-				struct i915_power_well *power_well)
-{
-	skl_set_power_well(dev_priv, power_well, false);
-}
-
 static void bxt_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 					   struct i915_power_well *power_well)
 {
-	bxt_ddi_phy_init(dev_priv, power_well->data);
+	bxt_ddi_phy_init(dev_priv, power_well->bxt.phy);
 }
 
 static void bxt_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
 					    struct i915_power_well *power_well)
 {
-	bxt_ddi_phy_uninit(dev_priv, power_well->data);
+	bxt_ddi_phy_uninit(dev_priv, power_well->bxt.phy);
 }
 
 static bool bxt_dpio_cmn_power_well_enabled(struct drm_i915_private *dev_priv,
 					    struct i915_power_well *power_well)
 {
-	return bxt_ddi_phy_is_enabled(dev_priv, power_well->data);
+	return bxt_ddi_phy_is_enabled(dev_priv, power_well->bxt.phy);
 }
 
 static void bxt_verify_ddi_phy_power_wells(struct drm_i915_private *dev_priv)
@@ -969,16 +669,16 @@ static void bxt_verify_ddi_phy_power_wells(struct drm_i915_private *dev_priv)
 
 	power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_A);
 	if (power_well->count > 0)
-		bxt_ddi_phy_verify_state(dev_priv, power_well->data);
+		bxt_ddi_phy_verify_state(dev_priv, power_well->bxt.phy);
 
 	power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_BC);
 	if (power_well->count > 0)
-		bxt_ddi_phy_verify_state(dev_priv, power_well->data);
+		bxt_ddi_phy_verify_state(dev_priv, power_well->bxt.phy);
 
 	if (IS_GEMINILAKE(dev_priv)) {
 		power_well = lookup_power_well(dev_priv, GLK_DPIO_CMN_C);
 		if (power_well->count > 0)
-			bxt_ddi_phy_verify_state(dev_priv, power_well->data);
+			bxt_ddi_phy_verify_state(dev_priv, power_well->bxt.phy);
 	}
 }
 
@@ -1076,7 +776,7 @@ static void i830_pipes_power_well_sync_hw(struct drm_i915_private *dev_priv,
 static void vlv_set_power_well(struct drm_i915_private *dev_priv,
 			       struct i915_power_well *power_well, bool enable)
 {
-	enum punit_power_well power_well_id = power_well->id;
+	enum i915_power_well_id power_well_id = power_well->id;
 	u32 mask;
 	u32 state;
 	u32 ctrl;
@@ -1124,7 +824,7 @@ static void vlv_power_well_disable(struct drm_i915_private *dev_priv,
 static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv,
 				   struct i915_power_well *power_well)
 {
-	int power_well_id = power_well->id;
+	enum i915_power_well_id power_well_id = power_well->id;
 	bool enabled = false;
 	u32 mask;
 	u32 state;
@@ -1311,8 +1011,9 @@ static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
 
 #define POWER_DOMAIN_MASK (GENMASK_ULL(POWER_DOMAIN_NUM - 1, 0))
 
-static struct i915_power_well *lookup_power_well(struct drm_i915_private *dev_priv,
-						 int power_well_id)
+static struct i915_power_well *
+lookup_power_well(struct drm_i915_private *dev_priv,
+		  enum i915_power_well_id power_well_id)
 {
 	struct i915_power_domains *power_domains = &dev_priv->power_domains;
 	int i;
@@ -1659,7 +1360,7 @@ void chv_phy_powergate_lanes(struct intel_encoder *encoder,
 static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv,
 					struct i915_power_well *power_well)
 {
-	enum pipe pipe = power_well->id;
+	enum pipe pipe = PIPE_A;
 	bool enabled;
 	u32 state, ctrl;
 
@@ -1689,7 +1390,7 @@ static void chv_set_pipe_power_well(struct drm_i915_private *dev_priv,
 				    struct i915_power_well *power_well,
 				    bool enable)
 {
-	enum pipe pipe = power_well->id;
+	enum pipe pipe = PIPE_A;
 	u32 state;
 	u32 ctrl;
 
@@ -1722,7 +1423,7 @@ out:
 static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv,
 				       struct i915_power_well *power_well)
 {
-	WARN_ON_ONCE(power_well->id != PIPE_A);
+	WARN_ON_ONCE(power_well->id != CHV_DISP_PW_PIPE_A);
 
 	chv_set_pipe_power_well(dev_priv, power_well, true);
 
@@ -1732,7 +1433,7 @@ static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv,
 static void chv_pipe_power_well_disable(struct drm_i915_private *dev_priv,
 					struct i915_power_well *power_well)
 {
-	WARN_ON_ONCE(power_well->id != PIPE_A);
+	WARN_ON_ONCE(power_well->id != CHV_DISP_PW_PIPE_A);
 
 	vlv_display_power_well_deinit(dev_priv);
 
@@ -1848,37 +1549,13 @@ void intel_display_power_put(struct drm_i915_private *dev_priv,
 	intel_runtime_pm_put(dev_priv);
 }
 
-#define HSW_DISPLAY_POWER_DOMAINS (			\
-	BIT_ULL(POWER_DOMAIN_PIPE_B) |			\
-	BIT_ULL(POWER_DOMAIN_PIPE_C) |			\
-	BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |		\
-	BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |		\
-	BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |		\
-	BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |		\
-	BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |		\
-	BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |		\
-	BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */	\
-	BIT_ULL(POWER_DOMAIN_VGA) |				\
-	BIT_ULL(POWER_DOMAIN_AUDIO) |			\
-	BIT_ULL(POWER_DOMAIN_INIT))
-
-#define BDW_DISPLAY_POWER_DOMAINS (			\
-	BIT_ULL(POWER_DOMAIN_PIPE_B) |			\
-	BIT_ULL(POWER_DOMAIN_PIPE_C) |			\
-	BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |		\
-	BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |		\
-	BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |		\
-	BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |		\
-	BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
-	BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |		\
-	BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */	\
-	BIT_ULL(POWER_DOMAIN_VGA) |				\
-	BIT_ULL(POWER_DOMAIN_AUDIO) |			\
+#define I830_PIPES_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_PIPE_A) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_B) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |	\
+	BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |	\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |	\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |	\
 	BIT_ULL(POWER_DOMAIN_INIT))
 
 #define VLV_DISPLAY_POWER_DOMAINS (		\
@@ -1961,13 +1638,201 @@ void intel_display_power_put(struct drm_i915_private *dev_priv,
 	BIT_ULL(POWER_DOMAIN_AUX_D) |		\
 	BIT_ULL(POWER_DOMAIN_INIT))
 
-#define I830_PIPES_POWER_DOMAINS (		\
-	BIT_ULL(POWER_DOMAIN_PIPE_A) |		\
-	BIT_ULL(POWER_DOMAIN_PIPE_B) |		\
-	BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |	\
-	BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |	\
-	BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |	\
-	BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |	\
+#define HSW_DISPLAY_POWER_DOMAINS (			\
+	BIT_ULL(POWER_DOMAIN_PIPE_B) |			\
+	BIT_ULL(POWER_DOMAIN_PIPE_C) |			\
+	BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |		\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |		\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |		\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */	\
+	BIT_ULL(POWER_DOMAIN_VGA) |				\
+	BIT_ULL(POWER_DOMAIN_AUDIO) |			\
+	BIT_ULL(POWER_DOMAIN_INIT))
+
+#define BDW_DISPLAY_POWER_DOMAINS (			\
+	BIT_ULL(POWER_DOMAIN_PIPE_B) |			\
+	BIT_ULL(POWER_DOMAIN_PIPE_C) |			\
+	BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |		\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |		\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |		\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */	\
+	BIT_ULL(POWER_DOMAIN_VGA) |				\
+	BIT_ULL(POWER_DOMAIN_AUDIO) |			\
+	BIT_ULL(POWER_DOMAIN_INIT))
+
+#define SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_B) |			\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_C) |			\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_E_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
+	BIT_ULL(POWER_DOMAIN_AUX_C) |			\
+	BIT_ULL(POWER_DOMAIN_AUX_D) |			\
+	BIT_ULL(POWER_DOMAIN_AUDIO) |			\
+	BIT_ULL(POWER_DOMAIN_VGA) |				\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define SKL_DISPLAY_DDI_IO_A_E_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_E_IO) |		\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define SKL_DISPLAY_DDI_IO_B_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |		\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define SKL_DISPLAY_DDI_IO_C_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |		\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define SKL_DISPLAY_DDI_IO_D_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |		\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define SKL_DISPLAY_DC_OFF_POWER_DOMAINS (		\
+	SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS |		\
+	BIT_ULL(POWER_DOMAIN_MODESET) |			\
+	BIT_ULL(POWER_DOMAIN_AUX_A) |			\
+	BIT_ULL(POWER_DOMAIN_INIT))
+
+#define BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_B) |			\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_C) |			\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_AUX_B) |			\
+	BIT_ULL(POWER_DOMAIN_AUX_C) |			\
+	BIT_ULL(POWER_DOMAIN_AUDIO) |			\
+	BIT_ULL(POWER_DOMAIN_VGA) |				\
+	BIT_ULL(POWER_DOMAIN_GMBUS) |			\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define BXT_DISPLAY_DC_OFF_POWER_DOMAINS (		\
+	BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS |		\
+	BIT_ULL(POWER_DOMAIN_MODESET) |			\
+	BIT_ULL(POWER_DOMAIN_AUX_A) |			\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define BXT_DPIO_CMN_A_POWER_DOMAINS (			\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_AUX_A) |			\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define BXT_DPIO_CMN_BC_POWER_DOMAINS (			\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_AUX_B) |			\
+	BIT_ULL(POWER_DOMAIN_AUX_C) |			\
+	BIT_ULL(POWER_DOMAIN_INIT))
+
+#define GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_B) |			\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_C) |			\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
+	BIT_ULL(POWER_DOMAIN_AUX_C) |			\
+	BIT_ULL(POWER_DOMAIN_AUDIO) |			\
+	BIT_ULL(POWER_DOMAIN_VGA) |				\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define GLK_DISPLAY_DDI_IO_A_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO))
+#define GLK_DISPLAY_DDI_IO_B_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO))
+#define GLK_DISPLAY_DDI_IO_C_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO))
+#define GLK_DPIO_CMN_A_POWER_DOMAINS (			\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_AUX_A) |			\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define GLK_DPIO_CMN_B_POWER_DOMAINS (			\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_AUX_B) |			\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define GLK_DPIO_CMN_C_POWER_DOMAINS (			\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_AUX_C) |			\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define GLK_DISPLAY_AUX_A_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_AUX_A) |		\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define GLK_DISPLAY_AUX_B_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_AUX_B) |		\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define GLK_DISPLAY_AUX_C_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_AUX_C) |		\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define GLK_DISPLAY_DC_OFF_POWER_DOMAINS (		\
+	GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS |		\
+	BIT_ULL(POWER_DOMAIN_MODESET) |			\
+	BIT_ULL(POWER_DOMAIN_AUX_A) |			\
+	BIT_ULL(POWER_DOMAIN_INIT))
+
+#define CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_B) |			\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_C) |			\
+	BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |		\
+	BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |		\
+	BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
+	BIT_ULL(POWER_DOMAIN_AUX_C) |			\
+	BIT_ULL(POWER_DOMAIN_AUX_D) |			\
+	BIT_ULL(POWER_DOMAIN_AUDIO) |			\
+	BIT_ULL(POWER_DOMAIN_VGA) |				\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define CNL_DISPLAY_DDI_A_IO_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO) |		\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define CNL_DISPLAY_DDI_B_IO_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |		\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define CNL_DISPLAY_DDI_C_IO_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |		\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define CNL_DISPLAY_DDI_D_IO_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |		\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define CNL_DISPLAY_AUX_A_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_AUX_A) |			\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define CNL_DISPLAY_AUX_B_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_AUX_B) |			\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define CNL_DISPLAY_AUX_C_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_AUX_C) |			\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define CNL_DISPLAY_AUX_D_POWER_DOMAINS (		\
+	BIT_ULL(POWER_DOMAIN_AUX_D) |			\
+	BIT_ULL(POWER_DOMAIN_INIT))
+#define CNL_DISPLAY_DC_OFF_POWER_DOMAINS (		\
+	CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS |		\
+	BIT_ULL(POWER_DOMAIN_MODESET) |			\
+	BIT_ULL(POWER_DOMAIN_AUX_A) |			\
 	BIT_ULL(POWER_DOMAIN_INIT))
 
 static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
@@ -1997,6 +1862,7 @@ static struct i915_power_well i9xx_always_on_power_well[] = {
 		.always_on = 1,
 		.domains = POWER_DOMAIN_MASK,
 		.ops = &i9xx_always_on_power_well_ops,
+		.id = I915_DISP_PW_ALWAYS_ON,
 	},
 };
 
@@ -2013,11 +1879,13 @@ static struct i915_power_well i830_power_wells[] = {
 		.always_on = 1,
 		.domains = POWER_DOMAIN_MASK,
 		.ops = &i9xx_always_on_power_well_ops,
+		.id = I915_DISP_PW_ALWAYS_ON,
 	},
 	{
 		.name = "pipes",
 		.domains = I830_PIPES_POWER_DOMAINS,
 		.ops = &i830_pipes_power_well_ops,
+		.id = I830_DISP_PW_PIPES,
 	},
 };
 
@@ -2028,13 +1896,6 @@ static const struct i915_power_well_ops hsw_power_well_ops = {
 	.is_enabled = hsw_power_well_enabled,
 };
 
-static const struct i915_power_well_ops skl_power_well_ops = {
-	.sync_hw = skl_power_well_sync_hw,
-	.enable = skl_power_well_enable,
-	.disable = skl_power_well_disable,
-	.is_enabled = skl_power_well_enabled,
-};
-
 static const struct i915_power_well_ops gen9_dc_off_power_well_ops = {
 	.sync_hw = i9xx_power_well_sync_hw_noop,
 	.enable = gen9_dc_off_power_well_enable,
@@ -2055,11 +1916,16 @@ static struct i915_power_well hsw_power_wells[] = {
 		.always_on = 1,
 		.domains = POWER_DOMAIN_MASK,
 		.ops = &i9xx_always_on_power_well_ops,
+		.id = I915_DISP_PW_ALWAYS_ON,
 	},
 	{
 		.name = "display",
 		.domains = HSW_DISPLAY_POWER_DOMAINS,
 		.ops = &hsw_power_well_ops,
+		.id = HSW_DISP_PW_GLOBAL,
+		{
+			.hsw.has_vga = true,
+		},
 	},
 };
 
@@ -2069,11 +1935,17 @@ static struct i915_power_well bdw_power_wells[] = {
 		.always_on = 1,
 		.domains = POWER_DOMAIN_MASK,
 		.ops = &i9xx_always_on_power_well_ops,
+		.id = I915_DISP_PW_ALWAYS_ON,
 	},
 	{
 		.name = "display",
 		.domains = BDW_DISPLAY_POWER_DOMAINS,
 		.ops = &hsw_power_well_ops,
+		.id = HSW_DISP_PW_GLOBAL,
+		{
+			.hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
+			.hsw.has_vga = true,
+		},
 	},
 };
 
@@ -2104,7 +1976,7 @@ static struct i915_power_well vlv_power_wells[] = {
 		.always_on = 1,
 		.domains = POWER_DOMAIN_MASK,
 		.ops = &i9xx_always_on_power_well_ops,
-		.id = PUNIT_POWER_WELL_ALWAYS_ON,
+		.id = I915_DISP_PW_ALWAYS_ON,
 	},
 	{
 		.name = "display",
@@ -2162,6 +2034,7 @@ static struct i915_power_well chv_power_wells[] = {
 		.always_on = 1,
 		.domains = POWER_DOMAIN_MASK,
 		.ops = &i9xx_always_on_power_well_ops,
+		.id = I915_DISP_PW_ALWAYS_ON,
 	},
 	{
 		.name = "display",
@@ -2171,7 +2044,7 @@ static struct i915_power_well chv_power_wells[] = {
 		 * required for any pipe to work.
 		 */
 		.domains = CHV_DISPLAY_POWER_DOMAINS,
-		.id = PIPE_A,
+		.id = CHV_DISP_PW_PIPE_A,
 		.ops = &chv_pipe_power_well_ops,
 	},
 	{
@@ -2189,7 +2062,7 @@ static struct i915_power_well chv_power_wells[] = {
 };
 
 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
-				    int power_well_id)
+					 enum i915_power_well_id power_well_id)
 {
 	struct i915_power_well *power_well;
 	bool ret;
@@ -2206,20 +2079,23 @@ static struct i915_power_well skl_power_wells[] = {
 		.always_on = 1,
 		.domains = POWER_DOMAIN_MASK,
 		.ops = &i9xx_always_on_power_well_ops,
-		.id = SKL_DISP_PW_ALWAYS_ON,
+		.id = I915_DISP_PW_ALWAYS_ON,
 	},
 	{
 		.name = "power well 1",
 		/* Handled by the DMC firmware */
 		.domains = 0,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = SKL_DISP_PW_1,
+		{
+			.hsw.has_fuses = true,
+		},
 	},
 	{
 		.name = "MISC IO power well",
 		/* Handled by the DMC firmware */
 		.domains = 0,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = SKL_DISP_PW_MISC_IO,
 	},
 	{
@@ -2231,31 +2107,36 @@ static struct i915_power_well skl_power_wells[] = {
 	{
 		.name = "power well 2",
 		.domains = SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = SKL_DISP_PW_2,
+		{
+			.hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
+			.hsw.has_vga = true,
+			.hsw.has_fuses = true,
+		},
 	},
 	{
 		.name = "DDI A/E IO power well",
 		.domains = SKL_DISPLAY_DDI_IO_A_E_POWER_DOMAINS,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = SKL_DISP_PW_DDI_A_E,
 	},
 	{
 		.name = "DDI B IO power well",
 		.domains = SKL_DISPLAY_DDI_IO_B_POWER_DOMAINS,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = SKL_DISP_PW_DDI_B,
 	},
 	{
 		.name = "DDI C IO power well",
 		.domains = SKL_DISPLAY_DDI_IO_C_POWER_DOMAINS,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = SKL_DISP_PW_DDI_C,
 	},
 	{
 		.name = "DDI D IO power well",
 		.domains = SKL_DISPLAY_DDI_IO_D_POWER_DOMAINS,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = SKL_DISP_PW_DDI_D,
 	},
 };
@@ -2266,12 +2147,16 @@ static struct i915_power_well bxt_power_wells[] = {
 		.always_on = 1,
 		.domains = POWER_DOMAIN_MASK,
 		.ops = &i9xx_always_on_power_well_ops,
+		.id = I915_DISP_PW_ALWAYS_ON,
 	},
 	{
 		.name = "power well 1",
 		.domains = 0,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = SKL_DISP_PW_1,
+		{
+			.hsw.has_fuses = true,
+		},
 	},
 	{
 		.name = "DC off",
@@ -2282,22 +2167,31 @@ static struct i915_power_well bxt_power_wells[] = {
 	{
 		.name = "power well 2",
 		.domains = BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = SKL_DISP_PW_2,
+		{
+			.hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
+			.hsw.has_vga = true,
+			.hsw.has_fuses = true,
+		},
 	},
 	{
 		.name = "dpio-common-a",
 		.domains = BXT_DPIO_CMN_A_POWER_DOMAINS,
 		.ops = &bxt_dpio_cmn_power_well_ops,
 		.id = BXT_DPIO_CMN_A,
-		.data = DPIO_PHY1,
+		{
+			.bxt.phy = DPIO_PHY1,
+		},
 	},
 	{
 		.name = "dpio-common-bc",
 		.domains = BXT_DPIO_CMN_BC_POWER_DOMAINS,
 		.ops = &bxt_dpio_cmn_power_well_ops,
 		.id = BXT_DPIO_CMN_BC,
-		.data = DPIO_PHY0,
+		{
+			.bxt.phy = DPIO_PHY0,
+		},
 	},
 };
 
@@ -2307,13 +2201,17 @@ static struct i915_power_well glk_power_wells[] = {
 		.always_on = 1,
 		.domains = POWER_DOMAIN_MASK,
 		.ops = &i9xx_always_on_power_well_ops,
+		.id = I915_DISP_PW_ALWAYS_ON,
 	},
 	{
 		.name = "power well 1",
 		/* Handled by the DMC firmware */
 		.domains = 0,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = SKL_DISP_PW_1,
+		{
+			.hsw.has_fuses = true,
+		},
 	},
 	{
 		.name = "DC off",
@@ -2324,64 +2222,75 @@ static struct i915_power_well glk_power_wells[] = {
 	{
 		.name = "power well 2",
 		.domains = GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = SKL_DISP_PW_2,
+		{
+			.hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
+			.hsw.has_vga = true,
+			.hsw.has_fuses = true,
+		},
 	},
 	{
 		.name = "dpio-common-a",
 		.domains = GLK_DPIO_CMN_A_POWER_DOMAINS,
 		.ops = &bxt_dpio_cmn_power_well_ops,
 		.id = BXT_DPIO_CMN_A,
-		.data = DPIO_PHY1,
+		{
+			.bxt.phy = DPIO_PHY1,
+		},
 	},
 	{
 		.name = "dpio-common-b",
 		.domains = GLK_DPIO_CMN_B_POWER_DOMAINS,
 		.ops = &bxt_dpio_cmn_power_well_ops,
 		.id = BXT_DPIO_CMN_BC,
-		.data = DPIO_PHY0,
+		{
+			.bxt.phy = DPIO_PHY0,
+		},
 	},
 	{
 		.name = "dpio-common-c",
 		.domains = GLK_DPIO_CMN_C_POWER_DOMAINS,
 		.ops = &bxt_dpio_cmn_power_well_ops,
 		.id = GLK_DPIO_CMN_C,
-		.data = DPIO_PHY2,
+		{
+			.bxt.phy = DPIO_PHY2,
+		},
 	},
 	{
 		.name = "AUX A",
 		.domains = GLK_DISPLAY_AUX_A_POWER_DOMAINS,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = GLK_DISP_PW_AUX_A,
 	},
 	{
 		.name = "AUX B",
 		.domains = GLK_DISPLAY_AUX_B_POWER_DOMAINS,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = GLK_DISP_PW_AUX_B,
 	},
 	{
 		.name = "AUX C",
 		.domains = GLK_DISPLAY_AUX_C_POWER_DOMAINS,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = GLK_DISP_PW_AUX_C,
 	},
 	{
 		.name = "DDI A IO power well",
 		.domains = GLK_DISPLAY_DDI_IO_A_POWER_DOMAINS,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = GLK_DISP_PW_DDI_A,
 	},
 	{
 		.name = "DDI B IO power well",
 		.domains = GLK_DISPLAY_DDI_IO_B_POWER_DOMAINS,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = SKL_DISP_PW_DDI_B,
 	},
 	{
 		.name = "DDI C IO power well",
 		.domains = GLK_DISPLAY_DDI_IO_C_POWER_DOMAINS,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = SKL_DISP_PW_DDI_C,
 	},
 };
@@ -2392,36 +2301,40 @@ static struct i915_power_well cnl_power_wells[] = {
 		.always_on = 1,
 		.domains = POWER_DOMAIN_MASK,
 		.ops = &i9xx_always_on_power_well_ops,
+		.id = I915_DISP_PW_ALWAYS_ON,
 	},
 	{
 		.name = "power well 1",
 		/* Handled by the DMC firmware */
 		.domains = 0,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = SKL_DISP_PW_1,
+		{
+			.hsw.has_fuses = true,
+		},
 	},
 	{
 		.name = "AUX A",
 		.domains = CNL_DISPLAY_AUX_A_POWER_DOMAINS,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = CNL_DISP_PW_AUX_A,
 	},
 	{
 		.name = "AUX B",
 		.domains = CNL_DISPLAY_AUX_B_POWER_DOMAINS,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = CNL_DISP_PW_AUX_B,
 	},
 	{
 		.name = "AUX C",
 		.domains = CNL_DISPLAY_AUX_C_POWER_DOMAINS,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = CNL_DISP_PW_AUX_C,
 	},
 	{
 		.name = "AUX D",
 		.domains = CNL_DISPLAY_AUX_D_POWER_DOMAINS,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = CNL_DISP_PW_AUX_D,
 	},
 	{
@@ -2433,31 +2346,36 @@ static struct i915_power_well cnl_power_wells[] = {
 	{
 		.name = "power well 2",
 		.domains = CNL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = SKL_DISP_PW_2,
+		{
+			.hsw.irq_pipe_mask = BIT(PIPE_B) | BIT(PIPE_C),
+			.hsw.has_vga = true,
+			.hsw.has_fuses = true,
+		},
 	},
 	{
 		.name = "DDI A IO power well",
 		.domains = CNL_DISPLAY_DDI_A_IO_POWER_DOMAINS,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = CNL_DISP_PW_DDI_A,
 	},
 	{
 		.name = "DDI B IO power well",
 		.domains = CNL_DISPLAY_DDI_B_IO_POWER_DOMAINS,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = SKL_DISP_PW_DDI_B,
 	},
 	{
 		.name = "DDI C IO power well",
 		.domains = CNL_DISPLAY_DDI_C_IO_POWER_DOMAINS,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = SKL_DISP_PW_DDI_C,
 	},
 	{
 		.name = "DDI D IO power well",
 		.domains = CNL_DISPLAY_DDI_D_IO_POWER_DOMAINS,
-		.ops = &skl_power_well_ops,
+		.ops = &hsw_power_well_ops,
 		.id = SKL_DISP_PW_DDI_D,
 	},
 };
@@ -2479,7 +2397,7 @@ static uint32_t get_allowed_dc_mask(const struct drm_i915_private *dev_priv,
 	int requested_dc;
 	int max_dc;
 
-	if (IS_GEN9_BC(dev_priv)) {
+	if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) {
 		max_dc = 2;
 		mask = 0;
 	} else if (IS_GEN9_LP(dev_priv)) {
@@ -2521,6 +2439,22 @@ static uint32_t get_allowed_dc_mask(const struct drm_i915_private *dev_priv,
 	return mask;
 }
 
+static void assert_power_well_ids_unique(struct drm_i915_private *dev_priv)
+{
+	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+	u64 power_well_ids;
+	int i;
+
+	power_well_ids = 0;
+	for (i = 0; i < power_domains->power_well_count; i++) {
+		enum i915_power_well_id id = power_domains->power_wells[i].id;
+
+		WARN_ON(id >= sizeof(power_well_ids) * 8);
+		WARN_ON(power_well_ids & BIT_ULL(id));
+		power_well_ids |= BIT_ULL(id);
+	}
+}
+
 #define set_power_wells(power_domains, __power_wells) ({		\
 	(power_domains)->power_wells = (__power_wells);			\
 	(power_domains)->power_well_count = ARRAY_SIZE(__power_wells);	\
@@ -2572,6 +2506,8 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv)
 		set_power_wells(power_domains, i9xx_always_on_power_well);
 	}
 
+	assert_power_well_ids_unique(dev_priv);
+
 	return 0;
 }
 
@@ -2694,13 +2630,18 @@ static void skl_display_core_uninit(struct drm_i915_private *dev_priv)
 
 	mutex_lock(&power_domains->lock);
 
-	well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO);
-	intel_power_well_disable(dev_priv, well);
-
+	/*
+	 * BSpec says to keep the MISC IO power well enabled here, only
+	 * remove our request for power well 1.
+	 * Note that even though the driver's request is removed power well 1
+	 * may stay enabled after this due to DMC's own request on it.
+	 */
 	well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
 	intel_power_well_disable(dev_priv, well);
 
 	mutex_unlock(&power_domains->lock);
+
+	usleep_range(10, 30);		/* 10 us delay per Bspec */
 }
 
 void bxt_display_core_init(struct drm_i915_private *dev_priv,
@@ -2751,13 +2692,19 @@ void bxt_display_core_uninit(struct drm_i915_private *dev_priv)
 
 	/* The spec doesn't call for removing the reset handshake flag */
 
-	/* Disable PG1 */
+	/*
+	 * Disable PW1 (PG1).
+	 * Note that even though the driver's request is removed power well 1
+	 * may stay enabled after this due to DMC's own request on it.
+	 */
 	mutex_lock(&power_domains->lock);
 
 	well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
 	intel_power_well_disable(dev_priv, well);
 
 	mutex_unlock(&power_domains->lock);
+
+	usleep_range(10, 30);		/* 10 us delay per Bspec */
 }
 
 #define CNL_PROCMON_IDX(val) \
@@ -2796,7 +2743,7 @@ static void cnl_display_core_init(struct drm_i915_private *dev_priv, bool resume
 
 	/* 2. Enable Comp */
 	val = I915_READ(CHICKEN_MISC_2);
-	val &= ~COMP_PWR_DOWN;
+	val &= ~CNL_COMP_PWR_DOWN;
 	I915_WRITE(CHICKEN_MISC_2, val);
 
 	val = I915_READ(CNL_PORT_COMP_DW3);
@@ -2821,7 +2768,10 @@ static void cnl_display_core_init(struct drm_i915_private *dev_priv, bool resume
 	val |= CL_POWER_DOWN_ENABLE;
 	I915_WRITE(CNL_PORT_CL1CM_DW5, val);
 
-	/* 4. Enable Power Well 1 (PG1) and Aux IO Power */
+	/*
+	 * 4. Enable Power Well 1 (PG1).
+	 *    The AUX IO power wells will be enabled on demand.
+	 */
 	mutex_lock(&power_domains->lock);
 	well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
 	intel_power_well_enable(dev_priv, well);
@@ -2853,15 +2803,21 @@ static void cnl_display_core_uninit(struct drm_i915_private *dev_priv)
 	/* 3. Disable CD clock */
 	cnl_uninit_cdclk(dev_priv);
 
-	/* 4. Disable Power Well 1 (PG1) and Aux IO Power */
+	/*
+	 * 4. Disable Power Well 1 (PG1).
+	 *    The AUX IO power wells are toggled on demand, so they are already
+	 *    disabled at this point.
+	 */
 	mutex_lock(&power_domains->lock);
 	well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
 	intel_power_well_disable(dev_priv, well);
 	mutex_unlock(&power_domains->lock);
 
+	usleep_range(10, 30);		/* 10 us delay per Bspec */
+
 	/* 5. Disable Comp */
 	val = I915_READ(CHICKEN_MISC_2);
-	val |= COMP_PWR_DOWN;
+	val |= CNL_COMP_PWR_DOWN;
 	I915_WRITE(CHICKEN_MISC_2, val);
 }
 
diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c
index 3f8f30b412cd..29a3b0f5bec7 100644
--- a/drivers/gpu/drm/i915/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/intel_sdvo.c
@@ -451,23 +451,24 @@ static const char * const cmd_status_names[] = {
 	"Scaling not supported"
 };
 
-static bool intel_sdvo_write_cmd(struct intel_sdvo *intel_sdvo, u8 cmd,
-				 const void *args, int args_len)
+static bool __intel_sdvo_write_cmd(struct intel_sdvo *intel_sdvo, u8 cmd,
+				   const void *args, int args_len,
+				   bool unlocked)
 {
 	u8 *buf, status;
 	struct i2c_msg *msgs;
 	int i, ret = true;
 
-        /* Would be simpler to allocate both in one go ? */        
+	/* Would be simpler to allocate both in one go ? */
 	buf = kzalloc(args_len * 2 + 2, GFP_KERNEL);
 	if (!buf)
 		return false;
 
 	msgs = kcalloc(args_len + 3, sizeof(*msgs), GFP_KERNEL);
 	if (!msgs) {
-	        kfree(buf);
+		kfree(buf);
 		return false;
-        }
+	}
 
 	intel_sdvo_debug_write(intel_sdvo, cmd, args, args_len);
 
@@ -498,7 +499,10 @@ static bool intel_sdvo_write_cmd(struct intel_sdvo *intel_sdvo, u8 cmd,
 	msgs[i+2].len = 1;
 	msgs[i+2].buf = &status;
 
-	ret = i2c_transfer(intel_sdvo->i2c, msgs, i+3);
+	if (unlocked)
+		ret = i2c_transfer(intel_sdvo->i2c, msgs, i+3);
+	else
+		ret = __i2c_transfer(intel_sdvo->i2c, msgs, i+3);
 	if (ret < 0) {
 		DRM_DEBUG_KMS("I2c transfer returned %d\n", ret);
 		ret = false;
@@ -516,6 +520,12 @@ out:
 	return ret;
 }
 
+static bool intel_sdvo_write_cmd(struct intel_sdvo *intel_sdvo, u8 cmd,
+				 const void *args, int args_len)
+{
+	return __intel_sdvo_write_cmd(intel_sdvo, cmd, args, args_len, true);
+}
+
 static bool intel_sdvo_read_response(struct intel_sdvo *intel_sdvo,
 				     void *response, int response_len)
 {
@@ -602,13 +612,13 @@ static int intel_sdvo_get_pixel_multiplier(const struct drm_display_mode *adjust
 		return 4;
 }
 
-static bool intel_sdvo_set_control_bus_switch(struct intel_sdvo *intel_sdvo,
-					      u8 ddc_bus)
+static bool __intel_sdvo_set_control_bus_switch(struct intel_sdvo *intel_sdvo,
+						u8 ddc_bus)
 {
 	/* This must be the immediately preceding write before the i2c xfer */
-	return intel_sdvo_write_cmd(intel_sdvo,
-				    SDVO_CMD_SET_CONTROL_BUS_SWITCH,
-				    &ddc_bus, 1);
+	return __intel_sdvo_write_cmd(intel_sdvo,
+				      SDVO_CMD_SET_CONTROL_BUS_SWITCH,
+				      &ddc_bus, 1, false);
 }
 
 static bool intel_sdvo_set_value(struct intel_sdvo *intel_sdvo, u8 cmd, const void *data, int len)
@@ -996,7 +1006,8 @@ static bool intel_sdvo_set_avi_infoframe(struct intel_sdvo *intel_sdvo,
 	ssize_t len;
 
 	ret = drm_hdmi_avi_infoframe_from_display_mode(&frame.avi,
-						       &pipe_config->base.adjusted_mode);
+						       &pipe_config->base.adjusted_mode,
+						       false);
 	if (ret < 0) {
 		DRM_ERROR("couldn't fill AVI infoframe\n");
 		return false;
@@ -1343,13 +1354,15 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder,
 		sdvox |= (9 << 19) | SDVO_BORDER_ENABLE;
 	}
 
-	if (INTEL_PCH_TYPE(dev_priv) >= PCH_CPT)
+	if (HAS_PCH_CPT(dev_priv))
 		sdvox |= SDVO_PIPE_SEL_CPT(crtc->pipe);
 	else
 		sdvox |= SDVO_PIPE_SEL(crtc->pipe);
 
-	if (crtc_state->has_audio)
+	if (crtc_state->has_audio) {
+		WARN_ON_ONCE(INTEL_GEN(dev_priv) < 4);
 		sdvox |= SDVO_AUDIO_ENABLE;
+	}
 
 	if (INTEL_GEN(dev_priv) >= 4) {
 		/* done in crtc_mode_set as the dpll_md reg must be written early */
@@ -1479,6 +1492,9 @@ static void intel_sdvo_get_config(struct intel_encoder *encoder,
 	if (sdvox & HDMI_COLOR_RANGE_16_235)
 		pipe_config->limited_color_range = true;
 
+	if (sdvox & SDVO_AUDIO_ENABLE)
+		pipe_config->has_audio = true;
+
 	if (intel_sdvo_get_value(intel_sdvo, SDVO_CMD_GET_ENCODE,
 				 &val, 1)) {
 		if (val == SDVO_ENCODE_HDMI)
@@ -2192,10 +2208,8 @@ intel_sdvo_connector_duplicate_state(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs intel_sdvo_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.detect = intel_sdvo_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.set_property = drm_atomic_helper_connector_set_property,
 	.atomic_get_property = intel_sdvo_connector_atomic_get_property,
 	.atomic_set_property = intel_sdvo_connector_atomic_set_property,
 	.late_register = intel_sdvo_connector_register,
@@ -2454,6 +2468,7 @@ static bool
 intel_sdvo_dvi_init(struct intel_sdvo *intel_sdvo, int device)
 {
 	struct drm_encoder *encoder = &intel_sdvo->base.base;
+	struct drm_i915_private *dev_priv = to_i915(encoder->dev);
 	struct drm_connector *connector;
 	struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
 	struct intel_connector *intel_connector;
@@ -2489,7 +2504,9 @@ intel_sdvo_dvi_init(struct intel_sdvo *intel_sdvo, int device)
 	encoder->encoder_type = DRM_MODE_ENCODER_TMDS;
 	connector->connector_type = DRM_MODE_CONNECTOR_DVID;
 
-	if (intel_sdvo_is_hdmi_connector(intel_sdvo, device)) {
+	/* gen3 doesn't do the hdmi bits in the SDVO register */
+	if (INTEL_GEN(dev_priv) >= 4 &&
+	    intel_sdvo_is_hdmi_connector(intel_sdvo, device)) {
 		connector->connector_type = DRM_MODE_CONNECTOR_HDMIA;
 		intel_sdvo->is_hdmi = true;
 	}
@@ -2925,7 +2942,7 @@ static int intel_sdvo_ddc_proxy_xfer(struct i2c_adapter *adapter,
 {
 	struct intel_sdvo *sdvo = adapter->algo_data;
 
-	if (!intel_sdvo_set_control_bus_switch(sdvo, sdvo->ddc_bus))
+	if (!__intel_sdvo_set_control_bus_switch(sdvo, sdvo->ddc_bus))
 		return -EIO;
 
 	return sdvo->i2c->algo->master_xfer(sdvo->i2c, msgs, num);
@@ -2942,6 +2959,33 @@ static const struct i2c_algorithm intel_sdvo_ddc_proxy = {
 	.functionality	= intel_sdvo_ddc_proxy_func
 };
 
+static void proxy_lock_bus(struct i2c_adapter *adapter,
+			   unsigned int flags)
+{
+	struct intel_sdvo *sdvo = adapter->algo_data;
+	sdvo->i2c->lock_ops->lock_bus(sdvo->i2c, flags);
+}
+
+static int proxy_trylock_bus(struct i2c_adapter *adapter,
+			     unsigned int flags)
+{
+	struct intel_sdvo *sdvo = adapter->algo_data;
+	return sdvo->i2c->lock_ops->trylock_bus(sdvo->i2c, flags);
+}
+
+static void proxy_unlock_bus(struct i2c_adapter *adapter,
+			     unsigned int flags)
+{
+	struct intel_sdvo *sdvo = adapter->algo_data;
+	sdvo->i2c->lock_ops->unlock_bus(sdvo->i2c, flags);
+}
+
+static const struct i2c_lock_operations proxy_lock_ops = {
+	.lock_bus =    proxy_lock_bus,
+	.trylock_bus = proxy_trylock_bus,
+	.unlock_bus =  proxy_unlock_bus,
+};
+
 static bool
 intel_sdvo_init_ddc_proxy(struct intel_sdvo *sdvo,
 			  struct drm_i915_private *dev_priv)
@@ -2954,6 +2998,7 @@ intel_sdvo_init_ddc_proxy(struct intel_sdvo *sdvo,
 	sdvo->ddc.dev.parent = &pdev->dev;
 	sdvo->ddc.algo_data = sdvo;
 	sdvo->ddc.algo = &intel_sdvo_ddc_proxy;
+	sdvo->ddc.lock_ops = &proxy_lock_ops;
 
 	return i2c_add_adapter(&sdvo->ddc) == 0;
 }
diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c
index 0c650c2cbca8..524933b01483 100644
--- a/drivers/gpu/drm/i915/intel_sprite.c
+++ b/drivers/gpu/drm/i915/intel_sprite.c
@@ -30,6 +30,7 @@
  * support.
  */
 #include <drm/drmP.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_rect.h>
@@ -176,7 +177,7 @@ void intel_pipe_update_start(struct intel_crtc *crtc)
  * re-enables interrupts and verifies the update was actually completed
  * before a vblank using the value of @start_vbl_count.
  */
-void intel_pipe_update_end(struct intel_crtc *crtc, struct intel_flip_work *work)
+void intel_pipe_update_end(struct intel_crtc *crtc)
 {
 	enum pipe pipe = crtc->pipe;
 	int scanline_end = intel_get_crtc_scanline(crtc);
@@ -184,12 +185,6 @@ void intel_pipe_update_end(struct intel_crtc *crtc, struct intel_flip_work *work
 	ktime_t end_vbl_time = ktime_get();
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 
-	if (work) {
-		work->flip_queued_vblank = end_vbl_count;
-		smp_mb__before_atomic();
-		atomic_set(&work->pending, 1);
-	}
-
 	trace_i915_pipe_update_end(crtc, end_vbl_count, scanline_end);
 
 	/* We're still in the vblank-evade critical section, this can't race.
@@ -244,6 +239,7 @@ skl_update_plane(struct intel_plane *plane,
 	u32 surf_addr = plane_state->main.offset;
 	unsigned int rotation = plane_state->base.rotation;
 	u32 stride = skl_plane_stride(fb, 0, rotation);
+	u32 aux_stride = skl_plane_stride(fb, 1, rotation);
 	int crtc_x = plane_state->base.dst.x1;
 	int crtc_y = plane_state->base.dst.y1;
 	uint32_t crtc_w = drm_rect_width(&plane_state->base.dst);
@@ -262,7 +258,7 @@ skl_update_plane(struct intel_plane *plane,
 
 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 
-	if (IS_GEMINILAKE(dev_priv)) {
+	if (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) {
 		I915_WRITE_FW(PLANE_COLOR_CTL(pipe, plane_id),
 			      PLANE_COLOR_PIPE_GAMMA_ENABLE |
 			      PLANE_COLOR_PIPE_CSC_ENABLE |
@@ -278,6 +274,10 @@ skl_update_plane(struct intel_plane *plane,
 	I915_WRITE_FW(PLANE_OFFSET(pipe, plane_id), (y << 16) | x);
 	I915_WRITE_FW(PLANE_STRIDE(pipe, plane_id), stride);
 	I915_WRITE_FW(PLANE_SIZE(pipe, plane_id), (src_h << 16) | src_w);
+	I915_WRITE_FW(PLANE_AUX_DIST(pipe, plane_id),
+		      (plane_state->aux.offset - surf_addr) | aux_stride);
+	I915_WRITE_FW(PLANE_AUX_OFFSET(pipe, plane_id),
+		      (plane_state->aux.y << 16) | plane_state->aux.x);
 
 	/* program plane scaler */
 	if (plane_state->scaler_id >= 0) {
@@ -1038,6 +1038,12 @@ static const uint32_t g4x_plane_formats[] = {
 	DRM_FORMAT_VYUY,
 };
 
+static const uint64_t i9xx_plane_format_modifiers[] = {
+	I915_FORMAT_MOD_X_TILED,
+	DRM_FORMAT_MOD_LINEAR,
+	DRM_FORMAT_MOD_INVALID
+};
+
 static const uint32_t snb_plane_formats[] = {
 	DRM_FORMAT_XBGR8888,
 	DRM_FORMAT_XRGB8888,
@@ -1073,6 +1079,122 @@ static uint32_t skl_plane_formats[] = {
 	DRM_FORMAT_VYUY,
 };
 
+static const uint64_t skl_plane_format_modifiers[] = {
+	I915_FORMAT_MOD_X_TILED,
+	DRM_FORMAT_MOD_LINEAR,
+	DRM_FORMAT_MOD_INVALID
+};
+
+static bool g4x_sprite_plane_format_mod_supported(struct drm_plane *plane,
+						  uint32_t format,
+						  uint64_t modifier)
+{
+	switch (format) {
+	case DRM_FORMAT_XBGR8888:
+	case DRM_FORMAT_XRGB8888:
+	case DRM_FORMAT_YUYV:
+	case DRM_FORMAT_YVYU:
+	case DRM_FORMAT_UYVY:
+	case DRM_FORMAT_VYUY:
+		if (modifier == DRM_FORMAT_MOD_LINEAR ||
+		    modifier == I915_FORMAT_MOD_X_TILED)
+			return true;
+		/* fall through */
+	default:
+		return false;
+	}
+}
+
+static bool vlv_sprite_plane_format_mod_supported(struct drm_plane *plane,
+						  uint32_t format,
+						  uint64_t modifier)
+{
+	switch (format) {
+	case DRM_FORMAT_YUYV:
+	case DRM_FORMAT_YVYU:
+	case DRM_FORMAT_UYVY:
+	case DRM_FORMAT_VYUY:
+	case DRM_FORMAT_RGB565:
+	case DRM_FORMAT_XRGB8888:
+	case DRM_FORMAT_ARGB8888:
+	case DRM_FORMAT_XBGR2101010:
+	case DRM_FORMAT_ABGR2101010:
+	case DRM_FORMAT_XBGR8888:
+	case DRM_FORMAT_ABGR8888:
+		if (modifier == DRM_FORMAT_MOD_LINEAR ||
+		    modifier == I915_FORMAT_MOD_X_TILED)
+			return true;
+		/* fall through */
+	default:
+		return false;
+	}
+}
+
+static bool skl_sprite_plane_format_mod_supported(struct drm_plane *plane,
+						  uint32_t format,
+						  uint64_t modifier)
+{
+	/* This is the same as primary plane since SKL has universal planes */
+	switch (format) {
+	case DRM_FORMAT_XRGB8888:
+	case DRM_FORMAT_XBGR8888:
+	case DRM_FORMAT_ARGB8888:
+	case DRM_FORMAT_ABGR8888:
+	case DRM_FORMAT_RGB565:
+	case DRM_FORMAT_XRGB2101010:
+	case DRM_FORMAT_XBGR2101010:
+	case DRM_FORMAT_YUYV:
+	case DRM_FORMAT_YVYU:
+	case DRM_FORMAT_UYVY:
+	case DRM_FORMAT_VYUY:
+		if (modifier == I915_FORMAT_MOD_Yf_TILED)
+			return true;
+		/* fall through */
+	case DRM_FORMAT_C8:
+		if (modifier == DRM_FORMAT_MOD_LINEAR ||
+		    modifier == I915_FORMAT_MOD_X_TILED ||
+		    modifier == I915_FORMAT_MOD_Y_TILED)
+			return true;
+		/* fall through */
+	default:
+		return false;
+	}
+}
+
+static bool intel_sprite_plane_format_mod_supported(struct drm_plane *plane,
+                                                    uint32_t format,
+                                                    uint64_t modifier)
+{
+	struct drm_i915_private *dev_priv = to_i915(plane->dev);
+
+	if (WARN_ON(modifier == DRM_FORMAT_MOD_INVALID))
+		return false;
+
+	if ((modifier >> 56) != DRM_FORMAT_MOD_VENDOR_INTEL &&
+	    modifier != DRM_FORMAT_MOD_LINEAR)
+		return false;
+
+	if (INTEL_GEN(dev_priv) >= 9)
+		return skl_sprite_plane_format_mod_supported(plane, format, modifier);
+	else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
+		return vlv_sprite_plane_format_mod_supported(plane, format, modifier);
+	else
+		return g4x_sprite_plane_format_mod_supported(plane, format, modifier);
+
+	unreachable();
+}
+
+static const struct drm_plane_funcs intel_sprite_plane_funcs = {
+        .update_plane = drm_atomic_helper_update_plane,
+        .disable_plane = drm_atomic_helper_disable_plane,
+        .destroy = intel_plane_destroy,
+        .atomic_get_property = intel_plane_atomic_get_property,
+        .atomic_set_property = intel_plane_atomic_set_property,
+        .atomic_duplicate_state = intel_plane_duplicate_state,
+        .atomic_destroy_state = intel_plane_destroy_state,
+        .format_mod_supported = intel_sprite_plane_format_mod_supported,
+};
+
 struct intel_plane *
 intel_sprite_plane_create(struct drm_i915_private *dev_priv,
 			  enum pipe pipe, int plane)
@@ -1081,6 +1203,7 @@ intel_sprite_plane_create(struct drm_i915_private *dev_priv,
 	struct intel_plane_state *state = NULL;
 	unsigned long possible_crtcs;
 	const uint32_t *plane_formats;
+	const uint64_t *modifiers;
 	unsigned int supported_rotations;
 	int num_plane_formats;
 	int ret;
@@ -1098,7 +1221,17 @@ intel_sprite_plane_create(struct drm_i915_private *dev_priv,
 	}
 	intel_plane->base.state = &state->base;
 
-	if (INTEL_GEN(dev_priv) >= 9) {
+	if (INTEL_GEN(dev_priv) >= 10) {
+		intel_plane->can_scale = true;
+		state->scaler_id = -1;
+
+		intel_plane->update_plane = skl_update_plane;
+		intel_plane->disable_plane = skl_disable_plane;
+
+		plane_formats = skl_plane_formats;
+		num_plane_formats = ARRAY_SIZE(skl_plane_formats);
+		modifiers = skl_plane_format_modifiers;
+	} else if (INTEL_GEN(dev_priv) >= 9) {
 		intel_plane->can_scale = true;
 		state->scaler_id = -1;
 
@@ -1107,6 +1240,7 @@ intel_sprite_plane_create(struct drm_i915_private *dev_priv,
 
 		plane_formats = skl_plane_formats;
 		num_plane_formats = ARRAY_SIZE(skl_plane_formats);
+		modifiers = skl_plane_format_modifiers;
 	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		intel_plane->can_scale = false;
 		intel_plane->max_downscale = 1;
@@ -1116,6 +1250,7 @@ intel_sprite_plane_create(struct drm_i915_private *dev_priv,
 
 		plane_formats = vlv_plane_formats;
 		num_plane_formats = ARRAY_SIZE(vlv_plane_formats);
+		modifiers = i9xx_plane_format_modifiers;
 	} else if (INTEL_GEN(dev_priv) >= 7) {
 		if (IS_IVYBRIDGE(dev_priv)) {
 			intel_plane->can_scale = true;
@@ -1130,6 +1265,7 @@ intel_sprite_plane_create(struct drm_i915_private *dev_priv,
 
 		plane_formats = snb_plane_formats;
 		num_plane_formats = ARRAY_SIZE(snb_plane_formats);
+		modifiers = i9xx_plane_format_modifiers;
 	} else {
 		intel_plane->can_scale = true;
 		intel_plane->max_downscale = 16;
@@ -1137,6 +1273,7 @@ intel_sprite_plane_create(struct drm_i915_private *dev_priv,
 		intel_plane->update_plane = g4x_update_plane;
 		intel_plane->disable_plane = g4x_disable_plane;
 
+		modifiers = i9xx_plane_format_modifiers;
 		if (IS_GEN6(dev_priv)) {
 			plane_formats = snb_plane_formats;
 			num_plane_formats = ARRAY_SIZE(snb_plane_formats);
@@ -1169,14 +1306,16 @@ intel_sprite_plane_create(struct drm_i915_private *dev_priv,
 
 	if (INTEL_GEN(dev_priv) >= 9)
 		ret = drm_universal_plane_init(&dev_priv->drm, &intel_plane->base,
-					       possible_crtcs, &intel_plane_funcs,
+					       possible_crtcs, &intel_sprite_plane_funcs,
 					       plane_formats, num_plane_formats,
+					       modifiers,
 					       DRM_PLANE_TYPE_OVERLAY,
 					       "plane %d%c", plane + 2, pipe_name(pipe));
 	else
 		ret = drm_universal_plane_init(&dev_priv->drm, &intel_plane->base,
-					       possible_crtcs, &intel_plane_funcs,
+					       possible_crtcs, &intel_sprite_plane_funcs,
 					       plane_formats, num_plane_formats,
+					       modifiers,
 					       DRM_PLANE_TYPE_OVERLAY,
 					       "sprite %c", sprite_name(pipe, plane));
 	if (ret)
diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c
index 784df024e230..906893c006d8 100644
--- a/drivers/gpu/drm/i915/intel_tv.c
+++ b/drivers/gpu/drm/i915/intel_tv.c
@@ -1407,11 +1407,9 @@ intel_tv_destroy(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs intel_tv_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.late_register = intel_connector_register,
 	.early_unregister = intel_connector_unregister,
 	.destroy = intel_tv_destroy,
-	.set_property = drm_atomic_helper_connector_set_property,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
diff --git a/drivers/gpu/drm/i915/intel_uc.c b/drivers/gpu/drm/i915/intel_uc.c
index 27e072cc96eb..0178ba42a0e5 100644
--- a/drivers/gpu/drm/i915/intel_uc.c
+++ b/drivers/gpu/drm/i915/intel_uc.c
@@ -94,7 +94,7 @@ void intel_uc_sanitize_options(struct drm_i915_private *dev_priv)
 		i915.enable_guc_submission = HAS_GUC_SCHED(dev_priv);
 }
 
-static void guc_write_irq_trigger(struct intel_guc *guc)
+static void gen8_guc_raise_irq(struct intel_guc *guc)
 {
 	struct drm_i915_private *dev_priv = guc_to_i915(guc);
 
@@ -109,7 +109,7 @@ void intel_uc_init_early(struct drm_i915_private *dev_priv)
 
 	mutex_init(&guc->send_mutex);
 	guc->send = intel_guc_send_nop;
-	guc->notify = guc_write_irq_trigger;
+	guc->notify = gen8_guc_raise_irq;
 }
 
 static void fetch_uc_fw(struct drm_i915_private *dev_priv,
diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
index 9882724bc2b6..1d7b879cc68c 100644
--- a/drivers/gpu/drm/i915/intel_uncore.c
+++ b/drivers/gpu/drm/i915/intel_uncore.c
@@ -643,7 +643,7 @@ find_fw_domain(struct drm_i915_private *dev_priv, u32 offset)
 	{ .start = (s), .end = (e), .domains = (d) }
 
 #define HAS_FWTABLE(dev_priv) \
-	(IS_GEN9(dev_priv) || \
+	(INTEL_GEN(dev_priv) >= 9 || \
 	 IS_CHERRYVIEW(dev_priv) || \
 	 IS_VALLEYVIEW(dev_priv))
 
@@ -1072,7 +1072,7 @@ static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv)
 		dev_priv->uncore.fw_clear = _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL);
 	}
 
-	if (IS_GEN9(dev_priv)) {
+	if (INTEL_GEN(dev_priv) >= 9) {
 		dev_priv->uncore.funcs.force_wake_get = fw_domains_get;
 		dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
 		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
@@ -1497,7 +1497,6 @@ static int gen6_reset_engines(struct drm_i915_private *dev_priv,
 		[VECS] = GEN6_GRDOM_VECS,
 	};
 	u32 hw_mask;
-	int ret;
 
 	if (engine_mask == ALL_ENGINES) {
 		hw_mask = GEN6_GRDOM_FULL;
@@ -1509,11 +1508,7 @@ static int gen6_reset_engines(struct drm_i915_private *dev_priv,
 			hw_mask |= hw_engine_mask[engine->id];
 	}
 
-	ret = gen6_hw_domain_reset(dev_priv, hw_mask);
-
-	intel_uncore_forcewake_reset(dev_priv, true);
-
-	return ret;
+	return gen6_hw_domain_reset(dev_priv, hw_mask);
 }
 
 /**
@@ -1719,6 +1714,17 @@ bool intel_has_gpu_reset(struct drm_i915_private *dev_priv)
 	return intel_get_gpu_reset(dev_priv) != NULL;
 }
 
+/*
+ * When GuC submission is enabled, GuC manages ELSP and can initiate the
+ * engine reset too. For now, fall back to full GPU reset if it is enabled.
+ */
+bool intel_has_reset_engine(struct drm_i915_private *dev_priv)
+{
+	return (dev_priv->info.has_reset_engine &&
+		!dev_priv->guc.execbuf_client &&
+		i915.reset >= 2);
+}
+
 int intel_guc_reset(struct drm_i915_private *dev_priv)
 {
 	int ret;
diff --git a/drivers/gpu/drm/i915/selftests/huge_gem_object.c b/drivers/gpu/drm/i915/selftests/huge_gem_object.c
index caf76af36aba..c5c7e8efbdd3 100644
--- a/drivers/gpu/drm/i915/selftests/huge_gem_object.c
+++ b/drivers/gpu/drm/i915/selftests/huge_gem_object.c
@@ -111,6 +111,7 @@ huge_gem_object(struct drm_i915_private *i915,
 		dma_addr_t dma_size)
 {
 	struct drm_i915_gem_object *obj;
+	unsigned int cache_level;
 
 	GEM_BUG_ON(!phys_size || phys_size > dma_size);
 	GEM_BUG_ON(!IS_ALIGNED(phys_size, PAGE_SIZE));
@@ -128,9 +129,8 @@ huge_gem_object(struct drm_i915_private *i915,
 
 	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
 	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
-	obj->cache_level = HAS_LLC(i915) ? I915_CACHE_LLC : I915_CACHE_NONE;
-	obj->cache_coherent = i915_gem_object_is_coherent(obj);
-	obj->cache_dirty = !obj->cache_coherent;
+	cache_level = HAS_LLC(i915) ? I915_CACHE_LLC : I915_CACHE_NONE;
+	i915_gem_object_set_cache_coherency(obj, cache_level);
 	obj->scratch = phys_size;
 
 	return obj;
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c b/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c
index 95d4aebc0181..35d778d70626 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c
@@ -241,7 +241,7 @@ static bool always_valid(struct drm_i915_private *i915)
 
 static bool needs_mi_store_dword(struct drm_i915_private *i915)
 {
-	return igt_can_mi_store_dword_imm(i915);
+	return intel_engine_can_store_dword(i915->engine[RCS]);
 }
 
 static const struct igt_coherency_mode {
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/selftests/i915_gem_context.c
index 12b85b3278cd..fb0a58fc8348 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_context.c
@@ -38,8 +38,6 @@ gpu_fill_dw(struct i915_vma *vma, u64 offset, unsigned long count, u32 value)
 	u32 *cmd;
 	int err;
 
-	GEM_BUG_ON(!igt_can_mi_store_dword_imm(vma->vm->i915));
-
 	size = (4 * count + 1) * sizeof(u32);
 	size = round_up(size, PAGE_SIZE);
 	obj = i915_gem_object_create_internal(vma->vm->i915, size);
@@ -123,6 +121,7 @@ static int gpu_fill(struct drm_i915_gem_object *obj,
 	int err;
 
 	GEM_BUG_ON(obj->base.size > vm->total);
+	GEM_BUG_ON(!intel_engine_can_store_dword(engine));
 
 	vma = i915_vma_instance(obj, vm, NULL);
 	if (IS_ERR(vma))
@@ -359,6 +358,9 @@ static int igt_ctx_exec(void *arg)
 		}
 
 		for_each_engine(engine, i915, id) {
+			if (!intel_engine_can_store_dword(engine))
+				continue;
+
 			if (!obj) {
 				obj = create_test_object(ctx, file, &objects);
 				if (IS_ERR(obj)) {
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
index 50710e3f1caa..6b132caffa18 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
@@ -197,6 +197,9 @@ static int lowlevel_hole(struct drm_i915_private *i915,
 {
 	I915_RND_STATE(seed_prng);
 	unsigned int size;
+	struct i915_vma mock_vma;
+
+	memset(&mock_vma, 0, sizeof(struct i915_vma));
 
 	/* Keep creating larger objects until one cannot fit into the hole */
 	for (size = 12; (hole_end - hole_start) >> size; size++) {
@@ -255,8 +258,11 @@ static int lowlevel_hole(struct drm_i915_private *i915,
 			    vm->allocate_va_range(vm, addr, BIT_ULL(size)))
 				break;
 
-			vm->insert_entries(vm, obj->mm.pages, addr,
-					   I915_CACHE_NONE, 0);
+			mock_vma.pages = obj->mm.pages;
+			mock_vma.node.size = BIT_ULL(size);
+			mock_vma.node.start = addr;
+
+			vm->insert_entries(vm, &mock_vma, I915_CACHE_NONE, 0);
 		}
 		count = n;
 
diff --git a/drivers/gpu/drm/i915/selftests/i915_random.c b/drivers/gpu/drm/i915/selftests/i915_random.c
index d044bf9a6feb..222c511bea49 100644
--- a/drivers/gpu/drm/i915/selftests/i915_random.c
+++ b/drivers/gpu/drm/i915/selftests/i915_random.c
@@ -62,7 +62,7 @@ unsigned int *i915_random_order(unsigned int count, struct rnd_state *state)
 {
 	unsigned int *order, i;
 
-	order = kmalloc_array(count, sizeof(*order), GFP_TEMPORARY);
+	order = kmalloc_array(count, sizeof(*order), GFP_KERNEL);
 	if (!order)
 		return order;
 
diff --git a/drivers/gpu/drm/i915/selftests/i915_vma.c b/drivers/gpu/drm/i915/selftests/i915_vma.c
index fb9072d5877f..2e86ec136b35 100644
--- a/drivers/gpu/drm/i915/selftests/i915_vma.c
+++ b/drivers/gpu/drm/i915/selftests/i915_vma.c
@@ -186,16 +186,20 @@ static int igt_vma_create(void *arg)
 				goto end;
 		}
 
-		list_for_each_entry_safe(ctx, cn, &contexts, link)
+		list_for_each_entry_safe(ctx, cn, &contexts, link) {
+			list_del_init(&ctx->link);
 			mock_context_close(ctx);
+		}
 	}
 
 end:
 	/* Final pass to lookup all created contexts */
 	err = create_vmas(i915, &objects, &contexts);
 out:
-	list_for_each_entry_safe(ctx, cn, &contexts, link)
+	list_for_each_entry_safe(ctx, cn, &contexts, link) {
+		list_del_init(&ctx->link);
 		mock_context_close(ctx);
+	}
 
 	list_for_each_entry_safe(obj, on, &objects, st_link)
 		i915_gem_object_put(obj);
diff --git a/drivers/gpu/drm/i915/selftests/intel_breadcrumbs.c b/drivers/gpu/drm/i915/selftests/intel_breadcrumbs.c
index 7276194c04f7..828904b7d468 100644
--- a/drivers/gpu/drm/i915/selftests/intel_breadcrumbs.c
+++ b/drivers/gpu/drm/i915/selftests/intel_breadcrumbs.c
@@ -117,12 +117,12 @@ static int igt_random_insert_remove(void *arg)
 
 	mock_engine_reset(engine);
 
-	waiters = kvmalloc_array(count, sizeof(*waiters), GFP_TEMPORARY);
+	waiters = kvmalloc_array(count, sizeof(*waiters), GFP_KERNEL);
 	if (!waiters)
 		goto out_engines;
 
 	bitmap = kcalloc(DIV_ROUND_UP(count, BITS_PER_LONG), sizeof(*bitmap),
-			 GFP_TEMPORARY);
+			 GFP_KERNEL);
 	if (!bitmap)
 		goto out_waiters;
 
@@ -187,12 +187,12 @@ static int igt_insert_complete(void *arg)
 
 	mock_engine_reset(engine);
 
-	waiters = kvmalloc_array(count, sizeof(*waiters), GFP_TEMPORARY);
+	waiters = kvmalloc_array(count, sizeof(*waiters), GFP_KERNEL);
 	if (!waiters)
 		goto out_engines;
 
 	bitmap = kcalloc(DIV_ROUND_UP(count, BITS_PER_LONG), sizeof(*bitmap),
-			 GFP_TEMPORARY);
+			 GFP_KERNEL);
 	if (!bitmap)
 		goto out_waiters;
 
@@ -368,7 +368,7 @@ static int igt_wakeup(void *arg)
 
 	mock_engine_reset(engine);
 
-	waiters = kvmalloc_array(count, sizeof(*waiters), GFP_TEMPORARY);
+	waiters = kvmalloc_array(count, sizeof(*waiters), GFP_KERNEL);
 	if (!waiters)
 		goto out_engines;
 
diff --git a/drivers/gpu/drm/i915/selftests/intel_hangcheck.c b/drivers/gpu/drm/i915/selftests/intel_hangcheck.c
index aa31d6c0cdfb..02e52a146ed8 100644
--- a/drivers/gpu/drm/i915/selftests/intel_hangcheck.c
+++ b/drivers/gpu/drm/i915/selftests/intel_hangcheck.c
@@ -22,8 +22,13 @@
  *
  */
 
+#include <linux/kthread.h>
+
 #include "../i915_selftest.h"
 
+#include "mock_context.h"
+#include "mock_drm.h"
+
 struct hang {
 	struct drm_i915_private *i915;
 	struct drm_i915_gem_object *hws;
@@ -248,9 +253,6 @@ static int igt_hang_sanitycheck(void *arg)
 
 	/* Basic check that we can execute our hanging batch */
 
-	if (!igt_can_mi_store_dword_imm(i915))
-		return 0;
-
 	mutex_lock(&i915->drm.struct_mutex);
 	err = hang_init(&h, i915);
 	if (err)
@@ -259,6 +261,9 @@ static int igt_hang_sanitycheck(void *arg)
 	for_each_engine(engine, i915, id) {
 		long timeout;
 
+		if (!intel_engine_can_store_dword(engine))
+			continue;
+
 		rq = hang_create_request(&h, engine, i915->kernel_context);
 		if (IS_ERR(rq)) {
 			err = PTR_ERR(rq);
@@ -292,6 +297,37 @@ unlock:
 	return err;
 }
 
+static void global_reset_lock(struct drm_i915_private *i915)
+{
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+
+	while (test_and_set_bit(I915_RESET_BACKOFF, &i915->gpu_error.flags))
+		wait_event(i915->gpu_error.reset_queue,
+			   !test_bit(I915_RESET_BACKOFF,
+				     &i915->gpu_error.flags));
+
+	for_each_engine(engine, i915, id) {
+		while (test_and_set_bit(I915_RESET_ENGINE + id,
+					&i915->gpu_error.flags))
+			wait_on_bit(&i915->gpu_error.flags,
+				    I915_RESET_ENGINE + id,
+				    TASK_UNINTERRUPTIBLE);
+	}
+}
+
+static void global_reset_unlock(struct drm_i915_private *i915)
+{
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+
+	for_each_engine(engine, i915, id)
+		clear_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags);
+
+	clear_bit(I915_RESET_BACKOFF, &i915->gpu_error.flags);
+	wake_up_all(&i915->gpu_error.reset_queue);
+}
+
 static int igt_global_reset(void *arg)
 {
 	struct drm_i915_private *i915 = arg;
@@ -300,13 +336,13 @@ static int igt_global_reset(void *arg)
 
 	/* Check that we can issue a global GPU reset */
 
-	set_bit(I915_RESET_BACKOFF, &i915->gpu_error.flags);
+	global_reset_lock(i915);
 	set_bit(I915_RESET_HANDOFF, &i915->gpu_error.flags);
 
 	mutex_lock(&i915->drm.struct_mutex);
 	reset_count = i915_reset_count(&i915->gpu_error);
 
-	i915_reset(i915);
+	i915_reset(i915, I915_RESET_QUIET);
 
 	if (i915_reset_count(&i915->gpu_error) == reset_count) {
 		pr_err("No GPU reset recorded!\n");
@@ -315,7 +351,214 @@ static int igt_global_reset(void *arg)
 	mutex_unlock(&i915->drm.struct_mutex);
 
 	GEM_BUG_ON(test_bit(I915_RESET_HANDOFF, &i915->gpu_error.flags));
-	clear_bit(I915_RESET_BACKOFF, &i915->gpu_error.flags);
+	global_reset_unlock(i915);
+
+	if (i915_terminally_wedged(&i915->gpu_error))
+		err = -EIO;
+
+	return err;
+}
+
+static int igt_reset_engine(void *arg)
+{
+	struct drm_i915_private *i915 = arg;
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+	unsigned int reset_count, reset_engine_count;
+	int err = 0;
+
+	/* Check that we can issue a global GPU and engine reset */
+
+	if (!intel_has_reset_engine(i915))
+		return 0;
+
+	for_each_engine(engine, i915, id) {
+		set_bit(I915_RESET_ENGINE + engine->id, &i915->gpu_error.flags);
+		reset_count = i915_reset_count(&i915->gpu_error);
+		reset_engine_count = i915_reset_engine_count(&i915->gpu_error,
+							     engine);
+
+		err = i915_reset_engine(engine, I915_RESET_QUIET);
+		if (err) {
+			pr_err("i915_reset_engine failed\n");
+			break;
+		}
+
+		if (i915_reset_count(&i915->gpu_error) != reset_count) {
+			pr_err("Full GPU reset recorded! (engine reset expected)\n");
+			err = -EINVAL;
+			break;
+		}
+
+		if (i915_reset_engine_count(&i915->gpu_error, engine) ==
+		    reset_engine_count) {
+			pr_err("No %s engine reset recorded!\n", engine->name);
+			err = -EINVAL;
+			break;
+		}
+
+		clear_bit(I915_RESET_ENGINE + engine->id,
+			  &i915->gpu_error.flags);
+	}
+
+	if (i915_terminally_wedged(&i915->gpu_error))
+		err = -EIO;
+
+	return err;
+}
+
+static int active_engine(void *data)
+{
+	struct intel_engine_cs *engine = data;
+	struct drm_i915_gem_request *rq[2] = {};
+	struct i915_gem_context *ctx[2];
+	struct drm_file *file;
+	unsigned long count = 0;
+	int err = 0;
+
+	file = mock_file(engine->i915);
+	if (IS_ERR(file))
+		return PTR_ERR(file);
+
+	mutex_lock(&engine->i915->drm.struct_mutex);
+	ctx[0] = live_context(engine->i915, file);
+	mutex_unlock(&engine->i915->drm.struct_mutex);
+	if (IS_ERR(ctx[0])) {
+		err = PTR_ERR(ctx[0]);
+		goto err_file;
+	}
+
+	mutex_lock(&engine->i915->drm.struct_mutex);
+	ctx[1] = live_context(engine->i915, file);
+	mutex_unlock(&engine->i915->drm.struct_mutex);
+	if (IS_ERR(ctx[1])) {
+		err = PTR_ERR(ctx[1]);
+		i915_gem_context_put(ctx[0]);
+		goto err_file;
+	}
+
+	while (!kthread_should_stop()) {
+		unsigned int idx = count++ & 1;
+		struct drm_i915_gem_request *old = rq[idx];
+		struct drm_i915_gem_request *new;
+
+		mutex_lock(&engine->i915->drm.struct_mutex);
+		new = i915_gem_request_alloc(engine, ctx[idx]);
+		if (IS_ERR(new)) {
+			mutex_unlock(&engine->i915->drm.struct_mutex);
+			err = PTR_ERR(new);
+			break;
+		}
+
+		rq[idx] = i915_gem_request_get(new);
+		i915_add_request(new);
+		mutex_unlock(&engine->i915->drm.struct_mutex);
+
+		if (old) {
+			i915_wait_request(old, 0, MAX_SCHEDULE_TIMEOUT);
+			i915_gem_request_put(old);
+		}
+	}
+
+	for (count = 0; count < ARRAY_SIZE(rq); count++)
+		i915_gem_request_put(rq[count]);
+
+err_file:
+	mock_file_free(engine->i915, file);
+	return err;
+}
+
+static int igt_reset_active_engines(void *arg)
+{
+	struct drm_i915_private *i915 = arg;
+	struct intel_engine_cs *engine, *active;
+	enum intel_engine_id id, tmp;
+	int err = 0;
+
+	/* Check that issuing a reset on one engine does not interfere
+	 * with any other engine.
+	 */
+
+	if (!intel_has_reset_engine(i915))
+		return 0;
+
+	for_each_engine(engine, i915, id) {
+		struct task_struct *threads[I915_NUM_ENGINES];
+		unsigned long resets[I915_NUM_ENGINES];
+		unsigned long global = i915_reset_count(&i915->gpu_error);
+		IGT_TIMEOUT(end_time);
+
+		memset(threads, 0, sizeof(threads));
+		for_each_engine(active, i915, tmp) {
+			struct task_struct *tsk;
+
+			if (active == engine)
+				continue;
+
+			resets[tmp] = i915_reset_engine_count(&i915->gpu_error,
+							      active);
+
+			tsk = kthread_run(active_engine, active,
+					  "igt/%s", active->name);
+			if (IS_ERR(tsk)) {
+				err = PTR_ERR(tsk);
+				goto unwind;
+			}
+
+			threads[tmp] = tsk;
+			get_task_struct(tsk);
+		}
+
+		set_bit(I915_RESET_ENGINE + engine->id, &i915->gpu_error.flags);
+		do {
+			err = i915_reset_engine(engine, I915_RESET_QUIET);
+			if (err) {
+				pr_err("i915_reset_engine(%s) failed, err=%d\n",
+				       engine->name, err);
+				break;
+			}
+		} while (time_before(jiffies, end_time));
+		clear_bit(I915_RESET_ENGINE + engine->id,
+			  &i915->gpu_error.flags);
+
+unwind:
+		for_each_engine(active, i915, tmp) {
+			int ret;
+
+			if (!threads[tmp])
+				continue;
+
+			ret = kthread_stop(threads[tmp]);
+			if (ret) {
+				pr_err("kthread for active engine %s failed, err=%d\n",
+				       active->name, ret);
+				if (!err)
+					err = ret;
+			}
+			put_task_struct(threads[tmp]);
+
+			if (resets[tmp] != i915_reset_engine_count(&i915->gpu_error,
+								   active)) {
+				pr_err("Innocent engine %s was reset (count=%ld)\n",
+				       active->name,
+				       i915_reset_engine_count(&i915->gpu_error,
+							       active) - resets[tmp]);
+				err = -EIO;
+			}
+		}
+
+		if (global != i915_reset_count(&i915->gpu_error)) {
+			pr_err("Global reset (count=%ld)!\n",
+			       i915_reset_count(&i915->gpu_error) - global);
+			err = -EIO;
+		}
+
+		if (err)
+			break;
+
+		cond_resched();
+	}
+
 	if (i915_terminally_wedged(&i915->gpu_error))
 		err = -EIO;
 
@@ -356,9 +599,12 @@ static int igt_wait_reset(void *arg)
 	long timeout;
 	int err;
 
+	if (!intel_engine_can_store_dword(i915->engine[RCS]))
+		return 0;
+
 	/* Check that we detect a stuck waiter and issue a reset */
 
-	set_bit(I915_RESET_BACKOFF, &i915->gpu_error.flags);
+	global_reset_lock(i915);
 
 	mutex_lock(&i915->drm.struct_mutex);
 	err = hang_init(&h, i915);
@@ -403,7 +649,7 @@ fini:
 	hang_fini(&h);
 unlock:
 	mutex_unlock(&i915->drm.struct_mutex);
-	clear_bit(I915_RESET_BACKOFF, &i915->gpu_error.flags);
+	global_reset_unlock(i915);
 
 	if (i915_terminally_wedged(&i915->gpu_error))
 		return -EIO;
@@ -421,10 +667,8 @@ static int igt_reset_queue(void *arg)
 
 	/* Check that we replay pending requests following a hang */
 
-	if (!igt_can_mi_store_dword_imm(i915))
-		return 0;
+	global_reset_lock(i915);
 
-	set_bit(I915_RESET_BACKOFF, &i915->gpu_error.flags);
 	mutex_lock(&i915->drm.struct_mutex);
 	err = hang_init(&h, i915);
 	if (err)
@@ -435,6 +679,9 @@ static int igt_reset_queue(void *arg)
 		IGT_TIMEOUT(end_time);
 		unsigned int count;
 
+		if (!intel_engine_can_store_dword(engine))
+			continue;
+
 		prev = hang_create_request(&h, engine, i915->kernel_context);
 		if (IS_ERR(prev)) {
 			err = PTR_ERR(prev);
@@ -471,7 +718,7 @@ static int igt_reset_queue(void *arg)
 
 			reset_count = fake_hangcheck(prev);
 
-			i915_reset(i915);
+			i915_reset(i915, I915_RESET_QUIET);
 
 			GEM_BUG_ON(test_bit(I915_RESET_HANDOFF,
 					    &i915->gpu_error.flags));
@@ -518,7 +765,7 @@ fini:
 	hang_fini(&h);
 unlock:
 	mutex_unlock(&i915->drm.struct_mutex);
-	clear_bit(I915_RESET_BACKOFF, &i915->gpu_error.flags);
+	global_reset_unlock(i915);
 
 	if (i915_terminally_wedged(&i915->gpu_error))
 		return -EIO;
@@ -526,13 +773,83 @@ unlock:
 	return err;
 }
 
+static int igt_handle_error(void *arg)
+{
+	struct drm_i915_private *i915 = arg;
+	struct intel_engine_cs *engine = i915->engine[RCS];
+	struct hang h;
+	struct drm_i915_gem_request *rq;
+	struct i915_gpu_state *error;
+	int err;
+
+	/* Check that we can issue a global GPU and engine reset */
+
+	if (!intel_has_reset_engine(i915))
+		return 0;
+
+	if (!intel_engine_can_store_dword(i915->engine[RCS]))
+		return 0;
+
+	mutex_lock(&i915->drm.struct_mutex);
+
+	err = hang_init(&h, i915);
+	if (err)
+		goto err_unlock;
+
+	rq = hang_create_request(&h, engine, i915->kernel_context);
+	if (IS_ERR(rq)) {
+		err = PTR_ERR(rq);
+		goto err_fini;
+	}
+
+	i915_gem_request_get(rq);
+	__i915_add_request(rq, true);
+
+	if (!wait_for_hang(&h, rq)) {
+		pr_err("Failed to start request %x\n", rq->fence.seqno);
+		err = -EIO;
+		goto err_request;
+	}
+
+	mutex_unlock(&i915->drm.struct_mutex);
+
+	/* Temporarily disable error capture */
+	error = xchg(&i915->gpu_error.first_error, (void *)-1);
+
+	engine->hangcheck.stalled = true;
+	engine->hangcheck.seqno = intel_engine_get_seqno(engine);
+
+	i915_handle_error(i915, intel_engine_flag(engine), "%s", __func__);
+
+	xchg(&i915->gpu_error.first_error, error);
+
+	mutex_lock(&i915->drm.struct_mutex);
+
+	if (rq->fence.error != -EIO) {
+		pr_err("Guilty request not identified!\n");
+		err = -EINVAL;
+		goto err_request;
+	}
+
+err_request:
+	i915_gem_request_put(rq);
+err_fini:
+	hang_fini(&h);
+err_unlock:
+	mutex_unlock(&i915->drm.struct_mutex);
+	return err;
+}
+
 int intel_hangcheck_live_selftests(struct drm_i915_private *i915)
 {
 	static const struct i915_subtest tests[] = {
 		SUBTEST(igt_hang_sanitycheck),
 		SUBTEST(igt_global_reset),
+		SUBTEST(igt_reset_engine),
+		SUBTEST(igt_reset_active_engines),
 		SUBTEST(igt_wait_reset),
 		SUBTEST(igt_reset_queue),
+		SUBTEST(igt_handle_error),
 	};
 
 	if (!intel_has_gpu_reset(i915))
diff --git a/drivers/gpu/drm/i915/selftests/intel_uncore.c b/drivers/gpu/drm/i915/selftests/intel_uncore.c
index 2d0fef2cfca6..3cac22eb47ce 100644
--- a/drivers/gpu/drm/i915/selftests/intel_uncore.c
+++ b/drivers/gpu/drm/i915/selftests/intel_uncore.c
@@ -127,7 +127,7 @@ static int intel_uncore_check_forcewake_domains(struct drm_i915_private *dev_pri
 		return 0;
 
 	valid = kzalloc(BITS_TO_LONGS(FW_RANGE) * sizeof(*valid),
-			GFP_TEMPORARY);
+			GFP_KERNEL);
 	if (!valid)
 		return -ENOMEM;
 
diff --git a/drivers/gpu/drm/i915/selftests/mock_context.c b/drivers/gpu/drm/i915/selftests/mock_context.c
index f8b9cc212b02..098ce643ad07 100644
--- a/drivers/gpu/drm/i915/selftests/mock_context.c
+++ b/drivers/gpu/drm/i915/selftests/mock_context.c
@@ -40,18 +40,13 @@ mock_context(struct drm_i915_private *i915,
 	INIT_LIST_HEAD(&ctx->link);
 	ctx->i915 = i915;
 
-	ctx->vma_lut.ht_bits = VMA_HT_BITS;
-	ctx->vma_lut.ht_size = BIT(VMA_HT_BITS);
-	ctx->vma_lut.ht = kcalloc(ctx->vma_lut.ht_size,
-				  sizeof(*ctx->vma_lut.ht),
-				  GFP_KERNEL);
-	if (!ctx->vma_lut.ht)
-		goto err_free;
-
-	ret = ida_simple_get(&i915->context_hw_ida,
+	INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL);
+	INIT_LIST_HEAD(&ctx->handles_list);
+
+	ret = ida_simple_get(&i915->contexts.hw_ida,
 			     0, MAX_CONTEXT_HW_ID, GFP_KERNEL);
 	if (ret < 0)
-		goto err_vma_ht;
+		goto err_handles;
 	ctx->hw_id = ret;
 
 	if (name) {
@@ -66,9 +61,7 @@ mock_context(struct drm_i915_private *i915,
 
 	return ctx;
 
-err_vma_ht:
-	kvfree(ctx->vma_lut.ht);
-err_free:
+err_handles:
 	kfree(ctx);
 	return NULL;
 
@@ -86,3 +79,20 @@ void mock_context_close(struct i915_gem_context *ctx)
 
 	i915_gem_context_put(ctx);
 }
+
+void mock_init_contexts(struct drm_i915_private *i915)
+{
+	INIT_LIST_HEAD(&i915->contexts.list);
+	ida_init(&i915->contexts.hw_ida);
+
+	INIT_WORK(&i915->contexts.free_work, contexts_free_worker);
+	init_llist_head(&i915->contexts.free_list);
+}
+
+struct i915_gem_context *
+live_context(struct drm_i915_private *i915, struct drm_file *file)
+{
+	lockdep_assert_held(&i915->drm.struct_mutex);
+
+	return i915_gem_create_context(i915, file->driver_priv);
+}
diff --git a/drivers/gpu/drm/i915/selftests/mock_context.h b/drivers/gpu/drm/i915/selftests/mock_context.h
index 2427e5c0916a..2f432c03d413 100644
--- a/drivers/gpu/drm/i915/selftests/mock_context.h
+++ b/drivers/gpu/drm/i915/selftests/mock_context.h
@@ -25,10 +25,15 @@
 #ifndef __MOCK_CONTEXT_H
 #define __MOCK_CONTEXT_H
 
+void mock_init_contexts(struct drm_i915_private *i915);
+
 struct i915_gem_context *
 mock_context(struct drm_i915_private *i915,
 	     const char *name);
 
 void mock_context_close(struct i915_gem_context *ctx);
 
+struct i915_gem_context *
+live_context(struct drm_i915_private *i915, struct drm_file *file);
+
 #endif /* !__MOCK_CONTEXT_H */
diff --git a/drivers/gpu/drm/i915/selftests/mock_engine.c b/drivers/gpu/drm/i915/selftests/mock_engine.c
index 5b18a2dc19a8..fc0fd7498689 100644
--- a/drivers/gpu/drm/i915/selftests/mock_engine.c
+++ b/drivers/gpu/drm/i915/selftests/mock_engine.c
@@ -123,10 +123,12 @@ static struct intel_ring *mock_ring(struct intel_engine_cs *engine)
 }
 
 struct intel_engine_cs *mock_engine(struct drm_i915_private *i915,
-				    const char *name)
+				    const char *name,
+				    int id)
 {
 	struct mock_engine *engine;
-	static int id;
+
+	GEM_BUG_ON(id >= I915_NUM_ENGINES);
 
 	engine = kzalloc(sizeof(*engine) + PAGE_SIZE, GFP_KERNEL);
 	if (!engine)
@@ -141,7 +143,7 @@ struct intel_engine_cs *mock_engine(struct drm_i915_private *i915,
 	/* minimal engine setup for requests */
 	engine->base.i915 = i915;
 	snprintf(engine->base.name, sizeof(engine->base.name), "%s", name);
-	engine->base.id = id++;
+	engine->base.id = id;
 	engine->base.status_page.page_addr = (void *)(engine + 1);
 
 	engine->base.context_pin = mock_context_pin;
diff --git a/drivers/gpu/drm/i915/selftests/mock_engine.h b/drivers/gpu/drm/i915/selftests/mock_engine.h
index e5e240216ba3..133d0c21790d 100644
--- a/drivers/gpu/drm/i915/selftests/mock_engine.h
+++ b/drivers/gpu/drm/i915/selftests/mock_engine.h
@@ -40,7 +40,8 @@ struct mock_engine {
 };
 
 struct intel_engine_cs *mock_engine(struct drm_i915_private *i915,
-				    const char *name);
+				    const char *name,
+				    int id);
 void mock_engine_flush(struct intel_engine_cs *engine);
 void mock_engine_reset(struct intel_engine_cs *engine);
 void mock_engine_free(struct intel_engine_cs *engine);
diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_device.c b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
index 8cdec455cf7d..678723430d78 100644
--- a/drivers/gpu/drm/i915/selftests/mock_gem_device.c
+++ b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
@@ -22,6 +22,7 @@
  *
  */
 
+#include <linux/pm_domain.h>
 #include <linux/pm_runtime.h>
 
 #include "mock_engine.h"
@@ -53,15 +54,17 @@ static void mock_device_release(struct drm_device *dev)
 
 	mutex_lock(&i915->drm.struct_mutex);
 	mock_device_flush(i915);
+	i915_gem_contexts_lost(i915);
 	mutex_unlock(&i915->drm.struct_mutex);
 
 	cancel_delayed_work_sync(&i915->gt.retire_work);
 	cancel_delayed_work_sync(&i915->gt.idle_work);
+	i915_gem_drain_workqueue(i915);
 
 	mutex_lock(&i915->drm.struct_mutex);
 	for_each_engine(engine, i915, id)
 		mock_engine_free(engine);
-	i915_gem_context_fini(i915);
+	i915_gem_contexts_fini(i915);
 	mutex_unlock(&i915->drm.struct_mutex);
 
 	drain_workqueue(i915->wq);
@@ -108,6 +111,23 @@ static void mock_idle_work_handler(struct work_struct *work)
 {
 }
 
+static int pm_domain_resume(struct device *dev)
+{
+	return pm_generic_runtime_resume(dev);
+}
+
+static int pm_domain_suspend(struct device *dev)
+{
+	return pm_generic_runtime_suspend(dev);
+}
+
+static struct dev_pm_domain pm_domain = {
+	.ops = {
+		.runtime_suspend = pm_domain_suspend,
+		.runtime_resume = pm_domain_resume,
+	},
+};
+
 struct drm_i915_private *mock_gem_device(void)
 {
 	struct drm_i915_private *i915;
@@ -126,8 +146,10 @@ struct drm_i915_private *mock_gem_device(void)
 	dev_set_name(&pdev->dev, "mock");
 	dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
 
+	dev_pm_domain_set(&pdev->dev, &pm_domain);
+	pm_runtime_enable(&pdev->dev);
 	pm_runtime_dont_use_autosuspend(&pdev->dev);
-	pm_runtime_get_sync(&pdev->dev);
+	WARN_ON(pm_runtime_get_sync(&pdev->dev));
 
 	i915 = (struct drm_i915_private *)(pdev + 1);
 	pci_set_drvdata(pdev, i915);
@@ -160,7 +182,7 @@ struct drm_i915_private *mock_gem_device(void)
 	INIT_LIST_HEAD(&i915->mm.unbound_list);
 	INIT_LIST_HEAD(&i915->mm.bound_list);
 
-	ida_init(&i915->context_hw_ida);
+	mock_init_contexts(i915);
 
 	INIT_DELAYED_WORK(&i915->gt.retire_work, mock_retire_work_handler);
 	INIT_DELAYED_WORK(&i915->gt.idle_work, mock_idle_work_handler);
@@ -204,7 +226,7 @@ struct drm_i915_private *mock_gem_device(void)
 	mutex_unlock(&i915->drm.struct_mutex);
 
 	mkwrite_device_info(i915)->ring_mask = BIT(0);
-	i915->engine[RCS] = mock_engine(i915, "mock");
+	i915->engine[RCS] = mock_engine(i915, "mock", RCS);
 	if (!i915->engine[RCS])
 		goto err_priorities;
 
diff --git a/drivers/gpu/drm/i915/selftests/mock_gtt.c b/drivers/gpu/drm/i915/selftests/mock_gtt.c
index a61309c7cb3e..f2118cf535a0 100644
--- a/drivers/gpu/drm/i915/selftests/mock_gtt.c
+++ b/drivers/gpu/drm/i915/selftests/mock_gtt.c
@@ -33,8 +33,7 @@ static void mock_insert_page(struct i915_address_space *vm,
 }
 
 static void mock_insert_entries(struct i915_address_space *vm,
-				struct sg_table *st,
-				u64 start,
+				struct i915_vma *vma,
 				enum i915_cache_level level, u32 flags)
 {
 }
diff --git a/drivers/gpu/drm/imx/imx-drm-core.c b/drivers/gpu/drm/imx/imx-drm-core.c
index 95e2181963d9..f91cb72d0830 100644
--- a/drivers/gpu/drm/imx/imx-drm-core.c
+++ b/drivers/gpu/drm/imx/imx-drm-core.c
@@ -115,7 +115,7 @@ static void imx_drm_atomic_commit_tail(struct drm_atomic_state *state)
 {
 	struct drm_device *dev = state->dev;
 	struct drm_plane *plane;
-	struct drm_plane_state *old_plane_state;
+	struct drm_plane_state *old_plane_state, *new_plane_state;
 	bool plane_disabling = false;
 	int i;
 
@@ -127,15 +127,15 @@ static void imx_drm_atomic_commit_tail(struct drm_atomic_state *state)
 
 	drm_atomic_helper_commit_modeset_enables(dev, state);
 
-	for_each_plane_in_state(state, plane, old_plane_state, i) {
-		if (drm_atomic_plane_disabling(old_plane_state, plane->state))
+	for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i) {
+		if (drm_atomic_plane_disabling(old_plane_state, new_plane_state))
 			plane_disabling = true;
 	}
 
 	if (plane_disabling) {
 		drm_atomic_helper_wait_for_vblanks(dev, state);
 
-		for_each_plane_in_state(state, plane, old_plane_state, i)
+		for_each_old_plane_in_state(state, plane, old_plane_state, i)
 			ipu_plane_disable_deferred(plane);
 
 	}
@@ -182,8 +182,6 @@ static struct drm_driver imx_drm_driver = {
 	.gem_free_object_unlocked = drm_gem_cma_free_object,
 	.gem_vm_ops		= &drm_gem_cma_vm_ops,
 	.dumb_create		= drm_gem_cma_dumb_create,
-	.dumb_map_offset	= drm_gem_cma_dumb_map_offset,
-	.dumb_destroy		= drm_gem_dumb_destroy,
 
 	.prime_handle_to_fd	= drm_gem_prime_handle_to_fd,
 	.prime_fd_to_handle	= drm_gem_prime_fd_to_handle,
diff --git a/drivers/gpu/drm/imx/imx-ldb.c b/drivers/gpu/drm/imx/imx-ldb.c
index 8b05ecb8fdef..56dd7a9a8e25 100644
--- a/drivers/gpu/drm/imx/imx-ldb.c
+++ b/drivers/gpu/drm/imx/imx-ldb.c
@@ -389,7 +389,6 @@ static int imx_ldb_encoder_atomic_check(struct drm_encoder *encoder,
 
 
 static const struct drm_connector_funcs imx_ldb_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = imx_drm_connector_destroy,
 	.reset = drm_atomic_helper_connector_reset,
diff --git a/drivers/gpu/drm/imx/imx-tve.c b/drivers/gpu/drm/imx/imx-tve.c
index 4826bb781723..bc27c2699464 100644
--- a/drivers/gpu/drm/imx/imx-tve.c
+++ b/drivers/gpu/drm/imx/imx-tve.c
@@ -341,7 +341,6 @@ static int imx_tve_atomic_check(struct drm_encoder *encoder,
 }
 
 static const struct drm_connector_funcs imx_tve_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = imx_drm_connector_destroy,
 	.reset = drm_atomic_helper_connector_reset,
diff --git a/drivers/gpu/drm/imx/ipuv3-crtc.c b/drivers/gpu/drm/imx/ipuv3-crtc.c
index 5456c15d962c..53e0b24beda6 100644
--- a/drivers/gpu/drm/imx/ipuv3-crtc.c
+++ b/drivers/gpu/drm/imx/ipuv3-crtc.c
@@ -50,7 +50,8 @@ static inline struct ipu_crtc *to_ipu_crtc(struct drm_crtc *crtc)
 	return container_of(crtc, struct ipu_crtc, base);
 }
 
-static void ipu_crtc_enable(struct drm_crtc *crtc)
+static void ipu_crtc_atomic_enable(struct drm_crtc *crtc,
+				   struct drm_crtc_state *old_state)
 {
 	struct ipu_crtc *ipu_crtc = to_ipu_crtc(crtc);
 	struct ipu_soc *ipu = dev_get_drvdata(ipu_crtc->dev->parent);
@@ -293,7 +294,7 @@ static const struct drm_crtc_helper_funcs ipu_helper_funcs = {
 	.atomic_check = ipu_crtc_atomic_check,
 	.atomic_begin = ipu_crtc_atomic_begin,
 	.atomic_disable = ipu_crtc_atomic_disable,
-	.enable = ipu_crtc_enable,
+	.atomic_enable = ipu_crtc_atomic_enable,
 };
 
 static void ipu_put_resources(struct ipu_crtc *ipu_crtc)
diff --git a/drivers/gpu/drm/imx/ipuv3-plane.c b/drivers/gpu/drm/imx/ipuv3-plane.c
index 6276bb834b4f..cf98596c7ce1 100644
--- a/drivers/gpu/drm/imx/ipuv3-plane.c
+++ b/drivers/gpu/drm/imx/ipuv3-plane.c
@@ -496,6 +496,27 @@ static int ipu_chan_assign_axi_id(int ipu_chan)
 	}
 }
 
+static void ipu_calculate_bursts(u32 width, u32 cpp, u32 stride,
+				 u8 *burstsize, u8 *num_bursts)
+{
+	const unsigned int width_bytes = width * cpp;
+	unsigned int npb, bursts;
+
+	/* Maximum number of pixels per burst without overshooting stride */
+	for (npb = 64 / cpp; npb > 0; --npb) {
+		if (round_up(width_bytes, npb * cpp) <= stride)
+			break;
+	}
+	*burstsize = npb;
+
+	/* Maximum number of consecutive bursts without overshooting stride */
+	for (bursts = 8; bursts > 1; bursts /= 2) {
+		if (round_up(width_bytes, npb * cpp * bursts) <= stride)
+			break;
+	}
+	*num_bursts = bursts;
+}
+
 static void ipu_plane_atomic_update(struct drm_plane *plane,
 				    struct drm_plane_state *old_state)
 {
@@ -509,6 +530,9 @@ static void ipu_plane_atomic_update(struct drm_plane *plane,
 	unsigned long alpha_eba = 0;
 	enum ipu_color_space ics;
 	unsigned int axi_id = 0;
+	const struct drm_format_info *info;
+	u8 burstsize, num_bursts;
+	u32 width, height;
 	int active;
 
 	if (ipu_plane->dp_flow == IPU_DP_FLOW_SYNC_FG)
@@ -525,8 +549,8 @@ static void ipu_plane_atomic_update(struct drm_plane *plane,
 		ipu_prg_channel_configure(ipu_plane->ipu_ch, axi_id,
 					  drm_rect_width(&state->src) >> 16,
 					  drm_rect_height(&state->src) >> 16,
-					  state->fb->pitches[0],
-					  state->fb->format->format, &eba);
+					  fb->pitches[0],
+					  fb->format->format, &eba);
 	}
 
 	if (old_state->fb && !drm_atomic_crtc_needs_modeset(crtc_state)) {
@@ -545,19 +569,17 @@ static void ipu_plane_atomic_update(struct drm_plane *plane,
 		return;
 	}
 
+	ics = ipu_drm_fourcc_to_colorspace(fb->format->format);
 	switch (ipu_plane->dp_flow) {
 	case IPU_DP_FLOW_SYNC_BG:
-		ipu_dp_setup_channel(ipu_plane->dp,
-					IPUV3_COLORSPACE_RGB,
-					IPUV3_COLORSPACE_RGB);
+		ipu_dp_setup_channel(ipu_plane->dp, ics, IPUV3_COLORSPACE_RGB);
 		ipu_dp_set_global_alpha(ipu_plane->dp, true, 0, true);
 		break;
 	case IPU_DP_FLOW_SYNC_FG:
-		ics = ipu_drm_fourcc_to_colorspace(state->fb->format->format);
 		ipu_dp_setup_channel(ipu_plane->dp, ics,
 					IPUV3_COLORSPACE_UNKNOWN);
 		/* Enable local alpha on partial plane */
-		switch (state->fb->format->format) {
+		switch (fb->format->format) {
 		case DRM_FORMAT_ARGB1555:
 		case DRM_FORMAT_ABGR1555:
 		case DRM_FORMAT_RGBA5551:
@@ -583,15 +605,21 @@ static void ipu_plane_atomic_update(struct drm_plane *plane,
 
 	ipu_dmfc_config_wait4eot(ipu_plane->dmfc, drm_rect_width(dst));
 
+	width = drm_rect_width(&state->src) >> 16;
+	height = drm_rect_height(&state->src) >> 16;
+	info = drm_format_info(fb->format->format);
+	ipu_calculate_bursts(width, info->cpp[0], fb->pitches[0],
+			     &burstsize, &num_bursts);
+
 	ipu_cpmem_zero(ipu_plane->ipu_ch);
-	ipu_cpmem_set_resolution(ipu_plane->ipu_ch,
-				 drm_rect_width(&state->src) >> 16,
-				 drm_rect_height(&state->src) >> 16);
-	ipu_cpmem_set_fmt(ipu_plane->ipu_ch, state->fb->format->format);
+	ipu_cpmem_set_resolution(ipu_plane->ipu_ch, width, height);
+	ipu_cpmem_set_fmt(ipu_plane->ipu_ch, fb->format->format);
+	ipu_cpmem_set_burstsize(ipu_plane->ipu_ch, burstsize);
 	ipu_cpmem_set_high_priority(ipu_plane->ipu_ch);
 	ipu_idmac_set_double_buffer(ipu_plane->ipu_ch, 1);
-	ipu_cpmem_set_stride(ipu_plane->ipu_ch, state->fb->pitches[0]);
+	ipu_cpmem_set_stride(ipu_plane->ipu_ch, fb->pitches[0]);
 	ipu_cpmem_set_axi_id(ipu_plane->ipu_ch, axi_id);
+
 	switch (fb->format->format) {
 	case DRM_FORMAT_YUV420:
 	case DRM_FORMAT_YVU420:
@@ -631,6 +659,7 @@ static void ipu_plane_atomic_update(struct drm_plane *plane,
 	case DRM_FORMAT_RGBX8888_A8:
 	case DRM_FORMAT_BGRX8888_A8:
 		alpha_eba = drm_plane_state_to_eba(state, 1);
+		num_bursts = 0;
 
 		dev_dbg(ipu_plane->base.dev->dev, "phys = %lu %lu, x = %d, y = %d",
 			eba, alpha_eba, state->src.x1 >> 16, state->src.y1 >> 16);
@@ -644,8 +673,7 @@ static void ipu_plane_atomic_update(struct drm_plane *plane,
 		ipu_cpmem_set_format_passthrough(ipu_plane->alpha_ch, 8);
 		ipu_cpmem_set_high_priority(ipu_plane->alpha_ch);
 		ipu_idmac_set_double_buffer(ipu_plane->alpha_ch, 1);
-		ipu_cpmem_set_stride(ipu_plane->alpha_ch,
-				     state->fb->pitches[1]);
+		ipu_cpmem_set_stride(ipu_plane->alpha_ch, fb->pitches[1]);
 		ipu_cpmem_set_burstsize(ipu_plane->alpha_ch, 16);
 		ipu_cpmem_set_buffer(ipu_plane->alpha_ch, 0, alpha_eba);
 		ipu_cpmem_set_buffer(ipu_plane->alpha_ch, 1, alpha_eba);
@@ -657,6 +685,7 @@ static void ipu_plane_atomic_update(struct drm_plane *plane,
 	}
 	ipu_cpmem_set_buffer(ipu_plane->ipu_ch, 0, eba);
 	ipu_cpmem_set_buffer(ipu_plane->ipu_ch, 1, eba);
+	ipu_idmac_lock_enable(ipu_plane->ipu_ch, num_bursts);
 	ipu_plane_enable(ipu_plane);
 }
 
@@ -675,7 +704,7 @@ int ipu_planes_assign_pre(struct drm_device *dev,
 	int available_pres = ipu_prg_max_active_channels();
 	int i;
 
-	for_each_plane_in_state(state, plane, plane_state, i) {
+	for_each_new_plane_in_state(state, plane, plane_state, i) {
 		struct ipu_plane_state *ipu_state =
 				to_ipu_plane_state(plane_state);
 		struct ipu_plane *ipu_plane = to_ipu_plane(plane);
@@ -718,8 +747,8 @@ struct ipu_plane *ipu_plane_init(struct drm_device *dev, struct ipu_soc *ipu,
 
 	ret = drm_universal_plane_init(dev, &ipu_plane->base, possible_crtcs,
 				       &ipu_plane_funcs, ipu_plane_formats,
-				       ARRAY_SIZE(ipu_plane_formats), type,
-				       NULL);
+				       ARRAY_SIZE(ipu_plane_formats),
+				       NULL, type, NULL);
 	if (ret) {
 		DRM_ERROR("failed to initialize plane\n");
 		kfree(ipu_plane);
diff --git a/drivers/gpu/drm/imx/parallel-display.c b/drivers/gpu/drm/imx/parallel-display.c
index 8aca20209cb8..8def97d75030 100644
--- a/drivers/gpu/drm/imx/parallel-display.c
+++ b/drivers/gpu/drm/imx/parallel-display.c
@@ -135,7 +135,6 @@ static int imx_pd_encoder_atomic_check(struct drm_encoder *encoder,
 }
 
 static const struct drm_connector_funcs imx_pd_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = imx_drm_connector_destroy,
 	.reset = drm_atomic_helper_connector_reset,
diff --git a/drivers/gpu/drm/lib/drm_random.c b/drivers/gpu/drm/lib/drm_random.c
index 7b12a68c3b54..a78c4b483e8d 100644
--- a/drivers/gpu/drm/lib/drm_random.c
+++ b/drivers/gpu/drm/lib/drm_random.c
@@ -28,7 +28,7 @@ unsigned int *drm_random_order(unsigned int count, struct rnd_state *state)
 {
 	unsigned int *order, i;
 
-	order = kmalloc_array(count, sizeof(*order), GFP_TEMPORARY);
+	order = kmalloc_array(count, sizeof(*order), GFP_KERNEL);
 	if (!order)
 		return order;
 
diff --git a/drivers/gpu/drm/mediatek/mtk_disp_color.c b/drivers/gpu/drm/mediatek/mtk_disp_color.c
index ef79a6d55646..f609b62b8be6 100644
--- a/drivers/gpu/drm/mediatek/mtk_disp_color.c
+++ b/drivers/gpu/drm/mediatek/mtk_disp_color.c
@@ -84,8 +84,8 @@ static int mtk_disp_color_bind(struct device *dev, struct device *master,
 
 	ret = mtk_ddp_comp_register(drm_dev, &priv->ddp_comp);
 	if (ret < 0) {
-		dev_err(dev, "Failed to register component %s: %d\n",
-			dev->of_node->full_name, ret);
+		dev_err(dev, "Failed to register component %pOF: %d\n",
+			dev->of_node, ret);
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/mediatek/mtk_disp_ovl.c b/drivers/gpu/drm/mediatek/mtk_disp_ovl.c
index 35bc5babdbf7..978782a77629 100644
--- a/drivers/gpu/drm/mediatek/mtk_disp_ovl.c
+++ b/drivers/gpu/drm/mediatek/mtk_disp_ovl.c
@@ -235,8 +235,8 @@ static int mtk_disp_ovl_bind(struct device *dev, struct device *master,
 
 	ret = mtk_ddp_comp_register(drm_dev, &priv->ddp_comp);
 	if (ret < 0) {
-		dev_err(dev, "Failed to register component %s: %d\n",
-			dev->of_node->full_name, ret);
+		dev_err(dev, "Failed to register component %pOF: %d\n",
+			dev->of_node, ret);
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/mediatek/mtk_disp_rdma.c b/drivers/gpu/drm/mediatek/mtk_disp_rdma.c
index b68a51376f83..585943c81e1f 100644
--- a/drivers/gpu/drm/mediatek/mtk_disp_rdma.c
+++ b/drivers/gpu/drm/mediatek/mtk_disp_rdma.c
@@ -155,8 +155,8 @@ static int mtk_disp_rdma_bind(struct device *dev, struct device *master,
 
 	ret = mtk_ddp_comp_register(drm_dev, &priv->ddp_comp);
 	if (ret < 0) {
-		dev_err(dev, "Failed to register component %s: %d\n",
-			dev->of_node->full_name, ret);
+		dev_err(dev, "Failed to register component %pOF: %d\n",
+			dev->of_node, ret);
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/mediatek/mtk_dpi.c b/drivers/gpu/drm/mediatek/mtk_dpi.c
index 32ca351ecd09..e80a603e5fb0 100644
--- a/drivers/gpu/drm/mediatek/mtk_dpi.c
+++ b/drivers/gpu/drm/mediatek/mtk_dpi.c
@@ -605,8 +605,8 @@ static int mtk_dpi_bind(struct device *dev, struct device *master, void *data)
 
 	ret = mtk_ddp_comp_register(drm_dev, &dpi->ddp_comp);
 	if (ret < 0) {
-		dev_err(dev, "Failed to register component %s: %d\n",
-			dev->of_node->full_name, ret);
+		dev_err(dev, "Failed to register component %pOF: %d\n",
+			dev->of_node, ret);
 		return ret;
 	}
 
@@ -710,7 +710,7 @@ static int mtk_dpi_probe(struct platform_device *pdev)
 	if (!bridge_node)
 		return -ENODEV;
 
-	dev_info(dev, "Found bridge node: %s\n", bridge_node->full_name);
+	dev_info(dev, "Found bridge node: %pOF\n", bridge_node);
 
 	dpi->bridge = of_drm_find_bridge(bridge_node);
 	of_node_put(bridge_node);
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_crtc.c b/drivers/gpu/drm/mediatek/mtk_drm_crtc.c
index cb32c9369f3a..658b8dd45b83 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_crtc.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_crtc.c
@@ -366,7 +366,8 @@ static void mtk_crtc_ddp_config(struct drm_crtc *crtc)
 	}
 }
 
-static void mtk_drm_crtc_enable(struct drm_crtc *crtc)
+static void mtk_drm_crtc_atomic_enable(struct drm_crtc *crtc,
+				       struct drm_crtc_state *old_state)
 {
 	struct mtk_drm_crtc *mtk_crtc = to_mtk_crtc(crtc);
 	struct mtk_ddp_comp *ovl = mtk_crtc->ddp_comp[0];
@@ -390,7 +391,8 @@ static void mtk_drm_crtc_enable(struct drm_crtc *crtc)
 	mtk_crtc->enabled = true;
 }
 
-static void mtk_drm_crtc_disable(struct drm_crtc *crtc)
+static void mtk_drm_crtc_atomic_disable(struct drm_crtc *crtc,
+					struct drm_crtc_state *old_state)
 {
 	struct mtk_drm_crtc *mtk_crtc = to_mtk_crtc(crtc);
 	struct mtk_ddp_comp *ovl = mtk_crtc->ddp_comp[0];
@@ -487,10 +489,10 @@ static const struct drm_crtc_funcs mtk_crtc_funcs = {
 static const struct drm_crtc_helper_funcs mtk_crtc_helper_funcs = {
 	.mode_fixup	= mtk_drm_crtc_mode_fixup,
 	.mode_set_nofb	= mtk_drm_crtc_mode_set_nofb,
-	.enable		= mtk_drm_crtc_enable,
-	.disable	= mtk_drm_crtc_disable,
 	.atomic_begin	= mtk_drm_crtc_atomic_begin,
 	.atomic_flush	= mtk_drm_crtc_atomic_flush,
+	.atomic_enable	= mtk_drm_crtc_atomic_enable,
+	.atomic_disable	= mtk_drm_crtc_atomic_disable,
 };
 
 static int mtk_drm_crtc_init(struct drm_device *drm,
@@ -577,8 +579,7 @@ int mtk_drm_crtc_create(struct drm_device *drm_dev,
 		node = priv->comp_node[comp_id];
 		comp = priv->ddp_comp[comp_id];
 		if (!comp) {
-			dev_err(dev, "Component %s not initialized\n",
-				node->full_name);
+			dev_err(dev, "Component %pOF not initialized\n", node);
 			ret = -ENODEV;
 			goto unprepare;
 		}
@@ -586,8 +587,8 @@ int mtk_drm_crtc_create(struct drm_device *drm_dev,
 		ret = clk_prepare(comp->clk);
 		if (ret) {
 			dev_err(dev,
-				"Failed to prepare clock for component %s: %d\n",
-				node->full_name, ret);
+				"Failed to prepare clock for component %pOF: %d\n",
+				node, ret);
 			goto unprepare;
 		}
 
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c b/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c
index 07d7ea2268ef..4672317e3ad1 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c
@@ -295,15 +295,13 @@ int mtk_ddp_comp_init(struct device *dev, struct device_node *node,
 	larb_node = of_parse_phandle(node, "mediatek,larb", 0);
 	if (!larb_node) {
 		dev_err(dev,
-			"Missing mediadek,larb phandle in %s node\n",
-			node->full_name);
+			"Missing mediadek,larb phandle in %pOF node\n", node);
 		return -EINVAL;
 	}
 
 	larb_pdev = of_find_device_by_node(larb_node);
 	if (!larb_pdev) {
-		dev_warn(dev, "Waiting for larb device %s\n",
-			 larb_node->full_name);
+		dev_warn(dev, "Waiting for larb device %pOF\n", larb_node);
 		of_node_put(larb_node);
 		return -EPROBE_DEFER;
 	}
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_drv.c b/drivers/gpu/drm/mediatek/mtk_drm_drv.c
index 41d2cffe953e..a2ca90fc403c 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_drv.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_drv.c
@@ -48,11 +48,11 @@ static void mtk_atomic_schedule(struct mtk_drm_private *private,
 static void mtk_atomic_wait_for_fences(struct drm_atomic_state *state)
 {
 	struct drm_plane *plane;
-	struct drm_plane_state *plane_state;
+	struct drm_plane_state *new_plane_state;
 	int i;
 
-	for_each_plane_in_state(state, plane, plane_state, i)
-		mtk_fb_wait(plane->state->fb);
+	for_each_new_plane_in_state(state, plane, new_plane_state, i)
+		mtk_fb_wait(new_plane_state->fb);
 }
 
 static void mtk_atomic_complete(struct mtk_drm_private *private,
@@ -109,7 +109,12 @@ static int mtk_atomic_commit(struct drm_device *drm,
 	mutex_lock(&private->commit.lock);
 	flush_work(&private->commit.work);
 
-	drm_atomic_helper_swap_state(state, true);
+	ret = drm_atomic_helper_swap_state(state, true);
+	if (ret) {
+		mutex_unlock(&private->commit.lock);
+		drm_atomic_helper_cleanup_planes(drm, state);
+		return ret;
+	}
 
 	drm_atomic_state_get(state);
 	if (async)
@@ -187,8 +192,8 @@ static int mtk_drm_kms_init(struct drm_device *drm)
 
 	pdev = of_find_device_by_node(private->mutex_node);
 	if (!pdev) {
-		dev_err(drm->dev, "Waiting for disp-mutex device %s\n",
-			private->mutex_node->full_name);
+		dev_err(drm->dev, "Waiting for disp-mutex device %pOF\n",
+			private->mutex_node);
 		of_node_put(private->mutex_node);
 		return -EPROBE_DEFER;
 	}
@@ -266,7 +271,6 @@ static void mtk_drm_kms_deinit(struct drm_device *drm)
 {
 	drm_kms_helper_poll_fini(drm);
 
-	drm_vblank_cleanup(drm);
 	component_unbind_all(drm->dev, drm);
 	drm_mode_config_cleanup(drm);
 }
@@ -289,8 +293,6 @@ static struct drm_driver mtk_drm_driver = {
 	.gem_free_object_unlocked = mtk_drm_gem_free_object,
 	.gem_vm_ops = &drm_gem_cma_vm_ops,
 	.dumb_create = mtk_drm_gem_dumb_create,
-	.dumb_map_offset = mtk_drm_gem_dumb_map_offset,
-	.dumb_destroy = drm_gem_dumb_destroy,
 
 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
 	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
@@ -417,8 +419,8 @@ static int mtk_drm_probe(struct platform_device *pdev)
 			continue;
 
 		if (!of_device_is_available(node)) {
-			dev_dbg(dev, "Skipping disabled component %s\n",
-				node->full_name);
+			dev_dbg(dev, "Skipping disabled component %pOF\n",
+				node);
 			continue;
 		}
 
@@ -431,8 +433,8 @@ static int mtk_drm_probe(struct platform_device *pdev)
 
 		comp_id = mtk_ddp_comp_get_id(node, comp_type);
 		if (comp_id < 0) {
-			dev_warn(dev, "Skipping unknown component %s\n",
-				 node->full_name);
+			dev_warn(dev, "Skipping unknown component %pOF\n",
+				 node);
 			continue;
 		}
 
@@ -448,8 +450,8 @@ static int mtk_drm_probe(struct platform_device *pdev)
 		    comp_type == MTK_DISP_RDMA ||
 		    comp_type == MTK_DSI ||
 		    comp_type == MTK_DPI) {
-			dev_info(dev, "Adding component match for %s\n",
-				 node->full_name);
+			dev_info(dev, "Adding component match for %pOF\n",
+				 node);
 			drm_of_component_match_add(dev, &match, compare_of,
 						   node);
 		} else {
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_fb.c b/drivers/gpu/drm/mediatek/mtk_drm_fb.c
index d4246c9dceae..0d8d506695f9 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_fb.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_fb.c
@@ -58,7 +58,7 @@ static void mtk_drm_fb_destroy(struct drm_framebuffer *fb)
 
 	drm_framebuffer_cleanup(fb);
 
-	drm_gem_object_unreference_unlocked(mtk_fb->gem_obj);
+	drm_gem_object_put_unlocked(mtk_fb->gem_obj);
 
 	kfree(mtk_fb);
 }
@@ -160,6 +160,6 @@ struct drm_framebuffer *mtk_drm_mode_fb_create(struct drm_device *dev,
 	return &mtk_fb->base;
 
 unreference:
-	drm_gem_object_unreference_unlocked(gem);
+	drm_gem_object_put_unlocked(gem);
 	return ERR_PTR(ret);
 }
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_gem.c b/drivers/gpu/drm/mediatek/mtk_drm_gem.c
index 7abc550ebc00..f595ac816b55 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_gem.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_gem.c
@@ -122,7 +122,7 @@ int mtk_drm_gem_dumb_create(struct drm_file *file_priv, struct drm_device *dev,
 		goto err_handle_create;
 
 	/* drop reference from allocate - handle holds it now. */
-	drm_gem_object_unreference_unlocked(&mtk_gem->base);
+	drm_gem_object_put_unlocked(&mtk_gem->base);
 
 	return 0;
 
@@ -131,31 +131,6 @@ err_handle_create:
 	return ret;
 }
 
-int mtk_drm_gem_dumb_map_offset(struct drm_file *file_priv,
-				struct drm_device *dev, uint32_t handle,
-				uint64_t *offset)
-{
-	struct drm_gem_object *obj;
-	int ret;
-
-	obj = drm_gem_object_lookup(file_priv, handle);
-	if (!obj) {
-		DRM_ERROR("failed to lookup gem object.\n");
-		return -EINVAL;
-	}
-
-	ret = drm_gem_create_mmap_offset(obj);
-	if (ret)
-		goto out;
-
-	*offset = drm_vma_node_offset_addr(&obj->vma_node);
-	DRM_DEBUG_KMS("offset = 0x%llx\n", *offset);
-
-out:
-	drm_gem_object_unreference_unlocked(obj);
-	return ret;
-}
-
 static int mtk_drm_gem_object_mmap(struct drm_gem_object *obj,
 				   struct vm_area_struct *vma)
 
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_gem.h b/drivers/gpu/drm/mediatek/mtk_drm_gem.h
index 2752718fa5b2..534639b43a1c 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_gem.h
+++ b/drivers/gpu/drm/mediatek/mtk_drm_gem.h
@@ -46,9 +46,6 @@ struct mtk_drm_gem_obj *mtk_drm_gem_create(struct drm_device *dev, size_t size,
 					   bool alloc_kmap);
 int mtk_drm_gem_dumb_create(struct drm_file *file_priv, struct drm_device *dev,
 			    struct drm_mode_create_dumb *args);
-int mtk_drm_gem_dumb_map_offset(struct drm_file *file_priv,
-				struct drm_device *dev, uint32_t handle,
-				uint64_t *offset);
 int mtk_drm_gem_mmap(struct file *filp, struct vm_area_struct *vma);
 int mtk_drm_gem_mmap_buf(struct drm_gem_object *obj,
 			 struct vm_area_struct *vma);
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_plane.c b/drivers/gpu/drm/mediatek/mtk_drm_plane.c
index 1a59b9ab4aa8..6f121891430f 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_plane.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_plane.c
@@ -175,7 +175,7 @@ int mtk_plane_init(struct drm_device *dev, struct drm_plane *plane,
 
 	err = drm_universal_plane_init(dev, plane, possible_crtcs,
 				       &mtk_plane_funcs, formats,
-				       ARRAY_SIZE(formats), type, NULL);
+				       ARRAY_SIZE(formats), NULL, type, NULL);
 	if (err) {
 		DRM_ERROR("failed to initialize plane\n");
 		return err;
diff --git a/drivers/gpu/drm/mediatek/mtk_dsi.c b/drivers/gpu/drm/mediatek/mtk_dsi.c
index 97253c8f813b..7e5e24c2152a 100644
--- a/drivers/gpu/drm/mediatek/mtk_dsi.c
+++ b/drivers/gpu/drm/mediatek/mtk_dsi.c
@@ -766,7 +766,6 @@ static const struct drm_encoder_helper_funcs mtk_dsi_encoder_helper_funcs = {
 };
 
 static const struct drm_connector_funcs mtk_dsi_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = drm_connector_cleanup,
 	.reset = drm_atomic_helper_connector_reset,
@@ -1048,8 +1047,8 @@ static int mtk_dsi_bind(struct device *dev, struct device *master, void *data)
 
 	ret = mtk_ddp_comp_register(drm, &dsi->ddp_comp);
 	if (ret < 0) {
-		dev_err(dev, "Failed to register component %s: %d\n",
-			dev->of_node->full_name, ret);
+		dev_err(dev, "Failed to register component %pOF: %d\n",
+			dev->of_node, ret);
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi.c b/drivers/gpu/drm/mediatek/mtk_hdmi.c
index 71eb4fbbfc85..690c67507cbc 100644
--- a/drivers/gpu/drm/mediatek/mtk_hdmi.c
+++ b/drivers/gpu/drm/mediatek/mtk_hdmi.c
@@ -975,7 +975,7 @@ static int mtk_hdmi_setup_avi_infoframe(struct mtk_hdmi *hdmi,
 	u8 buffer[17];
 	ssize_t err;
 
-	err = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode);
+	err = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode, false);
 	if (err < 0) {
 		dev_err(hdmi->dev,
 			"Failed to get AVI infoframe from mode: %zd\n", err);
@@ -1261,7 +1261,6 @@ static struct drm_encoder *mtk_hdmi_conn_best_enc(struct drm_connector *conn)
 }
 
 static const struct drm_connector_funcs mtk_hdmi_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.detect = hdmi_conn_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = hdmi_conn_destroy,
@@ -1456,8 +1455,8 @@ static int mtk_hdmi_dt_parse_pdata(struct mtk_hdmi *hdmi,
 
 	cec_pdev = of_find_device_by_node(cec_np);
 	if (!cec_pdev) {
-		dev_err(hdmi->dev, "Waiting for CEC device %s\n",
-			cec_np->full_name);
+		dev_err(hdmi->dev, "Waiting for CEC device %pOF\n",
+			cec_np);
 		return -EPROBE_DEFER;
 	}
 	hdmi->cec_dev = &cec_pdev->dev;
@@ -1501,8 +1500,8 @@ static int mtk_hdmi_dt_parse_pdata(struct mtk_hdmi *hdmi,
 
 	i2c_np = of_parse_phandle(remote, "ddc-i2c-bus", 0);
 	if (!i2c_np) {
-		dev_err(dev, "Failed to find ddc-i2c-bus node in %s\n",
-			remote->full_name);
+		dev_err(dev, "Failed to find ddc-i2c-bus node in %pOF\n",
+			remote);
 		of_node_put(remote);
 		return -EINVAL;
 	}
diff --git a/drivers/gpu/drm/meson/meson_crtc.c b/drivers/gpu/drm/meson/meson_crtc.c
index c986eb03b9d9..5155f0179b61 100644
--- a/drivers/gpu/drm/meson/meson_crtc.c
+++ b/drivers/gpu/drm/meson/meson_crtc.c
@@ -79,7 +79,8 @@ static const struct drm_crtc_funcs meson_crtc_funcs = {
 
 };
 
-static void meson_crtc_enable(struct drm_crtc *crtc)
+static void meson_crtc_atomic_enable(struct drm_crtc *crtc,
+				     struct drm_crtc_state *old_state)
 {
 	struct meson_crtc *meson_crtc = to_meson_crtc(crtc);
 	struct drm_crtc_state *crtc_state = crtc->state;
@@ -102,7 +103,8 @@ static void meson_crtc_enable(struct drm_crtc *crtc)
 	priv->viu.osd1_enabled = true;
 }
 
-static void meson_crtc_disable(struct drm_crtc *crtc)
+static void meson_crtc_atomic_disable(struct drm_crtc *crtc,
+				      struct drm_crtc_state *old_state)
 {
 	struct meson_crtc *meson_crtc = to_meson_crtc(crtc);
 	struct meson_drm *priv = meson_crtc->priv;
@@ -149,10 +151,10 @@ static void meson_crtc_atomic_flush(struct drm_crtc *crtc,
 }
 
 static const struct drm_crtc_helper_funcs meson_crtc_helper_funcs = {
-	.enable		= meson_crtc_enable,
-	.disable	= meson_crtc_disable,
 	.atomic_begin	= meson_crtc_atomic_begin,
 	.atomic_flush	= meson_crtc_atomic_flush,
+	.atomic_enable	= meson_crtc_atomic_enable,
+	.atomic_disable	= meson_crtc_atomic_disable,
 };
 
 void meson_crtc_irq(struct meson_drm *priv)
diff --git a/drivers/gpu/drm/meson/meson_drv.c b/drivers/gpu/drm/meson/meson_drv.c
index 4d98fac92795..7742c7d81ed8 100644
--- a/drivers/gpu/drm/meson/meson_drv.c
+++ b/drivers/gpu/drm/meson/meson_drv.c
@@ -116,8 +116,6 @@ static struct drm_driver meson_driver = {
 
 	/* GEM Ops */
 	.dumb_create		= drm_gem_cma_dumb_create,
-	.dumb_destroy		= drm_gem_dumb_destroy,
-	.dumb_map_offset	= drm_gem_cma_dumb_map_offset,
 	.gem_free_object_unlocked = drm_gem_cma_free_object,
 	.gem_vm_ops		= &drm_gem_cma_vm_ops,
 
@@ -303,9 +301,8 @@ static const struct component_master_ops meson_drv_master_ops = {
 
 static int compare_of(struct device *dev, void *data)
 {
-	DRM_DEBUG_DRIVER("Comparing of node %s with %s\n",
-			 of_node_full_name(dev->of_node),
-			 of_node_full_name(data));
+	DRM_DEBUG_DRIVER("Comparing of node %pOF with %pOF\n",
+			 dev->of_node, data);
 
 	return dev->of_node == data;
 }
diff --git a/drivers/gpu/drm/meson/meson_plane.c b/drivers/gpu/drm/meson/meson_plane.c
index a32d3b6e2e12..17e96fa47868 100644
--- a/drivers/gpu/drm/meson/meson_plane.c
+++ b/drivers/gpu/drm/meson/meson_plane.c
@@ -223,6 +223,7 @@ int meson_plane_create(struct meson_drm *priv)
 				 &meson_plane_funcs,
 				 supported_drm_formats,
 				 ARRAY_SIZE(supported_drm_formats),
+				 NULL,
 				 DRM_PLANE_TYPE_PRIMARY, "meson_primary_plane");
 
 	drm_plane_helper_add(plane, &meson_plane_helper_funcs);
diff --git a/drivers/gpu/drm/meson/meson_venc_cvbs.c b/drivers/gpu/drm/meson/meson_venc_cvbs.c
index 00775b397dba..79d95ca8a0c0 100644
--- a/drivers/gpu/drm/meson/meson_venc_cvbs.c
+++ b/drivers/gpu/drm/meson/meson_venc_cvbs.c
@@ -118,7 +118,6 @@ static int meson_cvbs_connector_mode_valid(struct drm_connector *connector,
 }
 
 static const struct drm_connector_funcs meson_cvbs_connector_funcs = {
-	.dpms			= drm_atomic_helper_connector_dpms,
 	.detect			= meson_cvbs_connector_detect,
 	.fill_modes		= drm_helper_probe_single_connector_modes,
 	.destroy		= meson_cvbs_connector_destroy,
diff --git a/drivers/gpu/drm/mga/mga_drv.c b/drivers/gpu/drm/mga/mga_drv.c
index 63ba0699d107..1aad27813c23 100644
--- a/drivers/gpu/drm/mga/mga_drv.c
+++ b/drivers/gpu/drm/mga/mga_drv.c
@@ -62,7 +62,6 @@ static struct drm_driver driver = {
 	.load = mga_driver_load,
 	.unload = mga_driver_unload,
 	.lastclose = mga_driver_lastclose,
-	.set_busid = drm_pci_set_busid,
 	.dma_quiescent = mga_driver_dma_quiescent,
 	.get_vblank_counter = mga_get_vblank_counter,
 	.enable_vblank = mga_enable_vblank,
@@ -90,12 +89,12 @@ static struct pci_driver mga_pci_driver = {
 static int __init mga_init(void)
 {
 	driver.num_ioctls = mga_max_ioctl;
-	return drm_pci_init(&driver, &mga_pci_driver);
+	return drm_legacy_pci_init(&driver, &mga_pci_driver);
 }
 
 static void __exit mga_exit(void)
 {
-	drm_pci_exit(&driver, &mga_pci_driver);
+	drm_legacy_pci_exit(&driver, &mga_pci_driver);
 }
 
 module_init(mga_init);
diff --git a/drivers/gpu/drm/mgag200/mgag200_cursor.c b/drivers/gpu/drm/mgag200/mgag200_cursor.c
index 2ac3fcbfea7b..968e20379d54 100644
--- a/drivers/gpu/drm/mgag200/mgag200_cursor.c
+++ b/drivers/gpu/drm/mgag200/mgag200_cursor.c
@@ -248,7 +248,7 @@ int mga_crtc_cursor_set(struct drm_crtc *crtc,
 out_unreserve1:
 	mgag200_bo_unreserve(pixels_2);
 out_unref:
-	drm_gem_object_unreference_unlocked(obj);
+	drm_gem_object_put_unlocked(obj);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/mgag200/mgag200_drv.c b/drivers/gpu/drm/mgag200/mgag200_drv.c
index 9ac007880328..74cdde2ee474 100644
--- a/drivers/gpu/drm/mgag200/mgag200_drv.c
+++ b/drivers/gpu/drm/mgag200/mgag200_drv.c
@@ -91,7 +91,6 @@ static struct drm_driver driver = {
 	.driver_features = DRIVER_GEM | DRIVER_MODESET,
 	.load = mgag200_driver_load,
 	.unload = mgag200_driver_unload,
-	.set_busid = drm_pci_set_busid,
 	.fops = &mgag200_driver_fops,
 	.name = DRIVER_NAME,
 	.desc = DRIVER_DESC,
@@ -103,7 +102,6 @@ static struct drm_driver driver = {
 	.gem_free_object_unlocked = mgag200_gem_free_object,
 	.dumb_create = mgag200_dumb_create,
 	.dumb_map_offset = mgag200_dumb_mmap_offset,
-	.dumb_destroy = drm_gem_dumb_destroy,
 };
 
 static struct pci_driver mgag200_pci_driver = {
@@ -120,12 +118,13 @@ static int __init mgag200_init(void)
 
 	if (mgag200_modeset == 0)
 		return -EINVAL;
-	return drm_pci_init(&driver, &mgag200_pci_driver);
+
+	return pci_register_driver(&mgag200_pci_driver);
 }
 
 static void __exit mgag200_exit(void)
 {
-	drm_pci_exit(&driver, &mgag200_pci_driver);
+	pci_unregister_driver(&mgag200_pci_driver);
 }
 
 module_init(mgag200_init);
diff --git a/drivers/gpu/drm/mgag200/mgag200_drv.h b/drivers/gpu/drm/mgag200/mgag200_drv.h
index c88b6ec88dd2..04f1dfba12e5 100644
--- a/drivers/gpu/drm/mgag200/mgag200_drv.h
+++ b/drivers/gpu/drm/mgag200/mgag200_drv.h
@@ -237,11 +237,6 @@ mgag200_bo(struct ttm_buffer_object *bo)
 {
 	return container_of(bo, struct mgag200_bo, bo);
 }
-				/* mgag200_crtc.c */
-void mga_crtc_fb_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
-			     u16 blue, int regno);
-void mga_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green,
-			     u16 *blue, int regno);
 
 				/* mgag200_mode.c */
 int mgag200_modeset_init(struct mga_device *mdev);
diff --git a/drivers/gpu/drm/mgag200/mgag200_fb.c b/drivers/gpu/drm/mgag200/mgag200_fb.c
index 5d3b1fac906f..30726c9fe28c 100644
--- a/drivers/gpu/drm/mgag200/mgag200_fb.c
+++ b/drivers/gpu/drm/mgag200/mgag200_fb.c
@@ -210,7 +210,6 @@ static int mgag200fb_create(struct drm_fb_helper *helper,
 
 	strcpy(info->fix.id, "mgadrmfb");
 
-	info->flags = FBINFO_DEFAULT | FBINFO_CAN_FORCE_OUTPUT;
 	info->fbops = &mgag200fb_ops;
 
 	/* setup aperture base/size for vesafb takeover */
@@ -233,7 +232,7 @@ static int mgag200fb_create(struct drm_fb_helper *helper,
 err_alloc_fbi:
 	vfree(sysram);
 err_sysram:
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 
 	return ret;
 }
@@ -246,7 +245,7 @@ static int mga_fbdev_destroy(struct drm_device *dev,
 	drm_fb_helper_unregister_fbi(&mfbdev->helper);
 
 	if (mfb->obj) {
-		drm_gem_object_unreference_unlocked(mfb->obj);
+		drm_gem_object_put_unlocked(mfb->obj);
 		mfb->obj = NULL;
 	}
 	drm_fb_helper_fini(&mfbdev->helper);
@@ -258,8 +257,6 @@ static int mga_fbdev_destroy(struct drm_device *dev,
 }
 
 static const struct drm_fb_helper_funcs mga_fb_helper_funcs = {
-	.gamma_set = mga_crtc_fb_gamma_set,
-	.gamma_get = mga_crtc_fb_gamma_get,
 	.fb_probe = mgag200fb_create,
 };
 
diff --git a/drivers/gpu/drm/mgag200/mgag200_main.c b/drivers/gpu/drm/mgag200/mgag200_main.c
index dce8a3eb5a10..780f983b0294 100644
--- a/drivers/gpu/drm/mgag200/mgag200_main.c
+++ b/drivers/gpu/drm/mgag200/mgag200_main.c
@@ -18,7 +18,7 @@ static void mga_user_framebuffer_destroy(struct drm_framebuffer *fb)
 {
 	struct mga_framebuffer *mga_fb = to_mga_framebuffer(fb);
 
-	drm_gem_object_unreference_unlocked(mga_fb->obj);
+	drm_gem_object_put_unlocked(mga_fb->obj);
 	drm_framebuffer_cleanup(fb);
 	kfree(fb);
 }
@@ -59,13 +59,13 @@ mgag200_user_framebuffer_create(struct drm_device *dev,
 
 	mga_fb = kzalloc(sizeof(*mga_fb), GFP_KERNEL);
 	if (!mga_fb) {
-		drm_gem_object_unreference_unlocked(obj);
+		drm_gem_object_put_unlocked(obj);
 		return ERR_PTR(-ENOMEM);
 	}
 
 	ret = mgag200_framebuffer_init(dev, mga_fb, mode_cmd, obj);
 	if (ret) {
-		drm_gem_object_unreference_unlocked(obj);
+		drm_gem_object_put_unlocked(obj);
 		kfree(mga_fb);
 		return ERR_PTR(ret);
 	}
@@ -317,7 +317,7 @@ int mgag200_dumb_create(struct drm_file *file,
 		return ret;
 
 	ret = drm_gem_handle_create(file, gobj, &handle);
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 	if (ret)
 		return ret;
 
@@ -366,6 +366,6 @@ mgag200_dumb_mmap_offset(struct drm_file *file,
 	bo = gem_to_mga_bo(obj);
 	*offset = mgag200_bo_mmap_offset(bo);
 
-	drm_gem_object_unreference_unlocked(obj);
+	drm_gem_object_put_unlocked(obj);
 	return 0;
 }
diff --git a/drivers/gpu/drm/mgag200/mgag200_mode.c b/drivers/gpu/drm/mgag200/mgag200_mode.c
index f4b53588e071..5e9cd4c0e8b6 100644
--- a/drivers/gpu/drm/mgag200/mgag200_mode.c
+++ b/drivers/gpu/drm/mgag200/mgag200_mode.c
@@ -27,15 +27,19 @@
 
 static void mga_crtc_load_lut(struct drm_crtc *crtc)
 {
-	struct mga_crtc *mga_crtc = to_mga_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct mga_device *mdev = dev->dev_private;
 	struct drm_framebuffer *fb = crtc->primary->fb;
+	u16 *r_ptr, *g_ptr, *b_ptr;
 	int i;
 
 	if (!crtc->enabled)
 		return;
 
+	r_ptr = crtc->gamma_store;
+	g_ptr = r_ptr + crtc->gamma_size;
+	b_ptr = g_ptr + crtc->gamma_size;
+
 	WREG8(DAC_INDEX + MGA1064_INDEX, 0);
 
 	if (fb && fb->format->cpp[0] * 8 == 16) {
@@ -46,25 +50,27 @@ static void mga_crtc_load_lut(struct drm_crtc *crtc)
 				if (i > (MGAG200_LUT_SIZE >> 1)) {
 					r = b = 0;
 				} else {
-					r = mga_crtc->lut_r[i << 1];
-					b = mga_crtc->lut_b[i << 1];
+					r = *r_ptr++ >> 8;
+					b = *b_ptr++ >> 8;
+					r_ptr++;
+					b_ptr++;
 				}
 			} else {
-				r = mga_crtc->lut_r[i];
-				b = mga_crtc->lut_b[i];
+				r = *r_ptr++ >> 8;
+				b = *b_ptr++ >> 8;
 			}
 			/* VGA registers */
 			WREG8(DAC_INDEX + MGA1064_COL_PAL, r);
-			WREG8(DAC_INDEX + MGA1064_COL_PAL, mga_crtc->lut_g[i]);
+			WREG8(DAC_INDEX + MGA1064_COL_PAL, *g_ptr++ >> 8);
 			WREG8(DAC_INDEX + MGA1064_COL_PAL, b);
 		}
 		return;
 	}
 	for (i = 0; i < MGAG200_LUT_SIZE; i++) {
 		/* VGA registers */
-		WREG8(DAC_INDEX + MGA1064_COL_PAL, mga_crtc->lut_r[i]);
-		WREG8(DAC_INDEX + MGA1064_COL_PAL, mga_crtc->lut_g[i]);
-		WREG8(DAC_INDEX + MGA1064_COL_PAL, mga_crtc->lut_b[i]);
+		WREG8(DAC_INDEX + MGA1064_COL_PAL, *r_ptr++ >> 8);
+		WREG8(DAC_INDEX + MGA1064_COL_PAL, *g_ptr++ >> 8);
+		WREG8(DAC_INDEX + MGA1064_COL_PAL, *b_ptr++ >> 8);
 	}
 }
 
@@ -1399,14 +1405,6 @@ static int mga_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
 			      u16 *blue, uint32_t size,
 			      struct drm_modeset_acquire_ctx *ctx)
 {
-	struct mga_crtc *mga_crtc = to_mga_crtc(crtc);
-	int i;
-
-	for (i = 0; i < size; i++) {
-		mga_crtc->lut_r[i] = red[i] >> 8;
-		mga_crtc->lut_g[i] = green[i] >> 8;
-		mga_crtc->lut_b[i] = blue[i] >> 8;
-	}
 	mga_crtc_load_lut(crtc);
 
 	return 0;
@@ -1455,14 +1453,12 @@ static const struct drm_crtc_helper_funcs mga_helper_funcs = {
 	.mode_set_base = mga_crtc_mode_set_base,
 	.prepare = mga_crtc_prepare,
 	.commit = mga_crtc_commit,
-	.load_lut = mga_crtc_load_lut,
 };
 
 /* CRTC setup */
 static void mga_crtc_init(struct mga_device *mdev)
 {
 	struct mga_crtc *mga_crtc;
-	int i;
 
 	mga_crtc = kzalloc(sizeof(struct mga_crtc) +
 			      (MGAG200FB_CONN_LIMIT * sizeof(struct drm_connector *)),
@@ -1476,37 +1472,9 @@ static void mga_crtc_init(struct mga_device *mdev)
 	drm_mode_crtc_set_gamma_size(&mga_crtc->base, MGAG200_LUT_SIZE);
 	mdev->mode_info.crtc = mga_crtc;
 
-	for (i = 0; i < MGAG200_LUT_SIZE; i++) {
-		mga_crtc->lut_r[i] = i;
-		mga_crtc->lut_g[i] = i;
-		mga_crtc->lut_b[i] = i;
-	}
-
 	drm_crtc_helper_add(&mga_crtc->base, &mga_helper_funcs);
 }
 
-/** Sets the color ramps on behalf of fbcon */
-void mga_crtc_fb_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
-			      u16 blue, int regno)
-{
-	struct mga_crtc *mga_crtc = to_mga_crtc(crtc);
-
-	mga_crtc->lut_r[regno] = red >> 8;
-	mga_crtc->lut_g[regno] = green >> 8;
-	mga_crtc->lut_b[regno] = blue >> 8;
-}
-
-/** Gets the color ramps on behalf of fbcon */
-void mga_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green,
-			      u16 *blue, int regno)
-{
-	struct mga_crtc *mga_crtc = to_mga_crtc(crtc);
-
-	*red = (u16)mga_crtc->lut_r[regno] << 8;
-	*green = (u16)mga_crtc->lut_g[regno] << 8;
-	*blue = (u16)mga_crtc->lut_b[regno] << 8;
-}
-
 /*
  * The encoder comes after the CRTC in the output pipeline, but before
  * the connector. It's responsible for ensuring that the digital
diff --git a/drivers/gpu/drm/msm/adreno/a3xx_gpu.c b/drivers/gpu/drm/msm/adreno/a3xx_gpu.c
index 0e3828ed1e46..7791313405b5 100644
--- a/drivers/gpu/drm/msm/adreno/a3xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a3xx_gpu.c
@@ -486,8 +486,6 @@ struct msm_gpu *a3xx_gpu_init(struct drm_device *dev)
 	adreno_gpu = &a3xx_gpu->base;
 	gpu = &adreno_gpu->base;
 
-	a3xx_gpu->pdev = pdev;
-
 	gpu->perfcntrs = perfcntrs;
 	gpu->num_perfcntrs = ARRAY_SIZE(perfcntrs);
 
diff --git a/drivers/gpu/drm/msm/adreno/a3xx_gpu.h b/drivers/gpu/drm/msm/adreno/a3xx_gpu.h
index 85ff66cbddd6..ab60dc9e344e 100644
--- a/drivers/gpu/drm/msm/adreno/a3xx_gpu.h
+++ b/drivers/gpu/drm/msm/adreno/a3xx_gpu.h
@@ -28,7 +28,6 @@
 
 struct a3xx_gpu {
 	struct adreno_gpu base;
-	struct platform_device *pdev;
 
 	/* if OCMEM is used for GMEM: */
 	uint32_t ocmem_base;
diff --git a/drivers/gpu/drm/msm/adreno/a4xx_gpu.c b/drivers/gpu/drm/msm/adreno/a4xx_gpu.c
index 19abf229b08d..58341ef6f15b 100644
--- a/drivers/gpu/drm/msm/adreno/a4xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a4xx_gpu.c
@@ -568,8 +568,6 @@ struct msm_gpu *a4xx_gpu_init(struct drm_device *dev)
 	adreno_gpu = &a4xx_gpu->base;
 	gpu = &adreno_gpu->base;
 
-	a4xx_gpu->pdev = pdev;
-
 	gpu->perfcntrs = NULL;
 	gpu->num_perfcntrs = 0;
 
diff --git a/drivers/gpu/drm/msm/adreno/a4xx_gpu.h b/drivers/gpu/drm/msm/adreno/a4xx_gpu.h
index 01247204ac92..f757184328a3 100644
--- a/drivers/gpu/drm/msm/adreno/a4xx_gpu.h
+++ b/drivers/gpu/drm/msm/adreno/a4xx_gpu.h
@@ -23,7 +23,6 @@
 
 struct a4xx_gpu {
 	struct adreno_gpu base;
-	struct platform_device *pdev;
 
 	/* if OCMEM is used for GMEM: */
 	uint32_t ocmem_base;
diff --git a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
index f9eae03aa1dc..17c59d839e6f 100644
--- a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
@@ -284,28 +284,14 @@ static int a5xx_me_init(struct msm_gpu *gpu)
 static struct drm_gem_object *a5xx_ucode_load_bo(struct msm_gpu *gpu,
 		const struct firmware *fw, u64 *iova)
 {
-	struct drm_device *drm = gpu->dev;
 	struct drm_gem_object *bo;
 	void *ptr;
 
-	bo = msm_gem_new_locked(drm, fw->size - 4, MSM_BO_UNCACHED);
-	if (IS_ERR(bo))
-		return bo;
+	ptr = msm_gem_kernel_new_locked(gpu->dev, fw->size - 4,
+		MSM_BO_UNCACHED | MSM_BO_GPU_READONLY, gpu->aspace, &bo, iova);
 
-	ptr = msm_gem_get_vaddr(bo);
-	if (!ptr) {
-		drm_gem_object_unreference(bo);
-		return ERR_PTR(-ENOMEM);
-	}
-
-	if (iova) {
-		int ret = msm_gem_get_iova(bo, gpu->aspace, iova);
-
-		if (ret) {
-			drm_gem_object_unreference(bo);
-			return ERR_PTR(ret);
-		}
-	}
+	if (IS_ERR(ptr))
+		return ERR_CAST(ptr);
 
 	memcpy(ptr, &fw->data[4], fw->size - 4);
 
@@ -372,8 +358,7 @@ static int a5xx_zap_shader_init(struct msm_gpu *gpu)
 {
 	static bool loaded;
 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
-	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
-	struct platform_device *pdev = a5xx_gpu->pdev;
+	struct platform_device *pdev = gpu->pdev;
 	int ret;
 
 	/*
@@ -410,6 +395,7 @@ static int a5xx_zap_shader_init(struct msm_gpu *gpu)
 	  A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \
 	  A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW | \
 	  A5XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
+	  A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT | \
 	  A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
 	  A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
 	  A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP)
@@ -812,6 +798,27 @@ static void a5xx_gpmu_err_irq(struct msm_gpu *gpu)
 	dev_err_ratelimited(gpu->dev->dev, "GPMU | voltage droop\n");
 }
 
+static void a5xx_fault_detect_irq(struct msm_gpu *gpu)
+{
+	struct drm_device *dev = gpu->dev;
+	struct msm_drm_private *priv = dev->dev_private;
+
+	dev_err(dev->dev, "gpu fault fence %x status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x\n",
+		gpu->funcs->last_fence(gpu),
+		gpu_read(gpu, REG_A5XX_RBBM_STATUS),
+		gpu_read(gpu, REG_A5XX_CP_RB_RPTR),
+		gpu_read(gpu, REG_A5XX_CP_RB_WPTR),
+		gpu_read64(gpu, REG_A5XX_CP_IB1_BASE, REG_A5XX_CP_IB1_BASE_HI),
+		gpu_read(gpu, REG_A5XX_CP_IB1_BUFSZ),
+		gpu_read64(gpu, REG_A5XX_CP_IB2_BASE, REG_A5XX_CP_IB2_BASE_HI),
+		gpu_read(gpu, REG_A5XX_CP_IB2_BUFSZ));
+
+	/* Turn off the hangcheck timer to keep it from bothering us */
+	del_timer(&gpu->hangcheck_timer);
+
+	queue_work(priv->wq, &gpu->recover_work);
+}
+
 #define RBBM_ERROR_MASK \
 	(A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \
 	A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \
@@ -838,6 +845,9 @@ static irqreturn_t a5xx_irq(struct msm_gpu *gpu)
 	if (status & A5XX_RBBM_INT_0_MASK_CP_HW_ERROR)
 		a5xx_cp_err_irq(gpu);
 
+	if (status & A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT)
+		a5xx_fault_detect_irq(gpu);
+
 	if (status & A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS)
 		a5xx_uche_err_irq(gpu);
 
@@ -1015,7 +1025,6 @@ struct msm_gpu *a5xx_gpu_init(struct drm_device *dev)
 	adreno_gpu = &a5xx_gpu->base;
 	gpu = &adreno_gpu->base;
 
-	a5xx_gpu->pdev = pdev;
 	adreno_gpu->registers = a5xx_registers;
 	adreno_gpu->reg_offsets = a5xx_register_offsets;
 
diff --git a/drivers/gpu/drm/msm/adreno/a5xx_gpu.h b/drivers/gpu/drm/msm/adreno/a5xx_gpu.h
index 1137092241d5..e94451685bf8 100644
--- a/drivers/gpu/drm/msm/adreno/a5xx_gpu.h
+++ b/drivers/gpu/drm/msm/adreno/a5xx_gpu.h
@@ -23,7 +23,6 @@
 
 struct a5xx_gpu {
 	struct adreno_gpu base;
-	struct platform_device *pdev;
 
 	struct drm_gem_object *pm4_bo;
 	uint64_t pm4_iova;
diff --git a/drivers/gpu/drm/msm/adreno/a5xx_power.c b/drivers/gpu/drm/msm/adreno/a5xx_power.c
index 87af6eea0483..04aab1dcae2b 100644
--- a/drivers/gpu/drm/msm/adreno/a5xx_power.c
+++ b/drivers/gpu/drm/msm/adreno/a5xx_power.c
@@ -294,16 +294,10 @@ void a5xx_gpmu_ucode_init(struct msm_gpu *gpu)
 	 */
 	bosize = (cmds_size + (cmds_size / TYPE4_MAX_PAYLOAD) + 1) << 2;
 
-	a5xx_gpu->gpmu_bo = msm_gem_new_locked(drm, bosize, MSM_BO_UNCACHED);
-	if (IS_ERR(a5xx_gpu->gpmu_bo))
-		goto err;
-
-	if (msm_gem_get_iova(a5xx_gpu->gpmu_bo, gpu->aspace,
-			&a5xx_gpu->gpmu_iova))
-		goto err;
-
-	ptr = msm_gem_get_vaddr(a5xx_gpu->gpmu_bo);
-	if (!ptr)
+	ptr = msm_gem_kernel_new_locked(drm, bosize,
+		MSM_BO_UNCACHED | MSM_BO_GPU_READONLY, gpu->aspace,
+		&a5xx_gpu->gpmu_bo, &a5xx_gpu->gpmu_iova);
+	if (IS_ERR(ptr))
 		goto err;
 
 	while (cmds_size > 0) {
diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
index 7414c6bbd582..c8b4ac254bb5 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
@@ -337,11 +337,6 @@ void adreno_wait_ring(struct msm_gpu *gpu, uint32_t ndwords)
 		DRM_ERROR("%s: timeout waiting for ringbuffer space\n", gpu->name);
 }
 
-static const char *iommu_ports[] = {
-		"gfx3d_user", "gfx3d_priv",
-		"gfx3d1_user", "gfx3d1_priv",
-};
-
 int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 		struct adreno_gpu *adreno_gpu, const struct adreno_gpu_funcs *funcs)
 {
@@ -373,15 +368,15 @@ int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 
 	adreno_gpu_config.ringsz = RB_SIZE;
 
+	pm_runtime_set_autosuspend_delay(&pdev->dev, DRM_MSM_INACTIVE_PERIOD);
+	pm_runtime_use_autosuspend(&pdev->dev);
+	pm_runtime_enable(&pdev->dev);
+
 	ret = msm_gpu_init(drm, pdev, &adreno_gpu->base, &funcs->base,
 			adreno_gpu->info->name, &adreno_gpu_config);
 	if (ret)
 		return ret;
 
-	pm_runtime_set_autosuspend_delay(&pdev->dev, DRM_MSM_INACTIVE_PERIOD);
-	pm_runtime_use_autosuspend(&pdev->dev);
-	pm_runtime_enable(&pdev->dev);
-
 	ret = request_firmware(&adreno_gpu->pm4, adreno_gpu->info->pm4fw, drm->dev);
 	if (ret) {
 		dev_err(drm->dev, "failed to load %s PM4 firmware: %d\n",
@@ -396,37 +391,17 @@ int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 		return ret;
 	}
 
-	if (gpu->aspace && gpu->aspace->mmu) {
-		struct msm_mmu *mmu = gpu->aspace->mmu;
-		ret = mmu->funcs->attach(mmu, iommu_ports,
-				ARRAY_SIZE(iommu_ports));
-		if (ret)
-			return ret;
-	}
+	adreno_gpu->memptrs = msm_gem_kernel_new(drm,
+		sizeof(*adreno_gpu->memptrs), MSM_BO_UNCACHED, gpu->aspace,
+		&adreno_gpu->memptrs_bo, &adreno_gpu->memptrs_iova);
 
-	adreno_gpu->memptrs_bo = msm_gem_new(drm, sizeof(*adreno_gpu->memptrs),
-			MSM_BO_UNCACHED);
-	if (IS_ERR(adreno_gpu->memptrs_bo)) {
-		ret = PTR_ERR(adreno_gpu->memptrs_bo);
-		adreno_gpu->memptrs_bo = NULL;
-		dev_err(drm->dev, "could not allocate memptrs: %d\n", ret);
-		return ret;
-	}
-
-	adreno_gpu->memptrs = msm_gem_get_vaddr(adreno_gpu->memptrs_bo);
 	if (IS_ERR(adreno_gpu->memptrs)) {
-		dev_err(drm->dev, "could not vmap memptrs\n");
-		return -ENOMEM;
-	}
-
-	ret = msm_gem_get_iova(adreno_gpu->memptrs_bo, gpu->aspace,
-			&adreno_gpu->memptrs_iova);
-	if (ret) {
-		dev_err(drm->dev, "could not map memptrs: %d\n", ret);
-		return ret;
+		ret = PTR_ERR(adreno_gpu->memptrs);
+		adreno_gpu->memptrs = NULL;
+		dev_err(drm->dev, "could not allocate memptrs: %d\n", ret);
 	}
 
-	return 0;
+	return ret;
 }
 
 void adreno_gpu_cleanup(struct adreno_gpu *adreno_gpu)
@@ -446,10 +421,4 @@ void adreno_gpu_cleanup(struct adreno_gpu *adreno_gpu)
 	release_firmware(adreno_gpu->pfp);
 
 	msm_gpu_cleanup(gpu);
-
-	if (gpu->aspace) {
-		gpu->aspace->mmu->funcs->detach(gpu->aspace->mmu,
-			iommu_ports, ARRAY_SIZE(iommu_ports));
-		msm_gem_address_space_put(gpu->aspace);
-	}
 }
diff --git a/drivers/gpu/drm/msm/dsi/dsi.c b/drivers/gpu/drm/msm/dsi/dsi.c
index 311c1c1e7d6c..98742d7af6dc 100644
--- a/drivers/gpu/drm/msm/dsi/dsi.c
+++ b/drivers/gpu/drm/msm/dsi/dsi.c
@@ -161,12 +161,17 @@ static const struct of_device_id dt_match[] = {
 	{}
 };
 
+static const struct dev_pm_ops dsi_pm_ops = {
+	SET_RUNTIME_PM_OPS(msm_dsi_runtime_suspend, msm_dsi_runtime_resume, NULL)
+};
+
 static struct platform_driver dsi_driver = {
 	.probe = dsi_dev_probe,
 	.remove = dsi_dev_remove,
 	.driver = {
 		.name = "msm_dsi",
 		.of_match_table = dt_match,
+		.pm = &dsi_pm_ops,
 	},
 };
 
diff --git a/drivers/gpu/drm/msm/dsi/dsi.h b/drivers/gpu/drm/msm/dsi/dsi.h
index 9e6017387efb..2302046197a8 100644
--- a/drivers/gpu/drm/msm/dsi/dsi.h
+++ b/drivers/gpu/drm/msm/dsi/dsi.h
@@ -179,6 +179,8 @@ void msm_dsi_host_destroy(struct mipi_dsi_host *host);
 int msm_dsi_host_modeset_init(struct mipi_dsi_host *host,
 					struct drm_device *dev);
 int msm_dsi_host_init(struct msm_dsi *msm_dsi);
+int msm_dsi_runtime_suspend(struct device *dev);
+int msm_dsi_runtime_resume(struct device *dev);
 
 /* dsi phy */
 struct msm_dsi_phy;
diff --git a/drivers/gpu/drm/msm/dsi/dsi_host.c b/drivers/gpu/drm/msm/dsi/dsi_host.c
index c7b612c3d771..dbb31a014419 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_host.c
+++ b/drivers/gpu/drm/msm/dsi/dsi_host.c
@@ -135,7 +135,6 @@ struct msm_dsi_host {
 	struct completion video_comp;
 	struct mutex dev_mutex;
 	struct mutex cmd_mutex;
-	struct mutex clk_mutex;
 	spinlock_t intr_lock; /* Protect interrupt ctrl register */
 
 	u32 err_work_state;
@@ -221,6 +220,8 @@ static const struct msm_dsi_cfg_handler *dsi_get_config(
 		goto put_gdsc;
 	}
 
+	pm_runtime_get_sync(dev);
+
 	ret = regulator_enable(gdsc_reg);
 	if (ret) {
 		pr_err("%s: unable to enable gdsc\n", __func__);
@@ -247,6 +248,7 @@ disable_clks:
 	clk_disable_unprepare(ahb_clk);
 disable_gdsc:
 	regulator_disable(gdsc_reg);
+	pm_runtime_put_autosuspend(dev);
 put_clk:
 	clk_put(ahb_clk);
 put_gdsc:
@@ -455,6 +457,34 @@ static void dsi_bus_clk_disable(struct msm_dsi_host *msm_host)
 		clk_disable_unprepare(msm_host->bus_clks[i]);
 }
 
+int msm_dsi_runtime_suspend(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct msm_dsi *msm_dsi = platform_get_drvdata(pdev);
+	struct mipi_dsi_host *host = msm_dsi->host;
+	struct msm_dsi_host *msm_host = to_msm_dsi_host(host);
+
+	if (!msm_host->cfg_hnd)
+		return 0;
+
+	dsi_bus_clk_disable(msm_host);
+
+	return 0;
+}
+
+int msm_dsi_runtime_resume(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct msm_dsi *msm_dsi = platform_get_drvdata(pdev);
+	struct mipi_dsi_host *host = msm_dsi->host;
+	struct msm_dsi_host *msm_host = to_msm_dsi_host(host);
+
+	if (!msm_host->cfg_hnd)
+		return 0;
+
+	return dsi_bus_clk_enable(msm_host);
+}
+
 static int dsi_link_clk_enable_6g(struct msm_dsi_host *msm_host)
 {
 	int ret;
@@ -596,35 +626,6 @@ static void dsi_link_clk_disable(struct msm_dsi_host *msm_host)
 	}
 }
 
-static int dsi_clk_ctrl(struct msm_dsi_host *msm_host, bool enable)
-{
-	int ret = 0;
-
-	mutex_lock(&msm_host->clk_mutex);
-	if (enable) {
-		ret = dsi_bus_clk_enable(msm_host);
-		if (ret) {
-			pr_err("%s: Can not enable bus clk, %d\n",
-				__func__, ret);
-			goto unlock_ret;
-		}
-		ret = dsi_link_clk_enable(msm_host);
-		if (ret) {
-			pr_err("%s: Can not enable link clk, %d\n",
-				__func__, ret);
-			dsi_bus_clk_disable(msm_host);
-			goto unlock_ret;
-		}
-	} else {
-		dsi_link_clk_disable(msm_host);
-		dsi_bus_clk_disable(msm_host);
-	}
-
-unlock_ret:
-	mutex_unlock(&msm_host->clk_mutex);
-	return ret;
-}
-
 static int dsi_calc_clk_rate(struct msm_dsi_host *msm_host)
 {
 	struct drm_display_mode *mode = msm_host->mode;
@@ -1699,6 +1700,7 @@ int msm_dsi_host_init(struct msm_dsi *msm_dsi)
 	}
 
 	msm_host->pdev = pdev;
+	msm_dsi->host = &msm_host->base;
 
 	ret = dsi_host_parse_dt(msm_host);
 	if (ret) {
@@ -1713,6 +1715,8 @@ int msm_dsi_host_init(struct msm_dsi *msm_dsi)
 		goto fail;
 	}
 
+	pm_runtime_enable(&pdev->dev);
+
 	msm_host->cfg_hnd = dsi_get_config(msm_host);
 	if (!msm_host->cfg_hnd) {
 		ret = -EINVAL;
@@ -1753,7 +1757,6 @@ int msm_dsi_host_init(struct msm_dsi *msm_dsi)
 	init_completion(&msm_host->video_comp);
 	mutex_init(&msm_host->dev_mutex);
 	mutex_init(&msm_host->cmd_mutex);
-	mutex_init(&msm_host->clk_mutex);
 	spin_lock_init(&msm_host->intr_lock);
 
 	/* setup workqueue */
@@ -1761,7 +1764,6 @@ int msm_dsi_host_init(struct msm_dsi *msm_dsi)
 	INIT_WORK(&msm_host->err_work, dsi_err_worker);
 	INIT_WORK(&msm_host->hpd_work, dsi_hpd_worker);
 
-	msm_dsi->host = &msm_host->base;
 	msm_dsi->id = msm_host->id;
 
 	DBG("Dsi Host %d initialized", msm_host->id);
@@ -1783,9 +1785,10 @@ void msm_dsi_host_destroy(struct mipi_dsi_host *host)
 		msm_host->workqueue = NULL;
 	}
 
-	mutex_destroy(&msm_host->clk_mutex);
 	mutex_destroy(&msm_host->cmd_mutex);
 	mutex_destroy(&msm_host->dev_mutex);
+
+	pm_runtime_disable(&msm_host->pdev->dev);
 }
 
 int msm_dsi_host_modeset_init(struct mipi_dsi_host *host,
@@ -1881,7 +1884,8 @@ int msm_dsi_host_xfer_prepare(struct mipi_dsi_host *host,
 	 * mdss interrupt is generated in mdp core clock domain
 	 * mdp clock need to be enabled to receive dsi interrupt
 	 */
-	dsi_clk_ctrl(msm_host, 1);
+	pm_runtime_get_sync(&msm_host->pdev->dev);
+	dsi_link_clk_enable(msm_host);
 
 	/* TODO: vote for bus bandwidth */
 
@@ -1911,7 +1915,8 @@ void msm_dsi_host_xfer_restore(struct mipi_dsi_host *host,
 
 	/* TODO: unvote for bus bandwidth */
 
-	dsi_clk_ctrl(msm_host, 0);
+	dsi_link_clk_disable(msm_host);
+	pm_runtime_put_autosuspend(&msm_host->pdev->dev);
 }
 
 int msm_dsi_host_cmd_tx(struct mipi_dsi_host *host,
@@ -2160,8 +2165,11 @@ int msm_dsi_host_enable(struct mipi_dsi_host *host)
 	 * and only turned on before MDP START.
 	 * This part of code should be enabled once mdp driver support it.
 	 */
-	/* if (msm_panel->mode == MSM_DSI_CMD_MODE)
-		dsi_clk_ctrl(msm_host, 0); */
+	/* if (msm_panel->mode == MSM_DSI_CMD_MODE) {
+	 *	dsi_link_clk_disable(msm_host);
+	 *	pm_runtime_put_autosuspend(&msm_host->pdev->dev);
+	 * }
+	 */
 
 	return 0;
 }
@@ -2217,9 +2225,11 @@ int msm_dsi_host_power_on(struct mipi_dsi_host *host,
 		goto unlock_ret;
 	}
 
-	ret = dsi_clk_ctrl(msm_host, 1);
+	pm_runtime_get_sync(&msm_host->pdev->dev);
+	ret = dsi_link_clk_enable(msm_host);
 	if (ret) {
-		pr_err("%s: failed to enable clocks. ret=%d\n", __func__, ret);
+		pr_err("%s: failed to enable link clocks. ret=%d\n",
+		       __func__, ret);
 		goto fail_disable_reg;
 	}
 
@@ -2243,7 +2253,8 @@ int msm_dsi_host_power_on(struct mipi_dsi_host *host,
 	return 0;
 
 fail_disable_clk:
-	dsi_clk_ctrl(msm_host, 0);
+	dsi_link_clk_disable(msm_host);
+	pm_runtime_put_autosuspend(&msm_host->pdev->dev);
 fail_disable_reg:
 	dsi_host_regulator_disable(msm_host);
 unlock_ret:
@@ -2268,7 +2279,8 @@ int msm_dsi_host_power_off(struct mipi_dsi_host *host)
 
 	pinctrl_pm_select_sleep_state(&msm_host->pdev->dev);
 
-	dsi_clk_ctrl(msm_host, 0);
+	dsi_link_clk_disable(msm_host);
+	pm_runtime_put_autosuspend(&msm_host->pdev->dev);
 
 	dsi_host_regulator_disable(msm_host);
 
diff --git a/drivers/gpu/drm/msm/dsi/dsi_manager.c b/drivers/gpu/drm/msm/dsi/dsi_manager.c
index a879ffa534b4..855248132b2b 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_manager.c
+++ b/drivers/gpu/drm/msm/dsi/dsi_manager.c
@@ -626,7 +626,6 @@ static void dsi_mgr_bridge_mode_set(struct drm_bridge *bridge,
 }
 
 static const struct drm_connector_funcs dsi_mgr_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.detect = dsi_mgr_connector_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = dsi_mgr_connector_destroy,
diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c
index 0c2eb9c9a1fc..7c9bf91bc22b 100644
--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c
+++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c
@@ -373,7 +373,7 @@ static int dsi_phy_enable_resource(struct msm_dsi_phy *phy)
 static void dsi_phy_disable_resource(struct msm_dsi_phy *phy)
 {
 	clk_disable_unprepare(phy->ahb_clk);
-	pm_runtime_put_sync(&phy->pdev->dev);
+	pm_runtime_put_autosuspend(&phy->pdev->dev);
 }
 
 static const struct of_device_id dsi_phy_dt_match[] = {
diff --git a/drivers/gpu/drm/msm/edp/edp_connector.c b/drivers/gpu/drm/msm/edp/edp_connector.c
index 5960628ceb93..6f3fc6b0f0a3 100644
--- a/drivers/gpu/drm/msm/edp/edp_connector.c
+++ b/drivers/gpu/drm/msm/edp/edp_connector.c
@@ -92,7 +92,6 @@ static int edp_connector_mode_valid(struct drm_connector *connector,
 }
 
 static const struct drm_connector_funcs edp_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.detect = edp_connector_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = edp_connector_destroy,
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi.c b/drivers/gpu/drm/msm/hdmi/hdmi.c
index a968cad509c2..17e069a133a4 100644
--- a/drivers/gpu/drm/msm/hdmi/hdmi.c
+++ b/drivers/gpu/drm/msm/hdmi/hdmi.c
@@ -239,6 +239,8 @@ static struct hdmi *msm_hdmi_init(struct platform_device *pdev)
 		hdmi->pwr_clks[i] = clk;
 	}
 
+	pm_runtime_enable(&pdev->dev);
+
 	hdmi->workq = alloc_ordered_workqueue("msm_hdmi", 0);
 
 	hdmi->i2c = msm_hdmi_i2c_init(hdmi);
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi_bridge.c b/drivers/gpu/drm/msm/hdmi/hdmi_bridge.c
index ae40e7179d4f..7e357077ed26 100644
--- a/drivers/gpu/drm/msm/hdmi/hdmi_bridge.c
+++ b/drivers/gpu/drm/msm/hdmi/hdmi_bridge.c
@@ -35,6 +35,8 @@ static void msm_hdmi_power_on(struct drm_bridge *bridge)
 	const struct hdmi_platform_config *config = hdmi->config;
 	int i, ret;
 
+	pm_runtime_get_sync(&hdmi->pdev->dev);
+
 	for (i = 0; i < config->pwr_reg_cnt; i++) {
 		ret = regulator_enable(hdmi->pwr_regs[i]);
 		if (ret) {
@@ -84,6 +86,8 @@ static void power_off(struct drm_bridge *bridge)
 					config->pwr_reg_names[i], ret);
 		}
 	}
+
+	pm_runtime_put_autosuspend(&hdmi->pdev->dev);
 }
 
 #define AVI_IFRAME_LINE_NUMBER 1
@@ -97,7 +101,7 @@ static void msm_hdmi_config_avi_infoframe(struct hdmi *hdmi)
 	u32 val;
 	int len;
 
-	drm_hdmi_avi_infoframe_from_display_mode(&frame.avi, mode);
+	drm_hdmi_avi_infoframe_from_display_mode(&frame.avi, mode, false);
 
 	len = hdmi_infoframe_pack(&frame, buffer, sizeof(buffer));
 	if (len < 0) {
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi_connector.c b/drivers/gpu/drm/msm/hdmi/hdmi_connector.c
index a2515b466ce5..c0848dfedd50 100644
--- a/drivers/gpu/drm/msm/hdmi/hdmi_connector.c
+++ b/drivers/gpu/drm/msm/hdmi/hdmi_connector.c
@@ -137,6 +137,36 @@ err:
 	return ret;
 }
 
+static void enable_hpd_clocks(struct hdmi *hdmi, bool enable)
+{
+	const struct hdmi_platform_config *config = hdmi->config;
+	struct device *dev = &hdmi->pdev->dev;
+	int i, ret;
+
+	if (enable) {
+		for (i = 0; i < config->hpd_clk_cnt; i++) {
+			if (config->hpd_freq && config->hpd_freq[i]) {
+				ret = clk_set_rate(hdmi->hpd_clks[i],
+						   config->hpd_freq[i]);
+				if (ret)
+					dev_warn(dev,
+						 "failed to set clk %s (%d)\n",
+						 config->hpd_clk_names[i], ret);
+			}
+
+			ret = clk_prepare_enable(hdmi->hpd_clks[i]);
+			if (ret) {
+				dev_err(dev,
+					"failed to enable hpd clk: %s (%d)\n",
+					config->hpd_clk_names[i], ret);
+			}
+		}
+	} else {
+		for (i = config->hpd_clk_cnt - 1; i >= 0; i--)
+			clk_disable_unprepare(hdmi->hpd_clks[i]);
+	}
+}
+
 static int hpd_enable(struct hdmi_connector *hdmi_connector)
 {
 	struct hdmi *hdmi = hdmi_connector->hdmi;
@@ -167,22 +197,8 @@ static int hpd_enable(struct hdmi_connector *hdmi_connector)
 		goto fail;
 	}
 
-	for (i = 0; i < config->hpd_clk_cnt; i++) {
-		if (config->hpd_freq && config->hpd_freq[i]) {
-			ret = clk_set_rate(hdmi->hpd_clks[i],
-					config->hpd_freq[i]);
-			if (ret)
-				dev_warn(dev, "failed to set clk %s (%d)\n",
-						config->hpd_clk_names[i], ret);
-		}
-
-		ret = clk_prepare_enable(hdmi->hpd_clks[i]);
-		if (ret) {
-			dev_err(dev, "failed to enable hpd clk: %s (%d)\n",
-					config->hpd_clk_names[i], ret);
-			goto fail;
-		}
-	}
+	pm_runtime_get_sync(dev);
+	enable_hpd_clocks(hdmi, true);
 
 	msm_hdmi_set_mode(hdmi, false);
 	msm_hdmi_phy_reset(hdmi);
@@ -225,8 +241,8 @@ static void hdp_disable(struct hdmi_connector *hdmi_connector)
 
 	msm_hdmi_set_mode(hdmi, false);
 
-	for (i = 0; i < config->hpd_clk_cnt; i++)
-		clk_disable_unprepare(hdmi->hpd_clks[i]);
+	enable_hpd_clocks(hdmi, false);
+	pm_runtime_put_autosuspend(dev);
 
 	ret = gpio_config(hdmi, false);
 	if (ret)
@@ -285,7 +301,16 @@ void msm_hdmi_connector_irq(struct drm_connector *connector)
 
 static enum drm_connector_status detect_reg(struct hdmi *hdmi)
 {
-	uint32_t hpd_int_status = hdmi_read(hdmi, REG_HDMI_HPD_INT_STATUS);
+	uint32_t hpd_int_status;
+
+	pm_runtime_get_sync(&hdmi->pdev->dev);
+	enable_hpd_clocks(hdmi, true);
+
+	hpd_int_status = hdmi_read(hdmi, REG_HDMI_HPD_INT_STATUS);
+
+	enable_hpd_clocks(hdmi, false);
+	pm_runtime_put_autosuspend(&hdmi->pdev->dev);
+
 	return (hpd_int_status & HDMI_HPD_INT_STATUS_CABLE_DETECTED) ?
 			connector_status_connected : connector_status_disconnected;
 }
@@ -407,7 +432,6 @@ static int msm_hdmi_connector_mode_valid(struct drm_connector *connector,
 }
 
 static const struct drm_connector_funcs hdmi_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.detect = hdmi_connector_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = hdmi_connector_destroy,
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_crtc.c b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_crtc.c
index 615e1def64d9..47fa2aba1983 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_crtc.c
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_crtc.c
@@ -279,7 +279,8 @@ static void mdp4_crtc_mode_set_nofb(struct drm_crtc *crtc)
 	}
 }
 
-static void mdp4_crtc_disable(struct drm_crtc *crtc)
+static void mdp4_crtc_atomic_disable(struct drm_crtc *crtc,
+				     struct drm_crtc_state *old_state)
 {
 	struct mdp4_crtc *mdp4_crtc = to_mdp4_crtc(crtc);
 	struct mdp4_kms *mdp4_kms = get_kms(crtc);
@@ -295,7 +296,8 @@ static void mdp4_crtc_disable(struct drm_crtc *crtc)
 	mdp4_crtc->enabled = false;
 }
 
-static void mdp4_crtc_enable(struct drm_crtc *crtc)
+static void mdp4_crtc_atomic_enable(struct drm_crtc *crtc,
+				    struct drm_crtc_state *old_state)
 {
 	struct mdp4_crtc *mdp4_crtc = to_mdp4_crtc(crtc);
 	struct mdp4_kms *mdp4_kms = get_kms(crtc);
@@ -482,7 +484,6 @@ static const struct drm_crtc_funcs mdp4_crtc_funcs = {
 	.set_config = drm_atomic_helper_set_config,
 	.destroy = mdp4_crtc_destroy,
 	.page_flip = drm_atomic_helper_page_flip,
-	.set_property = drm_atomic_helper_crtc_set_property,
 	.cursor_set = mdp4_crtc_cursor_set,
 	.cursor_move = mdp4_crtc_cursor_move,
 	.reset = drm_atomic_helper_crtc_reset,
@@ -492,11 +493,11 @@ static const struct drm_crtc_funcs mdp4_crtc_funcs = {
 
 static const struct drm_crtc_helper_funcs mdp4_crtc_helper_funcs = {
 	.mode_set_nofb = mdp4_crtc_mode_set_nofb,
-	.disable = mdp4_crtc_disable,
-	.enable = mdp4_crtc_enable,
 	.atomic_check = mdp4_crtc_atomic_check,
 	.atomic_begin = mdp4_crtc_atomic_begin,
 	.atomic_flush = mdp4_crtc_atomic_flush,
+	.atomic_enable = mdp4_crtc_atomic_enable,
+	.atomic_disable = mdp4_crtc_atomic_disable,
 };
 
 static void mdp4_crtc_vblank_irq(struct mdp_irq *irq, uint32_t irqstatus)
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.c b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.c
index bcd1f5cac72c..f7f087419ed8 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.c
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.c
@@ -114,7 +114,7 @@ static void mdp4_prepare_commit(struct msm_kms *kms, struct drm_atomic_state *st
 	mdp4_enable(mdp4_kms);
 
 	/* see 119ecb7fd */
-	for_each_crtc_in_state(state, crtc, crtc_state, i)
+	for_each_new_crtc_in_state(state, crtc, crtc_state, i)
 		drm_crtc_vblank_get(crtc);
 }
 
@@ -126,7 +126,7 @@ static void mdp4_complete_commit(struct msm_kms *kms, struct drm_atomic_state *s
 	struct drm_crtc_state *crtc_state;
 
 	/* see 119ecb7fd */
-	for_each_crtc_in_state(state, crtc, crtc_state, i)
+	for_each_new_crtc_in_state(state, crtc, crtc_state, i)
 		drm_crtc_vblank_put(crtc);
 
 	mdp4_disable(mdp4_kms);
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_lvds_connector.c b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_lvds_connector.c
index 353429b05733..e3b1c86b7aae 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_lvds_connector.c
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_lvds_connector.c
@@ -91,7 +91,6 @@ static int mdp4_lvds_connector_mode_valid(struct drm_connector *connector,
 }
 
 static const struct drm_connector_funcs mdp4_lvds_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.detect = mdp4_lvds_connector_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = mdp4_lvds_connector_destroy,
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_plane.c b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_plane.c
index a20e3d644523..7a1ad3af08e3 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_plane.c
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_plane.c
@@ -401,7 +401,7 @@ struct drm_plane *mdp4_plane_init(struct drm_device *dev,
 	type = private_plane ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY;
 	ret = drm_universal_plane_init(dev, plane, 0xff, &mdp4_plane_funcs,
 				 mdp4_plane->formats, mdp4_plane->nformats,
-				 type, NULL);
+				 NULL, type, NULL);
 	if (ret)
 		goto fail;
 
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_cmd_encoder.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_cmd_encoder.c
index aa7402e03f67..60790df91bfa 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_cmd_encoder.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_cmd_encoder.c
@@ -192,6 +192,7 @@ int mdp5_cmd_encoder_set_split_display(struct drm_encoder *encoder,
 {
 	struct mdp5_encoder *mdp5_cmd_enc = to_mdp5_encoder(encoder);
 	struct mdp5_kms *mdp5_kms;
+	struct device *dev;
 	int intf_num;
 	u32 data = 0;
 
@@ -214,14 +215,16 @@ int mdp5_cmd_encoder_set_split_display(struct drm_encoder *encoder,
 	/* Smart Panel, Sync mode */
 	data |= MDP5_SPLIT_DPL_UPPER_SMART_PANEL;
 
+	dev = &mdp5_kms->pdev->dev;
+
 	/* Make sure clocks are on when connectors calling this function. */
-	mdp5_enable(mdp5_kms);
+	pm_runtime_get_sync(dev);
 	mdp5_write(mdp5_kms, REG_MDP5_SPLIT_DPL_UPPER, data);
 
 	mdp5_write(mdp5_kms, REG_MDP5_SPLIT_DPL_LOWER,
 		   MDP5_SPLIT_DPL_LOWER_SMART_PANEL);
 	mdp5_write(mdp5_kms, REG_MDP5_SPLIT_DPL_EN, 1);
-	mdp5_disable(mdp5_kms);
+	pm_runtime_put_autosuspend(dev);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_crtc.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_crtc.c
index 735a87a699fa..6fcb58ab718c 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_crtc.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_crtc.c
@@ -409,11 +409,13 @@ static void mdp5_crtc_mode_set_nofb(struct drm_crtc *crtc)
 	spin_unlock_irqrestore(&mdp5_crtc->lm_lock, flags);
 }
 
-static void mdp5_crtc_disable(struct drm_crtc *crtc)
+static void mdp5_crtc_atomic_disable(struct drm_crtc *crtc,
+				     struct drm_crtc_state *old_state)
 {
 	struct mdp5_crtc *mdp5_crtc = to_mdp5_crtc(crtc);
 	struct mdp5_crtc_state *mdp5_cstate = to_mdp5_crtc_state(crtc->state);
 	struct mdp5_kms *mdp5_kms = get_kms(crtc);
+	struct device *dev = &mdp5_kms->pdev->dev;
 
 	DBG("%s", crtc->name);
 
@@ -424,23 +426,28 @@ static void mdp5_crtc_disable(struct drm_crtc *crtc)
 		mdp_irq_unregister(&mdp5_kms->base, &mdp5_crtc->pp_done);
 
 	mdp_irq_unregister(&mdp5_kms->base, &mdp5_crtc->err);
-	mdp5_disable(mdp5_kms);
+	pm_runtime_put_autosuspend(dev);
 
 	mdp5_crtc->enabled = false;
 }
 
-static void mdp5_crtc_enable(struct drm_crtc *crtc)
+static void mdp5_crtc_atomic_enable(struct drm_crtc *crtc,
+				    struct drm_crtc_state *old_state)
 {
 	struct mdp5_crtc *mdp5_crtc = to_mdp5_crtc(crtc);
 	struct mdp5_crtc_state *mdp5_cstate = to_mdp5_crtc_state(crtc->state);
 	struct mdp5_kms *mdp5_kms = get_kms(crtc);
+	struct device *dev = &mdp5_kms->pdev->dev;
 
 	DBG("%s", crtc->name);
 
 	if (WARN_ON(mdp5_crtc->enabled))
 		return;
 
-	mdp5_enable(mdp5_kms);
+	pm_runtime_get_sync(dev);
+
+	mdp5_crtc_mode_set_nofb(crtc);
+
 	mdp_irq_register(&mdp5_kms->base, &mdp5_crtc->err);
 
 	if (mdp5_cstate->cmd_mode)
@@ -531,7 +538,7 @@ static bool is_fullscreen(struct drm_crtc_state *cstate,
 		((pstate->crtc_y + pstate->crtc_h) >= cstate->mode.vdisplay);
 }
 
-enum mdp_mixer_stage_id get_start_stage(struct drm_crtc *crtc,
+static enum mdp_mixer_stage_id get_start_stage(struct drm_crtc *crtc,
 					struct drm_crtc_state *new_crtc_state,
 					struct drm_plane_state *bpstate)
 {
@@ -725,6 +732,7 @@ static int mdp5_crtc_cursor_set(struct drm_crtc *crtc,
 	struct mdp5_pipeline *pipeline = &mdp5_cstate->pipeline;
 	struct drm_device *dev = crtc->dev;
 	struct mdp5_kms *mdp5_kms = get_kms(crtc);
+	struct platform_device *pdev = mdp5_kms->pdev;
 	struct msm_kms *kms = &mdp5_kms->base.base;
 	struct drm_gem_object *cursor_bo, *old_bo = NULL;
 	uint32_t blendcfg, stride;
@@ -753,7 +761,7 @@ static int mdp5_crtc_cursor_set(struct drm_crtc *crtc,
 	if (!handle) {
 		DBG("Cursor off");
 		cursor_enable = false;
-		mdp5_enable(mdp5_kms);
+		pm_runtime_get_sync(&pdev->dev);
 		goto set_cursor;
 	}
 
@@ -768,6 +776,8 @@ static int mdp5_crtc_cursor_set(struct drm_crtc *crtc,
 	lm = mdp5_cstate->pipeline.mixer->lm;
 	stride = width * drm_format_plane_cpp(DRM_FORMAT_ARGB8888, 0);
 
+	pm_runtime_get_sync(&pdev->dev);
+
 	spin_lock_irqsave(&mdp5_crtc->cursor.lock, flags);
 	old_bo = mdp5_crtc->cursor.scanout_bo;
 
@@ -777,8 +787,6 @@ static int mdp5_crtc_cursor_set(struct drm_crtc *crtc,
 
 	get_roi(crtc, &roi_w, &roi_h);
 
-	mdp5_enable(mdp5_kms);
-
 	mdp5_write(mdp5_kms, REG_MDP5_LM_CURSOR_STRIDE(lm), stride);
 	mdp5_write(mdp5_kms, REG_MDP5_LM_CURSOR_FORMAT(lm),
 			MDP5_LM_CURSOR_FORMAT_FORMAT(CURSOR_FMT_ARGB8888));
@@ -796,6 +804,8 @@ static int mdp5_crtc_cursor_set(struct drm_crtc *crtc,
 
 	spin_unlock_irqrestore(&mdp5_crtc->cursor.lock, flags);
 
+	pm_runtime_put_autosuspend(&pdev->dev);
+
 set_cursor:
 	ret = mdp5_ctl_set_cursor(ctl, pipeline, 0, cursor_enable);
 	if (ret) {
@@ -807,7 +817,7 @@ set_cursor:
 	crtc_flush(crtc, flush_mask);
 
 end:
-	mdp5_disable(mdp5_kms);
+	pm_runtime_put_autosuspend(&pdev->dev);
 	if (old_bo) {
 		drm_flip_work_queue(&mdp5_crtc->unref_cursor_work, old_bo);
 		/* enable vblank to complete cursor work: */
@@ -840,7 +850,7 @@ static int mdp5_crtc_cursor_move(struct drm_crtc *crtc, int x, int y)
 
 	get_roi(crtc, &roi_w, &roi_h);
 
-	mdp5_enable(mdp5_kms);
+	pm_runtime_get_sync(&mdp5_kms->pdev->dev);
 
 	spin_lock_irqsave(&mdp5_crtc->cursor.lock, flags);
 	mdp5_write(mdp5_kms, REG_MDP5_LM_CURSOR_SIZE(lm),
@@ -853,7 +863,7 @@ static int mdp5_crtc_cursor_move(struct drm_crtc *crtc, int x, int y)
 
 	crtc_flush(crtc, flush_mask);
 
-	mdp5_disable(mdp5_kms);
+	pm_runtime_put_autosuspend(&mdp5_kms->pdev->dev);
 
 	return 0;
 }
@@ -925,7 +935,6 @@ static const struct drm_crtc_funcs mdp5_crtc_funcs = {
 	.set_config = drm_atomic_helper_set_config,
 	.destroy = mdp5_crtc_destroy,
 	.page_flip = drm_atomic_helper_page_flip,
-	.set_property = drm_atomic_helper_crtc_set_property,
 	.reset = mdp5_crtc_reset,
 	.atomic_duplicate_state = mdp5_crtc_duplicate_state,
 	.atomic_destroy_state = mdp5_crtc_destroy_state,
@@ -938,7 +947,6 @@ static const struct drm_crtc_funcs mdp5_crtc_no_lm_cursor_funcs = {
 	.set_config = drm_atomic_helper_set_config,
 	.destroy = mdp5_crtc_destroy,
 	.page_flip = drm_atomic_helper_page_flip,
-	.set_property = drm_atomic_helper_crtc_set_property,
 	.reset = mdp5_crtc_reset,
 	.atomic_duplicate_state = mdp5_crtc_duplicate_state,
 	.atomic_destroy_state = mdp5_crtc_destroy_state,
@@ -947,11 +955,11 @@ static const struct drm_crtc_funcs mdp5_crtc_no_lm_cursor_funcs = {
 
 static const struct drm_crtc_helper_funcs mdp5_crtc_helper_funcs = {
 	.mode_set_nofb = mdp5_crtc_mode_set_nofb,
-	.disable = mdp5_crtc_disable,
-	.enable = mdp5_crtc_enable,
 	.atomic_check = mdp5_crtc_atomic_check,
 	.atomic_begin = mdp5_crtc_atomic_begin,
 	.atomic_flush = mdp5_crtc_atomic_flush,
+	.atomic_enable = mdp5_crtc_atomic_enable,
+	.atomic_disable = mdp5_crtc_atomic_disable,
 };
 
 static void mdp5_crtc_vblank_irq(struct mdp_irq *irq, uint32_t irqstatus)
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_encoder.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_encoder.c
index 70bef51245af..5b851380d3f2 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_encoder.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_encoder.c
@@ -297,6 +297,10 @@ static void mdp5_encoder_enable(struct drm_encoder *encoder)
 {
 	struct mdp5_encoder *mdp5_encoder = to_mdp5_encoder(encoder);
 	struct mdp5_interface *intf = mdp5_encoder->intf;
+	/* this isn't right I think */
+	struct drm_crtc_state *cstate = encoder->crtc->state;
+
+	mdp5_encoder_mode_set(encoder, &cstate->mode, &cstate->adjusted_mode);
 
 	if (intf->mode == MDP5_INTF_DSI_MODE_COMMAND)
 		mdp5_cmd_encoder_enable(encoder);
@@ -320,7 +324,6 @@ static int mdp5_encoder_atomic_check(struct drm_encoder *encoder,
 }
 
 static const struct drm_encoder_helper_funcs mdp5_encoder_helper_funcs = {
-	.mode_set = mdp5_encoder_mode_set,
 	.disable = mdp5_encoder_disable,
 	.enable = mdp5_encoder_enable,
 	.atomic_check = mdp5_encoder_atomic_check,
@@ -350,6 +353,7 @@ int mdp5_vid_encoder_set_split_display(struct drm_encoder *encoder,
 	struct mdp5_encoder *mdp5_encoder = to_mdp5_encoder(encoder);
 	struct mdp5_encoder *mdp5_slave_enc = to_mdp5_encoder(slave_encoder);
 	struct mdp5_kms *mdp5_kms;
+	struct device *dev;
 	int intf_num;
 	u32 data = 0;
 
@@ -369,8 +373,10 @@ int mdp5_vid_encoder_set_split_display(struct drm_encoder *encoder,
 	else
 		return -EINVAL;
 
+	dev = &mdp5_kms->pdev->dev;
 	/* Make sure clocks are on when connectors calling this function. */
-	mdp5_enable(mdp5_kms);
+	pm_runtime_get_sync(dev);
+
 	/* Dumb Panel, Sync mode */
 	mdp5_write(mdp5_kms, REG_MDP5_SPLIT_DPL_UPPER, 0);
 	mdp5_write(mdp5_kms, REG_MDP5_SPLIT_DPL_LOWER, data);
@@ -378,7 +384,7 @@ int mdp5_vid_encoder_set_split_display(struct drm_encoder *encoder,
 
 	mdp5_ctl_pair(mdp5_encoder->ctl, mdp5_slave_enc->ctl, true);
 
-	mdp5_disable(mdp5_kms);
+	pm_runtime_put_autosuspend(dev);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_irq.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_irq.c
index 3ce8b9dec9c1..bb5deb00c899 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_irq.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_irq.c
@@ -49,16 +49,19 @@ static void mdp5_irq_error_handler(struct mdp_irq *irq, uint32_t irqstatus)
 void mdp5_irq_preinstall(struct msm_kms *kms)
 {
 	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(kms));
-	mdp5_enable(mdp5_kms);
+	struct device *dev = &mdp5_kms->pdev->dev;
+
+	pm_runtime_get_sync(dev);
 	mdp5_write(mdp5_kms, REG_MDP5_INTR_CLEAR, 0xffffffff);
 	mdp5_write(mdp5_kms, REG_MDP5_INTR_EN, 0x00000000);
-	mdp5_disable(mdp5_kms);
+	pm_runtime_put_autosuspend(dev);
 }
 
 int mdp5_irq_postinstall(struct msm_kms *kms)
 {
 	struct mdp_kms *mdp_kms = to_mdp_kms(kms);
 	struct mdp5_kms *mdp5_kms = to_mdp5_kms(mdp_kms);
+	struct device *dev = &mdp5_kms->pdev->dev;
 	struct mdp_irq *error_handler = &mdp5_kms->error_handler;
 
 	error_handler->irq = mdp5_irq_error_handler;
@@ -67,9 +70,9 @@ int mdp5_irq_postinstall(struct msm_kms *kms)
 			MDP5_IRQ_INTF2_UNDER_RUN |
 			MDP5_IRQ_INTF3_UNDER_RUN;
 
-	mdp5_enable(mdp5_kms);
+	pm_runtime_get_sync(dev);
 	mdp_irq_register(mdp_kms, error_handler);
-	mdp5_disable(mdp5_kms);
+	pm_runtime_put_autosuspend(dev);
 
 	return 0;
 }
@@ -77,9 +80,11 @@ int mdp5_irq_postinstall(struct msm_kms *kms)
 void mdp5_irq_uninstall(struct msm_kms *kms)
 {
 	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(kms));
-	mdp5_enable(mdp5_kms);
+	struct device *dev = &mdp5_kms->pdev->dev;
+
+	pm_runtime_get_sync(dev);
 	mdp5_write(mdp5_kms, REG_MDP5_INTR_EN, 0x00000000);
-	mdp5_disable(mdp5_kms);
+	pm_runtime_put_autosuspend(dev);
 }
 
 irqreturn_t mdp5_irq(struct msm_kms *kms)
@@ -109,11 +114,12 @@ irqreturn_t mdp5_irq(struct msm_kms *kms)
 int mdp5_enable_vblank(struct msm_kms *kms, struct drm_crtc *crtc)
 {
 	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(kms));
+	struct device *dev = &mdp5_kms->pdev->dev;
 
-	mdp5_enable(mdp5_kms);
+	pm_runtime_get_sync(dev);
 	mdp_update_vblank_mask(to_mdp_kms(kms),
 			mdp5_crtc_vblank(crtc), true);
-	mdp5_disable(mdp5_kms);
+	pm_runtime_put_autosuspend(dev);
 
 	return 0;
 }
@@ -121,9 +127,10 @@ int mdp5_enable_vblank(struct msm_kms *kms, struct drm_crtc *crtc)
 void mdp5_disable_vblank(struct msm_kms *kms, struct drm_crtc *crtc)
 {
 	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(kms));
+	struct device *dev = &mdp5_kms->pdev->dev;
 
-	mdp5_enable(mdp5_kms);
+	pm_runtime_get_sync(dev);
 	mdp_update_vblank_mask(to_mdp_kms(kms),
 			mdp5_crtc_vblank(crtc), false);
-	mdp5_disable(mdp5_kms);
+	pm_runtime_put_autosuspend(dev);
 }
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.c
index 1c603aef3c59..f7c0698fec40 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.c
@@ -30,11 +30,10 @@ static const char *iommu_ports[] = {
 static int mdp5_hw_init(struct msm_kms *kms)
 {
 	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(kms));
-	struct platform_device *pdev = mdp5_kms->pdev;
+	struct device *dev = &mdp5_kms->pdev->dev;
 	unsigned long flags;
 
-	pm_runtime_get_sync(&pdev->dev);
-	mdp5_enable(mdp5_kms);
+	pm_runtime_get_sync(dev);
 
 	/* Magic unknown register writes:
 	 *
@@ -66,8 +65,7 @@ static int mdp5_hw_init(struct msm_kms *kms)
 
 	mdp5_ctlm_hw_reset(mdp5_kms->ctlm);
 
-	mdp5_disable(mdp5_kms);
-	pm_runtime_put_sync(&pdev->dev);
+	pm_runtime_put_sync(dev);
 
 	return 0;
 }
@@ -111,8 +109,9 @@ static void mdp5_swap_state(struct msm_kms *kms, struct drm_atomic_state *state)
 static void mdp5_prepare_commit(struct msm_kms *kms, struct drm_atomic_state *state)
 {
 	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(kms));
+	struct device *dev = &mdp5_kms->pdev->dev;
 
-	mdp5_enable(mdp5_kms);
+	pm_runtime_get_sync(dev);
 
 	if (mdp5_kms->smp)
 		mdp5_smp_prepare_commit(mdp5_kms->smp, &mdp5_kms->state->smp);
@@ -121,11 +120,12 @@ static void mdp5_prepare_commit(struct msm_kms *kms, struct drm_atomic_state *st
 static void mdp5_complete_commit(struct msm_kms *kms, struct drm_atomic_state *state)
 {
 	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(kms));
+	struct device *dev = &mdp5_kms->pdev->dev;
 
 	if (mdp5_kms->smp)
 		mdp5_smp_complete_commit(mdp5_kms->smp, &mdp5_kms->state->smp);
 
-	mdp5_disable(mdp5_kms);
+	pm_runtime_put_autosuspend(dev);
 }
 
 static void mdp5_wait_for_crtc_commit_done(struct msm_kms *kms,
@@ -249,6 +249,9 @@ int mdp5_disable(struct mdp5_kms *mdp5_kms)
 {
 	DBG("");
 
+	mdp5_kms->enable_count--;
+	WARN_ON(mdp5_kms->enable_count < 0);
+
 	clk_disable_unprepare(mdp5_kms->ahb_clk);
 	clk_disable_unprepare(mdp5_kms->axi_clk);
 	clk_disable_unprepare(mdp5_kms->core_clk);
@@ -262,6 +265,8 @@ int mdp5_enable(struct mdp5_kms *mdp5_kms)
 {
 	DBG("");
 
+	mdp5_kms->enable_count++;
+
 	clk_prepare_enable(mdp5_kms->ahb_clk);
 	clk_prepare_enable(mdp5_kms->axi_clk);
 	clk_prepare_enable(mdp5_kms->core_clk);
@@ -486,11 +491,12 @@ fail:
 static void read_mdp_hw_revision(struct mdp5_kms *mdp5_kms,
 				 u32 *major, u32 *minor)
 {
+	struct device *dev = &mdp5_kms->pdev->dev;
 	u32 version;
 
-	mdp5_enable(mdp5_kms);
+	pm_runtime_get_sync(dev);
 	version = mdp5_read(mdp5_kms, REG_MDP5_HW_VERSION);
-	mdp5_disable(mdp5_kms);
+	pm_runtime_put_autosuspend(dev);
 
 	*major = FIELD(version, MDP5_HW_VERSION_MAJOR);
 	*minor = FIELD(version, MDP5_HW_VERSION_MINOR);
@@ -643,7 +649,7 @@ struct msm_kms *mdp5_kms_init(struct drm_device *dev)
 	 * have left things on, in which case we'll start getting faults if
 	 * we don't disable):
 	 */
-	mdp5_enable(mdp5_kms);
+	pm_runtime_get_sync(&pdev->dev);
 	for (i = 0; i < MDP5_INTF_NUM_MAX; i++) {
 		if (mdp5_cfg_intf_is_virtual(config->hw->intf.connect[i]) ||
 		    !config->hw->intf.base[i])
@@ -652,7 +658,6 @@ struct msm_kms *mdp5_kms_init(struct drm_device *dev)
 
 		mdp5_write(mdp5_kms, REG_MDP5_INTF_FRAME_LINE_COUNT_EN(i), 0x3);
 	}
-	mdp5_disable(mdp5_kms);
 	mdelay(16);
 
 	if (config->platform.iommu) {
@@ -678,6 +683,8 @@ struct msm_kms *mdp5_kms_init(struct drm_device *dev)
 		aspace = NULL;;
 	}
 
+	pm_runtime_put_autosuspend(&pdev->dev);
+
 	ret = modeset_init(mdp5_kms);
 	if (ret) {
 		dev_err(&pdev->dev, "modeset_init failed: %d\n", ret);
@@ -1005,6 +1012,30 @@ static int mdp5_dev_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static __maybe_unused int mdp5_runtime_suspend(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct mdp5_kms *mdp5_kms = platform_get_drvdata(pdev);
+
+	DBG("");
+
+	return mdp5_disable(mdp5_kms);
+}
+
+static __maybe_unused int mdp5_runtime_resume(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct mdp5_kms *mdp5_kms = platform_get_drvdata(pdev);
+
+	DBG("");
+
+	return mdp5_enable(mdp5_kms);
+}
+
+static const struct dev_pm_ops mdp5_pm_ops = {
+	SET_RUNTIME_PM_OPS(mdp5_runtime_suspend, mdp5_runtime_resume, NULL)
+};
+
 static const struct of_device_id mdp5_dt_match[] = {
 	{ .compatible = "qcom,mdp5", },
 	/* to support downstream DT files */
@@ -1019,6 +1050,7 @@ static struct platform_driver mdp5_driver = {
 	.driver = {
 		.name = "msm_mdp",
 		.of_match_table = mdp5_dt_match,
+		.pm = &mdp5_pm_ops,
 	},
 };
 
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.h b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.h
index 17caa0e8c8ae..9b3fe01089d1 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.h
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.h
@@ -76,6 +76,8 @@ struct mdp5_kms {
 	bool rpm_enabled;
 
 	struct mdp_irq error_handler;
+
+	int enable_count;
 };
 #define to_mdp5_kms(x) container_of(x, struct mdp5_kms, base)
 
@@ -167,11 +169,13 @@ struct mdp5_encoder {
 
 static inline void mdp5_write(struct mdp5_kms *mdp5_kms, u32 reg, u32 data)
 {
+	WARN_ON(mdp5_kms->enable_count <= 0);
 	msm_writel(data, mdp5_kms->mmio + reg);
 }
 
 static inline u32 mdp5_read(struct mdp5_kms *mdp5_kms, u32 reg)
 {
+	WARN_ON(mdp5_kms->enable_count <= 0);
 	return msm_readl(mdp5_kms->mmio + reg);
 }
 
@@ -255,9 +259,6 @@ static inline uint32_t lm2ppdone(struct mdp5_hw_mixer *mixer)
 	return MDP5_IRQ_PING_PONG_0_DONE << mixer->pp;
 }
 
-int mdp5_disable(struct mdp5_kms *mdp5_kms);
-int mdp5_enable(struct mdp5_kms *mdp5_kms);
-
 void mdp5_set_irqmask(struct mdp_kms *mdp_kms, uint32_t irqmask,
 		uint32_t old_irqmask);
 void mdp5_irq_preinstall(struct msm_kms *kms);
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_mdss.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_mdss.c
index 9c34d7824988..f2a0db7a8a03 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_mdss.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_mdss.c
@@ -31,6 +31,10 @@ struct msm_mdss {
 
 	struct regulator *vdd;
 
+	struct clk *ahb_clk;
+	struct clk *axi_clk;
+	struct clk *vsync_clk;
+
 	struct {
 		volatile unsigned long enabled_mask;
 		struct irq_domain *domain;
@@ -140,6 +144,51 @@ static int mdss_irq_domain_init(struct msm_mdss *mdss)
 	return 0;
 }
 
+int msm_mdss_enable(struct msm_mdss *mdss)
+{
+	DBG("");
+
+	clk_prepare_enable(mdss->ahb_clk);
+	if (mdss->axi_clk)
+		clk_prepare_enable(mdss->axi_clk);
+	if (mdss->vsync_clk)
+		clk_prepare_enable(mdss->vsync_clk);
+
+	return 0;
+}
+
+int msm_mdss_disable(struct msm_mdss *mdss)
+{
+	DBG("");
+
+	if (mdss->vsync_clk)
+		clk_disable_unprepare(mdss->vsync_clk);
+	if (mdss->axi_clk)
+		clk_disable_unprepare(mdss->axi_clk);
+	clk_disable_unprepare(mdss->ahb_clk);
+
+	return 0;
+}
+
+static int msm_mdss_get_clocks(struct msm_mdss *mdss)
+{
+	struct platform_device *pdev = to_platform_device(mdss->dev->dev);
+
+	mdss->ahb_clk = msm_clk_get(pdev, "iface");
+	if (IS_ERR(mdss->ahb_clk))
+		mdss->ahb_clk = NULL;
+
+	mdss->axi_clk = msm_clk_get(pdev, "bus");
+	if (IS_ERR(mdss->axi_clk))
+		mdss->axi_clk = NULL;
+
+	mdss->vsync_clk = msm_clk_get(pdev, "vsync");
+	if (IS_ERR(mdss->vsync_clk))
+		mdss->vsync_clk = NULL;
+
+	return 0;
+}
+
 void msm_mdss_destroy(struct drm_device *dev)
 {
 	struct msm_drm_private *priv = dev->dev_private;
@@ -153,8 +202,6 @@ void msm_mdss_destroy(struct drm_device *dev)
 
 	regulator_disable(mdss->vdd);
 
-	pm_runtime_put_sync(dev->dev);
-
 	pm_runtime_disable(dev->dev);
 }
 
@@ -190,6 +237,12 @@ int msm_mdss_init(struct drm_device *dev)
 		goto fail;
 	}
 
+	ret = msm_mdss_get_clocks(mdss);
+	if (ret) {
+		dev_err(dev->dev, "failed to get clocks: %d\n", ret);
+		goto fail;
+	}
+
 	/* Regulator to enable GDSCs in downstream kernels */
 	mdss->vdd = devm_regulator_get(dev->dev, "vdd");
 	if (IS_ERR(mdss->vdd)) {
@@ -221,12 +274,6 @@ int msm_mdss_init(struct drm_device *dev)
 
 	pm_runtime_enable(dev->dev);
 
-	/*
-	 * TODO: This is needed as the MDSS GDSC is only tied to MDSS's power
-	 * domain. Remove this once runtime PM is adapted for all the devices.
-	 */
-	pm_runtime_get_sync(dev->dev);
-
 	return 0;
 fail_irq:
 	regulator_disable(mdss->vdd);
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c
index 61f39c86dd09..4b22ac3413a1 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c
@@ -246,7 +246,6 @@ static const struct drm_plane_funcs mdp5_plane_funcs = {
 		.update_plane = drm_atomic_helper_update_plane,
 		.disable_plane = drm_atomic_helper_disable_plane,
 		.destroy = mdp5_plane_destroy,
-		.set_property = drm_atomic_helper_plane_set_property,
 		.atomic_set_property = mdp5_plane_atomic_set_property,
 		.atomic_get_property = mdp5_plane_atomic_get_property,
 		.reset = mdp5_plane_reset,
@@ -259,7 +258,6 @@ static const struct drm_plane_funcs mdp5_cursor_plane_funcs = {
 		.update_plane = mdp5_update_cursor_plane_legacy,
 		.disable_plane = drm_atomic_helper_disable_plane,
 		.destroy = mdp5_plane_destroy,
-		.set_property = drm_atomic_helper_plane_set_property,
 		.atomic_set_property = mdp5_plane_atomic_set_property,
 		.atomic_get_property = mdp5_plane_atomic_get_property,
 		.reset = mdp5_plane_reset,
@@ -1139,12 +1137,12 @@ struct drm_plane *mdp5_plane_init(struct drm_device *dev,
 		ret = drm_universal_plane_init(dev, plane, 0xff,
 				&mdp5_cursor_plane_funcs,
 				mdp5_plane->formats, mdp5_plane->nformats,
-				type, NULL);
+				NULL, type, NULL);
 	else
 		ret = drm_universal_plane_init(dev, plane, 0xff,
 				&mdp5_plane_funcs,
 				mdp5_plane->formats, mdp5_plane->nformats,
-				type, NULL);
+				NULL, type, NULL);
 	if (ret)
 		goto fail;
 
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_smp.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_smp.c
index 58f712d37e7f..ae4983d9d0a5 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_smp.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_smp.c
@@ -28,6 +28,13 @@ struct mdp5_smp {
 
 	int blk_cnt;
 	int blk_size;
+
+	/* register cache */
+	u32 alloc_w[22];
+	u32 alloc_r[22];
+	u32 pipe_reqprio_fifo_wm0[SSPP_MAX];
+	u32 pipe_reqprio_fifo_wm1[SSPP_MAX];
+	u32 pipe_reqprio_fifo_wm2[SSPP_MAX];
 };
 
 static inline
@@ -98,16 +105,15 @@ static int smp_request_block(struct mdp5_smp *smp,
 static void set_fifo_thresholds(struct mdp5_smp *smp,
 		enum mdp5_pipe pipe, int nblks)
 {
-	struct mdp5_kms *mdp5_kms = get_kms(smp);
 	u32 smp_entries_per_blk = smp->blk_size / (128 / BITS_PER_BYTE);
 	u32 val;
 
 	/* 1/4 of SMP pool that is being fetched */
 	val = (nblks * smp_entries_per_blk) / 4;
 
-	mdp5_write(mdp5_kms, REG_MDP5_PIPE_REQPRIO_FIFO_WM_0(pipe), val * 1);
-	mdp5_write(mdp5_kms, REG_MDP5_PIPE_REQPRIO_FIFO_WM_1(pipe), val * 2);
-	mdp5_write(mdp5_kms, REG_MDP5_PIPE_REQPRIO_FIFO_WM_2(pipe), val * 3);
+	smp->pipe_reqprio_fifo_wm0[pipe] = val * 1;
+	smp->pipe_reqprio_fifo_wm1[pipe] = val * 2;
+	smp->pipe_reqprio_fifo_wm2[pipe] = val * 3;
 }
 
 /*
@@ -222,7 +228,6 @@ void mdp5_smp_release(struct mdp5_smp *smp, struct mdp5_smp_state *state,
 static unsigned update_smp_state(struct mdp5_smp *smp,
 		u32 cid, mdp5_smp_state_t *assigned)
 {
-	struct mdp5_kms *mdp5_kms = get_kms(smp);
 	int cnt = smp->blk_cnt;
 	unsigned nblks = 0;
 	u32 blk, val;
@@ -231,7 +236,7 @@ static unsigned update_smp_state(struct mdp5_smp *smp,
 		int idx = blk / 3;
 		int fld = blk % 3;
 
-		val = mdp5_read(mdp5_kms, REG_MDP5_SMP_ALLOC_W_REG(idx));
+		val = smp->alloc_w[idx];
 
 		switch (fld) {
 		case 0:
@@ -248,8 +253,8 @@ static unsigned update_smp_state(struct mdp5_smp *smp,
 			break;
 		}
 
-		mdp5_write(mdp5_kms, REG_MDP5_SMP_ALLOC_W_REG(idx), val);
-		mdp5_write(mdp5_kms, REG_MDP5_SMP_ALLOC_R_REG(idx), val);
+		smp->alloc_w[idx] = val;
+		smp->alloc_r[idx] = val;
 
 		nblks++;
 	}
@@ -257,6 +262,39 @@ static unsigned update_smp_state(struct mdp5_smp *smp,
 	return nblks;
 }
 
+static void write_smp_alloc_regs(struct mdp5_smp *smp)
+{
+	struct mdp5_kms *mdp5_kms = get_kms(smp);
+	int i, num_regs;
+
+	num_regs = smp->blk_cnt / 3 + 1;
+
+	for (i = 0; i < num_regs; i++) {
+		mdp5_write(mdp5_kms, REG_MDP5_SMP_ALLOC_W_REG(i),
+			   smp->alloc_w[i]);
+		mdp5_write(mdp5_kms, REG_MDP5_SMP_ALLOC_R_REG(i),
+			   smp->alloc_r[i]);
+	}
+}
+
+static void write_smp_fifo_regs(struct mdp5_smp *smp)
+{
+	struct mdp5_kms *mdp5_kms = get_kms(smp);
+	int i;
+
+	for (i = 0; i < mdp5_kms->num_hwpipes; i++) {
+		struct mdp5_hw_pipe *hwpipe = mdp5_kms->hwpipes[i];
+		enum mdp5_pipe pipe = hwpipe->pipe;
+
+		mdp5_write(mdp5_kms, REG_MDP5_PIPE_REQPRIO_FIFO_WM_0(pipe),
+			   smp->pipe_reqprio_fifo_wm0[pipe]);
+		mdp5_write(mdp5_kms, REG_MDP5_PIPE_REQPRIO_FIFO_WM_1(pipe),
+			   smp->pipe_reqprio_fifo_wm1[pipe]);
+		mdp5_write(mdp5_kms, REG_MDP5_PIPE_REQPRIO_FIFO_WM_2(pipe),
+			   smp->pipe_reqprio_fifo_wm2[pipe]);
+	}
+}
+
 void mdp5_smp_prepare_commit(struct mdp5_smp *smp, struct mdp5_smp_state *state)
 {
 	enum mdp5_pipe pipe;
@@ -277,6 +315,9 @@ void mdp5_smp_prepare_commit(struct mdp5_smp *smp, struct mdp5_smp_state *state)
 		set_fifo_thresholds(smp, pipe, nblks);
 	}
 
+	write_smp_alloc_regs(smp);
+	write_smp_fifo_regs(smp);
+
 	state->assigned = 0;
 }
 
@@ -289,6 +330,8 @@ void mdp5_smp_complete_commit(struct mdp5_smp *smp, struct mdp5_smp_state *state
 		set_fifo_thresholds(smp, pipe, 0);
 	}
 
+	write_smp_fifo_regs(smp);
+
 	state->released = 0;
 }
 
diff --git a/drivers/gpu/drm/msm/msm_atomic.c b/drivers/gpu/drm/msm/msm_atomic.c
index 9633a68b14d7..025d454163b0 100644
--- a/drivers/gpu/drm/msm/msm_atomic.c
+++ b/drivers/gpu/drm/msm/msm_atomic.c
@@ -84,13 +84,13 @@ static void msm_atomic_wait_for_commit_done(struct drm_device *dev,
 		struct drm_atomic_state *old_state)
 {
 	struct drm_crtc *crtc;
-	struct drm_crtc_state *crtc_state;
+	struct drm_crtc_state *new_crtc_state;
 	struct msm_drm_private *priv = old_state->dev->dev_private;
 	struct msm_kms *kms = priv->kms;
 	int i;
 
-	for_each_crtc_in_state(old_state, crtc, crtc_state, i) {
-		if (!crtc->state->enable)
+	for_each_new_crtc_in_state(old_state, crtc, new_crtc_state, i) {
+		if (!new_crtc_state->active)
 			continue;
 
 		kms->funcs->wait_for_crtc_commit_done(kms, crtc);
@@ -195,7 +195,7 @@ int msm_atomic_commit(struct drm_device *dev,
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *crtc_state;
 	struct drm_plane *plane;
-	struct drm_plane_state *plane_state;
+	struct drm_plane_state *old_plane_state, *new_plane_state;
 	int i, ret;
 
 	ret = drm_atomic_helper_prepare_planes(dev, state);
@@ -211,19 +211,19 @@ int msm_atomic_commit(struct drm_device *dev,
 	/*
 	 * Figure out what crtcs we have:
 	 */
-	for_each_crtc_in_state(state, crtc, crtc_state, i)
+	for_each_new_crtc_in_state(state, crtc, crtc_state, i)
 		c->crtc_mask |= drm_crtc_mask(crtc);
 
 	/*
 	 * Figure out what fence to wait for:
 	 */
-	for_each_plane_in_state(state, plane, plane_state, i) {
-		if ((plane->state->fb != plane_state->fb) && plane_state->fb) {
-			struct drm_gem_object *obj = msm_framebuffer_bo(plane_state->fb, 0);
+	for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i) {
+		if ((new_plane_state->fb != old_plane_state->fb) && new_plane_state->fb) {
+			struct drm_gem_object *obj = msm_framebuffer_bo(new_plane_state->fb, 0);
 			struct msm_gem_object *msm_obj = to_msm_bo(obj);
 			struct dma_fence *fence = reservation_object_get_excl_rcu(msm_obj->resv);
 
-			drm_atomic_set_fence_for_plane(plane_state, fence);
+			drm_atomic_set_fence_for_plane(new_plane_state, fence);
 		}
 	}
 
@@ -232,20 +232,18 @@ int msm_atomic_commit(struct drm_device *dev,
 	 * mark our set of crtc's as busy:
 	 */
 	ret = start_atomic(dev->dev_private, c->crtc_mask);
-	if (ret) {
-		kfree(c);
-		goto error;
-	}
+	if (ret)
+		goto err_free;
+
+	BUG_ON(drm_atomic_helper_swap_state(state, false) < 0);
 
 	/*
 	 * This is the point of no return - everything below never fails except
 	 * when the hw goes bonghits. Which means we can commit the new state on
 	 * the software side now.
+	 *
+	 * swap driver private state while still holding state_lock
 	 */
-
-	drm_atomic_helper_swap_state(state, true);
-
-	/* swap driver private state while still holding state_lock */
 	if (to_kms_state(state)->state)
 		priv->kms->funcs->swap_state(priv->kms, state);
 
@@ -275,6 +273,8 @@ int msm_atomic_commit(struct drm_device *dev,
 
 	return 0;
 
+err_free:
+	kfree(c);
 error:
 	drm_atomic_helper_cleanup_planes(dev, state);
 	return ret;
diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c
index f49f6ac5585c..606df7bea97b 100644
--- a/drivers/gpu/drm/msm/msm_drv.c
+++ b/drivers/gpu/drm/msm/msm_drv.c
@@ -73,6 +73,10 @@ bool dumpstate = false;
 MODULE_PARM_DESC(dumpstate, "Dump KMS state on errors");
 module_param(dumpstate, bool, 0600);
 
+static bool modeset = true;
+MODULE_PARM_DESC(modeset, "Use kernel modesetting [KMS] (1=on (default), 0=disable)");
+module_param(modeset, bool, 0600);
+
 /*
  * Util/helpers:
  */
@@ -832,7 +836,6 @@ static struct drm_driver msm_driver = {
 	.gem_vm_ops         = &vm_ops,
 	.dumb_create        = msm_gem_dumb_create,
 	.dumb_map_offset    = msm_gem_dumb_map_offset,
-	.dumb_destroy       = drm_gem_dumb_destroy,
 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
 	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
 	.gem_prime_export   = drm_gem_prime_export,
@@ -879,8 +882,37 @@ static int msm_pm_resume(struct device *dev)
 }
 #endif
 
+#ifdef CONFIG_PM
+static int msm_runtime_suspend(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct msm_drm_private *priv = ddev->dev_private;
+
+	DBG("");
+
+	if (priv->mdss)
+		return msm_mdss_disable(priv->mdss);
+
+	return 0;
+}
+
+static int msm_runtime_resume(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct msm_drm_private *priv = ddev->dev_private;
+
+	DBG("");
+
+	if (priv->mdss)
+		return msm_mdss_enable(priv->mdss);
+
+	return 0;
+}
+#endif
+
 static const struct dev_pm_ops msm_pm_ops = {
 	SET_SYSTEM_SLEEP_PM_OPS(msm_pm_suspend, msm_pm_resume)
+	SET_RUNTIME_PM_OPS(msm_runtime_suspend, msm_runtime_resume, NULL)
 };
 
 /*
@@ -1104,6 +1136,9 @@ static struct platform_driver msm_platform_driver = {
 
 static int __init msm_drm_register(void)
 {
+	if (!modeset)
+		return -EINVAL;
+
 	DBG("init");
 	msm_mdp_register();
 	msm_dsi_register();
diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h
index fc8d24f7c084..5e8109c07560 100644
--- a/drivers/gpu/drm/msm/msm_drv.h
+++ b/drivers/gpu/drm/msm/msm_drv.h
@@ -55,8 +55,6 @@ struct msm_fence_cb;
 struct msm_gem_address_space;
 struct msm_gem_vma;
 
-#define NUM_DOMAINS 2    /* one for KMS, then one per gpu core (?) */
-
 struct msm_file_private {
 	/* currently we don't do anything useful with this.. but when
 	 * per-context address spaces are supported we'd keep track of
@@ -237,6 +235,12 @@ struct drm_gem_object *msm_gem_new(struct drm_device *dev,
 		uint32_t size, uint32_t flags);
 struct drm_gem_object *msm_gem_new_locked(struct drm_device *dev,
 		uint32_t size, uint32_t flags);
+void *msm_gem_kernel_new(struct drm_device *dev, uint32_t size,
+		uint32_t flags, struct msm_gem_address_space *aspace,
+		struct drm_gem_object **bo, uint64_t *iova);
+void *msm_gem_kernel_new_locked(struct drm_device *dev, uint32_t size,
+		uint32_t flags, struct msm_gem_address_space *aspace,
+		struct drm_gem_object **bo, uint64_t *iova);
 struct drm_gem_object *msm_gem_import(struct drm_device *dev,
 		struct dma_buf *dmabuf, struct sg_table *sgt);
 
@@ -248,10 +252,10 @@ uint32_t msm_framebuffer_iova(struct drm_framebuffer *fb,
 		struct msm_gem_address_space *aspace, int plane);
 struct drm_gem_object *msm_framebuffer_bo(struct drm_framebuffer *fb, int plane);
 const struct msm_format *msm_framebuffer_format(struct drm_framebuffer *fb);
-struct drm_framebuffer *msm_framebuffer_init(struct drm_device *dev,
-		const struct drm_mode_fb_cmd2 *mode_cmd, struct drm_gem_object **bos);
 struct drm_framebuffer *msm_framebuffer_create(struct drm_device *dev,
 		struct drm_file *file, const struct drm_mode_fb_cmd2 *mode_cmd);
+struct drm_framebuffer * msm_alloc_stolen_fb(struct drm_device *dev,
+		int w, int h, int p, uint32_t format);
 
 struct drm_fb_helper *msm_fbdev_init(struct drm_device *dev);
 void msm_fbdev_free(struct drm_device *dev);
diff --git a/drivers/gpu/drm/msm/msm_fb.c b/drivers/gpu/drm/msm/msm_fb.c
index 6ecb7b170316..fc175e724ad6 100644
--- a/drivers/gpu/drm/msm/msm_fb.c
+++ b/drivers/gpu/drm/msm/msm_fb.c
@@ -20,6 +20,7 @@
 
 #include "msm_drv.h"
 #include "msm_kms.h"
+#include "msm_gem.h"
 
 struct msm_framebuffer {
 	struct drm_framebuffer base;
@@ -28,6 +29,8 @@ struct msm_framebuffer {
 };
 #define to_msm_framebuffer(x) container_of(x, struct msm_framebuffer, base)
 
+static struct drm_framebuffer *msm_framebuffer_init(struct drm_device *dev,
+		const struct drm_mode_fb_cmd2 *mode_cmd, struct drm_gem_object **bos);
 
 static int msm_framebuffer_create_handle(struct drm_framebuffer *fb,
 		struct drm_file *file_priv,
@@ -161,7 +164,7 @@ out_unref:
 	return ERR_PTR(ret);
 }
 
-struct drm_framebuffer *msm_framebuffer_init(struct drm_device *dev,
+static struct drm_framebuffer *msm_framebuffer_init(struct drm_device *dev,
 		const struct drm_mode_fb_cmd2 *mode_cmd, struct drm_gem_object **bos)
 {
 	struct msm_drm_private *priv = dev->dev_private;
@@ -237,3 +240,43 @@ fail:
 
 	return ERR_PTR(ret);
 }
+
+struct drm_framebuffer *
+msm_alloc_stolen_fb(struct drm_device *dev, int w, int h, int p, uint32_t format)
+{
+	struct drm_mode_fb_cmd2 mode_cmd = {
+		.pixel_format = format,
+		.width = w,
+		.height = h,
+		.pitches = { p },
+	};
+	struct drm_gem_object *bo;
+	struct drm_framebuffer *fb;
+	int size;
+
+	/* allocate backing bo */
+	size = mode_cmd.pitches[0] * mode_cmd.height;
+	DBG("allocating %d bytes for fb %d", size, dev->primary->index);
+	bo = msm_gem_new(dev, size, MSM_BO_SCANOUT | MSM_BO_WC | MSM_BO_STOLEN);
+	if (IS_ERR(bo)) {
+		dev_warn(dev->dev, "could not allocate stolen bo\n");
+		/* try regular bo: */
+		bo = msm_gem_new(dev, size, MSM_BO_SCANOUT | MSM_BO_WC);
+	}
+	if (IS_ERR(bo)) {
+		dev_err(dev->dev, "failed to allocate buffer object\n");
+		return ERR_CAST(bo);
+	}
+
+	fb = msm_framebuffer_init(dev, &mode_cmd, &bo);
+	if (IS_ERR(fb)) {
+		dev_err(dev->dev, "failed to allocate fb\n");
+		/* note: if fb creation failed, we can't rely on fb destroy
+		 * to unref the bo:
+		 */
+		drm_gem_object_unreference_unlocked(bo);
+		return ERR_CAST(fb);
+	}
+
+	return fb;
+}
diff --git a/drivers/gpu/drm/msm/msm_fbdev.c b/drivers/gpu/drm/msm/msm_fbdev.c
index 5ecf4ff9a059..c178563fcd4d 100644
--- a/drivers/gpu/drm/msm/msm_fbdev.c
+++ b/drivers/gpu/drm/msm/msm_fbdev.c
@@ -19,7 +19,6 @@
 #include <drm/drm_fb_helper.h>
 
 #include "msm_drv.h"
-#include "msm_gem.h"
 #include "msm_kms.h"
 
 extern int msm_gem_mmap_obj(struct drm_gem_object *obj,
@@ -35,7 +34,6 @@ static int msm_fbdev_mmap(struct fb_info *info, struct vm_area_struct *vma);
 struct msm_fbdev {
 	struct drm_fb_helper base;
 	struct drm_framebuffer *fb;
-	struct drm_gem_object *bo;
 };
 
 static struct fb_ops msm_fb_ops = {
@@ -57,16 +55,16 @@ static int msm_fbdev_mmap(struct fb_info *info, struct vm_area_struct *vma)
 {
 	struct drm_fb_helper *helper = (struct drm_fb_helper *)info->par;
 	struct msm_fbdev *fbdev = to_msm_fbdev(helper);
-	struct drm_gem_object *drm_obj = fbdev->bo;
+	struct drm_gem_object *bo = msm_framebuffer_bo(fbdev->fb, 0);
 	int ret = 0;
 
-	ret = drm_gem_mmap_obj(drm_obj, drm_obj->size, vma);
+	ret = drm_gem_mmap_obj(bo, bo->size, vma);
 	if (ret) {
 		pr_err("%s:drm_gem_mmap_obj fail\n", __func__);
 		return ret;
 	}
 
-	return msm_gem_mmap_obj(drm_obj, vma);
+	return msm_gem_mmap_obj(bo, vma);
 }
 
 static int msm_fbdev_create(struct drm_fb_helper *helper,
@@ -76,47 +74,30 @@ static int msm_fbdev_create(struct drm_fb_helper *helper,
 	struct drm_device *dev = helper->dev;
 	struct msm_drm_private *priv = dev->dev_private;
 	struct drm_framebuffer *fb = NULL;
+	struct drm_gem_object *bo;
 	struct fb_info *fbi = NULL;
-	struct drm_mode_fb_cmd2 mode_cmd = {0};
 	uint64_t paddr;
-	int ret, size;
+	uint32_t format;
+	int ret, pitch;
+
+	format = drm_mode_legacy_fb_format(sizes->surface_bpp, sizes->surface_depth);
 
 	DBG("create fbdev: %dx%d@%d (%dx%d)", sizes->surface_width,
 			sizes->surface_height, sizes->surface_bpp,
 			sizes->fb_width, sizes->fb_height);
 
-	mode_cmd.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp,
-			sizes->surface_depth);
-
-	mode_cmd.width = sizes->surface_width;
-	mode_cmd.height = sizes->surface_height;
-
-	mode_cmd.pitches[0] = align_pitch(
-			mode_cmd.width, sizes->surface_bpp);
+	pitch = align_pitch(sizes->surface_width, sizes->surface_bpp);
+	fb = msm_alloc_stolen_fb(dev, sizes->surface_width,
+			sizes->surface_height, pitch, format);
 
-	/* allocate backing bo */
-	size = mode_cmd.pitches[0] * mode_cmd.height;
-	DBG("allocating %d bytes for fb %d", size, dev->primary->index);
-	fbdev->bo = msm_gem_new(dev, size, MSM_BO_SCANOUT |
-			MSM_BO_WC | MSM_BO_STOLEN);
-	if (IS_ERR(fbdev->bo)) {
-		ret = PTR_ERR(fbdev->bo);
-		fbdev->bo = NULL;
-		dev_err(dev->dev, "failed to allocate buffer object: %d\n", ret);
-		goto fail;
-	}
-
-	fb = msm_framebuffer_init(dev, &mode_cmd, &fbdev->bo);
 	if (IS_ERR(fb)) {
 		dev_err(dev->dev, "failed to allocate fb\n");
-		/* note: if fb creation failed, we can't rely on fb destroy
-		 * to unref the bo:
-		 */
-		drm_gem_object_unreference_unlocked(fbdev->bo);
 		ret = PTR_ERR(fb);
 		goto fail;
 	}
 
+	bo = msm_framebuffer_bo(fb, 0);
+
 	mutex_lock(&dev->struct_mutex);
 
 	/*
@@ -124,7 +105,7 @@ static int msm_fbdev_create(struct drm_fb_helper *helper,
 	 * in panic (ie. lock-safe, etc) we could avoid pinning the
 	 * buffer now:
 	 */
-	ret = msm_gem_get_iova(fbdev->bo, priv->kms->aspace, &paddr);
+	ret = msm_gem_get_iova(bo, priv->kms->aspace, &paddr);
 	if (ret) {
 		dev_err(dev->dev, "failed to get buffer obj iova: %d\n", ret);
 		goto fail_unlock;
@@ -143,7 +124,6 @@ static int msm_fbdev_create(struct drm_fb_helper *helper,
 	helper->fb = fb;
 
 	fbi->par = helper;
-	fbi->flags = FBINFO_DEFAULT;
 	fbi->fbops = &msm_fb_ops;
 
 	strcpy(fbi->fix.id, "msm");
@@ -153,14 +133,14 @@ static int msm_fbdev_create(struct drm_fb_helper *helper,
 
 	dev->mode_config.fb_base = paddr;
 
-	fbi->screen_base = msm_gem_get_vaddr(fbdev->bo);
+	fbi->screen_base = msm_gem_get_vaddr(bo);
 	if (IS_ERR(fbi->screen_base)) {
 		ret = PTR_ERR(fbi->screen_base);
 		goto fail_unlock;
 	}
-	fbi->screen_size = fbdev->bo->size;
+	fbi->screen_size = bo->size;
 	fbi->fix.smem_start = paddr;
-	fbi->fix.smem_len = fbdev->bo->size;
+	fbi->fix.smem_len = bo->size;
 
 	DBG("par=%p, %dx%d", fbi->par, fbi->var.xres, fbi->var.yres);
 	DBG("allocated %dx%d fb", fbdev->fb->width, fbdev->fb->height);
@@ -242,7 +222,9 @@ void msm_fbdev_free(struct drm_device *dev)
 
 	/* this will free the backing object */
 	if (fbdev->fb) {
-		msm_gem_put_vaddr(fbdev->bo);
+		struct drm_gem_object *bo =
+			msm_framebuffer_bo(fbdev->fb, 0);
+		msm_gem_put_vaddr(bo);
 		drm_framebuffer_remove(fbdev->fb);
 	}
 
diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c
index a0c60e738db8..f15821a0d900 100644
--- a/drivers/gpu/drm/msm/msm_gem.c
+++ b/drivers/gpu/drm/msm/msm_gem.c
@@ -1024,3 +1024,49 @@ fail:
 	drm_gem_object_unreference_unlocked(obj);
 	return ERR_PTR(ret);
 }
+
+static void *_msm_gem_kernel_new(struct drm_device *dev, uint32_t size,
+		uint32_t flags, struct msm_gem_address_space *aspace,
+		struct drm_gem_object **bo, uint64_t *iova, bool locked)
+{
+	void *vaddr;
+	struct drm_gem_object *obj = _msm_gem_new(dev, size, flags, locked);
+	int ret;
+
+	if (IS_ERR(obj))
+		return ERR_CAST(obj);
+
+	if (iova) {
+		ret = msm_gem_get_iova(obj, aspace, iova);
+		if (ret) {
+			drm_gem_object_unreference(obj);
+			return ERR_PTR(ret);
+		}
+	}
+
+	vaddr = msm_gem_get_vaddr(obj);
+	if (!vaddr) {
+		msm_gem_put_iova(obj, aspace);
+		drm_gem_object_unreference(obj);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	if (bo)
+		*bo = obj;
+
+	return vaddr;
+}
+
+void *msm_gem_kernel_new(struct drm_device *dev, uint32_t size,
+		uint32_t flags, struct msm_gem_address_space *aspace,
+		struct drm_gem_object **bo, uint64_t *iova)
+{
+	return _msm_gem_kernel_new(dev, size, flags, aspace, bo, iova, false);
+}
+
+void *msm_gem_kernel_new_locked(struct drm_device *dev, uint32_t size,
+		uint32_t flags, struct msm_gem_address_space *aspace,
+		struct drm_gem_object **bo, uint64_t *iova)
+{
+	return _msm_gem_kernel_new(dev, size, flags, aspace, bo, iova, true);
+}
diff --git a/drivers/gpu/drm/msm/msm_gem_submit.c b/drivers/gpu/drm/msm/msm_gem_submit.c
index 8a75c0bd8a78..5d0a75d4b249 100644
--- a/drivers/gpu/drm/msm/msm_gem_submit.c
+++ b/drivers/gpu/drm/msm/msm_gem_submit.c
@@ -40,7 +40,7 @@ static struct msm_gem_submit *submit_create(struct drm_device *dev,
 	if (sz > SIZE_MAX)
 		return NULL;
 
-	submit = kmalloc(sz, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
+	submit = kmalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
 	if (!submit)
 		return NULL;
 
diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c
index 9f3dbc236ab3..ffbff27600e0 100644
--- a/drivers/gpu/drm/msm/msm_gpu.c
+++ b/drivers/gpu/drm/msm/msm_gpu.c
@@ -562,11 +562,49 @@ static int get_clocks(struct platform_device *pdev, struct msm_gpu *gpu)
 	return 0;
 }
 
+static struct msm_gem_address_space *
+msm_gpu_create_address_space(struct msm_gpu *gpu, struct platform_device *pdev,
+		uint64_t va_start, uint64_t va_end)
+{
+	struct iommu_domain *iommu;
+	struct msm_gem_address_space *aspace;
+	int ret;
+
+	/*
+	 * Setup IOMMU.. eventually we will (I think) do this once per context
+	 * and have separate page tables per context.  For now, to keep things
+	 * simple and to get something working, just use a single address space:
+	 */
+	iommu = iommu_domain_alloc(&platform_bus_type);
+	if (!iommu)
+		return NULL;
+
+	iommu->geometry.aperture_start = va_start;
+	iommu->geometry.aperture_end = va_end;
+
+	dev_info(gpu->dev->dev, "%s: using IOMMU\n", gpu->name);
+
+	aspace = msm_gem_address_space_create(&pdev->dev, iommu, "gpu");
+	if (IS_ERR(aspace)) {
+		dev_err(gpu->dev->dev, "failed to init iommu: %ld\n",
+			PTR_ERR(aspace));
+		iommu_domain_free(iommu);
+		return ERR_CAST(aspace);
+	}
+
+	ret = aspace->mmu->funcs->attach(aspace->mmu, NULL, 0);
+	if (ret) {
+		msm_gem_address_space_put(aspace);
+		return ERR_PTR(ret);
+	}
+
+	return aspace;
+}
+
 int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 		struct msm_gpu *gpu, const struct msm_gpu_funcs *funcs,
 		const char *name, struct msm_gpu_config *config)
 {
-	struct iommu_domain *iommu;
 	int ret;
 
 	if (WARN_ON(gpu->num_perfcntrs > ARRAY_SIZE(gpu->last_cntrs)))
@@ -636,28 +674,19 @@ int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 	if (IS_ERR(gpu->gpu_cx))
 		gpu->gpu_cx = NULL;
 
-	/* Setup IOMMU.. eventually we will (I think) do this once per context
-	 * and have separate page tables per context.  For now, to keep things
-	 * simple and to get something working, just use a single address space:
-	 */
-	iommu = iommu_domain_alloc(&platform_bus_type);
-	if (iommu) {
-		iommu->geometry.aperture_start = config->va_start;
-		iommu->geometry.aperture_end = config->va_end;
-
-		dev_info(drm->dev, "%s: using IOMMU\n", name);
-		gpu->aspace = msm_gem_address_space_create(&pdev->dev,
-				iommu, "gpu");
-		if (IS_ERR(gpu->aspace)) {
-			ret = PTR_ERR(gpu->aspace);
-			dev_err(drm->dev, "failed to init iommu: %d\n", ret);
-			gpu->aspace = NULL;
-			iommu_domain_free(iommu);
-			goto fail;
-		}
+	gpu->pdev = pdev;
+	platform_set_drvdata(pdev, gpu);
+
+	bs_init(gpu);
 
-	} else {
+	gpu->aspace = msm_gpu_create_address_space(gpu, pdev,
+		config->va_start, config->va_end);
+
+	if (gpu->aspace == NULL)
 		dev_info(drm->dev, "%s: no IOMMU, fallback to VRAM carveout!\n", name);
+	else if (IS_ERR(gpu->aspace)) {
+		ret = PTR_ERR(gpu->aspace);
+		goto fail;
 	}
 
 	/* Create ringbuffer: */
@@ -669,14 +698,10 @@ int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 		goto fail;
 	}
 
-	gpu->pdev = pdev;
-	platform_set_drvdata(pdev, gpu);
-
-	bs_init(gpu);
-
 	return 0;
 
 fail:
+	platform_set_drvdata(pdev, NULL);
 	return ret;
 }
 
@@ -693,7 +718,9 @@ void msm_gpu_cleanup(struct msm_gpu *gpu)
 			msm_gem_put_iova(gpu->rb->bo, gpu->aspace);
 		msm_ringbuffer_destroy(gpu->rb);
 	}
-
-	if (gpu->fctx)
-		msm_fence_context_free(gpu->fctx);
+	if (gpu->aspace) {
+		gpu->aspace->mmu->funcs->detach(gpu->aspace->mmu,
+			NULL, 0);
+		msm_gem_address_space_put(gpu->aspace);
+	}
 }
diff --git a/drivers/gpu/drm/msm/msm_kms.h b/drivers/gpu/drm/msm/msm_kms.h
index a8f2ba5e5f07..17d5824417ad 100644
--- a/drivers/gpu/drm/msm/msm_kms.h
+++ b/drivers/gpu/drm/msm/msm_kms.h
@@ -99,5 +99,7 @@ struct msm_kms *mdp4_kms_init(struct drm_device *dev);
 struct msm_kms *mdp5_kms_init(struct drm_device *dev);
 int msm_mdss_init(struct drm_device *dev);
 void msm_mdss_destroy(struct drm_device *dev);
+int msm_mdss_enable(struct msm_mdss *mdss);
+int msm_mdss_disable(struct msm_mdss *mdss);
 
 #endif /* __MSM_KMS_H__ */
diff --git a/drivers/gpu/drm/msm/msm_ringbuffer.c b/drivers/gpu/drm/msm/msm_ringbuffer.c
index 791bca3c6a9c..bf065a540130 100644
--- a/drivers/gpu/drm/msm/msm_ringbuffer.c
+++ b/drivers/gpu/drm/msm/msm_ringbuffer.c
@@ -33,16 +33,14 @@ struct msm_ringbuffer *msm_ringbuffer_new(struct msm_gpu *gpu, int size)
 	}
 
 	ring->gpu = gpu;
-	ring->bo = msm_gem_new(gpu->dev, size, MSM_BO_WC);
-	if (IS_ERR(ring->bo)) {
-		ret = PTR_ERR(ring->bo);
-		ring->bo = NULL;
-		goto fail;
-	}
 
-	ring->start = msm_gem_get_vaddr(ring->bo);
+	/* Pass NULL for the iova pointer - we will map it later */
+	ring->start = msm_gem_kernel_new(gpu->dev, size, MSM_BO_WC,
+		gpu->aspace, &ring->bo, NULL);
+
 	if (IS_ERR(ring->start)) {
 		ret = PTR_ERR(ring->start);
+		ring->start = 0;
 		goto fail;
 	}
 	ring->end   = ring->start + (size / 4);
diff --git a/drivers/gpu/drm/mxsfb/mxsfb_drv.c b/drivers/gpu/drm/mxsfb/mxsfb_drv.c
index d1b9c34c7c00..7fbad9cb656e 100644
--- a/drivers/gpu/drm/mxsfb/mxsfb_drv.c
+++ b/drivers/gpu/drm/mxsfb/mxsfb_drv.c
@@ -190,7 +190,7 @@ static int mxsfb_load(struct drm_device *drm, unsigned long flags)
 	}
 
 	ret = drm_simple_display_pipe_init(drm, &mxsfb->pipe, &mxsfb_funcs,
-			mxsfb_formats, ARRAY_SIZE(mxsfb_formats),
+			mxsfb_formats, ARRAY_SIZE(mxsfb_formats), NULL,
 			&mxsfb->connector);
 	if (ret < 0) {
 		dev_err(drm->dev, "Cannot setup simple display pipe\n");
@@ -256,7 +256,6 @@ static void mxsfb_unload(struct drm_device *drm)
 
 	drm_kms_helper_poll_fini(drm);
 	drm_mode_config_cleanup(drm);
-	drm_vblank_cleanup(drm);
 
 	pm_runtime_get_sync(drm->dev);
 	drm_irq_uninstall(drm);
@@ -335,11 +334,9 @@ static struct drm_driver mxsfb_driver = {
 	.irq_uninstall		= mxsfb_irq_preinstall,
 	.enable_vblank		= mxsfb_enable_vblank,
 	.disable_vblank		= mxsfb_disable_vblank,
-	.gem_free_object	= drm_gem_cma_free_object,
+	.gem_free_object_unlocked = drm_gem_cma_free_object,
 	.gem_vm_ops		= &drm_gem_cma_vm_ops,
 	.dumb_create		= drm_gem_cma_dumb_create,
-	.dumb_map_offset	= drm_gem_cma_dumb_map_offset,
-	.dumb_destroy		= drm_gem_dumb_destroy,
 	.prime_handle_to_fd	= drm_gem_prime_handle_to_fd,
 	.prime_fd_to_handle	= drm_gem_prime_fd_to_handle,
 	.gem_prime_export	= drm_gem_prime_export,
diff --git a/drivers/gpu/drm/mxsfb/mxsfb_out.c b/drivers/gpu/drm/mxsfb/mxsfb_out.c
index f7d729aa09bd..e5edf016a439 100644
--- a/drivers/gpu/drm/mxsfb/mxsfb_out.c
+++ b/drivers/gpu/drm/mxsfb/mxsfb_out.c
@@ -74,7 +74,6 @@ static void mxsfb_panel_connector_destroy(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs mxsfb_panel_connector_funcs = {
-	.dpms			= drm_atomic_helper_connector_dpms,
 	.detect			= mxsfb_panel_connector_detect,
 	.fill_modes		= drm_helper_probe_single_connector_modes,
 	.destroy		= mxsfb_panel_connector_destroy,
diff --git a/drivers/gpu/drm/nouveau/dispnv04/crtc.c b/drivers/gpu/drm/nouveau/dispnv04/crtc.c
index 4b4b0b496262..6aa6ee16dcbd 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/crtc.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/crtc.c
@@ -764,13 +764,18 @@ nv_crtc_gamma_load(struct drm_crtc *crtc)
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
 	struct drm_device *dev = nv_crtc->base.dev;
 	struct rgb { uint8_t r, g, b; } __attribute__((packed)) *rgbs;
+	u16 *r, *g, *b;
 	int i;
 
 	rgbs = (struct rgb *)nv04_display(dev)->mode_reg.crtc_reg[nv_crtc->index].DAC;
+	r = crtc->gamma_store;
+	g = r + crtc->gamma_size;
+	b = g + crtc->gamma_size;
+
 	for (i = 0; i < 256; i++) {
-		rgbs[i].r = nv_crtc->lut.r[i] >> 8;
-		rgbs[i].g = nv_crtc->lut.g[i] >> 8;
-		rgbs[i].b = nv_crtc->lut.b[i] >> 8;
+		rgbs[i].r = *r++ >> 8;
+		rgbs[i].g = *g++ >> 8;
+		rgbs[i].b = *b++ >> 8;
 	}
 
 	nouveau_hw_load_state_palette(dev, nv_crtc->index, &nv04_display(dev)->mode_reg);
@@ -792,13 +797,6 @@ nv_crtc_gamma_set(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b,
 		  struct drm_modeset_acquire_ctx *ctx)
 {
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-	int i;
-
-	for (i = 0; i < size; i++) {
-		nv_crtc->lut.r[i] = r[i];
-		nv_crtc->lut.g[i] = g[i];
-		nv_crtc->lut.b[i] = b[i];
-	}
 
 	/* We need to know the depth before we upload, but it's possible to
 	 * get called before a framebuffer is bound.  If this is the case,
@@ -1095,25 +1093,51 @@ static const struct drm_crtc_helper_funcs nv04_crtc_helper_funcs = {
 	.mode_set = nv_crtc_mode_set,
 	.mode_set_base = nv04_crtc_mode_set_base,
 	.mode_set_base_atomic = nv04_crtc_mode_set_base_atomic,
-	.load_lut = nv_crtc_gamma_load,
 	.disable = nv_crtc_disable,
 };
 
+static const uint32_t modeset_formats[] = {
+        DRM_FORMAT_XRGB8888,
+        DRM_FORMAT_RGB565,
+        DRM_FORMAT_XRGB1555,
+};
+
+static struct drm_plane *
+create_primary_plane(struct drm_device *dev)
+{
+        struct drm_plane *primary;
+        int ret;
+
+        primary = kzalloc(sizeof(*primary), GFP_KERNEL);
+        if (primary == NULL) {
+                DRM_DEBUG_KMS("Failed to allocate primary plane\n");
+                return NULL;
+        }
+
+        /* possible_crtc's will be filled in later by crtc_init */
+        ret = drm_universal_plane_init(dev, primary, 0,
+                                       &drm_primary_helper_funcs,
+                                       modeset_formats,
+                                       ARRAY_SIZE(modeset_formats), NULL,
+                                       DRM_PLANE_TYPE_PRIMARY, NULL);
+        if (ret) {
+                kfree(primary);
+                primary = NULL;
+        }
+
+        return primary;
+}
+
 int
 nv04_crtc_create(struct drm_device *dev, int crtc_num)
 {
 	struct nouveau_crtc *nv_crtc;
-	int ret, i;
+	int ret;
 
 	nv_crtc = kzalloc(sizeof(*nv_crtc), GFP_KERNEL);
 	if (!nv_crtc)
 		return -ENOMEM;
 
-	for (i = 0; i < 256; i++) {
-		nv_crtc->lut.r[i] = i << 8;
-		nv_crtc->lut.g[i] = i << 8;
-		nv_crtc->lut.b[i] = i << 8;
-	}
 	nv_crtc->lut.depth = 0;
 
 	nv_crtc->index = crtc_num;
@@ -1122,7 +1146,9 @@ nv04_crtc_create(struct drm_device *dev, int crtc_num)
 	nv_crtc->save = nv_crtc_save;
 	nv_crtc->restore = nv_crtc_restore;
 
-	drm_crtc_init(dev, &nv_crtc->base, &nv04_crtc_funcs);
+	drm_crtc_init_with_planes(dev, &nv_crtc->base,
+                                  create_primary_plane(dev), NULL,
+                                  &nv04_crtc_funcs, NULL);
 	drm_crtc_helper_add(&nv_crtc->base, &nv04_crtc_helper_funcs);
 	drm_mode_crtc_set_gamma_size(&nv_crtc->base, 256);
 
diff --git a/drivers/gpu/drm/nouveau/dispnv04/overlay.c b/drivers/gpu/drm/nouveau/dispnv04/overlay.c
index e54944d23268..c8c2333f24ee 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/overlay.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/overlay.c
@@ -63,6 +63,7 @@ static uint32_t formats[] = {
 	DRM_FORMAT_YUYV,
 	DRM_FORMAT_UYVY,
 	DRM_FORMAT_NV12,
+	DRM_FORMAT_NV21,
 };
 
 /* Sine can be approximated with
@@ -90,6 +91,26 @@ cos_mul(int degrees, int factor)
 }
 
 static int
+verify_scaling(const struct drm_framebuffer *fb, uint8_t shift,
+               uint32_t src_x, uint32_t src_y, uint32_t src_w, uint32_t src_h,
+               uint32_t crtc_w, uint32_t crtc_h)
+{
+	if (crtc_w < (src_w >> shift) || crtc_h < (src_h >> shift)) {
+		DRM_DEBUG_KMS("Unsuitable framebuffer scaling: %dx%d -> %dx%d\n",
+			      src_w, src_h, crtc_w, crtc_h);
+		return -ERANGE;
+	}
+
+	if (src_x != 0 || src_y != 0) {
+		DRM_DEBUG_KMS("Unsuitable framebuffer offset: %d,%d\n",
+                              src_x, src_y);
+		return -ERANGE;
+	}
+
+	return 0;
+}
+
+static int
 nv10_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
 		  struct drm_framebuffer *fb, int crtc_x, int crtc_y,
 		  unsigned int crtc_w, unsigned int crtc_h,
@@ -107,7 +128,9 @@ nv10_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
 	bool flip = nv_plane->flip;
 	int soff = NV_PCRTC0_SIZE * nv_crtc->index;
 	int soff2 = NV_PCRTC0_SIZE * !nv_crtc->index;
-	int format, ret;
+	unsigned shift = drm->client.device.info.chipset >= 0x30 ? 1 : 3;
+	unsigned format = 0;
+	int ret;
 
 	/* Source parameters given in 16.16 fixed point, ignore fractional. */
 	src_x >>= 16;
@@ -115,18 +138,9 @@ nv10_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
 	src_w >>= 16;
 	src_h >>= 16;
 
-	format = ALIGN(src_w * 4, 0x100);
-
-	if (format > 0xffff)
-		return -ERANGE;
-
-	if (drm->client.device.info.chipset >= 0x30) {
-		if (crtc_w < (src_w >> 1) || crtc_h < (src_h >> 1))
-			return -ERANGE;
-	} else {
-		if (crtc_w < (src_w >> 3) || crtc_h < (src_h >> 3))
-			return -ERANGE;
-	}
+	ret = verify_scaling(fb, shift, 0, 0, src_w, src_h, crtc_w, crtc_h);
+	if (ret)
+		return ret;
 
 	ret = nouveau_bo_pin(nv_fb->nvbo, TTM_PL_FLAG_VRAM, false);
 	if (ret)
@@ -146,21 +160,23 @@ nv10_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
 	nvif_wr32(dev, NV_PVIDEO_POINT_OUT(flip), crtc_y << 16 | crtc_x);
 	nvif_wr32(dev, NV_PVIDEO_SIZE_OUT(flip), crtc_h << 16 | crtc_w);
 
-	if (fb->format->format != DRM_FORMAT_UYVY)
+	if (fb->format->format == DRM_FORMAT_YUYV ||
+	    fb->format->format == DRM_FORMAT_NV12)
 		format |= NV_PVIDEO_FORMAT_COLOR_LE_CR8YB8CB8YA8;
-	if (fb->format->format == DRM_FORMAT_NV12)
+	if (fb->format->format == DRM_FORMAT_NV12 ||
+	    fb->format->format == DRM_FORMAT_NV21)
 		format |= NV_PVIDEO_FORMAT_PLANAR;
 	if (nv_plane->iturbt_709)
 		format |= NV_PVIDEO_FORMAT_MATRIX_ITURBT709;
 	if (nv_plane->colorkey & (1 << 24))
 		format |= NV_PVIDEO_FORMAT_DISPLAY_COLOR_KEY;
 
-	if (fb->format->format == DRM_FORMAT_NV12) {
+	if (format & NV_PVIDEO_FORMAT_PLANAR) {
 		nvif_wr32(dev, NV_PVIDEO_UVPLANE_BASE(flip), 0);
 		nvif_wr32(dev, NV_PVIDEO_UVPLANE_OFFSET_BUFF(flip),
 			nv_fb->nvbo->bo.offset + fb->offsets[1]);
 	}
-	nvif_wr32(dev, NV_PVIDEO_FORMAT(flip), format);
+	nvif_wr32(dev, NV_PVIDEO_FORMAT(flip), format | fb->pitches[0]);
 	nvif_wr32(dev, NV_PVIDEO_STOP, 0);
 	/* TODO: wait for vblank? */
 	nvif_wr32(dev, NV_PVIDEO_BUFFER, flip ? 0x10 : 0x1);
@@ -357,7 +373,7 @@ nv04_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
 	struct nouveau_bo *cur = nv_plane->cur;
 	uint32_t overlay = 1;
 	int brightness = (nv_plane->brightness - 512) * 62 / 512;
-	int pitch, ret, i;
+	int ret, i;
 
 	/* Source parameters given in 16.16 fixed point, ignore fractional. */
 	src_x >>= 16;
@@ -365,17 +381,9 @@ nv04_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
 	src_w >>= 16;
 	src_h >>= 16;
 
-	pitch = ALIGN(src_w * 4, 0x100);
-
-	if (pitch > 0xffff)
-		return -ERANGE;
-
-	/* TODO: Compute an offset? Not sure how to do this for YUYV. */
-	if (src_x != 0 || src_y != 0)
-		return -ERANGE;
-
-	if (crtc_w < src_w || crtc_h < src_h)
-		return -ERANGE;
+	ret = verify_scaling(fb, 0, src_x, src_y, src_w, src_h, crtc_w, crtc_h);
+	if (ret)
+		return ret;
 
 	ret = nouveau_bo_pin(nv_fb->nvbo, TTM_PL_FLAG_VRAM, false);
 	if (ret)
@@ -389,8 +397,9 @@ nv04_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
 
 	for (i = 0; i < 2; i++) {
 		nvif_wr32(dev, NV_PVIDEO_BUFF0_START_ADDRESS + 4 * i,
-			nv_fb->nvbo->bo.offset);
-		nvif_wr32(dev, NV_PVIDEO_BUFF0_PITCH_LENGTH + 4 * i, pitch);
+			  nv_fb->nvbo->bo.offset);
+		nvif_wr32(dev, NV_PVIDEO_BUFF0_PITCH_LENGTH + 4 * i,
+			  fb->pitches[0]);
 		nvif_wr32(dev, NV_PVIDEO_BUFF0_OFFSET + 4 * i, 0);
 	}
 	nvif_wr32(dev, NV_PVIDEO_WINDOW_START, crtc_y << 16 | crtc_x);
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/conn.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/conn.h
index e8e77ee24776..deb477282dde 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/conn.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/conn.h
@@ -18,6 +18,7 @@ enum dcb_connector_type {
 	DCB_CONNECTOR_HDMI_C = 0x63,
 	DCB_CONNECTOR_DMS59_DP0 = 0x64,
 	DCB_CONNECTOR_DMS59_DP1 = 0x65,
+	DCB_CONNECTOR_WFD	= 0x70,
 	DCB_CONNECTOR_NONE = 0xff
 };
 
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/dcb.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/dcb.h
index 4892a65ddd48..903d117603d8 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/dcb.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios/dcb.h
@@ -6,6 +6,7 @@ enum dcb_output_type {
 	DCB_OUTPUT_TMDS		= 0x2,
 	DCB_OUTPUT_LVDS		= 0x3,
 	DCB_OUTPUT_DP		= 0x6,
+	DCB_OUTPUT_WFD		= 0x8,
 	DCB_OUTPUT_EOL		= 0xe,
 	DCB_OUTPUT_UNUSED	= 0xf,
 	DCB_OUTPUT_ANY = -1,
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h
index b268b96faece..1bfd93b85575 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h
@@ -96,4 +96,5 @@ int g84_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
 int gt215_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
 int gf119_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
 int gm107_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
+int gm200_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_bios.c b/drivers/gpu/drm/nouveau/nouveau_bios.c
index b998c33af18a..dd6fba55ad5d 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bios.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bios.c
@@ -351,11 +351,8 @@ static int parse_fp_mode_table(struct drm_device *dev, struct nvbios *bios)
 	struct lvdstableheader lth;
 
 	if (bios->fp.fptablepointer == 0x0) {
-		/* Apple cards don't have the fp table; the laptops use DDC */
-		/* The table is also missing on some x86 IGPs */
-#ifndef __powerpc__
-		NV_ERROR(drm, "Pointer to flat panel table invalid\n");
-#endif
+		/* Most laptop cards lack an fp table. They use DDC. */
+		NV_DEBUG(drm, "Pointer to flat panel table invalid\n");
 		bios->digital_min_front_porch = 0x4b;
 		return 0;
 	}
diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.c b/drivers/gpu/drm/nouveau/nouveau_connector.c
index dab78c660dd6..70d8e0d69ad5 100644
--- a/drivers/gpu/drm/nouveau/nouveau_connector.c
+++ b/drivers/gpu/drm/nouveau/nouveau_connector.c
@@ -770,9 +770,6 @@ nouveau_connector_set_property(struct drm_connector *connector,
 	struct drm_encoder *encoder = to_drm_encoder(nv_encoder);
 	int ret;
 
-	if (drm_drv_uses_atomic_modeset(connector->dev))
-		return drm_atomic_helper_connector_set_property(connector, property, value);
-
 	ret = connector->funcs->atomic_set_property(&nv_connector->base,
 						    &asyc->state,
 						    property, value);
@@ -1075,17 +1072,9 @@ nouveau_connector_helper_funcs = {
 	.best_encoder = nouveau_connector_best_encoder,
 };
 
-static int
-nouveau_connector_dpms(struct drm_connector *connector, int mode)
-{
-	if (drm_drv_uses_atomic_modeset(connector->dev))
-		return drm_atomic_helper_connector_dpms(connector, mode);
-	return drm_helper_connector_dpms(connector, mode);
-}
-
 static const struct drm_connector_funcs
 nouveau_connector_funcs = {
-	.dpms = nouveau_connector_dpms,
+	.dpms = drm_helper_connector_dpms,
 	.reset = nouveau_conn_reset,
 	.detect = nouveau_connector_detect,
 	.force = nouveau_connector_force,
@@ -1100,7 +1089,7 @@ nouveau_connector_funcs = {
 
 static const struct drm_connector_funcs
 nouveau_connector_funcs_lvds = {
-	.dpms = nouveau_connector_dpms,
+	.dpms = drm_helper_connector_dpms,
 	.reset = nouveau_conn_reset,
 	.detect = nouveau_connector_detect_lvds,
 	.force = nouveau_connector_force,
@@ -1195,6 +1184,7 @@ drm_conntype_from_dcb(enum dcb_connector_type dcb)
 	case DCB_CONNECTOR_HDMI_0   :
 	case DCB_CONNECTOR_HDMI_1   :
 	case DCB_CONNECTOR_HDMI_C   : return DRM_MODE_CONNECTOR_HDMIA;
+	case DCB_CONNECTOR_WFD	    : return DRM_MODE_CONNECTOR_VIRTUAL;
 	default:
 		break;
 	}
diff --git a/drivers/gpu/drm/nouveau/nouveau_crtc.h b/drivers/gpu/drm/nouveau/nouveau_crtc.h
index 050fcf30a0d2..b7a18fbee6dc 100644
--- a/drivers/gpu/drm/nouveau/nouveau_crtc.h
+++ b/drivers/gpu/drm/nouveau/nouveau_crtc.h
@@ -61,9 +61,6 @@ struct nouveau_crtc {
 
 	struct {
 		struct nouveau_bo *nvbo;
-		uint16_t r[256];
-		uint16_t g[256];
-		uint16_t b[256];
 		int depth;
 	} lut;
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c
index f362c9fa8b3b..2e7785f49e6d 100644
--- a/drivers/gpu/drm/nouveau/nouveau_display.c
+++ b/drivers/gpu/drm/nouveau/nouveau_display.c
@@ -159,8 +159,6 @@ nouveau_display_vblank_fini(struct drm_device *dev)
 {
 	struct drm_crtc *crtc;
 
-	drm_vblank_cleanup(dev);
-
 	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
 		struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
 		nvif_notify_fini(&nv_crtc->vblank);
@@ -233,9 +231,30 @@ nouveau_framebuffer_new(struct drm_device *dev,
 			struct nouveau_bo *nvbo,
 			struct nouveau_framebuffer **pfb)
 {
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_framebuffer *fb;
 	int ret;
 
+        /* YUV overlays have special requirements pre-NV50 */
+	if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA &&
+
+	    (mode_cmd->pixel_format == DRM_FORMAT_YUYV ||
+	     mode_cmd->pixel_format == DRM_FORMAT_UYVY ||
+	     mode_cmd->pixel_format == DRM_FORMAT_NV12 ||
+	     mode_cmd->pixel_format == DRM_FORMAT_NV21) &&
+	    (mode_cmd->pitches[0] & 0x3f || /* align 64 */
+	     mode_cmd->pitches[0] >= 0x10000 || /* at most 64k pitch */
+	     (mode_cmd->pitches[1] && /* pitches for planes must match */
+	      mode_cmd->pitches[0] != mode_cmd->pitches[1]))) {
+		struct drm_format_name_buf format_name;
+		DRM_DEBUG_KMS("Unsuitable framebuffer: format: %s; pitches: 0x%x\n 0x%x\n",
+			      drm_get_format_name(mode_cmd->pixel_format,
+						  &format_name),
+			      mode_cmd->pitches[0],
+			      mode_cmd->pitches[1]);
+		return -EINVAL;
+	}
+
 	if (!(fb = *pfb = kzalloc(sizeof(*fb), GFP_KERNEL)))
 		return -ENOMEM;
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c
index 90757af9bc73..595630d1fb9e 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.c
@@ -585,18 +585,18 @@ nouveau_do_suspend(struct drm_device *dev, bool runtime)
 	nouveau_led_suspend(dev);
 
 	if (dev->mode_config.num_crtc) {
-		NV_INFO(drm, "suspending console...\n");
+		NV_DEBUG(drm, "suspending console...\n");
 		nouveau_fbcon_set_suspend(dev, 1);
-		NV_INFO(drm, "suspending display...\n");
+		NV_DEBUG(drm, "suspending display...\n");
 		ret = nouveau_display_suspend(dev, runtime);
 		if (ret)
 			return ret;
 	}
 
-	NV_INFO(drm, "evicting buffers...\n");
+	NV_DEBUG(drm, "evicting buffers...\n");
 	ttm_bo_evict_mm(&drm->ttm.bdev, TTM_PL_VRAM);
 
-	NV_INFO(drm, "waiting for kernel channels to go idle...\n");
+	NV_DEBUG(drm, "waiting for kernel channels to go idle...\n");
 	if (drm->cechan) {
 		ret = nouveau_channel_idle(drm->cechan);
 		if (ret)
@@ -609,7 +609,7 @@ nouveau_do_suspend(struct drm_device *dev, bool runtime)
 			goto fail_display;
 	}
 
-	NV_INFO(drm, "suspending fence...\n");
+	NV_DEBUG(drm, "suspending fence...\n");
 	if (drm->fence && nouveau_fence(drm)->suspend) {
 		if (!nouveau_fence(drm)->suspend(drm)) {
 			ret = -ENOMEM;
@@ -617,7 +617,7 @@ nouveau_do_suspend(struct drm_device *dev, bool runtime)
 		}
 	}
 
-	NV_INFO(drm, "suspending object tree...\n");
+	NV_DEBUG(drm, "suspending object tree...\n");
 	ret = nvif_client_suspend(&drm->client.base);
 	if (ret)
 		goto fail_client;
@@ -630,7 +630,7 @@ fail_client:
 
 fail_display:
 	if (dev->mode_config.num_crtc) {
-		NV_INFO(drm, "resuming display...\n");
+		NV_DEBUG(drm, "resuming display...\n");
 		nouveau_display_resume(dev, runtime);
 	}
 	return ret;
@@ -641,19 +641,19 @@ nouveau_do_resume(struct drm_device *dev, bool runtime)
 {
 	struct nouveau_drm *drm = nouveau_drm(dev);
 
-	NV_INFO(drm, "resuming object tree...\n");
+	NV_DEBUG(drm, "resuming object tree...\n");
 	nvif_client_resume(&drm->client.base);
 
-	NV_INFO(drm, "resuming fence...\n");
+	NV_DEBUG(drm, "resuming fence...\n");
 	if (drm->fence && nouveau_fence(drm)->resume)
 		nouveau_fence(drm)->resume(drm);
 
 	nouveau_run_vbios_init(dev);
 
 	if (dev->mode_config.num_crtc) {
-		NV_INFO(drm, "resuming display...\n");
+		NV_DEBUG(drm, "resuming display...\n");
 		nouveau_display_resume(dev, runtime);
-		NV_INFO(drm, "resuming console...\n");
+		NV_DEBUG(drm, "resuming console...\n");
 		nouveau_fbcon_set_suspend(dev, 0);
 	}
 
@@ -998,7 +998,6 @@ driver_stub = {
 
 	.dumb_create = nouveau_display_dumb_create,
 	.dumb_map_offset = nouveau_display_dumb_map_offset,
-	.dumb_destroy = drm_gem_dumb_destroy,
 
 	.name = DRIVER_NAME,
 	.desc = DRIVER_DESC,
@@ -1098,7 +1097,6 @@ static int __init
 nouveau_drm_init(void)
 {
 	driver_pci = driver_stub;
-	driver_pci.set_busid = drm_pci_set_busid;
 	driver_platform = driver_stub;
 
 	nouveau_display_options();
@@ -1117,7 +1115,12 @@ nouveau_drm_init(void)
 
 	nouveau_register_dsm_handler();
 	nouveau_backlight_ctor();
-	return drm_pci_init(&driver_pci, &nouveau_drm_pci_driver);
+
+#ifdef CONFIG_PCI
+	return pci_register_driver(&nouveau_drm_pci_driver);
+#else
+	return 0;
+#endif
 }
 
 static void __exit
@@ -1126,7 +1129,9 @@ nouveau_drm_exit(void)
 	if (!nouveau_modeset)
 		return;
 
-	drm_pci_exit(&driver_pci, &nouveau_drm_pci_driver);
+#ifdef CONFIG_PCI
+	pci_unregister_driver(&nouveau_drm_pci_driver);
+#endif
 	nouveau_backlight_dtor();
 	nouveau_unregister_dsm_handler();
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.c b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
index 2665a078b6da..f7707849bb53 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
@@ -278,26 +278,6 @@ nouveau_fbcon_accel_init(struct drm_device *dev)
 		info->fbops = &nouveau_fbcon_ops;
 }
 
-static void nouveau_fbcon_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
-				    u16 blue, int regno)
-{
-	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-
-	nv_crtc->lut.r[regno] = red;
-	nv_crtc->lut.g[regno] = green;
-	nv_crtc->lut.b[regno] = blue;
-}
-
-static void nouveau_fbcon_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green,
-				    u16 *blue, int regno)
-{
-	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-
-	*red = nv_crtc->lut.r[regno];
-	*green = nv_crtc->lut.g[regno];
-	*blue = nv_crtc->lut.b[regno];
-}
-
 static void
 nouveau_fbcon_zfill(struct drm_device *dev, struct nouveau_fbdev *fbcon)
 {
@@ -467,8 +447,6 @@ void nouveau_fbcon_gpu_lockup(struct fb_info *info)
 }
 
 static const struct drm_fb_helper_funcs nouveau_fbcon_helper_funcs = {
-	.gamma_set = nouveau_fbcon_gamma_set,
-	.gamma_get = nouveau_fbcon_gamma_get,
 	.fb_probe = nouveau_fbcon_create,
 };
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c
index 999c35a25498..b0ad7fcefcf5 100644
--- a/drivers/gpu/drm/nouveau/nouveau_ttm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c
@@ -179,7 +179,8 @@ nouveau_gart_manager_new(struct ttm_mem_type_manager *man,
 }
 
 static void
-nouveau_gart_manager_debug(struct ttm_mem_type_manager *man, const char *prefix)
+nouveau_gart_manager_debug(struct ttm_mem_type_manager *man,
+			   struct drm_printer *printer)
 {
 }
 
@@ -252,7 +253,8 @@ nv04_gart_manager_new(struct ttm_mem_type_manager *man,
 }
 
 static void
-nv04_gart_manager_debug(struct ttm_mem_type_manager *man, const char *prefix)
+nv04_gart_manager_debug(struct ttm_mem_type_manager *man,
+			struct drm_printer *printer)
 {
 }
 
diff --git a/drivers/gpu/drm/nouveau/nv50_display.c b/drivers/gpu/drm/nouveau/nv50_display.c
index 2bc0dc985214..2dbf62a2ac41 100644
--- a/drivers/gpu/drm/nouveau/nv50_display.c
+++ b/drivers/gpu/drm/nouveau/nv50_display.c
@@ -1055,7 +1055,6 @@ nv50_wndw = {
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = nv50_wndw_destroy,
 	.reset = nv50_wndw_reset,
-	.set_property = drm_atomic_helper_plane_set_property,
 	.atomic_duplicate_state = nv50_wndw_atomic_duplicate_state,
 	.atomic_destroy_state = nv50_wndw_atomic_destroy_state,
 };
@@ -1083,8 +1082,9 @@ nv50_wndw_ctor(const struct nv50_wndw_func *func, struct drm_device *dev,
 	wndw->func = func;
 	wndw->dmac = dmac;
 
-	ret = drm_universal_plane_init(dev, &wndw->plane, 0, &nv50_wndw, format,
-				       nformat, type, "%s-%d", name, index);
+	ret = drm_universal_plane_init(dev, &wndw->plane, 0, &nv50_wndw,
+				       format, nformat, NULL,
+				       type, "%s-%d", name, index);
 	if (ret)
 		return ret;
 
@@ -2103,7 +2103,7 @@ nv50_head_atomic_check(struct drm_crtc *crtc, struct drm_crtc_state *state)
 
 	NV_ATOMIC(drm, "%s atomic_check %d\n", crtc->name, asyh->state.active);
 	if (asyh->state.active) {
-		for_each_connector_in_state(asyh->state.state, conn, conns, i) {
+		for_each_new_connector_in_state(asyh->state.state, conn, conns, i) {
 			if (conns->crtc == crtc) {
 				asyc = nouveau_conn_atom(conns);
 				break;
@@ -2204,28 +2204,29 @@ nv50_head_lut_load(struct drm_crtc *crtc)
 	struct nv50_disp *disp = nv50_disp(crtc->dev);
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
 	void __iomem *lut = nvbo_kmap_obj_iovirtual(nv_crtc->lut.nvbo);
+	u16 *r, *g, *b;
 	int i;
 
-	for (i = 0; i < 256; i++) {
-		u16 r = nv_crtc->lut.r[i] >> 2;
-		u16 g = nv_crtc->lut.g[i] >> 2;
-		u16 b = nv_crtc->lut.b[i] >> 2;
+	r = crtc->gamma_store;
+	g = r + crtc->gamma_size;
+	b = g + crtc->gamma_size;
 
+	for (i = 0; i < 256; i++) {
 		if (disp->disp->oclass < GF110_DISP) {
-			writew(r + 0x0000, lut + (i * 0x08) + 0);
-			writew(g + 0x0000, lut + (i * 0x08) + 2);
-			writew(b + 0x0000, lut + (i * 0x08) + 4);
+			writew((*r++ >> 2) + 0x0000, lut + (i * 0x08) + 0);
+			writew((*g++ >> 2) + 0x0000, lut + (i * 0x08) + 2);
+			writew((*b++ >> 2) + 0x0000, lut + (i * 0x08) + 4);
 		} else {
-			writew(r + 0x6000, lut + (i * 0x20) + 0);
-			writew(g + 0x6000, lut + (i * 0x20) + 2);
-			writew(b + 0x6000, lut + (i * 0x20) + 4);
+			/* 0x6000 interferes with the 14-bit color??? */
+			writew((*r++ >> 2) + 0x6000, lut + (i * 0x20) + 0);
+			writew((*g++ >> 2) + 0x6000, lut + (i * 0x20) + 2);
+			writew((*b++ >> 2) + 0x6000, lut + (i * 0x20) + 4);
 		}
 	}
 }
 
 static const struct drm_crtc_helper_funcs
 nv50_head_help = {
-	.load_lut = nv50_head_lut_load,
 	.atomic_check = nv50_head_atomic_check,
 };
 
@@ -2234,15 +2235,6 @@ nv50_head_gamma_set(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b,
 		    uint32_t size,
 		    struct drm_modeset_acquire_ctx *ctx)
 {
-	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-	u32 i;
-
-	for (i = 0; i < size; i++) {
-		nv_crtc->lut.r[i] = r[i];
-		nv_crtc->lut.g[i] = g[i];
-		nv_crtc->lut.b[i] = b[i];
-	}
-
 	nv50_head_lut_load(crtc);
 	return 0;
 }
@@ -2325,7 +2317,6 @@ nv50_head_func = {
 	.destroy = nv50_head_destroy,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
-	.set_property = drm_atomic_helper_crtc_set_property,
 	.atomic_duplicate_state = nv50_head_atomic_duplicate_state,
 	.atomic_destroy_state = nv50_head_atomic_destroy_state,
 };
@@ -2340,19 +2331,13 @@ nv50_head_create(struct drm_device *dev, int index)
 	struct nv50_base *base;
 	struct nv50_curs *curs;
 	struct drm_crtc *crtc;
-	int ret, i;
+	int ret;
 
 	head = kzalloc(sizeof(*head), GFP_KERNEL);
 	if (!head)
 		return -ENOMEM;
 
 	head->base.index = index;
-	for (i = 0; i < 256; i++) {
-		head->base.lut.r[i] = i << 8;
-		head->base.lut.g[i] = i << 8;
-		head->base.lut.b[i] = i << 8;
-	}
-
 	ret = nv50_base_new(drm, head, &base);
 	if (ret == 0)
 		ret = nv50_curs_new(drm, head, &curs);
@@ -2762,7 +2747,8 @@ nv50_hdmi_enable(struct drm_encoder *encoder, struct drm_display_mode *mode)
 	if (!drm_detect_hdmi_monitor(nv_connector->edid))
 		return;
 
-	ret = drm_hdmi_avi_infoframe_from_display_mode(&avi_frame.avi, mode);
+	ret = drm_hdmi_avi_infoframe_from_display_mode(&avi_frame.avi, mode,
+						       false);
 	if (!ret) {
 		/* We have an AVI InfoFrame, populate it to the display */
 		args.pwr.avi_infoframe_length
@@ -3119,11 +3105,9 @@ nv50_mstc_destroy(struct drm_connector *connector)
 
 static const struct drm_connector_funcs
 nv50_mstc = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.reset = nouveau_conn_reset,
 	.detect = nv50_mstc_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.set_property = drm_atomic_helper_connector_set_property,
 	.destroy = nv50_mstc_destroy,
 	.atomic_duplicate_state = nouveau_conn_atomic_duplicate_state,
 	.atomic_destroy_state = nouveau_conn_atomic_destroy_state,
@@ -3157,7 +3141,7 @@ nv50_mstc_new(struct nv50_mstm *mstm, struct drm_dp_mst_port *port,
 	mstc->connector.funcs->reset(&mstc->connector);
 	nouveau_conn_attach_properties(&mstc->connector);
 
-	for (i = 0; i < ARRAY_SIZE(mstm->msto) && mstm->msto; i++)
+	for (i = 0; i < ARRAY_SIZE(mstm->msto) && mstm->msto[i]; i++)
 		drm_mode_connector_attach_encoder(&mstc->connector, &mstm->msto[i]->encoder);
 
 	drm_object_attach_property(&mstc->connector.base, dev->mode_config.path_property, 0);
@@ -3913,9 +3897,9 @@ static void
 nv50_disp_atomic_commit_tail(struct drm_atomic_state *state)
 {
 	struct drm_device *dev = state->dev;
-	struct drm_crtc_state *crtc_state;
+	struct drm_crtc_state *new_crtc_state, *old_crtc_state;
 	struct drm_crtc *crtc;
-	struct drm_plane_state *plane_state;
+	struct drm_plane_state *new_plane_state;
 	struct drm_plane *plane;
 	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nv50_disp *disp = nv50_disp(dev);
@@ -3934,13 +3918,13 @@ nv50_disp_atomic_commit_tail(struct drm_atomic_state *state)
 		mutex_lock(&disp->mutex);
 
 	/* Disable head(s). */
-	for_each_crtc_in_state(state, crtc, crtc_state, i) {
-		struct nv50_head_atom *asyh = nv50_head_atom(crtc->state);
+	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
+		struct nv50_head_atom *asyh = nv50_head_atom(new_crtc_state);
 		struct nv50_head *head = nv50_head(crtc);
 
 		NV_ATOMIC(drm, "%s: clr %04x (set %04x)\n", crtc->name,
 			  asyh->clr.mask, asyh->set.mask);
-		if (crtc_state->active && !asyh->state.active)
+		if (old_crtc_state->active && !new_crtc_state->active)
 			drm_crtc_vblank_off(crtc);
 
 		if (asyh->clr.mask) {
@@ -3950,8 +3934,8 @@ nv50_disp_atomic_commit_tail(struct drm_atomic_state *state)
 	}
 
 	/* Disable plane(s). */
-	for_each_plane_in_state(state, plane, plane_state, i) {
-		struct nv50_wndw_atom *asyw = nv50_wndw_atom(plane->state);
+	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
+		struct nv50_wndw_atom *asyw = nv50_wndw_atom(new_plane_state);
 		struct nv50_wndw *wndw = nv50_wndw(plane);
 
 		NV_ATOMIC(drm, "%s: clr %02x (set %02x)\n", plane->name,
@@ -4016,8 +4000,8 @@ nv50_disp_atomic_commit_tail(struct drm_atomic_state *state)
 	}
 
 	/* Update head(s). */
-	for_each_crtc_in_state(state, crtc, crtc_state, i) {
-		struct nv50_head_atom *asyh = nv50_head_atom(crtc->state);
+	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
+		struct nv50_head_atom *asyh = nv50_head_atom(new_crtc_state);
 		struct nv50_head *head = nv50_head(crtc);
 
 		NV_ATOMIC(drm, "%s: set %04x (clr %04x)\n", crtc->name,
@@ -4028,17 +4012,17 @@ nv50_disp_atomic_commit_tail(struct drm_atomic_state *state)
 			interlock_core = 1;
 		}
 
-		if (asyh->state.active) {
-			if (!crtc_state->active)
+		if (new_crtc_state->active) {
+			if (!old_crtc_state->active)
 				drm_crtc_vblank_on(crtc);
-			if (asyh->state.event)
+			if (new_crtc_state->event)
 				drm_crtc_vblank_get(crtc);
 		}
 	}
 
 	/* Update plane(s). */
-	for_each_plane_in_state(state, plane, plane_state, i) {
-		struct nv50_wndw_atom *asyw = nv50_wndw_atom(plane->state);
+	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
+		struct nv50_wndw_atom *asyw = nv50_wndw_atom(new_plane_state);
 		struct nv50_wndw *wndw = nv50_wndw(plane);
 
 		NV_ATOMIC(drm, "%s: set %02x (clr %02x)\n", plane->name,
@@ -4068,25 +4052,26 @@ nv50_disp_atomic_commit_tail(struct drm_atomic_state *state)
 		mutex_unlock(&disp->mutex);
 
 	/* Wait for HW to signal completion. */
-	for_each_plane_in_state(state, plane, plane_state, i) {
-		struct nv50_wndw_atom *asyw = nv50_wndw_atom(plane->state);
+	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
+		struct nv50_wndw_atom *asyw = nv50_wndw_atom(new_plane_state);
 		struct nv50_wndw *wndw = nv50_wndw(plane);
 		int ret = nv50_wndw_wait_armed(wndw, asyw);
 		if (ret)
 			NV_ERROR(drm, "%s: timeout\n", plane->name);
 	}
 
-	for_each_crtc_in_state(state, crtc, crtc_state, i) {
-		if (crtc->state->event) {
+	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
+		if (new_crtc_state->event) {
 			unsigned long flags;
 			/* Get correct count/ts if racing with vblank irq */
-			if (crtc->state->active)
-				drm_accurate_vblank_count(crtc);
+			if (new_crtc_state->active)
+				drm_crtc_accurate_vblank_count(crtc);
 			spin_lock_irqsave(&crtc->dev->event_lock, flags);
-			drm_crtc_send_vblank_event(crtc, crtc->state->event);
+			drm_crtc_send_vblank_event(crtc, new_crtc_state->event);
 			spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
-			crtc->state->event = NULL;
-			if (crtc->state->active)
+
+			new_crtc_state->event = NULL;
+			if (new_crtc_state->active)
 				drm_crtc_vblank_put(crtc);
 		}
 	}
@@ -4111,7 +4096,7 @@ nv50_disp_atomic_commit(struct drm_device *dev,
 {
 	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nv50_disp *disp = nv50_disp(dev);
-	struct drm_plane_state *plane_state;
+	struct drm_plane_state *old_plane_state;
 	struct drm_plane *plane;
 	struct drm_crtc *crtc;
 	bool active = false;
@@ -4134,12 +4119,17 @@ nv50_disp_atomic_commit(struct drm_device *dev,
 	if (!nonblock) {
 		ret = drm_atomic_helper_wait_for_fences(dev, state, true);
 		if (ret)
-			goto done;
+			goto err_cleanup;
 	}
 
-	for_each_plane_in_state(state, plane, plane_state, i) {
-		struct nv50_wndw_atom *asyw = nv50_wndw_atom(plane_state);
+	ret = drm_atomic_helper_swap_state(state, true);
+	if (ret)
+		goto err_cleanup;
+
+	for_each_old_plane_in_state(state, plane, old_plane_state, i) {
+		struct nv50_wndw_atom *asyw = nv50_wndw_atom(old_plane_state);
 		struct nv50_wndw *wndw = nv50_wndw(plane);
+
 		if (asyw->set.image) {
 			asyw->ntfy.handle = wndw->dmac->sync.handle;
 			asyw->ntfy.offset = wndw->ntfy;
@@ -4150,7 +4140,6 @@ nv50_disp_atomic_commit(struct drm_device *dev,
 		}
 	}
 
-	drm_atomic_helper_swap_state(state, true);
 	drm_atomic_state_get(state);
 
 	if (nonblock)
@@ -4162,7 +4151,7 @@ nv50_disp_atomic_commit(struct drm_device *dev,
 		if (crtc->state->enable) {
 			if (!drm->have_disp_power_ref) {
 				drm->have_disp_power_ref = true;
-				return ret;
+				return 0;
 			}
 			active = true;
 			break;
@@ -4174,6 +4163,9 @@ nv50_disp_atomic_commit(struct drm_device *dev,
 		drm->have_disp_power_ref = false;
 	}
 
+err_cleanup:
+	if (ret)
+		drm_atomic_helper_cleanup_planes(dev, state);
 done:
 	pm_runtime_put_autosuspend(dev->dev);
 	return ret;
@@ -4200,18 +4192,19 @@ nv50_disp_outp_atomic_add(struct nv50_atom *atom, struct drm_encoder *encoder)
 
 static int
 nv50_disp_outp_atomic_check_clr(struct nv50_atom *atom,
-				struct drm_connector *connector)
+				struct drm_connector_state *old_connector_state)
 {
-	struct drm_encoder *encoder = connector->state->best_encoder;
-	struct drm_crtc_state *crtc_state;
+	struct drm_encoder *encoder = old_connector_state->best_encoder;
+	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
 	struct drm_crtc *crtc;
 	struct nv50_outp_atom *outp;
 
-	if (!(crtc = connector->state->crtc))
+	if (!(crtc = old_connector_state->crtc))
 		return 0;
 
-	crtc_state = drm_atomic_get_existing_crtc_state(&atom->state, crtc);
-	if (crtc->state->active && drm_atomic_crtc_needs_modeset(crtc_state)) {
+	old_crtc_state = drm_atomic_get_old_crtc_state(&atom->state, crtc);
+	new_crtc_state = drm_atomic_get_new_crtc_state(&atom->state, crtc);
+	if (old_crtc_state->active && drm_atomic_crtc_needs_modeset(new_crtc_state)) {
 		outp = nv50_disp_outp_atomic_add(atom, encoder);
 		if (IS_ERR(outp))
 			return PTR_ERR(outp);
@@ -4232,15 +4225,15 @@ nv50_disp_outp_atomic_check_set(struct nv50_atom *atom,
 				struct drm_connector_state *connector_state)
 {
 	struct drm_encoder *encoder = connector_state->best_encoder;
-	struct drm_crtc_state *crtc_state;
+	struct drm_crtc_state *new_crtc_state;
 	struct drm_crtc *crtc;
 	struct nv50_outp_atom *outp;
 
 	if (!(crtc = connector_state->crtc))
 		return 0;
 
-	crtc_state = drm_atomic_get_existing_crtc_state(&atom->state, crtc);
-	if (crtc_state->active && drm_atomic_crtc_needs_modeset(crtc_state)) {
+	new_crtc_state = drm_atomic_get_new_crtc_state(&atom->state, crtc);
+	if (new_crtc_state->active && drm_atomic_crtc_needs_modeset(new_crtc_state)) {
 		outp = nv50_disp_outp_atomic_add(atom, encoder);
 		if (IS_ERR(outp))
 			return PTR_ERR(outp);
@@ -4256,7 +4249,7 @@ static int
 nv50_disp_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
 {
 	struct nv50_atom *atom = nv50_atom(state);
-	struct drm_connector_state *connector_state;
+	struct drm_connector_state *old_connector_state, *new_connector_state;
 	struct drm_connector *connector;
 	int ret, i;
 
@@ -4264,12 +4257,12 @@ nv50_disp_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
 	if (ret)
 		return ret;
 
-	for_each_connector_in_state(state, connector, connector_state, i) {
-		ret = nv50_disp_outp_atomic_check_clr(atom, connector);
+	for_each_oldnew_connector_in_state(state, connector, old_connector_state, new_connector_state, i) {
+		ret = nv50_disp_outp_atomic_check_clr(atom, old_connector_state);
 		if (ret)
 			return ret;
 
-		ret = nv50_disp_outp_atomic_check_set(atom, connector_state);
+		ret = nv50_disp_outp_atomic_check_set(atom, new_connector_state);
 		if (ret)
 			return ret;
 	}
@@ -4458,11 +4451,13 @@ nv50_display_create(struct drm_device *dev)
 
 	/* create crtc objects to represent the hw heads */
 	if (disp->disp->oclass >= GF110_DISP)
-		crtcs = nvif_rd32(&device->object, 0x022448);
+		crtcs = nvif_rd32(&device->object, 0x612004) & 0xf;
 	else
-		crtcs = 2;
+		crtcs = 0x3;
 
-	for (i = 0; i < crtcs; i++) {
+	for (i = 0; i < fls(crtcs); i++) {
+		if (!(crtcs & (1 << i)))
+			continue;
 		ret = nv50_head_create(dev, i);
 		if (ret)
 			goto out;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
index 7bdc7a5ae723..e096a5d9c292 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
@@ -2043,6 +2043,7 @@ nv120_chipset = {
 	.mxm = nv50_mxm_new,
 	.pci = gk104_pci_new,
 	.pmu = gm107_pmu_new,
+	.therm = gm200_therm_new,
 	.secboot = gm200_secboot_new,
 	.timer = gk20a_timer_new,
 	.top = gk104_top_new,
@@ -2077,6 +2078,7 @@ nv124_chipset = {
 	.mxm = nv50_mxm_new,
 	.pci = gk104_pci_new,
 	.pmu = gm107_pmu_new,
+	.therm = gm200_therm_new,
 	.secboot = gm200_secboot_new,
 	.timer = gk20a_timer_new,
 	.top = gk104_top_new,
@@ -2111,6 +2113,7 @@ nv126_chipset = {
 	.mxm = nv50_mxm_new,
 	.pci = gk104_pci_new,
 	.pmu = gm107_pmu_new,
+	.therm = gm200_therm_new,
 	.secboot = gm200_secboot_new,
 	.timer = gk20a_timer_new,
 	.top = gk104_top_new,
@@ -2321,6 +2324,35 @@ nv137_chipset = {
 };
 
 static const struct nvkm_device_chip
+nv138_chipset = {
+	.name = "GP108",
+	.bar = gf100_bar_new,
+	.bios = nvkm_bios_new,
+	.bus = gf100_bus_new,
+	.devinit = gm200_devinit_new,
+	.fb = gp102_fb_new,
+	.fuse = gm107_fuse_new,
+	.gpio = gk104_gpio_new,
+	.i2c = gm200_i2c_new,
+	.ibus = gm200_ibus_new,
+	.imem = nv50_instmem_new,
+	.ltc = gp100_ltc_new,
+	.mc = gp100_mc_new,
+	.mmu = gf100_mmu_new,
+	.pci = gp100_pci_new,
+	.pmu = gp102_pmu_new,
+	.timer = gk20a_timer_new,
+	.top = gk104_top_new,
+	.ce[0] = gp102_ce_new,
+	.ce[1] = gp102_ce_new,
+	.ce[2] = gp102_ce_new,
+	.ce[3] = gp102_ce_new,
+	.disp = gp102_disp_new,
+	.dma = gf119_dma_new,
+	.fifo = gp100_fifo_new,
+};
+
+static const struct nvkm_device_chip
 nv13b_chipset = {
 	.name = "GP10B",
 	.bar = gk20a_bar_new,
@@ -2782,6 +2814,7 @@ nvkm_device_ctor(const struct nvkm_device_func *func,
 		case 0x134: device->chip = &nv134_chipset; break;
 		case 0x136: device->chip = &nv136_chipset; break;
 		case 0x137: device->chip = &nv137_chipset; break;
+		case 0x138: device->chip = &nv138_chipset; break;
 		case 0x13b: device->chip = &nv13b_chipset; break;
 		default:
 			nvdev_error(device, "unknown chipset (%08x)\n", boot0);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c
index 88582af8bd89..93a75e5b2791 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c
@@ -285,6 +285,10 @@ nvkm_disp_oneinit(struct nvkm_engine *engine)
 		case DCB_OUTPUT_DP:
 			ret = nvkm_dp_new(disp, i, &dcbE, &outp);
 			break;
+		case DCB_OUTPUT_WFD:
+			/* No support for WFD yet. */
+			ret = -ENODEV;
+			continue;
 		default:
 			nvkm_warn(subdev, "dcb %d type %d unknown\n",
 				  i, dcbE.type);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/headgf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/headgf119.c
index b33552757647..9fd7ae331308 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/headgf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/headgf119.c
@@ -92,5 +92,8 @@ gf119_head = {
 int
 gf119_head_new(struct nvkm_disp *disp, int id)
 {
+	struct nvkm_device *device = disp->engine.subdev.device;
+	if (!(nvkm_rd32(device, 0x612004) & (0x00000001 << id)))
+		return 0;
 	return nvkm_head_new_(&gf119_head, disp, id);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv31.c b/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv31.c
index 8a8895246d26..7fea7d45202f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv31.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv31.c
@@ -124,6 +124,8 @@ nv31_mpeg_tile(struct nvkm_engine *engine, int i, struct nvkm_fb_tile *tile)
 static bool
 nv31_mpeg_mthd_dma(struct nvkm_device *device, u32 mthd, u32 data)
 {
+	struct nv31_mpeg *mpeg = nv31_mpeg(device->mpeg);
+	struct nvkm_subdev *subdev = &mpeg->engine.subdev;
 	u32 inst = data << 4;
 	u32 dma0 = nvkm_rd32(device, 0x700000 + inst);
 	u32 dma1 = nvkm_rd32(device, 0x700004 + inst);
@@ -132,8 +134,11 @@ nv31_mpeg_mthd_dma(struct nvkm_device *device, u32 mthd, u32 data)
 	u32 size = dma1 + 1;
 
 	/* only allow linear DMA objects */
-	if (!(dma0 & 0x00002000))
+	if (!(dma0 & 0x00002000)) {
+		nvkm_error(subdev, "inst %08x dma0 %08x dma1 %08x dma2 %08x\n",
+			   inst, dma0, dma1, dma2);
 		return false;
+	}
 
 	if (mthd == 0x0190) {
 		/* DMA_CMD */
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv40.c
index 16de5bd94b14..b5ec7c504dc6 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv40.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv40.c
@@ -31,6 +31,8 @@ bool
 nv40_mpeg_mthd_dma(struct nvkm_device *device, u32 mthd, u32 data)
 {
 	struct nvkm_instmem *imem = device->imem;
+	struct nv31_mpeg *mpeg = nv31_mpeg(device->mpeg);
+	struct nvkm_subdev *subdev = &mpeg->engine.subdev;
 	u32 inst = data << 4;
 	u32 dma0 = nvkm_instmem_rd32(imem, inst + 0);
 	u32 dma1 = nvkm_instmem_rd32(imem, inst + 4);
@@ -39,8 +41,11 @@ nv40_mpeg_mthd_dma(struct nvkm_device *device, u32 mthd, u32 data)
 	u32 size = dma1 + 1;
 
 	/* only allow linear DMA objects */
-	if (!(dma0 & 0x00002000))
+	if (!(dma0 & 0x00002000)) {
+		nvkm_error(subdev, "inst %08x dma0 %08x dma1 %08x dma2 %08x\n",
+			   inst, dma0, dma1, dma2);
 		return false;
+	}
 
 	if (mthd == 0x0190) {
 		/* DMA_CMD */
diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/msgqueue.c b/drivers/gpu/drm/nouveau/nvkm/falcon/msgqueue.c
index d45d7947a964..77273b53672c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/falcon/msgqueue.c
+++ b/drivers/gpu/drm/nouveau/nvkm/falcon/msgqueue.c
@@ -251,7 +251,7 @@ cmd_write(struct nvkm_msgqueue *priv, struct nvkm_msgqueue_hdr *cmd,
 	  struct nvkm_msgqueue_queue *queue)
 {
 	const struct nvkm_subdev *subdev = priv->falcon->owner;
-	static unsigned long timeout = ~0;
+	static unsigned timeout = 2000;
 	unsigned long end_jiffies = jiffies + msecs_to_jiffies(timeout);
 	int ret = -EAGAIN;
 	bool commit = true;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c
index 6d8f21290aa2..676c167c95b9 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c
@@ -24,6 +24,7 @@
 #include "gf100.h"
 
 #include <core/gpuobj.h>
+#include <core/option.h>
 #include <subdev/fb.h>
 #include <subdev/mmu.h>
 
@@ -59,6 +60,8 @@ gf100_bar_ctor_vm(struct gf100_bar *bar, struct gf100_bar_vm *bar_vm,
 		return ret;
 
 	bar_len = device->func->resource_size(device, bar_nr);
+	if (bar_nr == 3 && bar->bar2_halve)
+		bar_len >>= 1;
 
 	ret = nvkm_vm_new(device, 0, bar_len, 0, key, &vm);
 	if (ret)
@@ -129,6 +132,8 @@ gf100_bar_init(struct nvkm_bar *base)
 
 	if (bar->bar[0].mem) {
 		addr = nvkm_memory_addr(bar->bar[0].mem) >> 12;
+		if (bar->bar2_halve)
+			addr |= 0x40000000;
 		nvkm_wr32(device, 0x001714, 0x80000000 | addr);
 	}
 
@@ -161,6 +166,7 @@ gf100_bar_new_(const struct nvkm_bar_func *func, struct nvkm_device *device,
 	if (!(bar = kzalloc(sizeof(*bar), GFP_KERNEL)))
 		return -ENOMEM;
 	nvkm_bar_ctor(func, device, index, &bar->base);
+	bar->bar2_halve = nvkm_boolopt(device->cfgopt, "NvBar2Halve", false);
 	*pbar = &bar->base;
 	return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h
index f7dea69640d8..20a5255362ba 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.h
@@ -11,6 +11,7 @@ struct gf100_bar_vm {
 
 struct gf100_bar {
 	struct nvkm_bar base;
+	bool bar2_halve;
 	struct gf100_bar_vm bar[2];
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c
index 3841ad6be99e..a239e73562c8 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c
@@ -60,12 +60,12 @@ gf100_fb_oneinit(struct nvkm_fb *base)
 	size = min(size, 0x1000);
 
 	ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, size, 0x1000,
-			      false, &fb->base.mmu_rd);
+			      true, &fb->base.mmu_rd);
 	if (ret)
 		return ret;
 
 	ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, size, 0x1000,
-			      false, &fb->base.mmu_wr);
+			      true, &fb->base.mmu_wr);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/base.c
index 4a57defc99b3..1399d923d446 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/base.c
@@ -171,7 +171,7 @@ nvkm_gpio_fini(struct nvkm_subdev *subdev, bool suspend)
 	return 0;
 }
 
-static struct dmi_system_id gpio_reset_ids[] = {
+static const struct dmi_system_id gpio_reset_ids[] = {
 	{
 		.ident = "Apple Macbook 10,1",
 		.matches = {
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gf100.c
index d2c4d6033abb..f93766418056 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gf100.c
@@ -27,6 +27,7 @@ static const struct nvkm_mc_map
 gf100_mc_reset[] = {
 	{ 0x00020000, NVKM_ENGINE_MSPDEC },
 	{ 0x00008000, NVKM_ENGINE_MSVLD },
+	{ 0x00002000, NVKM_SUBDEV_PMU, true },
 	{ 0x00001000, NVKM_ENGINE_GR },
 	{ 0x00000100, NVKM_ENGINE_FIFO },
 	{ 0x00000080, NVKM_ENGINE_CE1 },
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c
index eb9b278198b2..a4cb82495cee 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c
@@ -192,6 +192,10 @@ nvkm_pci_new_(const struct nvkm_pci_func *func, struct nvkm_device *device,
 		}
 	}
 
+#ifdef __BIG_ENDIAN
+	pci->msi = false;
+#endif
+
 	pci->msi = nvkm_boolopt(device->cfgopt, "NvMSI", pci->msi);
 	if (pci->msi && func->msi_rearm) {
 		pci->msi = pci_enable_msi(pci->pdev) == 0;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c
index 3306f9fe7140..ce70a193caa7 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c
@@ -75,7 +75,7 @@ nvkm_pmu_reset(struct nvkm_pmu *pmu)
 {
 	struct nvkm_device *device = pmu->subdev.device;
 
-	if (!(nvkm_rd32(device, 0x000200) & 0x00002000))
+	if (!pmu->func->enabled(pmu))
 		return 0;
 
 	/* Inhibit interrupts, and wait for idle. */
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gf100.c
index 0e36d4cb7201..0b458656e870 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gf100.c
@@ -24,13 +24,30 @@
 #include "priv.h"
 #include "fuc/gf100.fuc3.h"
 
+#include <subdev/mc.h>
+
+void
+gf100_pmu_reset(struct nvkm_pmu *pmu)
+{
+	struct nvkm_device *device = pmu->subdev.device;
+	nvkm_mc_disable(device, NVKM_SUBDEV_PMU);
+	nvkm_mc_enable(device, NVKM_SUBDEV_PMU);
+}
+
+bool
+gf100_pmu_enabled(struct nvkm_pmu *pmu)
+{
+	return nvkm_mc_enabled(pmu->subdev.device, NVKM_SUBDEV_PMU);
+}
+
 static const struct nvkm_pmu_func
 gf100_pmu = {
 	.code.data = gf100_pmu_code,
 	.code.size = sizeof(gf100_pmu_code),
 	.data.data = gf100_pmu_data,
 	.data.size = sizeof(gf100_pmu_data),
-	.reset = gt215_pmu_reset,
+	.enabled = gf100_pmu_enabled,
+	.reset = gf100_pmu_reset,
 	.init = gt215_pmu_init,
 	.fini = gt215_pmu_fini,
 	.intr = gt215_pmu_intr,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gf119.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gf119.c
index 0e4ba4248b15..3dfa79d4fb13 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gf119.c
@@ -30,7 +30,8 @@ gf119_pmu = {
 	.code.size = sizeof(gf119_pmu_code),
 	.data.data = gf119_pmu_data,
 	.data.size = sizeof(gf119_pmu_data),
-	.reset = gt215_pmu_reset,
+	.enabled = gf100_pmu_enabled,
+	.reset = gf100_pmu_reset,
 	.init = gt215_pmu_init,
 	.fini = gt215_pmu_fini,
 	.intr = gt215_pmu_intr,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk104.c
index 2ad858d825ac..8f7ec10fd2a4 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk104.c
@@ -109,7 +109,8 @@ gk104_pmu = {
 	.code.size = sizeof(gk104_pmu_code),
 	.data.data = gk104_pmu_data,
 	.data.size = sizeof(gk104_pmu_data),
-	.reset = gt215_pmu_reset,
+	.enabled = gf100_pmu_enabled,
+	.reset = gf100_pmu_reset,
 	.init = gt215_pmu_init,
 	.fini = gt215_pmu_fini,
 	.intr = gt215_pmu_intr,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk110.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk110.c
index fc4b8ecfdaeb..345741d55a56 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk110.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk110.c
@@ -88,7 +88,8 @@ gk110_pmu = {
 	.code.size = sizeof(gk110_pmu_code),
 	.data.data = gk110_pmu_data,
 	.data.size = sizeof(gk110_pmu_data),
-	.reset = gt215_pmu_reset,
+	.enabled = gf100_pmu_enabled,
+	.reset = gf100_pmu_reset,
 	.init = gt215_pmu_init,
 	.fini = gt215_pmu_fini,
 	.intr = gt215_pmu_intr,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk208.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk208.c
index e9a91277683a..e4acf7876ea1 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk208.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk208.c
@@ -30,7 +30,8 @@ gk208_pmu = {
 	.code.size = sizeof(gk208_pmu_code),
 	.data.data = gk208_pmu_data,
 	.data.size = sizeof(gk208_pmu_data),
-	.reset = gt215_pmu_reset,
+	.enabled = gf100_pmu_enabled,
+	.reset = gf100_pmu_reset,
 	.init = gt215_pmu_init,
 	.fini = gt215_pmu_fini,
 	.intr = gt215_pmu_intr,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk20a.c
index 978aae3c1001..05e81855c367 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk20a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk20a.c
@@ -196,9 +196,10 @@ gk20a_dvfs_data= {
 
 static const struct nvkm_pmu_func
 gk20a_pmu = {
+	.enabled = gf100_pmu_enabled,
 	.init = gk20a_pmu_init,
 	.fini = gk20a_pmu_fini,
-	.reset = gt215_pmu_reset,
+	.reset = gf100_pmu_reset,
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm107.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm107.c
index 9a248ed75f09..459df1ef9e70 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm107.c
@@ -32,7 +32,8 @@ gm107_pmu = {
 	.code.size = sizeof(gm107_pmu_code),
 	.data.data = gm107_pmu_data,
 	.data.size = sizeof(gm107_pmu_data),
-	.reset = gt215_pmu_reset,
+	.enabled = gf100_pmu_enabled,
+	.reset = gf100_pmu_reset,
 	.init = gt215_pmu_init,
 	.fini = gt215_pmu_fini,
 	.intr = gt215_pmu_intr,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm20b.c
index 44bef22bce52..31c843145c7a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm20b.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm20b.c
@@ -38,6 +38,7 @@ gm20b_pmu_recv(struct nvkm_pmu *pmu)
 
 static const struct nvkm_pmu_func
 gm20b_pmu = {
+	.enabled = gf100_pmu_enabled,
 	.intr = gt215_pmu_intr,
 	.recv = gm20b_pmu_recv,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp100.c
index 6c41c20c85a7..e210cd6af816 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp100.c
@@ -25,7 +25,8 @@
 
 static const struct nvkm_pmu_func
 gp100_pmu = {
-	.reset = gt215_pmu_reset,
+	.enabled = gf100_pmu_enabled,
+	.reset = gf100_pmu_reset,
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c
index f017352206c9..98c7a2a8afc4 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c
@@ -31,8 +31,15 @@ gp102_pmu_reset(struct nvkm_pmu *pmu)
 	nvkm_mask(device, 0x10a3c0, 0x00000001, 0x00000000);
 }
 
+static bool
+gp102_pmu_enabled(struct nvkm_pmu *pmu)
+{
+	return !(nvkm_rd32(pmu->subdev.device, 0x10a3c0) & 0x00000001);
+}
+
 static const struct nvkm_pmu_func
 gp102_pmu = {
+	.enabled = gp102_pmu_enabled,
 	.reset = gp102_pmu_reset,
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gt215.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gt215.c
index 90d428b3be97..e04216daea58 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gt215.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gt215.c
@@ -180,13 +180,19 @@ gt215_pmu_fini(struct nvkm_pmu *pmu)
 	nvkm_wr32(pmu->subdev.device, 0x10a014, 0x00000060);
 }
 
-void
+static void
 gt215_pmu_reset(struct nvkm_pmu *pmu)
 {
 	struct nvkm_device *device = pmu->subdev.device;
-	nvkm_mask(device, 0x000200, 0x00002000, 0x00000000);
-	nvkm_mask(device, 0x000200, 0x00002000, 0x00002000);
-	nvkm_rd32(device, 0x000200);
+	nvkm_mask(device, 0x022210, 0x00000001, 0x00000000);
+	nvkm_mask(device, 0x022210, 0x00000001, 0x00000001);
+	nvkm_rd32(device, 0x022210);
+}
+
+static bool
+gt215_pmu_enabled(struct nvkm_pmu *pmu)
+{
+	return nvkm_rd32(pmu->subdev.device, 0x022210) & 0x00000001;
 }
 
 int
@@ -241,6 +247,7 @@ gt215_pmu = {
 	.code.size = sizeof(gt215_pmu_code),
 	.data.data = gt215_pmu_data,
 	.data.size = sizeof(gt215_pmu_data),
+	.enabled = gt215_pmu_enabled,
 	.reset = gt215_pmu_reset,
 	.init = gt215_pmu_init,
 	.fini = gt215_pmu_fini,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h
index 096cba069f72..a4c48a10cd47 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h
@@ -20,6 +20,7 @@ struct nvkm_pmu_func {
 		u32  size;
 	} data;
 
+	bool (*enabled)(struct nvkm_pmu *);
 	void (*reset)(struct nvkm_pmu *);
 	int (*init)(struct nvkm_pmu *);
 	void (*fini)(struct nvkm_pmu *);
@@ -30,12 +31,14 @@ struct nvkm_pmu_func {
 	void (*pgob)(struct nvkm_pmu *, bool);
 };
 
-void gt215_pmu_reset(struct nvkm_pmu *);
 int gt215_pmu_init(struct nvkm_pmu *);
 void gt215_pmu_fini(struct nvkm_pmu *);
 void gt215_pmu_intr(struct nvkm_pmu *);
 void gt215_pmu_recv(struct nvkm_pmu *);
 int gt215_pmu_send(struct nvkm_pmu *, u32[2], u32, u32, u32, u32);
 
+bool gf100_pmu_enabled(struct nvkm_pmu *);
+void gf100_pmu_reset(struct nvkm_pmu *);
+
 void gk110_pmu_pgob(struct nvkm_pmu *, bool);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/Kbuild
index 135758ba3e28..2bafcc1d1818 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/Kbuild
@@ -11,3 +11,4 @@ nvkm-y += nvkm/subdev/therm/g84.o
 nvkm-y += nvkm/subdev/therm/gt215.o
 nvkm-y += nvkm/subdev/therm/gf119.o
 nvkm-y += nvkm/subdev/therm/gm107.o
+nvkm-y += nvkm/subdev/therm/gm200.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/g84.c
index 86e81930d8ee..96f8da40ac82 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/g84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/g84.c
@@ -203,7 +203,7 @@ g84_therm_fini(struct nvkm_therm *therm)
 	nvkm_wr32(device, 0x1100, 0x10000); /* PBUS */
 }
 
-static void
+void
 g84_therm_init(struct nvkm_therm *therm)
 {
 	g84_sensor_setup(therm);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gm200.c
new file mode 100644
index 000000000000..73dc78093d5d
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gm200.c
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2017 Karol Herbst
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Karol Herbst
+ */
+#include "priv.h"
+
+static const struct nvkm_therm_func
+gm200_therm = {
+	.init = g84_therm_init,
+	.fini = g84_therm_fini,
+	.temp_get = g84_temp_get,
+	.program_alarms = nvkm_therm_program_alarms_polling,
+};
+
+int
+gm200_therm_new(struct nvkm_device *device, int index,
+		struct nvkm_therm **ptherm)
+{
+	return nvkm_therm_new_(&gm200_therm, device, index, ptherm);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/priv.h
index 235a5d8daff6..1f46e371d7c4 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/priv.h
@@ -111,6 +111,7 @@ void g84_therm_fini(struct nvkm_therm *);
 
 int gt215_therm_fan_sense(struct nvkm_therm *);
 
+void g84_therm_init(struct nvkm_therm *);
 void gf119_therm_init(struct nvkm_therm *);
 
 int nvkm_fanpwm_create(struct nvkm_therm *, struct dcb_gpio_func *);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/temp.c b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/temp.c
index e93b2410c38b..ddb2b2c600ca 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/temp.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/temp.c
@@ -83,7 +83,7 @@ nvkm_therm_sensor_event(struct nvkm_therm *therm, enum nvkm_therm_thrs thrs,
 {
 	struct nvkm_subdev *subdev = &therm->subdev;
 	bool active;
-	const char *thresolds[] = {
+	static const char * const thresholds[] = {
 		"fanboost", "downclock", "critical", "shutdown"
 	};
 	int temperature = therm->func->temp_get(therm);
@@ -94,10 +94,10 @@ nvkm_therm_sensor_event(struct nvkm_therm *therm, enum nvkm_therm_thrs thrs,
 	if (dir == NVKM_THERM_THRS_FALLING)
 		nvkm_info(subdev,
 			  "temperature (%i C) went below the '%s' threshold\n",
-			  temperature, thresolds[thrs]);
+			  temperature, thresholds[thrs]);
 	else
 		nvkm_info(subdev, "temperature (%i C) hit the '%s' threshold\n",
-			  temperature, thresolds[thrs]);
+			  temperature, thresholds[thrs]);
 
 	active = (dir == NVKM_THERM_THRS_RISING);
 	switch (thrs) {
diff --git a/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c b/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c
index e1fa143a5625..542a76503fbd 100644
--- a/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c
+++ b/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c
@@ -198,6 +198,9 @@ static int tvc_probe(struct platform_device *pdev)
 	struct omap_dss_device *dssdev;
 	int r;
 
+	if (!pdev->dev.of_node)
+		return -ENODEV;
+
 	ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
 	if (!ddata)
 		return -ENOMEM;
diff --git a/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c b/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c
index 79cb69f1acf5..d9d25df6fc1b 100644
--- a/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c
+++ b/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c
@@ -15,6 +15,7 @@
 #include <linux/platform_device.h>
 #include <linux/of.h>
 #include <linux/of_gpio.h>
+#include <linux/mutex.h>
 
 #include <drm/drm_edid.h>
 
@@ -37,6 +38,10 @@ static const struct videomode hdmic_default_vm = {
 struct panel_drv_data {
 	struct omap_dss_device dssdev;
 	struct omap_dss_device *in;
+	void (*hpd_cb)(void *cb_data, enum drm_connector_status status);
+	void *hpd_cb_data;
+	bool hpd_enabled;
+	struct mutex hpd_lock;
 
 	struct device *dev;
 
@@ -167,6 +172,70 @@ static bool hdmic_detect(struct omap_dss_device *dssdev)
 		return in->ops.hdmi->detect(in);
 }
 
+static int hdmic_register_hpd_cb(struct omap_dss_device *dssdev,
+				 void (*cb)(void *cb_data,
+					    enum drm_connector_status status),
+				 void *cb_data)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	if (gpio_is_valid(ddata->hpd_gpio)) {
+		mutex_lock(&ddata->hpd_lock);
+		ddata->hpd_cb = cb;
+		ddata->hpd_cb_data = cb_data;
+		mutex_unlock(&ddata->hpd_lock);
+		return 0;
+	} else if (in->ops.hdmi->register_hpd_cb) {
+		return in->ops.hdmi->register_hpd_cb(in, cb, cb_data);
+	}
+
+	return -ENOTSUPP;
+}
+
+static void hdmic_unregister_hpd_cb(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	if (gpio_is_valid(ddata->hpd_gpio)) {
+		mutex_lock(&ddata->hpd_lock);
+		ddata->hpd_cb = NULL;
+		ddata->hpd_cb_data = NULL;
+		mutex_unlock(&ddata->hpd_lock);
+	} else if (in->ops.hdmi->unregister_hpd_cb) {
+		in->ops.hdmi->unregister_hpd_cb(in);
+	}
+}
+
+static void hdmic_enable_hpd(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	if (gpio_is_valid(ddata->hpd_gpio)) {
+		mutex_lock(&ddata->hpd_lock);
+		ddata->hpd_enabled = true;
+		mutex_unlock(&ddata->hpd_lock);
+	} else if (in->ops.hdmi->enable_hpd) {
+		in->ops.hdmi->enable_hpd(in);
+	}
+}
+
+static void hdmic_disable_hpd(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	struct omap_dss_device *in = ddata->in;
+
+	if (gpio_is_valid(ddata->hpd_gpio)) {
+		mutex_lock(&ddata->hpd_lock);
+		ddata->hpd_enabled = false;
+		mutex_unlock(&ddata->hpd_lock);
+	} else if (in->ops.hdmi->disable_hpd) {
+		in->ops.hdmi->disable_hpd(in);
+	}
+}
+
 static int hdmic_set_hdmi_mode(struct omap_dss_device *dssdev, bool hdmi_mode)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
@@ -197,10 +266,34 @@ static struct omap_dss_driver hdmic_driver = {
 
 	.read_edid		= hdmic_read_edid,
 	.detect			= hdmic_detect,
+	.register_hpd_cb	= hdmic_register_hpd_cb,
+	.unregister_hpd_cb	= hdmic_unregister_hpd_cb,
+	.enable_hpd		= hdmic_enable_hpd,
+	.disable_hpd		= hdmic_disable_hpd,
 	.set_hdmi_mode		= hdmic_set_hdmi_mode,
 	.set_hdmi_infoframe	= hdmic_set_infoframe,
 };
 
+static irqreturn_t hdmic_hpd_isr(int irq, void *data)
+{
+	struct panel_drv_data *ddata = data;
+
+	mutex_lock(&ddata->hpd_lock);
+	if (ddata->hpd_enabled && ddata->hpd_cb) {
+		enum drm_connector_status status;
+
+		if (hdmic_detect(&ddata->dssdev))
+			status = connector_status_connected;
+		else
+			status = connector_status_disconnected;
+
+		ddata->hpd_cb(ddata->hpd_cb_data, status);
+	}
+	mutex_unlock(&ddata->hpd_lock);
+
+	return IRQ_HANDLED;
+}
+
 static int hdmic_probe_of(struct platform_device *pdev)
 {
 	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
@@ -246,11 +339,22 @@ static int hdmic_probe(struct platform_device *pdev)
 	if (r)
 		return r;
 
+	mutex_init(&ddata->hpd_lock);
+
 	if (gpio_is_valid(ddata->hpd_gpio)) {
 		r = devm_gpio_request_one(&pdev->dev, ddata->hpd_gpio,
 				GPIOF_DIR_IN, "hdmi_hpd");
 		if (r)
 			goto err_reg;
+
+		r = devm_request_threaded_irq(&pdev->dev,
+				gpio_to_irq(ddata->hpd_gpio),
+				NULL, hdmic_hpd_isr,
+				IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING |
+				IRQF_ONESHOT,
+				"hdmic hpd", ddata);
+		if (r)
+			goto err_reg;
 	}
 
 	ddata->vm = hdmic_default_vm;
diff --git a/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c b/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c
index 58276a48112e..a9e9d667c55e 100644
--- a/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c
+++ b/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c
@@ -15,12 +15,17 @@
 #include <linux/slab.h>
 #include <linux/platform_device.h>
 #include <linux/gpio/consumer.h>
+#include <linux/mutex.h>
 
 #include "../dss/omapdss.h"
 
 struct panel_drv_data {
 	struct omap_dss_device dssdev;
 	struct omap_dss_device *in;
+	void (*hpd_cb)(void *cb_data, enum drm_connector_status status);
+	void *hpd_cb_data;
+	bool hpd_enabled;
+	struct mutex hpd_lock;
 
 	struct gpio_desc *ct_cp_hpd_gpio;
 	struct gpio_desc *ls_oe_gpio;
@@ -162,6 +167,49 @@ static bool tpd_detect(struct omap_dss_device *dssdev)
 	return gpiod_get_value_cansleep(ddata->hpd_gpio);
 }
 
+static int tpd_register_hpd_cb(struct omap_dss_device *dssdev,
+			       void (*cb)(void *cb_data,
+					  enum drm_connector_status status),
+			       void *cb_data)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+
+	mutex_lock(&ddata->hpd_lock);
+	ddata->hpd_cb = cb;
+	ddata->hpd_cb_data = cb_data;
+	mutex_unlock(&ddata->hpd_lock);
+
+	return 0;
+}
+
+static void tpd_unregister_hpd_cb(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+
+	mutex_lock(&ddata->hpd_lock);
+	ddata->hpd_cb = NULL;
+	ddata->hpd_cb_data = NULL;
+	mutex_unlock(&ddata->hpd_lock);
+}
+
+static void tpd_enable_hpd(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+
+	mutex_lock(&ddata->hpd_lock);
+	ddata->hpd_enabled = true;
+	mutex_unlock(&ddata->hpd_lock);
+}
+
+static void tpd_disable_hpd(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+
+	mutex_lock(&ddata->hpd_lock);
+	ddata->hpd_enabled = false;
+	mutex_unlock(&ddata->hpd_lock);
+}
+
 static int tpd_set_infoframe(struct omap_dss_device *dssdev,
 		const struct hdmi_avi_infoframe *avi)
 {
@@ -193,10 +241,34 @@ static const struct omapdss_hdmi_ops tpd_hdmi_ops = {
 
 	.read_edid		= tpd_read_edid,
 	.detect			= tpd_detect,
+	.register_hpd_cb	= tpd_register_hpd_cb,
+	.unregister_hpd_cb	= tpd_unregister_hpd_cb,
+	.enable_hpd		= tpd_enable_hpd,
+	.disable_hpd		= tpd_disable_hpd,
 	.set_infoframe		= tpd_set_infoframe,
 	.set_hdmi_mode		= tpd_set_hdmi_mode,
 };
 
+static irqreturn_t tpd_hpd_isr(int irq, void *data)
+{
+	struct panel_drv_data *ddata = data;
+
+	mutex_lock(&ddata->hpd_lock);
+	if (ddata->hpd_enabled && ddata->hpd_cb) {
+		enum drm_connector_status status;
+
+		if (tpd_detect(&ddata->dssdev))
+			status = connector_status_connected;
+		else
+			status = connector_status_disconnected;
+
+		ddata->hpd_cb(ddata->hpd_cb_data, status);
+	}
+	mutex_unlock(&ddata->hpd_lock);
+
+	return IRQ_HANDLED;
+}
+
 static int tpd_probe_of(struct platform_device *pdev)
 {
 	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
@@ -261,6 +333,15 @@ static int tpd_probe(struct platform_device *pdev)
 
 	ddata->hpd_gpio = gpio;
 
+	mutex_init(&ddata->hpd_lock);
+
+	r = devm_request_threaded_irq(&pdev->dev, gpiod_to_irq(ddata->hpd_gpio),
+		NULL, tpd_hpd_isr,
+		IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
+		"tpd12s015 hpd", ddata);
+	if (r)
+		goto err_gpio;
+
 	dssdev = &ddata->dssdev;
 	dssdev->ops.hdmi = &tpd_hdmi_ops;
 	dssdev->dev = &pdev->dev;
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-dpi.c b/drivers/gpu/drm/omapdrm/displays/panel-dpi.c
index 6468a765f3d1..e065f7e10cca 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-dpi.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-dpi.c
@@ -231,6 +231,9 @@ static int panel_dpi_probe(struct platform_device *pdev)
 	struct omap_dss_device *dssdev;
 	int r;
 
+	if (!pdev->dev.of_node)
+		return -ENODEV;
+
 	ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
 	if (ddata == NULL)
 		return -ENOMEM;
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c b/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c
index 76787a75a4dc..92c556ac22c7 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c
@@ -554,7 +554,7 @@ static struct attribute *dsicm_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group dsicm_attr_group = {
+static const struct attribute_group dsicm_attr_group = {
 	.attrs = dsicm_attrs,
 };
 
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c b/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c
index c90474afaebd..74d13969b9ca 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c
@@ -19,7 +19,7 @@
 
 #include "../dss/omapdss.h"
 
-static struct videomode lb035q02_vm = {
+static const struct videomode lb035q02_vm = {
 	.hactive = 320,
 	.vactive = 240,
 
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c b/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c
index 346aefdb015f..8e5bff4e5226 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c
@@ -503,7 +503,7 @@ static struct attribute *bldev_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group bldev_attr_group = {
+static const struct attribute_group bldev_attr_group = {
 	.attrs = bldev_attrs,
 };
 
@@ -720,6 +720,9 @@ static int acx565akm_probe(struct spi_device *spi)
 
 	dev_dbg(&spi->dev, "%s\n", __func__);
 
+	if (!spi->dev.of_node)
+		return -ENODEV;
+
 	spi->mode = SPI_MODE_3;
 
 	ddata = devm_kzalloc(&spi->dev, sizeof(*ddata), GFP_KERNEL);
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c
index cbf4c67c4933..0a38a0e8c925 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c
@@ -40,7 +40,7 @@ struct panel_drv_data {
 	struct spi_device *spi_dev;
 };
 
-static struct videomode td028ttec1_panel_vm = {
+static const struct videomode td028ttec1_panel_vm = {
 	.hactive	= 480,
 	.vactive	= 640,
 	.pixelclock	= 22153000,
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c
index 20c6d8fe215a..ac4a6d4d134c 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c
@@ -282,7 +282,7 @@ static struct attribute *tpo_td043_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group tpo_td043_attr_group = {
+static const struct attribute_group tpo_td043_attr_group = {
 	.attrs = tpo_td043_attrs,
 };
 
diff --git a/drivers/gpu/drm/omapdrm/dss/Makefile b/drivers/gpu/drm/omapdrm/dss/Makefile
index 688195e448c5..142ce5a02542 100644
--- a/drivers/gpu/drm/omapdrm/dss/Makefile
+++ b/drivers/gpu/drm/omapdrm/dss/Makefile
@@ -5,7 +5,7 @@ omapdss-base-y := base.o display.o dss-of.o output.o
 
 obj-$(CONFIG_OMAP2_DSS) += omapdss.o
 # Core DSS files
-omapdss-y := core.o dss.o dss_features.o dispc.o dispc_coefs.o \
+omapdss-y := core.o dss.o dispc.o dispc_coefs.o \
 	pll.o video-pll.o
 omapdss-$(CONFIG_OMAP2_DSS_DPI) += dpi.o
 omapdss-$(CONFIG_OMAP2_DSS_VENC) += venc.o
diff --git a/drivers/gpu/drm/omapdrm/dss/core.c b/drivers/gpu/drm/omapdrm/dss/core.c
index bdce4bfdf6e0..197ddbc1512b 100644
--- a/drivers/gpu/drm/omapdrm/dss/core.c
+++ b/drivers/gpu/drm/omapdrm/dss/core.c
@@ -24,182 +24,10 @@
 
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/clk.h>
-#include <linux/err.h>
 #include <linux/platform_device.h>
-#include <linux/seq_file.h>
-#include <linux/debugfs.h>
-#include <linux/io.h>
-#include <linux/device.h>
-#include <linux/regulator/consumer.h>
-#include <linux/suspend.h>
-#include <linux/slab.h>
 
 #include "omapdss.h"
 #include "dss.h"
-#include "dss_features.h"
-
-static struct {
-	struct platform_device *pdev;
-} core;
-
-enum omapdss_version omapdss_get_version(void)
-{
-	struct omap_dss_board_info *pdata = core.pdev->dev.platform_data;
-	return pdata->version;
-}
-EXPORT_SYMBOL(omapdss_get_version);
-
-int dss_dsi_enable_pads(int dsi_id, unsigned lane_mask)
-{
-	struct omap_dss_board_info *board_data = core.pdev->dev.platform_data;
-
-	if (!board_data->dsi_enable_pads)
-		return -ENOENT;
-
-	return board_data->dsi_enable_pads(dsi_id, lane_mask);
-}
-
-void dss_dsi_disable_pads(int dsi_id, unsigned lane_mask)
-{
-	struct omap_dss_board_info *board_data = core.pdev->dev.platform_data;
-
-	if (!board_data->dsi_disable_pads)
-		return;
-
-	return board_data->dsi_disable_pads(dsi_id, lane_mask);
-}
-
-int dss_set_min_bus_tput(struct device *dev, unsigned long tput)
-{
-	struct omap_dss_board_info *pdata = core.pdev->dev.platform_data;
-
-	if (pdata->set_min_bus_tput)
-		return pdata->set_min_bus_tput(dev, tput);
-	else
-		return 0;
-}
-
-#if defined(CONFIG_OMAP2_DSS_DEBUGFS)
-static int dss_debug_show(struct seq_file *s, void *unused)
-{
-	void (*func)(struct seq_file *) = s->private;
-	func(s);
-	return 0;
-}
-
-static int dss_debug_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, dss_debug_show, inode->i_private);
-}
-
-static const struct file_operations dss_debug_fops = {
-	.open           = dss_debug_open,
-	.read           = seq_read,
-	.llseek         = seq_lseek,
-	.release        = single_release,
-};
-
-static struct dentry *dss_debugfs_dir;
-
-static int dss_initialize_debugfs(void)
-{
-	dss_debugfs_dir = debugfs_create_dir("omapdss", NULL);
-	if (IS_ERR(dss_debugfs_dir)) {
-		int err = PTR_ERR(dss_debugfs_dir);
-		dss_debugfs_dir = NULL;
-		return err;
-	}
-
-	debugfs_create_file("clk", S_IRUGO, dss_debugfs_dir,
-			&dss_debug_dump_clocks, &dss_debug_fops);
-
-	return 0;
-}
-
-static void dss_uninitialize_debugfs(void)
-{
-	if (dss_debugfs_dir)
-		debugfs_remove_recursive(dss_debugfs_dir);
-}
-
-int dss_debugfs_create_file(const char *name, void (*write)(struct seq_file *))
-{
-	struct dentry *d;
-
-	d = debugfs_create_file(name, S_IRUGO, dss_debugfs_dir,
-			write, &dss_debug_fops);
-
-	return PTR_ERR_OR_ZERO(d);
-}
-#else /* CONFIG_OMAP2_DSS_DEBUGFS */
-static inline int dss_initialize_debugfs(void)
-{
-	return 0;
-}
-static inline void dss_uninitialize_debugfs(void)
-{
-}
-int dss_debugfs_create_file(const char *name, void (*write)(struct seq_file *))
-{
-	return 0;
-}
-#endif /* CONFIG_OMAP2_DSS_DEBUGFS */
-
-/* PLATFORM DEVICE */
-
-static void dss_disable_all_devices(void)
-{
-	struct omap_dss_device *dssdev = NULL;
-
-	for_each_dss_dev(dssdev) {
-		if (!dssdev->driver)
-			continue;
-
-		if (dssdev->state == OMAP_DSS_DISPLAY_ACTIVE)
-			dssdev->driver->disable(dssdev);
-	}
-}
-
-static int __init omap_dss_probe(struct platform_device *pdev)
-{
-	int r;
-
-	core.pdev = pdev;
-
-	dss_features_init(omapdss_get_version());
-
-	r = dss_initialize_debugfs();
-	if (r)
-		goto err_debugfs;
-
-	return 0;
-
-err_debugfs:
-
-	return r;
-}
-
-static int omap_dss_remove(struct platform_device *pdev)
-{
-	dss_uninitialize_debugfs();
-
-	return 0;
-}
-
-static void omap_dss_shutdown(struct platform_device *pdev)
-{
-	DSSDBG("shutdown\n");
-	dss_disable_all_devices();
-}
-
-static struct platform_driver omap_dss_driver = {
-	.remove         = omap_dss_remove,
-	.shutdown	= omap_dss_shutdown,
-	.driver         = {
-		.name   = "omapdss",
-	},
-};
 
 /* INIT */
 static int (*dss_output_drv_reg_funcs[])(void) __initdata = {
@@ -236,21 +64,25 @@ static void (*dss_output_drv_unreg_funcs[])(void) = {
 	dss_uninit_platform_driver,
 };
 
+static struct platform_device *omap_drm_device;
+
 static int __init omap_dss_init(void)
 {
 	int r;
 	int i;
 
-	r = platform_driver_probe(&omap_dss_driver, omap_dss_probe);
-	if (r)
-		return r;
-
 	for (i = 0; i < ARRAY_SIZE(dss_output_drv_reg_funcs); ++i) {
 		r = dss_output_drv_reg_funcs[i]();
 		if (r)
 			goto err_reg;
 	}
 
+	omap_drm_device = platform_device_register_simple("omapdrm", 0, NULL, 0);
+	if (IS_ERR(omap_drm_device)) {
+		r = PTR_ERR(omap_drm_device);
+		goto err_reg;
+	}
+
 	return 0;
 
 err_reg:
@@ -259,8 +91,6 @@ err_reg:
 			++i)
 		dss_output_drv_unreg_funcs[i]();
 
-	platform_driver_unregister(&omap_dss_driver);
-
 	return r;
 }
 
@@ -268,10 +98,10 @@ static void __exit omap_dss_exit(void)
 {
 	int i;
 
+	platform_device_unregister(omap_drm_device);
+
 	for (i = 0; i < ARRAY_SIZE(dss_output_drv_unreg_funcs); ++i)
 		dss_output_drv_unreg_funcs[i]();
-
-	platform_driver_unregister(&omap_dss_driver);
 }
 
 module_init(omap_dss_init);
diff --git a/drivers/gpu/drm/omapdrm/dss/dispc.c b/drivers/gpu/drm/omapdrm/dss/dispc.c
index fd7504b37e3b..0f4fdb221498 100644
--- a/drivers/gpu/drm/omapdrm/dss/dispc.c
+++ b/drivers/gpu/drm/omapdrm/dss/dispc.c
@@ -39,13 +39,14 @@
 #include <linux/mfd/syscon.h>
 #include <linux/regmap.h>
 #include <linux/of.h>
+#include <linux/of_device.h>
 #include <linux/component.h>
+#include <linux/sys_soc.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_blend.h>
 
 #include "omapdss.h"
 #include "dss.h"
-#include "dss_features.h"
 #include "dispc.h"
 
 /* DISPC */
@@ -63,6 +64,33 @@ enum omap_burst_size {
 #define REG_FLD_MOD(idx, val, start, end)				\
 	dispc_write_reg(idx, FLD_MOD(dispc_read_reg(idx), val, start, end))
 
+/* DISPC has feature id */
+enum dispc_feature_id {
+	FEAT_LCDENABLEPOL,
+	FEAT_LCDENABLESIGNAL,
+	FEAT_PCKFREEENABLE,
+	FEAT_FUNCGATED,
+	FEAT_MGR_LCD2,
+	FEAT_MGR_LCD3,
+	FEAT_LINEBUFFERSPLIT,
+	FEAT_ROWREPEATENABLE,
+	FEAT_RESIZECONF,
+	/* Independent core clk divider */
+	FEAT_CORE_CLK_DIV,
+	FEAT_HANDLE_UV_SEPARATE,
+	FEAT_ATTR2,
+	FEAT_CPR,
+	FEAT_PRELOAD,
+	FEAT_FIR_COEF_V,
+	FEAT_ALPHA_FIXED_ZORDER,
+	FEAT_ALPHA_FREE_ZORDER,
+	FEAT_FIFO_MERGE,
+	/* An unknown HW bug causing the normal FIFO thresholds not to work */
+	FEAT_OMAP3_DSI_FIFO_BUG,
+	FEAT_BURST_2D,
+	FEAT_MFLAG,
+};
+
 struct dispc_features {
 	u8 sw_start;
 	u8 fp_start;
@@ -76,6 +104,9 @@ struct dispc_features {
 	u16 mgr_height_max;
 	unsigned long max_lcd_pclk;
 	unsigned long max_tv_pclk;
+	unsigned int max_downscale;
+	unsigned int max_line_width;
+	unsigned int min_pcd;
 	int (*calc_scaling) (unsigned long pclk, unsigned long lclk,
 		const struct videomode *vm,
 		u16 width, u16 height, u16 out_width, u16 out_height,
@@ -86,6 +117,16 @@ struct dispc_features {
 		u16 width, u16 height, u16 out_width, u16 out_height,
 		bool mem_to_mem);
 	u8 num_fifos;
+	const enum dispc_feature_id *features;
+	unsigned int num_features;
+	const struct dss_reg_field *reg_fields;
+	const unsigned int num_reg_fields;
+	const enum omap_overlay_caps *overlay_caps;
+	const u32 **supported_color_modes;
+	unsigned int num_mgrs;
+	unsigned int num_ovls;
+	unsigned int buffer_size_unit;
+	unsigned int burst_size_unit;
 
 	/* swap GFX & WB fifos */
 	bool gfx_fifo_workaround:1;
@@ -180,6 +221,17 @@ enum mgr_reg_fields {
 	DISPC_MGR_FLD_NUM,
 };
 
+/* DISPC register field id */
+enum dispc_feat_reg_field {
+	FEAT_REG_FIRHINC,
+	FEAT_REG_FIRVINC,
+	FEAT_REG_FIFOHIGHTHRESHOLD,
+	FEAT_REG_FIFOLOWTHRESHOLD,
+	FEAT_REG_FIFOSIZE,
+	FEAT_REG_HORIZONTALACCU,
+	FEAT_REG_VERTICALACCU,
+};
+
 struct dispc_reg_field {
 	u16 reg;
 	u8 high;
@@ -343,6 +395,38 @@ static void mgr_fld_write(enum omap_channel channel,
 		spin_unlock_irqrestore(&dispc.control_lock, flags);
 }
 
+static int dispc_get_num_ovls(void)
+{
+	return dispc.feat->num_ovls;
+}
+
+static int dispc_get_num_mgrs(void)
+{
+	return dispc.feat->num_mgrs;
+}
+
+static void dispc_get_reg_field(enum dispc_feat_reg_field id,
+				u8 *start, u8 *end)
+{
+	if (id >= dispc.feat->num_reg_fields)
+		BUG();
+
+	*start = dispc.feat->reg_fields[id].start;
+	*end = dispc.feat->reg_fields[id].end;
+}
+
+static bool dispc_has_feature(enum dispc_feature_id id)
+{
+	unsigned int i;
+
+	for (i = 0; i < dispc.feat->num_features; i++) {
+		if (dispc.feat->features[i] == id)
+			return true;
+	}
+
+	return false;
+}
+
 #define SR(reg) \
 	dispc.ctx[DISPC_##reg / sizeof(u32)] = dispc_read_reg(DISPC_##reg)
 #define RR(reg) \
@@ -358,19 +442,19 @@ static void dispc_save_context(void)
 	SR(CONTROL);
 	SR(CONFIG);
 	SR(LINE_NUMBER);
-	if (dss_has_feature(FEAT_ALPHA_FIXED_ZORDER) ||
-			dss_has_feature(FEAT_ALPHA_FREE_ZORDER))
+	if (dispc_has_feature(FEAT_ALPHA_FIXED_ZORDER) ||
+			dispc_has_feature(FEAT_ALPHA_FREE_ZORDER))
 		SR(GLOBAL_ALPHA);
-	if (dss_has_feature(FEAT_MGR_LCD2)) {
+	if (dispc_has_feature(FEAT_MGR_LCD2)) {
 		SR(CONTROL2);
 		SR(CONFIG2);
 	}
-	if (dss_has_feature(FEAT_MGR_LCD3)) {
+	if (dispc_has_feature(FEAT_MGR_LCD3)) {
 		SR(CONTROL3);
 		SR(CONFIG3);
 	}
 
-	for (i = 0; i < dss_feat_get_num_mgrs(); i++) {
+	for (i = 0; i < dispc_get_num_mgrs(); i++) {
 		SR(DEFAULT_COLOR(i));
 		SR(TRANS_COLOR(i));
 		SR(SIZE_MGR(i));
@@ -385,14 +469,14 @@ static void dispc_save_context(void)
 		SR(DATA_CYCLE2(i));
 		SR(DATA_CYCLE3(i));
 
-		if (dss_has_feature(FEAT_CPR)) {
+		if (dispc_has_feature(FEAT_CPR)) {
 			SR(CPR_COEF_R(i));
 			SR(CPR_COEF_G(i));
 			SR(CPR_COEF_B(i));
 		}
 	}
 
-	for (i = 0; i < dss_feat_get_num_ovls(); i++) {
+	for (i = 0; i < dispc_get_num_ovls(); i++) {
 		SR(OVL_BA0(i));
 		SR(OVL_BA1(i));
 		SR(OVL_POSITION(i));
@@ -401,7 +485,7 @@ static void dispc_save_context(void)
 		SR(OVL_FIFO_THRESHOLD(i));
 		SR(OVL_ROW_INC(i));
 		SR(OVL_PIXEL_INC(i));
-		if (dss_has_feature(FEAT_PRELOAD))
+		if (dispc_has_feature(FEAT_PRELOAD))
 			SR(OVL_PRELOAD(i));
 		if (i == OMAP_DSS_GFX) {
 			SR(OVL_WINDOW_SKIP(i));
@@ -422,12 +506,12 @@ static void dispc_save_context(void)
 		for (j = 0; j < 5; j++)
 			SR(OVL_CONV_COEF(i, j));
 
-		if (dss_has_feature(FEAT_FIR_COEF_V)) {
+		if (dispc_has_feature(FEAT_FIR_COEF_V)) {
 			for (j = 0; j < 8; j++)
 				SR(OVL_FIR_COEF_V(i, j));
 		}
 
-		if (dss_has_feature(FEAT_HANDLE_UV_SEPARATE)) {
+		if (dispc_has_feature(FEAT_HANDLE_UV_SEPARATE)) {
 			SR(OVL_BA0_UV(i));
 			SR(OVL_BA1_UV(i));
 			SR(OVL_FIR2(i));
@@ -443,11 +527,11 @@ static void dispc_save_context(void)
 			for (j = 0; j < 8; j++)
 				SR(OVL_FIR_COEF_V2(i, j));
 		}
-		if (dss_has_feature(FEAT_ATTR2))
+		if (dispc_has_feature(FEAT_ATTR2))
 			SR(OVL_ATTRIBUTES2(i));
 	}
 
-	if (dss_has_feature(FEAT_CORE_CLK_DIV))
+	if (dispc_has_feature(FEAT_CORE_CLK_DIV))
 		SR(DIVISOR);
 
 	dispc.ctx_valid = true;
@@ -468,15 +552,15 @@ static void dispc_restore_context(void)
 	/*RR(CONTROL);*/
 	RR(CONFIG);
 	RR(LINE_NUMBER);
-	if (dss_has_feature(FEAT_ALPHA_FIXED_ZORDER) ||
-			dss_has_feature(FEAT_ALPHA_FREE_ZORDER))
+	if (dispc_has_feature(FEAT_ALPHA_FIXED_ZORDER) ||
+			dispc_has_feature(FEAT_ALPHA_FREE_ZORDER))
 		RR(GLOBAL_ALPHA);
-	if (dss_has_feature(FEAT_MGR_LCD2))
+	if (dispc_has_feature(FEAT_MGR_LCD2))
 		RR(CONFIG2);
-	if (dss_has_feature(FEAT_MGR_LCD3))
+	if (dispc_has_feature(FEAT_MGR_LCD3))
 		RR(CONFIG3);
 
-	for (i = 0; i < dss_feat_get_num_mgrs(); i++) {
+	for (i = 0; i < dispc_get_num_mgrs(); i++) {
 		RR(DEFAULT_COLOR(i));
 		RR(TRANS_COLOR(i));
 		RR(SIZE_MGR(i));
@@ -491,14 +575,14 @@ static void dispc_restore_context(void)
 		RR(DATA_CYCLE2(i));
 		RR(DATA_CYCLE3(i));
 
-		if (dss_has_feature(FEAT_CPR)) {
+		if (dispc_has_feature(FEAT_CPR)) {
 			RR(CPR_COEF_R(i));
 			RR(CPR_COEF_G(i));
 			RR(CPR_COEF_B(i));
 		}
 	}
 
-	for (i = 0; i < dss_feat_get_num_ovls(); i++) {
+	for (i = 0; i < dispc_get_num_ovls(); i++) {
 		RR(OVL_BA0(i));
 		RR(OVL_BA1(i));
 		RR(OVL_POSITION(i));
@@ -507,7 +591,7 @@ static void dispc_restore_context(void)
 		RR(OVL_FIFO_THRESHOLD(i));
 		RR(OVL_ROW_INC(i));
 		RR(OVL_PIXEL_INC(i));
-		if (dss_has_feature(FEAT_PRELOAD))
+		if (dispc_has_feature(FEAT_PRELOAD))
 			RR(OVL_PRELOAD(i));
 		if (i == OMAP_DSS_GFX) {
 			RR(OVL_WINDOW_SKIP(i));
@@ -528,12 +612,12 @@ static void dispc_restore_context(void)
 		for (j = 0; j < 5; j++)
 			RR(OVL_CONV_COEF(i, j));
 
-		if (dss_has_feature(FEAT_FIR_COEF_V)) {
+		if (dispc_has_feature(FEAT_FIR_COEF_V)) {
 			for (j = 0; j < 8; j++)
 				RR(OVL_FIR_COEF_V(i, j));
 		}
 
-		if (dss_has_feature(FEAT_HANDLE_UV_SEPARATE)) {
+		if (dispc_has_feature(FEAT_HANDLE_UV_SEPARATE)) {
 			RR(OVL_BA0_UV(i));
 			RR(OVL_BA1_UV(i));
 			RR(OVL_FIR2(i));
@@ -549,18 +633,18 @@ static void dispc_restore_context(void)
 			for (j = 0; j < 8; j++)
 				RR(OVL_FIR_COEF_V2(i, j));
 		}
-		if (dss_has_feature(FEAT_ATTR2))
+		if (dispc_has_feature(FEAT_ATTR2))
 			RR(OVL_ATTRIBUTES2(i));
 	}
 
-	if (dss_has_feature(FEAT_CORE_CLK_DIV))
+	if (dispc_has_feature(FEAT_CORE_CLK_DIV))
 		RR(DIVISOR);
 
 	/* enable last, because LCD & DIGIT enable are here */
 	RR(CONTROL);
-	if (dss_has_feature(FEAT_MGR_LCD2))
+	if (dispc_has_feature(FEAT_MGR_LCD2))
 		RR(CONTROL2);
-	if (dss_has_feature(FEAT_MGR_LCD3))
+	if (dispc_has_feature(FEAT_MGR_LCD3))
 		RR(CONTROL3);
 	/* clear spurious SYNC_LOST_DIGIT interrupts */
 	dispc_clear_irqstatus(DISPC_IRQ_SYNC_LOST_DIGIT);
@@ -779,7 +863,7 @@ static void dispc_ovl_write_color_conv_coef(enum omap_plane_id plane,
 static void dispc_setup_color_conv_coef(void)
 {
 	int i;
-	int num_ovl = dss_feat_get_num_ovls();
+	int num_ovl = dispc_get_num_ovls();
 	const struct color_conv_coef ctbl_bt601_5_ovl = {
 		/* YUV -> RGB */
 		298, 409, 0, 298, -208, -100, 298, 0, 517, 0,
@@ -868,10 +952,10 @@ static void dispc_ovl_enable_zorder_planes(void)
 {
 	int i;
 
-	if (!dss_has_feature(FEAT_ALPHA_FREE_ZORDER))
+	if (!dispc_has_feature(FEAT_ALPHA_FREE_ZORDER))
 		return;
 
-	for (i = 0; i < dss_feat_get_num_ovls(); i++)
+	for (i = 0; i < dispc_get_num_ovls(); i++)
 		REG_FLD_MOD(DISPC_OVL_ATTRIBUTES(i), 1, 25, 25);
 }
 
@@ -994,7 +1078,7 @@ static bool format_is_yuv(u32 fourcc)
 static void dispc_ovl_configure_burst_type(enum omap_plane_id plane,
 		enum omap_dss_rotation_type rotation_type)
 {
-	if (dss_has_feature(FEAT_BURST_2D) == 0)
+	if (dispc_has_feature(FEAT_BURST_2D) == 0)
 		return;
 
 	if (rotation_type == OMAP_DSS_ROT_TILER)
@@ -1025,7 +1109,7 @@ static void dispc_ovl_set_channel_out(enum omap_plane_id plane,
 	}
 
 	val = dispc_read_reg(DISPC_OVL_ATTRIBUTES(plane));
-	if (dss_has_feature(FEAT_MGR_LCD2)) {
+	if (dispc_has_feature(FEAT_MGR_LCD2)) {
 		switch (channel) {
 		case OMAP_DSS_CHANNEL_LCD:
 			chan = 0;
@@ -1040,7 +1124,7 @@ static void dispc_ovl_set_channel_out(enum omap_plane_id plane,
 			chan2 = 1;
 			break;
 		case OMAP_DSS_CHANNEL_LCD3:
-			if (dss_has_feature(FEAT_MGR_LCD3)) {
+			if (dispc_has_feature(FEAT_MGR_LCD3)) {
 				chan = 0;
 				chan2 = 2;
 			} else {
@@ -1089,7 +1173,7 @@ static enum omap_channel dispc_ovl_get_channel_out(enum omap_plane_id plane)
 	if (FLD_GET(val, shift, shift) == 1)
 		return OMAP_DSS_CHANNEL_DIGIT;
 
-	if (!dss_has_feature(FEAT_MGR_LCD2))
+	if (!dispc_has_feature(FEAT_MGR_LCD2))
 		return OMAP_DSS_CHANNEL_LCD;
 
 	switch (FLD_GET(val, 31, 30)) {
@@ -1128,7 +1212,7 @@ static void dispc_configure_burst_sizes(void)
 	const int burst_size = BURST_SIZE_X8;
 
 	/* Configure burst size always to maximum size */
-	for (i = 0; i < dss_feat_get_num_ovls(); ++i)
+	for (i = 0; i < dispc_get_num_ovls(); ++i)
 		dispc_ovl_set_burst_size(i, burst_size);
 	if (dispc.feat->has_writeback)
 		dispc_ovl_set_burst_size(OMAP_DSS_WB, burst_size);
@@ -1136,19 +1220,28 @@ static void dispc_configure_burst_sizes(void)
 
 static u32 dispc_ovl_get_burst_size(enum omap_plane_id plane)
 {
-	unsigned unit = dss_feat_get_burst_size_unit();
 	/* burst multiplier is always x8 (see dispc_configure_burst_sizes()) */
-	return unit * 8;
+	return dispc.feat->burst_size_unit * 8;
 }
 
-static const u32 *dispc_ovl_get_color_modes(enum omap_plane_id plane)
+static bool dispc_ovl_color_mode_supported(enum omap_plane_id plane, u32 fourcc)
 {
-	return dss_feat_get_supported_color_modes(plane);
+	const u32 *modes;
+	unsigned int i;
+
+	modes = dispc.feat->supported_color_modes[plane];
+
+	for (i = 0; modes[i]; ++i) {
+		if (modes[i] == fourcc)
+			return true;
+	}
+
+	return false;
 }
 
-static int dispc_get_num_ovls(void)
+static const u32 *dispc_ovl_get_color_modes(enum omap_plane_id plane)
 {
-	return dss_feat_get_num_ovls();
+	return dispc.feat->supported_color_modes[plane];
 }
 
 static void dispc_mgr_enable_cpr(enum omap_channel channel, bool enable)
@@ -1223,9 +1316,9 @@ static void dispc_init_fifos(void)
 	u32 unit;
 	int i;
 
-	unit = dss_feat_get_buffer_size_unit();
+	unit = dispc.feat->buffer_size_unit;
 
-	dss_feat_get_reg_field(FEAT_REG_FIFOSIZE, &start, &end);
+	dispc_get_reg_field(FEAT_REG_FIFOSIZE, &start, &end);
 
 	for (fifo = 0; fifo < dispc.feat->num_fifos; ++fifo) {
 		size = REG_GET(DISPC_OVL_FIFO_SIZE_STATUS(fifo), start, end);
@@ -1265,7 +1358,7 @@ static void dispc_init_fifos(void)
 	/*
 	 * Setup default fifo thresholds.
 	 */
-	for (i = 0; i < dss_feat_get_num_ovls(); ++i) {
+	for (i = 0; i < dispc_get_num_ovls(); ++i) {
 		u32 low, high;
 		const bool use_fifomerge = false;
 		const bool manual_update = false;
@@ -1307,7 +1400,7 @@ void dispc_ovl_set_fifo_threshold(enum omap_plane_id plane, u32 low,
 	u8 hi_start, hi_end, lo_start, lo_end;
 	u32 unit;
 
-	unit = dss_feat_get_buffer_size_unit();
+	unit = dispc.feat->buffer_size_unit;
 
 	WARN_ON(low % unit != 0);
 	WARN_ON(high % unit != 0);
@@ -1315,8 +1408,8 @@ void dispc_ovl_set_fifo_threshold(enum omap_plane_id plane, u32 low,
 	low /= unit;
 	high /= unit;
 
-	dss_feat_get_reg_field(FEAT_REG_FIFOHIGHTHRESHOLD, &hi_start, &hi_end);
-	dss_feat_get_reg_field(FEAT_REG_FIFOLOWTHRESHOLD, &lo_start, &lo_end);
+	dispc_get_reg_field(FEAT_REG_FIFOHIGHTHRESHOLD, &hi_start, &hi_end);
+	dispc_get_reg_field(FEAT_REG_FIFOLOWTHRESHOLD, &lo_start, &lo_end);
 
 	DSSDBG("fifo(%d) threshold (bytes), old %u/%u, new %u/%u\n",
 			plane,
@@ -1335,14 +1428,14 @@ void dispc_ovl_set_fifo_threshold(enum omap_plane_id plane, u32 low,
 	 * large for the preload field, set the threshold to the maximum value
 	 * that can be held by the preload register
 	 */
-	if (dss_has_feature(FEAT_PRELOAD) && dispc.feat->set_max_preload &&
+	if (dispc_has_feature(FEAT_PRELOAD) && dispc.feat->set_max_preload &&
 			plane != OMAP_DSS_WB)
 		dispc_write_reg(DISPC_OVL_PRELOAD(plane), min(high, 0xfffu));
 }
 
 void dispc_enable_fifomerge(bool enable)
 {
-	if (!dss_has_feature(FEAT_FIFO_MERGE)) {
+	if (!dispc_has_feature(FEAT_FIFO_MERGE)) {
 		WARN_ON(enable);
 		return;
 	}
@@ -1360,7 +1453,7 @@ void dispc_ovl_compute_fifo_thresholds(enum omap_plane_id plane,
 	 * buffer_units, and the fifo thresholds must be buffer_unit aligned.
 	 */
 
-	unsigned buf_unit = dss_feat_get_buffer_size_unit();
+	unsigned buf_unit = dispc.feat->buffer_size_unit;
 	unsigned ovl_fifo_size, total_fifo_size, burst_size;
 	int i;
 
@@ -1369,7 +1462,7 @@ void dispc_ovl_compute_fifo_thresholds(enum omap_plane_id plane,
 
 	if (use_fifomerge) {
 		total_fifo_size = 0;
-		for (i = 0; i < dss_feat_get_num_ovls(); ++i)
+		for (i = 0; i < dispc_get_num_ovls(); ++i)
 			total_fifo_size += dispc_ovl_get_fifo_size(i);
 	} else {
 		total_fifo_size = ovl_fifo_size;
@@ -1381,7 +1474,7 @@ void dispc_ovl_compute_fifo_thresholds(enum omap_plane_id plane,
 	 * combined fifo size
 	 */
 
-	if (manual_update && dss_has_feature(FEAT_OMAP3_DSI_FIFO_BUG)) {
+	if (manual_update && dispc_has_feature(FEAT_OMAP3_DSI_FIFO_BUG)) {
 		*fifo_low = ovl_fifo_size - burst_size * 2;
 		*fifo_high = total_fifo_size - burst_size;
 	} else if (plane == OMAP_DSS_WB) {
@@ -1435,9 +1528,9 @@ static void dispc_init_mflag(void)
 		(1 << 0) |	/* MFLAG_CTRL = force always on */
 		(0 << 2));	/* MFLAG_START = disable */
 
-	for (i = 0; i < dss_feat_get_num_ovls(); ++i) {
+	for (i = 0; i < dispc_get_num_ovls(); ++i) {
 		u32 size = dispc_ovl_get_fifo_size(i);
-		u32 unit = dss_feat_get_buffer_size_unit();
+		u32 unit = dispc.feat->buffer_size_unit;
 		u32 low, high;
 
 		dispc_ovl_set_mflag(i, true);
@@ -1456,7 +1549,7 @@ static void dispc_init_mflag(void)
 
 	if (dispc.feat->has_writeback) {
 		u32 size = dispc_ovl_get_fifo_size(OMAP_DSS_WB);
-		u32 unit = dss_feat_get_buffer_size_unit();
+		u32 unit = dispc.feat->buffer_size_unit;
 		u32 low, high;
 
 		dispc_ovl_set_mflag(OMAP_DSS_WB, true);
@@ -1483,10 +1576,8 @@ static void dispc_ovl_set_fir(enum omap_plane_id plane,
 	if (color_comp == DISPC_COLOR_COMPONENT_RGB_Y) {
 		u8 hinc_start, hinc_end, vinc_start, vinc_end;
 
-		dss_feat_get_reg_field(FEAT_REG_FIRHINC,
-					&hinc_start, &hinc_end);
-		dss_feat_get_reg_field(FEAT_REG_FIRVINC,
-					&vinc_start, &vinc_end);
+		dispc_get_reg_field(FEAT_REG_FIRHINC, &hinc_start, &hinc_end);
+		dispc_get_reg_field(FEAT_REG_FIRVINC, &vinc_start, &vinc_end);
 		val = FLD_VAL(vinc, vinc_start, vinc_end) |
 				FLD_VAL(hinc, hinc_start, hinc_end);
 
@@ -1503,8 +1594,8 @@ static void dispc_ovl_set_vid_accu0(enum omap_plane_id plane, int haccu,
 	u32 val;
 	u8 hor_start, hor_end, vert_start, vert_end;
 
-	dss_feat_get_reg_field(FEAT_REG_HORIZONTALACCU, &hor_start, &hor_end);
-	dss_feat_get_reg_field(FEAT_REG_VERTICALACCU, &vert_start, &vert_end);
+	dispc_get_reg_field(FEAT_REG_HORIZONTALACCU, &hor_start, &hor_end);
+	dispc_get_reg_field(FEAT_REG_VERTICALACCU, &vert_start, &vert_end);
 
 	val = FLD_VAL(vaccu, vert_start, vert_end) |
 			FLD_VAL(haccu, hor_start, hor_end);
@@ -1518,8 +1609,8 @@ static void dispc_ovl_set_vid_accu1(enum omap_plane_id plane, int haccu,
 	u32 val;
 	u8 hor_start, hor_end, vert_start, vert_end;
 
-	dss_feat_get_reg_field(FEAT_REG_HORIZONTALACCU, &hor_start, &hor_end);
-	dss_feat_get_reg_field(FEAT_REG_VERTICALACCU, &vert_start, &vert_end);
+	dispc_get_reg_field(FEAT_REG_HORIZONTALACCU, &hor_start, &hor_end);
+	dispc_get_reg_field(FEAT_REG_VERTICALACCU, &vert_start, &vert_end);
 
 	val = FLD_VAL(vaccu, vert_start, vert_end) |
 			FLD_VAL(haccu, hor_start, hor_end);
@@ -1671,14 +1762,14 @@ static void dispc_ovl_set_scaling_common(enum omap_plane_id plane,
 	l |= five_taps ? (1 << 21) : 0;
 
 	/* VRESIZECONF and HRESIZECONF */
-	if (dss_has_feature(FEAT_RESIZECONF)) {
+	if (dispc_has_feature(FEAT_RESIZECONF)) {
 		l &= ~(0x3 << 7);
 		l |= (orig_width <= out_width) ? 0 : (1 << 7);
 		l |= (orig_height <= out_height) ? 0 : (1 << 8);
 	}
 
 	/* LINEBUFFERSPLIT */
-	if (dss_has_feature(FEAT_LINEBUFFERSPLIT)) {
+	if (dispc_has_feature(FEAT_LINEBUFFERSPLIT)) {
 		l &= ~(0x1 << 22);
 		l |= five_taps ? (1 << 22) : 0;
 	}
@@ -1713,7 +1804,7 @@ static void dispc_ovl_set_scaling_uv(enum omap_plane_id plane,
 	int scale_y = out_height != orig_height;
 	bool chroma_upscale = plane != OMAP_DSS_WB;
 
-	if (!dss_has_feature(FEAT_HANDLE_UV_SEPARATE))
+	if (!dispc_has_feature(FEAT_HANDLE_UV_SEPARATE))
 		return;
 
 	if (!format_is_yuv(fourcc)) {
@@ -1860,11 +1951,11 @@ static void dispc_ovl_set_rotation_attrs(enum omap_plane_id plane, u8 rotation,
 		vidrot = 1;
 
 	REG_FLD_MOD(DISPC_OVL_ATTRIBUTES(plane), vidrot, 13, 12);
-	if (dss_has_feature(FEAT_ROWREPEATENABLE))
+	if (dispc_has_feature(FEAT_ROWREPEATENABLE))
 		REG_FLD_MOD(DISPC_OVL_ATTRIBUTES(plane),
 			row_repeat ? 1 : 0, 18, 18);
 
-	if (dss_feat_color_mode_supported(plane, DRM_FORMAT_NV12)) {
+	if (dispc_ovl_color_mode_supported(plane, DRM_FORMAT_NV12)) {
 		bool doublestride =
 			fourcc == DRM_FORMAT_NV12 &&
 			rotation_type == OMAP_DSS_ROT_TILER &&
@@ -2118,8 +2209,7 @@ static int dispc_ovl_calc_scaling_24xx(unsigned long pclk, unsigned long lclk,
 	int error;
 	u16 in_width, in_height;
 	int min_factor = min(*decim_x, *decim_y);
-	const int maxsinglelinewidth =
-			dss_feat_get_param_max(FEAT_PARAM_LINEWIDTH);
+	const int maxsinglelinewidth = dispc.feat->max_line_width;
 
 	*five_taps = false;
 
@@ -2163,8 +2253,7 @@ static int dispc_ovl_calc_scaling_34xx(unsigned long pclk, unsigned long lclk,
 {
 	int error;
 	u16 in_width, in_height;
-	const int maxsinglelinewidth =
-			dss_feat_get_param_max(FEAT_PARAM_LINEWIDTH);
+	const int maxsinglelinewidth = dispc.feat->max_line_width;
 
 	do {
 		in_height = height / *decim_y;
@@ -2249,9 +2338,8 @@ static int dispc_ovl_calc_scaling_44xx(unsigned long pclk, unsigned long lclk,
 	u16 in_width, in_width_max;
 	int decim_x_min = *decim_x;
 	u16 in_height = height / *decim_y;
-	const int maxsinglelinewidth =
-				dss_feat_get_param_max(FEAT_PARAM_LINEWIDTH);
-	const int maxdownscale = dss_feat_get_param_max(FEAT_PARAM_DOWNSCALE);
+	const int maxsinglelinewidth = dispc.feat->max_line_width;
+	const int maxdownscale = dispc.feat->max_downscale;
 
 	if (mem_to_mem) {
 		in_width_max = out_width * maxdownscale;
@@ -2311,7 +2399,7 @@ static int dispc_ovl_calc_scaling(unsigned long pclk, unsigned long lclk,
 		int *x_predecim, int *y_predecim, u16 pos_x,
 		enum omap_dss_rotation_type rotation_type, bool mem_to_mem)
 {
-	const int maxdownscale = dss_feat_get_param_max(FEAT_PARAM_DOWNSCALE);
+	const int maxdownscale = dispc.feat->max_downscale;
 	const int max_decim_limit = 16;
 	unsigned long core_clk = 0;
 	int decim_x, decim_y, ret;
@@ -2332,7 +2420,7 @@ static int dispc_ovl_calc_scaling(unsigned long pclk, unsigned long lclk,
 	} else {
 		*x_predecim = max_decim_limit;
 		*y_predecim = (rotation_type == OMAP_DSS_ROT_TILER &&
-				dss_has_feature(FEAT_BURST_2D)) ?
+				dispc_has_feature(FEAT_BURST_2D)) ?
 				2 : max_decim_limit;
 	}
 
@@ -2428,7 +2516,7 @@ static int dispc_ovl_setup_common(enum omap_plane_id plane,
 			out_height);
 	}
 
-	if (!dss_feat_color_mode_supported(plane, fourcc))
+	if (!dispc_ovl_color_mode_supported(plane, fourcc))
 		return -EINVAL;
 
 	r = dispc_ovl_calc_scaling(pclk, lclk, caps, vm, in_width,
@@ -2549,7 +2637,7 @@ static int dispc_ovl_setup(enum omap_plane_id plane,
 		enum omap_channel channel)
 {
 	int r;
-	enum omap_overlay_caps caps = dss_feat_get_overlay_caps(plane);
+	enum omap_overlay_caps caps = dispc.feat->overlay_caps[plane];
 	const bool replication = true;
 
 	DSSDBG("dispc_ovl_setup %d, pa %pad, pa_uv %pad, sw %d, %d,%d, %dx%d ->"
@@ -2647,12 +2735,12 @@ static int dispc_ovl_enable(enum omap_plane_id plane, bool enable)
 
 static enum omap_dss_output_id dispc_mgr_get_supported_outputs(enum omap_channel channel)
 {
-	return dss_feat_get_supported_outputs(channel);
+	return dss_get_supported_outputs(channel);
 }
 
 static void dispc_lcd_enable_signal_polarity(bool act_high)
 {
-	if (!dss_has_feature(FEAT_LCDENABLEPOL))
+	if (!dispc_has_feature(FEAT_LCDENABLEPOL))
 		return;
 
 	REG_FLD_MOD(DISPC_CONTROL, act_high ? 1 : 0, 29, 29);
@@ -2660,7 +2748,7 @@ static void dispc_lcd_enable_signal_polarity(bool act_high)
 
 void dispc_lcd_enable_signal(bool enable)
 {
-	if (!dss_has_feature(FEAT_LCDENABLESIGNAL))
+	if (!dispc_has_feature(FEAT_LCDENABLESIGNAL))
 		return;
 
 	REG_FLD_MOD(DISPC_CONTROL, enable ? 1 : 0, 28, 28);
@@ -2668,17 +2756,12 @@ void dispc_lcd_enable_signal(bool enable)
 
 void dispc_pck_free_enable(bool enable)
 {
-	if (!dss_has_feature(FEAT_PCKFREEENABLE))
+	if (!dispc_has_feature(FEAT_PCKFREEENABLE))
 		return;
 
 	REG_FLD_MOD(DISPC_CONTROL, enable ? 1 : 0, 27, 27);
 }
 
-static int dispc_get_num_mgrs(void)
-{
-	return dss_feat_get_num_mgrs();
-}
-
 static void dispc_mgr_enable_fifohandcheck(enum omap_channel channel, bool enable)
 {
 	mgr_fld_write(channel, DISPC_MGR_FLD_FIFOHANDCHECK, enable);
@@ -2718,7 +2801,7 @@ static void dispc_mgr_enable_trans_key(enum omap_channel ch, bool enable)
 static void dispc_mgr_enable_alpha_fixed_zorder(enum omap_channel ch,
 		bool enable)
 {
-	if (!dss_has_feature(FEAT_ALPHA_FIXED_ZORDER))
+	if (!dispc_has_feature(FEAT_ALPHA_FIXED_ZORDER))
 		return;
 
 	if (ch == OMAP_DSS_CHANNEL_LCD)
@@ -2735,7 +2818,7 @@ static void dispc_mgr_setup(enum omap_channel channel,
 	dispc_mgr_enable_trans_key(channel, info->trans_enabled);
 	dispc_mgr_enable_alpha_fixed_zorder(channel,
 			info->partial_alpha_enabled);
-	if (dss_has_feature(FEAT_CPR)) {
+	if (dispc_has_feature(FEAT_CPR)) {
 		dispc_mgr_enable_cpr(channel, info->cpr_enable);
 		dispc_mgr_set_cpr_coef(channel, &info->cpr_coefs);
 	}
@@ -3013,7 +3096,7 @@ static void dispc_mgr_set_lcd_divisor(enum omap_channel channel, u16 lck_div,
 	dispc_write_reg(DISPC_DIVISORo(channel),
 			FLD_VAL(lck_div, 23, 16) | FLD_VAL(pck_div, 7, 0));
 
-	if (!dss_has_feature(FEAT_CORE_CLK_DIV) &&
+	if (!dispc_has_feature(FEAT_CORE_CLK_DIV) &&
 			channel == OMAP_DSS_CHANNEL_LCD)
 		dispc.core_clk_rate = dispc_fclk_rate() / lck_div;
 }
@@ -3168,7 +3251,7 @@ void dispc_dump_clocks(struct seq_file *s)
 
 	seq_printf(s, "fck\t\t%-16lu\n", dispc_fclk_rate());
 
-	if (dss_has_feature(FEAT_CORE_CLK_DIV)) {
+	if (dispc_has_feature(FEAT_CORE_CLK_DIV)) {
 		seq_printf(s, "- DISPC-CORE-CLK -\n");
 		l = dispc_read_reg(DISPC_DIVISOR);
 		lcd = FLD_GET(l, 23, 16);
@@ -3179,9 +3262,9 @@ void dispc_dump_clocks(struct seq_file *s)
 
 	dispc_dump_clocks_channel(s, OMAP_DSS_CHANNEL_LCD);
 
-	if (dss_has_feature(FEAT_MGR_LCD2))
+	if (dispc_has_feature(FEAT_MGR_LCD2))
 		dispc_dump_clocks_channel(s, OMAP_DSS_CHANNEL_LCD2);
-	if (dss_has_feature(FEAT_MGR_LCD3))
+	if (dispc_has_feature(FEAT_MGR_LCD3))
 		dispc_dump_clocks_channel(s, OMAP_DSS_CHANNEL_LCD3);
 
 	dispc_runtime_put();
@@ -3221,18 +3304,18 @@ static void dispc_dump_regs(struct seq_file *s)
 	DUMPREG(DISPC_CAPABLE);
 	DUMPREG(DISPC_LINE_STATUS);
 	DUMPREG(DISPC_LINE_NUMBER);
-	if (dss_has_feature(FEAT_ALPHA_FIXED_ZORDER) ||
-			dss_has_feature(FEAT_ALPHA_FREE_ZORDER))
+	if (dispc_has_feature(FEAT_ALPHA_FIXED_ZORDER) ||
+			dispc_has_feature(FEAT_ALPHA_FREE_ZORDER))
 		DUMPREG(DISPC_GLOBAL_ALPHA);
-	if (dss_has_feature(FEAT_MGR_LCD2)) {
+	if (dispc_has_feature(FEAT_MGR_LCD2)) {
 		DUMPREG(DISPC_CONTROL2);
 		DUMPREG(DISPC_CONFIG2);
 	}
-	if (dss_has_feature(FEAT_MGR_LCD3)) {
+	if (dispc_has_feature(FEAT_MGR_LCD3)) {
 		DUMPREG(DISPC_CONTROL3);
 		DUMPREG(DISPC_CONFIG3);
 	}
-	if (dss_has_feature(FEAT_MFLAG))
+	if (dispc_has_feature(FEAT_MFLAG))
 		DUMPREG(DISPC_GLOBAL_MFLAG_ATTRIBUTE);
 
 #undef DUMPREG
@@ -3245,7 +3328,7 @@ static void dispc_dump_regs(struct seq_file *s)
 	p_names = mgr_names;
 
 	/* DISPC channel specific registers */
-	for (i = 0; i < dss_feat_get_num_mgrs(); i++) {
+	for (i = 0; i < dispc_get_num_mgrs(); i++) {
 		DUMPREG(i, DISPC_DEFAULT_COLOR);
 		DUMPREG(i, DISPC_TRANS_COLOR);
 		DUMPREG(i, DISPC_SIZE_MGR);
@@ -3262,7 +3345,7 @@ static void dispc_dump_regs(struct seq_file *s)
 		DUMPREG(i, DISPC_DATA_CYCLE2);
 		DUMPREG(i, DISPC_DATA_CYCLE3);
 
-		if (dss_has_feature(FEAT_CPR)) {
+		if (dispc_has_feature(FEAT_CPR)) {
 			DUMPREG(i, DISPC_CPR_COEF_R);
 			DUMPREG(i, DISPC_CPR_COEF_G);
 			DUMPREG(i, DISPC_CPR_COEF_B);
@@ -3271,7 +3354,7 @@ static void dispc_dump_regs(struct seq_file *s)
 
 	p_names = ovl_names;
 
-	for (i = 0; i < dss_feat_get_num_ovls(); i++) {
+	for (i = 0; i < dispc_get_num_ovls(); i++) {
 		DUMPREG(i, DISPC_OVL_BA0);
 		DUMPREG(i, DISPC_OVL_BA1);
 		DUMPREG(i, DISPC_OVL_POSITION);
@@ -3282,9 +3365,9 @@ static void dispc_dump_regs(struct seq_file *s)
 		DUMPREG(i, DISPC_OVL_ROW_INC);
 		DUMPREG(i, DISPC_OVL_PIXEL_INC);
 
-		if (dss_has_feature(FEAT_PRELOAD))
+		if (dispc_has_feature(FEAT_PRELOAD))
 			DUMPREG(i, DISPC_OVL_PRELOAD);
-		if (dss_has_feature(FEAT_MFLAG))
+		if (dispc_has_feature(FEAT_MFLAG))
 			DUMPREG(i, DISPC_OVL_MFLAG_THRESHOLD);
 
 		if (i == OMAP_DSS_GFX) {
@@ -3297,14 +3380,14 @@ static void dispc_dump_regs(struct seq_file *s)
 		DUMPREG(i, DISPC_OVL_PICTURE_SIZE);
 		DUMPREG(i, DISPC_OVL_ACCU0);
 		DUMPREG(i, DISPC_OVL_ACCU1);
-		if (dss_has_feature(FEAT_HANDLE_UV_SEPARATE)) {
+		if (dispc_has_feature(FEAT_HANDLE_UV_SEPARATE)) {
 			DUMPREG(i, DISPC_OVL_BA0_UV);
 			DUMPREG(i, DISPC_OVL_BA1_UV);
 			DUMPREG(i, DISPC_OVL_FIR2);
 			DUMPREG(i, DISPC_OVL_ACCU2_0);
 			DUMPREG(i, DISPC_OVL_ACCU2_1);
 		}
-		if (dss_has_feature(FEAT_ATTR2))
+		if (dispc_has_feature(FEAT_ATTR2))
 			DUMPREG(i, DISPC_OVL_ATTRIBUTES2);
 	}
 
@@ -3319,21 +3402,21 @@ static void dispc_dump_regs(struct seq_file *s)
 		DUMPREG(i, DISPC_OVL_ROW_INC);
 		DUMPREG(i, DISPC_OVL_PIXEL_INC);
 
-		if (dss_has_feature(FEAT_MFLAG))
+		if (dispc_has_feature(FEAT_MFLAG))
 			DUMPREG(i, DISPC_OVL_MFLAG_THRESHOLD);
 
 		DUMPREG(i, DISPC_OVL_FIR);
 		DUMPREG(i, DISPC_OVL_PICTURE_SIZE);
 		DUMPREG(i, DISPC_OVL_ACCU0);
 		DUMPREG(i, DISPC_OVL_ACCU1);
-		if (dss_has_feature(FEAT_HANDLE_UV_SEPARATE)) {
+		if (dispc_has_feature(FEAT_HANDLE_UV_SEPARATE)) {
 			DUMPREG(i, DISPC_OVL_BA0_UV);
 			DUMPREG(i, DISPC_OVL_BA1_UV);
 			DUMPREG(i, DISPC_OVL_FIR2);
 			DUMPREG(i, DISPC_OVL_ACCU2_0);
 			DUMPREG(i, DISPC_OVL_ACCU2_1);
 		}
-		if (dss_has_feature(FEAT_ATTR2))
+		if (dispc_has_feature(FEAT_ATTR2))
 			DUMPREG(i, DISPC_OVL_ATTRIBUTES2);
 	}
 
@@ -3349,7 +3432,7 @@ static void dispc_dump_regs(struct seq_file *s)
 	/* Video pipeline coefficient registers */
 
 	/* start from OMAP_DSS_VIDEO1 */
-	for (i = 1; i < dss_feat_get_num_ovls(); i++) {
+	for (i = 1; i < dispc_get_num_ovls(); i++) {
 		for (j = 0; j < 8; j++)
 			DUMPREG(i, DISPC_OVL_FIR_COEF_H, j);
 
@@ -3359,12 +3442,12 @@ static void dispc_dump_regs(struct seq_file *s)
 		for (j = 0; j < 5; j++)
 			DUMPREG(i, DISPC_OVL_CONV_COEF, j);
 
-		if (dss_has_feature(FEAT_FIR_COEF_V)) {
+		if (dispc_has_feature(FEAT_FIR_COEF_V)) {
 			for (j = 0; j < 8; j++)
 				DUMPREG(i, DISPC_OVL_FIR_COEF_V, j);
 		}
 
-		if (dss_has_feature(FEAT_HANDLE_UV_SEPARATE)) {
+		if (dispc_has_feature(FEAT_HANDLE_UV_SEPARATE)) {
 			for (j = 0; j < 8; j++)
 				DUMPREG(i, DISPC_OVL_FIR_COEF_H2, j);
 
@@ -3397,7 +3480,7 @@ int dispc_calc_clock_rates(unsigned long dispc_fclk_rate,
 	return 0;
 }
 
-bool dispc_div_calc(unsigned long dispc,
+bool dispc_div_calc(unsigned long dispc_freq,
 		unsigned long pck_min, unsigned long pck_max,
 		dispc_div_calc_func func, void *data)
 {
@@ -3415,19 +3498,19 @@ bool dispc_div_calc(unsigned long dispc,
 	min_fck_per_pck = 0;
 #endif
 
-	pckd_hw_min = dss_feat_get_param_min(FEAT_PARAM_DSS_PCD);
-	pckd_hw_max = dss_feat_get_param_max(FEAT_PARAM_DSS_PCD);
+	pckd_hw_min = dispc.feat->min_pcd;
+	pckd_hw_max = 255;
 
-	lck_max = dss_feat_get_param_max(FEAT_PARAM_DSS_FCK);
+	lck_max = dss_get_max_fck_rate();
 
 	pck_min = pck_min ? pck_min : 1;
 	pck_max = pck_max ? pck_max : ULONG_MAX;
 
-	lckd_start = max(DIV_ROUND_UP(dispc, lck_max), 1ul);
-	lckd_stop = min(dispc / pck_min, 255ul);
+	lckd_start = max(DIV_ROUND_UP(dispc_freq, lck_max), 1ul);
+	lckd_stop = min(dispc_freq / pck_min, 255ul);
 
 	for (lckd = lckd_start; lckd <= lckd_stop; ++lckd) {
-		lck = dispc / lckd;
+		lck = dispc_freq / lckd;
 
 		pckd_start = max(DIV_ROUND_UP(lck, pck_max), pckd_hw_min);
 		pckd_stop = min(lck / pck_min, pckd_hw_max);
@@ -3441,7 +3524,7 @@ bool dispc_div_calc(unsigned long dispc,
 			 * also. Thus we need to use the calculated lck. For
 			 * OMAP4+ the DISPC fclk is a separate clock.
 			 */
-			if (dss_has_feature(FEAT_CORE_CLK_DIV))
+			if (dispc_has_feature(FEAT_CORE_CLK_DIV))
 				fck = dispc_core_clk_rate();
 			else
 				fck = lck;
@@ -3556,10 +3639,10 @@ static void dispc_restore_gamma_tables(void)
 
 	dispc_mgr_write_gamma_table(OMAP_DSS_CHANNEL_DIGIT);
 
-	if (dss_has_feature(FEAT_MGR_LCD2))
+	if (dispc_has_feature(FEAT_MGR_LCD2))
 		dispc_mgr_write_gamma_table(OMAP_DSS_CHANNEL_LCD2);
 
-	if (dss_has_feature(FEAT_MGR_LCD3))
+	if (dispc_has_feature(FEAT_MGR_LCD3))
 		dispc_mgr_write_gamma_table(OMAP_DSS_CHANNEL_LCD3);
 }
 
@@ -3627,11 +3710,11 @@ static int dispc_init_gamma_tables(void)
 		u32 *gt;
 
 		if (channel == OMAP_DSS_CHANNEL_LCD2 &&
-		    !dss_has_feature(FEAT_MGR_LCD2))
+		    !dispc_has_feature(FEAT_MGR_LCD2))
 			continue;
 
 		if (channel == OMAP_DSS_CHANNEL_LCD3 &&
-		    !dss_has_feature(FEAT_MGR_LCD3))
+		    !dispc_has_feature(FEAT_MGR_LCD3))
 			continue;
 
 		gt = devm_kmalloc_array(&dispc.pdev->dev, gdesc->len,
@@ -3651,7 +3734,7 @@ static void _omap_dispc_initial_config(void)
 	u32 l;
 
 	/* Exclusively enable DISPC_CORE_CLK and set divider to 1 */
-	if (dss_has_feature(FEAT_CORE_CLK_DIV)) {
+	if (dispc_has_feature(FEAT_CORE_CLK_DIV)) {
 		l = dispc_read_reg(DISPC_DIVISOR);
 		/* Use DISPC_DIVISOR.LCD, instead of DISPC_DIVISOR1.LCD */
 		l = FLD_MOD(l, 1, 0, 0);
@@ -3669,7 +3752,7 @@ static void _omap_dispc_initial_config(void)
 	 * func-clock auto-gating. For newer versions
 	 * (dispc.feat->has_gamma_table) this enables tv-out gamma tables.
 	 */
-	if (dss_has_feature(FEAT_FUNCGATED) || dispc.feat->has_gamma_table)
+	if (dispc_has_feature(FEAT_FUNCGATED) || dispc.feat->has_gamma_table)
 		REG_FLD_MOD(DISPC_CONFIG, 1, 9, 9);
 
 	dispc_setup_color_conv_coef();
@@ -3685,10 +3768,272 @@ static void _omap_dispc_initial_config(void)
 	if (dispc.feat->mstandby_workaround)
 		REG_FLD_MOD(DISPC_MSTANDBY_CTRL, 1, 0, 0);
 
-	if (dss_has_feature(FEAT_MFLAG))
+	if (dispc_has_feature(FEAT_MFLAG))
 		dispc_init_mflag();
 }
 
+static const enum dispc_feature_id omap2_dispc_features_list[] = {
+	FEAT_LCDENABLEPOL,
+	FEAT_LCDENABLESIGNAL,
+	FEAT_PCKFREEENABLE,
+	FEAT_FUNCGATED,
+	FEAT_ROWREPEATENABLE,
+	FEAT_RESIZECONF,
+};
+
+static const enum dispc_feature_id omap3_dispc_features_list[] = {
+	FEAT_LCDENABLEPOL,
+	FEAT_LCDENABLESIGNAL,
+	FEAT_PCKFREEENABLE,
+	FEAT_FUNCGATED,
+	FEAT_LINEBUFFERSPLIT,
+	FEAT_ROWREPEATENABLE,
+	FEAT_RESIZECONF,
+	FEAT_CPR,
+	FEAT_PRELOAD,
+	FEAT_FIR_COEF_V,
+	FEAT_ALPHA_FIXED_ZORDER,
+	FEAT_FIFO_MERGE,
+	FEAT_OMAP3_DSI_FIFO_BUG,
+};
+
+static const enum dispc_feature_id am43xx_dispc_features_list[] = {
+	FEAT_LCDENABLEPOL,
+	FEAT_LCDENABLESIGNAL,
+	FEAT_PCKFREEENABLE,
+	FEAT_FUNCGATED,
+	FEAT_LINEBUFFERSPLIT,
+	FEAT_ROWREPEATENABLE,
+	FEAT_RESIZECONF,
+	FEAT_CPR,
+	FEAT_PRELOAD,
+	FEAT_FIR_COEF_V,
+	FEAT_ALPHA_FIXED_ZORDER,
+	FEAT_FIFO_MERGE,
+};
+
+static const enum dispc_feature_id omap4_dispc_features_list[] = {
+	FEAT_MGR_LCD2,
+	FEAT_CORE_CLK_DIV,
+	FEAT_HANDLE_UV_SEPARATE,
+	FEAT_ATTR2,
+	FEAT_CPR,
+	FEAT_PRELOAD,
+	FEAT_FIR_COEF_V,
+	FEAT_ALPHA_FREE_ZORDER,
+	FEAT_FIFO_MERGE,
+	FEAT_BURST_2D,
+};
+
+static const enum dispc_feature_id omap5_dispc_features_list[] = {
+	FEAT_MGR_LCD2,
+	FEAT_MGR_LCD3,
+	FEAT_CORE_CLK_DIV,
+	FEAT_HANDLE_UV_SEPARATE,
+	FEAT_ATTR2,
+	FEAT_CPR,
+	FEAT_PRELOAD,
+	FEAT_FIR_COEF_V,
+	FEAT_ALPHA_FREE_ZORDER,
+	FEAT_FIFO_MERGE,
+	FEAT_BURST_2D,
+	FEAT_MFLAG,
+};
+
+static const struct dss_reg_field omap2_dispc_reg_fields[] = {
+	[FEAT_REG_FIRHINC]			= { 11, 0 },
+	[FEAT_REG_FIRVINC]			= { 27, 16 },
+	[FEAT_REG_FIFOLOWTHRESHOLD]		= { 8, 0 },
+	[FEAT_REG_FIFOHIGHTHRESHOLD]		= { 24, 16 },
+	[FEAT_REG_FIFOSIZE]			= { 8, 0 },
+	[FEAT_REG_HORIZONTALACCU]		= { 9, 0 },
+	[FEAT_REG_VERTICALACCU]			= { 25, 16 },
+};
+
+static const struct dss_reg_field omap3_dispc_reg_fields[] = {
+	[FEAT_REG_FIRHINC]			= { 12, 0 },
+	[FEAT_REG_FIRVINC]			= { 28, 16 },
+	[FEAT_REG_FIFOLOWTHRESHOLD]		= { 11, 0 },
+	[FEAT_REG_FIFOHIGHTHRESHOLD]		= { 27, 16 },
+	[FEAT_REG_FIFOSIZE]			= { 10, 0 },
+	[FEAT_REG_HORIZONTALACCU]		= { 9, 0 },
+	[FEAT_REG_VERTICALACCU]			= { 25, 16 },
+};
+
+static const struct dss_reg_field omap4_dispc_reg_fields[] = {
+	[FEAT_REG_FIRHINC]			= { 12, 0 },
+	[FEAT_REG_FIRVINC]			= { 28, 16 },
+	[FEAT_REG_FIFOLOWTHRESHOLD]		= { 15, 0 },
+	[FEAT_REG_FIFOHIGHTHRESHOLD]		= { 31, 16 },
+	[FEAT_REG_FIFOSIZE]			= { 15, 0 },
+	[FEAT_REG_HORIZONTALACCU]		= { 10, 0 },
+	[FEAT_REG_VERTICALACCU]			= { 26, 16 },
+};
+
+static const enum omap_overlay_caps omap2_dispc_overlay_caps[] = {
+	/* OMAP_DSS_GFX */
+	OMAP_DSS_OVL_CAP_POS | OMAP_DSS_OVL_CAP_REPLICATION,
+
+	/* OMAP_DSS_VIDEO1 */
+	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_POS |
+		OMAP_DSS_OVL_CAP_REPLICATION,
+
+	/* OMAP_DSS_VIDEO2 */
+	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_POS |
+		OMAP_DSS_OVL_CAP_REPLICATION,
+};
+
+static const enum omap_overlay_caps omap3430_dispc_overlay_caps[] = {
+	/* OMAP_DSS_GFX */
+	OMAP_DSS_OVL_CAP_GLOBAL_ALPHA | OMAP_DSS_OVL_CAP_POS |
+		OMAP_DSS_OVL_CAP_REPLICATION,
+
+	/* OMAP_DSS_VIDEO1 */
+	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_POS |
+		OMAP_DSS_OVL_CAP_REPLICATION,
+
+	/* OMAP_DSS_VIDEO2 */
+	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_GLOBAL_ALPHA |
+		OMAP_DSS_OVL_CAP_POS | OMAP_DSS_OVL_CAP_REPLICATION,
+};
+
+static const enum omap_overlay_caps omap3630_dispc_overlay_caps[] = {
+	/* OMAP_DSS_GFX */
+	OMAP_DSS_OVL_CAP_GLOBAL_ALPHA | OMAP_DSS_OVL_CAP_PRE_MULT_ALPHA |
+		OMAP_DSS_OVL_CAP_POS | OMAP_DSS_OVL_CAP_REPLICATION,
+
+	/* OMAP_DSS_VIDEO1 */
+	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_POS |
+		OMAP_DSS_OVL_CAP_REPLICATION,
+
+	/* OMAP_DSS_VIDEO2 */
+	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_GLOBAL_ALPHA |
+		OMAP_DSS_OVL_CAP_PRE_MULT_ALPHA | OMAP_DSS_OVL_CAP_POS |
+		OMAP_DSS_OVL_CAP_REPLICATION,
+};
+
+static const enum omap_overlay_caps omap4_dispc_overlay_caps[] = {
+	/* OMAP_DSS_GFX */
+	OMAP_DSS_OVL_CAP_GLOBAL_ALPHA | OMAP_DSS_OVL_CAP_PRE_MULT_ALPHA |
+		OMAP_DSS_OVL_CAP_ZORDER | OMAP_DSS_OVL_CAP_POS |
+		OMAP_DSS_OVL_CAP_REPLICATION,
+
+	/* OMAP_DSS_VIDEO1 */
+	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_GLOBAL_ALPHA |
+		OMAP_DSS_OVL_CAP_PRE_MULT_ALPHA | OMAP_DSS_OVL_CAP_ZORDER |
+		OMAP_DSS_OVL_CAP_POS | OMAP_DSS_OVL_CAP_REPLICATION,
+
+	/* OMAP_DSS_VIDEO2 */
+	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_GLOBAL_ALPHA |
+		OMAP_DSS_OVL_CAP_PRE_MULT_ALPHA | OMAP_DSS_OVL_CAP_ZORDER |
+		OMAP_DSS_OVL_CAP_POS | OMAP_DSS_OVL_CAP_REPLICATION,
+
+	/* OMAP_DSS_VIDEO3 */
+	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_GLOBAL_ALPHA |
+		OMAP_DSS_OVL_CAP_PRE_MULT_ALPHA | OMAP_DSS_OVL_CAP_ZORDER |
+		OMAP_DSS_OVL_CAP_POS | OMAP_DSS_OVL_CAP_REPLICATION,
+};
+
+#define COLOR_ARRAY(arr...) (const u32[]) { arr, 0 }
+
+static const u32 *omap2_dispc_supported_color_modes[] = {
+
+	/* OMAP_DSS_GFX */
+	COLOR_ARRAY(
+	DRM_FORMAT_RGBX4444, DRM_FORMAT_RGB565,
+	DRM_FORMAT_XRGB8888, DRM_FORMAT_RGB888),
+
+	/* OMAP_DSS_VIDEO1 */
+	COLOR_ARRAY(
+	DRM_FORMAT_RGB565, DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_RGB888, DRM_FORMAT_YUYV,
+	DRM_FORMAT_UYVY),
+
+	/* OMAP_DSS_VIDEO2 */
+	COLOR_ARRAY(
+	DRM_FORMAT_RGB565, DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_RGB888, DRM_FORMAT_YUYV,
+	DRM_FORMAT_UYVY),
+};
+
+static const u32 *omap3_dispc_supported_color_modes[] = {
+	/* OMAP_DSS_GFX */
+	COLOR_ARRAY(
+	DRM_FORMAT_RGBX4444, DRM_FORMAT_ARGB4444,
+	DRM_FORMAT_RGB565, DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_RGB888, DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_RGBA8888, DRM_FORMAT_RGBX8888),
+
+	/* OMAP_DSS_VIDEO1 */
+	COLOR_ARRAY(
+	DRM_FORMAT_XRGB8888, DRM_FORMAT_RGB888,
+	DRM_FORMAT_RGBX4444, DRM_FORMAT_RGB565,
+	DRM_FORMAT_YUYV, DRM_FORMAT_UYVY),
+
+	/* OMAP_DSS_VIDEO2 */
+	COLOR_ARRAY(
+	DRM_FORMAT_RGBX4444, DRM_FORMAT_ARGB4444,
+	DRM_FORMAT_RGB565, DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_RGB888, DRM_FORMAT_YUYV,
+	DRM_FORMAT_UYVY, DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_RGBA8888, DRM_FORMAT_RGBX8888),
+};
+
+static const u32 *omap4_dispc_supported_color_modes[] = {
+	/* OMAP_DSS_GFX */
+	COLOR_ARRAY(
+	DRM_FORMAT_RGBX4444, DRM_FORMAT_ARGB4444,
+	DRM_FORMAT_RGB565, DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_RGB888, DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_RGBA8888, DRM_FORMAT_RGBX8888,
+	DRM_FORMAT_ARGB1555, DRM_FORMAT_XRGB4444,
+	DRM_FORMAT_RGBA4444, DRM_FORMAT_XRGB1555),
+
+	/* OMAP_DSS_VIDEO1 */
+	COLOR_ARRAY(
+	DRM_FORMAT_RGB565, DRM_FORMAT_RGBX4444,
+	DRM_FORMAT_YUYV, DRM_FORMAT_ARGB1555,
+	DRM_FORMAT_RGBA8888, DRM_FORMAT_NV12,
+	DRM_FORMAT_RGBA4444, DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_RGB888, DRM_FORMAT_UYVY,
+	DRM_FORMAT_ARGB4444, DRM_FORMAT_XRGB1555,
+	DRM_FORMAT_ARGB8888, DRM_FORMAT_XRGB4444,
+	DRM_FORMAT_RGBX8888),
+
+       /* OMAP_DSS_VIDEO2 */
+	COLOR_ARRAY(
+	DRM_FORMAT_RGB565, DRM_FORMAT_RGBX4444,
+	DRM_FORMAT_YUYV, DRM_FORMAT_ARGB1555,
+	DRM_FORMAT_RGBA8888, DRM_FORMAT_NV12,
+	DRM_FORMAT_RGBA4444, DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_RGB888, DRM_FORMAT_UYVY,
+	DRM_FORMAT_ARGB4444, DRM_FORMAT_XRGB1555,
+	DRM_FORMAT_ARGB8888, DRM_FORMAT_XRGB4444,
+	DRM_FORMAT_RGBX8888),
+
+	/* OMAP_DSS_VIDEO3 */
+	COLOR_ARRAY(
+	DRM_FORMAT_RGB565, DRM_FORMAT_RGBX4444,
+	DRM_FORMAT_YUYV, DRM_FORMAT_ARGB1555,
+	DRM_FORMAT_RGBA8888, DRM_FORMAT_NV12,
+	DRM_FORMAT_RGBA4444, DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_RGB888, DRM_FORMAT_UYVY,
+	DRM_FORMAT_ARGB4444, DRM_FORMAT_XRGB1555,
+	DRM_FORMAT_ARGB8888, DRM_FORMAT_XRGB4444,
+	DRM_FORMAT_RGBX8888),
+
+	/* OMAP_DSS_WB */
+	COLOR_ARRAY(
+	DRM_FORMAT_RGB565, DRM_FORMAT_RGBX4444,
+	DRM_FORMAT_YUYV, DRM_FORMAT_ARGB1555,
+	DRM_FORMAT_RGBA8888, DRM_FORMAT_NV12,
+	DRM_FORMAT_RGBA4444, DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_RGB888, DRM_FORMAT_UYVY,
+	DRM_FORMAT_ARGB4444, DRM_FORMAT_XRGB1555,
+	DRM_FORMAT_ARGB8888, DRM_FORMAT_XRGB4444,
+	DRM_FORMAT_RGBX8888),
+};
+
 static const struct dispc_features omap24xx_dispc_feats = {
 	.sw_start		=	5,
 	.fp_start		=	15,
@@ -3701,9 +4046,26 @@ static const struct dispc_features omap24xx_dispc_feats = {
 	.mgr_width_max		=	2048,
 	.mgr_height_max		=	2048,
 	.max_lcd_pclk		=	66500000,
+	.max_downscale		=	2,
+	/*
+	 * Assume the line width buffer to be 768 pixels as OMAP2 DISPC scaler
+	 * cannot scale an image width larger than 768.
+	 */
+	.max_line_width		=	768,
+	.min_pcd		=	2,
 	.calc_scaling		=	dispc_ovl_calc_scaling_24xx,
 	.calc_core_clk		=	calc_core_clk_24xx,
 	.num_fifos		=	3,
+	.features		=	omap2_dispc_features_list,
+	.num_features		=	ARRAY_SIZE(omap2_dispc_features_list),
+	.reg_fields		=	omap2_dispc_reg_fields,
+	.num_reg_fields		=	ARRAY_SIZE(omap2_dispc_reg_fields),
+	.overlay_caps		=	omap2_dispc_overlay_caps,
+	.supported_color_modes	=	omap2_dispc_supported_color_modes,
+	.num_mgrs		=	2,
+	.num_ovls		=	3,
+	.buffer_size_unit	=	1,
+	.burst_size_unit	=	8,
 	.no_framedone_tv	=	true,
 	.set_max_preload	=	false,
 	.last_pixel_inc_missing	=	true,
@@ -3722,9 +4084,22 @@ static const struct dispc_features omap34xx_rev1_0_dispc_feats = {
 	.mgr_height_max		=	2048,
 	.max_lcd_pclk		=	173000000,
 	.max_tv_pclk		=	59000000,
+	.max_downscale		=	4,
+	.max_line_width		=	1024,
+	.min_pcd		=	1,
 	.calc_scaling		=	dispc_ovl_calc_scaling_34xx,
 	.calc_core_clk		=	calc_core_clk_34xx,
 	.num_fifos		=	3,
+	.features		=	omap3_dispc_features_list,
+	.num_features		=	ARRAY_SIZE(omap3_dispc_features_list),
+	.reg_fields		=	omap3_dispc_reg_fields,
+	.num_reg_fields		=	ARRAY_SIZE(omap3_dispc_reg_fields),
+	.overlay_caps		=	omap3430_dispc_overlay_caps,
+	.supported_color_modes	=	omap3_dispc_supported_color_modes,
+	.num_mgrs		=	2,
+	.num_ovls		=	3,
+	.buffer_size_unit	=	1,
+	.burst_size_unit	=	8,
 	.no_framedone_tv	=	true,
 	.set_max_preload	=	false,
 	.last_pixel_inc_missing	=	true,
@@ -3743,9 +4118,90 @@ static const struct dispc_features omap34xx_rev3_0_dispc_feats = {
 	.mgr_height_max		=	2048,
 	.max_lcd_pclk		=	173000000,
 	.max_tv_pclk		=	59000000,
+	.max_downscale		=	4,
+	.max_line_width		=	1024,
+	.min_pcd		=	1,
+	.calc_scaling		=	dispc_ovl_calc_scaling_34xx,
+	.calc_core_clk		=	calc_core_clk_34xx,
+	.num_fifos		=	3,
+	.features		=	omap3_dispc_features_list,
+	.num_features		=	ARRAY_SIZE(omap3_dispc_features_list),
+	.reg_fields		=	omap3_dispc_reg_fields,
+	.num_reg_fields		=	ARRAY_SIZE(omap3_dispc_reg_fields),
+	.overlay_caps		=	omap3430_dispc_overlay_caps,
+	.supported_color_modes	=	omap3_dispc_supported_color_modes,
+	.num_mgrs		=	2,
+	.num_ovls		=	3,
+	.buffer_size_unit	=	1,
+	.burst_size_unit	=	8,
+	.no_framedone_tv	=	true,
+	.set_max_preload	=	false,
+	.last_pixel_inc_missing	=	true,
+};
+
+static const struct dispc_features omap36xx_dispc_feats = {
+	.sw_start		=	7,
+	.fp_start		=	19,
+	.bp_start		=	31,
+	.sw_max			=	256,
+	.vp_max			=	4095,
+	.hp_max			=	4096,
+	.mgr_width_start	=	10,
+	.mgr_height_start	=	26,
+	.mgr_width_max		=	2048,
+	.mgr_height_max		=	2048,
+	.max_lcd_pclk		=	173000000,
+	.max_tv_pclk		=	59000000,
+	.max_downscale		=	4,
+	.max_line_width		=	1024,
+	.min_pcd		=	1,
 	.calc_scaling		=	dispc_ovl_calc_scaling_34xx,
 	.calc_core_clk		=	calc_core_clk_34xx,
 	.num_fifos		=	3,
+	.features		=	omap3_dispc_features_list,
+	.num_features		=	ARRAY_SIZE(omap3_dispc_features_list),
+	.reg_fields		=	omap3_dispc_reg_fields,
+	.num_reg_fields		=	ARRAY_SIZE(omap3_dispc_reg_fields),
+	.overlay_caps		=	omap3630_dispc_overlay_caps,
+	.supported_color_modes	=	omap3_dispc_supported_color_modes,
+	.num_mgrs		=	2,
+	.num_ovls		=	3,
+	.buffer_size_unit	=	1,
+	.burst_size_unit	=	8,
+	.no_framedone_tv	=	true,
+	.set_max_preload	=	false,
+	.last_pixel_inc_missing	=	true,
+};
+
+static const struct dispc_features am43xx_dispc_feats = {
+	.sw_start		=	7,
+	.fp_start		=	19,
+	.bp_start		=	31,
+	.sw_max			=	256,
+	.vp_max			=	4095,
+	.hp_max			=	4096,
+	.mgr_width_start	=	10,
+	.mgr_height_start	=	26,
+	.mgr_width_max		=	2048,
+	.mgr_height_max		=	2048,
+	.max_lcd_pclk		=	173000000,
+	.max_tv_pclk		=	59000000,
+	.max_downscale		=	4,
+	.max_line_width		=	1024,
+	.min_pcd		=	1,
+	.calc_scaling		=	dispc_ovl_calc_scaling_34xx,
+	.calc_core_clk		=	calc_core_clk_34xx,
+	.num_fifos		=	3,
+	.features		=	am43xx_dispc_features_list,
+	.num_features		=	ARRAY_SIZE(am43xx_dispc_features_list),
+	.reg_fields		=	omap3_dispc_reg_fields,
+	.num_reg_fields		=	ARRAY_SIZE(omap3_dispc_reg_fields),
+	.overlay_caps		=	omap3430_dispc_overlay_caps,
+	.supported_color_modes	=	omap3_dispc_supported_color_modes,
+	.num_mgrs		=	1,
+	.num_ovls		=	3,
+	.buffer_size_unit	=	1,
+	.burst_size_unit	=	8,
 	.no_framedone_tv	=	true,
 	.set_max_preload	=	false,
 	.last_pixel_inc_missing	=	true,
@@ -3764,9 +4220,22 @@ static const struct dispc_features omap44xx_dispc_feats = {
 	.mgr_height_max		=	2048,
 	.max_lcd_pclk		=	170000000,
 	.max_tv_pclk		=	185625000,
+	.max_downscale		=	4,
+	.max_line_width		=	2048,
+	.min_pcd		=	1,
 	.calc_scaling		=	dispc_ovl_calc_scaling_44xx,
 	.calc_core_clk		=	calc_core_clk_44xx,
 	.num_fifos		=	5,
+	.features		=	omap4_dispc_features_list,
+	.num_features		=	ARRAY_SIZE(omap4_dispc_features_list),
+	.reg_fields		=	omap4_dispc_reg_fields,
+	.num_reg_fields		=	ARRAY_SIZE(omap4_dispc_reg_fields),
+	.overlay_caps		=	omap4_dispc_overlay_caps,
+	.supported_color_modes	=	omap4_dispc_supported_color_modes,
+	.num_mgrs		=	3,
+	.num_ovls		=	4,
+	.buffer_size_unit	=	16,
+	.burst_size_unit	=	16,
 	.gfx_fifo_workaround	=	true,
 	.set_max_preload	=	true,
 	.supports_sync_align	=	true,
@@ -3790,9 +4259,22 @@ static const struct dispc_features omap54xx_dispc_feats = {
 	.mgr_height_max		=	4096,
 	.max_lcd_pclk		=	170000000,
 	.max_tv_pclk		=	186000000,
+	.max_downscale		=	4,
+	.max_line_width		=	2048,
+	.min_pcd		=	1,
 	.calc_scaling		=	dispc_ovl_calc_scaling_44xx,
 	.calc_core_clk		=	calc_core_clk_44xx,
 	.num_fifos		=	5,
+	.features		=	omap5_dispc_features_list,
+	.num_features		=	ARRAY_SIZE(omap5_dispc_features_list),
+	.reg_fields		=	omap4_dispc_reg_fields,
+	.num_reg_fields		=	ARRAY_SIZE(omap4_dispc_reg_fields),
+	.overlay_caps		=	omap4_dispc_overlay_caps,
+	.supported_color_modes	=	omap4_dispc_supported_color_modes,
+	.num_mgrs		=	4,
+	.num_ovls		=	4,
+	.buffer_size_unit	=	16,
+	.burst_size_unit	=	16,
 	.gfx_fifo_workaround	=	true,
 	.mstandby_workaround	=	true,
 	.set_max_preload	=	true,
@@ -3804,54 +4286,6 @@ static const struct dispc_features omap54xx_dispc_feats = {
 	.has_gamma_i734_bug	=	true,
 };
 
-static int dispc_init_features(struct platform_device *pdev)
-{
-	const struct dispc_features *src;
-	struct dispc_features *dst;
-
-	dst = devm_kzalloc(&pdev->dev, sizeof(*dst), GFP_KERNEL);
-	if (!dst) {
-		dev_err(&pdev->dev, "Failed to allocate DISPC Features\n");
-		return -ENOMEM;
-	}
-
-	switch (omapdss_get_version()) {
-	case OMAPDSS_VER_OMAP24xx:
-		src = &omap24xx_dispc_feats;
-		break;
-
-	case OMAPDSS_VER_OMAP34xx_ES1:
-		src = &omap34xx_rev1_0_dispc_feats;
-		break;
-
-	case OMAPDSS_VER_OMAP34xx_ES3:
-	case OMAPDSS_VER_OMAP3630:
-	case OMAPDSS_VER_AM35xx:
-	case OMAPDSS_VER_AM43xx:
-		src = &omap34xx_rev3_0_dispc_feats;
-		break;
-
-	case OMAPDSS_VER_OMAP4430_ES1:
-	case OMAPDSS_VER_OMAP4430_ES2:
-	case OMAPDSS_VER_OMAP4:
-		src = &omap44xx_dispc_feats;
-		break;
-
-	case OMAPDSS_VER_OMAP5:
-	case OMAPDSS_VER_DRA7xx:
-		src = &omap54xx_dispc_feats;
-		break;
-
-	default:
-		return -ENODEV;
-	}
-
-	memcpy(dst, src, sizeof(*dst));
-	dispc.feat = dst;
-
-	return 0;
-}
-
 static irqreturn_t dispc_irq_handler(int irq, void *arg)
 {
 	if (!dispc.is_enabled)
@@ -4083,9 +4517,28 @@ static const struct dispc_ops dispc_ops = {
 };
 
 /* DISPC HW IP initialisation */
+static const struct of_device_id dispc_of_match[] = {
+	{ .compatible = "ti,omap2-dispc", .data = &omap24xx_dispc_feats },
+	{ .compatible = "ti,omap3-dispc", .data = &omap36xx_dispc_feats },
+	{ .compatible = "ti,omap4-dispc", .data = &omap44xx_dispc_feats },
+	{ .compatible = "ti,omap5-dispc", .data = &omap54xx_dispc_feats },
+	{ .compatible = "ti,dra7-dispc",  .data = &omap54xx_dispc_feats },
+	{},
+};
+
+static const struct soc_device_attribute dispc_soc_devices[] = {
+	{ .machine = "OMAP3[45]*",
+	  .revision = "ES[12].?",	.data = &omap34xx_rev1_0_dispc_feats },
+	{ .machine = "OMAP3[45]*",	.data = &omap34xx_rev3_0_dispc_feats },
+	{ .machine = "AM35*",		.data = &omap34xx_rev3_0_dispc_feats },
+	{ .machine = "AM43*",		.data = &am43xx_dispc_feats },
+	{ /* sentinel */ }
+};
+
 static int dispc_bind(struct device *dev, struct device *master, void *data)
 {
 	struct platform_device *pdev = to_platform_device(dev);
+	const struct soc_device_attribute *soc;
 	u32 rev;
 	int r = 0;
 	struct resource *dispc_mem;
@@ -4095,9 +4548,15 @@ static int dispc_bind(struct device *dev, struct device *master, void *data)
 
 	spin_lock_init(&dispc.control_lock);
 
-	r = dispc_init_features(dispc.pdev);
-	if (r)
-		return r;
+	/*
+	 * The OMAP3-based models can't be told apart using the compatible
+	 * string, use SoC device matching.
+	 */
+	soc = soc_device_match(dispc_soc_devices);
+	if (soc)
+		dispc.feat = soc->data;
+	else
+		dispc.feat = of_match_device(dispc_of_match, &pdev->dev)->data;
 
 	r = dispc_errata_i734_wa_init();
 	if (r)
@@ -4226,15 +4685,6 @@ static const struct dev_pm_ops dispc_pm_ops = {
 	.runtime_resume = dispc_runtime_resume,
 };
 
-static const struct of_device_id dispc_of_match[] = {
-	{ .compatible = "ti,omap2-dispc", },
-	{ .compatible = "ti,omap3-dispc", },
-	{ .compatible = "ti,omap4-dispc", },
-	{ .compatible = "ti,omap5-dispc", },
-	{ .compatible = "ti,dra7-dispc", },
-	{},
-};
-
 static struct platform_driver omap_dispchw_driver = {
 	.probe		= dispc_probe,
 	.remove         = dispc_remove,
diff --git a/drivers/gpu/drm/omapdrm/dss/dpi.c b/drivers/gpu/drm/omapdrm/dss/dpi.c
index 86dbb65a6c28..daf286fc8a40 100644
--- a/drivers/gpu/drm/omapdrm/dss/dpi.c
+++ b/drivers/gpu/drm/omapdrm/dss/dpi.c
@@ -32,13 +32,14 @@
 #include <linux/string.h>
 #include <linux/of.h>
 #include <linux/clk.h>
+#include <linux/sys_soc.h>
 
 #include "omapdss.h"
 #include "dss.h"
-#include "dss_features.h"
 
 struct dpi_data {
 	struct platform_device *pdev;
+	enum dss_model dss_model;
 
 	struct regulator *vdds_dsi_reg;
 	enum dss_clk_source clk_src;
@@ -99,25 +100,21 @@ static enum dss_clk_source dpi_get_clk_src_dra7xx(enum omap_channel channel)
 	return DSS_CLK_SRC_FCK;
 }
 
-static enum dss_clk_source dpi_get_clk_src(enum omap_channel channel)
+static enum dss_clk_source dpi_get_clk_src(struct dpi_data *dpi)
 {
+	enum omap_channel channel = dpi->output.dispc_channel;
+
 	/*
 	 * XXX we can't currently use DSI PLL for DPI with OMAP3, as the DSI PLL
 	 * would also be used for DISPC fclk. Meaning, when the DPI output is
 	 * disabled, DISPC clock will be disabled, and TV out will stop.
 	 */
-	switch (omapdss_get_version()) {
-	case OMAPDSS_VER_OMAP24xx:
-	case OMAPDSS_VER_OMAP34xx_ES1:
-	case OMAPDSS_VER_OMAP34xx_ES3:
-	case OMAPDSS_VER_OMAP3630:
-	case OMAPDSS_VER_AM35xx:
-	case OMAPDSS_VER_AM43xx:
+	switch (dpi->dss_model) {
+	case DSS_MODEL_OMAP2:
+	case DSS_MODEL_OMAP3:
 		return DSS_CLK_SRC_FCK;
 
-	case OMAPDSS_VER_OMAP4430_ES1:
-	case OMAPDSS_VER_OMAP4430_ES2:
-	case OMAPDSS_VER_OMAP4:
+	case DSS_MODEL_OMAP4:
 		switch (channel) {
 		case OMAP_DSS_CHANNEL_LCD:
 			return DSS_CLK_SRC_PLL1_1;
@@ -127,7 +124,7 @@ static enum dss_clk_source dpi_get_clk_src(enum omap_channel channel)
 			return DSS_CLK_SRC_FCK;
 		}
 
-	case OMAPDSS_VER_OMAP5:
+	case DSS_MODEL_OMAP5:
 		switch (channel) {
 		case OMAP_DSS_CHANNEL_LCD:
 			return DSS_CLK_SRC_PLL1_1;
@@ -138,7 +135,7 @@ static enum dss_clk_source dpi_get_clk_src(enum omap_channel channel)
 			return DSS_CLK_SRC_FCK;
 		}
 
-	case OMAPDSS_VER_DRA7xx:
+	case DSS_MODEL_DRA7:
 		return dpi_get_clk_src_dra7xx(channel);
 
 	default:
@@ -213,7 +210,7 @@ static bool dpi_calc_pll_cb(int n, int m, unsigned long fint,
 	ctx->pll_cinfo.clkdco = clkdco;
 
 	return dss_pll_hsdiv_calc_a(ctx->pll, clkdco,
-		ctx->pck_min, dss_feat_get_param_max(FEAT_PARAM_DSS_FCK),
+		ctx->pck_min, dss_get_max_fck_rate(),
 		dpi_calc_hsdiv_cb, ctx);
 }
 
@@ -403,19 +400,13 @@ static int dpi_display_enable(struct omap_dss_device *dssdev)
 
 	mutex_lock(&dpi->lock);
 
-	if (dss_has_feature(FEAT_DPI_USES_VDDS_DSI) && !dpi->vdds_dsi_reg) {
-		DSSERR("no VDSS_DSI regulator\n");
-		r = -ENODEV;
-		goto err_no_reg;
-	}
-
 	if (!out->dispc_channel_connected) {
 		DSSERR("failed to enable display: no output/manager\n");
 		r = -ENODEV;
 		goto err_no_out_mgr;
 	}
 
-	if (dss_has_feature(FEAT_DPI_USES_VDDS_DSI)) {
+	if (dpi->vdds_dsi_reg) {
 		r = regulator_enable(dpi->vdds_dsi_reg);
 		if (r)
 			goto err_reg_enable;
@@ -459,11 +450,10 @@ err_pll_init:
 err_src_sel:
 	dispc_runtime_put();
 err_get_dispc:
-	if (dss_has_feature(FEAT_DPI_USES_VDDS_DSI))
+	if (dpi->vdds_dsi_reg)
 		regulator_disable(dpi->vdds_dsi_reg);
 err_reg_enable:
 err_no_out_mgr:
-err_no_reg:
 	mutex_unlock(&dpi->lock);
 	return r;
 }
@@ -484,7 +474,7 @@ static void dpi_display_disable(struct omap_dss_device *dssdev)
 
 	dispc_runtime_put();
 
-	if (dss_has_feature(FEAT_DPI_USES_VDDS_DSI))
+	if (dpi->vdds_dsi_reg)
 		regulator_disable(dpi->vdds_dsi_reg);
 
 	mutex_unlock(&dpi->lock);
@@ -575,11 +565,21 @@ static int dpi_verify_pll(struct dss_pll *pll)
 	return 0;
 }
 
+static const struct soc_device_attribute dpi_soc_devices[] = {
+	{ .family = "OMAP3[456]*" },
+	{ .family = "[AD]M37*" },
+	{ /* sentinel */ }
+};
+
 static int dpi_init_regulator(struct dpi_data *dpi)
 {
 	struct regulator *vdds_dsi;
 
-	if (!dss_has_feature(FEAT_DPI_USES_VDDS_DSI))
+	/*
+	 * The DPI uses the DSI VDDS on OMAP34xx, OMAP35xx, OMAP36xx, AM37xx and
+	 * DM37xx only.
+	 */
+	if (!soc_device_match(dpi_soc_devices))
 		return 0;
 
 	if (dpi->vdds_dsi_reg)
@@ -604,7 +604,7 @@ static void dpi_init_pll(struct dpi_data *dpi)
 	if (dpi->pll)
 		return;
 
-	dpi->clk_src = dpi_get_clk_src(dpi->output.dispc_channel);
+	dpi->clk_src = dpi_get_clk_src(dpi);
 
 	pll = dss_pll_find_by_src(dpi->clk_src);
 	if (!pll)
@@ -624,18 +624,14 @@ static void dpi_init_pll(struct dpi_data *dpi)
  * the channel in some more dynamic manner, or get the channel as a user
  * parameter.
  */
-static enum omap_channel dpi_get_channel(int port_num)
+static enum omap_channel dpi_get_channel(struct dpi_data *dpi, int port_num)
 {
-	switch (omapdss_get_version()) {
-	case OMAPDSS_VER_OMAP24xx:
-	case OMAPDSS_VER_OMAP34xx_ES1:
-	case OMAPDSS_VER_OMAP34xx_ES3:
-	case OMAPDSS_VER_OMAP3630:
-	case OMAPDSS_VER_AM35xx:
-	case OMAPDSS_VER_AM43xx:
+	switch (dpi->dss_model) {
+	case DSS_MODEL_OMAP2:
+	case DSS_MODEL_OMAP3:
 		return OMAP_DSS_CHANNEL_LCD;
 
-	case OMAPDSS_VER_DRA7xx:
+	case DSS_MODEL_DRA7:
 		switch (port_num) {
 		case 2:
 			return OMAP_DSS_CHANNEL_LCD3;
@@ -646,12 +642,10 @@ static enum omap_channel dpi_get_channel(int port_num)
 			return OMAP_DSS_CHANNEL_LCD;
 		}
 
-	case OMAPDSS_VER_OMAP4430_ES1:
-	case OMAPDSS_VER_OMAP4430_ES2:
-	case OMAPDSS_VER_OMAP4:
+	case DSS_MODEL_OMAP4:
 		return OMAP_DSS_CHANNEL_LCD2;
 
-	case OMAPDSS_VER_OMAP5:
+	case DSS_MODEL_OMAP5:
 		return OMAP_DSS_CHANNEL_LCD3;
 
 	default:
@@ -716,10 +710,8 @@ static const struct omapdss_dpi_ops dpi_ops = {
 	.get_timings = dpi_get_timings,
 };
 
-static void dpi_init_output_port(struct platform_device *pdev,
-	struct device_node *port)
+static void dpi_init_output_port(struct dpi_data *dpi, struct device_node *port)
 {
-	struct dpi_data *dpi = port->data;
 	struct omap_dss_device *out = &dpi->output;
 	int r;
 	u32 port_num;
@@ -741,10 +733,10 @@ static void dpi_init_output_port(struct platform_device *pdev,
 		break;
 	}
 
-	out->dev = &pdev->dev;
+	out->dev = &dpi->pdev->dev;
 	out->id = OMAP_DSS_OUTPUT_DPI;
 	out->output_type = OMAP_DISPLAY_TYPE_DPI;
-	out->dispc_channel = dpi_get_channel(port_num);
+	out->dispc_channel = dpi_get_channel(dpi, port_num);
 	out->port_num = port_num;
 	out->ops.dpi = &dpi_ops;
 	out->owner = THIS_MODULE;
@@ -760,7 +752,8 @@ static void dpi_uninit_output_port(struct device_node *port)
 	omapdss_unregister_output(out);
 }
 
-int dpi_init_port(struct platform_device *pdev, struct device_node *port)
+int dpi_init_port(struct platform_device *pdev, struct device_node *port,
+		  enum dss_model dss_model)
 {
 	struct dpi_data *dpi;
 	struct device_node *ep;
@@ -786,11 +779,12 @@ int dpi_init_port(struct platform_device *pdev, struct device_node *port)
 	of_node_put(ep);
 
 	dpi->pdev = pdev;
+	dpi->dss_model = dss_model;
 	port->data = dpi;
 
 	mutex_init(&dpi->lock);
 
-	dpi_init_output_port(pdev, port);
+	dpi_init_output_port(dpi, port);
 
 	dpi->port_initialized = true;
 
diff --git a/drivers/gpu/drm/omapdrm/dss/dsi.c b/drivers/gpu/drm/omapdrm/dss/dsi.c
index 835f49004bc3..b56a05730314 100644
--- a/drivers/gpu/drm/omapdrm/dss/dsi.c
+++ b/drivers/gpu/drm/omapdrm/dss/dsi.c
@@ -20,6 +20,8 @@
 #define DSS_SUBSYS_NAME "DSI"
 
 #include <linux/kernel.h>
+#include <linux/mfd/syscon.h>
+#include <linux/regmap.h>
 #include <linux/io.h>
 #include <linux/clk.h>
 #include <linux/device.h>
@@ -42,12 +44,12 @@
 #include <linux/of_graph.h>
 #include <linux/of_platform.h>
 #include <linux/component.h>
+#include <linux/sys_soc.h>
 
 #include <video/mipi_display.h>
 
 #include "omapdss.h"
 #include "dss.h"
-#include "dss_features.h"
 
 #define DSI_CATCH_MISSING_TE
 
@@ -228,6 +230,12 @@ static int dsi_vc_send_null(struct omap_dss_device *dssdev, int channel);
 #define DSI_MAX_NR_ISRS                2
 #define DSI_MAX_NR_LANES	5
 
+enum dsi_model {
+	DSI_MODEL_OMAP3,
+	DSI_MODEL_OMAP4,
+	DSI_MODEL_OMAP5,
+};
+
 enum dsi_lane_function {
 	DSI_LANE_UNUSED	= 0,
 	DSI_LANE_CLK,
@@ -299,12 +307,36 @@ struct dsi_lp_clock_info {
 	u16 lp_clk_div;
 };
 
+struct dsi_module_id_data {
+	u32 address;
+	int id;
+};
+
+enum dsi_quirks {
+	DSI_QUIRK_PLL_PWR_BUG = (1 << 0),	/* DSI-PLL power command 0x3 is not working */
+	DSI_QUIRK_DCS_CMD_CONFIG_VC = (1 << 1),
+	DSI_QUIRK_VC_OCP_WIDTH = (1 << 2),
+	DSI_QUIRK_REVERSE_TXCLKESC = (1 << 3),
+	DSI_QUIRK_GNQ = (1 << 4),
+	DSI_QUIRK_PHY_DCC = (1 << 5),
+};
+
+struct dsi_of_data {
+	enum dsi_model model;
+	const struct dss_pll_hw *pll_hw;
+	const struct dsi_module_id_data *modules;
+	unsigned int max_fck_freq;
+	unsigned int max_pll_lpdiv;
+	enum dsi_quirks quirks;
+};
+
 struct dsi_data {
 	struct platform_device *pdev;
 	void __iomem *proto_base;
 	void __iomem *phy_base;
 	void __iomem *pll_base;
 
+	const struct dsi_of_data *data;
 	int module_id;
 
 	int irq;
@@ -312,6 +344,7 @@ struct dsi_data {
 	bool is_enabled;
 
 	struct clk *dss_clk;
+	struct regmap *syscon;
 
 	struct dispc_clock_info user_dispc_cinfo;
 	struct dss_pll_clock_info user_dsi_cinfo;
@@ -397,13 +430,6 @@ struct dsi_packet_sent_handler_data {
 	struct completion *completion;
 };
 
-struct dsi_module_id_data {
-	u32 address;
-	int id;
-};
-
-static const struct of_device_id dsi_of_match[];
-
 #ifdef DSI_PERF_MEASURE
 static bool dsi_perf;
 module_param(dsi_perf, bool, 0644);
@@ -1186,6 +1212,7 @@ static int dsi_regulator_init(struct platform_device *dsidev)
 
 static void _dsi_print_reset_status(struct platform_device *dsidev)
 {
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
 	u32 l;
 	int b0, b1, b2;
 
@@ -1194,7 +1221,7 @@ static void _dsi_print_reset_status(struct platform_device *dsidev)
 	 * I/O. */
 	l = dsi_read_reg(dsidev, DSI_DSIPHY_CFG5);
 
-	if (dss_has_feature(FEAT_DSI_REVERSE_TXCLKESC)) {
+	if (dsi->data->quirks & DSI_QUIRK_REVERSE_TXCLKESC) {
 		b0 = 28;
 		b1 = 27;
 		b2 = 26;
@@ -1297,7 +1324,7 @@ static int dsi_set_lp_clk_divisor(struct platform_device *dsidev)
 	unsigned long dsi_fclk;
 	unsigned lp_clk_div;
 	unsigned long lp_clk;
-	unsigned lpdiv_max = dss_feat_get_param_max(FEAT_PARAM_DSIPLL_LPDIV);
+	unsigned lpdiv_max = dsi->data->max_pll_lpdiv;
 
 
 	lp_clk_div = dsi->user_lp_cinfo.lp_clk_div;
@@ -1349,11 +1376,12 @@ enum dsi_pll_power_state {
 static int dsi_pll_power(struct platform_device *dsidev,
 		enum dsi_pll_power_state state)
 {
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
 	int t = 0;
 
 	/* DSI-PLL power command 0x3 is not working */
-	if (dss_has_feature(FEAT_DSI_PLL_PWR_BUG) &&
-			state == DSI_PLL_POWER_ON_DIV)
+	if ((dsi->data->quirks & DSI_QUIRK_PLL_PWR_BUG) &&
+	    state == DSI_PLL_POWER_ON_DIV)
 		state = DSI_PLL_POWER_ON_ALL;
 
 	/* PLL_PWR_CMD */
@@ -1373,11 +1401,12 @@ static int dsi_pll_power(struct platform_device *dsidev,
 }
 
 
-static void dsi_pll_calc_dsi_fck(struct dss_pll_clock_info *cinfo)
+static void dsi_pll_calc_dsi_fck(struct dsi_data *dsi,
+				 struct dss_pll_clock_info *cinfo)
 {
 	unsigned long max_dsi_fck;
 
-	max_dsi_fck = dss_feat_get_param_max(FEAT_PARAM_DSI_FCK);
+	max_dsi_fck = dsi->data->max_fck_freq;
 
 	cinfo->mX[HSDIV_DSI] = DIV_ROUND_UP(cinfo->clkdco, max_dsi_fck);
 	cinfo->clkout[HSDIV_DSI] = cinfo->clkdco / cinfo->mX[HSDIV_DSI];
@@ -1773,13 +1802,14 @@ static int dsi_cio_power(struct platform_device *dsidev,
 
 static unsigned dsi_get_line_buf_size(struct platform_device *dsidev)
 {
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
 	int val;
 
 	/* line buffer on OMAP3 is 1024 x 24bits */
 	/* XXX: for some reason using full buffer size causes
 	 * considerable TX slowdown with update sizes that fill the
 	 * whole buffer */
-	if (!dss_has_feature(FEAT_DSI_GNQ))
+	if (!(dsi->data->quirks & DSI_QUIRK_GNQ))
 		return 1023 * 3;
 
 	val = REG_GET(dsidev, DSI_GNQ, 14, 12); /* VP1_LINE_BUFFER_SIZE */
@@ -1872,6 +1902,7 @@ static inline unsigned ddr2ns(struct platform_device *dsidev, unsigned ddr)
 
 static void dsi_cio_timings(struct platform_device *dsidev)
 {
+	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
 	u32 r;
 	u32 ths_prepare, ths_prepare_ths_zero, ths_trail, ths_exit;
 	u32 tlpx_half, tclk_trail, tclk_zero;
@@ -1934,7 +1965,7 @@ static void dsi_cio_timings(struct platform_device *dsidev)
 	r = FLD_MOD(r, tclk_trail, 15, 8);
 	r = FLD_MOD(r, tclk_zero, 7, 0);
 
-	if (dss_has_feature(FEAT_DSI_PHY_DCC)) {
+	if (dsi->data->quirks & DSI_QUIRK_PHY_DCC) {
 		r = FLD_MOD(r, 0, 21, 21);	/* DCCEN = disable */
 		r = FLD_MOD(r, 1, 22, 22);	/* CLKINP_DIVBY2EN = enable */
 		r = FLD_MOD(r, 1, 23, 23);	/* CLKINP_SEL = enable */
@@ -2006,7 +2037,7 @@ static int dsi_cio_wait_tx_clk_esc_reset(struct platform_device *dsidev)
 	static const u8 offsets_new[] = { 24, 25, 26, 27, 28 };
 	const u8 *offsets;
 
-	if (dss_has_feature(FEAT_DSI_REVERSE_TXCLKESC))
+	if (dsi->data->quirks & DSI_QUIRK_REVERSE_TXCLKESC)
 		offsets = offsets_old;
 	else
 		offsets = offsets_new;
@@ -2060,6 +2091,83 @@ static unsigned dsi_get_lane_mask(struct platform_device *dsidev)
 	return mask;
 }
 
+/* OMAP4 CONTROL_DSIPHY */
+#define OMAP4_DSIPHY_SYSCON_OFFSET			0x78
+
+#define OMAP4_DSI2_LANEENABLE_SHIFT			29
+#define OMAP4_DSI2_LANEENABLE_MASK			(0x7 << 29)
+#define OMAP4_DSI1_LANEENABLE_SHIFT			24
+#define OMAP4_DSI1_LANEENABLE_MASK			(0x1f << 24)
+#define OMAP4_DSI1_PIPD_SHIFT				19
+#define OMAP4_DSI1_PIPD_MASK				(0x1f << 19)
+#define OMAP4_DSI2_PIPD_SHIFT				14
+#define OMAP4_DSI2_PIPD_MASK				(0x1f << 14)
+
+static int dsi_omap4_mux_pads(struct dsi_data *dsi, unsigned int lanes)
+{
+	u32 enable_mask, enable_shift;
+	u32 pipd_mask, pipd_shift;
+
+	if (dsi->module_id == 0) {
+		enable_mask = OMAP4_DSI1_LANEENABLE_MASK;
+		enable_shift = OMAP4_DSI1_LANEENABLE_SHIFT;
+		pipd_mask = OMAP4_DSI1_PIPD_MASK;
+		pipd_shift = OMAP4_DSI1_PIPD_SHIFT;
+	} else if (dsi->module_id == 1) {
+		enable_mask = OMAP4_DSI2_LANEENABLE_MASK;
+		enable_shift = OMAP4_DSI2_LANEENABLE_SHIFT;
+		pipd_mask = OMAP4_DSI2_PIPD_MASK;
+		pipd_shift = OMAP4_DSI2_PIPD_SHIFT;
+	} else {
+		return -ENODEV;
+	}
+
+	return regmap_update_bits(dsi->syscon, OMAP4_DSIPHY_SYSCON_OFFSET,
+		enable_mask | pipd_mask,
+		(lanes << enable_shift) | (lanes << pipd_shift));
+}
+
+/* OMAP5 CONTROL_DSIPHY */
+
+#define OMAP5_DSIPHY_SYSCON_OFFSET	0x74
+
+#define OMAP5_DSI1_LANEENABLE_SHIFT	24
+#define OMAP5_DSI2_LANEENABLE_SHIFT	19
+#define OMAP5_DSI_LANEENABLE_MASK	0x1f
+
+static int dsi_omap5_mux_pads(struct dsi_data *dsi, unsigned int lanes)
+{
+	u32 enable_shift;
+
+	if (dsi->module_id == 0)
+		enable_shift = OMAP5_DSI1_LANEENABLE_SHIFT;
+	else if (dsi->module_id == 1)
+		enable_shift = OMAP5_DSI2_LANEENABLE_SHIFT;
+	else
+		return -ENODEV;
+
+	return regmap_update_bits(dsi->syscon, OMAP5_DSIPHY_SYSCON_OFFSET,
+		OMAP5_DSI_LANEENABLE_MASK << enable_shift,
+		lanes << enable_shift);
+}
+
+static int dsi_enable_pads(struct dsi_data *dsi, unsigned int lane_mask)
+{
+	if (dsi->data->model == DSI_MODEL_OMAP4)
+		return dsi_omap4_mux_pads(dsi, lane_mask);
+	if (dsi->data->model == DSI_MODEL_OMAP5)
+		return dsi_omap5_mux_pads(dsi, lane_mask);
+	return 0;
+}
+
+static void dsi_disable_pads(struct dsi_data *dsi)
+{
+	if (dsi->data->model == DSI_MODEL_OMAP4)
+		dsi_omap4_mux_pads(dsi, 0);
+	else if (dsi->data->model == DSI_MODEL_OMAP5)
+		dsi_omap5_mux_pads(dsi, 0);
+}
+
 static int dsi_cio_init(struct platform_device *dsidev)
 {
 	struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
@@ -2068,7 +2176,7 @@ static int dsi_cio_init(struct platform_device *dsidev)
 
 	DSSDBG("DSI CIO init starts");
 
-	r = dss_dsi_enable_pads(dsi->module_id, dsi_get_lane_mask(dsidev));
+	r = dsi_enable_pads(dsi, dsi_get_lane_mask(dsidev));
 	if (r)
 		return r;
 
@@ -2178,7 +2286,7 @@ err_cio_pwr:
 		dsi_cio_disable_lane_override(dsidev);
 err_scp_clk_dom:
 	dsi_disable_scp_clk(dsidev);
-	dss_dsi_disable_pads(dsi->module_id, dsi_get_lane_mask(dsidev));
+	dsi_disable_pads(dsi);
 	return r;
 }
 
@@ -2191,7 +2299,7 @@ static void dsi_cio_uninit(struct platform_device *dsidev)
 
 	dsi_cio_power(dsidev, DSI_COMPLEXIO_POWER_OFF);
 	dsi_disable_scp_clk(dsidev);
-	dss_dsi_disable_pads(dsi->module_id, dsi_get_lane_mask(dsidev));
+	dsi_disable_pads(dsi);
 }
 
 static void dsi_config_tx_fifo(struct platform_device *dsidev,
@@ -2439,7 +2547,7 @@ static void dsi_vc_initial_config(struct platform_device *dsidev, int channel)
 	r = FLD_MOD(r, 1, 7, 7); /* CS_TX_EN */
 	r = FLD_MOD(r, 1, 8, 8); /* ECC_TX_EN */
 	r = FLD_MOD(r, 0, 9, 9); /* MODE_SPEED, high speed on/off */
-	if (dss_has_feature(FEAT_DSI_VC_OCP_WIDTH))
+	if (dsi->data->quirks & DSI_QUIRK_VC_OCP_WIDTH)
 		r = FLD_MOD(r, 3, 11, 10);	/* OCP_WIDTH = 32 bit */
 
 	r = FLD_MOD(r, 4, 29, 27); /* DMA_RX_REQ_NB = no dma */
@@ -2474,7 +2582,7 @@ static int dsi_vc_config_source(struct platform_device *dsidev, int channel,
 	REG_FLD_MOD(dsidev, DSI_VC_CTRL(channel), source, 1, 1);
 
 	/* DCS_CMD_ENABLE */
-	if (dss_has_feature(FEAT_DSI_DCS_CMD_CONFIG_VC)) {
+	if (dsi->data->quirks & DSI_QUIRK_DCS_CMD_CONFIG_VC) {
 		bool enable = source == DSI_VC_SOURCE_VP;
 		REG_FLD_MOD(dsidev, DSI_VC_CTRL(channel), enable, 30, 30);
 	}
@@ -3607,7 +3715,7 @@ static int dsi_proto_config(struct platform_device *dsidev)
 	r = FLD_MOD(r, 0, 8, 8);	/* VP_CLK_POL */
 	r = FLD_MOD(r, 1, 14, 14);	/* TRIGGER_RESET_MODE */
 	r = FLD_MOD(r, 1, 19, 19);	/* EOT_ENABLE */
-	if (!dss_has_feature(FEAT_DSI_DCS_CMD_CONFIG_VC)) {
+	if (!(dsi->data->quirks & DSI_QUIRK_DCS_CMD_CONFIG_VC)) {
 		r = FLD_MOD(r, 1, 24, 24);	/* DCS_CMD_ENABLE */
 		/* DCS_CMD_CODE, 1=start, 0=continue */
 		r = FLD_MOD(r, 0, 25, 25);
@@ -4450,6 +4558,7 @@ static bool dsi_cm_calc_pll_cb(int n, int m, unsigned long fint,
 		unsigned long clkdco, void *data)
 {
 	struct dsi_clk_calc_ctx *ctx = data;
+	struct dsi_data *dsi = dsi_get_dsidrv_data(ctx->dsidev);
 
 	ctx->dsi_cinfo.n = n;
 	ctx->dsi_cinfo.m = m;
@@ -4457,7 +4566,7 @@ static bool dsi_cm_calc_pll_cb(int n, int m, unsigned long fint,
 	ctx->dsi_cinfo.clkdco = clkdco;
 
 	return dss_pll_hsdiv_calc_a(ctx->pll, clkdco, ctx->req_pck_min,
-			dss_feat_get_param_max(FEAT_PARAM_DSS_FCK),
+			dsi->data->max_fck_freq,
 			dsi_cm_calc_hsdiv_cb, ctx);
 }
 
@@ -4749,6 +4858,7 @@ static bool dsi_vm_calc_pll_cb(int n, int m, unsigned long fint,
 		unsigned long clkdco, void *data)
 {
 	struct dsi_clk_calc_ctx *ctx = data;
+	struct dsi_data *dsi = dsi_get_dsidrv_data(ctx->dsidev);
 
 	ctx->dsi_cinfo.n = n;
 	ctx->dsi_cinfo.m = m;
@@ -4756,7 +4866,7 @@ static bool dsi_vm_calc_pll_cb(int n, int m, unsigned long fint,
 	ctx->dsi_cinfo.clkdco = clkdco;
 
 	return dss_pll_hsdiv_calc_a(ctx->pll, clkdco, ctx->req_pck_min,
-			dss_feat_get_param_max(FEAT_PARAM_DSS_FCK),
+			dsi->data->max_fck_freq,
 			dsi_vm_calc_hsdiv_cb, ctx);
 }
 
@@ -4827,7 +4937,7 @@ static int dsi_set_config(struct omap_dss_device *dssdev,
 		goto err;
 	}
 
-	dsi_pll_calc_dsi_fck(&ctx.dsi_cinfo);
+	dsi_pll_calc_dsi_fck(dsi, &ctx.dsi_cinfo);
 
 	r = dsi_lp_clock_calc(ctx.dsi_cinfo.clkout[HSDIV_DSI],
 		config->lp_clk_min, config->lp_clk_max, &dsi->user_lp_cinfo);
@@ -4857,24 +4967,14 @@ err:
  * the channel in some more dynamic manner, or get the channel as a user
  * parameter.
  */
-static enum omap_channel dsi_get_channel(int module_id)
+static enum omap_channel dsi_get_channel(struct dsi_data *dsi)
 {
-	switch (omapdss_get_version()) {
-	case OMAPDSS_VER_OMAP24xx:
-	case OMAPDSS_VER_AM43xx:
-		DSSWARN("DSI not supported\n");
+	switch (dsi->data->model) {
+	case DSI_MODEL_OMAP3:
 		return OMAP_DSS_CHANNEL_LCD;
 
-	case OMAPDSS_VER_OMAP34xx_ES1:
-	case OMAPDSS_VER_OMAP34xx_ES3:
-	case OMAPDSS_VER_OMAP3630:
-	case OMAPDSS_VER_AM35xx:
-		return OMAP_DSS_CHANNEL_LCD;
-
-	case OMAPDSS_VER_OMAP4430_ES1:
-	case OMAPDSS_VER_OMAP4430_ES2:
-	case OMAPDSS_VER_OMAP4:
-		switch (module_id) {
+	case DSI_MODEL_OMAP4:
+		switch (dsi->module_id) {
 		case 0:
 			return OMAP_DSS_CHANNEL_LCD;
 		case 1:
@@ -4884,8 +4984,8 @@ static enum omap_channel dsi_get_channel(int module_id)
 			return OMAP_DSS_CHANNEL_LCD;
 		}
 
-	case OMAPDSS_VER_OMAP5:
-		switch (module_id) {
+	case DSI_MODEL_OMAP5:
+		switch (dsi->module_id) {
 		case 0:
 			return OMAP_DSS_CHANNEL_LCD;
 		case 1:
@@ -5065,7 +5165,7 @@ static void dsi_init_output(struct platform_device *dsidev)
 
 	out->output_type = OMAP_DISPLAY_TYPE_DSI;
 	out->name = dsi->module_id == 0 ? "dsi.0" : "dsi.1";
-	out->dispc_channel = dsi_get_channel(dsi->module_id);
+	out->dispc_channel = dsi_get_channel(dsi);
 	out->ops.dsi = &dsi_ops;
 	out->owner = THIS_MODULE;
 
@@ -5240,29 +5340,7 @@ static int dsi_init_pll_data(struct platform_device *dsidev)
 	pll->id = dsi->module_id == 0 ? DSS_PLL_DSI1 : DSS_PLL_DSI2;
 	pll->clkin = clk;
 	pll->base = dsi->pll_base;
-
-	switch (omapdss_get_version()) {
-	case OMAPDSS_VER_OMAP34xx_ES1:
-	case OMAPDSS_VER_OMAP34xx_ES3:
-	case OMAPDSS_VER_OMAP3630:
-	case OMAPDSS_VER_AM35xx:
-		pll->hw = &dss_omap3_dsi_pll_hw;
-		break;
-
-	case OMAPDSS_VER_OMAP4430_ES1:
-	case OMAPDSS_VER_OMAP4430_ES2:
-	case OMAPDSS_VER_OMAP4:
-		pll->hw = &dss_omap4_dsi_pll_hw;
-		break;
-
-	case OMAPDSS_VER_OMAP5:
-		pll->hw = &dss_omap5_dsi_pll_hw;
-		break;
-
-	default:
-		return -ENODEV;
-	}
-
+	pll->hw = dsi->data->pll_hw;
 	pll->ops = &dsi_pll_ops;
 
 	r = dss_pll_register(pll);
@@ -5273,9 +5351,74 @@ static int dsi_init_pll_data(struct platform_device *dsidev)
 }
 
 /* DSI1 HW IP initialisation */
+static const struct dsi_of_data dsi_of_data_omap34xx = {
+	.model = DSI_MODEL_OMAP3,
+	.pll_hw = &dss_omap3_dsi_pll_hw,
+	.modules = (const struct dsi_module_id_data[]) {
+		{ .address = 0x4804fc00, .id = 0, },
+		{ },
+	},
+	.max_fck_freq = 173000000,
+	.max_pll_lpdiv = (1 << 13) - 1,
+	.quirks = DSI_QUIRK_REVERSE_TXCLKESC,
+};
+
+static const struct dsi_of_data dsi_of_data_omap36xx = {
+	.model = DSI_MODEL_OMAP3,
+	.pll_hw = &dss_omap3_dsi_pll_hw,
+	.modules = (const struct dsi_module_id_data[]) {
+		{ .address = 0x4804fc00, .id = 0, },
+		{ },
+	},
+	.max_fck_freq = 173000000,
+	.max_pll_lpdiv = (1 << 13) - 1,
+	.quirks = DSI_QUIRK_PLL_PWR_BUG,
+};
+
+static const struct dsi_of_data dsi_of_data_omap4 = {
+	.model = DSI_MODEL_OMAP4,
+	.pll_hw = &dss_omap4_dsi_pll_hw,
+	.modules = (const struct dsi_module_id_data[]) {
+		{ .address = 0x58004000, .id = 0, },
+		{ .address = 0x58005000, .id = 1, },
+		{ },
+	},
+	.max_fck_freq = 170000000,
+	.max_pll_lpdiv = (1 << 13) - 1,
+	.quirks = DSI_QUIRK_DCS_CMD_CONFIG_VC | DSI_QUIRK_VC_OCP_WIDTH
+		| DSI_QUIRK_GNQ,
+};
+
+static const struct dsi_of_data dsi_of_data_omap5 = {
+	.model = DSI_MODEL_OMAP5,
+	.pll_hw = &dss_omap5_dsi_pll_hw,
+	.modules = (const struct dsi_module_id_data[]) {
+		{ .address = 0x58004000, .id = 0, },
+		{ .address = 0x58009000, .id = 1, },
+		{ },
+	},
+	.max_fck_freq = 209250000,
+	.max_pll_lpdiv = (1 << 13) - 1,
+	.quirks = DSI_QUIRK_DCS_CMD_CONFIG_VC | DSI_QUIRK_VC_OCP_WIDTH
+		| DSI_QUIRK_GNQ | DSI_QUIRK_PHY_DCC,
+};
+
+static const struct of_device_id dsi_of_match[] = {
+	{ .compatible = "ti,omap3-dsi", .data = &dsi_of_data_omap36xx, },
+	{ .compatible = "ti,omap4-dsi", .data = &dsi_of_data_omap4, },
+	{ .compatible = "ti,omap5-dsi", .data = &dsi_of_data_omap5, },
+	{},
+};
+
+static const struct soc_device_attribute dsi_soc_devices[] = {
+	{ .machine = "OMAP3[45]*",	.data = &dsi_of_data_omap34xx },
+	{ .machine = "AM35*",		.data = &dsi_of_data_omap34xx },
+	{ /* sentinel */ }
+};
 static int dsi_bind(struct device *dev, struct device *master, void *data)
 {
 	struct platform_device *dsidev = to_platform_device(dev);
+	const struct soc_device_attribute *soc;
 	const struct dsi_module_id_data *d;
 	u32 rev;
 	int r, i;
@@ -5339,7 +5482,13 @@ static int dsi_bind(struct device *dev, struct device *master, void *data)
 		return r;
 	}
 
-	d = of_match_node(dsi_of_match, dsidev->dev.of_node)->data;
+	soc = soc_device_match(dsi_soc_devices);
+	if (soc)
+		dsi->data = soc->data;
+	else
+		dsi->data = of_match_node(dsi_of_match, dev->of_node)->data;
+
+	d = dsi->data->modules;
 	while (d->address != 0 && d->address != dsi_mem->start)
 		d++;
 
@@ -5350,6 +5499,24 @@ static int dsi_bind(struct device *dev, struct device *master, void *data)
 
 	dsi->module_id = d->id;
 
+	if (dsi->data->model == DSI_MODEL_OMAP4 ||
+	    dsi->data->model == DSI_MODEL_OMAP5) {
+		struct device_node *np;
+
+		/*
+		 * The OMAP4/5 display DT bindings don't reference the padconf
+		 * syscon. Our only option to retrieve it is to find it by name.
+		 */
+		np = of_find_node_by_name(NULL,
+			dsi->data->model == DSI_MODEL_OMAP4 ?
+			"omap4_padconf_global" : "omap5_padconf_global");
+		if (!np)
+			return -ENODEV;
+
+		dsi->syscon = syscon_node_to_regmap(np);
+		of_node_put(np);
+	}
+
 	/* DSI VCs initialization */
 	for (i = 0; i < ARRAY_SIZE(dsi->vc); i++) {
 		dsi->vc[i].source = DSI_VC_SOURCE_L4;
@@ -5375,7 +5542,7 @@ static int dsi_bind(struct device *dev, struct device *master, void *data)
 
 	/* DSI on OMAP3 doesn't have register DSI_GNQ, set number
 	 * of data to 3 by default */
-	if (dss_has_feature(FEAT_DSI_GNQ))
+	if (dsi->data->quirks & DSI_QUIRK_GNQ)
 		/* NB_DATA_LANES */
 		dsi->num_lanes_supported = 1 + REG_GET(dsidev, DSI_GNQ, 11, 9);
 	else
@@ -5495,30 +5662,6 @@ static const struct dev_pm_ops dsi_pm_ops = {
 	.runtime_resume = dsi_runtime_resume,
 };
 
-static const struct dsi_module_id_data dsi_of_data_omap3[] = {
-	{ .address = 0x4804fc00, .id = 0, },
-	{ },
-};
-
-static const struct dsi_module_id_data dsi_of_data_omap4[] = {
-	{ .address = 0x58004000, .id = 0, },
-	{ .address = 0x58005000, .id = 1, },
-	{ },
-};
-
-static const struct dsi_module_id_data dsi_of_data_omap5[] = {
-	{ .address = 0x58004000, .id = 0, },
-	{ .address = 0x58009000, .id = 1, },
-	{ },
-};
-
-static const struct of_device_id dsi_of_match[] = {
-	{ .compatible = "ti,omap3-dsi", .data = dsi_of_data_omap3, },
-	{ .compatible = "ti,omap4-dsi", .data = dsi_of_data_omap4, },
-	{ .compatible = "ti,omap5-dsi", .data = dsi_of_data_omap5, },
-	{},
-};
-
 static struct platform_driver omap_dsihw_driver = {
 	.probe		= dsi_probe,
 	.remove		= dsi_remove,
diff --git a/drivers/gpu/drm/omapdrm/dss/dss.c b/drivers/gpu/drm/omapdrm/dss/dss.c
index 99e22ca972c7..d1755f12236b 100644
--- a/drivers/gpu/drm/omapdrm/dss/dss.c
+++ b/drivers/gpu/drm/omapdrm/dss/dss.c
@@ -22,6 +22,7 @@
 
 #define DSS_SUBSYS_NAME "DSS"
 
+#include <linux/debugfs.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/io.h>
@@ -38,14 +39,15 @@
 #include <linux/mfd/syscon.h>
 #include <linux/regmap.h>
 #include <linux/of.h>
+#include <linux/of_device.h>
 #include <linux/of_graph.h>
 #include <linux/regulator/consumer.h>
 #include <linux/suspend.h>
 #include <linux/component.h>
+#include <linux/sys_soc.h>
 
 #include "omapdss.h"
 #include "dss.h"
-#include "dss_features.h"
 
 #define DSS_SZ_REGS			SZ_512
 
@@ -69,15 +71,24 @@ struct dss_reg {
 #define REG_FLD_MOD(idx, val, start, end) \
 	dss_write_reg(idx, FLD_MOD(dss_read_reg(idx), val, start, end))
 
+struct dss_ops {
+	int (*dpi_select_source)(int port, enum omap_channel channel);
+	int (*select_lcd_source)(enum omap_channel channel,
+		enum dss_clk_source clk_src);
+};
+
 struct dss_features {
+	enum dss_model model;
 	u8 fck_div_max;
+	unsigned int fck_freq_max;
 	u8 dss_fck_multiplier;
 	const char *parent_clk_name;
 	const enum omap_display_type *ports;
 	int num_ports;
-	int (*dpi_select_source)(int port, enum omap_channel channel);
-	int (*select_lcd_source)(enum omap_channel channel,
-		enum dss_clk_source clk_src);
+	const enum omap_dss_output_id *outputs;
+	const struct dss_ops *ops;
+	struct dss_reg_field dispc_clk_switch;
+	bool has_lcd_clk_src;
 };
 
 static struct {
@@ -139,8 +150,7 @@ static void dss_save_context(void)
 
 	SR(CONTROL);
 
-	if (dss_feat_get_supported_displays(OMAP_DSS_CHANNEL_LCD) &
-			OMAP_DISPLAY_TYPE_SDI) {
+	if (dss.feat->outputs[OMAP_DSS_CHANNEL_LCD] & OMAP_DSS_OUTPUT_SDI) {
 		SR(SDI_CONTROL);
 		SR(PLL_CONTROL);
 	}
@@ -159,8 +169,7 @@ static void dss_restore_context(void)
 
 	RR(CONTROL);
 
-	if (dss_feat_get_supported_displays(OMAP_DSS_CHANNEL_LCD) &
-			OMAP_DISPLAY_TYPE_SDI) {
+	if (dss.feat->outputs[OMAP_DSS_CHANNEL_LCD] & OMAP_DSS_OUTPUT_SDI) {
 		RR(SDI_CONTROL);
 		RR(PLL_CONTROL);
 	}
@@ -390,8 +399,7 @@ static void dss_dump_regs(struct seq_file *s)
 	DUMPREG(DSS_SYSSTATUS);
 	DUMPREG(DSS_CONTROL);
 
-	if (dss_feat_get_supported_displays(OMAP_DSS_CHANNEL_LCD) &
-			OMAP_DISPLAY_TYPE_SDI) {
+	if (dss.feat->outputs[OMAP_DSS_CHANNEL_LCD] & OMAP_DSS_OUTPUT_SDI) {
 		DUMPREG(DSS_SDI_CONTROL);
 		DUMPREG(DSS_PLL_CONTROL);
 		DUMPREG(DSS_SDI_STATUS);
@@ -419,14 +427,12 @@ static int dss_get_channel_index(enum omap_channel channel)
 static void dss_select_dispc_clk_source(enum dss_clk_source clk_src)
 {
 	int b;
-	u8 start, end;
 
 	/*
 	 * We always use PRCM clock as the DISPC func clock, except on DSS3,
 	 * where we don't have separate DISPC and LCD clock sources.
 	 */
-	if (WARN_ON(dss_has_feature(FEAT_LCD_CLK_SRC) &&
-		clk_src != DSS_CLK_SRC_FCK))
+	if (WARN_ON(dss.feat->has_lcd_clk_src && clk_src != DSS_CLK_SRC_FCK))
 		return;
 
 	switch (clk_src) {
@@ -444,9 +450,9 @@ static void dss_select_dispc_clk_source(enum dss_clk_source clk_src)
 		return;
 	}
 
-	dss_feat_get_reg_field(FEAT_REG_DISPC_CLK_SWITCH, &start, &end);
-
-	REG_FLD_MOD(DSS_CONTROL, b, start, end);	/* DISPC_CLK_SWITCH */
+	REG_FLD_MOD(DSS_CONTROL, b,			/* DISPC_CLK_SWITCH */
+		    dss.feat->dispc_clk_switch.start,
+		    dss.feat->dispc_clk_switch.end);
 
 	dss.dispc_clk_source = clk_src;
 }
@@ -570,13 +576,13 @@ void dss_select_lcd_clk_source(enum omap_channel channel,
 	int idx = dss_get_channel_index(channel);
 	int r;
 
-	if (!dss_has_feature(FEAT_LCD_CLK_SRC)) {
+	if (!dss.feat->has_lcd_clk_src) {
 		dss_select_dispc_clk_source(clk_src);
 		dss.lcd_clk_source[idx] = clk_src;
 		return;
 	}
 
-	r = dss.feat->select_lcd_source(channel, clk_src);
+	r = dss.feat->ops->select_lcd_source(channel, clk_src);
 	if (r)
 		return;
 
@@ -595,7 +601,7 @@ enum dss_clk_source dss_get_dsi_clk_source(int dsi_module)
 
 enum dss_clk_source dss_get_lcd_clk_source(enum omap_channel channel)
 {
-	if (dss_has_feature(FEAT_LCD_CLK_SRC)) {
+	if (dss.feat->has_lcd_clk_src) {
 		int idx = dss_get_channel_index(channel);
 		return dss.lcd_clk_source[idx];
 	} else {
@@ -615,7 +621,7 @@ bool dss_div_calc(unsigned long pck, unsigned long fck_min,
 	unsigned long prate;
 	unsigned m;
 
-	fck_hw_max = dss_feat_get_param_max(FEAT_PARAM_DSS_FCK);
+	fck_hw_max = dss.feat->fck_freq_max;
 
 	if (dss.parent_clk == NULL) {
 		unsigned pckd;
@@ -673,6 +679,16 @@ unsigned long dss_get_dispc_clk_rate(void)
 	return dss.dss_clk_rate;
 }
 
+unsigned long dss_get_max_fck_rate(void)
+{
+	return dss.feat->fck_freq_max;
+}
+
+enum omap_dss_output_id dss_get_supported_outputs(enum omap_channel channel)
+{
+	return dss.feat->outputs[channel];
+}
+
 static int dss_setup_default_clock(void)
 {
 	unsigned long max_dss_fck, prate;
@@ -680,7 +696,7 @@ static int dss_setup_default_clock(void)
 	unsigned fck_div;
 	int r;
 
-	max_dss_fck = dss_feat_get_param_max(FEAT_PARAM_DSS_FCK);
+	max_dss_fck = dss.feat->fck_freq_max;
 
 	if (dss.parent_clk == NULL) {
 		fck = clk_round_rate(dss.dss_clk, max_dss_fck);
@@ -721,27 +737,29 @@ void dss_set_dac_pwrdn_bgz(bool enable)
 
 void dss_select_hdmi_venc_clk_source(enum dss_hdmi_venc_clk_source_select src)
 {
-	enum omap_display_type dp;
-	dp = dss_feat_get_supported_displays(OMAP_DSS_CHANNEL_DIGIT);
+	enum omap_dss_output_id outputs;
+
+	outputs = dss.feat->outputs[OMAP_DSS_CHANNEL_DIGIT];
 
 	/* Complain about invalid selections */
-	WARN_ON((src == DSS_VENC_TV_CLK) && !(dp & OMAP_DISPLAY_TYPE_VENC));
-	WARN_ON((src == DSS_HDMI_M_PCLK) && !(dp & OMAP_DISPLAY_TYPE_HDMI));
+	WARN_ON((src == DSS_VENC_TV_CLK) && !(outputs & OMAP_DSS_OUTPUT_VENC));
+	WARN_ON((src == DSS_HDMI_M_PCLK) && !(outputs & OMAP_DSS_OUTPUT_HDMI));
 
 	/* Select only if we have options */
-	if ((dp & OMAP_DISPLAY_TYPE_VENC) && (dp & OMAP_DISPLAY_TYPE_HDMI))
+	if ((outputs & OMAP_DSS_OUTPUT_VENC) &&
+	    (outputs & OMAP_DSS_OUTPUT_HDMI))
 		REG_FLD_MOD(DSS_CONTROL, src, 15, 15);	/* VENC_HDMI_SWITCH */
 }
 
 enum dss_hdmi_venc_clk_source_select dss_get_hdmi_venc_clk_source(void)
 {
-	enum omap_display_type displays;
+	enum omap_dss_output_id outputs;
 
-	displays = dss_feat_get_supported_displays(OMAP_DSS_CHANNEL_DIGIT);
-	if ((displays & OMAP_DISPLAY_TYPE_HDMI) == 0)
+	outputs = dss.feat->outputs[OMAP_DSS_CHANNEL_DIGIT];
+	if ((outputs & OMAP_DSS_OUTPUT_HDMI) == 0)
 		return DSS_VENC_TV_CLK;
 
-	if ((displays & OMAP_DISPLAY_TYPE_VENC) == 0)
+	if ((outputs & OMAP_DSS_OUTPUT_VENC) == 0)
 		return DSS_HDMI_M_PCLK;
 
 	return REG_GET(DSS_CONTROL, 15, 15);
@@ -823,7 +841,7 @@ static int dss_dpi_select_source_dra7xx(int port, enum omap_channel channel)
 
 int dss_dpi_select_source(int port, enum omap_channel channel)
 {
-	return dss.feat->dpi_select_source(port, channel);
+	return dss.feat->ops->dpi_select_source(port, channel);
 }
 
 static int dss_get_clocks(void)
@@ -882,7 +900,7 @@ void dss_runtime_put(void)
 
 /* DEBUGFS */
 #if defined(CONFIG_OMAP2_DSS_DEBUGFS)
-void dss_debug_dump_clocks(struct seq_file *s)
+static void dss_debug_dump_clocks(struct seq_file *s)
 {
 	dss_dump_clocks(s);
 	dispc_dump_clocks(s);
@@ -890,8 +908,88 @@ void dss_debug_dump_clocks(struct seq_file *s)
 	dsi_dump_clocks(s);
 #endif
 }
-#endif
 
+static int dss_debug_show(struct seq_file *s, void *unused)
+{
+	void (*func)(struct seq_file *) = s->private;
+
+	func(s);
+	return 0;
+}
+
+static int dss_debug_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, dss_debug_show, inode->i_private);
+}
+
+static const struct file_operations dss_debug_fops = {
+	.open           = dss_debug_open,
+	.read           = seq_read,
+	.llseek         = seq_lseek,
+	.release        = single_release,
+};
+
+static struct dentry *dss_debugfs_dir;
+
+static int dss_initialize_debugfs(void)
+{
+	dss_debugfs_dir = debugfs_create_dir("omapdss", NULL);
+	if (IS_ERR(dss_debugfs_dir)) {
+		int err = PTR_ERR(dss_debugfs_dir);
+
+		dss_debugfs_dir = NULL;
+		return err;
+	}
+
+	debugfs_create_file("clk", S_IRUGO, dss_debugfs_dir,
+			&dss_debug_dump_clocks, &dss_debug_fops);
+
+	return 0;
+}
+
+static void dss_uninitialize_debugfs(void)
+{
+	if (dss_debugfs_dir)
+		debugfs_remove_recursive(dss_debugfs_dir);
+}
+
+int dss_debugfs_create_file(const char *name, void (*write)(struct seq_file *))
+{
+	struct dentry *d;
+
+	d = debugfs_create_file(name, S_IRUGO, dss_debugfs_dir,
+			write, &dss_debug_fops);
+
+	return PTR_ERR_OR_ZERO(d);
+}
+#else /* CONFIG_OMAP2_DSS_DEBUGFS */
+static inline int dss_initialize_debugfs(void)
+{
+	return 0;
+}
+static inline void dss_uninitialize_debugfs(void)
+{
+}
+#endif /* CONFIG_OMAP2_DSS_DEBUGFS */
+
+static const struct dss_ops dss_ops_omap2_omap3 = {
+	.dpi_select_source = &dss_dpi_select_source_omap2_omap3,
+};
+
+static const struct dss_ops dss_ops_omap4 = {
+	.dpi_select_source = &dss_dpi_select_source_omap4,
+	.select_lcd_source = &dss_lcd_clk_mux_omap4,
+};
+
+static const struct dss_ops dss_ops_omap5 = {
+	.dpi_select_source = &dss_dpi_select_source_omap5,
+	.select_lcd_source = &dss_lcd_clk_mux_omap5,
+};
+
+static const struct dss_ops dss_ops_dra7 = {
+	.dpi_select_source = &dss_dpi_select_source_dra7xx,
+	.select_lcd_source = &dss_lcd_clk_mux_dra7,
+};
 
 static const enum omap_display_type omap2plus_ports[] = {
 	OMAP_DISPLAY_TYPE_DPI,
@@ -908,130 +1006,168 @@ static const enum omap_display_type dra7xx_ports[] = {
 	OMAP_DISPLAY_TYPE_DPI,
 };
 
+static const enum omap_dss_output_id omap2_dss_supported_outputs[] = {
+	/* OMAP_DSS_CHANNEL_LCD */
+	OMAP_DSS_OUTPUT_DPI | OMAP_DSS_OUTPUT_DBI,
+
+	/* OMAP_DSS_CHANNEL_DIGIT */
+	OMAP_DSS_OUTPUT_VENC,
+};
+
+static const enum omap_dss_output_id omap3430_dss_supported_outputs[] = {
+	/* OMAP_DSS_CHANNEL_LCD */
+	OMAP_DSS_OUTPUT_DPI | OMAP_DSS_OUTPUT_DBI |
+	OMAP_DSS_OUTPUT_SDI | OMAP_DSS_OUTPUT_DSI1,
+
+	/* OMAP_DSS_CHANNEL_DIGIT */
+	OMAP_DSS_OUTPUT_VENC,
+};
+
+static const enum omap_dss_output_id omap3630_dss_supported_outputs[] = {
+	/* OMAP_DSS_CHANNEL_LCD */
+	OMAP_DSS_OUTPUT_DPI | OMAP_DSS_OUTPUT_DBI |
+	OMAP_DSS_OUTPUT_DSI1,
+
+	/* OMAP_DSS_CHANNEL_DIGIT */
+	OMAP_DSS_OUTPUT_VENC,
+};
+
+static const enum omap_dss_output_id am43xx_dss_supported_outputs[] = {
+	/* OMAP_DSS_CHANNEL_LCD */
+	OMAP_DSS_OUTPUT_DPI | OMAP_DSS_OUTPUT_DBI,
+};
+
+static const enum omap_dss_output_id omap4_dss_supported_outputs[] = {
+	/* OMAP_DSS_CHANNEL_LCD */
+	OMAP_DSS_OUTPUT_DBI | OMAP_DSS_OUTPUT_DSI1,
+
+	/* OMAP_DSS_CHANNEL_DIGIT */
+	OMAP_DSS_OUTPUT_VENC | OMAP_DSS_OUTPUT_HDMI,
+
+	/* OMAP_DSS_CHANNEL_LCD2 */
+	OMAP_DSS_OUTPUT_DPI | OMAP_DSS_OUTPUT_DBI |
+	OMAP_DSS_OUTPUT_DSI2,
+};
+
+static const enum omap_dss_output_id omap5_dss_supported_outputs[] = {
+	/* OMAP_DSS_CHANNEL_LCD */
+	OMAP_DSS_OUTPUT_DPI | OMAP_DSS_OUTPUT_DBI |
+	OMAP_DSS_OUTPUT_DSI1 | OMAP_DSS_OUTPUT_DSI2,
+
+	/* OMAP_DSS_CHANNEL_DIGIT */
+	OMAP_DSS_OUTPUT_HDMI,
+
+	/* OMAP_DSS_CHANNEL_LCD2 */
+	OMAP_DSS_OUTPUT_DPI | OMAP_DSS_OUTPUT_DBI |
+	OMAP_DSS_OUTPUT_DSI1,
+
+	/* OMAP_DSS_CHANNEL_LCD3 */
+	OMAP_DSS_OUTPUT_DPI | OMAP_DSS_OUTPUT_DBI |
+	OMAP_DSS_OUTPUT_DSI2,
+};
+
 static const struct dss_features omap24xx_dss_feats = {
+	.model			=	DSS_MODEL_OMAP2,
 	/*
 	 * fck div max is really 16, but the divider range has gaps. The range
 	 * from 1 to 6 has no gaps, so let's use that as a max.
 	 */
 	.fck_div_max		=	6,
+	.fck_freq_max		=	133000000,
 	.dss_fck_multiplier	=	2,
 	.parent_clk_name	=	"core_ck",
-	.dpi_select_source	=	&dss_dpi_select_source_omap2_omap3,
 	.ports			=	omap2plus_ports,
 	.num_ports		=	ARRAY_SIZE(omap2plus_ports),
+	.outputs		=	omap2_dss_supported_outputs,
+	.ops			=	&dss_ops_omap2_omap3,
+	.dispc_clk_switch	=	{ 0, 0 },
+	.has_lcd_clk_src	=	false,
 };
 
 static const struct dss_features omap34xx_dss_feats = {
+	.model			=	DSS_MODEL_OMAP3,
 	.fck_div_max		=	16,
+	.fck_freq_max		=	173000000,
 	.dss_fck_multiplier	=	2,
 	.parent_clk_name	=	"dpll4_ck",
-	.dpi_select_source	=	&dss_dpi_select_source_omap2_omap3,
 	.ports			=	omap34xx_ports,
+	.outputs		=	omap3430_dss_supported_outputs,
 	.num_ports		=	ARRAY_SIZE(omap34xx_ports),
+	.ops			=	&dss_ops_omap2_omap3,
+	.dispc_clk_switch	=	{ 0, 0 },
+	.has_lcd_clk_src	=	false,
 };
 
 static const struct dss_features omap3630_dss_feats = {
+	.model			=	DSS_MODEL_OMAP3,
 	.fck_div_max		=	32,
+	.fck_freq_max		=	173000000,
 	.dss_fck_multiplier	=	1,
 	.parent_clk_name	=	"dpll4_ck",
-	.dpi_select_source	=	&dss_dpi_select_source_omap2_omap3,
 	.ports			=	omap2plus_ports,
 	.num_ports		=	ARRAY_SIZE(omap2plus_ports),
+	.outputs		=	omap3630_dss_supported_outputs,
+	.ops			=	&dss_ops_omap2_omap3,
+	.dispc_clk_switch	=	{ 0, 0 },
+	.has_lcd_clk_src	=	false,
 };
 
 static const struct dss_features omap44xx_dss_feats = {
+	.model			=	DSS_MODEL_OMAP4,
 	.fck_div_max		=	32,
+	.fck_freq_max		=	186000000,
 	.dss_fck_multiplier	=	1,
 	.parent_clk_name	=	"dpll_per_x2_ck",
-	.dpi_select_source	=	&dss_dpi_select_source_omap4,
 	.ports			=	omap2plus_ports,
 	.num_ports		=	ARRAY_SIZE(omap2plus_ports),
-	.select_lcd_source	=	&dss_lcd_clk_mux_omap4,
+	.outputs		=	omap4_dss_supported_outputs,
+	.ops			=	&dss_ops_omap4,
+	.dispc_clk_switch	=	{ 9, 8 },
+	.has_lcd_clk_src	=	true,
 };
 
 static const struct dss_features omap54xx_dss_feats = {
+	.model			=	DSS_MODEL_OMAP5,
 	.fck_div_max		=	64,
+	.fck_freq_max		=	209250000,
 	.dss_fck_multiplier	=	1,
 	.parent_clk_name	=	"dpll_per_x2_ck",
-	.dpi_select_source	=	&dss_dpi_select_source_omap5,
 	.ports			=	omap2plus_ports,
 	.num_ports		=	ARRAY_SIZE(omap2plus_ports),
-	.select_lcd_source	=	&dss_lcd_clk_mux_omap5,
+	.outputs		=	omap5_dss_supported_outputs,
+	.ops			=	&dss_ops_omap5,
+	.dispc_clk_switch	=	{ 9, 7 },
+	.has_lcd_clk_src	=	true,
 };
 
 static const struct dss_features am43xx_dss_feats = {
+	.model			=	DSS_MODEL_OMAP3,
 	.fck_div_max		=	0,
+	.fck_freq_max		=	200000000,
 	.dss_fck_multiplier	=	0,
 	.parent_clk_name	=	NULL,
-	.dpi_select_source	=	&dss_dpi_select_source_omap2_omap3,
 	.ports			=	omap2plus_ports,
 	.num_ports		=	ARRAY_SIZE(omap2plus_ports),
+	.outputs		=	am43xx_dss_supported_outputs,
+	.ops			=	&dss_ops_omap2_omap3,
+	.dispc_clk_switch	=	{ 0, 0 },
+	.has_lcd_clk_src	=	true,
 };
 
 static const struct dss_features dra7xx_dss_feats = {
+	.model			=	DSS_MODEL_DRA7,
 	.fck_div_max		=	64,
+	.fck_freq_max		=	209250000,
 	.dss_fck_multiplier	=	1,
 	.parent_clk_name	=	"dpll_per_x2_ck",
-	.dpi_select_source	=	&dss_dpi_select_source_dra7xx,
 	.ports			=	dra7xx_ports,
 	.num_ports		=	ARRAY_SIZE(dra7xx_ports),
-	.select_lcd_source	=	&dss_lcd_clk_mux_dra7,
+	.outputs		=	omap5_dss_supported_outputs,
+	.ops			=	&dss_ops_dra7,
+	.dispc_clk_switch	=	{ 9, 7 },
+	.has_lcd_clk_src	=	true,
 };
 
-static int dss_init_features(struct platform_device *pdev)
-{
-	const struct dss_features *src;
-	struct dss_features *dst;
-
-	dst = devm_kzalloc(&pdev->dev, sizeof(*dst), GFP_KERNEL);
-	if (!dst) {
-		dev_err(&pdev->dev, "Failed to allocate local DSS Features\n");
-		return -ENOMEM;
-	}
-
-	switch (omapdss_get_version()) {
-	case OMAPDSS_VER_OMAP24xx:
-		src = &omap24xx_dss_feats;
-		break;
-
-	case OMAPDSS_VER_OMAP34xx_ES1:
-	case OMAPDSS_VER_OMAP34xx_ES3:
-	case OMAPDSS_VER_AM35xx:
-		src = &omap34xx_dss_feats;
-		break;
-
-	case OMAPDSS_VER_OMAP3630:
-		src = &omap3630_dss_feats;
-		break;
-
-	case OMAPDSS_VER_OMAP4430_ES1:
-	case OMAPDSS_VER_OMAP4430_ES2:
-	case OMAPDSS_VER_OMAP4:
-		src = &omap44xx_dss_feats;
-		break;
-
-	case OMAPDSS_VER_OMAP5:
-		src = &omap54xx_dss_feats;
-		break;
-
-	case OMAPDSS_VER_AM43xx:
-		src = &am43xx_dss_feats;
-		break;
-
-	case OMAPDSS_VER_DRA7xx:
-		src = &dra7xx_dss_feats;
-		break;
-
-	default:
-		return -ENODEV;
-	}
-
-	memcpy(dst, src, sizeof(*dst));
-	dss.feat = dst;
-
-	return 0;
-}
-
 static int dss_init_ports(struct platform_device *pdev)
 {
 	struct device_node *parent = pdev->dev.of_node;
@@ -1045,7 +1181,7 @@ static int dss_init_ports(struct platform_device *pdev)
 
 		switch (dss.feat->ports[i]) {
 		case OMAP_DISPLAY_TYPE_DPI:
-			dpi_init_port(pdev, port);
+			dpi_init_port(pdev, port, dss.feat->model);
 			break;
 		case OMAP_DISPLAY_TYPE_SDI:
 			sdi_init_port(pdev, port);
@@ -1144,6 +1280,23 @@ static int dss_video_pll_probe(struct platform_device *pdev)
 }
 
 /* DSS HW IP initialisation */
+static const struct of_device_id dss_of_match[] = {
+	{ .compatible = "ti,omap2-dss", .data = &omap24xx_dss_feats },
+	{ .compatible = "ti,omap3-dss", .data = &omap3630_dss_feats },
+	{ .compatible = "ti,omap4-dss", .data = &omap44xx_dss_feats },
+	{ .compatible = "ti,omap5-dss", .data = &omap54xx_dss_feats },
+	{ .compatible = "ti,dra7-dss",  .data = &dra7xx_dss_feats },
+	{},
+};
+MODULE_DEVICE_TABLE(of, dss_of_match);
+
+static const struct soc_device_attribute dss_soc_devices[] = {
+	{ .machine = "OMAP3430/3530", .data = &omap34xx_dss_feats },
+	{ .machine = "AM35??",        .data = &omap34xx_dss_feats },
+	{ .family  = "AM43xx",        .data = &am43xx_dss_feats },
+	{ /* sentinel */ }
+};
+
 static int dss_bind(struct device *dev)
 {
 	struct platform_device *pdev = to_platform_device(dev);
@@ -1151,12 +1304,6 @@ static int dss_bind(struct device *dev)
 	u32 rev;
 	int r;
 
-	dss.pdev = pdev;
-
-	r = dss_init_features(dss.pdev);
-	if (r)
-		return r;
-
 	dss_mem = platform_get_resource(dss.pdev, IORESOURCE_MEM, 0);
 	dss.base = devm_ioremap_resource(&pdev->dev, dss_mem);
 	if (IS_ERR(dss.base))
@@ -1288,15 +1435,34 @@ static int dss_add_child_component(struct device *dev, void *data)
 
 static int dss_probe(struct platform_device *pdev)
 {
+	const struct soc_device_attribute *soc;
 	struct component_match *match = NULL;
 	int r;
 
+	dss.pdev = pdev;
+
+	/*
+	 * The various OMAP3-based SoCs can't be told apart using the compatible
+	 * string, use SoC device matching.
+	 */
+	soc = soc_device_match(dss_soc_devices);
+	if (soc)
+		dss.feat = soc->data;
+	else
+		dss.feat = of_match_device(dss_of_match, &pdev->dev)->data;
+
+	r = dss_initialize_debugfs();
+	if (r)
+		return r;
+
 	/* add all the child devices as components */
 	device_for_each_child(&pdev->dev, &match, dss_add_child_component);
 
 	r = component_master_add_with_match(&pdev->dev, &dss_component_ops, match);
-	if (r)
+	if (r) {
+		dss_uninitialize_debugfs();
 		return r;
+	}
 
 	return 0;
 }
@@ -1304,9 +1470,27 @@ static int dss_probe(struct platform_device *pdev)
 static int dss_remove(struct platform_device *pdev)
 {
 	component_master_del(&pdev->dev, &dss_component_ops);
+
+	dss_uninitialize_debugfs();
+
 	return 0;
 }
 
+static void dss_shutdown(struct platform_device *pdev)
+{
+	struct omap_dss_device *dssdev = NULL;
+
+	DSSDBG("shutdown\n");
+
+	for_each_dss_dev(dssdev) {
+		if (!dssdev->driver)
+			continue;
+
+		if (dssdev->state == OMAP_DSS_DISPLAY_ACTIVE)
+			dssdev->driver->disable(dssdev);
+	}
+}
+
 static int dss_runtime_suspend(struct device *dev)
 {
 	dss_save_context();
@@ -1343,20 +1527,10 @@ static const struct dev_pm_ops dss_pm_ops = {
 	.runtime_resume = dss_runtime_resume,
 };
 
-static const struct of_device_id dss_of_match[] = {
-	{ .compatible = "ti,omap2-dss", },
-	{ .compatible = "ti,omap3-dss", },
-	{ .compatible = "ti,omap4-dss", },
-	{ .compatible = "ti,omap5-dss", },
-	{ .compatible = "ti,dra7-dss", },
-	{},
-};
-
-MODULE_DEVICE_TABLE(of, dss_of_match);
-
 static struct platform_driver omap_dsshw_driver = {
 	.probe		= dss_probe,
 	.remove		= dss_remove,
+	.shutdown	= dss_shutdown,
 	.driver         = {
 		.name   = "omapdss_dss",
 		.pm	= &dss_pm_ops,
diff --git a/drivers/gpu/drm/omapdrm/dss/dss.h b/drivers/gpu/drm/omapdrm/dss/dss.h
index 8dbf35f3ab23..ed465572491e 100644
--- a/drivers/gpu/drm/omapdrm/dss/dss.h
+++ b/drivers/gpu/drm/omapdrm/dss/dss.h
@@ -27,6 +27,9 @@
 
 #include "omapdss.h"
 
+#define MAX_DSS_LCD_MANAGERS	3
+#define MAX_NUM_DSI		2
+
 #ifdef pr_fmt
 #undef pr_fmt
 #endif
@@ -72,6 +75,14 @@
 #define FLD_MOD(orig, val, start, end) \
 	(((orig) & ~FLD_MASK(start, end)) | FLD_VAL(val, start, end))
 
+enum dss_model {
+	DSS_MODEL_OMAP2,
+	DSS_MODEL_OMAP3,
+	DSS_MODEL_OMAP4,
+	DSS_MODEL_OMAP5,
+	DSS_MODEL_DRA7,
+};
+
 enum dss_io_pad_mode {
 	DSS_IO_PAD_MODE_RESET,
 	DSS_IO_PAD_MODE_RFBI,
@@ -174,6 +185,9 @@ struct dss_pll_hw {
 	bool has_freqsel;
 	bool has_selfreqdco;
 	bool has_refsel;
+
+	/* DRA7 errata i886: use high N & M to avoid jitter */
+	bool errata_i886;
 };
 
 struct dss_pll {
@@ -192,6 +206,11 @@ struct dss_pll {
 	struct dss_pll_clock_info cinfo;
 };
 
+/* Defines a generic omap register field */
+struct dss_reg_field {
+	u8 start, end;
+};
+
 struct dispc_clock_info {
 	/* rates that we get with dividers below */
 	unsigned long lck;
@@ -219,10 +238,11 @@ struct seq_file;
 struct platform_device;
 
 /* core */
-int dss_dsi_enable_pads(int dsi_id, unsigned lane_mask);
-void dss_dsi_disable_pads(int dsi_id, unsigned lane_mask);
-int dss_set_min_bus_tput(struct device *dev, unsigned long tput);
-int dss_debugfs_create_file(const char *name, void (*write)(struct seq_file *));
+static inline int dss_set_min_bus_tput(struct device *dev, unsigned long tput)
+{
+	/* To be implemented when the OMAP platform will provide this feature */
+	return 0;
+}
 
 static inline bool dss_mgr_is_lcd(enum omap_channel id)
 {
@@ -234,6 +254,16 @@ static inline bool dss_mgr_is_lcd(enum omap_channel id)
 }
 
 /* DSS */
+#if defined(CONFIG_OMAP2_DSS_DEBUGFS)
+int dss_debugfs_create_file(const char *name, void (*write)(struct seq_file *));
+#else
+static inline int dss_debugfs_create_file(const char *name,
+					  void (*write)(struct seq_file *))
+{
+	return 0;
+}
+#endif /* CONFIG_OMAP2_DSS_DEBUGFS */
+
 int dss_init_platform_driver(void) __init;
 void dss_uninit_platform_driver(void);
 
@@ -241,6 +271,8 @@ int dss_runtime_get(void);
 void dss_runtime_put(void);
 
 unsigned long dss_get_dispc_clk_rate(void);
+unsigned long dss_get_max_fck_rate(void);
+enum omap_dss_output_id dss_get_supported_outputs(enum omap_channel channel);
 int dss_dpi_select_source(int port, enum omap_channel channel);
 void dss_select_hdmi_venc_clk_source(enum dss_hdmi_venc_clk_source_select);
 enum dss_hdmi_venc_clk_source_select dss_get_hdmi_venc_clk_source(void);
@@ -252,10 +284,6 @@ struct dss_pll *dss_video_pll_init(struct platform_device *pdev, int id,
 	struct regulator *regulator);
 void dss_video_pll_uninit(struct dss_pll *pll);
 
-#if defined(CONFIG_OMAP2_DSS_DEBUGFS)
-void dss_debug_dump_clocks(struct seq_file *s);
-#endif
-
 void dss_ctrl_pll_enable(enum dss_pll_id pll_id, bool enable);
 
 void dss_sdi_init(int datapairs);
@@ -312,11 +340,12 @@ void dsi_irq_handler(void);
 
 /* DPI */
 #ifdef CONFIG_OMAP2_DSS_DPI
-int dpi_init_port(struct platform_device *pdev, struct device_node *port);
+int dpi_init_port(struct platform_device *pdev, struct device_node *port,
+		  enum dss_model dss_model);
 void dpi_uninit_port(struct device_node *port);
 #else
 static inline int dpi_init_port(struct platform_device *pdev,
-		struct device_node *port)
+		struct device_node *port, enum dss_model dss_model)
 {
 	return 0;
 }
diff --git a/drivers/gpu/drm/omapdrm/dss/dss_features.c b/drivers/gpu/drm/omapdrm/dss/dss_features.c
deleted file mode 100644
index 0e599710dd95..000000000000
--- a/drivers/gpu/drm/omapdrm/dss/dss_features.c
+++ /dev/null
@@ -1,905 +0,0 @@
-/*
- * linux/drivers/video/omap2/dss/dss_features.c
- *
- * Copyright (C) 2010 Texas Instruments
- * Author: Archit Taneja <archit@ti.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/types.h>
-#include <linux/err.h>
-#include <linux/slab.h>
-#include <drm/drm_fourcc.h>
-
-#include "omapdss.h"
-#include "dss.h"
-#include "dss_features.h"
-
-/* Defines a generic omap register field */
-struct dss_reg_field {
-	u8 start, end;
-};
-
-struct dss_param_range {
-	int min, max;
-};
-
-struct omap_dss_features {
-	const struct dss_reg_field *reg_fields;
-	const int num_reg_fields;
-
-	const enum dss_feat_id *features;
-	const int num_features;
-
-	const int num_mgrs;
-	const int num_ovls;
-	const enum omap_display_type *supported_displays;
-	const enum omap_dss_output_id *supported_outputs;
-	const u32 **supported_color_modes;
-	const enum omap_overlay_caps *overlay_caps;
-	const struct dss_param_range *dss_params;
-
-	const u32 buffer_size_unit;
-	const u32 burst_size_unit;
-};
-
-/* This struct is assigned to one of the below during initialization */
-static const struct omap_dss_features *omap_current_dss_features;
-
-static const struct dss_reg_field omap2_dss_reg_fields[] = {
-	[FEAT_REG_FIRHINC]			= { 11, 0 },
-	[FEAT_REG_FIRVINC]			= { 27, 16 },
-	[FEAT_REG_FIFOLOWTHRESHOLD]		= { 8, 0 },
-	[FEAT_REG_FIFOHIGHTHRESHOLD]		= { 24, 16 },
-	[FEAT_REG_FIFOSIZE]			= { 8, 0 },
-	[FEAT_REG_HORIZONTALACCU]		= { 9, 0 },
-	[FEAT_REG_VERTICALACCU]			= { 25, 16 },
-	[FEAT_REG_DISPC_CLK_SWITCH]		= { 0, 0 },
-};
-
-static const struct dss_reg_field omap3_dss_reg_fields[] = {
-	[FEAT_REG_FIRHINC]			= { 12, 0 },
-	[FEAT_REG_FIRVINC]			= { 28, 16 },
-	[FEAT_REG_FIFOLOWTHRESHOLD]		= { 11, 0 },
-	[FEAT_REG_FIFOHIGHTHRESHOLD]		= { 27, 16 },
-	[FEAT_REG_FIFOSIZE]			= { 10, 0 },
-	[FEAT_REG_HORIZONTALACCU]		= { 9, 0 },
-	[FEAT_REG_VERTICALACCU]			= { 25, 16 },
-	[FEAT_REG_DISPC_CLK_SWITCH]		= { 0, 0 },
-};
-
-static const struct dss_reg_field am43xx_dss_reg_fields[] = {
-	[FEAT_REG_FIRHINC]			= { 12, 0 },
-	[FEAT_REG_FIRVINC]			= { 28, 16 },
-	[FEAT_REG_FIFOLOWTHRESHOLD]	= { 11, 0 },
-	[FEAT_REG_FIFOHIGHTHRESHOLD]		= { 27, 16 },
-	[FEAT_REG_FIFOSIZE]		= { 10, 0 },
-	[FEAT_REG_HORIZONTALACCU]		= { 9, 0 },
-	[FEAT_REG_VERTICALACCU]			= { 25, 16 },
-	[FEAT_REG_DISPC_CLK_SWITCH]		= { 0, 0 },
-};
-
-static const struct dss_reg_field omap4_dss_reg_fields[] = {
-	[FEAT_REG_FIRHINC]			= { 12, 0 },
-	[FEAT_REG_FIRVINC]			= { 28, 16 },
-	[FEAT_REG_FIFOLOWTHRESHOLD]		= { 15, 0 },
-	[FEAT_REG_FIFOHIGHTHRESHOLD]		= { 31, 16 },
-	[FEAT_REG_FIFOSIZE]			= { 15, 0 },
-	[FEAT_REG_HORIZONTALACCU]		= { 10, 0 },
-	[FEAT_REG_VERTICALACCU]			= { 26, 16 },
-	[FEAT_REG_DISPC_CLK_SWITCH]		= { 9, 8 },
-};
-
-static const struct dss_reg_field omap5_dss_reg_fields[] = {
-	[FEAT_REG_FIRHINC]			= { 12, 0 },
-	[FEAT_REG_FIRVINC]			= { 28, 16 },
-	[FEAT_REG_FIFOLOWTHRESHOLD]		= { 15, 0 },
-	[FEAT_REG_FIFOHIGHTHRESHOLD]		= { 31, 16 },
-	[FEAT_REG_FIFOSIZE]			= { 15, 0 },
-	[FEAT_REG_HORIZONTALACCU]		= { 10, 0 },
-	[FEAT_REG_VERTICALACCU]			= { 26, 16 },
-	[FEAT_REG_DISPC_CLK_SWITCH]		= { 9, 7 },
-};
-
-static const enum omap_display_type omap2_dss_supported_displays[] = {
-	/* OMAP_DSS_CHANNEL_LCD */
-	OMAP_DISPLAY_TYPE_DPI | OMAP_DISPLAY_TYPE_DBI,
-
-	/* OMAP_DSS_CHANNEL_DIGIT */
-	OMAP_DISPLAY_TYPE_VENC,
-};
-
-static const enum omap_display_type omap3430_dss_supported_displays[] = {
-	/* OMAP_DSS_CHANNEL_LCD */
-	OMAP_DISPLAY_TYPE_DPI | OMAP_DISPLAY_TYPE_DBI |
-	OMAP_DISPLAY_TYPE_SDI | OMAP_DISPLAY_TYPE_DSI,
-
-	/* OMAP_DSS_CHANNEL_DIGIT */
-	OMAP_DISPLAY_TYPE_VENC,
-};
-
-static const enum omap_display_type omap3630_dss_supported_displays[] = {
-	/* OMAP_DSS_CHANNEL_LCD */
-	OMAP_DISPLAY_TYPE_DPI | OMAP_DISPLAY_TYPE_DBI |
-	OMAP_DISPLAY_TYPE_DSI,
-
-	/* OMAP_DSS_CHANNEL_DIGIT */
-	OMAP_DISPLAY_TYPE_VENC,
-};
-
-static const enum omap_display_type am43xx_dss_supported_displays[] = {
-	/* OMAP_DSS_CHANNEL_LCD */
-	OMAP_DISPLAY_TYPE_DPI | OMAP_DISPLAY_TYPE_DBI,
-};
-
-static const enum omap_display_type omap4_dss_supported_displays[] = {
-	/* OMAP_DSS_CHANNEL_LCD */
-	OMAP_DISPLAY_TYPE_DBI | OMAP_DISPLAY_TYPE_DSI,
-
-	/* OMAP_DSS_CHANNEL_DIGIT */
-	OMAP_DISPLAY_TYPE_VENC | OMAP_DISPLAY_TYPE_HDMI,
-
-	/* OMAP_DSS_CHANNEL_LCD2 */
-	OMAP_DISPLAY_TYPE_DPI | OMAP_DISPLAY_TYPE_DBI |
-	OMAP_DISPLAY_TYPE_DSI,
-};
-
-static const enum omap_display_type omap5_dss_supported_displays[] = {
-	/* OMAP_DSS_CHANNEL_LCD */
-	OMAP_DISPLAY_TYPE_DPI | OMAP_DISPLAY_TYPE_DBI |
-	OMAP_DISPLAY_TYPE_DSI,
-
-	/* OMAP_DSS_CHANNEL_DIGIT */
-	OMAP_DISPLAY_TYPE_HDMI | OMAP_DISPLAY_TYPE_DPI,
-
-	/* OMAP_DSS_CHANNEL_LCD2 */
-	OMAP_DISPLAY_TYPE_DPI | OMAP_DISPLAY_TYPE_DBI |
-	OMAP_DISPLAY_TYPE_DSI,
-};
-
-static const enum omap_dss_output_id omap2_dss_supported_outputs[] = {
-	/* OMAP_DSS_CHANNEL_LCD */
-	OMAP_DSS_OUTPUT_DPI | OMAP_DSS_OUTPUT_DBI,
-
-	/* OMAP_DSS_CHANNEL_DIGIT */
-	OMAP_DSS_OUTPUT_VENC,
-};
-
-static const enum omap_dss_output_id omap3430_dss_supported_outputs[] = {
-	/* OMAP_DSS_CHANNEL_LCD */
-	OMAP_DSS_OUTPUT_DPI | OMAP_DSS_OUTPUT_DBI |
-	OMAP_DSS_OUTPUT_SDI | OMAP_DSS_OUTPUT_DSI1,
-
-	/* OMAP_DSS_CHANNEL_DIGIT */
-	OMAP_DSS_OUTPUT_VENC,
-};
-
-static const enum omap_dss_output_id omap3630_dss_supported_outputs[] = {
-	/* OMAP_DSS_CHANNEL_LCD */
-	OMAP_DSS_OUTPUT_DPI | OMAP_DSS_OUTPUT_DBI |
-	OMAP_DSS_OUTPUT_DSI1,
-
-	/* OMAP_DSS_CHANNEL_DIGIT */
-	OMAP_DSS_OUTPUT_VENC,
-};
-
-static const enum omap_dss_output_id am43xx_dss_supported_outputs[] = {
-	/* OMAP_DSS_CHANNEL_LCD */
-	OMAP_DSS_OUTPUT_DPI | OMAP_DSS_OUTPUT_DBI,
-};
-
-static const enum omap_dss_output_id omap4_dss_supported_outputs[] = {
-	/* OMAP_DSS_CHANNEL_LCD */
-	OMAP_DSS_OUTPUT_DBI | OMAP_DSS_OUTPUT_DSI1,
-
-	/* OMAP_DSS_CHANNEL_DIGIT */
-	OMAP_DSS_OUTPUT_VENC | OMAP_DSS_OUTPUT_HDMI,
-
-	/* OMAP_DSS_CHANNEL_LCD2 */
-	OMAP_DSS_OUTPUT_DPI | OMAP_DSS_OUTPUT_DBI |
-	OMAP_DSS_OUTPUT_DSI2,
-};
-
-static const enum omap_dss_output_id omap5_dss_supported_outputs[] = {
-	/* OMAP_DSS_CHANNEL_LCD */
-	OMAP_DSS_OUTPUT_DPI | OMAP_DSS_OUTPUT_DBI |
-	OMAP_DSS_OUTPUT_DSI1 | OMAP_DSS_OUTPUT_DSI2,
-
-	/* OMAP_DSS_CHANNEL_DIGIT */
-	OMAP_DSS_OUTPUT_HDMI,
-
-	/* OMAP_DSS_CHANNEL_LCD2 */
-	OMAP_DSS_OUTPUT_DPI | OMAP_DSS_OUTPUT_DBI |
-	OMAP_DSS_OUTPUT_DSI1,
-
-	/* OMAP_DSS_CHANNEL_LCD3 */
-	OMAP_DSS_OUTPUT_DPI | OMAP_DSS_OUTPUT_DBI |
-	OMAP_DSS_OUTPUT_DSI2,
-};
-
-#define COLOR_ARRAY(arr...) (const u32[]) { arr, 0 }
-
-static const u32 *omap2_dss_supported_color_modes[] = {
-
-	/* OMAP_DSS_GFX */
-	COLOR_ARRAY(
-	DRM_FORMAT_RGBX4444, DRM_FORMAT_RGB565,
-	DRM_FORMAT_XRGB8888, DRM_FORMAT_RGB888),
-
-	/* OMAP_DSS_VIDEO1 */
-	COLOR_ARRAY(
-	DRM_FORMAT_RGB565, DRM_FORMAT_XRGB8888,
-	DRM_FORMAT_RGB888, DRM_FORMAT_YUYV,
-	DRM_FORMAT_UYVY),
-
-	/* OMAP_DSS_VIDEO2 */
-	COLOR_ARRAY(
-	DRM_FORMAT_RGB565, DRM_FORMAT_XRGB8888,
-	DRM_FORMAT_RGB888, DRM_FORMAT_YUYV,
-	DRM_FORMAT_UYVY),
-};
-
-static const u32 *omap3_dss_supported_color_modes[] = {
-	/* OMAP_DSS_GFX */
-	COLOR_ARRAY(
-	DRM_FORMAT_RGBX4444, DRM_FORMAT_ARGB4444,
-	DRM_FORMAT_RGB565, DRM_FORMAT_XRGB8888,
-	DRM_FORMAT_RGB888, DRM_FORMAT_ARGB8888,
-	DRM_FORMAT_RGBA8888, DRM_FORMAT_RGBX8888),
-
-	/* OMAP_DSS_VIDEO1 */
-	COLOR_ARRAY(
-	DRM_FORMAT_XRGB8888, DRM_FORMAT_RGB888,
-	DRM_FORMAT_RGBX4444, DRM_FORMAT_RGB565,
-	DRM_FORMAT_YUYV, DRM_FORMAT_UYVY),
-
-	/* OMAP_DSS_VIDEO2 */
-	COLOR_ARRAY(
-	DRM_FORMAT_RGBX4444, DRM_FORMAT_ARGB4444,
-	DRM_FORMAT_RGB565, DRM_FORMAT_XRGB8888,
-	DRM_FORMAT_RGB888, DRM_FORMAT_YUYV,
-	DRM_FORMAT_UYVY, DRM_FORMAT_ARGB8888,
-	DRM_FORMAT_RGBA8888, DRM_FORMAT_RGBX8888),
-};
-
-static const u32 *omap4_dss_supported_color_modes[] = {
-	/* OMAP_DSS_GFX */
-	COLOR_ARRAY(
-	DRM_FORMAT_RGBX4444, DRM_FORMAT_ARGB4444,
-	DRM_FORMAT_RGB565, DRM_FORMAT_XRGB8888,
-	DRM_FORMAT_RGB888, DRM_FORMAT_ARGB8888,
-	DRM_FORMAT_RGBA8888, DRM_FORMAT_RGBX8888,
-	DRM_FORMAT_ARGB1555, DRM_FORMAT_XRGB4444,
-	DRM_FORMAT_RGBA4444, DRM_FORMAT_XRGB1555),
-
-	/* OMAP_DSS_VIDEO1 */
-	COLOR_ARRAY(
-	DRM_FORMAT_RGB565, DRM_FORMAT_RGBX4444,
-	DRM_FORMAT_YUYV, DRM_FORMAT_ARGB1555,
-	DRM_FORMAT_RGBA8888, DRM_FORMAT_NV12,
-	DRM_FORMAT_RGBA4444, DRM_FORMAT_XRGB8888,
-	DRM_FORMAT_RGB888, DRM_FORMAT_UYVY,
-	DRM_FORMAT_ARGB4444, DRM_FORMAT_XRGB1555,
-	DRM_FORMAT_ARGB8888, DRM_FORMAT_XRGB4444,
-	DRM_FORMAT_RGBX8888),
-
-       /* OMAP_DSS_VIDEO2 */
-	COLOR_ARRAY(
-	DRM_FORMAT_RGB565, DRM_FORMAT_RGBX4444,
-	DRM_FORMAT_YUYV, DRM_FORMAT_ARGB1555,
-	DRM_FORMAT_RGBA8888, DRM_FORMAT_NV12,
-	DRM_FORMAT_RGBA4444, DRM_FORMAT_XRGB8888,
-	DRM_FORMAT_RGB888, DRM_FORMAT_UYVY,
-	DRM_FORMAT_ARGB4444, DRM_FORMAT_XRGB1555,
-	DRM_FORMAT_ARGB8888, DRM_FORMAT_XRGB4444,
-	DRM_FORMAT_RGBX8888),
-
-	/* OMAP_DSS_VIDEO3 */
-	COLOR_ARRAY(
-	DRM_FORMAT_RGB565, DRM_FORMAT_RGBX4444,
-	DRM_FORMAT_YUYV, DRM_FORMAT_ARGB1555,
-	DRM_FORMAT_RGBA8888, DRM_FORMAT_NV12,
-	DRM_FORMAT_RGBA4444, DRM_FORMAT_XRGB8888,
-	DRM_FORMAT_RGB888, DRM_FORMAT_UYVY,
-	DRM_FORMAT_ARGB4444, DRM_FORMAT_XRGB1555,
-	DRM_FORMAT_ARGB8888, DRM_FORMAT_XRGB4444,
-	DRM_FORMAT_RGBX8888),
-
-	/* OMAP_DSS_WB */
-	COLOR_ARRAY(
-	DRM_FORMAT_RGB565, DRM_FORMAT_RGBX4444,
-	DRM_FORMAT_YUYV, DRM_FORMAT_ARGB1555,
-	DRM_FORMAT_RGBA8888, DRM_FORMAT_NV12,
-	DRM_FORMAT_RGBA4444, DRM_FORMAT_XRGB8888,
-	DRM_FORMAT_RGB888, DRM_FORMAT_UYVY,
-	DRM_FORMAT_ARGB4444, DRM_FORMAT_XRGB1555,
-	DRM_FORMAT_ARGB8888, DRM_FORMAT_XRGB4444,
-	DRM_FORMAT_RGBX8888),
-};
-
-static const enum omap_overlay_caps omap2_dss_overlay_caps[] = {
-	/* OMAP_DSS_GFX */
-	OMAP_DSS_OVL_CAP_POS | OMAP_DSS_OVL_CAP_REPLICATION,
-
-	/* OMAP_DSS_VIDEO1 */
-	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_POS |
-		OMAP_DSS_OVL_CAP_REPLICATION,
-
-	/* OMAP_DSS_VIDEO2 */
-	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_POS |
-		OMAP_DSS_OVL_CAP_REPLICATION,
-};
-
-static const enum omap_overlay_caps omap3430_dss_overlay_caps[] = {
-	/* OMAP_DSS_GFX */
-	OMAP_DSS_OVL_CAP_GLOBAL_ALPHA | OMAP_DSS_OVL_CAP_POS |
-		OMAP_DSS_OVL_CAP_REPLICATION,
-
-	/* OMAP_DSS_VIDEO1 */
-	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_POS |
-		OMAP_DSS_OVL_CAP_REPLICATION,
-
-	/* OMAP_DSS_VIDEO2 */
-	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_GLOBAL_ALPHA |
-		OMAP_DSS_OVL_CAP_POS | OMAP_DSS_OVL_CAP_REPLICATION,
-};
-
-static const enum omap_overlay_caps omap3630_dss_overlay_caps[] = {
-	/* OMAP_DSS_GFX */
-	OMAP_DSS_OVL_CAP_GLOBAL_ALPHA | OMAP_DSS_OVL_CAP_PRE_MULT_ALPHA |
-		OMAP_DSS_OVL_CAP_POS | OMAP_DSS_OVL_CAP_REPLICATION,
-
-	/* OMAP_DSS_VIDEO1 */
-	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_POS |
-		OMAP_DSS_OVL_CAP_REPLICATION,
-
-	/* OMAP_DSS_VIDEO2 */
-	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_GLOBAL_ALPHA |
-		OMAP_DSS_OVL_CAP_PRE_MULT_ALPHA | OMAP_DSS_OVL_CAP_POS |
-		OMAP_DSS_OVL_CAP_REPLICATION,
-};
-
-static const enum omap_overlay_caps omap4_dss_overlay_caps[] = {
-	/* OMAP_DSS_GFX */
-	OMAP_DSS_OVL_CAP_GLOBAL_ALPHA | OMAP_DSS_OVL_CAP_PRE_MULT_ALPHA |
-		OMAP_DSS_OVL_CAP_ZORDER | OMAP_DSS_OVL_CAP_POS |
-		OMAP_DSS_OVL_CAP_REPLICATION,
-
-	/* OMAP_DSS_VIDEO1 */
-	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_GLOBAL_ALPHA |
-		OMAP_DSS_OVL_CAP_PRE_MULT_ALPHA | OMAP_DSS_OVL_CAP_ZORDER |
-		OMAP_DSS_OVL_CAP_POS | OMAP_DSS_OVL_CAP_REPLICATION,
-
-	/* OMAP_DSS_VIDEO2 */
-	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_GLOBAL_ALPHA |
-		OMAP_DSS_OVL_CAP_PRE_MULT_ALPHA | OMAP_DSS_OVL_CAP_ZORDER |
-		OMAP_DSS_OVL_CAP_POS | OMAP_DSS_OVL_CAP_REPLICATION,
-
-	/* OMAP_DSS_VIDEO3 */
-	OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_GLOBAL_ALPHA |
-		OMAP_DSS_OVL_CAP_PRE_MULT_ALPHA | OMAP_DSS_OVL_CAP_ZORDER |
-		OMAP_DSS_OVL_CAP_POS | OMAP_DSS_OVL_CAP_REPLICATION,
-};
-
-static const struct dss_param_range omap2_dss_param_range[] = {
-	[FEAT_PARAM_DSS_FCK]			= { 0, 133000000 },
-	[FEAT_PARAM_DSS_PCD]			= { 2, 255 },
-	[FEAT_PARAM_DOWNSCALE]			= { 1, 2 },
-	/*
-	 * Assuming the line width buffer to be 768 pixels as OMAP2 DISPC
-	 * scaler cannot scale a image with width more than 768.
-	 */
-	[FEAT_PARAM_LINEWIDTH]			= { 1, 768 },
-};
-
-static const struct dss_param_range omap3_dss_param_range[] = {
-	[FEAT_PARAM_DSS_FCK]			= { 0, 173000000 },
-	[FEAT_PARAM_DSS_PCD]			= { 1, 255 },
-	[FEAT_PARAM_DSIPLL_LPDIV]		= { 1, (1 << 13) - 1},
-	[FEAT_PARAM_DSI_FCK]			= { 0, 173000000 },
-	[FEAT_PARAM_DOWNSCALE]			= { 1, 4 },
-	[FEAT_PARAM_LINEWIDTH]			= { 1, 1024 },
-};
-
-static const struct dss_param_range am43xx_dss_param_range[] = {
-	[FEAT_PARAM_DSS_FCK]			= { 0, 200000000 },
-	[FEAT_PARAM_DSS_PCD]			= { 1, 255 },
-	[FEAT_PARAM_DOWNSCALE]			= { 1, 4 },
-	[FEAT_PARAM_LINEWIDTH]			= { 1, 1024 },
-};
-
-static const struct dss_param_range omap4_dss_param_range[] = {
-	[FEAT_PARAM_DSS_FCK]			= { 0, 186000000 },
-	[FEAT_PARAM_DSS_PCD]			= { 1, 255 },
-	[FEAT_PARAM_DSIPLL_LPDIV]		= { 0, (1 << 13) - 1 },
-	[FEAT_PARAM_DSI_FCK]			= { 0, 170000000 },
-	[FEAT_PARAM_DOWNSCALE]			= { 1, 4 },
-	[FEAT_PARAM_LINEWIDTH]			= { 1, 2048 },
-};
-
-static const struct dss_param_range omap5_dss_param_range[] = {
-	[FEAT_PARAM_DSS_FCK]			= { 0, 209250000 },
-	[FEAT_PARAM_DSS_PCD]			= { 1, 255 },
-	[FEAT_PARAM_DSIPLL_LPDIV]		= { 0, (1 << 13) - 1 },
-	[FEAT_PARAM_DSI_FCK]			= { 0, 209250000 },
-	[FEAT_PARAM_DOWNSCALE]			= { 1, 4 },
-	[FEAT_PARAM_LINEWIDTH]			= { 1, 2048 },
-};
-
-static const enum dss_feat_id omap2_dss_feat_list[] = {
-	FEAT_LCDENABLEPOL,
-	FEAT_LCDENABLESIGNAL,
-	FEAT_PCKFREEENABLE,
-	FEAT_FUNCGATED,
-	FEAT_ROWREPEATENABLE,
-	FEAT_RESIZECONF,
-};
-
-static const enum dss_feat_id omap3430_dss_feat_list[] = {
-	FEAT_LCDENABLEPOL,
-	FEAT_LCDENABLESIGNAL,
-	FEAT_PCKFREEENABLE,
-	FEAT_FUNCGATED,
-	FEAT_LINEBUFFERSPLIT,
-	FEAT_ROWREPEATENABLE,
-	FEAT_RESIZECONF,
-	FEAT_DSI_REVERSE_TXCLKESC,
-	FEAT_VENC_REQUIRES_TV_DAC_CLK,
-	FEAT_CPR,
-	FEAT_PRELOAD,
-	FEAT_FIR_COEF_V,
-	FEAT_ALPHA_FIXED_ZORDER,
-	FEAT_FIFO_MERGE,
-	FEAT_OMAP3_DSI_FIFO_BUG,
-	FEAT_DPI_USES_VDDS_DSI,
-};
-
-static const enum dss_feat_id am35xx_dss_feat_list[] = {
-	FEAT_LCDENABLEPOL,
-	FEAT_LCDENABLESIGNAL,
-	FEAT_PCKFREEENABLE,
-	FEAT_FUNCGATED,
-	FEAT_LINEBUFFERSPLIT,
-	FEAT_ROWREPEATENABLE,
-	FEAT_RESIZECONF,
-	FEAT_DSI_REVERSE_TXCLKESC,
-	FEAT_VENC_REQUIRES_TV_DAC_CLK,
-	FEAT_CPR,
-	FEAT_PRELOAD,
-	FEAT_FIR_COEF_V,
-	FEAT_ALPHA_FIXED_ZORDER,
-	FEAT_FIFO_MERGE,
-	FEAT_OMAP3_DSI_FIFO_BUG,
-};
-
-static const enum dss_feat_id am43xx_dss_feat_list[] = {
-	FEAT_LCDENABLEPOL,
-	FEAT_LCDENABLESIGNAL,
-	FEAT_PCKFREEENABLE,
-	FEAT_FUNCGATED,
-	FEAT_LINEBUFFERSPLIT,
-	FEAT_ROWREPEATENABLE,
-	FEAT_RESIZECONF,
-	FEAT_CPR,
-	FEAT_PRELOAD,
-	FEAT_FIR_COEF_V,
-	FEAT_ALPHA_FIXED_ZORDER,
-	FEAT_FIFO_MERGE,
-};
-
-static const enum dss_feat_id omap3630_dss_feat_list[] = {
-	FEAT_LCDENABLEPOL,
-	FEAT_LCDENABLESIGNAL,
-	FEAT_PCKFREEENABLE,
-	FEAT_FUNCGATED,
-	FEAT_LINEBUFFERSPLIT,
-	FEAT_ROWREPEATENABLE,
-	FEAT_RESIZECONF,
-	FEAT_DSI_PLL_PWR_BUG,
-	FEAT_CPR,
-	FEAT_PRELOAD,
-	FEAT_FIR_COEF_V,
-	FEAT_ALPHA_FIXED_ZORDER,
-	FEAT_FIFO_MERGE,
-	FEAT_OMAP3_DSI_FIFO_BUG,
-	FEAT_DPI_USES_VDDS_DSI,
-};
-
-static const enum dss_feat_id omap4430_es1_0_dss_feat_list[] = {
-	FEAT_MGR_LCD2,
-	FEAT_CORE_CLK_DIV,
-	FEAT_LCD_CLK_SRC,
-	FEAT_DSI_DCS_CMD_CONFIG_VC,
-	FEAT_DSI_VC_OCP_WIDTH,
-	FEAT_DSI_GNQ,
-	FEAT_HANDLE_UV_SEPARATE,
-	FEAT_ATTR2,
-	FEAT_CPR,
-	FEAT_PRELOAD,
-	FEAT_FIR_COEF_V,
-	FEAT_ALPHA_FREE_ZORDER,
-	FEAT_FIFO_MERGE,
-	FEAT_BURST_2D,
-};
-
-static const enum dss_feat_id omap4430_es2_0_1_2_dss_feat_list[] = {
-	FEAT_MGR_LCD2,
-	FEAT_CORE_CLK_DIV,
-	FEAT_LCD_CLK_SRC,
-	FEAT_DSI_DCS_CMD_CONFIG_VC,
-	FEAT_DSI_VC_OCP_WIDTH,
-	FEAT_DSI_GNQ,
-	FEAT_HDMI_CTS_SWMODE,
-	FEAT_HANDLE_UV_SEPARATE,
-	FEAT_ATTR2,
-	FEAT_CPR,
-	FEAT_PRELOAD,
-	FEAT_FIR_COEF_V,
-	FEAT_ALPHA_FREE_ZORDER,
-	FEAT_FIFO_MERGE,
-	FEAT_BURST_2D,
-};
-
-static const enum dss_feat_id omap4_dss_feat_list[] = {
-	FEAT_MGR_LCD2,
-	FEAT_CORE_CLK_DIV,
-	FEAT_LCD_CLK_SRC,
-	FEAT_DSI_DCS_CMD_CONFIG_VC,
-	FEAT_DSI_VC_OCP_WIDTH,
-	FEAT_DSI_GNQ,
-	FEAT_HDMI_CTS_SWMODE,
-	FEAT_HDMI_AUDIO_USE_MCLK,
-	FEAT_HANDLE_UV_SEPARATE,
-	FEAT_ATTR2,
-	FEAT_CPR,
-	FEAT_PRELOAD,
-	FEAT_FIR_COEF_V,
-	FEAT_ALPHA_FREE_ZORDER,
-	FEAT_FIFO_MERGE,
-	FEAT_BURST_2D,
-};
-
-static const enum dss_feat_id omap5_dss_feat_list[] = {
-	FEAT_MGR_LCD2,
-	FEAT_MGR_LCD3,
-	FEAT_CORE_CLK_DIV,
-	FEAT_LCD_CLK_SRC,
-	FEAT_DSI_DCS_CMD_CONFIG_VC,
-	FEAT_DSI_VC_OCP_WIDTH,
-	FEAT_DSI_GNQ,
-	FEAT_HDMI_CTS_SWMODE,
-	FEAT_HDMI_AUDIO_USE_MCLK,
-	FEAT_HANDLE_UV_SEPARATE,
-	FEAT_ATTR2,
-	FEAT_CPR,
-	FEAT_PRELOAD,
-	FEAT_FIR_COEF_V,
-	FEAT_ALPHA_FREE_ZORDER,
-	FEAT_FIFO_MERGE,
-	FEAT_BURST_2D,
-	FEAT_DSI_PHY_DCC,
-	FEAT_MFLAG,
-};
-
-/* OMAP2 DSS Features */
-static const struct omap_dss_features omap2_dss_features = {
-	.reg_fields = omap2_dss_reg_fields,
-	.num_reg_fields = ARRAY_SIZE(omap2_dss_reg_fields),
-
-	.features = omap2_dss_feat_list,
-	.num_features = ARRAY_SIZE(omap2_dss_feat_list),
-
-	.num_mgrs = 2,
-	.num_ovls = 3,
-	.supported_displays = omap2_dss_supported_displays,
-	.supported_outputs = omap2_dss_supported_outputs,
-	.supported_color_modes = omap2_dss_supported_color_modes,
-	.overlay_caps = omap2_dss_overlay_caps,
-	.dss_params = omap2_dss_param_range,
-	.buffer_size_unit = 1,
-	.burst_size_unit = 8,
-};
-
-/* OMAP3 DSS Features */
-static const struct omap_dss_features omap3430_dss_features = {
-	.reg_fields = omap3_dss_reg_fields,
-	.num_reg_fields = ARRAY_SIZE(omap3_dss_reg_fields),
-
-	.features = omap3430_dss_feat_list,
-	.num_features = ARRAY_SIZE(omap3430_dss_feat_list),
-
-	.num_mgrs = 2,
-	.num_ovls = 3,
-	.supported_displays = omap3430_dss_supported_displays,
-	.supported_outputs = omap3430_dss_supported_outputs,
-	.supported_color_modes = omap3_dss_supported_color_modes,
-	.overlay_caps = omap3430_dss_overlay_caps,
-	.dss_params = omap3_dss_param_range,
-	.buffer_size_unit = 1,
-	.burst_size_unit = 8,
-};
-
-/*
- * AM35xx DSS Features. This is basically OMAP3 DSS Features without the
- * vdds_dsi regulator.
- */
-static const struct omap_dss_features am35xx_dss_features = {
-	.reg_fields = omap3_dss_reg_fields,
-	.num_reg_fields = ARRAY_SIZE(omap3_dss_reg_fields),
-
-	.features = am35xx_dss_feat_list,
-	.num_features = ARRAY_SIZE(am35xx_dss_feat_list),
-
-	.num_mgrs = 2,
-	.num_ovls = 3,
-	.supported_displays = omap3430_dss_supported_displays,
-	.supported_outputs = omap3430_dss_supported_outputs,
-	.supported_color_modes = omap3_dss_supported_color_modes,
-	.overlay_caps = omap3430_dss_overlay_caps,
-	.dss_params = omap3_dss_param_range,
-	.buffer_size_unit = 1,
-	.burst_size_unit = 8,
-};
-
-static const struct omap_dss_features am43xx_dss_features = {
-	.reg_fields = am43xx_dss_reg_fields,
-	.num_reg_fields = ARRAY_SIZE(am43xx_dss_reg_fields),
-
-	.features = am43xx_dss_feat_list,
-	.num_features = ARRAY_SIZE(am43xx_dss_feat_list),
-
-	.num_mgrs = 1,
-	.num_ovls = 3,
-	.supported_displays = am43xx_dss_supported_displays,
-	.supported_outputs = am43xx_dss_supported_outputs,
-	.supported_color_modes = omap3_dss_supported_color_modes,
-	.overlay_caps = omap3430_dss_overlay_caps,
-	.dss_params = am43xx_dss_param_range,
-	.buffer_size_unit = 1,
-	.burst_size_unit = 8,
-};
-
-static const struct omap_dss_features omap3630_dss_features = {
-	.reg_fields = omap3_dss_reg_fields,
-	.num_reg_fields = ARRAY_SIZE(omap3_dss_reg_fields),
-
-	.features = omap3630_dss_feat_list,
-	.num_features = ARRAY_SIZE(omap3630_dss_feat_list),
-
-	.num_mgrs = 2,
-	.num_ovls = 3,
-	.supported_displays = omap3630_dss_supported_displays,
-	.supported_outputs = omap3630_dss_supported_outputs,
-	.supported_color_modes = omap3_dss_supported_color_modes,
-	.overlay_caps = omap3630_dss_overlay_caps,
-	.dss_params = omap3_dss_param_range,
-	.buffer_size_unit = 1,
-	.burst_size_unit = 8,
-};
-
-/* OMAP4 DSS Features */
-/* For OMAP4430 ES 1.0 revision */
-static const struct omap_dss_features omap4430_es1_0_dss_features  = {
-	.reg_fields = omap4_dss_reg_fields,
-	.num_reg_fields = ARRAY_SIZE(omap4_dss_reg_fields),
-
-	.features = omap4430_es1_0_dss_feat_list,
-	.num_features = ARRAY_SIZE(omap4430_es1_0_dss_feat_list),
-
-	.num_mgrs = 3,
-	.num_ovls = 4,
-	.supported_displays = omap4_dss_supported_displays,
-	.supported_outputs = omap4_dss_supported_outputs,
-	.supported_color_modes = omap4_dss_supported_color_modes,
-	.overlay_caps = omap4_dss_overlay_caps,
-	.dss_params = omap4_dss_param_range,
-	.buffer_size_unit = 16,
-	.burst_size_unit = 16,
-};
-
-/* For OMAP4430 ES 2.0, 2.1 and 2.2 revisions */
-static const struct omap_dss_features omap4430_es2_0_1_2_dss_features = {
-	.reg_fields = omap4_dss_reg_fields,
-	.num_reg_fields = ARRAY_SIZE(omap4_dss_reg_fields),
-
-	.features = omap4430_es2_0_1_2_dss_feat_list,
-	.num_features = ARRAY_SIZE(omap4430_es2_0_1_2_dss_feat_list),
-
-	.num_mgrs = 3,
-	.num_ovls = 4,
-	.supported_displays = omap4_dss_supported_displays,
-	.supported_outputs = omap4_dss_supported_outputs,
-	.supported_color_modes = omap4_dss_supported_color_modes,
-	.overlay_caps = omap4_dss_overlay_caps,
-	.dss_params = omap4_dss_param_range,
-	.buffer_size_unit = 16,
-	.burst_size_unit = 16,
-};
-
-/* For all the other OMAP4 versions */
-static const struct omap_dss_features omap4_dss_features = {
-	.reg_fields = omap4_dss_reg_fields,
-	.num_reg_fields = ARRAY_SIZE(omap4_dss_reg_fields),
-
-	.features = omap4_dss_feat_list,
-	.num_features = ARRAY_SIZE(omap4_dss_feat_list),
-
-	.num_mgrs = 3,
-	.num_ovls = 4,
-	.supported_displays = omap4_dss_supported_displays,
-	.supported_outputs = omap4_dss_supported_outputs,
-	.supported_color_modes = omap4_dss_supported_color_modes,
-	.overlay_caps = omap4_dss_overlay_caps,
-	.dss_params = omap4_dss_param_range,
-	.buffer_size_unit = 16,
-	.burst_size_unit = 16,
-};
-
-/* OMAP5 DSS Features */
-static const struct omap_dss_features omap5_dss_features = {
-	.reg_fields = omap5_dss_reg_fields,
-	.num_reg_fields = ARRAY_SIZE(omap5_dss_reg_fields),
-
-	.features = omap5_dss_feat_list,
-	.num_features = ARRAY_SIZE(omap5_dss_feat_list),
-
-	.num_mgrs = 4,
-	.num_ovls = 4,
-	.supported_displays = omap5_dss_supported_displays,
-	.supported_outputs = omap5_dss_supported_outputs,
-	.supported_color_modes = omap4_dss_supported_color_modes,
-	.overlay_caps = omap4_dss_overlay_caps,
-	.dss_params = omap5_dss_param_range,
-	.buffer_size_unit = 16,
-	.burst_size_unit = 16,
-};
-
-/* Functions returning values related to a DSS feature */
-int dss_feat_get_num_mgrs(void)
-{
-	return omap_current_dss_features->num_mgrs;
-}
-
-int dss_feat_get_num_ovls(void)
-{
-	return omap_current_dss_features->num_ovls;
-}
-
-unsigned long dss_feat_get_param_min(enum dss_range_param param)
-{
-	return omap_current_dss_features->dss_params[param].min;
-}
-
-unsigned long dss_feat_get_param_max(enum dss_range_param param)
-{
-	return omap_current_dss_features->dss_params[param].max;
-}
-
-enum omap_display_type dss_feat_get_supported_displays(enum omap_channel channel)
-{
-	return omap_current_dss_features->supported_displays[channel];
-}
-
-enum omap_dss_output_id dss_feat_get_supported_outputs(enum omap_channel channel)
-{
-	return omap_current_dss_features->supported_outputs[channel];
-}
-
-const u32 *dss_feat_get_supported_color_modes(enum omap_plane_id plane)
-{
-	return omap_current_dss_features->supported_color_modes[plane];
-}
-
-enum omap_overlay_caps dss_feat_get_overlay_caps(enum omap_plane_id plane)
-{
-	return omap_current_dss_features->overlay_caps[plane];
-}
-
-bool dss_feat_color_mode_supported(enum omap_plane_id plane, u32 fourcc)
-{
-	const u32 *modes;
-	unsigned int i;
-
-	modes = omap_current_dss_features->supported_color_modes[plane];
-
-	for (i = 0; modes[i]; ++i) {
-		if (modes[i] == fourcc)
-			return true;
-	}
-
-	return false;
-}
-
-u32 dss_feat_get_buffer_size_unit(void)
-{
-	return omap_current_dss_features->buffer_size_unit;
-}
-
-u32 dss_feat_get_burst_size_unit(void)
-{
-	return omap_current_dss_features->burst_size_unit;
-}
-
-/* DSS has_feature check */
-bool dss_has_feature(enum dss_feat_id id)
-{
-	int i;
-	const enum dss_feat_id *features = omap_current_dss_features->features;
-	const int num_features = omap_current_dss_features->num_features;
-
-	for (i = 0; i < num_features; i++) {
-		if (features[i] == id)
-			return true;
-	}
-
-	return false;
-}
-
-void dss_feat_get_reg_field(enum dss_feat_reg_field id, u8 *start, u8 *end)
-{
-	if (id >= omap_current_dss_features->num_reg_fields)
-		BUG();
-
-	*start = omap_current_dss_features->reg_fields[id].start;
-	*end = omap_current_dss_features->reg_fields[id].end;
-}
-
-void dss_features_init(enum omapdss_version version)
-{
-	switch (version) {
-	case OMAPDSS_VER_OMAP24xx:
-		omap_current_dss_features = &omap2_dss_features;
-		break;
-
-	case OMAPDSS_VER_OMAP34xx_ES1:
-	case OMAPDSS_VER_OMAP34xx_ES3:
-		omap_current_dss_features = &omap3430_dss_features;
-		break;
-
-	case OMAPDSS_VER_OMAP3630:
-		omap_current_dss_features = &omap3630_dss_features;
-		break;
-
-	case OMAPDSS_VER_OMAP4430_ES1:
-		omap_current_dss_features = &omap4430_es1_0_dss_features;
-		break;
-
-	case OMAPDSS_VER_OMAP4430_ES2:
-		omap_current_dss_features = &omap4430_es2_0_1_2_dss_features;
-		break;
-
-	case OMAPDSS_VER_OMAP4:
-		omap_current_dss_features = &omap4_dss_features;
-		break;
-
-	case OMAPDSS_VER_OMAP5:
-	case OMAPDSS_VER_DRA7xx:
-		omap_current_dss_features = &omap5_dss_features;
-		break;
-
-	case OMAPDSS_VER_AM35xx:
-		omap_current_dss_features = &am35xx_dss_features;
-		break;
-
-	case OMAPDSS_VER_AM43xx:
-		omap_current_dss_features = &am43xx_dss_features;
-		break;
-
-	default:
-		DSSWARN("Unsupported OMAP version");
-		break;
-	}
-}
diff --git a/drivers/gpu/drm/omapdrm/dss/dss_features.h b/drivers/gpu/drm/omapdrm/dss/dss_features.h
deleted file mode 100644
index c36436d27ff5..000000000000
--- a/drivers/gpu/drm/omapdrm/dss/dss_features.h
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * linux/drivers/video/omap2/dss/dss_features.h
- *
- * Copyright (C) 2010 Texas Instruments
- * Author: Archit Taneja <archit@ti.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef __OMAP2_DSS_FEATURES_H
-#define __OMAP2_DSS_FEATURES_H
-
-#define MAX_DSS_MANAGERS	4
-#define MAX_DSS_OVERLAYS	4
-#define MAX_DSS_LCD_MANAGERS	3
-#define MAX_NUM_DSI		2
-
-/* DSS has feature id */
-enum dss_feat_id {
-	FEAT_LCDENABLEPOL,
-	FEAT_LCDENABLESIGNAL,
-	FEAT_PCKFREEENABLE,
-	FEAT_FUNCGATED,
-	FEAT_MGR_LCD2,
-	FEAT_MGR_LCD3,
-	FEAT_LINEBUFFERSPLIT,
-	FEAT_ROWREPEATENABLE,
-	FEAT_RESIZECONF,
-	/* Independent core clk divider */
-	FEAT_CORE_CLK_DIV,
-	FEAT_LCD_CLK_SRC,
-	/* DSI-PLL power command 0x3 is not working */
-	FEAT_DSI_PLL_PWR_BUG,
-	FEAT_DSI_DCS_CMD_CONFIG_VC,
-	FEAT_DSI_VC_OCP_WIDTH,
-	FEAT_DSI_REVERSE_TXCLKESC,
-	FEAT_DSI_GNQ,
-	FEAT_DPI_USES_VDDS_DSI,
-	FEAT_HDMI_CTS_SWMODE,
-	FEAT_HDMI_AUDIO_USE_MCLK,
-	FEAT_HANDLE_UV_SEPARATE,
-	FEAT_ATTR2,
-	FEAT_VENC_REQUIRES_TV_DAC_CLK,
-	FEAT_CPR,
-	FEAT_PRELOAD,
-	FEAT_FIR_COEF_V,
-	FEAT_ALPHA_FIXED_ZORDER,
-	FEAT_ALPHA_FREE_ZORDER,
-	FEAT_FIFO_MERGE,
-	/* An unknown HW bug causing the normal FIFO thresholds not to work */
-	FEAT_OMAP3_DSI_FIFO_BUG,
-	FEAT_BURST_2D,
-	FEAT_DSI_PHY_DCC,
-	FEAT_MFLAG,
-};
-
-/* DSS register field id */
-enum dss_feat_reg_field {
-	FEAT_REG_FIRHINC,
-	FEAT_REG_FIRVINC,
-	FEAT_REG_FIFOHIGHTHRESHOLD,
-	FEAT_REG_FIFOLOWTHRESHOLD,
-	FEAT_REG_FIFOSIZE,
-	FEAT_REG_HORIZONTALACCU,
-	FEAT_REG_VERTICALACCU,
-	FEAT_REG_DISPC_CLK_SWITCH,
-};
-
-enum dss_range_param {
-	FEAT_PARAM_DSS_FCK,
-	FEAT_PARAM_DSS_PCD,
-	FEAT_PARAM_DSIPLL_LPDIV,
-	FEAT_PARAM_DSI_FCK,
-	FEAT_PARAM_DOWNSCALE,
-	FEAT_PARAM_LINEWIDTH,
-};
-
-/* DSS Feature Functions */
-unsigned long dss_feat_get_param_min(enum dss_range_param param);
-unsigned long dss_feat_get_param_max(enum dss_range_param param);
-enum omap_overlay_caps dss_feat_get_overlay_caps(enum omap_plane_id plane);
-bool dss_feat_color_mode_supported(enum omap_plane_id plane,
-		u32 fourcc);
-
-u32 dss_feat_get_buffer_size_unit(void);	/* in bytes */
-u32 dss_feat_get_burst_size_unit(void);		/* in bytes */
-
-bool dss_has_feature(enum dss_feat_id id);
-void dss_feat_get_reg_field(enum dss_feat_reg_field id, u8 *start, u8 *end);
-void dss_features_init(enum omapdss_version version);
-
-enum omap_display_type dss_feat_get_supported_displays(enum omap_channel channel);
-enum omap_dss_output_id dss_feat_get_supported_outputs(enum omap_channel channel);
-
-int dss_feat_get_num_mgrs(void);
-int dss_feat_get_num_ovls(void);
-const u32 *dss_feat_get_supported_color_modes(enum omap_plane_id plane);
-
-#endif
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi.h b/drivers/gpu/drm/omapdrm/dss/hdmi.h
index fb6cccd02374..a820b394af09 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi.h
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi.h
@@ -234,6 +234,7 @@ struct hdmi_core_audio_config {
 struct hdmi_wp_data {
 	void __iomem *base;
 	phys_addr_t phys_base;
+	unsigned int version;
 };
 
 struct hdmi_pll_data {
@@ -245,15 +246,24 @@ struct hdmi_pll_data {
 	struct hdmi_wp_data *wp;
 };
 
+struct hdmi_phy_features {
+	bool bist_ctrl;
+	bool ldo_voltage;
+	unsigned long max_phy;
+};
+
 struct hdmi_phy_data {
 	void __iomem *base;
 
+	const struct hdmi_phy_features *features;
 	u8 lane_function[4];
 	u8 lane_polarity[4];
 };
 
 struct hdmi_core_data {
 	void __iomem *base;
+	bool cts_swmode;
+	bool audio_use_mclk;
 };
 
 static inline void hdmi_write_reg(void __iomem *base_addr, const u32 idx,
@@ -303,7 +313,8 @@ void hdmi_wp_video_config_timing(struct hdmi_wp_data *wp,
 		struct videomode *vm);
 void hdmi_wp_init_vid_fmt_timings(struct hdmi_video_format *video_fmt,
 		struct videomode *vm, struct hdmi_config *param);
-int hdmi_wp_init(struct platform_device *pdev, struct hdmi_wp_data *wp);
+int hdmi_wp_init(struct platform_device *pdev, struct hdmi_wp_data *wp,
+		 unsigned int version);
 phys_addr_t hdmi_wp_get_audio_dma_addr(struct hdmi_wp_data *wp);
 
 /* HDMI PLL funcs */
@@ -316,7 +327,8 @@ void hdmi_pll_uninit(struct hdmi_pll_data *hpll);
 int hdmi_phy_configure(struct hdmi_phy_data *phy, unsigned long hfbitclk,
 	unsigned long lfbitclk);
 void hdmi_phy_dump(struct hdmi_phy_data *phy, struct seq_file *s);
-int hdmi_phy_init(struct platform_device *pdev, struct hdmi_phy_data *phy);
+int hdmi_phy_init(struct platform_device *pdev, struct hdmi_phy_data *phy,
+		  unsigned int version);
 int hdmi_phy_parse_lanes(struct hdmi_phy_data *phy, const u32 *lanes);
 
 /* HDMI common funcs */
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi4.c b/drivers/gpu/drm/omapdrm/dss/hdmi4.c
index 284b4942b9ac..f169348da377 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi4.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi4.c
@@ -40,7 +40,6 @@
 #include "omapdss.h"
 #include "hdmi4_core.h"
 #include "dss.h"
-#include "dss_features.h"
 #include "hdmi.h"
 
 static struct omap_hdmi hdmi;
@@ -668,7 +667,7 @@ static int hdmi_audio_register(struct device *dev)
 {
 	struct omap_hdmi_audio_pdata pdata = {
 		.dev = dev,
-		.dss_version = omapdss_get_version(),
+		.version = 4,
 		.audio_dma_addr = hdmi_wp_get_audio_dma_addr(&hdmi.wp),
 		.ops = &hdmi_audio_ops,
 	};
@@ -700,7 +699,7 @@ static int hdmi4_bind(struct device *dev, struct device *master, void *data)
 	if (r)
 		return r;
 
-	r = hdmi_wp_init(pdev, &hdmi.wp);
+	r = hdmi_wp_init(pdev, &hdmi.wp, 4);
 	if (r)
 		return r;
 
@@ -708,7 +707,7 @@ static int hdmi4_bind(struct device *dev, struct device *master, void *data)
 	if (r)
 		return r;
 
-	r = hdmi_phy_init(pdev, &hdmi.phy);
+	r = hdmi_phy_init(pdev, &hdmi.phy, 4);
 	if (r)
 		goto err;
 
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi4_core.c b/drivers/gpu/drm/omapdrm/dss/hdmi4_core.c
index ed6001613405..365cf07daa01 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi4_core.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi4_core.c
@@ -31,11 +31,11 @@
 #include <linux/platform_device.h>
 #include <linux/string.h>
 #include <linux/seq_file.h>
+#include <linux/sys_soc.h>
 #include <sound/asound.h>
 #include <sound/asoundef.h>
 
 #include "hdmi4_core.h"
-#include "dss_features.h"
 
 #define HDMI_CORE_AV		0x500
 
@@ -757,10 +757,10 @@ int hdmi4_audio_config(struct hdmi_core_data *core, struct hdmi_wp_data *wp,
 	/* Audio clock regeneration settings */
 	acore.n = n;
 	acore.cts = cts;
-	if (dss_has_feature(FEAT_HDMI_CTS_SWMODE)) {
+	if (core->cts_swmode) {
 		acore.aud_par_busclk = 0;
 		acore.cts_mode = HDMI_AUDIO_CTS_MODE_SW;
-		acore.use_mclk = dss_has_feature(FEAT_HDMI_AUDIO_USE_MCLK);
+		acore.use_mclk = core->audio_use_mclk;
 	} else {
 		acore.aud_par_busclk = (((128 * 31) - 1) << 8);
 		acore.cts_mode = HDMI_AUDIO_CTS_MODE_HW;
@@ -884,10 +884,42 @@ void hdmi4_audio_stop(struct hdmi_core_data *core, struct hdmi_wp_data *wp)
 	hdmi_wp_audio_core_req_enable(wp, false);
 }
 
+struct hdmi4_features {
+	bool cts_swmode;
+	bool audio_use_mclk;
+};
+
+static const struct hdmi4_features hdmi4_es1_features = {
+	.cts_swmode = false,
+	.audio_use_mclk = false,
+};
+
+static const struct hdmi4_features hdmi4_es2_features = {
+	.cts_swmode = true,
+	.audio_use_mclk = false,
+};
+
+static const struct hdmi4_features hdmi4_es3_features = {
+	.cts_swmode = true,
+	.audio_use_mclk = true,
+};
+
+static const struct soc_device_attribute hdmi4_soc_devices[] = {
+	{ .family = "OMAP4", .revision = "ES1.?", .data = &hdmi4_es1_features },
+	{ .family = "OMAP4", .revision = "ES2.?", .data = &hdmi4_es2_features },
+	{ .family = "OMAP4",			  .data = &hdmi4_es3_features },
+	{ /* sentinel */ }
+};
+
 int hdmi4_core_init(struct platform_device *pdev, struct hdmi_core_data *core)
 {
+	const struct hdmi4_features *features;
 	struct resource *res;
 
+	features = soc_device_match(hdmi4_soc_devices)->data;
+	core->cts_swmode = features->cts_swmode;
+	core->audio_use_mclk = features->audio_use_mclk;
+
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "core");
 	core->base = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(core->base))
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi5.c b/drivers/gpu/drm/omapdrm/dss/hdmi5.c
index 441e1999d86a..b3221ca5bcd8 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi5.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi5.c
@@ -45,7 +45,6 @@
 #include "omapdss.h"
 #include "hdmi5_core.h"
 #include "dss.h"
-#include "dss_features.h"
 
 static struct omap_hdmi hdmi;
 
@@ -695,7 +694,7 @@ static int hdmi_audio_register(struct device *dev)
 {
 	struct omap_hdmi_audio_pdata pdata = {
 		.dev = dev,
-		.dss_version = omapdss_get_version(),
+		.version = 5,
 		.audio_dma_addr = hdmi_wp_get_audio_dma_addr(&hdmi.wp),
 		.ops = &hdmi_audio_ops,
 	};
@@ -732,7 +731,7 @@ static int hdmi5_bind(struct device *dev, struct device *master, void *data)
 	if (r)
 		return r;
 
-	r = hdmi_wp_init(pdev, &hdmi.wp);
+	r = hdmi_wp_init(pdev, &hdmi.wp, 5);
 	if (r)
 		return r;
 
@@ -740,7 +739,7 @@ static int hdmi5_bind(struct device *dev, struct device *master, void *data)
 	if (r)
 		return r;
 
-	r = hdmi_phy_init(pdev, &hdmi.phy);
+	r = hdmi_phy_init(pdev, &hdmi.phy, 5);
 	if (r)
 		goto err;
 
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi_phy.c b/drivers/gpu/drm/omapdrm/dss/hdmi_phy.c
index fb5e4c724b4b..a156292b1820 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi_phy.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi_phy.c
@@ -19,14 +19,6 @@
 #include "dss.h"
 #include "hdmi.h"
 
-struct hdmi_phy_features {
-	bool bist_ctrl;
-	bool ldo_voltage;
-	unsigned long max_phy;
-};
-
-static const struct hdmi_phy_features *phy_feat;
-
 void hdmi_phy_dump(struct hdmi_phy_data *phy, struct seq_file *s)
 {
 #define DUMPPHY(r) seq_printf(s, "%-35s %08x\n", #r,\
@@ -36,7 +28,7 @@ void hdmi_phy_dump(struct hdmi_phy_data *phy, struct seq_file *s)
 	DUMPPHY(HDMI_TXPHY_DIGITAL_CTRL);
 	DUMPPHY(HDMI_TXPHY_POWER_CTRL);
 	DUMPPHY(HDMI_TXPHY_PAD_CFG_CTRL);
-	if (phy_feat->bist_ctrl)
+	if (phy->features->bist_ctrl)
 		DUMPPHY(HDMI_TXPHY_BIST_CONTROL);
 }
 
@@ -146,7 +138,7 @@ int hdmi_phy_configure(struct hdmi_phy_data *phy, unsigned long hfbitclk,
 	 * In OMAP5+, the HFBITCLK must be divided by 2 before issuing the
 	 * HDMI_PHYPWRCMD_LDOON command.
 	*/
-	if (phy_feat->bist_ctrl)
+	if (phy->features->bist_ctrl)
 		REG_FLD_MOD(phy->base, HDMI_TXPHY_BIST_CONTROL, 1, 11, 11);
 
 	/*
@@ -155,7 +147,7 @@ int hdmi_phy_configure(struct hdmi_phy_data *phy, unsigned long hfbitclk,
 	 */
 	if (hfbitclk != lfbitclk)
 		freqout = 0;
-	else if (hfbitclk / 10 < phy_feat->max_phy)
+	else if (hfbitclk / 10 < phy->features->max_phy)
 		freqout = 1;
 	else
 		freqout = 2;
@@ -170,7 +162,7 @@ int hdmi_phy_configure(struct hdmi_phy_data *phy, unsigned long hfbitclk,
 	hdmi_write_reg(phy->base, HDMI_TXPHY_DIGITAL_CTRL, 0xF0000000);
 
 	/* Setup max LDO voltage */
-	if (phy_feat->ldo_voltage)
+	if (phy->features->ldo_voltage)
 		REG_FLD_MOD(phy->base, HDMI_TXPHY_POWER_CTRL, 0xB, 3, 0);
 
 	hdmi_phy_configure_lanes(phy);
@@ -190,47 +182,15 @@ static const struct hdmi_phy_features omap54xx_phy_feats = {
 	.max_phy	=	186000000,
 };
 
-static int hdmi_phy_init_features(struct platform_device *pdev)
-{
-	struct hdmi_phy_features *dst;
-	const struct hdmi_phy_features *src;
-
-	dst = devm_kzalloc(&pdev->dev, sizeof(*dst), GFP_KERNEL);
-	if (!dst) {
-		dev_err(&pdev->dev, "Failed to allocate HDMI PHY Features\n");
-		return -ENOMEM;
-	}
-
-	switch (omapdss_get_version()) {
-	case OMAPDSS_VER_OMAP4430_ES1:
-	case OMAPDSS_VER_OMAP4430_ES2:
-	case OMAPDSS_VER_OMAP4:
-		src = &omap44xx_phy_feats;
-		break;
-
-	case OMAPDSS_VER_OMAP5:
-	case OMAPDSS_VER_DRA7xx:
-		src = &omap54xx_phy_feats;
-		break;
-
-	default:
-		return -ENODEV;
-	}
-
-	memcpy(dst, src, sizeof(*dst));
-	phy_feat = dst;
-
-	return 0;
-}
-
-int hdmi_phy_init(struct platform_device *pdev, struct hdmi_phy_data *phy)
+int hdmi_phy_init(struct platform_device *pdev, struct hdmi_phy_data *phy,
+		  unsigned int version)
 {
-	int r;
 	struct resource *res;
 
-	r = hdmi_phy_init_features(pdev);
-	if (r)
-		return r;
+	if (version == 4)
+		phy->features = &omap44xx_phy_feats;
+	else
+		phy->features = &omap54xx_phy_feats;
 
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "phy");
 	phy->base = devm_ioremap_resource(&pdev->dev, res);
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi_pll.c b/drivers/gpu/drm/omapdrm/dss/hdmi_pll.c
index 46239358655a..55bee81f4dd5 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi_pll.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi_pll.c
@@ -71,7 +71,7 @@ static void hdmi_pll_disable(struct dss_pll *dsspll)
 	WARN_ON(r < 0 && r != -ENOSYS);
 }
 
-static const struct dss_pll_ops dsi_pll_ops = {
+static const struct dss_pll_ops hdmi_pll_ops = {
 	.enable = hdmi_pll_enable,
 	.disable = hdmi_pll_disable,
 	.set_config = dss_pll_write_config_type_b,
@@ -128,7 +128,8 @@ static const struct dss_pll_hw dss_omap5_hdmi_pll_hw = {
 	.has_refsel = true,
 };
 
-static int dsi_init_pll_data(struct platform_device *pdev, struct hdmi_pll_data *hpll)
+static int hdmi_init_pll_data(struct platform_device *pdev,
+			      struct hdmi_pll_data *hpll)
 {
 	struct dss_pll *pll = &hpll->pll;
 	struct clk *clk;
@@ -145,23 +146,12 @@ static int dsi_init_pll_data(struct platform_device *pdev, struct hdmi_pll_data
 	pll->base = hpll->base;
 	pll->clkin = clk;
 
-	switch (omapdss_get_version()) {
-	case OMAPDSS_VER_OMAP4430_ES1:
-	case OMAPDSS_VER_OMAP4430_ES2:
-	case OMAPDSS_VER_OMAP4:
+	if (hpll->wp->version == 4)
 		pll->hw = &dss_omap4_hdmi_pll_hw;
-		break;
-
-	case OMAPDSS_VER_OMAP5:
-	case OMAPDSS_VER_DRA7xx:
+	else
 		pll->hw = &dss_omap5_hdmi_pll_hw;
-		break;
-
-	default:
-		return -ENODEV;
-	}
 
-	pll->ops = &dsi_pll_ops;
+	pll->ops = &hdmi_pll_ops;
 
 	r = dss_pll_register(pll);
 	if (r)
@@ -184,7 +174,7 @@ int hdmi_pll_init(struct platform_device *pdev, struct hdmi_pll_data *pll,
 	if (IS_ERR(pll->base))
 		return PTR_ERR(pll->base);
 
-	r = dsi_init_pll_data(pdev, pll);
+	r = hdmi_init_pll_data(pdev, pll);
 	if (r) {
 		DSSERR("failed to init HDMI PLL\n");
 		return r;
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c b/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c
index ab129df2e310..88034fbe0e9f 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c
@@ -178,9 +178,7 @@ void hdmi_wp_video_config_timing(struct hdmi_wp_data *wp,
 	 * However, we don't support OMAP5 ES1 at all, so we can just check for
 	 * OMAP4 here.
 	 */
-	if (omapdss_get_version() == OMAPDSS_VER_OMAP4430_ES1 ||
-	    omapdss_get_version() == OMAPDSS_VER_OMAP4430_ES2 ||
-	    omapdss_get_version() == OMAPDSS_VER_OMAP4)
+	if (wp->version == 4)
 		hsync_len_offset = 0;
 
 	timing_h |= FLD_VAL(vm->hback_porch, 31, 20);
@@ -235,9 +233,7 @@ void hdmi_wp_audio_config_format(struct hdmi_wp_data *wp,
 	DSSDBG("Enter hdmi_wp_audio_config_format\n");
 
 	r = hdmi_read_reg(wp->base, HDMI_WP_AUDIO_CFG);
-	if (omapdss_get_version() == OMAPDSS_VER_OMAP4430_ES1 ||
-	    omapdss_get_version() == OMAPDSS_VER_OMAP4430_ES2 ||
-	    omapdss_get_version() == OMAPDSS_VER_OMAP4) {
+	if (wp->version == 4) {
 		r = FLD_MOD(r, aud_fmt->stereo_channels, 26, 24);
 		r = FLD_MOD(r, aud_fmt->active_chnnls_msk, 23, 16);
 	}
@@ -282,7 +278,8 @@ int hdmi_wp_audio_core_req_enable(struct hdmi_wp_data *wp, bool enable)
 	return 0;
 }
 
-int hdmi_wp_init(struct platform_device *pdev, struct hdmi_wp_data *wp)
+int hdmi_wp_init(struct platform_device *pdev, struct hdmi_wp_data *wp,
+		 unsigned int version)
 {
 	struct resource *res;
 
@@ -292,6 +289,7 @@ int hdmi_wp_init(struct platform_device *pdev, struct hdmi_wp_data *wp)
 		return PTR_ERR(wp->base);
 
 	wp->phys_base = res->start;
+	wp->version = version;
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/omapdrm/dss/omapdss.h b/drivers/gpu/drm/omapdrm/dss/omapdss.h
index 85953a0bc7c2..47a331670963 100644
--- a/drivers/gpu/drm/omapdrm/dss/omapdss.h
+++ b/drivers/gpu/drm/omapdrm/dss/omapdss.h
@@ -25,6 +25,7 @@
 #include <video/videomode.h>
 #include <linux/platform_data/omapdss.h>
 #include <uapi/drm/drm_mode.h>
+#include <drm/drm_crtc.h>
 
 #define DISPC_IRQ_FRAMEDONE		(1 << 0)
 #define DISPC_IRQ_VSYNC			(1 << 1)
@@ -241,13 +242,6 @@ struct omap_dss_dsi_config {
 	enum omap_dss_dsi_trans_mode trans_mode;
 };
 
-/* Hardcoded videomodes for tv. Venc only uses these to
- * identify the mode, and does not actually use the configs
- * itself. However, the configs should be something that
- * a normal monitor can also show */
-extern const struct videomode omap_dss_pal_vm;
-extern const struct videomode omap_dss_ntsc_vm;
-
 struct omap_dss_cpr_coefs {
 	s16 rr, rg, rb;
 	s16 gr, gg, gb;
@@ -403,6 +397,14 @@ struct omapdss_hdmi_ops {
 	int (*read_edid)(struct omap_dss_device *dssdev, u8 *buf, int len);
 	bool (*detect)(struct omap_dss_device *dssdev);
 
+	int (*register_hpd_cb)(struct omap_dss_device *dssdev,
+			       void (*cb)(void *cb_data,
+					  enum drm_connector_status status),
+			       void *cb_data);
+	void (*unregister_hpd_cb)(struct omap_dss_device *dssdev);
+	void (*enable_hpd)(struct omap_dss_device *dssdev);
+	void (*disable_hpd)(struct omap_dss_device *dssdev);
+
 	int (*set_hdmi_mode)(struct omap_dss_device *dssdev, bool hdmi_mode);
 	int (*set_infoframe)(struct omap_dss_device *dssdev,
 		const struct hdmi_avi_infoframe *avi);
@@ -567,12 +569,19 @@ struct omap_dss_driver {
 	int (*read_edid)(struct omap_dss_device *dssdev, u8 *buf, int len);
 	bool (*detect)(struct omap_dss_device *dssdev);
 
+	int (*register_hpd_cb)(struct omap_dss_device *dssdev,
+			       void (*cb)(void *cb_data,
+					  enum drm_connector_status status),
+			       void *cb_data);
+	void (*unregister_hpd_cb)(struct omap_dss_device *dssdev);
+	void (*enable_hpd)(struct omap_dss_device *dssdev);
+	void (*disable_hpd)(struct omap_dss_device *dssdev);
+
 	int (*set_hdmi_mode)(struct omap_dss_device *dssdev, bool hdmi_mode);
 	int (*set_hdmi_infoframe)(struct omap_dss_device *dssdev,
 		const struct hdmi_avi_infoframe *avi);
 };
 
-enum omapdss_version omapdss_get_version(void);
 bool omapdss_is_initialized(void);
 
 int omap_dss_register_driver(struct omap_dss_driver *);
diff --git a/drivers/gpu/drm/omapdrm/dss/pll.c b/drivers/gpu/drm/omapdrm/dss/pll.c
index 5e221302768b..9d9d9d42009b 100644
--- a/drivers/gpu/drm/omapdrm/dss/pll.c
+++ b/drivers/gpu/drm/omapdrm/dss/pll.c
@@ -215,8 +215,8 @@ bool dss_pll_calc_a(const struct dss_pll *pll, unsigned long clkin,
 		dss_pll_calc_func func, void *data)
 {
 	const struct dss_pll_hw *hw = pll->hw;
-	int n, n_min, n_max;
-	int m, m_min, m_max;
+	int n, n_start, n_stop, n_inc;
+	int m, m_start, m_stop, m_inc;
 	unsigned long fint, clkdco;
 	unsigned long pll_hw_max;
 	unsigned long fint_hw_min, fint_hw_max;
@@ -226,22 +226,33 @@ bool dss_pll_calc_a(const struct dss_pll *pll, unsigned long clkin,
 	fint_hw_min = hw->fint_min;
 	fint_hw_max = hw->fint_max;
 
-	n_min = max(DIV_ROUND_UP(clkin, fint_hw_max), 1ul);
-	n_max = min((unsigned)(clkin / fint_hw_min), hw->n_max);
+	n_start = max(DIV_ROUND_UP(clkin, fint_hw_max), 1ul);
+	n_stop = min((unsigned)(clkin / fint_hw_min), hw->n_max);
+	n_inc = 1;
+
+	if (hw->errata_i886) {
+		swap(n_start, n_stop);
+		n_inc = -1;
+	}
 
 	pll_max = pll_max ? pll_max : ULONG_MAX;
 
-	/* Try to find high N & M to avoid jitter (DRA7 errata i886) */
-	for (n = n_max; n >= n_min; --n) {
+	for (n = n_start; n != n_stop; n += n_inc) {
 		fint = clkin / n;
 
-		m_min = max(DIV_ROUND_UP(DIV_ROUND_UP(pll_min, fint), 2),
+		m_start = max(DIV_ROUND_UP(DIV_ROUND_UP(pll_min, fint), 2),
 				1ul);
-		m_max = min3((unsigned)(pll_max / fint / 2),
+		m_stop = min3((unsigned)(pll_max / fint / 2),
 				(unsigned)(pll_hw_max / fint / 2),
 				hw->m_max);
+		m_inc = 1;
+
+		if (hw->errata_i886) {
+			swap(m_start, m_stop);
+			m_inc = -1;
+		}
 
-		for (m = m_max; m >= m_min; --m) {
+		for (m = m_start; m != m_stop; m += m_inc) {
 			clkdco = 2 * m * fint;
 
 			if (func(n, m, fint, clkdco, data))
diff --git a/drivers/gpu/drm/omapdrm/dss/venc.c b/drivers/gpu/drm/omapdrm/dss/venc.c
index a6bfb3918b8d..d58da6f32693 100644
--- a/drivers/gpu/drm/omapdrm/dss/venc.c
+++ b/drivers/gpu/drm/omapdrm/dss/venc.c
@@ -37,10 +37,10 @@
 #include <linux/of.h>
 #include <linux/of_graph.h>
 #include <linux/component.h>
+#include <linux/sys_soc.h>
 
 #include "omapdss.h"
 #include "dss.h"
-#include "dss_features.h"
 
 /* Venc registers */
 #define VENC_REV_ID				0x00
@@ -263,7 +263,13 @@ static const struct venc_config venc_config_pal_bdghi = {
 	.fid_ext_start_y__fid_ext_offset_y	= 0x01380005,
 };
 
-const struct videomode omap_dss_pal_vm = {
+enum venc_videomode {
+	VENC_MODE_UNKNOWN,
+	VENC_MODE_PAL,
+	VENC_MODE_NTSC,
+};
+
+static const struct videomode omap_dss_pal_vm = {
 	.hactive	= 720,
 	.vactive	= 574,
 	.pixelclock	= 13500000,
@@ -279,9 +285,8 @@ const struct videomode omap_dss_pal_vm = {
 			  DISPLAY_FLAGS_PIXDATA_POSEDGE |
 			  DISPLAY_FLAGS_SYNC_NEGEDGE,
 };
-EXPORT_SYMBOL(omap_dss_pal_vm);
 
-const struct videomode omap_dss_ntsc_vm = {
+static const struct videomode omap_dss_ntsc_vm = {
 	.hactive	= 720,
 	.vactive	= 482,
 	.pixelclock	= 13500000,
@@ -297,7 +302,24 @@ const struct videomode omap_dss_ntsc_vm = {
 			  DISPLAY_FLAGS_PIXDATA_POSEDGE |
 			  DISPLAY_FLAGS_SYNC_NEGEDGE,
 };
-EXPORT_SYMBOL(omap_dss_ntsc_vm);
+
+static enum venc_videomode venc_get_videomode(const struct videomode *vm)
+{
+	if (!(vm->flags & DISPLAY_FLAGS_INTERLACED))
+		return VENC_MODE_UNKNOWN;
+
+	if (vm->pixelclock == omap_dss_pal_vm.pixelclock &&
+	    vm->hactive == omap_dss_pal_vm.hactive &&
+	    vm->vactive == omap_dss_pal_vm.vactive)
+		return VENC_MODE_PAL;
+
+	if (vm->pixelclock == omap_dss_ntsc_vm.pixelclock &&
+	    vm->hactive == omap_dss_ntsc_vm.hactive &&
+	    vm->vactive == omap_dss_ntsc_vm.vactive)
+		return VENC_MODE_NTSC;
+
+	return VENC_MODE_UNKNOWN;
+}
 
 static struct {
 	struct platform_device *pdev;
@@ -311,6 +333,7 @@ static struct {
 	struct videomode vm;
 	enum omap_dss_venc_type type;
 	bool invert_polarity;
+	bool requires_tv_dac_clk;
 
 	struct omap_dss_device output;
 } venc;
@@ -424,14 +447,14 @@ static void venc_runtime_put(void)
 
 static const struct venc_config *venc_timings_to_config(struct videomode *vm)
 {
-	if (memcmp(&omap_dss_pal_vm, vm, sizeof(*vm)) == 0)
+	switch (venc_get_videomode(vm)) {
+	default:
+		WARN_ON_ONCE(1);
+	case VENC_MODE_PAL:
 		return &venc_config_pal_trm;
-
-	if (memcmp(&omap_dss_ntsc_vm, vm, sizeof(*vm)) == 0)
+	case VENC_MODE_NTSC:
 		return &venc_config_ntsc_trm;
-
-	BUG();
-	return NULL;
+	}
 }
 
 static int venc_power_on(struct omap_dss_device *dssdev)
@@ -542,15 +565,28 @@ static void venc_display_disable(struct omap_dss_device *dssdev)
 static void venc_set_timings(struct omap_dss_device *dssdev,
 			     struct videomode *vm)
 {
+	struct videomode actual_vm;
+
 	DSSDBG("venc_set_timings\n");
 
 	mutex_lock(&venc.venc_lock);
 
+	switch (venc_get_videomode(vm)) {
+	default:
+		WARN_ON_ONCE(1);
+	case VENC_MODE_PAL:
+		actual_vm = omap_dss_pal_vm;
+		break;
+	case VENC_MODE_NTSC:
+		actual_vm = omap_dss_ntsc_vm;
+		break;
+	}
+
 	/* Reset WSS data when the TV standard changes. */
-	if (memcmp(&venc.vm, vm, sizeof(*vm)))
+	if (memcmp(&venc.vm, &actual_vm, sizeof(actual_vm)))
 		venc.wss_data = 0;
 
-	venc.vm = *vm;
+	venc.vm = actual_vm;
 
 	dispc_set_tv_pclk(13500000);
 
@@ -562,13 +598,13 @@ static int venc_check_timings(struct omap_dss_device *dssdev,
 {
 	DSSDBG("venc_check_timings\n");
 
-	if (memcmp(&omap_dss_pal_vm, vm, sizeof(*vm)) == 0)
-		return 0;
-
-	if (memcmp(&omap_dss_ntsc_vm, vm, sizeof(*vm)) == 0)
+	switch (venc_get_videomode(vm)) {
+	case VENC_MODE_PAL:
+	case VENC_MODE_NTSC:
 		return 0;
-
-	return -EINVAL;
+	default:
+		return -EINVAL;
+	}
 }
 
 static void venc_get_timings(struct omap_dss_device *dssdev,
@@ -693,7 +729,7 @@ static int venc_get_clocks(struct platform_device *pdev)
 {
 	struct clk *clk;
 
-	if (dss_has_feature(FEAT_VENC_REQUIRES_TV_DAC_CLK)) {
+	if (venc.requires_tv_dac_clk) {
 		clk = devm_clk_get(&pdev->dev, "tv_dac_clk");
 		if (IS_ERR(clk)) {
 			DSSERR("can't get tv_dac_clk\n");
@@ -828,6 +864,12 @@ err:
 }
 
 /* VENC HW IP initialisation */
+static const struct soc_device_attribute venc_soc_devices[] = {
+	{ .machine = "OMAP3[45]*" },
+	{ .machine = "AM35*" },
+	{ /* sentinel */ }
+};
+
 static int venc_bind(struct device *dev, struct device *master, void *data)
 {
 	struct platform_device *pdev = to_platform_device(dev);
@@ -837,6 +879,10 @@ static int venc_bind(struct device *dev, struct device *master, void *data)
 
 	venc.pdev = pdev;
 
+	/* The OMAP34xx, OMAP35xx and AM35xx VENC require the TV DAC clock. */
+	if (soc_device_match(venc_soc_devices))
+		venc.requires_tv_dac_clk = true;
+
 	mutex_init(&venc.venc_lock);
 
 	venc.wss_data = 0;
diff --git a/drivers/gpu/drm/omapdrm/dss/video-pll.c b/drivers/gpu/drm/omapdrm/dss/video-pll.c
index fbd1263a29a4..38a239cc5e04 100644
--- a/drivers/gpu/drm/omapdrm/dss/video-pll.c
+++ b/drivers/gpu/drm/omapdrm/dss/video-pll.c
@@ -19,7 +19,6 @@
 
 #include "omapdss.h"
 #include "dss.h"
-#include "dss_features.h"
 
 struct dss_video_pll {
 	struct dss_pll pll;
@@ -131,6 +130,8 @@ static const struct dss_pll_hw dss_dra7_video_pll_hw = {
 	.mX_lsb[3] = 5,
 
 	.has_refsel = true,
+
+	.errata_i886 = true,
 };
 
 struct dss_pll *dss_video_pll_init(struct platform_device *pdev, int id,
diff --git a/drivers/gpu/drm/omapdrm/omap_connector.c b/drivers/gpu/drm/omapdrm/omap_connector.c
index c24b6b783e9a..aa5ba9ae2191 100644
--- a/drivers/gpu/drm/omapdrm/omap_connector.c
+++ b/drivers/gpu/drm/omapdrm/omap_connector.c
@@ -35,6 +35,23 @@ struct omap_connector {
 	bool hdmi_mode;
 };
 
+static void omap_connector_hpd_cb(void *cb_data,
+				  enum drm_connector_status status)
+{
+	struct omap_connector *omap_connector = cb_data;
+	struct drm_connector *connector = &omap_connector->base;
+	struct drm_device *dev = connector->dev;
+	enum drm_connector_status old_status;
+
+	mutex_lock(&dev->mode_config.mutex);
+	old_status = connector->status;
+	connector->status = status;
+	mutex_unlock(&dev->mode_config.mutex);
+
+	if (old_status != status)
+		drm_kms_helper_hotplug_event(dev);
+}
+
 bool omap_connector_get_hdmi_mode(struct drm_connector *connector)
 {
 	struct omap_connector *omap_connector = to_omap_connector(connector);
@@ -75,6 +92,10 @@ static void omap_connector_destroy(struct drm_connector *connector)
 	struct omap_dss_device *dssdev = omap_connector->dssdev;
 
 	DBG("%s", omap_connector->dssdev->name);
+	if (connector->polled == DRM_CONNECTOR_POLL_HPD &&
+	    dssdev->driver->unregister_hpd_cb) {
+		dssdev->driver->unregister_hpd_cb(dssdev);
+	}
 	drm_connector_unregister(connector);
 	drm_connector_cleanup(connector);
 	kfree(omap_connector);
@@ -195,7 +216,6 @@ static int omap_connector_mode_valid(struct drm_connector *connector,
 }
 
 static const struct drm_connector_funcs omap_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.reset = drm_atomic_helper_connector_reset,
 	.detect = omap_connector_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
@@ -216,6 +236,7 @@ struct drm_connector *omap_connector_init(struct drm_device *dev,
 {
 	struct drm_connector *connector = NULL;
 	struct omap_connector *omap_connector;
+	bool hpd_supported = false;
 
 	DBG("%s", dssdev->name);
 
@@ -233,7 +254,20 @@ struct drm_connector *omap_connector_init(struct drm_device *dev,
 				connector_type);
 	drm_connector_helper_add(connector, &omap_connector_helper_funcs);
 
-	if (dssdev->driver->detect)
+	if (dssdev->driver->register_hpd_cb) {
+		int ret = dssdev->driver->register_hpd_cb(dssdev,
+							  omap_connector_hpd_cb,
+							  omap_connector);
+		if (!ret)
+			hpd_supported = true;
+		else if (ret != -ENOTSUPP)
+			DBG("%s: Failed to register HPD callback (%d).",
+			    dssdev->name, ret);
+	}
+
+	if (hpd_supported)
+		connector->polled = DRM_CONNECTOR_POLL_HPD;
+	else if (dssdev->driver->detect)
 		connector->polled = DRM_CONNECTOR_POLL_CONNECT |
 				    DRM_CONNECTOR_POLL_DISCONNECT;
 	else
diff --git a/drivers/gpu/drm/omapdrm/omap_crtc.c b/drivers/gpu/drm/omapdrm/omap_crtc.c
index dd0ef40ca469..cc85c16cbc2a 100644
--- a/drivers/gpu/drm/omapdrm/omap_crtc.c
+++ b/drivers/gpu/drm/omapdrm/omap_crtc.c
@@ -26,6 +26,16 @@
 
 #include "omap_drv.h"
 
+#define to_omap_crtc_state(x) container_of(x, struct omap_crtc_state, base)
+
+struct omap_crtc_state {
+	/* Must be first. */
+	struct drm_crtc_state base;
+	/* Shadow values for legacy userspace support. */
+	unsigned int rotation;
+	unsigned int zpos;
+};
+
 #define to_omap_crtc(x) container_of(x, struct omap_crtc, base)
 
 struct omap_crtc {
@@ -356,7 +366,8 @@ static void omap_crtc_arm_event(struct drm_crtc *crtc)
 	}
 }
 
-static void omap_crtc_enable(struct drm_crtc *crtc)
+static void omap_crtc_atomic_enable(struct drm_crtc *crtc,
+				    struct drm_crtc_state *old_state)
 {
 	struct omap_crtc *omap_crtc = to_omap_crtc(crtc);
 	int ret;
@@ -372,7 +383,8 @@ static void omap_crtc_enable(struct drm_crtc *crtc)
 	spin_unlock_irq(&crtc->dev->event_lock);
 }
 
-static void omap_crtc_disable(struct drm_crtc *crtc)
+static void omap_crtc_atomic_disable(struct drm_crtc *crtc,
+				     struct drm_crtc_state *old_state)
 {
 	struct omap_crtc *omap_crtc = to_omap_crtc(crtc);
 
@@ -443,6 +455,8 @@ static void omap_crtc_mode_set_nofb(struct drm_crtc *crtc)
 static int omap_crtc_atomic_check(struct drm_crtc *crtc,
 				struct drm_crtc_state *state)
 {
+	struct drm_plane_state *pri_state;
+
 	if (state->color_mgmt_changed && state->gamma_lut) {
 		uint length = state->gamma_lut->length /
 			sizeof(struct drm_color_lut);
@@ -451,6 +465,16 @@ static int omap_crtc_atomic_check(struct drm_crtc *crtc,
 			return -EINVAL;
 	}
 
+	pri_state = drm_atomic_get_new_plane_state(state->state, crtc->primary);
+	if (pri_state) {
+		struct omap_crtc_state *omap_crtc_state =
+			to_omap_crtc_state(state);
+
+		/* Mirror new values for zpos and rotation in omap_crtc_state */
+		omap_crtc_state->zpos = pri_state->zpos;
+		omap_crtc_state->rotation = pri_state->rotation;
+	}
+
 	return 0;
 }
 
@@ -496,39 +520,32 @@ static void omap_crtc_atomic_flush(struct drm_crtc *crtc,
 	spin_unlock_irq(&crtc->dev->event_lock);
 }
 
-static bool omap_crtc_is_plane_prop(struct drm_crtc *crtc,
-	struct drm_property *property)
-{
-	struct drm_device *dev = crtc->dev;
-	struct omap_drm_private *priv = dev->dev_private;
-
-	return property == priv->zorder_prop ||
-		property == crtc->primary->rotation_property;
-}
-
 static int omap_crtc_atomic_set_property(struct drm_crtc *crtc,
 					 struct drm_crtc_state *state,
 					 struct drm_property *property,
 					 uint64_t val)
 {
-	if (omap_crtc_is_plane_prop(crtc, property)) {
-		struct drm_plane_state *plane_state;
-		struct drm_plane *plane = crtc->primary;
-
-		/*
-		 * Delegate property set to the primary plane. Get the plane
-		 * state and set the property directly.
-		 */
-
-		plane_state = drm_atomic_get_plane_state(state->state, plane);
-		if (IS_ERR(plane_state))
-			return PTR_ERR(plane_state);
+	struct omap_drm_private *priv = crtc->dev->dev_private;
+	struct drm_plane_state *plane_state;
 
-		return drm_atomic_plane_set_property(plane, plane_state,
-				property, val);
-	}
+	/*
+	 * Delegate property set to the primary plane. Get the plane state and
+	 * set the property directly, the shadow copy will be assigned in the
+	 * omap_crtc_atomic_check callback. This way updates to plane state will
+	 * always be mirrored in the crtc state correctly.
+	 */
+	plane_state = drm_atomic_get_plane_state(state->state, crtc->primary);
+	if (IS_ERR(plane_state))
+		return PTR_ERR(plane_state);
+
+	if (property == crtc->primary->rotation_property)
+		plane_state->rotation = val;
+	else if (property == priv->zorder_prop)
+		plane_state->zpos = val;
+	else
+		return -EINVAL;
 
-	return -EINVAL;
+	return 0;
 }
 
 static int omap_crtc_atomic_get_property(struct drm_crtc *crtc,
@@ -536,28 +553,60 @@ static int omap_crtc_atomic_get_property(struct drm_crtc *crtc,
 					 struct drm_property *property,
 					 uint64_t *val)
 {
-	if (omap_crtc_is_plane_prop(crtc, property)) {
-		/*
-		 * Delegate property get to the primary plane. The
-		 * drm_atomic_plane_get_property() function isn't exported, but
-		 * can be called through drm_object_property_get_value() as that
-		 * will call drm_atomic_get_property() for atomic drivers.
-		 */
-		return drm_object_property_get_value(&crtc->primary->base,
-				property, val);
-	}
+	struct omap_drm_private *priv = crtc->dev->dev_private;
+	struct omap_crtc_state *omap_state = to_omap_crtc_state(state);
+
+	if (property == crtc->primary->rotation_property)
+		*val = omap_state->rotation;
+	else if (property == priv->zorder_prop)
+		*val = omap_state->zpos;
+	else
+		return -EINVAL;
+
+	return 0;
+}
+
+static void omap_crtc_reset(struct drm_crtc *crtc)
+{
+	if (crtc->state)
+		__drm_atomic_helper_crtc_destroy_state(crtc->state);
+
+	kfree(crtc->state);
+	crtc->state = kzalloc(sizeof(struct omap_crtc_state), GFP_KERNEL);
+
+	if (crtc->state)
+		crtc->state->crtc = crtc;
+}
+
+static struct drm_crtc_state *
+omap_crtc_duplicate_state(struct drm_crtc *crtc)
+{
+	struct omap_crtc_state *state, *current_state;
+
+	if (WARN_ON(!crtc->state))
+		return NULL;
+
+	current_state = to_omap_crtc_state(crtc->state);
+
+	state = kmalloc(sizeof(*state), GFP_KERNEL);
+	if (!state)
+		return NULL;
+
+	__drm_atomic_helper_crtc_duplicate_state(crtc, &state->base);
+
+	state->zpos = current_state->zpos;
+	state->rotation = current_state->rotation;
 
-	return -EINVAL;
+	return &state->base;
 }
 
 static const struct drm_crtc_funcs omap_crtc_funcs = {
-	.reset = drm_atomic_helper_crtc_reset,
+	.reset = omap_crtc_reset,
 	.set_config = drm_atomic_helper_set_config,
 	.destroy = omap_crtc_destroy,
 	.page_flip = drm_atomic_helper_page_flip,
 	.gamma_set = drm_atomic_helper_legacy_gamma_set,
-	.set_property = drm_atomic_helper_crtc_set_property,
-	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
+	.atomic_duplicate_state = omap_crtc_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
 	.atomic_set_property = omap_crtc_atomic_set_property,
 	.atomic_get_property = omap_crtc_atomic_get_property,
@@ -567,11 +616,11 @@ static const struct drm_crtc_funcs omap_crtc_funcs = {
 
 static const struct drm_crtc_helper_funcs omap_crtc_helper_funcs = {
 	.mode_set_nofb = omap_crtc_mode_set_nofb,
-	.disable = omap_crtc_disable,
-	.enable = omap_crtc_enable,
 	.atomic_check = omap_crtc_atomic_check,
 	.atomic_begin = omap_crtc_atomic_begin,
 	.atomic_flush = omap_crtc_atomic_flush,
+	.atomic_enable = omap_crtc_atomic_enable,
+	.atomic_disable = omap_crtc_atomic_disable,
 };
 
 /* -----------------------------------------------------------------------------
diff --git a/drivers/gpu/drm/omapdrm/omap_drv.c b/drivers/gpu/drm/omapdrm/omap_drv.c
index 022029ea6972..cdf5b0601eba 100644
--- a/drivers/gpu/drm/omapdrm/omap_drv.c
+++ b/drivers/gpu/drm/omapdrm/omap_drv.c
@@ -57,13 +57,13 @@ static void omap_fb_output_poll_changed(struct drm_device *dev)
 static void omap_atomic_wait_for_completion(struct drm_device *dev,
 					    struct drm_atomic_state *old_state)
 {
-	struct drm_crtc_state *old_crtc_state;
+	struct drm_crtc_state *new_crtc_state;
 	struct drm_crtc *crtc;
 	unsigned int i;
 	int ret;
 
-	for_each_crtc_in_state(old_state, crtc, old_crtc_state, i) {
-		if (!crtc->state->enable)
+	for_each_new_crtc_in_state(old_state, crtc, new_crtc_state, i) {
+		if (!new_crtc_state->active)
 			continue;
 
 		ret = omap_crtc_wait_pending(crtc);
@@ -84,23 +84,36 @@ static void omap_atomic_commit_tail(struct drm_atomic_state *old_state)
 	/* Apply the atomic update. */
 	drm_atomic_helper_commit_modeset_disables(dev, old_state);
 
-	/* With the current dss dispc implementation we have to enable
-	 * the new modeset before we can commit planes. The dispc ovl
-	 * configuration relies on the video mode configuration been
-	 * written into the HW when the ovl configuration is
-	 * calculated.
-	 *
-	 * This approach is not ideal because after a mode change the
-	 * plane update is executed only after the first vblank
-	 * interrupt. The dispc implementation should be fixed so that
-	 * it is able use uncommitted drm state information.
-	 */
-	drm_atomic_helper_commit_modeset_enables(dev, old_state);
-	omap_atomic_wait_for_completion(dev, old_state);
-
-	drm_atomic_helper_commit_planes(dev, old_state, 0);
-
-	drm_atomic_helper_commit_hw_done(old_state);
+	if (priv->omaprev != 0x3430) {
+		/* With the current dss dispc implementation we have to enable
+		 * the new modeset before we can commit planes. The dispc ovl
+		 * configuration relies on the video mode configuration been
+		 * written into the HW when the ovl configuration is
+		 * calculated.
+		 *
+		 * This approach is not ideal because after a mode change the
+		 * plane update is executed only after the first vblank
+		 * interrupt. The dispc implementation should be fixed so that
+		 * it is able use uncommitted drm state information.
+		 */
+		drm_atomic_helper_commit_modeset_enables(dev, old_state);
+		omap_atomic_wait_for_completion(dev, old_state);
+
+		drm_atomic_helper_commit_planes(dev, old_state, 0);
+
+		drm_atomic_helper_commit_hw_done(old_state);
+	} else {
+		/*
+		 * OMAP3 DSS seems to have issues with the work-around above,
+		 * resulting in endless sync losts if a crtc is enabled without
+		 * a plane. For now, skip the WA for OMAP3.
+		 */
+		drm_atomic_helper_commit_planes(dev, old_state, 0);
+
+		drm_atomic_helper_commit_modeset_enables(dev, old_state);
+
+		drm_atomic_helper_commit_hw_done(old_state);
+	}
 
 	/*
 	 * Wait for completion of the page flips to ensure that old buffers
@@ -324,6 +337,32 @@ static int omap_modeset_init(struct drm_device *dev)
 }
 
 /*
+ * Enable the HPD in external components if supported
+ */
+static void omap_modeset_enable_external_hpd(void)
+{
+	struct omap_dss_device *dssdev = NULL;
+
+	for_each_dss_dev(dssdev) {
+		if (dssdev->driver->enable_hpd)
+			dssdev->driver->enable_hpd(dssdev);
+	}
+}
+
+/*
+ * Disable the HPD in external components if supported
+ */
+static void omap_modeset_disable_external_hpd(void)
+{
+	struct omap_dss_device *dssdev = NULL;
+
+	for_each_dss_dev(dssdev) {
+		if (dssdev->driver->disable_hpd)
+			dssdev->driver->disable_hpd(dssdev);
+	}
+}
+
+/*
  * drm ioctl funcs
  */
 
@@ -438,44 +477,11 @@ static int dev_open(struct drm_device *dev, struct drm_file *file)
  */
 static void dev_lastclose(struct drm_device *dev)
 {
-	int i;
-
-	/* we don't support vga_switcheroo.. so just make sure the fbdev
-	 * mode is active
-	 */
 	struct omap_drm_private *priv = dev->dev_private;
 	int ret;
 
 	DBG("lastclose: dev=%p", dev);
 
-	/* need to restore default rotation state.. not sure
-	 * if there is a cleaner way to restore properties to
-	 * default state?  Maybe a flag that properties should
-	 * automatically be restored to default state on
-	 * lastclose?
-	 */
-	for (i = 0; i < priv->num_crtcs; i++) {
-		struct drm_crtc *crtc = priv->crtcs[i];
-
-		if (!crtc->primary->rotation_property)
-			continue;
-
-		drm_object_property_set_value(&crtc->base,
-					      crtc->primary->rotation_property,
-					      DRM_MODE_ROTATE_0);
-	}
-
-	for (i = 0; i < priv->num_planes; i++) {
-		struct drm_plane *plane = priv->planes[i];
-
-		if (!plane->rotation_property)
-			continue;
-
-		drm_object_property_set_value(&plane->base,
-					      plane->rotation_property,
-					      DRM_MODE_ROTATE_0);
-	}
-
 	if (priv->fbdev) {
 		ret = drm_fb_helper_restore_fbdev_mode_unlocked(priv->fbdev);
 		if (ret)
@@ -517,7 +523,6 @@ static struct drm_driver omap_drm_driver = {
 	.gem_vm_ops = &omap_gem_vm_ops,
 	.dumb_create = omap_gem_dumb_create,
 	.dumb_map_offset = omap_gem_dumb_map_offset,
-	.dumb_destroy = drm_gem_dumb_destroy,
 	.ioctls = ioctls,
 	.num_ioctls = DRM_OMAP_NUM_IOCTLS,
 	.fops = &omapdriver_fops,
@@ -550,6 +555,12 @@ static int pdev_probe(struct platform_device *pdev)
 	if (omapdss_is_initialized() == false)
 		return -EPROBE_DEFER;
 
+	ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to set the DMA mask\n");
+		return ret;
+	}
+
 	omap_crtc_pre_init();
 
 	ret = omap_connect_dssdevs();
@@ -603,6 +614,7 @@ static int pdev_probe(struct platform_device *pdev)
 	priv->fbdev = omap_fbdev_init(ddev);
 
 	drm_kms_helper_poll_init(ddev);
+	omap_modeset_enable_external_hpd();
 
 	/*
 	 * Register the DRM device with the core and the connectors with
@@ -615,6 +627,7 @@ static int pdev_probe(struct platform_device *pdev)
 	return 0;
 
 err_cleanup_helpers:
+	omap_modeset_disable_external_hpd();
 	drm_kms_helper_poll_fini(ddev);
 	if (priv->fbdev)
 		omap_fbdev_free(ddev);
@@ -643,6 +656,7 @@ static int pdev_remove(struct platform_device *pdev)
 
 	drm_dev_unregister(ddev);
 
+	omap_modeset_disable_external_hpd();
 	drm_kms_helper_poll_fini(ddev);
 
 	if (priv->fbdev)
@@ -734,7 +748,7 @@ static SIMPLE_DEV_PM_OPS(omapdrm_pm_ops, omap_drm_suspend, omap_drm_resume);
 
 static struct platform_driver pdev = {
 	.driver = {
-		.name = DRIVER_NAME,
+		.name = "omapdrm",
 		.pm = &omapdrm_pm_ops,
 	},
 	.probe = pdev_probe,
diff --git a/drivers/gpu/drm/omapdrm/omap_encoder.c b/drivers/gpu/drm/omapdrm/omap_encoder.c
index 86c977b7189a..624f5b50b755 100644
--- a/drivers/gpu/drm/omapdrm/omap_encoder.c
+++ b/drivers/gpu/drm/omapdrm/omap_encoder.c
@@ -85,7 +85,8 @@ static void omap_encoder_mode_set(struct drm_encoder *encoder,
 	if (hdmi_mode && dssdev->driver->set_hdmi_infoframe) {
 		struct hdmi_avi_infoframe avi;
 
-		r = drm_hdmi_avi_infoframe_from_display_mode(&avi, adjusted_mode);
+		r = drm_hdmi_avi_infoframe_from_display_mode(&avi, adjusted_mode,
+							     false);
 		if (r == 0)
 			dssdev->driver->set_hdmi_infoframe(dssdev, &avi);
 	}
diff --git a/drivers/gpu/drm/omapdrm/omap_fb.c b/drivers/gpu/drm/omapdrm/omap_fb.c
index ddf7a457951b..b1a762b70cbf 100644
--- a/drivers/gpu/drm/omapdrm/omap_fb.c
+++ b/drivers/gpu/drm/omapdrm/omap_fb.c
@@ -379,7 +379,7 @@ struct drm_framebuffer *omap_framebuffer_create(struct drm_device *dev,
 	return fb;
 
 error:
-	while (--i > 0)
+	while (--i >= 0)
 		drm_gem_object_unreference_unlocked(bos[i]);
 
 	return fb;
diff --git a/drivers/gpu/drm/omapdrm/omap_fbdev.c b/drivers/gpu/drm/omapdrm/omap_fbdev.c
index daf81a0a2899..9273118040b7 100644
--- a/drivers/gpu/drm/omapdrm/omap_fbdev.c
+++ b/drivers/gpu/drm/omapdrm/omap_fbdev.c
@@ -184,7 +184,6 @@ static int omap_fbdev_create(struct drm_fb_helper *helper,
 	helper->fb = fb;
 
 	fbi->par = helper;
-	fbi->flags = FBINFO_DEFAULT;
 	fbi->fbops = &omap_fb_ops;
 
 	strcpy(fbi->fix.id, MODULE_NAME);
diff --git a/drivers/gpu/drm/omapdrm/omap_gem_dmabuf.c b/drivers/gpu/drm/omapdrm/omap_gem_dmabuf.c
index 863a881dd7cd..afdbad5c866a 100644
--- a/drivers/gpu/drm/omapdrm/omap_gem_dmabuf.c
+++ b/drivers/gpu/drm/omapdrm/omap_gem_dmabuf.c
@@ -144,7 +144,7 @@ static int omap_gem_dmabuf_mmap(struct dma_buf *buffer,
 	return omap_gem_mmap_obj(obj, vma);
 }
 
-static struct dma_buf_ops omap_dmabuf_ops = {
+static const struct dma_buf_ops omap_dmabuf_ops = {
 	.map_dma_buf = omap_gem_map_dma_buf,
 	.unmap_dma_buf = omap_gem_unmap_dma_buf,
 	.release = drm_gem_dmabuf_release,
diff --git a/drivers/gpu/drm/omapdrm/omap_plane.c b/drivers/gpu/drm/omapdrm/omap_plane.c
index 2160f64548e0..15e5d5d325c6 100644
--- a/drivers/gpu/drm/omapdrm/omap_plane.c
+++ b/drivers/gpu/drm/omapdrm/omap_plane.c
@@ -235,7 +235,6 @@ static const struct drm_plane_funcs omap_plane_funcs = {
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.reset = omap_plane_reset,
 	.destroy = omap_plane_destroy,
-	.set_property = drm_atomic_helper_plane_set_property,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
 	.atomic_set_property = omap_plane_atomic_set_property,
@@ -291,7 +290,7 @@ struct drm_plane *omap_plane_init(struct drm_device *dev,
 
 	ret = drm_universal_plane_init(dev, plane, possible_crtcs,
 				       &omap_plane_funcs, formats,
-				       nformats, type, NULL);
+				       nformats, NULL, type, NULL);
 	if (ret < 0)
 		goto error;
 
diff --git a/drivers/gpu/drm/panel/panel-lvds.c b/drivers/gpu/drm/panel/panel-lvds.c
index 3216aa9a88d6..e2d57c01200b 100644
--- a/drivers/gpu/drm/panel/panel-lvds.c
+++ b/drivers/gpu/drm/panel/panel-lvds.c
@@ -143,14 +143,14 @@ static int panel_lvds_parse_dt(struct panel_lvds *lvds)
 
 	ret = of_property_read_u32(np, "width-mm", &lvds->width);
 	if (ret < 0) {
-		dev_err(lvds->dev, "%s: invalid or missing %s DT property\n",
-			of_node_full_name(np), "width-mm");
+		dev_err(lvds->dev, "%pOF: invalid or missing %s DT property\n",
+			np, "width-mm");
 		return -ENODEV;
 	}
 	ret = of_property_read_u32(np, "height-mm", &lvds->height);
 	if (ret < 0) {
-		dev_err(lvds->dev, "%s: invalid or missing %s DT property\n",
-			of_node_full_name(np), "height-mm");
+		dev_err(lvds->dev, "%pOF: invalid or missing %s DT property\n",
+			np, "height-mm");
 		return -ENODEV;
 	}
 
@@ -158,8 +158,8 @@ static int panel_lvds_parse_dt(struct panel_lvds *lvds)
 
 	ret = of_property_read_string(np, "data-mapping", &mapping);
 	if (ret < 0) {
-		dev_err(lvds->dev, "%s: invalid or missing %s DT property\n",
-			of_node_full_name(np), "data-mapping");
+		dev_err(lvds->dev, "%pOF: invalid or missing %s DT property\n",
+			np, "data-mapping");
 		return -ENODEV;
 	}
 
@@ -170,8 +170,8 @@ static int panel_lvds_parse_dt(struct panel_lvds *lvds)
 	} else if (!strcmp(mapping, "vesa-24")) {
 		lvds->bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG;
 	} else {
-		dev_err(lvds->dev, "%s: invalid or missing %s DT property\n",
-			of_node_full_name(np), "data-mapping");
+		dev_err(lvds->dev, "%pOF: invalid or missing %s DT property\n",
+			np, "data-mapping");
 		return -EINVAL;
 	}
 
diff --git a/drivers/gpu/drm/pl111/pl111_connector.c b/drivers/gpu/drm/pl111/pl111_connector.c
index 3f213d7e7692..d335f9a29ce4 100644
--- a/drivers/gpu/drm/pl111/pl111_connector.c
+++ b/drivers/gpu/drm/pl111/pl111_connector.c
@@ -69,7 +69,6 @@ const struct drm_connector_funcs connector_funcs = {
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = pl111_connector_destroy,
 	.detect = pl111_connector_detect,
-	.dpms = drm_atomic_helper_connector_dpms,
 	.reset = drm_atomic_helper_connector_reset,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
diff --git a/drivers/gpu/drm/pl111/pl111_display.c b/drivers/gpu/drm/pl111/pl111_display.c
index c6ca4f1bbd49..b58c988d9da0 100644
--- a/drivers/gpu/drm/pl111/pl111_display.c
+++ b/drivers/gpu/drm/pl111/pl111_display.c
@@ -23,6 +23,7 @@
 #include <drm/drmP.h>
 #include <drm/drm_panel.h>
 #include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_fb_cma_helper.h>
 
 #include "pl111_drm.h"
@@ -274,7 +275,7 @@ void pl111_disable_vblank(struct drm_device *drm, unsigned int crtc)
 static int pl111_display_prepare_fb(struct drm_simple_display_pipe *pipe,
 				    struct drm_plane_state *plane_state)
 {
-	return drm_fb_cma_prepare_fb(&pipe->plane, plane_state);
+	return drm_gem_fb_prepare_fb(&pipe->plane, plane_state);
 }
 
 static const struct drm_simple_display_pipe_funcs pl111_display_funcs = {
@@ -457,7 +458,7 @@ int pl111_display_init(struct drm_device *drm)
 	ret = drm_simple_display_pipe_init(drm, &priv->pipe,
 					   &pl111_display_funcs,
 					   formats, ARRAY_SIZE(formats),
-					   &priv->connector.connector);
+					   NULL, &priv->connector.connector);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/pl111/pl111_drv.c b/drivers/gpu/drm/pl111/pl111_drv.c
index ac8771be70b0..581c452cede1 100644
--- a/drivers/gpu/drm/pl111/pl111_drv.c
+++ b/drivers/gpu/drm/pl111/pl111_drv.c
@@ -66,14 +66,15 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_fb_cma_helper.h>
 
 #include "pl111_drm.h"
 
 #define DRIVER_DESC      "DRM module for PL111"
 
-static struct drm_mode_config_funcs mode_config_funcs = {
-	.fb_create = drm_fb_cma_create,
+static const struct drm_mode_config_funcs mode_config_funcs = {
+	.fb_create = drm_gem_fb_create,
 	.atomic_check = drm_atomic_helper_check,
 	.atomic_commit = drm_atomic_helper_commit,
 };
@@ -159,9 +160,7 @@ static struct drm_driver pl111_drm_driver = {
 	.minor = 0,
 	.patchlevel = 0,
 	.dumb_create = drm_gem_cma_dumb_create,
-	.dumb_destroy = drm_gem_dumb_destroy,
-	.dumb_map_offset = drm_gem_cma_dumb_map_offset,
-	.gem_free_object = drm_gem_cma_free_object,
+	.gem_free_object_unlocked = drm_gem_cma_free_object,
 	.gem_vm_ops = &drm_gem_cma_vm_ops,
 
 	.enable_vblank = pl111_enable_vblank,
diff --git a/drivers/gpu/drm/qxl/qxl_display.c b/drivers/gpu/drm/qxl/qxl_display.c
index 7babdd8f20fd..e1dd05423e86 100644
--- a/drivers/gpu/drm/qxl/qxl_display.c
+++ b/drivers/gpu/drm/qxl/qxl_display.c
@@ -378,10 +378,6 @@ qxl_framebuffer_init(struct drm_device *dev,
 	return 0;
 }
 
-static void qxl_crtc_dpms(struct drm_crtc *crtc, int mode)
-{
-}
-
 static bool qxl_crtc_mode_fixup(struct drm_crtc *crtc,
 				  const struct drm_display_mode *mode,
 				  struct drm_display_mode *adjusted_mode)
@@ -437,7 +433,7 @@ static void qxl_monitors_config_set(struct qxl_device *qdev,
 
 }
 
-void qxl_mode_set_nofb(struct drm_crtc *crtc)
+static void qxl_mode_set_nofb(struct drm_crtc *crtc)
 {
 	struct qxl_device *qdev = crtc->dev->dev_private;
 	struct qxl_crtc *qcrtc = to_qxl_crtc(crtc);
@@ -451,12 +447,14 @@ void qxl_mode_set_nofb(struct drm_crtc *crtc)
 
 }
 
-static void qxl_crtc_commit(struct drm_crtc *crtc)
+static void qxl_crtc_atomic_enable(struct drm_crtc *crtc,
+				   struct drm_crtc_state *old_state)
 {
 	DRM_DEBUG("\n");
 }
 
-static void qxl_crtc_disable(struct drm_crtc *crtc)
+static void qxl_crtc_atomic_disable(struct drm_crtc *crtc,
+				    struct drm_crtc_state *old_state)
 {
 	struct qxl_crtc *qcrtc = to_qxl_crtc(crtc);
 	struct qxl_device *qdev = crtc->dev->dev_private;
@@ -467,16 +465,15 @@ static void qxl_crtc_disable(struct drm_crtc *crtc)
 }
 
 static const struct drm_crtc_helper_funcs qxl_crtc_helper_funcs = {
-	.dpms = qxl_crtc_dpms,
-	.disable = qxl_crtc_disable,
 	.mode_fixup = qxl_crtc_mode_fixup,
 	.mode_set_nofb = qxl_mode_set_nofb,
-	.commit = qxl_crtc_commit,
 	.atomic_flush = qxl_crtc_atomic_flush,
+	.atomic_enable = qxl_crtc_atomic_enable,
+	.atomic_disable = qxl_crtc_atomic_disable,
 };
 
-int qxl_primary_atomic_check(struct drm_plane *plane,
-			     struct drm_plane_state *state)
+static int qxl_primary_atomic_check(struct drm_plane *plane,
+				    struct drm_plane_state *state)
 {
 	struct qxl_device *qdev = plane->dev->dev_private;
 	struct qxl_framebuffer *qfb;
@@ -551,8 +548,8 @@ static void qxl_primary_atomic_disable(struct drm_plane *plane,
 	}
 }
 
-int qxl_plane_atomic_check(struct drm_plane *plane,
-			   struct drm_plane_state *state)
+static int qxl_plane_atomic_check(struct drm_plane *plane,
+				  struct drm_plane_state *state)
 {
 	return 0;
 }
@@ -651,8 +648,8 @@ out_free_release:
 
 }
 
-void qxl_cursor_atomic_disable(struct drm_plane *plane,
-			       struct drm_plane_state *old_state)
+static void qxl_cursor_atomic_disable(struct drm_plane *plane,
+				      struct drm_plane_state *old_state)
 {
 	struct qxl_device *qdev = plane->dev->dev_private;
 	struct qxl_release *release;
@@ -679,8 +676,8 @@ void qxl_cursor_atomic_disable(struct drm_plane *plane,
 	qxl_release_fence_buffer_objects(release);
 }
 
-int qxl_plane_prepare_fb(struct drm_plane *plane,
-			 struct drm_plane_state *new_state)
+static int qxl_plane_prepare_fb(struct drm_plane *plane,
+				struct drm_plane_state *new_state)
 {
 	struct drm_gem_object *obj;
 	struct qxl_bo *user_bo;
@@ -791,7 +788,7 @@ static struct drm_plane *qxl_create_plane(struct qxl_device *qdev,
 
 	err = drm_universal_plane_init(&qdev->ddev, plane, possible_crtcs,
 				       funcs, formats, num_formats,
-				       type, NULL);
+				       NULL, type, NULL);
 	if (err)
 		goto free_plane;
 
diff --git a/drivers/gpu/drm/qxl/qxl_drv.c b/drivers/gpu/drm/qxl/qxl_drv.c
index c2fc201d9e1b..2445e75cf7ea 100644
--- a/drivers/gpu/drm/qxl/qxl_drv.c
+++ b/drivers/gpu/drm/qxl/qxl_drv.c
@@ -37,7 +37,6 @@
 #include "qxl_drv.h"
 #include "qxl_object.h"
 
-extern int qxl_max_ioctls;
 static const struct pci_device_id pciidlist[] = {
 	{ 0x1b36, 0x100, PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_DISPLAY_VGA << 8,
 	  0xffff00, 0 },
@@ -262,11 +261,8 @@ static struct drm_driver qxl_driver = {
 			   DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED |
 			   DRIVER_ATOMIC,
 
-	.set_busid = drm_pci_set_busid,
-
 	.dumb_create = qxl_mode_dumb_create,
 	.dumb_map_offset = qxl_mode_dumb_mmap,
-	.dumb_destroy = drm_gem_dumb_destroy,
 #if defined(CONFIG_DEBUG_FS)
 	.debugfs_init = qxl_debugfs_init,
 #endif
@@ -303,12 +299,12 @@ static int __init qxl_init(void)
 	if (qxl_modeset == 0)
 		return -EINVAL;
 	qxl_driver.num_ioctls = qxl_max_ioctls;
-	return drm_pci_init(&qxl_driver, &qxl_pci_driver);
+	return pci_register_driver(&qxl_pci_driver);
 }
 
 static void __exit qxl_exit(void)
 {
-	drm_pci_exit(&qxl_driver, &qxl_pci_driver);
+	pci_unregister_driver(&qxl_pci_driver);
 }
 
 module_init(qxl_init);
diff --git a/drivers/gpu/drm/qxl/qxl_drv.h b/drivers/gpu/drm/qxl/qxl_drv.h
index 3591d2330a09..3397a1907336 100644
--- a/drivers/gpu/drm/qxl/qxl_drv.h
+++ b/drivers/gpu/drm/qxl/qxl_drv.h
@@ -64,6 +64,7 @@
 
 extern int qxl_log_level;
 extern int qxl_num_crtc;
+extern int qxl_max_ioctls;
 
 enum {
 	QXL_INFO_LEVEL = 1,
diff --git a/drivers/gpu/drm/qxl/qxl_fb.c b/drivers/gpu/drm/qxl/qxl_fb.c
index 573e7e9a5f98..844c4a31ca13 100644
--- a/drivers/gpu/drm/qxl/qxl_fb.c
+++ b/drivers/gpu/drm/qxl/qxl_fb.c
@@ -275,7 +275,6 @@ static int qxlfb_create(struct qxl_fbdev *qfbdev,
 
 	drm_fb_helper_fill_fix(info, fb->pitches[0], fb->format->depth);
 
-	info->flags = FBINFO_DEFAULT | FBINFO_HWACCEL_COPYAREA | FBINFO_HWACCEL_FILLRECT;
 	info->fbops = &qxlfb_ops;
 
 	/*
diff --git a/drivers/gpu/drm/qxl/qxl_ioctl.c b/drivers/gpu/drm/qxl/qxl_ioctl.c
index 0b82a87916ae..31effed4a3c8 100644
--- a/drivers/gpu/drm/qxl/qxl_ioctl.c
+++ b/drivers/gpu/drm/qxl/qxl_ioctl.c
@@ -163,7 +163,7 @@ static int qxl_process_single_command(struct qxl_device *qdev,
 		return -EINVAL;
 
 	if (!access_ok(VERIFY_READ,
-		       (void *)(unsigned long)cmd->command,
+		       u64_to_user_ptr(cmd->command),
 		       cmd->command_size))
 		return -EFAULT;
 
@@ -183,7 +183,9 @@ static int qxl_process_single_command(struct qxl_device *qdev,
 
 	/* TODO copy slow path code from i915 */
 	fb_cmd = qxl_bo_kmap_atomic_page(qdev, cmd_bo, (release->release_offset & PAGE_SIZE));
-	unwritten = __copy_from_user_inatomic_nocache(fb_cmd + sizeof(union qxl_release_info) + (release->release_offset & ~PAGE_SIZE), (void *)(unsigned long)cmd->command, cmd->command_size);
+	unwritten = __copy_from_user_inatomic_nocache
+		(fb_cmd + sizeof(union qxl_release_info) + (release->release_offset & ~PAGE_SIZE),
+		 u64_to_user_ptr(cmd->command), cmd->command_size);
 
 	{
 		struct qxl_drawable *draw = fb_cmd;
@@ -201,10 +203,9 @@ static int qxl_process_single_command(struct qxl_device *qdev,
 	num_relocs = 0;
 	for (i = 0; i < cmd->relocs_num; ++i) {
 		struct drm_qxl_reloc reloc;
+		struct drm_qxl_reloc __user *u = u64_to_user_ptr(cmd->relocs);
 
-		if (copy_from_user(&reloc,
-				       &((struct drm_qxl_reloc *)(uintptr_t)cmd->relocs)[i],
-				       sizeof(reloc))) {
+		if (copy_from_user(&reloc, u + i, sizeof(reloc))) {
 			ret = -EFAULT;
 			goto out_free_bos;
 		}
@@ -282,10 +283,10 @@ static int qxl_execbuffer_ioctl(struct drm_device *dev, void *data,
 
 	for (cmd_num = 0; cmd_num < execbuffer->commands_num; ++cmd_num) {
 
-		struct drm_qxl_command *commands =
-			(struct drm_qxl_command *)(uintptr_t)execbuffer->commands;
+		struct drm_qxl_command __user *commands =
+			u64_to_user_ptr(execbuffer->commands);
 
-		if (copy_from_user(&user_cmd, &commands[cmd_num],
+		if (copy_from_user(&user_cmd, commands + cmd_num,
 				       sizeof(user_cmd)))
 			return -EFAULT;
 
diff --git a/drivers/gpu/drm/qxl/qxl_object.c b/drivers/gpu/drm/qxl/qxl_object.c
index 9a7eef7dd604..0a67ddf19c3d 100644
--- a/drivers/gpu/drm/qxl/qxl_object.c
+++ b/drivers/gpu/drm/qxl/qxl_object.c
@@ -221,7 +221,7 @@ struct qxl_bo *qxl_bo_ref(struct qxl_bo *bo)
 	return bo;
 }
 
-int __qxl_bo_pin(struct qxl_bo *bo, u32 domain, u64 *gpu_addr)
+static int __qxl_bo_pin(struct qxl_bo *bo, u32 domain, u64 *gpu_addr)
 {
 	struct drm_device *ddev = bo->gem_base.dev;
 	int r;
@@ -244,7 +244,7 @@ int __qxl_bo_pin(struct qxl_bo *bo, u32 domain, u64 *gpu_addr)
 	return r;
 }
 
-int __qxl_bo_unpin(struct qxl_bo *bo)
+static int __qxl_bo_unpin(struct qxl_bo *bo)
 {
 	struct drm_device *ddev = bo->gem_base.dev;
 	int r, i;
diff --git a/drivers/gpu/drm/qxl/qxl_ttm.c b/drivers/gpu/drm/qxl/qxl_ttm.c
index 87fc1dbd0a2f..7ecf8a4b9fe6 100644
--- a/drivers/gpu/drm/qxl/qxl_ttm.c
+++ b/drivers/gpu/drm/qxl/qxl_ttm.c
@@ -187,7 +187,7 @@ static void qxl_evict_flags(struct ttm_buffer_object *bo,
 				struct ttm_placement *placement)
 {
 	struct qxl_bo *qbo;
-	static struct ttm_place placements = {
+	static const struct ttm_place placements = {
 		.fpfn = 0,
 		.lpfn = 0,
 		.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM
diff --git a/drivers/gpu/drm/r128/r128_drv.c b/drivers/gpu/drm/r128/r128_drv.c
index a982be57d1ef..0d2b7e42b3a7 100644
--- a/drivers/gpu/drm/r128/r128_drv.c
+++ b/drivers/gpu/drm/r128/r128_drv.c
@@ -62,7 +62,6 @@ static struct drm_driver driver = {
 	.load = r128_driver_load,
 	.preclose = r128_driver_preclose,
 	.lastclose = r128_driver_lastclose,
-	.set_busid = drm_pci_set_busid,
 	.get_vblank_counter = r128_get_vblank_counter,
 	.enable_vblank = r128_enable_vblank,
 	.disable_vblank = r128_disable_vblank,
@@ -96,12 +95,12 @@ static int __init r128_init(void)
 {
 	driver.num_ioctls = r128_max_ioctl;
 
-	return drm_pci_init(&driver, &r128_pci_driver);
+	return drm_legacy_pci_init(&driver, &r128_pci_driver);
 }
 
 static void __exit r128_exit(void)
 {
-	drm_pci_exit(&driver, &r128_pci_driver);
+	drm_legacy_pci_exit(&driver, &r128_pci_driver);
 }
 
 module_init(r128_init);
diff --git a/drivers/gpu/drm/radeon/atombios_crtc.c b/drivers/gpu/drm/radeon/atombios_crtc.c
index 3c492a0aa6bd..02baaaf20e9d 100644
--- a/drivers/gpu/drm/radeon/atombios_crtc.c
+++ b/drivers/gpu/drm/radeon/atombios_crtc.c
@@ -2217,7 +2217,6 @@ static const struct drm_crtc_helper_funcs atombios_helper_funcs = {
 	.mode_set_base_atomic = atombios_crtc_set_base_atomic,
 	.prepare = atombios_crtc_prepare,
 	.commit = atombios_crtc_commit,
-	.load_lut = radeon_crtc_load_lut,
 	.disable = atombios_crtc_disable,
 };
 
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index 5008f3d4cccc..8cbaeec090c9 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -464,7 +464,7 @@ struct radeon_bo_list {
 	struct radeon_bo		*robj;
 	struct ttm_validate_buffer	tv;
 	uint64_t			gpu_offset;
-	unsigned			prefered_domains;
+	unsigned			preferred_domains;
 	unsigned			allowed_domains;
 	uint32_t			tiling_flags;
 };
@@ -924,7 +924,7 @@ struct radeon_vm_id {
 struct radeon_vm {
 	struct mutex		mutex;
 
-	struct rb_root		va;
+	struct rb_root_cached	va;
 
 	/* protecting invalidated and freed */
 	spinlock_t		status_lock;
@@ -2327,7 +2327,7 @@ struct radeon_device {
 	uint8_t				*bios;
 	bool				is_atom_bios;
 	uint16_t			bios_header_start;
-	struct radeon_bo		*stollen_vga_memory;
+	struct radeon_bo		*stolen_vga_memory;
 	/* Register mmio */
 	resource_size_t			rmmio_base;
 	resource_size_t			rmmio_size;
diff --git a/drivers/gpu/drm/radeon/radeon_acpi.c b/drivers/gpu/drm/radeon/radeon_acpi.c
index 6efbd65c929e..8d3251a10cd4 100644
--- a/drivers/gpu/drm/radeon/radeon_acpi.c
+++ b/drivers/gpu/drm/radeon/radeon_acpi.c
@@ -351,7 +351,7 @@ out:
  * handles it.
  * Returns NOTIFY code
  */
-int radeon_atif_handler(struct radeon_device *rdev,
+static int radeon_atif_handler(struct radeon_device *rdev,
 		struct acpi_bus_event *event)
 {
 	struct radeon_atif *atif = &rdev->atif;
diff --git a/drivers/gpu/drm/radeon/radeon_acpi.h b/drivers/gpu/drm/radeon/radeon_acpi.h
index 7af1977c2c68..35202a453e66 100644
--- a/drivers/gpu/drm/radeon/radeon_acpi.h
+++ b/drivers/gpu/drm/radeon/radeon_acpi.h
@@ -27,9 +27,6 @@
 struct radeon_device;
 struct acpi_bus_event;
 
-int radeon_atif_handler(struct radeon_device *rdev,
-		struct acpi_bus_event *event);
-
 /* AMD hw uses four ACPI control methods:
  * 1. ATIF
  * ARG0: (ACPI_INTEGER) function code
diff --git a/drivers/gpu/drm/radeon/radeon_audio.c b/drivers/gpu/drm/radeon/radeon_audio.c
index aaacac190d26..770e31f5fd1b 100644
--- a/drivers/gpu/drm/radeon/radeon_audio.c
+++ b/drivers/gpu/drm/radeon/radeon_audio.c
@@ -516,7 +516,7 @@ static int radeon_audio_set_avi_packet(struct drm_encoder *encoder,
 	if (!connector)
 		return -EINVAL;
 
-	err = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode);
+	err = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode, false);
 	if (err < 0) {
 		DRM_ERROR("failed to setup AVI infoframe: %d\n", err);
 		return err;
diff --git a/drivers/gpu/drm/radeon/radeon_connectors.c b/drivers/gpu/drm/radeon/radeon_connectors.c
index 27affbde058c..2f642cbefd8e 100644
--- a/drivers/gpu/drm/radeon/radeon_connectors.c
+++ b/drivers/gpu/drm/radeon/radeon_connectors.c
@@ -773,12 +773,15 @@ static int radeon_connector_set_property(struct drm_connector *connector, struct
 
 		if (connector->encoder->crtc) {
 			struct drm_crtc *crtc  = connector->encoder->crtc;
-			const struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private;
 			struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
 
 			radeon_crtc->output_csc = radeon_encoder->output_csc;
 
-			(*crtc_funcs->load_lut)(crtc);
+			/*
+			 * Our .gamma_set assumes the .gamma_store has been
+			 * prefilled and don't care about its arguments.
+			 */
+			crtc->funcs->gamma_set(crtc, NULL, NULL, NULL, 0, NULL);
 		}
 	}
 
diff --git a/drivers/gpu/drm/radeon/radeon_cs.c b/drivers/gpu/drm/radeon/radeon_cs.c
index 00b22af70f5c..1ae31dbc61c6 100644
--- a/drivers/gpu/drm/radeon/radeon_cs.c
+++ b/drivers/gpu/drm/radeon/radeon_cs.c
@@ -130,7 +130,7 @@ static int radeon_cs_parser_relocs(struct radeon_cs_parser *p)
 		     p->rdev->family == CHIP_RS880)) {
 
 			/* TODO: is this still needed for NI+ ? */
-			p->relocs[i].prefered_domains =
+			p->relocs[i].preferred_domains =
 				RADEON_GEM_DOMAIN_VRAM;
 
 			p->relocs[i].allowed_domains =
@@ -148,14 +148,14 @@ static int radeon_cs_parser_relocs(struct radeon_cs_parser *p)
 				return -EINVAL;
 			}
 
-			p->relocs[i].prefered_domains = domain;
+			p->relocs[i].preferred_domains = domain;
 			if (domain == RADEON_GEM_DOMAIN_VRAM)
 				domain |= RADEON_GEM_DOMAIN_GTT;
 			p->relocs[i].allowed_domains = domain;
 		}
 
 		if (radeon_ttm_tt_has_userptr(p->relocs[i].robj->tbo.ttm)) {
-			uint32_t domain = p->relocs[i].prefered_domains;
+			uint32_t domain = p->relocs[i].preferred_domains;
 			if (!(domain & RADEON_GEM_DOMAIN_GTT)) {
 				DRM_ERROR("Only RADEON_GEM_DOMAIN_GTT is "
 					  "allowed for userptr BOs\n");
@@ -163,7 +163,7 @@ static int radeon_cs_parser_relocs(struct radeon_cs_parser *p)
 			}
 			need_mmap_lock = true;
 			domain = RADEON_GEM_DOMAIN_GTT;
-			p->relocs[i].prefered_domains = domain;
+			p->relocs[i].preferred_domains = domain;
 			p->relocs[i].allowed_domains = domain;
 		}
 
@@ -437,7 +437,7 @@ static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error, bo
 			if (bo == NULL)
 				continue;
 
-			drm_gem_object_unreference_unlocked(&bo->gem_base);
+			drm_gem_object_put_unlocked(&bo->gem_base);
 		}
 	}
 	kfree(parser->track);
diff --git a/drivers/gpu/drm/radeon/radeon_cursor.c b/drivers/gpu/drm/radeon/radeon_cursor.c
index 4a4f9533c53b..91952277557e 100644
--- a/drivers/gpu/drm/radeon/radeon_cursor.c
+++ b/drivers/gpu/drm/radeon/radeon_cursor.c
@@ -307,7 +307,7 @@ int radeon_crtc_cursor_set2(struct drm_crtc *crtc,
 	robj = gem_to_radeon_bo(obj);
 	ret = radeon_bo_reserve(robj, false);
 	if (ret != 0) {
-		drm_gem_object_unreference_unlocked(obj);
+		drm_gem_object_put_unlocked(obj);
 		return ret;
 	}
 	/* Only 27 bit offset for legacy cursor */
@@ -317,7 +317,7 @@ int radeon_crtc_cursor_set2(struct drm_crtc *crtc,
 	radeon_bo_unreserve(robj);
 	if (ret) {
 		DRM_ERROR("Failed to pin new cursor BO (%d)\n", ret);
-		drm_gem_object_unreference_unlocked(obj);
+		drm_gem_object_put_unlocked(obj);
 		return ret;
 	}
 
@@ -352,7 +352,7 @@ unpin:
 			radeon_bo_unpin(robj);
 			radeon_bo_unreserve(robj);
 		}
-		drm_gem_object_unreference_unlocked(radeon_crtc->cursor_bo);
+		drm_gem_object_put_unlocked(radeon_crtc->cursor_bo);
 	}
 
 	radeon_crtc->cursor_bo = obj;
diff --git a/drivers/gpu/drm/radeon/radeon_display.c b/drivers/gpu/drm/radeon/radeon_display.c
index 17d3dafc8319..ddfe91efa61e 100644
--- a/drivers/gpu/drm/radeon/radeon_display.c
+++ b/drivers/gpu/drm/radeon/radeon_display.c
@@ -42,6 +42,7 @@ static void avivo_crtc_load_lut(struct drm_crtc *crtc)
 	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct radeon_device *rdev = dev->dev_private;
+	u16 *r, *g, *b;
 	int i;
 
 	DRM_DEBUG_KMS("%d\n", radeon_crtc->crtc_id);
@@ -60,11 +61,14 @@ static void avivo_crtc_load_lut(struct drm_crtc *crtc)
 	WREG32(AVIVO_DC_LUT_WRITE_EN_MASK, 0x0000003f);
 
 	WREG8(AVIVO_DC_LUT_RW_INDEX, 0);
+	r = crtc->gamma_store;
+	g = r + crtc->gamma_size;
+	b = g + crtc->gamma_size;
 	for (i = 0; i < 256; i++) {
 		WREG32(AVIVO_DC_LUT_30_COLOR,
-			     (radeon_crtc->lut_r[i] << 20) |
-			     (radeon_crtc->lut_g[i] << 10) |
-			     (radeon_crtc->lut_b[i] << 0));
+		       ((*r++ & 0xffc0) << 14) |
+		       ((*g++ & 0xffc0) << 4) |
+		       (*b++ >> 6));
 	}
 
 	/* Only change bit 0 of LUT_SEL, other bits are set elsewhere */
@@ -76,6 +80,7 @@ static void dce4_crtc_load_lut(struct drm_crtc *crtc)
 	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct radeon_device *rdev = dev->dev_private;
+	u16 *r, *g, *b;
 	int i;
 
 	DRM_DEBUG_KMS("%d\n", radeon_crtc->crtc_id);
@@ -93,11 +98,14 @@ static void dce4_crtc_load_lut(struct drm_crtc *crtc)
 	WREG32(EVERGREEN_DC_LUT_WRITE_EN_MASK + radeon_crtc->crtc_offset, 0x00000007);
 
 	WREG32(EVERGREEN_DC_LUT_RW_INDEX + radeon_crtc->crtc_offset, 0);
+	r = crtc->gamma_store;
+	g = r + crtc->gamma_size;
+	b = g + crtc->gamma_size;
 	for (i = 0; i < 256; i++) {
 		WREG32(EVERGREEN_DC_LUT_30_COLOR + radeon_crtc->crtc_offset,
-		       (radeon_crtc->lut_r[i] << 20) |
-		       (radeon_crtc->lut_g[i] << 10) |
-		       (radeon_crtc->lut_b[i] << 0));
+		       ((*r++ & 0xffc0) << 14) |
+		       ((*g++ & 0xffc0) << 4) |
+		       (*b++ >> 6));
 	}
 }
 
@@ -106,6 +114,7 @@ static void dce5_crtc_load_lut(struct drm_crtc *crtc)
 	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct radeon_device *rdev = dev->dev_private;
+	u16 *r, *g, *b;
 	int i;
 
 	DRM_DEBUG_KMS("%d\n", radeon_crtc->crtc_id);
@@ -135,11 +144,14 @@ static void dce5_crtc_load_lut(struct drm_crtc *crtc)
 	WREG32(EVERGREEN_DC_LUT_WRITE_EN_MASK + radeon_crtc->crtc_offset, 0x00000007);
 
 	WREG32(EVERGREEN_DC_LUT_RW_INDEX + radeon_crtc->crtc_offset, 0);
+	r = crtc->gamma_store;
+	g = r + crtc->gamma_size;
+	b = g + crtc->gamma_size;
 	for (i = 0; i < 256; i++) {
 		WREG32(EVERGREEN_DC_LUT_30_COLOR + radeon_crtc->crtc_offset,
-		       (radeon_crtc->lut_r[i] << 20) |
-		       (radeon_crtc->lut_g[i] << 10) |
-		       (radeon_crtc->lut_b[i] << 0));
+		       ((*r++ & 0xffc0) << 14) |
+		       ((*g++ & 0xffc0) << 4) |
+		       (*b++ >> 6));
 	}
 
 	WREG32(NI_DEGAMMA_CONTROL + radeon_crtc->crtc_offset,
@@ -172,6 +184,7 @@ static void legacy_crtc_load_lut(struct drm_crtc *crtc)
 	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct radeon_device *rdev = dev->dev_private;
+	u16 *r, *g, *b;
 	int i;
 	uint32_t dac2_cntl;
 
@@ -183,11 +196,14 @@ static void legacy_crtc_load_lut(struct drm_crtc *crtc)
 	WREG32(RADEON_DAC_CNTL2, dac2_cntl);
 
 	WREG8(RADEON_PALETTE_INDEX, 0);
+	r = crtc->gamma_store;
+	g = r + crtc->gamma_size;
+	b = g + crtc->gamma_size;
 	for (i = 0; i < 256; i++) {
 		WREG32(RADEON_PALETTE_30_DATA,
-			     (radeon_crtc->lut_r[i] << 20) |
-			     (radeon_crtc->lut_g[i] << 10) |
-			     (radeon_crtc->lut_b[i] << 0));
+		       ((*r++ & 0xffc0) << 14) |
+		       ((*g++ & 0xffc0) << 4) |
+		       (*b++ >> 6));
 	}
 }
 
@@ -209,41 +225,10 @@ void radeon_crtc_load_lut(struct drm_crtc *crtc)
 		legacy_crtc_load_lut(crtc);
 }
 
-/** Sets the color ramps on behalf of fbcon */
-void radeon_crtc_fb_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
-			      u16 blue, int regno)
-{
-	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
-
-	radeon_crtc->lut_r[regno] = red >> 6;
-	radeon_crtc->lut_g[regno] = green >> 6;
-	radeon_crtc->lut_b[regno] = blue >> 6;
-}
-
-/** Gets the color ramps on behalf of fbcon */
-void radeon_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green,
-			      u16 *blue, int regno)
-{
-	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
-
-	*red = radeon_crtc->lut_r[regno] << 6;
-	*green = radeon_crtc->lut_g[regno] << 6;
-	*blue = radeon_crtc->lut_b[regno] << 6;
-}
-
 static int radeon_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
 				 u16 *blue, uint32_t size,
 				 struct drm_modeset_acquire_ctx *ctx)
 {
-	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
-	int i;
-
-	/* userspace palettes are always correct as is */
-	for (i = 0; i < size; i++) {
-		radeon_crtc->lut_r[i] = red[i] >> 6;
-		radeon_crtc->lut_g[i] = green[i] >> 6;
-		radeon_crtc->lut_b[i] = blue[i] >> 6;
-	}
 	radeon_crtc_load_lut(crtc);
 
 	return 0;
@@ -282,7 +267,7 @@ static void radeon_unpin_work_func(struct work_struct *__work)
 	} else
 		DRM_ERROR("failed to reserve buffer after flip\n");
 
-	drm_gem_object_unreference_unlocked(&work->old_rbo->gem_base);
+	drm_gem_object_put_unlocked(&work->old_rbo->gem_base);
 	kfree(work);
 }
 
@@ -519,7 +504,7 @@ static int radeon_crtc_page_flip_target(struct drm_crtc *crtc,
 	obj = old_radeon_fb->obj;
 
 	/* take a reference to the old object */
-	drm_gem_object_reference(obj);
+	drm_gem_object_get(obj);
 	work->old_rbo = gem_to_radeon_bo(obj);
 
 	new_radeon_fb = to_radeon_framebuffer(fb);
@@ -618,7 +603,7 @@ pflip_cleanup:
 	radeon_bo_unreserve(new_rbo);
 
 cleanup:
-	drm_gem_object_unreference_unlocked(&work->old_rbo->gem_base);
+	drm_gem_object_put_unlocked(&work->old_rbo->gem_base);
 	dma_fence_put(work->fence);
 	kfree(work);
 	return r;
@@ -1303,7 +1288,7 @@ static void radeon_user_framebuffer_destroy(struct drm_framebuffer *fb)
 {
 	struct radeon_framebuffer *radeon_fb = to_radeon_framebuffer(fb);
 
-	drm_gem_object_unreference_unlocked(radeon_fb->obj);
+	drm_gem_object_put_unlocked(radeon_fb->obj);
 	drm_framebuffer_cleanup(fb);
 	kfree(radeon_fb);
 }
@@ -1363,14 +1348,14 @@ radeon_user_framebuffer_create(struct drm_device *dev,
 
 	radeon_fb = kzalloc(sizeof(*radeon_fb), GFP_KERNEL);
 	if (radeon_fb == NULL) {
-		drm_gem_object_unreference_unlocked(obj);
+		drm_gem_object_put_unlocked(obj);
 		return ERR_PTR(-ENOMEM);
 	}
 
 	ret = radeon_framebuffer_init(dev, radeon_fb, mode_cmd, obj);
 	if (ret) {
 		kfree(radeon_fb);
-		drm_gem_object_unreference_unlocked(obj);
+		drm_gem_object_put_unlocked(obj);
 		return ERR_PTR(ret);
 	}
 
@@ -1388,12 +1373,12 @@ static const struct drm_mode_config_funcs radeon_mode_funcs = {
 	.output_poll_changed = radeon_output_poll_changed
 };
 
-static struct drm_prop_enum_list radeon_tmds_pll_enum_list[] =
+static const struct drm_prop_enum_list radeon_tmds_pll_enum_list[] =
 {	{ 0, "driver" },
 	{ 1, "bios" },
 };
 
-static struct drm_prop_enum_list radeon_tv_std_enum_list[] =
+static const struct drm_prop_enum_list radeon_tv_std_enum_list[] =
 {	{ TV_STD_NTSC, "ntsc" },
 	{ TV_STD_PAL, "pal" },
 	{ TV_STD_PAL_M, "pal-m" },
@@ -1404,25 +1389,25 @@ static struct drm_prop_enum_list radeon_tv_std_enum_list[] =
 	{ TV_STD_SECAM, "secam" },
 };
 
-static struct drm_prop_enum_list radeon_underscan_enum_list[] =
+static const struct drm_prop_enum_list radeon_underscan_enum_list[] =
 {	{ UNDERSCAN_OFF, "off" },
 	{ UNDERSCAN_ON, "on" },
 	{ UNDERSCAN_AUTO, "auto" },
 };
 
-static struct drm_prop_enum_list radeon_audio_enum_list[] =
+static const struct drm_prop_enum_list radeon_audio_enum_list[] =
 {	{ RADEON_AUDIO_DISABLE, "off" },
 	{ RADEON_AUDIO_ENABLE, "on" },
 	{ RADEON_AUDIO_AUTO, "auto" },
 };
 
 /* XXX support different dither options? spatial, temporal, both, etc. */
-static struct drm_prop_enum_list radeon_dither_enum_list[] =
+static const struct drm_prop_enum_list radeon_dither_enum_list[] =
 {	{ RADEON_FMT_DITHER_DISABLE, "off" },
 	{ RADEON_FMT_DITHER_ENABLE, "on" },
 };
 
-static struct drm_prop_enum_list radeon_output_csc_enum_list[] =
+static const struct drm_prop_enum_list radeon_output_csc_enum_list[] =
 {	{ RADEON_OUTPUT_CSC_BYPASS, "bypass" },
 	{ RADEON_OUTPUT_CSC_TVRGB, "tvrgb" },
 	{ RADEON_OUTPUT_CSC_YCBCR601, "ycbcr601" },
diff --git a/drivers/gpu/drm/radeon/radeon_dp_mst.c b/drivers/gpu/drm/radeon/radeon_dp_mst.c
index 6598306dca9b..ebdf1b859cb6 100644
--- a/drivers/gpu/drm/radeon/radeon_dp_mst.c
+++ b/drivers/gpu/drm/radeon/radeon_dp_mst.c
@@ -300,9 +300,7 @@ static void radeon_dp_register_mst_connector(struct drm_connector *connector)
 	struct drm_device *dev = connector->dev;
 	struct radeon_device *rdev = dev->dev_private;
 
-	drm_modeset_lock_all(dev);
 	radeon_fb_add_connector(rdev, connector);
-	drm_modeset_unlock_all(dev);
 
 	drm_connector_register(connector);
 }
@@ -315,13 +313,8 @@ static void radeon_dp_destroy_mst_connector(struct drm_dp_mst_topology_mgr *mgr,
 	struct radeon_device *rdev = dev->dev_private;
 
 	drm_connector_unregister(connector);
-	/* need to nuke the connector */
-	drm_modeset_lock_all(dev);
-	/* dpms off */
 	radeon_fb_remove_connector(rdev, connector);
-
 	drm_connector_cleanup(connector);
-	drm_modeset_unlock_all(dev);
 
 	kfree(connector);
 	DRM_DEBUG_KMS("\n");
diff --git a/drivers/gpu/drm/radeon/radeon_drv.c b/drivers/gpu/drm/radeon/radeon_drv.c
index 74abd161237b..f4becad0a78c 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.c
+++ b/drivers/gpu/drm/radeon/radeon_drv.c
@@ -567,7 +567,6 @@ static struct drm_driver kms_driver = {
 	.open = radeon_driver_open_kms,
 	.postclose = radeon_driver_postclose_kms,
 	.lastclose = radeon_driver_lastclose_kms,
-	.set_busid = drm_pci_set_busid,
 	.unload = radeon_driver_unload_kms,
 	.get_vblank_counter = radeon_get_vblank_counter_kms,
 	.enable_vblank = radeon_enable_vblank_kms,
@@ -584,7 +583,6 @@ static struct drm_driver kms_driver = {
 	.gem_close_object = radeon_gem_object_close,
 	.dumb_create = radeon_mode_dumb_create,
 	.dumb_map_offset = radeon_mode_dumb_mmap,
-	.dumb_destroy = drm_gem_dumb_destroy,
 	.fops = &radeon_driver_kms_fops,
 
 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
@@ -642,14 +640,13 @@ static int __init radeon_init(void)
 		return -EINVAL;
 	}
 
-	/* let modprobe override vga console setting */
-	return drm_pci_init(driver, pdriver);
+	return pci_register_driver(pdriver);
 }
 
 static void __exit radeon_exit(void)
 {
 	radeon_kfd_fini();
-	drm_pci_exit(driver, pdriver);
+	pci_unregister_driver(pdriver);
 	radeon_unregister_atpx_handler();
 }
 
diff --git a/drivers/gpu/drm/radeon/radeon_fb.c b/drivers/gpu/drm/radeon/radeon_fb.c
index 356ad90a5238..fd25361ac681 100644
--- a/drivers/gpu/drm/radeon/radeon_fb.c
+++ b/drivers/gpu/drm/radeon/radeon_fb.c
@@ -118,7 +118,7 @@ static void radeonfb_destroy_pinned_object(struct drm_gem_object *gobj)
 		radeon_bo_unpin(rbo);
 		radeon_bo_unreserve(rbo);
 	}
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 }
 
 static int radeonfb_create_pinned_object(struct radeon_fbdev *rfbdev,
@@ -264,7 +264,6 @@ static int radeonfb_create(struct drm_fb_helper *helper,
 
 	drm_fb_helper_fill_fix(info, fb->pitches[0], fb->format->depth);
 
-	info->flags = FBINFO_DEFAULT | FBINFO_CAN_FORCE_OUTPUT;
 	info->fbops = &radeonfb_ops;
 
 	tmp = radeon_bo_gpu_offset(rbo) - rdev->mc.vram_start;
@@ -300,7 +299,7 @@ out:
 
 	}
 	if (fb && ret) {
-		drm_gem_object_unreference_unlocked(gobj);
+		drm_gem_object_put_unlocked(gobj);
 		drm_framebuffer_unregister_private(fb);
 		drm_framebuffer_cleanup(fb);
 		kfree(fb);
@@ -332,8 +331,6 @@ static int radeon_fbdev_destroy(struct drm_device *dev, struct radeon_fbdev *rfb
 }
 
 static const struct drm_fb_helper_funcs radeon_fb_helper_funcs = {
-	.gamma_set = radeon_crtc_fb_gamma_set,
-	.gamma_get = radeon_crtc_fb_gamma_get,
 	.fb_probe = radeonfb_create,
 };
 
@@ -347,9 +344,12 @@ int radeon_fbdev_init(struct radeon_device *rdev)
 	if (list_empty(&rdev->ddev->mode_config.connector_list))
 		return 0;
 
-	/* select 8 bpp console on RN50 or 16MB cards */
-	if (ASIC_IS_RN50(rdev) || rdev->mc.real_vram_size <= (32*1024*1024))
+	/* select 8 bpp console on 8MB cards, or 16 bpp on RN50 or 32MB */
+	if (rdev->mc.real_vram_size <= (8*1024*1024))
 		bpp_sel = 8;
+	else if (ASIC_IS_RN50(rdev) ||
+		 rdev->mc.real_vram_size <= (32*1024*1024))
+		bpp_sel = 16;
 
 	rfbdev = kzalloc(sizeof(struct radeon_fbdev), GFP_KERNEL);
 	if (!rfbdev)
diff --git a/drivers/gpu/drm/radeon/radeon_gem.c b/drivers/gpu/drm/radeon/radeon_gem.c
index 574bf7e6b118..3386452bd2f0 100644
--- a/drivers/gpu/drm/radeon/radeon_gem.c
+++ b/drivers/gpu/drm/radeon/radeon_gem.c
@@ -271,7 +271,7 @@ int radeon_gem_create_ioctl(struct drm_device *dev, void *data,
 	}
 	r = drm_gem_handle_create(filp, gobj, &handle);
 	/* drop reference from allocate - handle holds it now */
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 	if (r) {
 		up_read(&rdev->exclusive_lock);
 		r = radeon_gem_handle_lockup(rdev, r);
@@ -352,7 +352,7 @@ int radeon_gem_userptr_ioctl(struct drm_device *dev, void *data,
 
 	r = drm_gem_handle_create(filp, gobj, &handle);
 	/* drop reference from allocate - handle holds it now */
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 	if (r)
 		goto handle_lockup;
 
@@ -361,7 +361,7 @@ int radeon_gem_userptr_ioctl(struct drm_device *dev, void *data,
 	return 0;
 
 release_object:
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 
 handle_lockup:
 	up_read(&rdev->exclusive_lock);
@@ -395,7 +395,7 @@ int radeon_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 
 	r = radeon_gem_set_domain(gobj, args->read_domains, args->write_domain);
 
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 	up_read(&rdev->exclusive_lock);
 	r = radeon_gem_handle_lockup(robj->rdev, r);
 	return r;
@@ -414,11 +414,11 @@ int radeon_mode_dumb_mmap(struct drm_file *filp,
 	}
 	robj = gem_to_radeon_bo(gobj);
 	if (radeon_ttm_tt_has_userptr(robj->tbo.ttm)) {
-		drm_gem_object_unreference_unlocked(gobj);
+		drm_gem_object_put_unlocked(gobj);
 		return -EPERM;
 	}
 	*offset_p = radeon_bo_mmap_offset(robj);
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 	return 0;
 }
 
@@ -453,7 +453,7 @@ int radeon_gem_busy_ioctl(struct drm_device *dev, void *data,
 
 	cur_placement = ACCESS_ONCE(robj->tbo.mem.mem_type);
 	args->domain = radeon_mem_type_to_domain(cur_placement);
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 	return r;
 }
 
@@ -485,7 +485,7 @@ int radeon_gem_wait_idle_ioctl(struct drm_device *dev, void *data,
 	if (rdev->asic->mmio_hdp_flush &&
 	    radeon_mem_type_to_domain(cur_placement) == RADEON_GEM_DOMAIN_VRAM)
 		robj->rdev->asic->mmio_hdp_flush(rdev);
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 	r = radeon_gem_handle_lockup(rdev, r);
 	return r;
 }
@@ -504,7 +504,7 @@ int radeon_gem_set_tiling_ioctl(struct drm_device *dev, void *data,
 		return -ENOENT;
 	robj = gem_to_radeon_bo(gobj);
 	r = radeon_bo_set_tiling_flags(robj, args->tiling_flags, args->pitch);
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 	return r;
 }
 
@@ -527,7 +527,7 @@ int radeon_gem_get_tiling_ioctl(struct drm_device *dev, void *data,
 	radeon_bo_get_tiling_flags(rbo, &args->tiling_flags, &args->pitch);
 	radeon_bo_unreserve(rbo);
 out:
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 	return r;
 }
 
@@ -661,14 +661,14 @@ int radeon_gem_va_ioctl(struct drm_device *dev, void *data,
 	r = radeon_bo_reserve(rbo, false);
 	if (r) {
 		args->operation = RADEON_VA_RESULT_ERROR;
-		drm_gem_object_unreference_unlocked(gobj);
+		drm_gem_object_put_unlocked(gobj);
 		return r;
 	}
 	bo_va = radeon_vm_bo_find(&fpriv->vm, rbo);
 	if (!bo_va) {
 		args->operation = RADEON_VA_RESULT_ERROR;
 		radeon_bo_unreserve(rbo);
-		drm_gem_object_unreference_unlocked(gobj);
+		drm_gem_object_put_unlocked(gobj);
 		return -ENOENT;
 	}
 
@@ -695,7 +695,7 @@ int radeon_gem_va_ioctl(struct drm_device *dev, void *data,
 		args->operation = RADEON_VA_RESULT_ERROR;
 	}
 out:
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 	return r;
 }
 
@@ -736,7 +736,7 @@ int radeon_gem_op_ioctl(struct drm_device *dev, void *data,
 
 	radeon_bo_unreserve(robj);
 out:
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 	return r;
 }
 
@@ -762,7 +762,7 @@ int radeon_mode_dumb_create(struct drm_file *file_priv,
 
 	r = drm_gem_handle_create(file_priv, gobj, &handle);
 	/* drop reference from allocate - handle holds it now */
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 	if (r) {
 		return r;
 	}
diff --git a/drivers/gpu/drm/radeon/radeon_irq_kms.c b/drivers/gpu/drm/radeon/radeon_irq_kms.c
index 7aacb44df201..afaf10db47cc 100644
--- a/drivers/gpu/drm/radeon/radeon_irq_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_irq_kms.c
@@ -283,6 +283,10 @@ int radeon_irq_kms_init(struct radeon_device *rdev)
 	int r = 0;
 
 	spin_lock_init(&rdev->irq.lock);
+
+	/* Disable vblank irqs aggressively for power-saving */
+	rdev->ddev->vblank_disable_immediate = true;
+
 	r = drm_vblank_init(rdev->ddev, rdev->num_crtc);
 	if (r) {
 		return r;
@@ -324,7 +328,6 @@ int radeon_irq_kms_init(struct radeon_device *rdev)
  */
 void radeon_irq_kms_fini(struct radeon_device *rdev)
 {
-	drm_vblank_cleanup(rdev->ddev);
 	if (rdev->irq.installed) {
 		drm_irq_uninstall(rdev->ddev);
 		rdev->irq.installed = false;
diff --git a/drivers/gpu/drm/radeon/radeon_kfd.c b/drivers/gpu/drm/radeon/radeon_kfd.c
index a2ab6dcdf4a2..f6578c96925c 100644
--- a/drivers/gpu/drm/radeon/radeon_kfd.c
+++ b/drivers/gpu/drm/radeon/radeon_kfd.c
@@ -75,12 +75,14 @@ static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id,
 				uint32_t hpd_size, uint64_t hpd_gpu_addr);
 static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id);
 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
-			uint32_t queue_id, uint32_t __user *wptr);
+			uint32_t queue_id, uint32_t __user *wptr,
+			uint32_t wptr_shift, uint32_t wptr_mask,
+			struct mm_struct *mm);
 static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd);
 static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
 				uint32_t pipe_id, uint32_t queue_id);
 
-static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
+static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd, uint32_t reset_type,
 				unsigned int timeout, uint32_t pipe_id,
 				uint32_t queue_id);
 static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
@@ -482,7 +484,9 @@ static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
 }
 
 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
-			uint32_t queue_id, uint32_t __user *wptr)
+			uint32_t queue_id, uint32_t __user *wptr,
+			uint32_t wptr_shift, uint32_t wptr_mask,
+			struct mm_struct *mm)
 {
 	uint32_t wptr_shadow, is_wptr_shadow_valid;
 	struct cik_mqd *m;
@@ -636,7 +640,7 @@ static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
 	return false;
 }
 
-static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type,
+static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd, uint32_t reset_type,
 				unsigned int timeout, uint32_t pipe_id,
 				uint32_t queue_id)
 {
@@ -785,7 +789,8 @@ static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd,
 					unsigned int watch_point_id,
 					unsigned int reg_offset)
 {
-	return watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + reg_offset];
+	return watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + reg_offset]
+		/ 4;
 }
 
 static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd, uint8_t vmid)
diff --git a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
index ce6cb6666212..1f1856e0b1e0 100644
--- a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
+++ b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
@@ -1116,7 +1116,6 @@ static const struct drm_crtc_helper_funcs legacy_helper_funcs = {
 	.mode_set_base_atomic = radeon_crtc_set_base_atomic,
 	.prepare = radeon_crtc_prepare,
 	.commit = radeon_crtc_commit,
-	.load_lut = radeon_crtc_load_lut,
 	.disable = radeon_crtc_disable
 };
 
diff --git a/drivers/gpu/drm/radeon/radeon_mn.c b/drivers/gpu/drm/radeon/radeon_mn.c
index 896f2cf51e4e..1d62288b7ee3 100644
--- a/drivers/gpu/drm/radeon/radeon_mn.c
+++ b/drivers/gpu/drm/radeon/radeon_mn.c
@@ -50,7 +50,7 @@ struct radeon_mn {
 
 	/* objects protected by lock */
 	struct mutex		lock;
-	struct rb_root		objects;
+	struct rb_root_cached	objects;
 };
 
 struct radeon_mn_node {
@@ -75,8 +75,8 @@ static void radeon_mn_destroy(struct work_struct *work)
 	mutex_lock(&rdev->mn_lock);
 	mutex_lock(&rmn->lock);
 	hash_del(&rmn->node);
-	rbtree_postorder_for_each_entry_safe(node, next_node, &rmn->objects,
-					     it.rb) {
+	rbtree_postorder_for_each_entry_safe(node, next_node,
+					     &rmn->objects.rb_root, it.rb) {
 
 		interval_tree_remove(&node->it, &rmn->objects);
 		list_for_each_entry_safe(bo, next_bo, &node->bos, mn_list) {
@@ -205,7 +205,7 @@ static struct radeon_mn *radeon_mn_get(struct radeon_device *rdev)
 	rmn->mm = mm;
 	rmn->mn.ops = &radeon_mn_ops;
 	mutex_init(&rmn->lock);
-	rmn->objects = RB_ROOT;
+	rmn->objects = RB_ROOT_CACHED;
 	
 	r = __mmu_notifier_register(&rmn->mn, mm);
 	if (r)
diff --git a/drivers/gpu/drm/radeon/radeon_mode.h b/drivers/gpu/drm/radeon/radeon_mode.h
index 00f5ec5c12c7..da44ac234f64 100644
--- a/drivers/gpu/drm/radeon/radeon_mode.h
+++ b/drivers/gpu/drm/radeon/radeon_mode.h
@@ -935,10 +935,6 @@ extern void
 radeon_combios_encoder_crtc_scratch_regs(struct drm_encoder *encoder, int crtc);
 extern void
 radeon_combios_encoder_dpms_scratch_regs(struct drm_encoder *encoder, bool on);
-extern void radeon_crtc_fb_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
-				     u16 blue, int regno);
-extern void radeon_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green,
-				     u16 *blue, int regno);
 int radeon_framebuffer_init(struct drm_device *dev,
 			     struct radeon_framebuffer *rfb,
 			     const struct drm_mode_fb_cmd2 *mode_cmd,
diff --git a/drivers/gpu/drm/radeon/radeon_object.c b/drivers/gpu/drm/radeon/radeon_object.c
index 8b722297a05c..093594976126 100644
--- a/drivers/gpu/drm/radeon/radeon_object.c
+++ b/drivers/gpu/drm/radeon/radeon_object.c
@@ -445,7 +445,7 @@ void radeon_bo_force_delete(struct radeon_device *rdev)
 		list_del_init(&bo->list);
 		mutex_unlock(&bo->rdev->gem.mutex);
 		/* this should unref the ttm bo */
-		drm_gem_object_unreference_unlocked(&bo->gem_base);
+		drm_gem_object_put_unlocked(&bo->gem_base);
 	}
 }
 
@@ -546,7 +546,7 @@ int radeon_bo_list_validate(struct radeon_device *rdev,
 	list_for_each_entry(lobj, head, tv.head) {
 		struct radeon_bo *bo = lobj->robj;
 		if (!bo->pin_count) {
-			u32 domain = lobj->prefered_domains;
+			u32 domain = lobj->preferred_domains;
 			u32 allowed = lobj->allowed_domains;
 			u32 current_domain =
 				radeon_mem_type_to_domain(bo->tbo.mem.mem_type);
diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c
index faa021396da3..bf69bf9086bf 100644
--- a/drivers/gpu/drm/radeon/radeon_ttm.c
+++ b/drivers/gpu/drm/radeon/radeon_ttm.c
@@ -178,7 +178,7 @@ static int radeon_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
 static void radeon_evict_flags(struct ttm_buffer_object *bo,
 				struct ttm_placement *placement)
 {
-	static struct ttm_place placements = {
+	static const struct ttm_place placements = {
 		.fpfn = 0,
 		.lpfn = 0,
 		.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM
@@ -907,17 +907,17 @@ int radeon_ttm_init(struct radeon_device *rdev)
 
 	r = radeon_bo_create(rdev, 256 * 1024, PAGE_SIZE, true,
 			     RADEON_GEM_DOMAIN_VRAM, 0, NULL,
-			     NULL, &rdev->stollen_vga_memory);
+			     NULL, &rdev->stolen_vga_memory);
 	if (r) {
 		return r;
 	}
-	r = radeon_bo_reserve(rdev->stollen_vga_memory, false);
+	r = radeon_bo_reserve(rdev->stolen_vga_memory, false);
 	if (r)
 		return r;
-	r = radeon_bo_pin(rdev->stollen_vga_memory, RADEON_GEM_DOMAIN_VRAM, NULL);
-	radeon_bo_unreserve(rdev->stollen_vga_memory);
+	r = radeon_bo_pin(rdev->stolen_vga_memory, RADEON_GEM_DOMAIN_VRAM, NULL);
+	radeon_bo_unreserve(rdev->stolen_vga_memory);
 	if (r) {
-		radeon_bo_unref(&rdev->stollen_vga_memory);
+		radeon_bo_unref(&rdev->stolen_vga_memory);
 		return r;
 	}
 	DRM_INFO("radeon: %uM of VRAM memory ready\n",
@@ -946,13 +946,13 @@ void radeon_ttm_fini(struct radeon_device *rdev)
 	if (!rdev->mman.initialized)
 		return;
 	radeon_ttm_debugfs_fini(rdev);
-	if (rdev->stollen_vga_memory) {
-		r = radeon_bo_reserve(rdev->stollen_vga_memory, false);
+	if (rdev->stolen_vga_memory) {
+		r = radeon_bo_reserve(rdev->stolen_vga_memory, false);
 		if (r == 0) {
-			radeon_bo_unpin(rdev->stollen_vga_memory);
-			radeon_bo_unreserve(rdev->stollen_vga_memory);
+			radeon_bo_unpin(rdev->stolen_vga_memory);
+			radeon_bo_unreserve(rdev->stolen_vga_memory);
 		}
-		radeon_bo_unref(&rdev->stollen_vga_memory);
+		radeon_bo_unref(&rdev->stolen_vga_memory);
 	}
 	ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_VRAM);
 	ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_TT);
@@ -1030,19 +1030,17 @@ int radeon_mmap(struct file *filp, struct vm_area_struct *vma)
 static int radeon_mm_dump_table(struct seq_file *m, void *data)
 {
 	struct drm_info_node *node = (struct drm_info_node *)m->private;
-	unsigned ttm_pl = *(int *)node->info_ent->data;
+	unsigned ttm_pl = *(int*)node->info_ent->data;
 	struct drm_device *dev = node->minor->dev;
 	struct radeon_device *rdev = dev->dev_private;
-	struct drm_mm *mm = (struct drm_mm *)rdev->mman.bdev.man[ttm_pl].priv;
-	struct ttm_bo_global *glob = rdev->mman.bdev.glob;
+	struct ttm_mem_type_manager *man = &rdev->mman.bdev.man[ttm_pl];
 	struct drm_printer p = drm_seq_file_printer(m);
 
-	spin_lock(&glob->lru_lock);
-	drm_mm_print(mm, &p);
-	spin_unlock(&glob->lru_lock);
+	man->func->debug(man, &p);
 	return 0;
 }
 
+
 static int ttm_pl_vram = TTM_PL_VRAM;
 static int ttm_pl_tt = TTM_PL_TT;
 
diff --git a/drivers/gpu/drm/radeon/radeon_vm.c b/drivers/gpu/drm/radeon/radeon_vm.c
index 5f68245579a3..e5c0e635e371 100644
--- a/drivers/gpu/drm/radeon/radeon_vm.c
+++ b/drivers/gpu/drm/radeon/radeon_vm.c
@@ -139,7 +139,7 @@ struct radeon_bo_list *radeon_vm_get_bos(struct radeon_device *rdev,
 
 	/* add the vm page table to the list */
 	list[0].robj = vm->page_directory;
-	list[0].prefered_domains = RADEON_GEM_DOMAIN_VRAM;
+	list[0].preferred_domains = RADEON_GEM_DOMAIN_VRAM;
 	list[0].allowed_domains = RADEON_GEM_DOMAIN_VRAM;
 	list[0].tv.bo = &vm->page_directory->tbo;
 	list[0].tv.shared = true;
@@ -151,7 +151,7 @@ struct radeon_bo_list *radeon_vm_get_bos(struct radeon_device *rdev,
 			continue;
 
 		list[idx].robj = vm->page_tables[i].bo;
-		list[idx].prefered_domains = RADEON_GEM_DOMAIN_VRAM;
+		list[idx].preferred_domains = RADEON_GEM_DOMAIN_VRAM;
 		list[idx].allowed_domains = RADEON_GEM_DOMAIN_VRAM;
 		list[idx].tv.bo = &list[idx].robj->tbo;
 		list[idx].tv.shared = true;
@@ -1185,7 +1185,7 @@ int radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm)
 		vm->ids[i].last_id_use = NULL;
 	}
 	mutex_init(&vm->mutex);
-	vm->va = RB_ROOT;
+	vm->va = RB_ROOT_CACHED;
 	spin_lock_init(&vm->status_lock);
 	INIT_LIST_HEAD(&vm->invalidated);
 	INIT_LIST_HEAD(&vm->freed);
@@ -1232,10 +1232,11 @@ void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm)
 	struct radeon_bo_va *bo_va, *tmp;
 	int i, r;
 
-	if (!RB_EMPTY_ROOT(&vm->va)) {
+	if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
 		dev_err(rdev->dev, "still active bo inside vm\n");
 	}
-	rbtree_postorder_for_each_entry_safe(bo_va, tmp, &vm->va, it.rb) {
+	rbtree_postorder_for_each_entry_safe(bo_va, tmp,
+					     &vm->va.rb_root, it.rb) {
 		interval_tree_remove(&bo_va->it, &vm->va);
 		r = radeon_bo_reserve(bo_va->bo, false);
 		if (!r) {
diff --git a/drivers/gpu/drm/radeon/vce_v2_0.c b/drivers/gpu/drm/radeon/vce_v2_0.c
index fce214482e72..b0a43b68776d 100644
--- a/drivers/gpu/drm/radeon/vce_v2_0.c
+++ b/drivers/gpu/drm/radeon/vce_v2_0.c
@@ -104,6 +104,10 @@ static void vce_v2_0_disable_cg(struct radeon_device *rdev)
 	WREG32(VCE_CGTT_CLK_OVERRIDE, 7);
 }
 
+/*
+ * Local variable sw_cg is used for debugging purposes, in case we
+ * ran into problems with dynamic clock gating. Don't remove it.
+ */
 void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable)
 {
 	bool sw_cg = false;
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_crtc.c b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
index 345eff72f581..301ea1a8018e 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
@@ -13,6 +13,7 @@
 
 #include <linux/clk.h>
 #include <linux/mutex.h>
+#include <linux/sys_soc.h>
 
 #include <drm/drmP.h>
 #include <drm/drm_atomic.h>
@@ -129,10 +130,8 @@ static void rcar_du_dpll_divider(struct rcar_du_crtc *rcrtc,
 			for (fdpll = 1; fdpll < 32; fdpll++) {
 				unsigned long output;
 
-				/* 1/2 (FRQSEL=1) for duty rate 50% */
 				output = input * (n + 1) / (m + 1)
-				       / (fdpll + 1) / 2;
-
+				       / (fdpll + 1);
 				if (output >= 400000000)
 					continue;
 
@@ -158,6 +157,11 @@ done:
 		 best_diff);
 }
 
+static const struct soc_device_attribute rcar_du_r8a7795_es1[] = {
+	{ .soc_id = "r8a7795", .revision = "ES1.*" },
+	{ /* sentinel */ }
+};
+
 static void rcar_du_crtc_set_display_timing(struct rcar_du_crtc *rcrtc)
 {
 	const struct drm_display_mode *mode = &rcrtc->crtc.state->adjusted_mode;
@@ -168,7 +172,8 @@ static void rcar_du_crtc_set_display_timing(struct rcar_du_crtc *rcrtc)
 	u32 escr;
 	u32 div;
 
-	/* Compute the clock divisor and select the internal or external dot
+	/*
+	 * Compute the clock divisor and select the internal or external dot
 	 * clock based on the requested frequency.
 	 */
 	clk = clk_get_rate(rcrtc->clock);
@@ -185,7 +190,20 @@ static void rcar_du_crtc_set_display_timing(struct rcar_du_crtc *rcrtc)
 
 		extclk = clk_get_rate(rcrtc->extclock);
 		if (rcdu->info->dpll_ch & (1 << rcrtc->index)) {
-			rcar_du_dpll_divider(rcrtc, &dpll, extclk, mode_clock);
+			unsigned long target = mode_clock;
+
+			/*
+			 * The H3 ES1.x exhibits dot clock duty cycle stability
+			 * issues. We can work around them by configuring the
+			 * DPLL to twice the desired frequency, coupled with a
+			 * /2 post-divider. This isn't needed on other SoCs and
+			 * breaks HDMI output on M3-W for a currently unknown
+			 * reason, so restrict the workaround to H3 ES1.x.
+			 */
+			if (soc_device_match(rcar_du_r8a7795_es1))
+				target *= 2;
+
+			rcar_du_dpll_divider(rcrtc, &dpll, extclk, target);
 			extclk = dpll.output;
 		}
 
@@ -197,8 +215,6 @@ static void rcar_du_crtc_set_display_timing(struct rcar_du_crtc *rcrtc)
 
 		if (abs((long)extrate - (long)mode_clock) <
 		    abs((long)rate - (long)mode_clock)) {
-			dev_dbg(rcrtc->group->dev->dev,
-				"crtc%u: using external clock\n", rcrtc->index);
 
 			if (rcdu->info->dpll_ch & (1 << rcrtc->index)) {
 				u32 dpllcr = DPLLCR_CODE | DPLLCR_CLKE
@@ -215,12 +231,14 @@ static void rcar_du_crtc_set_display_timing(struct rcar_du_crtc *rcrtc)
 
 				rcar_du_group_write(rcrtc->group, DPLLCR,
 						    dpllcr);
-
-				escr = ESCR_DCLKSEL_DCLKIN | 1;
-			} else {
-				escr = ESCR_DCLKSEL_DCLKIN | extdiv;
 			}
+
+			escr = ESCR_DCLKSEL_DCLKIN | extdiv;
 		}
+
+		dev_dbg(rcrtc->group->dev->dev,
+			"mode clock %lu extrate %lu rate %lu ESCR 0x%08x\n",
+			mode_clock, extrate, rate, escr);
 	}
 
 	rcar_du_group_write(rcrtc->group, rcrtc->index % 2 ? ESCR2 : ESCR,
@@ -261,12 +279,14 @@ void rcar_du_crtc_route_output(struct drm_crtc *crtc,
 	struct rcar_du_crtc *rcrtc = to_rcar_crtc(crtc);
 	struct rcar_du_device *rcdu = rcrtc->group->dev;
 
-	/* Store the route from the CRTC output to the DU output. The DU will be
+	/*
+	 * Store the route from the CRTC output to the DU output. The DU will be
 	 * configured when starting the CRTC.
 	 */
 	rcrtc->outputs |= BIT(output);
 
-	/* Store RGB routing to DPAD0, the hardware will be configured when
+	/*
+	 * Store RGB routing to DPAD0, the hardware will be configured when
 	 * starting the CRTC.
 	 */
 	if (output == RCAR_DU_OUTPUT_DPAD0)
@@ -342,7 +362,8 @@ static void rcar_du_crtc_update_planes(struct rcar_du_crtc *rcrtc)
 		}
 	}
 
-	/* Update the planes to display timing and dot clock generator
+	/*
+	 * Update the planes to display timing and dot clock generator
 	 * associations.
 	 *
 	 * Updating the DPTSR register requires restarting the CRTC group,
@@ -431,14 +452,8 @@ static void rcar_du_crtc_wait_page_flip(struct rcar_du_crtc *rcrtc)
  * Start/Stop and Suspend/Resume
  */
 
-static void rcar_du_crtc_start(struct rcar_du_crtc *rcrtc)
+static void rcar_du_crtc_setup(struct rcar_du_crtc *rcrtc)
 {
-	struct drm_crtc *crtc = &rcrtc->crtc;
-	bool interlaced;
-
-	if (rcrtc->started)
-		return;
-
 	/* Set display off and background to black */
 	rcar_du_crtc_write(rcrtc, DOOR, DOOR_RGB(0, 0, 0));
 	rcar_du_crtc_write(rcrtc, BPOR, BPOR_RGB(0, 0, 0));
@@ -450,7 +465,20 @@ static void rcar_du_crtc_start(struct rcar_du_crtc *rcrtc)
 	/* Start with all planes disabled. */
 	rcar_du_group_write(rcrtc->group, rcrtc->index % 2 ? DS2PR : DS1PR, 0);
 
-	/* Select master sync mode. This enables display operation in master
+	/* Enable the VSP compositor. */
+	if (rcar_du_has(rcrtc->group->dev, RCAR_DU_FEATURE_VSP1_SOURCE))
+		rcar_du_vsp_enable(rcrtc);
+
+	/* Turn vertical blanking interrupt reporting on. */
+	drm_crtc_vblank_on(&rcrtc->crtc);
+}
+
+static void rcar_du_crtc_start(struct rcar_du_crtc *rcrtc)
+{
+	bool interlaced;
+
+	/*
+	 * Select master sync mode. This enables display operation in master
 	 * sync mode (with the HSYNC and VSYNC signals configured as outputs and
 	 * actively driven).
 	 */
@@ -460,38 +488,56 @@ static void rcar_du_crtc_start(struct rcar_du_crtc *rcrtc)
 			     DSYSR_TVM_MASTER);
 
 	rcar_du_group_start_stop(rcrtc->group, true);
+}
 
-	/* Enable the VSP compositor. */
-	if (rcar_du_has(rcrtc->group->dev, RCAR_DU_FEATURE_VSP1_SOURCE))
-		rcar_du_vsp_enable(rcrtc);
+static void rcar_du_crtc_disable_planes(struct rcar_du_crtc *rcrtc)
+{
+	struct rcar_du_device *rcdu = rcrtc->group->dev;
+	struct drm_crtc *crtc = &rcrtc->crtc;
+	u32 status;
 
-	/* Turn vertical blanking interrupt reporting back on. */
-	drm_crtc_vblank_on(crtc);
+	/* Make sure vblank interrupts are enabled. */
+	drm_crtc_vblank_get(crtc);
 
-	rcrtc->started = true;
+	/*
+	 * Disable planes and calculate how many vertical blanking interrupts we
+	 * have to wait for. If a vertical blanking interrupt has been triggered
+	 * but not processed yet, we don't know whether it occurred before or
+	 * after the planes got disabled. We thus have to wait for two vblank
+	 * interrupts in that case.
+	 */
+	spin_lock_irq(&rcrtc->vblank_lock);
+	rcar_du_group_write(rcrtc->group, rcrtc->index % 2 ? DS2PR : DS1PR, 0);
+	status = rcar_du_crtc_read(rcrtc, DSSR);
+	rcrtc->vblank_count = status & DSSR_VBK ? 2 : 1;
+	spin_unlock_irq(&rcrtc->vblank_lock);
+
+	if (!wait_event_timeout(rcrtc->vblank_wait, rcrtc->vblank_count == 0,
+				msecs_to_jiffies(100)))
+		dev_warn(rcdu->dev, "vertical blanking timeout\n");
+
+	drm_crtc_vblank_put(crtc);
 }
 
 static void rcar_du_crtc_stop(struct rcar_du_crtc *rcrtc)
 {
 	struct drm_crtc *crtc = &rcrtc->crtc;
 
-	if (!rcrtc->started)
-		return;
-
-	/* Disable all planes and wait for the change to take effect. This is
-	 * required as the DSnPR registers are updated on vblank, and no vblank
-	 * will occur once the CRTC is stopped. Disabling planes when starting
-	 * the CRTC thus wouldn't be enough as it would start scanning out
-	 * immediately from old frame buffers until the next vblank.
+	/*
+	 * Disable all planes and wait for the change to take effect. This is
+	 * required as the plane enable registers are updated on vblank, and no
+	 * vblank will occur once the CRTC is stopped. Disabling planes when
+	 * starting the CRTC thus wouldn't be enough as it would start scanning
+	 * out immediately from old frame buffers until the next vblank.
 	 *
 	 * This increases the CRTC stop delay, especially when multiple CRTCs
 	 * are stopped in one operation as we now wait for one vblank per CRTC.
 	 * Whether this can be improved needs to be researched.
 	 */
-	rcar_du_group_write(rcrtc->group, rcrtc->index % 2 ? DS2PR : DS1PR, 0);
-	drm_crtc_wait_one_vblank(crtc);
+	rcar_du_crtc_disable_planes(rcrtc);
 
-	/* Disable vertical blanking interrupt reporting. We first need to wait
+	/*
+	 * Disable vertical blanking interrupt reporting. We first need to wait
 	 * for page flip completion before stopping the CRTC as userspace
 	 * expects page flips to eventually complete.
 	 */
@@ -502,14 +548,13 @@ static void rcar_du_crtc_stop(struct rcar_du_crtc *rcrtc)
 	if (rcar_du_has(rcrtc->group->dev, RCAR_DU_FEATURE_VSP1_SOURCE))
 		rcar_du_vsp_disable(rcrtc);
 
-	/* Select switch sync mode. This stops display operation and configures
+	/*
+	 * Select switch sync mode. This stops display operation and configures
 	 * the HSYNC and VSYNC signals as inputs.
 	 */
 	rcar_du_crtc_clr_set(rcrtc, DSYSR, DSYSR_TVM_MASK, DSYSR_TVM_SWITCH);
 
 	rcar_du_group_start_stop(rcrtc->group, false);
-
-	rcrtc->started = false;
 }
 
 void rcar_du_crtc_suspend(struct rcar_du_crtc *rcrtc)
@@ -529,12 +574,10 @@ void rcar_du_crtc_resume(struct rcar_du_crtc *rcrtc)
 		return;
 
 	rcar_du_crtc_get(rcrtc);
-	rcar_du_crtc_start(rcrtc);
+	rcar_du_crtc_setup(rcrtc);
 
 	/* Commit the planes state. */
-	if (rcar_du_has(rcrtc->group->dev, RCAR_DU_FEATURE_VSP1_SOURCE)) {
-		rcar_du_vsp_enable(rcrtc);
-	} else {
+	if (!rcar_du_has(rcrtc->group->dev, RCAR_DU_FEATURE_VSP1_SOURCE)) {
 		for (i = 0; i < rcrtc->group->num_planes; ++i) {
 			struct rcar_du_plane *plane = &rcrtc->group->planes[i];
 
@@ -546,21 +589,33 @@ void rcar_du_crtc_resume(struct rcar_du_crtc *rcrtc)
 	}
 
 	rcar_du_crtc_update_planes(rcrtc);
+	rcar_du_crtc_start(rcrtc);
 }
 
 /* -----------------------------------------------------------------------------
  * CRTC Functions
  */
 
-static void rcar_du_crtc_enable(struct drm_crtc *crtc)
+static void rcar_du_crtc_atomic_enable(struct drm_crtc *crtc,
+				       struct drm_crtc_state *old_state)
 {
 	struct rcar_du_crtc *rcrtc = to_rcar_crtc(crtc);
 
-	rcar_du_crtc_get(rcrtc);
+	/*
+	 * If the CRTC has already been setup by the .atomic_begin() handler we
+	 * can skip the setup stage.
+	 */
+	if (!rcrtc->initialized) {
+		rcar_du_crtc_get(rcrtc);
+		rcar_du_crtc_setup(rcrtc);
+		rcrtc->initialized = true;
+	}
+
 	rcar_du_crtc_start(rcrtc);
 }
 
-static void rcar_du_crtc_disable(struct drm_crtc *crtc)
+static void rcar_du_crtc_atomic_disable(struct drm_crtc *crtc,
+					struct drm_crtc_state *old_state)
 {
 	struct rcar_du_crtc *rcrtc = to_rcar_crtc(crtc);
 
@@ -574,6 +629,7 @@ static void rcar_du_crtc_disable(struct drm_crtc *crtc)
 	}
 	spin_unlock_irq(&crtc->dev->event_lock);
 
+	rcrtc->initialized = false;
 	rcrtc->outputs = 0;
 }
 
@@ -582,6 +638,19 @@ static void rcar_du_crtc_atomic_begin(struct drm_crtc *crtc,
 {
 	struct rcar_du_crtc *rcrtc = to_rcar_crtc(crtc);
 
+	WARN_ON(!crtc->state->enable);
+
+	/*
+	 * If a mode set is in progress we can be called with the CRTC disabled.
+	 * We then need to first setup the CRTC in order to configure planes.
+	 * The .atomic_enable() handler will notice and skip the CRTC setup.
+	 */
+	if (!rcrtc->initialized) {
+		rcar_du_crtc_get(rcrtc);
+		rcar_du_crtc_setup(rcrtc);
+		rcrtc->initialized = true;
+	}
+
 	if (rcar_du_has(rcrtc->group->dev, RCAR_DU_FEATURE_VSP1_SOURCE))
 		rcar_du_vsp_atomic_begin(rcrtc);
 }
@@ -609,10 +678,10 @@ static void rcar_du_crtc_atomic_flush(struct drm_crtc *crtc,
 }
 
 static const struct drm_crtc_helper_funcs crtc_helper_funcs = {
-	.disable = rcar_du_crtc_disable,
-	.enable = rcar_du_crtc_enable,
 	.atomic_begin = rcar_du_crtc_atomic_begin,
 	.atomic_flush = rcar_du_crtc_atomic_flush,
+	.atomic_enable = rcar_du_crtc_atomic_enable,
+	.atomic_disable = rcar_du_crtc_atomic_disable,
 };
 
 static int rcar_du_crtc_enable_vblank(struct drm_crtc *crtc)
@@ -621,6 +690,7 @@ static int rcar_du_crtc_enable_vblank(struct drm_crtc *crtc)
 
 	rcar_du_crtc_write(rcrtc, DSRCR, DSRCR_VBCL);
 	rcar_du_crtc_set(rcrtc, DIER, DIER_VBE);
+	rcrtc->vblank_enable = true;
 
 	return 0;
 }
@@ -630,6 +700,7 @@ static void rcar_du_crtc_disable_vblank(struct drm_crtc *crtc)
 	struct rcar_du_crtc *rcrtc = to_rcar_crtc(crtc);
 
 	rcar_du_crtc_clr(rcrtc, DIER, DIER_VBE);
+	rcrtc->vblank_enable = false;
 }
 
 static const struct drm_crtc_funcs crtc_funcs = {
@@ -654,14 +725,30 @@ static irqreturn_t rcar_du_crtc_irq(int irq, void *arg)
 	irqreturn_t ret = IRQ_NONE;
 	u32 status;
 
+	spin_lock(&rcrtc->vblank_lock);
+
 	status = rcar_du_crtc_read(rcrtc, DSSR);
 	rcar_du_crtc_write(rcrtc, DSRCR, status & DSRCR_MASK);
 
-	if (status & DSSR_FRM) {
-		drm_crtc_handle_vblank(&rcrtc->crtc);
+	if (status & DSSR_VBK) {
+		/*
+		 * Wake up the vblank wait if the counter reaches 0. This must
+		 * be protected by the vblank_lock to avoid races in
+		 * rcar_du_crtc_disable_planes().
+		 */
+		if (rcrtc->vblank_count) {
+			if (--rcrtc->vblank_count == 0)
+				wake_up(&rcrtc->vblank_wait);
+		}
+	}
 
-		if (rcdu->info->gen < 3)
+	spin_unlock(&rcrtc->vblank_lock);
+
+	if (status & DSSR_VBK) {
+		if (rcdu->info->gen < 3) {
+			drm_crtc_handle_vblank(&rcrtc->crtc);
 			rcar_du_crtc_finish_page_flip(rcrtc);
+		}
 
 		ret = IRQ_HANDLED;
 	}
@@ -715,13 +802,15 @@ int rcar_du_crtc_create(struct rcar_du_group *rgrp, unsigned int index)
 	}
 
 	init_waitqueue_head(&rcrtc->flip_wait);
+	init_waitqueue_head(&rcrtc->vblank_wait);
+	spin_lock_init(&rcrtc->vblank_lock);
 
 	rcrtc->group = rgrp;
 	rcrtc->mmio_offset = mmio_offsets[index];
 	rcrtc->index = index;
 
 	if (rcar_du_has(rcdu, RCAR_DU_FEATURE_VSP1_SOURCE))
-		primary = &rcrtc->vsp->planes[0].plane;
+		primary = &rcrtc->vsp->planes[rcrtc->vsp_pipe].plane;
 	else
 		primary = &rgrp->planes[index % 2].plane;
 
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_crtc.h b/drivers/gpu/drm/rcar-du/rcar_du_crtc.h
index b199ed5adf36..fdc2bf99bda1 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_crtc.h
+++ b/drivers/gpu/drm/rcar-du/rcar_du_crtc.h
@@ -15,6 +15,7 @@
 #define __RCAR_DU_CRTC_H__
 
 #include <linux/mutex.h>
+#include <linux/spinlock.h>
 #include <linux/wait.h>
 
 #include <drm/drmP.h>
@@ -30,11 +31,17 @@ struct rcar_du_vsp;
  * @extclock: external pixel dot clock (optional)
  * @mmio_offset: offset of the CRTC registers in the DU MMIO block
  * @index: CRTC software and hardware index
- * @started: whether the CRTC has been started and is running
+ * @initialized: whether the CRTC has been initialized and clocks enabled
+ * @vblank_enable: whether vblank events are enabled on this CRTC
  * @event: event to post when the pending page flip completes
  * @flip_wait: wait queue used to signal page flip completion
+ * @vblank_lock: protects vblank_wait and vblank_count
+ * @vblank_wait: wait queue used to signal vertical blanking
+ * @vblank_count: number of vertical blanking interrupts to wait for
  * @outputs: bitmask of the outputs (enum rcar_du_output) driven by this CRTC
  * @group: CRTC group this CRTC belongs to
+ * @vsp: VSP feeding video to this CRTC
+ * @vsp_pipe: index of the VSP pipeline feeding video to this CRTC
  */
 struct rcar_du_crtc {
 	struct drm_crtc crtc;
@@ -43,15 +50,21 @@ struct rcar_du_crtc {
 	struct clk *extclock;
 	unsigned int mmio_offset;
 	unsigned int index;
-	bool started;
+	bool initialized;
 
+	bool vblank_enable;
 	struct drm_pending_vblank_event *event;
 	wait_queue_head_t flip_wait;
 
+	spinlock_t vblank_lock;
+	wait_queue_head_t vblank_wait;
+	unsigned int vblank_count;
+
 	unsigned int outputs;
 
 	struct rcar_du_group *group;
 	struct rcar_du_vsp *vsp;
+	unsigned int vsp_pipe;
 };
 
 #define to_rcar_crtc(c)	container_of(c, struct rcar_du_crtc, crtc)
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_drv.c b/drivers/gpu/drm/rcar-du/rcar_du_drv.c
index d6a0255181cc..d2f29e6b1112 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_drv.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_drv.c
@@ -39,7 +39,8 @@ static const struct rcar_du_device_info rcar_du_r8a7779_info = {
 	.features = 0,
 	.num_crtcs = 2,
 	.routes = {
-		/* R8A7779 has two RGB outputs and one (currently unsupported)
+		/*
+		 * R8A7779 has two RGB outputs and one (currently unsupported)
 		 * TCON output.
 		 */
 		[RCAR_DU_OUTPUT_DPAD0] = {
@@ -61,7 +62,8 @@ static const struct rcar_du_device_info rcar_du_r8a7790_info = {
 	.quirks = RCAR_DU_QUIRK_ALIGN_128B | RCAR_DU_QUIRK_LVDS_LANES,
 	.num_crtcs = 3,
 	.routes = {
-		/* R8A7790 has one RGB output, two LVDS outputs and one
+		/*
+		 * R8A7790 has one RGB output, two LVDS outputs and one
 		 * (currently unsupported) TCON output.
 		 */
 		[RCAR_DU_OUTPUT_DPAD0] = {
@@ -87,7 +89,8 @@ static const struct rcar_du_device_info rcar_du_r8a7791_info = {
 		  | RCAR_DU_FEATURE_EXT_CTRL_REGS,
 	.num_crtcs = 2,
 	.routes = {
-		/* R8A779[13] has one RGB output, one LVDS output and one
+		/*
+		 * R8A779[13] has one RGB output, one LVDS output and one
 		 * (currently unsupported) TCON output.
 		 */
 		[RCAR_DU_OUTPUT_DPAD0] = {
@@ -127,7 +130,8 @@ static const struct rcar_du_device_info rcar_du_r8a7794_info = {
 		  | RCAR_DU_FEATURE_EXT_CTRL_REGS,
 	.num_crtcs = 2,
 	.routes = {
-		/* R8A7794 has two RGB outputs and one (currently unsupported)
+		/*
+		 * R8A7794 has two RGB outputs and one (currently unsupported)
 		 * TCON output.
 		 */
 		[RCAR_DU_OUTPUT_DPAD0] = {
@@ -149,7 +153,8 @@ static const struct rcar_du_device_info rcar_du_r8a7795_info = {
 		  | RCAR_DU_FEATURE_VSP1_SOURCE,
 	.num_crtcs = 4,
 	.routes = {
-		/* R8A7795 has one RGB output, two HDMI outputs and one
+		/*
+		 * R8A7795 has one RGB output, two HDMI outputs and one
 		 * LVDS output.
 		 */
 		[RCAR_DU_OUTPUT_DPAD0] = {
@@ -180,19 +185,25 @@ static const struct rcar_du_device_info rcar_du_r8a7796_info = {
 		  | RCAR_DU_FEATURE_VSP1_SOURCE,
 	.num_crtcs = 3,
 	.routes = {
-		/* R8A7796 has one RGB output, one LVDS output and one
-		 * (currently unsupported) HDMI output.
+		/*
+		 * R8A7796 has one RGB output, one LVDS output and one HDMI
+		 * output.
 		 */
 		[RCAR_DU_OUTPUT_DPAD0] = {
 			.possible_crtcs = BIT(2),
 			.port = 0,
 		},
+		[RCAR_DU_OUTPUT_HDMI0] = {
+			.possible_crtcs = BIT(1),
+			.port = 1,
+		},
 		[RCAR_DU_OUTPUT_LVDS0] = {
 			.possible_crtcs = BIT(0),
 			.port = 2,
 		},
 	},
 	.num_lvds = 1,
+	.dpll_ch =  BIT(1),
 };
 
 static const struct of_device_id rcar_du_of_table[] = {
@@ -238,8 +249,6 @@ static struct drm_driver rcar_du_driver = {
 	.gem_prime_vunmap	= drm_gem_cma_prime_vunmap,
 	.gem_prime_mmap		= drm_gem_cma_prime_mmap,
 	.dumb_create		= rcar_du_dumb_create,
-	.dumb_map_offset	= drm_gem_cma_dumb_map_offset,
-	.dumb_destroy		= drm_gem_dumb_destroy,
 	.fops			= &rcar_du_fops,
 	.name			= "rcar-du",
 	.desc			= "Renesas R-Car Display Unit",
@@ -341,7 +350,8 @@ static int rcar_du_probe(struct platform_device *pdev)
 
 	ddev->irq_enabled = 1;
 
-	/* Register the DRM device with the core and the connectors with
+	/*
+	 * Register the DRM device with the core and the connectors with
 	 * sysfs.
 	 */
 	ret = drm_dev_register(ddev, 0);
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_encoder.c b/drivers/gpu/drm/rcar-du/rcar_du_encoder.c
index 3e048dd98b64..ba8d2804c1d1 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_encoder.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_encoder.c
@@ -186,8 +186,8 @@ int rcar_du_encoder_init(struct rcar_du_device *rcdu,
 	}
 
 	if (enc_node) {
-		dev_dbg(rcdu->dev, "initializing encoder %s for output %u\n",
-			of_node_full_name(enc_node), output);
+		dev_dbg(rcdu->dev, "initializing encoder %pOF for output %u\n",
+			enc_node, output);
 
 		/* Locate the DRM bridge from the encoder DT node. */
 		bridge = of_drm_find_bridge(enc_node);
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_group.c b/drivers/gpu/drm/rcar-du/rcar_du_group.c
index 64738fca96d0..2f37ea901873 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_group.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_group.c
@@ -64,7 +64,8 @@ static void rcar_du_group_setup_defr8(struct rcar_du_group *rgrp)
 	if (rcdu->info->gen < 3) {
 		defr8 |= DEFR8_DEFE8;
 
-		/* On Gen2 the DEFR8 register for the first group also controls
+		/*
+		 * On Gen2 the DEFR8 register for the first group also controls
 		 * RGB output routing to DPAD0 and VSPD1 routing to DU0/1/2 for
 		 * DU instances that support it.
 		 */
@@ -75,7 +76,8 @@ static void rcar_du_group_setup_defr8(struct rcar_du_group *rgrp)
 				defr8 |= DEFR8_VSCS;
 		}
 	} else {
-		/* On Gen3 VSPD routing can't be configured, but DPAD routing
+		/*
+		 * On Gen3 VSPD routing can't be configured, but DPAD routing
 		 * needs to be set despite having a single option available.
 		 */
 		u32 crtc = ffs(possible_crtcs) - 1;
@@ -124,7 +126,8 @@ static void rcar_du_group_setup(struct rcar_du_group *rgrp)
 	if (rcdu->info->gen >= 3)
 		rcar_du_group_write(rgrp, DEFR10, DEFR10_CODE | DEFR10_DEFE10);
 
-	/* Use DS1PR and DS2PR to configure planes priorities and connects the
+	/*
+	 * Use DS1PR and DS2PR to configure planes priorities and connects the
 	 * superposition 0 to DU0 pins. DU1 pins will be configured dynamically.
 	 */
 	rcar_du_group_write(rgrp, DORCR, DORCR_PG1D_DS1 | DORCR_DPRS);
@@ -177,7 +180,8 @@ static void __rcar_du_group_start_stop(struct rcar_du_group *rgrp, bool start)
 
 void rcar_du_group_start_stop(struct rcar_du_group *rgrp, bool start)
 {
-	/* Many of the configuration bits are only updated when the display
+	/*
+	 * Many of the configuration bits are only updated when the display
 	 * reset (DRES) bit in DSYSR is set to 1, disabling *both* CRTCs. Some
 	 * of those bits could be pre-configured, but others (especially the
 	 * bits related to plane assignment to display timing controllers) need
@@ -208,23 +212,32 @@ void rcar_du_group_restart(struct rcar_du_group *rgrp)
 
 int rcar_du_set_dpad0_vsp1_routing(struct rcar_du_device *rcdu)
 {
+	struct rcar_du_group *rgrp;
+	struct rcar_du_crtc *crtc;
+	unsigned int index;
 	int ret;
 
 	if (!rcar_du_has(rcdu, RCAR_DU_FEATURE_EXT_CTRL_REGS))
 		return 0;
 
-	/* RGB output routing to DPAD0 and VSP1D routing to DU0/1/2 are
-	 * configured in the DEFR8 register of the first group. As this function
-	 * can be called with the DU0 and DU1 CRTCs disabled, we need to enable
-	 * the first group clock before accessing the register.
+	/*
+	 * RGB output routing to DPAD0 and VSP1D routing to DU0/1/2 are
+	 * configured in the DEFR8 register of the first group on Gen2 and the
+	 * last group on Gen3. As this function can be called with the DU
+	 * channels of the corresponding CRTCs disabled, we need to enable the
+	 * group clock before accessing the register.
 	 */
-	ret = clk_prepare_enable(rcdu->crtcs[0].clock);
+	index = rcdu->info->gen < 3 ? 0 : DIV_ROUND_UP(rcdu->num_crtcs, 2) - 1;
+	rgrp = &rcdu->groups[index];
+	crtc = &rcdu->crtcs[index * 2];
+
+	ret = clk_prepare_enable(crtc->clock);
 	if (ret < 0)
 		return ret;
 
-	rcar_du_group_setup_defr8(&rcdu->groups[0]);
+	rcar_du_group_setup_defr8(rgrp);
 
-	clk_disable_unprepare(rcdu->crtcs[0].clock);
+	clk_disable_unprepare(crtc->clock);
 
 	return 0;
 }
@@ -236,7 +249,8 @@ int rcar_du_group_set_routing(struct rcar_du_group *rgrp)
 
 	dorcr &= ~(DORCR_PG2T | DORCR_DK2S | DORCR_PG2D_MASK);
 
-	/* Set the DPAD1 pins sources. Select CRTC 0 if explicitly requested and
+	/*
+	 * Set the DPAD1 pins sources. Select CRTC 0 if explicitly requested and
 	 * CRTC 1 in all other cases to avoid cloning CRTC 0 to DPAD0 and DPAD1
 	 * by default.
 	 */
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_kms.c b/drivers/gpu/drm/rcar-du/rcar_du_kms.c
index f4125c8ca902..7278b9703c15 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_kms.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_kms.c
@@ -96,7 +96,8 @@ static const struct rcar_du_format_info rcar_du_format_infos[] = {
 		.pnmr = PnMR_SPIM_TP_OFF | PnMR_DDDF_YC,
 		.edf = PnDDCR4_EDF_NONE,
 	},
-	/* The following formats are not supported on Gen2 and thus have no
+	/*
+	 * The following formats are not supported on Gen2 and thus have no
 	 * associated .pnmr or .edf settings.
 	 */
 	{
@@ -153,7 +154,8 @@ int rcar_du_dumb_create(struct drm_file *file, struct drm_device *dev,
 	unsigned int min_pitch = DIV_ROUND_UP(args->width * args->bpp, 8);
 	unsigned int align;
 
-	/* The R8A7779 DU requires a 16 pixels pitch alignment as documented,
+	/*
+	 * The R8A7779 DU requires a 16 pixels pitch alignment as documented,
 	 * but the R8A7790 DU seems to require a 128 bytes pitch alignment.
 	 */
 	if (rcar_du_needs(rcdu, RCAR_DU_QUIRK_ALIGN_128B))
@@ -255,12 +257,12 @@ static void rcar_du_atomic_commit_tail(struct drm_atomic_state *old_state)
 
 	/* Apply the atomic update. */
 	drm_atomic_helper_commit_modeset_disables(dev, old_state);
-	drm_atomic_helper_commit_modeset_enables(dev, old_state);
 	drm_atomic_helper_commit_planes(dev, old_state,
 					DRM_PLANE_COMMIT_ACTIVE_ONLY);
+	drm_atomic_helper_commit_modeset_enables(dev, old_state);
 
 	drm_atomic_helper_commit_hw_done(old_state);
-	drm_atomic_helper_wait_for_vblanks(dev, old_state);
+	drm_atomic_helper_wait_for_flip_done(dev, old_state);
 
 	drm_atomic_helper_cleanup_planes(dev, old_state);
 }
@@ -297,19 +299,19 @@ static int rcar_du_encoders_init_one(struct rcar_du_device *rcdu,
 	 */
 	entity = of_graph_get_remote_port_parent(ep->local_node);
 	if (!entity) {
-		dev_dbg(rcdu->dev, "unconnected endpoint %s, skipping\n",
-			ep->local_node->full_name);
+		dev_dbg(rcdu->dev, "unconnected endpoint %pOF, skipping\n",
+			ep->local_node);
 		return -ENODEV;
 	}
 
 	if (!of_device_is_available(entity)) {
 		dev_dbg(rcdu->dev,
-			"connected entity %s is disabled, skipping\n",
-			entity->full_name);
+			"connected entity %pOF is disabled, skipping\n",
+			entity);
 		return -ENODEV;
 	}
 
-	entity_ep_node = of_parse_phandle(ep->local_node, "remote-endpoint", 0);
+	entity_ep_node = of_graph_get_remote_endpoint(ep->local_node);
 
 	for_each_endpoint_of_node(entity, ep_node) {
 		if (ep_node == entity_ep_node)
@@ -325,8 +327,8 @@ static int rcar_du_encoders_init_one(struct rcar_du_device *rcdu,
 
 		if (!connector) {
 			dev_warn(rcdu->dev,
-				 "no connector for encoder %s, skipping\n",
-				 encoder->full_name);
+				 "no connector for encoder %pOF, skipping\n",
+				 encoder);
 			of_node_put(entity_ep_node);
 			of_node_put(encoder);
 			return -ENODEV;
@@ -348,8 +350,8 @@ static int rcar_du_encoders_init_one(struct rcar_du_device *rcdu,
 	ret = rcar_du_encoder_init(rcdu, output, encoder, connector);
 	if (ret && ret != -EPROBE_DEFER)
 		dev_warn(rcdu->dev,
-			 "failed to initialize encoder %s on output %u (%d), skipping\n",
-			 of_node_full_name(encoder), output, ret);
+			 "failed to initialize encoder %pOF on output %u (%d), skipping\n",
+			 encoder, output, ret);
 
 	of_node_put(encoder);
 	of_node_put(connector);
@@ -419,7 +421,8 @@ static int rcar_du_properties_init(struct rcar_du_device *rcdu)
 	if (rcdu->props.alpha == NULL)
 		return -ENOMEM;
 
-	/* The color key is expressed as an RGB888 triplet stored in a 32-bit
+	/*
+	 * The color key is expressed as an RGB888 triplet stored in a 32-bit
 	 * integer in XRGB8888 format. Bit 24 is used as a flag to disable (0)
 	 * or enable source color keying (1).
 	 */
@@ -432,6 +435,81 @@ static int rcar_du_properties_init(struct rcar_du_device *rcdu)
 	return 0;
 }
 
+static int rcar_du_vsps_init(struct rcar_du_device *rcdu)
+{
+	const struct device_node *np = rcdu->dev->of_node;
+	struct of_phandle_args args;
+	struct {
+		struct device_node *np;
+		unsigned int crtcs_mask;
+	} vsps[RCAR_DU_MAX_VSPS] = { { 0, }, };
+	unsigned int vsps_count = 0;
+	unsigned int cells;
+	unsigned int i;
+	int ret;
+
+	/*
+	 * First parse the DT vsps property to populate the list of VSPs. Each
+	 * entry contains a pointer to the VSP DT node and a bitmask of the
+	 * connected DU CRTCs.
+	 */
+	cells = of_property_count_u32_elems(np, "vsps") / rcdu->num_crtcs - 1;
+	if (cells > 1)
+		return -EINVAL;
+
+	for (i = 0; i < rcdu->num_crtcs; ++i) {
+		unsigned int j;
+
+		ret = of_parse_phandle_with_fixed_args(np, "vsps", cells, i,
+						       &args);
+		if (ret < 0)
+			goto error;
+
+		/*
+		 * Add the VSP to the list or update the corresponding existing
+		 * entry if the VSP has already been added.
+		 */
+		for (j = 0; j < vsps_count; ++j) {
+			if (vsps[j].np == args.np)
+				break;
+		}
+
+		if (j < vsps_count)
+			of_node_put(args.np);
+		else
+			vsps[vsps_count++].np = args.np;
+
+		vsps[j].crtcs_mask |= BIT(i);
+
+		/* Store the VSP pointer and pipe index in the CRTC. */
+		rcdu->crtcs[i].vsp = &rcdu->vsps[j];
+		rcdu->crtcs[i].vsp_pipe = cells >= 1 ? args.args[0] : 0;
+	}
+
+	/*
+	 * Then initialize all the VSPs from the node pointers and CRTCs bitmask
+	 * computed previously.
+	 */
+	for (i = 0; i < vsps_count; ++i) {
+		struct rcar_du_vsp *vsp = &rcdu->vsps[i];
+
+		vsp->index = i;
+		vsp->dev = rcdu;
+
+		ret = rcar_du_vsp_init(vsp, vsps[i].np, vsps[i].crtcs_mask);
+		if (ret < 0)
+			goto error;
+	}
+
+	return 0;
+
+error:
+	for (i = 0; i < ARRAY_SIZE(vsps); ++i)
+		of_node_put(vsps[i].np);
+
+	return ret;
+}
+
 int rcar_du_modeset_init(struct rcar_du_device *rcdu)
 {
 	static const unsigned int mmio_offsets[] = {
@@ -461,7 +539,8 @@ int rcar_du_modeset_init(struct rcar_du_device *rcdu)
 	if (ret < 0)
 		return ret;
 
-	/* Initialize vertical blanking interrupts handling. Start with vblank
+	/*
+	 * Initialize vertical blanking interrupts handling. Start with vblank
 	 * disabled for all CRTCs.
 	 */
 	ret = drm_vblank_init(dev, (1 << rcdu->info->num_crtcs) - 1);
@@ -481,7 +560,8 @@ int rcar_du_modeset_init(struct rcar_du_device *rcdu)
 		rgrp->index = i;
 		rgrp->num_crtcs = min(rcdu->num_crtcs - 2 * i, 2U);
 
-		/* If we have more than one CRTCs in this group pre-associate
+		/*
+		 * If we have more than one CRTCs in this group pre-associate
 		 * the low-order planes with CRTC 0 and the high-order planes
 		 * with CRTC 1 to minimize flicker occurring when the
 		 * association is changed.
@@ -499,17 +579,9 @@ int rcar_du_modeset_init(struct rcar_du_device *rcdu)
 
 	/* Initialize the compositors. */
 	if (rcar_du_has(rcdu, RCAR_DU_FEATURE_VSP1_SOURCE)) {
-		for (i = 0; i < rcdu->num_crtcs; ++i) {
-			struct rcar_du_vsp *vsp = &rcdu->vsps[i];
-
-			vsp->index = i;
-			vsp->dev = rcdu;
-			rcdu->crtcs[i].vsp = vsp;
-
-			ret = rcar_du_vsp_init(vsp);
-			if (ret < 0)
-				return ret;
-		}
+		ret = rcar_du_vsps_init(rcdu);
+		if (ret < 0)
+			return ret;
 	}
 
 	/* Create the CRTCs. */
@@ -537,7 +609,8 @@ int rcar_du_modeset_init(struct rcar_du_device *rcdu)
 
 	num_encoders = ret;
 
-	/* Set the possible CRTCs and possible clones. There's always at least
+	/*
+	 * Set the possible CRTCs and possible clones. There's always at least
 	 * one way for all encoders to clone each other, set all bits in the
 	 * possible clones field.
 	 */
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_lvdscon.c b/drivers/gpu/drm/rcar-du/rcar_du_lvdscon.c
index ee91481131ad..b373ad48ef5f 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_lvdscon.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_lvdscon.c
@@ -46,7 +46,6 @@ static void rcar_du_lvds_connector_destroy(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.reset = drm_atomic_helper_connector_reset,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = rcar_du_lvds_connector_destroy,
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_lvdsenc.c b/drivers/gpu/drm/rcar-du/rcar_du_lvdsenc.c
index 1661f6201210..12d22f3db1af 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_lvdsenc.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_lvdsenc.c
@@ -59,7 +59,8 @@ static void rcar_du_lvdsenc_start_gen2(struct rcar_du_lvdsenc *lvds,
 
 	rcar_lvds_write(lvds, LVDPLLCR, pllcr);
 
-	/* Select the input, hardcode mode 0, enable LVDS operation and turn
+	/*
+	 * Select the input, hardcode mode 0, enable LVDS operation and turn
 	 * bias circuitry on.
 	 */
 	lvdcr0 = (lvds->mode << LVDCR0_LVMD_SHIFT) | LVDCR0_BEN | LVDCR0_LVEN;
@@ -73,7 +74,8 @@ static void rcar_du_lvdsenc_start_gen2(struct rcar_du_lvdsenc *lvds,
 			LVDCR1_CHSTBY_GEN2(1) | LVDCR1_CHSTBY_GEN2(0) |
 			LVDCR1_CLKSTBY_GEN2);
 
-	/* Turn the PLL on, wait for the startup delay, and turn the output
+	/*
+	 * Turn the PLL on, wait for the startup delay, and turn the output
 	 * on.
 	 */
 	lvdcr0 |= LVDCR0_PLLON;
@@ -140,7 +142,8 @@ static int rcar_du_lvdsenc_start(struct rcar_du_lvdsenc *lvds,
 	if (ret < 0)
 		return ret;
 
-	/* Hardcode the channels and control signals routing for now.
+	/*
+	 * Hardcode the channels and control signals routing for now.
 	 *
 	 * HSYNC -> CTRL0
 	 * VSYNC -> CTRL1
@@ -202,7 +205,8 @@ void rcar_du_lvdsenc_atomic_check(struct rcar_du_lvdsenc *lvds,
 {
 	struct rcar_du_device *rcdu = lvds->dev;
 
-	/* The internal LVDS encoder has a restricted clock frequency operating
+	/*
+	 * The internal LVDS encoder has a restricted clock frequency operating
 	 * range (30MHz to 150MHz on Gen2, 25.175MHz to 148.5MHz on Gen3). Clamp
 	 * the clock accordingly.
 	 */
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_plane.c b/drivers/gpu/drm/rcar-du/rcar_du_plane.c
index dcde6288da6c..61833cc1c699 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_plane.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_plane.c
@@ -50,23 +50,21 @@
  * automatically when the core swaps the old and new states.
  */
 
-static bool rcar_du_plane_needs_realloc(struct rcar_du_plane *plane,
-					struct rcar_du_plane_state *new_state)
+static bool rcar_du_plane_needs_realloc(
+				const struct rcar_du_plane_state *old_state,
+				const struct rcar_du_plane_state *new_state)
 {
-	struct rcar_du_plane_state *cur_state;
-
-	cur_state = to_rcar_plane_state(plane->plane.state);
-
-	/* Lowering the number of planes doesn't strictly require reallocation
+	/*
+	 * Lowering the number of planes doesn't strictly require reallocation
 	 * as the extra hardware plane will be freed when committing, but doing
 	 * so could lead to more fragmentation.
 	 */
-	if (!cur_state->format ||
-	    cur_state->format->planes != new_state->format->planes)
+	if (!old_state->format ||
+	    old_state->format->planes != new_state->format->planes)
 		return true;
 
 	/* Reallocate hardware planes if the source has changed. */
-	if (cur_state->source != new_state->source)
+	if (old_state->source != new_state->source)
 		return true;
 
 	return false;
@@ -141,37 +139,43 @@ int rcar_du_atomic_check_planes(struct drm_device *dev,
 	unsigned int groups = 0;
 	unsigned int i;
 	struct drm_plane *drm_plane;
-	struct drm_plane_state *drm_plane_state;
+	struct drm_plane_state *old_drm_plane_state;
+	struct drm_plane_state *new_drm_plane_state;
 
 	/* Check if hardware planes need to be reallocated. */
-	for_each_plane_in_state(state, drm_plane, drm_plane_state, i) {
-		struct rcar_du_plane_state *plane_state;
+	for_each_oldnew_plane_in_state(state, drm_plane, old_drm_plane_state,
+				       new_drm_plane_state, i) {
+		struct rcar_du_plane_state *old_plane_state;
+		struct rcar_du_plane_state *new_plane_state;
 		struct rcar_du_plane *plane;
 		unsigned int index;
 
 		plane = to_rcar_plane(drm_plane);
-		plane_state = to_rcar_plane_state(drm_plane_state);
+		old_plane_state = to_rcar_plane_state(old_drm_plane_state);
+		new_plane_state = to_rcar_plane_state(new_drm_plane_state);
 
 		dev_dbg(rcdu->dev, "%s: checking plane (%u,%tu)\n", __func__,
 			plane->group->index, plane - plane->group->planes);
 
-		/* If the plane is being disabled we don't need to go through
+		/*
+		 * If the plane is being disabled we don't need to go through
 		 * the full reallocation procedure. Just mark the hardware
 		 * plane(s) as freed.
 		 */
-		if (!plane_state->format) {
+		if (!new_plane_state->format) {
 			dev_dbg(rcdu->dev, "%s: plane is being disabled\n",
 				__func__);
 			index = plane - plane->group->planes;
 			group_freed_planes[plane->group->index] |= 1 << index;
-			plane_state->hwindex = -1;
+			new_plane_state->hwindex = -1;
 			continue;
 		}
 
-		/* If the plane needs to be reallocated mark it as such, and
+		/*
+		 * If the plane needs to be reallocated mark it as such, and
 		 * mark the hardware plane(s) as free.
 		 */
-		if (rcar_du_plane_needs_realloc(plane, plane_state)) {
+		if (rcar_du_plane_needs_realloc(old_plane_state, new_plane_state)) {
 			dev_dbg(rcdu->dev, "%s: plane needs reallocation\n",
 				__func__);
 			groups |= 1 << plane->group->index;
@@ -179,14 +183,15 @@ int rcar_du_atomic_check_planes(struct drm_device *dev,
 
 			index = plane - plane->group->planes;
 			group_freed_planes[plane->group->index] |= 1 << index;
-			plane_state->hwindex = -1;
+			new_plane_state->hwindex = -1;
 		}
 	}
 
 	if (!needs_realloc)
 		return 0;
 
-	/* Grab all plane states for the groups that need reallocation to ensure
+	/*
+	 * Grab all plane states for the groups that need reallocation to ensure
 	 * locking and avoid racy updates. This serializes the update operation,
 	 * but there's not much we can do about it as that's the hardware
 	 * design.
@@ -204,14 +209,15 @@ int rcar_du_atomic_check_planes(struct drm_device *dev,
 
 		for (i = 0; i < group->num_planes; ++i) {
 			struct rcar_du_plane *plane = &group->planes[i];
-			struct rcar_du_plane_state *plane_state;
+			struct rcar_du_plane_state *new_plane_state;
 			struct drm_plane_state *s;
 
 			s = drm_atomic_get_plane_state(state, &plane->plane);
 			if (IS_ERR(s))
 				return PTR_ERR(s);
 
-			/* If the plane has been freed in the above loop its
+			/*
+			 * If the plane has been freed in the above loop its
 			 * hardware planes must not be added to the used planes
 			 * bitmask. However, the current state doesn't reflect
 			 * the free state yet, as we've modified the new state
@@ -226,16 +232,16 @@ int rcar_du_atomic_check_planes(struct drm_device *dev,
 				continue;
 			}
 
-			plane_state = to_rcar_plane_state(plane->plane.state);
-			used_planes |= rcar_du_plane_hwmask(plane_state);
+			new_plane_state = to_rcar_plane_state(s);
+			used_planes |= rcar_du_plane_hwmask(new_plane_state);
 
 			dev_dbg(rcdu->dev,
 				"%s: plane (%u,%tu) uses %u hwplanes (index %d)\n",
 				__func__, plane->group->index,
 				plane - plane->group->planes,
-				plane_state->format ?
-				plane_state->format->planes : 0,
-				plane_state->hwindex);
+				new_plane_state->format ?
+				new_plane_state->format->planes : 0,
+				new_plane_state->hwindex);
 		}
 
 		group_free_planes[index] = 0xff & ~used_planes;
@@ -246,40 +252,45 @@ int rcar_du_atomic_check_planes(struct drm_device *dev,
 	}
 
 	/* Reallocate hardware planes for each plane that needs it. */
-	for_each_plane_in_state(state, drm_plane, drm_plane_state, i) {
-		struct rcar_du_plane_state *plane_state;
+	for_each_oldnew_plane_in_state(state, drm_plane, old_drm_plane_state,
+				       new_drm_plane_state, i) {
+		struct rcar_du_plane_state *old_plane_state;
+		struct rcar_du_plane_state *new_plane_state;
 		struct rcar_du_plane *plane;
 		unsigned int crtc_planes;
 		unsigned int free;
 		int idx;
 
 		plane = to_rcar_plane(drm_plane);
-		plane_state = to_rcar_plane_state(drm_plane_state);
+		old_plane_state = to_rcar_plane_state(old_drm_plane_state);
+		new_plane_state = to_rcar_plane_state(new_drm_plane_state);
 
 		dev_dbg(rcdu->dev, "%s: allocating plane (%u,%tu)\n", __func__,
 			plane->group->index, plane - plane->group->planes);
 
-		/* Skip planes that are being disabled or don't need to be
+		/*
+		 * Skip planes that are being disabled or don't need to be
 		 * reallocated.
 		 */
-		if (!plane_state->format ||
-		    !rcar_du_plane_needs_realloc(plane, plane_state))
+		if (!new_plane_state->format ||
+		    !rcar_du_plane_needs_realloc(old_plane_state, new_plane_state))
 			continue;
 
-		/* Try to allocate the plane from the free planes currently
+		/*
+		 * Try to allocate the plane from the free planes currently
 		 * associated with the target CRTC to avoid restarting the CRTC
 		 * group and thus minimize flicker. If it fails fall back to
 		 * allocating from all free planes.
 		 */
-		crtc_planes = to_rcar_crtc(plane_state->state.crtc)->index % 2
+		crtc_planes = to_rcar_crtc(new_plane_state->state.crtc)->index % 2
 			    ? plane->group->dptsr_planes
 			    : ~plane->group->dptsr_planes;
 		free = group_free_planes[plane->group->index];
 
-		idx = rcar_du_plane_hwalloc(plane, plane_state,
+		idx = rcar_du_plane_hwalloc(plane, new_plane_state,
 					    free & crtc_planes);
 		if (idx < 0)
-			idx = rcar_du_plane_hwalloc(plane, plane_state,
+			idx = rcar_du_plane_hwalloc(plane, new_plane_state,
 						    free);
 		if (idx < 0) {
 			dev_dbg(rcdu->dev, "%s: no available hardware plane\n",
@@ -288,12 +299,12 @@ int rcar_du_atomic_check_planes(struct drm_device *dev,
 		}
 
 		dev_dbg(rcdu->dev, "%s: allocated %u hwplanes (index %u)\n",
-			__func__, plane_state->format->planes, idx);
+			__func__, new_plane_state->format->planes, idx);
 
-		plane_state->hwindex = idx;
+		new_plane_state->hwindex = idx;
 
 		group_free_planes[plane->group->index] &=
-			~rcar_du_plane_hwmask(plane_state);
+			~rcar_du_plane_hwmask(new_plane_state);
 
 		dev_dbg(rcdu->dev, "%s: group %u free planes mask 0x%02x\n",
 			__func__, plane->group->index,
@@ -351,14 +362,16 @@ static void rcar_du_plane_setup_scanout(struct rcar_du_group *rgrp,
 		dma[1] = 0;
 	}
 
-	/* Memory pitch (expressed in pixels). Must be doubled for interlaced
+	/*
+	 * Memory pitch (expressed in pixels). Must be doubled for interlaced
 	 * operation with 32bpp formats.
 	 */
 	rcar_du_plane_write(rgrp, index, PnMWR,
 			    (interlaced && state->format->bpp == 32) ?
 			    pitch * 2 : pitch);
 
-	/* The Y position is expressed in raster line units and must be doubled
+	/*
+	 * The Y position is expressed in raster line units and must be doubled
 	 * for 32bpp formats, according to the R8A7790 datasheet. No mention of
 	 * doubling the Y position is found in the R8A7779 datasheet, but the
 	 * rule seems to apply there as well.
@@ -396,7 +409,8 @@ static void rcar_du_plane_setup_mode(struct rcar_du_group *rgrp,
 	u32 colorkey;
 	u32 pnmr;
 
-	/* The PnALPHAR register controls alpha-blending in 16bpp formats
+	/*
+	 * The PnALPHAR register controls alpha-blending in 16bpp formats
 	 * (ARGB1555 and XRGB1555).
 	 *
 	 * For ARGB, set the alpha value to 0, and enable alpha-blending when
@@ -413,7 +427,8 @@ static void rcar_du_plane_setup_mode(struct rcar_du_group *rgrp,
 
 	pnmr = PnMR_BM_MD | state->format->pnmr;
 
-	/* Disable color keying when requested. YUV formats have the
+	/*
+	 * Disable color keying when requested. YUV formats have the
 	 * PnMR_SPIM_TP_OFF bit set in their pnmr field, disabling color keying
 	 * automatically.
 	 */
@@ -457,7 +472,8 @@ static void rcar_du_plane_setup_format_gen2(struct rcar_du_group *rgrp,
 	u32 ddcr2 = PnDDCR2_CODE;
 	u32 ddcr4;
 
-	/* Data format
+	/*
+	 * Data format
 	 *
 	 * The data format is selected by the DDDF field in PnMR and the EDF
 	 * field in DDCR4.
@@ -589,7 +605,8 @@ static void rcar_du_plane_atomic_update(struct drm_plane *plane,
 
 	rcar_du_plane_setup(rplane);
 
-	/* Check whether the source has changed from memory to live source or
+	/*
+	 * Check whether the source has changed from memory to live source or
 	 * from live source to memory. The source has been configured by the
 	 * VSPS bit in the PnDDCR4 register. Although the datasheet states that
 	 * the bit is updated during vertical blanking, it seems that updates
@@ -698,7 +715,6 @@ static const struct drm_plane_funcs rcar_du_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.reset = rcar_du_plane_reset,
-	.set_property = drm_atomic_helper_plane_set_property,
 	.destroy = drm_plane_cleanup,
 	.atomic_duplicate_state = rcar_du_plane_atomic_duplicate_state,
 	.atomic_destroy_state = rcar_du_plane_atomic_destroy_state,
@@ -726,7 +742,8 @@ int rcar_du_planes_init(struct rcar_du_group *rgrp)
 	unsigned int i;
 	int ret;
 
-	 /* Create one primary plane per CRTC in this group and seven overlay
+	 /*
+	  * Create one primary plane per CRTC in this group and seven overlay
 	  * planes.
 	  */
 	rgrp->num_planes = rgrp->num_crtcs + 7;
@@ -743,8 +760,8 @@ int rcar_du_planes_init(struct rcar_du_group *rgrp)
 
 		ret = drm_universal_plane_init(rcdu->ddev, &plane->plane, crtcs,
 					       &rcar_du_plane_funcs, formats,
-					       ARRAY_SIZE(formats), type,
-					       NULL);
+					       ARRAY_SIZE(formats),
+					       NULL, type, NULL);
 		if (ret < 0)
 			return ret;
 
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_plane.h b/drivers/gpu/drm/rcar-du/rcar_du_plane.h
index 8b91dd3a46e4..f62e09f195de 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_plane.h
+++ b/drivers/gpu/drm/rcar-du/rcar_du_plane.h
@@ -20,7 +20,8 @@
 struct rcar_du_format_info;
 struct rcar_du_group;
 
-/* The RCAR DU has 8 hardware planes, shared between primary and overlay planes.
+/*
+ * The RCAR DU has 8 hardware planes, shared between primary and overlay planes.
  * As using overlay planes requires at least one of the CRTCs being enabled, no
  * more than 7 overlay planes can be available. We thus create 1 primary plane
  * per CRTC and 7 overlay planes, for a total of up to 9 KMS planes.
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_vsp.c b/drivers/gpu/drm/rcar-du/rcar_du_vsp.c
index f870445ebc8d..2c96147bc444 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_vsp.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_vsp.c
@@ -19,6 +19,7 @@
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_plane_helper.h>
 
+#include <linux/bitops.h>
 #include <linux/dma-mapping.h>
 #include <linux/of_platform.h>
 #include <linux/scatterlist.h>
@@ -30,11 +31,15 @@
 #include "rcar_du_kms.h"
 #include "rcar_du_vsp.h"
 
-static void rcar_du_vsp_complete(void *private)
+static void rcar_du_vsp_complete(void *private, bool completed)
 {
 	struct rcar_du_crtc *crtc = private;
 
-	rcar_du_crtc_finish_page_flip(crtc);
+	if (crtc->vblank_enable)
+		drm_crtc_handle_vblank(&crtc->crtc);
+
+	if (completed)
+		rcar_du_crtc_finish_page_flip(crtc);
 }
 
 void rcar_du_vsp_enable(struct rcar_du_crtc *crtc)
@@ -73,7 +78,8 @@ void rcar_du_vsp_enable(struct rcar_du_crtc *crtc)
 
 	__rcar_du_plane_setup(crtc->group, &state);
 
-	/* Ensure that the plane source configuration takes effect by requesting
+	/*
+	 * Ensure that the plane source configuration takes effect by requesting
 	 * a restart of the group. See rcar_du_plane_atomic_update() for a more
 	 * detailed explanation.
 	 *
@@ -81,22 +87,22 @@ void rcar_du_vsp_enable(struct rcar_du_crtc *crtc)
 	 */
 	crtc->group->need_restart = true;
 
-	vsp1_du_setup_lif(crtc->vsp->vsp, &cfg);
+	vsp1_du_setup_lif(crtc->vsp->vsp, crtc->vsp_pipe, &cfg);
 }
 
 void rcar_du_vsp_disable(struct rcar_du_crtc *crtc)
 {
-	vsp1_du_setup_lif(crtc->vsp->vsp, NULL);
+	vsp1_du_setup_lif(crtc->vsp->vsp, crtc->vsp_pipe, NULL);
 }
 
 void rcar_du_vsp_atomic_begin(struct rcar_du_crtc *crtc)
 {
-	vsp1_du_atomic_begin(crtc->vsp->vsp);
+	vsp1_du_atomic_begin(crtc->vsp->vsp, crtc->vsp_pipe);
 }
 
 void rcar_du_vsp_atomic_flush(struct rcar_du_crtc *crtc)
 {
-	vsp1_du_atomic_flush(crtc->vsp->vsp);
+	vsp1_du_atomic_flush(crtc->vsp->vsp, crtc->vsp_pipe);
 }
 
 /* Keep the two tables in sync. */
@@ -162,6 +168,7 @@ static void rcar_du_vsp_plane_setup(struct rcar_du_vsp_plane *plane)
 {
 	struct rcar_du_vsp_plane_state *state =
 		to_rcar_vsp_plane_state(plane->plane.state);
+	struct rcar_du_crtc *crtc = to_rcar_crtc(state->state.crtc);
 	struct drm_framebuffer *fb = plane->plane.state->fb;
 	struct vsp1_du_atomic_config cfg = {
 		.pixelformat = 0,
@@ -192,7 +199,8 @@ static void rcar_du_vsp_plane_setup(struct rcar_du_vsp_plane *plane)
 		}
 	}
 
-	vsp1_du_atomic_update(plane->vsp->vsp, plane->index, &cfg);
+	vsp1_du_atomic_update(plane->vsp->vsp, crtc->vsp_pipe,
+			      plane->index, &cfg);
 }
 
 static int rcar_du_vsp_plane_prepare_fb(struct drm_plane *plane,
@@ -288,11 +296,13 @@ static void rcar_du_vsp_plane_atomic_update(struct drm_plane *plane,
 					struct drm_plane_state *old_state)
 {
 	struct rcar_du_vsp_plane *rplane = to_rcar_vsp_plane(plane);
+	struct rcar_du_crtc *crtc = to_rcar_crtc(old_state->crtc);
 
 	if (plane->state->crtc)
 		rcar_du_vsp_plane_setup(rplane);
 	else
-		vsp1_du_atomic_update(rplane->vsp->vsp, rplane->index, NULL);
+		vsp1_du_atomic_update(rplane->vsp->vsp, crtc->vsp_pipe,
+				      rplane->index, NULL);
 }
 
 static const struct drm_plane_helper_funcs rcar_du_vsp_plane_helper_funcs = {
@@ -383,7 +393,6 @@ static const struct drm_plane_funcs rcar_du_vsp_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.reset = rcar_du_vsp_plane_reset,
-	.set_property = drm_atomic_helper_plane_set_property,
 	.destroy = drm_plane_cleanup,
 	.atomic_duplicate_state = rcar_du_vsp_plane_atomic_duplicate_state,
 	.atomic_destroy_state = rcar_du_vsp_plane_atomic_destroy_state,
@@ -391,23 +400,17 @@ static const struct drm_plane_funcs rcar_du_vsp_plane_funcs = {
 	.atomic_get_property = rcar_du_vsp_plane_atomic_get_property,
 };
 
-int rcar_du_vsp_init(struct rcar_du_vsp *vsp)
+int rcar_du_vsp_init(struct rcar_du_vsp *vsp, struct device_node *np,
+		     unsigned int crtcs)
 {
 	struct rcar_du_device *rcdu = vsp->dev;
 	struct platform_device *pdev;
-	struct device_node *np;
+	unsigned int num_crtcs = hweight32(crtcs);
 	unsigned int i;
 	int ret;
 
 	/* Find the VSP device and initialize it. */
-	np = of_parse_phandle(rcdu->dev->of_node, "vsps", vsp->index);
-	if (!np) {
-		dev_err(rcdu->dev, "vsps node not found\n");
-		return -ENXIO;
-	}
-
 	pdev = of_find_device_by_node(np);
-	of_node_put(np);
 	if (!pdev)
 		return -ENXIO;
 
@@ -417,7 +420,8 @@ int rcar_du_vsp_init(struct rcar_du_vsp *vsp)
 	if (ret < 0)
 		return ret;
 
-	 /* The VSP2D (Gen3) has 5 RPFs, but the VSP1D (Gen2) is limited to
+	 /*
+	  * The VSP2D (Gen3) has 5 RPFs, but the VSP1D (Gen2) is limited to
 	  * 4 RPFs.
 	  */
 	vsp->num_planes = rcdu->info->gen >= 3 ? 5 : 4;
@@ -428,19 +432,19 @@ int rcar_du_vsp_init(struct rcar_du_vsp *vsp)
 		return -ENOMEM;
 
 	for (i = 0; i < vsp->num_planes; ++i) {
-		enum drm_plane_type type = i ? DRM_PLANE_TYPE_OVERLAY
-					 : DRM_PLANE_TYPE_PRIMARY;
+		enum drm_plane_type type = i < num_crtcs
+					 ? DRM_PLANE_TYPE_PRIMARY
+					 : DRM_PLANE_TYPE_OVERLAY;
 		struct rcar_du_vsp_plane *plane = &vsp->planes[i];
 
 		plane->vsp = vsp;
 		plane->index = i;
 
-		ret = drm_universal_plane_init(rcdu->ddev, &plane->plane,
-					       1 << vsp->index,
+		ret = drm_universal_plane_init(rcdu->ddev, &plane->plane, crtcs,
 					       &rcar_du_vsp_plane_funcs,
 					       formats_kms,
-					       ARRAY_SIZE(formats_kms), type,
-					       NULL);
+					       ARRAY_SIZE(formats_kms),
+					       NULL, type, NULL);
 		if (ret < 0)
 			return ret;
 
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_vsp.h b/drivers/gpu/drm/rcar-du/rcar_du_vsp.h
index 8861661590ff..f876c512163c 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_vsp.h
+++ b/drivers/gpu/drm/rcar-du/rcar_du_vsp.h
@@ -64,13 +64,19 @@ to_rcar_vsp_plane_state(struct drm_plane_state *state)
 }
 
 #ifdef CONFIG_DRM_RCAR_VSP
-int rcar_du_vsp_init(struct rcar_du_vsp *vsp);
+int rcar_du_vsp_init(struct rcar_du_vsp *vsp, struct device_node *np,
+		     unsigned int crtcs);
 void rcar_du_vsp_enable(struct rcar_du_crtc *crtc);
 void rcar_du_vsp_disable(struct rcar_du_crtc *crtc);
 void rcar_du_vsp_atomic_begin(struct rcar_du_crtc *crtc);
 void rcar_du_vsp_atomic_flush(struct rcar_du_crtc *crtc);
 #else
-static inline int rcar_du_vsp_init(struct rcar_du_vsp *vsp) { return -ENXIO; };
+static inline int rcar_du_vsp_init(struct rcar_du_vsp *vsp,
+				   struct device_node *np,
+				   unsigned int crtcs)
+{
+	return -ENXIO;
+}
 static inline void rcar_du_vsp_enable(struct rcar_du_crtc *crtc) { };
 static inline void rcar_du_vsp_disable(struct rcar_du_crtc *crtc) { };
 static inline void rcar_du_vsp_atomic_begin(struct rcar_du_crtc *crtc) { };
diff --git a/drivers/gpu/drm/rcar-du/rcar_dw_hdmi.c b/drivers/gpu/drm/rcar-du/rcar_dw_hdmi.c
index 7539626b8ebd..dc85b53d58ef 100644
--- a/drivers/gpu/drm/rcar-du/rcar_dw_hdmi.c
+++ b/drivers/gpu/drm/rcar-du/rcar_dw_hdmi.c
@@ -45,7 +45,7 @@ static int rcar_hdmi_phy_configure(struct dw_hdmi *hdmi,
 {
 	const struct rcar_hdmi_phy_params *params = rcar_hdmi_phy_params;
 
-	for (; params && params->mpixelclock != ~0UL; ++params) {
+	for (; params->mpixelclock != ~0UL; ++params) {
 		if (mpixelclock <= params->mpixelclock)
 			break;
 	}
diff --git a/drivers/gpu/drm/rockchip/cdn-dp-core.c b/drivers/gpu/drm/rockchip/cdn-dp-core.c
index 9b0b0588bbed..a57da051f516 100644
--- a/drivers/gpu/drm/rockchip/cdn-dp-core.c
+++ b/drivers/gpu/drm/rockchip/cdn-dp-core.c
@@ -254,7 +254,6 @@ static void cdn_dp_connector_destroy(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs cdn_dp_atomic_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.detect = cdn_dp_connector_detect,
 	.destroy = cdn_dp_connector_destroy,
 	.fill_modes = drm_helper_probe_single_connector_modes,
diff --git a/drivers/gpu/drm/rockchip/dw-mipi-dsi.c b/drivers/gpu/drm/rockchip/dw-mipi-dsi.c
index 21b9737662ae..9a20b9dc27c8 100644
--- a/drivers/gpu/drm/rockchip/dw-mipi-dsi.c
+++ b/drivers/gpu/drm/rockchip/dw-mipi-dsi.c
@@ -1080,7 +1080,6 @@ static void dw_mipi_dsi_drm_connector_destroy(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs dw_mipi_dsi_atomic_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = dw_mipi_dsi_drm_connector_destroy,
 	.reset = drm_atomic_helper_connector_reset,
diff --git a/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c b/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c
index f8208489724e..ccd5d595ada7 100644
--- a/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c
+++ b/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c
@@ -7,10 +7,12 @@
  * (at your option) any later version.
  */
 
+#include <linux/clk.h>
+#include <linux/mfd/syscon.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
-#include <linux/mfd/syscon.h>
 #include <linux/regmap.h>
+
 #include <drm/drm_of.h>
 #include <drm/drmP.h>
 #include <drm/drm_crtc_helper.h>
@@ -20,13 +22,32 @@
 #include "rockchip_drm_drv.h"
 #include "rockchip_drm_vop.h"
 
-#define GRF_SOC_CON6                    0x025c
-#define HDMI_SEL_VOP_LIT                (1 << 4)
+#define RK3288_GRF_SOC_CON6		0x025C
+#define RK3288_HDMI_LCDC_SEL		BIT(4)
+#define RK3399_GRF_SOC_CON20		0x6250
+#define RK3399_HDMI_LCDC_SEL		BIT(6)
+
+#define HIWORD_UPDATE(val, mask)	(val | (mask) << 16)
+
+/**
+ * struct rockchip_hdmi_chip_data - splite the grf setting of kind of chips
+ * @lcdsel_grf_reg: grf register offset of lcdc select
+ * @lcdsel_big: reg value of selecting vop big for HDMI
+ * @lcdsel_lit: reg value of selecting vop little for HDMI
+ */
+struct rockchip_hdmi_chip_data {
+	u32	lcdsel_grf_reg;
+	u32	lcdsel_big;
+	u32	lcdsel_lit;
+};
 
 struct rockchip_hdmi {
 	struct device *dev;
 	struct regmap *regmap;
 	struct drm_encoder encoder;
+	const struct rockchip_hdmi_chip_data *chip_data;
+	struct clk *vpll_clk;
+	struct clk *grf_clk;
 };
 
 #define to_rockchip_hdmi(x)	container_of(x, struct rockchip_hdmi, x)
@@ -143,6 +164,7 @@ static const struct dw_hdmi_phy_config rockchip_phy_config[] = {
 static int rockchip_hdmi_parse_dt(struct rockchip_hdmi *hdmi)
 {
 	struct device_node *np = hdmi->dev->of_node;
+	int ret;
 
 	hdmi->regmap = syscon_regmap_lookup_by_phandle(np, "rockchip,grf");
 	if (IS_ERR(hdmi->regmap)) {
@@ -150,6 +172,32 @@ static int rockchip_hdmi_parse_dt(struct rockchip_hdmi *hdmi)
 		return PTR_ERR(hdmi->regmap);
 	}
 
+	hdmi->vpll_clk = devm_clk_get(hdmi->dev, "vpll");
+	if (PTR_ERR(hdmi->vpll_clk) == -ENOENT) {
+		hdmi->vpll_clk = NULL;
+	} else if (PTR_ERR(hdmi->vpll_clk) == -EPROBE_DEFER) {
+		return -EPROBE_DEFER;
+	} else if (IS_ERR(hdmi->vpll_clk)) {
+		dev_err(hdmi->dev, "failed to get grf clock\n");
+		return PTR_ERR(hdmi->vpll_clk);
+	}
+
+	hdmi->grf_clk = devm_clk_get(hdmi->dev, "grf");
+	if (PTR_ERR(hdmi->grf_clk) == -ENOENT) {
+		hdmi->grf_clk = NULL;
+	} else if (PTR_ERR(hdmi->grf_clk) == -EPROBE_DEFER) {
+		return -EPROBE_DEFER;
+	} else if (IS_ERR(hdmi->grf_clk)) {
+		dev_err(hdmi->dev, "failed to get grf clock\n");
+		return PTR_ERR(hdmi->grf_clk);
+	}
+
+	ret = clk_prepare_enable(hdmi->vpll_clk);
+	if (ret) {
+		dev_err(hdmi->dev, "Failed to enable HDMI vpll: %d\n", ret);
+		return ret;
+	}
+
 	return 0;
 }
 
@@ -192,23 +240,36 @@ static void dw_hdmi_rockchip_encoder_mode_set(struct drm_encoder *encoder,
 					      struct drm_display_mode *mode,
 					      struct drm_display_mode *adj_mode)
 {
+	struct rockchip_hdmi *hdmi = to_rockchip_hdmi(encoder);
+
+	clk_set_rate(hdmi->vpll_clk, adj_mode->clock * 1000);
 }
 
 static void dw_hdmi_rockchip_encoder_enable(struct drm_encoder *encoder)
 {
 	struct rockchip_hdmi *hdmi = to_rockchip_hdmi(encoder);
 	u32 val;
-	int mux;
+	int ret;
 
-	mux = drm_of_encoder_active_endpoint_id(hdmi->dev->of_node, encoder);
-	if (mux)
-		val = HDMI_SEL_VOP_LIT | (HDMI_SEL_VOP_LIT << 16);
+	ret = drm_of_encoder_active_endpoint_id(hdmi->dev->of_node, encoder);
+	if (ret)
+		val = hdmi->chip_data->lcdsel_lit;
 	else
-		val = HDMI_SEL_VOP_LIT << 16;
+		val = hdmi->chip_data->lcdsel_big;
 
-	regmap_write(hdmi->regmap, GRF_SOC_CON6, val);
+	ret = clk_prepare_enable(hdmi->grf_clk);
+	if (ret < 0) {
+		dev_err(hdmi->dev, "failed to enable grfclk %d\n", ret);
+		return;
+	}
+
+	ret = regmap_write(hdmi->regmap, hdmi->chip_data->lcdsel_grf_reg, val);
+	if (ret != 0)
+		dev_err(hdmi->dev, "Could not write to GRF: %d\n", ret);
+
+	clk_disable_unprepare(hdmi->grf_clk);
 	dev_dbg(hdmi->dev, "vop %s output to hdmi\n",
-		(mux) ? "LIT" : "BIG");
+		ret ? "LIT" : "BIG");
 }
 
 static int
@@ -232,16 +293,40 @@ static const struct drm_encoder_helper_funcs dw_hdmi_rockchip_encoder_helper_fun
 	.atomic_check = dw_hdmi_rockchip_encoder_atomic_check,
 };
 
-static const struct dw_hdmi_plat_data rockchip_hdmi_drv_data = {
+static struct rockchip_hdmi_chip_data rk3288_chip_data = {
+	.lcdsel_grf_reg = RK3288_GRF_SOC_CON6,
+	.lcdsel_big = HIWORD_UPDATE(0, RK3288_HDMI_LCDC_SEL),
+	.lcdsel_lit = HIWORD_UPDATE(RK3288_HDMI_LCDC_SEL, RK3288_HDMI_LCDC_SEL),
+};
+
+static const struct dw_hdmi_plat_data rk3288_hdmi_drv_data = {
 	.mode_valid = dw_hdmi_rockchip_mode_valid,
 	.mpll_cfg   = rockchip_mpll_cfg,
 	.cur_ctr    = rockchip_cur_ctr,
 	.phy_config = rockchip_phy_config,
+	.phy_data = &rk3288_chip_data,
+};
+
+static struct rockchip_hdmi_chip_data rk3399_chip_data = {
+	.lcdsel_grf_reg = RK3399_GRF_SOC_CON20,
+	.lcdsel_big = HIWORD_UPDATE(0, RK3399_HDMI_LCDC_SEL),
+	.lcdsel_lit = HIWORD_UPDATE(RK3399_HDMI_LCDC_SEL, RK3399_HDMI_LCDC_SEL),
+};
+
+static const struct dw_hdmi_plat_data rk3399_hdmi_drv_data = {
+	.mode_valid = dw_hdmi_rockchip_mode_valid,
+	.mpll_cfg   = rockchip_mpll_cfg,
+	.cur_ctr    = rockchip_cur_ctr,
+	.phy_config = rockchip_phy_config,
+	.phy_data = &rk3399_chip_data,
 };
 
 static const struct of_device_id dw_hdmi_rockchip_dt_ids[] = {
 	{ .compatible = "rockchip,rk3288-dw-hdmi",
-	  .data = &rockchip_hdmi_drv_data
+	  .data = &rk3288_hdmi_drv_data
+	},
+	{ .compatible = "rockchip,rk3399-dw-hdmi",
+	  .data = &rk3399_hdmi_drv_data
 	},
 	{},
 };
@@ -268,6 +353,7 @@ static int dw_hdmi_rockchip_bind(struct device *dev, struct device *master,
 	match = of_match_node(dw_hdmi_rockchip_dt_ids, pdev->dev.of_node);
 	plat_data = match->data;
 	hdmi->dev = &pdev->dev;
+	hdmi->chip_data = plat_data->phy_data;
 	encoder = &hdmi->encoder;
 
 	encoder->possible_crtcs = drm_of_find_possible_crtcs(drm, dev->of_node);
diff --git a/drivers/gpu/drm/rockchip/inno_hdmi.c b/drivers/gpu/drm/rockchip/inno_hdmi.c
index 7d9b75eb6c44..7a251a54e792 100644
--- a/drivers/gpu/drm/rockchip/inno_hdmi.c
+++ b/drivers/gpu/drm/rockchip/inno_hdmi.c
@@ -294,7 +294,7 @@ static int inno_hdmi_config_video_avi(struct inno_hdmi *hdmi,
 	union hdmi_infoframe frame;
 	int rc;
 
-	rc = drm_hdmi_avi_infoframe_from_display_mode(&frame.avi, mode);
+	rc = drm_hdmi_avi_infoframe_from_display_mode(&frame.avi, mode, false);
 
 	if (hdmi->hdmi_data.enc_out_format == HDMI_COLORSPACE_YUV444)
 		frame.avi.colorspace = HDMI_COLORSPACE_YUV444;
@@ -592,8 +592,7 @@ static void inno_hdmi_connector_destroy(struct drm_connector *connector)
 	drm_connector_cleanup(connector);
 }
 
-static struct drm_connector_funcs inno_hdmi_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
+static const struct drm_connector_funcs inno_hdmi_connector_funcs = {
 	.fill_modes = inno_hdmi_probe_single_connector_modes,
 	.detect = inno_hdmi_connector_detect,
 	.destroy = inno_hdmi_connector_destroy,
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_drv.c b/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
index c16bc0a7115b..ff3d0f5efbb1 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
@@ -161,23 +161,21 @@ static int rockchip_drm_bind(struct device *dev)
 	 */
 	drm_dev->irq_enabled = true;
 
-	/* init kms poll for handling hpd */
-	drm_kms_helper_poll_init(drm_dev);
-
 	ret = rockchip_drm_fbdev_init(drm_dev);
 	if (ret)
-		goto err_kms_helper_poll_fini;
+		goto err_unbind_all;
+
+	/* init kms poll for handling hpd */
+	drm_kms_helper_poll_init(drm_dev);
 
 	ret = drm_dev_register(drm_dev, 0);
 	if (ret)
-		goto err_fbdev_fini;
+		goto err_kms_helper_poll_fini;
 
 	return 0;
-err_fbdev_fini:
-	rockchip_drm_fbdev_fini(drm_dev);
 err_kms_helper_poll_fini:
 	drm_kms_helper_poll_fini(drm_dev);
-	drm_vblank_cleanup(drm_dev);
+	rockchip_drm_fbdev_fini(drm_dev);
 err_unbind_all:
 	component_unbind_all(dev, drm_dev);
 err_mode_config_cleanup:
@@ -200,7 +198,6 @@ static void rockchip_drm_unbind(struct device *dev)
 	drm_kms_helper_poll_fini(drm_dev);
 
 	drm_atomic_helper_shutdown(drm_dev);
-	drm_vblank_cleanup(drm_dev);
 	component_unbind_all(dev, drm_dev);
 	drm_mode_config_cleanup(drm_dev);
 	rockchip_iommu_cleanup(drm_dev);
@@ -235,8 +232,6 @@ static struct drm_driver rockchip_drm_driver = {
 	.gem_vm_ops		= &drm_gem_cma_vm_ops,
 	.gem_free_object_unlocked = rockchip_gem_free_object,
 	.dumb_create		= rockchip_gem_dumb_create,
-	.dumb_map_offset	= rockchip_gem_dumb_map_offset,
-	.dumb_destroy		= drm_gem_dumb_destroy,
 	.prime_handle_to_fd	= drm_gem_prime_handle_to_fd,
 	.prime_fd_to_handle	= drm_gem_prime_fd_to_handle,
 	.gem_prime_import	= drm_gem_prime_import,
@@ -378,8 +373,8 @@ static int rockchip_drm_platform_of_probe(struct device *dev)
 
 		iommu = of_parse_phandle(port->parent, "iommus", 0);
 		if (!iommu || !of_device_is_available(iommu->parent)) {
-			dev_dbg(dev, "no iommu attached for %s, using non-iommu buffers\n",
-				port->parent->full_name);
+			dev_dbg(dev, "no iommu attached for %pOF, using non-iommu buffers\n",
+				port->parent);
 			/*
 			 * if there is a crtc not support iommu, force set all
 			 * crtc use non-iommu buffer.
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_fb.c b/drivers/gpu/drm/rockchip/rockchip_drm_fb.c
index 81f9548672b0..70773041785b 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_fb.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_fb.c
@@ -48,7 +48,7 @@ static void rockchip_drm_fb_destroy(struct drm_framebuffer *fb)
 	int i;
 
 	for (i = 0; i < ROCKCHIP_MAX_FB_BUFFER; i++)
-		drm_gem_object_unreference_unlocked(rockchip_fb->obj[i]);
+		drm_gem_object_put_unlocked(rockchip_fb->obj[i]);
 
 	drm_framebuffer_cleanup(fb);
 	kfree(rockchip_fb);
@@ -144,7 +144,7 @@ rockchip_user_fb_create(struct drm_device *dev, struct drm_file *file_priv,
 			width * drm_format_plane_cpp(mode_cmd->pixel_format, i);
 
 		if (obj->size < min_size) {
-			drm_gem_object_unreference_unlocked(obj);
+			drm_gem_object_put_unlocked(obj);
 			ret = -EINVAL;
 			goto err_gem_object_unreference;
 		}
@@ -161,40 +161,19 @@ rockchip_user_fb_create(struct drm_device *dev, struct drm_file *file_priv,
 
 err_gem_object_unreference:
 	for (i--; i >= 0; i--)
-		drm_gem_object_unreference_unlocked(objs[i]);
+		drm_gem_object_put_unlocked(objs[i]);
 	return ERR_PTR(ret);
 }
 
 static void rockchip_drm_output_poll_changed(struct drm_device *dev)
 {
 	struct rockchip_drm_private *private = dev->dev_private;
-	struct drm_fb_helper *fb_helper = &private->fbdev_helper;
 
-	if (fb_helper)
-		drm_fb_helper_hotplug_event(fb_helper);
-}
-
-static void
-rockchip_atomic_commit_tail(struct drm_atomic_state *state)
-{
-	struct drm_device *dev = state->dev;
-
-	drm_atomic_helper_commit_modeset_disables(dev, state);
-
-	drm_atomic_helper_commit_modeset_enables(dev, state);
-
-	drm_atomic_helper_commit_planes(dev, state,
-					DRM_PLANE_COMMIT_ACTIVE_ONLY);
-
-	drm_atomic_helper_commit_hw_done(state);
-
-	drm_atomic_helper_wait_for_vblanks(dev, state);
-
-	drm_atomic_helper_cleanup_planes(dev, state);
+	drm_fb_helper_hotplug_event(&private->fbdev_helper);
 }
 
 static const struct drm_mode_config_helper_funcs rockchip_mode_config_helpers = {
-	.atomic_commit_tail = rockchip_atomic_commit_tail,
+	.atomic_commit_tail = drm_atomic_helper_commit_tail_rpm,
 };
 
 static const struct drm_mode_config_funcs rockchip_drm_mode_config_funcs = {
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_fbdev.c b/drivers/gpu/drm/rockchip/rockchip_drm_fbdev.c
index ce946b9c57a9..724579ebf947 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_fbdev.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_fbdev.c
@@ -173,7 +173,7 @@ void rockchip_drm_fbdev_fini(struct drm_device *dev)
 	drm_fb_helper_unregister_fbi(helper);
 
 	if (helper->fb)
-		drm_framebuffer_unreference(helper->fb);
+		drm_framebuffer_put(helper->fb);
 
 	drm_fb_helper_fini(helper);
 }
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c
index b74ac717e56a..1869c8bb76c8 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c
@@ -383,7 +383,7 @@ rockchip_gem_create_with_handle(struct drm_file *file_priv,
 		goto err_handle_create;
 
 	/* drop reference from allocate - handle holds it now. */
-	drm_gem_object_unreference_unlocked(obj);
+	drm_gem_object_put_unlocked(obj);
 
 	return rk_obj;
 
@@ -393,32 +393,6 @@ err_handle_create:
 	return ERR_PTR(ret);
 }
 
-int rockchip_gem_dumb_map_offset(struct drm_file *file_priv,
-				 struct drm_device *dev, uint32_t handle,
-				 uint64_t *offset)
-{
-	struct drm_gem_object *obj;
-	int ret;
-
-	obj = drm_gem_object_lookup(file_priv, handle);
-	if (!obj) {
-		DRM_ERROR("failed to lookup gem object.\n");
-		return -EINVAL;
-	}
-
-	ret = drm_gem_create_mmap_offset(obj);
-	if (ret)
-		goto out;
-
-	*offset = drm_vma_node_offset_addr(&obj->vma_node);
-	DRM_DEBUG_KMS("offset = 0x%llx\n", *offset);
-
-out:
-	drm_gem_object_unreference_unlocked(obj);
-
-	return 0;
-}
-
 /*
  * rockchip_gem_dumb_create - (struct drm_driver)->dumb_create callback
  * function
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_gem.h b/drivers/gpu/drm/rockchip/rockchip_drm_gem.h
index 3f6ea4d18a5c..f237375582fb 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_gem.h
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_gem.h
@@ -57,7 +57,4 @@ void rockchip_gem_free_object(struct drm_gem_object *obj);
 int rockchip_gem_dumb_create(struct drm_file *file_priv,
 			     struct drm_device *dev,
 			     struct drm_mode_create_dumb *args);
-int rockchip_gem_dumb_map_offset(struct drm_file *file_priv,
-				 struct drm_device *dev, uint32_t handle,
-				 uint64_t *offset);
 #endif /* _ROCKCHIP_DRM_GEM_H */
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
index 2900f1410d95..bf9ed0e63973 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
@@ -42,33 +42,20 @@
 #include "rockchip_drm_psr.h"
 #include "rockchip_drm_vop.h"
 
-#define __REG_SET_RELAXED(x, off, mask, shift, v, write_mask) \
-		vop_mask_write(x, off, mask, shift, v, write_mask, true)
-
-#define __REG_SET_NORMAL(x, off, mask, shift, v, write_mask) \
-		vop_mask_write(x, off, mask, shift, v, write_mask, false)
-
-#define REG_SET(x, base, reg, v, mode) \
-		__REG_SET_##mode(x, base + reg.offset, \
-				 reg.mask, reg.shift, v, reg.write_mask)
-#define REG_SET_MASK(x, base, reg, mask, v, mode) \
-		__REG_SET_##mode(x, base + reg.offset, \
-				 mask, reg.shift, v, reg.write_mask)
-
 #define VOP_WIN_SET(x, win, name, v) \
-		REG_SET(x, win->base, win->phy->name, v, RELAXED)
+		vop_reg_set(vop, &win->phy->name, win->base, ~0, v, #name)
 #define VOP_SCL_SET(x, win, name, v) \
-		REG_SET(x, win->base, win->phy->scl->name, v, RELAXED)
+		vop_reg_set(vop, &win->phy->scl->name, win->base, ~0, v, #name)
 #define VOP_SCL_SET_EXT(x, win, name, v) \
-		REG_SET(x, win->base, win->phy->scl->ext->name, v, RELAXED)
-#define VOP_CTRL_SET(x, name, v) \
-		REG_SET(x, 0, (x)->data->ctrl->name, v, NORMAL)
+		vop_reg_set(vop, &win->phy->scl->ext->name, \
+			    win->base, ~0, v, #name)
 
-#define VOP_INTR_GET(vop, name) \
-		vop_read_reg(vop, 0, &vop->data->ctrl->name)
+#define VOP_INTR_SET_MASK(vop, name, mask, v) \
+		vop_reg_set(vop, &vop->data->intr->name, 0, mask, v, #name)
+
+#define VOP_REG_SET(vop, group, name, v) \
+		    vop_reg_set(vop, &vop->data->group->name, 0, ~0, v, #name)
 
-#define VOP_INTR_SET(vop, name, mask, v) \
-		REG_SET_MASK(vop, 0, vop->data->intr->name, mask, v, NORMAL)
 #define VOP_INTR_SET_TYPE(vop, name, type, v) \
 	do { \
 		int i, reg = 0, mask = 0; \
@@ -78,13 +65,13 @@
 				mask |= 1 << i; \
 			} \
 		} \
-		VOP_INTR_SET(vop, name, mask, reg); \
+		VOP_INTR_SET_MASK(vop, name, mask, reg); \
 	} while (0)
 #define VOP_INTR_GET_TYPE(vop, name, type) \
 		vop_get_intr_type(vop, &vop->data->intr->name, type)
 
 #define VOP_WIN_GET(x, win, name) \
-		vop_read_reg(x, win->base, &win->phy->name)
+		vop_read_reg(x, win->offset, win->phy->name)
 
 #define VOP_WIN_GET_YRGBADDR(vop, win) \
 		vop_readl(vop, win->base + win->phy->yrgb_mst.offset)
@@ -166,14 +153,22 @@ static inline uint32_t vop_read_reg(struct vop *vop, uint32_t base,
 	return (vop_readl(vop, base + reg->offset) >> reg->shift) & reg->mask;
 }
 
-static inline void vop_mask_write(struct vop *vop, uint32_t offset,
-				  uint32_t mask, uint32_t shift, uint32_t v,
-				  bool write_mask, bool relaxed)
+static void vop_reg_set(struct vop *vop, const struct vop_reg *reg,
+			uint32_t _offset, uint32_t _mask, uint32_t v,
+			const char *reg_name)
 {
-	if (!mask)
+	int offset, mask, shift;
+
+	if (!reg || !reg->mask) {
+		dev_dbg(vop->dev, "Warning: not support %s\n", reg_name);
 		return;
+	}
+
+	offset = reg->offset + _offset;
+	mask = reg->mask & _mask;
+	shift = reg->shift;
 
-	if (write_mask) {
+	if (reg->write_mask) {
 		v = ((v << shift) & 0xffff) | (mask << (shift + 16));
 	} else {
 		uint32_t cached_val = vop->regsbak[offset >> 2];
@@ -182,7 +177,7 @@ static inline void vop_mask_write(struct vop *vop, uint32_t offset,
 		vop->regsbak[offset >> 2] = v;
 	}
 
-	if (relaxed)
+	if (reg->relaxed)
 		writel_relaxed(v, vop->regs + offset);
 	else
 		writel(v, vop->regs + offset);
@@ -204,7 +199,7 @@ static inline uint32_t vop_get_intr_type(struct vop *vop,
 
 static inline void vop_cfg_done(struct vop *vop)
 {
-	VOP_CTRL_SET(vop, cfg_done, 1);
+	VOP_REG_SET(vop, common, cfg_done, 1);
 }
 
 static bool has_rb_swapped(uint32_t format)
@@ -556,7 +551,7 @@ static int vop_enable(struct drm_crtc *crtc)
 
 	spin_lock(&vop->reg_lock);
 
-	VOP_CTRL_SET(vop, standby, 0);
+	VOP_REG_SET(vop, common, standby, 1);
 
 	spin_unlock(&vop->reg_lock);
 
@@ -577,7 +572,8 @@ err_put_pm_runtime:
 	return ret;
 }
 
-static void vop_crtc_disable(struct drm_crtc *crtc)
+static void vop_crtc_atomic_disable(struct drm_crtc *crtc,
+				    struct drm_crtc_state *old_state)
 {
 	struct vop *vop = to_vop(crtc);
 
@@ -599,7 +595,7 @@ static void vop_crtc_disable(struct drm_crtc *crtc)
 
 	spin_lock(&vop->reg_lock);
 
-	VOP_CTRL_SET(vop, standby, 1);
+	VOP_REG_SET(vop, common, standby, 1);
 
 	spin_unlock(&vop->reg_lock);
 
@@ -870,7 +866,8 @@ static bool vop_crtc_mode_fixup(struct drm_crtc *crtc,
 	return true;
 }
 
-static void vop_crtc_enable(struct drm_crtc *crtc)
+static void vop_crtc_atomic_enable(struct drm_crtc *crtc,
+				   struct drm_crtc_state *old_state)
 {
 	struct vop *vop = to_vop(crtc);
 	const struct vop_data *vop_data = vop->data;
@@ -897,70 +894,34 @@ static void vop_crtc_enable(struct drm_crtc *crtc)
 		return;
 	}
 
-	/*
-	 * If dclk rate is zero, mean that scanout is stop,
-	 * we don't need wait any more.
-	 */
-	if (clk_get_rate(vop->dclk)) {
-		/*
-		 * Rk3288 vop timing register is immediately, when configure
-		 * display timing on display time, may cause tearing.
-		 *
-		 * Vop standby will take effect at end of current frame,
-		 * if dsp hold valid irq happen, it means standby complete.
-		 *
-		 * mode set:
-		 *    standby and wait complete --> |----
-		 *                                  | display time
-		 *                                  |----
-		 *                                  |---> dsp hold irq
-		 *     configure display timing --> |
-		 *         standby exit             |
-		 *                                  | new frame start.
-		 */
-
-		reinit_completion(&vop->dsp_hold_completion);
-		vop_dsp_hold_valid_irq_enable(vop);
-
-		spin_lock(&vop->reg_lock);
-
-		VOP_CTRL_SET(vop, standby, 1);
-
-		spin_unlock(&vop->reg_lock);
-
-		wait_for_completion(&vop->dsp_hold_completion);
-
-		vop_dsp_hold_valid_irq_disable(vop);
-	}
-
 	pin_pol = BIT(DCLK_INVERT);
 	pin_pol |= (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC) ?
 		   BIT(HSYNC_POSITIVE) : 0;
 	pin_pol |= (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC) ?
 		   BIT(VSYNC_POSITIVE) : 0;
-	VOP_CTRL_SET(vop, pin_pol, pin_pol);
+	VOP_REG_SET(vop, output, pin_pol, pin_pol);
 
 	switch (s->output_type) {
 	case DRM_MODE_CONNECTOR_LVDS:
-		VOP_CTRL_SET(vop, rgb_en, 1);
-		VOP_CTRL_SET(vop, rgb_pin_pol, pin_pol);
+		VOP_REG_SET(vop, output, rgb_en, 1);
+		VOP_REG_SET(vop, output, rgb_pin_pol, pin_pol);
 		break;
 	case DRM_MODE_CONNECTOR_eDP:
-		VOP_CTRL_SET(vop, edp_pin_pol, pin_pol);
-		VOP_CTRL_SET(vop, edp_en, 1);
+		VOP_REG_SET(vop, output, edp_pin_pol, pin_pol);
+		VOP_REG_SET(vop, output, edp_en, 1);
 		break;
 	case DRM_MODE_CONNECTOR_HDMIA:
-		VOP_CTRL_SET(vop, hdmi_pin_pol, pin_pol);
-		VOP_CTRL_SET(vop, hdmi_en, 1);
+		VOP_REG_SET(vop, output, hdmi_pin_pol, pin_pol);
+		VOP_REG_SET(vop, output, hdmi_en, 1);
 		break;
 	case DRM_MODE_CONNECTOR_DSI:
-		VOP_CTRL_SET(vop, mipi_pin_pol, pin_pol);
-		VOP_CTRL_SET(vop, mipi_en, 1);
+		VOP_REG_SET(vop, output, mipi_pin_pol, pin_pol);
+		VOP_REG_SET(vop, output, mipi_en, 1);
 		break;
 	case DRM_MODE_CONNECTOR_DisplayPort:
 		pin_pol &= ~BIT(DCLK_INVERT);
-		VOP_CTRL_SET(vop, dp_pin_pol, pin_pol);
-		VOP_CTRL_SET(vop, dp_en, 1);
+		VOP_REG_SET(vop, output, dp_pin_pol, pin_pol);
+		VOP_REG_SET(vop, output, dp_en, 1);
 		break;
 	default:
 		DRM_DEV_ERROR(vop->dev, "unsupported connector_type [%d]\n",
@@ -973,25 +934,25 @@ static void vop_crtc_enable(struct drm_crtc *crtc)
 	if (s->output_mode == ROCKCHIP_OUT_MODE_AAAA &&
 	    !(vop_data->feature & VOP_FEATURE_OUTPUT_RGB10))
 		s->output_mode = ROCKCHIP_OUT_MODE_P888;
-	VOP_CTRL_SET(vop, out_mode, s->output_mode);
+	VOP_REG_SET(vop, common, out_mode, s->output_mode);
 
-	VOP_CTRL_SET(vop, htotal_pw, (htotal << 16) | hsync_len);
+	VOP_REG_SET(vop, modeset, htotal_pw, (htotal << 16) | hsync_len);
 	val = hact_st << 16;
 	val |= hact_end;
-	VOP_CTRL_SET(vop, hact_st_end, val);
-	VOP_CTRL_SET(vop, hpost_st_end, val);
+	VOP_REG_SET(vop, modeset, hact_st_end, val);
+	VOP_REG_SET(vop, modeset, hpost_st_end, val);
 
-	VOP_CTRL_SET(vop, vtotal_pw, (vtotal << 16) | vsync_len);
+	VOP_REG_SET(vop, modeset, vtotal_pw, (vtotal << 16) | vsync_len);
 	val = vact_st << 16;
 	val |= vact_end;
-	VOP_CTRL_SET(vop, vact_st_end, val);
-	VOP_CTRL_SET(vop, vpost_st_end, val);
+	VOP_REG_SET(vop, modeset, vact_st_end, val);
+	VOP_REG_SET(vop, modeset, vpost_st_end, val);
 
-	VOP_CTRL_SET(vop, line_flag_num[0], vact_end);
+	VOP_REG_SET(vop, intr, line_flag_num[0], vact_end);
 
 	clk_set_rate(vop->dclk, adjusted_mode->clock * 1000);
 
-	VOP_CTRL_SET(vop, standby, 0);
+	VOP_REG_SET(vop, common, standby, 0);
 
 	rockchip_drm_psr_activate(&vop->crtc);
 }
@@ -1026,7 +987,7 @@ static void vop_crtc_atomic_flush(struct drm_crtc *crtc,
 				  struct drm_crtc_state *old_crtc_state)
 {
 	struct drm_atomic_state *old_state = old_crtc_state->state;
-	struct drm_plane_state *old_plane_state;
+	struct drm_plane_state *old_plane_state, *new_plane_state;
 	struct vop *vop = to_vop(crtc);
 	struct drm_plane *plane;
 	int i;
@@ -1057,14 +1018,15 @@ static void vop_crtc_atomic_flush(struct drm_crtc *crtc,
 	}
 	spin_unlock_irq(&crtc->dev->event_lock);
 
-	for_each_plane_in_state(old_state, plane, old_plane_state, i) {
+	for_each_oldnew_plane_in_state(old_state, plane, old_plane_state,
+				       new_plane_state, i) {
 		if (!old_plane_state->fb)
 			continue;
 
-		if (old_plane_state->fb == plane->state->fb)
+		if (old_plane_state->fb == new_plane_state->fb)
 			continue;
 
-		drm_framebuffer_reference(old_plane_state->fb);
+		drm_framebuffer_get(old_plane_state->fb);
 		drm_flip_work_queue(&vop->fb_unref_work, old_plane_state->fb);
 		set_bit(VOP_PENDING_FB_UNREF, &vop->pending);
 		WARN_ON(drm_crtc_vblank_get(crtc) != 0);
@@ -1078,11 +1040,11 @@ static void vop_crtc_atomic_begin(struct drm_crtc *crtc,
 }
 
 static const struct drm_crtc_helper_funcs vop_crtc_helper_funcs = {
-	.enable = vop_crtc_enable,
-	.disable = vop_crtc_disable,
 	.mode_fixup = vop_crtc_mode_fixup,
 	.atomic_flush = vop_crtc_atomic_flush,
 	.atomic_begin = vop_crtc_atomic_begin,
+	.atomic_enable = vop_crtc_atomic_enable,
+	.atomic_disable = vop_crtc_atomic_disable,
 };
 
 static void vop_crtc_destroy(struct drm_crtc *crtc)
@@ -1188,7 +1150,7 @@ static void vop_fb_unref_worker(struct drm_flip_work *work, void *val)
 	struct drm_framebuffer *fb = val;
 
 	drm_crtc_vblank_put(&vop->crtc);
-	drm_framebuffer_unreference(fb);
+	drm_framebuffer_put(fb);
 }
 
 static void vop_handle_vblank(struct vop *vop)
@@ -1289,7 +1251,7 @@ static int vop_create_crtc(struct vop *vop)
 					       0, &vop_plane_funcs,
 					       win_data->phy->data_formats,
 					       win_data->phy->nformats,
-					       win_data->type, NULL);
+					       NULL, win_data->type, NULL);
 		if (ret) {
 			DRM_DEV_ERROR(vop->dev, "failed to init plane %d\n",
 				      ret);
@@ -1328,7 +1290,7 @@ static int vop_create_crtc(struct vop *vop)
 					       &vop_plane_funcs,
 					       win_data->phy->data_formats,
 					       win_data->phy->nformats,
-					       win_data->type, NULL);
+					       NULL, win_data->type, NULL);
 		if (ret) {
 			DRM_DEV_ERROR(vop->dev, "failed to init overlay %d\n",
 				      ret);
@@ -1339,8 +1301,8 @@ static int vop_create_crtc(struct vop *vop)
 
 	port = of_get_child_by_name(dev->of_node, "port");
 	if (!port) {
-		DRM_DEV_ERROR(vop->dev, "no port node found in %s\n",
-			      dev->of_node->full_name);
+		DRM_DEV_ERROR(vop->dev, "no port node found in %pOF\n",
+			      dev->of_node);
 		ret = -ENOENT;
 		goto err_cleanup_crtc;
 	}
@@ -1394,7 +1356,6 @@ static void vop_destroy_crtc(struct vop *vop)
 static int vop_initial(struct vop *vop)
 {
 	const struct vop_data *vop_data = vop->data;
-	const struct vop_reg_data *init_table = vop_data->init_table;
 	struct reset_control *ahb_rst;
 	int i, ret;
 
@@ -1454,13 +1415,16 @@ static int vop_initial(struct vop *vop)
 
 	memcpy(vop->regsbak, vop->regs, vop->len);
 
-	for (i = 0; i < vop_data->table_size; i++)
-		vop_writel(vop, init_table[i].offset, init_table[i].value);
+	VOP_REG_SET(vop, misc, global_regdone_en, 1);
+	VOP_REG_SET(vop, common, dsp_blank, 0);
 
 	for (i = 0; i < vop_data->win_size; i++) {
 		const struct vop_win_data *win = &vop_data->win[i];
+		int channel = i * 2 + 1;
 
+		VOP_WIN_SET(vop, win, channel, (channel + 1) << 4 | channel);
 		VOP_WIN_SET(vop, win, enable, 0);
+		VOP_WIN_SET(vop, win, gate, 1);
 	}
 
 	vop_cfg_done(vop);
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.h b/drivers/gpu/drm/rockchip/rockchip_drm_vop.h
index 27eefbfcf3d0..56bbd2e2a8ef 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.h
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.h
@@ -15,6 +15,14 @@
 #ifndef _ROCKCHIP_DRM_VOP_H
 #define _ROCKCHIP_DRM_VOP_H
 
+/*
+ * major: IP major version, used for IP structure
+ * minor: big feature change under same structure
+ */
+#define VOP_VERSION(major, minor)	((major) << 8 | (minor))
+#define VOP_MAJOR(version)		((version) >> 8)
+#define VOP_MINOR(version)		((version) & 0xff)
+
 enum vop_data_format {
 	VOP_FMT_ARGB8888 = 0,
 	VOP_FMT_RGB888,
@@ -24,53 +32,58 @@ enum vop_data_format {
 	VOP_FMT_YUV444SP,
 };
 
-struct vop_reg_data {
-	uint32_t offset;
-	uint32_t value;
-};
-
 struct vop_reg {
-	uint32_t offset;
-	uint32_t shift;
 	uint32_t mask;
+	uint16_t offset;
+	uint8_t shift;
 	bool write_mask;
+	bool relaxed;
 };
 
-struct vop_ctrl {
-	struct vop_reg standby;
-	struct vop_reg data_blank;
-	struct vop_reg gate_en;
-	struct vop_reg mmu_en;
-	struct vop_reg rgb_en;
-	struct vop_reg edp_en;
-	struct vop_reg hdmi_en;
-	struct vop_reg mipi_en;
-	struct vop_reg dp_en;
-	struct vop_reg out_mode;
-	struct vop_reg dither_down;
-	struct vop_reg dither_up;
-	struct vop_reg pin_pol;
-	struct vop_reg rgb_pin_pol;
-	struct vop_reg hdmi_pin_pol;
-	struct vop_reg edp_pin_pol;
-	struct vop_reg mipi_pin_pol;
-	struct vop_reg dp_pin_pol;
-
+struct vop_modeset {
 	struct vop_reg htotal_pw;
 	struct vop_reg hact_st_end;
+	struct vop_reg hpost_st_end;
 	struct vop_reg vtotal_pw;
 	struct vop_reg vact_st_end;
-	struct vop_reg hpost_st_end;
 	struct vop_reg vpost_st_end;
+};
 
-	struct vop_reg line_flag_num[2];
+struct vop_output {
+	struct vop_reg pin_pol;
+	struct vop_reg dp_pin_pol;
+	struct vop_reg edp_pin_pol;
+	struct vop_reg hdmi_pin_pol;
+	struct vop_reg mipi_pin_pol;
+	struct vop_reg rgb_pin_pol;
+	struct vop_reg dp_en;
+	struct vop_reg edp_en;
+	struct vop_reg hdmi_en;
+	struct vop_reg mipi_en;
+	struct vop_reg rgb_en;
+};
 
+struct vop_common {
 	struct vop_reg cfg_done;
+	struct vop_reg dsp_blank;
+	struct vop_reg data_blank;
+	struct vop_reg dither_down;
+	struct vop_reg dither_up;
+	struct vop_reg gate_en;
+	struct vop_reg mmu_en;
+	struct vop_reg out_mode;
+	struct vop_reg standby;
+};
+
+struct vop_misc {
+	struct vop_reg global_regdone_en;
 };
 
 struct vop_intr {
 	const int *intrs;
 	uint32_t nintrs;
+
+	struct vop_reg line_flag_num[2];
 	struct vop_reg enable;
 	struct vop_reg clear;
 	struct vop_reg status;
@@ -115,6 +128,7 @@ struct vop_win_phy {
 	uint32_t nformats;
 
 	struct vop_reg enable;
+	struct vop_reg gate;
 	struct vop_reg format;
 	struct vop_reg rb_swap;
 	struct vop_reg act_info;
@@ -127,6 +141,7 @@ struct vop_win_phy {
 
 	struct vop_reg dst_alpha_ctl;
 	struct vop_reg src_alpha_ctl;
+	struct vop_reg channel;
 };
 
 struct vop_win_data {
@@ -136,10 +151,12 @@ struct vop_win_data {
 };
 
 struct vop_data {
-	const struct vop_reg_data *init_table;
-	unsigned int table_size;
-	const struct vop_ctrl *ctrl;
+	uint32_t version;
 	const struct vop_intr *intr;
+	const struct vop_common *common;
+	const struct vop_misc *misc;
+	const struct vop_modeset *modeset;
+	const struct vop_output *output;
 	const struct vop_win_data *win;
 	unsigned int win_size;
 
diff --git a/drivers/gpu/drm/rockchip/rockchip_vop_reg.c b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
index bafd698a28b1..94de7b9f6fde 100644
--- a/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
+++ b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
@@ -20,17 +20,23 @@
 #include "rockchip_drm_vop.h"
 #include "rockchip_vop_reg.h"
 
-#define VOP_REG(off, _mask, s) \
-		{.offset = off, \
+#define _VOP_REG(off, _mask, _shift, _write_mask, _relaxed) \
+		{ \
+		 .offset = off, \
 		 .mask = _mask, \
-		 .shift = s, \
-		 .write_mask = false,}
+		 .shift = _shift, \
+		 .write_mask = _write_mask, \
+		 .relaxed = _relaxed, \
+		}
 
-#define VOP_REG_MASK(off, _mask, s) \
-		{.offset = off, \
-		 .mask = _mask, \
-		 .shift = s, \
-		 .write_mask = true,}
+#define VOP_REG(off, _mask, _shift) \
+		_VOP_REG(off, _mask, _shift, false, true)
+
+#define VOP_REG_SYNC(off, _mask, _shift) \
+		_VOP_REG(off, _mask, _shift, false, false)
+
+#define VOP_REG_MASK_SYNC(off, _mask, _shift) \
+		_VOP_REG(off, _mask, _shift, true, false)
 
 static const uint32_t formats_win_full[] = {
 	DRM_FORMAT_XRGB8888,
@@ -110,32 +116,35 @@ static const int rk3036_vop_intrs[] = {
 static const struct vop_intr rk3036_intr = {
 	.intrs = rk3036_vop_intrs,
 	.nintrs = ARRAY_SIZE(rk3036_vop_intrs),
-	.status = VOP_REG(RK3036_INT_STATUS, 0xf, 0),
-	.enable = VOP_REG(RK3036_INT_STATUS, 0xf, 4),
-	.clear = VOP_REG(RK3036_INT_STATUS, 0xf, 8),
+	.line_flag_num[0] = VOP_REG(RK3036_INT_STATUS, 0xfff, 12),
+	.status = VOP_REG_SYNC(RK3036_INT_STATUS, 0xf, 0),
+	.enable = VOP_REG_SYNC(RK3036_INT_STATUS, 0xf, 4),
+	.clear = VOP_REG_SYNC(RK3036_INT_STATUS, 0xf, 8),
 };
 
-static const struct vop_ctrl rk3036_ctrl_data = {
-	.standby = VOP_REG(RK3036_SYS_CTRL, 0x1, 30),
-	.out_mode = VOP_REG(RK3036_DSP_CTRL0, 0xf, 0),
-	.pin_pol = VOP_REG(RK3036_DSP_CTRL0, 0xf, 4),
+static const struct vop_modeset rk3036_modeset = {
 	.htotal_pw = VOP_REG(RK3036_DSP_HTOTAL_HS_END, 0x1fff1fff, 0),
 	.hact_st_end = VOP_REG(RK3036_DSP_HACT_ST_END, 0x1fff1fff, 0),
 	.vtotal_pw = VOP_REG(RK3036_DSP_VTOTAL_VS_END, 0x1fff1fff, 0),
 	.vact_st_end = VOP_REG(RK3036_DSP_VACT_ST_END, 0x1fff1fff, 0),
-	.line_flag_num[0] = VOP_REG(RK3036_INT_STATUS, 0xfff, 12),
-	.cfg_done = VOP_REG(RK3036_REG_CFG_DONE, 0x1, 0),
 };
 
-static const struct vop_reg_data rk3036_vop_init_reg_table[] = {
-	{RK3036_DSP_CTRL1, 0x00000000},
+static const struct vop_output rk3036_output = {
+	.pin_pol = VOP_REG(RK3036_DSP_CTRL0, 0xf, 4),
+};
+
+static const struct vop_common rk3036_common = {
+	.standby = VOP_REG_SYNC(RK3036_SYS_CTRL, 0x1, 30),
+	.out_mode = VOP_REG(RK3036_DSP_CTRL0, 0xf, 0),
+	.dsp_blank = VOP_REG(RK3036_DSP_CTRL1, 0x1, 24),
+	.cfg_done = VOP_REG_SYNC(RK3036_REG_CFG_DONE, 0x1, 0),
 };
 
 static const struct vop_data rk3036_vop = {
-	.init_table = rk3036_vop_init_reg_table,
-	.table_size = ARRAY_SIZE(rk3036_vop_init_reg_table),
-	.ctrl = &rk3036_ctrl_data,
 	.intr = &rk3036_intr,
+	.common = &rk3036_common,
+	.modeset = &rk3036_modeset,
+	.output = &rk3036_output,
 	.win = rk3036_vop_win_data,
 	.win_size = ARRAY_SIZE(rk3036_vop_win_data),
 };
@@ -188,12 +197,14 @@ static const struct vop_win_phy rk3288_win01_data = {
 	.uv_vir = VOP_REG(RK3288_WIN0_VIR, 0x3fff, 16),
 	.src_alpha_ctl = VOP_REG(RK3288_WIN0_SRC_ALPHA_CTRL, 0xff, 0),
 	.dst_alpha_ctl = VOP_REG(RK3288_WIN0_DST_ALPHA_CTRL, 0xff, 0),
+	.channel = VOP_REG(RK3288_WIN0_CTRL2, 0xff, 0),
 };
 
 static const struct vop_win_phy rk3288_win23_data = {
 	.data_formats = formats_win_lite,
 	.nformats = ARRAY_SIZE(formats_win_lite),
-	.enable = VOP_REG(RK3288_WIN2_CTRL0, 0x1, 0),
+	.enable = VOP_REG(RK3288_WIN2_CTRL0, 0x1, 4),
+	.gate = VOP_REG(RK3288_WIN2_CTRL0, 0x1, 0),
 	.format = VOP_REG(RK3288_WIN2_CTRL0, 0x7, 1),
 	.rb_swap = VOP_REG(RK3288_WIN2_CTRL0, 0x1, 12),
 	.dsp_info = VOP_REG(RK3288_WIN2_DSP_INFO0, 0x0fff0fff, 0),
@@ -204,40 +215,33 @@ static const struct vop_win_phy rk3288_win23_data = {
 	.dst_alpha_ctl = VOP_REG(RK3288_WIN2_DST_ALPHA_CTRL, 0xff, 0),
 };
 
-static const struct vop_ctrl rk3288_ctrl_data = {
-	.standby = VOP_REG(RK3288_SYS_CTRL, 0x1, 22),
-	.gate_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 23),
-	.mmu_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 20),
-	.rgb_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 12),
-	.hdmi_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 13),
-	.edp_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 14),
-	.mipi_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 15),
-	.dither_down = VOP_REG(RK3288_DSP_CTRL1, 0xf, 1),
-	.dither_up = VOP_REG(RK3288_DSP_CTRL1, 0x1, 6),
-	.data_blank = VOP_REG(RK3288_DSP_CTRL0, 0x1, 19),
-	.out_mode = VOP_REG(RK3288_DSP_CTRL0, 0xf, 0),
-	.pin_pol = VOP_REG(RK3288_DSP_CTRL0, 0xf, 4),
+static const struct vop_modeset rk3288_modeset = {
 	.htotal_pw = VOP_REG(RK3288_DSP_HTOTAL_HS_END, 0x1fff1fff, 0),
 	.hact_st_end = VOP_REG(RK3288_DSP_HACT_ST_END, 0x1fff1fff, 0),
 	.vtotal_pw = VOP_REG(RK3288_DSP_VTOTAL_VS_END, 0x1fff1fff, 0),
 	.vact_st_end = VOP_REG(RK3288_DSP_VACT_ST_END, 0x1fff1fff, 0),
 	.hpost_st_end = VOP_REG(RK3288_POST_DSP_HACT_INFO, 0x1fff1fff, 0),
 	.vpost_st_end = VOP_REG(RK3288_POST_DSP_VACT_INFO, 0x1fff1fff, 0),
-	.line_flag_num[0] = VOP_REG(RK3288_INTR_CTRL0, 0x1fff, 12),
-	.cfg_done = VOP_REG(RK3288_REG_CFG_DONE, 0x1, 0),
 };
 
-static const struct vop_reg_data rk3288_init_reg_table[] = {
-	{RK3288_SYS_CTRL, 0x00c00000},
-	{RK3288_DSP_CTRL0, 0x00000000},
-	{RK3288_WIN0_CTRL0, 0x00000080},
-	{RK3288_WIN1_CTRL0, 0x00000080},
-	/* TODO: Win2/3 support multiple area function, but we haven't found
-	 * a suitable way to use it yet, so let's just use them as other windows
-	 * with only area 0 enabled.
-	 */
-	{RK3288_WIN2_CTRL0, 0x00000010},
-	{RK3288_WIN3_CTRL0, 0x00000010},
+static const struct vop_output rk3288_output = {
+	.pin_pol = VOP_REG(RK3288_DSP_CTRL0, 0xf, 4),
+	.rgb_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 12),
+	.hdmi_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 13),
+	.edp_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 14),
+	.mipi_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 15),
+};
+
+static const struct vop_common rk3288_common = {
+	.standby = VOP_REG_SYNC(RK3288_SYS_CTRL, 0x1, 22),
+	.gate_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 23),
+	.mmu_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 20),
+	.dither_down = VOP_REG(RK3288_DSP_CTRL1, 0xf, 1),
+	.dither_up = VOP_REG(RK3288_DSP_CTRL1, 0x1, 6),
+	.data_blank = VOP_REG(RK3288_DSP_CTRL0, 0x1, 19),
+	.dsp_blank = VOP_REG(RK3288_DSP_CTRL0, 0x3, 18),
+	.out_mode = VOP_REG(RK3288_DSP_CTRL0, 0xf, 0),
+	.cfg_done = VOP_REG_SYNC(RK3288_REG_CFG_DONE, 0x1, 0),
 };
 
 /*
@@ -267,50 +271,24 @@ static const int rk3288_vop_intrs[] = {
 static const struct vop_intr rk3288_vop_intr = {
 	.intrs = rk3288_vop_intrs,
 	.nintrs = ARRAY_SIZE(rk3288_vop_intrs),
+	.line_flag_num[0] = VOP_REG(RK3288_INTR_CTRL0, 0x1fff, 12),
 	.status = VOP_REG(RK3288_INTR_CTRL0, 0xf, 0),
 	.enable = VOP_REG(RK3288_INTR_CTRL0, 0xf, 4),
 	.clear = VOP_REG(RK3288_INTR_CTRL0, 0xf, 8),
 };
 
 static const struct vop_data rk3288_vop = {
-	.init_table = rk3288_init_reg_table,
-	.table_size = ARRAY_SIZE(rk3288_init_reg_table),
+	.version = VOP_VERSION(3, 1),
 	.feature = VOP_FEATURE_OUTPUT_RGB10,
 	.intr = &rk3288_vop_intr,
-	.ctrl = &rk3288_ctrl_data,
+	.common = &rk3288_common,
+	.modeset = &rk3288_modeset,
+	.output = &rk3288_output,
 	.win = rk3288_vop_win_data,
 	.win_size = ARRAY_SIZE(rk3288_vop_win_data),
 };
 
-static const struct vop_ctrl rk3399_ctrl_data = {
-	.standby = VOP_REG(RK3399_SYS_CTRL, 0x1, 22),
-	.gate_en = VOP_REG(RK3399_SYS_CTRL, 0x1, 23),
-	.dp_en = VOP_REG(RK3399_SYS_CTRL, 0x1, 11),
-	.rgb_en = VOP_REG(RK3399_SYS_CTRL, 0x1, 12),
-	.hdmi_en = VOP_REG(RK3399_SYS_CTRL, 0x1, 13),
-	.edp_en = VOP_REG(RK3399_SYS_CTRL, 0x1, 14),
-	.mipi_en = VOP_REG(RK3399_SYS_CTRL, 0x1, 15),
-	.dither_down = VOP_REG(RK3399_DSP_CTRL1, 0xf, 1),
-	.dither_up = VOP_REG(RK3399_DSP_CTRL1, 0x1, 6),
-	.data_blank = VOP_REG(RK3399_DSP_CTRL0, 0x1, 19),
-	.out_mode = VOP_REG(RK3399_DSP_CTRL0, 0xf, 0),
-	.rgb_pin_pol = VOP_REG(RK3399_DSP_CTRL1, 0xf, 16),
-	.dp_pin_pol = VOP_REG(RK3399_DSP_CTRL1, 0xf, 16),
-	.hdmi_pin_pol = VOP_REG(RK3399_DSP_CTRL1, 0xf, 20),
-	.edp_pin_pol = VOP_REG(RK3399_DSP_CTRL1, 0xf, 24),
-	.mipi_pin_pol = VOP_REG(RK3399_DSP_CTRL1, 0xf, 28),
-	.htotal_pw = VOP_REG(RK3399_DSP_HTOTAL_HS_END, 0x1fff1fff, 0),
-	.hact_st_end = VOP_REG(RK3399_DSP_HACT_ST_END, 0x1fff1fff, 0),
-	.vtotal_pw = VOP_REG(RK3399_DSP_VTOTAL_VS_END, 0x1fff1fff, 0),
-	.vact_st_end = VOP_REG(RK3399_DSP_VACT_ST_END, 0x1fff1fff, 0),
-	.hpost_st_end = VOP_REG(RK3399_POST_DSP_HACT_INFO, 0x1fff1fff, 0),
-	.vpost_st_end = VOP_REG(RK3399_POST_DSP_VACT_INFO, 0x1fff1fff, 0),
-	.line_flag_num[0] = VOP_REG(RK3399_LINE_FLAG, 0xffff, 0),
-	.line_flag_num[1] = VOP_REG(RK3399_LINE_FLAG, 0xffff, 16),
-	.cfg_done = VOP_REG_MASK(RK3399_REG_CFG_DONE, 0x1, 0),
-};
-
-static const int rk3399_vop_intrs[] = {
+static const int rk3368_vop_intrs[] = {
 	FS_INTR,
 	0, 0,
 	LINE_FLAG_INTR,
@@ -320,69 +298,232 @@ static const int rk3399_vop_intrs[] = {
 	DSP_HOLD_VALID_INTR,
 };
 
-static const struct vop_intr rk3399_vop_intr = {
-	.intrs = rk3399_vop_intrs,
-	.nintrs = ARRAY_SIZE(rk3399_vop_intrs),
-	.status = VOP_REG_MASK(RK3399_INTR_STATUS0, 0xffff, 0),
-	.enable = VOP_REG_MASK(RK3399_INTR_EN0, 0xffff, 0),
-	.clear = VOP_REG_MASK(RK3399_INTR_CLEAR0, 0xffff, 0),
+static const struct vop_intr rk3368_vop_intr = {
+	.intrs = rk3368_vop_intrs,
+	.nintrs = ARRAY_SIZE(rk3368_vop_intrs),
+	.line_flag_num[0] = VOP_REG(RK3368_LINE_FLAG, 0xffff, 0),
+	.line_flag_num[1] = VOP_REG(RK3368_LINE_FLAG, 0xffff, 16),
+	.status = VOP_REG_MASK_SYNC(RK3368_INTR_STATUS, 0x3fff, 0),
+	.enable = VOP_REG_MASK_SYNC(RK3368_INTR_EN, 0x3fff, 0),
+	.clear = VOP_REG_MASK_SYNC(RK3368_INTR_CLEAR, 0x3fff, 0),
 };
 
-static const struct vop_reg_data rk3399_init_reg_table[] = {
-	{RK3399_SYS_CTRL, 0x2000f800},
-	{RK3399_DSP_CTRL0, 0x00000000},
-	{RK3399_WIN0_CTRL0, 0x00000080},
-	{RK3399_WIN1_CTRL0, 0x00000080},
-	/* TODO: Win2/3 support multiple area function, but we haven't found
-	 * a suitable way to use it yet, so let's just use them as other windows
-	 * with only area 0 enabled.
-	 */
-	{RK3399_WIN2_CTRL0, 0x00000010},
-	{RK3399_WIN3_CTRL0, 0x00000010},
+static const struct vop_win_phy rk3368_win23_data = {
+	.data_formats = formats_win_lite,
+	.nformats = ARRAY_SIZE(formats_win_lite),
+	.gate = VOP_REG(RK3368_WIN2_CTRL0, 0x1, 0),
+	.enable = VOP_REG(RK3368_WIN2_CTRL0, 0x1, 4),
+	.format = VOP_REG(RK3368_WIN2_CTRL0, 0x3, 5),
+	.rb_swap = VOP_REG(RK3368_WIN2_CTRL0, 0x1, 20),
+	.dsp_info = VOP_REG(RK3368_WIN2_DSP_INFO0, 0x0fff0fff, 0),
+	.dsp_st = VOP_REG(RK3368_WIN2_DSP_ST0, 0x1fff1fff, 0),
+	.yrgb_mst = VOP_REG(RK3368_WIN2_MST0, 0xffffffff, 0),
+	.yrgb_vir = VOP_REG(RK3368_WIN2_VIR0_1, 0x1fff, 0),
+	.src_alpha_ctl = VOP_REG(RK3368_WIN2_SRC_ALPHA_CTRL, 0xff, 0),
+	.dst_alpha_ctl = VOP_REG(RK3368_WIN2_DST_ALPHA_CTRL, 0xff, 0),
+};
+
+static const struct vop_win_data rk3368_vop_win_data[] = {
+	{ .base = 0x00, .phy = &rk3288_win01_data,
+	  .type = DRM_PLANE_TYPE_PRIMARY },
+	{ .base = 0x40, .phy = &rk3288_win01_data,
+	  .type = DRM_PLANE_TYPE_OVERLAY },
+	{ .base = 0x00, .phy = &rk3368_win23_data,
+	  .type = DRM_PLANE_TYPE_OVERLAY },
+	{ .base = 0x50, .phy = &rk3368_win23_data,
+	  .type = DRM_PLANE_TYPE_CURSOR },
+};
+
+static const struct vop_output rk3368_output = {
+	.rgb_pin_pol = VOP_REG(RK3368_DSP_CTRL1, 0xf, 16),
+	.hdmi_pin_pol = VOP_REG(RK3368_DSP_CTRL1, 0xf, 20),
+	.edp_pin_pol = VOP_REG(RK3368_DSP_CTRL1, 0xf, 24),
+	.mipi_pin_pol = VOP_REG(RK3368_DSP_CTRL1, 0xf, 28),
+	.rgb_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 12),
+	.hdmi_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 13),
+	.edp_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 14),
+	.mipi_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 15),
+};
+
+static const struct vop_misc rk3368_misc = {
+	.global_regdone_en = VOP_REG(RK3368_SYS_CTRL, 0x1, 11),
+};
+
+static const struct vop_data rk3368_vop = {
+	.version = VOP_VERSION(3, 2),
+	.intr = &rk3368_vop_intr,
+	.common = &rk3288_common,
+	.modeset = &rk3288_modeset,
+	.output = &rk3368_output,
+	.misc = &rk3368_misc,
+	.win = rk3368_vop_win_data,
+	.win_size = ARRAY_SIZE(rk3368_vop_win_data),
+};
+
+static const struct vop_intr rk3366_vop_intr = {
+	.intrs = rk3368_vop_intrs,
+	.nintrs = ARRAY_SIZE(rk3368_vop_intrs),
+	.line_flag_num[0] = VOP_REG(RK3366_LINE_FLAG, 0xffff, 0),
+	.line_flag_num[1] = VOP_REG(RK3366_LINE_FLAG, 0xffff, 16),
+	.status = VOP_REG_MASK_SYNC(RK3366_INTR_STATUS0, 0xffff, 0),
+	.enable = VOP_REG_MASK_SYNC(RK3366_INTR_EN0, 0xffff, 0),
+	.clear = VOP_REG_MASK_SYNC(RK3366_INTR_CLEAR0, 0xffff, 0),
+};
+
+static const struct vop_data rk3366_vop = {
+	.version = VOP_VERSION(3, 4),
+	.intr = &rk3366_vop_intr,
+	.common = &rk3288_common,
+	.modeset = &rk3288_modeset,
+	.output = &rk3368_output,
+	.misc = &rk3368_misc,
+	.win = rk3368_vop_win_data,
+	.win_size = ARRAY_SIZE(rk3368_vop_win_data),
+};
+
+static const struct vop_output rk3399_output = {
+	.dp_pin_pol = VOP_REG(RK3399_DSP_CTRL1, 0xf, 16),
+	.rgb_pin_pol = VOP_REG(RK3368_DSP_CTRL1, 0xf, 16),
+	.hdmi_pin_pol = VOP_REG(RK3368_DSP_CTRL1, 0xf, 20),
+	.edp_pin_pol = VOP_REG(RK3368_DSP_CTRL1, 0xf, 24),
+	.mipi_pin_pol = VOP_REG(RK3368_DSP_CTRL1, 0xf, 28),
+	.dp_en = VOP_REG(RK3399_SYS_CTRL, 0x1, 11),
+	.rgb_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 12),
+	.hdmi_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 13),
+	.edp_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 14),
+	.mipi_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 15),
 };
 
 static const struct vop_data rk3399_vop_big = {
-	.init_table = rk3399_init_reg_table,
-	.table_size = ARRAY_SIZE(rk3399_init_reg_table),
+	.version = VOP_VERSION(3, 5),
 	.feature = VOP_FEATURE_OUTPUT_RGB10,
-	.intr = &rk3399_vop_intr,
-	.ctrl = &rk3399_ctrl_data,
-	/*
-	 * rk3399 vop big windows register layout is same as rk3288.
-	 */
-	.win = rk3288_vop_win_data,
-	.win_size = ARRAY_SIZE(rk3288_vop_win_data),
+	.intr = &rk3366_vop_intr,
+	.common = &rk3288_common,
+	.modeset = &rk3288_modeset,
+	.output = &rk3399_output,
+	.misc = &rk3368_misc,
+	.win = rk3368_vop_win_data,
+	.win_size = ARRAY_SIZE(rk3368_vop_win_data),
 };
 
 static const struct vop_win_data rk3399_vop_lit_win_data[] = {
 	{ .base = 0x00, .phy = &rk3288_win01_data,
 	  .type = DRM_PLANE_TYPE_PRIMARY },
-	{ .base = 0x00, .phy = &rk3288_win23_data,
+	{ .base = 0x00, .phy = &rk3368_win23_data,
 	  .type = DRM_PLANE_TYPE_CURSOR},
 };
 
 static const struct vop_data rk3399_vop_lit = {
-	.init_table = rk3399_init_reg_table,
-	.table_size = ARRAY_SIZE(rk3399_init_reg_table),
-	.intr = &rk3399_vop_intr,
-	.ctrl = &rk3399_ctrl_data,
-	/*
-	 * rk3399 vop lit windows register layout is same as rk3288,
-	 * but cut off the win1 and win3 windows.
-	 */
+	.version = VOP_VERSION(3, 6),
+	.intr = &rk3366_vop_intr,
+	.common = &rk3288_common,
+	.modeset = &rk3288_modeset,
+	.output = &rk3399_output,
+	.misc = &rk3368_misc,
 	.win = rk3399_vop_lit_win_data,
 	.win_size = ARRAY_SIZE(rk3399_vop_lit_win_data),
 };
 
+static const struct vop_win_data rk3228_vop_win_data[] = {
+	{ .base = 0x00, .phy = &rk3288_win01_data,
+	  .type = DRM_PLANE_TYPE_PRIMARY },
+	{ .base = 0x40, .phy = &rk3288_win01_data,
+	  .type = DRM_PLANE_TYPE_CURSOR },
+};
+
+static const struct vop_data rk3228_vop = {
+	.version = VOP_VERSION(3, 7),
+	.feature = VOP_FEATURE_OUTPUT_RGB10,
+	.intr = &rk3366_vop_intr,
+	.common = &rk3288_common,
+	.modeset = &rk3288_modeset,
+	.output = &rk3399_output,
+	.misc = &rk3368_misc,
+	.win = rk3228_vop_win_data,
+	.win_size = ARRAY_SIZE(rk3228_vop_win_data),
+};
+
+static const struct vop_modeset rk3328_modeset = {
+	.htotal_pw = VOP_REG(RK3328_DSP_HTOTAL_HS_END, 0x1fff1fff, 0),
+	.hact_st_end = VOP_REG(RK3328_DSP_HACT_ST_END, 0x1fff1fff, 0),
+	.vtotal_pw = VOP_REG(RK3328_DSP_VTOTAL_VS_END, 0x1fff1fff, 0),
+	.vact_st_end = VOP_REG(RK3328_DSP_VACT_ST_END, 0x1fff1fff, 0),
+	.hpost_st_end = VOP_REG(RK3328_POST_DSP_HACT_INFO, 0x1fff1fff, 0),
+	.vpost_st_end = VOP_REG(RK3328_POST_DSP_VACT_INFO, 0x1fff1fff, 0),
+};
+
+static const struct vop_output rk3328_output = {
+	.rgb_en = VOP_REG(RK3328_SYS_CTRL, 0x1, 12),
+	.hdmi_en = VOP_REG(RK3328_SYS_CTRL, 0x1, 13),
+	.edp_en = VOP_REG(RK3328_SYS_CTRL, 0x1, 14),
+	.mipi_en = VOP_REG(RK3328_SYS_CTRL, 0x1, 15),
+	.rgb_pin_pol = VOP_REG(RK3328_DSP_CTRL1, 0xf, 16),
+	.hdmi_pin_pol = VOP_REG(RK3328_DSP_CTRL1, 0xf, 20),
+	.edp_pin_pol = VOP_REG(RK3328_DSP_CTRL1, 0xf, 24),
+	.mipi_pin_pol = VOP_REG(RK3328_DSP_CTRL1, 0xf, 28),
+};
+
+static const struct vop_misc rk3328_misc = {
+	.global_regdone_en = VOP_REG(RK3328_SYS_CTRL, 0x1, 11),
+};
+
+static const struct vop_common rk3328_common = {
+	.standby = VOP_REG_SYNC(RK3328_SYS_CTRL, 0x1, 22),
+	.dither_down = VOP_REG(RK3328_DSP_CTRL1, 0xf, 1),
+	.dither_up = VOP_REG(RK3328_DSP_CTRL1, 0x1, 6),
+	.dsp_blank = VOP_REG(RK3328_DSP_CTRL0, 0x3, 18),
+	.out_mode = VOP_REG(RK3328_DSP_CTRL0, 0xf, 0),
+	.cfg_done = VOP_REG_SYNC(RK3328_REG_CFG_DONE, 0x1, 0),
+};
+
+static const struct vop_intr rk3328_vop_intr = {
+	.intrs = rk3368_vop_intrs,
+	.nintrs = ARRAY_SIZE(rk3368_vop_intrs),
+	.line_flag_num[0] = VOP_REG(RK3328_LINE_FLAG, 0xffff, 0),
+	.line_flag_num[1] = VOP_REG(RK3328_LINE_FLAG, 0xffff, 16),
+	.status = VOP_REG_MASK_SYNC(RK3328_INTR_STATUS0, 0xffff, 0),
+	.enable = VOP_REG_MASK_SYNC(RK3328_INTR_EN0, 0xffff, 0),
+	.clear = VOP_REG_MASK_SYNC(RK3328_INTR_CLEAR0, 0xffff, 0),
+};
+
+static const struct vop_win_data rk3328_vop_win_data[] = {
+	{ .base = 0xd0, .phy = &rk3288_win01_data,
+	  .type = DRM_PLANE_TYPE_PRIMARY },
+	{ .base = 0x1d0, .phy = &rk3288_win01_data,
+	  .type = DRM_PLANE_TYPE_OVERLAY },
+	{ .base = 0x2d0, .phy = &rk3288_win01_data,
+	  .type = DRM_PLANE_TYPE_CURSOR },
+};
+
+static const struct vop_data rk3328_vop = {
+	.version = VOP_VERSION(3, 8),
+	.feature = VOP_FEATURE_OUTPUT_RGB10,
+	.intr = &rk3328_vop_intr,
+	.common = &rk3328_common,
+	.modeset = &rk3328_modeset,
+	.output = &rk3328_output,
+	.misc = &rk3328_misc,
+	.win = rk3328_vop_win_data,
+	.win_size = ARRAY_SIZE(rk3328_vop_win_data),
+};
+
 static const struct of_device_id vop_driver_dt_match[] = {
 	{ .compatible = "rockchip,rk3036-vop",
 	  .data = &rk3036_vop },
 	{ .compatible = "rockchip,rk3288-vop",
 	  .data = &rk3288_vop },
+	{ .compatible = "rockchip,rk3368-vop",
+	  .data = &rk3368_vop },
+	{ .compatible = "rockchip,rk3366-vop",
+	  .data = &rk3366_vop },
 	{ .compatible = "rockchip,rk3399-vop-big",
 	  .data = &rk3399_vop_big },
 	{ .compatible = "rockchip,rk3399-vop-lit",
 	  .data = &rk3399_vop_lit },
+	{ .compatible = "rockchip,rk3228-vop",
+	  .data = &rk3228_vop },
+	{ .compatible = "rockchip,rk3328-vop",
+	  .data = &rk3328_vop },
 	{},
 };
 MODULE_DEVICE_TABLE(of, vop_driver_dt_match);
diff --git a/drivers/gpu/drm/rockchip/rockchip_vop_reg.h b/drivers/gpu/drm/rockchip/rockchip_vop_reg.h
index cd197260ece5..4a4799ff65de 100644
--- a/drivers/gpu/drm/rockchip/rockchip_vop_reg.h
+++ b/drivers/gpu/drm/rockchip/rockchip_vop_reg.h
@@ -41,6 +41,7 @@
 #define RK3288_WIN0_SRC_ALPHA_CTRL		0x0060
 #define RK3288_WIN0_DST_ALPHA_CTRL		0x0064
 #define RK3288_WIN0_FADING_CTRL			0x0068
+#define RK3288_WIN0_CTRL2			0x006c
 
 /* win1 register */
 #define RK3288_WIN1_CTRL0			0x0070
@@ -122,6 +123,717 @@
 #define RK3288_DSP_VACT_ST_END_F1		0x019c
 /* register definition end */
 
+/* rk3368 register definition */
+#define RK3368_REG_CFG_DONE			0x0000
+#define RK3368_VERSION_INFO			0x0004
+#define RK3368_SYS_CTRL				0x0008
+#define RK3368_SYS_CTRL1			0x000c
+#define RK3368_DSP_CTRL0			0x0010
+#define RK3368_DSP_CTRL1			0x0014
+#define RK3368_DSP_BG				0x0018
+#define RK3368_MCU_CTRL				0x001c
+#define RK3368_LINE_FLAG			0x0020
+#define RK3368_INTR_EN				0x0024
+#define RK3368_INTR_CLEAR			0x0028
+#define RK3368_INTR_STATUS			0x002c
+#define RK3368_WIN0_CTRL0			0x0030
+#define RK3368_WIN0_CTRL1			0x0034
+#define RK3368_WIN0_COLOR_KEY			0x0038
+#define RK3368_WIN0_VIR				0x003c
+#define RK3368_WIN0_YRGB_MST			0x0040
+#define RK3368_WIN0_CBR_MST			0x0044
+#define RK3368_WIN0_ACT_INFO			0x0048
+#define RK3368_WIN0_DSP_INFO			0x004c
+#define RK3368_WIN0_DSP_ST			0x0050
+#define RK3368_WIN0_SCL_FACTOR_YRGB		0x0054
+#define RK3368_WIN0_SCL_FACTOR_CBR		0x0058
+#define RK3368_WIN0_SCL_OFFSET			0x005c
+#define RK3368_WIN0_SRC_ALPHA_CTRL		0x0060
+#define RK3368_WIN0_DST_ALPHA_CTRL		0x0064
+#define RK3368_WIN0_FADING_CTRL			0x0068
+#define RK3368_WIN0_CTRL2			0x006c
+#define RK3368_WIN1_CTRL0			0x0070
+#define RK3368_WIN1_CTRL1			0x0074
+#define RK3368_WIN1_COLOR_KEY			0x0078
+#define RK3368_WIN1_VIR				0x007c
+#define RK3368_WIN1_YRGB_MST			0x0080
+#define RK3368_WIN1_CBR_MST			0x0084
+#define RK3368_WIN1_ACT_INFO			0x0088
+#define RK3368_WIN1_DSP_INFO			0x008c
+#define RK3368_WIN1_DSP_ST			0x0090
+#define RK3368_WIN1_SCL_FACTOR_YRGB		0x0094
+#define RK3368_WIN1_SCL_FACTOR_CBR		0x0098
+#define RK3368_WIN1_SCL_OFFSET			0x009c
+#define RK3368_WIN1_SRC_ALPHA_CTRL		0x00a0
+#define RK3368_WIN1_DST_ALPHA_CTRL		0x00a4
+#define RK3368_WIN1_FADING_CTRL			0x00a8
+#define RK3368_WIN1_CTRL2			0x00ac
+#define RK3368_WIN2_CTRL0			0x00b0
+#define RK3368_WIN2_CTRL1			0x00b4
+#define RK3368_WIN2_VIR0_1			0x00b8
+#define RK3368_WIN2_VIR2_3			0x00bc
+#define RK3368_WIN2_MST0			0x00c0
+#define RK3368_WIN2_DSP_INFO0			0x00c4
+#define RK3368_WIN2_DSP_ST0			0x00c8
+#define RK3368_WIN2_COLOR_KEY			0x00cc
+#define RK3368_WIN2_MST1			0x00d0
+#define RK3368_WIN2_DSP_INFO1			0x00d4
+#define RK3368_WIN2_DSP_ST1			0x00d8
+#define RK3368_WIN2_SRC_ALPHA_CTRL		0x00dc
+#define RK3368_WIN2_MST2			0x00e0
+#define RK3368_WIN2_DSP_INFO2			0x00e4
+#define RK3368_WIN2_DSP_ST2			0x00e8
+#define RK3368_WIN2_DST_ALPHA_CTRL		0x00ec
+#define RK3368_WIN2_MST3			0x00f0
+#define RK3368_WIN2_DSP_INFO3			0x00f4
+#define RK3368_WIN2_DSP_ST3			0x00f8
+#define RK3368_WIN2_FADING_CTRL			0x00fc
+#define RK3368_WIN3_CTRL0			0x0100
+#define RK3368_WIN3_CTRL1			0x0104
+#define RK3368_WIN3_VIR0_1			0x0108
+#define RK3368_WIN3_VIR2_3			0x010c
+#define RK3368_WIN3_MST0			0x0110
+#define RK3368_WIN3_DSP_INFO0			0x0114
+#define RK3368_WIN3_DSP_ST0			0x0118
+#define RK3368_WIN3_COLOR_KEY			0x011c
+#define RK3368_WIN3_MST1			0x0120
+#define RK3368_WIN3_DSP_INFO1			0x0124
+#define RK3368_WIN3_DSP_ST1			0x0128
+#define RK3368_WIN3_SRC_ALPHA_CTRL		0x012c
+#define RK3368_WIN3_MST2			0x0130
+#define RK3368_WIN3_DSP_INFO2			0x0134
+#define RK3368_WIN3_DSP_ST2			0x0138
+#define RK3368_WIN3_DST_ALPHA_CTRL		0x013c
+#define RK3368_WIN3_MST3			0x0140
+#define RK3368_WIN3_DSP_INFO3			0x0144
+#define RK3368_WIN3_DSP_ST3			0x0148
+#define RK3368_WIN3_FADING_CTRL			0x014c
+#define RK3368_HWC_CTRL0			0x0150
+#define RK3368_HWC_CTRL1			0x0154
+#define RK3368_HWC_MST				0x0158
+#define RK3368_HWC_DSP_ST			0x015c
+#define RK3368_HWC_SRC_ALPHA_CTRL		0x0160
+#define RK3368_HWC_DST_ALPHA_CTRL		0x0164
+#define RK3368_HWC_FADING_CTRL			0x0168
+#define RK3368_HWC_RESERVED1			0x016c
+#define RK3368_POST_DSP_HACT_INFO		0x0170
+#define RK3368_POST_DSP_VACT_INFO		0x0174
+#define RK3368_POST_SCL_FACTOR_YRGB		0x0178
+#define RK3368_POST_RESERVED			0x017c
+#define RK3368_POST_SCL_CTRL			0x0180
+#define RK3368_POST_DSP_VACT_INFO_F1		0x0184
+#define RK3368_DSP_HTOTAL_HS_END		0x0188
+#define RK3368_DSP_HACT_ST_END			0x018c
+#define RK3368_DSP_VTOTAL_VS_END		0x0190
+#define RK3368_DSP_VACT_ST_END			0x0194
+#define RK3368_DSP_VS_ST_END_F1			0x0198
+#define RK3368_DSP_VACT_ST_END_F1		0x019c
+#define RK3368_PWM_CTRL				0x01a0
+#define RK3368_PWM_PERIOD_HPR			0x01a4
+#define RK3368_PWM_DUTY_LPR			0x01a8
+#define RK3368_PWM_CNT				0x01ac
+#define RK3368_BCSH_COLOR_BAR			0x01b0
+#define RK3368_BCSH_BCS				0x01b4
+#define RK3368_BCSH_H				0x01b8
+#define RK3368_BCSH_CTRL			0x01bc
+#define RK3368_CABC_CTRL0			0x01c0
+#define RK3368_CABC_CTRL1			0x01c4
+#define RK3368_CABC_CTRL2			0x01c8
+#define RK3368_CABC_CTRL3			0x01cc
+#define RK3368_CABC_GAUSS_LINE0_0		0x01d0
+#define RK3368_CABC_GAUSS_LINE0_1		0x01d4
+#define RK3368_CABC_GAUSS_LINE1_0		0x01d8
+#define RK3368_CABC_GAUSS_LINE1_1		0x01dc
+#define RK3368_CABC_GAUSS_LINE2_0		0x01e0
+#define RK3368_CABC_GAUSS_LINE2_1		0x01e4
+#define RK3368_FRC_LOWER01_0			0x01e8
+#define RK3368_FRC_LOWER01_1			0x01ec
+#define RK3368_FRC_LOWER10_0			0x01f0
+#define RK3368_FRC_LOWER10_1			0x01f4
+#define RK3368_FRC_LOWER11_0			0x01f8
+#define RK3368_FRC_LOWER11_1			0x01fc
+#define RK3368_IFBDC_CTRL			0x0200
+#define RK3368_IFBDC_TILES_NUM			0x0204
+#define RK3368_IFBDC_FRAME_RST_CYCLE		0x0208
+#define RK3368_IFBDC_BASE_ADDR			0x020c
+#define RK3368_IFBDC_MB_SIZE			0x0210
+#define RK3368_IFBDC_CMP_INDEX_INIT		0x0214
+#define RK3368_IFBDC_VIR			0x0220
+#define RK3368_IFBDC_DEBUG0			0x0230
+#define RK3368_IFBDC_DEBUG1			0x0234
+#define RK3368_LATENCY_CTRL0			0x0250
+#define RK3368_RD_MAX_LATENCY_NUM0		0x0254
+#define RK3368_RD_LATENCY_THR_NUM0		0x0258
+#define RK3368_RD_LATENCY_SAMP_NUM0		0x025c
+#define RK3368_WIN0_DSP_BG			0x0260
+#define RK3368_WIN1_DSP_BG			0x0264
+#define RK3368_WIN2_DSP_BG			0x0268
+#define RK3368_WIN3_DSP_BG			0x026c
+#define RK3368_SCAN_LINE_NUM			0x0270
+#define RK3368_CABC_DEBUG0			0x0274
+#define RK3368_CABC_DEBUG1			0x0278
+#define RK3368_CABC_DEBUG2			0x027c
+#define RK3368_DBG_REG_000			0x0280
+#define RK3368_DBG_REG_001			0x0284
+#define RK3368_DBG_REG_002			0x0288
+#define RK3368_DBG_REG_003			0x028c
+#define RK3368_DBG_REG_004			0x0290
+#define RK3368_DBG_REG_005			0x0294
+#define RK3368_DBG_REG_006			0x0298
+#define RK3368_DBG_REG_007			0x029c
+#define RK3368_DBG_REG_008			0x02a0
+#define RK3368_DBG_REG_016			0x02c0
+#define RK3368_DBG_REG_017			0x02c4
+#define RK3368_DBG_REG_018			0x02c8
+#define RK3368_DBG_REG_019			0x02cc
+#define RK3368_DBG_REG_020			0x02d0
+#define RK3368_DBG_REG_021			0x02d4
+#define RK3368_DBG_REG_022			0x02d8
+#define RK3368_DBG_REG_023			0x02dc
+#define RK3368_DBG_REG_028			0x02f0
+#define RK3368_MMU_DTE_ADDR			0x0300
+#define RK3368_MMU_STATUS			0x0304
+#define RK3368_MMU_COMMAND			0x0308
+#define RK3368_MMU_PAGE_FAULT_ADDR		0x030c
+#define RK3368_MMU_ZAP_ONE_LINE			0x0310
+#define RK3368_MMU_INT_RAWSTAT			0x0314
+#define RK3368_MMU_INT_CLEAR			0x0318
+#define RK3368_MMU_INT_MASK			0x031c
+#define RK3368_MMU_INT_STATUS			0x0320
+#define RK3368_MMU_AUTO_GATING			0x0324
+#define RK3368_WIN2_LUT_ADDR			0x0400
+#define RK3368_WIN3_LUT_ADDR			0x0800
+#define RK3368_HWC_LUT_ADDR			0x0c00
+#define RK3368_GAMMA_LUT_ADDR			0x1000
+#define RK3368_CABC_GAMMA_LUT_ADDR		0x1800
+#define RK3368_MCU_BYPASS_WPORT			0x2200
+#define RK3368_MCU_BYPASS_RPORT			0x2300
+/* rk3368 register definition end */
+
+#define RK3366_REG_CFG_DONE			0x0000
+#define RK3366_VERSION_INFO			0x0004
+#define RK3366_SYS_CTRL				0x0008
+#define RK3366_SYS_CTRL1			0x000c
+#define RK3366_DSP_CTRL0			0x0010
+#define RK3366_DSP_CTRL1			0x0014
+#define RK3366_DSP_BG				0x0018
+#define RK3366_MCU_CTRL				0x001c
+#define RK3366_WB_CTRL0				0x0020
+#define RK3366_WB_CTRL1				0x0024
+#define RK3366_WB_YRGB_MST			0x0028
+#define RK3366_WB_CBR_MST			0x002c
+#define RK3366_WIN0_CTRL0			0x0030
+#define RK3366_WIN0_CTRL1			0x0034
+#define RK3366_WIN0_COLOR_KEY			0x0038
+#define RK3366_WIN0_VIR				0x003c
+#define RK3366_WIN0_YRGB_MST			0x0040
+#define RK3366_WIN0_CBR_MST			0x0044
+#define RK3366_WIN0_ACT_INFO			0x0048
+#define RK3366_WIN0_DSP_INFO			0x004c
+#define RK3366_WIN0_DSP_ST			0x0050
+#define RK3366_WIN0_SCL_FACTOR_YRGB		0x0054
+#define RK3366_WIN0_SCL_FACTOR_CBR		0x0058
+#define RK3366_WIN0_SCL_OFFSET			0x005c
+#define RK3366_WIN0_SRC_ALPHA_CTRL		0x0060
+#define RK3366_WIN0_DST_ALPHA_CTRL		0x0064
+#define RK3366_WIN0_FADING_CTRL			0x0068
+#define RK3366_WIN0_CTRL2			0x006c
+#define RK3366_WIN1_CTRL0			0x0070
+#define RK3366_WIN1_CTRL1			0x0074
+#define RK3366_WIN1_COLOR_KEY			0x0078
+#define RK3366_WIN1_VIR				0x007c
+#define RK3366_WIN1_YRGB_MST			0x0080
+#define RK3366_WIN1_CBR_MST			0x0084
+#define RK3366_WIN1_ACT_INFO			0x0088
+#define RK3366_WIN1_DSP_INFO			0x008c
+#define RK3366_WIN1_DSP_ST			0x0090
+#define RK3366_WIN1_SCL_FACTOR_YRGB		0x0094
+#define RK3366_WIN1_SCL_FACTOR_CBR		0x0098
+#define RK3366_WIN1_SCL_OFFSET			0x009c
+#define RK3366_WIN1_SRC_ALPHA_CTRL		0x00a0
+#define RK3366_WIN1_DST_ALPHA_CTRL		0x00a4
+#define RK3366_WIN1_FADING_CTRL			0x00a8
+#define RK3366_WIN1_CTRL2			0x00ac
+#define RK3366_WIN2_CTRL0			0x00b0
+#define RK3366_WIN2_CTRL1			0x00b4
+#define RK3366_WIN2_VIR0_1			0x00b8
+#define RK3366_WIN2_VIR2_3			0x00bc
+#define RK3366_WIN2_MST0			0x00c0
+#define RK3366_WIN2_DSP_INFO0			0x00c4
+#define RK3366_WIN2_DSP_ST0			0x00c8
+#define RK3366_WIN2_COLOR_KEY			0x00cc
+#define RK3366_WIN2_MST1			0x00d0
+#define RK3366_WIN2_DSP_INFO1			0x00d4
+#define RK3366_WIN2_DSP_ST1			0x00d8
+#define RK3366_WIN2_SRC_ALPHA_CTRL		0x00dc
+#define RK3366_WIN2_MST2			0x00e0
+#define RK3366_WIN2_DSP_INFO2			0x00e4
+#define RK3366_WIN2_DSP_ST2			0x00e8
+#define RK3366_WIN2_DST_ALPHA_CTRL		0x00ec
+#define RK3366_WIN2_MST3			0x00f0
+#define RK3366_WIN2_DSP_INFO3			0x00f4
+#define RK3366_WIN2_DSP_ST3			0x00f8
+#define RK3366_WIN2_FADING_CTRL			0x00fc
+#define RK3366_WIN3_CTRL0			0x0100
+#define RK3366_WIN3_CTRL1			0x0104
+#define RK3366_WIN3_VIR0_1			0x0108
+#define RK3366_WIN3_VIR2_3			0x010c
+#define RK3366_WIN3_MST0			0x0110
+#define RK3366_WIN3_DSP_INFO0			0x0114
+#define RK3366_WIN3_DSP_ST0			0x0118
+#define RK3366_WIN3_COLOR_KEY			0x011c
+#define RK3366_WIN3_MST1			0x0120
+#define RK3366_WIN3_DSP_INFO1			0x0124
+#define RK3366_WIN3_DSP_ST1			0x0128
+#define RK3366_WIN3_SRC_ALPHA_CTRL		0x012c
+#define RK3366_WIN3_MST2			0x0130
+#define RK3366_WIN3_DSP_INFO2			0x0134
+#define RK3366_WIN3_DSP_ST2			0x0138
+#define RK3366_WIN3_DST_ALPHA_CTRL		0x013c
+#define RK3366_WIN3_MST3			0x0140
+#define RK3366_WIN3_DSP_INFO3			0x0144
+#define RK3366_WIN3_DSP_ST3			0x0148
+#define RK3366_WIN3_FADING_CTRL			0x014c
+#define RK3366_HWC_CTRL0			0x0150
+#define RK3366_HWC_CTRL1			0x0154
+#define RK3366_HWC_MST				0x0158
+#define RK3366_HWC_DSP_ST			0x015c
+#define RK3366_HWC_SRC_ALPHA_CTRL		0x0160
+#define RK3366_HWC_DST_ALPHA_CTRL		0x0164
+#define RK3366_HWC_FADING_CTRL			0x0168
+#define RK3366_HWC_RESERVED1			0x016c
+#define RK3366_POST_DSP_HACT_INFO		0x0170
+#define RK3366_POST_DSP_VACT_INFO		0x0174
+#define RK3366_POST_SCL_FACTOR_YRGB		0x0178
+#define RK3366_POST_RESERVED			0x017c
+#define RK3366_POST_SCL_CTRL			0x0180
+#define RK3366_POST_DSP_VACT_INFO_F1		0x0184
+#define RK3366_DSP_HTOTAL_HS_END		0x0188
+#define RK3366_DSP_HACT_ST_END			0x018c
+#define RK3366_DSP_VTOTAL_VS_END		0x0190
+#define RK3366_DSP_VACT_ST_END			0x0194
+#define RK3366_DSP_VS_ST_END_F1			0x0198
+#define RK3366_DSP_VACT_ST_END_F1		0x019c
+#define RK3366_PWM_CTRL				0x01a0
+#define RK3366_PWM_PERIOD_HPR			0x01a4
+#define RK3366_PWM_DUTY_LPR			0x01a8
+#define RK3366_PWM_CNT				0x01ac
+#define RK3366_BCSH_COLOR_BAR			0x01b0
+#define RK3366_BCSH_BCS				0x01b4
+#define RK3366_BCSH_H				0x01b8
+#define RK3366_BCSH_CTRL			0x01bc
+#define RK3366_CABC_CTRL0			0x01c0
+#define RK3366_CABC_CTRL1			0x01c4
+#define RK3366_CABC_CTRL2			0x01c8
+#define RK3366_CABC_CTRL3			0x01cc
+#define RK3366_CABC_GAUSS_LINE0_0		0x01d0
+#define RK3366_CABC_GAUSS_LINE0_1		0x01d4
+#define RK3366_CABC_GAUSS_LINE1_0		0x01d8
+#define RK3366_CABC_GAUSS_LINE1_1		0x01dc
+#define RK3366_CABC_GAUSS_LINE2_0		0x01e0
+#define RK3366_CABC_GAUSS_LINE2_1		0x01e4
+#define RK3366_FRC_LOWER01_0			0x01e8
+#define RK3366_FRC_LOWER01_1			0x01ec
+#define RK3366_FRC_LOWER10_0			0x01f0
+#define RK3366_FRC_LOWER10_1			0x01f4
+#define RK3366_FRC_LOWER11_0			0x01f8
+#define RK3366_FRC_LOWER11_1			0x01fc
+#define RK3366_INTR_EN0				0x0280
+#define RK3366_INTR_CLEAR0			0x0284
+#define RK3366_INTR_STATUS0			0x0288
+#define RK3366_INTR_RAW_STATUS0			0x028c
+#define RK3366_INTR_EN1				0x0290
+#define RK3366_INTR_CLEAR1			0x0294
+#define RK3366_INTR_STATUS1			0x0298
+#define RK3366_INTR_RAW_STATUS1			0x029c
+#define RK3366_LINE_FLAG			0x02a0
+#define RK3366_VOP_STATUS			0x02a4
+#define RK3366_BLANKING_VALUE			0x02a8
+#define RK3366_WIN0_DSP_BG			0x02b0
+#define RK3366_WIN1_DSP_BG			0x02b4
+#define RK3366_WIN2_DSP_BG			0x02b8
+#define RK3366_WIN3_DSP_BG			0x02bc
+#define RK3366_WIN2_LUT_ADDR			0x0400
+#define RK3366_WIN3_LUT_ADDR			0x0800
+#define RK3366_HWC_LUT_ADDR			0x0c00
+#define RK3366_GAMMA0_LUT_ADDR			0x1000
+#define RK3366_GAMMA1_LUT_ADDR			0x1400
+#define RK3366_CABC_GAMMA_LUT_ADDR		0x1800
+#define RK3366_MCU_BYPASS_WPORT			0x2200
+#define RK3366_MCU_BYPASS_RPORT			0x2300
+#define RK3366_MMU_DTE_ADDR			0x2400
+#define RK3366_MMU_STATUS			0x2404
+#define RK3366_MMU_COMMAND			0x2408
+#define RK3366_MMU_PAGE_FAULT_ADDR		0x240c
+#define RK3366_MMU_ZAP_ONE_LINE			0x2410
+#define RK3366_MMU_INT_RAWSTAT			0x2414
+#define RK3366_MMU_INT_CLEAR			0x2418
+#define RK3366_MMU_INT_MASK			0x241c
+#define RK3366_MMU_INT_STATUS			0x2420
+#define RK3366_MMU_AUTO_GATING			0x2424
+
+/* rk3399 register definition */
+#define RK3399_REG_CFG_DONE			0x0000
+#define RK3399_VERSION_INFO			0x0004
+#define RK3399_SYS_CTRL				0x0008
+#define RK3399_SYS_CTRL1			0x000c
+#define RK3399_DSP_CTRL0			0x0010
+#define RK3399_DSP_CTRL1			0x0014
+#define RK3399_DSP_BG				0x0018
+#define RK3399_MCU_CTRL				0x001c
+#define RK3399_WB_CTRL0				0x0020
+#define RK3399_WB_CTRL1				0x0024
+#define RK3399_WB_YRGB_MST			0x0028
+#define RK3399_WB_CBR_MST			0x002c
+#define RK3399_WIN0_CTRL0			0x0030
+#define RK3399_WIN0_CTRL1			0x0034
+#define RK3399_WIN0_COLOR_KEY			0x0038
+#define RK3399_WIN0_VIR				0x003c
+#define RK3399_WIN0_YRGB_MST			0x0040
+#define RK3399_WIN0_CBR_MST			0x0044
+#define RK3399_WIN0_ACT_INFO			0x0048
+#define RK3399_WIN0_DSP_INFO			0x004c
+#define RK3399_WIN0_DSP_ST			0x0050
+#define RK3399_WIN0_SCL_FACTOR_YRGB		0x0054
+#define RK3399_WIN0_SCL_FACTOR_CBR		0x0058
+#define RK3399_WIN0_SCL_OFFSET			0x005c
+#define RK3399_WIN0_SRC_ALPHA_CTRL		0x0060
+#define RK3399_WIN0_DST_ALPHA_CTRL		0x0064
+#define RK3399_WIN0_FADING_CTRL			0x0068
+#define RK3399_WIN0_CTRL2			0x006c
+#define RK3399_WIN1_CTRL0			0x0070
+#define RK3399_WIN1_CTRL1			0x0074
+#define RK3399_WIN1_COLOR_KEY			0x0078
+#define RK3399_WIN1_VIR				0x007c
+#define RK3399_WIN1_YRGB_MST			0x0080
+#define RK3399_WIN1_CBR_MST			0x0084
+#define RK3399_WIN1_ACT_INFO			0x0088
+#define RK3399_WIN1_DSP_INFO			0x008c
+#define RK3399_WIN1_DSP_ST			0x0090
+#define RK3399_WIN1_SCL_FACTOR_YRGB		0x0094
+#define RK3399_WIN1_SCL_FACTOR_CBR		0x0098
+#define RK3399_WIN1_SCL_OFFSET			0x009c
+#define RK3399_WIN1_SRC_ALPHA_CTRL		0x00a0
+#define RK3399_WIN1_DST_ALPHA_CTRL		0x00a4
+#define RK3399_WIN1_FADING_CTRL			0x00a8
+#define RK3399_WIN1_CTRL2			0x00ac
+#define RK3399_WIN2_CTRL0			0x00b0
+#define RK3399_WIN2_CTRL1			0x00b4
+#define RK3399_WIN2_VIR0_1			0x00b8
+#define RK3399_WIN2_VIR2_3			0x00bc
+#define RK3399_WIN2_MST0			0x00c0
+#define RK3399_WIN2_DSP_INFO0			0x00c4
+#define RK3399_WIN2_DSP_ST0			0x00c8
+#define RK3399_WIN2_COLOR_KEY			0x00cc
+#define RK3399_WIN2_MST1			0x00d0
+#define RK3399_WIN2_DSP_INFO1			0x00d4
+#define RK3399_WIN2_DSP_ST1			0x00d8
+#define RK3399_WIN2_SRC_ALPHA_CTRL		0x00dc
+#define RK3399_WIN2_MST2			0x00e0
+#define RK3399_WIN2_DSP_INFO2			0x00e4
+#define RK3399_WIN2_DSP_ST2			0x00e8
+#define RK3399_WIN2_DST_ALPHA_CTRL		0x00ec
+#define RK3399_WIN2_MST3			0x00f0
+#define RK3399_WIN2_DSP_INFO3			0x00f4
+#define RK3399_WIN2_DSP_ST3			0x00f8
+#define RK3399_WIN2_FADING_CTRL			0x00fc
+#define RK3399_WIN3_CTRL0			0x0100
+#define RK3399_WIN3_CTRL1			0x0104
+#define RK3399_WIN3_VIR0_1			0x0108
+#define RK3399_WIN3_VIR2_3			0x010c
+#define RK3399_WIN3_MST0			0x0110
+#define RK3399_WIN3_DSP_INFO0			0x0114
+#define RK3399_WIN3_DSP_ST0			0x0118
+#define RK3399_WIN3_COLOR_KEY			0x011c
+#define RK3399_WIN3_MST1			0x0120
+#define RK3399_WIN3_DSP_INFO1			0x0124
+#define RK3399_WIN3_DSP_ST1			0x0128
+#define RK3399_WIN3_SRC_ALPHA_CTRL		0x012c
+#define RK3399_WIN3_MST2			0x0130
+#define RK3399_WIN3_DSP_INFO2			0x0134
+#define RK3399_WIN3_DSP_ST2			0x0138
+#define RK3399_WIN3_DST_ALPHA_CTRL		0x013c
+#define RK3399_WIN3_MST3			0x0140
+#define RK3399_WIN3_DSP_INFO3			0x0144
+#define RK3399_WIN3_DSP_ST3			0x0148
+#define RK3399_WIN3_FADING_CTRL			0x014c
+#define RK3399_HWC_CTRL0			0x0150
+#define RK3399_HWC_CTRL1			0x0154
+#define RK3399_HWC_MST				0x0158
+#define RK3399_HWC_DSP_ST			0x015c
+#define RK3399_HWC_SRC_ALPHA_CTRL		0x0160
+#define RK3399_HWC_DST_ALPHA_CTRL		0x0164
+#define RK3399_HWC_FADING_CTRL			0x0168
+#define RK3399_HWC_RESERVED1			0x016c
+#define RK3399_POST_DSP_HACT_INFO		0x0170
+#define RK3399_POST_DSP_VACT_INFO		0x0174
+#define RK3399_POST_SCL_FACTOR_YRGB		0x0178
+#define RK3399_POST_RESERVED			0x017c
+#define RK3399_POST_SCL_CTRL			0x0180
+#define RK3399_POST_DSP_VACT_INFO_F1		0x0184
+#define RK3399_DSP_HTOTAL_HS_END		0x0188
+#define RK3399_DSP_HACT_ST_END			0x018c
+#define RK3399_DSP_VTOTAL_VS_END		0x0190
+#define RK3399_DSP_VACT_ST_END			0x0194
+#define RK3399_DSP_VS_ST_END_F1			0x0198
+#define RK3399_DSP_VACT_ST_END_F1		0x019c
+#define RK3399_PWM_CTRL				0x01a0
+#define RK3399_PWM_PERIOD_HPR			0x01a4
+#define RK3399_PWM_DUTY_LPR			0x01a8
+#define RK3399_PWM_CNT				0x01ac
+#define RK3399_BCSH_COLOR_BAR			0x01b0
+#define RK3399_BCSH_BCS				0x01b4
+#define RK3399_BCSH_H				0x01b8
+#define RK3399_BCSH_CTRL			0x01bc
+#define RK3399_CABC_CTRL0			0x01c0
+#define RK3399_CABC_CTRL1			0x01c4
+#define RK3399_CABC_CTRL2			0x01c8
+#define RK3399_CABC_CTRL3			0x01cc
+#define RK3399_CABC_GAUSS_LINE0_0		0x01d0
+#define RK3399_CABC_GAUSS_LINE0_1		0x01d4
+#define RK3399_CABC_GAUSS_LINE1_0		0x01d8
+#define RK3399_CABC_GAUSS_LINE1_1		0x01dc
+#define RK3399_CABC_GAUSS_LINE2_0		0x01e0
+#define RK3399_CABC_GAUSS_LINE2_1		0x01e4
+#define RK3399_FRC_LOWER01_0			0x01e8
+#define RK3399_FRC_LOWER01_1			0x01ec
+#define RK3399_FRC_LOWER10_0			0x01f0
+#define RK3399_FRC_LOWER10_1			0x01f4
+#define RK3399_FRC_LOWER11_0			0x01f8
+#define RK3399_FRC_LOWER11_1			0x01fc
+#define RK3399_AFBCD0_CTRL			0x0200
+#define RK3399_AFBCD0_HDR_PTR			0x0204
+#define RK3399_AFBCD0_PIC_SIZE			0x0208
+#define RK3399_AFBCD0_STATUS			0x020c
+#define RK3399_AFBCD1_CTRL			0x0220
+#define RK3399_AFBCD1_HDR_PTR			0x0224
+#define RK3399_AFBCD1_PIC_SIZE			0x0228
+#define RK3399_AFBCD1_STATUS			0x022c
+#define RK3399_AFBCD2_CTRL			0x0240
+#define RK3399_AFBCD2_HDR_PTR			0x0244
+#define RK3399_AFBCD2_PIC_SIZE			0x0248
+#define RK3399_AFBCD2_STATUS			0x024c
+#define RK3399_AFBCD3_CTRL			0x0260
+#define RK3399_AFBCD3_HDR_PTR			0x0264
+#define RK3399_AFBCD3_PIC_SIZE			0x0268
+#define RK3399_AFBCD3_STATUS			0x026c
+#define RK3399_INTR_EN0				0x0280
+#define RK3399_INTR_CLEAR0			0x0284
+#define RK3399_INTR_STATUS0			0x0288
+#define RK3399_INTR_RAW_STATUS0			0x028c
+#define RK3399_INTR_EN1				0x0290
+#define RK3399_INTR_CLEAR1			0x0294
+#define RK3399_INTR_STATUS1			0x0298
+#define RK3399_INTR_RAW_STATUS1			0x029c
+#define RK3399_LINE_FLAG			0x02a0
+#define RK3399_VOP_STATUS			0x02a4
+#define RK3399_BLANKING_VALUE			0x02a8
+#define RK3399_MCU_BYPASS_PORT			0x02ac
+#define RK3399_WIN0_DSP_BG			0x02b0
+#define RK3399_WIN1_DSP_BG			0x02b4
+#define RK3399_WIN2_DSP_BG			0x02b8
+#define RK3399_WIN3_DSP_BG			0x02bc
+#define RK3399_YUV2YUV_WIN			0x02c0
+#define RK3399_YUV2YUV_POST			0x02c4
+#define RK3399_AUTO_GATING_EN			0x02cc
+#define RK3399_WIN0_CSC_COE			0x03a0
+#define RK3399_WIN1_CSC_COE			0x03c0
+#define RK3399_WIN2_CSC_COE			0x03e0
+#define RK3399_WIN3_CSC_COE			0x0400
+#define RK3399_HWC_CSC_COE			0x0420
+#define RK3399_BCSH_R2Y_CSC_COE			0x0440
+#define RK3399_BCSH_Y2R_CSC_COE			0x0460
+#define RK3399_POST_YUV2YUV_Y2R_COE		0x0480
+#define RK3399_POST_YUV2YUV_3X3_COE		0x04a0
+#define RK3399_POST_YUV2YUV_R2Y_COE		0x04c0
+#define RK3399_WIN0_YUV2YUV_Y2R			0x04e0
+#define RK3399_WIN0_YUV2YUV_3X3			0x0500
+#define RK3399_WIN0_YUV2YUV_R2Y			0x0520
+#define RK3399_WIN1_YUV2YUV_Y2R			0x0540
+#define RK3399_WIN1_YUV2YUV_3X3			0x0560
+#define RK3399_WIN1_YUV2YUV_R2Y			0x0580
+#define RK3399_WIN2_YUV2YUV_Y2R			0x05a0
+#define RK3399_WIN2_YUV2YUV_3X3			0x05c0
+#define RK3399_WIN2_YUV2YUV_R2Y			0x05e0
+#define RK3399_WIN3_YUV2YUV_Y2R			0x0600
+#define RK3399_WIN3_YUV2YUV_3X3			0x0620
+#define RK3399_WIN3_YUV2YUV_R2Y			0x0640
+#define RK3399_WIN2_LUT_ADDR			0x1000
+#define RK3399_WIN3_LUT_ADDR			0x1400
+#define RK3399_HWC_LUT_ADDR			0x1800
+#define RK3399_CABC_GAMMA_LUT_ADDR		0x1c00
+#define RK3399_GAMMA_LUT_ADDR			0x2000
+/* rk3399 register definition end */
+
+/* rk3328 register definition end */
+#define RK3328_REG_CFG_DONE			0x00000000
+#define RK3328_VERSION_INFO			0x00000004
+#define RK3328_SYS_CTRL				0x00000008
+#define RK3328_SYS_CTRL1			0x0000000c
+#define RK3328_DSP_CTRL0			0x00000010
+#define RK3328_DSP_CTRL1			0x00000014
+#define RK3328_DSP_BG				0x00000018
+#define RK3328_AUTO_GATING_EN			0x0000003c
+#define RK3328_LINE_FLAG			0x00000040
+#define RK3328_VOP_STATUS			0x00000044
+#define RK3328_BLANKING_VALUE			0x00000048
+#define RK3328_WIN0_DSP_BG			0x00000050
+#define RK3328_WIN1_DSP_BG			0x00000054
+#define RK3328_DBG_PERF_LATENCY_CTRL0		0x000000c0
+#define RK3328_DBG_PERF_RD_MAX_LATENCY_NUM0	0x000000c4
+#define RK3328_DBG_PERF_RD_LATENCY_THR_NUM0	0x000000c8
+#define RK3328_DBG_PERF_RD_LATENCY_SAMP_NUM0	0x000000cc
+#define RK3328_INTR_EN0				0x000000e0
+#define RK3328_INTR_CLEAR0			0x000000e4
+#define RK3328_INTR_STATUS0			0x000000e8
+#define RK3328_INTR_RAW_STATUS0			0x000000ec
+#define RK3328_INTR_EN1				0x000000f0
+#define RK3328_INTR_CLEAR1			0x000000f4
+#define RK3328_INTR_STATUS1			0x000000f8
+#define RK3328_INTR_RAW_STATUS1			0x000000fc
+#define RK3328_WIN0_CTRL0			0x00000100
+#define RK3328_WIN0_CTRL1			0x00000104
+#define RK3328_WIN0_COLOR_KEY			0x00000108
+#define RK3328_WIN0_VIR				0x0000010c
+#define RK3328_WIN0_YRGB_MST			0x00000110
+#define RK3328_WIN0_CBR_MST			0x00000114
+#define RK3328_WIN0_ACT_INFO			0x00000118
+#define RK3328_WIN0_DSP_INFO			0x0000011c
+#define RK3328_WIN0_DSP_ST			0x00000120
+#define RK3328_WIN0_SCL_FACTOR_YRGB		0x00000124
+#define RK3328_WIN0_SCL_FACTOR_CBR		0x00000128
+#define RK3328_WIN0_SCL_OFFSET			0x0000012c
+#define RK3328_WIN0_SRC_ALPHA_CTRL		0x00000130
+#define RK3328_WIN0_DST_ALPHA_CTRL		0x00000134
+#define RK3328_WIN0_FADING_CTRL			0x00000138
+#define RK3328_WIN0_CTRL2			0x0000013c
+#define RK3328_DBG_WIN0_REG0			0x000001f0
+#define RK3328_DBG_WIN0_REG1			0x000001f4
+#define RK3328_DBG_WIN0_REG2			0x000001f8
+#define RK3328_DBG_WIN0_RESERVED		0x000001fc
+#define RK3328_WIN1_CTRL0			0x00000200
+#define RK3328_WIN1_CTRL1			0x00000204
+#define RK3328_WIN1_COLOR_KEY			0x00000208
+#define RK3328_WIN1_VIR				0x0000020c
+#define RK3328_WIN1_YRGB_MST			0x00000210
+#define RK3328_WIN1_CBR_MST			0x00000214
+#define RK3328_WIN1_ACT_INFO			0x00000218
+#define RK3328_WIN1_DSP_INFO			0x0000021c
+#define RK3328_WIN1_DSP_ST			0x00000220
+#define RK3328_WIN1_SCL_FACTOR_YRGB		0x00000224
+#define RK3328_WIN1_SCL_FACTOR_CBR		0x00000228
+#define RK3328_WIN1_SCL_OFFSET			0x0000022c
+#define RK3328_WIN1_SRC_ALPHA_CTRL		0x00000230
+#define RK3328_WIN1_DST_ALPHA_CTRL		0x00000234
+#define RK3328_WIN1_FADING_CTRL			0x00000238
+#define RK3328_WIN1_CTRL2			0x0000023c
+#define RK3328_DBG_WIN1_REG0			0x000002f0
+#define RK3328_DBG_WIN1_REG1			0x000002f4
+#define RK3328_DBG_WIN1_REG2			0x000002f8
+#define RK3328_DBG_WIN1_RESERVED		0x000002fc
+#define RK3328_WIN2_CTRL0			0x00000300
+#define RK3328_WIN2_CTRL1			0x00000304
+#define RK3328_WIN2_COLOR_KEY			0x00000308
+#define RK3328_WIN2_VIR				0x0000030c
+#define RK3328_WIN2_YRGB_MST			0x00000310
+#define RK3328_WIN2_CBR_MST			0x00000314
+#define RK3328_WIN2_ACT_INFO			0x00000318
+#define RK3328_WIN2_DSP_INFO			0x0000031c
+#define RK3328_WIN2_DSP_ST			0x00000320
+#define RK3328_WIN2_SCL_FACTOR_YRGB		0x00000324
+#define RK3328_WIN2_SCL_FACTOR_CBR		0x00000328
+#define RK3328_WIN2_SCL_OFFSET			0x0000032c
+#define RK3328_WIN2_SRC_ALPHA_CTRL		0x00000330
+#define RK3328_WIN2_DST_ALPHA_CTRL		0x00000334
+#define RK3328_WIN2_FADING_CTRL			0x00000338
+#define RK3328_WIN2_CTRL2			0x0000033c
+#define RK3328_DBG_WIN2_REG0			0x000003f0
+#define RK3328_DBG_WIN2_REG1			0x000003f4
+#define RK3328_DBG_WIN2_REG2			0x000003f8
+#define RK3328_DBG_WIN2_RESERVED		0x000003fc
+#define RK3328_WIN3_CTRL0			0x00000400
+#define RK3328_WIN3_CTRL1			0x00000404
+#define RK3328_WIN3_COLOR_KEY			0x00000408
+#define RK3328_WIN3_VIR				0x0000040c
+#define RK3328_WIN3_YRGB_MST			0x00000410
+#define RK3328_WIN3_CBR_MST			0x00000414
+#define RK3328_WIN3_ACT_INFO			0x00000418
+#define RK3328_WIN3_DSP_INFO			0x0000041c
+#define RK3328_WIN3_DSP_ST			0x00000420
+#define RK3328_WIN3_SCL_FACTOR_YRGB		0x00000424
+#define RK3328_WIN3_SCL_FACTOR_CBR		0x00000428
+#define RK3328_WIN3_SCL_OFFSET			0x0000042c
+#define RK3328_WIN3_SRC_ALPHA_CTRL		0x00000430
+#define RK3328_WIN3_DST_ALPHA_CTRL		0x00000434
+#define RK3328_WIN3_FADING_CTRL			0x00000438
+#define RK3328_WIN3_CTRL2			0x0000043c
+#define RK3328_DBG_WIN3_REG0			0x000004f0
+#define RK3328_DBG_WIN3_REG1			0x000004f4
+#define RK3328_DBG_WIN3_REG2			0x000004f8
+#define RK3328_DBG_WIN3_RESERVED		0x000004fc
+
+#define RK3328_HWC_CTRL0			0x00000500
+#define RK3328_HWC_CTRL1			0x00000504
+#define RK3328_HWC_MST				0x00000508
+#define RK3328_HWC_DSP_ST			0x0000050c
+#define RK3328_HWC_SRC_ALPHA_CTRL		0x00000510
+#define RK3328_HWC_DST_ALPHA_CTRL		0x00000514
+#define RK3328_HWC_FADING_CTRL			0x00000518
+#define RK3328_HWC_RESERVED1			0x0000051c
+#define RK3328_POST_DSP_HACT_INFO		0x00000600
+#define RK3328_POST_DSP_VACT_INFO		0x00000604
+#define RK3328_POST_SCL_FACTOR_YRGB		0x00000608
+#define RK3328_POST_RESERVED			0x0000060c
+#define RK3328_POST_SCL_CTRL			0x00000610
+#define RK3328_POST_DSP_VACT_INFO_F1		0x00000614
+#define RK3328_DSP_HTOTAL_HS_END		0x00000618
+#define RK3328_DSP_HACT_ST_END			0x0000061c
+#define RK3328_DSP_VTOTAL_VS_END		0x00000620
+#define RK3328_DSP_VACT_ST_END			0x00000624
+#define RK3328_DSP_VS_ST_END_F1			0x00000628
+#define RK3328_DSP_VACT_ST_END_F1		0x0000062c
+#define RK3328_BCSH_COLOR_BAR			0x00000640
+#define RK3328_BCSH_BCS				0x00000644
+#define RK3328_BCSH_H				0x00000648
+#define RK3328_BCSH_CTRL			0x0000064c
+#define RK3328_FRC_LOWER01_0			0x00000678
+#define RK3328_FRC_LOWER01_1			0x0000067c
+#define RK3328_FRC_LOWER10_0			0x00000680
+#define RK3328_FRC_LOWER10_1			0x00000684
+#define RK3328_FRC_LOWER11_0			0x00000688
+#define RK3328_FRC_LOWER11_1			0x0000068c
+#define RK3328_DBG_POST_REG0			0x000006e8
+#define RK3328_DBG_POST_RESERVED		0x000006ec
+#define RK3328_DBG_DATAO			0x000006f0
+#define RK3328_DBG_DATAO_2			0x000006f4
+
+/* sdr to hdr */
+#define RK3328_SDR2HDR_CTRL			0x00000700
+#define RK3328_EOTF_OETF_Y0			0x00000704
+#define RK3328_RESERVED0001			0x00000708
+#define RK3328_RESERVED0002			0x0000070c
+#define RK3328_EOTF_OETF_Y1			0x00000710
+#define RK3328_EOTF_OETF_Y64			0x0000080c
+#define RK3328_OETF_DX_DXPOW1			0x00000810
+#define RK3328_OETF_DX_DXPOW64			0x0000090c
+#define RK3328_OETF_XN1				0x00000910
+#define RK3328_OETF_XN63			0x00000a08
+
+/* hdr to sdr */
+#define RK3328_HDR2SDR_CTRL			0x00000a10
+#define RK3328_HDR2SDR_SRC_RANGE		0x00000a14
+#define RK3328_HDR2SDR_NORMFACEETF		0x00000a18
+#define RK3328_RESERVED0003			0x00000a1c
+#define RK3328_HDR2SDR_DST_RANGE		0x00000a20
+#define RK3328_HDR2SDR_NORMFACCGAMMA		0x00000a24
+#define RK3328_EETF_OETF_Y0			0x00000a28
+#define RK3328_SAT_Y0				0x00000a2c
+#define RK3328_EETF_OETF_Y1			0x00000a30
+#define RK3328_SAT_Y1				0x00000ab0
+#define RK3328_SAT_Y8				0x00000acc
+
+#define RK3328_HWC_LUT_ADDR			0x00000c00
+
 /* rk3036 register definition */
 #define RK3036_SYS_CTRL			0x00
 #define RK3036_DSP_CTRL0		0x04
@@ -166,197 +878,4 @@
 #define RK3036_HWC_LUT_ADDR		0x800
 /* rk3036 register definition end */
 
-/* rk3399 register definition */
-#define RK3399_REG_CFG_DONE		0x00000
-#define RK3399_VERSION_INFO		0x00004
-#define RK3399_SYS_CTRL			0x00008
-#define RK3399_SYS_CTRL1		0x0000c
-#define RK3399_DSP_CTRL0		0x00010
-#define RK3399_DSP_CTRL1		0x00014
-#define RK3399_DSP_BG			0x00018
-#define RK3399_MCU_CTRL			0x0001c
-#define RK3399_WB_CTRL0			0x00020
-#define RK3399_WB_CTRL1			0x00024
-#define RK3399_WB_YRGB_MST		0x00028
-#define RK3399_WB_CBR_MST		0x0002c
-#define RK3399_WIN0_CTRL0		0x00030
-#define RK3399_WIN0_CTRL1		0x00034
-#define RK3399_WIN0_COLOR_KEY		0x00038
-#define RK3399_WIN0_VIR			0x0003c
-#define RK3399_WIN0_YRGB_MST		0x00040
-#define RK3399_WIN0_CBR_MST		0x00044
-#define RK3399_WIN0_ACT_INFO		0x00048
-#define RK3399_WIN0_DSP_INFO		0x0004c
-#define RK3399_WIN0_DSP_ST		0x00050
-#define RK3399_WIN0_SCL_FACTOR_YRGB	0x00054
-#define RK3399_WIN0_SCL_FACTOR_CBR	0x00058
-#define RK3399_WIN0_SCL_OFFSET		0x0005c
-#define RK3399_WIN0_SRC_ALPHA_CTRL	0x00060
-#define RK3399_WIN0_DST_ALPHA_CTRL	0x00064
-#define RK3399_WIN0_FADING_CTRL		0x00068
-#define RK3399_WIN0_CTRL2		0x0006c
-#define RK3399_WIN1_CTRL0		0x00070
-#define RK3399_WIN1_CTRL1		0x00074
-#define RK3399_WIN1_COLOR_KEY		0x00078
-#define RK3399_WIN1_VIR			0x0007c
-#define RK3399_WIN1_YRGB_MST		0x00080
-#define RK3399_WIN1_CBR_MST		0x00084
-#define RK3399_WIN1_ACT_INFO		0x00088
-#define RK3399_WIN1_DSP_INFO		0x0008c
-#define RK3399_WIN1_DSP_ST		0x00090
-#define RK3399_WIN1_SCL_FACTOR_YRGB	0x00094
-#define RK3399_WIN1_SCL_FACTOR_CBR	0x00098
-#define RK3399_WIN1_SCL_OFFSET		0x0009c
-#define RK3399_WIN1_SRC_ALPHA_CTRL	0x000a0
-#define RK3399_WIN1_DST_ALPHA_CTRL	0x000a4
-#define RK3399_WIN1_FADING_CTRL		0x000a8
-#define RK3399_WIN1_CTRL2		0x000ac
-#define RK3399_WIN2_CTRL0		0x000b0
-#define RK3399_WIN2_CTRL1		0x000b4
-#define RK3399_WIN2_VIR0_1		0x000b8
-#define RK3399_WIN2_VIR2_3		0x000bc
-#define RK3399_WIN2_MST0		0x000c0
-#define RK3399_WIN2_DSP_INFO0		0x000c4
-#define RK3399_WIN2_DSP_ST0		0x000c8
-#define RK3399_WIN2_COLOR_KEY		0x000cc
-#define RK3399_WIN2_MST1		0x000d0
-#define RK3399_WIN2_DSP_INFO1		0x000d4
-#define RK3399_WIN2_DSP_ST1		0x000d8
-#define RK3399_WIN2_SRC_ALPHA_CTRL	0x000dc
-#define RK3399_WIN2_MST2		0x000e0
-#define RK3399_WIN2_DSP_INFO2		0x000e4
-#define RK3399_WIN2_DSP_ST2		0x000e8
-#define RK3399_WIN2_DST_ALPHA_CTRL	0x000ec
-#define RK3399_WIN2_MST3		0x000f0
-#define RK3399_WIN2_DSP_INFO3		0x000f4
-#define RK3399_WIN2_DSP_ST3		0x000f8
-#define RK3399_WIN2_FADING_CTRL		0x000fc
-#define RK3399_WIN3_CTRL0		0x00100
-#define RK3399_WIN3_CTRL1		0x00104
-#define RK3399_WIN3_VIR0_1		0x00108
-#define RK3399_WIN3_VIR2_3		0x0010c
-#define RK3399_WIN3_MST0		0x00110
-#define RK3399_WIN3_DSP_INFO0		0x00114
-#define RK3399_WIN3_DSP_ST0		0x00118
-#define RK3399_WIN3_COLOR_KEY		0x0011c
-#define RK3399_WIN3_MST1		0x00120
-#define RK3399_WIN3_DSP_INFO1		0x00124
-#define RK3399_WIN3_DSP_ST1		0x00128
-#define RK3399_WIN3_SRC_ALPHA_CTRL	0x0012c
-#define RK3399_WIN3_MST2		0x00130
-#define RK3399_WIN3_DSP_INFO2		0x00134
-#define RK3399_WIN3_DSP_ST2		0x00138
-#define RK3399_WIN3_DST_ALPHA_CTRL	0x0013c
-#define RK3399_WIN3_MST3		0x00140
-#define RK3399_WIN3_DSP_INFO3		0x00144
-#define RK3399_WIN3_DSP_ST3		0x00148
-#define RK3399_WIN3_FADING_CTRL		0x0014c
-#define RK3399_HWC_CTRL0		0x00150
-#define RK3399_HWC_CTRL1		0x00154
-#define RK3399_HWC_MST			0x00158
-#define RK3399_HWC_DSP_ST		0x0015c
-#define RK3399_HWC_SRC_ALPHA_CTRL	0x00160
-#define RK3399_HWC_DST_ALPHA_CTRL	0x00164
-#define RK3399_HWC_FADING_CTRL		0x00168
-#define RK3399_HWC_RESERVED1		0x0016c
-#define RK3399_POST_DSP_HACT_INFO	0x00170
-#define RK3399_POST_DSP_VACT_INFO	0x00174
-#define RK3399_POST_SCL_FACTOR_YRGB	0x00178
-#define RK3399_POST_RESERVED		0x0017c
-#define RK3399_POST_SCL_CTRL		0x00180
-#define RK3399_POST_DSP_VACT_INFO_F1	0x00184
-#define RK3399_DSP_HTOTAL_HS_END	0x00188
-#define RK3399_DSP_HACT_ST_END		0x0018c
-#define RK3399_DSP_VTOTAL_VS_END	0x00190
-#define RK3399_DSP_VACT_ST_END		0x00194
-#define RK3399_DSP_VS_ST_END_F1		0x00198
-#define RK3399_DSP_VACT_ST_END_F1	0x0019c
-#define RK3399_PWM_CTRL			0x001a0
-#define RK3399_PWM_PERIOD_HPR		0x001a4
-#define RK3399_PWM_DUTY_LPR		0x001a8
-#define RK3399_PWM_CNT			0x001ac
-#define RK3399_BCSH_COLOR_BAR		0x001b0
-#define RK3399_BCSH_BCS			0x001b4
-#define RK3399_BCSH_H			0x001b8
-#define RK3399_BCSH_CTRL		0x001bc
-#define RK3399_CABC_CTRL0		0x001c0
-#define RK3399_CABC_CTRL1		0x001c4
-#define RK3399_CABC_CTRL2		0x001c8
-#define RK3399_CABC_CTRL3		0x001cc
-#define RK3399_CABC_GAUSS_LINE0_0	0x001d0
-#define RK3399_CABC_GAUSS_LINE0_1	0x001d4
-#define RK3399_CABC_GAUSS_LINE1_0	0x001d8
-#define RK3399_CABC_GAUSS_LINE1_1	0x001dc
-#define RK3399_CABC_GAUSS_LINE2_0	0x001e0
-#define RK3399_CABC_GAUSS_LINE2_1	0x001e4
-#define RK3399_FRC_LOWER01_0		0x001e8
-#define RK3399_FRC_LOWER01_1		0x001ec
-#define RK3399_FRC_LOWER10_0		0x001f0
-#define RK3399_FRC_LOWER10_1		0x001f4
-#define RK3399_FRC_LOWER11_0		0x001f8
-#define RK3399_FRC_LOWER11_1		0x001fc
-#define RK3399_AFBCD0_CTRL		0x00200
-#define RK3399_AFBCD0_HDR_PTR		0x00204
-#define RK3399_AFBCD0_PIC_SIZE		0x00208
-#define RK3399_AFBCD0_STATUS		0x0020c
-#define RK3399_AFBCD1_CTRL		0x00220
-#define RK3399_AFBCD1_HDR_PTR		0x00224
-#define RK3399_AFBCD1_PIC_SIZE		0x00228
-#define RK3399_AFBCD1_STATUS		0x0022c
-#define RK3399_AFBCD2_CTRL		0x00240
-#define RK3399_AFBCD2_HDR_PTR		0x00244
-#define RK3399_AFBCD2_PIC_SIZE		0x00248
-#define RK3399_AFBCD2_STATUS		0x0024c
-#define RK3399_AFBCD3_CTRL		0x00260
-#define RK3399_AFBCD3_HDR_PTR		0x00264
-#define RK3399_AFBCD3_PIC_SIZE		0x00268
-#define RK3399_AFBCD3_STATUS		0x0026c
-#define RK3399_INTR_EN0			0x00280
-#define RK3399_INTR_CLEAR0		0x00284
-#define RK3399_INTR_STATUS0		0x00288
-#define RK3399_INTR_RAW_STATUS0		0x0028c
-#define RK3399_INTR_EN1			0x00290
-#define RK3399_INTR_CLEAR1		0x00294
-#define RK3399_INTR_STATUS1		0x00298
-#define RK3399_INTR_RAW_STATUS1		0x0029c
-#define RK3399_LINE_FLAG		0x002a0
-#define RK3399_VOP_STATUS		0x002a4
-#define RK3399_BLANKING_VALUE		0x002a8
-#define RK3399_MCU_BYPASS_PORT		0x002ac
-#define RK3399_WIN0_DSP_BG		0x002b0
-#define RK3399_WIN1_DSP_BG		0x002b4
-#define RK3399_WIN2_DSP_BG		0x002b8
-#define RK3399_WIN3_DSP_BG		0x002bc
-#define RK3399_YUV2YUV_WIN		0x002c0
-#define RK3399_YUV2YUV_POST		0x002c4
-#define RK3399_AUTO_GATING_EN		0x002cc
-#define RK3399_WIN0_CSC_COE		0x003a0
-#define RK3399_WIN1_CSC_COE		0x003c0
-#define RK3399_WIN2_CSC_COE		0x003e0
-#define RK3399_WIN3_CSC_COE		0x00400
-#define RK3399_HWC_CSC_COE		0x00420
-#define RK3399_BCSH_R2Y_CSC_COE		0x00440
-#define RK3399_BCSH_Y2R_CSC_COE		0x00460
-#define RK3399_POST_YUV2YUV_Y2R_COE	0x00480
-#define RK3399_POST_YUV2YUV_3X3_COE	0x004a0
-#define RK3399_POST_YUV2YUV_R2Y_COE	0x004c0
-#define RK3399_WIN0_YUV2YUV_Y2R		0x004e0
-#define RK3399_WIN0_YUV2YUV_3X3		0x00500
-#define RK3399_WIN0_YUV2YUV_R2Y		0x00520
-#define RK3399_WIN1_YUV2YUV_Y2R		0x00540
-#define RK3399_WIN1_YUV2YUV_3X3		0x00560
-#define RK3399_WIN1_YUV2YUV_R2Y		0x00580
-#define RK3399_WIN2_YUV2YUV_Y2R		0x005a0
-#define RK3399_WIN2_YUV2YUV_3X3		0x005c0
-#define RK3399_WIN2_YUV2YUV_R2Y		0x005e0
-#define RK3399_WIN3_YUV2YUV_Y2R		0x00600
-#define RK3399_WIN3_YUV2YUV_3X3		0x00620
-#define RK3399_WIN3_YUV2YUV_R2Y		0x00640
-#define RK3399_WIN2_LUT_ADDR		0x01000
-#define RK3399_WIN3_LUT_ADDR		0x01400
-#define RK3399_HWC_LUT_ADDR		0x01800
-#define RK3399_CABC_GAMMA_LUT_ADDR	0x01c00
-#define RK3399_GAMMA_LUT_ADDR		0x02000
-/* rk3399 register definition end */
-
 #endif /* _ROCKCHIP_VOP_REG_H */
diff --git a/drivers/gpu/drm/savage/savage_drv.c b/drivers/gpu/drm/savage/savage_drv.c
index 78c6d8e9b42c..2bddeb8bf457 100644
--- a/drivers/gpu/drm/savage/savage_drv.c
+++ b/drivers/gpu/drm/savage/savage_drv.c
@@ -55,7 +55,6 @@ static struct drm_driver driver = {
 	.preclose = savage_reclaim_buffers,
 	.lastclose = savage_driver_lastclose,
 	.unload = savage_driver_unload,
-	.set_busid = drm_pci_set_busid,
 	.ioctls = savage_ioctls,
 	.dma_ioctl = savage_bci_buffers,
 	.fops = &savage_driver_fops,
@@ -75,12 +74,12 @@ static struct pci_driver savage_pci_driver = {
 static int __init savage_init(void)
 {
 	driver.num_ioctls = savage_max_ioctl;
-	return drm_pci_init(&driver, &savage_pci_driver);
+	return drm_legacy_pci_init(&driver, &savage_pci_driver);
 }
 
 static void __exit savage_exit(void)
 {
-	drm_pci_exit(&driver, &savage_pci_driver);
+	drm_legacy_pci_exit(&driver, &savage_pci_driver);
 }
 
 module_init(savage_init);
diff --git a/drivers/gpu/drm/selftests/test-drm_mm.c b/drivers/gpu/drm/selftests/test-drm_mm.c
index dfdd858eda0a..86eb4c185a28 100644
--- a/drivers/gpu/drm/selftests/test-drm_mm.c
+++ b/drivers/gpu/drm/selftests/test-drm_mm.c
@@ -1627,7 +1627,7 @@ static int igt_topdown(void *ignored)
 		goto err;
 
 	bitmap = kzalloc(count / BITS_PER_LONG * sizeof(unsigned long),
-			 GFP_TEMPORARY);
+			 GFP_KERNEL);
 	if (!bitmap)
 		goto err_nodes;
 
@@ -1741,7 +1741,7 @@ static int igt_bottomup(void *ignored)
 		goto err;
 
 	bitmap = kzalloc(count / BITS_PER_LONG * sizeof(unsigned long),
-			 GFP_TEMPORARY);
+			 GFP_KERNEL);
 	if (!bitmap)
 		goto err_nodes;
 
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_drv.c b/drivers/gpu/drm/shmobile/shmob_drm_drv.c
index 800d1d2c435d..592572554eb0 100644
--- a/drivers/gpu/drm/shmobile/shmob_drm_drv.c
+++ b/drivers/gpu/drm/shmobile/shmob_drm_drv.c
@@ -145,8 +145,6 @@ static struct drm_driver shmob_drm_driver = {
 	.gem_prime_vunmap	= drm_gem_cma_prime_vunmap,
 	.gem_prime_mmap		= drm_gem_cma_prime_mmap,
 	.dumb_create		= drm_gem_cma_dumb_create,
-	.dumb_map_offset	= drm_gem_cma_dumb_map_offset,
-	.dumb_destroy		= drm_gem_dumb_destroy,
 	.fops			= &shmob_drm_fops,
 	.name			= "shmob-drm",
 	.desc			= "Renesas SH Mobile DRM",
@@ -277,7 +275,7 @@ static int shmob_drm_probe(struct platform_device *pdev)
 	ret = drm_irq_install(ddev, platform_get_irq(pdev, 0));
 	if (ret < 0) {
 		dev_err(&pdev->dev, "failed to install IRQ handler\n");
-		goto err_vblank_cleanup;
+		goto err_modeset_cleanup;
 	}
 
 	/*
@@ -292,8 +290,6 @@ static int shmob_drm_probe(struct platform_device *pdev)
 
 err_irq_uninstall:
 	drm_irq_uninstall(ddev);
-err_vblank_cleanup:
-	drm_vblank_cleanup(ddev);
 err_modeset_cleanup:
 	drm_kms_helper_poll_fini(ddev);
 	drm_mode_config_cleanup(ddev);
diff --git a/drivers/gpu/drm/sis/sis_drv.c b/drivers/gpu/drm/sis/sis_drv.c
index 7f05da13ea5e..e04a92658cd7 100644
--- a/drivers/gpu/drm/sis/sis_drv.c
+++ b/drivers/gpu/drm/sis/sis_drv.c
@@ -104,7 +104,6 @@ static struct drm_driver driver = {
 	.open = sis_driver_open,
 	.preclose = sis_reclaim_buffers_locked,
 	.postclose = sis_driver_postclose,
-	.set_busid = drm_pci_set_busid,
 	.dma_quiescent = sis_idle,
 	.lastclose = sis_lastclose,
 	.ioctls = sis_ioctls,
@@ -125,12 +124,12 @@ static struct pci_driver sis_pci_driver = {
 static int __init sis_init(void)
 {
 	driver.num_ioctls = sis_max_ioctl;
-	return drm_pci_init(&driver, &sis_pci_driver);
+	return drm_legacy_pci_init(&driver, &sis_pci_driver);
 }
 
 static void __exit sis_exit(void)
 {
-	drm_pci_exit(&driver, &sis_pci_driver);
+	drm_legacy_pci_exit(&driver, &sis_pci_driver);
 }
 
 module_init(sis_init);
diff --git a/drivers/gpu/drm/sti/sti_crtc.c b/drivers/gpu/drm/sti/sti_crtc.c
index d45a4335df5d..e8a4d48e985a 100644
--- a/drivers/gpu/drm/sti/sti_crtc.c
+++ b/drivers/gpu/drm/sti/sti_crtc.c
@@ -20,7 +20,8 @@
 #include "sti_vid.h"
 #include "sti_vtg.h"
 
-static void sti_crtc_enable(struct drm_crtc *crtc)
+static void sti_crtc_atomic_enable(struct drm_crtc *crtc,
+				   struct drm_crtc_state *old_state)
 {
 	struct sti_mixer *mixer = to_sti_mixer(crtc);
 
@@ -31,7 +32,8 @@ static void sti_crtc_enable(struct drm_crtc *crtc)
 	drm_crtc_vblank_on(crtc);
 }
 
-static void sti_crtc_disabling(struct drm_crtc *crtc)
+static void sti_crtc_atomic_disable(struct drm_crtc *crtc,
+				    struct drm_crtc_state *old_state)
 {
 	struct sti_mixer *mixer = to_sti_mixer(crtc);
 
@@ -222,10 +224,10 @@ static void sti_crtc_atomic_flush(struct drm_crtc *crtc,
 }
 
 static const struct drm_crtc_helper_funcs sti_crtc_helper_funcs = {
-	.enable = sti_crtc_enable,
-	.disable = sti_crtc_disabling,
 	.mode_set_nofb = sti_crtc_mode_set_nofb,
 	.atomic_flush = sti_crtc_atomic_flush,
+	.atomic_enable = sti_crtc_atomic_enable,
+	.atomic_disable = sti_crtc_atomic_disable,
 };
 
 static void sti_crtc_destroy(struct drm_crtc *crtc)
diff --git a/drivers/gpu/drm/sti/sti_cursor.c b/drivers/gpu/drm/sti/sti_cursor.c
index 5b3a41f74f21..b709ebbec095 100644
--- a/drivers/gpu/drm/sti/sti_cursor.c
+++ b/drivers/gpu/drm/sti/sti_cursor.c
@@ -348,7 +348,6 @@ static const struct drm_plane_funcs sti_cursor_plane_helpers_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = sti_cursor_destroy,
-	.set_property = drm_atomic_helper_plane_set_property,
 	.reset = sti_plane_reset,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
@@ -392,7 +391,7 @@ struct drm_plane *sti_cursor_create(struct drm_device *drm_dev,
 				       &sti_cursor_plane_helpers_funcs,
 				       cursor_supported_formats,
 				       ARRAY_SIZE(cursor_supported_formats),
-				       DRM_PLANE_TYPE_CURSOR, NULL);
+				       NULL, DRM_PLANE_TYPE_CURSOR, NULL);
 	if (res) {
 		DRM_ERROR("Failed to initialize universal plane\n");
 		goto err_plane;
diff --git a/drivers/gpu/drm/sti/sti_drv.c b/drivers/gpu/drm/sti/sti_drv.c
index a4b574283269..1700c542cd93 100644
--- a/drivers/gpu/drm/sti/sti_drv.c
+++ b/drivers/gpu/drm/sti/sti_drv.c
@@ -175,8 +175,6 @@ static struct drm_driver sti_driver = {
 	.gem_free_object_unlocked = drm_gem_cma_free_object,
 	.gem_vm_ops = &drm_gem_cma_vm_ops,
 	.dumb_create = drm_gem_cma_dumb_create,
-	.dumb_map_offset = drm_gem_cma_dumb_map_offset,
-	.dumb_destroy = drm_gem_dumb_destroy,
 	.fops = &sti_driver_fops,
 
 	.enable_vblank = sti_crtc_enable_vblank,
@@ -237,7 +235,6 @@ static void sti_cleanup(struct drm_device *ddev)
 	}
 
 	drm_kms_helper_poll_fini(ddev);
-	drm_vblank_cleanup(ddev);
 	component_unbind_all(ddev->dev, ddev);
 	kfree(private);
 	ddev->dev_private = NULL;
diff --git a/drivers/gpu/drm/sti/sti_dvo.c b/drivers/gpu/drm/sti/sti_dvo.c
index 24ebc6b2f34d..852bf2293b05 100644
--- a/drivers/gpu/drm/sti/sti_dvo.c
+++ b/drivers/gpu/drm/sti/sti_dvo.c
@@ -412,7 +412,6 @@ static int sti_dvo_late_register(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs sti_dvo_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = sti_dvo_connector_detect,
 	.destroy = drm_connector_cleanup,
@@ -582,7 +581,7 @@ static int sti_dvo_remove(struct platform_device *pdev)
 	return 0;
 }
 
-static struct of_device_id dvo_of_match[] = {
+static const struct of_device_id dvo_of_match[] = {
 	{ .compatible = "st,stih407-dvo", },
 	{ /* end node */ }
 };
diff --git a/drivers/gpu/drm/sti/sti_gdp.c b/drivers/gpu/drm/sti/sti_gdp.c
index 5ee0503945c8..b65eea4f2c97 100644
--- a/drivers/gpu/drm/sti/sti_gdp.c
+++ b/drivers/gpu/drm/sti/sti_gdp.c
@@ -895,7 +895,6 @@ static const struct drm_plane_funcs sti_gdp_plane_helpers_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = sti_gdp_destroy,
-	.set_property = drm_atomic_helper_plane_set_property,
 	.reset = sti_plane_reset,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
@@ -931,7 +930,7 @@ struct drm_plane *sti_gdp_create(struct drm_device *drm_dev,
 				       &sti_gdp_plane_helpers_funcs,
 				       gdp_supported_formats,
 				       ARRAY_SIZE(gdp_supported_formats),
-				       type, NULL);
+				       NULL, type, NULL);
 	if (res) {
 		DRM_ERROR("Failed to initialize universal plane\n");
 		goto err;
diff --git a/drivers/gpu/drm/sti/sti_hda.c b/drivers/gpu/drm/sti/sti_hda.c
index d6ed909d9d75..cf65e32b5090 100644
--- a/drivers/gpu/drm/sti/sti_hda.c
+++ b/drivers/gpu/drm/sti/sti_hda.c
@@ -647,7 +647,6 @@ static int sti_hda_late_register(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs sti_hda_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = drm_connector_cleanup,
 	.reset = drm_atomic_helper_connector_reset,
diff --git a/drivers/gpu/drm/sti/sti_hdmi.c b/drivers/gpu/drm/sti/sti_hdmi.c
index a59c95a8081b..30f02d2fdd03 100644
--- a/drivers/gpu/drm/sti/sti_hdmi.c
+++ b/drivers/gpu/drm/sti/sti_hdmi.c
@@ -434,7 +434,7 @@ static int hdmi_avi_infoframe_config(struct sti_hdmi *hdmi)
 
 	DRM_DEBUG_DRIVER("\n");
 
-	ret = drm_hdmi_avi_infoframe_from_display_mode(&infoframe, mode);
+	ret = drm_hdmi_avi_infoframe_from_display_mode(&infoframe, mode, false);
 	if (ret < 0) {
 		DRM_ERROR("failed to setup AVI infoframe: %d\n", ret);
 		return ret;
@@ -1113,12 +1113,10 @@ static int sti_hdmi_late_register(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs sti_hdmi_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = sti_hdmi_connector_detect,
 	.destroy = drm_connector_cleanup,
 	.reset = drm_atomic_helper_connector_reset,
-	.set_property = drm_atomic_helper_connector_set_property,
 	.atomic_set_property = sti_hdmi_connector_set_property,
 	.atomic_get_property = sti_hdmi_connector_get_property,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
diff --git a/drivers/gpu/drm/sti/sti_hqvdp.c b/drivers/gpu/drm/sti/sti_hqvdp.c
index a1c161f77804..b19b3430b296 100644
--- a/drivers/gpu/drm/sti/sti_hqvdp.c
+++ b/drivers/gpu/drm/sti/sti_hqvdp.c
@@ -958,6 +958,7 @@ static void sti_hqvdp_start_xp70(struct sti_hqvdp *hqvdp)
 	}
 	if (i == POLL_MAX_ATTEMPT) {
 		DRM_ERROR("Could not reset\n");
+		clk_disable_unprepare(hqvdp->clk);
 		goto out;
 	}
 
@@ -994,6 +995,7 @@ static void sti_hqvdp_start_xp70(struct sti_hqvdp *hqvdp)
 	}
 	if (i == POLL_MAX_ATTEMPT) {
 		DRM_ERROR("Could not boot\n");
+		clk_disable_unprepare(hqvdp->clk);
 		goto out;
 	}
 
@@ -1081,6 +1083,7 @@ static int sti_hqvdp_atomic_check(struct drm_plane *drm_plane,
 					    &hqvdp->vtg_nb,
 					    crtc)) {
 			DRM_ERROR("Cannot register VTG notifier\n");
+			clk_disable_unprepare(hqvdp->clk_pix_main);
 			return -EINVAL;
 		}
 		hqvdp->vtg_registered = true;
@@ -1273,7 +1276,6 @@ static const struct drm_plane_funcs sti_hqvdp_plane_helpers_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = sti_hqvdp_destroy,
-	.set_property = drm_atomic_helper_plane_set_property,
 	.reset = sti_plane_reset,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
@@ -1295,7 +1297,7 @@ static struct drm_plane *sti_hqvdp_create(struct drm_device *drm_dev,
 				       &sti_hqvdp_plane_helpers_funcs,
 				       hqvdp_supported_formats,
 				       ARRAY_SIZE(hqvdp_supported_formats),
-				       DRM_PLANE_TYPE_OVERLAY, NULL);
+				       NULL, DRM_PLANE_TYPE_OVERLAY, NULL);
 	if (res) {
 		DRM_ERROR("Failed to initialize universal plane\n");
 		return NULL;
@@ -1395,7 +1397,7 @@ static int sti_hqvdp_remove(struct platform_device *pdev)
 	return 0;
 }
 
-static struct of_device_id hqvdp_of_match[] = {
+static const struct of_device_id hqvdp_of_match[] = {
 	{ .compatible = "st,stih407-hqvdp", },
 	{ /* end node */ }
 };
diff --git a/drivers/gpu/drm/stm/Kconfig b/drivers/gpu/drm/stm/Kconfig
index 8fe5b184b4e8..35367ada3bc1 100644
--- a/drivers/gpu/drm/stm/Kconfig
+++ b/drivers/gpu/drm/stm/Kconfig
@@ -4,7 +4,7 @@ config DRM_STM
 	select DRM_KMS_HELPER
 	select DRM_GEM_CMA_HELPER
 	select DRM_KMS_CMA_HELPER
-	select DRM_PANEL
+	select DRM_PANEL_BRIDGE
 	select VIDEOMODE_HELPERS
 	select FB_PROVIDE_GET_FB_UNMAPPED_AREA
 
@@ -13,3 +13,10 @@ config DRM_STM
 	  STMicroelectronics STM32 MCUs.
 	  To compile this driver as a module, choose M here: the module
 	  will be called stm-drm.
+
+config DRM_STM_DSI
+	tristate "STMicroelectronics specific extensions for Synopsys MIPI DSI"
+	depends on DRM_STM
+	select DRM_DW_MIPI_DSI
+	help
+	  Choose this option for MIPI DSI support on STMicroelectronics SoC.
diff --git a/drivers/gpu/drm/stm/Makefile b/drivers/gpu/drm/stm/Makefile
index a09ecf450218..d883adc365a2 100644
--- a/drivers/gpu/drm/stm/Makefile
+++ b/drivers/gpu/drm/stm/Makefile
@@ -2,4 +2,6 @@ stm-drm-y := \
 	drv.o \
 	ltdc.o
 
+obj-$(CONFIG_DRM_STM_DSI) += dw_mipi_dsi-stm.o
+
 obj-$(CONFIG_DRM_STM) += stm-drm.o
diff --git a/drivers/gpu/drm/stm/drv.c b/drivers/gpu/drm/stm/drv.c
index 83ab48f1fd00..b333b37f3f89 100644
--- a/drivers/gpu/drm/stm/drv.c
+++ b/drivers/gpu/drm/stm/drv.c
@@ -20,13 +20,6 @@
 
 #include "ltdc.h"
 
-#define DRIVER_NAME		"stm"
-#define DRIVER_DESC		"STMicroelectronics SoC DRM"
-#define DRIVER_DATE		"20170330"
-#define DRIVER_MAJOR		1
-#define DRIVER_MINOR		0
-#define DRIVER_PATCH_LEVEL	0
-
 #define STM_MAX_FB_WIDTH	2048
 #define STM_MAX_FB_HEIGHT	2048 /* same as width to handle orientation */
 
@@ -59,16 +52,14 @@ static struct drm_driver drv_driver = {
 	.driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME |
 			   DRIVER_ATOMIC,
 	.lastclose = drv_lastclose,
-	.name = DRIVER_NAME,
-	.desc = DRIVER_DESC,
-	.date = DRIVER_DATE,
-	.major = DRIVER_MAJOR,
-	.minor = DRIVER_MINOR,
-	.patchlevel = DRIVER_PATCH_LEVEL,
+	.name = "stm",
+	.desc = "STMicroelectronics SoC DRM",
+	.date = "20170330",
+	.major = 1,
+	.minor = 0,
+	.patchlevel = 0,
 	.fops = &drv_driver_fops,
 	.dumb_create = drm_gem_cma_dumb_create,
-	.dumb_map_offset = drm_gem_cma_dumb_map_offset,
-	.dumb_destroy = drm_gem_dumb_destroy,
 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
 	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
 	.gem_free_object_unlocked = drm_gem_cma_free_object,
@@ -206,7 +197,7 @@ static struct platform_driver stm_drm_platform_driver = {
 	.probe = stm_drm_platform_probe,
 	.remove = stm_drm_platform_remove,
 	.driver = {
-		.name = DRIVER_NAME,
+		.name = "stm32-display",
 		.of_match_table = drv_dt_ids,
 	},
 };
diff --git a/drivers/gpu/drm/stm/dw_mipi_dsi-stm.c b/drivers/gpu/drm/stm/dw_mipi_dsi-stm.c
new file mode 100644
index 000000000000..568c5d0461ea
--- /dev/null
+++ b/drivers/gpu/drm/stm/dw_mipi_dsi-stm.c
@@ -0,0 +1,352 @@
+/*
+ * Copyright (C) STMicroelectronics SA 2017
+ *
+ * Authors: Philippe Cornu <philippe.cornu@st.com>
+ *          Yannick Fertre <yannick.fertre@st.com>
+ *
+ * License terms:  GNU General Public License (GPL), version 2
+ */
+
+#include <linux/clk.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <drm/drmP.h>
+#include <drm/drm_mipi_dsi.h>
+#include <drm/bridge/dw_mipi_dsi.h>
+#include <video/mipi_display.h>
+
+/* DSI wrapper register & bit definitions */
+/* Note: registers are named as in the Reference Manual */
+#define DSI_WCFGR	0x0400		/* Wrapper ConFiGuration Reg */
+#define WCFGR_DSIM	BIT(0)		/* DSI Mode */
+#define WCFGR_COLMUX	GENMASK(3, 1)	/* COLor MUltipleXing */
+
+#define DSI_WCR		0x0404		/* Wrapper Control Reg */
+#define WCR_DSIEN	BIT(3)		/* DSI ENable */
+
+#define DSI_WISR	0x040C		/* Wrapper Interrupt and Status Reg */
+#define WISR_PLLLS	BIT(8)		/* PLL Lock Status */
+#define WISR_RRS	BIT(12)		/* Regulator Ready Status */
+
+#define DSI_WPCR0	0x0418		/* Wrapper Phy Conf Reg 0 */
+#define WPCR0_UIX4	GENMASK(5, 0)	/* Unit Interval X 4 */
+#define WPCR0_TDDL	BIT(16)		/* Turn Disable Data Lanes */
+
+#define DSI_WRPCR	0x0430		/* Wrapper Regulator & Pll Ctrl Reg */
+#define WRPCR_PLLEN	BIT(0)		/* PLL ENable */
+#define WRPCR_NDIV	GENMASK(8, 2)	/* pll loop DIVision Factor */
+#define WRPCR_IDF	GENMASK(14, 11)	/* pll Input Division Factor */
+#define WRPCR_ODF	GENMASK(17, 16)	/* pll Output Division Factor */
+#define WRPCR_REGEN	BIT(24)		/* REGulator ENable */
+#define WRPCR_BGREN	BIT(28)		/* BandGap Reference ENable */
+#define IDF_MIN		1
+#define IDF_MAX		7
+#define NDIV_MIN	10
+#define NDIV_MAX	125
+#define ODF_MIN		1
+#define ODF_MAX		8
+
+/* dsi color format coding according to the datasheet */
+enum dsi_color {
+	DSI_RGB565_CONF1,
+	DSI_RGB565_CONF2,
+	DSI_RGB565_CONF3,
+	DSI_RGB666_CONF1,
+	DSI_RGB666_CONF2,
+	DSI_RGB888,
+};
+
+#define LANE_MIN_KBPS	31250
+#define LANE_MAX_KBPS	500000
+
+/* Sleep & timeout for regulator on/off, pll lock/unlock & fifo empty */
+#define SLEEP_US	1000
+#define TIMEOUT_US	200000
+
+struct dw_mipi_dsi_stm {
+	void __iomem *base;
+	struct clk *pllref_clk;
+};
+
+static inline void dsi_write(struct dw_mipi_dsi_stm *dsi, u32 reg, u32 val)
+{
+	writel(val, dsi->base + reg);
+}
+
+static inline u32 dsi_read(struct dw_mipi_dsi_stm *dsi, u32 reg)
+{
+	return readl(dsi->base + reg);
+}
+
+static inline void dsi_set(struct dw_mipi_dsi_stm *dsi, u32 reg, u32 mask)
+{
+	dsi_write(dsi, reg, dsi_read(dsi, reg) | mask);
+}
+
+static inline void dsi_clear(struct dw_mipi_dsi_stm *dsi, u32 reg, u32 mask)
+{
+	dsi_write(dsi, reg, dsi_read(dsi, reg) & ~mask);
+}
+
+static inline void dsi_update_bits(struct dw_mipi_dsi_stm *dsi, u32 reg,
+				   u32 mask, u32 val)
+{
+	dsi_write(dsi, reg, (dsi_read(dsi, reg) & ~mask) | val);
+}
+
+static enum dsi_color dsi_color_from_mipi(enum mipi_dsi_pixel_format fmt)
+{
+	switch (fmt) {
+	case MIPI_DSI_FMT_RGB888:
+		return DSI_RGB888;
+	case MIPI_DSI_FMT_RGB666:
+		return DSI_RGB666_CONF2;
+	case MIPI_DSI_FMT_RGB666_PACKED:
+		return DSI_RGB666_CONF1;
+	case MIPI_DSI_FMT_RGB565:
+		return DSI_RGB565_CONF1;
+	default:
+		DRM_DEBUG_DRIVER("MIPI color invalid, so we use rgb888\n");
+	}
+	return DSI_RGB888;
+}
+
+static int dsi_pll_get_clkout_khz(int clkin_khz, int idf, int ndiv, int odf)
+{
+	/* prevent from division by 0 */
+	if (idf * odf)
+		return DIV_ROUND_CLOSEST(clkin_khz * ndiv, idf * odf);
+
+	return 0;
+}
+
+static int dsi_pll_get_params(int clkin_khz, int clkout_khz,
+			      int *idf, int *ndiv, int *odf)
+{
+	int i, o, n, n_min, n_max;
+	int fvco_min, fvco_max, delta, best_delta; /* all in khz */
+
+	/* Early checks preventing division by 0 & odd results */
+	if ((clkin_khz <= 0) || (clkout_khz <= 0))
+		return -EINVAL;
+
+	fvco_min = LANE_MIN_KBPS * 2 * ODF_MAX;
+	fvco_max = LANE_MAX_KBPS * 2 * ODF_MIN;
+
+	best_delta = 1000000; /* big started value (1000000khz) */
+
+	for (i = IDF_MIN; i <= IDF_MAX; i++) {
+		/* Compute ndiv range according to Fvco */
+		n_min = ((fvco_min * i) / (2 * clkin_khz)) + 1;
+		n_max = (fvco_max * i) / (2 * clkin_khz);
+
+		/* No need to continue idf loop if we reach ndiv max */
+		if (n_min >= NDIV_MAX)
+			break;
+
+		/* Clamp ndiv to valid values */
+		if (n_min < NDIV_MIN)
+			n_min = NDIV_MIN;
+		if (n_max > NDIV_MAX)
+			n_max = NDIV_MAX;
+
+		for (o = ODF_MIN; o <= ODF_MAX; o *= 2) {
+			n = DIV_ROUND_CLOSEST(i * o * clkout_khz, clkin_khz);
+			/* Check ndiv according to vco range */
+			if ((n < n_min) || (n > n_max))
+				continue;
+			/* Check if new delta is better & saves parameters */
+			delta = dsi_pll_get_clkout_khz(clkin_khz, i, n, o) -
+				clkout_khz;
+			if (delta < 0)
+				delta = -delta;
+			if (delta < best_delta) {
+				*idf = i;
+				*ndiv = n;
+				*odf = o;
+				best_delta = delta;
+			}
+			/* fast return in case of "perfect result" */
+			if (!delta)
+				return 0;
+		}
+	}
+
+	return 0;
+}
+
+static int dw_mipi_dsi_phy_init(void *priv_data)
+{
+	struct dw_mipi_dsi_stm *dsi = priv_data;
+	u32 val;
+	int ret;
+
+	/* Enable the regulator */
+	dsi_set(dsi, DSI_WRPCR, WRPCR_REGEN | WRPCR_BGREN);
+	ret = readl_poll_timeout(dsi->base + DSI_WISR, val, val & WISR_RRS,
+				 SLEEP_US, TIMEOUT_US);
+	if (ret)
+		DRM_DEBUG_DRIVER("!TIMEOUT! waiting REGU, let's continue\n");
+
+	/* Enable the DSI PLL & wait for its lock */
+	dsi_set(dsi, DSI_WRPCR, WRPCR_PLLEN);
+	ret = readl_poll_timeout(dsi->base + DSI_WISR, val, val & WISR_PLLLS,
+				 SLEEP_US, TIMEOUT_US);
+	if (ret)
+		DRM_DEBUG_DRIVER("!TIMEOUT! waiting PLL, let's continue\n");
+
+	/* Enable the DSI wrapper */
+	dsi_set(dsi, DSI_WCR, WCR_DSIEN);
+
+	return 0;
+}
+
+static int
+dw_mipi_dsi_get_lane_mbps(void *priv_data, struct drm_display_mode *mode,
+			  unsigned long mode_flags, u32 lanes, u32 format,
+			  unsigned int *lane_mbps)
+{
+	struct dw_mipi_dsi_stm *dsi = priv_data;
+	unsigned int idf, ndiv, odf, pll_in_khz, pll_out_khz;
+	int ret, bpp;
+	u32 val;
+
+	pll_in_khz = (unsigned int)(clk_get_rate(dsi->pllref_clk) / 1000);
+
+	/* Compute requested pll out */
+	bpp = mipi_dsi_pixel_format_to_bpp(format);
+	pll_out_khz = mode->clock * bpp / lanes;
+	/* Add 20% to pll out to be higher than pixel bw (burst mode only) */
+	pll_out_khz = (pll_out_khz * 12) / 10;
+	if (pll_out_khz > LANE_MAX_KBPS) {
+		pll_out_khz = LANE_MAX_KBPS;
+		DRM_WARN("Warning max phy mbps is used\n");
+	}
+	if (pll_out_khz < LANE_MIN_KBPS) {
+		pll_out_khz = LANE_MIN_KBPS;
+		DRM_WARN("Warning min phy mbps is used\n");
+	}
+
+	/* Compute best pll parameters */
+	idf = 0;
+	ndiv = 0;
+	odf = 0;
+	ret = dsi_pll_get_params(pll_in_khz, pll_out_khz, &idf, &ndiv, &odf);
+	if (ret)
+		DRM_WARN("Warning dsi_pll_get_params(): bad params\n");
+
+	/* Get the adjusted pll out value */
+	pll_out_khz = dsi_pll_get_clkout_khz(pll_in_khz, idf, ndiv, odf);
+
+	/* Set the PLL division factors */
+	dsi_update_bits(dsi, DSI_WRPCR,	WRPCR_NDIV | WRPCR_IDF | WRPCR_ODF,
+			(ndiv << 2) | (idf << 11) | ((ffs(odf) - 1) << 16));
+
+	/* Compute uix4 & set the bit period in high-speed mode */
+	val = 4000000 / pll_out_khz;
+	dsi_update_bits(dsi, DSI_WPCR0, WPCR0_UIX4, val);
+
+	/* Select video mode by resetting DSIM bit */
+	dsi_clear(dsi, DSI_WCFGR, WCFGR_DSIM);
+
+	/* Select the color coding */
+	dsi_update_bits(dsi, DSI_WCFGR, WCFGR_COLMUX,
+			dsi_color_from_mipi(format) << 1);
+
+	*lane_mbps = pll_out_khz / 1000;
+
+	DRM_DEBUG_DRIVER("pll_in %ukHz pll_out %ukHz lane_mbps %uMHz\n",
+			 pll_in_khz, pll_out_khz, *lane_mbps);
+
+	return 0;
+}
+
+static const struct dw_mipi_dsi_phy_ops dw_mipi_dsi_stm_phy_ops = {
+	.init = dw_mipi_dsi_phy_init,
+	.get_lane_mbps = dw_mipi_dsi_get_lane_mbps,
+};
+
+static struct dw_mipi_dsi_plat_data dw_mipi_dsi_stm_plat_data = {
+	.max_data_lanes = 2,
+	.phy_ops = &dw_mipi_dsi_stm_phy_ops,
+};
+
+static const struct of_device_id dw_mipi_dsi_stm_dt_ids[] = {
+	{ .compatible = "st,stm32-dsi", .data = &dw_mipi_dsi_stm_plat_data, },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, dw_mipi_dsi_stm_dt_ids);
+
+static int dw_mipi_dsi_stm_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct dw_mipi_dsi_stm *dsi;
+	struct resource *res;
+	int ret;
+
+	dsi = devm_kzalloc(dev, sizeof(*dsi), GFP_KERNEL);
+	if (!dsi)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res) {
+		DRM_ERROR("Unable to get resource\n");
+		return -ENODEV;
+	}
+
+	dsi->base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(dsi->base)) {
+		DRM_ERROR("Unable to get dsi registers\n");
+		return PTR_ERR(dsi->base);
+	}
+
+	dsi->pllref_clk = devm_clk_get(dev, "ref");
+	if (IS_ERR(dsi->pllref_clk)) {
+		ret = PTR_ERR(dsi->pllref_clk);
+		dev_err(dev, "Unable to get pll reference clock: %d\n", ret);
+		return ret;
+	}
+
+	ret = clk_prepare_enable(dsi->pllref_clk);
+	if (ret) {
+		dev_err(dev, "%s: Failed to enable pllref_clk\n", __func__);
+		return ret;
+	}
+
+	dw_mipi_dsi_stm_plat_data.base = dsi->base;
+	dw_mipi_dsi_stm_plat_data.priv_data = dsi;
+
+	ret = dw_mipi_dsi_probe(pdev, &dw_mipi_dsi_stm_plat_data);
+	if (ret) {
+		DRM_ERROR("Failed to initialize mipi dsi host\n");
+		clk_disable_unprepare(dsi->pllref_clk);
+	}
+
+	return ret;
+}
+
+static int dw_mipi_dsi_stm_remove(struct platform_device *pdev)
+{
+	struct dw_mipi_dsi_stm *dsi = dw_mipi_dsi_stm_plat_data.priv_data;
+
+	clk_disable_unprepare(dsi->pllref_clk);
+	dw_mipi_dsi_remove(pdev);
+
+	return 0;
+}
+
+static struct platform_driver dw_mipi_dsi_stm_driver = {
+	.probe		= dw_mipi_dsi_stm_probe,
+	.remove		= dw_mipi_dsi_stm_remove,
+	.driver		= {
+		.of_match_table = dw_mipi_dsi_stm_dt_ids,
+		.name	= "dw_mipi_dsi-stm",
+	},
+};
+
+module_platform_driver(dw_mipi_dsi_stm_driver);
+
+MODULE_AUTHOR("Philippe Cornu <philippe.cornu@st.com>");
+MODULE_AUTHOR("Yannick Fertre <yannick.fertre@st.com>");
+MODULE_DESCRIPTION("STMicroelectronics DW MIPI DSI host controller driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/stm/ltdc.c b/drivers/gpu/drm/stm/ltdc.c
index 1b9483d4f2a4..d394a03632c4 100644
--- a/drivers/gpu/drm/stm/ltdc.c
+++ b/drivers/gpu/drm/stm/ltdc.c
@@ -21,7 +21,7 @@
 #include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_of.h>
-#include <drm/drm_panel.h>
+#include <drm/drm_bridge.h>
 #include <drm/drm_plane_helper.h>
 
 #include <video/videomode.h>
@@ -42,52 +42,52 @@
  * an extra offset specified with reg_ofs.
  */
 #define REG_OFS_NONE	0
-#define REG_OFS_4	4 /* Insertion of "Layer Configuration 2" reg */
+#define REG_OFS_4	4		/* Insertion of "Layer Conf. 2" reg */
 #define REG_OFS		(ldev->caps.reg_ofs)
-#define LAY_OFS		0x80	/* Register Offset between 2 layers */
+#define LAY_OFS		0x80		/* Register Offset between 2 layers */
 
 /* Global register offsets */
-#define LTDC_IDR	0x0000 /* IDentification */
-#define LTDC_LCR	0x0004 /* Layer Count */
-#define LTDC_SSCR	0x0008 /* Synchronization Size Configuration */
-#define LTDC_BPCR	0x000C /* Back Porch Configuration */
-#define LTDC_AWCR	0x0010 /* Active Width Configuration */
-#define LTDC_TWCR	0x0014 /* Total Width Configuration */
-#define LTDC_GCR	0x0018 /* Global Control */
-#define LTDC_GC1R	0x001C /* Global Configuration 1 */
-#define LTDC_GC2R	0x0020 /* Global Configuration 2 */
-#define LTDC_SRCR	0x0024 /* Shadow Reload Configuration */
-#define LTDC_GACR	0x0028 /* GAmma Correction */
-#define LTDC_BCCR	0x002C /* Background Color Configuration */
-#define LTDC_IER	0x0034 /* Interrupt Enable */
-#define LTDC_ISR	0x0038 /* Interrupt Status */
-#define LTDC_ICR	0x003C /* Interrupt Clear */
-#define LTDC_LIPCR	0x0040 /* Line Interrupt Position Configuration */
-#define LTDC_CPSR	0x0044 /* Current Position Status */
-#define LTDC_CDSR	0x0048 /* Current Display Status */
+#define LTDC_IDR	0x0000		/* IDentification */
+#define LTDC_LCR	0x0004		/* Layer Count */
+#define LTDC_SSCR	0x0008		/* Synchronization Size Configuration */
+#define LTDC_BPCR	0x000C		/* Back Porch Configuration */
+#define LTDC_AWCR	0x0010		/* Active Width Configuration */
+#define LTDC_TWCR	0x0014		/* Total Width Configuration */
+#define LTDC_GCR	0x0018		/* Global Control */
+#define LTDC_GC1R	0x001C		/* Global Configuration 1 */
+#define LTDC_GC2R	0x0020		/* Global Configuration 2 */
+#define LTDC_SRCR	0x0024		/* Shadow Reload Configuration */
+#define LTDC_GACR	0x0028		/* GAmma Correction */
+#define LTDC_BCCR	0x002C		/* Background Color Configuration */
+#define LTDC_IER	0x0034		/* Interrupt Enable */
+#define LTDC_ISR	0x0038		/* Interrupt Status */
+#define LTDC_ICR	0x003C		/* Interrupt Clear */
+#define LTDC_LIPCR	0x0040		/* Line Interrupt Position Conf. */
+#define LTDC_CPSR	0x0044		/* Current Position Status */
+#define LTDC_CDSR	0x0048		/* Current Display Status */
 
 /* Layer register offsets */
-#define LTDC_L1LC1R	(0x0080)	   /* L1 Layer Configuration 1 */
-#define LTDC_L1LC2R	(0x0084)	   /* L1 Layer Configuration 2 */
-#define LTDC_L1CR	(0x0084 + REG_OFS) /* L1 Control */
-#define LTDC_L1WHPCR	(0x0088 + REG_OFS) /* L1 Window Hor Position Config */
-#define LTDC_L1WVPCR	(0x008C + REG_OFS) /* L1 Window Vert Position Config */
-#define LTDC_L1CKCR	(0x0090 + REG_OFS) /* L1 Color Keying Configuration */
-#define LTDC_L1PFCR	(0x0094 + REG_OFS) /* L1 Pixel Format Configuration */
-#define LTDC_L1CACR	(0x0098 + REG_OFS) /* L1 Constant Alpha Config */
-#define LTDC_L1DCCR	(0x009C + REG_OFS) /* L1 Default Color Configuration */
-#define LTDC_L1BFCR	(0x00A0 + REG_OFS) /* L1 Blend Factors Configuration */
-#define LTDC_L1FBBCR	(0x00A4 + REG_OFS) /* L1 FrameBuffer Bus Control */
-#define LTDC_L1AFBCR	(0x00A8 + REG_OFS) /* L1 AuxFB Control */
-#define LTDC_L1CFBAR	(0x00AC + REG_OFS) /* L1 Color FrameBuffer Address */
-#define LTDC_L1CFBLR	(0x00B0 + REG_OFS) /* L1 Color FrameBuffer Length */
-#define LTDC_L1CFBLNR	(0x00B4 + REG_OFS) /* L1 Color FrameBuffer Line Nb */
-#define LTDC_L1AFBAR	(0x00B8 + REG_OFS) /* L1 AuxFB Address */
-#define LTDC_L1AFBLR	(0x00BC + REG_OFS) /* L1 AuxFB Length */
-#define LTDC_L1AFBLNR	(0x00C0 + REG_OFS) /* L1 AuxFB Line Number */
-#define LTDC_L1CLUTWR	(0x00C4 + REG_OFS) /* L1 CLUT Write */
-#define LTDC_L1YS1R	(0x00E0 + REG_OFS) /* L1 YCbCr Scale 1 */
-#define LTDC_L1YS2R	(0x00E4 + REG_OFS) /* L1 YCbCr Scale 2 */
+#define LTDC_L1LC1R	(0x80)		/* L1 Layer Configuration 1 */
+#define LTDC_L1LC2R	(0x84)		/* L1 Layer Configuration 2 */
+#define LTDC_L1CR	(0x84 + REG_OFS)/* L1 Control */
+#define LTDC_L1WHPCR	(0x88 + REG_OFS)/* L1 Window Hor Position Config */
+#define LTDC_L1WVPCR	(0x8C + REG_OFS)/* L1 Window Vert Position Config */
+#define LTDC_L1CKCR	(0x90 + REG_OFS)/* L1 Color Keying Configuration */
+#define LTDC_L1PFCR	(0x94 + REG_OFS)/* L1 Pixel Format Configuration */
+#define LTDC_L1CACR	(0x98 + REG_OFS)/* L1 Constant Alpha Config */
+#define LTDC_L1DCCR	(0x9C + REG_OFS)/* L1 Default Color Configuration */
+#define LTDC_L1BFCR	(0xA0 + REG_OFS)/* L1 Blend Factors Configuration */
+#define LTDC_L1FBBCR	(0xA4 + REG_OFS)/* L1 FrameBuffer Bus Control */
+#define LTDC_L1AFBCR	(0xA8 + REG_OFS)/* L1 AuxFB Control */
+#define LTDC_L1CFBAR	(0xAC + REG_OFS)/* L1 Color FrameBuffer Address */
+#define LTDC_L1CFBLR	(0xB0 + REG_OFS)/* L1 Color FrameBuffer Length */
+#define LTDC_L1CFBLNR	(0xB4 + REG_OFS)/* L1 Color FrameBuffer Line Nb */
+#define LTDC_L1AFBAR	(0xB8 + REG_OFS)/* L1 AuxFB Address */
+#define LTDC_L1AFBLR	(0xBC + REG_OFS)/* L1 AuxFB Length */
+#define LTDC_L1AFBLNR	(0xC0 + REG_OFS)/* L1 AuxFB Line Number */
+#define LTDC_L1CLUTWR	(0xC4 + REG_OFS)/* L1 CLUT Write */
+#define LTDC_L1YS1R	(0xE0 + REG_OFS)/* L1 YCbCr Scale 1 */
+#define LTDC_L1YS2R	(0xE4 + REG_OFS)/* L1 YCbCr Scale 2 */
 
 /* Bit definitions */
 #define SSCR_VSH	GENMASK(10, 0)	/* Vertical Synchronization Height */
@@ -104,10 +104,10 @@
 
 #define GCR_LTDCEN	BIT(0)		/* LTDC ENable */
 #define GCR_DEN		BIT(16)		/* Dither ENable */
-#define GCR_PCPOL	BIT(28)		/* Pixel Clock POLarity */
-#define GCR_DEPOL	BIT(29)		/* Data Enable POLarity */
-#define GCR_VSPOL	BIT(30)		/* Vertical Synchro POLarity */
-#define GCR_HSPOL	BIT(31)		/* Horizontal Synchro POLarity */
+#define GCR_PCPOL	BIT(28)		/* Pixel Clock POLarity-Inverted */
+#define GCR_DEPOL	BIT(29)		/* Data Enable POLarity-High */
+#define GCR_VSPOL	BIT(30)		/* Vertical Synchro POLarity-High */
+#define GCR_HSPOL	BIT(31)		/* Horizontal Synchro POLarity-High */
 
 #define GC1R_WBCH	GENMASK(3, 0)	/* Width of Blue CHannel output */
 #define GC1R_WGCH	GENMASK(7, 4)	/* Width of Green Channel output */
@@ -172,60 +172,52 @@
 #define LXCFBLR_CFBLL	GENMASK(12, 0)	/* Color Frame Buffer Line Length */
 #define LXCFBLR_CFBP	GENMASK(28, 16)	/* Color Frame Buffer Pitch in bytes */
 
-#define LXCFBLNR_CFBLN	GENMASK(10, 0)	 /* Color Frame Buffer Line Number */
+#define LXCFBLNR_CFBLN	GENMASK(10, 0)	/* Color Frame Buffer Line Number */
 
-#define HSPOL_AL   0		/* Horizontal Sync POLarity Active Low */
-#define VSPOL_AL   0		/* Vertical Sync POLarity Active Low */
-#define DEPOL_AL   0		/* Data Enable POLarity Active Low */
-#define PCPOL_IPC  0		/* Input Pixel Clock */
-#define HSPOL_AH   GCR_HSPOL	/* Horizontal Sync POLarity Active High */
-#define VSPOL_AH   GCR_VSPOL	/* Vertical Sync POLarity Active High */
-#define DEPOL_AH   GCR_DEPOL	/* Data Enable POLarity Active High */
-#define PCPOL_IIPC GCR_PCPOL	/* Inverted Input Pixel Clock */
-#define CONSTA_MAX 0xFF		/* CONSTant Alpha MAX= 1.0 */
-#define BF1_PAXCA  0x600	/* Pixel Alpha x Constant Alpha */
-#define BF1_CA     0x400	/* Constant Alpha */
-#define BF2_1PAXCA 0x007	/* 1 - (Pixel Alpha x Constant Alpha) */
-#define BF2_1CA	   0x005	/* 1 - Constant Alpha */
+#define CONSTA_MAX	0xFF		/* CONSTant Alpha MAX= 1.0 */
+#define BF1_PAXCA	0x600		/* Pixel Alpha x Constant Alpha */
+#define BF1_CA		0x400		/* Constant Alpha */
+#define BF2_1PAXCA	0x007		/* 1 - (Pixel Alpha x Constant Alpha) */
+#define BF2_1CA		0x005		/* 1 - Constant Alpha */
 
-#define NB_PF           8       /* Max nb of HW pixel format */
+#define NB_PF		8		/* Max nb of HW pixel format */
 
 enum ltdc_pix_fmt {
 	PF_NONE,
 	/* RGB formats */
-	PF_ARGB8888,    /* ARGB [32 bits] */
-	PF_RGBA8888,    /* RGBA [32 bits] */
-	PF_RGB888,      /* RGB [24 bits] */
-	PF_RGB565,      /* RGB [16 bits] */
-	PF_ARGB1555,    /* ARGB A:1 bit RGB:15 bits [16 bits] */
-	PF_ARGB4444,    /* ARGB A:4 bits R/G/B: 4 bits each [16 bits] */
+	PF_ARGB8888,		/* ARGB [32 bits] */
+	PF_RGBA8888,		/* RGBA [32 bits] */
+	PF_RGB888,		/* RGB [24 bits] */
+	PF_RGB565,		/* RGB [16 bits] */
+	PF_ARGB1555,		/* ARGB A:1 bit RGB:15 bits [16 bits] */
+	PF_ARGB4444,		/* ARGB A:4 bits R/G/B: 4 bits each [16 bits] */
 	/* Indexed formats */
-	PF_L8,          /* Indexed 8 bits [8 bits] */
-	PF_AL44,        /* Alpha:4 bits + indexed 4 bits [8 bits] */
-	PF_AL88         /* Alpha:8 bits + indexed 8 bits [16 bits] */
+	PF_L8,			/* Indexed 8 bits [8 bits] */
+	PF_AL44,		/* Alpha:4 bits + indexed 4 bits [8 bits] */
+	PF_AL88			/* Alpha:8 bits + indexed 8 bits [16 bits] */
 };
 
 /* The index gives the encoding of the pixel format for an HW version */
 static const enum ltdc_pix_fmt ltdc_pix_fmt_a0[NB_PF] = {
-	PF_ARGB8888,	/* 0x00 */
-	PF_RGB888,	/* 0x01 */
-	PF_RGB565,	/* 0x02 */
-	PF_ARGB1555,	/* 0x03 */
-	PF_ARGB4444,	/* 0x04 */
-	PF_L8,		/* 0x05 */
-	PF_AL44,	/* 0x06 */
-	PF_AL88		/* 0x07 */
+	PF_ARGB8888,		/* 0x00 */
+	PF_RGB888,		/* 0x01 */
+	PF_RGB565,		/* 0x02 */
+	PF_ARGB1555,		/* 0x03 */
+	PF_ARGB4444,		/* 0x04 */
+	PF_L8,			/* 0x05 */
+	PF_AL44,		/* 0x06 */
+	PF_AL88			/* 0x07 */
 };
 
 static const enum ltdc_pix_fmt ltdc_pix_fmt_a1[NB_PF] = {
-	PF_ARGB8888,	/* 0x00 */
-	PF_RGB888,	/* 0x01 */
-	PF_RGB565,	/* 0x02 */
-	PF_RGBA8888,	/* 0x03 */
-	PF_AL44,	/* 0x04 */
-	PF_L8,		/* 0x05 */
-	PF_ARGB1555,	/* 0x06 */
-	PF_ARGB4444	/* 0x07 */
+	PF_ARGB8888,		/* 0x00 */
+	PF_RGB888,		/* 0x01 */
+	PF_RGB565,		/* 0x02 */
+	PF_RGBA8888,		/* 0x03 */
+	PF_AL44,		/* 0x04 */
+	PF_L8,			/* 0x05 */
+	PF_ARGB1555,		/* 0x06 */
+	PF_ARGB4444		/* 0x07 */
 };
 
 static inline u32 reg_read(void __iomem *base, u32 reg)
@@ -269,11 +261,6 @@ static inline struct ltdc_device *encoder_to_ltdc(struct drm_encoder *enc)
 	return (struct ltdc_device *)enc->dev->dev_private;
 }
 
-static inline struct ltdc_device *connector_to_ltdc(struct drm_connector *con)
-{
-	return (struct ltdc_device *)con->dev->dev_private;
-}
-
 static inline enum ltdc_pix_fmt to_ltdc_pixelformat(u32 drm_fmt)
 {
 	enum ltdc_pix_fmt pf;
@@ -307,7 +294,7 @@ static inline enum ltdc_pix_fmt to_ltdc_pixelformat(u32 drm_fmt)
 	default:
 		pf = PF_NONE;
 		break;
-	/* Note: There are no DRM_FORMAT for AL44 and AL88 */
+		/* Note: There are no DRM_FORMAT for AL44 and AL88 */
 	}
 
 	return pf;
@@ -330,8 +317,8 @@ static inline u32 to_drm_pixelformat(enum ltdc_pix_fmt pf)
 		return DRM_FORMAT_ARGB4444;
 	case PF_L8:
 		return DRM_FORMAT_C8;
-	case PF_AL44: /* No DRM support */
-	case PF_AL88: /* No DRM support */
+	case PF_AL44:		/* No DRM support */
+	case PF_AL88:		/* No DRM support */
 	case PF_NONE:
 	default:
 		return 0;
@@ -375,18 +362,8 @@ static irqreturn_t ltdc_irq(int irq, void *arg)
  * DRM_CRTC
  */
 
-static void ltdc_crtc_load_lut(struct drm_crtc *crtc)
-{
-	struct ltdc_device *ldev = crtc_to_ltdc(crtc);
-	unsigned int i, lay;
-
-	for (lay = 0; lay < ldev->caps.nb_layers; lay++)
-		for (i = 0; i < 256; i++)
-			reg_write(ldev->regs, LTDC_L1CLUTWR + lay * LAY_OFS,
-				  ldev->clut[i]);
-}
-
-static void ltdc_crtc_enable(struct drm_crtc *crtc)
+static void ltdc_crtc_atomic_enable(struct drm_crtc *crtc,
+				    struct drm_crtc_state *old_state)
 {
 	struct ltdc_device *ldev = crtc_to_ltdc(crtc);
 
@@ -407,7 +384,8 @@ static void ltdc_crtc_enable(struct drm_crtc *crtc)
 	drm_crtc_vblank_on(crtc);
 }
 
-static void ltdc_crtc_disable(struct drm_crtc *crtc)
+static void ltdc_crtc_atomic_disable(struct drm_crtc *crtc,
+				     struct drm_crtc_state *old_state)
 {
 	struct ltdc_device *ldev = crtc_to_ltdc(crtc);
 
@@ -462,20 +440,20 @@ static void ltdc_crtc_mode_set_nofb(struct drm_crtc *crtc)
 
 	clk_enable(ldev->pixel_clk);
 
-	/* Configures the HS, VS, DE and PC polarities. */
-	val = HSPOL_AL | VSPOL_AL | DEPOL_AL | PCPOL_IPC;
+	/* Configures the HS, VS, DE and PC polarities. Default Active Low */
+	val = 0;
 
 	if (vm.flags & DISPLAY_FLAGS_HSYNC_HIGH)
-		val |= HSPOL_AH;
+		val |= GCR_HSPOL;
 
 	if (vm.flags & DISPLAY_FLAGS_VSYNC_HIGH)
-		val |= VSPOL_AH;
+		val |= GCR_VSPOL;
 
 	if (vm.flags & DISPLAY_FLAGS_DE_HIGH)
-		val |= DEPOL_AH;
+		val |= GCR_DEPOL;
 
 	if (vm.flags & DISPLAY_FLAGS_PIXDATA_NEGEDGE)
-		val |= PCPOL_IIPC;
+		val |= GCR_PCPOL;
 
 	reg_update_bits(ldev->regs, LTDC_GCR,
 			GCR_HSPOL | GCR_VSPOL | GCR_DEPOL | GCR_PCPOL, val);
@@ -522,12 +500,11 @@ static void ltdc_crtc_atomic_flush(struct drm_crtc *crtc,
 	}
 }
 
-static struct drm_crtc_helper_funcs ltdc_crtc_helper_funcs = {
-	.load_lut = ltdc_crtc_load_lut,
-	.enable = ltdc_crtc_enable,
-	.disable = ltdc_crtc_disable,
+static const struct drm_crtc_helper_funcs ltdc_crtc_helper_funcs = {
 	.mode_set_nofb = ltdc_crtc_mode_set_nofb,
 	.atomic_flush = ltdc_crtc_atomic_flush,
+	.atomic_enable = ltdc_crtc_atomic_enable,
+	.atomic_disable = ltdc_crtc_atomic_disable,
 };
 
 int ltdc_crtc_enable_vblank(struct drm_device *ddev, unsigned int pipe)
@@ -548,7 +525,7 @@ void ltdc_crtc_disable_vblank(struct drm_device *ddev, unsigned int pipe)
 	reg_clear(ldev->regs, LTDC_IER, IER_LIE);
 }
 
-static struct drm_crtc_funcs ltdc_crtc_funcs = {
+static const struct drm_crtc_funcs ltdc_crtc_funcs = {
 	.destroy = drm_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
@@ -613,11 +590,11 @@ static void ltdc_plane_atomic_update(struct drm_plane *plane,
 	src_w = state->src_w >> 16;
 	src_h = state->src_h >> 16;
 
-	DRM_DEBUG_DRIVER(
-		"plane:%d fb:%d (%dx%d)@(%d,%d) -> (%dx%d)@(%d,%d)\n",
-		plane->base.id, fb->base.id,
-		src_w, src_h, src_x, src_y,
-		state->crtc_w, state->crtc_h, state->crtc_x, state->crtc_y);
+	DRM_DEBUG_DRIVER("plane:%d fb:%d (%dx%d)@(%d,%d) -> (%dx%d)@(%d,%d)\n",
+			 plane->base.id, fb->base.id,
+			 src_w, src_h, src_x, src_y,
+			 state->crtc_w, state->crtc_h,
+			 state->crtc_x, state->crtc_y);
 
 	bpcr = reg_read(ldev->regs, LTDC_BPCR);
 	ahbp = (bpcr & BPCR_AHBP) >> 16;
@@ -642,7 +619,7 @@ static void ltdc_plane_atomic_update(struct drm_plane *plane,
 	if (val == NB_PF) {
 		DRM_ERROR("Pixel format %.4s not supported\n",
 			  (char *)&fb->format->format);
-		val = 0; /* set by default ARGB 32 bits */
+		val = 0;	/* set by default ARGB 32 bits */
 	}
 	reg_update_bits(ldev->regs, LTDC_L1PFCR + lofs, LXPFCR_PF, val);
 
@@ -656,8 +633,7 @@ static void ltdc_plane_atomic_update(struct drm_plane *plane,
 
 	/* Specifies the constant alpha value */
 	val = CONSTA_MAX;
-	reg_update_bits(ldev->regs, LTDC_L1CACR + lofs,
-			LXCACR_CONSTA, val);
+	reg_update_bits(ldev->regs, LTDC_L1CACR + lofs, LXCACR_CONSTA, val);
 
 	/* Specifies the blending factors */
 	val = BF1_PAXCA | BF2_1PAXCA;
@@ -666,8 +642,7 @@ static void ltdc_plane_atomic_update(struct drm_plane *plane,
 
 	/* Configures the frame buffer line number */
 	val = y1 - y0 + 1;
-	reg_update_bits(ldev->regs, LTDC_L1CFBLNR + lofs,
-			LXCFBLNR_CFBLN, val);
+	reg_update_bits(ldev->regs, LTDC_L1CFBLNR + lofs, LXCFBLNR_CFBLN, val);
 
 	/* Sets the FB address */
 	paddr = (u32)drm_fb_cma_get_gem_addr(fb, state, 0);
@@ -706,11 +681,10 @@ static void ltdc_plane_atomic_disable(struct drm_plane *plane,
 			 oldstate->crtc->base.id, plane->base.id);
 }
 
-static struct drm_plane_funcs ltdc_plane_funcs = {
+static const struct drm_plane_funcs ltdc_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = drm_plane_cleanup,
-	.set_property = drm_atomic_helper_plane_set_property,
 	.reset = drm_atomic_helper_plane_reset,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
@@ -748,7 +722,7 @@ static struct drm_plane *ltdc_plane_create(struct drm_device *ddev,
 
 	ret = drm_universal_plane_init(ddev, plane, possible_crtcs,
 				       &ltdc_plane_funcs, formats, nb_fmt,
-				       type, NULL);
+				       NULL, type, NULL);
 	if (ret < 0)
 		return 0;
 
@@ -773,7 +747,7 @@ static int ltdc_crtc_init(struct drm_device *ddev, struct drm_crtc *crtc)
 	struct ltdc_device *ldev = ddev->dev_private;
 	struct drm_plane *primary, *overlay;
 	unsigned int i;
-	int res;
+	int ret;
 
 	primary = ltdc_plane_create(ddev, DRM_PLANE_TYPE_PRIMARY);
 	if (!primary) {
@@ -781,9 +755,9 @@ static int ltdc_crtc_init(struct drm_device *ddev, struct drm_crtc *crtc)
 		return -EINVAL;
 	}
 
-	res = drm_crtc_init_with_planes(ddev, crtc, primary, NULL,
+	ret = drm_crtc_init_with_planes(ddev, crtc, primary, NULL,
 					&ltdc_crtc_funcs, NULL);
-	if (res) {
+	if (ret) {
 		DRM_ERROR("Can not initialize CRTC\n");
 		goto cleanup;
 	}
@@ -796,7 +770,7 @@ static int ltdc_crtc_init(struct drm_device *ddev, struct drm_crtc *crtc)
 	for (i = 1; i < ldev->caps.nb_layers; i++) {
 		overlay = ltdc_plane_create(ddev, DRM_PLANE_TYPE_OVERLAY);
 		if (!overlay) {
-			res = -ENOMEM;
+			ret = -ENOMEM;
 			DRM_ERROR("Can not create overlay plane %d\n", i);
 			goto cleanup;
 		}
@@ -806,137 +780,42 @@ static int ltdc_crtc_init(struct drm_device *ddev, struct drm_crtc *crtc)
 
 cleanup:
 	ltdc_plane_destroy_all(ddev);
-	return res;
+	return ret;
 }
 
 /*
  * DRM_ENCODER
  */
 
-static void ltdc_rgb_encoder_enable(struct drm_encoder *encoder)
-{
-	struct ltdc_device *ldev = encoder_to_ltdc(encoder);
-
-	DRM_DEBUG_DRIVER("\n");
-
-	drm_panel_prepare(ldev->panel);
-	drm_panel_enable(ldev->panel);
-}
-
-static void ltdc_rgb_encoder_disable(struct drm_encoder *encoder)
-{
-	struct ltdc_device *ldev = encoder_to_ltdc(encoder);
-
-	DRM_DEBUG_DRIVER("\n");
-
-	drm_panel_disable(ldev->panel);
-	drm_panel_unprepare(ldev->panel);
-}
-
-static const struct drm_encoder_helper_funcs ltdc_rgb_encoder_helper_funcs = {
-	.enable = ltdc_rgb_encoder_enable,
-	.disable = ltdc_rgb_encoder_disable,
-};
-
-static const struct drm_encoder_funcs ltdc_rgb_encoder_funcs = {
+static const struct drm_encoder_funcs ltdc_encoder_funcs = {
 	.destroy = drm_encoder_cleanup,
 };
 
-static struct drm_encoder *ltdc_rgb_encoder_create(struct drm_device *ddev)
+static int ltdc_encoder_init(struct drm_device *ddev)
 {
+	struct ltdc_device *ldev = ddev->dev_private;
 	struct drm_encoder *encoder;
+	int ret;
 
 	encoder = devm_kzalloc(ddev->dev, sizeof(*encoder), GFP_KERNEL);
 	if (!encoder)
-		return NULL;
+		return -ENOMEM;
 
 	encoder->possible_crtcs = CRTC_MASK;
-	encoder->possible_clones = 0; /* No cloning support */
+	encoder->possible_clones = 0;	/* No cloning support */
 
-	drm_encoder_init(ddev, encoder, &ltdc_rgb_encoder_funcs,
+	drm_encoder_init(ddev, encoder, &ltdc_encoder_funcs,
 			 DRM_MODE_ENCODER_DPI, NULL);
 
-	drm_encoder_helper_add(encoder, &ltdc_rgb_encoder_helper_funcs);
-
-	DRM_DEBUG_DRIVER("RGB encoder:%d created\n", encoder->base.id);
-
-	return encoder;
-}
-
-/*
- * DRM_CONNECTOR
- */
-
-static int ltdc_rgb_connector_get_modes(struct drm_connector *connector)
-{
-	struct drm_device *ddev = connector->dev;
-	struct ltdc_device *ldev = ddev->dev_private;
-	int ret = 0;
-
-	DRM_DEBUG_DRIVER("\n");
-
-	if (ldev->panel)
-		ret = drm_panel_get_modes(ldev->panel);
-
-	return ret < 0 ? 0 : ret;
-}
-
-static struct drm_connector_helper_funcs ltdc_rgb_connector_helper_funcs = {
-	.get_modes = ltdc_rgb_connector_get_modes,
-};
-
-static enum drm_connector_status
-ltdc_rgb_connector_detect(struct drm_connector *connector, bool force)
-{
-	struct ltdc_device *ldev = connector_to_ltdc(connector);
-
-	return ldev->panel ? connector_status_connected :
-	       connector_status_disconnected;
-}
-
-static void ltdc_rgb_connector_destroy(struct drm_connector *connector)
-{
-	DRM_DEBUG_DRIVER("\n");
-
-	drm_connector_unregister(connector);
-	drm_connector_cleanup(connector);
-}
-
-static const struct drm_connector_funcs ltdc_rgb_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
-	.fill_modes = drm_helper_probe_single_connector_modes,
-	.detect = ltdc_rgb_connector_detect,
-	.destroy = ltdc_rgb_connector_destroy,
-	.reset = drm_atomic_helper_connector_reset,
-	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
-	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
-};
-
-struct drm_connector *ltdc_rgb_connector_create(struct drm_device *ddev)
-{
-	struct drm_connector *connector;
-	int err;
-
-	connector = devm_kzalloc(ddev->dev, sizeof(*connector), GFP_KERNEL);
-	if (!connector) {
-		DRM_ERROR("Failed to allocate connector\n");
-		return NULL;
-	}
-
-	connector->polled = DRM_CONNECTOR_POLL_HPD;
-
-	err = drm_connector_init(ddev, connector, &ltdc_rgb_connector_funcs,
-				 DRM_MODE_CONNECTOR_DPI);
-	if (err) {
-		DRM_ERROR("Failed to initialize connector\n");
-		return NULL;
+	ret = drm_bridge_attach(encoder, ldev->bridge, NULL);
+	if (ret) {
+		drm_encoder_cleanup(encoder);
+		return -EINVAL;
 	}
 
-	drm_connector_helper_add(connector, &ltdc_rgb_connector_helper_funcs);
-
-	DRM_DEBUG_DRIVER("RGB connector:%d created\n", connector->base.id);
+	DRM_DEBUG_DRIVER("Bridge encoder:%d created\n", encoder->base.id);
 
-	return connector;
+	return 0;
 }
 
 static int ltdc_get_caps(struct drm_device *ddev)
@@ -972,61 +851,26 @@ static int ltdc_get_caps(struct drm_device *ddev)
 	return 0;
 }
 
-static struct drm_panel *ltdc_get_panel(struct drm_device *ddev)
-{
-	struct device *dev = ddev->dev;
-	struct device_node *np = dev->of_node;
-	struct device_node *entity, *port = NULL;
-	struct drm_panel *panel = NULL;
-
-	DRM_DEBUG_DRIVER("\n");
-
-	/*
-	 * Parse ltdc node to get remote port and find RGB panel / HDMI slave
-	 * If a dsi or a bridge (hdmi, lvds...) is connected to ltdc,
-	 * a remote port & RGB panel will not be found.
-	 */
-	for_each_endpoint_of_node(np, entity) {
-		if (!of_device_is_available(entity))
-			continue;
-
-		port = of_graph_get_remote_port_parent(entity);
-		if (port) {
-			panel = of_drm_find_panel(port);
-			of_node_put(port);
-			if (panel) {
-				DRM_DEBUG_DRIVER("remote panel %s\n",
-						 port->full_name);
-			} else {
-				DRM_DEBUG_DRIVER("panel missing\n");
-				of_node_put(entity);
-			}
-		}
-	}
-
-	return panel;
-}
-
 int ltdc_load(struct drm_device *ddev)
 {
 	struct platform_device *pdev = to_platform_device(ddev->dev);
 	struct ltdc_device *ldev = ddev->dev_private;
 	struct device *dev = ddev->dev;
 	struct device_node *np = dev->of_node;
-	struct drm_encoder *encoder;
-	struct drm_connector *connector = NULL;
+	struct drm_bridge *bridge;
+	struct drm_panel *panel;
 	struct drm_crtc *crtc;
 	struct reset_control *rstc;
-	struct resource res;
+	struct resource *res;
 	int irq, ret, i;
 
 	DRM_DEBUG_DRIVER("\n");
 
-	ldev->panel = ltdc_get_panel(ddev);
-	if (!ldev->panel)
-		return -EPROBE_DEFER;
+	ret = drm_of_find_panel_or_bridge(np, 0, 0, &panel, &bridge);
+	if (ret)
+		return ret;
 
-	rstc = of_reset_control_get(np, NULL);
+	rstc = devm_reset_control_get_exclusive(dev, NULL);
 
 	mutex_init(&ldev->err_lock);
 
@@ -1041,15 +885,18 @@ int ltdc_load(struct drm_device *ddev)
 		return -ENODEV;
 	}
 
-	if (of_address_to_resource(np, 0, &res)) {
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res) {
 		DRM_ERROR("Unable to get resource\n");
-		return -ENODEV;
+		ret = -ENODEV;
+		goto err;
 	}
 
-	ldev->regs = devm_ioremap_resource(dev, &res);
+	ldev->regs = devm_ioremap_resource(dev, res);
 	if (IS_ERR(ldev->regs)) {
 		DRM_ERROR("Unable to get ltdc registers\n");
-		return PTR_ERR(ldev->regs);
+		ret = PTR_ERR(ldev->regs);
+		goto err;
 	}
 
 	for (i = 0; i < MAX_IRQ; i++) {
@@ -1062,7 +909,7 @@ int ltdc_load(struct drm_device *ddev)
 						dev_name(dev), ddev);
 		if (ret) {
 			DRM_ERROR("Failed to register LTDC interrupt\n");
-			return ret;
+			goto err;
 		}
 	}
 
@@ -1077,33 +924,27 @@ int ltdc_load(struct drm_device *ddev)
 	if (ret) {
 		DRM_ERROR("hardware identifier (0x%08x) not supported!\n",
 			  ldev->caps.hw_version);
-		return ret;
+		goto err;
 	}
 
 	DRM_INFO("ltdc hw version 0x%08x - ready\n", ldev->caps.hw_version);
 
-	if (ldev->panel) {
-		encoder = ltdc_rgb_encoder_create(ddev);
-		if (!encoder) {
-			DRM_ERROR("Failed to create RGB encoder\n");
-			ret = -EINVAL;
+	if (panel) {
+		bridge = drm_panel_bridge_add(panel, DRM_MODE_CONNECTOR_DPI);
+		if (IS_ERR(bridge)) {
+			DRM_ERROR("Failed to create panel-bridge\n");
+			ret = PTR_ERR(bridge);
 			goto err;
 		}
+		ldev->is_panel_bridge = true;
+	}
 
-		connector = ltdc_rgb_connector_create(ddev);
-		if (!connector) {
-			DRM_ERROR("Failed to create RGB connector\n");
-			ret = -EINVAL;
-			goto err;
-		}
+	ldev->bridge = bridge;
 
-		ret = drm_mode_connector_attach_encoder(connector, encoder);
-		if (ret) {
-			DRM_ERROR("Failed to attach connector to encoder\n");
-			goto err;
-		}
-
-		drm_panel_attach(ldev->panel, connector);
+	ret = ltdc_encoder_init(ddev);
+	if (ret) {
+		DRM_ERROR("Failed to init encoder\n");
+		goto err;
 	}
 
 	crtc = devm_kzalloc(dev, sizeof(*crtc), GFP_KERNEL);
@@ -1129,9 +970,10 @@ int ltdc_load(struct drm_device *ddev)
 	ddev->irq_enabled = 1;
 
 	return 0;
+
 err:
-	if (ldev->panel)
-		drm_panel_detach(ldev->panel);
+	if (ldev->is_panel_bridge)
+		drm_panel_bridge_remove(bridge);
 
 	clk_disable_unprepare(ldev->pixel_clk);
 
@@ -1144,8 +986,8 @@ void ltdc_unload(struct drm_device *ddev)
 
 	DRM_DEBUG_DRIVER("\n");
 
-	if (ldev->panel)
-		drm_panel_detach(ldev->panel);
+	if (ldev->is_panel_bridge)
+		drm_panel_bridge_remove(ldev->bridge);
 
 	clk_disable_unprepare(ldev->pixel_clk);
 }
diff --git a/drivers/gpu/drm/stm/ltdc.h b/drivers/gpu/drm/stm/ltdc.h
index d7a9c736ac1e..bc6d6f6419a9 100644
--- a/drivers/gpu/drm/stm/ltdc.h
+++ b/drivers/gpu/drm/stm/ltdc.h
@@ -24,10 +24,10 @@ struct ltdc_device {
 	struct drm_fbdev_cma *fbdev;
 	void __iomem *regs;
 	struct clk *pixel_clk;	/* lcd pixel clock */
-	struct drm_panel *panel;
+	struct drm_bridge *bridge;
+	bool is_panel_bridge;
 	struct mutex err_lock;	/* protecting error_status */
 	struct ltdc_caps caps;
-	u32 clut[256];		/* color look up table */
 	u32 error_status;
 	u32 irq_status;
 };
diff --git a/drivers/gpu/drm/sun4i/Kconfig b/drivers/gpu/drm/sun4i/Kconfig
index 5bcad8f5fb4f..06f05302ee75 100644
--- a/drivers/gpu/drm/sun4i/Kconfig
+++ b/drivers/gpu/drm/sun4i/Kconfig
@@ -13,17 +13,26 @@ config DRM_SUN4I
 	  Display Engine. If M is selected the module will be called
 	  sun4i-drm.
 
+if DRM_SUN4I
+
 config DRM_SUN4I_HDMI
        tristate "Allwinner A10 HDMI Controller Support"
-       depends on DRM_SUN4I
        default DRM_SUN4I
        help
 	  Choose this option if you have an Allwinner SoC with an HDMI
 	  controller.
 
+config DRM_SUN4I_HDMI_CEC
+       bool "Allwinner A10 HDMI CEC Support"
+       depends on DRM_SUN4I_HDMI
+       select CEC_CORE
+       depends on CEC_PIN
+       help
+	  Choose this option if you have an Allwinner SoC with an HDMI
+	  controller and want to use CEC.
+
 config DRM_SUN4I_BACKEND
 	tristate "Support for Allwinner A10 Display Engine Backend"
-	depends on DRM_SUN4I
 	default DRM_SUN4I
 	help
 	  Choose this option if you have an Allwinner SoC with the
@@ -33,10 +42,11 @@ config DRM_SUN4I_BACKEND
 
 config DRM_SUN8I_MIXER
 	tristate "Support for Allwinner Display Engine 2.0 Mixer"
-	depends on DRM_SUN4I
 	default MACH_SUN8I
 	help
 	  Choose this option if you have an Allwinner SoC with the
 	  Allwinner Display Engine 2.0, which has a mixer to do some
 	  graphics mixture and feed graphics to TCON, If M is
 	  selected the module will be called sun8i-mixer.
+
+endif
diff --git a/drivers/gpu/drm/sun4i/Makefile b/drivers/gpu/drm/sun4i/Makefile
index e29fd3a2ba9c..43c753cafc88 100644
--- a/drivers/gpu/drm/sun4i/Makefile
+++ b/drivers/gpu/drm/sun4i/Makefile
@@ -2,6 +2,7 @@ sun4i-drm-y += sun4i_drv.o
 sun4i-drm-y += sun4i_framebuffer.o
 
 sun4i-drm-hdmi-y += sun4i_hdmi_enc.o
+sun4i-drm-hdmi-y += sun4i_hdmi_i2c.o
 sun4i-drm-hdmi-y += sun4i_hdmi_ddc_clk.o
 sun4i-drm-hdmi-y += sun4i_hdmi_tmds_clk.o
 
diff --git a/drivers/gpu/drm/sun4i/sun4i_backend.c b/drivers/gpu/drm/sun4i/sun4i_backend.c
index cf480218daa5..ec5943627aa5 100644
--- a/drivers/gpu/drm/sun4i/sun4i_backend.c
+++ b/drivers/gpu/drm/sun4i/sun4i_backend.c
@@ -312,7 +312,7 @@ static int sun4i_backend_of_get_id(struct device_node *node)
 		struct device_node *remote;
 		u32 reg;
 
-		remote = of_parse_phandle(ep, "remote-endpoint", 0);
+		remote = of_graph_get_remote_endpoint(ep);
 		if (!remote)
 			continue;
 
diff --git a/drivers/gpu/drm/sun4i/sun4i_crtc.c b/drivers/gpu/drm/sun4i/sun4i_crtc.c
index f8c70439d1e2..d097c6f93ad0 100644
--- a/drivers/gpu/drm/sun4i/sun4i_crtc.c
+++ b/drivers/gpu/drm/sun4i/sun4i_crtc.c
@@ -69,7 +69,8 @@ static void sun4i_crtc_atomic_flush(struct drm_crtc *crtc,
 	}
 }
 
-static void sun4i_crtc_disable(struct drm_crtc *crtc)
+static void sun4i_crtc_atomic_disable(struct drm_crtc *crtc,
+				      struct drm_crtc_state *old_state)
 {
 	struct sun4i_crtc *scrtc = drm_crtc_to_sun4i_crtc(crtc);
 
@@ -86,7 +87,8 @@ static void sun4i_crtc_disable(struct drm_crtc *crtc)
 	}
 }
 
-static void sun4i_crtc_enable(struct drm_crtc *crtc)
+static void sun4i_crtc_atomic_enable(struct drm_crtc *crtc,
+				     struct drm_crtc_state *old_state)
 {
 	struct sun4i_crtc *scrtc = drm_crtc_to_sun4i_crtc(crtc);
 
@@ -98,8 +100,8 @@ static void sun4i_crtc_enable(struct drm_crtc *crtc)
 static const struct drm_crtc_helper_funcs sun4i_crtc_helper_funcs = {
 	.atomic_begin	= sun4i_crtc_atomic_begin,
 	.atomic_flush	= sun4i_crtc_atomic_flush,
-	.disable	= sun4i_crtc_disable,
-	.enable		= sun4i_crtc_enable,
+	.atomic_enable	= sun4i_crtc_atomic_enable,
+	.atomic_disable	= sun4i_crtc_atomic_disable,
 };
 
 static int sun4i_crtc_enable_vblank(struct drm_crtc *crtc)
diff --git a/drivers/gpu/drm/sun4i/sun4i_drv.c b/drivers/gpu/drm/sun4i/sun4i_drv.c
index abc7d8fe06b4..ace59651892f 100644
--- a/drivers/gpu/drm/sun4i/sun4i_drv.c
+++ b/drivers/gpu/drm/sun4i/sun4i_drv.c
@@ -25,12 +25,20 @@
 #include "sun4i_framebuffer.h"
 #include "sun4i_tcon.h"
 
+static void sun4i_drv_lastclose(struct drm_device *dev)
+{
+	struct sun4i_drv *drv = dev->dev_private;
+
+	drm_fbdev_cma_restore_mode(drv->fbdev);
+}
+
 DEFINE_DRM_GEM_CMA_FOPS(sun4i_drv_fops);
 
 static struct drm_driver sun4i_drv_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_PRIME | DRIVER_ATOMIC,
 
 	/* Generic Operations */
+	.lastclose		= sun4i_drv_lastclose,
 	.fops			= &sun4i_drv_fops,
 	.name			= "sun4i-drm",
 	.desc			= "Allwinner sun4i Display Engine",
@@ -40,8 +48,6 @@ static struct drm_driver sun4i_drv_driver = {
 
 	/* GEM Operations */
 	.dumb_create		= drm_gem_cma_dumb_create,
-	.dumb_destroy		= drm_gem_dumb_destroy,
-	.dumb_map_offset	= drm_gem_cma_dumb_map_offset,
 	.gem_free_object_unlocked = drm_gem_cma_free_object,
 	.gem_vm_ops		= &drm_gem_cma_vm_ops,
 
@@ -187,9 +193,9 @@ static bool sun4i_drv_node_is_tcon(struct device_node *node)
 
 static int compare_of(struct device *dev, void *data)
 {
-	DRM_DEBUG_DRIVER("Comparing of node %s with %s\n",
-			 of_node_full_name(dev->of_node),
-			 of_node_full_name(data));
+	DRM_DEBUG_DRIVER("Comparing of node %pOF with %pOF\n",
+			 dev->of_node,
+			 data);
 
 	return dev->of_node == data;
 }
@@ -219,8 +225,7 @@ static int sun4i_drv_add_endpoints(struct device *dev,
 
 	if (!sun4i_drv_node_is_frontend(node)) {
 		/* Add current component */
-		DRM_DEBUG_DRIVER("Adding component %s\n",
-				 of_node_full_name(node));
+		DRM_DEBUG_DRIVER("Adding component %pOF\n", node);
 		drm_of_component_match_add(dev, match, compare_of, node);
 		count++;
 	}
diff --git a/drivers/gpu/drm/sun4i/sun4i_hdmi.h b/drivers/gpu/drm/sun4i/sun4i_hdmi.h
index 2f2f2ff1ea63..1457750988da 100644
--- a/drivers/gpu/drm/sun4i/sun4i_hdmi.h
+++ b/drivers/gpu/drm/sun4i/sun4i_hdmi.h
@@ -15,6 +15,8 @@
 #include <drm/drm_connector.h>
 #include <drm/drm_encoder.h>
 
+#include <media/cec.h>
+
 #define SUN4I_HDMI_CTRL_REG		0x004
 #define SUN4I_HDMI_CTRL_ENABLE			BIT(31)
 
@@ -86,6 +88,11 @@
 #define SUN4I_HDMI_PLL_DBG0_TMDS_PARENT_MASK	BIT(21)
 #define SUN4I_HDMI_PLL_DBG0_TMDS_PARENT_SHIFT	21
 
+#define SUN4I_HDMI_CEC			0x214
+#define SUN4I_HDMI_CEC_ENABLE			BIT(11)
+#define SUN4I_HDMI_CEC_TX			BIT(9)
+#define SUN4I_HDMI_CEC_RX			BIT(8)
+
 #define SUN4I_HDMI_PKT_CTRL_REG(n)	(0x2f0 + (4 * (n)))
 #define SUN4I_HDMI_PKT_CTRL_TYPE(n, t)		((t) << (((n) % 4) * 4))
 
@@ -96,6 +103,7 @@
 #define SUN4I_HDMI_DDC_CTRL_ENABLE		BIT(31)
 #define SUN4I_HDMI_DDC_CTRL_START_CMD		BIT(30)
 #define SUN4I_HDMI_DDC_CTRL_FIFO_DIR_MASK	BIT(8)
+#define SUN4I_HDMI_DDC_CTRL_FIFO_DIR_WRITE	(1 << 8)
 #define SUN4I_HDMI_DDC_CTRL_FIFO_DIR_READ	(0 << 8)
 #define SUN4I_HDMI_DDC_CTRL_RESET		BIT(0)
 
@@ -105,14 +113,34 @@
 #define SUN4I_HDMI_DDC_ADDR_OFFSET(off)		(((off) & 0xff) << 8)
 #define SUN4I_HDMI_DDC_ADDR_SLAVE(addr)		((addr) & 0xff)
 
+#define SUN4I_HDMI_DDC_INT_STATUS_REG		0x50c
+#define SUN4I_HDMI_DDC_INT_STATUS_ILLEGAL_FIFO_OPERATION	BIT(7)
+#define SUN4I_HDMI_DDC_INT_STATUS_DDC_RX_FIFO_UNDERFLOW		BIT(6)
+#define SUN4I_HDMI_DDC_INT_STATUS_DDC_TX_FIFO_OVERFLOW		BIT(5)
+#define SUN4I_HDMI_DDC_INT_STATUS_FIFO_REQUEST			BIT(4)
+#define SUN4I_HDMI_DDC_INT_STATUS_ARBITRATION_ERROR		BIT(3)
+#define SUN4I_HDMI_DDC_INT_STATUS_ACK_ERROR			BIT(2)
+#define SUN4I_HDMI_DDC_INT_STATUS_BUS_ERROR			BIT(1)
+#define SUN4I_HDMI_DDC_INT_STATUS_TRANSFER_COMPLETE		BIT(0)
+
 #define SUN4I_HDMI_DDC_FIFO_CTRL_REG	0x510
 #define SUN4I_HDMI_DDC_FIFO_CTRL_CLEAR		BIT(31)
+#define SUN4I_HDMI_DDC_FIFO_CTRL_RX_THRES(n)	(((n) & 0xf) << 4)
+#define SUN4I_HDMI_DDC_FIFO_CTRL_RX_THRES_MASK	GENMASK(7, 4)
+#define SUN4I_HDMI_DDC_FIFO_CTRL_RX_THRES_MAX	(BIT(4) - 1)
+#define SUN4I_HDMI_DDC_FIFO_CTRL_TX_THRES(n)	((n) & 0xf)
+#define SUN4I_HDMI_DDC_FIFO_CTRL_TX_THRES_MASK	GENMASK(3, 0)
+#define SUN4I_HDMI_DDC_FIFO_CTRL_TX_THRES_MAX	(BIT(4) - 1)
 
 #define SUN4I_HDMI_DDC_FIFO_DATA_REG	0x518
+
 #define SUN4I_HDMI_DDC_BYTE_COUNT_REG	0x51c
+#define SUN4I_HDMI_DDC_BYTE_COUNT_MAX		(BIT(10) - 1)
 
 #define SUN4I_HDMI_DDC_CMD_REG		0x520
 #define SUN4I_HDMI_DDC_CMD_EXPLICIT_EDDC_READ	6
+#define SUN4I_HDMI_DDC_CMD_IMPLICIT_READ	5
+#define SUN4I_HDMI_DDC_CMD_IMPLICIT_WRITE	3
 
 #define SUN4I_HDMI_DDC_CLK_REG		0x528
 #define SUN4I_HDMI_DDC_CLK_M(m)			(((m) & 0x7) << 3)
@@ -146,12 +174,16 @@ struct sun4i_hdmi {
 	struct clk		*ddc_clk;
 	struct clk		*tmds_clk;
 
+	struct i2c_adapter	*i2c;
+
 	struct sun4i_drv	*drv;
 
 	bool			hdmi_monitor;
+	struct cec_adapter	*cec_adap;
 };
 
 int sun4i_ddc_create(struct sun4i_hdmi *hdmi, struct clk *clk);
 int sun4i_tmds_create(struct sun4i_hdmi *hdmi);
+int sun4i_hdmi_i2c_create(struct device *dev, struct sun4i_hdmi *hdmi);
 
 #endif /* _SUN4I_HDMI_H_ */
diff --git a/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c b/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c
index d3398f6250ef..9ea6cd5a1370 100644
--- a/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c
+++ b/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c
@@ -29,8 +29,6 @@
 #include "sun4i_hdmi.h"
 #include "sun4i_tcon.h"
 
-#define DDC_SEGMENT_ADDR	0x30
-
 static inline struct sun4i_hdmi *
 drm_encoder_to_sun4i_hdmi(struct drm_encoder *encoder)
 {
@@ -52,7 +50,7 @@ static int sun4i_hdmi_setup_avi_infoframes(struct sun4i_hdmi *hdmi,
 	u8 buffer[17];
 	int i, ret;
 
-	ret = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode);
+	ret = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode, false);
 	if (ret < 0) {
 		DRM_ERROR("Failed to get infoframes from mode\n");
 		return ret;
@@ -184,93 +182,13 @@ static const struct drm_encoder_funcs sun4i_hdmi_funcs = {
 	.destroy	= drm_encoder_cleanup,
 };
 
-static int sun4i_hdmi_read_sub_block(struct sun4i_hdmi *hdmi,
-				     unsigned int blk, unsigned int offset,
-				     u8 *buf, unsigned int count)
-{
-	unsigned long reg;
-	int i;
-
-	reg = readl(hdmi->base + SUN4I_HDMI_DDC_CTRL_REG);
-	reg &= ~SUN4I_HDMI_DDC_CTRL_FIFO_DIR_MASK;
-	writel(reg | SUN4I_HDMI_DDC_CTRL_FIFO_DIR_READ,
-	       hdmi->base + SUN4I_HDMI_DDC_CTRL_REG);
-
-	writel(SUN4I_HDMI_DDC_ADDR_SEGMENT(offset >> 8) |
-	       SUN4I_HDMI_DDC_ADDR_EDDC(DDC_SEGMENT_ADDR << 1) |
-	       SUN4I_HDMI_DDC_ADDR_OFFSET(offset) |
-	       SUN4I_HDMI_DDC_ADDR_SLAVE(DDC_ADDR),
-	       hdmi->base + SUN4I_HDMI_DDC_ADDR_REG);
-
-	reg = readl(hdmi->base + SUN4I_HDMI_DDC_FIFO_CTRL_REG);
-	writel(reg | SUN4I_HDMI_DDC_FIFO_CTRL_CLEAR,
-	       hdmi->base + SUN4I_HDMI_DDC_FIFO_CTRL_REG);
-
-	writel(count, hdmi->base + SUN4I_HDMI_DDC_BYTE_COUNT_REG);
-	writel(SUN4I_HDMI_DDC_CMD_EXPLICIT_EDDC_READ,
-	       hdmi->base + SUN4I_HDMI_DDC_CMD_REG);
-
-	reg = readl(hdmi->base + SUN4I_HDMI_DDC_CTRL_REG);
-	writel(reg | SUN4I_HDMI_DDC_CTRL_START_CMD,
-	       hdmi->base + SUN4I_HDMI_DDC_CTRL_REG);
-
-	if (readl_poll_timeout(hdmi->base + SUN4I_HDMI_DDC_CTRL_REG, reg,
-			       !(reg & SUN4I_HDMI_DDC_CTRL_START_CMD),
-			       100, 100000))
-		return -EIO;
-
-	for (i = 0; i < count; i++)
-		buf[i] = readb(hdmi->base + SUN4I_HDMI_DDC_FIFO_DATA_REG);
-
-	return 0;
-}
-
-static int sun4i_hdmi_read_edid_block(void *data, u8 *buf, unsigned int blk,
-				      size_t length)
-{
-	struct sun4i_hdmi *hdmi = data;
-	int retry = 2, i;
-
-	do {
-		for (i = 0; i < length; i += SUN4I_HDMI_DDC_FIFO_SIZE) {
-			unsigned char offset = blk * EDID_LENGTH + i;
-			unsigned int count = min((unsigned int)SUN4I_HDMI_DDC_FIFO_SIZE,
-						 length - i);
-			int ret;
-
-			ret = sun4i_hdmi_read_sub_block(hdmi, blk, offset,
-							buf + i, count);
-			if (ret)
-				return ret;
-		}
-	} while (!drm_edid_block_valid(buf, blk, true, NULL) && (retry--));
-
-	return 0;
-}
-
 static int sun4i_hdmi_get_modes(struct drm_connector *connector)
 {
 	struct sun4i_hdmi *hdmi = drm_connector_to_sun4i_hdmi(connector);
-	unsigned long reg;
 	struct edid *edid;
 	int ret;
 
-	/* Reset i2c controller */
-	writel(SUN4I_HDMI_DDC_CTRL_ENABLE | SUN4I_HDMI_DDC_CTRL_RESET,
-	       hdmi->base + SUN4I_HDMI_DDC_CTRL_REG);
-	if (readl_poll_timeout(hdmi->base + SUN4I_HDMI_DDC_CTRL_REG, reg,
-			       !(reg & SUN4I_HDMI_DDC_CTRL_RESET),
-			       100, 2000))
-		return -EIO;
-
-	writel(SUN4I_HDMI_DDC_LINE_CTRL_SDA_ENABLE |
-	       SUN4I_HDMI_DDC_LINE_CTRL_SCL_ENABLE,
-	       hdmi->base + SUN4I_HDMI_DDC_LINE_CTRL_REG);
-
-	clk_prepare_enable(hdmi->ddc_clk);
-	clk_set_rate(hdmi->ddc_clk, 100000);
-
-	edid = drm_do_get_edid(connector, sun4i_hdmi_read_edid_block, hdmi);
+	edid = drm_get_edid(connector, hdmi->i2c);
 	if (!edid)
 		return 0;
 
@@ -279,11 +197,10 @@ static int sun4i_hdmi_get_modes(struct drm_connector *connector)
 			 hdmi->hdmi_monitor ? "an HDMI" : "a DVI");
 
 	drm_mode_connector_update_edid_property(connector, edid);
+	cec_s_phys_addr_from_edid(hdmi->cec_adap, edid);
 	ret = drm_add_edid_modes(connector, edid);
 	kfree(edid);
 
-	clk_disable_unprepare(hdmi->ddc_clk);
-
 	return ret;
 }
 
@@ -299,14 +216,15 @@ sun4i_hdmi_connector_detect(struct drm_connector *connector, bool force)
 
 	if (readl_poll_timeout(hdmi->base + SUN4I_HDMI_HPD_REG, reg,
 			       reg & SUN4I_HDMI_HPD_HIGH,
-			       0, 500000))
+			       0, 500000)) {
+		cec_phys_addr_invalidate(hdmi->cec_adap);
 		return connector_status_disconnected;
+	}
 
 	return connector_status_connected;
 }
 
 static const struct drm_connector_funcs sun4i_hdmi_connector_funcs = {
-	.dpms			= drm_atomic_helper_connector_dpms,
 	.detect			= sun4i_hdmi_connector_detect,
 	.fill_modes		= drm_helper_probe_single_connector_modes,
 	.destroy		= drm_connector_cleanup,
@@ -315,6 +233,40 @@ static const struct drm_connector_funcs sun4i_hdmi_connector_funcs = {
 	.atomic_destroy_state	= drm_atomic_helper_connector_destroy_state,
 };
 
+#ifdef CONFIG_DRM_SUN4I_HDMI_CEC
+static bool sun4i_hdmi_cec_pin_read(struct cec_adapter *adap)
+{
+	struct sun4i_hdmi *hdmi = cec_get_drvdata(adap);
+
+	return readl(hdmi->base + SUN4I_HDMI_CEC) & SUN4I_HDMI_CEC_RX;
+}
+
+static void sun4i_hdmi_cec_pin_low(struct cec_adapter *adap)
+{
+	struct sun4i_hdmi *hdmi = cec_get_drvdata(adap);
+
+	/* Start driving the CEC pin low */
+	writel(SUN4I_HDMI_CEC_ENABLE, hdmi->base + SUN4I_HDMI_CEC);
+}
+
+static void sun4i_hdmi_cec_pin_high(struct cec_adapter *adap)
+{
+	struct sun4i_hdmi *hdmi = cec_get_drvdata(adap);
+
+	/*
+	 * Stop driving the CEC pin, the pull up will take over
+	 * unless another CEC device is driving the pin low.
+	 */
+	writel(0, hdmi->base + SUN4I_HDMI_CEC);
+}
+
+static const struct cec_pin_ops sun4i_hdmi_cec_pin_ops = {
+	.read = sun4i_hdmi_cec_pin_read,
+	.low = sun4i_hdmi_cec_pin_low,
+	.high = sun4i_hdmi_cec_pin_high,
+};
+#endif
+
 static int sun4i_hdmi_bind(struct device *dev, struct device *master,
 			   void *data)
 {
@@ -407,9 +359,9 @@ static int sun4i_hdmi_bind(struct device *dev, struct device *master,
 		SUN4I_HDMI_PLL_CTRL_PLL_EN;
 	writel(reg, hdmi->base + SUN4I_HDMI_PLL_CTRL_REG);
 
-	ret = sun4i_ddc_create(hdmi, hdmi->tmds_clk);
+	ret = sun4i_hdmi_i2c_create(dev, hdmi);
 	if (ret) {
-		dev_err(dev, "Couldn't create the DDC clock\n");
+		dev_err(dev, "Couldn't create the HDMI I2C adapter\n");
 		return ret;
 	}
 
@@ -422,13 +374,26 @@ static int sun4i_hdmi_bind(struct device *dev, struct device *master,
 			       NULL);
 	if (ret) {
 		dev_err(dev, "Couldn't initialise the HDMI encoder\n");
-		return ret;
+		goto err_del_i2c_adapter;
 	}
 
 	hdmi->encoder.possible_crtcs = drm_of_find_possible_crtcs(drm,
 								  dev->of_node);
-	if (!hdmi->encoder.possible_crtcs)
-		return -EPROBE_DEFER;
+	if (!hdmi->encoder.possible_crtcs) {
+		ret = -EPROBE_DEFER;
+		goto err_del_i2c_adapter;
+	}
+
+#ifdef CONFIG_DRM_SUN4I_HDMI_CEC
+	hdmi->cec_adap = cec_pin_allocate_adapter(&sun4i_hdmi_cec_pin_ops,
+		hdmi, "sun4i", CEC_CAP_TRANSMIT | CEC_CAP_LOG_ADDRS |
+		CEC_CAP_PASSTHROUGH | CEC_CAP_RC);
+	ret = PTR_ERR_OR_ZERO(hdmi->cec_adap);
+	if (ret < 0)
+		goto err_cleanup_connector;
+	writel(readl(hdmi->base + SUN4I_HDMI_CEC) & ~SUN4I_HDMI_CEC_TX,
+	       hdmi->base + SUN4I_HDMI_CEC);
+#endif
 
 	drm_connector_helper_add(&hdmi->connector,
 				 &sun4i_hdmi_connector_helper_funcs);
@@ -445,12 +410,18 @@ static int sun4i_hdmi_bind(struct device *dev, struct device *master,
 	hdmi->connector.polled = DRM_CONNECTOR_POLL_CONNECT |
 		DRM_CONNECTOR_POLL_DISCONNECT;
 
+	ret = cec_register_adapter(hdmi->cec_adap, dev);
+	if (ret < 0)
+		goto err_cleanup_connector;
 	drm_mode_connector_attach_encoder(&hdmi->connector, &hdmi->encoder);
 
 	return 0;
 
 err_cleanup_connector:
+	cec_delete_adapter(hdmi->cec_adap);
 	drm_encoder_cleanup(&hdmi->encoder);
+err_del_i2c_adapter:
+	i2c_del_adapter(hdmi->i2c);
 	return ret;
 }
 
@@ -459,8 +430,10 @@ static void sun4i_hdmi_unbind(struct device *dev, struct device *master,
 {
 	struct sun4i_hdmi *hdmi = dev_get_drvdata(dev);
 
+	cec_unregister_adapter(hdmi->cec_adap);
 	drm_connector_cleanup(&hdmi->connector);
 	drm_encoder_cleanup(&hdmi->encoder);
+	i2c_del_adapter(hdmi->i2c);
 }
 
 static const struct component_ops sun4i_hdmi_ops = {
diff --git a/drivers/gpu/drm/sun4i/sun4i_hdmi_i2c.c b/drivers/gpu/drm/sun4i/sun4i_hdmi_i2c.c
new file mode 100644
index 000000000000..2e42d09ab42e
--- /dev/null
+++ b/drivers/gpu/drm/sun4i/sun4i_hdmi_i2c.c
@@ -0,0 +1,220 @@
+/*
+ * Copyright (C) 2016 Maxime Ripard <maxime.ripard@free-electrons.com>
+ * Copyright (C) 2017 Jonathan Liu <net147@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
+ */
+
+#include <linux/clk.h>
+#include <linux/i2c.h>
+#include <linux/iopoll.h>
+
+#include "sun4i_hdmi.h"
+
+#define SUN4I_HDMI_DDC_INT_STATUS_ERROR_MASK ( \
+	SUN4I_HDMI_DDC_INT_STATUS_ILLEGAL_FIFO_OPERATION | \
+	SUN4I_HDMI_DDC_INT_STATUS_DDC_RX_FIFO_UNDERFLOW | \
+	SUN4I_HDMI_DDC_INT_STATUS_DDC_TX_FIFO_OVERFLOW | \
+	SUN4I_HDMI_DDC_INT_STATUS_ARBITRATION_ERROR | \
+	SUN4I_HDMI_DDC_INT_STATUS_ACK_ERROR | \
+	SUN4I_HDMI_DDC_INT_STATUS_BUS_ERROR \
+)
+
+/* FIFO request bit is set when FIFO level is above RX_THRESHOLD during read */
+#define RX_THRESHOLD SUN4I_HDMI_DDC_FIFO_CTRL_RX_THRES_MAX
+/* FIFO request bit is set when FIFO level is below TX_THRESHOLD during write */
+#define TX_THRESHOLD 1
+
+static int fifo_transfer(struct sun4i_hdmi *hdmi, u8 *buf, int len, bool read)
+{
+	/*
+	 * 1 byte takes 9 clock cycles (8 bits + 1 ACK) = 90 us for 100 kHz
+	 * clock. As clock rate is fixed, just round it up to 100 us.
+	 */
+	const unsigned long byte_time_ns = 100;
+	const u32 mask = SUN4I_HDMI_DDC_INT_STATUS_ERROR_MASK |
+			 SUN4I_HDMI_DDC_INT_STATUS_FIFO_REQUEST |
+			 SUN4I_HDMI_DDC_INT_STATUS_TRANSFER_COMPLETE;
+	u32 reg;
+
+	/* Limit transfer length by FIFO threshold */
+	len = min_t(int, len, read ? (RX_THRESHOLD + 1) :
+			      (SUN4I_HDMI_DDC_FIFO_SIZE - TX_THRESHOLD + 1));
+
+	/* Wait until error, FIFO request bit set or transfer complete */
+	if (readl_poll_timeout(hdmi->base + SUN4I_HDMI_DDC_INT_STATUS_REG, reg,
+			       reg & mask, len * byte_time_ns, 100000))
+		return -ETIMEDOUT;
+
+	if (reg & SUN4I_HDMI_DDC_INT_STATUS_ERROR_MASK)
+		return -EIO;
+
+	if (read)
+		readsb(hdmi->base + SUN4I_HDMI_DDC_FIFO_DATA_REG, buf, len);
+	else
+		writesb(hdmi->base + SUN4I_HDMI_DDC_FIFO_DATA_REG, buf, len);
+
+	/* Clear FIFO request bit */
+	writel(SUN4I_HDMI_DDC_INT_STATUS_FIFO_REQUEST,
+	       hdmi->base + SUN4I_HDMI_DDC_INT_STATUS_REG);
+
+	return len;
+}
+
+static int xfer_msg(struct sun4i_hdmi *hdmi, struct i2c_msg *msg)
+{
+	int i, len;
+	u32 reg;
+
+	/* Set FIFO direction */
+	reg = readl(hdmi->base + SUN4I_HDMI_DDC_CTRL_REG);
+	reg &= ~SUN4I_HDMI_DDC_CTRL_FIFO_DIR_MASK;
+	reg |= (msg->flags & I2C_M_RD) ?
+	       SUN4I_HDMI_DDC_CTRL_FIFO_DIR_READ :
+	       SUN4I_HDMI_DDC_CTRL_FIFO_DIR_WRITE;
+	writel(reg, hdmi->base + SUN4I_HDMI_DDC_CTRL_REG);
+
+	/* Set I2C address */
+	writel(SUN4I_HDMI_DDC_ADDR_SLAVE(msg->addr),
+	       hdmi->base + SUN4I_HDMI_DDC_ADDR_REG);
+
+	/* Set FIFO RX/TX thresholds and clear FIFO */
+	reg = readl(hdmi->base + SUN4I_HDMI_DDC_FIFO_CTRL_REG);
+	reg |= SUN4I_HDMI_DDC_FIFO_CTRL_CLEAR;
+	reg &= ~SUN4I_HDMI_DDC_FIFO_CTRL_RX_THRES_MASK;
+	reg |= SUN4I_HDMI_DDC_FIFO_CTRL_RX_THRES(RX_THRESHOLD);
+	reg &= ~SUN4I_HDMI_DDC_FIFO_CTRL_TX_THRES_MASK;
+	reg |= SUN4I_HDMI_DDC_FIFO_CTRL_TX_THRES(TX_THRESHOLD);
+	writel(reg, hdmi->base + SUN4I_HDMI_DDC_FIFO_CTRL_REG);
+	if (readl_poll_timeout(hdmi->base + SUN4I_HDMI_DDC_FIFO_CTRL_REG,
+			       reg,
+			       !(reg & SUN4I_HDMI_DDC_FIFO_CTRL_CLEAR),
+			       100, 2000))
+		return -EIO;
+
+	/* Set transfer length */
+	writel(msg->len, hdmi->base + SUN4I_HDMI_DDC_BYTE_COUNT_REG);
+
+	/* Set command */
+	writel(msg->flags & I2C_M_RD ?
+	       SUN4I_HDMI_DDC_CMD_IMPLICIT_READ :
+	       SUN4I_HDMI_DDC_CMD_IMPLICIT_WRITE,
+	       hdmi->base + SUN4I_HDMI_DDC_CMD_REG);
+
+	/* Clear interrupt status bits */
+	writel(SUN4I_HDMI_DDC_INT_STATUS_ERROR_MASK |
+	       SUN4I_HDMI_DDC_INT_STATUS_FIFO_REQUEST |
+	       SUN4I_HDMI_DDC_INT_STATUS_TRANSFER_COMPLETE,
+	       hdmi->base + SUN4I_HDMI_DDC_INT_STATUS_REG);
+
+	/* Start command */
+	reg = readl(hdmi->base + SUN4I_HDMI_DDC_CTRL_REG);
+	writel(reg | SUN4I_HDMI_DDC_CTRL_START_CMD,
+	       hdmi->base + SUN4I_HDMI_DDC_CTRL_REG);
+
+	/* Transfer bytes */
+	for (i = 0; i < msg->len; i += len) {
+		len = fifo_transfer(hdmi, msg->buf + i, msg->len - i,
+				    msg->flags & I2C_M_RD);
+		if (len <= 0)
+			return len;
+	}
+
+	/* Wait for command to finish */
+	if (readl_poll_timeout(hdmi->base + SUN4I_HDMI_DDC_CTRL_REG,
+			       reg,
+			       !(reg & SUN4I_HDMI_DDC_CTRL_START_CMD),
+			       100, 100000))
+		return -EIO;
+
+	/* Check for errors */
+	reg = readl(hdmi->base + SUN4I_HDMI_DDC_INT_STATUS_REG);
+	if ((reg & SUN4I_HDMI_DDC_INT_STATUS_ERROR_MASK) ||
+	    !(reg & SUN4I_HDMI_DDC_INT_STATUS_TRANSFER_COMPLETE)) {
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static int sun4i_hdmi_i2c_xfer(struct i2c_adapter *adap,
+			       struct i2c_msg *msgs, int num)
+{
+	struct sun4i_hdmi *hdmi = i2c_get_adapdata(adap);
+	u32 reg;
+	int err, i, ret = num;
+
+	for (i = 0; i < num; i++) {
+		if (!msgs[i].len)
+			return -EINVAL;
+		if (msgs[i].len > SUN4I_HDMI_DDC_BYTE_COUNT_MAX)
+			return -EINVAL;
+	}
+
+	/* Reset I2C controller */
+	writel(SUN4I_HDMI_DDC_CTRL_ENABLE | SUN4I_HDMI_DDC_CTRL_RESET,
+	       hdmi->base + SUN4I_HDMI_DDC_CTRL_REG);
+	if (readl_poll_timeout(hdmi->base + SUN4I_HDMI_DDC_CTRL_REG, reg,
+			       !(reg & SUN4I_HDMI_DDC_CTRL_RESET),
+			       100, 2000))
+		return -EIO;
+
+	writel(SUN4I_HDMI_DDC_LINE_CTRL_SDA_ENABLE |
+	       SUN4I_HDMI_DDC_LINE_CTRL_SCL_ENABLE,
+	       hdmi->base + SUN4I_HDMI_DDC_LINE_CTRL_REG);
+
+	clk_prepare_enable(hdmi->ddc_clk);
+	clk_set_rate(hdmi->ddc_clk, 100000);
+
+	for (i = 0; i < num; i++) {
+		err = xfer_msg(hdmi, &msgs[i]);
+		if (err) {
+			ret = err;
+			break;
+		}
+	}
+
+	clk_disable_unprepare(hdmi->ddc_clk);
+	return ret;
+}
+
+static u32 sun4i_hdmi_i2c_func(struct i2c_adapter *adap)
+{
+	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
+}
+
+static const struct i2c_algorithm sun4i_hdmi_i2c_algorithm = {
+	.master_xfer	= sun4i_hdmi_i2c_xfer,
+	.functionality	= sun4i_hdmi_i2c_func,
+};
+
+int sun4i_hdmi_i2c_create(struct device *dev, struct sun4i_hdmi *hdmi)
+{
+	struct i2c_adapter *adap;
+	int ret = 0;
+
+	ret = sun4i_ddc_create(hdmi, hdmi->tmds_clk);
+	if (ret)
+		return ret;
+
+	adap = devm_kzalloc(dev, sizeof(*adap), GFP_KERNEL);
+	if (!adap)
+		return -ENOMEM;
+
+	adap->owner = THIS_MODULE;
+	adap->class = I2C_CLASS_DDC;
+	adap->algo = &sun4i_hdmi_i2c_algorithm;
+	strlcpy(adap->name, "sun4i_hdmi_i2c adapter", sizeof(adap->name));
+	i2c_set_adapdata(adap, hdmi);
+
+	ret = i2c_add_adapter(adap);
+	if (ret)
+		return ret;
+
+	hdmi->i2c = adap;
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/sun4i/sun4i_layer.c b/drivers/gpu/drm/sun4i/sun4i_layer.c
index ead4f9d4c1ee..7bddf12548d3 100644
--- a/drivers/gpu/drm/sun4i/sun4i_layer.c
+++ b/drivers/gpu/drm/sun4i/sun4i_layer.c
@@ -25,12 +25,6 @@ struct sun4i_plane_desc {
 	       uint32_t                nformats;
 };
 
-static int sun4i_backend_layer_atomic_check(struct drm_plane *plane,
-					    struct drm_plane_state *state)
-{
-	return 0;
-}
-
 static void sun4i_backend_layer_atomic_disable(struct drm_plane *plane,
 					       struct drm_plane_state *old_state)
 {
@@ -52,8 +46,7 @@ static void sun4i_backend_layer_atomic_update(struct drm_plane *plane,
 	sun4i_backend_layer_enable(backend, layer->id, true);
 }
 
-static struct drm_plane_helper_funcs sun4i_backend_layer_helper_funcs = {
-	.atomic_check	= sun4i_backend_layer_atomic_check,
+static const struct drm_plane_helper_funcs sun4i_backend_layer_helper_funcs = {
 	.atomic_disable	= sun4i_backend_layer_atomic_disable,
 	.atomic_update	= sun4i_backend_layer_atomic_update,
 };
@@ -115,7 +108,7 @@ static struct sun4i_layer *sun4i_layer_init_one(struct drm_device *drm,
 	ret = drm_universal_plane_init(drm, &layer->plane, 0,
 				       &sun4i_backend_layer_funcs,
 				       plane->formats, plane->nformats,
-				       plane->type, NULL);
+				       NULL, plane->type, NULL);
 	if (ret) {
 		dev_err(drm->dev, "Couldn't initialize layer\n");
 		return ERR_PTR(ret);
diff --git a/drivers/gpu/drm/sun4i/sun4i_rgb.c b/drivers/gpu/drm/sun4i/sun4i_rgb.c
index 422b191faa77..7cd7090ad63a 100644
--- a/drivers/gpu/drm/sun4i/sun4i_rgb.c
+++ b/drivers/gpu/drm/sun4i/sun4i_rgb.c
@@ -119,8 +119,7 @@ sun4i_rgb_connector_destroy(struct drm_connector *connector)
 	drm_connector_cleanup(connector);
 }
 
-static struct drm_connector_funcs sun4i_rgb_con_funcs = {
-	.dpms			= drm_atomic_helper_connector_dpms,
+static const struct drm_connector_funcs sun4i_rgb_con_funcs = {
 	.fill_modes		= drm_helper_probe_single_connector_modes,
 	.destroy		= sun4i_rgb_connector_destroy,
 	.reset			= drm_atomic_helper_connector_reset,
@@ -128,13 +127,6 @@ static struct drm_connector_funcs sun4i_rgb_con_funcs = {
 	.atomic_destroy_state	= drm_atomic_helper_connector_destroy_state,
 };
 
-static int sun4i_rgb_atomic_check(struct drm_encoder *encoder,
-				  struct drm_crtc_state *crtc_state,
-				  struct drm_connector_state *conn_state)
-{
-	return 0;
-}
-
 static void sun4i_rgb_encoder_enable(struct drm_encoder *encoder)
 {
 	struct sun4i_rgb *rgb = drm_encoder_to_sun4i_rgb(encoder);
@@ -182,7 +174,6 @@ static void sun4i_rgb_encoder_mode_set(struct drm_encoder *encoder,
 }
 
 static struct drm_encoder_helper_funcs sun4i_rgb_enc_helper_funcs = {
-	.atomic_check	= sun4i_rgb_atomic_check,
 	.mode_set	= sun4i_rgb_encoder_mode_set,
 	.disable	= sun4i_rgb_encoder_disable,
 	.enable		= sun4i_rgb_encoder_enable,
diff --git a/drivers/gpu/drm/sun4i/sun4i_tcon.h b/drivers/gpu/drm/sun4i/sun4i_tcon.h
index e3c50ecdcd04..552c88ec16be 100644
--- a/drivers/gpu/drm/sun4i/sun4i_tcon.h
+++ b/drivers/gpu/drm/sun4i/sun4i_tcon.h
@@ -194,8 +194,6 @@ void sun4i_tcon_channel_enable(struct sun4i_tcon *tcon, int channel);
 void sun4i_tcon_enable_vblank(struct sun4i_tcon *tcon, bool enable);
 
 /* Mode Related Controls */
-void sun4i_tcon_switch_interlace(struct sun4i_tcon *tcon,
-				 bool enable);
 void sun4i_tcon_set_mux(struct sun4i_tcon *tcon, int channel,
 			struct drm_encoder *encoder);
 void sun4i_tcon0_mode_set(struct sun4i_tcon *tcon,
diff --git a/drivers/gpu/drm/sun4i/sun4i_tv.c b/drivers/gpu/drm/sun4i/sun4i_tv.c
index 338b9e5bb2a3..050cfd43c7a0 100644
--- a/drivers/gpu/drm/sun4i/sun4i_tv.c
+++ b/drivers/gpu/drm/sun4i/sun4i_tv.c
@@ -341,13 +341,6 @@ static void sun4i_tv_mode_to_drm_mode(const struct tv_mode *tv_mode,
 	mode->vtotal = mode->vsync_end  + tv_mode->vback_porch;
 }
 
-static int sun4i_tv_atomic_check(struct drm_encoder *encoder,
-				 struct drm_crtc_state *crtc_state,
-				 struct drm_connector_state *conn_state)
-{
-	return 0;
-}
-
 static void sun4i_tv_disable(struct drm_encoder *encoder)
 {
 	struct sun4i_tv *tv = drm_encoder_to_sun4i_tv(encoder);
@@ -489,7 +482,6 @@ static void sun4i_tv_mode_set(struct drm_encoder *encoder,
 }
 
 static struct drm_encoder_helper_funcs sun4i_tv_helper_funcs = {
-	.atomic_check	= sun4i_tv_atomic_check,
 	.disable	= sun4i_tv_disable,
 	.enable		= sun4i_tv_enable,
 	.mode_set	= sun4i_tv_mode_set,
@@ -545,8 +537,7 @@ sun4i_tv_comp_connector_destroy(struct drm_connector *connector)
 	drm_connector_cleanup(connector);
 }
 
-static struct drm_connector_funcs sun4i_tv_comp_connector_funcs = {
-	.dpms			= drm_atomic_helper_connector_dpms,
+static const struct drm_connector_funcs sun4i_tv_comp_connector_funcs = {
 	.fill_modes		= drm_helper_probe_single_connector_modes,
 	.destroy		= sun4i_tv_comp_connector_destroy,
 	.reset			= drm_atomic_helper_connector_reset,
diff --git a/drivers/gpu/drm/sun4i/sun8i_layer.c b/drivers/gpu/drm/sun4i/sun8i_layer.c
index e627eeece658..23810ff72684 100644
--- a/drivers/gpu/drm/sun4i/sun8i_layer.c
+++ b/drivers/gpu/drm/sun4i/sun8i_layer.c
@@ -90,7 +90,7 @@ static struct sun8i_layer *sun8i_layer_init_one(struct drm_device *drm,
 	ret = drm_universal_plane_init(drm, &layer->plane, 0,
 				       &sun8i_mixer_layer_funcs,
 				       plane->formats, plane->nformats,
-				       plane->type, NULL);
+				       NULL, plane->type, NULL);
 	if (ret) {
 		dev_err(drm->dev, "Couldn't initialize layer\n");
 		return ERR_PTR(ret);
diff --git a/drivers/gpu/drm/tdfx/tdfx_drv.c b/drivers/gpu/drm/tdfx/tdfx_drv.c
index c54138c3a376..3a1476818c65 100644
--- a/drivers/gpu/drm/tdfx/tdfx_drv.c
+++ b/drivers/gpu/drm/tdfx/tdfx_drv.c
@@ -55,7 +55,6 @@ static const struct file_operations tdfx_driver_fops = {
 
 static struct drm_driver driver = {
 	.driver_features = DRIVER_LEGACY,
-	.set_busid = drm_pci_set_busid,
 	.fops = &tdfx_driver_fops,
 	.name = DRIVER_NAME,
 	.desc = DRIVER_DESC,
@@ -72,12 +71,12 @@ static struct pci_driver tdfx_pci_driver = {
 
 static int __init tdfx_init(void)
 {
-	return drm_pci_init(&driver, &tdfx_pci_driver);
+	return drm_legacy_pci_init(&driver, &tdfx_pci_driver);
 }
 
 static void __exit tdfx_exit(void)
 {
-	drm_pci_exit(&driver, &tdfx_pci_driver);
+	drm_legacy_pci_exit(&driver, &tdfx_pci_driver);
 }
 
 module_init(tdfx_init);
diff --git a/drivers/gpu/drm/tegra/Kconfig b/drivers/gpu/drm/tegra/Kconfig
index 2db29d67193d..dc58ab140151 100644
--- a/drivers/gpu/drm/tegra/Kconfig
+++ b/drivers/gpu/drm/tegra/Kconfig
@@ -3,6 +3,7 @@ config DRM_TEGRA
 	depends on ARCH_TEGRA || (ARM && COMPILE_TEST)
 	depends on COMMON_CLK
 	depends on DRM
+	depends on OF
 	select DRM_KMS_HELPER
 	select DRM_MIPI_DSI
 	select DRM_PANEL
diff --git a/drivers/gpu/drm/tegra/Makefile b/drivers/gpu/drm/tegra/Makefile
index 6af3a9ad6565..8927784396e8 100644
--- a/drivers/gpu/drm/tegra/Makefile
+++ b/drivers/gpu/drm/tegra/Makefile
@@ -17,4 +17,6 @@ tegra-drm-y := \
 	falcon.o \
 	vic.o
 
+tegra-drm-y += trace.o
+
 obj-$(CONFIG_DRM_TEGRA) += tegra-drm.o
diff --git a/drivers/gpu/drm/tegra/dc.c b/drivers/gpu/drm/tegra/dc.c
index c875f11786b9..4df39112e38e 100644
--- a/drivers/gpu/drm/tegra/dc.c
+++ b/drivers/gpu/drm/tegra/dc.c
@@ -678,8 +678,8 @@ static struct drm_plane *tegra_dc_primary_plane_create(struct drm_device *drm,
 
 	err = drm_universal_plane_init(drm, &plane->base, possible_crtcs,
 				       &tegra_primary_plane_funcs, formats,
-				       num_formats, DRM_PLANE_TYPE_PRIMARY,
-				       NULL);
+				       num_formats, NULL,
+				       DRM_PLANE_TYPE_PRIMARY, NULL);
 	if (err < 0) {
 		kfree(plane);
 		return ERR_PTR(err);
@@ -844,8 +844,8 @@ static struct drm_plane *tegra_dc_cursor_plane_create(struct drm_device *drm,
 
 	err = drm_universal_plane_init(drm, &plane->base, 1 << dc->pipe,
 				       &tegra_cursor_plane_funcs, formats,
-				       num_formats, DRM_PLANE_TYPE_CURSOR,
-				       NULL);
+				       num_formats, NULL,
+				       DRM_PLANE_TYPE_CURSOR, NULL);
 	if (err < 0) {
 		kfree(plane);
 		return ERR_PTR(err);
@@ -906,8 +906,8 @@ static struct drm_plane *tegra_dc_overlay_plane_create(struct drm_device *drm,
 
 	err = drm_universal_plane_init(drm, &plane->base, 1 << dc->pipe,
 				       &tegra_overlay_plane_funcs, formats,
-				       num_formats, DRM_PLANE_TYPE_OVERLAY,
-				       NULL);
+				       num_formats, NULL,
+				       DRM_PLANE_TYPE_OVERLAY, NULL);
 	if (err < 0) {
 		kfree(plane);
 		return ERR_PTR(err);
@@ -1199,7 +1199,8 @@ static int tegra_dc_wait_idle(struct tegra_dc *dc, unsigned long timeout)
 	return -ETIMEDOUT;
 }
 
-static void tegra_crtc_disable(struct drm_crtc *crtc)
+static void tegra_crtc_atomic_disable(struct drm_crtc *crtc,
+				      struct drm_crtc_state *old_state)
 {
 	struct tegra_dc *dc = to_tegra_dc(crtc);
 	u32 value;
@@ -1243,7 +1244,8 @@ static void tegra_crtc_disable(struct drm_crtc *crtc)
 	pm_runtime_put_sync(dc->dev);
 }
 
-static void tegra_crtc_enable(struct drm_crtc *crtc)
+static void tegra_crtc_atomic_enable(struct drm_crtc *crtc,
+				     struct drm_crtc_state *old_state)
 {
 	struct drm_display_mode *mode = &crtc->state->adjusted_mode;
 	struct tegra_dc_state *state = to_dc_state(crtc->state);
@@ -1351,11 +1353,11 @@ static void tegra_crtc_atomic_flush(struct drm_crtc *crtc,
 }
 
 static const struct drm_crtc_helper_funcs tegra_crtc_helper_funcs = {
-	.disable = tegra_crtc_disable,
-	.enable = tegra_crtc_enable,
 	.atomic_check = tegra_crtc_atomic_check,
 	.atomic_begin = tegra_crtc_atomic_begin,
 	.atomic_flush = tegra_crtc_atomic_flush,
+	.atomic_enable = tegra_crtc_atomic_enable,
+	.atomic_disable = tegra_crtc_atomic_disable,
 };
 
 static irqreturn_t tegra_dc_irq(int irq, void *data)
diff --git a/drivers/gpu/drm/tegra/dpaux.c b/drivers/gpu/drm/tegra/dpaux.c
index 2fde44c3a1b3..e4da041ba89b 100644
--- a/drivers/gpu/drm/tegra/dpaux.c
+++ b/drivers/gpu/drm/tegra/dpaux.c
@@ -25,6 +25,7 @@
 
 #include "dpaux.h"
 #include "drm.h"
+#include "trace.h"
 
 static DEFINE_MUTEX(dpaux_lock);
 static LIST_HEAD(dpaux_list);
@@ -65,14 +66,19 @@ static inline struct tegra_dpaux *work_to_dpaux(struct work_struct *work)
 }
 
 static inline u32 tegra_dpaux_readl(struct tegra_dpaux *dpaux,
-				    unsigned long offset)
+				    unsigned int offset)
 {
-	return readl(dpaux->regs + (offset << 2));
+	u32 value = readl(dpaux->regs + (offset << 2));
+
+	trace_dpaux_readl(dpaux->dev, offset, value);
+
+	return value;
 }
 
 static inline void tegra_dpaux_writel(struct tegra_dpaux *dpaux,
-				      u32 value, unsigned long offset)
+				      u32 value, unsigned int offset)
 {
+	trace_dpaux_writel(dpaux->dev, offset, value);
 	writel(value, dpaux->regs + (offset << 2));
 }
 
diff --git a/drivers/gpu/drm/tegra/drm.c b/drivers/gpu/drm/tegra/drm.c
index 518f4b69ea53..597d563d636a 100644
--- a/drivers/gpu/drm/tegra/drm.c
+++ b/drivers/gpu/drm/tegra/drm.c
@@ -100,7 +100,12 @@ static int tegra_atomic_commit(struct drm_device *drm,
 	 * the software side now.
 	 */
 
-	drm_atomic_helper_swap_state(state, true);
+	err = drm_atomic_helper_swap_state(state, true);
+	if (err) {
+		mutex_unlock(&tegra->commit.lock);
+		drm_atomic_helper_cleanup_planes(drm, state);
+		return err;
+	}
 
 	drm_atomic_state_get(state);
 	if (nonblock)
@@ -214,12 +219,10 @@ static int tegra_drm_load(struct drm_device *drm, unsigned long flags)
 
 	err = tegra_drm_fb_init(drm);
 	if (err < 0)
-		goto vblank;
+		goto device;
 
 	return 0;
 
-vblank:
-	drm_vblank_cleanup(drm);
 device:
 	host1x_device_exit(device);
 fbdev:
@@ -248,7 +251,6 @@ static void tegra_drm_unload(struct drm_device *drm)
 	drm_kms_helper_poll_fini(drm);
 	tegra_drm_fb_exit(drm);
 	drm_mode_config_cleanup(drm);
-	drm_vblank_cleanup(drm);
 
 	err = host1x_device_exit(device);
 	if (err < 0)
@@ -304,8 +306,6 @@ host1x_bo_lookup(struct drm_file *file, u32 handle)
 	if (!gem)
 		return NULL;
 
-	drm_gem_object_unreference_unlocked(gem);
-
 	bo = to_tegra_bo(gem);
 	return &bo->base;
 }
@@ -394,8 +394,10 @@ int tegra_drm_submit(struct tegra_drm_context *context,
 		(void __user *)(uintptr_t)args->waitchks;
 	struct drm_tegra_syncpt syncpt;
 	struct host1x *host1x = dev_get_drvdata(drm->dev->parent);
+	struct drm_gem_object **refs;
 	struct host1x_syncpt *sp;
 	struct host1x_job *job;
+	unsigned int num_refs;
 	int err;
 
 	/* We don't yet support other than one syncpt_incr struct per submit */
@@ -417,6 +419,21 @@ int tegra_drm_submit(struct tegra_drm_context *context,
 	job->class = context->client->base.class;
 	job->serialize = true;
 
+	/*
+	 * Track referenced BOs so that they can be unreferenced after the
+	 * submission is complete.
+	 */
+	num_refs = num_cmdbufs + num_relocs * 2 + num_waitchks;
+
+	refs = kmalloc_array(num_refs, sizeof(*refs), GFP_KERNEL);
+	if (!refs) {
+		err = -ENOMEM;
+		goto put;
+	}
+
+	/* reuse as an iterator later */
+	num_refs = 0;
+
 	while (num_cmdbufs) {
 		struct drm_tegra_cmdbuf cmdbuf;
 		struct host1x_bo *bo;
@@ -445,6 +462,7 @@ int tegra_drm_submit(struct tegra_drm_context *context,
 
 		offset = (u64)cmdbuf.offset + (u64)cmdbuf.words * sizeof(u32);
 		obj = host1x_to_tegra_bo(bo);
+		refs[num_refs++] = &obj->gem;
 
 		/*
 		 * Gather buffer base address must be 4-bytes aligned,
@@ -474,6 +492,7 @@ int tegra_drm_submit(struct tegra_drm_context *context,
 
 		reloc = &job->relocarray[num_relocs];
 		obj = host1x_to_tegra_bo(reloc->cmdbuf.bo);
+		refs[num_refs++] = &obj->gem;
 
 		/*
 		 * The unaligned cmdbuf offset will cause an unaligned write
@@ -487,6 +506,7 @@ int tegra_drm_submit(struct tegra_drm_context *context,
 		}
 
 		obj = host1x_to_tegra_bo(reloc->target.bo);
+		refs[num_refs++] = &obj->gem;
 
 		if (reloc->target.offset >= obj->gem.size) {
 			err = -EINVAL;
@@ -506,6 +526,7 @@ int tegra_drm_submit(struct tegra_drm_context *context,
 			goto fail;
 
 		obj = host1x_to_tegra_bo(wait->bo);
+		refs[num_refs++] = &obj->gem;
 
 		/*
 		 * The unaligned offset will cause an unaligned write during
@@ -545,17 +566,20 @@ int tegra_drm_submit(struct tegra_drm_context *context,
 		goto fail;
 
 	err = host1x_job_submit(job);
-	if (err)
-		goto fail_submit;
+	if (err) {
+		host1x_job_unpin(job);
+		goto fail;
+	}
 
 	args->fence = job->syncpt_end;
 
-	host1x_job_put(job);
-	return 0;
-
-fail_submit:
-	host1x_job_unpin(job);
 fail:
+	while (num_refs--)
+		drm_gem_object_put_unlocked(refs[num_refs]);
+
+	kfree(refs);
+
+put:
 	host1x_job_put(job);
 	return err;
 }
@@ -591,7 +615,7 @@ static int tegra_gem_mmap(struct drm_device *drm, void *data,
 
 	args->offset = drm_vma_node_offset_addr(&bo->gem.vma_node);
 
-	drm_gem_object_unreference_unlocked(gem);
+	drm_gem_object_put_unlocked(gem);
 
 	return 0;
 }
@@ -858,7 +882,7 @@ static int tegra_gem_set_tiling(struct drm_device *drm, void *data,
 	bo->tiling.mode = mode;
 	bo->tiling.value = value;
 
-	drm_gem_object_unreference_unlocked(gem);
+	drm_gem_object_put_unlocked(gem);
 
 	return 0;
 }
@@ -898,7 +922,7 @@ static int tegra_gem_get_tiling(struct drm_device *drm, void *data,
 		break;
 	}
 
-	drm_gem_object_unreference_unlocked(gem);
+	drm_gem_object_put_unlocked(gem);
 
 	return err;
 }
@@ -923,7 +947,7 @@ static int tegra_gem_set_flags(struct drm_device *drm, void *data,
 	if (args->flags & DRM_TEGRA_GEM_BOTTOM_UP)
 		bo->flags |= TEGRA_BO_BOTTOM_UP;
 
-	drm_gem_object_unreference_unlocked(gem);
+	drm_gem_object_put_unlocked(gem);
 
 	return 0;
 }
@@ -945,7 +969,7 @@ static int tegra_gem_get_flags(struct drm_device *drm, void *data,
 	if (bo->flags & TEGRA_BO_BOTTOM_UP)
 		args->flags |= DRM_TEGRA_GEM_BOTTOM_UP;
 
-	drm_gem_object_unreference_unlocked(gem);
+	drm_gem_object_put_unlocked(gem);
 
 	return 0;
 }
@@ -953,20 +977,34 @@ static int tegra_gem_get_flags(struct drm_device *drm, void *data,
 
 static const struct drm_ioctl_desc tegra_drm_ioctls[] = {
 #ifdef CONFIG_DRM_TEGRA_STAGING
-	DRM_IOCTL_DEF_DRV(TEGRA_GEM_CREATE, tegra_gem_create, 0),
-	DRM_IOCTL_DEF_DRV(TEGRA_GEM_MMAP, tegra_gem_mmap, 0),
-	DRM_IOCTL_DEF_DRV(TEGRA_SYNCPT_READ, tegra_syncpt_read, 0),
-	DRM_IOCTL_DEF_DRV(TEGRA_SYNCPT_INCR, tegra_syncpt_incr, 0),
-	DRM_IOCTL_DEF_DRV(TEGRA_SYNCPT_WAIT, tegra_syncpt_wait, 0),
-	DRM_IOCTL_DEF_DRV(TEGRA_OPEN_CHANNEL, tegra_open_channel, 0),
-	DRM_IOCTL_DEF_DRV(TEGRA_CLOSE_CHANNEL, tegra_close_channel, 0),
-	DRM_IOCTL_DEF_DRV(TEGRA_GET_SYNCPT, tegra_get_syncpt, 0),
-	DRM_IOCTL_DEF_DRV(TEGRA_SUBMIT, tegra_submit, 0),
-	DRM_IOCTL_DEF_DRV(TEGRA_GET_SYNCPT_BASE, tegra_get_syncpt_base, 0),
-	DRM_IOCTL_DEF_DRV(TEGRA_GEM_SET_TILING, tegra_gem_set_tiling, 0),
-	DRM_IOCTL_DEF_DRV(TEGRA_GEM_GET_TILING, tegra_gem_get_tiling, 0),
-	DRM_IOCTL_DEF_DRV(TEGRA_GEM_SET_FLAGS, tegra_gem_set_flags, 0),
-	DRM_IOCTL_DEF_DRV(TEGRA_GEM_GET_FLAGS, tegra_gem_get_flags, 0),
+	DRM_IOCTL_DEF_DRV(TEGRA_GEM_CREATE, tegra_gem_create,
+			  DRM_UNLOCKED | DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(TEGRA_GEM_MMAP, tegra_gem_mmap,
+			  DRM_UNLOCKED | DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(TEGRA_SYNCPT_READ, tegra_syncpt_read,
+			  DRM_UNLOCKED | DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(TEGRA_SYNCPT_INCR, tegra_syncpt_incr,
+			  DRM_UNLOCKED | DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(TEGRA_SYNCPT_WAIT, tegra_syncpt_wait,
+			  DRM_UNLOCKED | DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(TEGRA_OPEN_CHANNEL, tegra_open_channel,
+			  DRM_UNLOCKED | DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(TEGRA_CLOSE_CHANNEL, tegra_close_channel,
+			  DRM_UNLOCKED | DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(TEGRA_GET_SYNCPT, tegra_get_syncpt,
+			  DRM_UNLOCKED | DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(TEGRA_SUBMIT, tegra_submit,
+			  DRM_UNLOCKED | DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(TEGRA_GET_SYNCPT_BASE, tegra_get_syncpt_base,
+			  DRM_UNLOCKED | DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(TEGRA_GEM_SET_TILING, tegra_gem_set_tiling,
+			  DRM_UNLOCKED | DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(TEGRA_GEM_GET_TILING, tegra_gem_get_tiling,
+			  DRM_UNLOCKED | DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(TEGRA_GEM_SET_FLAGS, tegra_gem_set_flags,
+			  DRM_UNLOCKED | DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(TEGRA_GEM_GET_FLAGS, tegra_gem_get_flags,
+			  DRM_UNLOCKED | DRM_RENDER_ALLOW),
 #endif
 };
 
@@ -1033,9 +1071,11 @@ static int tegra_debugfs_iova(struct seq_file *s, void *data)
 	struct tegra_drm *tegra = drm->dev_private;
 	struct drm_printer p = drm_seq_file_printer(s);
 
-	mutex_lock(&tegra->mm_lock);
-	drm_mm_print(&tegra->mm, &p);
-	mutex_unlock(&tegra->mm_lock);
+	if (tegra->domain) {
+		mutex_lock(&tegra->mm_lock);
+		drm_mm_print(&tegra->mm, &p);
+		mutex_unlock(&tegra->mm_lock);
+	}
 
 	return 0;
 }
@@ -1055,7 +1095,7 @@ static int tegra_debugfs_init(struct drm_minor *minor)
 
 static struct drm_driver tegra_drm_driver = {
 	.driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME |
-			   DRIVER_ATOMIC,
+			   DRIVER_ATOMIC | DRIVER_RENDER,
 	.load = tegra_drm_load,
 	.unload = tegra_drm_unload,
 	.open = tegra_drm_open,
@@ -1075,8 +1115,6 @@ static struct drm_driver tegra_drm_driver = {
 	.gem_prime_import = tegra_gem_prime_import,
 
 	.dumb_create = tegra_bo_dumb_create,
-	.dumb_map_offset = tegra_bo_dumb_map_offset,
-	.dumb_destroy = drm_gem_dumb_destroy,
 
 	.ioctls = tegra_drm_ioctls,
 	.num_ioctls = ARRAY_SIZE(tegra_drm_ioctls),
diff --git a/drivers/gpu/drm/tegra/drm.h b/drivers/gpu/drm/tegra/drm.h
index 6d6da01282f3..063f5d397526 100644
--- a/drivers/gpu/drm/tegra/drm.h
+++ b/drivers/gpu/drm/tegra/drm.h
@@ -23,6 +23,7 @@
 #include <drm/drm_fixed.h>
 
 #include "gem.h"
+#include "trace.h"
 
 struct reset_control;
 
@@ -172,14 +173,19 @@ static inline struct tegra_dc *to_tegra_dc(struct drm_crtc *crtc)
 }
 
 static inline void tegra_dc_writel(struct tegra_dc *dc, u32 value,
-				   unsigned long offset)
+				   unsigned int offset)
 {
+	trace_dc_writel(dc->dev, offset, value);
 	writel(value, dc->regs + (offset << 2));
 }
 
-static inline u32 tegra_dc_readl(struct tegra_dc *dc, unsigned long offset)
+static inline u32 tegra_dc_readl(struct tegra_dc *dc, unsigned int offset)
 {
-	return readl(dc->regs + (offset << 2));
+	u32 value = readl(dc->regs + (offset << 2));
+
+	trace_dc_readl(dc->dev, offset, value);
+
+	return value;
 }
 
 struct tegra_dc_window {
diff --git a/drivers/gpu/drm/tegra/dsi.c b/drivers/gpu/drm/tegra/dsi.c
index 3dea1216bafd..046649ec9441 100644
--- a/drivers/gpu/drm/tegra/dsi.c
+++ b/drivers/gpu/drm/tegra/dsi.c
@@ -28,6 +28,7 @@
 #include "drm.h"
 #include "dsi.h"
 #include "mipi-phy.h"
+#include "trace.h"
 
 struct tegra_dsi_state {
 	struct drm_connector_state base;
@@ -105,15 +106,20 @@ static struct tegra_dsi_state *tegra_dsi_get_state(struct tegra_dsi *dsi)
 	return to_dsi_state(dsi->output.connector.state);
 }
 
-static inline u32 tegra_dsi_readl(struct tegra_dsi *dsi, unsigned long reg)
+static inline u32 tegra_dsi_readl(struct tegra_dsi *dsi, unsigned int offset)
 {
-	return readl(dsi->regs + (reg << 2));
+	u32 value = readl(dsi->regs + (offset << 2));
+
+	trace_dsi_readl(dsi->dev, offset, value);
+
+	return value;
 }
 
 static inline void tegra_dsi_writel(struct tegra_dsi *dsi, u32 value,
-				    unsigned long reg)
+				    unsigned int offset)
 {
-	writel(value, dsi->regs + (reg << 2));
+	trace_dsi_writel(dsi->dev, offset, value);
+	writel(value, dsi->regs + (offset << 2));
 }
 
 static int tegra_dsi_show_regs(struct seq_file *s, void *data)
@@ -815,7 +821,6 @@ tegra_dsi_connector_duplicate_state(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs tegra_dsi_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.reset = tegra_dsi_connector_reset,
 	.detect = tegra_output_connector_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
diff --git a/drivers/gpu/drm/tegra/fb.c b/drivers/gpu/drm/tegra/fb.c
index 25acb73ee728..80540c1c66dc 100644
--- a/drivers/gpu/drm/tegra/fb.c
+++ b/drivers/gpu/drm/tegra/fb.c
@@ -88,7 +88,7 @@ static void tegra_fb_destroy(struct drm_framebuffer *framebuffer)
 			if (bo->pages)
 				vunmap(bo->vaddr);
 
-			drm_gem_object_unreference_unlocked(&bo->gem);
+			drm_gem_object_put_unlocked(&bo->gem);
 		}
 	}
 
@@ -195,7 +195,7 @@ struct drm_framebuffer *tegra_fb_create(struct drm_device *drm,
 
 unreference:
 	while (i--)
-		drm_gem_object_unreference_unlocked(&planes[i]->gem);
+		drm_gem_object_put_unlocked(&planes[i]->gem);
 
 	return ERR_PTR(err);
 }
@@ -242,7 +242,7 @@ static int tegra_fbdev_probe(struct drm_fb_helper *helper,
 	info = drm_fb_helper_alloc_fbi(helper);
 	if (IS_ERR(info)) {
 		dev_err(drm->dev, "failed to allocate framebuffer info\n");
-		drm_gem_object_unreference_unlocked(&bo->gem);
+		drm_gem_object_put_unlocked(&bo->gem);
 		return PTR_ERR(info);
 	}
 
@@ -251,7 +251,7 @@ static int tegra_fbdev_probe(struct drm_fb_helper *helper,
 		err = PTR_ERR(fbdev->fb);
 		dev_err(drm->dev, "failed to allocate DRM framebuffer: %d\n",
 			err);
-		drm_gem_object_unreference_unlocked(&bo->gem);
+		drm_gem_object_put_unlocked(&bo->gem);
 		return PTR_ERR(fbdev->fb);
 	}
 
diff --git a/drivers/gpu/drm/tegra/gem.c b/drivers/gpu/drm/tegra/gem.c
index 7a39a355678a..ab1e53d434e8 100644
--- a/drivers/gpu/drm/tegra/gem.c
+++ b/drivers/gpu/drm/tegra/gem.c
@@ -24,7 +24,7 @@ static void tegra_bo_put(struct host1x_bo *bo)
 {
 	struct tegra_bo *obj = host1x_to_tegra_bo(bo);
 
-	drm_gem_object_unreference_unlocked(&obj->gem);
+	drm_gem_object_put_unlocked(&obj->gem);
 }
 
 static dma_addr_t tegra_bo_pin(struct host1x_bo *bo, struct sg_table **sgt)
@@ -95,7 +95,7 @@ static struct host1x_bo *tegra_bo_get(struct host1x_bo *bo)
 {
 	struct tegra_bo *obj = host1x_to_tegra_bo(bo);
 
-	drm_gem_object_reference(&obj->gem);
+	drm_gem_object_get(&obj->gem);
 
 	return bo;
 }
@@ -325,7 +325,7 @@ struct tegra_bo *tegra_bo_create_with_handle(struct drm_file *file,
 		return ERR_PTR(err);
 	}
 
-	drm_gem_object_unreference_unlocked(&bo->gem);
+	drm_gem_object_put_unlocked(&bo->gem);
 
 	return bo;
 }
@@ -423,27 +423,6 @@ int tegra_bo_dumb_create(struct drm_file *file, struct drm_device *drm,
 	return 0;
 }
 
-int tegra_bo_dumb_map_offset(struct drm_file *file, struct drm_device *drm,
-			     u32 handle, u64 *offset)
-{
-	struct drm_gem_object *gem;
-	struct tegra_bo *bo;
-
-	gem = drm_gem_object_lookup(file, handle);
-	if (!gem) {
-		dev_err(drm->dev, "failed to lookup GEM object\n");
-		return -EINVAL;
-	}
-
-	bo = to_tegra_bo(gem);
-
-	*offset = drm_vma_node_offset_addr(&bo->gem.vma_node);
-
-	drm_gem_object_unreference_unlocked(gem);
-
-	return 0;
-}
-
 static int tegra_bo_fault(struct vm_fault *vmf)
 {
 	struct vm_area_struct *vma = vmf->vma;
@@ -481,30 +460,28 @@ const struct vm_operations_struct tegra_bo_vm_ops = {
 	.close = drm_gem_vm_close,
 };
 
-int tegra_drm_mmap(struct file *file, struct vm_area_struct *vma)
+static int tegra_gem_mmap(struct drm_gem_object *gem,
+			  struct vm_area_struct *vma)
 {
-	struct drm_gem_object *gem;
-	struct tegra_bo *bo;
-	int ret;
-
-	ret = drm_gem_mmap(file, vma);
-	if (ret)
-		return ret;
-
-	gem = vma->vm_private_data;
-	bo = to_tegra_bo(gem);
+	struct tegra_bo *bo = to_tegra_bo(gem);
 
 	if (!bo->pages) {
 		unsigned long vm_pgoff = vma->vm_pgoff;
+		int err;
 
+		/*
+		 * Clear the VM_PFNMAP flag that was set by drm_gem_mmap(),
+		 * and set the vm_pgoff (used as a fake buffer offset by DRM)
+		 * to 0 as we want to map the whole buffer.
+		 */
 		vma->vm_flags &= ~VM_PFNMAP;
 		vma->vm_pgoff = 0;
 
-		ret = dma_mmap_wc(gem->dev->dev, vma, bo->vaddr, bo->paddr,
+		err = dma_mmap_wc(gem->dev->dev, vma, bo->vaddr, bo->paddr,
 				  gem->size);
-		if (ret) {
+		if (err < 0) {
 			drm_gem_vm_close(vma);
-			return ret;
+			return err;
 		}
 
 		vma->vm_pgoff = vm_pgoff;
@@ -520,6 +497,20 @@ int tegra_drm_mmap(struct file *file, struct vm_area_struct *vma)
 	return 0;
 }
 
+int tegra_drm_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	struct drm_gem_object *gem;
+	int err;
+
+	err = drm_gem_mmap(file, vma);
+	if (err < 0)
+		return err;
+
+	gem = vma->vm_private_data;
+
+	return tegra_gem_mmap(gem, vma);
+}
+
 static struct sg_table *
 tegra_gem_prime_map_dma_buf(struct dma_buf_attachment *attach,
 			    enum dma_data_direction dir)
@@ -603,7 +594,14 @@ static void tegra_gem_prime_kunmap(struct dma_buf *buf, unsigned long page,
 
 static int tegra_gem_prime_mmap(struct dma_buf *buf, struct vm_area_struct *vma)
 {
-	return -EINVAL;
+	struct drm_gem_object *gem = buf->priv;
+	int err;
+
+	err = drm_gem_mmap_obj(gem, gem->size, vma);
+	if (err < 0)
+		return err;
+
+	return tegra_gem_mmap(gem, vma);
 }
 
 static void *tegra_gem_prime_vmap(struct dma_buf *buf)
@@ -654,7 +652,7 @@ struct drm_gem_object *tegra_gem_prime_import(struct drm_device *drm,
 		struct drm_gem_object *gem = buf->priv;
 
 		if (gem->dev == drm) {
-			drm_gem_object_reference(gem);
+			drm_gem_object_get(gem);
 			return gem;
 		}
 	}
diff --git a/drivers/gpu/drm/tegra/gem.h b/drivers/gpu/drm/tegra/gem.h
index 8b32a6fd586d..8eb9fd24ef0e 100644
--- a/drivers/gpu/drm/tegra/gem.h
+++ b/drivers/gpu/drm/tegra/gem.h
@@ -67,8 +67,6 @@ struct tegra_bo *tegra_bo_create_with_handle(struct drm_file *file,
 void tegra_bo_free_object(struct drm_gem_object *gem);
 int tegra_bo_dumb_create(struct drm_file *file, struct drm_device *drm,
 			 struct drm_mode_create_dumb *args);
-int tegra_bo_dumb_map_offset(struct drm_file *file, struct drm_device *drm,
-			     u32 handle, u64 *offset);
 
 int tegra_drm_mmap(struct file *file, struct vm_area_struct *vma);
 
diff --git a/drivers/gpu/drm/tegra/hdmi.c b/drivers/gpu/drm/tegra/hdmi.c
index cda0491ed6bf..5b9d83b71943 100644
--- a/drivers/gpu/drm/tegra/hdmi.c
+++ b/drivers/gpu/drm/tegra/hdmi.c
@@ -24,6 +24,7 @@
 #include "hdmi.h"
 #include "drm.h"
 #include "dc.h"
+#include "trace.h"
 
 #define HDMI_ELD_BUFFER_SIZE 96
 
@@ -100,14 +101,19 @@ enum {
 };
 
 static inline u32 tegra_hdmi_readl(struct tegra_hdmi *hdmi,
-				   unsigned long offset)
+				   unsigned int offset)
 {
-	return readl(hdmi->regs + (offset << 2));
+	u32 value = readl(hdmi->regs + (offset << 2));
+
+	trace_hdmi_readl(hdmi->dev, offset, value);
+
+	return value;
 }
 
 static inline void tegra_hdmi_writel(struct tegra_hdmi *hdmi, u32 value,
-				     unsigned long offset)
+				     unsigned int offset)
 {
+	trace_hdmi_writel(hdmi->dev, offset, value);
 	writel(value, hdmi->regs + (offset << 2));
 }
 
@@ -734,7 +740,7 @@ static void tegra_hdmi_setup_avi_infoframe(struct tegra_hdmi *hdmi,
 	u8 buffer[17];
 	ssize_t err;
 
-	err = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode);
+	err = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode, false);
 	if (err < 0) {
 		dev_err(hdmi->dev, "failed to setup AVI infoframe: %zd\n", err);
 		return;
@@ -902,7 +908,6 @@ tegra_hdmi_connector_detect(struct drm_connector *connector, bool force)
 }
 
 static const struct drm_connector_funcs tegra_hdmi_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.reset = drm_atomic_helper_connector_reset,
 	.detect = tegra_hdmi_connector_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
diff --git a/drivers/gpu/drm/tegra/rgb.c b/drivers/gpu/drm/tegra/rgb.c
index a131b44e2d6f..78ec5193741d 100644
--- a/drivers/gpu/drm/tegra/rgb.c
+++ b/drivers/gpu/drm/tegra/rgb.c
@@ -88,7 +88,6 @@ static void tegra_dc_write_regs(struct tegra_dc *dc,
 }
 
 static const struct drm_connector_funcs tegra_rgb_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.reset = drm_atomic_helper_connector_reset,
 	.detect = tegra_output_connector_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
diff --git a/drivers/gpu/drm/tegra/sor.c b/drivers/gpu/drm/tegra/sor.c
index a8f528925009..7ab1d1dc7cd7 100644
--- a/drivers/gpu/drm/tegra/sor.c
+++ b/drivers/gpu/drm/tegra/sor.c
@@ -26,6 +26,7 @@
 #include "dc.h"
 #include "drm.h"
 #include "sor.h"
+#include "trace.h"
 
 #define SOR_REKEY 0x38
 
@@ -232,14 +233,19 @@ static inline struct tegra_sor *to_sor(struct tegra_output *output)
 	return container_of(output, struct tegra_sor, output);
 }
 
-static inline u32 tegra_sor_readl(struct tegra_sor *sor, unsigned long offset)
+static inline u32 tegra_sor_readl(struct tegra_sor *sor, unsigned int offset)
 {
-	return readl(sor->regs + (offset << 2));
+	u32 value = readl(sor->regs + (offset << 2));
+
+	trace_sor_readl(sor->dev, offset, value);
+
+	return value;
 }
 
 static inline void tegra_sor_writel(struct tegra_sor *sor, u32 value,
-				    unsigned long offset)
+				    unsigned int offset)
 {
+	trace_sor_writel(sor->dev, offset, value);
 	writel(value, sor->regs + (offset << 2));
 }
 
@@ -1340,7 +1346,6 @@ tegra_sor_connector_duplicate_state(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs tegra_sor_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.reset = tegra_sor_connector_reset,
 	.detect = tegra_sor_connector_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
@@ -1904,7 +1909,7 @@ tegra_sor_hdmi_setup_avi_infoframe(struct tegra_sor *sor,
 	value &= ~INFOFRAME_CTRL_ENABLE;
 	tegra_sor_writel(sor, value, SOR_HDMI_AVI_INFOFRAME_CTRL);
 
-	err = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode);
+	err = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode, false);
 	if (err < 0) {
 		dev_err(sor->dev, "failed to setup AVI infoframe: %d\n", err);
 		return err;
diff --git a/drivers/gpu/drm/tegra/trace.c b/drivers/gpu/drm/tegra/trace.c
new file mode 100644
index 000000000000..006f65c72a34
--- /dev/null
+++ b/drivers/gpu/drm/tegra/trace.c
@@ -0,0 +1,2 @@
+#define CREATE_TRACE_POINTS
+#include "trace.h"
diff --git a/drivers/gpu/drm/tegra/trace.h b/drivers/gpu/drm/tegra/trace.h
new file mode 100644
index 000000000000..e9b7cdad5c4c
--- /dev/null
+++ b/drivers/gpu/drm/tegra/trace.h
@@ -0,0 +1,68 @@
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM tegra
+
+#if !defined(DRM_TEGRA_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)
+#define DRM_TEGRA_TRACE_H 1
+
+#include <linux/device.h>
+#include <linux/tracepoint.h>
+
+DECLARE_EVENT_CLASS(register_access,
+	TP_PROTO(struct device *dev, unsigned int offset, u32 value),
+	TP_ARGS(dev, offset, value),
+	TP_STRUCT__entry(
+		__field(struct device *, dev)
+		__field(unsigned int, offset)
+		__field(u32, value)
+	),
+	TP_fast_assign(
+		__entry->dev = dev;
+		__entry->offset = offset;
+		__entry->value = value;
+	),
+	TP_printk("%s %04x %08x", dev_name(__entry->dev), __entry->offset,
+		  __entry->value)
+);
+
+DEFINE_EVENT(register_access, dc_writel,
+	TP_PROTO(struct device *dev, unsigned int offset, u32 value),
+	TP_ARGS(dev, offset, value));
+DEFINE_EVENT(register_access, dc_readl,
+	TP_PROTO(struct device *dev, unsigned int offset, u32 value),
+	TP_ARGS(dev, offset, value));
+
+DEFINE_EVENT(register_access, hdmi_writel,
+	TP_PROTO(struct device *dev, unsigned int offset, u32 value),
+	TP_ARGS(dev, offset, value));
+DEFINE_EVENT(register_access, hdmi_readl,
+	TP_PROTO(struct device *dev, unsigned int offset, u32 value),
+	TP_ARGS(dev, offset, value));
+
+DEFINE_EVENT(register_access, dsi_writel,
+	TP_PROTO(struct device *dev, unsigned int offset, u32 value),
+	TP_ARGS(dev, offset, value));
+DEFINE_EVENT(register_access, dsi_readl,
+	TP_PROTO(struct device *dev, unsigned int offset, u32 value),
+	TP_ARGS(dev, offset, value));
+
+DEFINE_EVENT(register_access, dpaux_writel,
+	TP_PROTO(struct device *dev, unsigned int offset, u32 value),
+	TP_ARGS(dev, offset, value));
+DEFINE_EVENT(register_access, dpaux_readl,
+	TP_PROTO(struct device *dev, unsigned int offset, u32 value),
+	TP_ARGS(dev, offset, value));
+
+DEFINE_EVENT(register_access, sor_writel,
+	TP_PROTO(struct device *dev, unsigned int offset, u32 value),
+	TP_ARGS(dev, offset, value));
+DEFINE_EVENT(register_access, sor_readl,
+	TP_PROTO(struct device *dev, unsigned int offset, u32 value),
+	TP_ARGS(dev, offset, value));
+
+#endif /* DRM_TEGRA_TRACE_H */
+
+/* This part must be outside protection */
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#define TRACE_INCLUDE_FILE trace
+#include <trace/define_trace.h>
diff --git a/drivers/gpu/drm/tegra/vic.c b/drivers/gpu/drm/tegra/vic.c
index 47cb1aaa58b1..2448229fa653 100644
--- a/drivers/gpu/drm/tegra/vic.c
+++ b/drivers/gpu/drm/tegra/vic.c
@@ -258,12 +258,16 @@ static const struct tegra_drm_client_ops vic_ops = {
 	.submit = tegra_drm_submit,
 };
 
+#define NVIDIA_TEGRA_124_VIC_FIRMWARE "nvidia/tegra124/vic03_ucode.bin"
+
 static const struct vic_config vic_t124_config = {
-	.firmware = "nvidia/tegra124/vic03_ucode.bin",
+	.firmware = NVIDIA_TEGRA_124_VIC_FIRMWARE,
 };
 
+#define NVIDIA_TEGRA_210_VIC_FIRMWARE "nvidia/tegra210/vic04_ucode.bin"
+
 static const struct vic_config vic_t210_config = {
-	.firmware = "nvidia/tegra210/vic04_ucode.bin",
+	.firmware = NVIDIA_TEGRA_210_VIC_FIRMWARE,
 };
 
 static const struct of_device_id vic_match[] = {
@@ -394,3 +398,10 @@ struct platform_driver tegra_vic_driver = {
 	.probe = vic_probe,
 	.remove = vic_remove,
 };
+
+#if IS_ENABLED(CONFIG_ARCH_TEGRA_124_SOC)
+MODULE_FIRMWARE(NVIDIA_TEGRA_124_VIC_FIRMWARE);
+#endif
+#if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC)
+MODULE_FIRMWARE(NVIDIA_TEGRA_210_VIC_FIRMWARE);
+#endif
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
index d524ed0d5146..406fe4544b83 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
@@ -504,6 +504,12 @@ static void tilcdc_crtc_enable(struct drm_crtc *crtc)
 	mutex_unlock(&tilcdc_crtc->enable_lock);
 }
 
+static void tilcdc_crtc_atomic_enable(struct drm_crtc *crtc,
+				      struct drm_crtc_state *old_state)
+{
+	tilcdc_crtc_enable(crtc);
+}
+
 static void tilcdc_crtc_off(struct drm_crtc *crtc, bool shutdown)
 {
 	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
@@ -562,6 +568,12 @@ static void tilcdc_crtc_disable(struct drm_crtc *crtc)
 	tilcdc_crtc_off(crtc, false);
 }
 
+static void tilcdc_crtc_atomic_disable(struct drm_crtc *crtc,
+				       struct drm_crtc_state *old_state)
+{
+	tilcdc_crtc_disable(crtc);
+}
+
 void tilcdc_crtc_shutdown(struct drm_crtc *crtc)
 {
 	tilcdc_crtc_off(crtc, true);
@@ -729,9 +741,9 @@ static const struct drm_crtc_funcs tilcdc_crtc_funcs = {
 
 static const struct drm_crtc_helper_funcs tilcdc_crtc_helper_funcs = {
 		.mode_fixup     = tilcdc_crtc_mode_fixup,
-		.enable		= tilcdc_crtc_enable,
-		.disable	= tilcdc_crtc_disable,
 		.atomic_check	= tilcdc_crtc_atomic_check,
+		.atomic_enable	= tilcdc_crtc_atomic_enable,
+		.atomic_disable	= tilcdc_crtc_atomic_disable,
 };
 
 int tilcdc_crtc_max_width(struct drm_crtc *crtc)
@@ -1038,8 +1050,8 @@ int tilcdc_crtc_create(struct drm_device *dev)
 	if (priv->is_componentized) {
 		crtc->port = of_graph_get_port_by_id(dev->dev->of_node, 0);
 		if (!crtc->port) { /* This should never happen */
-			dev_err(dev->dev, "Port node not found in %s\n",
-				dev->dev->of_node->full_name);
+			dev_err(dev->dev, "Port node not found in %pOF\n",
+				dev->dev->of_node);
 			ret = -EINVAL;
 			goto fail;
 		}
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_drv.c b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
index d67e18983a7d..b0d70f943cec 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_drv.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
@@ -108,7 +108,11 @@ static int tilcdc_commit(struct drm_device *dev,
 	if (ret)
 		return ret;
 
-	drm_atomic_helper_swap_state(state, true);
+	ret = drm_atomic_helper_swap_state(state, true);
+	if (ret) {
+		drm_atomic_helper_cleanup_planes(dev, state);
+		return ret;
+	}
 
 	/*
 	 * Everything below can be run asynchronously without the need to grab
@@ -538,8 +542,6 @@ static struct drm_driver tilcdc_driver = {
 	.gem_free_object_unlocked = drm_gem_cma_free_object,
 	.gem_vm_ops         = &drm_gem_cma_vm_ops,
 	.dumb_create        = drm_gem_cma_dumb_create,
-	.dumb_map_offset    = drm_gem_cma_dumb_map_offset,
-	.dumb_destroy       = drm_gem_dumb_destroy,
 
 	.prime_handle_to_fd	= drm_gem_prime_handle_to_fd,
 	.prime_fd_to_handle	= drm_gem_prime_fd_to_handle,
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_panel.c b/drivers/gpu/drm/tilcdc/tilcdc_panel.c
index 28c3e2f44f64..1813a3623ce6 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_panel.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_panel.c
@@ -189,7 +189,6 @@ static struct drm_encoder *panel_connector_best_encoder(
 
 static const struct drm_connector_funcs panel_connector_funcs = {
 	.destroy            = panel_connector_destroy,
-	.dpms               = drm_atomic_helper_connector_dpms,
 	.fill_modes         = drm_helper_probe_single_connector_modes,
 	.reset              = drm_atomic_helper_connector_reset,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_plane.c b/drivers/gpu/drm/tilcdc/tilcdc_plane.c
index ba0d66c0d8ac..7667b038ae7f 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_plane.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_plane.c
@@ -28,7 +28,6 @@ static struct drm_plane_funcs tilcdc_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
 	.destroy	= drm_plane_cleanup,
-	.set_property	= drm_atomic_helper_plane_set_property,
 	.reset		= drm_atomic_helper_plane_reset,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c b/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c
index aabfad882e23..1e2dfb1b1d6b 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_tfp410.c
@@ -202,7 +202,6 @@ static struct drm_encoder *tfp410_connector_best_encoder(
 
 static const struct drm_connector_funcs tfp410_connector_funcs = {
 	.destroy            = tfp410_connector_destroy,
-	.dpms               = drm_atomic_helper_connector_dpms,
 	.detect             = tfp410_connector_detect,
 	.fill_modes         = drm_helper_probe_single_connector_modes,
 	.reset              = drm_atomic_helper_connector_reset,
diff --git a/drivers/gpu/drm/tinydrm/Kconfig b/drivers/gpu/drm/tinydrm/Kconfig
index 3504c53846da..2e790e7dced5 100644
--- a/drivers/gpu/drm/tinydrm/Kconfig
+++ b/drivers/gpu/drm/tinydrm/Kconfig
@@ -19,3 +19,26 @@ config TINYDRM_MI0283QT
 	help
 	  DRM driver for the Multi-Inno MI0283QT display panel
 	  If M is selected the module will be called mi0283qt.
+
+config TINYDRM_REPAPER
+	tristate "DRM support for Pervasive Displays RePaper panels (V231)"
+	depends on DRM_TINYDRM && SPI
+	depends on THERMAL || !THERMAL
+	help
+	  DRM driver for the following Pervasive Displays panels:
+	  1.44" TFT EPD Panel (E1144CS021)
+	  1.90" TFT EPD Panel (E1190CS021)
+	  2.00" TFT EPD Panel (E2200CS021)
+	  2.71" TFT EPD Panel (E2271CS021)
+
+	  If M is selected the module will be called repaper.
+
+config TINYDRM_ST7586
+	tristate "DRM support for Sitronix ST7586 display panels"
+	depends on DRM_TINYDRM && SPI
+	select TINYDRM_MIPI_DBI
+	help
+	  DRM driver for the following Sitronix ST7586 panels:
+	  * LEGO MINDSTORMS EV3
+
+	  If M is selected the module will be called st7586.
diff --git a/drivers/gpu/drm/tinydrm/Makefile b/drivers/gpu/drm/tinydrm/Makefile
index 7a3604cf4fc2..0c184bd1bb59 100644
--- a/drivers/gpu/drm/tinydrm/Makefile
+++ b/drivers/gpu/drm/tinydrm/Makefile
@@ -5,3 +5,5 @@ obj-$(CONFIG_TINYDRM_MIPI_DBI)		+= mipi-dbi.o
 
 # Displays
 obj-$(CONFIG_TINYDRM_MI0283QT)		+= mi0283qt.o
+obj-$(CONFIG_TINYDRM_REPAPER)		+= repaper.o
+obj-$(CONFIG_TINYDRM_ST7586)		+= st7586.o
diff --git a/drivers/gpu/drm/tinydrm/core/tinydrm-helpers.c b/drivers/gpu/drm/tinydrm/core/tinydrm-helpers.c
index d4cda3308ac7..bd6cce093a85 100644
--- a/drivers/gpu/drm/tinydrm/core/tinydrm-helpers.c
+++ b/drivers/gpu/drm/tinydrm/core/tinydrm-helpers.c
@@ -7,13 +7,15 @@
  * (at your option) any later version.
  */
 
-#include <drm/tinydrm/tinydrm.h>
-#include <drm/tinydrm/tinydrm-helpers.h>
 #include <linux/backlight.h>
+#include <linux/dma-buf.h>
 #include <linux/pm.h>
 #include <linux/spi/spi.h>
 #include <linux/swab.h>
 
+#include <drm/tinydrm/tinydrm.h>
+#include <drm/tinydrm/tinydrm-helpers.h>
+
 static unsigned int spi_max;
 module_param(spi_max, uint, 0400);
 MODULE_PARM_DESC(spi_max, "Set a lower SPI max transfer size");
@@ -181,6 +183,60 @@ void tinydrm_xrgb8888_to_rgb565(u16 *dst, void *vaddr,
 EXPORT_SYMBOL(tinydrm_xrgb8888_to_rgb565);
 
 /**
+ * tinydrm_xrgb8888_to_gray8 - Convert XRGB8888 to grayscale
+ * @dst: 8-bit grayscale destination buffer
+ * @vaddr: XRGB8888 source buffer
+ * @fb: DRM framebuffer
+ * @clip: Clip rectangle area to copy
+ *
+ * Drm doesn't have native monochrome or grayscale support.
+ * Such drivers can announce the commonly supported XR24 format to userspace
+ * and use this function to convert to the native format.
+ *
+ * Monochrome drivers will use the most significant bit,
+ * where 1 means foreground color and 0 background color.
+ *
+ * ITU BT.601 is used for the RGB -> luma (brightness) conversion.
+ */
+void tinydrm_xrgb8888_to_gray8(u8 *dst, void *vaddr, struct drm_framebuffer *fb,
+			       struct drm_clip_rect *clip)
+{
+	unsigned int len = (clip->x2 - clip->x1) * sizeof(u32);
+	unsigned int x, y;
+	void *buf;
+	u32 *src;
+
+	if (WARN_ON(fb->format->format != DRM_FORMAT_XRGB8888))
+		return;
+	/*
+	 * The cma memory is write-combined so reads are uncached.
+	 * Speed up by fetching one line at a time.
+	 */
+	buf = kmalloc(len, GFP_KERNEL);
+	if (!buf)
+		return;
+
+	for (y = clip->y1; y < clip->y2; y++) {
+		src = vaddr + (y * fb->pitches[0]);
+		src += clip->x1;
+		memcpy(buf, src, len);
+		src = buf;
+		for (x = clip->x1; x < clip->x2; x++) {
+			u8 r = (*src & 0x00ff0000) >> 16;
+			u8 g = (*src & 0x0000ff00) >> 8;
+			u8 b =  *src & 0x000000ff;
+
+			/* ITU BT.601: Y = 0.299 R + 0.587 G + 0.114 B */
+			*dst++ = (3 * r + 6 * g + b) / 10;
+			src++;
+		}
+	}
+
+	kfree(buf);
+}
+EXPORT_SYMBOL(tinydrm_xrgb8888_to_gray8);
+
+/**
  * tinydrm_of_find_backlight - Find backlight device in device-tree
  * @dev: Device
  *
diff --git a/drivers/gpu/drm/tinydrm/core/tinydrm-pipe.c b/drivers/gpu/drm/tinydrm/core/tinydrm-pipe.c
index ec43fb7ad9e4..177e9d861001 100644
--- a/drivers/gpu/drm/tinydrm/core/tinydrm-pipe.c
+++ b/drivers/gpu/drm/tinydrm/core/tinydrm-pipe.c
@@ -56,7 +56,7 @@ static const struct drm_connector_helper_funcs tinydrm_connector_hfuncs = {
 static enum drm_connector_status
 tinydrm_connector_detect(struct drm_connector *connector, bool force)
 {
-	if (drm_device_is_unplugged(connector->dev))
+	if (drm_dev_is_unplugged(connector->dev))
 		return connector_status_disconnected;
 
 	return connector->status;
@@ -71,7 +71,6 @@ static void tinydrm_connector_destroy(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs tinydrm_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.reset = drm_atomic_helper_connector_reset,
 	.detect = tinydrm_connector_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
@@ -225,7 +224,7 @@ tinydrm_display_pipe_init(struct tinydrm_device *tdev,
 		return PTR_ERR(connector);
 
 	ret = drm_simple_display_pipe_init(drm, &tdev->pipe, funcs, formats,
-					   format_count, connector);
+					   format_count, NULL, connector);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/tinydrm/mi0283qt.c b/drivers/gpu/drm/tinydrm/mi0283qt.c
index 482ff1c3db61..7e5bb7d6f655 100644
--- a/drivers/gpu/drm/tinydrm/mi0283qt.c
+++ b/drivers/gpu/drm/tinydrm/mi0283qt.c
@@ -195,8 +195,12 @@ static int mi0283qt_probe(struct spi_device *spi)
 
 	device_property_read_u32(dev, "rotation", &rotation);
 
-	ret = mipi_dbi_spi_init(spi, mipi, dc, &mi0283qt_pipe_funcs,
-				&mi0283qt_driver, &mi0283qt_mode, rotation);
+	ret = mipi_dbi_spi_init(spi, mipi, dc);
+	if (ret)
+		return ret;
+
+	ret = mipi_dbi_init(&spi->dev, mipi, &mi0283qt_pipe_funcs,
+			    &mi0283qt_driver, &mi0283qt_mode, rotation);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/tinydrm/mipi-dbi.c b/drivers/gpu/drm/tinydrm/mipi-dbi.c
index c83eeb7a34b0..2caeabcd3458 100644
--- a/drivers/gpu/drm/tinydrm/mipi-dbi.c
+++ b/drivers/gpu/drm/tinydrm/mipi-dbi.c
@@ -776,15 +776,12 @@ static int mipi_dbi_typec3_command(struct mipi_dbi *mipi, u8 cmd,
 /**
  * mipi_dbi_spi_init - Initialize MIPI DBI SPI interfaced controller
  * @spi: SPI device
- * @dc: D/C gpio (optional)
  * @mipi: &mipi_dbi structure to initialize
- * @pipe_funcs: Display pipe functions
- * @driver: DRM driver
- * @mode: Display mode
- * @rotation: Initial rotation in degrees Counter Clock Wise
+ * @dc: D/C gpio (optional)
  *
  * This function sets &mipi_dbi->command, enables &mipi->read_commands for the
- * usual read commands and initializes @mipi using mipi_dbi_init().
+ * usual read commands. It should be followed by a call to mipi_dbi_init() or
+ * a driver-specific init.
  *
  * If @dc is set, a Type C Option 3 interface is assumed, if not
  * Type C Option 1.
@@ -799,11 +796,7 @@ static int mipi_dbi_typec3_command(struct mipi_dbi *mipi, u8 cmd,
  * Zero on success, negative error code on failure.
  */
 int mipi_dbi_spi_init(struct spi_device *spi, struct mipi_dbi *mipi,
-		      struct gpio_desc *dc,
-		      const struct drm_simple_display_pipe_funcs *pipe_funcs,
-		      struct drm_driver *driver,
-		      const struct drm_display_mode *mode,
-		      unsigned int rotation)
+		      struct gpio_desc *dc)
 {
 	size_t tx_size = tinydrm_spi_max_transfer_size(spi, 0);
 	struct device *dev = &spi->dev;
@@ -849,7 +842,7 @@ int mipi_dbi_spi_init(struct spi_device *spi, struct mipi_dbi *mipi,
 			return -ENOMEM;
 	}
 
-	return mipi_dbi_init(dev, mipi, pipe_funcs, driver, mode, rotation);
+	return 0;
 }
 EXPORT_SYMBOL(mipi_dbi_spi_init);
 
diff --git a/drivers/gpu/drm/tinydrm/repaper.c b/drivers/gpu/drm/tinydrm/repaper.c
new file mode 100644
index 000000000000..30dc97b3ff21
--- /dev/null
+++ b/drivers/gpu/drm/tinydrm/repaper.c
@@ -0,0 +1,1117 @@
+/*
+ * DRM driver for Pervasive Displays RePaper branded e-ink panels
+ *
+ * Copyright 2013-2017 Pervasive Displays, Inc.
+ * Copyright 2017 Noralf Trønnes
+ *
+ * The driver supports:
+ * Material Film: Aurora Mb (V231)
+ * Driver IC: G2 (eTC)
+ *
+ * The controller code was taken from the userspace driver:
+ * https://github.com/repaper/gratis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/delay.h>
+#include <linux/dma-buf.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/sched/clock.h>
+#include <linux/spi/spi.h>
+#include <linux/thermal.h>
+
+#include <drm/tinydrm/tinydrm.h>
+#include <drm/tinydrm/tinydrm-helpers.h>
+
+#define REPAPER_RID_G2_COG_ID	0x12
+
+enum repaper_model {
+	E1144CS021 = 1,
+	E1190CS021,
+	E2200CS021,
+	E2271CS021,
+};
+
+enum repaper_stage {         /* Image pixel -> Display pixel */
+	REPAPER_COMPENSATE,  /* B -> W, W -> B (Current Image) */
+	REPAPER_WHITE,       /* B -> N, W -> W (Current Image) */
+	REPAPER_INVERSE,     /* B -> N, W -> B (New Image) */
+	REPAPER_NORMAL       /* B -> B, W -> W (New Image) */
+};
+
+enum repaper_epd_border_byte {
+	REPAPER_BORDER_BYTE_NONE,
+	REPAPER_BORDER_BYTE_ZERO,
+	REPAPER_BORDER_BYTE_SET,
+};
+
+struct repaper_epd {
+	struct tinydrm_device tinydrm;
+	struct spi_device *spi;
+
+	struct gpio_desc *panel_on;
+	struct gpio_desc *border;
+	struct gpio_desc *discharge;
+	struct gpio_desc *reset;
+	struct gpio_desc *busy;
+
+	struct thermal_zone_device *thermal;
+
+	unsigned int height;
+	unsigned int width;
+	unsigned int bytes_per_scan;
+	const u8 *channel_select;
+	unsigned int stage_time;
+	unsigned int factored_stage_time;
+	bool middle_scan;
+	bool pre_border_byte;
+	enum repaper_epd_border_byte border_byte;
+
+	u8 *line_buffer;
+	void *current_frame;
+
+	bool enabled;
+	bool cleared;
+	bool partial;
+};
+
+static inline struct repaper_epd *
+epd_from_tinydrm(struct tinydrm_device *tdev)
+{
+	return container_of(tdev, struct repaper_epd, tinydrm);
+}
+
+static int repaper_spi_transfer(struct spi_device *spi, u8 header,
+				const void *tx, void *rx, size_t len)
+{
+	void *txbuf = NULL, *rxbuf = NULL;
+	struct spi_transfer tr[2] = {};
+	u8 *headerbuf;
+	int ret;
+
+	headerbuf = kmalloc(1, GFP_KERNEL);
+	if (!headerbuf)
+		return -ENOMEM;
+
+	headerbuf[0] = header;
+	tr[0].tx_buf = headerbuf;
+	tr[0].len = 1;
+
+	/* Stack allocated tx? */
+	if (tx && len <= 32) {
+		txbuf = kmalloc(len, GFP_KERNEL);
+		if (!txbuf) {
+			ret = -ENOMEM;
+			goto out_free;
+		}
+		memcpy(txbuf, tx, len);
+	}
+
+	if (rx) {
+		rxbuf = kmalloc(len, GFP_KERNEL);
+		if (!rxbuf) {
+			ret = -ENOMEM;
+			goto out_free;
+		}
+	}
+
+	tr[1].tx_buf = txbuf ? txbuf : tx;
+	tr[1].rx_buf = rxbuf;
+	tr[1].len = len;
+
+	ndelay(80);
+	ret = spi_sync_transfer(spi, tr, 2);
+	if (rx && !ret)
+		memcpy(rx, rxbuf, len);
+
+out_free:
+	kfree(headerbuf);
+	kfree(txbuf);
+	kfree(rxbuf);
+
+	return ret;
+}
+
+static int repaper_write_buf(struct spi_device *spi, u8 reg,
+			     const u8 *buf, size_t len)
+{
+	int ret;
+
+	ret = repaper_spi_transfer(spi, 0x70, &reg, NULL, 1);
+	if (ret)
+		return ret;
+
+	return repaper_spi_transfer(spi, 0x72, buf, NULL, len);
+}
+
+static int repaper_write_val(struct spi_device *spi, u8 reg, u8 val)
+{
+	return repaper_write_buf(spi, reg, &val, 1);
+}
+
+static int repaper_read_val(struct spi_device *spi, u8 reg)
+{
+	int ret;
+	u8 val;
+
+	ret = repaper_spi_transfer(spi, 0x70, &reg, NULL, 1);
+	if (ret)
+		return ret;
+
+	ret = repaper_spi_transfer(spi, 0x73, NULL, &val, 1);
+
+	return ret ? ret : val;
+}
+
+static int repaper_read_id(struct spi_device *spi)
+{
+	int ret;
+	u8 id;
+
+	ret = repaper_spi_transfer(spi, 0x71, NULL, &id, 1);
+
+	return ret ? ret : id;
+}
+
+static void repaper_spi_mosi_low(struct spi_device *spi)
+{
+	const u8 buf[1] = { 0 };
+
+	spi_write(spi, buf, 1);
+}
+
+/* pixels on display are numbered from 1 so even is actually bits 1,3,5,... */
+static void repaper_even_pixels(struct repaper_epd *epd, u8 **pp,
+				const u8 *data, u8 fixed_value, const u8 *mask,
+				enum repaper_stage stage)
+{
+	unsigned int b;
+
+	for (b = 0; b < (epd->width / 8); b++) {
+		if (data) {
+			u8 pixels = data[b] & 0xaa;
+			u8 pixel_mask = 0xff;
+			u8 p1, p2, p3, p4;
+
+			if (mask) {
+				pixel_mask = (mask[b] ^ pixels) & 0xaa;
+				pixel_mask |= pixel_mask >> 1;
+			}
+
+			switch (stage) {
+			case REPAPER_COMPENSATE: /* B -> W, W -> B (Current) */
+				pixels = 0xaa | ((pixels ^ 0xaa) >> 1);
+				break;
+			case REPAPER_WHITE:      /* B -> N, W -> W (Current) */
+				pixels = 0x55 + ((pixels ^ 0xaa) >> 1);
+				break;
+			case REPAPER_INVERSE:    /* B -> N, W -> B (New) */
+				pixels = 0x55 | (pixels ^ 0xaa);
+				break;
+			case REPAPER_NORMAL:     /* B -> B, W -> W (New) */
+				pixels = 0xaa | (pixels >> 1);
+				break;
+			}
+
+			pixels = (pixels & pixel_mask) | (~pixel_mask & 0x55);
+			p1 = (pixels >> 6) & 0x03;
+			p2 = (pixels >> 4) & 0x03;
+			p3 = (pixels >> 2) & 0x03;
+			p4 = (pixels >> 0) & 0x03;
+			pixels = (p1 << 0) | (p2 << 2) | (p3 << 4) | (p4 << 6);
+			*(*pp)++ = pixels;
+		} else {
+			*(*pp)++ = fixed_value;
+		}
+	}
+}
+
+/* pixels on display are numbered from 1 so odd is actually bits 0,2,4,... */
+static void repaper_odd_pixels(struct repaper_epd *epd, u8 **pp,
+			       const u8 *data, u8 fixed_value, const u8 *mask,
+			       enum repaper_stage stage)
+{
+	unsigned int b;
+
+	for (b = epd->width / 8; b > 0; b--) {
+		if (data) {
+			u8 pixels = data[b - 1] & 0x55;
+			u8 pixel_mask = 0xff;
+
+			if (mask) {
+				pixel_mask = (mask[b - 1] ^ pixels) & 0x55;
+				pixel_mask |= pixel_mask << 1;
+			}
+
+			switch (stage) {
+			case REPAPER_COMPENSATE: /* B -> W, W -> B (Current) */
+				pixels = 0xaa | (pixels ^ 0x55);
+				break;
+			case REPAPER_WHITE:      /* B -> N, W -> W (Current) */
+				pixels = 0x55 + (pixels ^ 0x55);
+				break;
+			case REPAPER_INVERSE:    /* B -> N, W -> B (New) */
+				pixels = 0x55 | ((pixels ^ 0x55) << 1);
+				break;
+			case REPAPER_NORMAL:     /* B -> B, W -> W (New) */
+				pixels = 0xaa | pixels;
+				break;
+			}
+
+			pixels = (pixels & pixel_mask) | (~pixel_mask & 0x55);
+			*(*pp)++ = pixels;
+		} else {
+			*(*pp)++ = fixed_value;
+		}
+	}
+}
+
+/* interleave bits: (byte)76543210 -> (16 bit).7.6.5.4.3.2.1 */
+static inline u16 repaper_interleave_bits(u16 value)
+{
+	value = (value | (value << 4)) & 0x0f0f;
+	value = (value | (value << 2)) & 0x3333;
+	value = (value | (value << 1)) & 0x5555;
+
+	return value;
+}
+
+/* pixels on display are numbered from 1 */
+static void repaper_all_pixels(struct repaper_epd *epd, u8 **pp,
+			       const u8 *data, u8 fixed_value, const u8 *mask,
+			       enum repaper_stage stage)
+{
+	unsigned int b;
+
+	for (b = epd->width / 8; b > 0; b--) {
+		if (data) {
+			u16 pixels = repaper_interleave_bits(data[b - 1]);
+			u16 pixel_mask = 0xffff;
+
+			if (mask) {
+				pixel_mask = repaper_interleave_bits(mask[b - 1]);
+
+				pixel_mask = (pixel_mask ^ pixels) & 0x5555;
+				pixel_mask |= pixel_mask << 1;
+			}
+
+			switch (stage) {
+			case REPAPER_COMPENSATE: /* B -> W, W -> B (Current) */
+				pixels = 0xaaaa | (pixels ^ 0x5555);
+				break;
+			case REPAPER_WHITE:      /* B -> N, W -> W (Current) */
+				pixels = 0x5555 + (pixels ^ 0x5555);
+				break;
+			case REPAPER_INVERSE:    /* B -> N, W -> B (New) */
+				pixels = 0x5555 | ((pixels ^ 0x5555) << 1);
+				break;
+			case REPAPER_NORMAL:     /* B -> B, W -> W (New) */
+				pixels = 0xaaaa | pixels;
+				break;
+			}
+
+			pixels = (pixels & pixel_mask) | (~pixel_mask & 0x5555);
+			*(*pp)++ = pixels >> 8;
+			*(*pp)++ = pixels;
+		} else {
+			*(*pp)++ = fixed_value;
+			*(*pp)++ = fixed_value;
+		}
+	}
+}
+
+/* output one line of scan and data bytes to the display */
+static void repaper_one_line(struct repaper_epd *epd, unsigned int line,
+			     const u8 *data, u8 fixed_value, const u8 *mask,
+			     enum repaper_stage stage)
+{
+	u8 *p = epd->line_buffer;
+	unsigned int b;
+
+	repaper_spi_mosi_low(epd->spi);
+
+	if (epd->pre_border_byte)
+		*p++ = 0x00;
+
+	if (epd->middle_scan) {
+		/* data bytes */
+		repaper_odd_pixels(epd, &p, data, fixed_value, mask, stage);
+
+		/* scan line */
+		for (b = epd->bytes_per_scan; b > 0; b--) {
+			if (line / 4 == b - 1)
+				*p++ = 0x03 << (2 * (line & 0x03));
+			else
+				*p++ = 0x00;
+		}
+
+		/* data bytes */
+		repaper_even_pixels(epd, &p, data, fixed_value, mask, stage);
+	} else {
+		/*
+		 * even scan line, but as lines on display are numbered from 1,
+		 * line: 1,3,5,...
+		 */
+		for (b = 0; b < epd->bytes_per_scan; b++) {
+			if (0 != (line & 0x01) && line / 8 == b)
+				*p++ = 0xc0 >> (line & 0x06);
+			else
+				*p++ = 0x00;
+		}
+
+		/* data bytes */
+		repaper_all_pixels(epd, &p, data, fixed_value, mask, stage);
+
+		/*
+		 * odd scan line, but as lines on display are numbered from 1,
+		 * line: 0,2,4,6,...
+		 */
+		for (b = epd->bytes_per_scan; b > 0; b--) {
+			if (0 == (line & 0x01) && line / 8 == b - 1)
+				*p++ = 0x03 << (line & 0x06);
+			else
+				*p++ = 0x00;
+		}
+	}
+
+	switch (epd->border_byte) {
+	case REPAPER_BORDER_BYTE_NONE:
+		break;
+
+	case REPAPER_BORDER_BYTE_ZERO:
+		*p++ = 0x00;
+		break;
+
+	case REPAPER_BORDER_BYTE_SET:
+		switch (stage) {
+		case REPAPER_COMPENSATE:
+		case REPAPER_WHITE:
+		case REPAPER_INVERSE:
+			*p++ = 0x00;
+			break;
+		case REPAPER_NORMAL:
+			*p++ = 0xaa;
+			break;
+		}
+		break;
+	}
+
+	repaper_write_buf(epd->spi, 0x0a, epd->line_buffer,
+			  p - epd->line_buffer);
+
+	/* Output data to panel */
+	repaper_write_val(epd->spi, 0x02, 0x07);
+
+	repaper_spi_mosi_low(epd->spi);
+}
+
+static void repaper_frame_fixed(struct repaper_epd *epd, u8 fixed_value,
+				enum repaper_stage stage)
+{
+	unsigned int line;
+
+	for (line = 0; line < epd->height; line++)
+		repaper_one_line(epd, line, NULL, fixed_value, NULL, stage);
+}
+
+static void repaper_frame_data(struct repaper_epd *epd, const u8 *image,
+			       const u8 *mask, enum repaper_stage stage)
+{
+	unsigned int line;
+
+	if (!mask) {
+		for (line = 0; line < epd->height; line++) {
+			repaper_one_line(epd, line,
+					 &image[line * (epd->width / 8)],
+					 0, NULL, stage);
+		}
+	} else {
+		for (line = 0; line < epd->height; line++) {
+			size_t n = line * epd->width / 8;
+
+			repaper_one_line(epd, line, &image[n], 0, &mask[n],
+					 stage);
+		}
+	}
+}
+
+static void repaper_frame_fixed_repeat(struct repaper_epd *epd, u8 fixed_value,
+				       enum repaper_stage stage)
+{
+	u64 start = local_clock();
+	u64 end = start + (epd->factored_stage_time * 1000 * 1000);
+
+	do {
+		repaper_frame_fixed(epd, fixed_value, stage);
+	} while (local_clock() < end);
+}
+
+static void repaper_frame_data_repeat(struct repaper_epd *epd, const u8 *image,
+				      const u8 *mask, enum repaper_stage stage)
+{
+	u64 start = local_clock();
+	u64 end = start + (epd->factored_stage_time * 1000 * 1000);
+
+	do {
+		repaper_frame_data(epd, image, mask, stage);
+	} while (local_clock() < end);
+}
+
+static void repaper_get_temperature(struct repaper_epd *epd)
+{
+	int ret, temperature = 0;
+	unsigned int factor10x;
+
+	if (!epd->thermal)
+		return;
+
+	ret = thermal_zone_get_temp(epd->thermal, &temperature);
+	if (ret) {
+		dev_err(&epd->spi->dev, "Failed to get temperature (%d)\n",
+			ret);
+		return;
+	}
+
+	temperature /= 1000;
+
+	if (temperature <= -10)
+		factor10x = 170;
+	else if (temperature <= -5)
+		factor10x = 120;
+	else if (temperature <= 5)
+		factor10x = 80;
+	else if (temperature <= 10)
+		factor10x = 40;
+	else if (temperature <= 15)
+		factor10x = 30;
+	else if (temperature <= 20)
+		factor10x = 20;
+	else if (temperature <= 40)
+		factor10x = 10;
+	else
+		factor10x = 7;
+
+	epd->factored_stage_time = epd->stage_time * factor10x / 10;
+}
+
+static void repaper_gray8_to_mono_reversed(u8 *buf, u32 width, u32 height)
+{
+	u8 *gray8 = buf, *mono = buf;
+	int y, xb, i;
+
+	for (y = 0; y < height; y++)
+		for (xb = 0; xb < width / 8; xb++) {
+			u8 byte = 0x00;
+
+			for (i = 0; i < 8; i++) {
+				int x = xb * 8 + i;
+
+				byte >>= 1;
+				if (gray8[y * width + x] >> 7)
+					byte |= BIT(7);
+			}
+			*mono++ = byte;
+		}
+}
+
+static int repaper_fb_dirty(struct drm_framebuffer *fb,
+			    struct drm_file *file_priv,
+			    unsigned int flags, unsigned int color,
+			    struct drm_clip_rect *clips,
+			    unsigned int num_clips)
+{
+	struct drm_gem_cma_object *cma_obj = drm_fb_cma_get_gem_obj(fb, 0);
+	struct dma_buf_attachment *import_attach = cma_obj->base.import_attach;
+	struct tinydrm_device *tdev = fb->dev->dev_private;
+	struct repaper_epd *epd = epd_from_tinydrm(tdev);
+	struct drm_clip_rect clip;
+	u8 *buf = NULL;
+	int ret = 0;
+
+	/* repaper can't do partial updates */
+	clip.x1 = 0;
+	clip.x2 = fb->width;
+	clip.y1 = 0;
+	clip.y2 = fb->height;
+
+	mutex_lock(&tdev->dirty_lock);
+
+	if (!epd->enabled)
+		goto out_unlock;
+
+	/* fbdev can flush even when we're not interested */
+	if (tdev->pipe.plane.fb != fb)
+		goto out_unlock;
+
+	repaper_get_temperature(epd);
+
+	DRM_DEBUG("Flushing [FB:%d] st=%ums\n", fb->base.id,
+		  epd->factored_stage_time);
+
+	buf = kmalloc(fb->width * fb->height, GFP_KERNEL);
+	if (!buf) {
+		ret = -ENOMEM;
+		goto out_unlock;
+	}
+
+	if (import_attach) {
+		ret = dma_buf_begin_cpu_access(import_attach->dmabuf,
+					       DMA_FROM_DEVICE);
+		if (ret)
+			goto out_unlock;
+	}
+
+	tinydrm_xrgb8888_to_gray8(buf, cma_obj->vaddr, fb, &clip);
+
+	if (import_attach) {
+		ret = dma_buf_end_cpu_access(import_attach->dmabuf,
+					     DMA_FROM_DEVICE);
+		if (ret)
+			goto out_unlock;
+	}
+
+	repaper_gray8_to_mono_reversed(buf, fb->width, fb->height);
+
+	if (epd->partial) {
+		repaper_frame_data_repeat(epd, buf, epd->current_frame,
+					  REPAPER_NORMAL);
+	} else if (epd->cleared) {
+		repaper_frame_data_repeat(epd, epd->current_frame, NULL,
+					  REPAPER_COMPENSATE);
+		repaper_frame_data_repeat(epd, epd->current_frame, NULL,
+					  REPAPER_WHITE);
+		repaper_frame_data_repeat(epd, buf, NULL, REPAPER_INVERSE);
+		repaper_frame_data_repeat(epd, buf, NULL, REPAPER_NORMAL);
+
+		epd->partial = true;
+	} else {
+		/* Clear display (anything -> white) */
+		repaper_frame_fixed_repeat(epd, 0xff, REPAPER_COMPENSATE);
+		repaper_frame_fixed_repeat(epd, 0xff, REPAPER_WHITE);
+		repaper_frame_fixed_repeat(epd, 0xaa, REPAPER_INVERSE);
+		repaper_frame_fixed_repeat(epd, 0xaa, REPAPER_NORMAL);
+
+		/* Assuming a clear (white) screen output an image */
+		repaper_frame_fixed_repeat(epd, 0xaa, REPAPER_COMPENSATE);
+		repaper_frame_fixed_repeat(epd, 0xaa, REPAPER_WHITE);
+		repaper_frame_data_repeat(epd, buf, NULL, REPAPER_INVERSE);
+		repaper_frame_data_repeat(epd, buf, NULL, REPAPER_NORMAL);
+
+		epd->cleared = true;
+		epd->partial = true;
+	}
+
+	memcpy(epd->current_frame, buf, fb->width * fb->height / 8);
+
+	/*
+	 * An extra frame write is needed if pixels are set in the bottom line,
+	 * or else grey lines rises up from the pixels
+	 */
+	if (epd->pre_border_byte) {
+		unsigned int x;
+
+		for (x = 0; x < (fb->width / 8); x++)
+			if (buf[x + (fb->width * (fb->height - 1) / 8)]) {
+				repaper_frame_data_repeat(epd, buf,
+							  epd->current_frame,
+							  REPAPER_NORMAL);
+				break;
+			}
+	}
+
+out_unlock:
+	mutex_unlock(&tdev->dirty_lock);
+
+	if (ret)
+		dev_err(fb->dev->dev, "Failed to update display (%d)\n", ret);
+	kfree(buf);
+
+	return ret;
+}
+
+static const struct drm_framebuffer_funcs repaper_fb_funcs = {
+	.destroy	= drm_fb_cma_destroy,
+	.create_handle	= drm_fb_cma_create_handle,
+	.dirty		= repaper_fb_dirty,
+};
+
+static void power_off(struct repaper_epd *epd)
+{
+	/* Turn off power and all signals */
+	gpiod_set_value_cansleep(epd->reset, 0);
+	gpiod_set_value_cansleep(epd->panel_on, 0);
+	if (epd->border)
+		gpiod_set_value_cansleep(epd->border, 0);
+
+	/* Ensure SPI MOSI and CLOCK are Low before CS Low */
+	repaper_spi_mosi_low(epd->spi);
+
+	/* Discharge pulse */
+	gpiod_set_value_cansleep(epd->discharge, 1);
+	msleep(150);
+	gpiod_set_value_cansleep(epd->discharge, 0);
+}
+
+static void repaper_pipe_enable(struct drm_simple_display_pipe *pipe,
+				struct drm_crtc_state *crtc_state)
+{
+	struct tinydrm_device *tdev = pipe_to_tinydrm(pipe);
+	struct repaper_epd *epd = epd_from_tinydrm(tdev);
+	struct spi_device *spi = epd->spi;
+	struct device *dev = &spi->dev;
+	bool dc_ok = false;
+	int i, ret;
+
+	DRM_DEBUG_DRIVER("\n");
+
+	/* Power up sequence */
+	gpiod_set_value_cansleep(epd->reset, 0);
+	gpiod_set_value_cansleep(epd->panel_on, 0);
+	gpiod_set_value_cansleep(epd->discharge, 0);
+	if (epd->border)
+		gpiod_set_value_cansleep(epd->border, 0);
+	repaper_spi_mosi_low(spi);
+	usleep_range(5000, 10000);
+
+	gpiod_set_value_cansleep(epd->panel_on, 1);
+	/*
+	 * This delay comes from the repaper.org userspace driver, it's not
+	 * mentioned in the datasheet.
+	 */
+	usleep_range(10000, 15000);
+	gpiod_set_value_cansleep(epd->reset, 1);
+	if (epd->border)
+		gpiod_set_value_cansleep(epd->border, 1);
+	usleep_range(5000, 10000);
+	gpiod_set_value_cansleep(epd->reset, 0);
+	usleep_range(5000, 10000);
+	gpiod_set_value_cansleep(epd->reset, 1);
+	usleep_range(5000, 10000);
+
+	/* Wait for COG to become ready */
+	for (i = 100; i > 0; i--) {
+		if (!gpiod_get_value_cansleep(epd->busy))
+			break;
+
+		usleep_range(10, 100);
+	}
+
+	if (!i) {
+		dev_err(dev, "timeout waiting for panel to become ready.\n");
+		power_off(epd);
+		return;
+	}
+
+	repaper_read_id(spi);
+	ret = repaper_read_id(spi);
+	if (ret != REPAPER_RID_G2_COG_ID) {
+		if (ret < 0)
+			dev_err(dev, "failed to read chip (%d)\n", ret);
+		else
+			dev_err(dev, "wrong COG ID 0x%02x\n", ret);
+		power_off(epd);
+		return;
+	}
+
+	/* Disable OE */
+	repaper_write_val(spi, 0x02, 0x40);
+
+	ret = repaper_read_val(spi, 0x0f);
+	if (ret < 0 || !(ret & 0x80)) {
+		if (ret < 0)
+			dev_err(dev, "failed to read chip (%d)\n", ret);
+		else
+			dev_err(dev, "panel is reported broken\n");
+		power_off(epd);
+		return;
+	}
+
+	/* Power saving mode */
+	repaper_write_val(spi, 0x0b, 0x02);
+	/* Channel select */
+	repaper_write_buf(spi, 0x01, epd->channel_select, 8);
+	/* High power mode osc */
+	repaper_write_val(spi, 0x07, 0xd1);
+	/* Power setting */
+	repaper_write_val(spi, 0x08, 0x02);
+	/* Vcom level */
+	repaper_write_val(spi, 0x09, 0xc2);
+	/* Power setting */
+	repaper_write_val(spi, 0x04, 0x03);
+	/* Driver latch on */
+	repaper_write_val(spi, 0x03, 0x01);
+	/* Driver latch off */
+	repaper_write_val(spi, 0x03, 0x00);
+	usleep_range(5000, 10000);
+
+	/* Start chargepump */
+	for (i = 0; i < 4; ++i) {
+		/* Charge pump positive voltage on - VGH/VDL on */
+		repaper_write_val(spi, 0x05, 0x01);
+		msleep(240);
+
+		/* Charge pump negative voltage on - VGL/VDL on */
+		repaper_write_val(spi, 0x05, 0x03);
+		msleep(40);
+
+		/* Charge pump Vcom on - Vcom driver on */
+		repaper_write_val(spi, 0x05, 0x0f);
+		msleep(40);
+
+		/* check DC/DC */
+		ret = repaper_read_val(spi, 0x0f);
+		if (ret < 0) {
+			dev_err(dev, "failed to read chip (%d)\n", ret);
+			power_off(epd);
+			return;
+		}
+
+		if (ret & 0x40) {
+			dc_ok = true;
+			break;
+		}
+	}
+
+	if (!dc_ok) {
+		dev_err(dev, "dc/dc failed\n");
+		power_off(epd);
+		return;
+	}
+
+	/*
+	 * Output enable to disable
+	 * The userspace driver sets this to 0x04, but the datasheet says 0x06
+	 */
+	repaper_write_val(spi, 0x02, 0x04);
+
+	epd->enabled = true;
+	epd->partial = false;
+}
+
+static void repaper_pipe_disable(struct drm_simple_display_pipe *pipe)
+{
+	struct tinydrm_device *tdev = pipe_to_tinydrm(pipe);
+	struct repaper_epd *epd = epd_from_tinydrm(tdev);
+	struct spi_device *spi = epd->spi;
+	unsigned int line;
+
+	DRM_DEBUG_DRIVER("\n");
+
+	mutex_lock(&tdev->dirty_lock);
+	epd->enabled = false;
+	mutex_unlock(&tdev->dirty_lock);
+
+	/* Nothing frame */
+	for (line = 0; line < epd->height; line++)
+		repaper_one_line(epd, 0x7fffu, NULL, 0x00, NULL,
+				 REPAPER_COMPENSATE);
+
+	/* 2.7" */
+	if (epd->border) {
+		/* Dummy line */
+		repaper_one_line(epd, 0x7fffu, NULL, 0x00, NULL,
+				 REPAPER_COMPENSATE);
+		msleep(25);
+		gpiod_set_value_cansleep(epd->border, 0);
+		msleep(200);
+		gpiod_set_value_cansleep(epd->border, 1);
+	} else {
+		/* Border dummy line */
+		repaper_one_line(epd, 0x7fffu, NULL, 0x00, NULL,
+				 REPAPER_NORMAL);
+		msleep(200);
+	}
+
+	/* not described in datasheet */
+	repaper_write_val(spi, 0x0b, 0x00);
+	/* Latch reset turn on */
+	repaper_write_val(spi, 0x03, 0x01);
+	/* Power off charge pump Vcom */
+	repaper_write_val(spi, 0x05, 0x03);
+	/* Power off charge pump neg voltage */
+	repaper_write_val(spi, 0x05, 0x01);
+	msleep(120);
+	/* Discharge internal */
+	repaper_write_val(spi, 0x04, 0x80);
+	/* turn off all charge pumps */
+	repaper_write_val(spi, 0x05, 0x00);
+	/* Turn off osc */
+	repaper_write_val(spi, 0x07, 0x01);
+	msleep(50);
+
+	power_off(epd);
+}
+
+static const struct drm_simple_display_pipe_funcs repaper_pipe_funcs = {
+	.enable = repaper_pipe_enable,
+	.disable = repaper_pipe_disable,
+	.update = tinydrm_display_pipe_update,
+	.prepare_fb = tinydrm_display_pipe_prepare_fb,
+};
+
+static const uint32_t repaper_formats[] = {
+	DRM_FORMAT_XRGB8888,
+};
+
+static const struct drm_display_mode repaper_e1144cs021_mode = {
+	TINYDRM_MODE(128, 96, 29, 22),
+};
+
+static const u8 repaper_e1144cs021_cs[] = { 0x00, 0x00, 0x00, 0x00,
+					    0x00, 0x0f, 0xff, 0x00 };
+
+static const struct drm_display_mode repaper_e1190cs021_mode = {
+	TINYDRM_MODE(144, 128, 36, 32),
+};
+
+static const u8 repaper_e1190cs021_cs[] = { 0x00, 0x00, 0x00, 0x03,
+					    0xfc, 0x00, 0x00, 0xff };
+
+static const struct drm_display_mode repaper_e2200cs021_mode = {
+	TINYDRM_MODE(200, 96, 46, 22),
+};
+
+static const u8 repaper_e2200cs021_cs[] = { 0x00, 0x00, 0x00, 0x00,
+					    0x01, 0xff, 0xe0, 0x00 };
+
+static const struct drm_display_mode repaper_e2271cs021_mode = {
+	TINYDRM_MODE(264, 176, 57, 38),
+};
+
+static const u8 repaper_e2271cs021_cs[] = { 0x00, 0x00, 0x00, 0x7f,
+					    0xff, 0xfe, 0x00, 0x00 };
+
+DEFINE_DRM_GEM_CMA_FOPS(repaper_fops);
+
+static struct drm_driver repaper_driver = {
+	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_PRIME |
+				  DRIVER_ATOMIC,
+	.fops			= &repaper_fops,
+	TINYDRM_GEM_DRIVER_OPS,
+	.name			= "repaper",
+	.desc			= "Pervasive Displays RePaper e-ink panels",
+	.date			= "20170405",
+	.major			= 1,
+	.minor			= 0,
+};
+
+static const struct of_device_id repaper_of_match[] = {
+	{ .compatible = "pervasive,e1144cs021", .data = (void *)E1144CS021 },
+	{ .compatible = "pervasive,e1190cs021", .data = (void *)E1190CS021 },
+	{ .compatible = "pervasive,e2200cs021", .data = (void *)E2200CS021 },
+	{ .compatible = "pervasive,e2271cs021", .data = (void *)E2271CS021 },
+	{},
+};
+MODULE_DEVICE_TABLE(of, repaper_of_match);
+
+static const struct spi_device_id repaper_id[] = {
+	{ "e1144cs021", E1144CS021 },
+	{ "e1190cs021", E1190CS021 },
+	{ "e2200cs021", E2200CS021 },
+	{ "e2271cs021", E2271CS021 },
+	{ },
+};
+MODULE_DEVICE_TABLE(spi, repaper_id);
+
+static int repaper_probe(struct spi_device *spi)
+{
+	const struct drm_display_mode *mode;
+	const struct spi_device_id *spi_id;
+	const struct of_device_id *match;
+	struct device *dev = &spi->dev;
+	struct tinydrm_device *tdev;
+	enum repaper_model model;
+	const char *thermal_zone;
+	struct repaper_epd *epd;
+	size_t line_buffer_size;
+	int ret;
+
+	match = of_match_device(repaper_of_match, dev);
+	if (match) {
+		model = (enum repaper_model)match->data;
+	} else {
+		spi_id = spi_get_device_id(spi);
+		model = spi_id->driver_data;
+	}
+
+	/* The SPI device is used to allocate dma memory */
+	if (!dev->coherent_dma_mask) {
+		ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(32));
+		if (ret) {
+			dev_warn(dev, "Failed to set dma mask %d\n", ret);
+			return ret;
+		}
+	}
+
+	epd = devm_kzalloc(dev, sizeof(*epd), GFP_KERNEL);
+	if (!epd)
+		return -ENOMEM;
+
+	epd->spi = spi;
+
+	epd->panel_on = devm_gpiod_get(dev, "panel-on", GPIOD_OUT_LOW);
+	if (IS_ERR(epd->panel_on)) {
+		ret = PTR_ERR(epd->panel_on);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Failed to get gpio 'panel-on'\n");
+		return ret;
+	}
+
+	epd->discharge = devm_gpiod_get(dev, "discharge", GPIOD_OUT_LOW);
+	if (IS_ERR(epd->discharge)) {
+		ret = PTR_ERR(epd->discharge);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Failed to get gpio 'discharge'\n");
+		return ret;
+	}
+
+	epd->reset = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW);
+	if (IS_ERR(epd->reset)) {
+		ret = PTR_ERR(epd->reset);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Failed to get gpio 'reset'\n");
+		return ret;
+	}
+
+	epd->busy = devm_gpiod_get(dev, "busy", GPIOD_IN);
+	if (IS_ERR(epd->busy)) {
+		ret = PTR_ERR(epd->busy);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "Failed to get gpio 'busy'\n");
+		return ret;
+	}
+
+	if (!device_property_read_string(dev, "pervasive,thermal-zone",
+					 &thermal_zone)) {
+		epd->thermal = thermal_zone_get_zone_by_name(thermal_zone);
+		if (IS_ERR(epd->thermal)) {
+			dev_err(dev, "Failed to get thermal zone: %s\n",
+				thermal_zone);
+			return PTR_ERR(epd->thermal);
+		}
+	}
+
+	switch (model) {
+	case E1144CS021:
+		mode = &repaper_e1144cs021_mode;
+		epd->channel_select = repaper_e1144cs021_cs;
+		epd->stage_time = 480;
+		epd->bytes_per_scan = 96 / 4;
+		epd->middle_scan = true; /* data-scan-data */
+		epd->pre_border_byte = false;
+		epd->border_byte = REPAPER_BORDER_BYTE_ZERO;
+		break;
+
+	case E1190CS021:
+		mode = &repaper_e1190cs021_mode;
+		epd->channel_select = repaper_e1190cs021_cs;
+		epd->stage_time = 480;
+		epd->bytes_per_scan = 128 / 4 / 2;
+		epd->middle_scan = false; /* scan-data-scan */
+		epd->pre_border_byte = false;
+		epd->border_byte = REPAPER_BORDER_BYTE_SET;
+		break;
+
+	case E2200CS021:
+		mode = &repaper_e2200cs021_mode;
+		epd->channel_select = repaper_e2200cs021_cs;
+		epd->stage_time = 480;
+		epd->bytes_per_scan = 96 / 4;
+		epd->middle_scan = true; /* data-scan-data */
+		epd->pre_border_byte = true;
+		epd->border_byte = REPAPER_BORDER_BYTE_NONE;
+		break;
+
+	case E2271CS021:
+		epd->border = devm_gpiod_get(dev, "border", GPIOD_OUT_LOW);
+		if (IS_ERR(epd->border)) {
+			ret = PTR_ERR(epd->border);
+			if (ret != -EPROBE_DEFER)
+				dev_err(dev, "Failed to get gpio 'border'\n");
+			return ret;
+		}
+
+		mode = &repaper_e2271cs021_mode;
+		epd->channel_select = repaper_e2271cs021_cs;
+		epd->stage_time = 630;
+		epd->bytes_per_scan = 176 / 4;
+		epd->middle_scan = true; /* data-scan-data */
+		epd->pre_border_byte = true;
+		epd->border_byte = REPAPER_BORDER_BYTE_NONE;
+		break;
+
+	default:
+		return -ENODEV;
+	}
+
+	epd->width = mode->hdisplay;
+	epd->height = mode->vdisplay;
+	epd->factored_stage_time = epd->stage_time;
+
+	line_buffer_size = 2 * epd->width / 8 + epd->bytes_per_scan + 2;
+	epd->line_buffer = devm_kzalloc(dev, line_buffer_size, GFP_KERNEL);
+	if (!epd->line_buffer)
+		return -ENOMEM;
+
+	epd->current_frame = devm_kzalloc(dev, epd->width * epd->height / 8,
+					  GFP_KERNEL);
+	if (!epd->current_frame)
+		return -ENOMEM;
+
+	tdev = &epd->tinydrm;
+
+	ret = devm_tinydrm_init(dev, tdev, &repaper_fb_funcs, &repaper_driver);
+	if (ret)
+		return ret;
+
+	ret = tinydrm_display_pipe_init(tdev, &repaper_pipe_funcs,
+					DRM_MODE_CONNECTOR_VIRTUAL,
+					repaper_formats,
+					ARRAY_SIZE(repaper_formats), mode, 0);
+	if (ret)
+		return ret;
+
+	drm_mode_config_reset(tdev->drm);
+
+	ret = devm_tinydrm_register(tdev);
+	if (ret)
+		return ret;
+
+	spi_set_drvdata(spi, tdev);
+
+	DRM_DEBUG_DRIVER("Initialized %s:%s @%uMHz on minor %d\n",
+			 tdev->drm->driver->name, dev_name(dev),
+			 spi->max_speed_hz / 1000000,
+			 tdev->drm->primary->index);
+
+	return 0;
+}
+
+static void repaper_shutdown(struct spi_device *spi)
+{
+	struct tinydrm_device *tdev = spi_get_drvdata(spi);
+
+	tinydrm_shutdown(tdev);
+}
+
+static struct spi_driver repaper_spi_driver = {
+	.driver = {
+		.name = "repaper",
+		.owner = THIS_MODULE,
+		.of_match_table = repaper_of_match,
+	},
+	.id_table = repaper_id,
+	.probe = repaper_probe,
+	.shutdown = repaper_shutdown,
+};
+module_spi_driver(repaper_spi_driver);
+
+MODULE_DESCRIPTION("Pervasive Displays RePaper DRM driver");
+MODULE_AUTHOR("Noralf Trønnes");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/tinydrm/st7586.c b/drivers/gpu/drm/tinydrm/st7586.c
new file mode 100644
index 000000000000..b439956a07f4
--- /dev/null
+++ b/drivers/gpu/drm/tinydrm/st7586.c
@@ -0,0 +1,428 @@
+/*
+ * DRM driver for Sitronix ST7586 panels
+ *
+ * Copyright 2017 David Lechner <david@lechnology.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/delay.h>
+#include <linux/dma-buf.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/property.h>
+#include <linux/spi/spi.h>
+#include <video/mipi_display.h>
+
+#include <drm/tinydrm/mipi-dbi.h>
+#include <drm/tinydrm/tinydrm-helpers.h>
+
+/* controller-specific commands */
+#define ST7586_DISP_MODE_GRAY	0x38
+#define ST7586_DISP_MODE_MONO	0x39
+#define ST7586_ENABLE_DDRAM	0x3a
+#define ST7586_SET_DISP_DUTY	0xb0
+#define ST7586_SET_PART_DISP	0xb4
+#define ST7586_SET_NLINE_INV	0xb5
+#define ST7586_SET_VOP		0xc0
+#define ST7586_SET_BIAS_SYSTEM	0xc3
+#define ST7586_SET_BOOST_LEVEL	0xc4
+#define ST7586_SET_VOP_OFFSET	0xc7
+#define ST7586_ENABLE_ANALOG	0xd0
+#define ST7586_AUTO_READ_CTRL	0xd7
+#define ST7586_OTP_RW_CTRL	0xe0
+#define ST7586_OTP_CTRL_OUT	0xe1
+#define ST7586_OTP_READ		0xe3
+
+#define ST7586_DISP_CTRL_MX	BIT(6)
+#define ST7586_DISP_CTRL_MY	BIT(7)
+
+/*
+ * The ST7586 controller has an unusual pixel format where 2bpp grayscale is
+ * packed 3 pixels per byte with the first two pixels using 3 bits and the 3rd
+ * pixel using only 2 bits.
+ *
+ * |  D7  |  D6  |  D5  ||      |      || 2bpp |
+ * | (D4) | (D3) | (D2) ||  D1  |  D0  || GRAY |
+ * +------+------+------++------+------++------+
+ * |  1   |  1   |  1   ||  1   |  1   || 0  0 | black
+ * |  1   |  0   |  0   ||  1   |  0   || 0  1 | dark gray
+ * |  0   |  1   |  0   ||  0   |  1   || 1  0 | light gray
+ * |  0   |  0   |  0   ||  0   |  0   || 1  1 | white
+ */
+
+static const u8 st7586_lookup[] = { 0x7, 0x4, 0x2, 0x0 };
+
+static void st7586_xrgb8888_to_gray332(u8 *dst, void *vaddr,
+				       struct drm_framebuffer *fb,
+				       struct drm_clip_rect *clip)
+{
+	size_t len = (clip->x2 - clip->x1) * (clip->y2 - clip->y1);
+	unsigned int x, y;
+	u8 *src, *buf, val;
+
+	buf = kmalloc(len, GFP_KERNEL);
+	if (!buf)
+		return;
+
+	tinydrm_xrgb8888_to_gray8(buf, vaddr, fb, clip);
+	src = buf;
+
+	for (y = clip->y1; y < clip->y2; y++) {
+		for (x = clip->x1; x < clip->x2; x += 3) {
+			val = st7586_lookup[*src++ >> 6] << 5;
+			val |= st7586_lookup[*src++ >> 6] << 2;
+			val |= st7586_lookup[*src++ >> 6] >> 1;
+			*dst++ = val;
+		}
+	}
+
+	kfree(buf);
+}
+
+static int st7586_buf_copy(void *dst, struct drm_framebuffer *fb,
+			   struct drm_clip_rect *clip)
+{
+	struct drm_gem_cma_object *cma_obj = drm_fb_cma_get_gem_obj(fb, 0);
+	struct dma_buf_attachment *import_attach = cma_obj->base.import_attach;
+	void *src = cma_obj->vaddr;
+	int ret = 0;
+
+	if (import_attach) {
+		ret = dma_buf_begin_cpu_access(import_attach->dmabuf,
+					       DMA_FROM_DEVICE);
+		if (ret)
+			return ret;
+	}
+
+	st7586_xrgb8888_to_gray332(dst, src, fb, clip);
+
+	if (import_attach)
+		ret = dma_buf_end_cpu_access(import_attach->dmabuf,
+					     DMA_FROM_DEVICE);
+
+	return ret;
+}
+
+static int st7586_fb_dirty(struct drm_framebuffer *fb,
+			   struct drm_file *file_priv, unsigned int flags,
+			   unsigned int color, struct drm_clip_rect *clips,
+			   unsigned int num_clips)
+{
+	struct tinydrm_device *tdev = fb->dev->dev_private;
+	struct mipi_dbi *mipi = mipi_dbi_from_tinydrm(tdev);
+	struct drm_clip_rect clip;
+	int start, end;
+	int ret = 0;
+
+	mutex_lock(&tdev->dirty_lock);
+
+	if (!mipi->enabled)
+		goto out_unlock;
+
+	/* fbdev can flush even when we're not interested */
+	if (tdev->pipe.plane.fb != fb)
+		goto out_unlock;
+
+	tinydrm_merge_clips(&clip, clips, num_clips, flags, fb->width,
+			    fb->height);
+
+	/* 3 pixels per byte, so grow clip to nearest multiple of 3 */
+	clip.x1 = rounddown(clip.x1, 3);
+	clip.x2 = roundup(clip.x2, 3);
+
+	DRM_DEBUG("Flushing [FB:%d] x1=%u, x2=%u, y1=%u, y2=%u\n", fb->base.id,
+		  clip.x1, clip.x2, clip.y1, clip.y2);
+
+	ret = st7586_buf_copy(mipi->tx_buf, fb, &clip);
+	if (ret)
+		goto out_unlock;
+
+	/* Pixels are packed 3 per byte */
+	start = clip.x1 / 3;
+	end = clip.x2 / 3;
+
+	mipi_dbi_command(mipi, MIPI_DCS_SET_COLUMN_ADDRESS,
+			 (start >> 8) & 0xFF, start & 0xFF,
+			 (end >> 8) & 0xFF, (end - 1) & 0xFF);
+	mipi_dbi_command(mipi, MIPI_DCS_SET_PAGE_ADDRESS,
+			 (clip.y1 >> 8) & 0xFF, clip.y1 & 0xFF,
+			 (clip.y2 >> 8) & 0xFF, (clip.y2 - 1) & 0xFF);
+
+	ret = mipi_dbi_command_buf(mipi, MIPI_DCS_WRITE_MEMORY_START,
+				   (u8 *)mipi->tx_buf,
+				   (end - start) * (clip.y2 - clip.y1));
+
+out_unlock:
+	mutex_unlock(&tdev->dirty_lock);
+
+	if (ret)
+		dev_err_once(fb->dev->dev, "Failed to update display %d\n",
+			     ret);
+
+	return ret;
+}
+
+static const struct drm_framebuffer_funcs st7586_fb_funcs = {
+	.destroy	= drm_fb_cma_destroy,
+	.create_handle	= drm_fb_cma_create_handle,
+	.dirty		= st7586_fb_dirty,
+};
+
+static void st7586_pipe_enable(struct drm_simple_display_pipe *pipe,
+			       struct drm_crtc_state *crtc_state)
+{
+	struct tinydrm_device *tdev = pipe_to_tinydrm(pipe);
+	struct mipi_dbi *mipi = mipi_dbi_from_tinydrm(tdev);
+	struct drm_framebuffer *fb = pipe->plane.fb;
+	struct device *dev = tdev->drm->dev;
+	int ret;
+	u8 addr_mode;
+
+	DRM_DEBUG_KMS("\n");
+
+	mipi_dbi_hw_reset(mipi);
+	ret = mipi_dbi_command(mipi, ST7586_AUTO_READ_CTRL, 0x9f);
+	if (ret) {
+		dev_err(dev, "Error sending command %d\n", ret);
+		return;
+	}
+
+	mipi_dbi_command(mipi, ST7586_OTP_RW_CTRL, 0x00);
+
+	msleep(10);
+
+	mipi_dbi_command(mipi, ST7586_OTP_READ);
+
+	msleep(20);
+
+	mipi_dbi_command(mipi, ST7586_OTP_CTRL_OUT);
+	mipi_dbi_command(mipi, MIPI_DCS_EXIT_SLEEP_MODE);
+	mipi_dbi_command(mipi, MIPI_DCS_SET_DISPLAY_OFF);
+
+	msleep(50);
+
+	mipi_dbi_command(mipi, ST7586_SET_VOP_OFFSET, 0x00);
+	mipi_dbi_command(mipi, ST7586_SET_VOP, 0xe3, 0x00);
+	mipi_dbi_command(mipi, ST7586_SET_BIAS_SYSTEM, 0x02);
+	mipi_dbi_command(mipi, ST7586_SET_BOOST_LEVEL, 0x04);
+	mipi_dbi_command(mipi, ST7586_ENABLE_ANALOG, 0x1d);
+	mipi_dbi_command(mipi, ST7586_SET_NLINE_INV, 0x00);
+	mipi_dbi_command(mipi, ST7586_DISP_MODE_GRAY);
+	mipi_dbi_command(mipi, ST7586_ENABLE_DDRAM, 0x02);
+
+	switch (mipi->rotation) {
+	default:
+		addr_mode = 0x00;
+		break;
+	case 90:
+		addr_mode = ST7586_DISP_CTRL_MY;
+		break;
+	case 180:
+		addr_mode = ST7586_DISP_CTRL_MX | ST7586_DISP_CTRL_MY;
+		break;
+	case 270:
+		addr_mode = ST7586_DISP_CTRL_MX;
+		break;
+	}
+	mipi_dbi_command(mipi, MIPI_DCS_SET_ADDRESS_MODE, addr_mode);
+
+	mipi_dbi_command(mipi, ST7586_SET_DISP_DUTY, 0x7f);
+	mipi_dbi_command(mipi, ST7586_SET_PART_DISP, 0xa0);
+	mipi_dbi_command(mipi, MIPI_DCS_SET_PARTIAL_AREA, 0x00, 0x00, 0x00, 0x77);
+	mipi_dbi_command(mipi, MIPI_DCS_EXIT_INVERT_MODE);
+
+	msleep(100);
+
+	mipi_dbi_command(mipi, MIPI_DCS_SET_DISPLAY_ON);
+
+	mipi->enabled = true;
+
+	if (fb)
+		fb->funcs->dirty(fb, NULL, 0, 0, NULL, 0);
+}
+
+static void st7586_pipe_disable(struct drm_simple_display_pipe *pipe)
+{
+	struct tinydrm_device *tdev = pipe_to_tinydrm(pipe);
+	struct mipi_dbi *mipi = mipi_dbi_from_tinydrm(tdev);
+
+	DRM_DEBUG_KMS("\n");
+
+	if (!mipi->enabled)
+		return;
+
+	mipi_dbi_command(mipi, MIPI_DCS_SET_DISPLAY_OFF);
+	mipi->enabled = false;
+}
+
+static const u32 st7586_formats[] = {
+	DRM_FORMAT_XRGB8888,
+};
+
+static int st7586_init(struct device *dev, struct mipi_dbi *mipi,
+		const struct drm_simple_display_pipe_funcs *pipe_funcs,
+		struct drm_driver *driver, const struct drm_display_mode *mode,
+		unsigned int rotation)
+{
+	size_t bufsize = (mode->vdisplay + 2) / 3 * mode->hdisplay;
+	struct tinydrm_device *tdev = &mipi->tinydrm;
+	int ret;
+
+	mutex_init(&mipi->cmdlock);
+
+	mipi->tx_buf = devm_kmalloc(dev, bufsize, GFP_KERNEL);
+	if (!mipi->tx_buf)
+		return -ENOMEM;
+
+	ret = devm_tinydrm_init(dev, tdev, &st7586_fb_funcs, driver);
+	if (ret)
+		return ret;
+
+	ret = tinydrm_display_pipe_init(tdev, pipe_funcs,
+					DRM_MODE_CONNECTOR_VIRTUAL,
+					st7586_formats,
+					ARRAY_SIZE(st7586_formats),
+					mode, rotation);
+	if (ret)
+		return ret;
+
+	tdev->drm->mode_config.preferred_depth = 32;
+	mipi->rotation = rotation;
+
+	drm_mode_config_reset(tdev->drm);
+
+	DRM_DEBUG_KMS("preferred_depth=%u, rotation = %u\n",
+		      tdev->drm->mode_config.preferred_depth, rotation);
+
+	return 0;
+}
+
+static const struct drm_simple_display_pipe_funcs st7586_pipe_funcs = {
+	.enable		= st7586_pipe_enable,
+	.disable	= st7586_pipe_disable,
+	.update		= tinydrm_display_pipe_update,
+	.prepare_fb	= tinydrm_display_pipe_prepare_fb,
+};
+
+static const struct drm_display_mode st7586_mode = {
+	TINYDRM_MODE(178, 128, 37, 27),
+};
+
+DEFINE_DRM_GEM_CMA_FOPS(st7586_fops);
+
+static struct drm_driver st7586_driver = {
+	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_PRIME |
+				  DRIVER_ATOMIC,
+	.fops			= &st7586_fops,
+	TINYDRM_GEM_DRIVER_OPS,
+	.lastclose		= tinydrm_lastclose,
+	.debugfs_init		= mipi_dbi_debugfs_init,
+	.name			= "st7586",
+	.desc			= "Sitronix ST7586",
+	.date			= "20170801",
+	.major			= 1,
+	.minor			= 0,
+};
+
+static const struct of_device_id st7586_of_match[] = {
+	{ .compatible = "lego,ev3-lcd" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, st7586_of_match);
+
+static const struct spi_device_id st7586_id[] = {
+	{ "ev3-lcd", 0 },
+	{ },
+};
+MODULE_DEVICE_TABLE(spi, st7586_id);
+
+static int st7586_probe(struct spi_device *spi)
+{
+	struct device *dev = &spi->dev;
+	struct tinydrm_device *tdev;
+	struct mipi_dbi *mipi;
+	struct gpio_desc *a0;
+	u32 rotation = 0;
+	int ret;
+
+	mipi = devm_kzalloc(dev, sizeof(*mipi), GFP_KERNEL);
+	if (!mipi)
+		return -ENOMEM;
+
+	mipi->reset = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(mipi->reset)) {
+		dev_err(dev, "Failed to get gpio 'reset'\n");
+		return PTR_ERR(mipi->reset);
+	}
+
+	a0 = devm_gpiod_get(dev, "a0", GPIOD_OUT_LOW);
+	if (IS_ERR(a0)) {
+		dev_err(dev, "Failed to get gpio 'a0'\n");
+		return PTR_ERR(a0);
+	}
+
+	device_property_read_u32(dev, "rotation", &rotation);
+
+	ret = mipi_dbi_spi_init(spi, mipi, a0);
+	if (ret)
+		return ret;
+
+	/* Cannot read from this controller via SPI */
+	mipi->read_commands = NULL;
+
+	/*
+	 * we are using 8-bit data, so we are not actually swapping anything,
+	 * but setting mipi->swap_bytes makes mipi_dbi_typec3_command() do the
+	 * right thing and not use 16-bit transfers (which results in swapped
+	 * bytes on little-endian systems and causes out of order data to be
+	 * sent to the display).
+	 */
+	mipi->swap_bytes = true;
+
+	ret = st7586_init(&spi->dev, mipi, &st7586_pipe_funcs, &st7586_driver,
+			  &st7586_mode, rotation);
+	if (ret)
+		return ret;
+
+	tdev = &mipi->tinydrm;
+
+	ret = devm_tinydrm_register(tdev);
+	if (ret)
+		return ret;
+
+	spi_set_drvdata(spi, mipi);
+
+	DRM_DEBUG_DRIVER("Initialized %s:%s @%uMHz on minor %d\n",
+			 tdev->drm->driver->name, dev_name(dev),
+			 spi->max_speed_hz / 1000000,
+			 tdev->drm->primary->index);
+
+	return 0;
+}
+
+static void st7586_shutdown(struct spi_device *spi)
+{
+	struct mipi_dbi *mipi = spi_get_drvdata(spi);
+
+	tinydrm_shutdown(&mipi->tinydrm);
+}
+
+static struct spi_driver st7586_spi_driver = {
+	.driver = {
+		.name = "st7586",
+		.owner = THIS_MODULE,
+		.of_match_table = st7586_of_match,
+	},
+	.id_table = st7586_id,
+	.probe = st7586_probe,
+	.shutdown = st7586_shutdown,
+};
+module_spi_driver(st7586_spi_driver);
+
+MODULE_DESCRIPTION("Sitronix ST7586 DRM driver");
+MODULE_AUTHOR("David Lechner <david@lechnology.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index 22b57020790d..180ce6296416 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -70,6 +70,7 @@ static inline int ttm_mem_type_from_place(const struct ttm_place *place,
 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type)
 {
 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];
+	struct drm_printer p = drm_debug_printer(TTM_PFX);
 
 	pr_err("    has_type: %d\n", man->has_type);
 	pr_err("    use_type: %d\n", man->use_type);
@@ -79,7 +80,7 @@ static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type)
 	pr_err("    available_caching: 0x%08X\n", man->available_caching);
 	pr_err("    default_caching: 0x%08X\n", man->default_caching);
 	if (mem_type != TTM_PL_SYSTEM)
-		(*man->func->debug)(man, TTM_PFX);
+		(*man->func->debug)(man, &p);
 }
 
 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
@@ -108,8 +109,8 @@ static ssize_t ttm_bo_global_show(struct kobject *kobj,
 	struct ttm_bo_global *glob =
 		container_of(kobj, struct ttm_bo_global, kobj);
 
-	return snprintf(buffer, PAGE_SIZE, "%lu\n",
-			(unsigned long) atomic_read(&glob->bo_count));
+	return snprintf(buffer, PAGE_SIZE, "%d\n",
+				atomic_read(&glob->bo_count));
 }
 
 static struct attribute *ttm_bo_global_attrs[] = {
@@ -394,14 +395,33 @@ static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
 	ww_mutex_unlock (&bo->resv->lock);
 }
 
+static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo)
+{
+	int r;
+
+	if (bo->resv == &bo->ttm_resv)
+		return 0;
+
+	reservation_object_init(&bo->ttm_resv);
+	BUG_ON(!reservation_object_trylock(&bo->ttm_resv));
+
+	r = reservation_object_copy_fences(&bo->ttm_resv, bo->resv);
+	if (r) {
+		reservation_object_unlock(&bo->ttm_resv);
+		reservation_object_fini(&bo->ttm_resv);
+	}
+
+	return r;
+}
+
 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
 {
 	struct reservation_object_list *fobj;
 	struct dma_fence *fence;
 	int i;
 
-	fobj = reservation_object_get_list(bo->resv);
-	fence = reservation_object_get_excl(bo->resv);
+	fobj = reservation_object_get_list(&bo->ttm_resv);
+	fence = reservation_object_get_excl(&bo->ttm_resv);
 	if (fence && !fence->ops->signaled)
 		dma_fence_enable_sw_signaling(fence);
 
@@ -430,8 +450,19 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
 			ttm_bo_cleanup_memtype_use(bo);
 
 			return;
-		} else
-			ttm_bo_flush_all_fences(bo);
+		}
+
+		ret = ttm_bo_individualize_resv(bo);
+		if (ret) {
+			/* Last resort, if we fail to allocate memory for the
+			 * fences block for the BO to become idle and free it.
+			 */
+			spin_unlock(&glob->lru_lock);
+			ttm_bo_wait(bo, true, true);
+			ttm_bo_cleanup_memtype_use(bo);
+			return;
+		}
+		ttm_bo_flush_all_fences(bo);
 
 		/*
 		 * Make NO_EVICT bos immediately available to
@@ -443,6 +474,8 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
 			ttm_bo_add_to_lru(bo);
 		}
 
+		if (bo->resv != &bo->ttm_resv)
+			reservation_object_unlock(&bo->ttm_resv);
 		__ttm_bo_unreserve(bo);
 	}
 
@@ -471,17 +504,25 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo,
 					  bool no_wait_gpu)
 {
 	struct ttm_bo_global *glob = bo->glob;
+	struct reservation_object *resv;
 	int ret;
 
-	ret = ttm_bo_wait(bo, false, true);
+	if (unlikely(list_empty(&bo->ddestroy)))
+		resv = bo->resv;
+	else
+		resv = &bo->ttm_resv;
+
+	if (reservation_object_test_signaled_rcu(resv, true))
+		ret = 0;
+	else
+		ret = -EBUSY;
 
 	if (ret && !no_wait_gpu) {
 		long lret;
 		ww_mutex_unlock(&bo->resv->lock);
 		spin_unlock(&glob->lru_lock);
 
-		lret = reservation_object_wait_timeout_rcu(bo->resv,
-							   true,
+		lret = reservation_object_wait_timeout_rcu(resv, true,
 							   interruptible,
 							   30 * HZ);
 
@@ -505,13 +546,6 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo,
 			spin_unlock(&glob->lru_lock);
 			return 0;
 		}
-
-		/*
-		 * remove sync_obj with ttm_bo_wait, the wait should be
-		 * finished, and no new wait object should have been added.
-		 */
-		ret = ttm_bo_wait(bo, false, true);
-		WARN_ON(ret);
 	}
 
 	if (ret || unlikely(list_empty(&bo->ddestroy))) {
diff --git a/drivers/gpu/drm/ttm/ttm_bo_manager.c b/drivers/gpu/drm/ttm/ttm_bo_manager.c
index 90a6c0b03afc..a7c232dc39cb 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_manager.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_manager.c
@@ -136,13 +136,12 @@ static int ttm_bo_man_takedown(struct ttm_mem_type_manager *man)
 }
 
 static void ttm_bo_man_debug(struct ttm_mem_type_manager *man,
-			     const char *prefix)
+			     struct drm_printer *printer)
 {
 	struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv;
-	struct drm_printer p = drm_debug_printer(prefix);
 
 	spin_lock(&rman->lock);
-	drm_mm_print(&rman->mm, &p);
+	drm_mm_print(&rman->mm, printer);
 	spin_unlock(&rman->lock);
 }
 
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
index d0459b392e5e..c934ad5b3903 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_util.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -469,6 +469,7 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
 	 * TODO: Explicit member copy would probably be better here.
 	 */
 
+	atomic_inc(&bo->glob->bo_count);
 	INIT_LIST_HEAD(&fbo->ddestroy);
 	INIT_LIST_HEAD(&fbo->lru);
 	INIT_LIST_HEAD(&fbo->swap);
diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c
index b442d12f2f7d..c8ebb757e36b 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_vm.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c
@@ -39,6 +39,7 @@
 #include <linux/rbtree.h>
 #include <linux/module.h>
 #include <linux/uaccess.h>
+#include <linux/mem_encrypt.h>
 
 #define TTM_BO_VM_NUM_PREFAULT 16
 
@@ -230,9 +231,11 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf)
 	 * first page.
 	 */
 	for (i = 0; i < TTM_BO_VM_NUM_PREFAULT; ++i) {
-		if (bo->mem.bus.is_iomem)
+		if (bo->mem.bus.is_iomem) {
+			/* Iomem should not be marked encrypted */
+			cvma.vm_page_prot = pgprot_decrypted(cvma.vm_page_prot);
 			pfn = bdev->driver->io_mem_pfn(bo, page_offset);
-		else {
+		} else {
 			page = ttm->pages[page_offset];
 			if (unlikely(!page && i == 0)) {
 				retval = VM_FAULT_OOM;
@@ -294,10 +297,87 @@ static void ttm_bo_vm_close(struct vm_area_struct *vma)
 	vma->vm_private_data = NULL;
 }
 
+static int ttm_bo_vm_access_kmap(struct ttm_buffer_object *bo,
+				 unsigned long offset,
+				 void *buf, int len, int write)
+{
+	unsigned long page = offset >> PAGE_SHIFT;
+	unsigned long bytes_left = len;
+	int ret;
+
+	/* Copy a page at a time, that way no extra virtual address
+	 * mapping is needed
+	 */
+	offset -= page << PAGE_SHIFT;
+	do {
+		unsigned long bytes = min(bytes_left, PAGE_SIZE - offset);
+		struct ttm_bo_kmap_obj map;
+		void *ptr;
+		bool is_iomem;
+
+		ret = ttm_bo_kmap(bo, page, 1, &map);
+		if (ret)
+			return ret;
+
+		ptr = (uint8_t *)ttm_kmap_obj_virtual(&map, &is_iomem) + offset;
+		WARN_ON_ONCE(is_iomem);
+		if (write)
+			memcpy(ptr, buf, bytes);
+		else
+			memcpy(buf, ptr, bytes);
+		ttm_bo_kunmap(&map);
+
+		page++;
+		bytes_left -= bytes;
+		offset = 0;
+	} while (bytes_left);
+
+	return len;
+}
+
+static int ttm_bo_vm_access(struct vm_area_struct *vma, unsigned long addr,
+			    void *buf, int len, int write)
+{
+	unsigned long offset = (addr) - vma->vm_start;
+	struct ttm_buffer_object *bo = vma->vm_private_data;
+	int ret;
+
+	if (len < 1 || (offset + len) >> PAGE_SHIFT > bo->num_pages)
+		return -EIO;
+
+	ret = ttm_bo_reserve(bo, true, false, NULL);
+	if (ret)
+		return ret;
+
+	switch (bo->mem.mem_type) {
+	case TTM_PL_SYSTEM:
+		if (unlikely(bo->ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {
+			ret = ttm_tt_swapin(bo->ttm);
+			if (unlikely(ret != 0))
+				return ret;
+		}
+		/* fall through */
+	case TTM_PL_TT:
+		ret = ttm_bo_vm_access_kmap(bo, offset, buf, len, write);
+		break;
+	default:
+		if (bo->bdev->driver->access_memory)
+			ret = bo->bdev->driver->access_memory(
+				bo, offset, buf, len, write);
+		else
+			ret = -EIO;
+	}
+
+	ttm_bo_unreserve(bo);
+
+	return ret;
+}
+
 static const struct vm_operations_struct ttm_bo_vm_ops = {
 	.fault = ttm_bo_vm_fault,
 	.open = ttm_bo_vm_open,
-	.close = ttm_bo_vm_close
+	.close = ttm_bo_vm_close,
+	.access = ttm_bo_vm_access
 };
 
 static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev,
diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c
index eeddc1e48409..871599826773 100644
--- a/drivers/gpu/drm/ttm/ttm_page_alloc.c
+++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c
@@ -615,7 +615,7 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
 		} else {
 			pr_err("Failed to fill pool (%p)\n", pool);
 			/* If we have any pages left put them to the pool. */
-			list_for_each_entry(p, &pool->list, lru) {
+			list_for_each_entry(p, &new_pages, lru) {
 				++cpages;
 			}
 			list_splice(&new_pages, &pool->list);
diff --git a/drivers/gpu/drm/udl/udl_connector.c b/drivers/gpu/drm/udl/udl_connector.c
index d2f57c52f7db..9f9a49748d17 100644
--- a/drivers/gpu/drm/udl/udl_connector.c
+++ b/drivers/gpu/drm/udl/udl_connector.c
@@ -96,7 +96,7 @@ static int udl_mode_valid(struct drm_connector *connector,
 static enum drm_connector_status
 udl_detect(struct drm_connector *connector, bool force)
 {
-	if (drm_device_is_unplugged(connector->dev))
+	if (drm_dev_is_unplugged(connector->dev))
 		return connector_status_disconnected;
 	return connector_status_connected;
 }
diff --git a/drivers/gpu/drm/udl/udl_dmabuf.c b/drivers/gpu/drm/udl/udl_dmabuf.c
index 2e031a894813..2867ed155ff6 100644
--- a/drivers/gpu/drm/udl/udl_dmabuf.c
+++ b/drivers/gpu/drm/udl/udl_dmabuf.c
@@ -186,7 +186,7 @@ static int udl_dmabuf_mmap(struct dma_buf *dma_buf,
 	return -EINVAL;
 }
 
-static struct dma_buf_ops udl_dmabuf_ops = {
+static const struct dma_buf_ops udl_dmabuf_ops = {
 	.attach			= udl_attach_dma_buf,
 	.detach			= udl_detach_dma_buf,
 	.map_dma_buf		= udl_map_dma_buf,
diff --git a/drivers/gpu/drm/udl/udl_drv.c b/drivers/gpu/drm/udl/udl_drv.c
index cd8b01727734..31421b6b586e 100644
--- a/drivers/gpu/drm/udl/udl_drv.c
+++ b/drivers/gpu/drm/udl/udl_drv.c
@@ -11,11 +11,6 @@
 #include <drm/drm_crtc_helper.h>
 #include "udl_drv.h"
 
-static int udl_driver_set_busid(struct drm_device *d, struct drm_master *m)
-{
-	return 0;
-}
-
 static int udl_usb_suspend(struct usb_interface *interface,
 			   pm_message_t message)
 {
@@ -52,7 +47,6 @@ static struct drm_driver driver = {
 	.driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME,
 	.load = udl_driver_load,
 	.unload = udl_driver_unload,
-	.set_busid = udl_driver_set_busid,
 
 	/* gem hooks */
 	.gem_free_object = udl_gem_free_object,
@@ -60,7 +54,6 @@ static struct drm_driver driver = {
 
 	.dumb_create = udl_dumb_create,
 	.dumb_map_offset = udl_gem_mmap,
-	.dumb_destroy = drm_gem_dumb_destroy,
 	.fops = &udl_driver_fops,
 
 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
@@ -108,7 +101,7 @@ static void udl_usb_disconnect(struct usb_interface *interface)
 	drm_kms_helper_poll_disable(dev);
 	udl_fbdev_unplug(dev);
 	udl_drop_usb(dev);
-	drm_unplug_dev(dev);
+	drm_dev_unplug(dev);
 }
 
 /*
@@ -118,7 +111,7 @@ static void udl_usb_disconnect(struct usb_interface *interface)
  * which is compatible with all known USB 2.0 era graphics chips and firmware,
  * but allows DisplayLink to increment those for any future incompatible chips
  */
-static struct usb_device_id id_table[] = {
+static const struct usb_device_id id_table[] = {
 	{.idVendor = 0x17e9, .bInterfaceClass = 0xff,
 	 .bInterfaceSubClass = 0x00,
 	 .bInterfaceProtocol = 0x00,
diff --git a/drivers/gpu/drm/udl/udl_fb.c b/drivers/gpu/drm/udl/udl_fb.c
index 4a6500362564..b5b335c9b2bb 100644
--- a/drivers/gpu/drm/udl/udl_fb.c
+++ b/drivers/gpu/drm/udl/udl_fb.c
@@ -14,6 +14,7 @@
 #include <linux/slab.h>
 #include <linux/fb.h>
 #include <linux/dma-buf.h>
+#include <linux/mem_encrypt.h>
 
 #include <drm/drmP.h>
 #include <drm/drm_crtc.h>
@@ -169,6 +170,9 @@ static int udl_fb_mmap(struct fb_info *info, struct vm_area_struct *vma)
 	pr_notice("mmap() framebuffer addr:%lu size:%lu\n",
 		  pos, size);
 
+	/* We don't want the framebuffer to be mapped encrypted */
+	vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
+
 	while (size > 0) {
 		page = vmalloc_to_pfn((void *)pos);
 		if (remap_pfn_range(vma, start, page, PAGE_SIZE, PAGE_SHARED))
@@ -198,7 +202,7 @@ static int udl_fb_open(struct fb_info *info, int user)
 	struct udl_device *udl = dev->dev_private;
 
 	/* If the USB device is gone, we don't accept new opens */
-	if (drm_device_is_unplugged(udl->ddev))
+	if (drm_dev_is_unplugged(udl->ddev))
 		return -ENODEV;
 
 	ufbdev->fb_count++;
@@ -309,7 +313,7 @@ static void udl_user_framebuffer_destroy(struct drm_framebuffer *fb)
 	struct udl_framebuffer *ufb = to_udl_fb(fb);
 
 	if (ufb->obj)
-		drm_gem_object_unreference_unlocked(&ufb->obj->base);
+		drm_gem_object_put_unlocked(&ufb->obj->base);
 
 	drm_framebuffer_cleanup(fb);
 	kfree(ufb);
@@ -393,7 +397,6 @@ static int udlfb_create(struct drm_fb_helper *helper,
 	info->fix.smem_len = size;
 	info->fix.smem_start = (unsigned long)ufbdev->ufb.obj->vmapping;
 
-	info->flags = FBINFO_DEFAULT | FBINFO_CAN_FORCE_OUTPUT;
 	info->fbops = &udlfb_ops;
 	drm_fb_helper_fill_fix(info, fb->pitches[0], fb->format->depth);
 	drm_fb_helper_fill_var(info, &ufbdev->helper, sizes->fb_width, sizes->fb_height);
@@ -404,7 +407,7 @@ static int udlfb_create(struct drm_fb_helper *helper,
 
 	return ret;
 out_gfree:
-	drm_gem_object_unreference_unlocked(&ufbdev->ufb.obj->base);
+	drm_gem_object_put_unlocked(&ufbdev->ufb.obj->base);
 out:
 	return ret;
 }
@@ -420,7 +423,7 @@ static void udl_fbdev_destroy(struct drm_device *dev,
 	drm_fb_helper_fini(&ufbdev->helper);
 	drm_framebuffer_unregister_private(&ufbdev->ufb.base);
 	drm_framebuffer_cleanup(&ufbdev->ufb.base);
-	drm_gem_object_unreference_unlocked(&ufbdev->ufb.obj->base);
+	drm_gem_object_put_unlocked(&ufbdev->ufb.obj->base);
 }
 
 int udl_fbdev_init(struct drm_device *dev)
diff --git a/drivers/gpu/drm/udl/udl_gem.c b/drivers/gpu/drm/udl/udl_gem.c
index db9ceceba30e..dee6bd9a3dd1 100644
--- a/drivers/gpu/drm/udl/udl_gem.c
+++ b/drivers/gpu/drm/udl/udl_gem.c
@@ -52,7 +52,7 @@ udl_gem_create(struct drm_file *file,
 		return ret;
 	}
 
-	drm_gem_object_unreference_unlocked(&obj->base);
+	drm_gem_object_put_unlocked(&obj->base);
 	*handle_p = handle;
 	return 0;
 }
@@ -234,7 +234,7 @@ int udl_gem_mmap(struct drm_file *file, struct drm_device *dev,
 	*offset = drm_vma_node_offset_addr(&gobj->base.vma_node);
 
 out:
-	drm_gem_object_unreference(&gobj->base);
+	drm_gem_object_put(&gobj->base);
 unlock:
 	mutex_unlock(&dev->struct_mutex);
 	return ret;
diff --git a/drivers/gpu/drm/udl/udl_main.c b/drivers/gpu/drm/udl/udl_main.c
index a9d93b871a15..0328b2c7b210 100644
--- a/drivers/gpu/drm/udl/udl_main.c
+++ b/drivers/gpu/drm/udl/udl_main.c
@@ -371,8 +371,6 @@ void udl_driver_unload(struct drm_device *dev)
 {
 	struct udl_device *udl = dev->dev_private;
 
-	drm_vblank_cleanup(dev);
-
 	if (udl->urbs.count)
 		udl_free_urb_list(dev);
 
diff --git a/drivers/gpu/drm/vc4/Kconfig b/drivers/gpu/drm/vc4/Kconfig
index 4361bdcfd28a..fdae18aeab4f 100644
--- a/drivers/gpu/drm/vc4/Kconfig
+++ b/drivers/gpu/drm/vc4/Kconfig
@@ -19,3 +19,11 @@ config DRM_VC4
 	  This driver requires that "avoid_warnings=2" be present in
 	  the config.txt for the firmware, to keep it from smashing
 	  our display setup.
+
+config DRM_VC4_HDMI_CEC
+       bool "Broadcom VC4 HDMI CEC Support"
+       depends on DRM_VC4
+       select CEC_CORE
+       help
+	  Choose this option if you have a Broadcom VC4 GPU
+	  and want to use CEC.
diff --git a/drivers/gpu/drm/vc4/vc4_bo.c b/drivers/gpu/drm/vc4/vc4_bo.c
index 487f96412d35..3afdbf4bc10b 100644
--- a/drivers/gpu/drm/vc4/vc4_bo.c
+++ b/drivers/gpu/drm/vc4/vc4_bo.c
@@ -24,21 +24,35 @@
 #include "vc4_drv.h"
 #include "uapi/drm/vc4_drm.h"
 
+static const char * const bo_type_names[] = {
+	"kernel",
+	"V3D",
+	"V3D shader",
+	"dumb",
+	"binner",
+	"RCL",
+	"BCL",
+	"kernel BO cache",
+};
+
+static bool is_user_label(int label)
+{
+	return label >= VC4_BO_TYPE_COUNT;
+}
+
 static void vc4_bo_stats_dump(struct vc4_dev *vc4)
 {
-	DRM_INFO("num bos allocated: %d\n",
-		 vc4->bo_stats.num_allocated);
-	DRM_INFO("size bos allocated: %dkb\n",
-		 vc4->bo_stats.size_allocated / 1024);
-	DRM_INFO("num bos used: %d\n",
-		 vc4->bo_stats.num_allocated - vc4->bo_stats.num_cached);
-	DRM_INFO("size bos used: %dkb\n",
-		 (vc4->bo_stats.size_allocated -
-		  vc4->bo_stats.size_cached) / 1024);
-	DRM_INFO("num bos cached: %d\n",
-		 vc4->bo_stats.num_cached);
-	DRM_INFO("size bos cached: %dkb\n",
-		 vc4->bo_stats.size_cached / 1024);
+	int i;
+
+	for (i = 0; i < vc4->num_labels; i++) {
+		if (!vc4->bo_labels[i].num_allocated)
+			continue;
+
+		DRM_INFO("%30s: %6dkb BOs (%d)\n",
+			 vc4->bo_labels[i].name,
+			 vc4->bo_labels[i].size_allocated / 1024,
+			 vc4->bo_labels[i].num_allocated);
+	}
 }
 
 #ifdef CONFIG_DEBUG_FS
@@ -47,64 +61,133 @@ int vc4_bo_stats_debugfs(struct seq_file *m, void *unused)
 	struct drm_info_node *node = (struct drm_info_node *)m->private;
 	struct drm_device *dev = node->minor->dev;
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
-	struct vc4_bo_stats stats;
+	int i;
 
-	/* Take a snapshot of the current stats with the lock held. */
 	mutex_lock(&vc4->bo_lock);
-	stats = vc4->bo_stats;
+	for (i = 0; i < vc4->num_labels; i++) {
+		if (!vc4->bo_labels[i].num_allocated)
+			continue;
+
+		seq_printf(m, "%30s: %6dkb BOs (%d)\n",
+			   vc4->bo_labels[i].name,
+			   vc4->bo_labels[i].size_allocated / 1024,
+			   vc4->bo_labels[i].num_allocated);
+	}
 	mutex_unlock(&vc4->bo_lock);
 
-	seq_printf(m, "num bos allocated: %d\n",
-		   stats.num_allocated);
-	seq_printf(m, "size bos allocated: %dkb\n",
-		   stats.size_allocated / 1024);
-	seq_printf(m, "num bos used: %d\n",
-		   stats.num_allocated - stats.num_cached);
-	seq_printf(m, "size bos used: %dkb\n",
-		   (stats.size_allocated - stats.size_cached) / 1024);
-	seq_printf(m, "num bos cached: %d\n",
-		   stats.num_cached);
-	seq_printf(m, "size bos cached: %dkb\n",
-		   stats.size_cached / 1024);
-
 	return 0;
 }
 #endif
 
+/* Takes ownership of *name and returns the appropriate slot for it in
+ * the bo_labels[] array, extending it as necessary.
+ *
+ * This is inefficient and could use a hash table instead of walking
+ * an array and strcmp()ing.  However, the assumption is that user
+ * labeling will be infrequent (scanout buffers and other long-lived
+ * objects, or debug driver builds), so we can live with it for now.
+ */
+static int vc4_get_user_label(struct vc4_dev *vc4, const char *name)
+{
+	int i;
+	int free_slot = -1;
+
+	for (i = 0; i < vc4->num_labels; i++) {
+		if (!vc4->bo_labels[i].name) {
+			free_slot = i;
+		} else if (strcmp(vc4->bo_labels[i].name, name) == 0) {
+			kfree(name);
+			return i;
+		}
+	}
+
+	if (free_slot != -1) {
+		WARN_ON(vc4->bo_labels[free_slot].num_allocated != 0);
+		vc4->bo_labels[free_slot].name = name;
+		return free_slot;
+	} else {
+		u32 new_label_count = vc4->num_labels + 1;
+		struct vc4_label *new_labels =
+			krealloc(vc4->bo_labels,
+				 new_label_count * sizeof(*new_labels),
+				 GFP_KERNEL);
+
+		if (!new_labels) {
+			kfree(name);
+			return -1;
+		}
+
+		free_slot = vc4->num_labels;
+		vc4->bo_labels = new_labels;
+		vc4->num_labels = new_label_count;
+
+		vc4->bo_labels[free_slot].name = name;
+		vc4->bo_labels[free_slot].num_allocated = 0;
+		vc4->bo_labels[free_slot].size_allocated = 0;
+
+		return free_slot;
+	}
+}
+
+static void vc4_bo_set_label(struct drm_gem_object *gem_obj, int label)
+{
+	struct vc4_bo *bo = to_vc4_bo(gem_obj);
+	struct vc4_dev *vc4 = to_vc4_dev(gem_obj->dev);
+
+	lockdep_assert_held(&vc4->bo_lock);
+
+	if (label != -1) {
+		vc4->bo_labels[label].num_allocated++;
+		vc4->bo_labels[label].size_allocated += gem_obj->size;
+	}
+
+	vc4->bo_labels[bo->label].num_allocated--;
+	vc4->bo_labels[bo->label].size_allocated -= gem_obj->size;
+
+	if (vc4->bo_labels[bo->label].num_allocated == 0 &&
+	    is_user_label(bo->label)) {
+		/* Free user BO label slots on last unreference.
+		 * Slots are just where we track the stats for a given
+		 * name, and once a name is unused we can reuse that
+		 * slot.
+		 */
+		kfree(vc4->bo_labels[bo->label].name);
+		vc4->bo_labels[bo->label].name = NULL;
+	}
+
+	bo->label = label;
+}
+
 static uint32_t bo_page_index(size_t size)
 {
 	return (size / PAGE_SIZE) - 1;
 }
 
-/* Must be called with bo_lock held. */
 static void vc4_bo_destroy(struct vc4_bo *bo)
 {
 	struct drm_gem_object *obj = &bo->base.base;
 	struct vc4_dev *vc4 = to_vc4_dev(obj->dev);
 
+	lockdep_assert_held(&vc4->bo_lock);
+
+	vc4_bo_set_label(obj, -1);
+
 	if (bo->validated_shader) {
 		kfree(bo->validated_shader->texture_samples);
 		kfree(bo->validated_shader);
 		bo->validated_shader = NULL;
 	}
 
-	vc4->bo_stats.num_allocated--;
-	vc4->bo_stats.size_allocated -= obj->size;
-
 	reservation_object_fini(&bo->_resv);
 
 	drm_gem_cma_free_object(obj);
 }
 
-/* Must be called with bo_lock held. */
 static void vc4_bo_remove_from_cache(struct vc4_bo *bo)
 {
-	struct drm_gem_object *obj = &bo->base.base;
-	struct vc4_dev *vc4 = to_vc4_dev(obj->dev);
-
-	vc4->bo_stats.num_cached--;
-	vc4->bo_stats.size_cached -= obj->size;
+	struct vc4_dev *vc4 = to_vc4_dev(bo->base.base.dev);
 
+	lockdep_assert_held(&vc4->bo_lock);
 	list_del(&bo->unref_head);
 	list_del(&bo->size_head);
 }
@@ -165,7 +248,8 @@ static void vc4_bo_cache_purge(struct drm_device *dev)
 }
 
 static struct vc4_bo *vc4_bo_get_from_cache(struct drm_device *dev,
-					    uint32_t size)
+					    uint32_t size,
+					    enum vc4_kernel_bo_type type)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
 	uint32_t page_index = bo_page_index(size);
@@ -186,6 +270,8 @@ static struct vc4_bo *vc4_bo_get_from_cache(struct drm_device *dev,
 	kref_init(&bo->base.base.refcount);
 
 out:
+	if (bo)
+		vc4_bo_set_label(&bo->base.base, type);
 	mutex_unlock(&vc4->bo_lock);
 	return bo;
 }
@@ -208,8 +294,9 @@ struct drm_gem_object *vc4_create_object(struct drm_device *dev, size_t size)
 		return ERR_PTR(-ENOMEM);
 
 	mutex_lock(&vc4->bo_lock);
-	vc4->bo_stats.num_allocated++;
-	vc4->bo_stats.size_allocated += size;
+	bo->label = VC4_BO_TYPE_KERNEL;
+	vc4->bo_labels[VC4_BO_TYPE_KERNEL].num_allocated++;
+	vc4->bo_labels[VC4_BO_TYPE_KERNEL].size_allocated += size;
 	mutex_unlock(&vc4->bo_lock);
 	bo->resv = &bo->_resv;
 	reservation_object_init(bo->resv);
@@ -218,7 +305,7 @@ struct drm_gem_object *vc4_create_object(struct drm_device *dev, size_t size)
 }
 
 struct vc4_bo *vc4_bo_create(struct drm_device *dev, size_t unaligned_size,
-			     bool allow_unzeroed)
+			     bool allow_unzeroed, enum vc4_kernel_bo_type type)
 {
 	size_t size = roundup(unaligned_size, PAGE_SIZE);
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
@@ -229,7 +316,7 @@ struct vc4_bo *vc4_bo_create(struct drm_device *dev, size_t unaligned_size,
 		return ERR_PTR(-EINVAL);
 
 	/* First, try to get a vc4_bo from the kernel BO cache. */
-	bo = vc4_bo_get_from_cache(dev, size);
+	bo = vc4_bo_get_from_cache(dev, size, type);
 	if (bo) {
 		if (!allow_unzeroed)
 			memset(bo->base.vaddr, 0, bo->base.base.size);
@@ -251,7 +338,13 @@ struct vc4_bo *vc4_bo_create(struct drm_device *dev, size_t unaligned_size,
 			return ERR_PTR(-ENOMEM);
 		}
 	}
-	return to_vc4_bo(&cma_obj->base);
+	bo = to_vc4_bo(&cma_obj->base);
+
+	mutex_lock(&vc4->bo_lock);
+	vc4_bo_set_label(&cma_obj->base, type);
+	mutex_unlock(&vc4->bo_lock);
+
+	return bo;
 }
 
 int vc4_dumb_create(struct drm_file *file_priv,
@@ -268,22 +361,23 @@ int vc4_dumb_create(struct drm_file *file_priv,
 	if (args->size < args->pitch * args->height)
 		args->size = args->pitch * args->height;
 
-	bo = vc4_bo_create(dev, args->size, false);
+	bo = vc4_bo_create(dev, args->size, false, VC4_BO_TYPE_DUMB);
 	if (IS_ERR(bo))
 		return PTR_ERR(bo);
 
 	ret = drm_gem_handle_create(file_priv, &bo->base.base, &args->handle);
-	drm_gem_object_unreference_unlocked(&bo->base.base);
+	drm_gem_object_put_unlocked(&bo->base.base);
 
 	return ret;
 }
 
-/* Must be called with bo_lock held. */
 static void vc4_bo_cache_free_old(struct drm_device *dev)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
 	unsigned long expire_time = jiffies - msecs_to_jiffies(1000);
 
+	lockdep_assert_held(&vc4->bo_lock);
+
 	while (!list_empty(&vc4->bo_cache.time_list)) {
 		struct vc4_bo *bo = list_last_entry(&vc4->bo_cache.time_list,
 						    struct vc4_bo, unref_head);
@@ -348,8 +442,7 @@ void vc4_free_object(struct drm_gem_object *gem_bo)
 	list_add(&bo->size_head, cache_list);
 	list_add(&bo->unref_head, &vc4->bo_cache.time_list);
 
-	vc4->bo_stats.num_cached++;
-	vc4->bo_stats.size_cached += gem_bo->size;
+	vc4_bo_set_label(&bo->base.base, VC4_BO_TYPE_KERNEL_CACHE);
 
 	vc4_bo_cache_free_old(dev);
 
@@ -389,7 +482,7 @@ vc4_prime_export(struct drm_device *dev, struct drm_gem_object *obj, int flags)
 	struct vc4_bo *bo = to_vc4_bo(obj);
 
 	if (bo->validated_shader) {
-		DRM_ERROR("Attempting to export shader BO\n");
+		DRM_DEBUG("Attempting to export shader BO\n");
 		return ERR_PTR(-EINVAL);
 	}
 
@@ -410,7 +503,7 @@ int vc4_mmap(struct file *filp, struct vm_area_struct *vma)
 	bo = to_vc4_bo(gem_obj);
 
 	if (bo->validated_shader && (vma->vm_flags & VM_WRITE)) {
-		DRM_ERROR("mmaping of shader BOs for writing not allowed.\n");
+		DRM_DEBUG("mmaping of shader BOs for writing not allowed.\n");
 		return -EINVAL;
 	}
 
@@ -435,7 +528,7 @@ int vc4_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
 	struct vc4_bo *bo = to_vc4_bo(obj);
 
 	if (bo->validated_shader && (vma->vm_flags & VM_WRITE)) {
-		DRM_ERROR("mmaping of shader BOs for writing not allowed.\n");
+		DRM_DEBUG("mmaping of shader BOs for writing not allowed.\n");
 		return -EINVAL;
 	}
 
@@ -447,7 +540,7 @@ void *vc4_prime_vmap(struct drm_gem_object *obj)
 	struct vc4_bo *bo = to_vc4_bo(obj);
 
 	if (bo->validated_shader) {
-		DRM_ERROR("mmaping of shader BOs not allowed.\n");
+		DRM_DEBUG("mmaping of shader BOs not allowed.\n");
 		return ERR_PTR(-EINVAL);
 	}
 
@@ -483,12 +576,12 @@ int vc4_create_bo_ioctl(struct drm_device *dev, void *data,
 	 * We can't allocate from the BO cache, because the BOs don't
 	 * get zeroed, and that might leak data between users.
 	 */
-	bo = vc4_bo_create(dev, args->size, false);
+	bo = vc4_bo_create(dev, args->size, false, VC4_BO_TYPE_V3D);
 	if (IS_ERR(bo))
 		return PTR_ERR(bo);
 
 	ret = drm_gem_handle_create(file_priv, &bo->base.base, &args->handle);
-	drm_gem_object_unreference_unlocked(&bo->base.base);
+	drm_gem_object_put_unlocked(&bo->base.base);
 
 	return ret;
 }
@@ -501,14 +594,14 @@ int vc4_mmap_bo_ioctl(struct drm_device *dev, void *data,
 
 	gem_obj = drm_gem_object_lookup(file_priv, args->handle);
 	if (!gem_obj) {
-		DRM_ERROR("Failed to look up GEM BO %d\n", args->handle);
+		DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle);
 		return -EINVAL;
 	}
 
 	/* The mmap offset was set up at BO allocation time. */
 	args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node);
 
-	drm_gem_object_unreference_unlocked(gem_obj);
+	drm_gem_object_put_unlocked(gem_obj);
 	return 0;
 }
 
@@ -536,7 +629,7 @@ vc4_create_shader_bo_ioctl(struct drm_device *dev, void *data,
 		return -EINVAL;
 	}
 
-	bo = vc4_bo_create(dev, args->size, true);
+	bo = vc4_bo_create(dev, args->size, true, VC4_BO_TYPE_V3D_SHADER);
 	if (IS_ERR(bo))
 		return PTR_ERR(bo);
 
@@ -564,7 +657,7 @@ vc4_create_shader_bo_ioctl(struct drm_device *dev, void *data,
 	ret = drm_gem_handle_create(file_priv, &bo->base.base, &args->handle);
 
  fail:
-	drm_gem_object_unreference_unlocked(&bo->base.base);
+	drm_gem_object_put_unlocked(&bo->base.base);
 
 	return ret;
 }
@@ -605,13 +698,13 @@ int vc4_set_tiling_ioctl(struct drm_device *dev, void *data,
 
 	gem_obj = drm_gem_object_lookup(file_priv, args->handle);
 	if (!gem_obj) {
-		DRM_ERROR("Failed to look up GEM BO %d\n", args->handle);
+		DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle);
 		return -ENOENT;
 	}
 	bo = to_vc4_bo(gem_obj);
 	bo->t_format = t_format;
 
-	drm_gem_object_unreference_unlocked(gem_obj);
+	drm_gem_object_put_unlocked(gem_obj);
 
 	return 0;
 }
@@ -636,7 +729,7 @@ int vc4_get_tiling_ioctl(struct drm_device *dev, void *data,
 
 	gem_obj = drm_gem_object_lookup(file_priv, args->handle);
 	if (!gem_obj) {
-		DRM_ERROR("Failed to look up GEM BO %d\n", args->handle);
+		DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle);
 		return -ENOENT;
 	}
 	bo = to_vc4_bo(gem_obj);
@@ -646,14 +739,29 @@ int vc4_get_tiling_ioctl(struct drm_device *dev, void *data,
 	else
 		args->modifier = DRM_FORMAT_MOD_NONE;
 
-	drm_gem_object_unreference_unlocked(gem_obj);
+	drm_gem_object_put_unlocked(gem_obj);
 
 	return 0;
 }
 
-void vc4_bo_cache_init(struct drm_device *dev)
+int vc4_bo_cache_init(struct drm_device *dev)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	int i;
+
+	/* Create the initial set of BO labels that the kernel will
+	 * use.  This lets us avoid a bunch of string reallocation in
+	 * the kernel's draw and BO allocation paths.
+	 */
+	vc4->bo_labels = kcalloc(VC4_BO_TYPE_COUNT, sizeof(*vc4->bo_labels),
+				 GFP_KERNEL);
+	if (!vc4->bo_labels)
+		return -ENOMEM;
+	vc4->num_labels = VC4_BO_TYPE_COUNT;
+
+	BUILD_BUG_ON(ARRAY_SIZE(bo_type_names) != VC4_BO_TYPE_COUNT);
+	for (i = 0; i < VC4_BO_TYPE_COUNT; i++)
+		vc4->bo_labels[i].name = bo_type_names[i];
 
 	mutex_init(&vc4->bo_lock);
 
@@ -663,19 +771,66 @@ void vc4_bo_cache_init(struct drm_device *dev)
 	setup_timer(&vc4->bo_cache.time_timer,
 		    vc4_bo_cache_time_timer,
 		    (unsigned long)dev);
+
+	return 0;
 }
 
 void vc4_bo_cache_destroy(struct drm_device *dev)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	int i;
 
 	del_timer(&vc4->bo_cache.time_timer);
 	cancel_work_sync(&vc4->bo_cache.time_work);
 
 	vc4_bo_cache_purge(dev);
 
-	if (vc4->bo_stats.num_allocated) {
-		DRM_ERROR("Destroying BO cache while BOs still allocated:\n");
-		vc4_bo_stats_dump(vc4);
+	for (i = 0; i < vc4->num_labels; i++) {
+		if (vc4->bo_labels[i].num_allocated) {
+			DRM_ERROR("Destroying BO cache with %d %s "
+				  "BOs still allocated\n",
+				  vc4->bo_labels[i].num_allocated,
+				  vc4->bo_labels[i].name);
+		}
+
+		if (is_user_label(i))
+			kfree(vc4->bo_labels[i].name);
 	}
+	kfree(vc4->bo_labels);
+}
+
+int vc4_label_bo_ioctl(struct drm_device *dev, void *data,
+		       struct drm_file *file_priv)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	struct drm_vc4_label_bo *args = data;
+	char *name;
+	struct drm_gem_object *gem_obj;
+	int ret = 0, label;
+
+	if (!args->len)
+		return -EINVAL;
+
+	name = strndup_user(u64_to_user_ptr(args->name), args->len + 1);
+	if (IS_ERR(name))
+		return PTR_ERR(name);
+
+	gem_obj = drm_gem_object_lookup(file_priv, args->handle);
+	if (!gem_obj) {
+		DRM_ERROR("Failed to look up GEM BO %d\n", args->handle);
+		kfree(name);
+		return -ENOENT;
+	}
+
+	mutex_lock(&vc4->bo_lock);
+	label = vc4_get_user_label(vc4, name);
+	if (label != -1)
+		vc4_bo_set_label(gem_obj, label);
+	else
+		ret = -ENOMEM;
+	mutex_unlock(&vc4->bo_lock);
+
+	drm_gem_object_put_unlocked(gem_obj);
+
+	return ret;
 }
diff --git a/drivers/gpu/drm/vc4/vc4_crtc.c b/drivers/gpu/drm/vc4/vc4_crtc.c
index a12cc7ea99b6..ce1e3b9e14c9 100644
--- a/drivers/gpu/drm/vc4/vc4_crtc.c
+++ b/drivers/gpu/drm/vc4/vc4_crtc.c
@@ -479,7 +479,8 @@ static void require_hvs_enabled(struct drm_device *dev)
 		     SCALER_DISPCTRL_ENABLE);
 }
 
-static void vc4_crtc_disable(struct drm_crtc *crtc)
+static void vc4_crtc_atomic_disable(struct drm_crtc *crtc,
+				    struct drm_crtc_state *old_state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
@@ -518,6 +519,19 @@ static void vc4_crtc_disable(struct drm_crtc *crtc)
 	WARN_ON_ONCE((HVS_READ(SCALER_DISPSTATX(chan)) &
 		      (SCALER_DISPSTATX_FULL | SCALER_DISPSTATX_EMPTY)) !=
 		     SCALER_DISPSTATX_EMPTY);
+
+	/*
+	 * Make sure we issue a vblank event after disabling the CRTC if
+	 * someone was waiting it.
+	 */
+	if (crtc->state->event) {
+		unsigned long flags;
+
+		spin_lock_irqsave(&dev->event_lock, flags);
+		drm_crtc_send_vblank_event(crtc, crtc->state->event);
+		crtc->state->event = NULL;
+		spin_unlock_irqrestore(&dev->event_lock, flags);
+	}
 }
 
 static void vc4_crtc_update_dlist(struct drm_crtc *crtc)
@@ -548,7 +562,8 @@ static void vc4_crtc_update_dlist(struct drm_crtc *crtc)
 	}
 }
 
-static void vc4_crtc_enable(struct drm_crtc *crtc)
+static void vc4_crtc_atomic_enable(struct drm_crtc *crtc,
+				   struct drm_crtc_state *old_state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
@@ -577,18 +592,17 @@ static void vc4_crtc_enable(struct drm_crtc *crtc)
 		   CRTC_READ(PV_V_CONTROL) | PV_VCONTROL_VIDEN);
 }
 
-static bool vc4_crtc_mode_fixup(struct drm_crtc *crtc,
-				const struct drm_display_mode *mode,
-				struct drm_display_mode *adjusted_mode)
+static enum drm_mode_status vc4_crtc_mode_valid(struct drm_crtc *crtc,
+						const struct drm_display_mode *mode)
 {
 	/* Do not allow doublescan modes from user space */
-	if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) {
+	if (mode->flags & DRM_MODE_FLAG_DBLSCAN) {
 		DRM_DEBUG_KMS("[CRTC:%d] Doublescan mode rejected.\n",
 			      crtc->base.id);
-		return false;
+		return MODE_NO_DBLESCAN;
 	}
 
-	return true;
+	return MODE_OK;
 }
 
 static int vc4_crtc_atomic_check(struct drm_crtc *crtc,
@@ -682,14 +696,6 @@ static void vc4_disable_vblank(struct drm_crtc *crtc)
 	CRTC_WRITE(PV_INTEN, 0);
 }
 
-/* Must be called with the event lock held */
-bool vc4_event_pending(struct drm_crtc *crtc)
-{
-	struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
-
-	return !!vc4_crtc->event;
-}
-
 static void vc4_crtc_handle_page_flip(struct vc4_crtc *vc4_crtc)
 {
 	struct drm_crtc *crtc = &vc4_crtc->base;
@@ -757,7 +763,7 @@ vc4_async_page_flip_complete(struct vc4_seqno_cb *cb)
 	}
 
 	drm_crtc_vblank_put(crtc);
-	drm_framebuffer_unreference(flip_state->fb);
+	drm_framebuffer_put(flip_state->fb);
 	kfree(flip_state);
 
 	up(&vc4->async_modeset);
@@ -786,7 +792,7 @@ static int vc4_async_page_flip(struct drm_crtc *crtc,
 	if (!flip_state)
 		return -ENOMEM;
 
-	drm_framebuffer_reference(fb);
+	drm_framebuffer_get(fb);
 	flip_state->fb = fb;
 	flip_state->crtc = crtc;
 	flip_state->event = event;
@@ -794,7 +800,7 @@ static int vc4_async_page_flip(struct drm_crtc *crtc,
 	/* Make sure all other async modesetes have landed. */
 	ret = down_interruptible(&vc4->async_modeset);
 	if (ret) {
-		drm_framebuffer_unreference(fb);
+		drm_framebuffer_put(fb);
 		kfree(flip_state);
 		return ret;
 	}
@@ -885,11 +891,11 @@ static const struct drm_crtc_funcs vc4_crtc_funcs = {
 
 static const struct drm_crtc_helper_funcs vc4_crtc_helper_funcs = {
 	.mode_set_nofb = vc4_crtc_mode_set_nofb,
-	.disable = vc4_crtc_disable,
-	.enable = vc4_crtc_enable,
-	.mode_fixup = vc4_crtc_mode_fixup,
+	.mode_valid = vc4_crtc_mode_valid,
 	.atomic_check = vc4_crtc_atomic_check,
 	.atomic_flush = vc4_crtc_atomic_flush,
+	.atomic_enable = vc4_crtc_atomic_enable,
+	.atomic_disable = vc4_crtc_atomic_disable,
 };
 
 static const struct vc4_crtc_data pv0_data = {
diff --git a/drivers/gpu/drm/vc4/vc4_dpi.c b/drivers/gpu/drm/vc4/vc4_dpi.c
index 2e0fe46aeb2e..519cefef800d 100644
--- a/drivers/gpu/drm/vc4/vc4_dpi.c
+++ b/drivers/gpu/drm/vc4/vc4_dpi.c
@@ -224,20 +224,19 @@ static void vc4_dpi_encoder_enable(struct drm_encoder *encoder)
 		DRM_ERROR("Failed to set clock rate: %d\n", ret);
 }
 
-static bool vc4_dpi_encoder_mode_fixup(struct drm_encoder *encoder,
-				       const struct drm_display_mode *mode,
-				       struct drm_display_mode *adjusted_mode)
+static enum drm_mode_status vc4_dpi_encoder_mode_valid(struct drm_encoder *encoder,
+						       const struct drm_display_mode *mode)
 {
-	if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
-		return false;
+	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+		return MODE_NO_INTERLACE;
 
-	return true;
+	return MODE_OK;
 }
 
 static const struct drm_encoder_helper_funcs vc4_dpi_encoder_helper_funcs = {
 	.disable = vc4_dpi_encoder_disable,
 	.enable = vc4_dpi_encoder_enable,
-	.mode_fixup = vc4_dpi_encoder_mode_fixup,
+	.mode_valid = vc4_dpi_encoder_mode_valid,
 };
 
 static const struct of_device_id vc4_dpi_dt_match[] = {
diff --git a/drivers/gpu/drm/vc4/vc4_drv.c b/drivers/gpu/drm/vc4/vc4_drv.c
index c6b487c3d2b7..1c96edcb302b 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.c
+++ b/drivers/gpu/drm/vc4/vc4_drv.c
@@ -99,6 +99,7 @@ static int vc4_get_param_ioctl(struct drm_device *dev, void *data,
 	case DRM_VC4_PARAM_SUPPORTS_BRANCHES:
 	case DRM_VC4_PARAM_SUPPORTS_ETC1:
 	case DRM_VC4_PARAM_SUPPORTS_THREADED_FS:
+	case DRM_VC4_PARAM_SUPPORTS_FIXED_RCL_ORDER:
 		args->value = true;
 		break;
 	default:
@@ -140,6 +141,7 @@ static const struct drm_ioctl_desc vc4_drm_ioctls[] = {
 	DRM_IOCTL_DEF_DRV(VC4_GET_PARAM, vc4_get_param_ioctl, DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(VC4_SET_TILING, vc4_set_tiling_ioctl, DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(VC4_GET_TILING, vc4_get_tiling_ioctl, DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(VC4_LABEL_BO, vc4_label_bo_ioctl, DRM_RENDER_ALLOW),
 };
 
 static struct drm_driver vc4_drm_driver = {
@@ -178,8 +180,6 @@ static struct drm_driver vc4_drm_driver = {
 	.gem_prime_mmap = vc4_prime_mmap,
 
 	.dumb_create = vc4_dumb_create,
-	.dumb_map_offset = drm_gem_cma_dumb_map_offset,
-	.dumb_destroy = drm_gem_dumb_destroy,
 
 	.ioctls = vc4_drm_ioctls,
 	.num_ioctls = ARRAY_SIZE(vc4_drm_ioctls),
@@ -257,7 +257,9 @@ static int vc4_drm_bind(struct device *dev)
 	vc4->dev = drm;
 	drm->dev_private = vc4;
 
-	vc4_bo_cache_init(drm);
+	ret = vc4_bo_cache_init(drm);
+	if (ret)
+		goto dev_unref;
 
 	drm_mode_config_init(drm);
 
@@ -281,8 +283,9 @@ unbind_all:
 	component_unbind_all(dev, drm);
 gem_destroy:
 	vc4_gem_destroy(drm);
-	drm_dev_unref(drm);
 	vc4_bo_cache_destroy(drm);
+dev_unref:
+	drm_dev_unref(drm);
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/vc4/vc4_drv.h b/drivers/gpu/drm/vc4/vc4_drv.h
index df22698d62ee..87f2d8e5c134 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.h
+++ b/drivers/gpu/drm/vc4/vc4_drv.h
@@ -11,6 +11,24 @@
 #include <drm/drm_encoder.h>
 #include <drm/drm_gem_cma_helper.h>
 
+/* Don't forget to update vc4_bo.c: bo_type_names[] when adding to
+ * this.
+ */
+enum vc4_kernel_bo_type {
+	/* Any kernel allocation (gem_create_object hook) before it
+	 * gets another type set.
+	 */
+	VC4_BO_TYPE_KERNEL,
+	VC4_BO_TYPE_V3D,
+	VC4_BO_TYPE_V3D_SHADER,
+	VC4_BO_TYPE_DUMB,
+	VC4_BO_TYPE_BIN,
+	VC4_BO_TYPE_RCL,
+	VC4_BO_TYPE_BCL,
+	VC4_BO_TYPE_KERNEL_CACHE,
+	VC4_BO_TYPE_COUNT
+};
+
 struct vc4_dev {
 	struct drm_device *dev;
 
@@ -46,14 +64,14 @@ struct vc4_dev {
 		struct timer_list time_timer;
 	} bo_cache;
 
-	struct vc4_bo_stats {
+	u32 num_labels;
+	struct vc4_label {
+		const char *name;
 		u32 num_allocated;
 		u32 size_allocated;
-		u32 num_cached;
-		u32 size_cached;
-	} bo_stats;
+	} *bo_labels;
 
-	/* Protects bo_cache and the BO stats. */
+	/* Protects bo_cache and bo_labels. */
 	struct mutex bo_lock;
 
 	uint64_t dma_fence_context;
@@ -169,6 +187,11 @@ struct vc4_bo {
 	/* normally (resv == &_resv) except for imported bo's */
 	struct reservation_object *resv;
 	struct reservation_object _resv;
+
+	/* One of enum vc4_kernel_bo_type, or VC4_BO_TYPE_COUNT + i
+	 * for user-allocated labels.
+	 */
+	int label;
 };
 
 static inline struct vc4_bo *
@@ -460,7 +483,7 @@ struct vc4_validated_shader_info {
 struct drm_gem_object *vc4_create_object(struct drm_device *dev, size_t size);
 void vc4_free_object(struct drm_gem_object *gem_obj);
 struct vc4_bo *vc4_bo_create(struct drm_device *dev, size_t size,
-			     bool from_cache);
+			     bool from_cache, enum vc4_kernel_bo_type type);
 int vc4_dumb_create(struct drm_file *file_priv,
 		    struct drm_device *dev,
 		    struct drm_mode_create_dumb *args);
@@ -478,6 +501,8 @@ int vc4_get_tiling_ioctl(struct drm_device *dev, void *data,
 			 struct drm_file *file_priv);
 int vc4_get_hang_state_ioctl(struct drm_device *dev, void *data,
 			     struct drm_file *file_priv);
+int vc4_label_bo_ioctl(struct drm_device *dev, void *data,
+		       struct drm_file *file_priv);
 int vc4_mmap(struct file *filp, struct vm_area_struct *vma);
 struct reservation_object *vc4_prime_res_obj(struct drm_gem_object *obj);
 int vc4_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma);
@@ -485,13 +510,12 @@ struct drm_gem_object *vc4_prime_import_sg_table(struct drm_device *dev,
 						 struct dma_buf_attachment *attach,
 						 struct sg_table *sgt);
 void *vc4_prime_vmap(struct drm_gem_object *obj);
-void vc4_bo_cache_init(struct drm_device *dev);
+int vc4_bo_cache_init(struct drm_device *dev);
 void vc4_bo_cache_destroy(struct drm_device *dev);
 int vc4_bo_stats_debugfs(struct seq_file *m, void *arg);
 
 /* vc4_crtc.c */
 extern struct platform_driver vc4_crtc_driver;
-bool vc4_event_pending(struct drm_crtc *crtc);
 int vc4_crtc_debugfs_regs(struct seq_file *m, void *arg);
 bool vc4_crtc_get_scanoutpos(struct drm_device *dev, unsigned int crtc_id,
 			     bool in_vblank_irq, int *vpos, int *hpos,
diff --git a/drivers/gpu/drm/vc4/vc4_dsi.c b/drivers/gpu/drm/vc4/vc4_dsi.c
index 5e8b81eaa168..d1e0dc908048 100644
--- a/drivers/gpu/drm/vc4/vc4_dsi.c
+++ b/drivers/gpu/drm/vc4/vc4_dsi.c
@@ -736,18 +736,18 @@ static void vc4_dsi_latch_ulps(struct vc4_dsi *dsi, bool latch)
 /* Enters or exits Ultra Low Power State. */
 static void vc4_dsi_ulps(struct vc4_dsi *dsi, bool ulps)
 {
-	bool continuous = dsi->mode_flags & MIPI_DSI_CLOCK_NON_CONTINUOUS;
-	u32 phyc_ulps = ((continuous ? DSI_PORT_BIT(PHYC_CLANE_ULPS) : 0) |
+	bool non_continuous = dsi->mode_flags & MIPI_DSI_CLOCK_NON_CONTINUOUS;
+	u32 phyc_ulps = ((non_continuous ? DSI_PORT_BIT(PHYC_CLANE_ULPS) : 0) |
 			 DSI_PHYC_DLANE0_ULPS |
 			 (dsi->lanes > 1 ? DSI_PHYC_DLANE1_ULPS : 0) |
 			 (dsi->lanes > 2 ? DSI_PHYC_DLANE2_ULPS : 0) |
 			 (dsi->lanes > 3 ? DSI_PHYC_DLANE3_ULPS : 0));
-	u32 stat_ulps = ((continuous ? DSI1_STAT_PHY_CLOCK_ULPS : 0) |
+	u32 stat_ulps = ((non_continuous ? DSI1_STAT_PHY_CLOCK_ULPS : 0) |
 			 DSI1_STAT_PHY_D0_ULPS |
 			 (dsi->lanes > 1 ? DSI1_STAT_PHY_D1_ULPS : 0) |
 			 (dsi->lanes > 2 ? DSI1_STAT_PHY_D2_ULPS : 0) |
 			 (dsi->lanes > 3 ? DSI1_STAT_PHY_D3_ULPS : 0));
-	u32 stat_stop = ((continuous ? DSI1_STAT_PHY_CLOCK_STOP : 0) |
+	u32 stat_stop = ((non_continuous ? DSI1_STAT_PHY_CLOCK_STOP : 0) |
 			 DSI1_STAT_PHY_D0_STOP |
 			 (dsi->lanes > 1 ? DSI1_STAT_PHY_D1_STOP : 0) |
 			 (dsi->lanes > 2 ? DSI1_STAT_PHY_D2_STOP : 0) |
@@ -1035,7 +1035,17 @@ static void vc4_dsi_encoder_enable(struct drm_encoder *encoder)
 				     DSI_HS_DLT4_TRAIL) |
 		       VC4_SET_FIELD(0, DSI_HS_DLT4_ANLAT));
 
-	DSI_PORT_WRITE(HS_DLT5, VC4_SET_FIELD(dsi_hs_timing(ui_ns, 1000, 5000),
+	/* T_INIT is how long STOP is driven after power-up to
+	 * indicate to the slave (also coming out of power-up) that
+	 * master init is complete, and should be greater than the
+	 * maximum of two value: T_INIT,MASTER and T_INIT,SLAVE.  The
+	 * D-PHY spec gives a minimum 100us for T_INIT,MASTER and
+	 * T_INIT,SLAVE, while allowing protocols on top of it to give
+	 * greater minimums.  The vc4 firmware uses an extremely
+	 * conservative 5ms, and we maintain that here.
+	 */
+	DSI_PORT_WRITE(HS_DLT5, VC4_SET_FIELD(dsi_hs_timing(ui_ns,
+							    5 * 1000 * 1000, 0),
 					      DSI_HS_DLT5_INIT));
 
 	DSI_PORT_WRITE(HS_DLT6,
@@ -1626,14 +1636,10 @@ static void vc4_dsi_unbind(struct device *dev, struct device *master,
 
 	pm_runtime_disable(dev);
 
-	drm_bridge_remove(dsi->bridge);
 	vc4_dsi_encoder_destroy(dsi->encoder);
 
 	mipi_dsi_host_unregister(&dsi->dsi_host);
 
-	clk_disable_unprepare(dsi->pll_phy_clock);
-	clk_disable_unprepare(dsi->escape_clock);
-
 	if (dsi->port == 1)
 		vc4->dsi1 = NULL;
 }
diff --git a/drivers/gpu/drm/vc4/vc4_gem.c b/drivers/gpu/drm/vc4/vc4_gem.c
index d5b821ad06af..d0c6bfb68c4e 100644
--- a/drivers/gpu/drm/vc4/vc4_gem.c
+++ b/drivers/gpu/drm/vc4/vc4_gem.c
@@ -55,7 +55,7 @@ vc4_free_hang_state(struct drm_device *dev, struct vc4_hang_state *state)
 	unsigned int i;
 
 	for (i = 0; i < state->user_state.bo_count; i++)
-		drm_gem_object_unreference_unlocked(state->bo[i]);
+		drm_gem_object_put_unlocked(state->bo[i]);
 
 	kfree(state);
 }
@@ -119,7 +119,7 @@ vc4_get_hang_state_ioctl(struct drm_device *dev, void *data,
 		bo_state[i].size = vc4_bo->base.base.size;
 	}
 
-	if (copy_to_user((void __user *)(uintptr_t)get_state->bo,
+	if (copy_to_user(u64_to_user_ptr(get_state->bo),
 			 bo_state,
 			 state->bo_count * sizeof(*bo_state)))
 		ret = -EFAULT;
@@ -188,12 +188,12 @@ vc4_save_hang_state(struct drm_device *dev)
 			continue;
 
 		for (j = 0; j < exec[i]->bo_count; j++) {
-			drm_gem_object_reference(&exec[i]->bo[j]->base);
+			drm_gem_object_get(&exec[i]->bo[j]->base);
 			kernel_state->bo[j + prev_idx] = &exec[i]->bo[j]->base;
 		}
 
 		list_for_each_entry(bo, &exec[i]->unref_list, unref_head) {
-			drm_gem_object_reference(&bo->base.base);
+			drm_gem_object_get(&bo->base.base);
 			kernel_state->bo[j + prev_idx] = &bo->base.base;
 			j++;
 		}
@@ -659,7 +659,7 @@ vc4_cl_lookup_bos(struct drm_device *dev,
 		/* See comment on bo_index for why we have to check
 		 * this.
 		 */
-		DRM_ERROR("Rendering requires BOs to validate\n");
+		DRM_DEBUG("Rendering requires BOs to validate\n");
 		return -EINVAL;
 	}
 
@@ -678,8 +678,7 @@ vc4_cl_lookup_bos(struct drm_device *dev,
 		goto fail;
 	}
 
-	if (copy_from_user(handles,
-			   (void __user *)(uintptr_t)args->bo_handles,
+	if (copy_from_user(handles, u64_to_user_ptr(args->bo_handles),
 			   exec->bo_count * sizeof(uint32_t))) {
 		ret = -EFAULT;
 		DRM_ERROR("Failed to copy in GEM handles\n");
@@ -691,13 +690,13 @@ vc4_cl_lookup_bos(struct drm_device *dev,
 		struct drm_gem_object *bo = idr_find(&file_priv->object_idr,
 						     handles[i]);
 		if (!bo) {
-			DRM_ERROR("Failed to look up GEM BO %d: %d\n",
+			DRM_DEBUG("Failed to look up GEM BO %d: %d\n",
 				  i, handles[i]);
 			ret = -EINVAL;
 			spin_unlock(&file_priv->table_lock);
 			goto fail;
 		}
-		drm_gem_object_reference(bo);
+		drm_gem_object_get(bo);
 		exec->bo[i] = (struct drm_gem_cma_object *)bo;
 	}
 	spin_unlock(&file_priv->table_lock);
@@ -729,7 +728,7 @@ vc4_get_bcl(struct drm_device *dev, struct vc4_exec_info *exec)
 	    args->shader_rec_count >= (UINT_MAX /
 					  sizeof(struct vc4_shader_state)) ||
 	    temp_size < exec_size) {
-		DRM_ERROR("overflow in exec arguments\n");
+		DRM_DEBUG("overflow in exec arguments\n");
 		ret = -EINVAL;
 		goto fail;
 	}
@@ -755,27 +754,27 @@ vc4_get_bcl(struct drm_device *dev, struct vc4_exec_info *exec)
 	exec->shader_state_size = args->shader_rec_count;
 
 	if (copy_from_user(bin,
-			   (void __user *)(uintptr_t)args->bin_cl,
+			   u64_to_user_ptr(args->bin_cl),
 			   args->bin_cl_size)) {
 		ret = -EFAULT;
 		goto fail;
 	}
 
 	if (copy_from_user(exec->shader_rec_u,
-			   (void __user *)(uintptr_t)args->shader_rec,
+			   u64_to_user_ptr(args->shader_rec),
 			   args->shader_rec_size)) {
 		ret = -EFAULT;
 		goto fail;
 	}
 
 	if (copy_from_user(exec->uniforms_u,
-			   (void __user *)(uintptr_t)args->uniforms,
+			   u64_to_user_ptr(args->uniforms),
 			   args->uniforms_size)) {
 		ret = -EFAULT;
 		goto fail;
 	}
 
-	bo = vc4_bo_create(dev, exec_size, true);
+	bo = vc4_bo_create(dev, exec_size, true, VC4_BO_TYPE_BCL);
 	if (IS_ERR(bo)) {
 		DRM_ERROR("Couldn't allocate BO for binning\n");
 		ret = PTR_ERR(bo);
@@ -835,7 +834,7 @@ vc4_complete_exec(struct drm_device *dev, struct vc4_exec_info *exec)
 
 	if (exec->bo) {
 		for (i = 0; i < exec->bo_count; i++)
-			drm_gem_object_unreference_unlocked(&exec->bo[i]->base);
+			drm_gem_object_put_unlocked(&exec->bo[i]->base);
 		kvfree(exec->bo);
 	}
 
@@ -843,7 +842,7 @@ vc4_complete_exec(struct drm_device *dev, struct vc4_exec_info *exec)
 		struct vc4_bo *bo = list_first_entry(&exec->unref_list,
 						     struct vc4_bo, unref_head);
 		list_del(&bo->unref_head);
-		drm_gem_object_unreference_unlocked(&bo->base.base);
+		drm_gem_object_put_unlocked(&bo->base.base);
 	}
 
 	/* Free up the allocation of any bin slots we used. */
@@ -974,7 +973,7 @@ vc4_wait_bo_ioctl(struct drm_device *dev, void *data,
 
 	gem_obj = drm_gem_object_lookup(file_priv, args->handle);
 	if (!gem_obj) {
-		DRM_ERROR("Failed to look up GEM BO %d\n", args->handle);
+		DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle);
 		return -EINVAL;
 	}
 	bo = to_vc4_bo(gem_obj);
@@ -982,7 +981,7 @@ vc4_wait_bo_ioctl(struct drm_device *dev, void *data,
 	ret = vc4_wait_for_seqno_ioctl_helper(dev, bo->seqno,
 					      &args->timeout_ns);
 
-	drm_gem_object_unreference_unlocked(gem_obj);
+	drm_gem_object_put_unlocked(gem_obj);
 	return ret;
 }
 
@@ -1008,8 +1007,11 @@ vc4_submit_cl_ioctl(struct drm_device *dev, void *data,
 	struct ww_acquire_ctx acquire_ctx;
 	int ret = 0;
 
-	if ((args->flags & ~VC4_SUBMIT_CL_USE_CLEAR_COLOR) != 0) {
-		DRM_ERROR("Unknown flags: 0x%02x\n", args->flags);
+	if ((args->flags & ~(VC4_SUBMIT_CL_USE_CLEAR_COLOR |
+			     VC4_SUBMIT_CL_FIXED_RCL_ORDER |
+			     VC4_SUBMIT_CL_RCL_ORDER_INCREASING_X |
+			     VC4_SUBMIT_CL_RCL_ORDER_INCREASING_Y)) != 0) {
+		DRM_DEBUG("Unknown flags: 0x%02x\n", args->flags);
 		return -EINVAL;
 	}
 
@@ -1118,6 +1120,4 @@ vc4_gem_destroy(struct drm_device *dev)
 
 	if (vc4->hang_state)
 		vc4_free_hang_state(dev, vc4->hang_state);
-
-	vc4_bo_cache_destroy(dev);
 }
diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.c b/drivers/gpu/drm/vc4/vc4_hdmi.c
index ed63d4e85762..937da8dd65b8 100644
--- a/drivers/gpu/drm/vc4/vc4_hdmi.c
+++ b/drivers/gpu/drm/vc4/vc4_hdmi.c
@@ -57,9 +57,14 @@
 #include <sound/pcm_drm_eld.h>
 #include <sound/pcm_params.h>
 #include <sound/soc.h>
+#include "media/cec.h"
 #include "vc4_drv.h"
 #include "vc4_regs.h"
 
+#define HSM_CLOCK_FREQ 163682864
+#define CEC_CLOCK_FREQ 40000
+#define CEC_CLOCK_DIV  (HSM_CLOCK_FREQ / CEC_CLOCK_FREQ)
+
 /* HDMI audio information */
 struct vc4_hdmi_audio {
 	struct snd_soc_card card;
@@ -85,6 +90,11 @@ struct vc4_hdmi {
 	int hpd_gpio;
 	bool hpd_active_low;
 
+	struct cec_adapter *cec_adap;
+	struct cec_msg cec_rx_msg;
+	bool cec_tx_ok;
+	bool cec_irq_was_rx;
+
 	struct clk *pixel_clock;
 	struct clk *hsm_clock;
 };
@@ -149,6 +159,23 @@ static const struct {
 	HDMI_REG(VC4_HDMI_VERTB1),
 	HDMI_REG(VC4_HDMI_TX_PHY_RESET_CTL),
 	HDMI_REG(VC4_HDMI_TX_PHY_CTL0),
+
+	HDMI_REG(VC4_HDMI_CEC_CNTRL_1),
+	HDMI_REG(VC4_HDMI_CEC_CNTRL_2),
+	HDMI_REG(VC4_HDMI_CEC_CNTRL_3),
+	HDMI_REG(VC4_HDMI_CEC_CNTRL_4),
+	HDMI_REG(VC4_HDMI_CEC_CNTRL_5),
+	HDMI_REG(VC4_HDMI_CPU_STATUS),
+	HDMI_REG(VC4_HDMI_CPU_MASK_STATUS),
+
+	HDMI_REG(VC4_HDMI_CEC_RX_DATA_1),
+	HDMI_REG(VC4_HDMI_CEC_RX_DATA_2),
+	HDMI_REG(VC4_HDMI_CEC_RX_DATA_3),
+	HDMI_REG(VC4_HDMI_CEC_RX_DATA_4),
+	HDMI_REG(VC4_HDMI_CEC_TX_DATA_1),
+	HDMI_REG(VC4_HDMI_CEC_TX_DATA_2),
+	HDMI_REG(VC4_HDMI_CEC_TX_DATA_3),
+	HDMI_REG(VC4_HDMI_CEC_TX_DATA_4),
 };
 
 static const struct {
@@ -216,8 +243,8 @@ vc4_hdmi_connector_detect(struct drm_connector *connector, bool force)
 		if (gpio_get_value_cansleep(vc4->hdmi->hpd_gpio) ^
 		    vc4->hdmi->hpd_active_low)
 			return connector_status_connected;
-		else
-			return connector_status_disconnected;
+		cec_phys_addr_invalidate(vc4->hdmi->cec_adap);
+		return connector_status_disconnected;
 	}
 
 	if (drm_probe_ddc(vc4->hdmi->ddc))
@@ -225,8 +252,8 @@ vc4_hdmi_connector_detect(struct drm_connector *connector, bool force)
 
 	if (HDMI_READ(VC4_HDMI_HOTPLUG) & VC4_HDMI_HOTPLUG_CONNECTED)
 		return connector_status_connected;
-	else
-		return connector_status_disconnected;
+	cec_phys_addr_invalidate(vc4->hdmi->cec_adap);
+	return connector_status_disconnected;
 }
 
 static void vc4_hdmi_connector_destroy(struct drm_connector *connector)
@@ -247,6 +274,7 @@ static int vc4_hdmi_connector_get_modes(struct drm_connector *connector)
 	struct edid *edid;
 
 	edid = drm_get_edid(connector, vc4->hdmi->ddc);
+	cec_s_phys_addr_from_edid(vc4->hdmi->cec_adap, edid);
 	if (!edid)
 		return -ENODEV;
 
@@ -260,12 +288,12 @@ static int vc4_hdmi_connector_get_modes(struct drm_connector *connector)
 	drm_mode_connector_update_edid_property(connector, edid);
 	ret = drm_add_edid_modes(connector, edid);
 	drm_edid_to_eld(connector, edid);
+	kfree(edid);
 
 	return ret;
 }
 
 static const struct drm_connector_funcs vc4_hdmi_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.detect = vc4_hdmi_connector_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = vc4_hdmi_connector_destroy,
@@ -395,7 +423,7 @@ static void vc4_hdmi_set_avi_infoframe(struct drm_encoder *encoder)
 	union hdmi_infoframe frame;
 	int ret;
 
-	ret = drm_hdmi_avi_infoframe_from_display_mode(&frame.avi, mode);
+	ret = drm_hdmi_avi_infoframe_from_display_mode(&frame.avi, mode, false);
 	if (ret < 0) {
 		DRM_ERROR("couldn't fill AVI infoframe\n");
 		return;
@@ -463,11 +491,6 @@ static void vc4_hdmi_encoder_disable(struct drm_encoder *encoder)
 	HD_WRITE(VC4_HD_VID_CTL,
 		 HD_READ(VC4_HD_VID_CTL) & ~VC4_HD_VID_CTL_ENABLE);
 
-	HD_WRITE(VC4_HD_M_CTL, VC4_HD_M_SW_RST);
-	udelay(1);
-	HD_WRITE(VC4_HD_M_CTL, 0);
-
-	clk_disable_unprepare(hdmi->hsm_clock);
 	clk_disable_unprepare(hdmi->pixel_clock);
 
 	ret = pm_runtime_put(&hdmi->pdev->dev);
@@ -509,16 +532,6 @@ static void vc4_hdmi_encoder_enable(struct drm_encoder *encoder)
 		return;
 	}
 
-	/* This is the rate that is set by the firmware.  The number
-	 * needs to be a bit higher than the pixel clock rate
-	 * (generally 148.5Mhz).
-	 */
-	ret = clk_set_rate(hdmi->hsm_clock, 163682864);
-	if (ret) {
-		DRM_ERROR("Failed to set HSM clock rate: %d\n", ret);
-		return;
-	}
-
 	ret = clk_set_rate(hdmi->pixel_clock,
 			   mode->clock * 1000 *
 			   ((mode->flags & DRM_MODE_FLAG_DBLCLK) ? 2 : 1));
@@ -533,20 +546,6 @@ static void vc4_hdmi_encoder_enable(struct drm_encoder *encoder)
 		return;
 	}
 
-	ret = clk_prepare_enable(hdmi->hsm_clock);
-	if (ret) {
-		DRM_ERROR("Failed to turn on HDMI state machine clock: %d\n",
-			  ret);
-		clk_disable_unprepare(hdmi->pixel_clock);
-		return;
-	}
-
-	HD_WRITE(VC4_HD_M_CTL, VC4_HD_M_SW_RST);
-	udelay(1);
-	HD_WRITE(VC4_HD_M_CTL, 0);
-
-	HD_WRITE(VC4_HD_M_CTL, VC4_HD_M_ENABLE);
-
 	HDMI_WRITE(VC4_HDMI_SW_RESET_CONTROL,
 		   VC4_HDMI_SW_RESET_HDMI |
 		   VC4_HDMI_SW_RESET_FORMAT_DETECT);
@@ -1150,6 +1149,159 @@ static void vc4_hdmi_audio_cleanup(struct vc4_hdmi *hdmi)
 		snd_soc_unregister_codec(dev);
 }
 
+#ifdef CONFIG_DRM_VC4_HDMI_CEC
+static irqreturn_t vc4_cec_irq_handler_thread(int irq, void *priv)
+{
+	struct vc4_dev *vc4 = priv;
+	struct vc4_hdmi *hdmi = vc4->hdmi;
+
+	if (hdmi->cec_irq_was_rx) {
+		if (hdmi->cec_rx_msg.len)
+			cec_received_msg(hdmi->cec_adap, &hdmi->cec_rx_msg);
+	} else if (hdmi->cec_tx_ok) {
+		cec_transmit_done(hdmi->cec_adap, CEC_TX_STATUS_OK,
+				  0, 0, 0, 0);
+	} else {
+		/*
+		 * This CEC implementation makes 1 retry, so if we
+		 * get a NACK, then that means it made 2 attempts.
+		 */
+		cec_transmit_done(hdmi->cec_adap, CEC_TX_STATUS_NACK,
+				  0, 2, 0, 0);
+	}
+	return IRQ_HANDLED;
+}
+
+static void vc4_cec_read_msg(struct vc4_dev *vc4, u32 cntrl1)
+{
+	struct cec_msg *msg = &vc4->hdmi->cec_rx_msg;
+	unsigned int i;
+
+	msg->len = 1 + ((cntrl1 & VC4_HDMI_CEC_REC_WRD_CNT_MASK) >>
+					VC4_HDMI_CEC_REC_WRD_CNT_SHIFT);
+	for (i = 0; i < msg->len; i += 4) {
+		u32 val = HDMI_READ(VC4_HDMI_CEC_RX_DATA_1 + i);
+
+		msg->msg[i] = val & 0xff;
+		msg->msg[i + 1] = (val >> 8) & 0xff;
+		msg->msg[i + 2] = (val >> 16) & 0xff;
+		msg->msg[i + 3] = (val >> 24) & 0xff;
+	}
+}
+
+static irqreturn_t vc4_cec_irq_handler(int irq, void *priv)
+{
+	struct vc4_dev *vc4 = priv;
+	struct vc4_hdmi *hdmi = vc4->hdmi;
+	u32 stat = HDMI_READ(VC4_HDMI_CPU_STATUS);
+	u32 cntrl1, cntrl5;
+
+	if (!(stat & VC4_HDMI_CPU_CEC))
+		return IRQ_NONE;
+	hdmi->cec_rx_msg.len = 0;
+	cntrl1 = HDMI_READ(VC4_HDMI_CEC_CNTRL_1);
+	cntrl5 = HDMI_READ(VC4_HDMI_CEC_CNTRL_5);
+	hdmi->cec_irq_was_rx = cntrl5 & VC4_HDMI_CEC_RX_CEC_INT;
+	if (hdmi->cec_irq_was_rx) {
+		vc4_cec_read_msg(vc4, cntrl1);
+		cntrl1 |= VC4_HDMI_CEC_CLEAR_RECEIVE_OFF;
+		HDMI_WRITE(VC4_HDMI_CEC_CNTRL_1, cntrl1);
+		cntrl1 &= ~VC4_HDMI_CEC_CLEAR_RECEIVE_OFF;
+	} else {
+		hdmi->cec_tx_ok = cntrl1 & VC4_HDMI_CEC_TX_STATUS_GOOD;
+		cntrl1 &= ~VC4_HDMI_CEC_START_XMIT_BEGIN;
+	}
+	HDMI_WRITE(VC4_HDMI_CEC_CNTRL_1, cntrl1);
+	HDMI_WRITE(VC4_HDMI_CPU_CLEAR, VC4_HDMI_CPU_CEC);
+
+	return IRQ_WAKE_THREAD;
+}
+
+static int vc4_hdmi_cec_adap_enable(struct cec_adapter *adap, bool enable)
+{
+	struct vc4_dev *vc4 = cec_get_drvdata(adap);
+	/* clock period in microseconds */
+	const u32 usecs = 1000000 / CEC_CLOCK_FREQ;
+	u32 val = HDMI_READ(VC4_HDMI_CEC_CNTRL_5);
+
+	val &= ~(VC4_HDMI_CEC_TX_SW_RESET | VC4_HDMI_CEC_RX_SW_RESET |
+		 VC4_HDMI_CEC_CNT_TO_4700_US_MASK |
+		 VC4_HDMI_CEC_CNT_TO_4500_US_MASK);
+	val |= ((4700 / usecs) << VC4_HDMI_CEC_CNT_TO_4700_US_SHIFT) |
+	       ((4500 / usecs) << VC4_HDMI_CEC_CNT_TO_4500_US_SHIFT);
+
+	if (enable) {
+		HDMI_WRITE(VC4_HDMI_CEC_CNTRL_5, val |
+			   VC4_HDMI_CEC_TX_SW_RESET | VC4_HDMI_CEC_RX_SW_RESET);
+		HDMI_WRITE(VC4_HDMI_CEC_CNTRL_5, val);
+		HDMI_WRITE(VC4_HDMI_CEC_CNTRL_2,
+			 ((1500 / usecs) << VC4_HDMI_CEC_CNT_TO_1500_US_SHIFT) |
+			 ((1300 / usecs) << VC4_HDMI_CEC_CNT_TO_1300_US_SHIFT) |
+			 ((800 / usecs) << VC4_HDMI_CEC_CNT_TO_800_US_SHIFT) |
+			 ((600 / usecs) << VC4_HDMI_CEC_CNT_TO_600_US_SHIFT) |
+			 ((400 / usecs) << VC4_HDMI_CEC_CNT_TO_400_US_SHIFT));
+		HDMI_WRITE(VC4_HDMI_CEC_CNTRL_3,
+			 ((2750 / usecs) << VC4_HDMI_CEC_CNT_TO_2750_US_SHIFT) |
+			 ((2400 / usecs) << VC4_HDMI_CEC_CNT_TO_2400_US_SHIFT) |
+			 ((2050 / usecs) << VC4_HDMI_CEC_CNT_TO_2050_US_SHIFT) |
+			 ((1700 / usecs) << VC4_HDMI_CEC_CNT_TO_1700_US_SHIFT));
+		HDMI_WRITE(VC4_HDMI_CEC_CNTRL_4,
+			 ((4300 / usecs) << VC4_HDMI_CEC_CNT_TO_4300_US_SHIFT) |
+			 ((3900 / usecs) << VC4_HDMI_CEC_CNT_TO_3900_US_SHIFT) |
+			 ((3600 / usecs) << VC4_HDMI_CEC_CNT_TO_3600_US_SHIFT) |
+			 ((3500 / usecs) << VC4_HDMI_CEC_CNT_TO_3500_US_SHIFT));
+
+		HDMI_WRITE(VC4_HDMI_CPU_MASK_CLEAR, VC4_HDMI_CPU_CEC);
+	} else {
+		HDMI_WRITE(VC4_HDMI_CPU_MASK_SET, VC4_HDMI_CPU_CEC);
+		HDMI_WRITE(VC4_HDMI_CEC_CNTRL_5, val |
+			   VC4_HDMI_CEC_TX_SW_RESET | VC4_HDMI_CEC_RX_SW_RESET);
+	}
+	return 0;
+}
+
+static int vc4_hdmi_cec_adap_log_addr(struct cec_adapter *adap, u8 log_addr)
+{
+	struct vc4_dev *vc4 = cec_get_drvdata(adap);
+
+	HDMI_WRITE(VC4_HDMI_CEC_CNTRL_1,
+		   (HDMI_READ(VC4_HDMI_CEC_CNTRL_1) & ~VC4_HDMI_CEC_ADDR_MASK) |
+		   (log_addr & 0xf) << VC4_HDMI_CEC_ADDR_SHIFT);
+	return 0;
+}
+
+static int vc4_hdmi_cec_adap_transmit(struct cec_adapter *adap, u8 attempts,
+				      u32 signal_free_time, struct cec_msg *msg)
+{
+	struct vc4_dev *vc4 = cec_get_drvdata(adap);
+	u32 val;
+	unsigned int i;
+
+	for (i = 0; i < msg->len; i += 4)
+		HDMI_WRITE(VC4_HDMI_CEC_TX_DATA_1 + i,
+			   (msg->msg[i]) |
+			   (msg->msg[i + 1] << 8) |
+			   (msg->msg[i + 2] << 16) |
+			   (msg->msg[i + 3] << 24));
+
+	val = HDMI_READ(VC4_HDMI_CEC_CNTRL_1);
+	val &= ~VC4_HDMI_CEC_START_XMIT_BEGIN;
+	HDMI_WRITE(VC4_HDMI_CEC_CNTRL_1, val);
+	val &= ~VC4_HDMI_CEC_MESSAGE_LENGTH_MASK;
+	val |= (msg->len - 1) << VC4_HDMI_CEC_MESSAGE_LENGTH_SHIFT;
+	val |= VC4_HDMI_CEC_START_XMIT_BEGIN;
+
+	HDMI_WRITE(VC4_HDMI_CEC_CNTRL_1, val);
+	return 0;
+}
+
+static const struct cec_adap_ops vc4_hdmi_cec_adap_ops = {
+	.adap_enable = vc4_hdmi_cec_adap_enable,
+	.adap_log_addr = vc4_hdmi_cec_adap_log_addr,
+	.adap_transmit = vc4_hdmi_cec_adap_transmit,
+};
+#endif
+
 static int vc4_hdmi_bind(struct device *dev, struct device *master, void *data)
 {
 	struct platform_device *pdev = to_platform_device(dev);
@@ -1205,6 +1357,23 @@ static int vc4_hdmi_bind(struct device *dev, struct device *master, void *data)
 		return -EPROBE_DEFER;
 	}
 
+	/* This is the rate that is set by the firmware.  The number
+	 * needs to be a bit higher than the pixel clock rate
+	 * (generally 148.5Mhz).
+	 */
+	ret = clk_set_rate(hdmi->hsm_clock, HSM_CLOCK_FREQ);
+	if (ret) {
+		DRM_ERROR("Failed to set HSM clock rate: %d\n", ret);
+		goto err_put_i2c;
+	}
+
+	ret = clk_prepare_enable(hdmi->hsm_clock);
+	if (ret) {
+		DRM_ERROR("Failed to turn on HDMI state machine clock: %d\n",
+			  ret);
+		goto err_put_i2c;
+	}
+
 	/* Only use the GPIO HPD pin if present in the DT, otherwise
 	 * we'll use the HDMI core's register.
 	 */
@@ -1216,7 +1385,7 @@ static int vc4_hdmi_bind(struct device *dev, struct device *master, void *data)
 							 &hpd_gpio_flags);
 		if (hdmi->hpd_gpio < 0) {
 			ret = hdmi->hpd_gpio;
-			goto err_put_i2c;
+			goto err_unprepare_hsm;
 		}
 
 		hdmi->hpd_active_low = hpd_gpio_flags & OF_GPIO_ACTIVE_LOW;
@@ -1224,6 +1393,14 @@ static int vc4_hdmi_bind(struct device *dev, struct device *master, void *data)
 
 	vc4->hdmi = hdmi;
 
+	/* HDMI core must be enabled. */
+	if (!(HD_READ(VC4_HD_M_CTL) & VC4_HD_M_ENABLE)) {
+		HD_WRITE(VC4_HD_M_CTL, VC4_HD_M_SW_RST);
+		udelay(1);
+		HD_WRITE(VC4_HD_M_CTL, 0);
+
+		HD_WRITE(VC4_HD_M_CTL, VC4_HD_M_ENABLE);
+	}
 	pm_runtime_enable(dev);
 
 	drm_encoder_init(drm, hdmi->encoder, &vc4_hdmi_encoder_funcs,
@@ -1235,6 +1412,37 @@ static int vc4_hdmi_bind(struct device *dev, struct device *master, void *data)
 		ret = PTR_ERR(hdmi->connector);
 		goto err_destroy_encoder;
 	}
+#ifdef CONFIG_DRM_VC4_HDMI_CEC
+	hdmi->cec_adap = cec_allocate_adapter(&vc4_hdmi_cec_adap_ops,
+					      vc4, "vc4",
+					      CEC_CAP_TRANSMIT |
+					      CEC_CAP_LOG_ADDRS |
+					      CEC_CAP_PASSTHROUGH |
+					      CEC_CAP_RC, 1);
+	ret = PTR_ERR_OR_ZERO(hdmi->cec_adap);
+	if (ret < 0)
+		goto err_destroy_conn;
+	HDMI_WRITE(VC4_HDMI_CPU_MASK_SET, 0xffffffff);
+	value = HDMI_READ(VC4_HDMI_CEC_CNTRL_1);
+	value &= ~VC4_HDMI_CEC_DIV_CLK_CNT_MASK;
+	/*
+	 * Set the logical address to Unregistered and set the clock
+	 * divider: the hsm_clock rate and this divider setting will
+	 * give a 40 kHz CEC clock.
+	 */
+	value |= VC4_HDMI_CEC_ADDR_MASK |
+		 (4091 << VC4_HDMI_CEC_DIV_CLK_CNT_SHIFT);
+	HDMI_WRITE(VC4_HDMI_CEC_CNTRL_1, value);
+	ret = devm_request_threaded_irq(dev, platform_get_irq(pdev, 0),
+					vc4_cec_irq_handler,
+					vc4_cec_irq_handler_thread, 0,
+					"vc4 hdmi cec", vc4);
+	if (ret)
+		goto err_delete_cec_adap;
+	ret = cec_register_adapter(hdmi->cec_adap, dev);
+	if (ret < 0)
+		goto err_delete_cec_adap;
+#endif
 
 	ret = vc4_hdmi_audio_init(hdmi);
 	if (ret)
@@ -1242,8 +1450,16 @@ static int vc4_hdmi_bind(struct device *dev, struct device *master, void *data)
 
 	return 0;
 
+#ifdef CONFIG_DRM_VC4_HDMI_CEC
+err_delete_cec_adap:
+	cec_delete_adapter(hdmi->cec_adap);
+err_destroy_conn:
+	vc4_hdmi_connector_destroy(hdmi->connector);
+#endif
 err_destroy_encoder:
 	vc4_hdmi_encoder_destroy(hdmi->encoder);
+err_unprepare_hsm:
+	clk_disable_unprepare(hdmi->hsm_clock);
 	pm_runtime_disable(dev);
 err_put_i2c:
 	put_device(&hdmi->ddc->dev);
@@ -1259,10 +1475,11 @@ static void vc4_hdmi_unbind(struct device *dev, struct device *master,
 	struct vc4_hdmi *hdmi = vc4->hdmi;
 
 	vc4_hdmi_audio_cleanup(hdmi);
-
+	cec_unregister_adapter(hdmi->cec_adap);
 	vc4_hdmi_connector_destroy(hdmi->connector);
 	vc4_hdmi_encoder_destroy(hdmi->encoder);
 
+	clk_disable_unprepare(hdmi->hsm_clock);
 	pm_runtime_disable(dev);
 
 	put_device(&hdmi->ddc->dev);
diff --git a/drivers/gpu/drm/vc4/vc4_kms.c b/drivers/gpu/drm/vc4/vc4_kms.c
index bc6ecdc6f104..50c4959b5bd3 100644
--- a/drivers/gpu/drm/vc4/vc4_kms.c
+++ b/drivers/gpu/drm/vc4/vc4_kms.c
@@ -20,6 +20,7 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_plane_helper.h>
 #include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_gem_framebuffer_helper.h>
 #include "vc4_drv.h"
 
 static void vc4_output_poll_changed(struct drm_device *dev)
@@ -29,16 +30,9 @@ static void vc4_output_poll_changed(struct drm_device *dev)
 	drm_fbdev_cma_hotplug_event(vc4->fbdev);
 }
 
-struct vc4_commit {
-	struct drm_device *dev;
-	struct drm_atomic_state *state;
-	struct vc4_seqno_cb cb;
-};
-
 static void
-vc4_atomic_complete_commit(struct vc4_commit *c)
+vc4_atomic_complete_commit(struct drm_atomic_state *state)
 {
-	struct drm_atomic_state *state = c->state;
 	struct drm_device *dev = state->dev;
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
 
@@ -72,28 +66,14 @@ vc4_atomic_complete_commit(struct vc4_commit *c)
 	drm_atomic_state_put(state);
 
 	up(&vc4->async_modeset);
-
-	kfree(c);
 }
 
-static void
-vc4_atomic_complete_commit_seqno_cb(struct vc4_seqno_cb *cb)
+static void commit_work(struct work_struct *work)
 {
-	struct vc4_commit *c = container_of(cb, struct vc4_commit, cb);
-
-	vc4_atomic_complete_commit(c);
-}
-
-static struct vc4_commit *commit_init(struct drm_atomic_state *state)
-{
-	struct vc4_commit *c = kzalloc(sizeof(*c), GFP_KERNEL);
-
-	if (!c)
-		return NULL;
-	c->dev = state->dev;
-	c->state = state;
-
-	return c;
+	struct drm_atomic_state *state = container_of(work,
+						      struct drm_atomic_state,
+						      commit_work);
+	vc4_atomic_complete_commit(state);
 }
 
 /**
@@ -115,40 +95,29 @@ static int vc4_atomic_commit(struct drm_device *dev,
 {
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
 	int ret;
-	int i;
-	uint64_t wait_seqno = 0;
-	struct vc4_commit *c;
-	struct drm_plane *plane;
-	struct drm_plane_state *new_state;
-
-	c = commit_init(state);
-	if (!c)
-		return -ENOMEM;
 
 	ret = drm_atomic_helper_setup_commit(state, nonblock);
 	if (ret)
 		return ret;
 
+	INIT_WORK(&state->commit_work, commit_work);
+
 	ret = down_interruptible(&vc4->async_modeset);
-	if (ret) {
-		kfree(c);
+	if (ret)
 		return ret;
-	}
 
 	ret = drm_atomic_helper_prepare_planes(dev, state);
 	if (ret) {
-		kfree(c);
 		up(&vc4->async_modeset);
 		return ret;
 	}
 
-	for_each_plane_in_state(state, plane, new_state, i) {
-		if ((plane->state->fb != new_state->fb) && new_state->fb) {
-			struct drm_gem_cma_object *cma_bo =
-				drm_fb_cma_get_gem_obj(new_state->fb, 0);
-			struct vc4_bo *bo = to_vc4_bo(&cma_bo->base);
-
-			wait_seqno = max(bo->seqno, wait_seqno);
+	if (!nonblock) {
+		ret = drm_atomic_helper_wait_for_fences(dev, state, true);
+		if (ret) {
+			drm_atomic_helper_cleanup_planes(dev, state);
+			up(&vc4->async_modeset);
+			return ret;
 		}
 	}
 
@@ -158,7 +127,7 @@ static int vc4_atomic_commit(struct drm_device *dev,
 	 * the software side now.
 	 */
 
-	drm_atomic_helper_swap_state(state, true);
+	BUG_ON(drm_atomic_helper_swap_state(state, false) < 0);
 
 	/*
 	 * Everything below can be run asynchronously without the need to grab
@@ -177,13 +146,10 @@ static int vc4_atomic_commit(struct drm_device *dev,
 	 */
 
 	drm_atomic_state_get(state);
-	if (nonblock) {
-		vc4_queue_seqno_cb(dev, &c->cb, wait_seqno,
-				   vc4_atomic_complete_commit_seqno_cb);
-	} else {
-		vc4_wait_for_seqno(dev, wait_seqno, ~0ull, false);
-		vc4_atomic_complete_commit(c);
-	}
+	if (nonblock)
+		queue_work(system_unbound_wq, &state->commit_work);
+	else
+		vc4_atomic_complete_commit(state);
 
 	return 0;
 }
@@ -204,7 +170,7 @@ static struct drm_framebuffer *vc4_fb_create(struct drm_device *dev,
 		gem_obj = drm_gem_object_lookup(file_priv,
 						mode_cmd->handles[0]);
 		if (!gem_obj) {
-			DRM_ERROR("Failed to look up GEM BO %d\n",
+			DRM_DEBUG("Failed to look up GEM BO %d\n",
 				  mode_cmd->handles[0]);
 			return ERR_PTR(-ENOENT);
 		}
@@ -219,12 +185,12 @@ static struct drm_framebuffer *vc4_fb_create(struct drm_device *dev,
 			mode_cmd_local.modifier[0] = DRM_FORMAT_MOD_NONE;
 		}
 
-		drm_gem_object_unreference_unlocked(gem_obj);
+		drm_gem_object_put_unlocked(gem_obj);
 
 		mode_cmd = &mode_cmd_local;
 	}
 
-	return drm_fb_cma_create(dev, file_priv, mode_cmd);
+	return drm_gem_fb_create(dev, file_priv, mode_cmd);
 }
 
 static const struct drm_mode_config_funcs vc4_mode_funcs = {
@@ -241,6 +207,9 @@ int vc4_kms_load(struct drm_device *dev)
 
 	sema_init(&vc4->async_modeset, 1);
 
+	/* Set support for vblank irq fast disable, before drm_vblank_init() */
+	dev->vblank_disable_immediate = true;
+
 	ret = drm_vblank_init(dev, dev->mode_config.num_crtc);
 	if (ret < 0) {
 		dev_err(dev->dev, "failed to initialize vblank\n");
diff --git a/drivers/gpu/drm/vc4/vc4_plane.c b/drivers/gpu/drm/vc4/vc4_plane.c
index fa6809d8b0fe..2968b3ebb895 100644
--- a/drivers/gpu/drm/vc4/vc4_plane.c
+++ b/drivers/gpu/drm/vc4/vc4_plane.c
@@ -759,9 +759,26 @@ void vc4_plane_async_set_fb(struct drm_plane *plane, struct drm_framebuffer *fb)
 	vc4_state->dlist[vc4_state->ptr0_offset] = addr;
 }
 
+static int vc4_prepare_fb(struct drm_plane *plane,
+			  struct drm_plane_state *state)
+{
+	struct vc4_bo *bo;
+	struct dma_fence *fence;
+
+	if ((plane->state->fb == state->fb) || !state->fb)
+		return 0;
+
+	bo = to_vc4_bo(&drm_fb_cma_get_gem_obj(state->fb, 0)->base);
+	fence = reservation_object_get_excl_rcu(bo->resv);
+	drm_atomic_set_fence_for_plane(state, fence);
+
+	return 0;
+}
+
 static const struct drm_plane_helper_funcs vc4_plane_helper_funcs = {
 	.atomic_check = vc4_plane_atomic_check,
 	.atomic_update = vc4_plane_atomic_update,
+	.prepare_fb = vc4_prepare_fb,
 };
 
 static void vc4_plane_destroy(struct drm_plane *plane)
@@ -885,7 +902,7 @@ struct drm_plane *vc4_plane_init(struct drm_device *dev,
 	ret = drm_universal_plane_init(dev, plane, 0,
 				       &vc4_plane_funcs,
 				       formats, num_formats,
-				       type, NULL);
+				       NULL, type, NULL);
 
 	drm_plane_helper_add(plane, &vc4_plane_helper_funcs);
 
diff --git a/drivers/gpu/drm/vc4/vc4_regs.h b/drivers/gpu/drm/vc4/vc4_regs.h
index d382c34c1b9e..55677bd50f66 100644
--- a/drivers/gpu/drm/vc4/vc4_regs.h
+++ b/drivers/gpu/drm/vc4/vc4_regs.h
@@ -561,16 +561,129 @@
 # define VC4_HDMI_VERTB_VBP_MASK		VC4_MASK(8, 0)
 # define VC4_HDMI_VERTB_VBP_SHIFT		0
 
+#define VC4_HDMI_CEC_CNTRL_1			0x0e8
+/* Set when the transmission has ended. */
+# define VC4_HDMI_CEC_TX_EOM			BIT(31)
+/* If set, transmission was acked on the 1st or 2nd attempt (only one
+ * retry is attempted).  If in continuous mode, this means TX needs to
+ * be filled if !TX_EOM.
+ */
+# define VC4_HDMI_CEC_TX_STATUS_GOOD		BIT(30)
+# define VC4_HDMI_CEC_RX_EOM			BIT(29)
+# define VC4_HDMI_CEC_RX_STATUS_GOOD		BIT(28)
+/* Number of bytes received for the message. */
+# define VC4_HDMI_CEC_REC_WRD_CNT_MASK		VC4_MASK(27, 24)
+# define VC4_HDMI_CEC_REC_WRD_CNT_SHIFT		24
+/* Sets continuous receive mode.  Generates interrupt after each 8
+ * bytes to signal that RX_DATA should be consumed, and at RX_EOM.
+ *
+ * If disabled, maximum 16 bytes will be received (including header),
+ * and interrupt at RX_EOM.  Later bytes will be acked but not put
+ * into the RX_DATA.
+ */
+# define VC4_HDMI_CEC_RX_CONTINUE		BIT(23)
+# define VC4_HDMI_CEC_TX_CONTINUE		BIT(22)
+/* Set this after a CEC interrupt. */
+# define VC4_HDMI_CEC_CLEAR_RECEIVE_OFF		BIT(21)
+/* Starts a TX.  Will wait for appropriate idel time before CEC
+ * activity. Must be cleared in between transmits.
+ */
+# define VC4_HDMI_CEC_START_XMIT_BEGIN		BIT(20)
+# define VC4_HDMI_CEC_MESSAGE_LENGTH_MASK	VC4_MASK(19, 16)
+# define VC4_HDMI_CEC_MESSAGE_LENGTH_SHIFT	16
+/* Device's CEC address */
+# define VC4_HDMI_CEC_ADDR_MASK			VC4_MASK(15, 12)
+# define VC4_HDMI_CEC_ADDR_SHIFT		12
+/* Divides off of HSM clock to generate CEC bit clock. */
+/* With the current defaults the CEC bit clock is 40 kHz = 25 usec */
+# define VC4_HDMI_CEC_DIV_CLK_CNT_MASK		VC4_MASK(11, 0)
+# define VC4_HDMI_CEC_DIV_CLK_CNT_SHIFT		0
+
+/* Set these fields to how many bit clock cycles get to that many
+ * microseconds.
+ */
+#define VC4_HDMI_CEC_CNTRL_2			0x0ec
+# define VC4_HDMI_CEC_CNT_TO_1500_US_MASK	VC4_MASK(30, 24)
+# define VC4_HDMI_CEC_CNT_TO_1500_US_SHIFT	24
+# define VC4_HDMI_CEC_CNT_TO_1300_US_MASK	VC4_MASK(23, 17)
+# define VC4_HDMI_CEC_CNT_TO_1300_US_SHIFT	17
+# define VC4_HDMI_CEC_CNT_TO_800_US_MASK	VC4_MASK(16, 11)
+# define VC4_HDMI_CEC_CNT_TO_800_US_SHIFT	11
+# define VC4_HDMI_CEC_CNT_TO_600_US_MASK	VC4_MASK(10, 5)
+# define VC4_HDMI_CEC_CNT_TO_600_US_SHIFT	5
+# define VC4_HDMI_CEC_CNT_TO_400_US_MASK	VC4_MASK(4, 0)
+# define VC4_HDMI_CEC_CNT_TO_400_US_SHIFT	0
+
+#define VC4_HDMI_CEC_CNTRL_3			0x0f0
+# define VC4_HDMI_CEC_CNT_TO_2750_US_MASK	VC4_MASK(31, 24)
+# define VC4_HDMI_CEC_CNT_TO_2750_US_SHIFT	24
+# define VC4_HDMI_CEC_CNT_TO_2400_US_MASK	VC4_MASK(23, 16)
+# define VC4_HDMI_CEC_CNT_TO_2400_US_SHIFT	16
+# define VC4_HDMI_CEC_CNT_TO_2050_US_MASK	VC4_MASK(15, 8)
+# define VC4_HDMI_CEC_CNT_TO_2050_US_SHIFT	8
+# define VC4_HDMI_CEC_CNT_TO_1700_US_MASK	VC4_MASK(7, 0)
+# define VC4_HDMI_CEC_CNT_TO_1700_US_SHIFT	0
+
+#define VC4_HDMI_CEC_CNTRL_4			0x0f4
+# define VC4_HDMI_CEC_CNT_TO_4300_US_MASK	VC4_MASK(31, 24)
+# define VC4_HDMI_CEC_CNT_TO_4300_US_SHIFT	24
+# define VC4_HDMI_CEC_CNT_TO_3900_US_MASK	VC4_MASK(23, 16)
+# define VC4_HDMI_CEC_CNT_TO_3900_US_SHIFT	16
+# define VC4_HDMI_CEC_CNT_TO_3600_US_MASK	VC4_MASK(15, 8)
+# define VC4_HDMI_CEC_CNT_TO_3600_US_SHIFT	8
+# define VC4_HDMI_CEC_CNT_TO_3500_US_MASK	VC4_MASK(7, 0)
+# define VC4_HDMI_CEC_CNT_TO_3500_US_SHIFT	0
+
+#define VC4_HDMI_CEC_CNTRL_5			0x0f8
+# define VC4_HDMI_CEC_TX_SW_RESET		BIT(27)
+# define VC4_HDMI_CEC_RX_SW_RESET		BIT(26)
+# define VC4_HDMI_CEC_PAD_SW_RESET		BIT(25)
+# define VC4_HDMI_CEC_MUX_TP_OUT_CEC		BIT(24)
+# define VC4_HDMI_CEC_RX_CEC_INT		BIT(23)
+# define VC4_HDMI_CEC_CLK_PRELOAD_MASK		VC4_MASK(22, 16)
+# define VC4_HDMI_CEC_CLK_PRELOAD_SHIFT		16
+# define VC4_HDMI_CEC_CNT_TO_4700_US_MASK	VC4_MASK(15, 8)
+# define VC4_HDMI_CEC_CNT_TO_4700_US_SHIFT	8
+# define VC4_HDMI_CEC_CNT_TO_4500_US_MASK	VC4_MASK(7, 0)
+# define VC4_HDMI_CEC_CNT_TO_4500_US_SHIFT	0
+
+/* Transmit data, first byte is low byte of the 32-bit reg.  MSB of
+ * each byte transmitted first.
+ */
+#define VC4_HDMI_CEC_TX_DATA_1			0x0fc
+#define VC4_HDMI_CEC_TX_DATA_2			0x100
+#define VC4_HDMI_CEC_TX_DATA_3			0x104
+#define VC4_HDMI_CEC_TX_DATA_4			0x108
+#define VC4_HDMI_CEC_RX_DATA_1			0x10c
+#define VC4_HDMI_CEC_RX_DATA_2			0x110
+#define VC4_HDMI_CEC_RX_DATA_3			0x114
+#define VC4_HDMI_CEC_RX_DATA_4			0x118
+
 #define VC4_HDMI_TX_PHY_RESET_CTL		0x2c0
 
 #define VC4_HDMI_TX_PHY_CTL0			0x2c4
 # define VC4_HDMI_TX_PHY_RNG_PWRDN		BIT(25)
 
+/* Interrupt status bits */
+#define VC4_HDMI_CPU_STATUS			0x340
+#define VC4_HDMI_CPU_SET			0x344
+#define VC4_HDMI_CPU_CLEAR			0x348
+# define VC4_HDMI_CPU_CEC			BIT(6)
+# define VC4_HDMI_CPU_HOTPLUG			BIT(0)
+
+#define VC4_HDMI_CPU_MASK_STATUS		0x34c
+#define VC4_HDMI_CPU_MASK_SET			0x350
+#define VC4_HDMI_CPU_MASK_CLEAR			0x354
+
 #define VC4_HDMI_GCP(x)				(0x400 + ((x) * 0x4))
 #define VC4_HDMI_RAM_PACKET(x)			(0x400 + ((x) * 0x24))
 #define VC4_HDMI_PACKET_STRIDE			0x24
 
 #define VC4_HD_M_CTL				0x00c
+/* Debug: Current receive value on the CEC pad. */
+# define VC4_HD_CECRXD				BIT(9)
+/* Debug: Override CEC output to 0. */
+# define VC4_HD_CECOVR				BIT(8)
 # define VC4_HD_M_REGISTER_FILE_STANDBY		(3 << 6)
 # define VC4_HD_M_RAM_STANDBY			(3 << 4)
 # define VC4_HD_M_SW_RST			BIT(2)
diff --git a/drivers/gpu/drm/vc4/vc4_render_cl.c b/drivers/gpu/drm/vc4/vc4_render_cl.c
index 5dc19429d4ae..273984f71ae2 100644
--- a/drivers/gpu/drm/vc4/vc4_render_cl.c
+++ b/drivers/gpu/drm/vc4/vc4_render_cl.c
@@ -261,8 +261,17 @@ static int vc4_create_rcl_bo(struct drm_device *dev, struct vc4_exec_info *exec,
 	uint8_t max_y_tile = args->max_y_tile;
 	uint8_t xtiles = max_x_tile - min_x_tile + 1;
 	uint8_t ytiles = max_y_tile - min_y_tile + 1;
-	uint8_t x, y;
+	uint8_t xi, yi;
 	uint32_t size, loop_body_size;
+	bool positive_x = true;
+	bool positive_y = true;
+
+	if (args->flags & VC4_SUBMIT_CL_FIXED_RCL_ORDER) {
+		if (!(args->flags & VC4_SUBMIT_CL_RCL_ORDER_INCREASING_X))
+			positive_x = false;
+		if (!(args->flags & VC4_SUBMIT_CL_RCL_ORDER_INCREASING_Y))
+			positive_y = false;
+	}
 
 	size = VC4_PACKET_TILE_RENDERING_MODE_CONFIG_SIZE;
 	loop_body_size = VC4_PACKET_TILE_COORDINATES_SIZE;
@@ -320,7 +329,7 @@ static int vc4_create_rcl_bo(struct drm_device *dev, struct vc4_exec_info *exec,
 
 	size += xtiles * ytiles * loop_body_size;
 
-	setup->rcl = &vc4_bo_create(dev, size, true)->base;
+	setup->rcl = &vc4_bo_create(dev, size, true, VC4_BO_TYPE_RCL)->base;
 	if (IS_ERR(setup->rcl))
 		return PTR_ERR(setup->rcl);
 	list_add_tail(&to_vc4_bo(&setup->rcl->base)->unref_head,
@@ -354,10 +363,12 @@ static int vc4_create_rcl_bo(struct drm_device *dev, struct vc4_exec_info *exec,
 	rcl_u16(setup, args->height);
 	rcl_u16(setup, args->color_write.bits);
 
-	for (y = min_y_tile; y <= max_y_tile; y++) {
-		for (x = min_x_tile; x <= max_x_tile; x++) {
-			bool first = (x == min_x_tile && y == min_y_tile);
-			bool last = (x == max_x_tile && y == max_y_tile);
+	for (yi = 0; yi < ytiles; yi++) {
+		int y = positive_y ? min_y_tile + yi : max_y_tile - yi;
+		for (xi = 0; xi < xtiles; xi++) {
+			int x = positive_x ? min_x_tile + xi : max_x_tile - xi;
+			bool first = (xi == 0 && yi == 0);
+			bool last = (xi == xtiles - 1 && yi == ytiles - 1);
 
 			emit_tile(exec, setup, x, y, first, last);
 		}
@@ -378,14 +389,14 @@ static int vc4_full_res_bounds_check(struct vc4_exec_info *exec,
 	u32 render_tiles_stride = DIV_ROUND_UP(exec->args->width, 32);
 
 	if (surf->offset > obj->base.size) {
-		DRM_ERROR("surface offset %d > BO size %zd\n",
+		DRM_DEBUG("surface offset %d > BO size %zd\n",
 			  surf->offset, obj->base.size);
 		return -EINVAL;
 	}
 
 	if ((obj->base.size - surf->offset) / VC4_TILE_BUFFER_SIZE <
 	    render_tiles_stride * args->max_y_tile + args->max_x_tile) {
-		DRM_ERROR("MSAA tile %d, %d out of bounds "
+		DRM_DEBUG("MSAA tile %d, %d out of bounds "
 			  "(bo size %zd, offset %d).\n",
 			  args->max_x_tile, args->max_y_tile,
 			  obj->base.size,
@@ -401,7 +412,7 @@ static int vc4_rcl_msaa_surface_setup(struct vc4_exec_info *exec,
 				      struct drm_vc4_submit_rcl_surface *surf)
 {
 	if (surf->flags != 0 || surf->bits != 0) {
-		DRM_ERROR("MSAA surface had nonzero flags/bits\n");
+		DRM_DEBUG("MSAA surface had nonzero flags/bits\n");
 		return -EINVAL;
 	}
 
@@ -415,7 +426,7 @@ static int vc4_rcl_msaa_surface_setup(struct vc4_exec_info *exec,
 	exec->rcl_write_bo[exec->rcl_write_bo_count++] = *obj;
 
 	if (surf->offset & 0xf) {
-		DRM_ERROR("MSAA write must be 16b aligned.\n");
+		DRM_DEBUG("MSAA write must be 16b aligned.\n");
 		return -EINVAL;
 	}
 
@@ -437,7 +448,7 @@ static int vc4_rcl_surface_setup(struct vc4_exec_info *exec,
 	int ret;
 
 	if (surf->flags & ~VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
-		DRM_ERROR("Extra flags set\n");
+		DRM_DEBUG("Extra flags set\n");
 		return -EINVAL;
 	}
 
@@ -453,12 +464,12 @@ static int vc4_rcl_surface_setup(struct vc4_exec_info *exec,
 
 	if (surf->flags & VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
 		if (surf == &exec->args->zs_write) {
-			DRM_ERROR("general zs write may not be a full-res.\n");
+			DRM_DEBUG("general zs write may not be a full-res.\n");
 			return -EINVAL;
 		}
 
 		if (surf->bits != 0) {
-			DRM_ERROR("load/store general bits set with "
+			DRM_DEBUG("load/store general bits set with "
 				  "full res load/store.\n");
 			return -EINVAL;
 		}
@@ -473,19 +484,19 @@ static int vc4_rcl_surface_setup(struct vc4_exec_info *exec,
 	if (surf->bits & ~(VC4_LOADSTORE_TILE_BUFFER_TILING_MASK |
 			   VC4_LOADSTORE_TILE_BUFFER_BUFFER_MASK |
 			   VC4_LOADSTORE_TILE_BUFFER_FORMAT_MASK)) {
-		DRM_ERROR("Unknown bits in load/store: 0x%04x\n",
+		DRM_DEBUG("Unknown bits in load/store: 0x%04x\n",
 			  surf->bits);
 		return -EINVAL;
 	}
 
 	if (tiling > VC4_TILING_FORMAT_LT) {
-		DRM_ERROR("Bad tiling format\n");
+		DRM_DEBUG("Bad tiling format\n");
 		return -EINVAL;
 	}
 
 	if (buffer == VC4_LOADSTORE_TILE_BUFFER_ZS) {
 		if (format != 0) {
-			DRM_ERROR("No color format should be set for ZS\n");
+			DRM_DEBUG("No color format should be set for ZS\n");
 			return -EINVAL;
 		}
 		cpp = 4;
@@ -499,16 +510,16 @@ static int vc4_rcl_surface_setup(struct vc4_exec_info *exec,
 			cpp = 4;
 			break;
 		default:
-			DRM_ERROR("Bad tile buffer format\n");
+			DRM_DEBUG("Bad tile buffer format\n");
 			return -EINVAL;
 		}
 	} else {
-		DRM_ERROR("Bad load/store buffer %d.\n", buffer);
+		DRM_DEBUG("Bad load/store buffer %d.\n", buffer);
 		return -EINVAL;
 	}
 
 	if (surf->offset & 0xf) {
-		DRM_ERROR("load/store buffer must be 16b aligned.\n");
+		DRM_DEBUG("load/store buffer must be 16b aligned.\n");
 		return -EINVAL;
 	}
 
@@ -533,7 +544,7 @@ vc4_rcl_render_config_surface_setup(struct vc4_exec_info *exec,
 	int cpp;
 
 	if (surf->flags != 0) {
-		DRM_ERROR("No flags supported on render config.\n");
+		DRM_DEBUG("No flags supported on render config.\n");
 		return -EINVAL;
 	}
 
@@ -541,7 +552,7 @@ vc4_rcl_render_config_surface_setup(struct vc4_exec_info *exec,
 			   VC4_RENDER_CONFIG_FORMAT_MASK |
 			   VC4_RENDER_CONFIG_MS_MODE_4X |
 			   VC4_RENDER_CONFIG_DECIMATE_MODE_4X)) {
-		DRM_ERROR("Unknown bits in render config: 0x%04x\n",
+		DRM_DEBUG("Unknown bits in render config: 0x%04x\n",
 			  surf->bits);
 		return -EINVAL;
 	}
@@ -556,7 +567,7 @@ vc4_rcl_render_config_surface_setup(struct vc4_exec_info *exec,
 	exec->rcl_write_bo[exec->rcl_write_bo_count++] = *obj;
 
 	if (tiling > VC4_TILING_FORMAT_LT) {
-		DRM_ERROR("Bad tiling format\n");
+		DRM_DEBUG("Bad tiling format\n");
 		return -EINVAL;
 	}
 
@@ -569,7 +580,7 @@ vc4_rcl_render_config_surface_setup(struct vc4_exec_info *exec,
 		cpp = 4;
 		break;
 	default:
-		DRM_ERROR("Bad tile buffer format\n");
+		DRM_DEBUG("Bad tile buffer format\n");
 		return -EINVAL;
 	}
 
@@ -590,7 +601,7 @@ int vc4_get_rcl(struct drm_device *dev, struct vc4_exec_info *exec)
 
 	if (args->min_x_tile > args->max_x_tile ||
 	    args->min_y_tile > args->max_y_tile) {
-		DRM_ERROR("Bad render tile set (%d,%d)-(%d,%d)\n",
+		DRM_DEBUG("Bad render tile set (%d,%d)-(%d,%d)\n",
 			  args->min_x_tile, args->min_y_tile,
 			  args->max_x_tile, args->max_y_tile);
 		return -EINVAL;
@@ -599,7 +610,7 @@ int vc4_get_rcl(struct drm_device *dev, struct vc4_exec_info *exec)
 	if (has_bin &&
 	    (args->max_x_tile > exec->bin_tiles_x ||
 	     args->max_y_tile > exec->bin_tiles_y)) {
-		DRM_ERROR("Render tiles (%d,%d) outside of bin config "
+		DRM_DEBUG("Render tiles (%d,%d) outside of bin config "
 			  "(%d,%d)\n",
 			  args->max_x_tile, args->max_y_tile,
 			  exec->bin_tiles_x, exec->bin_tiles_y);
@@ -642,7 +653,7 @@ int vc4_get_rcl(struct drm_device *dev, struct vc4_exec_info *exec)
 	 */
 	if (!setup.color_write && !setup.zs_write &&
 	    !setup.msaa_color_write && !setup.msaa_zs_write) {
-		DRM_ERROR("RCL requires color or Z/S write\n");
+		DRM_DEBUG("RCL requires color or Z/S write\n");
 		return -EINVAL;
 	}
 
diff --git a/drivers/gpu/drm/vc4/vc4_v3d.c b/drivers/gpu/drm/vc4/vc4_v3d.c
index 8c723da71f66..622cd43840b8 100644
--- a/drivers/gpu/drm/vc4/vc4_v3d.c
+++ b/drivers/gpu/drm/vc4/vc4_v3d.c
@@ -236,7 +236,8 @@ vc4_allocate_bin_bo(struct drm_device *drm)
 	INIT_LIST_HEAD(&list);
 
 	while (true) {
-		struct vc4_bo *bo = vc4_bo_create(drm, size, true);
+		struct vc4_bo *bo = vc4_bo_create(drm, size, true,
+						  VC4_BO_TYPE_BIN);
 
 		if (IS_ERR(bo)) {
 			ret = PTR_ERR(bo);
diff --git a/drivers/gpu/drm/vc4/vc4_validate.c b/drivers/gpu/drm/vc4/vc4_validate.c
index 814b512c6b9a..2db485abb186 100644
--- a/drivers/gpu/drm/vc4/vc4_validate.c
+++ b/drivers/gpu/drm/vc4/vc4_validate.c
@@ -109,7 +109,7 @@ vc4_use_bo(struct vc4_exec_info *exec, uint32_t hindex)
 	struct vc4_bo *bo;
 
 	if (hindex >= exec->bo_count) {
-		DRM_ERROR("BO index %d greater than BO count %d\n",
+		DRM_DEBUG("BO index %d greater than BO count %d\n",
 			  hindex, exec->bo_count);
 		return NULL;
 	}
@@ -117,7 +117,7 @@ vc4_use_bo(struct vc4_exec_info *exec, uint32_t hindex)
 	bo = to_vc4_bo(&obj->base);
 
 	if (bo->validated_shader) {
-		DRM_ERROR("Trying to use shader BO as something other than "
+		DRM_DEBUG("Trying to use shader BO as something other than "
 			  "a shader\n");
 		return NULL;
 	}
@@ -172,7 +172,7 @@ vc4_check_tex_size(struct vc4_exec_info *exec, struct drm_gem_cma_object *fbo,
 	 * our math.
 	 */
 	if (width > 4096 || height > 4096) {
-		DRM_ERROR("Surface dimensions (%d,%d) too large",
+		DRM_DEBUG("Surface dimensions (%d,%d) too large",
 			  width, height);
 		return false;
 	}
@@ -191,7 +191,7 @@ vc4_check_tex_size(struct vc4_exec_info *exec, struct drm_gem_cma_object *fbo,
 		aligned_height = round_up(height, utile_h);
 		break;
 	default:
-		DRM_ERROR("buffer tiling %d unsupported\n", tiling_format);
+		DRM_DEBUG("buffer tiling %d unsupported\n", tiling_format);
 		return false;
 	}
 
@@ -200,7 +200,7 @@ vc4_check_tex_size(struct vc4_exec_info *exec, struct drm_gem_cma_object *fbo,
 
 	if (size + offset < size ||
 	    size + offset > fbo->base.size) {
-		DRM_ERROR("Overflow in %dx%d (%dx%d) fbo size (%d + %d > %zd)\n",
+		DRM_DEBUG("Overflow in %dx%d (%dx%d) fbo size (%d + %d > %zd)\n",
 			  width, height,
 			  aligned_width, aligned_height,
 			  size, offset, fbo->base.size);
@@ -214,7 +214,7 @@ static int
 validate_flush(VALIDATE_ARGS)
 {
 	if (!validate_bin_pos(exec, untrusted, exec->args->bin_cl_size - 1)) {
-		DRM_ERROR("Bin CL must end with VC4_PACKET_FLUSH\n");
+		DRM_DEBUG("Bin CL must end with VC4_PACKET_FLUSH\n");
 		return -EINVAL;
 	}
 	exec->found_flush = true;
@@ -226,13 +226,13 @@ static int
 validate_start_tile_binning(VALIDATE_ARGS)
 {
 	if (exec->found_start_tile_binning_packet) {
-		DRM_ERROR("Duplicate VC4_PACKET_START_TILE_BINNING\n");
+		DRM_DEBUG("Duplicate VC4_PACKET_START_TILE_BINNING\n");
 		return -EINVAL;
 	}
 	exec->found_start_tile_binning_packet = true;
 
 	if (!exec->found_tile_binning_mode_config_packet) {
-		DRM_ERROR("missing VC4_PACKET_TILE_BINNING_MODE_CONFIG\n");
+		DRM_DEBUG("missing VC4_PACKET_TILE_BINNING_MODE_CONFIG\n");
 		return -EINVAL;
 	}
 
@@ -243,7 +243,7 @@ static int
 validate_increment_semaphore(VALIDATE_ARGS)
 {
 	if (!validate_bin_pos(exec, untrusted, exec->args->bin_cl_size - 2)) {
-		DRM_ERROR("Bin CL must end with "
+		DRM_DEBUG("Bin CL must end with "
 			  "VC4_PACKET_INCREMENT_SEMAPHORE\n");
 		return -EINVAL;
 	}
@@ -264,7 +264,7 @@ validate_indexed_prim_list(VALIDATE_ARGS)
 
 	/* Check overflow condition */
 	if (exec->shader_state_count == 0) {
-		DRM_ERROR("shader state must precede primitives\n");
+		DRM_DEBUG("shader state must precede primitives\n");
 		return -EINVAL;
 	}
 	shader_state = &exec->shader_state[exec->shader_state_count - 1];
@@ -281,7 +281,7 @@ validate_indexed_prim_list(VALIDATE_ARGS)
 
 	if (offset > ib->base.size ||
 	    (ib->base.size - offset) / index_size < length) {
-		DRM_ERROR("IB access overflow (%d + %d*%d > %zd)\n",
+		DRM_DEBUG("IB access overflow (%d + %d*%d > %zd)\n",
 			  offset, length, index_size, ib->base.size);
 		return -EINVAL;
 	}
@@ -301,13 +301,13 @@ validate_gl_array_primitive(VALIDATE_ARGS)
 
 	/* Check overflow condition */
 	if (exec->shader_state_count == 0) {
-		DRM_ERROR("shader state must precede primitives\n");
+		DRM_DEBUG("shader state must precede primitives\n");
 		return -EINVAL;
 	}
 	shader_state = &exec->shader_state[exec->shader_state_count - 1];
 
 	if (length + base_index < length) {
-		DRM_ERROR("primitive vertex count overflow\n");
+		DRM_DEBUG("primitive vertex count overflow\n");
 		return -EINVAL;
 	}
 	max_index = length + base_index - 1;
@@ -324,7 +324,7 @@ validate_gl_shader_state(VALIDATE_ARGS)
 	uint32_t i = exec->shader_state_count++;
 
 	if (i >= exec->shader_state_size) {
-		DRM_ERROR("More requests for shader states than declared\n");
+		DRM_DEBUG("More requests for shader states than declared\n");
 		return -EINVAL;
 	}
 
@@ -332,7 +332,7 @@ validate_gl_shader_state(VALIDATE_ARGS)
 	exec->shader_state[i].max_index = 0;
 
 	if (exec->shader_state[i].addr & ~0xf) {
-		DRM_ERROR("high bits set in GL shader rec reference\n");
+		DRM_DEBUG("high bits set in GL shader rec reference\n");
 		return -EINVAL;
 	}
 
@@ -356,7 +356,7 @@ validate_tile_binning_config(VALIDATE_ARGS)
 	int bin_slot;
 
 	if (exec->found_tile_binning_mode_config_packet) {
-		DRM_ERROR("Duplicate VC4_PACKET_TILE_BINNING_MODE_CONFIG\n");
+		DRM_DEBUG("Duplicate VC4_PACKET_TILE_BINNING_MODE_CONFIG\n");
 		return -EINVAL;
 	}
 	exec->found_tile_binning_mode_config_packet = true;
@@ -368,14 +368,14 @@ validate_tile_binning_config(VALIDATE_ARGS)
 
 	if (exec->bin_tiles_x == 0 ||
 	    exec->bin_tiles_y == 0) {
-		DRM_ERROR("Tile binning config of %dx%d too small\n",
+		DRM_DEBUG("Tile binning config of %dx%d too small\n",
 			  exec->bin_tiles_x, exec->bin_tiles_y);
 		return -EINVAL;
 	}
 
 	if (flags & (VC4_BIN_CONFIG_DB_NON_MS |
 		     VC4_BIN_CONFIG_TILE_BUFFER_64BIT)) {
-		DRM_ERROR("unsupported binning config flags 0x%02x\n", flags);
+		DRM_DEBUG("unsupported binning config flags 0x%02x\n", flags);
 		return -EINVAL;
 	}
 
@@ -493,20 +493,20 @@ vc4_validate_bin_cl(struct drm_device *dev,
 		const struct cmd_info *info;
 
 		if (cmd >= ARRAY_SIZE(cmd_info)) {
-			DRM_ERROR("0x%08x: packet %d out of bounds\n",
+			DRM_DEBUG("0x%08x: packet %d out of bounds\n",
 				  src_offset, cmd);
 			return -EINVAL;
 		}
 
 		info = &cmd_info[cmd];
 		if (!info->name) {
-			DRM_ERROR("0x%08x: packet %d invalid\n",
+			DRM_DEBUG("0x%08x: packet %d invalid\n",
 				  src_offset, cmd);
 			return -EINVAL;
 		}
 
 		if (src_offset + info->len > len) {
-			DRM_ERROR("0x%08x: packet %d (%s) length 0x%08x "
+			DRM_DEBUG("0x%08x: packet %d (%s) length 0x%08x "
 				  "exceeds bounds (0x%08x)\n",
 				  src_offset, cmd, info->name, info->len,
 				  src_offset + len);
@@ -519,7 +519,7 @@ vc4_validate_bin_cl(struct drm_device *dev,
 		if (info->func && info->func(exec,
 					     dst_pkt + 1,
 					     src_pkt + 1)) {
-			DRM_ERROR("0x%08x: packet %d (%s) failed to validate\n",
+			DRM_DEBUG("0x%08x: packet %d (%s) failed to validate\n",
 				  src_offset, cmd, info->name);
 			return -EINVAL;
 		}
@@ -537,7 +537,7 @@ vc4_validate_bin_cl(struct drm_device *dev,
 	exec->ct0ea = exec->ct0ca + dst_offset;
 
 	if (!exec->found_start_tile_binning_packet) {
-		DRM_ERROR("Bin CL missing VC4_PACKET_START_TILE_BINNING\n");
+		DRM_DEBUG("Bin CL missing VC4_PACKET_START_TILE_BINNING\n");
 		return -EINVAL;
 	}
 
@@ -549,7 +549,7 @@ vc4_validate_bin_cl(struct drm_device *dev,
 	 * semaphore increment.
 	 */
 	if (!exec->found_increment_semaphore_packet || !exec->found_flush) {
-		DRM_ERROR("Bin CL missing VC4_PACKET_INCREMENT_SEMAPHORE + "
+		DRM_DEBUG("Bin CL missing VC4_PACKET_INCREMENT_SEMAPHORE + "
 			  "VC4_PACKET_FLUSH\n");
 		return -EINVAL;
 	}
@@ -588,11 +588,11 @@ reloc_tex(struct vc4_exec_info *exec,
 		uint32_t remaining_size = tex->base.size - p0;
 
 		if (p0 > tex->base.size - 4) {
-			DRM_ERROR("UBO offset greater than UBO size\n");
+			DRM_DEBUG("UBO offset greater than UBO size\n");
 			goto fail;
 		}
 		if (p1 > remaining_size - 4) {
-			DRM_ERROR("UBO clamp would allow reads "
+			DRM_DEBUG("UBO clamp would allow reads "
 				  "outside of UBO\n");
 			goto fail;
 		}
@@ -612,14 +612,14 @@ reloc_tex(struct vc4_exec_info *exec,
 		if (VC4_GET_FIELD(p3, VC4_TEX_P2_PTYPE) ==
 		    VC4_TEX_P2_PTYPE_CUBE_MAP_STRIDE) {
 			if (cube_map_stride) {
-				DRM_ERROR("Cube map stride set twice\n");
+				DRM_DEBUG("Cube map stride set twice\n");
 				goto fail;
 			}
 
 			cube_map_stride = p3 & VC4_TEX_P2_CMST_MASK;
 		}
 		if (!cube_map_stride) {
-			DRM_ERROR("Cube map stride not set\n");
+			DRM_DEBUG("Cube map stride not set\n");
 			goto fail;
 		}
 	}
@@ -660,7 +660,7 @@ reloc_tex(struct vc4_exec_info *exec,
 	case VC4_TEXTURE_TYPE_RGBA64:
 	case VC4_TEXTURE_TYPE_YUV422R:
 	default:
-		DRM_ERROR("Texture format %d unsupported\n", type);
+		DRM_DEBUG("Texture format %d unsupported\n", type);
 		goto fail;
 	}
 	utile_w = utile_width(cpp);
@@ -713,7 +713,7 @@ reloc_tex(struct vc4_exec_info *exec,
 		level_size = aligned_width * cpp * aligned_height;
 
 		if (offset < level_size) {
-			DRM_ERROR("Level %d (%dx%d -> %dx%d) size %db "
+			DRM_DEBUG("Level %d (%dx%d -> %dx%d) size %db "
 				  "overflowed buffer bounds (offset %d)\n",
 				  i, level_width, level_height,
 				  aligned_width, aligned_height,
@@ -764,7 +764,7 @@ validate_gl_shader_rec(struct drm_device *dev,
 
 	nr_relocs = ARRAY_SIZE(shader_reloc_offsets) + nr_attributes;
 	if (nr_relocs * 4 > exec->shader_rec_size) {
-		DRM_ERROR("overflowed shader recs reading %d handles "
+		DRM_DEBUG("overflowed shader recs reading %d handles "
 			  "from %d bytes left\n",
 			  nr_relocs, exec->shader_rec_size);
 		return -EINVAL;
@@ -774,7 +774,7 @@ validate_gl_shader_rec(struct drm_device *dev,
 	exec->shader_rec_size -= nr_relocs * 4;
 
 	if (packet_size > exec->shader_rec_size) {
-		DRM_ERROR("overflowed shader recs copying %db packet "
+		DRM_DEBUG("overflowed shader recs copying %db packet "
 			  "from %d bytes left\n",
 			  packet_size, exec->shader_rec_size);
 		return -EINVAL;
@@ -794,7 +794,7 @@ validate_gl_shader_rec(struct drm_device *dev,
 
 	for (i = 0; i < shader_reloc_count; i++) {
 		if (src_handles[i] > exec->bo_count) {
-			DRM_ERROR("Shader handle %d too big\n", src_handles[i]);
+			DRM_DEBUG("Shader handle %d too big\n", src_handles[i]);
 			return -EINVAL;
 		}
 
@@ -810,13 +810,13 @@ validate_gl_shader_rec(struct drm_device *dev,
 
 	if (((*(uint16_t *)pkt_u & VC4_SHADER_FLAG_FS_SINGLE_THREAD) == 0) !=
 	    to_vc4_bo(&bo[0]->base)->validated_shader->is_threaded) {
-		DRM_ERROR("Thread mode of CL and FS do not match\n");
+		DRM_DEBUG("Thread mode of CL and FS do not match\n");
 		return -EINVAL;
 	}
 
 	if (to_vc4_bo(&bo[1]->base)->validated_shader->is_threaded ||
 	    to_vc4_bo(&bo[2]->base)->validated_shader->is_threaded) {
-		DRM_ERROR("cs and vs cannot be threaded\n");
+		DRM_DEBUG("cs and vs cannot be threaded\n");
 		return -EINVAL;
 	}
 
@@ -831,7 +831,7 @@ validate_gl_shader_rec(struct drm_device *dev,
 		*(uint32_t *)(pkt_v + o) = bo[i]->paddr + src_offset;
 
 		if (src_offset != 0) {
-			DRM_ERROR("Shaders must be at offset 0 of "
+			DRM_DEBUG("Shaders must be at offset 0 of "
 				  "the BO.\n");
 			return -EINVAL;
 		}
@@ -842,7 +842,7 @@ validate_gl_shader_rec(struct drm_device *dev,
 
 		if (validated_shader->uniforms_src_size >
 		    exec->uniforms_size) {
-			DRM_ERROR("Uniforms src buffer overflow\n");
+			DRM_DEBUG("Uniforms src buffer overflow\n");
 			return -EINVAL;
 		}
 
@@ -900,7 +900,7 @@ validate_gl_shader_rec(struct drm_device *dev,
 
 		if (vbo->base.size < offset ||
 		    vbo->base.size - offset < attr_size) {
-			DRM_ERROR("BO offset overflow (%d + %d > %zu)\n",
+			DRM_DEBUG("BO offset overflow (%d + %d > %zu)\n",
 				  offset, attr_size, vbo->base.size);
 			return -EINVAL;
 		}
@@ -909,7 +909,7 @@ validate_gl_shader_rec(struct drm_device *dev,
 			max_index = ((vbo->base.size - offset - attr_size) /
 				     stride);
 			if (state->max_index > max_index) {
-				DRM_ERROR("primitives use index %d out of "
+				DRM_DEBUG("primitives use index %d out of "
 					  "supplied %d\n",
 					  state->max_index, max_index);
 				return -EINVAL;
diff --git a/drivers/gpu/drm/vc4/vc4_validate_shaders.c b/drivers/gpu/drm/vc4/vc4_validate_shaders.c
index 0b2df5c6efb4..d3f15bf60900 100644
--- a/drivers/gpu/drm/vc4/vc4_validate_shaders.c
+++ b/drivers/gpu/drm/vc4/vc4_validate_shaders.c
@@ -200,7 +200,7 @@ check_tmu_write(struct vc4_validated_shader_info *validated_shader,
 		uint32_t clamp_reg, clamp_offset;
 
 		if (sig == QPU_SIG_SMALL_IMM) {
-			DRM_ERROR("direct TMU read used small immediate\n");
+			DRM_DEBUG("direct TMU read used small immediate\n");
 			return false;
 		}
 
@@ -209,7 +209,7 @@ check_tmu_write(struct vc4_validated_shader_info *validated_shader,
 		 */
 		if (is_mul ||
 		    QPU_GET_FIELD(inst, QPU_OP_ADD) != QPU_A_ADD) {
-			DRM_ERROR("direct TMU load wasn't an add\n");
+			DRM_DEBUG("direct TMU load wasn't an add\n");
 			return false;
 		}
 
@@ -220,13 +220,13 @@ check_tmu_write(struct vc4_validated_shader_info *validated_shader,
 		 */
 		clamp_reg = raddr_add_a_to_live_reg_index(inst);
 		if (clamp_reg == ~0) {
-			DRM_ERROR("direct TMU load wasn't clamped\n");
+			DRM_DEBUG("direct TMU load wasn't clamped\n");
 			return false;
 		}
 
 		clamp_offset = validation_state->live_min_clamp_offsets[clamp_reg];
 		if (clamp_offset == ~0) {
-			DRM_ERROR("direct TMU load wasn't clamped\n");
+			DRM_DEBUG("direct TMU load wasn't clamped\n");
 			return false;
 		}
 
@@ -238,7 +238,7 @@ check_tmu_write(struct vc4_validated_shader_info *validated_shader,
 
 		if (!(add_b == QPU_MUX_A && raddr_a == QPU_R_UNIF) &&
 		    !(add_b == QPU_MUX_B && raddr_b == QPU_R_UNIF)) {
-			DRM_ERROR("direct TMU load didn't add to a uniform\n");
+			DRM_DEBUG("direct TMU load didn't add to a uniform\n");
 			return false;
 		}
 
@@ -246,14 +246,14 @@ check_tmu_write(struct vc4_validated_shader_info *validated_shader,
 	} else {
 		if (raddr_a == QPU_R_UNIF || (sig != QPU_SIG_SMALL_IMM &&
 					      raddr_b == QPU_R_UNIF)) {
-			DRM_ERROR("uniform read in the same instruction as "
+			DRM_DEBUG("uniform read in the same instruction as "
 				  "texture setup.\n");
 			return false;
 		}
 	}
 
 	if (validation_state->tmu_write_count[tmu] >= 4) {
-		DRM_ERROR("TMU%d got too many parameters before dispatch\n",
+		DRM_DEBUG("TMU%d got too many parameters before dispatch\n",
 			  tmu);
 		return false;
 	}
@@ -265,7 +265,7 @@ check_tmu_write(struct vc4_validated_shader_info *validated_shader,
 	 */
 	if (!is_direct) {
 		if (validation_state->needs_uniform_address_update) {
-			DRM_ERROR("Texturing with undefined uniform address\n");
+			DRM_DEBUG("Texturing with undefined uniform address\n");
 			return false;
 		}
 
@@ -336,35 +336,35 @@ validate_uniform_address_write(struct vc4_validated_shader_info *validated_shade
 	case QPU_SIG_LOAD_TMU1:
 		break;
 	default:
-		DRM_ERROR("uniforms address change must be "
+		DRM_DEBUG("uniforms address change must be "
 			  "normal math\n");
 		return false;
 	}
 
 	if (is_mul || QPU_GET_FIELD(inst, QPU_OP_ADD) != QPU_A_ADD) {
-		DRM_ERROR("Uniform address reset must be an ADD.\n");
+		DRM_DEBUG("Uniform address reset must be an ADD.\n");
 		return false;
 	}
 
 	if (QPU_GET_FIELD(inst, QPU_COND_ADD) != QPU_COND_ALWAYS) {
-		DRM_ERROR("Uniform address reset must be unconditional.\n");
+		DRM_DEBUG("Uniform address reset must be unconditional.\n");
 		return false;
 	}
 
 	if (QPU_GET_FIELD(inst, QPU_PACK) != QPU_PACK_A_NOP &&
 	    !(inst & QPU_PM)) {
-		DRM_ERROR("No packing allowed on uniforms reset\n");
+		DRM_DEBUG("No packing allowed on uniforms reset\n");
 		return false;
 	}
 
 	if (add_lri == -1) {
-		DRM_ERROR("First argument of uniform address write must be "
+		DRM_DEBUG("First argument of uniform address write must be "
 			  "an immediate value.\n");
 		return false;
 	}
 
 	if (validation_state->live_immediates[add_lri] != expected_offset) {
-		DRM_ERROR("Resetting uniforms with offset %db instead of %db\n",
+		DRM_DEBUG("Resetting uniforms with offset %db instead of %db\n",
 			  validation_state->live_immediates[add_lri],
 			  expected_offset);
 		return false;
@@ -372,7 +372,7 @@ validate_uniform_address_write(struct vc4_validated_shader_info *validated_shade
 
 	if (!(add_b == QPU_MUX_A && raddr_a == QPU_R_UNIF) &&
 	    !(add_b == QPU_MUX_B && raddr_b == QPU_R_UNIF)) {
-		DRM_ERROR("Second argument of uniform address write must be "
+		DRM_DEBUG("Second argument of uniform address write must be "
 			  "a uniform.\n");
 		return false;
 	}
@@ -417,7 +417,7 @@ check_reg_write(struct vc4_validated_shader_info *validated_shader,
 	switch (waddr) {
 	case QPU_W_UNIFORMS_ADDRESS:
 		if (is_b) {
-			DRM_ERROR("relative uniforms address change "
+			DRM_DEBUG("relative uniforms address change "
 				  "unsupported\n");
 			return false;
 		}
@@ -452,11 +452,11 @@ check_reg_write(struct vc4_validated_shader_info *validated_shader,
 		/* XXX: I haven't thought about these, so don't support them
 		 * for now.
 		 */
-		DRM_ERROR("Unsupported waddr %d\n", waddr);
+		DRM_DEBUG("Unsupported waddr %d\n", waddr);
 		return false;
 
 	case QPU_W_VPM_ADDR:
-		DRM_ERROR("General VPM DMA unsupported\n");
+		DRM_DEBUG("General VPM DMA unsupported\n");
 		return false;
 
 	case QPU_W_VPM:
@@ -559,7 +559,7 @@ check_instruction_writes(struct vc4_validated_shader_info *validated_shader,
 	bool ok;
 
 	if (is_tmu_write(waddr_add) && is_tmu_write(waddr_mul)) {
-		DRM_ERROR("ADD and MUL both set up textures\n");
+		DRM_DEBUG("ADD and MUL both set up textures\n");
 		return false;
 	}
 
@@ -588,7 +588,7 @@ check_branch(uint64_t inst,
 	 * there's no need for it.
 	 */
 	if (waddr_add != QPU_W_NOP || waddr_mul != QPU_W_NOP) {
-		DRM_ERROR("branch instruction at %d wrote a register.\n",
+		DRM_DEBUG("branch instruction at %d wrote a register.\n",
 			  validation_state->ip);
 		return false;
 	}
@@ -614,7 +614,7 @@ check_instruction_reads(struct vc4_validated_shader_info *validated_shader,
 		validated_shader->uniforms_size += 4;
 
 		if (validation_state->needs_uniform_address_update) {
-			DRM_ERROR("Uniform read with undefined uniform "
+			DRM_DEBUG("Uniform read with undefined uniform "
 				  "address\n");
 			return false;
 		}
@@ -660,19 +660,19 @@ vc4_validate_branches(struct vc4_shader_validation_state *validation_state)
 			continue;
 
 		if (ip - last_branch < 4) {
-			DRM_ERROR("Branch at %d during delay slots\n", ip);
+			DRM_DEBUG("Branch at %d during delay slots\n", ip);
 			return false;
 		}
 		last_branch = ip;
 
 		if (inst & QPU_BRANCH_REG) {
-			DRM_ERROR("branching from register relative "
+			DRM_DEBUG("branching from register relative "
 				  "not supported\n");
 			return false;
 		}
 
 		if (!(inst & QPU_BRANCH_REL)) {
-			DRM_ERROR("relative branching required\n");
+			DRM_DEBUG("relative branching required\n");
 			return false;
 		}
 
@@ -682,13 +682,13 @@ vc4_validate_branches(struct vc4_shader_validation_state *validation_state)
 		 * end of the shader object.
 		 */
 		if (branch_imm % sizeof(inst) != 0) {
-			DRM_ERROR("branch target not aligned\n");
+			DRM_DEBUG("branch target not aligned\n");
 			return false;
 		}
 
 		branch_target_ip = after_delay_ip + (branch_imm >> 3);
 		if (branch_target_ip >= validation_state->max_ip) {
-			DRM_ERROR("Branch at %d outside of shader (ip %d/%d)\n",
+			DRM_DEBUG("Branch at %d outside of shader (ip %d/%d)\n",
 				  ip, branch_target_ip,
 				  validation_state->max_ip);
 			return false;
@@ -699,7 +699,7 @@ vc4_validate_branches(struct vc4_shader_validation_state *validation_state)
 		 * the shader.
 		 */
 		if (after_delay_ip >= validation_state->max_ip) {
-			DRM_ERROR("Branch at %d continues past shader end "
+			DRM_DEBUG("Branch at %d continues past shader end "
 				  "(%d/%d)\n",
 				  ip, after_delay_ip, validation_state->max_ip);
 			return false;
@@ -709,7 +709,7 @@ vc4_validate_branches(struct vc4_shader_validation_state *validation_state)
 	}
 
 	if (max_branch_target > validation_state->max_ip - 3) {
-		DRM_ERROR("Branch landed after QPU_SIG_PROG_END");
+		DRM_DEBUG("Branch landed after QPU_SIG_PROG_END");
 		return false;
 	}
 
@@ -750,7 +750,7 @@ vc4_handle_branch_target(struct vc4_shader_validation_state *validation_state)
 		return true;
 
 	if (texturing_in_progress(validation_state)) {
-		DRM_ERROR("Branch target landed during TMU setup\n");
+		DRM_DEBUG("Branch target landed during TMU setup\n");
 		return false;
 	}
 
@@ -837,7 +837,7 @@ vc4_validate_shader(struct drm_gem_cma_object *shader_obj)
 		case QPU_SIG_LAST_THREAD_SWITCH:
 			if (!check_instruction_writes(validated_shader,
 						      &validation_state)) {
-				DRM_ERROR("Bad write at ip %d\n", ip);
+				DRM_DEBUG("Bad write at ip %d\n", ip);
 				goto fail;
 			}
 
@@ -855,7 +855,7 @@ vc4_validate_shader(struct drm_gem_cma_object *shader_obj)
 				validated_shader->is_threaded = true;
 
 				if (ip < last_thread_switch_ip + 3) {
-					DRM_ERROR("Thread switch too soon after "
+					DRM_DEBUG("Thread switch too soon after "
 						  "last switch at ip %d\n", ip);
 					goto fail;
 				}
@@ -867,7 +867,7 @@ vc4_validate_shader(struct drm_gem_cma_object *shader_obj)
 		case QPU_SIG_LOAD_IMM:
 			if (!check_instruction_writes(validated_shader,
 						      &validation_state)) {
-				DRM_ERROR("Bad LOAD_IMM write at ip %d\n", ip);
+				DRM_DEBUG("Bad LOAD_IMM write at ip %d\n", ip);
 				goto fail;
 			}
 			break;
@@ -878,14 +878,14 @@ vc4_validate_shader(struct drm_gem_cma_object *shader_obj)
 				goto fail;
 
 			if (ip < last_thread_switch_ip + 3) {
-				DRM_ERROR("Branch in thread switch at ip %d",
+				DRM_DEBUG("Branch in thread switch at ip %d",
 					  ip);
 				goto fail;
 			}
 
 			break;
 		default:
-			DRM_ERROR("Unsupported QPU signal %d at "
+			DRM_DEBUG("Unsupported QPU signal %d at "
 				  "instruction %d\n", sig, ip);
 			goto fail;
 		}
@@ -898,7 +898,7 @@ vc4_validate_shader(struct drm_gem_cma_object *shader_obj)
 	}
 
 	if (ip == validation_state.max_ip) {
-		DRM_ERROR("shader failed to terminate before "
+		DRM_DEBUG("shader failed to terminate before "
 			  "shader BO end at %zd\n",
 			  shader_obj->base.size);
 		goto fail;
@@ -907,7 +907,7 @@ vc4_validate_shader(struct drm_gem_cma_object *shader_obj)
 	/* Might corrupt other thread */
 	if (validated_shader->is_threaded &&
 	    validation_state.all_registers_used) {
-		DRM_ERROR("Shader uses threading, but uses the upper "
+		DRM_DEBUG("Shader uses threading, but uses the upper "
 			  "half of the registers, too\n");
 		goto fail;
 	}
diff --git a/drivers/gpu/drm/vc4/vc4_vec.c b/drivers/gpu/drm/vc4/vc4_vec.c
index 09c1e05765fa..3a9a302247a2 100644
--- a/drivers/gpu/drm/vc4/vc4_vec.c
+++ b/drivers/gpu/drm/vc4/vc4_vec.c
@@ -366,10 +366,8 @@ static int vc4_vec_connector_get_modes(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs vc4_vec_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.detect = vc4_vec_connector_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.set_property = drm_atomic_helper_connector_set_property,
 	.destroy = vc4_vec_connector_destroy,
 	.reset = drm_atomic_helper_connector_reset,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
diff --git a/drivers/gpu/drm/vgem/vgem_drv.c b/drivers/gpu/drm/vgem/vgem_drv.c
index 18f401b442c2..2524ff116f00 100644
--- a/drivers/gpu/drm/vgem/vgem_drv.c
+++ b/drivers/gpu/drm/vgem/vgem_drv.c
@@ -52,6 +52,7 @@ static void vgem_gem_free_object(struct drm_gem_object *obj)
 	struct drm_vgem_gem_object *vgem_obj = to_vgem_bo(obj);
 
 	kvfree(vgem_obj->pages);
+	mutex_destroy(&vgem_obj->pages_lock);
 
 	if (obj->import_attach)
 		drm_prime_gem_destroy(obj, vgem_obj->table);
@@ -76,11 +77,15 @@ static int vgem_gem_fault(struct vm_fault *vmf)
 	if (page_offset > num_pages)
 		return VM_FAULT_SIGBUS;
 
+	ret = -ENOENT;
+	mutex_lock(&obj->pages_lock);
 	if (obj->pages) {
 		get_page(obj->pages[page_offset]);
 		vmf->page = obj->pages[page_offset];
 		ret = 0;
-	} else {
+	}
+	mutex_unlock(&obj->pages_lock);
+	if (ret) {
 		struct page *page;
 
 		page = shmem_read_mapping_page(
@@ -161,6 +166,8 @@ static struct drm_vgem_gem_object *__vgem_gem_create(struct drm_device *dev,
 		return ERR_PTR(ret);
 	}
 
+	mutex_init(&obj->pages_lock);
+
 	return obj;
 }
 
@@ -183,7 +190,7 @@ static struct drm_gem_object *vgem_gem_create(struct drm_device *dev,
 		return ERR_CAST(obj);
 
 	ret = drm_gem_handle_create(file, &obj->base, handle);
-	drm_gem_object_unreference_unlocked(&obj->base);
+	drm_gem_object_put_unlocked(&obj->base);
 	if (ret)
 		goto err;
 
@@ -238,7 +245,7 @@ static int vgem_gem_dumb_map(struct drm_file *file, struct drm_device *dev,
 
 	*offset = drm_vma_node_offset_addr(&obj->vma_node);
 unref:
-	drm_gem_object_unreference_unlocked(obj);
+	drm_gem_object_put_unlocked(obj);
 
 	return ret;
 }
@@ -271,40 +278,70 @@ static const struct file_operations vgem_driver_fops = {
 	.poll		= drm_poll,
 	.read		= drm_read,
 	.unlocked_ioctl = drm_ioctl,
+	.compat_ioctl	= drm_compat_ioctl,
 	.release	= drm_release,
 };
 
+static struct page **vgem_pin_pages(struct drm_vgem_gem_object *bo)
+{
+	mutex_lock(&bo->pages_lock);
+	if (bo->pages_pin_count++ == 0) {
+		struct page **pages;
+
+		pages = drm_gem_get_pages(&bo->base);
+		if (IS_ERR(pages)) {
+			bo->pages_pin_count--;
+			mutex_unlock(&bo->pages_lock);
+			return pages;
+		}
+
+		bo->pages = pages;
+	}
+	mutex_unlock(&bo->pages_lock);
+
+	return bo->pages;
+}
+
+static void vgem_unpin_pages(struct drm_vgem_gem_object *bo)
+{
+	mutex_lock(&bo->pages_lock);
+	if (--bo->pages_pin_count == 0) {
+		drm_gem_put_pages(&bo->base, bo->pages, true, true);
+		bo->pages = NULL;
+	}
+	mutex_unlock(&bo->pages_lock);
+}
+
 static int vgem_prime_pin(struct drm_gem_object *obj)
 {
+	struct drm_vgem_gem_object *bo = to_vgem_bo(obj);
 	long n_pages = obj->size >> PAGE_SHIFT;
 	struct page **pages;
 
-	/* Flush the object from the CPU cache so that importers can rely
-	 * on coherent indirect access via the exported dma-address.
-	 */
-	pages = drm_gem_get_pages(obj);
+	pages = vgem_pin_pages(bo);
 	if (IS_ERR(pages))
 		return PTR_ERR(pages);
 
+	/* Flush the object from the CPU cache so that importers can rely
+	 * on coherent indirect access via the exported dma-address.
+	 */
 	drm_clflush_pages(pages, n_pages);
-	drm_gem_put_pages(obj, pages, true, false);
 
 	return 0;
 }
 
-static struct sg_table *vgem_prime_get_sg_table(struct drm_gem_object *obj)
+static void vgem_prime_unpin(struct drm_gem_object *obj)
 {
-	struct sg_table *st;
-	struct page **pages;
+	struct drm_vgem_gem_object *bo = to_vgem_bo(obj);
 
-	pages = drm_gem_get_pages(obj);
-	if (IS_ERR(pages))
-		return ERR_CAST(pages);
+	vgem_unpin_pages(bo);
+}
 
-	st = drm_prime_pages_to_sg(pages, obj->size >> PAGE_SHIFT);
-	drm_gem_put_pages(obj, pages, false, false);
+static struct sg_table *vgem_prime_get_sg_table(struct drm_gem_object *obj)
+{
+	struct drm_vgem_gem_object *bo = to_vgem_bo(obj);
 
-	return st;
+	return drm_prime_pages_to_sg(bo->pages, bo->base.size >> PAGE_SHIFT);
 }
 
 static struct drm_gem_object* vgem_prime_import(struct drm_device *dev,
@@ -333,6 +370,8 @@ static struct drm_gem_object *vgem_prime_import_sg_table(struct drm_device *dev,
 		__vgem_gem_destroy(obj);
 		return ERR_PTR(-ENOMEM);
 	}
+
+	obj->pages_pin_count++; /* perma-pinned */
 	drm_prime_sg_to_page_addr_arrays(obj->table, obj->pages, NULL,
 					npages);
 	return &obj->base;
@@ -340,23 +379,23 @@ static struct drm_gem_object *vgem_prime_import_sg_table(struct drm_device *dev,
 
 static void *vgem_prime_vmap(struct drm_gem_object *obj)
 {
+	struct drm_vgem_gem_object *bo = to_vgem_bo(obj);
 	long n_pages = obj->size >> PAGE_SHIFT;
 	struct page **pages;
-	void *addr;
 
-	pages = drm_gem_get_pages(obj);
+	pages = vgem_pin_pages(bo);
 	if (IS_ERR(pages))
 		return NULL;
 
-	addr = vmap(pages, n_pages, 0, pgprot_writecombine(PAGE_KERNEL));
-	drm_gem_put_pages(obj, pages, false, false);
-
-	return addr;
+	return vmap(pages, n_pages, 0, pgprot_writecombine(PAGE_KERNEL));
 }
 
 static void vgem_prime_vunmap(struct drm_gem_object *obj, void *vaddr)
 {
+	struct drm_vgem_gem_object *bo = to_vgem_bo(obj);
+
 	vunmap(vaddr);
+	vgem_unpin_pages(bo);
 }
 
 static int vgem_prime_mmap(struct drm_gem_object *obj,
@@ -409,6 +448,7 @@ static struct drm_driver vgem_driver = {
 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
 	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
 	.gem_prime_pin = vgem_prime_pin,
+	.gem_prime_unpin = vgem_prime_unpin,
 	.gem_prime_import = vgem_prime_import,
 	.gem_prime_export = drm_gem_prime_export,
 	.gem_prime_import_sg_table = vgem_prime_import_sg_table,
diff --git a/drivers/gpu/drm/vgem/vgem_drv.h b/drivers/gpu/drm/vgem/vgem_drv.h
index 1aae01419112..5c8f6d619ff3 100644
--- a/drivers/gpu/drm/vgem/vgem_drv.h
+++ b/drivers/gpu/drm/vgem/vgem_drv.h
@@ -43,7 +43,11 @@ struct vgem_file {
 #define to_vgem_bo(x) container_of(x, struct drm_vgem_gem_object, base)
 struct drm_vgem_gem_object {
 	struct drm_gem_object base;
+
 	struct page **pages;
+	unsigned int pages_pin_count;
+	struct mutex pages_lock;
+
 	struct sg_table *table;
 };
 
diff --git a/drivers/gpu/drm/vgem/vgem_fence.c b/drivers/gpu/drm/vgem/vgem_fence.c
index 3109c8308eb5..8fd52f211e9d 100644
--- a/drivers/gpu/drm/vgem/vgem_fence.c
+++ b/drivers/gpu/drm/vgem/vgem_fence.c
@@ -213,7 +213,7 @@ err_fence:
 		dma_fence_put(fence);
 	}
 err:
-	drm_gem_object_unreference_unlocked(obj);
+	drm_gem_object_put_unlocked(obj);
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/via/via_drv.c b/drivers/gpu/drm/via/via_drv.c
index 9e0e5392b6ec..aaf766f7cca2 100644
--- a/drivers/gpu/drm/via/via_drv.c
+++ b/drivers/gpu/drm/via/via_drv.c
@@ -77,7 +77,6 @@ static struct drm_driver driver = {
 	.open = via_driver_open,
 	.preclose = via_reclaim_buffers_locked,
 	.postclose = via_driver_postclose,
-	.set_busid = drm_pci_set_busid,
 	.context_dtor = via_final_context,
 	.get_vblank_counter = via_get_vblank_counter,
 	.enable_vblank = via_enable_vblank,
@@ -107,12 +106,12 @@ static int __init via_init(void)
 {
 	driver.num_ioctls = via_max_ioctl;
 	via_init_command_verifier();
-	return drm_pci_init(&driver, &via_pci_driver);
+	return drm_legacy_pci_init(&driver, &via_pci_driver);
 }
 
 static void __exit via_exit(void)
 {
-	drm_pci_exit(&driver, &via_pci_driver);
+	drm_legacy_pci_exit(&driver, &via_pci_driver);
 }
 
 module_init(via_init);
diff --git a/drivers/gpu/drm/virtio/virtgpu_display.c b/drivers/gpu/drm/virtio/virtgpu_display.c
index d51bd4521f17..b6d52055a11f 100644
--- a/drivers/gpu/drm/virtio/virtgpu_display.c
+++ b/drivers/gpu/drm/virtio/virtgpu_display.c
@@ -113,11 +113,13 @@ static void virtio_gpu_crtc_mode_set_nofb(struct drm_crtc *crtc)
 				   crtc->mode.vdisplay, 0, 0);
 }
 
-static void virtio_gpu_crtc_enable(struct drm_crtc *crtc)
+static void virtio_gpu_crtc_atomic_enable(struct drm_crtc *crtc,
+					  struct drm_crtc_state *old_state)
 {
 }
 
-static void virtio_gpu_crtc_disable(struct drm_crtc *crtc)
+static void virtio_gpu_crtc_atomic_disable(struct drm_crtc *crtc,
+					   struct drm_crtc_state *old_state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct virtio_gpu_device *vgdev = dev->dev_private;
@@ -145,11 +147,11 @@ static void virtio_gpu_crtc_atomic_flush(struct drm_crtc *crtc,
 }
 
 static const struct drm_crtc_helper_funcs virtio_gpu_crtc_helper_funcs = {
-	.enable        = virtio_gpu_crtc_enable,
-	.disable       = virtio_gpu_crtc_disable,
 	.mode_set_nofb = virtio_gpu_crtc_mode_set_nofb,
 	.atomic_check  = virtio_gpu_crtc_atomic_check,
 	.atomic_flush  = virtio_gpu_crtc_atomic_flush,
+	.atomic_enable = virtio_gpu_crtc_atomic_enable,
+	.atomic_disable = virtio_gpu_crtc_atomic_disable,
 };
 
 static void virtio_gpu_enc_mode_set(struct drm_encoder *encoder,
@@ -250,7 +252,6 @@ static void virtio_gpu_conn_destroy(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs virtio_gpu_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.detect = virtio_gpu_conn_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = virtio_gpu_conn_destroy,
diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.c b/drivers/gpu/drm/virtio/virtgpu_drv.c
index 63d35c7e416c..49a3d8d5a249 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drv.c
+++ b/drivers/gpu/drm/virtio/virtgpu_drv.c
@@ -122,7 +122,6 @@ static struct drm_driver driver = {
 
 	.dumb_create = virtio_gpu_mode_dumb_create,
 	.dumb_map_offset = virtio_gpu_mode_dumb_mmap,
-	.dumb_destroy = virtio_gpu_mode_dumb_destroy,
 
 #if defined(CONFIG_DEBUG_FS)
 	.debugfs_init = virtio_gpu_debugfs_init,
diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.h b/drivers/gpu/drm/virtio/virtgpu_drv.h
index 3a66abb8fd50..da2fb585fea4 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drv.h
+++ b/drivers/gpu/drm/virtio/virtgpu_drv.h
@@ -236,9 +236,6 @@ struct virtio_gpu_object *virtio_gpu_alloc_object(struct drm_device *dev,
 int virtio_gpu_mode_dumb_create(struct drm_file *file_priv,
 				struct drm_device *dev,
 				struct drm_mode_create_dumb *args);
-int virtio_gpu_mode_dumb_destroy(struct drm_file *file_priv,
-				 struct drm_device *dev,
-				 uint32_t handle);
 int virtio_gpu_mode_dumb_mmap(struct drm_file *file_priv,
 			      struct drm_device *dev,
 			      uint32_t handle, uint64_t *offset_p);
diff --git a/drivers/gpu/drm/virtio/virtgpu_fb.c b/drivers/gpu/drm/virtio/virtgpu_fb.c
index 33df067b11c1..15d18fd0c64b 100644
--- a/drivers/gpu/drm/virtio/virtgpu_fb.c
+++ b/drivers/gpu/drm/virtio/virtgpu_fb.c
@@ -273,7 +273,6 @@ static int virtio_gpufb_create(struct drm_fb_helper *helper,
 	vfbdev->helper.fb = fb;
 
 	strcpy(info->fix.id, "virtiodrmfb");
-	info->flags = FBINFO_DEFAULT;
 	info->fbops = &virtio_gpufb_ops;
 	info->pixmap.flags = FB_PIXMAP_SYSTEM;
 
@@ -309,7 +308,7 @@ static int virtio_gpu_fbdev_destroy(struct drm_device *dev,
 
 	return 0;
 }
-static struct drm_fb_helper_funcs virtio_gpu_fb_helper_funcs = {
+static const struct drm_fb_helper_funcs virtio_gpu_fb_helper_funcs = {
 	.fb_probe = virtio_gpufb_create,
 };
 
diff --git a/drivers/gpu/drm/virtio/virtgpu_gem.c b/drivers/gpu/drm/virtio/virtgpu_gem.c
index cc025d8fbe19..72ad7b103448 100644
--- a/drivers/gpu/drm/virtio/virtgpu_gem.c
+++ b/drivers/gpu/drm/virtio/virtgpu_gem.c
@@ -118,13 +118,6 @@ fail:
 	return ret;
 }
 
-int virtio_gpu_mode_dumb_destroy(struct drm_file *file_priv,
-				 struct drm_device *dev,
-				 uint32_t handle)
-{
-	return drm_gem_handle_delete(file_priv, handle);
-}
-
 int virtio_gpu_mode_dumb_mmap(struct drm_file *file_priv,
 			      struct drm_device *dev,
 			      uint32_t handle, uint64_t *offset_p)
diff --git a/drivers/gpu/drm/virtio/virtgpu_plane.c b/drivers/gpu/drm/virtio/virtgpu_plane.c
index adcdbd0abef6..71ba455af915 100644
--- a/drivers/gpu/drm/virtio/virtgpu_plane.c
+++ b/drivers/gpu/drm/virtio/virtgpu_plane.c
@@ -298,7 +298,7 @@ struct drm_plane *virtio_gpu_plane_init(struct virtio_gpu_device *vgdev,
 	ret = drm_universal_plane_init(dev, plane, 1 << index,
 				       &virtio_gpu_plane_funcs,
 				       formats, nformats,
-				       type, NULL);
+				       NULL, type, NULL);
 	if (ret)
 		goto err_plane_init;
 
diff --git a/drivers/gpu/drm/virtio/virtgpu_ttm.c b/drivers/gpu/drm/virtio/virtgpu_ttm.c
index c1f2af4ca4ca..cd389c5eaef5 100644
--- a/drivers/gpu/drm/virtio/virtgpu_ttm.c
+++ b/drivers/gpu/drm/virtio/virtgpu_ttm.c
@@ -192,7 +192,7 @@ static int ttm_bo_man_takedown(struct ttm_mem_type_manager *man)
 }
 
 static void ttm_bo_man_debug(struct ttm_mem_type_manager *man,
-			     const char *prefix)
+			     struct drm_printer *printer)
 {
 }
 
@@ -234,7 +234,7 @@ static int virtio_gpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
 static void virtio_gpu_evict_flags(struct ttm_buffer_object *bo,
 				struct ttm_placement *placement)
 {
-	static struct ttm_place placements = {
+	static const struct ttm_place placements = {
 		.fpfn  = 0,
 		.lpfn  = 0,
 		.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM,
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_cmdbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_cmdbuf.c
index 86178796de6c..c706ad30411b 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_cmdbuf.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_cmdbuf.c
@@ -51,6 +51,7 @@ struct vmw_cmdbuf_context {
 	struct list_head hw_submitted;
 	struct list_head preempted;
 	unsigned num_hw_submitted;
+	bool block_submission;
 };
 
 /**
@@ -60,6 +61,9 @@ struct vmw_cmdbuf_context {
  * kernel command submissions, @cur.
  * @space_mutex: Mutex to protect against starvation when we allocate
  * main pool buffer space.
+ * @error_mutex: Mutex to serialize the work queue error handling.
+ * Note this is not needed if the same workqueue handler
+ * can't race with itself...
  * @work: A struct work_struct implementeing command buffer error handling.
  * Immutable.
  * @dev_priv: Pointer to the device private struct. Immutable.
@@ -85,7 +89,6 @@ struct vmw_cmdbuf_context {
  * Internal protection.
  * @dheaders: Pool of DMA memory for device command buffer headers with trailing
  * space for inline data. Internal protection.
- * @tasklet: Tasklet struct for irq processing. Immutable.
  * @alloc_queue: Wait queue for processes waiting to allocate command buffer
  * space.
  * @idle_queue: Wait queue for processes waiting for command buffer idle.
@@ -102,6 +105,7 @@ struct vmw_cmdbuf_context {
 struct vmw_cmdbuf_man {
 	struct mutex cur_mutex;
 	struct mutex space_mutex;
+	struct mutex error_mutex;
 	struct work_struct work;
 	struct vmw_private *dev_priv;
 	struct vmw_cmdbuf_context ctx[SVGA_CB_CONTEXT_MAX];
@@ -117,7 +121,6 @@ struct vmw_cmdbuf_man {
 	spinlock_t lock;
 	struct dma_pool *headers;
 	struct dma_pool *dheaders;
-	struct tasklet_struct tasklet;
 	wait_queue_head_t alloc_queue;
 	wait_queue_head_t idle_queue;
 	bool irq_on;
@@ -181,12 +184,13 @@ struct vmw_cmdbuf_alloc_info {
 };
 
 /* Loop over each context in the command buffer manager. */
-#define for_each_cmdbuf_ctx(_man, _i, _ctx) \
+#define for_each_cmdbuf_ctx(_man, _i, _ctx)				\
 	for (_i = 0, _ctx = &(_man)->ctx[0]; (_i) < SVGA_CB_CONTEXT_MAX; \
 	     ++(_i), ++(_ctx))
 
-static int vmw_cmdbuf_startstop(struct vmw_cmdbuf_man *man, bool enable);
-
+static int vmw_cmdbuf_startstop(struct vmw_cmdbuf_man *man, u32 context,
+				bool enable);
+static int vmw_cmdbuf_preempt(struct vmw_cmdbuf_man *man, u32 context);
 
 /**
  * vmw_cmdbuf_cur_lock - Helper to lock the cur_mutex.
@@ -278,9 +282,9 @@ void vmw_cmdbuf_header_free(struct vmw_cmdbuf_header *header)
 		vmw_cmdbuf_header_inline_free(header);
 		return;
 	}
-	spin_lock_bh(&man->lock);
+	spin_lock(&man->lock);
 	__vmw_cmdbuf_header_free(header);
-	spin_unlock_bh(&man->lock);
+	spin_unlock(&man->lock);
 }
 
 
@@ -331,7 +335,8 @@ static void vmw_cmdbuf_ctx_submit(struct vmw_cmdbuf_man *man,
 				  struct vmw_cmdbuf_context *ctx)
 {
 	while (ctx->num_hw_submitted < man->max_hw_submitted &&
-	      !list_empty(&ctx->submitted)) {
+	       !list_empty(&ctx->submitted) &&
+	       !ctx->block_submission) {
 		struct vmw_cmdbuf_header *entry;
 		SVGACBStatus status;
 
@@ -386,12 +391,17 @@ static void vmw_cmdbuf_ctx_process(struct vmw_cmdbuf_man *man,
 			__vmw_cmdbuf_header_free(entry);
 			break;
 		case SVGA_CB_STATUS_COMMAND_ERROR:
-		case SVGA_CB_STATUS_CB_HEADER_ERROR:
+			entry->cb_header->status = SVGA_CB_STATUS_NONE;
 			list_add_tail(&entry->list, &man->error);
 			schedule_work(&man->work);
 			break;
 		case SVGA_CB_STATUS_PREEMPTED:
-			list_add(&entry->list, &ctx->preempted);
+			entry->cb_header->status = SVGA_CB_STATUS_NONE;
+			list_add_tail(&entry->list, &ctx->preempted);
+			break;
+		case SVGA_CB_STATUS_CB_HEADER_ERROR:
+			WARN_ONCE(true, "Command buffer header error.\n");
+			__vmw_cmdbuf_header_free(entry);
 			break;
 		default:
 			WARN_ONCE(true, "Undefined command buffer status.\n");
@@ -468,20 +478,17 @@ static void vmw_cmdbuf_ctx_add(struct vmw_cmdbuf_man *man,
 }
 
 /**
- * vmw_cmdbuf_man_tasklet - The main part of the command buffer interrupt
- * handler implemented as a tasklet.
+ * vmw_cmdbuf_irqthread - The main part of the command buffer interrupt
+ * handler implemented as a threaded irq task.
  *
- * @data: Tasklet closure. A pointer to the command buffer manager cast to
- * an unsigned long.
+ * @man: Pointer to the command buffer manager.
  *
- * The bottom half (tasklet) of the interrupt handler simply calls into the
+ * The bottom half of the interrupt handler simply calls into the
  * command buffer processor to free finished buffers and submit any
  * queued buffers to hardware.
  */
-static void vmw_cmdbuf_man_tasklet(unsigned long data)
+void vmw_cmdbuf_irqthread(struct vmw_cmdbuf_man *man)
 {
-	struct vmw_cmdbuf_man *man = (struct vmw_cmdbuf_man *) data;
-
 	spin_lock(&man->lock);
 	vmw_cmdbuf_man_process(man);
 	spin_unlock(&man->lock);
@@ -502,24 +509,112 @@ static void vmw_cmdbuf_work_func(struct work_struct *work)
 		container_of(work, struct vmw_cmdbuf_man, work);
 	struct vmw_cmdbuf_header *entry, *next;
 	uint32_t dummy;
-	bool restart = false;
+	bool restart[SVGA_CB_CONTEXT_MAX];
+	bool send_fence = false;
+	struct list_head restart_head[SVGA_CB_CONTEXT_MAX];
+	int i;
+	struct vmw_cmdbuf_context *ctx;
 
-	spin_lock_bh(&man->lock);
+	for_each_cmdbuf_ctx(man, i, ctx) {
+		INIT_LIST_HEAD(&restart_head[i]);
+		restart[i] = false;
+	}
+
+	mutex_lock(&man->error_mutex);
+	spin_lock(&man->lock);
 	list_for_each_entry_safe(entry, next, &man->error, list) {
-		restart = true;
-		DRM_ERROR("Command buffer error.\n");
+		SVGACBHeader *cb_hdr = entry->cb_header;
+		SVGA3dCmdHeader *header = (SVGA3dCmdHeader *)
+			(entry->cmd + cb_hdr->errorOffset);
+		u32 error_cmd_size, new_start_offset;
+		const char *cmd_name;
+
+		list_del_init(&entry->list);
+		restart[entry->cb_context] = true;
+
+		if (!vmw_cmd_describe(header, &error_cmd_size, &cmd_name)) {
+			DRM_ERROR("Unknown command causing device error.\n");
+			DRM_ERROR("Command buffer offset is %lu\n",
+				  (unsigned long) cb_hdr->errorOffset);
+			__vmw_cmdbuf_header_free(entry);
+			send_fence = true;
+			continue;
+		}
 
-		list_del(&entry->list);
-		__vmw_cmdbuf_header_free(entry);
-		wake_up_all(&man->idle_queue);
+		DRM_ERROR("Command \"%s\" causing device error.\n", cmd_name);
+		DRM_ERROR("Command buffer offset is %lu\n",
+			  (unsigned long) cb_hdr->errorOffset);
+		DRM_ERROR("Command size is %lu\n",
+			  (unsigned long) error_cmd_size);
+
+		new_start_offset = cb_hdr->errorOffset + error_cmd_size;
+
+		if (new_start_offset >= cb_hdr->length) {
+			__vmw_cmdbuf_header_free(entry);
+			send_fence = true;
+			continue;
+		}
+
+		if (man->using_mob)
+			cb_hdr->ptr.mob.mobOffset += new_start_offset;
+		else
+			cb_hdr->ptr.pa += (u64) new_start_offset;
+
+		entry->cmd += new_start_offset;
+		cb_hdr->length -= new_start_offset;
+		cb_hdr->errorOffset = 0;
+		cb_hdr->offset = 0;
+		list_add_tail(&entry->list, &restart_head[entry->cb_context]);
+		man->ctx[entry->cb_context].block_submission = true;
+	}
+	spin_unlock(&man->lock);
+
+	/* Preempt all contexts with errors */
+	for_each_cmdbuf_ctx(man, i, ctx) {
+		if (ctx->block_submission && vmw_cmdbuf_preempt(man, i))
+			DRM_ERROR("Failed preempting command buffer "
+				  "context %u.\n", i);
+	}
+
+	spin_lock(&man->lock);
+	for_each_cmdbuf_ctx(man, i, ctx) {
+		if (!ctx->block_submission)
+			continue;
+
+		/* Move preempted command buffers to the preempted queue. */
+		vmw_cmdbuf_ctx_process(man, ctx, &dummy);
+
+		/*
+		 * Add the preempted queue after the command buffer
+		 * that caused an error.
+		 */
+		list_splice_init(&ctx->preempted, restart_head[i].prev);
+
+		/*
+		 * Finally add all command buffers first in the submitted
+		 * queue, to rerun them.
+		 */
+		list_splice_init(&restart_head[i], &ctx->submitted);
+
+		ctx->block_submission = false;
 	}
-	spin_unlock_bh(&man->lock);
 
-	if (restart && vmw_cmdbuf_startstop(man, true))
-		DRM_ERROR("Failed restarting command buffer context 0.\n");
+	vmw_cmdbuf_man_process(man);
+	spin_unlock(&man->lock);
+
+	for_each_cmdbuf_ctx(man, i, ctx) {
+		if (restart[i] && vmw_cmdbuf_startstop(man, i, true))
+			DRM_ERROR("Failed restarting command buffer "
+				  "context %u.\n", i);
+	}
 
 	/* Send a new fence in case one was removed */
-	vmw_fifo_send_fence(man->dev_priv, &dummy);
+	if (send_fence) {
+		vmw_fifo_send_fence(man->dev_priv, &dummy);
+		wake_up_all(&man->idle_queue);
+	}
+
+	mutex_unlock(&man->error_mutex);
 }
 
 /**
@@ -536,7 +631,7 @@ static bool vmw_cmdbuf_man_idle(struct vmw_cmdbuf_man *man,
 	bool idle = false;
 	int i;
 
-	spin_lock_bh(&man->lock);
+	spin_lock(&man->lock);
 	vmw_cmdbuf_man_process(man);
 	for_each_cmdbuf_ctx(man, i, ctx) {
 		if (!list_empty(&ctx->submitted) ||
@@ -548,7 +643,7 @@ static bool vmw_cmdbuf_man_idle(struct vmw_cmdbuf_man *man,
 	idle = list_empty(&man->error);
 
 out_unlock:
-	spin_unlock_bh(&man->lock);
+	spin_unlock(&man->lock);
 
 	return idle;
 }
@@ -571,7 +666,7 @@ static void __vmw_cmdbuf_cur_flush(struct vmw_cmdbuf_man *man)
 	if (!cur)
 		return;
 
-	spin_lock_bh(&man->lock);
+	spin_lock(&man->lock);
 	if (man->cur_pos == 0) {
 		__vmw_cmdbuf_header_free(cur);
 		goto out_unlock;
@@ -580,7 +675,7 @@ static void __vmw_cmdbuf_cur_flush(struct vmw_cmdbuf_man *man)
 	man->cur->cb_header->length = man->cur_pos;
 	vmw_cmdbuf_ctx_add(man, man->cur, SVGA_CB_CONTEXT_0);
 out_unlock:
-	spin_unlock_bh(&man->lock);
+	spin_unlock(&man->lock);
 	man->cur = NULL;
 	man->cur_pos = 0;
 }
@@ -673,14 +768,14 @@ static bool vmw_cmdbuf_try_alloc(struct vmw_cmdbuf_man *man,
 		return true;
  
 	memset(info->node, 0, sizeof(*info->node));
-	spin_lock_bh(&man->lock);
+	spin_lock(&man->lock);
 	ret = drm_mm_insert_node(&man->mm, info->node, info->page_size);
 	if (ret) {
 		vmw_cmdbuf_man_process(man);
 		ret = drm_mm_insert_node(&man->mm, info->node, info->page_size);
 	}
 
-	spin_unlock_bh(&man->lock);
+	spin_unlock(&man->lock);
 	info->done = !ret;
 
 	return info->done;
@@ -801,9 +896,9 @@ static int vmw_cmdbuf_space_pool(struct vmw_cmdbuf_man *man,
 	return 0;
 
 out_no_cb_header:
-	spin_lock_bh(&man->lock);
+	spin_lock(&man->lock);
 	drm_mm_remove_node(&header->node);
-	spin_unlock_bh(&man->lock);
+	spin_unlock(&man->lock);
 
 	return ret;
 }
@@ -1023,18 +1118,6 @@ void vmw_cmdbuf_commit(struct vmw_cmdbuf_man *man, size_t size,
 	vmw_cmdbuf_cur_unlock(man);
 }
 
-/**
- * vmw_cmdbuf_tasklet_schedule - Schedule the interrupt handler bottom half.
- *
- * @man: The command buffer manager.
- */
-void vmw_cmdbuf_tasklet_schedule(struct vmw_cmdbuf_man *man)
-{
-	if (!man)
-		return;
-
-	tasklet_schedule(&man->tasklet);
-}
 
 /**
  * vmw_cmdbuf_send_device_command - Send a command through the device context.
@@ -1059,9 +1142,9 @@ static int vmw_cmdbuf_send_device_command(struct vmw_cmdbuf_man *man,
 	memcpy(cmd, command, size);
 	header->cb_header->length = size;
 	header->cb_context = SVGA_CB_CONTEXT_DEVICE;
-	spin_lock_bh(&man->lock);
+	spin_lock(&man->lock);
 	status = vmw_cmdbuf_header_submit(header);
-	spin_unlock_bh(&man->lock);
+	spin_unlock(&man->lock);
 	vmw_cmdbuf_header_free(header);
 
 	if (status != SVGA_CB_STATUS_COMPLETED) {
@@ -1074,6 +1157,29 @@ static int vmw_cmdbuf_send_device_command(struct vmw_cmdbuf_man *man,
 }
 
 /**
+ * vmw_cmdbuf_preempt - Send a preempt command through the device
+ * context.
+ *
+ * @man: The command buffer manager.
+ *
+ * Synchronously sends a preempt command.
+ */
+static int vmw_cmdbuf_preempt(struct vmw_cmdbuf_man *man, u32 context)
+{
+	struct {
+		uint32 id;
+		SVGADCCmdPreempt body;
+	} __packed cmd;
+
+	cmd.id = SVGA_DC_CMD_PREEMPT;
+	cmd.body.context = SVGA_CB_CONTEXT_0 + context;
+	cmd.body.ignoreIDZero = 0;
+
+	return vmw_cmdbuf_send_device_command(man, &cmd, sizeof(cmd));
+}
+
+
+/**
  * vmw_cmdbuf_startstop - Send a start / stop command through the device
  * context.
  *
@@ -1082,7 +1188,7 @@ static int vmw_cmdbuf_send_device_command(struct vmw_cmdbuf_man *man,
  *
  * Synchronously sends a device start / stop context command.
  */
-static int vmw_cmdbuf_startstop(struct vmw_cmdbuf_man *man,
+static int vmw_cmdbuf_startstop(struct vmw_cmdbuf_man *man, u32 context,
 				bool enable)
 {
 	struct {
@@ -1092,7 +1198,7 @@ static int vmw_cmdbuf_startstop(struct vmw_cmdbuf_man *man,
 
 	cmd.id = SVGA_DC_CMD_START_STOP_CONTEXT;
 	cmd.body.enable = (enable) ? 1 : 0;
-	cmd.body.context = SVGA_CB_CONTEXT_0;
+	cmd.body.context = SVGA_CB_CONTEXT_0 + context;
 
 	return vmw_cmdbuf_send_device_command(man, &cmd, sizeof(cmd));
 }
@@ -1191,7 +1297,7 @@ struct vmw_cmdbuf_man *vmw_cmdbuf_man_create(struct vmw_private *dev_priv)
 {
 	struct vmw_cmdbuf_man *man;
 	struct vmw_cmdbuf_context *ctx;
-	int i;
+	unsigned int i;
 	int ret;
 
 	if (!(dev_priv->capabilities & SVGA_CAP_COMMAND_BUFFERS))
@@ -1226,8 +1332,7 @@ struct vmw_cmdbuf_man *vmw_cmdbuf_man_create(struct vmw_private *dev_priv)
 	spin_lock_init(&man->lock);
 	mutex_init(&man->cur_mutex);
 	mutex_init(&man->space_mutex);
-	tasklet_init(&man->tasklet, vmw_cmdbuf_man_tasklet,
-		     (unsigned long) man);
+	mutex_init(&man->error_mutex);
 	man->default_size = VMW_CMDBUF_INLINE_SIZE;
 	init_waitqueue_head(&man->alloc_queue);
 	init_waitqueue_head(&man->idle_queue);
@@ -1236,11 +1341,14 @@ struct vmw_cmdbuf_man *vmw_cmdbuf_man_create(struct vmw_private *dev_priv)
 	INIT_WORK(&man->work, &vmw_cmdbuf_work_func);
 	vmw_generic_waiter_add(dev_priv, SVGA_IRQFLAG_ERROR,
 			       &dev_priv->error_waiters);
-	ret = vmw_cmdbuf_startstop(man, true);
-	if (ret) {
-		DRM_ERROR("Failed starting command buffer context 0.\n");
-		vmw_cmdbuf_man_destroy(man);
-		return ERR_PTR(ret);
+	for_each_cmdbuf_ctx(man, i, ctx) {
+		ret = vmw_cmdbuf_startstop(man, i, true);
+		if (ret) {
+			DRM_ERROR("Failed starting command buffer "
+				  "context %u.\n", i);
+			vmw_cmdbuf_man_destroy(man);
+			return ERR_PTR(ret);
+		}
 	}
 
 	return man;
@@ -1290,18 +1398,24 @@ void vmw_cmdbuf_remove_pool(struct vmw_cmdbuf_man *man)
  */
 void vmw_cmdbuf_man_destroy(struct vmw_cmdbuf_man *man)
 {
+	struct vmw_cmdbuf_context *ctx;
+	unsigned int i;
+
 	WARN_ON_ONCE(man->has_pool);
 	(void) vmw_cmdbuf_idle(man, false, 10*HZ);
-	if (vmw_cmdbuf_startstop(man, false))
-		DRM_ERROR("Failed stopping command buffer context 0.\n");
+
+	for_each_cmdbuf_ctx(man, i, ctx)
+		if (vmw_cmdbuf_startstop(man, i, false))
+			DRM_ERROR("Failed stopping command buffer "
+				  "context %u.\n", i);
 
 	vmw_generic_waiter_remove(man->dev_priv, SVGA_IRQFLAG_ERROR,
 				  &man->dev_priv->error_waiters);
-	tasklet_kill(&man->tasklet);
 	(void) cancel_work_sync(&man->work);
 	dma_pool_destroy(man->dheaders);
 	dma_pool_destroy(man->headers);
 	mutex_destroy(&man->cur_mutex);
 	mutex_destroy(&man->space_mutex);
+	mutex_destroy(&man->error_mutex);
 	kfree(man);
 }
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
index 4436d53ae16c..e84fee3ec4f3 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
@@ -36,7 +36,6 @@
 #include <drm/ttm/ttm_module.h>
 #include <linux/dma_remapping.h>
 
-#define VMWGFX_DRIVER_NAME "vmwgfx"
 #define VMWGFX_DRIVER_DESC "Linux drm driver for VMware graphics devices"
 #define VMWGFX_CHIP_SVGAII 0
 #define VMW_FB_RESERVATION 0
@@ -825,7 +824,7 @@ static int vmw_driver_load(struct drm_device *dev, unsigned long chipset)
 	}
 
 	if (dev_priv->capabilities & SVGA_CAP_IRQMASK) {
-		ret = drm_irq_install(dev, dev->pdev->irq);
+		ret = vmw_irq_install(dev, dev->pdev->irq);
 		if (ret != 0) {
 			DRM_ERROR("Failed installing irq: %d\n", ret);
 			goto out_no_irq;
@@ -937,7 +936,7 @@ out_no_bdev:
 	vmw_fence_manager_takedown(dev_priv->fman);
 out_no_fman:
 	if (dev_priv->capabilities & SVGA_CAP_IRQMASK)
-		drm_irq_uninstall(dev_priv->dev);
+		vmw_irq_uninstall(dev_priv->dev);
 out_no_irq:
 	if (dev_priv->stealth)
 		pci_release_region(dev->pdev, 2);
@@ -990,7 +989,7 @@ static void vmw_driver_unload(struct drm_device *dev)
 	vmw_release_device_late(dev_priv);
 	vmw_fence_manager_takedown(dev_priv->fman);
 	if (dev_priv->capabilities & SVGA_CAP_IRQMASK)
-		drm_irq_uninstall(dev_priv->dev);
+		vmw_irq_uninstall(dev_priv->dev);
 	if (dev_priv->stealth)
 		pci_release_region(dev->pdev, 2);
 	else
@@ -1516,10 +1515,6 @@ static struct drm_driver driver = {
 	.load = vmw_driver_load,
 	.unload = vmw_driver_unload,
 	.lastclose = vmw_lastclose,
-	.irq_preinstall = vmw_irq_preinstall,
-	.irq_postinstall = vmw_irq_postinstall,
-	.irq_uninstall = vmw_irq_uninstall,
-	.irq_handler = vmw_irq_handler,
 	.get_vblank_counter = vmw_get_vblank_counter,
 	.enable_vblank = vmw_enable_vblank,
 	.disable_vblank = vmw_disable_vblank,
@@ -1531,7 +1526,6 @@ static struct drm_driver driver = {
 	.master_drop = vmw_master_drop,
 	.open = vmw_driver_open,
 	.postclose = vmw_postclose,
-	.set_busid = drm_pci_set_busid,
 
 	.dumb_create = vmw_dumb_create,
 	.dumb_map_offset = vmw_dumb_map_offset,
@@ -1571,7 +1565,7 @@ static int __init vmwgfx_init(void)
 	if (vgacon_text_force())
 		return -EINVAL;
 
-	ret = drm_pci_init(&driver, &vmw_pci_driver);
+	ret = pci_register_driver(&vmw_pci_driver);
 	if (ret)
 		DRM_ERROR("Failed initializing DRM.\n");
 	return ret;
@@ -1579,7 +1573,7 @@ static int __init vmwgfx_init(void)
 
 static void __exit vmwgfx_exit(void)
 {
-	drm_pci_exit(&driver, &vmw_pci_driver);
+	pci_unregister_driver(&vmw_pci_driver);
 }
 
 module_init(vmwgfx_init);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
index 4b948fba9eec..7e5f30e234b1 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
@@ -40,10 +40,12 @@
 #include <drm/ttm/ttm_execbuf_util.h>
 #include <drm/ttm/ttm_module.h>
 #include "vmwgfx_fence.h"
+#include <linux/sync_file.h>
 
-#define VMWGFX_DRIVER_DATE "20170607"
+#define VMWGFX_DRIVER_NAME "vmwgfx"
+#define VMWGFX_DRIVER_DATE "20170612"
 #define VMWGFX_DRIVER_MAJOR 2
-#define VMWGFX_DRIVER_MINOR 13
+#define VMWGFX_DRIVER_MINOR 14
 #define VMWGFX_DRIVER_PATCHLEVEL 0
 #define VMWGFX_FILE_PAGE_OFFSET 0x00100000
 #define VMWGFX_FIFO_STATIC_SIZE (1024*1024)
@@ -351,6 +353,12 @@ struct vmw_otable_batch {
 	struct ttm_buffer_object *otable_bo;
 };
 
+enum {
+	VMW_IRQTHREAD_FENCE,
+	VMW_IRQTHREAD_CMDBUF,
+	VMW_IRQTHREAD_MAX
+};
+
 struct vmw_private {
 	struct ttm_bo_device bdev;
 	struct ttm_bo_global_ref bo_global_ref;
@@ -529,6 +537,7 @@ struct vmw_private {
 	struct vmw_otable_batch otable_batch;
 
 	struct vmw_cmdbuf_man *cman;
+	DECLARE_BITMAP(irqthread_pending, VMW_IRQTHREAD_MAX);
 };
 
 static inline struct vmw_surface *vmw_res_to_srf(struct vmw_resource *res)
@@ -561,24 +570,21 @@ static inline struct vmw_master *vmw_master(struct drm_master *master)
 static inline void vmw_write(struct vmw_private *dev_priv,
 			     unsigned int offset, uint32_t value)
 {
-	unsigned long irq_flags;
-
-	spin_lock_irqsave(&dev_priv->hw_lock, irq_flags);
+	spin_lock(&dev_priv->hw_lock);
 	outl(offset, dev_priv->io_start + VMWGFX_INDEX_PORT);
 	outl(value, dev_priv->io_start + VMWGFX_VALUE_PORT);
-	spin_unlock_irqrestore(&dev_priv->hw_lock, irq_flags);
+	spin_unlock(&dev_priv->hw_lock);
 }
 
 static inline uint32_t vmw_read(struct vmw_private *dev_priv,
 				unsigned int offset)
 {
-	unsigned long irq_flags;
 	u32 val;
 
-	spin_lock_irqsave(&dev_priv->hw_lock, irq_flags);
+	spin_lock(&dev_priv->hw_lock);
 	outl(offset, dev_priv->io_start + VMWGFX_INDEX_PORT);
 	val = inl(dev_priv->io_start + VMWGFX_VALUE_PORT);
-	spin_unlock_irqrestore(&dev_priv->hw_lock, irq_flags);
+	spin_unlock(&dev_priv->hw_lock);
 
 	return val;
 }
@@ -821,7 +827,8 @@ extern int vmw_execbuf_process(struct drm_file *file_priv,
 			       uint32_t dx_context_handle,
 			       struct drm_vmw_fence_rep __user
 			       *user_fence_rep,
-			       struct vmw_fence_obj **out_fence);
+			       struct vmw_fence_obj **out_fence,
+			       uint32_t flags);
 extern void __vmw_execbuf_release_pinned_bo(struct vmw_private *dev_priv,
 					    struct vmw_fence_obj *fence);
 extern void vmw_execbuf_release_pinned_bo(struct vmw_private *dev_priv);
@@ -836,23 +843,23 @@ extern void vmw_execbuf_copy_fence_user(struct vmw_private *dev_priv,
 					struct drm_vmw_fence_rep __user
 					*user_fence_rep,
 					struct vmw_fence_obj *fence,
-					uint32_t fence_handle);
+					uint32_t fence_handle,
+					int32_t out_fence_fd,
+					struct sync_file *sync_file);
 extern int vmw_validate_single_buffer(struct vmw_private *dev_priv,
 				      struct ttm_buffer_object *bo,
 				      bool interruptible,
 				      bool validate_as_mob);
-
+bool vmw_cmd_describe(const void *buf, u32 *size, char const **cmd);
 
 /**
  * IRQs and wating - vmwgfx_irq.c
  */
 
-extern irqreturn_t vmw_irq_handler(int irq, void *arg);
 extern int vmw_wait_seqno(struct vmw_private *dev_priv, bool lazy,
 			  uint32_t seqno, bool interruptible,
 			  unsigned long timeout);
-extern void vmw_irq_preinstall(struct drm_device *dev);
-extern int vmw_irq_postinstall(struct drm_device *dev);
+extern int vmw_irq_install(struct drm_device *dev, int irq);
 extern void vmw_irq_uninstall(struct drm_device *dev);
 extern bool vmw_seqno_passed(struct vmw_private *dev_priv,
 				uint32_t seqno);
@@ -1150,13 +1157,13 @@ extern void *vmw_cmdbuf_reserve(struct vmw_cmdbuf_man *man, size_t size,
 extern void vmw_cmdbuf_commit(struct vmw_cmdbuf_man *man, size_t size,
 			      struct vmw_cmdbuf_header *header,
 			      bool flush);
-extern void vmw_cmdbuf_tasklet_schedule(struct vmw_cmdbuf_man *man);
 extern void *vmw_cmdbuf_alloc(struct vmw_cmdbuf_man *man,
 			      size_t size, bool interruptible,
 			      struct vmw_cmdbuf_header **p_header);
 extern void vmw_cmdbuf_header_free(struct vmw_cmdbuf_header *header);
 extern int vmw_cmdbuf_cur_flush(struct vmw_cmdbuf_man *man,
 				bool interruptible);
+extern void vmw_cmdbuf_irqthread(struct vmw_cmdbuf_man *man);
 
 
 /**
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
index 2cfb3c93f42a..21c62a34e558 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
@@ -24,6 +24,7 @@
  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  *
  **************************************************************************/
+#include <linux/sync_file.h>
 
 #include "vmwgfx_drv.h"
 #include "vmwgfx_reg.h"
@@ -112,11 +113,12 @@ struct vmw_cmd_entry {
 	bool user_allow;
 	bool gb_disable;
 	bool gb_enable;
+	const char *cmd_name;
 };
 
 #define VMW_CMD_DEF(_cmd, _func, _user_allow, _gb_disable, _gb_enable)	\
 	[(_cmd) - SVGA_3D_CMD_BASE] = {(_func), (_user_allow),\
-				       (_gb_disable), (_gb_enable)}
+				       (_gb_disable), (_gb_enable), #_cmd}
 
 static int vmw_resource_context_res_add(struct vmw_private *dev_priv,
 					struct vmw_sw_context *sw_context,
@@ -3302,6 +3304,8 @@ static const struct vmw_cmd_entry vmw_cmd_entries[SVGA_3D_CMD_MAX] = {
 		    true, false, true),
 	VMW_CMD_DEF(SVGA_3D_CMD_NOP, &vmw_cmd_ok,
 		    true, false, true),
+	VMW_CMD_DEF(SVGA_3D_CMD_NOP_ERROR, &vmw_cmd_ok,
+		    true, false, true),
 	VMW_CMD_DEF(SVGA_3D_CMD_ENABLE_GART, &vmw_cmd_invalid,
 		    false, false, true),
 	VMW_CMD_DEF(SVGA_3D_CMD_DISABLE_GART, &vmw_cmd_invalid,
@@ -3469,6 +3473,51 @@ static const struct vmw_cmd_entry vmw_cmd_entries[SVGA_3D_CMD_MAX] = {
 		    true, false, true),
 };
 
+bool vmw_cmd_describe(const void *buf, u32 *size, char const **cmd)
+{
+	u32 cmd_id = ((u32 *) buf)[0];
+
+	if (cmd_id >= SVGA_CMD_MAX) {
+		SVGA3dCmdHeader *header = (SVGA3dCmdHeader *) buf;
+		const struct vmw_cmd_entry *entry;
+
+		*size = header->size + sizeof(SVGA3dCmdHeader);
+		cmd_id = header->id;
+		if (cmd_id >= SVGA_3D_CMD_MAX)
+			return false;
+
+		cmd_id -= SVGA_3D_CMD_BASE;
+		entry = &vmw_cmd_entries[cmd_id];
+		*cmd = entry->cmd_name;
+		return true;
+	}
+
+	switch (cmd_id) {
+	case SVGA_CMD_UPDATE:
+		*cmd = "SVGA_CMD_UPDATE";
+		*size = sizeof(u32) + sizeof(SVGAFifoCmdUpdate);
+		break;
+	case SVGA_CMD_DEFINE_GMRFB:
+		*cmd = "SVGA_CMD_DEFINE_GMRFB";
+		*size = sizeof(u32) + sizeof(SVGAFifoCmdDefineGMRFB);
+		break;
+	case SVGA_CMD_BLIT_GMRFB_TO_SCREEN:
+		*cmd = "SVGA_CMD_BLIT_GMRFB_TO_SCREEN";
+		*size = sizeof(u32) + sizeof(SVGAFifoCmdBlitGMRFBToScreen);
+		break;
+	case SVGA_CMD_BLIT_SCREEN_TO_GMRFB:
+		*cmd = "SVGA_CMD_BLIT_SCREEN_TO_GMRFB";
+		*size = sizeof(u32) + sizeof(SVGAFifoCmdBlitGMRFBToScreen);
+		break;
+	default:
+		*cmd = "UNKNOWN";
+		*size = 0;
+		return false;
+	}
+
+	return true;
+}
+
 static int vmw_cmd_check(struct vmw_private *dev_priv,
 			 struct vmw_sw_context *sw_context,
 			 void *buf, uint32_t *size)
@@ -3781,6 +3830,8 @@ int vmw_execbuf_fence_commands(struct drm_file *file_priv,
  * which the information should be copied.
  * @fence: Pointer to the fenc object.
  * @fence_handle: User-space fence handle.
+ * @out_fence_fd: exported file descriptor for the fence.  -1 if not used
+ * @sync_file:  Only used to clean up in case of an error in this function.
  *
  * This function copies fence information to user-space. If copying fails,
  * The user-space struct drm_vmw_fence_rep::error member is hopefully
@@ -3796,7 +3847,9 @@ vmw_execbuf_copy_fence_user(struct vmw_private *dev_priv,
 			    int ret,
 			    struct drm_vmw_fence_rep __user *user_fence_rep,
 			    struct vmw_fence_obj *fence,
-			    uint32_t fence_handle)
+			    uint32_t fence_handle,
+			    int32_t out_fence_fd,
+			    struct sync_file *sync_file)
 {
 	struct drm_vmw_fence_rep fence_rep;
 
@@ -3806,6 +3859,7 @@ vmw_execbuf_copy_fence_user(struct vmw_private *dev_priv,
 	memset(&fence_rep, 0, sizeof(fence_rep));
 
 	fence_rep.error = ret;
+	fence_rep.fd = out_fence_fd;
 	if (ret == 0) {
 		BUG_ON(fence == NULL);
 
@@ -3828,6 +3882,14 @@ vmw_execbuf_copy_fence_user(struct vmw_private *dev_priv,
 	 * and unreference the handle.
 	 */
 	if (unlikely(ret != 0) && (fence_rep.error == 0)) {
+		if (sync_file)
+			fput(sync_file->file);
+
+		if (fence_rep.fd != -1) {
+			put_unused_fd(fence_rep.fd);
+			fence_rep.fd = -1;
+		}
+
 		ttm_ref_object_base_unref(vmw_fp->tfile,
 					  fence_handle, TTM_REF_USAGE);
 		DRM_ERROR("Fence copy error. Syncing.\n");
@@ -4003,7 +4065,8 @@ int vmw_execbuf_process(struct drm_file *file_priv,
 			uint64_t throttle_us,
 			uint32_t dx_context_handle,
 			struct drm_vmw_fence_rep __user *user_fence_rep,
-			struct vmw_fence_obj **out_fence)
+			struct vmw_fence_obj **out_fence,
+			uint32_t flags)
 {
 	struct vmw_sw_context *sw_context = &dev_priv->ctx;
 	struct vmw_fence_obj *fence = NULL;
@@ -4013,20 +4076,33 @@ int vmw_execbuf_process(struct drm_file *file_priv,
 	struct ww_acquire_ctx ticket;
 	uint32_t handle;
 	int ret;
+	int32_t out_fence_fd = -1;
+	struct sync_file *sync_file = NULL;
+
+
+	if (flags & DRM_VMW_EXECBUF_FLAG_EXPORT_FENCE_FD) {
+		out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
+		if (out_fence_fd < 0) {
+			DRM_ERROR("Failed to get a fence file descriptor.\n");
+			return out_fence_fd;
+		}
+	}
 
 	if (throttle_us) {
 		ret = vmw_wait_lag(dev_priv, &dev_priv->fifo.marker_queue,
 				   throttle_us);
 
 		if (ret)
-			return ret;
+			goto out_free_fence_fd;
 	}
 
 	kernel_commands = vmw_execbuf_cmdbuf(dev_priv, user_commands,
 					     kernel_commands, command_size,
 					     &header);
-	if (IS_ERR(kernel_commands))
-		return PTR_ERR(kernel_commands);
+	if (IS_ERR(kernel_commands)) {
+		ret = PTR_ERR(kernel_commands);
+		goto out_free_fence_fd;
+	}
 
 	ret = mutex_lock_interruptible(&dev_priv->cmdbuf_mutex);
 	if (ret) {
@@ -4162,8 +4238,32 @@ int vmw_execbuf_process(struct drm_file *file_priv,
 		__vmw_execbuf_release_pinned_bo(dev_priv, fence);
 
 	vmw_clear_validations(sw_context);
+
+	/*
+	 * If anything fails here, give up trying to export the fence
+	 * and do a sync since the user mode will not be able to sync
+	 * the fence itself.  This ensures we are still functionally
+	 * correct.
+	 */
+	if (flags & DRM_VMW_EXECBUF_FLAG_EXPORT_FENCE_FD) {
+
+		sync_file = sync_file_create(&fence->base);
+		if (!sync_file) {
+			DRM_ERROR("Unable to create sync file for fence\n");
+			put_unused_fd(out_fence_fd);
+			out_fence_fd = -1;
+
+			(void) vmw_fence_obj_wait(fence, false, false,
+						  VMW_FENCE_WAIT_TIMEOUT);
+		} else {
+			/* Link the fence with the FD created earlier */
+			fd_install(out_fence_fd, sync_file->file);
+		}
+	}
+
 	vmw_execbuf_copy_fence_user(dev_priv, vmw_fpriv(file_priv), ret,
-				    user_fence_rep, fence, handle);
+				    user_fence_rep, fence, handle,
+				    out_fence_fd, sync_file);
 
 	/* Don't unreference when handing fence out */
 	if (unlikely(out_fence != NULL)) {
@@ -4214,6 +4314,9 @@ out_unlock:
 out_free_header:
 	if (header)
 		vmw_cmdbuf_header_free(header);
+out_free_fence_fd:
+	if (out_fence_fd >= 0)
+		put_unused_fd(out_fence_fd);
 
 	return ret;
 }
@@ -4366,6 +4469,7 @@ int vmw_execbuf_ioctl(struct drm_device *dev, unsigned long data,
 	static const size_t copy_offset[] = {
 		offsetof(struct drm_vmw_execbuf_arg, context_handle),
 		sizeof(struct drm_vmw_execbuf_arg)};
+	struct dma_fence *in_fence = NULL;
 
 	if (unlikely(size < copy_offset[0])) {
 		DRM_ERROR("Invalid command size, ioctl %d\n",
@@ -4401,15 +4505,25 @@ int vmw_execbuf_ioctl(struct drm_device *dev, unsigned long data,
 		arg.context_handle = (uint32_t) -1;
 		break;
 	case 2:
-		if (arg.pad64 != 0) {
-			DRM_ERROR("Unused IOCTL data not set to zero.\n");
-			return -EINVAL;
-		}
-		break;
 	default:
 		break;
 	}
 
+
+	/* If imported a fence FD from elsewhere, then wait on it */
+	if (arg.flags & DRM_VMW_EXECBUF_FLAG_IMPORT_FENCE_FD) {
+		in_fence = sync_file_get_fence(arg.imported_fence_fd);
+
+		if (!in_fence) {
+			DRM_ERROR("Cannot get imported fence\n");
+			return -EINVAL;
+		}
+
+		ret = vmw_wait_dma_fence(dev_priv->fman, in_fence);
+		if (ret)
+			goto out;
+	}
+
 	ret = ttm_read_lock(&dev_priv->reservation_sem, true);
 	if (unlikely(ret != 0))
 		return ret;
@@ -4419,12 +4533,16 @@ int vmw_execbuf_ioctl(struct drm_device *dev, unsigned long data,
 				  NULL, arg.command_size, arg.throttle_us,
 				  arg.context_handle,
 				  (void __user *)(unsigned long)arg.fence_rep,
-				  NULL);
+				  NULL,
+				  arg.flags);
 	ttm_read_unlock(&dev_priv->reservation_sem);
 	if (unlikely(ret != 0))
-		return ret;
+		goto out;
 
 	vmw_kms_cursor_post_execbuf(dev_priv);
 
-	return 0;
+out:
+	if (in_fence)
+		dma_fence_put(in_fence);
+	return ret;
 }
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
index 6f4cb4678cbc..d23a18aae476 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
@@ -779,7 +779,6 @@ int vmw_fb_init(struct vmw_private *vmw_priv)
 	info->screen_base = (char __iomem *)par->vmalloc;
 	info->screen_size = fb_size;
 
-	info->flags = FBINFO_DEFAULT;
 	info->fbops = &vmw_fb_ops;
 
 	/* 24 depth per default */
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
index b8bc5bc7de7e..3bbad22b3748 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
@@ -114,12 +114,11 @@ static void vmw_fence_obj_destroy(struct dma_fence *f)
 		container_of(f, struct vmw_fence_obj, base);
 
 	struct vmw_fence_manager *fman = fman_from_fence(fence);
-	unsigned long irq_flags;
 
-	spin_lock_irqsave(&fman->lock, irq_flags);
+	spin_lock(&fman->lock);
 	list_del_init(&fence->head);
 	--fman->num_fence_objects;
-	spin_unlock_irqrestore(&fman->lock, irq_flags);
+	spin_unlock(&fman->lock);
 	fence->destroy(fence);
 }
 
@@ -252,10 +251,10 @@ static void vmw_fence_work_func(struct work_struct *work)
 		INIT_LIST_HEAD(&list);
 		mutex_lock(&fman->goal_irq_mutex);
 
-		spin_lock_irq(&fman->lock);
+		spin_lock(&fman->lock);
 		list_splice_init(&fman->cleanup_list, &list);
 		seqno_valid = fman->seqno_valid;
-		spin_unlock_irq(&fman->lock);
+		spin_unlock(&fman->lock);
 
 		if (!seqno_valid && fman->goal_irq_on) {
 			fman->goal_irq_on = false;
@@ -305,15 +304,14 @@ struct vmw_fence_manager *vmw_fence_manager_init(struct vmw_private *dev_priv)
 
 void vmw_fence_manager_takedown(struct vmw_fence_manager *fman)
 {
-	unsigned long irq_flags;
 	bool lists_empty;
 
 	(void) cancel_work_sync(&fman->work);
 
-	spin_lock_irqsave(&fman->lock, irq_flags);
+	spin_lock(&fman->lock);
 	lists_empty = list_empty(&fman->fence_list) &&
 		list_empty(&fman->cleanup_list);
-	spin_unlock_irqrestore(&fman->lock, irq_flags);
+	spin_unlock(&fman->lock);
 
 	BUG_ON(!lists_empty);
 	kfree(fman);
@@ -323,7 +321,6 @@ static int vmw_fence_obj_init(struct vmw_fence_manager *fman,
 			      struct vmw_fence_obj *fence, u32 seqno,
 			      void (*destroy) (struct vmw_fence_obj *fence))
 {
-	unsigned long irq_flags;
 	int ret = 0;
 
 	dma_fence_init(&fence->base, &vmw_fence_ops, &fman->lock,
@@ -331,7 +328,7 @@ static int vmw_fence_obj_init(struct vmw_fence_manager *fman,
 	INIT_LIST_HEAD(&fence->seq_passed_actions);
 	fence->destroy = destroy;
 
-	spin_lock_irqsave(&fman->lock, irq_flags);
+	spin_lock(&fman->lock);
 	if (unlikely(fman->fifo_down)) {
 		ret = -EBUSY;
 		goto out_unlock;
@@ -340,7 +337,7 @@ static int vmw_fence_obj_init(struct vmw_fence_manager *fman,
 	++fman->num_fence_objects;
 
 out_unlock:
-	spin_unlock_irqrestore(&fman->lock, irq_flags);
+	spin_unlock(&fman->lock);
 	return ret;
 
 }
@@ -489,11 +486,9 @@ rerun:
 
 void vmw_fences_update(struct vmw_fence_manager *fman)
 {
-	unsigned long irq_flags;
-
-	spin_lock_irqsave(&fman->lock, irq_flags);
+	spin_lock(&fman->lock);
 	__vmw_fences_update(fman);
-	spin_unlock_irqrestore(&fman->lock, irq_flags);
+	spin_unlock(&fman->lock);
 }
 
 bool vmw_fence_obj_signaled(struct vmw_fence_obj *fence)
@@ -650,6 +645,51 @@ out_no_object:
 
 
 /**
+ * vmw_wait_dma_fence - Wait for a dma fence
+ *
+ * @fman: pointer to a fence manager
+ * @fence: DMA fence to wait on
+ *
+ * This function handles the case when the fence is actually a fence
+ * array.  If that's the case, it'll wait on each of the child fence
+ */
+int vmw_wait_dma_fence(struct vmw_fence_manager *fman,
+		       struct dma_fence *fence)
+{
+	struct dma_fence_array *fence_array;
+	int ret = 0;
+	int i;
+
+
+	if (dma_fence_is_signaled(fence))
+		return 0;
+
+	if (!dma_fence_is_array(fence))
+		return dma_fence_wait(fence, true);
+
+	/* From i915: Note that if the fence-array was created in
+	 * signal-on-any mode, we should *not* decompose it into its individual
+	 * fences. However, we don't currently store which mode the fence-array
+	 * is operating in. Fortunately, the only user of signal-on-any is
+	 * private to amdgpu and we should not see any incoming fence-array
+	 * from sync-file being in signal-on-any mode.
+	 */
+
+	fence_array = to_dma_fence_array(fence);
+	for (i = 0; i < fence_array->num_fences; i++) {
+		struct dma_fence *child = fence_array->fences[i];
+
+		ret = dma_fence_wait(child, true);
+
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+
+
+/**
  * vmw_fence_fifo_down - signal all unsignaled fence objects.
  */
 
@@ -663,14 +703,14 @@ void vmw_fence_fifo_down(struct vmw_fence_manager *fman)
 	 * restart when we've released the fman->lock.
 	 */
 
-	spin_lock_irq(&fman->lock);
+	spin_lock(&fman->lock);
 	fman->fifo_down = true;
 	while (!list_empty(&fman->fence_list)) {
 		struct vmw_fence_obj *fence =
 			list_entry(fman->fence_list.prev, struct vmw_fence_obj,
 				   head);
 		dma_fence_get(&fence->base);
-		spin_unlock_irq(&fman->lock);
+		spin_unlock(&fman->lock);
 
 		ret = vmw_fence_obj_wait(fence, false, false,
 					 VMW_FENCE_WAIT_TIMEOUT);
@@ -686,18 +726,16 @@ void vmw_fence_fifo_down(struct vmw_fence_manager *fman)
 
 		BUG_ON(!list_empty(&fence->head));
 		dma_fence_put(&fence->base);
-		spin_lock_irq(&fman->lock);
+		spin_lock(&fman->lock);
 	}
-	spin_unlock_irq(&fman->lock);
+	spin_unlock(&fman->lock);
 }
 
 void vmw_fence_fifo_up(struct vmw_fence_manager *fman)
 {
-	unsigned long irq_flags;
-
-	spin_lock_irqsave(&fman->lock, irq_flags);
+	spin_lock(&fman->lock);
 	fman->fifo_down = false;
-	spin_unlock_irqrestore(&fman->lock, irq_flags);
+	spin_unlock(&fman->lock);
 }
 
 
@@ -812,9 +850,9 @@ int vmw_fence_obj_signaled_ioctl(struct drm_device *dev, void *data,
 	arg->signaled = vmw_fence_obj_signaled(fence);
 
 	arg->signaled_flags = arg->flags;
-	spin_lock_irq(&fman->lock);
+	spin_lock(&fman->lock);
 	arg->passed_seqno = dev_priv->last_read_seqno;
-	spin_unlock_irq(&fman->lock);
+	spin_unlock(&fman->lock);
 
 	ttm_base_object_unref(&base);
 
@@ -841,8 +879,7 @@ int vmw_fence_obj_unref_ioctl(struct drm_device *dev, void *data,
  *
  * This function is called when the seqno of the fence where @action is
  * attached has passed. It queues the event on the submitter's event list.
- * This function is always called from atomic context, and may be called
- * from irq context.
+ * This function is always called from atomic context.
  */
 static void vmw_event_fence_action_seq_passed(struct vmw_fence_action *action)
 {
@@ -851,13 +888,13 @@ static void vmw_event_fence_action_seq_passed(struct vmw_fence_action *action)
 	struct drm_device *dev = eaction->dev;
 	struct drm_pending_event *event = eaction->event;
 	struct drm_file *file_priv;
-	unsigned long irq_flags;
+
 
 	if (unlikely(event == NULL))
 		return;
 
 	file_priv = event->file_priv;
-	spin_lock_irqsave(&dev->event_lock, irq_flags);
+	spin_lock_irq(&dev->event_lock);
 
 	if (likely(eaction->tv_sec != NULL)) {
 		struct timeval tv;
@@ -869,7 +906,7 @@ static void vmw_event_fence_action_seq_passed(struct vmw_fence_action *action)
 
 	drm_send_event_locked(dev, eaction->event);
 	eaction->event = NULL;
-	spin_unlock_irqrestore(&dev->event_lock, irq_flags);
+	spin_unlock_irq(&dev->event_lock);
 }
 
 /**
@@ -904,11 +941,10 @@ static void vmw_fence_obj_add_action(struct vmw_fence_obj *fence,
 			      struct vmw_fence_action *action)
 {
 	struct vmw_fence_manager *fman = fman_from_fence(fence);
-	unsigned long irq_flags;
 	bool run_update = false;
 
 	mutex_lock(&fman->goal_irq_mutex);
-	spin_lock_irqsave(&fman->lock, irq_flags);
+	spin_lock(&fman->lock);
 
 	fman->pending_actions[action->type]++;
 	if (dma_fence_is_signaled_locked(&fence->base)) {
@@ -927,7 +963,7 @@ static void vmw_fence_obj_add_action(struct vmw_fence_obj *fence,
 		run_update = vmw_fence_goal_check_locked(fence);
 	}
 
-	spin_unlock_irqrestore(&fman->lock, irq_flags);
+	spin_unlock(&fman->lock);
 
 	if (run_update) {
 		if (!fman->goal_irq_on) {
@@ -1114,7 +1150,7 @@ int vmw_fence_event_ioctl(struct drm_device *dev, void *data,
 	}
 
 	vmw_execbuf_copy_fence_user(dev_priv, vmw_fp, 0, user_fence_rep, fence,
-				    handle);
+				    handle, -1, NULL);
 	vmw_fence_obj_unreference(&fence);
 	return 0;
 out_no_create:
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.h b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.h
index d9d85aa6ed20..20224dba9d8e 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.h
@@ -28,6 +28,7 @@
 #ifndef _VMWGFX_FENCE_H_
 
 #include <linux/dma-fence.h>
+#include <linux/dma-fence-array.h>
 
 #define VMW_FENCE_WAIT_TIMEOUT (5*HZ)
 
@@ -102,6 +103,9 @@ extern int vmw_user_fence_create(struct drm_file *file_priv,
 				 struct vmw_fence_obj **p_fence,
 				 uint32_t *p_handle);
 
+extern int vmw_wait_dma_fence(struct vmw_fence_manager *fman,
+			      struct dma_fence *fence);
+
 extern void vmw_fence_fifo_up(struct vmw_fence_manager *fman);
 
 extern void vmw_fence_fifo_down(struct vmw_fence_manager *fman);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_gmrid_manager.c b/drivers/gpu/drm/vmwgfx/vmwgfx_gmrid_manager.c
index d2b03d4a3c86..f2f9d88131f2 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_gmrid_manager.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_gmrid_manager.c
@@ -157,9 +157,9 @@ static int vmw_gmrid_man_takedown(struct ttm_mem_type_manager *man)
 }
 
 static void vmw_gmrid_man_debug(struct ttm_mem_type_manager *man,
-				const char *prefix)
+				struct drm_printer *printer)
 {
-	pr_info("%s: No debug info available for the GMR id manager\n", prefix);
+	drm_printf(printer, "No debug info available for the GMR id manager\n");
 }
 
 const struct ttm_mem_type_manager_func vmw_gmrid_manager_func = {
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c b/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c
index 0c7e1723292c..b9239ba067c4 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c
@@ -30,11 +30,56 @@
 
 #define VMW_FENCE_WRAP (1 << 24)
 
-irqreturn_t vmw_irq_handler(int irq, void *arg)
+/**
+ * vmw_thread_fn - Deferred (process context) irq handler
+ *
+ * @irq: irq number
+ * @arg: Closure argument. Pointer to a struct drm_device cast to void *
+ *
+ * This function implements the deferred part of irq processing.
+ * The function is guaranteed to run at least once after the
+ * vmw_irq_handler has returned with IRQ_WAKE_THREAD.
+ *
+ */
+static irqreturn_t vmw_thread_fn(int irq, void *arg)
+{
+	struct drm_device *dev = (struct drm_device *)arg;
+	struct vmw_private *dev_priv = vmw_priv(dev);
+	irqreturn_t ret = IRQ_NONE;
+
+	if (test_and_clear_bit(VMW_IRQTHREAD_FENCE,
+			       dev_priv->irqthread_pending)) {
+		vmw_fences_update(dev_priv->fman);
+		wake_up_all(&dev_priv->fence_queue);
+		ret = IRQ_HANDLED;
+	}
+
+	if (test_and_clear_bit(VMW_IRQTHREAD_CMDBUF,
+			       dev_priv->irqthread_pending)) {
+		vmw_cmdbuf_irqthread(dev_priv->cman);
+		ret = IRQ_HANDLED;
+	}
+
+	return ret;
+}
+
+/**
+ * vmw_irq_handler irq handler
+ *
+ * @irq: irq number
+ * @arg: Closure argument. Pointer to a struct drm_device cast to void *
+ *
+ * This function implements the quick part of irq processing.
+ * The function performs fast actions like clearing the device interrupt
+ * flags and also reasonably quick actions like waking processes waiting for
+ * FIFO space. Other IRQ actions are deferred to the IRQ thread.
+ */
+static irqreturn_t vmw_irq_handler(int irq, void *arg)
 {
 	struct drm_device *dev = (struct drm_device *)arg;
 	struct vmw_private *dev_priv = vmw_priv(dev);
 	uint32_t status, masked_status;
+	irqreturn_t ret = IRQ_HANDLED;
 
 	status = inl(dev_priv->io_start + VMWGFX_IRQSTATUS_PORT);
 	masked_status = status & READ_ONCE(dev_priv->irq_mask);
@@ -45,20 +90,21 @@ irqreturn_t vmw_irq_handler(int irq, void *arg)
 	if (!status)
 		return IRQ_NONE;
 
-	if (masked_status & (SVGA_IRQFLAG_ANY_FENCE |
-			     SVGA_IRQFLAG_FENCE_GOAL)) {
-		vmw_fences_update(dev_priv->fman);
-		wake_up_all(&dev_priv->fence_queue);
-	}
-
 	if (masked_status & SVGA_IRQFLAG_FIFO_PROGRESS)
 		wake_up_all(&dev_priv->fifo_queue);
 
-	if (masked_status & (SVGA_IRQFLAG_COMMAND_BUFFER |
-			     SVGA_IRQFLAG_ERROR))
-		vmw_cmdbuf_tasklet_schedule(dev_priv->cman);
+	if ((masked_status & (SVGA_IRQFLAG_ANY_FENCE |
+			      SVGA_IRQFLAG_FENCE_GOAL)) &&
+	    !test_and_set_bit(VMW_IRQTHREAD_FENCE, dev_priv->irqthread_pending))
+		ret = IRQ_WAKE_THREAD;
 
-	return IRQ_HANDLED;
+	if ((masked_status & (SVGA_IRQFLAG_COMMAND_BUFFER |
+			      SVGA_IRQFLAG_ERROR)) &&
+	    !test_and_set_bit(VMW_IRQTHREAD_CMDBUF,
+			      dev_priv->irqthread_pending))
+		ret = IRQ_WAKE_THREAD;
+
+	return ret;
 }
 
 static bool vmw_fifo_idle(struct vmw_private *dev_priv, uint32_t seqno)
@@ -281,23 +327,15 @@ int vmw_wait_seqno(struct vmw_private *dev_priv,
 	return ret;
 }
 
-void vmw_irq_preinstall(struct drm_device *dev)
+static void vmw_irq_preinstall(struct drm_device *dev)
 {
 	struct vmw_private *dev_priv = vmw_priv(dev);
 	uint32_t status;
 
-	if (!(dev_priv->capabilities & SVGA_CAP_IRQMASK))
-		return;
-
 	status = inl(dev_priv->io_start + VMWGFX_IRQSTATUS_PORT);
 	outl(status, dev_priv->io_start + VMWGFX_IRQSTATUS_PORT);
 }
 
-int vmw_irq_postinstall(struct drm_device *dev)
-{
-	return 0;
-}
-
 void vmw_irq_uninstall(struct drm_device *dev)
 {
 	struct vmw_private *dev_priv = vmw_priv(dev);
@@ -306,8 +344,41 @@ void vmw_irq_uninstall(struct drm_device *dev)
 	if (!(dev_priv->capabilities & SVGA_CAP_IRQMASK))
 		return;
 
+	if (!dev->irq_enabled)
+		return;
+
 	vmw_write(dev_priv, SVGA_REG_IRQMASK, 0);
 
 	status = inl(dev_priv->io_start + VMWGFX_IRQSTATUS_PORT);
 	outl(status, dev_priv->io_start + VMWGFX_IRQSTATUS_PORT);
+
+	dev->irq_enabled = false;
+	free_irq(dev->irq, dev);
+}
+
+/**
+ * vmw_irq_install - Install the irq handlers
+ *
+ * @dev:  Pointer to the drm device.
+ * @irq:  The irq number.
+ * Return:  Zero if successful. Negative number otherwise.
+ */
+int vmw_irq_install(struct drm_device *dev, int irq)
+{
+	int ret;
+
+	if (dev->irq_enabled)
+		return -EBUSY;
+
+	vmw_irq_preinstall(dev);
+
+	ret = request_threaded_irq(irq, vmw_irq_handler, vmw_thread_fn,
+				   IRQF_SHARED, VMWGFX_DRIVER_NAME, dev);
+	if (ret < 0)
+		return ret;
+
+	dev->irq_enabled = true;
+	dev->irq = irq;
+
+	return ret;
 }
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index 61e06f0e8cd3..b850562fbdd6 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -1536,7 +1536,7 @@ err_out:
  * RETURNS
  * Zero for success or -errno
  */
-int
+static int
 vmw_kms_atomic_check_modeset(struct drm_device *dev,
 			     struct drm_atomic_state *state)
 {
@@ -1545,8 +1545,7 @@ vmw_kms_atomic_check_modeset(struct drm_device *dev,
 	struct vmw_private *dev_priv = vmw_priv(dev);
 	int i;
 
-
-	for_each_crtc_in_state(state, crtc, crtc_state, i) {
+	for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
 		unsigned long requested_bb_mem = 0;
 
 		if (dev_priv->active_display_unit == vmw_du_screen_target) {
@@ -1567,10 +1566,34 @@ vmw_kms_atomic_check_modeset(struct drm_device *dev,
 }
 
 
+/**
+ * vmw_kms_atomic_commit - Perform an atomic state commit
+ *
+ * @dev: DRM device
+ * @state: the driver state object
+ * @nonblock: Whether nonblocking behaviour is requested
+ *
+ * This is a simple wrapper around drm_atomic_helper_commit() for
+ * us to clear the nonblocking value.
+ *
+ * Nonblocking commits currently cause synchronization issues
+ * for vmwgfx.
+ *
+ * RETURNS
+ * Zero for success or negative error code on failure.
+ */
+int vmw_kms_atomic_commit(struct drm_device *dev,
+			  struct drm_atomic_state *state,
+			  bool nonblock)
+{
+	return drm_atomic_helper_commit(dev, state, false);
+}
+
+
 static const struct drm_mode_config_funcs vmw_kms_funcs = {
 	.fb_create = vmw_kms_fb_create,
 	.atomic_check = vmw_kms_atomic_check_modeset,
-	.atomic_commit = drm_atomic_helper_commit,
+	.atomic_commit = vmw_kms_atomic_commit,
 };
 
 static int vmw_kms_generic_present(struct vmw_private *dev_priv,
@@ -1667,7 +1690,7 @@ int vmw_kms_init(struct vmw_private *dev_priv)
 
 int vmw_kms_close(struct vmw_private *dev_priv)
 {
-	int ret;
+	int ret = 0;
 
 	/*
 	 * Docs says we should take the lock before calling this function
@@ -1675,11 +1698,7 @@ int vmw_kms_close(struct vmw_private *dev_priv)
 	 * drm_encoder_cleanup which takes the lock we deadlock.
 	 */
 	drm_mode_config_cleanup(dev_priv->dev);
-	if (dev_priv->active_display_unit == vmw_du_screen_object)
-		ret = vmw_kms_sou_close_display(dev_priv);
-	else if (dev_priv->active_display_unit == vmw_du_screen_target)
-		ret = vmw_kms_stdu_close_display(dev_priv);
-	else
+	if (dev_priv->active_display_unit == vmw_du_legacy)
 		ret = vmw_kms_ldu_close_display(dev_priv);
 
 	return ret;
@@ -2499,7 +2518,7 @@ void vmw_kms_helper_buffer_finish(struct vmw_private *dev_priv,
 	if (file_priv)
 		vmw_execbuf_copy_fence_user(dev_priv, vmw_fpriv(file_priv),
 					    ret, user_fence_rep, fence,
-					    handle);
+					    handle, -1, NULL);
 	if (out_fence)
 		*out_fence = fence;
 	else
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
index 5f8d678ae675..ff9c8389ff21 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
@@ -390,7 +390,6 @@ int vmw_kms_update_proxy(struct vmw_resource *res,
  * Screen Objects display functions - vmwgfx_scrn.c
  */
 int vmw_kms_sou_init_display(struct vmw_private *dev_priv);
-int vmw_kms_sou_close_display(struct vmw_private *dev_priv);
 int vmw_kms_sou_do_surface_dirty(struct vmw_private *dev_priv,
 				 struct vmw_framebuffer *framebuffer,
 				 struct drm_clip_rect *clips,
@@ -418,7 +417,6 @@ int vmw_kms_sou_readback(struct vmw_private *dev_priv,
  * Screen Target Display Unit functions - vmwgfx_stdu.c
  */
 int vmw_kms_stdu_init_display(struct vmw_private *dev_priv);
-int vmw_kms_stdu_close_display(struct vmw_private *dev_priv);
 int vmw_kms_stdu_surface_dirty(struct vmw_private *dev_priv,
 			       struct vmw_framebuffer *framebuffer,
 			       struct drm_clip_rect *clips,
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
index d3987bcf53f8..b8a09807c5de 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
@@ -203,19 +203,7 @@ static void vmw_ldu_crtc_mode_set_nofb(struct drm_crtc *crtc)
 }
 
 /**
- * vmw_ldu_crtc_helper_prepare - Noop
- *
- * @crtc: CRTC associated with the new screen
- *
- * Prepares the CRTC for a mode set, but we don't need to do anything here.
- *
- */
-static void vmw_ldu_crtc_helper_prepare(struct drm_crtc *crtc)
-{
-}
-
-/**
- * vmw_ldu_crtc_helper_commit - Noop
+ * vmw_ldu_crtc_atomic_enable - Noop
  *
  * @crtc: CRTC associated with the new screen
  *
@@ -224,16 +212,18 @@ static void vmw_ldu_crtc_helper_prepare(struct drm_crtc *crtc)
  * but since for LDU the display plane is closely tied to the
  * CRTC, it makes more sense to do those at plane update time.
  */
-static void vmw_ldu_crtc_helper_commit(struct drm_crtc *crtc)
+static void vmw_ldu_crtc_atomic_enable(struct drm_crtc *crtc,
+				       struct drm_crtc_state *old_state)
 {
 }
 
 /**
- * vmw_ldu_crtc_helper_disable - Turns off CRTC
+ * vmw_ldu_crtc_atomic_disable - Turns off CRTC
  *
  * @crtc: CRTC to be turned off
  */
-static void vmw_ldu_crtc_helper_disable(struct drm_crtc *crtc)
+static void vmw_ldu_crtc_atomic_disable(struct drm_crtc *crtc,
+					struct drm_crtc_state *old_state)
 {
 }
 
@@ -388,13 +378,12 @@ drm_plane_helper_funcs vmw_ldu_primary_plane_helper_funcs = {
 };
 
 static const struct drm_crtc_helper_funcs vmw_ldu_crtc_helper_funcs = {
-	.prepare = vmw_ldu_crtc_helper_prepare,
-	.commit = vmw_ldu_crtc_helper_commit,
-	.disable = vmw_ldu_crtc_helper_disable,
 	.mode_set_nofb = vmw_ldu_crtc_mode_set_nofb,
 	.atomic_check = vmw_du_crtc_atomic_check,
 	.atomic_begin = vmw_du_crtc_atomic_begin,
 	.atomic_flush = vmw_du_crtc_atomic_flush,
+	.atomic_enable = vmw_ldu_crtc_atomic_enable,
+	.atomic_disable = vmw_ldu_crtc_atomic_disable,
 };
 
 
@@ -439,7 +428,7 @@ static int vmw_ldu_init(struct vmw_private *dev_priv, unsigned unit)
 				       0, &vmw_ldu_plane_funcs,
 				       vmw_primary_plane_formats,
 				       ARRAY_SIZE(vmw_primary_plane_formats),
-				       DRM_PLANE_TYPE_PRIMARY, NULL);
+				       NULL, DRM_PLANE_TYPE_PRIMARY, NULL);
 	if (ret) {
 		DRM_ERROR("Failed to initialize primary plane");
 		goto err_free;
@@ -454,7 +443,7 @@ static int vmw_ldu_init(struct vmw_private *dev_priv, unsigned unit)
 			0, &vmw_ldu_cursor_funcs,
 			vmw_cursor_plane_formats,
 			ARRAY_SIZE(vmw_cursor_plane_formats),
-			DRM_PLANE_TYPE_CURSOR, NULL);
+			NULL, DRM_PLANE_TYPE_CURSOR, NULL);
 	if (ret) {
 		DRM_ERROR("Failed to initialize cursor plane");
 		drm_plane_cleanup(&ldu->base.primary);
@@ -582,13 +571,9 @@ err_free:
 
 int vmw_kms_ldu_close_display(struct vmw_private *dev_priv)
 {
-	struct drm_device *dev = dev_priv->dev;
-
 	if (!dev_priv->ldu_priv)
 		return -ENOSYS;
 
-	drm_vblank_cleanup(dev);
-
 	BUG_ON(!list_empty(&dev_priv->ldu_priv->active));
 
 	kfree(dev_priv->ldu_priv);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c b/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
index 8d7dc9def7c2..d1552d3e0652 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
@@ -270,22 +270,24 @@ static void vmw_sou_crtc_helper_prepare(struct drm_crtc *crtc)
 }
 
 /**
- * vmw_sou_crtc_helper_commit - Noop
+ * vmw_sou_crtc_atomic_enable - Noop
  *
  * @crtc: CRTC associated with the new screen
  *
  * This is called after a mode set has been completed.
  */
-static void vmw_sou_crtc_helper_commit(struct drm_crtc *crtc)
+static void vmw_sou_crtc_atomic_enable(struct drm_crtc *crtc,
+				       struct drm_crtc_state *old_state)
 {
 }
 
 /**
- * vmw_sou_crtc_helper_disable - Turns off CRTC
+ * vmw_sou_crtc_atomic_disable - Turns off CRTC
  *
  * @crtc: CRTC to be turned off
  */
-static void vmw_sou_crtc_helper_disable(struct drm_crtc *crtc)
+static void vmw_sou_crtc_atomic_disable(struct drm_crtc *crtc,
+					struct drm_crtc_state *old_state)
 {
 	struct vmw_private *dev_priv;
 	struct vmw_screen_object_unit *sou;
@@ -573,12 +575,12 @@ drm_plane_helper_funcs vmw_sou_primary_plane_helper_funcs = {
 
 static const struct drm_crtc_helper_funcs vmw_sou_crtc_helper_funcs = {
 	.prepare = vmw_sou_crtc_helper_prepare,
-	.commit = vmw_sou_crtc_helper_commit,
-	.disable = vmw_sou_crtc_helper_disable,
 	.mode_set_nofb = vmw_sou_crtc_mode_set_nofb,
 	.atomic_check = vmw_du_crtc_atomic_check,
 	.atomic_begin = vmw_du_crtc_atomic_begin,
 	.atomic_flush = vmw_du_crtc_atomic_flush,
+	.atomic_enable = vmw_sou_crtc_atomic_enable,
+	.atomic_disable = vmw_sou_crtc_atomic_disable,
 };
 
 
@@ -622,7 +624,7 @@ static int vmw_sou_init(struct vmw_private *dev_priv, unsigned unit)
 				       0, &vmw_sou_plane_funcs,
 				       vmw_primary_plane_formats,
 				       ARRAY_SIZE(vmw_primary_plane_formats),
-				       DRM_PLANE_TYPE_PRIMARY, NULL);
+				       NULL, DRM_PLANE_TYPE_PRIMARY, NULL);
 	if (ret) {
 		DRM_ERROR("Failed to initialize primary plane");
 		goto err_free;
@@ -637,7 +639,7 @@ static int vmw_sou_init(struct vmw_private *dev_priv, unsigned unit)
 			0, &vmw_sou_cursor_funcs,
 			vmw_cursor_plane_formats,
 			ARRAY_SIZE(vmw_cursor_plane_formats),
-			DRM_PLANE_TYPE_CURSOR, NULL);
+			NULL, DRM_PLANE_TYPE_CURSOR, NULL);
 	if (ret) {
 		DRM_ERROR("Failed to initialize cursor plane");
 		drm_plane_cleanup(&sou->base.primary);
@@ -746,15 +748,6 @@ int vmw_kms_sou_init_display(struct vmw_private *dev_priv)
 	return 0;
 }
 
-int vmw_kms_sou_close_display(struct vmw_private *dev_priv)
-{
-	struct drm_device *dev = dev_priv->dev;
-
-	drm_vblank_cleanup(dev);
-
-	return 0;
-}
-
 static int do_dmabuf_define_gmrfb(struct vmw_private *dev_priv,
 				  struct vmw_framebuffer *framebuffer)
 {
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
index 5284e8d2f7ba..ca3afae2db1f 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
@@ -412,7 +412,8 @@ static void vmw_stdu_crtc_helper_prepare(struct drm_crtc *crtc)
 }
 
 
-static void vmw_stdu_crtc_helper_commit(struct drm_crtc *crtc)
+static void vmw_stdu_crtc_atomic_enable(struct drm_crtc *crtc,
+					struct drm_crtc_state *old_state)
 {
 	struct vmw_private *dev_priv;
 	struct vmw_screen_target_display_unit *stdu;
@@ -432,7 +433,8 @@ static void vmw_stdu_crtc_helper_commit(struct drm_crtc *crtc)
 		vmw_kms_del_active(dev_priv, &stdu->base);
 }
 
-static void vmw_stdu_crtc_helper_disable(struct drm_crtc *crtc)
+static void vmw_stdu_crtc_atomic_disable(struct drm_crtc *crtc,
+					 struct drm_crtc_state *old_state)
 {
 	struct vmw_private *dev_priv;
 	struct vmw_screen_target_display_unit *stdu;
@@ -1415,12 +1417,12 @@ drm_plane_helper_funcs vmw_stdu_primary_plane_helper_funcs = {
 
 static const struct drm_crtc_helper_funcs vmw_stdu_crtc_helper_funcs = {
 	.prepare = vmw_stdu_crtc_helper_prepare,
-	.commit = vmw_stdu_crtc_helper_commit,
-	.disable = vmw_stdu_crtc_helper_disable,
 	.mode_set_nofb = vmw_stdu_crtc_mode_set_nofb,
 	.atomic_check = vmw_du_crtc_atomic_check,
 	.atomic_begin = vmw_du_crtc_atomic_begin,
 	.atomic_flush = vmw_du_crtc_atomic_flush,
+	.atomic_enable = vmw_stdu_crtc_atomic_enable,
+	.atomic_disable = vmw_stdu_crtc_atomic_disable,
 };
 
 
@@ -1473,7 +1475,7 @@ static int vmw_stdu_init(struct vmw_private *dev_priv, unsigned unit)
 				       0, &vmw_stdu_plane_funcs,
 				       vmw_primary_plane_formats,
 				       ARRAY_SIZE(vmw_primary_plane_formats),
-				       DRM_PLANE_TYPE_PRIMARY, NULL);
+				       NULL, DRM_PLANE_TYPE_PRIMARY, NULL);
 	if (ret) {
 		DRM_ERROR("Failed to initialize primary plane");
 		goto err_free;
@@ -1488,7 +1490,7 @@ static int vmw_stdu_init(struct vmw_private *dev_priv, unsigned unit)
 			0, &vmw_stdu_cursor_funcs,
 			vmw_cursor_plane_formats,
 			ARRAY_SIZE(vmw_cursor_plane_formats),
-			DRM_PLANE_TYPE_CURSOR, NULL);
+			NULL, DRM_PLANE_TYPE_CURSOR, NULL);
 	if (ret) {
 		DRM_ERROR("Failed to initialize cursor plane");
 		drm_plane_cleanup(&stdu->base.primary);
@@ -1651,36 +1653,11 @@ int vmw_kms_stdu_init_display(struct vmw_private *dev_priv)
 
 		if (unlikely(ret != 0)) {
 			DRM_ERROR("Failed to initialize STDU %d", i);
-			goto err_vblank_cleanup;
+			return ret;
 		}
 	}
 
 	DRM_INFO("Screen Target Display device initialized\n");
 
 	return 0;
-
-err_vblank_cleanup:
-	drm_vblank_cleanup(dev);
-	return ret;
-}
-
-
-
-/**
- * vmw_kms_stdu_close_display - Cleans up after vmw_kms_stdu_init_display
- *
- * @dev_priv: VMW DRM device
- *
- * Frees up any resources allocated by vmw_kms_stdu_init_display
- *
- * RETURNS:
- * 0 on success
- */
-int vmw_kms_stdu_close_display(struct vmw_private *dev_priv)
-{
-	struct drm_device *dev = dev_priv->dev;
-
-	drm_vblank_cleanup(dev);
-
-	return 0;
 }
diff --git a/drivers/gpu/drm/zte/zx_drm_drv.c b/drivers/gpu/drm/zte/zx_drm_drv.c
index f46c855d274b..45244828fc1f 100644
--- a/drivers/gpu/drm/zte/zx_drm_drv.c
+++ b/drivers/gpu/drm/zte/zx_drm_drv.c
@@ -59,11 +59,9 @@ static struct drm_driver zx_drm_driver = {
 	.driver_features = DRIVER_GEM | DRIVER_MODESET | DRIVER_PRIME |
 			   DRIVER_ATOMIC,
 	.lastclose = zx_drm_lastclose,
-	.gem_free_object = drm_gem_cma_free_object,
+	.gem_free_object_unlocked = drm_gem_cma_free_object,
 	.gem_vm_ops = &drm_gem_cma_vm_ops,
 	.dumb_create = drm_gem_cma_dumb_create,
-	.dumb_map_offset = drm_gem_cma_dumb_map_offset,
-	.dumb_destroy = drm_gem_dumb_destroy,
 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
 	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
 	.gem_prime_export = drm_gem_prime_export,
@@ -149,7 +147,6 @@ out_fbdev_fini:
 out_poll_fini:
 	drm_kms_helper_poll_fini(drm);
 	drm_mode_config_cleanup(drm);
-	drm_vblank_cleanup(drm);
 out_unbind:
 	component_unbind_all(dev, drm);
 out_unregister:
@@ -171,7 +168,6 @@ static void zx_drm_unbind(struct device *dev)
 	}
 	drm_kms_helper_poll_fini(drm);
 	drm_mode_config_cleanup(drm);
-	drm_vblank_cleanup(drm);
 	component_unbind_all(dev, drm);
 	dev_set_drvdata(dev, NULL);
 	drm->dev_private = NULL;
diff --git a/drivers/gpu/drm/zte/zx_hdmi.c b/drivers/gpu/drm/zte/zx_hdmi.c
index 0df7366e594b..b8abb1b496ff 100644
--- a/drivers/gpu/drm/zte/zx_hdmi.c
+++ b/drivers/gpu/drm/zte/zx_hdmi.c
@@ -124,7 +124,7 @@ static int zx_hdmi_config_video_avi(struct zx_hdmi *hdmi,
 	union hdmi_infoframe frame;
 	int ret;
 
-	ret = drm_hdmi_avi_infoframe_from_display_mode(&frame.avi, mode);
+	ret = drm_hdmi_avi_infoframe_from_display_mode(&frame.avi, mode, false);
 	if (ret) {
 		DRM_DEV_ERROR(hdmi->dev, "failed to get avi infoframe: %d\n",
 			      ret);
@@ -300,7 +300,6 @@ zx_hdmi_connector_detect(struct drm_connector *connector, bool force)
 }
 
 static const struct drm_connector_funcs zx_hdmi_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = zx_hdmi_connector_detect,
 	.destroy = drm_connector_cleanup,
diff --git a/drivers/gpu/drm/zte/zx_plane.c b/drivers/gpu/drm/zte/zx_plane.c
index 4a6252720c10..18e763493264 100644
--- a/drivers/gpu/drm/zte/zx_plane.c
+++ b/drivers/gpu/drm/zte/zx_plane.c
@@ -540,7 +540,7 @@ int zx_plane_init(struct drm_device *drm, struct zx_plane *zplane,
 
 	ret = drm_universal_plane_init(drm, plane, VOU_CRTC_MASK,
 				       &zx_plane_funcs, formats, format_count,
-				       type, NULL);
+				       NULL, type, NULL);
 	if (ret) {
 		DRM_DEV_ERROR(dev, "failed to init universal plane: %d\n", ret);
 		return ret;
diff --git a/drivers/gpu/drm/zte/zx_tvenc.c b/drivers/gpu/drm/zte/zx_tvenc.c
index b56dc69843fc..0de1a71ca4e0 100644
--- a/drivers/gpu/drm/zte/zx_tvenc.c
+++ b/drivers/gpu/drm/zte/zx_tvenc.c
@@ -269,7 +269,6 @@ static struct drm_connector_helper_funcs zx_tvenc_connector_helper_funcs = {
 };
 
 static const struct drm_connector_funcs zx_tvenc_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = drm_connector_cleanup,
 	.reset = drm_atomic_helper_connector_reset,
diff --git a/drivers/gpu/drm/zte/zx_vga.c b/drivers/gpu/drm/zte/zx_vga.c
index 1e0811f775cb..3e7e33cd3dfa 100644
--- a/drivers/gpu/drm/zte/zx_vga.c
+++ b/drivers/gpu/drm/zte/zx_vga.c
@@ -138,7 +138,6 @@ zx_vga_connector_detect(struct drm_connector *connector, bool force)
 }
 
 static const struct drm_connector_funcs zx_vga_connector_funcs = {
-	.dpms = drm_atomic_helper_connector_dpms,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.detect = zx_vga_connector_detect,
 	.destroy = drm_connector_cleanup,
diff --git a/drivers/gpu/drm/zte/zx_vou.c b/drivers/gpu/drm/zte/zx_vou.c
index 5fbd10b60ee5..7491813131f3 100644
--- a/drivers/gpu/drm/zte/zx_vou.c
+++ b/drivers/gpu/drm/zte/zx_vou.c
@@ -350,7 +350,8 @@ static inline void vou_chn_set_update(struct zx_crtc *zcrtc)
 	zx_writel(zcrtc->chnreg + CHN_UPDATE, 1);
 }
 
-static void zx_crtc_enable(struct drm_crtc *crtc)
+static void zx_crtc_atomic_enable(struct drm_crtc *crtc,
+				  struct drm_crtc_state *old_state)
 {
 	struct drm_display_mode *mode = &crtc->state->adjusted_mode;
 	bool interlaced = mode->flags & DRM_MODE_FLAG_INTERLACE;
@@ -454,7 +455,8 @@ static void zx_crtc_enable(struct drm_crtc *crtc)
 		DRM_DEV_ERROR(vou->dev, "failed to enable pixclk: %d\n", ret);
 }
 
-static void zx_crtc_disable(struct drm_crtc *crtc)
+static void zx_crtc_atomic_disable(struct drm_crtc *crtc,
+				   struct drm_crtc_state *old_state)
 {
 	struct zx_crtc *zcrtc = to_zx_crtc(crtc);
 	const struct zx_crtc_bits *bits = zcrtc->bits;
@@ -490,9 +492,9 @@ static void zx_crtc_atomic_flush(struct drm_crtc *crtc,
 }
 
 static const struct drm_crtc_helper_funcs zx_crtc_helper_funcs = {
-	.enable = zx_crtc_enable,
-	.disable = zx_crtc_disable,
 	.atomic_flush = zx_crtc_atomic_flush,
+	.atomic_enable = zx_crtc_atomic_enable,
+	.atomic_disable = zx_crtc_atomic_disable,
 };
 
 static int zx_vou_enable_vblank(struct drm_crtc *crtc)
diff --git a/drivers/gpu/host1x/bus.c b/drivers/gpu/host1x/bus.c
index a048e3ac523d..f9cde03030fd 100644
--- a/drivers/gpu/host1x/bus.c
+++ b/drivers/gpu/host1x/bus.c
@@ -41,13 +41,15 @@ struct host1x_subdev {
 /**
  * host1x_subdev_add() - add a new subdevice with an associated device node
  * @device: host1x device to add the subdevice to
- * @driver: host1x driver
  * @np: device node
  */
 static int host1x_subdev_add(struct host1x_device *device,
+			     struct host1x_driver *driver,
 			     struct device_node *np)
 {
 	struct host1x_subdev *subdev;
+	struct device_node *child;
+	int err;
 
 	subdev = kzalloc(sizeof(*subdev), GFP_KERNEL);
 	if (!subdev)
@@ -60,6 +62,19 @@ static int host1x_subdev_add(struct host1x_device *device,
 	list_add_tail(&subdev->list, &device->subdevs);
 	mutex_unlock(&device->subdevs_lock);
 
+	/* recursively add children */
+	for_each_child_of_node(np, child) {
+		if (of_match_node(driver->subdevs, child) &&
+		    of_device_is_available(child)) {
+			err = host1x_subdev_add(device, driver, child);
+			if (err < 0) {
+				/* XXX cleanup? */
+				of_node_put(child);
+				return err;
+			}
+		}
+	}
+
 	return 0;
 }
 
@@ -88,7 +103,7 @@ static int host1x_device_parse_dt(struct host1x_device *device,
 	for_each_child_of_node(device->dev.parent->of_node, np) {
 		if (of_match_node(driver->subdevs, np) &&
 		    of_device_is_available(np)) {
-			err = host1x_subdev_add(device, np);
+			err = host1x_subdev_add(device, driver, np);
 			if (err < 0) {
 				of_node_put(np);
 				return err;
diff --git a/drivers/gpu/host1x/dev.c b/drivers/gpu/host1x/dev.c
index 778272514164..7f22c5c37660 100644
--- a/drivers/gpu/host1x/dev.c
+++ b/drivers/gpu/host1x/dev.c
@@ -134,8 +134,8 @@ static int host1x_probe(struct platform_device *pdev)
 
 	syncpt_irq = platform_get_irq(pdev, 0);
 	if (syncpt_irq < 0) {
-		dev_err(&pdev->dev, "failed to get IRQ\n");
-		return -ENXIO;
+		dev_err(&pdev->dev, "failed to get IRQ: %d\n", syncpt_irq);
+		return syncpt_irq;
 	}
 
 	host = devm_kzalloc(&pdev->dev, sizeof(*host), GFP_KERNEL);
diff --git a/drivers/gpu/host1x/hw/intr_hw.c b/drivers/gpu/host1x/hw/intr_hw.c
index dacb8009a605..37ebb51703fa 100644
--- a/drivers/gpu/host1x/hw/intr_hw.c
+++ b/drivers/gpu/host1x/hw/intr_hw.c
@@ -33,10 +33,10 @@ static void host1x_intr_syncpt_handle(struct host1x_syncpt *syncpt)
 	unsigned int id = syncpt->id;
 	struct host1x *host = syncpt->host;
 
-	host1x_sync_writel(host, BIT_MASK(id),
-		HOST1X_SYNC_SYNCPT_THRESH_INT_DISABLE(BIT_WORD(id)));
-	host1x_sync_writel(host, BIT_MASK(id),
-		HOST1X_SYNC_SYNCPT_THRESH_CPU0_INT_STATUS(BIT_WORD(id)));
+	host1x_sync_writel(host, BIT(id % 32),
+		HOST1X_SYNC_SYNCPT_THRESH_INT_DISABLE(id / 32));
+	host1x_sync_writel(host, BIT(id % 32),
+		HOST1X_SYNC_SYNCPT_THRESH_CPU0_INT_STATUS(id / 32));
 
 	schedule_work(&syncpt->intr.work);
 }
@@ -50,9 +50,9 @@ static irqreturn_t syncpt_thresh_isr(int irq, void *dev_id)
 	for (i = 0; i < DIV_ROUND_UP(host->info->nb_pts, 32); i++) {
 		reg = host1x_sync_readl(host,
 			HOST1X_SYNC_SYNCPT_THRESH_CPU0_INT_STATUS(i));
-		for_each_set_bit(id, &reg, BITS_PER_LONG) {
+		for_each_set_bit(id, &reg, 32) {
 			struct host1x_syncpt *syncpt =
-				host->syncpt + (i * BITS_PER_LONG + id);
+				host->syncpt + (i * 32 + id);
 			host1x_intr_syncpt_handle(syncpt);
 		}
 	}
@@ -117,17 +117,17 @@ static void _host1x_intr_set_syncpt_threshold(struct host1x *host,
 static void _host1x_intr_enable_syncpt_intr(struct host1x *host,
 					    unsigned int id)
 {
-	host1x_sync_writel(host, BIT_MASK(id),
-		HOST1X_SYNC_SYNCPT_THRESH_INT_ENABLE_CPU0(BIT_WORD(id)));
+	host1x_sync_writel(host, BIT(id % 32),
+		HOST1X_SYNC_SYNCPT_THRESH_INT_ENABLE_CPU0(id / 32));
 }
 
 static void _host1x_intr_disable_syncpt_intr(struct host1x *host,
 					     unsigned int id)
 {
-	host1x_sync_writel(host, BIT_MASK(id),
-		HOST1X_SYNC_SYNCPT_THRESH_INT_DISABLE(BIT_WORD(id)));
-	host1x_sync_writel(host, BIT_MASK(id),
-		HOST1X_SYNC_SYNCPT_THRESH_CPU0_INT_STATUS(BIT_WORD(id)));
+	host1x_sync_writel(host, BIT(id % 32),
+		HOST1X_SYNC_SYNCPT_THRESH_INT_DISABLE(id / 32));
+	host1x_sync_writel(host, BIT(id % 32),
+		HOST1X_SYNC_SYNCPT_THRESH_CPU0_INT_STATUS(id / 32));
 }
 
 static int _host1x_free_syncpt_irq(struct host1x *host)
diff --git a/drivers/gpu/host1x/hw/syncpt_hw.c b/drivers/gpu/host1x/hw/syncpt_hw.c
index c93f74fcce72..7b0270d60742 100644
--- a/drivers/gpu/host1x/hw/syncpt_hw.c
+++ b/drivers/gpu/host1x/hw/syncpt_hw.c
@@ -89,7 +89,7 @@ static int syncpt_cpu_incr(struct host1x_syncpt *sp)
 	    host1x_syncpt_idle(sp))
 		return -EINVAL;
 
-	host1x_sync_writel(host, BIT_MASK(sp->id),
+	host1x_sync_writel(host, BIT(sp->id % 32),
 			   HOST1X_SYNC_SYNCPT_CPU_INCR(reg_offset));
 	wmb();
 
diff --git a/drivers/gpu/host1x/job.c b/drivers/gpu/host1x/job.c
index bee504406cfc..db509ab8874e 100644
--- a/drivers/gpu/host1x/job.c
+++ b/drivers/gpu/host1x/job.c
@@ -197,10 +197,6 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job)
 		}
 
 		phys_addr = host1x_bo_pin(reloc->target.bo, &sgt);
-		if (!phys_addr) {
-			err = -EINVAL;
-			goto unpin;
-		}
 
 		job->addr_phys[job->num_unpins] = phys_addr;
 		job->unpins[job->num_unpins].bo = reloc->target.bo;
@@ -225,10 +221,6 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job)
 		}
 
 		phys_addr = host1x_bo_pin(g->bo, &sgt);
-		if (!phys_addr) {
-			err = -EINVAL;
-			goto unpin;
-		}
 
 		if (!IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) && host->domain) {
 			for_each_sg(sgt->sgl, sg, sgt->nents, j)
diff --git a/drivers/gpu/ipu-v3/Kconfig b/drivers/gpu/ipu-v3/Kconfig
index 08766c6e7856..87a20b3dcf7a 100644
--- a/drivers/gpu/ipu-v3/Kconfig
+++ b/drivers/gpu/ipu-v3/Kconfig
@@ -1,6 +1,7 @@
 config IMX_IPUV3_CORE
 	tristate "IPUv3 core support"
 	depends on SOC_IMX5 || SOC_IMX6Q || ARCH_MULTIPLATFORM
+	depends on DRM || !DRM # if DRM=m, this can't be 'y'
 	select GENERIC_IRQ_CHIP
 	help
 	  Choose this if you have a i.MX5/6 system and want to use the Image
diff --git a/drivers/gpu/ipu-v3/ipu-common.c b/drivers/gpu/ipu-v3/ipu-common.c
index 960d816ad7f7..6a573d21d3cc 100644
--- a/drivers/gpu/ipu-v3/ipu-common.c
+++ b/drivers/gpu/ipu-v3/ipu-common.c
@@ -1217,8 +1217,8 @@ static int ipu_add_client_devices(struct ipu_soc *ipu, unsigned long ipu_base)
 		of_node = of_graph_get_port_by_id(dev->of_node, i);
 		if (!of_node) {
 			dev_info(dev,
-				 "no port@%d node in %s, not using %s%d\n",
-				 i, dev->of_node->full_name,
+				 "no port@%d node in %pOF, not using %s%d\n",
+				 i, dev->of_node,
 				 (i / 2) ? "DI" : "CSI", i % 2);
 			continue;
 		}
diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig
index 3cd60f460b61..0a3117cc29e7 100644
--- a/drivers/hid/Kconfig
+++ b/drivers/hid/Kconfig
@@ -924,7 +924,7 @@ config HID_UDRAW_PS3
 
 config HID_WACOM
 	tristate "Wacom Intuos/Graphire tablet support (USB)"
-	depends on HID
+	depends on USB_HID
 	select POWER_SUPPLY
 	select NEW_LEDS
 	select LEDS_CLASS
diff --git a/drivers/hid/hid-asus.c b/drivers/hid/hid-asus.c
index a4a3c38bc145..50c294be8324 100644
--- a/drivers/hid/hid-asus.c
+++ b/drivers/hid/hid-asus.c
@@ -29,6 +29,7 @@
 #include <linux/hid.h>
 #include <linux/module.h>
 #include <linux/input/mt.h>
+#include <linux/usb.h> /* For to_usb_interface for T100 touchpad intf check */
 
 #include "hid-ids.h"
 
@@ -38,24 +39,19 @@ MODULE_AUTHOR("Victor Vlasenko <victor.vlasenko@sysgears.com>");
 MODULE_AUTHOR("Frederik Wenigwieser <frederik.wenigwieser@gmail.com>");
 MODULE_DESCRIPTION("Asus HID Keyboard and TouchPad");
 
+#define T100_TPAD_INTF 2
+
+#define T100CHI_MOUSE_REPORT_ID 0x06
 #define FEATURE_REPORT_ID 0x0d
 #define INPUT_REPORT_ID 0x5d
 #define FEATURE_KBD_REPORT_ID 0x5a
-
-#define INPUT_REPORT_SIZE 28
 #define FEATURE_KBD_REPORT_SIZE 16
 
 #define SUPPORT_KBD_BACKLIGHT BIT(0)
 
-#define MAX_CONTACTS 5
-
-#define MAX_X 2794
-#define MAX_Y 1758
 #define MAX_TOUCH_MAJOR 8
 #define MAX_PRESSURE 128
 
-#define CONTACT_DATA_SIZE 5
-
 #define BTN_LEFT_MASK 0x01
 #define CONTACT_TOOL_TYPE_MASK 0x80
 #define CONTACT_X_MSB_MASK 0xf0
@@ -70,6 +66,7 @@ MODULE_DESCRIPTION("Asus HID Keyboard and TouchPad");
 #define QUIRK_NO_CONSUMER_USAGES	BIT(4)
 #define QUIRK_USE_KBD_BACKLIGHT		BIT(5)
 #define QUIRK_T100_KEYBOARD		BIT(6)
+#define QUIRK_T100CHI			BIT(7)
 
 #define I2C_KEYBOARD_QUIRKS			(QUIRK_FIX_NOTEBOOK_REPORT | \
 						 QUIRK_NO_INIT_REPORTS | \
@@ -88,19 +85,62 @@ struct asus_kbd_leds {
 	bool removed;
 };
 
+struct asus_touchpad_info {
+	int max_x;
+	int max_y;
+	int res_x;
+	int res_y;
+	int contact_size;
+	int max_contacts;
+};
+
 struct asus_drvdata {
 	unsigned long quirks;
 	struct input_dev *input;
 	struct asus_kbd_leds *kbd_backlight;
+	const struct asus_touchpad_info *tp;
 	bool enable_backlight;
 };
 
-static void asus_report_contact_down(struct input_dev *input,
+static const struct asus_touchpad_info asus_i2c_tp = {
+	.max_x = 2794,
+	.max_y = 1758,
+	.contact_size = 5,
+	.max_contacts = 5,
+};
+
+static const struct asus_touchpad_info asus_t100ta_tp = {
+	.max_x = 2240,
+	.max_y = 1120,
+	.res_x = 30, /* units/mm */
+	.res_y = 27, /* units/mm */
+	.contact_size = 5,
+	.max_contacts = 5,
+};
+
+static const struct asus_touchpad_info asus_t100chi_tp = {
+	.max_x = 2640,
+	.max_y = 1320,
+	.res_x = 31, /* units/mm */
+	.res_y = 29, /* units/mm */
+	.contact_size = 3,
+	.max_contacts = 4,
+};
+
+static void asus_report_contact_down(struct asus_drvdata *drvdat,
 		int toolType, u8 *data)
 {
-	int touch_major, pressure;
-	int x = (data[0] & CONTACT_X_MSB_MASK) << 4 | data[1];
-	int y = MAX_Y - ((data[0] & CONTACT_Y_MSB_MASK) << 8 | data[2]);
+	struct input_dev *input = drvdat->input;
+	int touch_major, pressure, x, y;
+
+	x = (data[0] & CONTACT_X_MSB_MASK) << 4 | data[1];
+	y = drvdat->tp->max_y - ((data[0] & CONTACT_Y_MSB_MASK) << 8 | data[2]);
+
+	input_report_abs(input, ABS_MT_POSITION_X, x);
+	input_report_abs(input, ABS_MT_POSITION_Y, y);
+
+	if (drvdat->tp->contact_size < 5)
+		return;
 
 	if (toolType == MT_TOOL_PALM) {
 		touch_major = MAX_TOUCH_MAJOR;
@@ -110,19 +150,20 @@ static void asus_report_contact_down(struct input_dev *input,
 		pressure = data[4] & CONTACT_PRESSURE_MASK;
 	}
 
-	input_report_abs(input, ABS_MT_POSITION_X, x);
-	input_report_abs(input, ABS_MT_POSITION_Y, y);
 	input_report_abs(input, ABS_MT_TOUCH_MAJOR, touch_major);
 	input_report_abs(input, ABS_MT_PRESSURE, pressure);
 }
 
 /* Required for Synaptics Palm Detection */
-static void asus_report_tool_width(struct input_dev *input)
+static void asus_report_tool_width(struct asus_drvdata *drvdat)
 {
-	struct input_mt *mt = input->mt;
+	struct input_mt *mt = drvdat->input->mt;
 	struct input_mt_slot *oldest;
 	int oldid, count, i;
 
+	if (drvdat->tp->contact_size < 5)
+		return;
+
 	oldest = NULL;
 	oldid = mt->trkid;
 	count = 0;
@@ -141,35 +182,42 @@ static void asus_report_tool_width(struct input_dev *input)
 	}
 
 	if (oldest) {
-		input_report_abs(input, ABS_TOOL_WIDTH,
+		input_report_abs(drvdat->input, ABS_TOOL_WIDTH,
 			input_mt_get_value(oldest, ABS_MT_TOUCH_MAJOR));
 	}
 }
 
-static void asus_report_input(struct input_dev *input, u8 *data)
+static int asus_report_input(struct asus_drvdata *drvdat, u8 *data, int size)
 {
-	int i;
+	int i, toolType = MT_TOOL_FINGER;
 	u8 *contactData = data + 2;
 
-	for (i = 0; i < MAX_CONTACTS; i++) {
+	if (size != 3 + drvdat->tp->contact_size * drvdat->tp->max_contacts)
+		return 0;
+
+	for (i = 0; i < drvdat->tp->max_contacts; i++) {
 		bool down = !!(data[1] & BIT(i+3));
-		int toolType = contactData[3] & CONTACT_TOOL_TYPE_MASK ?
+
+		if (drvdat->tp->contact_size >= 5)
+			toolType = contactData[3] & CONTACT_TOOL_TYPE_MASK ?
 						MT_TOOL_PALM : MT_TOOL_FINGER;
 
-		input_mt_slot(input, i);
-		input_mt_report_slot_state(input, toolType, down);
+		input_mt_slot(drvdat->input, i);
+		input_mt_report_slot_state(drvdat->input, toolType, down);
 
 		if (down) {
-			asus_report_contact_down(input, toolType, contactData);
-			contactData += CONTACT_DATA_SIZE;
+			asus_report_contact_down(drvdat, toolType, contactData);
+			contactData += drvdat->tp->contact_size;
 		}
 	}
 
-	input_report_key(input, BTN_LEFT, data[1] & BTN_LEFT_MASK);
-	asus_report_tool_width(input);
+	input_report_key(drvdat->input, BTN_LEFT, data[1] & BTN_LEFT_MASK);
+	asus_report_tool_width(drvdat);
+
+	input_mt_sync_frame(drvdat->input);
+	input_sync(drvdat->input);
 
-	input_mt_sync_frame(input);
-	input_sync(input);
+	return 1;
 }
 
 static int asus_raw_event(struct hid_device *hdev,
@@ -177,12 +225,8 @@ static int asus_raw_event(struct hid_device *hdev,
 {
 	struct asus_drvdata *drvdata = hid_get_drvdata(hdev);
 
-	if (drvdata->quirks & QUIRK_IS_MULTITOUCH &&
-					 data[0] == INPUT_REPORT_ID &&
-						size == INPUT_REPORT_SIZE) {
-		asus_report_input(drvdata->input, data);
-		return 1;
-	}
+	if (drvdata->tp && data[0] == INPUT_REPORT_ID)
+		return asus_report_input(drvdata, data, size);
 
 	return 0;
 }
@@ -334,19 +378,35 @@ static int asus_input_configured(struct hid_device *hdev, struct hid_input *hi)
 	struct input_dev *input = hi->input;
 	struct asus_drvdata *drvdata = hid_get_drvdata(hdev);
 
-	if (drvdata->quirks & QUIRK_IS_MULTITOUCH) {
+	/* T100CHI uses MULTI_INPUT, bind the touchpad to the mouse hid_input */
+	if (drvdata->quirks & QUIRK_T100CHI &&
+	    hi->report->id != T100CHI_MOUSE_REPORT_ID)
+		return 0;
+
+	if (drvdata->tp) {
 		int ret;
 
-		input_set_abs_params(input, ABS_MT_POSITION_X, 0, MAX_X, 0, 0);
-		input_set_abs_params(input, ABS_MT_POSITION_Y, 0, MAX_Y, 0, 0);
-		input_set_abs_params(input, ABS_TOOL_WIDTH, 0, MAX_TOUCH_MAJOR, 0, 0);
-		input_set_abs_params(input, ABS_MT_TOUCH_MAJOR, 0, MAX_TOUCH_MAJOR, 0, 0);
-		input_set_abs_params(input, ABS_MT_PRESSURE, 0, MAX_PRESSURE, 0, 0);
+		input_set_abs_params(input, ABS_MT_POSITION_X, 0,
+				     drvdata->tp->max_x, 0, 0);
+		input_set_abs_params(input, ABS_MT_POSITION_Y, 0,
+				     drvdata->tp->max_y, 0, 0);
+		input_abs_set_res(input, ABS_MT_POSITION_X, drvdata->tp->res_x);
+		input_abs_set_res(input, ABS_MT_POSITION_Y, drvdata->tp->res_y);
+
+		if (drvdata->tp->contact_size >= 5) {
+			input_set_abs_params(input, ABS_TOOL_WIDTH, 0,
+					     MAX_TOUCH_MAJOR, 0, 0);
+			input_set_abs_params(input, ABS_MT_TOUCH_MAJOR, 0,
+					     MAX_TOUCH_MAJOR, 0, 0);
+			input_set_abs_params(input, ABS_MT_PRESSURE, 0,
+					      MAX_PRESSURE, 0, 0);
+		}
 
 		__set_bit(BTN_LEFT, input->keybit);
 		__set_bit(INPUT_PROP_BUTTONPAD, input->propbit);
 
-		ret = input_mt_init_slots(input, MAX_CONTACTS, INPUT_MT_POINTER);
+		ret = input_mt_init_slots(input, drvdata->tp->max_contacts,
+					  INPUT_MT_POINTER);
 
 		if (ret) {
 			hid_err(hdev, "Asus input mt init slots failed: %d\n", ret);
@@ -378,6 +438,26 @@ static int asus_input_mapping(struct hid_device *hdev,
 		return -1;
 	}
 
+	/*
+	 * Ignore a bunch of bogus collections in the T100CHI descriptor.
+	 * This avoids a bunch of non-functional hid_input devices getting
+	 * created because of the T100CHI using HID_QUIRK_MULTI_INPUT.
+	 */
+	if (drvdata->quirks & QUIRK_T100CHI) {
+		if (field->application == (HID_UP_GENDESK | 0x0080) ||
+		    usage->hid == (HID_UP_GENDEVCTRLS | 0x0024) ||
+		    usage->hid == (HID_UP_GENDEVCTRLS | 0x0025) ||
+		    usage->hid == (HID_UP_GENDEVCTRLS | 0x0026))
+			return -1;
+		/*
+		 * We use the hid_input for the mouse report for the touchpad,
+		 * keep the left button, to avoid the core removing it.
+		 */
+		if (field->application == HID_GD_MOUSE &&
+		    usage->hid != (HID_UP_BUTTON | 1))
+			return -1;
+	}
+
 	/* ASUS-specific keyboard hotkeys */
 	if ((usage->hid & HID_USAGE_PAGE) == 0xff310000) {
 		set_bit(EV_REP, hi->input->evbit);
@@ -496,7 +576,7 @@ static int __maybe_unused asus_reset_resume(struct hid_device *hdev)
 {
 	struct asus_drvdata *drvdata = hid_get_drvdata(hdev);
 
-	if (drvdata->quirks & QUIRK_IS_MULTITOUCH)
+	if (drvdata->tp)
 		return asus_start_multitouch(hdev);
 
 	return 0;
@@ -517,6 +597,28 @@ static int asus_probe(struct hid_device *hdev, const struct hid_device_id *id)
 
 	drvdata->quirks = id->driver_data;
 
+	if (drvdata->quirks & QUIRK_IS_MULTITOUCH)
+		drvdata->tp = &asus_i2c_tp;
+
+	if (drvdata->quirks & QUIRK_T100_KEYBOARD) {
+		struct usb_interface *intf = to_usb_interface(hdev->dev.parent);
+
+		if (intf->altsetting->desc.bInterfaceNumber == T100_TPAD_INTF) {
+			drvdata->quirks = QUIRK_SKIP_INPUT_MAPPING;
+			drvdata->tp = &asus_t100ta_tp;
+		}
+	}
+
+	if (drvdata->quirks & QUIRK_T100CHI) {
+		/*
+		 * All functionality is on a single HID interface and for
+		 * userspace the touchpad must be a separate input_dev.
+		 */
+		hdev->quirks |= HID_QUIRK_MULTI_INPUT |
+				HID_QUIRK_NO_EMPTY_INPUT;
+		drvdata->tp = &asus_t100chi_tp;
+	}
+
 	if (drvdata->quirks & QUIRK_NO_INIT_REPORTS)
 		hdev->quirks |= HID_QUIRK_NO_INIT_REPORTS;
 
@@ -538,13 +640,13 @@ static int asus_probe(struct hid_device *hdev, const struct hid_device_id *id)
 		goto err_stop_hw;
 	}
 
-	if (drvdata->quirks & QUIRK_IS_MULTITOUCH) {
+	if (drvdata->tp) {
 		drvdata->input->name = "Asus TouchPad";
 	} else {
 		drvdata->input->name = "Asus Keyboard";
 	}
 
-	if (drvdata->quirks & QUIRK_IS_MULTITOUCH) {
+	if (drvdata->tp) {
 		ret = asus_start_multitouch(hdev);
 		if (ret)
 			goto err_stop_hw;
@@ -578,11 +680,34 @@ static __u8 *asus_report_fixup(struct hid_device *hdev, __u8 *rdesc,
 		hid_info(hdev, "Fixing up Asus notebook report descriptor\n");
 		rdesc[55] = 0xdd;
 	}
+	/* For the T100TA keyboard dock */
 	if (drvdata->quirks & QUIRK_T100_KEYBOARD &&
 		 *rsize == 76 && rdesc[73] == 0x81 && rdesc[74] == 0x01) {
 		hid_info(hdev, "Fixing up Asus T100 keyb report descriptor\n");
 		rdesc[74] &= ~HID_MAIN_ITEM_CONSTANT;
 	}
+	/* For the T100CHI keyboard dock */
+	if (drvdata->quirks & QUIRK_T100CHI &&
+		 *rsize == 403 && rdesc[388] == 0x09 && rdesc[389] == 0x76) {
+		/*
+		 * Change Usage (76h) to Usage Minimum (00h), Usage Maximum
+		 * (FFh) and clear the flags in the Input() byte.
+		 * Note the descriptor has a bogus 0 byte at the end so we
+		 * only need 1 extra byte.
+		 */
+		*rsize = 404;
+		rdesc = kmemdup(rdesc, *rsize, GFP_KERNEL);
+		if (!rdesc)
+			return NULL;
+
+		hid_info(hdev, "Fixing up T100CHI keyb report descriptor\n");
+		memmove(rdesc + 392, rdesc + 390, 12);
+		rdesc[388] = 0x19;
+		rdesc[389] = 0x00;
+		rdesc[390] = 0x29;
+		rdesc[391] = 0xff;
+		rdesc[402] = 0x00;
+	}
 
 	return rdesc;
 }
@@ -602,6 +727,9 @@ static const struct hid_device_id asus_devices[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_CHICONY, USB_DEVICE_ID_ASUS_AK1D) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_TURBOX, USB_DEVICE_ID_ASUS_MD_5110) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_JESS, USB_DEVICE_ID_ASUS_MD_5112) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ASUSTEK,
+		USB_DEVICE_ID_ASUSTEK_T100CHI_KEYBOARD), QUIRK_T100CHI },
+
 	{ }
 };
 MODULE_DEVICE_TABLE(hid, asus_devices);
diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c
index 9017dcc14502..9bc91160819b 100644
--- a/drivers/hid/hid-core.c
+++ b/drivers/hid/hid-core.c
@@ -1982,6 +1982,7 @@ static const struct hid_device_id hid_have_special_driver[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_T100_KEYBOARD) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_JESS, USB_DEVICE_ID_ASUS_MD_5112) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_TURBOX, USB_DEVICE_ID_ASUS_MD_5110) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_T100CHI_KEYBOARD) },
 #endif
 #if IS_ENABLED(CONFIG_HID_AUREAL)
 	{ HID_USB_DEVICE(USB_VENDOR_ID_AUREAL, USB_DEVICE_ID_AUREAL_W01RN) },
@@ -2487,11 +2488,9 @@ static int hid_device_probe(struct device *dev)
 	const struct hid_device_id *id;
 	int ret = 0;
 
-	if (down_interruptible(&hdev->driver_lock))
-		return -EINTR;
 	if (down_interruptible(&hdev->driver_input_lock)) {
 		ret = -EINTR;
-		goto unlock_driver_lock;
+		goto end;
 	}
 	hdev->io_started = false;
 
@@ -2518,8 +2517,7 @@ static int hid_device_probe(struct device *dev)
 unlock:
 	if (!hdev->io_started)
 		up(&hdev->driver_input_lock);
-unlock_driver_lock:
-	up(&hdev->driver_lock);
+end:
 	return ret;
 }
 
@@ -2529,11 +2527,9 @@ static int hid_device_remove(struct device *dev)
 	struct hid_driver *hdrv;
 	int ret = 0;
 
-	if (down_interruptible(&hdev->driver_lock))
-		return -EINTR;
 	if (down_interruptible(&hdev->driver_input_lock)) {
 		ret = -EINTR;
-		goto unlock_driver_lock;
+		goto end;
 	}
 	hdev->io_started = false;
 
@@ -2549,8 +2545,7 @@ static int hid_device_remove(struct device *dev)
 
 	if (!hdev->io_started)
 		up(&hdev->driver_input_lock);
-unlock_driver_lock:
-	up(&hdev->driver_lock);
+end:
 	return ret;
 }
 
@@ -3008,7 +3003,6 @@ struct hid_device *hid_allocate_device(void)
 	init_waitqueue_head(&hdev->debug_wait);
 	INIT_LIST_HEAD(&hdev->debug_list);
 	spin_lock_init(&hdev->debug_list_lock);
-	sema_init(&hdev->driver_lock, 1);
 	sema_init(&hdev->driver_input_lock, 1);
 	mutex_init(&hdev->ll_open_lock);
 
diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h
index c9ba4c6db74c..b397a14ab970 100644
--- a/drivers/hid/hid-ids.h
+++ b/drivers/hid/hid-ids.h
@@ -176,6 +176,8 @@
 #define USB_DEVICE_ID_ASUSTEK_LCM	0x1726
 #define USB_DEVICE_ID_ASUSTEK_LCM2	0x175b
 #define USB_DEVICE_ID_ASUSTEK_T100_KEYBOARD	0x17e0
+#define USB_DEVICE_ID_ASUSTEK_T100CHI_KEYBOARD	0x8502
+#define USB_DEVICE_ID_ASUSTEK_T304_KEYBOARD	0x184a
 #define USB_DEVICE_ID_ASUSTEK_I2C_KEYBOARD	0x8585
 #define USB_DEVICE_ID_ASUSTEK_I2C_TOUCHPAD	0x0101
 #define USB_DEVICE_ID_ASUSTEK_ROG_KEYBOARD1 0x1854
@@ -666,7 +668,8 @@
 #define USB_VENDOR_ID_LOGITECH		0x046d
 #define USB_DEVICE_ID_LOGITECH_AUDIOHUB 0x0a0e
 #define USB_DEVICE_ID_LOGITECH_T651	0xb00c
-#define USB_DEVICE_ID_LOGITECH_C077	0xc007
+#define USB_DEVICE_ID_LOGITECH_C007	0xc007
+#define USB_DEVICE_ID_LOGITECH_C077	0xc077
 #define USB_DEVICE_ID_LOGITECH_RECEIVER	0xc101
 #define USB_DEVICE_ID_LOGITECH_HARMONY_FIRST  0xc110
 #define USB_DEVICE_ID_LOGITECH_HARMONY_LAST 0xc14f
diff --git a/drivers/hid/hid-input.c b/drivers/hid/hid-input.c
index ccdff1ee1f0c..199f6a01fc62 100644
--- a/drivers/hid/hid-input.c
+++ b/drivers/hid/hid-input.c
@@ -340,13 +340,45 @@ static unsigned find_battery_quirk(struct hid_device *hdev)
 	return quirks;
 }
 
+static int hidinput_scale_battery_capacity(struct hid_device *dev,
+					   int value)
+{
+	if (dev->battery_min < dev->battery_max &&
+	    value >= dev->battery_min && value <= dev->battery_max)
+		value = ((value - dev->battery_min) * 100) /
+			(dev->battery_max - dev->battery_min);
+
+	return value;
+}
+
+static int hidinput_query_battery_capacity(struct hid_device *dev)
+{
+	u8 *buf;
+	int ret;
+
+	buf = kmalloc(2, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	ret = hid_hw_raw_request(dev, dev->battery_report_id, buf, 2,
+				 dev->battery_report_type, HID_REQ_GET_REPORT);
+	if (ret != 2) {
+		kfree(buf);
+		return -ENODATA;
+	}
+
+	ret = hidinput_scale_battery_capacity(dev, buf[1]);
+	kfree(buf);
+	return ret;
+}
+
 static int hidinput_get_battery_property(struct power_supply *psy,
 					 enum power_supply_property prop,
 					 union power_supply_propval *val)
 {
 	struct hid_device *dev = power_supply_get_drvdata(psy);
+	int value;
 	int ret = 0;
-	__u8 *buf;
 
 	switch (prop) {
 	case POWER_SUPPLY_PROP_PRESENT:
@@ -355,29 +387,15 @@ static int hidinput_get_battery_property(struct power_supply *psy,
 		break;
 
 	case POWER_SUPPLY_PROP_CAPACITY:
-
-		buf = kmalloc(2 * sizeof(__u8), GFP_KERNEL);
-		if (!buf) {
-			ret = -ENOMEM;
-			break;
-		}
-		ret = hid_hw_raw_request(dev, dev->battery_report_id, buf, 2,
-					 dev->battery_report_type,
-					 HID_REQ_GET_REPORT);
-
-		if (ret != 2) {
-			ret = -ENODATA;
-			kfree(buf);
-			break;
+		if (dev->battery_report_type == HID_FEATURE_REPORT) {
+			value = hidinput_query_battery_capacity(dev);
+			if (value < 0)
+				return value;
+		} else  {
+			value = dev->battery_capacity;
 		}
-		ret = 0;
 
-		if (dev->battery_min < dev->battery_max &&
-		    buf[1] >= dev->battery_min &&
-		    buf[1] <= dev->battery_max)
-			val->intval = (100 * (buf[1] - dev->battery_min)) /
-				(dev->battery_max - dev->battery_min);
-		kfree(buf);
+		val->intval = value;
 		break;
 
 	case POWER_SUPPLY_PROP_MODEL_NAME:
@@ -385,7 +403,22 @@ static int hidinput_get_battery_property(struct power_supply *psy,
 		break;
 
 	case POWER_SUPPLY_PROP_STATUS:
-		val->intval = POWER_SUPPLY_STATUS_DISCHARGING;
+		if (!dev->battery_reported &&
+		    dev->battery_report_type == HID_FEATURE_REPORT) {
+			value = hidinput_query_battery_capacity(dev);
+			if (value < 0)
+				return value;
+
+			dev->battery_capacity = value;
+			dev->battery_reported = true;
+		}
+
+		if (!dev->battery_reported)
+			val->intval = POWER_SUPPLY_STATUS_UNKNOWN;
+		else if (dev->battery_capacity == 100)
+			val->intval = POWER_SUPPLY_STATUS_FULL;
+		else
+			val->intval = POWER_SUPPLY_STATUS_DISCHARGING;
 		break;
 
 	case POWER_SUPPLY_PROP_SCOPE:
@@ -400,18 +433,16 @@ static int hidinput_get_battery_property(struct power_supply *psy,
 	return ret;
 }
 
-static bool hidinput_setup_battery(struct hid_device *dev, unsigned report_type, struct hid_field *field)
+static int hidinput_setup_battery(struct hid_device *dev, unsigned report_type, struct hid_field *field)
 {
-	struct power_supply_desc *psy_desc = NULL;
+	struct power_supply_desc *psy_desc;
 	struct power_supply_config psy_cfg = { .drv_data = dev, };
 	unsigned quirks;
 	s32 min, max;
+	int error;
 
-	if (field->usage->hid != HID_DC_BATTERYSTRENGTH)
-		return false;	/* no match */
-
-	if (dev->battery != NULL)
-		goto out;	/* already initialized? */
+	if (dev->battery)
+		return 0;	/* already initialized? */
 
 	quirks = find_battery_quirk(dev);
 
@@ -419,16 +450,18 @@ static bool hidinput_setup_battery(struct hid_device *dev, unsigned report_type,
 		dev->bus, dev->vendor, dev->product, dev->version, quirks);
 
 	if (quirks & HID_BATTERY_QUIRK_IGNORE)
-		goto out;
+		return 0;
 
 	psy_desc = kzalloc(sizeof(*psy_desc), GFP_KERNEL);
-	if (psy_desc == NULL)
-		goto out;
-
-	psy_desc->name = kasprintf(GFP_KERNEL, "hid-%s-battery", dev->uniq);
-	if (psy_desc->name == NULL) {
-		kfree(psy_desc);
-		goto out;
+	if (!psy_desc)
+		return -ENOMEM;
+
+	psy_desc->name = kasprintf(GFP_KERNEL, "hid-%s-battery",
+				   strlen(dev->uniq) ?
+					dev->uniq : dev_name(&dev->dev));
+	if (!psy_desc->name) {
+		error = -ENOMEM;
+		goto err_free_mem;
 	}
 
 	psy_desc->type = POWER_SUPPLY_TYPE_BATTERY;
@@ -455,17 +488,20 @@ static bool hidinput_setup_battery(struct hid_device *dev, unsigned report_type,
 
 	dev->battery = power_supply_register(&dev->dev, psy_desc, &psy_cfg);
 	if (IS_ERR(dev->battery)) {
-		hid_warn(dev, "can't register power supply: %ld\n",
-				PTR_ERR(dev->battery));
-		kfree(psy_desc->name);
-		kfree(psy_desc);
-		dev->battery = NULL;
-	} else {
-		power_supply_powers(dev->battery, &dev->dev);
+		error = PTR_ERR(dev->battery);
+		hid_warn(dev, "can't register power supply: %d\n", error);
+		goto err_free_name;
 	}
 
-out:
-	return true;
+	power_supply_powers(dev->battery, &dev->dev);
+	return 0;
+
+err_free_name:
+	kfree(psy_desc->name);
+err_free_mem:
+	kfree(psy_desc);
+	dev->battery = NULL;
+	return error;
 }
 
 static void hidinput_cleanup_battery(struct hid_device *dev)
@@ -481,16 +517,39 @@ static void hidinput_cleanup_battery(struct hid_device *dev)
 	kfree(psy_desc);
 	dev->battery = NULL;
 }
+
+static void hidinput_update_battery(struct hid_device *dev, int value)
+{
+	int capacity;
+
+	if (!dev->battery)
+		return;
+
+	if (value == 0 || value < dev->battery_min || value > dev->battery_max)
+		return;
+
+	capacity = hidinput_scale_battery_capacity(dev, value);
+
+	if (!dev->battery_reported || capacity != dev->battery_capacity) {
+		dev->battery_capacity = capacity;
+		dev->battery_reported = true;
+		power_supply_changed(dev->battery);
+	}
+}
 #else  /* !CONFIG_HID_BATTERY_STRENGTH */
-static bool hidinput_setup_battery(struct hid_device *dev, unsigned report_type,
-				   struct hid_field *field)
+static int hidinput_setup_battery(struct hid_device *dev, unsigned report_type,
+				  struct hid_field *field)
 {
-	return false;
+	return 0;
 }
 
 static void hidinput_cleanup_battery(struct hid_device *dev)
 {
 }
+
+static void hidinput_update_battery(struct hid_device *dev, int value)
+{
+}
 #endif	/* CONFIG_HID_BATTERY_STRENGTH */
 
 static void hidinput_configure_usage(struct hid_input *hidinput, struct hid_field *field,
@@ -710,6 +769,11 @@ static void hidinput_configure_usage(struct hid_input *hidinput, struct hid_fiel
 			}
 			break;
 
+		case 0x3b: /* Battery Strength */
+			hidinput_setup_battery(device, HID_INPUT_REPORT, field);
+			usage->type = EV_PWR;
+			goto ignore;
+
 		case 0x3c: /* Invert */
 			map_key_clear(BTN_TOOL_RUBBER);
 			break;
@@ -944,11 +1008,13 @@ static void hidinput_configure_usage(struct hid_input *hidinput, struct hid_fiel
 		break;
 
 	case HID_UP_GENDEVCTRLS:
-		if (hidinput_setup_battery(device, HID_INPUT_REPORT, field))
+		switch (usage->hid) {
+		case HID_DC_BATTERYSTRENGTH:
+			hidinput_setup_battery(device, HID_INPUT_REPORT, field);
+			usage->type = EV_PWR;
 			goto ignore;
-		else
-			goto unknown;
-		break;
+		}
+		goto unknown;
 
 	case HID_UP_HPVENDOR:	/* Reported on a Dutch layout HP5308 */
 		set_bit(EV_REP, input->evbit);
@@ -1031,7 +1097,6 @@ mapped:
 	if (usage->code > max)
 		goto ignore;
 
-
 	if (usage->type == EV_ABS) {
 
 		int a = field->logical_minimum;
@@ -1090,14 +1155,19 @@ void hidinput_hid_event(struct hid_device *hid, struct hid_field *field, struct
 	struct input_dev *input;
 	unsigned *quirks = &hid->quirks;
 
-	if (!field->hidinput)
+	if (!usage->type)
 		return;
 
-	input = field->hidinput->input;
+	if (usage->type == EV_PWR) {
+		hidinput_update_battery(hid, value);
+		return;
+	}
 
-	if (!usage->type)
+	if (!field->hidinput)
 		return;
 
+	input = field->hidinput->input;
+
 	if (usage->hat_min < usage->hat_max || usage->hat_dir) {
 		int hat_dir = usage->hat_dir;
 		if (!hat_dir)
@@ -1373,6 +1443,7 @@ static void report_features(struct hid_device *hid)
 	struct hid_driver *drv = hid->driver;
 	struct hid_report_enum *rep_enum;
 	struct hid_report *rep;
+	struct hid_usage *usage;
 	int i, j;
 
 	rep_enum = &hid->report_enum[HID_FEATURE_REPORT];
@@ -1383,12 +1454,15 @@ static void report_features(struct hid_device *hid)
 				continue;
 
 			for (j = 0; j < rep->field[i]->maxusage; j++) {
+				usage = &rep->field[i]->usage[j];
+
 				/* Verify if Battery Strength feature is available */
-				hidinput_setup_battery(hid, HID_FEATURE_REPORT, rep->field[i]);
+				if (usage->hid == HID_DC_BATTERYSTRENGTH)
+					hidinput_setup_battery(hid, HID_FEATURE_REPORT,
+							       rep->field[i]);
 
 				if (drv->feature_mapping)
-					drv->feature_mapping(hid, rep->field[i],
-							     rep->field[i]->usage + j);
+					drv->feature_mapping(hid, rep->field[i], usage);
 			}
 		}
 }
diff --git a/drivers/hid/hid-logitech-hidpp.c b/drivers/hid/hid-logitech-hidpp.c
index 501e16a9227d..614054af904a 100644
--- a/drivers/hid/hid-logitech-hidpp.c
+++ b/drivers/hid/hid-logitech-hidpp.c
@@ -2926,7 +2926,7 @@ static struct attribute *sysfs_attrs[] = {
 	NULL
 };
 
-static struct attribute_group ps_attribute_group = {
+static const struct attribute_group ps_attribute_group = {
 	.attrs = sysfs_attrs
 };
 
diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c
index aff20f4b6d97..440b999304a5 100644
--- a/drivers/hid/hid-multitouch.c
+++ b/drivers/hid/hid-multitouch.c
@@ -72,6 +72,7 @@ MODULE_LICENSE("GPL");
 #define MT_QUIRK_FIX_CONST_CONTACT_ID	BIT(14)
 #define MT_QUIRK_TOUCH_SIZE_SCALING	BIT(15)
 #define MT_QUIRK_STICKY_FINGERS		BIT(16)
+#define MT_QUIRK_ASUS_CUSTOM_UP		BIT(17)
 
 #define MT_INPUTMODE_TOUCHSCREEN	0x02
 #define MT_INPUTMODE_TOUCHPAD		0x03
@@ -169,6 +170,7 @@ static void mt_post_parse(struct mt_device *td);
 #define MT_CLS_GENERALTOUCH_TWOFINGERS		0x0108
 #define MT_CLS_GENERALTOUCH_PWT_TENFINGERS	0x0109
 #define MT_CLS_LG				0x010a
+#define MT_CLS_ASUS				0x010b
 #define MT_CLS_VTL				0x0110
 #define MT_CLS_GOOGLE				0x0111
 
@@ -290,6 +292,10 @@ static struct mt_class mt_classes[] = {
 			MT_QUIRK_IGNORE_DUPLICATES |
 			MT_QUIRK_HOVERING |
 			MT_QUIRK_CONTACT_CNT_ACCURATE },
+	{ .name = MT_CLS_ASUS,
+		.quirks = MT_QUIRK_ALWAYS_VALID |
+			MT_QUIRK_CONTACT_CNT_ACCURATE |
+			MT_QUIRK_ASUS_CUSTOM_UP },
 	{ .name = MT_CLS_VTL,
 		.quirks = MT_QUIRK_ALWAYS_VALID |
 			MT_QUIRK_CONTACT_CNT_ACCURATE |
@@ -341,7 +347,7 @@ static struct attribute *sysfs_attrs[] = {
 	NULL
 };
 
-static struct attribute_group mt_attribute_group = {
+static const struct attribute_group mt_attribute_group = {
 	.attrs = sysfs_attrs
 };
 
@@ -905,6 +911,8 @@ static int mt_touch_input_configured(struct hid_device *hdev,
 	return 0;
 }
 
+#define mt_map_key_clear(c)	hid_map_usage_clear(hi, usage, bit, \
+						    max, EV_KEY, (c))
 static int mt_input_mapping(struct hid_device *hdev, struct hid_input *hi,
 		struct hid_field *field, struct hid_usage *usage,
 		unsigned long **bit, int *max)
@@ -922,10 +930,36 @@ static int mt_input_mapping(struct hid_device *hdev, struct hid_input *hi,
 	    field->application != HID_DG_PEN &&
 	    field->application != HID_DG_TOUCHPAD &&
 	    field->application != HID_GD_KEYBOARD &&
-	    field->application != HID_CP_CONSUMER_CONTROL)
+	    field->application != HID_CP_CONSUMER_CONTROL &&
+	    field->application != HID_GD_WIRELESS_RADIO_CTLS &&
+	    !(field->application == HID_VD_ASUS_CUSTOM_MEDIA_KEYS &&
+	      td->mtclass.quirks & MT_QUIRK_ASUS_CUSTOM_UP))
 		return -1;
 
 	/*
+	 * Some Asus keyboard+touchpad devices have the hotkeys defined in the
+	 * touchpad report descriptor. We need to treat these as an array to
+	 * map usages to input keys.
+	 */
+	if (field->application == HID_VD_ASUS_CUSTOM_MEDIA_KEYS &&
+	    td->mtclass.quirks & MT_QUIRK_ASUS_CUSTOM_UP &&
+	    (usage->hid & HID_USAGE_PAGE) == HID_UP_CUSTOM) {
+		set_bit(EV_REP, hi->input->evbit);
+		if (field->flags & HID_MAIN_ITEM_VARIABLE)
+			field->flags &= ~HID_MAIN_ITEM_VARIABLE;
+		switch (usage->hid & HID_USAGE) {
+		case 0x10: mt_map_key_clear(KEY_BRIGHTNESSDOWN);	break;
+		case 0x20: mt_map_key_clear(KEY_BRIGHTNESSUP);		break;
+		case 0x35: mt_map_key_clear(KEY_DISPLAY_OFF);		break;
+		case 0x6b: mt_map_key_clear(KEY_F21);			break;
+		case 0x6c: mt_map_key_clear(KEY_SLEEP);			break;
+		default:
+			return -1;
+		}
+		return 1;
+	}
+
+	/*
 	 * some egalax touchscreens have "application == HID_DG_TOUCHSCREEN"
 	 * for the stylus.
 	 * The check for mt_report_id ensures we don't process
@@ -1133,6 +1167,12 @@ static int mt_input_configured(struct hid_device *hdev, struct hid_input *hi)
 		case HID_CP_CONSUMER_CONTROL:
 			suffix = "Consumer Control";
 			break;
+		case HID_GD_WIRELESS_RADIO_CTLS:
+			suffix = "Wireless Radio Control";
+			break;
+		case HID_VD_ASUS_CUSTOM_MEDIA_KEYS:
+			suffix = "Custom Media Keys";
+			break;
 		default:
 			suffix = "UNKNOWN";
 			break;
@@ -1384,6 +1424,12 @@ static const struct hid_device_id mt_devices[] = {
 		MT_USB_DEVICE(USB_VENDOR_ID_ANTON,
 			USB_DEVICE_ID_ANTON_TOUCH_PAD) },
 
+	/* Asus T304UA */
+	{ .driver_data = MT_CLS_ASUS,
+		HID_DEVICE(BUS_USB, HID_GROUP_MULTITOUCH_WIN_8,
+			USB_VENDOR_ID_ASUSTEK,
+			USB_DEVICE_ID_ASUSTEK_T304_KEYBOARD) },
+
 	/* Atmel panels */
 	{ .driver_data = MT_CLS_SERIAL,
 		MT_USB_DEVICE(USB_VENDOR_ID_ATMEL,
diff --git a/drivers/hid/hid-ntrig.c b/drivers/hid/hid-ntrig.c
index 1b0084d4af2e..3d121d8ee980 100644
--- a/drivers/hid/hid-ntrig.c
+++ b/drivers/hid/hid-ntrig.c
@@ -445,7 +445,7 @@ static struct attribute *sysfs_attrs[] = {
 	NULL
 };
 
-static struct attribute_group ntrig_attribute_group = {
+static const struct attribute_group ntrig_attribute_group = {
 	.attrs = sysfs_attrs
 };
 
diff --git a/drivers/hid/hid-picolcd_cir.c b/drivers/hid/hid-picolcd_cir.c
index 8ffbb6f65a65..32747b7f917e 100644
--- a/drivers/hid/hid-picolcd_cir.c
+++ b/drivers/hid/hid-picolcd_cir.c
@@ -113,10 +113,10 @@ int picolcd_init_cir(struct picolcd_data *data, struct hid_report *report)
 		return -ENOMEM;
 
 	rdev->priv             = data;
-	rdev->allowed_protocols = RC_BIT_ALL_IR_DECODER;
+	rdev->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
 	rdev->open             = picolcd_cir_open;
 	rdev->close            = picolcd_cir_close;
-	rdev->input_name       = data->hdev->name;
+	rdev->device_name      = data->hdev->name;
 	rdev->input_phys       = data->hdev->phys;
 	rdev->input_id.bustype = data->hdev->bus;
 	rdev->input_id.vendor  = data->hdev->vendor;
diff --git a/drivers/hid/hid-prodikeys.c b/drivers/hid/hid-prodikeys.c
index f095bf8a3aa9..49c4bd34b3c5 100644
--- a/drivers/hid/hid-prodikeys.c
+++ b/drivers/hid/hid-prodikeys.c
@@ -593,7 +593,7 @@ static void pcmidi_in_trigger(struct snd_rawmidi_substream *substream, int up)
 	pm->in_triggered = up;
 }
 
-static struct snd_rawmidi_ops pcmidi_in_ops = {
+static const struct snd_rawmidi_ops pcmidi_in_ops = {
 	.open = pcmidi_in_open,
 	.close = pcmidi_in_close,
 	.trigger = pcmidi_in_trigger
diff --git a/drivers/hid/hid-sensor-custom.c b/drivers/hid/hid-sensor-custom.c
index 3a84aaf1418b..0bcf041368c7 100644
--- a/drivers/hid/hid-sensor-custom.c
+++ b/drivers/hid/hid-sensor-custom.c
@@ -276,7 +276,7 @@ static struct attribute *enable_sensor_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group enable_sensor_attr_group = {
+static const struct attribute_group enable_sensor_attr_group = {
 	.attrs = enable_sensor_attrs,
 };
 
@@ -823,7 +823,7 @@ static int hid_sensor_custom_remove(struct platform_device *pdev)
 	return 0;
 }
 
-static struct platform_device_id hid_sensor_custom_ids[] = {
+static const struct platform_device_id hid_sensor_custom_ids[] = {
 	{
 		.name = "HID-SENSOR-2000e1",
 	},
diff --git a/drivers/hid/hid-sensor-hub.c b/drivers/hid/hid-sensor-hub.c
index 4ef73374a8f9..25363fc571bc 100644
--- a/drivers/hid/hid-sensor-hub.c
+++ b/drivers/hid/hid-sensor-hub.c
@@ -579,54 +579,6 @@ void sensor_hub_device_close(struct hid_sensor_hub_device *hsdev)
 }
 EXPORT_SYMBOL_GPL(sensor_hub_device_close);
 
-static __u8 *sensor_hub_report_fixup(struct hid_device *hdev, __u8 *rdesc,
-		unsigned int *rsize)
-{
-	int index;
-	struct sensor_hub_data *sd =  hid_get_drvdata(hdev);
-	unsigned char report_block[] = {
-				0x0a,  0x16, 0x03, 0x15, 0x00, 0x25, 0x05};
-	unsigned char power_block[] = {
-				0x0a,  0x19, 0x03, 0x15, 0x00, 0x25, 0x05};
-
-	if (!(sd->quirks & HID_SENSOR_HUB_ENUM_QUIRK)) {
-		hid_dbg(hdev, "No Enum quirks\n");
-		return rdesc;
-	}
-
-	/* Looks for power and report state usage id and force to 1 */
-	for (index = 0; index < *rsize; ++index) {
-		if (((*rsize - index) > sizeof(report_block)) &&
-			!memcmp(&rdesc[index], report_block,
-						sizeof(report_block))) {
-			rdesc[index + 4] = 0x01;
-			index += sizeof(report_block);
-		}
-		if (((*rsize - index) > sizeof(power_block)) &&
-			!memcmp(&rdesc[index], power_block,
-						sizeof(power_block))) {
-			rdesc[index + 4] = 0x01;
-			index += sizeof(power_block);
-		}
-	}
-
-	/* Checks if the report descriptor of Thinkpad Helix 2 has a logical
-	 * minimum for magnetic flux axis greater than the maximum */
-	if (hdev->product == USB_DEVICE_ID_TEXAS_INSTRUMENTS_LENOVO_YOGA &&
-		*rsize == 2558 && rdesc[913] == 0x17 && rdesc[914] == 0x40 &&
-		rdesc[915] == 0x81 && rdesc[916] == 0x08 &&
-		rdesc[917] == 0x00 && rdesc[918] == 0x27 &&
-		rdesc[921] == 0x07 && rdesc[922] == 0x00) {
-		/* Sets negative logical minimum for mag x, y and z */
-		rdesc[914] = rdesc[935] = rdesc[956] = 0xc0;
-		rdesc[915] = rdesc[936] = rdesc[957] = 0x7e;
-		rdesc[916] = rdesc[937] = rdesc[958] = 0xf7;
-		rdesc[917] = rdesc[938] = rdesc[959] = 0xff;
-	}
-
-	return rdesc;
-}
-
 static int sensor_hub_probe(struct hid_device *hdev,
 				const struct hid_device_id *id)
 {
@@ -778,51 +730,6 @@ static void sensor_hub_remove(struct hid_device *hdev)
 }
 
 static const struct hid_device_id sensor_hub_devices[] = {
-	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, USB_VENDOR_ID_INTEL_0,
-			USB_DEVICE_ID_INTEL_HID_SENSOR_0),
-			.driver_data = HID_SENSOR_HUB_ENUM_QUIRK},
-	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, USB_VENDOR_ID_INTEL_1,
-			USB_DEVICE_ID_INTEL_HID_SENSOR_0),
-			.driver_data = HID_SENSOR_HUB_ENUM_QUIRK},
-	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, USB_VENDOR_ID_INTEL_1,
-			USB_DEVICE_ID_INTEL_HID_SENSOR_1),
-			.driver_data = HID_SENSOR_HUB_ENUM_QUIRK},
-	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, USB_VENDOR_ID_MICROSOFT,
-			USB_DEVICE_ID_MS_SURFACE_PRO_2),
-			.driver_data = HID_SENSOR_HUB_ENUM_QUIRK},
-	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, USB_VENDOR_ID_MICROSOFT,
-			USB_DEVICE_ID_MS_TOUCH_COVER_2),
-			.driver_data = HID_SENSOR_HUB_ENUM_QUIRK},
-	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, USB_VENDOR_ID_MICROSOFT,
-			USB_DEVICE_ID_MS_TYPE_COVER_2),
-			.driver_data = HID_SENSOR_HUB_ENUM_QUIRK},
-	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, USB_VENDOR_ID_MICROSOFT,
-			0x07bd), /* Microsoft Surface 3 */
-			.driver_data = HID_SENSOR_HUB_ENUM_QUIRK},
-	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, USB_VENDOR_ID_MICROCHIP,
-			0x0f01), /* MM7150 */
-			.driver_data = HID_SENSOR_HUB_ENUM_QUIRK},
-	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, USB_VENDOR_ID_STM_0,
-			USB_DEVICE_ID_STM_HID_SENSOR),
-			.driver_data = HID_SENSOR_HUB_ENUM_QUIRK},
-	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, USB_VENDOR_ID_STM_0,
-			USB_DEVICE_ID_STM_HID_SENSOR_1),
-			.driver_data = HID_SENSOR_HUB_ENUM_QUIRK},
-	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, USB_VENDOR_ID_TEXAS_INSTRUMENTS,
-			USB_DEVICE_ID_TEXAS_INSTRUMENTS_LENOVO_YOGA),
-			.driver_data = HID_SENSOR_HUB_ENUM_QUIRK},
-	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, USB_VENDOR_ID_ITE,
-			USB_DEVICE_ID_ITE_LENOVO_YOGA),
-			.driver_data = HID_SENSOR_HUB_ENUM_QUIRK},
-	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, USB_VENDOR_ID_ITE,
-			USB_DEVICE_ID_ITE_LENOVO_YOGA2),
-			.driver_data = HID_SENSOR_HUB_ENUM_QUIRK},
-	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, USB_VENDOR_ID_ITE,
-			USB_DEVICE_ID_ITE_LENOVO_YOGA900),
-			.driver_data = HID_SENSOR_HUB_ENUM_QUIRK},
-	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, USB_VENDOR_ID_INTEL_0,
-			0x22D8),
-			.driver_data = HID_SENSOR_HUB_ENUM_QUIRK},
 	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_SENSOR_HUB, HID_ANY_ID,
 		     HID_ANY_ID) },
 	{ }
@@ -835,7 +742,6 @@ static struct hid_driver sensor_hub_driver = {
 	.probe = sensor_hub_probe,
 	.remove = sensor_hub_remove,
 	.raw_event = sensor_hub_raw_event,
-	.report_fixup = sensor_hub_report_fixup,
 #ifdef CONFIG_PM
 	.suspend = sensor_hub_suspend,
 	.resume = sensor_hub_resume,
diff --git a/drivers/hid/i2c-hid/i2c-hid.c b/drivers/hid/i2c-hid/i2c-hid.c
index 046f692fd0a2..77396145d2d0 100644
--- a/drivers/hid/i2c-hid/i2c-hid.c
+++ b/drivers/hid/i2c-hid/i2c-hid.c
@@ -780,7 +780,7 @@ static int i2c_hid_power(struct hid_device *hid, int lvl)
 	return 0;
 }
 
-static struct hid_ll_driver i2c_hid_ll_driver = {
+struct hid_ll_driver i2c_hid_ll_driver = {
 	.parse = i2c_hid_parse,
 	.start = i2c_hid_start,
 	.stop = i2c_hid_stop,
@@ -790,6 +790,7 @@ static struct hid_ll_driver i2c_hid_ll_driver = {
 	.output_report = i2c_hid_output_report,
 	.raw_request = i2c_hid_raw_request,
 };
+EXPORT_SYMBOL_GPL(i2c_hid_ll_driver);
 
 static int i2c_hid_init_irq(struct i2c_client *client)
 {
diff --git a/drivers/hid/uhid.c b/drivers/hid/uhid.c
index 7f8ff39ed44b..6f819f144cb4 100644
--- a/drivers/hid/uhid.c
+++ b/drivers/hid/uhid.c
@@ -369,7 +369,7 @@ static int uhid_hid_output_report(struct hid_device *hid, __u8 *buf,
 	return uhid_hid_output_raw(hid, buf, count, HID_OUTPUT_REPORT);
 }
 
-static struct hid_ll_driver uhid_hid_driver = {
+struct hid_ll_driver uhid_hid_driver = {
 	.start = uhid_hid_start,
 	.stop = uhid_hid_stop,
 	.open = uhid_hid_open,
@@ -378,6 +378,7 @@ static struct hid_ll_driver uhid_hid_driver = {
 	.raw_request = uhid_hid_raw_request,
 	.output_report = uhid_hid_output_report,
 };
+EXPORT_SYMBOL_GPL(uhid_hid_driver);
 
 #ifdef CONFIG_COMPAT
 
diff --git a/drivers/hid/usbhid/hid-core.c b/drivers/hid/usbhid/hid-core.c
index c008847e0b20..089bad8a9a21 100644
--- a/drivers/hid/usbhid/hid-core.c
+++ b/drivers/hid/usbhid/hid-core.c
@@ -1265,7 +1265,7 @@ static int usbhid_idle(struct hid_device *hid, int report, int idle,
 	return hid_set_idle(dev, ifnum, report, idle);
 }
 
-static struct hid_ll_driver usb_hid_driver = {
+struct hid_ll_driver usb_hid_driver = {
 	.parse = usbhid_parse,
 	.start = usbhid_start,
 	.stop = usbhid_stop,
@@ -1278,6 +1278,7 @@ static struct hid_ll_driver usb_hid_driver = {
 	.output_report = usbhid_output_report,
 	.idle = usbhid_idle,
 };
+EXPORT_SYMBOL_GPL(usb_hid_driver);
 
 static int usbhid_probe(struct usb_interface *intf, const struct usb_device_id *id)
 {
diff --git a/drivers/hid/usbhid/hid-quirks.c b/drivers/hid/usbhid/hid-quirks.c
index a88e7c7bea0a..a83fa76655b9 100644
--- a/drivers/hid/usbhid/hid-quirks.c
+++ b/drivers/hid/usbhid/hid-quirks.c
@@ -99,6 +99,7 @@ static const struct hid_blacklist {
 	{ USB_VENDOR_ID_HP, USB_PRODUCT_ID_HP_LOGITECH_OEM_USB_OPTICAL_MOUSE_0A4A, HID_QUIRK_ALWAYS_POLL },
 	{ USB_VENDOR_ID_HP, USB_PRODUCT_ID_HP_LOGITECH_OEM_USB_OPTICAL_MOUSE_0B4A, HID_QUIRK_ALWAYS_POLL },
 	{ USB_VENDOR_ID_HP, USB_PRODUCT_ID_HP_PIXART_OEM_USB_OPTICAL_MOUSE, HID_QUIRK_ALWAYS_POLL },
+	{ USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_C007, HID_QUIRK_ALWAYS_POLL },
 	{ USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_C077, HID_QUIRK_ALWAYS_POLL },
 	{ USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_KEYBOARD_G710_PLUS, HID_QUIRK_NOGET },
 	{ USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_MOUSE_C01A, HID_QUIRK_ALWAYS_POLL },
diff --git a/drivers/hid/usbhid/usbkbd.c b/drivers/hid/usbhid/usbkbd.c
index 7fb2d1e4f5dd..ed01dc425d29 100644
--- a/drivers/hid/usbhid/usbkbd.c
+++ b/drivers/hid/usbhid/usbkbd.c
@@ -392,7 +392,7 @@ static void usb_kbd_disconnect(struct usb_interface *intf)
 	}
 }
 
-static struct usb_device_id usb_kbd_id_table [] = {
+static const struct usb_device_id usb_kbd_id_table[] = {
 	{ USB_INTERFACE_INFO(USB_INTERFACE_CLASS_HID, USB_INTERFACE_SUBCLASS_BOOT,
 		USB_INTERFACE_PROTOCOL_KEYBOARD) },
 	{ }						/* Terminating entry */
diff --git a/drivers/hid/usbhid/usbmouse.c b/drivers/hid/usbhid/usbmouse.c
index dd911c5241d8..589ad7c15a58 100644
--- a/drivers/hid/usbhid/usbmouse.c
+++ b/drivers/hid/usbhid/usbmouse.c
@@ -226,7 +226,7 @@ static void usb_mouse_disconnect(struct usb_interface *intf)
 	}
 }
 
-static struct usb_device_id usb_mouse_id_table [] = {
+static const struct usb_device_id usb_mouse_id_table[] = {
 	{ USB_INTERFACE_INFO(USB_INTERFACE_CLASS_HID, USB_INTERFACE_SUBCLASS_BOOT,
 		USB_INTERFACE_PROTOCOL_MOUSE) },
 	{ }	/* Terminating entry */
diff --git a/drivers/hid/wacom_sys.c b/drivers/hid/wacom_sys.c
index 838c1ebfffa9..e82a696a1d07 100644
--- a/drivers/hid/wacom_sys.c
+++ b/drivers/hid/wacom_sys.c
@@ -1671,10 +1671,7 @@ static ssize_t wacom_show_remote_mode(struct kobject *kobj,
 	u8 mode;
 
 	mode = wacom->led.groups[index].select;
-	if (mode >= 0 && mode < 3)
-		return snprintf(buf, PAGE_SIZE, "%d\n", mode);
-	else
-		return snprintf(buf, PAGE_SIZE, "%d\n", -1);
+	return sprintf(buf, "%d\n", mode < 3 ? mode : -1);
 }
 
 #define DEVICE_EKR_ATTR_GROUP(SET_ID)					\
@@ -2028,41 +2025,37 @@ static void wacom_update_name(struct wacom *wacom, const char *suffix)
 
 	/* Generic devices name unspecified */
 	if ((features->type == HID_GENERIC) && !strcmp("Wacom HID", features->name)) {
-		if (strstr(wacom->hdev->name, "Wacom") ||
-		    strstr(wacom->hdev->name, "wacom") ||
-		    strstr(wacom->hdev->name, "WACOM")) {
-			/* name is in HID descriptor, use it */
-			strlcpy(name, wacom->hdev->name, sizeof(name));
-
-			/* strip out excess whitespaces */
-			while (1) {
-				char *gap = strstr(name, "  ");
-				if (gap == NULL)
-					break;
-				/* shift everything including the terminator */
-				memmove(gap, gap+1, strlen(gap));
-			}
+		char *product_name = wacom->hdev->name;
 
-			/* strip off excessive prefixing */
-			if (strstr(name, "Wacom Co.,Ltd. Wacom ") == name) {
-				int n = strlen(name);
-				int x = strlen("Wacom Co.,Ltd. ");
-				memmove(name, name+x, n-x+1);
-			}
-			if (strstr(name, "Wacom Co., Ltd. Wacom ") == name) {
-				int n = strlen(name);
-				int x = strlen("Wacom Co., Ltd. ");
-				memmove(name, name+x, n-x+1);
-			}
+		if (hid_is_using_ll_driver(wacom->hdev, &usb_hid_driver)) {
+			struct usb_interface *intf = to_usb_interface(wacom->hdev->dev.parent);
+			struct usb_device *dev = interface_to_usbdev(intf);
+			product_name = dev->product;
+		}
 
-			/* get rid of trailing whitespace */
-			if (name[strlen(name)-1] == ' ')
-				name[strlen(name)-1] = '\0';
+		if (wacom->hdev->bus == BUS_I2C) {
+			snprintf(name, sizeof(name), "%s %X",
+				 features->name, wacom->hdev->product);
+		} else if (strstr(product_name, "Wacom") ||
+			   strstr(product_name, "wacom") ||
+			   strstr(product_name, "WACOM")) {
+			strlcpy(name, product_name, sizeof(name));
 		} else {
-			/* no meaningful name retrieved. use product ID */
-			snprintf(name, sizeof(name),
-				 "%s %X", features->name, wacom->hdev->product);
+			snprintf(name, sizeof(name), "Wacom %s", product_name);
 		}
+
+		/* strip out excess whitespaces */
+		while (1) {
+			char *gap = strstr(name, "  ");
+			if (gap == NULL)
+				break;
+			/* shift everything including the terminator */
+			memmove(gap, gap+1, strlen(gap));
+		}
+
+		/* get rid of trailing whitespace */
+		if (name[strlen(name)-1] == ' ')
+			name[strlen(name)-1] = '\0';
 	} else {
 		strlcpy(name, features->name, sizeof(name));
 	}
diff --git a/drivers/hid/wacom_wac.c b/drivers/hid/wacom_wac.c
index 9f940293ede4..bb17d7bbefd3 100644
--- a/drivers/hid/wacom_wac.c
+++ b/drivers/hid/wacom_wac.c
@@ -1846,7 +1846,13 @@ static void wacom_wac_pad_usage_mapping(struct hid_device *hdev,
 		features->device_type |= WACOM_DEVICETYPE_PAD;
 		break;
 	case WACOM_HID_WD_TOUCHRINGSTATUS:
-		wacom_map_usage(input, usage, field, EV_ABS, ABS_WHEEL, 0);
+		/*
+		 * Only set up type/code association. Completely mapping
+		 * this usage may overwrite the axis resolution and range.
+		 */
+		usage->type = EV_ABS;
+		usage->code = ABS_WHEEL;
+		set_bit(EV_ABS, input->evbit);
 		features->device_type |= WACOM_DEVICETYPE_PAD;
 		break;
 	case WACOM_HID_WD_BUTTONCONFIG:
diff --git a/drivers/hv/Kconfig b/drivers/hv/Kconfig
index c29cd5387a35..50b89ea0e60f 100644
--- a/drivers/hv/Kconfig
+++ b/drivers/hv/Kconfig
@@ -3,6 +3,7 @@ menu "Microsoft Hyper-V guest support"
 config HYPERV
 	tristate "Microsoft Hyper-V client drivers"
 	depends on X86 && ACPI && PCI && X86_LOCAL_APIC && HYPERVISOR_GUEST
+	select PARAVIRT
 	help
 	  Select this option to run Linux as a Hyper-V client operating
 	  system.
diff --git a/drivers/hv/channel.c b/drivers/hv/channel.c
index e57cc40cb768..efd5db743319 100644
--- a/drivers/hv/channel.c
+++ b/drivers/hv/channel.c
@@ -177,6 +177,11 @@ int vmbus_open(struct vmbus_channel *newchannel, u32 send_ringbuffer_size,
 		      &vmbus_connection.chn_msg_list);
 	spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 
+	if (newchannel->rescind) {
+		err = -ENODEV;
+		goto error_free_gpadl;
+	}
+
 	ret = vmbus_post_msg(open_msg,
 			     sizeof(struct vmbus_channel_open_channel), true);
 
@@ -421,6 +426,11 @@ int vmbus_establish_gpadl(struct vmbus_channel *channel, void *kbuffer,
 
 	spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
 
+	if (channel->rescind) {
+		ret = -ENODEV;
+		goto cleanup;
+	}
+
 	ret = vmbus_post_msg(gpadlmsg, msginfo->msgsize -
 			     sizeof(*msginfo), true);
 	if (ret != 0)
@@ -494,6 +504,10 @@ int vmbus_teardown_gpadl(struct vmbus_channel *channel, u32 gpadl_handle)
 	list_add_tail(&info->msglistentry,
 		      &vmbus_connection.chn_msg_list);
 	spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
+
+	if (channel->rescind)
+		goto post_msg_err;
+
 	ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_gpadl_teardown),
 			     true);
 
@@ -647,9 +661,23 @@ void vmbus_close(struct vmbus_channel *channel)
 }
 EXPORT_SYMBOL_GPL(vmbus_close);
 
-int vmbus_sendpacket_ctl(struct vmbus_channel *channel, void *buffer,
-			 u32 bufferlen, u64 requestid,
-			 enum vmbus_packet_type type, u32 flags)
+/**
+ * vmbus_sendpacket() - Send the specified buffer on the given channel
+ * @channel: Pointer to vmbus_channel structure.
+ * @buffer: Pointer to the buffer you want to receive the data into.
+ * @bufferlen: Maximum size of what the the buffer will hold
+ * @requestid: Identifier of the request
+ * @type: Type of packet that is being send e.g. negotiate, time
+ * packet etc.
+ *
+ * Sends data in @buffer directly to hyper-v via the vmbus
+ * This will send the data unparsed to hyper-v.
+ *
+ * Mainly used by Hyper-V drivers.
+ */
+int vmbus_sendpacket(struct vmbus_channel *channel, void *buffer,
+			   u32 bufferlen, u64 requestid,
+			   enum vmbus_packet_type type, u32 flags)
 {
 	struct vmpacket_descriptor desc;
 	u32 packetlen = sizeof(struct vmpacket_descriptor) + bufferlen;
@@ -676,42 +704,19 @@ int vmbus_sendpacket_ctl(struct vmbus_channel *channel, void *buffer,
 
 	return hv_ringbuffer_write(channel, bufferlist, num_vecs);
 }
-EXPORT_SYMBOL(vmbus_sendpacket_ctl);
-
-/**
- * vmbus_sendpacket() - Send the specified buffer on the given channel
- * @channel: Pointer to vmbus_channel structure.
- * @buffer: Pointer to the buffer you want to receive the data into.
- * @bufferlen: Maximum size of what the the buffer will hold
- * @requestid: Identifier of the request
- * @type: Type of packet that is being send e.g. negotiate, time
- * packet etc.
- *
- * Sends data in @buffer directly to hyper-v via the vmbus
- * This will send the data unparsed to hyper-v.
- *
- * Mainly used by Hyper-V drivers.
- */
-int vmbus_sendpacket(struct vmbus_channel *channel, void *buffer,
-			   u32 bufferlen, u64 requestid,
-			   enum vmbus_packet_type type, u32 flags)
-{
-	return vmbus_sendpacket_ctl(channel, buffer, bufferlen, requestid,
-				    type, flags);
-}
 EXPORT_SYMBOL(vmbus_sendpacket);
 
 /*
- * vmbus_sendpacket_pagebuffer_ctl - Send a range of single-page buffer
+ * vmbus_sendpacket_pagebuffer - Send a range of single-page buffer
  * packets using a GPADL Direct packet type. This interface allows you
  * to control notifying the host. This will be useful for sending
  * batched data. Also the sender can control the send flags
  * explicitly.
  */
-int vmbus_sendpacket_pagebuffer_ctl(struct vmbus_channel *channel,
-				    struct hv_page_buffer pagebuffers[],
-				    u32 pagecount, void *buffer, u32 bufferlen,
-				    u64 requestid, u32 flags)
+int vmbus_sendpacket_pagebuffer(struct vmbus_channel *channel,
+				struct hv_page_buffer pagebuffers[],
+				u32 pagecount, void *buffer, u32 bufferlen,
+				u64 requestid)
 {
 	int i;
 	struct vmbus_channel_packet_page_buffer desc;
@@ -736,7 +741,7 @@ int vmbus_sendpacket_pagebuffer_ctl(struct vmbus_channel *channel,
 
 	/* Setup the descriptor */
 	desc.type = VM_PKT_DATA_USING_GPA_DIRECT;
-	desc.flags = flags;
+	desc.flags = VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED;
 	desc.dataoffset8 = descsize >> 3; /* in 8-bytes granularity */
 	desc.length8 = (u16)(packetlen_aligned >> 3);
 	desc.transactionid = requestid;
@@ -757,24 +762,6 @@ int vmbus_sendpacket_pagebuffer_ctl(struct vmbus_channel *channel,
 
 	return hv_ringbuffer_write(channel, bufferlist, 3);
 }
-EXPORT_SYMBOL_GPL(vmbus_sendpacket_pagebuffer_ctl);
-
-/*
- * vmbus_sendpacket_pagebuffer - Send a range of single-page buffer
- * packets using a GPADL Direct packet type.
- */
-int vmbus_sendpacket_pagebuffer(struct vmbus_channel *channel,
-				     struct hv_page_buffer pagebuffers[],
-				     u32 pagecount, void *buffer, u32 bufferlen,
-				     u64 requestid)
-{
-	u32 flags = VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED;
-
-	return vmbus_sendpacket_pagebuffer_ctl(channel, pagebuffers, pagecount,
-					       buffer, bufferlen,
-					       requestid, flags);
-
-}
 EXPORT_SYMBOL_GPL(vmbus_sendpacket_pagebuffer);
 
 /*
@@ -814,62 +801,6 @@ int vmbus_sendpacket_mpb_desc(struct vmbus_channel *channel,
 }
 EXPORT_SYMBOL_GPL(vmbus_sendpacket_mpb_desc);
 
-/*
- * vmbus_sendpacket_multipagebuffer - Send a multi-page buffer packet
- * using a GPADL Direct packet type.
- */
-int vmbus_sendpacket_multipagebuffer(struct vmbus_channel *channel,
-				struct hv_multipage_buffer *multi_pagebuffer,
-				void *buffer, u32 bufferlen, u64 requestid)
-{
-	struct vmbus_channel_packet_multipage_buffer desc;
-	u32 descsize;
-	u32 packetlen;
-	u32 packetlen_aligned;
-	struct kvec bufferlist[3];
-	u64 aligned_data = 0;
-	u32 pfncount = NUM_PAGES_SPANNED(multi_pagebuffer->offset,
-					 multi_pagebuffer->len);
-
-	if (pfncount > MAX_MULTIPAGE_BUFFER_COUNT)
-		return -EINVAL;
-
-	/*
-	 * Adjust the size down since vmbus_channel_packet_multipage_buffer is
-	 * the largest size we support
-	 */
-	descsize = sizeof(struct vmbus_channel_packet_multipage_buffer) -
-			  ((MAX_MULTIPAGE_BUFFER_COUNT - pfncount) *
-			  sizeof(u64));
-	packetlen = descsize + bufferlen;
-	packetlen_aligned = ALIGN(packetlen, sizeof(u64));
-
-
-	/* Setup the descriptor */
-	desc.type = VM_PKT_DATA_USING_GPA_DIRECT;
-	desc.flags = VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED;
-	desc.dataoffset8 = descsize >> 3; /* in 8-bytes granularity */
-	desc.length8 = (u16)(packetlen_aligned >> 3);
-	desc.transactionid = requestid;
-	desc.rangecount = 1;
-
-	desc.range.len = multi_pagebuffer->len;
-	desc.range.offset = multi_pagebuffer->offset;
-
-	memcpy(desc.range.pfn_array, multi_pagebuffer->pfn_array,
-	       pfncount * sizeof(u64));
-
-	bufferlist[0].iov_base = &desc;
-	bufferlist[0].iov_len = descsize;
-	bufferlist[1].iov_base = buffer;
-	bufferlist[1].iov_len = bufferlen;
-	bufferlist[2].iov_base = &aligned_data;
-	bufferlist[2].iov_len = (packetlen_aligned - packetlen);
-
-	return hv_ringbuffer_write(channel, bufferlist, 3);
-}
-EXPORT_SYMBOL_GPL(vmbus_sendpacket_multipagebuffer);
-
 /**
  * vmbus_recvpacket() - Retrieve the user packet on the specified channel
  * @channel: Pointer to vmbus_channel structure.
diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c
index 4bbb8dea4727..060df71c2e8b 100644
--- a/drivers/hv/channel_mgmt.c
+++ b/drivers/hv/channel_mgmt.c
@@ -451,6 +451,12 @@ static void vmbus_process_offer(struct vmbus_channel *newchannel)
 	/* Make sure this is a new offer */
 	mutex_lock(&vmbus_connection.channel_mutex);
 
+	/*
+	 * Now that we have acquired the channel_mutex,
+	 * we can release the potentially racing rescind thread.
+	 */
+	atomic_dec(&vmbus_connection.offer_in_progress);
+
 	list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) {
 		if (!uuid_le_cmp(channel->offermsg.offer.if_type,
 			newchannel->offermsg.offer.if_type) &&
@@ -481,7 +487,6 @@ static void vmbus_process_offer(struct vmbus_channel *newchannel)
 			channel->num_sc++;
 			spin_unlock_irqrestore(&channel->lock, flags);
 		} else {
-			atomic_dec(&vmbus_connection.offer_in_progress);
 			goto err_free_chan;
 		}
 	}
@@ -510,7 +515,6 @@ static void vmbus_process_offer(struct vmbus_channel *newchannel)
 	if (!fnew) {
 		if (channel->sc_creation_callback != NULL)
 			channel->sc_creation_callback(newchannel);
-		atomic_dec(&vmbus_connection.offer_in_progress);
 		return;
 	}
 
@@ -541,7 +545,7 @@ static void vmbus_process_offer(struct vmbus_channel *newchannel)
 		goto err_deq_chan;
 	}
 
-	atomic_dec(&vmbus_connection.offer_in_progress);
+	newchannel->probe_done = true;
 	return;
 
 err_deq_chan:
@@ -599,7 +603,7 @@ static void init_vp_index(struct vmbus_channel *channel, u16 dev_type)
 		 */
 		channel->numa_node = 0;
 		channel->target_cpu = 0;
-		channel->target_vp = hv_context.vp_index[0];
+		channel->target_vp = hv_cpu_number_to_vp_number(0);
 		return;
 	}
 
@@ -683,7 +687,7 @@ static void init_vp_index(struct vmbus_channel *channel, u16 dev_type)
 	}
 
 	channel->target_cpu = cur_cpu;
-	channel->target_vp = hv_context.vp_index[cur_cpu];
+	channel->target_vp = hv_cpu_number_to_vp_number(cur_cpu);
 }
 
 static void vmbus_wait_for_unload(void)
@@ -805,21 +809,12 @@ static void vmbus_onoffer(struct vmbus_channel_message_header *hdr)
 	/*
 	 * Setup state for signalling the host.
 	 */
-	newchannel->sig_event = (struct hv_input_signal_event *)
-				(ALIGN((unsigned long)
-				&newchannel->sig_buf,
-				HV_HYPERCALL_PARAM_ALIGN));
-
-	newchannel->sig_event->connectionid.asu32 = 0;
-	newchannel->sig_event->connectionid.u.id = VMBUS_EVENT_CONNECTION_ID;
-	newchannel->sig_event->flag_number = 0;
-	newchannel->sig_event->rsvdz = 0;
+	newchannel->sig_event = VMBUS_EVENT_CONNECTION_ID;
 
 	if (vmbus_proto_version != VERSION_WS2008) {
 		newchannel->is_dedicated_interrupt =
 				(offer->is_dedicated_interrupt != 0);
-		newchannel->sig_event->connectionid.u.id =
-				offer->connection_id;
+		newchannel->sig_event = offer->connection_id;
 	}
 
 	memcpy(&newchannel->offermsg, offer,
@@ -882,8 +877,27 @@ static void vmbus_onoffer_rescind(struct vmbus_channel_message_header *hdr)
 	channel->rescind = true;
 	spin_unlock_irqrestore(&channel->lock, flags);
 
+	/*
+	 * Now that we have posted the rescind state, perform
+	 * rescind related cleanup.
+	 */
 	vmbus_rescind_cleanup(channel);
 
+	/*
+	 * Now wait for offer handling to complete.
+	 */
+	while (READ_ONCE(channel->probe_done) == false) {
+		/*
+		 * We wait here until any channel offer is currently
+		 * being processed.
+		 */
+		msleep(1);
+	}
+
+	/*
+	 * At this point, the rescind handling can proceed safely.
+	 */
+
 	if (channel->device_obj) {
 		if (channel->chn_rescind_callback) {
 			channel->chn_rescind_callback(channel);
@@ -1228,8 +1242,7 @@ struct vmbus_channel *vmbus_get_outgoing_channel(struct vmbus_channel *primary)
 		return outgoing_channel;
 	}
 
-	cur_cpu = hv_context.vp_index[get_cpu()];
-	put_cpu();
+	cur_cpu = hv_cpu_number_to_vp_number(smp_processor_id());
 	list_for_each_safe(cur, tmp, &primary->sc_list) {
 		cur_channel = list_entry(cur, struct vmbus_channel, sc_list);
 		if (cur_channel->state != CHANNEL_OPENED_STATE)
diff --git a/drivers/hv/connection.c b/drivers/hv/connection.c
index 59c11ff90d12..f41901f80b64 100644
--- a/drivers/hv/connection.c
+++ b/drivers/hv/connection.c
@@ -32,6 +32,8 @@
 #include <linux/hyperv.h>
 #include <linux/export.h>
 #include <asm/hyperv.h>
+#include <asm/mshyperv.h>
+
 #include "hyperv_vmbus.h"
 
 
@@ -94,7 +96,8 @@ static int vmbus_negotiate_version(struct vmbus_channel_msginfo *msginfo,
 	 * the CPU attempting to connect may not be CPU 0.
 	 */
 	if (version >= VERSION_WIN8_1) {
-		msg->target_vcpu = hv_context.vp_index[smp_processor_id()];
+		msg->target_vcpu =
+			hv_cpu_number_to_vp_number(smp_processor_id());
 		vmbus_connection.connect_cpu = smp_processor_id();
 	} else {
 		msg->target_vcpu = 0;
@@ -406,6 +409,6 @@ void vmbus_set_event(struct vmbus_channel *channel)
 	if (!channel->is_dedicated_interrupt)
 		vmbus_send_interrupt(child_relid);
 
-	hv_do_hypercall(HVCALL_SIGNAL_EVENT, channel->sig_event, NULL);
+	hv_do_fast_hypercall8(HVCALL_SIGNAL_EVENT, channel->sig_event);
 }
 EXPORT_SYMBOL_GPL(vmbus_set_event);
diff --git a/drivers/hv/hv.c b/drivers/hv/hv.c
index 2ea12207caa0..8267439dd1ee 100644
--- a/drivers/hv/hv.c
+++ b/drivers/hv/hv.c
@@ -234,7 +234,6 @@ int hv_synic_init(unsigned int cpu)
 	union hv_synic_siefp siefp;
 	union hv_synic_sint shared_sint;
 	union hv_synic_scontrol sctrl;
-	u64 vp_index;
 
 	/* Setup the Synic's message page */
 	hv_get_simp(simp.as_uint64);
@@ -276,14 +275,6 @@ int hv_synic_init(unsigned int cpu)
 	hv_context.synic_initialized = true;
 
 	/*
-	 * Setup the mapping between Hyper-V's notion
-	 * of cpuid and Linux' notion of cpuid.
-	 * This array will be indexed using Linux cpuid.
-	 */
-	hv_get_vp_index(vp_index);
-	hv_context.vp_index[cpu] = (u32)vp_index;
-
-	/*
 	 * Register the per-cpu clockevent source.
 	 */
 	if (ms_hyperv.features & HV_X64_MSR_SYNTIMER_AVAILABLE)
diff --git a/drivers/hv/hv_balloon.c b/drivers/hv/hv_balloon.c
index f5728deff893..db0e6652d7ef 100644
--- a/drivers/hv/hv_balloon.c
+++ b/drivers/hv/hv_balloon.c
@@ -584,10 +584,6 @@ static int hv_memory_notifier(struct notifier_block *nb, unsigned long val,
 
 	switch (val) {
 	case MEM_ONLINE:
-		spin_lock_irqsave(&dm_device.ha_lock, flags);
-		dm_device.num_pages_onlined += mem->nr_pages;
-		spin_unlock_irqrestore(&dm_device.ha_lock, flags);
-		/* Fall through */
 	case MEM_CANCEL_ONLINE:
 		if (dm_device.ha_waiting) {
 			dm_device.ha_waiting = false;
@@ -644,6 +640,9 @@ static void hv_page_online_one(struct hv_hotadd_state *has, struct page *pg)
 	__online_page_set_limits(pg);
 	__online_page_increment_counters(pg);
 	__online_page_free(pg);
+
+	WARN_ON_ONCE(!spin_is_locked(&dm_device.ha_lock));
+	dm_device.num_pages_onlined++;
 }
 
 static void hv_bring_pgs_online(struct hv_hotadd_state *has,
@@ -1036,8 +1035,8 @@ static void process_info(struct hv_dynmem_device *dm, struct dm_info_msg *msg)
 		if (info_hdr->data_size == sizeof(__u64)) {
 			__u64 *max_page_count = (__u64 *)&info_hdr[1];
 
-			pr_info("INFO_TYPE_MAX_PAGE_CNT = %llu\n",
-				*max_page_count);
+			pr_info("Max. dynamic memory size: %llu MB\n",
+				(*max_page_count) >> (20 - PAGE_SHIFT));
 		}
 
 		break;
@@ -1656,6 +1655,7 @@ static int balloon_probe(struct hv_device *dev,
 	}
 
 	dm_device.state = DM_INITIALIZED;
+	last_post_time = jiffies;
 
 	return 0;
 
diff --git a/drivers/hv/hv_kvp.c b/drivers/hv/hv_kvp.c
index 9a90b915b5be..5eed1e7da15c 100644
--- a/drivers/hv/hv_kvp.c
+++ b/drivers/hv/hv_kvp.c
@@ -304,7 +304,7 @@ static int process_ob_ipinfo(void *in_msg, void *out_msg, int op)
 				strlen((char *)in->body.kvp_ip_val.adapter_id),
 				UTF16_HOST_ENDIAN,
 				(wchar_t *)out->kvp_ip_val.adapter_id,
-				MAX_IP_ADDR_SIZE);
+				MAX_ADAPTER_ID_SIZE);
 		if (len < 0)
 			return len;
 
diff --git a/drivers/hv/hyperv_vmbus.h b/drivers/hv/hyperv_vmbus.h
index 1b6a5e0dfa75..49569f8fe038 100644
--- a/drivers/hv/hyperv_vmbus.h
+++ b/drivers/hv/hyperv_vmbus.h
@@ -229,17 +229,6 @@ struct hv_context {
 	struct hv_per_cpu_context __percpu *cpu_context;
 
 	/*
-	 * Hypervisor's notion of virtual processor ID is different from
-	 * Linux' notion of CPU ID. This information can only be retrieved
-	 * in the context of the calling CPU. Setup a map for easy access
-	 * to this information:
-	 *
-	 * vp_index[a] is the Hyper-V's processor ID corresponding to
-	 * Linux cpuid 'a'.
-	 */
-	u32 vp_index[NR_CPUS];
-
-	/*
 	 * To manage allocations in a NUMA node.
 	 * Array indexed by numa node ID.
 	 */
diff --git a/drivers/hv/ring_buffer.c b/drivers/hv/ring_buffer.c
index 1f450c39a9b0..12eb8caa4263 100644
--- a/drivers/hv/ring_buffer.c
+++ b/drivers/hv/ring_buffer.c
@@ -29,6 +29,7 @@
 #include <linux/uio.h>
 #include <linux/vmalloc.h>
 #include <linux/slab.h>
+#include <linux/prefetch.h>
 
 #include "hyperv_vmbus.h"
 
@@ -94,30 +95,6 @@ hv_set_next_write_location(struct hv_ring_buffer_info *ring_info,
 	ring_info->ring_buffer->write_index = next_write_location;
 }
 
-/* Get the next read location for the specified ring buffer. */
-static inline u32
-hv_get_next_read_location(const struct hv_ring_buffer_info *ring_info)
-{
-	return ring_info->ring_buffer->read_index;
-}
-
-/*
- * Get the next read location + offset for the specified ring buffer.
- * This allows the caller to skip.
- */
-static inline u32
-hv_get_next_readlocation_withoffset(const struct hv_ring_buffer_info *ring_info,
-				    u32 offset)
-{
-	u32 next = ring_info->ring_buffer->read_index;
-
-	next += offset;
-	if (next >= ring_info->ring_datasize)
-		next -= ring_info->ring_datasize;
-
-	return next;
-}
-
 /* Set the next read location for the specified ring buffer. */
 static inline void
 hv_set_next_read_location(struct hv_ring_buffer_info *ring_info,
@@ -142,29 +119,6 @@ hv_get_ring_bufferindices(struct hv_ring_buffer_info *ring_info)
 }
 
 /*
- * Helper routine to copy to source from ring buffer.
- * Assume there is enough room. Handles wrap-around in src case only!!
- */
-static u32 hv_copyfrom_ringbuffer(
-	const struct hv_ring_buffer_info *ring_info,
-	void				*dest,
-	u32				destlen,
-	u32				start_read_offset)
-{
-	void *ring_buffer = hv_get_ring_buffer(ring_info);
-	u32 ring_buffer_size = hv_get_ring_buffersize(ring_info);
-
-	memcpy(dest, ring_buffer + start_read_offset, destlen);
-
-	start_read_offset += destlen;
-	if (start_read_offset >= ring_buffer_size)
-		start_read_offset -= ring_buffer_size;
-
-	return start_read_offset;
-}
-
-
-/*
  * Helper routine to copy from source to ring buffer.
  * Assume there is enough room. Handles wrap-around in dest case only!!
  */
@@ -334,33 +288,22 @@ int hv_ringbuffer_write(struct vmbus_channel *channel,
 	return 0;
 }
 
-static inline void
-init_cached_read_index(struct hv_ring_buffer_info *rbi)
-{
-	rbi->cached_read_index = rbi->ring_buffer->read_index;
-}
-
 int hv_ringbuffer_read(struct vmbus_channel *channel,
 		       void *buffer, u32 buflen, u32 *buffer_actual_len,
 		       u64 *requestid, bool raw)
 {
-	u32 bytes_avail_toread;
-	u32 next_read_location;
-	u64 prev_indices = 0;
-	struct vmpacket_descriptor desc;
-	u32 offset;
-	u32 packetlen;
-	struct hv_ring_buffer_info *inring_info = &channel->inbound;
-
-	if (buflen <= 0)
+	struct vmpacket_descriptor *desc;
+	u32 packetlen, offset;
+
+	if (unlikely(buflen == 0))
 		return -EINVAL;
 
 	*buffer_actual_len = 0;
 	*requestid = 0;
 
-	bytes_avail_toread = hv_get_bytes_to_read(inring_info);
 	/* Make sure there is something to read */
-	if (bytes_avail_toread < sizeof(desc)) {
+	desc = hv_pkt_iter_first(channel);
+	if (desc == NULL) {
 		/*
 		 * No error is set when there is even no header, drivers are
 		 * supposed to analyze buffer_actual_len.
@@ -368,48 +311,22 @@ int hv_ringbuffer_read(struct vmbus_channel *channel,
 		return 0;
 	}
 
-	init_cached_read_index(inring_info);
-
-	next_read_location = hv_get_next_read_location(inring_info);
-	next_read_location = hv_copyfrom_ringbuffer(inring_info, &desc,
-						    sizeof(desc),
-						    next_read_location);
-
-	offset = raw ? 0 : (desc.offset8 << 3);
-	packetlen = (desc.len8 << 3) - offset;
+	offset = raw ? 0 : (desc->offset8 << 3);
+	packetlen = (desc->len8 << 3) - offset;
 	*buffer_actual_len = packetlen;
-	*requestid = desc.trans_id;
-
-	if (bytes_avail_toread < packetlen + offset)
-		return -EAGAIN;
+	*requestid = desc->trans_id;
 
-	if (packetlen > buflen)
+	if (unlikely(packetlen > buflen))
 		return -ENOBUFS;
 
-	next_read_location =
-		hv_get_next_readlocation_withoffset(inring_info, offset);
+	/* since ring is double mapped, only one copy is necessary */
+	memcpy(buffer, (const char *)desc + offset, packetlen);
 
-	next_read_location = hv_copyfrom_ringbuffer(inring_info,
-						buffer,
-						packetlen,
-						next_read_location);
+	/* Advance ring index to next packet descriptor */
+	__hv_pkt_iter_next(channel, desc);
 
-	next_read_location = hv_copyfrom_ringbuffer(inring_info,
-						&prev_indices,
-						sizeof(u64),
-						next_read_location);
-
-	/*
-	 * Make sure all reads are done before we update the read index since
-	 * the writer may start writing to the read area once the read index
-	 * is updated.
-	 */
-	virt_mb();
-
-	/* Update the read index */
-	hv_set_next_read_location(inring_info, next_read_location);
-
-	hv_signal_on_read(channel);
+	/* Notify host of update */
+	hv_pkt_iter_close(channel);
 
 	return 0;
 }
@@ -440,14 +357,16 @@ static u32 hv_pkt_iter_avail(const struct hv_ring_buffer_info *rbi)
 struct vmpacket_descriptor *hv_pkt_iter_first(struct vmbus_channel *channel)
 {
 	struct hv_ring_buffer_info *rbi = &channel->inbound;
-
-	/* set state for later hv_signal_on_read() */
-	init_cached_read_index(rbi);
+	struct vmpacket_descriptor *desc;
 
 	if (hv_pkt_iter_avail(rbi) < sizeof(struct vmpacket_descriptor))
 		return NULL;
 
-	return hv_get_ring_buffer(rbi) + rbi->priv_read_index;
+	desc = hv_get_ring_buffer(rbi) + rbi->priv_read_index;
+	if (desc)
+		prefetch((char *)desc + (desc->len8 << 3));
+
+	return desc;
 }
 EXPORT_SYMBOL_GPL(hv_pkt_iter_first);
 
@@ -471,10 +390,7 @@ __hv_pkt_iter_next(struct vmbus_channel *channel,
 		rbi->priv_read_index -= dsize;
 
 	/* more data? */
-	if (hv_pkt_iter_avail(rbi) < sizeof(struct vmpacket_descriptor))
-		return NULL;
-	else
-		return hv_get_ring_buffer(rbi) + rbi->priv_read_index;
+	return hv_pkt_iter_first(channel);
 }
 EXPORT_SYMBOL_GPL(__hv_pkt_iter_next);
 
@@ -484,6 +400,7 @@ EXPORT_SYMBOL_GPL(__hv_pkt_iter_next);
 void hv_pkt_iter_close(struct vmbus_channel *channel)
 {
 	struct hv_ring_buffer_info *rbi = &channel->inbound;
+	u32 orig_write_sz = hv_get_bytes_to_write(rbi);
 
 	/*
 	 * Make sure all reads are done before we update the read index since
@@ -493,6 +410,40 @@ void hv_pkt_iter_close(struct vmbus_channel *channel)
 	virt_rmb();
 	rbi->ring_buffer->read_index = rbi->priv_read_index;
 
-	hv_signal_on_read(channel);
+	/*
+	 * Issue a full memory barrier before making the signaling decision.
+	 * Here is the reason for having this barrier:
+	 * If the reading of the pend_sz (in this function)
+	 * were to be reordered and read before we commit the new read
+	 * index (in the calling function)  we could
+	 * have a problem. If the host were to set the pending_sz after we
+	 * have sampled pending_sz and go to sleep before we commit the
+	 * read index, we could miss sending the interrupt. Issue a full
+	 * memory barrier to address this.
+	 */
+	virt_mb();
+
+	/* If host has disabled notifications then skip */
+	if (rbi->ring_buffer->interrupt_mask)
+		return;
+
+	if (rbi->ring_buffer->feature_bits.feat_pending_send_sz) {
+		u32 pending_sz = READ_ONCE(rbi->ring_buffer->pending_send_sz);
+
+		/*
+		 * If there was space before we began iteration,
+		 * then host was not blocked. Also handles case where
+		 * pending_sz is zero then host has nothing pending
+		 * and does not need to be signaled.
+		 */
+		if (orig_write_sz > pending_sz)
+			return;
+
+		/* If pending write will not fit, don't give false hope. */
+		if (hv_get_bytes_to_write(rbi) < pending_sz)
+			return;
+	}
+
+	vmbus_setevent(channel);
 }
 EXPORT_SYMBOL_GPL(hv_pkt_iter_close);
diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c
index ed84e96715a0..a9d49f6f6501 100644
--- a/drivers/hv/vmbus_drv.c
+++ b/drivers/hv/vmbus_drv.c
@@ -940,6 +940,9 @@ static void vmbus_chan_sched(struct hv_per_cpu_context *hv_cpu)
 			if (channel->offermsg.child_relid != relid)
 				continue;
 
+			if (channel->rescind)
+				continue;
+
 			switch (channel->callback_mode) {
 			case HV_CALL_ISR:
 				vmbus_channel_isr(channel);
@@ -1451,23 +1454,6 @@ void vmbus_free_mmio(resource_size_t start, resource_size_t size)
 }
 EXPORT_SYMBOL_GPL(vmbus_free_mmio);
 
-/**
- * vmbus_cpu_number_to_vp_number() - Map CPU to VP.
- * @cpu_number: CPU number in Linux terms
- *
- * This function returns the mapping between the Linux processor
- * number and the hypervisor's virtual processor number, useful
- * in making hypercalls and such that talk about specific
- * processors.
- *
- * Return: Virtual processor number in Hyper-V terms
- */
-int vmbus_cpu_number_to_vp_number(int cpu_number)
-{
-	return hv_context.vp_index[cpu_number];
-}
-EXPORT_SYMBOL_GPL(vmbus_cpu_number_to_vp_number);
-
 static int vmbus_acpi_add(struct acpi_device *device)
 {
 	acpi_status result;
diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig
index 5ef2814345ef..d65431417b17 100644
--- a/drivers/hwmon/Kconfig
+++ b/drivers/hwmon/Kconfig
@@ -343,6 +343,7 @@ config SENSORS_ASB100
 
 config SENSORS_ASPEED
 	tristate "ASPEED AST2400/AST2500 PWM and Fan tach driver"
+	depends on THERMAL || THERMAL=n
 	select REGMAP
 	help
 	  This driver provides support for ASPEED AST2400/AST2500 PWM
@@ -790,6 +791,13 @@ config SENSORS_LTC4261
 	  This driver can also be built as a module. If so, the module will
 	  be called ltc4261.
 
+config SENSORS_LTQ_CPUTEMP
+	bool "Lantiq cpu temperature sensor driver"
+	depends on LANTIQ
+	help
+	  If you say yes here you get support for the temperature
+	  sensor inside your CPU.
+
 config SENSORS_MAX1111
 	tristate "Maxim MAX1111 Serial 8-bit ADC chip and compatibles"
 	depends on SPI_MASTER
diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile
index d4641a9f16c1..c84d9784be98 100644
--- a/drivers/hwmon/Makefile
+++ b/drivers/hwmon/Makefile
@@ -110,6 +110,7 @@ obj-$(CONFIG_SENSORS_LTC4222)	+= ltc4222.o
 obj-$(CONFIG_SENSORS_LTC4245)	+= ltc4245.o
 obj-$(CONFIG_SENSORS_LTC4260)	+= ltc4260.o
 obj-$(CONFIG_SENSORS_LTC4261)	+= ltc4261.o
+obj-$(CONFIG_SENSORS_LTQ_CPUTEMP) += ltq-cputemp.o
 obj-$(CONFIG_SENSORS_MAX1111)	+= max1111.o
 obj-$(CONFIG_SENSORS_MAX16065)	+= max16065.o
 obj-$(CONFIG_SENSORS_MAX1619)	+= max1619.o
diff --git a/drivers/hwmon/acpi_power_meter.c b/drivers/hwmon/acpi_power_meter.c
index 579bdf93be43..14a94d90c028 100644
--- a/drivers/hwmon/acpi_power_meter.c
+++ b/drivers/hwmon/acpi_power_meter.c
@@ -973,7 +973,7 @@ static int __init enable_cap_knobs(const struct dmi_system_id *d)
 	return 0;
 }
 
-static struct dmi_system_id __initdata pm_dmi_table[] = {
+static const struct dmi_system_id pm_dmi_table[] __initconst = {
 	{
 		enable_cap_knobs, "IBM Active Energy Manager",
 		{
diff --git a/drivers/hwmon/adc128d818.c b/drivers/hwmon/adc128d818.c
index a557b46dbe8e..bd2ca315c9d8 100644
--- a/drivers/hwmon/adc128d818.c
+++ b/drivers/hwmon/adc128d818.c
@@ -384,7 +384,7 @@ static struct attribute *adc128_attrs[] = {
 	NULL
 };
 
-static struct attribute_group adc128_group = {
+static const struct attribute_group adc128_group = {
 	.attrs = adc128_attrs,
 	.is_visible = adc128_is_visible,
 };
diff --git a/drivers/hwmon/ads1015.c b/drivers/hwmon/ads1015.c
index 357b42607164..98c704d3366a 100644
--- a/drivers/hwmon/ads1015.c
+++ b/drivers/hwmon/ads1015.c
@@ -191,24 +191,23 @@ static int ads1015_get_channels_config_of(struct i2c_client *client)
 		unsigned int data_rate = ADS1015_DEFAULT_DATA_RATE;
 
 		if (of_property_read_u32(node, "reg", &pval)) {
-			dev_err(&client->dev, "invalid reg on %s\n",
-				node->full_name);
+			dev_err(&client->dev, "invalid reg on %pOF\n", node);
 			continue;
 		}
 
 		channel = pval;
 		if (channel >= ADS1015_CHANNELS) {
 			dev_err(&client->dev,
-				"invalid channel index %d on %s\n",
-				channel, node->full_name);
+				"invalid channel index %d on %pOF\n",
+				channel, node);
 			continue;
 		}
 
 		if (!of_property_read_u32(node, "ti,gain", &pval)) {
 			pga = pval;
 			if (pga > 6) {
-				dev_err(&client->dev, "invalid gain on %s\n",
-					node->full_name);
+				dev_err(&client->dev, "invalid gain on %pOF\n",
+					node);
 				return -EINVAL;
 			}
 		}
@@ -217,8 +216,7 @@ static int ads1015_get_channels_config_of(struct i2c_client *client)
 			data_rate = pval;
 			if (data_rate > 7) {
 				dev_err(&client->dev,
-					"invalid data_rate on %s\n",
-					node->full_name);
+					"invalid data_rate on %pOF\n", node);
 				return -EINVAL;
 			}
 		}
diff --git a/drivers/hwmon/adt7475.c b/drivers/hwmon/adt7475.c
index 1baa213a60bd..9ef84998c7f3 100644
--- a/drivers/hwmon/adt7475.c
+++ b/drivers/hwmon/adt7475.c
@@ -1319,14 +1319,14 @@ static struct attribute *vid_attrs[] = {
 	NULL
 };
 
-static struct attribute_group adt7475_attr_group = { .attrs = adt7475_attrs };
-static struct attribute_group fan4_attr_group = { .attrs = fan4_attrs };
-static struct attribute_group pwm2_attr_group = { .attrs = pwm2_attrs };
-static struct attribute_group in0_attr_group = { .attrs = in0_attrs };
-static struct attribute_group in3_attr_group = { .attrs = in3_attrs };
-static struct attribute_group in4_attr_group = { .attrs = in4_attrs };
-static struct attribute_group in5_attr_group = { .attrs = in5_attrs };
-static struct attribute_group vid_attr_group = { .attrs = vid_attrs };
+static const struct attribute_group adt7475_attr_group = { .attrs = adt7475_attrs };
+static const struct attribute_group fan4_attr_group = { .attrs = fan4_attrs };
+static const struct attribute_group pwm2_attr_group = { .attrs = pwm2_attrs };
+static const struct attribute_group in0_attr_group = { .attrs = in0_attrs };
+static const struct attribute_group in3_attr_group = { .attrs = in3_attrs };
+static const struct attribute_group in4_attr_group = { .attrs = in4_attrs };
+static const struct attribute_group in5_attr_group = { .attrs = in5_attrs };
+static const struct attribute_group vid_attr_group = { .attrs = vid_attrs };
 
 static int adt7475_detect(struct i2c_client *client,
 			  struct i2c_board_info *info)
diff --git a/drivers/hwmon/applesmc.c b/drivers/hwmon/applesmc.c
index 76c34f4fde13..5c677ba44014 100644
--- a/drivers/hwmon/applesmc.c
+++ b/drivers/hwmon/applesmc.c
@@ -1247,7 +1247,7 @@ static int applesmc_dmi_match(const struct dmi_system_id *id)
  * Note that DMI_MATCH(...,"MacBook") will match "MacBookPro1,1".
  * So we need to put "Apple MacBook Pro" before "Apple MacBook".
  */
-static __initdata struct dmi_system_id applesmc_whitelist[] = {
+static const struct dmi_system_id applesmc_whitelist[] __initconst = {
 	{ applesmc_dmi_match, "Apple MacBook Air", {
 	  DMI_MATCH(DMI_BOARD_VENDOR, "Apple"),
 	  DMI_MATCH(DMI_PRODUCT_NAME, "MacBookAir") },
diff --git a/drivers/hwmon/asc7621.c b/drivers/hwmon/asc7621.c
index c77644d45a02..4875e99b59c9 100644
--- a/drivers/hwmon/asc7621.c
+++ b/drivers/hwmon/asc7621.c
@@ -512,7 +512,7 @@ static ssize_t show_pwm_ac(struct device *dev,
 {
 	SETUP_SHOW_DATA_PARAM(dev, attr);
 	u8 config, altbit, regval;
-	const u8 map[] = {
+	static const u8 map[] = {
 		0x01, 0x02, 0x04, 0x1f, 0x00, 0x06, 0x07, 0x10,
 		0x08, 0x0f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f
 	};
@@ -533,7 +533,7 @@ static ssize_t store_pwm_ac(struct device *dev,
 	SETUP_STORE_DATA_PARAM(dev, attr);
 	unsigned long reqval;
 	u8 currval, config, altbit, newval;
-	const u16 map[] = {
+	static const u16 map[] = {
 		0x04, 0x00, 0x01, 0xff, 0x02, 0xff, 0x05, 0x06,
 		0x08, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0f,
 		0x07, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
diff --git a/drivers/hwmon/aspeed-pwm-tacho.c b/drivers/hwmon/aspeed-pwm-tacho.c
index ddfe66bdff86..69b97d45e3cb 100644
--- a/drivers/hwmon/aspeed-pwm-tacho.c
+++ b/drivers/hwmon/aspeed-pwm-tacho.c
@@ -20,6 +20,7 @@
 #include <linux/platform_device.h>
 #include <linux/sysfs.h>
 #include <linux/regmap.h>
+#include <linux/thermal.h>
 
 /* ASPEED PWM & FAN Tach Register Definition */
 #define ASPEED_PTCR_CTRL		0x00
@@ -166,6 +167,18 @@
 /* How long we sleep in us while waiting for an RPM result. */
 #define ASPEED_RPM_STATUS_SLEEP_USEC	500
 
+#define MAX_CDEV_NAME_LEN 16
+
+struct aspeed_cooling_device {
+	char name[16];
+	struct aspeed_pwm_tacho_data *priv;
+	struct thermal_cooling_device *tcdev;
+	int pwm_port;
+	u8 *cooling_levels;
+	u8 max_state;
+	u8 cur_state;
+};
+
 struct aspeed_pwm_tacho_data {
 	struct regmap *regmap;
 	unsigned long clk_freq;
@@ -180,6 +193,7 @@ struct aspeed_pwm_tacho_data {
 	u8 pwm_port_type[8];
 	u8 pwm_port_fan_ctrl[8];
 	u8 fan_tach_ch_source[16];
+	struct aspeed_cooling_device *cdev[8];
 	const struct attribute_group *groups[3];
 };
 
@@ -765,6 +779,94 @@ static void aspeed_create_fan_tach_channel(struct aspeed_pwm_tacho_data *priv,
 	}
 }
 
+static int
+aspeed_pwm_cz_get_max_state(struct thermal_cooling_device *tcdev,
+			    unsigned long *state)
+{
+	struct aspeed_cooling_device *cdev = tcdev->devdata;
+
+	*state = cdev->max_state;
+
+	return 0;
+}
+
+static int
+aspeed_pwm_cz_get_cur_state(struct thermal_cooling_device *tcdev,
+			    unsigned long *state)
+{
+	struct aspeed_cooling_device *cdev = tcdev->devdata;
+
+	*state = cdev->cur_state;
+
+	return 0;
+}
+
+static int
+aspeed_pwm_cz_set_cur_state(struct thermal_cooling_device *tcdev,
+			    unsigned long state)
+{
+	struct aspeed_cooling_device *cdev = tcdev->devdata;
+
+	if (state > cdev->max_state)
+		return -EINVAL;
+
+	cdev->cur_state = state;
+	cdev->priv->pwm_port_fan_ctrl[cdev->pwm_port] =
+					cdev->cooling_levels[cdev->cur_state];
+	aspeed_set_pwm_port_fan_ctrl(cdev->priv, cdev->pwm_port,
+				     cdev->cooling_levels[cdev->cur_state]);
+
+	return 0;
+}
+
+static const struct thermal_cooling_device_ops aspeed_pwm_cool_ops = {
+	.get_max_state = aspeed_pwm_cz_get_max_state,
+	.get_cur_state = aspeed_pwm_cz_get_cur_state,
+	.set_cur_state = aspeed_pwm_cz_set_cur_state,
+};
+
+static int aspeed_create_pwm_cooling(struct device *dev,
+				     struct device_node *child,
+				     struct aspeed_pwm_tacho_data *priv,
+				     u32 pwm_port, u8 num_levels)
+{
+	int ret;
+	struct aspeed_cooling_device *cdev;
+
+	cdev = devm_kzalloc(dev, sizeof(*cdev), GFP_KERNEL);
+
+	if (!cdev)
+		return -ENOMEM;
+
+	cdev->cooling_levels = devm_kzalloc(dev, num_levels, GFP_KERNEL);
+	if (!cdev->cooling_levels)
+		return -ENOMEM;
+
+	cdev->max_state = num_levels - 1;
+	ret = of_property_read_u8_array(child, "cooling-levels",
+					cdev->cooling_levels,
+					num_levels);
+	if (ret) {
+		dev_err(dev, "Property 'cooling-levels' cannot be read.\n");
+		return ret;
+	}
+	snprintf(cdev->name, MAX_CDEV_NAME_LEN, "%s%d", child->name, pwm_port);
+
+	cdev->tcdev = thermal_of_cooling_device_register(child,
+							 cdev->name,
+							 cdev,
+							 &aspeed_pwm_cool_ops);
+	if (IS_ERR(cdev->tcdev))
+		return PTR_ERR(cdev->tcdev);
+
+	cdev->priv = priv;
+	cdev->pwm_port = pwm_port;
+
+	priv->cdev[pwm_port] = cdev;
+
+	return 0;
+}
+
 static int aspeed_create_fan(struct device *dev,
 			     struct device_node *child,
 			     struct aspeed_pwm_tacho_data *priv)
@@ -778,6 +880,15 @@ static int aspeed_create_fan(struct device *dev,
 		return ret;
 	aspeed_create_pwm_port(priv, (u8)pwm_port);
 
+	ret = of_property_count_u8_elems(child, "cooling-levels");
+
+	if (ret > 0) {
+		ret = aspeed_create_pwm_cooling(dev, child, priv, pwm_port,
+						ret);
+		if (ret)
+			return ret;
+	}
+
 	count = of_property_count_u8_elems(child, "aspeed,fan-tach-ch");
 	if (count < 1)
 		return -EINVAL;
@@ -834,9 +945,10 @@ static int aspeed_pwm_tacho_probe(struct platform_device *pdev)
 
 	for_each_child_of_node(np, child) {
 		ret = aspeed_create_fan(dev, child, priv);
-		of_node_put(child);
-		if (ret)
+		if (ret) {
+			of_node_put(child);
 			return ret;
+		}
 	}
 
 	priv->groups[0] = &pwm_dev_group;
diff --git a/drivers/hwmon/da9052-hwmon.c b/drivers/hwmon/da9052-hwmon.c
index c9832bfacfe5..97a62f5b9ea4 100644
--- a/drivers/hwmon/da9052-hwmon.c
+++ b/drivers/hwmon/da9052-hwmon.c
@@ -20,13 +20,19 @@
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/platform_device.h>
+#include <linux/property.h>
 
 #include <linux/mfd/da9052/da9052.h>
 #include <linux/mfd/da9052/reg.h>
+#include <linux/regulator/consumer.h>
 
 struct da9052_hwmon {
-	struct da9052	*da9052;
-	struct mutex	hwmon_lock;
+	struct da9052		*da9052;
+	struct mutex		hwmon_lock;
+	bool			tsi_as_adc;
+	int			tsiref_mv;
+	struct regulator	*tsiref;
+	struct completion	tsidone;
 };
 
 static const char * const input_names[] = {
@@ -37,6 +43,10 @@ static const char * const input_names[] = {
 	[DA9052_ADC_IN4]	=	"ADC IN4",
 	[DA9052_ADC_IN5]	=	"ADC IN5",
 	[DA9052_ADC_IN6]	=	"ADC IN6",
+	[DA9052_ADC_TSI_XP]	=	"ADC TS X+",
+	[DA9052_ADC_TSI_YP]	=	"ADC TS Y+",
+	[DA9052_ADC_TSI_XN]	=	"ADC TS X-",
+	[DA9052_ADC_TSI_YN]	=	"ADC TS Y-",
 	[DA9052_ADC_TJUNC]	=	"BATTERY JUNCTION TEMP",
 	[DA9052_ADC_VBBAT]	=	"BACK-UP BATTERY VOLTAGE",
 };
@@ -59,6 +69,11 @@ static inline int vbbat_reg_to_mv(int value)
 	return DIV_ROUND_CLOSEST(value * 5000, 1023);
 }
 
+static inline int input_tsireg_to_mv(struct da9052_hwmon *hwmon, int value)
+{
+	return DIV_ROUND_CLOSEST(value * hwmon->tsiref_mv, 1023);
+}
+
 static inline int da9052_enable_vddout_channel(struct da9052 *da9052)
 {
 	return da9052_reg_update(da9052, DA9052_ADC_CONT_REG,
@@ -154,6 +169,97 @@ static ssize_t da9052_read_misc_channel(struct device *dev,
 	return sprintf(buf, "%d\n", input_reg_to_mv(ret));
 }
 
+static int da9052_request_tsi_read(struct da9052_hwmon *hwmon, int channel)
+{
+	u8 val = DA9052_TSICONTB_TSIMAN;
+
+	switch (channel) {
+	case DA9052_ADC_TSI_XP:
+		val |= DA9052_TSICONTB_TSIMUX_XP;
+		break;
+	case DA9052_ADC_TSI_YP:
+		val |= DA9052_TSICONTB_TSIMUX_YP;
+		break;
+	case DA9052_ADC_TSI_XN:
+		val |= DA9052_TSICONTB_TSIMUX_XN;
+		break;
+	case DA9052_ADC_TSI_YN:
+		val |= DA9052_TSICONTB_TSIMUX_YN;
+		break;
+	}
+
+	return da9052_reg_write(hwmon->da9052, DA9052_TSI_CONT_B_REG, val);
+}
+
+static int da9052_get_tsi_result(struct da9052_hwmon *hwmon, int channel)
+{
+	u8 regs[3];
+	int msb, lsb, err;
+
+	/* block read to avoid separation of MSB and LSB */
+	err = da9052_group_read(hwmon->da9052, DA9052_TSI_X_MSB_REG,
+				ARRAY_SIZE(regs), regs);
+	if (err)
+		return err;
+
+	switch (channel) {
+	case DA9052_ADC_TSI_XP:
+	case DA9052_ADC_TSI_XN:
+		msb = regs[0] << DA9052_TSILSB_TSIXL_BITS;
+		lsb = regs[2] & DA9052_TSILSB_TSIXL;
+		lsb >>= DA9052_TSILSB_TSIXL_SHIFT;
+		break;
+	case DA9052_ADC_TSI_YP:
+	case DA9052_ADC_TSI_YN:
+		msb = regs[1] << DA9052_TSILSB_TSIYL_BITS;
+		lsb = regs[2] & DA9052_TSILSB_TSIYL;
+		lsb >>= DA9052_TSILSB_TSIYL_SHIFT;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return msb | lsb;
+}
+
+
+static ssize_t __da9052_read_tsi(struct device *dev, int channel)
+{
+	struct da9052_hwmon *hwmon = dev_get_drvdata(dev);
+	int ret;
+
+	reinit_completion(&hwmon->tsidone);
+
+	ret = da9052_request_tsi_read(hwmon, channel);
+	if (ret < 0)
+		return ret;
+
+	/* Wait for an conversion done interrupt */
+	if (!wait_for_completion_timeout(&hwmon->tsidone,
+					 msecs_to_jiffies(500)))
+		return -ETIMEDOUT;
+
+	return da9052_get_tsi_result(hwmon, channel);
+}
+
+static ssize_t da9052_read_tsi(struct device *dev,
+			       struct device_attribute *devattr,
+			       char *buf)
+{
+	struct da9052_hwmon *hwmon = dev_get_drvdata(dev);
+	int channel = to_sensor_dev_attr(devattr)->index;
+	int ret;
+
+	mutex_lock(&hwmon->hwmon_lock);
+	ret = __da9052_read_tsi(dev, channel);
+	mutex_unlock(&hwmon->hwmon_lock);
+
+	if (ret < 0)
+		return ret;
+	else
+		return sprintf(buf, "%d\n", input_tsireg_to_mv(hwmon, ret));
+}
+
 static ssize_t da9052_read_tjunc(struct device *dev,
 				 struct device_attribute *devattr, char *buf)
 {
@@ -196,43 +302,82 @@ static ssize_t show_label(struct device *dev,
 		       input_names[to_sensor_dev_attr(devattr)->index]);
 }
 
-static SENSOR_DEVICE_ATTR(in0_input, S_IRUGO, da9052_read_vddout, NULL,
+static umode_t da9052_channel_is_visible(struct kobject *kobj,
+					 struct attribute *attr, int index)
+{
+	struct device *dev = container_of(kobj, struct device, kobj);
+	struct da9052_hwmon *hwmon = dev_get_drvdata(dev);
+	struct device_attribute *dattr = container_of(attr,
+				struct device_attribute, attr);
+	struct sensor_device_attribute *sattr = to_sensor_dev_attr(dattr);
+
+	if (!hwmon->tsi_as_adc) {
+		switch (sattr->index) {
+		case DA9052_ADC_TSI_XP:
+		case DA9052_ADC_TSI_YP:
+		case DA9052_ADC_TSI_XN:
+		case DA9052_ADC_TSI_YN:
+			return 0;
+		}
+	}
+
+	return attr->mode;
+}
+
+static SENSOR_DEVICE_ATTR(in0_input, 0444, da9052_read_vddout, NULL,
 			  DA9052_ADC_VDDOUT);
-static SENSOR_DEVICE_ATTR(in0_label, S_IRUGO, show_label, NULL,
+static SENSOR_DEVICE_ATTR(in0_label, 0444, show_label, NULL,
 			  DA9052_ADC_VDDOUT);
-static SENSOR_DEVICE_ATTR(in3_input, S_IRUGO, da9052_read_vbat, NULL,
+static SENSOR_DEVICE_ATTR(in3_input, 0444, da9052_read_vbat, NULL,
 			  DA9052_ADC_VBAT);
-static SENSOR_DEVICE_ATTR(in3_label, S_IRUGO, show_label, NULL,
+static SENSOR_DEVICE_ATTR(in3_label, 0444, show_label, NULL,
 			  DA9052_ADC_VBAT);
-static SENSOR_DEVICE_ATTR(in4_input, S_IRUGO, da9052_read_misc_channel, NULL,
+static SENSOR_DEVICE_ATTR(in4_input, 0444, da9052_read_misc_channel, NULL,
 			  DA9052_ADC_IN4);
-static SENSOR_DEVICE_ATTR(in4_label, S_IRUGO, show_label, NULL,
+static SENSOR_DEVICE_ATTR(in4_label, 0444, show_label, NULL,
 			  DA9052_ADC_IN4);
-static SENSOR_DEVICE_ATTR(in5_input, S_IRUGO, da9052_read_misc_channel, NULL,
+static SENSOR_DEVICE_ATTR(in5_input, 0444, da9052_read_misc_channel, NULL,
 			  DA9052_ADC_IN5);
-static SENSOR_DEVICE_ATTR(in5_label, S_IRUGO, show_label, NULL,
+static SENSOR_DEVICE_ATTR(in5_label, 0444, show_label, NULL,
 			  DA9052_ADC_IN5);
-static SENSOR_DEVICE_ATTR(in6_input, S_IRUGO, da9052_read_misc_channel, NULL,
+static SENSOR_DEVICE_ATTR(in6_input, 0444, da9052_read_misc_channel, NULL,
 			  DA9052_ADC_IN6);
-static SENSOR_DEVICE_ATTR(in6_label, S_IRUGO, show_label, NULL,
+static SENSOR_DEVICE_ATTR(in6_label, 0444, show_label, NULL,
 			  DA9052_ADC_IN6);
-static SENSOR_DEVICE_ATTR(in9_input, S_IRUGO, da9052_read_vbbat, NULL,
+static SENSOR_DEVICE_ATTR(in9_input, 0444, da9052_read_vbbat, NULL,
 			  DA9052_ADC_VBBAT);
-static SENSOR_DEVICE_ATTR(in9_label, S_IRUGO, show_label, NULL,
+static SENSOR_DEVICE_ATTR(in9_label, 0444, show_label, NULL,
 			  DA9052_ADC_VBBAT);
 
-static SENSOR_DEVICE_ATTR(curr1_input, S_IRUGO, da9052_read_ich, NULL,
+static SENSOR_DEVICE_ATTR(in70_input, 0444, da9052_read_tsi, NULL,
+			  DA9052_ADC_TSI_XP);
+static SENSOR_DEVICE_ATTR(in70_label, 0444, show_label, NULL,
+			  DA9052_ADC_TSI_XP);
+static SENSOR_DEVICE_ATTR(in71_input, 0444, da9052_read_tsi, NULL,
+			  DA9052_ADC_TSI_XN);
+static SENSOR_DEVICE_ATTR(in71_label, 0444, show_label, NULL,
+			  DA9052_ADC_TSI_XN);
+static SENSOR_DEVICE_ATTR(in72_input, 0444, da9052_read_tsi, NULL,
+			  DA9052_ADC_TSI_YP);
+static SENSOR_DEVICE_ATTR(in72_label, 0444, show_label, NULL,
+			  DA9052_ADC_TSI_YP);
+static SENSOR_DEVICE_ATTR(in73_input, 0444, da9052_read_tsi, NULL,
+			  DA9052_ADC_TSI_YN);
+static SENSOR_DEVICE_ATTR(in73_label, 0444, show_label, NULL,
+			  DA9052_ADC_TSI_YN);
+
+static SENSOR_DEVICE_ATTR(curr1_input, 0444, da9052_read_ich, NULL,
 			  DA9052_ADC_ICH);
-static SENSOR_DEVICE_ATTR(curr1_label, S_IRUGO, show_label, NULL,
+static SENSOR_DEVICE_ATTR(curr1_label, 0444, show_label, NULL,
 			  DA9052_ADC_ICH);
 
-static SENSOR_DEVICE_ATTR(temp2_input, S_IRUGO, da9052_read_tbat, NULL,
+static SENSOR_DEVICE_ATTR(temp2_input, 0444, da9052_read_tbat, NULL,
 			  DA9052_ADC_TBAT);
-static SENSOR_DEVICE_ATTR(temp2_label, S_IRUGO, show_label, NULL,
+static SENSOR_DEVICE_ATTR(temp2_label, 0444, show_label, NULL,
 			  DA9052_ADC_TBAT);
-static SENSOR_DEVICE_ATTR(temp8_input, S_IRUGO, da9052_read_tjunc, NULL,
+static SENSOR_DEVICE_ATTR(temp8_input, 0444, da9052_read_tjunc, NULL,
 			  DA9052_ADC_TJUNC);
-static SENSOR_DEVICE_ATTR(temp8_label, S_IRUGO, show_label, NULL,
+static SENSOR_DEVICE_ATTR(temp8_label, 0444, show_label, NULL,
 			  DA9052_ADC_TJUNC);
 
 static struct attribute *da9052_attrs[] = {
@@ -246,6 +391,14 @@ static struct attribute *da9052_attrs[] = {
 	&sensor_dev_attr_in5_label.dev_attr.attr,
 	&sensor_dev_attr_in6_input.dev_attr.attr,
 	&sensor_dev_attr_in6_label.dev_attr.attr,
+	&sensor_dev_attr_in70_input.dev_attr.attr,
+	&sensor_dev_attr_in70_label.dev_attr.attr,
+	&sensor_dev_attr_in71_input.dev_attr.attr,
+	&sensor_dev_attr_in71_label.dev_attr.attr,
+	&sensor_dev_attr_in72_input.dev_attr.attr,
+	&sensor_dev_attr_in72_label.dev_attr.attr,
+	&sensor_dev_attr_in73_input.dev_attr.attr,
+	&sensor_dev_attr_in73_label.dev_attr.attr,
 	&sensor_dev_attr_in9_input.dev_attr.attr,
 	&sensor_dev_attr_in9_label.dev_attr.attr,
 	&sensor_dev_attr_curr1_input.dev_attr.attr,
@@ -257,29 +410,117 @@ static struct attribute *da9052_attrs[] = {
 	NULL
 };
 
-ATTRIBUTE_GROUPS(da9052);
+static const struct attribute_group da9052_group = {
+	.attrs = da9052_attrs,
+	.is_visible = da9052_channel_is_visible,
+};
+__ATTRIBUTE_GROUPS(da9052);
+
+static irqreturn_t da9052_tsi_datardy_irq(int irq, void *data)
+{
+	struct da9052_hwmon *hwmon = data;
+
+	complete(&hwmon->tsidone);
+	return IRQ_HANDLED;
+}
 
 static int da9052_hwmon_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct da9052_hwmon *hwmon;
 	struct device *hwmon_dev;
+	int err;
 
 	hwmon = devm_kzalloc(dev, sizeof(struct da9052_hwmon), GFP_KERNEL);
 	if (!hwmon)
 		return -ENOMEM;
 
+	platform_set_drvdata(pdev, hwmon);
+
 	mutex_init(&hwmon->hwmon_lock);
 	hwmon->da9052 = dev_get_drvdata(pdev->dev.parent);
 
+	init_completion(&hwmon->tsidone);
+
+	hwmon->tsi_as_adc =
+		device_property_read_bool(pdev->dev.parent, "dlg,tsi-as-adc");
+
+	if (hwmon->tsi_as_adc) {
+		hwmon->tsiref = devm_regulator_get(pdev->dev.parent, "tsiref");
+		if (IS_ERR(hwmon->tsiref)) {
+			err = PTR_ERR(hwmon->tsiref);
+			dev_err(&pdev->dev, "failed to get tsiref: %d", err);
+			return err;
+		}
+
+		err = regulator_enable(hwmon->tsiref);
+		if (err)
+			return err;
+
+		hwmon->tsiref_mv = regulator_get_voltage(hwmon->tsiref);
+		if (hwmon->tsiref_mv < 0) {
+			err = hwmon->tsiref_mv;
+			goto exit_regulator;
+		}
+
+		/* convert from microvolt (DT) to millivolt (hwmon) */
+		hwmon->tsiref_mv /= 1000;
+
+		/* TSIREF limits from datasheet */
+		if (hwmon->tsiref_mv < 1800 || hwmon->tsiref_mv > 2600) {
+			dev_err(hwmon->da9052->dev, "invalid TSIREF voltage: %d",
+				hwmon->tsiref_mv);
+			err = -ENXIO;
+			goto exit_regulator;
+		}
+
+		/* disable touchscreen features */
+		da9052_reg_write(hwmon->da9052, DA9052_TSI_CONT_A_REG, 0x00);
+
+		err = da9052_request_irq(hwmon->da9052, DA9052_IRQ_TSIREADY,
+					 "tsiready-irq", da9052_tsi_datardy_irq,
+					 hwmon);
+		if (err) {
+			dev_err(&pdev->dev, "Failed to register TSIRDY IRQ: %d",
+				err);
+			goto exit_regulator;
+		}
+	}
+
 	hwmon_dev = devm_hwmon_device_register_with_groups(dev, "da9052",
 							   hwmon,
 							   da9052_groups);
-	return PTR_ERR_OR_ZERO(hwmon_dev);
+	err = PTR_ERR_OR_ZERO(hwmon_dev);
+	if (err)
+		goto exit_irq;
+
+	return 0;
+
+exit_irq:
+	if (hwmon->tsi_as_adc)
+		da9052_free_irq(hwmon->da9052, DA9052_IRQ_TSIREADY, hwmon);
+exit_regulator:
+	if (hwmon->tsiref)
+		regulator_disable(hwmon->tsiref);
+
+	return err;
+}
+
+static int da9052_hwmon_remove(struct platform_device *pdev)
+{
+	struct da9052_hwmon *hwmon = platform_get_drvdata(pdev);
+
+	if (hwmon->tsi_as_adc) {
+		da9052_free_irq(hwmon->da9052, DA9052_IRQ_TSIREADY, hwmon);
+		regulator_disable(hwmon->tsiref);
+	}
+
+	return 0;
 }
 
 static struct platform_driver da9052_hwmon_driver = {
 	.probe = da9052_hwmon_probe,
+	.remove = da9052_hwmon_remove,
 	.driver = {
 		.name = "da9052-hwmon",
 	},
diff --git a/drivers/hwmon/dell-smm-hwmon.c b/drivers/hwmon/dell-smm-hwmon.c
index 3189246302a6..c7c9e95e58a8 100644
--- a/drivers/hwmon/dell-smm-hwmon.c
+++ b/drivers/hwmon/dell-smm-hwmon.c
@@ -890,7 +890,7 @@ static const struct i8k_config_data i8k_config_data[] = {
 	},
 };
 
-static struct dmi_system_id i8k_dmi_table[] __initdata = {
+static const struct dmi_system_id i8k_dmi_table[] __initconst = {
 	{
 		.ident = "Dell Inspiron",
 		.matches = {
@@ -1013,7 +1013,7 @@ MODULE_DEVICE_TABLE(dmi, i8k_dmi_table);
  * of affected Dell machines for which we disallow I8K_SMM_GET_FAN_TYPE call.
  * See bug: https://bugzilla.kernel.org/show_bug.cgi?id=100121
  */
-static struct dmi_system_id i8k_blacklist_fan_type_dmi_table[] __initdata = {
+static const struct dmi_system_id i8k_blacklist_fan_type_dmi_table[] __initconst = {
 	{
 		.ident = "Dell Studio XPS 8000",
 		.matches = {
diff --git a/drivers/hwmon/ftsteutates.c b/drivers/hwmon/ftsteutates.c
index 0f0277e7aae5..0801f48a41f7 100644
--- a/drivers/hwmon/ftsteutates.c
+++ b/drivers/hwmon/ftsteutates.c
@@ -60,7 +60,7 @@
 
 static const unsigned short normal_i2c[] = { 0x73, I2C_CLIENT_END };
 
-static struct i2c_device_id fts_id[] = {
+static const struct i2c_device_id fts_id[] = {
 	{ "ftsteutates", 0 },
 	{ }
 };
@@ -435,6 +435,7 @@ clear_temp_alarm(struct device *dev, struct device_attribute *devattr,
 		goto error;
 
 	data->valid = false;
+	ret = count;
 error:
 	mutex_unlock(&data->update_lock);
 	return ret;
@@ -508,6 +509,7 @@ clear_fan_alarm(struct device *dev, struct device_attribute *devattr,
 		goto error;
 
 	data->valid = false;
+	ret = count;
 error:
 	mutex_unlock(&data->update_lock);
 	return ret;
diff --git a/drivers/hwmon/hwmon.c b/drivers/hwmon/hwmon.c
index dd6e17c1076b..c9790e2c3440 100644
--- a/drivers/hwmon/hwmon.c
+++ b/drivers/hwmon/hwmon.c
@@ -85,7 +85,7 @@ static umode_t hwmon_dev_name_is_visible(struct kobject *kobj,
 	return attr->mode;
 }
 
-static struct attribute_group hwmon_dev_attr_group = {
+static const struct attribute_group hwmon_dev_attr_group = {
 	.attrs		= hwmon_dev_attrs,
 	.is_visible	= hwmon_dev_name_is_visible,
 };
@@ -135,7 +135,7 @@ static int hwmon_thermal_get_temp(void *data, int *temp)
 	return 0;
 }
 
-static struct thermal_zone_of_device_ops hwmon_thermal_ops = {
+static const struct thermal_zone_of_device_ops hwmon_thermal_ops = {
 	.get_temp = hwmon_thermal_get_temp,
 };
 
diff --git a/drivers/hwmon/i5k_amb.c b/drivers/hwmon/i5k_amb.c
index a5a9f457b7f7..9397d2f0e79a 100644
--- a/drivers/hwmon/i5k_amb.c
+++ b/drivers/hwmon/i5k_amb.c
@@ -495,7 +495,7 @@ static struct {
 };
 
 #ifdef MODULE
-static struct pci_device_id i5k_amb_ids[] = {
+static const struct pci_device_id i5k_amb_ids[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_5000_ERR) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_5400_ERR) },
 	{ 0, }
diff --git a/drivers/hwmon/it87.c b/drivers/hwmon/it87.c
index 4dfc7238313e..f8499cb95fec 100644
--- a/drivers/hwmon/it87.c
+++ b/drivers/hwmon/it87.c
@@ -497,12 +497,14 @@ static const struct it87_devices it87_devices[] = {
 #define has_vin3_5v(data)	((data)->features & FEAT_VIN3_5V)
 
 struct it87_sio_data {
+	int sioaddr;
 	enum chips type;
 	/* Values read from Super-I/O config space */
 	u8 revision;
 	u8 vid_value;
 	u8 beep_pin;
 	u8 internal;	/* Internal sensors can be labeled */
+	bool need_in7_reroute;
 	/* Features skipped based on config or DMI */
 	u16 skip_in;
 	u8 skip_vid;
@@ -517,6 +519,7 @@ struct it87_sio_data {
  */
 struct it87_data {
 	const struct attribute_group *groups[7];
+	int sioaddr;
 	enum chips type;
 	u32 features;
 	u8 peci_mask;
@@ -532,6 +535,7 @@ struct it87_data {
 	u16 in_internal;	/* Bitfield, internal sensors (for labels) */
 	u16 has_in;		/* Bitfield, voltage sensors enabled */
 	u8 in[NUM_VIN][3];		/* [nr][0]=in, [1]=min, [2]=max */
+	bool need_in7_reroute;
 	u8 has_fan;		/* Bitfield, fans enabled */
 	u16 fan[NUM_FAN][2];	/* Register values, [nr][0]=fan, [1]=min */
 	u8 has_temp;		/* Bitfield, temp sensors enabled */
@@ -2487,6 +2491,7 @@ static int __init it87_find(int sioaddr, unsigned short *address,
 	}
 
 	err = 0;
+	sio_data->sioaddr = sioaddr;
 	sio_data->revision = superio_inb(sioaddr, DEVREV) & 0x0f;
 	pr_info("Found IT%04x%s chip at 0x%x, revision %d\n", chip_type,
 		it87_devices[sio_data->type].suffix,
@@ -2575,6 +2580,7 @@ static int __init it87_find(int sioaddr, unsigned short *address,
 				reg2c |= BIT(1);
 				superio_outb(sioaddr, IT87_SIO_PINX2_REG,
 					     reg2c);
+				sio_data->need_in7_reroute = true;
 				pr_notice("Routing internal VCCH5V to in7.\n");
 			}
 			pr_notice("in7 routed to internal voltage divider, with external pin disabled.\n");
@@ -2761,13 +2767,13 @@ static int __init it87_find(int sioaddr, unsigned short *address,
 		uart6 = sio_data->type == it8782 && (reg & BIT(2));
 
 		/*
-		 * The IT8720F has no VIN7 pin, so VCCH should always be
+		 * The IT8720F has no VIN7 pin, so VCCH5V should always be
 		 * routed internally to VIN7 with an internal divider.
 		 * Curiously, there still is a configuration bit to control
 		 * this, which means it can be set incorrectly. And even
 		 * more curiously, many boards out there are improperly
 		 * configured, even though the IT8720F datasheet claims
-		 * that the internal routing of VCCH to VIN7 is the default
+		 * that the internal routing of VCCH5V to VIN7 is the default
 		 * setting. So we force the internal routing in this case.
 		 *
 		 * On IT8782F, VIN7 is multiplexed with one of the UART6 pins.
@@ -2777,7 +2783,8 @@ static int __init it87_find(int sioaddr, unsigned short *address,
 		if ((sio_data->type == it8720 || uart6) && !(reg & BIT(1))) {
 			reg |= BIT(1);
 			superio_outb(sioaddr, IT87_SIO_PINX2_REG, reg);
-			pr_notice("Routing internal VCCH to in7\n");
+			sio_data->need_in7_reroute = true;
+			pr_notice("Routing internal VCCH5V to in7\n");
 		}
 		if (reg & BIT(0))
 			sio_data->internal |= BIT(0);
@@ -2828,13 +2835,89 @@ exit:
 	return err;
 }
 
+/*
+ * Some chips seem to have default value 0xff for all limit
+ * registers. For low voltage limits it makes no sense and triggers
+ * alarms, so change to 0 instead. For high temperature limits, it
+ * means -1 degree C, which surprisingly doesn't trigger an alarm,
+ * but is still confusing, so change to 127 degrees C.
+ */
+static void it87_check_limit_regs(struct it87_data *data)
+{
+	int i, reg;
+
+	for (i = 0; i < NUM_VIN_LIMIT; i++) {
+		reg = it87_read_value(data, IT87_REG_VIN_MIN(i));
+		if (reg == 0xff)
+			it87_write_value(data, IT87_REG_VIN_MIN(i), 0);
+	}
+	for (i = 0; i < NUM_TEMP_LIMIT; i++) {
+		reg = it87_read_value(data, IT87_REG_TEMP_HIGH(i));
+		if (reg == 0xff)
+			it87_write_value(data, IT87_REG_TEMP_HIGH(i), 127);
+	}
+}
+
+/* Check if voltage monitors are reset manually or by some reason */
+static void it87_check_voltage_monitors_reset(struct it87_data *data)
+{
+	int reg;
+
+	reg = it87_read_value(data, IT87_REG_VIN_ENABLE);
+	if ((reg & 0xff) == 0) {
+		/* Enable all voltage monitors */
+		it87_write_value(data, IT87_REG_VIN_ENABLE, 0xff);
+	}
+}
+
+/* Check if tachometers are reset manually or by some reason */
+static void it87_check_tachometers_reset(struct platform_device *pdev)
+{
+	struct it87_sio_data *sio_data = dev_get_platdata(&pdev->dev);
+	struct it87_data *data = platform_get_drvdata(pdev);
+	u8 mask, fan_main_ctrl;
+
+	mask = 0x70 & ~(sio_data->skip_fan << 4);
+	fan_main_ctrl = it87_read_value(data, IT87_REG_FAN_MAIN_CTRL);
+	if ((fan_main_ctrl & mask) == 0) {
+		/* Enable all fan tachometers */
+		fan_main_ctrl |= mask;
+		it87_write_value(data, IT87_REG_FAN_MAIN_CTRL,
+				 fan_main_ctrl);
+	}
+}
+
+/* Set tachometers to 16-bit mode if needed */
+static void it87_check_tachometers_16bit_mode(struct platform_device *pdev)
+{
+	struct it87_data *data = platform_get_drvdata(pdev);
+	int reg;
+
+	if (!has_fan16_config(data))
+		return;
+
+	reg = it87_read_value(data, IT87_REG_FAN_16BIT);
+	if (~reg & 0x07 & data->has_fan) {
+		dev_dbg(&pdev->dev,
+			"Setting fan1-3 to 16-bit mode\n");
+		it87_write_value(data, IT87_REG_FAN_16BIT,
+				 reg | 0x07);
+	}
+}
+
+static void it87_start_monitoring(struct it87_data *data)
+{
+	it87_write_value(data, IT87_REG_CONFIG,
+			 (it87_read_value(data, IT87_REG_CONFIG) & 0x3e)
+			 | (update_vbat ? 0x41 : 0x01));
+}
+
 /* Called when we have found a new IT87. */
 static void it87_init_device(struct platform_device *pdev)
 {
 	struct it87_sio_data *sio_data = dev_get_platdata(&pdev->dev);
 	struct it87_data *data = platform_get_drvdata(pdev);
 	int tmp, i;
-	u8 mask;
 
 	/*
 	 * For each PWM channel:
@@ -2855,23 +2938,7 @@ static void it87_init_device(struct platform_device *pdev)
 		data->auto_pwm[i][3] = 0x7f;	/* Full speed, hard-coded */
 	}
 
-	/*
-	 * Some chips seem to have default value 0xff for all limit
-	 * registers. For low voltage limits it makes no sense and triggers
-	 * alarms, so change to 0 instead. For high temperature limits, it
-	 * means -1 degree C, which surprisingly doesn't trigger an alarm,
-	 * but is still confusing, so change to 127 degrees C.
-	 */
-	for (i = 0; i < NUM_VIN_LIMIT; i++) {
-		tmp = it87_read_value(data, IT87_REG_VIN_MIN(i));
-		if (tmp == 0xff)
-			it87_write_value(data, IT87_REG_VIN_MIN(i), 0);
-	}
-	for (i = 0; i < NUM_TEMP_LIMIT; i++) {
-		tmp = it87_read_value(data, IT87_REG_TEMP_HIGH(i));
-		if (tmp == 0xff)
-			it87_write_value(data, IT87_REG_TEMP_HIGH(i), 127);
-	}
+	it87_check_limit_regs(data);
 
 	/*
 	 * Temperature channels are not forcibly enabled, as they can be
@@ -2880,38 +2947,19 @@ static void it87_init_device(struct platform_device *pdev)
 	 * run-time through the temp{1-3}_type sysfs accessors if needed.
 	 */
 
-	/* Check if voltage monitors are reset manually or by some reason */
-	tmp = it87_read_value(data, IT87_REG_VIN_ENABLE);
-	if ((tmp & 0xff) == 0) {
-		/* Enable all voltage monitors */
-		it87_write_value(data, IT87_REG_VIN_ENABLE, 0xff);
-	}
+	it87_check_voltage_monitors_reset(data);
+
+	it87_check_tachometers_reset(pdev);
 
-	/* Check if tachometers are reset manually or by some reason */
-	mask = 0x70 & ~(sio_data->skip_fan << 4);
 	data->fan_main_ctrl = it87_read_value(data, IT87_REG_FAN_MAIN_CTRL);
-	if ((data->fan_main_ctrl & mask) == 0) {
-		/* Enable all fan tachometers */
-		data->fan_main_ctrl |= mask;
-		it87_write_value(data, IT87_REG_FAN_MAIN_CTRL,
-				 data->fan_main_ctrl);
-	}
 	data->has_fan = (data->fan_main_ctrl >> 4) & 0x07;
 
-	tmp = it87_read_value(data, IT87_REG_FAN_16BIT);
-
-	/* Set tachometers to 16-bit mode if needed */
-	if (has_fan16_config(data)) {
-		if (~tmp & 0x07 & data->has_fan) {
-			dev_dbg(&pdev->dev,
-				"Setting fan1-3 to 16-bit mode\n");
-			it87_write_value(data, IT87_REG_FAN_16BIT,
-					 tmp | 0x07);
-		}
-	}
+	it87_check_tachometers_16bit_mode(pdev);
 
 	/* Check for additional fans */
 	if (has_five_fans(data)) {
+		tmp = it87_read_value(data, IT87_REG_FAN_16BIT);
+
 		if (tmp & BIT(4))
 			data->has_fan |= BIT(3); /* fan4 enabled */
 		if (tmp & BIT(5))
@@ -2933,10 +2981,7 @@ static void it87_init_device(struct platform_device *pdev)
 			sio_data->skip_pwm |= BIT(5);
 	}
 
-	/* Start monitoring */
-	it87_write_value(data, IT87_REG_CONFIG,
-			 (it87_read_value(data, IT87_REG_CONFIG) & 0x3e)
-			 | (update_vbat ? 0x41 : 0x01));
+	it87_start_monitoring(data);
 }
 
 /* Return 1 if and only if the PWM interface is safe to use */
@@ -2986,8 +3031,6 @@ static int it87_check_pwm(struct device *dev)
 				 "PWM configuration is too broken to be fixed\n");
 		}
 
-		dev_info(dev,
-			 "Detected broken BIOS defaults, disabling PWM interface\n");
 		return 0;
 	} else if (fix_pwm_polarity) {
 		dev_info(dev,
@@ -3020,6 +3063,7 @@ static int it87_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	data->addr = res->start;
+	data->sioaddr = sio_data->sioaddr;
 	data->type = sio_data->type;
 	data->features = it87_devices[sio_data->type].features;
 	data->peci_mask = it87_devices[sio_data->type].peci_mask;
@@ -3058,6 +3102,9 @@ static int it87_probe(struct platform_device *pdev)
 
 	/* Check PWM configuration */
 	enable_pwm_interface = it87_check_pwm(dev);
+	if (!enable_pwm_interface)
+		dev_info(dev,
+			 "Detected broken BIOS defaults, disabling PWM interface\n");
 
 	/* Starting with IT8721F, we handle scaling of internal voltages */
 	if (has_12mv_adc(data)) {
@@ -3085,6 +3132,7 @@ static int it87_probe(struct platform_device *pdev)
 	}
 
 	data->in_internal = sio_data->internal;
+	data->need_in7_reroute = sio_data->need_in7_reroute;
 	data->has_in = 0x3ff & ~sio_data->skip_in;
 
 	if (has_six_temp(data)) {
@@ -3140,9 +3188,71 @@ static int it87_probe(struct platform_device *pdev)
 	return PTR_ERR_OR_ZERO(hwmon_dev);
 }
 
+static void __maybe_unused it87_resume_sio(struct platform_device *pdev)
+{
+	struct it87_data *data = dev_get_drvdata(&pdev->dev);
+	int err;
+	int reg2c;
+
+	if (!data->need_in7_reroute)
+		return;
+
+	err = superio_enter(data->sioaddr);
+	if (err) {
+		dev_warn(&pdev->dev,
+			 "Unable to enter Super I/O to reroute in7 (%d)",
+			 err);
+		return;
+	}
+
+	superio_select(data->sioaddr, GPIO);
+
+	reg2c = superio_inb(data->sioaddr, IT87_SIO_PINX2_REG);
+	if (!(reg2c & BIT(1))) {
+		dev_dbg(&pdev->dev,
+			"Routing internal VCCH5V to in7 again");
+
+		reg2c |= BIT(1);
+		superio_outb(data->sioaddr, IT87_SIO_PINX2_REG,
+			     reg2c);
+	}
+
+	superio_exit(data->sioaddr);
+}
+
+static int __maybe_unused it87_resume(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct it87_data *data = dev_get_drvdata(dev);
+
+	it87_resume_sio(pdev);
+
+	mutex_lock(&data->update_lock);
+
+	it87_check_pwm(dev);
+	it87_check_limit_regs(data);
+	it87_check_voltage_monitors_reset(data);
+	it87_check_tachometers_reset(pdev);
+	it87_check_tachometers_16bit_mode(pdev);
+
+	it87_start_monitoring(data);
+
+	/* force update */
+	data->valid = 0;
+
+	mutex_unlock(&data->update_lock);
+
+	it87_update_device(dev);
+
+	return 0;
+}
+
+static SIMPLE_DEV_PM_OPS(it87_dev_pm_ops, NULL, it87_resume);
+
 static struct platform_driver it87_driver = {
 	.driver = {
 		.name	= DRVNAME,
+		.pm     = &it87_dev_pm_ops,
 	},
 	.probe	= it87_probe,
 };
diff --git a/drivers/hwmon/jc42.c b/drivers/hwmon/jc42.c
index 1bf22eff0b08..5f11dc014ed6 100644
--- a/drivers/hwmon/jc42.c
+++ b/drivers/hwmon/jc42.c
@@ -72,6 +72,8 @@ static const unsigned short normal_i2c[] = {
 #define NXP_MANID		0x1131  /* NXP Semiconductors */
 #define ONS_MANID		0x1b09  /* ON Semiconductor */
 #define STM_MANID		0x104a  /* ST Microelectronics */
+#define GT_MANID		0x1c68	/* Giantec */
+#define GT_MANID2		0x132d	/* Giantec, 2nd mfg ID */
 
 /* Supported chips */
 
@@ -86,6 +88,13 @@ static const unsigned short normal_i2c[] = {
 #define AT30TSE004_DEVID	0x2200
 #define AT30TSE004_DEVID_MASK	0xffff
 
+/* Giantec */
+#define GT30TS00_DEVID		0x2200
+#define GT30TS00_DEVID_MASK	0xff00
+
+#define GT34TS02_DEVID		0x3300
+#define GT34TS02_DEVID_MASK	0xff00
+
 /* IDT */
 #define TSE2004_DEVID		0x2200
 #define TSE2004_DEVID_MASK	0xff00
@@ -130,6 +139,12 @@ static const unsigned short normal_i2c[] = {
 #define CAT6095_DEVID		0x0800	/* Also matches CAT34TS02 */
 #define CAT6095_DEVID_MASK	0xffe0
 
+#define CAT34TS02C_DEVID	0x0a00
+#define CAT34TS02C_DEVID_MASK	0xfff0
+
+#define CAT34TS04_DEVID		0x2200
+#define CAT34TS04_DEVID_MASK	0xfff0
+
 /* ST Microelectronics */
 #define STTS424_DEVID		0x0101
 #define STTS424_DEVID_MASK	0xffff
@@ -158,6 +173,8 @@ static struct jc42_chips jc42_chips[] = {
 	{ ADT_MANID, ADT7408_DEVID, ADT7408_DEVID_MASK },
 	{ ATMEL_MANID, AT30TS00_DEVID, AT30TS00_DEVID_MASK },
 	{ ATMEL_MANID2, AT30TSE004_DEVID, AT30TSE004_DEVID_MASK },
+	{ GT_MANID, GT30TS00_DEVID, GT30TS00_DEVID_MASK },
+	{ GT_MANID2, GT34TS02_DEVID, GT34TS02_DEVID_MASK },
 	{ IDT_MANID, TSE2004_DEVID, TSE2004_DEVID_MASK },
 	{ IDT_MANID, TS3000_DEVID, TS3000_DEVID_MASK },
 	{ IDT_MANID, TS3001_DEVID, TS3001_DEVID_MASK },
@@ -170,6 +187,8 @@ static struct jc42_chips jc42_chips[] = {
 	{ MCP_MANID, MCP9843_DEVID, MCP9843_DEVID_MASK },
 	{ NXP_MANID, SE97_DEVID, SE97_DEVID_MASK },
 	{ ONS_MANID, CAT6095_DEVID, CAT6095_DEVID_MASK },
+	{ ONS_MANID, CAT34TS02C_DEVID, CAT34TS02C_DEVID_MASK },
+	{ ONS_MANID, CAT34TS04_DEVID, CAT34TS04_DEVID_MASK },
 	{ NXP_MANID, SE98_DEVID, SE98_DEVID_MASK },
 	{ STM_MANID, STTS424_DEVID, STTS424_DEVID_MASK },
 	{ STM_MANID, STTS424E_DEVID, STTS424E_DEVID_MASK },
diff --git a/drivers/hwmon/ltq-cputemp.c b/drivers/hwmon/ltq-cputemp.c
new file mode 100644
index 000000000000..1d33f94594c1
--- /dev/null
+++ b/drivers/hwmon/ltq-cputemp.c
@@ -0,0 +1,163 @@
+/* Lantiq cpu temperature sensor driver
+ *
+ * Copyright (C) 2017 Florian Eckert <fe@dev.tdt.de>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version
+ *
+ * This program is distributed in the hope that it will be useful
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>
+ */
+
+#include <linux/bitops.h>
+#include <linux/delay.h>
+#include <linux/hwmon.h>
+#include <linux/hwmon-sysfs.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+
+#include <lantiq_soc.h>
+
+/* gphy1 configuration register contains cpu temperature */
+#define CGU_GPHY1_CR   0x0040
+#define CGU_TEMP_PD    BIT(19)
+
+static void ltq_cputemp_enable(void)
+{
+	ltq_cgu_w32(ltq_cgu_r32(CGU_GPHY1_CR) | CGU_TEMP_PD, CGU_GPHY1_CR);
+}
+
+static void ltq_cputemp_disable(void *data)
+{
+	ltq_cgu_w32(ltq_cgu_r32(CGU_GPHY1_CR) & ~CGU_TEMP_PD, CGU_GPHY1_CR);
+}
+
+static int ltq_read(struct device *dev, enum hwmon_sensor_types type,
+		    u32 attr, int channel, long *temp)
+{
+	int value;
+
+	switch (attr) {
+	case hwmon_temp_input:
+		/* get the temperature including one decimal place */
+		value = (ltq_cgu_r32(CGU_GPHY1_CR) >> 9) & 0x01FF;
+		value = value * 5;
+		/* range -38 to +154 °C, register value zero is -38.0 °C */
+		value -= 380;
+		/* scale temp to millidegree */
+		value = value * 100;
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	*temp = value;
+	return 0;
+}
+
+static umode_t ltq_is_visible(const void *_data, enum hwmon_sensor_types type,
+			      u32 attr, int channel)
+{
+	if (type != hwmon_temp)
+		return 0;
+
+	switch (attr) {
+	case hwmon_temp_input:
+		return 0444;
+	default:
+		return 0;
+	}
+}
+
+static const u32 ltq_chip_config[] = {
+	HWMON_C_REGISTER_TZ,
+	0
+};
+
+static const struct hwmon_channel_info ltq_chip = {
+	.type = hwmon_chip,
+	.config = ltq_chip_config,
+};
+
+static const u32 ltq_temp_config[] = {
+	HWMON_T_INPUT,
+	0
+};
+
+static const struct hwmon_channel_info ltq_temp = {
+	.type = hwmon_temp,
+	.config = ltq_temp_config,
+};
+
+static const struct hwmon_channel_info *ltq_info[] = {
+	&ltq_chip,
+	&ltq_temp,
+	NULL
+};
+
+static const struct hwmon_ops ltq_hwmon_ops = {
+	.is_visible = ltq_is_visible,
+	.read = ltq_read,
+};
+
+static const struct hwmon_chip_info ltq_chip_info = {
+	.ops = &ltq_hwmon_ops,
+	.info = ltq_info,
+};
+
+static int ltq_cputemp_probe(struct platform_device *pdev)
+{
+	struct device *hwmon_dev;
+	int err = 0;
+
+	/* available on vr9 v1.2 SoCs only */
+	if (ltq_soc_type() != SOC_TYPE_VR9_2)
+		return -ENODEV;
+
+	err = devm_add_action(&pdev->dev, ltq_cputemp_disable, NULL);
+	if (err)
+		return err;
+
+	ltq_cputemp_enable();
+
+	hwmon_dev = devm_hwmon_device_register_with_info(&pdev->dev,
+							 "ltq_cputemp",
+							 NULL,
+							 &ltq_chip_info,
+							 NULL);
+
+	if (IS_ERR(hwmon_dev)) {
+		dev_err(&pdev->dev, "Failed to register as hwmon device");
+		return PTR_ERR(hwmon_dev);
+	}
+
+	return 0;
+}
+
+const struct of_device_id ltq_cputemp_match[] = {
+	{ .compatible = "lantiq,cputemp" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, ltq_cputemp_match);
+
+static struct platform_driver ltq_cputemp_driver = {
+	.probe = ltq_cputemp_probe,
+	.driver = {
+		.name = "ltq-cputemp",
+		.of_match_table = ltq_cputemp_match,
+	},
+};
+
+module_platform_driver(ltq_cputemp_driver);
+
+MODULE_AUTHOR("Florian Eckert <fe@dev.tdt.de>");
+MODULE_DESCRIPTION("Lantiq cpu temperature sensor driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hwmon/nct7802.c b/drivers/hwmon/nct7802.c
index 12b94b094c0d..2876c18ed841 100644
--- a/drivers/hwmon/nct7802.c
+++ b/drivers/hwmon/nct7802.c
@@ -704,7 +704,7 @@ static umode_t nct7802_temp_is_visible(struct kobject *kobj,
 	return attr->mode;
 }
 
-static struct attribute_group nct7802_temp_group = {
+static const struct attribute_group nct7802_temp_group = {
 	.attrs = nct7802_temp_attrs,
 	.is_visible = nct7802_temp_is_visible,
 };
@@ -802,7 +802,7 @@ static umode_t nct7802_in_is_visible(struct kobject *kobj,
 	return attr->mode;
 }
 
-static struct attribute_group nct7802_in_group = {
+static const struct attribute_group nct7802_in_group = {
 	.attrs = nct7802_in_attrs,
 	.is_visible = nct7802_in_is_visible,
 };
@@ -880,7 +880,7 @@ static umode_t nct7802_fan_is_visible(struct kobject *kobj,
 	return attr->mode;
 }
 
-static struct attribute_group nct7802_fan_group = {
+static const struct attribute_group nct7802_fan_group = {
 	.attrs = nct7802_fan_attrs,
 	.is_visible = nct7802_fan_is_visible,
 };
@@ -898,7 +898,7 @@ static struct attribute *nct7802_pwm_attrs[] = {
 	NULL
 };
 
-static struct attribute_group nct7802_pwm_group = {
+static const struct attribute_group nct7802_pwm_group = {
 	.attrs = nct7802_pwm_attrs,
 };
 
@@ -1011,7 +1011,7 @@ static struct attribute *nct7802_auto_point_attrs[] = {
 	NULL
 };
 
-static struct attribute_group nct7802_auto_point_group = {
+static const struct attribute_group nct7802_auto_point_group = {
 	.attrs = nct7802_auto_point_attrs,
 };
 
diff --git a/drivers/hwmon/pmbus/Kconfig b/drivers/hwmon/pmbus/Kconfig
index 68d717a3fd59..40019325b517 100644
--- a/drivers/hwmon/pmbus/Kconfig
+++ b/drivers/hwmon/pmbus/Kconfig
@@ -37,6 +37,15 @@ config SENSORS_ADM1275
 	  This driver can also be built as a module. If so, the module will
 	  be called adm1275.
 
+config SENSORS_IBM_CFFPS
+	tristate "IBM Common Form Factor Power Supply"
+	help
+	  If you say yes here you get hardware monitoring support for the IBM
+	  Common Form Factor power supply.
+
+	  This driver can also be built as a module. If so, the module will
+	  be called ibm-cffps.
+
 config SENSORS_IR35221
 	tristate "Infineon IR35221"
 	default n
@@ -135,6 +144,15 @@ config SENSORS_TPS40422
 	  This driver can also be built as a module. If so, the module will
 	  be called tps40422.
 
+config SENSORS_TPS53679
+	tristate "TI TPS53679"
+	help
+	  If you say yes here you get hardware monitoring support for TI
+	  TPS53679.
+
+	  This driver can also be built as a module. If so, the module will
+	  be called tps53679.
+
 config SENSORS_UCD9000
 	tristate "TI UCD90120, UCD90124, UCD90160, UCD9090, UCD90910"
 	default n
diff --git a/drivers/hwmon/pmbus/Makefile b/drivers/hwmon/pmbus/Makefile
index 75bb7ca619d9..459a6be3390e 100644
--- a/drivers/hwmon/pmbus/Makefile
+++ b/drivers/hwmon/pmbus/Makefile
@@ -5,6 +5,7 @@
 obj-$(CONFIG_PMBUS)		+= pmbus_core.o
 obj-$(CONFIG_SENSORS_PMBUS)	+= pmbus.o
 obj-$(CONFIG_SENSORS_ADM1275)	+= adm1275.o
+obj-$(CONFIG_SENSORS_IBM_CFFPS)	+= ibm-cffps.o
 obj-$(CONFIG_SENSORS_IR35221)	+= ir35221.o
 obj-$(CONFIG_SENSORS_LM25066)	+= lm25066.o
 obj-$(CONFIG_SENSORS_LTC2978)	+= ltc2978.o
@@ -14,6 +15,7 @@ obj-$(CONFIG_SENSORS_MAX20751)	+= max20751.o
 obj-$(CONFIG_SENSORS_MAX34440)	+= max34440.o
 obj-$(CONFIG_SENSORS_MAX8688)	+= max8688.o
 obj-$(CONFIG_SENSORS_TPS40422)	+= tps40422.o
+obj-$(CONFIG_SENSORS_TPS53679)	+= tps53679.o
 obj-$(CONFIG_SENSORS_UCD9000)	+= ucd9000.o
 obj-$(CONFIG_SENSORS_UCD9200)	+= ucd9200.o
 obj-$(CONFIG_SENSORS_ZL6100)	+= zl6100.o
diff --git a/drivers/hwmon/pmbus/ibm-cffps.c b/drivers/hwmon/pmbus/ibm-cffps.c
new file mode 100644
index 000000000000..cb56da6834e5
--- /dev/null
+++ b/drivers/hwmon/pmbus/ibm-cffps.c
@@ -0,0 +1,151 @@
+/*
+ * Copyright 2017 IBM Corp.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/bitops.h>
+#include <linux/device.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+
+#include "pmbus.h"
+
+/* STATUS_MFR_SPECIFIC bits */
+#define CFFPS_MFR_FAN_FAULT			BIT(0)
+#define CFFPS_MFR_THERMAL_FAULT			BIT(1)
+#define CFFPS_MFR_OV_FAULT			BIT(2)
+#define CFFPS_MFR_UV_FAULT			BIT(3)
+#define CFFPS_MFR_PS_KILL			BIT(4)
+#define CFFPS_MFR_OC_FAULT			BIT(5)
+#define CFFPS_MFR_VAUX_FAULT			BIT(6)
+#define CFFPS_MFR_CURRENT_SHARE_WARNING		BIT(7)
+
+static int ibm_cffps_read_byte_data(struct i2c_client *client, int page,
+				    int reg)
+{
+	int rc, mfr;
+
+	switch (reg) {
+	case PMBUS_STATUS_VOUT:
+	case PMBUS_STATUS_IOUT:
+	case PMBUS_STATUS_TEMPERATURE:
+	case PMBUS_STATUS_FAN_12:
+		rc = pmbus_read_byte_data(client, page, reg);
+		if (rc < 0)
+			return rc;
+
+		mfr = pmbus_read_byte_data(client, page,
+					   PMBUS_STATUS_MFR_SPECIFIC);
+		if (mfr < 0)
+			/*
+			 * Return the status register instead of an error,
+			 * since we successfully read status.
+			 */
+			return rc;
+
+		/* Add MFR_SPECIFIC bits to the standard pmbus status regs. */
+		if (reg == PMBUS_STATUS_FAN_12) {
+			if (mfr & CFFPS_MFR_FAN_FAULT)
+				rc |= PB_FAN_FAN1_FAULT;
+		} else if (reg == PMBUS_STATUS_TEMPERATURE) {
+			if (mfr & CFFPS_MFR_THERMAL_FAULT)
+				rc |= PB_TEMP_OT_FAULT;
+		} else if (reg == PMBUS_STATUS_VOUT) {
+			if (mfr & (CFFPS_MFR_OV_FAULT | CFFPS_MFR_VAUX_FAULT))
+				rc |= PB_VOLTAGE_OV_FAULT;
+			if (mfr & CFFPS_MFR_UV_FAULT)
+				rc |= PB_VOLTAGE_UV_FAULT;
+		} else if (reg == PMBUS_STATUS_IOUT) {
+			if (mfr & CFFPS_MFR_OC_FAULT)
+				rc |= PB_IOUT_OC_FAULT;
+			if (mfr & CFFPS_MFR_CURRENT_SHARE_WARNING)
+				rc |= PB_CURRENT_SHARE_FAULT;
+		}
+		break;
+	default:
+		rc = -ENODATA;
+		break;
+	}
+
+	return rc;
+}
+
+static int ibm_cffps_read_word_data(struct i2c_client *client, int page,
+				    int reg)
+{
+	int rc, mfr;
+
+	switch (reg) {
+	case PMBUS_STATUS_WORD:
+		rc = pmbus_read_word_data(client, page, reg);
+		if (rc < 0)
+			return rc;
+
+		mfr = pmbus_read_byte_data(client, page,
+					   PMBUS_STATUS_MFR_SPECIFIC);
+		if (mfr < 0)
+			/*
+			 * Return the status register instead of an error,
+			 * since we successfully read status.
+			 */
+			return rc;
+
+		if (mfr & CFFPS_MFR_PS_KILL)
+			rc |= PB_STATUS_OFF;
+		break;
+	default:
+		rc = -ENODATA;
+		break;
+	}
+
+	return rc;
+}
+
+static struct pmbus_driver_info ibm_cffps_info = {
+	.pages = 1,
+	.func[0] = PMBUS_HAVE_VIN | PMBUS_HAVE_VOUT | PMBUS_HAVE_IOUT |
+		PMBUS_HAVE_PIN | PMBUS_HAVE_FAN12 | PMBUS_HAVE_TEMP |
+		PMBUS_HAVE_TEMP2 | PMBUS_HAVE_TEMP3 | PMBUS_HAVE_STATUS_VOUT |
+		PMBUS_HAVE_STATUS_IOUT | PMBUS_HAVE_STATUS_INPUT |
+		PMBUS_HAVE_STATUS_TEMP | PMBUS_HAVE_STATUS_FAN12,
+	.read_byte_data = ibm_cffps_read_byte_data,
+	.read_word_data = ibm_cffps_read_word_data,
+};
+
+static int ibm_cffps_probe(struct i2c_client *client,
+			   const struct i2c_device_id *id)
+{
+	return pmbus_do_probe(client, id, &ibm_cffps_info);
+}
+
+static const struct i2c_device_id ibm_cffps_id[] = {
+	{ "ibm_cffps1", 1 },
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, ibm_cffps_id);
+
+static const struct of_device_id ibm_cffps_of_match[] = {
+	{ .compatible = "ibm,cffps1" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, ibm_cffps_of_match);
+
+static struct i2c_driver ibm_cffps_driver = {
+	.driver = {
+		.name = "ibm-cffps",
+		.of_match_table = ibm_cffps_of_match,
+	},
+	.probe = ibm_cffps_probe,
+	.remove = pmbus_do_remove,
+	.id_table = ibm_cffps_id,
+};
+
+module_i2c_driver(ibm_cffps_driver);
+
+MODULE_AUTHOR("Eddie James");
+MODULE_DESCRIPTION("PMBus driver for IBM Common Form Factor power supplies");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hwmon/pmbus/lm25066.c b/drivers/hwmon/pmbus/lm25066.c
index a3d912cd3b8d..10d17fb8f283 100644
--- a/drivers/hwmon/pmbus/lm25066.c
+++ b/drivers/hwmon/pmbus/lm25066.c
@@ -28,7 +28,7 @@
 #include <linux/i2c.h>
 #include "pmbus.h"
 
-enum chips { lm25056, lm25063, lm25066, lm5064, lm5066 };
+enum chips { lm25056, lm25063, lm25066, lm5064, lm5066, lm5066i };
 
 #define LM25066_READ_VAUX		0xd0
 #define LM25066_MFR_READ_IIN		0xd1
@@ -65,7 +65,7 @@ struct __coeff {
 #define PSC_CURRENT_IN_L	(PSC_NUM_CLASSES)
 #define PSC_POWER_L		(PSC_NUM_CLASSES + 1)
 
-static struct __coeff lm25066_coeff[5][PSC_NUM_CLASSES + 2] = {
+static struct __coeff lm25066_coeff[6][PSC_NUM_CLASSES + 2] = {
 	[lm25056] = {
 		[PSC_VOLTAGE_IN] = {
 			.m = 16296,
@@ -210,6 +210,41 @@ static struct __coeff lm25066_coeff[5][PSC_NUM_CLASSES + 2] = {
 			.m = 16,
 		},
 	},
+	[lm5066i] = {
+		[PSC_VOLTAGE_IN] = {
+			.m = 4617,
+			.b = -140,
+			.R = -2,
+		},
+		[PSC_VOLTAGE_OUT] = {
+			.m = 4602,
+			.b = 500,
+			.R = -2,
+		},
+		[PSC_CURRENT_IN] = {
+			.m = 15076,
+			.b = -504,
+			.R = -2,
+		},
+		[PSC_CURRENT_IN_L] = {
+			.m = 7645,
+			.b = 100,
+			.R = -2,
+		},
+		[PSC_POWER] = {
+			.m = 1701,
+			.b = -4000,
+			.R = -3,
+		},
+		[PSC_POWER_L] = {
+			.m = 861,
+			.b = -965,
+			.R = -3,
+		},
+		[PSC_TEMPERATURE] = {
+			.m = 16,
+		},
+	},
 };
 
 struct lm25066_data {
@@ -250,6 +285,7 @@ static int lm25066_read_word_data(struct i2c_client *client, int page, int reg)
 			ret = DIV_ROUND_CLOSEST(ret * 70, 453);
 			break;
 		case lm5066:
+		case lm5066i:
 			/* VIN: 2.18 mV VAUX: 725 uV LSB */
 			ret = DIV_ROUND_CLOSEST(ret * 725, 2180);
 			break;
@@ -488,16 +524,18 @@ static int lm25066_probe(struct i2c_client *client,
 	info->m[PSC_VOLTAGE_OUT] = coeff[PSC_VOLTAGE_OUT].m;
 	info->b[PSC_VOLTAGE_OUT] = coeff[PSC_VOLTAGE_OUT].b;
 	info->R[PSC_VOLTAGE_OUT] = coeff[PSC_VOLTAGE_OUT].R;
-	info->b[PSC_CURRENT_IN] = coeff[PSC_CURRENT_IN].b;
 	info->R[PSC_CURRENT_IN] = coeff[PSC_CURRENT_IN].R;
-	info->b[PSC_POWER] = coeff[PSC_POWER].b;
 	info->R[PSC_POWER] = coeff[PSC_POWER].R;
 	if (config & LM25066_DEV_SETUP_CL) {
 		info->m[PSC_CURRENT_IN] = coeff[PSC_CURRENT_IN_L].m;
+		info->b[PSC_CURRENT_IN] = coeff[PSC_CURRENT_IN_L].b;
 		info->m[PSC_POWER] = coeff[PSC_POWER_L].m;
+		info->b[PSC_POWER] = coeff[PSC_POWER_L].b;
 	} else {
 		info->m[PSC_CURRENT_IN] = coeff[PSC_CURRENT_IN].m;
+		info->b[PSC_CURRENT_IN] = coeff[PSC_CURRENT_IN].b;
 		info->m[PSC_POWER] = coeff[PSC_POWER].m;
+		info->b[PSC_POWER] = coeff[PSC_POWER].b;
 	}
 
 	return pmbus_do_probe(client, id, info);
@@ -509,6 +547,7 @@ static const struct i2c_device_id lm25066_id[] = {
 	{"lm25066", lm25066},
 	{"lm5064", lm5064},
 	{"lm5066", lm5066},
+	{"lm5066i", lm5066i},
 	{ }
 };
 
diff --git a/drivers/hwmon/pmbus/pmbus.h b/drivers/hwmon/pmbus/pmbus.h
index bfcb13bae34b..4efa2bd4f6d8 100644
--- a/drivers/hwmon/pmbus/pmbus.h
+++ b/drivers/hwmon/pmbus/pmbus.h
@@ -341,7 +341,7 @@ enum pmbus_sensor_classes {
 #define PMBUS_HAVE_STATUS_VMON	BIT(19)
 
 enum pmbus_data_format { linear = 0, direct, vid };
-enum vrm_version { vr11 = 0, vr12 };
+enum vrm_version { vr11 = 0, vr12, vr13 };
 
 struct pmbus_driver_info {
 	int pages;		/* Total number of pages */
diff --git a/drivers/hwmon/pmbus/pmbus_core.c b/drivers/hwmon/pmbus/pmbus_core.c
index f1eff6b6c798..302f0aef59de 100644
--- a/drivers/hwmon/pmbus/pmbus_core.c
+++ b/drivers/hwmon/pmbus/pmbus_core.c
@@ -19,6 +19,7 @@
  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  */
 
+#include <linux/debugfs.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/init.h>
@@ -101,6 +102,7 @@ struct pmbus_data {
 	int num_attributes;
 	struct attribute_group group;
 	const struct attribute_group *groups[2];
+	struct dentry *debugfs;		/* debugfs device directory */
 
 	struct pmbus_sensor *sensors;
 
@@ -112,12 +114,20 @@ struct pmbus_data {
 	 * A single status register covers multiple attributes,
 	 * so we keep them all together.
 	 */
-	u8 status[PB_NUM_STATUS_REG];
-	u8 status_register;
+	u16 status[PB_NUM_STATUS_REG];
+
+	bool has_status_word;		/* device uses STATUS_WORD register */
+	int (*read_status)(struct i2c_client *client, int page);
 
 	u8 currpage;
 };
 
+struct pmbus_debugfs_entry {
+	struct i2c_client *client;
+	u8 page;
+	u8 reg;
+};
+
 void pmbus_clear_cache(struct i2c_client *client)
 {
 	struct pmbus_data *data = i2c_get_clientdata(client);
@@ -324,7 +334,7 @@ static int pmbus_check_status_cml(struct i2c_client *client)
 	struct pmbus_data *data = i2c_get_clientdata(client);
 	int status, status2;
 
-	status = _pmbus_read_byte_data(client, -1, data->status_register);
+	status = data->read_status(client, -1);
 	if (status < 0 || (status & PB_STATUS_CML)) {
 		status2 = _pmbus_read_byte_data(client, -1, PMBUS_STATUS_CML);
 		if (status2 < 0 || (status2 & PB_CML_FAULT_INVALID_COMMAND))
@@ -348,6 +358,23 @@ static bool pmbus_check_register(struct i2c_client *client,
 	return rv >= 0;
 }
 
+static bool pmbus_check_status_register(struct i2c_client *client, int page)
+{
+	int status;
+	struct pmbus_data *data = i2c_get_clientdata(client);
+
+	status = data->read_status(client, page);
+	if (status >= 0 && !(data->flags & PMBUS_SKIP_STATUS_CHECK) &&
+	    (status & PB_STATUS_CML)) {
+		status = _pmbus_read_byte_data(client, -1, PMBUS_STATUS_CML);
+		if (status < 0 || (status & PB_CML_FAULT_INVALID_COMMAND))
+			status = -EIO;
+	}
+
+	pmbus_clear_fault_page(client, -1);
+	return status >= 0;
+}
+
 bool pmbus_check_byte_register(struct i2c_client *client, int page, int reg)
 {
 	return pmbus_check_register(client, _pmbus_read_byte_data, page, reg);
@@ -394,8 +421,7 @@ static struct pmbus_data *pmbus_update_device(struct device *dev)
 
 		for (i = 0; i < info->pages; i++) {
 			data->status[PB_STATUS_BASE + i]
-			    = _pmbus_read_byte_data(client, i,
-						    data->status_register);
+			    = data->read_status(client, i);
 			for (j = 0; j < ARRAY_SIZE(pmbus_status); j++) {
 				struct _pmbus_status *s = &pmbus_status[j];
 
@@ -531,6 +557,10 @@ static long pmbus_reg2data_vid(struct pmbus_data *data,
 		if (val >= 0x01)
 			rv = 250 + (val - 1) * 5;
 		break;
+	case vr13:
+		if (val >= 0x01)
+			rv = 500 + (val - 1) * 10;
+		break;
 	}
 	return rv;
 }
@@ -716,10 +746,10 @@ static int pmbus_get_boolean(struct pmbus_data *data, struct pmbus_boolean *b,
 {
 	struct pmbus_sensor *s1 = b->s1;
 	struct pmbus_sensor *s2 = b->s2;
-	u16 reg = (index >> 8) & 0xffff;
-	u8 mask = index & 0xff;
+	u16 reg = (index >> 16) & 0xffff;
+	u16 mask = index & 0xffff;
 	int ret, status;
-	u8 regval;
+	u16 regval;
 
 	status = data->status[reg];
 	if (status < 0)
@@ -860,7 +890,7 @@ static int pmbus_add_boolean(struct pmbus_data *data,
 			     const char *name, const char *type, int seq,
 			     struct pmbus_sensor *s1,
 			     struct pmbus_sensor *s2,
-			     u16 reg, u8 mask)
+			     u16 reg, u16 mask)
 {
 	struct pmbus_boolean *boolean;
 	struct sensor_device_attribute *a;
@@ -876,7 +906,7 @@ static int pmbus_add_boolean(struct pmbus_data *data,
 	boolean->s1 = s1;
 	boolean->s2 = s2;
 	pmbus_attr_init(a, boolean->name, S_IRUGO, pmbus_show_boolean, NULL,
-			(reg << 8) | mask);
+			(reg << 16) | mask);
 
 	return pmbus_add_attribute(data, &a->dev_attr.attr);
 }
@@ -962,7 +992,7 @@ struct pmbus_limit_attr {
  */
 struct pmbus_sensor_attr {
 	u16 reg;			/* sensor register */
-	u8 gbit;			/* generic status bit */
+	u16 gbit;			/* generic status bit */
 	u8 nlimit;			/* # of limit registers */
 	enum pmbus_sensor_classes class;/* sensor class */
 	const char *label;		/* sensor label */
@@ -1028,6 +1058,7 @@ static int pmbus_add_sensor_attrs_one(struct i2c_client *client,
 				      const struct pmbus_sensor_attr *attr)
 {
 	struct pmbus_sensor *base;
+	bool upper = !!(attr->gbit & 0xff00);	/* need to check STATUS_WORD */
 	int ret;
 
 	if (attr->label) {
@@ -1048,11 +1079,12 @@ static int pmbus_add_sensor_attrs_one(struct i2c_client *client,
 		/*
 		 * Add generic alarm attribute only if there are no individual
 		 * alarm attributes, if there is a global alarm bit, and if
-		 * the generic status register for this page is accessible.
+		 * the generic status register (word or byte, depending on
+		 * which global bit is set) for this page is accessible.
 		 */
 		if (!ret && attr->gbit &&
-		    pmbus_check_byte_register(client, page,
-					      data->status_register)) {
+		    (!upper || (upper && data->has_status_word)) &&
+		    pmbus_check_status_register(client, page)) {
 			ret = pmbus_add_boolean(data, name, "alarm", index,
 						NULL, NULL,
 						PB_STATUS_BASE + page,
@@ -1308,6 +1340,7 @@ static const struct pmbus_sensor_attr current_attributes[] = {
 		.func = PMBUS_HAVE_IIN,
 		.sfunc = PMBUS_HAVE_STATUS_INPUT,
 		.sbase = PB_STATUS_INPUT_BASE,
+		.gbit = PB_STATUS_INPUT,
 		.limit = iin_limit_attrs,
 		.nlimit = ARRAY_SIZE(iin_limit_attrs),
 	}, {
@@ -1392,6 +1425,7 @@ static const struct pmbus_sensor_attr power_attributes[] = {
 		.func = PMBUS_HAVE_PIN,
 		.sfunc = PMBUS_HAVE_STATUS_INPUT,
 		.sbase = PB_STATUS_INPUT_BASE,
+		.gbit = PB_STATUS_INPUT,
 		.limit = pin_limit_attrs,
 		.nlimit = ARRAY_SIZE(pin_limit_attrs),
 	}, {
@@ -1729,6 +1763,16 @@ static int pmbus_identify_common(struct i2c_client *client,
 	return 0;
 }
 
+static int pmbus_read_status_byte(struct i2c_client *client, int page)
+{
+	return _pmbus_read_byte_data(client, page, PMBUS_STATUS_BYTE);
+}
+
+static int pmbus_read_status_word(struct i2c_client *client, int page)
+{
+	return _pmbus_read_word_data(client, page, PMBUS_STATUS_WORD);
+}
+
 static int pmbus_init_common(struct i2c_client *client, struct pmbus_data *data,
 			     struct pmbus_driver_info *info)
 {
@@ -1736,19 +1780,21 @@ static int pmbus_init_common(struct i2c_client *client, struct pmbus_data *data,
 	int page, ret;
 
 	/*
-	 * Some PMBus chips don't support PMBUS_STATUS_BYTE, so try
-	 * to use PMBUS_STATUS_WORD instead if that is the case.
+	 * Some PMBus chips don't support PMBUS_STATUS_WORD, so try
+	 * to use PMBUS_STATUS_BYTE instead if that is the case.
 	 * Bail out if both registers are not supported.
 	 */
-	data->status_register = PMBUS_STATUS_BYTE;
-	ret = i2c_smbus_read_byte_data(client, PMBUS_STATUS_BYTE);
-	if (ret < 0 || ret == 0xff) {
-		data->status_register = PMBUS_STATUS_WORD;
-		ret = i2c_smbus_read_word_data(client, PMBUS_STATUS_WORD);
-		if (ret < 0 || ret == 0xffff) {
+	data->read_status = pmbus_read_status_word;
+	ret = i2c_smbus_read_word_data(client, PMBUS_STATUS_WORD);
+	if (ret < 0 || ret == 0xffff) {
+		data->read_status = pmbus_read_status_byte;
+		ret = i2c_smbus_read_byte_data(client, PMBUS_STATUS_BYTE);
+		if (ret < 0 || ret == 0xff) {
 			dev_err(dev, "PMBus status register not found\n");
 			return -ENODEV;
 		}
+	} else {
+		data->has_status_word = true;
 	}
 
 	/* Enable PEC if the controller supports it */
@@ -1859,6 +1905,184 @@ static int pmbus_regulator_register(struct pmbus_data *data)
 }
 #endif
 
+static struct dentry *pmbus_debugfs_dir;	/* pmbus debugfs directory */
+
+#if IS_ENABLED(CONFIG_DEBUG_FS)
+static int pmbus_debugfs_get(void *data, u64 *val)
+{
+	int rc;
+	struct pmbus_debugfs_entry *entry = data;
+
+	rc = _pmbus_read_byte_data(entry->client, entry->page, entry->reg);
+	if (rc < 0)
+		return rc;
+
+	*val = rc;
+
+	return 0;
+}
+DEFINE_DEBUGFS_ATTRIBUTE(pmbus_debugfs_ops, pmbus_debugfs_get, NULL,
+			 "0x%02llx\n");
+
+static int pmbus_debugfs_get_status(void *data, u64 *val)
+{
+	int rc;
+	struct pmbus_debugfs_entry *entry = data;
+	struct pmbus_data *pdata = i2c_get_clientdata(entry->client);
+
+	rc = pdata->read_status(entry->client, entry->page);
+	if (rc < 0)
+		return rc;
+
+	*val = rc;
+
+	return 0;
+}
+DEFINE_DEBUGFS_ATTRIBUTE(pmbus_debugfs_ops_status, pmbus_debugfs_get_status,
+			 NULL, "0x%04llx\n");
+
+static int pmbus_init_debugfs(struct i2c_client *client,
+			      struct pmbus_data *data)
+{
+	int i, idx = 0;
+	char name[PMBUS_NAME_SIZE];
+	struct pmbus_debugfs_entry *entries;
+
+	if (!pmbus_debugfs_dir)
+		return -ENODEV;
+
+	/*
+	 * Create the debugfs directory for this device. Use the hwmon device
+	 * name to avoid conflicts (hwmon numbers are globally unique).
+	 */
+	data->debugfs = debugfs_create_dir(dev_name(data->hwmon_dev),
+					   pmbus_debugfs_dir);
+	if (IS_ERR_OR_NULL(data->debugfs)) {
+		data->debugfs = NULL;
+		return -ENODEV;
+	}
+
+	/* Allocate the max possible entries we need. */
+	entries = devm_kzalloc(data->dev,
+			       sizeof(*entries) * (data->info->pages * 10),
+			       GFP_KERNEL);
+	if (!entries)
+		return -ENOMEM;
+
+	for (i = 0; i < data->info->pages; ++i) {
+		/* Check accessibility of status register if it's not page 0 */
+		if (!i || pmbus_check_status_register(client, i)) {
+			/* No need to set reg as we have special read op. */
+			entries[idx].client = client;
+			entries[idx].page = i;
+			scnprintf(name, PMBUS_NAME_SIZE, "status%d", i);
+			debugfs_create_file(name, 0444, data->debugfs,
+					    &entries[idx++],
+					    &pmbus_debugfs_ops_status);
+		}
+
+		if (data->info->func[i] & PMBUS_HAVE_STATUS_VOUT) {
+			entries[idx].client = client;
+			entries[idx].page = i;
+			entries[idx].reg = PMBUS_STATUS_VOUT;
+			scnprintf(name, PMBUS_NAME_SIZE, "status%d_vout", i);
+			debugfs_create_file(name, 0444, data->debugfs,
+					    &entries[idx++],
+					    &pmbus_debugfs_ops);
+		}
+
+		if (data->info->func[i] & PMBUS_HAVE_STATUS_IOUT) {
+			entries[idx].client = client;
+			entries[idx].page = i;
+			entries[idx].reg = PMBUS_STATUS_IOUT;
+			scnprintf(name, PMBUS_NAME_SIZE, "status%d_iout", i);
+			debugfs_create_file(name, 0444, data->debugfs,
+					    &entries[idx++],
+					    &pmbus_debugfs_ops);
+		}
+
+		if (data->info->func[i] & PMBUS_HAVE_STATUS_INPUT) {
+			entries[idx].client = client;
+			entries[idx].page = i;
+			entries[idx].reg = PMBUS_STATUS_INPUT;
+			scnprintf(name, PMBUS_NAME_SIZE, "status%d_input", i);
+			debugfs_create_file(name, 0444, data->debugfs,
+					    &entries[idx++],
+					    &pmbus_debugfs_ops);
+		}
+
+		if (data->info->func[i] & PMBUS_HAVE_STATUS_TEMP) {
+			entries[idx].client = client;
+			entries[idx].page = i;
+			entries[idx].reg = PMBUS_STATUS_TEMPERATURE;
+			scnprintf(name, PMBUS_NAME_SIZE, "status%d_temp", i);
+			debugfs_create_file(name, 0444, data->debugfs,
+					    &entries[idx++],
+					    &pmbus_debugfs_ops);
+		}
+
+		if (pmbus_check_byte_register(client, i, PMBUS_STATUS_CML)) {
+			entries[idx].client = client;
+			entries[idx].page = i;
+			entries[idx].reg = PMBUS_STATUS_CML;
+			scnprintf(name, PMBUS_NAME_SIZE, "status%d_cml", i);
+			debugfs_create_file(name, 0444, data->debugfs,
+					    &entries[idx++],
+					    &pmbus_debugfs_ops);
+		}
+
+		if (pmbus_check_byte_register(client, i, PMBUS_STATUS_OTHER)) {
+			entries[idx].client = client;
+			entries[idx].page = i;
+			entries[idx].reg = PMBUS_STATUS_OTHER;
+			scnprintf(name, PMBUS_NAME_SIZE, "status%d_other", i);
+			debugfs_create_file(name, 0444, data->debugfs,
+					    &entries[idx++],
+					    &pmbus_debugfs_ops);
+		}
+
+		if (pmbus_check_byte_register(client, i,
+					      PMBUS_STATUS_MFR_SPECIFIC)) {
+			entries[idx].client = client;
+			entries[idx].page = i;
+			entries[idx].reg = PMBUS_STATUS_MFR_SPECIFIC;
+			scnprintf(name, PMBUS_NAME_SIZE, "status%d_mfr", i);
+			debugfs_create_file(name, 0444, data->debugfs,
+					    &entries[idx++],
+					    &pmbus_debugfs_ops);
+		}
+
+		if (data->info->func[i] & PMBUS_HAVE_STATUS_FAN12) {
+			entries[idx].client = client;
+			entries[idx].page = i;
+			entries[idx].reg = PMBUS_STATUS_FAN_12;
+			scnprintf(name, PMBUS_NAME_SIZE, "status%d_fan12", i);
+			debugfs_create_file(name, 0444, data->debugfs,
+					    &entries[idx++],
+					    &pmbus_debugfs_ops);
+		}
+
+		if (data->info->func[i] & PMBUS_HAVE_STATUS_FAN34) {
+			entries[idx].client = client;
+			entries[idx].page = i;
+			entries[idx].reg = PMBUS_STATUS_FAN_34;
+			scnprintf(name, PMBUS_NAME_SIZE, "status%d_fan34", i);
+			debugfs_create_file(name, 0444, data->debugfs,
+					    &entries[idx++],
+					    &pmbus_debugfs_ops);
+		}
+	}
+
+	return 0;
+}
+#else
+static int pmbus_init_debugfs(struct i2c_client *client,
+			      struct pmbus_data *data)
+{
+	return 0;
+}
+#endif	/* IS_ENABLED(CONFIG_DEBUG_FS) */
+
 int pmbus_do_probe(struct i2c_client *client, const struct i2c_device_id *id,
 		   struct pmbus_driver_info *info)
 {
@@ -1918,6 +2142,10 @@ int pmbus_do_probe(struct i2c_client *client, const struct i2c_device_id *id,
 	if (ret)
 		goto out_unregister;
 
+	ret = pmbus_init_debugfs(client, data);
+	if (ret)
+		dev_warn(dev, "Failed to register debugfs\n");
+
 	return 0;
 
 out_unregister:
@@ -1931,12 +2159,32 @@ EXPORT_SYMBOL_GPL(pmbus_do_probe);
 int pmbus_do_remove(struct i2c_client *client)
 {
 	struct pmbus_data *data = i2c_get_clientdata(client);
+
+	debugfs_remove_recursive(data->debugfs);
+
 	hwmon_device_unregister(data->hwmon_dev);
 	kfree(data->group.attrs);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(pmbus_do_remove);
 
+static int __init pmbus_core_init(void)
+{
+	pmbus_debugfs_dir = debugfs_create_dir("pmbus", NULL);
+	if (IS_ERR(pmbus_debugfs_dir))
+		pmbus_debugfs_dir = NULL;
+
+	return 0;
+}
+
+static void __exit pmbus_core_exit(void)
+{
+	debugfs_remove_recursive(pmbus_debugfs_dir);
+}
+
+module_init(pmbus_core_init);
+module_exit(pmbus_core_exit);
+
 MODULE_AUTHOR("Guenter Roeck");
 MODULE_DESCRIPTION("PMBus core driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/hwmon/pmbus/tps53679.c b/drivers/hwmon/pmbus/tps53679.c
new file mode 100644
index 000000000000..85b515cd9df0
--- /dev/null
+++ b/drivers/hwmon/pmbus/tps53679.c
@@ -0,0 +1,113 @@
+/*
+ * Hardware monitoring driver for Texas Instruments TPS53679
+ *
+ * Copyright (c) 2017 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2017 Vadim Pasternak <vadimp@mellanox.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include "pmbus.h"
+
+#define TPS53679_PROT_VR12_5MV		0x01 /* VR12.0 mode, 5-mV DAC */
+#define TPS53679_PROT_VR12_5_10MV	0x02 /* VR12.5 mode, 10-mV DAC */
+#define TPS53679_PROT_VR13_10MV		0x04 /* VR13.0 mode, 10-mV DAC */
+#define TPS53679_PROT_IMVP8_5MV		0x05 /* IMVP8 mode, 5-mV DAC */
+#define TPS53679_PROT_VR13_5MV		0x07 /* VR13.0 mode, 5-mV DAC */
+#define TPS53679_PAGE_NUM		2
+
+static int tps53679_identify(struct i2c_client *client,
+			     struct pmbus_driver_info *info)
+{
+	u8 vout_params;
+	int ret;
+
+	/* Read the register with VOUT scaling value.*/
+	ret = pmbus_read_byte_data(client, 0, PMBUS_VOUT_MODE);
+	if (ret < 0)
+		return ret;
+
+	vout_params = ret & GENMASK(4, 0);
+
+	switch (vout_params) {
+	case TPS53679_PROT_VR13_10MV:
+	case TPS53679_PROT_VR12_5_10MV:
+		info->vrm_version = vr13;
+		break;
+	case TPS53679_PROT_VR13_5MV:
+	case TPS53679_PROT_VR12_5MV:
+	case TPS53679_PROT_IMVP8_5MV:
+		info->vrm_version = vr12;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static struct pmbus_driver_info tps53679_info = {
+	.pages = TPS53679_PAGE_NUM,
+	.format[PSC_VOLTAGE_IN] = linear,
+	.format[PSC_VOLTAGE_OUT] = vid,
+	.format[PSC_TEMPERATURE] = linear,
+	.format[PSC_CURRENT_OUT] = linear,
+	.format[PSC_POWER] = linear,
+	.func[0] = PMBUS_HAVE_VIN | PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT |
+		PMBUS_HAVE_IOUT | PMBUS_HAVE_STATUS_IOUT |
+		PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP |
+		PMBUS_HAVE_POUT,
+	.func[1] = PMBUS_HAVE_VIN | PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT |
+		PMBUS_HAVE_IOUT | PMBUS_HAVE_STATUS_IOUT |
+		PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP |
+		PMBUS_HAVE_POUT,
+	.identify = tps53679_identify,
+};
+
+static int tps53679_probe(struct i2c_client *client,
+			  const struct i2c_device_id *id)
+{
+	return pmbus_do_probe(client, id, &tps53679_info);
+}
+
+static const struct i2c_device_id tps53679_id[] = {
+	{"tps53679", 0},
+	{}
+};
+
+MODULE_DEVICE_TABLE(i2c, tps53679_id);
+
+static const struct of_device_id tps53679_of_match[] = {
+	{.compatible = "ti,tps53679"},
+	{}
+};
+MODULE_DEVICE_TABLE(of, tps53679_of_match);
+
+static struct i2c_driver tps53679_driver = {
+	.driver = {
+		.name = "tps53679",
+		.of_match_table = of_match_ptr(tps53679_of_match),
+	},
+	.probe = tps53679_probe,
+	.remove = pmbus_do_remove,
+	.id_table = tps53679_id,
+};
+
+module_i2c_driver(tps53679_driver);
+
+MODULE_AUTHOR("Vadim Pasternak <vadimp@mellanox.com>");
+MODULE_DESCRIPTION("PMBus driver for Texas Instruments TPS53679");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hwmon/scpi-hwmon.c b/drivers/hwmon/scpi-hwmon.c
index a586480a7ca9..7e49da50bc69 100644
--- a/drivers/hwmon/scpi-hwmon.c
+++ b/drivers/hwmon/scpi-hwmon.c
@@ -120,7 +120,7 @@ scpi_show_label(struct device *dev, struct device_attribute *attr, char *buf)
 	return sprintf(buf, "%s\n", sensor->info.name);
 }
 
-static struct thermal_zone_of_device_ops scpi_sensor_ops = {
+static const struct thermal_zone_of_device_ops scpi_sensor_ops = {
 	.get_temp = scpi_read_temp,
 };
 
diff --git a/drivers/hwmon/stts751.c b/drivers/hwmon/stts751.c
index d56251d6eec2..3f940fb67dc6 100644
--- a/drivers/hwmon/stts751.c
+++ b/drivers/hwmon/stts751.c
@@ -718,6 +718,10 @@ static int stts751_read_chip_config(struct stts751_priv *priv)
 	ret = i2c_smbus_read_byte_data(priv->client, STTS751_REG_RATE);
 	if (ret < 0)
 		return ret;
+	if (ret >= ARRAY_SIZE(stts751_intervals)) {
+		dev_err(priv->dev, "Unrecognized conversion rate 0x%x\n", ret);
+		return -ENODEV;
+	}
 	priv->interval = ret;
 
 	ret = stts751_read_reg16(priv, &priv->event_max,
diff --git a/drivers/hwtracing/coresight/Kconfig b/drivers/hwtracing/coresight/Kconfig
index 8d55d6d79015..ef9cb3c164e1 100644
--- a/drivers/hwtracing/coresight/Kconfig
+++ b/drivers/hwtracing/coresight/Kconfig
@@ -70,13 +70,13 @@ config CORESIGHT_SOURCE_ETM4X
 	  for instruction level tracing. Depending on the implemented version
 	  data tracing may also be available.
 
-config CORESIGHT_QCOM_REPLICATOR
-	bool "Qualcomm CoreSight Replicator driver"
+config CORESIGHT_DYNAMIC_REPLICATOR
+	bool "CoreSight Programmable Replicator driver"
 	depends on CORESIGHT_LINKS_AND_SINKS
 	help
-	  This enables support for Qualcomm CoreSight link driver. The
-	  programmable ATB replicator sends the ATB trace stream from the
-	  ETB/ETF to the TPIUi and ETR.
+	  This enables support for dynamic CoreSight replicator link driver.
+	  The programmable ATB replicator allows independent filtering of the
+	  trace data based on the traceid.
 
 config CORESIGHT_STM
 	bool "CoreSight System Trace Macrocell driver"
diff --git a/drivers/hwtracing/coresight/Makefile b/drivers/hwtracing/coresight/Makefile
index 433d59025eb6..5bae90ce794d 100644
--- a/drivers/hwtracing/coresight/Makefile
+++ b/drivers/hwtracing/coresight/Makefile
@@ -14,6 +14,6 @@ obj-$(CONFIG_CORESIGHT_SOURCE_ETM3X) += coresight-etm3x.o coresight-etm-cp14.o \
 					coresight-etm3x-sysfs.o
 obj-$(CONFIG_CORESIGHT_SOURCE_ETM4X) += coresight-etm4x.o \
 					coresight-etm4x-sysfs.o
-obj-$(CONFIG_CORESIGHT_QCOM_REPLICATOR) += coresight-replicator-qcom.o
+obj-$(CONFIG_CORESIGHT_DYNAMIC_REPLICATOR) += coresight-dynamic-replicator.o
 obj-$(CONFIG_CORESIGHT_STM) += coresight-stm.o
 obj-$(CONFIG_CORESIGHT_CPU_DEBUG) += coresight-cpu-debug.o
diff --git a/drivers/hwtracing/coresight/coresight-cpu-debug.c b/drivers/hwtracing/coresight/coresight-cpu-debug.c
index 64a77e00eaa6..6ea62c62ff27 100644
--- a/drivers/hwtracing/coresight/coresight-cpu-debug.c
+++ b/drivers/hwtracing/coresight/coresight-cpu-debug.c
@@ -667,7 +667,7 @@ static int debug_remove(struct amba_device *adev)
 	return 0;
 }
 
-static struct amba_id debug_ids[] = {
+static const struct amba_id debug_ids[] = {
 	{       /* Debug for Cortex-A53 */
 		.id	= 0x000bbd03,
 		.mask	= 0x000fffff,
diff --git a/drivers/hwtracing/coresight/coresight-replicator-qcom.c b/drivers/hwtracing/coresight/coresight-dynamic-replicator.c
index 0a3d15f0b009..accc2056f7c6 100644
--- a/drivers/hwtracing/coresight/coresight-replicator-qcom.c
+++ b/drivers/hwtracing/coresight/coresight-dynamic-replicator.c
@@ -95,6 +95,28 @@ static const struct coresight_ops replicator_cs_ops = {
 	.link_ops	= &replicator_link_ops,
 };
 
+#define coresight_replicator_reg(name, offset) \
+	coresight_simple_reg32(struct replicator_state, name, offset)
+
+coresight_replicator_reg(idfilter0, REPLICATOR_IDFILTER0);
+coresight_replicator_reg(idfilter1, REPLICATOR_IDFILTER1);
+
+static struct attribute *replicator_mgmt_attrs[] = {
+	&dev_attr_idfilter0.attr,
+	&dev_attr_idfilter1.attr,
+	NULL,
+};
+
+static const struct attribute_group replicator_mgmt_group = {
+	.attrs = replicator_mgmt_attrs,
+	.name = "mgmt",
+};
+
+static const struct attribute_group *replicator_groups[] = {
+	&replicator_mgmt_group,
+	NULL,
+};
+
 static int replicator_probe(struct amba_device *adev, const struct amba_id *id)
 {
 	int ret;
@@ -139,11 +161,11 @@ static int replicator_probe(struct amba_device *adev, const struct amba_id *id)
 	desc.ops = &replicator_cs_ops;
 	desc.pdata = adev->dev.platform_data;
 	desc.dev = &adev->dev;
+	desc.groups = replicator_groups;
 	drvdata->csdev = coresight_register(&desc);
 	if (IS_ERR(drvdata->csdev))
 		return PTR_ERR(drvdata->csdev);
 
-	dev_info(dev, "%s initialized\n", (char *)id->data);
 	return 0;
 }
 
@@ -175,18 +197,22 @@ static const struct dev_pm_ops replicator_dev_pm_ops = {
 			   NULL)
 };
 
-static struct amba_id replicator_ids[] = {
+static const struct amba_id replicator_ids[] = {
 	{
 		.id     = 0x0003b909,
 		.mask   = 0x0003ffff,
-		.data	= "REPLICATOR 1.0",
+	},
+	{
+		/* Coresight SoC-600 */
+		.id     = 0x000bb9ec,
+		.mask   = 0x000fffff,
 	},
 	{ 0, 0 },
 };
 
 static struct amba_driver replicator_driver = {
 	.drv = {
-		.name	= "coresight-replicator-qcom",
+		.name	= "coresight-dynamic-replicator",
 		.pm	= &replicator_dev_pm_ops,
 		.suppress_bind_attrs = true,
 	},
diff --git a/drivers/hwtracing/coresight/coresight-etb10.c b/drivers/hwtracing/coresight/coresight-etb10.c
index d5b96423e1a5..56ecd7aff5eb 100644
--- a/drivers/hwtracing/coresight/coresight-etb10.c
+++ b/drivers/hwtracing/coresight/coresight-etb10.c
@@ -200,8 +200,10 @@ static void etb_disable_hw(struct etb_drvdata *drvdata)
 
 static void etb_dump_hw(struct etb_drvdata *drvdata)
 {
+	bool lost = false;
 	int i;
 	u8 *buf_ptr;
+	const u32 *barrier;
 	u32 read_data, depth;
 	u32 read_ptr, write_ptr;
 	u32 frame_off, frame_endoff;
@@ -223,20 +225,26 @@ static void etb_dump_hw(struct etb_drvdata *drvdata)
 	}
 
 	if ((readl_relaxed(drvdata->base + ETB_STATUS_REG)
-		      & ETB_STATUS_RAM_FULL) == 0)
+		      & ETB_STATUS_RAM_FULL) == 0) {
 		writel_relaxed(0x0, drvdata->base + ETB_RAM_READ_POINTER);
-	else
+	} else {
 		writel_relaxed(write_ptr, drvdata->base + ETB_RAM_READ_POINTER);
+		lost = true;
+	}
 
 	depth = drvdata->buffer_depth;
 	buf_ptr = drvdata->buf;
+	barrier = barrier_pkt;
 	for (i = 0; i < depth; i++) {
 		read_data = readl_relaxed(drvdata->base +
 					  ETB_RAM_READ_DATA_REG);
-		*buf_ptr++ = read_data >> 0;
-		*buf_ptr++ = read_data >> 8;
-		*buf_ptr++ = read_data >> 16;
-		*buf_ptr++ = read_data >> 24;
+		if (lost && *barrier) {
+			read_data = *barrier;
+			barrier++;
+		}
+
+		*(u32 *)buf_ptr = read_data;
+		buf_ptr += 4;
 	}
 
 	if (frame_off) {
@@ -353,8 +361,10 @@ static void etb_update_buffer(struct coresight_device *csdev,
 			      struct perf_output_handle *handle,
 			      void *sink_config)
 {
+	bool lost = false;
 	int i, cur;
 	u8 *buf_ptr;
+	const u32 *barrier;
 	u32 read_ptr, write_ptr, capacity;
 	u32 status, read_data, to_read;
 	unsigned long offset;
@@ -366,8 +376,8 @@ static void etb_update_buffer(struct coresight_device *csdev,
 
 	capacity = drvdata->buffer_depth * ETB_FRAME_SIZE_WORDS;
 
-	CS_UNLOCK(drvdata->base);
 	etb_disable_hw(drvdata);
+	CS_UNLOCK(drvdata->base);
 
 	/* unit is in words, not bytes */
 	read_ptr = readl_relaxed(drvdata->base + ETB_RAM_READ_POINTER);
@@ -384,7 +394,7 @@ static void etb_update_buffer(struct coresight_device *csdev,
 			(unsigned long)write_ptr);
 
 		write_ptr &= ~(ETB_FRAME_SIZE_WORDS - 1);
-		perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
+		lost = true;
 	}
 
 	/*
@@ -395,7 +405,7 @@ static void etb_update_buffer(struct coresight_device *csdev,
 	 */
 	status = readl_relaxed(drvdata->base + ETB_STATUS_REG);
 	if (status & ETB_STATUS_RAM_FULL) {
-		perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
+		lost = true;
 		to_read = capacity;
 		read_ptr = write_ptr;
 	} else {
@@ -428,22 +438,30 @@ static void etb_update_buffer(struct coresight_device *csdev,
 		if (read_ptr > (drvdata->buffer_depth - 1))
 			read_ptr -= drvdata->buffer_depth;
 		/* let the decoder know we've skipped ahead */
-		perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
+		lost = true;
 	}
 
+	if (lost)
+		perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
+
 	/* finally tell HW where we want to start reading from */
 	writel_relaxed(read_ptr, drvdata->base + ETB_RAM_READ_POINTER);
 
 	cur = buf->cur;
 	offset = buf->offset;
+	barrier = barrier_pkt;
+
 	for (i = 0; i < to_read; i += 4) {
 		buf_ptr = buf->data_pages[cur] + offset;
 		read_data = readl_relaxed(drvdata->base +
 					  ETB_RAM_READ_DATA_REG);
-		*buf_ptr++ = read_data >> 0;
-		*buf_ptr++ = read_data >> 8;
-		*buf_ptr++ = read_data >> 16;
-		*buf_ptr++ = read_data >> 24;
+		if (lost && *barrier) {
+			read_data = *barrier;
+			barrier++;
+		}
+
+		*(u32 *)buf_ptr = read_data;
+		buf_ptr += 4;
 
 		offset += 4;
 		if (offset >= PAGE_SIZE) {
@@ -557,17 +575,17 @@ static const struct file_operations etb_fops = {
 	.llseek		= no_llseek,
 };
 
-#define coresight_etb10_simple_func(name, offset)                       \
-	coresight_simple_func(struct etb_drvdata, NULL, name, offset)
+#define coresight_etb10_reg(name, offset)		\
+	coresight_simple_reg32(struct etb_drvdata, name, offset)
 
-coresight_etb10_simple_func(rdp, ETB_RAM_DEPTH_REG);
-coresight_etb10_simple_func(sts, ETB_STATUS_REG);
-coresight_etb10_simple_func(rrp, ETB_RAM_READ_POINTER);
-coresight_etb10_simple_func(rwp, ETB_RAM_WRITE_POINTER);
-coresight_etb10_simple_func(trg, ETB_TRG);
-coresight_etb10_simple_func(ctl, ETB_CTL_REG);
-coresight_etb10_simple_func(ffsr, ETB_FFSR);
-coresight_etb10_simple_func(ffcr, ETB_FFCR);
+coresight_etb10_reg(rdp, ETB_RAM_DEPTH_REG);
+coresight_etb10_reg(sts, ETB_STATUS_REG);
+coresight_etb10_reg(rrp, ETB_RAM_READ_POINTER);
+coresight_etb10_reg(rwp, ETB_RAM_WRITE_POINTER);
+coresight_etb10_reg(trg, ETB_TRG);
+coresight_etb10_reg(ctl, ETB_CTL_REG);
+coresight_etb10_reg(ffsr, ETB_FFSR);
+coresight_etb10_reg(ffcr, ETB_FFCR);
 
 static struct attribute *coresight_etb_mgmt_attrs[] = {
 	&dev_attr_rdp.attr,
@@ -728,7 +746,7 @@ static const struct dev_pm_ops etb_dev_pm_ops = {
 	SET_RUNTIME_PM_OPS(etb_runtime_suspend, etb_runtime_resume, NULL)
 };
 
-static struct amba_id etb_ids[] = {
+static const struct amba_id etb_ids[] = {
 	{
 		.id	= 0x0003b907,
 		.mask	= 0x0003ffff,
diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c
index 8f546f59a3fd..8a0ad77574e7 100644
--- a/drivers/hwtracing/coresight/coresight-etm-perf.c
+++ b/drivers/hwtracing/coresight/coresight-etm-perf.c
@@ -53,14 +53,16 @@ static DEFINE_PER_CPU(struct coresight_device *, csdev_src);
 /* ETMv3.5/PTM's ETMCR is 'config' */
 PMU_FORMAT_ATTR(cycacc,		"config:" __stringify(ETM_OPT_CYCACC));
 PMU_FORMAT_ATTR(timestamp,	"config:" __stringify(ETM_OPT_TS));
+PMU_FORMAT_ATTR(retstack,	"config:" __stringify(ETM_OPT_RETSTK));
 
 static struct attribute *etm_config_formats_attr[] = {
 	&format_attr_cycacc.attr,
 	&format_attr_timestamp.attr,
+	&format_attr_retstack.attr,
 	NULL,
 };
 
-static struct attribute_group etm_pmu_format_group = {
+static const struct attribute_group etm_pmu_format_group = {
 	.name   = "format",
 	.attrs  = etm_config_formats_attr,
 };
diff --git a/drivers/hwtracing/coresight/coresight-etm.h b/drivers/hwtracing/coresight/coresight-etm.h
index ad063d7444e1..70b0a248c321 100644
--- a/drivers/hwtracing/coresight/coresight-etm.h
+++ b/drivers/hwtracing/coresight/coresight-etm.h
@@ -106,6 +106,7 @@
 #define ETMTECR1_START_STOP	BIT(25)
 /* ETMCCER - 0x1E8 */
 #define ETMCCER_TIMESTAMP	BIT(22)
+#define ETMCCER_RETSTACK	BIT(23)
 
 #define ETM_MODE_EXCLUDE	BIT(0)
 #define ETM_MODE_CYCACC		BIT(1)
diff --git a/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c
index ca98ad13bb8c..6e547ec6fead 100644
--- a/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c
+++ b/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c
@@ -1232,19 +1232,19 @@ static struct attribute *coresight_etm_attrs[] = {
 	NULL,
 };
 
-#define coresight_etm3x_simple_func(name, offset)			\
-	coresight_simple_func(struct etm_drvdata, NULL, name, offset)
-
-coresight_etm3x_simple_func(etmccr, ETMCCR);
-coresight_etm3x_simple_func(etmccer, ETMCCER);
-coresight_etm3x_simple_func(etmscr, ETMSCR);
-coresight_etm3x_simple_func(etmidr, ETMIDR);
-coresight_etm3x_simple_func(etmcr, ETMCR);
-coresight_etm3x_simple_func(etmtraceidr, ETMTRACEIDR);
-coresight_etm3x_simple_func(etmteevr, ETMTEEVR);
-coresight_etm3x_simple_func(etmtssvr, ETMTSSCR);
-coresight_etm3x_simple_func(etmtecr1, ETMTECR1);
-coresight_etm3x_simple_func(etmtecr2, ETMTECR2);
+#define coresight_etm3x_reg(name, offset)			\
+	coresight_simple_reg32(struct etm_drvdata, name, offset)
+
+coresight_etm3x_reg(etmccr, ETMCCR);
+coresight_etm3x_reg(etmccer, ETMCCER);
+coresight_etm3x_reg(etmscr, ETMSCR);
+coresight_etm3x_reg(etmidr, ETMIDR);
+coresight_etm3x_reg(etmcr, ETMCR);
+coresight_etm3x_reg(etmtraceidr, ETMTRACEIDR);
+coresight_etm3x_reg(etmteevr, ETMTEEVR);
+coresight_etm3x_reg(etmtssvr, ETMTSSCR);
+coresight_etm3x_reg(etmtecr1, ETMTECR1);
+coresight_etm3x_reg(etmtecr2, ETMTECR2);
 
 static struct attribute *coresight_etm_mgmt_attrs[] = {
 	&dev_attr_etmccr.attr,
diff --git a/drivers/hwtracing/coresight/coresight-etm3x.c b/drivers/hwtracing/coresight/coresight-etm3x.c
index 93ee8fc539be..e5b1ec57dbde 100644
--- a/drivers/hwtracing/coresight/coresight-etm3x.c
+++ b/drivers/hwtracing/coresight/coresight-etm3x.c
@@ -243,6 +243,8 @@ void etm_set_default(struct etm_config *config)
 	}
 
 	config->ctxid_mask = 0x0;
+	/* Setting default to 1024 as per TRM recommendation */
+	config->sync_freq = 0x400;
 }
 
 void etm_config_trace_mode(struct etm_config *config)
@@ -308,7 +310,9 @@ void etm_config_trace_mode(struct etm_config *config)
 	config->addr_type[1] = ETM_ADDR_TYPE_RANGE;
 }
 
-#define ETM3X_SUPPORTED_OPTIONS (ETMCR_CYC_ACC | ETMCR_TIMESTAMP_EN)
+#define ETM3X_SUPPORTED_OPTIONS (ETMCR_CYC_ACC | \
+				 ETMCR_TIMESTAMP_EN | \
+				 ETMCR_RETURN_STACK)
 
 static int etm_parse_event_config(struct etm_drvdata *drvdata,
 				  struct perf_event *event)
@@ -339,14 +343,24 @@ static int etm_parse_event_config(struct etm_drvdata *drvdata,
 		etm_config_trace_mode(config);
 
 	/*
-	 * At this time only cycle accurate and timestamp options are
-	 * available.
+	 * At this time only cycle accurate, return stack  and timestamp
+	 * options are available.
 	 */
 	if (attr->config & ~ETM3X_SUPPORTED_OPTIONS)
 		return -EINVAL;
 
 	config->ctrl = attr->config;
 
+	/*
+	 * Possible to have cores with PTM (supports ret stack) and ETM
+	 * (never has ret stack) on the same SoC. So if we have a request
+	 * for return stack that can't be honoured on this core then
+	 * clear the bit - trace will still continue normally
+	 */
+	if ((config->ctrl & ETMCR_RETURN_STACK) &&
+	    !(drvdata->etmccer & ETMCCER_RETSTACK))
+		config->ctrl &= ~ETMCR_RETURN_STACK;
+
 	return 0;
 }
 
@@ -885,7 +899,7 @@ static const struct dev_pm_ops etm_dev_pm_ops = {
 	SET_RUNTIME_PM_OPS(etm_runtime_suspend, etm_runtime_resume, NULL)
 };
 
-static struct amba_id etm_ids[] = {
+static const struct amba_id etm_ids[] = {
 	{	/* ETM 3.3 */
 		.id	= 0x0003b921,
 		.mask	= 0x0003ffff,
diff --git a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c
index b9b1e9c8f4c4..4e6eab53e34e 100644
--- a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c
+++ b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c
@@ -2066,23 +2066,23 @@ static u32 etmv4_cross_read(const struct device *dev, u32 offset)
 	return reg.data;
 }
 
-#define coresight_etm4x_simple_func(name, offset)			\
-	coresight_simple_func(struct etmv4_drvdata, NULL, name, offset)
+#define coresight_etm4x_reg(name, offset)			\
+	coresight_simple_reg32(struct etmv4_drvdata, name, offset)
 
 #define coresight_etm4x_cross_read(name, offset)			\
 	coresight_simple_func(struct etmv4_drvdata, etmv4_cross_read,	\
 			      name, offset)
 
-coresight_etm4x_simple_func(trcpdcr, TRCPDCR);
-coresight_etm4x_simple_func(trcpdsr, TRCPDSR);
-coresight_etm4x_simple_func(trclsr, TRCLSR);
-coresight_etm4x_simple_func(trcauthstatus, TRCAUTHSTATUS);
-coresight_etm4x_simple_func(trcdevid, TRCDEVID);
-coresight_etm4x_simple_func(trcdevtype, TRCDEVTYPE);
-coresight_etm4x_simple_func(trcpidr0, TRCPIDR0);
-coresight_etm4x_simple_func(trcpidr1, TRCPIDR1);
-coresight_etm4x_simple_func(trcpidr2, TRCPIDR2);
-coresight_etm4x_simple_func(trcpidr3, TRCPIDR3);
+coresight_etm4x_reg(trcpdcr, TRCPDCR);
+coresight_etm4x_reg(trcpdsr, TRCPDSR);
+coresight_etm4x_reg(trclsr, TRCLSR);
+coresight_etm4x_reg(trcauthstatus, TRCAUTHSTATUS);
+coresight_etm4x_reg(trcdevid, TRCDEVID);
+coresight_etm4x_reg(trcdevtype, TRCDEVTYPE);
+coresight_etm4x_reg(trcpidr0, TRCPIDR0);
+coresight_etm4x_reg(trcpidr1, TRCPIDR1);
+coresight_etm4x_reg(trcpidr2, TRCPIDR2);
+coresight_etm4x_reg(trcpidr3, TRCPIDR3);
 coresight_etm4x_cross_read(trcoslsr, TRCOSLSR);
 coresight_etm4x_cross_read(trcconfig, TRCCONFIGR);
 coresight_etm4x_cross_read(trctraceid, TRCTRACEIDR);
diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c b/drivers/hwtracing/coresight/coresight-etm4x.c
index 532adc9dd32a..cf364a514c12 100644
--- a/drivers/hwtracing/coresight/coresight-etm4x.c
+++ b/drivers/hwtracing/coresight/coresight-etm4x.c
@@ -224,6 +224,10 @@ static int etm4_parse_event_config(struct etmv4_drvdata *drvdata,
 	if (attr->config & BIT(ETM_OPT_TS))
 		/* bit[11], Global timestamp tracing bit */
 		config->cfg |= BIT(11);
+	/* return stack - enable if selected and supported */
+	if ((attr->config & BIT(ETM_OPT_RETSTK)) && drvdata->retstack)
+		/* bit[12], Return stack enable bit */
+		config->cfg |= BIT(12);
 
 out:
 	return ret;
@@ -1048,7 +1052,7 @@ err_arch_supported:
 	return ret;
 }
 
-static struct amba_id etm4_ids[] = {
+static const struct amba_id etm4_ids[] = {
 	{       /* ETM 4.0 - Cortex-A53  */
 		.id	= 0x000bb95d,
 		.mask	= 0x000fffff,
diff --git a/drivers/hwtracing/coresight/coresight-funnel.c b/drivers/hwtracing/coresight/coresight-funnel.c
index 860fe6ef5632..77642e0e955b 100644
--- a/drivers/hwtracing/coresight/coresight-funnel.c
+++ b/drivers/hwtracing/coresight/coresight-funnel.c
@@ -246,11 +246,16 @@ static const struct dev_pm_ops funnel_dev_pm_ops = {
 	SET_RUNTIME_PM_OPS(funnel_runtime_suspend, funnel_runtime_resume, NULL)
 };
 
-static struct amba_id funnel_ids[] = {
+static const struct amba_id funnel_ids[] = {
 	{
 		.id     = 0x0003b908,
 		.mask   = 0x0003ffff,
 	},
+	{
+		/* Coresight SoC-600 */
+		.id     = 0x000bb9eb,
+		.mask   = 0x000fffff,
+	},
 	{ 0, 0},
 };
 
diff --git a/drivers/hwtracing/coresight/coresight-priv.h b/drivers/hwtracing/coresight/coresight-priv.h
index 5f662d82052c..f1d0e21d8cab 100644
--- a/drivers/hwtracing/coresight/coresight-priv.h
+++ b/drivers/hwtracing/coresight/coresight-priv.h
@@ -39,23 +39,33 @@
 #define ETM_MODE_EXCL_USER	BIT(31)
 
 typedef u32 (*coresight_read_fn)(const struct device *, u32 offset);
-#define coresight_simple_func(type, func, name, offset)			\
+#define __coresight_simple_func(type, func, name, lo_off, hi_off)	\
 static ssize_t name##_show(struct device *_dev,				\
 			   struct device_attribute *attr, char *buf)	\
 {									\
 	type *drvdata = dev_get_drvdata(_dev->parent);			\
 	coresight_read_fn fn = func;					\
-	u32 val;							\
+	u64 val;							\
 	pm_runtime_get_sync(_dev->parent);				\
 	if (fn)								\
-		val = fn(_dev->parent, offset);				\
+		val = (u64)fn(_dev->parent, lo_off);			\
 	else								\
-		val = readl_relaxed(drvdata->base + offset);		\
+		val = coresight_read_reg_pair(drvdata->base,		\
+						 lo_off, hi_off);	\
 	pm_runtime_put_sync(_dev->parent);				\
-	return scnprintf(buf, PAGE_SIZE, "0x%x\n", val);		\
+	return scnprintf(buf, PAGE_SIZE, "0x%llx\n", val);		\
 }									\
 static DEVICE_ATTR_RO(name)
 
+#define coresight_simple_func(type, func, name, offset)			\
+	__coresight_simple_func(type, func, name, offset, -1)
+#define coresight_simple_reg32(type, name, offset)			\
+	__coresight_simple_func(type, NULL, name, offset, -1)
+#define coresight_simple_reg64(type, name, lo_off, hi_off)		\
+	__coresight_simple_func(type, NULL, name, lo_off, hi_off)
+
+extern const u32 barrier_pkt[5];
+
 enum etm_addr_type {
 	ETM_ADDR_TYPE_NONE,
 	ETM_ADDR_TYPE_SINGLE,
@@ -106,6 +116,25 @@ static inline void CS_UNLOCK(void __iomem *addr)
 	} while (0);
 }
 
+static inline u64
+coresight_read_reg_pair(void __iomem *addr, s32 lo_offset, s32 hi_offset)
+{
+	u64 val;
+
+	val = readl_relaxed(addr + lo_offset);
+	val |= (hi_offset < 0) ? 0 :
+	       (u64)readl_relaxed(addr + hi_offset) << 32;
+	return val;
+}
+
+static inline void coresight_write_reg_pair(void __iomem *addr, u64 val,
+						 s32 lo_offset, s32 hi_offset)
+{
+	writel_relaxed((u32)val, addr + lo_offset);
+	if (hi_offset >= 0)
+		writel_relaxed((u32)(val >> 32), addr + hi_offset);
+}
+
 void coresight_disable_path(struct list_head *path);
 int coresight_enable_path(struct list_head *path, u32 mode);
 struct coresight_device *coresight_get_sink(struct list_head *path);
diff --git a/drivers/hwtracing/coresight/coresight-stm.c b/drivers/hwtracing/coresight/coresight-stm.c
index 93fc26f01bab..92a780a6df1d 100644
--- a/drivers/hwtracing/coresight/coresight-stm.c
+++ b/drivers/hwtracing/coresight/coresight-stm.c
@@ -276,7 +276,7 @@ static void stm_disable(struct coresight_device *csdev,
 		spin_unlock(&drvdata->spinlock);
 
 		/* Wait until the engine has completely stopped */
-		coresight_timeout(drvdata, STMTCSR, STMTCSR_BUSY_BIT, 0);
+		coresight_timeout(drvdata->base, STMTCSR, STMTCSR_BUSY_BIT, 0);
 
 		pm_runtime_put(drvdata->dev);
 
@@ -307,7 +307,8 @@ static inline bool stm_addr_unaligned(const void *addr, u8 write_bytes)
 	return ((unsigned long)addr & (write_bytes - 1));
 }
 
-static void stm_send(void *addr, const void *data, u32 size, u8 write_bytes)
+static void stm_send(void __iomem *addr, const void *data,
+		     u32 size, u8 write_bytes)
 {
 	u8 paload[8];
 
@@ -414,7 +415,7 @@ static ssize_t notrace stm_generic_packet(struct stm_data *stm_data,
 				  unsigned int size,
 				  const unsigned char *payload)
 {
-	unsigned long ch_addr;
+	void __iomem *ch_addr;
 	struct stm_drvdata *drvdata = container_of(stm_data,
 						   struct stm_drvdata, stm);
 
@@ -424,7 +425,7 @@ static ssize_t notrace stm_generic_packet(struct stm_data *stm_data,
 	if (channel >= drvdata->numsp)
 		return -EINVAL;
 
-	ch_addr = (unsigned long)stm_channel_addr(drvdata, channel);
+	ch_addr = stm_channel_addr(drvdata, channel);
 
 	flags = (flags == STP_PACKET_TIMESTAMPED) ? STM_FLAG_TIMESTAMPED : 0;
 	flags |= test_bit(channel, drvdata->chs.guaranteed) ?
@@ -437,20 +438,20 @@ static ssize_t notrace stm_generic_packet(struct stm_data *stm_data,
 
 	switch (packet) {
 	case STP_PACKET_FLAG:
-		ch_addr |= stm_channel_off(STM_PKT_TYPE_FLAG, flags);
+		ch_addr += stm_channel_off(STM_PKT_TYPE_FLAG, flags);
 
 		/*
 		 * The generic STM core sets a size of '0' on flag packets.
 		 * As such send a flag packet of size '1' and tell the
 		 * core we did so.
 		 */
-		stm_send((void *)ch_addr, payload, 1, drvdata->write_bytes);
+		stm_send(ch_addr, payload, 1, drvdata->write_bytes);
 		size = 1;
 		break;
 
 	case STP_PACKET_DATA:
-		ch_addr |= stm_channel_off(STM_PKT_TYPE_DATA, flags);
-		stm_send((void *)ch_addr, payload, size,
+		ch_addr += stm_channel_off(STM_PKT_TYPE_DATA, flags);
+		stm_send(ch_addr, payload, size,
 				drvdata->write_bytes);
 		break;
 
@@ -635,21 +636,21 @@ static ssize_t traceid_store(struct device *dev,
 }
 static DEVICE_ATTR_RW(traceid);
 
-#define coresight_stm_simple_func(name, offset)	\
-	coresight_simple_func(struct stm_drvdata, NULL, name, offset)
-
-coresight_stm_simple_func(tcsr, STMTCSR);
-coresight_stm_simple_func(tsfreqr, STMTSFREQR);
-coresight_stm_simple_func(syncr, STMSYNCR);
-coresight_stm_simple_func(sper, STMSPER);
-coresight_stm_simple_func(spter, STMSPTER);
-coresight_stm_simple_func(privmaskr, STMPRIVMASKR);
-coresight_stm_simple_func(spscr, STMSPSCR);
-coresight_stm_simple_func(spmscr, STMSPMSCR);
-coresight_stm_simple_func(spfeat1r, STMSPFEAT1R);
-coresight_stm_simple_func(spfeat2r, STMSPFEAT2R);
-coresight_stm_simple_func(spfeat3r, STMSPFEAT3R);
-coresight_stm_simple_func(devid, CORESIGHT_DEVID);
+#define coresight_stm_reg(name, offset)	\
+	coresight_simple_reg32(struct stm_drvdata, name, offset)
+
+coresight_stm_reg(tcsr, STMTCSR);
+coresight_stm_reg(tsfreqr, STMTSFREQR);
+coresight_stm_reg(syncr, STMSYNCR);
+coresight_stm_reg(sper, STMSPER);
+coresight_stm_reg(spter, STMSPTER);
+coresight_stm_reg(privmaskr, STMPRIVMASKR);
+coresight_stm_reg(spscr, STMSPSCR);
+coresight_stm_reg(spmscr, STMSPMSCR);
+coresight_stm_reg(spfeat1r, STMSPFEAT1R);
+coresight_stm_reg(spfeat2r, STMSPFEAT2R);
+coresight_stm_reg(spfeat3r, STMSPFEAT3R);
+coresight_stm_reg(devid, CORESIGHT_DEVID);
 
 static struct attribute *coresight_stm_attrs[] = {
 	&dev_attr_hwevent_enable.attr,
@@ -914,7 +915,7 @@ static const struct dev_pm_ops stm_dev_pm_ops = {
 	SET_RUNTIME_PM_OPS(stm_runtime_suspend, stm_runtime_resume, NULL)
 };
 
-static struct amba_id stm_ids[] = {
+static const struct amba_id stm_ids[] = {
 	{
 		.id     = 0x0003b962,
 		.mask   = 0x0003ffff,
diff --git a/drivers/hwtracing/coresight/coresight-tmc-etf.c b/drivers/hwtracing/coresight/coresight-tmc-etf.c
index e3b9fb82eb8d..e2513b786242 100644
--- a/drivers/hwtracing/coresight/coresight-tmc-etf.c
+++ b/drivers/hwtracing/coresight/coresight-tmc-etf.c
@@ -43,17 +43,34 @@ static void tmc_etb_enable_hw(struct tmc_drvdata *drvdata)
 
 static void tmc_etb_dump_hw(struct tmc_drvdata *drvdata)
 {
+	bool lost = false;
 	char *bufp;
-	u32 read_data;
+	const u32 *barrier;
+	u32 read_data, status;
 	int i;
 
+	/*
+	 * Get a hold of the status register and see if a wrap around
+	 * has occurred.
+	 */
+	status = readl_relaxed(drvdata->base + TMC_STS);
+	if (status & TMC_STS_FULL)
+		lost = true;
+
 	bufp = drvdata->buf;
 	drvdata->len = 0;
+	barrier = barrier_pkt;
 	while (1) {
 		for (i = 0; i < drvdata->memwidth; i++) {
 			read_data = readl_relaxed(drvdata->base + TMC_RRD);
 			if (read_data == 0xFFFFFFFF)
 				return;
+
+			if (lost && *barrier) {
+				read_data = *barrier;
+				barrier++;
+			}
+
 			memcpy(bufp, &read_data, 4);
 			bufp += 4;
 			drvdata->len += 4;
@@ -369,9 +386,11 @@ static void tmc_update_etf_buffer(struct coresight_device *csdev,
 				  struct perf_output_handle *handle,
 				  void *sink_config)
 {
+	bool lost = false;
 	int i, cur;
+	const u32 *barrier;
 	u32 *buf_ptr;
-	u32 read_ptr, write_ptr;
+	u64 read_ptr, write_ptr;
 	u32 status, to_read;
 	unsigned long offset;
 	struct cs_buffers *buf = sink_config;
@@ -388,8 +407,8 @@ static void tmc_update_etf_buffer(struct coresight_device *csdev,
 
 	tmc_flush_and_stop(drvdata);
 
-	read_ptr = readl_relaxed(drvdata->base + TMC_RRP);
-	write_ptr = readl_relaxed(drvdata->base + TMC_RWP);
+	read_ptr = tmc_read_rrp(drvdata);
+	write_ptr = tmc_read_rwp(drvdata);
 
 	/*
 	 * Get a hold of the status register and see if a wrap around
@@ -397,7 +416,7 @@ static void tmc_update_etf_buffer(struct coresight_device *csdev,
 	 */
 	status = readl_relaxed(drvdata->base + TMC_STS);
 	if (status & TMC_STS_FULL) {
-		perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
+		lost = true;
 		to_read = drvdata->size;
 	} else {
 		to_read = CIRC_CNT(write_ptr, read_ptr, drvdata->size);
@@ -441,18 +460,27 @@ static void tmc_update_etf_buffer(struct coresight_device *csdev,
 		if (read_ptr > (drvdata->size - 1))
 			read_ptr -= drvdata->size;
 		/* Tell the HW */
-		writel_relaxed(read_ptr, drvdata->base + TMC_RRP);
-		perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
+		tmc_write_rrp(drvdata, read_ptr);
+		lost = true;
 	}
 
+	if (lost)
+		perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
+
 	cur = buf->cur;
 	offset = buf->offset;
+	barrier = barrier_pkt;
 
 	/* for every byte to read */
 	for (i = 0; i < to_read; i += 4) {
 		buf_ptr = buf->data_pages[cur] + offset;
 		*buf_ptr = readl_relaxed(drvdata->base + TMC_RRD);
 
+		if (lost && *barrier) {
+			*buf_ptr = *barrier;
+			barrier++;
+		}
+
 		offset += 4;
 		if (offset >= PAGE_SIZE) {
 			offset = 0;
diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c
index 5d312699b3b9..68fbc8f7450e 100644
--- a/drivers/hwtracing/coresight/coresight-tmc-etr.c
+++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c
@@ -22,7 +22,7 @@
 
 static void tmc_etr_enable_hw(struct tmc_drvdata *drvdata)
 {
-	u32 axictl;
+	u32 axictl, sts;
 
 	/* Zero out the memory to help with debug */
 	memset(drvdata->vaddr, 0, drvdata->size);
@@ -36,17 +36,29 @@ static void tmc_etr_enable_hw(struct tmc_drvdata *drvdata)
 	writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
 
 	axictl = readl_relaxed(drvdata->base + TMC_AXICTL);
-	axictl |= TMC_AXICTL_WR_BURST_16;
-	writel_relaxed(axictl, drvdata->base + TMC_AXICTL);
-	axictl &= ~TMC_AXICTL_SCT_GAT_MODE;
-	writel_relaxed(axictl, drvdata->base + TMC_AXICTL);
-	axictl = (axictl &
-		  ~(TMC_AXICTL_PROT_CTL_B0 | TMC_AXICTL_PROT_CTL_B1)) |
-		  TMC_AXICTL_PROT_CTL_B1;
+	axictl &= ~TMC_AXICTL_CLEAR_MASK;
+	axictl |= (TMC_AXICTL_PROT_CTL_B1 | TMC_AXICTL_WR_BURST_16);
+	axictl |= TMC_AXICTL_AXCACHE_OS;
+
+	if (tmc_etr_has_cap(drvdata, TMC_ETR_AXI_ARCACHE)) {
+		axictl &= ~TMC_AXICTL_ARCACHE_MASK;
+		axictl |= TMC_AXICTL_ARCACHE_OS;
+	}
+
 	writel_relaxed(axictl, drvdata->base + TMC_AXICTL);
+	tmc_write_dba(drvdata, drvdata->paddr);
+	/*
+	 * If the TMC pointers must be programmed before the session,
+	 * we have to set it properly (i.e, RRP/RWP to base address and
+	 * STS to "not full").
+	 */
+	if (tmc_etr_has_cap(drvdata, TMC_ETR_SAVE_RESTORE)) {
+		tmc_write_rrp(drvdata, drvdata->paddr);
+		tmc_write_rwp(drvdata, drvdata->paddr);
+		sts = readl_relaxed(drvdata->base + TMC_STS) & ~TMC_STS_FULL;
+		writel_relaxed(sts, drvdata->base + TMC_STS);
+	}
 
-	writel_relaxed(drvdata->paddr, drvdata->base + TMC_DBALO);
-	writel_relaxed(0x0, drvdata->base + TMC_DBAHI);
 	writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
 		       TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
 		       TMC_FFCR_TRIGON_TRIGIN,
@@ -59,9 +71,12 @@ static void tmc_etr_enable_hw(struct tmc_drvdata *drvdata)
 
 static void tmc_etr_dump_hw(struct tmc_drvdata *drvdata)
 {
-	u32 rwp, val;
+	const u32 *barrier;
+	u32 val;
+	u32 *temp;
+	u64 rwp;
 
-	rwp = readl_relaxed(drvdata->base + TMC_RWP);
+	rwp = tmc_read_rwp(drvdata);
 	val = readl_relaxed(drvdata->base + TMC_STS);
 
 	/*
@@ -71,6 +86,16 @@ static void tmc_etr_dump_hw(struct tmc_drvdata *drvdata)
 	if (val & TMC_STS_FULL) {
 		drvdata->buf = drvdata->vaddr + rwp - drvdata->paddr;
 		drvdata->len = drvdata->size;
+
+		barrier = barrier_pkt;
+		temp = (u32 *)drvdata->buf;
+
+		while (*barrier) {
+			*temp = *barrier;
+			temp++;
+			barrier++;
+		}
+
 	} else {
 		drvdata->buf = drvdata->vaddr;
 		drvdata->len = rwp - drvdata->paddr;
diff --git a/drivers/hwtracing/coresight/coresight-tmc.c b/drivers/hwtracing/coresight/coresight-tmc.c
index 864488793f09..2ff4a66a3caa 100644
--- a/drivers/hwtracing/coresight/coresight-tmc.c
+++ b/drivers/hwtracing/coresight/coresight-tmc.c
@@ -217,20 +217,24 @@ static enum tmc_mem_intf_width tmc_get_memwidth(u32 devid)
 	return memwidth;
 }
 
-#define coresight_tmc_simple_func(name, offset)			\
-	coresight_simple_func(struct tmc_drvdata, NULL, name, offset)
-
-coresight_tmc_simple_func(rsz, TMC_RSZ);
-coresight_tmc_simple_func(sts, TMC_STS);
-coresight_tmc_simple_func(rrp, TMC_RRP);
-coresight_tmc_simple_func(rwp, TMC_RWP);
-coresight_tmc_simple_func(trg, TMC_TRG);
-coresight_tmc_simple_func(ctl, TMC_CTL);
-coresight_tmc_simple_func(ffsr, TMC_FFSR);
-coresight_tmc_simple_func(ffcr, TMC_FFCR);
-coresight_tmc_simple_func(mode, TMC_MODE);
-coresight_tmc_simple_func(pscr, TMC_PSCR);
-coresight_tmc_simple_func(devid, CORESIGHT_DEVID);
+#define coresight_tmc_reg(name, offset)			\
+	coresight_simple_reg32(struct tmc_drvdata, name, offset)
+#define coresight_tmc_reg64(name, lo_off, hi_off)	\
+	coresight_simple_reg64(struct tmc_drvdata, name, lo_off, hi_off)
+
+coresight_tmc_reg(rsz, TMC_RSZ);
+coresight_tmc_reg(sts, TMC_STS);
+coresight_tmc_reg(trg, TMC_TRG);
+coresight_tmc_reg(ctl, TMC_CTL);
+coresight_tmc_reg(ffsr, TMC_FFSR);
+coresight_tmc_reg(ffcr, TMC_FFCR);
+coresight_tmc_reg(mode, TMC_MODE);
+coresight_tmc_reg(pscr, TMC_PSCR);
+coresight_tmc_reg(axictl, TMC_AXICTL);
+coresight_tmc_reg(devid, CORESIGHT_DEVID);
+coresight_tmc_reg64(rrp, TMC_RRP, TMC_RRPHI);
+coresight_tmc_reg64(rwp, TMC_RWP, TMC_RWPHI);
+coresight_tmc_reg64(dba, TMC_DBALO, TMC_DBAHI);
 
 static struct attribute *coresight_tmc_mgmt_attrs[] = {
 	&dev_attr_rsz.attr,
@@ -244,6 +248,8 @@ static struct attribute *coresight_tmc_mgmt_attrs[] = {
 	&dev_attr_mode.attr,
 	&dev_attr_pscr.attr,
 	&dev_attr_devid.attr,
+	&dev_attr_dba.attr,
+	&dev_attr_axictl.attr,
 	NULL,
 };
 
@@ -293,6 +299,42 @@ const struct attribute_group *coresight_tmc_groups[] = {
 	NULL,
 };
 
+/* Detect and initialise the capabilities of a TMC ETR */
+static int tmc_etr_setup_caps(struct tmc_drvdata *drvdata,
+			     u32 devid, void *dev_caps)
+{
+	u32 dma_mask = 0;
+
+	/* Set the unadvertised capabilities */
+	tmc_etr_init_caps(drvdata, (u32)(unsigned long)dev_caps);
+
+	if (!(devid & TMC_DEVID_NOSCAT))
+		tmc_etr_set_cap(drvdata, TMC_ETR_SG);
+
+	/* Check if the AXI address width is available */
+	if (devid & TMC_DEVID_AXIAW_VALID)
+		dma_mask = ((devid >> TMC_DEVID_AXIAW_SHIFT) &
+				TMC_DEVID_AXIAW_MASK);
+
+	/*
+	 * Unless specified in the device configuration, ETR uses a 40-bit
+	 * AXI master in place of the embedded SRAM of ETB/ETF.
+	 */
+	switch (dma_mask) {
+	case 32:
+	case 40:
+	case 44:
+	case 48:
+	case 52:
+		dev_info(drvdata->dev, "Detected dma mask %dbits\n", dma_mask);
+		break;
+	default:
+		dma_mask = 40;
+	}
+
+	return dma_set_mask_and_coherent(drvdata->dev, DMA_BIT_MASK(dma_mask));
+}
+
 static int tmc_probe(struct amba_device *adev, const struct amba_id *id)
 {
 	int ret = 0;
@@ -354,25 +396,29 @@ static int tmc_probe(struct amba_device *adev, const struct amba_id *id)
 	desc.dev = dev;
 	desc.groups = coresight_tmc_groups;
 
-	if (drvdata->config_type == TMC_CONFIG_TYPE_ETB) {
+	switch (drvdata->config_type) {
+	case TMC_CONFIG_TYPE_ETB:
 		desc.type = CORESIGHT_DEV_TYPE_SINK;
 		desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_BUFFER;
 		desc.ops = &tmc_etb_cs_ops;
-	} else if (drvdata->config_type == TMC_CONFIG_TYPE_ETR) {
+		break;
+	case TMC_CONFIG_TYPE_ETR:
 		desc.type = CORESIGHT_DEV_TYPE_SINK;
 		desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_BUFFER;
 		desc.ops = &tmc_etr_cs_ops;
-		/*
-		 * ETR configuration uses a 40-bit AXI master in place of
-		 * the embedded SRAM of ETB/ETF.
-		 */
-		ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40));
+		ret = tmc_etr_setup_caps(drvdata, devid, id->data);
 		if (ret)
 			goto out;
-	} else {
+		break;
+	case TMC_CONFIG_TYPE_ETF:
 		desc.type = CORESIGHT_DEV_TYPE_LINKSINK;
 		desc.subtype.link_subtype = CORESIGHT_DEV_SUBTYPE_LINK_FIFO;
 		desc.ops = &tmc_etf_cs_ops;
+		break;
+	default:
+		pr_err("%s: Unsupported TMC config\n", pdata->name);
+		ret = -EINVAL;
+		goto out;
 	}
 
 	drvdata->csdev = coresight_register(&desc);
@@ -391,11 +437,27 @@ out:
 	return ret;
 }
 
-static struct amba_id tmc_ids[] = {
+static const struct amba_id tmc_ids[] = {
 	{
 		.id     = 0x0003b961,
 		.mask   = 0x0003ffff,
 	},
+	{
+		/* Coresight SoC 600 TMC-ETR/ETS */
+		.id	= 0x000bb9e8,
+		.mask	= 0x000fffff,
+		.data	= (void *)(unsigned long)CORESIGHT_SOC_600_ETR_CAPS,
+	},
+	{
+		/* Coresight SoC 600 TMC-ETB */
+		.id	= 0x000bb9e9,
+		.mask	= 0x000fffff,
+	},
+	{
+		/* Coresight SoC 600 TMC-ETF */
+		.id	= 0x000bb9ea,
+		.mask	= 0x000fffff,
+	},
 	{ 0, 0},
 };
 
diff --git a/drivers/hwtracing/coresight/coresight-tmc.h b/drivers/hwtracing/coresight/coresight-tmc.h
index 51c01851533e..8df7a813f537 100644
--- a/drivers/hwtracing/coresight/coresight-tmc.h
+++ b/drivers/hwtracing/coresight/coresight-tmc.h
@@ -54,11 +54,32 @@
 #define TMC_STS_TMCREADY_BIT	2
 #define TMC_STS_FULL		BIT(0)
 #define TMC_STS_TRIGGERED	BIT(1)
-/* TMC_AXICTL - 0x110 */
+/*
+ * TMC_AXICTL - 0x110
+ *
+ * TMC AXICTL format for SoC-400
+ *	Bits [0-1]	: ProtCtrlBit0-1
+ *	Bits [2-5]	: CacheCtrlBits 0-3 (AXCACHE)
+ *	Bit  6		: Reserved
+ *	Bit  7		: ScatterGatherMode
+ *	Bits [8-11]	: WrBurstLen
+ *	Bits [12-31]	: Reserved.
+ * TMC AXICTL format for SoC-600, as above except:
+ *	Bits [2-5]	: AXI WCACHE
+ *	Bits [16-19]	: AXI RCACHE
+ *	Bits [20-31]	: Reserved
+ */
+#define TMC_AXICTL_CLEAR_MASK 0xfbf
+#define TMC_AXICTL_ARCACHE_MASK (0xf << 16)
+
 #define TMC_AXICTL_PROT_CTL_B0	BIT(0)
 #define TMC_AXICTL_PROT_CTL_B1	BIT(1)
 #define TMC_AXICTL_SCT_GAT_MODE	BIT(7)
 #define TMC_AXICTL_WR_BURST_16	0xF00
+/* Write-back Read and Write-allocate */
+#define TMC_AXICTL_AXCACHE_OS	(0xf << 2)
+#define TMC_AXICTL_ARCACHE_OS	(0xf << 16)
+
 /* TMC_FFCR - 0x304 */
 #define TMC_FFCR_FLUSHMAN_BIT	6
 #define TMC_FFCR_EN_FMT		BIT(0)
@@ -69,6 +90,12 @@
 #define TMC_FFCR_STOP_ON_FLUSH	BIT(12)
 
 
+#define TMC_DEVID_NOSCAT	BIT(24)
+
+#define TMC_DEVID_AXIAW_VALID	BIT(16)
+#define TMC_DEVID_AXIAW_SHIFT	17
+#define TMC_DEVID_AXIAW_MASK	0x7f
+
 enum tmc_config_type {
 	TMC_CONFIG_TYPE_ETB,
 	TMC_CONFIG_TYPE_ETR,
@@ -88,6 +115,24 @@ enum tmc_mem_intf_width {
 	TMC_MEM_INTF_WIDTH_256BITS	= 8,
 };
 
+/* TMC ETR Capability bit definitions */
+#define TMC_ETR_SG			(0x1U << 0)
+/* ETR has separate read/write cache encodings */
+#define TMC_ETR_AXI_ARCACHE		(0x1U << 1)
+/*
+ * TMC_ETR_SAVE_RESTORE - Values of RRP/RWP/STS.Full are
+ * retained when TMC leaves Disabled state, allowing us to continue
+ * the tracing from a point where we stopped. This also implies that
+ * the RRP/RWP/STS.Full should always be programmed to the correct
+ * value. Unfortunately this is not advertised by the hardware,
+ * so we have to rely on PID of the IP to detect the functionality.
+ */
+#define TMC_ETR_SAVE_RESTORE		(0x1U << 2)
+
+/* Coresight SoC-600 TMC-ETR unadvertised capabilities */
+#define CORESIGHT_SOC_600_ETR_CAPS	\
+	(TMC_ETR_SAVE_RESTORE | TMC_ETR_AXI_ARCACHE)
+
 /**
  * struct tmc_drvdata - specifics associated to an TMC component
  * @base:	memory mapped base address for this component.
@@ -104,6 +149,8 @@ enum tmc_mem_intf_width {
  * @config_type: TMC variant, must be of type @tmc_config_type.
  * @memwidth:	width of the memory interface databus, in bytes.
  * @trigger_cntr: amount of words to store after a trigger.
+ * @etr_caps:	Bitmask of capabilities of the TMC ETR, inferred from the
+ *		device configuration register (DEVID)
  */
 struct tmc_drvdata {
 	void __iomem		*base;
@@ -121,6 +168,7 @@ struct tmc_drvdata {
 	enum tmc_config_type	config_type;
 	enum tmc_mem_intf_width	memwidth;
 	u32			trigger_cntr;
+	u32			etr_caps;
 };
 
 /* Generic functions */
@@ -139,4 +187,39 @@ extern const struct coresight_ops tmc_etf_cs_ops;
 int tmc_read_prepare_etr(struct tmc_drvdata *drvdata);
 int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata);
 extern const struct coresight_ops tmc_etr_cs_ops;
+
+
+#define TMC_REG_PAIR(name, lo_off, hi_off)				\
+static inline u64							\
+tmc_read_##name(struct tmc_drvdata *drvdata)				\
+{									\
+	return coresight_read_reg_pair(drvdata->base, lo_off, hi_off);	\
+}									\
+static inline void							\
+tmc_write_##name(struct tmc_drvdata *drvdata, u64 val)			\
+{									\
+	coresight_write_reg_pair(drvdata->base, val, lo_off, hi_off);	\
+}
+
+TMC_REG_PAIR(rrp, TMC_RRP, TMC_RRPHI)
+TMC_REG_PAIR(rwp, TMC_RWP, TMC_RWPHI)
+TMC_REG_PAIR(dba, TMC_DBALO, TMC_DBAHI)
+
+/* Initialise the caps from unadvertised static capabilities of the device */
+static inline void tmc_etr_init_caps(struct tmc_drvdata *drvdata, u32 dev_caps)
+{
+	WARN_ON(drvdata->etr_caps);
+	drvdata->etr_caps = dev_caps;
+}
+
+static inline void tmc_etr_set_cap(struct tmc_drvdata *drvdata, u32 cap)
+{
+	drvdata->etr_caps |= cap;
+}
+
+static inline bool tmc_etr_has_cap(struct tmc_drvdata *drvdata, u32 cap)
+{
+	return !!(drvdata->etr_caps & cap);
+}
+
 #endif
diff --git a/drivers/hwtracing/coresight/coresight-tpiu.c b/drivers/hwtracing/coresight/coresight-tpiu.c
index 0673baf0f2f5..d7a3e453016d 100644
--- a/drivers/hwtracing/coresight/coresight-tpiu.c
+++ b/drivers/hwtracing/coresight/coresight-tpiu.c
@@ -192,7 +192,7 @@ static const struct dev_pm_ops tpiu_dev_pm_ops = {
 	SET_RUNTIME_PM_OPS(tpiu_runtime_suspend, tpiu_runtime_resume, NULL)
 };
 
-static struct amba_id tpiu_ids[] = {
+static const struct amba_id tpiu_ids[] = {
 	{
 		.id	= 0x0003b912,
 		.mask	= 0x0003ffff,
@@ -201,6 +201,11 @@ static struct amba_id tpiu_ids[] = {
 		.id	= 0x0004b912,
 		.mask	= 0x0007ffff,
 	},
+	{
+		/* Coresight SoC-600 */
+		.id	= 0x000bb9e7,
+		.mask	= 0x000fffff,
+	},
 	{ 0, 0},
 };
 
diff --git a/drivers/hwtracing/coresight/coresight.c b/drivers/hwtracing/coresight/coresight.c
index 6a0202b7384f..b8091bef21dc 100644
--- a/drivers/hwtracing/coresight/coresight.c
+++ b/drivers/hwtracing/coresight/coresight.c
@@ -53,6 +53,14 @@ static DEFINE_PER_CPU(struct list_head *, tracer_path);
  */
 static struct list_head *stm_path;
 
+/*
+ * When losing synchronisation a new barrier packet needs to be inserted at the
+ * beginning of the data collected in a buffer.  That way the decoder knows that
+ * it needs to look for another sync sequence.
+ */
+const u32 barrier_pkt[5] = {0x7fffffff, 0x7fffffff,
+			    0x7fffffff, 0x7fffffff, 0x0};
+
 static int coresight_id_match(struct device *dev, void *data)
 {
 	int trace_id, i_trace_id;
diff --git a/drivers/hwtracing/intel_th/core.c b/drivers/hwtracing/intel_th/core.c
index 8da567abc0ce..1a023e30488c 100644
--- a/drivers/hwtracing/intel_th/core.c
+++ b/drivers/hwtracing/intel_th/core.c
@@ -101,17 +101,53 @@ out_pm:
 	return ret;
 }
 
+static void intel_th_device_remove(struct intel_th_device *thdev);
+
 static int intel_th_remove(struct device *dev)
 {
 	struct intel_th_driver *thdrv = to_intel_th_driver(dev->driver);
 	struct intel_th_device *thdev = to_intel_th_device(dev);
-	struct intel_th_device *hub = to_intel_th_device(dev->parent);
+	struct intel_th_device *hub = to_intel_th_hub(thdev);
 	int err;
 
 	if (thdev->type == INTEL_TH_SWITCH) {
+		struct intel_th *th = to_intel_th(hub);
+		int i, lowest;
+
+		/* disconnect outputs */
 		err = device_for_each_child(dev, thdev, intel_th_child_remove);
 		if (err)
 			return err;
+
+		/*
+		 * Remove outputs, that is, hub's children: they are created
+		 * at hub's probe time by having the hub call
+		 * intel_th_output_enable() for each of them.
+		 */
+		for (i = 0, lowest = -1; i < th->num_thdevs; i++) {
+			/*
+			 * Move the non-output devices from higher up the
+			 * th->thdev[] array to lower positions to maintain
+			 * a contiguous array.
+			 */
+			if (th->thdev[i]->type != INTEL_TH_OUTPUT) {
+				if (lowest >= 0) {
+					th->thdev[lowest] = th->thdev[i];
+					th->thdev[i] = NULL;
+					++lowest;
+				}
+
+				continue;
+			}
+
+			if (lowest == -1)
+				lowest = i;
+
+			intel_th_device_remove(th->thdev[i]);
+			th->thdev[i] = NULL;
+		}
+
+		th->num_thdevs = lowest;
 	}
 
 	if (thdrv->attr_group)
@@ -156,21 +192,6 @@ static struct device_type intel_th_source_device_type = {
 	.release	= intel_th_device_release,
 };
 
-static struct intel_th *to_intel_th(struct intel_th_device *thdev)
-{
-	/*
-	 * subdevice tree is flat: if this one is not a switch, its
-	 * parent must be
-	 */
-	if (thdev->type != INTEL_TH_SWITCH)
-		thdev = to_intel_th_hub(thdev);
-
-	if (WARN_ON_ONCE(!thdev || thdev->type != INTEL_TH_SWITCH))
-		return NULL;
-
-	return dev_get_drvdata(thdev->dev.parent);
-}
-
 static char *intel_th_output_devnode(struct device *dev, umode_t *mode,
 				     kuid_t *uid, kgid_t *gid)
 {
@@ -205,6 +226,7 @@ static int intel_th_output_activate(struct intel_th_device *thdev)
 {
 	struct intel_th_driver *thdrv =
 		to_intel_th_driver_or_null(thdev->dev.driver);
+	struct intel_th *th = to_intel_th(thdev);
 	int ret = 0;
 
 	if (!thdrv)
@@ -215,15 +237,28 @@ static int intel_th_output_activate(struct intel_th_device *thdev)
 
 	pm_runtime_get_sync(&thdev->dev);
 
+	if (th->activate)
+		ret = th->activate(th);
+	if (ret)
+		goto fail_put;
+
 	if (thdrv->activate)
 		ret = thdrv->activate(thdev);
 	else
 		intel_th_trace_enable(thdev);
 
-	if (ret) {
-		pm_runtime_put(&thdev->dev);
-		module_put(thdrv->driver.owner);
-	}
+	if (ret)
+		goto fail_deactivate;
+
+	return 0;
+
+fail_deactivate:
+	if (th->deactivate)
+		th->deactivate(th);
+
+fail_put:
+	pm_runtime_put(&thdev->dev);
+	module_put(thdrv->driver.owner);
 
 	return ret;
 }
@@ -232,6 +267,7 @@ static void intel_th_output_deactivate(struct intel_th_device *thdev)
 {
 	struct intel_th_driver *thdrv =
 		to_intel_th_driver_or_null(thdev->dev.driver);
+	struct intel_th *th = to_intel_th(thdev);
 
 	if (!thdrv)
 		return;
@@ -241,6 +277,9 @@ static void intel_th_output_deactivate(struct intel_th_device *thdev)
 	else
 		intel_th_trace_disable(thdev);
 
+	if (th->deactivate)
+		th->deactivate(th);
+
 	pm_runtime_put(&thdev->dev);
 	module_put(thdrv->driver.owner);
 }
@@ -326,10 +365,10 @@ intel_th_device_alloc(struct intel_th *th, unsigned int type, const char *name,
 	struct device *parent;
 	struct intel_th_device *thdev;
 
-	if (type == INTEL_TH_SWITCH)
-		parent = th->dev;
-	else
+	if (type == INTEL_TH_OUTPUT)
 		parent = &th->hub->dev;
+	else
+		parent = th->dev;
 
 	thdev = kzalloc(sizeof(*thdev) + strlen(name) + 1, GFP_KERNEL);
 	if (!thdev)
@@ -392,13 +431,14 @@ static const struct intel_th_subdevice {
 	unsigned		otype;
 	unsigned		scrpd;
 	int			id;
-} intel_th_subdevices[TH_SUBDEVICE_MAX] = {
+} intel_th_subdevices[] = {
 	{
 		.nres	= 1,
 		.res	= {
 			{
+				/* Handle TSCU from GTH driver */
 				.start	= REG_GTH_OFFSET,
-				.end	= REG_GTH_OFFSET + REG_GTH_LENGTH - 1,
+				.end	= REG_TSCU_OFFSET + REG_TSCU_LENGTH - 1,
 				.flags	= IORESOURCE_MEM,
 			},
 		},
@@ -483,6 +523,21 @@ static const struct intel_th_subdevice {
 		.nres	= 1,
 		.res	= {
 			{
+				.start	= REG_PTI_OFFSET,
+				.end	= REG_PTI_OFFSET + REG_PTI_LENGTH - 1,
+				.flags	= IORESOURCE_MEM,
+			},
+		},
+		.id	= -1,
+		.name	= "lpp",
+		.type	= INTEL_TH_OUTPUT,
+		.otype	= GTH_LPP,
+		.scrpd	= SCRPD_PTI_IS_PRIM_DEST,
+	},
+	{
+		.nres	= 1,
+		.res	= {
+			{
 				.start	= REG_DCIH_OFFSET,
 				.end	= REG_DCIH_OFFSET + REG_DCIH_LENGTH - 1,
 				.flags	= IORESOURCE_MEM,
@@ -526,98 +581,182 @@ static inline void intel_th_request_hub_module_flush(struct intel_th *th)
 }
 #endif /* CONFIG_MODULES */
 
-static int intel_th_populate(struct intel_th *th, struct resource *devres,
-			     unsigned int ndevres, int irq)
+static struct intel_th_device *
+intel_th_subdevice_alloc(struct intel_th *th,
+			 const struct intel_th_subdevice *subdev)
 {
+	struct intel_th_device *thdev;
 	struct resource res[3];
 	unsigned int req = 0;
-	int src, dst, err;
+	int r, err;
 
-	/* create devices for each intel_th_subdevice */
-	for (src = 0, dst = 0; src < ARRAY_SIZE(intel_th_subdevices); src++) {
-		const struct intel_th_subdevice *subdev =
-			&intel_th_subdevices[src];
-		struct intel_th_device *thdev;
-		int r;
+	thdev = intel_th_device_alloc(th, subdev->type, subdev->name,
+				      subdev->id);
+	if (!thdev)
+		return ERR_PTR(-ENOMEM);
 
-		/* only allow SOURCE and SWITCH devices in host mode */
-		if (host_mode && subdev->type == INTEL_TH_OUTPUT)
-			continue;
+	thdev->drvdata = th->drvdata;
+
+	memcpy(res, subdev->res,
+	       sizeof(struct resource) * subdev->nres);
+
+	for (r = 0; r < subdev->nres; r++) {
+		struct resource *devres = th->resource;
+		int bar = TH_MMIO_CONFIG;
+
+		/*
+		 * Take .end == 0 to mean 'take the whole bar',
+		 * .start then tells us which bar it is. Default to
+		 * TH_MMIO_CONFIG.
+		 */
+		if (!res[r].end && res[r].flags == IORESOURCE_MEM) {
+			bar = res[r].start;
+			res[r].start = 0;
+			res[r].end = resource_size(&devres[bar]) - 1;
+		}
+
+		if (res[r].flags & IORESOURCE_MEM) {
+			res[r].start	+= devres[bar].start;
+			res[r].end	+= devres[bar].start;
 
-		thdev = intel_th_device_alloc(th, subdev->type, subdev->name,
-					      subdev->id);
-		if (!thdev) {
-			err = -ENOMEM;
-			goto kill_subdevs;
+			dev_dbg(th->dev, "%s:%d @ %pR\n",
+				subdev->name, r, &res[r]);
+		} else if (res[r].flags & IORESOURCE_IRQ) {
+			res[r].start	= th->irq;
 		}
+	}
 
-		memcpy(res, subdev->res,
-		       sizeof(struct resource) * subdev->nres);
+	err = intel_th_device_add_resources(thdev, res, subdev->nres);
+	if (err) {
+		put_device(&thdev->dev);
+		goto fail_put_device;
+	}
 
-		for (r = 0; r < subdev->nres; r++) {
-			int bar = TH_MMIO_CONFIG;
+	if (subdev->type == INTEL_TH_OUTPUT) {
+		thdev->dev.devt = MKDEV(th->major, th->num_thdevs);
+		thdev->output.type = subdev->otype;
+		thdev->output.port = -1;
+		thdev->output.scratchpad = subdev->scrpd;
+	} else if (subdev->type == INTEL_TH_SWITCH) {
+		thdev->host_mode = host_mode;
+		th->hub = thdev;
+	}
 
-			/*
-			 * Take .end == 0 to mean 'take the whole bar',
-			 * .start then tells us which bar it is. Default to
-			 * TH_MMIO_CONFIG.
-			 */
-			if (!res[r].end && res[r].flags == IORESOURCE_MEM) {
-				bar = res[r].start;
-				res[r].start = 0;
-				res[r].end = resource_size(&devres[bar]) - 1;
-			}
+	err = device_add(&thdev->dev);
+	if (err) {
+		put_device(&thdev->dev);
+		goto fail_free_res;
+	}
 
-			if (res[r].flags & IORESOURCE_MEM) {
-				res[r].start	+= devres[bar].start;
-				res[r].end	+= devres[bar].start;
+	/* need switch driver to be loaded to enumerate the rest */
+	if (subdev->type == INTEL_TH_SWITCH && !req) {
+		err = intel_th_request_hub_module(th);
+		if (!err)
+			req++;
+	}
 
-				dev_dbg(th->dev, "%s:%d @ %pR\n",
-					subdev->name, r, &res[r]);
-			} else if (res[r].flags & IORESOURCE_IRQ) {
-				res[r].start	= irq;
-			}
-		}
+	return thdev;
 
-		err = intel_th_device_add_resources(thdev, res, subdev->nres);
-		if (err) {
-			put_device(&thdev->dev);
-			goto kill_subdevs;
-		}
+fail_free_res:
+	kfree(thdev->resource);
 
-		if (subdev->type == INTEL_TH_OUTPUT) {
-			thdev->dev.devt = MKDEV(th->major, dst);
-			thdev->output.type = subdev->otype;
-			thdev->output.port = -1;
-			thdev->output.scratchpad = subdev->scrpd;
-		} else if (subdev->type == INTEL_TH_SWITCH) {
-			thdev->host_mode = host_mode;
-		}
+fail_put_device:
+	put_device(&thdev->dev);
+
+	return ERR_PTR(err);
+}
 
-		err = device_add(&thdev->dev);
-		if (err) {
-			put_device(&thdev->dev);
-			goto kill_subdevs;
+/**
+ * intel_th_output_enable() - find and enable a device for a given output type
+ * @th:		Intel TH instance
+ * @otype:	output type
+ *
+ * Go through the unallocated output devices, find the first one whos type
+ * matches @otype and instantiate it. These devices are removed when the hub
+ * device is removed, see intel_th_remove().
+ */
+int intel_th_output_enable(struct intel_th *th, unsigned int otype)
+{
+	struct intel_th_device *thdev;
+	int src = 0, dst = 0;
+
+	for (src = 0, dst = 0; dst <= th->num_thdevs; src++, dst++) {
+		for (; src < ARRAY_SIZE(intel_th_subdevices); src++) {
+			if (intel_th_subdevices[src].type != INTEL_TH_OUTPUT)
+				continue;
+
+			if (intel_th_subdevices[src].otype != otype)
+				continue;
+
+			break;
 		}
 
-		/* need switch driver to be loaded to enumerate the rest */
-		if (subdev->type == INTEL_TH_SWITCH && !req) {
-			th->hub = thdev;
-			err = intel_th_request_hub_module(th);
-			if (!err)
-				req++;
+		/* no unallocated matching subdevices */
+		if (src == ARRAY_SIZE(intel_th_subdevices))
+			return -ENODEV;
+
+		for (; dst < th->num_thdevs; dst++) {
+			if (th->thdev[dst]->type != INTEL_TH_OUTPUT)
+				continue;
+
+			if (th->thdev[dst]->output.type != otype)
+				continue;
+
+			break;
 		}
 
-		th->thdev[dst++] = thdev;
+		/*
+		 * intel_th_subdevices[src] matches our requirements and is
+		 * not matched in th::thdev[]
+		 */
+		if (dst == th->num_thdevs)
+			goto found;
 	}
 
+	return -ENODEV;
+
+found:
+	thdev = intel_th_subdevice_alloc(th, &intel_th_subdevices[src]);
+	if (IS_ERR(thdev))
+		return PTR_ERR(thdev);
+
+	th->thdev[th->num_thdevs++] = thdev;
+
 	return 0;
+}
+EXPORT_SYMBOL_GPL(intel_th_output_enable);
+
+static int intel_th_populate(struct intel_th *th)
+{
+	int src;
+
+	/* create devices for each intel_th_subdevice */
+	for (src = 0; src < ARRAY_SIZE(intel_th_subdevices); src++) {
+		const struct intel_th_subdevice *subdev =
+			&intel_th_subdevices[src];
+		struct intel_th_device *thdev;
+
+		/* only allow SOURCE and SWITCH devices in host mode */
+		if (host_mode && subdev->type == INTEL_TH_OUTPUT)
+			continue;
 
-kill_subdevs:
-	for (; dst >= 0; dst--)
-		intel_th_device_remove(th->thdev[dst]);
+		/*
+		 * don't enable port OUTPUTs in this path; SWITCH enables them
+		 * via intel_th_output_enable()
+		 */
+		if (subdev->type == INTEL_TH_OUTPUT &&
+		    subdev->otype != GTH_NONE)
+			continue;
+
+		thdev = intel_th_subdevice_alloc(th, subdev);
+		/* note: caller should free subdevices from th::thdev[] */
+		if (IS_ERR(thdev))
+			return PTR_ERR(thdev);
+
+		th->thdev[th->num_thdevs++] = thdev;
+	}
 
-	return err;
+	return 0;
 }
 
 static int match_devt(struct device *dev, void *data)
@@ -670,8 +809,8 @@ static const struct file_operations intel_th_output_fops = {
  * @irq:	irq number
  */
 struct intel_th *
-intel_th_alloc(struct device *dev, struct resource *devres,
-	       unsigned int ndevres, int irq)
+intel_th_alloc(struct device *dev, struct intel_th_drvdata *drvdata,
+	       struct resource *devres, unsigned int ndevres, int irq)
 {
 	struct intel_th *th;
 	int err;
@@ -693,6 +832,11 @@ intel_th_alloc(struct device *dev, struct resource *devres,
 		goto err_ida;
 	}
 	th->dev = dev;
+	th->drvdata = drvdata;
+
+	th->resource = devres;
+	th->num_resources = ndevres;
+	th->irq = irq;
 
 	dev_set_drvdata(dev, th);
 
@@ -700,18 +844,15 @@ intel_th_alloc(struct device *dev, struct resource *devres,
 	pm_runtime_put(dev);
 	pm_runtime_allow(dev);
 
-	err = intel_th_populate(th, devres, ndevres, irq);
-	if (err)
-		goto err_chrdev;
+	err = intel_th_populate(th);
+	if (err) {
+		/* free the subdevices and undo everything */
+		intel_th_free(th);
+		return ERR_PTR(err);
+	}
 
 	return th;
 
-err_chrdev:
-	pm_runtime_forbid(dev);
-
-	__unregister_chrdev(th->major, 0, TH_POSSIBLE_OUTPUTS,
-			    "intel_th/output");
-
 err_ida:
 	ida_simple_remove(&intel_th_ida, th->id);
 
@@ -727,11 +868,15 @@ void intel_th_free(struct intel_th *th)
 	int i;
 
 	intel_th_request_hub_module_flush(th);
-	for (i = 0; i < TH_SUBDEVICE_MAX; i++)
-		if (th->thdev[i] && th->thdev[i] != th->hub)
-			intel_th_device_remove(th->thdev[i]);
 
 	intel_th_device_remove(th->hub);
+	for (i = 0; i < th->num_thdevs; i++) {
+		if (th->thdev[i] != th->hub)
+			intel_th_device_remove(th->thdev[i]);
+		th->thdev[i] = NULL;
+	}
+
+	th->num_thdevs = 0;
 
 	pm_runtime_get_sync(th->dev);
 	pm_runtime_forbid(th->dev);
diff --git a/drivers/hwtracing/intel_th/gth.c b/drivers/hwtracing/intel_th/gth.c
index dd32d0bad687..018678ec3c13 100644
--- a/drivers/hwtracing/intel_th/gth.c
+++ b/drivers/hwtracing/intel_th/gth.c
@@ -285,16 +285,16 @@ gth_output_parm_get(struct gth_device *gth, int port, unsigned int parm)
  */
 static int intel_th_gth_reset(struct gth_device *gth)
 {
-	u32 scratchpad;
+	u32 reg;
 	int port, i;
 
-	scratchpad = ioread32(gth->base + REG_GTH_SCRPD0);
-	if (scratchpad & SCRPD_DEBUGGER_IN_USE)
+	reg = ioread32(gth->base + REG_GTH_SCRPD0);
+	if (reg & SCRPD_DEBUGGER_IN_USE)
 		return -EBUSY;
 
 	/* Always save/restore STH and TU registers in S0ix entry/exit */
-	scratchpad |= SCRPD_STH_IS_ENABLED | SCRPD_TRIGGER_IS_ENABLED;
-	iowrite32(scratchpad, gth->base + REG_GTH_SCRPD0);
+	reg |= SCRPD_STH_IS_ENABLED | SCRPD_TRIGGER_IS_ENABLED;
+	iowrite32(reg, gth->base + REG_GTH_SCRPD0);
 
 	/* output ports */
 	for (port = 0; port < 8; port++) {
@@ -512,6 +512,15 @@ static void intel_th_gth_disable(struct intel_th_device *thdev,
 	iowrite32(reg, gth->base + REG_GTH_SCRPD0);
 }
 
+static void gth_tscu_resync(struct gth_device *gth)
+{
+	u32 reg;
+
+	reg = ioread32(gth->base + REG_TSCU_TSUCTRL);
+	reg &= ~TSUCTRL_CTCRESYNC;
+	iowrite32(reg, gth->base + REG_TSCU_TSUCTRL);
+}
+
 /**
  * intel_th_gth_enable() - enable tracing to an output device
  * @thdev:	GTH device
@@ -524,6 +533,7 @@ static void intel_th_gth_enable(struct intel_th_device *thdev,
 				struct intel_th_output *output)
 {
 	struct gth_device *gth = dev_get_drvdata(&thdev->dev);
+	struct intel_th *th = to_intel_th(thdev);
 	u32 scr = 0xfc0000, scrpd;
 	int master;
 
@@ -539,6 +549,9 @@ static void intel_th_gth_enable(struct intel_th_device *thdev,
 	output->active = true;
 	spin_unlock(&gth->gth_lock);
 
+	if (INTEL_TH_CAP(th, tscu_enable))
+		gth_tscu_resync(gth);
+
 	scrpd = ioread32(gth->base + REG_GTH_SCRPD0);
 	scrpd |= output->scratchpad;
 	iowrite32(scrpd, gth->base + REG_GTH_SCRPD0);
@@ -639,6 +652,7 @@ intel_th_gth_set_output(struct intel_th_device *thdev, unsigned int master)
 static int intel_th_gth_probe(struct intel_th_device *thdev)
 {
 	struct device *dev = &thdev->dev;
+	struct intel_th *th = dev_get_drvdata(dev->parent);
 	struct gth_device *gth;
 	struct resource *res;
 	void __iomem *base;
@@ -660,6 +674,8 @@ static int intel_th_gth_probe(struct intel_th_device *thdev)
 	gth->base = base;
 	spin_lock_init(&gth->gth_lock);
 
+	dev_set_drvdata(dev, gth);
+
 	/*
 	 * Host mode can be signalled via SW means or via SCRPD_DEBUGGER_IN_USE
 	 * bit. Either way, don't reset HW in this case, and don't export any
@@ -667,7 +683,7 @@ static int intel_th_gth_probe(struct intel_th_device *thdev)
 	 * drivers to ports, see intel_th_gth_assign().
 	 */
 	if (thdev->host_mode)
-		goto done;
+		return 0;
 
 	ret = intel_th_gth_reset(gth);
 	if (ret) {
@@ -676,7 +692,7 @@ static int intel_th_gth_probe(struct intel_th_device *thdev)
 
 		thdev->host_mode = true;
 
-		goto done;
+		return 0;
 	}
 
 	for (i = 0; i < TH_CONFIGURABLE_MASTERS + 1; i++)
@@ -687,6 +703,13 @@ static int intel_th_gth_probe(struct intel_th_device *thdev)
 		gth->output[i].index = i;
 		gth->output[i].port_type =
 			gth_output_parm_get(gth, i, TH_OUTPUT_PARM(port));
+		if (gth->output[i].port_type == GTH_NONE)
+			continue;
+
+		ret = intel_th_output_enable(th, gth->output[i].port_type);
+		/* -ENODEV is ok, we just won't have that device enumerated */
+		if (ret && ret != -ENODEV)
+			return ret;
 	}
 
 	if (intel_th_output_attributes(gth) ||
@@ -698,9 +721,6 @@ static int intel_th_gth_probe(struct intel_th_device *thdev)
 		return -ENOMEM;
 	}
 
-done:
-	dev_set_drvdata(dev, gth);
-
 	return 0;
 }
 
diff --git a/drivers/hwtracing/intel_th/gth.h b/drivers/hwtracing/intel_th/gth.h
index 56f0d2620577..f3d234251a12 100644
--- a/drivers/hwtracing/intel_th/gth.h
+++ b/drivers/hwtracing/intel_th/gth.h
@@ -55,9 +55,14 @@ enum {
 	REG_GTH_SCRPD1		= 0xe4, /* ScratchPad[1] */
 	REG_GTH_SCRPD2		= 0xe8, /* ScratchPad[2] */
 	REG_GTH_SCRPD3		= 0xec, /* ScratchPad[3] */
+	REG_TSCU_TSUCTRL	= 0x2000, /* TSCU control register */
+	REG_TSCU_TSCUSTAT	= 0x2004, /* TSCU status register */
 };
 
 /* waiting for Pipeline Empty bit(s) to assert for GTH */
 #define GTH_PLE_WAITLOOP_DEPTH	10000
 
+#define TSUCTRL_CTCRESYNC	BIT(0)
+#define TSCUSTAT_CTCSYNCING	BIT(1)
+
 #endif /* __INTEL_TH_GTH_H__ */
diff --git a/drivers/hwtracing/intel_th/intel_th.h b/drivers/hwtracing/intel_th/intel_th.h
index 3096e7054f6d..99ad563fc40d 100644
--- a/drivers/hwtracing/intel_th/intel_th.h
+++ b/drivers/hwtracing/intel_th/intel_th.h
@@ -48,8 +48,19 @@ struct intel_th_output {
 };
 
 /**
+ * struct intel_th_drvdata - describes hardware capabilities and quirks
+ * @tscu_enable:	device needs SW to enable time stamping unit
+ */
+struct intel_th_drvdata {
+	unsigned int	tscu_enable        : 1;
+};
+
+#define INTEL_TH_CAP(_th, _cap) ((_th)->drvdata ? (_th)->drvdata->_cap : 0)
+
+/**
  * struct intel_th_device - device on the intel_th bus
  * @dev:		device
+ * @drvdata:		hardware capabilities/quirks
  * @resource:		array of resources available to this device
  * @num_resources:	number of resources in @resource array
  * @type:		INTEL_TH_{SOURCE,OUTPUT,SWITCH}
@@ -59,11 +70,12 @@ struct intel_th_output {
  * @name:		device name to match the driver
  */
 struct intel_th_device {
-	struct device	dev;
-	struct resource	*resource;
-	unsigned int	num_resources;
-	unsigned int	type;
-	int		id;
+	struct device		dev;
+	struct intel_th_drvdata *drvdata;
+	struct resource		*resource;
+	unsigned int		num_resources;
+	unsigned int		type;
+	int			id;
 
 	/* INTEL_TH_SWITCH specific */
 	bool			host_mode;
@@ -96,6 +108,17 @@ intel_th_device_get_resource(struct intel_th_device *thdev, unsigned int type,
 	return NULL;
 }
 
+/*
+ * GTH, output ports configuration
+ */
+enum {
+	GTH_NONE = 0,
+	GTH_MSU,	/* memory/usb */
+	GTH_CTP,	/* Common Trace Port */
+	GTH_LPP,	/* Low Power Path */
+	GTH_PTI,	/* MIPI-PTI */
+};
+
 /**
  * intel_th_output_assigned() - if an output device is assigned to a switch port
  * @thdev:	the output device
@@ -106,7 +129,8 @@ static inline bool
 intel_th_output_assigned(struct intel_th_device *thdev)
 {
 	return thdev->type == INTEL_TH_OUTPUT &&
-		thdev->output.port >= 0;
+		(thdev->output.port >= 0 ||
+		 thdev->output.type == GTH_NONE);
 }
 
 /**
@@ -161,8 +185,18 @@ struct intel_th_driver {
 #define to_intel_th_driver_or_null(_d)		\
 	((_d) ? to_intel_th_driver(_d) : NULL)
 
+/*
+ * Subdevice tree structure is as follows:
+ * + struct intel_th device (pci; dev_{get,set}_drvdata()
+ *   + struct intel_th_device INTEL_TH_SWITCH (GTH)
+ *     + struct intel_th_device INTEL_TH_OUTPUT (MSU, PTI)
+ *   + struct intel_th_device INTEL_TH_SOURCE (STH)
+ *
+ * In other words, INTEL_TH_OUTPUT devices are children of INTEL_TH_SWITCH;
+ * INTEL_TH_SWITCH and INTEL_TH_SOURCE are children of the intel_th device.
+ */
 static inline struct intel_th_device *
-to_intel_th_hub(struct intel_th_device *thdev)
+to_intel_th_parent(struct intel_th_device *thdev)
 {
 	struct device *parent = thdev->dev.parent;
 
@@ -172,9 +206,20 @@ to_intel_th_hub(struct intel_th_device *thdev)
 	return to_intel_th_device(parent);
 }
 
+static inline struct intel_th *to_intel_th(struct intel_th_device *thdev)
+{
+	if (thdev->type == INTEL_TH_OUTPUT)
+		thdev = to_intel_th_parent(thdev);
+
+	if (WARN_ON_ONCE(!thdev || thdev->type == INTEL_TH_OUTPUT))
+		return NULL;
+
+	return dev_get_drvdata(thdev->dev.parent);
+}
+
 struct intel_th *
-intel_th_alloc(struct device *dev, struct resource *devres,
-	       unsigned int ndevres, int irq);
+intel_th_alloc(struct device *dev, struct intel_th_drvdata *drvdata,
+	       struct resource *devres, unsigned int ndevres, int irq);
 void intel_th_free(struct intel_th *th);
 
 int intel_th_driver_register(struct intel_th_driver *thdrv);
@@ -184,6 +229,7 @@ int intel_th_trace_enable(struct intel_th_device *thdev);
 int intel_th_trace_disable(struct intel_th_device *thdev);
 int intel_th_set_output(struct intel_th_device *thdev,
 			unsigned int master);
+int intel_th_output_enable(struct intel_th *th, unsigned int otype);
 
 enum {
 	TH_MMIO_CONFIG = 0,
@@ -191,8 +237,9 @@ enum {
 	TH_MMIO_END,
 };
 
-#define TH_SUBDEVICE_MAX	6
 #define TH_POSSIBLE_OUTPUTS	8
+/* Total number of possible subdevices: outputs + GTH + STH */
+#define TH_SUBDEVICE_MAX	(TH_POSSIBLE_OUTPUTS + 2)
 #define TH_CONFIGURABLE_MASTERS 256
 #define TH_MSC_MAX		2
 
@@ -201,6 +248,10 @@ enum {
  * @dev:	driver core's device
  * @thdev:	subdevices
  * @hub:	"switch" subdevice (GTH)
+ * @resource:	resources of the entire controller
+ * @num_thdevs:	number of devices in the @thdev array
+ * @num_resources:	number or resources in the @resource array
+ * @irq:	irq number
  * @id:		this Intel TH controller's device ID in the system
  * @major:	device node major for output devices
  */
@@ -209,6 +260,14 @@ struct intel_th {
 
 	struct intel_th_device	*thdev[TH_SUBDEVICE_MAX];
 	struct intel_th_device	*hub;
+	struct intel_th_drvdata	*drvdata;
+
+	struct resource		*resource;
+	int			(*activate)(struct intel_th *);
+	void			(*deactivate)(struct intel_th *);
+	unsigned int		num_thdevs;
+	unsigned int		num_resources;
+	int			irq;
 
 	int			id;
 	int			major;
@@ -220,6 +279,17 @@ struct intel_th {
 #endif
 };
 
+static inline struct intel_th_device *
+to_intel_th_hub(struct intel_th_device *thdev)
+{
+	if (thdev->type == INTEL_TH_SWITCH)
+		return thdev;
+	else if (thdev->type == INTEL_TH_OUTPUT)
+		return to_intel_th_parent(thdev);
+
+	return to_intel_th(thdev)->hub;
+}
+
 /*
  * Register windows
  */
@@ -228,6 +298,10 @@ enum {
 	REG_GTH_OFFSET		= 0x0000,
 	REG_GTH_LENGTH		= 0x2000,
 
+	/* Timestamp counter unit (TSCU) */
+	REG_TSCU_OFFSET		= 0x2000,
+	REG_TSCU_LENGTH		= 0x1000,
+
 	/* Software Trace Hub (STH) [0x4000..0x4fff] */
 	REG_STH_OFFSET		= 0x4000,
 	REG_STH_LENGTH		= 0x2000,
@@ -250,16 +324,6 @@ enum {
 };
 
 /*
- * GTH, output ports configuration
- */
-enum {
-	GTH_NONE = 0,
-	GTH_MSU,	/* memory/usb */
-	GTH_CTP,	/* Common Trace Port */
-	GTH_PTI = 4,	/* MIPI-PTI */
-};
-
-/*
  * Scratchpad bits: tell firmware and external debuggers
  * what we are up to.
  */
diff --git a/drivers/hwtracing/intel_th/msu.c b/drivers/hwtracing/intel_th/msu.c
index dbbe31df74df..dfb57eaa9f22 100644
--- a/drivers/hwtracing/intel_th/msu.c
+++ b/drivers/hwtracing/intel_th/msu.c
@@ -709,17 +709,17 @@ static int msc_buffer_win_alloc(struct msc *msc, unsigned int nr_blocks)
 	}
 
 	for (i = 0; i < nr_blocks; i++) {
-		win->block[i].bdesc = dma_alloc_coherent(msc_dev(msc), size,
-							 &win->block[i].addr,
-							 GFP_KERNEL);
+		win->block[i].bdesc =
+			dma_alloc_coherent(msc_dev(msc)->parent->parent, size,
+					   &win->block[i].addr, GFP_KERNEL);
+
+		if (!win->block[i].bdesc)
+			goto err_nomem;
 
 #ifdef CONFIG_X86
 		/* Set the page as uncached */
 		set_memory_uc((unsigned long)win->block[i].bdesc, 1);
 #endif
-
-		if (!win->block[i].bdesc)
-			goto err_nomem;
 	}
 
 	win->msc = msc;
diff --git a/drivers/hwtracing/intel_th/pci.c b/drivers/hwtracing/intel_th/pci.c
index 590cf90dd21a..bc9cebc30526 100644
--- a/drivers/hwtracing/intel_th/pci.c
+++ b/drivers/hwtracing/intel_th/pci.c
@@ -27,9 +27,53 @@
 
 #define BAR_MASK (BIT(TH_MMIO_CONFIG) | BIT(TH_MMIO_SW))
 
+#define PCI_REG_NPKDSC	0x80
+#define NPKDSC_TSACT	BIT(5)
+
+static int intel_th_pci_activate(struct intel_th *th)
+{
+	struct pci_dev *pdev = to_pci_dev(th->dev);
+	u32 npkdsc;
+	int err;
+
+	if (!INTEL_TH_CAP(th, tscu_enable))
+		return 0;
+
+	err = pci_read_config_dword(pdev, PCI_REG_NPKDSC, &npkdsc);
+	if (!err) {
+		npkdsc |= NPKDSC_TSACT;
+		err = pci_write_config_dword(pdev, PCI_REG_NPKDSC, npkdsc);
+	}
+
+	if (err)
+		dev_err(&pdev->dev, "failed to read NPKDSC register\n");
+
+	return err;
+}
+
+static void intel_th_pci_deactivate(struct intel_th *th)
+{
+	struct pci_dev *pdev = to_pci_dev(th->dev);
+	u32 npkdsc;
+	int err;
+
+	if (!INTEL_TH_CAP(th, tscu_enable))
+		return;
+
+	err = pci_read_config_dword(pdev, PCI_REG_NPKDSC, &npkdsc);
+	if (!err) {
+		npkdsc |= NPKDSC_TSACT;
+		err = pci_write_config_dword(pdev, PCI_REG_NPKDSC, npkdsc);
+	}
+
+	if (err)
+		dev_err(&pdev->dev, "failed to read NPKDSC register\n");
+}
+
 static int intel_th_pci_probe(struct pci_dev *pdev,
 			      const struct pci_device_id *id)
 {
+	struct intel_th_drvdata *drvdata = (void *)id->driver_data;
 	struct intel_th *th;
 	int err;
 
@@ -41,11 +85,16 @@ static int intel_th_pci_probe(struct pci_dev *pdev,
 	if (err)
 		return err;
 
-	th = intel_th_alloc(&pdev->dev, pdev->resource,
+	th = intel_th_alloc(&pdev->dev, drvdata, pdev->resource,
 			    DEVICE_COUNT_RESOURCE, pdev->irq);
 	if (IS_ERR(th))
 		return PTR_ERR(th);
 
+	th->activate   = intel_th_pci_activate;
+	th->deactivate = intel_th_pci_deactivate;
+
+	pci_set_master(pdev);
+
 	return 0;
 }
 
@@ -56,6 +105,10 @@ static void intel_th_pci_remove(struct pci_dev *pdev)
 	intel_th_free(th);
 }
 
+static const struct intel_th_drvdata intel_th_2x = {
+	.tscu_enable	= 1,
+};
+
 static const struct pci_device_id intel_th_pci_id_table[] = {
 	{
 		PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x9d26),
@@ -93,7 +146,17 @@ static const struct pci_device_id intel_th_pci_id_table[] = {
 	{
 		/* Gemini Lake */
 		PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x318e),
-		.driver_data = (kernel_ulong_t)0,
+		.driver_data = (kernel_ulong_t)&intel_th_2x,
+	},
+	{
+		/* Cannon Lake H */
+		PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xa326),
+		.driver_data = (kernel_ulong_t)&intel_th_2x,
+	},
+	{
+		/* Cannon Lake LP */
+		PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x9da6),
+		.driver_data = (kernel_ulong_t)&intel_th_2x,
 	},
 	{ 0 },
 };
diff --git a/drivers/hwtracing/intel_th/pti.c b/drivers/hwtracing/intel_th/pti.c
index 35738b5bfccd..e96a1fcb57b2 100644
--- a/drivers/hwtracing/intel_th/pti.c
+++ b/drivers/hwtracing/intel_th/pti.c
@@ -1,7 +1,7 @@
 /*
  * Intel(R) Trace Hub PTI output driver
  *
- * Copyright (C) 2014-2015 Intel Corporation.
+ * Copyright (C) 2014-2016 Intel Corporation.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -34,6 +34,8 @@ struct pti_device {
 	unsigned int		freeclk;
 	unsigned int		clkdiv;
 	unsigned int		patgen;
+	unsigned int		lpp_dest_mask;
+	unsigned int		lpp_dest;
 };
 
 /* map PTI widths to MODE settings of PTI_CTL register */
@@ -163,6 +165,7 @@ static int intel_th_pti_activate(struct intel_th_device *thdev)
 		ctl |= PTI_FCEN;
 	ctl |= pti->mode << __ffs(PTI_MODE);
 	ctl |= pti->clkdiv << __ffs(PTI_CLKDIV);
+	ctl |= pti->lpp_dest << __ffs(LPP_DEST);
 
 	iowrite32(ctl, pti->base + REG_PTI_CTL);
 
@@ -192,6 +195,15 @@ static void read_hw_config(struct pti_device *pti)
 		pti->mode = pti_width_mode(4);
 	if (!pti->clkdiv)
 		pti->clkdiv = 1;
+
+	if (pti->thdev->output.type == GTH_LPP) {
+		if (ctl & LPP_PTIPRESENT)
+			pti->lpp_dest_mask |= LPP_DEST_PTI;
+		if (ctl & LPP_BSSBPRESENT)
+			pti->lpp_dest_mask |= LPP_DEST_EXI;
+		if (ctl & LPP_DEST)
+			pti->lpp_dest = 1;
+	}
 }
 
 static int intel_th_pti_probe(struct intel_th_device *thdev)
@@ -239,10 +251,103 @@ static struct intel_th_driver intel_th_pti_driver = {
 	},
 };
 
-module_driver(intel_th_pti_driver,
-	      intel_th_driver_register,
-	      intel_th_driver_unregister);
+static const char * const lpp_dest_str[] = { "pti", "exi" };
+
+static ssize_t lpp_dest_show(struct device *dev, struct device_attribute *attr,
+			     char *buf)
+{
+	struct pti_device *pti = dev_get_drvdata(dev);
+	ssize_t ret = 0;
+	int i;
+
+	for (i = ARRAY_SIZE(lpp_dest_str) - 1; i >= 0; i--) {
+		const char *fmt = pti->lpp_dest == i ? "[%s] " : "%s ";
+
+		if (!(pti->lpp_dest_mask & BIT(i)))
+			continue;
+
+		ret += scnprintf(buf + ret, PAGE_SIZE - ret,
+				 fmt, lpp_dest_str[i]);
+	}
+
+	if (ret)
+		buf[ret - 1] = '\n';
+
+	return ret;
+}
+
+static ssize_t lpp_dest_store(struct device *dev, struct device_attribute *attr,
+			      const char *buf, size_t size)
+{
+	struct pti_device *pti = dev_get_drvdata(dev);
+	ssize_t ret = -EINVAL;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(lpp_dest_str); i++)
+		if (sysfs_streq(buf, lpp_dest_str[i]))
+			break;
+
+	if (i < ARRAY_SIZE(lpp_dest_str) && pti->lpp_dest_mask & BIT(i)) {
+		pti->lpp_dest = i;
+		ret = size;
+	}
+
+	return ret;
+}
+
+static DEVICE_ATTR_RW(lpp_dest);
+
+static struct attribute *lpp_output_attrs[] = {
+	&dev_attr_mode.attr,
+	&dev_attr_freerunning_clock.attr,
+	&dev_attr_clock_divider.attr,
+	&dev_attr_lpp_dest.attr,
+	NULL,
+};
+
+static struct attribute_group lpp_output_group = {
+	.attrs	= lpp_output_attrs,
+};
+
+static struct intel_th_driver intel_th_lpp_driver = {
+	.probe		= intel_th_pti_probe,
+	.remove		= intel_th_pti_remove,
+	.activate	= intel_th_pti_activate,
+	.deactivate	= intel_th_pti_deactivate,
+	.attr_group	= &lpp_output_group,
+	.driver	= {
+		.name	= "lpp",
+		.owner	= THIS_MODULE,
+	},
+};
+
+static int __init intel_th_pti_lpp_init(void)
+{
+	int err;
+
+	err = intel_th_driver_register(&intel_th_pti_driver);
+	if (err)
+		return err;
+
+	err = intel_th_driver_register(&intel_th_lpp_driver);
+	if (err) {
+		intel_th_driver_unregister(&intel_th_pti_driver);
+		return err;
+	}
+
+	return 0;
+}
+
+module_init(intel_th_pti_lpp_init);
+
+static void __exit intel_th_pti_lpp_exit(void)
+{
+	intel_th_driver_unregister(&intel_th_pti_driver);
+	intel_th_driver_unregister(&intel_th_lpp_driver);
+}
+
+module_exit(intel_th_pti_lpp_exit);
 
 MODULE_LICENSE("GPL v2");
-MODULE_DESCRIPTION("Intel(R) Trace Hub PTI output driver");
+MODULE_DESCRIPTION("Intel(R) Trace Hub PTI/LPP output driver");
 MODULE_AUTHOR("Alexander Shishkin <alexander.shishkin@linux.intel.com>");
diff --git a/drivers/hwtracing/intel_th/pti.h b/drivers/hwtracing/intel_th/pti.h
index 20883f5628cf..30827be67b4c 100644
--- a/drivers/hwtracing/intel_th/pti.h
+++ b/drivers/hwtracing/intel_th/pti.h
@@ -23,7 +23,15 @@ enum {
 #define PTI_EN		BIT(0)
 #define PTI_FCEN	BIT(1)
 #define PTI_MODE	0xf0
+#define LPP_PTIPRESENT	BIT(8)
+#define LPP_BSSBPRESENT	BIT(9)
 #define PTI_CLKDIV	0x000f0000
 #define PTI_PATGENMODE	0x00f00000
+#define LPP_DEST	BIT(25)
+#define LPP_BSSBACT	BIT(30)
+#define LPP_LPPBUSY	BIT(31)
+
+#define LPP_DEST_PTI	BIT(0)
+#define LPP_DEST_EXI	BIT(1)
 
 #endif /* __INTEL_TH_STH_H__ */
diff --git a/drivers/hwtracing/stm/core.c b/drivers/hwtracing/stm/core.c
index 0e731143f6a4..9414900575d8 100644
--- a/drivers/hwtracing/stm/core.c
+++ b/drivers/hwtracing/stm/core.c
@@ -566,7 +566,7 @@ static int stm_char_policy_set_ioctl(struct stm_file *stmf, void __user *arg)
 	if (copy_from_user(&size, arg, sizeof(size)))
 		return -EFAULT;
 
-	if (size >= PATH_MAX + sizeof(*id))
+	if (size < sizeof(*id) || size >= PATH_MAX + sizeof(*id))
 		return -EINVAL;
 
 	/*
diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig
index 65fa29591d21..c06dce2c1da7 100644
--- a/drivers/i2c/busses/Kconfig
+++ b/drivers/i2c/busses/Kconfig
@@ -189,6 +189,14 @@ config I2C_PIIX4
 	  This driver can also be built as a module.  If so, the module
 	  will be called i2c-piix4.
 
+config I2C_CHT_WC
+	tristate "Intel Cherry Trail Whiskey Cove PMIC smbus controller"
+	depends on INTEL_SOC_PMIC_CHTWC
+	help
+	  If you say yes to this option, support will be included for the
+	  SMBus controller found in the Intel Cherry Trail Whiskey Cove PMIC
+	  found on some Intel Cherry Trail systems.
+
 config I2C_NFORCE2
 	tristate "Nvidia nForce2, nForce3 and nForce4"
 	depends on PCI
@@ -328,6 +336,16 @@ config I2C_POWERMAC
 
 comment "I2C system bus drivers (mostly embedded / system-on-chip)"
 
+config I2C_ALTERA
+	tristate "Altera Soft IP I2C"
+	depends on (ARCH_SOCFPGA || NIOS2) && OF
+	help
+	  If you say yes to this option, support will be included for the
+	  Altera Soft IP I2C interfaces on SoCFPGA and Nios2 architectures.
+
+	  This driver can also be built as a module.  If so, the module
+	  will be called i2c-altera.
+
 config I2C_ASPEED
 	tristate "Aspeed I2C Controller"
 	depends on ARCH_ASPEED || COMPILE_TEST
@@ -900,6 +918,13 @@ config I2C_SIRF
 	  This driver can also be built as a module.  If so, the module
 	  will be called i2c-sirf.
 
+config I2C_SPRD
+	bool "Spreadtrum I2C interface"
+	depends on I2C=y && ARCH_SPRD
+	help
+	  If you say yes to this option, support will be included for the
+	  Spreadtrum I2C interface.
+
 config I2C_ST
 	tristate "STMicroelectronics SSC I2C support"
 	depends on ARCH_STI
@@ -920,6 +945,16 @@ config I2C_STM32F4
 	  This driver can also be built as module. If so, the module
 	  will be called i2c-stm32f4.
 
+config I2C_STM32F7
+	tristate "STMicroelectronics STM32F7 I2C support"
+	depends on ARCH_STM32 || COMPILE_TEST
+	help
+	  Enable this option to add support for STM32 I2C controller embedded
+	  in STM32F7 SoCs.
+
+	  This driver can also be built as module. If so, the module
+	  will be called i2c-stm32f7.
+
 config I2C_STU300
 	tristate "ST Microelectronics DDC I2C interface"
 	depends on MACH_U300
diff --git a/drivers/i2c/busses/Makefile b/drivers/i2c/busses/Makefile
index 1b2fc815a4d8..47f3ac9a695a 100644
--- a/drivers/i2c/busses/Makefile
+++ b/drivers/i2c/busses/Makefile
@@ -12,6 +12,7 @@ obj-$(CONFIG_I2C_ALI15X3)	+= i2c-ali15x3.o
 obj-$(CONFIG_I2C_AMD756)	+= i2c-amd756.o
 obj-$(CONFIG_I2C_AMD756_S4882)	+= i2c-amd756-s4882.o
 obj-$(CONFIG_I2C_AMD8111)	+= i2c-amd8111.o
+obj-$(CONFIG_I2C_CHT_WC)	+= i2c-cht-wc.o
 obj-$(CONFIG_I2C_I801)		+= i2c-i801.o
 obj-$(CONFIG_I2C_ISCH)		+= i2c-isch.o
 obj-$(CONFIG_I2C_ISMT)		+= i2c-ismt.o
@@ -29,6 +30,7 @@ obj-$(CONFIG_I2C_HYDRA)		+= i2c-hydra.o
 obj-$(CONFIG_I2C_POWERMAC)	+= i2c-powermac.o
 
 # Embedded system I2C/SMBus host controller drivers
+obj-$(CONFIG_I2C_ALTERA)	+= i2c-altera.o
 obj-$(CONFIG_I2C_ASPEED)	+= i2c-aspeed.o
 obj-$(CONFIG_I2C_AT91)		+= i2c-at91.o
 obj-$(CONFIG_I2C_AU1550)	+= i2c-au1550.o
@@ -89,8 +91,10 @@ obj-$(CONFIG_I2C_SH7760)	+= i2c-sh7760.o
 obj-$(CONFIG_I2C_SH_MOBILE)	+= i2c-sh_mobile.o
 obj-$(CONFIG_I2C_SIMTEC)	+= i2c-simtec.o
 obj-$(CONFIG_I2C_SIRF)		+= i2c-sirf.o
+obj-$(CONFIG_I2C_SPRD)		+= i2c-sprd.o
 obj-$(CONFIG_I2C_ST)		+= i2c-st.o
 obj-$(CONFIG_I2C_STM32F4)	+= i2c-stm32f4.o
+obj-$(CONFIG_I2C_STM32F7)	+= i2c-stm32f7.o
 obj-$(CONFIG_I2C_STU300)	+= i2c-stu300.o
 obj-$(CONFIG_I2C_SUN6I_P2WI)	+= i2c-sun6i-p2wi.o
 obj-$(CONFIG_I2C_TEGRA)		+= i2c-tegra.o
diff --git a/drivers/i2c/busses/i2c-altera.c b/drivers/i2c/busses/i2c-altera.c
new file mode 100644
index 000000000000..f5e1941e65b5
--- /dev/null
+++ b/drivers/i2c/busses/i2c-altera.c
@@ -0,0 +1,511 @@
+/*
+ *  Copyright Intel Corporation (C) 2017.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Based on the i2c-axxia.c driver.
+ */
+#include <linux/clk.h>
+#include <linux/clkdev.h>
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/iopoll.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/platform_device.h>
+
+#define ALTR_I2C_TFR_CMD	0x00	/* Transfer Command register */
+#define     ALTR_I2C_TFR_CMD_STA	BIT(9)	/* send START before byte */
+#define     ALTR_I2C_TFR_CMD_STO	BIT(8)	/* send STOP after byte */
+#define     ALTR_I2C_TFR_CMD_RW_D	BIT(0)	/* Direction of transfer */
+#define ALTR_I2C_RX_DATA	0x04	/* RX data FIFO register */
+#define ALTR_I2C_CTRL		0x08	/* Control register */
+#define     ALTR_I2C_CTRL_RXT_SHFT	4	/* RX FIFO Threshold */
+#define     ALTR_I2C_CTRL_TCT_SHFT	2	/* TFER CMD FIFO Threshold */
+#define     ALTR_I2C_CTRL_BSPEED	BIT(1)	/* Bus Speed (1=Fast) */
+#define     ALTR_I2C_CTRL_EN	BIT(0)	/* Enable Core (1=Enable) */
+#define ALTR_I2C_ISER		0x0C	/* Interrupt Status Enable register */
+#define     ALTR_I2C_ISER_RXOF_EN	BIT(4)	/* Enable RX OVERFLOW IRQ */
+#define     ALTR_I2C_ISER_ARB_EN	BIT(3)	/* Enable ARB LOST IRQ */
+#define     ALTR_I2C_ISER_NACK_EN	BIT(2)	/* Enable NACK DET IRQ */
+#define     ALTR_I2C_ISER_RXRDY_EN	BIT(1)	/* Enable RX Ready IRQ */
+#define     ALTR_I2C_ISER_TXRDY_EN	BIT(0)	/* Enable TX Ready IRQ */
+#define ALTR_I2C_ISR		0x10	/* Interrupt Status register */
+#define     ALTR_I2C_ISR_RXOF		BIT(4)	/* RX OVERFLOW IRQ */
+#define     ALTR_I2C_ISR_ARB		BIT(3)	/* ARB LOST IRQ */
+#define     ALTR_I2C_ISR_NACK		BIT(2)	/* NACK DET IRQ */
+#define     ALTR_I2C_ISR_RXRDY		BIT(1)	/* RX Ready IRQ */
+#define     ALTR_I2C_ISR_TXRDY		BIT(0)	/* TX Ready IRQ */
+#define ALTR_I2C_STATUS		0x14	/* Status register */
+#define     ALTR_I2C_STAT_CORE		BIT(0)	/* Core Status (0=idle) */
+#define ALTR_I2C_TC_FIFO_LVL	0x18	/* Transfer FIFO LVL register */
+#define ALTR_I2C_RX_FIFO_LVL	0x1C	/* Receive FIFO LVL register */
+#define ALTR_I2C_SCL_LOW	0x20	/* SCL low count register */
+#define ALTR_I2C_SCL_HIGH	0x24	/* SCL high count register */
+#define ALTR_I2C_SDA_HOLD	0x28	/* SDA hold count register */
+
+#define ALTR_I2C_ALL_IRQ	(ALTR_I2C_ISR_RXOF | ALTR_I2C_ISR_ARB | \
+				 ALTR_I2C_ISR_NACK | ALTR_I2C_ISR_RXRDY | \
+				 ALTR_I2C_ISR_TXRDY)
+
+#define ALTR_I2C_THRESHOLD	0	/* IRQ Threshold at 1 element */
+#define ALTR_I2C_DFLT_FIFO_SZ	4
+#define ALTR_I2C_TIMEOUT	100000	/* 100ms */
+#define ALTR_I2C_XFER_TIMEOUT	(msecs_to_jiffies(250))
+
+/**
+ * altr_i2c_dev - I2C device context
+ * @base: pointer to register struct
+ * @msg: pointer to current message
+ * @msg_len: number of bytes transferred in msg
+ * @msg_err: error code for completed message
+ * @msg_complete: xfer completion object
+ * @dev: device reference
+ * @adapter: core i2c abstraction
+ * @i2c_clk: clock reference for i2c input clock
+ * @bus_clk_rate: current i2c bus clock rate
+ * @buf: ptr to msg buffer for easier use.
+ * @fifo_size: size of the FIFO passed in.
+ * @isr_mask: cached copy of local ISR enables.
+ * @isr_status: cached copy of local ISR status.
+ * @lock: spinlock for IRQ synchronization.
+ */
+struct altr_i2c_dev {
+	void __iomem *base;
+	struct i2c_msg *msg;
+	size_t msg_len;
+	int msg_err;
+	struct completion msg_complete;
+	struct device *dev;
+	struct i2c_adapter adapter;
+	struct clk *i2c_clk;
+	u32 bus_clk_rate;
+	u8 *buf;
+	u32 fifo_size;
+	u32 isr_mask;
+	u32 isr_status;
+	spinlock_t lock;	/* IRQ synchronization */
+};
+
+static void
+altr_i2c_int_enable(struct altr_i2c_dev *idev, u32 mask, bool enable)
+{
+	unsigned long flags;
+	u32 int_en;
+
+	spin_lock_irqsave(&idev->lock, flags);
+
+	int_en = readl(idev->base + ALTR_I2C_ISER);
+	if (enable)
+		idev->isr_mask = int_en | mask;
+	else
+		idev->isr_mask = int_en & ~mask;
+
+	writel(idev->isr_mask, idev->base + ALTR_I2C_ISER);
+
+	spin_unlock_irqrestore(&idev->lock, flags);
+}
+
+static void altr_i2c_int_clear(struct altr_i2c_dev *idev, u32 mask)
+{
+	u32 int_en = readl(idev->base + ALTR_I2C_ISR);
+
+	writel(int_en | mask, idev->base + ALTR_I2C_ISR);
+}
+
+static void altr_i2c_core_disable(struct altr_i2c_dev *idev)
+{
+	u32 tmp = readl(idev->base + ALTR_I2C_CTRL);
+
+	writel(tmp & ~ALTR_I2C_CTRL_EN, idev->base + ALTR_I2C_CTRL);
+}
+
+static void altr_i2c_core_enable(struct altr_i2c_dev *idev)
+{
+	u32 tmp = readl(idev->base + ALTR_I2C_CTRL);
+
+	writel(tmp | ALTR_I2C_CTRL_EN, idev->base + ALTR_I2C_CTRL);
+}
+
+static void altr_i2c_reset(struct altr_i2c_dev *idev)
+{
+	altr_i2c_core_disable(idev);
+	altr_i2c_core_enable(idev);
+}
+
+static inline void altr_i2c_stop(struct altr_i2c_dev *idev)
+{
+	writel(ALTR_I2C_TFR_CMD_STO, idev->base + ALTR_I2C_TFR_CMD);
+}
+
+static void altr_i2c_init(struct altr_i2c_dev *idev)
+{
+	u32 divisor = clk_get_rate(idev->i2c_clk) / idev->bus_clk_rate;
+	u32 clk_mhz = clk_get_rate(idev->i2c_clk) / 1000000;
+	u32 tmp = (ALTR_I2C_THRESHOLD << ALTR_I2C_CTRL_RXT_SHFT) |
+		  (ALTR_I2C_THRESHOLD << ALTR_I2C_CTRL_TCT_SHFT);
+	u32 t_high, t_low;
+
+	if (idev->bus_clk_rate <= 100000) {
+		tmp &= ~ALTR_I2C_CTRL_BSPEED;
+		/* Standard mode SCL 50/50 */
+		t_high = divisor * 1 / 2;
+		t_low = divisor * 1 / 2;
+	} else {
+		tmp |= ALTR_I2C_CTRL_BSPEED;
+		/* Fast mode SCL 33/66 */
+		t_high = divisor * 1 / 3;
+		t_low = divisor * 2 / 3;
+	}
+	writel(tmp, idev->base + ALTR_I2C_CTRL);
+
+	dev_dbg(idev->dev, "rate=%uHz per_clk=%uMHz -> ratio=1:%u\n",
+		idev->bus_clk_rate, clk_mhz, divisor);
+
+	/* Reset controller */
+	altr_i2c_reset(idev);
+
+	/* SCL High Time */
+	writel(t_high, idev->base + ALTR_I2C_SCL_HIGH);
+	/* SCL Low Time */
+	writel(t_low, idev->base + ALTR_I2C_SCL_LOW);
+	/* SDA Hold Time, 300ns */
+	writel(div_u64(300 * clk_mhz, 1000), idev->base + ALTR_I2C_SDA_HOLD);
+
+	/* Mask all master interrupt bits */
+	altr_i2c_int_enable(idev, ALTR_I2C_ALL_IRQ, false);
+}
+
+/**
+ * altr_i2c_transfer - On the last byte to be transmitted, send
+ * a Stop bit on the last byte.
+ */
+static void altr_i2c_transfer(struct altr_i2c_dev *idev, u32 data)
+{
+	/* On the last byte to be transmitted, send STOP */
+	if (idev->msg_len == 1)
+		data |= ALTR_I2C_TFR_CMD_STO;
+	if (idev->msg_len > 0)
+		writel(data, idev->base + ALTR_I2C_TFR_CMD);
+}
+
+/**
+ * altr_i2c_empty_rx_fifo - Fetch data from RX FIFO until end of
+ * transfer. Send a Stop bit on the last byte.
+ */
+static void altr_i2c_empty_rx_fifo(struct altr_i2c_dev *idev)
+{
+	size_t rx_fifo_avail = readl(idev->base + ALTR_I2C_RX_FIFO_LVL);
+	int bytes_to_transfer = min(rx_fifo_avail, idev->msg_len);
+
+	while (bytes_to_transfer-- > 0) {
+		*idev->buf++ = readl(idev->base + ALTR_I2C_RX_DATA);
+		idev->msg_len--;
+		altr_i2c_transfer(idev, 0);
+	}
+}
+
+/**
+ * altr_i2c_fill_tx_fifo - Fill TX FIFO from current message buffer.
+ * @return: Number of bytes left to transfer.
+ */
+static int altr_i2c_fill_tx_fifo(struct altr_i2c_dev *idev)
+{
+	size_t tx_fifo_avail = idev->fifo_size - readl(idev->base +
+						       ALTR_I2C_TC_FIFO_LVL);
+	int bytes_to_transfer = min(tx_fifo_avail, idev->msg_len);
+	int ret = idev->msg_len - bytes_to_transfer;
+
+	while (bytes_to_transfer-- > 0) {
+		altr_i2c_transfer(idev, *idev->buf++);
+		idev->msg_len--;
+	}
+
+	return ret;
+}
+
+static irqreturn_t altr_i2c_isr_quick(int irq, void *_dev)
+{
+	struct altr_i2c_dev *idev = _dev;
+	irqreturn_t ret = IRQ_HANDLED;
+
+	/* Read IRQ status but only interested in Enabled IRQs. */
+	idev->isr_status = readl(idev->base + ALTR_I2C_ISR) & idev->isr_mask;
+	if (idev->isr_status)
+		ret = IRQ_WAKE_THREAD;
+
+	return ret;
+}
+
+static irqreturn_t altr_i2c_isr(int irq, void *_dev)
+{
+	int ret;
+	bool read, finish = false;
+	struct altr_i2c_dev *idev = _dev;
+	u32 status = idev->isr_status;
+
+	if (!idev->msg) {
+		dev_warn(idev->dev, "unexpected interrupt\n");
+		altr_i2c_int_clear(idev, ALTR_I2C_ALL_IRQ);
+		return IRQ_HANDLED;
+	}
+	read = (idev->msg->flags & I2C_M_RD) != 0;
+
+	/* handle Lost Arbitration */
+	if (unlikely(status & ALTR_I2C_ISR_ARB)) {
+		altr_i2c_int_clear(idev, ALTR_I2C_ISR_ARB);
+		idev->msg_err = -EAGAIN;
+		finish = true;
+	} else if (unlikely(status & ALTR_I2C_ISR_NACK)) {
+		dev_dbg(idev->dev, "Could not get ACK\n");
+		idev->msg_err = -ENXIO;
+		altr_i2c_int_clear(idev, ALTR_I2C_ISR_NACK);
+		altr_i2c_stop(idev);
+		finish = true;
+	} else if (read && unlikely(status & ALTR_I2C_ISR_RXOF)) {
+		/* handle RX FIFO Overflow */
+		altr_i2c_empty_rx_fifo(idev);
+		altr_i2c_int_clear(idev, ALTR_I2C_ISR_RXRDY);
+		altr_i2c_stop(idev);
+		dev_err(idev->dev, "RX FIFO Overflow\n");
+		finish = true;
+	} else if (read && (status & ALTR_I2C_ISR_RXRDY)) {
+		/* RX FIFO needs service? */
+		altr_i2c_empty_rx_fifo(idev);
+		altr_i2c_int_clear(idev, ALTR_I2C_ISR_RXRDY);
+		if (!idev->msg_len)
+			finish = true;
+	} else if (!read && (status & ALTR_I2C_ISR_TXRDY)) {
+		/* TX FIFO needs service? */
+		altr_i2c_int_clear(idev, ALTR_I2C_ISR_TXRDY);
+		if (idev->msg_len > 0)
+			altr_i2c_fill_tx_fifo(idev);
+		else
+			finish = true;
+	} else {
+		dev_warn(idev->dev, "Unexpected interrupt: 0x%x\n", status);
+		altr_i2c_int_clear(idev, ALTR_I2C_ALL_IRQ);
+	}
+
+	if (finish) {
+		/* Wait for the Core to finish */
+		ret = readl_poll_timeout_atomic(idev->base + ALTR_I2C_STATUS,
+						status,
+						!(status & ALTR_I2C_STAT_CORE),
+						1, ALTR_I2C_TIMEOUT);
+		if (ret)
+			dev_err(idev->dev, "message timeout\n");
+		altr_i2c_int_enable(idev, ALTR_I2C_ALL_IRQ, false);
+		altr_i2c_int_clear(idev, ALTR_I2C_ALL_IRQ);
+		complete(&idev->msg_complete);
+		dev_dbg(idev->dev, "Message Complete\n");
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int altr_i2c_xfer_msg(struct altr_i2c_dev *idev, struct i2c_msg *msg)
+{
+	u32 imask = ALTR_I2C_ISR_RXOF | ALTR_I2C_ISR_ARB | ALTR_I2C_ISR_NACK;
+	unsigned long time_left;
+	u32 value;
+	u8 addr = i2c_8bit_addr_from_msg(msg);
+
+	idev->msg = msg;
+	idev->msg_len = msg->len;
+	idev->buf = msg->buf;
+	idev->msg_err = 0;
+	reinit_completion(&idev->msg_complete);
+	altr_i2c_core_enable(idev);
+
+	/* Make sure RX FIFO is empty */
+	do {
+		readl(idev->base + ALTR_I2C_RX_DATA);
+	} while (readl(idev->base + ALTR_I2C_RX_FIFO_LVL));
+
+	writel(ALTR_I2C_TFR_CMD_STA | addr, idev->base + ALTR_I2C_TFR_CMD);
+
+	if ((msg->flags & I2C_M_RD) != 0) {
+		imask |= ALTR_I2C_ISER_RXOF_EN | ALTR_I2C_ISER_RXRDY_EN;
+		altr_i2c_int_enable(idev, imask, true);
+		/* write the first byte to start the RX */
+		altr_i2c_transfer(idev, 0);
+	} else {
+		imask |= ALTR_I2C_ISR_TXRDY;
+		altr_i2c_int_enable(idev, imask, true);
+		altr_i2c_fill_tx_fifo(idev);
+	}
+
+	time_left = wait_for_completion_timeout(&idev->msg_complete,
+						ALTR_I2C_XFER_TIMEOUT);
+	altr_i2c_int_enable(idev, imask, false);
+
+	value = readl(idev->base + ALTR_I2C_STATUS) & ALTR_I2C_STAT_CORE;
+	if (value)
+		dev_err(idev->dev, "Core Status not IDLE...\n");
+
+	if (time_left == 0) {
+		idev->msg_err = -ETIMEDOUT;
+		dev_dbg(idev->dev, "Transaction timed out.\n");
+	}
+
+	altr_i2c_core_disable(idev);
+
+	return idev->msg_err;
+}
+
+static int
+altr_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
+{
+	struct altr_i2c_dev *idev = i2c_get_adapdata(adap);
+	int i, ret;
+
+	for (i = 0; i < num; i++) {
+		ret = altr_i2c_xfer_msg(idev, msgs++);
+		if (ret)
+			return ret;
+	}
+	return num;
+}
+
+static u32 altr_i2c_func(struct i2c_adapter *adap)
+{
+	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
+}
+
+static const struct i2c_algorithm altr_i2c_algo = {
+	.master_xfer = altr_i2c_xfer,
+	.functionality = altr_i2c_func,
+};
+
+static int altr_i2c_probe(struct platform_device *pdev)
+{
+	struct altr_i2c_dev *idev = NULL;
+	struct resource *res;
+	int irq, ret;
+	u32 val;
+
+	idev = devm_kzalloc(&pdev->dev, sizeof(*idev), GFP_KERNEL);
+	if (!idev)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	idev->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(idev->base))
+		return PTR_ERR(idev->base);
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		dev_err(&pdev->dev, "missing interrupt resource\n");
+		return irq;
+	}
+
+	idev->i2c_clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(idev->i2c_clk)) {
+		dev_err(&pdev->dev, "missing clock\n");
+		return PTR_ERR(idev->i2c_clk);
+	}
+
+	idev->dev = &pdev->dev;
+	init_completion(&idev->msg_complete);
+	spin_lock_init(&idev->lock);
+
+	val = device_property_read_u32(idev->dev, "fifo-size",
+				       &idev->fifo_size);
+	if (val) {
+		dev_err(&pdev->dev, "FIFO size set to default of %d\n",
+			ALTR_I2C_DFLT_FIFO_SZ);
+		idev->fifo_size = ALTR_I2C_DFLT_FIFO_SZ;
+	}
+
+	val = device_property_read_u32(idev->dev, "clock-frequency",
+				       &idev->bus_clk_rate);
+	if (val) {
+		dev_err(&pdev->dev, "Default to 100kHz\n");
+		idev->bus_clk_rate = 100000;	/* default clock rate */
+	}
+
+	if (idev->bus_clk_rate > 400000) {
+		dev_err(&pdev->dev, "invalid clock-frequency %d\n",
+			idev->bus_clk_rate);
+		return -EINVAL;
+	}
+
+	ret = devm_request_threaded_irq(&pdev->dev, irq, altr_i2c_isr_quick,
+					altr_i2c_isr, IRQF_ONESHOT,
+					pdev->name, idev);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to claim IRQ %d\n", irq);
+		return ret;
+	}
+
+	ret = clk_prepare_enable(idev->i2c_clk);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to enable clock\n");
+		return ret;
+	}
+
+	altr_i2c_init(idev);
+
+	i2c_set_adapdata(&idev->adapter, idev);
+	strlcpy(idev->adapter.name, pdev->name, sizeof(idev->adapter.name));
+	idev->adapter.owner = THIS_MODULE;
+	idev->adapter.algo = &altr_i2c_algo;
+	idev->adapter.dev.parent = &pdev->dev;
+	idev->adapter.dev.of_node = pdev->dev.of_node;
+
+	platform_set_drvdata(pdev, idev);
+
+	ret = i2c_add_adapter(&idev->adapter);
+	if (ret) {
+		clk_disable_unprepare(idev->i2c_clk);
+		return ret;
+	}
+	dev_info(&pdev->dev, "Altera SoftIP I2C Probe Complete\n");
+
+	return 0;
+}
+
+static int altr_i2c_remove(struct platform_device *pdev)
+{
+	struct altr_i2c_dev *idev = platform_get_drvdata(pdev);
+
+	clk_disable_unprepare(idev->i2c_clk);
+	i2c_del_adapter(&idev->adapter);
+
+	return 0;
+}
+
+/* Match table for of_platform binding */
+static const struct of_device_id altr_i2c_of_match[] = {
+	{ .compatible = "altr,softip-i2c-v1.0" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, altr_i2c_of_match);
+
+static struct platform_driver altr_i2c_driver = {
+	.probe = altr_i2c_probe,
+	.remove = altr_i2c_remove,
+	.driver = {
+		.name = "altera-i2c",
+		.of_match_table = altr_i2c_of_match,
+	},
+};
+
+module_platform_driver(altr_i2c_driver);
+
+MODULE_DESCRIPTION("Altera Soft IP I2C bus driver");
+MODULE_AUTHOR("Thor Thayer <thor.thayer@linux.intel.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/i2c/busses/i2c-aspeed.c b/drivers/i2c/busses/i2c-aspeed.c
index f19348328a71..284f8670dbeb 100644
--- a/drivers/i2c/busses/i2c-aspeed.c
+++ b/drivers/i2c/busses/i2c-aspeed.c
@@ -53,6 +53,9 @@
 #define ASPEED_I2CD_MASTER_EN				BIT(0)
 
 /* 0x04 : I2CD Clock and AC Timing Control Register #1 */
+#define ASPEED_I2CD_TIME_TBUF_MASK			GENMASK(31, 28)
+#define ASPEED_I2CD_TIME_THDSTA_MASK			GENMASK(27, 24)
+#define ASPEED_I2CD_TIME_TACST_MASK			GENMASK(23, 20)
 #define ASPEED_I2CD_TIME_SCL_HIGH_SHIFT			16
 #define ASPEED_I2CD_TIME_SCL_HIGH_MASK			GENMASK(19, 16)
 #define ASPEED_I2CD_TIME_SCL_LOW_SHIFT			12
@@ -132,6 +135,7 @@ struct aspeed_i2c_bus {
 	/* Synchronizes I/O mem access to base. */
 	spinlock_t			lock;
 	struct completion		cmd_complete;
+	u32				(*get_clk_reg_val)(u32 divisor);
 	unsigned long			parent_clk_frequency;
 	u32				bus_frequency;
 	/* Transaction state. */
@@ -410,10 +414,11 @@ static bool aspeed_i2c_master_irq(struct aspeed_i2c_bus *bus)
 	}
 
 	/* We are in an invalid state; reset bus to a known state. */
-	if (!bus->msgs && bus->master_state != ASPEED_I2C_MASTER_STOP) {
+	if (!bus->msgs) {
 		dev_err(bus->dev, "bus in unknown state");
 		bus->cmd_err = -EIO;
-		aspeed_i2c_do_stop(bus);
+		if (bus->master_state != ASPEED_I2C_MASTER_STOP)
+			aspeed_i2c_do_stop(bus);
 		goto out_no_complete;
 	}
 	msg = &bus->msgs[bus->msgs_index];
@@ -674,7 +679,7 @@ static const struct i2c_algorithm aspeed_i2c_algo = {
 #endif /* CONFIG_I2C_SLAVE */
 };
 
-static u32 aspeed_i2c_get_clk_reg_val(u32 divisor)
+static u32 aspeed_i2c_get_clk_reg_val(u32 clk_high_low_max, u32 divisor)
 {
 	u32 base_clk, clk_high, clk_low, tmp;
 
@@ -694,16 +699,22 @@ static u32 aspeed_i2c_get_clk_reg_val(u32 divisor)
 	 * Thus,
 	 *	SCL_freq = APB_freq /
 	 *		((1 << base_clk) * (clk_high + 1 + clk_low + 1))
-	 * The documentation recommends clk_high >= 8 and clk_low >= 7 when
-	 * possible; this last constraint gives us the following solution:
+	 * The documentation recommends clk_high >= clk_high_max / 2 and
+	 * clk_low >= clk_low_max / 2 - 1 when possible; this last constraint
+	 * gives us the following solution:
 	 */
-	base_clk = divisor > 33 ? ilog2((divisor - 1) / 32) + 1 : 0;
-	tmp = divisor / (1 << base_clk);
-	clk_high = tmp / 2 + tmp % 2;
-	clk_low = tmp - clk_high;
+	base_clk = divisor > clk_high_low_max ?
+			ilog2((divisor - 1) / clk_high_low_max) + 1 : 0;
+	tmp = (divisor + (1 << base_clk) - 1) >> base_clk;
+	clk_low = tmp / 2;
+	clk_high = tmp - clk_low;
+
+	if (clk_high)
+		clk_high--;
+
+	if (clk_low)
+		clk_low--;
 
-	clk_high -= 1;
-	clk_low -= 1;
 
 	return ((clk_high << ASPEED_I2CD_TIME_SCL_HIGH_SHIFT)
 		& ASPEED_I2CD_TIME_SCL_HIGH_MASK)
@@ -712,13 +723,35 @@ static u32 aspeed_i2c_get_clk_reg_val(u32 divisor)
 			| (base_clk & ASPEED_I2CD_TIME_BASE_DIVISOR_MASK);
 }
 
+static u32 aspeed_i2c_24xx_get_clk_reg_val(u32 divisor)
+{
+	/*
+	 * clk_high and clk_low are each 3 bits wide, so each can hold a max
+	 * value of 8 giving a clk_high_low_max of 16.
+	 */
+	return aspeed_i2c_get_clk_reg_val(16, divisor);
+}
+
+static u32 aspeed_i2c_25xx_get_clk_reg_val(u32 divisor)
+{
+	/*
+	 * clk_high and clk_low are each 4 bits wide, so each can hold a max
+	 * value of 16 giving a clk_high_low_max of 32.
+	 */
+	return aspeed_i2c_get_clk_reg_val(32, divisor);
+}
+
 /* precondition: bus.lock has been acquired. */
 static int aspeed_i2c_init_clk(struct aspeed_i2c_bus *bus)
 {
 	u32 divisor, clk_reg_val;
 
-	divisor = bus->parent_clk_frequency / bus->bus_frequency;
-	clk_reg_val = aspeed_i2c_get_clk_reg_val(divisor);
+	divisor = DIV_ROUND_UP(bus->parent_clk_frequency, bus->bus_frequency);
+	clk_reg_val = readl(bus->base + ASPEED_I2C_AC_TIMING_REG1);
+	clk_reg_val &= (ASPEED_I2CD_TIME_TBUF_MASK |
+			ASPEED_I2CD_TIME_THDSTA_MASK |
+			ASPEED_I2CD_TIME_TACST_MASK);
+	clk_reg_val |= bus->get_clk_reg_val(divisor);
 	writel(clk_reg_val, bus->base + ASPEED_I2C_AC_TIMING_REG1);
 	writel(ASPEED_NO_TIMEOUT_CTRL, bus->base + ASPEED_I2C_AC_TIMING_REG2);
 
@@ -777,8 +810,22 @@ static int aspeed_i2c_reset(struct aspeed_i2c_bus *bus)
 	return ret;
 }
 
+static const struct of_device_id aspeed_i2c_bus_of_table[] = {
+	{
+		.compatible = "aspeed,ast2400-i2c-bus",
+		.data = aspeed_i2c_24xx_get_clk_reg_val,
+	},
+	{
+		.compatible = "aspeed,ast2500-i2c-bus",
+		.data = aspeed_i2c_25xx_get_clk_reg_val,
+	},
+	{ },
+};
+MODULE_DEVICE_TABLE(of, aspeed_i2c_bus_of_table);
+
 static int aspeed_i2c_probe_bus(struct platform_device *pdev)
 {
+	const struct of_device_id *match;
 	struct aspeed_i2c_bus *bus;
 	struct clk *parent_clk;
 	struct resource *res;
@@ -808,6 +855,12 @@ static int aspeed_i2c_probe_bus(struct platform_device *pdev)
 		bus->bus_frequency = 100000;
 	}
 
+	match = of_match_node(aspeed_i2c_bus_of_table, pdev->dev.of_node);
+	if (!match)
+		bus->get_clk_reg_val = aspeed_i2c_24xx_get_clk_reg_val;
+	else
+		bus->get_clk_reg_val = match->data;
+
 	/* Initialize the I2C adapter */
 	spin_lock_init(&bus->lock);
 	init_completion(&bus->cmd_complete);
@@ -869,13 +922,6 @@ static int aspeed_i2c_remove_bus(struct platform_device *pdev)
 	return 0;
 }
 
-static const struct of_device_id aspeed_i2c_bus_of_table[] = {
-	{ .compatible = "aspeed,ast2400-i2c-bus", },
-	{ .compatible = "aspeed,ast2500-i2c-bus", },
-	{ },
-};
-MODULE_DEVICE_TABLE(of, aspeed_i2c_bus_of_table);
-
 static struct platform_driver aspeed_i2c_bus_driver = {
 	.probe		= aspeed_i2c_probe_bus,
 	.remove		= aspeed_i2c_remove_bus,
diff --git a/drivers/i2c/busses/i2c-at91.c b/drivers/i2c/busses/i2c-at91.c
index 38dd61d621df..bfd1fdff64a9 100644
--- a/drivers/i2c/busses/i2c-at91.c
+++ b/drivers/i2c/busses/i2c-at91.c
@@ -809,7 +809,7 @@ out:
  * The hardware can handle at most two messages concatenated by a
  * repeated start via it's internal address feature.
  */
-static struct i2c_adapter_quirks at91_twi_quirks = {
+static const struct i2c_adapter_quirks at91_twi_quirks = {
 	.flags = I2C_AQ_COMB | I2C_AQ_COMB_WRITE_FIRST | I2C_AQ_COMB_SAME_ADDR,
 	.max_comb_1st_msg_len = 3,
 };
diff --git a/drivers/i2c/busses/i2c-bcm-iproc.c b/drivers/i2c/busses/i2c-bcm-iproc.c
index 318df559adc5..4c8c3bc4669c 100644
--- a/drivers/i2c/busses/i2c-bcm-iproc.c
+++ b/drivers/i2c/busses/i2c-bcm-iproc.c
@@ -510,8 +510,7 @@ static int bcm_iproc_i2c_remove(struct platform_device *pdev)
 
 static int bcm_iproc_i2c_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct bcm_iproc_i2c_dev *iproc_i2c = platform_get_drvdata(pdev);
+	struct bcm_iproc_i2c_dev *iproc_i2c = dev_get_drvdata(dev);
 
 	/* make sure there's no pending interrupt when we go into suspend */
 	writel(0, iproc_i2c->base + IE_OFFSET);
@@ -526,8 +525,7 @@ static int bcm_iproc_i2c_suspend(struct device *dev)
 
 static int bcm_iproc_i2c_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct bcm_iproc_i2c_dev *iproc_i2c = platform_get_drvdata(pdev);
+	struct bcm_iproc_i2c_dev *iproc_i2c = dev_get_drvdata(dev);
 	int ret;
 	u32 val;
 
diff --git a/drivers/i2c/busses/i2c-bfin-twi.c b/drivers/i2c/busses/i2c-bfin-twi.c
index 9fe942b8c610..ff3343186a82 100644
--- a/drivers/i2c/busses/i2c-bfin-twi.c
+++ b/drivers/i2c/busses/i2c-bfin-twi.c
@@ -21,7 +21,6 @@
 #include <linux/interrupt.h>
 #include <linux/platform_device.h>
 #include <linux/delay.h>
-#include <linux/i2c/bfin_twi.h>
 
 #include <asm/irq.h>
 #include <asm/portmux.h>
diff --git a/drivers/i2c/busses/i2c-cadence.c b/drivers/i2c/busses/i2c-cadence.c
index 75d80161931f..b13605718291 100644
--- a/drivers/i2c/busses/i2c-cadence.c
+++ b/drivers/i2c/busses/i2c-cadence.c
@@ -826,8 +826,7 @@ static int cdns_i2c_clk_notifier_cb(struct notifier_block *nb, unsigned long
  */
 static int __maybe_unused cdns_i2c_runtime_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct cdns_i2c *xi2c = platform_get_drvdata(pdev);
+	struct cdns_i2c *xi2c = dev_get_drvdata(dev);
 
 	clk_disable(xi2c->clk);
 
@@ -844,8 +843,7 @@ static int __maybe_unused cdns_i2c_runtime_suspend(struct device *dev)
  */
 static int __maybe_unused cdns_i2c_runtime_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct cdns_i2c *xi2c = platform_get_drvdata(pdev);
+	struct cdns_i2c *xi2c = dev_get_drvdata(dev);
 	int ret;
 
 	ret = clk_enable(xi2c->clk);
diff --git a/drivers/i2c/busses/i2c-cht-wc.c b/drivers/i2c/busses/i2c-cht-wc.c
new file mode 100644
index 000000000000..190bbbc7bfee
--- /dev/null
+++ b/drivers/i2c/busses/i2c-cht-wc.c
@@ -0,0 +1,363 @@
+/*
+ * Intel CHT Whiskey Cove PMIC I2C Master driver
+ * Copyright (C) 2017 Hans de Goede <hdegoede@redhat.com>
+ *
+ * Based on various non upstream patches to support the CHT Whiskey Cove PMIC:
+ * Copyright (C) 2011 - 2014 Intel Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version
+ * 2 as published by the Free Software Foundation, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/completion.h>
+#include <linux/delay.h>
+#include <linux/i2c.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/irqdomain.h>
+#include <linux/mfd/intel_soc_pmic.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#define CHT_WC_I2C_CTRL			0x5e24
+#define CHT_WC_I2C_CTRL_WR		BIT(0)
+#define CHT_WC_I2C_CTRL_RD		BIT(1)
+#define CHT_WC_I2C_CLIENT_ADDR		0x5e25
+#define CHT_WC_I2C_REG_OFFSET		0x5e26
+#define CHT_WC_I2C_WRDATA		0x5e27
+#define CHT_WC_I2C_RDDATA		0x5e28
+
+#define CHT_WC_EXTCHGRIRQ		0x6e0a
+#define CHT_WC_EXTCHGRIRQ_CLIENT_IRQ	BIT(0)
+#define CHT_WC_EXTCHGRIRQ_WRITE_IRQ	BIT(1)
+#define CHT_WC_EXTCHGRIRQ_READ_IRQ	BIT(2)
+#define CHT_WC_EXTCHGRIRQ_NACK_IRQ	BIT(3)
+#define CHT_WC_EXTCHGRIRQ_ADAP_IRQMASK	((u8)GENMASK(3, 1))
+#define CHT_WC_EXTCHGRIRQ_MSK		0x6e17
+
+struct cht_wc_i2c_adap {
+	struct i2c_adapter adapter;
+	wait_queue_head_t wait;
+	struct irq_chip irqchip;
+	struct mutex adap_lock;
+	struct mutex irqchip_lock;
+	struct regmap *regmap;
+	struct irq_domain *irq_domain;
+	struct i2c_client *client;
+	int client_irq;
+	u8 irq_mask;
+	u8 old_irq_mask;
+	int read_data;
+	bool io_error;
+	bool done;
+};
+
+static irqreturn_t cht_wc_i2c_adap_thread_handler(int id, void *data)
+{
+	struct cht_wc_i2c_adap *adap = data;
+	int ret, reg;
+
+	mutex_lock(&adap->adap_lock);
+
+	/* Read IRQs */
+	ret = regmap_read(adap->regmap, CHT_WC_EXTCHGRIRQ, &reg);
+	if (ret) {
+		dev_err(&adap->adapter.dev, "Error reading extchgrirq reg\n");
+		mutex_unlock(&adap->adap_lock);
+		return IRQ_NONE;
+	}
+
+	reg &= ~adap->irq_mask;
+
+	/* Reads must be acked after reading the received data. */
+	ret = regmap_read(adap->regmap, CHT_WC_I2C_RDDATA, &adap->read_data);
+	if (ret)
+		adap->io_error = true;
+
+	/*
+	 * Immediately ack IRQs, so that if new IRQs arrives while we're
+	 * handling the previous ones our irq will re-trigger when we're done.
+	 */
+	ret = regmap_write(adap->regmap, CHT_WC_EXTCHGRIRQ, reg);
+	if (ret)
+		dev_err(&adap->adapter.dev, "Error writing extchgrirq reg\n");
+
+	if (reg & CHT_WC_EXTCHGRIRQ_ADAP_IRQMASK) {
+		adap->io_error |= !!(reg & CHT_WC_EXTCHGRIRQ_NACK_IRQ);
+		adap->done = true;
+	}
+
+	mutex_unlock(&adap->adap_lock);
+
+	if (reg & CHT_WC_EXTCHGRIRQ_ADAP_IRQMASK)
+		wake_up(&adap->wait);
+
+	/*
+	 * Do NOT use handle_nested_irq here, the client irq handler will
+	 * likely want to do i2c transfers and the i2c controller uses this
+	 * interrupt handler as well, so running the client irq handler from
+	 * this thread will cause things to lock up.
+	 */
+	if (reg & CHT_WC_EXTCHGRIRQ_CLIENT_IRQ) {
+		/*
+		 * generic_handle_irq expects local IRQs to be disabled
+		 * as normally it is called from interrupt context.
+		 */
+		local_irq_disable();
+		generic_handle_irq(adap->client_irq);
+		local_irq_enable();
+	}
+
+	return IRQ_HANDLED;
+}
+
+static u32 cht_wc_i2c_adap_master_func(struct i2c_adapter *adap)
+{
+	/* This i2c adapter only supports SMBUS byte transfers */
+	return I2C_FUNC_SMBUS_BYTE_DATA;
+}
+
+static int cht_wc_i2c_adap_smbus_xfer(struct i2c_adapter *_adap, u16 addr,
+				      unsigned short flags, char read_write,
+				      u8 command, int size,
+				      union i2c_smbus_data *data)
+{
+	struct cht_wc_i2c_adap *adap = i2c_get_adapdata(_adap);
+	int ret;
+
+	mutex_lock(&adap->adap_lock);
+	adap->io_error = false;
+	adap->done = false;
+	mutex_unlock(&adap->adap_lock);
+
+	ret = regmap_write(adap->regmap, CHT_WC_I2C_CLIENT_ADDR, addr);
+	if (ret)
+		return ret;
+
+	if (read_write == I2C_SMBUS_WRITE) {
+		ret = regmap_write(adap->regmap, CHT_WC_I2C_WRDATA, data->byte);
+		if (ret)
+			return ret;
+	}
+
+	ret = regmap_write(adap->regmap, CHT_WC_I2C_REG_OFFSET, command);
+	if (ret)
+		return ret;
+
+	ret = regmap_write(adap->regmap, CHT_WC_I2C_CTRL,
+			   (read_write == I2C_SMBUS_WRITE) ?
+			   CHT_WC_I2C_CTRL_WR : CHT_WC_I2C_CTRL_RD);
+	if (ret)
+		return ret;
+
+	ret = wait_event_timeout(adap->wait, adap->done, msecs_to_jiffies(30));
+	if (ret == 0) {
+		/*
+		 * The CHT GPIO controller serializes all IRQs, sometimes
+		 * causing significant delays, check status manually.
+		 */
+		cht_wc_i2c_adap_thread_handler(0, adap);
+		if (!adap->done)
+			return -ETIMEDOUT;
+	}
+
+	ret = 0;
+	mutex_lock(&adap->adap_lock);
+	if (adap->io_error)
+		ret = -EIO;
+	else if (read_write == I2C_SMBUS_READ)
+		data->byte = adap->read_data;
+	mutex_unlock(&adap->adap_lock);
+
+	return ret;
+}
+
+static const struct i2c_algorithm cht_wc_i2c_adap_algo = {
+	.functionality = cht_wc_i2c_adap_master_func,
+	.smbus_xfer = cht_wc_i2c_adap_smbus_xfer,
+};
+
+/**** irqchip for the client connected to the extchgr i2c adapter ****/
+static void cht_wc_i2c_irq_lock(struct irq_data *data)
+{
+	struct cht_wc_i2c_adap *adap = irq_data_get_irq_chip_data(data);
+
+	mutex_lock(&adap->irqchip_lock);
+}
+
+static void cht_wc_i2c_irq_sync_unlock(struct irq_data *data)
+{
+	struct cht_wc_i2c_adap *adap = irq_data_get_irq_chip_data(data);
+	int ret;
+
+	if (adap->irq_mask != adap->old_irq_mask) {
+		ret = regmap_write(adap->regmap, CHT_WC_EXTCHGRIRQ_MSK,
+				   adap->irq_mask);
+		if (ret == 0)
+			adap->old_irq_mask = adap->irq_mask;
+		else
+			dev_err(&adap->adapter.dev, "Error writing EXTCHGRIRQ_MSK\n");
+	}
+
+	mutex_unlock(&adap->irqchip_lock);
+}
+
+static void cht_wc_i2c_irq_enable(struct irq_data *data)
+{
+	struct cht_wc_i2c_adap *adap = irq_data_get_irq_chip_data(data);
+
+	adap->irq_mask &= ~CHT_WC_EXTCHGRIRQ_CLIENT_IRQ;
+}
+
+static void cht_wc_i2c_irq_disable(struct irq_data *data)
+{
+	struct cht_wc_i2c_adap *adap = irq_data_get_irq_chip_data(data);
+
+	adap->irq_mask |= CHT_WC_EXTCHGRIRQ_CLIENT_IRQ;
+}
+
+static const struct irq_chip cht_wc_i2c_irq_chip = {
+	.irq_bus_lock		= cht_wc_i2c_irq_lock,
+	.irq_bus_sync_unlock	= cht_wc_i2c_irq_sync_unlock,
+	.irq_disable		= cht_wc_i2c_irq_disable,
+	.irq_enable		= cht_wc_i2c_irq_enable,
+	.name			= "cht_wc_ext_chrg_irq_chip",
+};
+
+static const struct property_entry bq24190_props[] = {
+	PROPERTY_ENTRY_STRING("extcon-name", "cht_wcove_pwrsrc"),
+	PROPERTY_ENTRY_BOOL("omit-battery-class"),
+	PROPERTY_ENTRY_BOOL("disable-reset"),
+	{ }
+};
+
+static int cht_wc_i2c_adap_i2c_probe(struct platform_device *pdev)
+{
+	struct intel_soc_pmic *pmic = dev_get_drvdata(pdev->dev.parent);
+	struct cht_wc_i2c_adap *adap;
+	struct i2c_board_info board_info = {
+		.type = "bq24190",
+		.addr = 0x6b,
+		.properties = bq24190_props,
+	};
+	int ret, reg, irq;
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		dev_err(&pdev->dev, "Error missing irq resource\n");
+		return -EINVAL;
+	}
+
+	adap = devm_kzalloc(&pdev->dev, sizeof(*adap), GFP_KERNEL);
+	if (!adap)
+		return -ENOMEM;
+
+	init_waitqueue_head(&adap->wait);
+	mutex_init(&adap->adap_lock);
+	mutex_init(&adap->irqchip_lock);
+	adap->irqchip = cht_wc_i2c_irq_chip;
+	adap->regmap = pmic->regmap;
+	adap->adapter.owner = THIS_MODULE;
+	adap->adapter.class = I2C_CLASS_HWMON;
+	adap->adapter.algo = &cht_wc_i2c_adap_algo;
+	strlcpy(adap->adapter.name, "PMIC I2C Adapter",
+		sizeof(adap->adapter.name));
+	adap->adapter.dev.parent = &pdev->dev;
+
+	/* Clear and activate i2c-adapter interrupts, disable client IRQ */
+	adap->old_irq_mask = adap->irq_mask = ~CHT_WC_EXTCHGRIRQ_ADAP_IRQMASK;
+
+	ret = regmap_read(adap->regmap, CHT_WC_I2C_RDDATA, &reg);
+	if (ret)
+		return ret;
+
+	ret = regmap_write(adap->regmap, CHT_WC_EXTCHGRIRQ, ~adap->irq_mask);
+	if (ret)
+		return ret;
+
+	ret = regmap_write(adap->regmap, CHT_WC_EXTCHGRIRQ_MSK, adap->irq_mask);
+	if (ret)
+		return ret;
+
+	/* Alloc and register client IRQ */
+	adap->irq_domain = irq_domain_add_linear(pdev->dev.of_node, 1,
+						 &irq_domain_simple_ops, NULL);
+	if (!adap->irq_domain)
+		return -ENOMEM;
+
+	adap->client_irq = irq_create_mapping(adap->irq_domain, 0);
+	if (!adap->client_irq) {
+		ret = -ENOMEM;
+		goto remove_irq_domain;
+	}
+
+	irq_set_chip_data(adap->client_irq, adap);
+	irq_set_chip_and_handler(adap->client_irq, &adap->irqchip,
+				 handle_simple_irq);
+
+	ret = devm_request_threaded_irq(&pdev->dev, irq, NULL,
+					cht_wc_i2c_adap_thread_handler,
+					IRQF_ONESHOT, "PMIC I2C Adapter", adap);
+	if (ret)
+		goto remove_irq_domain;
+
+	i2c_set_adapdata(&adap->adapter, adap);
+	ret = i2c_add_adapter(&adap->adapter);
+	if (ret)
+		goto remove_irq_domain;
+
+	board_info.irq = adap->client_irq;
+	adap->client = i2c_new_device(&adap->adapter, &board_info);
+	if (!adap->client) {
+		ret = -ENOMEM;
+		goto del_adapter;
+	}
+
+	platform_set_drvdata(pdev, adap);
+	return 0;
+
+del_adapter:
+	i2c_del_adapter(&adap->adapter);
+remove_irq_domain:
+	irq_domain_remove(adap->irq_domain);
+	return ret;
+}
+
+static int cht_wc_i2c_adap_i2c_remove(struct platform_device *pdev)
+{
+	struct cht_wc_i2c_adap *adap = platform_get_drvdata(pdev);
+
+	i2c_unregister_device(adap->client);
+	i2c_del_adapter(&adap->adapter);
+	irq_domain_remove(adap->irq_domain);
+
+	return 0;
+}
+
+static struct platform_device_id cht_wc_i2c_adap_id_table[] = {
+	{ .name = "cht_wcove_ext_chgr" },
+	{},
+};
+MODULE_DEVICE_TABLE(platform, cht_wc_i2c_adap_id_table);
+
+static struct platform_driver cht_wc_i2c_adap_driver = {
+	.probe = cht_wc_i2c_adap_i2c_probe,
+	.remove = cht_wc_i2c_adap_i2c_remove,
+	.driver = {
+		.name = "cht_wcove_ext_chgr",
+	},
+	.id_table = cht_wc_i2c_adap_id_table,
+};
+module_platform_driver(cht_wc_i2c_adap_driver);
+
+MODULE_DESCRIPTION("Intel CHT Whiskey Cove PMIC I2C Master driver");
+MODULE_AUTHOR("Hans de Goede <hdegoede@redhat.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/i2c/busses/i2c-cpm.c b/drivers/i2c/busses/i2c-cpm.c
index d89bde2c5da2..8a8ca945561b 100644
--- a/drivers/i2c/busses/i2c-cpm.c
+++ b/drivers/i2c/busses/i2c-cpm.c
@@ -413,7 +413,7 @@ static const struct i2c_algorithm cpm_i2c_algo = {
 };
 
 /* CPM_MAX_READ is also limiting writes according to the code! */
-static struct i2c_adapter_quirks cpm_i2c_quirks = {
+static const struct i2c_adapter_quirks cpm_i2c_quirks = {
 	.max_num_msgs = CPM_MAXBD,
 	.max_read_len = CPM_MAX_READ,
 	.max_write_len = CPM_MAX_READ,
diff --git a/drivers/i2c/busses/i2c-davinci.c b/drivers/i2c/busses/i2c-davinci.c
index 9e7ef5cf5d49..b8c43535f16c 100644
--- a/drivers/i2c/busses/i2c-davinci.c
+++ b/drivers/i2c/busses/i2c-davinci.c
@@ -733,7 +733,7 @@ static inline void i2c_davinci_cpufreq_deregister(struct davinci_i2c_dev *dev)
 }
 #endif
 
-static struct i2c_algorithm i2c_davinci_algo = {
+static const struct i2c_algorithm i2c_davinci_algo = {
 	.master_xfer	= i2c_davinci_xfer,
 	.functionality	= i2c_davinci_func,
 };
@@ -801,7 +801,7 @@ static int davinci_i2c_probe(struct platform_device *pdev)
 
 	dev->clk = devm_clk_get(&pdev->dev, NULL);
 	if (IS_ERR(dev->clk))
-		return -ENODEV;
+		return PTR_ERR(dev->clk);
 	clk_prepare_enable(dev->clk);
 
 	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
@@ -876,8 +876,7 @@ static int davinci_i2c_remove(struct platform_device *pdev)
 #ifdef CONFIG_PM
 static int davinci_i2c_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct davinci_i2c_dev *i2c_dev = platform_get_drvdata(pdev);
+	struct davinci_i2c_dev *i2c_dev = dev_get_drvdata(dev);
 
 	/* put I2C into reset */
 	davinci_i2c_reset_ctrl(i2c_dev, 0);
@@ -888,8 +887,7 @@ static int davinci_i2c_suspend(struct device *dev)
 
 static int davinci_i2c_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct davinci_i2c_dev *i2c_dev = platform_get_drvdata(pdev);
+	struct davinci_i2c_dev *i2c_dev = dev_get_drvdata(dev);
 
 	clk_prepare_enable(i2c_dev->clk);
 	/* take I2C out of reset */
diff --git a/drivers/i2c/busses/i2c-designware-platdrv.c b/drivers/i2c/busses/i2c-designware-platdrv.c
index 143a8fd582b4..0e65b97842b4 100644
--- a/drivers/i2c/busses/i2c-designware-platdrv.c
+++ b/drivers/i2c/busses/i2c-designware-platdrv.c
@@ -198,8 +198,7 @@ static void i2c_dw_configure_slave(struct dw_i2c_dev *dev)
 	dev->functionality = I2C_FUNC_SLAVE | DW_IC_DEFAULT_FUNCTIONALITY;
 
 	dev->slave_cfg = DW_IC_CON_RX_FIFO_FULL_HLD_CTRL |
-			 DW_IC_CON_RESTART_EN | DW_IC_CON_STOP_DET_IFADDRESSED |
-			 DW_IC_CON_SPEED_FAST;
+			 DW_IC_CON_RESTART_EN | DW_IC_CON_STOP_DET_IFADDRESSED;
 
 	dev->mode = DW_IC_SLAVE;
 
@@ -257,7 +256,8 @@ static int dw_i2c_plat_probe(struct platform_device *pdev)
 	struct dw_i2c_dev *dev;
 	u32 acpi_speed, ht = 0;
 	struct resource *mem;
-	int irq, ret;
+	int i, irq, ret;
+	const int supported_speeds[] = { 0, 100000, 400000, 1000000, 3400000 };
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0)
@@ -298,9 +298,16 @@ static int dw_i2c_plat_probe(struct platform_device *pdev)
 	}
 
 	acpi_speed = i2c_acpi_find_bus_speed(&pdev->dev);
-	/* Some broken DSTDs use 1MiHz instead of 1MHz */
-	if (acpi_speed == 1048576)
-		acpi_speed = 1000000;
+	/*
+	 * Some DSTDs use a non standard speed, round down to the lowest
+	 * standard speed.
+	 */
+	for (i = 1; i < ARRAY_SIZE(supported_speeds); i++) {
+		if (acpi_speed < supported_speeds[i])
+			break;
+	}
+	acpi_speed = supported_speeds[i - 1];
+
 	/*
 	 * Find bus speed from the "clock-frequency" device property, ACPI
 	 * or by using fast mode if neither is set.
@@ -430,10 +437,9 @@ static void dw_i2c_plat_complete(struct device *dev)
 #endif
 
 #ifdef CONFIG_PM
-static int dw_i2c_plat_suspend(struct device *dev)
+static int dw_i2c_plat_runtime_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct dw_i2c_dev *i_dev = platform_get_drvdata(pdev);
+	struct dw_i2c_dev *i_dev = dev_get_drvdata(dev);
 
 	i_dev->disable(i_dev);
 	i2c_dw_plat_prepare_clk(i_dev, false);
@@ -443,8 +449,7 @@ static int dw_i2c_plat_suspend(struct device *dev)
 
 static int dw_i2c_plat_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct dw_i2c_dev *i_dev = platform_get_drvdata(pdev);
+	struct dw_i2c_dev *i_dev = dev_get_drvdata(dev);
 
 	i2c_dw_plat_prepare_clk(i_dev, true);
 	i_dev->init(i_dev);
@@ -452,11 +457,21 @@ static int dw_i2c_plat_resume(struct device *dev)
 	return 0;
 }
 
+#ifdef CONFIG_PM_SLEEP
+static int dw_i2c_plat_suspend(struct device *dev)
+{
+	pm_runtime_resume(dev);
+	return dw_i2c_plat_runtime_suspend(dev);
+}
+#endif
+
 static const struct dev_pm_ops dw_i2c_dev_pm_ops = {
 	.prepare = dw_i2c_plat_prepare,
 	.complete = dw_i2c_plat_complete,
 	SET_SYSTEM_SLEEP_PM_OPS(dw_i2c_plat_suspend, dw_i2c_plat_resume)
-	SET_RUNTIME_PM_OPS(dw_i2c_plat_suspend, dw_i2c_plat_resume, NULL)
+	SET_RUNTIME_PM_OPS(dw_i2c_plat_runtime_suspend,
+			   dw_i2c_plat_resume,
+			   NULL)
 };
 
 #define DW_I2C_DEV_PMOPS (&dw_i2c_dev_pm_ops)
diff --git a/drivers/i2c/busses/i2c-designware-slave.c b/drivers/i2c/busses/i2c-designware-slave.c
index 0548c7ea578c..ea9578ab19a1 100644
--- a/drivers/i2c/busses/i2c-designware-slave.c
+++ b/drivers/i2c/busses/i2c-designware-slave.c
@@ -177,6 +177,8 @@ static int i2c_dw_reg_slave(struct i2c_client *slave)
 		return -EBUSY;
 	if (slave->flags & I2C_CLIENT_TEN)
 		return -EAFNOSUPPORT;
+	pm_runtime_get_sync(dev->dev);
+
 	/*
 	 * Set slave address in the IC_SAR register,
 	 * the address to which the DW_apb_i2c responds.
@@ -205,6 +207,7 @@ static int i2c_dw_unreg_slave(struct i2c_client *slave)
 	dev->disable_int(dev);
 	dev->disable(dev);
 	dev->slave = NULL;
+	pm_runtime_put(dev->dev);
 
 	return 0;
 }
@@ -272,7 +275,7 @@ static int i2c_dw_irq_handler_slave(struct dw_i2c_dev *dev)
 	slave_activity = ((dw_readl(dev, DW_IC_STATUS) &
 		DW_IC_STATUS_SLAVE_ACTIVITY) >> 6);
 
-	if (!enabled || !(raw_stat & ~DW_IC_INTR_ACTIVITY))
+	if (!enabled || !(raw_stat & ~DW_IC_INTR_ACTIVITY) || !dev->slave)
 		return 0;
 
 	dev_dbg(dev->dev,
@@ -343,7 +346,7 @@ static irqreturn_t i2c_dw_isr_slave(int this_irq, void *dev_id)
 	return IRQ_RETVAL(ret);
 }
 
-static struct i2c_algorithm i2c_dw_algo = {
+static const struct i2c_algorithm i2c_dw_algo = {
 	.functionality = i2c_dw_func,
 	.reg_slave = i2c_dw_reg_slave,
 	.unreg_slave = i2c_dw_unreg_slave,
@@ -382,7 +385,6 @@ int i2c_dw_probe_slave(struct dw_i2c_dev *dev)
 	ret = i2c_add_numbered_adapter(adap);
 	if (ret)
 		dev_err(dev->dev, "failure adding adapter: %d\n", ret);
-	pm_runtime_put_noidle(dev->dev);
 
 	return ret;
 }
diff --git a/drivers/i2c/busses/i2c-exynos5.c b/drivers/i2c/busses/i2c-exynos5.c
index 23ed4d67ecad..3855e0b11877 100644
--- a/drivers/i2c/busses/i2c-exynos5.c
+++ b/drivers/i2c/busses/i2c-exynos5.c
@@ -803,8 +803,7 @@ static int exynos5_i2c_remove(struct platform_device *pdev)
 #ifdef CONFIG_PM_SLEEP
 static int exynos5_i2c_suspend_noirq(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct exynos5_i2c *i2c = platform_get_drvdata(pdev);
+	struct exynos5_i2c *i2c = dev_get_drvdata(dev);
 
 	i2c->suspended = 1;
 
@@ -815,8 +814,7 @@ static int exynos5_i2c_suspend_noirq(struct device *dev)
 
 static int exynos5_i2c_resume_noirq(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct exynos5_i2c *i2c = platform_get_drvdata(pdev);
+	struct exynos5_i2c *i2c = dev_get_drvdata(dev);
 	int ret = 0;
 
 	ret = clk_prepare_enable(i2c->clk);
diff --git a/drivers/i2c/busses/i2c-gpio.c b/drivers/i2c/busses/i2c-gpio.c
index 34cfc0ebdcb9..0ef8fcc6ac3a 100644
--- a/drivers/i2c/busses/i2c-gpio.c
+++ b/drivers/i2c/busses/i2c-gpio.c
@@ -98,8 +98,8 @@ static int of_i2c_gpio_get_pins(struct device_node *np,
 		return -EPROBE_DEFER;
 
 	if (!gpio_is_valid(*sda_pin) || !gpio_is_valid(*scl_pin)) {
-		pr_err("%s: invalid GPIO pins, sda=%d/scl=%d\n",
-		       np->full_name, *sda_pin, *scl_pin);
+		pr_err("%pOF: invalid GPIO pins, sda=%d/scl=%d\n",
+		       np, *sda_pin, *scl_pin);
 		return -ENODEV;
 	}
 
diff --git a/drivers/i2c/busses/i2c-hix5hd2.c b/drivers/i2c/busses/i2c-hix5hd2.c
index ae7f3180f7e8..bb68957d3da5 100644
--- a/drivers/i2c/busses/i2c-hix5hd2.c
+++ b/drivers/i2c/busses/i2c-hix5hd2.c
@@ -505,8 +505,7 @@ static int hix5hd2_i2c_remove(struct platform_device *pdev)
 #ifdef CONFIG_PM
 static int hix5hd2_i2c_runtime_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct hix5hd2_i2c_priv *priv = platform_get_drvdata(pdev);
+	struct hix5hd2_i2c_priv *priv = dev_get_drvdata(dev);
 
 	clk_disable_unprepare(priv->clk);
 
@@ -515,8 +514,7 @@ static int hix5hd2_i2c_runtime_suspend(struct device *dev)
 
 static int hix5hd2_i2c_runtime_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct hix5hd2_i2c_priv *priv = platform_get_drvdata(pdev);
+	struct hix5hd2_i2c_priv *priv = dev_get_drvdata(dev);
 
 	clk_prepare_enable(priv->clk);
 	hix5hd2_i2c_init(priv);
diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c
index c9536e17d6ff..e114e4e00d29 100644
--- a/drivers/i2c/busses/i2c-i801.c
+++ b/drivers/i2c/busses/i2c-i801.c
@@ -1332,6 +1332,7 @@ static void i801_add_tco(struct i801_priv *priv)
 	u32 tco_base, tco_ctl;
 	u32 base_addr, ctrl_val;
 	u64 base64_addr;
+	u8 hidden;
 
 	if (!(priv->features & FEATURE_TCO))
 		return;
@@ -1376,8 +1377,10 @@ static void i801_add_tco(struct i801_priv *priv)
 
 	devfn = PCI_DEVFN(PCI_SLOT(pci_dev->devfn), 1);
 
-	/* Unhide the P2SB device */
-	pci_bus_write_config_byte(pci_dev->bus, devfn, 0xe1, 0x0);
+	/* Unhide the P2SB device, if it is hidden */
+	pci_bus_read_config_byte(pci_dev->bus, devfn, 0xe1, &hidden);
+	if (hidden)
+		pci_bus_write_config_byte(pci_dev->bus, devfn, 0xe1, 0x0);
 
 	pci_bus_read_config_dword(pci_dev->bus, devfn, SBREG_BAR, &base_addr);
 	base64_addr = base_addr & 0xfffffff0;
@@ -1385,8 +1388,9 @@ static void i801_add_tco(struct i801_priv *priv)
 	pci_bus_read_config_dword(pci_dev->bus, devfn, SBREG_BAR + 0x4, &base_addr);
 	base64_addr |= (u64)base_addr << 32;
 
-	/* Hide the P2SB device */
-	pci_bus_write_config_byte(pci_dev->bus, devfn, 0xe1, 0x1);
+	/* Hide the P2SB device, if it was hidden before */
+	if (hidden)
+		pci_bus_write_config_byte(pci_dev->bus, devfn, 0xe1, hidden);
 	spin_unlock(&p2sb_spinlock);
 
 	res = &tco_res[ICH_RES_MEM_OFF];
diff --git a/drivers/i2c/busses/i2c-ismt.c b/drivers/i2c/busses/i2c-ismt.c
index e98e44e584a4..22ffcb73c185 100644
--- a/drivers/i2c/busses/i2c-ismt.c
+++ b/drivers/i2c/busses/i2c-ismt.c
@@ -341,8 +341,10 @@ static int ismt_process_desc(const struct ismt_desc *desc,
 			break;
 		case I2C_SMBUS_BLOCK_DATA:
 		case I2C_SMBUS_I2C_BLOCK_DATA:
-			memcpy(&data->block[1], dma_buffer, desc->rxbytes);
-			data->block[0] = desc->rxbytes;
+			if (desc->rxbytes != dma_buffer[0] + 1)
+				return -EMSGSIZE;
+
+			memcpy(data->block, dma_buffer, desc->rxbytes);
 			break;
 		}
 		return 0;
diff --git a/drivers/i2c/busses/i2c-kempld.c b/drivers/i2c/busses/i2c-kempld.c
index 25993d2e64bf..e879190b5d1d 100644
--- a/drivers/i2c/busses/i2c-kempld.c
+++ b/drivers/i2c/busses/i2c-kempld.c
@@ -289,7 +289,7 @@ static const struct i2c_algorithm kempld_i2c_algorithm = {
 	.functionality	= kempld_i2c_func,
 };
 
-static struct i2c_adapter kempld_i2c_adapter = {
+static const struct i2c_adapter kempld_i2c_adapter = {
 	.owner		= THIS_MODULE,
 	.name		= "i2c-kempld",
 	.class		= I2C_CLASS_HWMON | I2C_CLASS_SPD,
diff --git a/drivers/i2c/busses/i2c-lpc2k.c b/drivers/i2c/busses/i2c-lpc2k.c
index 9b1fef455a89..59167c018ae7 100644
--- a/drivers/i2c/busses/i2c-lpc2k.c
+++ b/drivers/i2c/busses/i2c-lpc2k.c
@@ -457,8 +457,7 @@ static int i2c_lpc2k_remove(struct platform_device *dev)
 #ifdef CONFIG_PM
 static int i2c_lpc2k_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct lpc2k_i2c *i2c = platform_get_drvdata(pdev);
+	struct lpc2k_i2c *i2c = dev_get_drvdata(dev);
 
 	clk_disable(i2c->clk);
 
@@ -467,8 +466,7 @@ static int i2c_lpc2k_suspend(struct device *dev)
 
 static int i2c_lpc2k_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct lpc2k_i2c *i2c = platform_get_drvdata(pdev);
+	struct lpc2k_i2c *i2c = dev_get_drvdata(dev);
 
 	clk_enable(i2c->clk);
 	i2c_lpc2k_reset(i2c);
diff --git a/drivers/i2c/busses/i2c-mlxcpld.c b/drivers/i2c/busses/i2c-mlxcpld.c
index d271e6a0954c..4c28fa28ce76 100644
--- a/drivers/i2c/busses/i2c-mlxcpld.c
+++ b/drivers/i2c/busses/i2c-mlxcpld.c
@@ -433,7 +433,7 @@ static const struct i2c_algorithm mlxcpld_i2c_algo = {
 	.functionality	= mlxcpld_i2c_func
 };
 
-static struct i2c_adapter_quirks mlxcpld_i2c_quirks = {
+static const struct i2c_adapter_quirks mlxcpld_i2c_quirks = {
 	.flags = I2C_AQ_COMB_WRITE_THEN_READ,
 	.max_read_len = MLXCPLD_I2C_DATA_REG_SZ - MLXCPLD_I2C_MAX_ADDR_LEN,
 	.max_write_len = MLXCPLD_I2C_DATA_REG_SZ,
diff --git a/drivers/i2c/busses/i2c-mt65xx.c b/drivers/i2c/busses/i2c-mt65xx.c
index 45d61714c81b..09d288ce0ddb 100644
--- a/drivers/i2c/busses/i2c-mt65xx.c
+++ b/drivers/i2c/busses/i2c-mt65xx.c
@@ -50,7 +50,6 @@
 #define I2C_FS_START_CON		0x1800
 #define I2C_TIME_CLR_VALUE		0x0000
 #define I2C_TIME_DEFAULT_VALUE		0x0003
-#define I2C_FS_TIME_INIT_VALUE		0x1303
 #define I2C_WRRD_TRANAC_VALUE		0x0002
 #define I2C_RD_TRANAC_VALUE		0x0001
 
@@ -154,6 +153,7 @@ struct mtk_i2c {
 	bool use_push_pull;		/* IO config push-pull mode */
 
 	u16 irq_stat;			/* interrupt status */
+	unsigned int clk_src_div;
 	unsigned int speed_hz;		/* The speed in transfer */
 	enum mtk_trans_op op;
 	u16 timing_reg;
@@ -172,6 +172,10 @@ static const struct i2c_adapter_quirks mt6577_i2c_quirks = {
 	.max_comb_2nd_msg_len = 31,
 };
 
+static const struct i2c_adapter_quirks mt7622_i2c_quirks = {
+	.max_num_msgs = 255,
+};
+
 static const struct mtk_i2c_compatible mt6577_compat = {
 	.quirks = &mt6577_i2c_quirks,
 	.pmic_i2c = 0,
@@ -190,6 +194,15 @@ static const struct mtk_i2c_compatible mt6589_compat = {
 	.support_33bits = 0,
 };
 
+static const struct mtk_i2c_compatible mt7622_compat = {
+	.quirks = &mt7622_i2c_quirks,
+	.pmic_i2c = 0,
+	.dcm = 1,
+	.auto_restart = 1,
+	.aux_len_reg = 1,
+	.support_33bits = 0,
+};
+
 static const struct mtk_i2c_compatible mt8173_compat = {
 	.pmic_i2c = 0,
 	.dcm = 1,
@@ -201,6 +214,7 @@ static const struct mtk_i2c_compatible mt8173_compat = {
 static const struct of_device_id mtk_i2c_of_match[] = {
 	{ .compatible = "mediatek,mt6577-i2c", .data = &mt6577_compat },
 	{ .compatible = "mediatek,mt6589-i2c", .data = &mt6589_compat },
+	{ .compatible = "mediatek,mt7622-i2c", .data = &mt7622_compat },
 	{ .compatible = "mediatek,mt8173-i2c", .data = &mt8173_compat },
 	{}
 };
@@ -285,23 +299,20 @@ static void mtk_i2c_init_hw(struct mtk_i2c *i2c)
  * less than or equal to i2c->speed_hz. The calculation try to get
  * sample_cnt and step_cn
  */
-static int mtk_i2c_set_speed(struct mtk_i2c *i2c, unsigned int parent_clk,
-			     unsigned int clock_div)
+static int mtk_i2c_calculate_speed(struct mtk_i2c *i2c, unsigned int clk_src,
+				   unsigned int target_speed,
+				   unsigned int *timing_step_cnt,
+				   unsigned int *timing_sample_cnt)
 {
-	unsigned int clk_src;
 	unsigned int step_cnt;
 	unsigned int sample_cnt;
 	unsigned int max_step_cnt;
-	unsigned int target_speed;
 	unsigned int base_sample_cnt = MAX_SAMPLE_CNT_DIV;
 	unsigned int base_step_cnt;
 	unsigned int opt_div;
 	unsigned int best_mul;
 	unsigned int cnt_mul;
 
-	clk_src = parent_clk / clock_div;
-	target_speed = i2c->speed_hz;
-
 	if (target_speed > MAX_HS_MODE_SPEED)
 		target_speed = MAX_HS_MODE_SPEED;
 
@@ -347,16 +358,48 @@ static int mtk_i2c_set_speed(struct mtk_i2c *i2c, unsigned int parent_clk,
 		return -EINVAL;
 	}
 
-	step_cnt--;
-	sample_cnt--;
+	*timing_step_cnt = step_cnt - 1;
+	*timing_sample_cnt = sample_cnt - 1;
+
+	return 0;
+}
+
+static int mtk_i2c_set_speed(struct mtk_i2c *i2c, unsigned int parent_clk)
+{
+	unsigned int clk_src;
+	unsigned int step_cnt;
+	unsigned int sample_cnt;
+	unsigned int target_speed;
+	int ret;
+
+	clk_src = parent_clk / i2c->clk_src_div;
+	target_speed = i2c->speed_hz;
 
 	if (target_speed > MAX_FS_MODE_SPEED) {
+		/* Set master code speed register */
+		ret = mtk_i2c_calculate_speed(i2c, clk_src, MAX_FS_MODE_SPEED,
+					      &step_cnt, &sample_cnt);
+		if (ret < 0)
+			return ret;
+
+		i2c->timing_reg = (sample_cnt << 8) | step_cnt;
+
 		/* Set the high speed mode register */
-		i2c->timing_reg = I2C_FS_TIME_INIT_VALUE;
+		ret = mtk_i2c_calculate_speed(i2c, clk_src, target_speed,
+					      &step_cnt, &sample_cnt);
+		if (ret < 0)
+			return ret;
+
 		i2c->high_speed_reg = I2C_TIME_DEFAULT_VALUE |
 			(sample_cnt << 12) | (step_cnt << 8);
 	} else {
-		i2c->timing_reg = (sample_cnt << 8) | (step_cnt << 0);
+		ret = mtk_i2c_calculate_speed(i2c, clk_src, target_speed,
+					      &step_cnt, &sample_cnt);
+		if (ret < 0)
+			return ret;
+
+		i2c->timing_reg = (sample_cnt << 8) | step_cnt;
+
 		/* Disable the high speed transaction */
 		i2c->high_speed_reg = I2C_TIME_CLR_VALUE;
 	}
@@ -647,8 +690,7 @@ static const struct i2c_algorithm mtk_i2c_algorithm = {
 	.functionality = mtk_i2c_functionality,
 };
 
-static int mtk_i2c_parse_dt(struct device_node *np, struct mtk_i2c *i2c,
-			    unsigned int *clk_src_div)
+static int mtk_i2c_parse_dt(struct device_node *np, struct mtk_i2c *i2c)
 {
 	int ret;
 
@@ -656,11 +698,11 @@ static int mtk_i2c_parse_dt(struct device_node *np, struct mtk_i2c *i2c,
 	if (ret < 0)
 		i2c->speed_hz = I2C_DEFAULT_SPEED;
 
-	ret = of_property_read_u32(np, "clock-div", clk_src_div);
+	ret = of_property_read_u32(np, "clock-div", &i2c->clk_src_div);
 	if (ret < 0)
 		return ret;
 
-	if (*clk_src_div == 0)
+	if (i2c->clk_src_div == 0)
 		return -EINVAL;
 
 	i2c->have_pmic = of_property_read_bool(np, "mediatek,have-pmic");
@@ -676,7 +718,6 @@ static int mtk_i2c_probe(struct platform_device *pdev)
 	int ret = 0;
 	struct mtk_i2c *i2c;
 	struct clk *clk;
-	unsigned int clk_src_div;
 	struct resource *res;
 	int irq;
 
@@ -684,7 +725,7 @@ static int mtk_i2c_probe(struct platform_device *pdev)
 	if (!i2c)
 		return -ENOMEM;
 
-	ret = mtk_i2c_parse_dt(pdev->dev.of_node, i2c, &clk_src_div);
+	ret = mtk_i2c_parse_dt(pdev->dev.of_node, i2c);
 	if (ret)
 		return -EINVAL;
 
@@ -745,7 +786,7 @@ static int mtk_i2c_probe(struct platform_device *pdev)
 
 	strlcpy(i2c->adap.name, I2C_DRV_NAME, sizeof(i2c->adap.name));
 
-	ret = mtk_i2c_set_speed(i2c, clk_get_rate(clk), clk_src_div);
+	ret = mtk_i2c_set_speed(i2c, clk_get_rate(clk));
 	if (ret) {
 		dev_err(&pdev->dev, "Failed to set the speed.\n");
 		return -EINVAL;
diff --git a/drivers/i2c/busses/i2c-mv64xxx.c b/drivers/i2c/busses/i2c-mv64xxx.c
index 5c4db65c5019..a832c45276a4 100644
--- a/drivers/i2c/busses/i2c-mv64xxx.c
+++ b/drivers/i2c/busses/i2c-mv64xxx.c
@@ -820,7 +820,7 @@ mv64xxx_of_config(struct mv64xxx_i2c_data *drv_data,
 		goto out;
 	}
 
-	drv_data->rstc = devm_reset_control_get_optional(dev, NULL);
+	drv_data->rstc = devm_reset_control_get_optional_exclusive(dev, NULL);
 	if (IS_ERR(drv_data->rstc)) {
 		rc = PTR_ERR(drv_data->rstc);
 		goto out;
@@ -975,8 +975,7 @@ mv64xxx_i2c_remove(struct platform_device *dev)
 #ifdef CONFIG_PM
 static int mv64xxx_i2c_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct mv64xxx_i2c_data *drv_data = platform_get_drvdata(pdev);
+	struct mv64xxx_i2c_data *drv_data = dev_get_drvdata(dev);
 
 	mv64xxx_i2c_hw_init(drv_data);
 
diff --git a/drivers/i2c/busses/i2c-nomadik.c b/drivers/i2c/busses/i2c-nomadik.c
index da6609d62848..49c7c0c91486 100644
--- a/drivers/i2c/busses/i2c-nomadik.c
+++ b/drivers/i2c/busses/i2c-nomadik.c
@@ -1088,7 +1088,7 @@ static struct i2c_vendor_data vendor_db8500 = {
 	.fifodepth = 32, /* Guessed from TFTR/RFTR = 15 */
 };
 
-static struct amba_id nmk_i2c_ids[] = {
+static const struct amba_id nmk_i2c_ids[] = {
 	{
 		.id	= 0x00180024,
 		.mask	= 0x00ffffff,
diff --git a/drivers/i2c/busses/i2c-ocores.c b/drivers/i2c/busses/i2c-ocores.c
index 34f1889a4073..8c42ca7107b2 100644
--- a/drivers/i2c/busses/i2c-ocores.c
+++ b/drivers/i2c/busses/i2c-ocores.c
@@ -276,7 +276,7 @@ static const struct i2c_algorithm ocores_algorithm = {
 	.functionality = ocores_func,
 };
 
-static struct i2c_adapter ocores_adapter = {
+static const struct i2c_adapter ocores_adapter = {
 	.owner = THIS_MODULE,
 	.name = "i2c-ocores",
 	.class = I2C_CLASS_DEPRECATED,
diff --git a/drivers/i2c/busses/i2c-octeon-platdrv.c b/drivers/i2c/busses/i2c-octeon-platdrv.c
index 917524ce6890..64bda83e65ac 100644
--- a/drivers/i2c/busses/i2c-octeon-platdrv.c
+++ b/drivers/i2c/busses/i2c-octeon-platdrv.c
@@ -126,7 +126,7 @@ static const struct i2c_algorithm octeon_i2c_algo = {
 	.functionality = octeon_i2c_functionality,
 };
 
-static struct i2c_adapter octeon_i2c_ops = {
+static const struct i2c_adapter octeon_i2c_ops = {
 	.owner = THIS_MODULE,
 	.name = "OCTEON adapter",
 	.algo = &octeon_i2c_algo,
diff --git a/drivers/i2c/busses/i2c-opal.c b/drivers/i2c/busses/i2c-opal.c
index 11e2a1fc10e9..0aabb7eca0c5 100644
--- a/drivers/i2c/busses/i2c-opal.c
+++ b/drivers/i2c/busses/i2c-opal.c
@@ -204,7 +204,7 @@ static const struct i2c_algorithm i2c_opal_algo = {
  * For two messages, we basically support simple smbus transactions of a
  * write-then-anything.
  */
-static struct i2c_adapter_quirks i2c_opal_quirks = {
+static const struct i2c_adapter_quirks i2c_opal_quirks = {
 	.flags = I2C_AQ_COMB | I2C_AQ_COMB_WRITE_FIRST | I2C_AQ_COMB_SAME_ADDR,
 	.max_comb_1st_msg_len = 4,
 };
diff --git a/drivers/i2c/busses/i2c-pmcmsp.c b/drivers/i2c/busses/i2c-pmcmsp.c
index 217c78711d65..2aa0e83174c5 100644
--- a/drivers/i2c/busses/i2c-pmcmsp.c
+++ b/drivers/i2c/busses/i2c-pmcmsp.c
@@ -577,7 +577,7 @@ static u32 pmcmsptwi_i2c_func(struct i2c_adapter *adapter)
 		I2C_FUNC_SMBUS_WORD_DATA | I2C_FUNC_SMBUS_PROC_CALL;
 }
 
-static struct i2c_adapter_quirks pmcmsptwi_i2c_quirks = {
+static const struct i2c_adapter_quirks pmcmsptwi_i2c_quirks = {
 	.flags = I2C_AQ_COMB_WRITE_THEN_READ,
 	.max_write_len = MSP_MAX_BYTES_PER_RW,
 	.max_read_len = MSP_MAX_BYTES_PER_RW,
@@ -587,7 +587,7 @@ static struct i2c_adapter_quirks pmcmsptwi_i2c_quirks = {
 
 /* -- Initialization -- */
 
-static struct i2c_algorithm pmcmsptwi_algo = {
+static const struct i2c_algorithm pmcmsptwi_algo = {
 	.master_xfer	= pmcmsptwi_master_xfer,
 	.functionality	= pmcmsptwi_i2c_func,
 };
diff --git a/drivers/i2c/busses/i2c-pnx.c b/drivers/i2c/busses/i2c-pnx.c
index fd5f9d2bf6d9..42d6b3a226f8 100644
--- a/drivers/i2c/busses/i2c-pnx.c
+++ b/drivers/i2c/busses/i2c-pnx.c
@@ -590,7 +590,7 @@ static u32 i2c_pnx_func(struct i2c_adapter *adapter)
 	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
 }
 
-static struct i2c_algorithm pnx_algorithm = {
+static const struct i2c_algorithm pnx_algorithm = {
 	.master_xfer = i2c_pnx_xfer,
 	.functionality = i2c_pnx_func,
 };
diff --git a/drivers/i2c/busses/i2c-powermac.c b/drivers/i2c/busses/i2c-powermac.c
index b0d9dee14a7e..f2a2067525ef 100644
--- a/drivers/i2c/busses/i2c-powermac.c
+++ b/drivers/i2c/busses/i2c-powermac.c
@@ -197,7 +197,7 @@ static const struct i2c_algorithm i2c_powermac_algorithm = {
 	.functionality	= i2c_powermac_func,
 };
 
-static struct i2c_adapter_quirks i2c_powermac_quirks = {
+static const struct i2c_adapter_quirks i2c_powermac_quirks = {
 	.max_num_msgs = 1,
 };
 
@@ -234,7 +234,7 @@ static u32 i2c_powermac_get_addr(struct i2c_adapter *adap,
 	else if (!strcmp(node->name, "deq"))
 		return 0x34;
 
-	dev_warn(&adap->dev, "No i2c address for %s\n", node->full_name);
+	dev_warn(&adap->dev, "No i2c address for %pOF\n", node);
 
 	return 0xffffffff;
 }
@@ -315,8 +315,7 @@ static bool i2c_powermac_get_type(struct i2c_adapter *adap,
 		}
 	}
 
-	dev_err(&adap->dev, "i2c-powermac: modalias failure"
-		" on %s\n", node->full_name);
+	dev_err(&adap->dev, "i2c-powermac: modalias failure on %pOF\n", node);
 	return false;
 }
 
@@ -348,8 +347,7 @@ static void i2c_powermac_register_devices(struct i2c_adapter *adap,
 		if (!pmac_i2c_match_adapter(node, adap))
 			continue;
 
-		dev_dbg(&adap->dev, "i2c-powermac: register %s\n",
-			node->full_name);
+		dev_dbg(&adap->dev, "i2c-powermac: register %pOF\n", node);
 
 		/*
 		 * Keep track of some device existence to handle
@@ -372,7 +370,7 @@ static void i2c_powermac_register_devices(struct i2c_adapter *adap,
 		newdev = i2c_new_device(adap, &info);
 		if (!newdev) {
 			dev_err(&adap->dev, "i2c-powermac: Failure to register"
-				" %s\n", node->full_name);
+				" %pOF\n", node);
 			of_node_put(node);
 			/* We do not dispose of the interrupt mapping on
 			 * purpose. It's not necessary (interrupt cannot be
diff --git a/drivers/i2c/busses/i2c-puv3.c b/drivers/i2c/busses/i2c-puv3.c
index 0c8b1571886d..287088b8c4c8 100644
--- a/drivers/i2c/busses/i2c-puv3.c
+++ b/drivers/i2c/busses/i2c-puv3.c
@@ -175,7 +175,7 @@ static u32 puv3_i2c_func(struct i2c_adapter *adapter)
 	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
 }
 
-static struct i2c_algorithm puv3_i2c_algorithm = {
+static const struct i2c_algorithm puv3_i2c_algorithm = {
 	.master_xfer	= puv3_i2c_xfer,
 	.functionality	= puv3_i2c_func,
 };
diff --git a/drivers/i2c/busses/i2c-pxa.c b/drivers/i2c/busses/i2c-pxa.c
index 6cf333ecc8b8..600d264e080c 100644
--- a/drivers/i2c/busses/i2c-pxa.c
+++ b/drivers/i2c/busses/i2c-pxa.c
@@ -1346,8 +1346,7 @@ static int i2c_pxa_remove(struct platform_device *dev)
 #ifdef CONFIG_PM
 static int i2c_pxa_suspend_noirq(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct pxa_i2c *i2c = platform_get_drvdata(pdev);
+	struct pxa_i2c *i2c = dev_get_drvdata(dev);
 
 	clk_disable(i2c->clk);
 
@@ -1356,8 +1355,7 @@ static int i2c_pxa_suspend_noirq(struct device *dev)
 
 static int i2c_pxa_resume_noirq(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct pxa_i2c *i2c = platform_get_drvdata(pdev);
+	struct pxa_i2c *i2c = dev_get_drvdata(dev);
 
 	clk_enable(i2c->clk);
 	i2c_pxa_reset(i2c);
diff --git a/drivers/i2c/busses/i2c-qup.c b/drivers/i2c/busses/i2c-qup.c
index 1902d8ac9753..08f8e0107642 100644
--- a/drivers/i2c/busses/i2c-qup.c
+++ b/drivers/i2c/busses/i2c-qup.c
@@ -1396,7 +1396,7 @@ static const struct i2c_algorithm qup_i2c_algo_v2 = {
  * the end of the read, the length of the read is specified as one byte
  * which limits the possible read to 256 (QUP_READ_LIMIT) bytes.
  */
-static struct i2c_adapter_quirks qup_i2c_quirks = {
+static const struct i2c_adapter_quirks qup_i2c_quirks = {
 	.max_read_len = QUP_READ_LIMIT,
 };
 
diff --git a/drivers/i2c/busses/i2c-rcar.c b/drivers/i2c/busses/i2c-rcar.c
index 93c1a54981df..15d764afec3b 100644
--- a/drivers/i2c/busses/i2c-rcar.c
+++ b/drivers/i2c/busses/i2c-rcar.c
@@ -625,9 +625,8 @@ static struct dma_chan *rcar_i2c_request_dma_chan(struct device *dev,
 
 	chan = dma_request_chan(dev, chan_name);
 	if (IS_ERR(chan)) {
-		ret = PTR_ERR(chan);
-		dev_dbg(dev, "request_channel failed for %s (%d)\n",
-			chan_name, ret);
+		dev_dbg(dev, "request_channel failed for %s (%ld)\n",
+			chan_name, PTR_ERR(chan));
 		return chan;
 	}
 
diff --git a/drivers/i2c/busses/i2c-rk3x.c b/drivers/i2c/busses/i2c-rk3x.c
index df220666d627..fe234578380a 100644
--- a/drivers/i2c/busses/i2c-rk3x.c
+++ b/drivers/i2c/busses/i2c-rk3x.c
@@ -1131,6 +1131,11 @@ static const struct i2c_algorithm rk3x_i2c_algorithm = {
 	.functionality		= rk3x_i2c_func,
 };
 
+static const struct rk3x_i2c_soc_data rv1108_soc_data = {
+	.grf_offset = -1,
+	.calc_timings = rk3x_i2c_v1_calc_timings,
+};
+
 static const struct rk3x_i2c_soc_data rk3066_soc_data = {
 	.grf_offset = 0x154,
 	.calc_timings = rk3x_i2c_v0_calc_timings,
@@ -1158,6 +1163,10 @@ static const struct rk3x_i2c_soc_data rk3399_soc_data = {
 
 static const struct of_device_id rk3x_i2c_match[] = {
 	{
+		.compatible = "rockchip,rv1108-i2c",
+		.data = (void *)&rv1108_soc_data
+	},
+	{
 		.compatible = "rockchip,rk3066-i2c",
 		.data = (void *)&rk3066_soc_data
 	},
diff --git a/drivers/i2c/busses/i2c-s3c2410.c b/drivers/i2c/busses/i2c-s3c2410.c
index 499af26e736e..5d97510ee48b 100644
--- a/drivers/i2c/busses/i2c-s3c2410.c
+++ b/drivers/i2c/busses/i2c-s3c2410.c
@@ -1246,8 +1246,7 @@ static int s3c24xx_i2c_remove(struct platform_device *pdev)
 #ifdef CONFIG_PM_SLEEP
 static int s3c24xx_i2c_suspend_noirq(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct s3c24xx_i2c *i2c = platform_get_drvdata(pdev);
+	struct s3c24xx_i2c *i2c = dev_get_drvdata(dev);
 
 	i2c->suspended = 1;
 
@@ -1259,8 +1258,7 @@ static int s3c24xx_i2c_suspend_noirq(struct device *dev)
 
 static int s3c24xx_i2c_resume_noirq(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct s3c24xx_i2c *i2c = platform_get_drvdata(pdev);
+	struct s3c24xx_i2c *i2c = dev_get_drvdata(dev);
 	int ret;
 
 	if (!IS_ERR(i2c->sysreg))
diff --git a/drivers/i2c/busses/i2c-sh_mobile.c b/drivers/i2c/busses/i2c-sh_mobile.c
index 2e097d97d258..6f2aaeb7c4fa 100644
--- a/drivers/i2c/busses/i2c-sh_mobile.c
+++ b/drivers/i2c/busses/i2c-sh_mobile.c
@@ -561,8 +561,8 @@ static struct dma_chan *sh_mobile_i2c_request_dma_chan(struct device *dev,
 
 	chan = dma_request_slave_channel_reason(dev, chan_name);
 	if (IS_ERR(chan)) {
-		ret = PTR_ERR(chan);
-		dev_dbg(dev, "request_channel failed for %s (%d)\n", chan_name, ret);
+		dev_dbg(dev, "request_channel failed for %s (%ld)\n", chan_name,
+			PTR_ERR(chan));
 		return chan;
 	}
 
diff --git a/drivers/i2c/busses/i2c-simtec.c b/drivers/i2c/busses/i2c-simtec.c
index b4685bb9b5d7..adca51a99487 100644
--- a/drivers/i2c/busses/i2c-simtec.c
+++ b/drivers/i2c/busses/i2c-simtec.c
@@ -127,8 +127,7 @@ static int simtec_i2c_probe(struct platform_device *dev)
 	iounmap(pd->reg);
 
  err_res:
-	release_resource(pd->ioarea);
-	kfree(pd->ioarea);
+	release_mem_region(pd->ioarea->start, size);
 
  err:
 	kfree(pd);
@@ -142,8 +141,7 @@ static int simtec_i2c_remove(struct platform_device *dev)
 	i2c_del_adapter(&pd->adap);
 
 	iounmap(pd->reg);
-	release_resource(pd->ioarea);
-	kfree(pd->ioarea);
+	release_mem_region(pd->ioarea->start, resource_size(pd->ioarea));
 	kfree(pd);
 
 	return 0;
diff --git a/drivers/i2c/busses/i2c-sirf.c b/drivers/i2c/busses/i2c-sirf.c
index 95e81d0f72b4..2fd8b6d00391 100644
--- a/drivers/i2c/busses/i2c-sirf.c
+++ b/drivers/i2c/busses/i2c-sirf.c
@@ -421,8 +421,7 @@ static int i2c_sirfsoc_remove(struct platform_device *pdev)
 #ifdef CONFIG_PM
 static int i2c_sirfsoc_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct i2c_adapter *adapter = platform_get_drvdata(pdev);
+	struct i2c_adapter *adapter = dev_get_drvdata(dev);
 	struct sirfsoc_i2c *siic = adapter->algo_data;
 
 	clk_enable(siic->clk);
@@ -434,8 +433,7 @@ static int i2c_sirfsoc_suspend(struct device *dev)
 
 static int i2c_sirfsoc_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct i2c_adapter *adapter = platform_get_drvdata(pdev);
+	struct i2c_adapter *adapter = dev_get_drvdata(dev);
 	struct sirfsoc_i2c *siic = adapter->algo_data;
 
 	clk_enable(siic->clk);
diff --git a/drivers/i2c/busses/i2c-sprd.c b/drivers/i2c/busses/i2c-sprd.c
new file mode 100644
index 000000000000..22e08ae1704f
--- /dev/null
+++ b/drivers/i2c/busses/i2c-sprd.c
@@ -0,0 +1,646 @@
+/*
+ * Copyright (C) 2017 Spreadtrum Communications Inc.
+ *
+ * SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+
+#define I2C_CTL			0x00
+#define I2C_ADDR_CFG		0x04
+#define I2C_COUNT		0x08
+#define I2C_RX			0x0c
+#define I2C_TX			0x10
+#define I2C_STATUS		0x14
+#define I2C_HSMODE_CFG		0x18
+#define I2C_VERSION		0x1c
+#define ADDR_DVD0		0x20
+#define ADDR_DVD1		0x24
+#define ADDR_STA0_DVD		0x28
+#define ADDR_RST		0x2c
+
+/* I2C_CTL */
+#define STP_EN			BIT(20)
+#define FIFO_AF_LVL_MASK	GENMASK(19, 16)
+#define FIFO_AF_LVL		16
+#define FIFO_AE_LVL_MASK	GENMASK(15, 12)
+#define FIFO_AE_LVL		12
+#define I2C_DMA_EN		BIT(11)
+#define FULL_INTEN		BIT(10)
+#define EMPTY_INTEN		BIT(9)
+#define I2C_DVD_OPT		BIT(8)
+#define I2C_OUT_OPT		BIT(7)
+#define I2C_TRIM_OPT		BIT(6)
+#define I2C_HS_MODE		BIT(4)
+#define I2C_MODE		BIT(3)
+#define I2C_EN			BIT(2)
+#define I2C_INT_EN		BIT(1)
+#define I2C_START		BIT(0)
+
+/* I2C_STATUS */
+#define SDA_IN			BIT(21)
+#define SCL_IN			BIT(20)
+#define FIFO_FULL		BIT(4)
+#define FIFO_EMPTY		BIT(3)
+#define I2C_INT			BIT(2)
+#define I2C_RX_ACK		BIT(1)
+#define I2C_BUSY		BIT(0)
+
+/* ADDR_RST */
+#define I2C_RST			BIT(0)
+
+#define I2C_FIFO_DEEP		12
+#define I2C_FIFO_FULL_THLD	15
+#define I2C_FIFO_EMPTY_THLD	4
+#define I2C_DATA_STEP		8
+#define I2C_ADDR_DVD0_CALC(high, low)	\
+	((((high) & GENMASK(15, 0)) << 16) | ((low) & GENMASK(15, 0)))
+#define I2C_ADDR_DVD1_CALC(high, low)	\
+	(((high) & GENMASK(31, 16)) | (((low) & GENMASK(31, 16)) >> 16))
+
+/* timeout (ms) for pm runtime autosuspend */
+#define SPRD_I2C_PM_TIMEOUT	1000
+
+/* SPRD i2c data structure */
+struct sprd_i2c {
+	struct i2c_adapter adap;
+	struct device *dev;
+	void __iomem *base;
+	struct i2c_msg *msg;
+	struct clk *clk;
+	u32 src_clk;
+	u32 bus_freq;
+	struct completion complete;
+	u8 *buf;
+	u32 count;
+	int irq;
+	int err;
+};
+
+static void sprd_i2c_set_count(struct sprd_i2c *i2c_dev, u32 count)
+{
+	writel(count, i2c_dev->base + I2C_COUNT);
+}
+
+static void sprd_i2c_send_stop(struct sprd_i2c *i2c_dev, int stop)
+{
+	u32 tmp = readl(i2c_dev->base + I2C_CTL);
+
+	if (stop)
+		writel(tmp & ~STP_EN, i2c_dev->base + I2C_CTL);
+	else
+		writel(tmp | STP_EN, i2c_dev->base + I2C_CTL);
+}
+
+static void sprd_i2c_clear_start(struct sprd_i2c *i2c_dev)
+{
+	u32 tmp = readl(i2c_dev->base + I2C_CTL);
+
+	writel(tmp & ~I2C_START, i2c_dev->base + I2C_CTL);
+}
+
+static void sprd_i2c_clear_ack(struct sprd_i2c *i2c_dev)
+{
+	u32 tmp = readl(i2c_dev->base + I2C_STATUS);
+
+	writel(tmp & ~I2C_RX_ACK, i2c_dev->base + I2C_STATUS);
+}
+
+static void sprd_i2c_clear_irq(struct sprd_i2c *i2c_dev)
+{
+	u32 tmp = readl(i2c_dev->base + I2C_STATUS);
+
+	writel(tmp & ~I2C_INT, i2c_dev->base + I2C_STATUS);
+}
+
+static void sprd_i2c_reset_fifo(struct sprd_i2c *i2c_dev)
+{
+	writel(I2C_RST, i2c_dev->base + ADDR_RST);
+}
+
+static void sprd_i2c_set_devaddr(struct sprd_i2c *i2c_dev, struct i2c_msg *m)
+{
+	writel(m->addr << 1, i2c_dev->base + I2C_ADDR_CFG);
+}
+
+static void sprd_i2c_write_bytes(struct sprd_i2c *i2c_dev, u8 *buf, u32 len)
+{
+	u32 i;
+
+	for (i = 0; i < len; i++)
+		writeb(buf[i], i2c_dev->base + I2C_TX);
+}
+
+static void sprd_i2c_read_bytes(struct sprd_i2c *i2c_dev, u8 *buf, u32 len)
+{
+	u32 i;
+
+	for (i = 0; i < len; i++)
+		buf[i] = readb(i2c_dev->base + I2C_RX);
+}
+
+static void sprd_i2c_set_full_thld(struct sprd_i2c *i2c_dev, u32 full_thld)
+{
+	u32 tmp = readl(i2c_dev->base + I2C_CTL);
+
+	tmp &= ~FIFO_AF_LVL_MASK;
+	tmp |= full_thld << FIFO_AF_LVL;
+	writel(tmp, i2c_dev->base + I2C_CTL);
+};
+
+static void sprd_i2c_set_empty_thld(struct sprd_i2c *i2c_dev, u32 empty_thld)
+{
+	u32 tmp = readl(i2c_dev->base + I2C_CTL);
+
+	tmp &= ~FIFO_AE_LVL_MASK;
+	tmp |= empty_thld << FIFO_AE_LVL;
+	writel(tmp, i2c_dev->base + I2C_CTL);
+};
+
+static void sprd_i2c_set_fifo_full_int(struct sprd_i2c *i2c_dev, int enable)
+{
+	u32 tmp = readl(i2c_dev->base + I2C_CTL);
+
+	if (enable)
+		tmp |= FULL_INTEN;
+	else
+		tmp &= ~FULL_INTEN;
+
+	writel(tmp, i2c_dev->base + I2C_CTL);
+};
+
+static void sprd_i2c_set_fifo_empty_int(struct sprd_i2c *i2c_dev, int enable)
+{
+	u32 tmp = readl(i2c_dev->base + I2C_CTL);
+
+	if (enable)
+		tmp |= EMPTY_INTEN;
+	else
+		tmp &= ~EMPTY_INTEN;
+
+	writel(tmp, i2c_dev->base + I2C_CTL);
+};
+
+static void sprd_i2c_opt_start(struct sprd_i2c *i2c_dev)
+{
+	u32 tmp = readl(i2c_dev->base + I2C_CTL);
+
+	writel(tmp | I2C_START, i2c_dev->base + I2C_CTL);
+}
+
+static void sprd_i2c_opt_mode(struct sprd_i2c *i2c_dev, int rw)
+{
+	u32 cmd = readl(i2c_dev->base + I2C_CTL) & ~I2C_MODE;
+
+	writel(cmd | rw << 3, i2c_dev->base + I2C_CTL);
+}
+
+static void sprd_i2c_data_transfer(struct sprd_i2c *i2c_dev)
+{
+	u32 i2c_count = i2c_dev->count;
+	u32 need_tran = i2c_count <= I2C_FIFO_DEEP ? i2c_count : I2C_FIFO_DEEP;
+	struct i2c_msg *msg = i2c_dev->msg;
+
+	if (msg->flags & I2C_M_RD) {
+		sprd_i2c_read_bytes(i2c_dev, i2c_dev->buf, I2C_FIFO_FULL_THLD);
+		i2c_dev->count -= I2C_FIFO_FULL_THLD;
+		i2c_dev->buf += I2C_FIFO_FULL_THLD;
+
+		/*
+		 * If the read data count is larger than rx fifo full threshold,
+		 * we should enable the rx fifo full interrupt to read data
+		 * again.
+		 */
+		if (i2c_dev->count >= I2C_FIFO_FULL_THLD)
+			sprd_i2c_set_fifo_full_int(i2c_dev, 1);
+	} else {
+		sprd_i2c_write_bytes(i2c_dev, i2c_dev->buf, need_tran);
+		i2c_dev->buf += need_tran;
+		i2c_dev->count -= need_tran;
+
+		/*
+		 * If the write data count is arger than tx fifo depth which
+		 * means we can not write all data in one time, then we should
+		 * enable the tx fifo empty interrupt to write again.
+		 */
+		if (i2c_count > I2C_FIFO_DEEP)
+			sprd_i2c_set_fifo_empty_int(i2c_dev, 1);
+	}
+}
+
+static int sprd_i2c_handle_msg(struct i2c_adapter *i2c_adap,
+			       struct i2c_msg *msg, bool is_last_msg)
+{
+	struct sprd_i2c *i2c_dev = i2c_adap->algo_data;
+
+	i2c_dev->msg = msg;
+	i2c_dev->buf = msg->buf;
+	i2c_dev->count = msg->len;
+
+	reinit_completion(&i2c_dev->complete);
+	sprd_i2c_reset_fifo(i2c_dev);
+	sprd_i2c_set_devaddr(i2c_dev, msg);
+	sprd_i2c_set_count(i2c_dev, msg->len);
+
+	if (msg->flags & I2C_M_RD) {
+		sprd_i2c_opt_mode(i2c_dev, 1);
+		sprd_i2c_send_stop(i2c_dev, 1);
+	} else {
+		sprd_i2c_opt_mode(i2c_dev, 0);
+		sprd_i2c_send_stop(i2c_dev, !!is_last_msg);
+	}
+
+	/*
+	 * We should enable rx fifo full interrupt to get data when receiving
+	 * full data.
+	 */
+	if (msg->flags & I2C_M_RD)
+		sprd_i2c_set_fifo_full_int(i2c_dev, 1);
+	else
+		sprd_i2c_data_transfer(i2c_dev);
+
+	sprd_i2c_opt_start(i2c_dev);
+
+	wait_for_completion(&i2c_dev->complete);
+
+	return i2c_dev->err;
+}
+
+static int sprd_i2c_master_xfer(struct i2c_adapter *i2c_adap,
+				struct i2c_msg *msgs, int num)
+{
+	struct sprd_i2c *i2c_dev = i2c_adap->algo_data;
+	int im, ret;
+
+	ret = pm_runtime_get_sync(i2c_dev->dev);
+	if (ret < 0)
+		return ret;
+
+	for (im = 0; im < num - 1; im++) {
+		ret = sprd_i2c_handle_msg(i2c_adap, &msgs[im], 0);
+		if (ret)
+			goto err_msg;
+	}
+
+	ret = sprd_i2c_handle_msg(i2c_adap, &msgs[im++], 1);
+
+err_msg:
+	pm_runtime_mark_last_busy(i2c_dev->dev);
+	pm_runtime_put_autosuspend(i2c_dev->dev);
+
+	return ret < 0 ? ret : im;
+}
+
+static u32 sprd_i2c_func(struct i2c_adapter *adap)
+{
+	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
+}
+
+static const struct i2c_algorithm sprd_i2c_algo = {
+	.master_xfer = sprd_i2c_master_xfer,
+	.functionality = sprd_i2c_func,
+};
+
+static void sprd_i2c_set_clk(struct sprd_i2c *i2c_dev, u32 freq)
+{
+	u32 apb_clk = i2c_dev->src_clk;
+	/*
+	 * From I2C databook, the prescale calculation formula:
+	 * prescale = freq_i2c / (4 * freq_scl) - 1;
+	 */
+	u32 i2c_dvd = apb_clk / (4 * freq) - 1;
+	/*
+	 * From I2C databook, the high period of SCL clock is recommended as
+	 * 40% (2/5), and the low period of SCL clock is recommended as 60%
+	 * (3/5), then the formula should be:
+	 * high = (prescale * 2 * 2) / 5
+	 * low = (prescale * 2 * 3) / 5
+	 */
+	u32 high = ((i2c_dvd << 1) * 2) / 5;
+	u32 low = ((i2c_dvd << 1) * 3) / 5;
+	u32 div0 = I2C_ADDR_DVD0_CALC(high, low);
+	u32 div1 = I2C_ADDR_DVD1_CALC(high, low);
+
+	writel(div0, i2c_dev->base + ADDR_DVD0);
+	writel(div1, i2c_dev->base + ADDR_DVD1);
+
+	/* Start hold timing = hold time(us) * source clock */
+	if (freq == 400000)
+		writel((6 * apb_clk) / 10000000, i2c_dev->base + ADDR_STA0_DVD);
+	else if (freq == 100000)
+		writel((4 * apb_clk) / 1000000, i2c_dev->base + ADDR_STA0_DVD);
+}
+
+static void sprd_i2c_enable(struct sprd_i2c *i2c_dev)
+{
+	u32 tmp = I2C_DVD_OPT;
+
+	writel(tmp, i2c_dev->base + I2C_CTL);
+
+	sprd_i2c_set_full_thld(i2c_dev, I2C_FIFO_FULL_THLD);
+	sprd_i2c_set_empty_thld(i2c_dev, I2C_FIFO_EMPTY_THLD);
+
+	sprd_i2c_set_clk(i2c_dev, i2c_dev->bus_freq);
+	sprd_i2c_reset_fifo(i2c_dev);
+	sprd_i2c_clear_irq(i2c_dev);
+
+	tmp = readl(i2c_dev->base + I2C_CTL);
+	writel(tmp | I2C_EN | I2C_INT_EN, i2c_dev->base + I2C_CTL);
+}
+
+static irqreturn_t sprd_i2c_isr_thread(int irq, void *dev_id)
+{
+	struct sprd_i2c *i2c_dev = dev_id;
+	struct i2c_msg *msg = i2c_dev->msg;
+	bool ack = !(readl(i2c_dev->base + I2C_STATUS) & I2C_RX_ACK);
+	u32 i2c_count = readl(i2c_dev->base + I2C_COUNT);
+	u32 i2c_tran;
+
+	if (msg->flags & I2C_M_RD)
+		i2c_tran = i2c_dev->count >= I2C_FIFO_FULL_THLD;
+	else
+		i2c_tran = i2c_count;
+
+	/*
+	 * If we got one ACK from slave when writing data, and we did not
+	 * finish this transmission (i2c_tran is not zero), then we should
+	 * continue to write data.
+	 *
+	 * For reading data, ack is always true, if i2c_tran is not 0 which
+	 * means we still need to contine to read data from slave.
+	 */
+	if (i2c_tran && ack) {
+		sprd_i2c_data_transfer(i2c_dev);
+		return IRQ_HANDLED;
+	}
+
+	i2c_dev->err = 0;
+
+	/*
+	 * If we did not get one ACK from slave when writing data, we should
+	 * return -EIO to notify users.
+	 */
+	if (!ack)
+		i2c_dev->err = -EIO;
+	else if (msg->flags & I2C_M_RD && i2c_dev->count)
+		sprd_i2c_read_bytes(i2c_dev, i2c_dev->buf, i2c_dev->count);
+
+	/* Transmission is done and clear ack and start operation */
+	sprd_i2c_clear_ack(i2c_dev);
+	sprd_i2c_clear_start(i2c_dev);
+	complete(&i2c_dev->complete);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t sprd_i2c_isr(int irq, void *dev_id)
+{
+	struct sprd_i2c *i2c_dev = dev_id;
+	struct i2c_msg *msg = i2c_dev->msg;
+	u32 i2c_count = readl(i2c_dev->base + I2C_COUNT);
+	bool ack = !(readl(i2c_dev->base + I2C_STATUS) & I2C_RX_ACK);
+	u32 i2c_tran;
+
+	if (msg->flags & I2C_M_RD)
+		i2c_tran = i2c_dev->count >= I2C_FIFO_FULL_THLD;
+	else
+		i2c_tran = i2c_count;
+
+	/*
+	 * If we did not get one ACK from slave when writing data, then we
+	 * should finish this transmission since we got some errors.
+	 *
+	 * When writing data, if i2c_tran == 0 which means we have writen
+	 * done all data, then we can finish this transmission.
+	 *
+	 * When reading data, if conut < rx fifo full threshold, which
+	 * means we can read all data in one time, then we can finish this
+	 * transmission too.
+	 */
+	if (!i2c_tran || !ack) {
+		sprd_i2c_clear_start(i2c_dev);
+		sprd_i2c_clear_irq(i2c_dev);
+	}
+
+	sprd_i2c_set_fifo_empty_int(i2c_dev, 0);
+	sprd_i2c_set_fifo_full_int(i2c_dev, 0);
+
+	return IRQ_WAKE_THREAD;
+}
+
+static int sprd_i2c_clk_init(struct sprd_i2c *i2c_dev)
+{
+	struct clk *clk_i2c, *clk_parent;
+
+	clk_i2c = devm_clk_get(i2c_dev->dev, "i2c");
+	if (IS_ERR(clk_i2c)) {
+		dev_warn(i2c_dev->dev, "i2c%d can't get the i2c clock\n",
+			 i2c_dev->adap.nr);
+		clk_i2c = NULL;
+	}
+
+	clk_parent = devm_clk_get(i2c_dev->dev, "source");
+	if (IS_ERR(clk_parent)) {
+		dev_warn(i2c_dev->dev, "i2c%d can't get the source clock\n",
+			 i2c_dev->adap.nr);
+		clk_parent = NULL;
+	}
+
+	if (clk_set_parent(clk_i2c, clk_parent))
+		i2c_dev->src_clk = clk_get_rate(clk_i2c);
+	else
+		i2c_dev->src_clk = 26000000;
+
+	dev_dbg(i2c_dev->dev, "i2c%d set source clock is %d\n",
+		i2c_dev->adap.nr, i2c_dev->src_clk);
+
+	i2c_dev->clk = devm_clk_get(i2c_dev->dev, "enable");
+	if (IS_ERR(i2c_dev->clk)) {
+		dev_warn(i2c_dev->dev, "i2c%d can't get the enable clock\n",
+			 i2c_dev->adap.nr);
+		i2c_dev->clk = NULL;
+	}
+
+	return 0;
+}
+
+static int sprd_i2c_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct sprd_i2c *i2c_dev;
+	struct resource *res;
+	u32 prop;
+	int ret;
+
+	pdev->id = of_alias_get_id(dev->of_node, "i2c");
+
+	i2c_dev = devm_kzalloc(dev, sizeof(struct sprd_i2c), GFP_KERNEL);
+	if (!i2c_dev)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	i2c_dev->base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(i2c_dev->base))
+		return PTR_ERR(i2c_dev->base);
+
+	i2c_dev->irq = platform_get_irq(pdev, 0);
+	if (i2c_dev->irq < 0) {
+		dev_err(&pdev->dev, "failed to get irq resource\n");
+		return i2c_dev->irq;
+	}
+
+	i2c_set_adapdata(&i2c_dev->adap, i2c_dev);
+	init_completion(&i2c_dev->complete);
+	snprintf(i2c_dev->adap.name, sizeof(i2c_dev->adap.name),
+		 "%s", "sprd-i2c");
+
+	i2c_dev->bus_freq = 100000;
+	i2c_dev->adap.owner = THIS_MODULE;
+	i2c_dev->dev = dev;
+	i2c_dev->adap.retries = 3;
+	i2c_dev->adap.algo = &sprd_i2c_algo;
+	i2c_dev->adap.algo_data = i2c_dev;
+	i2c_dev->adap.dev.parent = dev;
+	i2c_dev->adap.nr = pdev->id;
+	i2c_dev->adap.dev.of_node = dev->of_node;
+
+	if (!of_property_read_u32(dev->of_node, "clock-frequency", &prop))
+		i2c_dev->bus_freq = prop;
+
+	/* We only support 100k and 400k now, otherwise will return error. */
+	if (i2c_dev->bus_freq != 100000 && i2c_dev->bus_freq != 400000)
+		return -EINVAL;
+
+	sprd_i2c_clk_init(i2c_dev);
+	platform_set_drvdata(pdev, i2c_dev);
+
+	ret = clk_prepare_enable(i2c_dev->clk);
+	if (ret)
+		return ret;
+
+	sprd_i2c_enable(i2c_dev);
+
+	pm_runtime_set_autosuspend_delay(i2c_dev->dev, SPRD_I2C_PM_TIMEOUT);
+	pm_runtime_use_autosuspend(i2c_dev->dev);
+	pm_runtime_set_active(i2c_dev->dev);
+	pm_runtime_enable(i2c_dev->dev);
+
+	ret = pm_runtime_get_sync(i2c_dev->dev);
+	if (ret < 0)
+		goto err_rpm_put;
+
+	ret = devm_request_threaded_irq(dev, i2c_dev->irq,
+		sprd_i2c_isr, sprd_i2c_isr_thread,
+		IRQF_NO_SUSPEND | IRQF_ONESHOT,
+		pdev->name, i2c_dev);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to request irq %d\n", i2c_dev->irq);
+		goto err_rpm_put;
+	}
+
+	ret = i2c_add_numbered_adapter(&i2c_dev->adap);
+	if (ret) {
+		dev_err(&pdev->dev, "add adapter failed\n");
+		goto err_rpm_put;
+	}
+
+	pm_runtime_mark_last_busy(i2c_dev->dev);
+	pm_runtime_put_autosuspend(i2c_dev->dev);
+	return 0;
+
+err_rpm_put:
+	pm_runtime_put_noidle(i2c_dev->dev);
+	pm_runtime_disable(i2c_dev->dev);
+	clk_disable_unprepare(i2c_dev->clk);
+	return ret;
+}
+
+static int sprd_i2c_remove(struct platform_device *pdev)
+{
+	struct sprd_i2c *i2c_dev = platform_get_drvdata(pdev);
+	int ret;
+
+	ret = pm_runtime_get_sync(i2c_dev->dev);
+	if (ret < 0)
+		return ret;
+
+	i2c_del_adapter(&i2c_dev->adap);
+	clk_disable_unprepare(i2c_dev->clk);
+
+	pm_runtime_put_noidle(i2c_dev->dev);
+	pm_runtime_disable(i2c_dev->dev);
+
+	return 0;
+}
+
+static int __maybe_unused sprd_i2c_suspend_noirq(struct device *pdev)
+{
+	return pm_runtime_force_suspend(pdev);
+}
+
+static int __maybe_unused sprd_i2c_resume_noirq(struct device *pdev)
+{
+	return pm_runtime_force_resume(pdev);
+}
+
+static int __maybe_unused sprd_i2c_runtime_suspend(struct device *pdev)
+{
+	struct sprd_i2c *i2c_dev = dev_get_drvdata(pdev);
+
+	clk_disable_unprepare(i2c_dev->clk);
+
+	return 0;
+}
+
+static int __maybe_unused sprd_i2c_runtime_resume(struct device *pdev)
+{
+	struct sprd_i2c *i2c_dev = dev_get_drvdata(pdev);
+	int ret;
+
+	ret = clk_prepare_enable(i2c_dev->clk);
+	if (ret)
+		return ret;
+
+	sprd_i2c_enable(i2c_dev);
+
+	return 0;
+}
+
+static const struct dev_pm_ops sprd_i2c_pm_ops = {
+	SET_RUNTIME_PM_OPS(sprd_i2c_runtime_suspend,
+			   sprd_i2c_runtime_resume, NULL)
+
+	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(sprd_i2c_suspend_noirq,
+				      sprd_i2c_resume_noirq)
+};
+
+static const struct of_device_id sprd_i2c_of_match[] = {
+	{ .compatible = "sprd,sc9860-i2c", },
+};
+
+static struct platform_driver sprd_i2c_driver = {
+	.probe = sprd_i2c_probe,
+	.remove = sprd_i2c_remove,
+	.driver = {
+		   .name = "sprd-i2c",
+		   .of_match_table = sprd_i2c_of_match,
+		   .pm = &sprd_i2c_pm_ops,
+	},
+};
+
+static int sprd_i2c_init(void)
+{
+	return platform_driver_register(&sprd_i2c_driver);
+}
+arch_initcall_sync(sprd_i2c_init);
diff --git a/drivers/i2c/busses/i2c-st.c b/drivers/i2c/busses/i2c-st.c
index 1eb9fa82dcfd..9e62f893958a 100644
--- a/drivers/i2c/busses/i2c-st.c
+++ b/drivers/i2c/busses/i2c-st.c
@@ -745,8 +745,7 @@ static int st_i2c_xfer(struct i2c_adapter *i2c_adap,
 #ifdef CONFIG_PM_SLEEP
 static int st_i2c_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct st_i2c_dev *i2c_dev = platform_get_drvdata(pdev);
+	struct st_i2c_dev *i2c_dev = dev_get_drvdata(dev);
 
 	if (i2c_dev->busy)
 		return -EBUSY;
diff --git a/drivers/i2c/busses/i2c-stm32.h b/drivers/i2c/busses/i2c-stm32.h
new file mode 100644
index 000000000000..dab51761f8c5
--- /dev/null
+++ b/drivers/i2c/busses/i2c-stm32.h
@@ -0,0 +1,20 @@
+/*
+ * i2c-stm32.h
+ *
+ * Copyright (C) M'boumba Cedric Madianga 2017
+ * Author: M'boumba Cedric Madianga <cedric.madianga@gmail.com>
+ *
+ * License terms:  GNU General Public License (GPL), version 2
+ */
+
+#ifndef _I2C_STM32_H
+#define _I2C_STM32_H
+
+enum stm32_i2c_speed {
+	STM32_I2C_SPEED_STANDARD, /* 100 kHz */
+	STM32_I2C_SPEED_FAST, /* 400 kHz */
+	STM32_I2C_SPEED_FAST_PLUS, /* 1 MHz */
+	STM32_I2C_SPEED_END,
+};
+
+#endif /* _I2C_STM32_H */
diff --git a/drivers/i2c/busses/i2c-stm32f4.c b/drivers/i2c/busses/i2c-stm32f4.c
index f9dd7e86b861..4ec108496f15 100644
--- a/drivers/i2c/busses/i2c-stm32f4.c
+++ b/drivers/i2c/busses/i2c-stm32f4.c
@@ -27,6 +27,8 @@
 #include <linux/platform_device.h>
 #include <linux/reset.h>
 
+#include "i2c-stm32.h"
+
 /* STM32F4 I2C offset registers */
 #define STM32F4_I2C_CR1			0x00
 #define STM32F4_I2C_CR2			0x04
@@ -90,12 +92,6 @@
 #define STM32F4_I2C_MAX_FREQ		46U
 #define HZ_TO_MHZ			1000000
 
-enum stm32f4_i2c_speed {
-	STM32F4_I2C_SPEED_STANDARD, /* 100 kHz */
-	STM32F4_I2C_SPEED_FAST, /* 400 kHz */
-	STM32F4_I2C_SPEED_END,
-};
-
 /**
  * struct stm32f4_i2c_msg - client specific data
  * @addr: 8-bit slave addr, including r/w bit
@@ -159,7 +155,7 @@ static int stm32f4_i2c_set_periph_clk_freq(struct stm32f4_i2c_dev *i2c_dev)
 	i2c_dev->parent_rate = clk_get_rate(i2c_dev->clk);
 	freq = DIV_ROUND_UP(i2c_dev->parent_rate, HZ_TO_MHZ);
 
-	if (i2c_dev->speed == STM32F4_I2C_SPEED_STANDARD) {
+	if (i2c_dev->speed == STM32_I2C_SPEED_STANDARD) {
 		/*
 		 * To reach 100 kHz, the parent clk frequency should be between
 		 * a minimum value of 2 MHz and a maximum value of 46 MHz due
@@ -216,7 +212,7 @@ static void stm32f4_i2c_set_rise_time(struct stm32f4_i2c_dev *i2c_dev)
 	 * is not higher than 46 MHz . As a result trise is at most 4 bits wide
 	 * and so fits into the TRISE bits [5:0].
 	 */
-	if (i2c_dev->speed == STM32F4_I2C_SPEED_STANDARD)
+	if (i2c_dev->speed == STM32_I2C_SPEED_STANDARD)
 		trise = freq + 1;
 	else
 		trise = freq * 3 / 10 + 1;
@@ -230,7 +226,7 @@ static void stm32f4_i2c_set_speed_mode(struct stm32f4_i2c_dev *i2c_dev)
 	u32 val;
 	u32 ccr = 0;
 
-	if (i2c_dev->speed == STM32F4_I2C_SPEED_STANDARD) {
+	if (i2c_dev->speed == STM32_I2C_SPEED_STANDARD) {
 		/*
 		 * In standard mode:
 		 * t_scl_high = t_scl_low = CCR * I2C parent clk period
@@ -751,7 +747,7 @@ static u32 stm32f4_i2c_func(struct i2c_adapter *adap)
 	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
 }
 
-static struct i2c_algorithm stm32f4_i2c_algo = {
+static const struct i2c_algorithm stm32f4_i2c_algo = {
 	.master_xfer = stm32f4_i2c_xfer,
 	.functionality = stm32f4_i2c_func,
 };
@@ -798,7 +794,7 @@ static int stm32f4_i2c_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	rst = devm_reset_control_get(&pdev->dev, NULL);
+	rst = devm_reset_control_get_exclusive(&pdev->dev, NULL);
 	if (IS_ERR(rst)) {
 		dev_err(&pdev->dev, "Error: Missing controller reset\n");
 		ret = PTR_ERR(rst);
@@ -808,10 +804,10 @@ static int stm32f4_i2c_probe(struct platform_device *pdev)
 	udelay(2);
 	reset_control_deassert(rst);
 
-	i2c_dev->speed = STM32F4_I2C_SPEED_STANDARD;
+	i2c_dev->speed = STM32_I2C_SPEED_STANDARD;
 	ret = of_property_read_u32(np, "clock-frequency", &clk_rate);
 	if (!ret && clk_rate >= 400000)
-		i2c_dev->speed = STM32F4_I2C_SPEED_FAST;
+		i2c_dev->speed = STM32_I2C_SPEED_FAST;
 
 	i2c_dev->dev = &pdev->dev;
 
diff --git a/drivers/i2c/busses/i2c-stm32f7.c b/drivers/i2c/busses/i2c-stm32f7.c
new file mode 100644
index 000000000000..47c67b0ca896
--- /dev/null
+++ b/drivers/i2c/busses/i2c-stm32f7.c
@@ -0,0 +1,972 @@
+/*
+ * Driver for STMicroelectronics STM32F7 I2C controller
+ *
+ * This I2C controller is described in the STM32F75xxx and STM32F74xxx Soc
+ * reference manual.
+ * Please see below a link to the documentation:
+ * http://www.st.com/resource/en/reference_manual/dm00124865.pdf
+ *
+ * Copyright (C) M'boumba Cedric Madianga 2017
+ * Author: M'boumba Cedric Madianga <cedric.madianga@gmail.com>
+ *
+ * This driver is based on i2c-stm32f4.c
+ *
+ * License terms:  GNU General Public License (GPL), version 2
+ */
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+#include <linux/slab.h>
+
+#include "i2c-stm32.h"
+
+/* STM32F7 I2C registers */
+#define STM32F7_I2C_CR1				0x00
+#define STM32F7_I2C_CR2				0x04
+#define STM32F7_I2C_TIMINGR			0x10
+#define STM32F7_I2C_ISR				0x18
+#define STM32F7_I2C_ICR				0x1C
+#define STM32F7_I2C_RXDR			0x24
+#define STM32F7_I2C_TXDR			0x28
+
+/* STM32F7 I2C control 1 */
+#define STM32F7_I2C_CR1_ANFOFF			BIT(12)
+#define STM32F7_I2C_CR1_ERRIE			BIT(7)
+#define STM32F7_I2C_CR1_TCIE			BIT(6)
+#define STM32F7_I2C_CR1_STOPIE			BIT(5)
+#define STM32F7_I2C_CR1_NACKIE			BIT(4)
+#define STM32F7_I2C_CR1_ADDRIE			BIT(3)
+#define STM32F7_I2C_CR1_RXIE			BIT(2)
+#define STM32F7_I2C_CR1_TXIE			BIT(1)
+#define STM32F7_I2C_CR1_PE			BIT(0)
+#define STM32F7_I2C_ALL_IRQ_MASK		(STM32F7_I2C_CR1_ERRIE \
+						| STM32F7_I2C_CR1_TCIE \
+						| STM32F7_I2C_CR1_STOPIE \
+						| STM32F7_I2C_CR1_NACKIE \
+						| STM32F7_I2C_CR1_RXIE \
+						| STM32F7_I2C_CR1_TXIE)
+
+/* STM32F7 I2C control 2 */
+#define STM32F7_I2C_CR2_RELOAD			BIT(24)
+#define STM32F7_I2C_CR2_NBYTES_MASK		GENMASK(23, 16)
+#define STM32F7_I2C_CR2_NBYTES(n)		(((n) & 0xff) << 16)
+#define STM32F7_I2C_CR2_NACK			BIT(15)
+#define STM32F7_I2C_CR2_STOP			BIT(14)
+#define STM32F7_I2C_CR2_START			BIT(13)
+#define STM32F7_I2C_CR2_RD_WRN			BIT(10)
+#define STM32F7_I2C_CR2_SADD7_MASK		GENMASK(7, 1)
+#define STM32F7_I2C_CR2_SADD7(n)		(((n) & 0x7f) << 1)
+
+/* STM32F7 I2C Interrupt Status */
+#define STM32F7_I2C_ISR_BUSY			BIT(15)
+#define STM32F7_I2C_ISR_ARLO			BIT(9)
+#define STM32F7_I2C_ISR_BERR			BIT(8)
+#define STM32F7_I2C_ISR_TCR			BIT(7)
+#define STM32F7_I2C_ISR_TC			BIT(6)
+#define STM32F7_I2C_ISR_STOPF			BIT(5)
+#define STM32F7_I2C_ISR_NACKF			BIT(4)
+#define STM32F7_I2C_ISR_RXNE			BIT(2)
+#define STM32F7_I2C_ISR_TXIS			BIT(1)
+
+/* STM32F7 I2C Interrupt Clear */
+#define STM32F7_I2C_ICR_ARLOCF			BIT(9)
+#define STM32F7_I2C_ICR_BERRCF			BIT(8)
+#define STM32F7_I2C_ICR_STOPCF			BIT(5)
+#define STM32F7_I2C_ICR_NACKCF			BIT(4)
+
+/* STM32F7 I2C Timing */
+#define STM32F7_I2C_TIMINGR_PRESC(n)		(((n) & 0xf) << 28)
+#define STM32F7_I2C_TIMINGR_SCLDEL(n)		(((n) & 0xf) << 20)
+#define STM32F7_I2C_TIMINGR_SDADEL(n)		(((n) & 0xf) << 16)
+#define STM32F7_I2C_TIMINGR_SCLH(n)		(((n) & 0xff) << 8)
+#define STM32F7_I2C_TIMINGR_SCLL(n)		((n) & 0xff)
+
+#define STM32F7_I2C_MAX_LEN			0xff
+
+#define STM32F7_I2C_DNF_DEFAULT			0
+#define STM32F7_I2C_DNF_MAX			16
+
+#define STM32F7_I2C_ANALOG_FILTER_ENABLE	1
+#define STM32F7_I2C_ANALOG_FILTER_DELAY_MIN	50	/* ns */
+#define STM32F7_I2C_ANALOG_FILTER_DELAY_MAX	260	/* ns */
+
+#define STM32F7_I2C_RISE_TIME_DEFAULT		25	/* ns */
+#define STM32F7_I2C_FALL_TIME_DEFAULT		10	/* ns */
+
+#define STM32F7_PRESC_MAX			BIT(4)
+#define STM32F7_SCLDEL_MAX			BIT(4)
+#define STM32F7_SDADEL_MAX			BIT(4)
+#define STM32F7_SCLH_MAX			BIT(8)
+#define STM32F7_SCLL_MAX			BIT(8)
+
+/**
+ * struct stm32f7_i2c_spec - private i2c specification timing
+ * @rate: I2C bus speed (Hz)
+ * @rate_min: 80% of I2C bus speed (Hz)
+ * @rate_max: 100% of I2C bus speed (Hz)
+ * @fall_max: Max fall time of both SDA and SCL signals (ns)
+ * @rise_max: Max rise time of both SDA and SCL signals (ns)
+ * @hddat_min: Min data hold time (ns)
+ * @vddat_max: Max data valid time (ns)
+ * @sudat_min: Min data setup time (ns)
+ * @l_min: Min low period of the SCL clock (ns)
+ * @h_min: Min high period of the SCL clock (ns)
+ */
+struct stm32f7_i2c_spec {
+	u32 rate;
+	u32 rate_min;
+	u32 rate_max;
+	u32 fall_max;
+	u32 rise_max;
+	u32 hddat_min;
+	u32 vddat_max;
+	u32 sudat_min;
+	u32 l_min;
+	u32 h_min;
+};
+
+/**
+ * struct stm32f7_i2c_setup - private I2C timing setup parameters
+ * @speed: I2C speed mode (standard, Fast Plus)
+ * @speed_freq: I2C speed frequency  (Hz)
+ * @clock_src: I2C clock source frequency (Hz)
+ * @rise_time: Rise time (ns)
+ * @fall_time: Fall time (ns)
+ * @dnf: Digital filter coefficient (0-16)
+ * @analog_filter: Analog filter delay (On/Off)
+ */
+struct stm32f7_i2c_setup {
+	enum stm32_i2c_speed speed;
+	u32 speed_freq;
+	u32 clock_src;
+	u32 rise_time;
+	u32 fall_time;
+	u8 dnf;
+	bool analog_filter;
+};
+
+/**
+ * struct stm32f7_i2c_timings - private I2C output parameters
+ * @prec: Prescaler value
+ * @scldel: Data setup time
+ * @sdadel: Data hold time
+ * @sclh: SCL high period (master mode)
+ * @sclh: SCL low period (master mode)
+ */
+struct stm32f7_i2c_timings {
+	struct list_head node;
+	u8 presc;
+	u8 scldel;
+	u8 sdadel;
+	u8 sclh;
+	u8 scll;
+};
+
+/**
+ * struct stm32f7_i2c_msg - client specific data
+ * @addr: 8-bit slave addr, including r/w bit
+ * @count: number of bytes to be transferred
+ * @buf: data buffer
+ * @result: result of the transfer
+ * @stop: last I2C msg to be sent, i.e. STOP to be generated
+ */
+struct stm32f7_i2c_msg {
+	u8 addr;
+	u32 count;
+	u8 *buf;
+	int result;
+	bool stop;
+};
+
+/**
+ * struct stm32f7_i2c_dev - private data of the controller
+ * @adap: I2C adapter for this controller
+ * @dev: device for this controller
+ * @base: virtual memory area
+ * @complete: completion of I2C message
+ * @clk: hw i2c clock
+ * @speed: I2C clock frequency of the controller. Standard, Fast or Fast+
+ * @msg: Pointer to data to be written
+ * @msg_num: number of I2C messages to be executed
+ * @msg_id: message identifiant
+ * @f7_msg: customized i2c msg for driver usage
+ * @setup: I2C timing input setup
+ * @timing: I2C computed timings
+ */
+struct stm32f7_i2c_dev {
+	struct i2c_adapter adap;
+	struct device *dev;
+	void __iomem *base;
+	struct completion complete;
+	struct clk *clk;
+	int speed;
+	struct i2c_msg *msg;
+	unsigned int msg_num;
+	unsigned int msg_id;
+	struct stm32f7_i2c_msg f7_msg;
+	struct stm32f7_i2c_setup *setup;
+	struct stm32f7_i2c_timings timing;
+};
+
+/**
+ * All these values are coming from I2C Specification, Version 6.0, 4th of
+ * April 2014.
+ *
+ * Table10. Characteristics of the SDA and SCL bus lines for Standard, Fast,
+ * and Fast-mode Plus I2C-bus devices
+ */
+static struct stm32f7_i2c_spec i2c_specs[] = {
+	[STM32_I2C_SPEED_STANDARD] = {
+		.rate = 100000,
+		.rate_min = 80000,
+		.rate_max = 100000,
+		.fall_max = 300,
+		.rise_max = 1000,
+		.hddat_min = 0,
+		.vddat_max = 3450,
+		.sudat_min = 250,
+		.l_min = 4700,
+		.h_min = 4000,
+	},
+	[STM32_I2C_SPEED_FAST] = {
+		.rate = 400000,
+		.rate_min = 320000,
+		.rate_max = 400000,
+		.fall_max = 300,
+		.rise_max = 300,
+		.hddat_min = 0,
+		.vddat_max = 900,
+		.sudat_min = 100,
+		.l_min = 1300,
+		.h_min = 600,
+	},
+	[STM32_I2C_SPEED_FAST_PLUS] = {
+		.rate = 1000000,
+		.rate_min = 800000,
+		.rate_max = 1000000,
+		.fall_max = 100,
+		.rise_max = 120,
+		.hddat_min = 0,
+		.vddat_max = 450,
+		.sudat_min = 50,
+		.l_min = 500,
+		.h_min = 260,
+	},
+};
+
+struct stm32f7_i2c_setup stm32f7_setup = {
+	.rise_time = STM32F7_I2C_RISE_TIME_DEFAULT,
+	.fall_time = STM32F7_I2C_FALL_TIME_DEFAULT,
+	.dnf = STM32F7_I2C_DNF_DEFAULT,
+	.analog_filter = STM32F7_I2C_ANALOG_FILTER_ENABLE,
+};
+
+static inline void stm32f7_i2c_set_bits(void __iomem *reg, u32 mask)
+{
+	writel_relaxed(readl_relaxed(reg) | mask, reg);
+}
+
+static inline void stm32f7_i2c_clr_bits(void __iomem *reg, u32 mask)
+{
+	writel_relaxed(readl_relaxed(reg) & ~mask, reg);
+}
+
+static int stm32f7_i2c_compute_timing(struct stm32f7_i2c_dev *i2c_dev,
+				      struct stm32f7_i2c_setup *setup,
+				      struct stm32f7_i2c_timings *output)
+{
+	u32 p_prev = STM32F7_PRESC_MAX;
+	u32 i2cclk = DIV_ROUND_CLOSEST(NSEC_PER_SEC,
+				       setup->clock_src);
+	u32 i2cbus = DIV_ROUND_CLOSEST(NSEC_PER_SEC,
+				       setup->speed_freq);
+	u32 clk_error_prev = i2cbus;
+	u32 tsync;
+	u32 af_delay_min, af_delay_max;
+	u32 dnf_delay;
+	u32 clk_min, clk_max;
+	int sdadel_min, sdadel_max;
+	int scldel_min;
+	struct stm32f7_i2c_timings *v, *_v, *s;
+	struct list_head solutions;
+	u16 p, l, a, h;
+	int ret = 0;
+
+	if (setup->speed >= STM32_I2C_SPEED_END) {
+		dev_err(i2c_dev->dev, "speed out of bound {%d/%d}\n",
+			setup->speed, STM32_I2C_SPEED_END - 1);
+		return -EINVAL;
+	}
+
+	if ((setup->rise_time > i2c_specs[setup->speed].rise_max) ||
+	    (setup->fall_time > i2c_specs[setup->speed].fall_max)) {
+		dev_err(i2c_dev->dev,
+			"timings out of bound Rise{%d>%d}/Fall{%d>%d}\n",
+			setup->rise_time, i2c_specs[setup->speed].rise_max,
+			setup->fall_time, i2c_specs[setup->speed].fall_max);
+		return -EINVAL;
+	}
+
+	if (setup->dnf > STM32F7_I2C_DNF_MAX) {
+		dev_err(i2c_dev->dev,
+			"DNF out of bound %d/%d\n",
+			setup->dnf, STM32F7_I2C_DNF_MAX);
+		return -EINVAL;
+	}
+
+	if (setup->speed_freq > i2c_specs[setup->speed].rate) {
+		dev_err(i2c_dev->dev, "ERROR: Freq {%d/%d}\n",
+			setup->speed_freq, i2c_specs[setup->speed].rate);
+		return -EINVAL;
+	}
+
+	/*  Analog and Digital Filters */
+	af_delay_min =
+		(setup->analog_filter ?
+		 STM32F7_I2C_ANALOG_FILTER_DELAY_MIN : 0);
+	af_delay_max =
+		(setup->analog_filter ?
+		 STM32F7_I2C_ANALOG_FILTER_DELAY_MAX : 0);
+	dnf_delay = setup->dnf * i2cclk;
+
+	sdadel_min = setup->fall_time - i2c_specs[setup->speed].hddat_min -
+		af_delay_min - (setup->dnf + 3) * i2cclk;
+
+	sdadel_max = i2c_specs[setup->speed].vddat_max - setup->rise_time -
+		af_delay_max - (setup->dnf + 4) * i2cclk;
+
+	scldel_min = setup->rise_time + i2c_specs[setup->speed].sudat_min;
+
+	if (sdadel_min < 0)
+		sdadel_min = 0;
+	if (sdadel_max < 0)
+		sdadel_max = 0;
+
+	dev_dbg(i2c_dev->dev, "SDADEL(min/max): %i/%i, SCLDEL(Min): %i\n",
+		sdadel_min, sdadel_max, scldel_min);
+
+	INIT_LIST_HEAD(&solutions);
+	/* Compute possible values for PRESC, SCLDEL and SDADEL */
+	for (p = 0; p < STM32F7_PRESC_MAX; p++) {
+		for (l = 0; l < STM32F7_SCLDEL_MAX; l++) {
+			u32 scldel = (l + 1) * (p + 1) * i2cclk;
+
+			if (scldel < scldel_min)
+				continue;
+
+			for (a = 0; a < STM32F7_SDADEL_MAX; a++) {
+				u32 sdadel = (a * (p + 1) + 1) * i2cclk;
+
+				if (((sdadel >= sdadel_min) &&
+				     (sdadel <= sdadel_max)) &&
+				    (p != p_prev)) {
+					v = kmalloc(sizeof(*v), GFP_KERNEL);
+					if (!v) {
+						ret = -ENOMEM;
+						goto exit;
+					}
+
+					v->presc = p;
+					v->scldel = l;
+					v->sdadel = a;
+					p_prev = p;
+
+					list_add_tail(&v->node,
+						      &solutions);
+				}
+			}
+		}
+	}
+
+	if (list_empty(&solutions)) {
+		dev_err(i2c_dev->dev, "no Prescaler solution\n");
+		ret = -EPERM;
+		goto exit;
+	}
+
+	tsync = af_delay_min + dnf_delay + (2 * i2cclk);
+	s = NULL;
+	clk_max = NSEC_PER_SEC / i2c_specs[setup->speed].rate_min;
+	clk_min = NSEC_PER_SEC / i2c_specs[setup->speed].rate_max;
+
+	/*
+	 * Among Prescaler possibilities discovered above figures out SCL Low
+	 * and High Period. Provided:
+	 * - SCL Low Period has to be higher than SCL Clock Low Period
+	 *   defined by I2C Specification. I2C Clock has to be lower than
+	 *   (SCL Low Period - Analog/Digital filters) / 4.
+	 * - SCL High Period has to be lower than SCL Clock High Period
+	 *   defined by I2C Specification
+	 * - I2C Clock has to be lower than SCL High Period
+	 */
+	list_for_each_entry(v, &solutions, node) {
+		u32 prescaler = (v->presc + 1) * i2cclk;
+
+		for (l = 0; l < STM32F7_SCLL_MAX; l++) {
+			u32 tscl_l = (l + 1) * prescaler + tsync;
+
+			if ((tscl_l < i2c_specs[setup->speed].l_min) ||
+			    (i2cclk >=
+			     ((tscl_l - af_delay_min - dnf_delay) / 4))) {
+				continue;
+			}
+
+			for (h = 0; h < STM32F7_SCLH_MAX; h++) {
+				u32 tscl_h = (h + 1) * prescaler + tsync;
+				u32 tscl = tscl_l + tscl_h +
+					setup->rise_time + setup->fall_time;
+
+				if ((tscl >= clk_min) && (tscl <= clk_max) &&
+				    (tscl_h >= i2c_specs[setup->speed].h_min) &&
+				    (i2cclk < tscl_h)) {
+					int clk_error = tscl - i2cbus;
+
+					if (clk_error < 0)
+						clk_error = -clk_error;
+
+					if (clk_error < clk_error_prev) {
+						clk_error_prev = clk_error;
+						v->scll = l;
+						v->sclh = h;
+						s = v;
+					}
+				}
+			}
+		}
+	}
+
+	if (!s) {
+		dev_err(i2c_dev->dev, "no solution at all\n");
+		ret = -EPERM;
+		goto exit;
+	}
+
+	output->presc = s->presc;
+	output->scldel = s->scldel;
+	output->sdadel = s->sdadel;
+	output->scll = s->scll;
+	output->sclh = s->sclh;
+
+	dev_dbg(i2c_dev->dev,
+		"Presc: %i, scldel: %i, sdadel: %i, scll: %i, sclh: %i\n",
+		output->presc,
+		output->scldel, output->sdadel,
+		output->scll, output->sclh);
+
+exit:
+	/* Release list and memory */
+	list_for_each_entry_safe(v, _v, &solutions, node) {
+		list_del(&v->node);
+		kfree(v);
+	}
+
+	return ret;
+}
+
+static int stm32f7_i2c_setup_timing(struct stm32f7_i2c_dev *i2c_dev,
+				    struct stm32f7_i2c_setup *setup)
+{
+	int ret = 0;
+
+	setup->speed = i2c_dev->speed;
+	setup->speed_freq = i2c_specs[setup->speed].rate;
+	setup->clock_src = clk_get_rate(i2c_dev->clk);
+
+	if (!setup->clock_src) {
+		dev_err(i2c_dev->dev, "clock rate is 0\n");
+		return -EINVAL;
+	}
+
+	do {
+		ret = stm32f7_i2c_compute_timing(i2c_dev, setup,
+						 &i2c_dev->timing);
+		if (ret) {
+			dev_err(i2c_dev->dev,
+				"failed to compute I2C timings.\n");
+			if (i2c_dev->speed > STM32_I2C_SPEED_STANDARD) {
+				i2c_dev->speed--;
+				setup->speed = i2c_dev->speed;
+				setup->speed_freq =
+					i2c_specs[setup->speed].rate;
+				dev_warn(i2c_dev->dev,
+					 "downgrade I2C Speed Freq to (%i)\n",
+					 i2c_specs[setup->speed].rate);
+			} else {
+				break;
+			}
+		}
+	} while (ret);
+
+	if (ret) {
+		dev_err(i2c_dev->dev, "Impossible to compute I2C timings.\n");
+		return ret;
+	}
+
+	dev_dbg(i2c_dev->dev, "I2C Speed(%i), Freq(%i), Clk Source(%i)\n",
+		setup->speed, setup->speed_freq, setup->clock_src);
+	dev_dbg(i2c_dev->dev, "I2C Rise(%i) and Fall(%i) Time\n",
+		setup->rise_time, setup->fall_time);
+	dev_dbg(i2c_dev->dev, "I2C Analog Filter(%s), DNF(%i)\n",
+		(setup->analog_filter ? "On" : "Off"), setup->dnf);
+
+	return 0;
+}
+
+static void stm32f7_i2c_hw_config(struct stm32f7_i2c_dev *i2c_dev)
+{
+	struct stm32f7_i2c_timings *t = &i2c_dev->timing;
+	u32 timing = 0;
+
+	/* Timing settings */
+	timing |= STM32F7_I2C_TIMINGR_PRESC(t->presc);
+	timing |= STM32F7_I2C_TIMINGR_SCLDEL(t->scldel);
+	timing |= STM32F7_I2C_TIMINGR_SDADEL(t->sdadel);
+	timing |= STM32F7_I2C_TIMINGR_SCLH(t->sclh);
+	timing |= STM32F7_I2C_TIMINGR_SCLL(t->scll);
+	writel_relaxed(timing, i2c_dev->base + STM32F7_I2C_TIMINGR);
+
+	/* Enable I2C */
+	if (i2c_dev->setup->analog_filter)
+		stm32f7_i2c_clr_bits(i2c_dev->base + STM32F7_I2C_CR1,
+				     STM32F7_I2C_CR1_ANFOFF);
+	else
+		stm32f7_i2c_set_bits(i2c_dev->base + STM32F7_I2C_CR1,
+				     STM32F7_I2C_CR1_ANFOFF);
+	stm32f7_i2c_set_bits(i2c_dev->base + STM32F7_I2C_CR1,
+			     STM32F7_I2C_CR1_PE);
+}
+
+static void stm32f7_i2c_write_tx_data(struct stm32f7_i2c_dev *i2c_dev)
+{
+	struct stm32f7_i2c_msg *f7_msg = &i2c_dev->f7_msg;
+	void __iomem *base = i2c_dev->base;
+
+	if (f7_msg->count) {
+		writeb_relaxed(*f7_msg->buf++, base + STM32F7_I2C_TXDR);
+		f7_msg->count--;
+	}
+}
+
+static void stm32f7_i2c_read_rx_data(struct stm32f7_i2c_dev *i2c_dev)
+{
+	struct stm32f7_i2c_msg *f7_msg = &i2c_dev->f7_msg;
+	void __iomem *base = i2c_dev->base;
+
+	if (f7_msg->count) {
+		*f7_msg->buf++ = readb_relaxed(base + STM32F7_I2C_RXDR);
+		f7_msg->count--;
+	}
+}
+
+static void stm32f7_i2c_reload(struct stm32f7_i2c_dev *i2c_dev)
+{
+	struct stm32f7_i2c_msg *f7_msg = &i2c_dev->f7_msg;
+	u32 cr2;
+
+	cr2 = readl_relaxed(i2c_dev->base + STM32F7_I2C_CR2);
+
+	cr2 &= ~STM32F7_I2C_CR2_NBYTES_MASK;
+	if (f7_msg->count > STM32F7_I2C_MAX_LEN) {
+		cr2 |= STM32F7_I2C_CR2_NBYTES(STM32F7_I2C_MAX_LEN);
+	} else {
+		cr2 &= ~STM32F7_I2C_CR2_RELOAD;
+		cr2 |= STM32F7_I2C_CR2_NBYTES(f7_msg->count);
+	}
+
+	writel_relaxed(cr2, i2c_dev->base + STM32F7_I2C_CR2);
+}
+
+static int stm32f7_i2c_wait_free_bus(struct stm32f7_i2c_dev *i2c_dev)
+{
+	u32 status;
+	int ret;
+
+	ret = readl_relaxed_poll_timeout(i2c_dev->base + STM32F7_I2C_ISR,
+					 status,
+					 !(status & STM32F7_I2C_ISR_BUSY),
+					 10, 1000);
+	if (ret) {
+		dev_dbg(i2c_dev->dev, "bus busy\n");
+		ret = -EBUSY;
+	}
+
+	return ret;
+}
+
+static void stm32f7_i2c_xfer_msg(struct stm32f7_i2c_dev *i2c_dev,
+				 struct i2c_msg *msg)
+{
+	struct stm32f7_i2c_msg *f7_msg = &i2c_dev->f7_msg;
+	void __iomem *base = i2c_dev->base;
+	u32 cr1, cr2;
+
+	f7_msg->addr = msg->addr;
+	f7_msg->buf = msg->buf;
+	f7_msg->count = msg->len;
+	f7_msg->result = 0;
+	f7_msg->stop = (i2c_dev->msg_id >= i2c_dev->msg_num - 1);
+
+	reinit_completion(&i2c_dev->complete);
+
+	cr1 = readl_relaxed(base + STM32F7_I2C_CR1);
+	cr2 = readl_relaxed(base + STM32F7_I2C_CR2);
+
+	/* Set transfer direction */
+	cr2 &= ~STM32F7_I2C_CR2_RD_WRN;
+	if (msg->flags & I2C_M_RD)
+		cr2 |= STM32F7_I2C_CR2_RD_WRN;
+
+	/* Set slave address */
+	cr2 &= ~STM32F7_I2C_CR2_SADD7_MASK;
+	cr2 |= STM32F7_I2C_CR2_SADD7(f7_msg->addr);
+
+	/* Set nb bytes to transfer and reload if needed */
+	cr2 &= ~(STM32F7_I2C_CR2_NBYTES_MASK | STM32F7_I2C_CR2_RELOAD);
+	if (f7_msg->count > STM32F7_I2C_MAX_LEN) {
+		cr2 |= STM32F7_I2C_CR2_NBYTES(STM32F7_I2C_MAX_LEN);
+		cr2 |= STM32F7_I2C_CR2_RELOAD;
+	} else {
+		cr2 |= STM32F7_I2C_CR2_NBYTES(f7_msg->count);
+	}
+
+	/* Enable NACK, STOP, error and transfer complete interrupts */
+	cr1 |= STM32F7_I2C_CR1_ERRIE | STM32F7_I2C_CR1_TCIE |
+		STM32F7_I2C_CR1_STOPIE | STM32F7_I2C_CR1_NACKIE;
+
+	/* Clear TX/RX interrupt */
+	cr1 &= ~(STM32F7_I2C_CR1_RXIE | STM32F7_I2C_CR1_TXIE);
+
+	/* Enable RX/TX interrupt according to msg direction */
+	if (msg->flags & I2C_M_RD)
+		cr1 |= STM32F7_I2C_CR1_RXIE;
+	else
+		cr1 |= STM32F7_I2C_CR1_TXIE;
+
+	/* Configure Start/Repeated Start */
+	cr2 |= STM32F7_I2C_CR2_START;
+
+	/* Write configurations registers */
+	writel_relaxed(cr1, base + STM32F7_I2C_CR1);
+	writel_relaxed(cr2, base + STM32F7_I2C_CR2);
+}
+
+static void stm32f7_i2c_disable_irq(struct stm32f7_i2c_dev *i2c_dev, u32 mask)
+{
+	stm32f7_i2c_clr_bits(i2c_dev->base + STM32F7_I2C_CR1, mask);
+}
+
+static irqreturn_t stm32f7_i2c_isr_event(int irq, void *data)
+{
+	struct stm32f7_i2c_dev *i2c_dev = data;
+	struct stm32f7_i2c_msg *f7_msg = &i2c_dev->f7_msg;
+	void __iomem *base = i2c_dev->base;
+	u32 status, mask;
+
+	status = readl_relaxed(i2c_dev->base + STM32F7_I2C_ISR);
+
+	/* Tx empty */
+	if (status & STM32F7_I2C_ISR_TXIS)
+		stm32f7_i2c_write_tx_data(i2c_dev);
+
+	/* RX not empty */
+	if (status & STM32F7_I2C_ISR_RXNE)
+		stm32f7_i2c_read_rx_data(i2c_dev);
+
+	/* NACK received */
+	if (status & STM32F7_I2C_ISR_NACKF) {
+		dev_dbg(i2c_dev->dev, "<%s>: Receive NACK\n", __func__);
+		writel_relaxed(STM32F7_I2C_ICR_NACKCF, base + STM32F7_I2C_ICR);
+		f7_msg->result = -ENXIO;
+	}
+
+	/* STOP detection flag */
+	if (status & STM32F7_I2C_ISR_STOPF) {
+		/* Disable interrupts */
+		stm32f7_i2c_disable_irq(i2c_dev, STM32F7_I2C_ALL_IRQ_MASK);
+
+		/* Clear STOP flag */
+		writel_relaxed(STM32F7_I2C_ICR_STOPCF, base + STM32F7_I2C_ICR);
+
+		complete(&i2c_dev->complete);
+	}
+
+	/* Transfer complete */
+	if (status & STM32F7_I2C_ISR_TC) {
+		if (f7_msg->stop) {
+			mask = STM32F7_I2C_CR2_STOP;
+			stm32f7_i2c_set_bits(base + STM32F7_I2C_CR2, mask);
+		} else {
+			i2c_dev->msg_id++;
+			i2c_dev->msg++;
+			stm32f7_i2c_xfer_msg(i2c_dev, i2c_dev->msg);
+		}
+	}
+
+	/*
+	 * Transfer Complete Reload: 255 data bytes have been transferred
+	 * We have to prepare the I2C controller to transfer the remaining
+	 * data.
+	 */
+	if (status & STM32F7_I2C_ISR_TCR)
+		stm32f7_i2c_reload(i2c_dev);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t stm32f7_i2c_isr_error(int irq, void *data)
+{
+	struct stm32f7_i2c_dev *i2c_dev = data;
+	struct stm32f7_i2c_msg *f7_msg = &i2c_dev->f7_msg;
+	void __iomem *base = i2c_dev->base;
+	struct device *dev = i2c_dev->dev;
+	u32 status;
+
+	status = readl_relaxed(i2c_dev->base + STM32F7_I2C_ISR);
+
+	/* Bus error */
+	if (status & STM32F7_I2C_ISR_BERR) {
+		dev_err(dev, "<%s>: Bus error\n", __func__);
+		writel_relaxed(STM32F7_I2C_ICR_BERRCF, base + STM32F7_I2C_ICR);
+		f7_msg->result = -EIO;
+	}
+
+	/* Arbitration loss */
+	if (status & STM32F7_I2C_ISR_ARLO) {
+		dev_dbg(dev, "<%s>: Arbitration loss\n", __func__);
+		writel_relaxed(STM32F7_I2C_ICR_ARLOCF, base + STM32F7_I2C_ICR);
+		f7_msg->result = -EAGAIN;
+	}
+
+	stm32f7_i2c_disable_irq(i2c_dev, STM32F7_I2C_ALL_IRQ_MASK);
+
+	complete(&i2c_dev->complete);
+
+	return IRQ_HANDLED;
+}
+
+static int stm32f7_i2c_xfer(struct i2c_adapter *i2c_adap,
+			    struct i2c_msg msgs[], int num)
+{
+	struct stm32f7_i2c_dev *i2c_dev = i2c_get_adapdata(i2c_adap);
+	struct stm32f7_i2c_msg *f7_msg = &i2c_dev->f7_msg;
+	unsigned long time_left;
+	int ret;
+
+	i2c_dev->msg = msgs;
+	i2c_dev->msg_num = num;
+	i2c_dev->msg_id = 0;
+
+	ret = clk_enable(i2c_dev->clk);
+	if (ret) {
+		dev_err(i2c_dev->dev, "Failed to enable clock\n");
+		return ret;
+	}
+
+	ret = stm32f7_i2c_wait_free_bus(i2c_dev);
+	if (ret)
+		goto clk_free;
+
+	stm32f7_i2c_xfer_msg(i2c_dev, msgs);
+
+	time_left = wait_for_completion_timeout(&i2c_dev->complete,
+						i2c_dev->adap.timeout);
+	ret = f7_msg->result;
+
+	if (!time_left) {
+		dev_dbg(i2c_dev->dev, "Access to slave 0x%x timed out\n",
+			i2c_dev->msg->addr);
+		ret = -ETIMEDOUT;
+	}
+
+clk_free:
+	clk_disable(i2c_dev->clk);
+
+	return (ret < 0) ? ret : num;
+}
+
+static u32 stm32f7_i2c_func(struct i2c_adapter *adap)
+{
+	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
+}
+
+static struct i2c_algorithm stm32f7_i2c_algo = {
+	.master_xfer = stm32f7_i2c_xfer,
+	.functionality = stm32f7_i2c_func,
+};
+
+static int stm32f7_i2c_probe(struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct stm32f7_i2c_dev *i2c_dev;
+	const struct stm32f7_i2c_setup *setup;
+	struct resource *res;
+	u32 irq_error, irq_event, clk_rate, rise_time, fall_time;
+	struct i2c_adapter *adap;
+	struct reset_control *rst;
+	int ret;
+
+	i2c_dev = devm_kzalloc(&pdev->dev, sizeof(*i2c_dev), GFP_KERNEL);
+	if (!i2c_dev)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	i2c_dev->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(i2c_dev->base))
+		return PTR_ERR(i2c_dev->base);
+
+	irq_event = irq_of_parse_and_map(np, 0);
+	if (!irq_event) {
+		dev_err(&pdev->dev, "IRQ event missing or invalid\n");
+		return -EINVAL;
+	}
+
+	irq_error = irq_of_parse_and_map(np, 1);
+	if (!irq_error) {
+		dev_err(&pdev->dev, "IRQ error missing or invalid\n");
+		return -EINVAL;
+	}
+
+	i2c_dev->clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(i2c_dev->clk)) {
+		dev_err(&pdev->dev, "Error: Missing controller clock\n");
+		return PTR_ERR(i2c_dev->clk);
+	}
+	ret = clk_prepare_enable(i2c_dev->clk);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to prepare_enable clock\n");
+		return ret;
+	}
+
+	i2c_dev->speed = STM32_I2C_SPEED_STANDARD;
+	ret = device_property_read_u32(&pdev->dev, "clock-frequency",
+				       &clk_rate);
+	if (!ret && clk_rate >= 1000000)
+		i2c_dev->speed = STM32_I2C_SPEED_FAST_PLUS;
+	else if (!ret && clk_rate >= 400000)
+		i2c_dev->speed = STM32_I2C_SPEED_FAST;
+	else if (!ret && clk_rate >= 100000)
+		i2c_dev->speed = STM32_I2C_SPEED_STANDARD;
+
+	rst = devm_reset_control_get(&pdev->dev, NULL);
+	if (IS_ERR(rst)) {
+		dev_err(&pdev->dev, "Error: Missing controller reset\n");
+		ret = PTR_ERR(rst);
+		goto clk_free;
+	}
+	reset_control_assert(rst);
+	udelay(2);
+	reset_control_deassert(rst);
+
+	i2c_dev->dev = &pdev->dev;
+
+	ret = devm_request_irq(&pdev->dev, irq_event, stm32f7_i2c_isr_event, 0,
+			       pdev->name, i2c_dev);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to request irq event %i\n",
+			irq_event);
+		goto clk_free;
+	}
+
+	ret = devm_request_irq(&pdev->dev, irq_error, stm32f7_i2c_isr_error, 0,
+			       pdev->name, i2c_dev);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to request irq error %i\n",
+			irq_error);
+		goto clk_free;
+	}
+
+	setup = of_device_get_match_data(&pdev->dev);
+	i2c_dev->setup->rise_time = setup->rise_time;
+	i2c_dev->setup->fall_time = setup->fall_time;
+	i2c_dev->setup->dnf = setup->dnf;
+	i2c_dev->setup->analog_filter = setup->analog_filter;
+
+	ret = device_property_read_u32(i2c_dev->dev, "i2c-scl-rising-time-ns",
+				       &rise_time);
+	if (!ret)
+		i2c_dev->setup->rise_time = rise_time;
+
+	ret = device_property_read_u32(i2c_dev->dev, "i2c-scl-falling-time-ns",
+				       &fall_time);
+	if (!ret)
+		i2c_dev->setup->fall_time = fall_time;
+
+	ret = stm32f7_i2c_setup_timing(i2c_dev, i2c_dev->setup);
+	if (ret)
+		goto clk_free;
+
+	stm32f7_i2c_hw_config(i2c_dev);
+
+	adap = &i2c_dev->adap;
+	i2c_set_adapdata(adap, i2c_dev);
+	snprintf(adap->name, sizeof(adap->name), "STM32F7 I2C(%pa)",
+		 &res->start);
+	adap->owner = THIS_MODULE;
+	adap->timeout = 2 * HZ;
+	adap->retries = 3;
+	adap->algo = &stm32f7_i2c_algo;
+	adap->dev.parent = &pdev->dev;
+	adap->dev.of_node = pdev->dev.of_node;
+
+	init_completion(&i2c_dev->complete);
+
+	ret = i2c_add_adapter(adap);
+	if (ret)
+		goto clk_free;
+
+	platform_set_drvdata(pdev, i2c_dev);
+
+	clk_disable(i2c_dev->clk);
+
+	dev_info(i2c_dev->dev, "STM32F7 I2C-%d bus adapter\n", adap->nr);
+
+	return 0;
+
+clk_free:
+	clk_disable_unprepare(i2c_dev->clk);
+
+	return ret;
+}
+
+static int stm32f7_i2c_remove(struct platform_device *pdev)
+{
+	struct stm32f7_i2c_dev *i2c_dev = platform_get_drvdata(pdev);
+
+	i2c_del_adapter(&i2c_dev->adap);
+
+	clk_unprepare(i2c_dev->clk);
+
+	return 0;
+}
+
+static const struct of_device_id stm32f7_i2c_match[] = {
+	{ .compatible = "st,stm32f7-i2c", .data = &stm32f7_setup},
+	{},
+};
+MODULE_DEVICE_TABLE(of, stm32f7_i2c_match);
+
+static struct platform_driver stm32f7_i2c_driver = {
+	.driver = {
+		.name = "stm32f7-i2c",
+		.of_match_table = stm32f7_i2c_match,
+	},
+	.probe = stm32f7_i2c_probe,
+	.remove = stm32f7_i2c_remove,
+};
+
+module_platform_driver(stm32f7_i2c_driver);
+
+MODULE_AUTHOR("M'boumba Cedric Madianga <cedric.madianga@gmail.com>");
+MODULE_DESCRIPTION("STMicroelectronics STM32F7 I2C driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/i2c/busses/i2c-sun6i-p2wi.c b/drivers/i2c/busses/i2c-sun6i-p2wi.c
index 7668e2e9b8fd..7c07ce116e38 100644
--- a/drivers/i2c/busses/i2c-sun6i-p2wi.c
+++ b/drivers/i2c/busses/i2c-sun6i-p2wi.c
@@ -223,8 +223,8 @@ static int p2wi_probe(struct platform_device *pdev)
 	if (childnp) {
 		ret = of_property_read_u32(childnp, "reg", &slave_addr);
 		if (ret) {
-			dev_err(dev, "invalid slave address on node %s\n",
-				childnp->full_name);
+			dev_err(dev, "invalid slave address on node %pOF\n",
+				childnp);
 			return -EINVAL;
 		}
 
@@ -258,7 +258,7 @@ static int p2wi_probe(struct platform_device *pdev)
 
 	parent_clk_freq = clk_get_rate(p2wi->clk);
 
-	p2wi->rstc = devm_reset_control_get(dev, NULL);
+	p2wi->rstc = devm_reset_control_get_exclusive(dev, NULL);
 	if (IS_ERR(p2wi->rstc)) {
 		ret = PTR_ERR(p2wi->rstc);
 		dev_err(dev, "failed to retrieve reset controller: %d\n", ret);
diff --git a/drivers/i2c/busses/i2c-taos-evm.c b/drivers/i2c/busses/i2c-taos-evm.c
index 210ca82f8aa0..addd90a8cb59 100644
--- a/drivers/i2c/busses/i2c-taos-evm.c
+++ b/drivers/i2c/busses/i2c-taos-evm.c
@@ -291,7 +291,7 @@ static void taos_disconnect(struct serio *serio)
 	dev_info(&serio->dev, "Disconnected from TAOS EVM\n");
 }
 
-static struct serio_device_id taos_serio_ids[] = {
+static const struct serio_device_id taos_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_TAOSEVM,
diff --git a/drivers/i2c/busses/i2c-tegra.c b/drivers/i2c/busses/i2c-tegra.c
index 4af9bbae20df..60292d243e24 100644
--- a/drivers/i2c/busses/i2c-tegra.c
+++ b/drivers/i2c/busses/i2c-tegra.c
@@ -793,7 +793,7 @@ static const struct i2c_algorithm tegra_i2c_algo = {
 };
 
 /* payload size is only 12 bit */
-static struct i2c_adapter_quirks tegra_i2c_quirks = {
+static const struct i2c_adapter_quirks tegra_i2c_quirks = {
 	.max_read_len = 4096,
 	.max_write_len = 4096,
 };
@@ -911,7 +911,7 @@ static int tegra_i2c_probe(struct platform_device *pdev)
 	i2c_dev->cont_id = pdev->id;
 	i2c_dev->dev = &pdev->dev;
 
-	i2c_dev->rst = devm_reset_control_get(&pdev->dev, "i2c");
+	i2c_dev->rst = devm_reset_control_get_exclusive(&pdev->dev, "i2c");
 	if (IS_ERR(i2c_dev->rst)) {
 		dev_err(&pdev->dev, "missing controller reset\n");
 		return PTR_ERR(i2c_dev->rst);
diff --git a/drivers/i2c/busses/i2c-thunderx-pcidrv.c b/drivers/i2c/busses/i2c-thunderx-pcidrv.c
index ea35a895b568..df0976f4432a 100644
--- a/drivers/i2c/busses/i2c-thunderx-pcidrv.c
+++ b/drivers/i2c/busses/i2c-thunderx-pcidrv.c
@@ -75,7 +75,7 @@ static const struct i2c_algorithm thunderx_i2c_algo = {
 	.functionality = thunderx_i2c_functionality,
 };
 
-static struct i2c_adapter thunderx_i2c_ops = {
+static const struct i2c_adapter thunderx_i2c_ops = {
 	.owner	= THIS_MODULE,
 	.name	= "ThunderX adapter",
 	.algo	= &thunderx_i2c_algo,
diff --git a/drivers/i2c/busses/i2c-uniphier-f.c b/drivers/i2c/busses/i2c-uniphier-f.c
index beee31892295..9918bdd81619 100644
--- a/drivers/i2c/busses/i2c-uniphier-f.c
+++ b/drivers/i2c/busses/i2c-uniphier-f.c
@@ -97,6 +97,7 @@ struct uniphier_fi2c_priv {
 	int error;
 	unsigned int flags;
 	unsigned int busy_cnt;
+	unsigned int clk_cycle;
 };
 
 static void uniphier_fi2c_fill_txfifo(struct uniphier_fi2c_priv *priv,
@@ -461,9 +462,9 @@ static struct i2c_bus_recovery_info uniphier_fi2c_bus_recovery_info = {
 	.unprepare_recovery = uniphier_fi2c_unprepare_recovery,
 };
 
-static void uniphier_fi2c_hw_init(struct uniphier_fi2c_priv *priv,
-				  u32 bus_speed, unsigned long clk_rate)
+static void uniphier_fi2c_hw_init(struct uniphier_fi2c_priv *priv)
 {
+	unsigned int cyc = priv->clk_cycle;
 	u32 tmp;
 
 	tmp = readl(priv->membase + UNIPHIER_FI2C_CR);
@@ -472,12 +473,10 @@ static void uniphier_fi2c_hw_init(struct uniphier_fi2c_priv *priv,
 
 	uniphier_fi2c_reset(priv);
 
-	tmp = clk_rate / bus_speed;
-
-	writel(tmp, priv->membase + UNIPHIER_FI2C_CYC);
-	writel(tmp / 2, priv->membase + UNIPHIER_FI2C_LCTL);
-	writel(tmp / 2, priv->membase + UNIPHIER_FI2C_SSUT);
-	writel(tmp / 16, priv->membase + UNIPHIER_FI2C_DSUT);
+	writel(cyc, priv->membase + UNIPHIER_FI2C_CYC);
+	writel(cyc / 2, priv->membase + UNIPHIER_FI2C_LCTL);
+	writel(cyc / 2, priv->membase + UNIPHIER_FI2C_SSUT);
+	writel(cyc / 16, priv->membase + UNIPHIER_FI2C_DSUT);
 
 	uniphier_fi2c_prepare_operation(priv);
 }
@@ -531,6 +530,7 @@ static int uniphier_fi2c_probe(struct platform_device *pdev)
 		goto disable_clk;
 	}
 
+	priv->clk_cycle = clk_rate / bus_speed;
 	init_completion(&priv->comp);
 	priv->adap.owner = THIS_MODULE;
 	priv->adap.algo = &uniphier_fi2c_algo;
@@ -541,7 +541,7 @@ static int uniphier_fi2c_probe(struct platform_device *pdev)
 	i2c_set_adapdata(&priv->adap, priv);
 	platform_set_drvdata(pdev, priv);
 
-	uniphier_fi2c_hw_init(priv, bus_speed, clk_rate);
+	uniphier_fi2c_hw_init(priv);
 
 	ret = devm_request_irq(dev, irq, uniphier_fi2c_interrupt, 0,
 			       pdev->name, priv);
@@ -568,6 +568,33 @@ static int uniphier_fi2c_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static int __maybe_unused uniphier_fi2c_suspend(struct device *dev)
+{
+	struct uniphier_fi2c_priv *priv = dev_get_drvdata(dev);
+
+	clk_disable_unprepare(priv->clk);
+
+	return 0;
+}
+
+static int __maybe_unused uniphier_fi2c_resume(struct device *dev)
+{
+	struct uniphier_fi2c_priv *priv = dev_get_drvdata(dev);
+	int ret;
+
+	ret = clk_prepare_enable(priv->clk);
+	if (ret)
+		return ret;
+
+	uniphier_fi2c_hw_init(priv);
+
+	return 0;
+}
+
+static const struct dev_pm_ops uniphier_fi2c_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(uniphier_fi2c_suspend, uniphier_fi2c_resume)
+};
+
 static const struct of_device_id uniphier_fi2c_match[] = {
 	{ .compatible = "socionext,uniphier-fi2c" },
 	{ /* sentinel */ }
@@ -580,6 +607,7 @@ static struct platform_driver uniphier_fi2c_drv = {
 	.driver = {
 		.name  = "uniphier-fi2c",
 		.of_match_table = uniphier_fi2c_match,
+		.pm = &uniphier_fi2c_pm_ops,
 	},
 };
 module_platform_driver(uniphier_fi2c_drv);
diff --git a/drivers/i2c/busses/i2c-uniphier.c b/drivers/i2c/busses/i2c-uniphier.c
index 777c0fe93653..bb181b088291 100644
--- a/drivers/i2c/busses/i2c-uniphier.c
+++ b/drivers/i2c/busses/i2c-uniphier.c
@@ -53,6 +53,7 @@ struct uniphier_i2c_priv {
 	void __iomem *membase;
 	struct clk *clk;
 	unsigned int busy_cnt;
+	unsigned int clk_cycle;
 };
 
 static irqreturn_t uniphier_i2c_interrupt(int irq, void *dev_id)
@@ -316,13 +317,13 @@ static struct i2c_bus_recovery_info uniphier_i2c_bus_recovery_info = {
 	.unprepare_recovery = uniphier_i2c_unprepare_recovery,
 };
 
-static void uniphier_i2c_hw_init(struct uniphier_i2c_priv *priv,
-				 u32 bus_speed, unsigned long clk_rate)
+static void uniphier_i2c_hw_init(struct uniphier_i2c_priv *priv)
 {
+	unsigned int cyc = priv->clk_cycle;
+
 	uniphier_i2c_reset(priv, true);
 
-	writel((clk_rate / bus_speed / 2 << 16) | (clk_rate / bus_speed),
-	       priv->membase + UNIPHIER_I2C_CLK);
+	writel((cyc / 2 << 16) | cyc, priv->membase + UNIPHIER_I2C_CLK);
 
 	uniphier_i2c_reset(priv, false);
 }
@@ -376,6 +377,7 @@ static int uniphier_i2c_probe(struct platform_device *pdev)
 		goto disable_clk;
 	}
 
+	priv->clk_cycle = clk_rate / bus_speed;
 	init_completion(&priv->comp);
 	priv->adap.owner = THIS_MODULE;
 	priv->adap.algo = &uniphier_i2c_algo;
@@ -386,7 +388,7 @@ static int uniphier_i2c_probe(struct platform_device *pdev)
 	i2c_set_adapdata(&priv->adap, priv);
 	platform_set_drvdata(pdev, priv);
 
-	uniphier_i2c_hw_init(priv, bus_speed, clk_rate);
+	uniphier_i2c_hw_init(priv);
 
 	ret = devm_request_irq(dev, irq, uniphier_i2c_interrupt, 0, pdev->name,
 			       priv);
@@ -413,6 +415,33 @@ static int uniphier_i2c_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static int __maybe_unused uniphier_i2c_suspend(struct device *dev)
+{
+	struct uniphier_i2c_priv *priv = dev_get_drvdata(dev);
+
+	clk_disable_unprepare(priv->clk);
+
+	return 0;
+}
+
+static int __maybe_unused uniphier_i2c_resume(struct device *dev)
+{
+	struct uniphier_i2c_priv *priv = dev_get_drvdata(dev);
+	int ret;
+
+	ret = clk_prepare_enable(priv->clk);
+	if (ret)
+		return ret;
+
+	uniphier_i2c_hw_init(priv);
+
+	return 0;
+}
+
+static const struct dev_pm_ops uniphier_i2c_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(uniphier_i2c_suspend, uniphier_i2c_resume)
+};
+
 static const struct of_device_id uniphier_i2c_match[] = {
 	{ .compatible = "socionext,uniphier-i2c" },
 	{ /* sentinel */ }
@@ -425,6 +454,7 @@ static struct platform_driver uniphier_i2c_drv = {
 	.driver = {
 		.name  = "uniphier-i2c",
 		.of_match_table = uniphier_i2c_match,
+		.pm = &uniphier_i2c_pm_ops,
 	},
 };
 module_platform_driver(uniphier_i2c_drv);
diff --git a/drivers/i2c/busses/i2c-versatile.c b/drivers/i2c/busses/i2c-versatile.c
index c73d2d22009e..f1ab2a637ec0 100644
--- a/drivers/i2c/busses/i2c-versatile.c
+++ b/drivers/i2c/busses/i2c-versatile.c
@@ -55,7 +55,7 @@ static int i2c_versatile_getscl(void *data)
 	return !!(readl(i2c->base + I2C_CONTROL) & SCL);
 }
 
-static struct i2c_algo_bit_data i2c_versatile_algo = {
+static const struct i2c_algo_bit_data i2c_versatile_algo = {
 	.setsda	= i2c_versatile_setsda,
 	.setscl = i2c_versatile_setscl,
 	.getsda	= i2c_versatile_getsda,
diff --git a/drivers/i2c/busses/i2c-xiic.c b/drivers/i2c/busses/i2c-xiic.c
index 66bce3b311a1..ae6ed254e01d 100644
--- a/drivers/i2c/busses/i2c-xiic.c
+++ b/drivers/i2c/busses/i2c-xiic.c
@@ -721,7 +721,7 @@ static const struct i2c_algorithm xiic_algorithm = {
 	.functionality = xiic_func,
 };
 
-static struct i2c_adapter xiic_adapter = {
+static const struct i2c_adapter xiic_adapter = {
 	.owner = THIS_MODULE,
 	.name = DRIVER_NAME,
 	.class = I2C_CLASS_DEPRECATED,
@@ -853,8 +853,7 @@ MODULE_DEVICE_TABLE(of, xiic_of_match);
 
 static int __maybe_unused cdns_i2c_runtime_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct xiic_i2c *i2c = platform_get_drvdata(pdev);
+	struct xiic_i2c *i2c = dev_get_drvdata(dev);
 
 	clk_disable(i2c->clk);
 
@@ -863,8 +862,7 @@ static int __maybe_unused cdns_i2c_runtime_suspend(struct device *dev)
 
 static int __maybe_unused cdns_i2c_runtime_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct xiic_i2c *i2c = platform_get_drvdata(pdev);
+	struct xiic_i2c *i2c = dev_get_drvdata(dev);
 	int ret;
 
 	ret = clk_enable(i2c->clk);
diff --git a/drivers/i2c/i2c-core-base.c b/drivers/i2c/i2c-core-base.c
index 12822a4b8f8f..56e46581b84b 100644
--- a/drivers/i2c/i2c-core-base.c
+++ b/drivers/i2c/i2c-core-base.c
@@ -353,8 +353,8 @@ static int i2c_device_probe(struct device *dev)
 	}
 
 	/*
-	 * An I2C ID table is not mandatory, if and only if, a suitable Device
-	 * Tree match table entry is supplied for the probing device.
+	 * An I2C ID table is not mandatory, if and only if, a suitable OF
+	 * or ACPI ID table is supplied for the probing device.
 	 */
 	if (!driver->id_table &&
 	    !i2c_acpi_match_device(dev->driver->acpi_match_table, client) &&
diff --git a/drivers/i2c/i2c-core-of.c b/drivers/i2c/i2c-core-of.c
index ccf82fdbcd8e..8d474bb1dc15 100644
--- a/drivers/i2c/i2c-core-of.c
+++ b/drivers/i2c/i2c-core-of.c
@@ -32,18 +32,17 @@ static struct i2c_client *of_i2c_register_device(struct i2c_adapter *adap,
 	u32 addr;
 	int len;
 
-	dev_dbg(&adap->dev, "of_i2c: register %s\n", node->full_name);
+	dev_dbg(&adap->dev, "of_i2c: register %pOF\n", node);
 
 	if (of_modalias_node(node, info.type, sizeof(info.type)) < 0) {
-		dev_err(&adap->dev, "of_i2c: modalias failure on %s\n",
-			node->full_name);
+		dev_err(&adap->dev, "of_i2c: modalias failure on %pOF\n",
+			node);
 		return ERR_PTR(-EINVAL);
 	}
 
 	addr_be = of_get_property(node, "reg", &len);
 	if (!addr_be || (len < sizeof(*addr_be))) {
-		dev_err(&adap->dev, "of_i2c: invalid reg on %s\n",
-			node->full_name);
+		dev_err(&adap->dev, "of_i2c: invalid reg on %pOF\n", node);
 		return ERR_PTR(-EINVAL);
 	}
 
@@ -59,8 +58,8 @@ static struct i2c_client *of_i2c_register_device(struct i2c_adapter *adap,
 	}
 
 	if (i2c_check_addr_validity(addr, info.flags)) {
-		dev_err(&adap->dev, "of_i2c: invalid addr=%x on %s\n",
-			addr, node->full_name);
+		dev_err(&adap->dev, "of_i2c: invalid addr=%x on %pOF\n",
+			addr, node);
 		return ERR_PTR(-EINVAL);
 	}
 
@@ -76,8 +75,7 @@ static struct i2c_client *of_i2c_register_device(struct i2c_adapter *adap,
 
 	result = i2c_new_device(adap, &info);
 	if (result == NULL) {
-		dev_err(&adap->dev, "of_i2c: Failure registering %s\n",
-			node->full_name);
+		dev_err(&adap->dev, "of_i2c: Failure registering %pOF\n", node);
 		of_node_put(node);
 		return ERR_PTR(-EINVAL);
 	}
@@ -106,8 +104,8 @@ void of_i2c_register_devices(struct i2c_adapter *adap)
 		client = of_i2c_register_device(adap, node);
 		if (IS_ERR(client)) {
 			dev_warn(&adap->dev,
-				 "Failed to create I2C device for %s\n",
-				 node->full_name);
+				 "Failed to create I2C device for %pOF\n",
+				 node);
 			of_node_clear_flag(node, OF_POPULATED);
 		}
 	}
@@ -243,8 +241,8 @@ static int of_i2c_notify(struct notifier_block *nb, unsigned long action,
 		put_device(&adap->dev);
 
 		if (IS_ERR(client)) {
-			dev_err(&adap->dev, "failed to create client for '%s'\n",
-				 rd->dn->full_name);
+			dev_err(&adap->dev, "failed to create client for '%pOF'\n",
+				 rd->dn);
 			of_node_clear_flag(rd->dn, OF_POPULATED);
 			return notifier_from_errno(PTR_ERR(client));
 		}
diff --git a/drivers/i2c/muxes/Kconfig b/drivers/i2c/muxes/Kconfig
index 17121329bb79..0f5c8fc36625 100644
--- a/drivers/i2c/muxes/Kconfig
+++ b/drivers/i2c/muxes/Kconfig
@@ -8,7 +8,7 @@ menu "Multiplexer I2C Chip support"
 config I2C_ARB_GPIO_CHALLENGE
 	tristate "GPIO-based I2C arbitration"
 	depends on GPIOLIB || COMPILE_TEST
-	depends on OF
+	depends on OF || COMPILE_TEST
 	help
 	  If you say yes to this option, support will be included for an
 	  I2C multimaster arbitration scheme using GPIOs and a challenge &
@@ -76,6 +76,7 @@ config I2C_MUX_PCA954x
 config I2C_MUX_PINCTRL
 	tristate "pinctrl-based I2C multiplexer"
 	depends on PINCTRL
+	depends on OF || COMPILE_TEST
 	help
 	  If you say yes to this option, support will be included for an I2C
 	  multiplexer that uses the pinctrl subsystem, i.e. pin multiplexing.
diff --git a/drivers/i2c/muxes/i2c-demux-pinctrl.c b/drivers/i2c/muxes/i2c-demux-pinctrl.c
index 3e6fe1760d82..33ce032cb701 100644
--- a/drivers/i2c/muxes/i2c-demux-pinctrl.c
+++ b/drivers/i2c/muxes/i2c-demux-pinctrl.c
@@ -167,8 +167,8 @@ static ssize_t available_masters_show(struct device *dev,
 	int count = 0, i;
 
 	for (i = 0; i < priv->num_chan && count < PAGE_SIZE; i++)
-		count += scnprintf(buf + count, PAGE_SIZE - count, "%d:%s%c",
-				   i, priv->chan[i].parent_np->full_name,
+		count += scnprintf(buf + count, PAGE_SIZE - count, "%d:%pOF%c",
+				   i, priv->chan[i].parent_np,
 				   i == priv->num_chan - 1 ? '\n' : ' ');
 
 	return count;
diff --git a/drivers/i2c/muxes/i2c-mux-mlxcpld.c b/drivers/i2c/muxes/i2c-mux-mlxcpld.c
index e53f2abd1350..12ad8d65faf6 100644
--- a/drivers/i2c/muxes/i2c-mux-mlxcpld.c
+++ b/drivers/i2c/muxes/i2c-mux-mlxcpld.c
@@ -38,9 +38,9 @@
 #include <linux/io.h>
 #include <linux/init.h>
 #include <linux/module.h>
+#include <linux/platform_data/x86/mlxcpld.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
-#include <linux/i2c/mlxcpld.h>
 
 #define CPLD_MUX_MAX_NCHANS	8
 
diff --git a/drivers/i2c/muxes/i2c-mux-pca9541.c b/drivers/i2c/muxes/i2c-mux-pca9541.c
index 9e318c9516c7..6a39adaf433f 100644
--- a/drivers/i2c/muxes/i2c-mux-pca9541.c
+++ b/drivers/i2c/muxes/i2c-mux-pca9541.c
@@ -16,15 +16,14 @@
  * warranty of any kind, whether express or implied.
  */
 
-#include <linux/module.h>
-#include <linux/jiffies.h>
 #include <linux/delay.h>
-#include <linux/slab.h>
 #include <linux/device.h>
 #include <linux/i2c.h>
 #include <linux/i2c-mux.h>
-
-#include <linux/i2c/pca954x.h>
+#include <linux/jiffies.h>
+#include <linux/module.h>
+#include <linux/platform_data/pca954x.h>
+#include <linux/slab.h>
 
 /*
  * The PCA9541 is a bus master selector. It supports two I2C masters connected
diff --git a/drivers/i2c/muxes/i2c-mux-pca954x.c b/drivers/i2c/muxes/i2c-mux-pca954x.c
index f1751c290af6..7b992db38021 100644
--- a/drivers/i2c/muxes/i2c-mux-pca954x.c
+++ b/drivers/i2c/muxes/i2c-mux-pca954x.c
@@ -39,13 +39,13 @@
 #include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
 #include <linux/i2c-mux.h>
-#include <linux/i2c/pca954x.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
 #include <linux/of_irq.h>
+#include <linux/platform_data/pca954x.h>
 #include <linux/pm.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
diff --git a/drivers/i2c/muxes/i2c-mux-pinctrl.c b/drivers/i2c/muxes/i2c-mux-pinctrl.c
index 7c0c264b07bc..cc6818aabab5 100644
--- a/drivers/i2c/muxes/i2c-mux-pinctrl.c
+++ b/drivers/i2c/muxes/i2c-mux-pinctrl.c
@@ -20,17 +20,14 @@
 #include <linux/i2c-mux.h>
 #include <linux/module.h>
 #include <linux/pinctrl/consumer.h>
-#include <linux/i2c-mux-pinctrl.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/of.h>
 #include "../../pinctrl/core.h"
 
 struct i2c_mux_pinctrl {
-	struct i2c_mux_pinctrl_platform_data *pdata;
 	struct pinctrl *pinctrl;
 	struct pinctrl_state **states;
-	struct pinctrl_state *state_idle;
 };
 
 static int i2c_mux_pinctrl_select(struct i2c_mux_core *muxc, u32 chan)
@@ -42,85 +39,9 @@ static int i2c_mux_pinctrl_select(struct i2c_mux_core *muxc, u32 chan)
 
 static int i2c_mux_pinctrl_deselect(struct i2c_mux_core *muxc, u32 chan)
 {
-	struct i2c_mux_pinctrl *mux = i2c_mux_priv(muxc);
-
-	return pinctrl_select_state(mux->pinctrl, mux->state_idle);
+	return i2c_mux_pinctrl_select(muxc, muxc->num_adapters);
 }
 
-#ifdef CONFIG_OF
-static int i2c_mux_pinctrl_parse_dt(struct i2c_mux_pinctrl *mux,
-				    struct platform_device *pdev)
-{
-	struct device_node *np = pdev->dev.of_node;
-	int num_names, i, ret;
-	struct device_node *adapter_np;
-	struct i2c_adapter *adapter;
-
-	if (!np)
-		return 0;
-
-	mux->pdata = devm_kzalloc(&pdev->dev, sizeof(*mux->pdata), GFP_KERNEL);
-	if (!mux->pdata)
-		return -ENOMEM;
-
-	num_names = of_property_count_strings(np, "pinctrl-names");
-	if (num_names < 0) {
-		dev_err(&pdev->dev, "Cannot parse pinctrl-names: %d\n",
-			num_names);
-		return num_names;
-	}
-
-	mux->pdata->pinctrl_states = devm_kzalloc(&pdev->dev,
-		sizeof(*mux->pdata->pinctrl_states) * num_names,
-		GFP_KERNEL);
-	if (!mux->pdata->pinctrl_states)
-		return -ENOMEM;
-
-	for (i = 0; i < num_names; i++) {
-		ret = of_property_read_string_index(np, "pinctrl-names", i,
-			&mux->pdata->pinctrl_states[mux->pdata->bus_count]);
-		if (ret < 0) {
-			dev_err(&pdev->dev, "Cannot parse pinctrl-names: %d\n",
-				ret);
-			return ret;
-		}
-		if (!strcmp(mux->pdata->pinctrl_states[mux->pdata->bus_count],
-			    "idle")) {
-			if (i != num_names - 1) {
-				dev_err(&pdev->dev,
-					"idle state must be last\n");
-				return -EINVAL;
-			}
-			mux->pdata->pinctrl_state_idle = "idle";
-		} else {
-			mux->pdata->bus_count++;
-		}
-	}
-
-	adapter_np = of_parse_phandle(np, "i2c-parent", 0);
-	if (!adapter_np) {
-		dev_err(&pdev->dev, "Cannot parse i2c-parent\n");
-		return -ENODEV;
-	}
-	adapter = of_find_i2c_adapter_by_node(adapter_np);
-	of_node_put(adapter_np);
-	if (!adapter) {
-		dev_err(&pdev->dev, "Cannot find parent bus\n");
-		return -EPROBE_DEFER;
-	}
-	mux->pdata->parent_bus_num = i2c_adapter_id(adapter);
-	put_device(&adapter->dev);
-
-	return 0;
-}
-#else
-static inline int i2c_mux_pinctrl_parse_dt(struct i2c_mux_pinctrl *mux,
-					   struct platform_device *pdev)
-{
-	return 0;
-}
-#endif
-
 static struct i2c_adapter *i2c_mux_pinctrl_root_adapter(
 	struct pinctrl_state *state)
 {
@@ -141,110 +62,108 @@ static struct i2c_adapter *i2c_mux_pinctrl_root_adapter(
 	return root;
 }
 
+static struct i2c_adapter *i2c_mux_pinctrl_parent_adapter(struct device *dev)
+{
+	struct device_node *np = dev->of_node;
+	struct device_node *parent_np;
+	struct i2c_adapter *parent;
+
+	parent_np = of_parse_phandle(np, "i2c-parent", 0);
+	if (!parent_np) {
+		dev_err(dev, "Cannot parse i2c-parent\n");
+		return ERR_PTR(-ENODEV);
+	}
+	parent = of_find_i2c_adapter_by_node(parent_np);
+	of_node_put(parent_np);
+	if (!parent)
+		return ERR_PTR(-EPROBE_DEFER);
+
+	return parent;
+}
+
 static int i2c_mux_pinctrl_probe(struct platform_device *pdev)
 {
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
 	struct i2c_mux_core *muxc;
 	struct i2c_mux_pinctrl *mux;
+	struct i2c_adapter *parent;
 	struct i2c_adapter *root;
-	int i, ret;
-
-	mux = devm_kzalloc(&pdev->dev, sizeof(*mux), GFP_KERNEL);
-	if (!mux) {
-		ret = -ENOMEM;
-		goto err;
-	}
+	int num_names, i, ret;
+	const char *name;
 
-	mux->pdata = dev_get_platdata(&pdev->dev);
-	if (!mux->pdata) {
-		ret = i2c_mux_pinctrl_parse_dt(mux, pdev);
-		if (ret < 0)
-			goto err;
-	}
-	if (!mux->pdata) {
-		dev_err(&pdev->dev, "Missing platform data\n");
-		ret = -ENODEV;
-		goto err;
+	num_names = of_property_count_strings(np, "pinctrl-names");
+	if (num_names < 0) {
+		dev_err(dev, "Cannot parse pinctrl-names: %d\n",
+			num_names);
+		return num_names;
 	}
 
-	mux->states = devm_kzalloc(&pdev->dev,
-				   sizeof(*mux->states) * mux->pdata->bus_count,
-				   GFP_KERNEL);
-	if (!mux->states) {
-		dev_err(&pdev->dev, "Cannot allocate states\n");
-		ret = -ENOMEM;
-		goto err;
-	}
+	parent = i2c_mux_pinctrl_parent_adapter(dev);
+	if (IS_ERR(parent))
+		return PTR_ERR(parent);
 
-	muxc = i2c_mux_alloc(NULL, &pdev->dev, mux->pdata->bus_count, 0, 0,
-			     i2c_mux_pinctrl_select, NULL);
+	muxc = i2c_mux_alloc(parent, dev, num_names,
+			     sizeof(*mux) + num_names * sizeof(*mux->states),
+			     0, i2c_mux_pinctrl_select, NULL);
 	if (!muxc) {
 		ret = -ENOMEM;
-		goto err;
+		goto err_put_parent;
 	}
-	muxc->priv = mux;
+	mux = i2c_mux_priv(muxc);
+	mux->states = (struct pinctrl_state **)(mux + 1);
 
 	platform_set_drvdata(pdev, muxc);
 
-	mux->pinctrl = devm_pinctrl_get(&pdev->dev);
+	mux->pinctrl = devm_pinctrl_get(dev);
 	if (IS_ERR(mux->pinctrl)) {
 		ret = PTR_ERR(mux->pinctrl);
-		dev_err(&pdev->dev, "Cannot get pinctrl: %d\n", ret);
-		goto err;
+		dev_err(dev, "Cannot get pinctrl: %d\n", ret);
+		goto err_put_parent;
 	}
-	for (i = 0; i < mux->pdata->bus_count; i++) {
-		mux->states[i] = pinctrl_lookup_state(mux->pinctrl,
-						mux->pdata->pinctrl_states[i]);
+
+	for (i = 0; i < num_names; i++) {
+		ret = of_property_read_string_index(np, "pinctrl-names", i,
+						    &name);
+		if (ret < 0) {
+			dev_err(dev, "Cannot parse pinctrl-names: %d\n", ret);
+			goto err_put_parent;
+		}
+
+		mux->states[i] = pinctrl_lookup_state(mux->pinctrl, name);
 		if (IS_ERR(mux->states[i])) {
 			ret = PTR_ERR(mux->states[i]);
-			dev_err(&pdev->dev,
-				"Cannot look up pinctrl state %s: %d\n",
-				mux->pdata->pinctrl_states[i], ret);
-			goto err;
-		}
-	}
-	if (mux->pdata->pinctrl_state_idle) {
-		mux->state_idle = pinctrl_lookup_state(mux->pinctrl,
-						mux->pdata->pinctrl_state_idle);
-		if (IS_ERR(mux->state_idle)) {
-			ret = PTR_ERR(mux->state_idle);
-			dev_err(&pdev->dev,
-				"Cannot look up pinctrl state %s: %d\n",
-				mux->pdata->pinctrl_state_idle, ret);
-			goto err;
+			dev_err(dev, "Cannot look up pinctrl state %s: %d\n",
+				name, ret);
+			goto err_put_parent;
 		}
 
-		muxc->deselect = i2c_mux_pinctrl_deselect;
-	}
+		if (strcmp(name, "idle"))
+			continue;
 
-	muxc->parent = i2c_get_adapter(mux->pdata->parent_bus_num);
-	if (!muxc->parent) {
-		dev_err(&pdev->dev, "Parent adapter (%d) not found\n",
-			mux->pdata->parent_bus_num);
-		ret = -EPROBE_DEFER;
-		goto err;
+		if (i != num_names - 1) {
+			dev_err(dev, "idle state must be last\n");
+			ret = -EINVAL;
+			goto err_put_parent;
+		}
+		muxc->deselect = i2c_mux_pinctrl_deselect;
 	}
 
 	root = i2c_root_adapter(&muxc->parent->dev);
 
 	muxc->mux_locked = true;
-	for (i = 0; i < mux->pdata->bus_count; i++) {
+	for (i = 0; i < num_names; i++) {
 		if (root != i2c_mux_pinctrl_root_adapter(mux->states[i])) {
 			muxc->mux_locked = false;
 			break;
 		}
 	}
-	if (muxc->mux_locked && mux->pdata->pinctrl_state_idle &&
-	    root != i2c_mux_pinctrl_root_adapter(mux->state_idle))
-		muxc->mux_locked = false;
-
 	if (muxc->mux_locked)
-		dev_info(&pdev->dev, "mux-locked i2c mux\n");
+		dev_info(dev, "mux-locked i2c mux\n");
 
-	for (i = 0; i < mux->pdata->bus_count; i++) {
-		u32 bus = mux->pdata->base_bus_num ?
-				(mux->pdata->base_bus_num + i) : 0;
-
-		ret = i2c_mux_add_adapter(muxc, bus, i, 0);
+	/* Do not add any adapter for the idle state (if it's there at all). */
+	for (i = 0; i < num_names - !!muxc->deselect; i++) {
+		ret = i2c_mux_add_adapter(muxc, 0, i, 0);
 		if (ret)
 			goto err_del_adapter;
 	}
@@ -253,8 +172,9 @@ static int i2c_mux_pinctrl_probe(struct platform_device *pdev)
 
 err_del_adapter:
 	i2c_mux_del_adapters(muxc);
-	i2c_put_adapter(muxc->parent);
-err:
+err_put_parent:
+	i2c_put_adapter(parent);
+
 	return ret;
 }
 
@@ -264,16 +184,15 @@ static int i2c_mux_pinctrl_remove(struct platform_device *pdev)
 
 	i2c_mux_del_adapters(muxc);
 	i2c_put_adapter(muxc->parent);
+
 	return 0;
 }
 
-#ifdef CONFIG_OF
 static const struct of_device_id i2c_mux_pinctrl_of_match[] = {
 	{ .compatible = "i2c-mux-pinctrl", },
 	{},
 };
 MODULE_DEVICE_TABLE(of, i2c_mux_pinctrl_of_match);
-#endif
 
 static struct platform_driver i2c_mux_pinctrl_driver = {
 	.driver	= {
diff --git a/drivers/ide/ide-floppy.c b/drivers/ide/ide-floppy.c
index 627b1f62a749..3ddd88219906 100644
--- a/drivers/ide/ide-floppy.c
+++ b/drivers/ide/ide-floppy.c
@@ -72,7 +72,7 @@ static int ide_floppy_callback(ide_drive_t *drive, int dsc)
 		drive->failed_pc = NULL;
 
 	if (pc->c[0] == GPCMD_READ_10 || pc->c[0] == GPCMD_WRITE_10 ||
-	    (req_op(rq) == REQ_OP_SCSI_IN || req_op(rq) == REQ_OP_SCSI_OUT))
+	    blk_rq_is_scsi(rq))
 		uptodate = 1; /* FIXME */
 	else if (pc->c[0] == GPCMD_REQUEST_SENSE) {
 
diff --git a/drivers/ide/pmac.c b/drivers/ide/pmac.c
index 0c5d3a99468e..c5b902b86b44 100644
--- a/drivers/ide/pmac.c
+++ b/drivers/ide/pmac.c
@@ -1145,8 +1145,8 @@ static int pmac_ide_macio_attach(struct macio_dev *mdev,
 		return -ENOMEM;
 
 	if (macio_resource_count(mdev) == 0) {
-		printk(KERN_WARNING "ide-pmac: no address for %s\n",
-				    mdev->ofdev.dev.of_node->full_name);
+		printk(KERN_WARNING "ide-pmac: no address for %pOF\n",
+				    mdev->ofdev.dev.of_node);
 		rc = -ENXIO;
 		goto out_free_pmif;
 	}
@@ -1154,7 +1154,7 @@ static int pmac_ide_macio_attach(struct macio_dev *mdev,
 	/* Request memory resource for IO ports */
 	if (macio_request_resource(mdev, 0, "ide-pmac (ports)")) {
 		printk(KERN_ERR "ide-pmac: can't request MMIO resource for "
-				"%s!\n", mdev->ofdev.dev.of_node->full_name);
+				"%pOF!\n", mdev->ofdev.dev.of_node);
 		rc = -EBUSY;
 		goto out_free_pmif;
 	}
@@ -1165,8 +1165,8 @@ static int pmac_ide_macio_attach(struct macio_dev *mdev,
 	 * where that happens though...
 	 */
 	if (macio_irq_count(mdev) == 0) {
-		printk(KERN_WARNING "ide-pmac: no intrs for device %s, using "
-				    "13\n", mdev->ofdev.dev.of_node->full_name);
+		printk(KERN_WARNING "ide-pmac: no intrs for device %pOF, using "
+				    "13\n", mdev->ofdev.dev.of_node);
 		irq = irq_create_mapping(NULL, 13);
 	} else
 		irq = macio_irq(mdev, 0);
@@ -1183,8 +1183,8 @@ static int pmac_ide_macio_attach(struct macio_dev *mdev,
 	if (macio_resource_count(mdev) >= 2) {
 		if (macio_request_resource(mdev, 1, "ide-pmac (dma)"))
 			printk(KERN_WARNING "ide-pmac: can't request DMA "
-					    "resource for %s!\n",
-					    mdev->ofdev.dev.of_node->full_name);
+					    "resource for %pOF!\n",
+					    mdev->ofdev.dev.of_node);
 		else
 			pmif->dma_regs = ioremap(macio_resource_start(mdev, 1), 0x1000);
 	} else
@@ -1274,7 +1274,7 @@ static int pmac_ide_pci_attach(struct pci_dev *pdev,
 
 	if (pci_enable_device(pdev)) {
 		printk(KERN_WARNING "ide-pmac: Can't enable PCI device for "
-				    "%s\n", np->full_name);
+				    "%pOF\n", np);
 		rc = -ENXIO;
 		goto out_free_pmif;
 	}
@@ -1282,7 +1282,7 @@ static int pmac_ide_pci_attach(struct pci_dev *pdev,
 			
 	if (pci_request_regions(pdev, "Kauai ATA")) {
 		printk(KERN_ERR "ide-pmac: Cannot obtain PCI resources for "
-				"%s\n", np->full_name);
+				"%pOF\n", np);
 		rc = -ENXIO;
 		goto out_free_pmif;
 	}
diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c
index c2ae819a871c..5dc7ea4b6bc4 100644
--- a/drivers/idle/intel_idle.c
+++ b/drivers/idle/intel_idle.c
@@ -97,7 +97,7 @@ static const struct idle_cpu *icpu;
 static struct cpuidle_device __percpu *intel_idle_cpuidle_devices;
 static int intel_idle(struct cpuidle_device *dev,
 			struct cpuidle_driver *drv, int index);
-static void intel_idle_freeze(struct cpuidle_device *dev,
+static void intel_idle_s2idle(struct cpuidle_device *dev,
 			      struct cpuidle_driver *drv, int index);
 static struct cpuidle_state *cpuidle_state_table;
 
@@ -132,7 +132,7 @@ static struct cpuidle_state nehalem_cstates[] = {
 		.exit_latency = 3,
 		.target_residency = 6,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C1E",
 		.desc = "MWAIT 0x01",
@@ -140,7 +140,7 @@ static struct cpuidle_state nehalem_cstates[] = {
 		.exit_latency = 10,
 		.target_residency = 20,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C3",
 		.desc = "MWAIT 0x10",
@@ -148,7 +148,7 @@ static struct cpuidle_state nehalem_cstates[] = {
 		.exit_latency = 20,
 		.target_residency = 80,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C6",
 		.desc = "MWAIT 0x20",
@@ -156,7 +156,7 @@ static struct cpuidle_state nehalem_cstates[] = {
 		.exit_latency = 200,
 		.target_residency = 800,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.enter = NULL }
 };
@@ -169,7 +169,7 @@ static struct cpuidle_state snb_cstates[] = {
 		.exit_latency = 2,
 		.target_residency = 2,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C1E",
 		.desc = "MWAIT 0x01",
@@ -177,7 +177,7 @@ static struct cpuidle_state snb_cstates[] = {
 		.exit_latency = 10,
 		.target_residency = 20,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C3",
 		.desc = "MWAIT 0x10",
@@ -185,7 +185,7 @@ static struct cpuidle_state snb_cstates[] = {
 		.exit_latency = 80,
 		.target_residency = 211,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C6",
 		.desc = "MWAIT 0x20",
@@ -193,7 +193,7 @@ static struct cpuidle_state snb_cstates[] = {
 		.exit_latency = 104,
 		.target_residency = 345,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C7",
 		.desc = "MWAIT 0x30",
@@ -201,7 +201,7 @@ static struct cpuidle_state snb_cstates[] = {
 		.exit_latency = 109,
 		.target_residency = 345,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.enter = NULL }
 };
@@ -214,7 +214,7 @@ static struct cpuidle_state byt_cstates[] = {
 		.exit_latency = 1,
 		.target_residency = 1,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C6N",
 		.desc = "MWAIT 0x58",
@@ -222,7 +222,7 @@ static struct cpuidle_state byt_cstates[] = {
 		.exit_latency = 300,
 		.target_residency = 275,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C6S",
 		.desc = "MWAIT 0x52",
@@ -230,7 +230,7 @@ static struct cpuidle_state byt_cstates[] = {
 		.exit_latency = 500,
 		.target_residency = 560,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C7",
 		.desc = "MWAIT 0x60",
@@ -238,7 +238,7 @@ static struct cpuidle_state byt_cstates[] = {
 		.exit_latency = 1200,
 		.target_residency = 4000,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C7S",
 		.desc = "MWAIT 0x64",
@@ -246,7 +246,7 @@ static struct cpuidle_state byt_cstates[] = {
 		.exit_latency = 10000,
 		.target_residency = 20000,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.enter = NULL }
 };
@@ -259,7 +259,7 @@ static struct cpuidle_state cht_cstates[] = {
 		.exit_latency = 1,
 		.target_residency = 1,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C6N",
 		.desc = "MWAIT 0x58",
@@ -267,7 +267,7 @@ static struct cpuidle_state cht_cstates[] = {
 		.exit_latency = 80,
 		.target_residency = 275,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C6S",
 		.desc = "MWAIT 0x52",
@@ -275,7 +275,7 @@ static struct cpuidle_state cht_cstates[] = {
 		.exit_latency = 200,
 		.target_residency = 560,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C7",
 		.desc = "MWAIT 0x60",
@@ -283,7 +283,7 @@ static struct cpuidle_state cht_cstates[] = {
 		.exit_latency = 1200,
 		.target_residency = 4000,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C7S",
 		.desc = "MWAIT 0x64",
@@ -291,7 +291,7 @@ static struct cpuidle_state cht_cstates[] = {
 		.exit_latency = 10000,
 		.target_residency = 20000,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.enter = NULL }
 };
@@ -304,7 +304,7 @@ static struct cpuidle_state ivb_cstates[] = {
 		.exit_latency = 1,
 		.target_residency = 1,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C1E",
 		.desc = "MWAIT 0x01",
@@ -312,7 +312,7 @@ static struct cpuidle_state ivb_cstates[] = {
 		.exit_latency = 10,
 		.target_residency = 20,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C3",
 		.desc = "MWAIT 0x10",
@@ -320,7 +320,7 @@ static struct cpuidle_state ivb_cstates[] = {
 		.exit_latency = 59,
 		.target_residency = 156,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C6",
 		.desc = "MWAIT 0x20",
@@ -328,7 +328,7 @@ static struct cpuidle_state ivb_cstates[] = {
 		.exit_latency = 80,
 		.target_residency = 300,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C7",
 		.desc = "MWAIT 0x30",
@@ -336,7 +336,7 @@ static struct cpuidle_state ivb_cstates[] = {
 		.exit_latency = 87,
 		.target_residency = 300,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.enter = NULL }
 };
@@ -349,7 +349,7 @@ static struct cpuidle_state ivt_cstates[] = {
 		.exit_latency = 1,
 		.target_residency = 1,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C1E",
 		.desc = "MWAIT 0x01",
@@ -357,7 +357,7 @@ static struct cpuidle_state ivt_cstates[] = {
 		.exit_latency = 10,
 		.target_residency = 80,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C3",
 		.desc = "MWAIT 0x10",
@@ -365,7 +365,7 @@ static struct cpuidle_state ivt_cstates[] = {
 		.exit_latency = 59,
 		.target_residency = 156,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C6",
 		.desc = "MWAIT 0x20",
@@ -373,7 +373,7 @@ static struct cpuidle_state ivt_cstates[] = {
 		.exit_latency = 82,
 		.target_residency = 300,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.enter = NULL }
 };
@@ -386,7 +386,7 @@ static struct cpuidle_state ivt_cstates_4s[] = {
 		.exit_latency = 1,
 		.target_residency = 1,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C1E",
 		.desc = "MWAIT 0x01",
@@ -394,7 +394,7 @@ static struct cpuidle_state ivt_cstates_4s[] = {
 		.exit_latency = 10,
 		.target_residency = 250,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C3",
 		.desc = "MWAIT 0x10",
@@ -402,7 +402,7 @@ static struct cpuidle_state ivt_cstates_4s[] = {
 		.exit_latency = 59,
 		.target_residency = 300,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C6",
 		.desc = "MWAIT 0x20",
@@ -410,7 +410,7 @@ static struct cpuidle_state ivt_cstates_4s[] = {
 		.exit_latency = 84,
 		.target_residency = 400,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.enter = NULL }
 };
@@ -423,7 +423,7 @@ static struct cpuidle_state ivt_cstates_8s[] = {
 		.exit_latency = 1,
 		.target_residency = 1,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C1E",
 		.desc = "MWAIT 0x01",
@@ -431,7 +431,7 @@ static struct cpuidle_state ivt_cstates_8s[] = {
 		.exit_latency = 10,
 		.target_residency = 500,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C3",
 		.desc = "MWAIT 0x10",
@@ -439,7 +439,7 @@ static struct cpuidle_state ivt_cstates_8s[] = {
 		.exit_latency = 59,
 		.target_residency = 600,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C6",
 		.desc = "MWAIT 0x20",
@@ -447,7 +447,7 @@ static struct cpuidle_state ivt_cstates_8s[] = {
 		.exit_latency = 88,
 		.target_residency = 700,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.enter = NULL }
 };
@@ -460,7 +460,7 @@ static struct cpuidle_state hsw_cstates[] = {
 		.exit_latency = 2,
 		.target_residency = 2,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C1E",
 		.desc = "MWAIT 0x01",
@@ -468,7 +468,7 @@ static struct cpuidle_state hsw_cstates[] = {
 		.exit_latency = 10,
 		.target_residency = 20,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C3",
 		.desc = "MWAIT 0x10",
@@ -476,7 +476,7 @@ static struct cpuidle_state hsw_cstates[] = {
 		.exit_latency = 33,
 		.target_residency = 100,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C6",
 		.desc = "MWAIT 0x20",
@@ -484,7 +484,7 @@ static struct cpuidle_state hsw_cstates[] = {
 		.exit_latency = 133,
 		.target_residency = 400,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C7s",
 		.desc = "MWAIT 0x32",
@@ -492,7 +492,7 @@ static struct cpuidle_state hsw_cstates[] = {
 		.exit_latency = 166,
 		.target_residency = 500,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C8",
 		.desc = "MWAIT 0x40",
@@ -500,7 +500,7 @@ static struct cpuidle_state hsw_cstates[] = {
 		.exit_latency = 300,
 		.target_residency = 900,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C9",
 		.desc = "MWAIT 0x50",
@@ -508,7 +508,7 @@ static struct cpuidle_state hsw_cstates[] = {
 		.exit_latency = 600,
 		.target_residency = 1800,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C10",
 		.desc = "MWAIT 0x60",
@@ -516,7 +516,7 @@ static struct cpuidle_state hsw_cstates[] = {
 		.exit_latency = 2600,
 		.target_residency = 7700,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.enter = NULL }
 };
@@ -528,7 +528,7 @@ static struct cpuidle_state bdw_cstates[] = {
 		.exit_latency = 2,
 		.target_residency = 2,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C1E",
 		.desc = "MWAIT 0x01",
@@ -536,7 +536,7 @@ static struct cpuidle_state bdw_cstates[] = {
 		.exit_latency = 10,
 		.target_residency = 20,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C3",
 		.desc = "MWAIT 0x10",
@@ -544,7 +544,7 @@ static struct cpuidle_state bdw_cstates[] = {
 		.exit_latency = 40,
 		.target_residency = 100,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C6",
 		.desc = "MWAIT 0x20",
@@ -552,7 +552,7 @@ static struct cpuidle_state bdw_cstates[] = {
 		.exit_latency = 133,
 		.target_residency = 400,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C7s",
 		.desc = "MWAIT 0x32",
@@ -560,7 +560,7 @@ static struct cpuidle_state bdw_cstates[] = {
 		.exit_latency = 166,
 		.target_residency = 500,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C8",
 		.desc = "MWAIT 0x40",
@@ -568,7 +568,7 @@ static struct cpuidle_state bdw_cstates[] = {
 		.exit_latency = 300,
 		.target_residency = 900,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C9",
 		.desc = "MWAIT 0x50",
@@ -576,7 +576,7 @@ static struct cpuidle_state bdw_cstates[] = {
 		.exit_latency = 600,
 		.target_residency = 1800,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C10",
 		.desc = "MWAIT 0x60",
@@ -584,7 +584,7 @@ static struct cpuidle_state bdw_cstates[] = {
 		.exit_latency = 2600,
 		.target_residency = 7700,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.enter = NULL }
 };
@@ -597,7 +597,7 @@ static struct cpuidle_state skl_cstates[] = {
 		.exit_latency = 2,
 		.target_residency = 2,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C1E",
 		.desc = "MWAIT 0x01",
@@ -605,7 +605,7 @@ static struct cpuidle_state skl_cstates[] = {
 		.exit_latency = 10,
 		.target_residency = 20,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C3",
 		.desc = "MWAIT 0x10",
@@ -613,7 +613,7 @@ static struct cpuidle_state skl_cstates[] = {
 		.exit_latency = 70,
 		.target_residency = 100,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C6",
 		.desc = "MWAIT 0x20",
@@ -621,7 +621,7 @@ static struct cpuidle_state skl_cstates[] = {
 		.exit_latency = 85,
 		.target_residency = 200,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C7s",
 		.desc = "MWAIT 0x33",
@@ -629,7 +629,7 @@ static struct cpuidle_state skl_cstates[] = {
 		.exit_latency = 124,
 		.target_residency = 800,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C8",
 		.desc = "MWAIT 0x40",
@@ -637,7 +637,7 @@ static struct cpuidle_state skl_cstates[] = {
 		.exit_latency = 200,
 		.target_residency = 800,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C9",
 		.desc = "MWAIT 0x50",
@@ -645,7 +645,7 @@ static struct cpuidle_state skl_cstates[] = {
 		.exit_latency = 480,
 		.target_residency = 5000,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C10",
 		.desc = "MWAIT 0x60",
@@ -653,7 +653,7 @@ static struct cpuidle_state skl_cstates[] = {
 		.exit_latency = 890,
 		.target_residency = 5000,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.enter = NULL }
 };
@@ -666,7 +666,7 @@ static struct cpuidle_state skx_cstates[] = {
 		.exit_latency = 2,
 		.target_residency = 2,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C1E",
 		.desc = "MWAIT 0x01",
@@ -674,7 +674,7 @@ static struct cpuidle_state skx_cstates[] = {
 		.exit_latency = 10,
 		.target_residency = 20,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C6",
 		.desc = "MWAIT 0x20",
@@ -682,7 +682,7 @@ static struct cpuidle_state skx_cstates[] = {
 		.exit_latency = 133,
 		.target_residency = 600,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.enter = NULL }
 };
@@ -695,7 +695,7 @@ static struct cpuidle_state atom_cstates[] = {
 		.exit_latency = 10,
 		.target_residency = 20,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C2",
 		.desc = "MWAIT 0x10",
@@ -703,7 +703,7 @@ static struct cpuidle_state atom_cstates[] = {
 		.exit_latency = 20,
 		.target_residency = 80,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C4",
 		.desc = "MWAIT 0x30",
@@ -711,7 +711,7 @@ static struct cpuidle_state atom_cstates[] = {
 		.exit_latency = 100,
 		.target_residency = 400,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C6",
 		.desc = "MWAIT 0x52",
@@ -719,7 +719,7 @@ static struct cpuidle_state atom_cstates[] = {
 		.exit_latency = 140,
 		.target_residency = 560,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.enter = NULL }
 };
@@ -731,7 +731,7 @@ static struct cpuidle_state tangier_cstates[] = {
 		.exit_latency = 1,
 		.target_residency = 4,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C4",
 		.desc = "MWAIT 0x30",
@@ -739,7 +739,7 @@ static struct cpuidle_state tangier_cstates[] = {
 		.exit_latency = 100,
 		.target_residency = 400,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C6",
 		.desc = "MWAIT 0x52",
@@ -747,7 +747,7 @@ static struct cpuidle_state tangier_cstates[] = {
 		.exit_latency = 140,
 		.target_residency = 560,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C7",
 		.desc = "MWAIT 0x60",
@@ -755,7 +755,7 @@ static struct cpuidle_state tangier_cstates[] = {
 		.exit_latency = 1200,
 		.target_residency = 4000,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C9",
 		.desc = "MWAIT 0x64",
@@ -763,7 +763,7 @@ static struct cpuidle_state tangier_cstates[] = {
 		.exit_latency = 10000,
 		.target_residency = 20000,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.enter = NULL }
 };
@@ -775,7 +775,7 @@ static struct cpuidle_state avn_cstates[] = {
 		.exit_latency = 2,
 		.target_residency = 2,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C6",
 		.desc = "MWAIT 0x51",
@@ -783,7 +783,7 @@ static struct cpuidle_state avn_cstates[] = {
 		.exit_latency = 15,
 		.target_residency = 45,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.enter = NULL }
 };
@@ -795,7 +795,7 @@ static struct cpuidle_state knl_cstates[] = {
 		.exit_latency = 1,
 		.target_residency = 2,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze },
+		.enter_s2idle = intel_idle_s2idle },
 	{
 		.name = "C6",
 		.desc = "MWAIT 0x10",
@@ -803,7 +803,7 @@ static struct cpuidle_state knl_cstates[] = {
 		.exit_latency = 120,
 		.target_residency = 500,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze },
+		.enter_s2idle = intel_idle_s2idle },
 	{
 		.enter = NULL }
 };
@@ -816,7 +816,7 @@ static struct cpuidle_state bxt_cstates[] = {
 		.exit_latency = 2,
 		.target_residency = 2,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C1E",
 		.desc = "MWAIT 0x01",
@@ -824,7 +824,7 @@ static struct cpuidle_state bxt_cstates[] = {
 		.exit_latency = 10,
 		.target_residency = 20,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C6",
 		.desc = "MWAIT 0x20",
@@ -832,7 +832,7 @@ static struct cpuidle_state bxt_cstates[] = {
 		.exit_latency = 133,
 		.target_residency = 133,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C7s",
 		.desc = "MWAIT 0x31",
@@ -840,7 +840,7 @@ static struct cpuidle_state bxt_cstates[] = {
 		.exit_latency = 155,
 		.target_residency = 155,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C8",
 		.desc = "MWAIT 0x40",
@@ -848,7 +848,7 @@ static struct cpuidle_state bxt_cstates[] = {
 		.exit_latency = 1000,
 		.target_residency = 1000,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C9",
 		.desc = "MWAIT 0x50",
@@ -856,7 +856,7 @@ static struct cpuidle_state bxt_cstates[] = {
 		.exit_latency = 2000,
 		.target_residency = 2000,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C10",
 		.desc = "MWAIT 0x60",
@@ -864,7 +864,7 @@ static struct cpuidle_state bxt_cstates[] = {
 		.exit_latency = 10000,
 		.target_residency = 10000,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.enter = NULL }
 };
@@ -877,7 +877,7 @@ static struct cpuidle_state dnv_cstates[] = {
 		.exit_latency = 2,
 		.target_residency = 2,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C1E",
 		.desc = "MWAIT 0x01",
@@ -885,7 +885,7 @@ static struct cpuidle_state dnv_cstates[] = {
 		.exit_latency = 10,
 		.target_residency = 20,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.name = "C6",
 		.desc = "MWAIT 0x20",
@@ -893,7 +893,7 @@ static struct cpuidle_state dnv_cstates[] = {
 		.exit_latency = 50,
 		.target_residency = 500,
 		.enter = &intel_idle,
-		.enter_freeze = intel_idle_freeze, },
+		.enter_s2idle = intel_idle_s2idle, },
 	{
 		.enter = NULL }
 };
@@ -913,16 +913,15 @@ static __cpuidle int intel_idle(struct cpuidle_device *dev,
 	struct cpuidle_state *state = &drv->states[index];
 	unsigned long eax = flg2MWAIT(state->flags);
 	unsigned int cstate;
-	int cpu = smp_processor_id();
 
 	cstate = (((eax) >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK) + 1;
 
 	/*
-	 * leave_mm() to avoid costly and often unnecessary wakeups
-	 * for flushing the user TLB's associated with the active mm.
+	 * NB: if CPUIDLE_FLAG_TLB_FLUSHED is set, this idle transition
+	 * will probably flush the TLB.  It's not guaranteed to flush
+	 * the TLB, though, so it's not clear that we can do anything
+	 * useful with this knowledge.
 	 */
-	if (state->flags & CPUIDLE_FLAG_TLB_FLUSHED)
-		leave_mm(cpu);
 
 	if (!(lapic_timer_reliable_states & (1 << (cstate))))
 		tick_broadcast_enter();
@@ -936,12 +935,12 @@ static __cpuidle int intel_idle(struct cpuidle_device *dev,
 }
 
 /**
- * intel_idle_freeze - simplified "enter" callback routine for suspend-to-idle
+ * intel_idle_s2idle - simplified "enter" callback routine for suspend-to-idle
  * @dev: cpuidle_device
  * @drv: cpuidle driver
  * @index: state index
  */
-static void intel_idle_freeze(struct cpuidle_device *dev,
+static void intel_idle_s2idle(struct cpuidle_device *dev,
 			     struct cpuidle_driver *drv, int index)
 {
 	unsigned long ecx = 1; /* break on interrupt flag */
@@ -1331,13 +1330,14 @@ static void __init intel_idle_cpuidle_driver_init(void)
 
 	intel_idle_state_table_update();
 
+	cpuidle_poll_state_init(drv);
 	drv->state_count = 1;
 
 	for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) {
 		int num_substates, mwait_hint, mwait_cstate;
 
 		if ((cpuidle_state_table[cstate].enter == NULL) &&
-		    (cpuidle_state_table[cstate].enter_freeze == NULL))
+		    (cpuidle_state_table[cstate].enter_s2idle == NULL))
 			break;
 
 		if (cstate + 1 > max_cstate) {
diff --git a/drivers/iio/accel/bma180.c b/drivers/iio/accel/bma180.c
index efc67739c28f..3dec972ca672 100644
--- a/drivers/iio/accel/bma180.c
+++ b/drivers/iio/accel/bma180.c
@@ -842,7 +842,7 @@ static SIMPLE_DEV_PM_OPS(bma180_pm_ops, bma180_suspend, bma180_resume);
 #define BMA180_PM_OPS NULL
 #endif
 
-static struct i2c_device_id bma180_ids[] = {
+static const struct i2c_device_id bma180_ids[] = {
 	{ "bma180", BMA180 },
 	{ "bma250", BMA250 },
 	{ }
diff --git a/drivers/iio/accel/bmc150-accel-i2c.c b/drivers/iio/accel/bmc150-accel-i2c.c
index 8ca8041267ef..f85014fbaa12 100644
--- a/drivers/iio/accel/bmc150-accel-i2c.c
+++ b/drivers/iio/accel/bmc150-accel-i2c.c
@@ -64,6 +64,7 @@ static const struct acpi_device_id bmc150_accel_acpi_match[] = {
 	{"BMA250E",	bma250e},
 	{"BMA222E",	bma222e},
 	{"BMA0280",	bma280},
+	{"BOSC0200"},
 	{ },
 };
 MODULE_DEVICE_TABLE(acpi, bmc150_accel_acpi_match);
diff --git a/drivers/iio/accel/da311.c b/drivers/iio/accel/da311.c
index 537cfa8b6edf..c0c1620d2a2f 100644
--- a/drivers/iio/accel/da311.c
+++ b/drivers/iio/accel/da311.c
@@ -139,7 +139,7 @@ static int da311_register_mask_write(struct i2c_client *client, u16 addr,
 /* Init sequence taken from the android driver */
 static int da311_reset(struct i2c_client *client)
 {
-	const struct {
+	static const struct {
 		u16 addr;
 		u8 mask;
 		u8 data;
diff --git a/drivers/iio/accel/sca3000.c b/drivers/iio/accel/sca3000.c
index cb1d83fa19a0..39ab210c44f6 100644
--- a/drivers/iio/accel/sca3000.c
+++ b/drivers/iio/accel/sca3000.c
@@ -36,7 +36,7 @@
 #define   SCA3000_LOCKED				BIT(5)
 #define   SCA3000_EEPROM_CS_ERROR			BIT(1)
 #define   SCA3000_SPI_FRAME_ERROR			BIT(0)
- 
+
 /* All reads done using register decrement so no need to directly access LSBs */
 #define SCA3000_REG_X_MSB_ADDR				0x05
 #define SCA3000_REG_Y_MSB_ADDR				0x07
@@ -74,7 +74,7 @@
 #define SCA3000_REG_INT_STATUS_ADDR			0x16
 #define   SCA3000_REG_INT_STATUS_THREE_QUARTERS		BIT(7)
 #define   SCA3000_REG_INT_STATUS_HALF			BIT(6)
-	
+
 #define SCA3000_INT_STATUS_FREE_FALL			BIT(3)
 #define SCA3000_INT_STATUS_Y_TRIGGER			BIT(2)
 #define SCA3000_INT_STATUS_X_TRIGGER			BIT(1)
@@ -124,7 +124,7 @@
 
 #define SCA3000_REG_INT_MASK_ADDR			0x21
 #define   SCA3000_REG_INT_MASK_PROT_MASK		0x1C
- 
+
 #define   SCA3000_REG_INT_MASK_RING_THREE_QUARTER	BIT(7)
 #define   SCA3000_REG_INT_MASK_RING_HALF		BIT(6)
 
diff --git a/drivers/iio/accel/st_accel.h b/drivers/iio/accel/st_accel.h
index 3ad44ce7ae82..0fe521609a3a 100644
--- a/drivers/iio/accel/st_accel.h
+++ b/drivers/iio/accel/st_accel.h
@@ -29,10 +29,13 @@ enum st_accel_type {
 	LIS2DH12,
 	LIS3L02DQ,
 	LNG2DM,
+	H3LIS331DL,
+	LIS331DL,
+	LIS3LV02DL,
 	ST_ACCEL_MAX,
 };
 
-#define H3LIS331DL_DRIVER_NAME		"h3lis331dl_accel"
+#define H3LIS331DL_ACCEL_DEV_NAME	"h3lis331dl_accel"
 #define LIS3LV02DL_ACCEL_DEV_NAME	"lis3lv02dl_accel"
 #define LSM303DLHC_ACCEL_DEV_NAME	"lsm303dlhc_accel"
 #define LIS3DH_ACCEL_DEV_NAME		"lis3dh"
diff --git a/drivers/iio/accel/st_accel_core.c b/drivers/iio/accel/st_accel_core.c
index e44f62bf9caa..752856b3a849 100644
--- a/drivers/iio/accel/st_accel_core.c
+++ b/drivers/iio/accel/st_accel_core.c
@@ -161,7 +161,7 @@ static const struct st_sensor_settings st_accel_sensors_settings[] = {
 		.drdy_irq = {
 			.addr = 0x22,
 			.mask_int1 = 0x10,
-			.mask_int2 = 0x08,
+			.mask_int2 = 0x00,
 			.addr_ihl = 0x25,
 			.mask_ihl = 0x02,
 			.addr_stat_drdy = ST_SENSORS_DEFAULT_STAT_ADDR,
@@ -464,7 +464,7 @@ static const struct st_sensor_settings st_accel_sensors_settings[] = {
 		.wai = 0x32,
 		.wai_addr = ST_SENSORS_DEFAULT_WAI_ADDRESS,
 		.sensors_supported = {
-			[0] = H3LIS331DL_DRIVER_NAME,
+			[0] = H3LIS331DL_ACCEL_DEV_NAME,
 		},
 		.ch = (struct iio_chan_spec *)st_accel_12bit_channels,
 		.odr = {
@@ -637,7 +637,7 @@ static const struct st_sensor_settings st_accel_sensors_settings[] = {
 		.drdy_irq = {
 			.addr = 0x22,
 			.mask_int1 = 0x10,
-			.mask_int2 = 0x08,
+			.mask_int2 = 0x00,
 			.addr_ihl = 0x25,
 			.mask_ihl = 0x02,
 			.addr_stat_drdy = ST_SENSORS_DEFAULT_STAT_ADDR,
diff --git a/drivers/iio/accel/st_accel_i2c.c b/drivers/iio/accel/st_accel_i2c.c
index 543f0ad7fd7e..18cafb9f2468 100644
--- a/drivers/iio/accel/st_accel_i2c.c
+++ b/drivers/iio/accel/st_accel_i2c.c
@@ -84,7 +84,7 @@ static const struct of_device_id st_accel_of_match[] = {
 	},
 	{
 		.compatible = "st,h3lis331dl-accel",
-		.data = H3LIS331DL_DRIVER_NAME,
+		.data = H3LIS331DL_ACCEL_DEV_NAME,
 	},
 	{
 		.compatible = "st,lis3l02dq",
@@ -126,6 +126,9 @@ static const struct i2c_device_id st_accel_id_table[] = {
 	{ LIS2DH12_ACCEL_DEV_NAME, LIS2DH12 },
 	{ LIS3L02DQ_ACCEL_DEV_NAME, LIS3L02DQ },
 	{ LNG2DM_ACCEL_DEV_NAME, LNG2DM },
+	{ H3LIS331DL_ACCEL_DEV_NAME, H3LIS331DL },
+	{ LIS331DL_ACCEL_DEV_NAME, LIS331DL },
+	{ LIS3LV02DL_ACCEL_DEV_NAME, LIS3LV02DL },
 	{},
 };
 MODULE_DEVICE_TABLE(i2c, st_accel_id_table);
@@ -144,7 +147,8 @@ static int st_accel_i2c_probe(struct i2c_client *client,
 	adata = iio_priv(indio_dev);
 
 	if (client->dev.of_node) {
-		st_sensors_of_i2c_probe(client, st_accel_of_match);
+		st_sensors_of_name_probe(&client->dev, st_accel_of_match,
+					 client->name, sizeof(client->name));
 	} else if (ACPI_HANDLE(&client->dev)) {
 		ret = st_sensors_match_acpi_device(&client->dev);
 		if ((ret < 0) || (ret >= ST_ACCEL_MAX))
diff --git a/drivers/iio/accel/st_accel_spi.c b/drivers/iio/accel/st_accel_spi.c
index 1a867f5563a4..915fa49085f7 100644
--- a/drivers/iio/accel/st_accel_spi.c
+++ b/drivers/iio/accel/st_accel_spi.c
@@ -18,6 +18,77 @@
 #include <linux/iio/common/st_sensors_spi.h>
 #include "st_accel.h"
 
+#ifdef CONFIG_OF
+/*
+ * For new single-chip sensors use <device_name> as compatible string.
+ * For old single-chip devices keep <device_name>-accel to maintain
+ * compatibility
+ */
+static const struct of_device_id st_accel_of_match[] = {
+	{
+		/* An older compatible */
+		.compatible = "st,lis302dl-spi",
+		.data = LIS3LV02DL_ACCEL_DEV_NAME,
+	},
+	{
+		.compatible = "st,lis3lv02dl-accel",
+		.data = LIS3LV02DL_ACCEL_DEV_NAME,
+	},
+	{
+		.compatible = "st,lis3dh-accel",
+		.data = LIS3DH_ACCEL_DEV_NAME,
+	},
+	{
+		.compatible = "st,lsm330d-accel",
+		.data = LSM330D_ACCEL_DEV_NAME,
+	},
+	{
+		.compatible = "st,lsm330dl-accel",
+		.data = LSM330DL_ACCEL_DEV_NAME,
+	},
+	{
+		.compatible = "st,lsm330dlc-accel",
+		.data = LSM330DLC_ACCEL_DEV_NAME,
+	},
+	{
+		.compatible = "st,lis331dlh-accel",
+		.data = LIS331DLH_ACCEL_DEV_NAME,
+	},
+	{
+		.compatible = "st,lsm330-accel",
+		.data = LSM330_ACCEL_DEV_NAME,
+	},
+	{
+		.compatible = "st,lsm303agr-accel",
+		.data = LSM303AGR_ACCEL_DEV_NAME,
+	},
+	{
+		.compatible = "st,lis2dh12-accel",
+		.data = LIS2DH12_ACCEL_DEV_NAME,
+	},
+	{
+		.compatible = "st,lis3l02dq",
+		.data = LIS3L02DQ_ACCEL_DEV_NAME,
+	},
+	{
+		.compatible = "st,lng2dm-accel",
+		.data = LNG2DM_ACCEL_DEV_NAME,
+	},
+	{
+		.compatible = "st,h3lis331dl-accel",
+		.data = H3LIS331DL_ACCEL_DEV_NAME,
+	},
+	{
+		.compatible = "st,lis331dl-accel",
+		.data = LIS331DL_ACCEL_DEV_NAME,
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(of, st_accel_of_match);
+#else
+#define st_accel_of_match	NULL
+#endif
+
 static int st_accel_spi_probe(struct spi_device *spi)
 {
 	struct iio_dev *indio_dev;
@@ -30,6 +101,8 @@ static int st_accel_spi_probe(struct spi_device *spi)
 
 	adata = iio_priv(indio_dev);
 
+	st_sensors_of_name_probe(&spi->dev, st_accel_of_match,
+				 spi->modalias, sizeof(spi->modalias));
 	st_sensors_spi_configure(indio_dev, spi, adata);
 
 	err = st_accel_common_probe(indio_dev);
@@ -57,22 +130,17 @@ static const struct spi_device_id st_accel_id_table[] = {
 	{ LIS2DH12_ACCEL_DEV_NAME },
 	{ LIS3L02DQ_ACCEL_DEV_NAME },
 	{ LNG2DM_ACCEL_DEV_NAME },
+	{ H3LIS331DL_ACCEL_DEV_NAME },
+	{ LIS331DL_ACCEL_DEV_NAME },
+	{ LIS3LV02DL_ACCEL_DEV_NAME },
 	{},
 };
 MODULE_DEVICE_TABLE(spi, st_accel_id_table);
 
-#ifdef CONFIG_OF
-static const struct of_device_id lis302dl_spi_dt_ids[] = {
-	{ .compatible = "st,lis302dl-spi" },
-	{}
-};
-MODULE_DEVICE_TABLE(of, lis302dl_spi_dt_ids);
-#endif
-
 static struct spi_driver st_accel_driver = {
 	.driver = {
 		.name = "st-accel-spi",
-		.of_match_table = of_match_ptr(lis302dl_spi_dt_ids),
+		.of_match_table = of_match_ptr(st_accel_of_match),
 	},
 	.probe = st_accel_spi_probe,
 	.remove = st_accel_spi_remove,
diff --git a/drivers/iio/adc/Kconfig b/drivers/iio/adc/Kconfig
index 614fa41559b1..57625653fcb6 100644
--- a/drivers/iio/adc/Kconfig
+++ b/drivers/iio/adc/Kconfig
@@ -158,6 +158,7 @@ config AT91_SAMA5D2_ADC
 	tristate "Atmel AT91 SAMA5D2 ADC"
 	depends on ARCH_AT91 || COMPILE_TEST
 	depends on HAS_IOMEM
+	select IIO_TRIGGERED_BUFFER
 	help
 	  Say yes here to build support for Atmel SAMA5D2 ADC which is
 	  available on SAMA5D2 SoC family.
@@ -239,6 +240,15 @@ config DA9150_GPADC
 	  To compile this driver as a module, choose M here: the module will be
 	  called berlin2-adc.
 
+config DLN2_ADC
+	tristate "Diolan DLN-2 ADC driver support"
+	depends on MFD_DLN2
+	help
+	  Say yes here to build support for Diolan DLN-2 ADC.
+
+	  This driver can also be built as a module. If so, the module will be
+	  called adc_dln2.
+
 config ENVELOPE_DETECTOR
 	tristate "Envelope detector using a DAC and a comparator"
 	depends on OF
@@ -249,6 +259,17 @@ config ENVELOPE_DETECTOR
 	  To compile this driver as a module, choose M here: the module will be
 	  called envelope-detector.
 
+config EP93XX_ADC
+	tristate "Cirrus Logic EP93XX ADC driver"
+	depends on ARCH_EP93XX
+	help
+	  Driver for the ADC module on the EP93XX series of SoC from Cirrus Logic.
+	  It's recommended to switch on CONFIG_HIGH_RES_TIMERS option, in this
+	  case driver will reduce its CPU usage by 90% in some use cases.
+
+	  To compile this driver as a module, choose M here: the module will be
+	  called ep93xx_adc.
+
 config EXYNOS_ADC
 	tristate "Exynos ADC driver support"
 	depends on ARCH_EXYNOS || ARCH_S3C24XX || ARCH_S3C64XX || (OF && COMPILE_TEST)
@@ -322,7 +343,7 @@ config INA2XX_ADC
 	  This driver is mutually exclusive with the HWMON version.
 
 config IMX7D_ADC
-	tristate "IMX7D ADC driver"
+	tristate "Freescale IMX7D ADC driver"
 	depends on ARCH_MXC || COMPILE_TEST
 	depends on HAS_IOMEM
 	help
@@ -362,6 +383,16 @@ config LPC32XX_ADC
 	  activate only one via device tree selection.  Provides direct access
 	  via sysfs.
 
+config LTC2471
+	tristate "Linear Technology LTC2471 and LTC2473 ADC driver"
+	depends on I2C
+	help
+	  Say yes here to build support for Linear Technology LTC2471 and
+	  LTC2473 16-bit I2C ADC.
+
+	  This driver can also be built as a module. If so, the module will
+	  be called ltc2471.
+
 config LTC2485
 	tristate "Linear Technology LTC2485 ADC driver"
 	depends on I2C
diff --git a/drivers/iio/adc/Makefile b/drivers/iio/adc/Makefile
index b546736a5541..9874e05f52d7 100644
--- a/drivers/iio/adc/Makefile
+++ b/drivers/iio/adc/Makefile
@@ -24,7 +24,9 @@ obj-$(CONFIG_BERLIN2_ADC) += berlin2-adc.o
 obj-$(CONFIG_CC10001_ADC) += cc10001_adc.o
 obj-$(CONFIG_CPCAP_ADC) += cpcap-adc.o
 obj-$(CONFIG_DA9150_GPADC) += da9150-gpadc.o
+obj-$(CONFIG_DLN2_ADC) += dln2-adc.o
 obj-$(CONFIG_ENVELOPE_DETECTOR) += envelope-detector.o
+obj-$(CONFIG_EP93XX_ADC) += ep93xx_adc.o
 obj-$(CONFIG_EXYNOS_ADC) += exynos_adc.o
 obj-$(CONFIG_FSL_MX25_ADC) += fsl-imx25-gcq.o
 obj-$(CONFIG_HI8435) += hi8435.o
@@ -34,6 +36,7 @@ obj-$(CONFIG_INA2XX_ADC) += ina2xx-adc.o
 obj-$(CONFIG_LP8788_ADC) += lp8788_adc.o
 obj-$(CONFIG_LPC18XX_ADC) += lpc18xx_adc.o
 obj-$(CONFIG_LPC32XX_ADC) += lpc32xx_adc.o
+obj-$(CONFIG_LTC2471) += ltc2471.o
 obj-$(CONFIG_LTC2485) += ltc2485.o
 obj-$(CONFIG_LTC2497) += ltc2497.o
 obj-$(CONFIG_MAX1027) += max1027.o
diff --git a/drivers/iio/adc/ad7766.c b/drivers/iio/adc/ad7766.c
index 75cca42b6e70..ce45037295d8 100644
--- a/drivers/iio/adc/ad7766.c
+++ b/drivers/iio/adc/ad7766.c
@@ -103,8 +103,7 @@ static int ad7766_preenable(struct iio_dev *indio_dev)
 		return ret;
 	}
 
-	if (ad7766->pd_gpio)
-		gpiod_set_value(ad7766->pd_gpio, 0);
+	gpiod_set_value(ad7766->pd_gpio, 0);
 
 	return 0;
 }
@@ -113,8 +112,7 @@ static int ad7766_postdisable(struct iio_dev *indio_dev)
 {
 	struct ad7766 *ad7766 = iio_priv(indio_dev);
 
-	if (ad7766->pd_gpio)
-		gpiod_set_value(ad7766->pd_gpio, 1);
+	gpiod_set_value(ad7766->pd_gpio, 1);
 
 	/*
 	 * The PD pin is synchronous to the clock, so give it some time to
diff --git a/drivers/iio/adc/at91-sama5d2_adc.c b/drivers/iio/adc/at91-sama5d2_adc.c
index e10dca3ed74b..bc5b38e3a147 100644
--- a/drivers/iio/adc/at91-sama5d2_adc.c
+++ b/drivers/iio/adc/at91-sama5d2_adc.c
@@ -25,6 +25,11 @@
 #include <linux/wait.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
+#include <linux/iio/buffer.h>
+#include <linux/iio/trigger.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/iio/triggered_buffer.h>
+#include <linux/pinctrl/consumer.h>
 #include <linux/regulator/consumer.h>
 
 /* Control Register */
@@ -132,6 +137,17 @@
 #define AT91_SAMA5D2_PRESSR	0xbc
 /* Trigger Register */
 #define AT91_SAMA5D2_TRGR	0xc0
+/* Mask for TRGMOD field of TRGR register */
+#define AT91_SAMA5D2_TRGR_TRGMOD_MASK GENMASK(2, 0)
+/* No trigger, only software trigger can start conversions */
+#define AT91_SAMA5D2_TRGR_TRGMOD_NO_TRIGGER 0
+/* Trigger Mode external trigger rising edge */
+#define AT91_SAMA5D2_TRGR_TRGMOD_EXT_TRIG_RISE 1
+/* Trigger Mode external trigger falling edge */
+#define AT91_SAMA5D2_TRGR_TRGMOD_EXT_TRIG_FALL 2
+/* Trigger Mode external trigger any edge */
+#define AT91_SAMA5D2_TRGR_TRGMOD_EXT_TRIG_ANY 3
+
 /* Correction Select Register */
 #define AT91_SAMA5D2_COSR	0xd0
 /* Correction Value Register */
@@ -145,14 +161,29 @@
 /* Version Register */
 #define AT91_SAMA5D2_VERSION	0xfc
 
+#define AT91_SAMA5D2_HW_TRIG_CNT 3
+#define AT91_SAMA5D2_SINGLE_CHAN_CNT 12
+#define AT91_SAMA5D2_DIFF_CHAN_CNT 6
+
+/*
+ * Maximum number of bytes to hold conversion from all channels
+ * plus the timestamp
+ */
+#define AT91_BUFFER_MAX_BYTES ((AT91_SAMA5D2_SINGLE_CHAN_CNT +		\
+				AT91_SAMA5D2_DIFF_CHAN_CNT) * 2 + 8)
+
+#define AT91_BUFFER_MAX_HWORDS (AT91_BUFFER_MAX_BYTES / 2)
+
 #define AT91_SAMA5D2_CHAN_SINGLE(num, addr)				\
 	{								\
 		.type = IIO_VOLTAGE,					\
 		.channel = num,						\
 		.address = addr,					\
+		.scan_index = num,					\
 		.scan_type = {						\
 			.sign = 'u',					\
 			.realbits = 12,					\
+			.storagebits = 16,				\
 		},							\
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),		\
 		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),	\
@@ -168,9 +199,11 @@
 		.channel = num,						\
 		.channel2 = num2,					\
 		.address = addr,					\
+		.scan_index = num + AT91_SAMA5D2_SINGLE_CHAN_CNT,	\
 		.scan_type = {						\
 			.sign = 's',					\
 			.realbits = 12,					\
+			.storagebits = 16,				\
 		},							\
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),		\
 		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),	\
@@ -188,6 +221,12 @@ struct at91_adc_soc_info {
 	unsigned			max_sample_rate;
 };
 
+struct at91_adc_trigger {
+	char				*name;
+	unsigned int			trgmod_value;
+	unsigned int			edge_type;
+};
+
 struct at91_adc_state {
 	void __iomem			*base;
 	int				irq;
@@ -195,11 +234,14 @@ struct at91_adc_state {
 	struct regulator		*reg;
 	struct regulator		*vref;
 	int				vref_uv;
+	struct iio_trigger		*trig;
+	const struct at91_adc_trigger	*selected_trig;
 	const struct iio_chan_spec	*chan;
 	bool				conversion_done;
 	u32				conversion_value;
 	struct at91_adc_soc_info	soc_info;
 	wait_queue_head_t		wq_data_available;
+	u16				buffer[AT91_BUFFER_MAX_HWORDS];
 	/*
 	 * lock to prevent concurrent 'single conversion' requests through
 	 * sysfs.
@@ -207,6 +249,24 @@ struct at91_adc_state {
 	struct mutex			lock;
 };
 
+static const struct at91_adc_trigger at91_adc_trigger_list[] = {
+	{
+		.name = "external_rising",
+		.trgmod_value = AT91_SAMA5D2_TRGR_TRGMOD_EXT_TRIG_RISE,
+		.edge_type = IRQ_TYPE_EDGE_RISING,
+	},
+	{
+		.name = "external_falling",
+		.trgmod_value = AT91_SAMA5D2_TRGR_TRGMOD_EXT_TRIG_FALL,
+		.edge_type = IRQ_TYPE_EDGE_FALLING,
+	},
+	{
+		.name = "external_any",
+		.trgmod_value = AT91_SAMA5D2_TRGR_TRGMOD_EXT_TRIG_ANY,
+		.edge_type = IRQ_TYPE_EDGE_BOTH,
+	},
+};
+
 static const struct iio_chan_spec at91_adc_channels[] = {
 	AT91_SAMA5D2_CHAN_SINGLE(0, 0x50),
 	AT91_SAMA5D2_CHAN_SINGLE(1, 0x54),
@@ -226,12 +286,132 @@ static const struct iio_chan_spec at91_adc_channels[] = {
 	AT91_SAMA5D2_CHAN_DIFF(6, 7, 0x68),
 	AT91_SAMA5D2_CHAN_DIFF(8, 9, 0x70),
 	AT91_SAMA5D2_CHAN_DIFF(10, 11, 0x78),
+	IIO_CHAN_SOFT_TIMESTAMP(AT91_SAMA5D2_SINGLE_CHAN_CNT
+				+ AT91_SAMA5D2_DIFF_CHAN_CNT + 1),
+};
+
+static int at91_adc_configure_trigger(struct iio_trigger *trig, bool state)
+{
+	struct iio_dev *indio = iio_trigger_get_drvdata(trig);
+	struct at91_adc_state *st = iio_priv(indio);
+	u32 status = at91_adc_readl(st, AT91_SAMA5D2_TRGR);
+	u8 bit;
+
+	/* clear TRGMOD */
+	status &= ~AT91_SAMA5D2_TRGR_TRGMOD_MASK;
+
+	if (state)
+		status |= st->selected_trig->trgmod_value;
+
+	/* set/unset hw trigger */
+	at91_adc_writel(st, AT91_SAMA5D2_TRGR, status);
+
+	for_each_set_bit(bit, indio->active_scan_mask, indio->num_channels) {
+		struct iio_chan_spec const *chan = indio->channels + bit;
+
+		if (state) {
+			at91_adc_writel(st, AT91_SAMA5D2_CHER,
+					BIT(chan->channel));
+			at91_adc_writel(st, AT91_SAMA5D2_IER,
+					BIT(chan->channel));
+		} else {
+			at91_adc_writel(st, AT91_SAMA5D2_IDR,
+					BIT(chan->channel));
+			at91_adc_writel(st, AT91_SAMA5D2_CHDR,
+					BIT(chan->channel));
+		}
+	}
+
+	return 0;
+}
+
+static int at91_adc_reenable_trigger(struct iio_trigger *trig)
+{
+	struct iio_dev *indio = iio_trigger_get_drvdata(trig);
+	struct at91_adc_state *st = iio_priv(indio);
+
+	enable_irq(st->irq);
+
+	/* Needed to ACK the DRDY interruption */
+	at91_adc_readl(st, AT91_SAMA5D2_LCDR);
+	return 0;
+}
+
+static const struct iio_trigger_ops at91_adc_trigger_ops = {
+	.owner = THIS_MODULE,
+	.set_trigger_state = &at91_adc_configure_trigger,
+	.try_reenable = &at91_adc_reenable_trigger,
 };
 
+static struct iio_trigger *at91_adc_allocate_trigger(struct iio_dev *indio,
+						     char *trigger_name)
+{
+	struct iio_trigger *trig;
+	int ret;
+
+	trig = devm_iio_trigger_alloc(&indio->dev, "%s-dev%d-%s", indio->name,
+				      indio->id, trigger_name);
+	if (!trig)
+		return NULL;
+
+	trig->dev.parent = indio->dev.parent;
+	iio_trigger_set_drvdata(trig, indio);
+	trig->ops = &at91_adc_trigger_ops;
+
+	ret = devm_iio_trigger_register(&indio->dev, trig);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return trig;
+}
+
+static int at91_adc_trigger_init(struct iio_dev *indio)
+{
+	struct at91_adc_state *st = iio_priv(indio);
+
+	st->trig = at91_adc_allocate_trigger(indio, st->selected_trig->name);
+	if (IS_ERR(st->trig)) {
+		dev_err(&indio->dev,
+			"could not allocate trigger\n");
+		return PTR_ERR(st->trig);
+	}
+
+	return 0;
+}
+
+static irqreturn_t at91_adc_trigger_handler(int irq, void *p)
+{
+	struct iio_poll_func *pf = p;
+	struct iio_dev *indio = pf->indio_dev;
+	struct at91_adc_state *st = iio_priv(indio);
+	int i = 0;
+	u8 bit;
+
+	for_each_set_bit(bit, indio->active_scan_mask, indio->num_channels) {
+		struct iio_chan_spec const *chan = indio->channels + bit;
+
+		st->buffer[i] = at91_adc_readl(st, chan->address);
+		i++;
+	}
+
+	iio_push_to_buffers_with_timestamp(indio, st->buffer, pf->timestamp);
+
+	iio_trigger_notify_done(indio->trig);
+
+	return IRQ_HANDLED;
+}
+
+static int at91_adc_buffer_init(struct iio_dev *indio)
+{
+	return devm_iio_triggered_buffer_setup(&indio->dev, indio,
+			&iio_pollfunc_store_time,
+			&at91_adc_trigger_handler, NULL);
+}
+
 static unsigned at91_adc_startup_time(unsigned startup_time_min,
 				      unsigned adc_clk_khz)
 {
-	const unsigned startup_lookup[] = {
+	static const unsigned int startup_lookup[] = {
 		  0,   8,  16,  24,
 		 64,  80,  96, 112,
 		512, 576, 640, 704,
@@ -293,14 +473,18 @@ static irqreturn_t at91_adc_interrupt(int irq, void *private)
 	u32 status = at91_adc_readl(st, AT91_SAMA5D2_ISR);
 	u32 imr = at91_adc_readl(st, AT91_SAMA5D2_IMR);
 
-	if (status & imr) {
+	if (!(status & imr))
+		return IRQ_NONE;
+
+	if (iio_buffer_enabled(indio)) {
+		disable_irq_nosync(irq);
+		iio_trigger_poll(indio->trig);
+	} else {
 		st->conversion_value = at91_adc_readl(st, st->chan->address);
 		st->conversion_done = true;
 		wake_up_interruptible(&st->wq_data_available);
-		return IRQ_HANDLED;
 	}
-
-	return IRQ_NONE;
+	return IRQ_HANDLED;
 }
 
 static int at91_adc_read_raw(struct iio_dev *indio_dev,
@@ -313,6 +497,11 @@ static int at91_adc_read_raw(struct iio_dev *indio_dev,
 
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
+		/* we cannot use software trigger if hw trigger enabled */
+		ret = iio_device_claim_direct_mode(indio_dev);
+		if (ret)
+			return ret;
+
 		mutex_lock(&st->lock);
 
 		st->chan = chan;
@@ -344,6 +533,8 @@ static int at91_adc_read_raw(struct iio_dev *indio_dev,
 		at91_adc_writel(st, AT91_SAMA5D2_CHDR, BIT(chan->channel));
 
 		mutex_unlock(&st->lock);
+
+		iio_device_release_direct_mode(indio_dev);
 		return ret;
 
 	case IIO_CHAN_INFO_SCALE:
@@ -386,12 +577,27 @@ static const struct iio_info at91_adc_info = {
 	.driver_module = THIS_MODULE,
 };
 
+static void at91_adc_hw_init(struct at91_adc_state *st)
+{
+	at91_adc_writel(st, AT91_SAMA5D2_CR, AT91_SAMA5D2_CR_SWRST);
+	at91_adc_writel(st, AT91_SAMA5D2_IDR, 0xffffffff);
+	/*
+	 * Transfer field must be set to 2 according to the datasheet and
+	 * allows different analog settings for each channel.
+	 */
+	at91_adc_writel(st, AT91_SAMA5D2_MR,
+			AT91_SAMA5D2_MR_TRANSFER(2) | AT91_SAMA5D2_MR_ANACH);
+
+	at91_adc_setup_samp_freq(st, st->soc_info.min_sample_rate);
+}
+
 static int at91_adc_probe(struct platform_device *pdev)
 {
 	struct iio_dev *indio_dev;
 	struct at91_adc_state *st;
 	struct resource	*res;
-	int ret;
+	int ret, i;
+	u32 edge_type;
 
 	indio_dev = devm_iio_device_alloc(&pdev->dev, sizeof(*st));
 	if (!indio_dev)
@@ -432,6 +638,27 @@ static int at91_adc_probe(struct platform_device *pdev)
 		return ret;
 	}
 
+	ret = of_property_read_u32(pdev->dev.of_node,
+				   "atmel,trigger-edge-type", &edge_type);
+	if (ret) {
+		dev_err(&pdev->dev,
+			"invalid or missing value for atmel,trigger-edge-type\n");
+		return ret;
+	}
+
+	st->selected_trig = NULL;
+
+	for (i = 0; i < AT91_SAMA5D2_HW_TRIG_CNT; i++)
+		if (at91_adc_trigger_list[i].edge_type == edge_type) {
+			st->selected_trig = &at91_adc_trigger_list[i];
+			break;
+		}
+
+	if (!st->selected_trig) {
+		dev_err(&pdev->dev, "invalid external trigger edge value\n");
+		return -EINVAL;
+	}
+
 	init_waitqueue_head(&st->wq_data_available);
 	mutex_init(&st->lock);
 
@@ -482,16 +709,7 @@ static int at91_adc_probe(struct platform_device *pdev)
 		goto vref_disable;
 	}
 
-	at91_adc_writel(st, AT91_SAMA5D2_CR, AT91_SAMA5D2_CR_SWRST);
-	at91_adc_writel(st, AT91_SAMA5D2_IDR, 0xffffffff);
-	/*
-	 * Transfer field must be set to 2 according to the datasheet and
-	 * allows different analog settings for each channel.
-	 */
-	at91_adc_writel(st, AT91_SAMA5D2_MR,
-			AT91_SAMA5D2_MR_TRANSFER(2) | AT91_SAMA5D2_MR_ANACH);
-
-	at91_adc_setup_samp_freq(st, st->soc_info.min_sample_rate);
+	at91_adc_hw_init(st);
 
 	ret = clk_prepare_enable(st->per_clk);
 	if (ret)
@@ -499,10 +717,25 @@ static int at91_adc_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, indio_dev);
 
+	ret = at91_adc_buffer_init(indio_dev);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "couldn't initialize the buffer.\n");
+		goto per_clk_disable_unprepare;
+	}
+
+	ret = at91_adc_trigger_init(indio_dev);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "couldn't setup the triggers.\n");
+		goto per_clk_disable_unprepare;
+	}
+
 	ret = iio_device_register(indio_dev);
 	if (ret < 0)
 		goto per_clk_disable_unprepare;
 
+	dev_info(&pdev->dev, "setting up trigger as %s\n",
+		 st->selected_trig->name);
+
 	dev_info(&pdev->dev, "version: %x\n",
 		 readl_relaxed(st->base + AT91_SAMA5D2_VERSION));
 
@@ -532,6 +765,69 @@ static int at91_adc_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static __maybe_unused int at91_adc_suspend(struct device *dev)
+{
+	struct iio_dev *indio_dev =
+			platform_get_drvdata(to_platform_device(dev));
+	struct at91_adc_state *st = iio_priv(indio_dev);
+
+	/*
+	 * Do a sofware reset of the ADC before we go to suspend.
+	 * this will ensure that all pins are free from being muxed by the ADC
+	 * and can be used by for other devices.
+	 * Otherwise, ADC will hog them and we can't go to suspend mode.
+	 */
+	at91_adc_writel(st, AT91_SAMA5D2_CR, AT91_SAMA5D2_CR_SWRST);
+
+	clk_disable_unprepare(st->per_clk);
+	regulator_disable(st->vref);
+	regulator_disable(st->reg);
+
+	return pinctrl_pm_select_sleep_state(dev);
+}
+
+static __maybe_unused int at91_adc_resume(struct device *dev)
+{
+	struct iio_dev *indio_dev =
+			platform_get_drvdata(to_platform_device(dev));
+	struct at91_adc_state *st = iio_priv(indio_dev);
+	int ret;
+
+	ret = pinctrl_pm_select_default_state(dev);
+	if (ret)
+		goto resume_failed;
+
+	ret = regulator_enable(st->reg);
+	if (ret)
+		goto resume_failed;
+
+	ret = regulator_enable(st->vref);
+	if (ret)
+		goto reg_disable_resume;
+
+	ret = clk_prepare_enable(st->per_clk);
+	if (ret)
+		goto vref_disable_resume;
+
+	at91_adc_hw_init(st);
+
+	/* reconfiguring trigger hardware state */
+	if (iio_buffer_enabled(indio_dev))
+		at91_adc_configure_trigger(st->trig, true);
+
+	return 0;
+
+vref_disable_resume:
+	regulator_disable(st->vref);
+reg_disable_resume:
+	regulator_disable(st->reg);
+resume_failed:
+	dev_err(&indio_dev->dev, "failed to resume\n");
+	return ret;
+}
+
+static SIMPLE_DEV_PM_OPS(at91_adc_pm_ops, at91_adc_suspend, at91_adc_resume);
+
 static const struct of_device_id at91_adc_dt_match[] = {
 	{
 		.compatible = "atmel,sama5d2-adc",
@@ -547,6 +843,7 @@ static struct platform_driver at91_adc_driver = {
 	.driver = {
 		.name = "at91-sama5d2_adc",
 		.of_match_table = at91_adc_dt_match,
+		.pm = &at91_adc_pm_ops,
 	},
 };
 module_platform_driver(at91_adc_driver)
diff --git a/drivers/iio/adc/at91_adc.c b/drivers/iio/adc/at91_adc.c
index 34b928cefeed..15109728cae7 100644
--- a/drivers/iio/adc/at91_adc.c
+++ b/drivers/iio/adc/at91_adc.c
@@ -799,7 +799,7 @@ static u32 calc_startup_ticks_9x5(u32 startup_time, u32 adc_clk_khz)
 	 * For sama5d3x and at91sam9x5, the formula changes to:
 	 * Startup Time = <lookup_table_value> / ADC Clock
 	 */
-	const int startup_lookup[] = {
+	static const int startup_lookup[] = {
 		0,   8,   16,  24,
 		64,  80,  96,  112,
 		512, 576, 640, 704,
diff --git a/drivers/iio/adc/dln2-adc.c b/drivers/iio/adc/dln2-adc.c
new file mode 100644
index 000000000000..ab8d6aed5085
--- /dev/null
+++ b/drivers/iio/adc/dln2-adc.c
@@ -0,0 +1,722 @@
+/*
+ * Driver for the Diolan DLN-2 USB-ADC adapter
+ *
+ * Copyright (c) 2017 Jack Andersen
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation, version 2.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/platform_device.h>
+#include <linux/mfd/dln2.h>
+
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+#include <linux/iio/trigger.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/iio/triggered_buffer.h>
+#include <linux/iio/buffer.h>
+#include <linux/iio/kfifo_buf.h>
+
+#define DLN2_ADC_MOD_NAME "dln2-adc"
+
+#define DLN2_ADC_ID             0x06
+
+#define DLN2_ADC_GET_CHANNEL_COUNT	DLN2_CMD(0x01, DLN2_ADC_ID)
+#define DLN2_ADC_ENABLE			DLN2_CMD(0x02, DLN2_ADC_ID)
+#define DLN2_ADC_DISABLE		DLN2_CMD(0x03, DLN2_ADC_ID)
+#define DLN2_ADC_CHANNEL_ENABLE		DLN2_CMD(0x05, DLN2_ADC_ID)
+#define DLN2_ADC_CHANNEL_DISABLE	DLN2_CMD(0x06, DLN2_ADC_ID)
+#define DLN2_ADC_SET_RESOLUTION		DLN2_CMD(0x08, DLN2_ADC_ID)
+#define DLN2_ADC_CHANNEL_GET_VAL	DLN2_CMD(0x0A, DLN2_ADC_ID)
+#define DLN2_ADC_CHANNEL_GET_ALL_VAL	DLN2_CMD(0x0B, DLN2_ADC_ID)
+#define DLN2_ADC_CHANNEL_SET_CFG	DLN2_CMD(0x0C, DLN2_ADC_ID)
+#define DLN2_ADC_CHANNEL_GET_CFG	DLN2_CMD(0x0D, DLN2_ADC_ID)
+#define DLN2_ADC_CONDITION_MET_EV	DLN2_CMD(0x10, DLN2_ADC_ID)
+
+#define DLN2_ADC_EVENT_NONE		0
+#define DLN2_ADC_EVENT_BELOW		1
+#define DLN2_ADC_EVENT_LEVEL_ABOVE	2
+#define DLN2_ADC_EVENT_OUTSIDE		3
+#define DLN2_ADC_EVENT_INSIDE		4
+#define DLN2_ADC_EVENT_ALWAYS		5
+
+#define DLN2_ADC_MAX_CHANNELS 8
+#define DLN2_ADC_DATA_BITS 10
+
+/*
+ * Plays similar role to iio_demux_table in subsystem core; except allocated
+ * in a fixed 8-element array.
+ */
+struct dln2_adc_demux_table {
+	unsigned int from;
+	unsigned int to;
+	unsigned int length;
+};
+
+struct dln2_adc {
+	struct platform_device *pdev;
+	struct iio_chan_spec iio_channels[DLN2_ADC_MAX_CHANNELS + 1];
+	int port, trigger_chan;
+	struct iio_trigger *trig;
+	struct mutex mutex;
+	/* Cached sample period in milliseconds */
+	unsigned int sample_period;
+	/* Demux table */
+	unsigned int demux_count;
+	struct dln2_adc_demux_table demux[DLN2_ADC_MAX_CHANNELS];
+	/* Precomputed timestamp padding offset and length */
+	unsigned int ts_pad_offset, ts_pad_length;
+};
+
+struct dln2_adc_port_chan {
+	u8 port;
+	u8 chan;
+};
+
+struct dln2_adc_get_all_vals {
+	__le16 channel_mask;
+	__le16 values[DLN2_ADC_MAX_CHANNELS];
+};
+
+static void dln2_adc_add_demux(struct dln2_adc *dln2,
+	unsigned int in_loc, unsigned int out_loc,
+	unsigned int length)
+{
+	struct dln2_adc_demux_table *p = dln2->demux_count ?
+		&dln2->demux[dln2->demux_count - 1] : NULL;
+
+	if (p && p->from + p->length == in_loc &&
+		p->to + p->length == out_loc) {
+		p->length += length;
+	} else if (dln2->demux_count < DLN2_ADC_MAX_CHANNELS) {
+		p = &dln2->demux[dln2->demux_count++];
+		p->from = in_loc;
+		p->to = out_loc;
+		p->length = length;
+	}
+}
+
+static void dln2_adc_update_demux(struct dln2_adc *dln2)
+{
+	int in_ind = -1, out_ind;
+	unsigned int in_loc = 0, out_loc = 0;
+	struct iio_dev *indio_dev = platform_get_drvdata(dln2->pdev);
+
+	/* Clear out any old demux */
+	dln2->demux_count = 0;
+
+	/* Optimize all 8-channels case */
+	if (indio_dev->masklength &&
+	    (*indio_dev->active_scan_mask & 0xff) == 0xff) {
+		dln2_adc_add_demux(dln2, 0, 0, 16);
+		dln2->ts_pad_offset = 0;
+		dln2->ts_pad_length = 0;
+		return;
+	}
+
+	/* Build demux table from fixed 8-channels to active_scan_mask */
+	for_each_set_bit(out_ind,
+			 indio_dev->active_scan_mask,
+			 indio_dev->masklength) {
+		/* Handle timestamp separately */
+		if (out_ind == DLN2_ADC_MAX_CHANNELS)
+			break;
+		for (++in_ind; in_ind != out_ind; ++in_ind)
+			in_loc += 2;
+		dln2_adc_add_demux(dln2, in_loc, out_loc, 2);
+		out_loc += 2;
+		in_loc += 2;
+	}
+
+	if (indio_dev->scan_timestamp) {
+		size_t ts_offset = indio_dev->scan_bytes / sizeof(int64_t) - 1;
+
+		dln2->ts_pad_offset = out_loc;
+		dln2->ts_pad_length = ts_offset * sizeof(int64_t) - out_loc;
+	} else {
+		dln2->ts_pad_offset = 0;
+		dln2->ts_pad_length = 0;
+	}
+}
+
+static int dln2_adc_get_chan_count(struct dln2_adc *dln2)
+{
+	int ret;
+	u8 port = dln2->port;
+	u8 count;
+	int olen = sizeof(count);
+
+	ret = dln2_transfer(dln2->pdev, DLN2_ADC_GET_CHANNEL_COUNT,
+			    &port, sizeof(port), &count, &olen);
+	if (ret < 0) {
+		dev_dbg(&dln2->pdev->dev, "Problem in %s\n", __func__);
+		return ret;
+	}
+	if (olen < sizeof(count))
+		return -EPROTO;
+
+	return count;
+}
+
+static int dln2_adc_set_port_resolution(struct dln2_adc *dln2)
+{
+	int ret;
+	struct dln2_adc_port_chan port_chan = {
+		.port = dln2->port,
+		.chan = DLN2_ADC_DATA_BITS,
+	};
+
+	ret = dln2_transfer_tx(dln2->pdev, DLN2_ADC_SET_RESOLUTION,
+			       &port_chan, sizeof(port_chan));
+	if (ret < 0)
+		dev_dbg(&dln2->pdev->dev, "Problem in %s\n", __func__);
+
+	return ret;
+}
+
+static int dln2_adc_set_chan_enabled(struct dln2_adc *dln2,
+				     int channel, bool enable)
+{
+	int ret;
+	struct dln2_adc_port_chan port_chan = {
+		.port = dln2->port,
+		.chan = channel,
+	};
+	u16 cmd = enable ? DLN2_ADC_CHANNEL_ENABLE : DLN2_ADC_CHANNEL_DISABLE;
+
+	ret = dln2_transfer_tx(dln2->pdev, cmd, &port_chan, sizeof(port_chan));
+	if (ret < 0)
+		dev_dbg(&dln2->pdev->dev, "Problem in %s\n", __func__);
+
+	return ret;
+}
+
+static int dln2_adc_set_port_enabled(struct dln2_adc *dln2, bool enable,
+				     u16 *conflict_out)
+{
+	int ret;
+	u8 port = dln2->port;
+	__le16 conflict;
+	int olen = sizeof(conflict);
+	u16 cmd = enable ? DLN2_ADC_ENABLE : DLN2_ADC_DISABLE;
+
+	if (conflict_out)
+		*conflict_out = 0;
+
+	ret = dln2_transfer(dln2->pdev, cmd, &port, sizeof(port),
+			    &conflict, &olen);
+	if (ret < 0) {
+		dev_dbg(&dln2->pdev->dev, "Problem in %s(%d)\n",
+			__func__, (int)enable);
+		if (conflict_out && enable && olen >= sizeof(conflict))
+			*conflict_out = le16_to_cpu(conflict);
+		return ret;
+	}
+	if (enable && olen < sizeof(conflict))
+		return -EPROTO;
+
+	return ret;
+}
+
+static int dln2_adc_set_chan_period(struct dln2_adc *dln2,
+	unsigned int channel, unsigned int period)
+{
+	int ret;
+	struct {
+		struct dln2_adc_port_chan port_chan;
+		__u8 type;
+		__le16 period;
+		__le16 low;
+		__le16 high;
+	} __packed set_cfg = {
+		.port_chan.port = dln2->port,
+		.port_chan.chan = channel,
+		.type = period ? DLN2_ADC_EVENT_ALWAYS : DLN2_ADC_EVENT_NONE,
+		.period = cpu_to_le16(period)
+	};
+
+	ret = dln2_transfer_tx(dln2->pdev, DLN2_ADC_CHANNEL_SET_CFG,
+			       &set_cfg, sizeof(set_cfg));
+	if (ret < 0)
+		dev_dbg(&dln2->pdev->dev, "Problem in %s\n", __func__);
+
+	return ret;
+}
+
+static int dln2_adc_read(struct dln2_adc *dln2, unsigned int channel)
+{
+	int ret, i;
+	struct iio_dev *indio_dev = platform_get_drvdata(dln2->pdev);
+	u16 conflict;
+	__le16 value;
+	int olen = sizeof(value);
+	struct dln2_adc_port_chan port_chan = {
+		.port = dln2->port,
+		.chan = channel,
+	};
+
+	ret = iio_device_claim_direct_mode(indio_dev);
+	if (ret < 0)
+		return ret;
+
+	ret = dln2_adc_set_chan_enabled(dln2, channel, true);
+	if (ret < 0)
+		goto release_direct;
+
+	ret = dln2_adc_set_port_enabled(dln2, true, &conflict);
+	if (ret < 0) {
+		if (conflict) {
+			dev_err(&dln2->pdev->dev,
+				"ADC pins conflict with mask %04X\n",
+				(int)conflict);
+			ret = -EBUSY;
+		}
+		goto disable_chan;
+	}
+
+	/*
+	 * Call GET_VAL twice due to initial zero-return immediately after
+	 * enabling channel.
+	 */
+	for (i = 0; i < 2; ++i) {
+		ret = dln2_transfer(dln2->pdev, DLN2_ADC_CHANNEL_GET_VAL,
+				    &port_chan, sizeof(port_chan),
+				    &value, &olen);
+		if (ret < 0) {
+			dev_dbg(&dln2->pdev->dev, "Problem in %s\n", __func__);
+			goto disable_port;
+		}
+		if (olen < sizeof(value)) {
+			ret = -EPROTO;
+			goto disable_port;
+		}
+	}
+
+	ret = le16_to_cpu(value);
+
+disable_port:
+	dln2_adc_set_port_enabled(dln2, false, NULL);
+disable_chan:
+	dln2_adc_set_chan_enabled(dln2, channel, false);
+release_direct:
+	iio_device_release_direct_mode(indio_dev);
+
+	return ret;
+}
+
+static int dln2_adc_read_all(struct dln2_adc *dln2,
+			     struct dln2_adc_get_all_vals *get_all_vals)
+{
+	int ret;
+	__u8 port = dln2->port;
+	int olen = sizeof(*get_all_vals);
+
+	ret = dln2_transfer(dln2->pdev, DLN2_ADC_CHANNEL_GET_ALL_VAL,
+			    &port, sizeof(port), get_all_vals, &olen);
+	if (ret < 0) {
+		dev_dbg(&dln2->pdev->dev, "Problem in %s\n", __func__);
+		return ret;
+	}
+	if (olen < sizeof(*get_all_vals))
+		return -EPROTO;
+
+	return ret;
+}
+
+static int dln2_adc_read_raw(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     int *val,
+			     int *val2,
+			     long mask)
+{
+	int ret;
+	unsigned int microhertz;
+	struct dln2_adc *dln2 = iio_priv(indio_dev);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		mutex_lock(&dln2->mutex);
+		ret = dln2_adc_read(dln2, chan->channel);
+		mutex_unlock(&dln2->mutex);
+
+		if (ret < 0)
+			return ret;
+
+		*val = ret;
+		return IIO_VAL_INT;
+
+	case IIO_CHAN_INFO_SCALE:
+		/*
+		 * Voltage reference is fixed at 3.3v
+		 *  3.3 / (1 << 10) * 1000000000
+		 */
+		*val = 0;
+		*val2 = 3222656;
+		return IIO_VAL_INT_PLUS_NANO;
+
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		if (dln2->sample_period) {
+			microhertz = 1000000000 / dln2->sample_period;
+			*val = microhertz / 1000000;
+			*val2 = microhertz % 1000000;
+		} else {
+			*val = 0;
+			*val2 = 0;
+		}
+
+		return IIO_VAL_INT_PLUS_MICRO;
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static int dln2_adc_write_raw(struct iio_dev *indio_dev,
+			      struct iio_chan_spec const *chan,
+			      int val,
+			      int val2,
+			      long mask)
+{
+	int ret;
+	unsigned int microhertz;
+	struct dln2_adc *dln2 = iio_priv(indio_dev);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		microhertz = 1000000 * val + val2;
+
+		mutex_lock(&dln2->mutex);
+
+		dln2->sample_period =
+			microhertz ? 1000000000 / microhertz : UINT_MAX;
+		if (dln2->sample_period > 65535) {
+			dln2->sample_period = 65535;
+			dev_warn(&dln2->pdev->dev,
+				 "clamping period to 65535ms\n");
+		}
+
+		/*
+		 * The first requested channel is arbitrated as a shared
+		 * trigger source, so only one event is registered with the
+		 * DLN. The event handler will then read all enabled channel
+		 * values using DLN2_ADC_CHANNEL_GET_ALL_VAL to maintain
+		 * synchronization between ADC readings.
+		 */
+		if (dln2->trigger_chan != -1)
+			ret = dln2_adc_set_chan_period(dln2,
+				dln2->trigger_chan, dln2->sample_period);
+		else
+			ret = 0;
+
+		mutex_unlock(&dln2->mutex);
+
+		return ret;
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static int dln2_update_scan_mode(struct iio_dev *indio_dev,
+				 const unsigned long *scan_mask)
+{
+	struct dln2_adc *dln2 = iio_priv(indio_dev);
+	int chan_count = indio_dev->num_channels - 1;
+	int ret, i, j;
+
+	mutex_lock(&dln2->mutex);
+
+	for (i = 0; i < chan_count; ++i) {
+		ret = dln2_adc_set_chan_enabled(dln2, i,
+						test_bit(i, scan_mask));
+		if (ret < 0) {
+			for (j = 0; j < i; ++j)
+				dln2_adc_set_chan_enabled(dln2, j, false);
+			mutex_unlock(&dln2->mutex);
+			dev_err(&dln2->pdev->dev,
+				"Unable to enable ADC channel %d\n", i);
+			return -EBUSY;
+		}
+	}
+
+	dln2_adc_update_demux(dln2);
+
+	mutex_unlock(&dln2->mutex);
+
+	return 0;
+}
+
+#define DLN2_ADC_CHAN(lval, idx) {					\
+	lval.type = IIO_VOLTAGE;					\
+	lval.channel = idx;						\
+	lval.indexed = 1;						\
+	lval.info_mask_separate = BIT(IIO_CHAN_INFO_RAW);		\
+	lval.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SCALE) |	\
+				       BIT(IIO_CHAN_INFO_SAMP_FREQ);	\
+	lval.scan_index = idx;						\
+	lval.scan_type.sign = 'u';					\
+	lval.scan_type.realbits = DLN2_ADC_DATA_BITS;			\
+	lval.scan_type.storagebits = 16;				\
+	lval.scan_type.endianness = IIO_LE;				\
+}
+
+/* Assignment version of IIO_CHAN_SOFT_TIMESTAMP */
+#define IIO_CHAN_SOFT_TIMESTAMP_ASSIGN(lval, _si) {	\
+	lval.type = IIO_TIMESTAMP;			\
+	lval.channel = -1;				\
+	lval.scan_index = _si;				\
+	lval.scan_type.sign = 's';			\
+	lval.scan_type.realbits = 64;			\
+	lval.scan_type.storagebits = 64;		\
+}
+
+static const struct iio_info dln2_adc_info = {
+	.read_raw = dln2_adc_read_raw,
+	.write_raw = dln2_adc_write_raw,
+	.update_scan_mode = dln2_update_scan_mode,
+	.driver_module = THIS_MODULE,
+};
+
+static irqreturn_t dln2_adc_trigger_h(int irq, void *p)
+{
+	struct iio_poll_func *pf = p;
+	struct iio_dev *indio_dev = pf->indio_dev;
+	struct {
+		__le16 values[DLN2_ADC_MAX_CHANNELS];
+		int64_t timestamp_space;
+	} data;
+	struct dln2_adc_get_all_vals dev_data;
+	struct dln2_adc *dln2 = iio_priv(indio_dev);
+	const struct dln2_adc_demux_table *t;
+	int ret, i;
+
+	mutex_lock(&dln2->mutex);
+	ret = dln2_adc_read_all(dln2, &dev_data);
+	mutex_unlock(&dln2->mutex);
+	if (ret < 0)
+		goto done;
+
+	/* Demux operation */
+	for (i = 0; i < dln2->demux_count; ++i) {
+		t = &dln2->demux[i];
+		memcpy((void *)data.values + t->to,
+		       (void *)dev_data.values + t->from, t->length);
+	}
+
+	/* Zero padding space between values and timestamp */
+	if (dln2->ts_pad_length)
+		memset((void *)data.values + dln2->ts_pad_offset,
+		       0, dln2->ts_pad_length);
+
+	iio_push_to_buffers_with_timestamp(indio_dev, &data,
+					   iio_get_time_ns(indio_dev));
+
+done:
+	iio_trigger_notify_done(indio_dev->trig);
+	return IRQ_HANDLED;
+}
+
+static int dln2_adc_triggered_buffer_postenable(struct iio_dev *indio_dev)
+{
+	int ret;
+	struct dln2_adc *dln2 = iio_priv(indio_dev);
+	u16 conflict;
+	unsigned int trigger_chan;
+
+	mutex_lock(&dln2->mutex);
+
+	/* Enable ADC */
+	ret = dln2_adc_set_port_enabled(dln2, true, &conflict);
+	if (ret < 0) {
+		mutex_unlock(&dln2->mutex);
+		dev_dbg(&dln2->pdev->dev, "Problem in %s\n", __func__);
+		if (conflict) {
+			dev_err(&dln2->pdev->dev,
+				"ADC pins conflict with mask %04X\n",
+				(int)conflict);
+			ret = -EBUSY;
+		}
+		return ret;
+	}
+
+	/* Assign trigger channel based on first enabled channel */
+	trigger_chan = find_first_bit(indio_dev->active_scan_mask,
+				      indio_dev->masklength);
+	if (trigger_chan < DLN2_ADC_MAX_CHANNELS) {
+		dln2->trigger_chan = trigger_chan;
+		ret = dln2_adc_set_chan_period(dln2, dln2->trigger_chan,
+					       dln2->sample_period);
+		mutex_unlock(&dln2->mutex);
+		if (ret < 0) {
+			dev_dbg(&dln2->pdev->dev, "Problem in %s\n", __func__);
+			return ret;
+		}
+	} else {
+		dln2->trigger_chan = -1;
+		mutex_unlock(&dln2->mutex);
+	}
+
+	return iio_triggered_buffer_postenable(indio_dev);
+}
+
+static int dln2_adc_triggered_buffer_predisable(struct iio_dev *indio_dev)
+{
+	int ret;
+	struct dln2_adc *dln2 = iio_priv(indio_dev);
+
+	mutex_lock(&dln2->mutex);
+
+	/* Disable trigger channel */
+	if (dln2->trigger_chan != -1) {
+		dln2_adc_set_chan_period(dln2, dln2->trigger_chan, 0);
+		dln2->trigger_chan = -1;
+	}
+
+	/* Disable ADC */
+	ret = dln2_adc_set_port_enabled(dln2, false, NULL);
+
+	mutex_unlock(&dln2->mutex);
+	if (ret < 0) {
+		dev_dbg(&dln2->pdev->dev, "Problem in %s\n", __func__);
+		return ret;
+	}
+
+	return iio_triggered_buffer_predisable(indio_dev);
+}
+
+static const struct iio_buffer_setup_ops dln2_adc_buffer_setup_ops = {
+	.postenable = dln2_adc_triggered_buffer_postenable,
+	.predisable = dln2_adc_triggered_buffer_predisable,
+};
+
+static void dln2_adc_event(struct platform_device *pdev, u16 echo,
+			   const void *data, int len)
+{
+	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
+	struct dln2_adc *dln2 = iio_priv(indio_dev);
+
+	/* Called via URB completion handler */
+	iio_trigger_poll(dln2->trig);
+}
+
+static const struct iio_trigger_ops dln2_adc_trigger_ops = {
+	.owner = THIS_MODULE,
+};
+
+static int dln2_adc_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct dln2_adc *dln2;
+	struct dln2_platform_data *pdata = dev_get_platdata(&pdev->dev);
+	struct iio_dev *indio_dev;
+	int i, ret, chans;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*dln2));
+	if (!indio_dev) {
+		dev_err(dev, "failed allocating iio device\n");
+		return -ENOMEM;
+	}
+
+	dln2 = iio_priv(indio_dev);
+	dln2->pdev = pdev;
+	dln2->port = pdata->port;
+	dln2->trigger_chan = -1;
+	mutex_init(&dln2->mutex);
+
+	platform_set_drvdata(pdev, indio_dev);
+
+	ret = dln2_adc_set_port_resolution(dln2);
+	if (ret < 0) {
+		dev_err(dev, "failed to set ADC resolution to 10 bits\n");
+		return ret;
+	}
+
+	chans = dln2_adc_get_chan_count(dln2);
+	if (chans < 0) {
+		dev_err(dev, "failed to get channel count: %d\n", chans);
+		return chans;
+	}
+	if (chans > DLN2_ADC_MAX_CHANNELS) {
+		chans = DLN2_ADC_MAX_CHANNELS;
+		dev_warn(dev, "clamping channels to %d\n",
+			 DLN2_ADC_MAX_CHANNELS);
+	}
+
+	for (i = 0; i < chans; ++i)
+		DLN2_ADC_CHAN(dln2->iio_channels[i], i)
+	IIO_CHAN_SOFT_TIMESTAMP_ASSIGN(dln2->iio_channels[i], i);
+
+	indio_dev->name = DLN2_ADC_MOD_NAME;
+	indio_dev->dev.parent = dev;
+	indio_dev->info = &dln2_adc_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = dln2->iio_channels;
+	indio_dev->num_channels = chans + 1;
+	indio_dev->setup_ops = &dln2_adc_buffer_setup_ops;
+
+	dln2->trig = devm_iio_trigger_alloc(dev, "%s-dev%d",
+					    indio_dev->name, indio_dev->id);
+	if (!dln2->trig) {
+		dev_err(dev, "failed to allocate trigger\n");
+		return -ENOMEM;
+	}
+	dln2->trig->ops = &dln2_adc_trigger_ops;
+	iio_trigger_set_drvdata(dln2->trig, dln2);
+	devm_iio_trigger_register(dev, dln2->trig);
+	iio_trigger_set_immutable(indio_dev, dln2->trig);
+
+	ret = devm_iio_triggered_buffer_setup(dev, indio_dev, NULL,
+					      dln2_adc_trigger_h,
+					      &dln2_adc_buffer_setup_ops);
+	if (ret) {
+		dev_err(dev, "failed to allocate triggered buffer: %d\n", ret);
+		return ret;
+	}
+
+	ret = dln2_register_event_cb(pdev, DLN2_ADC_CONDITION_MET_EV,
+				     dln2_adc_event);
+	if (ret) {
+		dev_err(dev, "failed to setup DLN2 periodic event: %d\n", ret);
+		return ret;
+	}
+
+	ret = iio_device_register(indio_dev);
+	if (ret) {
+		dev_err(dev, "failed to register iio device: %d\n", ret);
+		goto unregister_event;
+	}
+
+	return ret;
+
+unregister_event:
+	dln2_unregister_event_cb(pdev, DLN2_ADC_CONDITION_MET_EV);
+
+	return ret;
+}
+
+static int dln2_adc_remove(struct platform_device *pdev)
+{
+	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
+
+	iio_device_unregister(indio_dev);
+	dln2_unregister_event_cb(pdev, DLN2_ADC_CONDITION_MET_EV);
+	return 0;
+}
+
+static struct platform_driver dln2_adc_driver = {
+	.driver.name	= DLN2_ADC_MOD_NAME,
+	.probe		= dln2_adc_probe,
+	.remove		= dln2_adc_remove,
+};
+
+module_platform_driver(dln2_adc_driver);
+
+MODULE_AUTHOR("Jack Andersen <jackoalan@gmail.com");
+MODULE_DESCRIPTION("Driver for the Diolan DLN2 ADC interface");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:dln2-adc");
diff --git a/drivers/iio/adc/ep93xx_adc.c b/drivers/iio/adc/ep93xx_adc.c
new file mode 100644
index 000000000000..a179ac476c6d
--- /dev/null
+++ b/drivers/iio/adc/ep93xx_adc.c
@@ -0,0 +1,255 @@
+/*
+ * Driver for ADC module on the Cirrus Logic EP93xx series of SoCs
+ *
+ * Copyright (C) 2015 Alexander Sverdlin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * The driver uses polling to get the conversion status. According to EP93xx
+ * datasheets, reading ADCResult register starts the conversion, but user is also
+ * responsible for ensuring that delay between adjacent conversion triggers is
+ * long enough so that maximum allowed conversion rate is not exceeded. This
+ * basically renders IRQ mode unusable.
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/iio/iio.h>
+#include <linux/io.h>
+#include <linux/irqflags.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/platform_device.h>
+
+/*
+ * This code could benefit from real HR Timers, but jiffy granularity would
+ * lower ADC conversion rate down to CONFIG_HZ, so we fallback to busy wait
+ * in such case.
+ *
+ * HR Timers-based version loads CPU only up to 10% during back to back ADC
+ * conversion, while busy wait-based version consumes whole CPU power.
+ */
+#ifdef CONFIG_HIGH_RES_TIMERS
+#define ep93xx_adc_delay(usmin, usmax) usleep_range(usmin, usmax)
+#else
+#define ep93xx_adc_delay(usmin, usmax) udelay(usmin)
+#endif
+
+#define EP93XX_ADC_RESULT	0x08
+#define   EP93XX_ADC_SDR	BIT(31)
+#define EP93XX_ADC_SWITCH	0x18
+#define EP93XX_ADC_SW_LOCK	0x20
+
+struct ep93xx_adc_priv {
+	struct clk *clk;
+	void __iomem *base;
+	int lastch;
+	struct mutex lock;
+};
+
+#define EP93XX_ADC_CH(index, dname, swcfg) {			\
+	.type = IIO_VOLTAGE,					\
+	.indexed = 1,						\
+	.channel = index,					\
+	.address = swcfg,					\
+	.datasheet_name = dname,				\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),		\
+	.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SCALE) |	\
+				   BIT(IIO_CHAN_INFO_OFFSET),	\
+}
+
+/*
+ * Numbering scheme for channels 0..4 is defined in EP9301 and EP9302 datasheets.
+ * EP9307, EP9312 and EP9312 have 3 channels more (total 8), but the numbering is
+ * not defined. So the last three are numbered randomly, let's say.
+ */
+static const struct iio_chan_spec ep93xx_adc_channels[8] = {
+	EP93XX_ADC_CH(0, "YM",	0x608),
+	EP93XX_ADC_CH(1, "SXP",	0x680),
+	EP93XX_ADC_CH(2, "SXM",	0x640),
+	EP93XX_ADC_CH(3, "SYP",	0x620),
+	EP93XX_ADC_CH(4, "SYM",	0x610),
+	EP93XX_ADC_CH(5, "XP",	0x601),
+	EP93XX_ADC_CH(6, "XM",	0x602),
+	EP93XX_ADC_CH(7, "YP",	0x604),
+};
+
+static int ep93xx_read_raw(struct iio_dev *iiodev,
+			   struct iio_chan_spec const *channel, int *value,
+			   int *shift, long mask)
+{
+	struct ep93xx_adc_priv *priv = iio_priv(iiodev);
+	unsigned long timeout;
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		mutex_lock(&priv->lock);
+		if (priv->lastch != channel->channel) {
+			priv->lastch = channel->channel;
+			/*
+			 * Switch register is software-locked, unlocking must be
+			 * immediately followed by write
+			 */
+			local_irq_disable();
+			writel_relaxed(0xAA, priv->base + EP93XX_ADC_SW_LOCK);
+			writel_relaxed(channel->address,
+				       priv->base + EP93XX_ADC_SWITCH);
+			local_irq_enable();
+			/*
+			 * Settling delay depends on module clock and could be
+			 * 2ms or 500us
+			 */
+			ep93xx_adc_delay(2000, 2000);
+		}
+		/* Start the conversion, eventually discarding old result */
+		readl_relaxed(priv->base + EP93XX_ADC_RESULT);
+		/* Ensure maximum conversion rate is not exceeded */
+		ep93xx_adc_delay(DIV_ROUND_UP(1000000, 925),
+				 DIV_ROUND_UP(1000000, 925));
+		/* At this point conversion must be completed, but anyway... */
+		ret = IIO_VAL_INT;
+		timeout = jiffies + msecs_to_jiffies(1) + 1;
+		while (1) {
+			u32 t;
+
+			t = readl_relaxed(priv->base + EP93XX_ADC_RESULT);
+			if (t & EP93XX_ADC_SDR) {
+				*value = sign_extend32(t, 15);
+				break;
+			}
+
+			if (time_after(jiffies, timeout)) {
+				dev_err(&iiodev->dev, "Conversion timeout\n");
+				ret = -ETIMEDOUT;
+				break;
+			}
+
+			cpu_relax();
+		}
+		mutex_unlock(&priv->lock);
+		return ret;
+
+	case IIO_CHAN_INFO_OFFSET:
+		/* According to datasheet, range is -25000..25000 */
+		*value = 25000;
+		return IIO_VAL_INT;
+
+	case IIO_CHAN_INFO_SCALE:
+		/* Typical supply voltage is 3.3v */
+		*value = (1ULL << 32) * 3300 / 50000;
+		*shift = 32;
+		return IIO_VAL_FRACTIONAL_LOG2;
+	}
+
+	return -EINVAL;
+}
+
+static const struct iio_info ep93xx_adc_info = {
+	.driver_module = THIS_MODULE,
+	.read_raw = ep93xx_read_raw,
+};
+
+static int ep93xx_adc_probe(struct platform_device *pdev)
+{
+	int ret;
+	struct iio_dev *iiodev;
+	struct ep93xx_adc_priv *priv;
+	struct clk *pclk;
+	struct resource *res;
+
+	iiodev = devm_iio_device_alloc(&pdev->dev, sizeof(*priv));
+	if (!iiodev)
+		return -ENOMEM;
+	priv = iio_priv(iiodev);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res) {
+		dev_err(&pdev->dev, "Cannot obtain memory resource\n");
+		return -ENXIO;
+	}
+	priv->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(priv->base)) {
+		dev_err(&pdev->dev, "Cannot map memory resource\n");
+		return PTR_ERR(priv->base);
+	}
+
+	iiodev->dev.parent = &pdev->dev;
+	iiodev->name = dev_name(&pdev->dev);
+	iiodev->modes = INDIO_DIRECT_MODE;
+	iiodev->info = &ep93xx_adc_info;
+	iiodev->num_channels = ARRAY_SIZE(ep93xx_adc_channels);
+	iiodev->channels = ep93xx_adc_channels;
+
+	priv->lastch = -1;
+	mutex_init(&priv->lock);
+
+	platform_set_drvdata(pdev, iiodev);
+
+	priv->clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(priv->clk)) {
+		dev_err(&pdev->dev, "Cannot obtain clock\n");
+		return PTR_ERR(priv->clk);
+	}
+
+	pclk = clk_get_parent(priv->clk);
+	if (!pclk) {
+		dev_warn(&pdev->dev, "Cannot obtain parent clock\n");
+	} else {
+		/*
+		 * This is actually a place for improvement:
+		 * EP93xx ADC supports two clock divisors -- 4 and 16,
+		 * resulting in conversion rates 3750 and 925 samples per second
+		 * with 500us or 2ms settling time respectively.
+		 * One might find this interesting enough to be configurable.
+		 */
+		ret = clk_set_rate(priv->clk, clk_get_rate(pclk) / 16);
+		if (ret)
+			dev_warn(&pdev->dev, "Cannot set clock rate\n");
+		/*
+		 * We can tolerate rate setting failure because the module should
+		 * work in any case.
+		 */
+	}
+
+	ret = clk_enable(priv->clk);
+	if (ret) {
+		dev_err(&pdev->dev, "Cannot enable clock\n");
+		return ret;
+	}
+
+	ret = iio_device_register(iiodev);
+	if (ret)
+		clk_disable(priv->clk);
+
+	return ret;
+}
+
+static int ep93xx_adc_remove(struct platform_device *pdev)
+{
+	struct iio_dev *iiodev = platform_get_drvdata(pdev);
+	struct ep93xx_adc_priv *priv = iio_priv(iiodev);
+
+	iio_device_unregister(iiodev);
+	clk_disable(priv->clk);
+
+	return 0;
+}
+
+static struct platform_driver ep93xx_adc_driver = {
+	.driver = {
+		.name = "ep93xx-adc",
+	},
+	.probe = ep93xx_adc_probe,
+	.remove = ep93xx_adc_remove,
+};
+module_platform_driver(ep93xx_adc_driver);
+
+MODULE_AUTHOR("Alexander Sverdlin <alexander.sverdlin@gmail.com>");
+MODULE_DESCRIPTION("Cirrus Logic EP93XX ADC driver");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:ep93xx-adc");
diff --git a/drivers/iio/adc/ina2xx-adc.c b/drivers/iio/adc/ina2xx-adc.c
index 232c0b80d658..f387b972e4f4 100644
--- a/drivers/iio/adc/ina2xx-adc.c
+++ b/drivers/iio/adc/ina2xx-adc.c
@@ -44,6 +44,7 @@
 
 #define INA226_MASK_ENABLE		0x06
 #define INA226_CVRF			BIT(3)
+#define INA219_CNVR			BIT(1)
 
 #define INA2XX_MAX_REGISTERS            8
 
@@ -592,6 +593,7 @@ static int ina2xx_work_buffer(struct iio_dev *indio_dev)
 	int bit, ret, i = 0;
 	s64 time_a, time_b;
 	unsigned int alert;
+	int cnvr_need_clear = 0;
 
 	time_a = iio_get_time_ns(indio_dev);
 
@@ -603,22 +605,30 @@ static int ina2xx_work_buffer(struct iio_dev *indio_dev)
 	 * we check the ConVersionReadyFlag.
 	 * On hardware that supports using the ALERT pin to toggle a
 	 * GPIO a triggered buffer could be used instead.
-	 * For now, we pay for that extra read of the ALERT register
+	 * For now, we do an extra read of the MASK_ENABLE register (INA226)
+	 * resp. the BUS_VOLTAGE register (INA219).
 	 */
 	if (!chip->allow_async_readout)
 		do {
-			ret = regmap_read(chip->regmap, INA226_MASK_ENABLE,
-					  &alert);
+			if (chip->config->chip_id == ina226) {
+				ret = regmap_read(chip->regmap,
+						  INA226_MASK_ENABLE, &alert);
+				alert &= INA226_CVRF;
+			} else {
+				ret = regmap_read(chip->regmap,
+						  INA2XX_BUS_VOLTAGE, &alert);
+				alert &= INA219_CNVR;
+				cnvr_need_clear = alert;
+			}
+
 			if (ret < 0)
 				return ret;
 
-			alert &= INA226_CVRF;
 		} while (!alert);
 
 	/*
-	 * Single register reads: bulk_read will not work with ina226
-	 * as there is no auto-increment of the address register for
-	 * data length longer than 16bits.
+	 * Single register reads: bulk_read will not work with ina226/219
+	 * as there is no auto-increment of the register pointer.
 	 */
 	for_each_set_bit(bit, indio_dev->active_scan_mask,
 			 indio_dev->masklength) {
@@ -630,6 +640,18 @@ static int ina2xx_work_buffer(struct iio_dev *indio_dev)
 			return ret;
 
 		data[i++] = val;
+
+		if (INA2XX_SHUNT_VOLTAGE + bit == INA2XX_POWER)
+			cnvr_need_clear = 0;
+	}
+
+	/* Dummy read on INA219 power register to clear CNVR flag */
+	if (cnvr_need_clear && chip->config->chip_id == ina219) {
+		unsigned int val;
+
+		ret = regmap_read(chip->regmap, INA2XX_POWER, &val);
+		if (ret < 0)
+			return ret;
 	}
 
 	time_b = iio_get_time_ns(indio_dev);
@@ -644,7 +666,7 @@ static int ina2xx_capture_thread(void *data)
 {
 	struct iio_dev *indio_dev = data;
 	struct ina2xx_chip_info *chip = iio_priv(indio_dev);
-	unsigned int sampling_us = SAMPLING_PERIOD(chip);
+	int sampling_us = SAMPLING_PERIOD(chip);
 	int buffer_us;
 
 	/*
diff --git a/drivers/iio/adc/ltc2471.c b/drivers/iio/adc/ltc2471.c
new file mode 100644
index 000000000000..29b7ed60cdb0
--- /dev/null
+++ b/drivers/iio/adc/ltc2471.c
@@ -0,0 +1,160 @@
+/*
+ * Driver for Linear Technology LTC2471 and LTC2473 voltage monitors
+ * The LTC2473 is identical to the 2471, but reports a differential signal.
+ *
+ * Copyright (C) 2017 Topic Embedded Products
+ * Author: Mike Looijmans <mike.looijmans@topic.nl>
+ *
+ * License: GPLv2
+ */
+
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+
+enum ltc2471_chips {
+	ltc2471,
+	ltc2473,
+};
+
+struct ltc2471_data {
+	struct i2c_client *client;
+};
+
+/* Reference voltage is 1.25V */
+#define LTC2471_VREF 1250
+
+/* Read two bytes from the I2C bus to obtain the ADC result */
+static int ltc2471_get_value(struct i2c_client *client)
+{
+	int ret;
+	__be16 buf;
+
+	ret = i2c_master_recv(client, (char *)&buf, sizeof(buf));
+	if (ret < 0)
+		return ret;
+	if (ret != sizeof(buf))
+		return -EIO;
+
+	/* MSB first */
+	return be16_to_cpu(buf);
+}
+
+static int ltc2471_read_raw(struct iio_dev *indio_dev,
+			    struct iio_chan_spec const *chan,
+			    int *val, int *val2, long info)
+{
+	struct ltc2471_data *data = iio_priv(indio_dev);
+	int ret;
+
+	switch (info) {
+	case IIO_CHAN_INFO_RAW:
+		ret = ltc2471_get_value(data->client);
+		if (ret < 0)
+			return ret;
+		*val = ret;
+		return IIO_VAL_INT;
+
+	case IIO_CHAN_INFO_SCALE:
+		if (chan->differential)
+			/* Output ranges from -VREF to +VREF */
+			*val = 2 * LTC2471_VREF;
+		else
+			/* Output ranges from 0 to VREF */
+			*val = LTC2471_VREF;
+		*val2 = 16;	/* 16 data bits */
+		return IIO_VAL_FRACTIONAL_LOG2;
+
+	case IIO_CHAN_INFO_OFFSET:
+		/* Only differential chip has this property */
+		*val = -LTC2471_VREF;
+		return IIO_VAL_INT;
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct iio_chan_spec ltc2471_channel[] = {
+	{
+		.type = IIO_VOLTAGE,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),
+	},
+};
+
+static const struct iio_chan_spec ltc2473_channel[] = {
+	{
+		.type = IIO_VOLTAGE,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE) |
+					    BIT(IIO_CHAN_INFO_OFFSET),
+		.differential = 1,
+	},
+};
+
+static const struct iio_info ltc2471_info = {
+	.read_raw = ltc2471_read_raw,
+	.driver_module = THIS_MODULE,
+};
+
+static int ltc2471_i2c_probe(struct i2c_client *client,
+			     const struct i2c_device_id *id)
+{
+	struct iio_dev *indio_dev;
+	struct ltc2471_data *data;
+	int ret;
+
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C))
+		return -EOPNOTSUPP;
+
+	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	data = iio_priv(indio_dev);
+	data->client = client;
+
+	indio_dev->dev.parent = &client->dev;
+	indio_dev->name = id->name;
+	indio_dev->info = &ltc2471_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	if (id->driver_data == ltc2473)
+		indio_dev->channels = ltc2473_channel;
+	else
+		indio_dev->channels = ltc2471_channel;
+	indio_dev->num_channels = 1;
+
+	/* Trigger once to start conversion and check if chip is there */
+	ret = ltc2471_get_value(client);
+	if (ret < 0) {
+		dev_err(&client->dev, "Cannot read from device.\n");
+		return ret;
+	}
+
+	return devm_iio_device_register(&client->dev, indio_dev);
+}
+
+static const struct i2c_device_id ltc2471_i2c_id[] = {
+	{ "ltc2471", ltc2471 },
+	{ "ltc2473", ltc2473 },
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, ltc2471_i2c_id);
+
+static struct i2c_driver ltc2471_i2c_driver = {
+	.driver = {
+		.name = "ltc2471",
+	},
+	.probe    = ltc2471_i2c_probe,
+	.id_table = ltc2471_i2c_id,
+};
+
+module_i2c_driver(ltc2471_i2c_driver);
+
+MODULE_DESCRIPTION("LTC2471/LTC2473 ADC driver");
+MODULE_AUTHOR("Topic Embedded Products");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/adc/ltc2497.c b/drivers/iio/adc/ltc2497.c
index 2691b10023f5..5bf8011dcde9 100644
--- a/drivers/iio/adc/ltc2497.c
+++ b/drivers/iio/adc/ltc2497.c
@@ -11,6 +11,7 @@
 #include <linux/delay.h>
 #include <linux/i2c.h>
 #include <linux/iio/iio.h>
+#include <linux/iio/driver.h>
 #include <linux/iio/sysfs.h>
 #include <linux/module.h>
 #include <linux/of.h>
@@ -127,13 +128,14 @@ static int ltc2497_read_raw(struct iio_dev *indio_dev,
 	}
 }
 
-#define LTC2497_CHAN(_chan, _addr) { \
+#define LTC2497_CHAN(_chan, _addr, _ds_name) { \
 	.type = IIO_VOLTAGE, \
 	.indexed = 1, \
 	.channel = (_chan), \
 	.address = (_addr | (_chan / 2) | ((_chan & 1) ? LTC2497_SIGN : 0)), \
 	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW), \
 	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE), \
+	.datasheet_name = (_ds_name), \
 }
 
 #define LTC2497_CHAN_DIFF(_chan, _addr) { \
@@ -148,22 +150,22 @@ static int ltc2497_read_raw(struct iio_dev *indio_dev,
 }
 
 static const struct iio_chan_spec ltc2497_channel[] = {
-	LTC2497_CHAN(0, LTC2497_SGL),
-	LTC2497_CHAN(1, LTC2497_SGL),
-	LTC2497_CHAN(2, LTC2497_SGL),
-	LTC2497_CHAN(3, LTC2497_SGL),
-	LTC2497_CHAN(4, LTC2497_SGL),
-	LTC2497_CHAN(5, LTC2497_SGL),
-	LTC2497_CHAN(6, LTC2497_SGL),
-	LTC2497_CHAN(7, LTC2497_SGL),
-	LTC2497_CHAN(8, LTC2497_SGL),
-	LTC2497_CHAN(9, LTC2497_SGL),
-	LTC2497_CHAN(10, LTC2497_SGL),
-	LTC2497_CHAN(11, LTC2497_SGL),
-	LTC2497_CHAN(12, LTC2497_SGL),
-	LTC2497_CHAN(13, LTC2497_SGL),
-	LTC2497_CHAN(14, LTC2497_SGL),
-	LTC2497_CHAN(15, LTC2497_SGL),
+	LTC2497_CHAN(0, LTC2497_SGL, "CH0"),
+	LTC2497_CHAN(1, LTC2497_SGL, "CH1"),
+	LTC2497_CHAN(2, LTC2497_SGL, "CH2"),
+	LTC2497_CHAN(3, LTC2497_SGL, "CH3"),
+	LTC2497_CHAN(4, LTC2497_SGL, "CH4"),
+	LTC2497_CHAN(5, LTC2497_SGL, "CH5"),
+	LTC2497_CHAN(6, LTC2497_SGL, "CH6"),
+	LTC2497_CHAN(7, LTC2497_SGL, "CH7"),
+	LTC2497_CHAN(8, LTC2497_SGL, "CH8"),
+	LTC2497_CHAN(9, LTC2497_SGL, "CH9"),
+	LTC2497_CHAN(10, LTC2497_SGL, "CH10"),
+	LTC2497_CHAN(11, LTC2497_SGL, "CH11"),
+	LTC2497_CHAN(12, LTC2497_SGL, "CH12"),
+	LTC2497_CHAN(13, LTC2497_SGL, "CH13"),
+	LTC2497_CHAN(14, LTC2497_SGL, "CH14"),
+	LTC2497_CHAN(15, LTC2497_SGL, "CH15"),
 	LTC2497_CHAN_DIFF(0, LTC2497_DIFF),
 	LTC2497_CHAN_DIFF(1, LTC2497_DIFF),
 	LTC2497_CHAN_DIFF(2, LTC2497_DIFF),
@@ -192,6 +194,7 @@ static int ltc2497_probe(struct i2c_client *client,
 {
 	struct iio_dev *indio_dev;
 	struct ltc2497_st *st;
+	struct iio_map *plat_data;
 	int ret;
 
 	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C |
@@ -221,19 +224,31 @@ static int ltc2497_probe(struct i2c_client *client,
 	if (ret < 0)
 		return ret;
 
+	if (client->dev.platform_data) {
+		plat_data = ((struct iio_map *)client->dev.platform_data);
+		ret = iio_map_array_register(indio_dev, plat_data);
+		if (ret) {
+			dev_err(&indio_dev->dev, "iio map err: %d\n", ret);
+			goto err_regulator_disable;
+		}
+	}
+
 	ret = i2c_smbus_write_byte(st->client, LTC2497_CONFIG_DEFAULT);
 	if (ret < 0)
-		goto err_regulator_disable;
+		goto err_array_unregister;
 
 	st->addr_prev = LTC2497_CONFIG_DEFAULT;
 	st->time_prev = ktime_get();
 
 	ret = iio_device_register(indio_dev);
 	if (ret < 0)
-		goto err_regulator_disable;
+		goto err_array_unregister;
 
 	return 0;
 
+err_array_unregister:
+	iio_map_array_unregister(indio_dev);
+
 err_regulator_disable:
 	regulator_disable(st->ref);
 
@@ -245,6 +260,7 @@ static int ltc2497_remove(struct i2c_client *client)
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct ltc2497_st *st = iio_priv(indio_dev);
 
+	iio_map_array_unregister(indio_dev);
 	iio_device_unregister(indio_dev);
 	regulator_disable(st->ref);
 
diff --git a/drivers/iio/adc/max9611.c b/drivers/iio/adc/max9611.c
index b0526e4b9530..b1dd17cbce58 100644
--- a/drivers/iio/adc/max9611.c
+++ b/drivers/iio/adc/max9611.c
@@ -549,8 +549,8 @@ static int max9611_probe(struct i2c_client *client,
 	ret = of_property_read_u32(of_node, shunt_res_prop, &of_shunt);
 	if (ret) {
 		dev_err(&client->dev,
-			"Missing %s property for %s node\n",
-			shunt_res_prop, of_node->full_name);
+			"Missing %s property for %pOF node\n",
+			shunt_res_prop, of_node);
 		return ret;
 	}
 	max9611->shunt_resistor_uohm = of_shunt;
diff --git a/drivers/iio/adc/mcp3422.c b/drivers/iio/adc/mcp3422.c
index 254135e07792..63de705086ed 100644
--- a/drivers/iio/adc/mcp3422.c
+++ b/drivers/iio/adc/mcp3422.c
@@ -379,10 +379,12 @@ static int mcp3422_probe(struct i2c_client *client,
 
 	/* meaningful default configuration */
 	config = (MCP3422_CONT_SAMPLING
-		| MCP3422_CHANNEL_VALUE(1)
+		| MCP3422_CHANNEL_VALUE(0)
 		| MCP3422_PGA_VALUE(MCP3422_PGA_1)
 		| MCP3422_SAMPLE_RATE_VALUE(MCP3422_SRATE_240));
-	mcp3422_update_config(adc, config);
+	err = mcp3422_update_config(adc, config);
+	if (err < 0)
+		return err;
 
 	err = devm_iio_device_register(&client->dev, indio_dev);
 	if (err < 0)
diff --git a/drivers/iio/adc/meson_saradc.c b/drivers/iio/adc/meson_saradc.c
index 83da50ed73ab..2e8dbb89c8c9 100644
--- a/drivers/iio/adc/meson_saradc.c
+++ b/drivers/iio/adc/meson_saradc.c
@@ -572,8 +572,8 @@ static int meson_sar_adc_clk_init(struct iio_dev *indio_dev,
 	struct clk_init_data init;
 	const char *clk_parents[1];
 
-	init.name = devm_kasprintf(&indio_dev->dev, GFP_KERNEL, "%s#adc_div",
-				   of_node_full_name(indio_dev->dev.of_node));
+	init.name = devm_kasprintf(&indio_dev->dev, GFP_KERNEL, "%pOF#adc_div",
+				   indio_dev->dev.of_node);
 	init.flags = 0;
 	init.ops = &clk_divider_ops;
 	clk_parents[0] = __clk_get_name(priv->clkin);
@@ -591,8 +591,8 @@ static int meson_sar_adc_clk_init(struct iio_dev *indio_dev,
 	if (WARN_ON(IS_ERR(priv->adc_div_clk)))
 		return PTR_ERR(priv->adc_div_clk);
 
-	init.name = devm_kasprintf(&indio_dev->dev, GFP_KERNEL, "%s#adc_en",
-				   of_node_full_name(indio_dev->dev.of_node));
+	init.name = devm_kasprintf(&indio_dev->dev, GFP_KERNEL, "%pOF#adc_en",
+				   indio_dev->dev.of_node);
 	init.flags = CLK_SET_RATE_PARENT;
 	init.ops = &clk_gate_ops;
 	clk_parents[0] = __clk_get_name(priv->adc_div_clk);
@@ -915,6 +915,11 @@ static int meson_sar_adc_probe(struct platform_device *pdev)
 	init_completion(&priv->done);
 
 	match = of_match_device(meson_sar_adc_of_match, &pdev->dev);
+	if (!match) {
+		dev_err(&pdev->dev, "failed to match device\n");
+		return -ENODEV;
+	}
+
 	priv->data = match->data;
 
 	indio_dev->name = priv->data->name;
diff --git a/drivers/iio/adc/mt6577_auxadc.c b/drivers/iio/adc/mt6577_auxadc.c
index 2d104c828041..414cf44bf19d 100644
--- a/drivers/iio/adc/mt6577_auxadc.c
+++ b/drivers/iio/adc/mt6577_auxadc.c
@@ -184,6 +184,37 @@ static const struct iio_info mt6577_auxadc_info = {
 	.read_raw = &mt6577_auxadc_read_raw,
 };
 
+static int __maybe_unused mt6577_auxadc_resume(struct device *dev)
+{
+	struct iio_dev *indio_dev = dev_get_drvdata(dev);
+	struct mt6577_auxadc_device *adc_dev = iio_priv(indio_dev);
+	int ret;
+
+	ret = clk_prepare_enable(adc_dev->adc_clk);
+	if (ret) {
+		pr_err("failed to enable auxadc clock\n");
+		return ret;
+	}
+
+	mt6577_auxadc_mod_reg(adc_dev->reg_base + MT6577_AUXADC_MISC,
+			      MT6577_AUXADC_PDN_EN, 0);
+	mdelay(MT6577_AUXADC_POWER_READY_MS);
+
+	return 0;
+}
+
+static int __maybe_unused mt6577_auxadc_suspend(struct device *dev)
+{
+	struct iio_dev *indio_dev = dev_get_drvdata(dev);
+	struct mt6577_auxadc_device *adc_dev = iio_priv(indio_dev);
+
+	mt6577_auxadc_mod_reg(adc_dev->reg_base + MT6577_AUXADC_MISC,
+			      0, MT6577_AUXADC_PDN_EN);
+	clk_disable_unprepare(adc_dev->adc_clk);
+
+	return 0;
+}
+
 static int mt6577_auxadc_probe(struct platform_device *pdev)
 {
 	struct mt6577_auxadc_device *adc_dev;
@@ -269,8 +300,13 @@ static int mt6577_auxadc_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static SIMPLE_DEV_PM_OPS(mt6577_auxadc_pm_ops,
+			 mt6577_auxadc_suspend,
+			 mt6577_auxadc_resume);
+
 static const struct of_device_id mt6577_auxadc_of_match[] = {
 	{ .compatible = "mediatek,mt2701-auxadc", },
+	{ .compatible = "mediatek,mt7622-auxadc", },
 	{ .compatible = "mediatek,mt8173-auxadc", },
 	{ }
 };
@@ -280,6 +316,7 @@ static struct platform_driver mt6577_auxadc_driver = {
 	.driver = {
 		.name   = "mt6577-auxadc",
 		.of_match_table = mt6577_auxadc_of_match,
+		.pm = &mt6577_auxadc_pm_ops,
 	},
 	.probe	= mt6577_auxadc_probe,
 	.remove	= mt6577_auxadc_remove,
diff --git a/drivers/iio/adc/rockchip_saradc.c b/drivers/iio/adc/rockchip_saradc.c
index ae6d3324f518..5f612d694b33 100644
--- a/drivers/iio/adc/rockchip_saradc.c
+++ b/drivers/iio/adc/rockchip_saradc.c
@@ -224,6 +224,11 @@ static int rockchip_saradc_probe(struct platform_device *pdev)
 	info = iio_priv(indio_dev);
 
 	match = of_match_device(rockchip_saradc_match, &pdev->dev);
+	if (!match) {
+		dev_err(&pdev->dev, "failed to match device\n");
+		return -ENODEV;
+	}
+
 	info->data = match->data;
 
 	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
@@ -235,7 +240,8 @@ static int rockchip_saradc_probe(struct platform_device *pdev)
 	 * The reset should be an optional property, as it should work
 	 * with old devicetrees as well
 	 */
-	info->reset = devm_reset_control_get(&pdev->dev, "saradc-apb");
+	info->reset = devm_reset_control_get_exclusive(&pdev->dev,
+						       "saradc-apb");
 	if (IS_ERR(info->reset)) {
 		ret = PTR_ERR(info->reset);
 		if (ret != -ENOENT)
diff --git a/drivers/iio/adc/stm32-adc-core.c b/drivers/iio/adc/stm32-adc-core.c
index e09233b03c05..804198eb0eef 100644
--- a/drivers/iio/adc/stm32-adc-core.c
+++ b/drivers/iio/adc/stm32-adc-core.c
@@ -64,7 +64,7 @@
 #define STM32H7_CKMODE_MASK		GENMASK(17, 16)
 
 /* STM32 H7 maximum analog clock rate (from datasheet) */
-#define STM32H7_ADC_MAX_CLK_RATE	72000000
+#define STM32H7_ADC_MAX_CLK_RATE	36000000
 
 /**
  * stm32_adc_common_regs - stm32 common registers, compatible dependent data
@@ -148,14 +148,14 @@ static int stm32f4_adc_clk_sel(struct platform_device *pdev,
 		return -EINVAL;
 	}
 
-	priv->common.rate = rate;
+	priv->common.rate = rate / stm32f4_pclk_div[i];
 	val = readl_relaxed(priv->common.base + STM32F4_ADC_CCR);
 	val &= ~STM32F4_ADC_ADCPRE_MASK;
 	val |= i << STM32F4_ADC_ADCPRE_SHIFT;
 	writel_relaxed(val, priv->common.base + STM32F4_ADC_CCR);
 
 	dev_dbg(&pdev->dev, "Using analog clock source at %ld kHz\n",
-		rate / (stm32f4_pclk_div[i] * 1000));
+		priv->common.rate / 1000);
 
 	return 0;
 }
@@ -172,7 +172,7 @@ struct stm32h7_adc_ck_spec {
 	int div;
 };
 
-const struct stm32h7_adc_ck_spec stm32h7_adc_ckmodes_spec[] = {
+static const struct stm32h7_adc_ck_spec stm32h7_adc_ckmodes_spec[] = {
 	/* 00: CK_ADC[1..3]: Asynchronous clock modes */
 	{ 0, 0, 1 },
 	{ 0, 1, 2 },
@@ -250,7 +250,7 @@ static int stm32h7_adc_clk_sel(struct platform_device *pdev,
 
 out:
 	/* rate used later by each ADC instance to control BOOST mode */
-	priv->common.rate = rate;
+	priv->common.rate = rate / div;
 
 	/* Set common clock mode and prescaler */
 	val = readl_relaxed(priv->common.base + STM32H7_ADC_CCR);
@@ -260,7 +260,7 @@ out:
 	writel_relaxed(val, priv->common.base + STM32H7_ADC_CCR);
 
 	dev_dbg(&pdev->dev, "Using %s clock/%d source at %ld kHz\n",
-		ckmode ? "bus" : "adc", div, rate / (div * 1000));
+		ckmode ? "bus" : "adc", div, priv->common.rate / 1000);
 
 	return 0;
 }
diff --git a/drivers/iio/adc/stm32-adc.c b/drivers/iio/adc/stm32-adc.c
index 5bfcc1f13105..e3c15f88075f 100644
--- a/drivers/iio/adc/stm32-adc.c
+++ b/drivers/iio/adc/stm32-adc.c
@@ -25,6 +25,7 @@
 #include <linux/dmaengine.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/buffer.h>
+#include <linux/iio/timer/stm32-lptim-trigger.h>
 #include <linux/iio/timer/stm32-timer-trigger.h>
 #include <linux/iio/trigger.h>
 #include <linux/iio/trigger_consumer.h>
@@ -83,6 +84,8 @@
 #define STM32H7_ADC_IER			0x04
 #define STM32H7_ADC_CR			0x08
 #define STM32H7_ADC_CFGR		0x0C
+#define STM32H7_ADC_SMPR1		0x14
+#define STM32H7_ADC_SMPR2		0x18
 #define STM32H7_ADC_PCSEL		0x1C
 #define STM32H7_ADC_SQR1		0x30
 #define STM32H7_ADC_SQR2		0x34
@@ -151,6 +154,7 @@ enum stm32h7_adc_dmngt {
 #define STM32H7_BOOST_CLKRATE		20000000UL
 
 #define STM32_ADC_MAX_SQ		16	/* SQ1..SQ16 */
+#define STM32_ADC_MAX_SMP		7	/* SMPx range is [0..7] */
 #define STM32_ADC_TIMEOUT_US		100000
 #define STM32_ADC_TIMEOUT	(msecs_to_jiffies(STM32_ADC_TIMEOUT_US / 1000))
 
@@ -182,6 +186,11 @@ enum stm32_adc_extsel {
 	STM32_EXT13,
 	STM32_EXT14,
 	STM32_EXT15,
+	STM32_EXT16,
+	STM32_EXT17,
+	STM32_EXT18,
+	STM32_EXT19,
+	STM32_EXT20,
 };
 
 /**
@@ -227,6 +236,8 @@ struct stm32_adc_regs {
  * @exten:		trigger control register & bitfield
  * @extsel:		trigger selection register & bitfield
  * @res:		resolution selection register & bitfield
+ * @smpr:		smpr1 & smpr2 registers offset array
+ * @smp_bits:		smpr1 & smpr2 index and bitfields
  */
 struct stm32_adc_regspec {
 	const u32 dr;
@@ -236,6 +247,8 @@ struct stm32_adc_regspec {
 	const struct stm32_adc_regs exten;
 	const struct stm32_adc_regs extsel;
 	const struct stm32_adc_regs res;
+	const u32 smpr[2];
+	const struct stm32_adc_regs *smp_bits;
 };
 
 struct stm32_adc;
@@ -251,6 +264,7 @@ struct stm32_adc;
  * @start_conv:		routine to start conversions
  * @stop_conv:		routine to stop conversions
  * @unprepare:		optional unprepare routine (disable, power-down)
+ * @smp_cycles:		programmable sampling time (ADC clock cycles)
  */
 struct stm32_adc_cfg {
 	const struct stm32_adc_regspec	*regs;
@@ -262,6 +276,7 @@ struct stm32_adc_cfg {
 	void (*start_conv)(struct stm32_adc *, bool dma);
 	void (*stop_conv)(struct stm32_adc *);
 	void (*unprepare)(struct stm32_adc *);
+	const unsigned int *smp_cycles;
 };
 
 /**
@@ -283,6 +298,7 @@ struct stm32_adc_cfg {
  * @rx_dma_buf:		dma rx buffer bus address
  * @rx_buf_sz:		dma rx buffer size
  * @pcsel		bitmask to preselect channels on some devices
+ * @smpr_val:		sampling time settings (e.g. smpr1 / smpr2)
  * @cal:		optional calibration data on some devices
  */
 struct stm32_adc {
@@ -303,6 +319,7 @@ struct stm32_adc {
 	dma_addr_t		rx_dma_buf;
 	unsigned int		rx_buf_sz;
 	u32			pcsel;
+	u32			smpr_val[2];
 	struct stm32_adc_calib	cal;
 };
 
@@ -431,6 +448,39 @@ static struct stm32_adc_trig_info stm32f4_adc_trigs[] = {
 	{}, /* sentinel */
 };
 
+/**
+ * stm32f4_smp_bits[] - describe sampling time register index & bit fields
+ * Sorted so it can be indexed by channel number.
+ */
+static const struct stm32_adc_regs stm32f4_smp_bits[] = {
+	/* STM32F4_ADC_SMPR2: smpr[] index, mask, shift for SMP0 to SMP9 */
+	{ 1, GENMASK(2, 0), 0 },
+	{ 1, GENMASK(5, 3), 3 },
+	{ 1, GENMASK(8, 6), 6 },
+	{ 1, GENMASK(11, 9), 9 },
+	{ 1, GENMASK(14, 12), 12 },
+	{ 1, GENMASK(17, 15), 15 },
+	{ 1, GENMASK(20, 18), 18 },
+	{ 1, GENMASK(23, 21), 21 },
+	{ 1, GENMASK(26, 24), 24 },
+	{ 1, GENMASK(29, 27), 27 },
+	/* STM32F4_ADC_SMPR1, smpr[] index, mask, shift for SMP10 to SMP18 */
+	{ 0, GENMASK(2, 0), 0 },
+	{ 0, GENMASK(5, 3), 3 },
+	{ 0, GENMASK(8, 6), 6 },
+	{ 0, GENMASK(11, 9), 9 },
+	{ 0, GENMASK(14, 12), 12 },
+	{ 0, GENMASK(17, 15), 15 },
+	{ 0, GENMASK(20, 18), 18 },
+	{ 0, GENMASK(23, 21), 21 },
+	{ 0, GENMASK(26, 24), 24 },
+};
+
+/* STM32F4 programmable sampling time (ADC clock cycles) */
+static const unsigned int stm32f4_adc_smp_cycles[STM32_ADC_MAX_SMP + 1] = {
+	3, 15, 28, 56, 84, 112, 144, 480,
+};
+
 static const struct stm32_adc_regspec stm32f4_adc_regspec = {
 	.dr = STM32F4_ADC_DR,
 	.ier_eoc = { STM32F4_ADC_CR1, STM32F4_EOCIE },
@@ -440,6 +490,8 @@ static const struct stm32_adc_regspec stm32f4_adc_regspec = {
 	.extsel = { STM32F4_ADC_CR2, STM32F4_EXTSEL_MASK,
 		    STM32F4_EXTSEL_SHIFT },
 	.res = { STM32F4_ADC_CR1, STM32F4_RES_MASK, STM32F4_RES_SHIFT },
+	.smpr = { STM32F4_ADC_SMPR1, STM32F4_ADC_SMPR2 },
+	.smp_bits = stm32f4_smp_bits,
 };
 
 static const struct stm32_adc_regs stm32h7_sq[STM32_ADC_MAX_SQ + 1] = {
@@ -480,9 +532,46 @@ static struct stm32_adc_trig_info stm32h7_adc_trigs[] = {
 	{ TIM4_TRGO, STM32_EXT12 },
 	{ TIM6_TRGO, STM32_EXT13 },
 	{ TIM3_CH4, STM32_EXT15 },
+	{ LPTIM1_OUT, STM32_EXT18 },
+	{ LPTIM2_OUT, STM32_EXT19 },
+	{ LPTIM3_OUT, STM32_EXT20 },
 	{},
 };
 
+/**
+ * stm32h7_smp_bits - describe sampling time register index & bit fields
+ * Sorted so it can be indexed by channel number.
+ */
+static const struct stm32_adc_regs stm32h7_smp_bits[] = {
+	/* STM32H7_ADC_SMPR1, smpr[] index, mask, shift for SMP0 to SMP9 */
+	{ 0, GENMASK(2, 0), 0 },
+	{ 0, GENMASK(5, 3), 3 },
+	{ 0, GENMASK(8, 6), 6 },
+	{ 0, GENMASK(11, 9), 9 },
+	{ 0, GENMASK(14, 12), 12 },
+	{ 0, GENMASK(17, 15), 15 },
+	{ 0, GENMASK(20, 18), 18 },
+	{ 0, GENMASK(23, 21), 21 },
+	{ 0, GENMASK(26, 24), 24 },
+	{ 0, GENMASK(29, 27), 27 },
+	/* STM32H7_ADC_SMPR2, smpr[] index, mask, shift for SMP10 to SMP19 */
+	{ 1, GENMASK(2, 0), 0 },
+	{ 1, GENMASK(5, 3), 3 },
+	{ 1, GENMASK(8, 6), 6 },
+	{ 1, GENMASK(11, 9), 9 },
+	{ 1, GENMASK(14, 12), 12 },
+	{ 1, GENMASK(17, 15), 15 },
+	{ 1, GENMASK(20, 18), 18 },
+	{ 1, GENMASK(23, 21), 21 },
+	{ 1, GENMASK(26, 24), 24 },
+	{ 1, GENMASK(29, 27), 27 },
+};
+
+/* STM32H7 programmable sampling time (ADC clock cycles, rounded down) */
+static const unsigned int stm32h7_adc_smp_cycles[STM32_ADC_MAX_SMP + 1] = {
+	1, 2, 8, 16, 32, 64, 387, 810,
+};
+
 static const struct stm32_adc_regspec stm32h7_adc_regspec = {
 	.dr = STM32H7_ADC_DR,
 	.ier_eoc = { STM32H7_ADC_IER, STM32H7_EOCIE },
@@ -492,6 +581,8 @@ static const struct stm32_adc_regspec stm32h7_adc_regspec = {
 	.extsel = { STM32H7_ADC_CFGR, STM32H7_EXTSEL_MASK,
 		    STM32H7_EXTSEL_SHIFT },
 	.res = { STM32H7_ADC_CFGR, STM32H7_RES_MASK, STM32H7_RES_SHIFT },
+	.smpr = { STM32H7_ADC_SMPR1, STM32H7_ADC_SMPR2 },
+	.smp_bits = stm32h7_smp_bits,
 };
 
 /**
@@ -933,6 +1024,7 @@ static void stm32h7_adc_unprepare(struct stm32_adc *adc)
  * @scan_mask: channels to be converted
  *
  * Conversion sequence :
+ * Apply sampling time settings for all channels.
  * Configure ADC scan sequence based on selected channels in scan_mask.
  * Add channels to SQR registers, from scan_mask LSB to MSB, then
  * program sequence len.
@@ -946,6 +1038,10 @@ static int stm32_adc_conf_scan_seq(struct iio_dev *indio_dev,
 	u32 val, bit;
 	int i = 0;
 
+	/* Apply sampling time settings */
+	stm32_adc_writel(adc, adc->cfg->regs->smpr[0], adc->smpr_val[0]);
+	stm32_adc_writel(adc, adc->cfg->regs->smpr[1], adc->smpr_val[1]);
+
 	for_each_set_bit(bit, scan_mask, indio_dev->masklength) {
 		chan = indio_dev->channels + bit;
 		/*
@@ -995,7 +1091,8 @@ static int stm32_adc_get_trig_extsel(struct iio_dev *indio_dev,
 		 * Checking both stm32 timer trigger type and trig name
 		 * should be safe against arbitrary trigger names.
 		 */
-		if (is_stm32_timer_trigger(trig) &&
+		if ((is_stm32_timer_trigger(trig) ||
+		     is_stm32_lptim_trigger(trig)) &&
 		    !strcmp(adc->cfg->trigs[i].name, trig->name)) {
 			return adc->cfg->trigs[i].extsel;
 		}
@@ -1079,6 +1176,7 @@ static const struct iio_enum stm32_adc_trig_pol = {
  * @res: conversion result
  *
  * The function performs a single conversion on a given channel:
+ * - Apply sampling time settings
  * - Program sequencer with one channel (e.g. in SQ1 with len = 1)
  * - Use SW trigger
  * - Start conversion, then wait for interrupt completion.
@@ -1103,6 +1201,10 @@ static int stm32_adc_single_conv(struct iio_dev *indio_dev,
 			return ret;
 	}
 
+	/* Apply sampling time settings */
+	stm32_adc_writel(adc, regs->smpr[0], adc->smpr_val[0]);
+	stm32_adc_writel(adc, regs->smpr[1], adc->smpr_val[1]);
+
 	/* Program chan number in regular sequence (SQ1) */
 	val = stm32_adc_readl(adc, regs->sqr[1].reg);
 	val &= ~regs->sqr[1].mask;
@@ -1507,10 +1609,28 @@ static int stm32_adc_of_get_resolution(struct iio_dev *indio_dev)
 	return 0;
 }
 
+static void stm32_adc_smpr_init(struct stm32_adc *adc, int channel, u32 smp_ns)
+{
+	const struct stm32_adc_regs *smpr = &adc->cfg->regs->smp_bits[channel];
+	u32 period_ns, shift = smpr->shift, mask = smpr->mask;
+	unsigned int smp, r = smpr->reg;
+
+	/* Determine sampling time (ADC clock cycles) */
+	period_ns = NSEC_PER_SEC / adc->common->rate;
+	for (smp = 0; smp <= STM32_ADC_MAX_SMP; smp++)
+		if ((period_ns * adc->cfg->smp_cycles[smp]) >= smp_ns)
+			break;
+	if (smp > STM32_ADC_MAX_SMP)
+		smp = STM32_ADC_MAX_SMP;
+
+	/* pre-build sampling time registers (e.g. smpr1, smpr2) */
+	adc->smpr_val[r] = (adc->smpr_val[r] & ~mask) | (smp << shift);
+}
+
 static void stm32_adc_chan_init_one(struct iio_dev *indio_dev,
 				    struct iio_chan_spec *chan,
 				    const struct stm32_adc_chan_spec *channel,
-				    int scan_index)
+				    int scan_index, u32 smp)
 {
 	struct stm32_adc *adc = iio_priv(indio_dev);
 
@@ -1526,6 +1646,9 @@ static void stm32_adc_chan_init_one(struct iio_dev *indio_dev,
 	chan->scan_type.storagebits = 16;
 	chan->ext_info = stm32_adc_ext_info;
 
+	/* Prepare sampling time settings */
+	stm32_adc_smpr_init(adc, chan->channel, smp);
+
 	/* pre-build selected channels mask */
 	adc->pcsel |= BIT(chan->channel);
 }
@@ -1538,8 +1661,8 @@ static int stm32_adc_chan_of_init(struct iio_dev *indio_dev)
 	struct property *prop;
 	const __be32 *cur;
 	struct iio_chan_spec *channels;
-	int scan_index = 0, num_channels;
-	u32 val;
+	int scan_index = 0, num_channels, ret;
+	u32 val, smp = 0;
 
 	num_channels = of_property_count_u32_elems(node, "st,adc-channels");
 	if (num_channels < 0 ||
@@ -1548,6 +1671,13 @@ static int stm32_adc_chan_of_init(struct iio_dev *indio_dev)
 		return num_channels < 0 ? num_channels : -EINVAL;
 	}
 
+	/* Optional sample time is provided either for each, or all channels */
+	ret = of_property_count_u32_elems(node, "st,min-sample-time-nsecs");
+	if (ret > 1 && ret != num_channels) {
+		dev_err(&indio_dev->dev, "Invalid st,min-sample-time-nsecs\n");
+		return -EINVAL;
+	}
+
 	channels = devm_kcalloc(&indio_dev->dev, num_channels,
 				sizeof(struct iio_chan_spec), GFP_KERNEL);
 	if (!channels)
@@ -1558,9 +1688,19 @@ static int stm32_adc_chan_of_init(struct iio_dev *indio_dev)
 			dev_err(&indio_dev->dev, "Invalid channel %d\n", val);
 			return -EINVAL;
 		}
+
+		/*
+		 * Using of_property_read_u32_index(), smp value will only be
+		 * modified if valid u32 value can be decoded. This allows to
+		 * get either no value, 1 shared value for all indexes, or one
+		 * value per channel.
+		 */
+		of_property_read_u32_index(node, "st,min-sample-time-nsecs",
+					   scan_index, &smp);
+
 		stm32_adc_chan_init_one(indio_dev, &channels[scan_index],
 					&adc_info->channels[val],
-					scan_index);
+					scan_index, smp);
 		scan_index++;
 	}
 
@@ -1634,7 +1774,7 @@ static int stm32_adc_probe(struct platform_device *pdev)
 	indio_dev->dev.parent = &pdev->dev;
 	indio_dev->dev.of_node = pdev->dev.of_node;
 	indio_dev->info = &stm32_adc_iio_info;
-	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->modes = INDIO_DIRECT_MODE | INDIO_HARDWARE_TRIGGERED;
 
 	platform_set_drvdata(pdev, adc);
 
@@ -1755,6 +1895,7 @@ static const struct stm32_adc_cfg stm32f4_adc_cfg = {
 	.clk_required = true,
 	.start_conv = stm32f4_adc_start_conv,
 	.stop_conv = stm32f4_adc_stop_conv,
+	.smp_cycles = stm32f4_adc_smp_cycles,
 };
 
 static const struct stm32_adc_cfg stm32h7_adc_cfg = {
@@ -1766,6 +1907,7 @@ static const struct stm32_adc_cfg stm32h7_adc_cfg = {
 	.stop_conv = stm32h7_adc_stop_conv,
 	.prepare = stm32h7_adc_prepare,
 	.unprepare = stm32h7_adc_unprepare,
+	.smp_cycles = stm32h7_adc_smp_cycles,
 };
 
 static const struct of_device_id stm32_adc_of_match[] = {
diff --git a/drivers/iio/adc/ti-ads1015.c b/drivers/iio/adc/ti-ads1015.c
index 884b8e461b17..d1210024f6bc 100644
--- a/drivers/iio/adc/ti-ads1015.c
+++ b/drivers/iio/adc/ti-ads1015.c
@@ -17,6 +17,7 @@
 #include <linux/module.h>
 #include <linux/of_device.h>
 #include <linux/init.h>
+#include <linux/irq.h>
 #include <linux/i2c.h>
 #include <linux/regmap.h>
 #include <linux/pm_runtime.h>
@@ -28,6 +29,7 @@
 #include <linux/iio/iio.h>
 #include <linux/iio/types.h>
 #include <linux/iio/sysfs.h>
+#include <linux/iio/events.h>
 #include <linux/iio/buffer.h>
 #include <linux/iio/triggered_buffer.h>
 #include <linux/iio/trigger_consumer.h>
@@ -36,17 +38,38 @@
 
 #define ADS1015_CONV_REG	0x00
 #define ADS1015_CFG_REG		0x01
+#define ADS1015_LO_THRESH_REG	0x02
+#define ADS1015_HI_THRESH_REG	0x03
 
+#define ADS1015_CFG_COMP_QUE_SHIFT	0
+#define ADS1015_CFG_COMP_LAT_SHIFT	2
+#define ADS1015_CFG_COMP_POL_SHIFT	3
+#define ADS1015_CFG_COMP_MODE_SHIFT	4
 #define ADS1015_CFG_DR_SHIFT	5
 #define ADS1015_CFG_MOD_SHIFT	8
 #define ADS1015_CFG_PGA_SHIFT	9
 #define ADS1015_CFG_MUX_SHIFT	12
 
+#define ADS1015_CFG_COMP_QUE_MASK	GENMASK(1, 0)
+#define ADS1015_CFG_COMP_LAT_MASK	BIT(2)
+#define ADS1015_CFG_COMP_POL_MASK	BIT(2)
+#define ADS1015_CFG_COMP_MODE_MASK	BIT(4)
 #define ADS1015_CFG_DR_MASK	GENMASK(7, 5)
 #define ADS1015_CFG_MOD_MASK	BIT(8)
 #define ADS1015_CFG_PGA_MASK	GENMASK(11, 9)
 #define ADS1015_CFG_MUX_MASK	GENMASK(14, 12)
 
+/* Comparator queue and disable field */
+#define ADS1015_CFG_COMP_DISABLE	3
+
+/* Comparator polarity field */
+#define ADS1015_CFG_COMP_POL_LOW	0
+#define ADS1015_CFG_COMP_POL_HIGH	1
+
+/* Comparator mode field */
+#define ADS1015_CFG_COMP_MODE_TRAD	0
+#define ADS1015_CFG_COMP_MODE_WINDOW	1
+
 /* device operating modes */
 #define ADS1015_CONTINUOUS	0
 #define ADS1015_SINGLESHOT	1
@@ -81,18 +104,36 @@ static const unsigned int ads1115_data_rate[] = {
 	8, 16, 32, 64, 128, 250, 475, 860
 };
 
-static const struct {
-	int scale;
-	int uscale;
-} ads1015_scale[] = {
-	{3, 0},
-	{2, 0},
-	{1, 0},
-	{0, 500000},
-	{0, 250000},
-	{0, 125000},
-	{0, 125000},
-	{0, 125000},
+/*
+ * Translation from PGA bits to full-scale positive and negative input voltage
+ * range in mV
+ */
+static int ads1015_fullscale_range[] = {
+	6144, 4096, 2048, 1024, 512, 256, 256, 256
+};
+
+/*
+ * Translation from COMP_QUE field value to the number of successive readings
+ * exceed the threshold values before an interrupt is generated
+ */
+static const int ads1015_comp_queue[] = { 1, 2, 4 };
+
+static const struct iio_event_spec ads1015_events[] = {
+	{
+		.type = IIO_EV_TYPE_THRESH,
+		.dir = IIO_EV_DIR_RISING,
+		.mask_separate = BIT(IIO_EV_INFO_VALUE) |
+				BIT(IIO_EV_INFO_ENABLE),
+	}, {
+		.type = IIO_EV_TYPE_THRESH,
+		.dir = IIO_EV_DIR_FALLING,
+		.mask_separate = BIT(IIO_EV_INFO_VALUE),
+	}, {
+		.type = IIO_EV_TYPE_THRESH,
+		.dir = IIO_EV_DIR_EITHER,
+		.mask_separate = BIT(IIO_EV_INFO_ENABLE) |
+				BIT(IIO_EV_INFO_PERIOD),
+	},
 };
 
 #define ADS1015_V_CHAN(_chan, _addr) {				\
@@ -111,6 +152,8 @@ static const struct {
 		.shift = 4,					\
 		.endianness = IIO_CPU,				\
 	},							\
+	.event_spec = ads1015_events,				\
+	.num_event_specs = ARRAY_SIZE(ads1015_events),		\
 	.datasheet_name = "AIN"#_chan,				\
 }
 
@@ -132,6 +175,8 @@ static const struct {
 		.shift = 4,					\
 		.endianness = IIO_CPU,				\
 	},							\
+	.event_spec = ads1015_events,				\
+	.num_event_specs = ARRAY_SIZE(ads1015_events),		\
 	.datasheet_name = "AIN"#_chan"-AIN"#_chan2,		\
 }
 
@@ -150,6 +195,8 @@ static const struct {
 		.storagebits = 16,				\
 		.endianness = IIO_CPU,				\
 	},							\
+	.event_spec = ads1015_events,				\
+	.num_event_specs = ARRAY_SIZE(ads1015_events),		\
 	.datasheet_name = "AIN"#_chan,				\
 }
 
@@ -170,9 +217,17 @@ static const struct {
 		.storagebits = 16,				\
 		.endianness = IIO_CPU,				\
 	},							\
+	.event_spec = ads1015_events,				\
+	.num_event_specs = ARRAY_SIZE(ads1015_events),		\
 	.datasheet_name = "AIN"#_chan"-AIN"#_chan2,		\
 }
 
+struct ads1015_thresh_data {
+	unsigned int comp_queue;
+	int high_thresh;
+	int low_thresh;
+};
+
 struct ads1015_data {
 	struct regmap *regmap;
 	/*
@@ -182,18 +237,54 @@ struct ads1015_data {
 	struct mutex lock;
 	struct ads1015_channel_data channel_data[ADS1015_CHANNELS];
 
+	unsigned int event_channel;
+	unsigned int comp_mode;
+	struct ads1015_thresh_data thresh_data[ADS1015_CHANNELS];
+
 	unsigned int *data_rate;
+	/*
+	 * Set to true when the ADC is switched to the continuous-conversion
+	 * mode and exits from a power-down state.  This flag is used to avoid
+	 * getting the stale result from the conversion register.
+	 */
+	bool conv_invalid;
 };
 
+static bool ads1015_event_channel_enabled(struct ads1015_data *data)
+{
+	return (data->event_channel != ADS1015_CHANNELS);
+}
+
+static void ads1015_event_channel_enable(struct ads1015_data *data, int chan,
+					 int comp_mode)
+{
+	WARN_ON(ads1015_event_channel_enabled(data));
+
+	data->event_channel = chan;
+	data->comp_mode = comp_mode;
+}
+
+static void ads1015_event_channel_disable(struct ads1015_data *data, int chan)
+{
+	data->event_channel = ADS1015_CHANNELS;
+}
+
 static bool ads1015_is_writeable_reg(struct device *dev, unsigned int reg)
 {
-	return (reg == ADS1015_CFG_REG);
+	switch (reg) {
+	case ADS1015_CFG_REG:
+	case ADS1015_LO_THRESH_REG:
+	case ADS1015_HI_THRESH_REG:
+		return true;
+	default:
+		return false;
+	}
 }
 
 static const struct regmap_config ads1015_regmap_config = {
 	.reg_bits = 8,
 	.val_bits = 16,
-	.max_register = ADS1015_CFG_REG,
+	.max_register = ADS1015_HI_THRESH_REG,
 	.writeable_reg = ads1015_is_writeable_reg,
 };
 
@@ -235,33 +326,51 @@ static int ads1015_set_power_state(struct ads1015_data *data, bool on)
 		ret = pm_runtime_put_autosuspend(dev);
 	}
 
-	return ret;
+	return ret < 0 ? ret : 0;
 }
 
 static
 int ads1015_get_adc_result(struct ads1015_data *data, int chan, int *val)
 {
 	int ret, pga, dr, conv_time;
-	bool change;
+	unsigned int old, mask, cfg;
 
 	if (chan < 0 || chan >= ADS1015_CHANNELS)
 		return -EINVAL;
 
+	ret = regmap_read(data->regmap, ADS1015_CFG_REG, &old);
+	if (ret)
+		return ret;
+
 	pga = data->channel_data[chan].pga;
 	dr = data->channel_data[chan].data_rate;
+	mask = ADS1015_CFG_MUX_MASK | ADS1015_CFG_PGA_MASK |
+		ADS1015_CFG_DR_MASK;
+	cfg = chan << ADS1015_CFG_MUX_SHIFT | pga << ADS1015_CFG_PGA_SHIFT |
+		dr << ADS1015_CFG_DR_SHIFT;
+
+	if (ads1015_event_channel_enabled(data)) {
+		mask |= ADS1015_CFG_COMP_QUE_MASK | ADS1015_CFG_COMP_MODE_MASK;
+		cfg |= data->thresh_data[chan].comp_queue <<
+				ADS1015_CFG_COMP_QUE_SHIFT |
+			data->comp_mode <<
+				ADS1015_CFG_COMP_MODE_SHIFT;
+	}
 
-	ret = regmap_update_bits_check(data->regmap, ADS1015_CFG_REG,
-				       ADS1015_CFG_MUX_MASK |
-				       ADS1015_CFG_PGA_MASK,
-				       chan << ADS1015_CFG_MUX_SHIFT |
-				       pga << ADS1015_CFG_PGA_SHIFT,
-				       &change);
-	if (ret < 0)
+	cfg = (old & ~mask) | (cfg & mask);
+
+	ret = regmap_write(data->regmap, ADS1015_CFG_REG, cfg);
+	if (ret)
 		return ret;
 
-	if (change) {
-		conv_time = DIV_ROUND_UP(USEC_PER_SEC, data->data_rate[dr]);
+	if (old != cfg || data->conv_invalid) {
+		int dr_old = (old & ADS1015_CFG_DR_MASK) >>
+				ADS1015_CFG_DR_SHIFT;
+
+		conv_time = DIV_ROUND_UP(USEC_PER_SEC, data->data_rate[dr_old]);
+		conv_time += DIV_ROUND_UP(USEC_PER_SEC, data->data_rate[dr]);
 		usleep_range(conv_time, conv_time + 1);
+		data->conv_invalid = false;
 	}
 
 	return regmap_read(data->regmap, ADS1015_CONV_REG, val);
@@ -298,52 +407,36 @@ err:
 	return IRQ_HANDLED;
 }
 
-static int ads1015_set_scale(struct ads1015_data *data, int chan,
+static int ads1015_set_scale(struct ads1015_data *data,
+			     struct iio_chan_spec const *chan,
 			     int scale, int uscale)
 {
-	int i, ret, rindex = -1;
-
-	for (i = 0; i < ARRAY_SIZE(ads1015_scale); i++)
-		if (ads1015_scale[i].scale == scale &&
-		    ads1015_scale[i].uscale == uscale) {
-			rindex = i;
-			break;
+	int i;
+	int fullscale = div_s64((scale * 1000000LL + uscale) <<
+				(chan->scan_type.realbits - 1), 1000000);
+
+	for (i = 0; i < ARRAY_SIZE(ads1015_fullscale_range); i++) {
+		if (ads1015_fullscale_range[i] == fullscale) {
+			data->channel_data[chan->address].pga = i;
+			return 0;
 		}
-	if (rindex < 0)
-		return -EINVAL;
-
-	ret = regmap_update_bits(data->regmap, ADS1015_CFG_REG,
-				 ADS1015_CFG_PGA_MASK,
-				 rindex << ADS1015_CFG_PGA_SHIFT);
-	if (ret < 0)
-		return ret;
-
-	data->channel_data[chan].pga = rindex;
+	}
 
-	return 0;
+	return -EINVAL;
 }
 
 static int ads1015_set_data_rate(struct ads1015_data *data, int chan, int rate)
 {
-	int i, ret, rindex = -1;
+	int i;
 
-	for (i = 0; i < ARRAY_SIZE(ads1015_data_rate); i++)
+	for (i = 0; i < ARRAY_SIZE(ads1015_data_rate); i++) {
 		if (data->data_rate[i] == rate) {
-			rindex = i;
-			break;
+			data->channel_data[chan].data_rate = i;
+			return 0;
 		}
-	if (rindex < 0)
-		return -EINVAL;
-
-	ret = regmap_update_bits(data->regmap, ADS1015_CFG_REG,
-				 ADS1015_CFG_DR_MASK,
-				 rindex << ADS1015_CFG_DR_SHIFT);
-	if (ret < 0)
-		return ret;
-
-	data->channel_data[chan].data_rate = rindex;
+	}
 
-	return 0;
+	return -EINVAL;
 }
 
 static int ads1015_read_raw(struct iio_dev *indio_dev,
@@ -353,41 +446,47 @@ static int ads1015_read_raw(struct iio_dev *indio_dev,
 	int ret, idx;
 	struct ads1015_data *data = iio_priv(indio_dev);
 
-	mutex_lock(&indio_dev->mlock);
 	mutex_lock(&data->lock);
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW: {
 		int shift = chan->scan_type.shift;
 
-		if (iio_buffer_enabled(indio_dev)) {
-			ret = -EBUSY;
+		ret = iio_device_claim_direct_mode(indio_dev);
+		if (ret)
 			break;
+
+		if (ads1015_event_channel_enabled(data) &&
+				data->event_channel != chan->address) {
+			ret = -EBUSY;
+			goto release_direct;
 		}
 
 		ret = ads1015_set_power_state(data, true);
 		if (ret < 0)
-			break;
+			goto release_direct;
 
 		ret = ads1015_get_adc_result(data, chan->address, val);
 		if (ret < 0) {
 			ads1015_set_power_state(data, false);
-			break;
+			goto release_direct;
 		}
 
 		*val = sign_extend32(*val >> shift, 15 - shift);
 
 		ret = ads1015_set_power_state(data, false);
 		if (ret < 0)
-			break;
+			goto release_direct;
 
 		ret = IIO_VAL_INT;
+release_direct:
+		iio_device_release_direct_mode(indio_dev);
 		break;
 	}
 	case IIO_CHAN_INFO_SCALE:
 		idx = data->channel_data[chan->address].pga;
-		*val = ads1015_scale[idx].scale;
-		*val2 = ads1015_scale[idx].uscale;
-		ret = IIO_VAL_INT_PLUS_MICRO;
+		*val = ads1015_fullscale_range[idx];
+		*val2 = chan->scan_type.realbits - 1;
+		ret = IIO_VAL_FRACTIONAL_LOG2;
 		break;
 	case IIO_CHAN_INFO_SAMP_FREQ:
 		idx = data->channel_data[chan->address].data_rate;
@@ -399,7 +498,6 @@ static int ads1015_read_raw(struct iio_dev *indio_dev,
 		break;
 	}
 	mutex_unlock(&data->lock);
-	mutex_unlock(&indio_dev->mlock);
 
 	return ret;
 }
@@ -414,7 +512,7 @@ static int ads1015_write_raw(struct iio_dev *indio_dev,
 	mutex_lock(&data->lock);
 	switch (mask) {
 	case IIO_CHAN_INFO_SCALE:
-		ret = ads1015_set_scale(data, chan->address, val, val2);
+		ret = ads1015_set_scale(data, chan, val, val2);
 		break;
 	case IIO_CHAN_INFO_SAMP_FREQ:
 		ret = ads1015_set_data_rate(data, chan->address, val);
@@ -428,8 +526,254 @@ static int ads1015_write_raw(struct iio_dev *indio_dev,
 	return ret;
 }
 
+static int ads1015_read_event(struct iio_dev *indio_dev,
+	const struct iio_chan_spec *chan, enum iio_event_type type,
+	enum iio_event_direction dir, enum iio_event_info info, int *val,
+	int *val2)
+{
+	struct ads1015_data *data = iio_priv(indio_dev);
+	int ret;
+	unsigned int comp_queue;
+	int period;
+	int dr;
+
+	mutex_lock(&data->lock);
+
+	switch (info) {
+	case IIO_EV_INFO_VALUE:
+		*val = (dir == IIO_EV_DIR_RISING) ?
+			data->thresh_data[chan->address].high_thresh :
+			data->thresh_data[chan->address].low_thresh;
+		ret = IIO_VAL_INT;
+		break;
+	case IIO_EV_INFO_PERIOD:
+		dr = data->channel_data[chan->address].data_rate;
+		comp_queue = data->thresh_data[chan->address].comp_queue;
+		period = ads1015_comp_queue[comp_queue] *
+			USEC_PER_SEC / data->data_rate[dr];
+
+		*val = period / USEC_PER_SEC;
+		*val2 = period % USEC_PER_SEC;
+		ret = IIO_VAL_INT_PLUS_MICRO;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	mutex_unlock(&data->lock);
+
+	return ret;
+}
+
+static int ads1015_write_event(struct iio_dev *indio_dev,
+	const struct iio_chan_spec *chan, enum iio_event_type type,
+	enum iio_event_direction dir, enum iio_event_info info, int val,
+	int val2)
+{
+	struct ads1015_data *data = iio_priv(indio_dev);
+	int realbits = chan->scan_type.realbits;
+	int ret = 0;
+	long long period;
+	int i;
+	int dr;
+
+	mutex_lock(&data->lock);
+
+	switch (info) {
+	case IIO_EV_INFO_VALUE:
+		if (val >= 1 << (realbits - 1) || val < -1 << (realbits - 1)) {
+			ret = -EINVAL;
+			break;
+		}
+		if (dir == IIO_EV_DIR_RISING)
+			data->thresh_data[chan->address].high_thresh = val;
+		else
+			data->thresh_data[chan->address].low_thresh = val;
+		break;
+	case IIO_EV_INFO_PERIOD:
+		dr = data->channel_data[chan->address].data_rate;
+		period = val * USEC_PER_SEC + val2;
+
+		for (i = 0; i < ARRAY_SIZE(ads1015_comp_queue) - 1; i++) {
+			if (period <= ads1015_comp_queue[i] *
+					USEC_PER_SEC / data->data_rate[dr])
+				break;
+		}
+		data->thresh_data[chan->address].comp_queue = i;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	mutex_unlock(&data->lock);
+
+	return ret;
+}
+
+static int ads1015_read_event_config(struct iio_dev *indio_dev,
+	const struct iio_chan_spec *chan, enum iio_event_type type,
+	enum iio_event_direction dir)
+{
+	struct ads1015_data *data = iio_priv(indio_dev);
+	int ret = 0;
+
+	mutex_lock(&data->lock);
+	if (data->event_channel == chan->address) {
+		switch (dir) {
+		case IIO_EV_DIR_RISING:
+			ret = 1;
+			break;
+		case IIO_EV_DIR_EITHER:
+			ret = (data->comp_mode == ADS1015_CFG_COMP_MODE_WINDOW);
+			break;
+		default:
+			ret = -EINVAL;
+			break;
+		}
+	}
+	mutex_unlock(&data->lock);
+
+	return ret;
+}
+
+static int ads1015_enable_event_config(struct ads1015_data *data,
+	const struct iio_chan_spec *chan, int comp_mode)
+{
+	int low_thresh = data->thresh_data[chan->address].low_thresh;
+	int high_thresh = data->thresh_data[chan->address].high_thresh;
+	int ret;
+	unsigned int val;
+
+	if (ads1015_event_channel_enabled(data)) {
+		if (data->event_channel != chan->address ||
+			(data->comp_mode == ADS1015_CFG_COMP_MODE_TRAD &&
+				comp_mode == ADS1015_CFG_COMP_MODE_WINDOW))
+			return -EBUSY;
+
+		return 0;
+	}
+
+	if (comp_mode == ADS1015_CFG_COMP_MODE_TRAD) {
+		low_thresh = max(-1 << (chan->scan_type.realbits - 1),
+				high_thresh - 1);
+	}
+	ret = regmap_write(data->regmap, ADS1015_LO_THRESH_REG,
+			low_thresh << chan->scan_type.shift);
+	if (ret)
+		return ret;
+
+	ret = regmap_write(data->regmap, ADS1015_HI_THRESH_REG,
+			high_thresh << chan->scan_type.shift);
+	if (ret)
+		return ret;
+
+	ret = ads1015_set_power_state(data, true);
+	if (ret < 0)
+		return ret;
+
+	ads1015_event_channel_enable(data, chan->address, comp_mode);
+
+	ret = ads1015_get_adc_result(data, chan->address, &val);
+	if (ret) {
+		ads1015_event_channel_disable(data, chan->address);
+		ads1015_set_power_state(data, false);
+	}
+
+	return ret;
+}
+
+static int ads1015_disable_event_config(struct ads1015_data *data,
+	const struct iio_chan_spec *chan, int comp_mode)
+{
+	int ret;
+
+	if (!ads1015_event_channel_enabled(data))
+		return 0;
+
+	if (data->event_channel != chan->address)
+		return 0;
+
+	if (data->comp_mode == ADS1015_CFG_COMP_MODE_TRAD &&
+			comp_mode == ADS1015_CFG_COMP_MODE_WINDOW)
+		return 0;
+
+	ret = regmap_update_bits(data->regmap, ADS1015_CFG_REG,
+				ADS1015_CFG_COMP_QUE_MASK,
+				ADS1015_CFG_COMP_DISABLE <<
+					ADS1015_CFG_COMP_QUE_SHIFT);
+	if (ret)
+		return ret;
+
+	ads1015_event_channel_disable(data, chan->address);
+
+	return ads1015_set_power_state(data, false);
+}
+
+static int ads1015_write_event_config(struct iio_dev *indio_dev,
+	const struct iio_chan_spec *chan, enum iio_event_type type,
+	enum iio_event_direction dir, int state)
+{
+	struct ads1015_data *data = iio_priv(indio_dev);
+	int ret;
+	int comp_mode = (dir == IIO_EV_DIR_EITHER) ?
+		ADS1015_CFG_COMP_MODE_WINDOW : ADS1015_CFG_COMP_MODE_TRAD;
+
+	mutex_lock(&data->lock);
+
+	/* Prevent from enabling both buffer and event at a time */
+	ret = iio_device_claim_direct_mode(indio_dev);
+	if (ret) {
+		mutex_unlock(&data->lock);
+		return ret;
+	}
+
+	if (state)
+		ret = ads1015_enable_event_config(data, chan, comp_mode);
+	else
+		ret = ads1015_disable_event_config(data, chan, comp_mode);
+
+	iio_device_release_direct_mode(indio_dev);
+	mutex_unlock(&data->lock);
+
+	return ret;
+}
+
+static irqreturn_t ads1015_event_handler(int irq, void *priv)
+{
+	struct iio_dev *indio_dev = priv;
+	struct ads1015_data *data = iio_priv(indio_dev);
+	int val;
+	int ret;
+
+	/* Clear the latched ALERT/RDY pin */
+	ret = regmap_read(data->regmap, ADS1015_CONV_REG, &val);
+	if (ret)
+		return IRQ_HANDLED;
+
+	if (ads1015_event_channel_enabled(data)) {
+		enum iio_event_direction dir;
+		u64 code;
+
+		dir = data->comp_mode == ADS1015_CFG_COMP_MODE_TRAD ?
+					IIO_EV_DIR_RISING : IIO_EV_DIR_EITHER;
+		code = IIO_UNMOD_EVENT_CODE(IIO_VOLTAGE, data->event_channel,
+					IIO_EV_TYPE_THRESH, dir);
+		iio_push_event(indio_dev, code, iio_get_time_ns(indio_dev));
+	}
+
+	return IRQ_HANDLED;
+}
+
 static int ads1015_buffer_preenable(struct iio_dev *indio_dev)
 {
+	struct ads1015_data *data = iio_priv(indio_dev);
+
+	/* Prevent from enabling both buffer and event at a time */
+	if (ads1015_event_channel_enabled(data))
+		return -EBUSY;
+
 	return ads1015_set_power_state(iio_priv(indio_dev), true);
 }
 
@@ -446,7 +790,10 @@ static const struct iio_buffer_setup_ops ads1015_buffer_setup_ops = {
 	.validate_scan_mask = &iio_validate_scan_mask_onehot,
 };
 
-static IIO_CONST_ATTR(scale_available, "3 2 1 0.5 0.25 0.125");
+static IIO_CONST_ATTR_NAMED(ads1015_scale_available, scale_available,
+	"3 2 1 0.5 0.25 0.125");
+static IIO_CONST_ATTR_NAMED(ads1115_scale_available, scale_available,
+	"0.1875 0.125 0.0625 0.03125 0.015625 0.007813");
 
 static IIO_CONST_ATTR_NAMED(ads1015_sampling_frequency_available,
 	sampling_frequency_available, "128 250 490 920 1600 2400 3300");
@@ -454,7 +801,7 @@ static IIO_CONST_ATTR_NAMED(ads1115_sampling_frequency_available,
 	sampling_frequency_available, "8 16 32 64 128 250 475 860");
 
 static struct attribute *ads1015_attributes[] = {
-	&iio_const_attr_scale_available.dev_attr.attr,
+	&iio_const_attr_ads1015_scale_available.dev_attr.attr,
 	&iio_const_attr_ads1015_sampling_frequency_available.dev_attr.attr,
 	NULL,
 };
@@ -464,7 +811,7 @@ static const struct attribute_group ads1015_attribute_group = {
 };
 
 static struct attribute *ads1115_attributes[] = {
-	&iio_const_attr_scale_available.dev_attr.attr,
+	&iio_const_attr_ads1115_scale_available.dev_attr.attr,
 	&iio_const_attr_ads1115_sampling_frequency_available.dev_attr.attr,
 	NULL,
 };
@@ -477,6 +824,10 @@ static const struct iio_info ads1015_info = {
 	.driver_module	= THIS_MODULE,
 	.read_raw	= ads1015_read_raw,
 	.write_raw	= ads1015_write_raw,
+	.read_event_value = ads1015_read_event,
+	.write_event_value = ads1015_write_event,
+	.read_event_config = ads1015_read_event_config,
+	.write_event_config = ads1015_write_event_config,
 	.attrs          = &ads1015_attribute_group,
 };
 
@@ -484,6 +835,10 @@ static const struct iio_info ads1115_info = {
 	.driver_module	= THIS_MODULE,
 	.read_raw	= ads1015_read_raw,
 	.write_raw	= ads1015_write_raw,
+	.read_event_value = ads1015_read_event,
+	.write_event_value = ads1015_write_event,
+	.read_event_config = ads1015_read_event_config,
+	.write_event_config = ads1015_write_event_config,
 	.attrs          = &ads1115_attribute_group,
 };
 
@@ -505,24 +860,24 @@ static int ads1015_get_channels_config_of(struct i2c_client *client)
 		unsigned int data_rate = ADS1015_DEFAULT_DATA_RATE;
 
 		if (of_property_read_u32(node, "reg", &pval)) {
-			dev_err(&client->dev, "invalid reg on %s\n",
-				node->full_name);
+			dev_err(&client->dev, "invalid reg on %pOF\n",
+				node);
 			continue;
 		}
 
 		channel = pval;
 		if (channel >= ADS1015_CHANNELS) {
 			dev_err(&client->dev,
-				"invalid channel index %d on %s\n",
-				channel, node->full_name);
+				"invalid channel index %d on %pOF\n",
+				channel, node);
 			continue;
 		}
 
 		if (!of_property_read_u32(node, "ti,gain", &pval)) {
 			pga = pval;
 			if (pga > 6) {
-				dev_err(&client->dev, "invalid gain on %s\n",
-					node->full_name);
+				dev_err(&client->dev, "invalid gain on %pOF\n",
+					node);
 				of_node_put(node);
 				return -EINVAL;
 			}
@@ -532,8 +887,8 @@ static int ads1015_get_channels_config_of(struct i2c_client *client)
 			data_rate = pval;
 			if (data_rate > 7) {
 				dev_err(&client->dev,
-					"invalid data_rate on %s\n",
-					node->full_name);
+					"invalid data_rate on %pOF\n",
+					node);
 				of_node_put(node);
 				return -EINVAL;
 			}
@@ -573,6 +928,13 @@ static void ads1015_get_channels_config(struct i2c_client *client)
 	}
 }
 
+static int ads1015_set_conv_mode(struct ads1015_data *data, int mode)
+{
+	return regmap_update_bits(data->regmap, ADS1015_CFG_REG,
+				  ADS1015_CFG_MOD_MASK,
+				  mode << ADS1015_CFG_MOD_SHIFT);
+}
+
 static int ads1015_probe(struct i2c_client *client,
 			 const struct i2c_device_id *id)
 {
@@ -580,6 +942,7 @@ static int ads1015_probe(struct i2c_client *client,
 	struct ads1015_data *data;
 	int ret;
 	enum chip_ids chip;
+	int i;
 
 	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
 	if (!indio_dev)
@@ -614,6 +977,18 @@ static int ads1015_probe(struct i2c_client *client,
 		break;
 	}
 
+	data->event_channel = ADS1015_CHANNELS;
+	/*
+	 * Set default lower and upper threshold to min and max value
+	 * respectively.
+	 */
+	for (i = 0; i < ADS1015_CHANNELS; i++) {
+		int realbits = indio_dev->channels[i].scan_type.realbits;
+
+		data->thresh_data[i].low_thresh = -1 << (realbits - 1);
+		data->thresh_data[i].high_thresh = (1 << (realbits - 1)) - 1;
+	}
+
 	/* we need to keep this ABI the same as used by hwmon ADS1015 driver */
 	ads1015_get_channels_config(client);
 
@@ -623,16 +998,56 @@ static int ads1015_probe(struct i2c_client *client,
 		return PTR_ERR(data->regmap);
 	}
 
-	ret = iio_triggered_buffer_setup(indio_dev, NULL,
-					 ads1015_trigger_handler,
-					 &ads1015_buffer_setup_ops);
+	ret = devm_iio_triggered_buffer_setup(&client->dev, indio_dev, NULL,
+					      ads1015_trigger_handler,
+					      &ads1015_buffer_setup_ops);
 	if (ret < 0) {
 		dev_err(&client->dev, "iio triggered buffer setup failed\n");
 		return ret;
 	}
+
+	if (client->irq) {
+		unsigned long irq_trig =
+			irqd_get_trigger_type(irq_get_irq_data(client->irq));
+		unsigned int cfg_comp_mask = ADS1015_CFG_COMP_QUE_MASK |
+			ADS1015_CFG_COMP_LAT_MASK | ADS1015_CFG_COMP_POL_MASK;
+		unsigned int cfg_comp =
+			ADS1015_CFG_COMP_DISABLE << ADS1015_CFG_COMP_QUE_SHIFT |
+			1 << ADS1015_CFG_COMP_LAT_SHIFT;
+
+		switch (irq_trig) {
+		case IRQF_TRIGGER_LOW:
+			cfg_comp |= ADS1015_CFG_COMP_POL_LOW;
+			break;
+		case IRQF_TRIGGER_HIGH:
+			cfg_comp |= ADS1015_CFG_COMP_POL_HIGH;
+			break;
+		default:
+			return -EINVAL;
+		}
+
+		ret = regmap_update_bits(data->regmap, ADS1015_CFG_REG,
+					cfg_comp_mask, cfg_comp);
+		if (ret)
+			return ret;
+
+		ret = devm_request_threaded_irq(&client->dev, client->irq,
+						NULL, ads1015_event_handler,
+						irq_trig | IRQF_ONESHOT,
+						client->name, indio_dev);
+		if (ret)
+			return ret;
+	}
+
+	ret = ads1015_set_conv_mode(data, ADS1015_CONTINUOUS);
+	if (ret)
+		return ret;
+
+	data->conv_invalid = true;
+
 	ret = pm_runtime_set_active(&client->dev);
 	if (ret)
-		goto err_buffer_cleanup;
+		return ret;
 	pm_runtime_set_autosuspend_delay(&client->dev, ADS1015_SLEEP_DELAY_MS);
 	pm_runtime_use_autosuspend(&client->dev);
 	pm_runtime_enable(&client->dev);
@@ -640,15 +1055,10 @@ static int ads1015_probe(struct i2c_client *client,
 	ret = iio_device_register(indio_dev);
 	if (ret < 0) {
 		dev_err(&client->dev, "Failed to register IIO device\n");
-		goto err_buffer_cleanup;
+		return ret;
 	}
 
 	return 0;
-
-err_buffer_cleanup:
-	iio_triggered_buffer_cleanup(indio_dev);
-
-	return ret;
 }
 
 static int ads1015_remove(struct i2c_client *client)
@@ -662,12 +1072,8 @@ static int ads1015_remove(struct i2c_client *client)
 	pm_runtime_set_suspended(&client->dev);
 	pm_runtime_put_noidle(&client->dev);
 
-	iio_triggered_buffer_cleanup(indio_dev);
-
 	/* power down single shot mode */
-	return regmap_update_bits(data->regmap, ADS1015_CFG_REG,
-				  ADS1015_CFG_MOD_MASK,
-				  ADS1015_SINGLESHOT << ADS1015_CFG_MOD_SHIFT);
+	return ads1015_set_conv_mode(data, ADS1015_SINGLESHOT);
 }
 
 #ifdef CONFIG_PM
@@ -676,19 +1082,20 @@ static int ads1015_runtime_suspend(struct device *dev)
 	struct iio_dev *indio_dev = i2c_get_clientdata(to_i2c_client(dev));
 	struct ads1015_data *data = iio_priv(indio_dev);
 
-	return regmap_update_bits(data->regmap, ADS1015_CFG_REG,
-				  ADS1015_CFG_MOD_MASK,
-				  ADS1015_SINGLESHOT << ADS1015_CFG_MOD_SHIFT);
+	return ads1015_set_conv_mode(data, ADS1015_SINGLESHOT);
 }
 
 static int ads1015_runtime_resume(struct device *dev)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(to_i2c_client(dev));
 	struct ads1015_data *data = iio_priv(indio_dev);
+	int ret;
 
-	return regmap_update_bits(data->regmap, ADS1015_CFG_REG,
-				  ADS1015_CFG_MOD_MASK,
-				  ADS1015_CONTINUOUS << ADS1015_CFG_MOD_SHIFT);
+	ret = ads1015_set_conv_mode(data, ADS1015_CONTINUOUS);
+	if (!ret)
+		data->conv_invalid = true;
+
+	return ret;
 }
 #endif
 
diff --git a/drivers/iio/adc/ti-ads7950.c b/drivers/iio/adc/ti-ads7950.c
index 16a06633332c..a376190914ad 100644
--- a/drivers/iio/adc/ti-ads7950.c
+++ b/drivers/iio/adc/ti-ads7950.c
@@ -21,6 +21,7 @@
  * GNU General Public License for more details.
  */
 
+#include <linux/acpi.h>
 #include <linux/bitops.h>
 #include <linux/device.h>
 #include <linux/err.h>
@@ -37,6 +38,12 @@
 #include <linux/iio/trigger_consumer.h>
 #include <linux/iio/triggered_buffer.h>
 
+/*
+ * In case of ACPI, we use the 5000 mV as default for the reference pin.
+ * Device tree users encode that via the vref-supply regulator.
+ */
+#define TI_ADS7950_VA_MV_ACPI_DEFAULT	5000
+
 #define TI_ADS7950_CR_MANUAL	BIT(12)
 #define TI_ADS7950_CR_WRITE	BIT(11)
 #define TI_ADS7950_CR_CHAN(ch)	((ch) << 7)
@@ -58,6 +65,7 @@ struct ti_ads7950_state {
 	struct spi_message	scan_single_msg;
 
 	struct regulator	*reg;
+	unsigned int		vref_mv;
 
 	unsigned int		settings;
 
@@ -305,11 +313,15 @@ static int ti_ads7950_get_range(struct ti_ads7950_state *st)
 {
 	int vref;
 
-	vref = regulator_get_voltage(st->reg);
-	if (vref < 0)
-		return vref;
+	if (st->vref_mv) {
+		vref = st->vref_mv;
+	} else {
+		vref = regulator_get_voltage(st->reg);
+		if (vref < 0)
+			return vref;
 
-	vref /= 1000;
+		vref /= 1000;
+	}
 
 	if (st->settings & TI_ADS7950_CR_RANGE_5V)
 		vref *= 2;
@@ -411,6 +423,10 @@ static int ti_ads7950_probe(struct spi_device *spi)
 	spi_message_init_with_transfers(&st->scan_single_msg,
 					st->scan_single_xfer, 3);
 
+	/* Use hard coded value for reference voltage in ACPI case */
+	if (ACPI_COMPANION(&spi->dev))
+		st->vref_mv = TI_ADS7950_VA_MV_ACPI_DEFAULT;
+
 	st->reg = devm_regulator_get(&spi->dev, "vref");
 	if (IS_ERR(st->reg)) {
 		dev_err(&spi->dev, "Failed get get regulator \"vref\"\n");
@@ -475,9 +491,27 @@ static const struct spi_device_id ti_ads7950_id[] = {
 };
 MODULE_DEVICE_TABLE(spi, ti_ads7950_id);
 
+static const struct of_device_id ads7950_of_table[] = {
+	{ .compatible = "ti,ads7950", .data = &ti_ads7950_chip_info[TI_ADS7950] },
+	{ .compatible = "ti,ads7951", .data = &ti_ads7950_chip_info[TI_ADS7951] },
+	{ .compatible = "ti,ads7952", .data = &ti_ads7950_chip_info[TI_ADS7952] },
+	{ .compatible = "ti,ads7953", .data = &ti_ads7950_chip_info[TI_ADS7953] },
+	{ .compatible = "ti,ads7954", .data = &ti_ads7950_chip_info[TI_ADS7954] },
+	{ .compatible = "ti,ads7955", .data = &ti_ads7950_chip_info[TI_ADS7955] },
+	{ .compatible = "ti,ads7956", .data = &ti_ads7950_chip_info[TI_ADS7956] },
+	{ .compatible = "ti,ads7957", .data = &ti_ads7950_chip_info[TI_ADS7957] },
+	{ .compatible = "ti,ads7958", .data = &ti_ads7950_chip_info[TI_ADS7958] },
+	{ .compatible = "ti,ads7959", .data = &ti_ads7950_chip_info[TI_ADS7959] },
+	{ .compatible = "ti,ads7960", .data = &ti_ads7950_chip_info[TI_ADS7960] },
+	{ .compatible = "ti,ads7961", .data = &ti_ads7950_chip_info[TI_ADS7961] },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, ads7950_of_table);
+
 static struct spi_driver ti_ads7950_driver = {
 	.driver = {
 		.name	= "ads7950",
+		.of_match_table = ads7950_of_table,
 	},
 	.probe		= ti_ads7950_probe,
 	.remove		= ti_ads7950_remove,
diff --git a/drivers/iio/adc/twl4030-madc.c b/drivers/iio/adc/twl4030-madc.c
index bd3d37fc2144..1edd99f0c5e5 100644
--- a/drivers/iio/adc/twl4030-madc.c
+++ b/drivers/iio/adc/twl4030-madc.c
@@ -35,7 +35,7 @@
 #include <linux/delay.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
-#include <linux/i2c/twl.h>
+#include <linux/mfd/twl.h>
 #include <linux/module.h>
 #include <linux/stddef.h>
 #include <linux/mutex.h>
diff --git a/drivers/iio/adc/twl6030-gpadc.c b/drivers/iio/adc/twl6030-gpadc.c
index becbb0aef232..bc0e60b9da45 100644
--- a/drivers/iio/adc/twl6030-gpadc.c
+++ b/drivers/iio/adc/twl6030-gpadc.c
@@ -33,7 +33,7 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/of_platform.h>
-#include <linux/i2c/twl.h>
+#include <linux/mfd/twl.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 
diff --git a/drivers/iio/adc/xilinx-xadc-events.c b/drivers/iio/adc/xilinx-xadc-events.c
index 6d5c2a6f4e6e..dc0670308253 100644
--- a/drivers/iio/adc/xilinx-xadc-events.c
+++ b/drivers/iio/adc/xilinx-xadc-events.c
@@ -68,7 +68,7 @@ void xadc_handle_events(struct iio_dev *indio_dev, unsigned long events)
 		xadc_handle_event(indio_dev, i);
 }
 
-static unsigned xadc_get_threshold_offset(const struct iio_chan_spec *chan,
+static unsigned int xadc_get_threshold_offset(const struct iio_chan_spec *chan,
 	enum iio_event_direction dir)
 {
 	unsigned int offset;
@@ -90,26 +90,24 @@ static unsigned xadc_get_threshold_offset(const struct iio_chan_spec *chan,
 
 static unsigned int xadc_get_alarm_mask(const struct iio_chan_spec *chan)
 {
-	if (chan->type == IIO_TEMP) {
+	if (chan->type == IIO_TEMP)
 		return XADC_ALARM_OT_MASK;
-	} else {
-		switch (chan->channel) {
-		case 0:
-			return XADC_ALARM_VCCINT_MASK;
-		case 1:
-			return XADC_ALARM_VCCAUX_MASK;
-		case 2:
-			return XADC_ALARM_VCCBRAM_MASK;
-		case 3:
-			return XADC_ALARM_VCCPINT_MASK;
-		case 4:
-			return XADC_ALARM_VCCPAUX_MASK;
-		case 5:
-			return XADC_ALARM_VCCODDR_MASK;
-		default:
-			/* We will never get here */
-			return 0;
-		}
+	switch (chan->channel) {
+	case 0:
+		return XADC_ALARM_VCCINT_MASK;
+	case 1:
+		return XADC_ALARM_VCCAUX_MASK;
+	case 2:
+		return XADC_ALARM_VCCBRAM_MASK;
+	case 3:
+		return XADC_ALARM_VCCPINT_MASK;
+	case 4:
+		return XADC_ALARM_VCCPAUX_MASK;
+	case 5:
+		return XADC_ALARM_VCCODDR_MASK;
+	default:
+		/* We will never get here */
+		return 0;
 	}
 }
 
diff --git a/drivers/iio/adc/xilinx-xadc.h b/drivers/iio/adc/xilinx-xadc.h
index f6f081965647..62edbdae1244 100644
--- a/drivers/iio/adc/xilinx-xadc.h
+++ b/drivers/iio/adc/xilinx-xadc.h
@@ -71,13 +71,13 @@ struct xadc {
 };
 
 struct xadc_ops {
-	int (*read)(struct xadc *, unsigned int, uint16_t *);
-	int (*write)(struct xadc *, unsigned int, uint16_t);
+	int (*read)(struct xadc *xadc, unsigned int reg, uint16_t *val);
+	int (*write)(struct xadc *xadc, unsigned int reg, uint16_t val);
 	int (*setup)(struct platform_device *pdev, struct iio_dev *indio_dev,
 			int irq);
-	void (*update_alarm)(struct xadc *, unsigned int);
-	unsigned long (*get_dclk_rate)(struct xadc *);
-	irqreturn_t (*interrupt_handler)(int, void *);
+	void (*update_alarm)(struct xadc *xadc, unsigned int alarm);
+	unsigned long (*get_dclk_rate)(struct xadc *xadc);
+	irqreturn_t (*interrupt_handler)(int irq, void *devid);
 
 	unsigned int flags;
 };
diff --git a/drivers/iio/chemical/Kconfig b/drivers/iio/chemical/Kconfig
index cea7f9857a1f..5cb5be7612b4 100644
--- a/drivers/iio/chemical/Kconfig
+++ b/drivers/iio/chemical/Kconfig
@@ -21,6 +21,15 @@ config ATLAS_PH_SENSOR
 	 To compile this driver as module, choose M here: the
 	 module will be called atlas-ph-sensor.
 
+config CCS811
+	tristate "AMS CCS811 VOC sensor"
+	depends on I2C
+    select IIO_BUFFER
+    select IIO_TRIGGERED_BUFFER
+	help
+	  Say Y here to build I2C interface support for the AMS
+	  CCS811 VOC (Volatile Organic Compounds) sensor
+
 config IAQCORE
 	tristate "AMS iAQ-Core VOC sensors"
 	depends on I2C
diff --git a/drivers/iio/chemical/Makefile b/drivers/iio/chemical/Makefile
index b02202b41289..a629b29d1e0b 100644
--- a/drivers/iio/chemical/Makefile
+++ b/drivers/iio/chemical/Makefile
@@ -4,5 +4,6 @@
 
 # When adding new entries keep the list in alphabetical order
 obj-$(CONFIG_ATLAS_PH_SENSOR)	+= atlas-ph-sensor.o
+obj-$(CONFIG_CCS811)		+= ccs811.o
 obj-$(CONFIG_IAQCORE)		+= ams-iaq-core.o
 obj-$(CONFIG_VZ89X)		+= vz89x.o
diff --git a/drivers/iio/chemical/ccs811.c b/drivers/iio/chemical/ccs811.c
new file mode 100644
index 000000000000..840a6cbd5f0f
--- /dev/null
+++ b/drivers/iio/chemical/ccs811.c
@@ -0,0 +1,405 @@
+/*
+ * ccs811.c - Support for AMS CCS811 VOC Sensor
+ *
+ * Copyright (C) 2017 Narcisa Vasile <narcisaanamaria12@gmail.com>
+ *
+ * Datasheet: ams.com/content/download/951091/2269479/CCS811_DS000459_3-00.pdf
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * IIO driver for AMS CCS811 (I2C address 0x5A/0x5B set by ADDR Low/High)
+ *
+ * TODO:
+ * 1. Make the drive mode selectable form userspace
+ * 2. Add support for interrupts
+ * 3. Adjust time to wait for data to be ready based on selected operation mode
+ * 4. Read error register and put the information in logs
+ */
+
+#include <linux/delay.h>
+#include <linux/i2c.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/buffer.h>
+#include <linux/iio/triggered_buffer.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/module.h>
+
+#define CCS811_STATUS		0x00
+#define CCS811_MEAS_MODE	0x01
+#define CCS811_ALG_RESULT_DATA	0x02
+#define CCS811_RAW_DATA		0x03
+#define CCS811_HW_ID		0x20
+#define CCS881_HW_ID_VALUE	0x81
+#define CCS811_HW_VERSION	0x21
+#define CCS811_HW_VERSION_VALUE	0x10
+#define CCS811_HW_VERSION_MASK	0xF0
+#define CCS811_ERR		0xE0
+/* Used to transition from boot to application mode */
+#define CCS811_APP_START	0xF4
+
+/* Status register flags */
+#define CCS811_STATUS_ERROR		BIT(0)
+#define CCS811_STATUS_DATA_READY	BIT(3)
+#define CCS811_STATUS_APP_VALID_MASK	BIT(4)
+#define CCS811_STATUS_APP_VALID_LOADED	BIT(4)
+/*
+ * Value of FW_MODE bit of STATUS register describes the sensor's state:
+ * 0: Firmware is in boot mode, this allows new firmware to be loaded
+ * 1: Firmware is in application mode. CCS811 is ready to take ADC measurements
+ */
+#define CCS811_STATUS_FW_MODE_MASK	BIT(7)
+#define CCS811_STATUS_FW_MODE_APPLICATION	BIT(7)
+
+/* Measurement modes */
+#define CCS811_MODE_IDLE	0x00
+#define CCS811_MODE_IAQ_1SEC	0x10
+#define CCS811_MODE_IAQ_10SEC	0x20
+#define CCS811_MODE_IAQ_60SEC	0x30
+#define CCS811_MODE_RAW_DATA	0x40
+
+#define CCS811_VOLTAGE_MASK	0x3FF
+
+struct ccs811_reading {
+	__be16 co2;
+	__be16 voc;
+	u8 status;
+	u8 error;
+	__be16 resistance;
+} __attribute__((__packed__));
+
+struct ccs811_data {
+	struct i2c_client *client;
+	struct mutex lock; /* Protect readings */
+	struct ccs811_reading buffer;
+};
+
+static const struct iio_chan_spec ccs811_channels[] = {
+	{
+		.type = IIO_CURRENT,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_SCALE),
+		.scan_index = -1,
+	}, {
+		.type = IIO_VOLTAGE,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_SCALE),
+		.scan_index = -1,
+	}, {
+		.type = IIO_CONCENTRATION,
+		.channel2 = IIO_MOD_CO2,
+		.modified = 1,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_OFFSET) |
+				      BIT(IIO_CHAN_INFO_SCALE),
+		.scan_index = 0,
+		.scan_type = {
+			.sign = 'u',
+			.realbits = 16,
+			.storagebits = 16,
+			.endianness = IIO_BE,
+		},
+	}, {
+		.type = IIO_CONCENTRATION,
+		.channel2 = IIO_MOD_VOC,
+		.modified = 1,
+		.info_mask_separate =  BIT(IIO_CHAN_INFO_RAW) |
+				       BIT(IIO_CHAN_INFO_SCALE),
+		.scan_index = 1,
+		.scan_type = {
+			.sign = 'u',
+			.realbits = 16,
+			.storagebits = 16,
+			.endianness = IIO_BE,
+		},
+	},
+	IIO_CHAN_SOFT_TIMESTAMP(2),
+};
+
+/*
+ * The CCS811 powers-up in boot mode. A setup write to CCS811_APP_START will
+ * transition the sensor to application mode.
+ */
+static int ccs811_start_sensor_application(struct i2c_client *client)
+{
+	int ret;
+
+	ret = i2c_smbus_read_byte_data(client, CCS811_STATUS);
+	if (ret < 0)
+		return ret;
+
+	if ((ret & CCS811_STATUS_APP_VALID_MASK) !=
+	    CCS811_STATUS_APP_VALID_LOADED)
+		return -EIO;
+
+	ret = i2c_smbus_write_byte(client, CCS811_APP_START);
+	if (ret < 0)
+		return ret;
+
+	ret = i2c_smbus_read_byte_data(client, CCS811_STATUS);
+	if (ret < 0)
+		return ret;
+
+	if ((ret & CCS811_STATUS_FW_MODE_MASK) !=
+	    CCS811_STATUS_FW_MODE_APPLICATION) {
+		dev_err(&client->dev, "Application failed to start. Sensor is still in boot mode.\n");
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static int ccs811_setup(struct i2c_client *client)
+{
+	int ret;
+
+	ret = ccs811_start_sensor_application(client);
+	if (ret < 0)
+		return ret;
+
+	return i2c_smbus_write_byte_data(client, CCS811_MEAS_MODE,
+					 CCS811_MODE_IAQ_1SEC);
+}
+
+static int ccs811_get_measurement(struct ccs811_data *data)
+{
+	int ret, tries = 11;
+
+	/* Maximum waiting time: 1s, as measurements are made every second */
+	while (tries-- > 0) {
+		ret = i2c_smbus_read_byte_data(data->client, CCS811_STATUS);
+		if (ret < 0)
+			return ret;
+
+		if ((ret & CCS811_STATUS_DATA_READY) || tries == 0)
+			break;
+		msleep(100);
+	}
+	if (!(ret & CCS811_STATUS_DATA_READY))
+		return -EIO;
+
+	return i2c_smbus_read_i2c_block_data(data->client,
+					    CCS811_ALG_RESULT_DATA, 8,
+					    (char *)&data->buffer);
+}
+
+static int ccs811_read_raw(struct iio_dev *indio_dev,
+			   struct iio_chan_spec const *chan,
+			   int *val, int *val2, long mask)
+{
+	struct ccs811_data *data = iio_priv(indio_dev);
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		mutex_lock(&data->lock);
+		ret = ccs811_get_measurement(data);
+		if (ret < 0) {
+			mutex_unlock(&data->lock);
+			return ret;
+		}
+
+		switch (chan->type) {
+		case IIO_VOLTAGE:
+			*val = be16_to_cpu(data->buffer.resistance) &
+					   CCS811_VOLTAGE_MASK;
+			ret = IIO_VAL_INT;
+			break;
+		case IIO_CURRENT:
+			*val = be16_to_cpu(data->buffer.resistance) >> 10;
+			ret = IIO_VAL_INT;
+			break;
+		case IIO_CONCENTRATION:
+			switch (chan->channel2) {
+			case IIO_MOD_CO2:
+				*val = be16_to_cpu(data->buffer.co2);
+				ret =  IIO_VAL_INT;
+				break;
+			case IIO_MOD_VOC:
+				*val = be16_to_cpu(data->buffer.voc);
+				ret = IIO_VAL_INT;
+				break;
+			default:
+				ret = -EINVAL;
+			}
+			break;
+		default:
+			ret = -EINVAL;
+		}
+		mutex_unlock(&data->lock);
+
+		return ret;
+
+	case IIO_CHAN_INFO_SCALE:
+		switch (chan->type) {
+		case IIO_VOLTAGE:
+			*val = 1;
+			*val2 = 612903;
+			return IIO_VAL_INT_PLUS_MICRO;
+		case IIO_CURRENT:
+			*val = 0;
+			*val2 = 1000;
+			return IIO_VAL_INT_PLUS_MICRO;
+		case IIO_CONCENTRATION:
+			switch (chan->channel2) {
+			case IIO_MOD_CO2:
+				*val = 0;
+				*val2 = 12834;
+				return IIO_VAL_INT_PLUS_MICRO;
+			case IIO_MOD_VOC:
+				*val = 0;
+				*val2 = 84246;
+				return IIO_VAL_INT_PLUS_MICRO;
+			default:
+				return -EINVAL;
+			}
+		default:
+			return -EINVAL;
+		}
+	case IIO_CHAN_INFO_OFFSET:
+		if (!(chan->type == IIO_CONCENTRATION &&
+		      chan->channel2 == IIO_MOD_CO2))
+			return -EINVAL;
+		*val = -400;
+		return IIO_VAL_INT;
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct iio_info ccs811_info = {
+	.read_raw = ccs811_read_raw,
+	.driver_module = THIS_MODULE,
+};
+
+static irqreturn_t ccs811_trigger_handler(int irq, void *p)
+{
+	struct iio_poll_func *pf = p;
+	struct iio_dev *indio_dev = pf->indio_dev;
+	struct ccs811_data *data = iio_priv(indio_dev);
+	struct i2c_client *client = data->client;
+	s16 buf[8]; /* s16 eCO2 + s16 TVOC + padding + 8 byte timestamp */
+	int ret;
+
+	ret = i2c_smbus_read_i2c_block_data(client, CCS811_ALG_RESULT_DATA, 4,
+					    (u8 *)&buf);
+	if (ret != 4) {
+		dev_err(&client->dev, "cannot read sensor data\n");
+		goto err;
+	}
+
+	iio_push_to_buffers_with_timestamp(indio_dev, buf,
+					   iio_get_time_ns(indio_dev));
+
+err:
+	iio_trigger_notify_done(indio_dev->trig);
+
+	return IRQ_HANDLED;
+}
+
+static int ccs811_probe(struct i2c_client *client,
+			const struct i2c_device_id *id)
+{
+	struct iio_dev *indio_dev;
+	struct ccs811_data *data;
+	int ret;
+
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_WRITE_BYTE
+				     | I2C_FUNC_SMBUS_BYTE_DATA
+				     | I2C_FUNC_SMBUS_READ_I2C_BLOCK))
+		return -EOPNOTSUPP;
+
+	/* Check hardware id (should be 0x81 for this family of devices) */
+	ret = i2c_smbus_read_byte_data(client, CCS811_HW_ID);
+	if (ret < 0)
+		return ret;
+
+	if (ret != CCS881_HW_ID_VALUE) {
+		dev_err(&client->dev, "hardware id doesn't match CCS81x\n");
+		return -ENODEV;
+	}
+
+	ret = i2c_smbus_read_byte_data(client, CCS811_HW_VERSION);
+	if (ret < 0)
+		return ret;
+
+	if ((ret & CCS811_HW_VERSION_MASK) != CCS811_HW_VERSION_VALUE) {
+		dev_err(&client->dev, "no CCS811 sensor\n");
+		return -ENODEV;
+	}
+
+	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	ret = ccs811_setup(client);
+	if (ret < 0)
+		return ret;
+
+	data = iio_priv(indio_dev);
+	i2c_set_clientdata(client, indio_dev);
+	data->client = client;
+
+	mutex_init(&data->lock);
+
+	indio_dev->dev.parent = &client->dev;
+	indio_dev->name = id->name;
+	indio_dev->info = &ccs811_info;
+
+	indio_dev->channels = ccs811_channels;
+	indio_dev->num_channels = ARRAY_SIZE(ccs811_channels);
+
+	ret = iio_triggered_buffer_setup(indio_dev, NULL,
+					 ccs811_trigger_handler, NULL);
+
+	if (ret < 0) {
+		dev_err(&client->dev, "triggered buffer setup failed\n");
+		goto err_poweroff;
+	}
+
+	ret = iio_device_register(indio_dev);
+	if (ret < 0) {
+		dev_err(&client->dev, "unable to register iio device\n");
+		goto err_buffer_cleanup;
+	}
+	return 0;
+
+err_buffer_cleanup:
+	iio_triggered_buffer_cleanup(indio_dev);
+err_poweroff:
+	i2c_smbus_write_byte_data(client, CCS811_MEAS_MODE, CCS811_MODE_IDLE);
+
+	return ret;
+}
+
+static int ccs811_remove(struct i2c_client *client)
+{
+	struct iio_dev *indio_dev = i2c_get_clientdata(client);
+
+	iio_device_unregister(indio_dev);
+	iio_triggered_buffer_cleanup(indio_dev);
+
+	return i2c_smbus_write_byte_data(client, CCS811_MEAS_MODE,
+					 CCS811_MODE_IDLE);
+}
+
+static const struct i2c_device_id ccs811_id[] = {
+	{"ccs811", 0},
+	{	}
+};
+MODULE_DEVICE_TABLE(i2c, ccs811_id);
+
+static struct i2c_driver ccs811_driver = {
+	.driver = {
+		.name = "ccs811",
+	},
+	.probe = ccs811_probe,
+	.remove = ccs811_remove,
+	.id_table = ccs811_id,
+};
+module_i2c_driver(ccs811_driver);
+
+MODULE_AUTHOR("Narcisa Vasile <narcisaanamaria12@gmail.com>");
+MODULE_DESCRIPTION("CCS811 volatile organic compounds sensor");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/common/hid-sensors/hid-sensor-attributes.c b/drivers/iio/common/hid-sensors/hid-sensor-attributes.c
index f5d4d786e193..ed3849d6fc6a 100644
--- a/drivers/iio/common/hid-sensors/hid-sensor-attributes.c
+++ b/drivers/iio/common/hid-sensors/hid-sensor-attributes.c
@@ -473,6 +473,9 @@ int hid_sensor_parse_common_attributes(struct hid_sensor_hub_device *hsdev,
 					HID_USAGE_SENSOR_PROY_POWER_STATE,
 					&st->power_state);
 
+	st->power_state.logical_minimum = 1;
+	st->report_state.logical_minimum = 1;
+
 	sensor_hub_input_get_attribute_info(hsdev,
 			HID_FEATURE_REPORT, usage_id,
 			HID_USAGE_SENSOR_PROP_SENSITIVITY_ABS,
diff --git a/drivers/iio/common/hid-sensors/hid-sensor-trigger.c b/drivers/iio/common/hid-sensors/hid-sensor-trigger.c
index 16ade0a0327b..0e4b379ada45 100644
--- a/drivers/iio/common/hid-sensors/hid-sensor-trigger.c
+++ b/drivers/iio/common/hid-sensors/hid-sensor-trigger.c
@@ -111,8 +111,6 @@ static int _hid_sensor_power_state(struct hid_sensor_common *st, bool state)
 	s32 poll_value = 0;
 
 	if (state) {
-		if (!atomic_read(&st->user_requested_state))
-			return 0;
 		if (sensor_hub_device_open(st->hsdev))
 			return -EIO;
 
@@ -161,6 +159,9 @@ static int _hid_sensor_power_state(struct hid_sensor_common *st, bool state)
 				       &report_val);
 	}
 
+	pr_debug("HID_SENSOR %s set power_state %d report_state %d\n",
+		 st->pdev->name, state_val, report_val);
+
 	sensor_hub_get_feature(st->hsdev, st->power_state.report_id,
 			       st->power_state.index,
 			       sizeof(state_val), &state_val);
@@ -182,6 +183,7 @@ int hid_sensor_power_state(struct hid_sensor_common *st, bool state)
 		ret = pm_runtime_get_sync(&st->pdev->dev);
 	else {
 		pm_runtime_mark_last_busy(&st->pdev->dev);
+		pm_runtime_use_autosuspend(&st->pdev->dev);
 		ret = pm_runtime_put_autosuspend(&st->pdev->dev);
 	}
 	if (ret < 0) {
@@ -285,8 +287,6 @@ int hid_sensor_setup_trigger(struct iio_dev *indio_dev, const char *name,
 	/* Default to 3 seconds, but can be changed from sysfs */
 	pm_runtime_set_autosuspend_delay(&attrb->pdev->dev,
 					 3000);
-	pm_runtime_use_autosuspend(&attrb->pdev->dev);
-
 	return ret;
 error_unreg_trigger:
 	iio_trigger_unregister(trig);
diff --git a/drivers/iio/common/st_sensors/st_sensors_core.c b/drivers/iio/common/st_sensors/st_sensors_core.c
index 6e6a1ecc99dd..d99bb1460fe2 100644
--- a/drivers/iio/common/st_sensors/st_sensors_core.c
+++ b/drivers/iio/common/st_sensors/st_sensors_core.c
@@ -15,6 +15,7 @@
 #include <linux/iio/iio.h>
 #include <linux/regulator/consumer.h>
 #include <linux/of.h>
+#include <linux/of_device.h>
 #include <asm/unaligned.h>
 #include <linux/iio/common/st_sensors.h>
 
@@ -345,6 +346,36 @@ static struct st_sensors_platform_data *st_sensors_of_probe(struct device *dev,
 
 	return pdata;
 }
+
+/**
+ * st_sensors_of_name_probe() - device tree probe for ST sensor name
+ * @dev: driver model representation of the device.
+ * @match: the OF match table for the device, containing compatible strings
+ *	but also a .data field with the corresponding internal kernel name
+ *	used by this sensor.
+ * @name: device name buffer reference.
+ * @len: device name buffer length.
+ *
+ * In effect this function matches a compatible string to an internal kernel
+ * name for a certain sensor device, so that the rest of the autodetection can
+ * rely on that name from this point on. I2C/SPI devices will be renamed
+ * to match the internal kernel convention.
+ */
+void st_sensors_of_name_probe(struct device *dev,
+			      const struct of_device_id *match,
+			      char *name, int len)
+{
+	const struct of_device_id *of_id;
+
+	of_id = of_match_device(match, dev);
+	if (!of_id || !of_id->data)
+		return;
+
+	/* The name from the OF match takes precedence if present */
+	strncpy(name, of_id->data, len);
+	name[len - 1] = '\0';
+}
+EXPORT_SYMBOL(st_sensors_of_name_probe);
 #else
 static struct st_sensors_platform_data *st_sensors_of_probe(struct device *dev,
 		struct st_sensors_platform_data *defdata)
diff --git a/drivers/iio/common/st_sensors/st_sensors_i2c.c b/drivers/iio/common/st_sensors/st_sensors_i2c.c
index c83df4dbfcd7..b81e48e9f27e 100644
--- a/drivers/iio/common/st_sensors/st_sensors_i2c.c
+++ b/drivers/iio/common/st_sensors/st_sensors_i2c.c
@@ -79,35 +79,6 @@ void st_sensors_i2c_configure(struct iio_dev *indio_dev,
 }
 EXPORT_SYMBOL(st_sensors_i2c_configure);
 
-#ifdef CONFIG_OF
-/**
- * st_sensors_of_i2c_probe() - device tree probe for ST I2C sensors
- * @client: the I2C client device for the sensor
- * @match: the OF match table for the device, containing compatible strings
- *	but also a .data field with the corresponding internal kernel name
- *	used by this sensor.
- *
- * In effect this function matches a compatible string to an internal kernel
- * name for a certain sensor device, so that the rest of the autodetection can
- * rely on that name from this point on. I2C client devices will be renamed
- * to match the internal kernel convention.
- */
-void st_sensors_of_i2c_probe(struct i2c_client *client,
-			     const struct of_device_id *match)
-{
-	const struct of_device_id *of_id;
-
-	of_id = of_match_device(match, &client->dev);
-	if (!of_id)
-		return;
-
-	/* The name from the OF match takes precedence if present */
-	strncpy(client->name, of_id->data, sizeof(client->name));
-	client->name[sizeof(client->name) - 1] = '\0';
-}
-EXPORT_SYMBOL(st_sensors_of_i2c_probe);
-#endif
-
 #ifdef CONFIG_ACPI
 int st_sensors_match_acpi_device(struct device *dev)
 {
diff --git a/drivers/iio/counter/Kconfig b/drivers/iio/counter/Kconfig
index b37e5fc03149..474e1ac4e7c0 100644
--- a/drivers/iio/counter/Kconfig
+++ b/drivers/iio/counter/Kconfig
@@ -21,4 +21,13 @@ config 104_QUAD_8
 	  The base port addresses for the devices may be configured via the base
 	  array module parameter.
 
+config STM32_LPTIMER_CNT
+	tristate "STM32 LP Timer encoder counter driver"
+	depends on MFD_STM32_LPTIMER || COMPILE_TEST
+	help
+	  Select this option to enable STM32 Low-Power Timer quadrature encoder
+	  and counter driver.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called stm32-lptimer-cnt.
 endmenu
diff --git a/drivers/iio/counter/Makefile b/drivers/iio/counter/Makefile
index 007e88411648..1b9a896eb488 100644
--- a/drivers/iio/counter/Makefile
+++ b/drivers/iio/counter/Makefile
@@ -5,3 +5,4 @@
 # When adding new entries keep the list in alphabetical order
 
 obj-$(CONFIG_104_QUAD_8)	+= 104-quad-8.o
+obj-$(CONFIG_STM32_LPTIMER_CNT)	+= stm32-lptimer-cnt.o
diff --git a/drivers/iio/counter/stm32-lptimer-cnt.c b/drivers/iio/counter/stm32-lptimer-cnt.c
new file mode 100644
index 000000000000..1c5909bb1605
--- /dev/null
+++ b/drivers/iio/counter/stm32-lptimer-cnt.c
@@ -0,0 +1,383 @@
+/*
+ * STM32 Low-Power Timer Encoder and Counter driver
+ *
+ * Copyright (C) STMicroelectronics 2017
+ *
+ * Author: Fabrice Gasnier <fabrice.gasnier@st.com>
+ *
+ * Inspired by 104-quad-8 and stm32-timer-trigger drivers.
+ *
+ * License terms:  GNU General Public License (GPL), version 2
+ */
+
+#include <linux/bitfield.h>
+#include <linux/iio/iio.h>
+#include <linux/mfd/stm32-lptimer.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+struct stm32_lptim_cnt {
+	struct device *dev;
+	struct regmap *regmap;
+	struct clk *clk;
+	u32 preset;
+	u32 polarity;
+	u32 quadrature_mode;
+};
+
+static int stm32_lptim_is_enabled(struct stm32_lptim_cnt *priv)
+{
+	u32 val;
+	int ret;
+
+	ret = regmap_read(priv->regmap, STM32_LPTIM_CR, &val);
+	if (ret)
+		return ret;
+
+	return FIELD_GET(STM32_LPTIM_ENABLE, val);
+}
+
+static int stm32_lptim_set_enable_state(struct stm32_lptim_cnt *priv,
+					int enable)
+{
+	int ret;
+	u32 val;
+
+	val = FIELD_PREP(STM32_LPTIM_ENABLE, enable);
+	ret = regmap_write(priv->regmap, STM32_LPTIM_CR, val);
+	if (ret)
+		return ret;
+
+	if (!enable) {
+		clk_disable(priv->clk);
+		return 0;
+	}
+
+	/* LP timer must be enabled before writing CMP & ARR */
+	ret = regmap_write(priv->regmap, STM32_LPTIM_ARR, priv->preset);
+	if (ret)
+		return ret;
+
+	ret = regmap_write(priv->regmap, STM32_LPTIM_CMP, 0);
+	if (ret)
+		return ret;
+
+	/* ensure CMP & ARR registers are properly written */
+	ret = regmap_read_poll_timeout(priv->regmap, STM32_LPTIM_ISR, val,
+				       (val & STM32_LPTIM_CMPOK_ARROK),
+				       100, 1000);
+	if (ret)
+		return ret;
+
+	ret = regmap_write(priv->regmap, STM32_LPTIM_ICR,
+			   STM32_LPTIM_CMPOKCF_ARROKCF);
+	if (ret)
+		return ret;
+
+	ret = clk_enable(priv->clk);
+	if (ret) {
+		regmap_write(priv->regmap, STM32_LPTIM_CR, 0);
+		return ret;
+	}
+
+	/* Start LP timer in continuous mode */
+	return regmap_update_bits(priv->regmap, STM32_LPTIM_CR,
+				  STM32_LPTIM_CNTSTRT, STM32_LPTIM_CNTSTRT);
+}
+
+static int stm32_lptim_setup(struct stm32_lptim_cnt *priv, int enable)
+{
+	u32 mask = STM32_LPTIM_ENC | STM32_LPTIM_COUNTMODE |
+		   STM32_LPTIM_CKPOL | STM32_LPTIM_PRESC;
+	u32 val;
+
+	/* Setup LP timer encoder/counter and polarity, without prescaler */
+	if (priv->quadrature_mode)
+		val = enable ? STM32_LPTIM_ENC : 0;
+	else
+		val = enable ? STM32_LPTIM_COUNTMODE : 0;
+	val |= FIELD_PREP(STM32_LPTIM_CKPOL, enable ? priv->polarity : 0);
+
+	return regmap_update_bits(priv->regmap, STM32_LPTIM_CFGR, mask, val);
+}
+
+static int stm32_lptim_write_raw(struct iio_dev *indio_dev,
+				 struct iio_chan_spec const *chan,
+				 int val, int val2, long mask)
+{
+	struct stm32_lptim_cnt *priv = iio_priv(indio_dev);
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_ENABLE:
+		if (val < 0 || val > 1)
+			return -EINVAL;
+
+		/* Check nobody uses the timer, or already disabled/enabled */
+		ret = stm32_lptim_is_enabled(priv);
+		if ((ret < 0) || (!ret && !val))
+			return ret;
+		if (val && ret)
+			return -EBUSY;
+
+		ret = stm32_lptim_setup(priv, val);
+		if (ret)
+			return ret;
+		return stm32_lptim_set_enable_state(priv, val);
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static int stm32_lptim_read_raw(struct iio_dev *indio_dev,
+				struct iio_chan_spec const *chan,
+				int *val, int *val2, long mask)
+{
+	struct stm32_lptim_cnt *priv = iio_priv(indio_dev);
+	u32 dat;
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		ret = regmap_read(priv->regmap, STM32_LPTIM_CNT, &dat);
+		if (ret)
+			return ret;
+		*val = dat;
+		return IIO_VAL_INT;
+
+	case IIO_CHAN_INFO_ENABLE:
+		ret = stm32_lptim_is_enabled(priv);
+		if (ret < 0)
+			return ret;
+		*val = ret;
+		return IIO_VAL_INT;
+
+	case IIO_CHAN_INFO_SCALE:
+		/* Non-quadrature mode: scale = 1 */
+		*val = 1;
+		*val2 = 0;
+		if (priv->quadrature_mode) {
+			/*
+			 * Quadrature encoder mode:
+			 * - both edges, quarter cycle, scale is 0.25
+			 * - either rising/falling edge scale is 0.5
+			 */
+			if (priv->polarity > 1)
+				*val2 = 2;
+			else
+				*val2 = 1;
+		}
+		return IIO_VAL_FRACTIONAL_LOG2;
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct iio_info stm32_lptim_cnt_iio_info = {
+	.read_raw = stm32_lptim_read_raw,
+	.write_raw = stm32_lptim_write_raw,
+	.driver_module = THIS_MODULE,
+};
+
+static const char *const stm32_lptim_quadrature_modes[] = {
+	"non-quadrature",
+	"quadrature",
+};
+
+static int stm32_lptim_get_quadrature_mode(struct iio_dev *indio_dev,
+					   const struct iio_chan_spec *chan)
+{
+	struct stm32_lptim_cnt *priv = iio_priv(indio_dev);
+
+	return priv->quadrature_mode;
+}
+
+static int stm32_lptim_set_quadrature_mode(struct iio_dev *indio_dev,
+					   const struct iio_chan_spec *chan,
+					   unsigned int type)
+{
+	struct stm32_lptim_cnt *priv = iio_priv(indio_dev);
+
+	if (stm32_lptim_is_enabled(priv))
+		return -EBUSY;
+
+	priv->quadrature_mode = type;
+
+	return 0;
+}
+
+static const struct iio_enum stm32_lptim_quadrature_mode_en = {
+	.items = stm32_lptim_quadrature_modes,
+	.num_items = ARRAY_SIZE(stm32_lptim_quadrature_modes),
+	.get = stm32_lptim_get_quadrature_mode,
+	.set = stm32_lptim_set_quadrature_mode,
+};
+
+static const char * const stm32_lptim_cnt_polarity[] = {
+	"rising-edge", "falling-edge", "both-edges",
+};
+
+static int stm32_lptim_cnt_get_polarity(struct iio_dev *indio_dev,
+					const struct iio_chan_spec *chan)
+{
+	struct stm32_lptim_cnt *priv = iio_priv(indio_dev);
+
+	return priv->polarity;
+}
+
+static int stm32_lptim_cnt_set_polarity(struct iio_dev *indio_dev,
+					const struct iio_chan_spec *chan,
+					unsigned int type)
+{
+	struct stm32_lptim_cnt *priv = iio_priv(indio_dev);
+
+	if (stm32_lptim_is_enabled(priv))
+		return -EBUSY;
+
+	priv->polarity = type;
+
+	return 0;
+}
+
+static const struct iio_enum stm32_lptim_cnt_polarity_en = {
+	.items = stm32_lptim_cnt_polarity,
+	.num_items = ARRAY_SIZE(stm32_lptim_cnt_polarity),
+	.get = stm32_lptim_cnt_get_polarity,
+	.set = stm32_lptim_cnt_set_polarity,
+};
+
+static ssize_t stm32_lptim_cnt_get_preset(struct iio_dev *indio_dev,
+					  uintptr_t private,
+					  const struct iio_chan_spec *chan,
+					  char *buf)
+{
+	struct stm32_lptim_cnt *priv = iio_priv(indio_dev);
+
+	return snprintf(buf, PAGE_SIZE, "%u\n", priv->preset);
+}
+
+static ssize_t stm32_lptim_cnt_set_preset(struct iio_dev *indio_dev,
+					  uintptr_t private,
+					  const struct iio_chan_spec *chan,
+					  const char *buf, size_t len)
+{
+	struct stm32_lptim_cnt *priv = iio_priv(indio_dev);
+	int ret;
+
+	if (stm32_lptim_is_enabled(priv))
+		return -EBUSY;
+
+	ret = kstrtouint(buf, 0, &priv->preset);
+	if (ret)
+		return ret;
+
+	if (priv->preset > STM32_LPTIM_MAX_ARR)
+		return -EINVAL;
+
+	return len;
+}
+
+/* LP timer with encoder */
+static const struct iio_chan_spec_ext_info stm32_lptim_enc_ext_info[] = {
+	{
+		.name = "preset",
+		.shared = IIO_SEPARATE,
+		.read = stm32_lptim_cnt_get_preset,
+		.write = stm32_lptim_cnt_set_preset,
+	},
+	IIO_ENUM("polarity", IIO_SEPARATE, &stm32_lptim_cnt_polarity_en),
+	IIO_ENUM_AVAILABLE("polarity", &stm32_lptim_cnt_polarity_en),
+	IIO_ENUM("quadrature_mode", IIO_SEPARATE,
+		 &stm32_lptim_quadrature_mode_en),
+	IIO_ENUM_AVAILABLE("quadrature_mode", &stm32_lptim_quadrature_mode_en),
+	{}
+};
+
+static const struct iio_chan_spec stm32_lptim_enc_channels = {
+	.type = IIO_COUNT,
+	.channel = 0,
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+			      BIT(IIO_CHAN_INFO_ENABLE) |
+			      BIT(IIO_CHAN_INFO_SCALE),
+	.ext_info = stm32_lptim_enc_ext_info,
+	.indexed = 1,
+};
+
+/* LP timer without encoder (counter only) */
+static const struct iio_chan_spec_ext_info stm32_lptim_cnt_ext_info[] = {
+	{
+		.name = "preset",
+		.shared = IIO_SEPARATE,
+		.read = stm32_lptim_cnt_get_preset,
+		.write = stm32_lptim_cnt_set_preset,
+	},
+	IIO_ENUM("polarity", IIO_SEPARATE, &stm32_lptim_cnt_polarity_en),
+	IIO_ENUM_AVAILABLE("polarity", &stm32_lptim_cnt_polarity_en),
+	{}
+};
+
+static const struct iio_chan_spec stm32_lptim_cnt_channels = {
+	.type = IIO_COUNT,
+	.channel = 0,
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+			      BIT(IIO_CHAN_INFO_ENABLE) |
+			      BIT(IIO_CHAN_INFO_SCALE),
+	.ext_info = stm32_lptim_cnt_ext_info,
+	.indexed = 1,
+};
+
+static int stm32_lptim_cnt_probe(struct platform_device *pdev)
+{
+	struct stm32_lptimer *ddata = dev_get_drvdata(pdev->dev.parent);
+	struct stm32_lptim_cnt *priv;
+	struct iio_dev *indio_dev;
+
+	if (IS_ERR_OR_NULL(ddata))
+		return -EINVAL;
+
+	indio_dev = devm_iio_device_alloc(&pdev->dev, sizeof(*priv));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	priv = iio_priv(indio_dev);
+	priv->dev = &pdev->dev;
+	priv->regmap = ddata->regmap;
+	priv->clk = ddata->clk;
+	priv->preset = STM32_LPTIM_MAX_ARR;
+
+	indio_dev->name = dev_name(&pdev->dev);
+	indio_dev->dev.parent = &pdev->dev;
+	indio_dev->dev.of_node = pdev->dev.of_node;
+	indio_dev->info = &stm32_lptim_cnt_iio_info;
+	if (ddata->has_encoder)
+		indio_dev->channels = &stm32_lptim_enc_channels;
+	else
+		indio_dev->channels = &stm32_lptim_cnt_channels;
+	indio_dev->num_channels = 1;
+
+	platform_set_drvdata(pdev, priv);
+
+	return devm_iio_device_register(&pdev->dev, indio_dev);
+}
+
+static const struct of_device_id stm32_lptim_cnt_of_match[] = {
+	{ .compatible = "st,stm32-lptimer-counter", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, stm32_lptim_cnt_of_match);
+
+static struct platform_driver stm32_lptim_cnt_driver = {
+	.probe = stm32_lptim_cnt_probe,
+	.driver = {
+		.name = "stm32-lptimer-counter",
+		.of_match_table = stm32_lptim_cnt_of_match,
+	},
+};
+module_platform_driver(stm32_lptim_cnt_driver);
+
+MODULE_AUTHOR("Fabrice Gasnier <fabrice.gasnier@st.com>");
+MODULE_ALIAS("platform:stm32-lptimer-counter");
+MODULE_DESCRIPTION("STMicroelectronics STM32 LPTIM counter driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/dac/stm32-dac-core.c b/drivers/iio/dac/stm32-dac-core.c
index 75e48788c7ea..55026fe1c610 100644
--- a/drivers/iio/dac/stm32-dac-core.c
+++ b/drivers/iio/dac/stm32-dac-core.c
@@ -42,6 +42,14 @@ struct stm32_dac_priv {
 	struct stm32_dac_common common;
 };
 
+/**
+ * struct stm32_dac_cfg - DAC configuration
+ * @has_hfsel: DAC has high frequency control
+ */
+struct stm32_dac_cfg {
+	bool has_hfsel;
+};
+
 static struct stm32_dac_priv *to_stm32_dac_priv(struct stm32_dac_common *com)
 {
 	return container_of(com, struct stm32_dac_priv, common);
@@ -57,6 +65,7 @@ static const struct regmap_config stm32_dac_regmap_cfg = {
 static int stm32_dac_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
+	const struct stm32_dac_cfg *cfg;
 	struct stm32_dac_priv *priv;
 	struct regmap *regmap;
 	struct resource *res;
@@ -69,6 +78,8 @@ static int stm32_dac_probe(struct platform_device *pdev)
 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
+	cfg = (const struct stm32_dac_cfg *)
+		of_match_device(dev->driver->of_match_table, dev)->data;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	mmio = devm_ioremap_resource(dev, res);
@@ -114,19 +125,23 @@ static int stm32_dac_probe(struct platform_device *pdev)
 		goto err_vref;
 	}
 
-	priv->rst = devm_reset_control_get(dev, NULL);
+	priv->rst = devm_reset_control_get_exclusive(dev, NULL);
 	if (!IS_ERR(priv->rst)) {
 		reset_control_assert(priv->rst);
 		udelay(2);
 		reset_control_deassert(priv->rst);
 	}
 
-	/* When clock speed is higher than 80MHz, set HFSEL */
-	priv->common.hfsel = (clk_get_rate(priv->pclk) > 80000000UL);
-	ret = regmap_update_bits(regmap, STM32_DAC_CR, STM32H7_DAC_CR_HFSEL,
-				 priv->common.hfsel ? STM32H7_DAC_CR_HFSEL : 0);
-	if (ret)
-		goto err_pclk;
+	if (cfg && cfg->has_hfsel) {
+		/* When clock speed is higher than 80MHz, set HFSEL */
+		priv->common.hfsel = (clk_get_rate(priv->pclk) > 80000000UL);
+		ret = regmap_update_bits(regmap, STM32_DAC_CR,
+					 STM32H7_DAC_CR_HFSEL,
+					 priv->common.hfsel ?
+					 STM32H7_DAC_CR_HFSEL : 0);
+		if (ret)
+			goto err_pclk;
+	}
 
 	platform_set_drvdata(pdev, &priv->common);
 
@@ -158,8 +173,17 @@ static int stm32_dac_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static const struct stm32_dac_cfg stm32h7_dac_cfg = {
+	.has_hfsel = true,
+};
+
 static const struct of_device_id stm32_dac_of_match[] = {
-	{ .compatible = "st,stm32h7-dac-core", },
+	{
+		.compatible = "st,stm32f4-dac-core",
+	}, {
+		.compatible = "st,stm32h7-dac-core",
+		.data = (void *)&stm32h7_dac_cfg,
+	},
 	{},
 };
 MODULE_DEVICE_TABLE(of, stm32_dac_of_match);
diff --git a/drivers/iio/dac/stm32-dac.c b/drivers/iio/dac/stm32-dac.c
index 50f8ec091058..c1864e8aa851 100644
--- a/drivers/iio/dac/stm32-dac.c
+++ b/drivers/iio/dac/stm32-dac.c
@@ -268,7 +268,7 @@ static int stm32_dac_chan_of_init(struct iio_dev *indio_dev)
 			break;
 	}
 	if (i >= ARRAY_SIZE(stm32_dac_channels)) {
-		dev_err(&indio_dev->dev, "Invalid st,dac-channel\n");
+		dev_err(&indio_dev->dev, "Invalid reg property\n");
 		return -EINVAL;
 	}
 
diff --git a/drivers/iio/gyro/mpu3050-core.c b/drivers/iio/gyro/mpu3050-core.c
index 2be2a5d287e6..e0d241a9aa30 100644
--- a/drivers/iio/gyro/mpu3050-core.c
+++ b/drivers/iio/gyro/mpu3050-core.c
@@ -1063,11 +1063,6 @@ static int mpu3050_trigger_probe(struct iio_dev *indio_dev, int irq)
 	case IRQF_TRIGGER_RISING:
 		dev_info(&indio_dev->dev,
 			 "pulse interrupts on the rising edge\n");
-		if (mpu3050->irq_opendrain) {
-			dev_info(&indio_dev->dev,
-				 "rising edge incompatible with open drain\n");
-			mpu3050->irq_opendrain = false;
-		}
 		break;
 	case IRQF_TRIGGER_FALLING:
 		mpu3050->irq_actl = true;
@@ -1078,11 +1073,6 @@ static int mpu3050_trigger_probe(struct iio_dev *indio_dev, int irq)
 		mpu3050->irq_latch = true;
 		dev_info(&indio_dev->dev,
 			 "interrupts active high level\n");
-		if (mpu3050->irq_opendrain) {
-			dev_info(&indio_dev->dev,
-				 "active high incompatible with open drain\n");
-			mpu3050->irq_opendrain = false;
-		}
 		/*
 		 * With level IRQs, we mask the IRQ until it is processed,
 		 * but with edge IRQs (pulses) we can queue several interrupts
diff --git a/drivers/iio/gyro/st_gyro.h b/drivers/iio/gyro/st_gyro.h
index a5c5c4e29add..48923ae6ac3b 100644
--- a/drivers/iio/gyro/st_gyro.h
+++ b/drivers/iio/gyro/st_gyro.h
@@ -19,6 +19,7 @@
 #define LSM330DL_GYRO_DEV_NAME		"lsm330dl_gyro"
 #define LSM330DLC_GYRO_DEV_NAME		"lsm330dlc_gyro"
 #define L3GD20_GYRO_DEV_NAME		"l3gd20"
+#define L3GD20H_GYRO_DEV_NAME		"l3gd20h"
 #define L3G4IS_GYRO_DEV_NAME		"l3g4is_ui"
 #define LSM330_GYRO_DEV_NAME		"lsm330_gyro"
 #define LSM9DS0_GYRO_DEV_NAME		"lsm9ds0_gyro"
diff --git a/drivers/iio/gyro/st_gyro_core.c b/drivers/iio/gyro/st_gyro_core.c
index 2a42b3d583e8..e366422e8512 100644
--- a/drivers/iio/gyro/st_gyro_core.c
+++ b/drivers/iio/gyro/st_gyro_core.c
@@ -35,6 +35,7 @@
 #define ST_GYRO_DEFAULT_OUT_Z_L_ADDR		0x2c
 
 /* FULLSCALE */
+#define ST_GYRO_FS_AVL_245DPS			245
 #define ST_GYRO_FS_AVL_250DPS			250
 #define ST_GYRO_FS_AVL_500DPS			500
 #define ST_GYRO_FS_AVL_2000DPS			2000
@@ -196,17 +197,17 @@ static const struct st_sensor_settings st_gyro_sensors_settings[] = {
 		.wai = 0xd7,
 		.wai_addr = ST_SENSORS_DEFAULT_WAI_ADDRESS,
 		.sensors_supported = {
-			[0] = L3GD20_GYRO_DEV_NAME,
+			[0] = L3GD20H_GYRO_DEV_NAME,
 		},
 		.ch = (struct iio_chan_spec *)st_gyro_16bit_channels,
 		.odr = {
 			.addr = 0x20,
 			.mask = 0xc0,
 			.odr_avl = {
-				{ .hz = 95, .value = 0x00, },
-				{ .hz = 190, .value = 0x01, },
-				{ .hz = 380, .value = 0x02, },
-				{ .hz = 760, .value = 0x03, },
+				{ .hz = 100, .value = 0x00, },
+				{ .hz = 200, .value = 0x01, },
+				{ .hz = 400, .value = 0x02, },
+				{ .hz = 800, .value = 0x03, },
 			},
 		},
 		.pw = {
@@ -224,7 +225,7 @@ static const struct st_sensor_settings st_gyro_sensors_settings[] = {
 			.mask = 0x30,
 			.fs_avl = {
 				[0] = {
-					.num = ST_GYRO_FS_AVL_250DPS,
+					.num = ST_GYRO_FS_AVL_245DPS,
 					.value = 0x00,
 					.gain = IIO_DEGREE_TO_RAD(8750),
 				},
diff --git a/drivers/iio/gyro/st_gyro_i2c.c b/drivers/iio/gyro/st_gyro_i2c.c
index 40056b821036..b405b82b9177 100644
--- a/drivers/iio/gyro/st_gyro_i2c.c
+++ b/drivers/iio/gyro/st_gyro_i2c.c
@@ -41,6 +41,10 @@ static const struct of_device_id st_gyro_of_match[] = {
 		.data = L3GD20_GYRO_DEV_NAME,
 	},
 	{
+		.compatible = "st,l3gd20h-gyro",
+		.data = L3GD20H_GYRO_DEV_NAME,
+	},
+	{
 		.compatible = "st,l3g4is-gyro",
 		.data = L3G4IS_GYRO_DEV_NAME,
 	},
@@ -71,7 +75,8 @@ static int st_gyro_i2c_probe(struct i2c_client *client,
 		return -ENOMEM;
 
 	gdata = iio_priv(indio_dev);
-	st_sensors_of_i2c_probe(client, st_gyro_of_match);
+	st_sensors_of_name_probe(&client->dev, st_gyro_of_match,
+				 client->name, sizeof(client->name));
 
 	st_sensors_i2c_configure(indio_dev, client, gdata);
 
@@ -95,6 +100,7 @@ static const struct i2c_device_id st_gyro_id_table[] = {
 	{ LSM330DL_GYRO_DEV_NAME },
 	{ LSM330DLC_GYRO_DEV_NAME },
 	{ L3GD20_GYRO_DEV_NAME },
+	{ L3GD20H_GYRO_DEV_NAME },
 	{ L3G4IS_GYRO_DEV_NAME },
 	{ LSM330_GYRO_DEV_NAME },
 	{ LSM9DS0_GYRO_DEV_NAME },
diff --git a/drivers/iio/gyro/st_gyro_spi.c b/drivers/iio/gyro/st_gyro_spi.c
index fbf2faed501c..0b52ed577dc2 100644
--- a/drivers/iio/gyro/st_gyro_spi.c
+++ b/drivers/iio/gyro/st_gyro_spi.c
@@ -18,6 +18,56 @@
 #include <linux/iio/common/st_sensors_spi.h>
 #include "st_gyro.h"
 
+#ifdef CONFIG_OF
+/*
+ * For new single-chip sensors use <device_name> as compatible string.
+ * For old single-chip devices keep <device_name>-gyro to maintain
+ * compatibility
+ */
+static const struct of_device_id st_gyro_of_match[] = {
+	{
+		.compatible = "st,l3g4200d-gyro",
+		.data = L3G4200D_GYRO_DEV_NAME,
+	},
+	{
+		.compatible = "st,lsm330d-gyro",
+		.data = LSM330D_GYRO_DEV_NAME,
+	},
+	{
+		.compatible = "st,lsm330dl-gyro",
+		.data = LSM330DL_GYRO_DEV_NAME,
+	},
+	{
+		.compatible = "st,lsm330dlc-gyro",
+		.data = LSM330DLC_GYRO_DEV_NAME,
+	},
+	{
+		.compatible = "st,l3gd20-gyro",
+		.data = L3GD20_GYRO_DEV_NAME,
+	},
+	{
+		.compatible = "st,l3gd20h-gyro",
+		.data = L3GD20H_GYRO_DEV_NAME,
+	},
+	{
+		.compatible = "st,l3g4is-gyro",
+		.data = L3G4IS_GYRO_DEV_NAME,
+	},
+	{
+		.compatible = "st,lsm330-gyro",
+		.data = LSM330_GYRO_DEV_NAME,
+	},
+	{
+		.compatible = "st,lsm9ds0-gyro",
+		.data = LSM9DS0_GYRO_DEV_NAME,
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(of, st_gyro_of_match);
+#else
+#define st_gyro_of_match	NULL
+#endif
+
 static int st_gyro_spi_probe(struct spi_device *spi)
 {
 	struct iio_dev *indio_dev;
@@ -30,6 +80,8 @@ static int st_gyro_spi_probe(struct spi_device *spi)
 
 	gdata = iio_priv(indio_dev);
 
+	st_sensors_of_name_probe(&spi->dev, st_gyro_of_match,
+				 spi->modalias, sizeof(spi->modalias));
 	st_sensors_spi_configure(indio_dev, spi, gdata);
 
 	err = st_gyro_common_probe(indio_dev);
@@ -52,6 +104,7 @@ static const struct spi_device_id st_gyro_id_table[] = {
 	{ LSM330DL_GYRO_DEV_NAME },
 	{ LSM330DLC_GYRO_DEV_NAME },
 	{ L3GD20_GYRO_DEV_NAME },
+	{ L3GD20H_GYRO_DEV_NAME },
 	{ L3G4IS_GYRO_DEV_NAME },
 	{ LSM330_GYRO_DEV_NAME },
 	{ LSM9DS0_GYRO_DEV_NAME },
@@ -62,6 +115,7 @@ MODULE_DEVICE_TABLE(spi, st_gyro_id_table);
 static struct spi_driver st_gyro_driver = {
 	.driver = {
 		.name = "st-gyro-spi",
+		.of_match_table = of_match_ptr(st_gyro_of_match),
 	},
 	.probe = st_gyro_spi_probe,
 	.remove = st_gyro_spi_remove,
diff --git a/drivers/iio/humidity/Kconfig b/drivers/iio/humidity/Kconfig
index 14b9ce453d9d..2c0fc9a400b8 100644
--- a/drivers/iio/humidity/Kconfig
+++ b/drivers/iio/humidity/Kconfig
@@ -31,7 +31,8 @@ config HDC100X
 	select IIO_TRIGGERED_BUFFER
 	help
 	  Say yes here to build support for the Texas Instruments
-	  HDC1000 and HDC1008 relative humidity and temperature sensors.
+	  HDC1000, HDC1008, HDC1010, HDC1050, and HDC1080 relative
+	  humidity and temperature sensors.
 
 	  To compile this driver as a module, choose M here: the module
 	  will be called hdc100x.
diff --git a/drivers/iio/humidity/hdc100x.c b/drivers/iio/humidity/hdc100x.c
index aa17115f54c9..7851bd90ef64 100644
--- a/drivers/iio/humidity/hdc100x.c
+++ b/drivers/iio/humidity/hdc100x.c
@@ -13,6 +13,12 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  * GNU General Public License for more details.
  *
+ * Datasheets:
+ * http://www.ti.com/product/HDC1000/datasheet
+ * http://www.ti.com/product/HDC1008/datasheet
+ * http://www.ti.com/product/HDC1010/datasheet
+ * http://www.ti.com/product/HDC1050/datasheet
+ * http://www.ti.com/product/HDC1080/datasheet
  */
 
 #include <linux/delay.h>
@@ -414,13 +420,29 @@ static int hdc100x_remove(struct i2c_client *client)
 
 static const struct i2c_device_id hdc100x_id[] = {
 	{ "hdc100x", 0 },
+	{ "hdc1000", 0 },
+	{ "hdc1008", 0 },
+	{ "hdc1010", 0 },
+	{ "hdc1050", 0 },
+	{ "hdc1080", 0 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, hdc100x_id);
 
+static const struct of_device_id hdc100x_dt_ids[] = {
+	{ .compatible = "ti,hdc1000" },
+	{ .compatible = "ti,hdc1008" },
+	{ .compatible = "ti,hdc1010" },
+	{ .compatible = "ti,hdc1050" },
+	{ .compatible = "ti,hdc1080" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, hdc100x_dt_ids);
+
 static struct i2c_driver hdc100x_driver = {
 	.driver = {
 		.name	= "hdc100x",
+		.of_match_table = of_match_ptr(hdc100x_dt_ids),
 	},
 	.probe = hdc100x_probe,
 	.remove = hdc100x_remove,
diff --git a/drivers/iio/humidity/hts221.h b/drivers/iio/humidity/hts221.h
index 94510266e0a5..51d021966222 100644
--- a/drivers/iio/humidity/hts221.h
+++ b/drivers/iio/humidity/hts221.h
@@ -30,12 +30,6 @@ struct hts221_transfer_function {
 	int (*write)(struct device *dev, u8 addr, int len, u8 *data);
 };
 
-#define HTS221_AVG_DEPTH	8
-struct hts221_avg_avl {
-	u16 avg;
-	u8 val;
-};
-
 enum hts221_sensor_type {
 	HTS221_SENSOR_H,
 	HTS221_SENSOR_T,
@@ -66,10 +60,9 @@ struct hts221_hw {
 
 extern const struct dev_pm_ops hts221_pm_ops;
 
-int hts221_config_drdy(struct hts221_hw *hw, bool enable);
+int hts221_write_with_mask(struct hts221_hw *hw, u8 addr, u8 mask, u8 val);
 int hts221_probe(struct iio_dev *iio_dev);
-int hts221_power_on(struct hts221_hw *hw);
-int hts221_power_off(struct hts221_hw *hw);
+int hts221_set_enable(struct hts221_hw *hw, bool enable);
 int hts221_allocate_buffers(struct hts221_hw *hw);
 int hts221_allocate_trigger(struct hts221_hw *hw);
 
diff --git a/drivers/iio/humidity/hts221_buffer.c b/drivers/iio/humidity/hts221_buffer.c
index 7d19a3da7ab7..9690dfe9a844 100644
--- a/drivers/iio/humidity/hts221_buffer.c
+++ b/drivers/iio/humidity/hts221_buffer.c
@@ -20,8 +20,16 @@
 #include <linux/iio/triggered_buffer.h>
 #include <linux/iio/buffer.h>
 
+#include <linux/platform_data/st_sensors_pdata.h>
+
 #include "hts221.h"
 
+#define HTS221_REG_DRDY_HL_ADDR		0x22
+#define HTS221_REG_DRDY_HL_MASK		BIT(7)
+#define HTS221_REG_DRDY_PP_OD_ADDR	0x22
+#define HTS221_REG_DRDY_PP_OD_MASK	BIT(6)
+#define HTS221_REG_DRDY_EN_ADDR		0x22
+#define HTS221_REG_DRDY_EN_MASK		BIT(2)
 #define HTS221_REG_STATUS_ADDR		0x27
 #define HTS221_RH_DRDY_MASK		BIT(1)
 #define HTS221_TEMP_DRDY_MASK		BIT(0)
@@ -30,8 +38,12 @@ static int hts221_trig_set_state(struct iio_trigger *trig, bool state)
 {
 	struct iio_dev *iio_dev = iio_trigger_get_drvdata(trig);
 	struct hts221_hw *hw = iio_priv(iio_dev);
+	int err;
+
+	err = hts221_write_with_mask(hw, HTS221_REG_DRDY_EN_ADDR,
+				     HTS221_REG_DRDY_EN_MASK, state);
 
-	return hts221_config_drdy(hw, state);
+	return err < 0 ? err : 0;
 }
 
 static const struct iio_trigger_ops hts221_trigger_ops = {
@@ -67,6 +79,9 @@ static irqreturn_t hts221_trigger_handler_thread(int irq, void *private)
 int hts221_allocate_trigger(struct hts221_hw *hw)
 {
 	struct iio_dev *iio_dev = iio_priv_to_dev(hw);
+	bool irq_active_low = false, open_drain = false;
+	struct device_node *np = hw->dev->of_node;
+	struct st_sensors_platform_data *pdata;
 	unsigned long irq_type;
 	int err;
 
@@ -76,6 +91,10 @@ int hts221_allocate_trigger(struct hts221_hw *hw)
 	case IRQF_TRIGGER_HIGH:
 	case IRQF_TRIGGER_RISING:
 		break;
+	case IRQF_TRIGGER_LOW:
+	case IRQF_TRIGGER_FALLING:
+		irq_active_low = true;
+		break;
 	default:
 		dev_info(hw->dev,
 			 "mode %lx unsupported, using IRQF_TRIGGER_RISING\n",
@@ -84,6 +103,24 @@ int hts221_allocate_trigger(struct hts221_hw *hw)
 		break;
 	}
 
+	err = hts221_write_with_mask(hw, HTS221_REG_DRDY_HL_ADDR,
+				     HTS221_REG_DRDY_HL_MASK, irq_active_low);
+	if (err < 0)
+		return err;
+
+	pdata = (struct st_sensors_platform_data *)hw->dev->platform_data;
+	if ((np && of_property_read_bool(np, "drive-open-drain")) ||
+	    (pdata && pdata->open_drain)) {
+		irq_type |= IRQF_SHARED;
+		open_drain = true;
+	}
+
+	err = hts221_write_with_mask(hw, HTS221_REG_DRDY_PP_OD_ADDR,
+				     HTS221_REG_DRDY_PP_OD_MASK,
+				     open_drain);
+	if (err < 0)
+		return err;
+
 	err = devm_request_threaded_irq(hw->dev, hw->irq, NULL,
 					hts221_trigger_handler_thread,
 					irq_type | IRQF_ONESHOT,
@@ -109,12 +146,12 @@ int hts221_allocate_trigger(struct hts221_hw *hw)
 
 static int hts221_buffer_preenable(struct iio_dev *iio_dev)
 {
-	return hts221_power_on(iio_priv(iio_dev));
+	return hts221_set_enable(iio_priv(iio_dev), true);
 }
 
 static int hts221_buffer_postdisable(struct iio_dev *iio_dev)
 {
-	return hts221_power_off(iio_priv(iio_dev));
+	return hts221_set_enable(iio_priv(iio_dev), false);
 }
 
 static const struct iio_buffer_setup_ops hts221_buffer_ops = {
diff --git a/drivers/iio/humidity/hts221_core.c b/drivers/iio/humidity/hts221_core.c
index a56da3999e00..32524a8dc66f 100644
--- a/drivers/iio/humidity/hts221_core.c
+++ b/drivers/iio/humidity/hts221_core.c
@@ -23,7 +23,6 @@
 
 #define HTS221_REG_CNTRL1_ADDR		0x20
 #define HTS221_REG_CNTRL2_ADDR		0x21
-#define HTS221_REG_CNTRL3_ADDR		0x22
 
 #define HTS221_REG_AVG_ADDR		0x10
 #define HTS221_REG_H_OUT_L		0x28
@@ -32,30 +31,9 @@
 #define HTS221_HUMIDITY_AVG_MASK	0x07
 #define HTS221_TEMP_AVG_MASK		0x38
 
-#define HTS221_ODR_MASK			0x87
+#define HTS221_ODR_MASK			0x03
 #define HTS221_BDU_MASK			BIT(2)
-
-#define HTS221_DRDY_MASK		BIT(2)
-
-#define HTS221_ENABLE_SENSOR		BIT(7)
-
-#define HTS221_HUMIDITY_AVG_4		0x00 /* 0.4 %RH */
-#define HTS221_HUMIDITY_AVG_8		0x01 /* 0.3 %RH */
-#define HTS221_HUMIDITY_AVG_16		0x02 /* 0.2 %RH */
-#define HTS221_HUMIDITY_AVG_32		0x03 /* 0.15 %RH */
-#define HTS221_HUMIDITY_AVG_64		0x04 /* 0.1 %RH */
-#define HTS221_HUMIDITY_AVG_128		0x05 /* 0.07 %RH */
-#define HTS221_HUMIDITY_AVG_256		0x06 /* 0.05 %RH */
-#define HTS221_HUMIDITY_AVG_512		0x07 /* 0.03 %RH */
-
-#define HTS221_TEMP_AVG_2		0x00 /* 0.08 degC */
-#define HTS221_TEMP_AVG_4		0x08 /* 0.05 degC */
-#define HTS221_TEMP_AVG_8		0x10 /* 0.04 degC */
-#define HTS221_TEMP_AVG_16		0x18 /* 0.03 degC */
-#define HTS221_TEMP_AVG_32		0x20 /* 0.02 degC */
-#define HTS221_TEMP_AVG_64		0x28 /* 0.015 degC */
-#define HTS221_TEMP_AVG_128		0x30 /* 0.01 degC */
-#define HTS221_TEMP_AVG_256		0x38 /* 0.007 degC */
+#define HTS221_ENABLE_MASK		BIT(7)
 
 /* calibration registers */
 #define HTS221_REG_0RH_CAL_X_H		0x36
@@ -73,10 +51,11 @@ struct hts221_odr {
 	u8 val;
 };
 
+#define HTS221_AVG_DEPTH		8
 struct hts221_avg {
 	u8 addr;
 	u8 mask;
-	struct hts221_avg_avl avg_avl[HTS221_AVG_DEPTH];
+	u16 avg_avl[HTS221_AVG_DEPTH];
 };
 
 static const struct hts221_odr hts221_odr_table[] = {
@@ -90,28 +69,28 @@ static const struct hts221_avg hts221_avg_list[] = {
 		.addr = HTS221_REG_AVG_ADDR,
 		.mask = HTS221_HUMIDITY_AVG_MASK,
 		.avg_avl = {
-			{ 4, HTS221_HUMIDITY_AVG_4 },
-			{ 8, HTS221_HUMIDITY_AVG_8 },
-			{ 16, HTS221_HUMIDITY_AVG_16 },
-			{ 32, HTS221_HUMIDITY_AVG_32 },
-			{ 64, HTS221_HUMIDITY_AVG_64 },
-			{ 128, HTS221_HUMIDITY_AVG_128 },
-			{ 256, HTS221_HUMIDITY_AVG_256 },
-			{ 512, HTS221_HUMIDITY_AVG_512 },
+			4, /* 0.4 %RH */
+			8, /* 0.3 %RH */
+			16, /* 0.2 %RH */
+			32, /* 0.15 %RH */
+			64, /* 0.1 %RH */
+			128, /* 0.07 %RH */
+			256, /* 0.05 %RH */
+			512, /* 0.03 %RH */
 		},
 	},
 	{
 		.addr = HTS221_REG_AVG_ADDR,
 		.mask = HTS221_TEMP_AVG_MASK,
 		.avg_avl = {
-			{ 2, HTS221_TEMP_AVG_2 },
-			{ 4, HTS221_TEMP_AVG_4 },
-			{ 8, HTS221_TEMP_AVG_8 },
-			{ 16, HTS221_TEMP_AVG_16 },
-			{ 32, HTS221_TEMP_AVG_32 },
-			{ 64, HTS221_TEMP_AVG_64 },
-			{ 128, HTS221_TEMP_AVG_128 },
-			{ 256, HTS221_TEMP_AVG_256 },
+			2, /* 0.08 degC */
+			4, /* 0.05 degC */
+			8, /* 0.04 degC */
+			16, /* 0.03 degC */
+			32, /* 0.02 degC */
+			64, /* 0.015 degC */
+			128, /* 0.01 degC */
+			256, /* 0.007 degC */
 		},
 	},
 };
@@ -152,8 +131,7 @@ static const struct iio_chan_spec hts221_channels[] = {
 	IIO_CHAN_SOFT_TIMESTAMP(2),
 };
 
-static int hts221_write_with_mask(struct hts221_hw *hw, u8 addr, u8 mask,
-				  u8 val)
+int hts221_write_with_mask(struct hts221_hw *hw, u8 addr, u8 mask, u8 val)
 {
 	u8 data;
 	int err;
@@ -166,7 +144,7 @@ static int hts221_write_with_mask(struct hts221_hw *hw, u8 addr, u8 mask,
 		goto unlock;
 	}
 
-	data = (data & ~mask) | (val & mask);
+	data = (data & ~mask) | ((val << __ffs(mask)) & mask);
 
 	err = hw->tf->write(hw->dev, addr, sizeof(data), &data);
 	if (err < 0)
@@ -199,21 +177,9 @@ static int hts221_check_whoami(struct hts221_hw *hw)
 	return 0;
 }
 
-int hts221_config_drdy(struct hts221_hw *hw, bool enable)
-{
-	u8 val = enable ? BIT(2) : 0;
-	int err;
-
-	err = hts221_write_with_mask(hw, HTS221_REG_CNTRL3_ADDR,
-				     HTS221_DRDY_MASK, val);
-
-	return err < 0 ? err : 0;
-}
-
 static int hts221_update_odr(struct hts221_hw *hw, u8 odr)
 {
 	int i, err;
-	u8 val;
 
 	for (i = 0; i < ARRAY_SIZE(hts221_odr_table); i++)
 		if (hts221_odr_table[i].hz == odr)
@@ -222,9 +188,8 @@ static int hts221_update_odr(struct hts221_hw *hw, u8 odr)
 	if (i == ARRAY_SIZE(hts221_odr_table))
 		return -EINVAL;
 
-	val = HTS221_ENABLE_SENSOR | HTS221_BDU_MASK | hts221_odr_table[i].val;
 	err = hts221_write_with_mask(hw, HTS221_REG_CNTRL1_ADDR,
-				     HTS221_ODR_MASK, val);
+				     HTS221_ODR_MASK, hts221_odr_table[i].val);
 	if (err < 0)
 		return err;
 
@@ -241,14 +206,13 @@ static int hts221_update_avg(struct hts221_hw *hw,
 	const struct hts221_avg *avg = &hts221_avg_list[type];
 
 	for (i = 0; i < HTS221_AVG_DEPTH; i++)
-		if (avg->avg_avl[i].avg == val)
+		if (avg->avg_avl[i] == val)
 			break;
 
 	if (i == HTS221_AVG_DEPTH)
 		return -EINVAL;
 
-	err = hts221_write_with_mask(hw, avg->addr, avg->mask,
-				     avg->avg_avl[i].val);
+	err = hts221_write_with_mask(hw, avg->addr, avg->mask, i);
 	if (err < 0)
 		return err;
 
@@ -283,7 +247,7 @@ hts221_sysfs_rh_oversampling_avail(struct device *dev,
 
 	for (i = 0; i < ARRAY_SIZE(avg->avg_avl); i++)
 		len += scnprintf(buf + len, PAGE_SIZE - len, "%d ",
-				 avg->avg_avl[i].avg);
+				 avg->avg_avl[i]);
 	buf[len - 1] = '\n';
 
 	return len;
@@ -300,36 +264,22 @@ hts221_sysfs_temp_oversampling_avail(struct device *dev,
 
 	for (i = 0; i < ARRAY_SIZE(avg->avg_avl); i++)
 		len += scnprintf(buf + len, PAGE_SIZE - len, "%d ",
-				 avg->avg_avl[i].avg);
+				 avg->avg_avl[i]);
 	buf[len - 1] = '\n';
 
 	return len;
 }
 
-int hts221_power_on(struct hts221_hw *hw)
-{
-	int err;
-
-	err = hts221_update_odr(hw, hw->odr);
-	if (err < 0)
-		return err;
-
-	hw->enabled = true;
-
-	return 0;
-}
-
-int hts221_power_off(struct hts221_hw *hw)
+int hts221_set_enable(struct hts221_hw *hw, bool enable)
 {
-	__le16 data = 0;
 	int err;
 
-	err = hw->tf->write(hw->dev, HTS221_REG_CNTRL1_ADDR, sizeof(data),
-			    (u8 *)&data);
+	err = hts221_write_with_mask(hw, HTS221_REG_CNTRL1_ADDR,
+				     HTS221_ENABLE_MASK, enable);
 	if (err < 0)
 		return err;
 
-	hw->enabled = false;
+	hw->enabled = enable;
 
 	return 0;
 }
@@ -484,7 +434,7 @@ static int hts221_read_oneshot(struct hts221_hw *hw, u8 addr, int *val)
 	u8 data[HTS221_DATA_SIZE];
 	int err;
 
-	err = hts221_power_on(hw);
+	err = hts221_set_enable(hw, true);
 	if (err < 0)
 		return err;
 
@@ -494,7 +444,7 @@ static int hts221_read_oneshot(struct hts221_hw *hw, u8 addr, int *val)
 	if (err < 0)
 		return err;
 
-	hts221_power_off(hw);
+	hts221_set_enable(hw, false);
 
 	*val = (s16)get_unaligned_le16(data);
 
@@ -534,13 +484,13 @@ static int hts221_read_raw(struct iio_dev *iio_dev,
 		case IIO_HUMIDITYRELATIVE:
 			avg = &hts221_avg_list[HTS221_SENSOR_H];
 			idx = hw->sensors[HTS221_SENSOR_H].cur_avg_idx;
-			*val = avg->avg_avl[idx].avg;
+			*val = avg->avg_avl[idx];
 			ret = IIO_VAL_INT;
 			break;
 		case IIO_TEMP:
 			avg = &hts221_avg_list[HTS221_SENSOR_T];
 			idx = hw->sensors[HTS221_SENSOR_T].cur_avg_idx;
-			*val = avg->avg_avl[idx].avg;
+			*val = avg->avg_avl[idx];
 			ret = IIO_VAL_INT;
 			break;
 		default:
@@ -644,8 +594,6 @@ int hts221_probe(struct iio_dev *iio_dev)
 	if (err < 0)
 		return err;
 
-	hw->odr = hts221_odr_table[0].hz;
-
 	iio_dev->modes = INDIO_DIRECT_MODE;
 	iio_dev->dev.parent = hw->dev;
 	iio_dev->available_scan_masks = hts221_scan_masks;
@@ -654,6 +602,16 @@ int hts221_probe(struct iio_dev *iio_dev)
 	iio_dev->name = HTS221_DEV_NAME;
 	iio_dev->info = &hts221_info;
 
+	/* enable Block Data Update */
+	err = hts221_write_with_mask(hw, HTS221_REG_CNTRL1_ADDR,
+				     HTS221_BDU_MASK, 1);
+	if (err < 0)
+		return err;
+
+	err = hts221_update_odr(hw, hts221_odr_table[0].hz);
+	if (err < 0)
+		return err;
+
 	/* configure humidity sensor */
 	err = hts221_parse_rh_caldata(hw);
 	if (err < 0) {
@@ -661,7 +619,7 @@ int hts221_probe(struct iio_dev *iio_dev)
 		return err;
 	}
 
-	data = hts221_avg_list[HTS221_SENSOR_H].avg_avl[3].avg;
+	data = hts221_avg_list[HTS221_SENSOR_H].avg_avl[3];
 	err = hts221_update_avg(hw, HTS221_SENSOR_H, data);
 	if (err < 0) {
 		dev_err(hw->dev, "failed to set rh oversampling ratio\n");
@@ -676,7 +634,7 @@ int hts221_probe(struct iio_dev *iio_dev)
 		return err;
 	}
 
-	data = hts221_avg_list[HTS221_SENSOR_T].avg_avl[3].avg;
+	data = hts221_avg_list[HTS221_SENSOR_T].avg_avl[3];
 	err = hts221_update_avg(hw, HTS221_SENSOR_T, data);
 	if (err < 0) {
 		dev_err(hw->dev,
@@ -702,11 +660,10 @@ static int __maybe_unused hts221_suspend(struct device *dev)
 {
 	struct iio_dev *iio_dev = dev_get_drvdata(dev);
 	struct hts221_hw *hw = iio_priv(iio_dev);
-	__le16 data = 0;
 	int err;
 
-	err = hw->tf->write(hw->dev, HTS221_REG_CNTRL1_ADDR, sizeof(data),
-			    (u8 *)&data);
+	err = hts221_write_with_mask(hw, HTS221_REG_CNTRL1_ADDR,
+				     HTS221_ENABLE_MASK, false);
 
 	return err < 0 ? err : 0;
 }
@@ -718,7 +675,8 @@ static int __maybe_unused hts221_resume(struct device *dev)
 	int err = 0;
 
 	if (hw->enabled)
-		err = hts221_update_odr(hw, hw->odr);
+		err = hts221_write_with_mask(hw, HTS221_REG_CNTRL1_ADDR,
+					     HTS221_ENABLE_MASK, true);
 
 	return err;
 }
diff --git a/drivers/iio/humidity/htu21.c b/drivers/iio/humidity/htu21.c
index 0fbbd8c40894..2c4b9be85a05 100644
--- a/drivers/iio/humidity/htu21.c
+++ b/drivers/iio/humidity/htu21.c
@@ -238,11 +238,19 @@ static const struct i2c_device_id htu21_id[] = {
 };
 MODULE_DEVICE_TABLE(i2c, htu21_id);
 
+static const struct of_device_id htu21_of_match[] = {
+	{ .compatible = "meas,htu21", },
+	{ .compatible = "meas,ms8607-humidity", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, htu21_of_match);
+
 static struct i2c_driver htu21_driver = {
 	.probe = htu21_probe,
 	.id_table = htu21_id,
 	.driver = {
 		   .name = "htu21",
+		   .of_match_table = of_match_ptr(htu21_of_match),
 		   },
 };
 
diff --git a/drivers/iio/imu/adis16400_core.c b/drivers/iio/imu/adis16400_core.c
index fb7c0dbed51c..9b697d35dbef 100644
--- a/drivers/iio/imu/adis16400_core.c
+++ b/drivers/iio/imu/adis16400_core.c
@@ -217,7 +217,7 @@ static int adis16400_set_freq(struct adis16400_state *st, unsigned int freq)
 	return adis_write_reg_8(&st->adis, ADIS16400_SMPL_PRD, val);
 }
 
-static const unsigned adis16400_3db_divisors[] = {
+static const unsigned int adis16400_3db_divisors[] = {
 	[0] = 2, /* Special case */
 	[1] = 6,
 	[2] = 12,
@@ -890,7 +890,7 @@ static const struct adis_data adis16400_data = {
 static void adis16400_setup_chan_mask(struct adis16400_state *st)
 {
 	const struct adis16400_chip_info *chip_info = st->variant;
-	unsigned i;
+	unsigned int i;
 
 	for (i = 0; i < chip_info->num_channels; i++) {
 		const struct iio_chan_spec *ch = &chip_info->channels[i];
diff --git a/drivers/iio/imu/adis16480.c b/drivers/iio/imu/adis16480.c
index 8cf84d3488b2..12898424d838 100644
--- a/drivers/iio/imu/adis16480.c
+++ b/drivers/iio/imu/adis16480.c
@@ -696,7 +696,7 @@ static const struct adis16480_chip_info adis16480_chip_info[] = {
 		.gyro_max_val = IIO_RAD_TO_DEGREE(22500),
 		.gyro_max_scale = 450,
 		.accel_max_val = IIO_M_S_2_TO_G(12500),
-		.accel_max_scale = 5,
+		.accel_max_scale = 10,
 	},
 	[ADIS16485] = {
 		.channels = adis16485_channels,
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
index 2a72acc6e049..e2737dc71b67 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
@@ -31,6 +31,8 @@
 #include <linux/iio/iio.h>
 #include <linux/iio/buffer.h>
 
+#include <linux/platform_data/st_sensors_pdata.h>
+
 #include "st_lsm6dsx.h"
 
 #define ST_LSM6DSX_REG_FIFO_THL_ADDR		0x06
@@ -39,6 +41,8 @@
 #define ST_LSM6DSX_REG_FIFO_DEC_GXL_ADDR	0x08
 #define ST_LSM6DSX_REG_HLACTIVE_ADDR		0x12
 #define ST_LSM6DSX_REG_HLACTIVE_MASK		BIT(5)
+#define ST_LSM6DSX_REG_PP_OD_ADDR		0x12
+#define ST_LSM6DSX_REG_PP_OD_MASK		BIT(4)
 #define ST_LSM6DSX_REG_FIFO_MODE_ADDR		0x0a
 #define ST_LSM6DSX_FIFO_MODE_MASK		GENMASK(2, 0)
 #define ST_LSM6DSX_FIFO_ODR_MASK		GENMASK(6, 3)
@@ -417,6 +421,8 @@ static const struct iio_buffer_setup_ops st_lsm6dsx_buffer_ops = {
 
 int st_lsm6dsx_fifo_setup(struct st_lsm6dsx_hw *hw)
 {
+	struct device_node *np = hw->dev->of_node;
+	struct st_sensors_platform_data *pdata;
 	struct iio_buffer *buffer;
 	unsigned long irq_type;
 	bool irq_active_low;
@@ -444,6 +450,17 @@ int st_lsm6dsx_fifo_setup(struct st_lsm6dsx_hw *hw)
 	if (err < 0)
 		return err;
 
+	pdata = (struct st_sensors_platform_data *)hw->dev->platform_data;
+	if ((np && of_property_read_bool(np, "drive-open-drain")) ||
+	    (pdata && pdata->open_drain)) {
+		err = st_lsm6dsx_write_with_mask(hw, ST_LSM6DSX_REG_PP_OD_ADDR,
+						 ST_LSM6DSX_REG_PP_OD_MASK, 1);
+		if (err < 0)
+			return err;
+
+		irq_type |= IRQF_SHARED;
+	}
+
 	err = devm_request_threaded_irq(hw->dev, hw->irq,
 					st_lsm6dsx_handler_irq,
 					st_lsm6dsx_handler_thread,
diff --git a/drivers/iio/inkern.c b/drivers/iio/inkern.c
index da3d06b073bb..069defcc6d9b 100644
--- a/drivers/iio/inkern.c
+++ b/drivers/iio/inkern.c
@@ -44,7 +44,7 @@ int iio_map_array_register(struct iio_dev *indio_dev, struct iio_map *maps)
 		}
 		mapi->map = &maps[i];
 		mapi->indio_dev = indio_dev;
-		list_add(&mapi->l, &iio_map_list);
+		list_add_tail(&mapi->l, &iio_map_list);
 		i++;
 	}
 error_ret:
@@ -205,8 +205,8 @@ static struct iio_channel *of_iio_channel_get_by_name(struct device_node *np,
 		if (!IS_ERR(chan) || PTR_ERR(chan) == -EPROBE_DEFER)
 			break;
 		else if (name && index >= 0) {
-			pr_err("ERROR: could not get IIO channel %s:%s(%i)\n",
-				np->full_name, name ? name : "", index);
+			pr_err("ERROR: could not get IIO channel %pOF:%s(%i)\n",
+				np, name ? name : "", index);
 			return NULL;
 		}
 
diff --git a/drivers/iio/light/apds9300.c b/drivers/iio/light/apds9300.c
index 649b26f67813..05eacd1ee40f 100644
--- a/drivers/iio/light/apds9300.c
+++ b/drivers/iio/light/apds9300.c
@@ -505,7 +505,7 @@ static SIMPLE_DEV_PM_OPS(apds9300_pm_ops, apds9300_suspend, apds9300_resume);
 #define APDS9300_PM_OPS NULL
 #endif
 
-static struct i2c_device_id apds9300_id[] = {
+static const struct i2c_device_id apds9300_id[] = {
 	{ APDS9300_DRV_NAME, 0 },
 	{ }
 };
diff --git a/drivers/iio/light/rpr0521.c b/drivers/iio/light/rpr0521.c
index 9d0c2e859bb2..a6efa12613a2 100644
--- a/drivers/iio/light/rpr0521.c
+++ b/drivers/iio/light/rpr0521.c
@@ -9,7 +9,7 @@
  *
  * IIO driver for RPR-0521RS (7-bit I2C slave address 0x38).
  *
- * TODO: illuminance channel, buffer
+ * TODO: illuminance channel
  */
 
 #include <linux/module.h>
@@ -20,6 +20,10 @@
 #include <linux/acpi.h>
 
 #include <linux/iio/iio.h>
+#include <linux/iio/buffer.h>
+#include <linux/iio/trigger.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/iio/triggered_buffer.h>
 #include <linux/iio/sysfs.h>
 #include <linux/pm_runtime.h>
 
@@ -30,6 +34,7 @@
 #define RPR0521_REG_PXS_DATA		0x44 /* 16-bit, little endian */
 #define RPR0521_REG_ALS_DATA0		0x46 /* 16-bit, little endian */
 #define RPR0521_REG_ALS_DATA1		0x48 /* 16-bit, little endian */
+#define RPR0521_REG_INTERRUPT		0x4A
 #define RPR0521_REG_PS_OFFSET_LSB	0x53
 #define RPR0521_REG_ID			0x92
 
@@ -42,16 +47,31 @@
 #define RPR0521_ALS_DATA1_GAIN_SHIFT	2
 #define RPR0521_PXS_GAIN_MASK		GENMASK(5, 4)
 #define RPR0521_PXS_GAIN_SHIFT		4
+#define RPR0521_PXS_PERSISTENCE_MASK	GENMASK(3, 0)
+#define RPR0521_INTERRUPT_INT_TRIG_PS_MASK	BIT(0)
+#define RPR0521_INTERRUPT_INT_TRIG_ALS_MASK	BIT(1)
+#define RPR0521_INTERRUPT_INT_REASSERT_MASK	BIT(3)
+#define RPR0521_INTERRUPT_ALS_INT_STATUS_MASK	BIT(6)
+#define RPR0521_INTERRUPT_PS_INT_STATUS_MASK	BIT(7)
 
 #define RPR0521_MODE_ALS_ENABLE		BIT(7)
 #define RPR0521_MODE_ALS_DISABLE	0x00
 #define RPR0521_MODE_PXS_ENABLE		BIT(6)
 #define RPR0521_MODE_PXS_DISABLE	0x00
+#define RPR0521_PXS_PERSISTENCE_DRDY	0x00
+
+#define RPR0521_INTERRUPT_INT_TRIG_PS_ENABLE	BIT(0)
+#define RPR0521_INTERRUPT_INT_TRIG_PS_DISABLE	0x00
+#define RPR0521_INTERRUPT_INT_TRIG_ALS_ENABLE	BIT(1)
+#define RPR0521_INTERRUPT_INT_TRIG_ALS_DISABLE	0x00
+#define RPR0521_INTERRUPT_INT_REASSERT_ENABLE	BIT(3)
+#define RPR0521_INTERRUPT_INT_REASSERT_DISABLE	0x00
 
 #define RPR0521_MANUFACT_ID		0xE0
 #define RPR0521_DEFAULT_MEAS_TIME	0x06 /* ALS - 100ms, PXS - 100ms */
 
 #define RPR0521_DRV_NAME		"RPR0521"
+#define RPR0521_IRQ_NAME		"rpr0521_event"
 #define RPR0521_REGMAP_NAME		"rpr0521_regmap"
 
 #define RPR0521_SLEEP_DELAY_MS	2000
@@ -167,6 +187,9 @@ struct rpr0521_data {
 	bool als_dev_en;
 	bool pxs_dev_en;
 
+	struct iio_trigger *drdy_trigger0;
+	s64 irq_timestamp;
+
 	/* optimize runtime pm ops - enable/disable device only if needed */
 	bool als_ps_need_en;
 	bool pxs_ps_need_en;
@@ -196,6 +219,19 @@ static const struct attribute_group rpr0521_attribute_group = {
 	.attrs = rpr0521_attributes,
 };
 
+/* Order of the channel data in buffer */
+enum rpr0521_scan_index_order {
+	RPR0521_CHAN_INDEX_PXS,
+	RPR0521_CHAN_INDEX_BOTH,
+	RPR0521_CHAN_INDEX_IR,
+};
+
+static const unsigned long rpr0521_available_scan_masks[] = {
+	BIT(RPR0521_CHAN_INDEX_PXS) | BIT(RPR0521_CHAN_INDEX_BOTH) |
+	BIT(RPR0521_CHAN_INDEX_IR),
+	0
+};
+
 static const struct iio_chan_spec rpr0521_channels[] = {
 	{
 		.type = IIO_PROXIMITY,
@@ -204,6 +240,13 @@ static const struct iio_chan_spec rpr0521_channels[] = {
 			BIT(IIO_CHAN_INFO_OFFSET) |
 			BIT(IIO_CHAN_INFO_SCALE),
 		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
+		.scan_index = RPR0521_CHAN_INDEX_PXS,
+		.scan_type = {
+			.sign = 'u',
+			.realbits = 16,
+			.storagebits = 16,
+			.endianness = IIO_LE,
+		},
 	},
 	{
 		.type = IIO_INTENSITY,
@@ -213,6 +256,13 @@ static const struct iio_chan_spec rpr0521_channels[] = {
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
 			BIT(IIO_CHAN_INFO_SCALE),
 		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
+		.scan_index = RPR0521_CHAN_INDEX_BOTH,
+		.scan_type = {
+			.sign = 'u',
+			.realbits = 16,
+			.storagebits = 16,
+			.endianness = IIO_LE,
+		},
 	},
 	{
 		.type = IIO_INTENSITY,
@@ -222,6 +272,13 @@ static const struct iio_chan_spec rpr0521_channels[] = {
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
 			BIT(IIO_CHAN_INFO_SCALE),
 		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
+		.scan_index = RPR0521_CHAN_INDEX_IR,
+		.scan_type = {
+			.sign = 'u',
+			.realbits = 16,
+			.storagebits = 16,
+			.endianness = IIO_LE,
+		},
 	},
 };
 
@@ -330,6 +387,198 @@ static int rpr0521_set_power_state(struct rpr0521_data *data, bool on,
 	return 0;
 }
 
+/* Interrupt register tells if this sensor caused the interrupt or not. */
+static inline bool rpr0521_is_triggered(struct rpr0521_data *data)
+{
+	int ret;
+	int reg;
+
+	ret = regmap_read(data->regmap, RPR0521_REG_INTERRUPT, &reg);
+	if (ret < 0)
+		return false;   /* Reg read failed. */
+	if (reg &
+	    (RPR0521_INTERRUPT_ALS_INT_STATUS_MASK |
+	    RPR0521_INTERRUPT_PS_INT_STATUS_MASK))
+		return true;
+	else
+		return false;   /* Int not from this sensor. */
+}
+
+/* IRQ to trigger handler */
+static irqreturn_t rpr0521_drdy_irq_handler(int irq, void *private)
+{
+	struct iio_dev *indio_dev = private;
+	struct rpr0521_data *data = iio_priv(indio_dev);
+
+	data->irq_timestamp = iio_get_time_ns(indio_dev);
+	/*
+	 * We need to wake the thread to read the interrupt reg. It
+	 * is not possible to do that here because regmap_read takes a
+	 * mutex.
+	 */
+
+	return IRQ_WAKE_THREAD;
+}
+
+static irqreturn_t rpr0521_drdy_irq_thread(int irq, void *private)
+{
+	struct iio_dev *indio_dev = private;
+	struct rpr0521_data *data = iio_priv(indio_dev);
+
+	if (rpr0521_is_triggered(data)) {
+		iio_trigger_poll_chained(data->drdy_trigger0);
+		return IRQ_HANDLED;
+	}
+
+	return IRQ_NONE;
+}
+
+static irqreturn_t rpr0521_trigger_consumer_store_time(int irq, void *p)
+{
+	struct iio_poll_func *pf = p;
+	struct iio_dev *indio_dev = pf->indio_dev;
+
+	/* Other trigger polls store time here. */
+	if (!iio_trigger_using_own(indio_dev))
+		pf->timestamp = iio_get_time_ns(indio_dev);
+
+	return IRQ_WAKE_THREAD;
+}
+
+static irqreturn_t rpr0521_trigger_consumer_handler(int irq, void *p)
+{
+	struct iio_poll_func *pf = p;
+	struct iio_dev *indio_dev = pf->indio_dev;
+	struct rpr0521_data *data = iio_priv(indio_dev);
+	int err;
+
+	u8 buffer[16]; /* 3 16-bit channels + padding + ts */
+
+	/* Use irq timestamp when reasonable. */
+	if (iio_trigger_using_own(indio_dev) && data->irq_timestamp) {
+		pf->timestamp = data->irq_timestamp;
+		data->irq_timestamp = 0;
+	}
+	/* Other chained trigger polls get timestamp only here. */
+	if (!pf->timestamp)
+		pf->timestamp = iio_get_time_ns(indio_dev);
+
+	err = regmap_bulk_read(data->regmap, RPR0521_REG_PXS_DATA,
+		&buffer,
+		(3 * 2) + 1);	/* 3 * 16-bit + (discarded) int clear reg. */
+	if (!err)
+		iio_push_to_buffers_with_timestamp(indio_dev,
+						   buffer, pf->timestamp);
+	else
+		dev_err(&data->client->dev,
+			"Trigger consumer can't read from sensor.\n");
+	pf->timestamp = 0;
+
+	iio_trigger_notify_done(indio_dev->trig);
+
+	return IRQ_HANDLED;
+}
+
+static int rpr0521_write_int_enable(struct rpr0521_data *data)
+{
+	int err;
+
+	/* Interrupt after each measurement */
+	err = regmap_update_bits(data->regmap, RPR0521_REG_PXS_CTRL,
+		RPR0521_PXS_PERSISTENCE_MASK,
+		RPR0521_PXS_PERSISTENCE_DRDY);
+	if (err) {
+		dev_err(&data->client->dev, "PS control reg write fail.\n");
+		return -EBUSY;
+		}
+
+	/* Ignore latch and mode because of drdy */
+	err = regmap_write(data->regmap, RPR0521_REG_INTERRUPT,
+		RPR0521_INTERRUPT_INT_REASSERT_DISABLE |
+		RPR0521_INTERRUPT_INT_TRIG_ALS_DISABLE |
+		RPR0521_INTERRUPT_INT_TRIG_PS_ENABLE
+		);
+	if (err) {
+		dev_err(&data->client->dev, "Interrupt setup write fail.\n");
+		return -EBUSY;
+		}
+
+	return 0;
+}
+
+static int rpr0521_write_int_disable(struct rpr0521_data *data)
+{
+	/* Don't care of clearing mode, assert and latch. */
+	return regmap_write(data->regmap, RPR0521_REG_INTERRUPT,
+				RPR0521_INTERRUPT_INT_TRIG_ALS_DISABLE |
+				RPR0521_INTERRUPT_INT_TRIG_PS_DISABLE
+				);
+}
+
+/*
+ * Trigger producer enable / disable. Note that there will be trigs only when
+ * measurement data is ready to be read.
+ */
+static int rpr0521_pxs_drdy_set_state(struct iio_trigger *trigger,
+	bool enable_drdy)
+{
+	struct iio_dev *indio_dev = iio_trigger_get_drvdata(trigger);
+	struct rpr0521_data *data = iio_priv(indio_dev);
+	int err;
+
+	if (enable_drdy)
+		err = rpr0521_write_int_enable(data);
+	else
+		err = rpr0521_write_int_disable(data);
+	if (err)
+		dev_err(&data->client->dev, "rpr0521_pxs_drdy_set_state failed\n");
+
+	return err;
+}
+
+static const struct iio_trigger_ops rpr0521_trigger_ops = {
+	.set_trigger_state = rpr0521_pxs_drdy_set_state,
+	.owner = THIS_MODULE,
+	};
+
+
+static int rpr0521_buffer_preenable(struct iio_dev *indio_dev)
+{
+	int err;
+	struct rpr0521_data *data = iio_priv(indio_dev);
+
+	mutex_lock(&data->lock);
+	err = rpr0521_set_power_state(data, true,
+		(RPR0521_MODE_PXS_MASK | RPR0521_MODE_ALS_MASK));
+	mutex_unlock(&data->lock);
+	if (err)
+		dev_err(&data->client->dev, "_buffer_preenable fail\n");
+
+	return err;
+}
+
+static int rpr0521_buffer_postdisable(struct iio_dev *indio_dev)
+{
+	int err;
+	struct rpr0521_data *data = iio_priv(indio_dev);
+
+	mutex_lock(&data->lock);
+	err = rpr0521_set_power_state(data, false,
+		(RPR0521_MODE_PXS_MASK | RPR0521_MODE_ALS_MASK));
+	mutex_unlock(&data->lock);
+	if (err)
+		dev_err(&data->client->dev, "_buffer_postdisable fail\n");
+
+	return err;
+}
+
+static const struct iio_buffer_setup_ops rpr0521_buffer_setup_ops = {
+	.preenable = rpr0521_buffer_preenable,
+	.postenable = iio_triggered_buffer_postenable,
+	.predisable = iio_triggered_buffer_predisable,
+	.postdisable = rpr0521_buffer_postdisable,
+};
+
 static int rpr0521_get_gain(struct rpr0521_data *data, int chan,
 			    int *val, int *val2)
 {
@@ -473,6 +722,7 @@ static int rpr0521_read_raw(struct iio_dev *indio_dev,
 {
 	struct rpr0521_data *data = iio_priv(indio_dev);
 	int ret;
+	int busy;
 	u8 device_mask;
 	__le16 raw_data;
 
@@ -481,26 +731,30 @@ static int rpr0521_read_raw(struct iio_dev *indio_dev,
 		if (chan->type != IIO_INTENSITY && chan->type != IIO_PROXIMITY)
 			return -EINVAL;
 
+		busy = iio_device_claim_direct_mode(indio_dev);
+		if (busy)
+			return -EBUSY;
+
 		device_mask = rpr0521_data_reg[chan->address].device_mask;
 
 		mutex_lock(&data->lock);
 		ret = rpr0521_set_power_state(data, true, device_mask);
-		if (ret < 0) {
-			mutex_unlock(&data->lock);
-			return ret;
-		}
+		if (ret < 0)
+			goto rpr0521_read_raw_out;
 
 		ret = regmap_bulk_read(data->regmap,
 				       rpr0521_data_reg[chan->address].address,
 				       &raw_data, sizeof(raw_data));
 		if (ret < 0) {
 			rpr0521_set_power_state(data, false, device_mask);
-			mutex_unlock(&data->lock);
-			return ret;
+			goto rpr0521_read_raw_out;
 		}
 
 		ret = rpr0521_set_power_state(data, false, device_mask);
+
+rpr0521_read_raw_out:
 		mutex_unlock(&data->lock);
+		iio_device_release_direct_mode(indio_dev);
 		if (ret < 0)
 			return ret;
 
@@ -617,12 +871,15 @@ static int rpr0521_init(struct rpr0521_data *data)
 		return ret;
 #endif
 
+	data->irq_timestamp = 0;
+
 	return 0;
 }
 
 static int rpr0521_poweroff(struct rpr0521_data *data)
 {
 	int ret;
+	int tmp;
 
 	ret = regmap_update_bits(data->regmap, RPR0521_REG_MODE_CTRL,
 				 RPR0521_MODE_ALS_MASK |
@@ -635,6 +892,16 @@ static int rpr0521_poweroff(struct rpr0521_data *data)
 	data->als_dev_en = false;
 	data->pxs_dev_en = false;
 
+	/*
+	 * Int pin keeps state after power off. Set pin to high impedance
+	 * mode to prevent power drain.
+	 */
+	ret = regmap_read(data->regmap, RPR0521_REG_INTERRUPT, &tmp);
+	if (ret) {
+		dev_err(&data->client->dev, "Failed to reset int pin.\n");
+		return ret;
+	}
+
 	return 0;
 }
 
@@ -707,6 +974,61 @@ static int rpr0521_probe(struct i2c_client *client,
 	pm_runtime_set_autosuspend_delay(&client->dev, RPR0521_SLEEP_DELAY_MS);
 	pm_runtime_use_autosuspend(&client->dev);
 
+	/*
+	 * If sensor write/read is needed in _probe after _use_autosuspend,
+	 * sensor needs to be _resumed first using rpr0521_set_power_state().
+	 */
+
+	/* IRQ to trigger setup */
+	if (client->irq) {
+		/* Trigger0 producer setup */
+		data->drdy_trigger0 = devm_iio_trigger_alloc(
+			indio_dev->dev.parent,
+			"%s-dev%d", indio_dev->name, indio_dev->id);
+		if (!data->drdy_trigger0) {
+			ret = -ENOMEM;
+			goto err_pm_disable;
+		}
+		data->drdy_trigger0->dev.parent = indio_dev->dev.parent;
+		data->drdy_trigger0->ops = &rpr0521_trigger_ops;
+		indio_dev->available_scan_masks = rpr0521_available_scan_masks;
+		iio_trigger_set_drvdata(data->drdy_trigger0, indio_dev);
+
+		/* Ties irq to trigger producer handler. */
+		ret = devm_request_threaded_irq(&client->dev, client->irq,
+			rpr0521_drdy_irq_handler, rpr0521_drdy_irq_thread,
+			IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
+			RPR0521_IRQ_NAME, indio_dev);
+		if (ret < 0) {
+			dev_err(&client->dev, "request irq %d for trigger0 failed\n",
+				client->irq);
+			goto err_pm_disable;
+			}
+
+		ret = devm_iio_trigger_register(indio_dev->dev.parent,
+						data->drdy_trigger0);
+		if (ret) {
+			dev_err(&client->dev, "iio trigger register failed\n");
+			goto err_pm_disable;
+		}
+
+		/*
+		 * Now whole pipe from physical interrupt (irq defined by
+		 * devicetree to device) to trigger0 output is set up.
+		 */
+
+		/* Trigger consumer setup */
+		ret = devm_iio_triggered_buffer_setup(indio_dev->dev.parent,
+			indio_dev,
+			rpr0521_trigger_consumer_store_time,
+			rpr0521_trigger_consumer_handler,
+			&rpr0521_buffer_setup_ops);
+		if (ret < 0) {
+			dev_err(&client->dev, "iio triggered buffer setup failed\n");
+			goto err_pm_disable;
+		}
+	}
+
 	ret = iio_device_register(indio_dev);
 	if (ret)
 		goto err_pm_disable;
diff --git a/drivers/iio/light/tcs3472.c b/drivers/iio/light/tcs3472.c
index 3aa71e34ae28..09e6ca5e332e 100644
--- a/drivers/iio/light/tcs3472.c
+++ b/drivers/iio/light/tcs3472.c
@@ -11,6 +11,8 @@
  * 7-bit I2C slave address 0x39 (TCS34721, TCS34723) or 0x29 (TCS34725,
  * TCS34727)
  *
+ * Datasheet: http://ams.com/eng/content/download/319364/1117183/file/TCS3472_Datasheet_EN_v2.pdf
+ *
  * TODO: interrupt support, thresholds, wait time
  */
 
@@ -169,7 +171,7 @@ static int tcs3472_write_raw(struct iio_dev *indio_dev,
 		for (i = 0; i < 256; i++) {
 			if (val2 == (256 - i) * 2400) {
 				data->atime = i;
-				return i2c_smbus_write_word_data(
+				return i2c_smbus_write_byte_data(
 					data->client, TCS3472_ATIME,
 					data->atime);
 			}
diff --git a/drivers/iio/light/tsl2583.c b/drivers/iio/light/tsl2583.c
index 1679181d2bdd..fb711ed4862e 100644
--- a/drivers/iio/light/tsl2583.c
+++ b/drivers/iio/light/tsl2583.c
@@ -924,7 +924,7 @@ static const struct dev_pm_ops tsl2583_pm_ops = {
 	SET_RUNTIME_PM_OPS(tsl2583_suspend, tsl2583_resume, NULL)
 };
 
-static struct i2c_device_id tsl2583_idtable[] = {
+static const struct i2c_device_id tsl2583_idtable[] = {
 	{ "tsl2580", 0 },
 	{ "tsl2581", 1 },
 	{ "tsl2583", 2 },
diff --git a/drivers/iio/magnetometer/Kconfig b/drivers/iio/magnetometer/Kconfig
index 421ad90a5fbe..ed9d776d01af 100644
--- a/drivers/iio/magnetometer/Kconfig
+++ b/drivers/iio/magnetometer/Kconfig
@@ -13,8 +13,8 @@ config AK8974
 	select IIO_BUFFER
 	select IIO_TRIGGERED_BUFFER
 	help
-	  Say yes here to build support for Asahi Kasei AK8974 or
-	  AMI305 I2C-based 3-axis magnetometer chips.
+	  Say yes here to build support for Asahi Kasei AK8974, AMI305 or
+	  AMI306 I2C-based 3-axis magnetometer chips.
 
 	  To compile this driver as a module, choose M here: the module
 	  will be called ak8974.
diff --git a/drivers/iio/magnetometer/ak8974.c b/drivers/iio/magnetometer/ak8974.c
index e13370dc9b1c..0bff76e96950 100644
--- a/drivers/iio/magnetometer/ak8974.c
+++ b/drivers/iio/magnetometer/ak8974.c
@@ -20,6 +20,7 @@
 #include <linux/mutex.h>
 #include <linux/delay.h>
 #include <linux/bitops.h>
+#include <linux/random.h>
 #include <linux/regmap.h>
 #include <linux/regulator/consumer.h>
 #include <linux/pm_runtime.h>
@@ -36,7 +37,7 @@
  * and MSB is at the next higher address.
  */
 
-/* These registers are common for AK8974 and AMI305 */
+/* These registers are common for AK8974 and AMI30x */
 #define AK8974_SELFTEST		0x0C
 #define AK8974_SELFTEST_IDLE	0x55
 #define AK8974_SELFTEST_OK	0xAA
@@ -44,6 +45,7 @@
 #define AK8974_INFO		0x0D
 
 #define AK8974_WHOAMI		0x0F
+#define AK8974_WHOAMI_VALUE_AMI306 0x46
 #define AK8974_WHOAMI_VALUE_AMI305 0x47
 #define AK8974_WHOAMI_VALUE_AK8974 0x48
 
@@ -73,6 +75,35 @@
 #define AK8974_TEMP		0x31
 #define AMI305_TEMP		0x60
 
+/* AMI306-specific control register */
+#define AMI306_CTRL4		0x5C
+
+/* AMI306 factory calibration data */
+
+/* fine axis sensitivity */
+#define AMI306_FINEOUTPUT_X	0x90
+#define AMI306_FINEOUTPUT_Y	0x92
+#define AMI306_FINEOUTPUT_Z	0x94
+
+/* axis sensitivity */
+#define AMI306_SENS_X		0x96
+#define AMI306_SENS_Y		0x98
+#define AMI306_SENS_Z		0x9A
+
+/* axis cross-interference */
+#define AMI306_GAIN_PARA_XZ	0x9C
+#define AMI306_GAIN_PARA_XY	0x9D
+#define AMI306_GAIN_PARA_YZ	0x9E
+#define AMI306_GAIN_PARA_YX	0x9F
+#define AMI306_GAIN_PARA_ZY	0xA0
+#define AMI306_GAIN_PARA_ZX	0xA1
+
+/* offset at ZERO magnetic field */
+#define AMI306_OFFZERO_X	0xF8
+#define AMI306_OFFZERO_Y	0xFA
+#define AMI306_OFFZERO_Z	0xFC
+
+
 #define AK8974_INT_X_HIGH	BIT(7) /* Axis over +threshold  */
 #define AK8974_INT_Y_HIGH	BIT(6)
 #define AK8974_INT_Z_HIGH	BIT(5)
@@ -158,6 +189,26 @@ struct ak8974 {
 static const char ak8974_reg_avdd[] = "avdd";
 static const char ak8974_reg_dvdd[] = "dvdd";
 
+static int ak8974_get_u16_val(struct ak8974 *ak8974, u8 reg, u16 *val)
+{
+	int ret;
+	__le16 bulk;
+
+	ret = regmap_bulk_read(ak8974->map, reg, &bulk, 2);
+	if (ret)
+		return ret;
+	*val = le16_to_cpu(bulk);
+
+	return 0;
+}
+
+static int ak8974_set_u16_val(struct ak8974 *ak8974, u8 reg, u16 val)
+{
+	__le16 bulk = cpu_to_le16(val);
+
+	return regmap_bulk_write(ak8974->map, reg, &bulk, 2);
+}
+
 static int ak8974_set_power(struct ak8974 *ak8974, bool mode)
 {
 	int ret;
@@ -209,6 +260,12 @@ static int ak8974_configure(struct ak8974 *ak8974)
 	ret = regmap_write(ak8974->map, AK8974_CTRL3, 0);
 	if (ret)
 		return ret;
+	if (ak8974->variant == AK8974_WHOAMI_VALUE_AMI306) {
+		/* magic from datasheet: set high-speed measurement mode */
+		ret = ak8974_set_u16_val(ak8974, AMI306_CTRL4, 0xA07E);
+		if (ret)
+			return ret;
+	}
 	ret = regmap_write(ak8974->map, AK8974_INT_CTRL, AK8974_INT_CTRL_POL);
 	if (ret)
 		return ret;
@@ -388,17 +445,18 @@ static int ak8974_selftest(struct ak8974 *ak8974)
 	return 0;
 }
 
-static int ak8974_get_u16_val(struct ak8974 *ak8974, u8 reg, u16 *val)
+static void ak8974_read_calib_data(struct ak8974 *ak8974, unsigned int reg,
+				   __le16 *tab, size_t tab_size)
 {
-	int ret;
-	__le16 bulk;
-
-	ret = regmap_bulk_read(ak8974->map, reg, &bulk, 2);
-	if (ret)
-		return ret;
-	*val = le16_to_cpu(bulk);
-
-	return 0;
+	int ret = regmap_bulk_read(ak8974->map, reg, tab, tab_size);
+	if (ret) {
+		memset(tab, 0xFF, tab_size);
+		dev_warn(&ak8974->i2c->dev,
+			 "can't read calibration data (regs %u..%zu): %d\n",
+			 reg, reg + tab_size - 1, ret);
+	} else {
+		add_device_randomness(tab, tab_size);
+	}
 }
 
 static int ak8974_detect(struct ak8974 *ak8974)
@@ -413,9 +471,13 @@ static int ak8974_detect(struct ak8974 *ak8974)
 	if (ret)
 		return ret;
 
+	name = "ami305";
+
 	switch (whoami) {
+	case AK8974_WHOAMI_VALUE_AMI306:
+		name = "ami306";
+		/* fall-through */
 	case AK8974_WHOAMI_VALUE_AMI305:
-		name = "ami305";
 		ret = regmap_read(ak8974->map, AMI305_VER, &fw);
 		if (ret)
 			return ret;
@@ -423,6 +485,7 @@ static int ak8974_detect(struct ak8974 *ak8974)
 		ret = ak8974_get_u16_val(ak8974, AMI305_SN, &sn);
 		if (ret)
 			return ret;
+		add_device_randomness(&sn, sizeof(sn));
 		dev_info(&ak8974->i2c->dev,
 			 "detected %s, FW ver %02x, S/N: %04x\n",
 			 name, fw, sn);
@@ -440,6 +503,33 @@ static int ak8974_detect(struct ak8974 *ak8974)
 	ak8974->name = name;
 	ak8974->variant = whoami;
 
+	if (whoami == AK8974_WHOAMI_VALUE_AMI306) {
+		__le16 fab_data1[9], fab_data2[3];
+		int i;
+
+		ak8974_read_calib_data(ak8974, AMI306_FINEOUTPUT_X,
+				       fab_data1, sizeof(fab_data1));
+		ak8974_read_calib_data(ak8974, AMI306_OFFZERO_X,
+				       fab_data2, sizeof(fab_data2));
+
+		for (i = 0; i < 3; ++i) {
+			static const char axis[3] = "XYZ";
+			static const char pgaxis[6] = "ZYZXYX";
+			unsigned offz = le16_to_cpu(fab_data2[i]) & 0x7F;
+			unsigned fine = le16_to_cpu(fab_data1[i]);
+			unsigned sens = le16_to_cpu(fab_data1[i + 3]);
+			unsigned pgain1 = le16_to_cpu(fab_data1[i + 6]);
+			unsigned pgain2 = pgain1 >> 8;
+
+			pgain1 &= 0xFF;
+
+			dev_info(&ak8974->i2c->dev,
+				 "factory calibration for axis %c: offz=%u sens=%u fine=%u pga%c=%u pga%c=%u\n",
+				 axis[i], offz, sens, fine, pgaxis[i * 2],
+				 pgain1, pgaxis[i * 2 + 1], pgain2);
+		}
+	}
+
 	return 0;
 }
 
@@ -602,19 +692,27 @@ static bool ak8974_writeable_reg(struct device *dev, unsigned int reg)
 	case AMI305_OFFSET_Y + 1:
 	case AMI305_OFFSET_Z:
 	case AMI305_OFFSET_Z + 1:
-		if (ak8974->variant == AK8974_WHOAMI_VALUE_AMI305)
-			return true;
-		return false;
+		return ak8974->variant == AK8974_WHOAMI_VALUE_AMI305 ||
+		       ak8974->variant == AK8974_WHOAMI_VALUE_AMI306;
+	case AMI306_CTRL4:
+	case AMI306_CTRL4 + 1:
+		return ak8974->variant == AK8974_WHOAMI_VALUE_AMI306;
 	default:
 		return false;
 	}
 }
 
+static bool ak8974_precious_reg(struct device *dev, unsigned int reg)
+{
+	return reg == AK8974_INT_CLEAR;
+}
+
 static const struct regmap_config ak8974_regmap_config = {
 	.reg_bits = 8,
 	.val_bits = 8,
 	.max_register = 0xff,
 	.writeable_reg = ak8974_writeable_reg,
+	.precious_reg = ak8974_precious_reg,
 };
 
 static int ak8974_probe(struct i2c_client *i2c,
@@ -678,7 +776,7 @@ static int ak8974_probe(struct i2c_client *i2c,
 
 	ret = ak8974_detect(ak8974);
 	if (ret) {
-		dev_err(&i2c->dev, "neither AK8974 nor AMI305 found\n");
+		dev_err(&i2c->dev, "neither AK8974 nor AMI30x found\n");
 		goto power_off;
 	}
 
@@ -827,6 +925,7 @@ static const struct dev_pm_ops ak8974_dev_pm_ops = {
 
 static const struct i2c_device_id ak8974_id[] = {
 	{"ami305", 0 },
+	{"ami306", 0 },
 	{"ak8974", 0 },
 	{}
 };
@@ -850,7 +949,7 @@ static struct i2c_driver ak8974_driver = {
 };
 module_i2c_driver(ak8974_driver);
 
-MODULE_DESCRIPTION("AK8974 and AMI305 3-axis magnetometer driver");
+MODULE_DESCRIPTION("AK8974 and AMI30x 3-axis magnetometer driver");
 MODULE_AUTHOR("Samu Onkalo");
 MODULE_AUTHOR("Linus Walleij");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/magnetometer/ak8975.c b/drivers/iio/magnetometer/ak8975.c
index 825369fb1c57..4ff883942f7b 100644
--- a/drivers/iio/magnetometer/ak8975.c
+++ b/drivers/iio/magnetometer/ak8975.c
@@ -784,6 +784,7 @@ static const struct iio_info ak8975_info = {
 	.driver_module = THIS_MODULE,
 };
 
+#ifdef CONFIG_ACPI
 static const struct acpi_device_id ak_acpi_match[] = {
 	{"AK8975", AK8975},
 	{"AK8963", AK8963},
@@ -793,6 +794,7 @@ static const struct acpi_device_id ak_acpi_match[] = {
 	{ },
 };
 MODULE_DEVICE_TABLE(acpi, ak_acpi_match);
+#endif
 
 static const char *ak8975_match_acpi_device(struct device *dev,
 					    enum asahi_compass_chipset *chipset)
diff --git a/drivers/iio/magnetometer/st_magn.h b/drivers/iio/magnetometer/st_magn.h
index 9daca4681922..8fe51ce427bd 100644
--- a/drivers/iio/magnetometer/st_magn.h
+++ b/drivers/iio/magnetometer/st_magn.h
@@ -19,6 +19,7 @@
 #define LSM303DLM_MAGN_DEV_NAME		"lsm303dlm_magn"
 #define LIS3MDL_MAGN_DEV_NAME		"lis3mdl"
 #define LSM303AGR_MAGN_DEV_NAME		"lsm303agr_magn"
+#define LIS2MDL_MAGN_DEV_NAME		"lis2mdl"
 
 int st_magn_common_probe(struct iio_dev *indio_dev);
 void st_magn_common_remove(struct iio_dev *indio_dev);
diff --git a/drivers/iio/magnetometer/st_magn_core.c b/drivers/iio/magnetometer/st_magn_core.c
index 8e1b0861fbe4..e68368b5b2a3 100644
--- a/drivers/iio/magnetometer/st_magn_core.c
+++ b/drivers/iio/magnetometer/st_magn_core.c
@@ -315,7 +315,7 @@ static const struct st_sensor_settings st_magn_sensors_settings[] = {
 				},
 			},
 		},
-		.multi_read_bit = false,
+		.multi_read_bit = true,
 		.bootime = 2,
 	},
 	{
@@ -323,6 +323,7 @@ static const struct st_sensor_settings st_magn_sensors_settings[] = {
 		.wai_addr = 0x4f,
 		.sensors_supported = {
 			[0] = LSM303AGR_MAGN_DEV_NAME,
+			[1] = LIS2MDL_MAGN_DEV_NAME,
 		},
 		.ch = (struct iio_chan_spec *)st_magn_3_16bit_channels,
 		.odr = {
@@ -356,9 +357,7 @@ static const struct st_sensor_settings st_magn_sensors_settings[] = {
 		.drdy_irq = {
 			.addr = 0x62,
 			.mask_int1 = 0x01,
-			.addr_ihl = 0x63,
-			.mask_ihl = 0x04,
-			.addr_stat_drdy = ST_SENSORS_DEFAULT_STAT_ADDR,
+			.addr_stat_drdy = 0x67,
 		},
 		.multi_read_bit = false,
 		.bootime = 2,
diff --git a/drivers/iio/magnetometer/st_magn_i2c.c b/drivers/iio/magnetometer/st_magn_i2c.c
index 8aa37af306ed..feaa28cf6a77 100644
--- a/drivers/iio/magnetometer/st_magn_i2c.c
+++ b/drivers/iio/magnetometer/st_magn_i2c.c
@@ -40,6 +40,10 @@ static const struct of_device_id st_magn_of_match[] = {
 		.compatible = "st,lsm303agr-magn",
 		.data = LSM303AGR_MAGN_DEV_NAME,
 	},
+	{
+		.compatible = "st,lis2mdl",
+		.data = LIS2MDL_MAGN_DEV_NAME,
+	},
 	{},
 };
 MODULE_DEVICE_TABLE(of, st_magn_of_match);
@@ -59,7 +63,8 @@ static int st_magn_i2c_probe(struct i2c_client *client,
 		return -ENOMEM;
 
 	mdata = iio_priv(indio_dev);
-	st_sensors_of_i2c_probe(client, st_magn_of_match);
+	st_sensors_of_name_probe(&client->dev, st_magn_of_match,
+				 client->name, sizeof(client->name));
 
 	st_sensors_i2c_configure(indio_dev, client, mdata);
 
@@ -84,6 +89,7 @@ static const struct i2c_device_id st_magn_id_table[] = {
 	{ LSM303DLM_MAGN_DEV_NAME },
 	{ LIS3MDL_MAGN_DEV_NAME },
 	{ LSM303AGR_MAGN_DEV_NAME },
+	{ LIS2MDL_MAGN_DEV_NAME },
 	{},
 };
 MODULE_DEVICE_TABLE(i2c, st_magn_id_table);
diff --git a/drivers/iio/magnetometer/st_magn_spi.c b/drivers/iio/magnetometer/st_magn_spi.c
index f3cb4dc05391..7b7cd08fcc32 100644
--- a/drivers/iio/magnetometer/st_magn_spi.c
+++ b/drivers/iio/magnetometer/st_magn_spi.c
@@ -18,6 +18,32 @@
 #include <linux/iio/common/st_sensors_spi.h>
 #include "st_magn.h"
 
+#ifdef CONFIG_OF
+/*
+ * For new single-chip sensors use <device_name> as compatible string.
+ * For old single-chip devices keep <device_name>-magn to maintain
+ * compatibility
+ */
+static const struct of_device_id st_magn_of_match[] = {
+	{
+		.compatible = "st,lis3mdl-magn",
+		.data = LIS3MDL_MAGN_DEV_NAME,
+	},
+	{
+		.compatible = "st,lsm303agr-magn",
+		.data = LSM303AGR_MAGN_DEV_NAME,
+	},
+	{
+		.compatible = "st,lis2mdl",
+		.data = LIS2MDL_MAGN_DEV_NAME,
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(of, st_magn_of_match);
+#else
+#define st_magn_of_match	NULL
+#endif
+
 static int st_magn_spi_probe(struct spi_device *spi)
 {
 	struct iio_dev *indio_dev;
@@ -30,6 +56,8 @@ static int st_magn_spi_probe(struct spi_device *spi)
 
 	mdata = iio_priv(indio_dev);
 
+	st_sensors_of_name_probe(&spi->dev, st_magn_of_match,
+				 spi->modalias, sizeof(spi->modalias));
 	st_sensors_spi_configure(indio_dev, spi, mdata);
 
 	err = st_magn_common_probe(indio_dev);
@@ -50,6 +78,7 @@ static int st_magn_spi_remove(struct spi_device *spi)
 static const struct spi_device_id st_magn_id_table[] = {
 	{ LIS3MDL_MAGN_DEV_NAME },
 	{ LSM303AGR_MAGN_DEV_NAME },
+	{ LIS2MDL_MAGN_DEV_NAME },
 	{},
 };
 MODULE_DEVICE_TABLE(spi, st_magn_id_table);
@@ -57,6 +86,7 @@ MODULE_DEVICE_TABLE(spi, st_magn_id_table);
 static struct spi_driver st_magn_driver = {
 	.driver = {
 		.name = "st-magn-spi",
+		.of_match_table = of_match_ptr(st_magn_of_match),
 	},
 	.probe = st_magn_spi_probe,
 	.remove = st_magn_spi_remove,
diff --git a/drivers/iio/orientation/hid-sensor-rotation.c b/drivers/iio/orientation/hid-sensor-rotation.c
index e9fa86c87db5..98fe0c5df380 100644
--- a/drivers/iio/orientation/hid-sensor-rotation.c
+++ b/drivers/iio/orientation/hid-sensor-rotation.c
@@ -238,7 +238,7 @@ static int dev_rot_parse_report(struct platform_device *pdev,
 static int hid_dev_rot_probe(struct platform_device *pdev)
 {
 	int ret;
-	static char *name;
+	char *name;
 	struct iio_dev *indio_dev;
 	struct dev_rot_state *rot_state;
 	struct hid_sensor_hub_device *hsdev = pdev->dev.platform_data;
diff --git a/drivers/iio/pressure/bmp280-core.c b/drivers/iio/pressure/bmp280-core.c
index d82b788374b6..0d2ea3ee371b 100644
--- a/drivers/iio/pressure/bmp280-core.c
+++ b/drivers/iio/pressure/bmp280-core.c
@@ -282,6 +282,11 @@ static int bmp280_read_temp(struct bmp280_data *data,
 	}
 
 	adc_temp = be32_to_cpu(tmp) >> 12;
+	if (adc_temp == BMP280_TEMP_SKIPPED) {
+		/* reading was skipped */
+		dev_err(data->dev, "reading temperature skipped\n");
+		return -EIO;
+	}
 	comp_temp = bmp280_compensate_temp(data, adc_temp);
 
 	/*
@@ -317,6 +322,11 @@ static int bmp280_read_press(struct bmp280_data *data,
 	}
 
 	adc_press = be32_to_cpu(tmp) >> 12;
+	if (adc_press == BMP280_PRESS_SKIPPED) {
+		/* reading was skipped */
+		dev_err(data->dev, "reading pressure skipped\n");
+		return -EIO;
+	}
 	comp_press = bmp280_compensate_press(data, adc_press);
 
 	*val = comp_press;
@@ -345,6 +355,11 @@ static int bmp280_read_humid(struct bmp280_data *data, int *val, int *val2)
 	}
 
 	adc_humidity = be16_to_cpu(tmp);
+	if (adc_humidity == BMP280_HUMIDITY_SKIPPED) {
+		/* reading was skipped */
+		dev_err(data->dev, "reading humidity skipped\n");
+		return -EIO;
+	}
 	comp_humidity = bmp280_compensate_humidity(data, adc_humidity);
 
 	*val = comp_humidity;
@@ -597,14 +612,20 @@ static const struct bmp280_chip_info bmp280_chip_info = {
 
 static int bme280_chip_config(struct bmp280_data *data)
 {
-	int ret = bmp280_chip_config(data);
+	int ret;
 	u8 osrs = BMP280_OSRS_HUMIDITIY_X(data->oversampling_humid + 1);
 
+	/*
+	 * Oversampling of humidity must be set before oversampling of
+	 * temperature/pressure is set to become effective.
+	 */
+	ret = regmap_update_bits(data->regmap, BMP280_REG_CTRL_HUMIDITY,
+				  BMP280_OSRS_HUMIDITY_MASK, osrs);
+
 	if (ret < 0)
 		return ret;
 
-	return regmap_update_bits(data->regmap, BMP280_REG_CTRL_HUMIDITY,
-				  BMP280_OSRS_HUMIDITY_MASK, osrs);
+	return bmp280_chip_config(data);
 }
 
 static const struct bmp280_chip_info bme280_chip_info = {
diff --git a/drivers/iio/pressure/bmp280.h b/drivers/iio/pressure/bmp280.h
index 2c770e13be0e..61347438b779 100644
--- a/drivers/iio/pressure/bmp280.h
+++ b/drivers/iio/pressure/bmp280.h
@@ -96,6 +96,11 @@
 #define BME280_CHIP_ID			0x60
 #define BMP280_SOFT_RESET_VAL		0xB6
 
+/* BMP280 register skipped special values */
+#define BMP280_TEMP_SKIPPED		0x80000
+#define BMP280_PRESS_SKIPPED		0x80000
+#define BMP280_HUMIDITY_SKIPPED		0x8000
+
 /* Regmap configurations */
 extern const struct regmap_config bmp180_regmap_config;
 extern const struct regmap_config bmp280_regmap_config;
diff --git a/drivers/iio/pressure/ms5637.c b/drivers/iio/pressure/ms5637.c
index 953ffbc0ef96..c413f8a84a63 100644
--- a/drivers/iio/pressure/ms5637.c
+++ b/drivers/iio/pressure/ms5637.c
@@ -181,11 +181,21 @@ static const struct i2c_device_id ms5637_id[] = {
 };
 MODULE_DEVICE_TABLE(i2c, ms5637_id);
 
+static const struct of_device_id ms5637_of_match[] = {
+	{ .compatible = "meas,ms5637", },
+	{ .compatible = "meas,ms5805", },
+	{ .compatible = "meas,ms5837", },
+	{ .compatible = "meas,ms8607-temppressure", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, ms5637_of_match);
+
 static struct i2c_driver ms5637_driver = {
 	.probe = ms5637_probe,
 	.id_table = ms5637_id,
 	.driver = {
-		   .name = "ms5637"
+		   .name = "ms5637",
+		   .of_match_table = of_match_ptr(ms5637_of_match),
 		   },
 };
 
diff --git a/drivers/iio/pressure/st_pressure_core.c b/drivers/iio/pressure/st_pressure_core.c
index f1bce05ffa13..34611a8ea2ce 100644
--- a/drivers/iio/pressure/st_pressure_core.c
+++ b/drivers/iio/pressure/st_pressure_core.c
@@ -390,7 +390,7 @@ static const struct st_sensor_settings st_press_sensors_settings[] = {
 		.drdy_irq = {
 			.addr = 0x23,
 			.mask_int1 = 0x01,
-			.mask_int2 = 0x10,
+			.mask_int2 = 0x00,
 			.addr_ihl = 0x22,
 			.mask_ihl = 0x80,
 			.addr_od = 0x22,
@@ -449,7 +449,7 @@ static const struct st_sensor_settings st_press_sensors_settings[] = {
 		.drdy_irq = {
 			.addr = 0x12,
 			.mask_int1 = 0x04,
-			.mask_int2 = 0x08,
+			.mask_int2 = 0x00,
 			.addr_ihl = 0x12,
 			.mask_ihl = 0x80,
 			.addr_od = 0x12,
diff --git a/drivers/iio/pressure/st_pressure_i2c.c b/drivers/iio/pressure/st_pressure_i2c.c
index 17417a4d5a5f..7f15e927fa2b 100644
--- a/drivers/iio/pressure/st_pressure_i2c.c
+++ b/drivers/iio/pressure/st_pressure_i2c.c
@@ -77,7 +77,8 @@ static int st_press_i2c_probe(struct i2c_client *client,
 	press_data = iio_priv(indio_dev);
 
 	if (client->dev.of_node) {
-		st_sensors_of_i2c_probe(client, st_press_of_match);
+		st_sensors_of_name_probe(&client->dev, st_press_of_match,
+					 client->name, sizeof(client->name));
 	} else if (ACPI_HANDLE(&client->dev)) {
 		ret = st_sensors_match_acpi_device(&client->dev);
 		if ((ret < 0) || (ret >= ST_PRESS_MAX))
diff --git a/drivers/iio/pressure/st_pressure_spi.c b/drivers/iio/pressure/st_pressure_spi.c
index 550508025af1..f5ebd36bb4bf 100644
--- a/drivers/iio/pressure/st_pressure_spi.c
+++ b/drivers/iio/pressure/st_pressure_spi.c
@@ -18,6 +18,36 @@
 #include <linux/iio/common/st_sensors_spi.h>
 #include "st_pressure.h"
 
+#ifdef CONFIG_OF
+/*
+ * For new single-chip sensors use <device_name> as compatible string.
+ * For old single-chip devices keep <device_name>-press to maintain
+ * compatibility
+ */
+static const struct of_device_id st_press_of_match[] = {
+	{
+		.compatible = "st,lps001wp-press",
+		.data = LPS001WP_PRESS_DEV_NAME,
+	},
+	{
+		.compatible = "st,lps25h-press",
+		.data = LPS25H_PRESS_DEV_NAME,
+	},
+	{
+		.compatible = "st,lps331ap-press",
+		.data = LPS331AP_PRESS_DEV_NAME,
+	},
+	{
+		.compatible = "st,lps22hb-press",
+		.data = LPS22HB_PRESS_DEV_NAME,
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(of, st_press_of_match);
+#else
+#define st_press_of_match	NULL
+#endif
+
 static int st_press_spi_probe(struct spi_device *spi)
 {
 	struct iio_dev *indio_dev;
@@ -30,6 +60,8 @@ static int st_press_spi_probe(struct spi_device *spi)
 
 	press_data = iio_priv(indio_dev);
 
+	st_sensors_of_name_probe(&spi->dev, st_press_of_match,
+				 spi->modalias, sizeof(spi->modalias));
 	st_sensors_spi_configure(indio_dev, spi, press_data);
 
 	err = st_press_common_probe(indio_dev);
@@ -58,6 +90,7 @@ MODULE_DEVICE_TABLE(spi, st_press_id_table);
 static struct spi_driver st_press_driver = {
 	.driver = {
 		.name = "st-press-spi",
+		.of_match_table = of_match_ptr(st_press_of_match),
 	},
 	.probe = st_press_spi_probe,
 	.remove = st_press_spi_remove,
diff --git a/drivers/iio/pressure/zpa2326.c b/drivers/iio/pressure/zpa2326.c
index c92a95f9f52c..ebfb1de7377f 100644
--- a/drivers/iio/pressure/zpa2326.c
+++ b/drivers/iio/pressure/zpa2326.c
@@ -141,14 +141,14 @@ struct zpa2326_private {
 	struct regulator               *vdd;
 };
 
-#define zpa2326_err(_idev, _format, _arg...) \
-	dev_err(_idev->dev.parent, _format, ##_arg)
+#define zpa2326_err(idev, fmt, ...)					\
+	dev_err(idev->dev.parent, fmt "\n", ##__VA_ARGS__)
 
-#define zpa2326_warn(_idev, _format, _arg...) \
-	dev_warn(_idev->dev.parent, _format, ##_arg)
+#define zpa2326_warn(idev, fmt, ...)					\
+	dev_warn(idev->dev.parent, fmt "\n", ##__VA_ARGS__)
 
-#define zpa2326_dbg(_idev, _format, _arg...) \
-	dev_dbg(_idev->dev.parent, _format, ##_arg)
+#define zpa2326_dbg(idev, fmt, ...)					\
+	dev_dbg(idev->dev.parent, fmt "\n", ##__VA_ARGS__)
 
 bool zpa2326_isreg_writeable(struct device *dev, unsigned int reg)
 {
diff --git a/drivers/iio/proximity/Kconfig b/drivers/iio/proximity/Kconfig
index 5b81a8c9d438..ae070950f920 100644
--- a/drivers/iio/proximity/Kconfig
+++ b/drivers/iio/proximity/Kconfig
@@ -57,12 +57,12 @@ config SX9500
 	  module will be called sx9500.
 
 config SRF08
-	tristate "Devantech SRF08 ultrasonic ranger sensor"
+	tristate "Devantech SRF02/SRF08/SRF10 ultrasonic ranger sensor"
 	depends on I2C
 	help
-	  Say Y here to build a driver for Devantech SRF08 ultrasonic
-	  ranger sensor. This driver can be used to measure the distance
-	  of objects.
+	  Say Y here to build a driver for Devantech SRF02/SRF08/SRF10
+	  ultrasonic ranger sensors with i2c interface.
+	  This driver can be used to measure the distance of objects.
 
 	  To compile this driver as a module, choose M here: the
 	  module will be called srf08.
diff --git a/drivers/iio/proximity/srf08.c b/drivers/iio/proximity/srf08.c
index 49316cbf7c60..9380d545aab1 100644
--- a/drivers/iio/proximity/srf08.c
+++ b/drivers/iio/proximity/srf08.c
@@ -1,14 +1,18 @@
 /*
- * srf08.c - Support for Devantech SRF08 ultrasonic ranger
+ * srf08.c - Support for Devantech SRFxx ultrasonic ranger
+ *           with i2c interface
+ * actually supported are srf02, srf08, srf10
  *
- * Copyright (c) 2016 Andreas Klinger <ak@it-klinger.de>
+ * Copyright (c) 2016, 2017 Andreas Klinger <ak@it-klinger.de>
  *
  * This file is subject to the terms and conditions of version 2 of
- * the GNU General Public License.  See the file COPYING in the main
+ * the GNU General Public License. See the file COPYING in the main
  * directory of this archive for more details.
  *
  * For details about the device see:
  * http://www.robot-electronics.co.uk/htm/srf08tech.html
+ * http://www.robot-electronics.co.uk/htm/srf10tech.htm
+ * http://www.robot-electronics.co.uk/htm/srf02tech.htm
  */
 
 #include <linux/err.h>
@@ -18,6 +22,9 @@
 #include <linux/bitops.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
+#include <linux/iio/buffer.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/iio/triggered_buffer.h>
 
 /* registers of SRF08 device */
 #define SRF08_WRITE_COMMAND	0x00	/* Command Register */
@@ -30,14 +37,46 @@
 
 #define SRF08_CMD_RANGING_CM	0x51	/* Ranging Mode - Result in cm */
 
-#define SRF08_DEFAULT_GAIN	1025	/* default analogue value of Gain */
-#define SRF08_DEFAULT_RANGE	6020	/* default value of Range in mm */
+enum srf08_sensor_type {
+	SRF02,
+	SRF08,
+	SRF10,
+	SRF_MAX_TYPE
+};
+
+struct srf08_chip_info {
+	const int		*sensitivity_avail;
+	int			num_sensitivity_avail;
+	int			sensitivity_default;
+
+	/* default value of Range in mm */
+	int			range_default;
+};
 
 struct srf08_data {
 	struct i2c_client	*client;
-	int			sensitivity;		/* Gain */
-	int			range_mm;		/* max. Range in mm */
+
+	/*
+	 * Gain in the datasheet is called sensitivity here to distinct it
+	 * from the gain used with amplifiers of adc's
+	 */
+	int			sensitivity;
+
+	/* max. Range in mm */
+	int			range_mm;
 	struct mutex		lock;
+
+	/*
+	 * triggered buffer
+	 * 1x16-bit channel + 3x16 padding + 4x16 timestamp
+	 */
+	s16			buffer[8];
+
+	/* Sensor-Type */
+	enum srf08_sensor_type	sensor_type;
+
+	/* Chip-specific information */
+	const struct srf08_chip_info	*chip_info;
 };
 
 /*
@@ -47,11 +86,42 @@ struct srf08_data {
  * But with ADC's this term is already used differently and that's why it
  * is called "Sensitivity" here.
  */
-static const int srf08_sensitivity[] = {
+static const struct srf08_chip_info srf02_chip_info = {
+	.sensitivity_avail	= NULL,
+	.num_sensitivity_avail	= 0,
+	.sensitivity_default	= 0,
+
+	.range_default		= 0,
+};
+
+static const int srf08_sensitivity_avail[] = {
 	 94,  97, 100, 103, 107, 110, 114, 118,
 	123, 128, 133, 139, 145, 152, 159, 168,
 	177, 187, 199, 212, 227, 245, 265, 288,
-	317, 352, 395, 450, 524, 626, 777, 1025 };
+	317, 352, 395, 450, 524, 626, 777, 1025
+	};
+
+static const struct srf08_chip_info srf08_chip_info = {
+	.sensitivity_avail	= srf08_sensitivity_avail,
+	.num_sensitivity_avail	= ARRAY_SIZE(srf08_sensitivity_avail),
+	.sensitivity_default	= 1025,
+
+	.range_default		= 6020,
+};
+
+static const int srf10_sensitivity_avail[] = {
+	 40,  40,  50,  60,  70,  80, 100, 120,
+	140, 200, 250, 300, 350, 400, 500, 600,
+	700,
+	};
+
+static const struct srf08_chip_info srf10_chip_info = {
+	.sensitivity_avail	= srf10_sensitivity_avail,
+	.num_sensitivity_avail	= ARRAY_SIZE(srf10_sensitivity_avail),
+	.sensitivity_default	= 700,
+
+	.range_default		= 6020,
+};
 
 static int srf08_read_ranging(struct srf08_data *data)
 {
@@ -110,6 +180,29 @@ static int srf08_read_ranging(struct srf08_data *data)
 	return ret;
 }
 
+static irqreturn_t srf08_trigger_handler(int irq, void *p)
+{
+	struct iio_poll_func *pf = p;
+	struct iio_dev *indio_dev = pf->indio_dev;
+	struct srf08_data *data = iio_priv(indio_dev);
+	s16 sensor_data;
+
+	sensor_data = srf08_read_ranging(data);
+	if (sensor_data < 0)
+		goto err;
+
+	mutex_lock(&data->lock);
+
+	data->buffer[0] = sensor_data;
+	iio_push_to_buffers_with_timestamp(indio_dev,
+						data->buffer, pf->timestamp);
+
+	mutex_unlock(&data->lock);
+err:
+	iio_trigger_notify_done(indio_dev->trig);
+	return IRQ_HANDLED;
+}
+
 static int srf08_read_raw(struct iio_dev *indio_dev,
 			    struct iio_chan_spec const *channel, int *val,
 			    int *val2, long mask)
@@ -225,9 +318,13 @@ static ssize_t srf08_show_sensitivity_available(struct device *dev,
 				struct device_attribute *attr, char *buf)
 {
 	int i, len = 0;
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct srf08_data *data = iio_priv(indio_dev);
 
-	for (i = 0; i < ARRAY_SIZE(srf08_sensitivity); i++)
-		len += sprintf(buf + len, "%d ", srf08_sensitivity[i]);
+	for (i = 0; i < data->chip_info->num_sensitivity_avail; i++)
+		if (data->chip_info->sensitivity_avail[i])
+			len += sprintf(buf + len, "%d ",
+				data->chip_info->sensitivity_avail[i]);
 
 	len += sprintf(buf + len, "\n");
 
@@ -256,19 +353,21 @@ static ssize_t srf08_write_sensitivity(struct srf08_data *data,
 	int ret, i;
 	u8 regval;
 
-	for (i = 0; i < ARRAY_SIZE(srf08_sensitivity); i++)
-		if (val == srf08_sensitivity[i]) {
+	if (!val)
+		return -EINVAL;
+
+	for (i = 0; i < data->chip_info->num_sensitivity_avail; i++)
+		if (val && (val == data->chip_info->sensitivity_avail[i])) {
 			regval = i;
 			break;
 		}
 
-	if (i >= ARRAY_SIZE(srf08_sensitivity))
+	if (i >= data->chip_info->num_sensitivity_avail)
 		return -EINVAL;
 
 	mutex_lock(&data->lock);
 
-	ret = i2c_smbus_write_byte_data(client,
-						SRF08_WRITE_MAX_GAIN, regval);
+	ret = i2c_smbus_write_byte_data(client, SRF08_WRITE_MAX_GAIN, regval);
 	if (ret < 0) {
 		dev_err(&client->dev, "write_sensitivity - err: %d\n", ret);
 		mutex_unlock(&data->lock);
@@ -323,7 +422,15 @@ static const struct iio_chan_spec srf08_channels[] = {
 		.info_mask_separate =
 				BIT(IIO_CHAN_INFO_RAW) |
 				BIT(IIO_CHAN_INFO_SCALE),
+		.scan_index = 0,
+		.scan_type = {
+			.sign = 's',
+			.realbits = 16,
+			.storagebits = 16,
+			.endianness = IIO_CPU,
+		},
 	},
+	IIO_CHAN_SOFT_TIMESTAMP(1),
 };
 
 static const struct iio_info srf08_info = {
@@ -332,6 +439,15 @@ static const struct iio_info srf08_info = {
 	.driver_module = THIS_MODULE,
 };
 
+/*
+ * srf02 don't have an adjustable range or sensitivity,
+ * so we don't need attributes at all
+ */
+static const struct iio_info srf02_info = {
+	.read_raw = srf08_read_raw,
+	.driver_module = THIS_MODULE,
+};
+
 static int srf08_probe(struct i2c_client *client,
 					 const struct i2c_device_id *id)
 {
@@ -352,34 +468,84 @@ static int srf08_probe(struct i2c_client *client,
 	data = iio_priv(indio_dev);
 	i2c_set_clientdata(client, indio_dev);
 	data->client = client;
+	data->sensor_type = (enum srf08_sensor_type)id->driver_data;
+
+	switch (data->sensor_type) {
+	case SRF02:
+		data->chip_info = &srf02_chip_info;
+		indio_dev->info = &srf02_info;
+		break;
+	case SRF08:
+		data->chip_info = &srf08_chip_info;
+		indio_dev->info = &srf08_info;
+		break;
+	case SRF10:
+		data->chip_info = &srf10_chip_info;
+		indio_dev->info = &srf08_info;
+		break;
+	default:
+		return -EINVAL;
+	}
 
-	indio_dev->name = "srf08";
+	indio_dev->name = id->name;
 	indio_dev->dev.parent = &client->dev;
 	indio_dev->modes = INDIO_DIRECT_MODE;
-	indio_dev->info = &srf08_info;
 	indio_dev->channels = srf08_channels;
 	indio_dev->num_channels = ARRAY_SIZE(srf08_channels);
 
 	mutex_init(&data->lock);
 
-	/*
-	 * set default values of device here
-	 * these register values cannot be read from the hardware
-	 * therefore set driver specific default values
-	 */
-	ret = srf08_write_range_mm(data, SRF08_DEFAULT_RANGE);
-	if (ret < 0)
+	ret = devm_iio_triggered_buffer_setup(&client->dev, indio_dev,
+			iio_pollfunc_store_time, srf08_trigger_handler, NULL);
+	if (ret < 0) {
+		dev_err(&client->dev, "setup of iio triggered buffer failed\n");
 		return ret;
+	}
 
-	ret = srf08_write_sensitivity(data, SRF08_DEFAULT_GAIN);
-	if (ret < 0)
-		return ret;
+	if (data->chip_info->range_default) {
+		/*
+		 * set default range of device in mm here
+		 * these register values cannot be read from the hardware
+		 * therefore set driver specific default values
+		 *
+		 * srf02 don't have a default value so it'll be omitted
+		 */
+		ret = srf08_write_range_mm(data,
+					data->chip_info->range_default);
+		if (ret < 0)
+			return ret;
+	}
+
+	if (data->chip_info->sensitivity_default) {
+		/*
+		 * set default sensitivity of device here
+		 * these register values cannot be read from the hardware
+		 * therefore set driver specific default values
+		 *
+		 * srf02 don't have a default value so it'll be omitted
+		 */
+		ret = srf08_write_sensitivity(data,
+				data->chip_info->sensitivity_default);
+		if (ret < 0)
+			return ret;
+	}
 
 	return devm_iio_device_register(&client->dev, indio_dev);
 }
 
+static const struct of_device_id of_srf08_match[] = {
+	{ .compatible = "devantech,srf02", (void *)SRF02},
+	{ .compatible = "devantech,srf08", (void *)SRF08},
+	{ .compatible = "devantech,srf10", (void *)SRF10},
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, of_srf08_match);
+
 static const struct i2c_device_id srf08_id[] = {
-	{ "srf08", 0 },
+	{ "srf02", SRF02 },
+	{ "srf08", SRF08 },
+	{ "srf10", SRF10 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, srf08_id);
@@ -387,6 +553,7 @@ MODULE_DEVICE_TABLE(i2c, srf08_id);
 static struct i2c_driver srf08_driver = {
 	.driver = {
 		.name	= "srf08",
+		.of_match_table	= of_srf08_match,
 	},
 	.probe = srf08_probe,
 	.id_table = srf08_id,
@@ -394,5 +561,5 @@ static struct i2c_driver srf08_driver = {
 module_i2c_driver(srf08_driver);
 
 MODULE_AUTHOR("Andreas Klinger <ak@it-klinger.de>");
-MODULE_DESCRIPTION("Devantech SRF08 ultrasonic ranger driver");
+MODULE_DESCRIPTION("Devantech SRF02/SRF08/SRF10 i2c ultrasonic ranger driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/iio/temperature/tsys01.c b/drivers/iio/temperature/tsys01.c
index 3e60c6189d98..d8aa211d76e4 100644
--- a/drivers/iio/temperature/tsys01.c
+++ b/drivers/iio/temperature/tsys01.c
@@ -214,11 +214,18 @@ static const struct i2c_device_id tsys01_id[] = {
 };
 MODULE_DEVICE_TABLE(i2c, tsys01_id);
 
+static const struct of_device_id tsys01_of_match[] = {
+	{ .compatible = "meas,tsys01", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, tsys01_of_match);
+
 static struct i2c_driver tsys01_driver = {
 	.probe = tsys01_i2c_probe,
 	.id_table = tsys01_id,
 	.driver = {
 		   .name = "tsys01",
+		   .of_match_table = of_match_ptr(tsys01_of_match),
 		   },
 };
 
diff --git a/drivers/iio/trigger/Kconfig b/drivers/iio/trigger/Kconfig
index e4d4e63434db..a633d2c8e805 100644
--- a/drivers/iio/trigger/Kconfig
+++ b/drivers/iio/trigger/Kconfig
@@ -24,6 +24,17 @@ config IIO_INTERRUPT_TRIGGER
 	  To compile this driver as a module, choose M here: the
 	  module will be called iio-trig-interrupt.
 
+config IIO_STM32_LPTIMER_TRIGGER
+	tristate "STM32 Low-Power Timer Trigger"
+	depends on MFD_STM32_LPTIMER || COMPILE_TEST
+	help
+	  Select this option to enable STM32 Low-Power Timer Trigger.
+	  This can be used as trigger source for STM32 internal ADC
+	  and/or DAC.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called stm32-lptimer-trigger.
+
 config IIO_STM32_TIMER_TRIGGER
 	tristate "STM32 Timer Trigger"
 	depends on (ARCH_STM32 && OF && MFD_STM32_TIMERS) || COMPILE_TEST
diff --git a/drivers/iio/trigger/Makefile b/drivers/iio/trigger/Makefile
index 5c4ecd380653..0a72a2a76cb2 100644
--- a/drivers/iio/trigger/Makefile
+++ b/drivers/iio/trigger/Makefile
@@ -6,6 +6,7 @@
 
 obj-$(CONFIG_IIO_HRTIMER_TRIGGER) += iio-trig-hrtimer.o
 obj-$(CONFIG_IIO_INTERRUPT_TRIGGER) += iio-trig-interrupt.o
+obj-$(CONFIG_IIO_STM32_LPTIMER_TRIGGER) += stm32-lptimer-trigger.o
 obj-$(CONFIG_IIO_STM32_TIMER_TRIGGER) += stm32-timer-trigger.o
 obj-$(CONFIG_IIO_SYSFS_TRIGGER) += iio-trig-sysfs.o
 obj-$(CONFIG_IIO_TIGHTLOOP_TRIGGER) += iio-trig-loop.o
diff --git a/drivers/iio/trigger/stm32-lptimer-trigger.c b/drivers/iio/trigger/stm32-lptimer-trigger.c
new file mode 100644
index 000000000000..241eae6a4306
--- /dev/null
+++ b/drivers/iio/trigger/stm32-lptimer-trigger.c
@@ -0,0 +1,118 @@
+/*
+ * STM32 Low-Power Timer Trigger driver
+ *
+ * Copyright (C) STMicroelectronics 2017
+ *
+ * Author: Fabrice Gasnier <fabrice.gasnier@st.com>.
+ *
+ * License terms:  GNU General Public License (GPL), version 2
+ *
+ * Inspired by Benjamin Gaignard's stm32-timer-trigger driver
+ */
+
+#include <linux/iio/timer/stm32-lptim-trigger.h>
+#include <linux/mfd/stm32-lptimer.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+/* List Low-Power Timer triggers */
+static const char * const stm32_lptim_triggers[] = {
+	LPTIM1_OUT,
+	LPTIM2_OUT,
+	LPTIM3_OUT,
+};
+
+struct stm32_lptim_trigger {
+	struct device *dev;
+	const char *trg;
+};
+
+static int stm32_lptim_validate_device(struct iio_trigger *trig,
+				       struct iio_dev *indio_dev)
+{
+	if (indio_dev->modes & INDIO_HARDWARE_TRIGGERED)
+		return 0;
+
+	return -EINVAL;
+}
+
+static const struct iio_trigger_ops stm32_lptim_trigger_ops = {
+	.owner = THIS_MODULE,
+	.validate_device = stm32_lptim_validate_device,
+};
+
+/**
+ * is_stm32_lptim_trigger
+ * @trig: trigger to be checked
+ *
+ * return true if the trigger is a valid STM32 IIO Low-Power Timer Trigger
+ * either return false
+ */
+bool is_stm32_lptim_trigger(struct iio_trigger *trig)
+{
+	return (trig->ops == &stm32_lptim_trigger_ops);
+}
+EXPORT_SYMBOL(is_stm32_lptim_trigger);
+
+static int stm32_lptim_setup_trig(struct stm32_lptim_trigger *priv)
+{
+	struct iio_trigger *trig;
+
+	trig = devm_iio_trigger_alloc(priv->dev, "%s", priv->trg);
+	if  (!trig)
+		return -ENOMEM;
+
+	trig->dev.parent = priv->dev->parent;
+	trig->ops = &stm32_lptim_trigger_ops;
+	iio_trigger_set_drvdata(trig, priv);
+
+	return devm_iio_trigger_register(priv->dev, trig);
+}
+
+static int stm32_lptim_trigger_probe(struct platform_device *pdev)
+{
+	struct stm32_lptim_trigger *priv;
+	u32 index;
+	int ret;
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	if (of_property_read_u32(pdev->dev.of_node, "reg", &index))
+		return -EINVAL;
+
+	if (index >= ARRAY_SIZE(stm32_lptim_triggers))
+		return -EINVAL;
+
+	priv->dev = &pdev->dev;
+	priv->trg = stm32_lptim_triggers[index];
+
+	ret = stm32_lptim_setup_trig(priv);
+	if (ret)
+		return ret;
+
+	platform_set_drvdata(pdev, priv);
+
+	return 0;
+}
+
+static const struct of_device_id stm32_lptim_trig_of_match[] = {
+	{ .compatible = "st,stm32-lptimer-trigger", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, stm32_lptim_trig_of_match);
+
+static struct platform_driver stm32_lptim_trigger_driver = {
+	.probe = stm32_lptim_trigger_probe,
+	.driver = {
+		.name = "stm32-lptimer-trigger",
+		.of_match_table = stm32_lptim_trig_of_match,
+	},
+};
+module_platform_driver(stm32_lptim_trigger_driver);
+
+MODULE_AUTHOR("Fabrice Gasnier <fabrice.gasnier@st.com>");
+MODULE_ALIAS("platform:stm32-lptimer-trigger");
+MODULE_DESCRIPTION("STMicroelectronics STM32 LPTIM trigger driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/trigger/stm32-timer-trigger.c b/drivers/iio/trigger/stm32-timer-trigger.c
index d22bc56dd9fc..9b9053494daf 100644
--- a/drivers/iio/trigger/stm32-timer-trigger.c
+++ b/drivers/iio/trigger/stm32-timer-trigger.c
@@ -13,6 +13,7 @@
 #include <linux/mfd/stm32-timers.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/of_device.h>
 
 #define MAX_TRIGGERS 7
 #define MAX_VALIDS 5
@@ -28,9 +29,14 @@ static const void *triggers_table[][MAX_TRIGGERS] = {
 	{ TIM7_TRGO,},
 	{ TIM8_TRGO, TIM8_TRGO2, TIM8_CH1, TIM8_CH2, TIM8_CH3, TIM8_CH4,},
 	{ TIM9_TRGO, TIM9_CH1, TIM9_CH2,},
-	{ }, /* timer 10 */
-	{ }, /* timer 11 */
+	{ TIM10_OC1,},
+	{ TIM11_OC1,},
 	{ TIM12_TRGO, TIM12_CH1, TIM12_CH2,},
+	{ TIM13_OC1,},
+	{ TIM14_OC1,},
+	{ TIM15_TRGO,},
+	{ TIM16_OC1,},
+	{ TIM17_OC1,},
 };
 
 /* List the triggers accepted by each timer */
@@ -43,10 +49,30 @@ static const void *valids_table[][MAX_VALIDS] = {
 	{ }, /* timer 6 */
 	{ }, /* timer 7 */
 	{ TIM1_TRGO, TIM2_TRGO, TIM4_TRGO, TIM5_TRGO,},
-	{ TIM2_TRGO, TIM3_TRGO,},
+	{ TIM2_TRGO, TIM3_TRGO, TIM10_OC1, TIM11_OC1,},
+	{ }, /* timer 10 */
+	{ }, /* timer 11 */
+	{ TIM4_TRGO, TIM5_TRGO, TIM13_OC1, TIM14_OC1,},
+};
+
+static const void *stm32h7_valids_table[][MAX_VALIDS] = {
+	{ TIM15_TRGO, TIM2_TRGO, TIM3_TRGO, TIM4_TRGO,},
+	{ TIM1_TRGO, TIM8_TRGO, TIM3_TRGO, TIM4_TRGO,},
+	{ TIM1_TRGO, TIM2_TRGO, TIM15_TRGO, TIM4_TRGO,},
+	{ TIM1_TRGO, TIM2_TRGO, TIM3_TRGO, TIM8_TRGO,},
+	{ TIM1_TRGO, TIM8_TRGO, TIM3_TRGO, TIM4_TRGO,},
+	{ }, /* timer 6 */
+	{ }, /* timer 7 */
+	{ TIM1_TRGO, TIM2_TRGO, TIM4_TRGO, TIM5_TRGO,},
+	{ }, /* timer 9 */
 	{ }, /* timer 10 */
 	{ }, /* timer 11 */
-	{ TIM4_TRGO, TIM5_TRGO,},
+	{ TIM4_TRGO, TIM5_TRGO, TIM13_OC1, TIM14_OC1,},
+	{ }, /* timer 13 */
+	{ }, /* timer 14 */
+	{ TIM1_TRGO, TIM3_TRGO, TIM16_OC1, TIM17_OC1,},
+	{ }, /* timer 16 */
+	{ }, /* timer 17 */
 };
 
 struct stm32_timer_trigger {
@@ -59,11 +85,21 @@ struct stm32_timer_trigger {
 	bool has_trgo2;
 };
 
+struct stm32_timer_trigger_cfg {
+	const void *(*valids_table)[MAX_VALIDS];
+	const unsigned int num_valids_table;
+};
+
 static bool stm32_timer_is_trgo2_name(const char *name)
 {
 	return !!strstr(name, "trgo2");
 }
 
+static bool stm32_timer_is_trgo_name(const char *name)
+{
+	return (!!strstr(name, "trgo") && !strstr(name, "trgo2"));
+}
+
 static int stm32_timer_start(struct stm32_timer_trigger *priv,
 			     struct iio_trigger *trig,
 			     unsigned int frequency)
@@ -328,6 +364,7 @@ static int stm32_setup_iio_triggers(struct stm32_timer_trigger *priv)
 
 	while (cur && *cur) {
 		struct iio_trigger *trig;
+		bool cur_is_trgo = stm32_timer_is_trgo_name(*cur);
 		bool cur_is_trgo2 = stm32_timer_is_trgo2_name(*cur);
 
 		if (cur_is_trgo2 && !priv->has_trgo2) {
@@ -344,10 +381,9 @@ static int stm32_setup_iio_triggers(struct stm32_timer_trigger *priv)
 
 		/*
 		 * sampling frequency and master mode attributes
-		 * should only be available on trgo trigger which
-		 * is always the first in the list.
+		 * should only be available on trgo/trgo2 triggers
 		 */
-		if (cur == priv->triggers || cur_is_trgo2)
+		if (cur_is_trgo || cur_is_trgo2)
 			trig->dev.groups = stm32_trigger_attr_groups;
 
 		iio_trigger_set_drvdata(trig, priv);
@@ -366,34 +402,32 @@ static int stm32_counter_read_raw(struct iio_dev *indio_dev,
 				  int *val, int *val2, long mask)
 {
 	struct stm32_timer_trigger *priv = iio_priv(indio_dev);
+	u32 dat;
 
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
-	{
-		u32 cnt;
-
-		regmap_read(priv->regmap, TIM_CNT, &cnt);
-		*val = cnt;
+		regmap_read(priv->regmap, TIM_CNT, &dat);
+		*val = dat;
+		return IIO_VAL_INT;
 
+	case IIO_CHAN_INFO_ENABLE:
+		regmap_read(priv->regmap, TIM_CR1, &dat);
+		*val = (dat & TIM_CR1_CEN) ? 1 : 0;
 		return IIO_VAL_INT;
-	}
-	case IIO_CHAN_INFO_SCALE:
-	{
-		u32 smcr;
 
-		regmap_read(priv->regmap, TIM_SMCR, &smcr);
-		smcr &= TIM_SMCR_SMS;
+	case IIO_CHAN_INFO_SCALE:
+		regmap_read(priv->regmap, TIM_SMCR, &dat);
+		dat &= TIM_SMCR_SMS;
 
 		*val = 1;
 		*val2 = 0;
 
 		/* in quadrature case scale = 0.25 */
-		if (smcr == 3)
+		if (dat == 3)
 			*val2 = 2;
 
 		return IIO_VAL_FRACTIONAL_LOG2;
 	}
-	}
 
 	return -EINVAL;
 }
@@ -403,15 +437,31 @@ static int stm32_counter_write_raw(struct iio_dev *indio_dev,
 				   int val, int val2, long mask)
 {
 	struct stm32_timer_trigger *priv = iio_priv(indio_dev);
+	u32 dat;
 
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
-		regmap_write(priv->regmap, TIM_CNT, val);
+		return regmap_write(priv->regmap, TIM_CNT, val);
 
-		return IIO_VAL_INT;
 	case IIO_CHAN_INFO_SCALE:
 		/* fixed scale */
 		return -EINVAL;
+
+	case IIO_CHAN_INFO_ENABLE:
+		if (val) {
+			regmap_read(priv->regmap, TIM_CR1, &dat);
+			if (!(dat & TIM_CR1_CEN))
+				clk_enable(priv->clk);
+			regmap_update_bits(priv->regmap, TIM_CR1, TIM_CR1_CEN,
+					   TIM_CR1_CEN);
+		} else {
+			regmap_read(priv->regmap, TIM_CR1, &dat);
+			regmap_update_bits(priv->regmap, TIM_CR1, TIM_CR1_CEN,
+					   0);
+			if (dat & TIM_CR1_CEN)
+				clk_disable(priv->clk);
+		}
+		return 0;
 	}
 
 	return -EINVAL;
@@ -471,7 +521,7 @@ static int stm32_get_trigger_mode(struct iio_dev *indio_dev,
 
 	regmap_read(priv->regmap, TIM_SMCR, &smcr);
 
-	return smcr == TIM_SMCR_SMS ? 0 : -EINVAL;
+	return (smcr & TIM_SMCR_SMS) == TIM_SMCR_SMS ? 0 : -EINVAL;
 }
 
 static const struct iio_enum stm32_trigger_mode_enum = {
@@ -507,9 +557,19 @@ static int stm32_set_enable_mode(struct iio_dev *indio_dev,
 {
 	struct stm32_timer_trigger *priv = iio_priv(indio_dev);
 	int sms = stm32_enable_mode2sms(mode);
+	u32 val;
 
 	if (sms < 0)
 		return sms;
+	/*
+	 * Triggered mode sets CEN bit automatically by hardware. So, first
+	 * enable counter clock, so it can use it. Keeps it in sync with CEN.
+	 */
+	if (sms == 6) {
+		regmap_read(priv->regmap, TIM_CR1, &val);
+		if (!(val & TIM_CR1_CEN))
+			clk_enable(priv->clk);
+	}
 
 	regmap_update_bits(priv->regmap, TIM_SMCR, TIM_SMCR_SMS, sms);
 
@@ -571,11 +631,14 @@ static int stm32_get_quadrature_mode(struct iio_dev *indio_dev,
 {
 	struct stm32_timer_trigger *priv = iio_priv(indio_dev);
 	u32 smcr;
+	int mode;
 
 	regmap_read(priv->regmap, TIM_SMCR, &smcr);
-	smcr &= TIM_SMCR_SMS;
+	mode = (smcr & TIM_SMCR_SMS) - 1;
+	if ((mode < 0) || (mode > ARRAY_SIZE(stm32_quadrature_modes)))
+		return -EINVAL;
 
-	return smcr - 1;
+	return mode;
 }
 
 static const struct iio_enum stm32_quadrature_mode_enum = {
@@ -592,13 +655,20 @@ static const char *const stm32_count_direction_states[] = {
 
 static int stm32_set_count_direction(struct iio_dev *indio_dev,
 				     const struct iio_chan_spec *chan,
-				     unsigned int mode)
+				     unsigned int dir)
 {
 	struct stm32_timer_trigger *priv = iio_priv(indio_dev);
+	u32 val;
+	int mode;
 
-	regmap_update_bits(priv->regmap, TIM_CR1, TIM_CR1_DIR, mode);
+	/* In encoder mode, direction is RO (given by TI1/TI2 signals) */
+	regmap_read(priv->regmap, TIM_SMCR, &val);
+	mode = (val & TIM_SMCR_SMS) - 1;
+	if ((mode >= 0) || (mode < ARRAY_SIZE(stm32_quadrature_modes)))
+		return -EBUSY;
 
-	return 0;
+	return regmap_update_bits(priv->regmap, TIM_CR1, TIM_CR1_DIR,
+				  dir ? TIM_CR1_DIR : 0);
 }
 
 static int stm32_get_count_direction(struct iio_dev *indio_dev,
@@ -609,7 +679,7 @@ static int stm32_get_count_direction(struct iio_dev *indio_dev,
 
 	regmap_read(priv->regmap, TIM_CR1, &cr1);
 
-	return (cr1 & TIM_CR1_DIR);
+	return ((cr1 & TIM_CR1_DIR) ? 1 : 0);
 }
 
 static const struct iio_enum stm32_count_direction_enum = {
@@ -672,7 +742,9 @@ static const struct iio_chan_spec_ext_info stm32_trigger_count_info[] = {
 static const struct iio_chan_spec stm32_trigger_channel = {
 	.type = IIO_COUNT,
 	.channel = 0,
-	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE),
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+			      BIT(IIO_CHAN_INFO_ENABLE) |
+			      BIT(IIO_CHAN_INFO_SCALE),
 	.ext_info = stm32_trigger_count_info,
 	.indexed = 1
 };
@@ -734,18 +806,22 @@ static int stm32_timer_trigger_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct stm32_timer_trigger *priv;
 	struct stm32_timers *ddata = dev_get_drvdata(pdev->dev.parent);
+	const struct stm32_timer_trigger_cfg *cfg;
 	unsigned int index;
 	int ret;
 
 	if (of_property_read_u32(dev->of_node, "reg", &index))
 		return -EINVAL;
 
+	cfg = (const struct stm32_timer_trigger_cfg *)
+		of_match_device(dev->driver->of_match_table, dev)->data;
+
 	if (index >= ARRAY_SIZE(triggers_table) ||
-	    index >= ARRAY_SIZE(valids_table))
+	    index >= cfg->num_valids_table)
 		return -EINVAL;
 
 	/* Create an IIO device only if we have triggers to be validated */
-	if (*valids_table[index])
+	if (*cfg->valids_table[index])
 		priv = stm32_setup_counter_device(dev);
 	else
 		priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
@@ -758,7 +834,7 @@ static int stm32_timer_trigger_probe(struct platform_device *pdev)
 	priv->clk = ddata->clk;
 	priv->max_arr = ddata->max_arr;
 	priv->triggers = triggers_table[index];
-	priv->valids = valids_table[index];
+	priv->valids = cfg->valids_table[index];
 	stm32_timer_detect_trgo2(priv);
 
 	ret = stm32_setup_iio_triggers(priv);
@@ -770,8 +846,24 @@ static int stm32_timer_trigger_probe(struct platform_device *pdev)
 	return 0;
 }
 
+static const struct stm32_timer_trigger_cfg stm32_timer_trg_cfg = {
+	.valids_table = valids_table,
+	.num_valids_table = ARRAY_SIZE(valids_table),
+};
+
+static const struct stm32_timer_trigger_cfg stm32h7_timer_trg_cfg = {
+	.valids_table = stm32h7_valids_table,
+	.num_valids_table = ARRAY_SIZE(stm32h7_valids_table),
+};
+
 static const struct of_device_id stm32_trig_of_match[] = {
-	{ .compatible = "st,stm32-timer-trigger", },
+	{
+		.compatible = "st,stm32-timer-trigger",
+		.data = (void *)&stm32_timer_trg_cfg,
+	}, {
+		.compatible = "st,stm32h7-timer-trigger",
+		.data = (void *)&stm32h7_timer_trg_cfg,
+	},
 	{ /* end node */ },
 };
 MODULE_DEVICE_TABLE(of, stm32_trig_of_match);
diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig
index 234fe01904e7..3726205c8704 100644
--- a/drivers/infiniband/Kconfig
+++ b/drivers/infiniband/Kconfig
@@ -34,6 +34,15 @@ config INFINIBAND_USER_ACCESS
 	  libibverbs, libibcm and a hardware driver library from
 	  <http://www.openfabrics.org/git/>.
 
+config INFINIBAND_EXP_USER_ACCESS
+	bool "Allow experimental support for Infiniband ABI"
+	depends on INFINIBAND_USER_ACCESS
+	---help---
+	  IOCTL based ABI support for Infiniband. This allows userspace
+	  to invoke the experimental IOCTL based ABI.
+	  These commands are parsed via per-device parsing tree and
+	  enables per-device features.
+
 config INFINIBAND_USER_MEM
 	bool
 	depends on INFINIBAND_USER_ACCESS != n
diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile
index e3cdafff8ece..b4df164f71a6 100644
--- a/drivers/infiniband/core/Makefile
+++ b/drivers/infiniband/core/Makefile
@@ -11,7 +11,8 @@ ib_core-y :=			packer.o ud_header.o verbs.o cq.o rw.o sysfs.o \
 				device.o fmr_pool.o cache.o netlink.o \
 				roce_gid_mgmt.o mr_pool.o addr.o sa_query.o \
 				multicast.o mad.o smi.o agent.o mad_rmpp.o \
-				security.o
+				security.o nldev.o
+
 ib_core-$(CONFIG_INFINIBAND_USER_MEM) += umem.o
 ib_core-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += umem_odp.o umem_rbtree.o
 ib_core-$(CONFIG_CGROUP_RDMA) += cgroup.o
@@ -31,4 +32,5 @@ ib_umad-y :=			user_mad.o
 ib_ucm-y :=			ucm.o
 
 ib_uverbs-y :=			uverbs_main.o uverbs_cmd.o uverbs_marshall.o \
-				rdma_core.o uverbs_std_types.o
+				rdma_core.o uverbs_std_types.o uverbs_ioctl.o \
+				uverbs_ioctl_merge.o
diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c
index 437522ca97b4..12523f630b61 100644
--- a/drivers/infiniband/core/addr.c
+++ b/drivers/infiniband/core/addr.c
@@ -130,13 +130,11 @@ static void ib_nl_process_good_ip_rsep(const struct nlmsghdr *nlh)
 }
 
 int ib_nl_handle_ip_res_resp(struct sk_buff *skb,
-			     struct netlink_callback *cb)
+			     struct nlmsghdr *nlh,
+			     struct netlink_ext_ack *extack)
 {
-	const struct nlmsghdr *nlh = (struct nlmsghdr *)cb->nlh;
-
 	if ((nlh->nlmsg_flags & NLM_F_REQUEST) ||
-	    !(NETLINK_CB(skb).sk) ||
-	    !netlink_capable(skb, CAP_NET_ADMIN))
+	    !(NETLINK_CB(skb).sk))
 		return -EPERM;
 
 	if (ib_nl_is_good_ip_resp(nlh))
@@ -186,7 +184,7 @@ static int ib_nl_ip_send_msg(struct rdma_dev_addr *dev_addr,
 
 	/* Repair the nlmsg header length */
 	nlmsg_end(skb, nlh);
-	ibnl_multicast(skb, nlh, RDMA_NL_GROUP_LS, GFP_KERNEL);
+	rdma_nl_multicast(skb, RDMA_NL_GROUP_LS, GFP_KERNEL);
 
 	/* Make the request retry, so when we get the response from userspace
 	 * we will have something.
@@ -326,7 +324,7 @@ static void queue_req(struct addr_req *req)
 static int ib_nl_fetch_ha(struct dst_entry *dst, struct rdma_dev_addr *dev_addr,
 			  const void *daddr, u32 seq, u16 family)
 {
-	if (ibnl_chk_listeners(RDMA_NL_GROUP_LS))
+	if (rdma_nl_chk_listeners(RDMA_NL_GROUP_LS))
 		return -EADDRNOTAVAIL;
 
 	/* We fill in what we can, the response will fill the rest */
diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c
index efc94304dee3..77515638c55c 100644
--- a/drivers/infiniband/core/cache.c
+++ b/drivers/infiniband/core/cache.c
@@ -1199,30 +1199,23 @@ int ib_cache_setup_one(struct ib_device *device)
 	device->cache.ports =
 		kzalloc(sizeof(*device->cache.ports) *
 			(rdma_end_port(device) - rdma_start_port(device) + 1), GFP_KERNEL);
-	if (!device->cache.ports) {
-		err = -ENOMEM;
-		goto out;
-	}
+	if (!device->cache.ports)
+		return -ENOMEM;
 
 	err = gid_table_setup_one(device);
-	if (err)
-		goto out;
+	if (err) {
+		kfree(device->cache.ports);
+		device->cache.ports = NULL;
+		return err;
+	}
 
 	for (p = 0; p <= rdma_end_port(device) - rdma_start_port(device); ++p)
 		ib_cache_update(device, p + rdma_start_port(device), true);
 
 	INIT_IB_EVENT_HANDLER(&device->cache.event_handler,
 			      device, ib_cache_event);
-	err = ib_register_event_handler(&device->cache.event_handler);
-	if (err)
-		goto err;
-
+	ib_register_event_handler(&device->cache.event_handler);
 	return 0;
-
-err:
-	gid_table_cleanup_one(device);
-out:
-	return err;
 }
 
 void ib_cache_release_one(struct ib_device *device)
diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c
index 2b4d613a3474..4c4b46586af2 100644
--- a/drivers/infiniband/core/cm.c
+++ b/drivers/infiniband/core/cm.c
@@ -373,11 +373,19 @@ out:
 	return ret;
 }
 
-static int cm_alloc_response_msg(struct cm_port *port,
-				 struct ib_mad_recv_wc *mad_recv_wc,
-				 struct ib_mad_send_buf **msg)
+static struct ib_mad_send_buf *cm_alloc_response_msg_no_ah(struct cm_port *port,
+							   struct ib_mad_recv_wc *mad_recv_wc)
+{
+	return ib_create_send_mad(port->mad_agent, 1, mad_recv_wc->wc->pkey_index,
+				  0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
+				  GFP_ATOMIC,
+				  IB_MGMT_BASE_VERSION);
+}
+
+static int cm_create_response_msg_ah(struct cm_port *port,
+				     struct ib_mad_recv_wc *mad_recv_wc,
+				     struct ib_mad_send_buf *msg)
 {
-	struct ib_mad_send_buf *m;
 	struct ib_ah *ah;
 
 	ah = ib_create_ah_from_wc(port->mad_agent->qp->pd, mad_recv_wc->wc,
@@ -385,27 +393,40 @@ static int cm_alloc_response_msg(struct cm_port *port,
 	if (IS_ERR(ah))
 		return PTR_ERR(ah);
 
-	m = ib_create_send_mad(port->mad_agent, 1, mad_recv_wc->wc->pkey_index,
-			       0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
-			       GFP_ATOMIC,
-			       IB_MGMT_BASE_VERSION);
-	if (IS_ERR(m)) {
-		rdma_destroy_ah(ah);
-		return PTR_ERR(m);
-	}
-	m->ah = ah;
-	*msg = m;
+	msg->ah = ah;
 	return 0;
 }
 
 static void cm_free_msg(struct ib_mad_send_buf *msg)
 {
-	rdma_destroy_ah(msg->ah);
+	if (msg->ah)
+		rdma_destroy_ah(msg->ah);
 	if (msg->context[0])
 		cm_deref_id(msg->context[0]);
 	ib_free_send_mad(msg);
 }
 
+static int cm_alloc_response_msg(struct cm_port *port,
+				 struct ib_mad_recv_wc *mad_recv_wc,
+				 struct ib_mad_send_buf **msg)
+{
+	struct ib_mad_send_buf *m;
+	int ret;
+
+	m = cm_alloc_response_msg_no_ah(port, mad_recv_wc);
+	if (IS_ERR(m))
+		return PTR_ERR(m);
+
+	ret = cm_create_response_msg_ah(port, mad_recv_wc, m);
+	if (ret) {
+		cm_free_msg(m);
+		return ret;
+	}
+
+	*msg = m;
+	return 0;
+}
+
 static void * cm_copy_private_data(const void *private_data,
 				   u8 private_data_len)
 {
@@ -1175,6 +1196,11 @@ static void cm_format_req(struct cm_req_msg *req_msg,
 {
 	struct sa_path_rec *pri_path = param->primary_path;
 	struct sa_path_rec *alt_path = param->alternate_path;
+	bool pri_ext = false;
+
+	if (pri_path->rec_type == SA_PATH_REC_TYPE_OPA)
+		pri_ext = opa_is_extended_lid(pri_path->opa.dlid,
+					      pri_path->opa.slid);
 
 	cm_format_mad_hdr(&req_msg->hdr, CM_REQ_ATTR_ID,
 			  cm_form_tid(cm_id_priv, CM_MSG_SEQUENCE_REQ));
@@ -1202,18 +1228,24 @@ static void cm_format_req(struct cm_req_msg *req_msg,
 		cm_req_set_srq(req_msg, param->srq);
 	}
 
+	req_msg->primary_local_gid = pri_path->sgid;
+	req_msg->primary_remote_gid = pri_path->dgid;
+	if (pri_ext) {
+		req_msg->primary_local_gid.global.interface_id
+			= OPA_MAKE_ID(be32_to_cpu(pri_path->opa.slid));
+		req_msg->primary_remote_gid.global.interface_id
+			= OPA_MAKE_ID(be32_to_cpu(pri_path->opa.dlid));
+	}
 	if (pri_path->hop_limit <= 1) {
-		req_msg->primary_local_lid =
+		req_msg->primary_local_lid = pri_ext ? 0 :
 			htons(ntohl(sa_path_get_slid(pri_path)));
-		req_msg->primary_remote_lid =
+		req_msg->primary_remote_lid = pri_ext ? 0 :
 			htons(ntohl(sa_path_get_dlid(pri_path)));
 	} else {
 		/* Work-around until there's a way to obtain remote LID info */
 		req_msg->primary_local_lid = IB_LID_PERMISSIVE;
 		req_msg->primary_remote_lid = IB_LID_PERMISSIVE;
 	}
-	req_msg->primary_local_gid = pri_path->sgid;
-	req_msg->primary_remote_gid = pri_path->dgid;
 	cm_req_set_primary_flow_label(req_msg, pri_path->flow_label);
 	cm_req_set_primary_packet_rate(req_msg, pri_path->rate);
 	req_msg->primary_traffic_class = pri_path->traffic_class;
@@ -1225,17 +1257,29 @@ static void cm_format_req(struct cm_req_msg *req_msg,
 			       pri_path->packet_life_time));
 
 	if (alt_path) {
+		bool alt_ext = false;
+
+		if (alt_path->rec_type == SA_PATH_REC_TYPE_OPA)
+			alt_ext = opa_is_extended_lid(alt_path->opa.dlid,
+						      alt_path->opa.slid);
+
+		req_msg->alt_local_gid = alt_path->sgid;
+		req_msg->alt_remote_gid = alt_path->dgid;
+		if (alt_ext) {
+			req_msg->alt_local_gid.global.interface_id
+				= OPA_MAKE_ID(be32_to_cpu(alt_path->opa.slid));
+			req_msg->alt_remote_gid.global.interface_id
+				= OPA_MAKE_ID(be32_to_cpu(alt_path->opa.dlid));
+		}
 		if (alt_path->hop_limit <= 1) {
-			req_msg->alt_local_lid =
+			req_msg->alt_local_lid = alt_ext ? 0 :
 				htons(ntohl(sa_path_get_slid(alt_path)));
-			req_msg->alt_remote_lid =
+			req_msg->alt_remote_lid = alt_ext ? 0 :
 				htons(ntohl(sa_path_get_dlid(alt_path)));
 		} else {
 			req_msg->alt_local_lid = IB_LID_PERMISSIVE;
 			req_msg->alt_remote_lid = IB_LID_PERMISSIVE;
 		}
-		req_msg->alt_local_gid = alt_path->sgid;
-		req_msg->alt_remote_gid = alt_path->dgid;
 		cm_req_set_alt_flow_label(req_msg,
 					  alt_path->flow_label);
 		cm_req_set_alt_packet_rate(req_msg, alt_path->rate);
@@ -1405,16 +1449,63 @@ static inline int cm_is_active_peer(__be64 local_ca_guid, __be64 remote_ca_guid,
 		 (be32_to_cpu(local_qpn) > be32_to_cpu(remote_qpn))));
 }
 
+static bool cm_req_has_alt_path(struct cm_req_msg *req_msg)
+{
+	return ((req_msg->alt_local_lid) ||
+		(ib_is_opa_gid(&req_msg->alt_local_gid)));
+}
+
+static void cm_path_set_rec_type(struct ib_device *ib_device, u8 port_num,
+				 struct sa_path_rec *path, union ib_gid *gid)
+{
+	if (ib_is_opa_gid(gid) && rdma_cap_opa_ah(ib_device, port_num))
+		path->rec_type = SA_PATH_REC_TYPE_OPA;
+	else
+		path->rec_type = SA_PATH_REC_TYPE_IB;
+}
+
+static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg,
+					struct sa_path_rec *primary_path,
+					struct sa_path_rec *alt_path)
+{
+	u32 lid;
+
+	if (primary_path->rec_type != SA_PATH_REC_TYPE_OPA) {
+		sa_path_set_dlid(primary_path,
+				 htonl(ntohs(req_msg->primary_local_lid)));
+		sa_path_set_slid(primary_path,
+				 htonl(ntohs(req_msg->primary_remote_lid)));
+	} else {
+		lid = opa_get_lid_from_gid(&req_msg->primary_local_gid);
+		sa_path_set_dlid(primary_path, cpu_to_be32(lid));
+
+		lid = opa_get_lid_from_gid(&req_msg->primary_remote_gid);
+		sa_path_set_slid(primary_path, cpu_to_be32(lid));
+	}
+
+	if (!cm_req_has_alt_path(req_msg))
+		return;
+
+	if (alt_path->rec_type != SA_PATH_REC_TYPE_OPA) {
+		sa_path_set_dlid(alt_path,
+				 htonl(ntohs(req_msg->alt_local_lid)));
+		sa_path_set_slid(alt_path,
+				 htonl(ntohs(req_msg->alt_remote_lid)));
+	} else {
+		lid = opa_get_lid_from_gid(&req_msg->alt_local_gid);
+		sa_path_set_dlid(alt_path, cpu_to_be32(lid));
+
+		lid = opa_get_lid_from_gid(&req_msg->alt_remote_gid);
+		sa_path_set_slid(alt_path, cpu_to_be32(lid));
+	}
+}
+
 static void cm_format_paths_from_req(struct cm_req_msg *req_msg,
 				     struct sa_path_rec *primary_path,
 				     struct sa_path_rec *alt_path)
 {
 	primary_path->dgid = req_msg->primary_local_gid;
 	primary_path->sgid = req_msg->primary_remote_gid;
-	sa_path_set_dlid(primary_path,
-			 htonl(ntohs(req_msg->primary_local_lid)));
-	sa_path_set_slid(primary_path,
-			 htonl(ntohs(req_msg->primary_remote_lid)));
 	primary_path->flow_label = cm_req_get_primary_flow_label(req_msg);
 	primary_path->hop_limit = req_msg->primary_hop_limit;
 	primary_path->traffic_class = req_msg->primary_traffic_class;
@@ -1431,13 +1522,9 @@ static void cm_format_paths_from_req(struct cm_req_msg *req_msg,
 	primary_path->packet_life_time -= (primary_path->packet_life_time > 0);
 	primary_path->service_id = req_msg->service_id;
 
-	if (req_msg->alt_local_lid) {
+	if (cm_req_has_alt_path(req_msg)) {
 		alt_path->dgid = req_msg->alt_local_gid;
 		alt_path->sgid = req_msg->alt_remote_gid;
-		sa_path_set_dlid(alt_path,
-				 htonl(ntohs(req_msg->alt_local_lid)));
-		sa_path_set_slid(alt_path,
-				 htonl(ntohs(req_msg->alt_remote_lid)));
 		alt_path->flow_label = cm_req_get_alt_flow_label(req_msg);
 		alt_path->hop_limit = req_msg->alt_hop_limit;
 		alt_path->traffic_class = req_msg->alt_traffic_class;
@@ -1454,6 +1541,7 @@ static void cm_format_paths_from_req(struct cm_req_msg *req_msg,
 		alt_path->packet_life_time -= (alt_path->packet_life_time > 0);
 		alt_path->service_id = req_msg->service_id;
 	}
+	cm_format_path_lid_from_req(req_msg, primary_path, alt_path);
 }
 
 static u16 cm_get_bth_pkey(struct cm_work *work)
@@ -1703,7 +1791,7 @@ static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc)
 {
 	if (!cm_req_get_primary_subnet_local(req_msg)) {
 		if (req_msg->primary_local_lid == IB_LID_PERMISSIVE) {
-			req_msg->primary_local_lid = cpu_to_be16(wc->slid);
+			req_msg->primary_local_lid = ib_lid_be16(wc->slid);
 			cm_req_set_primary_sl(req_msg, wc->sl);
 		}
 
@@ -1713,7 +1801,7 @@ static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc)
 
 	if (!cm_req_get_alt_subnet_local(req_msg)) {
 		if (req_msg->alt_local_lid == IB_LID_PERMISSIVE) {
-			req_msg->alt_local_lid = cpu_to_be16(wc->slid);
+			req_msg->alt_local_lid = ib_lid_be16(wc->slid);
 			cm_req_set_alt_sl(req_msg, wc->sl);
 		}
 
@@ -1784,9 +1872,12 @@ static int cm_req_handler(struct cm_work *work)
 					 dev_net(gid_attr.ndev));
 			dev_put(gid_attr.ndev);
 		} else {
-			work->path[0].rec_type = SA_PATH_REC_TYPE_IB;
+			cm_path_set_rec_type(work->port->cm_dev->ib_device,
+					     work->port->port_num,
+					     &work->path[0],
+					     &req_msg->primary_local_gid);
 		}
-		if (req_msg->alt_local_lid)
+		if (cm_req_has_alt_path(req_msg))
 			work->path[1].rec_type = work->path[0].rec_type;
 		cm_format_paths_from_req(req_msg, &work->path[0],
 					 &work->path[1]);
@@ -1811,16 +1902,19 @@ static int cm_req_handler(struct cm_work *work)
 					 dev_net(gid_attr.ndev));
 			dev_put(gid_attr.ndev);
 		} else {
-			work->path[0].rec_type = SA_PATH_REC_TYPE_IB;
+			cm_path_set_rec_type(work->port->cm_dev->ib_device,
+					     work->port->port_num,
+					     &work->path[0],
+					     &req_msg->primary_local_gid);
 		}
-		if (req_msg->alt_local_lid)
+		if (cm_req_has_alt_path(req_msg))
 			work->path[1].rec_type = work->path[0].rec_type;
 		ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_GID,
 			       &work->path[0].sgid, sizeof work->path[0].sgid,
 			       NULL, 0);
 		goto rejected;
 	}
-	if (req_msg->alt_local_lid) {
+	if (cm_req_has_alt_path(req_msg)) {
 		ret = cm_init_av_by_path(&work->path[1], &cm_id_priv->alt_av,
 					 cm_id_priv);
 		if (ret) {
@@ -2424,7 +2518,8 @@ static int cm_dreq_handler(struct cm_work *work)
 	case IB_CM_TIMEWAIT:
 		atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
 				counter[CM_DREQ_COUNTER]);
-		if (cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg))
+		msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc);
+		if (IS_ERR(msg))
 			goto unlock;
 
 		cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
@@ -2432,7 +2527,8 @@ static int cm_dreq_handler(struct cm_work *work)
 			       cm_id_priv->private_data_len);
 		spin_unlock_irq(&cm_id_priv->lock);
 
-		if (ib_post_send_mad(msg, NULL))
+		if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
+		    ib_post_send_mad(msg, NULL))
 			cm_free_msg(msg);
 		goto deref;
 	case IB_CM_DREQ_RCVD:
@@ -2843,6 +2939,11 @@ static void cm_format_lap(struct cm_lap_msg *lap_msg,
 			  const void *private_data,
 			  u8 private_data_len)
 {
+	bool alt_ext = false;
+
+	if (alternate_path->rec_type == SA_PATH_REC_TYPE_OPA)
+		alt_ext = opa_is_extended_lid(alternate_path->opa.dlid,
+					      alternate_path->opa.slid);
 	cm_format_mad_hdr(&lap_msg->hdr, CM_LAP_ATTR_ID,
 			  cm_form_tid(cm_id_priv, CM_MSG_SEQUENCE_LAP));
 	lap_msg->local_comm_id = cm_id_priv->id.local_id;
@@ -2856,6 +2957,12 @@ static void cm_format_lap(struct cm_lap_msg *lap_msg,
 		htons(ntohl(sa_path_get_dlid(alternate_path)));
 	lap_msg->alt_local_gid = alternate_path->sgid;
 	lap_msg->alt_remote_gid = alternate_path->dgid;
+	if (alt_ext) {
+		lap_msg->alt_local_gid.global.interface_id
+			= OPA_MAKE_ID(be32_to_cpu(alternate_path->opa.slid));
+		lap_msg->alt_remote_gid.global.interface_id
+			= OPA_MAKE_ID(be32_to_cpu(alternate_path->opa.dlid));
+	}
 	cm_lap_set_flow_label(lap_msg, alternate_path->flow_label);
 	cm_lap_set_traffic_class(lap_msg, alternate_path->traffic_class);
 	lap_msg->alt_hop_limit = alternate_path->hop_limit;
@@ -2924,16 +3031,29 @@ out:	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 }
 EXPORT_SYMBOL(ib_send_cm_lap);
 
+static void cm_format_path_lid_from_lap(struct cm_lap_msg *lap_msg,
+					struct sa_path_rec *path)
+{
+	u32 lid;
+
+	if (path->rec_type != SA_PATH_REC_TYPE_OPA) {
+		sa_path_set_dlid(path, htonl(ntohs(lap_msg->alt_local_lid)));
+		sa_path_set_slid(path, htonl(ntohs(lap_msg->alt_remote_lid)));
+	} else {
+		lid = opa_get_lid_from_gid(&lap_msg->alt_local_gid);
+		sa_path_set_dlid(path, cpu_to_be32(lid));
+
+		lid = opa_get_lid_from_gid(&lap_msg->alt_remote_gid);
+		sa_path_set_slid(path, cpu_to_be32(lid));
+	}
+}
+
 static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv,
 				    struct sa_path_rec *path,
 				    struct cm_lap_msg *lap_msg)
 {
-	memset(path, 0, sizeof *path);
-	path->rec_type = SA_PATH_REC_TYPE_IB;
 	path->dgid = lap_msg->alt_local_gid;
 	path->sgid = lap_msg->alt_remote_gid;
-	sa_path_set_dlid(path, htonl(ntohs(lap_msg->alt_local_lid)));
-	sa_path_set_slid(path, htonl(ntohs(lap_msg->alt_remote_lid)));
 	path->flow_label = cm_lap_get_flow_label(lap_msg);
 	path->hop_limit = lap_msg->alt_hop_limit;
 	path->traffic_class = cm_lap_get_traffic_class(lap_msg);
@@ -2947,6 +3067,7 @@ static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv,
 	path->packet_life_time_selector = IB_SA_EQ;
 	path->packet_life_time = cm_lap_get_local_ack_timeout(lap_msg);
 	path->packet_life_time -= (path->packet_life_time > 0);
+	cm_format_path_lid_from_lap(lap_msg, path);
 }
 
 static int cm_lap_handler(struct cm_work *work)
@@ -2965,6 +3086,11 @@ static int cm_lap_handler(struct cm_work *work)
 		return -EINVAL;
 
 	param = &work->cm_event.param.lap_rcvd;
+	memset(&work->path[0], 0, sizeof(work->path[1]));
+	cm_path_set_rec_type(work->port->cm_dev->ib_device,
+			     work->port->port_num,
+			     &work->path[0],
+			     &lap_msg->alt_local_gid);
 	param->alternate_path = &work->path[0];
 	cm_format_path_from_lap(cm_id_priv, param->alternate_path, lap_msg);
 	work->cm_event.private_data = &lap_msg->private_data;
@@ -2980,7 +3106,8 @@ static int cm_lap_handler(struct cm_work *work)
 	case IB_CM_MRA_LAP_SENT:
 		atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
 				counter[CM_LAP_COUNTER]);
-		if (cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg))
+		msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc);
+		if (IS_ERR(msg))
 			goto unlock;
 
 		cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
@@ -2990,7 +3117,8 @@ static int cm_lap_handler(struct cm_work *work)
 			      cm_id_priv->private_data_len);
 		spin_unlock_irq(&cm_id_priv->lock);
 
-		if (ib_post_send_mad(msg, NULL))
+		if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
+		    ib_post_send_mad(msg, NULL))
 			cm_free_msg(msg);
 		goto deref;
 	case IB_CM_LAP_RCVD:
@@ -4201,7 +4329,7 @@ static int __init ib_cm_init(void)
 		goto error1;
 	}
 
-	cm.wq = create_workqueue("ib_cm");
+	cm.wq = alloc_workqueue("ib_cm", 0, 1);
 	if (!cm.wq) {
 		ret = -ENOMEM;
 		goto error2;
diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 0eb393237ba2..852c8fec8088 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -72,6 +72,7 @@ MODULE_LICENSE("Dual BSD/GPL");
 #define CMA_MAX_CM_RETRIES 15
 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
 #define CMA_IBOE_PACKET_LIFETIME 18
+#define CMA_PREFERRED_ROCE_GID_TYPE IB_GID_TYPE_ROCE_UDP_ENCAP
 
 static const char * const cma_events[] = {
 	[RDMA_CM_EVENT_ADDR_RESOLVED]	 = "address resolved",
@@ -3998,7 +3999,8 @@ static void iboe_mcast_work_handler(struct work_struct *work)
 	kfree(mw);
 }
 
-static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid)
+static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid,
+			      enum ib_gid_type gid_type)
 {
 	struct sockaddr_in *sin = (struct sockaddr_in *)addr;
 	struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr;
@@ -4008,8 +4010,8 @@ static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid)
 	} else if (addr->sa_family == AF_INET6) {
 		memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
 	} else {
-		mgid->raw[0] = 0xff;
-		mgid->raw[1] = 0x0e;
+		mgid->raw[0] = (gid_type == IB_GID_TYPE_IB) ? 0xff : 0;
+		mgid->raw[1] = (gid_type == IB_GID_TYPE_IB) ? 0x0e : 0;
 		mgid->raw[2] = 0;
 		mgid->raw[3] = 0;
 		mgid->raw[4] = 0;
@@ -4050,7 +4052,9 @@ static int cma_iboe_join_multicast(struct rdma_id_private *id_priv,
 		goto out1;
 	}
 
-	cma_iboe_set_mgid(addr, &mc->multicast.ib->rec.mgid);
+	gid_type = id_priv->cma_dev->default_gid_type[id_priv->id.port_num -
+		   rdma_start_port(id_priv->cma_dev->device)];
+	cma_iboe_set_mgid(addr, &mc->multicast.ib->rec.mgid, gid_type);
 
 	mc->multicast.ib->rec.pkey = cpu_to_be16(0xffff);
 	if (id_priv->id.ps == RDMA_PS_UDP)
@@ -4066,8 +4070,6 @@ static int cma_iboe_join_multicast(struct rdma_id_private *id_priv,
 	mc->multicast.ib->rec.hop_limit = 1;
 	mc->multicast.ib->rec.mtu = iboe_get_mtu(ndev->mtu);
 
-	gid_type = id_priv->cma_dev->default_gid_type[id_priv->id.port_num -
-		   rdma_start_port(id_priv->cma_dev->device)];
 	if (addr->sa_family == AF_INET) {
 		if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
 			mc->multicast.ib->rec.hop_limit = IPV6_DEFAULT_HOPLIMIT;
@@ -4280,8 +4282,12 @@ static void cma_add_one(struct ib_device *device)
 	for (i = rdma_start_port(device); i <= rdma_end_port(device); i++) {
 		supported_gids = roce_gid_type_mask_support(device, i);
 		WARN_ON(!supported_gids);
-		cma_dev->default_gid_type[i - rdma_start_port(device)] =
-			find_first_bit(&supported_gids, BITS_PER_LONG);
+		if (supported_gids & (1 << CMA_PREFERRED_ROCE_GID_TYPE))
+			cma_dev->default_gid_type[i - rdma_start_port(device)] =
+				CMA_PREFERRED_ROCE_GID_TYPE;
+		else
+			cma_dev->default_gid_type[i - rdma_start_port(device)] =
+				find_first_bit(&supported_gids, BITS_PER_LONG);
 		cma_dev->default_roce_tos[i - rdma_start_port(device)] = 0;
 	}
 
@@ -4452,9 +4458,8 @@ out:
 	return skb->len;
 }
 
-static const struct ibnl_client_cbs cma_cb_table[] = {
-	[RDMA_NL_RDMA_CM_ID_STATS] = { .dump = cma_get_id_stats,
-				       .module = THIS_MODULE },
+static const struct rdma_nl_cbs cma_cb_table[] = {
+	[RDMA_NL_RDMA_CM_ID_STATS] = { .dump = cma_get_id_stats},
 };
 
 static int cma_init_net(struct net *net)
@@ -4506,9 +4511,7 @@ static int __init cma_init(void)
 	if (ret)
 		goto err;
 
-	if (ibnl_add_client(RDMA_NL_RDMA_CM, ARRAY_SIZE(cma_cb_table),
-			    cma_cb_table))
-		pr_warn("RDMA CMA: failed to add netlink callback\n");
+	rdma_nl_register(RDMA_NL_RDMA_CM, cma_cb_table);
 	cma_configfs_init();
 
 	return 0;
@@ -4525,7 +4528,7 @@ err_wq:
 static void __exit cma_cleanup(void)
 {
 	cma_configfs_exit();
-	ibnl_remove_client(RDMA_NL_RDMA_CM);
+	rdma_nl_unregister(RDMA_NL_RDMA_CM);
 	ib_unregister_client(&cma_client);
 	unregister_netdevice_notifier(&cma_nb);
 	rdma_addr_unregister_client(&addr_client);
@@ -4534,5 +4537,7 @@ static void __exit cma_cleanup(void)
 	destroy_workqueue(cma_wq);
 }
 
+MODULE_ALIAS_RDMA_NETLINK(RDMA_NL_RDMA_CM, 1);
+
 module_init(cma_init);
 module_exit(cma_cleanup);
diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h
index 11ae67514e13..a1d687a664f8 100644
--- a/drivers/infiniband/core/core_priv.h
+++ b/drivers/infiniband/core/core_priv.h
@@ -38,6 +38,7 @@
 #include <linux/cgroup_rdma.h>
 
 #include <rdma/ib_verbs.h>
+#include <rdma/opa_addr.h>
 #include <rdma/ib_mad.h>
 #include "mad_priv.h"
 
@@ -102,6 +103,14 @@ void ib_enum_all_roce_netdevs(roce_netdev_filter filter,
 			      roce_netdev_callback cb,
 			      void *cookie);
 
+typedef int (*nldev_callback)(struct ib_device *device,
+			      struct sk_buff *skb,
+			      struct netlink_callback *cb,
+			      unsigned int idx);
+
+int ib_enum_all_devs(nldev_callback nldev_cb, struct sk_buff *skb,
+		     struct netlink_callback *cb);
+
 enum ib_cache_gid_default_mode {
 	IB_CACHE_GID_DEFAULT_MODE_SET,
 	IB_CACHE_GID_DEFAULT_MODE_DELETE
@@ -179,8 +188,8 @@ void ib_mad_cleanup(void);
 int ib_sa_init(void);
 void ib_sa_cleanup(void);
 
-int ibnl_init(void);
-void ibnl_cleanup(void);
+int rdma_nl_init(void);
+void rdma_nl_exit(void);
 
 /**
  * Check if there are any listeners to the netlink group
@@ -190,11 +199,14 @@ void ibnl_cleanup(void);
 int ibnl_chk_listeners(unsigned int group);
 
 int ib_nl_handle_resolve_resp(struct sk_buff *skb,
-			      struct netlink_callback *cb);
+			      struct nlmsghdr *nlh,
+			      struct netlink_ext_ack *extack);
 int ib_nl_handle_set_timeout(struct sk_buff *skb,
-			     struct netlink_callback *cb);
+			     struct nlmsghdr *nlh,
+			     struct netlink_ext_ack *extack);
 int ib_nl_handle_ip_res_resp(struct sk_buff *skb,
-			     struct netlink_callback *cb);
+			     struct nlmsghdr *nlh,
+			     struct netlink_ext_ack *extack);
 
 int ib_get_cached_subnet_prefix(struct ib_device *device,
 				u8                port_num,
@@ -301,4 +313,9 @@ static inline int ib_mad_enforce_security(struct ib_mad_agent_private *map,
 	return 0;
 }
 #endif
+
+struct ib_device *__ib_device_get_by_index(u32 ifindex);
+/* RDMA device netlink */
+void nldev_init(void);
+void nldev_exit(void);
 #endif /* _CORE_PRIV_H */
diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c
index a5dfab6adf49..84fc32a2c8b3 100644
--- a/drivers/infiniband/core/device.c
+++ b/drivers/infiniband/core/device.c
@@ -134,6 +134,17 @@ static int ib_device_check_mandatory(struct ib_device *device)
 	return 0;
 }
 
+struct ib_device *__ib_device_get_by_index(u32 index)
+{
+	struct ib_device *device;
+
+	list_for_each_entry(device, &device_list, core_list)
+		if (device->index == index)
+			return device;
+
+	return NULL;
+}
+
 static struct ib_device *__ib_device_get_by_name(const char *name)
 {
 	struct ib_device *device;
@@ -145,7 +156,6 @@ static struct ib_device *__ib_device_get_by_name(const char *name)
 	return NULL;
 }
 
-
 static int alloc_name(char *name)
 {
 	unsigned long *inuse;
@@ -326,10 +336,10 @@ static int read_port_immutable(struct ib_device *device)
 	return 0;
 }
 
-void ib_get_device_fw_str(struct ib_device *dev, char *str, size_t str_len)
+void ib_get_device_fw_str(struct ib_device *dev, char *str)
 {
 	if (dev->get_dev_fw_str)
-		dev->get_dev_fw_str(dev, str, str_len);
+		dev->get_dev_fw_str(dev, str);
 	else
 		str[0] = '\0';
 }
@@ -395,6 +405,30 @@ static int ib_security_change(struct notifier_block *nb, unsigned long event,
 }
 
 /**
+ *	__dev_new_index	-	allocate an device index
+ *
+ *	Returns a suitable unique value for a new device interface
+ *	number.  It assumes that there are less than 2^32-1 ib devices
+ *	will be present in the system.
+ */
+static u32 __dev_new_index(void)
+{
+	/*
+	 * The device index to allow stable naming.
+	 * Similar to struct net -> ifindex.
+	 */
+	static u32 index;
+
+	for (;;) {
+		if (!(++index))
+			index = 1;
+
+		if (!__ib_device_get_by_index(index))
+			return index;
+	}
+}
+
+/**
  * ib_register_device - Register an IB device with IB core
  * @device:Device to register
  *
@@ -489,9 +523,10 @@ int ib_register_device(struct ib_device *device,
 	device->reg_state = IB_DEV_REGISTERED;
 
 	list_for_each_entry(client, &client_list, list)
-		if (client->add && !add_client_context(device, client))
+		if (!add_client_context(device, client) && client->add)
 			client->add(device);
 
+	device->index = __dev_new_index();
 	down_write(&lists_rwsem);
 	list_add_tail(&device->core_list, &device_list);
 	up_write(&lists_rwsem);
@@ -537,10 +572,11 @@ void ib_unregister_device(struct ib_device *device)
 	}
 	up_read(&lists_rwsem);
 
-	mutex_unlock(&device_mutex);
-
 	ib_device_unregister_rdmacg(device);
 	ib_device_unregister_sysfs(device);
+
+	mutex_unlock(&device_mutex);
+
 	ib_cache_cleanup_one(device);
 
 	ib_security_destroy_port_pkey_list(device);
@@ -577,7 +613,7 @@ int ib_register_client(struct ib_client *client)
 	mutex_lock(&device_mutex);
 
 	list_for_each_entry(device, &device_list, core_list)
-		if (client->add && !add_client_context(device, client))
+		if (!add_client_context(device, client) && client->add)
 			client->add(device);
 
 	down_write(&lists_rwsem);
@@ -711,7 +747,7 @@ EXPORT_SYMBOL(ib_set_client_data);
  * chapter 11 of the InfiniBand Architecture Specification).  This
  * callback may occur in interrupt context.
  */
-int ib_register_event_handler  (struct ib_event_handler *event_handler)
+void ib_register_event_handler(struct ib_event_handler *event_handler)
 {
 	unsigned long flags;
 
@@ -719,8 +755,6 @@ int ib_register_event_handler  (struct ib_event_handler *event_handler)
 	list_add_tail(&event_handler->list,
 		      &event_handler->device->event_handler_list);
 	spin_unlock_irqrestore(&event_handler->device->event_handler_lock, flags);
-
-	return 0;
 }
 EXPORT_SYMBOL(ib_register_event_handler);
 
@@ -731,15 +765,13 @@ EXPORT_SYMBOL(ib_register_event_handler);
  * Unregister an event handler registered with
  * ib_register_event_handler().
  */
-int ib_unregister_event_handler(struct ib_event_handler *event_handler)
+void ib_unregister_event_handler(struct ib_event_handler *event_handler)
 {
 	unsigned long flags;
 
 	spin_lock_irqsave(&event_handler->device->event_handler_lock, flags);
 	list_del(&event_handler->list);
 	spin_unlock_irqrestore(&event_handler->device->event_handler_lock, flags);
-
-	return 0;
 }
 EXPORT_SYMBOL(ib_unregister_event_handler);
 
@@ -893,6 +925,31 @@ void ib_enum_all_roce_netdevs(roce_netdev_filter filter,
 }
 
 /**
+ * ib_enum_all_devs - enumerate all ib_devices
+ * @cb: Callback to call for each found ib_device
+ *
+ * Enumerates all ib_devices and calls callback() on each device.
+ */
+int ib_enum_all_devs(nldev_callback nldev_cb, struct sk_buff *skb,
+		     struct netlink_callback *cb)
+{
+	struct ib_device *dev;
+	unsigned int idx = 0;
+	int ret = 0;
+
+	down_read(&lists_rwsem);
+	list_for_each_entry(dev, &device_list, core_list) {
+		ret = nldev_cb(dev, skb, cb, idx);
+		if (ret)
+			break;
+		idx++;
+	}
+
+	up_read(&lists_rwsem);
+	return ret;
+}
+
+/**
  * ib_query_pkey - Get P_Key table entry
  * @device:Device to query
  * @port_num:Port number to query
@@ -944,14 +1001,17 @@ int ib_modify_port(struct ib_device *device,
 		   u8 port_num, int port_modify_mask,
 		   struct ib_port_modify *port_modify)
 {
-	if (!device->modify_port)
-		return -ENOSYS;
+	int rc;
 
 	if (!rdma_is_port_valid(device, port_num))
 		return -EINVAL;
 
-	return device->modify_port(device, port_num, port_modify_mask,
-				   port_modify);
+	if (device->modify_port)
+		rc = device->modify_port(device, port_num, port_modify_mask,
+					   port_modify);
+	else
+		rc = rdma_protocol_roce(device, port_num) ? 0 : -ENOSYS;
+	return rc;
 }
 EXPORT_SYMBOL(ib_modify_port);
 
@@ -1086,29 +1146,21 @@ struct net_device *ib_get_net_dev_by_params(struct ib_device *dev,
 }
 EXPORT_SYMBOL(ib_get_net_dev_by_params);
 
-static struct ibnl_client_cbs ibnl_ls_cb_table[] = {
+static const struct rdma_nl_cbs ibnl_ls_cb_table[] = {
 	[RDMA_NL_LS_OP_RESOLVE] = {
-		.dump = ib_nl_handle_resolve_resp,
-		.module = THIS_MODULE },
+		.doit = ib_nl_handle_resolve_resp,
+		.flags = RDMA_NL_ADMIN_PERM,
+	},
 	[RDMA_NL_LS_OP_SET_TIMEOUT] = {
-		.dump = ib_nl_handle_set_timeout,
-		.module = THIS_MODULE },
+		.doit = ib_nl_handle_set_timeout,
+		.flags = RDMA_NL_ADMIN_PERM,
+	},
 	[RDMA_NL_LS_OP_IP_RESOLVE] = {
-		.dump = ib_nl_handle_ip_res_resp,
-		.module = THIS_MODULE },
+		.doit = ib_nl_handle_ip_res_resp,
+		.flags = RDMA_NL_ADMIN_PERM,
+	},
 };
 
-static int ib_add_ibnl_clients(void)
-{
-	return ibnl_add_client(RDMA_NL_LS, ARRAY_SIZE(ibnl_ls_cb_table),
-			       ibnl_ls_cb_table);
-}
-
-static void ib_remove_ibnl_clients(void)
-{
-	ibnl_remove_client(RDMA_NL_LS);
-}
-
 static int __init ib_core_init(void)
 {
 	int ret;
@@ -1130,9 +1182,9 @@ static int __init ib_core_init(void)
 		goto err_comp;
 	}
 
-	ret = ibnl_init();
+	ret = rdma_nl_init();
 	if (ret) {
-		pr_warn("Couldn't init IB netlink interface\n");
+		pr_warn("Couldn't init IB netlink interface: err %d\n", ret);
 		goto err_sysfs;
 	}
 
@@ -1154,24 +1206,18 @@ static int __init ib_core_init(void)
 		goto err_mad;
 	}
 
-	ret = ib_add_ibnl_clients();
-	if (ret) {
-		pr_warn("Couldn't register ibnl clients\n");
-		goto err_sa;
-	}
-
 	ret = register_lsm_notifier(&ibdev_lsm_nb);
 	if (ret) {
 		pr_warn("Couldn't register LSM notifier. ret %d\n", ret);
-		goto err_ibnl_clients;
+		goto err_sa;
 	}
 
+	nldev_init();
+	rdma_nl_register(RDMA_NL_LS, ibnl_ls_cb_table);
 	ib_cache_setup();
 
 	return 0;
 
-err_ibnl_clients:
-	ib_remove_ibnl_clients();
 err_sa:
 	ib_sa_cleanup();
 err_mad:
@@ -1179,7 +1225,7 @@ err_mad:
 err_addr:
 	addr_cleanup();
 err_ibnl:
-	ibnl_cleanup();
+	rdma_nl_exit();
 err_sysfs:
 	class_unregister(&ib_class);
 err_comp:
@@ -1191,18 +1237,21 @@ err:
 
 static void __exit ib_core_cleanup(void)
 {
-	unregister_lsm_notifier(&ibdev_lsm_nb);
 	ib_cache_cleanup();
-	ib_remove_ibnl_clients();
+	nldev_exit();
+	rdma_nl_unregister(RDMA_NL_LS);
+	unregister_lsm_notifier(&ibdev_lsm_nb);
 	ib_sa_cleanup();
 	ib_mad_cleanup();
 	addr_cleanup();
-	ibnl_cleanup();
+	rdma_nl_exit();
 	class_unregister(&ib_class);
 	destroy_workqueue(ib_comp_wq);
 	/* Make sure that any pending umem accounting work is done. */
 	destroy_workqueue(ib_wq);
 }
 
+MODULE_ALIAS_RDMA_NETLINK(RDMA_NL_LS, 4);
+
 module_init(ib_core_init);
 module_exit(ib_core_cleanup);
diff --git a/drivers/infiniband/core/iwcm.c b/drivers/infiniband/core/iwcm.c
index 31661b5c1743..fcf42f6bb82a 100644
--- a/drivers/infiniband/core/iwcm.c
+++ b/drivers/infiniband/core/iwcm.c
@@ -80,7 +80,7 @@ const char *__attribute_const__ iwcm_reject_msg(int reason)
 }
 EXPORT_SYMBOL(iwcm_reject_msg);
 
-static struct ibnl_client_cbs iwcm_nl_cb_table[] = {
+static struct rdma_nl_cbs iwcm_nl_cb_table[] = {
 	[RDMA_NL_IWPM_REG_PID] = {.dump = iwpm_register_pid_cb},
 	[RDMA_NL_IWPM_ADD_MAPPING] = {.dump = iwpm_add_mapping_cb},
 	[RDMA_NL_IWPM_QUERY_MAPPING] = {.dump = iwpm_add_and_query_mapping_cb},
@@ -1175,13 +1175,9 @@ static int __init iw_cm_init(void)
 	ret = iwpm_init(RDMA_NL_IWCM);
 	if (ret)
 		pr_err("iw_cm: couldn't init iwpm\n");
-
-	ret = ibnl_add_client(RDMA_NL_IWCM, ARRAY_SIZE(iwcm_nl_cb_table),
-			      iwcm_nl_cb_table);
-	if (ret)
-		pr_err("iw_cm: couldn't register netlink callbacks\n");
-
-	iwcm_wq = alloc_ordered_workqueue("iw_cm_wq", WQ_MEM_RECLAIM);
+	else
+		rdma_nl_register(RDMA_NL_IWCM, iwcm_nl_cb_table);
+	iwcm_wq = alloc_ordered_workqueue("iw_cm_wq", 0);
 	if (!iwcm_wq)
 		return -ENOMEM;
 
@@ -1200,9 +1196,11 @@ static void __exit iw_cm_cleanup(void)
 {
 	unregister_net_sysctl_table(iwcm_ctl_table_hdr);
 	destroy_workqueue(iwcm_wq);
-	ibnl_remove_client(RDMA_NL_IWCM);
+	rdma_nl_unregister(RDMA_NL_IWCM);
 	iwpm_exit(RDMA_NL_IWCM);
 }
 
+MODULE_ALIAS_RDMA_NETLINK(RDMA_NL_IWCM, 2);
+
 module_init(iw_cm_init);
 module_exit(iw_cm_cleanup);
diff --git a/drivers/infiniband/core/iwpm_msg.c b/drivers/infiniband/core/iwpm_msg.c
index a0e7c16d8bd8..30825bb9b8e9 100644
--- a/drivers/infiniband/core/iwpm_msg.c
+++ b/drivers/infiniband/core/iwpm_msg.c
@@ -42,7 +42,6 @@ int iwpm_valid_pid(void)
 {
 	return iwpm_user_pid > 0;
 }
-EXPORT_SYMBOL(iwpm_valid_pid);
 
 /*
  * iwpm_register_pid - Send a netlink query to user space
@@ -104,7 +103,7 @@ int iwpm_register_pid(struct iwpm_dev_data *pm_msg, u8 nl_client)
 	pr_debug("%s: Multicasting a nlmsg (dev = %s ifname = %s iwpm = %s)\n",
 		__func__, pm_msg->dev_name, pm_msg->if_name, iwpm_ulib_name);
 
-	ret = ibnl_multicast(skb, nlh, RDMA_NL_GROUP_IWPM, GFP_KERNEL);
+	ret = rdma_nl_multicast(skb, RDMA_NL_GROUP_IWPM, GFP_KERNEL);
 	if (ret) {
 		skb = NULL; /* skb is freed in the netlink send-op handling */
 		iwpm_user_pid = IWPM_PID_UNAVAILABLE;
@@ -122,7 +121,6 @@ pid_query_error:
 		iwpm_free_nlmsg_request(&nlmsg_request->kref);
 	return ret;
 }
-EXPORT_SYMBOL(iwpm_register_pid);
 
 /*
  * iwpm_add_mapping - Send a netlink add mapping message
@@ -174,7 +172,7 @@ int iwpm_add_mapping(struct iwpm_sa_data *pm_msg, u8 nl_client)
 		goto add_mapping_error;
 	nlmsg_request->req_buffer = pm_msg;
 
-	ret = ibnl_unicast(skb, nlh, iwpm_user_pid);
+	ret = rdma_nl_unicast_wait(skb, iwpm_user_pid);
 	if (ret) {
 		skb = NULL; /* skb is freed in the netlink send-op handling */
 		iwpm_user_pid = IWPM_PID_UNDEFINED;
@@ -191,7 +189,6 @@ add_mapping_error:
 		iwpm_free_nlmsg_request(&nlmsg_request->kref);
 	return ret;
 }
-EXPORT_SYMBOL(iwpm_add_mapping);
 
 /*
  * iwpm_add_and_query_mapping - Send a netlink add and query
@@ -251,7 +248,7 @@ int iwpm_add_and_query_mapping(struct iwpm_sa_data *pm_msg, u8 nl_client)
 		goto query_mapping_error;
 	nlmsg_request->req_buffer = pm_msg;
 
-	ret = ibnl_unicast(skb, nlh, iwpm_user_pid);
+	ret = rdma_nl_unicast_wait(skb, iwpm_user_pid);
 	if (ret) {
 		skb = NULL; /* skb is freed in the netlink send-op handling */
 		err_str = "Unable to send a nlmsg";
@@ -267,7 +264,6 @@ query_mapping_error:
 		iwpm_free_nlmsg_request(&nlmsg_request->kref);
 	return ret;
 }
-EXPORT_SYMBOL(iwpm_add_and_query_mapping);
 
 /*
  * iwpm_remove_mapping - Send a netlink remove mapping message
@@ -312,7 +308,7 @@ int iwpm_remove_mapping(struct sockaddr_storage *local_addr, u8 nl_client)
 	if (ret)
 		goto remove_mapping_error;
 
-	ret = ibnl_unicast(skb, nlh, iwpm_user_pid);
+	ret = rdma_nl_unicast_wait(skb, iwpm_user_pid);
 	if (ret) {
 		skb = NULL; /* skb is freed in the netlink send-op handling */
 		iwpm_user_pid = IWPM_PID_UNDEFINED;
@@ -328,7 +324,6 @@ remove_mapping_error:
 		dev_kfree_skb_any(skb);
 	return ret;
 }
-EXPORT_SYMBOL(iwpm_remove_mapping);
 
 /* netlink attribute policy for the received response to register pid request */
 static const struct nla_policy resp_reg_policy[IWPM_NLA_RREG_PID_MAX] = {
@@ -397,7 +392,6 @@ register_pid_response_exit:
 	up(&nlmsg_request->sem);
 	return 0;
 }
-EXPORT_SYMBOL(iwpm_register_pid_cb);
 
 /* netlink attribute policy for the received response to add mapping request */
 static const struct nla_policy resp_add_policy[IWPM_NLA_RMANAGE_MAPPING_MAX] = {
@@ -466,7 +460,6 @@ add_mapping_response_exit:
 	up(&nlmsg_request->sem);
 	return 0;
 }
-EXPORT_SYMBOL(iwpm_add_mapping_cb);
 
 /* netlink attribute policy for the response to add and query mapping request
  * and response with remote address info */
@@ -558,7 +551,6 @@ query_mapping_response_exit:
 	up(&nlmsg_request->sem);
 	return 0;
 }
-EXPORT_SYMBOL(iwpm_add_and_query_mapping_cb);
 
 /*
  * iwpm_remote_info_cb - Process a port mapper message, containing
@@ -627,7 +619,6 @@ int iwpm_remote_info_cb(struct sk_buff *skb, struct netlink_callback *cb)
 			"remote_info: Mapped remote sockaddr:");
 	return ret;
 }
-EXPORT_SYMBOL(iwpm_remote_info_cb);
 
 /* netlink attribute policy for the received request for mapping info */
 static const struct nla_policy resp_mapinfo_policy[IWPM_NLA_MAPINFO_REQ_MAX] = {
@@ -677,7 +668,6 @@ int iwpm_mapping_info_cb(struct sk_buff *skb, struct netlink_callback *cb)
 	ret = iwpm_send_mapinfo(nl_client, iwpm_user_pid);
 	return ret;
 }
-EXPORT_SYMBOL(iwpm_mapping_info_cb);
 
 /* netlink attribute policy for the received mapping info ack */
 static const struct nla_policy ack_mapinfo_policy[IWPM_NLA_MAPINFO_NUM_MAX] = {
@@ -707,7 +697,6 @@ int iwpm_ack_mapping_info_cb(struct sk_buff *skb, struct netlink_callback *cb)
 	atomic_set(&echo_nlmsg_seq, cb->nlh->nlmsg_seq);
 	return 0;
 }
-EXPORT_SYMBOL(iwpm_ack_mapping_info_cb);
 
 /* netlink attribute policy for the received port mapper error message */
 static const struct nla_policy map_error_policy[IWPM_NLA_ERR_MAX] = {
@@ -751,4 +740,3 @@ int iwpm_mapping_error_cb(struct sk_buff *skb, struct netlink_callback *cb)
 	up(&nlmsg_request->sem);
 	return 0;
 }
-EXPORT_SYMBOL(iwpm_mapping_error_cb);
diff --git a/drivers/infiniband/core/iwpm_util.c b/drivers/infiniband/core/iwpm_util.c
index f13870e69ccd..c81c55942626 100644
--- a/drivers/infiniband/core/iwpm_util.c
+++ b/drivers/infiniband/core/iwpm_util.c
@@ -54,8 +54,6 @@ static struct iwpm_admin_data iwpm_admin;
 int iwpm_init(u8 nl_client)
 {
 	int ret = 0;
-	if (iwpm_valid_client(nl_client))
-		return -EINVAL;
 	mutex_lock(&iwpm_admin_lock);
 	if (atomic_read(&iwpm_admin.refcount) == 0) {
 		iwpm_hash_bucket = kzalloc(IWPM_MAPINFO_HASH_SIZE *
@@ -83,7 +81,6 @@ init_exit:
 	}
 	return ret;
 }
-EXPORT_SYMBOL(iwpm_init);
 
 static void free_hash_bucket(void);
 static void free_reminfo_bucket(void);
@@ -109,7 +106,6 @@ int iwpm_exit(u8 nl_client)
 	iwpm_set_registration(nl_client, IWPM_REG_UNDEF);
 	return 0;
 }
-EXPORT_SYMBOL(iwpm_exit);
 
 static struct hlist_head *get_mapinfo_hash_bucket(struct sockaddr_storage *,
 					       struct sockaddr_storage *);
@@ -148,7 +144,6 @@ int iwpm_create_mapinfo(struct sockaddr_storage *local_sockaddr,
 	spin_unlock_irqrestore(&iwpm_mapinfo_lock, flags);
 	return ret;
 }
-EXPORT_SYMBOL(iwpm_create_mapinfo);
 
 int iwpm_remove_mapinfo(struct sockaddr_storage *local_sockaddr,
 			struct sockaddr_storage *mapped_local_addr)
@@ -184,7 +179,6 @@ remove_mapinfo_exit:
 	spin_unlock_irqrestore(&iwpm_mapinfo_lock, flags);
 	return ret;
 }
-EXPORT_SYMBOL(iwpm_remove_mapinfo);
 
 static void free_hash_bucket(void)
 {
@@ -297,7 +291,6 @@ get_remote_info_exit:
 	spin_unlock_irqrestore(&iwpm_reminfo_lock, flags);
 	return ret;
 }
-EXPORT_SYMBOL(iwpm_get_remote_info);
 
 struct iwpm_nlmsg_request *iwpm_get_nlmsg_request(__u32 nlmsg_seq,
 					u8 nl_client, gfp_t gfp)
@@ -383,15 +376,11 @@ int iwpm_get_nlmsg_seq(void)
 
 int iwpm_valid_client(u8 nl_client)
 {
-	if (nl_client >= RDMA_NL_NUM_CLIENTS)
-		return 0;
 	return iwpm_admin.client_list[nl_client];
 }
 
 void iwpm_set_valid(u8 nl_client, int valid)
 {
-	if (nl_client >= RDMA_NL_NUM_CLIENTS)
-		return;
 	iwpm_admin.client_list[nl_client] = valid;
 }
 
@@ -608,7 +597,7 @@ static int send_mapinfo_num(u32 mapping_num, u8 nl_client, int iwpm_pid)
 				&mapping_num, IWPM_NLA_MAPINFO_SEND_NUM);
 	if (ret)
 		goto mapinfo_num_error;
-	ret = ibnl_unicast(skb, nlh, iwpm_pid);
+	ret = rdma_nl_unicast(skb, iwpm_pid);
 	if (ret) {
 		skb = NULL;
 		err_str = "Unable to send a nlmsg";
@@ -637,7 +626,7 @@ static int send_nlmsg_done(struct sk_buff *skb, u8 nl_client, int iwpm_pid)
 		return -ENOMEM;
 	}
 	nlh->nlmsg_type = NLMSG_DONE;
-	ret = ibnl_unicast(skb, (struct nlmsghdr *)skb->data, iwpm_pid);
+	ret = rdma_nl_unicast(skb, iwpm_pid);
 	if (ret)
 		pr_warn("%s Unable to send a nlmsg\n", __func__);
 	return ret;
diff --git a/drivers/infiniband/core/mad_rmpp.c b/drivers/infiniband/core/mad_rmpp.c
index 0d3cca0a8890..e5cf09c66fe6 100644
--- a/drivers/infiniband/core/mad_rmpp.c
+++ b/drivers/infiniband/core/mad_rmpp.c
@@ -64,7 +64,7 @@ struct mad_rmpp_recv {
 
 	__be64 tid;
 	u32 src_qp;
-	u16 slid;
+	u32 slid;
 	u8 mgmt_class;
 	u8 class_version;
 	u8 method;
diff --git a/drivers/infiniband/core/netlink.c b/drivers/infiniband/core/netlink.c
index 94931c474d41..b12e58787c3d 100644
--- a/drivers/infiniband/core/netlink.c
+++ b/drivers/infiniband/core/netlink.c
@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2017 Mellanox Technologies Inc.  All rights reserved.
  * Copyright (c) 2010 Voltaire Inc.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
@@ -37,239 +38,267 @@
 #include <net/net_namespace.h>
 #include <net/sock.h>
 #include <rdma/rdma_netlink.h>
+#include <linux/module.h>
 #include "core_priv.h"
 
-struct ibnl_client {
-	struct list_head		list;
-	int				index;
-	int				nops;
-	const struct ibnl_client_cbs   *cb_table;
-};
+#include "core_priv.h"
 
-static DEFINE_MUTEX(ibnl_mutex);
+static DEFINE_MUTEX(rdma_nl_mutex);
 static struct sock *nls;
-static LIST_HEAD(client_list);
+static struct {
+	const struct rdma_nl_cbs   *cb_table;
+} rdma_nl_types[RDMA_NL_NUM_CLIENTS];
 
-int ibnl_chk_listeners(unsigned int group)
+int rdma_nl_chk_listeners(unsigned int group)
 {
-	if (netlink_has_listeners(nls, group) == 0)
-		return -1;
-	return 0;
+	return (netlink_has_listeners(nls, group)) ? 0 : -1;
 }
+EXPORT_SYMBOL(rdma_nl_chk_listeners);
 
-int ibnl_add_client(int index, int nops,
-		    const struct ibnl_client_cbs cb_table[])
+static bool is_nl_msg_valid(unsigned int type, unsigned int op)
 {
-	struct ibnl_client *cur;
-	struct ibnl_client *nl_client;
+	static const unsigned int max_num_ops[RDMA_NL_NUM_CLIENTS] = {
+		[RDMA_NL_RDMA_CM] = RDMA_NL_RDMA_CM_NUM_OPS,
+		[RDMA_NL_IWCM] = RDMA_NL_IWPM_NUM_OPS,
+		[RDMA_NL_LS] = RDMA_NL_LS_NUM_OPS,
+		[RDMA_NL_NLDEV] = RDMA_NLDEV_NUM_OPS,
+	};
 
-	nl_client = kmalloc(sizeof *nl_client, GFP_KERNEL);
-	if (!nl_client)
-		return -ENOMEM;
+	/*
+	 * This BUILD_BUG_ON is intended to catch addition of new
+	 * RDMA netlink protocol without updating the array above.
+	 */
+	BUILD_BUG_ON(RDMA_NL_NUM_CLIENTS != 6);
 
-	nl_client->index	= index;
-	nl_client->nops		= nops;
-	nl_client->cb_table	= cb_table;
+	if (type >= RDMA_NL_NUM_CLIENTS)
+		return false;
 
-	mutex_lock(&ibnl_mutex);
+	return (op < max_num_ops[type]) ? true : false;
+}
 
-	list_for_each_entry(cur, &client_list, list) {
-		if (cur->index == index) {
-			pr_warn("Client for %d already exists\n", index);
-			mutex_unlock(&ibnl_mutex);
-			kfree(nl_client);
-			return -EINVAL;
-		}
+static bool is_nl_valid(unsigned int type, unsigned int op)
+{
+	const struct rdma_nl_cbs *cb_table;
+
+	if (!is_nl_msg_valid(type, op))
+		return false;
+
+	cb_table = rdma_nl_types[type].cb_table;
+#ifdef CONFIG_MODULES
+	if (!cb_table) {
+		mutex_unlock(&rdma_nl_mutex);
+		request_module("rdma-netlink-subsys-%d", type);
+		mutex_lock(&rdma_nl_mutex);
+		cb_table = rdma_nl_types[type].cb_table;
 	}
+#endif
 
-	list_add_tail(&nl_client->list, &client_list);
-
-	mutex_unlock(&ibnl_mutex);
-
-	return 0;
+	if (!cb_table || (!cb_table[op].dump && !cb_table[op].doit))
+		return false;
+	return true;
 }
-EXPORT_SYMBOL(ibnl_add_client);
 
-int ibnl_remove_client(int index)
+void rdma_nl_register(unsigned int index,
+		      const struct rdma_nl_cbs cb_table[])
 {
-	struct ibnl_client *cur, *next;
-
-	mutex_lock(&ibnl_mutex);
-	list_for_each_entry_safe(cur, next, &client_list, list) {
-		if (cur->index == index) {
-			list_del(&(cur->list));
-			mutex_unlock(&ibnl_mutex);
-			kfree(cur);
-			return 0;
-		}
+	mutex_lock(&rdma_nl_mutex);
+	if (!is_nl_msg_valid(index, 0)) {
+		/*
+		 * All clients are not interesting in success/failure of
+		 * this call. They want to see the print to error log and
+		 * continue their initialization. Print warning for them,
+		 * because it is programmer's error to be here.
+		 */
+		mutex_unlock(&rdma_nl_mutex);
+		WARN(true,
+		     "The not-valid %u index was supplied to RDMA netlink\n",
+		     index);
+		return;
 	}
-	pr_warn("Can't remove callback for client idx %d. Not found\n", index);
-	mutex_unlock(&ibnl_mutex);
 
-	return -EINVAL;
+	if (rdma_nl_types[index].cb_table) {
+		mutex_unlock(&rdma_nl_mutex);
+		WARN(true,
+		     "The %u index is already registered in RDMA netlink\n",
+		     index);
+		return;
+	}
+
+	rdma_nl_types[index].cb_table = cb_table;
+	mutex_unlock(&rdma_nl_mutex);
+}
+EXPORT_SYMBOL(rdma_nl_register);
+
+void rdma_nl_unregister(unsigned int index)
+{
+	mutex_lock(&rdma_nl_mutex);
+	rdma_nl_types[index].cb_table = NULL;
+	mutex_unlock(&rdma_nl_mutex);
 }
-EXPORT_SYMBOL(ibnl_remove_client);
+EXPORT_SYMBOL(rdma_nl_unregister);
 
 void *ibnl_put_msg(struct sk_buff *skb, struct nlmsghdr **nlh, int seq,
 		   int len, int client, int op, int flags)
 {
-	unsigned char *prev_tail;
-
-	prev_tail = skb_tail_pointer(skb);
-	*nlh = nlmsg_put(skb, 0, seq, RDMA_NL_GET_TYPE(client, op),
-			 len, flags);
+	*nlh = nlmsg_put(skb, 0, seq, RDMA_NL_GET_TYPE(client, op), len, flags);
 	if (!*nlh)
-		goto out_nlmsg_trim;
-	(*nlh)->nlmsg_len = skb_tail_pointer(skb) - prev_tail;
+		return NULL;
 	return nlmsg_data(*nlh);
-
-out_nlmsg_trim:
-	nlmsg_trim(skb, prev_tail);
-	return NULL;
 }
 EXPORT_SYMBOL(ibnl_put_msg);
 
 int ibnl_put_attr(struct sk_buff *skb, struct nlmsghdr *nlh,
 		  int len, void *data, int type)
 {
-	unsigned char *prev_tail;
-
-	prev_tail = skb_tail_pointer(skb);
-	if (nla_put(skb, type, len, data))
-		goto nla_put_failure;
-	nlh->nlmsg_len += skb_tail_pointer(skb) - prev_tail;
+	if (nla_put(skb, type, len, data)) {
+		nlmsg_cancel(skb, nlh);
+		return -EMSGSIZE;
+	}
 	return 0;
-
-nla_put_failure:
-	nlmsg_trim(skb, prev_tail - nlh->nlmsg_len);
-	return -EMSGSIZE;
 }
 EXPORT_SYMBOL(ibnl_put_attr);
 
-static int ibnl_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh,
-			struct netlink_ext_ack *extack)
+static int rdma_nl_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh,
+			   struct netlink_ext_ack *extack)
 {
-	struct ibnl_client *client;
 	int type = nlh->nlmsg_type;
-	int index = RDMA_NL_GET_CLIENT(type);
+	unsigned int index = RDMA_NL_GET_CLIENT(type);
 	unsigned int op = RDMA_NL_GET_OP(type);
+	const struct rdma_nl_cbs *cb_table;
+
+	if (!is_nl_valid(index, op))
+		return -EINVAL;
+
+	cb_table = rdma_nl_types[index].cb_table;
 
-	list_for_each_entry(client, &client_list, list) {
-		if (client->index == index) {
-			if (op >= client->nops || !client->cb_table[op].dump)
-				return -EINVAL;
-
-			/*
-			 * For response or local service set_timeout request,
-			 * there is no need to use netlink_dump_start.
-			 */
-			if (!(nlh->nlmsg_flags & NLM_F_REQUEST) ||
-			    (index == RDMA_NL_LS &&
-			     op == RDMA_NL_LS_OP_SET_TIMEOUT)) {
-				struct netlink_callback cb = {
-					.skb = skb,
-					.nlh = nlh,
-					.dump = client->cb_table[op].dump,
-					.module = client->cb_table[op].module,
-				};
-
-				return cb.dump(skb, &cb);
-			}
-
-			{
-				struct netlink_dump_control c = {
-					.dump = client->cb_table[op].dump,
-					.module = client->cb_table[op].module,
-				};
-				return netlink_dump_start(nls, skb, nlh, &c);
-			}
-		}
+	if ((cb_table[op].flags & RDMA_NL_ADMIN_PERM) &&
+	    !netlink_capable(skb, CAP_NET_ADMIN))
+		return -EPERM;
+
+	/* FIXME: Convert IWCM to properly handle doit callbacks */
+	if ((nlh->nlmsg_flags & NLM_F_DUMP) || index == RDMA_NL_RDMA_CM ||
+	    index == RDMA_NL_IWCM) {
+		struct netlink_dump_control c = {
+			.dump = cb_table[op].dump,
+		};
+		return netlink_dump_start(nls, skb, nlh, &c);
 	}
 
-	pr_info("Index %d wasn't found in client list\n", index);
-	return -EINVAL;
+	if (cb_table[op].doit)
+		return cb_table[op].doit(skb, nlh, extack);
+
+	return 0;
 }
 
-static void ibnl_rcv_reply_skb(struct sk_buff *skb)
+/*
+ * This function is similar to netlink_rcv_skb with one exception:
+ * It calls to the callback for the netlink messages without NLM_F_REQUEST
+ * flag. These messages are intended for RDMA_NL_LS consumer, so it is allowed
+ * for that consumer only.
+ */
+static int rdma_nl_rcv_skb(struct sk_buff *skb, int (*cb)(struct sk_buff *,
+						   struct nlmsghdr *,
+						   struct netlink_ext_ack *))
 {
+	struct netlink_ext_ack extack = {};
 	struct nlmsghdr *nlh;
-	int msglen;
+	int err;
 
-	/*
-	 * Process responses until there is no more message or the first
-	 * request. Generally speaking, it is not recommended to mix responses
-	 * with requests.
-	 */
 	while (skb->len >= nlmsg_total_size(0)) {
+		int msglen;
+
 		nlh = nlmsg_hdr(skb);
+		err = 0;
 
 		if (nlh->nlmsg_len < NLMSG_HDRLEN || skb->len < nlh->nlmsg_len)
-			return;
-
-		/* Handle response only */
-		if (nlh->nlmsg_flags & NLM_F_REQUEST)
-			return;
-
-		ibnl_rcv_msg(skb, nlh, NULL);
+			return 0;
 
+		/*
+		 * Generally speaking, the only requests are handled
+		 * by the kernel, but RDMA_NL_LS is different, because it
+		 * runs backward netlink scheme. Kernel initiates messages
+		 * and waits for reply with data to keep pathrecord cache
+		 * in sync.
+		 */
+		if (!(nlh->nlmsg_flags & NLM_F_REQUEST) &&
+		    (RDMA_NL_GET_CLIENT(nlh->nlmsg_type) != RDMA_NL_LS))
+			goto ack;
+
+		/* Skip control messages */
+		if (nlh->nlmsg_type < NLMSG_MIN_TYPE)
+			goto ack;
+
+		err = cb(skb, nlh, &extack);
+		if (err == -EINTR)
+			goto skip;
+
+ack:
+		if (nlh->nlmsg_flags & NLM_F_ACK || err)
+			netlink_ack(skb, nlh, err, &extack);
+
+skip:
 		msglen = NLMSG_ALIGN(nlh->nlmsg_len);
 		if (msglen > skb->len)
 			msglen = skb->len;
 		skb_pull(skb, msglen);
 	}
+
+	return 0;
 }
 
-static void ibnl_rcv(struct sk_buff *skb)
+static void rdma_nl_rcv(struct sk_buff *skb)
 {
-	mutex_lock(&ibnl_mutex);
-	ibnl_rcv_reply_skb(skb);
-	netlink_rcv_skb(skb, &ibnl_rcv_msg);
-	mutex_unlock(&ibnl_mutex);
+	mutex_lock(&rdma_nl_mutex);
+	rdma_nl_rcv_skb(skb, &rdma_nl_rcv_msg);
+	mutex_unlock(&rdma_nl_mutex);
 }
 
-int ibnl_unicast(struct sk_buff *skb, struct nlmsghdr *nlh,
-			__u32 pid)
+int rdma_nl_unicast(struct sk_buff *skb, u32 pid)
+{
+	int err;
+
+	err = netlink_unicast(nls, skb, pid, MSG_DONTWAIT);
+	return (err < 0) ? err : 0;
+}
+EXPORT_SYMBOL(rdma_nl_unicast);
+
+int rdma_nl_unicast_wait(struct sk_buff *skb, __u32 pid)
 {
 	int err;
 
 	err = netlink_unicast(nls, skb, pid, 0);
 	return (err < 0) ? err : 0;
 }
-EXPORT_SYMBOL(ibnl_unicast);
+EXPORT_SYMBOL(rdma_nl_unicast_wait);
 
-int ibnl_multicast(struct sk_buff *skb, struct nlmsghdr *nlh,
-			unsigned int group, gfp_t flags)
+int rdma_nl_multicast(struct sk_buff *skb, unsigned int group, gfp_t flags)
 {
 	return nlmsg_multicast(nls, skb, 0, group, flags);
 }
-EXPORT_SYMBOL(ibnl_multicast);
+EXPORT_SYMBOL(rdma_nl_multicast);
 
-int __init ibnl_init(void)
+int __init rdma_nl_init(void)
 {
 	struct netlink_kernel_cfg cfg = {
-		.input	= ibnl_rcv,
+		.input	= rdma_nl_rcv,
 	};
 
 	nls = netlink_kernel_create(&init_net, NETLINK_RDMA, &cfg);
-	if (!nls) {
-		pr_warn("Failed to create netlink socket\n");
+	if (!nls)
 		return -ENOMEM;
-	}
 
 	nls->sk_sndtimeo = 10 * HZ;
 	return 0;
 }
 
-void ibnl_cleanup(void)
+void rdma_nl_exit(void)
 {
-	struct ibnl_client *cur, *next;
+	int idx;
 
-	mutex_lock(&ibnl_mutex);
-	list_for_each_entry_safe(cur, next, &client_list, list) {
-		list_del(&(cur->list));
-		kfree(cur);
-	}
-	mutex_unlock(&ibnl_mutex);
+	for (idx = 0; idx < RDMA_NL_NUM_CLIENTS; idx++)
+		rdma_nl_unregister(idx);
 
 	netlink_kernel_release(nls);
 }
+
+MODULE_ALIAS_NET_PF_PROTO(PF_NETLINK, NETLINK_RDMA);
diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c
new file mode 100644
index 000000000000..3ba24c428c3b
--- /dev/null
+++ b/drivers/infiniband/core/nldev.c
@@ -0,0 +1,325 @@
+/*
+ * Copyright (c) 2017 Mellanox Technologies. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the names of the copyright holders nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/module.h>
+#include <net/netlink.h>
+#include <rdma/rdma_netlink.h>
+
+#include "core_priv.h"
+
+static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = {
+	[RDMA_NLDEV_ATTR_DEV_INDEX]     = { .type = NLA_U32 },
+	[RDMA_NLDEV_ATTR_DEV_NAME]	= { .type = NLA_NUL_STRING,
+					    .len = IB_DEVICE_NAME_MAX - 1},
+	[RDMA_NLDEV_ATTR_PORT_INDEX]	= { .type = NLA_U32 },
+	[RDMA_NLDEV_ATTR_FW_VERSION]	= { .type = NLA_NUL_STRING,
+					    .len = IB_FW_VERSION_NAME_MAX - 1},
+	[RDMA_NLDEV_ATTR_NODE_GUID]	= { .type = NLA_U64 },
+	[RDMA_NLDEV_ATTR_SYS_IMAGE_GUID] = { .type = NLA_U64 },
+	[RDMA_NLDEV_ATTR_SUBNET_PREFIX]	= { .type = NLA_U64 },
+	[RDMA_NLDEV_ATTR_LID]		= { .type = NLA_U32 },
+	[RDMA_NLDEV_ATTR_SM_LID]	= { .type = NLA_U32 },
+	[RDMA_NLDEV_ATTR_LMC]		= { .type = NLA_U8 },
+	[RDMA_NLDEV_ATTR_PORT_STATE]	= { .type = NLA_U8 },
+	[RDMA_NLDEV_ATTR_PORT_PHYS_STATE] = { .type = NLA_U8 },
+	[RDMA_NLDEV_ATTR_DEV_NODE_TYPE] = { .type = NLA_U8 },
+};
+
+static int fill_dev_info(struct sk_buff *msg, struct ib_device *device)
+{
+	char fw[IB_FW_VERSION_NAME_MAX];
+
+	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_DEV_INDEX, device->index))
+		return -EMSGSIZE;
+	if (nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_NAME, device->name))
+		return -EMSGSIZE;
+	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, rdma_end_port(device)))
+		return -EMSGSIZE;
+
+	BUILD_BUG_ON(sizeof(device->attrs.device_cap_flags) != sizeof(u64));
+	if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_CAP_FLAGS,
+			      device->attrs.device_cap_flags, 0))
+		return -EMSGSIZE;
+
+	ib_get_device_fw_str(device, fw);
+	/* Device without FW has strlen(fw) */
+	if (strlen(fw) && nla_put_string(msg, RDMA_NLDEV_ATTR_FW_VERSION, fw))
+		return -EMSGSIZE;
+
+	if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_NODE_GUID,
+			      be64_to_cpu(device->node_guid), 0))
+		return -EMSGSIZE;
+	if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_SYS_IMAGE_GUID,
+			      be64_to_cpu(device->attrs.sys_image_guid), 0))
+		return -EMSGSIZE;
+	if (nla_put_u8(msg, RDMA_NLDEV_ATTR_DEV_NODE_TYPE, device->node_type))
+		return -EMSGSIZE;
+	return 0;
+}
+
+static int fill_port_info(struct sk_buff *msg,
+			  struct ib_device *device, u32 port)
+{
+	struct ib_port_attr attr;
+	int ret;
+
+	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_DEV_INDEX, device->index))
+		return -EMSGSIZE;
+	if (nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_NAME, device->name))
+		return -EMSGSIZE;
+	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port))
+		return -EMSGSIZE;
+
+	ret = ib_query_port(device, port, &attr);
+	if (ret)
+		return ret;
+
+	BUILD_BUG_ON(sizeof(attr.port_cap_flags) > sizeof(u64));
+	if (nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_CAP_FLAGS,
+			      (u64)attr.port_cap_flags, 0))
+		return -EMSGSIZE;
+	if (rdma_protocol_ib(device, port) &&
+	    nla_put_u64_64bit(msg, RDMA_NLDEV_ATTR_SUBNET_PREFIX,
+			      attr.subnet_prefix, 0))
+		return -EMSGSIZE;
+	if (rdma_protocol_ib(device, port)) {
+		if (nla_put_u32(msg, RDMA_NLDEV_ATTR_LID, attr.lid))
+			return -EMSGSIZE;
+		if (nla_put_u32(msg, RDMA_NLDEV_ATTR_SM_LID, attr.sm_lid))
+			return -EMSGSIZE;
+		if (nla_put_u8(msg, RDMA_NLDEV_ATTR_LMC, attr.lmc))
+			return -EMSGSIZE;
+	}
+	if (nla_put_u8(msg, RDMA_NLDEV_ATTR_PORT_STATE, attr.state))
+		return -EMSGSIZE;
+	if (nla_put_u8(msg, RDMA_NLDEV_ATTR_PORT_PHYS_STATE, attr.phys_state))
+		return -EMSGSIZE;
+	return 0;
+}
+
+static int nldev_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
+			  struct netlink_ext_ack *extack)
+{
+	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
+	struct ib_device *device;
+	struct sk_buff *msg;
+	u32 index;
+	int err;
+
+	err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
+			  nldev_policy, extack);
+	if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
+		return -EINVAL;
+
+	index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
+
+	device = __ib_device_get_by_index(index);
+	if (!device)
+		return -EINVAL;
+
+	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+	if (!msg)
+		return -ENOMEM;
+
+	nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
+			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET),
+			0, 0);
+
+	err = fill_dev_info(msg, device);
+	if (err) {
+		nlmsg_free(msg);
+		return err;
+	}
+
+	nlmsg_end(msg, nlh);
+
+	return rdma_nl_unicast(msg, NETLINK_CB(skb).portid);
+}
+
+static int _nldev_get_dumpit(struct ib_device *device,
+			     struct sk_buff *skb,
+			     struct netlink_callback *cb,
+			     unsigned int idx)
+{
+	int start = cb->args[0];
+	struct nlmsghdr *nlh;
+
+	if (idx < start)
+		return 0;
+
+	nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
+			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET),
+			0, NLM_F_MULTI);
+
+	if (fill_dev_info(skb, device)) {
+		nlmsg_cancel(skb, nlh);
+		goto out;
+	}
+
+	nlmsg_end(skb, nlh);
+
+	idx++;
+
+out:	cb->args[0] = idx;
+	return skb->len;
+}
+
+static int nldev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
+{
+	/*
+	 * There is no need to take lock, because
+	 * we are relying on ib_core's lists_rwsem
+	 */
+	return ib_enum_all_devs(_nldev_get_dumpit, skb, cb);
+}
+
+static int nldev_port_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
+			       struct netlink_ext_ack *extack)
+{
+	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
+	struct ib_device *device;
+	struct sk_buff *msg;
+	u32 index;
+	u32 port;
+	int err;
+
+	err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
+			  nldev_policy, extack);
+	if (err || !tb[RDMA_NLDEV_ATTR_PORT_INDEX])
+		return -EINVAL;
+
+	index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
+	device = __ib_device_get_by_index(index);
+	if (!device)
+		return -EINVAL;
+
+	port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
+	if (!rdma_is_port_valid(device, port))
+		return -EINVAL;
+
+	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+	if (!msg)
+		return -ENOMEM;
+
+	nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
+			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_GET),
+			0, 0);
+
+	err = fill_port_info(msg, device, port);
+	if (err) {
+		nlmsg_free(msg);
+		return err;
+	}
+
+	nlmsg_end(msg, nlh);
+
+	return rdma_nl_unicast(msg, NETLINK_CB(skb).portid);
+}
+
+static int nldev_port_get_dumpit(struct sk_buff *skb,
+				 struct netlink_callback *cb)
+{
+	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
+	struct ib_device *device;
+	int start = cb->args[0];
+	struct nlmsghdr *nlh;
+	u32 idx = 0;
+	u32 ifindex;
+	int err;
+	u32 p;
+
+	err = nlmsg_parse(cb->nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
+			  nldev_policy, NULL);
+	if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
+		return -EINVAL;
+
+	ifindex = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
+	device = __ib_device_get_by_index(ifindex);
+	if (!device)
+		return -EINVAL;
+
+	for (p = rdma_start_port(device); p <= rdma_end_port(device); ++p) {
+		/*
+		 * The dumpit function returns all information from specific
+		 * index. This specific index is taken from the netlink
+		 * messages request sent by user and it is available
+		 * in cb->args[0].
+		 *
+		 * Usually, the user doesn't fill this field and it causes
+		 * to return everything.
+		 *
+		 */
+		if (idx < start) {
+			idx++;
+			continue;
+		}
+
+		nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid,
+				cb->nlh->nlmsg_seq,
+				RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
+						 RDMA_NLDEV_CMD_PORT_GET),
+				0, NLM_F_MULTI);
+
+		if (fill_port_info(skb, device, p)) {
+			nlmsg_cancel(skb, nlh);
+			goto out;
+		}
+		idx++;
+		nlmsg_end(skb, nlh);
+	}
+
+out:	cb->args[0] = idx;
+	return skb->len;
+}
+
+static const struct rdma_nl_cbs nldev_cb_table[] = {
+	[RDMA_NLDEV_CMD_GET] = {
+		.doit = nldev_get_doit,
+		.dump = nldev_get_dumpit,
+	},
+	[RDMA_NLDEV_CMD_PORT_GET] = {
+		.doit = nldev_port_get_doit,
+		.dump = nldev_port_get_dumpit,
+	},
+};
+
+void __init nldev_init(void)
+{
+	rdma_nl_register(RDMA_NL_NLDEV, nldev_cb_table);
+}
+
+void __exit nldev_exit(void)
+{
+	rdma_nl_unregister(RDMA_NL_NLDEV);
+}
+
+MODULE_ALIAS_RDMA_NETLINK(RDMA_NL_NLDEV, 5);
diff --git a/drivers/infiniband/core/rdma_core.c b/drivers/infiniband/core/rdma_core.c
index 41c31a2bf093..85b5ee4defa4 100644
--- a/drivers/infiniband/core/rdma_core.c
+++ b/drivers/infiniband/core/rdma_core.c
@@ -35,10 +35,57 @@
 #include <rdma/ib_verbs.h>
 #include <rdma/uverbs_types.h>
 #include <linux/rcupdate.h>
+#include <rdma/uverbs_ioctl.h>
+#include <rdma/rdma_user_ioctl.h>
 #include "uverbs.h"
 #include "core_priv.h"
 #include "rdma_core.h"
 
+int uverbs_ns_idx(u16 *id, unsigned int ns_count)
+{
+	int ret = (*id & UVERBS_ID_NS_MASK) >> UVERBS_ID_NS_SHIFT;
+
+	if (ret >= ns_count)
+		return -EINVAL;
+
+	*id &= ~UVERBS_ID_NS_MASK;
+	return ret;
+}
+
+const struct uverbs_object_spec *uverbs_get_object(const struct ib_device *ibdev,
+						   uint16_t object)
+{
+	const struct uverbs_root_spec *object_hash = ibdev->specs_root;
+	const struct uverbs_object_spec_hash *objects;
+	int ret = uverbs_ns_idx(&object, object_hash->num_buckets);
+
+	if (ret < 0)
+		return NULL;
+
+	objects = object_hash->object_buckets[ret];
+
+	if (object >= objects->num_objects)
+		return NULL;
+
+	return objects->objects[object];
+}
+
+const struct uverbs_method_spec *uverbs_get_method(const struct uverbs_object_spec *object,
+						   uint16_t method)
+{
+	const struct uverbs_method_spec_hash *methods;
+	int ret = uverbs_ns_idx(&method, object->num_buckets);
+
+	if (ret < 0)
+		return NULL;
+
+	methods = object->method_buckets[ret];
+	if (method >= methods->num_methods)
+		return NULL;
+
+	return methods->methods[method];
+}
+
 void uverbs_uobject_get(struct ib_uobject *uobject)
 {
 	kref_get(&uobject->ref);
@@ -404,6 +451,41 @@ int __must_check rdma_remove_commit_uobject(struct ib_uobject *uobj)
 	return ret;
 }
 
+static int null_obj_type_class_remove_commit(struct ib_uobject *uobj,
+					     enum rdma_remove_reason why)
+{
+	return 0;
+}
+
+static const struct uverbs_obj_type null_obj_type = {
+	.type_class = &((const struct uverbs_obj_type_class){
+			.remove_commit = null_obj_type_class_remove_commit,
+			/* be cautious */
+			.needs_kfree_rcu = true}),
+};
+
+int rdma_explicit_destroy(struct ib_uobject *uobject)
+{
+	int ret;
+	struct ib_ucontext *ucontext = uobject->context;
+
+	/* Cleanup is running. Calling this should have been impossible */
+	if (!down_read_trylock(&ucontext->cleanup_rwsem)) {
+		WARN(true, "ib_uverbs: Cleanup is running while removing an uobject\n");
+		return 0;
+	}
+	lockdep_check(uobject, true);
+	ret = uobject->type->type_class->remove_commit(uobject,
+						       RDMA_REMOVE_DESTROY);
+	if (ret)
+		return ret;
+
+	uobject->type = &null_obj_type;
+
+	up_read(&ucontext->cleanup_rwsem);
+	return 0;
+}
+
 static void alloc_commit_idr_uobject(struct ib_uobject *uobj)
 {
 	uverbs_uobject_add(uobj);
@@ -625,3 +707,100 @@ const struct uverbs_obj_type_class uverbs_fd_class = {
 	.needs_kfree_rcu = false,
 };
 
+struct ib_uobject *uverbs_get_uobject_from_context(const struct uverbs_obj_type *type_attrs,
+						   struct ib_ucontext *ucontext,
+						   enum uverbs_obj_access access,
+						   int id)
+{
+	switch (access) {
+	case UVERBS_ACCESS_READ:
+		return rdma_lookup_get_uobject(type_attrs, ucontext, id, false);
+	case UVERBS_ACCESS_DESTROY:
+	case UVERBS_ACCESS_WRITE:
+		return rdma_lookup_get_uobject(type_attrs, ucontext, id, true);
+	case UVERBS_ACCESS_NEW:
+		return rdma_alloc_begin_uobject(type_attrs, ucontext);
+	default:
+		WARN_ON(true);
+		return ERR_PTR(-EOPNOTSUPP);
+	}
+}
+
+int uverbs_finalize_object(struct ib_uobject *uobj,
+			   enum uverbs_obj_access access,
+			   bool commit)
+{
+	int ret = 0;
+
+	/*
+	 * refcounts should be handled at the object level and not at the
+	 * uobject level. Refcounts of the objects themselves are done in
+	 * handlers.
+	 */
+
+	switch (access) {
+	case UVERBS_ACCESS_READ:
+		rdma_lookup_put_uobject(uobj, false);
+		break;
+	case UVERBS_ACCESS_WRITE:
+		rdma_lookup_put_uobject(uobj, true);
+		break;
+	case UVERBS_ACCESS_DESTROY:
+		if (commit)
+			ret = rdma_remove_commit_uobject(uobj);
+		else
+			rdma_lookup_put_uobject(uobj, true);
+		break;
+	case UVERBS_ACCESS_NEW:
+		if (commit)
+			ret = rdma_alloc_commit_uobject(uobj);
+		else
+			rdma_alloc_abort_uobject(uobj);
+		break;
+	default:
+		WARN_ON(true);
+		ret = -EOPNOTSUPP;
+	}
+
+	return ret;
+}
+
+int uverbs_finalize_objects(struct uverbs_attr_bundle *attrs_bundle,
+			    struct uverbs_attr_spec_hash * const *spec_hash,
+			    size_t num,
+			    bool commit)
+{
+	unsigned int i;
+	int ret = 0;
+
+	for (i = 0; i < num; i++) {
+		struct uverbs_attr_bundle_hash *curr_bundle =
+			&attrs_bundle->hash[i];
+		const struct uverbs_attr_spec_hash *curr_spec_bucket =
+			spec_hash[i];
+		unsigned int j;
+
+		for (j = 0; j < curr_bundle->num_attrs; j++) {
+			struct uverbs_attr *attr;
+			const struct uverbs_attr_spec *spec;
+
+			if (!uverbs_attr_is_valid_in_hash(curr_bundle, j))
+				continue;
+
+			attr = &curr_bundle->attrs[j];
+			spec = &curr_spec_bucket->attrs[j];
+
+			if (spec->type == UVERBS_ATTR_TYPE_IDR ||
+			    spec->type == UVERBS_ATTR_TYPE_FD) {
+				int current_ret;
+
+				current_ret = uverbs_finalize_object(attr->obj_attr.uobject,
+								     spec->obj.access,
+								     commit);
+				if (!ret)
+					ret = current_ret;
+			}
+		}
+	}
+	return ret;
+}
diff --git a/drivers/infiniband/core/rdma_core.h b/drivers/infiniband/core/rdma_core.h
index 1b82e7ff7fe8..1efcf93238dd 100644
--- a/drivers/infiniband/core/rdma_core.h
+++ b/drivers/infiniband/core/rdma_core.h
@@ -39,9 +39,15 @@
 
 #include <linux/idr.h>
 #include <rdma/uverbs_types.h>
+#include <rdma/uverbs_ioctl.h>
 #include <rdma/ib_verbs.h>
 #include <linux/mutex.h>
 
+int uverbs_ns_idx(u16 *id, unsigned int ns_count);
+const struct uverbs_object_spec *uverbs_get_object(const struct ib_device *ibdev,
+						   uint16_t object);
+const struct uverbs_method_spec *uverbs_get_method(const struct uverbs_object_spec *object,
+						   uint16_t method);
 /*
  * These functions initialize the context and cleanups its uobjects.
  * The context has a list of objects which is protected by a mutex
@@ -75,4 +81,40 @@ void uverbs_uobject_put(struct ib_uobject *uobject);
  */
 void uverbs_close_fd(struct file *f);
 
+/*
+ * Get an ib_uobject that corresponds to the given id from ucontext, assuming
+ * the object is from the given type. Lock it to the required access when
+ * applicable.
+ * This function could create (access == NEW), destroy (access == DESTROY)
+ * or unlock (access == READ || access == WRITE) objects if required.
+ * The action will be finalized only when uverbs_finalize_object or
+ * uverbs_finalize_objects are called.
+ */
+struct ib_uobject *uverbs_get_uobject_from_context(const struct uverbs_obj_type *type_attrs,
+						   struct ib_ucontext *ucontext,
+						   enum uverbs_obj_access access,
+						   int id);
+int uverbs_finalize_object(struct ib_uobject *uobj,
+			   enum uverbs_obj_access access,
+			   bool commit);
+/*
+ * Note that certain finalize stages could return a status:
+ *   (a) alloc_commit could return a failure if the object is committed at the
+ *       same time when the context is destroyed.
+ *   (b) remove_commit could fail if the object wasn't destroyed successfully.
+ * Since multiple objects could be finalized in one transaction, it is very NOT
+ * recommended to have several finalize actions which have side effects.
+ * For example, it's NOT recommended to have a certain action which has both
+ * a commit action and a destroy action or two destroy objects in the same
+ * action. The rule of thumb is to have one destroy or commit action with
+ * multiple lookups.
+ * The first non zero return value of finalize_object is returned from this
+ * function. For example, this could happen when we couldn't destroy an
+ * object.
+ */
+int uverbs_finalize_objects(struct uverbs_attr_bundle *attrs_bundle,
+			    struct uverbs_attr_spec_hash * const *spec_hash,
+			    size_t num,
+			    bool commit);
+
 #endif /* RDMA_CORE_H */
diff --git a/drivers/infiniband/core/roce_gid_mgmt.c b/drivers/infiniband/core/roce_gid_mgmt.c
index 94a9eefb3cfc..90e3889b7fbe 100644
--- a/drivers/infiniband/core/roce_gid_mgmt.c
+++ b/drivers/infiniband/core/roce_gid_mgmt.c
@@ -44,6 +44,8 @@
 
 static struct workqueue_struct *gid_cache_wq;
 
+static struct workqueue_struct *gid_cache_wq;
+
 enum gid_op_type {
 	GID_DEL = 0,
 	GID_ADD
diff --git a/drivers/infiniband/core/rw.c b/drivers/infiniband/core/rw.c
index dbfd854c32c9..6ca607e8e293 100644
--- a/drivers/infiniband/core/rw.c
+++ b/drivers/infiniband/core/rw.c
@@ -643,6 +643,30 @@ void rdma_rw_ctx_destroy_signature(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
 }
 EXPORT_SYMBOL(rdma_rw_ctx_destroy_signature);
 
+/**
+ * rdma_rw_mr_factor - return number of MRs required for a payload
+ * @device:	device handling the connection
+ * @port_num:	port num to which the connection is bound
+ * @maxpages:	maximum payload pages per rdma_rw_ctx
+ *
+ * Returns the number of MRs the device requires to move @maxpayload
+ * bytes. The returned value is used during transport creation to
+ * compute max_rdma_ctxts and the size of the transport's Send and
+ * Send Completion Queues.
+ */
+unsigned int rdma_rw_mr_factor(struct ib_device *device, u8 port_num,
+			       unsigned int maxpages)
+{
+	unsigned int mr_pages;
+
+	if (rdma_rw_can_use_mr(device, port_num))
+		mr_pages = rdma_rw_fr_page_list_len(device);
+	else
+		mr_pages = device->attrs.max_sge_rd;
+	return DIV_ROUND_UP(maxpages, mr_pages);
+}
+EXPORT_SYMBOL(rdma_rw_mr_factor);
+
 void rdma_rw_init_qp(struct ib_device *dev, struct ib_qp_init_attr *attr)
 {
 	u32 factor;
diff --git a/drivers/infiniband/core/sa_query.c b/drivers/infiniband/core/sa_query.c
index 70fa4cabe48e..ab5e1024fea9 100644
--- a/drivers/infiniband/core/sa_query.c
+++ b/drivers/infiniband/core/sa_query.c
@@ -50,6 +50,7 @@
 #include <uapi/rdma/ib_user_sa.h>
 #include <rdma/ib_marshall.h>
 #include <rdma/ib_addr.h>
+#include <rdma/opa_addr.h>
 #include "sa.h"
 #include "core_priv.h"
 
@@ -861,7 +862,7 @@ static int ib_nl_send_msg(struct ib_sa_query *query, gfp_t gfp_mask)
 	/* Repair the nlmsg header length */
 	nlmsg_end(skb, nlh);
 
-	ret = ibnl_multicast(skb, nlh, RDMA_NL_GROUP_LS, gfp_mask);
+	ret = rdma_nl_multicast(skb, RDMA_NL_GROUP_LS, gfp_mask);
 	if (!ret)
 		ret = len;
 	else
@@ -1021,9 +1022,9 @@ static void ib_nl_request_timeout(struct work_struct *work)
 }
 
 int ib_nl_handle_set_timeout(struct sk_buff *skb,
-			     struct netlink_callback *cb)
+			     struct nlmsghdr *nlh,
+			     struct netlink_ext_ack *extack)
 {
-	const struct nlmsghdr *nlh = (struct nlmsghdr *)cb->nlh;
 	int timeout, delta, abs_delta;
 	const struct nlattr *attr;
 	unsigned long flags;
@@ -1033,8 +1034,7 @@ int ib_nl_handle_set_timeout(struct sk_buff *skb,
 	int ret;
 
 	if (!(nlh->nlmsg_flags & NLM_F_REQUEST) ||
-	    !(NETLINK_CB(skb).sk) ||
-	    !netlink_capable(skb, CAP_NET_ADMIN))
+	    !(NETLINK_CB(skb).sk))
 		return -EPERM;
 
 	ret = nla_parse(tb, LS_NLA_TYPE_MAX - 1, nlmsg_data(nlh),
@@ -1098,9 +1098,9 @@ static inline int ib_nl_is_good_resolve_resp(const struct nlmsghdr *nlh)
 }
 
 int ib_nl_handle_resolve_resp(struct sk_buff *skb,
-			      struct netlink_callback *cb)
+			      struct nlmsghdr *nlh,
+			      struct netlink_ext_ack *extack)
 {
-	const struct nlmsghdr *nlh = (struct nlmsghdr *)cb->nlh;
 	unsigned long flags;
 	struct ib_sa_query *query;
 	struct ib_mad_send_buf *send_buf;
@@ -1109,8 +1109,7 @@ int ib_nl_handle_resolve_resp(struct sk_buff *skb,
 	int ret;
 
 	if ((nlh->nlmsg_flags & NLM_F_REQUEST) ||
-	    !(NETLINK_CB(skb).sk) ||
-	    !netlink_capable(skb, CAP_NET_ADMIN))
+	    !(NETLINK_CB(skb).sk))
 		return -EPERM;
 
 	spin_lock_irqsave(&ib_nl_request_lock, flags);
@@ -1241,6 +1240,11 @@ int ib_init_ah_from_path(struct ib_device *device, u8 port_num,
 	ah_attr->type = rdma_ah_find_type(device, port_num);
 
 	rdma_ah_set_dlid(ah_attr, be32_to_cpu(sa_path_get_dlid(rec)));
+
+	if ((ah_attr->type == RDMA_AH_ATTR_TYPE_OPA) &&
+	    (rdma_ah_get_dlid(ah_attr) == be16_to_cpu(IB_LID_PERMISSIVE)))
+		rdma_ah_set_make_grd(ah_attr, true);
+
 	rdma_ah_set_sl(ah_attr, rec->sl);
 	rdma_ah_set_path_bits(ah_attr, be32_to_cpu(sa_path_get_slid(rec)) &
 			      get_src_path_mask(device, port_num));
@@ -1420,7 +1424,7 @@ static int send_mad(struct ib_sa_query *query, int timeout_ms, gfp_t gfp_mask)
 
 	if ((query->flags & IB_SA_ENABLE_LOCAL_SERVICE) &&
 	    (!(query->flags & IB_SA_QUERY_OPA))) {
-		if (!ibnl_chk_listeners(RDMA_NL_GROUP_LS)) {
+		if (!rdma_nl_chk_listeners(RDMA_NL_GROUP_LS)) {
 			if (!ib_nl_make_request(query, gfp_mask))
 				return id;
 		}
@@ -2290,12 +2294,15 @@ static void update_sm_ah(struct work_struct *work)
 	rdma_ah_set_sl(&ah_attr, port_attr.sm_sl);
 	rdma_ah_set_port_num(&ah_attr, port->port_num);
 	if (port_attr.grh_required) {
-		rdma_ah_set_ah_flags(&ah_attr, IB_AH_GRH);
-
-		rdma_ah_set_subnet_prefix(&ah_attr,
-					  cpu_to_be64(port_attr.subnet_prefix));
-		rdma_ah_set_interface_id(&ah_attr,
-					 cpu_to_be64(IB_SA_WELL_KNOWN_GUID));
+		if (ah_attr.type == RDMA_AH_ATTR_TYPE_OPA) {
+			rdma_ah_set_make_grd(&ah_attr, true);
+		} else {
+			rdma_ah_set_ah_flags(&ah_attr, IB_AH_GRH);
+			rdma_ah_set_subnet_prefix(&ah_attr,
+						  cpu_to_be64(port_attr.subnet_prefix));
+			rdma_ah_set_interface_id(&ah_attr,
+						 cpu_to_be64(IB_SA_WELL_KNOWN_GUID));
+		}
 	}
 
 	new_ah->ah = rdma_create_ah(port->agent->qp->pd, &ah_attr);
@@ -2410,8 +2417,7 @@ static void ib_sa_add_one(struct ib_device *device)
 	 */
 
 	INIT_IB_EVENT_HANDLER(&sa_dev->event_handler, device, ib_sa_event);
-	if (ib_register_event_handler(&sa_dev->event_handler))
-		goto err;
+	ib_register_event_handler(&sa_dev->event_handler);
 
 	for (i = 0; i <= e - s; ++i) {
 		if (rdma_cap_ib_sa(device, i + 1))
diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c
index 7ebe1ef23652..abc5ab581f82 100644
--- a/drivers/infiniband/core/sysfs.c
+++ b/drivers/infiniband/core/sysfs.c
@@ -1210,8 +1210,8 @@ static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr,
 {
 	struct ib_device *dev = container_of(device, struct ib_device, dev);
 
-	ib_get_device_fw_str(dev, buf, PAGE_SIZE);
-	strlcat(buf, "\n", PAGE_SIZE);
+	ib_get_device_fw_str(dev, buf);
+	strlcat(buf, "\n", IB_FW_VERSION_NAME_MAX);
 	return strlen(buf);
 }
 
diff --git a/drivers/infiniband/core/ucm.c b/drivers/infiniband/core/ucm.c
index 112099c86a19..f2a7f62c2834 100644
--- a/drivers/infiniband/core/ucm.c
+++ b/drivers/infiniband/core/ucm.c
@@ -618,7 +618,7 @@ static ssize_t ib_ucm_init_qp_attr(struct ib_ucm_file *file,
 	if (result)
 		goto out;
 
-	ib_copy_qp_attr_to_user(&resp, &qp_attr);
+	ib_copy_qp_attr_to_user(ctx->cm_id->device, &resp, &qp_attr);
 
 	if (copy_to_user((void __user *)(unsigned long)cmd.response,
 			 &resp, sizeof(resp)))
diff --git a/drivers/infiniband/core/ucma.c b/drivers/infiniband/core/ucma.c
index 276f0ef835bd..eb85b546e223 100644
--- a/drivers/infiniband/core/ucma.c
+++ b/drivers/infiniband/core/ucma.c
@@ -248,14 +248,15 @@ static void ucma_copy_conn_event(struct rdma_ucm_conn_param *dst,
 	dst->qp_num = src->qp_num;
 }
 
-static void ucma_copy_ud_event(struct rdma_ucm_ud_param *dst,
+static void ucma_copy_ud_event(struct ib_device *device,
+			       struct rdma_ucm_ud_param *dst,
 			       struct rdma_ud_param *src)
 {
 	if (src->private_data_len)
 		memcpy(dst->private_data, src->private_data,
 		       src->private_data_len);
 	dst->private_data_len = src->private_data_len;
-	ib_copy_ah_attr_to_user(&dst->ah_attr, &src->ah_attr);
+	ib_copy_ah_attr_to_user(device, &dst->ah_attr, &src->ah_attr);
 	dst->qp_num = src->qp_num;
 	dst->qkey = src->qkey;
 }
@@ -335,7 +336,8 @@ static int ucma_event_handler(struct rdma_cm_id *cm_id,
 	uevent->resp.event = event->event;
 	uevent->resp.status = event->status;
 	if (cm_id->qp_type == IB_QPT_UD)
-		ucma_copy_ud_event(&uevent->resp.param.ud, &event->param.ud);
+		ucma_copy_ud_event(cm_id->device, &uevent->resp.param.ud,
+				   &event->param.ud);
 	else
 		ucma_copy_conn_event(&uevent->resp.param.conn,
 				     &event->param.conn);
@@ -1157,7 +1159,7 @@ static ssize_t ucma_init_qp_attr(struct ucma_file *file,
 	if (ret)
 		goto out;
 
-	ib_copy_qp_attr_to_user(&resp, &qp_attr);
+	ib_copy_qp_attr_to_user(ctx->cm_id->device, &resp, &qp_attr);
 	if (copy_to_user((void __user *)(unsigned long)cmd.response,
 			 &resp, sizeof(resp)))
 		ret = -EFAULT;
diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c
index 8c4ec564e495..55e8f5ed8b3c 100644
--- a/drivers/infiniband/core/umem_odp.c
+++ b/drivers/infiniband/core/umem_odp.c
@@ -166,24 +166,6 @@ static int invalidate_page_trampoline(struct ib_umem *item, u64 start,
 	return 0;
 }
 
-static void ib_umem_notifier_invalidate_page(struct mmu_notifier *mn,
-					     struct mm_struct *mm,
-					     unsigned long address)
-{
-	struct ib_ucontext *context = container_of(mn, struct ib_ucontext, mn);
-
-	if (!context->invalidate_range)
-		return;
-
-	ib_ucontext_notifier_start_account(context);
-	down_read(&context->umem_rwsem);
-	rbt_ib_umem_for_each_in_range(&context->umem_tree, address,
-				      address + PAGE_SIZE,
-				      invalidate_page_trampoline, NULL);
-	up_read(&context->umem_rwsem);
-	ib_ucontext_notifier_end_account(context);
-}
-
 static int invalidate_range_start_trampoline(struct ib_umem *item, u64 start,
 					     u64 end, void *cookie)
 {
@@ -237,7 +219,6 @@ static void ib_umem_notifier_invalidate_range_end(struct mmu_notifier *mn,
 
 static const struct mmu_notifier_ops ib_umem_notifiers = {
 	.release                    = ib_umem_notifier_release,
-	.invalidate_page            = ib_umem_notifier_invalidate_page,
 	.invalidate_range_start     = ib_umem_notifier_invalidate_range_start,
 	.invalidate_range_end       = ib_umem_notifier_invalidate_range_end,
 };
diff --git a/drivers/infiniband/core/umem_rbtree.c b/drivers/infiniband/core/umem_rbtree.c
index d176597b4d78..fc801920e341 100644
--- a/drivers/infiniband/core/umem_rbtree.c
+++ b/drivers/infiniband/core/umem_rbtree.c
@@ -72,7 +72,7 @@ INTERVAL_TREE_DEFINE(struct umem_odp_node, rb, u64, __subtree_last,
 /* @last is not a part of the interval. See comment for function
  * node_last.
  */
-int rbt_ib_umem_for_each_in_range(struct rb_root *root,
+int rbt_ib_umem_for_each_in_range(struct rb_root_cached *root,
 				  u64 start, u64 last,
 				  umem_call_back cb,
 				  void *cookie)
@@ -95,7 +95,7 @@ int rbt_ib_umem_for_each_in_range(struct rb_root *root,
 }
 EXPORT_SYMBOL(rbt_ib_umem_for_each_in_range);
 
-struct ib_umem_odp *rbt_ib_umem_lookup(struct rb_root *root,
+struct ib_umem_odp *rbt_ib_umem_lookup(struct rb_root_cached *root,
 				       u64 addr, u64 length)
 {
 	struct umem_odp_node *node;
diff --git a/drivers/infiniband/core/user_mad.c b/drivers/infiniband/core/user_mad.c
index 36a6f5c8914c..c1696e6084b2 100644
--- a/drivers/infiniband/core/user_mad.c
+++ b/drivers/infiniband/core/user_mad.c
@@ -229,7 +229,7 @@ static void recv_handler(struct ib_mad_agent *agent,
 	packet->mad.hdr.status	   = 0;
 	packet->mad.hdr.length	   = hdr_size(file) + mad_recv_wc->mad_len;
 	packet->mad.hdr.qpn	   = cpu_to_be32(mad_recv_wc->wc->src_qp);
-	packet->mad.hdr.lid	   = cpu_to_be16(mad_recv_wc->wc->slid);
+	packet->mad.hdr.lid	   = ib_lid_be16(mad_recv_wc->wc->slid);
 	packet->mad.hdr.sl	   = mad_recv_wc->wc->sl;
 	packet->mad.hdr.path_bits  = mad_recv_wc->wc->dlid_path_bits;
 	packet->mad.hdr.pkey_index = mad_recv_wc->wc->pkey_index;
diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h
index 64d494a64daf..37c8903e7fd0 100644
--- a/drivers/infiniband/core/uverbs.h
+++ b/drivers/infiniband/core/uverbs.h
@@ -100,6 +100,7 @@ struct ib_uverbs_device {
 	struct mutex				lists_mutex; /* protect lists */
 	struct list_head			uverbs_file_list;
 	struct list_head			uverbs_events_file_list;
+	struct uverbs_root_spec			*specs_root;
 };
 
 struct ib_uverbs_event_queue {
@@ -218,6 +219,8 @@ int uverbs_dealloc_mw(struct ib_mw *mw);
 void ib_uverbs_detach_umcast(struct ib_qp *qp,
 			     struct ib_uqp_object *uobj);
 
+long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
+
 struct ib_uverbs_flow_spec {
 	union {
 		union {
diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index c551d2b275fd..4ab30d832ac5 100644
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -91,9 +91,10 @@ ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file,
 		goto err;
 	}
 
-	INIT_UDATA(&udata, buf + sizeof cmd,
-		   (unsigned long) cmd.response + sizeof resp,
-		   in_len - sizeof cmd, out_len - sizeof resp);
+	INIT_UDATA(&udata, buf + sizeof(cmd),
+		   (unsigned long) cmd.response + sizeof(resp),
+		   in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
+		   out_len - sizeof(resp));
 
 	ret = ib_rdmacg_try_charge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE);
 	if (ret)
@@ -117,7 +118,7 @@ ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file,
 	ucontext->closing = 0;
 
 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
-	ucontext->umem_tree = RB_ROOT;
+	ucontext->umem_tree = RB_ROOT_CACHED;
 	init_rwsem(&ucontext->umem_rwsem);
 	ucontext->odp_mrs_count = 0;
 	INIT_LIST_HEAD(&ucontext->no_private_counters);
@@ -275,8 +276,14 @@ ssize_t ib_uverbs_query_port(struct ib_uverbs_file *file,
 	resp.bad_pkey_cntr   = attr.bad_pkey_cntr;
 	resp.qkey_viol_cntr  = attr.qkey_viol_cntr;
 	resp.pkey_tbl_len    = attr.pkey_tbl_len;
-	resp.lid 	     = attr.lid;
-	resp.sm_lid 	     = attr.sm_lid;
+
+	if (rdma_cap_opa_ah(ib_dev, cmd.port_num)) {
+		resp.lid     = OPA_TO_IB_UCAST_LID(attr.lid);
+		resp.sm_lid  = OPA_TO_IB_UCAST_LID(attr.sm_lid);
+	} else {
+		resp.lid     = ib_lid_cpu16(attr.lid);
+		resp.sm_lid  = ib_lid_cpu16(attr.sm_lid);
+	}
 	resp.lmc 	     = attr.lmc;
 	resp.max_vl_num      = attr.max_vl_num;
 	resp.sm_sl 	     = attr.sm_sl;
@@ -313,9 +320,10 @@ ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file,
 	if (copy_from_user(&cmd, buf, sizeof cmd))
 		return -EFAULT;
 
-	INIT_UDATA(&udata, buf + sizeof cmd,
-		   (unsigned long) cmd.response + sizeof resp,
-		   in_len - sizeof cmd, out_len - sizeof resp);
+	INIT_UDATA(&udata, buf + sizeof(cmd),
+		   (unsigned long) cmd.response + sizeof(resp),
+                   in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
+                   out_len - sizeof(resp));
 
 	uobj  = uobj_alloc(uobj_get_type(pd), file->ucontext);
 	if (IS_ERR(uobj))
@@ -482,9 +490,10 @@ ssize_t ib_uverbs_open_xrcd(struct ib_uverbs_file *file,
 	if (copy_from_user(&cmd, buf, sizeof cmd))
 		return -EFAULT;
 
-	INIT_UDATA(&udata, buf + sizeof cmd,
-		   (unsigned long) cmd.response + sizeof resp,
-		   in_len - sizeof cmd, out_len - sizeof  resp);
+	INIT_UDATA(&udata, buf + sizeof(cmd),
+		   (unsigned long) cmd.response + sizeof(resp),
+                   in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
+                   out_len - sizeof(resp));
 
 	mutex_lock(&file->device->xrcd_tree_mutex);
 
@@ -646,9 +655,10 @@ ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file,
 	if (copy_from_user(&cmd, buf, sizeof cmd))
 		return -EFAULT;
 
-	INIT_UDATA(&udata, buf + sizeof cmd,
-		   (unsigned long) cmd.response + sizeof resp,
-		   in_len - sizeof cmd, out_len - sizeof resp);
+	INIT_UDATA(&udata, buf + sizeof(cmd),
+		   (unsigned long) cmd.response + sizeof(resp),
+                   in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
+                   out_len - sizeof(resp));
 
 	if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))
 		return -EINVAL;
@@ -740,7 +750,8 @@ ssize_t ib_uverbs_rereg_mr(struct ib_uverbs_file *file,
 
 	INIT_UDATA(&udata, buf + sizeof(cmd),
 		   (unsigned long) cmd.response + sizeof(resp),
-		   in_len - sizeof(cmd), out_len - sizeof(resp));
+                   in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
+                   out_len - sizeof(resp));
 
 	if (cmd.flags & ~IB_MR_REREG_SUPPORTED || !cmd.flags)
 		return -EINVAL;
@@ -1015,7 +1026,7 @@ static struct ib_ucq_object *create_cq(struct ib_uverbs_file *file,
 	cq->uobject       = &obj->uobject;
 	cq->comp_handler  = ib_uverbs_comp_handler;
 	cq->event_handler = ib_uverbs_cq_event_handler;
-	cq->cq_context    = &ev_file->ev_queue;
+	cq->cq_context    = ev_file ? &ev_file->ev_queue : NULL;
 	atomic_set(&cq->usecnt, 0);
 
 	obj->uobject.object = cq;
@@ -1080,7 +1091,8 @@ ssize_t ib_uverbs_create_cq(struct ib_uverbs_file *file,
 
 	INIT_UDATA(&uhw, buf + sizeof(cmd),
 		   (unsigned long)cmd.response + sizeof(resp),
-		   in_len - sizeof(cmd), out_len - sizeof(resp));
+		   in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
+		   out_len - sizeof(resp));
 
 	memset(&cmd_ex, 0, sizeof(cmd_ex));
 	cmd_ex.user_handle = cmd.user_handle;
@@ -1161,9 +1173,10 @@ ssize_t ib_uverbs_resize_cq(struct ib_uverbs_file *file,
 	if (copy_from_user(&cmd, buf, sizeof cmd))
 		return -EFAULT;
 
-	INIT_UDATA(&udata, buf + sizeof cmd,
-		   (unsigned long) cmd.response + sizeof resp,
-		   in_len - sizeof cmd, out_len - sizeof resp);
+	INIT_UDATA(&udata, buf + sizeof(cmd),
+		   (unsigned long) cmd.response + sizeof(resp),
+		   in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
+		   out_len - sizeof(resp));
 
 	cq = uobj_get_obj_read(cq, cmd.cq_handle, file->ucontext);
 	if (!cq)
@@ -1185,7 +1198,8 @@ out:
 	return ret ? ret : in_len;
 }
 
-static int copy_wc_to_user(void __user *dest, struct ib_wc *wc)
+static int copy_wc_to_user(struct ib_device *ib_dev, void __user *dest,
+			   struct ib_wc *wc)
 {
 	struct ib_uverbs_wc tmp;
 
@@ -1199,7 +1213,10 @@ static int copy_wc_to_user(void __user *dest, struct ib_wc *wc)
 	tmp.src_qp		= wc->src_qp;
 	tmp.wc_flags		= wc->wc_flags;
 	tmp.pkey_index		= wc->pkey_index;
-	tmp.slid		= wc->slid;
+	if (rdma_cap_opa_ah(ib_dev, wc->port_num))
+		tmp.slid	= OPA_TO_IB_UCAST_LID(wc->slid);
+	else
+		tmp.slid	= ib_lid_cpu16(wc->slid);
 	tmp.sl			= wc->sl;
 	tmp.dlid_path_bits	= wc->dlid_path_bits;
 	tmp.port_num		= wc->port_num;
@@ -1243,7 +1260,7 @@ ssize_t ib_uverbs_poll_cq(struct ib_uverbs_file *file,
 		if (!ret)
 			break;
 
-		ret = copy_wc_to_user(data_ptr, &wc);
+		ret = copy_wc_to_user(ib_dev, data_ptr, &wc);
 		if (ret)
 			goto out_put;
 
@@ -1383,8 +1400,9 @@ static int create_qp(struct ib_uverbs_file *file,
 		attr.rwq_ind_tbl = ind_tbl;
 	}
 
-	if ((cmd_sz >= offsetof(typeof(*cmd), reserved1) +
-		       sizeof(cmd->reserved1)) && cmd->reserved1) {
+	if (cmd_sz > sizeof(*cmd) &&
+	    !ib_is_udata_cleared(ucore, sizeof(*cmd),
+				 cmd_sz - sizeof(*cmd))) {
 		ret = -EOPNOTSUPP;
 		goto err_put;
 	}
@@ -1420,7 +1438,7 @@ static int create_qp(struct ib_uverbs_file *file,
 			if (cmd->is_srq) {
 				srq = uobj_get_obj_read(srq, cmd->srq_handle,
 							file->ucontext);
-				if (!srq || srq->srq_type != IB_SRQT_BASIC) {
+				if (!srq || srq->srq_type == IB_SRQT_XRC) {
 					ret = -EINVAL;
 					goto err_put;
 				}
@@ -1482,11 +1500,21 @@ static int create_qp(struct ib_uverbs_file *file,
 				IB_QP_CREATE_MANAGED_SEND |
 				IB_QP_CREATE_MANAGED_RECV |
 				IB_QP_CREATE_SCATTER_FCS |
-				IB_QP_CREATE_CVLAN_STRIPPING)) {
+				IB_QP_CREATE_CVLAN_STRIPPING |
+				IB_QP_CREATE_SOURCE_QPN)) {
 		ret = -EINVAL;
 		goto err_put;
 	}
 
+	if (attr.create_flags & IB_QP_CREATE_SOURCE_QPN) {
+		if (!capable(CAP_NET_RAW)) {
+			ret = -EPERM;
+			goto err_put;
+		}
+
+		attr.source_qpn = cmd->source_qpn;
+	}
+
 	buf = (void *)cmd + sizeof(*cmd);
 	if (cmd_sz > sizeof(*cmd))
 		if (!(buf[0] == 0 && !memcmp(buf, buf + 1,
@@ -1522,6 +1550,7 @@ static int create_qp(struct ib_uverbs_file *file,
 		qp->qp_type	  = attr.qp_type;
 		atomic_set(&qp->usecnt, 0);
 		atomic_inc(&pd->usecnt);
+		qp->port = 0;
 		if (attr.send_cq)
 			atomic_inc(&attr.send_cq->usecnt);
 		if (attr.recv_cq)
@@ -1721,9 +1750,10 @@ ssize_t ib_uverbs_open_qp(struct ib_uverbs_file *file,
 	if (copy_from_user(&cmd, buf, sizeof cmd))
 		return -EFAULT;
 
-	INIT_UDATA(&udata, buf + sizeof cmd,
-		   (unsigned long) cmd.response + sizeof resp,
-		   in_len - sizeof cmd, out_len - sizeof resp);
+	INIT_UDATA(&udata, buf + sizeof(cmd),
+		   (unsigned long) cmd.response + sizeof(resp),
+		   in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
+		   out_len - sizeof(resp));
 
 	obj  = (struct ib_uqp_object *)uobj_alloc(uobj_get_type(qp),
 						  file->ucontext);
@@ -1790,6 +1820,28 @@ err_put:
 	return ret;
 }
 
+static void copy_ah_attr_to_uverbs(struct ib_uverbs_qp_dest *uverb_attr,
+				   struct rdma_ah_attr *rdma_attr)
+{
+	const struct ib_global_route   *grh;
+
+	uverb_attr->dlid              = rdma_ah_get_dlid(rdma_attr);
+	uverb_attr->sl                = rdma_ah_get_sl(rdma_attr);
+	uverb_attr->src_path_bits     = rdma_ah_get_path_bits(rdma_attr);
+	uverb_attr->static_rate       = rdma_ah_get_static_rate(rdma_attr);
+	uverb_attr->is_global         = !!(rdma_ah_get_ah_flags(rdma_attr) &
+					 IB_AH_GRH);
+	if (uverb_attr->is_global) {
+		grh = rdma_ah_read_grh(rdma_attr);
+		memcpy(uverb_attr->dgid, grh->dgid.raw, 16);
+		uverb_attr->flow_label        = grh->flow_label;
+		uverb_attr->sgid_index        = grh->sgid_index;
+		uverb_attr->hop_limit         = grh->hop_limit;
+		uverb_attr->traffic_class     = grh->traffic_class;
+	}
+	uverb_attr->port_num          = rdma_ah_get_port_num(rdma_attr);
+}
+
 ssize_t ib_uverbs_query_qp(struct ib_uverbs_file *file,
 			   struct ib_device *ib_dev,
 			   const char __user *buf, int in_len,
@@ -1800,7 +1852,6 @@ ssize_t ib_uverbs_query_qp(struct ib_uverbs_file *file,
 	struct ib_qp                   *qp;
 	struct ib_qp_attr              *attr;
 	struct ib_qp_init_attr         *init_attr;
-	const struct ib_global_route   *grh;
 	int                            ret;
 
 	if (copy_from_user(&cmd, buf, sizeof cmd))
@@ -1850,39 +1901,8 @@ ssize_t ib_uverbs_query_qp(struct ib_uverbs_file *file,
 	resp.alt_port_num           = attr->alt_port_num;
 	resp.alt_timeout            = attr->alt_timeout;
 
-	resp.dest.dlid              = rdma_ah_get_dlid(&attr->ah_attr);
-	resp.dest.sl                = rdma_ah_get_sl(&attr->ah_attr);
-	resp.dest.src_path_bits     = rdma_ah_get_path_bits(&attr->ah_attr);
-	resp.dest.static_rate       = rdma_ah_get_static_rate(&attr->ah_attr);
-	resp.dest.is_global         = !!(rdma_ah_get_ah_flags(&attr->ah_attr) &
-					 IB_AH_GRH);
-	if (resp.dest.is_global) {
-		grh = rdma_ah_read_grh(&attr->ah_attr);
-		memcpy(resp.dest.dgid, grh->dgid.raw, 16);
-		resp.dest.flow_label        = grh->flow_label;
-		resp.dest.sgid_index        = grh->sgid_index;
-		resp.dest.hop_limit         = grh->hop_limit;
-		resp.dest.traffic_class     = grh->traffic_class;
-	}
-	resp.dest.port_num          = rdma_ah_get_port_num(&attr->ah_attr);
-
-	resp.alt_dest.dlid          = rdma_ah_get_dlid(&attr->alt_ah_attr);
-	resp.alt_dest.sl            = rdma_ah_get_sl(&attr->alt_ah_attr);
-	resp.alt_dest.src_path_bits = rdma_ah_get_path_bits(&attr->alt_ah_attr);
-	resp.alt_dest.static_rate
-			= rdma_ah_get_static_rate(&attr->alt_ah_attr);
-	resp.alt_dest.is_global
-			= !!(rdma_ah_get_ah_flags(&attr->alt_ah_attr) &
-						  IB_AH_GRH);
-	if (resp.alt_dest.is_global) {
-		grh = rdma_ah_read_grh(&attr->alt_ah_attr);
-		memcpy(resp.alt_dest.dgid, grh->dgid.raw, 16);
-		resp.alt_dest.flow_label    = grh->flow_label;
-		resp.alt_dest.sgid_index    = grh->sgid_index;
-		resp.alt_dest.hop_limit     = grh->hop_limit;
-		resp.alt_dest.traffic_class = grh->traffic_class;
-	}
-	resp.alt_dest.port_num      = rdma_ah_get_port_num(&attr->alt_ah_attr);
+	copy_ah_attr_to_uverbs(&resp.dest, &attr->ah_attr);
+	copy_ah_attr_to_uverbs(&resp.alt_dest, &attr->alt_ah_attr);
 
 	resp.max_send_wr            = init_attr->cap.max_send_wr;
 	resp.max_recv_wr            = init_attr->cap.max_recv_wr;
@@ -1916,6 +1936,29 @@ static int modify_qp_mask(enum ib_qp_type qp_type, int mask)
 	}
 }
 
+static void copy_ah_attr_from_uverbs(struct ib_device *dev,
+				     struct rdma_ah_attr *rdma_attr,
+				     struct ib_uverbs_qp_dest *uverb_attr)
+{
+	rdma_attr->type = rdma_ah_find_type(dev, uverb_attr->port_num);
+	if (uverb_attr->is_global) {
+		rdma_ah_set_grh(rdma_attr, NULL,
+				uverb_attr->flow_label,
+				uverb_attr->sgid_index,
+				uverb_attr->hop_limit,
+				uverb_attr->traffic_class);
+		rdma_ah_set_dgid_raw(rdma_attr, uverb_attr->dgid);
+	} else {
+		rdma_ah_set_ah_flags(rdma_attr, 0);
+	}
+	rdma_ah_set_dlid(rdma_attr, uverb_attr->dlid);
+	rdma_ah_set_sl(rdma_attr, uverb_attr->sl);
+	rdma_ah_set_path_bits(rdma_attr, uverb_attr->src_path_bits);
+	rdma_ah_set_static_rate(rdma_attr, uverb_attr->static_rate);
+	rdma_ah_set_port_num(rdma_attr, uverb_attr->port_num);
+	rdma_ah_set_make_grd(rdma_attr, false);
+}
+
 static int modify_qp(struct ib_uverbs_file *file,
 		     struct ib_uverbs_ex_modify_qp *cmd, struct ib_udata *udata)
 {
@@ -1962,47 +2005,13 @@ static int modify_qp(struct ib_uverbs_file *file,
 	attr->alt_timeout	  = cmd->base.alt_timeout;
 	attr->rate_limit	  = cmd->rate_limit;
 
-	attr->ah_attr.type = rdma_ah_find_type(qp->device,
-					       cmd->base.dest.port_num);
-	if (cmd->base.dest.is_global) {
-		rdma_ah_set_grh(&attr->ah_attr, NULL,
-				cmd->base.dest.flow_label,
-				cmd->base.dest.sgid_index,
-				cmd->base.dest.hop_limit,
-				cmd->base.dest.traffic_class);
-		rdma_ah_set_dgid_raw(&attr->ah_attr, cmd->base.dest.dgid);
-	} else {
-		rdma_ah_set_ah_flags(&attr->ah_attr, 0);
-	}
-	rdma_ah_set_dlid(&attr->ah_attr, cmd->base.dest.dlid);
-	rdma_ah_set_sl(&attr->ah_attr, cmd->base.dest.sl);
-	rdma_ah_set_path_bits(&attr->ah_attr, cmd->base.dest.src_path_bits);
-	rdma_ah_set_static_rate(&attr->ah_attr, cmd->base.dest.static_rate);
-	rdma_ah_set_port_num(&attr->ah_attr,
-			     cmd->base.dest.port_num);
-
-	attr->alt_ah_attr.type = rdma_ah_find_type(qp->device,
-						   cmd->base.dest.port_num);
-	if (cmd->base.alt_dest.is_global) {
-		rdma_ah_set_grh(&attr->alt_ah_attr, NULL,
-				cmd->base.alt_dest.flow_label,
-				cmd->base.alt_dest.sgid_index,
-				cmd->base.alt_dest.hop_limit,
-				cmd->base.alt_dest.traffic_class);
-		rdma_ah_set_dgid_raw(&attr->alt_ah_attr,
-				     cmd->base.alt_dest.dgid);
-	} else {
-		rdma_ah_set_ah_flags(&attr->alt_ah_attr, 0);
-	}
+	if (cmd->base.attr_mask & IB_QP_AV)
+		copy_ah_attr_from_uverbs(qp->device, &attr->ah_attr,
+					 &cmd->base.dest);
 
-	rdma_ah_set_dlid(&attr->alt_ah_attr, cmd->base.alt_dest.dlid);
-	rdma_ah_set_sl(&attr->alt_ah_attr, cmd->base.alt_dest.sl);
-	rdma_ah_set_path_bits(&attr->alt_ah_attr,
-			      cmd->base.alt_dest.src_path_bits);
-	rdma_ah_set_static_rate(&attr->alt_ah_attr,
-				cmd->base.alt_dest.static_rate);
-	rdma_ah_set_port_num(&attr->alt_ah_attr,
-			     cmd->base.alt_dest.port_num);
+	if (cmd->base.attr_mask & IB_QP_ALT_PATH)
+		copy_ah_attr_from_uverbs(qp->device, &attr->alt_ah_attr,
+					 &cmd->base.alt_dest);
 
 	ret = ib_modify_qp_with_udata(qp, attr,
 				      modify_qp_mask(qp->qp_type,
@@ -2034,7 +2043,8 @@ ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file *file,
 		return -EOPNOTSUPP;
 
 	INIT_UDATA(&udata, buf + sizeof(cmd.base), NULL,
-		   in_len - sizeof(cmd.base), out_len);
+		   in_len - sizeof(cmd.base) - sizeof(struct ib_uverbs_cmd_hdr),
+		   out_len);
 
 	ret = modify_qp(file, &cmd, &udata);
 	if (ret)
@@ -2540,7 +2550,8 @@ ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file,
 
 	INIT_UDATA(&udata, buf + sizeof(cmd),
 		   (unsigned long)cmd.response + sizeof(resp),
-		   in_len - sizeof(cmd), out_len - sizeof(resp));
+		   in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
+		   out_len - sizeof(resp));
 
 	uobj  = uobj_alloc(uobj_get_type(ah), file->ucontext);
 	if (IS_ERR(uobj))
@@ -2553,6 +2564,7 @@ ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file,
 	}
 
 	attr.type = rdma_ah_find_type(ib_dev, cmd.attr.port_num);
+	rdma_ah_set_make_grd(&attr, false);
 	rdma_ah_set_dlid(&attr, cmd.attr.dlid);
 	rdma_ah_set_sl(&attr, cmd.attr.sl);
 	rdma_ah_set_path_bits(&attr, cmd.attr.src_path_bits);
@@ -3469,6 +3481,9 @@ static int __uverbs_create_xsrq(struct ib_uverbs_file *file,
 	if (IS_ERR(obj))
 		return PTR_ERR(obj);
 
+	if (cmd->srq_type == IB_SRQT_TM)
+		attr.ext.tag_matching.max_num_tags = cmd->max_num_tags;
+
 	if (cmd->srq_type == IB_SRQT_XRC) {
 		xrcd_uobj = uobj_get_read(uobj_get_type(xrcd), cmd->xrcd_handle,
 					  file->ucontext);
@@ -3485,10 +3500,12 @@ static int __uverbs_create_xsrq(struct ib_uverbs_file *file,
 
 		obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
 		atomic_inc(&obj->uxrcd->refcnt);
+	}
 
-		attr.ext.xrc.cq  = uobj_get_obj_read(cq, cmd->cq_handle,
-						     file->ucontext);
-		if (!attr.ext.xrc.cq) {
+	if (ib_srq_has_cq(cmd->srq_type)) {
+		attr.ext.cq  = uobj_get_obj_read(cq, cmd->cq_handle,
+						 file->ucontext);
+		if (!attr.ext.cq) {
 			ret = -EINVAL;
 			goto err_put_xrcd;
 		}
@@ -3523,10 +3540,13 @@ static int __uverbs_create_xsrq(struct ib_uverbs_file *file,
 	srq->event_handler = attr.event_handler;
 	srq->srq_context   = attr.srq_context;
 
+	if (ib_srq_has_cq(cmd->srq_type)) {
+		srq->ext.cq       = attr.ext.cq;
+		atomic_inc(&attr.ext.cq->usecnt);
+	}
+
 	if (cmd->srq_type == IB_SRQT_XRC) {
-		srq->ext.xrc.cq   = attr.ext.xrc.cq;
 		srq->ext.xrc.xrcd = attr.ext.xrc.xrcd;
-		atomic_inc(&attr.ext.xrc.cq->usecnt);
 		atomic_inc(&attr.ext.xrc.xrcd->usecnt);
 	}
 
@@ -3549,10 +3569,12 @@ static int __uverbs_create_xsrq(struct ib_uverbs_file *file,
 		goto err_copy;
 	}
 
-	if (cmd->srq_type == IB_SRQT_XRC) {
+	if (cmd->srq_type == IB_SRQT_XRC)
 		uobj_put_read(xrcd_uobj);
-		uobj_put_obj_read(attr.ext.xrc.cq);
-	}
+
+	if (ib_srq_has_cq(cmd->srq_type))
+		uobj_put_obj_read(attr.ext.cq);
+
 	uobj_put_obj_read(pd);
 	uobj_alloc_commit(&obj->uevent.uobject);
 
@@ -3565,8 +3587,8 @@ err_put:
 	uobj_put_obj_read(pd);
 
 err_put_cq:
-	if (cmd->srq_type == IB_SRQT_XRC)
-		uobj_put_obj_read(attr.ext.xrc.cq);
+	if (ib_srq_has_cq(cmd->srq_type))
+		uobj_put_obj_read(attr.ext.cq);
 
 err_put_xrcd:
 	if (cmd->srq_type == IB_SRQT_XRC) {
@@ -3596,6 +3618,7 @@ ssize_t ib_uverbs_create_srq(struct ib_uverbs_file *file,
 	if (copy_from_user(&cmd, buf, sizeof cmd))
 		return -EFAULT;
 
+	memset(&xcmd, 0, sizeof(xcmd));
 	xcmd.response	 = cmd.response;
 	xcmd.user_handle = cmd.user_handle;
 	xcmd.srq_type	 = IB_SRQT_BASIC;
@@ -3604,10 +3627,10 @@ ssize_t ib_uverbs_create_srq(struct ib_uverbs_file *file,
 	xcmd.max_sge	 = cmd.max_sge;
 	xcmd.srq_limit	 = cmd.srq_limit;
 
-	INIT_UDATA(&udata, buf + sizeof cmd,
-		   (unsigned long) cmd.response + sizeof resp,
-		   in_len - sizeof cmd - sizeof(struct ib_uverbs_cmd_hdr),
-		   out_len - sizeof resp);
+	INIT_UDATA(&udata, buf + sizeof(cmd),
+		   (unsigned long) cmd.response + sizeof(resp),
+		   in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
+		   out_len - sizeof(resp));
 
 	ret = __uverbs_create_xsrq(file, ib_dev, &xcmd, &udata);
 	if (ret)
@@ -3631,10 +3654,10 @@ ssize_t ib_uverbs_create_xsrq(struct ib_uverbs_file *file,
 	if (copy_from_user(&cmd, buf, sizeof cmd))
 		return -EFAULT;
 
-	INIT_UDATA(&udata, buf + sizeof cmd,
-		   (unsigned long) cmd.response + sizeof resp,
-		   in_len - sizeof cmd - sizeof(struct ib_uverbs_cmd_hdr),
-		   out_len - sizeof resp);
+	INIT_UDATA(&udata, buf + sizeof(cmd),
+		   (unsigned long) cmd.response + sizeof(resp),
+		   in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
+		   out_len - sizeof(resp));
 
 	ret = __uverbs_create_xsrq(file, ib_dev, &cmd, &udata);
 	if (ret)
@@ -3845,6 +3868,16 @@ int ib_uverbs_ex_query_device(struct ib_uverbs_file *file,
 
 	resp.raw_packet_caps = attr.raw_packet_caps;
 	resp.response_length += sizeof(resp.raw_packet_caps);
+
+	if (ucore->outlen < resp.response_length + sizeof(resp.xrq_caps))
+		goto end;
+
+	resp.xrq_caps.max_rndv_hdr_size = attr.xrq_caps.max_rndv_hdr_size;
+	resp.xrq_caps.max_num_tags      = attr.xrq_caps.max_num_tags;
+	resp.xrq_caps.max_ops		= attr.xrq_caps.max_ops;
+	resp.xrq_caps.max_sge		= attr.xrq_caps.max_sge;
+	resp.xrq_caps.flags		= attr.xrq_caps.flags;
+	resp.response_length += sizeof(resp.xrq_caps);
 end:
 	err = ib_copy_to_udata(ucore, &resp, resp.response_length);
 	return err;
diff --git a/drivers/infiniband/core/uverbs_ioctl.c b/drivers/infiniband/core/uverbs_ioctl.c
new file mode 100644
index 000000000000..5286ad57d903
--- /dev/null
+++ b/drivers/infiniband/core/uverbs_ioctl.c
@@ -0,0 +1,364 @@
+/*
+ * Copyright (c) 2017, Mellanox Technologies inc.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <rdma/rdma_user_ioctl.h>
+#include <rdma/uverbs_ioctl.h>
+#include "rdma_core.h"
+#include "uverbs.h"
+
+static int uverbs_process_attr(struct ib_device *ibdev,
+			       struct ib_ucontext *ucontext,
+			       const struct ib_uverbs_attr *uattr,
+			       u16 attr_id,
+			       const struct uverbs_attr_spec_hash *attr_spec_bucket,
+			       struct uverbs_attr_bundle_hash *attr_bundle_h,
+			       struct ib_uverbs_attr __user *uattr_ptr)
+{
+	const struct uverbs_attr_spec *spec;
+	struct uverbs_attr *e;
+	const struct uverbs_object_spec *object;
+	struct uverbs_obj_attr *o_attr;
+	struct uverbs_attr *elements = attr_bundle_h->attrs;
+
+	if (uattr->reserved)
+		return -EINVAL;
+
+	if (attr_id >= attr_spec_bucket->num_attrs) {
+		if (uattr->flags & UVERBS_ATTR_F_MANDATORY)
+			return -EINVAL;
+		else
+			return 0;
+	}
+
+	spec = &attr_spec_bucket->attrs[attr_id];
+	e = &elements[attr_id];
+	e->uattr = uattr_ptr;
+
+	switch (spec->type) {
+	case UVERBS_ATTR_TYPE_PTR_IN:
+	case UVERBS_ATTR_TYPE_PTR_OUT:
+		if (uattr->len < spec->len ||
+		    (!(spec->flags & UVERBS_ATTR_SPEC_F_MIN_SZ) &&
+		     uattr->len > spec->len))
+			return -EINVAL;
+
+		e->ptr_attr.data = uattr->data;
+		e->ptr_attr.len = uattr->len;
+		e->ptr_attr.flags = uattr->flags;
+		break;
+
+	case UVERBS_ATTR_TYPE_IDR:
+		if (uattr->data >> 32)
+			return -EINVAL;
+	/* fall through */
+	case UVERBS_ATTR_TYPE_FD:
+		if (uattr->len != 0 || !ucontext || uattr->data > INT_MAX)
+			return -EINVAL;
+
+		o_attr = &e->obj_attr;
+		object = uverbs_get_object(ibdev, spec->obj.obj_type);
+		if (!object)
+			return -EINVAL;
+		o_attr->type = object->type_attrs;
+
+		o_attr->id = (int)uattr->data;
+		o_attr->uobject = uverbs_get_uobject_from_context(
+					o_attr->type,
+					ucontext,
+					spec->obj.access,
+					o_attr->id);
+
+		if (IS_ERR(o_attr->uobject))
+			return PTR_ERR(o_attr->uobject);
+
+		if (spec->obj.access == UVERBS_ACCESS_NEW) {
+			u64 id = o_attr->uobject->id;
+
+			/* Copy the allocated id to the user-space */
+			if (put_user(id, &e->uattr->data)) {
+				uverbs_finalize_object(o_attr->uobject,
+						       UVERBS_ACCESS_NEW,
+						       false);
+				return -EFAULT;
+			}
+		}
+
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	set_bit(attr_id, attr_bundle_h->valid_bitmap);
+	return 0;
+}
+
+static int uverbs_uattrs_process(struct ib_device *ibdev,
+				 struct ib_ucontext *ucontext,
+				 const struct ib_uverbs_attr *uattrs,
+				 size_t num_uattrs,
+				 const struct uverbs_method_spec *method,
+				 struct uverbs_attr_bundle *attr_bundle,
+				 struct ib_uverbs_attr __user *uattr_ptr)
+{
+	size_t i;
+	int ret = 0;
+	int num_given_buckets = 0;
+
+	for (i = 0; i < num_uattrs; i++) {
+		const struct ib_uverbs_attr *uattr = &uattrs[i];
+		u16 attr_id = uattr->attr_id;
+		struct uverbs_attr_spec_hash *attr_spec_bucket;
+
+		ret = uverbs_ns_idx(&attr_id, method->num_buckets);
+		if (ret < 0) {
+			if (uattr->flags & UVERBS_ATTR_F_MANDATORY) {
+				uverbs_finalize_objects(attr_bundle,
+							method->attr_buckets,
+							num_given_buckets,
+							false);
+				return ret;
+			}
+			continue;
+		}
+
+		/*
+		 * ret is the found ns, so increase num_given_buckets if
+		 * necessary.
+		 */
+		if (ret >= num_given_buckets)
+			num_given_buckets = ret + 1;
+
+		attr_spec_bucket = method->attr_buckets[ret];
+		ret = uverbs_process_attr(ibdev, ucontext, uattr, attr_id,
+					  attr_spec_bucket, &attr_bundle->hash[ret],
+					  uattr_ptr++);
+		if (ret) {
+			uverbs_finalize_objects(attr_bundle,
+						method->attr_buckets,
+						num_given_buckets,
+						false);
+			return ret;
+		}
+	}
+
+	return num_given_buckets;
+}
+
+static int uverbs_validate_kernel_mandatory(const struct uverbs_method_spec *method_spec,
+					    struct uverbs_attr_bundle *attr_bundle)
+{
+	unsigned int i;
+
+	for (i = 0; i < attr_bundle->num_buckets; i++) {
+		struct uverbs_attr_spec_hash *attr_spec_bucket =
+			method_spec->attr_buckets[i];
+
+		if (!bitmap_subset(attr_spec_bucket->mandatory_attrs_bitmask,
+				   attr_bundle->hash[i].valid_bitmap,
+				   attr_spec_bucket->num_attrs))
+			return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int uverbs_handle_method(struct ib_uverbs_attr __user *uattr_ptr,
+				const struct ib_uverbs_attr *uattrs,
+				size_t num_uattrs,
+				struct ib_device *ibdev,
+				struct ib_uverbs_file *ufile,
+				const struct uverbs_method_spec *method_spec,
+				struct uverbs_attr_bundle *attr_bundle)
+{
+	int ret;
+	int finalize_ret;
+	int num_given_buckets;
+
+	num_given_buckets = uverbs_uattrs_process(ibdev, ufile->ucontext, uattrs,
+						  num_uattrs, method_spec,
+						  attr_bundle, uattr_ptr);
+	if (num_given_buckets <= 0)
+		return -EINVAL;
+
+	attr_bundle->num_buckets = num_given_buckets;
+	ret = uverbs_validate_kernel_mandatory(method_spec, attr_bundle);
+	if (ret)
+		goto cleanup;
+
+	ret = method_spec->handler(ibdev, ufile, attr_bundle);
+cleanup:
+	finalize_ret = uverbs_finalize_objects(attr_bundle,
+					       method_spec->attr_buckets,
+					       attr_bundle->num_buckets,
+					       !ret);
+
+	return ret ? ret : finalize_ret;
+}
+
+#define UVERBS_OPTIMIZE_USING_STACK_SZ  256
+static long ib_uverbs_cmd_verbs(struct ib_device *ib_dev,
+				struct ib_uverbs_file *file,
+				struct ib_uverbs_ioctl_hdr *hdr,
+				void __user *buf)
+{
+	const struct uverbs_object_spec *object_spec;
+	const struct uverbs_method_spec *method_spec;
+	long err = 0;
+	unsigned int i;
+	struct {
+		struct ib_uverbs_attr		*uattrs;
+		struct uverbs_attr_bundle	*uverbs_attr_bundle;
+	} *ctx = NULL;
+	struct uverbs_attr *curr_attr;
+	unsigned long *curr_bitmap;
+	size_t ctx_size;
+#ifdef UVERBS_OPTIMIZE_USING_STACK_SZ
+	uintptr_t data[UVERBS_OPTIMIZE_USING_STACK_SZ / sizeof(uintptr_t)];
+#endif
+
+	if (hdr->reserved)
+		return -EINVAL;
+
+	object_spec = uverbs_get_object(ib_dev, hdr->object_id);
+	if (!object_spec)
+		return -EOPNOTSUPP;
+
+	method_spec = uverbs_get_method(object_spec, hdr->method_id);
+	if (!method_spec)
+		return -EOPNOTSUPP;
+
+	if ((method_spec->flags & UVERBS_ACTION_FLAG_CREATE_ROOT) ^ !file->ucontext)
+		return -EINVAL;
+
+	ctx_size = sizeof(*ctx) +
+		   sizeof(struct uverbs_attr_bundle) +
+		   sizeof(struct uverbs_attr_bundle_hash) * method_spec->num_buckets +
+		   sizeof(*ctx->uattrs) * hdr->num_attrs +
+		   sizeof(*ctx->uverbs_attr_bundle->hash[0].attrs) *
+		   method_spec->num_child_attrs +
+		   sizeof(*ctx->uverbs_attr_bundle->hash[0].valid_bitmap) *
+			(method_spec->num_child_attrs / BITS_PER_LONG +
+			 method_spec->num_buckets);
+
+#ifdef UVERBS_OPTIMIZE_USING_STACK_SZ
+	if (ctx_size <= UVERBS_OPTIMIZE_USING_STACK_SZ)
+		ctx = (void *)data;
+
+	if (!ctx)
+#endif
+	ctx = kmalloc(ctx_size, GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+
+	ctx->uverbs_attr_bundle = (void *)ctx + sizeof(*ctx);
+	ctx->uattrs = (void *)(ctx->uverbs_attr_bundle + 1) +
+			      (sizeof(ctx->uverbs_attr_bundle->hash[0]) *
+			       method_spec->num_buckets);
+	curr_attr = (void *)(ctx->uattrs + hdr->num_attrs);
+	curr_bitmap = (void *)(curr_attr + method_spec->num_child_attrs);
+
+	/*
+	 * We just fill the pointers and num_attrs here. The data itself will be
+	 * filled at a later stage (uverbs_process_attr)
+	 */
+	for (i = 0; i < method_spec->num_buckets; i++) {
+		unsigned int curr_num_attrs = method_spec->attr_buckets[i]->num_attrs;
+
+		ctx->uverbs_attr_bundle->hash[i].attrs = curr_attr;
+		curr_attr += curr_num_attrs;
+		ctx->uverbs_attr_bundle->hash[i].num_attrs = curr_num_attrs;
+		ctx->uverbs_attr_bundle->hash[i].valid_bitmap = curr_bitmap;
+		bitmap_zero(curr_bitmap, curr_num_attrs);
+		curr_bitmap += BITS_TO_LONGS(curr_num_attrs);
+	}
+
+	err = copy_from_user(ctx->uattrs, buf,
+			     sizeof(*ctx->uattrs) * hdr->num_attrs);
+	if (err) {
+		err = -EFAULT;
+		goto out;
+	}
+
+	err = uverbs_handle_method(buf, ctx->uattrs, hdr->num_attrs, ib_dev,
+				   file, method_spec, ctx->uverbs_attr_bundle);
+out:
+#ifdef UVERBS_OPTIMIZE_USING_STACK_SZ
+	if (ctx_size > UVERBS_OPTIMIZE_USING_STACK_SZ)
+#endif
+	kfree(ctx);
+	return err;
+}
+
+#define IB_UVERBS_MAX_CMD_SZ 4096
+
+long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+{
+	struct ib_uverbs_file *file = filp->private_data;
+	struct ib_uverbs_ioctl_hdr __user *user_hdr =
+		(struct ib_uverbs_ioctl_hdr __user *)arg;
+	struct ib_uverbs_ioctl_hdr hdr;
+	struct ib_device *ib_dev;
+	int srcu_key;
+	long err;
+
+	srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
+	ib_dev = srcu_dereference(file->device->ib_dev,
+				  &file->device->disassociate_srcu);
+	if (!ib_dev) {
+		err = -EIO;
+		goto out;
+	}
+
+	if (cmd == RDMA_VERBS_IOCTL) {
+		err = copy_from_user(&hdr, user_hdr, sizeof(hdr));
+
+		if (err || hdr.length > IB_UVERBS_MAX_CMD_SZ ||
+		    hdr.length != sizeof(hdr) + hdr.num_attrs * sizeof(struct ib_uverbs_attr)) {
+			err = -EINVAL;
+			goto out;
+		}
+
+		if (hdr.reserved) {
+			err = -EOPNOTSUPP;
+			goto out;
+		}
+
+		err = ib_uverbs_cmd_verbs(ib_dev, file, &hdr,
+					  (__user void *)arg + sizeof(hdr));
+	} else {
+		err = -ENOIOCTLCMD;
+	}
+out:
+	srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
+
+	return err;
+}
diff --git a/drivers/infiniband/core/uverbs_ioctl_merge.c b/drivers/infiniband/core/uverbs_ioctl_merge.c
new file mode 100644
index 000000000000..76ddb6564578
--- /dev/null
+++ b/drivers/infiniband/core/uverbs_ioctl_merge.c
@@ -0,0 +1,665 @@
+/*
+ * Copyright (c) 2017, Mellanox Technologies inc.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <rdma/uverbs_ioctl.h>
+#include <rdma/rdma_user_ioctl.h>
+#include <linux/bitops.h>
+#include "uverbs.h"
+
+#define UVERBS_NUM_NS (UVERBS_ID_NS_MASK >> UVERBS_ID_NS_SHIFT)
+#define GET_NS_ID(idx) (((idx) & UVERBS_ID_NS_MASK) >> UVERBS_ID_NS_SHIFT)
+#define GET_ID(idx) ((idx) & ~UVERBS_ID_NS_MASK)
+
+#define _for_each_element(elem, tmpi, tmpj, hashes, num_buckets_offset,	       \
+			  buckets_offset)				       \
+	for (tmpj = 0,							       \
+	     elem = (*(const void ***)((hashes)[tmpi] +			       \
+				       (buckets_offset)))[0];	               \
+	     tmpj < *(size_t *)((hashes)[tmpi] + (num_buckets_offset));        \
+	     tmpj++)						               \
+		if ((elem = ((*(const void ***)(hashes[tmpi] +		       \
+						(buckets_offset)))[tmpj])))
+
+/*
+ * Iterate all elements of a few @hashes. The number of given hashes is
+ * indicated by @num_hashes. The offset of the number of buckets in the hash is
+ * represented by @num_buckets_offset, while the offset of the buckets array in
+ * the hash structure is represented by @buckets_offset. tmpi and tmpj are two
+ * short (or int) based indices that are given by the user. tmpi iterates over
+ * the different hashes. @elem points the current element in the hashes[tmpi]
+ * bucket we are looping on. To be honest, @hashes representation isn't exactly
+ * a hash, but more a collection of elements. These elements' ids are treated
+ * in a hash like manner, where the first upper bits are the bucket number.
+ * These elements are later mapped into a perfect-hash.
+ */
+#define for_each_element(elem, tmpi, tmpj, hashes, num_hashes,                 \
+			 num_buckets_offset, buckets_offset)		       \
+	for (tmpi = 0; tmpi < (num_hashes); tmpi++)		               \
+		_for_each_element(elem, tmpi, tmpj, hashes, num_buckets_offset,\
+				  buckets_offset)
+
+#define get_elements_iterators_entry_above(iters, num_elements, elements,     \
+					  num_objects_fld, objects_fld, bucket,\
+					  min_id)			       \
+	get_elements_above_id((const void **)iters, num_elements,       \
+				     (const void **)(elements),		       \
+				     offsetof(typeof(**elements),	       \
+					      num_objects_fld),		       \
+				     offsetof(typeof(**elements), objects_fld),\
+				     offsetof(typeof(***(*elements)->objects_fld), id),\
+				     bucket, min_id)
+
+#define get_objects_above_id(iters, num_trees, trees, bucket, min_id)	       \
+	get_elements_iterators_entry_above(iters, num_trees, trees,	       \
+					   num_objects, objects, bucket, min_id)
+
+#define get_methods_above_id(method_iters, num_iters, iters, bucket, min_id)\
+	get_elements_iterators_entry_above(method_iters, num_iters, iters,     \
+					   num_methods, methods, bucket, min_id)
+
+#define get_attrs_above_id(attrs_iters, num_iters, iters, bucket, min_id)\
+	get_elements_iterators_entry_above(attrs_iters, num_iters, iters,      \
+					   num_attrs, attrs, bucket, min_id)
+
+/*
+ * get_elements_above_id get a few hashes represented by @elements and
+ * @num_elements. The hashes fields are described by @num_offset, @data_offset
+ * and @id_offset in the same way as required by for_each_element. The function
+ * returns an array of @iters, represents an array of elements in the hashes
+ * buckets, which their ids are the smallest ids in all hashes but are all
+ * larger than the id given by min_id. Elements are only added to the iters
+ * array if their id belongs to the bucket @bucket. The number of elements in
+ * the returned array is returned by the function. @min_id is also updated to
+ * reflect the new min_id of all elements in iters.
+ */
+static size_t get_elements_above_id(const void **iters,
+				    unsigned int num_elements,
+				    const void **elements,
+				    size_t num_offset,
+				    size_t data_offset,
+				    size_t id_offset,
+				    u16 bucket,
+				    short *min_id)
+{
+	size_t num_iters = 0;
+	short min = SHRT_MAX;
+	const void *elem;
+	int i, j, last_stored = -1;
+
+	for_each_element(elem, i, j, elements, num_elements, num_offset,
+			 data_offset) {
+		u16 id = *(u16 *)(elem + id_offset);
+
+		if (GET_NS_ID(id) != bucket)
+			continue;
+
+		if (GET_ID(id) < *min_id ||
+		    (min != SHRT_MAX && GET_ID(id) > min))
+			continue;
+
+		/*
+		 * We first iterate all hashes represented by @elements. When
+		 * we do, we try to find an element @elem in the bucket @bucket
+		 * which its id is min. Since we can't ensure the user sorted
+		 * the elements in increasing order, we override this hash's
+		 * minimal id element we found, if a new element with a smaller
+		 * id was just found.
+		 */
+		iters[last_stored == i ? num_iters - 1 : num_iters++] = elem;
+		last_stored = i;
+		min = GET_ID(id);
+	}
+
+	/*
+	 * We only insert to our iters array an element, if its id is smaller
+	 * than all previous ids. Therefore, the final iters array is sorted so
+	 * that smaller ids are in the end of the array.
+	 * Therefore, we need to clean the beginning of the array to make sure
+	 * all ids of final elements are equal to min.
+	 */
+	for (i = num_iters - 1; i >= 0 &&
+	     GET_ID(*(u16 *)(iters[i] + id_offset)) == min; i--)
+		;
+
+	num_iters -= i + 1;
+	memmove(iters, iters + i + 1, sizeof(*iters) * num_iters);
+
+	*min_id = min;
+	return num_iters;
+}
+
+#define find_max_element_entry_id(num_elements, elements, num_objects_fld, \
+				  objects_fld, bucket)			   \
+	find_max_element_id(num_elements, (const void **)(elements),	   \
+			    offsetof(typeof(**elements), num_objects_fld),    \
+			    offsetof(typeof(**elements), objects_fld),	      \
+			    offsetof(typeof(***(*elements)->objects_fld), id),\
+			    bucket)
+
+static short find_max_element_ns_id(unsigned int num_elements,
+				    const void **elements,
+				    size_t num_offset,
+				    size_t data_offset,
+				    size_t id_offset)
+{
+	short max_ns = SHRT_MIN;
+	const void *elem;
+	int i, j;
+
+	for_each_element(elem, i, j, elements, num_elements, num_offset,
+			 data_offset) {
+		u16 id = *(u16 *)(elem + id_offset);
+
+		if (GET_NS_ID(id) > max_ns)
+			max_ns = GET_NS_ID(id);
+	}
+
+	return max_ns;
+}
+
+static short find_max_element_id(unsigned int num_elements,
+				 const void **elements,
+				 size_t num_offset,
+				 size_t data_offset,
+				 size_t id_offset,
+				 u16 bucket)
+{
+	short max_id = SHRT_MIN;
+	const void *elem;
+	int i, j;
+
+	for_each_element(elem, i, j, elements, num_elements, num_offset,
+			 data_offset) {
+		u16 id = *(u16 *)(elem + id_offset);
+
+		if (GET_NS_ID(id) == bucket &&
+		    GET_ID(id) > max_id)
+			max_id = GET_ID(id);
+	}
+	return max_id;
+}
+
+#define find_max_element_entry_id(num_elements, elements, num_objects_fld,   \
+				  objects_fld, bucket)			      \
+	find_max_element_id(num_elements, (const void **)(elements),	      \
+			    offsetof(typeof(**elements), num_objects_fld),    \
+			    offsetof(typeof(**elements), objects_fld),	      \
+			    offsetof(typeof(***(*elements)->objects_fld), id),\
+			    bucket)
+
+#define find_max_element_ns_entry_id(num_elements, elements,		    \
+				     num_objects_fld, objects_fld)	    \
+	find_max_element_ns_id(num_elements, (const void **)(elements),	    \
+			      offsetof(typeof(**elements), num_objects_fld),\
+			      offsetof(typeof(**elements), objects_fld),    \
+			      offsetof(typeof(***(*elements)->objects_fld), id))
+
+/*
+ * find_max_xxxx_ns_id gets a few elements. Each element is described by an id
+ * which its upper bits represents a namespace. It finds the max namespace. This
+ * could be used in order to know how many buckets do we need to allocate. If no
+ * elements exist, SHRT_MIN is returned. Namespace represents here different
+ * buckets. The common example is "common bucket" and "driver bucket".
+ *
+ * find_max_xxxx_id gets a few elements and a bucket. Each element is described
+ * by an id which its upper bits represent a namespace. It returns the max id
+ * which is contained in the same namespace defined in @bucket. This could be
+ * used in order to know how many elements do we need to allocate in the bucket.
+ * If no elements exist, SHRT_MIN is returned.
+ */
+
+#define find_max_object_id(num_trees, trees, bucket)			\
+		find_max_element_entry_id(num_trees, trees, num_objects,\
+					  objects, bucket)
+#define find_max_object_ns_id(num_trees, trees)			\
+		find_max_element_ns_entry_id(num_trees, trees,		\
+					     num_objects, objects)
+
+#define find_max_method_id(num_iters, iters, bucket)			\
+		find_max_element_entry_id(num_iters, iters, num_methods,\
+					  methods, bucket)
+#define find_max_method_ns_id(num_iters, iters)			\
+		find_max_element_ns_entry_id(num_iters, iters,		\
+					     num_methods, methods)
+
+#define find_max_attr_id(num_iters, iters, bucket)			\
+		find_max_element_entry_id(num_iters, iters, num_attrs,  \
+					  attrs, bucket)
+#define find_max_attr_ns_id(num_iters, iters)				\
+		find_max_element_ns_entry_id(num_iters, iters,		\
+					     num_attrs, attrs)
+
+static void free_method(struct uverbs_method_spec *method)
+{
+	unsigned int i;
+
+	if (!method)
+		return;
+
+	for (i = 0; i < method->num_buckets; i++)
+		kfree(method->attr_buckets[i]);
+
+	kfree(method);
+}
+
+#define IS_ATTR_OBJECT(attr) ((attr)->type == UVERBS_ATTR_TYPE_IDR || \
+			      (attr)->type == UVERBS_ATTR_TYPE_FD)
+
+/*
+ * This function gets array of size @num_method_defs which contains pointers to
+ * method definitions @method_defs. The function allocates an
+ * uverbs_method_spec structure and initializes its number of buckets and the
+ * elements in buckets to the correct attributes. While doing that, it
+ * validates that there aren't conflicts between attributes of different
+ * method_defs.
+ */
+static struct uverbs_method_spec *build_method_with_attrs(const struct uverbs_method_def **method_defs,
+							  size_t num_method_defs)
+{
+	int bucket_idx;
+	int max_attr_buckets = 0;
+	size_t num_attr_buckets = 0;
+	int res = 0;
+	struct uverbs_method_spec *method = NULL;
+	const struct uverbs_attr_def **attr_defs;
+	unsigned int num_of_singularities = 0;
+
+	max_attr_buckets = find_max_attr_ns_id(num_method_defs, method_defs);
+	if (max_attr_buckets >= 0)
+		num_attr_buckets = max_attr_buckets + 1;
+
+	method = kzalloc(sizeof(*method) +
+			 num_attr_buckets * sizeof(*method->attr_buckets),
+			 GFP_KERNEL);
+	if (!method)
+		return ERR_PTR(-ENOMEM);
+
+	method->num_buckets = num_attr_buckets;
+	attr_defs = kcalloc(num_method_defs, sizeof(*attr_defs), GFP_KERNEL);
+	if (!attr_defs) {
+		res = -ENOMEM;
+		goto free_method;
+	}
+	for (bucket_idx = 0; bucket_idx < method->num_buckets; bucket_idx++) {
+		short min_id = SHRT_MIN;
+		int attr_max_bucket = 0;
+		struct uverbs_attr_spec_hash *hash = NULL;
+
+		attr_max_bucket = find_max_attr_id(num_method_defs, method_defs,
+						   bucket_idx);
+		if (attr_max_bucket < 0)
+			continue;
+
+		hash = kzalloc(sizeof(*hash) +
+			       ALIGN(sizeof(*hash->attrs) * (attr_max_bucket + 1),
+				     sizeof(long)) +
+			       BITS_TO_LONGS(attr_max_bucket) * sizeof(long),
+			       GFP_KERNEL);
+		if (!hash) {
+			res = -ENOMEM;
+			goto free;
+		}
+		hash->num_attrs = attr_max_bucket + 1;
+		method->num_child_attrs += hash->num_attrs;
+		hash->mandatory_attrs_bitmask = (void *)(hash + 1) +
+						 ALIGN(sizeof(*hash->attrs) *
+						       (attr_max_bucket + 1),
+						       sizeof(long));
+
+		method->attr_buckets[bucket_idx] = hash;
+
+		do {
+			size_t			 num_attr_defs;
+			struct uverbs_attr_spec	*attr;
+			bool attr_obj_with_special_access;
+
+			num_attr_defs =
+				get_attrs_above_id(attr_defs,
+						   num_method_defs,
+						   method_defs,
+						   bucket_idx,
+						   &min_id);
+			/* Last attr in bucket */
+			if (!num_attr_defs)
+				break;
+
+			if (num_attr_defs > 1) {
+				/*
+				 * We don't allow two attribute definitions for
+				 * the same attribute. This is usually a
+				 * programmer error. If required, it's better to
+				 * just add a new attribute to capture the new
+				 * semantics.
+				 */
+				res = -EEXIST;
+				goto free;
+			}
+
+			attr = &hash->attrs[min_id];
+			memcpy(attr, &attr_defs[0]->attr, sizeof(*attr));
+
+			attr_obj_with_special_access = IS_ATTR_OBJECT(attr) &&
+				   (attr->obj.access == UVERBS_ACCESS_NEW ||
+				    attr->obj.access == UVERBS_ACCESS_DESTROY);
+			num_of_singularities +=  !!attr_obj_with_special_access;
+			if (WARN(num_of_singularities > 1,
+				 "ib_uverbs: Method contains more than one object attr (%d) with new/destroy access\n",
+				 min_id) ||
+			    WARN(attr_obj_with_special_access &&
+				 !(attr->flags & UVERBS_ATTR_SPEC_F_MANDATORY),
+				 "ib_uverbs: Tried to merge attr (%d) but it's an object with new/destroy aceess but isn't mandatory\n",
+				 min_id) ||
+			    WARN(IS_ATTR_OBJECT(attr) &&
+				 attr->flags & UVERBS_ATTR_SPEC_F_MIN_SZ,
+				 "ib_uverbs: Tried to merge attr (%d) but it's an object with min_sz flag\n",
+				 min_id)) {
+				res = -EINVAL;
+				goto free;
+			}
+
+			if (attr->flags & UVERBS_ATTR_SPEC_F_MANDATORY)
+				set_bit(min_id, hash->mandatory_attrs_bitmask);
+			min_id++;
+
+		} while (1);
+	}
+	kfree(attr_defs);
+	return method;
+
+free:
+	kfree(attr_defs);
+free_method:
+	free_method(method);
+	return ERR_PTR(res);
+}
+
+static void free_object(struct uverbs_object_spec *object)
+{
+	unsigned int i, j;
+
+	if (!object)
+		return;
+
+	for (i = 0; i < object->num_buckets; i++) {
+		struct uverbs_method_spec_hash	*method_buckets =
+			object->method_buckets[i];
+
+		if (!method_buckets)
+			continue;
+
+		for (j = 0; j < method_buckets->num_methods; j++)
+			free_method(method_buckets->methods[j]);
+
+		kfree(method_buckets);
+	}
+
+	kfree(object);
+}
+
+/*
+ * This function gets array of size @num_object_defs which contains pointers to
+ * object definitions @object_defs. The function allocated an
+ * uverbs_object_spec structure and initialize its number of buckets and the
+ * elements in buckets to the correct methods. While doing that, it
+ * sorts out the correct relationship between conflicts in the same method.
+ */
+static struct uverbs_object_spec *build_object_with_methods(const struct uverbs_object_def **object_defs,
+							    size_t num_object_defs)
+{
+	u16 bucket_idx;
+	int max_method_buckets = 0;
+	u16 num_method_buckets = 0;
+	int res = 0;
+	struct uverbs_object_spec *object = NULL;
+	const struct uverbs_method_def **method_defs;
+
+	max_method_buckets = find_max_method_ns_id(num_object_defs, object_defs);
+	if (max_method_buckets >= 0)
+		num_method_buckets = max_method_buckets + 1;
+
+	object = kzalloc(sizeof(*object) +
+			 num_method_buckets *
+			 sizeof(*object->method_buckets), GFP_KERNEL);
+	if (!object)
+		return ERR_PTR(-ENOMEM);
+
+	object->num_buckets = num_method_buckets;
+	method_defs = kcalloc(num_object_defs, sizeof(*method_defs), GFP_KERNEL);
+	if (!method_defs) {
+		res = -ENOMEM;
+		goto free_object;
+	}
+
+	for (bucket_idx = 0; bucket_idx < object->num_buckets; bucket_idx++) {
+		short min_id = SHRT_MIN;
+		int methods_max_bucket = 0;
+		struct uverbs_method_spec_hash *hash = NULL;
+
+		methods_max_bucket = find_max_method_id(num_object_defs, object_defs,
+							bucket_idx);
+		if (methods_max_bucket < 0)
+			continue;
+
+		hash = kzalloc(sizeof(*hash) +
+			       sizeof(*hash->methods) * (methods_max_bucket + 1),
+			       GFP_KERNEL);
+		if (!hash) {
+			res = -ENOMEM;
+			goto free;
+		}
+
+		hash->num_methods = methods_max_bucket + 1;
+		object->method_buckets[bucket_idx] = hash;
+
+		do {
+			size_t				num_method_defs;
+			struct uverbs_method_spec	*method;
+			int i;
+
+			num_method_defs =
+				get_methods_above_id(method_defs,
+						     num_object_defs,
+						     object_defs,
+						     bucket_idx,
+						     &min_id);
+			/* Last method in bucket */
+			if (!num_method_defs)
+				break;
+
+			method = build_method_with_attrs(method_defs,
+							 num_method_defs);
+			if (IS_ERR(method)) {
+				res = PTR_ERR(method);
+				goto free;
+			}
+
+			/*
+			 * The last tree which is given as an argument to the
+			 * merge overrides previous method handler.
+			 * Therefore, we iterate backwards and search for the
+			 * first handler which != NULL. This also defines the
+			 * set of flags used for this handler.
+			 */
+			for (i = num_object_defs - 1;
+			     i >= 0 && !method_defs[i]->handler; i--)
+				;
+			hash->methods[min_id++] = method;
+			/* NULL handler isn't allowed */
+			if (WARN(i < 0,
+				 "ib_uverbs: tried to merge function id %d, but all handlers are NULL\n",
+				 min_id)) {
+				res = -EINVAL;
+				goto free;
+			}
+			method->handler = method_defs[i]->handler;
+			method->flags = method_defs[i]->flags;
+
+		} while (1);
+	}
+	kfree(method_defs);
+	return object;
+
+free:
+	kfree(method_defs);
+free_object:
+	free_object(object);
+	return ERR_PTR(res);
+}
+
+void uverbs_free_spec_tree(struct uverbs_root_spec *root)
+{
+	unsigned int i, j;
+
+	if (!root)
+		return;
+
+	for (i = 0; i < root->num_buckets; i++) {
+		struct uverbs_object_spec_hash *object_hash =
+			root->object_buckets[i];
+
+		if (!object_hash)
+			continue;
+
+		for (j = 0; j < object_hash->num_objects; j++)
+			free_object(object_hash->objects[j]);
+
+		kfree(object_hash);
+	}
+
+	kfree(root);
+}
+EXPORT_SYMBOL(uverbs_free_spec_tree);
+
+struct uverbs_root_spec *uverbs_alloc_spec_tree(unsigned int num_trees,
+						const struct uverbs_object_tree_def **trees)
+{
+	u16 bucket_idx;
+	short max_object_buckets = 0;
+	size_t num_objects_buckets = 0;
+	struct uverbs_root_spec *root_spec = NULL;
+	const struct uverbs_object_def **object_defs;
+	int i;
+	int res = 0;
+
+	max_object_buckets = find_max_object_ns_id(num_trees, trees);
+	/*
+	 * Devices which don't want to support ib_uverbs, should just allocate
+	 * an empty parsing tree. Every user-space command won't hit any valid
+	 * entry in the parsing tree and thus will fail.
+	 */
+	if (max_object_buckets >= 0)
+		num_objects_buckets = max_object_buckets + 1;
+
+	root_spec = kzalloc(sizeof(*root_spec) +
+			    num_objects_buckets * sizeof(*root_spec->object_buckets),
+			    GFP_KERNEL);
+	if (!root_spec)
+		return ERR_PTR(-ENOMEM);
+	root_spec->num_buckets = num_objects_buckets;
+
+	object_defs = kcalloc(num_trees, sizeof(*object_defs),
+			      GFP_KERNEL);
+	if (!object_defs) {
+		res = -ENOMEM;
+		goto free_root;
+	}
+
+	for (bucket_idx = 0; bucket_idx < root_spec->num_buckets; bucket_idx++) {
+		short min_id = SHRT_MIN;
+		short objects_max_bucket;
+		struct uverbs_object_spec_hash *hash = NULL;
+
+		objects_max_bucket = find_max_object_id(num_trees, trees,
+							bucket_idx);
+		if (objects_max_bucket < 0)
+			continue;
+
+		hash = kzalloc(sizeof(*hash) +
+			       sizeof(*hash->objects) * (objects_max_bucket + 1),
+			       GFP_KERNEL);
+		if (!hash) {
+			res = -ENOMEM;
+			goto free;
+		}
+		hash->num_objects = objects_max_bucket + 1;
+		root_spec->object_buckets[bucket_idx] = hash;
+
+		do {
+			size_t				num_object_defs;
+			struct uverbs_object_spec	*object;
+
+			num_object_defs = get_objects_above_id(object_defs,
+							       num_trees,
+							       trees,
+							       bucket_idx,
+							       &min_id);
+			/* Last object in bucket */
+			if (!num_object_defs)
+				break;
+
+			object = build_object_with_methods(object_defs,
+							   num_object_defs);
+			if (IS_ERR(object)) {
+				res = PTR_ERR(object);
+				goto free;
+			}
+
+			/*
+			 * The last tree which is given as an argument to the
+			 * merge overrides previous object's type_attrs.
+			 * Therefore, we iterate backwards and search for the
+			 * first type_attrs which != NULL.
+			 */
+			for (i = num_object_defs - 1;
+			     i >= 0 && !object_defs[i]->type_attrs; i--)
+				;
+			/*
+			 * NULL is a valid type_attrs. It means an object we
+			 * can't instantiate (like DEVICE).
+			 */
+			object->type_attrs = i < 0 ? NULL :
+				object_defs[i]->type_attrs;
+
+			hash->objects[min_id++] = object;
+		} while (1);
+	}
+
+	kfree(object_defs);
+	return root_spec;
+
+free:
+	kfree(object_defs);
+free_root:
+	uverbs_free_spec_tree(root_spec);
+	return ERR_PTR(res);
+}
+EXPORT_SYMBOL(uverbs_alloc_spec_tree);
diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c
index c023e2c81b8f..dc2aed6fb21b 100644
--- a/drivers/infiniband/core/uverbs_main.c
+++ b/drivers/infiniband/core/uverbs_main.c
@@ -49,6 +49,7 @@
 #include <linux/uaccess.h>
 
 #include <rdma/ib.h>
+#include <rdma/uverbs_std_types.h>
 
 #include "uverbs.h"
 #include "core_priv.h"
@@ -595,7 +596,6 @@ struct file *ib_uverbs_alloc_async_event_file(struct ib_uverbs_file *uverbs_file
 {
 	struct ib_uverbs_async_event_file *ev_file;
 	struct file *filp;
-	int ret;
 
 	ev_file = kzalloc(sizeof(*ev_file), GFP_KERNEL);
 	if (!ev_file)
@@ -621,21 +621,11 @@ struct file *ib_uverbs_alloc_async_event_file(struct ib_uverbs_file *uverbs_file
 	INIT_IB_EVENT_HANDLER(&uverbs_file->event_handler,
 			      ib_dev,
 			      ib_uverbs_event_handler);
-	ret = ib_register_event_handler(&uverbs_file->event_handler);
-	if (ret)
-		goto err_put_file;
-
+	ib_register_event_handler(&uverbs_file->event_handler);
 	/* At that point async file stuff was fully set */
 
 	return filp;
 
-err_put_file:
-	fput(filp);
-	kref_put(&uverbs_file->async_file->ref,
-		 ib_uverbs_release_async_event_file);
-	uverbs_file->async_file = NULL;
-	return ERR_PTR(ret);
-
 err_put_refs:
 	kref_put(&ev_file->uverbs_file->ref, ib_uverbs_release_file);
 	kref_put(&ev_file->ref, ib_uverbs_release_async_event_file);
@@ -949,6 +939,9 @@ static const struct file_operations uverbs_fops = {
 	.open	 = ib_uverbs_open,
 	.release = ib_uverbs_close,
 	.llseek	 = no_llseek,
+#if IS_ENABLED(CONFIG_INFINIBAND_EXP_USER_ACCESS)
+	.unlocked_ioctl = ib_uverbs_ioctl,
+#endif
 };
 
 static const struct file_operations uverbs_mmap_fops = {
@@ -958,6 +951,9 @@ static const struct file_operations uverbs_mmap_fops = {
 	.open	 = ib_uverbs_open,
 	.release = ib_uverbs_close,
 	.llseek	 = no_llseek,
+#if IS_ENABLED(CONFIG_INFINIBAND_EXP_USER_ACCESS)
+	.unlocked_ioctl = ib_uverbs_ioctl,
+#endif
 };
 
 static struct ib_client uverbs_client = {
@@ -1108,6 +1104,18 @@ static void ib_uverbs_add_one(struct ib_device *device)
 	if (device_create_file(uverbs_dev->dev, &dev_attr_abi_version))
 		goto err_class;
 
+	if (!device->specs_root) {
+		const struct uverbs_object_tree_def *default_root[] = {
+			uverbs_default_get_objects()};
+
+		uverbs_dev->specs_root = uverbs_alloc_spec_tree(1,
+								default_root);
+		if (IS_ERR(uverbs_dev->specs_root))
+			goto err_class;
+
+		device->specs_root = uverbs_dev->specs_root;
+	}
+
 	ib_set_client_data(device, &uverbs_client, uverbs_dev);
 
 	return;
@@ -1153,7 +1161,6 @@ static void ib_uverbs_free_hw_resources(struct ib_uverbs_device *uverbs_dev,
 		kref_get(&file->ref);
 		mutex_unlock(&uverbs_dev->lists_mutex);
 
-		ib_uverbs_event_handler(&file->event_handler, &event);
 
 		mutex_lock(&file->cleanup_mutex);
 		ucontext = file->ucontext;
@@ -1170,6 +1177,7 @@ static void ib_uverbs_free_hw_resources(struct ib_uverbs_device *uverbs_dev,
 			 * for example due to freeing the resources
 			 * (e.g mmput).
 			 */
+			ib_uverbs_event_handler(&file->event_handler, &event);
 			ib_dev->disassociate_ucontext(ucontext);
 			mutex_lock(&file->cleanup_mutex);
 			ib_uverbs_cleanup_ucontext(file, ucontext, true);
@@ -1239,6 +1247,11 @@ static void ib_uverbs_remove_one(struct ib_device *device, void *client_data)
 		ib_uverbs_comp_dev(uverbs_dev);
 	if (wait_clients)
 		wait_for_completion(&uverbs_dev->comp);
+	if (uverbs_dev->specs_root) {
+		uverbs_free_spec_tree(uverbs_dev->specs_root);
+		device->specs_root = NULL;
+	}
+
 	kobject_put(&uverbs_dev->kobj);
 }
 
diff --git a/drivers/infiniband/core/uverbs_marshall.c b/drivers/infiniband/core/uverbs_marshall.c
index 94fd989c9060..bd0acf376af0 100644
--- a/drivers/infiniband/core/uverbs_marshall.c
+++ b/drivers/infiniband/core/uverbs_marshall.c
@@ -33,10 +33,47 @@
 #include <linux/export.h>
 #include <rdma/ib_marshall.h>
 
-void ib_copy_ah_attr_to_user(struct ib_uverbs_ah_attr *dst,
-			     struct rdma_ah_attr *src)
+#define OPA_DEFAULT_GID_PREFIX cpu_to_be64(0xfe80000000000000ULL)
+static int rdma_ah_conv_opa_to_ib(struct ib_device *dev,
+				  struct rdma_ah_attr *ib,
+				  struct rdma_ah_attr *opa)
 {
+	struct ib_port_attr port_attr;
+	int ret = 0;
+
+	/* Do structure copy and the over-write fields */
+	*ib = *opa;
+
+	ib->type = RDMA_AH_ATTR_TYPE_IB;
+	rdma_ah_set_grh(ib, NULL, 0, 0, 1, 0);
+
+	if (ib_query_port(dev, opa->port_num, &port_attr)) {
+		/* Set to default subnet to indicate error */
+		rdma_ah_set_subnet_prefix(ib, OPA_DEFAULT_GID_PREFIX);
+		ret = -EINVAL;
+	} else {
+		rdma_ah_set_subnet_prefix(ib,
+					  cpu_to_be64(port_attr.subnet_prefix));
+	}
+	rdma_ah_set_interface_id(ib, OPA_MAKE_ID(rdma_ah_get_dlid(opa)));
+	return ret;
+}
+
+void ib_copy_ah_attr_to_user(struct ib_device *device,
+			     struct ib_uverbs_ah_attr *dst,
+			     struct rdma_ah_attr *ah_attr)
+{
+	struct rdma_ah_attr *src = ah_attr;
+	struct rdma_ah_attr conv_ah;
+
 	memset(&dst->grh.reserved, 0, sizeof(dst->grh.reserved));
+
+	if ((ah_attr->type == RDMA_AH_ATTR_TYPE_OPA) &&
+	    (rdma_ah_get_dlid(ah_attr) >=
+	     be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
+	    (!rdma_ah_conv_opa_to_ib(device, &conv_ah, ah_attr)))
+		src = &conv_ah;
+
 	dst->dlid		   = rdma_ah_get_dlid(src);
 	dst->sl			   = rdma_ah_get_sl(src);
 	dst->src_path_bits	   = rdma_ah_get_path_bits(src);
@@ -57,7 +94,8 @@ void ib_copy_ah_attr_to_user(struct ib_uverbs_ah_attr *dst,
 }
 EXPORT_SYMBOL(ib_copy_ah_attr_to_user);
 
-void ib_copy_qp_attr_to_user(struct ib_uverbs_qp_attr *dst,
+void ib_copy_qp_attr_to_user(struct ib_device *device,
+			     struct ib_uverbs_qp_attr *dst,
 			     struct ib_qp_attr *src)
 {
 	dst->qp_state	        = src->qp_state;
@@ -76,8 +114,8 @@ void ib_copy_qp_attr_to_user(struct ib_uverbs_qp_attr *dst,
 	dst->max_recv_sge	= src->cap.max_recv_sge;
 	dst->max_inline_data	= src->cap.max_inline_data;
 
-	ib_copy_ah_attr_to_user(&dst->ah_attr, &src->ah_attr);
-	ib_copy_ah_attr_to_user(&dst->alt_ah_attr, &src->alt_ah_attr);
+	ib_copy_ah_attr_to_user(device, &dst->ah_attr, &src->ah_attr);
+	ib_copy_ah_attr_to_user(device, &dst->alt_ah_attr, &src->alt_ah_attr);
 
 	dst->pkey_index		= src->pkey_index;
 	dst->alt_pkey_index	= src->alt_pkey_index;
diff --git a/drivers/infiniband/core/uverbs_std_types.c b/drivers/infiniband/core/uverbs_std_types.c
index ef293379f37a..0a98579700ec 100644
--- a/drivers/infiniband/core/uverbs_std_types.c
+++ b/drivers/infiniband/core/uverbs_std_types.c
@@ -209,67 +209,244 @@ static int uverbs_hot_unplug_completion_event_file(struct ib_uobject_file *uobj_
 	return 0;
 };
 
-const struct uverbs_obj_fd_type uverbs_type_attrs_comp_channel = {
-	.type = UVERBS_TYPE_ALLOC_FD(sizeof(struct ib_uverbs_completion_event_file), 0),
-	.context_closed = uverbs_hot_unplug_completion_event_file,
-	.fops = &uverbs_event_fops,
-	.name = "[infinibandevent]",
-	.flags = O_RDONLY,
-};
+/*
+ * This spec is used in order to pass information to the hardware driver in a
+ * legacy way. Every verb that could get driver specific data should get this
+ * spec.
+ */
+static const struct uverbs_attr_def uverbs_uhw_compat_in =
+	UVERBS_ATTR_PTR_IN_SZ(UVERBS_UHW_IN, 0, UA_FLAGS(UVERBS_ATTR_SPEC_F_MIN_SZ));
+static const struct uverbs_attr_def uverbs_uhw_compat_out =
+	UVERBS_ATTR_PTR_OUT_SZ(UVERBS_UHW_OUT, 0, UA_FLAGS(UVERBS_ATTR_SPEC_F_MIN_SZ));
 
-const struct uverbs_obj_idr_type uverbs_type_attrs_cq = {
-	.type = UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_ucq_object), 0),
-	.destroy_object = uverbs_free_cq,
-};
+static void create_udata(struct uverbs_attr_bundle *ctx,
+			 struct ib_udata *udata)
+{
+	/*
+	 * This is for ease of conversion. The purpose is to convert all drivers
+	 * to use uverbs_attr_bundle instead of ib_udata.
+	 * Assume attr == 0 is input and attr == 1 is output.
+	 */
+	void __user *inbuf;
+	size_t inbuf_len = 0;
+	void __user *outbuf;
+	size_t outbuf_len = 0;
+	const struct uverbs_attr *uhw_in =
+		uverbs_attr_get(ctx, UVERBS_UHW_IN);
+	const struct uverbs_attr *uhw_out =
+		uverbs_attr_get(ctx, UVERBS_UHW_OUT);
+
+	if (!IS_ERR(uhw_in)) {
+		inbuf = uhw_in->ptr_attr.ptr;
+		inbuf_len = uhw_in->ptr_attr.len;
+	}
 
-const struct uverbs_obj_idr_type uverbs_type_attrs_qp = {
-	.type = UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uqp_object), 0),
-	.destroy_object = uverbs_free_qp,
-};
+	if (!IS_ERR(uhw_out)) {
+		outbuf = uhw_out->ptr_attr.ptr;
+		outbuf_len = uhw_out->ptr_attr.len;
+	}
 
-const struct uverbs_obj_idr_type uverbs_type_attrs_mw = {
-	.type = UVERBS_TYPE_ALLOC_IDR(0),
-	.destroy_object = uverbs_free_mw,
-};
+	INIT_UDATA_BUF_OR_NULL(udata, inbuf, outbuf, inbuf_len, outbuf_len);
+}
 
-const struct uverbs_obj_idr_type uverbs_type_attrs_mr = {
-	/* 1 is used in order to free the MR after all the MWs */
-	.type = UVERBS_TYPE_ALLOC_IDR(1),
-	.destroy_object = uverbs_free_mr,
-};
+static int uverbs_create_cq_handler(struct ib_device *ib_dev,
+				    struct ib_uverbs_file *file,
+				    struct uverbs_attr_bundle *attrs)
+{
+	struct ib_ucontext *ucontext = file->ucontext;
+	struct ib_ucq_object           *obj;
+	struct ib_udata uhw;
+	int ret;
+	u64 user_handle;
+	struct ib_cq_init_attr attr = {};
+	struct ib_cq                   *cq;
+	struct ib_uverbs_completion_event_file    *ev_file = NULL;
+	const struct uverbs_attr *ev_file_attr;
+	struct ib_uobject *ev_file_uobj;
+
+	if (!(ib_dev->uverbs_cmd_mask & 1ULL << IB_USER_VERBS_CMD_CREATE_CQ))
+		return -EOPNOTSUPP;
+
+	ret = uverbs_copy_from(&attr.comp_vector, attrs, CREATE_CQ_COMP_VECTOR);
+	if (!ret)
+		ret = uverbs_copy_from(&attr.cqe, attrs, CREATE_CQ_CQE);
+	if (!ret)
+		ret = uverbs_copy_from(&user_handle, attrs, CREATE_CQ_USER_HANDLE);
+	if (ret)
+		return ret;
 
-const struct uverbs_obj_idr_type uverbs_type_attrs_srq = {
-	.type = UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_usrq_object), 0),
-	.destroy_object = uverbs_free_srq,
-};
+	/* Optional param, if it doesn't exist, we get -ENOENT and skip it */
+	if (uverbs_copy_from(&attr.flags, attrs, CREATE_CQ_FLAGS) == -EFAULT)
+		return -EFAULT;
 
-const struct uverbs_obj_idr_type uverbs_type_attrs_ah = {
-	.type = UVERBS_TYPE_ALLOC_IDR(0),
-	.destroy_object = uverbs_free_ah,
-};
+	ev_file_attr = uverbs_attr_get(attrs, CREATE_CQ_COMP_CHANNEL);
+	if (!IS_ERR(ev_file_attr)) {
+		ev_file_uobj = ev_file_attr->obj_attr.uobject;
 
-const struct uverbs_obj_idr_type uverbs_type_attrs_flow = {
-	.type = UVERBS_TYPE_ALLOC_IDR(0),
-	.destroy_object = uverbs_free_flow,
-};
+		ev_file = container_of(ev_file_uobj,
+				       struct ib_uverbs_completion_event_file,
+				       uobj_file.uobj);
+		uverbs_uobject_get(ev_file_uobj);
+	}
 
-const struct uverbs_obj_idr_type uverbs_type_attrs_wq = {
-	.type = UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uwq_object), 0),
-	.destroy_object = uverbs_free_wq,
-};
+	if (attr.comp_vector >= ucontext->ufile->device->num_comp_vectors) {
+		ret = -EINVAL;
+		goto err_event_file;
+	}
 
-const struct uverbs_obj_idr_type uverbs_type_attrs_rwq_ind_table = {
-	.type = UVERBS_TYPE_ALLOC_IDR(0),
-	.destroy_object = uverbs_free_rwq_ind_tbl,
-};
+	obj = container_of(uverbs_attr_get(attrs, CREATE_CQ_HANDLE)->obj_attr.uobject,
+			   typeof(*obj), uobject);
+	obj->uverbs_file	   = ucontext->ufile;
+	obj->comp_events_reported  = 0;
+	obj->async_events_reported = 0;
+	INIT_LIST_HEAD(&obj->comp_list);
+	INIT_LIST_HEAD(&obj->async_list);
+
+	/* Temporary, only until drivers get the new uverbs_attr_bundle */
+	create_udata(attrs, &uhw);
+
+	cq = ib_dev->create_cq(ib_dev, &attr, ucontext, &uhw);
+	if (IS_ERR(cq)) {
+		ret = PTR_ERR(cq);
+		goto err_event_file;
+	}
 
-const struct uverbs_obj_idr_type uverbs_type_attrs_xrcd = {
-	.type = UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uxrcd_object), 0),
-	.destroy_object = uverbs_free_xrcd,
-};
+	cq->device        = ib_dev;
+	cq->uobject       = &obj->uobject;
+	cq->comp_handler  = ib_uverbs_comp_handler;
+	cq->event_handler = ib_uverbs_cq_event_handler;
+	cq->cq_context    = &ev_file->ev_queue;
+	obj->uobject.object = cq;
+	obj->uobject.user_handle = user_handle;
+	atomic_set(&cq->usecnt, 0);
+
+	ret = uverbs_copy_to(attrs, CREATE_CQ_RESP_CQE, &cq->cqe);
+	if (ret)
+		goto err_cq;
 
-const struct uverbs_obj_idr_type uverbs_type_attrs_pd = {
-	/* 2 is used in order to free the PD after MRs */
-	.type = UVERBS_TYPE_ALLOC_IDR(2),
-	.destroy_object = uverbs_free_pd,
+	return 0;
+err_cq:
+	ib_destroy_cq(cq);
+
+err_event_file:
+	if (ev_file)
+		uverbs_uobject_put(ev_file_uobj);
+	return ret;
 };
+
+static DECLARE_UVERBS_METHOD(
+	uverbs_method_cq_create, UVERBS_CQ_CREATE, uverbs_create_cq_handler,
+	&UVERBS_ATTR_IDR(CREATE_CQ_HANDLE, UVERBS_OBJECT_CQ, UVERBS_ACCESS_NEW,
+			 UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)),
+	&UVERBS_ATTR_PTR_IN(CREATE_CQ_CQE, u32,
+			    UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)),
+	&UVERBS_ATTR_PTR_IN(CREATE_CQ_USER_HANDLE, u64,
+			    UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)),
+	&UVERBS_ATTR_FD(CREATE_CQ_COMP_CHANNEL, UVERBS_OBJECT_COMP_CHANNEL,
+			UVERBS_ACCESS_READ),
+	&UVERBS_ATTR_PTR_IN(CREATE_CQ_COMP_VECTOR, u32,
+			    UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)),
+	&UVERBS_ATTR_PTR_IN(CREATE_CQ_FLAGS, u32),
+	&UVERBS_ATTR_PTR_OUT(CREATE_CQ_RESP_CQE, u32,
+			     UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)),
+	&uverbs_uhw_compat_in, &uverbs_uhw_compat_out);
+
+static int uverbs_destroy_cq_handler(struct ib_device *ib_dev,
+				     struct ib_uverbs_file *file,
+				     struct uverbs_attr_bundle *attrs)
+{
+	struct ib_uverbs_destroy_cq_resp resp;
+	struct ib_uobject *uobj =
+		uverbs_attr_get(attrs, DESTROY_CQ_HANDLE)->obj_attr.uobject;
+	struct ib_ucq_object *obj = container_of(uobj, struct ib_ucq_object,
+						 uobject);
+	int ret;
+
+	if (!(ib_dev->uverbs_cmd_mask & 1ULL << IB_USER_VERBS_CMD_DESTROY_CQ))
+		return -EOPNOTSUPP;
+
+	ret = rdma_explicit_destroy(uobj);
+	if (ret)
+		return ret;
+
+	resp.comp_events_reported  = obj->comp_events_reported;
+	resp.async_events_reported = obj->async_events_reported;
+
+	return uverbs_copy_to(attrs, DESTROY_CQ_RESP, &resp);
+}
+
+static DECLARE_UVERBS_METHOD(
+	uverbs_method_cq_destroy, UVERBS_CQ_DESTROY, uverbs_destroy_cq_handler,
+	&UVERBS_ATTR_IDR(DESTROY_CQ_HANDLE, UVERBS_OBJECT_CQ,
+			 UVERBS_ACCESS_DESTROY,
+			 UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)),
+	&UVERBS_ATTR_PTR_OUT(DESTROY_CQ_RESP, struct ib_uverbs_destroy_cq_resp,
+			     UA_FLAGS(UVERBS_ATTR_SPEC_F_MANDATORY)));
+
+DECLARE_UVERBS_OBJECT(uverbs_object_comp_channel,
+		      UVERBS_OBJECT_COMP_CHANNEL,
+		      &UVERBS_TYPE_ALLOC_FD(0,
+					      sizeof(struct ib_uverbs_completion_event_file),
+					      uverbs_hot_unplug_completion_event_file,
+					      &uverbs_event_fops,
+					      "[infinibandevent]", O_RDONLY));
+
+DECLARE_UVERBS_OBJECT(uverbs_object_cq, UVERBS_OBJECT_CQ,
+		      &UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_ucq_object), 0,
+						  uverbs_free_cq),
+		      &uverbs_method_cq_create,
+		      &uverbs_method_cq_destroy);
+
+DECLARE_UVERBS_OBJECT(uverbs_object_qp, UVERBS_OBJECT_QP,
+		      &UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uqp_object), 0,
+						  uverbs_free_qp));
+
+DECLARE_UVERBS_OBJECT(uverbs_object_mw, UVERBS_OBJECT_MW,
+		      &UVERBS_TYPE_ALLOC_IDR(0, uverbs_free_mw));
+
+DECLARE_UVERBS_OBJECT(uverbs_object_mr, UVERBS_OBJECT_MR,
+		      /* 1 is used in order to free the MR after all the MWs */
+		      &UVERBS_TYPE_ALLOC_IDR(1, uverbs_free_mr));
+
+DECLARE_UVERBS_OBJECT(uverbs_object_srq, UVERBS_OBJECT_SRQ,
+		      &UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_usrq_object), 0,
+						  uverbs_free_srq));
+
+DECLARE_UVERBS_OBJECT(uverbs_object_ah, UVERBS_OBJECT_AH,
+		      &UVERBS_TYPE_ALLOC_IDR(0, uverbs_free_ah));
+
+DECLARE_UVERBS_OBJECT(uverbs_object_flow, UVERBS_OBJECT_FLOW,
+		      &UVERBS_TYPE_ALLOC_IDR(0, uverbs_free_flow));
+
+DECLARE_UVERBS_OBJECT(uverbs_object_wq, UVERBS_OBJECT_WQ,
+		      &UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uwq_object), 0,
+						  uverbs_free_wq));
+
+DECLARE_UVERBS_OBJECT(uverbs_object_rwq_ind_table,
+		      UVERBS_OBJECT_RWQ_IND_TBL,
+		      &UVERBS_TYPE_ALLOC_IDR(0, uverbs_free_rwq_ind_tbl));
+
+DECLARE_UVERBS_OBJECT(uverbs_object_xrcd, UVERBS_OBJECT_XRCD,
+		      &UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uxrcd_object), 0,
+						  uverbs_free_xrcd));
+
+DECLARE_UVERBS_OBJECT(uverbs_object_pd, UVERBS_OBJECT_PD,
+		      /* 2 is used in order to free the PD after MRs */
+		      &UVERBS_TYPE_ALLOC_IDR(2, uverbs_free_pd));
+
+DECLARE_UVERBS_OBJECT(uverbs_object_device, UVERBS_OBJECT_DEVICE, NULL);
+
+DECLARE_UVERBS_OBJECT_TREE(uverbs_default_objects,
+			   &uverbs_object_device,
+			   &uverbs_object_pd,
+			   &uverbs_object_mr,
+			   &uverbs_object_comp_channel,
+			   &uverbs_object_cq,
+			   &uverbs_object_qp,
+			   &uverbs_object_ah,
+			   &uverbs_object_mw,
+			   &uverbs_object_srq,
+			   &uverbs_object_flow,
+			   &uverbs_object_wq,
+			   &uverbs_object_rwq_ind_table,
+			   &uverbs_object_xrcd);
diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index 7f8fe443df46..ee9e27dc799b 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -180,39 +180,29 @@ EXPORT_SYMBOL(ib_rate_to_mbps);
 __attribute_const__ enum rdma_transport_type
 rdma_node_get_transport(enum rdma_node_type node_type)
 {
-	switch (node_type) {
-	case RDMA_NODE_IB_CA:
-	case RDMA_NODE_IB_SWITCH:
-	case RDMA_NODE_IB_ROUTER:
-		return RDMA_TRANSPORT_IB;
-	case RDMA_NODE_RNIC:
-		return RDMA_TRANSPORT_IWARP;
-	case RDMA_NODE_USNIC:
+
+	if (node_type == RDMA_NODE_USNIC)
 		return RDMA_TRANSPORT_USNIC;
-	case RDMA_NODE_USNIC_UDP:
+	if (node_type == RDMA_NODE_USNIC_UDP)
 		return RDMA_TRANSPORT_USNIC_UDP;
-	default:
-		BUG();
-		return 0;
-	}
+	if (node_type == RDMA_NODE_RNIC)
+		return RDMA_TRANSPORT_IWARP;
+
+	return RDMA_TRANSPORT_IB;
 }
 EXPORT_SYMBOL(rdma_node_get_transport);
 
 enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device, u8 port_num)
 {
+	enum rdma_transport_type lt;
 	if (device->get_link_layer)
 		return device->get_link_layer(device, port_num);
 
-	switch (rdma_node_get_transport(device->node_type)) {
-	case RDMA_TRANSPORT_IB:
+	lt = rdma_node_get_transport(device->node_type);
+	if (lt == RDMA_TRANSPORT_IB)
 		return IB_LINK_LAYER_INFINIBAND;
-	case RDMA_TRANSPORT_IWARP:
-	case RDMA_TRANSPORT_USNIC:
-	case RDMA_TRANSPORT_USNIC_UDP:
-		return IB_LINK_LAYER_ETHERNET;
-	default:
-		return IB_LINK_LAYER_UNSPECIFIED;
-	}
+
+	return IB_LINK_LAYER_ETHERNET;
 }
 EXPORT_SYMBOL(rdma_port_get_link_layer);
 
@@ -478,6 +468,8 @@ int ib_init_ah_from_wc(struct ib_device *device, u8 port_num,
 	union ib_gid dgid;
 	union ib_gid sgid;
 
+	might_sleep();
+
 	memset(ah_attr, 0, sizeof *ah_attr);
 	ah_attr->type = rdma_ah_find_type(device, port_num);
 	if (rdma_cap_eth_ah(device, port_num)) {
@@ -632,11 +624,13 @@ struct ib_srq *ib_create_srq(struct ib_pd *pd,
 		srq->event_handler = srq_init_attr->event_handler;
 		srq->srq_context   = srq_init_attr->srq_context;
 		srq->srq_type      = srq_init_attr->srq_type;
+		if (ib_srq_has_cq(srq->srq_type)) {
+			srq->ext.cq   = srq_init_attr->ext.cq;
+			atomic_inc(&srq->ext.cq->usecnt);
+		}
 		if (srq->srq_type == IB_SRQT_XRC) {
 			srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd;
-			srq->ext.xrc.cq   = srq_init_attr->ext.xrc.cq;
 			atomic_inc(&srq->ext.xrc.xrcd->usecnt);
-			atomic_inc(&srq->ext.xrc.cq->usecnt);
 		}
 		atomic_inc(&pd->usecnt);
 		atomic_set(&srq->usecnt, 0);
@@ -677,18 +671,18 @@ int ib_destroy_srq(struct ib_srq *srq)
 
 	pd = srq->pd;
 	srq_type = srq->srq_type;
-	if (srq_type == IB_SRQT_XRC) {
+	if (ib_srq_has_cq(srq_type))
+		cq = srq->ext.cq;
+	if (srq_type == IB_SRQT_XRC)
 		xrcd = srq->ext.xrc.xrcd;
-		cq = srq->ext.xrc.cq;
-	}
 
 	ret = srq->device->destroy_srq(srq);
 	if (!ret) {
 		atomic_dec(&pd->usecnt);
-		if (srq_type == IB_SRQT_XRC) {
+		if (srq_type == IB_SRQT_XRC)
 			atomic_dec(&xrcd->usecnt);
+		if (ib_srq_has_cq(srq_type))
 			atomic_dec(&cq->usecnt);
-		}
 	}
 
 	return ret;
@@ -838,6 +832,7 @@ struct ib_qp *ib_create_qp(struct ib_pd *pd,
 	spin_lock_init(&qp->mr_lock);
 	INIT_LIST_HEAD(&qp->rdma_mrs);
 	INIT_LIST_HEAD(&qp->sig_mrs);
+	qp->port = 0;
 
 	if (qp_init_attr->qp_type == IB_QPT_XRC_TGT)
 		return ib_create_xrc_qp(qp, qp_init_attr);
@@ -1243,6 +1238,18 @@ int ib_resolve_eth_dmac(struct ib_device *device,
 	if (rdma_link_local_addr((struct in6_addr *)grh->dgid.raw)) {
 		rdma_get_ll_mac((struct in6_addr *)grh->dgid.raw,
 				ah_attr->roce.dmac);
+		return 0;
+	}
+	if (rdma_is_multicast_addr((struct in6_addr *)ah_attr->grh.dgid.raw)) {
+		if (ipv6_addr_v4mapped((struct in6_addr *)ah_attr->grh.dgid.raw)) {
+			__be32 addr = 0;
+
+			memcpy(&addr, ah_attr->grh.dgid.raw + 12, 4);
+			ip_eth_mc_map(addr, (char *)ah_attr->roce.dmac);
+		} else {
+			ipv6_eth_mc_map((struct in6_addr *)ah_attr->grh.dgid.raw,
+					(char *)ah_attr->roce.dmac);
+		}
 	} else {
 		union ib_gid		sgid;
 		struct ib_gid_attr	sgid_attr;
@@ -1297,10 +1304,69 @@ int ib_modify_qp_with_udata(struct ib_qp *qp, struct ib_qp_attr *attr,
 		if (ret)
 			return ret;
 	}
-	return ib_security_modify_qp(qp, attr, attr_mask, udata);
+	ret = ib_security_modify_qp(qp, attr, attr_mask, udata);
+	if (!ret && (attr_mask & IB_QP_PORT))
+		qp->port = attr->port_num;
+
+	return ret;
 }
 EXPORT_SYMBOL(ib_modify_qp_with_udata);
 
+int ib_get_eth_speed(struct ib_device *dev, u8 port_num, u8 *speed, u8 *width)
+{
+	int rc;
+	u32 netdev_speed;
+	struct net_device *netdev;
+	struct ethtool_link_ksettings lksettings;
+
+	if (rdma_port_get_link_layer(dev, port_num) != IB_LINK_LAYER_ETHERNET)
+		return -EINVAL;
+
+	if (!dev->get_netdev)
+		return -EOPNOTSUPP;
+
+	netdev = dev->get_netdev(dev, port_num);
+	if (!netdev)
+		return -ENODEV;
+
+	rtnl_lock();
+	rc = __ethtool_get_link_ksettings(netdev, &lksettings);
+	rtnl_unlock();
+
+	dev_put(netdev);
+
+	if (!rc) {
+		netdev_speed = lksettings.base.speed;
+	} else {
+		netdev_speed = SPEED_1000;
+		pr_warn("%s speed is unknown, defaulting to %d\n", netdev->name,
+			netdev_speed);
+	}
+
+	if (netdev_speed <= SPEED_1000) {
+		*width = IB_WIDTH_1X;
+		*speed = IB_SPEED_SDR;
+	} else if (netdev_speed <= SPEED_10000) {
+		*width = IB_WIDTH_1X;
+		*speed = IB_SPEED_FDR10;
+	} else if (netdev_speed <= SPEED_20000) {
+		*width = IB_WIDTH_4X;
+		*speed = IB_SPEED_DDR;
+	} else if (netdev_speed <= SPEED_25000) {
+		*width = IB_WIDTH_1X;
+		*speed = IB_SPEED_EDR;
+	} else if (netdev_speed <= SPEED_40000) {
+		*width = IB_WIDTH_4X;
+		*speed = IB_SPEED_FDR10;
+	} else {
+		*width = IB_WIDTH_4X;
+		*speed = IB_SPEED_EDR;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(ib_get_eth_speed);
+
 int ib_modify_qp(struct ib_qp *qp,
 		 struct ib_qp_attr *qp_attr,
 		 int qp_attr_mask)
@@ -1568,15 +1634,53 @@ EXPORT_SYMBOL(ib_dealloc_fmr);
 
 /* Multicast groups */
 
+static bool is_valid_mcast_lid(struct ib_qp *qp, u16 lid)
+{
+	struct ib_qp_init_attr init_attr = {};
+	struct ib_qp_attr attr = {};
+	int num_eth_ports = 0;
+	int port;
+
+	/* If QP state >= init, it is assigned to a port and we can check this
+	 * port only.
+	 */
+	if (!ib_query_qp(qp, &attr, IB_QP_STATE | IB_QP_PORT, &init_attr)) {
+		if (attr.qp_state >= IB_QPS_INIT) {
+			if (qp->device->get_link_layer(qp->device, attr.port_num) !=
+			    IB_LINK_LAYER_INFINIBAND)
+				return true;
+			goto lid_check;
+		}
+	}
+
+	/* Can't get a quick answer, iterate over all ports */
+	for (port = 0; port < qp->device->phys_port_cnt; port++)
+		if (qp->device->get_link_layer(qp->device, port) !=
+		    IB_LINK_LAYER_INFINIBAND)
+			num_eth_ports++;
+
+	/* If we have at lease one Ethernet port, RoCE annex declares that
+	 * multicast LID should be ignored. We can't tell at this step if the
+	 * QP belongs to an IB or Ethernet port.
+	 */
+	if (num_eth_ports)
+		return true;
+
+	/* If all the ports are IB, we can check according to IB spec. */
+lid_check:
+	return !(lid < be16_to_cpu(IB_MULTICAST_LID_BASE) ||
+		 lid == be16_to_cpu(IB_LID_PERMISSIVE));
+}
+
 int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
 {
 	int ret;
 
 	if (!qp->device->attach_mcast)
 		return -ENOSYS;
-	if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD ||
-	    lid < be16_to_cpu(IB_MULTICAST_LID_BASE) ||
-	    lid == be16_to_cpu(IB_LID_PERMISSIVE))
+
+	if (!rdma_is_multicast_addr((struct in6_addr *)gid->raw) ||
+	    qp->qp_type != IB_QPT_UD || !is_valid_mcast_lid(qp, lid))
 		return -EINVAL;
 
 	ret = qp->device->attach_mcast(qp, gid, lid);
@@ -1592,9 +1696,9 @@ int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
 
 	if (!qp->device->detach_mcast)
 		return -ENOSYS;
-	if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD ||
-	    lid < be16_to_cpu(IB_MULTICAST_LID_BASE) ||
-	    lid == be16_to_cpu(IB_LID_PERMISSIVE))
+
+	if (!rdma_is_multicast_addr((struct in6_addr *)gid->raw) ||
+	    qp->qp_type != IB_QPT_UD || !is_valid_mcast_lid(qp, lid))
 		return -EINVAL;
 
 	ret = qp->device->detach_mcast(qp, gid, lid);
diff --git a/drivers/infiniband/hw/bnxt_re/Kconfig b/drivers/infiniband/hw/bnxt_re/Kconfig
index 19982a4a9bba..18f5ed082f41 100644
--- a/drivers/infiniband/hw/bnxt_re/Kconfig
+++ b/drivers/infiniband/hw/bnxt_re/Kconfig
@@ -1,6 +1,7 @@
 config INFINIBAND_BNXT_RE
     tristate "Broadcom Netxtreme HCA support"
     depends on ETHERNET && NETDEVICES && PCI && INET && DCB
+    depends on MAY_USE_DEVLINK
     select NET_VENDOR_BROADCOM
     select BNXT
     ---help---
diff --git a/drivers/infiniband/hw/bnxt_re/Makefile b/drivers/infiniband/hw/bnxt_re/Makefile
index 036f84efbc73..afbaa0e20670 100644
--- a/drivers/infiniband/hw/bnxt_re/Makefile
+++ b/drivers/infiniband/hw/bnxt_re/Makefile
@@ -3,4 +3,4 @@ ccflags-y := -Idrivers/net/ethernet/broadcom/bnxt
 obj-$(CONFIG_INFINIBAND_BNXT_RE) += bnxt_re.o
 bnxt_re-y := main.o ib_verbs.o \
 	     qplib_res.o qplib_rcfw.o	\
-	     qplib_sp.o qplib_fp.o
+	     qplib_sp.o qplib_fp.o  hw_counters.o
diff --git a/drivers/infiniband/hw/bnxt_re/bnxt_re.h b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
index 85527532c49d..b3ad37fec578 100644
--- a/drivers/infiniband/hw/bnxt_re/bnxt_re.h
+++ b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
@@ -85,7 +85,7 @@ struct bnxt_re_sqp_entries {
 };
 
 #define BNXT_RE_MIN_MSIX		2
-#define BNXT_RE_MAX_MSIX		16
+#define BNXT_RE_MAX_MSIX		9
 #define BNXT_RE_AEQ_IDX			0
 #define BNXT_RE_NQ_IDX			1
 
@@ -116,7 +116,7 @@ struct bnxt_re_dev {
 	struct bnxt_qplib_rcfw		rcfw;
 
 	/* NQ */
-	struct bnxt_qplib_nq		nq;
+	struct bnxt_qplib_nq		nq[BNXT_RE_MAX_MSIX];
 
 	/* Device Resources */
 	struct bnxt_qplib_dev_attr	dev_attr;
@@ -140,6 +140,7 @@ struct bnxt_re_dev {
 	struct bnxt_re_qp		*qp1_sqp;
 	struct bnxt_re_ah		*sqp_ah;
 	struct bnxt_re_sqp_entries sqp_tbl[1024];
+	atomic_t nq_alloc_cnt;
 };
 
 #define to_bnxt_re_dev(ptr, member)	\
diff --git a/drivers/infiniband/hw/bnxt_re/hw_counters.c b/drivers/infiniband/hw/bnxt_re/hw_counters.c
new file mode 100644
index 000000000000..7b28219eba46
--- /dev/null
+++ b/drivers/infiniband/hw/bnxt_re/hw_counters.c
@@ -0,0 +1,114 @@
+/*
+ * Broadcom NetXtreme-E RoCE driver.
+ *
+ * Copyright (c) 2016 - 2017, Broadcom. All rights reserved.  The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: Statistics
+ *
+ */
+
+#include <linux/interrupt.h>
+#include <linux/types.h>
+#include <linux/spinlock.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/pci.h>
+#include <linux/prefetch.h>
+#include <linux/delay.h>
+
+#include <rdma/ib_addr.h>
+
+#include "bnxt_ulp.h"
+#include "roce_hsi.h"
+#include "qplib_res.h"
+#include "qplib_sp.h"
+#include "qplib_fp.h"
+#include "qplib_rcfw.h"
+#include "bnxt_re.h"
+#include "hw_counters.h"
+
+static const char * const bnxt_re_stat_name[] = {
+	[BNXT_RE_ACTIVE_QP]           =  "active_qps",
+	[BNXT_RE_ACTIVE_SRQ]          =  "active_srqs",
+	[BNXT_RE_ACTIVE_CQ]           =  "active_cqs",
+	[BNXT_RE_ACTIVE_MR]           =  "active_mrs",
+	[BNXT_RE_ACTIVE_MW]           =  "active_mws",
+	[BNXT_RE_RX_PKTS]             =  "rx_pkts",
+	[BNXT_RE_RX_BYTES]            =  "rx_bytes",
+	[BNXT_RE_TX_PKTS]             =  "tx_pkts",
+	[BNXT_RE_TX_BYTES]            =  "tx_bytes",
+	[BNXT_RE_RECOVERABLE_ERRORS]  =  "recoverable_errors"
+};
+
+int bnxt_re_ib_get_hw_stats(struct ib_device *ibdev,
+			    struct rdma_hw_stats *stats,
+			    u8 port, int index)
+{
+	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
+	struct ctx_hw_stats *bnxt_re_stats = rdev->qplib_ctx.stats.dma;
+
+	if (!port || !stats)
+		return -EINVAL;
+
+	stats->value[BNXT_RE_ACTIVE_QP] = atomic_read(&rdev->qp_count);
+	stats->value[BNXT_RE_ACTIVE_SRQ] = atomic_read(&rdev->srq_count);
+	stats->value[BNXT_RE_ACTIVE_CQ] = atomic_read(&rdev->cq_count);
+	stats->value[BNXT_RE_ACTIVE_MR] = atomic_read(&rdev->mr_count);
+	stats->value[BNXT_RE_ACTIVE_MW] = atomic_read(&rdev->mw_count);
+	if (bnxt_re_stats) {
+		stats->value[BNXT_RE_RECOVERABLE_ERRORS] =
+			le64_to_cpu(bnxt_re_stats->tx_bcast_pkts);
+		stats->value[BNXT_RE_RX_PKTS] =
+			le64_to_cpu(bnxt_re_stats->rx_ucast_pkts);
+		stats->value[BNXT_RE_RX_BYTES] =
+			le64_to_cpu(bnxt_re_stats->rx_ucast_bytes);
+		stats->value[BNXT_RE_TX_PKTS] =
+			le64_to_cpu(bnxt_re_stats->tx_ucast_pkts);
+		stats->value[BNXT_RE_TX_BYTES] =
+			le64_to_cpu(bnxt_re_stats->tx_ucast_bytes);
+	}
+	return ARRAY_SIZE(bnxt_re_stat_name);
+}
+
+struct rdma_hw_stats *bnxt_re_ib_alloc_hw_stats(struct ib_device *ibdev,
+						u8 port_num)
+{
+	BUILD_BUG_ON(ARRAY_SIZE(bnxt_re_stat_name) != BNXT_RE_NUM_COUNTERS);
+	/* We support only per port stats */
+	if (!port_num)
+		return NULL;
+
+	return rdma_alloc_hw_stats_struct(bnxt_re_stat_name,
+					  ARRAY_SIZE(bnxt_re_stat_name),
+					  RDMA_HW_STATS_DEFAULT_LIFESPAN);
+}
diff --git a/drivers/infiniband/hw/bnxt_re/hw_counters.h b/drivers/infiniband/hw/bnxt_re/hw_counters.h
new file mode 100644
index 000000000000..be0dc0093b58
--- /dev/null
+++ b/drivers/infiniband/hw/bnxt_re/hw_counters.h
@@ -0,0 +1,62 @@
+/*
+ * Broadcom NetXtreme-E RoCE driver.
+ *
+ * Copyright (c) 2016 - 2017, Broadcom. All rights reserved.  The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: Statistics (header)
+ *
+ */
+
+#ifndef __BNXT_RE_HW_STATS_H__
+#define __BNXT_RE_HW_STATS_H__
+
+enum bnxt_re_hw_stats {
+	BNXT_RE_ACTIVE_QP,
+	BNXT_RE_ACTIVE_SRQ,
+	BNXT_RE_ACTIVE_CQ,
+	BNXT_RE_ACTIVE_MR,
+	BNXT_RE_ACTIVE_MW,
+	BNXT_RE_RX_PKTS,
+	BNXT_RE_RX_BYTES,
+	BNXT_RE_TX_PKTS,
+	BNXT_RE_TX_BYTES,
+	BNXT_RE_RECOVERABLE_ERRORS,
+	BNXT_RE_NUM_COUNTERS
+};
+
+struct rdma_hw_stats *bnxt_re_ib_alloc_hw_stats(struct ib_device *ibdev,
+						u8 port_num);
+int bnxt_re_ib_get_hw_stats(struct ib_device *ibdev,
+			    struct rdma_hw_stats *stats,
+			    u8 port, int index);
+#endif /* __BNXT_RE_HW_STATS_H__ */
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index f0e01b3ac711..01eee15bbd65 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -223,50 +223,6 @@ int bnxt_re_modify_device(struct ib_device *ibdev,
 	return 0;
 }
 
-static void __to_ib_speed_width(struct net_device *netdev, u8 *speed, u8 *width)
-{
-	struct ethtool_link_ksettings lksettings;
-	u32 espeed;
-
-	if (netdev->ethtool_ops && netdev->ethtool_ops->get_link_ksettings) {
-		memset(&lksettings, 0, sizeof(lksettings));
-		rtnl_lock();
-		netdev->ethtool_ops->get_link_ksettings(netdev, &lksettings);
-		rtnl_unlock();
-		espeed = lksettings.base.speed;
-	} else {
-		espeed = SPEED_UNKNOWN;
-	}
-	switch (espeed) {
-	case SPEED_1000:
-		*speed = IB_SPEED_SDR;
-		*width = IB_WIDTH_1X;
-		break;
-	case SPEED_10000:
-		*speed = IB_SPEED_QDR;
-		*width = IB_WIDTH_1X;
-		break;
-	case SPEED_20000:
-		*speed = IB_SPEED_DDR;
-		*width = IB_WIDTH_4X;
-		break;
-	case SPEED_25000:
-		*speed = IB_SPEED_EDR;
-		*width = IB_WIDTH_1X;
-		break;
-	case SPEED_40000:
-		*speed = IB_SPEED_QDR;
-		*width = IB_WIDTH_4X;
-		break;
-	case SPEED_50000:
-		break;
-	default:
-		*speed = IB_SPEED_SDR;
-		*width = IB_WIDTH_1X;
-		break;
-	}
-}
-
 /* Port */
 int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num,
 		       struct ib_port_attr *port_attr)
@@ -308,25 +264,9 @@ int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num,
 	 * IB stack to avoid race in the NETDEV_UNREG path
 	 */
 	if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags))
-		__to_ib_speed_width(rdev->netdev, &port_attr->active_speed,
-				    &port_attr->active_width);
-	return 0;
-}
-
-int bnxt_re_modify_port(struct ib_device *ibdev, u8 port_num,
-			int port_modify_mask,
-			struct ib_port_modify *port_modify)
-{
-	switch (port_modify_mask) {
-	case IB_PORT_SHUTDOWN:
-		break;
-	case IB_PORT_INIT_TYPE:
-		break;
-	case IB_PORT_RESET_QKEY_CNTR:
-		break;
-	default:
-		break;
-	}
+		if (ib_get_eth_speed(ibdev, port_num, &port_attr->active_speed,
+				     &port_attr->active_width))
+			return -EINVAL;
 	return 0;
 }
 
@@ -846,6 +786,7 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp)
 	struct bnxt_re_dev *rdev = qp->rdev;
 	int rc;
 
+	bnxt_qplib_del_flush_qp(&qp->qplib_qp);
 	rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
 	if (rc) {
 		dev_err(rdev_to_dev(rdev), "Failed to destroy HW QP");
@@ -860,6 +801,7 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp)
 			return rc;
 		}
 
+		bnxt_qplib_del_flush_qp(&qp->qplib_qp);
 		rc = bnxt_qplib_destroy_qp(&rdev->qplib_res,
 					   &rdev->qp1_sqp->qplib_qp);
 		if (rc) {
@@ -969,7 +911,6 @@ static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah
 	if (!ah)
 		return NULL;
 
-	memset(ah, 0, sizeof(*ah));
 	ah->rdev = rdev;
 	ah->qplib_ah.pd = &pd->qplib_pd;
 
@@ -1016,7 +957,6 @@ static struct bnxt_re_qp *bnxt_re_create_shadow_qp
 	if (!qp)
 		return NULL;
 
-	memset(qp, 0, sizeof(*qp));
 	qp->rdev = rdev;
 
 	/* Initialize the shadow QP structure from the QP1 values */
@@ -1404,6 +1344,21 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
 		}
 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
 		qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state);
+
+		if (!qp->sumem &&
+		    qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
+			dev_dbg(rdev_to_dev(rdev),
+				"Move QP = %p to flush list\n",
+				qp);
+			bnxt_qplib_add_flush_qp(&qp->qplib_qp);
+		}
+		if (!qp->sumem &&
+		    qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {
+			dev_dbg(rdev_to_dev(rdev),
+				"Move QP = %p out of flush list\n",
+				qp);
+			bnxt_qplib_del_flush_qp(&qp->qplib_qp);
+		}
 	}
 	if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) {
 		qp->qplib_qp.modify_flags |=
@@ -2333,6 +2288,7 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq)
 	struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
 	struct bnxt_re_dev *rdev = cq->rdev;
 	int rc;
+	struct bnxt_qplib_nq *nq = cq->qplib_cq.nq;
 
 	rc = bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
 	if (rc) {
@@ -2347,7 +2303,7 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq)
 		kfree(cq);
 	}
 	atomic_dec(&rdev->cq_count);
-	rdev->nq.budget--;
+	nq->budget--;
 	return 0;
 }
 
@@ -2361,6 +2317,8 @@ struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev,
 	struct bnxt_re_cq *cq = NULL;
 	int rc, entries;
 	int cqe = attr->cqe;
+	struct bnxt_qplib_nq *nq = NULL;
+	unsigned int nq_alloc_cnt;
 
 	/* Validate CQ fields */
 	if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
@@ -2412,8 +2370,15 @@ struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev,
 		cq->qplib_cq.sghead = NULL;
 		cq->qplib_cq.nmap = 0;
 	}
+	/*
+	 * Allocating the NQ in a round robin fashion. nq_alloc_cnt is a
+	 * used for getting the NQ index.
+	 */
+	nq_alloc_cnt = atomic_inc_return(&rdev->nq_alloc_cnt);
+	nq = &rdev->nq[nq_alloc_cnt % (rdev->num_msix - 1)];
 	cq->qplib_cq.max_wqe = entries;
-	cq->qplib_cq.cnq_hw_ring_id = rdev->nq.ring_id;
+	cq->qplib_cq.cnq_hw_ring_id = nq->ring_id;
+	cq->qplib_cq.nq	= nq;
 
 	rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq);
 	if (rc) {
@@ -2423,7 +2388,7 @@ struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev,
 
 	cq->ib_cq.cqe = entries;
 	cq->cq_period = cq->qplib_cq.period;
-	rdev->nq.budget++;
+	nq->budget++;
 
 	atomic_inc(&rdev->cq_count);
 
@@ -2921,6 +2886,10 @@ int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)
 					sq->send_phantom = false;
 			}
 		}
+		if (ncqe < budget)
+			ncqe += bnxt_qplib_process_flush_list(&cq->qplib_cq,
+							      cqe + ncqe,
+							      budget - ncqe);
 
 		if (!ncqe)
 			break;
@@ -3410,7 +3379,7 @@ int bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx)
 					    &rdev->qplib_res.dpi_tbl,
 					    &uctx->dpi);
 		if (rc)
-			dev_err(rdev_to_dev(rdev), "Deallocte HW DPI failed!");
+			dev_err(rdev_to_dev(rdev), "Deallocate HW DPI failed!");
 			/* Don't fail, continue*/
 		uctx->dpi.dbr = NULL;
 	}
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
index a0bb7e33d7ca..1df11ed272ea 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
@@ -141,9 +141,6 @@ int bnxt_re_modify_device(struct ib_device *ibdev,
 			  struct ib_device_modify *device_modify);
 int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num,
 		       struct ib_port_attr *port_attr);
-int bnxt_re_modify_port(struct ib_device *ibdev, u8 port_num,
-			int port_modify_mask,
-			struct ib_port_modify *port_modify);
 int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num,
 			       struct ib_port_immutable *immutable);
 int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num,
diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c
index ceae2d92fb08..82d1cbc27aee 100644
--- a/drivers/infiniband/hw/bnxt_re/main.c
+++ b/drivers/infiniband/hw/bnxt_re/main.c
@@ -64,13 +64,14 @@
 #include "ib_verbs.h"
 #include <rdma/bnxt_re-abi.h>
 #include "bnxt.h"
+#include "hw_counters.h"
+
 static char version[] =
 		BNXT_RE_DESC " v" ROCE_DRV_MODULE_VERSION "\n";
 
 MODULE_AUTHOR("Eddie Wai <eddie.wai@broadcom.com>");
 MODULE_DESCRIPTION(BNXT_RE_DESC " Driver");
 MODULE_LICENSE("Dual BSD/GPL");
-MODULE_VERSION(ROCE_DRV_MODULE_VERSION);
 
 /* globals */
 static struct list_head bnxt_re_dev_list = LIST_HEAD_INIT(bnxt_re_dev_list);
@@ -162,7 +163,7 @@ static int bnxt_re_free_msix(struct bnxt_re_dev *rdev, bool lock_wait)
 
 static int bnxt_re_request_msix(struct bnxt_re_dev *rdev)
 {
-	int rc = 0, num_msix_want = BNXT_RE_MIN_MSIX, num_msix_got;
+	int rc = 0, num_msix_want = BNXT_RE_MAX_MSIX, num_msix_got;
 	struct bnxt_en_dev *en_dev;
 
 	if (!rdev)
@@ -170,6 +171,8 @@ static int bnxt_re_request_msix(struct bnxt_re_dev *rdev)
 
 	en_dev = rdev->en_dev;
 
+	num_msix_want = min_t(u32, BNXT_RE_MAX_MSIX, num_online_cpus());
+
 	rtnl_lock();
 	num_msix_got = en_dev->en_ops->bnxt_request_msix(en_dev, BNXT_ROCE_ULP,
 							 rdev->msix_entries,
@@ -474,7 +477,6 @@ static int bnxt_re_register_ib(struct bnxt_re_dev *rdev)
 	ibdev->modify_device		= bnxt_re_modify_device;
 
 	ibdev->query_port		= bnxt_re_query_port;
-	ibdev->modify_port		= bnxt_re_modify_port;
 	ibdev->get_port_immutable	= bnxt_re_get_port_immutable;
 	ibdev->query_pkey		= bnxt_re_query_pkey;
 	ibdev->query_gid		= bnxt_re_query_gid;
@@ -513,6 +515,8 @@ static int bnxt_re_register_ib(struct bnxt_re_dev *rdev)
 	ibdev->alloc_ucontext		= bnxt_re_alloc_ucontext;
 	ibdev->dealloc_ucontext		= bnxt_re_dealloc_ucontext;
 	ibdev->mmap			= bnxt_re_mmap;
+	ibdev->get_hw_stats             = bnxt_re_ib_get_hw_stats;
+	ibdev->alloc_hw_stats           = bnxt_re_ib_alloc_hw_stats;
 
 	return ib_register_device(ibdev, NULL);
 }
@@ -653,8 +657,12 @@ static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq,
 
 static void bnxt_re_cleanup_res(struct bnxt_re_dev *rdev)
 {
-	if (rdev->nq.hwq.max_elements)
-		bnxt_qplib_disable_nq(&rdev->nq);
+	int i;
+
+	if (rdev->nq[0].hwq.max_elements) {
+		for (i = 1; i < rdev->num_msix; i++)
+			bnxt_qplib_disable_nq(&rdev->nq[i - 1]);
+	}
 
 	if (rdev->qplib_res.rcfw)
 		bnxt_qplib_cleanup_res(&rdev->qplib_res);
@@ -662,31 +670,41 @@ static void bnxt_re_cleanup_res(struct bnxt_re_dev *rdev)
 
 static int bnxt_re_init_res(struct bnxt_re_dev *rdev)
 {
-	int rc = 0;
+	int rc = 0, i;
 
 	bnxt_qplib_init_res(&rdev->qplib_res);
 
-	if (rdev->msix_entries[BNXT_RE_NQ_IDX].vector <= 0)
-		return -EINVAL;
+	for (i = 1; i < rdev->num_msix ; i++) {
+		rc = bnxt_qplib_enable_nq(rdev->en_dev->pdev, &rdev->nq[i - 1],
+					  i - 1, rdev->msix_entries[i].vector,
+					  rdev->msix_entries[i].db_offset,
+					  &bnxt_re_cqn_handler, NULL);
 
-	rc = bnxt_qplib_enable_nq(rdev->en_dev->pdev, &rdev->nq,
-				  rdev->msix_entries[BNXT_RE_NQ_IDX].vector,
-				  rdev->msix_entries[BNXT_RE_NQ_IDX].db_offset,
-				  &bnxt_re_cqn_handler,
-				  NULL);
+		if (rc) {
+			dev_err(rdev_to_dev(rdev),
+				"Failed to enable NQ with rc = 0x%x", rc);
+			goto fail;
+		}
+	}
+	return 0;
+fail:
+	return rc;
+}
 
-	if (rc)
-		dev_err(rdev_to_dev(rdev), "Failed to enable NQ: %#x", rc);
+static void bnxt_re_free_nq_res(struct bnxt_re_dev *rdev, bool lock_wait)
+{
+	int i;
 
-	return rc;
+	for (i = 0; i < rdev->num_msix - 1; i++) {
+		bnxt_re_net_ring_free(rdev, rdev->nq[i].ring_id, lock_wait);
+		bnxt_qplib_free_nq(&rdev->nq[i]);
+	}
 }
 
 static void bnxt_re_free_res(struct bnxt_re_dev *rdev, bool lock_wait)
 {
-	if (rdev->nq.hwq.max_elements) {
-		bnxt_re_net_ring_free(rdev, rdev->nq.ring_id, lock_wait);
-		bnxt_qplib_free_nq(&rdev->nq);
-	}
+	bnxt_re_free_nq_res(rdev, lock_wait);
+
 	if (rdev->qplib_res.dpi_tbl.max) {
 		bnxt_qplib_dealloc_dpi(&rdev->qplib_res,
 				       &rdev->qplib_res.dpi_tbl,
@@ -700,7 +718,7 @@ static void bnxt_re_free_res(struct bnxt_re_dev *rdev, bool lock_wait)
 
 static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev)
 {
-	int rc = 0;
+	int rc = 0, i;
 
 	/* Configure and allocate resources for qplib */
 	rdev->qplib_res.rcfw = &rdev->rcfw;
@@ -717,30 +735,42 @@ static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev)
 				  &rdev->dpi_privileged,
 				  rdev);
 	if (rc)
-		goto fail;
+		goto dealloc_res;
 
-	rdev->nq.hwq.max_elements = BNXT_RE_MAX_CQ_COUNT +
-				    BNXT_RE_MAX_SRQC_COUNT + 2;
-	rc = bnxt_qplib_alloc_nq(rdev->en_dev->pdev, &rdev->nq);
-	if (rc) {
-		dev_err(rdev_to_dev(rdev),
-			"Failed to allocate NQ memory: %#x", rc);
-		goto fail;
-	}
-	rc = bnxt_re_net_ring_alloc
-			(rdev, rdev->nq.hwq.pbl[PBL_LVL_0].pg_map_arr,
-			 rdev->nq.hwq.pbl[rdev->nq.hwq.level].pg_count,
-			 HWRM_RING_ALLOC_CMPL, BNXT_QPLIB_NQE_MAX_CNT - 1,
-			 rdev->msix_entries[BNXT_RE_NQ_IDX].ring_idx,
-			 &rdev->nq.ring_id);
-	if (rc) {
-		dev_err(rdev_to_dev(rdev),
-			"Failed to allocate NQ ring: %#x", rc);
-		goto free_nq;
+	for (i = 0; i < rdev->num_msix - 1; i++) {
+		rdev->nq[i].hwq.max_elements = BNXT_RE_MAX_CQ_COUNT +
+			BNXT_RE_MAX_SRQC_COUNT + 2;
+		rc = bnxt_qplib_alloc_nq(rdev->en_dev->pdev, &rdev->nq[i]);
+		if (rc) {
+			dev_err(rdev_to_dev(rdev), "Alloc Failed NQ%d rc:%#x",
+				i, rc);
+			goto dealloc_dpi;
+		}
+		rc = bnxt_re_net_ring_alloc
+			(rdev, rdev->nq[i].hwq.pbl[PBL_LVL_0].pg_map_arr,
+			 rdev->nq[i].hwq.pbl[rdev->nq[i].hwq.level].pg_count,
+			 HWRM_RING_ALLOC_CMPL,
+			 BNXT_QPLIB_NQE_MAX_CNT - 1,
+			 rdev->msix_entries[i + 1].ring_idx,
+			 &rdev->nq[i].ring_id);
+		if (rc) {
+			dev_err(rdev_to_dev(rdev),
+				"Failed to allocate NQ fw id with rc = 0x%x",
+				rc);
+			goto free_nq;
+		}
 	}
 	return 0;
 free_nq:
-	bnxt_qplib_free_nq(&rdev->nq);
+	for (i = 0; i < rdev->num_msix - 1; i++)
+		bnxt_qplib_free_nq(&rdev->nq[i]);
+dealloc_dpi:
+	bnxt_qplib_dealloc_dpi(&rdev->qplib_res,
+			       &rdev->qplib_res.dpi_tbl,
+			       &rdev->dpi_privileged);
+dealloc_res:
+	bnxt_qplib_free_res(&rdev->qplib_res);
+
 fail:
 	rdev->qplib_res.rcfw = NULL;
 	return rc;
@@ -835,6 +865,42 @@ static void bnxt_re_dev_stop(struct bnxt_re_dev *rdev)
 	mutex_unlock(&rdev->qp_lock);
 }
 
+static int bnxt_re_update_gid(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
+	struct bnxt_qplib_gid gid;
+	u16 gid_idx, index;
+	int rc = 0;
+
+	if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags))
+		return 0;
+
+	if (!sgid_tbl) {
+		dev_err(rdev_to_dev(rdev), "QPLIB: SGID table not allocated");
+		return -EINVAL;
+	}
+
+	for (index = 0; index < sgid_tbl->active; index++) {
+		gid_idx = sgid_tbl->hw_id[index];
+
+		if (!memcmp(&sgid_tbl->tbl[index], &bnxt_qplib_gid_zero,
+			    sizeof(bnxt_qplib_gid_zero)))
+			continue;
+		/* need to modify the VLAN enable setting of non VLAN GID only
+		 * as setting is done for VLAN GID while adding GID
+		 */
+		if (sgid_tbl->vlan[index])
+			continue;
+
+		memcpy(&gid, &sgid_tbl->tbl[index], sizeof(gid));
+
+		rc = bnxt_qplib_update_sgid(sgid_tbl, &gid, gid_idx,
+					    rdev->qplib_res.netdev->dev_addr);
+	}
+
+	return rc;
+}
+
 static u32 bnxt_re_get_priority_mask(struct bnxt_re_dev *rdev)
 {
 	u32 prio_map = 0, tmp_map = 0;
@@ -854,8 +920,6 @@ static u32 bnxt_re_get_priority_mask(struct bnxt_re_dev *rdev)
 	tmp_map = dcb_ieee_getapp_mask(netdev, &app);
 	prio_map |= tmp_map;
 
-	if (!prio_map)
-		prio_map = -EFAULT;
 	return prio_map;
 }
 
@@ -881,10 +945,7 @@ static int bnxt_re_setup_qos(struct bnxt_re_dev *rdev)
 	int rc;
 
 	/* Get priority for roce */
-	rc = bnxt_re_get_priority_mask(rdev);
-	if (rc < 0)
-		return rc;
-	prio_map = (u8)rc;
+	prio_map = bnxt_re_get_priority_mask(rdev);
 
 	if (prio_map == rdev->cur_prio_map)
 		return 0;
@@ -906,6 +967,16 @@ static int bnxt_re_setup_qos(struct bnxt_re_dev *rdev)
 		return rc;
 	}
 
+	/* Actual priorities are not programmed as they are already
+	 * done by L2 driver; just enable or disable priority vlan tagging
+	 */
+	if ((prio_map == 0 && rdev->qplib_res.prio) ||
+	    (prio_map != 0 && !rdev->qplib_res.prio)) {
+		rdev->qplib_res.prio = prio_map ? true : false;
+
+		bnxt_re_update_gid(rdev);
+	}
+
 	return 0;
 }
 
@@ -998,7 +1069,8 @@ static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev)
 	/* Establish RCFW Communication Channel to initialize the context
 	 * memory for the function and all child VFs
 	 */
-	rc = bnxt_qplib_alloc_rcfw_channel(rdev->en_dev->pdev, &rdev->rcfw);
+	rc = bnxt_qplib_alloc_rcfw_channel(rdev->en_dev->pdev, &rdev->rcfw,
+					   BNXT_RE_MAX_QPC_COUNT);
 	if (rc)
 		goto fail;
 
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
index 9af1514e5944..e8afc47f8949 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
@@ -51,6 +51,168 @@
 #include "qplib_fp.h"
 
 static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq);
+static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp);
+
+static void bnxt_qplib_cancel_phantom_processing(struct bnxt_qplib_qp *qp)
+{
+	qp->sq.condition = false;
+	qp->sq.send_phantom = false;
+	qp->sq.single = false;
+}
+
+/* Flush list */
+static void __bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp)
+{
+	struct bnxt_qplib_cq *scq, *rcq;
+
+	scq = qp->scq;
+	rcq = qp->rcq;
+
+	if (!qp->sq.flushed) {
+		dev_dbg(&scq->hwq.pdev->dev,
+			"QPLIB: FP: Adding to SQ Flush list = %p",
+			qp);
+		bnxt_qplib_cancel_phantom_processing(qp);
+		list_add_tail(&qp->sq_flush, &scq->sqf_head);
+		qp->sq.flushed = true;
+	}
+	if (!qp->srq) {
+		if (!qp->rq.flushed) {
+			dev_dbg(&rcq->hwq.pdev->dev,
+				"QPLIB: FP: Adding to RQ Flush list = %p",
+				qp);
+			list_add_tail(&qp->rq_flush, &rcq->rqf_head);
+			qp->rq.flushed = true;
+		}
+	}
+}
+
+void bnxt_qplib_acquire_cq_locks(struct bnxt_qplib_qp *qp,
+				 unsigned long *flags)
+	__acquires(&qp->scq->hwq.lock) __acquires(&qp->rcq->hwq.lock)
+{
+	spin_lock_irqsave(&qp->scq->hwq.lock, *flags);
+	if (qp->scq == qp->rcq)
+		__acquire(&qp->rcq->hwq.lock);
+	else
+		spin_lock(&qp->rcq->hwq.lock);
+}
+
+void bnxt_qplib_release_cq_locks(struct bnxt_qplib_qp *qp,
+				 unsigned long *flags)
+	__releases(&qp->scq->hwq.lock) __releases(&qp->rcq->hwq.lock)
+{
+	if (qp->scq == qp->rcq)
+		__release(&qp->rcq->hwq.lock);
+	else
+		spin_unlock(&qp->rcq->hwq.lock);
+	spin_unlock_irqrestore(&qp->scq->hwq.lock, *flags);
+}
+
+static struct bnxt_qplib_cq *bnxt_qplib_find_buddy_cq(struct bnxt_qplib_qp *qp,
+						      struct bnxt_qplib_cq *cq)
+{
+	struct bnxt_qplib_cq *buddy_cq = NULL;
+
+	if (qp->scq == qp->rcq)
+		buddy_cq = NULL;
+	else if (qp->scq == cq)
+		buddy_cq = qp->rcq;
+	else
+		buddy_cq = qp->scq;
+	return buddy_cq;
+}
+
+static void bnxt_qplib_lock_buddy_cq(struct bnxt_qplib_qp *qp,
+				     struct bnxt_qplib_cq *cq)
+	__acquires(&buddy_cq->hwq.lock)
+{
+	struct bnxt_qplib_cq *buddy_cq = NULL;
+
+	buddy_cq = bnxt_qplib_find_buddy_cq(qp, cq);
+	if (!buddy_cq)
+		__acquire(&cq->hwq.lock);
+	else
+		spin_lock(&buddy_cq->hwq.lock);
+}
+
+static void bnxt_qplib_unlock_buddy_cq(struct bnxt_qplib_qp *qp,
+				       struct bnxt_qplib_cq *cq)
+	__releases(&buddy_cq->hwq.lock)
+{
+	struct bnxt_qplib_cq *buddy_cq = NULL;
+
+	buddy_cq = bnxt_qplib_find_buddy_cq(qp, cq);
+	if (!buddy_cq)
+		__release(&cq->hwq.lock);
+	else
+		spin_unlock(&buddy_cq->hwq.lock);
+}
+
+void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp)
+{
+	unsigned long flags;
+
+	bnxt_qplib_acquire_cq_locks(qp, &flags);
+	__bnxt_qplib_add_flush_qp(qp);
+	bnxt_qplib_release_cq_locks(qp, &flags);
+}
+
+static void __bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp)
+{
+	struct bnxt_qplib_cq *scq, *rcq;
+
+	scq = qp->scq;
+	rcq = qp->rcq;
+
+	if (qp->sq.flushed) {
+		qp->sq.flushed = false;
+		list_del(&qp->sq_flush);
+	}
+	if (!qp->srq) {
+		if (qp->rq.flushed) {
+			qp->rq.flushed = false;
+			list_del(&qp->rq_flush);
+		}
+	}
+}
+
+void bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp)
+{
+	unsigned long flags;
+
+	bnxt_qplib_acquire_cq_locks(qp, &flags);
+	__clean_cq(qp->scq, (u64)(unsigned long)qp);
+	qp->sq.hwq.prod = 0;
+	qp->sq.hwq.cons = 0;
+	__clean_cq(qp->rcq, (u64)(unsigned long)qp);
+	qp->rq.hwq.prod = 0;
+	qp->rq.hwq.cons = 0;
+
+	__bnxt_qplib_del_flush_qp(qp);
+	bnxt_qplib_release_cq_locks(qp, &flags);
+}
+
+static void bnxt_qpn_cqn_sched_task(struct work_struct *work)
+{
+	struct bnxt_qplib_nq_work *nq_work =
+			container_of(work, struct bnxt_qplib_nq_work, work);
+
+	struct bnxt_qplib_cq *cq = nq_work->cq;
+	struct bnxt_qplib_nq *nq = nq_work->nq;
+
+	if (cq && nq) {
+		spin_lock_bh(&cq->compl_lock);
+		if (atomic_read(&cq->arm_state) && nq->cqn_handler) {
+			dev_dbg(&nq->pdev->dev,
+				"%s:Trigger cq  = %p event nq = %p\n",
+				__func__, cq, nq);
+			nq->cqn_handler(nq, cq);
+		}
+		spin_unlock_bh(&cq->compl_lock);
+	}
+	kfree(nq_work);
+}
 
 static void bnxt_qplib_free_qp_hdr_buf(struct bnxt_qplib_res *res,
 				       struct bnxt_qplib_qp *qp)
@@ -119,6 +281,7 @@ static void bnxt_qplib_service_nq(unsigned long data)
 	struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data;
 	struct bnxt_qplib_hwq *hwq = &nq->hwq;
 	struct nq_base *nqe, **nq_ptr;
+	struct bnxt_qplib_cq *cq;
 	int num_cqne_processed = 0;
 	u32 sw_cons, raw_cons;
 	u16 type;
@@ -143,15 +306,17 @@ static void bnxt_qplib_service_nq(unsigned long data)
 			q_handle = le32_to_cpu(nqcne->cq_handle_low);
 			q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high)
 						     << 32;
-			bnxt_qplib_arm_cq_enable((struct bnxt_qplib_cq *)
-						 ((unsigned long)q_handle));
-			if (!nq->cqn_handler(nq, (struct bnxt_qplib_cq *)
-						 ((unsigned long)q_handle)))
+			cq = (struct bnxt_qplib_cq *)(unsigned long)q_handle;
+			bnxt_qplib_arm_cq_enable(cq);
+			spin_lock_bh(&cq->compl_lock);
+			atomic_set(&cq->arm_state, 0);
+			if (!nq->cqn_handler(nq, (cq)))
 				num_cqne_processed++;
 			else
 				dev_warn(&nq->pdev->dev,
 					 "QPLIB: cqn - type 0x%x not handled",
 					 type);
+			spin_unlock_bh(&cq->compl_lock);
 			break;
 		}
 		case NQ_BASE_TYPE_DBQ_EVENT:
@@ -190,12 +355,17 @@ static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance)
 
 void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq)
 {
+	if (nq->cqn_wq) {
+		destroy_workqueue(nq->cqn_wq);
+		nq->cqn_wq = NULL;
+	}
 	/* Make sure the HW is stopped! */
 	synchronize_irq(nq->vector);
 	tasklet_disable(&nq->worker);
 	tasklet_kill(&nq->worker);
 
 	if (nq->requested) {
+		irq_set_affinity_hint(nq->vector, NULL);
 		free_irq(nq->vector, nq);
 		nq->requested = false;
 	}
@@ -209,14 +379,14 @@ void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq)
 }
 
 int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
-			 int msix_vector, int bar_reg_offset,
+			 int nq_idx, int msix_vector, int bar_reg_offset,
 			 int (*cqn_handler)(struct bnxt_qplib_nq *nq,
 					    struct bnxt_qplib_cq *),
 			 int (*srqn_handler)(struct bnxt_qplib_nq *nq,
 					     void *, u8 event))
 {
 	resource_size_t nq_base;
-	int rc;
+	int rc = -1;
 
 	nq->pdev = pdev;
 	nq->vector = msix_vector;
@@ -227,14 +397,31 @@ int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
 
 	tasklet_init(&nq->worker, bnxt_qplib_service_nq, (unsigned long)nq);
 
+	/* Have a task to schedule CQ notifiers in post send case */
+	nq->cqn_wq  = create_singlethread_workqueue("bnxt_qplib_nq");
+	if (!nq->cqn_wq)
+		goto fail;
+
 	nq->requested = false;
-	rc = request_irq(nq->vector, bnxt_qplib_nq_irq, 0, "bnxt_qplib_nq", nq);
+	memset(nq->name, 0, 32);
+	sprintf(nq->name, "bnxt_qplib_nq-%d", nq_idx);
+	rc = request_irq(nq->vector, bnxt_qplib_nq_irq, 0, nq->name, nq);
 	if (rc) {
 		dev_err(&nq->pdev->dev,
 			"Failed to request IRQ for NQ: %#x", rc);
 		bnxt_qplib_disable_nq(nq);
 		goto fail;
 	}
+
+	cpumask_clear(&nq->mask);
+	cpumask_set_cpu(nq_idx, &nq->mask);
+	rc = irq_set_affinity_hint(nq->vector, &nq->mask);
+	if (rc) {
+		dev_warn(&nq->pdev->dev,
+			 "QPLIB: set affinity failed; vector: %d nq_idx: %d\n",
+			 nq->vector, nq_idx);
+	}
+
 	nq->requested = true;
 	nq->bar_reg = NQ_CONS_PCI_BAR_REGION;
 	nq->bar_reg_off = bar_reg_offset;
@@ -258,8 +445,10 @@ fail:
 
 void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq)
 {
-	if (nq->hwq.max_elements)
+	if (nq->hwq.max_elements) {
 		bnxt_qplib_free_hwq(nq->pdev, &nq->hwq);
+		nq->hwq.max_elements = 0;
+	}
 }
 
 int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq)
@@ -401,8 +590,8 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 
 	qp->id = le32_to_cpu(resp.xid);
 	qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
-	sq->flush_in_progress = false;
-	rq->flush_in_progress = false;
+	rcfw->qp_tbl[qp->id].qp_id = qp->id;
+	rcfw->qp_tbl[qp->id].qp_handle = (void *)qp;
 
 	return 0;
 
@@ -615,8 +804,10 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 
 	qp->id = le32_to_cpu(resp.xid);
 	qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
-	sq->flush_in_progress = false;
-	rq->flush_in_progress = false;
+	INIT_LIST_HEAD(&qp->sq_flush);
+	INIT_LIST_HEAD(&qp->rq_flush);
+	rcfw->qp_tbl[qp->id].qp_id = qp->id;
+	rcfw->qp_tbl[qp->id].qp_handle = (void *)qp;
 
 	return 0;
 
@@ -963,13 +1154,19 @@ int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res,
 	u16 cmd_flags = 0;
 	int rc;
 
+	rcfw->qp_tbl[qp->id].qp_id = BNXT_QPLIB_QP_ID_INVALID;
+	rcfw->qp_tbl[qp->id].qp_handle = NULL;
+
 	RCFW_CMD_PREP(req, DESTROY_QP, cmd_flags);
 
 	req.qp_cid = cpu_to_le32(qp->id);
 	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
 					  (void *)&resp, NULL, 0);
-	if (rc)
+	if (rc) {
+		rcfw->qp_tbl[qp->id].qp_id = qp->id;
+		rcfw->qp_tbl[qp->id].qp_handle = qp;
 		return rc;
+	}
 
 	/* Must walk the associated CQs to nullified the QP ptr */
 	spin_lock_irqsave(&qp->scq->hwq.lock, flags);
@@ -1074,14 +1271,21 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
 	struct bnxt_qplib_swq *swq;
 	struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr;
 	struct sq_sge *hw_sge;
+	struct bnxt_qplib_nq_work *nq_work = NULL;
+	bool sch_handler = false;
 	u32 sw_prod;
 	u8 wqe_size16;
 	int i, rc = 0, data_len = 0, pkt_num = 0;
 	__le32 temp32;
 
 	if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS) {
-		rc = -EINVAL;
-		goto done;
+		if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
+			sch_handler = true;
+			dev_dbg(&sq->hwq.pdev->dev,
+				"%s Error QP. Scheduling for poll_cq\n",
+				__func__);
+			goto queue_err;
+		}
 	}
 
 	if (bnxt_qplib_queue_full(sq)) {
@@ -1301,12 +1505,35 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
 			((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) &
 			 SQ_PSN_SEARCH_NEXT_PSN_MASK));
 	}
-
+queue_err:
+	if (sch_handler) {
+		/* Store the ULP info in the software structures */
+		sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
+		swq = &sq->swq[sw_prod];
+		swq->wr_id = wqe->wr_id;
+		swq->type = wqe->type;
+		swq->flags = wqe->flags;
+		if (qp->sig_type)
+			swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP;
+		swq->start_psn = sq->psn & BTH_PSN_MASK;
+	}
 	sq->hwq.prod++;
-
 	qp->wqe_cnt++;
 
 done:
+	if (sch_handler) {
+		nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);
+		if (nq_work) {
+			nq_work->cq = qp->scq;
+			nq_work->nq = qp->scq->nq;
+			INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);
+			queue_work(qp->scq->nq->cqn_wq, &nq_work->work);
+		} else {
+			dev_err(&sq->hwq.pdev->dev,
+				"QPLIB: FP: Failed to allocate SQ nq_work!");
+			rc = -ENOMEM;
+		}
+	}
 	return rc;
 }
 
@@ -1334,15 +1561,17 @@ int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
 	struct bnxt_qplib_q *rq = &qp->rq;
 	struct rq_wqe *rqe, **rqe_ptr;
 	struct sq_sge *hw_sge;
+	struct bnxt_qplib_nq_work *nq_work = NULL;
+	bool sch_handler = false;
 	u32 sw_prod;
 	int i, rc = 0;
 
 	if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
-		dev_err(&rq->hwq.pdev->dev,
-			"QPLIB: FP: QP (0x%x) is in the 0x%x state",
-			qp->id, qp->state);
-		rc = -EINVAL;
-		goto done;
+		sch_handler = true;
+		dev_dbg(&rq->hwq.pdev->dev,
+			"%s Error QP. Scheduling for poll_cq\n",
+			__func__);
+		goto queue_err;
 	}
 	if (bnxt_qplib_queue_full(rq)) {
 		dev_err(&rq->hwq.pdev->dev,
@@ -1378,7 +1607,27 @@ int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
 	/* Supply the rqe->wr_id index to the wr_id_tbl for now */
 	rqe->wr_id[0] = cpu_to_le32(sw_prod);
 
+queue_err:
+	if (sch_handler) {
+		/* Store the ULP info in the software structures */
+		sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
+		rq->swq[sw_prod].wr_id = wqe->wr_id;
+	}
+
 	rq->hwq.prod++;
+	if (sch_handler) {
+		nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);
+		if (nq_work) {
+			nq_work->cq = qp->rcq;
+			nq_work->nq = qp->rcq->nq;
+			INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);
+			queue_work(qp->rcq->nq->cqn_wq, &nq_work->work);
+		} else {
+			dev_err(&rq->hwq.pdev->dev,
+				"QPLIB: FP: Failed to allocate RQ nq_work!");
+			rc = -ENOMEM;
+		}
+	}
 done:
 	return rc;
 }
@@ -1471,6 +1720,9 @@ int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
 	cq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem;
 	cq->period = BNXT_QPLIB_QUEUE_START_PERIOD;
 	init_waitqueue_head(&cq->waitq);
+	INIT_LIST_HEAD(&cq->sqf_head);
+	INIT_LIST_HEAD(&cq->rqf_head);
+	spin_lock_init(&cq->compl_lock);
 
 	bnxt_qplib_arm_cq_enable(cq);
 	return 0;
@@ -1513,9 +1765,13 @@ static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp,
 	while (*budget) {
 		sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
 		if (sw_cons == sw_prod) {
-			sq->flush_in_progress = false;
 			break;
 		}
+		/* Skip the FENCE WQE completions */
+		if (sq->swq[sw_cons].wr_id == BNXT_QPLIB_FENCE_WRID) {
+			bnxt_qplib_cancel_phantom_processing(qp);
+			goto skip_compl;
+		}
 		memset(cqe, 0, sizeof(*cqe));
 		cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR;
 		cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
@@ -1525,6 +1781,7 @@ static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp,
 		cqe->type = sq->swq[sw_cons].type;
 		cqe++;
 		(*budget)--;
+skip_compl:
 		sq->hwq.cons++;
 	}
 	*pcqe = cqe;
@@ -1536,11 +1793,24 @@ static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp,
 }
 
 static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp,
-		      int opcode, struct bnxt_qplib_cqe **pcqe, int *budget)
+		      struct bnxt_qplib_cqe **pcqe, int *budget)
 {
 	struct bnxt_qplib_cqe *cqe;
 	u32 sw_prod, sw_cons;
 	int rc = 0;
+	int opcode = 0;
+
+	switch (qp->type) {
+	case CMDQ_CREATE_QP1_TYPE_GSI:
+		opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1;
+		break;
+	case CMDQ_CREATE_QP_TYPE_RC:
+		opcode = CQ_BASE_CQE_TYPE_RES_RC;
+		break;
+	case CMDQ_CREATE_QP_TYPE_UD:
+		opcode = CQ_BASE_CQE_TYPE_RES_UD;
+		break;
+	}
 
 	/* Flush the rest of the RQ */
 	sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
@@ -1567,6 +1837,21 @@ static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp,
 	return rc;
 }
 
+void bnxt_qplib_mark_qp_error(void *qp_handle)
+{
+	struct bnxt_qplib_qp *qp = qp_handle;
+
+	if (!qp)
+		return;
+
+	/* Must block new posting of SQ and RQ */
+	qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
+	bnxt_qplib_cancel_phantom_processing(qp);
+
+	/* Add qp to flush list of the CQ */
+	__bnxt_qplib_add_flush_qp(qp);
+}
+
 /* Note: SQE is valid from sw_sq_cons up to cqe_sq_cons (exclusive)
  *       CQE is track from sw_cq_cons to max_element but valid only if VALID=1
  */
@@ -1694,10 +1979,12 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,
 			cqe_sq_cons, sq->hwq.max_elements);
 		return -EINVAL;
 	}
-	/* If we were in the middle of flushing the SQ, continue */
-	if (sq->flush_in_progress)
-		goto flush;
 
+	if (qp->sq.flushed) {
+		dev_dbg(&cq->hwq.pdev->dev,
+			"%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
+		goto done;
+	}
 	/* Require to walk the sq's swq to fabricate CQEs for all previously
 	 * signaled SWQEs due to CQE aggregation from the current sq cons
 	 * to the cqe_sq_cons
@@ -1733,11 +2020,9 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,
 				sw_sq_cons, cqe->wr_id, cqe->status);
 			cqe++;
 			(*budget)--;
-			sq->flush_in_progress = true;
-			/* Must block new posting of SQ and RQ */
-			qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
-			sq->condition = false;
-			sq->single = false;
+			bnxt_qplib_lock_buddy_cq(qp, cq);
+			bnxt_qplib_mark_qp_error(qp);
+			bnxt_qplib_unlock_buddy_cq(qp, cq);
 		} else {
 			if (swq->flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
 				/* Before we complete, do WA 9060 */
@@ -1768,15 +2053,6 @@ out:
 	 * the WC for this CQE
 	 */
 	sq->single = false;
-	if (!sq->flush_in_progress)
-		goto done;
-flush:
-	/* Require to walk the sq's swq to fabricate CQEs for all
-	 * previously posted SWQEs due to the error CQE received
-	 */
-	rc = __flush_sq(sq, qp, pcqe, budget);
-	if (!rc)
-		sq->flush_in_progress = false;
 done:
 	return rc;
 }
@@ -1798,6 +2074,12 @@ static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq,
 		dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL");
 		return -EINVAL;
 	}
+	if (qp->rq.flushed) {
+		dev_dbg(&cq->hwq.pdev->dev,
+			"%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
+		goto done;
+	}
+
 	cqe = *pcqe;
 	cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
 	cqe->length = le32_to_cpu(hwcqe->length);
@@ -1817,8 +2099,6 @@ static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq,
 			wr_id_idx, rq->hwq.max_elements);
 		return -EINVAL;
 	}
-	if (rq->flush_in_progress)
-		goto flush_rq;
 
 	cqe->wr_id = rq->swq[wr_id_idx].wr_id;
 	cqe++;
@@ -1827,12 +2107,13 @@ static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq,
 	*pcqe = cqe;
 
 	if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
-		rq->flush_in_progress = true;
-flush_rq:
-		rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_RC, pcqe, budget);
-		if (!rc)
-			rq->flush_in_progress = false;
+		 /* Add qp to flush list of the CQ */
+		bnxt_qplib_lock_buddy_cq(qp, cq);
+		__bnxt_qplib_add_flush_qp(qp);
+		bnxt_qplib_unlock_buddy_cq(qp, cq);
 	}
+
+done:
 	return rc;
 }
 
@@ -1853,6 +2134,11 @@ static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,
 		dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL");
 		return -EINVAL;
 	}
+	if (qp->rq.flushed) {
+		dev_dbg(&cq->hwq.pdev->dev,
+			"%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
+		goto done;
+	}
 	cqe = *pcqe;
 	cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
 	cqe->length = le32_to_cpu(hwcqe->length);
@@ -1876,8 +2162,6 @@ static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,
 			wr_id_idx, rq->hwq.max_elements);
 		return -EINVAL;
 	}
-	if (rq->flush_in_progress)
-		goto flush_rq;
 
 	cqe->wr_id = rq->swq[wr_id_idx].wr_id;
 	cqe++;
@@ -1886,12 +2170,12 @@ static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,
 	*pcqe = cqe;
 
 	if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
-		rq->flush_in_progress = true;
-flush_rq:
-		rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_UD, pcqe, budget);
-		if (!rc)
-			rq->flush_in_progress = false;
+		/* Add qp to flush list of the CQ */
+		bnxt_qplib_lock_buddy_cq(qp, cq);
+		__bnxt_qplib_add_flush_qp(qp);
+		bnxt_qplib_unlock_buddy_cq(qp, cq);
 	}
+done:
 	return rc;
 }
 
@@ -1932,6 +2216,11 @@ static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq,
 			"QPLIB: process_cq Raw/QP1 qp is NULL");
 		return -EINVAL;
 	}
+	if (qp->rq.flushed) {
+		dev_dbg(&cq->hwq.pdev->dev,
+			"%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
+		goto done;
+	}
 	cqe = *pcqe;
 	cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
 	cqe->flags = le16_to_cpu(hwcqe->flags);
@@ -1960,8 +2249,6 @@ static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq,
 			wr_id_idx, rq->hwq.max_elements);
 		return -EINVAL;
 	}
-	if (rq->flush_in_progress)
-		goto flush_rq;
 
 	cqe->wr_id = rq->swq[wr_id_idx].wr_id;
 	cqe++;
@@ -1970,13 +2257,13 @@ static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq,
 	*pcqe = cqe;
 
 	if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
-		rq->flush_in_progress = true;
-flush_rq:
-		rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_RAWETH_QP1, pcqe,
-				budget);
-		if (!rc)
-			rq->flush_in_progress = false;
+		/* Add qp to flush list of the CQ */
+		bnxt_qplib_lock_buddy_cq(qp, cq);
+		__bnxt_qplib_add_flush_qp(qp);
+		bnxt_qplib_unlock_buddy_cq(qp, cq);
 	}
+
+done:
 	return rc;
 }
 
@@ -1990,7 +2277,6 @@ static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq,
 	struct bnxt_qplib_cqe *cqe;
 	u32 sw_cons = 0, cqe_cons;
 	int rc = 0;
-	u8 opcode = 0;
 
 	/* Check the Status */
 	if (hwcqe->status != CQ_TERMINAL_STATUS_OK)
@@ -2005,6 +2291,7 @@ static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq,
 			"QPLIB: FP: CQ Process terminal qp is NULL");
 		return -EINVAL;
 	}
+
 	/* Must block new posting of SQ and RQ */
 	qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
 
@@ -2023,9 +2310,12 @@ static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq,
 			cqe_cons, sq->hwq.max_elements);
 		goto do_rq;
 	}
-	/* If we were in the middle of flushing, continue */
-	if (sq->flush_in_progress)
-		goto flush_sq;
+
+	if (qp->sq.flushed) {
+		dev_dbg(&cq->hwq.pdev->dev,
+			"%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
+		goto sq_done;
+	}
 
 	/* Terminal CQE can also include aggregated successful CQEs prior.
 	 * So we must complete all CQEs from the current sq's cons to the
@@ -2055,11 +2345,6 @@ static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq,
 		rc = -EAGAIN;
 		goto sq_done;
 	}
-	sq->flush_in_progress = true;
-flush_sq:
-	rc = __flush_sq(sq, qp, pcqe, budget);
-	if (!rc)
-		sq->flush_in_progress = false;
 sq_done:
 	if (rc)
 		return rc;
@@ -2075,26 +2360,23 @@ do_rq:
 			cqe_cons, rq->hwq.max_elements);
 		goto done;
 	}
+
+	if (qp->rq.flushed) {
+		dev_dbg(&cq->hwq.pdev->dev,
+			"%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
+		rc = 0;
+		goto done;
+	}
+
 	/* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR
 	 * from the current rq->cons to the rq->prod regardless what the
 	 * rq->cons the terminal CQE indicates
 	 */
-	rq->flush_in_progress = true;
-	switch (qp->type) {
-	case CMDQ_CREATE_QP1_TYPE_GSI:
-		opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1;
-		break;
-	case CMDQ_CREATE_QP_TYPE_RC:
-		opcode = CQ_BASE_CQE_TYPE_RES_RC;
-		break;
-	case CMDQ_CREATE_QP_TYPE_UD:
-		opcode = CQ_BASE_CQE_TYPE_RES_UD;
-		break;
-	}
 
-	rc = __flush_rq(rq, qp, opcode, pcqe, budget);
-	if (!rc)
-		rq->flush_in_progress = false;
+	/* Add qp to flush list of the CQ */
+	bnxt_qplib_lock_buddy_cq(qp, cq);
+	__bnxt_qplib_add_flush_qp(qp);
+	bnxt_qplib_unlock_buddy_cq(qp, cq);
 done:
 	return rc;
 }
@@ -2115,6 +2397,33 @@ static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq,
 	return 0;
 }
 
+int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq,
+				  struct bnxt_qplib_cqe *cqe,
+				  int num_cqes)
+{
+	struct bnxt_qplib_qp *qp = NULL;
+	u32 budget = num_cqes;
+	unsigned long flags;
+
+	spin_lock_irqsave(&cq->hwq.lock, flags);
+	list_for_each_entry(qp, &cq->sqf_head, sq_flush) {
+		dev_dbg(&cq->hwq.pdev->dev,
+			"QPLIB: FP: Flushing SQ QP= %p",
+			qp);
+		__flush_sq(&qp->sq, qp, &cqe, &budget);
+	}
+
+	list_for_each_entry(qp, &cq->rqf_head, rq_flush) {
+		dev_dbg(&cq->hwq.pdev->dev,
+			"QPLIB: FP: Flushing RQ QP= %p",
+			qp);
+		__flush_rq(&qp->rq, qp, &cqe, &budget);
+	}
+	spin_unlock_irqrestore(&cq->hwq.lock, flags);
+
+	return num_cqes - budget;
+}
+
 int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
 		       int num_cqes, struct bnxt_qplib_qp **lib_qp)
 {
@@ -2205,6 +2514,7 @@ void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
 	spin_lock_irqsave(&cq->hwq.lock, flags);
 	if (arm_type)
 		bnxt_qplib_arm_cq(cq, arm_type);
-
+	/* Using cq->arm_state variable to track whether to issue cq handler */
+	atomic_set(&cq->arm_state, 1);
 	spin_unlock_irqrestore(&cq->hwq.lock, flags);
 }
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.h b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
index 19176e06c98a..8ead70ca1c1d 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
@@ -220,19 +220,20 @@ struct bnxt_qplib_q {
 	u16				q_full_delta;
 	u16				max_sge;
 	u32				psn;
-	bool				flush_in_progress;
 	bool				condition;
 	bool				single;
 	bool				send_phantom;
 	u32				phantom_wqe_cnt;
 	u32				phantom_cqe_cnt;
 	u32				next_cq_cons;
+	bool				flushed;
 };
 
 struct bnxt_qplib_qp {
 	struct bnxt_qplib_pd		*pd;
 	struct bnxt_qplib_dpi		*dpi;
 	u64				qp_handle;
+#define        BNXT_QPLIB_QP_ID_INVALID        0xFFFFFFFF
 	u32				id;
 	u8				type;
 	u8				sig_type;
@@ -296,6 +297,8 @@ struct bnxt_qplib_qp {
 	dma_addr_t			sq_hdr_buf_map;
 	void				*rq_hdr_buf;
 	dma_addr_t			rq_hdr_buf_map;
+	struct list_head		sq_flush;
+	struct list_head		rq_flush;
 };
 
 #define BNXT_QPLIB_MAX_CQE_ENTRY_SIZE	sizeof(struct cq_base)
@@ -351,6 +354,7 @@ struct bnxt_qplib_cq {
 	u16				period;
 	struct bnxt_qplib_hwq		hwq;
 	u32				cnq_hw_ring_id;
+	struct bnxt_qplib_nq		*nq;
 	bool				resize_in_progress;
 	struct scatterlist		*sghead;
 	u32				nmap;
@@ -360,6 +364,9 @@ struct bnxt_qplib_cq {
 	unsigned long			flags;
 #define CQ_FLAGS_RESIZE_IN_PROG		1
 	wait_queue_head_t		waitq;
+	struct list_head		sqf_head, rqf_head;
+	atomic_t			arm_state;
+	spinlock_t			compl_lock; /* synch CQ handlers */
 };
 
 #define BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE	sizeof(struct xrrq_irrq)
@@ -400,6 +407,7 @@ struct bnxt_qplib_nq {
 	struct pci_dev			*pdev;
 
 	int				vector;
+	cpumask_t			mask;
 	int				budget;
 	bool				requested;
 	struct tasklet_struct		worker;
@@ -417,11 +425,19 @@ struct bnxt_qplib_nq {
 						(struct bnxt_qplib_nq *nq,
 						 void *srq,
 						 u8 event);
+	struct workqueue_struct         *cqn_wq;
+	char                            name[32];
+};
+
+struct bnxt_qplib_nq_work {
+	struct work_struct      work;
+	struct bnxt_qplib_nq    *nq;
+	struct bnxt_qplib_cq    *cq;
 };
 
 void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq);
 int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
-			 int msix_vector, int bar_reg_offset,
+			 int nq_idx, int msix_vector, int bar_reg_offset,
 			 int (*cqn_handler)(struct bnxt_qplib_nq *nq,
 					    struct bnxt_qplib_cq *cq),
 			 int (*srqn_handler)(struct bnxt_qplib_nq *nq,
@@ -453,4 +469,13 @@ bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq);
 void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type);
 void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq);
 int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq);
+void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp);
+void bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp);
+void bnxt_qplib_acquire_cq_locks(struct bnxt_qplib_qp *qp,
+				 unsigned long *flags);
+void bnxt_qplib_release_cq_locks(struct bnxt_qplib_qp *qp,
+				 unsigned long *flags);
+int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq,
+				  struct bnxt_qplib_cqe *cqe,
+				  int num_cqes);
 #endif /* __BNXT_QPLIB_FP_H__ */
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
index 16e42754dbec..391bb7006e8f 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
@@ -44,6 +44,9 @@
 #include "roce_hsi.h"
 #include "qplib_res.h"
 #include "qplib_rcfw.h"
+#include "qplib_sp.h"
+#include "qplib_fp.h"
+
 static void bnxt_qplib_service_creq(unsigned long data);
 
 /* Hardware communication channel */
@@ -279,16 +282,29 @@ static int bnxt_qplib_process_qp_event(struct bnxt_qplib_rcfw *rcfw,
 				       struct creq_qp_event *qp_event)
 {
 	struct bnxt_qplib_hwq *cmdq = &rcfw->cmdq;
+	struct creq_qp_error_notification *err_event;
 	struct bnxt_qplib_crsq *crsqe;
 	unsigned long flags;
+	struct bnxt_qplib_qp *qp;
 	u16 cbit, blocked = 0;
 	u16 cookie;
 	__le16  mcookie;
+	u32 qp_id;
 
 	switch (qp_event->event) {
 	case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION:
+		err_event = (struct creq_qp_error_notification *)qp_event;
+		qp_id = le32_to_cpu(err_event->xid);
+		qp = rcfw->qp_tbl[qp_id].qp_handle;
 		dev_dbg(&rcfw->pdev->dev,
 			"QPLIB: Received QP error notification");
+		dev_dbg(&rcfw->pdev->dev,
+			"QPLIB: qpid 0x%x, req_err=0x%x, resp_err=0x%x\n",
+			qp_id, err_event->req_err_state_reason,
+			err_event->res_err_state_reason);
+		bnxt_qplib_acquire_cq_locks(qp, &flags);
+		bnxt_qplib_mark_qp_error(qp);
+		bnxt_qplib_release_cq_locks(qp, &flags);
 		break;
 	default:
 		/* Command Response */
@@ -507,6 +523,7 @@ skip_ctx_setup:
 
 void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_rcfw *rcfw)
 {
+	kfree(rcfw->qp_tbl);
 	kfree(rcfw->crsqe_tbl);
 	bnxt_qplib_free_hwq(rcfw->pdev, &rcfw->cmdq);
 	bnxt_qplib_free_hwq(rcfw->pdev, &rcfw->creq);
@@ -514,7 +531,8 @@ void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_rcfw *rcfw)
 }
 
 int bnxt_qplib_alloc_rcfw_channel(struct pci_dev *pdev,
-				  struct bnxt_qplib_rcfw *rcfw)
+				  struct bnxt_qplib_rcfw *rcfw,
+				  int qp_tbl_sz)
 {
 	rcfw->pdev = pdev;
 	rcfw->creq.max_elements = BNXT_QPLIB_CREQE_MAX_CNT;
@@ -541,6 +559,12 @@ int bnxt_qplib_alloc_rcfw_channel(struct pci_dev *pdev,
 	if (!rcfw->crsqe_tbl)
 		goto fail;
 
+	rcfw->qp_tbl_size = qp_tbl_sz;
+	rcfw->qp_tbl = kcalloc(qp_tbl_sz, sizeof(struct bnxt_qplib_qp_node),
+			       GFP_KERNEL);
+	if (!rcfw->qp_tbl)
+		goto fail;
+
 	return 0;
 
 fail:
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h
index 09ce121770cd..0ed312f17c8d 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h
@@ -148,6 +148,11 @@ struct bnxt_qplib_rcfw_sbuf {
 	u32 size;
 };
 
+struct bnxt_qplib_qp_node {
+	u32 qp_id;              /* QP id */
+	void *qp_handle;        /* ptr to qplib_qp */
+};
+
 /* RCFW Communication Channels */
 struct bnxt_qplib_rcfw {
 	struct pci_dev		*pdev;
@@ -181,11 +186,13 @@ struct bnxt_qplib_rcfw {
 	/* Actual Cmd and Resp Queues */
 	struct bnxt_qplib_hwq	cmdq;
 	struct bnxt_qplib_crsq	*crsqe_tbl;
+	int qp_tbl_size;
+	struct bnxt_qplib_qp_node *qp_tbl;
 };
 
 void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_rcfw *rcfw);
 int bnxt_qplib_alloc_rcfw_channel(struct pci_dev *pdev,
-				  struct bnxt_qplib_rcfw *rcfw);
+				  struct bnxt_qplib_rcfw *rcfw, int qp_tbl_sz);
 void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw);
 int bnxt_qplib_enable_rcfw_channel(struct pci_dev *pdev,
 				   struct bnxt_qplib_rcfw *rcfw,
@@ -207,4 +214,5 @@ int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw,
 int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw);
 int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw,
 			 struct bnxt_qplib_ctx *ctx, int is_virtfn);
+void bnxt_qplib_mark_qp_error(void *qp_handle);
 #endif /* __BNXT_QPLIB_RCFW_H__ */
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c
index 62447b3badec..4e101704e801 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_res.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c
@@ -468,9 +468,11 @@ static void bnxt_qplib_free_sgid_tbl(struct bnxt_qplib_res *res,
 	kfree(sgid_tbl->tbl);
 	kfree(sgid_tbl->hw_id);
 	kfree(sgid_tbl->ctx);
+	kfree(sgid_tbl->vlan);
 	sgid_tbl->tbl = NULL;
 	sgid_tbl->hw_id = NULL;
 	sgid_tbl->ctx = NULL;
+	sgid_tbl->vlan = NULL;
 	sgid_tbl->max = 0;
 	sgid_tbl->active = 0;
 }
@@ -491,8 +493,15 @@ static int bnxt_qplib_alloc_sgid_tbl(struct bnxt_qplib_res *res,
 	if (!sgid_tbl->ctx)
 		goto out_free2;
 
+	sgid_tbl->vlan = kcalloc(max, sizeof(u8), GFP_KERNEL);
+	if (!sgid_tbl->vlan)
+		goto out_free3;
+
 	sgid_tbl->max = max;
 	return 0;
+out_free3:
+	kfree(sgid_tbl->ctx);
+	sgid_tbl->ctx = NULL;
 out_free2:
 	kfree(sgid_tbl->hw_id);
 	sgid_tbl->hw_id = NULL;
@@ -514,6 +523,7 @@ static void bnxt_qplib_cleanup_sgid_tbl(struct bnxt_qplib_res *res,
 	}
 	memset(sgid_tbl->tbl, 0, sizeof(struct bnxt_qplib_gid) * sgid_tbl->max);
 	memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max);
+	memset(sgid_tbl->vlan, 0, sizeof(u8) * sgid_tbl->max);
 	sgid_tbl->active = 0;
 }
 
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.h b/drivers/infiniband/hw/bnxt_re/qplib_res.h
index 2e4855509719..e87207526d2c 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_res.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_res.h
@@ -116,6 +116,7 @@ struct bnxt_qplib_sgid_tbl {
 	u16				max;
 	u16				active;
 	void				*ctx;
+	u8				*vlan;
 };
 
 struct bnxt_qplib_pkey_tbl {
@@ -188,6 +189,7 @@ struct bnxt_qplib_res {
 	struct bnxt_qplib_sgid_tbl	sgid_tbl;
 	struct bnxt_qplib_pkey_tbl	pkey_tbl;
 	struct bnxt_qplib_dpi_tbl	dpi_tbl;
+	bool				prio;
 };
 
 #define to_bnxt_qplib(ptr, type, member)	\
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.c b/drivers/infiniband/hw/bnxt_re/qplib_sp.c
index ef91ab786dd4..e277e54a05eb 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_sp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.c
@@ -213,6 +213,7 @@ int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
 	}
 	memcpy(&sgid_tbl->tbl[index], &bnxt_qplib_gid_zero,
 	       sizeof(bnxt_qplib_gid_zero));
+	sgid_tbl->vlan[index] = 0;
 	sgid_tbl->active--;
 	dev_dbg(&res->pdev->dev,
 		"QPLIB: SGID deleted hw_id[0x%x] = 0x%x active = 0x%x",
@@ -265,28 +266,32 @@ int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
 		struct cmdq_add_gid req;
 		struct creq_add_gid_resp resp;
 		u16 cmd_flags = 0;
-		u32 temp32[4];
-		u16 temp16[3];
 		int rc;
 
 		RCFW_CMD_PREP(req, ADD_GID, cmd_flags);
 
-		memcpy(temp32, gid->data, sizeof(struct bnxt_qplib_gid));
-		req.gid[0] = cpu_to_be32(temp32[3]);
-		req.gid[1] = cpu_to_be32(temp32[2]);
-		req.gid[2] = cpu_to_be32(temp32[1]);
-		req.gid[3] = cpu_to_be32(temp32[0]);
-		if (vlan_id != 0xFFFF)
-			req.vlan = cpu_to_le16((vlan_id &
-					CMDQ_ADD_GID_VLAN_VLAN_ID_MASK) |
-					CMDQ_ADD_GID_VLAN_TPID_TPID_8100 |
-					CMDQ_ADD_GID_VLAN_VLAN_EN);
+		req.gid[0] = cpu_to_be32(((u32 *)gid->data)[3]);
+		req.gid[1] = cpu_to_be32(((u32 *)gid->data)[2]);
+		req.gid[2] = cpu_to_be32(((u32 *)gid->data)[1]);
+		req.gid[3] = cpu_to_be32(((u32 *)gid->data)[0]);
+		/*
+		 * driver should ensure that all RoCE traffic is always VLAN
+		 * tagged if RoCE traffic is running on non-zero VLAN ID or
+		 * RoCE traffic is running on non-zero Priority.
+		 */
+		if ((vlan_id != 0xFFFF) || res->prio) {
+			if (vlan_id != 0xFFFF)
+				req.vlan = cpu_to_le16
+				(vlan_id & CMDQ_ADD_GID_VLAN_VLAN_ID_MASK);
+			req.vlan |= cpu_to_le16
+					(CMDQ_ADD_GID_VLAN_TPID_TPID_8100 |
+					 CMDQ_ADD_GID_VLAN_VLAN_EN);
+		}
 
 		/* MAC in network format */
-		memcpy(temp16, smac, 6);
-		req.src_mac[0] = cpu_to_be16(temp16[0]);
-		req.src_mac[1] = cpu_to_be16(temp16[1]);
-		req.src_mac[2] = cpu_to_be16(temp16[2]);
+		req.src_mac[0] = cpu_to_be16(((u16 *)smac)[0]);
+		req.src_mac[1] = cpu_to_be16(((u16 *)smac)[1]);
+		req.src_mac[2] = cpu_to_be16(((u16 *)smac)[2]);
 
 		rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
 						  (void *)&resp, NULL, 0);
@@ -297,6 +302,9 @@ int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
 	/* Add GID to the sgid_tbl */
 	memcpy(&sgid_tbl->tbl[free_idx], gid, sizeof(*gid));
 	sgid_tbl->active++;
+	if (vlan_id != 0xFFFF)
+		sgid_tbl->vlan[free_idx] = 1;
+
 	dev_dbg(&res->pdev->dev,
 		"QPLIB: SGID added hw_id[0x%x] = 0x%x active = 0x%x",
 		 free_idx, sgid_tbl->hw_id[free_idx], sgid_tbl->active);
@@ -306,6 +314,43 @@ int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
 	return 0;
 }
 
+int bnxt_qplib_update_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
+			   struct bnxt_qplib_gid *gid, u16 gid_idx,
+			   u8 *smac)
+{
+	struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl,
+						   struct bnxt_qplib_res,
+						   sgid_tbl);
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct creq_modify_gid_resp resp;
+	struct cmdq_modify_gid req;
+	int rc;
+	u16 cmd_flags = 0;
+
+	RCFW_CMD_PREP(req, MODIFY_GID, cmd_flags);
+
+	req.gid[0] = cpu_to_be32(((u32 *)gid->data)[3]);
+	req.gid[1] = cpu_to_be32(((u32 *)gid->data)[2]);
+	req.gid[2] = cpu_to_be32(((u32 *)gid->data)[1]);
+	req.gid[3] = cpu_to_be32(((u32 *)gid->data)[0]);
+	if (res->prio) {
+		req.vlan |= cpu_to_le16
+			(CMDQ_ADD_GID_VLAN_TPID_TPID_8100 |
+			 CMDQ_ADD_GID_VLAN_VLAN_EN);
+	}
+
+	/* MAC in network format */
+	req.src_mac[0] = cpu_to_be16(((u16 *)smac)[0]);
+	req.src_mac[1] = cpu_to_be16(((u16 *)smac)[1]);
+	req.src_mac[2] = cpu_to_be16(((u16 *)smac)[2]);
+
+	req.gid_index = cpu_to_le16(gid_idx);
+
+	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
+					  (void *)&resp, NULL, 0);
+	return rc;
+}
+
 /* pkeys */
 int bnxt_qplib_get_pkey(struct bnxt_qplib_res *res,
 			struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 index,
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.h b/drivers/infiniband/hw/bnxt_re/qplib_sp.h
index 2ce7e2a32cf0..11322582f5e4 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_sp.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.h
@@ -135,6 +135,8 @@ int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
 int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
 			struct bnxt_qplib_gid *gid, u8 *mac, u16 vlan_id,
 			bool update, u32 *index);
+int bnxt_qplib_update_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
+			   struct bnxt_qplib_gid *gid, u16 gid_idx, u8 *smac);
 int bnxt_qplib_get_pkey(struct bnxt_qplib_res *res,
 			struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 index,
 			u16 *pkey);
diff --git a/drivers/infiniband/hw/bnxt_re/roce_hsi.h b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
index fc23477ac52f..eeb55b2db57e 100644
--- a/drivers/infiniband/hw/bnxt_re/roce_hsi.h
+++ b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
@@ -1473,8 +1473,8 @@ struct cmdq_modify_gid {
 	u8 resp_size;
 	u8 reserved8;
 	__le64 resp_addr;
-	__le32 gid[4];
-	__le16 src_mac[3];
+	__be32 gid[4];
+	__be16 src_mac[3];
 	__le16 vlan;
 	#define CMDQ_MODIFY_GID_VLAN_VLAN_ID_MASK		    0xfffUL
 	#define CMDQ_MODIFY_GID_VLAN_VLAN_ID_SFT		    0
diff --git a/drivers/infiniband/hw/cxgb3/iwch.c b/drivers/infiniband/hw/cxgb3/iwch.c
index 47b2ce2ef203..591de319c178 100644
--- a/drivers/infiniband/hw/cxgb3/iwch.c
+++ b/drivers/infiniband/hw/cxgb3/iwch.c
@@ -45,7 +45,6 @@
 MODULE_AUTHOR("Boyd Faulkner, Steve Wise");
 MODULE_DESCRIPTION("Chelsio T3 RDMA Driver");
 MODULE_LICENSE("Dual BSD/GPL");
-MODULE_VERSION(DRV_VERSION);
 
 static void open_rnic_dev(struct t3cdev *);
 static void close_rnic_dev(struct t3cdev *);
diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c
index 0cd0c1fa27d4..099e76f3758a 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_provider.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c
@@ -1336,8 +1336,7 @@ static int iwch_port_immutable(struct ib_device *ibdev, u8 port_num,
 	return 0;
 }
 
-static void get_dev_fw_ver_str(struct ib_device *ibdev, char *str,
-			       size_t str_len)
+static void get_dev_fw_ver_str(struct ib_device *ibdev, char *str)
 {
 	struct iwch_dev *iwch_dev = to_iwch_dev(ibdev);
 	struct ethtool_drvinfo info;
@@ -1345,7 +1344,7 @@ static void get_dev_fw_ver_str(struct ib_device *ibdev, char *str,
 
 	pr_debug("%s dev 0x%p\n", __func__, iwch_dev);
 	lldev->ethtool_ops->get_drvinfo(lldev, &info);
-	snprintf(str, str_len, "%s", info.fw_version);
+	snprintf(str, IB_FW_VERSION_NAME_MAX, "%s", info.fw_version);
 }
 
 int iwch_register_device(struct iwch_dev *dev)
diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c
index e49b34c3b136..ceaa2fa54d32 100644
--- a/drivers/infiniband/hw/cxgb4/cm.c
+++ b/drivers/infiniband/hw/cxgb4/cm.c
@@ -2871,7 +2871,6 @@ static int close_con_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
 		return 0;
 
 	pr_debug("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
-	BUG_ON(!ep);
 
 	/* The cm_id may be null if we failed to connect */
 	mutex_lock(&ep->com.mutex);
diff --git a/drivers/infiniband/hw/cxgb4/device.c b/drivers/infiniband/hw/cxgb4/device.c
index ae0b79aeea2e..fc886f81b885 100644
--- a/drivers/infiniband/hw/cxgb4/device.c
+++ b/drivers/infiniband/hw/cxgb4/device.c
@@ -44,7 +44,6 @@
 MODULE_AUTHOR("Steve Wise");
 MODULE_DESCRIPTION("Chelsio T4/T5 RDMA Driver");
 MODULE_LICENSE("Dual BSD/GPL");
-MODULE_VERSION(DRV_VERSION);
 
 static int allow_db_fc_on_t5;
 module_param(allow_db_fc_on_t5, int, 0644);
diff --git a/drivers/infiniband/hw/cxgb4/mem.c b/drivers/infiniband/hw/cxgb4/mem.c
index 5332f06b99ba..c2fba76becd4 100644
--- a/drivers/infiniband/hw/cxgb4/mem.c
+++ b/drivers/infiniband/hw/cxgb4/mem.c
@@ -661,7 +661,7 @@ struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd,
 	rhp = php->rhp;
 
 	if (mr_type != IB_MR_TYPE_MEM_REG ||
-	    max_num_sg > t4_max_fr_depth(&rhp->rdev.lldi.ulptx_memwrite_dsgl &&
+	    max_num_sg > t4_max_fr_depth(rhp->rdev.lldi.ulptx_memwrite_dsgl &&
 					 use_dsgl))
 		return ERR_PTR(-EINVAL);
 
diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c
index 0771e9a4d061..346e8334279a 100644
--- a/drivers/infiniband/hw/cxgb4/provider.c
+++ b/drivers/infiniband/hw/cxgb4/provider.c
@@ -517,14 +517,13 @@ static int c4iw_port_immutable(struct ib_device *ibdev, u8 port_num,
 	return 0;
 }
 
-static void get_dev_fw_str(struct ib_device *dev, char *str,
-			   size_t str_len)
+static void get_dev_fw_str(struct ib_device *dev, char *str)
 {
 	struct c4iw_dev *c4iw_dev = container_of(dev, struct c4iw_dev,
 						 ibdev);
 	pr_debug("%s dev 0x%p\n", __func__, dev);
 
-	snprintf(str, str_len, "%u.%u.%u.%u",
+	snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u.%u.%u",
 		 FW_HDR_FW_VER_MAJOR_G(c4iw_dev->rdev.lldi.fw_vers),
 		 FW_HDR_FW_VER_MINOR_G(c4iw_dev->rdev.lldi.fw_vers),
 		 FW_HDR_FW_VER_MICRO_G(c4iw_dev->rdev.lldi.fw_vers),
diff --git a/drivers/infiniband/hw/hfi1/Kconfig b/drivers/infiniband/hw/hfi1/Kconfig
index f6ea0881765a..7b146b67a80f 100644
--- a/drivers/infiniband/hw/hfi1/Kconfig
+++ b/drivers/infiniband/hw/hfi1/Kconfig
@@ -13,13 +13,6 @@ config HFI1_DEBUG_SDMA_ORDER
 	---help---
 	This is a debug flag to test for out of order
 	sdma completions for unit testing
-config HFI1_VERBS_31BIT_PSN
-	bool "HFI1 enable 31 bit PSN"
-	depends on INFINIBAND_HFI1
-	default y
-	---help---
-	Setting this enables 31 BIT PSN
-	For verbs RC/UC
 config SDMA_VERBOSITY
 	bool "Config SDMA Verbosity"
 	depends on INFINIBAND_HFI1
diff --git a/drivers/infiniband/hw/hfi1/Makefile b/drivers/infiniband/hw/hfi1/Makefile
index 88085f65432e..66d538c033b0 100644
--- a/drivers/infiniband/hw/hfi1/Makefile
+++ b/drivers/infiniband/hw/hfi1/Makefile
@@ -8,7 +8,7 @@
 obj-$(CONFIG_INFINIBAND_HFI1) += hfi1.o
 
 hfi1-y := affinity.o chip.o device.o driver.o efivar.o \
-	eprom.o file_ops.o firmware.o \
+	eprom.o exp_rcv.o file_ops.o firmware.o \
 	init.o intr.o mad.o mmu_rb.o pcie.o pio.o pio_copy.o platform.o \
 	qp.o qsfp.o rc.o ruc.o sdma.o sysfs.o trace.o \
 	uc.o ud.o user_exp_rcv.o user_pages.o user_sdma.o verbs.o \
diff --git a/drivers/infiniband/hw/hfi1/affinity.c b/drivers/infiniband/hw/hfi1/affinity.c
index e2cd2cd3b28a..a97055dd4fbd 100644
--- a/drivers/infiniband/hw/hfi1/affinity.c
+++ b/drivers/infiniband/hw/hfi1/affinity.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015, 2016 Intel Corporation.
+ * Copyright(c) 2015 - 2017 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -335,10 +335,10 @@ static void hfi1_update_sdma_affinity(struct hfi1_msix_entry *msix, int cpu)
 	sde->cpu = cpu;
 	cpumask_clear(&msix->mask);
 	cpumask_set_cpu(cpu, &msix->mask);
-	dd_dev_dbg(dd, "IRQ vector: %u, type %s engine %u -> cpu: %d\n",
-		   msix->msix.vector, irq_type_names[msix->type],
+	dd_dev_dbg(dd, "IRQ: %u, type %s engine %u -> cpu: %d\n",
+		   msix->irq, irq_type_names[msix->type],
 		   sde->this_idx, cpu);
-	irq_set_affinity_hint(msix->msix.vector, &msix->mask);
+	irq_set_affinity_hint(msix->irq, &msix->mask);
 
 	/*
 	 * Set the new cpu in the hfi1_affinity_node and clean
@@ -387,7 +387,7 @@ static void hfi1_setup_sdma_notifier(struct hfi1_msix_entry *msix)
 {
 	struct irq_affinity_notify *notify = &msix->notify;
 
-	notify->irq = msix->msix.vector;
+	notify->irq = msix->irq;
 	notify->notify = hfi1_irq_notifier_notify;
 	notify->release = hfi1_irq_notifier_release;
 
@@ -472,10 +472,10 @@ static int get_irq_affinity(struct hfi1_devdata *dd,
 	}
 
 	cpumask_set_cpu(cpu, &msix->mask);
-	dd_dev_info(dd, "IRQ vector: %u, type %s %s -> cpu: %d\n",
-		    msix->msix.vector, irq_type_names[msix->type],
+	dd_dev_info(dd, "IRQ: %u, type %s %s -> cpu: %d\n",
+		    msix->irq, irq_type_names[msix->type],
 		    extra, cpu);
-	irq_set_affinity_hint(msix->msix.vector, &msix->mask);
+	irq_set_affinity_hint(msix->irq, &msix->mask);
 
 	if (msix->type == IRQ_SDMA) {
 		sde->cpu = cpu;
@@ -533,7 +533,7 @@ void hfi1_put_irq_affinity(struct hfi1_devdata *dd,
 		}
 	}
 
-	irq_set_affinity_hint(msix->msix.vector, NULL);
+	irq_set_affinity_hint(msix->irq, NULL);
 	cpumask_clear(&msix->mask);
 	mutex_unlock(&node_affinity.lock);
 }
diff --git a/drivers/infiniband/hw/hfi1/affinity.h b/drivers/infiniband/hw/hfi1/affinity.h
index e78c7aa094e0..2a1e374169c0 100644
--- a/drivers/infiniband/hw/hfi1/affinity.h
+++ b/drivers/infiniband/hw/hfi1/affinity.h
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015, 2016 Intel Corporation.
+ * Copyright(c) 2015 - 2017 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -75,24 +75,26 @@ struct hfi1_msix_entry;
 /* Initialize non-HT cpu cores mask */
 void init_real_cpu_mask(void);
 /* Initialize driver affinity data */
-int hfi1_dev_affinity_init(struct hfi1_devdata *);
+int hfi1_dev_affinity_init(struct hfi1_devdata *dd);
 /*
  * Set IRQ affinity to a CPU. The function will determine the
  * CPU and set the affinity to it.
  */
-int hfi1_get_irq_affinity(struct hfi1_devdata *, struct hfi1_msix_entry *);
+int hfi1_get_irq_affinity(struct hfi1_devdata *dd,
+			  struct hfi1_msix_entry *msix);
 /*
  * Remove the IRQ's CPU affinity. This function also updates
  * any internal CPU tracking data
  */
-void hfi1_put_irq_affinity(struct hfi1_devdata *, struct hfi1_msix_entry *);
+void hfi1_put_irq_affinity(struct hfi1_devdata *dd,
+			   struct hfi1_msix_entry *msix);
 /*
  * Determine a CPU affinity for a user process, if the process does not
  * have an affinity set yet.
  */
-int hfi1_get_proc_affinity(int);
+int hfi1_get_proc_affinity(int node);
 /* Release a CPU used by a user process. */
-void hfi1_put_proc_affinity(int);
+void hfi1_put_proc_affinity(int cpu);
 
 struct hfi1_affinity_node {
 	int node;
diff --git a/drivers/infiniband/hw/hfi1/aspm.h b/drivers/infiniband/hw/hfi1/aspm.h
index 794e6814a531..522b40ed9937 100644
--- a/drivers/infiniband/hw/hfi1/aspm.h
+++ b/drivers/infiniband/hw/hfi1/aspm.h
@@ -237,14 +237,17 @@ static inline void aspm_disable_all(struct hfi1_devdata *dd)
 {
 	struct hfi1_ctxtdata *rcd;
 	unsigned long flags;
-	unsigned i;
+	u16 i;
 
 	for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
-		rcd = dd->rcd[i];
-		del_timer_sync(&rcd->aspm_timer);
-		spin_lock_irqsave(&rcd->aspm_lock, flags);
-		rcd->aspm_intr_enable = false;
-		spin_unlock_irqrestore(&rcd->aspm_lock, flags);
+		rcd = hfi1_rcd_get_by_index(dd, i);
+		if (rcd) {
+			del_timer_sync(&rcd->aspm_timer);
+			spin_lock_irqsave(&rcd->aspm_lock, flags);
+			rcd->aspm_intr_enable = false;
+			spin_unlock_irqrestore(&rcd->aspm_lock, flags);
+			hfi1_rcd_put(rcd);
+		}
 	}
 
 	aspm_disable(dd);
@@ -256,7 +259,7 @@ static inline void aspm_enable_all(struct hfi1_devdata *dd)
 {
 	struct hfi1_ctxtdata *rcd;
 	unsigned long flags;
-	unsigned i;
+	u16 i;
 
 	aspm_enable(dd);
 
@@ -264,11 +267,14 @@ static inline void aspm_enable_all(struct hfi1_devdata *dd)
 		return;
 
 	for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
-		rcd = dd->rcd[i];
-		spin_lock_irqsave(&rcd->aspm_lock, flags);
-		rcd->aspm_intr_enable = true;
-		rcd->aspm_enabled = true;
-		spin_unlock_irqrestore(&rcd->aspm_lock, flags);
+		rcd = hfi1_rcd_get_by_index(dd, i);
+		if (rcd) {
+			spin_lock_irqsave(&rcd->aspm_lock, flags);
+			rcd->aspm_intr_enable = true;
+			rcd->aspm_enabled = true;
+			spin_unlock_irqrestore(&rcd->aspm_lock, flags);
+			hfi1_rcd_put(rcd);
+		}
 	}
 }
 
@@ -284,13 +290,18 @@ static inline void aspm_ctx_init(struct hfi1_ctxtdata *rcd)
 
 static inline void aspm_init(struct hfi1_devdata *dd)
 {
-	unsigned i;
+	struct hfi1_ctxtdata *rcd;
+	u16 i;
 
 	spin_lock_init(&dd->aspm_lock);
 	dd->aspm_supported = aspm_hw_l1_supported(dd);
 
-	for (i = 0; i < dd->first_dyn_alloc_ctxt; i++)
-		aspm_ctx_init(dd->rcd[i]);
+	for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
+		rcd = hfi1_rcd_get_by_index(dd, i);
+		if (rcd)
+			aspm_ctx_init(rcd);
+		hfi1_rcd_put(rcd);
+	}
 
 	/* Start with ASPM disabled */
 	aspm_hw_set_l1_ent_latency(dd);
diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c
index 94b54850ec75..b2ed4b9cda6e 100644
--- a/drivers/infiniband/hw/hfi1/chip.c
+++ b/drivers/infiniband/hw/hfi1/chip.c
@@ -1012,14 +1012,15 @@ static struct flag_table dc8051_info_err_flags[] = {
  */
 static struct flag_table dc8051_info_host_msg_flags[] = {
 	FLAG_ENTRY0("Host request done", 0x0001),
-	FLAG_ENTRY0("BC SMA message", 0x0002),
-	FLAG_ENTRY0("BC PWR_MGM message", 0x0004),
+	FLAG_ENTRY0("BC PWR_MGM message", 0x0002),
+	FLAG_ENTRY0("BC SMA message", 0x0004),
 	FLAG_ENTRY0("BC Unknown message (BCC)", 0x0008),
 	FLAG_ENTRY0("BC Unknown message (LCB)", 0x0010),
 	FLAG_ENTRY0("External device config request", 0x0020),
 	FLAG_ENTRY0("VerifyCap all frames received", 0x0040),
 	FLAG_ENTRY0("LinkUp achieved", 0x0080),
 	FLAG_ENTRY0("Link going down", 0x0100),
+	FLAG_ENTRY0("Link width downgraded", 0x0200),
 };
 
 static u32 encoded_size(u32 size);
@@ -1064,8 +1065,13 @@ static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data,
 static int read_idle_sma(struct hfi1_devdata *dd, u64 *data);
 static int thermal_init(struct hfi1_devdata *dd);
 
+static void update_statusp(struct hfi1_pportdata *ppd, u32 state);
 static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state,
 				  int msecs);
+static void log_state_transition(struct hfi1_pportdata *ppd, u32 state);
+static void log_physical_state(struct hfi1_pportdata *ppd, u32 state);
+static int wait_physical_linkstate(struct hfi1_pportdata *ppd, u32 state,
+				   int msecs);
 static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc);
 static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr);
 static void handle_temp_err(struct hfi1_devdata *dd);
@@ -1294,25 +1300,71 @@ CNTR_ELEM(#name, \
 	  CNTR_SYNTH, \
 	  access_ibp_##cntr)
 
+/**
+ * hfi_addr_from_offset - return addr for readq/writeq
+ * @dd - the dd device
+ * @offset - the offset of the CSR within bar0
+ *
+ * This routine selects the appropriate base address
+ * based on the indicated offset.
+ */
+static inline void __iomem *hfi1_addr_from_offset(
+	const struct hfi1_devdata *dd,
+	u32 offset)
+{
+	if (offset >= dd->base2_start)
+		return dd->kregbase2 + (offset - dd->base2_start);
+	return dd->kregbase1 + offset;
+}
+
+/**
+ * read_csr - read CSR at the indicated offset
+ * @dd - the dd device
+ * @offset - the offset of the CSR within bar0
+ *
+ * Return: the value read or all FF's if there
+ * is no mapping
+ */
 u64 read_csr(const struct hfi1_devdata *dd, u32 offset)
 {
-	if (dd->flags & HFI1_PRESENT) {
-		return readq((void __iomem *)dd->kregbase + offset);
-	}
+	if (dd->flags & HFI1_PRESENT)
+		return readq(hfi1_addr_from_offset(dd, offset));
 	return -1;
 }
 
+/**
+ * write_csr - write CSR at the indicated offset
+ * @dd - the dd device
+ * @offset - the offset of the CSR within bar0
+ * @value - value to write
+ */
 void write_csr(const struct hfi1_devdata *dd, u32 offset, u64 value)
 {
-	if (dd->flags & HFI1_PRESENT)
-		writeq(value, (void __iomem *)dd->kregbase + offset);
+	if (dd->flags & HFI1_PRESENT) {
+		void __iomem *base = hfi1_addr_from_offset(dd, offset);
+
+		/* avoid write to RcvArray */
+		if (WARN_ON(offset >= RCV_ARRAY && offset < dd->base2_start))
+			return;
+		writeq(value, base);
+	}
 }
 
+/**
+ * get_csr_addr - return te iomem address for offset
+ * @dd - the dd device
+ * @offset - the offset of the CSR within bar0
+ *
+ * Return: The iomem address to use in subsequent
+ * writeq/readq operations.
+ */
 void __iomem *get_csr_addr(
-	struct hfi1_devdata *dd,
+	const struct hfi1_devdata *dd,
 	u32 offset)
 {
-	return (void __iomem *)dd->kregbase + offset;
+	if (dd->flags & HFI1_PRESENT)
+		return hfi1_addr_from_offset(dd, offset);
+	return NULL;
 }
 
 static inline u64 read_write_csr(const struct hfi1_devdata *dd, u32 csr,
@@ -5496,7 +5548,7 @@ static void update_rcverr_timer(unsigned long opaque)
 		set_link_down_reason(
 		ppd, OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN, 0,
 		OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN);
-		queue_work(ppd->hfi1_wq, &ppd->link_bounce_work);
+		queue_work(ppd->link_wq, &ppd->link_bounce_work);
 	}
 	dd->rcv_ovfl_cnt = (u32)cur_ovfl_cnt;
 
@@ -6051,7 +6103,7 @@ static void handle_qsfp_int(struct hfi1_devdata *dd, u32 src_ctx, u64 reg)
 				 * will not happen. We have to do it here
 				 * before turning the DC off.
 				 */
-				queue_work(ppd->hfi1_wq, &ppd->link_down_work);
+				queue_work(ppd->link_wq, &ppd->link_down_work);
 			}
 		} else {
 			dd_dev_info(dd, "%s: QSFP module inserted\n",
@@ -6086,7 +6138,7 @@ static void handle_qsfp_int(struct hfi1_devdata *dd, u32 src_ctx, u64 reg)
 
 	/* Schedule the QSFP work only if there is a cable attached. */
 	if (qsfp_mod_present(ppd))
-		queue_work(ppd->hfi1_wq, &ppd->qsfp_info.qsfp_work);
+		queue_work(ppd->link_wq, &ppd->qsfp_info.qsfp_work);
 }
 
 static int request_host_lcb_access(struct hfi1_devdata *dd)
@@ -6735,13 +6787,17 @@ static void wait_for_freeze_status(struct hfi1_devdata *dd, int freeze)
 static void rxe_freeze(struct hfi1_devdata *dd)
 {
 	int i;
+	struct hfi1_ctxtdata *rcd;
 
 	/* disable port */
 	clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
 
 	/* disable all receive contexts */
-	for (i = 0; i < dd->num_rcv_contexts; i++)
-		hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS, i);
+	for (i = 0; i < dd->num_rcv_contexts; i++) {
+		rcd = hfi1_rcd_get_by_index(dd, i);
+		hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS, rcd);
+		hfi1_rcd_put(rcd);
+	}
 }
 
 /*
@@ -6753,21 +6809,24 @@ static void rxe_freeze(struct hfi1_devdata *dd)
 static void rxe_kernel_unfreeze(struct hfi1_devdata *dd)
 {
 	u32 rcvmask;
-	int i;
+	u16 i;
+	struct hfi1_ctxtdata *rcd;
 
 	/* enable all kernel contexts */
 	for (i = 0; i < dd->num_rcv_contexts; i++) {
-		struct hfi1_ctxtdata *rcd = dd->rcd[i];
+		rcd = hfi1_rcd_get_by_index(dd, i);
 
 		/* Ensure all non-user contexts(including vnic) are enabled */
-		if (!rcd || !rcd->sc || (rcd->sc->type == SC_USER))
+		if (!rcd || !rcd->sc || (rcd->sc->type == SC_USER)) {
+			hfi1_rcd_put(rcd);
 			continue;
-
+		}
 		rcvmask = HFI1_RCVCTRL_CTXT_ENB;
 		/* HFI1_RCVCTRL_TAILUPD_[ENB|DIS] needs to be set explicitly */
-		rcvmask |= HFI1_CAP_KGET_MASK(dd->rcd[i]->flags, DMA_RTAIL) ?
+		rcvmask |= HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) ?
 			HFI1_RCVCTRL_TAILUPD_ENB : HFI1_RCVCTRL_TAILUPD_DIS;
-		hfi1_rcvctrl(dd, rcvmask, i);
+		hfi1_rcvctrl(dd, rcvmask, rcd);
+		hfi1_rcd_put(rcd);
 	}
 
 	/* enable port */
@@ -6906,7 +6965,7 @@ static void reset_neighbor_info(struct hfi1_pportdata *ppd)
 
 static const char * const link_down_reason_strs[] = {
 	[OPA_LINKDOWN_REASON_NONE] = "None",
-	[OPA_LINKDOWN_REASON_RCV_ERROR_0] = "Recive error 0",
+	[OPA_LINKDOWN_REASON_RCV_ERROR_0] = "Receive error 0",
 	[OPA_LINKDOWN_REASON_BAD_PKT_LEN] = "Bad packet length",
 	[OPA_LINKDOWN_REASON_PKT_TOO_LONG] = "Packet too long",
 	[OPA_LINKDOWN_REASON_PKT_TOO_SHORT] = "Packet too short",
@@ -6996,6 +7055,7 @@ void handle_link_down(struct work_struct *work)
 	/* Go offline first, then deal with reading/writing through 8051 */
 	was_up = !!(ppd->host_link_state & HLS_UP);
 	set_link_state(ppd, HLS_DN_OFFLINE);
+	xchg(&ppd->is_link_down_queued, 0);
 
 	if (was_up) {
 		lcl_reason = 0;
@@ -7330,7 +7390,7 @@ void handle_verify_cap(struct work_struct *work)
 	struct hfi1_devdata *dd = ppd->dd;
 	u64 reg;
 	u8 power_management;
-	u8 continious;
+	u8 continuous;
 	u8 vcu;
 	u8 vau;
 	u8 z;
@@ -7349,7 +7409,7 @@ void handle_verify_cap(struct work_struct *work)
 	lcb_shutdown(dd, 0);
 	adjust_lcb_for_fpga_serdes(dd);
 
-	read_vc_remote_phy(dd, &power_management, &continious);
+	read_vc_remote_phy(dd, &power_management, &continuous);
 	read_vc_remote_fabric(dd, &vau, &z, &vcu, &vl15buf,
 			      &partner_supported_crc);
 	read_vc_remote_link_width(dd, &remote_tx_rate, &link_widths);
@@ -7363,7 +7423,7 @@ void handle_verify_cap(struct work_struct *work)
 	get_link_widths(dd, &active_tx, &active_rx);
 	dd_dev_info(dd,
 		    "Peer PHY: power management 0x%x, continuous updates 0x%x\n",
-		    (int)power_management, (int)continious);
+		    (int)power_management, (int)continuous);
 	dd_dev_info(dd,
 		    "Peer Fabric: vAU %d, Z %d, vCU %d, vl15 credits 0x%x, CRC sizes 0x%x\n",
 		    (int)vau, (int)z, (int)vcu, (int)vl15buf,
@@ -7689,12 +7749,12 @@ static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg)
 			host_msg &= ~(u64)HOST_REQ_DONE;
 		}
 		if (host_msg & BC_SMA_MSG) {
-			queue_work(ppd->hfi1_wq, &ppd->sma_message_work);
+			queue_work(ppd->link_wq, &ppd->sma_message_work);
 			host_msg &= ~(u64)BC_SMA_MSG;
 		}
 		if (host_msg & LINKUP_ACHIEVED) {
 			dd_dev_info(dd, "8051: Link up\n");
-			queue_work(ppd->hfi1_wq, &ppd->link_up_work);
+			queue_work(ppd->link_wq, &ppd->link_up_work);
 			host_msg &= ~(u64)LINKUP_ACHIEVED;
 		}
 		if (host_msg & EXT_DEVICE_CFG_REQ) {
@@ -7702,7 +7762,7 @@ static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg)
 			host_msg &= ~(u64)EXT_DEVICE_CFG_REQ;
 		}
 		if (host_msg & VERIFY_CAP_FRAME) {
-			queue_work(ppd->hfi1_wq, &ppd->link_vc_work);
+			queue_work(ppd->link_wq, &ppd->link_vc_work);
 			host_msg &= ~(u64)VERIFY_CAP_FRAME;
 		}
 		if (host_msg & LINK_GOING_DOWN) {
@@ -7717,7 +7777,7 @@ static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg)
 			host_msg &= ~(u64)LINK_GOING_DOWN;
 		}
 		if (host_msg & LINK_WIDTH_DOWNGRADED) {
-			queue_work(ppd->hfi1_wq, &ppd->link_downgrade_work);
+			queue_work(ppd->link_wq, &ppd->link_downgrade_work);
 			host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED;
 		}
 		if (host_msg) {
@@ -7752,15 +7812,22 @@ static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg)
 	if (queue_link_down) {
 		/*
 		 * if the link is already going down or disabled, do not
-		 * queue another
+		 * queue another. If there's a link down entry already
+		 * queued, don't queue another one.
 		 */
 		if ((ppd->host_link_state &
 		    (HLS_GOING_OFFLINE | HLS_LINK_COOLDOWN)) ||
 		    ppd->link_enabled == 0) {
-			dd_dev_info(dd, "%s: not queuing link down\n",
-				    __func__);
+			dd_dev_info(dd, "%s: not queuing link down. host_link_state %x, link_enabled %x\n",
+				    __func__, ppd->host_link_state,
+				    ppd->link_enabled);
 		} else {
-			queue_work(ppd->hfi1_wq, &ppd->link_down_work);
+			if (xchg(&ppd->is_link_down_queued, 1) == 1)
+				dd_dev_info(dd,
+					    "%s: link down request already queued\n",
+					    __func__);
+			else
+				queue_work(ppd->link_wq, &ppd->link_down_work);
 		}
 	}
 }
@@ -7968,7 +8035,7 @@ static void handle_dcc_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
 		dd_dev_info_ratelimited(dd, "%s: PortErrorAction bounce\n",
 					__func__);
 		set_link_down_reason(ppd, lcl_reason, 0, lcl_reason);
-		queue_work(ppd->hfi1_wq, &ppd->link_bounce_work);
+		queue_work(ppd->link_wq, &ppd->link_bounce_work);
 	}
 }
 
@@ -8052,7 +8119,7 @@ static void is_rcv_avail_int(struct hfi1_devdata *dd, unsigned int source)
 	char *err_detail;
 
 	if (likely(source < dd->num_rcv_contexts)) {
-		rcd = dd->rcd[source];
+		rcd = hfi1_rcd_get_by_index(dd, source);
 		if (rcd) {
 			/* Check for non-user contexts, including vnic */
 			if ((source < dd->first_dyn_alloc_ctxt) ||
@@ -8060,6 +8127,8 @@ static void is_rcv_avail_int(struct hfi1_devdata *dd, unsigned int source)
 				rcd->do_interrupt(rcd, 0);
 			else
 				handle_user_interrupt(rcd);
+
+			hfi1_rcd_put(rcd);
 			return;	/* OK */
 		}
 		/* received an interrupt, but no rcd */
@@ -8081,12 +8150,14 @@ static void is_rcv_urgent_int(struct hfi1_devdata *dd, unsigned int source)
 	char *err_detail;
 
 	if (likely(source < dd->num_rcv_contexts)) {
-		rcd = dd->rcd[source];
+		rcd = hfi1_rcd_get_by_index(dd, source);
 		if (rcd) {
 			/* only pay attention to user urgent interrupts */
 			if ((source >= dd->first_dyn_alloc_ctxt) &&
 			    (!rcd->sc || (rcd->sc->type == SC_USER)))
 				handle_user_interrupt(rcd);
+
+			hfi1_rcd_put(rcd);
 			return;	/* OK */
 		}
 		/* received an interrupt, but no rcd */
@@ -8219,8 +8290,8 @@ static irqreturn_t sdma_interrupt(int irq, void *data)
 		/* handle the interrupt(s) */
 		sdma_engine_interrupt(sde, status);
 	} else {
-		dd_dev_err(dd, "SDMA engine %u interrupt, but no status bits set\n",
-			   sde->this_idx);
+		dd_dev_err_ratelimited(dd, "SDMA engine %u interrupt, but no status bits set\n",
+				       sde->this_idx);
 	}
 	return IRQ_HANDLED;
 }
@@ -8291,7 +8362,7 @@ static irqreturn_t receive_context_interrupt(int irq, void *data)
 	int disposition;
 	int present;
 
-	trace_hfi1_receive_interrupt(dd, rcd->ctxt);
+	trace_hfi1_receive_interrupt(dd, rcd);
 	this_cpu_inc(*dd->int_counter);
 	aspm_ctx_disable(rcd);
 
@@ -8781,6 +8852,20 @@ static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id,
 			& REMOTE_DEVICE_REV_MASK;
 }
 
+int write_host_interface_version(struct hfi1_devdata *dd, u8 version)
+{
+	u32 frame;
+	u32 mask;
+
+	mask = (HOST_INTERFACE_VERSION_MASK << HOST_INTERFACE_VERSION_SHIFT);
+	read_8051_config(dd, RESERVED_REGISTERS, GENERAL_CONFIG, &frame);
+	/* Clear, then set field */
+	frame &= ~mask;
+	frame |= ((u32)version << HOST_INTERFACE_VERSION_SHIFT);
+	return load_8051_config(dd, RESERVED_REGISTERS, GENERAL_CONFIG,
+				frame);
+}
+
 void read_misc_status(struct hfi1_devdata *dd, u8 *ver_major, u8 *ver_minor,
 		      u8 *ver_patch)
 {
@@ -9257,12 +9342,6 @@ int start_link(struct hfi1_pportdata *ppd)
 	 */
 	tune_serdes(ppd);
 
-	if (!ppd->link_enabled) {
-		dd_dev_info(ppd->dd,
-			    "%s: stopping link start because link is disabled\n",
-			    __func__);
-		return 0;
-	}
 	if (!ppd->driver_link_ready) {
 		dd_dev_info(ppd->dd,
 			    "%s: stopping link start because driver is not ready\n",
@@ -9373,13 +9452,13 @@ static int handle_qsfp_error_conditions(struct hfi1_pportdata *ppd,
 
 	if ((qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_ALARM) ||
 	    (qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_WARNING))
-		dd_dev_info(dd, "%s: QSFP cable temperature too high\n",
-			    __func__);
+		dd_dev_err(dd, "%s: QSFP cable temperature too high\n",
+			   __func__);
 
 	if ((qsfp_interrupt_status[0] & QSFP_LOW_TEMP_ALARM) ||
 	    (qsfp_interrupt_status[0] & QSFP_LOW_TEMP_WARNING))
-		dd_dev_info(dd, "%s: QSFP cable temperature too low\n",
-			    __func__);
+		dd_dev_err(dd, "%s: QSFP cable temperature too low\n",
+			   __func__);
 
 	/*
 	 * The remaining alarms/warnings don't matter if the link is down.
@@ -9389,75 +9468,75 @@ static int handle_qsfp_error_conditions(struct hfi1_pportdata *ppd,
 
 	if ((qsfp_interrupt_status[1] & QSFP_HIGH_VCC_ALARM) ||
 	    (qsfp_interrupt_status[1] & QSFP_HIGH_VCC_WARNING))
-		dd_dev_info(dd, "%s: QSFP supply voltage too high\n",
-			    __func__);
+		dd_dev_err(dd, "%s: QSFP supply voltage too high\n",
+			   __func__);
 
 	if ((qsfp_interrupt_status[1] & QSFP_LOW_VCC_ALARM) ||
 	    (qsfp_interrupt_status[1] & QSFP_LOW_VCC_WARNING))
-		dd_dev_info(dd, "%s: QSFP supply voltage too low\n",
-			    __func__);
+		dd_dev_err(dd, "%s: QSFP supply voltage too low\n",
+			   __func__);
 
 	/* Byte 2 is vendor specific */
 
 	if ((qsfp_interrupt_status[3] & QSFP_HIGH_POWER_ALARM) ||
 	    (qsfp_interrupt_status[3] & QSFP_HIGH_POWER_WARNING))
-		dd_dev_info(dd, "%s: Cable RX channel 1/2 power too high\n",
-			    __func__);
+		dd_dev_err(dd, "%s: Cable RX channel 1/2 power too high\n",
+			   __func__);
 
 	if ((qsfp_interrupt_status[3] & QSFP_LOW_POWER_ALARM) ||
 	    (qsfp_interrupt_status[3] & QSFP_LOW_POWER_WARNING))
-		dd_dev_info(dd, "%s: Cable RX channel 1/2 power too low\n",
-			    __func__);
+		dd_dev_err(dd, "%s: Cable RX channel 1/2 power too low\n",
+			   __func__);
 
 	if ((qsfp_interrupt_status[4] & QSFP_HIGH_POWER_ALARM) ||
 	    (qsfp_interrupt_status[4] & QSFP_HIGH_POWER_WARNING))
-		dd_dev_info(dd, "%s: Cable RX channel 3/4 power too high\n",
-			    __func__);
+		dd_dev_err(dd, "%s: Cable RX channel 3/4 power too high\n",
+			   __func__);
 
 	if ((qsfp_interrupt_status[4] & QSFP_LOW_POWER_ALARM) ||
 	    (qsfp_interrupt_status[4] & QSFP_LOW_POWER_WARNING))
-		dd_dev_info(dd, "%s: Cable RX channel 3/4 power too low\n",
-			    __func__);
+		dd_dev_err(dd, "%s: Cable RX channel 3/4 power too low\n",
+			   __func__);
 
 	if ((qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_ALARM) ||
 	    (qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_WARNING))
-		dd_dev_info(dd, "%s: Cable TX channel 1/2 bias too high\n",
-			    __func__);
+		dd_dev_err(dd, "%s: Cable TX channel 1/2 bias too high\n",
+			   __func__);
 
 	if ((qsfp_interrupt_status[5] & QSFP_LOW_BIAS_ALARM) ||
 	    (qsfp_interrupt_status[5] & QSFP_LOW_BIAS_WARNING))
-		dd_dev_info(dd, "%s: Cable TX channel 1/2 bias too low\n",
-			    __func__);
+		dd_dev_err(dd, "%s: Cable TX channel 1/2 bias too low\n",
+			   __func__);
 
 	if ((qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_ALARM) ||
 	    (qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_WARNING))
-		dd_dev_info(dd, "%s: Cable TX channel 3/4 bias too high\n",
-			    __func__);
+		dd_dev_err(dd, "%s: Cable TX channel 3/4 bias too high\n",
+			   __func__);
 
 	if ((qsfp_interrupt_status[6] & QSFP_LOW_BIAS_ALARM) ||
 	    (qsfp_interrupt_status[6] & QSFP_LOW_BIAS_WARNING))
-		dd_dev_info(dd, "%s: Cable TX channel 3/4 bias too low\n",
-			    __func__);
+		dd_dev_err(dd, "%s: Cable TX channel 3/4 bias too low\n",
+			   __func__);
 
 	if ((qsfp_interrupt_status[7] & QSFP_HIGH_POWER_ALARM) ||
 	    (qsfp_interrupt_status[7] & QSFP_HIGH_POWER_WARNING))
-		dd_dev_info(dd, "%s: Cable TX channel 1/2 power too high\n",
-			    __func__);
+		dd_dev_err(dd, "%s: Cable TX channel 1/2 power too high\n",
+			   __func__);
 
 	if ((qsfp_interrupt_status[7] & QSFP_LOW_POWER_ALARM) ||
 	    (qsfp_interrupt_status[7] & QSFP_LOW_POWER_WARNING))
-		dd_dev_info(dd, "%s: Cable TX channel 1/2 power too low\n",
-			    __func__);
+		dd_dev_err(dd, "%s: Cable TX channel 1/2 power too low\n",
+			   __func__);
 
 	if ((qsfp_interrupt_status[8] & QSFP_HIGH_POWER_ALARM) ||
 	    (qsfp_interrupt_status[8] & QSFP_HIGH_POWER_WARNING))
-		dd_dev_info(dd, "%s: Cable TX channel 3/4 power too high\n",
-			    __func__);
+		dd_dev_err(dd, "%s: Cable TX channel 3/4 power too high\n",
+			   __func__);
 
 	if ((qsfp_interrupt_status[8] & QSFP_LOW_POWER_ALARM) ||
 	    (qsfp_interrupt_status[8] & QSFP_LOW_POWER_WARNING))
-		dd_dev_info(dd, "%s: Cable TX channel 3/4 power too low\n",
-			    __func__);
+		dd_dev_err(dd, "%s: Cable TX channel 3/4 power too low\n",
+			   __func__);
 
 	/* Bytes 9-10 and 11-12 are reserved */
 	/* Bytes 13-15 are vendor specific */
@@ -9480,6 +9559,13 @@ void qsfp_event(struct work_struct *work)
 	if (!qsfp_mod_present(ppd))
 		return;
 
+	if (ppd->host_link_state == HLS_DN_DISABLE) {
+		dd_dev_info(ppd->dd,
+			    "%s: stopping link start because link is disabled\n",
+			    __func__);
+		return;
+	}
+
 	/*
 	 * Turn DC back on after cable has been re-inserted. Up until
 	 * now, the DC has been in reset to save power.
@@ -9635,7 +9721,7 @@ static void try_start_link(struct hfi1_pportdata *ppd)
 			    "QSFP not responding, waiting and retrying %d\n",
 			    (int)ppd->qsfp_retry_count);
 		ppd->qsfp_retry_count++;
-		queue_delayed_work(ppd->hfi1_wq, &ppd->start_link_work,
+		queue_delayed_work(ppd->link_wq, &ppd->start_link_work,
 				   msecs_to_jiffies(QSFP_RETRY_WAIT));
 		return;
 	}
@@ -9742,17 +9828,6 @@ static inline int init_cpu_counters(struct hfi1_devdata *dd)
 	return 0;
 }
 
-static const char * const pt_names[] = {
-	"expected",
-	"eager",
-	"invalid"
-};
-
-static const char *pt_name(u32 type)
-{
-	return type >= ARRAY_SIZE(pt_names) ? "unknown" : pt_names[type];
-}
-
 /*
  * index is the index into the receive array
  */
@@ -9760,35 +9835,34 @@ void hfi1_put_tid(struct hfi1_devdata *dd, u32 index,
 		  u32 type, unsigned long pa, u16 order)
 {
 	u64 reg;
-	void __iomem *base = (dd->rcvarray_wc ? dd->rcvarray_wc :
-			      (dd->kregbase + RCV_ARRAY));
 
 	if (!(dd->flags & HFI1_PRESENT))
 		goto done;
 
-	if (type == PT_INVALID) {
+	if (type == PT_INVALID || type == PT_INVALID_FLUSH) {
 		pa = 0;
+		order = 0;
 	} else if (type > PT_INVALID) {
 		dd_dev_err(dd,
 			   "unexpected receive array type %u for index %u, not handled\n",
 			   type, index);
 		goto done;
 	}
-
-	hfi1_cdbg(TID, "type %s, index 0x%x, pa 0x%lx, bsize 0x%lx",
-		  pt_name(type), index, pa, (unsigned long)order);
+	trace_hfi1_put_tid(dd, index, type, pa, order);
 
 #define RT_ADDR_SHIFT 12	/* 4KB kernel address boundary */
 	reg = RCV_ARRAY_RT_WRITE_ENABLE_SMASK
 		| (u64)order << RCV_ARRAY_RT_BUF_SIZE_SHIFT
 		| ((pa >> RT_ADDR_SHIFT) & RCV_ARRAY_RT_ADDR_MASK)
 					<< RCV_ARRAY_RT_ADDR_SHIFT;
-	writeq(reg, base + (index * 8));
+	trace_hfi1_write_rcvarray(dd->rcvarray_wc + (index * 8), reg);
+	writeq(reg, dd->rcvarray_wc + (index * 8));
 
-	if (type == PT_EAGER)
+	if (type == PT_EAGER || type == PT_INVALID_FLUSH || (index & 3) == 3)
 		/*
-		 * Eager entries are written one-by-one so we have to push them
-		 * after we write the entry.
+		 * Eager entries are written and flushed
+		 *
+		 * Expected entries are flushed every 4 writes
 		 */
 		flush_wc();
 done:
@@ -9810,15 +9884,6 @@ void hfi1_clear_tids(struct hfi1_ctxtdata *rcd)
 		hfi1_put_tid(dd, i, PT_INVALID, 0, 0);
 }
 
-struct ib_header *hfi1_get_msgheader(
-	struct hfi1_devdata *dd, __le32 *rhf_addr)
-{
-	u32 offset = rhf_hdrq_offset(rhf_to_cpu(rhf_addr));
-
-	return (struct ib_header *)
-		(rhf_addr - dd->rhf_offset + offset);
-}
-
 static const char * const ib_cfg_name_strings[] = {
 	"HFI1_IB_CFG_LIDLMC",
 	"HFI1_IB_CFG_LWID_DG_ENB",
@@ -10010,10 +10075,16 @@ static void set_lidlmc(struct hfi1_pportdata *ppd)
 	struct hfi1_devdata *dd = ppd->dd;
 	u32 mask = ~((1U << ppd->lmc) - 1);
 	u64 c1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG1);
+	u32 lid;
 
+	/*
+	 * Program 0 in CSR if port lid is extended. This prevents
+	 * 9B packets being sent out for large lids.
+	 */
+	lid = (ppd->lid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) ? 0 : ppd->lid;
 	c1 &= ~(DCC_CFG_PORT_CONFIG1_TARGET_DLID_SMASK
 		| DCC_CFG_PORT_CONFIG1_DLID_MASK_SMASK);
-	c1 |= ((ppd->lid & DCC_CFG_PORT_CONFIG1_TARGET_DLID_MASK)
+	c1 |= ((lid & DCC_CFG_PORT_CONFIG1_TARGET_DLID_MASK)
 			<< DCC_CFG_PORT_CONFIG1_TARGET_DLID_SHIFT) |
 	      ((mask & DCC_CFG_PORT_CONFIG1_DLID_MASK_MASK)
 			<< DCC_CFG_PORT_CONFIG1_DLID_MASK_SHIFT);
@@ -10024,7 +10095,7 @@ static void set_lidlmc(struct hfi1_pportdata *ppd)
 	 */
 	sreg = ((mask & SEND_CTXT_CHECK_SLID_MASK_MASK) <<
 			SEND_CTXT_CHECK_SLID_MASK_SHIFT) |
-	       (((ppd->lid & mask) & SEND_CTXT_CHECK_SLID_VALUE_MASK) <<
+	       (((lid & mask) & SEND_CTXT_CHECK_SLID_VALUE_MASK) <<
 			SEND_CTXT_CHECK_SLID_VALUE_SHIFT);
 
 	for (i = 0; i < dd->chip_send_contexts; i++) {
@@ -10034,29 +10105,7 @@ static void set_lidlmc(struct hfi1_pportdata *ppd)
 	}
 
 	/* Now we have to do the same thing for the sdma engines */
-	sdma_update_lmc(dd, mask, ppd->lid);
-}
-
-static int wait_phy_linkstate(struct hfi1_devdata *dd, u32 state, u32 msecs)
-{
-	unsigned long timeout;
-	u32 curr_state;
-
-	timeout = jiffies + msecs_to_jiffies(msecs);
-	while (1) {
-		curr_state = read_physical_state(dd);
-		if (curr_state == state)
-			break;
-		if (time_after(jiffies, timeout)) {
-			dd_dev_err(dd,
-				   "timeout waiting for phy link state 0x%x, current state is 0x%x\n",
-				   state, curr_state);
-			return -ETIMEDOUT;
-		}
-		usleep_range(1950, 2050); /* sleep 2ms-ish */
-	}
-
-	return 0;
+	sdma_update_lmc(dd, mask, lid);
 }
 
 static const char *state_completed_string(u32 completed)
@@ -10238,8 +10287,10 @@ static void force_logical_link_state_down(struct hfi1_pportdata *ppd)
 	write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP, 0);
 	write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK, 0);
 
-	/* call again to adjust ppd->statusp, if needed */
-	get_logical_state(ppd);
+	/* adjust ppd->statusp, if needed */
+	update_statusp(ppd, IB_PORT_DOWN);
+
+	dd_dev_info(ppd->dd, "logical state forced to LINK_DOWN\n");
 }
 
 /*
@@ -10253,49 +10304,35 @@ static void force_logical_link_state_down(struct hfi1_pportdata *ppd)
 static int goto_offline(struct hfi1_pportdata *ppd, u8 rem_reason)
 {
 	struct hfi1_devdata *dd = ppd->dd;
-	u32 pstate, previous_state;
+	u32 previous_state;
 	int ret;
-	int do_transition;
-	int do_wait;
 
 	update_lcb_cache(dd);
 
 	previous_state = ppd->host_link_state;
 	ppd->host_link_state = HLS_GOING_OFFLINE;
-	pstate = read_physical_state(dd);
-	if (pstate == PLS_OFFLINE) {
-		do_transition = 0;	/* in right state */
-		do_wait = 0;		/* ...no need to wait */
-	} else if ((pstate & 0xf0) == PLS_OFFLINE) {
-		do_transition = 0;	/* in an offline transient state */
-		do_wait = 1;		/* ...wait for it to settle */
-	} else {
-		do_transition = 1;	/* need to move to offline */
-		do_wait = 1;		/* ...will need to wait */
-	}
 
-	if (do_transition) {
-		ret = set_physical_link_state(dd,
-					      (rem_reason << 8) | PLS_OFFLINE);
+	/* start offline transition */
+	ret = set_physical_link_state(dd, (rem_reason << 8) | PLS_OFFLINE);
 
-		if (ret != HCMD_SUCCESS) {
-			dd_dev_err(dd,
-				   "Failed to transition to Offline link state, return %d\n",
-				   ret);
-			return -EINVAL;
-		}
-		if (ppd->offline_disabled_reason ==
-				HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE))
-			ppd->offline_disabled_reason =
-			HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT);
+	if (ret != HCMD_SUCCESS) {
+		dd_dev_err(dd,
+			   "Failed to transition to Offline link state, return %d\n",
+			   ret);
+		return -EINVAL;
 	}
+	if (ppd->offline_disabled_reason ==
+			HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE))
+		ppd->offline_disabled_reason =
+		HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT);
 
-	if (do_wait) {
-		/* it can take a while for the link to go down */
-		ret = wait_phy_linkstate(dd, PLS_OFFLINE, 10000);
-		if (ret < 0)
-			return ret;
-	}
+	/*
+	 * Wait for offline transition. It can take a while for
+	 * the link to go down.
+	 */
+	ret = wait_physical_linkstate(ppd, PLS_OFFLINE, 10000);
+	if (ret < 0)
+		return ret;
 
 	/*
 	 * Now in charge of LCB - must be after the physical state is
@@ -10415,11 +10452,11 @@ static const char *link_state_reason_name(struct hfi1_pportdata *ppd, u32 state)
 }
 
 /*
- * driver_physical_state - convert the driver's notion of a port's
+ * driver_pstate - convert the driver's notion of a port's
  * state (an HLS_*) into a physical state (a {IB,OPA}_PORTPHYSSTATE_*).
  * Return -1 (converted to a u32) to indicate error.
  */
-u32 driver_physical_state(struct hfi1_pportdata *ppd)
+u32 driver_pstate(struct hfi1_pportdata *ppd)
 {
 	switch (ppd->host_link_state) {
 	case HLS_UP_INIT:
@@ -10449,11 +10486,11 @@ u32 driver_physical_state(struct hfi1_pportdata *ppd)
 }
 
 /*
- * driver_logical_state - convert the driver's notion of a port's
+ * driver_lstate - convert the driver's notion of a port's
  * state (an HLS_*) into a logical state (a IB_PORT_*). Return -1
  * (converted to a u32) to indicate error.
  */
-u32 driver_logical_state(struct hfi1_pportdata *ppd)
+u32 driver_lstate(struct hfi1_pportdata *ppd)
 {
 	if (ppd->host_link_state && (ppd->host_link_state & HLS_DOWN))
 		return IB_PORT_DOWN;
@@ -10484,6 +10521,14 @@ void set_link_down_reason(struct hfi1_pportdata *ppd, u8 lcl_reason,
 }
 
 /*
+ * Verify if BCT for data VLs is non-zero.
+ */
+static inline bool data_vls_operational(struct hfi1_pportdata *ppd)
+{
+	return !!ppd->actual_vls_operational;
+}
+
+/*
  * Change the physical and/or logical link state.
  *
  * Do not call this routine while inside an interrupt.  It contains
@@ -10545,38 +10590,58 @@ int set_link_state(struct hfi1_pportdata *ppd, u32 state)
 			goto unexpected;
 		}
 
+		/*
+		 * Wait for Link_Up physical state.
+		 * Physical and Logical states should already be
+		 * be transitioned to LinkUp and LinkInit respectively.
+		 */
+		ret = wait_physical_linkstate(ppd, PLS_LINKUP, 1000);
+		if (ret) {
+			dd_dev_err(dd,
+				   "%s: physical state did not change to LINK-UP\n",
+				   __func__);
+			break;
+		}
+
 		ret = wait_logical_linkstate(ppd, IB_PORT_INIT, 1000);
 		if (ret) {
 			dd_dev_err(dd,
 				   "%s: logical state did not change to INIT\n",
 				   __func__);
-		} else {
-			/* clear old transient LINKINIT_REASON code */
-			if (ppd->linkinit_reason >= OPA_LINKINIT_REASON_CLEAR)
-				ppd->linkinit_reason =
-					OPA_LINKINIT_REASON_LINKUP;
+			break;
+		}
 
-			/* enable the port */
-			add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
+		/* clear old transient LINKINIT_REASON code */
+		if (ppd->linkinit_reason >= OPA_LINKINIT_REASON_CLEAR)
+			ppd->linkinit_reason =
+				OPA_LINKINIT_REASON_LINKUP;
 
-			handle_linkup_change(dd, 1);
-			ppd->host_link_state = HLS_UP_INIT;
-		}
+		/* enable the port */
+		add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
+
+		handle_linkup_change(dd, 1);
+		ppd->host_link_state = HLS_UP_INIT;
 		break;
 	case HLS_UP_ARMED:
 		if (ppd->host_link_state != HLS_UP_INIT)
 			goto unexpected;
 
-		ppd->host_link_state = HLS_UP_ARMED;
+		if (!data_vls_operational(ppd)) {
+			dd_dev_err(dd,
+				   "%s: data VLs not operational\n", __func__);
+			ret = -EINVAL;
+			break;
+		}
+
 		set_logical_state(dd, LSTATE_ARMED);
 		ret = wait_logical_linkstate(ppd, IB_PORT_ARMED, 1000);
 		if (ret) {
-			/* logical state didn't change, stay at init */
-			ppd->host_link_state = HLS_UP_INIT;
 			dd_dev_err(dd,
 				   "%s: logical state did not change to ARMED\n",
 				   __func__);
+			break;
 		}
+		ppd->host_link_state = HLS_UP_ARMED;
 		/*
 		 * The simulator does not currently implement SMA messages,
 		 * so neighbor_normal is not set.  Set it here when we first
@@ -10589,18 +10654,16 @@ int set_link_state(struct hfi1_pportdata *ppd, u32 state)
 		if (ppd->host_link_state != HLS_UP_ARMED)
 			goto unexpected;
 
-		ppd->host_link_state = HLS_UP_ACTIVE;
 		set_logical_state(dd, LSTATE_ACTIVE);
 		ret = wait_logical_linkstate(ppd, IB_PORT_ACTIVE, 1000);
 		if (ret) {
-			/* logical state didn't change, stay at armed */
-			ppd->host_link_state = HLS_UP_ARMED;
 			dd_dev_err(dd,
 				   "%s: logical state did not change to ACTIVE\n",
 				   __func__);
 		} else {
 			/* tell all engines to go running */
 			sdma_all_running(dd);
+			ppd->host_link_state = HLS_UP_ACTIVE;
 
 			/* Signal the IB layer that the port has went active */
 			event.device = &dd->verbs_dev.rdi.ibdev;
@@ -10658,6 +10721,8 @@ int set_link_state(struct hfi1_pportdata *ppd, u32 state)
 		 */
 		if (ret)
 			goto_offline(ppd, 0);
+		else
+			log_physical_state(ppd, PLS_POLLING);
 		break;
 	case HLS_DN_DISABLE:
 		/* link is disabled */
@@ -10682,6 +10747,13 @@ int set_link_state(struct hfi1_pportdata *ppd, u32 state)
 				ret = -EINVAL;
 				break;
 			}
+			ret = wait_physical_linkstate(ppd, PLS_DISABLED, 10000);
+			if (ret) {
+				dd_dev_err(dd,
+					   "%s: physical state did not change to DISABLED\n",
+					   __func__);
+				break;
+			}
 			dc_shutdown(dd);
 		}
 		ppd->host_link_state = HLS_DN_DISABLE;
@@ -10699,6 +10771,7 @@ int set_link_state(struct hfi1_pportdata *ppd, u32 state)
 		if (ppd->host_link_state != HLS_DN_POLL)
 			goto unexpected;
 		ppd->host_link_state = HLS_VERIFY_CAP;
+		log_physical_state(ppd, PLS_CONFIGPHY_VERIFYCAP);
 		break;
 	case HLS_GOING_UP:
 		if (ppd->host_link_state != HLS_VERIFY_CAP)
@@ -11693,16 +11766,18 @@ static u32 encoded_size(u32 size)
 	return 0x1;	/* if invalid, go with the minimum size */
 }
 
-void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op, int ctxt)
+void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op,
+		  struct hfi1_ctxtdata *rcd)
 {
-	struct hfi1_ctxtdata *rcd;
 	u64 rcvctrl, reg;
 	int did_enable = 0;
+	u16 ctxt;
 
-	rcd = dd->rcd[ctxt];
 	if (!rcd)
 		return;
 
+	ctxt = rcd->ctxt;
+
 	hfi1_cdbg(RCVCTRL, "ctxt %d op 0x%x", ctxt, op);
 
 	rcvctrl = read_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL);
@@ -12604,20 +12679,8 @@ const char *opa_pstate_name(u32 pstate)
 	return "unknown";
 }
 
-/*
- * Read the hardware link state and set the driver's cached value of it.
- * Return the (new) current value.
- */
-u32 get_logical_state(struct hfi1_pportdata *ppd)
+static void update_statusp(struct hfi1_pportdata *ppd, u32 state)
 {
-	u32 new_state;
-
-	new_state = chip_to_opa_lstate(ppd->dd, read_logical_state(ppd->dd));
-	if (new_state != ppd->lstate) {
-		dd_dev_info(ppd->dd, "logical state changed to %s (0x%x)\n",
-			    opa_lstate_name(new_state), new_state);
-		ppd->lstate = new_state;
-	}
 	/*
 	 * Set port status flags in the page mapped into userspace
 	 * memory. Do it here to ensure a reliable state - this is
@@ -12627,7 +12690,7 @@ u32 get_logical_state(struct hfi1_pportdata *ppd)
 	 * function.
 	 */
 	if (ppd->statusp) {
-		switch (ppd->lstate) {
+		switch (state) {
 		case IB_PORT_DOWN:
 		case IB_PORT_INIT:
 			*ppd->statusp &= ~(HFI1_STATUS_IB_CONF |
@@ -12641,10 +12704,9 @@ u32 get_logical_state(struct hfi1_pportdata *ppd)
 			break;
 		}
 	}
-	return ppd->lstate;
 }
 
-/**
+/*
  * wait_logical_linkstate - wait for an IB link state change to occur
  * @ppd: port device
  * @state: the state to wait for
@@ -12658,35 +12720,88 @@ static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state,
 				  int msecs)
 {
 	unsigned long timeout;
+	u32 new_state;
 
 	timeout = jiffies + msecs_to_jiffies(msecs);
 	while (1) {
-		if (get_logical_state(ppd) == state)
-			return 0;
-		if (time_after(jiffies, timeout))
+		new_state = chip_to_opa_lstate(ppd->dd,
+					       read_logical_state(ppd->dd));
+		if (new_state == state)
 			break;
+		if (time_after(jiffies, timeout)) {
+			dd_dev_err(ppd->dd,
+				   "timeout waiting for link state 0x%x\n",
+				   state);
+			return -ETIMEDOUT;
+		}
 		msleep(20);
 	}
-	dd_dev_err(ppd->dd, "timeout waiting for link state 0x%x\n", state);
 
-	return -ETIMEDOUT;
+	update_statusp(ppd, state);
+	dd_dev_info(ppd->dd,
+		    "logical state changed to %s (0x%x)\n",
+		    opa_lstate_name(state),
+		    state);
+	return 0;
 }
 
-u8 hfi1_ibphys_portstate(struct hfi1_pportdata *ppd)
+static void log_state_transition(struct hfi1_pportdata *ppd, u32 state)
 {
-	u32 pstate;
-	u32 ib_pstate;
+	u32 ib_pstate = chip_to_opa_pstate(ppd->dd, state);
 
-	pstate = read_physical_state(ppd->dd);
-	ib_pstate = chip_to_opa_pstate(ppd->dd, pstate);
-	if (ppd->last_pstate != ib_pstate) {
-		dd_dev_info(ppd->dd,
-			    "%s: physical state changed to %s (0x%x), phy 0x%x\n",
-			    __func__, opa_pstate_name(ib_pstate), ib_pstate,
-			    pstate);
-		ppd->last_pstate = ib_pstate;
+	dd_dev_info(ppd->dd,
+		    "physical state changed to %s (0x%x), phy 0x%x\n",
+		    opa_pstate_name(ib_pstate), ib_pstate, state);
+}
+
+/*
+ * Read the physical hardware link state and check if it matches host
+ * drivers anticipated state.
+ */
+static void log_physical_state(struct hfi1_pportdata *ppd, u32 state)
+{
+	u32 read_state = read_physical_state(ppd->dd);
+
+	if (read_state == state) {
+		log_state_transition(ppd, state);
+	} else {
+		dd_dev_err(ppd->dd,
+			   "anticipated phy link state 0x%x, read 0x%x\n",
+			   state, read_state);
 	}
-	return ib_pstate;
+}
+
+/*
+ * wait_physical_linkstate - wait for an physical link state change to occur
+ * @ppd: port device
+ * @state: the state to wait for
+ * @msecs: the number of milliseconds to wait
+ *
+ * Wait up to msecs milliseconds for physical link state change to occur.
+ * Returns 0 if state reached, otherwise -ETIMEDOUT.
+ */
+static int wait_physical_linkstate(struct hfi1_pportdata *ppd, u32 state,
+				   int msecs)
+{
+	u32 read_state;
+	unsigned long timeout;
+
+	timeout = jiffies + msecs_to_jiffies(msecs);
+	while (1) {
+		read_state = read_physical_state(ppd->dd);
+		if (read_state == state)
+			break;
+		if (time_after(jiffies, timeout)) {
+			dd_dev_err(ppd->dd,
+				   "timeout waiting for phy link state 0x%x\n",
+				   state);
+			return -ETIMEDOUT;
+		}
+		usleep_range(1950, 2050); /* sleep 2ms-ish */
+	}
+
+	log_state_transition(ppd, state);
+	return 0;
 }
 
 #define CLEAR_STATIC_RATE_CONTROL_SMASK(r) \
@@ -12809,30 +12924,24 @@ static void clean_up_interrupts(struct hfi1_devdata *dd)
 		for (i = 0; i < dd->num_msix_entries; i++, me++) {
 			if (!me->arg) /* => no irq, no affinity */
 				continue;
-			hfi1_put_irq_affinity(dd, &dd->msix_entries[i]);
-			free_irq(me->msix.vector, me->arg);
+			hfi1_put_irq_affinity(dd, me);
+			free_irq(me->irq, me->arg);
 		}
+
+		/* clean structures */
+		kfree(dd->msix_entries);
+		dd->msix_entries = NULL;
+		dd->num_msix_entries = 0;
 	} else {
 		/* INTx */
 		if (dd->requested_intx_irq) {
 			free_irq(dd->pcidev->irq, dd);
 			dd->requested_intx_irq = 0;
 		}
-	}
-
-	/* turn off interrupts */
-	if (dd->num_msix_entries) {
-		/* MSI-X */
-		pci_disable_msix(dd->pcidev);
-	} else {
-		/* INTx */
 		disable_intx(dd->pcidev);
 	}
 
-	/* clean structures */
-	kfree(dd->msix_entries);
-	dd->msix_entries = NULL;
-	dd->num_msix_entries = 0;
+	pci_free_irq_vectors(dd->pcidev);
 }
 
 /*
@@ -12953,7 +13062,7 @@ static int request_msix_irqs(struct hfi1_devdata *dd)
 			me->type = IRQ_SDMA;
 		} else if (first_rx <= i && i < last_rx) {
 			idx = i - first_rx;
-			rcd = dd->rcd[idx];
+			rcd = hfi1_rcd_get_by_index(dd, idx);
 			if (rcd) {
 				/*
 				 * Set the interrupt register and mask for this
@@ -12972,6 +13081,7 @@ static int request_msix_irqs(struct hfi1_devdata *dd)
 				remap_intr(dd, IS_RCVAVAIL_START + idx, i);
 				me->type = IRQ_RCVCTXT;
 				rcd->msix_intr = i;
+				hfi1_rcd_put(rcd);
 			}
 		} else {
 			/* not in our expected range - complain, then
@@ -12986,13 +13096,21 @@ static int request_msix_irqs(struct hfi1_devdata *dd)
 			continue;
 		/* make sure the name is terminated */
 		me->name[sizeof(me->name) - 1] = 0;
+		me->irq = pci_irq_vector(dd->pcidev, i);
+		/*
+		 * On err return me->irq.  Don't need to clear this
+		 * because 'arg' has not been set, and cleanup will
+		 * do the right thing.
+		 */
+		if (me->irq < 0)
+			return me->irq;
 
-		ret = request_threaded_irq(me->msix.vector, handler, thread, 0,
+		ret = request_threaded_irq(me->irq, handler, thread, 0,
 					   me->name, arg);
 		if (ret) {
 			dd_dev_err(dd,
-				   "unable to allocate %s interrupt, vector %d, index %d, err %d\n",
-				   err_info, me->msix.vector, idx, ret);
+				   "unable to allocate %s interrupt, irq %d, index %d, err %d\n",
+				   err_info, me->irq, idx, ret);
 			return ret;
 		}
 		/*
@@ -13003,8 +13121,7 @@ static int request_msix_irqs(struct hfi1_devdata *dd)
 
 		ret = hfi1_get_irq_affinity(dd, me);
 		if (ret)
-			dd_dev_err(dd,
-				   "unable to pin IRQ %d\n", ret);
+			dd_dev_err(dd, "unable to pin IRQ %d\n", ret);
 	}
 
 	return ret;
@@ -13023,7 +13140,7 @@ void hfi1_vnic_synchronize_irq(struct hfi1_devdata *dd)
 		struct hfi1_ctxtdata *rcd = dd->vnic.ctxt[i];
 		struct hfi1_msix_entry *me = &dd->msix_entries[rcd->msix_intr];
 
-		synchronize_irq(me->msix.vector);
+		synchronize_irq(me->irq);
 	}
 }
 
@@ -13036,7 +13153,7 @@ void hfi1_reset_vnic_msix_info(struct hfi1_ctxtdata *rcd)
 		return;
 
 	hfi1_put_irq_affinity(dd, me);
-	free_irq(me->msix.vector, me->arg);
+	free_irq(me->irq, me->arg);
 
 	me->arg = NULL;
 }
@@ -13064,14 +13181,19 @@ void hfi1_set_vnic_msix_info(struct hfi1_ctxtdata *rcd)
 		 DRIVER_NAME "_%d kctxt%d", dd->unit, idx);
 	me->name[sizeof(me->name) - 1] = 0;
 	me->type = IRQ_RCVCTXT;
-
+	me->irq = pci_irq_vector(dd->pcidev, rcd->msix_intr);
+	if (me->irq < 0) {
+		dd_dev_err(dd, "vnic irq vector request (idx %d) fail %d\n",
+			   idx, me->irq);
+		return;
+	}
 	remap_intr(dd, IS_RCVAVAIL_START + idx, rcd->msix_intr);
 
-	ret = request_threaded_irq(me->msix.vector, receive_context_interrupt,
+	ret = request_threaded_irq(me->irq, receive_context_interrupt,
 				   receive_context_thread, 0, me->name, arg);
 	if (ret) {
-		dd_dev_err(dd, "vnic irq request (vector %d, idx %d) fail %d\n",
-			   me->msix.vector, idx, ret);
+		dd_dev_err(dd, "vnic irq request (irq %d, idx %d) fail %d\n",
+			   me->irq, idx, ret);
 		return;
 	}
 	/*
@@ -13084,7 +13206,7 @@ void hfi1_set_vnic_msix_info(struct hfi1_ctxtdata *rcd)
 	if (ret) {
 		dd_dev_err(dd,
 			   "unable to pin IRQ %d\n", ret);
-		free_irq(me->msix.vector, me->arg);
+		free_irq(me->irq, me->arg);
 	}
 }
 
@@ -13107,9 +13229,8 @@ static void reset_interrupts(struct hfi1_devdata *dd)
 
 static int set_up_interrupts(struct hfi1_devdata *dd)
 {
-	struct hfi1_msix_entry *entries;
-	u32 total, request;
-	int i, ret;
+	u32 total;
+	int ret, request;
 	int single_interrupt = 0; /* we expect to have all the interrupts */
 
 	/*
@@ -13121,39 +13242,31 @@ static int set_up_interrupts(struct hfi1_devdata *dd)
 	 */
 	total = 1 + dd->num_sdma + dd->n_krcv_queues + HFI1_NUM_VNIC_CTXT;
 
-	entries = kcalloc(total, sizeof(*entries), GFP_KERNEL);
-	if (!entries) {
-		ret = -ENOMEM;
-		goto fail;
-	}
-	/* 1-1 MSI-X entry assignment */
-	for (i = 0; i < total; i++)
-		entries[i].msix.entry = i;
-
 	/* ask for MSI-X interrupts */
-	request = total;
-	request_msix(dd, &request, entries);
-
-	if (request == 0) {
+	request = request_msix(dd, total);
+	if (request < 0) {
+		ret = request;
+		goto fail;
+	} else if (request == 0) {
 		/* using INTx */
 		/* dd->num_msix_entries already zero */
-		kfree(entries);
 		single_interrupt = 1;
 		dd_dev_err(dd, "MSI-X failed, using INTx interrupts\n");
+	} else if (request < total) {
+		/* using MSI-X, with reduced interrupts */
+		dd_dev_err(dd, "reduced interrupt found, wanted %u, got %u\n",
+			   total, request);
+		ret = -EINVAL;
+		goto fail;
 	} else {
-		/* using MSI-X */
-		dd->num_msix_entries = request;
-		dd->msix_entries = entries;
-
-		if (request != total) {
-			/* using MSI-X, with reduced interrupts */
-			dd_dev_err(
-				dd,
-				"cannot handle reduced interrupt case, want %u, got %u\n",
-				total, request);
-			ret = -EINVAL;
+		dd->msix_entries = kcalloc(total, sizeof(*dd->msix_entries),
+					   GFP_KERNEL);
+		if (!dd->msix_entries) {
+			ret = -ENOMEM;
 			goto fail;
 		}
+		/* using MSI-X */
+		dd->num_msix_entries = total;
 		dd_dev_info(dd, "%u MSI-X interrupts allocated\n", total);
 	}
 
@@ -13396,8 +13509,7 @@ static void write_uninitialized_csrs_and_memories(struct hfi1_devdata *dd)
 
 	/* RcvArray */
 	for (i = 0; i < dd->chip_rcv_array_count; i++)
-		write_csr(dd, RCV_ARRAY + (8 * i),
-			  RCV_ARRAY_RT_WRITE_ENABLE_SMASK);
+		hfi1_put_tid(dd, i, PT_INVALID_FLUSH, 0, 0);
 
 	/* RcvQPMapTable */
 	for (i = 0; i < 32; i++)
@@ -13831,9 +13943,10 @@ static void init_sc2vl_tables(struct hfi1_devdata *dd)
  * a reset following the (possible) FLR in this routine.
  *
  */
-static void init_chip(struct hfi1_devdata *dd)
+static int init_chip(struct hfi1_devdata *dd)
 {
 	int i;
+	int ret = 0;
 
 	/*
 	 * Put the HFI CSRs in a known state.
@@ -13881,12 +13994,22 @@ static void init_chip(struct hfi1_devdata *dd)
 		pcie_flr(dd->pcidev);
 
 		/* restore command and BARs */
-		restore_pci_variables(dd);
+		ret = restore_pci_variables(dd);
+		if (ret) {
+			dd_dev_err(dd, "%s: Could not restore PCI variables\n",
+				   __func__);
+			return ret;
+		}
 
 		if (is_ax(dd)) {
 			dd_dev_info(dd, "Resetting CSRs with FLR\n");
 			pcie_flr(dd->pcidev);
-			restore_pci_variables(dd);
+			ret = restore_pci_variables(dd);
+			if (ret) {
+				dd_dev_err(dd, "%s: Could not restore PCI variables\n",
+					   __func__);
+				return ret;
+			}
 		}
 	} else {
 		dd_dev_info(dd, "Resetting CSRs with writes\n");
@@ -13914,6 +14037,7 @@ static void init_chip(struct hfi1_devdata *dd)
 	write_csr(dd, ASIC_QSFP1_OUT, 0x1f);
 	write_csr(dd, ASIC_QSFP2_OUT, 0x1f);
 	init_chip_resources(dd);
+	return ret;
 }
 
 static void init_early_variables(struct hfi1_devdata *dd)
@@ -14365,6 +14489,7 @@ void hfi1_deinit_vnic_rsm(struct hfi1_devdata *dd)
 static void init_rxe(struct hfi1_devdata *dd)
 {
 	struct rsm_map_table *rmt;
+	u64 val;
 
 	/* enable all receive errors */
 	write_csr(dd, RCV_ERR_MASK, ~0ull);
@@ -14389,6 +14514,11 @@ static void init_rxe(struct hfi1_devdata *dd)
 	 * (64 bytes).  Max_Payload_Size is possibly modified upward in
 	 * tune_pcie_caps() which is called after this routine.
 	 */
+
+	/* Have 16 bytes (4DW) of bypass header available in header queue */
+	val = read_csr(dd, RCV_BYPASS);
+	val |= (4ull << 16);
+	write_csr(dd, RCV_BYPASS, val);
 }
 
 static void init_other(struct hfi1_devdata *dd)
@@ -14470,99 +14600,86 @@ static void init_txe(struct hfi1_devdata *dd)
 		write_csr(dd, SEND_CM_TIMER_CTRL, HFI1_CREDIT_RETURN_RATE);
 }
 
-int hfi1_set_ctxt_jkey(struct hfi1_devdata *dd, unsigned ctxt, u16 jkey)
+int hfi1_set_ctxt_jkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd,
+		       u16 jkey)
 {
-	struct hfi1_ctxtdata *rcd = dd->rcd[ctxt];
-	unsigned sctxt;
-	int ret = 0;
+	u8 hw_ctxt;
 	u64 reg;
 
-	if (!rcd || !rcd->sc) {
-		ret = -EINVAL;
-		goto done;
-	}
-	sctxt = rcd->sc->hw_context;
+	if (!rcd || !rcd->sc)
+		return -EINVAL;
+
+	hw_ctxt = rcd->sc->hw_context;
 	reg = SEND_CTXT_CHECK_JOB_KEY_MASK_SMASK | /* mask is always 1's */
 		((jkey & SEND_CTXT_CHECK_JOB_KEY_VALUE_MASK) <<
 		 SEND_CTXT_CHECK_JOB_KEY_VALUE_SHIFT);
 	/* JOB_KEY_ALLOW_PERMISSIVE is not allowed by default */
 	if (HFI1_CAP_KGET_MASK(rcd->flags, ALLOW_PERM_JKEY))
 		reg |= SEND_CTXT_CHECK_JOB_KEY_ALLOW_PERMISSIVE_SMASK;
-	write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_JOB_KEY, reg);
+	write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_JOB_KEY, reg);
 	/*
 	 * Enable send-side J_KEY integrity check, unless this is A0 h/w
 	 */
 	if (!is_ax(dd)) {
-		reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
+		reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE);
 		reg |= SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
-		write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
+		write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg);
 	}
 
 	/* Enable J_KEY check on receive context. */
 	reg = RCV_KEY_CTRL_JOB_KEY_ENABLE_SMASK |
 		((jkey & RCV_KEY_CTRL_JOB_KEY_VALUE_MASK) <<
 		 RCV_KEY_CTRL_JOB_KEY_VALUE_SHIFT);
-	write_kctxt_csr(dd, ctxt, RCV_KEY_CTRL, reg);
-done:
-	return ret;
+	write_kctxt_csr(dd, rcd->ctxt, RCV_KEY_CTRL, reg);
+
+	return 0;
 }
 
-int hfi1_clear_ctxt_jkey(struct hfi1_devdata *dd, unsigned ctxt)
+int hfi1_clear_ctxt_jkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
 {
-	struct hfi1_ctxtdata *rcd = dd->rcd[ctxt];
-	unsigned sctxt;
-	int ret = 0;
+	u8 hw_ctxt;
 	u64 reg;
 
-	if (!rcd || !rcd->sc) {
-		ret = -EINVAL;
-		goto done;
-	}
-	sctxt = rcd->sc->hw_context;
-	write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_JOB_KEY, 0);
+	if (!rcd || !rcd->sc)
+		return -EINVAL;
+
+	hw_ctxt = rcd->sc->hw_context;
+	write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_JOB_KEY, 0);
 	/*
 	 * Disable send-side J_KEY integrity check, unless this is A0 h/w.
 	 * This check would not have been enabled for A0 h/w, see
 	 * set_ctxt_jkey().
 	 */
 	if (!is_ax(dd)) {
-		reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
+		reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE);
 		reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
-		write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
+		write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg);
 	}
 	/* Turn off the J_KEY on the receive side */
-	write_kctxt_csr(dd, ctxt, RCV_KEY_CTRL, 0);
-done:
-	return ret;
+	write_kctxt_csr(dd, rcd->ctxt, RCV_KEY_CTRL, 0);
+
+	return 0;
 }
 
-int hfi1_set_ctxt_pkey(struct hfi1_devdata *dd, unsigned ctxt, u16 pkey)
+int hfi1_set_ctxt_pkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd,
+		       u16 pkey)
 {
-	struct hfi1_ctxtdata *rcd;
-	unsigned sctxt;
-	int ret = 0;
+	u8 hw_ctxt;
 	u64 reg;
 
-	if (ctxt < dd->num_rcv_contexts) {
-		rcd = dd->rcd[ctxt];
-	} else {
-		ret = -EINVAL;
-		goto done;
-	}
-	if (!rcd || !rcd->sc) {
-		ret = -EINVAL;
-		goto done;
-	}
-	sctxt = rcd->sc->hw_context;
+	if (!rcd || !rcd->sc)
+		return -EINVAL;
+
+	hw_ctxt = rcd->sc->hw_context;
 	reg = ((u64)pkey & SEND_CTXT_CHECK_PARTITION_KEY_VALUE_MASK) <<
 		SEND_CTXT_CHECK_PARTITION_KEY_VALUE_SHIFT;
-	write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_PARTITION_KEY, reg);
-	reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE);
+	write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_PARTITION_KEY, reg);
+	reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE);
 	reg |= SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK;
 	reg &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_KDETH_PACKETS_SMASK;
-	write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg);
-done:
-	return ret;
+	write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg);
+
+	return 0;
 }
 
 int hfi1_clear_ctxt_pkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *ctxt)
@@ -14573,9 +14690,6 @@ int hfi1_clear_ctxt_pkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *ctxt)
 	if (!ctxt || !ctxt->sc)
 		return -EINVAL;
 
-	if (ctxt->ctxt >= dd->num_rcv_contexts)
-		return -EINVAL;
-
 	hw_ctxt = ctxt->sc->hw_context;
 	reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE);
 	reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK;
@@ -14773,7 +14887,6 @@ struct hfi1_devdata *hfi1_init_dd(struct pci_dev *pdev,
 		}
 		ppd->vls_supported = num_vls;
 		ppd->vls_operational = ppd->vls_supported;
-		ppd->actual_vls_operational = ppd->vls_supported;
 		/* Set the default MTU. */
 		for (vl = 0; vl < num_vls; vl++)
 			dd->vld[vl].mtu = hfi1_max_mtu;
@@ -14782,7 +14895,6 @@ struct hfi1_devdata *hfi1_init_dd(struct pci_dev *pdev,
 		 * Set the initial values to reasonable default, will be set
 		 * for real when link is up.
 		 */
-		ppd->lstate = IB_PORT_DOWN;
 		ppd->overrun_threshold = 0x4;
 		ppd->phy_error_threshold = 0xf;
 		ppd->port_crc_mode_enabled = link_crc_mask;
@@ -14793,7 +14905,6 @@ struct hfi1_devdata *hfi1_init_dd(struct pci_dev *pdev,
 		/* start in offline */
 		ppd->host_link_state = HLS_DN_OFFLINE;
 		init_vl_arb_caches(ppd);
-		ppd->last_pstate = 0xff; /* invalid value */
 	}
 
 	dd->link_default = HLS_DN_POLL;
@@ -14807,6 +14918,11 @@ struct hfi1_devdata *hfi1_init_dd(struct pci_dev *pdev,
 	if (ret < 0)
 		goto bail_free;
 
+	/* Save PCI space registers to rewrite after device reset */
+	ret = save_pci_variables(dd);
+	if (ret < 0)
+		goto bail_cleanup;
+
 	/* verify that reads actually work, save revision for reset check */
 	dd->revision = read_csr(dd, CCE_REVISION);
 	if (dd->revision == ~(u64)0) {
@@ -14899,7 +15015,9 @@ struct hfi1_devdata *hfi1_init_dd(struct pci_dev *pdev,
 		goto bail_cleanup;
 
 	/* obtain chip sizes, reset chip CSRs */
-	init_chip(dd);
+	ret = init_chip(dd);
+	if (ret)
+		goto bail_cleanup;
 
 	/* read in the PCIe link speed information */
 	ret = pcie_speeds(dd);
@@ -14974,10 +15092,16 @@ struct hfi1_devdata *hfi1_init_dd(struct pci_dev *pdev,
 	if (ret)
 		goto bail_cleanup;
 
-	ret = hfi1_create_ctxts(dd);
+	ret = hfi1_create_kctxts(dd);
 	if (ret)
 		goto bail_cleanup;
 
+	/*
+	 * Initialize aspm, to be done after gen3 transition and setting up
+	 * contexts and before enabling interrupts
+	 */
+	aspm_init(dd);
+
 	dd->rcvhdrsize = DEFAULT_RCVHDRSIZE;
 	/*
 	 * rcd[0] is guaranteed to be valid by this point. Also, all
@@ -14996,7 +15120,7 @@ struct hfi1_devdata *hfi1_init_dd(struct pci_dev *pdev,
 			goto bail_cleanup;
 	}
 
-	/* use contexts created by hfi1_create_ctxts */
+	/* use contexts created by hfi1_create_kctxts */
 	ret = set_up_interrupts(dd);
 	if (ret)
 		goto bail_cleanup;
diff --git a/drivers/infiniband/hw/hfi1/chip.h b/drivers/infiniband/hw/hfi1/chip.h
index cbe455d9ab8b..b8345a60a0fb 100644
--- a/drivers/infiniband/hw/hfi1/chip.h
+++ b/drivers/infiniband/hw/hfi1/chip.h
@@ -384,6 +384,7 @@
 #define VERIFY_CAP_LOCAL_FABRIC	     0x08
 #define VERIFY_CAP_LOCAL_LINK_WIDTH  0x09
 #define LOCAL_DEVICE_ID		     0x0a
+#define RESERVED_REGISTERS	     0x0b
 #define LOCAL_LNI_INFO		     0x0c
 #define REMOTE_LNI_INFO              0x0d
 #define MISC_STATUS		     0x0e
@@ -506,6 +507,9 @@
 #define DOWN_REMOTE_REASON_SHIFT 16
 #define DOWN_REMOTE_REASON_MASK  0xff
 
+#define HOST_INTERFACE_VERSION_SHIFT 16
+#define HOST_INTERFACE_VERSION_MASK  0xff
+
 /* verify capability PHY power management bits */
 #define PWRM_BER_CONTROL	0x1
 #define PWRM_BANDWIDTH_CONTROL	0x2
@@ -605,11 +609,11 @@ int read_lcb_csr(struct hfi1_devdata *dd, u32 offset, u64 *data);
 int write_lcb_csr(struct hfi1_devdata *dd, u32 offset, u64 data);
 
 void __iomem *get_csr_addr(
-	struct hfi1_devdata *dd,
+	const struct hfi1_devdata *dd,
 	u32 offset);
 
 static inline void __iomem *get_kctxt_csr_addr(
-	struct hfi1_devdata *dd,
+	const struct hfi1_devdata *dd,
 	int ctxt,
 	u32 offset0)
 {
@@ -644,7 +648,6 @@ u64 create_pbc(struct hfi1_pportdata *ppd, u64 flags, int srate_mbs, u32 vl,
 #define NUM_PCIE_SERDES 16	/* number of PCIe serdes on the SBus */
 extern const u8 pcie_serdes_broadcast[];
 extern const u8 pcie_pcs_addrs[2][NUM_PCIE_SERDES];
-extern uint platform_config_load;
 
 /* SBus commands */
 #define RESET_SBUS_RECEIVER 0x20
@@ -704,6 +707,7 @@ int read_8051_data(struct hfi1_devdata *dd, u32 addr, u32 len, u64 *result);
 /* chip.c */
 void read_misc_status(struct hfi1_devdata *dd, u8 *ver_major, u8 *ver_minor,
 		      u8 *ver_patch);
+int write_host_interface_version(struct hfi1_devdata *dd, u8 version);
 void read_guid(struct hfi1_devdata *dd);
 int wait_fm_ready(struct hfi1_devdata *dd, u32 mstimeout);
 void set_link_down_reason(struct hfi1_pportdata *ppd, u8 lcl_reason,
@@ -743,11 +747,10 @@ int is_ax(struct hfi1_devdata *dd);
 int is_bx(struct hfi1_devdata *dd);
 u32 read_physical_state(struct hfi1_devdata *dd);
 u32 chip_to_opa_pstate(struct hfi1_devdata *dd, u32 chip_pstate);
-u32 get_logical_state(struct hfi1_pportdata *ppd);
 const char *opa_lstate_name(u32 lstate);
 const char *opa_pstate_name(u32 pstate);
-u32 driver_physical_state(struct hfi1_pportdata *ppd);
-u32 driver_logical_state(struct hfi1_pportdata *ppd);
+u32 driver_pstate(struct hfi1_pportdata *ppd);
+u32 driver_lstate(struct hfi1_pportdata *ppd);
 
 int acquire_lcb_access(struct hfi1_devdata *dd, int sleep_ok);
 int release_lcb_access(struct hfi1_devdata *dd, int sleep_ok);
@@ -1347,21 +1350,21 @@ enum {
 u64 get_all_cpu_total(u64 __percpu *cntr);
 void hfi1_start_cleanup(struct hfi1_devdata *dd);
 void hfi1_clear_tids(struct hfi1_ctxtdata *rcd);
-struct ib_header *hfi1_get_msgheader(
-				struct hfi1_devdata *dd, __le32 *rhf_addr);
 void hfi1_init_ctxt(struct send_context *sc);
 void hfi1_put_tid(struct hfi1_devdata *dd, u32 index,
 		  u32 type, unsigned long pa, u16 order);
 void hfi1_quiet_serdes(struct hfi1_pportdata *ppd);
-void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op, int ctxt);
+void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op,
+		  struct hfi1_ctxtdata *rcd);
 u32 hfi1_read_cntrs(struct hfi1_devdata *dd, char **namep, u64 **cntrp);
 u32 hfi1_read_portcntrs(struct hfi1_pportdata *ppd, char **namep, u64 **cntrp);
-u8 hfi1_ibphys_portstate(struct hfi1_pportdata *ppd);
 int hfi1_get_ib_cfg(struct hfi1_pportdata *ppd, int which);
 int hfi1_set_ib_cfg(struct hfi1_pportdata *ppd, int which, u32 val);
-int hfi1_set_ctxt_jkey(struct hfi1_devdata *dd, unsigned ctxt, u16 jkey);
-int hfi1_clear_ctxt_jkey(struct hfi1_devdata *dd, unsigned ctxt);
-int hfi1_set_ctxt_pkey(struct hfi1_devdata *dd, unsigned ctxt, u16 pkey);
+int hfi1_set_ctxt_jkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd,
+		       u16 jkey);
+int hfi1_clear_ctxt_jkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *ctxt);
+int hfi1_set_ctxt_pkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *ctxt,
+		       u16 pkey);
 int hfi1_clear_ctxt_pkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *ctxt);
 void hfi1_read_link_quality(struct hfi1_devdata *dd, u8 *link_quality);
 void hfi1_init_vnic_rsm(struct hfi1_devdata *dd);
diff --git a/drivers/infiniband/hw/hfi1/common.h b/drivers/infiniband/hw/hfi1/common.h
index 995d62c7f9a7..3e27794ec750 100644
--- a/drivers/infiniband/hw/hfi1/common.h
+++ b/drivers/infiniband/hw/hfi1/common.h
@@ -325,22 +325,15 @@ struct diag_pkt {
 #define HFI1_LRH_BTH 0x0002      /* 1. word of IB LRH - next header: BTH */
 
 /* misc. */
+#define SC15_PACKET 0xF
 #define SIZE_OF_CRC 1
+#define SIZE_OF_LT 1
 
 #define LIM_MGMT_P_KEY       0x7FFF
 #define FULL_MGMT_P_KEY      0xFFFF
 
 #define DEFAULT_P_KEY LIM_MGMT_P_KEY
 
-/**
- * 0xF8 - 4 bits of multicast range and 1 bit for collective range
- * Example: For 24 bit LID space,
- * Multicast range: 0xF00000 to 0xF7FFFF
- * Collective range: 0xF80000 to 0xFFFFFE
- */
-#define HFI1_MCAST_NR 0x4 /* Number of top bits set */
-#define HFI1_COLLECTIVE_NR 0x1 /* Number of bits after MCAST_NR */
-
 #define HFI1_PSM_IOC_BASE_SEQ 0x0
 
 static inline __u64 rhf_to_cpu(const __le32 *rbuf)
diff --git a/drivers/infiniband/hw/hfi1/debugfs.c b/drivers/infiniband/hw/hfi1/debugfs.c
index e9fa3c293e42..36ae1fd86502 100644
--- a/drivers/infiniband/hw/hfi1/debugfs.c
+++ b/drivers/infiniband/hw/hfi1/debugfs.c
@@ -1,4 +1,3 @@
-#ifdef CONFIG_DEBUG_FS
 /*
  * Copyright(c) 2015-2017 Intel Corporation.
  *
@@ -173,12 +172,15 @@ static int _opcode_stats_seq_show(struct seq_file *s, void *v)
 	u64 n_packets = 0, n_bytes = 0;
 	struct hfi1_ibdev *ibd = (struct hfi1_ibdev *)s->private;
 	struct hfi1_devdata *dd = dd_from_dev(ibd);
+	struct hfi1_ctxtdata *rcd;
 
 	for (j = 0; j < dd->first_dyn_alloc_ctxt; j++) {
-		if (!dd->rcd[j])
-			continue;
-		n_packets += dd->rcd[j]->opstats->stats[i].n_packets;
-		n_bytes += dd->rcd[j]->opstats->stats[i].n_bytes;
+		rcd = hfi1_rcd_get_by_index(dd, j);
+		if (rcd) {
+			n_packets += rcd->opstats->stats[i].n_packets;
+			n_bytes += rcd->opstats->stats[i].n_bytes;
+		}
+		hfi1_rcd_put(rcd);
 	}
 	if (!n_packets && !n_bytes)
 		return SEQ_SKIP;
@@ -231,6 +233,7 @@ static int _ctx_stats_seq_show(struct seq_file *s, void *v)
 	u64 n_packets = 0;
 	struct hfi1_ibdev *ibd = (struct hfi1_ibdev *)s->private;
 	struct hfi1_devdata *dd = dd_from_dev(ibd);
+	struct hfi1_ctxtdata *rcd;
 
 	if (v == SEQ_START_TOKEN) {
 		seq_puts(s, "Ctx:npkts\n");
@@ -240,11 +243,14 @@ static int _ctx_stats_seq_show(struct seq_file *s, void *v)
 	spos = v;
 	i = *spos;
 
-	if (!dd->rcd[i])
+	rcd = hfi1_rcd_get_by_index(dd, i);
+	if (!rcd)
 		return SEQ_SKIP;
 
-	for (j = 0; j < ARRAY_SIZE(dd->rcd[i]->opstats->stats); j++)
-		n_packets += dd->rcd[i]->opstats->stats[j].n_packets;
+	for (j = 0; j < ARRAY_SIZE(rcd->opstats->stats); j++)
+		n_packets += rcd->opstats->stats[j].n_packets;
+
+	hfi1_rcd_put(rcd);
 
 	if (!n_packets)
 		return SEQ_SKIP;
@@ -260,10 +266,10 @@ DEBUGFS_FILE_OPS(ctx_stats);
 static void *_qp_stats_seq_start(struct seq_file *s, loff_t *pos)
 	__acquires(RCU)
 {
-	struct qp_iter *iter;
+	struct rvt_qp_iter *iter;
 	loff_t n = *pos;
 
-	iter = qp_iter_init(s->private);
+	iter = rvt_qp_iter_init(s->private, 0, NULL);
 
 	/* stop calls rcu_read_unlock */
 	rcu_read_lock();
@@ -272,7 +278,7 @@ static void *_qp_stats_seq_start(struct seq_file *s, loff_t *pos)
 		return NULL;
 
 	do {
-		if (qp_iter_next(iter)) {
+		if (rvt_qp_iter_next(iter)) {
 			kfree(iter);
 			return NULL;
 		}
@@ -285,11 +291,11 @@ static void *_qp_stats_seq_next(struct seq_file *s, void *iter_ptr,
 				loff_t *pos)
 	__must_hold(RCU)
 {
-	struct qp_iter *iter = iter_ptr;
+	struct rvt_qp_iter *iter = iter_ptr;
 
 	(*pos)++;
 
-	if (qp_iter_next(iter)) {
+	if (rvt_qp_iter_next(iter)) {
 		kfree(iter);
 		return NULL;
 	}
@@ -305,7 +311,7 @@ static void _qp_stats_seq_stop(struct seq_file *s, void *iter_ptr)
 
 static int _qp_stats_seq_show(struct seq_file *s, void *iter_ptr)
 {
-	struct qp_iter *iter = iter_ptr;
+	struct rvt_qp_iter *iter = iter_ptr;
 
 	if (!iter)
 		return 0;
@@ -361,6 +367,52 @@ DEBUGFS_SEQ_FILE_OPS(sdes);
 DEBUGFS_SEQ_FILE_OPEN(sdes)
 DEBUGFS_FILE_OPS(sdes);
 
+static void *_rcds_seq_start(struct seq_file *s, loff_t *pos)
+{
+	struct hfi1_ibdev *ibd;
+	struct hfi1_devdata *dd;
+
+	ibd = (struct hfi1_ibdev *)s->private;
+	dd = dd_from_dev(ibd);
+	if (!dd->rcd || *pos >= dd->n_krcv_queues)
+		return NULL;
+	return pos;
+}
+
+static void *_rcds_seq_next(struct seq_file *s, void *v, loff_t *pos)
+{
+	struct hfi1_ibdev *ibd = (struct hfi1_ibdev *)s->private;
+	struct hfi1_devdata *dd = dd_from_dev(ibd);
+
+	++*pos;
+	if (!dd->rcd || *pos >= dd->n_krcv_queues)
+		return NULL;
+	return pos;
+}
+
+static void _rcds_seq_stop(struct seq_file *s, void *v)
+{
+}
+
+static int _rcds_seq_show(struct seq_file *s, void *v)
+{
+	struct hfi1_ibdev *ibd = (struct hfi1_ibdev *)s->private;
+	struct hfi1_devdata *dd = dd_from_dev(ibd);
+	struct hfi1_ctxtdata *rcd;
+	loff_t *spos = v;
+	loff_t i = *spos;
+
+	rcd = hfi1_rcd_get_by_index(dd, i);
+	if (rcd)
+		seqfile_dump_rcd(s, rcd);
+	hfi1_rcd_put(rcd);
+	return 0;
+}
+
+DEBUGFS_SEQ_FILE_OPS(rcds);
+DEBUGFS_SEQ_FILE_OPEN(rcds)
+DEBUGFS_FILE_OPS(rcds);
+
 /* read the per-device counters */
 static ssize_t dev_counters_read(struct file *file, char __user *buf,
 				 size_t count, loff_t *ppos)
@@ -1098,12 +1150,15 @@ static int _fault_stats_seq_show(struct seq_file *s, void *v)
 	u64 n_packets = 0, n_bytes = 0;
 	struct hfi1_ibdev *ibd = (struct hfi1_ibdev *)s->private;
 	struct hfi1_devdata *dd = dd_from_dev(ibd);
+	struct hfi1_ctxtdata *rcd;
 
 	for (j = 0; j < dd->first_dyn_alloc_ctxt; j++) {
-		if (!dd->rcd[j])
-			continue;
-		n_packets += dd->rcd[j]->opstats->stats[i].n_packets;
-		n_bytes += dd->rcd[j]->opstats->stats[i].n_bytes;
+		rcd = hfi1_rcd_get_by_index(dd, j);
+		if (rcd) {
+			n_packets += rcd->opstats->stats[i].n_packets;
+			n_bytes += rcd->opstats->stats[i].n_bytes;
+		}
+		hfi1_rcd_put(rcd);
 	}
 	if (!n_packets && !n_bytes)
 		return SEQ_SKIP;
@@ -1311,6 +1366,7 @@ void hfi1_dbg_ibdev_init(struct hfi1_ibdev *ibd)
 	DEBUGFS_SEQ_FILE_CREATE(ctx_stats, ibd->hfi1_ibdev_dbg, ibd);
 	DEBUGFS_SEQ_FILE_CREATE(qp_stats, ibd->hfi1_ibdev_dbg, ibd);
 	DEBUGFS_SEQ_FILE_CREATE(sdes, ibd->hfi1_ibdev_dbg, ibd);
+	DEBUGFS_SEQ_FILE_CREATE(rcds, ibd->hfi1_ibdev_dbg, ibd);
 	DEBUGFS_SEQ_FILE_CREATE(sdma_cpu_list, ibd->hfi1_ibdev_dbg, ibd);
 	/* dev counter files */
 	for (i = 0; i < ARRAY_SIZE(cntr_ops); i++)
@@ -1478,5 +1534,3 @@ void hfi1_dbg_exit(void)
 	debugfs_remove_recursive(hfi1_dbg_root);
 	hfi1_dbg_root = NULL;
 }
-
-#endif
diff --git a/drivers/infiniband/hw/hfi1/driver.c b/drivers/infiniband/hw/hfi1/driver.c
index a50870e455a3..7372cc00cb2d 100644
--- a/drivers/infiniband/hw/hfi1/driver.c
+++ b/drivers/infiniband/hw/hfi1/driver.c
@@ -96,7 +96,6 @@ MODULE_PARM_DESC(cap_mask, "Bit mask of enabled/disabled HW features");
 
 MODULE_LICENSE("Dual BSD/GPL");
 MODULE_DESCRIPTION("Intel Omni-Path Architecture driver");
-MODULE_VERSION(HFI1_DRIVER_VERSION);
 
 /*
  * MAX_PKT_RCV is the max # if packets processed per receive interrupt.
@@ -196,7 +195,7 @@ int hfi1_count_active_units(void)
 
 	spin_lock_irqsave(&hfi1_devs_lock, flags);
 	list_for_each_entry(dd, &hfi1_dev_list, list) {
-		if (!(dd->flags & HFI1_PRESENT) || !dd->kregbase)
+		if (!(dd->flags & HFI1_PRESENT) || !dd->kregbase1)
 			continue;
 		for (pidx = 0; pidx < dd->num_pports; ++pidx) {
 			ppd = dd->pport + pidx;
@@ -224,6 +223,27 @@ static inline void *get_egrbuf(const struct hfi1_ctxtdata *rcd, u64 rhf,
 			(offset * RCV_BUF_BLOCK_SIZE));
 }
 
+static inline void *hfi1_get_header(struct hfi1_devdata *dd,
+				    __le32 *rhf_addr)
+{
+	u32 offset = rhf_hdrq_offset(rhf_to_cpu(rhf_addr));
+
+	return (void *)(rhf_addr - dd->rhf_offset + offset);
+}
+
+static inline struct ib_header *hfi1_get_msgheader(struct hfi1_devdata *dd,
+						   __le32 *rhf_addr)
+{
+	return (struct ib_header *)hfi1_get_header(dd, rhf_addr);
+}
+
+static inline struct hfi1_16b_header
+		*hfi1_get_16B_header(struct hfi1_devdata *dd,
+				     __le32 *rhf_addr)
+{
+	return (struct hfi1_16b_header *)hfi1_get_header(dd, rhf_addr);
+}
+
 /*
  * Validate and encode the a given RcvArray Buffer size.
  * The function will check whether the given size falls within
@@ -249,7 +269,7 @@ static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd,
 {
 	struct ib_header *rhdr = packet->hdr;
 	u32 rte = rhf_rcv_type_err(packet->rhf);
-	int lnh = ib_get_lnh(rhdr);
+	u32 mlid_base;
 	struct hfi1_ibport *ibp = rcd_to_iport(rcd);
 	struct hfi1_devdata *dd = ppd->dd;
 	struct rvt_dev_info *rdi = &dd->verbs_dev.rdi;
@@ -257,37 +277,47 @@ static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd,
 	if (packet->rhf & (RHF_VCRC_ERR | RHF_ICRC_ERR))
 		return;
 
+	if (packet->etype == RHF_RCV_TYPE_BYPASS) {
+		goto drop;
+	} else {
+		u8 lnh = ib_get_lnh(rhdr);
+
+		mlid_base = be16_to_cpu(IB_MULTICAST_LID_BASE);
+		if (lnh == HFI1_LRH_BTH) {
+			packet->ohdr = &rhdr->u.oth;
+		} else if (lnh == HFI1_LRH_GRH) {
+			packet->ohdr = &rhdr->u.l.oth;
+			packet->grh = &rhdr->u.l.grh;
+		} else {
+			goto drop;
+		}
+	}
+
 	if (packet->rhf & RHF_TID_ERR) {
 		/* For TIDERR and RC QPs preemptively schedule a NAK */
-		struct ib_other_headers *ohdr = NULL;
 		u32 tlen = rhf_pkt_len(packet->rhf); /* in bytes */
-		u16 lid  = ib_get_dlid(rhdr);
+		u32 dlid = ib_get_dlid(rhdr);
 		u32 qp_num;
-		u32 rcv_flags = 0;
 
 		/* Sanity check packet */
 		if (tlen < 24)
 			goto drop;
 
 		/* Check for GRH */
-		if (lnh == HFI1_LRH_BTH) {
-			ohdr = &rhdr->u.oth;
-		} else if (lnh == HFI1_LRH_GRH) {
+		if (packet->grh) {
 			u32 vtf;
+			struct ib_grh *grh = packet->grh;
 
-			ohdr = &rhdr->u.l.oth;
-			if (rhdr->u.l.grh.next_hdr != IB_GRH_NEXT_HDR)
+			if (grh->next_hdr != IB_GRH_NEXT_HDR)
 				goto drop;
-			vtf = be32_to_cpu(rhdr->u.l.grh.version_tclass_flow);
+			vtf = be32_to_cpu(grh->version_tclass_flow);
 			if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION)
 				goto drop;
-			rcv_flags |= HFI1_HAS_GRH;
-		} else {
-			goto drop;
 		}
+
 		/* Get the destination QP number. */
-		qp_num = be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK;
-		if (lid < be16_to_cpu(IB_MULTICAST_LID_BASE)) {
+		qp_num = ib_bth_get_qpn(packet->ohdr);
+		if (dlid < mlid_base) {
 			struct rvt_qp *qp;
 			unsigned long flags;
 
@@ -312,11 +342,7 @@ static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd,
 
 			switch (qp->ibqp.qp_type) {
 			case IB_QPT_RC:
-				hfi1_rc_hdrerr(
-					rcd,
-					rhdr,
-					rcv_flags,
-					qp);
+				hfi1_rc_hdrerr(rcd, packet, qp);
 				break;
 			default:
 				/* For now don't handle any other QP types */
@@ -332,9 +358,8 @@ static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd,
 	switch (rte) {
 	case RHF_RTE_ERROR_OP_CODE_ERR:
 	{
-		u32 opcode;
 		void *ebuf = NULL;
-		__be32 *bth = NULL;
+		u8 opcode;
 
 		if (rhf_use_egr_bfr(packet->rhf))
 			ebuf = packet->ebuf;
@@ -342,16 +367,7 @@ static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd,
 		if (!ebuf)
 			goto drop; /* this should never happen */
 
-		if (lnh == HFI1_LRH_BTH)
-			bth = (__be32 *)ebuf;
-		else if (lnh == HFI1_LRH_GRH)
-			bth = (__be32 *)((char *)ebuf + sizeof(struct ib_grh));
-		else
-			goto drop;
-
-		opcode = be32_to_cpu(bth[0]) >> 24;
-		opcode &= 0xff;
-
+		opcode = ib_bth_get_opcode(packet->ohdr);
 		if (opcode == IB_OPCODE_CNP) {
 			/*
 			 * Only in pre-B0 h/w is the CNP_OPCODE handled
@@ -365,7 +381,7 @@ static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd,
 			sc5 = hfi1_9B_get_sc5(rhdr, packet->rhf);
 			sl = ibp->sc_to_sl[sc5];
 
-			lqpn = be32_to_cpu(bth[1]) & RVT_QPN_MASK;
+			lqpn = ib_bth_get_qpn(packet->ohdr);
 			rcu_read_lock();
 			qp = rvt_lookup_qpn(rdi, &ibp->rvp, lqpn);
 			if (!qp) {
@@ -415,33 +431,39 @@ static inline void init_packet(struct hfi1_ctxtdata *rcd,
 	packet->rhf = rhf_to_cpu(packet->rhf_addr);
 	packet->rhqoff = rcd->head;
 	packet->numpkt = 0;
-	packet->rcv_flags = 0;
 }
 
 void hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt,
 			       bool do_cnp)
 {
 	struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
-	struct ib_header *hdr = pkt->hdr;
 	struct ib_other_headers *ohdr = pkt->ohdr;
-	struct ib_grh *grh = NULL;
+	struct ib_grh *grh = pkt->grh;
 	u32 rqpn = 0, bth1;
-	u16 rlid, dlid = ib_get_dlid(hdr);
-	u8 sc, svc_type;
+	u16 pkey, rlid, dlid = ib_get_dlid(pkt->hdr);
+	u8 hdr_type, sc, svc_type;
 	bool is_mcast = false;
 
-	if (pkt->rcv_flags & HFI1_HAS_GRH)
-		grh = &hdr->u.l.grh;
+	if (pkt->etype == RHF_RCV_TYPE_BYPASS) {
+		is_mcast = hfi1_is_16B_mcast(dlid);
+		pkey = hfi1_16B_get_pkey(pkt->hdr);
+		sc = hfi1_16B_get_sc(pkt->hdr);
+		hdr_type = HFI1_PKT_TYPE_16B;
+	} else {
+		is_mcast = (dlid > be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
+			   (dlid != be16_to_cpu(IB_LID_PERMISSIVE));
+		pkey = ib_bth_get_pkey(ohdr);
+		sc = hfi1_9B_get_sc5(pkt->hdr, pkt->rhf);
+		hdr_type = HFI1_PKT_TYPE_9B;
+	}
 
 	switch (qp->ibqp.qp_type) {
 	case IB_QPT_SMI:
 	case IB_QPT_GSI:
 	case IB_QPT_UD:
-		rlid = ib_get_slid(hdr);
-		rqpn = be32_to_cpu(ohdr->u.ud.deth[1]) & RVT_QPN_MASK;
+		rlid = ib_get_slid(pkt->hdr);
+		rqpn = ib_get_sqpn(pkt->ohdr);
 		svc_type = IB_CC_SVCTYPE_UD;
-		is_mcast = (dlid > be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
-			(dlid != be16_to_cpu(IB_LID_PERMISSIVE));
 		break;
 	case IB_QPT_UC:
 		rlid = rdma_ah_get_dlid(&qp->remote_ah_attr);
@@ -457,14 +479,11 @@ void hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt,
 		return;
 	}
 
-	sc = hfi1_9B_get_sc5(hdr, pkt->rhf);
-
 	bth1 = be32_to_cpu(ohdr->bth[1]);
-	if (do_cnp && (bth1 & IB_FECN_SMASK)) {
-		u16 pkey = (u16)be32_to_cpu(ohdr->bth[0]);
-
-		return_cnp(ibp, qp, rqpn, pkey, dlid, rlid, sc, grh);
-	}
+	/* Call appropriate CNP handler */
+	if (do_cnp && (bth1 & IB_FECN_SMASK))
+		hfi1_handle_cnp_tbl[hdr_type](ibp, qp, rqpn, pkey,
+					      dlid, rlid, sc, grh);
 
 	if (!is_mcast && (bth1 & IB_BECN_SMASK)) {
 		struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
@@ -591,9 +610,10 @@ static void __prescan_rxq(struct hfi1_packet *packet)
 
 		if (lnh == HFI1_LRH_BTH) {
 			packet->ohdr = &hdr->u.oth;
+			packet->grh = NULL;
 		} else if (lnh == HFI1_LRH_GRH) {
 			packet->ohdr = &hdr->u.l.oth;
-			packet->rcv_flags |= HFI1_HAS_GRH;
+			packet->grh = &hdr->u.l.grh;
 		} else {
 			goto next; /* just in case */
 		}
@@ -698,10 +718,8 @@ static inline int process_rcv_packet(struct hfi1_packet *packet, int thread)
 {
 	int ret;
 
-	packet->hdr = hfi1_get_msgheader(packet->rcd->dd,
-					 packet->rhf_addr);
-	packet->hlen = (u8 *)packet->rhf_addr - (u8 *)packet->hdr;
 	packet->etype = rhf_rcv_type(packet->rhf);
+
 	/* total length */
 	packet->tlen = rhf_pkt_len(packet->rhf); /* in bytes */
 	/* retrieve eager buffer details */
@@ -759,7 +777,7 @@ static inline void process_rcv_update(int last, struct hfi1_packet *packet)
 			       packet->etail, 0, 0);
 		packet->updegr = 0;
 	}
-	packet->rcv_flags = 0;
+	packet->grh = NULL;
 }
 
 static inline void finish_packet(struct hfi1_packet *packet)
@@ -837,9 +855,10 @@ bail:
 	return last;
 }
 
-static inline void set_nodma_rtail(struct hfi1_devdata *dd, u8 ctxt)
+static inline void set_nodma_rtail(struct hfi1_devdata *dd, u16 ctxt)
 {
-	int i;
+	struct hfi1_ctxtdata *rcd;
+	u16 i;
 
 	/*
 	 * For dynamically allocated kernel contexts (like vnic) switch
@@ -847,19 +866,28 @@ static inline void set_nodma_rtail(struct hfi1_devdata *dd, u8 ctxt)
 	 * interrupt handler for all statically allocated kernel contexts.
 	 */
 	if (ctxt >= dd->first_dyn_alloc_ctxt) {
-		dd->rcd[ctxt]->do_interrupt =
-			&handle_receive_interrupt_nodma_rtail;
+		rcd = hfi1_rcd_get_by_index(dd, ctxt);
+		if (rcd) {
+			rcd->do_interrupt =
+				&handle_receive_interrupt_nodma_rtail;
+			hfi1_rcd_put(rcd);
+		}
 		return;
 	}
 
-	for (i = HFI1_CTRL_CTXT + 1; i < dd->first_dyn_alloc_ctxt; i++)
-		dd->rcd[i]->do_interrupt =
-			&handle_receive_interrupt_nodma_rtail;
+	for (i = HFI1_CTRL_CTXT + 1; i < dd->first_dyn_alloc_ctxt; i++) {
+		rcd = hfi1_rcd_get_by_index(dd, i);
+		if (rcd)
+			rcd->do_interrupt =
+				&handle_receive_interrupt_nodma_rtail;
+		hfi1_rcd_put(rcd);
+	}
 }
 
-static inline void set_dma_rtail(struct hfi1_devdata *dd, u8 ctxt)
+static inline void set_dma_rtail(struct hfi1_devdata *dd, u16 ctxt)
 {
-	int i;
+	struct hfi1_ctxtdata *rcd;
+	u16 i;
 
 	/*
 	 * For dynamically allocated kernel contexts (like vnic) switch
@@ -867,27 +895,39 @@ static inline void set_dma_rtail(struct hfi1_devdata *dd, u8 ctxt)
 	 * interrupt handler for all statically allocated kernel contexts.
 	 */
 	if (ctxt >= dd->first_dyn_alloc_ctxt) {
-		dd->rcd[ctxt]->do_interrupt =
-			&handle_receive_interrupt_dma_rtail;
+		rcd = hfi1_rcd_get_by_index(dd, ctxt);
+		if (rcd) {
+			rcd->do_interrupt =
+				&handle_receive_interrupt_dma_rtail;
+			hfi1_rcd_put(rcd);
+		}
 		return;
 	}
 
-	for (i = HFI1_CTRL_CTXT + 1; i < dd->first_dyn_alloc_ctxt; i++)
-		dd->rcd[i]->do_interrupt =
-			&handle_receive_interrupt_dma_rtail;
+	for (i = HFI1_CTRL_CTXT + 1; i < dd->first_dyn_alloc_ctxt; i++) {
+		rcd = hfi1_rcd_get_by_index(dd, i);
+		if (rcd)
+			rcd->do_interrupt =
+				&handle_receive_interrupt_dma_rtail;
+		hfi1_rcd_put(rcd);
+	}
 }
 
 void set_all_slowpath(struct hfi1_devdata *dd)
 {
-	int i;
+	struct hfi1_ctxtdata *rcd;
+	u16 i;
 
 	/* HFI1_CTRL_CTXT must always use the slow path interrupt handler */
 	for (i = HFI1_CTRL_CTXT + 1; i < dd->num_rcv_contexts; i++) {
-		struct hfi1_ctxtdata *rcd = dd->rcd[i];
-
+		rcd = hfi1_rcd_get_by_index(dd, i);
+		if (!rcd)
+			continue;
 		if ((i < dd->first_dyn_alloc_ctxt) ||
-		    (rcd && rcd->sc && (rcd->sc->type == SC_KERNEL)))
+		    (rcd->sc && (rcd->sc->type == SC_KERNEL))) {
 			rcd->do_interrupt = &handle_receive_interrupt;
+		}
+		hfi1_rcd_put(rcd);
 	}
 }
 
@@ -896,20 +936,30 @@ static inline int set_armed_to_active(struct hfi1_ctxtdata *rcd,
 				      struct hfi1_devdata *dd)
 {
 	struct work_struct *lsaw = &rcd->ppd->linkstate_active_work;
-	struct ib_header *hdr = hfi1_get_msgheader(packet->rcd->dd,
-						   packet->rhf_addr);
 	u8 etype = rhf_rcv_type(packet->rhf);
+	u8 sc = SC15_PACKET;
+
+	if (etype == RHF_RCV_TYPE_IB) {
+		struct ib_header *hdr = hfi1_get_msgheader(packet->rcd->dd,
+							   packet->rhf_addr);
+		sc = hfi1_9B_get_sc5(hdr, packet->rhf);
+	} else if (etype == RHF_RCV_TYPE_BYPASS) {
+		struct hfi1_16b_header *hdr = hfi1_get_16B_header(
+						packet->rcd->dd,
+						packet->rhf_addr);
+		sc = hfi1_16B_get_sc(hdr);
+	}
+	if (sc != SC15_PACKET) {
+		int hwstate = driver_lstate(rcd->ppd);
 
-	if (etype == RHF_RCV_TYPE_IB &&
-	    hfi1_9B_get_sc5(hdr, packet->rhf) != 0xf) {
-		int hwstate = read_logical_state(dd);
-
-		if (hwstate != LSTATE_ACTIVE) {
-			dd_dev_info(dd, "Unexpected link state %d\n", hwstate);
+		if (hwstate != IB_PORT_ACTIVE) {
+			dd_dev_info(dd,
+				    "Unexpected link state %s\n",
+				    opa_lstate_name(hwstate));
 			return 0;
 		}
 
-		queue_work(rcd->ppd->hfi1_wq, lsaw);
+		queue_work(rcd->ppd->link_wq, lsaw);
 		return 1;
 	}
 	return 0;
@@ -1063,7 +1113,8 @@ void receive_interrupt_work(struct work_struct *work)
 	struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
 						  linkstate_active_work);
 	struct hfi1_devdata *dd = ppd->dd;
-	int i;
+	struct hfi1_ctxtdata *rcd;
+	u16 i;
 
 	/* Received non-SC15 packet implies neighbor_normal */
 	ppd->neighbor_normal = 1;
@@ -1073,8 +1124,12 @@ void receive_interrupt_work(struct work_struct *work)
 	 * Interrupt all statically allocated kernel contexts that could
 	 * have had an interrupt during auto activation.
 	 */
-	for (i = HFI1_CTRL_CTXT; i < dd->first_dyn_alloc_ctxt; i++)
-		force_recv_intr(dd->rcd[i]);
+	for (i = HFI1_CTRL_CTXT; i < dd->first_dyn_alloc_ctxt; i++) {
+		rcd = hfi1_rcd_get_by_index(dd, i);
+		if (rcd)
+			force_recv_intr(rcd);
+		hfi1_rcd_put(rcd);
+	}
 }
 
 /*
@@ -1264,10 +1319,9 @@ void hfi1_start_led_override(struct hfi1_pportdata *ppd, unsigned int timeon,
  */
 int hfi1_reset_device(int unit)
 {
-	int ret, i;
+	int ret;
 	struct hfi1_devdata *dd = hfi1_lookup(unit);
 	struct hfi1_pportdata *ppd;
-	unsigned long flags;
 	int pidx;
 
 	if (!dd) {
@@ -1277,7 +1331,7 @@ int hfi1_reset_device(int unit)
 
 	dd_dev_info(dd, "Reset on unit %u requested\n", unit);
 
-	if (!dd->kregbase || !(dd->flags & HFI1_PRESENT)) {
+	if (!dd->kregbase1 || !(dd->flags & HFI1_PRESENT)) {
 		dd_dev_info(dd,
 			    "Invalid unit number %u or not initialized or not present\n",
 			    unit);
@@ -1285,17 +1339,15 @@ int hfi1_reset_device(int unit)
 		goto bail;
 	}
 
-	spin_lock_irqsave(&dd->uctxt_lock, flags);
+	/* If there are any user/vnic contexts, we cannot reset */
+	mutex_lock(&hfi1_mutex);
 	if (dd->rcd)
-		for (i = dd->first_dyn_alloc_ctxt;
-		     i < dd->num_rcv_contexts; i++) {
-			if (!dd->rcd[i])
-				continue;
-			spin_unlock_irqrestore(&dd->uctxt_lock, flags);
+		if (hfi1_stats.sps_ctxts) {
+			mutex_unlock(&hfi1_mutex);
 			ret = -EBUSY;
 			goto bail;
 		}
-	spin_unlock_irqrestore(&dd->uctxt_lock, flags);
+	mutex_unlock(&hfi1_mutex);
 
 	for (pidx = 0; pidx < dd->num_pports; ++pidx) {
 		ppd = dd->pport + pidx;
@@ -1321,6 +1373,162 @@ bail:
 	return ret;
 }
 
+static inline void hfi1_setup_ib_header(struct hfi1_packet *packet)
+{
+	packet->hdr = (struct hfi1_ib_message_header *)
+			hfi1_get_msgheader(packet->rcd->dd,
+					   packet->rhf_addr);
+	packet->hlen = (u8 *)packet->rhf_addr - (u8 *)packet->hdr;
+}
+
+static int hfi1_bypass_ingress_pkt_check(struct hfi1_packet *packet)
+{
+	struct hfi1_pportdata *ppd = packet->rcd->ppd;
+
+	/* slid and dlid cannot be 0 */
+	if ((!packet->slid) || (!packet->dlid))
+		return -EINVAL;
+
+	/* Compare port lid with incoming packet dlid */
+	if ((!(hfi1_is_16B_mcast(packet->dlid))) &&
+	    (packet->dlid !=
+		opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE), 16B))) {
+		if (packet->dlid != ppd->lid)
+			return -EINVAL;
+	}
+
+	/* No multicast packets with SC15 */
+	if ((hfi1_is_16B_mcast(packet->dlid)) && (packet->sc == 0xF))
+		return -EINVAL;
+
+	/* Packets with permissive DLID always on SC15 */
+	if ((packet->dlid == opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE),
+					 16B)) &&
+	    (packet->sc != 0xF))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int hfi1_setup_9B_packet(struct hfi1_packet *packet)
+{
+	struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd);
+	struct ib_header *hdr;
+	u8 lnh;
+
+	hfi1_setup_ib_header(packet);
+	hdr = packet->hdr;
+
+	lnh = ib_get_lnh(hdr);
+	if (lnh == HFI1_LRH_BTH) {
+		packet->ohdr = &hdr->u.oth;
+		packet->grh = NULL;
+	} else if (lnh == HFI1_LRH_GRH) {
+		u32 vtf;
+
+		packet->ohdr = &hdr->u.l.oth;
+		packet->grh = &hdr->u.l.grh;
+		if (packet->grh->next_hdr != IB_GRH_NEXT_HDR)
+			goto drop;
+		vtf = be32_to_cpu(packet->grh->version_tclass_flow);
+		if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION)
+			goto drop;
+	} else {
+		goto drop;
+	}
+
+	/* Query commonly used fields from packet header */
+	packet->payload = packet->ebuf;
+	packet->opcode = ib_bth_get_opcode(packet->ohdr);
+	packet->slid = ib_get_slid(hdr);
+	packet->dlid = ib_get_dlid(hdr);
+	if (unlikely((packet->dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
+		     (packet->dlid != be16_to_cpu(IB_LID_PERMISSIVE))))
+		packet->dlid += opa_get_mcast_base(OPA_MCAST_NR) -
+				be16_to_cpu(IB_MULTICAST_LID_BASE);
+	packet->sl = ib_get_sl(hdr);
+	packet->sc = hfi1_9B_get_sc5(hdr, packet->rhf);
+	packet->pad = ib_bth_get_pad(packet->ohdr);
+	packet->extra_byte = 0;
+	packet->fecn = ib_bth_get_fecn(packet->ohdr);
+	packet->becn = ib_bth_get_becn(packet->ohdr);
+
+	return 0;
+drop:
+	ibp->rvp.n_pkt_drops++;
+	return -EINVAL;
+}
+
+static int hfi1_setup_bypass_packet(struct hfi1_packet *packet)
+{
+	/*
+	 * Bypass packets have a different header/payload split
+	 * compared to an IB packet.
+	 * Current split is set such that 16 bytes of the actual
+	 * header is in the header buffer and the remining is in
+	 * the eager buffer. We chose 16 since hfi1 driver only
+	 * supports 16B bypass packets and we will be able to
+	 * receive the entire LRH with such a split.
+	 */
+
+	struct hfi1_ctxtdata *rcd = packet->rcd;
+	struct hfi1_pportdata *ppd = rcd->ppd;
+	struct hfi1_ibport *ibp = &ppd->ibport_data;
+	u8 l4;
+	u8 grh_len;
+
+	packet->hdr = (struct hfi1_16b_header *)
+			hfi1_get_16B_header(packet->rcd->dd,
+					    packet->rhf_addr);
+	packet->hlen = (u8 *)packet->rhf_addr - (u8 *)packet->hdr;
+
+	l4 = hfi1_16B_get_l4(packet->hdr);
+	if (l4 == OPA_16B_L4_IB_LOCAL) {
+		grh_len = 0;
+		packet->ohdr = packet->ebuf;
+		packet->grh = NULL;
+	} else if (l4 == OPA_16B_L4_IB_GLOBAL) {
+		u32 vtf;
+
+		grh_len = sizeof(struct ib_grh);
+		packet->ohdr = packet->ebuf + grh_len;
+		packet->grh = packet->ebuf;
+		if (packet->grh->next_hdr != IB_GRH_NEXT_HDR)
+			goto drop;
+		vtf = be32_to_cpu(packet->grh->version_tclass_flow);
+		if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION)
+			goto drop;
+	} else {
+		goto drop;
+	}
+
+	/* Query commonly used fields from packet header */
+	packet->opcode = ib_bth_get_opcode(packet->ohdr);
+	packet->hlen = hdr_len_by_opcode[packet->opcode] + 8 + grh_len;
+	packet->payload = packet->ebuf + packet->hlen - (4 * sizeof(u32));
+	packet->slid = hfi1_16B_get_slid(packet->hdr);
+	packet->dlid = hfi1_16B_get_dlid(packet->hdr);
+	if (unlikely(hfi1_is_16B_mcast(packet->dlid)))
+		packet->dlid += opa_get_mcast_base(OPA_MCAST_NR) -
+				opa_get_lid(opa_get_mcast_base(OPA_MCAST_NR),
+					    16B);
+	packet->sc = hfi1_16B_get_sc(packet->hdr);
+	packet->sl = ibp->sc_to_sl[packet->sc];
+	packet->pad = hfi1_16B_bth_get_pad(packet->ohdr);
+	packet->extra_byte = SIZE_OF_LT;
+	packet->fecn = hfi1_16B_get_fecn(packet->hdr);
+	packet->becn = hfi1_16B_get_becn(packet->hdr);
+
+	if (hfi1_bypass_ingress_pkt_check(packet))
+		goto drop;
+
+	return 0;
+drop:
+	hfi1_cdbg(PKT, "%s: packet dropped\n", __func__);
+	ibp->rvp.n_pkt_drops++;
+	return -EINVAL;
+}
+
 void handle_eflags(struct hfi1_packet *packet)
 {
 	struct hfi1_ctxtdata *rcd = packet->rcd;
@@ -1351,6 +1559,9 @@ int process_receive_ib(struct hfi1_packet *packet)
 	if (unlikely(hfi1_dbg_fault_packet(packet)))
 		return RHF_RCV_CONTINUE;
 
+	if (hfi1_setup_9B_packet(packet))
+		return RHF_RCV_CONTINUE;
+
 	trace_hfi1_rcvhdr(packet->rcd->ppd->dd,
 			  packet->rcd->ctxt,
 			  rhf_err_flags(packet->rhf),
@@ -1380,8 +1591,8 @@ static inline bool hfi1_is_vnic_packet(struct hfi1_packet *packet)
 	if (packet->rcd->is_vnic)
 		return true;
 
-	if ((HFI1_GET_L2_TYPE(packet->ebuf) == OPA_VNIC_L2_TYPE) &&
-	    (HFI1_GET_L4_TYPE(packet->ebuf) == OPA_VNIC_L4_ETHR))
+	if ((hfi1_16B_get_l2(packet->ebuf) == OPA_16B_L2_TYPE) &&
+	    (hfi1_16B_get_l4(packet->ebuf) == OPA_16B_L4_ETHR))
 		return true;
 
 	return false;
@@ -1391,25 +1602,38 @@ int process_receive_bypass(struct hfi1_packet *packet)
 {
 	struct hfi1_devdata *dd = packet->rcd->dd;
 
-	if (unlikely(rhf_err_flags(packet->rhf))) {
-		handle_eflags(packet);
-	} else if (hfi1_is_vnic_packet(packet)) {
+	if (hfi1_is_vnic_packet(packet)) {
 		hfi1_vnic_bypass_rcv(packet);
 		return RHF_RCV_CONTINUE;
 	}
 
-	dd_dev_err(dd, "Unsupported bypass packet. Dropping\n");
-	incr_cntr64(&dd->sw_rcv_bypass_packet_errors);
-	if (!(dd->err_info_rcvport.status_and_code & OPA_EI_STATUS_SMASK)) {
-		u64 *flits = packet->ebuf;
+	if (hfi1_setup_bypass_packet(packet))
+		return RHF_RCV_CONTINUE;
+
+	if (unlikely(rhf_err_flags(packet->rhf))) {
+		handle_eflags(packet);
+		return RHF_RCV_CONTINUE;
+	}
 
-		if (flits && !(packet->rhf & RHF_LEN_ERR)) {
-			dd->err_info_rcvport.packet_flit1 = flits[0];
-			dd->err_info_rcvport.packet_flit2 =
-				packet->tlen > sizeof(flits[0]) ? flits[1] : 0;
+	if (hfi1_16B_get_l2(packet->hdr) == 0x2) {
+		hfi1_16B_rcv(packet);
+	} else {
+		dd_dev_err(dd,
+			   "Bypass packets other than 16B are not supported in normal operation. Dropping\n");
+		incr_cntr64(&dd->sw_rcv_bypass_packet_errors);
+		if (!(dd->err_info_rcvport.status_and_code &
+		      OPA_EI_STATUS_SMASK)) {
+			u64 *flits = packet->ebuf;
+
+			if (flits && !(packet->rhf & RHF_LEN_ERR)) {
+				dd->err_info_rcvport.packet_flit1 = flits[0];
+				dd->err_info_rcvport.packet_flit2 =
+					packet->tlen > sizeof(flits[0]) ?
+					flits[1] : 0;
+			}
+			dd->err_info_rcvport.status_and_code |=
+				(OPA_EI_STATUS_SMASK | BAD_L2_ERR);
 		}
-		dd->err_info_rcvport.status_and_code |=
-			(OPA_EI_STATUS_SMASK | BAD_L2_ERR);
 	}
 	return RHF_RCV_CONTINUE;
 }
@@ -1422,6 +1646,7 @@ int process_receive_error(struct hfi1_packet *packet)
 		 rhf_rcv_type_err(packet->rhf) == 3))
 		return RHF_RCV_CONTINUE;
 
+	hfi1_setup_ib_header(packet);
 	handle_eflags(packet);
 
 	if (unlikely(rhf_err_flags(packet->rhf)))
@@ -1435,6 +1660,8 @@ int kdeth_process_expected(struct hfi1_packet *packet)
 {
 	if (unlikely(hfi1_dbg_fault_packet(packet)))
 		return RHF_RCV_CONTINUE;
+
+	hfi1_setup_ib_header(packet);
 	if (unlikely(rhf_err_flags(packet->rhf)))
 		handle_eflags(packet);
 
@@ -1445,6 +1672,7 @@ int kdeth_process_expected(struct hfi1_packet *packet)
 
 int kdeth_process_eager(struct hfi1_packet *packet)
 {
+	hfi1_setup_ib_header(packet);
 	if (unlikely(rhf_err_flags(packet->rhf)))
 		handle_eflags(packet);
 	if (unlikely(hfi1_dbg_fault_packet(packet)))
@@ -1461,3 +1689,62 @@ int process_receive_invalid(struct hfi1_packet *packet)
 		   rhf_rcv_type(packet->rhf));
 	return RHF_RCV_CONTINUE;
 }
+
+void seqfile_dump_rcd(struct seq_file *s, struct hfi1_ctxtdata *rcd)
+{
+	struct hfi1_packet packet;
+	struct ps_mdata mdata;
+
+	seq_printf(s, "Rcd %u: RcvHdr cnt %u entsize %u %s head %llu tail %llu\n",
+		   rcd->ctxt, rcd->rcvhdrq_cnt, rcd->rcvhdrqentsize,
+		   HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) ?
+		   "dma_rtail" : "nodma_rtail",
+		   read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD) &
+		   RCV_HDR_HEAD_HEAD_MASK,
+		   read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL));
+
+	init_packet(rcd, &packet);
+	init_ps_mdata(&mdata, &packet);
+
+	while (1) {
+		struct hfi1_devdata *dd = rcd->dd;
+		__le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head +
+					 dd->rhf_offset;
+		struct ib_header *hdr;
+		u64 rhf = rhf_to_cpu(rhf_addr);
+		u32 etype = rhf_rcv_type(rhf), qpn;
+		u8 opcode;
+		u32 psn;
+		u8 lnh;
+
+		if (ps_done(&mdata, rhf, rcd))
+			break;
+
+		if (ps_skip(&mdata, rhf, rcd))
+			goto next;
+
+		if (etype > RHF_RCV_TYPE_IB)
+			goto next;
+
+		packet.hdr = hfi1_get_msgheader(dd, rhf_addr);
+		hdr = packet.hdr;
+
+		lnh = be16_to_cpu(hdr->lrh[0]) & 3;
+
+		if (lnh == HFI1_LRH_BTH)
+			packet.ohdr = &hdr->u.oth;
+		else if (lnh == HFI1_LRH_GRH)
+			packet.ohdr = &hdr->u.l.oth;
+		else
+			goto next; /* just in case */
+
+		opcode = (be32_to_cpu(packet.ohdr->bth[0]) >> 24);
+		qpn = be32_to_cpu(packet.ohdr->bth[1]) & RVT_QPN_MASK;
+		psn = mask_psn(be32_to_cpu(packet.ohdr->bth[2]));
+
+		seq_printf(s, "\tEnt %u: opcode 0x%x, qpn 0x%x, psn 0x%x\n",
+			   mdata.ps_head, opcode, qpn, psn);
+next:
+		update_ps_mdata(&mdata, rcd);
+	}
+}
diff --git a/drivers/infiniband/hw/hfi1/eprom.c b/drivers/infiniband/hw/hfi1/eprom.c
index 26da124c88e2..d46b17107901 100644
--- a/drivers/infiniband/hw/hfi1/eprom.c
+++ b/drivers/infiniband/hw/hfi1/eprom.c
@@ -250,7 +250,6 @@ static int read_partition_platform_config(struct hfi1_devdata *dd, void **data,
 {
 	void *buffer;
 	void *p;
-	u32 length;
 	int ret;
 
 	buffer = kmalloc(P1_SIZE, GFP_KERNEL);
@@ -265,13 +264,13 @@ static int read_partition_platform_config(struct hfi1_devdata *dd, void **data,
 
 	/* scan for image magic that may trail the actual data */
 	p = strnstr(buffer, IMAGE_TRAIL_MAGIC, P1_SIZE);
-	if (p)
-		length = p - buffer;
-	else
-		length = P1_SIZE;
+	if (!p) {
+		kfree(buffer);
+		return -ENOENT;
+	}
 
 	*data = buffer;
-	*size = length;
+	*size = p - buffer;
 	return 0;
 }
 
diff --git a/drivers/infiniband/hw/hfi1/exp_rcv.c b/drivers/infiniband/hw/hfi1/exp_rcv.c
new file mode 100644
index 000000000000..0af91675acc6
--- /dev/null
+++ b/drivers/infiniband/hw/hfi1/exp_rcv.c
@@ -0,0 +1,114 @@
+/*
+ * Copyright(c) 2017 Intel Corporation.
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * BSD LICENSE
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  - Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  - Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  - Neither the name of Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "exp_rcv.h"
+#include "trace.h"
+
+/**
+ * exp_tid_group_init - initialize exp_tid_set
+ * @set - the set
+ */
+void hfi1_exp_tid_group_init(struct exp_tid_set *set)
+{
+	INIT_LIST_HEAD(&set->list);
+	set->count = 0;
+}
+
+/**
+ * alloc_ctxt_rcv_groups - initialize expected receive groups
+ * @rcd - the context to add the groupings to
+ */
+int hfi1_alloc_ctxt_rcv_groups(struct hfi1_ctxtdata *rcd)
+{
+	struct hfi1_devdata *dd = rcd->dd;
+	u32 tidbase;
+	struct tid_group *grp;
+	int i;
+
+	tidbase = rcd->expected_base;
+	for (i = 0; i < rcd->expected_count /
+		     dd->rcv_entries.group_size; i++) {
+		grp = kzalloc(sizeof(*grp), GFP_KERNEL);
+		if (!grp)
+			goto bail;
+		grp->size = dd->rcv_entries.group_size;
+		grp->base = tidbase;
+		tid_group_add_tail(grp, &rcd->tid_group_list);
+		tidbase += dd->rcv_entries.group_size;
+	}
+
+	return 0;
+bail:
+	hfi1_free_ctxt_rcv_groups(rcd);
+	return -ENOMEM;
+}
+
+/**
+ * free_ctxt_rcv_groups - free  expected receive groups
+ * @rcd - the context to free
+ *
+ * The routine dismantles the expect receive linked
+ * list and clears any tids associated with the receive
+ * context.
+ *
+ * This should only be called for kernel contexts and the
+ * a base user context.
+ */
+void hfi1_free_ctxt_rcv_groups(struct hfi1_ctxtdata *rcd)
+{
+	struct tid_group *grp, *gptr;
+
+	WARN_ON(!EXP_TID_SET_EMPTY(rcd->tid_full_list));
+	WARN_ON(!EXP_TID_SET_EMPTY(rcd->tid_used_list));
+
+	list_for_each_entry_safe(grp, gptr, &rcd->tid_group_list.list, list) {
+		tid_group_remove(grp, &rcd->tid_group_list);
+		kfree(grp);
+	}
+
+	hfi1_clear_tids(rcd);
+}
diff --git a/drivers/infiniband/hw/hfi1/exp_rcv.h b/drivers/infiniband/hw/hfi1/exp_rcv.h
new file mode 100644
index 000000000000..08719047628a
--- /dev/null
+++ b/drivers/infiniband/hw/hfi1/exp_rcv.h
@@ -0,0 +1,190 @@
+#ifndef _HFI1_EXP_RCV_H
+#define _HFI1_EXP_RCV_H
+/*
+ * Copyright(c) 2017 Intel Corporation.
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * BSD LICENSE
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  - Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  - Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  - Neither the name of Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "hfi.h"
+
+#define EXP_TID_SET_EMPTY(set) (set.count == 0 && list_empty(&set.list))
+
+#define EXP_TID_TIDLEN_MASK   0x7FFULL
+#define EXP_TID_TIDLEN_SHIFT  0
+#define EXP_TID_TIDCTRL_MASK  0x3ULL
+#define EXP_TID_TIDCTRL_SHIFT 20
+#define EXP_TID_TIDIDX_MASK   0x3FFULL
+#define EXP_TID_TIDIDX_SHIFT  22
+#define EXP_TID_GET(tid, field)	\
+	(((tid) >> EXP_TID_TID##field##_SHIFT) & EXP_TID_TID##field##_MASK)
+
+#define EXP_TID_SET(field, value)			\
+	(((value) & EXP_TID_TID##field##_MASK) <<	\
+	 EXP_TID_TID##field##_SHIFT)
+#define EXP_TID_CLEAR(tid, field) ({					\
+		(tid) &= ~(EXP_TID_TID##field##_MASK <<			\
+			   EXP_TID_TID##field##_SHIFT);			\
+		})
+#define EXP_TID_RESET(tid, field, value) do {				\
+		EXP_TID_CLEAR(tid, field);				\
+		(tid) |= EXP_TID_SET(field, (value));			\
+	} while (0)
+
+/*
+ * Define fields in the KDETH header so we can update the header
+ * template.
+ */
+#define KDETH_OFFSET_SHIFT        0
+#define KDETH_OFFSET_MASK         0x7fff
+#define KDETH_OM_SHIFT            15
+#define KDETH_OM_MASK             0x1
+#define KDETH_TID_SHIFT           16
+#define KDETH_TID_MASK            0x3ff
+#define KDETH_TIDCTRL_SHIFT       26
+#define KDETH_TIDCTRL_MASK        0x3
+#define KDETH_INTR_SHIFT          28
+#define KDETH_INTR_MASK           0x1
+#define KDETH_SH_SHIFT            29
+#define KDETH_SH_MASK             0x1
+#define KDETH_KVER_SHIFT          30
+#define KDETH_KVER_MASK           0x3
+#define KDETH_JKEY_SHIFT          0x0
+#define KDETH_JKEY_MASK           0xff
+#define KDETH_HCRC_UPPER_SHIFT    16
+#define KDETH_HCRC_UPPER_MASK     0xff
+#define KDETH_HCRC_LOWER_SHIFT    24
+#define KDETH_HCRC_LOWER_MASK     0xff
+
+#define KDETH_GET(val, field)						\
+	(((le32_to_cpu((val))) >> KDETH_##field##_SHIFT) & KDETH_##field##_MASK)
+#define KDETH_SET(dw, field, val) do {					\
+		u32 dwval = le32_to_cpu(dw);				\
+		dwval &= ~(KDETH_##field##_MASK << KDETH_##field##_SHIFT); \
+		dwval |= (((val) & KDETH_##field##_MASK) << \
+			  KDETH_##field##_SHIFT);			\
+		dw = cpu_to_le32(dwval);				\
+	} while (0)
+
+#define KDETH_RESET(dw, field, val) ({ dw = 0; KDETH_SET(dw, field, val); })
+
+/* KDETH OM multipliers and switch over point */
+#define KDETH_OM_SMALL     4
+#define KDETH_OM_SMALL_SHIFT     2
+#define KDETH_OM_LARGE     64
+#define KDETH_OM_LARGE_SHIFT     6
+#define KDETH_OM_MAX_SIZE  (1 << ((KDETH_OM_LARGE / KDETH_OM_SMALL) + 1))
+
+struct tid_group {
+	struct list_head list;
+	u32 base;
+	u8 size;
+	u8 used;
+	u8 map;
+};
+
+/*
+ * Write an "empty" RcvArray entry.
+ * This function exists so the TID registaration code can use it
+ * to write to unused/unneeded entries and still take advantage
+ * of the WC performance improvements. The HFI will ignore this
+ * write to the RcvArray entry.
+ */
+static inline void rcv_array_wc_fill(struct hfi1_devdata *dd, u32 index)
+{
+	/*
+	 * Doing the WC fill writes only makes sense if the device is
+	 * present and the RcvArray has been mapped as WC memory.
+	 */
+	if ((dd->flags & HFI1_PRESENT) && dd->rcvarray_wc) {
+		writeq(0, dd->rcvarray_wc + (index * 8));
+		if ((index & 3) == 3)
+			flush_wc();
+	}
+}
+
+static inline void tid_group_add_tail(struct tid_group *grp,
+				      struct exp_tid_set *set)
+{
+	list_add_tail(&grp->list, &set->list);
+	set->count++;
+}
+
+static inline void tid_group_remove(struct tid_group *grp,
+				    struct exp_tid_set *set)
+{
+	list_del_init(&grp->list);
+	set->count--;
+}
+
+static inline void tid_group_move(struct tid_group *group,
+				  struct exp_tid_set *s1,
+				  struct exp_tid_set *s2)
+{
+	tid_group_remove(group, s1);
+	tid_group_add_tail(group, s2);
+}
+
+static inline struct tid_group *tid_group_pop(struct exp_tid_set *set)
+{
+	struct tid_group *grp =
+		list_first_entry(&set->list, struct tid_group, list);
+	list_del_init(&grp->list);
+	set->count--;
+	return grp;
+}
+
+static inline u32 rcventry2tidinfo(u32 rcventry)
+{
+	u32 pair = rcventry & ~0x1;
+
+	return EXP_TID_SET(IDX, pair >> 1) |
+		EXP_TID_SET(CTRL, 1 << (rcventry - pair));
+}
+
+int hfi1_alloc_ctxt_rcv_groups(struct hfi1_ctxtdata *rcd);
+void hfi1_free_ctxt_rcv_groups(struct hfi1_ctxtdata *rcd);
+void hfi1_exp_tid_group_init(struct exp_tid_set *set);
+
+#endif /* _HFI1_EXP_RCV_H */
diff --git a/drivers/infiniband/hw/hfi1/file_ops.c b/drivers/infiniband/hw/hfi1/file_ops.c
index 3158128d57e8..2bc89260235a 100644
--- a/drivers/infiniband/hw/hfi1/file_ops.c
+++ b/drivers/infiniband/hw/hfi1/file_ops.c
@@ -58,10 +58,10 @@
 #include "device.h"
 #include "common.h"
 #include "trace.h"
+#include "mmu_rb.h"
 #include "user_sdma.h"
 #include "user_exp_rcv.h"
 #include "aspm.h"
-#include "mmu_rb.h"
 
 #undef pr_fmt
 #define pr_fmt(fmt) DRIVER_NAME ": " fmt
@@ -79,21 +79,25 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma);
 
 static u64 kvirt_to_phys(void *addr);
 static int assign_ctxt(struct hfi1_filedata *fd, struct hfi1_user_info *uinfo);
-static int init_subctxts(struct hfi1_ctxtdata *uctxt,
-			 const struct hfi1_user_info *uinfo);
-static int init_user_ctxt(struct hfi1_filedata *fd);
+static void init_subctxts(struct hfi1_ctxtdata *uctxt,
+			  const struct hfi1_user_info *uinfo);
+static int init_user_ctxt(struct hfi1_filedata *fd,
+			  struct hfi1_ctxtdata *uctxt);
 static void user_init(struct hfi1_ctxtdata *uctxt);
 static int get_ctxt_info(struct hfi1_filedata *fd, void __user *ubase,
 			 __u32 len);
 static int get_base_info(struct hfi1_filedata *fd, void __user *ubase,
 			 __u32 len);
-static int setup_base_ctxt(struct hfi1_filedata *fd);
+static int setup_base_ctxt(struct hfi1_filedata *fd,
+			   struct hfi1_ctxtdata *uctxt);
 static int setup_subctxt(struct hfi1_ctxtdata *uctxt);
 
 static int find_sub_ctxt(struct hfi1_filedata *fd,
 			 const struct hfi1_user_info *uinfo);
 static int allocate_ctxt(struct hfi1_filedata *fd, struct hfi1_devdata *dd,
-			 struct hfi1_user_info *uinfo);
+			 struct hfi1_user_info *uinfo,
+			 struct hfi1_ctxtdata **cd);
+static void deallocate_ctxt(struct hfi1_ctxtdata *uctxt);
 static unsigned int poll_urgent(struct file *fp, struct poll_table_struct *pt);
 static unsigned int poll_next(struct file *fp, struct poll_table_struct *pt);
 static int user_event_ack(struct hfi1_ctxtdata *uctxt, u16 subctxt,
@@ -116,7 +120,7 @@ static const struct file_operations hfi1_file_ops = {
 	.llseek = noop_llseek,
 };
 
-static struct vm_operations_struct vm_ops = {
+static const struct vm_operations_struct vm_ops = {
 	.fault = vma_fault,
 };
 
@@ -181,7 +185,7 @@ static int hfi1_file_open(struct inode *inode, struct file *fp)
 					       struct hfi1_devdata,
 					       user_cdev);
 
-	if (!((dd->flags & HFI1_PRESENT) && dd->kregbase))
+	if (!((dd->flags & HFI1_PRESENT) && dd->kregbase1))
 		return -EINVAL;
 
 	if (!atomic_inc_not_zero(&dd->user_refcount))
@@ -267,12 +271,14 @@ static long hfi1_file_ioctl(struct file *fp, unsigned int cmd,
 			/*
 			 * Copy the number of tidlist entries we used
 			 * and the length of the buffer we registered.
-			 * These fields are adjacent in the structure so
-			 * we can copy them at the same time.
 			 */
 			addr = arg + offsetof(struct hfi1_tid_info, tidcnt);
 			if (copy_to_user((void __user *)addr, &tinfo.tidcnt,
-					 sizeof(tinfo.tidcnt) +
+					 sizeof(tinfo.tidcnt)))
+				return -EFAULT;
+
+			addr = arg + offsetof(struct hfi1_tid_info, length);
+			if (copy_to_user((void __user *)addr, &tinfo.length,
 					 sizeof(tinfo.length)))
 				ret = -EFAULT;
 		}
@@ -388,8 +394,7 @@ static long hfi1_file_ioctl(struct file *fp, unsigned int cmd,
 
 			sc_disable(sc);
 			ret = sc_enable(sc);
-			hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_ENB,
-				     uctxt->ctxt);
+			hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_ENB, uctxt);
 		} else {
 			ret = sc_restart(sc);
 		}
@@ -425,8 +430,7 @@ static ssize_t hfi1_write_iter(struct kiocb *kiocb, struct iov_iter *from)
 	if (!iter_is_iovec(from) || !dim)
 		return -EINVAL;
 
-	hfi1_cdbg(SDMA, "SDMA request from %u:%u (%lu)",
-		  fd->uctxt->ctxt, fd->subctxt, dim);
+	trace_hfi1_sdma_request(fd->dd, fd->uctxt->ctxt, fd->subctxt, dim);
 
 	if (atomic_read(&pq->n_reqs) == pq->n_max_reqs)
 		return -ENOSPC;
@@ -752,12 +756,11 @@ static int hfi1_file_close(struct inode *inode, struct file *fp)
 	if (!uctxt)
 		goto done;
 
-	hfi1_cdbg(PROC, "freeing ctxt %u:%u", uctxt->ctxt, fdata->subctxt);
-	mutex_lock(&hfi1_mutex);
+	hfi1_cdbg(PROC, "closing ctxt %u:%u", uctxt->ctxt, fdata->subctxt);
 
 	flush_wc();
 	/* drain user sdma queue */
-	hfi1_user_sdma_free_queues(fdata);
+	hfi1_user_sdma_free_queues(fdata, uctxt);
 
 	/* release the cpu */
 	hfi1_put_proc_affinity(fdata->rec_cpu_num);
@@ -766,6 +769,13 @@ static int hfi1_file_close(struct inode *inode, struct file *fp)
 	hfi1_user_exp_rcv_free(fdata);
 
 	/*
+	 * fdata->uctxt is used in the above cleanup.  It is not ready to be
+	 * removed until here.
+	 */
+	fdata->uctxt = NULL;
+	hfi1_rcd_put(uctxt);
+
+	/*
 	 * Clear any left over, unhandled events so the next process that
 	 * gets this context doesn't get confused.
 	 */
@@ -773,13 +783,14 @@ static int hfi1_file_close(struct inode *inode, struct file *fp)
 			   HFI1_MAX_SHARED_CTXTS) + fdata->subctxt;
 	*ev = 0;
 
+	spin_lock_irqsave(&dd->uctxt_lock, flags);
 	__clear_bit(fdata->subctxt, uctxt->in_use_ctxts);
 	if (!bitmap_empty(uctxt->in_use_ctxts, HFI1_MAX_SHARED_CTXTS)) {
-		mutex_unlock(&hfi1_mutex);
+		spin_unlock_irqrestore(&dd->uctxt_lock, flags);
 		goto done;
 	}
+	spin_unlock_irqrestore(&dd->uctxt_lock, flags);
 
-	spin_lock_irqsave(&dd->uctxt_lock, flags);
 	/*
 	 * Disable receive context and interrupt available, reset all
 	 * RcvCtxtCtrl bits to default values.
@@ -790,34 +801,24 @@ static int hfi1_file_close(struct inode *inode, struct file *fp)
 		     HFI1_RCVCTRL_TAILUPD_DIS |
 		     HFI1_RCVCTRL_ONE_PKT_EGR_DIS |
 		     HFI1_RCVCTRL_NO_RHQ_DROP_DIS |
-		     HFI1_RCVCTRL_NO_EGR_DROP_DIS, uctxt->ctxt);
+		     HFI1_RCVCTRL_NO_EGR_DROP_DIS, uctxt);
 	/* Clear the context's J_KEY */
-	hfi1_clear_ctxt_jkey(dd, uctxt->ctxt);
+	hfi1_clear_ctxt_jkey(dd, uctxt);
 	/*
-	 * Reset context integrity checks to default.
-	 * (writes to CSRs probably belong in chip.c)
+	 * If a send context is allocated, reset context integrity
+	 * checks to default and disable the send context.
 	 */
-	write_kctxt_csr(dd, uctxt->sc->hw_context, SEND_CTXT_CHECK_ENABLE,
-			hfi1_pkt_default_send_ctxt_mask(dd, uctxt->sc->type));
-	sc_disable(uctxt->sc);
-	spin_unlock_irqrestore(&dd->uctxt_lock, flags);
-
-	dd->rcd[uctxt->ctxt] = NULL;
+	if (uctxt->sc) {
+		set_pio_integrity(uctxt->sc);
+		sc_disable(uctxt->sc);
+	}
 
-	hfi1_user_exp_rcv_grp_free(uctxt);
+	hfi1_free_ctxt_rcv_groups(uctxt);
 	hfi1_clear_ctxt_pkey(dd, uctxt);
 
-	uctxt->rcvwait_to = 0;
-	uctxt->piowait_to = 0;
-	uctxt->rcvnowait = 0;
-	uctxt->pionowait = 0;
 	uctxt->event_flags = 0;
 
-	hfi1_stats.sps_ctxts--;
-	if (++dd->freectxts == dd->num_user_contexts)
-		aspm_enable_all(dd);
-	mutex_unlock(&hfi1_mutex);
-	hfi1_free_ctxtdata(dd, uctxt);
+	deallocate_ctxt(uctxt);
 done:
 	mmdrop(fdata->mm);
 	kobject_put(&dd->kobj);
@@ -845,135 +846,211 @@ static u64 kvirt_to_phys(void *addr)
 	return paddr;
 }
 
+/**
+ * complete_subctxt
+ * @fd: valid filedata pointer
+ *
+ * Sub-context info can only be set up after the base context
+ * has been completed.  This is indicated by the clearing of the
+ * HFI1_CTXT_BASE_UINIT bit.
+ *
+ * Wait for the bit to be cleared, and then complete the subcontext
+ * initialization.
+ *
+ */
+static int complete_subctxt(struct hfi1_filedata *fd)
+{
+	int ret;
+	unsigned long flags;
+
+	/*
+	 * sub-context info can only be set up after the base context
+	 * has been completed.
+	 */
+	ret = wait_event_interruptible(
+		fd->uctxt->wait,
+		!test_bit(HFI1_CTXT_BASE_UNINIT, &fd->uctxt->event_flags));
+
+	if (test_bit(HFI1_CTXT_BASE_FAILED, &fd->uctxt->event_flags))
+		ret = -ENOMEM;
+
+	/* Finish the sub-context init */
+	if (!ret) {
+		fd->rec_cpu_num = hfi1_get_proc_affinity(fd->uctxt->numa_id);
+		ret = init_user_ctxt(fd, fd->uctxt);
+	}
+
+	if (ret) {
+		hfi1_rcd_put(fd->uctxt);
+		fd->uctxt = NULL;
+		spin_lock_irqsave(&fd->dd->uctxt_lock, flags);
+		__clear_bit(fd->subctxt, fd->uctxt->in_use_ctxts);
+		spin_unlock_irqrestore(&fd->dd->uctxt_lock, flags);
+	}
+
+	return ret;
+}
+
 static int assign_ctxt(struct hfi1_filedata *fd, struct hfi1_user_info *uinfo)
 {
 	int ret;
 	unsigned int swmajor, swminor;
+	struct hfi1_ctxtdata *uctxt = NULL;
 
 	swmajor = uinfo->userversion >> 16;
 	if (swmajor != HFI1_USER_SWMAJOR)
 		return -ENODEV;
 
+	if (uinfo->subctxt_cnt > HFI1_MAX_SHARED_CTXTS)
+		return -EINVAL;
+
 	swminor = uinfo->userversion & 0xffff;
 
+	/*
+	 * Acquire the mutex to protect against multiple creations of what
+	 * could be a shared base context.
+	 */
 	mutex_lock(&hfi1_mutex);
 	/*
-	 * Get a sub context if necessary.
+	 * Get a sub context if available  (fd->uctxt will be set).
 	 * ret < 0 error, 0 no context, 1 sub-context found
 	 */
-	ret = 0;
-	if (uinfo->subctxt_cnt) {
-		ret = find_sub_ctxt(fd, uinfo);
-		if (ret > 0)
-			fd->rec_cpu_num =
-				hfi1_get_proc_affinity(fd->uctxt->numa_id);
-	}
+	ret = find_sub_ctxt(fd, uinfo);
 
 	/*
-	 * Allocate a base context if context sharing is not required or we
-	 * couldn't find a sub context.
+	 * Allocate a base context if context sharing is not required or a
+	 * sub context wasn't found.
 	 */
 	if (!ret)
-		ret = allocate_ctxt(fd, fd->dd, uinfo);
+		ret = allocate_ctxt(fd, fd->dd, uinfo, &uctxt);
 
 	mutex_unlock(&hfi1_mutex);
 
-	/* Depending on the context type, do the appropriate init */
-	if (ret > 0) {
-		/*
-		 * sub-context info can only be set up after the base
-		 * context has been completed.
-		 */
-		ret = wait_event_interruptible(fd->uctxt->wait, !test_bit(
-					       HFI1_CTXT_BASE_UNINIT,
-					       &fd->uctxt->event_flags));
-		if (test_bit(HFI1_CTXT_BASE_FAILED, &fd->uctxt->event_flags)) {
-			clear_bit(fd->subctxt, fd->uctxt->in_use_ctxts);
-			return -ENOMEM;
-		}
-		/* The only thing a sub context needs is the user_xxx stuff */
-		if (!ret)
-			ret = init_user_ctxt(fd);
-
-		if (ret)
-			clear_bit(fd->subctxt, fd->uctxt->in_use_ctxts);
-	} else if (!ret) {
-		ret = setup_base_ctxt(fd);
-		if (fd->uctxt->subctxt_cnt) {
-			/* If there is an error, set the failed bit. */
-			if (ret)
-				set_bit(HFI1_CTXT_BASE_FAILED,
-					&fd->uctxt->event_flags);
+	/* Depending on the context type, finish the appropriate init */
+	switch (ret) {
+	case 0:
+		ret = setup_base_ctxt(fd, uctxt);
+		if (uctxt->subctxt_cnt) {
 			/*
-			 * Base context is done, notify anybody using a
-			 * sub-context that is waiting for this completion
+			 * Base context is done (successfully or not), notify
+			 * anybody using a sub-context that is waiting for
+			 * this completion.
 			 */
-			clear_bit(HFI1_CTXT_BASE_UNINIT,
-				  &fd->uctxt->event_flags);
-			wake_up(&fd->uctxt->wait);
+			clear_bit(HFI1_CTXT_BASE_UNINIT, &uctxt->event_flags);
+			wake_up(&uctxt->wait);
 		}
+		break;
+	case 1:
+		ret = complete_subctxt(fd);
+		break;
+	default:
+		break;
 	}
 
 	return ret;
 }
 
-/*
- * The hfi1_mutex must be held when this function is called.  It is
- * necessary to ensure serialized access to the bitmask in_use_ctxts.
+/**
+ * match_ctxt
+ * @fd: valid filedata pointer
+ * @uinfo: user info to compare base context with
+ * @uctxt: context to compare uinfo to.
+ *
+ * Compare the given context with the given information to see if it
+ * can be used for a sub context.
  */
-static int find_sub_ctxt(struct hfi1_filedata *fd,
-			 const struct hfi1_user_info *uinfo)
+static int match_ctxt(struct hfi1_filedata *fd,
+		      const struct hfi1_user_info *uinfo,
+		      struct hfi1_ctxtdata *uctxt)
 {
-	int i;
 	struct hfi1_devdata *dd = fd->dd;
+	unsigned long flags;
 	u16 subctxt;
 
-	for (i = dd->first_dyn_alloc_ctxt; i < dd->num_rcv_contexts; i++) {
-		struct hfi1_ctxtdata *uctxt = dd->rcd[i];
+	/* Skip dynamically allocated kernel contexts */
+	if (uctxt->sc && (uctxt->sc->type == SC_KERNEL))
+		return 0;
 
-		/* Skip ctxts which are not yet open */
-		if (!uctxt ||
-		    bitmap_empty(uctxt->in_use_ctxts,
-				 HFI1_MAX_SHARED_CTXTS))
-			continue;
+	/* Skip ctxt if it doesn't match the requested one */
+	if (memcmp(uctxt->uuid, uinfo->uuid, sizeof(uctxt->uuid)) ||
+	    uctxt->jkey != generate_jkey(current_uid()) ||
+	    uctxt->subctxt_id != uinfo->subctxt_id ||
+	    uctxt->subctxt_cnt != uinfo->subctxt_cnt)
+		return 0;
 
-		/* Skip dynamically allocted kernel contexts */
-		if (uctxt->sc && (uctxt->sc->type == SC_KERNEL))
-			continue;
+	/* Verify the sharing process matches the base */
+	if (uctxt->userversion != uinfo->userversion)
+		return -EINVAL;
 
-		/* Skip ctxt if it doesn't match the requested one */
-		if (memcmp(uctxt->uuid, uinfo->uuid,
-			   sizeof(uctxt->uuid)) ||
-		    uctxt->jkey != generate_jkey(current_uid()) ||
-		    uctxt->subctxt_id != uinfo->subctxt_id ||
-		    uctxt->subctxt_cnt != uinfo->subctxt_cnt)
-			continue;
+	/* Find an unused sub context */
+	spin_lock_irqsave(&dd->uctxt_lock, flags);
+	if (bitmap_empty(uctxt->in_use_ctxts, HFI1_MAX_SHARED_CTXTS)) {
+		/* context is being closed, do not use */
+		spin_unlock_irqrestore(&dd->uctxt_lock, flags);
+		return 0;
+	}
 
-		/* Verify the sharing process matches the master */
-		if (uctxt->userversion != uinfo->userversion)
-			return -EINVAL;
+	subctxt = find_first_zero_bit(uctxt->in_use_ctxts,
+				      HFI1_MAX_SHARED_CTXTS);
+	if (subctxt >= uctxt->subctxt_cnt) {
+		spin_unlock_irqrestore(&dd->uctxt_lock, flags);
+		return -EBUSY;
+	}
 
-		/* Find an unused context */
-		subctxt = find_first_zero_bit(uctxt->in_use_ctxts,
-					      HFI1_MAX_SHARED_CTXTS);
-		if (subctxt >= uctxt->subctxt_cnt)
-			return -EBUSY;
+	fd->subctxt = subctxt;
+	__set_bit(fd->subctxt, uctxt->in_use_ctxts);
+	spin_unlock_irqrestore(&dd->uctxt_lock, flags);
 
-		fd->uctxt = uctxt;
-		fd->subctxt = subctxt;
-		__set_bit(fd->subctxt, uctxt->in_use_ctxts);
+	fd->uctxt = uctxt;
+	hfi1_rcd_get(uctxt);
+
+	return 1;
+}
 
-		return 1;
+/**
+ * find_sub_ctxt
+ * @fd: valid filedata pointer
+ * @uinfo: matching info to use to find a possible context to share.
+ *
+ * The hfi1_mutex must be held when this function is called.  It is
+ * necessary to ensure serialized creation of shared contexts.
+ *
+ * Return:
+ *    0      No sub-context found
+ *    1      Subcontext found and allocated
+ *    errno  EINVAL (incorrect parameters)
+ *           EBUSY (all sub contexts in use)
+ */
+static int find_sub_ctxt(struct hfi1_filedata *fd,
+			 const struct hfi1_user_info *uinfo)
+{
+	struct hfi1_ctxtdata *uctxt;
+	struct hfi1_devdata *dd = fd->dd;
+	u16 i;
+	int ret;
+
+	if (!uinfo->subctxt_cnt)
+		return 0;
+
+	for (i = dd->first_dyn_alloc_ctxt; i < dd->num_rcv_contexts; i++) {
+		uctxt = hfi1_rcd_get_by_index(dd, i);
+		if (uctxt) {
+			ret = match_ctxt(fd, uinfo, uctxt);
+			hfi1_rcd_put(uctxt);
+			/* value of != 0 will return */
+			if (ret)
+				return ret;
+		}
 	}
 
 	return 0;
 }
 
 static int allocate_ctxt(struct hfi1_filedata *fd, struct hfi1_devdata *dd,
-			 struct hfi1_user_info *uinfo)
+			 struct hfi1_user_info *uinfo,
+			 struct hfi1_ctxtdata **rcd)
 {
 	struct hfi1_ctxtdata *uctxt;
-	unsigned int ctxt;
 	int ret, numa;
 
 	if (dd->flags & HFI1_FROZEN) {
@@ -987,22 +1064,9 @@ static int allocate_ctxt(struct hfi1_filedata *fd, struct hfi1_devdata *dd,
 		return -EIO;
 	}
 
-	/*
-	 * This check is sort of redundant to the next EBUSY error. It would
-	 * also indicate an inconsistancy in the driver if this value was
-	 * zero, but there were still contexts available.
-	 */
 	if (!dd->freectxts)
 		return -EBUSY;
 
-	for (ctxt = dd->first_dyn_alloc_ctxt;
-	     ctxt < dd->num_rcv_contexts; ctxt++)
-		if (!dd->rcd[ctxt])
-			break;
-
-	if (ctxt == dd->num_rcv_contexts)
-		return -EBUSY;
-
 	/*
 	 * If we don't have a NUMA node requested, preference is towards
 	 * device NUMA node.
@@ -1012,11 +1076,10 @@ static int allocate_ctxt(struct hfi1_filedata *fd, struct hfi1_devdata *dd,
 		numa = cpu_to_node(fd->rec_cpu_num);
 	else
 		numa = numa_node_id();
-	uctxt = hfi1_create_ctxtdata(dd->pport, ctxt, numa);
-	if (!uctxt) {
-		dd_dev_err(dd,
-			   "Unable to allocate ctxtdata memory, failing open\n");
-		return -ENOMEM;
+	ret = hfi1_create_ctxtdata(dd->pport, numa, &uctxt);
+	if (ret < 0) {
+		dd_dev_err(dd, "user ctxtdata allocation failed\n");
+		return ret;
 	}
 	hfi1_cdbg(PROC, "[%u:%u] pid %u assigned to CPU %d (NUMA %u)",
 		  uctxt->ctxt, fd->subctxt, current->pid, fd->rec_cpu_num,
@@ -1025,8 +1088,7 @@ static int allocate_ctxt(struct hfi1_filedata *fd, struct hfi1_devdata *dd,
 	/*
 	 * Allocate and enable a PIO send context.
 	 */
-	uctxt->sc = sc_alloc(dd, SC_USER, uctxt->rcvhdrqentsize,
-			     uctxt->dd->node);
+	uctxt->sc = sc_alloc(dd, SC_USER, uctxt->rcvhdrqentsize, dd->node);
 	if (!uctxt->sc) {
 		ret = -ENOMEM;
 		goto ctxdata_free;
@@ -1038,28 +1100,19 @@ static int allocate_ctxt(struct hfi1_filedata *fd, struct hfi1_devdata *dd,
 		goto ctxdata_free;
 
 	/*
-	 * Setup sub context resources if the user-level has requested
+	 * Setup sub context information if the user-level has requested
 	 * sub contexts.
 	 * This has to be done here so the rest of the sub-contexts find the
-	 * proper master.
+	 * proper base context.
 	 */
-	if (uinfo->subctxt_cnt) {
-		ret = init_subctxts(uctxt, uinfo);
-		/*
-		 * On error, we don't need to disable and de-allocate the
-		 * send context because it will be done during file close
-		 */
-		if (ret)
-			goto ctxdata_free;
-	}
+	if (uinfo->subctxt_cnt)
+		init_subctxts(uctxt, uinfo);
 	uctxt->userversion = uinfo->userversion;
 	uctxt->flags = hfi1_cap_mask; /* save current flag state */
 	init_waitqueue_head(&uctxt->wait);
 	strlcpy(uctxt->comm, current->comm, sizeof(uctxt->comm));
 	memcpy(uctxt->uuid, uinfo->uuid, sizeof(uctxt->uuid));
 	uctxt->jkey = generate_jkey(current_uid());
-	INIT_LIST_HEAD(&uctxt->sdma_queues);
-	spin_lock_init(&uctxt->sdma_qlock);
 	hfi1_stats.sps_ctxts++;
 	/*
 	 * Disable ASPM when there are open user/PSM contexts to avoid
@@ -1067,31 +1120,33 @@ static int allocate_ctxt(struct hfi1_filedata *fd, struct hfi1_devdata *dd,
 	 */
 	if (dd->freectxts-- == dd->num_user_contexts)
 		aspm_disable_all(dd);
-	fd->uctxt = uctxt;
+
+	*rcd = uctxt;
 
 	return 0;
 
 ctxdata_free:
-	dd->rcd[ctxt] = NULL;
-	hfi1_free_ctxtdata(dd, uctxt);
+	hfi1_free_ctxt(uctxt);
 	return ret;
 }
 
-static int init_subctxts(struct hfi1_ctxtdata *uctxt,
-			 const struct hfi1_user_info *uinfo)
+static void deallocate_ctxt(struct hfi1_ctxtdata *uctxt)
 {
-	u16 num_subctxts;
+	mutex_lock(&hfi1_mutex);
+	hfi1_stats.sps_ctxts--;
+	if (++uctxt->dd->freectxts == uctxt->dd->num_user_contexts)
+		aspm_enable_all(uctxt->dd);
+	mutex_unlock(&hfi1_mutex);
 
-	num_subctxts = uinfo->subctxt_cnt;
-	if (num_subctxts > HFI1_MAX_SHARED_CTXTS)
-		return -EINVAL;
+	hfi1_free_ctxt(uctxt);
+}
 
+static void init_subctxts(struct hfi1_ctxtdata *uctxt,
+			  const struct hfi1_user_info *uinfo)
+{
 	uctxt->subctxt_cnt = uinfo->subctxt_cnt;
 	uctxt->subctxt_id = uinfo->subctxt_id;
-	uctxt->redirect_seq_cnt = 1;
 	set_bit(HFI1_CTXT_BASE_UNINIT, &uctxt->event_flags);
-
-	return 0;
 }
 
 static int setup_subctxt(struct hfi1_ctxtdata *uctxt)
@@ -1153,7 +1208,7 @@ static void user_init(struct hfi1_ctxtdata *uctxt)
 		clear_rcvhdrtail(uctxt);
 
 	/* Setup J_KEY before enabling the context */
-	hfi1_set_ctxt_jkey(uctxt->dd, uctxt->ctxt, uctxt->jkey);
+	hfi1_set_ctxt_jkey(uctxt->dd, uctxt, uctxt->jkey);
 
 	rcvctrl_ops = HFI1_RCVCTRL_CTXT_ENB;
 	if (HFI1_CAP_UGET_MASK(uctxt->flags, HDRSUPP))
@@ -1179,7 +1234,7 @@ static void user_init(struct hfi1_ctxtdata *uctxt)
 		rcvctrl_ops |= HFI1_RCVCTRL_TAILUPD_ENB;
 	else
 		rcvctrl_ops |= HFI1_RCVCTRL_TAILUPD_DIS;
-	hfi1_rcvctrl(uctxt->dd, rcvctrl_ops, uctxt->ctxt);
+	hfi1_rcvctrl(uctxt->dd, rcvctrl_ops, uctxt);
 }
 
 static int get_ctxt_info(struct hfi1_filedata *fd, void __user *ubase,
@@ -1223,23 +1278,25 @@ static int get_ctxt_info(struct hfi1_filedata *fd, void __user *ubase,
 	return ret;
 }
 
-static int init_user_ctxt(struct hfi1_filedata *fd)
+static int init_user_ctxt(struct hfi1_filedata *fd,
+			  struct hfi1_ctxtdata *uctxt)
 {
-	struct hfi1_ctxtdata *uctxt = fd->uctxt;
 	int ret;
 
 	ret = hfi1_user_sdma_alloc_queues(uctxt, fd);
 	if (ret)
 		return ret;
 
-	ret = hfi1_user_exp_rcv_init(fd);
+	ret = hfi1_user_exp_rcv_init(fd, uctxt);
+	if (ret)
+		hfi1_user_sdma_free_queues(fd, uctxt);
 
 	return ret;
 }
 
-static int setup_base_ctxt(struct hfi1_filedata *fd)
+static int setup_base_ctxt(struct hfi1_filedata *fd,
+			   struct hfi1_ctxtdata *uctxt)
 {
-	struct hfi1_ctxtdata *uctxt = fd->uctxt;
 	struct hfi1_devdata *dd = uctxt->dd;
 	int ret = 0;
 
@@ -1260,20 +1317,27 @@ static int setup_base_ctxt(struct hfi1_filedata *fd)
 	if (ret)
 		goto setup_failed;
 
-	ret = hfi1_user_exp_rcv_grp_init(fd);
+	ret = hfi1_alloc_ctxt_rcv_groups(uctxt);
 	if (ret)
 		goto setup_failed;
 
-	ret = init_user_ctxt(fd);
+	ret = init_user_ctxt(fd, uctxt);
 	if (ret)
 		goto setup_failed;
 
 	user_init(uctxt);
 
+	/* Now that the context is set up, the fd can get a reference. */
+	fd->uctxt = uctxt;
+	hfi1_rcd_get(uctxt);
+
 	return 0;
 
 setup_failed:
-	hfi1_free_ctxtdata(dd, uctxt);
+	/* Set the failed bit so sub-context init can do the right thing */
+	set_bit(HFI1_CTXT_BASE_FAILED, &uctxt->event_flags);
+	deallocate_ctxt(uctxt);
+
 	return ret;
 }
 
@@ -1390,7 +1454,7 @@ static unsigned int poll_next(struct file *fp,
 	spin_lock_irq(&dd->uctxt_lock);
 	if (hdrqempty(uctxt)) {
 		set_bit(HFI1_CTXT_WAITING_RCV, &uctxt->event_flags);
-		hfi1_rcvctrl(dd, HFI1_RCVCTRL_INTRAVAIL_ENB, uctxt->ctxt);
+		hfi1_rcvctrl(dd, HFI1_RCVCTRL_INTRAVAIL_ENB, uctxt);
 		pollflag = 0;
 	} else {
 		pollflag = POLLIN | POLLRDNORM;
@@ -1409,19 +1473,14 @@ int hfi1_set_uevent_bits(struct hfi1_pportdata *ppd, const int evtbit)
 {
 	struct hfi1_ctxtdata *uctxt;
 	struct hfi1_devdata *dd = ppd->dd;
-	unsigned ctxt;
-	int ret = 0;
-	unsigned long flags;
+	u16 ctxt;
 
-	if (!dd->events) {
-		ret = -EINVAL;
-		goto done;
-	}
+	if (!dd->events)
+		return -EINVAL;
 
-	spin_lock_irqsave(&dd->uctxt_lock, flags);
 	for (ctxt = dd->first_dyn_alloc_ctxt; ctxt < dd->num_rcv_contexts;
 	     ctxt++) {
-		uctxt = dd->rcd[ctxt];
+		uctxt = hfi1_rcd_get_by_index(dd, ctxt);
 		if (uctxt) {
 			unsigned long *evs = dd->events +
 				(uctxt->ctxt - dd->first_dyn_alloc_ctxt) *
@@ -1434,11 +1493,11 @@ int hfi1_set_uevent_bits(struct hfi1_pportdata *ppd, const int evtbit)
 			set_bit(evtbit, evs);
 			for (i = 1; i < uctxt->subctxt_cnt; i++)
 				set_bit(evtbit, evs + i);
+			hfi1_rcd_put(uctxt);
 		}
 	}
-	spin_unlock_irqrestore(&dd->uctxt_lock, flags);
-done:
-	return ret;
+
+	return 0;
 }
 
 /**
@@ -1475,7 +1534,7 @@ static int manage_rcvq(struct hfi1_ctxtdata *uctxt, u16 subctxt,
 	} else {
 		rcvctrl_op = HFI1_RCVCTRL_CTXT_DIS;
 	}
-	hfi1_rcvctrl(dd, rcvctrl_op, uctxt->ctxt);
+	hfi1_rcvctrl(dd, rcvctrl_op, uctxt);
 	/* always; new head should be equal to new tail; see above */
 bail:
 	return 0;
@@ -1525,7 +1584,7 @@ static int set_ctxt_pkey(struct hfi1_ctxtdata *uctxt, u16 subctxt, u16 pkey)
 		}
 
 	if (intable)
-		ret = hfi1_set_ctxt_pkey(dd, uctxt->ctxt, pkey);
+		ret = hfi1_set_ctxt_pkey(dd, uctxt, pkey);
 done:
 	return ret;
 }
diff --git a/drivers/infiniband/hw/hfi1/firmware.c b/drivers/infiniband/hw/hfi1/firmware.c
index 4042c11b2742..5aea8f47e670 100644
--- a/drivers/infiniband/hw/hfi1/firmware.c
+++ b/drivers/infiniband/hw/hfi1/firmware.c
@@ -64,30 +64,22 @@
 #define DEFAULT_FW_FABRIC_NAME "hfi1_fabric.fw"
 #define DEFAULT_FW_SBUS_NAME "hfi1_sbus.fw"
 #define DEFAULT_FW_PCIE_NAME "hfi1_pcie.fw"
-#define DEFAULT_PLATFORM_CONFIG_NAME "hfi1_platform.dat"
 #define ALT_FW_8051_NAME_ASIC "hfi1_dc8051_d.fw"
 #define ALT_FW_FABRIC_NAME "hfi1_fabric_d.fw"
 #define ALT_FW_SBUS_NAME "hfi1_sbus_d.fw"
 #define ALT_FW_PCIE_NAME "hfi1_pcie_d.fw"
+#define HOST_INTERFACE_VERSION 1
 
 static uint fw_8051_load = 1;
 static uint fw_fabric_serdes_load = 1;
 static uint fw_pcie_serdes_load = 1;
 static uint fw_sbus_load = 1;
 
-/*
- * Access required in platform.c
- * Maintains state of whether the platform config was fetched via the
- * fallback option
- */
-uint platform_config_load;
-
 /* Firmware file names get set in hfi1_firmware_init() based on the above */
 static char *fw_8051_name;
 static char *fw_fabric_serdes_name;
 static char *fw_sbus_name;
 static char *fw_pcie_serdes_name;
-static char *platform_config_name;
 
 #define SBUS_MAX_POLL_COUNT 100
 #define SBUS_COUNTER(reg, name) \
@@ -177,7 +169,6 @@ static struct firmware_details fw_8051;
 static struct firmware_details fw_fabric;
 static struct firmware_details fw_pcie;
 static struct firmware_details fw_sbus;
-static const struct firmware *platform_config;
 
 /* flags for turn_off_spicos() */
 #define SPICO_SBUS   0x1
@@ -615,6 +606,14 @@ retry:
 		fw_fabric_serdes_name = ALT_FW_FABRIC_NAME;
 		fw_sbus_name = ALT_FW_SBUS_NAME;
 		fw_pcie_serdes_name = ALT_FW_PCIE_NAME;
+
+		/*
+		 * Add a delay before obtaining and loading debug firmware.
+		 * Authorization will fail if the delay between firmware
+		 * authorization events is shorter than 50us. Add 100us to
+		 * make a delay time safe.
+		 */
+		usleep_range(100, 120);
 	}
 
 	if (fw_sbus_load) {
@@ -675,7 +674,6 @@ done:
 static int obtain_firmware(struct hfi1_devdata *dd)
 {
 	unsigned long timeout;
-	int err = 0;
 
 	mutex_lock(&fw_mutex);
 
@@ -699,38 +697,11 @@ static int obtain_firmware(struct hfi1_devdata *dd)
 	}
 	/* not in FW_TRY state */
 
-	if (fw_state == FW_FINAL) {
-		if (platform_config) {
-			dd->platform_config.data = platform_config->data;
-			dd->platform_config.size = platform_config->size;
-		}
-		goto done;	/* already acquired */
-	} else if (fw_state == FW_ERR) {
-		goto done;	/* already tried and failed */
-	}
-	/* fw_state is FW_EMPTY */
-
 	/* set fw_state to FW_TRY, FW_FINAL, or FW_ERR, and fw_err */
-	__obtain_firmware(dd);
-
-	if (platform_config_load) {
-		platform_config = NULL;
-		err = request_firmware(&platform_config, platform_config_name,
-				       &dd->pcidev->dev);
-		if (err) {
-			platform_config = NULL;
-			dd_dev_err(dd,
-				   "%s: No default platform config file found\n",
-				   __func__);
-			goto done;
-		}
-		dd->platform_config.data = platform_config->data;
-		dd->platform_config.size = platform_config->size;
-	}
+	if (fw_state == FW_EMPTY)
+		__obtain_firmware(dd);
 
-done:
 	mutex_unlock(&fw_mutex);
-
 	return fw_err;
 }
 
@@ -752,9 +723,6 @@ void dispose_firmware(void)
 	dispose_one_firmware(&fw_pcie);
 	dispose_one_firmware(&fw_sbus);
 
-	release_firmware(platform_config);
-	platform_config = NULL;
-
 	/* retain the error state, otherwise revert to empty */
 	if (fw_state != FW_ERR)
 		fw_state = FW_EMPTY;
@@ -1079,6 +1047,13 @@ static int load_8051_firmware(struct hfi1_devdata *dd,
 	dd_dev_info(dd, "8051 firmware version %d.%d.%d\n",
 		    (int)ver_major, (int)ver_minor, (int)ver_patch);
 	dd->dc8051_ver = dc8051_ver(ver_major, ver_minor, ver_patch);
+	ret = write_host_interface_version(dd, HOST_INTERFACE_VERSION);
+	if (ret != HCMD_SUCCESS) {
+		dd_dev_err(dd,
+			   "Failed to set host interface version, return 0x%x\n",
+			   ret);
+		return -EIO;
+	}
 
 	return 0;
 }
@@ -1709,10 +1684,8 @@ int hfi1_firmware_init(struct hfi1_devdata *dd)
 	}
 
 	/* no 8051 or QSFP on simulator */
-	if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR) {
+	if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
 		fw_8051_load = 0;
-		platform_config_load = 0;
-	}
 
 	if (!fw_8051_name) {
 		if (dd->icode == ICODE_RTL_SILICON)
@@ -1726,8 +1699,6 @@ int hfi1_firmware_init(struct hfi1_devdata *dd)
 		fw_sbus_name = DEFAULT_FW_SBUS_NAME;
 	if (!fw_pcie_serdes_name)
 		fw_pcie_serdes_name = DEFAULT_FW_PCIE_NAME;
-	if (!platform_config_name)
-		platform_config_name = DEFAULT_PLATFORM_CONFIG_NAME;
 
 	return obtain_firmware(dd);
 }
@@ -1773,6 +1744,7 @@ static int check_meta_version(struct hfi1_devdata *dd, u32 *system_table)
 int parse_platform_config(struct hfi1_devdata *dd)
 {
 	struct platform_config_cache *pcfgcache = &dd->pcfg_cache;
+	struct hfi1_pportdata *ppd = dd->pport;
 	u32 *ptr = NULL;
 	u32 header1 = 0, header2 = 0, magic_num = 0, crc = 0, file_length = 0;
 	u32 record_idx = 0, table_type = 0, table_length_dwords = 0;
@@ -1784,7 +1756,7 @@ int parse_platform_config(struct hfi1_devdata *dd)
 	 * scratch register bitmap, thus there is no platform config to parse.
 	 * Skip parsing in these situations.
 	 */
-	if (is_integrated(dd) && !platform_config_load)
+	if (ppd->config_from_scratch)
 		return 0;
 
 	if (!dd->platform_config.data) {
@@ -2073,13 +2045,14 @@ int get_platform_config_field(struct hfi1_devdata *dd,
 	int ret = 0, wlen = 0, seek = 0;
 	u32 field_len_bits = 0, field_start_bits = 0, *src_ptr = NULL;
 	struct platform_config_cache *pcfgcache = &dd->pcfg_cache;
+	struct hfi1_pportdata *ppd = dd->pport;
 
 	if (data)
 		memset(data, 0, len);
 	else
 		return -EINVAL;
 
-	if (is_integrated(dd) && !platform_config_load) {
+	if (ppd->config_from_scratch) {
 		/*
 		 * Use saved configuration from ppd for integrated platforms
 		 */
diff --git a/drivers/infiniband/hw/hfi1/hfi.h b/drivers/infiniband/hw/hfi1/hfi.h
index 414a04a481c2..3ac9c307a285 100644
--- a/drivers/infiniband/hw/hfi1/hfi.h
+++ b/drivers/infiniband/hw/hfi1/hfi.h
@@ -66,9 +66,11 @@
 #include <linux/i2c.h>
 #include <linux/i2c-algo-bit.h>
 #include <rdma/ib_hdrs.h>
+#include <rdma/opa_addr.h>
 #include <linux/rhashtable.h>
 #include <linux/netdevice.h>
 #include <rdma/rdma_vt.h>
+#include <rdma/opa_addr.h>
 
 #include "chip_registers.h"
 #include "common.h"
@@ -213,13 +215,11 @@ struct hfi1_ctxtdata {
 
 	/* dynamic receive available interrupt timeout */
 	u32 rcvavail_timeout;
-	/*
-	 * number of opens (including slave sub-contexts) on this instance
-	 * (ignoring forks, dup, etc. for now)
-	 */
-	int cnt;
+	/* Reference count the base context usage */
+	struct kref kref;
+
 	/* Device context index */
-	unsigned ctxt;
+	u16 ctxt;
 	/*
 	 * non-zero if ctxt can be shared, and defines the maximum number of
 	 * sub-contexts for this device context.
@@ -245,24 +245,10 @@ struct hfi1_ctxtdata {
 
 	/* lock protecting all Expected TID data */
 	struct mutex exp_lock;
-	/* number of pio bufs for this ctxt (all procs, if shared) */
-	u32 piocnt;
-	/* first pio buffer for this ctxt */
-	u32 pio_base;
-	/* chip offset of PIO buffers for this ctxt */
-	u32 piobufs;
 	/* per-context configuration flags */
 	unsigned long flags;
 	/* per-context event flags for fileops/intr communication */
 	unsigned long event_flags;
-	/* WAIT_RCV that timed out, no interrupt */
-	u32 rcvwait_to;
-	/* WAIT_PIO that timed out, no interrupt */
-	u32 piowait_to;
-	/* WAIT_RCV already happened, no wait */
-	u32 rcvnowait;
-	/* WAIT_PIO already happened, no wait */
-	u32 pionowait;
 	/* total number of polled urgent packets */
 	u32 urgent;
 	/* saved total number of polled urgent packets for poll edge trigger */
@@ -289,10 +275,8 @@ struct hfi1_ctxtdata {
 	u16 poll_type;
 	/* receive packet sequence counter */
 	u8 seq_cnt;
-	u8 redirect_seq_cnt;
 	/* ctxt rcvhdrq head offset */
 	u32 head;
-	u32 pkt_count;
 	/* QPs waiting for context processing */
 	struct list_head qp_wait_list;
 	/* interrupt handling */
@@ -301,15 +285,6 @@ struct hfi1_ctxtdata {
 	unsigned numa_id; /* numa node of this context */
 	/* verbs stats per CTX */
 	struct hfi1_opcode_stats_perctx *opstats;
-	/*
-	 * This is the kernel thread that will keep making
-	 * progress on the user sdma requests behind the scenes.
-	 * There is one per context (shared contexts use the master's).
-	 */
-	struct task_struct *progress;
-	struct list_head sdma_queues;
-	/* protect sdma queues */
-	spinlock_t sdma_qlock;
 
 	/* Is ASPM interrupt supported for this context */
 	bool aspm_intr_supported;
@@ -352,23 +327,150 @@ struct hfi1_ctxtdata {
 struct hfi1_packet {
 	void *ebuf;
 	void *hdr;
+	void *payload;
 	struct hfi1_ctxtdata *rcd;
 	__le32 *rhf_addr;
 	struct rvt_qp *qp;
 	struct ib_other_headers *ohdr;
+	struct ib_grh *grh;
 	u64 rhf;
 	u32 maxcnt;
 	u32 rhqoff;
+	u32 dlid;
+	u32 slid;
 	u16 tlen;
 	s16 etail;
 	u8 hlen;
 	u8 numpkt;
 	u8 rsize;
 	u8 updegr;
-	u8 rcv_flags;
 	u8 etype;
+	u8 extra_byte;
+	u8 pad;
+	u8 sc;
+	u8 sl;
+	u8 opcode;
+	bool becn;
+	bool fecn;
 };
 
+/* Packet types */
+#define HFI1_PKT_TYPE_9B  0
+#define HFI1_PKT_TYPE_16B 1
+
+/*
+ * OPA 16B Header
+ */
+#define OPA_16B_L4_MASK		0xFFull
+#define OPA_16B_SC_MASK		0x1F00000ull
+#define OPA_16B_SC_SHIFT	20
+#define OPA_16B_LID_MASK	0xFFFFFull
+#define OPA_16B_DLID_MASK	0xF000ull
+#define OPA_16B_DLID_SHIFT	20
+#define OPA_16B_DLID_HIGH_SHIFT	12
+#define OPA_16B_SLID_MASK	0xF00ull
+#define OPA_16B_SLID_SHIFT	20
+#define OPA_16B_SLID_HIGH_SHIFT	8
+#define OPA_16B_BECN_MASK       0x80000000ull
+#define OPA_16B_BECN_SHIFT      31
+#define OPA_16B_FECN_MASK       0x10000000ull
+#define OPA_16B_FECN_SHIFT      28
+#define OPA_16B_L2_MASK		0x60000000ull
+#define OPA_16B_L2_SHIFT	29
+#define OPA_16B_PKEY_MASK	0xFFFF0000ull
+#define OPA_16B_PKEY_SHIFT	16
+#define OPA_16B_LEN_MASK	0x7FF00000ull
+#define OPA_16B_LEN_SHIFT	20
+#define OPA_16B_RC_MASK		0xE000000ull
+#define OPA_16B_RC_SHIFT	25
+#define OPA_16B_AGE_MASK	0xFF0000ull
+#define OPA_16B_AGE_SHIFT	16
+#define OPA_16B_ENTROPY_MASK	0xFFFFull
+
+/*
+ * OPA 16B L2/L4 Encodings
+ */
+#define OPA_16B_L2_TYPE		0x02
+#define OPA_16B_L4_IB_LOCAL	0x09
+#define OPA_16B_L4_IB_GLOBAL	0x0A
+#define OPA_16B_L4_ETHR		OPA_VNIC_L4_ETHR
+
+static inline u8 hfi1_16B_get_l4(struct hfi1_16b_header *hdr)
+{
+	return (u8)(hdr->lrh[2] & OPA_16B_L4_MASK);
+}
+
+static inline u8 hfi1_16B_get_sc(struct hfi1_16b_header *hdr)
+{
+	return (u8)((hdr->lrh[1] & OPA_16B_SC_MASK) >> OPA_16B_SC_SHIFT);
+}
+
+static inline u32 hfi1_16B_get_dlid(struct hfi1_16b_header *hdr)
+{
+	return (u32)((hdr->lrh[1] & OPA_16B_LID_MASK) |
+		     (((hdr->lrh[2] & OPA_16B_DLID_MASK) >>
+		     OPA_16B_DLID_HIGH_SHIFT) << OPA_16B_DLID_SHIFT));
+}
+
+static inline u32 hfi1_16B_get_slid(struct hfi1_16b_header *hdr)
+{
+	return (u32)((hdr->lrh[0] & OPA_16B_LID_MASK) |
+		     (((hdr->lrh[2] & OPA_16B_SLID_MASK) >>
+		     OPA_16B_SLID_HIGH_SHIFT) << OPA_16B_SLID_SHIFT));
+}
+
+static inline u8 hfi1_16B_get_becn(struct hfi1_16b_header *hdr)
+{
+	return (u8)((hdr->lrh[0] & OPA_16B_BECN_MASK) >> OPA_16B_BECN_SHIFT);
+}
+
+static inline u8 hfi1_16B_get_fecn(struct hfi1_16b_header *hdr)
+{
+	return (u8)((hdr->lrh[1] & OPA_16B_FECN_MASK) >> OPA_16B_FECN_SHIFT);
+}
+
+static inline u8 hfi1_16B_get_l2(struct hfi1_16b_header *hdr)
+{
+	return (u8)((hdr->lrh[1] & OPA_16B_L2_MASK) >> OPA_16B_L2_SHIFT);
+}
+
+static inline u16 hfi1_16B_get_pkey(struct hfi1_16b_header *hdr)
+{
+	return (u16)((hdr->lrh[2] & OPA_16B_PKEY_MASK) >> OPA_16B_PKEY_SHIFT);
+}
+
+static inline u8 hfi1_16B_get_rc(struct hfi1_16b_header *hdr)
+{
+	return (u8)((hdr->lrh[1] & OPA_16B_RC_MASK) >> OPA_16B_RC_SHIFT);
+}
+
+static inline u8 hfi1_16B_get_age(struct hfi1_16b_header *hdr)
+{
+	return (u8)((hdr->lrh[3] & OPA_16B_AGE_MASK) >> OPA_16B_AGE_SHIFT);
+}
+
+static inline u16 hfi1_16B_get_len(struct hfi1_16b_header *hdr)
+{
+	return (u16)((hdr->lrh[0] & OPA_16B_LEN_MASK) >> OPA_16B_LEN_SHIFT);
+}
+
+static inline u16 hfi1_16B_get_entropy(struct hfi1_16b_header *hdr)
+{
+	return (u16)(hdr->lrh[3] & OPA_16B_ENTROPY_MASK);
+}
+
+#define OPA_16B_MAKE_QW(low_dw, high_dw) (((u64)(high_dw) << 32) | (low_dw))
+
+/*
+ * BTH
+ */
+#define OPA_16B_BTH_PAD_MASK	7
+static inline u8 hfi1_16B_bth_get_pad(struct ib_other_headers *ohdr)
+{
+	return (u8)((be32_to_cpu(ohdr->bth[0]) >> IB_BTH_PAD_SHIFT) &
+		   OPA_16B_BTH_PAD_MASK);
+}
+
 struct rvt_sge_state;
 
 /*
@@ -512,7 +614,7 @@ static inline void incr_cntr32(u32 *cntr)
 #define MAX_NAME_SIZE 64
 struct hfi1_msix_entry {
 	enum irq_type type;
-	struct msix_entry msix;
+	int irq;
 	void *arg;
 	char name[MAX_NAME_SIZE];
 	cpumask_t mask;
@@ -575,6 +677,9 @@ struct hfi1_pportdata {
 	u8  default_atten;
 	u8  max_power_class;
 
+	/* did we read platform config from scratch registers? */
+	bool config_from_scratch;
+
 	/* GUIDs for this interface, in host order, guids[0] is a port guid */
 	u64 guids[HFI1_GUIDS_PER_PORT];
 
@@ -593,6 +698,7 @@ struct hfi1_pportdata {
 	/* SendDMA related entries */
 
 	struct workqueue_struct *hfi1_wq;
+	struct workqueue_struct *link_wq;
 
 	/* move out of interrupt context */
 	struct work_struct link_vc_work;
@@ -607,8 +713,6 @@ struct hfi1_pportdata {
 	struct mutex hls_lock;
 	u32 host_link_state;
 
-	u32 lstate;	/* logical link state */
-
 	/* these are the "32 bit" regs */
 
 	u32 ibmtu; /* The MTU programmed for this unit */
@@ -619,7 +723,7 @@ struct hfi1_pportdata {
 	u32 ibmaxlen;
 	u32 current_egress_rate; /* units [10^6 bits/sec] */
 	/* LID programmed for this instance */
-	u16 lid;
+	u32 lid;
 	/* list of pkeys programmed; 0 if not set */
 	u16 pkeys[MAX_PKEY_VALUES];
 	u16 link_width_supported;
@@ -654,12 +758,12 @@ struct hfi1_pportdata {
 	u8 link_enabled;	/* link enabled? */
 	u8 linkinit_reason;
 	u8 local_tx_rate;	/* rate given to 8051 firmware */
-	u8 last_pstate;		/* info only */
 	u8 qsfp_retry_count;
 
 	/* placeholders for IB MAD packet settings */
 	u8 overrun_threshold;
 	u8 phy_error_threshold;
+	unsigned int is_link_down_queued;
 
 	/* Used to override LED behavior for things like maintenance beaconing*/
 	/*
@@ -756,6 +860,10 @@ struct hfi1_pportdata {
 typedef int (*rhf_rcv_function_ptr)(struct hfi1_packet *packet);
 
 typedef void (*opcode_handler)(struct hfi1_packet *packet);
+typedef void (*hfi1_make_req)(struct rvt_qp *qp,
+			      struct hfi1_pkt_state *ps,
+			      struct rvt_swqe *wqe);
+
 
 /* return values for the RHF receive functions */
 #define RHF_RCV_CONTINUE  0	/* keep going */
@@ -860,12 +968,15 @@ struct hfi1_devdata {
 	struct device *diag_device;
 	struct device *ui_device;
 
-	/* mem-mapped pointer to base of chip regs */
-	u8 __iomem *kregbase;
-	/* end of mem-mapped chip space excluding sendbuf and user regs */
-	u8 __iomem *kregend;
-	/* physical address of chip for io_remap, etc. */
+	/* first mapping up to RcvArray */
+	u8 __iomem *kregbase1;
 	resource_size_t physaddr;
+
+	/* second uncached mapping from RcvArray to pio send buffers */
+	u8 __iomem *kregbase2;
+	/* for detecting offset above kregbase2 address */
+	u32 base2_start;
+
 	/* Per VL data. Enough for all VLs but not all elements are set/used. */
 	struct per_vl_data vld[PER_VL_SEND_CONTEXTS];
 	/* send context data */
@@ -953,8 +1064,7 @@ struct hfi1_devdata {
 	u64 __iomem *egrtidbase;
 	spinlock_t sendctrl_lock; /* protect changes to SendCtrl */
 	spinlock_t rcvctrl_lock; /* protect changes to RcvCtrl */
-	/* around rcd and (user ctxts) ctxt_cnt use (intr vs free) */
-	spinlock_t uctxt_lock; /* rcd and user context changes */
+	spinlock_t uctxt_lock; /* protect rcd changes */
 	struct mutex dc8051_lock; /* exclusive access to 8051 */
 	struct workqueue_struct *update_cntr_wq;
 	struct work_struct update_cntr_work;
@@ -1229,9 +1339,10 @@ static inline bool hfi1_vnic_is_rsm_full(struct hfi1_devdata *dd, int spare)
 #define dc8051_ver_patch(a) ((a) & 0x0000ff)
 
 /* f_put_tid types */
-#define PT_EXPECTED 0
-#define PT_EAGER    1
-#define PT_INVALID  2
+#define PT_EXPECTED       0
+#define PT_EAGER          1
+#define PT_INVALID_FLUSH  2
+#define PT_INVALID        3
 
 struct tid_rb_node;
 struct mmu_rb_node;
@@ -1276,13 +1387,16 @@ void handle_user_interrupt(struct hfi1_ctxtdata *rcd);
 
 int hfi1_create_rcvhdrq(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd);
 int hfi1_setup_eagerbufs(struct hfi1_ctxtdata *rcd);
-int hfi1_create_ctxts(struct hfi1_devdata *dd);
-struct hfi1_ctxtdata *hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, u32 ctxt,
-					   int numa);
+int hfi1_create_kctxts(struct hfi1_devdata *dd);
+int hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, int numa,
+			 struct hfi1_ctxtdata **rcd);
+void hfi1_free_ctxt(struct hfi1_ctxtdata *rcd);
 void hfi1_init_pportdata(struct pci_dev *pdev, struct hfi1_pportdata *ppd,
 			 struct hfi1_devdata *dd, u8 hw_pidx, u8 port);
 void hfi1_free_ctxtdata(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd);
-
+int hfi1_rcd_put(struct hfi1_ctxtdata *rcd);
+void hfi1_rcd_get(struct hfi1_ctxtdata *rcd);
+struct hfi1_ctxtdata *hfi1_rcd_get_by_index(struct hfi1_devdata *dd, u16 ctxt);
 int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread);
 int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread);
 int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread);
@@ -1292,6 +1406,13 @@ void hfi1_set_vnic_msix_info(struct hfi1_ctxtdata *rcd);
 void hfi1_reset_vnic_msix_info(struct hfi1_ctxtdata *rcd);
 
 extern const struct pci_device_id hfi1_pci_tbl[];
+void hfi1_make_ud_req_9B(struct rvt_qp *qp,
+			 struct hfi1_pkt_state *ps,
+			 struct rvt_swqe *wqe);
+
+void hfi1_make_ud_req_16B(struct rvt_qp *qp,
+			  struct hfi1_pkt_state *ps,
+			  struct rvt_swqe *wqe);
 
 /* receive packet handler dispositions */
 #define RCV_PKT_OK      0x0 /* keep going */
@@ -1306,21 +1427,6 @@ static inline __le32 *get_rhf_addr(struct hfi1_ctxtdata *rcd)
 
 int hfi1_reset_device(int);
 
-/* return the driver's idea of the logical OPA port state */
-static inline u32 driver_lstate(struct hfi1_pportdata *ppd)
-{
-	/*
-	 * The driver does some processing from the time the logical
-	 * link state is at INIT to the time the SM can be notified
-	 * as such. Return IB_PORT_DOWN until the software state
-	 * is ready.
-	 */
-	if (ppd->lstate == IB_PORT_INIT && !(ppd->host_link_state & HLS_UP))
-		return IB_PORT_DOWN;
-	else
-		return ppd->lstate;
-}
-
 void receive_interrupt_work(struct work_struct *work);
 
 /* extract service channel from header and rhf */
@@ -1413,13 +1519,25 @@ static inline u32 egress_cycles(u32 len, u32 rate)
 }
 
 void set_link_ipg(struct hfi1_pportdata *ppd);
-void process_becn(struct hfi1_pportdata *ppd, u8 sl,  u16 rlid, u32 lqpn,
+void process_becn(struct hfi1_pportdata *ppd, u8 sl, u32 rlid, u32 lqpn,
 		  u32 rqpn, u8 svc_type);
 void return_cnp(struct hfi1_ibport *ibp, struct rvt_qp *qp, u32 remote_qpn,
 		u32 pkey, u32 slid, u32 dlid, u8 sc5,
 		const struct ib_grh *old_grh);
+void return_cnp_16B(struct hfi1_ibport *ibp, struct rvt_qp *qp,
+		    u32 remote_qpn, u32 pkey, u32 slid, u32 dlid,
+		    u8 sc5, const struct ib_grh *old_grh);
+typedef void (*hfi1_handle_cnp)(struct hfi1_ibport *ibp, struct rvt_qp *qp,
+				u32 remote_qpn, u32 pkey, u32 slid, u32 dlid,
+				u8 sc5, const struct ib_grh *old_grh);
+
+/* We support only two types - 9B and 16B for now */
+static const hfi1_handle_cnp hfi1_handle_cnp_tbl[2] = {
+	[HFI1_PKT_TYPE_9B] = &return_cnp,
+	[HFI1_PKT_TYPE_16B] = &return_cnp_16B
+};
 #define PKEY_CHECK_INVALID -1
-int egress_pkey_check(struct hfi1_pportdata *ppd, __be16 *lrh, __be32 *bth,
+int egress_pkey_check(struct hfi1_pportdata *ppd, u32 slid, u16 pkey,
 		      u8 sc5, int8_t s_pkey_index);
 
 #define PACKET_EGRESS_TIMEOUT 350
@@ -1522,9 +1640,9 @@ static void ingress_pkey_table_fail(struct hfi1_pportdata *ppd, u16 pkey,
  * by HW and rcv_pkey_check function should be called instead.
  */
 static inline int ingress_pkey_check(struct hfi1_pportdata *ppd, u16 pkey,
-				     u8 sc5, u8 idx, u16 slid)
+				     u8 sc5, u8 idx, u32 slid, bool force)
 {
-	if (!(ppd->part_enforce & HFI1_PART_ENFORCE_IN))
+	if (!(force) && !(ppd->part_enforce & HFI1_PART_ENFORCE_IN))
 		return 0;
 
 	/* If SC15, pkey[0:14] must be 0x7fff */
@@ -1658,12 +1776,22 @@ static inline bool process_ecn(struct rvt_qp *qp, struct hfi1_packet *pkt,
 			       bool do_cnp)
 {
 	struct ib_other_headers *ohdr = pkt->ohdr;
-	u32 bth1;
 
-	bth1 = be32_to_cpu(ohdr->bth[1]);
-	if (unlikely(bth1 & (IB_BECN_SMASK | IB_FECN_SMASK))) {
+	u32 bth1;
+	bool becn = false;
+	bool fecn = false;
+
+	if (pkt->etype == RHF_RCV_TYPE_BYPASS) {
+		fecn = hfi1_16B_get_fecn(pkt->hdr);
+		becn = hfi1_16B_get_becn(pkt->hdr);
+	} else {
+		bth1 = be32_to_cpu(ohdr->bth[1]);
+		fecn = bth1 & IB_FECN_SMASK;
+		becn = bth1 & IB_BECN_SMASK;
+	}
+	if (unlikely(fecn || becn)) {
 		hfi1_process_ecn_slowpath(qp, pkt, do_cnp);
-		return !!(bth1 & IB_FECN_SMASK);
+		return fecn;
 	}
 	return false;
 }
@@ -1829,10 +1957,9 @@ void hfi1_pcie_cleanup(struct pci_dev *pdev);
 int hfi1_pcie_ddinit(struct hfi1_devdata *dd, struct pci_dev *pdev);
 void hfi1_pcie_ddcleanup(struct hfi1_devdata *);
 int pcie_speeds(struct hfi1_devdata *dd);
-void request_msix(struct hfi1_devdata *dd, u32 *nent,
-		  struct hfi1_msix_entry *entry);
-void hfi1_enable_intx(struct pci_dev *pdev);
-void restore_pci_variables(struct hfi1_devdata *dd);
+int request_msix(struct hfi1_devdata *dd, u32 msireq);
+int restore_pci_variables(struct hfi1_devdata *dd);
+int save_pci_variables(struct hfi1_devdata *dd);
 int do_pcie_gen3_transition(struct hfi1_devdata *dd);
 int parse_platform_config(struct hfi1_devdata *dd);
 int get_platform_config_field(struct hfi1_devdata *dd,
@@ -1860,6 +1987,7 @@ int process_receive_error(struct hfi1_packet *packet);
 int kdeth_process_expected(struct hfi1_packet *packet);
 int kdeth_process_eager(struct hfi1_packet *packet);
 int process_receive_invalid(struct hfi1_packet *packet);
+void seqfile_dump_rcd(struct seq_file *s, struct hfi1_ctxtdata *rcd);
 
 /* global module parameter variables */
 extern unsigned int hfi1_max_mtu;
@@ -1991,9 +2119,15 @@ static inline u64 hfi1_pkt_base_sdma_integrity(struct hfi1_devdata *dd)
 #define dd_dev_emerg(dd, fmt, ...) \
 	dev_emerg(&(dd)->pcidev->dev, "%s: " fmt, \
 		  get_unit_name((dd)->unit), ##__VA_ARGS__)
+
 #define dd_dev_err(dd, fmt, ...) \
 	dev_err(&(dd)->pcidev->dev, "%s: " fmt, \
 			get_unit_name((dd)->unit), ##__VA_ARGS__)
+
+#define dd_dev_err_ratelimited(dd, fmt, ...) \
+	dev_err_ratelimited(&(dd)->pcidev->dev, "%s: " fmt, \
+			get_unit_name((dd)->unit), ##__VA_ARGS__)
+
 #define dd_dev_warn(dd, fmt, ...) \
 	dev_warn(&(dd)->pcidev->dev, "%s: " fmt, \
 			get_unit_name((dd)->unit), ##__VA_ARGS__)
@@ -2087,52 +2221,220 @@ int hfi1_tempsense_rd(struct hfi1_devdata *dd, struct hfi1_temp *temp);
 #define DD_DEV_ENTRY(dd)       __string(dev, dev_name(&(dd)->pcidev->dev))
 #define DD_DEV_ASSIGN(dd)      __assign_str(dev, dev_name(&(dd)->pcidev->dev))
 
-#define packettype_name(etype) { RHF_RCV_TYPE_##etype, #etype }
-#define show_packettype(etype)                  \
-__print_symbolic(etype,                         \
-	packettype_name(EXPECTED),              \
-	packettype_name(EAGER),                 \
-	packettype_name(IB),                    \
-	packettype_name(ERROR),                 \
-	packettype_name(BYPASS))
-
-#define ib_opcode_name(opcode) { IB_OPCODE_##opcode, #opcode  }
-#define show_ib_opcode(opcode)                             \
-__print_symbolic(opcode,                                   \
-	ib_opcode_name(RC_SEND_FIRST),                     \
-	ib_opcode_name(RC_SEND_MIDDLE),                    \
-	ib_opcode_name(RC_SEND_LAST),                      \
-	ib_opcode_name(RC_SEND_LAST_WITH_IMMEDIATE),       \
-	ib_opcode_name(RC_SEND_ONLY),                      \
-	ib_opcode_name(RC_SEND_ONLY_WITH_IMMEDIATE),       \
-	ib_opcode_name(RC_RDMA_WRITE_FIRST),               \
-	ib_opcode_name(RC_RDMA_WRITE_MIDDLE),              \
-	ib_opcode_name(RC_RDMA_WRITE_LAST),                \
-	ib_opcode_name(RC_RDMA_WRITE_LAST_WITH_IMMEDIATE), \
-	ib_opcode_name(RC_RDMA_WRITE_ONLY),                \
-	ib_opcode_name(RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE), \
-	ib_opcode_name(RC_RDMA_READ_REQUEST),              \
-	ib_opcode_name(RC_RDMA_READ_RESPONSE_FIRST),       \
-	ib_opcode_name(RC_RDMA_READ_RESPONSE_MIDDLE),      \
-	ib_opcode_name(RC_RDMA_READ_RESPONSE_LAST),        \
-	ib_opcode_name(RC_RDMA_READ_RESPONSE_ONLY),        \
-	ib_opcode_name(RC_ACKNOWLEDGE),                    \
-	ib_opcode_name(RC_ATOMIC_ACKNOWLEDGE),             \
-	ib_opcode_name(RC_COMPARE_SWAP),                   \
-	ib_opcode_name(RC_FETCH_ADD),                      \
-	ib_opcode_name(UC_SEND_FIRST),                     \
-	ib_opcode_name(UC_SEND_MIDDLE),                    \
-	ib_opcode_name(UC_SEND_LAST),                      \
-	ib_opcode_name(UC_SEND_LAST_WITH_IMMEDIATE),       \
-	ib_opcode_name(UC_SEND_ONLY),                      \
-	ib_opcode_name(UC_SEND_ONLY_WITH_IMMEDIATE),       \
-	ib_opcode_name(UC_RDMA_WRITE_FIRST),               \
-	ib_opcode_name(UC_RDMA_WRITE_MIDDLE),              \
-	ib_opcode_name(UC_RDMA_WRITE_LAST),                \
-	ib_opcode_name(UC_RDMA_WRITE_LAST_WITH_IMMEDIATE), \
-	ib_opcode_name(UC_RDMA_WRITE_ONLY),                \
-	ib_opcode_name(UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE), \
-	ib_opcode_name(UD_SEND_ONLY),                      \
-	ib_opcode_name(UD_SEND_ONLY_WITH_IMMEDIATE),       \
-	ib_opcode_name(CNP))
+static inline void hfi1_update_ah_attr(struct ib_device *ibdev,
+				       struct rdma_ah_attr *attr)
+{
+	struct hfi1_pportdata *ppd;
+	struct hfi1_ibport *ibp;
+	u32 dlid = rdma_ah_get_dlid(attr);
+
+	/*
+	 * Kernel clients may not have setup GRH information
+	 * Set that here.
+	 */
+	ibp = to_iport(ibdev, rdma_ah_get_port_num(attr));
+	ppd = ppd_from_ibp(ibp);
+	if ((((dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) ||
+	      (ppd->lid >= be16_to_cpu(IB_MULTICAST_LID_BASE))) &&
+	    (dlid != be32_to_cpu(OPA_LID_PERMISSIVE)) &&
+	    (dlid != be16_to_cpu(IB_LID_PERMISSIVE)) &&
+	    (!(rdma_ah_get_ah_flags(attr) & IB_AH_GRH))) ||
+	    (rdma_ah_get_make_grd(attr))) {
+		rdma_ah_set_ah_flags(attr, IB_AH_GRH);
+		rdma_ah_set_interface_id(attr, OPA_MAKE_ID(dlid));
+		rdma_ah_set_subnet_prefix(attr, ibp->rvp.gid_prefix);
+	}
+}
+
+/*
+ * hfi1_check_mcast- Check if the given lid is
+ * in the OPA multicast range.
+ *
+ * The LID might either reside in ah.dlid or might be
+ * in the GRH of the address handle as DGID if extended
+ * addresses are in use.
+ */
+static inline bool hfi1_check_mcast(u32 lid)
+{
+	return ((lid >= opa_get_mcast_base(OPA_MCAST_NR)) &&
+		(lid != be32_to_cpu(OPA_LID_PERMISSIVE)));
+}
+
+#define opa_get_lid(lid, format)	\
+	__opa_get_lid(lid, OPA_PORT_PACKET_FORMAT_##format)
+
+/* Convert a lid to a specific lid space */
+static inline u32 __opa_get_lid(u32 lid, u8 format)
+{
+	bool is_mcast = hfi1_check_mcast(lid);
+
+	switch (format) {
+	case OPA_PORT_PACKET_FORMAT_8B:
+	case OPA_PORT_PACKET_FORMAT_10B:
+		if (is_mcast)
+			return (lid - opa_get_mcast_base(OPA_MCAST_NR) +
+				0xF0000);
+		return lid & 0xFFFFF;
+	case OPA_PORT_PACKET_FORMAT_16B:
+		if (is_mcast)
+			return (lid - opa_get_mcast_base(OPA_MCAST_NR) +
+				0xF00000);
+		return lid & 0xFFFFFF;
+	case OPA_PORT_PACKET_FORMAT_9B:
+		if (is_mcast)
+			return (lid -
+				opa_get_mcast_base(OPA_MCAST_NR) +
+				be16_to_cpu(IB_MULTICAST_LID_BASE));
+		else
+			return lid & 0xFFFF;
+	default:
+		return lid;
+	}
+}
+
+/* Return true if the given lid is the OPA 16B multicast range */
+static inline bool hfi1_is_16B_mcast(u32 lid)
+{
+	return ((lid >=
+		opa_get_lid(opa_get_mcast_base(OPA_MCAST_NR), 16B)) &&
+		(lid != opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE), 16B)));
+}
+
+static inline void hfi1_make_opa_lid(struct rdma_ah_attr *attr)
+{
+	const struct ib_global_route *grh = rdma_ah_read_grh(attr);
+	u32 dlid = rdma_ah_get_dlid(attr);
+
+	/* Modify ah_attr.dlid to be in the 32 bit LID space.
+	 * This is how the address will be laid out:
+	 * Assuming MCAST_NR to be 4,
+	 * 32 bit permissive LID = 0xFFFFFFFF
+	 * Multicast LID range = 0xFFFFFFFE to 0xF0000000
+	 * Unicast LID range = 0xEFFFFFFF to 1
+	 * Invalid LID = 0
+	 */
+	if (ib_is_opa_gid(&grh->dgid))
+		dlid = opa_get_lid_from_gid(&grh->dgid);
+	else if ((dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
+		 (dlid != be16_to_cpu(IB_LID_PERMISSIVE)) &&
+		 (dlid != be32_to_cpu(OPA_LID_PERMISSIVE)))
+		dlid = dlid - be16_to_cpu(IB_MULTICAST_LID_BASE) +
+			opa_get_mcast_base(OPA_MCAST_NR);
+	else if (dlid == be16_to_cpu(IB_LID_PERMISSIVE))
+		dlid = be32_to_cpu(OPA_LID_PERMISSIVE);
+
+	rdma_ah_set_dlid(attr, dlid);
+}
+
+static inline u8 hfi1_get_packet_type(u32 lid)
+{
+	/* 9B if lid > 0xF0000000 */
+	if (lid >= opa_get_mcast_base(OPA_MCAST_NR))
+		return HFI1_PKT_TYPE_9B;
+
+	/* 16B if lid > 0xC000 */
+	if (lid >= opa_get_lid(opa_get_mcast_base(OPA_MCAST_NR), 9B))
+		return HFI1_PKT_TYPE_16B;
+
+	return HFI1_PKT_TYPE_9B;
+}
+
+static inline bool hfi1_get_hdr_type(u32 lid, struct rdma_ah_attr *attr)
+{
+	/*
+	 * If there was an incoming 16B packet with permissive
+	 * LIDs, OPA GIDs would have been programmed when those
+	 * packets were received. A 16B packet will have to
+	 * be sent in response to that packet. Return a 16B
+	 * header type if that's the case.
+	 */
+	if (rdma_ah_get_dlid(attr) == be32_to_cpu(OPA_LID_PERMISSIVE))
+		return (ib_is_opa_gid(&rdma_ah_read_grh(attr)->dgid)) ?
+			HFI1_PKT_TYPE_16B : HFI1_PKT_TYPE_9B;
+
+	/*
+	 * Return a 16B header type if either the the destination
+	 * or source lid is extended.
+	 */
+	if (hfi1_get_packet_type(rdma_ah_get_dlid(attr)) == HFI1_PKT_TYPE_16B)
+		return HFI1_PKT_TYPE_16B;
+
+	return hfi1_get_packet_type(lid);
+}
+
+static inline void hfi1_make_ext_grh(struct hfi1_packet *packet,
+				     struct ib_grh *grh, u32 slid,
+				     u32 dlid)
+{
+	struct hfi1_ibport *ibp = &packet->rcd->ppd->ibport_data;
+	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
+
+	if (!ibp)
+		return;
+
+	grh->hop_limit = 1;
+	grh->sgid.global.subnet_prefix = ibp->rvp.gid_prefix;
+	if (slid == opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE), 16B))
+		grh->sgid.global.interface_id =
+			OPA_MAKE_ID(be32_to_cpu(OPA_LID_PERMISSIVE));
+	else
+		grh->sgid.global.interface_id = OPA_MAKE_ID(slid);
+
+	/*
+	 * Upper layers (like mad) may compare the dgid in the
+	 * wc that is obtained here with the sgid_index in
+	 * the wr. Since sgid_index in wr is always 0 for
+	 * extended lids, set the dgid here to the default
+	 * IB gid.
+	 */
+	grh->dgid.global.subnet_prefix = ibp->rvp.gid_prefix;
+	grh->dgid.global.interface_id =
+		cpu_to_be64(ppd->guids[HFI1_PORT_GUID_INDEX]);
+}
+
+static inline int hfi1_get_16b_padding(u32 hdr_size, u32 payload)
+{
+	return -(hdr_size + payload + (SIZE_OF_CRC << 2) +
+		     SIZE_OF_LT) & 0x7;
+}
+
+static inline void hfi1_make_ib_hdr(struct ib_header *hdr,
+				    u16 lrh0, u16 len,
+				    u16 dlid, u16 slid)
+{
+	hdr->lrh[0] = cpu_to_be16(lrh0);
+	hdr->lrh[1] = cpu_to_be16(dlid);
+	hdr->lrh[2] = cpu_to_be16(len);
+	hdr->lrh[3] = cpu_to_be16(slid);
+}
+
+static inline void hfi1_make_16b_hdr(struct hfi1_16b_header *hdr,
+				     u32 slid, u32 dlid,
+				     u16 len, u16 pkey,
+				     u8 becn, u8 fecn, u8 l4,
+				     u8 sc)
+{
+	u32 lrh0 = 0;
+	u32 lrh1 = 0x40000000;
+	u32 lrh2 = 0;
+	u32 lrh3 = 0;
+
+	lrh0 = (lrh0 & ~OPA_16B_BECN_MASK) | (becn << OPA_16B_BECN_SHIFT);
+	lrh0 = (lrh0 & ~OPA_16B_LEN_MASK) | (len << OPA_16B_LEN_SHIFT);
+	lrh0 = (lrh0 & ~OPA_16B_LID_MASK)  | (slid & OPA_16B_LID_MASK);
+	lrh1 = (lrh1 & ~OPA_16B_FECN_MASK) | (fecn << OPA_16B_FECN_SHIFT);
+	lrh1 = (lrh1 & ~OPA_16B_SC_MASK) | (sc << OPA_16B_SC_SHIFT);
+	lrh1 = (lrh1 & ~OPA_16B_LID_MASK) | (dlid & OPA_16B_LID_MASK);
+	lrh2 = (lrh2 & ~OPA_16B_SLID_MASK) |
+		((slid >> OPA_16B_SLID_SHIFT) << OPA_16B_SLID_HIGH_SHIFT);
+	lrh2 = (lrh2 & ~OPA_16B_DLID_MASK) |
+		((dlid >> OPA_16B_DLID_SHIFT) << OPA_16B_DLID_HIGH_SHIFT);
+	lrh2 = (lrh2 & ~OPA_16B_PKEY_MASK) | (pkey << OPA_16B_PKEY_SHIFT);
+	lrh2 = (lrh2 & ~OPA_16B_L4_MASK) | l4;
+
+	hdr->lrh[0] = lrh0;
+	hdr->lrh[1] = lrh1;
+	hdr->lrh[2] = lrh2;
+	hdr->lrh[3] = lrh3;
+}
 #endif                          /* _HFI1_KERNEL_H */
diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c
index 4a11d4da4c92..fba77001c3a7 100644
--- a/drivers/infiniband/hw/hfi1/init.c
+++ b/drivers/infiniband/hw/hfi1/init.c
@@ -67,6 +67,7 @@
 #include "aspm.h"
 #include "affinity.h"
 #include "vnic.h"
+#include "exp_rcv.h"
 
 #undef pr_fmt
 #define pr_fmt(fmt) DRIVER_NAME ": " fmt
@@ -125,85 +126,198 @@ static struct idr hfi1_unit_table;
 u32 hfi1_cpulist_count;
 unsigned long *hfi1_cpulist;
 
-/*
- * Common code for creating the receive context array.
- */
-int hfi1_create_ctxts(struct hfi1_devdata *dd)
+static int hfi1_create_kctxt(struct hfi1_devdata *dd,
+			     struct hfi1_pportdata *ppd)
 {
-	unsigned i;
+	struct hfi1_ctxtdata *rcd;
 	int ret;
 
 	/* Control context has to be always 0 */
 	BUILD_BUG_ON(HFI1_CTRL_CTXT != 0);
 
+	ret = hfi1_create_ctxtdata(ppd, dd->node, &rcd);
+	if (ret < 0) {
+		dd_dev_err(dd, "Kernel receive context allocation failed\n");
+		return ret;
+	}
+
+	/*
+	 * Set up the kernel context flags here and now because they use
+	 * default values for all receive side memories.  User contexts will
+	 * be handled as they are created.
+	 */
+	rcd->flags = HFI1_CAP_KGET(MULTI_PKT_EGR) |
+		HFI1_CAP_KGET(NODROP_RHQ_FULL) |
+		HFI1_CAP_KGET(NODROP_EGR_FULL) |
+		HFI1_CAP_KGET(DMA_RTAIL);
+
+	/* Control context must use DMA_RTAIL */
+	if (rcd->ctxt == HFI1_CTRL_CTXT)
+		rcd->flags |= HFI1_CAP_DMA_RTAIL;
+	rcd->seq_cnt = 1;
+
+	rcd->sc = sc_alloc(dd, SC_ACK, rcd->rcvhdrqentsize, dd->node);
+	if (!rcd->sc) {
+		dd_dev_err(dd, "Kernel send context allocation failed\n");
+		return -ENOMEM;
+	}
+	hfi1_init_ctxt(rcd->sc);
+
+	return 0;
+}
+
+/*
+ * Create the receive context array and one or more kernel contexts
+ */
+int hfi1_create_kctxts(struct hfi1_devdata *dd)
+{
+	u16 i;
+	int ret;
+
 	dd->rcd = kzalloc_node(dd->num_rcv_contexts * sizeof(*dd->rcd),
 			       GFP_KERNEL, dd->node);
 	if (!dd->rcd)
-		goto nomem;
+		return -ENOMEM;
 
-	/* create one or more kernel contexts */
 	for (i = 0; i < dd->first_dyn_alloc_ctxt; ++i) {
-		struct hfi1_pportdata *ppd;
-		struct hfi1_ctxtdata *rcd;
+		ret = hfi1_create_kctxt(dd, dd->pport);
+		if (ret)
+			goto bail;
+	}
 
-		ppd = dd->pport + (i % dd->num_pports);
+	return 0;
+bail:
+	for (i = 0; dd->rcd && i < dd->first_dyn_alloc_ctxt; ++i)
+		hfi1_free_ctxt(dd->rcd[i]);
 
-		/* dd->rcd[i] gets assigned inside the callee */
-		rcd = hfi1_create_ctxtdata(ppd, i, dd->node);
-		if (!rcd) {
-			dd_dev_err(dd,
-				   "Unable to allocate kernel receive context, failing\n");
-			goto nomem;
-		}
-		/*
-		 * Set up the kernel context flags here and now because they
-		 * use default values for all receive side memories.  User
-		 * contexts will be handled as they are created.
-		 */
-		rcd->flags = HFI1_CAP_KGET(MULTI_PKT_EGR) |
-			HFI1_CAP_KGET(NODROP_RHQ_FULL) |
-			HFI1_CAP_KGET(NODROP_EGR_FULL) |
-			HFI1_CAP_KGET(DMA_RTAIL);
-
-		/* Control context must use DMA_RTAIL */
-		if (rcd->ctxt == HFI1_CTRL_CTXT)
-			rcd->flags |= HFI1_CAP_DMA_RTAIL;
-		rcd->seq_cnt = 1;
-
-		rcd->sc = sc_alloc(dd, SC_ACK, rcd->rcvhdrqentsize, dd->node);
-		if (!rcd->sc) {
-			dd_dev_err(dd,
-				   "Unable to allocate kernel send context, failing\n");
-			goto nomem;
-		}
+	/* All the contexts should be freed, free the array */
+	kfree(dd->rcd);
+	dd->rcd = NULL;
+	return ret;
+}
+
+/*
+ * Helper routines for the receive context reference count (rcd and uctxt).
+ */
+static void hfi1_rcd_init(struct hfi1_ctxtdata *rcd)
+{
+	kref_init(&rcd->kref);
+}
 
-		hfi1_init_ctxt(rcd->sc);
+/**
+ * hfi1_rcd_free - When reference is zero clean up.
+ * @kref: pointer to an initialized rcd data structure
+ *
+ */
+static void hfi1_rcd_free(struct kref *kref)
+{
+	unsigned long flags;
+	struct hfi1_ctxtdata *rcd =
+		container_of(kref, struct hfi1_ctxtdata, kref);
+
+	hfi1_free_ctxtdata(rcd->dd, rcd);
+
+	spin_lock_irqsave(&rcd->dd->uctxt_lock, flags);
+	rcd->dd->rcd[rcd->ctxt] = NULL;
+	spin_unlock_irqrestore(&rcd->dd->uctxt_lock, flags);
+
+	kfree(rcd);
+}
+
+/**
+ * hfi1_rcd_put - decrement reference for rcd
+ * @rcd: pointer to an initialized rcd data structure
+ *
+ * Use this to put a reference after the init.
+ */
+int hfi1_rcd_put(struct hfi1_ctxtdata *rcd)
+{
+	if (rcd)
+		return kref_put(&rcd->kref, hfi1_rcd_free);
+
+	return 0;
+}
+
+/**
+ * hfi1_rcd_get - increment reference for rcd
+ * @rcd: pointer to an initialized rcd data structure
+ *
+ * Use this to get a reference after the init.
+ */
+void hfi1_rcd_get(struct hfi1_ctxtdata *rcd)
+{
+	kref_get(&rcd->kref);
+}
+
+/**
+ * allocate_rcd_index - allocate an rcd index from the rcd array
+ * @dd: pointer to a valid devdata structure
+ * @rcd: rcd data structure to assign
+ * @index: pointer to index that is allocated
+ *
+ * Find an empty index in the rcd array, and assign the given rcd to it.
+ * If the array is full, we are EBUSY.
+ *
+ */
+static int allocate_rcd_index(struct hfi1_devdata *dd,
+			      struct hfi1_ctxtdata *rcd, u16 *index)
+{
+	unsigned long flags;
+	u16 ctxt;
+
+	spin_lock_irqsave(&dd->uctxt_lock, flags);
+	for (ctxt = 0; ctxt < dd->num_rcv_contexts; ctxt++)
+		if (!dd->rcd[ctxt])
+			break;
+
+	if (ctxt < dd->num_rcv_contexts) {
+		rcd->ctxt = ctxt;
+		dd->rcd[ctxt] = rcd;
+		hfi1_rcd_init(rcd);
 	}
+	spin_unlock_irqrestore(&dd->uctxt_lock, flags);
 
-	/*
-	 * Initialize aspm, to be done after gen3 transition and setting up
-	 * contexts and before enabling interrupts
-	 */
-	aspm_init(dd);
+	if (ctxt >= dd->num_rcv_contexts)
+		return -EBUSY;
+
+	*index = ctxt;
 
 	return 0;
-nomem:
-	ret = -ENOMEM;
+}
 
-	if (dd->rcd) {
-		for (i = 0; i < dd->num_rcv_contexts; ++i)
-			hfi1_free_ctxtdata(dd, dd->rcd[i]);
+/**
+ * hfi1_rcd_get_by_index
+ * @dd: pointer to a valid devdata structure
+ * @ctxt: the index of an possilbe rcd
+ *
+ * We need to protect access to the rcd array.  If access is needed to
+ * one or more index, get the protecting spinlock and then increment the
+ * kref.
+ *
+ * The caller is responsible for making the _put().
+ *
+ */
+struct hfi1_ctxtdata *hfi1_rcd_get_by_index(struct hfi1_devdata *dd, u16 ctxt)
+{
+	unsigned long flags;
+	struct hfi1_ctxtdata *rcd = NULL;
+
+	spin_lock_irqsave(&dd->uctxt_lock, flags);
+	if (dd->rcd[ctxt]) {
+		rcd = dd->rcd[ctxt];
+		hfi1_rcd_get(rcd);
 	}
-	kfree(dd->rcd);
-	dd->rcd = NULL;
-	return ret;
+	spin_unlock_irqrestore(&dd->uctxt_lock, flags);
+
+	return rcd;
 }
 
 /*
- * Common code for user and kernel context setup.
+ * Common code for user and kernel context create and setup.
+ * NOTE: the initial kref is done here (hf1_rcd_init()).
  */
-struct hfi1_ctxtdata *hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, u32 ctxt,
-					   int numa)
+int hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, int numa,
+			 struct hfi1_ctxtdata **context)
 {
 	struct hfi1_devdata *dd = ppd->dd;
 	struct hfi1_ctxtdata *rcd;
@@ -217,20 +331,30 @@ struct hfi1_ctxtdata *hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, u32 ctxt,
 	rcd = kzalloc_node(sizeof(*rcd), GFP_KERNEL, numa);
 	if (rcd) {
 		u32 rcvtids, max_entries;
-
-		hfi1_cdbg(PROC, "setting up context %u\n", ctxt);
+		u16 ctxt;
+		int ret;
+
+		ret = allocate_rcd_index(dd, rcd, &ctxt);
+		if (ret) {
+			*context = NULL;
+			kfree(rcd);
+			return ret;
+		}
 
 		INIT_LIST_HEAD(&rcd->qp_wait_list);
+		hfi1_exp_tid_group_init(&rcd->tid_group_list);
+		hfi1_exp_tid_group_init(&rcd->tid_used_list);
+		hfi1_exp_tid_group_init(&rcd->tid_full_list);
 		rcd->ppd = ppd;
 		rcd->dd = dd;
 		__set_bit(0, rcd->in_use_ctxts);
-		rcd->ctxt = ctxt;
-		dd->rcd[ctxt] = rcd;
 		rcd->numa_id = numa;
 		rcd->rcv_array_groups = dd->rcv_entries.ngroups;
 
 		mutex_init(&rcd->exp_lock);
 
+		hfi1_cdbg(PROC, "setting up context %u\n", rcd->ctxt);
+
 		/*
 		 * Calculate the context's RcvArray entry starting point.
 		 * We do this here because we have to take into account all
@@ -328,14 +452,30 @@ struct hfi1_ctxtdata *hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, u32 ctxt,
 			if (!rcd->opstats)
 				goto bail;
 		}
+
+		*context = rcd;
+		return 0;
 	}
-	return rcd;
+
 bail:
-	dd->rcd[ctxt] = NULL;
-	kfree(rcd->egrbufs.rcvtids);
-	kfree(rcd->egrbufs.buffers);
-	kfree(rcd);
-	return NULL;
+	*context = NULL;
+	hfi1_free_ctxt(rcd);
+	return -ENOMEM;
+}
+
+/**
+ * hfi1_free_ctxt
+ * @rcd: pointer to an initialized rcd data structure
+ *
+ * This wrapper is the free function that matches hfi1_create_ctxtdata().
+ * When a context is done being used (kernel or user), this function is called
+ * for the "final" put to match the kref init from hf1i_create_ctxtdata().
+ * Other users of the context do a get/put sequence to make sure that the
+ * structure isn't removed while in use.
+ */
+void hfi1_free_ctxt(struct hfi1_ctxtdata *rcd)
+{
+	hfi1_rcd_put(rcd);
 }
 
 /*
@@ -483,7 +623,6 @@ void hfi1_init_pportdata(struct pci_dev *pdev, struct hfi1_pportdata *ppd,
 
 	ppd->pkeys[default_pkey_idx] = DEFAULT_P_KEY;
 	ppd->part_enforce |= HFI1_PART_ENFORCE_IN;
-	ppd->part_enforce |= HFI1_PART_ENFORCE_OUT;
 
 	if (loopback) {
 		hfi1_early_err(&pdev->dev,
@@ -559,16 +698,19 @@ static int loadtime_init(struct hfi1_devdata *dd)
 static int init_after_reset(struct hfi1_devdata *dd)
 {
 	int i;
-
+	struct hfi1_ctxtdata *rcd;
 	/*
 	 * Ensure chip does no sends or receives, tail updates, or
 	 * pioavail updates while we re-initialize.  This is mostly
 	 * for the driver data structures, not chip registers.
 	 */
-	for (i = 0; i < dd->num_rcv_contexts; i++)
+	for (i = 0; i < dd->num_rcv_contexts; i++) {
+		rcd = hfi1_rcd_get_by_index(dd, i);
 		hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS |
-				  HFI1_RCVCTRL_INTRAVAIL_DIS |
-				  HFI1_RCVCTRL_TAILUPD_DIS, i);
+			     HFI1_RCVCTRL_INTRAVAIL_DIS |
+			     HFI1_RCVCTRL_TAILUPD_DIS, rcd);
+		hfi1_rcd_put(rcd);
+	}
 	pio_send_control(dd, PSC_GLOBAL_DISABLE);
 	for (i = 0; i < dd->num_send_contexts; i++)
 		sc_disable(dd->send_contexts[i].sc);
@@ -578,8 +720,9 @@ static int init_after_reset(struct hfi1_devdata *dd)
 
 static void enable_chip(struct hfi1_devdata *dd)
 {
+	struct hfi1_ctxtdata *rcd;
 	u32 rcvmask;
-	u32 i;
+	u16 i;
 
 	/* enable PIO send */
 	pio_send_control(dd, PSC_GLOBAL_ENABLE);
@@ -589,17 +732,21 @@ static void enable_chip(struct hfi1_devdata *dd)
 	 * Other ctxts done as user opens and initializes them.
 	 */
 	for (i = 0; i < dd->first_dyn_alloc_ctxt; ++i) {
+		rcd = hfi1_rcd_get_by_index(dd, i);
+		if (!rcd)
+			continue;
 		rcvmask = HFI1_RCVCTRL_CTXT_ENB | HFI1_RCVCTRL_INTRAVAIL_ENB;
-		rcvmask |= HFI1_CAP_KGET_MASK(dd->rcd[i]->flags, DMA_RTAIL) ?
+		rcvmask |= HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) ?
 			HFI1_RCVCTRL_TAILUPD_ENB : HFI1_RCVCTRL_TAILUPD_DIS;
-		if (!HFI1_CAP_KGET_MASK(dd->rcd[i]->flags, MULTI_PKT_EGR))
+		if (!HFI1_CAP_KGET_MASK(rcd->flags, MULTI_PKT_EGR))
 			rcvmask |= HFI1_RCVCTRL_ONE_PKT_EGR_ENB;
-		if (HFI1_CAP_KGET_MASK(dd->rcd[i]->flags, NODROP_RHQ_FULL))
+		if (HFI1_CAP_KGET_MASK(rcd->flags, NODROP_RHQ_FULL))
 			rcvmask |= HFI1_RCVCTRL_NO_RHQ_DROP_ENB;
-		if (HFI1_CAP_KGET_MASK(dd->rcd[i]->flags, NODROP_EGR_FULL))
+		if (HFI1_CAP_KGET_MASK(rcd->flags, NODROP_EGR_FULL))
 			rcvmask |= HFI1_RCVCTRL_NO_EGR_DROP_ENB;
-		hfi1_rcvctrl(dd, rcvmask, i);
-		sc_enable(dd->rcd[i]->sc);
+		hfi1_rcvctrl(dd, rcvmask, rcd);
+		sc_enable(rcd->sc);
+		hfi1_rcd_put(rcd);
 	}
 }
 
@@ -624,6 +771,20 @@ static int create_workqueues(struct hfi1_devdata *dd)
 			if (!ppd->hfi1_wq)
 				goto wq_error;
 		}
+		if (!ppd->link_wq) {
+			/*
+			 * Make the link workqueue single-threaded to enforce
+			 * serialization.
+			 */
+			ppd->link_wq =
+				alloc_workqueue(
+				    "hfi_link_%d_%d",
+				    WQ_SYSFS | WQ_MEM_RECLAIM | WQ_UNBOUND,
+				    1, /* max_active */
+				    dd->unit, pidx);
+			if (!ppd->link_wq)
+				goto wq_error;
+		}
 	}
 	return 0;
 wq_error:
@@ -634,6 +795,10 @@ wq_error:
 			destroy_workqueue(ppd->hfi1_wq);
 			ppd->hfi1_wq = NULL;
 		}
+		if (ppd->link_wq) {
+			destroy_workqueue(ppd->link_wq);
+			ppd->link_wq = NULL;
+		}
 	}
 	return -ENOMEM;
 }
@@ -656,7 +821,8 @@ wq_error:
 int hfi1_init(struct hfi1_devdata *dd, int reinit)
 {
 	int ret = 0, pidx, lastfail = 0;
-	unsigned i, len;
+	unsigned long len;
+	u16 i;
 	struct hfi1_ctxtdata *rcd;
 	struct hfi1_pportdata *ppd;
 
@@ -725,7 +891,7 @@ int hfi1_init(struct hfi1_devdata *dd, int reinit)
 		 * existing, and re-allocate.
 		 * Need to re-create rest of ctxt 0 ctxtdata as well.
 		 */
-		rcd = dd->rcd[i];
+		rcd = hfi1_rcd_get_by_index(dd, i);
 		if (!rcd)
 			continue;
 
@@ -739,6 +905,7 @@ int hfi1_init(struct hfi1_devdata *dd, int reinit)
 				   "failed to allocate kernel ctxt's rcvhdrq and/or egr bufs\n");
 			ret = lastfail;
 		}
+		hfi1_rcd_put(rcd);
 	}
 
 	/* Allocate enough memory for user event notification. */
@@ -858,6 +1025,7 @@ static void stop_timers(struct hfi1_devdata *dd)
 static void shutdown_device(struct hfi1_devdata *dd)
 {
 	struct hfi1_pportdata *ppd;
+	struct hfi1_ctxtdata *rcd;
 	unsigned pidx;
 	int i;
 
@@ -876,12 +1044,15 @@ static void shutdown_device(struct hfi1_devdata *dd)
 
 	for (pidx = 0; pidx < dd->num_pports; ++pidx) {
 		ppd = dd->pport + pidx;
-		for (i = 0; i < dd->num_rcv_contexts; i++)
+		for (i = 0; i < dd->num_rcv_contexts; i++) {
+			rcd = hfi1_rcd_get_by_index(dd, i);
 			hfi1_rcvctrl(dd, HFI1_RCVCTRL_TAILUPD_DIS |
-					  HFI1_RCVCTRL_CTXT_DIS |
-					  HFI1_RCVCTRL_INTRAVAIL_DIS |
-					  HFI1_RCVCTRL_PKEY_DIS |
-					  HFI1_RCVCTRL_ONE_PKT_EGR_DIS, i);
+				     HFI1_RCVCTRL_CTXT_DIS |
+				     HFI1_RCVCTRL_INTRAVAIL_DIS |
+				     HFI1_RCVCTRL_PKEY_DIS |
+				     HFI1_RCVCTRL_ONE_PKT_EGR_DIS, rcd);
+			hfi1_rcd_put(rcd);
+		}
 		/*
 		 * Gracefully stop all sends allowing any in progress to
 		 * trickle out first.
@@ -917,6 +1088,10 @@ static void shutdown_device(struct hfi1_devdata *dd)
 			destroy_workqueue(ppd->hfi1_wq);
 			ppd->hfi1_wq = NULL;
 		}
+		if (ppd->link_wq) {
+			destroy_workqueue(ppd->link_wq);
+			ppd->link_wq = NULL;
+		}
 	}
 	sdma_exit(dd);
 }
@@ -927,14 +1102,11 @@ static void shutdown_device(struct hfi1_devdata *dd)
  * @rcd: the ctxtdata structure
  *
  * free up any allocated data for a context
- * This should not touch anything that would affect a simultaneous
- * re-allocation of context data, because it is called after hfi1_mutex
- * is released (and can be called from reinit as well).
  * It should never change any chip state, or global driver state.
  */
 void hfi1_free_ctxtdata(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
 {
-	unsigned e;
+	u32 e;
 
 	if (!rcd)
 		return;
@@ -953,6 +1125,7 @@ void hfi1_free_ctxtdata(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
 
 	/* all the RcvArray entries should have been cleared by now */
 	kfree(rcd->egrbufs.rcvtids);
+	rcd->egrbufs.rcvtids = NULL;
 
 	for (e = 0; e < rcd->egrbufs.alloced; e++) {
 		if (rcd->egrbufs.buffers[e].dma)
@@ -962,13 +1135,21 @@ void hfi1_free_ctxtdata(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
 					  rcd->egrbufs.buffers[e].dma);
 	}
 	kfree(rcd->egrbufs.buffers);
+	rcd->egrbufs.alloced = 0;
+	rcd->egrbufs.buffers = NULL;
 
 	sc_free(rcd->sc);
+	rcd->sc = NULL;
+
 	vfree(rcd->subctxt_uregbase);
 	vfree(rcd->subctxt_rcvegrbuf);
 	vfree(rcd->subctxt_rcvhdr_base);
 	kfree(rcd->opstats);
-	kfree(rcd);
+
+	rcd->subctxt_uregbase = NULL;
+	rcd->subctxt_rcvegrbuf = NULL;
+	rcd->subctxt_rcvhdr_base = NULL;
+	rcd->opstats = NULL;
 }
 
 /*
@@ -1311,8 +1492,6 @@ static void cleanup_device_data(struct hfi1_devdata *dd)
 {
 	int ctxt;
 	int pidx;
-	struct hfi1_ctxtdata **tmp;
-	unsigned long flags;
 
 	/* users can't do anything more with chip */
 	for (pidx = 0; pidx < dd->num_pports; ++pidx) {
@@ -1337,18 +1516,6 @@ static void cleanup_device_data(struct hfi1_devdata *dd)
 
 	free_credit_return(dd);
 
-	/*
-	 * Free any resources still in use (usually just kernel contexts)
-	 * at unload; we do for ctxtcnt, because that's what we allocate.
-	 * We acquire lock to be really paranoid that rcd isn't being
-	 * accessed from some interrupt-related code (that should not happen,
-	 * but best to be sure).
-	 */
-	spin_lock_irqsave(&dd->uctxt_lock, flags);
-	tmp = dd->rcd;
-	dd->rcd = NULL;
-	spin_unlock_irqrestore(&dd->uctxt_lock, flags);
-
 	if (dd->rcvhdrtail_dummy_kvaddr) {
 		dma_free_coherent(&dd->pcidev->dev, sizeof(u64),
 				  (void *)dd->rcvhdrtail_dummy_kvaddr,
@@ -1356,16 +1523,22 @@ static void cleanup_device_data(struct hfi1_devdata *dd)
 		dd->rcvhdrtail_dummy_kvaddr = NULL;
 	}
 
-	for (ctxt = 0; tmp && ctxt < dd->num_rcv_contexts; ctxt++) {
-		struct hfi1_ctxtdata *rcd = tmp[ctxt];
+	/*
+	 * Free any resources still in use (usually just kernel contexts)
+	 * at unload; we do for ctxtcnt, because that's what we allocate.
+	 */
+	for (ctxt = 0; dd->rcd && ctxt < dd->num_rcv_contexts; ctxt++) {
+		struct hfi1_ctxtdata *rcd = dd->rcd[ctxt];
 
-		tmp[ctxt] = NULL; /* debugging paranoia */
 		if (rcd) {
 			hfi1_clear_tids(rcd);
-			hfi1_free_ctxtdata(dd, rcd);
+			hfi1_free_ctxt(rcd);
 		}
 	}
-	kfree(tmp);
+
+	kfree(dd->rcd);
+	dd->rcd = NULL;
+
 	free_pio_map(dd);
 	/* must follow rcv context free - need to remove rcv's hooks */
 	for (ctxt = 0; ctxt < dd->num_send_contexts; ctxt++)
@@ -1532,6 +1705,10 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 				destroy_workqueue(ppd->hfi1_wq);
 				ppd->hfi1_wq = NULL;
 			}
+			if (ppd->link_wq) {
+				destroy_workqueue(ppd->link_wq);
+				ppd->link_wq = NULL;
+			}
 		}
 		if (!j)
 			hfi1_device_remove(dd);
diff --git a/drivers/infiniband/hw/hfi1/intr.c b/drivers/infiniband/hw/hfi1/intr.c
index 04a5082d5ac5..96845dfed5c5 100644
--- a/drivers/infiniband/hw/hfi1/intr.c
+++ b/drivers/infiniband/hw/hfi1/intr.c
@@ -164,6 +164,7 @@ void handle_linkup_change(struct hfi1_devdata *dd, u32 linkup)
 		ppd->linkup = 0;
 
 		/* clear HW details of the previous connection */
+		ppd->actual_vls_operational = 0;
 		reset_link_credits(dd);
 
 		/* freeze after a link down to guarantee a clean egress */
@@ -196,7 +197,7 @@ void handle_user_interrupt(struct hfi1_ctxtdata *rcd)
 
 	if (test_and_clear_bit(HFI1_CTXT_WAITING_RCV, &rcd->event_flags)) {
 		wake_up_interruptible(&rcd->wait);
-		hfi1_rcvctrl(dd, HFI1_RCVCTRL_INTRAVAIL_DIS, rcd->ctxt);
+		hfi1_rcvctrl(dd, HFI1_RCVCTRL_INTRAVAIL_DIS, rcd);
 	} else if (test_and_clear_bit(HFI1_CTXT_WAITING_URG,
 							&rcd->event_flags)) {
 		rcd->urgent++;
diff --git a/drivers/infiniband/hw/hfi1/iowait.h b/drivers/infiniband/hw/hfi1/iowait.h
index d9740ddea6f1..591697d85eed 100644
--- a/drivers/infiniband/hw/hfi1/iowait.h
+++ b/drivers/infiniband/hw/hfi1/iowait.h
@@ -106,7 +106,9 @@ struct iowait {
 		struct sdma_engine *sde,
 		struct iowait *wait,
 		struct sdma_txreq *tx,
-		unsigned seq);
+		uint seq,
+		bool pkts_sent
+		);
 	void (*wakeup)(struct iowait *wait, int reason);
 	void (*sdma_drained)(struct iowait *wait);
 	seqlock_t *lock;
@@ -118,6 +120,7 @@ struct iowait {
 	u32 count;
 	u32 tx_limit;
 	u32 tx_count;
+	u8 starved_cnt;
 };
 
 #define SDMA_AVAIL_REASON 0
@@ -143,7 +146,8 @@ static inline void iowait_init(
 		struct sdma_engine *sde,
 		struct iowait *wait,
 		struct sdma_txreq *tx,
-		unsigned seq),
+		uint seq,
+		bool pkts_sent),
 	void (*wakeup)(struct iowait *wait, int reason),
 	void (*sdma_drained)(struct iowait *wait))
 {
@@ -305,4 +309,66 @@ static inline struct sdma_txreq *iowait_get_txhead(struct iowait *wait)
 	return tx;
 }
 
+/**
+ * iowait_queue - Put the iowait on a wait queue
+ * @pkts_sent: have some packets been sent before queuing?
+ * @w: the iowait struct
+ * @wait_head: the wait queue
+ *
+ * This function is called to insert an iowait struct into a
+ * wait queue after a resource (eg, sdma decriptor or pio
+ * buffer) is run out.
+ */
+static inline void iowait_queue(bool pkts_sent, struct iowait *w,
+				struct list_head *wait_head)
+{
+	/*
+	 * To play fair, insert the iowait at the tail of the wait queue if it
+	 * has already sent some packets; Otherwise, put it at the head.
+	 */
+	if (pkts_sent) {
+		list_add_tail(&w->list, wait_head);
+		w->starved_cnt = 0;
+	} else {
+		list_add(&w->list, wait_head);
+		w->starved_cnt++;
+	}
+}
+
+/**
+ * iowait_starve_clear - clear the wait queue's starve count
+ * @pkts_sent: have some packets been sent?
+ * @w: the iowait struct
+ *
+ * This function is called to clear the starve count. If no
+ * packets have been sent, the starve count will not be cleared.
+ */
+static inline void iowait_starve_clear(bool pkts_sent, struct iowait *w)
+{
+	if (pkts_sent)
+		w->starved_cnt = 0;
+}
+
+/**
+ * iowait_starve_find_max - Find the maximum of the starve count
+ * @w: the iowait struct
+ * @max: a variable containing the max starve count
+ * @idx: the index of the current iowait in an array
+ * @max_idx: a variable containing the array index for the
+ *         iowait entry that has the max starve count
+ *
+ * This function is called to compare the starve count of a
+ * given iowait with the given max starve count. The max starve
+ * count and the index will be updated if the iowait's start
+ * count is larger.
+ */
+static inline void iowait_starve_find_max(struct iowait *w, u8 *max,
+					  uint idx, uint *max_idx)
+{
+	if (w->starved_cnt > *max) {
+		*max = w->starved_cnt;
+		*max_idx = idx;
+	}
+}
+
 #endif
diff --git a/drivers/infiniband/hw/hfi1/mad.c b/drivers/infiniband/hw/hfi1/mad.c
index 5977673a52d4..f4c0ffc040cc 100644
--- a/drivers/infiniband/hw/hfi1/mad.c
+++ b/drivers/infiniband/hw/hfi1/mad.c
@@ -46,6 +46,7 @@
  */
 
 #include <linux/net.h>
+#include <rdma/opa_addr.h>
 #define OPA_NUM_PKEY_BLOCKS_PER_SMP (OPA_SMP_DR_DATA_SIZE \
 			/ (OPA_PARTITION_TABLE_BLK_SIZE * sizeof(u16)))
 
@@ -59,6 +60,24 @@
 #define OPA_LINK_WIDTH_RESET_OLD 0x0fff
 #define OPA_LINK_WIDTH_RESET 0xffff
 
+struct trap_node {
+	struct list_head list;
+	struct opa_mad_notice_attr data;
+	__be64 tid;
+	int len;
+	u32 retry;
+	u8 in_use;
+	u8 repress;
+};
+
+static int smp_length_check(u32 data_size, u32 request_len)
+{
+	if (unlikely(request_len < data_size))
+		return -EINVAL;
+
+	return 0;
+}
+
 static int reply(struct ib_mad_hdr *smp)
 {
 	/*
@@ -89,28 +108,222 @@ void hfi1_event_pkey_change(struct hfi1_devdata *dd, u8 port)
 	ib_dispatch_event(&event);
 }
 
-static void send_trap(struct hfi1_ibport *ibp, void *data, unsigned len)
+/*
+ * If the port is down, clean up all pending traps.  We need to be careful
+ * with the given trap, because it may be queued.
+ */
+static void cleanup_traps(struct hfi1_ibport *ibp, struct trap_node *trap)
+{
+	struct trap_node *node, *q;
+	unsigned long flags;
+	struct list_head trap_list;
+	int i;
+
+	for (i = 0; i < RVT_MAX_TRAP_LISTS; i++) {
+		spin_lock_irqsave(&ibp->rvp.lock, flags);
+		list_replace_init(&ibp->rvp.trap_lists[i].list, &trap_list);
+		ibp->rvp.trap_lists[i].list_len = 0;
+		spin_unlock_irqrestore(&ibp->rvp.lock, flags);
+
+		/*
+		 * Remove all items from the list, freeing all the non-given
+		 * traps.
+		 */
+		list_for_each_entry_safe(node, q, &trap_list, list) {
+			list_del(&node->list);
+			if (node != trap)
+				kfree(node);
+		}
+	}
+
+	/*
+	 * If this wasn't on one of the lists it would not be freed.  If it
+	 * was on the list, it is now safe to free.
+	 */
+	kfree(trap);
+}
+
+static struct trap_node *check_and_add_trap(struct hfi1_ibport *ibp,
+					    struct trap_node *trap)
+{
+	struct trap_node *node;
+	struct trap_list *trap_list;
+	unsigned long flags;
+	unsigned long timeout;
+	int found = 0;
+	unsigned int queue_id;
+	static int trap_count;
+
+	queue_id = trap->data.generic_type & 0x0F;
+	if (queue_id >= RVT_MAX_TRAP_LISTS) {
+		trap_count++;
+		pr_err_ratelimited("hfi1: Invalid trap 0x%0x dropped. Total dropped: %d\n",
+				   trap->data.generic_type, trap_count);
+		kfree(trap);
+		return NULL;
+	}
+
+	/*
+	 * Since the retry (handle timeout) does not remove a trap request
+	 * from the list, all we have to do is compare the node.
+	 */
+	spin_lock_irqsave(&ibp->rvp.lock, flags);
+	trap_list = &ibp->rvp.trap_lists[queue_id];
+
+	list_for_each_entry(node, &trap_list->list, list) {
+		if (node == trap) {
+			node->retry++;
+			found = 1;
+			break;
+		}
+	}
+
+	/* If it is not on the list, add it, limited to RVT-MAX_TRAP_LEN. */
+	if (!found) {
+		if (trap_list->list_len < RVT_MAX_TRAP_LEN) {
+			trap_list->list_len++;
+			list_add_tail(&trap->list, &trap_list->list);
+		} else {
+			pr_warn_ratelimited("hfi1: Maximum trap limit reached for 0x%0x traps\n",
+					    trap->data.generic_type);
+			kfree(trap);
+		}
+	}
+
+	/*
+	 * Next check to see if there is a timer pending.  If not, set it up
+	 * and get the first trap from the list.
+	 */
+	node = NULL;
+	if (!timer_pending(&ibp->rvp.trap_timer)) {
+		/*
+		 * o14-2
+		 * If the time out is set we have to wait until it expires
+		 * before the trap can be sent.
+		 * This should be > RVT_TRAP_TIMEOUT
+		 */
+		timeout = (RVT_TRAP_TIMEOUT *
+			   (1UL << ibp->rvp.subnet_timeout)) / 1000;
+		mod_timer(&ibp->rvp.trap_timer,
+			  jiffies + usecs_to_jiffies(timeout));
+		node = list_first_entry(&trap_list->list, struct trap_node,
+					list);
+		node->in_use = 1;
+	}
+	spin_unlock_irqrestore(&ibp->rvp.lock, flags);
+
+	return node;
+}
+
+static void subn_handle_opa_trap_repress(struct hfi1_ibport *ibp,
+					 struct opa_smp *smp)
+{
+	struct trap_list *trap_list;
+	struct trap_node *trap;
+	unsigned long flags;
+	int i;
+
+	if (smp->attr_id != IB_SMP_ATTR_NOTICE)
+		return;
+
+	spin_lock_irqsave(&ibp->rvp.lock, flags);
+	for (i = 0; i < RVT_MAX_TRAP_LISTS; i++) {
+		trap_list = &ibp->rvp.trap_lists[i];
+		trap = list_first_entry_or_null(&trap_list->list,
+						struct trap_node, list);
+		if (trap && trap->tid == smp->tid) {
+			if (trap->in_use) {
+				trap->repress = 1;
+			} else {
+				trap_list->list_len--;
+				list_del(&trap->list);
+				kfree(trap);
+			}
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&ibp->rvp.lock, flags);
+}
+
+static void hfi1_update_sm_ah_attr(struct hfi1_ibport *ibp,
+				   struct rdma_ah_attr *attr, u32 dlid)
+{
+	rdma_ah_set_dlid(attr, dlid);
+	rdma_ah_set_port_num(attr, ppd_from_ibp(ibp)->port);
+	if (dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) {
+		struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);
+
+		rdma_ah_set_ah_flags(attr, IB_AH_GRH);
+		grh->sgid_index = 0;
+		grh->hop_limit = 1;
+		grh->dgid.global.subnet_prefix =
+			ibp->rvp.gid_prefix;
+		grh->dgid.global.interface_id = OPA_MAKE_ID(dlid);
+	}
+}
+
+static int hfi1_modify_qp0_ah(struct hfi1_ibport *ibp,
+			      struct rvt_ah *ah, u32 dlid)
+{
+	struct rdma_ah_attr attr;
+	struct rvt_qp *qp0;
+	int ret = -EINVAL;
+
+	memset(&attr, 0, sizeof(attr));
+	attr.type = ah->ibah.type;
+	hfi1_update_sm_ah_attr(ibp, &attr, dlid);
+	rcu_read_lock();
+	qp0 = rcu_dereference(ibp->rvp.qp[0]);
+	if (qp0)
+		ret = rdma_modify_ah(&ah->ibah, &attr);
+	rcu_read_unlock();
+	return ret;
+}
+
+static struct ib_ah *hfi1_create_qp0_ah(struct hfi1_ibport *ibp, u32 dlid)
+{
+	struct rdma_ah_attr attr;
+	struct ib_ah *ah = ERR_PTR(-EINVAL);
+	struct rvt_qp *qp0;
+	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
+	struct hfi1_devdata *dd = dd_from_ppd(ppd);
+	u8 port_num = ppd->port;
+
+	memset(&attr, 0, sizeof(attr));
+	attr.type = rdma_ah_find_type(&dd->verbs_dev.rdi.ibdev, port_num);
+	hfi1_update_sm_ah_attr(ibp, &attr, dlid);
+	rcu_read_lock();
+	qp0 = rcu_dereference(ibp->rvp.qp[0]);
+	if (qp0)
+		ah = rdma_create_ah(qp0->ibqp.pd, &attr);
+	rcu_read_unlock();
+	return ah;
+}
+
+static void send_trap(struct hfi1_ibport *ibp, struct trap_node *trap)
 {
 	struct ib_mad_send_buf *send_buf;
 	struct ib_mad_agent *agent;
 	struct opa_smp *smp;
-	int ret;
 	unsigned long flags;
-	unsigned long timeout;
 	int pkey_idx;
 	u32 qpn = ppd_from_ibp(ibp)->sm_trap_qp;
 
 	agent = ibp->rvp.send_agent;
-	if (!agent)
+	if (!agent) {
+		cleanup_traps(ibp, trap);
 		return;
+	}
 
 	/* o14-3.2.1 */
-	if (ppd_from_ibp(ibp)->lstate != IB_PORT_ACTIVE)
+	if (driver_lstate(ppd_from_ibp(ibp)) != IB_PORT_ACTIVE) {
+		cleanup_traps(ibp, trap);
 		return;
+	}
 
-	/* o14-2 */
-	if (ibp->rvp.trap_timeout && time_before(jiffies,
-						 ibp->rvp.trap_timeout))
+	/* Add the trap to the list if necessary and see if we can send it */
+	trap = check_and_add_trap(ibp, trap);
+	if (!trap)
 		return;
 
 	pkey_idx = hfi1_lookup_pkey_idx(ibp, LIM_MGMT_P_KEY);
@@ -131,11 +344,21 @@ static void send_trap(struct hfi1_ibport *ibp, void *data, unsigned len)
 	smp->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
 	smp->class_version = OPA_SM_CLASS_VERSION;
 	smp->method = IB_MGMT_METHOD_TRAP;
-	ibp->rvp.tid++;
-	smp->tid = cpu_to_be64(ibp->rvp.tid);
+
+	/* Only update the transaction ID for new traps (o13-5). */
+	if (trap->tid == 0) {
+		ibp->rvp.tid++;
+		/* make sure that tid != 0 */
+		if (ibp->rvp.tid == 0)
+			ibp->rvp.tid++;
+		trap->tid = cpu_to_be64(ibp->rvp.tid);
+	}
+	smp->tid = trap->tid;
+
 	smp->attr_id = IB_SMP_ATTR_NOTICE;
 	/* o14-1: smp->mkey = 0; */
-	memcpy(smp->route.lid.data, data, len);
+
+	memcpy(smp->route.lid.data, &trap->data, trap->len);
 
 	spin_lock_irqsave(&ibp->rvp.lock, flags);
 	if (!ibp->rvp.sm_ah) {
@@ -144,65 +367,101 @@ static void send_trap(struct hfi1_ibport *ibp, void *data, unsigned len)
 
 			ah = hfi1_create_qp0_ah(ibp, ibp->rvp.sm_lid);
 			if (IS_ERR(ah)) {
-				ret = PTR_ERR(ah);
-			} else {
-				send_buf->ah = ah;
-				ibp->rvp.sm_ah = ibah_to_rvtah(ah);
-				ret = 0;
+				spin_unlock_irqrestore(&ibp->rvp.lock, flags);
+				return;
 			}
+			send_buf->ah = ah;
+			ibp->rvp.sm_ah = ibah_to_rvtah(ah);
 		} else {
-			ret = -EINVAL;
+			spin_unlock_irqrestore(&ibp->rvp.lock, flags);
+			return;
 		}
 	} else {
 		send_buf->ah = &ibp->rvp.sm_ah->ibah;
-		ret = 0;
 	}
+
+	/*
+	 * If the trap was repressed while things were getting set up, don't
+	 * bother sending it. This could happen for a retry.
+	 */
+	if (trap->repress) {
+		list_del(&trap->list);
+		spin_unlock_irqrestore(&ibp->rvp.lock, flags);
+		kfree(trap);
+		ib_free_send_mad(send_buf);
+		return;
+	}
+
+	trap->in_use = 0;
 	spin_unlock_irqrestore(&ibp->rvp.lock, flags);
 
-	if (!ret)
-		ret = ib_post_send_mad(send_buf, NULL);
-	if (!ret) {
-		/* 4.096 usec. */
-		timeout = (4096 * (1UL << ibp->rvp.subnet_timeout)) / 1000;
-		ibp->rvp.trap_timeout = jiffies + usecs_to_jiffies(timeout);
-	} else {
+	if (ib_post_send_mad(send_buf, NULL))
 		ib_free_send_mad(send_buf);
-		ibp->rvp.trap_timeout = 0;
+}
+
+void hfi1_handle_trap_timer(unsigned long data)
+{
+	struct hfi1_ibport *ibp = (struct hfi1_ibport *)data;
+	struct trap_node *trap = NULL;
+	unsigned long flags;
+	int i;
+
+	/* Find the trap with the highest priority */
+	spin_lock_irqsave(&ibp->rvp.lock, flags);
+	for (i = 0; !trap && i < RVT_MAX_TRAP_LISTS; i++) {
+		trap = list_first_entry_or_null(&ibp->rvp.trap_lists[i].list,
+						struct trap_node, list);
 	}
+	spin_unlock_irqrestore(&ibp->rvp.lock, flags);
+
+	if (trap)
+		send_trap(ibp, trap);
+}
+
+static struct trap_node *create_trap_node(u8 type, __be16 trap_num, u32 lid)
+{
+	struct trap_node *trap;
+
+	trap = kzalloc(sizeof(*trap), GFP_ATOMIC);
+	if (!trap)
+		return NULL;
+
+	INIT_LIST_HEAD(&trap->list);
+	trap->data.generic_type = type;
+	trap->data.prod_type_lsb = IB_NOTICE_PROD_CA;
+	trap->data.trap_num = trap_num;
+	trap->data.issuer_lid = cpu_to_be32(lid);
+
+	return trap;
 }
 
 /*
- * Send a bad [PQ]_Key trap (ch. 14.3.8).
+ * Send a bad P_Key trap (ch. 14.3.8).
  */
-void hfi1_bad_pqkey(struct hfi1_ibport *ibp, __be16 trap_num, u32 key, u32 sl,
-		    u32 qp1, u32 qp2, u16 lid1, u16 lid2)
+void hfi1_bad_pkey(struct hfi1_ibport *ibp, u32 key, u32 sl,
+		   u32 qp1, u32 qp2, u32 lid1, u32 lid2)
 {
-	struct opa_mad_notice_attr data;
+	struct trap_node *trap;
 	u32 lid = ppd_from_ibp(ibp)->lid;
-	u32 _lid1 = lid1;
-	u32 _lid2 = lid2;
-
-	memset(&data, 0, sizeof(data));
 
-	if (trap_num == OPA_TRAP_BAD_P_KEY)
-		ibp->rvp.pkey_violations++;
-	else
-		ibp->rvp.qkey_violations++;
 	ibp->rvp.n_pkt_drops++;
+	ibp->rvp.pkey_violations++;
+
+	trap = create_trap_node(IB_NOTICE_TYPE_SECURITY, OPA_TRAP_BAD_P_KEY,
+				lid);
+	if (!trap)
+		return;
 
 	/* Send violation trap */
-	data.generic_type = IB_NOTICE_TYPE_SECURITY;
-	data.prod_type_lsb = IB_NOTICE_PROD_CA;
-	data.trap_num = trap_num;
-	data.issuer_lid = cpu_to_be32(lid);
-	data.ntc_257_258.lid1 = cpu_to_be32(_lid1);
-	data.ntc_257_258.lid2 = cpu_to_be32(_lid2);
-	data.ntc_257_258.key = cpu_to_be32(key);
-	data.ntc_257_258.sl = sl << 3;
-	data.ntc_257_258.qp1 = cpu_to_be32(qp1);
-	data.ntc_257_258.qp2 = cpu_to_be32(qp2);
-
-	send_trap(ibp, &data, sizeof(data));
+	trap->data.ntc_257_258.lid1 = cpu_to_be32(lid1);
+	trap->data.ntc_257_258.lid2 = cpu_to_be32(lid2);
+	trap->data.ntc_257_258.key = cpu_to_be32(key);
+	trap->data.ntc_257_258.sl = sl << 3;
+	trap->data.ntc_257_258.qp1 = cpu_to_be32(qp1);
+	trap->data.ntc_257_258.qp2 = cpu_to_be32(qp2);
+
+	trap->len = sizeof(trap->data);
+	send_trap(ibp, trap);
 }
 
 /*
@@ -211,34 +470,36 @@ void hfi1_bad_pqkey(struct hfi1_ibport *ibp, __be16 trap_num, u32 key, u32 sl,
 static void bad_mkey(struct hfi1_ibport *ibp, struct ib_mad_hdr *mad,
 		     __be64 mkey, __be32 dr_slid, u8 return_path[], u8 hop_cnt)
 {
-	struct opa_mad_notice_attr data;
+	struct trap_node *trap;
 	u32 lid = ppd_from_ibp(ibp)->lid;
 
-	memset(&data, 0, sizeof(data));
+	trap = create_trap_node(IB_NOTICE_TYPE_SECURITY, OPA_TRAP_BAD_M_KEY,
+				lid);
+	if (!trap)
+		return;
+
 	/* Send violation trap */
-	data.generic_type = IB_NOTICE_TYPE_SECURITY;
-	data.prod_type_lsb = IB_NOTICE_PROD_CA;
-	data.trap_num = OPA_TRAP_BAD_M_KEY;
-	data.issuer_lid = cpu_to_be32(lid);
-	data.ntc_256.lid = data.issuer_lid;
-	data.ntc_256.method = mad->method;
-	data.ntc_256.attr_id = mad->attr_id;
-	data.ntc_256.attr_mod = mad->attr_mod;
-	data.ntc_256.mkey = mkey;
+	trap->data.ntc_256.lid = trap->data.issuer_lid;
+	trap->data.ntc_256.method = mad->method;
+	trap->data.ntc_256.attr_id = mad->attr_id;
+	trap->data.ntc_256.attr_mod = mad->attr_mod;
+	trap->data.ntc_256.mkey = mkey;
 	if (mad->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) {
-		data.ntc_256.dr_slid = dr_slid;
-		data.ntc_256.dr_trunc_hop = IB_NOTICE_TRAP_DR_NOTICE;
-		if (hop_cnt > ARRAY_SIZE(data.ntc_256.dr_rtn_path)) {
-			data.ntc_256.dr_trunc_hop |=
+		trap->data.ntc_256.dr_slid = dr_slid;
+		trap->data.ntc_256.dr_trunc_hop = IB_NOTICE_TRAP_DR_NOTICE;
+		if (hop_cnt > ARRAY_SIZE(trap->data.ntc_256.dr_rtn_path)) {
+			trap->data.ntc_256.dr_trunc_hop |=
 				IB_NOTICE_TRAP_DR_TRUNC;
-			hop_cnt = ARRAY_SIZE(data.ntc_256.dr_rtn_path);
+			hop_cnt = ARRAY_SIZE(trap->data.ntc_256.dr_rtn_path);
 		}
-		data.ntc_256.dr_trunc_hop |= hop_cnt;
-		memcpy(data.ntc_256.dr_rtn_path, return_path,
+		trap->data.ntc_256.dr_trunc_hop |= hop_cnt;
+		memcpy(trap->data.ntc_256.dr_rtn_path, return_path,
 		       hop_cnt);
 	}
 
-	send_trap(ibp, &data, sizeof(data));
+	trap->len = sizeof(trap->data);
+
+	send_trap(ibp, trap);
 }
 
 /*
@@ -246,22 +507,24 @@ static void bad_mkey(struct hfi1_ibport *ibp, struct ib_mad_hdr *mad,
  */
 void hfi1_cap_mask_chg(struct rvt_dev_info *rdi, u8 port_num)
 {
-	struct opa_mad_notice_attr data;
+	struct trap_node *trap;
 	struct hfi1_ibdev *verbs_dev = dev_from_rdi(rdi);
 	struct hfi1_devdata *dd = dd_from_dev(verbs_dev);
 	struct hfi1_ibport *ibp = &dd->pport[port_num - 1].ibport_data;
 	u32 lid = ppd_from_ibp(ibp)->lid;
 
-	memset(&data, 0, sizeof(data));
+	trap = create_trap_node(IB_NOTICE_TYPE_INFO,
+				OPA_TRAP_CHANGE_CAPABILITY,
+				lid);
+	if (!trap)
+		return;
 
-	data.generic_type = IB_NOTICE_TYPE_INFO;
-	data.prod_type_lsb = IB_NOTICE_PROD_CA;
-	data.trap_num = OPA_TRAP_CHANGE_CAPABILITY;
-	data.issuer_lid = cpu_to_be32(lid);
-	data.ntc_144.lid = data.issuer_lid;
-	data.ntc_144.new_cap_mask = cpu_to_be32(ibp->rvp.port_cap_flags);
+	trap->data.ntc_144.lid = trap->data.issuer_lid;
+	trap->data.ntc_144.new_cap_mask = cpu_to_be32(ibp->rvp.port_cap_flags);
+	trap->data.ntc_144.cap_mask3 = cpu_to_be16(ibp->rvp.port_cap3_flags);
 
-	send_trap(ibp, &data, sizeof(data));
+	trap->len = sizeof(trap->data);
+	send_trap(ibp, trap);
 }
 
 /*
@@ -269,19 +532,19 @@ void hfi1_cap_mask_chg(struct rvt_dev_info *rdi, u8 port_num)
  */
 void hfi1_sys_guid_chg(struct hfi1_ibport *ibp)
 {
-	struct opa_mad_notice_attr data;
+	struct trap_node *trap;
 	u32 lid = ppd_from_ibp(ibp)->lid;
 
-	memset(&data, 0, sizeof(data));
+	trap = create_trap_node(IB_NOTICE_TYPE_INFO, OPA_TRAP_CHANGE_SYSGUID,
+				lid);
+	if (!trap)
+		return;
 
-	data.generic_type = IB_NOTICE_TYPE_INFO;
-	data.prod_type_lsb = IB_NOTICE_PROD_CA;
-	data.trap_num = OPA_TRAP_CHANGE_SYSGUID;
-	data.issuer_lid = cpu_to_be32(lid);
-	data.ntc_145.new_sys_guid = ib_hfi1_sys_image_guid;
-	data.ntc_145.lid = data.issuer_lid;
+	trap->data.ntc_145.new_sys_guid = ib_hfi1_sys_image_guid;
+	trap->data.ntc_145.lid = trap->data.issuer_lid;
 
-	send_trap(ibp, &data, sizeof(data));
+	trap->len = sizeof(trap->data);
+	send_trap(ibp, trap);
 }
 
 /*
@@ -289,29 +552,30 @@ void hfi1_sys_guid_chg(struct hfi1_ibport *ibp)
  */
 void hfi1_node_desc_chg(struct hfi1_ibport *ibp)
 {
-	struct opa_mad_notice_attr data;
+	struct trap_node *trap;
 	u32 lid = ppd_from_ibp(ibp)->lid;
 
-	memset(&data, 0, sizeof(data));
+	trap = create_trap_node(IB_NOTICE_TYPE_INFO,
+				OPA_TRAP_CHANGE_CAPABILITY,
+				lid);
+	if (!trap)
+		return;
 
-	data.generic_type = IB_NOTICE_TYPE_INFO;
-	data.prod_type_lsb = IB_NOTICE_PROD_CA;
-	data.trap_num = OPA_TRAP_CHANGE_CAPABILITY;
-	data.issuer_lid = cpu_to_be32(lid);
-	data.ntc_144.lid = data.issuer_lid;
-	data.ntc_144.change_flags =
+	trap->data.ntc_144.lid = trap->data.issuer_lid;
+	trap->data.ntc_144.change_flags =
 		cpu_to_be16(OPA_NOTICE_TRAP_NODE_DESC_CHG);
 
-	send_trap(ibp, &data, sizeof(data));
+	trap->len = sizeof(trap->data);
+	send_trap(ibp, trap);
 }
 
 static int __subn_get_opa_nodedesc(struct opa_smp *smp, u32 am,
 				   u8 *data, struct ib_device *ibdev,
-				   u8 port, u32 *resp_len)
+				   u8 port, u32 *resp_len, u32 max_len)
 {
 	struct opa_node_description *nd;
 
-	if (am) {
+	if (am || smp_length_check(sizeof(*nd), max_len)) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)smp);
 	}
@@ -328,7 +592,7 @@ static int __subn_get_opa_nodedesc(struct opa_smp *smp, u32 am,
 
 static int __subn_get_opa_nodeinfo(struct opa_smp *smp, u32 am, u8 *data,
 				   struct ib_device *ibdev, u8 port,
-				   u32 *resp_len)
+				   u32 *resp_len, u32 max_len)
 {
 	struct opa_node_info *ni;
 	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);
@@ -338,6 +602,7 @@ static int __subn_get_opa_nodeinfo(struct opa_smp *smp, u32 am, u8 *data,
 
 	/* GUID 0 is illegal */
 	if (am || pidx >= dd->num_pports || ibdev->node_guid == 0 ||
+	    smp_length_check(sizeof(*ni), max_len) ||
 	    get_sguid(to_iport(ibdev, port), HFI1_PORT_GUID_INDEX) == 0) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)smp);
@@ -519,7 +784,7 @@ void read_ltp_rtt(struct hfi1_devdata *dd)
 
 static int __subn_get_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,
 				   struct ib_device *ibdev, u8 port,
-				   u32 *resp_len)
+				   u32 *resp_len, u32 max_len)
 {
 	int i;
 	struct hfi1_devdata *dd;
@@ -535,7 +800,7 @@ static int __subn_get_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,
 	u32 buffer_units;
 	u64 tmp = 0;
 
-	if (num_ports != 1) {
+	if (num_ports != 1 || smp_length_check(sizeof(*pi), max_len)) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)smp);
 	}
@@ -605,7 +870,7 @@ static int __subn_get_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,
 		ppd->offline_disabled_reason;
 
 	pi->port_states.portphysstate_portstate =
-		(hfi1_ibphys_portstate(ppd) << 4) | state;
+		(driver_pstate(ppd) << 4) | state;
 
 	pi->mkeyprotect_lmc = (ibp->rvp.mkeyprot << 6) | ppd->lmc;
 
@@ -704,13 +969,9 @@ static int __subn_get_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,
 	buffer_units |= (dd->vl15_init << 11) & OPA_PI_MASK_BUF_UNIT_VL15_INIT;
 	pi->buffer_units = cpu_to_be32(buffer_units);
 
-	pi->opa_cap_mask = cpu_to_be16(OPA_CAP_MASK3_IsSharedSpaceSupported |
-				       OPA_CAP_MASK3_IsEthOnFabricSupported);
-	/* Driver does not support mcast/collective configuration */
-	pi->opa_cap_mask &=
-		cpu_to_be16(~OPA_CAP_MASK3_IsAddrRangeConfigSupported);
-	pi->collectivemask_multicastmask = ((HFI1_COLLECTIVE_NR & 0x7)
-					    << 3 | (HFI1_MCAST_NR & 0x7));
+	pi->opa_cap_mask = cpu_to_be16(ibp->rvp.port_cap3_flags);
+	pi->collectivemask_multicastmask = ((OPA_COLLECTIVE_NR & 0x7)
+					    << 3 | (OPA_MCAST_NR & 0x7));
 
 	/* HFI supports a replay buffer 128 LTPs in size */
 	pi->replay_depth.buffer = 0x80;
@@ -748,7 +1009,7 @@ static int get_pkeys(struct hfi1_devdata *dd, u8 port, u16 *pkeys)
 
 static int __subn_get_opa_pkeytable(struct opa_smp *smp, u32 am, u8 *data,
 				    struct ib_device *ibdev, u8 port,
-				    u32 *resp_len)
+				    u32 *resp_len, u32 max_len)
 {
 	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);
 	u32 n_blocks_req = OPA_AM_NBLK(am);
@@ -771,6 +1032,11 @@ static int __subn_get_opa_pkeytable(struct opa_smp *smp, u32 am, u8 *data,
 
 	size = (n_blocks_req * OPA_PARTITION_TABLE_BLK_SIZE) * sizeof(u16);
 
+	if (smp_length_check(size, max_len)) {
+		smp->status |= IB_SMP_INVALID_FIELD;
+		return reply((struct ib_mad_hdr *)smp);
+	}
+
 	if (start_block + n_blocks_req > n_blocks_avail ||
 	    n_blocks_req > OPA_NUM_PKEY_BLOCKS_PER_SMP) {
 		pr_warn("OPA Get PKey AM Invalid : s 0x%x; req 0x%x; "
@@ -915,8 +1181,8 @@ static int physical_transition_allowed(int old, int new)
 static int port_states_transition_allowed(struct hfi1_pportdata *ppd,
 					  u32 logical_new, u32 physical_new)
 {
-	u32 physical_old = driver_physical_state(ppd);
-	u32 logical_old = driver_logical_state(ppd);
+	u32 physical_old = driver_pstate(ppd);
+	u32 logical_old = driver_lstate(ppd);
 	int ret, logical_allowed, physical_allowed;
 
 	ret = logical_transition_allowed(logical_old, logical_new);
@@ -1074,7 +1340,7 @@ static int set_port_states(struct hfi1_pportdata *ppd, struct opa_smp *smp,
  */
 static int __subn_set_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,
 				   struct ib_device *ibdev, u8 port,
-				   u32 *resp_len)
+				   u32 *resp_len, u32 max_len)
 {
 	struct opa_port_info *pi = (struct opa_port_info *)data;
 	struct ib_event event;
@@ -1083,8 +1349,8 @@ static int __subn_set_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,
 	struct hfi1_ibport *ibp;
 	u8 clientrereg;
 	unsigned long flags;
-	u32 smlid, opa_lid; /* tmp vars to hold LID values */
-	u16 lid;
+	u32 smlid;
+	u32 lid;
 	u8 ls_old, ls_new, ps_new;
 	u8 vls;
 	u8 msl;
@@ -1095,27 +1361,26 @@ static int __subn_set_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,
 	int ret, i, invalid = 0, call_set_mtu = 0;
 	int call_link_downgrade_policy = 0;
 
-	if (num_ports != 1) {
+	if (num_ports != 1 ||
+	    smp_length_check(sizeof(*pi), max_len)) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)smp);
 	}
 
-	opa_lid = be32_to_cpu(pi->lid);
-	if (opa_lid & 0xFFFF0000) {
-		pr_warn("OPA_PortInfo lid out of range: %X\n", opa_lid);
+	lid = be32_to_cpu(pi->lid);
+	if (lid & 0xFF000000) {
+		pr_warn("OPA_PortInfo lid out of range: %X\n", lid);
 		smp->status |= IB_SMP_INVALID_FIELD;
 		goto get_only;
 	}
 
-	lid = (u16)(opa_lid & 0x0000FFFF);
 
 	smlid = be32_to_cpu(pi->sm_lid);
-	if (smlid & 0xFFFF0000) {
+	if (smlid & 0xFF000000) {
 		pr_warn("OPA_PortInfo SM lid out of range: %X\n", smlid);
 		smp->status |= IB_SMP_INVALID_FIELD;
 		goto get_only;
 	}
-	smlid &= 0x0000FFFF;
 
 	clientrereg = (pi->clientrereg_subnettimeout &
 			OPA_PI_MASK_CLIENT_REREGISTER);
@@ -1130,12 +1395,16 @@ static int __subn_set_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,
 	ls_old = driver_lstate(ppd);
 
 	ibp->rvp.mkey = pi->mkey;
-	ibp->rvp.gid_prefix = pi->subnet_prefix;
+	if (ibp->rvp.gid_prefix != pi->subnet_prefix) {
+		ibp->rvp.gid_prefix = pi->subnet_prefix;
+		event.event = IB_EVENT_GID_CHANGE;
+		ib_dispatch_event(&event);
+	}
 	ibp->rvp.mkey_lease_period = be16_to_cpu(pi->mkey_lease_period);
 
 	/* Must be a valid unicast LID address. */
 	if ((lid == 0 && ls_old > IB_PORT_INIT) ||
-	    lid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) {
+	     (hfi1_is_16B_mcast(lid))) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		pr_warn("SubnSet(OPA_PortInfo) lid invalid 0x%x\n",
 			lid);
@@ -1148,6 +1417,16 @@ static int __subn_set_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,
 		hfi1_set_lid(ppd, lid, pi->mkeyprotect_lmc & OPA_PI_MASK_LMC);
 		event.event = IB_EVENT_LID_CHANGE;
 		ib_dispatch_event(&event);
+
+		if (HFI1_PORT_GUID_INDEX + 1 < HFI1_GUIDS_PER_PORT) {
+			/* Manufacture GID from LID to support extended
+			 * addresses
+			 */
+			ppd->guids[HFI1_PORT_GUID_INDEX + 1] =
+				be64_to_cpu(OPA_MAKE_ID(lid));
+			event.event = IB_EVENT_GID_CHANGE;
+			ib_dispatch_event(&event);
+		}
 	}
 
 	msl = pi->smsl & OPA_PI_MASK_SMSL;
@@ -1158,7 +1437,7 @@ static int __subn_set_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,
 
 	/* Must be a valid unicast LID address. */
 	if ((smlid == 0 && ls_old > IB_PORT_INIT) ||
-	    smlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) {
+	     (hfi1_is_16B_mcast(smlid))) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		pr_warn("SubnSet(OPA_PortInfo) smlid invalid 0x%x\n", smlid);
 	} else if (smlid != ibp->rvp.sm_lid || msl != ibp->rvp.sm_sl) {
@@ -1166,7 +1445,7 @@ static int __subn_set_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,
 		spin_lock_irqsave(&ibp->rvp.lock, flags);
 		if (ibp->rvp.sm_ah) {
 			if (smlid != ibp->rvp.sm_lid)
-				rdma_ah_set_dlid(&ibp->rvp.sm_ah->attr, smlid);
+				hfi1_modify_qp0_ah(ibp, ibp->rvp.sm_ah, smlid);
 			if (msl != ibp->rvp.sm_sl)
 				rdma_ah_set_sl(&ibp->rvp.sm_ah->attr, msl);
 		}
@@ -1346,7 +1625,8 @@ static int __subn_set_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,
 	if (ret)
 		return ret;
 
-	ret = __subn_get_opa_portinfo(smp, am, data, ibdev, port, resp_len);
+	ret = __subn_get_opa_portinfo(smp, am, data, ibdev, port, resp_len,
+				      max_len);
 
 	/* restore re-reg bit per o14-12.2.1 */
 	pi->clientrereg_subnettimeout |= clientrereg;
@@ -1363,7 +1643,8 @@ static int __subn_set_opa_portinfo(struct opa_smp *smp, u32 am, u8 *data,
 	return ret;
 
 get_only:
-	return __subn_get_opa_portinfo(smp, am, data, ibdev, port, resp_len);
+	return __subn_get_opa_portinfo(smp, am, data, ibdev, port, resp_len,
+				       max_len);
 }
 
 /**
@@ -1424,7 +1705,7 @@ static int set_pkeys(struct hfi1_devdata *dd, u8 port, u16 *pkeys)
 
 static int __subn_set_opa_pkeytable(struct opa_smp *smp, u32 am, u8 *data,
 				    struct ib_device *ibdev, u8 port,
-				    u32 *resp_len)
+				    u32 *resp_len, u32 max_len)
 {
 	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);
 	u32 n_blocks_sent = OPA_AM_NBLK(am);
@@ -1434,6 +1715,7 @@ static int __subn_set_opa_pkeytable(struct opa_smp *smp, u32 am, u8 *data,
 	int i;
 	u16 n_blocks_avail;
 	unsigned npkeys = hfi1_get_npkeys(dd);
+	u32 size = 0;
 
 	if (n_blocks_sent == 0) {
 		pr_warn("OPA Get PKey AM Invalid : P = %d; B = 0x%x; N = 0x%x\n",
@@ -1444,6 +1726,13 @@ static int __subn_set_opa_pkeytable(struct opa_smp *smp, u32 am, u8 *data,
 
 	n_blocks_avail = (u16)(npkeys / OPA_PARTITION_TABLE_BLK_SIZE) + 1;
 
+	size = sizeof(u16) * (n_blocks_sent * OPA_PARTITION_TABLE_BLK_SIZE);
+
+	if (smp_length_check(size, max_len)) {
+		smp->status |= IB_SMP_INVALID_FIELD;
+		return reply((struct ib_mad_hdr *)smp);
+	}
+
 	if (start_block + n_blocks_sent > n_blocks_avail ||
 	    n_blocks_sent > OPA_NUM_PKEY_BLOCKS_PER_SMP) {
 		pr_warn("OPA Set PKey AM Invalid : s 0x%x; req 0x%x; avail 0x%x; blk/smp 0x%lx\n",
@@ -1461,7 +1750,8 @@ static int __subn_set_opa_pkeytable(struct opa_smp *smp, u32 am, u8 *data,
 		return reply((struct ib_mad_hdr *)smp);
 	}
 
-	return __subn_get_opa_pkeytable(smp, am, data, ibdev, port, resp_len);
+	return __subn_get_opa_pkeytable(smp, am, data, ibdev, port, resp_len,
+					max_len);
 }
 
 #define ILLEGAL_VL 12
@@ -1522,14 +1812,14 @@ static int get_sc2vlt_tables(struct hfi1_devdata *dd, void *data)
 
 static int __subn_get_opa_sl_to_sc(struct opa_smp *smp, u32 am, u8 *data,
 				   struct ib_device *ibdev, u8 port,
-				   u32 *resp_len)
+				   u32 *resp_len, u32 max_len)
 {
 	struct hfi1_ibport *ibp = to_iport(ibdev, port);
 	u8 *p = data;
 	size_t size = ARRAY_SIZE(ibp->sl_to_sc); /* == 32 */
 	unsigned i;
 
-	if (am) {
+	if (am || smp_length_check(size, max_len)) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)smp);
 	}
@@ -1545,14 +1835,15 @@ static int __subn_get_opa_sl_to_sc(struct opa_smp *smp, u32 am, u8 *data,
 
 static int __subn_set_opa_sl_to_sc(struct opa_smp *smp, u32 am, u8 *data,
 				   struct ib_device *ibdev, u8 port,
-				   u32 *resp_len)
+				   u32 *resp_len, u32 max_len)
 {
 	struct hfi1_ibport *ibp = to_iport(ibdev, port);
 	u8 *p = data;
+	size_t size = ARRAY_SIZE(ibp->sl_to_sc);
 	int i;
 	u8 sc;
 
-	if (am) {
+	if (am || smp_length_check(size, max_len)) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)smp);
 	}
@@ -1567,19 +1858,20 @@ static int __subn_set_opa_sl_to_sc(struct opa_smp *smp, u32 am, u8 *data,
 		}
 	}
 
-	return __subn_get_opa_sl_to_sc(smp, am, data, ibdev, port, resp_len);
+	return __subn_get_opa_sl_to_sc(smp, am, data, ibdev, port, resp_len,
+				       max_len);
 }
 
 static int __subn_get_opa_sc_to_sl(struct opa_smp *smp, u32 am, u8 *data,
 				   struct ib_device *ibdev, u8 port,
-				   u32 *resp_len)
+				   u32 *resp_len, u32 max_len)
 {
 	struct hfi1_ibport *ibp = to_iport(ibdev, port);
 	u8 *p = data;
 	size_t size = ARRAY_SIZE(ibp->sc_to_sl); /* == 32 */
 	unsigned i;
 
-	if (am) {
+	if (am || smp_length_check(size, max_len)) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)smp);
 	}
@@ -1595,13 +1887,14 @@ static int __subn_get_opa_sc_to_sl(struct opa_smp *smp, u32 am, u8 *data,
 
 static int __subn_set_opa_sc_to_sl(struct opa_smp *smp, u32 am, u8 *data,
 				   struct ib_device *ibdev, u8 port,
-				   u32 *resp_len)
+				   u32 *resp_len, u32 max_len)
 {
 	struct hfi1_ibport *ibp = to_iport(ibdev, port);
+	size_t size = ARRAY_SIZE(ibp->sc_to_sl);
 	u8 *p = data;
 	int i;
 
-	if (am) {
+	if (am || smp_length_check(size, max_len)) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)smp);
 	}
@@ -1609,19 +1902,20 @@ static int __subn_set_opa_sc_to_sl(struct opa_smp *smp, u32 am, u8 *data,
 	for (i = 0; i < ARRAY_SIZE(ibp->sc_to_sl); i++)
 		ibp->sc_to_sl[i] = *p++;
 
-	return __subn_get_opa_sc_to_sl(smp, am, data, ibdev, port, resp_len);
+	return __subn_get_opa_sc_to_sl(smp, am, data, ibdev, port, resp_len,
+				       max_len);
 }
 
 static int __subn_get_opa_sc_to_vlt(struct opa_smp *smp, u32 am, u8 *data,
 				    struct ib_device *ibdev, u8 port,
-				    u32 *resp_len)
+				    u32 *resp_len, u32 max_len)
 {
 	u32 n_blocks = OPA_AM_NBLK(am);
 	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);
 	void *vp = (void *)data;
 	size_t size = 4 * sizeof(u64);
 
-	if (n_blocks != 1) {
+	if (n_blocks != 1 || smp_length_check(size, max_len)) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)smp);
 	}
@@ -1636,7 +1930,7 @@ static int __subn_get_opa_sc_to_vlt(struct opa_smp *smp, u32 am, u8 *data,
 
 static int __subn_set_opa_sc_to_vlt(struct opa_smp *smp, u32 am, u8 *data,
 				    struct ib_device *ibdev, u8 port,
-				    u32 *resp_len)
+				    u32 *resp_len, u32 max_len)
 {
 	u32 n_blocks = OPA_AM_NBLK(am);
 	int async_update = OPA_AM_ASYNC(am);
@@ -1644,8 +1938,15 @@ static int __subn_set_opa_sc_to_vlt(struct opa_smp *smp, u32 am, u8 *data,
 	void *vp = (void *)data;
 	struct hfi1_pportdata *ppd;
 	int lstate;
+	/*
+	 * set_sc2vlt_tables writes the information contained in *data
+	 * to four 64-bit registers SendSC2VLt[0-3]. We need to make
+	 * sure *max_len is not greater than the total size of the four
+	 * SendSC2VLt[0-3] registers.
+	 */
+	size_t size = 4 * sizeof(u64);
 
-	if (n_blocks != 1 || async_update) {
+	if (n_blocks != 1 || async_update || smp_length_check(size, max_len)) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)smp);
 	}
@@ -1665,27 +1966,28 @@ static int __subn_set_opa_sc_to_vlt(struct opa_smp *smp, u32 am, u8 *data,
 
 	set_sc2vlt_tables(dd, vp);
 
-	return __subn_get_opa_sc_to_vlt(smp, am, data, ibdev, port, resp_len);
+	return __subn_get_opa_sc_to_vlt(smp, am, data, ibdev, port, resp_len,
+					max_len);
 }
 
 static int __subn_get_opa_sc_to_vlnt(struct opa_smp *smp, u32 am, u8 *data,
 				     struct ib_device *ibdev, u8 port,
-				     u32 *resp_len)
+				     u32 *resp_len, u32 max_len)
 {
 	u32 n_blocks = OPA_AM_NPORT(am);
 	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);
 	struct hfi1_pportdata *ppd;
 	void *vp = (void *)data;
-	int size;
+	int size = sizeof(struct sc2vlnt);
 
-	if (n_blocks != 1) {
+	if (n_blocks != 1 || smp_length_check(size, max_len)) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)smp);
 	}
 
 	ppd = dd->pport + (port - 1);
 
-	size = fm_get_table(ppd, FM_TBL_SC2VLNT, vp);
+	fm_get_table(ppd, FM_TBL_SC2VLNT, vp);
 
 	if (resp_len)
 		*resp_len += size;
@@ -1695,15 +1997,16 @@ static int __subn_get_opa_sc_to_vlnt(struct opa_smp *smp, u32 am, u8 *data,
 
 static int __subn_set_opa_sc_to_vlnt(struct opa_smp *smp, u32 am, u8 *data,
 				     struct ib_device *ibdev, u8 port,
-				     u32 *resp_len)
+				     u32 *resp_len, u32 max_len)
 {
 	u32 n_blocks = OPA_AM_NPORT(am);
 	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);
 	struct hfi1_pportdata *ppd;
 	void *vp = (void *)data;
 	int lstate;
+	int size = sizeof(struct sc2vlnt);
 
-	if (n_blocks != 1) {
+	if (n_blocks != 1 || smp_length_check(size, max_len)) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)smp);
 	}
@@ -1721,12 +2024,12 @@ static int __subn_set_opa_sc_to_vlnt(struct opa_smp *smp, u32 am, u8 *data,
 	fm_set_table(ppd, FM_TBL_SC2VLNT, vp);
 
 	return __subn_get_opa_sc_to_vlnt(smp, am, data, ibdev, port,
-					 resp_len);
+					 resp_len, max_len);
 }
 
 static int __subn_get_opa_psi(struct opa_smp *smp, u32 am, u8 *data,
 			      struct ib_device *ibdev, u8 port,
-			      u32 *resp_len)
+			      u32 *resp_len, u32 max_len)
 {
 	u32 nports = OPA_AM_NPORT(am);
 	u32 start_of_sm_config = OPA_AM_START_SM_CFG(am);
@@ -1735,7 +2038,7 @@ static int __subn_get_opa_psi(struct opa_smp *smp, u32 am, u8 *data,
 	struct hfi1_pportdata *ppd;
 	struct opa_port_state_info *psi = (struct opa_port_state_info *)data;
 
-	if (nports != 1) {
+	if (nports != 1 || smp_length_check(sizeof(*psi), max_len)) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)smp);
 	}
@@ -1755,7 +2058,7 @@ static int __subn_get_opa_psi(struct opa_smp *smp, u32 am, u8 *data,
 		ppd->offline_disabled_reason;
 
 	psi->port_states.portphysstate_portstate =
-		(hfi1_ibphys_portstate(ppd) << 4) | (lstate & 0xf);
+		(driver_pstate(ppd) << 4) | (lstate & 0xf);
 	psi->link_width_downgrade_tx_active =
 		cpu_to_be16(ppd->link_width_downgrade_tx_active);
 	psi->link_width_downgrade_rx_active =
@@ -1768,7 +2071,7 @@ static int __subn_get_opa_psi(struct opa_smp *smp, u32 am, u8 *data,
 
 static int __subn_set_opa_psi(struct opa_smp *smp, u32 am, u8 *data,
 			      struct ib_device *ibdev, u8 port,
-			      u32 *resp_len)
+			      u32 *resp_len, u32 max_len)
 {
 	u32 nports = OPA_AM_NPORT(am);
 	u32 start_of_sm_config = OPA_AM_START_SM_CFG(am);
@@ -1779,7 +2082,7 @@ static int __subn_set_opa_psi(struct opa_smp *smp, u32 am, u8 *data,
 	struct opa_port_state_info *psi = (struct opa_port_state_info *)data;
 	int ret, invalid = 0;
 
-	if (nports != 1) {
+	if (nports != 1 || smp_length_check(sizeof(*psi), max_len)) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)smp);
 	}
@@ -1809,19 +2112,21 @@ static int __subn_set_opa_psi(struct opa_smp *smp, u32 am, u8 *data,
 	if (invalid)
 		smp->status |= IB_SMP_INVALID_FIELD;
 
-	return __subn_get_opa_psi(smp, am, data, ibdev, port, resp_len);
+	return __subn_get_opa_psi(smp, am, data, ibdev, port, resp_len,
+				  max_len);
 }
 
 static int __subn_get_opa_cable_info(struct opa_smp *smp, u32 am, u8 *data,
 				     struct ib_device *ibdev, u8 port,
-				     u32 *resp_len)
+				     u32 *resp_len, u32 max_len)
 {
 	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);
 	u32 addr = OPA_AM_CI_ADDR(am);
 	u32 len = OPA_AM_CI_LEN(am) + 1;
 	int ret;
 
-	if (dd->pport->port_type != PORT_TYPE_QSFP) {
+	if (dd->pport->port_type != PORT_TYPE_QSFP ||
+	    smp_length_check(len, max_len)) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)smp);
 	}
@@ -1864,21 +2169,22 @@ static int __subn_get_opa_cable_info(struct opa_smp *smp, u32 am, u8 *data,
 }
 
 static int __subn_get_opa_bct(struct opa_smp *smp, u32 am, u8 *data,
-			      struct ib_device *ibdev, u8 port, u32 *resp_len)
+			      struct ib_device *ibdev, u8 port, u32 *resp_len,
+			      u32 max_len)
 {
 	u32 num_ports = OPA_AM_NPORT(am);
 	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);
 	struct hfi1_pportdata *ppd;
 	struct buffer_control *p = (struct buffer_control *)data;
-	int size;
+	int size = sizeof(struct buffer_control);
 
-	if (num_ports != 1) {
+	if (num_ports != 1 || smp_length_check(size, max_len)) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)smp);
 	}
 
 	ppd = dd->pport + (port - 1);
-	size = fm_get_table(ppd, FM_TBL_BUFFER_CONTROL, p);
+	fm_get_table(ppd, FM_TBL_BUFFER_CONTROL, p);
 	trace_bct_get(dd, p);
 	if (resp_len)
 		*resp_len += size;
@@ -1887,14 +2193,15 @@ static int __subn_get_opa_bct(struct opa_smp *smp, u32 am, u8 *data,
 }
 
 static int __subn_set_opa_bct(struct opa_smp *smp, u32 am, u8 *data,
-			      struct ib_device *ibdev, u8 port, u32 *resp_len)
+			      struct ib_device *ibdev, u8 port, u32 *resp_len,
+			      u32 max_len)
 {
 	u32 num_ports = OPA_AM_NPORT(am);
 	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);
 	struct hfi1_pportdata *ppd;
 	struct buffer_control *p = (struct buffer_control *)data;
 
-	if (num_ports != 1) {
+	if (num_ports != 1 || smp_length_check(sizeof(*p), max_len)) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)smp);
 	}
@@ -1905,41 +2212,43 @@ static int __subn_set_opa_bct(struct opa_smp *smp, u32 am, u8 *data,
 		return reply((struct ib_mad_hdr *)smp);
 	}
 
-	return __subn_get_opa_bct(smp, am, data, ibdev, port, resp_len);
+	return __subn_get_opa_bct(smp, am, data, ibdev, port, resp_len,
+				  max_len);
 }
 
 static int __subn_get_opa_vl_arb(struct opa_smp *smp, u32 am, u8 *data,
 				 struct ib_device *ibdev, u8 port,
-				 u32 *resp_len)
+				 u32 *resp_len, u32 max_len)
 {
 	struct hfi1_pportdata *ppd = ppd_from_ibp(to_iport(ibdev, port));
 	u32 num_ports = OPA_AM_NPORT(am);
 	u8 section = (am & 0x00ff0000) >> 16;
 	u8 *p = data;
-	int size = 0;
+	int size = 256;
 
-	if (num_ports != 1) {
+	if (num_ports != 1 || smp_length_check(size, max_len)) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)smp);
 	}
 
 	switch (section) {
 	case OPA_VLARB_LOW_ELEMENTS:
-		size = fm_get_table(ppd, FM_TBL_VL_LOW_ARB, p);
+		fm_get_table(ppd, FM_TBL_VL_LOW_ARB, p);
 		break;
 	case OPA_VLARB_HIGH_ELEMENTS:
-		size = fm_get_table(ppd, FM_TBL_VL_HIGH_ARB, p);
+		fm_get_table(ppd, FM_TBL_VL_HIGH_ARB, p);
 		break;
 	case OPA_VLARB_PREEMPT_ELEMENTS:
-		size = fm_get_table(ppd, FM_TBL_VL_PREEMPT_ELEMS, p);
+		fm_get_table(ppd, FM_TBL_VL_PREEMPT_ELEMS, p);
 		break;
 	case OPA_VLARB_PREEMPT_MATRIX:
-		size = fm_get_table(ppd, FM_TBL_VL_PREEMPT_MATRIX, p);
+		fm_get_table(ppd, FM_TBL_VL_PREEMPT_MATRIX, p);
 		break;
 	default:
 		pr_warn("OPA SubnGet(VL Arb) AM Invalid : 0x%x\n",
 			be32_to_cpu(smp->attr_mod));
 		smp->status |= IB_SMP_INVALID_FIELD;
+		size = 0;
 		break;
 	}
 
@@ -1951,14 +2260,15 @@ static int __subn_get_opa_vl_arb(struct opa_smp *smp, u32 am, u8 *data,
 
 static int __subn_set_opa_vl_arb(struct opa_smp *smp, u32 am, u8 *data,
 				 struct ib_device *ibdev, u8 port,
-				 u32 *resp_len)
+				 u32 *resp_len, u32 max_len)
 {
 	struct hfi1_pportdata *ppd = ppd_from_ibp(to_iport(ibdev, port));
 	u32 num_ports = OPA_AM_NPORT(am);
 	u8 section = (am & 0x00ff0000) >> 16;
 	u8 *p = data;
+	int size = 256;
 
-	if (num_ports != 1) {
+	if (num_ports != 1 || smp_length_check(size, max_len)) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)smp);
 	}
@@ -1986,7 +2296,8 @@ static int __subn_set_opa_vl_arb(struct opa_smp *smp, u32 am, u8 *data,
 		break;
 	}
 
-	return __subn_get_opa_vl_arb(smp, am, data, ibdev, port, resp_len);
+	return __subn_get_opa_vl_arb(smp, am, data, ibdev, port, resp_len,
+				     max_len);
 }
 
 struct opa_pma_mad {
@@ -3282,13 +3593,18 @@ struct opa_congestion_info_attr {
 
 static int __subn_get_opa_cong_info(struct opa_smp *smp, u32 am, u8 *data,
 				    struct ib_device *ibdev, u8 port,
-				    u32 *resp_len)
+				    u32 *resp_len, u32 max_len)
 {
 	struct opa_congestion_info_attr *p =
 		(struct opa_congestion_info_attr *)data;
 	struct hfi1_ibport *ibp = to_iport(ibdev, port);
 	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
 
+	if (smp_length_check(sizeof(*p), max_len)) {
+		smp->status |= IB_SMP_INVALID_FIELD;
+		return reply((struct ib_mad_hdr *)smp);
+	}
+
 	p->congestion_info = 0;
 	p->control_table_cap = ppd->cc_max_table_entries;
 	p->congestion_log_length = OPA_CONG_LOG_ELEMS;
@@ -3301,7 +3617,7 @@ static int __subn_get_opa_cong_info(struct opa_smp *smp, u32 am, u8 *data,
 
 static int __subn_get_opa_cong_setting(struct opa_smp *smp, u32 am,
 				       u8 *data, struct ib_device *ibdev,
-				       u8 port, u32 *resp_len)
+				       u8 port, u32 *resp_len, u32 max_len)
 {
 	int i;
 	struct opa_congestion_setting_attr *p =
@@ -3311,6 +3627,11 @@ static int __subn_get_opa_cong_setting(struct opa_smp *smp, u32 am,
 	struct opa_congestion_setting_entry_shadow *entries;
 	struct cc_state *cc_state;
 
+	if (smp_length_check(sizeof(*p), max_len)) {
+		smp->status |= IB_SMP_INVALID_FIELD;
+		return reply((struct ib_mad_hdr *)smp);
+	}
+
 	rcu_read_lock();
 
 	cc_state = get_cc_state(ppd);
@@ -3385,7 +3706,7 @@ static void apply_cc_state(struct hfi1_pportdata *ppd)
 
 static int __subn_set_opa_cong_setting(struct opa_smp *smp, u32 am, u8 *data,
 				       struct ib_device *ibdev, u8 port,
-				       u32 *resp_len)
+				       u32 *resp_len, u32 max_len)
 {
 	struct opa_congestion_setting_attr *p =
 		(struct opa_congestion_setting_attr *)data;
@@ -3394,6 +3715,11 @@ static int __subn_set_opa_cong_setting(struct opa_smp *smp, u32 am, u8 *data,
 	struct opa_congestion_setting_entry_shadow *entries;
 	int i;
 
+	if (smp_length_check(sizeof(*p), max_len)) {
+		smp->status |= IB_SMP_INVALID_FIELD;
+		return reply((struct ib_mad_hdr *)smp);
+	}
+
 	/*
 	 * Save details from packet into the ppd.  Hold the cc_state_lock so
 	 * our information is consistent with anyone trying to apply the state.
@@ -3415,12 +3741,12 @@ static int __subn_set_opa_cong_setting(struct opa_smp *smp, u32 am, u8 *data,
 	apply_cc_state(ppd);
 
 	return __subn_get_opa_cong_setting(smp, am, data, ibdev, port,
-					   resp_len);
+					   resp_len, max_len);
 }
 
 static int __subn_get_opa_hfi1_cong_log(struct opa_smp *smp, u32 am,
 					u8 *data, struct ib_device *ibdev,
-					u8 port, u32 *resp_len)
+					u8 port, u32 *resp_len, u32 max_len)
 {
 	struct hfi1_ibport *ibp = to_iport(ibdev, port);
 	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
@@ -3428,7 +3754,7 @@ static int __subn_get_opa_hfi1_cong_log(struct opa_smp *smp, u32 am,
 	s64 ts;
 	int i;
 
-	if (am != 0) {
+	if (am || smp_length_check(sizeof(*cong_log), max_len)) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)smp);
 	}
@@ -3486,7 +3812,7 @@ static int __subn_get_opa_hfi1_cong_log(struct opa_smp *smp, u32 am,
 
 static int __subn_get_opa_cc_table(struct opa_smp *smp, u32 am, u8 *data,
 				   struct ib_device *ibdev, u8 port,
-				   u32 *resp_len)
+				   u32 *resp_len, u32 max_len)
 {
 	struct ib_cc_table_attr *cc_table_attr =
 		(struct ib_cc_table_attr *)data;
@@ -3498,9 +3824,10 @@ static int __subn_get_opa_cc_table(struct opa_smp *smp, u32 am, u8 *data,
 	int i, j;
 	u32 sentry, eentry;
 	struct cc_state *cc_state;
+	u32 size = sizeof(u16) * (IB_CCT_ENTRIES * n_blocks + 1);
 
 	/* sanity check n_blocks, start_block */
-	if (n_blocks == 0 ||
+	if (n_blocks == 0 || smp_length_check(size, max_len) ||
 	    start_block + n_blocks > ppd->cc_max_table_entries) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)smp);
@@ -3530,14 +3857,14 @@ static int __subn_get_opa_cc_table(struct opa_smp *smp, u32 am, u8 *data,
 	rcu_read_unlock();
 
 	if (resp_len)
-		*resp_len += sizeof(u16) * (IB_CCT_ENTRIES * n_blocks + 1);
+		*resp_len += size;
 
 	return reply((struct ib_mad_hdr *)smp);
 }
 
 static int __subn_set_opa_cc_table(struct opa_smp *smp, u32 am, u8 *data,
 				   struct ib_device *ibdev, u8 port,
-				   u32 *resp_len)
+				   u32 *resp_len, u32 max_len)
 {
 	struct ib_cc_table_attr *p = (struct ib_cc_table_attr *)data;
 	struct hfi1_ibport *ibp = to_iport(ibdev, port);
@@ -3548,9 +3875,10 @@ static int __subn_set_opa_cc_table(struct opa_smp *smp, u32 am, u8 *data,
 	int i, j;
 	u32 sentry, eentry;
 	u16 ccti_limit;
+	u32 size = sizeof(u16) * (IB_CCT_ENTRIES * n_blocks + 1);
 
 	/* sanity check n_blocks, start_block */
-	if (n_blocks == 0 ||
+	if (n_blocks == 0 || smp_length_check(size, max_len) ||
 	    start_block + n_blocks > ppd->cc_max_table_entries) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)smp);
@@ -3581,7 +3909,8 @@ static int __subn_set_opa_cc_table(struct opa_smp *smp, u32 am, u8 *data,
 	/* now apply the information */
 	apply_cc_state(ppd);
 
-	return __subn_get_opa_cc_table(smp, am, data, ibdev, port, resp_len);
+	return __subn_get_opa_cc_table(smp, am, data, ibdev, port, resp_len,
+				       max_len);
 }
 
 struct opa_led_info {
@@ -3594,7 +3923,7 @@ struct opa_led_info {
 
 static int __subn_get_opa_led_info(struct opa_smp *smp, u32 am, u8 *data,
 				   struct ib_device *ibdev, u8 port,
-				   u32 *resp_len)
+				   u32 *resp_len, u32 max_len)
 {
 	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);
 	struct hfi1_pportdata *ppd = dd->pport;
@@ -3602,7 +3931,7 @@ static int __subn_get_opa_led_info(struct opa_smp *smp, u32 am, u8 *data,
 	u32 nport = OPA_AM_NPORT(am);
 	u32 is_beaconing_active;
 
-	if (nport != 1) {
+	if (nport != 1 || smp_length_check(sizeof(*p), max_len)) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)smp);
 	}
@@ -3624,14 +3953,14 @@ static int __subn_get_opa_led_info(struct opa_smp *smp, u32 am, u8 *data,
 
 static int __subn_set_opa_led_info(struct opa_smp *smp, u32 am, u8 *data,
 				   struct ib_device *ibdev, u8 port,
-				   u32 *resp_len)
+				   u32 *resp_len, u32 max_len)
 {
 	struct hfi1_devdata *dd = dd_from_ibdev(ibdev);
 	struct opa_led_info *p = (struct opa_led_info *)data;
 	u32 nport = OPA_AM_NPORT(am);
 	int on = !!(be32_to_cpu(p->rsvd_led_mask) & OPA_LED_MASK);
 
-	if (nport != 1) {
+	if (nport != 1 || smp_length_check(sizeof(*p), max_len)) {
 		smp->status |= IB_SMP_INVALID_FIELD;
 		return reply((struct ib_mad_hdr *)smp);
 	}
@@ -3641,12 +3970,13 @@ static int __subn_set_opa_led_info(struct opa_smp *smp, u32 am, u8 *data,
 	else
 		shutdown_led_override(dd->pport);
 
-	return __subn_get_opa_led_info(smp, am, data, ibdev, port, resp_len);
+	return __subn_get_opa_led_info(smp, am, data, ibdev, port, resp_len,
+				       max_len);
 }
 
 static int subn_get_opa_sma(__be16 attr_id, struct opa_smp *smp, u32 am,
 			    u8 *data, struct ib_device *ibdev, u8 port,
-			    u32 *resp_len)
+			    u32 *resp_len, u32 max_len)
 {
 	int ret;
 	struct hfi1_ibport *ibp = to_iport(ibdev, port);
@@ -3654,71 +3984,71 @@ static int subn_get_opa_sma(__be16 attr_id, struct opa_smp *smp, u32 am,
 	switch (attr_id) {
 	case IB_SMP_ATTR_NODE_DESC:
 		ret = __subn_get_opa_nodedesc(smp, am, data, ibdev, port,
-					      resp_len);
+					      resp_len, max_len);
 		break;
 	case IB_SMP_ATTR_NODE_INFO:
 		ret = __subn_get_opa_nodeinfo(smp, am, data, ibdev, port,
-					      resp_len);
+					      resp_len, max_len);
 		break;
 	case IB_SMP_ATTR_PORT_INFO:
 		ret = __subn_get_opa_portinfo(smp, am, data, ibdev, port,
-					      resp_len);
+					      resp_len, max_len);
 		break;
 	case IB_SMP_ATTR_PKEY_TABLE:
 		ret = __subn_get_opa_pkeytable(smp, am, data, ibdev, port,
-					       resp_len);
+					       resp_len, max_len);
 		break;
 	case OPA_ATTRIB_ID_SL_TO_SC_MAP:
 		ret = __subn_get_opa_sl_to_sc(smp, am, data, ibdev, port,
-					      resp_len);
+					      resp_len, max_len);
 		break;
 	case OPA_ATTRIB_ID_SC_TO_SL_MAP:
 		ret = __subn_get_opa_sc_to_sl(smp, am, data, ibdev, port,
-					      resp_len);
+					      resp_len, max_len);
 		break;
 	case OPA_ATTRIB_ID_SC_TO_VLT_MAP:
 		ret = __subn_get_opa_sc_to_vlt(smp, am, data, ibdev, port,
-					       resp_len);
+					       resp_len, max_len);
 		break;
 	case OPA_ATTRIB_ID_SC_TO_VLNT_MAP:
 		ret = __subn_get_opa_sc_to_vlnt(smp, am, data, ibdev, port,
-						resp_len);
+						resp_len, max_len);
 		break;
 	case OPA_ATTRIB_ID_PORT_STATE_INFO:
 		ret = __subn_get_opa_psi(smp, am, data, ibdev, port,
-					 resp_len);
+					 resp_len, max_len);
 		break;
 	case OPA_ATTRIB_ID_BUFFER_CONTROL_TABLE:
 		ret = __subn_get_opa_bct(smp, am, data, ibdev, port,
-					 resp_len);
+					 resp_len, max_len);
 		break;
 	case OPA_ATTRIB_ID_CABLE_INFO:
 		ret = __subn_get_opa_cable_info(smp, am, data, ibdev, port,
-						resp_len);
+						resp_len, max_len);
 		break;
 	case IB_SMP_ATTR_VL_ARB_TABLE:
 		ret = __subn_get_opa_vl_arb(smp, am, data, ibdev, port,
-					    resp_len);
+					    resp_len, max_len);
 		break;
 	case OPA_ATTRIB_ID_CONGESTION_INFO:
 		ret = __subn_get_opa_cong_info(smp, am, data, ibdev, port,
-					       resp_len);
+					       resp_len, max_len);
 		break;
 	case OPA_ATTRIB_ID_HFI_CONGESTION_SETTING:
 		ret = __subn_get_opa_cong_setting(smp, am, data, ibdev,
-						  port, resp_len);
+						  port, resp_len, max_len);
 		break;
 	case OPA_ATTRIB_ID_HFI_CONGESTION_LOG:
 		ret = __subn_get_opa_hfi1_cong_log(smp, am, data, ibdev,
-						   port, resp_len);
+						   port, resp_len, max_len);
 		break;
 	case OPA_ATTRIB_ID_CONGESTION_CONTROL_TABLE:
 		ret = __subn_get_opa_cc_table(smp, am, data, ibdev, port,
-					      resp_len);
+					      resp_len, max_len);
 		break;
 	case IB_SMP_ATTR_LED_INFO:
 		ret = __subn_get_opa_led_info(smp, am, data, ibdev, port,
-					      resp_len);
+					      resp_len, max_len);
 		break;
 	case IB_SMP_ATTR_SM_INFO:
 		if (ibp->rvp.port_cap_flags & IB_PORT_SM_DISABLED)
@@ -3736,7 +4066,7 @@ static int subn_get_opa_sma(__be16 attr_id, struct opa_smp *smp, u32 am,
 
 static int subn_set_opa_sma(__be16 attr_id, struct opa_smp *smp, u32 am,
 			    u8 *data, struct ib_device *ibdev, u8 port,
-			    u32 *resp_len)
+			    u32 *resp_len, u32 max_len)
 {
 	int ret;
 	struct hfi1_ibport *ibp = to_iport(ibdev, port);
@@ -3744,51 +4074,51 @@ static int subn_set_opa_sma(__be16 attr_id, struct opa_smp *smp, u32 am,
 	switch (attr_id) {
 	case IB_SMP_ATTR_PORT_INFO:
 		ret = __subn_set_opa_portinfo(smp, am, data, ibdev, port,
-					      resp_len);
+					      resp_len, max_len);
 		break;
 	case IB_SMP_ATTR_PKEY_TABLE:
 		ret = __subn_set_opa_pkeytable(smp, am, data, ibdev, port,
-					       resp_len);
+					       resp_len, max_len);
 		break;
 	case OPA_ATTRIB_ID_SL_TO_SC_MAP:
 		ret = __subn_set_opa_sl_to_sc(smp, am, data, ibdev, port,
-					      resp_len);
+					      resp_len, max_len);
 		break;
 	case OPA_ATTRIB_ID_SC_TO_SL_MAP:
 		ret = __subn_set_opa_sc_to_sl(smp, am, data, ibdev, port,
-					      resp_len);
+					      resp_len, max_len);
 		break;
 	case OPA_ATTRIB_ID_SC_TO_VLT_MAP:
 		ret = __subn_set_opa_sc_to_vlt(smp, am, data, ibdev, port,
-					       resp_len);
+					       resp_len, max_len);
 		break;
 	case OPA_ATTRIB_ID_SC_TO_VLNT_MAP:
 		ret = __subn_set_opa_sc_to_vlnt(smp, am, data, ibdev, port,
-						resp_len);
+						resp_len, max_len);
 		break;
 	case OPA_ATTRIB_ID_PORT_STATE_INFO:
 		ret = __subn_set_opa_psi(smp, am, data, ibdev, port,
-					 resp_len);
+					 resp_len, max_len);
 		break;
 	case OPA_ATTRIB_ID_BUFFER_CONTROL_TABLE:
 		ret = __subn_set_opa_bct(smp, am, data, ibdev, port,
-					 resp_len);
+					 resp_len, max_len);
 		break;
 	case IB_SMP_ATTR_VL_ARB_TABLE:
 		ret = __subn_set_opa_vl_arb(smp, am, data, ibdev, port,
-					    resp_len);
+					    resp_len, max_len);
 		break;
 	case OPA_ATTRIB_ID_HFI_CONGESTION_SETTING:
 		ret = __subn_set_opa_cong_setting(smp, am, data, ibdev,
-						  port, resp_len);
+						  port, resp_len, max_len);
 		break;
 	case OPA_ATTRIB_ID_CONGESTION_CONTROL_TABLE:
 		ret = __subn_set_opa_cc_table(smp, am, data, ibdev, port,
-					      resp_len);
+					      resp_len, max_len);
 		break;
 	case IB_SMP_ATTR_LED_INFO:
 		ret = __subn_set_opa_led_info(smp, am, data, ibdev, port,
-					      resp_len);
+					      resp_len, max_len);
 		break;
 	case IB_SMP_ATTR_SM_INFO:
 		if (ibp->rvp.port_cap_flags & IB_PORT_SM_DISABLED)
@@ -3844,7 +4174,10 @@ static int subn_get_opa_aggregate(struct opa_smp *smp,
 		memset(next_smp + sizeof(*agg), 0, agg_data_len);
 
 		(void)subn_get_opa_sma(agg->attr_id, smp, am, agg->data,
-					ibdev, port, NULL);
+				       ibdev, port, NULL, (u32)agg_data_len);
+
+		if (smp->status & IB_SMP_INVALID_FIELD)
+			break;
 		if (smp->status & ~IB_SMP_DIRECTION) {
 			set_aggr_error(agg);
 			return reply((struct ib_mad_hdr *)smp);
@@ -3887,7 +4220,9 @@ static int subn_set_opa_aggregate(struct opa_smp *smp,
 		}
 
 		(void)subn_set_opa_sma(agg->attr_id, smp, am, agg->data,
-					ibdev, port, NULL);
+				       ibdev, port, NULL, (u32)agg_data_len);
+		if (smp->status & IB_SMP_INVALID_FIELD)
+			break;
 		if (smp->status & ~IB_SMP_DIRECTION) {
 			set_aggr_error(agg);
 			return reply((struct ib_mad_hdr *)smp);
@@ -3958,7 +4293,7 @@ static int opa_local_smp_check(struct hfi1_ibport *ibp,
 			       const struct ib_wc *in_wc)
 {
 	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
-	u16 slid = in_wc->slid;
+	u16 slid = ib_lid_cpu16(in_wc->slid);
 	u16 pkey;
 
 	if (in_wc->pkey_index >= ARRAY_SIZE(ppd->pkeys))
@@ -3997,12 +4332,13 @@ static int process_subn_opa(struct ib_device *ibdev, int mad_flags,
 	struct opa_smp *smp = (struct opa_smp *)out_mad;
 	struct hfi1_ibport *ibp = to_iport(ibdev, port);
 	u8 *data;
-	u32 am;
+	u32 am, data_size;
 	__be16 attr_id;
 	int ret;
 
 	*out_mad = *in_mad;
 	data = opa_get_smp_data(smp);
+	data_size = (u32)opa_get_smp_data_size(smp);
 
 	am = be32_to_cpu(smp->attr_mod);
 	attr_id = smp->attr_id;
@@ -4046,7 +4382,8 @@ static int process_subn_opa(struct ib_device *ibdev, int mad_flags,
 		default:
 			clear_opa_smp_data(smp);
 			ret = subn_get_opa_sma(attr_id, smp, am, data,
-					       ibdev, port, resp_len);
+					       ibdev, port, resp_len,
+					       data_size);
 			break;
 		case OPA_ATTRIB_ID_AGGREGATE:
 			ret = subn_get_opa_aggregate(smp, ibdev, port,
@@ -4058,7 +4395,8 @@ static int process_subn_opa(struct ib_device *ibdev, int mad_flags,
 		switch (attr_id) {
 		default:
 			ret = subn_set_opa_sma(attr_id, smp, am, data,
-					       ibdev, port, resp_len);
+					       ibdev, port, resp_len,
+					       data_size);
 			break;
 		case OPA_ATTRIB_ID_AGGREGATE:
 			ret = subn_set_opa_aggregate(smp, ibdev, port,
@@ -4077,6 +4415,11 @@ static int process_subn_opa(struct ib_device *ibdev, int mad_flags,
 		 */
 		ret = IB_MAD_RESULT_SUCCESS;
 		break;
+	case IB_MGMT_METHOD_TRAP_REPRESS:
+		subn_handle_opa_trap_repress(ibp, smp);
+		/* Always successful */
+		ret = IB_MAD_RESULT_SUCCESS;
+		break;
 	default:
 		smp->status |= IB_SMP_UNSUP_METHOD;
 		ret = reply((struct ib_mad_hdr *)smp);
diff --git a/drivers/infiniband/hw/hfi1/mad.h b/drivers/infiniband/hw/hfi1/mad.h
index 5aa3fd1be653..4c1245072093 100644
--- a/drivers/infiniband/hw/hfi1/mad.h
+++ b/drivers/infiniband/hw/hfi1/mad.h
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015, 2016 Intel Corporation.
+ * Copyright(c) 2015 - 2017 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -115,7 +115,7 @@ struct opa_mad_notice_attr {
 			__be32	lid;		/* LID where change occurred */
 			__be32	new_cap_mask;	/* new capability mask */
 			__be16	reserved2;
-			__be16	cap_mask;
+			__be16	cap_mask3;
 			__be16	change_flags;	/* low 4 bits only */
 		} __packed ntc_144;
 
@@ -428,5 +428,6 @@ struct sc2vlnt {
 		    COUNTER_MASK(1, 4))
 
 void hfi1_event_pkey_change(struct hfi1_devdata *dd, u8 port);
+void hfi1_handle_trap_timer(unsigned long data);
 
 #endif				/* _HFI1_MAD_H */
diff --git a/drivers/infiniband/hw/hfi1/mmu_rb.c b/drivers/infiniband/hw/hfi1/mmu_rb.c
index ccbf52c8ff6f..175002c046ed 100644
--- a/drivers/infiniband/hw/hfi1/mmu_rb.c
+++ b/drivers/infiniband/hw/hfi1/mmu_rb.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2016 Intel Corporation.
+ * Copyright(c) 2016 - 2017 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -54,7 +54,7 @@
 
 struct mmu_rb_handler {
 	struct mmu_notifier mn;
-	struct rb_root root;
+	struct rb_root_cached root;
 	void *ops_arg;
 	spinlock_t lock;        /* protect the RB tree */
 	struct mmu_rb_ops *ops;
@@ -67,8 +67,6 @@ struct mmu_rb_handler {
 
 static unsigned long mmu_node_start(struct mmu_rb_node *);
 static unsigned long mmu_node_last(struct mmu_rb_node *);
-static inline void mmu_notifier_page(struct mmu_notifier *, struct mm_struct *,
-				     unsigned long);
 static inline void mmu_notifier_range_start(struct mmu_notifier *,
 					    struct mm_struct *,
 					    unsigned long, unsigned long);
@@ -82,7 +80,6 @@ static void do_remove(struct mmu_rb_handler *handler,
 static void handle_remove(struct work_struct *work);
 
 static const struct mmu_notifier_ops mn_opts = {
-	.invalidate_page = mmu_notifier_page,
 	.invalidate_range_start = mmu_notifier_range_start,
 };
 
@@ -111,7 +108,7 @@ int hfi1_mmu_rb_register(void *ops_arg, struct mm_struct *mm,
 	if (!handlr)
 		return -ENOMEM;
 
-	handlr->root = RB_ROOT;
+	handlr->root = RB_ROOT_CACHED;
 	handlr->ops = ops;
 	handlr->ops_arg = ops_arg;
 	INIT_HLIST_NODE(&handlr->mn.hlist);
@@ -152,9 +149,9 @@ void hfi1_mmu_rb_unregister(struct mmu_rb_handler *handler)
 	INIT_LIST_HEAD(&del_list);
 
 	spin_lock_irqsave(&handler->lock, flags);
-	while ((node = rb_first(&handler->root))) {
+	while ((node = rb_first_cached(&handler->root))) {
 		rbnode = rb_entry(node, struct mmu_rb_node, node);
-		rb_erase(node, &handler->root);
+		rb_erase_cached(node, &handler->root);
 		/* move from LRU list to delete list */
 		list_move(&rbnode->list, &del_list);
 	}
@@ -172,9 +169,8 @@ int hfi1_mmu_rb_insert(struct mmu_rb_handler *handler,
 	unsigned long flags;
 	int ret = 0;
 
+	trace_hfi1_mmu_rb_insert(mnode->addr, mnode->len);
 	spin_lock_irqsave(&handler->lock, flags);
-	hfi1_cdbg(MMU, "Inserting node addr 0x%llx, len %u", mnode->addr,
-		  mnode->len);
 	node = __mmu_rb_search(handler, mnode->addr, mnode->len);
 	if (node) {
 		ret = -EINVAL;
@@ -200,7 +196,7 @@ static struct mmu_rb_node *__mmu_rb_search(struct mmu_rb_handler *handler,
 {
 	struct mmu_rb_node *node = NULL;
 
-	hfi1_cdbg(MMU, "Searching for addr 0x%llx, len %u", addr, len);
+	trace_hfi1_mmu_rb_search(addr, len);
 	if (!handler->ops->filter) {
 		node = __mmu_int_rb_iter_first(&handler->root, addr,
 					       (addr + len) - 1);
@@ -217,21 +213,27 @@ static struct mmu_rb_node *__mmu_rb_search(struct mmu_rb_handler *handler,
 	return node;
 }
 
-struct mmu_rb_node *hfi1_mmu_rb_extract(struct mmu_rb_handler *handler,
-					unsigned long addr, unsigned long len)
+bool hfi1_mmu_rb_remove_unless_exact(struct mmu_rb_handler *handler,
+				     unsigned long addr, unsigned long len,
+				     struct mmu_rb_node **rb_node)
 {
 	struct mmu_rb_node *node;
 	unsigned long flags;
+	bool ret = false;
 
 	spin_lock_irqsave(&handler->lock, flags);
 	node = __mmu_rb_search(handler, addr, len);
 	if (node) {
+		if (node->addr == addr && node->len == len)
+			goto unlock;
 		__mmu_int_rb_remove(node, &handler->root);
 		list_del(&node->list); /* remove from LRU list */
+		ret = true;
 	}
+unlock:
 	spin_unlock_irqrestore(&handler->lock, flags);
-
-	return node;
+	*rb_node = node;
+	return ret;
 }
 
 void hfi1_mmu_rb_evict(struct mmu_rb_handler *handler, void *evict_arg)
@@ -275,8 +277,7 @@ void hfi1_mmu_rb_remove(struct mmu_rb_handler *handler,
 	unsigned long flags;
 
 	/* Validity of handler and node pointers has been checked by caller. */
-	hfi1_cdbg(MMU, "Removing node addr 0x%llx, len %u", node->addr,
-		  node->len);
+	trace_hfi1_mmu_rb_remove(node->addr, node->len);
 	spin_lock_irqsave(&handler->lock, flags);
 	__mmu_int_rb_remove(node, &handler->root);
 	list_del(&node->list); /* remove from LRU list */
@@ -285,12 +286,6 @@ void hfi1_mmu_rb_remove(struct mmu_rb_handler *handler,
 	handler->ops->remove(handler->ops_arg, node);
 }
 
-static inline void mmu_notifier_page(struct mmu_notifier *mn,
-				     struct mm_struct *mm, unsigned long addr)
-{
-	mmu_notifier_mem_invalidate(mn, mm, addr, addr + PAGE_SIZE);
-}
-
 static inline void mmu_notifier_range_start(struct mmu_notifier *mn,
 					    struct mm_struct *mm,
 					    unsigned long start,
@@ -305,7 +300,7 @@ static void mmu_notifier_mem_invalidate(struct mmu_notifier *mn,
 {
 	struct mmu_rb_handler *handler =
 		container_of(mn, struct mmu_rb_handler, mn);
-	struct rb_root *root = &handler->root;
+	struct rb_root_cached *root = &handler->root;
 	struct mmu_rb_node *node, *ptr = NULL;
 	unsigned long flags;
 	bool added = false;
@@ -315,8 +310,7 @@ static void mmu_notifier_mem_invalidate(struct mmu_notifier *mn,
 	     node; node = ptr) {
 		/* Guard against node removal. */
 		ptr = __mmu_int_rb_iter_next(node, start, end - 1);
-		hfi1_cdbg(MMU, "Invalidating node addr 0x%llx, len %u",
-			  node->addr, node->len);
+		trace_hfi1_mmu_mem_invalidate(node->addr, node->len);
 		if (handler->ops->invalidate(handler->ops_arg, node)) {
 			__mmu_int_rb_remove(node, root);
 			/* move from LRU list to delete list */
diff --git a/drivers/infiniband/hw/hfi1/mmu_rb.h b/drivers/infiniband/hw/hfi1/mmu_rb.h
index 754f6ebf13fb..f04cec1e99d1 100644
--- a/drivers/infiniband/hw/hfi1/mmu_rb.h
+++ b/drivers/infiniband/hw/hfi1/mmu_rb.h
@@ -81,7 +81,8 @@ int hfi1_mmu_rb_insert(struct mmu_rb_handler *handler,
 void hfi1_mmu_rb_evict(struct mmu_rb_handler *handler, void *evict_arg);
 void hfi1_mmu_rb_remove(struct mmu_rb_handler *handler,
 			struct mmu_rb_node *mnode);
-struct mmu_rb_node *hfi1_mmu_rb_extract(struct mmu_rb_handler *handler,
-					unsigned long addr, unsigned long len);
+bool hfi1_mmu_rb_remove_unless_exact(struct mmu_rb_handler *handler,
+				     unsigned long addr, unsigned long len,
+				     struct mmu_rb_node **rb_node);
 
 #endif /* _HFI1_MMU_RB_H */
diff --git a/drivers/infiniband/hw/hfi1/opa_compat.h b/drivers/infiniband/hw/hfi1/opa_compat.h
index 6ef3c1cbdcd7..774215b95df5 100644
--- a/drivers/infiniband/hw/hfi1/opa_compat.h
+++ b/drivers/infiniband/hw/hfi1/opa_compat.h
@@ -84,7 +84,8 @@ static inline u8 port_states_to_phys_state(struct opa_port_states *ps)
 /*
  * OPA port physical states
  * IB Volume 1, Table 146 PortInfo/IB Volume 2 Section 5.4.2(1) PortPhysState
- * values.
+ * values are the same in OmniPath Architecture. OPA leverages some of the same
+ * concepts as InfiniBand, but has a few other states as well.
  *
  * When writing, only values 0-3 are valid, other values are ignored.
  * When reading, 0 is reserved.
@@ -92,6 +93,8 @@ static inline u8 port_states_to_phys_state(struct opa_port_states *ps)
  * Returned by the ibphys_portstate() routine.
  */
 enum opa_port_phys_state {
+	/* Values 0-7 have the same meaning in OPA as in InfiniBand. */
+
 	IB_PORTPHYSSTATE_NOP = 0,
 	/* 1 is reserved */
 	IB_PORTPHYSSTATE_POLLING = 2,
@@ -101,9 +104,23 @@ enum opa_port_phys_state {
 	IB_PORTPHYSSTATE_LINK_ERROR_RECOVERY = 6,
 	IB_PORTPHYSSTATE_PHY_TEST = 7,
 	/* 8 is reserved */
+
+	/*
+	 * Offline: Port is quiet (transmitters disabled) due to lack of
+	 * physical media, unsupported media, or transition between link up
+	 * and next link up attempt
+	 */
 	OPA_PORTPHYSSTATE_OFFLINE = 9,
-	OPA_PORTPHYSSTATE_GANGED = 10,
+
+	/* 10 is reserved */
+
+	/*
+	 * Phy_Test: Specific test patterns are transmitted, and receiver BER
+	 * can be monitored. This facilitates signal integrity testing for the
+	 * physical layer of the port.
+	 */
 	OPA_PORTPHYSSTATE_TEST = 11,
+
 	OPA_PORTPHYSSTATE_MAX = 11,
 	/* values 12-15 are reserved/ignored */
 };
diff --git a/drivers/infiniband/hw/hfi1/pcie.c b/drivers/infiniband/hw/hfi1/pcie.c
index 6a9f6f9819e1..82447b7cdda1 100644
--- a/drivers/infiniband/hw/hfi1/pcie.c
+++ b/drivers/infiniband/hw/hfi1/pcie.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015, 2016 Intel Corporation.
+ * Copyright(c) 2015 - 2017 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -68,7 +68,7 @@
 /*
  * Code to adjust PCIe capabilities.
  */
-static void tune_pcie_caps(struct hfi1_devdata *);
+static int tune_pcie_caps(struct hfi1_devdata *);
 
 /*
  * Do all the common PCIe setup and initialization.
@@ -161,6 +161,7 @@ int hfi1_pcie_ddinit(struct hfi1_devdata *dd, struct pci_dev *pdev)
 {
 	unsigned long len;
 	resource_size_t addr;
+	int ret = 0;
 
 	dd->pcidev = pdev;
 	pci_set_drvdata(pdev, dd);
@@ -179,47 +180,54 @@ int hfi1_pcie_ddinit(struct hfi1_devdata *dd, struct pci_dev *pdev)
 		return -EINVAL;
 	}
 
-	dd->kregbase = ioremap_nocache(addr, TXE_PIO_SEND);
-	if (!dd->kregbase)
+	dd->kregbase1 = ioremap_nocache(addr, RCV_ARRAY);
+	if (!dd->kregbase1) {
+		dd_dev_err(dd, "UC mapping of kregbase1 failed\n");
 		return -ENOMEM;
+	}
+	dd_dev_info(dd, "UC base1: %p for %x\n", dd->kregbase1, RCV_ARRAY);
+	dd->chip_rcv_array_count = readq(dd->kregbase1 + RCV_ARRAY_CNT);
+	dd_dev_info(dd, "RcvArray count: %u\n", dd->chip_rcv_array_count);
+	dd->base2_start  = RCV_ARRAY + dd->chip_rcv_array_count * 8;
+
+	dd->kregbase2 = ioremap_nocache(
+		addr + dd->base2_start,
+		TXE_PIO_SEND - dd->base2_start);
+	if (!dd->kregbase2) {
+		dd_dev_err(dd, "UC mapping of kregbase2 failed\n");
+		goto nomem;
+	}
+	dd_dev_info(dd, "UC base2: %p for %x\n", dd->kregbase2,
+		    TXE_PIO_SEND - dd->base2_start);
 
 	dd->piobase = ioremap_wc(addr + TXE_PIO_SEND, TXE_PIO_SIZE);
 	if (!dd->piobase) {
-		iounmap(dd->kregbase);
-		return -ENOMEM;
+		dd_dev_err(dd, "WC mapping of send buffers failed\n");
+		goto nomem;
 	}
+	dd_dev_info(dd, "WC piobase: %p\n for %x", dd->piobase, TXE_PIO_SIZE);
 
-	dd->flags |= HFI1_PRESENT;	/* now register routines work */
-
-	dd->kregend = dd->kregbase + TXE_PIO_SEND;
 	dd->physaddr = addr;        /* used for io_remap, etc. */
 
 	/*
-	 * Re-map the chip's RcvArray as write-combining to allow us
+	 * Map the chip's RcvArray as write-combining to allow us
 	 * to write an entire cacheline worth of entries in one shot.
-	 * If this re-map fails, just continue - the RcvArray programming
-	 * function will handle both cases.
 	 */
-	dd->chip_rcv_array_count = read_csr(dd, RCV_ARRAY_CNT);
 	dd->rcvarray_wc = ioremap_wc(addr + RCV_ARRAY,
 				     dd->chip_rcv_array_count * 8);
-	dd_dev_info(dd, "WC Remapped RcvArray: %p\n", dd->rcvarray_wc);
-	/*
-	 * Save BARs and command to rewrite after device reset.
-	 */
-	pci_read_config_dword(dd->pcidev, PCI_BASE_ADDRESS_0, &dd->pcibar0);
-	pci_read_config_dword(dd->pcidev, PCI_BASE_ADDRESS_1, &dd->pcibar1);
-	pci_read_config_dword(dd->pcidev, PCI_ROM_ADDRESS, &dd->pci_rom);
-	pci_read_config_word(dd->pcidev, PCI_COMMAND, &dd->pci_command);
-	pcie_capability_read_word(dd->pcidev, PCI_EXP_DEVCTL, &dd->pcie_devctl);
-	pcie_capability_read_word(dd->pcidev, PCI_EXP_LNKCTL, &dd->pcie_lnkctl);
-	pcie_capability_read_word(dd->pcidev, PCI_EXP_DEVCTL2,
-				  &dd->pcie_devctl2);
-	pci_read_config_dword(dd->pcidev, PCI_CFG_MSIX0, &dd->pci_msix0);
-	pci_read_config_dword(dd->pcidev, PCIE_CFG_SPCIE1, &dd->pci_lnkctl3);
-	pci_read_config_dword(dd->pcidev, PCIE_CFG_TPH2, &dd->pci_tph2);
+	if (!dd->rcvarray_wc) {
+		dd_dev_err(dd, "WC mapping of receive array failed\n");
+		goto nomem;
+	}
+	dd_dev_info(dd, "WC RcvArray: %p for %x\n",
+		    dd->rcvarray_wc, dd->chip_rcv_array_count * 8);
 
+	dd->flags |= HFI1_PRESENT;	/* chip.c CSR routines now work */
 	return 0;
+nomem:
+	ret = -ENOMEM;
+	hfi1_pcie_ddcleanup(dd);
+	return ret;
 }
 
 /*
@@ -229,59 +237,19 @@ int hfi1_pcie_ddinit(struct hfi1_devdata *dd, struct pci_dev *pdev)
  */
 void hfi1_pcie_ddcleanup(struct hfi1_devdata *dd)
 {
-	u64 __iomem *base = (void __iomem *)dd->kregbase;
-
 	dd->flags &= ~HFI1_PRESENT;
-	dd->kregbase = NULL;
-	iounmap(base);
+	if (dd->kregbase1)
+		iounmap(dd->kregbase1);
+	dd->kregbase1 = NULL;
+	if (dd->kregbase2)
+		iounmap(dd->kregbase2);
+	dd->kregbase2 = NULL;
 	if (dd->rcvarray_wc)
 		iounmap(dd->rcvarray_wc);
+	dd->rcvarray_wc = NULL;
 	if (dd->piobase)
 		iounmap(dd->piobase);
-}
-
-static void msix_setup(struct hfi1_devdata *dd, int pos, u32 *msixcnt,
-		       struct hfi1_msix_entry *hfi1_msix_entry)
-{
-	int ret;
-	int nvec = *msixcnt;
-	struct msix_entry *msix_entry;
-	int i;
-
-	/*
-	 * We can't pass hfi1_msix_entry array to msix_setup
-	 * so use a dummy msix_entry array and copy the allocated
-	 * irq back to the hfi1_msix_entry array.
-	 */
-	msix_entry = kmalloc_array(nvec, sizeof(*msix_entry), GFP_KERNEL);
-	if (!msix_entry) {
-		ret = -ENOMEM;
-		goto do_intx;
-	}
-
-	for (i = 0; i < nvec; i++)
-		msix_entry[i] = hfi1_msix_entry[i].msix;
-
-	ret = pci_enable_msix_range(dd->pcidev, msix_entry, 1, nvec);
-	if (ret < 0)
-		goto free_msix_entry;
-	nvec = ret;
-
-	for (i = 0; i < nvec; i++)
-		hfi1_msix_entry[i].msix = msix_entry[i];
-
-	kfree(msix_entry);
-	*msixcnt = nvec;
-	return;
-
-free_msix_entry:
-	kfree(msix_entry);
-
-do_intx:
-	dd_dev_err(dd, "pci_enable_msix_range %d vectors failed: %d, falling back to INTx\n",
-		   nvec, ret);
-	*msixcnt = 0;
-	hfi1_enable_intx(dd->pcidev);
+	dd->piobase = NULL;
 }
 
 /* return the PCIe link speed from the given link status */
@@ -314,8 +282,14 @@ static u32 extract_width(u16 linkstat)
 static void update_lbus_info(struct hfi1_devdata *dd)
 {
 	u16 linkstat;
+	int ret;
+
+	ret = pcie_capability_read_word(dd->pcidev, PCI_EXP_LNKSTA, &linkstat);
+	if (ret) {
+		dd_dev_err(dd, "Unable to read from PCI config\n");
+		return;
+	}
 
-	pcie_capability_read_word(dd->pcidev, PCI_EXP_LNKSTA, &linkstat);
 	dd->lbus_width = extract_width(linkstat);
 	dd->lbus_speed = extract_speed(linkstat);
 	snprintf(dd->lbus_info, sizeof(dd->lbus_info),
@@ -330,6 +304,7 @@ int pcie_speeds(struct hfi1_devdata *dd)
 {
 	u32 linkcap;
 	struct pci_dev *parent = dd->pcidev->bus->self;
+	int ret;
 
 	if (!pci_is_pcie(dd->pcidev)) {
 		dd_dev_err(dd, "Can't find PCI Express capability!\n");
@@ -339,7 +314,12 @@ int pcie_speeds(struct hfi1_devdata *dd)
 	/* find if our max speed is Gen3 and parent supports Gen3 speeds */
 	dd->link_gen3_capable = 1;
 
-	pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &linkcap);
+	ret = pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &linkcap);
+	if (ret) {
+		dd_dev_err(dd, "Unable to read from PCI config\n");
+		return ret;
+	}
+
 	if ((linkcap & PCI_EXP_LNKCAP_SLS) != GEN3_SPEED_VECTOR) {
 		dd_dev_info(dd,
 			    "This HFI is not Gen3 capable, max speed 0x%x, need 0x3\n",
@@ -364,49 +344,150 @@ int pcie_speeds(struct hfi1_devdata *dd)
 }
 
 /*
- * Returns in *nent:
- *	- actual number of interrupts allocated
+ * Returns:
+ *	- actual number of interrupts allocated or
  *	- 0 if fell back to INTx.
+ *      - error
  */
-void request_msix(struct hfi1_devdata *dd, u32 *nent,
-		  struct hfi1_msix_entry *entry)
+int request_msix(struct hfi1_devdata *dd, u32 msireq)
 {
-	int pos;
+	int nvec, ret;
 
-	pos = dd->pcidev->msix_cap;
-	if (*nent && pos) {
-		msix_setup(dd, pos, nent, entry);
-		/* did it, either MSI-X or INTx */
-	} else {
-		*nent = 0;
-		hfi1_enable_intx(dd->pcidev);
+	nvec = pci_alloc_irq_vectors(dd->pcidev, 1, msireq,
+				     PCI_IRQ_MSIX | PCI_IRQ_LEGACY);
+	if (nvec < 0) {
+		dd_dev_err(dd, "pci_alloc_irq_vectors() failed: %d\n", nvec);
+		return nvec;
 	}
 
-	tune_pcie_caps(dd);
+	ret = tune_pcie_caps(dd);
+	if (ret) {
+		dd_dev_err(dd, "tune_pcie_caps() failed: %d\n", ret);
+		pci_free_irq_vectors(dd->pcidev);
+		return ret;
+	}
+
+	/* check for legacy IRQ */
+	if (nvec == 1 && !dd->pcidev->msix_enabled)
+		return 0;
+
+	return nvec;
 }
 
-void hfi1_enable_intx(struct pci_dev *pdev)
+/* restore command and BARs after a reset has wiped them out */
+int restore_pci_variables(struct hfi1_devdata *dd)
 {
-	/* first, turn on INTx */
-	pci_intx(pdev, 1);
-	/* then turn off MSI-X */
-	pci_disable_msix(pdev);
+	int ret = 0;
+
+	ret = pci_write_config_word(dd->pcidev, PCI_COMMAND, dd->pci_command);
+	if (ret)
+		goto error;
+
+	ret = pci_write_config_dword(dd->pcidev, PCI_BASE_ADDRESS_0,
+				     dd->pcibar0);
+	if (ret)
+		goto error;
+
+	ret = pci_write_config_dword(dd->pcidev, PCI_BASE_ADDRESS_1,
+				     dd->pcibar1);
+	if (ret)
+		goto error;
+
+	ret = pci_write_config_dword(dd->pcidev, PCI_ROM_ADDRESS, dd->pci_rom);
+	if (ret)
+		goto error;
+
+	ret = pcie_capability_write_word(dd->pcidev, PCI_EXP_DEVCTL,
+					 dd->pcie_devctl);
+	if (ret)
+		goto error;
+
+	ret = pcie_capability_write_word(dd->pcidev, PCI_EXP_LNKCTL,
+					 dd->pcie_lnkctl);
+	if (ret)
+		goto error;
+
+	ret = pcie_capability_write_word(dd->pcidev, PCI_EXP_DEVCTL2,
+					 dd->pcie_devctl2);
+	if (ret)
+		goto error;
+
+	ret = pci_write_config_dword(dd->pcidev, PCI_CFG_MSIX0, dd->pci_msix0);
+	if (ret)
+		goto error;
+
+	ret = pci_write_config_dword(dd->pcidev, PCIE_CFG_SPCIE1,
+				     dd->pci_lnkctl3);
+	if (ret)
+		goto error;
+
+	ret = pci_write_config_dword(dd->pcidev, PCIE_CFG_TPH2, dd->pci_tph2);
+	if (ret)
+		goto error;
+
+	return 0;
+
+error:
+	dd_dev_err(dd, "Unable to write to PCI config\n");
+	return ret;
 }
 
-/* restore command and BARs after a reset has wiped them out */
-void restore_pci_variables(struct hfi1_devdata *dd)
+/* Save BARs and command to rewrite after device reset */
+int save_pci_variables(struct hfi1_devdata *dd)
 {
-	pci_write_config_word(dd->pcidev, PCI_COMMAND, dd->pci_command);
-	pci_write_config_dword(dd->pcidev, PCI_BASE_ADDRESS_0, dd->pcibar0);
-	pci_write_config_dword(dd->pcidev, PCI_BASE_ADDRESS_1, dd->pcibar1);
-	pci_write_config_dword(dd->pcidev, PCI_ROM_ADDRESS, dd->pci_rom);
-	pcie_capability_write_word(dd->pcidev, PCI_EXP_DEVCTL, dd->pcie_devctl);
-	pcie_capability_write_word(dd->pcidev, PCI_EXP_LNKCTL, dd->pcie_lnkctl);
-	pcie_capability_write_word(dd->pcidev, PCI_EXP_DEVCTL2,
-				   dd->pcie_devctl2);
-	pci_write_config_dword(dd->pcidev, PCI_CFG_MSIX0, dd->pci_msix0);
-	pci_write_config_dword(dd->pcidev, PCIE_CFG_SPCIE1, dd->pci_lnkctl3);
-	pci_write_config_dword(dd->pcidev, PCIE_CFG_TPH2, dd->pci_tph2);
+	int ret = 0;
+
+	ret = pci_read_config_dword(dd->pcidev, PCI_BASE_ADDRESS_0,
+				    &dd->pcibar0);
+	if (ret)
+		goto error;
+
+	ret = pci_read_config_dword(dd->pcidev, PCI_BASE_ADDRESS_1,
+				    &dd->pcibar1);
+	if (ret)
+		goto error;
+
+	ret = pci_read_config_dword(dd->pcidev, PCI_ROM_ADDRESS, &dd->pci_rom);
+	if (ret)
+		goto error;
+
+	ret = pci_read_config_word(dd->pcidev, PCI_COMMAND, &dd->pci_command);
+	if (ret)
+		goto error;
+
+	ret = pcie_capability_read_word(dd->pcidev, PCI_EXP_DEVCTL,
+					&dd->pcie_devctl);
+	if (ret)
+		goto error;
+
+	ret = pcie_capability_read_word(dd->pcidev, PCI_EXP_LNKCTL,
+					&dd->pcie_lnkctl);
+	if (ret)
+		goto error;
+
+	ret = pcie_capability_read_word(dd->pcidev, PCI_EXP_DEVCTL2,
+					&dd->pcie_devctl2);
+	if (ret)
+		goto error;
+
+	ret = pci_read_config_dword(dd->pcidev, PCI_CFG_MSIX0, &dd->pci_msix0);
+	if (ret)
+		goto error;
+
+	ret = pci_read_config_dword(dd->pcidev, PCIE_CFG_SPCIE1,
+				    &dd->pci_lnkctl3);
+	if (ret)
+		goto error;
+
+	ret = pci_read_config_dword(dd->pcidev, PCIE_CFG_TPH2, &dd->pci_tph2);
+	if (ret)
+		goto error;
+
+	return 0;
+
+error:
+	dd_dev_err(dd, "Unable to read from PCI config\n");
+	return ret;
 }
 
 /*
@@ -421,21 +502,33 @@ uint aspm_mode = ASPM_MODE_DISABLED;
 module_param_named(aspm, aspm_mode, uint, S_IRUGO);
 MODULE_PARM_DESC(aspm, "PCIe ASPM: 0: disable, 1: enable, 2: dynamic");
 
-static void tune_pcie_caps(struct hfi1_devdata *dd)
+static int tune_pcie_caps(struct hfi1_devdata *dd)
 {
 	struct pci_dev *parent;
 	u16 rc_mpss, rc_mps, ep_mpss, ep_mps;
 	u16 rc_mrrs, ep_mrrs, max_mrrs, ectl;
+	int ret;
 
 	/*
 	 * Turn on extended tags in DevCtl in case the BIOS has turned it off
 	 * to improve WFR SDMA bandwidth
 	 */
-	pcie_capability_read_word(dd->pcidev, PCI_EXP_DEVCTL, &ectl);
+	ret = pcie_capability_read_word(dd->pcidev,
+					PCI_EXP_DEVCTL, &ectl);
+	if (ret) {
+		dd_dev_err(dd, "Unable to read from PCI config\n");
+		return ret;
+	}
+
 	if (!(ectl & PCI_EXP_DEVCTL_EXT_TAG)) {
 		dd_dev_info(dd, "Enabling PCIe extended tags\n");
 		ectl |= PCI_EXP_DEVCTL_EXT_TAG;
-		pcie_capability_write_word(dd->pcidev, PCI_EXP_DEVCTL, ectl);
+		ret = pcie_capability_write_word(dd->pcidev,
+						 PCI_EXP_DEVCTL, ectl);
+		if (ret) {
+			dd_dev_err(dd, "Unable to write to PCI config\n");
+			return ret;
+		}
 	}
 	/* Find out supported and configured values for parent (root) */
 	parent = dd->pcidev->bus->self;
@@ -444,14 +537,14 @@ static void tune_pcie_caps(struct hfi1_devdata *dd)
 	 * access to the upstream component.
 	 */
 	if (!parent)
-		return;
+		return -EINVAL;
 	if (!pci_is_root_bus(parent->bus)) {
 		dd_dev_info(dd, "Parent not root\n");
-		return;
+		return -EINVAL;
 	}
 
 	if (!pci_is_pcie(parent) || !pci_is_pcie(dd->pcidev))
-		return;
+		return -EINVAL;
 	rc_mpss = parent->pcie_mpss;
 	rc_mps = ffs(pcie_get_mps(parent)) - 8;
 	/* Find out supported and configured values for endpoint (us) */
@@ -497,6 +590,8 @@ static void tune_pcie_caps(struct hfi1_devdata *dd)
 		ep_mrrs = max_mrrs;
 		pcie_set_readrq(dd->pcidev, ep_mrrs);
 	}
+
+	return 0;
 }
 
 /* End of PCIe capability tuning */
@@ -728,6 +823,7 @@ static int load_eq_table(struct hfi1_devdata *dd, const u8 eq[11][3], u8 fs,
 	u32 violation;
 	u32 i;
 	u8 c_minus1, c0, c_plus1;
+	int ret;
 
 	for (i = 0; i < 11; i++) {
 		/* set index */
@@ -739,8 +835,14 @@ static int load_eq_table(struct hfi1_devdata *dd, const u8 eq[11][3], u8 fs,
 		pci_write_config_dword(pdev, PCIE_CFG_REG_PL102,
 				       eq_value(c_minus1, c0, c_plus1));
 		/* check if these coefficients violate EQ rules */
-		pci_read_config_dword(dd->pcidev, PCIE_CFG_REG_PL105,
-				      &violation);
+		ret = pci_read_config_dword(dd->pcidev,
+					    PCIE_CFG_REG_PL105, &violation);
+		if (ret) {
+			dd_dev_err(dd, "Unable to read from PCI config\n");
+			hit_error = 1;
+			break;
+		}
+
 		if (violation
 		    & PCIE_CFG_REG_PL105_GEN3_EQ_VIOLATE_COEF_RULES_SMASK){
 			if (hit_error == 0) {
@@ -1194,7 +1296,13 @@ retry:
 	 * that it is Gen3 capable earlier.
 	 */
 	dd_dev_info(dd, "%s: setting parent target link speed\n", __func__);
-	pcie_capability_read_word(parent, PCI_EXP_LNKCTL2, &lnkctl2);
+	ret = pcie_capability_read_word(parent, PCI_EXP_LNKCTL2, &lnkctl2);
+	if (ret) {
+		dd_dev_err(dd, "Unable to read from PCI config\n");
+		return_error = 1;
+		goto done;
+	}
+
 	dd_dev_info(dd, "%s: ..old link control2: 0x%x\n", __func__,
 		    (u32)lnkctl2);
 	/* only write to parent if target is not as high as ours */
@@ -1203,20 +1311,37 @@ retry:
 		lnkctl2 |= target_vector;
 		dd_dev_info(dd, "%s: ..new link control2: 0x%x\n", __func__,
 			    (u32)lnkctl2);
-		pcie_capability_write_word(parent, PCI_EXP_LNKCTL2, lnkctl2);
+		ret = pcie_capability_write_word(parent,
+						 PCI_EXP_LNKCTL2, lnkctl2);
+		if (ret) {
+			dd_dev_err(dd, "Unable to write to PCI config\n");
+			return_error = 1;
+			goto done;
+		}
 	} else {
 		dd_dev_info(dd, "%s: ..target speed is OK\n", __func__);
 	}
 
 	dd_dev_info(dd, "%s: setting target link speed\n", __func__);
-	pcie_capability_read_word(dd->pcidev, PCI_EXP_LNKCTL2, &lnkctl2);
+	ret = pcie_capability_read_word(dd->pcidev, PCI_EXP_LNKCTL2, &lnkctl2);
+	if (ret) {
+		dd_dev_err(dd, "Unable to read from PCI config\n");
+		return_error = 1;
+		goto done;
+	}
+
 	dd_dev_info(dd, "%s: ..old link control2: 0x%x\n", __func__,
 		    (u32)lnkctl2);
 	lnkctl2 &= ~LNKCTL2_TARGET_LINK_SPEED_MASK;
 	lnkctl2 |= target_vector;
 	dd_dev_info(dd, "%s: ..new link control2: 0x%x\n", __func__,
 		    (u32)lnkctl2);
-	pcie_capability_write_word(dd->pcidev, PCI_EXP_LNKCTL2, lnkctl2);
+	ret = pcie_capability_write_word(dd->pcidev, PCI_EXP_LNKCTL2, lnkctl2);
+	if (ret) {
+		dd_dev_err(dd, "Unable to write to PCI config\n");
+		return_error = 1;
+		goto done;
+	}
 
 	/* step 5h: arm gasket logic */
 	/* hold DC in reset across the SBR */
@@ -1266,7 +1391,14 @@ retry:
 
 	/* restore PCI space registers we know were reset */
 	dd_dev_info(dd, "%s: calling restore_pci_variables\n", __func__);
-	restore_pci_variables(dd);
+	ret = restore_pci_variables(dd);
+	if (ret) {
+		dd_dev_err(dd, "%s: Could not restore PCI variables\n",
+			   __func__);
+		return_error = 1;
+		goto done;
+	}
+
 	/* restore firmware control */
 	write_csr(dd, MISC_CFG_FW_CTRL, fw_ctrl);
 
@@ -1296,7 +1428,13 @@ retry:
 	setextled(dd, 0);
 
 	/* check for any per-lane errors */
-	pci_read_config_dword(dd->pcidev, PCIE_CFG_SPCIE2, &reg32);
+	ret = pci_read_config_dword(dd->pcidev, PCIE_CFG_SPCIE2, &reg32);
+	if (ret) {
+		dd_dev_err(dd, "Unable to read from PCI config\n");
+		return_error = 1;
+		goto done;
+	}
+
 	dd_dev_info(dd, "%s: per-lane errors: 0x%x\n", __func__, reg32);
 
 	/* extract status, look for our HFI */
diff --git a/drivers/infiniband/hw/hfi1/pio.c b/drivers/infiniband/hw/hfi1/pio.c
index ed72b5aca139..7108a4b5e94c 100644
--- a/drivers/infiniband/hw/hfi1/pio.c
+++ b/drivers/infiniband/hw/hfi1/pio.c
@@ -1012,7 +1012,7 @@ static void sc_wait_for_packet_egress(struct send_context *sc, int pause)
 				   "%s: context %u(%u) timeout waiting for packets to egress, remaining count %u, bouncing link\n",
 				   __func__, sc->sw_index,
 				   sc->hw_context, (u32)reg);
-			queue_work(dd->pport->hfi1_wq,
+			queue_work(dd->pport->link_wq,
 				   &dd->pport->link_bounce_work);
 			break;
 		}
@@ -1568,7 +1568,8 @@ static void sc_piobufavail(struct send_context *sc)
 	struct rvt_qp *qp;
 	struct hfi1_qp_priv *priv;
 	unsigned long flags;
-	unsigned i, n = 0;
+	uint i, n = 0, max_idx = 0;
+	u8 max_starved_cnt = 0;
 
 	if (dd->send_contexts[sc->sw_index].type != SC_KERNEL &&
 	    dd->send_contexts[sc->sw_index].type != SC_VL15)
@@ -1591,6 +1592,7 @@ static void sc_piobufavail(struct send_context *sc)
 		priv = qp->priv;
 		list_del_init(&priv->s_iowait.list);
 		priv->s_iowait.lock = NULL;
+		iowait_starve_find_max(wait, &max_starved_cnt, n, &max_idx);
 		/* refcount held until actual wake up */
 		qps[n++] = qp;
 	}
@@ -1605,9 +1607,14 @@ static void sc_piobufavail(struct send_context *sc)
 	}
 	write_sequnlock_irqrestore(&dev->iowait_lock, flags);
 
-	for (i = 0; i < n; i++)
-		hfi1_qp_wakeup(qps[i],
+	/* Wake up the most starved one first */
+	if (n)
+		hfi1_qp_wakeup(qps[max_idx],
 			       RVT_S_WAIT_PIO | RVT_S_WAIT_PIO_DRAIN);
+	for (i = 0; i < n; i++)
+		if (i != max_idx)
+			hfi1_qp_wakeup(qps[i],
+				       RVT_S_WAIT_PIO | RVT_S_WAIT_PIO_DRAIN);
 }
 
 /* translate a send credit update to a bit code of reasons */
diff --git a/drivers/infiniband/hw/hfi1/platform.c b/drivers/infiniband/hw/hfi1/platform.c
index 838fe84e285a..a8af96d2b1b0 100644
--- a/drivers/infiniband/hw/hfi1/platform.c
+++ b/drivers/infiniband/hw/hfi1/platform.c
@@ -45,10 +45,14 @@
  *
  */
 
+#include <linux/firmware.h>
+
 #include "hfi.h"
 #include "efivar.h"
 #include "eprom.h"
 
+#define DEFAULT_PLATFORM_CONFIG_NAME "hfi1_platform.dat"
+
 static int validate_scratch_checksum(struct hfi1_devdata *dd)
 {
 	u64 checksum = 0, temp_scratch = 0;
@@ -58,8 +62,13 @@ static int validate_scratch_checksum(struct hfi1_devdata *dd)
 	version = (temp_scratch & BITMAP_VERSION_SMASK) >> BITMAP_VERSION_SHIFT;
 
 	/* Prevent power on default of all zeroes from passing checksum */
-	if (!version)
+	if (!version) {
+		dd_dev_err(dd, "%s: Config bitmap uninitialized\n", __func__);
+		dd_dev_err(dd,
+			   "%s: Please update your BIOS to support active channels\n",
+			   __func__);
 		return 0;
+	}
 
 	/*
 	 * ASIC scratch 0 only contains the checksum and bitmap version as
@@ -84,6 +93,8 @@ static int validate_scratch_checksum(struct hfi1_devdata *dd)
 
 	if (checksum + temp_scratch == 0xFFFF)
 		return 1;
+
+	dd_dev_err(dd, "%s: Configuration bitmap corrupted\n", __func__);
 	return 0;
 }
 
@@ -131,25 +142,22 @@ static void save_platform_config_fields(struct hfi1_devdata *dd)
 
 	ppd->max_power_class = (temp_scratch & QSFP_MAX_POWER_SMASK) >>
 				QSFP_MAX_POWER_SHIFT;
+
+	ppd->config_from_scratch = true;
 }
 
 void get_platform_config(struct hfi1_devdata *dd)
 {
 	int ret = 0;
-	unsigned long size = 0;
 	u8 *temp_platform_config = NULL;
 	u32 esize;
+	const struct firmware *platform_config_file = NULL;
 
 	if (is_integrated(dd)) {
 		if (validate_scratch_checksum(dd)) {
 			save_platform_config_fields(dd);
 			return;
 		}
-		dd_dev_err(dd, "%s: Config bitmap corrupted/uninitialized\n",
-			   __func__);
-		dd_dev_err(dd,
-			   "%s: Please update your BIOS to support active channels\n",
-			   __func__);
 	} else {
 		ret = eprom_read_platform_config(dd,
 						 (void **)&temp_platform_config,
@@ -160,36 +168,37 @@ void get_platform_config(struct hfi1_devdata *dd)
 			dd->platform_config.size = esize;
 			return;
 		}
-		/* fail, try EFI variable */
-
-		ret = read_hfi1_efi_var(dd, "configuration", &size,
-					(void **)&temp_platform_config);
-		if (!ret) {
-			dd->platform_config.data = temp_platform_config;
-			dd->platform_config.size = size;
-			return;
-		}
 	}
 	dd_dev_err(dd,
 		   "%s: Failed to get platform config, falling back to sub-optimal default file\n",
 		   __func__);
-	/* fall back to request firmware */
-	platform_config_load = 1;
-}
 
-void free_platform_config(struct hfi1_devdata *dd)
-{
-	if (!platform_config_load) {
-		/*
-		 * was loaded from EFI or the EPROM, release memory
-		 * allocated by read_efi_var/eprom_read_platform_config
-		 */
-		kfree(dd->platform_config.data);
+	ret = request_firmware(&platform_config_file,
+			       DEFAULT_PLATFORM_CONFIG_NAME,
+			       &dd->pcidev->dev);
+	if (ret) {
+		dd_dev_err(dd,
+			   "%s: No default platform config file found\n",
+			   __func__);
+		return;
 	}
+
 	/*
-	 * else do nothing, dispose_firmware will release
-	 * struct firmware platform_config on driver exit
+	 * Allocate separate memory block to store data and free firmware
+	 * structure. This allows free_platform_config to treat EPROM and
+	 * fallback configs in the same manner.
 	 */
+	dd->platform_config.data = kmemdup(platform_config_file->data,
+					   platform_config_file->size,
+					   GFP_KERNEL);
+	dd->platform_config.size = platform_config_file->size;
+	release_firmware(platform_config_file);
+}
+
+void free_platform_config(struct hfi1_devdata *dd)
+{
+	/* Release memory allocated for eprom or fallback file read. */
+	kfree(dd->platform_config.data);
 }
 
 void get_port_type(struct hfi1_pportdata *ppd)
@@ -242,7 +251,7 @@ static int qual_power(struct hfi1_pportdata *ppd)
 
 	if (ppd->offline_disabled_reason ==
 			HFI1_ODR_MASK(OPA_LINKDOWN_REASON_POWER_POLICY)) {
-		dd_dev_info(
+		dd_dev_err(
 			ppd->dd,
 			"%s: Port disabled due to system power restrictions\n",
 			__func__);
@@ -268,7 +277,7 @@ static int qual_bitrate(struct hfi1_pportdata *ppd)
 
 	if (ppd->offline_disabled_reason ==
 			HFI1_ODR_MASK(OPA_LINKDOWN_REASON_LINKSPEED_POLICY)) {
-		dd_dev_info(
+		dd_dev_err(
 			ppd->dd,
 			"%s: Cable failed bitrate check, disabling port\n",
 			__func__);
@@ -709,15 +718,15 @@ static void apply_tunings(
 		ret = load_8051_config(ppd->dd, DC_HOST_COMM_SETTINGS,
 				       GENERAL_CONFIG, config_data);
 		if (ret != HCMD_SUCCESS)
-			dd_dev_info(ppd->dd,
-				    "%s: Failed set ext device config params\n",
-				    __func__);
+			dd_dev_err(ppd->dd,
+				   "%s: Failed set ext device config params\n",
+				   __func__);
 	}
 
 	if (tx_preset_index == OPA_INVALID_INDEX) {
 		if (ppd->port_type == PORT_TYPE_QSFP && limiting_active)
-			dd_dev_info(ppd->dd, "%s: Invalid Tx preset index\n",
-				    __func__);
+			dd_dev_err(ppd->dd, "%s: Invalid Tx preset index\n",
+				   __func__);
 		return;
 	}
 
@@ -900,7 +909,7 @@ static int tune_qsfp(struct hfi1_pportdata *ppd,
 	case 0xD: /* fallthrough */
 	case 0xF:
 	default:
-		dd_dev_info(ppd->dd, "%s: Unknown/unsupported cable\n",
+		dd_dev_warn(ppd->dd, "%s: Unknown/unsupported cable\n",
 			    __func__);
 		break;
 	}
@@ -935,6 +944,21 @@ void tune_serdes(struct hfi1_pportdata *ppd)
 	if (loopback != LOOPBACK_NONE ||
 	    ppd->dd->icode == ICODE_FUNCTIONAL_SIMULATOR) {
 		ppd->driver_link_ready = 1;
+
+		if (qsfp_mod_present(ppd)) {
+			ret = acquire_chip_resource(ppd->dd,
+						    qsfp_resource(ppd->dd),
+						    QSFP_WAIT);
+			if (ret) {
+				dd_dev_err(ppd->dd, "%s: hfi%d: cannot lock i2c chain\n",
+					   __func__, (int)ppd->dd->hfi1_id);
+				goto bail;
+			}
+
+			refresh_qsfp_cache(ppd, &ppd->qsfp_info);
+			release_chip_resource(ppd->dd, qsfp_resource(ppd->dd));
+		}
+
 		return;
 	}
 
@@ -942,7 +966,7 @@ void tune_serdes(struct hfi1_pportdata *ppd)
 	case PORT_TYPE_DISCONNECTED:
 		ppd->offline_disabled_reason =
 			HFI1_ODR_MASK(OPA_LINKDOWN_REASON_DISCONNECTED);
-		dd_dev_info(dd, "%s: Port disconnected, disabling port\n",
+		dd_dev_warn(dd, "%s: Port disconnected, disabling port\n",
 			    __func__);
 		goto bail;
 	case PORT_TYPE_FIXED:
@@ -1027,7 +1051,7 @@ void tune_serdes(struct hfi1_pportdata *ppd)
 		}
 		break;
 	default:
-		dd_dev_info(ppd->dd, "%s: Unknown port type\n", __func__);
+		dd_dev_warn(ppd->dd, "%s: Unknown port type\n", __func__);
 		ppd->port_type = PORT_TYPE_UNKNOWN;
 		tuning_method = OPA_UNKNOWN_TUNING;
 		total_atten = 0;
diff --git a/drivers/infiniband/hw/hfi1/qp.c b/drivers/infiniband/hw/hfi1/qp.c
index 1a7af9f60c13..4b01ccd895b4 100644
--- a/drivers/infiniband/hw/hfi1/qp.c
+++ b/drivers/infiniband/hw/hfi1/qp.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015, 2016 Intel Corporation.
+ * Copyright(c) 2015 - 2017 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -68,17 +68,12 @@ static int iowait_sleep(
 	struct sdma_engine *sde,
 	struct iowait *wait,
 	struct sdma_txreq *stx,
-	unsigned seq);
+	unsigned int seq,
+	bool pkts_sent);
 static void iowait_wakeup(struct iowait *wait, int reason);
 static void iowait_sdma_drained(struct iowait *wait);
 static void qp_pio_drain(struct rvt_qp *qp);
 
-static inline unsigned mk_qpn(struct rvt_qpn_table *qpt,
-			      struct rvt_qpn_map *map, unsigned off)
-{
-	return (map - qpt->map) * RVT_BITS_PER_PAGE + off;
-}
-
 const struct rvt_operation_params hfi1_post_parms[RVT_OPERATION_MAX] = {
 [IB_WR_RDMA_WRITE] = {
 	.length = sizeof(struct ib_rdma_wr),
@@ -237,6 +232,31 @@ int hfi1_check_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr,
 	return 0;
 }
 
+/*
+ * qp_set_16b - Set the hdr_type based on whether the slid or the
+ * dlid in the connection is extended. Only applicable for RC and UC
+ * QPs. UD QPs determine this on the fly from the ah in the wqe
+ */
+static inline void qp_set_16b(struct rvt_qp *qp)
+{
+	struct hfi1_pportdata *ppd;
+	struct hfi1_ibport *ibp;
+	struct hfi1_qp_priv *priv = qp->priv;
+
+	/* Update ah_attr to account for extended LIDs */
+	hfi1_update_ah_attr(qp->ibqp.device, &qp->remote_ah_attr);
+
+	/* Create 32 bit LIDs */
+	hfi1_make_opa_lid(&qp->remote_ah_attr);
+
+	if (!(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH))
+		return;
+
+	ibp = to_iport(qp->ibqp.device, qp->port_num);
+	ppd = ppd_from_ibp(ibp);
+	priv->hdr_type = hfi1_get_hdr_type(ppd->lid, &qp->remote_ah_attr);
+}
+
 void hfi1_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr,
 		    int attr_mask, struct ib_udata *udata)
 {
@@ -247,6 +267,7 @@ void hfi1_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr,
 		priv->s_sc = ah_to_sc(ibqp->device, &qp->remote_ah_attr);
 		priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc);
 		priv->s_sendcontext = qp_to_send_context(qp, priv->s_sc);
+		qp_set_16b(qp);
 	}
 
 	if (attr_mask & IB_QP_PATH_MIG_STATE &&
@@ -256,6 +277,7 @@ void hfi1_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr,
 		priv->s_sc = ah_to_sc(ibqp->device, &qp->remote_ah_attr);
 		priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc);
 		priv->s_sendcontext = qp_to_send_context(qp, priv->s_sc);
+		qp_set_16b(qp);
 	}
 }
 
@@ -377,7 +399,8 @@ static int iowait_sleep(
 	struct sdma_engine *sde,
 	struct iowait *wait,
 	struct sdma_txreq *stx,
-	unsigned seq)
+	uint seq,
+	bool pkts_sent)
 {
 	struct verbs_txreq *tx = container_of(stx, struct verbs_txreq, txreq);
 	struct rvt_qp *qp;
@@ -408,7 +431,8 @@ static int iowait_sleep(
 
 			ibp->rvp.n_dmawait++;
 			qp->s_flags |= RVT_S_WAIT_DMA_DESC;
-			list_add_tail(&priv->s_iowait.list, &sde->dmawait);
+			iowait_queue(pkts_sent, &priv->s_iowait,
+				     &sde->dmawait);
 			priv->s_iowait.lock = &dev->iowait_lock;
 			trace_hfi1_qpsleep(qp, RVT_S_WAIT_DMA_DESC);
 			rvt_get_qp(qp);
@@ -506,82 +530,6 @@ struct send_context *qp_to_send_context(struct rvt_qp *qp, u8 sc5)
 					  sc5);
 }
 
-struct qp_iter {
-	struct hfi1_ibdev *dev;
-	struct rvt_qp *qp;
-	int specials;
-	int n;
-};
-
-struct qp_iter *qp_iter_init(struct hfi1_ibdev *dev)
-{
-	struct qp_iter *iter;
-
-	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
-	if (!iter)
-		return NULL;
-
-	iter->dev = dev;
-	iter->specials = dev->rdi.ibdev.phys_port_cnt * 2;
-
-	return iter;
-}
-
-int qp_iter_next(struct qp_iter *iter)
-{
-	struct hfi1_ibdev *dev = iter->dev;
-	int n = iter->n;
-	int ret = 1;
-	struct rvt_qp *pqp = iter->qp;
-	struct rvt_qp *qp;
-
-	/*
-	 * The approach is to consider the special qps
-	 * as an additional table entries before the
-	 * real hash table.  Since the qp code sets
-	 * the qp->next hash link to NULL, this works just fine.
-	 *
-	 * iter->specials is 2 * # ports
-	 *
-	 * n = 0..iter->specials is the special qp indices
-	 *
-	 * n = iter->specials..dev->rdi.qp_dev->qp_table_size+iter->specials are
-	 * the potential hash bucket entries
-	 *
-	 */
-	for (; n <  dev->rdi.qp_dev->qp_table_size + iter->specials; n++) {
-		if (pqp) {
-			qp = rcu_dereference(pqp->next);
-		} else {
-			if (n < iter->specials) {
-				struct hfi1_pportdata *ppd;
-				struct hfi1_ibport *ibp;
-				int pidx;
-
-				pidx = n % dev->rdi.ibdev.phys_port_cnt;
-				ppd = &dd_from_dev(dev)->pport[pidx];
-				ibp = &ppd->ibport_data;
-
-				if (!(n & 1))
-					qp = rcu_dereference(ibp->rvp.qp[0]);
-				else
-					qp = rcu_dereference(ibp->rvp.qp[1]);
-			} else {
-				qp = rcu_dereference(
-					dev->rdi.qp_dev->qp_table[
-						(n - iter->specials)]);
-			}
-		}
-		pqp = qp;
-		if (qp) {
-			iter->qp = qp;
-			iter->n = n;
-			return 0;
-		}
-	}
-	return ret;
-}
-
 static const char * const qp_type_str[] = {
 	"SMI", "GSI", "RC", "UC", "UD",
 };
@@ -595,19 +543,27 @@ static int qp_idle(struct rvt_qp *qp)
 		qp->s_tail == qp->s_head;
 }
 
-void qp_iter_print(struct seq_file *s, struct qp_iter *iter)
+/**
+ * qp_iter_print - print the qp information to seq_file
+ * @s: the seq_file to emit the qp information on
+ * @iter: the iterator for the qp hash list
+ */
+void qp_iter_print(struct seq_file *s, struct rvt_qp_iter *iter)
 {
 	struct rvt_swqe *wqe;
 	struct rvt_qp *qp = iter->qp;
 	struct hfi1_qp_priv *priv = qp->priv;
 	struct sdma_engine *sde;
 	struct send_context *send_context;
+	struct rvt_ack_entry *e = NULL;
 
 	sde = qp_to_sdma_engine(qp, priv->s_sc);
 	wqe = rvt_get_swqe_ptr(qp, qp->s_last);
 	send_context = qp_to_send_context(qp, priv->s_sc);
+	if (qp->s_ack_queue)
+		e = &qp->s_ack_queue[qp->s_tail_ack_queue];
 	seq_printf(s,
-		   "N %d %s QP %x R %u %s %u %u %u f=%x %u %u %u %u %u %u SPSN %x %x %x %x %x RPSN %x (%u %u %u %u %u %u %u) RQP %x LID %x SL %u MTU %u %u %u %u %u SDE %p,%u SC %p,%u SCQ %u %u PID %d\n",
+		   "N %d %s QP %x R %u %s %u %u %u f=%x %u %u %u %u %u %u SPSN %x %x %x %x %x RPSN %x S(%u %u %u %u %u %u %u) R(%u %u %u) RQP %x LID %x SL %u MTU %u %u %u %u %u SDE %p,%u SC %p,%u SCQ %u %u PID %d OS %x %x E %x %x %x\n",
 		   iter->n,
 		   qp_idle(qp) ? "I" : "B",
 		   qp->ibqp.qp_num,
@@ -630,6 +586,10 @@ void qp_iter_print(struct seq_file *s, struct qp_iter *iter)
 		   qp->s_last, qp->s_acked, qp->s_cur,
 		   qp->s_tail, qp->s_head, qp->s_size,
 		   qp->s_avail,
+		   /* ack_queue ring pointers, size */
+		   qp->s_tail_ack_queue, qp->r_head_ack_queue,
+		   rvt_max_atomic(&to_idev(qp->ibqp.device)->rdi),
+		   /* remote QP info  */
 		   qp->remote_qpn,
 		   rdma_ah_get_dlid(&qp->remote_ah_attr),
 		   rdma_ah_get_sl(&qp->remote_ah_attr),
@@ -644,7 +604,13 @@ void qp_iter_print(struct seq_file *s, struct qp_iter *iter)
 		   send_context ? send_context->sw_index : 0,
 		   ibcq_to_rvtcq(qp->ibqp.send_cq)->queue->head,
 		   ibcq_to_rvtcq(qp->ibqp.send_cq)->queue->tail,
-		   qp->pid);
+		   qp->pid,
+		   qp->s_state,
+		   qp->s_ack_state,
+		   /* ack queue information */
+		   e ? e->opcode : 0,
+		   e ? e->psn : 0,
+		   e ? e->lpsn : 0);
 }
 
 void *qp_priv_alloc(struct rvt_dev_info *rdi, struct rvt_qp *qp)
@@ -750,6 +716,7 @@ void hfi1_migrate_qp(struct rvt_qp *qp)
 	qp->s_flags |= RVT_S_AHG_CLEAR;
 	priv->s_sc = ah_to_sc(qp->ibqp.device, &qp->remote_ah_attr);
 	priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc);
+	qp_set_16b(qp);
 
 	ev.device = qp->ibqp.device;
 	ev.element.qp = &qp->ibqp;
@@ -832,6 +799,45 @@ void notify_error_qp(struct rvt_qp *qp)
 }
 
 /**
+ * hfi1_qp_iter_cb - callback for iterator
+ * @qp - the qp
+ * @v - the sl in low bits of v
+ *
+ * This is called from the iterator callback to work
+ * on an individual qp.
+ */
+static void hfi1_qp_iter_cb(struct rvt_qp *qp, u64 v)
+{
+	int lastwqe;
+	struct ib_event ev;
+	struct hfi1_ibport *ibp =
+		to_iport(qp->ibqp.device, qp->port_num);
+	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
+	u8 sl = (u8)v;
+
+	if (qp->port_num != ppd->port ||
+	    (qp->ibqp.qp_type != IB_QPT_UC &&
+	     qp->ibqp.qp_type != IB_QPT_RC) ||
+	    rdma_ah_get_sl(&qp->remote_ah_attr) != sl ||
+	    !(ib_rvt_state_ops[qp->state] & RVT_POST_SEND_OK))
+		return;
+
+	spin_lock_irq(&qp->r_lock);
+	spin_lock(&qp->s_hlock);
+	spin_lock(&qp->s_lock);
+	lastwqe = rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
+	spin_unlock(&qp->s_lock);
+	spin_unlock(&qp->s_hlock);
+	spin_unlock_irq(&qp->r_lock);
+	if (lastwqe) {
+		ev.device = qp->ibqp.device;
+		ev.element.qp = &qp->ibqp;
+		ev.event = IB_EVENT_QP_LAST_WQE_REACHED;
+		qp->ibqp.event_handler(&ev, qp->ibqp.qp_context);
+	}
+}
+
+/**
  * hfi1_error_port_qps - put a port's RC/UC qps into error state
  * @ibp: the ibport.
  * @sl: the service level.
@@ -842,44 +848,8 @@ void notify_error_qp(struct rvt_qp *qp)
  */
 void hfi1_error_port_qps(struct hfi1_ibport *ibp, u8 sl)
 {
-	struct rvt_qp *qp = NULL;
 	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
 	struct hfi1_ibdev *dev = &ppd->dd->verbs_dev;
-	int n;
-	int lastwqe;
-	struct ib_event ev;
-
-	rcu_read_lock();
-
-	/* Deal only with RC/UC qps that use the given SL. */
-	for (n = 0; n < dev->rdi.qp_dev->qp_table_size; n++) {
-		for (qp = rcu_dereference(dev->rdi.qp_dev->qp_table[n]); qp;
-			qp = rcu_dereference(qp->next)) {
-			if (qp->port_num == ppd->port &&
-			    (qp->ibqp.qp_type == IB_QPT_UC ||
-			     qp->ibqp.qp_type == IB_QPT_RC) &&
-			    rdma_ah_get_sl(&qp->remote_ah_attr) == sl &&
-			    (ib_rvt_state_ops[qp->state] &
-			     RVT_POST_SEND_OK)) {
-				spin_lock_irq(&qp->r_lock);
-				spin_lock(&qp->s_hlock);
-				spin_lock(&qp->s_lock);
-				lastwqe = rvt_error_qp(qp,
-						       IB_WC_WR_FLUSH_ERR);
-				spin_unlock(&qp->s_lock);
-				spin_unlock(&qp->s_hlock);
-				spin_unlock_irq(&qp->r_lock);
-				if (lastwqe) {
-					ev.device = qp->ibqp.device;
-					ev.element.qp = &qp->ibqp;
-					ev.event =
-						IB_EVENT_QP_LAST_WQE_REACHED;
-					qp->ibqp.event_handler(&ev,
-						qp->ibqp.qp_context);
-				}
-			}
-		}
-	}
 
-	rcu_read_unlock();
+	rvt_qp_iter(&dev->rdi, sl, hfi1_qp_iter_cb);
 }
diff --git a/drivers/infiniband/hw/hfi1/qp.h b/drivers/infiniband/hw/hfi1/qp.h
index 6fe542b6a927..c06d2f8348e0 100644
--- a/drivers/infiniband/hw/hfi1/qp.h
+++ b/drivers/infiniband/hw/hfi1/qp.h
@@ -1,7 +1,7 @@
 #ifndef _QP_H
 #define _QP_H
 /*
- * Copyright(c) 2015, 2016 Intel Corporation.
+ * Copyright(c) 2015 - 2017 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -94,26 +94,7 @@ void hfi1_qp_wakeup(struct rvt_qp *qp, u32 flag);
 struct sdma_engine *qp_to_sdma_engine(struct rvt_qp *qp, u8 sc5);
 struct send_context *qp_to_send_context(struct rvt_qp *qp, u8 sc5);
 
-struct qp_iter;
-
-/**
- * qp_iter_init - initialize the iterator for the qp hash list
- * @dev: the hfi1_ibdev
- */
-struct qp_iter *qp_iter_init(struct hfi1_ibdev *dev);
-
-/**
- * qp_iter_next - Find the next qp in the hash list
- * @iter: the iterator for the qp hash list
- */
-int qp_iter_next(struct qp_iter *iter);
-
-/**
- * qp_iter_print - print the qp information to seq_file
- * @s: the seq_file to emit the qp information on
- * @iter: the iterator for the qp hash list
- */
-void qp_iter_print(struct seq_file *s, struct qp_iter *iter);
+void qp_iter_print(struct seq_file *s, struct rvt_qp_iter *iter);
 
 void _hfi1_schedule_send(struct rvt_qp *qp);
 void hfi1_schedule_send(struct rvt_qp *qp);
diff --git a/drivers/infiniband/hw/hfi1/rc.c b/drivers/infiniband/hw/hfi1/rc.c
index 1080778a1f7c..e1cf0c08ca6f 100644
--- a/drivers/infiniband/hw/hfi1/rc.c
+++ b/drivers/infiniband/hw/hfi1/rc.c
@@ -100,8 +100,12 @@ static int make_rc_ack(struct hfi1_ibdev *dev, struct rvt_qp *qp,
 	if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
 		goto bail;
 
-	/* header size in 32-bit words LRH+BTH = (8+12)/4. */
-	hwords = 5;
+	if (priv->hdr_type == HFI1_PKT_TYPE_9B)
+		/* header size in 32-bit words LRH+BTH = (8+12)/4. */
+		hwords = 5;
+	else
+		/* header size in 32-bit words 16B LRH+BTH = (16+12)/4. */
+		hwords = 7;
 
 	switch (qp->s_ack_state) {
 	case OP(RDMA_READ_RESPONSE_LAST):
@@ -258,8 +262,7 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 	struct ib_other_headers *ohdr;
 	struct rvt_sge_state *ss;
 	struct rvt_swqe *wqe;
-	/* header size in 32-bit words LRH+BTH = (8+12)/4. */
-	u32 hwords = 5;
+	u32 hwords;
 	u32 len;
 	u32 bth0 = 0;
 	u32 bth2;
@@ -273,9 +276,23 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 	if (IS_ERR(ps->s_txreq))
 		goto bail_no_tx;
 
-	ohdr = &ps->s_txreq->phdr.hdr.u.oth;
-	if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)
-		ohdr = &ps->s_txreq->phdr.hdr.u.l.oth;
+	ps->s_txreq->phdr.hdr.hdr_type = priv->hdr_type;
+	if (priv->hdr_type == HFI1_PKT_TYPE_9B) {
+		/* header size in 32-bit words LRH+BTH = (8+12)/4. */
+		hwords = 5;
+		if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)
+			ohdr = &ps->s_txreq->phdr.hdr.ibh.u.l.oth;
+		else
+			ohdr = &ps->s_txreq->phdr.hdr.ibh.u.oth;
+	} else {
+		/* header size in 32-bit words 16B LRH+BTH = (16+12)/4. */
+		hwords = 7;
+		if ((rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) &&
+		    (hfi1_check_mcast(rdma_ah_get_dlid(&qp->remote_ah_attr))))
+			ohdr = &ps->s_txreq->phdr.hdr.opah.u.l.oth;
+		else
+			ohdr = &ps->s_txreq->phdr.hdr.opah.u.oth;
+	}
 
 	/* Sending responses has higher priority over sending requests. */
 	if ((qp->s_flags & RVT_S_RESP_PENDING) &&
@@ -425,7 +442,7 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 		case IB_WR_RDMA_WRITE:
 			if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
 				qp->s_lsn++;
-			/* FALLTHROUGH */
+			goto no_flow_control;
 		case IB_WR_RDMA_WRITE_WITH_IMM:
 			/* If no credit, return. */
 			if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT) &&
@@ -433,6 +450,7 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 				qp->s_flags |= RVT_S_WAIT_SSN_CREDIT;
 				goto bail;
 			}
+no_flow_control:
 			put_ib_reth_vaddr(
 				wqe->rdma_wr.remote_addr,
 				&ohdr->u.rc.reth);
@@ -703,6 +721,154 @@ bail_no_tx:
 	return 0;
 }
 
+static inline void hfi1_make_bth_aeth(struct rvt_qp *qp,
+				      struct ib_other_headers *ohdr,
+				      u32 bth0, u32 bth1)
+{
+	if (qp->r_nak_state)
+		ohdr->u.aeth = cpu_to_be32((qp->r_msn & IB_MSN_MASK) |
+					    (qp->r_nak_state <<
+					     IB_AETH_CREDIT_SHIFT));
+	else
+		ohdr->u.aeth = rvt_compute_aeth(qp);
+
+	ohdr->bth[0] = cpu_to_be32(bth0);
+	ohdr->bth[1] = cpu_to_be32(bth1 | qp->remote_qpn);
+	ohdr->bth[2] = cpu_to_be32(mask_psn(qp->r_ack_psn));
+}
+
+static inline void hfi1_queue_rc_ack(struct rvt_qp *qp, bool is_fecn)
+{
+	struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
+	unsigned long flags;
+
+	spin_lock_irqsave(&qp->s_lock, flags);
+	if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
+		goto unlock;
+	this_cpu_inc(*ibp->rvp.rc_qacks);
+	qp->s_flags |= RVT_S_ACK_PENDING | RVT_S_RESP_PENDING;
+	qp->s_nak_state = qp->r_nak_state;
+	qp->s_ack_psn = qp->r_ack_psn;
+	if (is_fecn)
+		qp->s_flags |= RVT_S_ECN;
+
+	/* Schedule the send tasklet. */
+	hfi1_schedule_send(qp);
+unlock:
+	spin_unlock_irqrestore(&qp->s_lock, flags);
+}
+
+static inline void hfi1_make_rc_ack_9B(struct rvt_qp *qp,
+				       struct hfi1_opa_header *opa_hdr,
+				       u8 sc5, bool is_fecn,
+				       u64 *pbc_flags, u32 *hwords,
+				       u32 *nwords)
+{
+	struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
+	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
+	struct ib_header *hdr = &opa_hdr->ibh;
+	struct ib_other_headers *ohdr;
+	u16 lrh0 = HFI1_LRH_BTH;
+	u16 pkey;
+	u32 bth0, bth1;
+
+	opa_hdr->hdr_type = HFI1_PKT_TYPE_9B;
+	ohdr = &hdr->u.oth;
+	/* header size in 32-bit words LRH+BTH+AETH = (8+12+4)/4 */
+	*hwords = 6;
+
+	if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)) {
+		*hwords += hfi1_make_grh(ibp, &hdr->u.l.grh,
+					 rdma_ah_read_grh(&qp->remote_ah_attr),
+					 *hwords - 2, SIZE_OF_CRC);
+		ohdr = &hdr->u.l.oth;
+		lrh0 = HFI1_LRH_GRH;
+	}
+	/* set PBC_DC_INFO bit (aka SC[4]) in pbc_flags */
+	*pbc_flags |= ((!!(sc5 & 0x10)) << PBC_DC_INFO_SHIFT);
+
+	/* read pkey_index w/o lock (its atomic) */
+	pkey = hfi1_get_pkey(ibp, qp->s_pkey_index);
+
+	lrh0 |= (sc5 & IB_SC_MASK) << IB_SC_SHIFT |
+		(rdma_ah_get_sl(&qp->remote_ah_attr) & IB_SL_MASK) <<
+			IB_SL_SHIFT;
+
+	hfi1_make_ib_hdr(hdr, lrh0, *hwords + SIZE_OF_CRC,
+			 opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr), 9B),
+			 ppd->lid | rdma_ah_get_path_bits(&qp->remote_ah_attr));
+
+	bth0 = pkey | (OP(ACKNOWLEDGE) << 24);
+	if (qp->s_mig_state == IB_MIG_MIGRATED)
+		bth0 |= IB_BTH_MIG_REQ;
+	bth1 = (!!is_fecn) << IB_BECN_SHIFT;
+	hfi1_make_bth_aeth(qp, ohdr, bth0, bth1);
+}
+
+static inline void hfi1_make_rc_ack_16B(struct rvt_qp *qp,
+					struct hfi1_opa_header *opa_hdr,
+					u8 sc5, bool is_fecn,
+					u64 *pbc_flags, u32 *hwords,
+					u32 *nwords)
+{
+	struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
+	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
+	struct hfi1_16b_header *hdr = &opa_hdr->opah;
+	struct ib_other_headers *ohdr;
+	u32 bth0, bth1;
+	u16 len, pkey;
+	u8 becn = !!is_fecn;
+	u8 l4 = OPA_16B_L4_IB_LOCAL;
+	u8 extra_bytes;
+
+	opa_hdr->hdr_type = HFI1_PKT_TYPE_16B;
+	ohdr = &hdr->u.oth;
+	/* header size in 32-bit words 16B LRH+BTH+AETH = (16+12+4)/4 */
+	*hwords = 8;
+	extra_bytes = hfi1_get_16b_padding(*hwords << 2, 0);
+	*nwords = SIZE_OF_CRC + ((extra_bytes + SIZE_OF_LT) >> 2);
+
+	if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) &&
+	    hfi1_check_mcast(rdma_ah_get_dlid(&qp->remote_ah_attr))) {
+		*hwords += hfi1_make_grh(ibp, &hdr->u.l.grh,
+					 rdma_ah_read_grh(&qp->remote_ah_attr),
+					 *hwords - 4, *nwords);
+		ohdr = &hdr->u.l.oth;
+		l4 = OPA_16B_L4_IB_GLOBAL;
+	}
+	*pbc_flags |= PBC_PACKET_BYPASS | PBC_INSERT_BYPASS_ICRC;
+
+	/* read pkey_index w/o lock (its atomic) */
+	pkey = hfi1_get_pkey(ibp, qp->s_pkey_index);
+
+	/* Convert dwords to flits */
+	len = (*hwords + *nwords) >> 1;
+
+	hfi1_make_16b_hdr(hdr,
+			  ppd->lid | rdma_ah_get_path_bits(&qp->remote_ah_attr),
+			  opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr),
+				      16B),
+			  len, pkey, becn, 0, l4, sc5);
+
+	bth0 = pkey | (OP(ACKNOWLEDGE) << 24);
+	bth0 |= extra_bytes << 20;
+	if (qp->s_mig_state == IB_MIG_MIGRATED)
+		bth1 = OPA_BTH_MIG_REQ;
+	hfi1_make_bth_aeth(qp, ohdr, bth0, bth1);
+}
+
+typedef void (*hfi1_make_rc_ack)(struct rvt_qp *qp,
+				 struct hfi1_opa_header *opa_hdr,
+				 u8 sc5, bool is_fecn,
+				 u64 *pbc_flags, u32 *hwords,
+				 u32 *nwords);
+
+/* We support only two types - 9B and 16B for now */
+static const hfi1_make_rc_ack hfi1_make_rc_ack_tbl[2] = {
+	[HFI1_PKT_TYPE_9B] = &hfi1_make_rc_ack_9B,
+	[HFI1_PKT_TYPE_16B] = &hfi1_make_rc_ack_16B
+};
+
 /**
  * hfi1_send_rc_ack - Construct an ACK packet and send it
  * @qp: a pointer to the QP
@@ -711,83 +877,48 @@ bail_no_tx:
  * Note that RDMA reads and atomics are handled in the
  * send side QP state and send engine.
  */
-void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp,
-		      int is_fecn)
+void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd,
+		      struct rvt_qp *qp, bool is_fecn)
 {
 	struct hfi1_ibport *ibp = rcd_to_iport(rcd);
+	struct hfi1_qp_priv *priv = qp->priv;
 	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
+	u8 sc5 = ibp->sl_to_sc[rdma_ah_get_sl(&qp->remote_ah_attr)];
 	u64 pbc, pbc_flags = 0;
-	u16 lrh0;
-	u16 sc5;
-	u32 bth0;
-	u32 hwords;
-	u32 vl, plen;
-	struct send_context *sc;
+	u32 hwords = 0;
+	u32 nwords = 0;
+	u32 plen;
 	struct pio_buf *pbuf;
-	struct ib_header hdr;
-	struct ib_other_headers *ohdr;
-	unsigned long flags;
+	struct hfi1_opa_header opa_hdr;
 
 	/* clear the defer count */
 	qp->r_adefered = 0;
 
 	/* Don't send ACK or NAK if a RDMA read or atomic is pending. */
-	if (qp->s_flags & RVT_S_RESP_PENDING)
-		goto queue_ack;
+	if (qp->s_flags & RVT_S_RESP_PENDING) {
+		hfi1_queue_rc_ack(qp, is_fecn);
+		return;
+	}
 
 	/* Ensure s_rdma_ack_cnt changes are committed */
 	smp_read_barrier_depends();
-	if (qp->s_rdma_ack_cnt)
-		goto queue_ack;
-
-	/* Construct the header */
-	/* header size in 32-bit words LRH+BTH+AETH = (8+12+4)/4 */
-	hwords = 6;
-	if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)) {
-		hwords += hfi1_make_grh(ibp, &hdr.u.l.grh,
-					rdma_ah_read_grh(&qp->remote_ah_attr),
-					hwords, 0);
-		ohdr = &hdr.u.l.oth;
-		lrh0 = HFI1_LRH_GRH;
-	} else {
-		ohdr = &hdr.u.oth;
-		lrh0 = HFI1_LRH_BTH;
+	if (qp->s_rdma_ack_cnt) {
+		hfi1_queue_rc_ack(qp, is_fecn);
+		return;
 	}
-	/* read pkey_index w/o lock (its atomic) */
-	bth0 = hfi1_get_pkey(ibp, qp->s_pkey_index) | (OP(ACKNOWLEDGE) << 24);
-	if (qp->s_mig_state == IB_MIG_MIGRATED)
-		bth0 |= IB_BTH_MIG_REQ;
-	if (qp->r_nak_state)
-		ohdr->u.aeth = cpu_to_be32((qp->r_msn & IB_MSN_MASK) |
-					    (qp->r_nak_state <<
-					     IB_AETH_CREDIT_SHIFT));
-	else
-		ohdr->u.aeth = rvt_compute_aeth(qp);
-	sc5 = ibp->sl_to_sc[rdma_ah_get_sl(&qp->remote_ah_attr)];
-	/* set PBC_DC_INFO bit (aka SC[4]) in pbc_flags */
-	pbc_flags |= ((!!(sc5 & 0x10)) << PBC_DC_INFO_SHIFT);
-	lrh0 |= (sc5 & 0xf) << 12 | (rdma_ah_get_sl(&qp->remote_ah_attr)
-				     & 0xf) << 4;
-	hdr.lrh[0] = cpu_to_be16(lrh0);
-	hdr.lrh[1] = cpu_to_be16(rdma_ah_get_dlid(&qp->remote_ah_attr));
-	hdr.lrh[2] = cpu_to_be16(hwords + SIZE_OF_CRC);
-	hdr.lrh[3] = cpu_to_be16(ppd->lid |
-				 rdma_ah_get_path_bits(&qp->remote_ah_attr));
-	ohdr->bth[0] = cpu_to_be32(bth0);
-	ohdr->bth[1] = cpu_to_be32(qp->remote_qpn);
-	ohdr->bth[1] |= cpu_to_be32((!!is_fecn) << IB_BECN_SHIFT);
-	ohdr->bth[2] = cpu_to_be32(mask_psn(qp->r_ack_psn));
 
 	/* Don't try to send ACKs if the link isn't ACTIVE */
 	if (driver_lstate(ppd) != IB_PORT_ACTIVE)
 		return;
 
-	sc = rcd->sc;
-	plen = 2 /* PBC */ + hwords;
-	vl = sc_to_vlt(ppd->dd, sc5);
-	pbc = create_pbc(ppd, pbc_flags, qp->srate_mbps, vl, plen);
+	/* Make the appropriate header */
+	hfi1_make_rc_ack_tbl[priv->hdr_type](qp, &opa_hdr, sc5, is_fecn,
+					     &pbc_flags, &hwords, &nwords);
 
-	pbuf = sc_buffer_alloc(sc, plen, NULL, NULL);
+	plen = 2 /* PBC */ + hwords + nwords;
+	pbc = create_pbc(ppd, pbc_flags, qp->srate_mbps,
+			 sc_to_vlt(ppd->dd, sc5), plen);
+	pbuf = sc_buffer_alloc(rcd->sc, plen, NULL, NULL);
 	if (!pbuf) {
 		/*
 		 * We have no room to send at the moment.  Pass
@@ -795,31 +926,18 @@ void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp,
 		 * so that when enough buffer space becomes available,
 		 * the ACK is sent ahead of other outgoing packets.
 		 */
-		goto queue_ack;
+		hfi1_queue_rc_ack(qp, is_fecn);
+		return;
 	}
-
-	trace_ack_output_ibhdr(dd_from_ibdev(qp->ibqp.device), &hdr);
+	trace_ack_output_ibhdr(dd_from_ibdev(qp->ibqp.device),
+			       &opa_hdr, ib_is_sc5(sc5));
 
 	/* write the pbc and data */
-	ppd->dd->pio_inline_send(ppd->dd, pbuf, pbc, &hdr, hwords);
-
+	ppd->dd->pio_inline_send(ppd->dd, pbuf, pbc,
+				 (priv->hdr_type == HFI1_PKT_TYPE_9B ?
+				 (void *)&opa_hdr.ibh :
+				 (void *)&opa_hdr.opah), hwords);
 	return;
-
-queue_ack:
-	spin_lock_irqsave(&qp->s_lock, flags);
-	if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
-		goto unlock;
-	this_cpu_inc(*ibp->rvp.rc_qacks);
-	qp->s_flags |= RVT_S_ACK_PENDING | RVT_S_RESP_PENDING;
-	qp->s_nak_state = qp->r_nak_state;
-	qp->s_ack_psn = qp->r_ack_psn;
-	if (is_fecn)
-		qp->s_flags |= RVT_S_ECN;
-
-	/* Schedule the send engine. */
-	hfi1_schedule_send(qp);
-unlock:
-	spin_unlock_irqrestore(&qp->s_lock, flags);
 }
 
 /**
@@ -984,10 +1102,13 @@ static void reset_sending_psn(struct rvt_qp *qp, u32 psn)
 /*
  * This should be called with the QP s_lock held and interrupts disabled.
  */
-void hfi1_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr)
+void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah)
 {
 	struct ib_other_headers *ohdr;
+	struct hfi1_qp_priv *priv = qp->priv;
 	struct rvt_swqe *wqe;
+	struct ib_header *hdr = NULL;
+	struct hfi1_16b_header *hdr_16b = NULL;
 	u32 opcode;
 	u32 psn;
 
@@ -996,10 +1117,22 @@ void hfi1_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr)
 		return;
 
 	/* Find out where the BTH is */
-	if (ib_get_lnh(hdr) == HFI1_LRH_BTH)
-		ohdr = &hdr->u.oth;
-	else
-		ohdr = &hdr->u.l.oth;
+	if (priv->hdr_type == HFI1_PKT_TYPE_9B) {
+		hdr = &opah->ibh;
+		if (ib_get_lnh(hdr) == HFI1_LRH_BTH)
+			ohdr = &hdr->u.oth;
+		else
+			ohdr = &hdr->u.l.oth;
+	} else {
+		u8 l4;
+
+		hdr_16b = &opah->opah;
+		l4  = hfi1_16B_get_l4(hdr_16b);
+		if (l4 == OPA_16B_L4_IB_LOCAL)
+			ohdr = &hdr_16b->u.oth;
+		else
+			ohdr = &hdr_16b->u.l.oth;
+	}
 
 	opcode = ib_bth_get_opcode(ohdr);
 	if (opcode >= OP(RDMA_READ_RESPONSE_FIRST) &&
@@ -1009,7 +1142,7 @@ void hfi1_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr)
 		return;
 	}
 
-	psn = be32_to_cpu(ohdr->bth[2]);
+	psn = ib_bth_get_psn(ohdr);
 	reset_sending_psn(qp, psn);
 
 	/*
@@ -1399,36 +1532,34 @@ static void rdma_seq_err(struct rvt_qp *qp, struct hfi1_ibport *ibp, u32 psn,
 
 /**
  * rc_rcv_resp - process an incoming RC response packet
- * @ibp: the port this packet came in on
- * @ohdr: the other headers for this packet
- * @data: the packet data
- * @tlen: the packet length
- * @qp: the QP for this packet
- * @opcode: the opcode for this packet
- * @psn: the packet sequence number for this packet
- * @hdrsize: the header length
- * @pmtu: the path MTU
+ * @packet: data packet information
  *
  * This is called from hfi1_rc_rcv() to process an incoming RC response
  * packet for the given QP.
  * Called at interrupt level.
  */
-static void rc_rcv_resp(struct hfi1_ibport *ibp,
-			struct ib_other_headers *ohdr,
-			void *data, u32 tlen, struct rvt_qp *qp,
-			u32 opcode, u32 psn, u32 hdrsize, u32 pmtu,
-			struct hfi1_ctxtdata *rcd)
+static void rc_rcv_resp(struct hfi1_packet *packet)
 {
+	struct hfi1_ctxtdata *rcd = packet->rcd;
+	void *data = packet->payload;
+	u32 tlen = packet->tlen;
+	struct rvt_qp *qp = packet->qp;
+	struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
+	struct ib_other_headers *ohdr = packet->ohdr;
 	struct rvt_swqe *wqe;
 	enum ib_wc_status status;
 	unsigned long flags;
 	int diff;
-	u32 pad;
-	u32 aeth;
 	u64 val;
+	u32 aeth;
+	u32 psn = ib_bth_get_psn(packet->ohdr);
+	u32 pmtu = qp->pmtu;
+	u16 hdrsize = packet->hlen;
+	u8 opcode = packet->opcode;
+	u8 pad = packet->pad;
+	u8 extra_bytes = pad + packet->extra_byte + (SIZE_OF_CRC << 2);
 
 	spin_lock_irqsave(&qp->s_lock, flags);
-
 	trace_hfi1_ack(qp, psn);
 
 	/* Ignore invalid responses. */
@@ -1494,7 +1625,7 @@ static void rc_rcv_resp(struct hfi1_ibport *ibp,
 		if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
 			goto ack_op_err;
 read_middle:
-		if (unlikely(tlen != (hdrsize + pmtu + 4)))
+		if (unlikely(tlen != (hdrsize + pmtu + extra_bytes)))
 			goto ack_len_err;
 		if (unlikely(pmtu >= qp->s_rdma_read_len))
 			goto ack_len_err;
@@ -1526,13 +1657,11 @@ read_middle:
 		aeth = be32_to_cpu(ohdr->u.aeth);
 		if (!do_rc_ack(qp, aeth, psn, opcode, 0, rcd))
 			goto ack_done;
-		/* Get the number of bytes the message was padded by. */
-		pad = ib_bth_get_pad(ohdr);
 		/*
 		 * Check that the data size is >= 0 && <= pmtu.
 		 * Remember to account for ICRC (4).
 		 */
-		if (unlikely(tlen < (hdrsize + pad + 4)))
+		if (unlikely(tlen < (hdrsize + extra_bytes)))
 			goto ack_len_err;
 		/*
 		 * If this is a response to a resent RDMA read, we
@@ -1550,16 +1679,14 @@ read_middle:
 			goto ack_seq_err;
 		if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
 			goto ack_op_err;
-		/* Get the number of bytes the message was padded by. */
-		pad = ib_bth_get_pad(ohdr);
 		/*
 		 * Check that the data size is >= 1 && <= pmtu.
 		 * Remember to account for ICRC (4).
 		 */
-		if (unlikely(tlen <= (hdrsize + pad + 4)))
+		if (unlikely(tlen <= (hdrsize + extra_bytes)))
 			goto ack_len_err;
 read_last:
-		tlen -= hdrsize + pad + 4;
+		tlen -= hdrsize + extra_bytes;
 		if (unlikely(tlen != qp->s_rdma_read_len))
 			goto ack_len_err;
 		aeth = be32_to_cpu(ohdr->u.aeth);
@@ -1844,7 +1971,7 @@ static void log_cca_event(struct hfi1_pportdata *ppd, u8 sl, u32 rlid,
 	spin_unlock_irqrestore(&ppd->cc_log_lock, flags);
 }
 
-void process_becn(struct hfi1_pportdata *ppd, u8 sl, u16 rlid, u32 lqpn,
+void process_becn(struct hfi1_pportdata *ppd, u8 sl, u32 rlid, u32 lqpn,
 		  u32 rqpn, u8 svc_type)
 {
 	struct cca_timer *cca_timer;
@@ -1901,12 +2028,7 @@ void process_becn(struct hfi1_pportdata *ppd, u8 sl, u16 rlid, u32 lqpn,
 
 /**
  * hfi1_rc_rcv - process an incoming RC packet
- * @rcd: the context pointer
- * @hdr: the header of this packet
- * @rcv_flags: flags relevant to rcv processing
- * @data: the packet data
- * @tlen: the packet length
- * @qp: the QP for this packet
+ * @packet: data packet information
  *
  * This is called from qp_rcv() to process an incoming RC packet
  * for the given QP.
@@ -1915,17 +2037,16 @@ void process_becn(struct hfi1_pportdata *ppd, u8 sl, u16 rlid, u32 lqpn,
 void hfi1_rc_rcv(struct hfi1_packet *packet)
 {
 	struct hfi1_ctxtdata *rcd = packet->rcd;
-	struct ib_header *hdr = packet->hdr;
-	u32 rcv_flags = packet->rcv_flags;
-	void *data = packet->ebuf;
+	void *data = packet->payload;
 	u32 tlen = packet->tlen;
 	struct rvt_qp *qp = packet->qp;
 	struct hfi1_ibport *ibp = rcd_to_iport(rcd);
 	struct ib_other_headers *ohdr = packet->ohdr;
-	u32 bth0, opcode;
+	u32 bth0 = be32_to_cpu(ohdr->bth[0]);
+	u32 opcode = packet->opcode;
 	u32 hdrsize = packet->hlen;
-	u32 psn;
-	u32 pad;
+	u32 psn = ib_bth_get_psn(packet->ohdr);
+	u32 pad = packet->pad;
 	struct ib_wc wc;
 	u32 pmtu = qp->pmtu;
 	int diff;
@@ -1935,17 +2056,15 @@ void hfi1_rc_rcv(struct hfi1_packet *packet)
 	bool is_fecn = false;
 	bool copy_last = false;
 	u32 rkey;
+	u8 extra_bytes = pad + packet->extra_byte + (SIZE_OF_CRC << 2);
 
 	lockdep_assert_held(&qp->r_lock);
-	bth0 = be32_to_cpu(ohdr->bth[0]);
-	if (hfi1_ruc_check_hdr(ibp, hdr, rcv_flags & HFI1_HAS_GRH, qp, bth0))
+
+	if (hfi1_ruc_check_hdr(ibp, packet))
 		return;
 
 	is_fecn = process_ecn(qp, packet, false);
 
-	psn = be32_to_cpu(ohdr->bth[2]);
-	opcode = ib_bth_get_opcode(ohdr);
-
 	/*
 	 * Process responses (ACKs) before anything else.  Note that the
 	 * packet sequence number will be for something in the send work
@@ -1954,8 +2073,7 @@ void hfi1_rc_rcv(struct hfi1_packet *packet)
 	 */
 	if (opcode >= OP(RDMA_READ_RESPONSE_FIRST) &&
 	    opcode <= OP(ATOMIC_ACKNOWLEDGE)) {
-		rc_rcv_resp(ibp, ohdr, data, tlen, qp, opcode, psn,
-			    hdrsize, pmtu, rcd);
+		rc_rcv_resp(packet);
 		if (is_fecn)
 			goto send_ack;
 		return;
@@ -2022,7 +2140,12 @@ void hfi1_rc_rcv(struct hfi1_packet *packet)
 	case OP(RDMA_WRITE_MIDDLE):
 send_middle:
 		/* Check for invalid length PMTU or posted rwqe len. */
-		if (unlikely(tlen != (hdrsize + pmtu + 4)))
+		/*
+		 * There will be no padding for 9B packet but 16B packets
+		 * will come in with some padding since we always add
+		 * CRC and LT bytes which will need to be flit aligned
+		 */
+		if (unlikely(tlen != (hdrsize + pmtu + extra_bytes)))
 			goto nack_inv;
 		qp->r_rcv_len += pmtu;
 		if (unlikely(qp->r_rcv_len > qp->r_len))
@@ -2074,14 +2197,12 @@ no_immediate_data:
 		wc.wc_flags = 0;
 		wc.ex.imm_data = 0;
 send_last:
-		/* Get the number of bytes the message was padded by. */
-		pad = ib_bth_get_pad(ohdr);
 		/* Check for invalid length. */
 		/* LAST len should be >= 1 */
-		if (unlikely(tlen < (hdrsize + pad + 4)))
+		if (unlikely(tlen < (hdrsize + extra_bytes)))
 			goto nack_inv;
-		/* Don't count the CRC. */
-		tlen -= (hdrsize + pad + 4);
+		/* Don't count the CRC(and padding and LT byte for 16B). */
+		tlen -= (hdrsize + extra_bytes);
 		wc.byte_len = tlen + qp->r_rcv_len;
 		if (unlikely(wc.byte_len > qp->r_len))
 			goto nack_inv;
@@ -2368,28 +2489,19 @@ send_ack:
 
 void hfi1_rc_hdrerr(
 	struct hfi1_ctxtdata *rcd,
-	struct ib_header *hdr,
-	u32 rcv_flags,
+	struct hfi1_packet *packet,
 	struct rvt_qp *qp)
 {
-	int has_grh = rcv_flags & HFI1_HAS_GRH;
-	struct ib_other_headers *ohdr;
 	struct hfi1_ibport *ibp = rcd_to_iport(rcd);
 	int diff;
 	u32 opcode;
-	u32 psn, bth0;
-
-	/* Check for GRH */
-	ohdr = &hdr->u.oth;
-	if (has_grh)
-		ohdr = &hdr->u.l.oth;
+	u32 psn;
 
-	bth0 = be32_to_cpu(ohdr->bth[0]);
-	if (hfi1_ruc_check_hdr(ibp, hdr, has_grh, qp, bth0))
+	if (hfi1_ruc_check_hdr(ibp, packet))
 		return;
 
-	psn = be32_to_cpu(ohdr->bth[2]);
-	opcode = ib_bth_get_opcode(ohdr);
+	psn = ib_bth_get_psn(packet->ohdr);
+	opcode = ib_bth_get_opcode(packet->ohdr);
 
 	/* Only deal with RDMA Writes for now */
 	if (opcode < IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST) {
diff --git a/drivers/infiniband/hw/hfi1/ruc.c b/drivers/infiniband/hw/hfi1/ruc.c
index 3a17daba28a9..b3291f0fde9a 100644
--- a/drivers/infiniband/hw/hfi1/ruc.c
+++ b/drivers/infiniband/hw/hfi1/ruc.c
@@ -74,8 +74,10 @@ static int init_sge(struct rvt_qp *qp, struct rvt_rwqe *wqe)
 		if (wqe->sg_list[i].length == 0)
 			continue;
 		/* Check LKEY */
-		if (!rvt_lkey_ok(rkt, pd, j ? &ss->sg_list[j - 1] : &ss->sge,
-				 &wqe->sg_list[i], IB_ACCESS_LOCAL_WRITE))
+		ret = rvt_lkey_ok(rkt, pd, j ? &ss->sg_list[j - 1] : &ss->sge,
+				  NULL, &wqe->sg_list[i],
+				  IB_ACCESS_LOCAL_WRITE);
+		if (unlikely(ret <= 0))
 			goto bad_lkey;
 		qp->r_len += wqe->sg_list[i].length;
 		j++;
@@ -214,100 +216,104 @@ static int gid_ok(union ib_gid *gid, __be64 gid_prefix, __be64 id)
  *
  * The s_lock will be acquired around the hfi1_migrate_qp() call.
  */
-int hfi1_ruc_check_hdr(struct hfi1_ibport *ibp, struct ib_header *hdr,
-		       int has_grh, struct rvt_qp *qp, u32 bth0)
+int hfi1_ruc_check_hdr(struct hfi1_ibport *ibp, struct hfi1_packet *packet)
 {
 	__be64 guid;
 	unsigned long flags;
+	struct rvt_qp *qp = packet->qp;
 	u8 sc5 = ibp->sl_to_sc[rdma_ah_get_sl(&qp->remote_ah_attr)];
+	u32 dlid = packet->dlid;
+	u32 slid = packet->slid;
+	u32 sl = packet->sl;
+	int migrated;
+	u32 bth0, bth1;
+	u16 pkey;
+
+	bth0 = be32_to_cpu(packet->ohdr->bth[0]);
+	bth1 = be32_to_cpu(packet->ohdr->bth[1]);
+	if (packet->etype == RHF_RCV_TYPE_BYPASS) {
+		pkey = hfi1_16B_get_pkey(packet->hdr);
+		migrated = bth1 & OPA_BTH_MIG_REQ;
+	} else {
+		pkey = ib_bth_get_pkey(packet->ohdr);
+		migrated = bth0 & IB_BTH_MIG_REQ;
+	}
 
-	if (qp->s_mig_state == IB_MIG_ARMED && (bth0 & IB_BTH_MIG_REQ)) {
-		if (!has_grh) {
-			if (rdma_ah_get_ah_flags(&qp->alt_ah_attr) &
-			    IB_AH_GRH)
-				goto err;
+	if (qp->s_mig_state == IB_MIG_ARMED && migrated) {
+		if (!packet->grh) {
+			if ((rdma_ah_get_ah_flags(&qp->alt_ah_attr) &
+			     IB_AH_GRH) &&
+			    (packet->etype != RHF_RCV_TYPE_BYPASS))
+				return 1;
 		} else {
 			const struct ib_global_route *grh;
 
 			if (!(rdma_ah_get_ah_flags(&qp->alt_ah_attr) &
 			      IB_AH_GRH))
-				goto err;
+				return 1;
 			grh = rdma_ah_read_grh(&qp->alt_ah_attr);
 			guid = get_sguid(ibp, grh->sgid_index);
-			if (!gid_ok(&hdr->u.l.grh.dgid, ibp->rvp.gid_prefix,
+			if (!gid_ok(&packet->grh->dgid, ibp->rvp.gid_prefix,
 				    guid))
-				goto err;
+				return 1;
 			if (!gid_ok(
-				&hdr->u.l.grh.sgid,
+				&packet->grh->sgid,
 				grh->dgid.global.subnet_prefix,
 				grh->dgid.global.interface_id))
-				goto err;
+				return 1;
 		}
-		if (unlikely(rcv_pkey_check(ppd_from_ibp(ibp), (u16)bth0, sc5,
-					    ib_get_slid(hdr)))) {
-			hfi1_bad_pqkey(ibp, OPA_TRAP_BAD_P_KEY,
-				       (u16)bth0,
-				       ib_get_sl(hdr),
-				       0, qp->ibqp.qp_num,
-				       ib_get_slid(hdr),
-				       ib_get_dlid(hdr));
-			goto err;
+		if (unlikely(rcv_pkey_check(ppd_from_ibp(ibp), pkey,
+					    sc5, slid))) {
+			hfi1_bad_pkey(ibp, pkey, sl, 0, qp->ibqp.qp_num,
+				      slid, dlid);
+			return 1;
 		}
 		/* Validate the SLID. See Ch. 9.6.1.5 and 17.2.8 */
-		if (ib_get_slid(hdr) !=
-			rdma_ah_get_dlid(&qp->alt_ah_attr) ||
+		if (slid != rdma_ah_get_dlid(&qp->alt_ah_attr) ||
 		    ppd_from_ibp(ibp)->port !=
 			rdma_ah_get_port_num(&qp->alt_ah_attr))
-			goto err;
+			return 1;
 		spin_lock_irqsave(&qp->s_lock, flags);
 		hfi1_migrate_qp(qp);
 		spin_unlock_irqrestore(&qp->s_lock, flags);
 	} else {
-		if (!has_grh) {
-			if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) &
-						 IB_AH_GRH)
-				goto err;
+		if (!packet->grh) {
+			if ((rdma_ah_get_ah_flags(&qp->remote_ah_attr) &
+			     IB_AH_GRH) &&
+			    (packet->etype != RHF_RCV_TYPE_BYPASS))
+				return 1;
 		} else {
 			const struct ib_global_route *grh;
 
 			if (!(rdma_ah_get_ah_flags(&qp->remote_ah_attr) &
 						   IB_AH_GRH))
-				goto err;
+				return 1;
 			grh = rdma_ah_read_grh(&qp->remote_ah_attr);
 			guid = get_sguid(ibp, grh->sgid_index);
-			if (!gid_ok(&hdr->u.l.grh.dgid, ibp->rvp.gid_prefix,
+			if (!gid_ok(&packet->grh->dgid, ibp->rvp.gid_prefix,
 				    guid))
-				goto err;
+				return 1;
 			if (!gid_ok(
-			     &hdr->u.l.grh.sgid,
+			     &packet->grh->sgid,
 			     grh->dgid.global.subnet_prefix,
 			     grh->dgid.global.interface_id))
-				goto err;
+				return 1;
 		}
-		if (unlikely(rcv_pkey_check(ppd_from_ibp(ibp), (u16)bth0, sc5,
-					    ib_get_slid(hdr)))) {
-			hfi1_bad_pqkey(ibp, OPA_TRAP_BAD_P_KEY,
-				       (u16)bth0,
-				       ib_get_sl(hdr),
-				       0, qp->ibqp.qp_num,
-				       ib_get_slid(hdr),
-				       ib_get_dlid(hdr));
-			goto err;
+		if (unlikely(rcv_pkey_check(ppd_from_ibp(ibp), pkey,
+					    sc5, slid))) {
+			hfi1_bad_pkey(ibp, pkey, sl, 0, qp->ibqp.qp_num,
+				      slid, dlid);
+			return 1;
 		}
 		/* Validate the SLID. See Ch. 9.6.1.5 */
-		if (ib_get_slid(hdr) !=
-			rdma_ah_get_dlid(&qp->remote_ah_attr) ||
+		if ((slid != rdma_ah_get_dlid(&qp->remote_ah_attr)) ||
 		    ppd_from_ibp(ibp)->port != qp->port_num)
-			goto err;
-		if (qp->s_mig_state == IB_MIG_REARM &&
-		    !(bth0 & IB_BTH_MIG_REQ))
+			return 1;
+		if (qp->s_mig_state == IB_MIG_REARM && !migrated)
 			qp->s_mig_state = IB_MIG_ARMED;
 	}
 
 	return 0;
-
-err:
-	return 1;
 }
 
 /**
@@ -643,7 +649,7 @@ done:
  * @ibp: a pointer to the IB port
  * @hdr: a pointer to the GRH header being constructed
  * @grh: the global route address to send to
- * @hwords: the number of 32 bit words of header being sent
+ * @hwords: size of header after grh being sent in dwords
  * @nwords: the number of 32 bit words of data being sent
  *
  * Return the size of the header in 32 bit words.
@@ -655,7 +661,7 @@ u32 hfi1_make_grh(struct hfi1_ibport *ibp, struct ib_grh *hdr,
 		cpu_to_be32((IB_GRH_VERSION << IB_GRH_VERSION_SHIFT) |
 			    (grh->traffic_class << IB_GRH_TCLASS_SHIFT) |
 			    (grh->flow_label << IB_GRH_FLOW_SHIFT));
-	hdr->paylen = cpu_to_be16((hwords - 2 + nwords + SIZE_OF_CRC) << 2);
+	hdr->paylen = cpu_to_be16((hwords + nwords) << 2);
 	/* next_hdr is defined by C8-7 in ch. 8.4.1 */
 	hdr->next_hdr = IB_GRH_NEXT_HDR;
 	hdr->hop_limit = grh->hop_limit;
@@ -671,7 +677,8 @@ u32 hfi1_make_grh(struct hfi1_ibport *ibp, struct ib_grh *hdr,
 	return sizeof(struct ib_grh) / sizeof(u32);
 }
 
-#define BTH2_OFFSET (offsetof(struct hfi1_sdma_header, hdr.u.oth.bth[2]) / 4)
+#define BTH2_OFFSET (offsetof(struct hfi1_sdma_header, \
+			      hdr.ibh.u.oth.bth[2]) / 4)
 
 /**
  * build_ahg - create ahg in s_ahg
@@ -728,73 +735,186 @@ static inline void build_ahg(struct rvt_qp *qp, u32 npsn)
 	}
 }
 
-void hfi1_make_ruc_header(struct rvt_qp *qp, struct ib_other_headers *ohdr,
-			  u32 bth0, u32 bth2, int middle,
-			  struct hfi1_pkt_state *ps)
+static inline void hfi1_make_ruc_bth(struct rvt_qp *qp,
+				     struct ib_other_headers *ohdr,
+				     u32 bth0, u32 bth1, u32 bth2)
+{
+	bth1 |= qp->remote_qpn;
+	ohdr->bth[0] = cpu_to_be32(bth0);
+	ohdr->bth[1] = cpu_to_be32(bth1);
+	ohdr->bth[2] = cpu_to_be32(bth2);
+}
+
+static inline void hfi1_make_ruc_header_16B(struct rvt_qp *qp,
+					    struct ib_other_headers *ohdr,
+					    u32 bth0, u32 bth2, int middle,
+					    struct hfi1_pkt_state *ps)
 {
 	struct hfi1_qp_priv *priv = qp->priv;
 	struct hfi1_ibport *ibp = ps->ibp;
-	u16 lrh0;
-	u32 nwords;
-	u32 extra_bytes;
-	u32 bth1;
-
-	/* Construct the header. */
-	extra_bytes = -ps->s_txreq->s_cur_size & 3;
-	nwords = (ps->s_txreq->s_cur_size + extra_bytes) >> 2;
-	lrh0 = HFI1_LRH_BTH;
+	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
+	u32 bth1 = 0;
+	u32 slid;
+	u16 pkey = hfi1_get_pkey(ibp, qp->s_pkey_index);
+	u8 l4 = OPA_16B_L4_IB_LOCAL;
+	u8 extra_bytes = hfi1_get_16b_padding((qp->s_hdrwords << 2),
+				   ps->s_txreq->s_cur_size);
+	u32 nwords = SIZE_OF_CRC + ((ps->s_txreq->s_cur_size +
+				 extra_bytes + SIZE_OF_LT) >> 2);
+	u8 becn = 0;
+
+	if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) &&
+	    hfi1_check_mcast(rdma_ah_get_dlid(&qp->remote_ah_attr))) {
+		struct ib_grh *grh;
+		struct ib_global_route *grd =
+			rdma_ah_retrieve_grh(&qp->remote_ah_attr);
+		int hdrwords;
+
+		/*
+		 * Ensure OPA GIDs are transformed to IB gids
+		 * before creating the GRH.
+		 */
+		if (grd->sgid_index == OPA_GID_INDEX)
+			grd->sgid_index = 0;
+		grh = &ps->s_txreq->phdr.hdr.opah.u.l.grh;
+		l4 = OPA_16B_L4_IB_GLOBAL;
+		hdrwords = qp->s_hdrwords - 4;
+		qp->s_hdrwords += hfi1_make_grh(ibp, grh, grd,
+						hdrwords, nwords);
+		middle = 0;
+	}
+
+	if (qp->s_mig_state == IB_MIG_MIGRATED)
+		bth1 |= OPA_BTH_MIG_REQ;
+	else
+		middle = 0;
+
+	if (middle)
+		build_ahg(qp, bth2);
+	else
+		qp->s_flags &= ~RVT_S_AHG_VALID;
+
+	bth0 |= pkey;
+	bth0 |= extra_bytes << 20;
+	if (qp->s_flags & RVT_S_ECN) {
+		qp->s_flags &= ~RVT_S_ECN;
+		/* we recently received a FECN, so return a BECN */
+		becn = 1;
+	}
+	hfi1_make_ruc_bth(qp, ohdr, bth0, bth1, bth2);
+
+	if (!ppd->lid)
+		slid = be32_to_cpu(OPA_LID_PERMISSIVE);
+	else
+		slid = ppd->lid |
+			(rdma_ah_get_path_bits(&qp->remote_ah_attr) &
+			((1 << ppd->lmc) - 1));
+
+	hfi1_make_16b_hdr(&ps->s_txreq->phdr.hdr.opah,
+			  slid,
+			  opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr),
+				      16B),
+			  (qp->s_hdrwords + nwords) >> 1,
+			  pkey, becn, 0, l4, priv->s_sc);
+}
+
+static inline void hfi1_make_ruc_header_9B(struct rvt_qp *qp,
+					   struct ib_other_headers *ohdr,
+					   u32 bth0, u32 bth2, int middle,
+					   struct hfi1_pkt_state *ps)
+{
+	struct hfi1_qp_priv *priv = qp->priv;
+	struct hfi1_ibport *ibp = ps->ibp;
+	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
+	u32 bth1 = 0;
+	u16 pkey = hfi1_get_pkey(ibp, qp->s_pkey_index);
+	u16 lrh0 = HFI1_LRH_BTH;
+	u16 slid;
+	u8 extra_bytes = -ps->s_txreq->s_cur_size & 3;
+	u32 nwords = SIZE_OF_CRC + ((ps->s_txreq->s_cur_size +
+					 extra_bytes) >> 2);
+
 	if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)) {
+		struct ib_grh *grh = &ps->s_txreq->phdr.hdr.ibh.u.l.grh;
+		int hdrwords = qp->s_hdrwords - 2;
+
+		lrh0 = HFI1_LRH_GRH;
 		qp->s_hdrwords +=
-			hfi1_make_grh(ibp,
-				      &ps->s_txreq->phdr.hdr.u.l.grh,
+			hfi1_make_grh(ibp, grh,
 				      rdma_ah_read_grh(&qp->remote_ah_attr),
-				      qp->s_hdrwords, nwords);
-		lrh0 = HFI1_LRH_GRH;
+				      hdrwords, nwords);
 		middle = 0;
 	}
 	lrh0 |= (priv->s_sc & 0xf) << 12 |
 		(rdma_ah_get_sl(&qp->remote_ah_attr) & 0xf) << 4;
-	/*
-	 * reset s_ahg/AHG fields
-	 *
-	 * This insures that the ahgentry/ahgcount
-	 * are at a non-AHG default to protect
-	 * build_verbs_tx_desc() from using
-	 * an include ahgidx.
-	 *
-	 * build_ahg() will modify as appropriate
-	 * to use the AHG feature.
-	 */
-	priv->s_ahg->tx_flags = 0;
-	priv->s_ahg->ahgcount = 0;
-	priv->s_ahg->ahgidx = 0;
+
 	if (qp->s_mig_state == IB_MIG_MIGRATED)
 		bth0 |= IB_BTH_MIG_REQ;
 	else
 		middle = 0;
+
 	if (middle)
 		build_ahg(qp, bth2);
 	else
 		qp->s_flags &= ~RVT_S_AHG_VALID;
-	ps->s_txreq->phdr.hdr.lrh[0] = cpu_to_be16(lrh0);
-	ps->s_txreq->phdr.hdr.lrh[1] =
-		cpu_to_be16(rdma_ah_get_dlid(&qp->remote_ah_attr));
-	ps->s_txreq->phdr.hdr.lrh[2] =
-		cpu_to_be16(qp->s_hdrwords + nwords + SIZE_OF_CRC);
-	ps->s_txreq->phdr.hdr.lrh[3] =
-		cpu_to_be16(ppd_from_ibp(ibp)->lid |
-			    rdma_ah_get_path_bits(&qp->remote_ah_attr));
-	bth0 |= hfi1_get_pkey(ibp, qp->s_pkey_index);
+
+	bth0 |= pkey;
 	bth0 |= extra_bytes << 20;
-	ohdr->bth[0] = cpu_to_be32(bth0);
-	bth1 = qp->remote_qpn;
 	if (qp->s_flags & RVT_S_ECN) {
 		qp->s_flags &= ~RVT_S_ECN;
 		/* we recently received a FECN, so return a BECN */
 		bth1 |= (IB_BECN_MASK << IB_BECN_SHIFT);
 	}
-	ohdr->bth[1] = cpu_to_be32(bth1);
-	ohdr->bth[2] = cpu_to_be32(bth2);
+	hfi1_make_ruc_bth(qp, ohdr, bth0, bth1, bth2);
+
+	if (!ppd->lid)
+		slid = be16_to_cpu(IB_LID_PERMISSIVE);
+	else
+		slid = ppd->lid |
+			(rdma_ah_get_path_bits(&qp->remote_ah_attr) &
+			((1 << ppd->lmc) - 1));
+	hfi1_make_ib_hdr(&ps->s_txreq->phdr.hdr.ibh,
+			 lrh0,
+			 qp->s_hdrwords + nwords,
+			 opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr), 9B),
+			 ppd_from_ibp(ibp)->lid |
+				rdma_ah_get_path_bits(&qp->remote_ah_attr));
+}
+
+typedef void (*hfi1_make_ruc_hdr)(struct rvt_qp *qp,
+				  struct ib_other_headers *ohdr,
+				  u32 bth0, u32 bth2, int middle,
+				  struct hfi1_pkt_state *ps);
+
+/* We support only two types - 9B and 16B for now */
+static const hfi1_make_ruc_hdr hfi1_ruc_header_tbl[2] = {
+	[HFI1_PKT_TYPE_9B] = &hfi1_make_ruc_header_9B,
+	[HFI1_PKT_TYPE_16B] = &hfi1_make_ruc_header_16B
+};
+
+void hfi1_make_ruc_header(struct rvt_qp *qp, struct ib_other_headers *ohdr,
+			  u32 bth0, u32 bth2, int middle,
+			  struct hfi1_pkt_state *ps)
+{
+	struct hfi1_qp_priv *priv = qp->priv;
+
+	/*
+	 * reset s_ahg/AHG fields
+	 *
+	 * This insures that the ahgentry/ahgcount
+	 * are at a non-AHG default to protect
+	 * build_verbs_tx_desc() from using
+	 * an include ahgidx.
+	 *
+	 * build_ahg() will modify as appropriate
+	 * to use the AHG feature.
+	 */
+	priv->s_ahg->tx_flags = 0;
+	priv->s_ahg->ahgcount = 0;
+	priv->s_ahg->ahgidx = 0;
+
+	/* Make the appropriate header */
+	hfi1_ruc_header_tbl[priv->hdr_type](qp, ohdr, bth0, bth2, middle, ps);
 }
 
 /* when sending, force a reschedule every one of these periods */
@@ -816,6 +936,8 @@ void hfi1_make_ruc_header(struct rvt_qp *qp, struct ib_other_headers *ohdr,
 static bool schedule_send_yield(struct rvt_qp *qp,
 				struct hfi1_pkt_state *ps)
 {
+	ps->pkts_sent = true;
+
 	if (unlikely(time_after(jiffies, ps->timeout))) {
 		if (!ps->in_thread ||
 		    workqueue_congested(ps->cpu, ps->ppd->hfi1_wq)) {
@@ -912,6 +1034,7 @@ void hfi1_do_send(struct rvt_qp *qp, bool in_thread)
 	ps.timeout = jiffies + ps.timeout_int;
 	ps.cpu = priv->s_sde ? priv->s_sde->cpu :
 			cpumask_first(cpumask_of_node(ps.ppd->dd->node));
+	ps.pkts_sent = false;
 
 	/* insure a pre-built packet is handled  */
 	ps.s_txreq = get_waiting_verbs_txreq(qp);
@@ -934,7 +1057,7 @@ void hfi1_do_send(struct rvt_qp *qp, bool in_thread)
 			spin_lock_irqsave(&qp->s_lock, ps.flags);
 		}
 	} while (make_req(qp, &ps));
-
+	iowait_starve_clear(ps.pkts_sent, &priv->s_iowait);
 	spin_unlock_irqrestore(&qp->s_lock, ps.flags);
 }
 
diff --git a/drivers/infiniband/hw/hfi1/sdma.c b/drivers/infiniband/hw/hfi1/sdma.c
index bfd0d5187e9b..6781bcdb10b3 100644
--- a/drivers/infiniband/hw/hfi1/sdma.c
+++ b/drivers/infiniband/hw/hfi1/sdma.c
@@ -246,7 +246,7 @@ static void __sdma_process_event(
 	enum sdma_events event);
 static void dump_sdma_state(struct sdma_engine *sde);
 static void sdma_make_progress(struct sdma_engine *sde, u64 status);
-static void sdma_desc_avail(struct sdma_engine *sde, unsigned avail);
+static void sdma_desc_avail(struct sdma_engine *sde, uint avail);
 static void sdma_flush_descq(struct sdma_engine *sde);
 
 /**
@@ -325,7 +325,7 @@ static void sdma_wait_for_packet_egress(struct sdma_engine *sde,
 			/* timed out - bounce the link */
 			dd_dev_err(dd, "%s: engine %u timeout waiting for packets to egress, remaining count %u, bouncing link\n",
 				   __func__, sde->this_idx, (u32)reg);
-			queue_work(dd->pport->hfi1_wq,
+			queue_work(dd->pport->link_wq,
 				   &dd->pport->link_bounce_work);
 			break;
 		}
@@ -1340,10 +1340,8 @@ static void sdma_clean(struct hfi1_devdata *dd, size_t num_engines)
  * @dd: hfi1_devdata
  * @port: port number (currently only zero)
  *
- * sdma_init initializes the specified number of engines.
- *
- * The code initializes each sde, its csrs.  Interrupts
- * are not required to be enabled.
+ * Initializes each sde and its csrs.
+ * Interrupts are not required to be enabled.
  *
  * Returns:
  * 0 - success, -errno on failure
@@ -1764,13 +1762,14 @@ retry:
  *
  * This is called with head_lock held.
  */
-static void sdma_desc_avail(struct sdma_engine *sde, unsigned avail)
+static void sdma_desc_avail(struct sdma_engine *sde, uint avail)
 {
 	struct iowait *wait, *nw;
 	struct iowait *waits[SDMA_WAIT_BATCH_SIZE];
-	unsigned i, n = 0, seq;
+	uint i, n = 0, seq, max_idx = 0;
 	struct sdma_txreq *stx;
 	struct hfi1_ibdev *dev = &sde->dd->verbs_dev;
+	u8 max_starved_cnt = 0;
 
 #ifdef CONFIG_SDMA_VERBOSITY
 	dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
@@ -1805,6 +1804,9 @@ static void sdma_desc_avail(struct sdma_engine *sde, unsigned avail)
 				if (num_desc > avail)
 					break;
 				avail -= num_desc;
+				/* Find the most starved wait memeber */
+				iowait_starve_find_max(wait, &max_starved_cnt,
+						       n, &max_idx);
 				list_del_init(&wait->list);
 				waits[n++] = wait;
 			}
@@ -1813,8 +1815,13 @@ static void sdma_desc_avail(struct sdma_engine *sde, unsigned avail)
 		}
 	} while (read_seqretry(&dev->iowait_lock, seq));
 
+	/* Schedule the most starved one first */
+	if (n)
+		waits[max_idx]->wakeup(waits[max_idx], SDMA_AVAIL_REASON);
+
 	for (i = 0; i < n; i++)
-		waits[i]->wakeup(waits[i], SDMA_AVAIL_REASON);
+		if (i != max_idx)
+			waits[i]->wakeup(waits[i], SDMA_AVAIL_REASON);
 }
 
 /* head_lock must be held */
@@ -2351,7 +2358,8 @@ static inline u16 submit_tx(struct sdma_engine *sde, struct sdma_txreq *tx)
 static int sdma_check_progress(
 	struct sdma_engine *sde,
 	struct iowait *wait,
-	struct sdma_txreq *tx)
+	struct sdma_txreq *tx,
+	bool pkts_sent)
 {
 	int ret;
 
@@ -2364,7 +2372,7 @@ static int sdma_check_progress(
 
 		seq = raw_seqcount_begin(
 			(const seqcount_t *)&sde->head_lock.seqcount);
-		ret = wait->sleep(sde, wait, tx, seq);
+		ret = wait->sleep(sde, wait, tx, seq, pkts_sent);
 		if (ret == -EAGAIN)
 			sde->desc_avail = sdma_descq_freecnt(sde);
 	} else {
@@ -2378,6 +2386,7 @@ static int sdma_check_progress(
  * @sde: sdma engine to use
  * @wait: wait structure to use when full (may be NULL)
  * @tx: sdma_txreq to submit
+ * @pkts_sent: has any packet been sent yet?
  *
  * The call submits the tx into the ring.  If a iowait structure is non-NULL
  * the packet will be queued to the list in wait.
@@ -2389,7 +2398,8 @@ static int sdma_check_progress(
  */
 int sdma_send_txreq(struct sdma_engine *sde,
 		    struct iowait *wait,
-		    struct sdma_txreq *tx)
+		    struct sdma_txreq *tx,
+		    bool pkts_sent)
 {
 	int ret = 0;
 	u16 tail;
@@ -2431,7 +2441,7 @@ unlock_noconn:
 	ret = -ECOMM;
 	goto unlock;
 nodesc:
-	ret = sdma_check_progress(sde, wait, tx);
+	ret = sdma_check_progress(sde, wait, tx, pkts_sent);
 	if (ret == -EAGAIN) {
 		ret = 0;
 		goto retry;
@@ -2500,8 +2510,10 @@ retry:
 	}
 update_tail:
 	total_count = submit_count + flush_count;
-	if (wait)
+	if (wait) {
 		iowait_sdma_add(wait, total_count);
+		iowait_starve_clear(submit_count > 0, wait);
+	}
 	if (tail != INVALID_TAIL)
 		sdma_update_tail(sde, tail);
 	spin_unlock_irqrestore(&sde->tail_lock, flags);
@@ -2529,7 +2541,7 @@ unlock_noconn:
 	ret = -ECOMM;
 	goto update_tail;
 nodesc:
-	ret = sdma_check_progress(sde, wait, tx);
+	ret = sdma_check_progress(sde, wait, tx, submit_count > 0);
 	if (ret == -EAGAIN) {
 		ret = 0;
 		goto retry;
diff --git a/drivers/infiniband/hw/hfi1/sdma.h b/drivers/infiniband/hw/hfi1/sdma.h
index 64f10b8b5db8..107011d8613b 100644
--- a/drivers/infiniband/hw/hfi1/sdma.h
+++ b/drivers/infiniband/hw/hfi1/sdma.h
@@ -852,7 +852,8 @@ struct iowait;
 
 int sdma_send_txreq(struct sdma_engine *sde,
 		    struct iowait *wait,
-		    struct sdma_txreq *tx);
+		    struct sdma_txreq *tx,
+		    bool pkts_sent);
 int sdma_send_txlist(struct sdma_engine *sde,
 		     struct iowait *wait,
 		     struct list_head *tx_list,
diff --git a/drivers/infiniband/hw/hfi1/sysfs.c b/drivers/infiniband/hw/hfi1/sysfs.c
index 2f3bbcac1e34..6d2702ef34ac 100644
--- a/drivers/infiniband/hw/hfi1/sysfs.c
+++ b/drivers/infiniband/hw/hfi1/sysfs.c
@@ -95,7 +95,7 @@ static void port_release(struct kobject *kobj)
 	/* nothing to do since memory is freed by hfi1_free_devdata() */
 }
 
-static struct bin_attribute cc_table_bin_attr = {
+static const struct bin_attribute cc_table_bin_attr = {
 	.attr = {.name = "cc_table_bin", .mode = 0444},
 	.read = read_cc_table_bin,
 	.size = PAGE_SIZE,
@@ -137,7 +137,7 @@ static ssize_t read_cc_setting_bin(struct file *filp, struct kobject *kobj,
 	return count;
 }
 
-static struct bin_attribute cc_setting_bin_attr = {
+static const struct bin_attribute cc_setting_bin_attr = {
 	.attr = {.name = "cc_settings_bin", .mode = 0444},
 	.read = read_cc_setting_bin,
 	.size = PAGE_SIZE,
diff --git a/drivers/infiniband/hw/hfi1/trace.c b/drivers/infiniband/hw/hfi1/trace.c
index eafae487face..9938bb983ce6 100644
--- a/drivers/infiniband/hw/hfi1/trace.c
+++ b/drivers/infiniband/hw/hfi1/trace.c
@@ -47,7 +47,7 @@
 #define CREATE_TRACE_POINTS
 #include "trace.h"
 
-u8 ibhdr_exhdr_len(struct ib_header *hdr)
+static u8 __get_ib_hdr_len(struct ib_header *hdr)
 {
 	struct ib_other_headers *ohdr;
 	u8 opcode;
@@ -61,13 +61,69 @@ u8 ibhdr_exhdr_len(struct ib_header *hdr)
 	       0 : hdr_len_by_opcode[opcode] - (12 + 8);
 }
 
-#define IMM_PRN  "imm %d"
-#define RETH_PRN "reth vaddr 0x%.16llx rkey 0x%.8x dlen 0x%.8x"
-#define AETH_PRN "aeth syn 0x%.2x %s msn 0x%.8x"
-#define DETH_PRN "deth qkey 0x%.8x sqpn 0x%.6x"
-#define IETH_PRN "ieth rkey 0x%.8x"
-#define ATOMICACKETH_PRN "origdata %llx"
-#define ATOMICETH_PRN "vaddr 0x%llx rkey 0x%.8x sdata %llx cdata %llx"
+static u8 __get_16b_hdr_len(struct hfi1_16b_header *hdr)
+{
+	struct ib_other_headers *ohdr;
+	u8 opcode;
+
+	if (hfi1_16B_get_l4(hdr) == OPA_16B_L4_IB_LOCAL)
+		ohdr = &hdr->u.oth;
+	else
+		ohdr = &hdr->u.l.oth;
+	opcode = ib_bth_get_opcode(ohdr);
+	return hdr_len_by_opcode[opcode] == 0 ?
+	       0 : hdr_len_by_opcode[opcode] - (12 + 8 + 8);
+}
+
+u8 hfi1_trace_packet_hdr_len(struct hfi1_packet *packet)
+{
+	if (packet->etype != RHF_RCV_TYPE_BYPASS)
+		return __get_ib_hdr_len(packet->hdr);
+	else
+		return __get_16b_hdr_len(packet->hdr);
+}
+
+u8 hfi1_trace_opa_hdr_len(struct hfi1_opa_header *opa_hdr)
+{
+	if (!opa_hdr->hdr_type)
+		return __get_ib_hdr_len(&opa_hdr->ibh);
+	else
+		return __get_16b_hdr_len(&opa_hdr->opah);
+}
+
+const char *hfi1_trace_get_packet_str(struct hfi1_packet *packet)
+{
+	if (packet->etype != RHF_RCV_TYPE_BYPASS)
+		return "IB";
+
+	switch (hfi1_16B_get_l2(packet->hdr)) {
+	case 0:
+		return "0";
+	case 1:
+		return "1";
+	case 2:
+		return "16B";
+	case 3:
+		return "9B";
+	}
+	return "";
+}
+
+const char *hfi1_trace_get_packet_type_str(u8 l4)
+{
+	if (l4)
+		return "16B";
+	else
+		return "9B";
+}
+
+#define IMM_PRN  "imm:%d"
+#define RETH_PRN "reth vaddr:0x%.16llx rkey:0x%.8x dlen:0x%.8x"
+#define AETH_PRN "aeth syn:0x%.2x %s msn:0x%.8x"
+#define DETH_PRN "deth qkey:0x%.8x sqpn:0x%.6x"
+#define IETH_PRN "ieth rkey:0x%.8x"
+#define ATOMICACKETH_PRN "origdata:%llx"
+#define ATOMICETH_PRN "vaddr:0x%llx rkey:0x%.8x sdata:%llx cdata:%llx"
 
 #define OP(transport, op) IB_OPCODE_## transport ## _ ## op
 
@@ -84,6 +140,125 @@ static const char *parse_syndrome(u8 syndrome)
 	return "";
 }
 
+void hfi1_trace_parse_9b_bth(struct ib_other_headers *ohdr,
+			     u8 *ack, u8 *becn, u8 *fecn, u8 *mig,
+			     u8 *se, u8 *pad, u8 *opcode, u8 *tver,
+			     u16 *pkey, u32 *psn, u32 *qpn)
+{
+	*ack = ib_bth_get_ackreq(ohdr);
+	*becn = ib_bth_get_becn(ohdr);
+	*fecn = ib_bth_get_fecn(ohdr);
+	*mig = ib_bth_get_migreq(ohdr);
+	*se = ib_bth_get_se(ohdr);
+	*pad = ib_bth_get_pad(ohdr);
+	*opcode = ib_bth_get_opcode(ohdr);
+	*tver = ib_bth_get_tver(ohdr);
+	*pkey = ib_bth_get_pkey(ohdr);
+	*psn = ib_bth_get_psn(ohdr);
+	*qpn = ib_bth_get_qpn(ohdr);
+}
+
+void hfi1_trace_parse_16b_bth(struct ib_other_headers *ohdr,
+			      u8 *ack, u8 *mig, u8 *opcode,
+			      u8 *pad, u8 *se, u8 *tver,
+			      u32 *psn, u32 *qpn)
+{
+	*ack = ib_bth_get_ackreq(ohdr);
+	*mig = ib_bth_get_migreq(ohdr);
+	*opcode = ib_bth_get_opcode(ohdr);
+	*pad = ib_bth_get_pad(ohdr);
+	*se = ib_bth_get_se(ohdr);
+	*tver = ib_bth_get_tver(ohdr);
+	*psn = ib_bth_get_psn(ohdr);
+	*qpn = ib_bth_get_qpn(ohdr);
+}
+
+void hfi1_trace_parse_9b_hdr(struct ib_header *hdr, bool sc5,
+			     u8 *lnh, u8 *lver, u8 *sl, u8 *sc,
+			     u16 *len, u32 *dlid, u32 *slid)
+{
+	*lnh = ib_get_lnh(hdr);
+	*lver = ib_get_lver(hdr);
+	*sl = ib_get_sl(hdr);
+	*sc = ib_get_sc(hdr) | (sc5 << 4);
+	*len = ib_get_len(hdr);
+	*dlid = ib_get_dlid(hdr);
+	*slid = ib_get_slid(hdr);
+}
+
+void hfi1_trace_parse_16b_hdr(struct hfi1_16b_header *hdr,
+			      u8 *age, u8 *becn, u8 *fecn,
+			      u8 *l4, u8 *rc, u8 *sc,
+			      u16 *entropy, u16 *len, u16 *pkey,
+			      u32 *dlid, u32 *slid)
+{
+	*age = hfi1_16B_get_age(hdr);
+	*becn = hfi1_16B_get_becn(hdr);
+	*fecn = hfi1_16B_get_fecn(hdr);
+	*l4 = hfi1_16B_get_l4(hdr);
+	*rc = hfi1_16B_get_rc(hdr);
+	*sc = hfi1_16B_get_sc(hdr);
+	*entropy = hfi1_16B_get_entropy(hdr);
+	*len = hfi1_16B_get_len(hdr);
+	*pkey = hfi1_16B_get_pkey(hdr);
+	*dlid = hfi1_16B_get_dlid(hdr);
+	*slid = hfi1_16B_get_slid(hdr);
+}
+
+#define LRH_PRN "len:%d sc:%d dlid:0x%.4x slid:0x%.4x "
+#define LRH_9B_PRN "lnh:%d,%s lver:%d sl:%d"
+#define LRH_16B_PRN "age:%d becn:%d fecn:%d l4:%d " \
+		    "rc:%d sc:%d pkey:0x%.4x entropy:0x%.4x"
+const char *hfi1_trace_fmt_lrh(struct trace_seq *p, bool bypass,
+			       u8 age, u8 becn, u8 fecn, u8 l4,
+			       u8 lnh, const char *lnh_name, u8 lver,
+			       u8 rc, u8 sc, u8 sl, u16 entropy,
+			       u16 len, u16 pkey, u32 dlid, u32 slid)
+{
+	const char *ret = trace_seq_buffer_ptr(p);
+
+	trace_seq_printf(p, LRH_PRN, len, sc, dlid, slid);
+
+	if (bypass)
+		trace_seq_printf(p, LRH_16B_PRN,
+				 age, becn, fecn, l4, rc, sc, pkey, entropy);
+
+	else
+		trace_seq_printf(p, LRH_9B_PRN,
+				 lnh, lnh_name, lver, sl);
+	trace_seq_putc(p, 0);
+
+	return ret;
+}
+
+#define BTH_9B_PRN \
+	"op:0x%.2x,%s se:%d m:%d pad:%d tver:%d pkey:0x%.4x " \
+	"f:%d b:%d qpn:0x%.6x a:%d psn:0x%.8x"
+#define BTH_16B_PRN \
+	"op:0x%.2x,%s se:%d m:%d pad:%d tver:%d " \
+	"qpn:0x%.6x a:%d psn:0x%.8x"
+const char *hfi1_trace_fmt_bth(struct trace_seq *p, bool bypass,
+			       u8 ack, u8 becn, u8 fecn, u8 mig,
+			       u8 se, u8 pad, u8 opcode, const char *opname,
+			       u8 tver, u16 pkey, u32 psn, u32 qpn)
+{
+	const char *ret = trace_seq_buffer_ptr(p);
+
+	if (bypass)
+		trace_seq_printf(p, BTH_16B_PRN,
+				 opcode, opname,
+				 se, mig, pad, tver, qpn, ack, psn);
+
+	else
+		trace_seq_printf(p, BTH_9B_PRN,
+				 opcode, opname,
+				 se, mig, pad, tver, pkey, fecn, becn,
+				 qpn, ack, psn);
+	trace_seq_putc(p, 0);
+
+	return ret;
+}
+
 const char *parse_everbs_hdrs(
 	struct trace_seq *p,
 	u8 opcode,
diff --git a/drivers/infiniband/hw/hfi1/trace.h b/drivers/infiniband/hw/hfi1/trace.h
index 92dc88f013c9..af50c0793450 100644
--- a/drivers/infiniband/hw/hfi1/trace.h
+++ b/drivers/infiniband/hw/hfi1/trace.h
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015, 2016 Intel Corporation.
+ * Copyright(c) 2015 - 2017 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -51,3 +51,4 @@
 #include "trace_rc.h"
 #include "trace_rx.h"
 #include "trace_tx.h"
+#include "trace_mmu.h"
diff --git a/drivers/infiniband/hw/hfi1/trace_ibhdrs.h b/drivers/infiniband/hw/hfi1/trace_ibhdrs.h
index 090f6b506953..6721f84dafa5 100644
--- a/drivers/infiniband/hw/hfi1/trace_ibhdrs.h
+++ b/drivers/infiniband/hw/hfi1/trace_ibhdrs.h
@@ -55,8 +55,79 @@
 #undef TRACE_SYSTEM
 #define TRACE_SYSTEM hfi1_ibhdrs
 
+#define ib_opcode_name(opcode) { IB_OPCODE_##opcode, #opcode  }
+#define show_ib_opcode(opcode)                             \
+__print_symbolic(opcode,                                   \
+	ib_opcode_name(RC_SEND_FIRST),                     \
+	ib_opcode_name(RC_SEND_MIDDLE),                    \
+	ib_opcode_name(RC_SEND_LAST),                      \
+	ib_opcode_name(RC_SEND_LAST_WITH_IMMEDIATE),       \
+	ib_opcode_name(RC_SEND_ONLY),                      \
+	ib_opcode_name(RC_SEND_ONLY_WITH_IMMEDIATE),       \
+	ib_opcode_name(RC_RDMA_WRITE_FIRST),               \
+	ib_opcode_name(RC_RDMA_WRITE_MIDDLE),              \
+	ib_opcode_name(RC_RDMA_WRITE_LAST),                \
+	ib_opcode_name(RC_RDMA_WRITE_LAST_WITH_IMMEDIATE), \
+	ib_opcode_name(RC_RDMA_WRITE_ONLY),                \
+	ib_opcode_name(RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE), \
+	ib_opcode_name(RC_RDMA_READ_REQUEST),              \
+	ib_opcode_name(RC_RDMA_READ_RESPONSE_FIRST),       \
+	ib_opcode_name(RC_RDMA_READ_RESPONSE_MIDDLE),      \
+	ib_opcode_name(RC_RDMA_READ_RESPONSE_LAST),        \
+	ib_opcode_name(RC_RDMA_READ_RESPONSE_ONLY),        \
+	ib_opcode_name(RC_ACKNOWLEDGE),                    \
+	ib_opcode_name(RC_ATOMIC_ACKNOWLEDGE),             \
+	ib_opcode_name(RC_COMPARE_SWAP),                   \
+	ib_opcode_name(RC_FETCH_ADD),                      \
+	ib_opcode_name(UC_SEND_FIRST),                     \
+	ib_opcode_name(UC_SEND_MIDDLE),                    \
+	ib_opcode_name(UC_SEND_LAST),                      \
+	ib_opcode_name(UC_SEND_LAST_WITH_IMMEDIATE),       \
+	ib_opcode_name(UC_SEND_ONLY),                      \
+	ib_opcode_name(UC_SEND_ONLY_WITH_IMMEDIATE),       \
+	ib_opcode_name(UC_RDMA_WRITE_FIRST),               \
+	ib_opcode_name(UC_RDMA_WRITE_MIDDLE),              \
+	ib_opcode_name(UC_RDMA_WRITE_LAST),                \
+	ib_opcode_name(UC_RDMA_WRITE_LAST_WITH_IMMEDIATE), \
+	ib_opcode_name(UC_RDMA_WRITE_ONLY),                \
+	ib_opcode_name(UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE), \
+	ib_opcode_name(UD_SEND_ONLY),                      \
+	ib_opcode_name(UD_SEND_ONLY_WITH_IMMEDIATE),       \
+	ib_opcode_name(CNP))
+
 u8 ibhdr_exhdr_len(struct ib_header *hdr);
 const char *parse_everbs_hdrs(struct trace_seq *p, u8 opcode, void *ehdrs);
+u8 hfi1_trace_opa_hdr_len(struct hfi1_opa_header *opah);
+u8 hfi1_trace_packet_hdr_len(struct hfi1_packet *packet);
+const char *hfi1_trace_get_packet_type_str(u8 l4);
+const char *hfi1_trace_get_packet_str(struct hfi1_packet *packet);
+void hfi1_trace_parse_9b_bth(struct ib_other_headers *ohdr,
+			     u8 *ack, u8 *becn, u8 *fecn, u8 *mig,
+			     u8 *se, u8 *pad, u8 *opcode, u8 *tver,
+			     u16 *pkey, u32 *psn, u32 *qpn);
+void hfi1_trace_parse_9b_hdr(struct ib_header *hdr, bool sc5,
+			     u8 *lnh, u8 *lver, u8 *sl, u8 *sc,
+			     u16 *len, u32 *dlid, u32 *slid);
+void hfi1_trace_parse_16b_bth(struct ib_other_headers *ohdr,
+			      u8 *ack, u8 *mig, u8 *opcode,
+			      u8 *pad, u8 *se, u8 *tver,
+			      u32 *psn, u32 *qpn);
+void hfi1_trace_parse_16b_hdr(struct hfi1_16b_header *hdr,
+			      u8 *age, u8 *becn, u8 *fecn,
+			      u8 *l4, u8 *rc, u8 *sc,
+			      u16 *entropy, u16 *len, u16 *pkey,
+			      u32 *dlid, u32 *slid);
+
+const char *hfi1_trace_fmt_lrh(struct trace_seq *p, bool bypass,
+			       u8 age, u8 becn, u8 fecn, u8 l4,
+			       u8 lnh, const char *lnh_name, u8 lver,
+			       u8 rc, u8 sc, u8 sl, u16 entropy,
+			       u16 len, u16 pkey, u32 dlid, u32 slid);
+
+const char *hfi1_trace_fmt_bth(struct trace_seq *p, bool bypass,
+			       u8 ack, u8 becn, u8 fecn, u8 mig,
+			       u8 se, u8 pad, u8 opcode, const char *opname,
+			       u8 tver, u16 pkey, u32 psn, u32 qpn);
 
 #define __parse_ib_ehdrs(op, ehdrs) parse_everbs_hdrs(p, op, ehdrs)
 
@@ -65,140 +136,303 @@ const char *parse_everbs_hdrs(struct trace_seq *p, u8 opcode, void *ehdrs);
 __print_symbolic(lrh,                    \
 	lrh_name(LRH_BTH),               \
 	lrh_name(LRH_GRH))
+#define PKT_ENTRY(pkt)	__string(ptype,  hfi1_trace_get_packet_str(packet))
+#define PKT_ASSIGN(pkt) __assign_str(ptype, hfi1_trace_get_packet_str(packet))
 
-#define LRH_PRN "vl %d lver %d sl %d lnh %d,%s dlid %.4x len %d slid %.4x"
-#define BTH_PRN \
-	"op 0x%.2x,%s se %d m %d pad %d tver %d pkey 0x%.4x " \
-	"f %d b %d qpn 0x%.6x a %d psn 0x%.8x"
-#define EHDR_PRN "%s"
-
-DECLARE_EVENT_CLASS(hfi1_ibhdr_template,
+DECLARE_EVENT_CLASS(hfi1_input_ibhdr_template,
 		    TP_PROTO(struct hfi1_devdata *dd,
-			     struct ib_header *hdr),
-		    TP_ARGS(dd, hdr),
+			     struct hfi1_packet *packet,
+			     bool sc5),
+		    TP_ARGS(dd, packet, sc5),
 		    TP_STRUCT__entry(
 			DD_DEV_ENTRY(dd)
-			/* LRH */
-			__field(u8, vl)
+			PKT_ENTRY(packet)
+			__field(bool, bypass)
+			__field(u8, ack)
+			__field(u8, age)
+			__field(u8, becn)
+			__field(u8, fecn)
+			__field(u8, l4)
+			__field(u8, lnh)
 			__field(u8, lver)
+			__field(u8, mig)
+			__field(u8, opcode)
+			__field(u8, pad)
+			__field(u8, rc)
+			__field(u8, sc)
+			__field(u8, se)
 			__field(u8, sl)
-			__field(u8, lnh)
-			__field(u16, dlid)
+			__field(u8, tver)
+			__field(u16, entropy)
 			__field(u16, len)
-			__field(u16, slid)
-			/* BTH */
+			__field(u16, pkey)
+			__field(u32, dlid)
+			__field(u32, psn)
+			__field(u32, qpn)
+			__field(u32, slid)
+			/* extended headers */
+			__dynamic_array(u8, ehdrs,
+					hfi1_trace_packet_hdr_len(packet))
+			),
+		    TP_fast_assign(
+			DD_DEV_ASSIGN(dd);
+			PKT_ASSIGN(packet);
+
+			if (packet->etype == RHF_RCV_TYPE_BYPASS) {
+				__entry->bypass = true;
+				hfi1_trace_parse_16b_hdr(packet->hdr,
+							 &__entry->age,
+							 &__entry->becn,
+							 &__entry->fecn,
+							 &__entry->l4,
+							 &__entry->rc,
+							 &__entry->sc,
+							 &__entry->entropy,
+							 &__entry->len,
+							 &__entry->pkey,
+							 &__entry->dlid,
+							 &__entry->slid);
+
+				  hfi1_trace_parse_16b_bth(packet->ohdr,
+							   &__entry->ack,
+							   &__entry->mig,
+							   &__entry->opcode,
+							   &__entry->pad,
+							   &__entry->se,
+							   &__entry->tver,
+							   &__entry->psn,
+							   &__entry->qpn);
+			} else {
+				__entry->bypass = false;
+				hfi1_trace_parse_9b_hdr(packet->hdr, sc5,
+							&__entry->lnh,
+							&__entry->lver,
+							&__entry->sl,
+							&__entry->sc,
+							&__entry->len,
+							&__entry->dlid,
+							&__entry->slid);
+
+				  hfi1_trace_parse_9b_bth(packet->ohdr,
+							  &__entry->ack,
+							  &__entry->becn,
+							  &__entry->fecn,
+							  &__entry->mig,
+							  &__entry->se,
+							  &__entry->pad,
+							  &__entry->opcode,
+							  &__entry->tver,
+							  &__entry->pkey,
+							  &__entry->psn,
+							  &__entry->qpn);
+				}
+				/* extended headers */
+				memcpy(__get_dynamic_array(ehdrs),
+				       &packet->ohdr->u,
+				       __get_dynamic_array_len(ehdrs));
+			 ),
+		    TP_printk("[%s] (%s) %s %s hlen:%d %s",
+			      __get_str(dev),
+			      __get_str(ptype),
+			      hfi1_trace_fmt_lrh(p,
+						 __entry->bypass,
+						 __entry->age,
+						 __entry->becn,
+						 __entry->fecn,
+						 __entry->l4,
+						 __entry->lnh,
+						 show_lnh(__entry->lnh),
+						 __entry->lver,
+						 __entry->rc,
+						 __entry->sc,
+						 __entry->sl,
+						 __entry->entropy,
+						 __entry->len,
+						 __entry->pkey,
+						 __entry->dlid,
+						 __entry->slid),
+			      hfi1_trace_fmt_bth(p,
+						 __entry->bypass,
+						 __entry->ack,
+						 __entry->becn,
+						 __entry->fecn,
+						 __entry->mig,
+						 __entry->se,
+						 __entry->pad,
+						 __entry->opcode,
+						 show_ib_opcode(__entry->opcode),
+						 __entry->tver,
+						 __entry->pkey,
+						 __entry->psn,
+						 __entry->qpn),
+			      /* extended headers */
+			      __get_dynamic_array_len(ehdrs),
+			      __parse_ib_ehdrs(
+					__entry->opcode,
+					(void *)__get_dynamic_array(ehdrs))
+			     )
+);
+
+DEFINE_EVENT(hfi1_input_ibhdr_template, input_ibhdr,
+	     TP_PROTO(struct hfi1_devdata *dd,
+		      struct hfi1_packet *packet, bool sc5),
+	     TP_ARGS(dd, packet, sc5));
+
+DECLARE_EVENT_CLASS(hfi1_output_ibhdr_template,
+		    TP_PROTO(struct hfi1_devdata *dd,
+			     struct hfi1_opa_header *opah, bool sc5),
+		    TP_ARGS(dd, opah, sc5),
+		    TP_STRUCT__entry(
+			DD_DEV_ENTRY(dd)
+			__field(bool, bypass)
+			__field(u8, ack)
+			__field(u8, age)
+			__field(u8, becn)
+			__field(u8, fecn)
+			__field(u8, l4)
+			__field(u8, lnh)
+			__field(u8, lver)
+			__field(u8, mig)
 			__field(u8, opcode)
-			__field(u8, se)
-			__field(u8, m)
 			__field(u8, pad)
+			__field(u8, rc)
+			__field(u8, sc)
+			__field(u8, se)
+			__field(u8, sl)
 			__field(u8, tver)
+			__field(u16, entropy)
+			__field(u16, len)
 			__field(u16, pkey)
-			__field(u8, f)
-			__field(u8, b)
-			__field(u32, qpn)
-			__field(u8, a)
+			__field(u32, dlid)
 			__field(u32, psn)
+			__field(u32, qpn)
+			__field(u32, slid)
 			/* extended headers */
-			__dynamic_array(u8, ehdrs, ibhdr_exhdr_len(hdr))
+			__dynamic_array(u8, ehdrs,
+					hfi1_trace_opa_hdr_len(opah))
 			),
-		      TP_fast_assign(
+		    TP_fast_assign(
 			struct ib_other_headers *ohdr;
 
 			DD_DEV_ASSIGN(dd);
-			/* LRH */
-			__entry->vl =
-			(u8)(be16_to_cpu(hdr->lrh[0]) >> 12);
-			__entry->lver =
-			(u8)(be16_to_cpu(hdr->lrh[0]) >> 8) & 0xf;
-			__entry->sl =
-			(u8)(be16_to_cpu(hdr->lrh[0]) >> 4) & 0xf;
-			__entry->lnh =
-			(u8)(be16_to_cpu(hdr->lrh[0]) & 3);
-			__entry->dlid =
-			be16_to_cpu(hdr->lrh[1]);
-			/* allow for larger len */
-			__entry->len =
-			be16_to_cpu(hdr->lrh[2]);
-			__entry->slid =
-			be16_to_cpu(hdr->lrh[3]);
-			/* BTH */
-			if (__entry->lnh == HFI1_LRH_BTH)
-			ohdr = &hdr->u.oth;
-			else
-			ohdr = &hdr->u.l.oth;
-			__entry->opcode =
-			(be32_to_cpu(ohdr->bth[0]) >> 24) & 0xff;
-			__entry->se =
-			(be32_to_cpu(ohdr->bth[0]) >> 23) & 1;
-			__entry->m =
-			(be32_to_cpu(ohdr->bth[0]) >> 22) & 1;
-			__entry->pad =
-			(be32_to_cpu(ohdr->bth[0]) >> 20) & 3;
-			__entry->tver =
-			(be32_to_cpu(ohdr->bth[0]) >> 16) & 0xf;
-			__entry->pkey =
-			be32_to_cpu(ohdr->bth[0]) & 0xffff;
-			__entry->f =
-			(be32_to_cpu(ohdr->bth[1]) >> IB_FECN_SHIFT) &
-			IB_FECN_MASK;
-			__entry->b =
-			(be32_to_cpu(ohdr->bth[1]) >> IB_BECN_SHIFT) &
-			IB_BECN_MASK;
-			__entry->qpn =
-			be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK;
-			__entry->a =
-			(be32_to_cpu(ohdr->bth[2]) >> 31) & 1;
-			/* allow for larger PSN */
-			__entry->psn =
-			be32_to_cpu(ohdr->bth[2]) & 0x7fffffff;
+
+			if (opah->hdr_type)  {
+				__entry->bypass = true;
+				hfi1_trace_parse_16b_hdr(&opah->opah,
+							 &__entry->age,
+							 &__entry->becn,
+							 &__entry->fecn,
+							 &__entry->l4,
+							 &__entry->rc,
+							 &__entry->sc,
+							 &__entry->entropy,
+							 &__entry->len,
+							 &__entry->pkey,
+							 &__entry->dlid,
+							 &__entry->slid);
+
+				if (entry->l4 == OPA_16B_L4_IB_LOCAL)
+					ohdr = &opah->opah.u.oth;
+				else
+					ohdr = &opah->opah.u.l.oth;
+				hfi1_trace_parse_16b_bth(ohdr,
+							 &__entry->ack,
+							 &__entry->mig,
+							 &__entry->opcode,
+							 &__entry->pad,
+							 &__entry->se,
+							 &__entry->tver,
+							 &__entry->psn,
+							 &__entry->qpn);
+			} else {
+				__entry->bypass = false;
+				hfi1_trace_parse_9b_hdr(&opah->ibh, sc5,
+							&__entry->lnh,
+							&__entry->lver,
+							&__entry->sl,
+							&__entry->sc,
+							&__entry->len,
+							&__entry->dlid,
+							&__entry->slid);
+				if (entry->lnh == HFI1_LRH_BTH)
+					ohdr = &opah->ibh.u.oth;
+				else
+					ohdr = &opah->ibh.u.l.oth;
+				hfi1_trace_parse_9b_bth(ohdr,
+							&__entry->ack,
+							&__entry->becn,
+							&__entry->fecn,
+							&__entry->mig,
+							&__entry->se,
+							&__entry->pad,
+							&__entry->opcode,
+							&__entry->tver,
+							&__entry->pkey,
+							&__entry->psn,
+							&__entry->qpn);
+			}
+
 			/* extended headers */
-			memcpy(__get_dynamic_array(ehdrs), &ohdr->u,
-			       ibhdr_exhdr_len(hdr));
-			),
-		TP_printk("[%s] " LRH_PRN " " BTH_PRN " " EHDR_PRN,
-			  __get_str(dev),
-			  /* LRH */
-			  __entry->vl,
-			  __entry->lver,
-			  __entry->sl,
-			  __entry->lnh, show_lnh(__entry->lnh),
-			  __entry->dlid,
-			  __entry->len,
-			  __entry->slid,
-			  /* BTH */
-			  __entry->opcode, show_ib_opcode(__entry->opcode),
-			  __entry->se,
-			  __entry->m,
-			  __entry->pad,
-			  __entry->tver,
-			  __entry->pkey,
-			  __entry->f,
-			  __entry->b,
-			  __entry->qpn,
-			  __entry->a,
-			  __entry->psn,
-			  /* extended headers */
-			  __parse_ib_ehdrs(
-				__entry->opcode,
-				(void *)__get_dynamic_array(ehdrs))
-			)
+			memcpy(__get_dynamic_array(ehdrs),
+			       &ohdr->u, __get_dynamic_array_len(ehdrs));
+		    ),
+		    TP_printk("[%s] (%s) %s %s hlen:%d %s",
+			      __get_str(dev),
+			      hfi1_trace_get_packet_type_str(__entry->l4),
+			      hfi1_trace_fmt_lrh(p,
+						 __entry->bypass,
+						 __entry->age,
+						 __entry->becn,
+						 __entry->fecn,
+						 __entry->l4,
+						 __entry->lnh,
+						 show_lnh(__entry->lnh),
+						 __entry->lver,
+						 __entry->rc,
+						 __entry->sc,
+						 __entry->sl,
+						 __entry->entropy,
+						 __entry->len,
+						 __entry->pkey,
+						 __entry->dlid,
+						 __entry->slid),
+			      hfi1_trace_fmt_bth(p,
+						 __entry->bypass,
+						 __entry->ack,
+						 __entry->becn,
+						 __entry->fecn,
+						 __entry->mig,
+						 __entry->se,
+						 __entry->pad,
+						 __entry->opcode,
+						 show_ib_opcode(__entry->opcode),
+						 __entry->tver,
+						 __entry->pkey,
+						 __entry->psn,
+						 __entry->qpn),
+			      /* extended headers */
+			      __get_dynamic_array_len(ehdrs),
+			      __parse_ib_ehdrs(
+					__entry->opcode,
+					(void *)__get_dynamic_array(ehdrs))
+			     )
 );
 
-DEFINE_EVENT(hfi1_ibhdr_template, input_ibhdr,
-	     TP_PROTO(struct hfi1_devdata *dd, struct ib_header *hdr),
-	     TP_ARGS(dd, hdr));
+DEFINE_EVENT(hfi1_output_ibhdr_template, pio_output_ibhdr,
+	     TP_PROTO(struct hfi1_devdata *dd,
+		      struct hfi1_opa_header *opah, bool sc5),
+	     TP_ARGS(dd, opah, sc5));
 
-DEFINE_EVENT(hfi1_ibhdr_template, pio_output_ibhdr,
-	     TP_PROTO(struct hfi1_devdata *dd, struct ib_header *hdr),
-	     TP_ARGS(dd, hdr));
+DEFINE_EVENT(hfi1_output_ibhdr_template, ack_output_ibhdr,
+	     TP_PROTO(struct hfi1_devdata *dd,
+		      struct hfi1_opa_header *opah, bool sc5),
+	     TP_ARGS(dd, opah, sc5));
 
-DEFINE_EVENT(hfi1_ibhdr_template, ack_output_ibhdr,
-	     TP_PROTO(struct hfi1_devdata *dd, struct ib_header *hdr),
-	     TP_ARGS(dd, hdr));
+DEFINE_EVENT(hfi1_output_ibhdr_template, sdma_output_ibhdr,
+	     TP_PROTO(struct hfi1_devdata *dd,
+		      struct hfi1_opa_header *opah, bool sc5),
+	     TP_ARGS(dd, opah, sc5));
 
-DEFINE_EVENT(hfi1_ibhdr_template, sdma_output_ibhdr,
-	     TP_PROTO(struct hfi1_devdata *dd, struct ib_header *hdr),
-	     TP_ARGS(dd, hdr));
 
 #endif /* __HFI1_TRACE_IBHDRS_H */
 
diff --git a/drivers/infiniband/hw/hfi1/trace_misc.h b/drivers/infiniband/hw/hfi1/trace_misc.h
index deac77ddaeab..8db2253523ff 100644
--- a/drivers/infiniband/hw/hfi1/trace_misc.h
+++ b/drivers/infiniband/hw/hfi1/trace_misc.h
@@ -72,6 +72,26 @@ TRACE_EVENT(hfi1_interrupt,
 		      __entry->src)
 );
 
+DECLARE_EVENT_CLASS(
+	hfi1_csr_template,
+	TP_PROTO(void __iomem *addr, u64 value),
+	TP_ARGS(addr, value),
+	TP_STRUCT__entry(
+		__field(void __iomem *, addr)
+		__field(u64, value)
+	),
+	TP_fast_assign(
+		__entry->addr = addr;
+		__entry->value = value;
+	),
+	TP_printk("addr %p value %llx", __entry->addr, __entry->value)
+);
+
+DEFINE_EVENT(
+	hfi1_csr_template, hfi1_write_rcvarray,
+	TP_PROTO(void __iomem *addr, u64 value),
+	TP_ARGS(addr, value));
+
 #ifdef CONFIG_FAULT_INJECTION
 TRACE_EVENT(hfi1_fault_opcode,
 	    TP_PROTO(struct rvt_qp *qp, u8 opcode),
diff --git a/drivers/infiniband/hw/hfi1/trace_mmu.h b/drivers/infiniband/hw/hfi1/trace_mmu.h
new file mode 100644
index 000000000000..3b7abbc382c2
--- /dev/null
+++ b/drivers/infiniband/hw/hfi1/trace_mmu.h
@@ -0,0 +1,95 @@
+/*
+ * Copyright(c) 2017 Intel Corporation.
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * BSD LICENSE
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  - Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  - Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  - Neither the name of Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+#if !defined(__HFI1_TRACE_MMU_H) || defined(TRACE_HEADER_MULTI_READ)
+#define __HFI1_TRACE_MMU_H
+
+#include <linux/tracepoint.h>
+#include <linux/trace_seq.h>
+
+#include "hfi.h"
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM hfi1_mmu
+
+DECLARE_EVENT_CLASS(hfi1_mmu_rb_template,
+		    TP_PROTO(unsigned long addr, unsigned long len),
+		    TP_ARGS(addr, len),
+		    TP_STRUCT__entry(__field(unsigned long, addr)
+				     __field(unsigned long, len)
+			    ),
+		    TP_fast_assign(__entry->addr = addr;
+				   __entry->len = len;
+			    ),
+		    TP_printk("MMU node addr 0x%lx, len %lu",
+			      __entry->addr,
+			      __entry->len
+			    )
+);
+
+DEFINE_EVENT(hfi1_mmu_rb_template, hfi1_mmu_rb_insert,
+	     TP_PROTO(unsigned long addr, unsigned long len),
+	     TP_ARGS(addr, len));
+
+DEFINE_EVENT(hfi1_mmu_rb_template, hfi1_mmu_rb_search,
+	     TP_PROTO(unsigned long addr, unsigned long len),
+	     TP_ARGS(addr, len));
+
+DEFINE_EVENT(hfi1_mmu_rb_template, hfi1_mmu_rb_remove,
+	     TP_PROTO(unsigned long addr, unsigned long len),
+	     TP_ARGS(addr, len));
+
+DEFINE_EVENT(hfi1_mmu_rb_template, hfi1_mmu_mem_invalidate,
+	     TP_PROTO(unsigned long addr, unsigned long len),
+	     TP_ARGS(addr, len));
+
+#endif /* __HFI1_TRACE_RC_H */
+
+#undef TRACE_INCLUDE_PATH
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_PATH .
+#define TRACE_INCLUDE_FILE trace_mmu
+#include <trace/define_trace.h>
diff --git a/drivers/infiniband/hw/hfi1/trace_rx.h b/drivers/infiniband/hw/hfi1/trace_rx.h
index f77e59fb43fe..f9909d240dcc 100644
--- a/drivers/infiniband/hw/hfi1/trace_rx.h
+++ b/drivers/infiniband/hw/hfi1/trace_rx.h
@@ -52,9 +52,25 @@
 
 #include "hfi.h"
 
+#define tidtype_name(type) { PT_##type, #type }
+#define show_tidtype(type)                   \
+__print_symbolic(type,                       \
+	tidtype_name(EXPECTED),              \
+	tidtype_name(EAGER),                 \
+	tidtype_name(INVALID))               \
+
 #undef TRACE_SYSTEM
 #define TRACE_SYSTEM hfi1_rx
 
+#define packettype_name(etype) { RHF_RCV_TYPE_##etype, #etype }
+#define show_packettype(etype)                  \
+__print_symbolic(etype,                         \
+	packettype_name(EXPECTED),              \
+	packettype_name(EAGER),                 \
+	packettype_name(IB),                    \
+	packettype_name(ERROR),                 \
+	packettype_name(BYPASS))
+
 TRACE_EVENT(hfi1_rcvhdr,
 	    TP_PROTO(struct hfi1_devdata *dd,
 		     u32 ctxt,
@@ -98,24 +114,24 @@ TRACE_EVENT(hfi1_rcvhdr,
 );
 
 TRACE_EVENT(hfi1_receive_interrupt,
-	    TP_PROTO(struct hfi1_devdata *dd, u32 ctxt),
-	    TP_ARGS(dd, ctxt),
+	    TP_PROTO(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd),
+	    TP_ARGS(dd, rcd),
 	    TP_STRUCT__entry(DD_DEV_ENTRY(dd)
 			     __field(u32, ctxt)
 			     __field(u8, slow_path)
 			     __field(u8, dma_rtail)
 			     ),
 	    TP_fast_assign(DD_DEV_ASSIGN(dd);
-			__entry->ctxt = ctxt;
-			if (dd->rcd[ctxt]->do_interrupt ==
+			__entry->ctxt = rcd->ctxt;
+			if (rcd->do_interrupt ==
 			    &handle_receive_interrupt) {
 				__entry->slow_path = 1;
 				__entry->dma_rtail = 0xFF;
-			} else if (dd->rcd[ctxt]->do_interrupt ==
+			} else if (rcd->do_interrupt ==
 					&handle_receive_interrupt_dma_rtail){
 				__entry->dma_rtail = 1;
 				__entry->slow_path = 0;
-			} else if (dd->rcd[ctxt]->do_interrupt ==
+			} else if (rcd->do_interrupt ==
 					&handle_receive_interrupt_nodma_rtail) {
 				__entry->dma_rtail = 0;
 				__entry->slow_path = 0;
@@ -129,7 +145,8 @@ TRACE_EVENT(hfi1_receive_interrupt,
 		      )
 );
 
-TRACE_EVENT(hfi1_exp_tid_reg,
+DECLARE_EVENT_CLASS(
+	    hfi1_exp_tid_reg_unreg,
 	    TP_PROTO(unsigned int ctxt, u16 subctxt, u32 rarr,
 		     u32 npages, unsigned long va, unsigned long pa,
 		     dma_addr_t dma),
@@ -163,38 +180,45 @@ TRACE_EVENT(hfi1_exp_tid_reg,
 		      )
 	);
 
-TRACE_EVENT(hfi1_exp_tid_unreg,
-	    TP_PROTO(unsigned int ctxt, u16 subctxt, u32 rarr, u32 npages,
-		     unsigned long va, unsigned long pa, dma_addr_t dma),
-	    TP_ARGS(ctxt, subctxt, rarr, npages, va, pa, dma),
-	    TP_STRUCT__entry(
-			     __field(unsigned int, ctxt)
-			     __field(u16, subctxt)
-			     __field(u32, rarr)
-			     __field(u32, npages)
-			     __field(unsigned long, va)
-			     __field(unsigned long, pa)
-			     __field(dma_addr_t, dma)
-			     ),
-	    TP_fast_assign(
-			   __entry->ctxt = ctxt;
-			   __entry->subctxt = subctxt;
-			   __entry->rarr = rarr;
-			   __entry->npages = npages;
-			   __entry->va = va;
-			   __entry->pa = pa;
-			   __entry->dma = dma;
-			   ),
-	    TP_printk("[%u:%u] entry:%u, %u pages @ 0x%lx, va:0x%lx dma:0x%llx",
-		      __entry->ctxt,
-		      __entry->subctxt,
-		      __entry->rarr,
-		      __entry->npages,
-		      __entry->pa,
-		      __entry->va,
-		      __entry->dma
-		      )
-	);
+DEFINE_EVENT(
+	hfi1_exp_tid_reg_unreg, hfi1_exp_tid_unreg,
+	TP_PROTO(unsigned int ctxt, u16 subctxt, u32 rarr, u32 npages,
+		 unsigned long va, unsigned long pa, dma_addr_t dma),
+	TP_ARGS(ctxt, subctxt, rarr, npages, va, pa, dma));
+
+DEFINE_EVENT(
+	hfi1_exp_tid_reg_unreg, hfi1_exp_tid_reg,
+	TP_PROTO(unsigned int ctxt, u16 subctxt, u32 rarr, u32 npages,
+		 unsigned long va, unsigned long pa, dma_addr_t dma),
+	TP_ARGS(ctxt, subctxt, rarr, npages, va, pa, dma));
+
+TRACE_EVENT(
+	hfi1_put_tid,
+	TP_PROTO(struct hfi1_devdata *dd,
+		 u32 index, u32 type, unsigned long pa, u16 order),
+	TP_ARGS(dd, index, type, pa, order),
+	TP_STRUCT__entry(
+		DD_DEV_ENTRY(dd)
+		__field(unsigned long, pa);
+		__field(u32, index);
+		__field(u32, type);
+		__field(u16, order);
+	),
+	TP_fast_assign(
+		DD_DEV_ASSIGN(dd);
+		__entry->pa = pa;
+		__entry->index = index;
+		__entry->type = type;
+		__entry->order = order;
+	),
+	TP_printk("[%s] type %s pa %lx index %u order %u",
+		  __get_str(dev),
+		  show_tidtype(__entry->type),
+		  __entry->pa,
+		  __entry->index,
+		  __entry->order
+	)
+);
 
 TRACE_EVENT(hfi1_exp_tid_inval,
 	    TP_PROTO(unsigned int ctxt, u16 subctxt, unsigned long va, u32 rarr,
diff --git a/drivers/infiniband/hw/hfi1/trace_tx.h b/drivers/infiniband/hw/hfi1/trace_tx.h
index c59809a7f121..c57af3b31fe1 100644
--- a/drivers/infiniband/hw/hfi1/trace_tx.h
+++ b/drivers/infiniband/hw/hfi1/trace_tx.h
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015, 2016 Intel Corporation.
+ * Copyright(c) 2015 - 2017 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -198,6 +198,140 @@ TRACE_EVENT(hfi1_sdma_engine_select,
 		      )
 );
 
+TRACE_EVENT(hfi1_sdma_user_free_queues,
+	    TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u16 subctxt),
+	    TP_ARGS(dd, ctxt, subctxt),
+	    TP_STRUCT__entry(DD_DEV_ENTRY(dd)
+			     __field(u16, ctxt)
+			     __field(u16, subctxt)
+			     ),
+	    TP_fast_assign(DD_DEV_ASSIGN(dd);
+			   __entry->ctxt = ctxt;
+			   __entry->subctxt = subctxt;
+			   ),
+	    TP_printk("[%s] SDMA [%u:%u] Freeing user SDMA queues",
+		      __get_str(dev),
+		      __entry->ctxt,
+		      __entry->subctxt
+		      )
+);
+
+TRACE_EVENT(hfi1_sdma_user_process_request,
+	    TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u16 subctxt,
+		     u16 comp_idx),
+	    TP_ARGS(dd, ctxt, subctxt, comp_idx),
+	    TP_STRUCT__entry(DD_DEV_ENTRY(dd)
+			     __field(u16, ctxt)
+			     __field(u16, subctxt)
+			     __field(u16, comp_idx)
+			     ),
+	    TP_fast_assign(DD_DEV_ASSIGN(dd);
+			   __entry->ctxt = ctxt;
+			   __entry->subctxt = subctxt;
+			   __entry->comp_idx = comp_idx;
+			   ),
+	    TP_printk("[%s] SDMA [%u:%u] Using req/comp entry: %u",
+		      __get_str(dev),
+		      __entry->ctxt,
+		      __entry->subctxt,
+		      __entry->comp_idx
+		      )
+);
+
+DECLARE_EVENT_CLASS(
+	hfi1_sdma_value_template,
+	TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u16 subctxt, u16 comp_idx,
+		 u32 value),
+	TP_ARGS(dd, ctxt, subctxt, comp_idx, value),
+	TP_STRUCT__entry(DD_DEV_ENTRY(dd)
+			 __field(u16, ctxt)
+			 __field(u16, subctxt)
+			 __field(u16, comp_idx)
+			 __field(u32, value)
+		),
+	TP_fast_assign(DD_DEV_ASSIGN(dd);
+		       __entry->ctxt = ctxt;
+		       __entry->subctxt = subctxt;
+		       __entry->comp_idx = comp_idx;
+		       __entry->value = value;
+		),
+	TP_printk("[%s] SDMA [%u:%u:%u] value: %u",
+		  __get_str(dev),
+		  __entry->ctxt,
+		  __entry->subctxt,
+		  __entry->comp_idx,
+		  __entry->value
+		)
+);
+
+DEFINE_EVENT(hfi1_sdma_value_template, hfi1_sdma_user_initial_tidoffset,
+	     TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u16 subctxt,
+		      u16 comp_idx, u32 tidoffset),
+	     TP_ARGS(dd, ctxt, subctxt, comp_idx, tidoffset));
+
+DEFINE_EVENT(hfi1_sdma_value_template, hfi1_sdma_user_data_length,
+	     TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u16 subctxt,
+		      u16 comp_idx, u32 data_len),
+	     TP_ARGS(dd, ctxt, subctxt, comp_idx, data_len));
+
+DEFINE_EVENT(hfi1_sdma_value_template, hfi1_sdma_user_compute_length,
+	     TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u16 subctxt,
+		      u16 comp_idx, u32 data_len),
+	     TP_ARGS(dd, ctxt, subctxt, comp_idx, data_len));
+
+TRACE_EVENT(hfi1_sdma_user_tid_info,
+	    TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u16 subctxt,
+		     u16 comp_idx, u32 tidoffset, u32 units, u8 shift),
+	    TP_ARGS(dd, ctxt, subctxt, comp_idx, tidoffset, units, shift),
+	    TP_STRUCT__entry(DD_DEV_ENTRY(dd)
+			     __field(u16, ctxt)
+			     __field(u16, subctxt)
+			     __field(u16, comp_idx)
+			     __field(u32, tidoffset)
+			     __field(u32, units)
+			     __field(u8, shift)
+			     ),
+	    TP_fast_assign(DD_DEV_ASSIGN(dd);
+			   __entry->ctxt = ctxt;
+			   __entry->subctxt = subctxt;
+			   __entry->comp_idx = comp_idx;
+			   __entry->tidoffset = tidoffset;
+			   __entry->units = units;
+			   __entry->shift = shift;
+			   ),
+	    TP_printk("[%s] SDMA [%u:%u:%u] TID offset %ubytes %uunits om %u",
+		      __get_str(dev),
+		      __entry->ctxt,
+		      __entry->subctxt,
+		      __entry->comp_idx,
+		      __entry->tidoffset,
+		      __entry->units,
+		      __entry->shift
+		      )
+);
+
+TRACE_EVENT(hfi1_sdma_request,
+	    TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u16 subctxt,
+		     unsigned long dim),
+	    TP_ARGS(dd, ctxt, subctxt, dim),
+	    TP_STRUCT__entry(DD_DEV_ENTRY(dd)
+			     __field(u16, ctxt)
+			     __field(u16, subctxt)
+			     __field(unsigned long, dim)
+			     ),
+	    TP_fast_assign(DD_DEV_ASSIGN(dd);
+			   __entry->ctxt = ctxt;
+			   __entry->subctxt = subctxt;
+			   __entry->dim = dim;
+			   ),
+	    TP_printk("[%s] SDMA from %u:%u (%lu)",
+		      __get_str(dev),
+		      __entry->ctxt,
+		      __entry->subctxt,
+		      __entry->dim
+		      )
+);
+
 DECLARE_EVENT_CLASS(hfi1_sdma_engine_class,
 		    TP_PROTO(struct sdma_engine *sde, u64 status),
 		    TP_ARGS(sde, status),
diff --git a/drivers/infiniband/hw/hfi1/uc.c b/drivers/infiniband/hw/hfi1/uc.c
index 5da1e4546543..0b646173ca22 100644
--- a/drivers/infiniband/hw/hfi1/uc.c
+++ b/drivers/infiniband/hw/hfi1/uc.c
@@ -65,7 +65,7 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 	struct hfi1_qp_priv *priv = qp->priv;
 	struct ib_other_headers *ohdr;
 	struct rvt_swqe *wqe;
-	u32 hwords = 5;
+	u32 hwords;
 	u32 bth0 = 0;
 	u32 len;
 	u32 pmtu = qp->pmtu;
@@ -93,9 +93,23 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 		goto done_free_tx;
 	}
 
-	ohdr = &ps->s_txreq->phdr.hdr.u.oth;
-	if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)
-		ohdr = &ps->s_txreq->phdr.hdr.u.l.oth;
+	ps->s_txreq->phdr.hdr.hdr_type = priv->hdr_type;
+	if (priv->hdr_type == HFI1_PKT_TYPE_9B) {
+		/* header size in 32-bit words LRH+BTH = (8+12)/4. */
+		hwords = 5;
+		if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)
+			ohdr = &ps->s_txreq->phdr.hdr.ibh.u.l.oth;
+		else
+			ohdr = &ps->s_txreq->phdr.hdr.ibh.u.oth;
+	} else {
+		/* header size in 32-bit words 16B LRH+BTH = (16+12)/4. */
+		hwords = 7;
+		if ((rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) &&
+		    (hfi1_check_mcast(rdma_ah_get_dlid(&qp->remote_ah_attr))))
+			ohdr = &ps->s_txreq->phdr.hdr.opah.u.l.oth;
+		else
+			ohdr = &ps->s_txreq->phdr.hdr.opah.u.oth;
+	}
 
 	/* Get the next send request. */
 	wqe = rvt_get_swqe_ptr(qp, qp->s_cur);
@@ -297,31 +311,26 @@ bail_no_tx:
 void hfi1_uc_rcv(struct hfi1_packet *packet)
 {
 	struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd);
-	struct ib_header *hdr = packet->hdr;
-	u32 rcv_flags = packet->rcv_flags;
-	void *data = packet->ebuf;
+	void *data = packet->payload;
 	u32 tlen = packet->tlen;
 	struct rvt_qp *qp = packet->qp;
 	struct ib_other_headers *ohdr = packet->ohdr;
-	u32 bth0, opcode;
+	u32 opcode = packet->opcode;
 	u32 hdrsize = packet->hlen;
 	u32 psn;
-	u32 pad;
+	u32 pad = packet->pad;
 	struct ib_wc wc;
 	u32 pmtu = qp->pmtu;
 	struct ib_reth *reth;
-	int has_grh = rcv_flags & HFI1_HAS_GRH;
 	int ret;
+	u8 extra_bytes = pad + packet->extra_byte + (SIZE_OF_CRC << 2);
 
-	bth0 = be32_to_cpu(ohdr->bth[0]);
-	if (hfi1_ruc_check_hdr(ibp, hdr, has_grh, qp, bth0))
+	if (hfi1_ruc_check_hdr(ibp, packet))
 		return;
 
 	process_ecn(qp, packet, true);
 
-	psn = be32_to_cpu(ohdr->bth[2]);
-	opcode = ib_bth_get_opcode(ohdr);
-
+	psn = ib_bth_get_psn(ohdr);
 	/* Compare the PSN verses the expected PSN. */
 	if (unlikely(cmp_psn(psn, qp->r_psn) != 0)) {
 		/*
@@ -414,7 +423,12 @@ send_first:
 		/* FALLTHROUGH */
 	case OP(SEND_MIDDLE):
 		/* Check for invalid length PMTU or posted rwqe len. */
-		if (unlikely(tlen != (hdrsize + pmtu + 4)))
+		/*
+		 * There will be no padding for 9B packet but 16B packets
+		 * will come in with some padding since we always add
+		 * CRC and LT bytes which will need to be flit aligned
+		 */
+		if (unlikely(tlen != (hdrsize + pmtu + extra_bytes)))
 			goto rewind;
 		qp->r_rcv_len += pmtu;
 		if (unlikely(qp->r_rcv_len > qp->r_len))
@@ -432,14 +446,12 @@ no_immediate_data:
 		wc.ex.imm_data = 0;
 		wc.wc_flags = 0;
 send_last:
-		/* Get the number of bytes the message was padded by. */
-		pad = ib_bth_get_pad(ohdr);
 		/* Check for invalid length. */
 		/* LAST len should be >= 1 */
-		if (unlikely(tlen < (hdrsize + pad + 4)))
+		if (unlikely(tlen < (hdrsize + extra_bytes)))
 			goto rewind;
 		/* Don't count the CRC. */
-		tlen -= (hdrsize + pad + 4);
+		tlen -= (hdrsize + extra_bytes);
 		wc.byte_len = tlen + qp->r_rcv_len;
 		if (unlikely(wc.byte_len > qp->r_len))
 			goto rewind;
@@ -527,14 +539,12 @@ rdma_first:
 rdma_last_imm:
 		wc.wc_flags = IB_WC_WITH_IMM;
 
-		/* Get the number of bytes the message was padded by. */
-		pad = ib_bth_get_pad(ohdr);
 		/* Check for invalid length. */
 		/* LAST len should be >= 1 */
 		if (unlikely(tlen < (hdrsize + pad + 4)))
 			goto drop;
 		/* Don't count the CRC. */
-		tlen -= (hdrsize + pad + 4);
+		tlen -= (hdrsize + extra_bytes);
 		if (unlikely(tlen + qp->r_rcv_len != qp->r_len))
 			goto drop;
 		if (test_and_clear_bit(RVT_R_REWIND_SGE, &qp->r_aflags)) {
@@ -554,14 +564,12 @@ rdma_last_imm:
 
 	case OP(RDMA_WRITE_LAST):
 rdma_last:
-		/* Get the number of bytes the message was padded by. */
-		pad = ib_bth_get_pad(ohdr);
 		/* Check for invalid length. */
 		/* LAST len should be >= 1 */
 		if (unlikely(tlen < (hdrsize + pad + 4)))
 			goto drop;
 		/* Don't count the CRC. */
-		tlen -= (hdrsize + pad + 4);
+		tlen -= (hdrsize + extra_bytes);
 		if (unlikely(tlen + qp->r_rcv_len != qp->r_len))
 			goto drop;
 		hfi1_copy_sge(&qp->r_sge, data, tlen, true, false);
diff --git a/drivers/infiniband/hw/hfi1/ud.c b/drivers/infiniband/hw/hfi1/ud.c
index 6a4e95cefae5..2ba74fdd6f15 100644
--- a/drivers/infiniband/hw/hfi1/ud.c
+++ b/drivers/infiniband/hw/hfi1/ud.c
@@ -53,6 +53,12 @@
 #include "verbs_txreq.h"
 #include "qp.h"
 
+/* We support only two types - 9B and 16B for now */
+static const hfi1_make_req hfi1_make_ud_req_tbl[2] = {
+	[HFI1_PKT_TYPE_9B] = &hfi1_make_ud_req_9B,
+	[HFI1_PKT_TYPE_16B] = &hfi1_make_ud_req_16B
+};
+
 /**
  * ud_loopback - handle send on loopback QPs
  * @sqp: the sending QP
@@ -67,6 +73,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe)
 {
 	struct hfi1_ibport *ibp = to_iport(sqp->ibqp.device, sqp->port_num);
 	struct hfi1_pportdata *ppd;
+	struct hfi1_qp_priv *priv = sqp->priv;
 	struct rvt_qp *qp;
 	struct rdma_ah_attr *ah_attr;
 	unsigned long flags;
@@ -102,18 +109,19 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe)
 
 	if (qp->ibqp.qp_num > 1) {
 		u16 pkey;
-		u16 slid;
+		u32 slid;
 		u8 sc5 = ibp->sl_to_sc[rdma_ah_get_sl(ah_attr)];
 
 		pkey = hfi1_get_pkey(ibp, sqp->s_pkey_index);
 		slid = ppd->lid | (rdma_ah_get_path_bits(ah_attr) &
 				   ((1 << ppd->lmc) - 1));
 		if (unlikely(ingress_pkey_check(ppd, pkey, sc5,
-						qp->s_pkey_index, slid))) {
-			hfi1_bad_pqkey(ibp, OPA_TRAP_BAD_P_KEY, pkey,
-				       rdma_ah_get_sl(ah_attr),
-				       sqp->ibqp.qp_num, qp->ibqp.qp_num,
-				       slid, rdma_ah_get_dlid(ah_attr));
+						qp->s_pkey_index,
+						slid, false))) {
+			hfi1_bad_pkey(ibp, pkey,
+				      rdma_ah_get_sl(ah_attr),
+				      sqp->ibqp.qp_num, qp->ibqp.qp_num,
+				      slid, rdma_ah_get_dlid(ah_attr));
 			goto drop;
 		}
 	}
@@ -128,18 +136,8 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe)
 
 		qkey = (int)swqe->ud_wr.remote_qkey < 0 ?
 			sqp->qkey : swqe->ud_wr.remote_qkey;
-		if (unlikely(qkey != qp->qkey)) {
-			u16 lid;
-
-			lid = ppd->lid | (rdma_ah_get_path_bits(ah_attr) &
-					  ((1 << ppd->lmc) - 1));
-			hfi1_bad_pqkey(ibp, OPA_TRAP_BAD_Q_KEY, qkey,
-				       rdma_ah_get_sl(ah_attr),
-				       sqp->ibqp.qp_num, qp->ibqp.qp_num,
-				       lid,
-				       rdma_ah_get_dlid(ah_attr));
-			goto drop;
-		}
+		if (unlikely(qkey != qp->qkey))
+			goto drop; /* silently drop per IBTA spec */
 	}
 
 	/*
@@ -185,9 +183,33 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe)
 
 	if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) {
 		struct ib_grh grh;
-		const struct ib_global_route *grd = rdma_ah_read_grh(ah_attr);
+		struct ib_global_route grd = *(rdma_ah_read_grh(ah_attr));
+
+		/*
+		 * For loopback packets with extended LIDs, the
+		 * sgid_index in the GRH is 0 and the dgid is
+		 * OPA GID of the sender. While creating a response
+		 * to the loopback packet, IB core creates the new
+		 * sgid_index from the DGID and that will be the
+		 * OPA_GID_INDEX. The new dgid is from the sgid
+		 * index and that will be in the IB GID format.
+		 *
+		 * We now have a case where the sent packet had a
+		 * different sgid_index and dgid compared to the
+		 * one that was received in response.
+		 *
+		 * Fix this inconsistency.
+		 */
+		if (priv->hdr_type == HFI1_PKT_TYPE_16B) {
+			if (grd.sgid_index == 0)
+				grd.sgid_index = OPA_GID_INDEX;
+
+			if (ib_is_opa_gid(&grd.dgid))
+				grd.dgid.global.interface_id =
+				cpu_to_be64(ppd->guids[HFI1_PORT_GUID_INDEX]);
+		}
 
-		hfi1_make_grh(ibp, &grh, grd, 0, 0);
+		hfi1_make_grh(ibp, &grh, &grd, 0, 0);
 		hfi1_copy_sge(&qp->r_sge, &grh,
 			      sizeof(grh), true, false);
 		wc.wc_flags |= IB_WC_GRH;
@@ -244,7 +266,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe)
 		wc.pkey_index = 0;
 	}
 	wc.slid = ppd->lid | (rdma_ah_get_path_bits(ah_attr) &
-			      ((1 << ppd->lmc) - 1));
+				   ((1 << ppd->lmc) - 1));
 	/* Check for loopback when the port lid is not set */
 	if (wc.slid == 0 && sqp->ibqp.qp_type == IB_QPT_GSI)
 		wc.slid = be16_to_cpu(IB_LID_PERMISSIVE);
@@ -261,6 +283,183 @@ drop:
 	rcu_read_unlock();
 }
 
+static void hfi1_make_bth_deth(struct rvt_qp *qp, struct rvt_swqe *wqe,
+			       struct ib_other_headers *ohdr,
+			       u16 *pkey, u32 extra_bytes, bool bypass)
+{
+	u32 bth0;
+	struct hfi1_ibport *ibp;
+
+	ibp = to_iport(qp->ibqp.device, qp->port_num);
+	if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) {
+		ohdr->u.ud.imm_data = wqe->wr.ex.imm_data;
+		bth0 = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE << 24;
+	} else {
+		bth0 = IB_OPCODE_UD_SEND_ONLY << 24;
+	}
+
+	if (wqe->wr.send_flags & IB_SEND_SOLICITED)
+		bth0 |= IB_BTH_SOLICITED;
+	bth0 |= extra_bytes << 20;
+	if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_SMI)
+		*pkey = hfi1_get_pkey(ibp, wqe->ud_wr.pkey_index);
+	else
+		*pkey = hfi1_get_pkey(ibp, qp->s_pkey_index);
+	if (!bypass)
+		bth0 |= *pkey;
+	ohdr->bth[0] = cpu_to_be32(bth0);
+	ohdr->bth[1] = cpu_to_be32(wqe->ud_wr.remote_qpn);
+	ohdr->bth[2] = cpu_to_be32(mask_psn(wqe->psn));
+	/*
+	 * Qkeys with the high order bit set mean use the
+	 * qkey from the QP context instead of the WR (see 10.2.5).
+	 */
+	ohdr->u.ud.deth[0] = cpu_to_be32((int)wqe->ud_wr.remote_qkey < 0 ?
+					 qp->qkey : wqe->ud_wr.remote_qkey);
+	ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num);
+}
+
+void hfi1_make_ud_req_9B(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
+			 struct rvt_swqe *wqe)
+{
+	u32 nwords, extra_bytes;
+	u16 len, slid, dlid, pkey;
+	u16 lrh0 = 0;
+	u8 sc5;
+	struct hfi1_qp_priv *priv = qp->priv;
+	struct ib_other_headers *ohdr;
+	struct rdma_ah_attr *ah_attr;
+	struct hfi1_pportdata *ppd;
+	struct hfi1_ibport *ibp;
+	struct ib_grh *grh;
+
+	ibp = to_iport(qp->ibqp.device, qp->port_num);
+	ppd = ppd_from_ibp(ibp);
+	ah_attr = &ibah_to_rvtah(wqe->ud_wr.ah)->attr;
+
+	extra_bytes = -wqe->length & 3;
+	nwords = ((wqe->length + extra_bytes) >> 2) + SIZE_OF_CRC;
+	/* header size in dwords LRH+BTH+DETH = (8+12+8)/4. */
+	qp->s_hdrwords = 7;
+	if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM)
+		qp->s_hdrwords++;
+
+	if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) {
+		grh = &ps->s_txreq->phdr.hdr.ibh.u.l.grh;
+		qp->s_hdrwords += hfi1_make_grh(ibp, grh,
+						rdma_ah_read_grh(ah_attr),
+						qp->s_hdrwords - 2, nwords);
+		lrh0 = HFI1_LRH_GRH;
+		ohdr = &ps->s_txreq->phdr.hdr.ibh.u.l.oth;
+	} else {
+		lrh0 = HFI1_LRH_BTH;
+		ohdr = &ps->s_txreq->phdr.hdr.ibh.u.oth;
+	}
+
+	sc5 = ibp->sl_to_sc[rdma_ah_get_sl(ah_attr)];
+	lrh0 |= (rdma_ah_get_sl(ah_attr) & 0xf) << 4;
+	if (qp->ibqp.qp_type == IB_QPT_SMI) {
+		lrh0 |= 0xF000; /* Set VL (see ch. 13.5.3.1) */
+		priv->s_sc = 0xf;
+	} else {
+		lrh0 |= (sc5 & 0xf) << 12;
+		priv->s_sc = sc5;
+	}
+
+	dlid = opa_get_lid(rdma_ah_get_dlid(ah_attr), 9B);
+	if (dlid == be16_to_cpu(IB_LID_PERMISSIVE)) {
+		slid = be16_to_cpu(IB_LID_PERMISSIVE);
+	} else {
+		u16 lid = (u16)ppd->lid;
+
+		if (lid) {
+			lid |= rdma_ah_get_path_bits(ah_attr) &
+				((1 << ppd->lmc) - 1);
+			slid = lid;
+		} else {
+			slid = be16_to_cpu(IB_LID_PERMISSIVE);
+		}
+	}
+	hfi1_make_bth_deth(qp, wqe, ohdr, &pkey, extra_bytes, false);
+	len = qp->s_hdrwords + nwords;
+
+	/* Setup the packet */
+	ps->s_txreq->phdr.hdr.hdr_type = HFI1_PKT_TYPE_9B;
+	hfi1_make_ib_hdr(&ps->s_txreq->phdr.hdr.ibh,
+			 lrh0, len, dlid, slid);
+}
+
+void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
+			  struct rvt_swqe *wqe)
+{
+	struct hfi1_qp_priv *priv = qp->priv;
+	struct ib_other_headers *ohdr;
+	struct rdma_ah_attr *ah_attr;
+	struct hfi1_pportdata *ppd;
+	struct hfi1_ibport *ibp;
+	u32 dlid, slid, nwords, extra_bytes;
+	u16 len, pkey;
+	u8 l4, sc5;
+
+	ibp = to_iport(qp->ibqp.device, qp->port_num);
+	ppd = ppd_from_ibp(ibp);
+	ah_attr = &ibah_to_rvtah(wqe->ud_wr.ah)->attr;
+	/* header size in dwords 16B LRH+BTH+DETH = (16+12+8)/4. */
+	qp->s_hdrwords = 9;
+	if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM)
+		qp->s_hdrwords++;
+
+	/* SW provides space for CRC and LT for bypass packets. */
+	extra_bytes = hfi1_get_16b_padding((qp->s_hdrwords << 2),
+					   wqe->length);
+	nwords = ((wqe->length + extra_bytes + SIZE_OF_LT) >> 2) + SIZE_OF_CRC;
+
+	if ((rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) &&
+	    hfi1_check_mcast(rdma_ah_get_dlid(ah_attr))) {
+		struct ib_grh *grh;
+		struct ib_global_route *grd = rdma_ah_retrieve_grh(ah_attr);
+		/*
+		 * Ensure OPA GIDs are transformed to IB gids
+		 * before creating the GRH.
+		 */
+		if (grd->sgid_index == OPA_GID_INDEX) {
+			dd_dev_warn(ppd->dd, "Bad sgid_index. sgid_index: %d\n",
+				    grd->sgid_index);
+			grd->sgid_index = 0;
+		}
+		grh = &ps->s_txreq->phdr.hdr.opah.u.l.grh;
+		qp->s_hdrwords += hfi1_make_grh(ibp, grh, grd,
+					qp->s_hdrwords - 4, nwords);
+		ohdr = &ps->s_txreq->phdr.hdr.opah.u.l.oth;
+		l4 = OPA_16B_L4_IB_GLOBAL;
+	} else {
+		ohdr = &ps->s_txreq->phdr.hdr.opah.u.oth;
+		l4 = OPA_16B_L4_IB_LOCAL;
+	}
+
+	sc5 = ibp->sl_to_sc[rdma_ah_get_sl(ah_attr)];
+	if (qp->ibqp.qp_type == IB_QPT_SMI)
+		priv->s_sc = 0xf;
+	else
+		priv->s_sc = sc5;
+
+	dlid = opa_get_lid(rdma_ah_get_dlid(ah_attr), 16B);
+	if (!ppd->lid)
+		slid = be32_to_cpu(OPA_LID_PERMISSIVE);
+	else
+		slid = ppd->lid | (rdma_ah_get_path_bits(ah_attr) &
+			   ((1 << ppd->lmc) - 1));
+
+	hfi1_make_bth_deth(qp, wqe, ohdr, &pkey, extra_bytes, true);
+	/* Convert dwords to flits */
+	len = (qp->s_hdrwords + nwords) >> 1;
+
+	/* Setup the packet */
+	ps->s_txreq->phdr.hdr.hdr_type = HFI1_PKT_TYPE_16B;
+	hfi1_make_16b_hdr(&ps->s_txreq->phdr.hdr.opah,
+			  slid, dlid, len, pkey, 0, 0, l4, priv->s_sc);
+}
+
 /**
  * hfi1_make_ud_req - construct a UD request packet
  * @qp: the QP
@@ -272,18 +471,12 @@ drop:
 int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 {
 	struct hfi1_qp_priv *priv = qp->priv;
-	struct ib_other_headers *ohdr;
 	struct rdma_ah_attr *ah_attr;
 	struct hfi1_pportdata *ppd;
 	struct hfi1_ibport *ibp;
 	struct rvt_swqe *wqe;
-	u32 nwords;
-	u32 extra_bytes;
-	u32 bth0;
-	u16 lrh0;
-	u16 lid;
 	int next_cur;
-	u8 sc5;
+	u32 lid;
 
 	ps->s_txreq = get_txreq(ps->dev, qp);
 	if (IS_ERR(ps->s_txreq))
@@ -320,13 +513,14 @@ int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 	ibp = to_iport(qp->ibqp.device, qp->port_num);
 	ppd = ppd_from_ibp(ibp);
 	ah_attr = &ibah_to_rvtah(wqe->ud_wr.ah)->attr;
-	if (rdma_ah_get_dlid(ah_attr) < be16_to_cpu(IB_MULTICAST_LID_BASE) ||
-	    rdma_ah_get_dlid(ah_attr) == be16_to_cpu(IB_LID_PERMISSIVE)) {
+	priv->hdr_type = hfi1_get_hdr_type(ppd->lid, ah_attr);
+	if ((!hfi1_check_mcast(rdma_ah_get_dlid(ah_attr))) ||
+	    (rdma_ah_get_dlid(ah_attr) == be32_to_cpu(OPA_LID_PERMISSIVE))) {
 		lid = rdma_ah_get_dlid(ah_attr) & ~((1 << ppd->lmc) - 1);
 		if (unlikely(!loopback &&
-			     (lid == ppd->lid ||
-			      (lid == be16_to_cpu(IB_LID_PERMISSIVE) &&
-			      qp->ibqp.qp_type == IB_QPT_GSI)))) {
+			     ((lid == ppd->lid) ||
+			      ((lid == be32_to_cpu(OPA_LID_PERMISSIVE)) &&
+			       (qp->ibqp.qp_type == IB_QPT_GSI))))) {
 			unsigned long tflags = ps->flags;
 			/*
 			 * If DMAs are in progress, we can't generate
@@ -350,11 +544,6 @@ int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 	}
 
 	qp->s_cur = next_cur;
-	extra_bytes = -wqe->length & 3;
-	nwords = (wqe->length + extra_bytes) >> 2;
-
-	/* header size in 32-bit words LRH+BTH+DETH = (8+12+8)/4. */
-	qp->s_hdrwords = 7;
 	ps->s_txreq->s_cur_size = wqe->length;
 	ps->s_txreq->ss = &qp->s_sge;
 	qp->s_srate = rdma_ah_get_static_rate(ah_attr);
@@ -365,77 +554,12 @@ int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 	qp->s_sge.num_sge = wqe->wr.num_sge;
 	qp->s_sge.total_len = wqe->length;
 
-	if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) {
-		/* Header size in 32-bit words. */
-		qp->s_hdrwords += hfi1_make_grh(ibp,
-						&ps->s_txreq->phdr.hdr.u.l.grh,
-						rdma_ah_read_grh(ah_attr),
-						qp->s_hdrwords, nwords);
-		lrh0 = HFI1_LRH_GRH;
-		ohdr = &ps->s_txreq->phdr.hdr.u.l.oth;
-		/*
-		 * Don't worry about sending to locally attached multicast
-		 * QPs.  It is unspecified by the spec. what happens.
-		 */
-	} else {
-		/* Header size in 32-bit words. */
-		lrh0 = HFI1_LRH_BTH;
-		ohdr = &ps->s_txreq->phdr.hdr.u.oth;
-	}
-	if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) {
-		qp->s_hdrwords++;
-		ohdr->u.ud.imm_data = wqe->wr.ex.imm_data;
-		bth0 = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE << 24;
-	} else {
-		bth0 = IB_OPCODE_UD_SEND_ONLY << 24;
-	}
-	sc5 = ibp->sl_to_sc[rdma_ah_get_sl(ah_attr)];
-	lrh0 |= (rdma_ah_get_sl(ah_attr) & 0xf) << 4;
-	if (qp->ibqp.qp_type == IB_QPT_SMI) {
-		lrh0 |= 0xF000; /* Set VL (see ch. 13.5.3.1) */
-		priv->s_sc = 0xf;
-	} else {
-		lrh0 |= (sc5 & 0xf) << 12;
-		priv->s_sc = sc5;
-	}
+	/* Make the appropriate header */
+	hfi1_make_ud_req_tbl[priv->hdr_type](qp, ps, qp->s_wqe);
 	priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc);
 	ps->s_txreq->sde = priv->s_sde;
 	priv->s_sendcontext = qp_to_send_context(qp, priv->s_sc);
 	ps->s_txreq->psc = priv->s_sendcontext;
-	ps->s_txreq->phdr.hdr.lrh[0] = cpu_to_be16(lrh0);
-	ps->s_txreq->phdr.hdr.lrh[1] =
-		cpu_to_be16(rdma_ah_get_dlid(ah_attr));
-	ps->s_txreq->phdr.hdr.lrh[2] =
-		cpu_to_be16(qp->s_hdrwords + nwords + SIZE_OF_CRC);
-	if (rdma_ah_get_dlid(ah_attr) == be16_to_cpu(IB_LID_PERMISSIVE)) {
-		ps->s_txreq->phdr.hdr.lrh[3] = IB_LID_PERMISSIVE;
-	} else {
-		lid = ppd->lid;
-		if (lid) {
-			lid |= rdma_ah_get_path_bits(ah_attr) &
-				((1 << ppd->lmc) - 1);
-			ps->s_txreq->phdr.hdr.lrh[3] = cpu_to_be16(lid);
-		} else {
-			ps->s_txreq->phdr.hdr.lrh[3] = IB_LID_PERMISSIVE;
-		}
-	}
-	if (wqe->wr.send_flags & IB_SEND_SOLICITED)
-		bth0 |= IB_BTH_SOLICITED;
-	bth0 |= extra_bytes << 20;
-	if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_SMI)
-		bth0 |= hfi1_get_pkey(ibp, wqe->ud_wr.pkey_index);
-	else
-		bth0 |= hfi1_get_pkey(ibp, qp->s_pkey_index);
-	ohdr->bth[0] = cpu_to_be32(bth0);
-	ohdr->bth[1] = cpu_to_be32(wqe->ud_wr.remote_qpn);
-	ohdr->bth[2] = cpu_to_be32(mask_psn(wqe->psn));
-	/*
-	 * Qkeys with the high order bit set mean use the
-	 * qkey from the QP context instead of the WR (see 10.2.5).
-	 */
-	ohdr->u.ud.deth[0] = cpu_to_be32((int)wqe->ud_wr.remote_qkey < 0 ?
-					 qp->qkey : wqe->ud_wr.remote_qkey);
-	ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num);
 	/* disarm any ahg */
 	priv->s_ahg->ahgcount = 0;
 	priv->s_ahg->ahgidx = 0;
@@ -505,6 +629,64 @@ int hfi1_lookup_pkey_idx(struct hfi1_ibport *ibp, u16 pkey)
 	return -1;
 }
 
+void return_cnp_16B(struct hfi1_ibport *ibp, struct rvt_qp *qp,
+		    u32 remote_qpn, u32 pkey, u32 slid, u32 dlid,
+		    u8 sc5, const struct ib_grh *old_grh)
+{
+	u64 pbc, pbc_flags = 0;
+	u32 bth0, plen, vl, hwords = 7;
+	u16 len;
+	u8 l4;
+	struct hfi1_16b_header hdr;
+	struct ib_other_headers *ohdr;
+	struct pio_buf *pbuf;
+	struct send_context *ctxt = qp_to_send_context(qp, sc5);
+	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
+	u32 nwords;
+
+	/* Populate length */
+	nwords = ((hfi1_get_16b_padding(hwords << 2, 0) +
+		   SIZE_OF_LT) >> 2) + SIZE_OF_CRC;
+	if (old_grh) {
+		struct ib_grh *grh = &hdr.u.l.grh;
+
+		grh->version_tclass_flow = old_grh->version_tclass_flow;
+		grh->paylen = cpu_to_be16((hwords - 4 + nwords) << 2);
+		grh->hop_limit = 0xff;
+		grh->sgid = old_grh->dgid;
+		grh->dgid = old_grh->sgid;
+		ohdr = &hdr.u.l.oth;
+		l4 = OPA_16B_L4_IB_GLOBAL;
+		hwords += sizeof(struct ib_grh) / sizeof(u32);
+	} else {
+		ohdr = &hdr.u.oth;
+		l4 = OPA_16B_L4_IB_LOCAL;
+	}
+
+	/* BIT 16 to 19 is TVER. Bit 20 to 22 is pad cnt */
+	bth0 = (IB_OPCODE_CNP << 24) | (1 << 16) |
+	       (hfi1_get_16b_padding(hwords << 2, 0) << 20);
+	ohdr->bth[0] = cpu_to_be32(bth0);
+
+	ohdr->bth[1] = cpu_to_be32(remote_qpn);
+	ohdr->bth[2] = 0; /* PSN 0 */
+
+	/* Convert dwords to flits */
+	len = (hwords + nwords) >> 1;
+	hfi1_make_16b_hdr(&hdr, slid, dlid, len, pkey, 1, 0, l4, sc5);
+
+	plen = 2 /* PBC */ + hwords + nwords;
+	pbc_flags |= PBC_PACKET_BYPASS | PBC_INSERT_BYPASS_ICRC;
+	vl = sc_to_vlt(ppd->dd, sc5);
+	pbc = create_pbc(ppd, pbc_flags, qp->srate_mbps, vl, plen);
+	if (ctxt) {
+		pbuf = sc_buffer_alloc(ctxt, plen, NULL, NULL);
+		if (pbuf)
+			ppd->dd->pio_inline_send(ppd->dd, pbuf, pbc,
+						 &hdr, hwords);
+	}
+}
+
 void return_cnp(struct hfi1_ibport *ibp, struct rvt_qp *qp, u32 remote_qpn,
 		u32 pkey, u32 slid, u32 dlid, u8 sc5,
 		const struct ib_grh *old_grh)
@@ -543,13 +725,9 @@ void return_cnp(struct hfi1_ibport *ibp, struct rvt_qp *qp, u32 remote_qpn,
 	ohdr->bth[1] = cpu_to_be32(remote_qpn | (1 << IB_BECN_SHIFT));
 	ohdr->bth[2] = 0; /* PSN 0 */
 
-	hdr.lrh[0] = cpu_to_be16(lrh0);
-	hdr.lrh[1] = cpu_to_be16(dlid);
-	hdr.lrh[2] = cpu_to_be16(hwords + SIZE_OF_CRC);
-	hdr.lrh[3] = cpu_to_be16(slid);
-
+	hfi1_make_ib_hdr(&hdr, lrh0, hwords + SIZE_OF_CRC, dlid, slid);
 	plen = 2 /* PBC */ + hwords;
-	pbc_flags |= (!!(sc5 & 0x10)) << PBC_DC_INFO_SHIFT;
+	pbc_flags |= (ib_is_sc5(sc5) << PBC_DC_INFO_SHIFT);
 	vl = sc_to_vlt(ppd->dd, sc5);
 	pbc = create_pbc(ppd, pbc_flags, qp->srate_mbps, vl, plen);
 	if (ctxt) {
@@ -668,37 +846,45 @@ static int opa_smp_check(struct hfi1_ibport *ibp, u16 pkey, u8 sc5,
 void hfi1_ud_rcv(struct hfi1_packet *packet)
 {
 	struct ib_other_headers *ohdr = packet->ohdr;
-	int opcode;
 	u32 hdrsize = packet->hlen;
 	struct ib_wc wc;
 	u32 qkey;
 	u32 src_qp;
-	u16 dlid, pkey;
+	u16 pkey;
 	int mgmt_pkey_idx = -1;
 	struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd);
 	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
 	struct ib_header *hdr = packet->hdr;
-	u32 rcv_flags = packet->rcv_flags;
-	void *data = packet->ebuf;
+	void *data = packet->payload;
 	u32 tlen = packet->tlen;
 	struct rvt_qp *qp = packet->qp;
-	bool has_grh = rcv_flags & HFI1_HAS_GRH;
-	u8 sc5 = hfi1_9B_get_sc5(hdr, packet->rhf);
-	u32 bth1;
-	u8 sl_from_sc, sl;
-	u16 slid;
+	u8 sc5 = packet->sc;
+	u8 sl_from_sc;
+	u8 opcode = packet->opcode;
+	u8 sl = packet->sl;
+	u32 dlid = packet->dlid;
+	u32 slid = packet->slid;
 	u8 extra_bytes;
+	bool dlid_is_permissive;
+	bool slid_is_permissive;
 
-	qkey = be32_to_cpu(ohdr->u.ud.deth[0]);
-	src_qp = be32_to_cpu(ohdr->u.ud.deth[1]) & RVT_QPN_MASK;
-	dlid = ib_get_dlid(hdr);
-	bth1 = be32_to_cpu(ohdr->bth[1]);
-	slid = ib_get_slid(hdr);
-	pkey = ib_bth_get_pkey(ohdr);
-	opcode = ib_bth_get_opcode(ohdr);
-	sl = ib_get_sl(hdr);
-	extra_bytes = ib_bth_get_pad(ohdr);
-	extra_bytes += (SIZE_OF_CRC << 2);
+	extra_bytes = packet->pad + packet->extra_byte + (SIZE_OF_CRC << 2);
+	qkey = ib_get_qkey(ohdr);
+	src_qp = ib_get_sqpn(ohdr);
+
+	if (packet->etype == RHF_RCV_TYPE_BYPASS) {
+		u32 permissive_lid =
+			opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE), 16B);
+
+		pkey = hfi1_16B_get_pkey(packet->hdr);
+		dlid_is_permissive = (dlid == permissive_lid);
+		slid_is_permissive = (slid == permissive_lid);
+	} else {
+		hdr = packet->hdr;
+		pkey = ib_bth_get_pkey(ohdr);
+		dlid_is_permissive = (dlid == be16_to_cpu(IB_LID_PERMISSIVE));
+		slid_is_permissive = (slid == be16_to_cpu(IB_LID_PERMISSIVE));
+	}
 	sl_from_sc = ibp->sc_to_sl[sc5];
 
 	process_ecn(qp, packet, (opcode != IB_OPCODE_CNP));
@@ -716,8 +902,7 @@ void hfi1_ud_rcv(struct hfi1_packet *packet)
 	 * and the QKEY matches (see 9.6.1.4.1 and 9.6.1.5.1).
 	 */
 	if (qp->ibqp.qp_num) {
-		if (unlikely(hdr->lrh[1] == IB_LID_PERMISSIVE ||
-			     hdr->lrh[3] == IB_LID_PERMISSIVE))
+		if (unlikely(dlid_is_permissive || slid_is_permissive))
 			goto drop;
 		if (qp->ibqp.qp_num > 1) {
 			if (unlikely(rcv_pkey_check(ppd, pkey, sc5, slid))) {
@@ -727,10 +912,10 @@ void hfi1_ud_rcv(struct hfi1_packet *packet)
 				 * for invalid pkeys is optional according to
 				 * IB spec (release 1.3, section 10.9.4)
 				 */
-				hfi1_bad_pqkey(ibp, OPA_TRAP_BAD_P_KEY,
-					       pkey, sl,
-					       src_qp, qp->ibqp.qp_num,
-					       slid, dlid);
+				hfi1_bad_pkey(ibp,
+					      pkey, sl,
+					      src_qp, qp->ibqp.qp_num,
+					      slid, dlid);
 				return;
 			}
 		} else {
@@ -739,12 +924,9 @@ void hfi1_ud_rcv(struct hfi1_packet *packet)
 			if (mgmt_pkey_idx < 0)
 				goto drop;
 		}
-		if (unlikely(qkey != qp->qkey)) {
-			hfi1_bad_pqkey(ibp, OPA_TRAP_BAD_Q_KEY, qkey, sl,
-				       src_qp, qp->ibqp.qp_num,
-				       slid, dlid);
+		if (unlikely(qkey != qp->qkey)) /* Silent drop */
 			return;
-		}
+
 		/* Drop invalid MAD packets (see 13.5.3.1). */
 		if (unlikely(qp->ibqp.qp_num == 1 &&
 			     (tlen > 2048 || (sc5 == 0xF))))
@@ -758,8 +940,7 @@ void hfi1_ud_rcv(struct hfi1_packet *packet)
 
 		if (tlen > 2048)
 			goto drop;
-		if ((hdr->lrh[1] == IB_LID_PERMISSIVE ||
-		     hdr->lrh[3] == IB_LID_PERMISSIVE) &&
+		if ((dlid_is_permissive || slid_is_permissive) &&
 		    smp->mgmt_class != IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)
 			goto drop;
 
@@ -811,8 +992,19 @@ void hfi1_ud_rcv(struct hfi1_packet *packet)
 		qp->r_flags |= RVT_R_REUSE_SGE;
 		goto drop;
 	}
-	if (has_grh) {
-		hfi1_copy_sge(&qp->r_sge, &hdr->u.l.grh,
+	if (packet->grh) {
+		hfi1_copy_sge(&qp->r_sge, packet->grh,
+			      sizeof(struct ib_grh), true, false);
+		wc.wc_flags |= IB_WC_GRH;
+	} else if (packet->etype == RHF_RCV_TYPE_BYPASS) {
+		struct ib_grh grh;
+		/*
+		 * Assuming we only created 16B on the send side
+		 * if we want to use large LIDs, since GRH was stripped
+		 * out when creating 16B, add back the GRH here.
+		 */
+		hfi1_make_ext_grh(packet, &grh, slid, dlid);
+		hfi1_copy_sge(&qp->r_sge, &grh,
 			      sizeof(struct ib_grh), true, false);
 		wc.wc_flags |= IB_WC_GRH;
 	} else {
@@ -845,14 +1037,15 @@ void hfi1_ud_rcv(struct hfi1_packet *packet)
 	} else {
 		wc.pkey_index = 0;
 	}
-
+	if (slid_is_permissive)
+		slid = be32_to_cpu(OPA_LID_PERMISSIVE);
 	wc.slid = slid;
 	wc.sl = sl_from_sc;
 
 	/*
 	 * Save the LMC lower bits if the destination LID is a unicast LID.
 	 */
-	wc.dlid_path_bits = dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE) ? 0 :
+	wc.dlid_path_bits = hfi1_check_mcast(dlid) ? 0 :
 		dlid & ((1 << ppd_from_ibp(ibp)->lmc) - 1);
 	wc.port_num = qp->port_num;
 	/* Signal completion event if the solicited bit is set. */
diff --git a/drivers/infiniband/hw/hfi1/user_exp_rcv.c b/drivers/infiniband/hw/hfi1/user_exp_rcv.c
index a8f0aa4722f6..6f6c14df383e 100644
--- a/drivers/infiniband/hw/hfi1/user_exp_rcv.c
+++ b/drivers/infiniband/hw/hfi1/user_exp_rcv.c
@@ -47,58 +47,28 @@
 #include <asm/page.h>
 #include <linux/string.h>
 
+#include "mmu_rb.h"
 #include "user_exp_rcv.h"
 #include "trace.h"
-#include "mmu_rb.h"
-
-struct tid_group {
-	struct list_head list;
-	u32 base;
-	u8 size;
-	u8 used;
-	u8 map;
-};
-
-struct tid_rb_node {
-	struct mmu_rb_node mmu;
-	unsigned long phys;
-	struct tid_group *grp;
-	u32 rcventry;
-	dma_addr_t dma_addr;
-	bool freed;
-	unsigned npages;
-	struct page *pages[0];
-};
-
-struct tid_pageset {
-	u16 idx;
-	u16 count;
-};
-
-#define EXP_TID_SET_EMPTY(set) (set.count == 0 && list_empty(&set.list))
-
-#define num_user_pages(vaddr, len)				       \
-	(1 + (((((unsigned long)(vaddr) +			       \
-		 (unsigned long)(len) - 1) & PAGE_MASK) -	       \
-	       ((unsigned long)vaddr & PAGE_MASK)) >> PAGE_SHIFT))
 
 static void unlock_exp_tids(struct hfi1_ctxtdata *uctxt,
 			    struct exp_tid_set *set,
 			    struct hfi1_filedata *fd);
-static u32 find_phys_blocks(struct page **pages, unsigned npages,
-			    struct tid_pageset *list);
-static int set_rcvarray_entry(struct hfi1_filedata *fd, unsigned long vaddr,
+static u32 find_phys_blocks(struct tid_user_buf *tidbuf, unsigned int npages);
+static int set_rcvarray_entry(struct hfi1_filedata *fd,
+			      struct tid_user_buf *tbuf,
 			      u32 rcventry, struct tid_group *grp,
-			      struct page **pages, unsigned npages);
+			      u16 pageidx, unsigned int npages);
 static int tid_rb_insert(void *arg, struct mmu_rb_node *node);
 static void cacheless_tid_rb_remove(struct hfi1_filedata *fdata,
 				    struct tid_rb_node *tnode);
 static void tid_rb_remove(void *arg, struct mmu_rb_node *node);
 static int tid_rb_invalidate(void *arg, struct mmu_rb_node *mnode);
-static int program_rcvarray(struct hfi1_filedata *fd, unsigned long vaddr,
-			    struct tid_group *grp, struct tid_pageset *sets,
-			    unsigned start, u16 count, struct page **pages,
-			    u32 *tidlist, unsigned *tididx, unsigned *pmapped);
+static int program_rcvarray(struct hfi1_filedata *fd, struct tid_user_buf *,
+			    struct tid_group *grp,
+			    unsigned int start, u16 count,
+			    u32 *tidlist, unsigned int *tididx,
+			    unsigned int *pmapped);
 static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo,
 			      struct tid_group **grp);
 static void clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node);
@@ -109,96 +79,14 @@ static struct mmu_rb_ops tid_rb_ops = {
 	.invalidate = tid_rb_invalidate
 };
 
-static inline u32 rcventry2tidinfo(u32 rcventry)
-{
-	u32 pair = rcventry & ~0x1;
-
-	return EXP_TID_SET(IDX, pair >> 1) |
-		EXP_TID_SET(CTRL, 1 << (rcventry - pair));
-}
-
-static inline void exp_tid_group_init(struct exp_tid_set *set)
-{
-	INIT_LIST_HEAD(&set->list);
-	set->count = 0;
-}
-
-static inline void tid_group_remove(struct tid_group *grp,
-				    struct exp_tid_set *set)
-{
-	list_del_init(&grp->list);
-	set->count--;
-}
-
-static inline void tid_group_add_tail(struct tid_group *grp,
-				      struct exp_tid_set *set)
-{
-	list_add_tail(&grp->list, &set->list);
-	set->count++;
-}
-
-static inline struct tid_group *tid_group_pop(struct exp_tid_set *set)
-{
-	struct tid_group *grp =
-		list_first_entry(&set->list, struct tid_group, list);
-	list_del_init(&grp->list);
-	set->count--;
-	return grp;
-}
-
-static inline void tid_group_move(struct tid_group *group,
-				  struct exp_tid_set *s1,
-				  struct exp_tid_set *s2)
-{
-	tid_group_remove(group, s1);
-	tid_group_add_tail(group, s2);
-}
-
-int hfi1_user_exp_rcv_grp_init(struct hfi1_filedata *fd)
-{
-	struct hfi1_ctxtdata *uctxt = fd->uctxt;
-	struct hfi1_devdata *dd = fd->dd;
-	u32 tidbase;
-	u32 i;
-	struct tid_group *grp, *gptr;
-
-	exp_tid_group_init(&uctxt->tid_group_list);
-	exp_tid_group_init(&uctxt->tid_used_list);
-	exp_tid_group_init(&uctxt->tid_full_list);
-
-	tidbase = uctxt->expected_base;
-	for (i = 0; i < uctxt->expected_count /
-		     dd->rcv_entries.group_size; i++) {
-		grp = kzalloc(sizeof(*grp), GFP_KERNEL);
-		if (!grp)
-			goto grp_failed;
-
-		grp->size = dd->rcv_entries.group_size;
-		grp->base = tidbase;
-		tid_group_add_tail(grp, &uctxt->tid_group_list);
-		tidbase += dd->rcv_entries.group_size;
-	}
-
-	return 0;
-
-grp_failed:
-	list_for_each_entry_safe(grp, gptr, &uctxt->tid_group_list.list,
-				 list) {
-		list_del_init(&grp->list);
-		kfree(grp);
-	}
-
-	return -ENOMEM;
-}
-
 /*
  * Initialize context and file private data needed for Expected
  * receive caching. This needs to be done after the context has
  * been configured with the eager/expected RcvEntry counts.
  */
-int hfi1_user_exp_rcv_init(struct hfi1_filedata *fd)
+int hfi1_user_exp_rcv_init(struct hfi1_filedata *fd,
+			   struct hfi1_ctxtdata *uctxt)
 {
-	struct hfi1_ctxtdata *uctxt = fd->uctxt;
 	struct hfi1_devdata *dd = uctxt->dd;
 	int ret = 0;
 
@@ -266,18 +154,6 @@ int hfi1_user_exp_rcv_init(struct hfi1_filedata *fd)
 	return ret;
 }
 
-void hfi1_user_exp_rcv_grp_free(struct hfi1_ctxtdata *uctxt)
-{
-	struct tid_group *grp, *gptr;
-
-	list_for_each_entry_safe(grp, gptr, &uctxt->tid_group_list.list,
-				 list) {
-		list_del_init(&grp->list);
-		kfree(grp);
-	}
-	hfi1_clear_tids(uctxt);
-}
-
 void hfi1_user_exp_rcv_free(struct hfi1_filedata *fd)
 {
 	struct hfi1_ctxtdata *uctxt = fd->uctxt;
@@ -302,21 +178,90 @@ void hfi1_user_exp_rcv_free(struct hfi1_filedata *fd)
 	fd->entry_to_rb = NULL;
 }
 
-/*
- * Write an "empty" RcvArray entry.
- * This function exists so the TID registaration code can use it
- * to write to unused/unneeded entries and still take advantage
- * of the WC performance improvements. The HFI will ignore this
- * write to the RcvArray entry.
+/**
+ * Release pinned receive buffer pages.
+ *
+ * @mapped - true if the pages have been DMA mapped. false otherwise.
+ * @idx - Index of the first page to unpin.
+ * @npages - No of pages to unpin.
+ *
+ * If the pages have been DMA mapped (indicated by mapped parameter), their
+ * info will be passed via a struct tid_rb_node. If they haven't been mapped,
+ * their info will be passed via a struct tid_user_buf.
+ */
+static void unpin_rcv_pages(struct hfi1_filedata *fd,
+			    struct tid_user_buf *tidbuf,
+			    struct tid_rb_node *node,
+			    unsigned int idx,
+			    unsigned int npages,
+			    bool mapped)
+{
+	struct page **pages;
+	struct hfi1_devdata *dd = fd->uctxt->dd;
+
+	if (mapped) {
+		pci_unmap_single(dd->pcidev, node->dma_addr,
+				 node->mmu.len, PCI_DMA_FROMDEVICE);
+		pages = &node->pages[idx];
+	} else {
+		pages = &tidbuf->pages[idx];
+	}
+	hfi1_release_user_pages(fd->mm, pages, npages, mapped);
+	fd->tid_n_pinned -= npages;
+}
+
+/**
+ * Pin receive buffer pages.
  */
-static inline void rcv_array_wc_fill(struct hfi1_devdata *dd, u32 index)
+static int pin_rcv_pages(struct hfi1_filedata *fd, struct tid_user_buf *tidbuf)
 {
+	int pinned;
+	unsigned int npages;
+	unsigned long vaddr = tidbuf->vaddr;
+	struct page **pages = NULL;
+	struct hfi1_devdata *dd = fd->uctxt->dd;
+
+	/* Get the number of pages the user buffer spans */
+	npages = num_user_pages(vaddr, tidbuf->length);
+	if (!npages)
+		return -EINVAL;
+
+	if (npages > fd->uctxt->expected_count) {
+		dd_dev_err(dd, "Expected buffer too big\n");
+		return -EINVAL;
+	}
+
+	/* Verify that access is OK for the user buffer */
+	if (!access_ok(VERIFY_WRITE, (void __user *)vaddr,
+		       npages * PAGE_SIZE)) {
+		dd_dev_err(dd, "Fail vaddr %p, %u pages, !access_ok\n",
+			   (void *)vaddr, npages);
+		return -EFAULT;
+	}
+	/* Allocate the array of struct page pointers needed for pinning */
+	pages = kcalloc(npages, sizeof(*pages), GFP_KERNEL);
+	if (!pages)
+		return -ENOMEM;
+
 	/*
-	 * Doing the WC fill writes only makes sense if the device is
-	 * present and the RcvArray has been mapped as WC memory.
+	 * Pin all the pages of the user buffer. If we can't pin all the
+	 * pages, accept the amount pinned so far and program only that.
+	 * User space knows how to deal with partially programmed buffers.
 	 */
-	if ((dd->flags & HFI1_PRESENT) && dd->rcvarray_wc)
-		writeq(0, dd->rcvarray_wc + (index * 8));
+	if (!hfi1_can_pin_pages(dd, fd->mm, fd->tid_n_pinned, npages)) {
+		kfree(pages);
+		return -ENOMEM;
+	}
+
+	pinned = hfi1_acquire_user_pages(fd->mm, vaddr, npages, true, pages);
+	if (pinned <= 0) {
+		kfree(pages);
+		return pinned;
+	}
+	tidbuf->pages = pages;
+	tidbuf->npages = npages;
+	fd->tid_n_pinned += pinned;
+	return pinned;
 }
 
 /*
@@ -374,62 +319,33 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd,
 	int ret = 0, need_group = 0, pinned;
 	struct hfi1_ctxtdata *uctxt = fd->uctxt;
 	struct hfi1_devdata *dd = uctxt->dd;
-	unsigned npages, ngroups, pageidx = 0, pageset_count, npagesets,
+	unsigned int ngroups, pageidx = 0, pageset_count,
 		tididx = 0, mapped, mapped_pages = 0;
-	unsigned long vaddr = tinfo->vaddr;
-	struct page **pages = NULL;
 	u32 *tidlist = NULL;
-	struct tid_pageset *pagesets = NULL;
+	struct tid_user_buf *tidbuf;
 
-	/* Get the number of pages the user buffer spans */
-	npages = num_user_pages(vaddr, tinfo->length);
-	if (!npages)
-		return -EINVAL;
-
-	if (npages > uctxt->expected_count) {
-		dd_dev_err(dd, "Expected buffer too big\n");
-		return -EINVAL;
-	}
-
-	/* Verify that access is OK for the user buffer */
-	if (!access_ok(VERIFY_WRITE, (void __user *)vaddr,
-		       npages * PAGE_SIZE)) {
-		dd_dev_err(dd, "Fail vaddr %p, %u pages, !access_ok\n",
-			   (void *)vaddr, npages);
-		return -EFAULT;
-	}
-
-	pagesets = kcalloc(uctxt->expected_count, sizeof(*pagesets),
-			   GFP_KERNEL);
-	if (!pagesets)
+	tidbuf = kzalloc(sizeof(*tidbuf), GFP_KERNEL);
+	if (!tidbuf)
 		return -ENOMEM;
 
-	/* Allocate the array of struct page pointers needed for pinning */
-	pages = kcalloc(npages, sizeof(*pages), GFP_KERNEL);
-	if (!pages) {
-		ret = -ENOMEM;
-		goto bail;
-	}
-
-	/*
-	 * Pin all the pages of the user buffer. If we can't pin all the
-	 * pages, accept the amount pinned so far and program only that.
-	 * User space knows how to deal with partially programmed buffers.
-	 */
-	if (!hfi1_can_pin_pages(dd, fd->mm, fd->tid_n_pinned, npages)) {
-		ret = -ENOMEM;
-		goto bail;
+	tidbuf->vaddr = tinfo->vaddr;
+	tidbuf->length = tinfo->length;
+	tidbuf->psets = kcalloc(uctxt->expected_count, sizeof(*tidbuf->psets),
+				GFP_KERNEL);
+	if (!tidbuf->psets) {
+		kfree(tidbuf);
+		return -ENOMEM;
 	}
 
-	pinned = hfi1_acquire_user_pages(fd->mm, vaddr, npages, true, pages);
+	pinned = pin_rcv_pages(fd, tidbuf);
 	if (pinned <= 0) {
-		ret = pinned;
-		goto bail;
+		kfree(tidbuf->psets);
+		kfree(tidbuf);
+		return pinned;
 	}
-	fd->tid_n_pinned += npages;
 
 	/* Find sets of physically contiguous pages */
-	npagesets = find_phys_blocks(pages, pinned, pagesets);
+	tidbuf->n_psets = find_phys_blocks(tidbuf, pinned);
 
 	/*
 	 * We don't need to access this under a lock since tid_used is per
@@ -437,10 +353,10 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd,
 	 * and hfi1_user_exp_rcv_setup() at the same time.
 	 */
 	spin_lock(&fd->tid_lock);
-	if (fd->tid_used + npagesets > fd->tid_limit)
+	if (fd->tid_used + tidbuf->n_psets > fd->tid_limit)
 		pageset_count = fd->tid_limit - fd->tid_used;
 	else
-		pageset_count = npagesets;
+		pageset_count = tidbuf->n_psets;
 	spin_unlock(&fd->tid_lock);
 
 	if (!pageset_count)
@@ -468,9 +384,9 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd,
 		struct tid_group *grp =
 			tid_group_pop(&uctxt->tid_group_list);
 
-		ret = program_rcvarray(fd, vaddr, grp, pagesets,
+		ret = program_rcvarray(fd, tidbuf, grp,
 				       pageidx, dd->rcv_entries.group_size,
-				       pages, tidlist, &tididx, &mapped);
+				       tidlist, &tididx, &mapped);
 		/*
 		 * If there was a failure to program the RcvArray
 		 * entries for the entire group, reset the grp fields
@@ -514,8 +430,8 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd,
 			unsigned use = min_t(unsigned, pageset_count - pageidx,
 					     grp->size - grp->used);
 
-			ret = program_rcvarray(fd, vaddr, grp, pagesets,
-					       pageidx, use, pages, tidlist,
+			ret = program_rcvarray(fd, tidbuf, grp,
+					       pageidx, use, tidlist,
 					       &tididx, &mapped);
 			if (ret < 0) {
 				hfi1_cdbg(TID,
@@ -575,16 +491,14 @@ nomem:
 	 * If not everything was mapped (due to insufficient RcvArray entries,
 	 * for example), unpin all unmapped pages so we can pin them nex time.
 	 */
-	if (mapped_pages != pinned) {
-		hfi1_release_user_pages(fd->mm, &pages[mapped_pages],
-					pinned - mapped_pages,
-					false);
-		fd->tid_n_pinned -= pinned - mapped_pages;
-	}
+	if (mapped_pages != pinned)
+		unpin_rcv_pages(fd, tidbuf, NULL, mapped_pages,
+				(pinned - mapped_pages), false);
 bail:
-	kfree(pagesets);
-	kfree(pages);
+	kfree(tidbuf->psets);
 	kfree(tidlist);
+	kfree(tidbuf->pages);
+	kfree(tidbuf);
 	return ret > 0 ? 0 : ret;
 }
 
@@ -674,11 +588,12 @@ int hfi1_user_exp_rcv_invalid(struct hfi1_filedata *fd,
 	return ret;
 }
 
-static u32 find_phys_blocks(struct page **pages, unsigned npages,
-			    struct tid_pageset *list)
+static u32 find_phys_blocks(struct tid_user_buf *tidbuf, unsigned int npages)
 {
 	unsigned pagecount, pageidx, setcount = 0, i;
 	unsigned long pfn, this_pfn;
+	struct page **pages = tidbuf->pages;
+	struct tid_pageset *list = tidbuf->psets;
 
 	if (!npages)
 		return 0;
@@ -741,13 +656,13 @@ static u32 find_phys_blocks(struct page **pages, unsigned npages,
 /**
  * program_rcvarray() - program an RcvArray group with receive buffers
  * @fd: filedata pointer
- * @vaddr: starting user virtual address
+ * @tbuf: pointer to struct tid_user_buf that has the user buffer starting
+ *	  virtual address, buffer length, page pointers, pagesets (array of
+ *	  struct tid_pageset holding information on physically contiguous
+ *	  chunks from the user buffer), and other fields.
  * @grp: RcvArray group
- * @sets: array of struct tid_pageset holding information on physically
- *        contiguous chunks from the user buffer
  * @start: starting index into sets array
  * @count: number of struct tid_pageset's to program
- * @pages: an array of struct page * for the user buffer
  * @tidlist: the array of u32 elements when the information about the
  *           programmed RcvArray entries is to be encoded.
  * @tididx: starting offset into tidlist
@@ -765,11 +680,11 @@ static u32 find_phys_blocks(struct page **pages, unsigned npages,
  * -ENOMEM or -EFAULT on error from set_rcvarray_entry(), or
  * number of RcvArray entries programmed.
  */
-static int program_rcvarray(struct hfi1_filedata *fd, unsigned long vaddr,
+static int program_rcvarray(struct hfi1_filedata *fd, struct tid_user_buf *tbuf,
 			    struct tid_group *grp,
-			    struct tid_pageset *sets,
-			    unsigned start, u16 count, struct page **pages,
-			    u32 *tidlist, unsigned *tididx, unsigned *pmapped)
+			    unsigned int start, u16 count,
+			    u32 *tidlist, unsigned int *tididx,
+			    unsigned int *pmapped)
 {
 	struct hfi1_ctxtdata *uctxt = fd->uctxt;
 	struct hfi1_devdata *dd = uctxt->dd;
@@ -808,11 +723,11 @@ static int program_rcvarray(struct hfi1_filedata *fd, unsigned long vaddr,
 		}
 
 		rcventry = grp->base + useidx;
-		npages = sets[setidx].count;
-		pageidx = sets[setidx].idx;
+		npages = tbuf->psets[setidx].count;
+		pageidx = tbuf->psets[setidx].idx;
 
-		ret = set_rcvarray_entry(fd, vaddr + (pageidx * PAGE_SIZE),
-					 rcventry, grp, pages + pageidx,
+		ret = set_rcvarray_entry(fd, tbuf,
+					 rcventry, grp, pageidx,
 					 npages);
 		if (ret)
 			return ret;
@@ -833,15 +748,17 @@ static int program_rcvarray(struct hfi1_filedata *fd, unsigned long vaddr,
 	return idx;
 }
 
-static int set_rcvarray_entry(struct hfi1_filedata *fd, unsigned long vaddr,
+static int set_rcvarray_entry(struct hfi1_filedata *fd,
+			      struct tid_user_buf *tbuf,
 			      u32 rcventry, struct tid_group *grp,
-			      struct page **pages, unsigned npages)
+			      u16 pageidx, unsigned int npages)
 {
 	int ret;
 	struct hfi1_ctxtdata *uctxt = fd->uctxt;
 	struct tid_rb_node *node;
 	struct hfi1_devdata *dd = uctxt->dd;
 	dma_addr_t phys;
+	struct page **pages = tbuf->pages + pageidx;
 
 	/*
 	 * Allocate the node first so we can handle a potential
@@ -862,7 +779,7 @@ static int set_rcvarray_entry(struct hfi1_filedata *fd, unsigned long vaddr,
 		return -EFAULT;
 	}
 
-	node->mmu.addr = vaddr;
+	node->mmu.addr = tbuf->vaddr + (pageidx * PAGE_SIZE);
 	node->mmu.len = npages * PAGE_SIZE;
 	node->phys = page_to_phys(pages[0]);
 	node->npages = npages;
@@ -935,17 +852,13 @@ static void clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node)
 				 node->npages, node->mmu.addr, node->phys,
 				 node->dma_addr);
 
-	hfi1_put_tid(dd, node->rcventry, PT_INVALID, 0, 0);
 	/*
 	 * Make sure device has seen the write before we unpin the
 	 * pages.
 	 */
-	flush_wc();
+	hfi1_put_tid(dd, node->rcventry, PT_INVALID_FLUSH, 0, 0);
 
-	pci_unmap_single(dd->pcidev, node->dma_addr, node->mmu.len,
-			 PCI_DMA_FROMDEVICE);
-	hfi1_release_user_pages(fd->mm, node->pages, node->npages, true);
-	fd->tid_n_pinned -= node->npages;
+	unpin_rcv_pages(fd, NULL, node, 0, node->npages, true);
 
 	node->grp->used--;
 	node->grp->map &= ~(1 << (node->rcventry - node->grp->base));
diff --git a/drivers/infiniband/hw/hfi1/user_exp_rcv.h b/drivers/infiniband/hw/hfi1/user_exp_rcv.h
index 5250c897298d..e383cc01a2bf 100644
--- a/drivers/infiniband/hw/hfi1/user_exp_rcv.h
+++ b/drivers/infiniband/hw/hfi1/user_exp_rcv.h
@@ -49,30 +49,44 @@
 
 #include "hfi.h"
 
-#define EXP_TID_TIDLEN_MASK   0x7FFULL
-#define EXP_TID_TIDLEN_SHIFT  0
-#define EXP_TID_TIDCTRL_MASK  0x3ULL
-#define EXP_TID_TIDCTRL_SHIFT 20
-#define EXP_TID_TIDIDX_MASK   0x3FFULL
-#define EXP_TID_TIDIDX_SHIFT  22
-#define EXP_TID_GET(tid, field)	\
-	(((tid) >> EXP_TID_TID##field##_SHIFT) & EXP_TID_TID##field##_MASK)
+#include "exp_rcv.h"
 
-#define EXP_TID_SET(field, value)			\
-	(((value) & EXP_TID_TID##field##_MASK) <<	\
-	 EXP_TID_TID##field##_SHIFT)
-#define EXP_TID_CLEAR(tid, field) ({					\
-		(tid) &= ~(EXP_TID_TID##field##_MASK <<			\
-			   EXP_TID_TID##field##_SHIFT);			\
-		})
-#define EXP_TID_RESET(tid, field, value) do {				\
-		EXP_TID_CLEAR(tid, field);				\
-		(tid) |= EXP_TID_SET(field, (value));			\
-	} while (0)
+struct tid_pageset {
+	u16 idx;
+	u16 count;
+};
 
-void hfi1_user_exp_rcv_grp_free(struct hfi1_ctxtdata *uctxt);
-int hfi1_user_exp_rcv_grp_init(struct hfi1_filedata *fd);
-int hfi1_user_exp_rcv_init(struct hfi1_filedata *fd);
+struct tid_user_buf {
+	unsigned long vaddr;
+	unsigned long length;
+	unsigned int npages;
+	struct page **pages;
+	struct tid_pageset *psets;
+	unsigned int n_psets;
+};
+
+struct tid_rb_node {
+	struct mmu_rb_node mmu;
+	unsigned long phys;
+	struct tid_group *grp;
+	u32 rcventry;
+	dma_addr_t dma_addr;
+	bool freed;
+	unsigned int npages;
+	struct page *pages[0];
+};
+
+static inline int num_user_pages(unsigned long addr,
+				 unsigned long len)
+{
+	const unsigned long spage = addr & PAGE_MASK;
+	const unsigned long epage = (addr + len - 1) & PAGE_MASK;
+
+	return 1 + ((epage - spage) >> PAGE_SHIFT);
+}
+
+int hfi1_user_exp_rcv_init(struct hfi1_filedata *fd,
+			   struct hfi1_ctxtdata *uctxt);
 void hfi1_user_exp_rcv_free(struct hfi1_filedata *fd);
 int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd,
 			    struct hfi1_tid_info *tinfo);
diff --git a/drivers/infiniband/hw/hfi1/user_sdma.c b/drivers/infiniband/hw/hfi1/user_sdma.c
index d55339f5d73b..c0c0e0445cbf 100644
--- a/drivers/infiniband/hw/hfi1/user_sdma.c
+++ b/drivers/infiniband/hw/hfi1/user_sdma.c
@@ -64,224 +64,20 @@
 
 #include "hfi.h"
 #include "sdma.h"
+#include "mmu_rb.h"
 #include "user_sdma.h"
 #include "verbs.h"  /* for the headers */
 #include "common.h" /* for struct hfi1_tid_info */
 #include "trace.h"
-#include "mmu_rb.h"
 
 static uint hfi1_sdma_comp_ring_size = 128;
 module_param_named(sdma_comp_size, hfi1_sdma_comp_ring_size, uint, S_IRUGO);
 MODULE_PARM_DESC(sdma_comp_size, "Size of User SDMA completion ring. Default: 128");
 
-/* The maximum number of Data io vectors per message/request */
-#define MAX_VECTORS_PER_REQ 8
-/*
- * Maximum number of packet to send from each message/request
- * before moving to the next one.
- */
-#define MAX_PKTS_PER_QUEUE 16
-
-#define num_pages(x) (1 + ((((x) - 1) & PAGE_MASK) >> PAGE_SHIFT))
-
-#define req_opcode(x) \
-	(((x) >> HFI1_SDMA_REQ_OPCODE_SHIFT) & HFI1_SDMA_REQ_OPCODE_MASK)
-#define req_version(x) \
-	(((x) >> HFI1_SDMA_REQ_VERSION_SHIFT) & HFI1_SDMA_REQ_OPCODE_MASK)
-#define req_iovcnt(x) \
-	(((x) >> HFI1_SDMA_REQ_IOVCNT_SHIFT) & HFI1_SDMA_REQ_IOVCNT_MASK)
-
-/* Number of BTH.PSN bits used for sequence number in expected rcvs */
-#define BTH_SEQ_MASK 0x7ffull
-
-/*
- * Define fields in the KDETH header so we can update the header
- * template.
- */
-#define KDETH_OFFSET_SHIFT        0
-#define KDETH_OFFSET_MASK         0x7fff
-#define KDETH_OM_SHIFT            15
-#define KDETH_OM_MASK             0x1
-#define KDETH_TID_SHIFT           16
-#define KDETH_TID_MASK            0x3ff
-#define KDETH_TIDCTRL_SHIFT       26
-#define KDETH_TIDCTRL_MASK        0x3
-#define KDETH_INTR_SHIFT          28
-#define KDETH_INTR_MASK           0x1
-#define KDETH_SH_SHIFT            29
-#define KDETH_SH_MASK             0x1
-#define KDETH_HCRC_UPPER_SHIFT    16
-#define KDETH_HCRC_UPPER_MASK     0xff
-#define KDETH_HCRC_LOWER_SHIFT    24
-#define KDETH_HCRC_LOWER_MASK     0xff
-
-#define AHG_KDETH_INTR_SHIFT 12
-#define AHG_KDETH_SH_SHIFT   13
-
-#define PBC2LRH(x) ((((x) & 0xfff) << 2) - 4)
-#define LRH2PBC(x) ((((x) >> 2) + 1) & 0xfff)
-
-#define KDETH_GET(val, field)						\
-	(((le32_to_cpu((val))) >> KDETH_##field##_SHIFT) & KDETH_##field##_MASK)
-#define KDETH_SET(dw, field, val) do {					\
-		u32 dwval = le32_to_cpu(dw);				\
-		dwval &= ~(KDETH_##field##_MASK << KDETH_##field##_SHIFT); \
-		dwval |= (((val) & KDETH_##field##_MASK) << \
-			  KDETH_##field##_SHIFT);			\
-		dw = cpu_to_le32(dwval);				\
-	} while (0)
-
-#define AHG_HEADER_SET(arr, idx, dw, bit, width, value)			\
-	do {								\
-		if ((idx) < ARRAY_SIZE((arr)))				\
-			(arr)[(idx++)] = sdma_build_ahg_descriptor(	\
-				(__force u16)(value), (dw), (bit),	\
-							(width));	\
-		else							\
-			return -ERANGE;					\
-	} while (0)
-
-/* KDETH OM multipliers and switch over point */
-#define KDETH_OM_SMALL     4
-#define KDETH_OM_SMALL_SHIFT     2
-#define KDETH_OM_LARGE     64
-#define KDETH_OM_LARGE_SHIFT     6
-#define KDETH_OM_MAX_SIZE  (1 << ((KDETH_OM_LARGE / KDETH_OM_SMALL) + 1))
-
-/* Tx request flag bits */
-#define TXREQ_FLAGS_REQ_ACK   BIT(0)      /* Set the ACK bit in the header */
-#define TXREQ_FLAGS_REQ_DISABLE_SH BIT(1) /* Disable header suppression */
-
-/* SDMA request flag bits */
-#define SDMA_REQ_FOR_THREAD 1
-#define SDMA_REQ_SEND_DONE  2
-#define SDMA_REQ_HAS_ERROR  3
-#define SDMA_REQ_DONE_ERROR 4
-
-#define SDMA_PKT_Q_INACTIVE BIT(0)
-#define SDMA_PKT_Q_ACTIVE   BIT(1)
-#define SDMA_PKT_Q_DEFERRED BIT(2)
-
-/*
- * Maximum retry attempts to submit a TX request
- * before putting the process to sleep.
- */
-#define MAX_DEFER_RETRY_COUNT 1
-
 static unsigned initial_pkt_count = 8;
 
-#define SDMA_IOWAIT_TIMEOUT 1000 /* in milliseconds */
-
-struct sdma_mmu_node;
-
-struct user_sdma_iovec {
-	struct list_head list;
-	struct iovec iov;
-	/* number of pages in this vector */
-	unsigned npages;
-	/* array of pinned pages for this vector */
-	struct page **pages;
-	/*
-	 * offset into the virtual address space of the vector at
-	 * which we last left off.
-	 */
-	u64 offset;
-	struct sdma_mmu_node *node;
-};
-
-struct sdma_mmu_node {
-	struct mmu_rb_node rb;
-	struct hfi1_user_sdma_pkt_q *pq;
-	atomic_t refcount;
-	struct page **pages;
-	unsigned npages;
-};
-
-/* evict operation argument */
-struct evict_data {
-	u32 cleared;	/* count evicted so far */
-	u32 target;	/* target count to evict */
-};
-
-struct user_sdma_request {
-	struct sdma_req_info info;
-	struct hfi1_user_sdma_pkt_q *pq;
-	struct hfi1_user_sdma_comp_q *cq;
-	/* This is the original header from user space */
-	struct hfi1_pkt_header hdr;
-	/*
-	 * Pointer to the SDMA engine for this request.
-	 * Since different request could be on different VLs,
-	 * each request will need it's own engine pointer.
-	 */
-	struct sdma_engine *sde;
-	s8 ahg_idx;
-	u32 ahg[9];
-	/*
-	 * KDETH.Offset (Eager) field
-	 * We need to remember the initial value so the headers
-	 * can be updated properly.
-	 */
-	u32 koffset;
-	/*
-	 * KDETH.OFFSET (TID) field
-	 * The offset can cover multiple packets, depending on the
-	 * size of the TID entry.
-	 */
-	u32 tidoffset;
-	/*
-	 * We copy the iovs for this request (based on
-	 * info.iovcnt). These are only the data vectors
-	 */
-	unsigned data_iovs;
-	/* total length of the data in the request */
-	u32 data_len;
-	/* progress index moving along the iovs array */
-	unsigned iov_idx;
-	struct user_sdma_iovec iovs[MAX_VECTORS_PER_REQ];
-	/* number of elements copied to the tids array */
-	u16 n_tids;
-	/* TID array values copied from the tid_iov vector */
-	u32 *tids;
-	u16 tididx;
-	u32 sent;
-	u64 seqnum;
-	u64 seqcomp;
-	u64 seqsubmitted;
-	struct list_head txps;
-	unsigned long flags;
-	/* status of the last txreq completed */
-	int status;
-};
-
-/*
- * A single txreq could span up to 3 physical pages when the MTU
- * is sufficiently large (> 4K). Each of the IOV pointers also
- * needs it's own set of flags so the vector has been handled
- * independently of each other.
- */
-struct user_sdma_txreq {
-	/* Packet header for the txreq */
-	struct hfi1_pkt_header hdr;
-	struct sdma_txreq txreq;
-	struct list_head list;
-	struct user_sdma_request *req;
-	u16 flags;
-	unsigned busycount;
-	u64 seqnum;
-};
-
-#define SDMA_DBG(req, fmt, ...)				     \
-	hfi1_cdbg(SDMA, "[%u:%u:%u:%u] " fmt, (req)->pq->dd->unit, \
-		 (req)->pq->ctxt, (req)->pq->subctxt, (req)->info.comp_idx, \
-		 ##__VA_ARGS__)
-#define SDMA_Q_DBG(pq, fmt, ...)			 \
-	hfi1_cdbg(SDMA, "[%u:%u:%u] " fmt, (pq)->dd->unit, (pq)->ctxt, \
-		 (pq)->subctxt, ##__VA_ARGS__)
-
 static int user_sdma_send_pkts(struct user_sdma_request *req,
 			       unsigned maxpkts);
-static int num_user_pages(const struct iovec *iov);
 static void user_sdma_txreq_cb(struct sdma_txreq *txreq, int status);
 static inline void pq_update(struct hfi1_user_sdma_pkt_q *pq);
 static void user_sdma_free_request(struct user_sdma_request *req, bool unpin);
@@ -307,7 +103,8 @@ static int defer_packet_queue(
 	struct sdma_engine *sde,
 	struct iowait *wait,
 	struct sdma_txreq *txreq,
-	unsigned int seq);
+	uint seq,
+	bool pkts_sent);
 static void activate_packet_queue(struct iowait *wait, int reason);
 static bool sdma_rb_filter(struct mmu_rb_node *node, unsigned long addr,
 			   unsigned long len);
@@ -329,7 +126,8 @@ static int defer_packet_queue(
 	struct sdma_engine *sde,
 	struct iowait *wait,
 	struct sdma_txreq *txreq,
-	unsigned seq)
+	uint seq,
+	bool pkts_sent)
 {
 	struct hfi1_user_sdma_pkt_q *pq =
 		container_of(wait, struct hfi1_user_sdma_pkt_q, busy);
@@ -349,7 +147,7 @@ static int defer_packet_queue(
 	xchg(&pq->state, SDMA_PKT_Q_DEFERRED);
 	write_seqlock(&dev->iowait_lock);
 	if (list_empty(&pq->busy.list))
-		list_add_tail(&pq->busy.list, &sde->dmawait);
+		iowait_queue(pkts_sent, &pq->busy, &sde->dmawait);
 	write_sequnlock(&dev->iowait_lock);
 	return -EBUSY;
 eagain:
@@ -364,13 +162,6 @@ static void activate_packet_queue(struct iowait *wait, int reason)
 	wake_up(&wait->wait_dma);
 };
 
-static void sdma_kmem_cache_ctor(void *obj)
-{
-	struct user_sdma_txreq *tx = obj;
-
-	memset(tx, 0, sizeof(*tx));
-}
-
 int hfi1_user_sdma_alloc_queues(struct hfi1_ctxtdata *uctxt,
 				struct hfi1_filedata *fd)
 {
@@ -379,7 +170,6 @@ int hfi1_user_sdma_alloc_queues(struct hfi1_ctxtdata *uctxt,
 	struct hfi1_devdata *dd;
 	struct hfi1_user_sdma_comp_q *cq;
 	struct hfi1_user_sdma_pkt_q *pq;
-	unsigned long flags;
 
 	if (!uctxt || !fd)
 		return -EBADF;
@@ -393,7 +183,6 @@ int hfi1_user_sdma_alloc_queues(struct hfi1_ctxtdata *uctxt,
 	if (!pq)
 		return -ENOMEM;
 
-	INIT_LIST_HEAD(&pq->list);
 	pq->dd = dd;
 	pq->ctxt = uctxt->ctxt;
 	pq->subctxt = fd->subctxt;
@@ -426,7 +215,7 @@ int hfi1_user_sdma_alloc_queues(struct hfi1_ctxtdata *uctxt,
 					    sizeof(struct user_sdma_txreq),
 					    L1_CACHE_BYTES,
 					    SLAB_HWCACHE_ALIGN,
-					    sdma_kmem_cache_ctor);
+					    NULL);
 	if (!pq->txreq_cache) {
 		dd_dev_err(dd, "[%u] Failed to allocate TxReq cache\n",
 			   uctxt->ctxt);
@@ -454,10 +243,6 @@ int hfi1_user_sdma_alloc_queues(struct hfi1_ctxtdata *uctxt,
 	fd->pq = pq;
 	fd->cq = cq;
 
-	spin_lock_irqsave(&uctxt->sdma_qlock, flags);
-	list_add(&pq->list, &uctxt->sdma_queues);
-	spin_unlock_irqrestore(&uctxt->sdma_qlock, flags);
-
 	return 0;
 
 pq_mmu_fail:
@@ -476,22 +261,17 @@ pq_reqs_nomem:
 	return ret;
 }
 
-int hfi1_user_sdma_free_queues(struct hfi1_filedata *fd)
+int hfi1_user_sdma_free_queues(struct hfi1_filedata *fd,
+			       struct hfi1_ctxtdata *uctxt)
 {
-	struct hfi1_ctxtdata *uctxt = fd->uctxt;
 	struct hfi1_user_sdma_pkt_q *pq;
-	unsigned long flags;
 
-	hfi1_cdbg(SDMA, "[%u:%u:%u] Freeing user SDMA queues", uctxt->dd->unit,
-		  uctxt->ctxt, fd->subctxt);
+	trace_hfi1_sdma_user_free_queues(uctxt->dd, uctxt->ctxt, fd->subctxt);
+
 	pq = fd->pq;
 	if (pq) {
 		if (pq->handler)
 			hfi1_mmu_rb_unregister(pq->handler);
-		spin_lock_irqsave(&uctxt->sdma_qlock, flags);
-		if (!list_empty(&pq->list))
-			list_del_init(&pq->list);
-		spin_unlock_irqrestore(&uctxt->sdma_qlock, flags);
 		iowait_sdma_drain(&pq->busy);
 		/* Wait until all requests have been freed. */
 		wait_event_interruptible(
@@ -546,6 +326,8 @@ int hfi1_user_sdma_process_request(struct hfi1_filedata *fd,
 	struct sdma_req_info info;
 	struct user_sdma_request *req;
 	u8 opcode, sc, vl;
+	u16 pkey;
+	u32 slid;
 	int req_queued = 0;
 	u16 dlid;
 	u32 selector;
@@ -567,7 +349,6 @@ int hfi1_user_sdma_process_request(struct hfi1_filedata *fd,
 
 	trace_hfi1_sdma_user_reqinfo(dd, uctxt->ctxt, fd->subctxt,
 				     (u16 *)&info);
-
 	if (info.comp_idx >= hfi1_sdma_comp_ring_size) {
 		hfi1_cdbg(SDMA,
 			  "[%u:%u:%u:%u] Invalid comp index",
@@ -604,15 +385,23 @@ int hfi1_user_sdma_process_request(struct hfi1_filedata *fd,
 	/*
 	 * All safety checks have been done and this request has been claimed.
 	 */
-	hfi1_cdbg(SDMA, "[%u:%u:%u] Using req/comp entry %u\n", dd->unit,
-		  uctxt->ctxt, fd->subctxt, info.comp_idx);
+	trace_hfi1_sdma_user_process_request(dd, uctxt->ctxt, fd->subctxt,
+					     info.comp_idx);
 	req = pq->reqs + info.comp_idx;
-	memset(req, 0, sizeof(*req));
 	req->data_iovs = req_iovcnt(info.ctrl) - 1; /* subtract header vector */
+	req->data_len  = 0;
 	req->pq = pq;
 	req->cq = cq;
 	req->status = -1;
 	req->ahg_idx = -1;
+	req->iov_idx = 0;
+	req->sent = 0;
+	req->seqnum = 0;
+	req->seqcomp = 0;
+	req->seqsubmitted = 0;
+	req->tids = NULL;
+	req->done = 0;
+	req->has_error = 0;
 	INIT_LIST_HEAD(&req->txps);
 
 	memcpy(&req->info, &info, sizeof(info));
@@ -671,8 +460,9 @@ int hfi1_user_sdma_process_request(struct hfi1_filedata *fd,
 	}
 
 	/* Checking P_KEY for requests from user-space */
-	if (egress_pkey_check(dd->pport, req->hdr.lrh, req->hdr.bth, sc,
-			      PKEY_CHECK_INVALID)) {
+	pkey = (u16)be32_to_cpu(req->hdr.bth[0]);
+	slid = be16_to_cpu(req->hdr.lrh[3]);
+	if (egress_pkey_check(dd->pport, slid, pkey, sc, PKEY_CHECK_INVALID)) {
 		ret = -EINVAL;
 		goto free_req;
 	}
@@ -696,24 +486,27 @@ int hfi1_user_sdma_process_request(struct hfi1_filedata *fd,
 	req->tidoffset = KDETH_GET(req->hdr.kdeth.ver_tid_offset, OFFSET) *
 		(KDETH_GET(req->hdr.kdeth.ver_tid_offset, OM) ?
 		 KDETH_OM_LARGE : KDETH_OM_SMALL);
-	SDMA_DBG(req, "Initial TID offset %u", req->tidoffset);
+	trace_hfi1_sdma_user_initial_tidoffset(dd, uctxt->ctxt, fd->subctxt,
+					       info.comp_idx, req->tidoffset);
 	idx++;
 
 	/* Save all the IO vector structures */
 	for (i = 0; i < req->data_iovs; i++) {
+		req->iovs[i].offset = 0;
 		INIT_LIST_HEAD(&req->iovs[i].list);
 		memcpy(&req->iovs[i].iov,
 		       iovec + idx++,
 		       sizeof(req->iovs[i].iov));
 		ret = pin_vector_pages(req, &req->iovs[i]);
 		if (ret) {
+			req->data_iovs = i;
 			req->status = ret;
 			goto free_req;
 		}
 		req->data_len += req->iovs[i].iov.iov_len;
 	}
-	SDMA_DBG(req, "total data length %u", req->data_len);
-
+	trace_hfi1_sdma_user_data_length(dd, uctxt->ctxt, fd->subctxt,
+					 info.comp_idx, req->data_len);
 	if (pcount > req->info.npkts)
 		pcount = req->info.npkts;
 	/*
@@ -749,6 +542,7 @@ int hfi1_user_sdma_process_request(struct hfi1_filedata *fd,
 		}
 		req->tids = tmp;
 		req->n_tids = ntids;
+		req->tididx = 0;
 		idx++;
 	}
 
@@ -791,12 +585,12 @@ int hfi1_user_sdma_process_request(struct hfi1_filedata *fd,
 	 * request have been submitted to the SDMA engine. However, it
 	 * will not wait for send completions.
 	 */
-	while (!test_bit(SDMA_REQ_SEND_DONE, &req->flags)) {
+	while (req->seqsubmitted != req->info.npkts) {
 		ret = user_sdma_send_pkts(req, pcount);
 		if (ret < 0) {
 			if (ret != -EBUSY) {
 				req->status = ret;
-				set_bit(SDMA_REQ_DONE_ERROR, &req->flags);
+				WRITE_ONCE(req->has_error, 1);
 				if (ACCESS_ONCE(req->seqcomp) ==
 				    req->seqsubmitted - 1)
 					goto free_req;
@@ -867,7 +661,11 @@ static inline u32 compute_data_length(struct user_sdma_request *req,
 	} else {
 		len = min(req->data_len - req->sent, (u32)req->info.fragsize);
 	}
-	SDMA_DBG(req, "Data Length = %u", len);
+	trace_hfi1_sdma_user_compute_length(req->pq->dd,
+					    req->pq->ctxt,
+					    req->pq->subctxt,
+					    req->info.comp_idx,
+					    len);
 	return len;
 }
 
@@ -884,6 +682,84 @@ static inline u32 get_lrh_len(struct hfi1_pkt_header hdr, u32 len)
 	return ((sizeof(hdr) - sizeof(hdr.pbc)) + 4 + len);
 }
 
+static int user_sdma_txadd_ahg(struct user_sdma_request *req,
+			       struct user_sdma_txreq *tx,
+			       u32 datalen)
+{
+	int ret;
+	u16 pbclen = le16_to_cpu(req->hdr.pbc[0]);
+	u32 lrhlen = get_lrh_len(req->hdr, pad_len(datalen));
+	struct hfi1_user_sdma_pkt_q *pq = req->pq;
+
+	/*
+	 * Copy the request header into the tx header
+	 * because the HW needs a cacheline-aligned
+	 * address.
+	 * This copy can be optimized out if the hdr
+	 * member of user_sdma_request were also
+	 * cacheline aligned.
+	 */
+	memcpy(&tx->hdr, &req->hdr, sizeof(tx->hdr));
+	if (PBC2LRH(pbclen) != lrhlen) {
+		pbclen = (pbclen & 0xf000) | LRH2PBC(lrhlen);
+		tx->hdr.pbc[0] = cpu_to_le16(pbclen);
+	}
+	ret = check_header_template(req, &tx->hdr, lrhlen, datalen);
+	if (ret)
+		return ret;
+	ret = sdma_txinit_ahg(&tx->txreq, SDMA_TXREQ_F_AHG_COPY,
+			      sizeof(tx->hdr) + datalen, req->ahg_idx,
+			      0, NULL, 0, user_sdma_txreq_cb);
+	if (ret)
+		return ret;
+	ret = sdma_txadd_kvaddr(pq->dd, &tx->txreq, &tx->hdr, sizeof(tx->hdr));
+	if (ret)
+		sdma_txclean(pq->dd, &tx->txreq);
+	return ret;
+}
+
+static int user_sdma_txadd(struct user_sdma_request *req,
+			   struct user_sdma_txreq *tx,
+			   struct user_sdma_iovec *iovec, u32 datalen,
+			   u32 *queued_ptr, u32 *data_sent_ptr,
+			   u64 *iov_offset_ptr)
+{
+	int ret;
+	unsigned int pageidx, len;
+	unsigned long base, offset;
+	u64 iov_offset = *iov_offset_ptr;
+	u32 queued = *queued_ptr, data_sent = *data_sent_ptr;
+	struct hfi1_user_sdma_pkt_q *pq = req->pq;
+
+	base = (unsigned long)iovec->iov.iov_base;
+	offset = offset_in_page(base + iovec->offset + iov_offset);
+	pageidx = (((iovec->offset + iov_offset + base) - (base & PAGE_MASK)) >>
+		   PAGE_SHIFT);
+	len = offset + req->info.fragsize > PAGE_SIZE ?
+		PAGE_SIZE - offset : req->info.fragsize;
+	len = min((datalen - queued), len);
+	ret = sdma_txadd_page(pq->dd, &tx->txreq, iovec->pages[pageidx],
+			      offset, len);
+	if (ret) {
+		SDMA_DBG(req, "SDMA txreq add page failed %d\n", ret);
+		return ret;
+	}
+	iov_offset += len;
+	queued += len;
+	data_sent += len;
+	if (unlikely(queued < datalen && pageidx == iovec->npages &&
+		     req->iov_idx < req->data_iovs - 1)) {
+		iovec->offset += iov_offset;
+		iovec = &req->iovs[++req->iov_idx];
+		iov_offset = 0;
+	}
+
+	*queued_ptr = queued;
+	*data_sent_ptr = data_sent;
+	*iov_offset_ptr = iov_offset;
+	return ret;
+}
+
 static int user_sdma_send_pkts(struct user_sdma_request *req, unsigned maxpkts)
 {
 	int ret = 0, count;
@@ -898,10 +774,8 @@ static int user_sdma_send_pkts(struct user_sdma_request *req, unsigned maxpkts)
 	pq = req->pq;
 
 	/* If tx completion has reported an error, we are done. */
-	if (test_bit(SDMA_REQ_HAS_ERROR, &req->flags)) {
-		set_bit(SDMA_REQ_DONE_ERROR, &req->flags);
+	if (READ_ONCE(req->has_error))
 		return -EFAULT;
-	}
 
 	/*
 	 * Check if we might have sent the entire request already
@@ -924,10 +798,8 @@ static int user_sdma_send_pkts(struct user_sdma_request *req, unsigned maxpkts)
 		 * with errors. If so, we are not going to process any
 		 * more packets from this request.
 		 */
-		if (test_bit(SDMA_REQ_HAS_ERROR, &req->flags)) {
-			set_bit(SDMA_REQ_DONE_ERROR, &req->flags);
+		if (READ_ONCE(req->has_error))
 			return -EFAULT;
-		}
 
 		tx = kmem_cache_alloc(pq->txreq_cache, GFP_KERNEL);
 		if (!tx)
@@ -984,39 +856,9 @@ static int user_sdma_send_pkts(struct user_sdma_request *req, unsigned maxpkts)
 
 		if (req->ahg_idx >= 0) {
 			if (!req->seqnum) {
-				u16 pbclen = le16_to_cpu(req->hdr.pbc[0]);
-				u32 lrhlen = get_lrh_len(req->hdr,
-							 pad_len(datalen));
-				/*
-				 * Copy the request header into the tx header
-				 * because the HW needs a cacheline-aligned
-				 * address.
-				 * This copy can be optimized out if the hdr
-				 * member of user_sdma_request were also
-				 * cacheline aligned.
-				 */
-				memcpy(&tx->hdr, &req->hdr, sizeof(tx->hdr));
-				if (PBC2LRH(pbclen) != lrhlen) {
-					pbclen = (pbclen & 0xf000) |
-						LRH2PBC(lrhlen);
-					tx->hdr.pbc[0] = cpu_to_le16(pbclen);
-				}
-				ret = check_header_template(req, &tx->hdr,
-							    lrhlen, datalen);
+				ret = user_sdma_txadd_ahg(req, tx, datalen);
 				if (ret)
 					goto free_tx;
-				ret = sdma_txinit_ahg(&tx->txreq,
-						      SDMA_TXREQ_F_AHG_COPY,
-						      sizeof(tx->hdr) + datalen,
-						      req->ahg_idx, 0, NULL, 0,
-						      user_sdma_txreq_cb);
-				if (ret)
-					goto free_tx;
-				ret = sdma_txadd_kvaddr(pq->dd, &tx->txreq,
-							&tx->hdr,
-							sizeof(tx->hdr));
-				if (ret)
-					goto free_txreq;
 			} else {
 				int changes;
 
@@ -1024,11 +866,6 @@ static int user_sdma_send_pkts(struct user_sdma_request *req, unsigned maxpkts)
 							       datalen);
 				if (changes < 0)
 					goto free_tx;
-				sdma_txinit_ahg(&tx->txreq,
-						SDMA_TXREQ_F_USE_AHG,
-						datalen, req->ahg_idx, changes,
-						req->ahg, sizeof(req->hdr),
-						user_sdma_txreq_cb);
 			}
 		} else {
 			ret = sdma_txinit(&tx->txreq, 0, sizeof(req->hdr) +
@@ -1052,35 +889,10 @@ static int user_sdma_send_pkts(struct user_sdma_request *req, unsigned maxpkts)
 		 */
 		while (queued < datalen &&
 		       (req->sent + data_sent) < req->data_len) {
-			unsigned long base, offset;
-			unsigned pageidx, len;
-
-			base = (unsigned long)iovec->iov.iov_base;
-			offset = offset_in_page(base + iovec->offset +
-						iov_offset);
-			pageidx = (((iovec->offset + iov_offset +
-				     base) - (base & PAGE_MASK)) >> PAGE_SHIFT);
-			len = offset + req->info.fragsize > PAGE_SIZE ?
-				PAGE_SIZE - offset : req->info.fragsize;
-			len = min((datalen - queued), len);
-			ret = sdma_txadd_page(pq->dd, &tx->txreq,
-					      iovec->pages[pageidx],
-					      offset, len);
-			if (ret) {
-				SDMA_DBG(req, "SDMA txreq add page failed %d\n",
-					 ret);
+			ret = user_sdma_txadd(req, tx, iovec, datalen,
+					      &queued, &data_sent, &iov_offset);
+			if (ret)
 				goto free_txreq;
-			}
-			iov_offset += len;
-			queued += len;
-			data_sent += len;
-			if (unlikely(queued < datalen &&
-				     pageidx == iovec->npages &&
-				     req->iov_idx < req->data_iovs - 1)) {
-				iovec->offset += iov_offset;
-				iovec = &req->iovs[++req->iov_idx];
-				iov_offset = 0;
-			}
 		}
 		/*
 		 * The txreq was submitted successfully so we can update
@@ -1105,7 +917,7 @@ dosend:
 	ret = sdma_send_txlist(req->sde, &pq->busy, &req->txps, &count);
 	req->seqsubmitted += count;
 	if (req->seqsubmitted == req->info.npkts) {
-		set_bit(SDMA_REQ_SEND_DONE, &req->flags);
+		WRITE_ONCE(req->done, 1);
 		/*
 		 * The txreq has already been submitted to the HW queue
 		 * so we can free the AHG entry now. Corruption will not
@@ -1124,19 +936,6 @@ free_tx:
 	return ret;
 }
 
-/*
- * How many pages in this iovec element?
- */
-static inline int num_user_pages(const struct iovec *iov)
-{
-	const unsigned long addr  = (unsigned long)iov->iov_base;
-	const unsigned long len   = iov->iov_len;
-	const unsigned long spage = addr & PAGE_MASK;
-	const unsigned long epage = (addr + len - 1) & PAGE_MASK;
-
-	return 1 + ((epage - spage) >> PAGE_SHIFT);
-}
-
 static u32 sdma_cache_evict(struct hfi1_user_sdma_pkt_q *pq, u32 npages)
 {
 	struct evict_data evict_data;
@@ -1147,22 +946,82 @@ static u32 sdma_cache_evict(struct hfi1_user_sdma_pkt_q *pq, u32 npages)
 	return evict_data.cleared;
 }
 
+static int pin_sdma_pages(struct user_sdma_request *req,
+			  struct user_sdma_iovec *iovec,
+			  struct sdma_mmu_node *node,
+			  int npages)
+{
+	int pinned, cleared;
+	struct page **pages;
+	struct hfi1_user_sdma_pkt_q *pq = req->pq;
+
+	pages = kcalloc(npages, sizeof(*pages), GFP_KERNEL);
+	if (!pages) {
+		SDMA_DBG(req, "Failed page array alloc");
+		return -ENOMEM;
+	}
+	memcpy(pages, node->pages, node->npages * sizeof(*pages));
+
+	npages -= node->npages;
+retry:
+	if (!hfi1_can_pin_pages(pq->dd, pq->mm,
+				atomic_read(&pq->n_locked), npages)) {
+		cleared = sdma_cache_evict(pq, npages);
+		if (cleared >= npages)
+			goto retry;
+	}
+	pinned = hfi1_acquire_user_pages(pq->mm,
+					 ((unsigned long)iovec->iov.iov_base +
+					 (node->npages * PAGE_SIZE)), npages, 0,
+					 pages + node->npages);
+	if (pinned < 0) {
+		kfree(pages);
+		return pinned;
+	}
+	if (pinned != npages) {
+		unpin_vector_pages(pq->mm, pages, node->npages, pinned);
+		return -EFAULT;
+	}
+	kfree(node->pages);
+	node->rb.len = iovec->iov.iov_len;
+	node->pages = pages;
+	atomic_add(pinned, &pq->n_locked);
+	return pinned;
+}
+
+static void unpin_sdma_pages(struct sdma_mmu_node *node)
+{
+	if (node->npages) {
+		unpin_vector_pages(node->pq->mm, node->pages, 0, node->npages);
+		atomic_sub(node->npages, &node->pq->n_locked);
+	}
+}
+
 static int pin_vector_pages(struct user_sdma_request *req,
 			    struct user_sdma_iovec *iovec)
 {
-	int ret = 0, pinned, npages, cleared;
-	struct page **pages;
+	int ret = 0, pinned, npages;
 	struct hfi1_user_sdma_pkt_q *pq = req->pq;
 	struct sdma_mmu_node *node = NULL;
 	struct mmu_rb_node *rb_node;
-
-	rb_node = hfi1_mmu_rb_extract(pq->handler,
-				      (unsigned long)iovec->iov.iov_base,
-				      iovec->iov.iov_len);
-	if (rb_node)
+	struct iovec *iov;
+	bool extracted;
+
+	extracted =
+		hfi1_mmu_rb_remove_unless_exact(pq->handler,
+						(unsigned long)
+						iovec->iov.iov_base,
+						iovec->iov.iov_len, &rb_node);
+	if (rb_node) {
 		node = container_of(rb_node, struct sdma_mmu_node, rb);
-	else
-		rb_node = NULL;
+		if (!extracted) {
+			atomic_inc(&node->refcount);
+			iovec->pages = node->pages;
+			iovec->npages = node->npages;
+			iovec->node = node;
+			return 0;
+		}
+	}
 
 	if (!node) {
 		node = kzalloc(sizeof(*node), GFP_KERNEL);
@@ -1174,46 +1033,16 @@ static int pin_vector_pages(struct user_sdma_request *req,
 		atomic_set(&node->refcount, 0);
 	}
 
-	npages = num_user_pages(&iovec->iov);
+	iov = &iovec->iov;
+	npages = num_user_pages((unsigned long)iov->iov_base, iov->iov_len);
 	if (node->npages < npages) {
-		pages = kcalloc(npages, sizeof(*pages), GFP_KERNEL);
-		if (!pages) {
-			SDMA_DBG(req, "Failed page array alloc");
-			ret = -ENOMEM;
-			goto bail;
-		}
-		memcpy(pages, node->pages, node->npages * sizeof(*pages));
-
-		npages -= node->npages;
-
-retry:
-		if (!hfi1_can_pin_pages(pq->dd, pq->mm,
-					atomic_read(&pq->n_locked), npages)) {
-			cleared = sdma_cache_evict(pq, npages);
-			if (cleared >= npages)
-				goto retry;
-		}
-		pinned = hfi1_acquire_user_pages(pq->mm,
-			((unsigned long)iovec->iov.iov_base +
-			 (node->npages * PAGE_SIZE)), npages, 0,
-			pages + node->npages);
+		pinned = pin_sdma_pages(req, iovec, node, npages);
 		if (pinned < 0) {
-			kfree(pages);
 			ret = pinned;
 			goto bail;
 		}
-		if (pinned != npages) {
-			unpin_vector_pages(pq->mm, pages, node->npages,
-					   pinned);
-			ret = -EFAULT;
-			goto bail;
-		}
-		kfree(node->pages);
-		node->rb.len = iovec->iov.iov_len;
-		node->pages = pages;
 		node->npages += pinned;
 		npages = node->npages;
-		atomic_add(pinned, &pq->n_locked);
 	}
 	iovec->pages = node->pages;
 	iovec->npages = npages;
@@ -1221,14 +1050,12 @@ retry:
 
 	ret = hfi1_mmu_rb_insert(req->pq->handler, &node->rb);
 	if (ret) {
-		atomic_sub(node->npages, &pq->n_locked);
 		iovec->node = NULL;
 		goto bail;
 	}
 	return 0;
 bail:
-	if (rb_node)
-		unpin_vector_pages(pq->mm, node->pages, 0, node->npages);
+	unpin_sdma_pages(node);
 	kfree(node);
 	return ret;
 }
@@ -1408,9 +1235,10 @@ static int set_txreq_header(struct user_sdma_request *req,
 		 * Set the KDETH.OFFSET and KDETH.OM based on size of
 		 * transfer.
 		 */
-		SDMA_DBG(req, "TID offset %ubytes %uunits om%u",
-			 req->tidoffset, req->tidoffset >> omfactor,
-			 omfactor != KDETH_OM_SMALL_SHIFT);
+		trace_hfi1_sdma_user_tid_info(
+			pq->dd, pq->ctxt, pq->subctxt, req->info.comp_idx,
+			req->tidoffset, req->tidoffset >> omfactor,
+			omfactor != KDETH_OM_SMALL_SHIFT);
 		KDETH_SET(hdr->kdeth.ver_tid_offset, OFFSET,
 			  req->tidoffset >> omfactor);
 		KDETH_SET(hdr->kdeth.ver_tid_offset, OM,
@@ -1423,21 +1251,22 @@ done:
 }
 
 static int set_txreq_header_ahg(struct user_sdma_request *req,
-				struct user_sdma_txreq *tx, u32 len)
+				struct user_sdma_txreq *tx, u32 datalen)
 {
+	u32 ahg[AHG_KDETH_ARRAY_SIZE];
 	int diff = 0;
 	u8 omfactor; /* KDETH.OM */
 	struct hfi1_user_sdma_pkt_q *pq = req->pq;
 	struct hfi1_pkt_header *hdr = &req->hdr;
 	u16 pbclen = le16_to_cpu(hdr->pbc[0]);
-	u32 val32, tidval = 0, lrhlen = get_lrh_len(*hdr, pad_len(len));
+	u32 val32, tidval = 0, lrhlen = get_lrh_len(*hdr, pad_len(datalen));
 
 	if (PBC2LRH(pbclen) != lrhlen) {
 		/* PBC.PbcLengthDWs */
-		AHG_HEADER_SET(req->ahg, diff, 0, 0, 12,
+		AHG_HEADER_SET(ahg, diff, 0, 0, 12,
 			       cpu_to_le16(LRH2PBC(lrhlen)));
 		/* LRH.PktLen (we need the full 16 bits due to byte swap) */
-		AHG_HEADER_SET(req->ahg, diff, 3, 0, 16,
+		AHG_HEADER_SET(ahg, diff, 3, 0, 16,
 			       cpu_to_be16(lrhlen >> 2));
 	}
 
@@ -1449,13 +1278,12 @@ static int set_txreq_header_ahg(struct user_sdma_request *req,
 		(HFI1_CAP_IS_KSET(EXTENDED_PSN) ? 0x7fffffff : 0xffffff);
 	if (unlikely(tx->flags & TXREQ_FLAGS_REQ_ACK))
 		val32 |= 1UL << 31;
-	AHG_HEADER_SET(req->ahg, diff, 6, 0, 16, cpu_to_be16(val32 >> 16));
-	AHG_HEADER_SET(req->ahg, diff, 6, 16, 16, cpu_to_be16(val32 & 0xffff));
+	AHG_HEADER_SET(ahg, diff, 6, 0, 16, cpu_to_be16(val32 >> 16));
+	AHG_HEADER_SET(ahg, diff, 6, 16, 16, cpu_to_be16(val32 & 0xffff));
 	/* KDETH.Offset */
-	AHG_HEADER_SET(req->ahg, diff, 15, 0, 16,
+	AHG_HEADER_SET(ahg, diff, 15, 0, 16,
 		       cpu_to_le16(req->koffset & 0xffff));
-	AHG_HEADER_SET(req->ahg, diff, 15, 16, 16,
-		       cpu_to_le16(req->koffset >> 16));
+	AHG_HEADER_SET(ahg, diff, 15, 16, 16, cpu_to_le16(req->koffset >> 16));
 	if (req_opcode(req->info.ctrl) == EXPECTED) {
 		__le16 val;
 
@@ -1473,9 +1301,8 @@ static int set_txreq_header_ahg(struct user_sdma_request *req,
 			 * we have to check again.
 			 */
 			if (++req->tididx > req->n_tids - 1 ||
-			    !req->tids[req->tididx]) {
+			    !req->tids[req->tididx])
 				return -EINVAL;
-			}
 			tidval = req->tids[req->tididx];
 		}
 		omfactor = ((EXP_TID_GET(tidval, LEN) *
@@ -1483,7 +1310,7 @@ static int set_txreq_header_ahg(struct user_sdma_request *req,
 				 KDETH_OM_MAX_SIZE) ? KDETH_OM_LARGE_SHIFT :
 				 KDETH_OM_SMALL_SHIFT;
 		/* KDETH.OM and KDETH.OFFSET (TID) */
-		AHG_HEADER_SET(req->ahg, diff, 7, 0, 16,
+		AHG_HEADER_SET(ahg, diff, 7, 0, 16,
 			       ((!!(omfactor - KDETH_OM_SMALL_SHIFT)) << 15 |
 				((req->tidoffset >> omfactor)
 				 & 0x7fff)));
@@ -1503,12 +1330,20 @@ static int set_txreq_header_ahg(struct user_sdma_request *req,
 					     AHG_KDETH_INTR_SHIFT));
 		}
 
-		AHG_HEADER_SET(req->ahg, diff, 7, 16, 14, val);
+		AHG_HEADER_SET(ahg, diff, 7, 16, 14, val);
 	}
+	if (diff < 0)
+		return diff;
 
 	trace_hfi1_sdma_user_header_ahg(pq->dd, pq->ctxt, pq->subctxt,
 					req->info.comp_idx, req->sde->this_idx,
-					req->ahg_idx, req->ahg, diff, tidval);
+					req->ahg_idx, ahg, diff, tidval);
+	sdma_txinit_ahg(&tx->txreq,
+			SDMA_TXREQ_F_USE_AHG,
+			datalen, req->ahg_idx, diff,
+			ahg, sizeof(req->hdr),
+			user_sdma_txreq_cb);
+
 	return diff;
 }
 
@@ -1537,7 +1372,7 @@ static void user_sdma_txreq_cb(struct sdma_txreq *txreq, int status)
 	if (status != SDMA_TXREQ_S_OK) {
 		SDMA_DBG(req, "SDMA completion with error %d",
 			 status);
-		set_bit(SDMA_REQ_HAS_ERROR, &req->flags);
+		WRITE_ONCE(req->has_error, 1);
 	}
 
 	req->seqcomp = tx->seqnum;
@@ -1556,8 +1391,8 @@ static void user_sdma_txreq_cb(struct sdma_txreq *txreq, int status)
 		if (status != SDMA_TXREQ_S_OK)
 			req->status = status;
 		if (req->seqcomp == (ACCESS_ONCE(req->seqsubmitted) - 1) &&
-		    (test_bit(SDMA_REQ_SEND_DONE, &req->flags) ||
-		     test_bit(SDMA_REQ_DONE_ERROR, &req->flags))) {
+		    (READ_ONCE(req->done) ||
+		     READ_ONCE(req->has_error))) {
 			user_sdma_free_request(req, false);
 			pq_update(pq);
 			set_comp_state(pq, cq, idx, ERROR, req->status);
@@ -1611,8 +1446,6 @@ static inline void set_comp_state(struct hfi1_user_sdma_pkt_q *pq,
 				  u16 idx, enum hfi1_sdma_comp_state state,
 				  int ret)
 {
-	hfi1_cdbg(SDMA, "[%u:%u:%u:%u] Setting completion status %u %d",
-		  pq->dd->unit, pq->ctxt, pq->subctxt, idx, state, ret);
 	if (state == ERROR)
 		cq->comps[idx].errcode = -ret;
 	smp_wmb(); /* make sure errcode is visible first */
@@ -1667,10 +1500,7 @@ static void sdma_rb_remove(void *arg, struct mmu_rb_node *mnode)
 	struct sdma_mmu_node *node =
 		container_of(mnode, struct sdma_mmu_node, rb);
 
-	atomic_sub(node->npages, &node->pq->n_locked);
-
-	unpin_vector_pages(node->pq->mm, node->pages, 0, node->npages);
-
+	unpin_sdma_pages(node);
 	kfree(node);
 }
 
diff --git a/drivers/infiniband/hw/hfi1/user_sdma.h b/drivers/infiniband/hw/hfi1/user_sdma.h
index e5b10aefe212..9b8bb5634c0d 100644
--- a/drivers/infiniband/hw/hfi1/user_sdma.h
+++ b/drivers/infiniband/hw/hfi1/user_sdma.h
@@ -53,11 +53,68 @@
 #include "iowait.h"
 #include "user_exp_rcv.h"
 
+/* The maximum number of Data io vectors per message/request */
+#define MAX_VECTORS_PER_REQ 8
+/*
+ * Maximum number of packet to send from each message/request
+ * before moving to the next one.
+ */
+#define MAX_PKTS_PER_QUEUE 16
+
+#define num_pages(x) (1 + ((((x) - 1) & PAGE_MASK) >> PAGE_SHIFT))
+
+#define req_opcode(x) \
+	(((x) >> HFI1_SDMA_REQ_OPCODE_SHIFT) & HFI1_SDMA_REQ_OPCODE_MASK)
+#define req_version(x) \
+	(((x) >> HFI1_SDMA_REQ_VERSION_SHIFT) & HFI1_SDMA_REQ_OPCODE_MASK)
+#define req_iovcnt(x) \
+	(((x) >> HFI1_SDMA_REQ_IOVCNT_SHIFT) & HFI1_SDMA_REQ_IOVCNT_MASK)
+
+/* Number of BTH.PSN bits used for sequence number in expected rcvs */
+#define BTH_SEQ_MASK 0x7ffull
+
+#define AHG_KDETH_INTR_SHIFT 12
+#define AHG_KDETH_SH_SHIFT   13
+#define AHG_KDETH_ARRAY_SIZE  9
+
+#define PBC2LRH(x) ((((x) & 0xfff) << 2) - 4)
+#define LRH2PBC(x) ((((x) >> 2) + 1) & 0xfff)
+
+#define AHG_HEADER_SET(arr, idx, dw, bit, width, value)			\
+	do {								\
+		if ((idx) < ARRAY_SIZE((arr)))				\
+			(arr)[(idx++)] = sdma_build_ahg_descriptor(	\
+				(__force u16)(value), (dw), (bit),	\
+							(width));	\
+		else							\
+			return -ERANGE;					\
+	} while (0)
+
+/* Tx request flag bits */
+#define TXREQ_FLAGS_REQ_ACK   BIT(0)      /* Set the ACK bit in the header */
+#define TXREQ_FLAGS_REQ_DISABLE_SH BIT(1) /* Disable header suppression */
+
+#define SDMA_PKT_Q_INACTIVE BIT(0)
+#define SDMA_PKT_Q_ACTIVE   BIT(1)
+#define SDMA_PKT_Q_DEFERRED BIT(2)
+
+/*
+ * Maximum retry attempts to submit a TX request
+ * before putting the process to sleep.
+ */
+#define MAX_DEFER_RETRY_COUNT 1
+
+#define SDMA_IOWAIT_TIMEOUT 1000 /* in milliseconds */
+
+#define SDMA_DBG(req, fmt, ...)				     \
+	hfi1_cdbg(SDMA, "[%u:%u:%u:%u] " fmt, (req)->pq->dd->unit, \
+		 (req)->pq->ctxt, (req)->pq->subctxt, (req)->info.comp_idx, \
+		 ##__VA_ARGS__)
+
 extern uint extended_psn;
 
 struct hfi1_user_sdma_pkt_q {
-	struct list_head list;
-	unsigned ctxt;
+	u16 ctxt;
 	u16 subctxt;
 	u16 n_max_reqs;
 	atomic_t n_reqs;
@@ -80,9 +137,115 @@ struct hfi1_user_sdma_comp_q {
 	struct hfi1_sdma_comp_entry *comps;
 };
 
+struct sdma_mmu_node {
+	struct mmu_rb_node rb;
+	struct hfi1_user_sdma_pkt_q *pq;
+	atomic_t refcount;
+	struct page **pages;
+	unsigned int npages;
+};
+
+struct user_sdma_iovec {
+	struct list_head list;
+	struct iovec iov;
+	/* number of pages in this vector */
+	unsigned int npages;
+	/* array of pinned pages for this vector */
+	struct page **pages;
+	/*
+	 * offset into the virtual address space of the vector at
+	 * which we last left off.
+	 */
+	u64 offset;
+	struct sdma_mmu_node *node;
+};
+
+/* evict operation argument */
+struct evict_data {
+	u32 cleared;	/* count evicted so far */
+	u32 target;	/* target count to evict */
+};
+
+struct user_sdma_request {
+	/* This is the original header from user space */
+	struct hfi1_pkt_header hdr;
+
+	/* Read mostly fields */
+	struct hfi1_user_sdma_pkt_q *pq ____cacheline_aligned_in_smp;
+	struct hfi1_user_sdma_comp_q *cq;
+	/*
+	 * Pointer to the SDMA engine for this request.
+	 * Since different request could be on different VLs,
+	 * each request will need it's own engine pointer.
+	 */
+	struct sdma_engine *sde;
+	struct sdma_req_info info;
+	/* TID array values copied from the tid_iov vector */
+	u32 *tids;
+	/* total length of the data in the request */
+	u32 data_len;
+	/* number of elements copied to the tids array */
+	u16 n_tids;
+	/*
+	 * We copy the iovs for this request (based on
+	 * info.iovcnt). These are only the data vectors
+	 */
+	u8 data_iovs;
+	s8 ahg_idx;
+
+	/* Writeable fields shared with interrupt */
+	u64 seqcomp ____cacheline_aligned_in_smp;
+	u64 seqsubmitted;
+	/* status of the last txreq completed */
+	int status;
+
+	/* Send side fields */
+	struct list_head txps ____cacheline_aligned_in_smp;
+	u64 seqnum;
+	/*
+	 * KDETH.OFFSET (TID) field
+	 * The offset can cover multiple packets, depending on the
+	 * size of the TID entry.
+	 */
+	u32 tidoffset;
+	/*
+	 * KDETH.Offset (Eager) field
+	 * We need to remember the initial value so the headers
+	 * can be updated properly.
+	 */
+	u32 koffset;
+	u32 sent;
+	/* TID index copied from the tid_iov vector */
+	u16 tididx;
+	/* progress index moving along the iovs array */
+	u8 iov_idx;
+	u8 done;
+	u8 has_error;
+
+	struct user_sdma_iovec iovs[MAX_VECTORS_PER_REQ];
+} ____cacheline_aligned_in_smp;
+
+/*
+ * A single txreq could span up to 3 physical pages when the MTU
+ * is sufficiently large (> 4K). Each of the IOV pointers also
+ * needs it's own set of flags so the vector has been handled
+ * independently of each other.
+ */
+struct user_sdma_txreq {
+	/* Packet header for the txreq */
+	struct hfi1_pkt_header hdr;
+	struct sdma_txreq txreq;
+	struct list_head list;
+	struct user_sdma_request *req;
+	u16 flags;
+	unsigned int busycount;
+	u64 seqnum;
+};
+
 int hfi1_user_sdma_alloc_queues(struct hfi1_ctxtdata *uctxt,
 				struct hfi1_filedata *fd);
-int hfi1_user_sdma_free_queues(struct hfi1_filedata *fd);
+int hfi1_user_sdma_free_queues(struct hfi1_filedata *fd,
+			       struct hfi1_ctxtdata *uctxt);
 int hfi1_user_sdma_process_request(struct hfi1_filedata *fd,
 				   struct iovec *iovec, unsigned long dim,
 				   unsigned long *count);
diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c
index 2d19f9bb434d..e232f3c608b4 100644
--- a/drivers/infiniband/hw/hfi1/verbs.c
+++ b/drivers/infiniband/hw/hfi1/verbs.c
@@ -53,6 +53,7 @@
 #include <linux/rculist.h>
 #include <linux/mm.h>
 #include <linux/vmalloc.h>
+#include <rdma/opa_addr.h>
 
 #include "hfi.h"
 #include "common.h"
@@ -508,13 +509,14 @@ again:
 /*
  * Make sure the QP is ready and able to accept the given opcode.
  */
-static inline opcode_handler qp_ok(int opcode, struct hfi1_packet *packet)
+static inline opcode_handler qp_ok(struct hfi1_packet *packet)
 {
 	if (!(ib_rvt_state_ops[packet->qp->state] & RVT_PROCESS_RECV_OK))
 		return NULL;
-	if (((opcode & RVT_OPCODE_QP_MASK) == packet->qp->allowed_ops) ||
-	    (opcode == IB_OPCODE_CNP))
-		return opcode_handler_tbl[opcode];
+	if (((packet->opcode & RVT_OPCODE_QP_MASK) ==
+	     packet->qp->allowed_ops) ||
+	    (packet->opcode == IB_OPCODE_CNP))
+		return opcode_handler_tbl[packet->opcode];
 
 	return NULL;
 }
@@ -548,69 +550,54 @@ static u64 hfi1_fault_tx(struct rvt_qp *qp, u8 opcode, u64 pbc)
 	return pbc;
 }
 
-/**
- * hfi1_ib_rcv - process an incoming packet
- * @packet: data packet information
- *
- * This is called to process an incoming packet at interrupt level.
- *
- * Tlen is the length of the header + data + CRC in bytes.
- */
-void hfi1_ib_rcv(struct hfi1_packet *packet)
+static int hfi1_do_pkey_check(struct hfi1_packet *packet)
 {
 	struct hfi1_ctxtdata *rcd = packet->rcd;
-	struct ib_header *hdr = packet->hdr;
-	u32 tlen = packet->tlen;
+	struct hfi1_pportdata *ppd = rcd->ppd;
+	struct hfi1_16b_header *hdr = packet->hdr;
+	u16 pkey;
+
+	/* Pkey check needed only for bypass packets */
+	if (packet->etype != RHF_RCV_TYPE_BYPASS)
+		return 0;
+
+	/* Perform pkey check */
+	pkey = hfi1_16B_get_pkey(hdr);
+	return ingress_pkey_check(ppd, pkey, packet->sc,
+				  packet->qp->s_pkey_index,
+				  packet->slid, true);
+}
+
+static inline void hfi1_handle_packet(struct hfi1_packet *packet,
+				      bool is_mcast)
+{
+	u32 qp_num;
+	struct hfi1_ctxtdata *rcd = packet->rcd;
 	struct hfi1_pportdata *ppd = rcd->ppd;
 	struct hfi1_ibport *ibp = rcd_to_iport(rcd);
 	struct rvt_dev_info *rdi = &ppd->dd->verbs_dev.rdi;
 	opcode_handler packet_handler;
 	unsigned long flags;
-	u32 qp_num;
-	int lnh;
-	u8 opcode;
-	u16 lid;
-
-	/* Check for GRH */
-	lnh = ib_get_lnh(hdr);
-	if (lnh == HFI1_LRH_BTH) {
-		packet->ohdr = &hdr->u.oth;
-	} else if (lnh == HFI1_LRH_GRH) {
-		u32 vtf;
-
-		packet->ohdr = &hdr->u.l.oth;
-		if (hdr->u.l.grh.next_hdr != IB_GRH_NEXT_HDR)
-			goto drop;
-		vtf = be32_to_cpu(hdr->u.l.grh.version_tclass_flow);
-		if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION)
-			goto drop;
-		packet->rcv_flags |= HFI1_HAS_GRH;
-	} else {
-		goto drop;
-	}
-
-	trace_input_ibhdr(rcd->dd, hdr);
 
-	opcode = ib_bth_get_opcode(packet->ohdr);
-	inc_opstats(tlen, &rcd->opstats->stats[opcode]);
+	inc_opstats(packet->tlen, &rcd->opstats->stats[packet->opcode]);
 
-	/* Get the destination QP number. */
-	qp_num = be32_to_cpu(packet->ohdr->bth[1]) & RVT_QPN_MASK;
-	lid = ib_get_dlid(hdr);
-	if (unlikely((lid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
-		     (lid != be16_to_cpu(IB_LID_PERMISSIVE)))) {
+	if (unlikely(is_mcast)) {
 		struct rvt_mcast *mcast;
 		struct rvt_mcast_qp *p;
 
-		if (lnh != HFI1_LRH_GRH)
+		if (!packet->grh)
 			goto drop;
-		mcast = rvt_mcast_find(&ibp->rvp, &hdr->u.l.grh.dgid, lid);
+		mcast = rvt_mcast_find(&ibp->rvp,
+				       &packet->grh->dgid,
+				       opa_get_lid(packet->dlid, 9B));
 		if (!mcast)
 			goto drop;
 		list_for_each_entry_rcu(p, &mcast->qp_list, list) {
 			packet->qp = p->qp;
+			if (hfi1_do_pkey_check(packet))
+				goto drop;
 			spin_lock_irqsave(&packet->qp->r_lock, flags);
-			packet_handler = qp_ok(opcode, packet);
+			packet_handler = qp_ok(packet);
 			if (likely(packet_handler))
 				packet_handler(packet);
 			else
@@ -624,19 +611,22 @@ void hfi1_ib_rcv(struct hfi1_packet *packet)
 		if (atomic_dec_return(&mcast->refcount) <= 1)
 			wake_up(&mcast->wait);
 	} else {
+		/* Get the destination QP number. */
+		qp_num = ib_bth_get_qpn(packet->ohdr);
 		rcu_read_lock();
 		packet->qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num);
-		if (!packet->qp) {
-			rcu_read_unlock();
-			goto drop;
-		}
-		if (unlikely(hfi1_dbg_fault_opcode(packet->qp, opcode,
-						   true))) {
-			rcu_read_unlock();
-			goto drop;
-		}
+		if (!packet->qp)
+			goto unlock_drop;
+
+		if (hfi1_do_pkey_check(packet))
+			goto unlock_drop;
+
+		if (unlikely(hfi1_dbg_fault_opcode(packet->qp, packet->opcode,
+						   true)))
+			goto unlock_drop;
+
 		spin_lock_irqsave(&packet->qp->r_lock, flags);
-		packet_handler = qp_ok(opcode, packet);
+		packet_handler = qp_ok(packet);
 		if (likely(packet_handler))
 			packet_handler(packet);
 		else
@@ -645,11 +635,34 @@ void hfi1_ib_rcv(struct hfi1_packet *packet)
 		rcu_read_unlock();
 	}
 	return;
-
+unlock_drop:
+	rcu_read_unlock();
 drop:
 	ibp->rvp.n_pkt_drops++;
 }
 
+/**
+ * hfi1_ib_rcv - process an incoming packet
+ * @packet: data packet information
+ *
+ * This is called to process an incoming packet at interrupt level.
+ */
+void hfi1_ib_rcv(struct hfi1_packet *packet)
+{
+	struct hfi1_ctxtdata *rcd = packet->rcd;
+
+	trace_input_ibhdr(rcd->dd, packet, !!(rhf_dc_info(packet->rhf)));
+	hfi1_handle_packet(packet, hfi1_check_mcast(packet->dlid));
+}
+
+void hfi1_16B_rcv(struct hfi1_packet *packet)
+{
+	struct hfi1_ctxtdata *rcd = packet->rcd;
+
+	trace_input_ibhdr(rcd->dd, packet, false);
+	hfi1_handle_packet(packet, hfi1_check_mcast(packet->dlid));
+}
+
 /*
  * This is called from a timer to check for QPs
  * which need kernel memory in order to send a packet.
@@ -696,7 +709,7 @@ static void verbs_sdma_complete(
 	if (tx->wqe) {
 		hfi1_send_complete(qp, tx->wqe, IB_WC_SUCCESS);
 	} else if (qp->ibqp.qp_type == IB_QPT_RC) {
-		struct ib_header *hdr;
+		struct hfi1_opa_header *hdr;
 
 		hdr = &tx->phdr.hdr;
 		hfi1_rc_send_complete(qp, hdr);
@@ -799,12 +812,27 @@ static int build_verbs_tx_desc(
 	int ret = 0;
 	struct hfi1_sdma_header *phdr = &tx->phdr;
 	u16 hdrbytes = tx->hdr_dwords << 2;
+	u32 *hdr;
+	u8 extra_bytes = 0;
+	static char trail_buf[12]; /* CRC = 4, LT = 1, Pad = 0 to 7 bytes */
 
+	if (tx->phdr.hdr.hdr_type) {
+		/*
+		 * hdrbytes accounts for PBC. Need to subtract 8 bytes
+		 * before calculating padding.
+		 */
+		extra_bytes = hfi1_get_16b_padding(hdrbytes - 8, length) +
+			      (SIZE_OF_CRC << 2) + SIZE_OF_LT;
+		hdr = (u32 *)&phdr->hdr.opah;
+	} else {
+		hdr = (u32 *)&phdr->hdr.ibh;
+	}
 	if (!ahg_info->ahgcount) {
 		ret = sdma_txinit_ahg(
 			&tx->txreq,
 			ahg_info->tx_flags,
-			hdrbytes + length,
+			hdrbytes + length +
+			extra_bytes,
 			ahg_info->ahgidx,
 			0,
 			NULL,
@@ -834,8 +862,17 @@ static int build_verbs_tx_desc(
 			goto bail_txadd;
 	}
 	/* add the ulp payload - if any. tx->ss can be NULL for acks */
-	if (tx->ss)
+	if (tx->ss) {
 		ret = build_verbs_ulp_payload(sde, length, tx);
+		if (ret)
+			goto bail_txadd;
+	}
+
+	/* add icrc, lt byte, and padding to flit */
+	if (extra_bytes != 0)
+		ret = sdma_txadd_kvaddr(sde->dd, &tx->txreq,
+					trail_buf, extra_bytes);
+
 bail_txadd:
 	return ret;
 }
@@ -847,26 +884,42 @@ int hfi1_verbs_send_dma(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
 	struct hfi1_ahg_info *ahg_info = priv->s_ahg;
 	u32 hdrwords = qp->s_hdrwords;
 	u32 len = ps->s_txreq->s_cur_size;
-	u32 plen = hdrwords + ((len + 3) >> 2) + 2; /* includes pbc */
+	u32 plen;
 	struct hfi1_ibdev *dev = ps->dev;
 	struct hfi1_pportdata *ppd = ps->ppd;
 	struct verbs_txreq *tx;
 	u8 sc5 = priv->s_sc;
-
 	int ret;
+	u32 dwords;
+	bool bypass = false;
+
+	if (ps->s_txreq->phdr.hdr.hdr_type) {
+		u8 extra_bytes = hfi1_get_16b_padding((hdrwords << 2), len);
+
+		dwords = (len + extra_bytes + (SIZE_OF_CRC << 2) +
+			  SIZE_OF_LT) >> 2;
+		bypass = true;
+	} else {
+		dwords = (len + 3) >> 2;
+	}
+	plen = hdrwords + dwords + 2;
 
 	tx = ps->s_txreq;
 	if (!sdma_txreq_built(&tx->txreq)) {
 		if (likely(pbc == 0)) {
 			u32 vl = sc_to_vlt(dd_from_ibdev(qp->ibqp.device), sc5);
-			u8 opcode = get_opcode(&tx->phdr.hdr);
 
 			/* No vl15 here */
-			/* set PBC_DC_INFO bit (aka SC[4]) in pbc_flags */
-			pbc |= (!!(sc5 & 0x10)) << PBC_DC_INFO_SHIFT;
+			/* set PBC_DC_INFO bit (aka SC[4]) in pbc */
+			if (ps->s_txreq->phdr.hdr.hdr_type)
+				pbc |= PBC_PACKET_BYPASS |
+				       PBC_INSERT_BYPASS_ICRC;
+			else
+				pbc |= (ib_is_sc5(sc5) << PBC_DC_INFO_SHIFT);
 
-			if (unlikely(hfi1_dbg_fault_opcode(qp, opcode, false)))
-				pbc = hfi1_fault_tx(qp, opcode, pbc);
+			if (unlikely(hfi1_dbg_fault_opcode(qp, ps->opcode,
+							   false)))
+				pbc = hfi1_fault_tx(qp, ps->opcode, pbc);
 			pbc = create_pbc(ppd,
 					 pbc,
 					 qp->srate_mbps,
@@ -878,14 +931,15 @@ int hfi1_verbs_send_dma(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
 		if (unlikely(ret))
 			goto bail_build;
 	}
-	ret =  sdma_send_txreq(tx->sde, &priv->s_iowait, &tx->txreq);
+	ret =  sdma_send_txreq(tx->sde, &priv->s_iowait, &tx->txreq,
+			       ps->pkts_sent);
 	if (unlikely(ret < 0)) {
 		if (ret == -ECOMM)
 			goto bail_ecomm;
 		return ret;
 	}
 	trace_sdma_output_ibhdr(dd_from_ibdev(qp->ibqp.device),
-				&ps->s_txreq->phdr.hdr);
+				&ps->s_txreq->phdr.hdr, ib_is_sc5(sc5));
 	return ret;
 
 bail_ecomm:
@@ -935,7 +989,8 @@ static int pio_wait(struct rvt_qp *qp,
 			dev->n_piodrain += !!(flag & RVT_S_WAIT_PIO_DRAIN);
 			qp->s_flags |= flag;
 			was_empty = list_empty(&sc->piowait);
-			list_add_tail(&priv->s_iowait.list, &sc->piowait);
+			iowait_queue(ps->pkts_sent, &priv->s_iowait,
+				     &sc->piowait);
 			priv->s_iowait.lock = &dev->iowait_lock;
 			trace_hfi1_qpsleep(qp, RVT_S_WAIT_PIO);
 			rvt_get_qp(qp);
@@ -967,10 +1022,10 @@ int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
 	u32 hdrwords = qp->s_hdrwords;
 	struct rvt_sge_state *ss = ps->s_txreq->ss;
 	u32 len = ps->s_txreq->s_cur_size;
-	u32 dwords = (len + 3) >> 2;
-	u32 plen = hdrwords + dwords + 2; /* includes pbc */
+	u32 dwords;
+	u32 plen;
 	struct hfi1_pportdata *ppd = ps->ppd;
-	u32 *hdr = (u32 *)&ps->s_txreq->phdr.hdr;
+	u32 *hdr;
 	u8 sc5;
 	unsigned long flags = 0;
 	struct send_context *sc;
@@ -978,6 +1033,23 @@ int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
 	int wc_status = IB_WC_SUCCESS;
 	int ret = 0;
 	pio_release_cb cb = NULL;
+	u32 lrh0_16b;
+	bool bypass = false;
+	u8 extra_bytes = 0;
+
+	if (ps->s_txreq->phdr.hdr.hdr_type) {
+		u8 pad_size = hfi1_get_16b_padding((hdrwords << 2), len);
+
+		extra_bytes = pad_size + (SIZE_OF_CRC << 2) + SIZE_OF_LT;
+		dwords = (len + extra_bytes) >> 2;
+		hdr = (u32 *)&ps->s_txreq->phdr.hdr.opah;
+		lrh0_16b = ps->s_txreq->phdr.hdr.opah.lrh[0];
+		bypass = true;
+	} else {
+		dwords = (len + 3) >> 2;
+		hdr = (u32 *)&ps->s_txreq->phdr.hdr.ibh;
+	}
+	plen = hdrwords + dwords + 2;
 
 	/* only RC/UC use complete */
 	switch (qp->ibqp.qp_type) {
@@ -995,13 +1067,14 @@ int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
 
 	if (likely(pbc == 0)) {
 		u8 vl = sc_to_vlt(dd_from_ibdev(qp->ibqp.device), sc5);
-		struct verbs_txreq *tx = ps->s_txreq;
-		u8 opcode = get_opcode(&tx->phdr.hdr);
 
-		/* set PBC_DC_INFO bit (aka SC[4]) in pbc_flags */
-		pbc |= (!!(sc5 & 0x10)) << PBC_DC_INFO_SHIFT;
-		if (unlikely(hfi1_dbg_fault_opcode(qp, opcode, false)))
-			pbc = hfi1_fault_tx(qp, opcode, pbc);
+		/* set PBC_DC_INFO bit (aka SC[4]) in pbc */
+		if (ps->s_txreq->phdr.hdr.hdr_type)
+			pbc |= PBC_PACKET_BYPASS | PBC_INSERT_BYPASS_ICRC;
+		else
+			pbc |= (ib_is_sc5(sc5) << PBC_DC_INFO_SHIFT);
+		if (unlikely(hfi1_dbg_fault_opcode(qp, ps->opcode, false)))
+			pbc = hfi1_fault_tx(qp, ps->opcode, pbc);
 		pbc = create_pbc(ppd, pbc, qp->srate_mbps, vl, plen);
 	}
 	if (cb)
@@ -1038,11 +1111,12 @@ int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
 		}
 	}
 
-	if (len == 0) {
+	if (dwords == 0) {
 		pio_copy(ppd->dd, pbuf, pbc, hdr, hdrwords);
 	} else {
+		seg_pio_copy_start(pbuf, pbc,
+				   hdr, hdrwords * 4);
 		if (ss) {
-			seg_pio_copy_start(pbuf, pbc, hdr, hdrwords * 4);
 			while (len) {
 				void *addr = ss->sge.vaddr;
 				u32 slen = ss->sge.length;
@@ -1053,12 +1127,24 @@ int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
 				seg_pio_copy_mid(pbuf, addr, slen);
 				len -= slen;
 			}
-			seg_pio_copy_end(pbuf);
 		}
+		/*
+		 * Bypass packet will need to copy additional
+		 * bytes to accommodate for CRC and LT bytes
+		 */
+		if (extra_bytes) {
+			u8 *empty_buf;
+
+			empty_buf = kcalloc(extra_bytes, sizeof(u8),
+					    GFP_KERNEL);
+			seg_pio_copy_mid(pbuf, empty_buf, extra_bytes);
+			kfree(empty_buf);
+		}
+		seg_pio_copy_end(pbuf);
 	}
 
 	trace_pio_output_ibhdr(dd_from_ibdev(qp->ibqp.device),
-			       &ps->s_txreq->phdr.hdr);
+			       &ps->s_txreq->phdr.hdr, ib_is_sc5(sc5));
 
 pio_bail:
 	if (qp->s_wqe) {
@@ -1104,10 +1190,10 @@ static inline int egress_pkey_matches_entry(u16 pkey, u16 ent)
 
 /**
  * egress_pkey_check - check P_KEY of a packet
- * @ppd:    Physical IB port data
- * @lrh: Local route header
- * @bth: Base transport header
- * @sc5:    SC for packet
+ * @ppd:  Physical IB port data
+ * @slid: SLID for packet
+ * @bkey: PKEY for header
+ * @sc5:  SC for packet
  * @s_pkey_index: It will be used for look up optimization for kernel contexts
  * only. If it is negative value, then it means user contexts is calling this
  * function.
@@ -1116,19 +1202,16 @@ static inline int egress_pkey_matches_entry(u16 pkey, u16 ent)
  *
  * Return: 0 on success, otherwise, 1
  */
-int egress_pkey_check(struct hfi1_pportdata *ppd, __be16 *lrh, __be32 *bth,
+int egress_pkey_check(struct hfi1_pportdata *ppd, u32 slid, u16 pkey,
 		      u8 sc5, int8_t s_pkey_index)
 {
 	struct hfi1_devdata *dd;
 	int i;
-	u16 pkey;
 	int is_user_ctxt_mechanism = (s_pkey_index < 0);
 
 	if (!(ppd->part_enforce & HFI1_PART_ENFORCE_OUT))
 		return 0;
 
-	pkey = (u16)be32_to_cpu(bth[0]);
-
 	/* If SC15, pkey[0:14] must be 0x7fff */
 	if ((sc5 == 0xf) && ((pkey & PKEY_LOW_15_MASK) != PKEY_LOW_15_MASK))
 		goto bad;
@@ -1161,8 +1244,6 @@ bad:
 		dd = ppd->dd;
 		if (!(dd->err_info_xmit_constraint.status &
 		      OPA_EI_STATUS_SMASK)) {
-			u16 slid = be16_to_cpu(lrh[3]);
-
 			dd->err_info_xmit_constraint.status |=
 				OPA_EI_STATUS_SMASK;
 			dd->err_info_xmit_constraint.slid = slid;
@@ -1179,11 +1260,11 @@ bad:
  * and size
  */
 static inline send_routine get_send_routine(struct rvt_qp *qp,
-					    struct verbs_txreq *tx)
+					    struct hfi1_pkt_state *ps)
 {
 	struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device);
 	struct hfi1_qp_priv *priv = qp->priv;
-	struct ib_header *h = &tx->phdr.hdr;
+	struct verbs_txreq *tx = ps->s_txreq;
 
 	if (unlikely(!(dd->flags & HFI1_HAS_SEND_DMA)))
 		return dd->process_pio_send;
@@ -1195,11 +1276,9 @@ static inline send_routine get_send_routine(struct rvt_qp *qp,
 		break;
 	case IB_QPT_UC:
 	case IB_QPT_RC: {
-		u8 op = get_opcode(h);
-
 		if (piothreshold &&
 		    tx->s_cur_size <= min(piothreshold, qp->pmtu) &&
-		    (BIT(op & OPMASK) & pio_opmask[op >> 5]) &&
+		    (BIT(ps->opcode & OPMASK) & pio_opmask[ps->opcode >> 5]) &&
 		    iowait_sdma_pending(&priv->s_iowait) == 0 &&
 		    !sdma_txreq_built(&tx->txreq))
 			return dd->process_pio_send;
@@ -1224,25 +1303,38 @@ int hfi1_verbs_send(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 	struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device);
 	struct hfi1_qp_priv *priv = qp->priv;
 	struct ib_other_headers *ohdr;
-	struct ib_header *hdr;
 	send_routine sr;
 	int ret;
-	u8 lnh;
+	u16 pkey;
+	u32 slid;
 
-	hdr = &ps->s_txreq->phdr.hdr;
 	/* locate the pkey within the headers */
-	lnh = ib_get_lnh(hdr);
-	if (lnh == HFI1_LRH_GRH)
-		ohdr = &hdr->u.l.oth;
-	else
-		ohdr = &hdr->u.oth;
-
-	sr = get_send_routine(qp, ps->s_txreq);
-	ret = egress_pkey_check(dd->pport,
-				hdr->lrh,
-				ohdr->bth,
-				priv->s_sc,
-				qp->s_pkey_index);
+	if (ps->s_txreq->phdr.hdr.hdr_type) {
+		struct hfi1_16b_header *hdr = &ps->s_txreq->phdr.hdr.opah;
+		u8 l4 = hfi1_16B_get_l4(hdr);
+
+		if (l4 == OPA_16B_L4_IB_GLOBAL)
+			ohdr = &hdr->u.l.oth;
+		else
+			ohdr = &hdr->u.oth;
+		slid = hfi1_16B_get_slid(hdr);
+		pkey = hfi1_16B_get_pkey(hdr);
+	} else {
+		struct ib_header *hdr = &ps->s_txreq->phdr.hdr.ibh;
+		u8 lnh = ib_get_lnh(hdr);
+
+		if (lnh == HFI1_LRH_GRH)
+			ohdr = &hdr->u.l.oth;
+		else
+			ohdr = &hdr->u.oth;
+		slid = ib_get_slid(hdr);
+		pkey = ib_bth_get_pkey(ohdr);
+	}
+
+	ps->opcode = ib_bth_get_opcode(ohdr);
+	sr = get_send_routine(qp, ps);
+	ret = egress_pkey_check(dd->pport, slid, pkey,
+				priv->s_sc, qp->s_pkey_index);
 	if (unlikely(ret)) {
 		/*
 		 * The value we are returning here does not get propagated to
@@ -1361,14 +1453,14 @@ static int query_port(struct rvt_dev_info *rdi, u8 port_num,
 	struct hfi1_ibdev *verbs_dev = dev_from_rdi(rdi);
 	struct hfi1_devdata *dd = dd_from_dev(verbs_dev);
 	struct hfi1_pportdata *ppd = &dd->pport[port_num - 1];
-	u16 lid = ppd->lid;
+	u32 lid = ppd->lid;
 
 	/* props being zeroed by the caller, avoid zeroing it here */
 	props->lid = lid ? lid : 0;
 	props->lmc = ppd->lmc;
 	/* OPA logical states match IB logical states */
 	props->state = driver_lstate(ppd);
-	props->phys_state = hfi1_ibphys_portstate(ppd);
+	props->phys_state = driver_pstate(ppd);
 	props->gid_tbl_len = HFI1_GUIDS_PER_PORT;
 	props->active_width = (u8)opa_width_to_ib(ppd->link_width_active);
 	/* see rate_show() in ib core/sysfs.c */
@@ -1388,6 +1480,15 @@ static int query_port(struct rvt_dev_info *rdi, u8 port_num,
 	props->active_mtu = !valid_ib_mtu(ppd->ibmtu) ? props->max_mtu :
 		mtu_to_enum(ppd->ibmtu, IB_MTU_2048);
 
+	/*
+	 * sm_lid of 0xFFFF needs special handling so that it can
+	 * be differentiated from a permissve LID of 0xFFFF.
+	 * We set the grh_required flag here so the SA can program
+	 * the DGID in the address handle appropriately
+	 */
+	if (props->sm_lid == be16_to_cpu(IB_LID_PERMISSIVE))
+		props->grh_required = true;
+
 	return 0;
 }
 
@@ -1473,6 +1574,10 @@ static int hfi1_check_ah(struct ib_device *ibdev, struct rdma_ah_attr *ah_attr)
 	struct hfi1_devdata *dd;
 	u8 sc5;
 
+	if (hfi1_check_mcast(rdma_ah_get_dlid(ah_attr)) &&
+	    !(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH))
+		return -EINVAL;
+
 	/* test the mapping for validity */
 	ibp = to_iport(ibdev, rdma_ah_get_port_num(ah_attr));
 	ppd = ppd_from_ibp(ibp);
@@ -1491,6 +1596,7 @@ static void hfi1_notify_new_ah(struct ib_device *ibdev,
 	struct hfi1_pportdata *ppd;
 	struct hfi1_devdata *dd;
 	u8 sc5;
+	struct rdma_ah_attr *attr = &ah->attr;
 
 	/*
 	 * Do not trust reading anything from rvt_ah at this point as it is not
@@ -1500,33 +1606,14 @@ static void hfi1_notify_new_ah(struct ib_device *ibdev,
 	ibp = to_iport(ibdev, rdma_ah_get_port_num(ah_attr));
 	ppd = ppd_from_ibp(ibp);
 	sc5 = ibp->sl_to_sc[rdma_ah_get_sl(&ah->attr)];
+	hfi1_update_ah_attr(ibdev, attr);
+	hfi1_make_opa_lid(attr);
 	dd = dd_from_ppd(ppd);
 	ah->vl = sc_to_vlt(dd, sc5);
 	if (ah->vl < num_vls || ah->vl == 15)
 		ah->log_pmtu = ilog2(dd->vld[ah->vl].mtu);
 }
 
-struct ib_ah *hfi1_create_qp0_ah(struct hfi1_ibport *ibp, u16 dlid)
-{
-	struct rdma_ah_attr attr;
-	struct ib_ah *ah = ERR_PTR(-EINVAL);
-	struct rvt_qp *qp0;
-	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
-	struct hfi1_devdata *dd = dd_from_ppd(ppd);
-	u8 port_num = ppd->port;
-
-	memset(&attr, 0, sizeof(attr));
-	attr.type = rdma_ah_find_type(&dd->verbs_dev.rdi.ibdev, port_num);
-	rdma_ah_set_dlid(&attr, dlid);
-	rdma_ah_set_port_num(&attr, ppd_from_ibp(ibp)->port);
-	rcu_read_lock();
-	qp0 = rcu_dereference(ibp->rvp.qp[0]);
-	if (qp0)
-		ah = rdma_create_ah(qp0->ibqp.pd, &attr);
-	rcu_read_unlock();
-	return ah;
-}
-
 /**
  * hfi1_get_npkeys - return the size of the PKEY table for context 0
  * @dd: the hfi1_ib device
@@ -1547,13 +1634,22 @@ static void init_ibport(struct hfi1_pportdata *ppd)
 		ibp->sc_to_sl[i] = i;
 	}
 
+	for (i = 0; i < RVT_MAX_TRAP_LISTS ; i++)
+		INIT_LIST_HEAD(&ibp->rvp.trap_lists[i].list);
+	setup_timer(&ibp->rvp.trap_timer, hfi1_handle_trap_timer,
+		    (unsigned long)ibp);
+
 	spin_lock_init(&ibp->rvp.lock);
 	/* Set the prefix to the default value (see ch. 4.1.1) */
 	ibp->rvp.gid_prefix = IB_DEFAULT_GID_PREFIX;
 	ibp->rvp.sm_lid = 0;
-	/* Below should only set bits defined in OPA PortInfo.CapabilityMask */
+	/*
+	 * Below should only set bits defined in OPA PortInfo.CapabilityMask
+	 * and PortInfo.CapabilityMask3
+	 */
 	ibp->rvp.port_cap_flags = IB_PORT_AUTO_MIGR_SUP |
 		IB_PORT_CAP_MASK_NOTICE_SUP;
+	ibp->rvp.port_cap3_flags = OPA_CAP_MASK3_IsSharedSpaceSupported;
 	ibp->rvp.pma_counter_select[0] = IB_PMA_PORT_XMIT_DATA;
 	ibp->rvp.pma_counter_select[1] = IB_PMA_PORT_RCV_DATA;
 	ibp->rvp.pma_counter_select[2] = IB_PMA_PORT_XMIT_PKTS;
@@ -1564,14 +1660,13 @@ static void init_ibport(struct hfi1_pportdata *ppd)
 	RCU_INIT_POINTER(ibp->rvp.qp[1], NULL);
 }
 
-static void hfi1_get_dev_fw_str(struct ib_device *ibdev, char *str,
-				size_t str_len)
+static void hfi1_get_dev_fw_str(struct ib_device *ibdev, char *str)
 {
 	struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
 	struct hfi1_ibdev *dev = dev_from_rdi(rdi);
 	u32 ver = dd_from_dev(dev)->dc8051_ver;
 
-	snprintf(str, str_len, "%u.%u.%u", dc8051_ver_maj(ver),
+	snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u.%u", dc8051_ver_maj(ver),
 		 dc8051_ver_min(ver), dc8051_ver_patch(ver));
 }
 
@@ -1816,7 +1911,8 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd)
 	dd->verbs_dev.rdi.dparms.psn_mask = PSN_MASK;
 	dd->verbs_dev.rdi.dparms.psn_shift = PSN_SHIFT;
 	dd->verbs_dev.rdi.dparms.psn_modify_mask = PSN_MODIFY_MASK;
-	dd->verbs_dev.rdi.dparms.core_cap_flags = RDMA_CORE_PORT_INTEL_OPA;
+	dd->verbs_dev.rdi.dparms.core_cap_flags = RDMA_CORE_PORT_INTEL_OPA |
+						RDMA_CORE_CAP_OPA_AH;
 	dd->verbs_dev.rdi.dparms.max_mad_size = OPA_MGMT_MAD_SIZE;
 
 	dd->verbs_dev.rdi.driver_f.qp_priv_alloc = qp_priv_alloc;
diff --git a/drivers/infiniband/hw/hfi1/verbs.h b/drivers/infiniband/hw/hfi1/verbs.h
index cd635d0c1d3b..87d1285a3340 100644
--- a/drivers/infiniband/hw/hfi1/verbs.h
+++ b/drivers/infiniband/hw/hfi1/verbs.h
@@ -95,6 +95,7 @@ struct hfi1_packet;
 #define HFI1_VENDOR_IPG		cpu_to_be16(0xFFA0)
 
 #define IB_DEFAULT_GID_PREFIX	cpu_to_be64(0xfe80000000000000ULL)
+#define OPA_BTH_MIG_REQ		BIT(31)
 
 #define RC_OP(x) IB_OPCODE_RC_##x
 #define UC_OP(x) IB_OPCODE_UC_##x
@@ -104,6 +105,25 @@ enum {
 	HFI1_HAS_GRH = (1 << 0),
 };
 
+struct hfi1_16b_header {
+	u32 lrh[4];
+	union {
+		struct {
+			struct ib_grh grh;
+			struct ib_other_headers oth;
+		} l;
+		struct ib_other_headers oth;
+	} u;
+} __packed;
+
+struct hfi1_opa_header {
+	union {
+		struct ib_header ibh; /* 9B header */
+		struct hfi1_16b_header opah; /* 16B header */
+	};
+	u8 hdr_type; /* 9B or 16B */
+} __packed;
+
 struct hfi1_ahg_info {
 	u32 ahgdesc[2];
 	u16 tx_flags;
@@ -113,7 +133,7 @@ struct hfi1_ahg_info {
 
 struct hfi1_sdma_header {
 	__le64 pbc;
-	struct ib_header hdr;
+	struct hfi1_opa_header hdr;
 } __packed;
 
 /*
@@ -127,6 +147,7 @@ struct hfi1_qp_priv {
 	u8 s_sc;		                  /* SC[0..4] for next packet */
 	struct iowait s_iowait;
 	struct rvt_qp *owner;
+	u8 hdr_type; /* 9B or 16B */
 };
 
 /*
@@ -142,7 +163,9 @@ struct hfi1_pkt_state {
 	unsigned long timeout;
 	unsigned long timeout_int;
 	int cpu;
+	u8 opcode;
 	bool in_thread;
+	bool pkts_sent;
 };
 
 #define HFI1_PSN_CREDIT  16
@@ -236,8 +259,8 @@ static inline int hfi1_send_ok(struct rvt_qp *qp)
 /*
  * This must be called with s_lock held.
  */
-void hfi1_bad_pqkey(struct hfi1_ibport *ibp, __be16 trap_num, u32 key, u32 sl,
-		    u32 qp1, u32 qp2, u16 lid1, u16 lid2);
+void hfi1_bad_pkey(struct hfi1_ibport *ibp, u32 key, u32 sl,
+		   u32 qp1, u32 qp2, u32 lid1, u32 lid2);
 void hfi1_cap_mask_chg(struct rvt_dev_info *rdi, u8 port_num);
 void hfi1_sys_guid_chg(struct hfi1_ibport *ibp);
 void hfi1_node_desc_chg(struct hfi1_ibport *ibp);
@@ -257,13 +280,8 @@ int hfi1_process_mad(struct ib_device *ibdev, int mad_flags, u8 port,
  * necessarily be at least one bit less than
  * the container holding the PSN.
  */
-#ifndef CONFIG_HFI1_VERBS_31BIT_PSN
-#define PSN_MASK 0xFFFFFF
-#define PSN_SHIFT 8
-#else
 #define PSN_MASK 0x7FFFFFFF
 #define PSN_SHIFT 1
-#endif
 #define PSN_MODIFY_MASK 0xFFFFFF
 
 /*
@@ -307,15 +325,12 @@ void hfi1_rc_rcv(struct hfi1_packet *packet);
 
 void hfi1_rc_hdrerr(
 	struct hfi1_ctxtdata *rcd,
-	struct ib_header *hdr,
-	u32 rcv_flags,
+	struct hfi1_packet *packet,
 	struct rvt_qp *qp);
 
 u8 ah_to_sc(struct ib_device *ibdev, struct rdma_ah_attr *ah_attr);
 
-struct ib_ah *hfi1_create_qp0_ah(struct hfi1_ibport *ibp, u16 dlid);
-
-void hfi1_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr);
+void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah);
 
 void hfi1_ud_rcv(struct hfi1_packet *packet);
 
@@ -336,18 +351,7 @@ int hfi1_check_send_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe);
 extern const u32 rc_only_opcode;
 extern const u32 uc_only_opcode;
 
-static inline u8 get_opcode(struct ib_header *h)
-{
-	u16 lnh = be16_to_cpu(h->lrh[0]) & 3;
-
-	if (lnh == IB_LNH_IBA_LOCAL)
-		return be32_to_cpu(h->u.oth.bth[0]) >> 24;
-	else
-		return be32_to_cpu(h->u.l.oth.bth[0]) >> 24;
-}
-
-int hfi1_ruc_check_hdr(struct hfi1_ibport *ibp, struct ib_header *hdr,
-		       int has_grh, struct rvt_qp *qp, u32 bth0);
+int hfi1_ruc_check_hdr(struct hfi1_ibport *ibp, struct hfi1_packet *packet);
 
 u32 hfi1_make_grh(struct hfi1_ibport *ibp, struct ib_grh *hdr,
 		  const struct ib_global_route *grh, u32 hwords, u32 nwords);
@@ -365,7 +369,8 @@ void hfi1_do_send(struct rvt_qp *qp, bool in_thread);
 void hfi1_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe,
 			enum ib_wc_status status);
 
-void hfi1_send_rc_ack(struct hfi1_ctxtdata *, struct rvt_qp *qp, int is_fecn);
+void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp,
+		      bool is_fecn);
 
 int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps);
 
@@ -379,6 +384,8 @@ void hfi1_unregister_ib_device(struct hfi1_devdata *);
 
 void hfi1_ib_rcv(struct hfi1_packet *packet);
 
+void hfi1_16B_rcv(struct hfi1_packet *packet);
+
 unsigned hfi1_get_npkeys(struct hfi1_devdata *);
 
 int hfi1_verbs_send_dma(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
diff --git a/drivers/infiniband/hw/hfi1/verbs_txreq.c b/drivers/infiniband/hw/hfi1/verbs_txreq.c
index 5d23172c470f..873e48ea923f 100644
--- a/drivers/infiniband/hw/hfi1/verbs_txreq.c
+++ b/drivers/infiniband/hw/hfi1/verbs_txreq.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2016 Intel Corporation.
+ * Copyright(c) 2016 - 2017 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -119,13 +119,6 @@ out:
 	return tx;
 }
 
-static void verbs_txreq_kmem_cache_ctor(void *obj)
-{
-	struct verbs_txreq *tx = (struct verbs_txreq *)obj;
-
-	memset(tx, 0, sizeof(*tx));
-}
-
 int verbs_txreq_init(struct hfi1_ibdev *dev)
 {
 	char buf[TXREQ_LEN];
@@ -135,7 +128,7 @@ int verbs_txreq_init(struct hfi1_ibdev *dev)
 	dev->verbs_txreq_cache = kmem_cache_create(buf,
 						   sizeof(struct verbs_txreq),
 						   0, SLAB_HWCACHE_ALIGN,
-						   verbs_txreq_kmem_cache_ctor);
+						   NULL);
 	if (!dev->verbs_txreq_cache)
 		return -ENOMEM;
 	return 0;
diff --git a/drivers/infiniband/hw/hfi1/vnic.h b/drivers/infiniband/hw/hfi1/vnic.h
index 4a621cde4abb..5ae781514e32 100644
--- a/drivers/infiniband/hw/hfi1/vnic.h
+++ b/drivers/infiniband/hw/hfi1/vnic.h
@@ -54,21 +54,6 @@
 #define HFI1_VNIC_MAX_TXQ     16
 #define HFI1_VNIC_MAX_PAD     12
 
-/* L2 header definitions */
-#define HFI1_L2_TYPE_OFFSET     0x7
-#define HFI1_L2_TYPE_SHFT       0x5
-#define HFI1_L2_TYPE_MASK       0x3
-
-#define HFI1_GET_L2_TYPE(hdr)                                            \
-	((*((u8 *)(hdr) + HFI1_L2_TYPE_OFFSET) >> HFI1_L2_TYPE_SHFT) &   \
-	 HFI1_L2_TYPE_MASK)
-
-/* L4 type definitions */
-#define HFI1_L4_TYPE_OFFSET 8
-
-#define HFI1_GET_L4_TYPE(data)   \
-	(*((u8 *)(data) + HFI1_L4_TYPE_OFFSET))
-
 /* L4 header definitions */
 #define HFI1_VNIC_L4_HDR_OFFSET  OPA_VNIC_L2_HDR_LEN
 
@@ -103,6 +88,7 @@ struct hfi1_vnic_sdma {
 	struct sdma_txreq stx;
 	unsigned int state;
 	u8 q_idx;
+	bool pkts_sent;
 };
 
 /**
diff --git a/drivers/infiniband/hw/hfi1/vnic_main.c b/drivers/infiniband/hw/hfi1/vnic_main.c
index 339f0cdd56d6..f419cbb05928 100644
--- a/drivers/infiniband/hw/hfi1/vnic_main.c
+++ b/drivers/infiniband/hw/hfi1/vnic_main.c
@@ -95,7 +95,7 @@ static int setup_vnic_ctxt(struct hfi1_devdata *dd, struct hfi1_ctxtdata *uctxt)
 	if (HFI1_CAP_KGET_MASK(uctxt->flags, DMA_RTAIL))
 		rcvctrl_ops |= HFI1_RCVCTRL_TAILUPD_ENB;
 
-	hfi1_rcvctrl(uctxt->dd, rcvctrl_ops, uctxt->ctxt);
+	hfi1_rcvctrl(uctxt->dd, rcvctrl_ops, uctxt);
 
 	uctxt->is_vnic = true;
 done:
@@ -106,22 +106,13 @@ static int allocate_vnic_ctxt(struct hfi1_devdata *dd,
 			      struct hfi1_ctxtdata **vnic_ctxt)
 {
 	struct hfi1_ctxtdata *uctxt;
-	unsigned int ctxt;
 	int ret;
 
 	if (dd->flags & HFI1_FROZEN)
 		return -EIO;
 
-	for (ctxt = dd->first_dyn_alloc_ctxt;
-	     ctxt < dd->num_rcv_contexts; ctxt++)
-		if (!dd->rcd[ctxt])
-			break;
-
-	if (ctxt == dd->num_rcv_contexts)
-		return -EBUSY;
-
-	uctxt = hfi1_create_ctxtdata(dd->pport, ctxt, dd->node);
-	if (!uctxt) {
+	ret = hfi1_create_ctxtdata(dd->pport, dd->node, &uctxt);
+	if (ret < 0) {
 		dd_dev_err(dd, "Unable to create ctxtdata, failing open\n");
 		return -ENOMEM;
 	}
@@ -155,12 +146,7 @@ static int allocate_vnic_ctxt(struct hfi1_devdata *dd,
 
 	return ret;
 bail:
-	/*
-	 * hfi1_free_ctxtdata() also releases send_context
-	 * structure if uctxt->sc is not null
-	 */
-	dd->rcd[uctxt->ctxt] = NULL;
-	hfi1_free_ctxtdata(dd, uctxt);
+	hfi1_free_ctxt(uctxt);
 	dd_dev_dbg(dd, "vnic allocation failed. rc %d\n", ret);
 	return ret;
 }
@@ -168,15 +154,12 @@ bail:
 static void deallocate_vnic_ctxt(struct hfi1_devdata *dd,
 				 struct hfi1_ctxtdata *uctxt)
 {
-	unsigned long flags;
-
 	dd_dev_dbg(dd, "closing vnic context %d\n", uctxt->ctxt);
 	flush_wc();
 
 	if (dd->num_msix_entries)
 		hfi1_reset_vnic_msix_info(uctxt);
 
-	spin_lock_irqsave(&dd->uctxt_lock, flags);
 	/*
 	 * Disable receive context and interrupt available, reset all
 	 * RcvCtxtCtrl bits to default values.
@@ -186,7 +169,7 @@ static void deallocate_vnic_ctxt(struct hfi1_devdata *dd,
 		     HFI1_RCVCTRL_INTRAVAIL_DIS |
 		     HFI1_RCVCTRL_ONE_PKT_EGR_DIS |
 		     HFI1_RCVCTRL_NO_RHQ_DROP_DIS |
-		     HFI1_RCVCTRL_NO_EGR_DROP_DIS, uctxt->ctxt);
+		     HFI1_RCVCTRL_NO_EGR_DROP_DIS, uctxt);
 	/*
 	 * VNIC contexts are allocated from user context pool.
 	 * Release them back to user context pool.
@@ -199,16 +182,15 @@ static void deallocate_vnic_ctxt(struct hfi1_devdata *dd,
 	sc_disable(uctxt->sc);
 
 	dd->send_contexts[uctxt->sc->sw_index].type = SC_USER;
-	spin_unlock_irqrestore(&dd->uctxt_lock, flags);
 
-	dd->rcd[uctxt->ctxt] = NULL;
 	uctxt->event_flags = 0;
 
 	hfi1_clear_tids(uctxt);
 	hfi1_clear_ctxt_pkey(dd, uctxt);
 
 	hfi1_stats.sps_ctxts--;
-	hfi1_free_ctxtdata(dd, uctxt);
+
+	hfi1_free_ctxt(uctxt);
 }
 
 void hfi1_vnic_setup(struct hfi1_devdata *dd)
@@ -582,8 +564,8 @@ void hfi1_vnic_bypass_rcv(struct hfi1_packet *packet)
 	int l4_type, vesw_id = -1;
 	u8 q_idx;
 
-	l4_type = HFI1_GET_L4_TYPE(packet->ebuf);
-	if (likely(l4_type == OPA_VNIC_L4_ETHR)) {
+	l4_type = hfi1_16B_get_l4(packet->ebuf);
+	if (likely(l4_type == OPA_16B_L4_ETHR)) {
 		vesw_id = HFI1_VNIC_GET_VESWID(packet->ebuf);
 		vinfo = idr_find(&dd->vnic.vesw_idr, vesw_id);
 
@@ -751,6 +733,7 @@ static int hfi1_vnic_init(struct hfi1_vnic_vport_info *vinfo)
 		rc = hfi1_vnic_allot_ctxt(dd, &dd->vnic.ctxt[i]);
 		if (rc)
 			break;
+		hfi1_rcd_get(dd->vnic.ctxt[i]);
 		dd->vnic.ctxt[i]->vnic_q_idx = i;
 	}
 
@@ -762,6 +745,7 @@ static int hfi1_vnic_init(struct hfi1_vnic_vport_info *vinfo)
 		 */
 		while (i-- > dd->vnic.num_ctxt) {
 			deallocate_vnic_ctxt(dd, dd->vnic.ctxt[i]);
+			hfi1_rcd_put(dd->vnic.ctxt[i]);
 			dd->vnic.ctxt[i] = NULL;
 		}
 		goto alloc_fail;
@@ -791,6 +775,7 @@ static void hfi1_vnic_deinit(struct hfi1_vnic_vport_info *vinfo)
 	if (--dd->vnic.num_vports == 0) {
 		for (i = 0; i < dd->vnic.num_ctxt; i++) {
 			deallocate_vnic_ctxt(dd, dd->vnic.ctxt[i]);
+			hfi1_rcd_put(dd->vnic.ctxt[i]);
 			dd->vnic.ctxt[i] = NULL;
 		}
 		hfi1_deinit_vnic_rsm(dd);
diff --git a/drivers/infiniband/hw/hfi1/vnic_sdma.c b/drivers/infiniband/hw/hfi1/vnic_sdma.c
index 51a817d3aa14..c3c96c5869ed 100644
--- a/drivers/infiniband/hw/hfi1/vnic_sdma.c
+++ b/drivers/infiniband/hw/hfi1/vnic_sdma.c
@@ -198,11 +198,16 @@ int hfi1_vnic_send_dma(struct hfi1_devdata *dd, u8 q_idx,
 		goto free_desc;
 	tx->retry_count = 0;
 
-	ret = sdma_send_txreq(sde, &vnic_sdma->wait, &tx->txreq);
+	ret = sdma_send_txreq(sde, &vnic_sdma->wait, &tx->txreq,
+			      vnic_sdma->pkts_sent);
 	/* When -ECOMM, sdma callback will be called with ABORT status */
 	if (unlikely(ret && unlikely(ret != -ECOMM)))
 		goto free_desc;
 
+	if (!ret) {
+		vnic_sdma->pkts_sent = true;
+		iowait_starve_clear(vnic_sdma->pkts_sent, &vnic_sdma->wait);
+	}
 	return ret;
 
 free_desc:
@@ -211,6 +216,8 @@ free_desc:
 tx_err:
 	if (ret != -EBUSY)
 		dev_kfree_skb_any(skb);
+	else
+		vnic_sdma->pkts_sent = false;
 	return ret;
 }
 
@@ -225,7 +232,8 @@ tx_err:
 static int hfi1_vnic_sdma_sleep(struct sdma_engine *sde,
 				struct iowait *wait,
 				struct sdma_txreq *txreq,
-				unsigned int seq)
+				uint seq,
+				bool pkts_sent)
 {
 	struct hfi1_vnic_sdma *vnic_sdma =
 		container_of(wait, struct hfi1_vnic_sdma, wait);
@@ -239,7 +247,7 @@ static int hfi1_vnic_sdma_sleep(struct sdma_engine *sde,
 	vnic_sdma->state = HFI1_VNIC_SDMA_Q_DEFERRED;
 	write_seqlock(&dev->iowait_lock);
 	if (list_empty(&vnic_sdma->wait.list))
-		list_add_tail(&vnic_sdma->wait.list, &sde->dmawait);
+		iowait_queue(pkts_sent, wait, &sde->dmawait);
 	write_sequnlock(&dev->iowait_lock);
 	return -EBUSY;
 }
@@ -295,22 +303,15 @@ void hfi1_vnic_sdma_init(struct hfi1_vnic_vport_info *vinfo)
 	}
 }
 
-static void hfi1_vnic_txreq_kmem_cache_ctor(void *obj)
-{
-	struct vnic_txreq *tx = (struct vnic_txreq *)obj;
-
-	memset(tx, 0, sizeof(*tx));
-}
-
 int hfi1_vnic_txreq_init(struct hfi1_devdata *dd)
 {
 	char buf[HFI1_VNIC_TXREQ_NAME_LEN];
 
 	snprintf(buf, sizeof(buf), "hfi1_%u_vnic_txreq_cache", dd->unit);
 	dd->vnic.txreq_cache = kmem_cache_create(buf,
-					  sizeof(struct vnic_txreq),
-					  0, SLAB_HWCACHE_ALIGN,
-					  hfi1_vnic_txreq_kmem_cache_ctor);
+						 sizeof(struct vnic_txreq),
+						 0, SLAB_HWCACHE_ALIGN,
+						 NULL);
 	if (!dd->vnic.txreq_cache)
 		return -ENOMEM;
 	return 0;
diff --git a/drivers/infiniband/hw/hns/Kconfig b/drivers/infiniband/hw/hns/Kconfig
index e1a6e055cd60..61c93bbd230d 100644
--- a/drivers/infiniband/hw/hns/Kconfig
+++ b/drivers/infiniband/hw/hns/Kconfig
@@ -1,7 +1,7 @@
 config INFINIBAND_HNS
 	tristate "HNS RoCE Driver"
 	depends on NET_VENDOR_HISILICON
-	depends on ARM64 && HNS && HNS_DSAF && HNS_ENET
+	depends on (ARM64 || (COMPILE_TEST && 64BIT)) && HNS && HNS_DSAF && HNS_ENET
 	---help---
 	  This is a RoCE/RDMA driver for the Hisilicon RoCE engine. The engine
 	  is used in Hisilicon Hi1610 and more further ICT SoC.
diff --git a/drivers/infiniband/hw/hns/hns_roce_ah.c b/drivers/infiniband/hw/hns/hns_roce_ah.c
index f78a733a63ec..d545302b8ef8 100644
--- a/drivers/infiniband/hw/hns/hns_roce_ah.c
+++ b/drivers/infiniband/hw/hns/hns_roce_ah.c
@@ -64,8 +64,10 @@ struct ib_ah *hns_roce_create_ah(struct ib_pd *ibpd,
 	} else {
 		u8 *dmac = rdma_ah_retrieve_dmac(ah_attr);
 
-		if (!dmac)
+		if (!dmac) {
+			kfree(ah);
 			return ERR_PTR(-EINVAL);
+		}
 		memcpy(ah->av.mac, dmac, ETH_ALEN);
 	}
 
diff --git a/drivers/infiniband/hw/hns/hns_roce_alloc.c b/drivers/infiniband/hw/hns/hns_roce_alloc.c
index 605962f2828c..e1b433cdd5e2 100644
--- a/drivers/infiniband/hw/hns/hns_roce_alloc.c
+++ b/drivers/infiniband/hw/hns/hns_roce_alloc.c
@@ -32,6 +32,7 @@
  */
 
 #include <linux/platform_device.h>
+#include <linux/vmalloc.h>
 #include "hns_roce_device.h"
 
 int hns_roce_bitmap_alloc(struct hns_roce_bitmap *bitmap, unsigned long *obj)
diff --git a/drivers/infiniband/hw/hns/hns_roce_eq.c b/drivers/infiniband/hw/hns/hns_roce_eq.c
index 50f864935a0e..b0f43735de1a 100644
--- a/drivers/infiniband/hw/hns/hns_roce_eq.c
+++ b/drivers/infiniband/hw/hns/hns_roce_eq.c
@@ -31,6 +31,7 @@
  */
 
 #include <linux/platform_device.h>
+#include <linux/interrupt.h>
 #include "hns_roce_common.h"
 #include "hns_roce_device.h"
 #include "hns_roce_eq.h"
@@ -292,7 +293,7 @@ static int hns_roce_aeq_int(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
 			dev_warn(dev, "Unhandled event %d on EQ %d at index %u\n",
 				 event_type, eq->eqn, eq->cons_index);
 			break;
-		};
+		}
 
 		eq->cons_index++;
 		aeqes_found = 1;
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
index 2540b65e242c..747efd1ae5a6 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
@@ -2023,7 +2023,6 @@ int hns_roce_v1_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 	struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
 	u32 notification_flag;
 	u32 doorbell[2];
-	int ret = 0;
 
 	notification_flag = (flags & IB_CQ_SOLICITED_MASK) ==
 			    IB_CQ_SOLICITED ? CQ_DB_REQ_NOT : CQ_DB_REQ_NOT_SOL;
@@ -2043,7 +2042,7 @@ int hns_roce_v1_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 
 	hns_roce_write64_k(doorbell, hr_cq->cq_db_l);
 
-	return ret;
+	return 0;
 }
 
 static int hns_roce_v1_poll_one(struct hns_roce_cq *hr_cq,
diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c
index 80fc01ffd8bd..e387360e3780 100644
--- a/drivers/infiniband/hw/hns/hns_roce_mr.c
+++ b/drivers/infiniband/hw/hns/hns_roce_mr.c
@@ -32,6 +32,7 @@
  */
 
 #include <linux/platform_device.h>
+#include <linux/vmalloc.h>
 #include <rdma/ib_umem.h>
 #include "hns_roce_device.h"
 #include "hns_roce_cmd.h"
diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c
index 054c52699090..f5dd21c2d275 100644
--- a/drivers/infiniband/hw/hns/hns_roce_qp.c
+++ b/drivers/infiniband/hw/hns/hns_roce_qp.c
@@ -799,7 +799,7 @@ bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq,
 
 	cur = hr_wq->head - hr_wq->tail;
 	if (likely(cur + nreq < hr_wq->max_post))
-		return 0;
+		return false;
 
 	hr_cq = to_hr_cq(ib_cq);
 	spin_lock(&hr_cq->lock);
diff --git a/drivers/infiniband/hw/i40iw/i40iw_cm.c b/drivers/infiniband/hw/i40iw/i40iw_cm.c
index 5a2fa743676c..14f36ba4e5be 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_cm.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_cm.c
@@ -1582,15 +1582,14 @@ static enum i40iw_status_code i40iw_del_multiple_qhash(
 }
 
 /**
- * i40iw_netdev_vlan_ipv6 - Gets the netdev and mac
+ * i40iw_netdev_vlan_ipv6 - Gets the netdev and vlan
  * @addr: local IPv6 address
  * @vlan_id: vlan id for the given IPv6 address
- * @mac: mac address for the given IPv6 address
  *
  * Returns the net_device of the IPv6 address and also sets the
- * vlan id and mac for that address.
+ * vlan id for that address.
  */
-static struct net_device *i40iw_netdev_vlan_ipv6(u32 *addr, u16 *vlan_id, u8 *mac)
+static struct net_device *i40iw_netdev_vlan_ipv6(u32 *addr, u16 *vlan_id)
 {
 	struct net_device *ip_dev = NULL;
 	struct in6_addr laddr6;
@@ -1600,15 +1599,11 @@ static struct net_device *i40iw_netdev_vlan_ipv6(u32 *addr, u16 *vlan_id, u8 *ma
 	i40iw_copy_ip_htonl(laddr6.in6_u.u6_addr32, addr);
 	if (vlan_id)
 		*vlan_id = I40IW_NO_VLAN;
-	if (mac)
-		eth_zero_addr(mac);
 	rcu_read_lock();
 	for_each_netdev_rcu(&init_net, ip_dev) {
 		if (ipv6_chk_addr(&init_net, &laddr6, ip_dev, 1)) {
 			if (vlan_id)
 				*vlan_id = rdma_vlan_dev_vlan_id(ip_dev);
-			if (ip_dev->dev_addr && mac)
-				ether_addr_copy(mac, ip_dev->dev_addr);
 			break;
 		}
 	}
@@ -3588,7 +3583,7 @@ int i40iw_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 		cm_node->vlan_id = i40iw_get_vlan_ipv4(cm_node->loc_addr);
 	} else {
 		cm_node->ipv4 = false;
-		i40iw_netdev_vlan_ipv6(cm_node->loc_addr, &cm_node->vlan_id, NULL);
+		i40iw_netdev_vlan_ipv6(cm_node->loc_addr, &cm_node->vlan_id);
 	}
 	i40iw_debug(cm_node->dev,
 		    I40IW_DEBUG_CM,
@@ -3687,8 +3682,6 @@ int i40iw_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 
 	cm_node->accelerated = 1;
 	if (cm_node->accept_pend) {
-		if (!cm_node->listener)
-			i40iw_pr_err("cm_node->listener NULL for passive node\n");
 		atomic_dec(&cm_node->listener->pend_accepts_cnt);
 		cm_node->accept_pend = 0;
 	}
@@ -3789,7 +3782,7 @@ int i40iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 				    raddr6->sin6_addr.in6_u.u6_addr32);
 		cm_info.loc_port = ntohs(laddr6->sin6_port);
 		cm_info.rem_port = ntohs(raddr6->sin6_port);
-		i40iw_netdev_vlan_ipv6(cm_info.loc_addr, &cm_info.vlan_id, NULL);
+		i40iw_netdev_vlan_ipv6(cm_info.loc_addr, &cm_info.vlan_id);
 	}
 	cm_info.cm_id = cm_id;
 	cm_info.tos = cm_id->tos;
@@ -3931,8 +3924,7 @@ int i40iw_create_listen(struct iw_cm_id *cm_id, int backlog)
 		cm_info.loc_port = ntohs(laddr6->sin6_port);
 		if (ipv6_addr_type(&laddr6->sin6_addr) != IPV6_ADDR_ANY)
 			i40iw_netdev_vlan_ipv6(cm_info.loc_addr,
-					       &cm_info.vlan_id,
-					       NULL);
+					       &cm_info.vlan_id);
 		else
 			wildcard = true;
 	}
@@ -4056,12 +4048,7 @@ static void i40iw_cm_event_connected(struct i40iw_cm_event *event)
 	i40iw_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL);
 
 	cm_node->accelerated = 1;
-	if (cm_node->accept_pend) {
-		if (!cm_node->listener)
-			i40iw_pr_err("listener is null for passive node\n");
-		atomic_dec(&cm_node->listener->pend_accepts_cnt);
-		cm_node->accept_pend = 0;
-	}
+
 	return;
 
 error:
diff --git a/drivers/infiniband/hw/i40iw/i40iw_ctrl.c b/drivers/infiniband/hw/i40iw/i40iw_ctrl.c
index 9ec1ae9a82c9..d1f5345f04f0 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_ctrl.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_ctrl.c
@@ -54,6 +54,17 @@ static inline void i40iw_insert_wqe_hdr(u64 *wqe, u64 header)
 	set_64bit_val(wqe, 24, header);
 }
 
+void i40iw_check_cqp_progress(struct i40iw_cqp_timeout *cqp_timeout, struct i40iw_sc_dev *dev)
+{
+	if (cqp_timeout->compl_cqp_cmds != dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS]) {
+		cqp_timeout->compl_cqp_cmds = dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS];
+		cqp_timeout->count = 0;
+	} else {
+		if (dev->cqp_cmd_stats[OP_REQUESTED_COMMANDS] != cqp_timeout->compl_cqp_cmds)
+			cqp_timeout->count++;
+	}
+}
+
 /**
  * i40iw_get_cqp_reg_info - get head and tail for cqp using registers
  * @cqp: struct for cqp hw
@@ -130,20 +141,32 @@ static enum i40iw_status_code i40iw_sc_parse_fpm_commit_buf(
 	u64 base = 0;
 	u32 i, j;
 	u32 k = 0;
-	u32 low;
 
 	/* copy base values in obj_info */
-	for (i = I40IW_HMC_IW_QP, j = 0;
-			i <= I40IW_HMC_IW_PBLE; i++, j += 8) {
+	for (i = I40IW_HMC_IW_QP, j = 0; i <= I40IW_HMC_IW_PBLE; i++, j += 8) {
+		if ((i == I40IW_HMC_IW_SRQ) ||
+			(i == I40IW_HMC_IW_FSIMC) ||
+			(i == I40IW_HMC_IW_FSIAV)) {
+			info[i].base = 0;
+			info[i].cnt = 0;
+			continue;
+		}
 		get_64bit_val(buf, j, &temp);
 		info[i].base = RS_64_1(temp, 32) * 512;
 		if (info[i].base > base) {
 			base = info[i].base;
 			k = i;
 		}
-		low = (u32)(temp);
-		if (low)
-			info[i].cnt = low;
+		if (i == I40IW_HMC_IW_APBVT_ENTRY) {
+			info[i].cnt = 1;
+			continue;
+		}
+		if (i == I40IW_HMC_IW_QP)
+			info[i].cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_QPS);
+		else if (i == I40IW_HMC_IW_CQ)
+			info[i].cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_CQS);
+		else
+			info[i].cnt = (u32)(temp);
 	}
 	size = info[k].cnt * info[k].size + info[k].base;
 	if (size & 0x1FFFFF)
@@ -155,6 +178,31 @@ static enum i40iw_status_code i40iw_sc_parse_fpm_commit_buf(
 }
 
 /**
+ * i40iw_sc_decode_fpm_query() - Decode a 64 bit value into max count and size
+ * @buf: ptr to fpm query buffer
+ * @buf_idx: index into buf
+ * @info: ptr to i40iw_hmc_obj_info struct
+ * @rsrc_idx: resource index into info
+ *
+ * Decode a 64 bit value from fpm query buffer into max count and size
+ */
+static u64 i40iw_sc_decode_fpm_query(u64 *buf,
+					    u32 buf_idx,
+					    struct i40iw_hmc_obj_info *obj_info,
+					    u32 rsrc_idx)
+{
+	u64 temp;
+	u32 size;
+
+	get_64bit_val(buf, buf_idx, &temp);
+	obj_info[rsrc_idx].max_cnt = (u32)temp;
+	size = (u32)RS_64_1(temp, 32);
+	obj_info[rsrc_idx].size = LS_64_1(1, size);
+
+	return temp;
+}
+
+/**
  * i40iw_sc_parse_fpm_query_buf() - parses fpm query buffer
  * @buf: ptr to fpm query buffer
  * @info: ptr to i40iw_hmc_obj_info struct
@@ -168,9 +216,9 @@ static enum i40iw_status_code i40iw_sc_parse_fpm_query_buf(
 				struct i40iw_hmc_info *hmc_info,
 				struct i40iw_hmc_fpm_misc *hmc_fpm_misc)
 {
-	u64 temp;
 	struct i40iw_hmc_obj_info *obj_info;
-	u32 i, j, size;
+	u64 temp;
+	u32 size;
 	u16 max_pe_sds;
 
 	obj_info = hmc_info->hmc_obj;
@@ -185,41 +233,52 @@ static enum i40iw_status_code i40iw_sc_parse_fpm_query_buf(
 	hmc_fpm_misc->max_sds = max_pe_sds;
 	hmc_info->sd_table.sd_cnt = max_pe_sds + hmc_info->first_sd_index;
 
-	for (i = I40IW_HMC_IW_QP, j = 8;
-	     i <= I40IW_HMC_IW_ARP; i++, j += 8) {
-		get_64bit_val(buf, j, &temp);
-		if (i == I40IW_HMC_IW_QP)
-			obj_info[i].max_cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_QPS);
-		else if (i == I40IW_HMC_IW_CQ)
-			obj_info[i].max_cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_CQS);
-		else
-			obj_info[i].max_cnt = (u32)temp;
+	get_64bit_val(buf, 8, &temp);
+	obj_info[I40IW_HMC_IW_QP].max_cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_QPS);
+	size = (u32)RS_64_1(temp, 32);
+	obj_info[I40IW_HMC_IW_QP].size = LS_64_1(1, size);
 
-		size = (u32)RS_64_1(temp, 32);
-		obj_info[i].size = ((u64)1 << size);
-	}
-	for (i = I40IW_HMC_IW_MR, j = 48;
-			i <= I40IW_HMC_IW_PBLE; i++, j += 8) {
-		get_64bit_val(buf, j, &temp);
-		obj_info[i].max_cnt = (u32)temp;
-		size = (u32)RS_64_1(temp, 32);
-		obj_info[i].size = LS_64_1(1, size);
-	}
+	get_64bit_val(buf, 16, &temp);
+	obj_info[I40IW_HMC_IW_CQ].max_cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_CQS);
+	size = (u32)RS_64_1(temp, 32);
+	obj_info[I40IW_HMC_IW_CQ].size = LS_64_1(1, size);
+
+	i40iw_sc_decode_fpm_query(buf, 32, obj_info, I40IW_HMC_IW_HTE);
+	i40iw_sc_decode_fpm_query(buf, 40, obj_info, I40IW_HMC_IW_ARP);
+
+	obj_info[I40IW_HMC_IW_APBVT_ENTRY].size = 8192;
+	obj_info[I40IW_HMC_IW_APBVT_ENTRY].max_cnt = 1;
+
+	i40iw_sc_decode_fpm_query(buf, 48, obj_info, I40IW_HMC_IW_MR);
+	i40iw_sc_decode_fpm_query(buf, 56, obj_info, I40IW_HMC_IW_XF);
 
-	get_64bit_val(buf, 120, &temp);
-	hmc_fpm_misc->max_ceqs = (u8)RS_64(temp, I40IW_QUERY_FPM_MAX_CEQS);
-	get_64bit_val(buf, 120, &temp);
-	hmc_fpm_misc->ht_multiplier = RS_64(temp, I40IW_QUERY_FPM_HTMULTIPLIER);
-	get_64bit_val(buf, 120, &temp);
-	hmc_fpm_misc->timer_bucket = RS_64(temp, I40IW_QUERY_FPM_TIMERBUCKET);
 	get_64bit_val(buf, 64, &temp);
+	obj_info[I40IW_HMC_IW_XFFL].max_cnt = (u32)temp;
+	obj_info[I40IW_HMC_IW_XFFL].size = 4;
 	hmc_fpm_misc->xf_block_size = RS_64(temp, I40IW_QUERY_FPM_XFBLOCKSIZE);
 	if (!hmc_fpm_misc->xf_block_size)
 		return I40IW_ERR_INVALID_SIZE;
+
+	i40iw_sc_decode_fpm_query(buf, 72, obj_info, I40IW_HMC_IW_Q1);
+
 	get_64bit_val(buf, 80, &temp);
+	obj_info[I40IW_HMC_IW_Q1FL].max_cnt = (u32)temp;
+	obj_info[I40IW_HMC_IW_Q1FL].size = 4;
 	hmc_fpm_misc->q1_block_size = RS_64(temp, I40IW_QUERY_FPM_Q1BLOCKSIZE);
 	if (!hmc_fpm_misc->q1_block_size)
 		return I40IW_ERR_INVALID_SIZE;
+
+	i40iw_sc_decode_fpm_query(buf, 88, obj_info, I40IW_HMC_IW_TIMER);
+
+	get_64bit_val(buf, 112, &temp);
+	obj_info[I40IW_HMC_IW_PBLE].max_cnt = (u32)temp;
+	obj_info[I40IW_HMC_IW_PBLE].size = 8;
+
+	get_64bit_val(buf, 120, &temp);
+	hmc_fpm_misc->max_ceqs = (u8)RS_64(temp, I40IW_QUERY_FPM_MAX_CEQS);
+	hmc_fpm_misc->ht_multiplier = RS_64(temp, I40IW_QUERY_FPM_HTMULTIPLIER);
+	hmc_fpm_misc->timer_bucket = RS_64(temp, I40IW_QUERY_FPM_TIMERBUCKET);
+
 	return 0;
 }
 
@@ -3392,13 +3451,6 @@ enum i40iw_status_code i40iw_sc_init_iw_hmc(struct i40iw_sc_dev *dev, u8 hmc_fn_
 		hmc_info->sd_table.sd_entry = virt_mem.va;
 	}
 
-	/* fill size of objects which are fixed */
-	hmc_info->hmc_obj[I40IW_HMC_IW_XFFL].size = 4;
-	hmc_info->hmc_obj[I40IW_HMC_IW_Q1FL].size = 4;
-	hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].size = 8;
-	hmc_info->hmc_obj[I40IW_HMC_IW_APBVT_ENTRY].size = 8192;
-	hmc_info->hmc_obj[I40IW_HMC_IW_APBVT_ENTRY].max_cnt = 1;
-
 	return ret_code;
 }
 
@@ -4840,7 +4892,7 @@ void i40iw_vsi_stats_free(struct i40iw_sc_vsi *vsi)
 {
 	u8 fcn_id = vsi->fcn_id;
 
-	if ((vsi->stats_fcn_id_alloc) && (fcn_id != I40IW_INVALID_FCN_ID))
+	if (vsi->stats_fcn_id_alloc && fcn_id < I40IW_MAX_STATS_COUNT)
 		vsi->dev->fcn_id_array[fcn_id] = false;
 	i40iw_hw_stats_stop_timer(vsi);
 }
diff --git a/drivers/infiniband/hw/i40iw/i40iw_d.h b/drivers/infiniband/hw/i40iw/i40iw_d.h
index a39ac12b6a7e..2ebaadbed379 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_d.h
+++ b/drivers/infiniband/hw/i40iw/i40iw_d.h
@@ -1507,8 +1507,8 @@ enum {
 	I40IW_CQ0_ALIGNMENT_MASK =		(256 - 1),
 	I40IW_HOST_CTX_ALIGNMENT_MASK =		(4 - 1),
 	I40IW_SHADOWAREA_MASK =			(128 - 1),
-	I40IW_FPM_QUERY_BUF_ALIGNMENT_MASK =	0,
-	I40IW_FPM_COMMIT_BUF_ALIGNMENT_MASK =	0
+	I40IW_FPM_QUERY_BUF_ALIGNMENT_MASK =	(4 - 1),
+	I40IW_FPM_COMMIT_BUF_ALIGNMENT_MASK =	(4 - 1)
 };
 
 enum i40iw_alignment {
diff --git a/drivers/infiniband/hw/i40iw/i40iw_main.c b/drivers/infiniband/hw/i40iw/i40iw_main.c
index ae8463ff59a7..cc742c3132c6 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_main.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_main.c
@@ -77,7 +77,6 @@ MODULE_PARM_DESC(mpa_version, "MPA version to be used in MPA Req/Resp 1 or 2");
 MODULE_AUTHOR("Intel Corporation, <e1000-rdma@lists.sourceforge.net>");
 MODULE_DESCRIPTION("Intel(R) Ethernet Connection X722 iWARP RDMA Driver");
 MODULE_LICENSE("Dual BSD/GPL");
-MODULE_VERSION(DRV_VERSION);
 
 static struct i40e_client i40iw_client;
 static char i40iw_client_name[I40E_CLIENT_STR_LENGTH] = "i40iw";
diff --git a/drivers/infiniband/hw/i40iw/i40iw_p.h b/drivers/infiniband/hw/i40iw/i40iw_p.h
index 28a92fee0822..e217a1259f57 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_p.h
+++ b/drivers/infiniband/hw/i40iw/i40iw_p.h
@@ -35,11 +35,13 @@
 #ifndef I40IW_P_H
 #define I40IW_P_H
 
-#define PAUSE_TIMER_VALUE  0xFFFF
-#define REFRESH_THRESHOLD  0x7FFF
-#define HIGH_THRESHOLD     0x800
-#define LOW_THRESHOLD      0x200
-#define ALL_TC2PFC         0xFF
+#define PAUSE_TIMER_VALUE       0xFFFF
+#define REFRESH_THRESHOLD       0x7FFF
+#define HIGH_THRESHOLD          0x800
+#define LOW_THRESHOLD           0x200
+#define ALL_TC2PFC              0xFF
+#define CQP_COMPL_WAIT_TIME     0x3E8
+#define CQP_TIMEOUT_THRESHOLD   5
 
 void i40iw_debug_buf(struct i40iw_sc_dev *dev, enum i40iw_debug_flag mask,
 		     char *desc, u64 *buf, u32 size);
@@ -51,6 +53,8 @@ void i40iw_sc_cqp_post_sq(struct i40iw_sc_cqp *cqp);
 
 u64 *i40iw_sc_cqp_get_next_send_wqe(struct i40iw_sc_cqp *cqp, u64 scratch);
 
+void i40iw_check_cqp_progress(struct i40iw_cqp_timeout *cqp_timeout, struct i40iw_sc_dev *dev);
+
 enum i40iw_status_code i40iw_sc_mr_fast_register(struct i40iw_sc_qp *qp,
 						 struct i40iw_fast_reg_stag_info *info,
 						 bool post_sq);
diff --git a/drivers/infiniband/hw/i40iw/i40iw_pble.c b/drivers/infiniband/hw/i40iw/i40iw_pble.c
index c87ba1617087..540aab5e502d 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_pble.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_pble.c
@@ -269,10 +269,8 @@ static enum i40iw_status_code add_bp_pages(struct i40iw_sc_dev *dev,
 	status = i40iw_add_sd_table_entry(dev->hw, hmc_info,
 					  info->idx.sd_idx, I40IW_SD_TYPE_PAGED,
 					  I40IW_HMC_DIRECT_BP_SIZE);
-	if (status) {
-		i40iw_free_vmalloc_mem(dev->hw, chunk);
-		return status;
-	}
+	if (status)
+		goto error;
 	if (!dev->is_pf) {
 		status = i40iw_vchnl_vf_add_hmc_objs(dev, I40IW_HMC_IW_PBLE,
 						     fpm_to_idx(pble_rsrc,
@@ -280,8 +278,7 @@ static enum i40iw_status_code add_bp_pages(struct i40iw_sc_dev *dev,
 						     (info->pages << PBLE_512_SHIFT));
 		if (status) {
 			i40iw_pr_err("allocate PBLEs in the PF.  Error %i\n", status);
-			i40iw_free_vmalloc_mem(dev->hw, chunk);
-			return status;
+			goto error;
 		}
 	}
 	addr = chunk->vaddr;
diff --git a/drivers/infiniband/hw/i40iw/i40iw_puda.c b/drivers/infiniband/hw/i40iw/i40iw_puda.c
index 71050c5d29a0..c2cab20c4bc5 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_puda.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_puda.c
@@ -685,7 +685,7 @@ static enum i40iw_status_code i40iw_puda_cq_create(struct i40iw_puda_rsrc *rsrc)
 	cqsize = rsrc->cq_size * (sizeof(struct i40iw_cqe));
 	tsize = cqsize + sizeof(struct i40iw_cq_shadow_area);
 	ret = i40iw_allocate_dma_mem(dev->hw, &rsrc->cqmem, tsize,
-				     I40IW_CQ0_ALIGNMENT_MASK);
+				     I40IW_CQ0_ALIGNMENT);
 	if (ret)
 		return ret;
 
@@ -949,14 +949,16 @@ enum i40iw_status_code i40iw_puda_create_rsrc(struct i40iw_sc_vsi *vsi,
 		ret = i40iw_puda_qp_create(rsrc);
 	}
 	if (ret) {
-		i40iw_debug(dev, I40IW_DEBUG_PUDA, "[%s] error qp_create\n", __func__);
+		i40iw_debug(dev, I40IW_DEBUG_PUDA, "[%s] error qp_create\n",
+			    __func__);
 		goto error;
 	}
 	rsrc->completion = PUDA_QP_CREATED;
 
 	ret = i40iw_puda_allocbufs(rsrc, info->tx_buf_cnt + info->rq_size);
 	if (ret) {
-		i40iw_debug(dev, I40IW_DEBUG_PUDA, "[%s] error allloc_buf\n", __func__);
+		i40iw_debug(dev, I40IW_DEBUG_PUDA, "[%s] error alloc_buf\n",
+			    __func__);
 		goto error;
 	}
 
diff --git a/drivers/infiniband/hw/i40iw/i40iw_status.h b/drivers/infiniband/hw/i40iw/i40iw_status.h
index 91c421762f06..f7013f11d808 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_status.h
+++ b/drivers/infiniband/hw/i40iw/i40iw_status.h
@@ -62,7 +62,7 @@ enum i40iw_status_code {
 	I40IW_ERR_INVALID_ALIGNMENT = -23,
 	I40IW_ERR_FLUSHED_QUEUE = -24,
 	I40IW_ERR_INVALID_PUSH_PAGE_INDEX = -25,
-	I40IW_ERR_INVALID_IMM_DATA_SIZE = -26,
+	I40IW_ERR_INVALID_INLINE_DATA_SIZE = -26,
 	I40IW_ERR_TIMEOUT = -27,
 	I40IW_ERR_OPCODE_MISMATCH = -28,
 	I40IW_ERR_CQP_COMPL_ERROR = -29,
diff --git a/drivers/infiniband/hw/i40iw/i40iw_type.h b/drivers/infiniband/hw/i40iw/i40iw_type.h
index 959ec81fba99..63118f6d5ab4 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_type.h
+++ b/drivers/infiniband/hw/i40iw/i40iw_type.h
@@ -1345,4 +1345,9 @@ struct i40iw_virtchnl_work_info {
 	void *worker_vf_dev;
 };
 
+struct i40iw_cqp_timeout {
+	u64 compl_cqp_cmds;
+	u8 count;
+};
+
 #endif
diff --git a/drivers/infiniband/hw/i40iw/i40iw_uk.c b/drivers/infiniband/hw/i40iw/i40iw_uk.c
index b0d3a0e8a9b5..0aadb7a0d1aa 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_uk.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_uk.c
@@ -435,7 +435,7 @@ static enum i40iw_status_code i40iw_inline_rdma_write(struct i40iw_qp_uk *qp,
 
 	op_info = &info->op.inline_rdma_write;
 	if (op_info->len > I40IW_MAX_INLINE_DATA_SIZE)
-		return I40IW_ERR_INVALID_IMM_DATA_SIZE;
+		return I40IW_ERR_INVALID_INLINE_DATA_SIZE;
 
 	ret_code = i40iw_inline_data_size_to_wqesize(op_info->len, &wqe_size);
 	if (ret_code)
@@ -511,7 +511,7 @@ static enum i40iw_status_code i40iw_inline_send(struct i40iw_qp_uk *qp,
 
 	op_info = &info->op.inline_send;
 	if (op_info->len > I40IW_MAX_INLINE_DATA_SIZE)
-		return I40IW_ERR_INVALID_IMM_DATA_SIZE;
+		return I40IW_ERR_INVALID_INLINE_DATA_SIZE;
 
 	ret_code = i40iw_inline_data_size_to_wqesize(op_info->len, &wqe_size);
 	if (ret_code)
@@ -784,7 +784,7 @@ static enum i40iw_status_code i40iw_cq_poll_completion(struct i40iw_cq_uk *cq,
 	get_64bit_val(cqe, 0, &qword0);
 	get_64bit_val(cqe, 16, &qword2);
 
-	info->tcp_seq_num = (u8)RS_64(qword0, I40IWCQ_TCPSEQNUM);
+	info->tcp_seq_num = (u32)RS_64(qword0, I40IWCQ_TCPSEQNUM);
 
 	info->qp_id = (u32)RS_64(qword2, I40IWCQ_QPID);
 
@@ -912,7 +912,7 @@ enum i40iw_status_code i40iw_get_wqe_shift(u32 wqdepth, u32 sge, u32 inline_data
 	return 0;
 }
 
-static struct i40iw_qp_uk_ops iw_qp_uk_ops = {
+static const struct i40iw_qp_uk_ops iw_qp_uk_ops = {
 	.iw_qp_post_wr = i40iw_qp_post_wr,
 	.iw_qp_ring_push_db = i40iw_qp_ring_push_db,
 	.iw_rdma_write = i40iw_rdma_write,
@@ -926,14 +926,14 @@ static struct i40iw_qp_uk_ops iw_qp_uk_ops = {
 	.iw_post_nop = i40iw_nop
 };
 
-static struct i40iw_cq_ops iw_cq_ops = {
+static const struct i40iw_cq_ops iw_cq_ops = {
 	.iw_cq_request_notification = i40iw_cq_request_notification,
 	.iw_cq_poll_completion = i40iw_cq_poll_completion,
 	.iw_cq_post_entries = i40iw_cq_post_entries,
 	.iw_cq_clean = i40iw_clean_cq
 };
 
-static struct i40iw_device_uk_ops iw_device_uk_ops = {
+static const struct i40iw_device_uk_ops iw_device_uk_ops = {
 	.iwarp_cq_uk_init = i40iw_cq_uk_init,
 	.iwarp_qp_uk_init = i40iw_qp_uk_init,
 };
@@ -1187,7 +1187,7 @@ enum i40iw_status_code i40iw_inline_data_size_to_wqesize(u32 data_size,
 							 u8 *wqe_size)
 {
 	if (data_size > I40IW_MAX_INLINE_DATA_SIZE)
-		return I40IW_ERR_INVALID_IMM_DATA_SIZE;
+		return I40IW_ERR_INVALID_INLINE_DATA_SIZE;
 
 	if (data_size <= 16)
 		*wqe_size = I40IW_QP_WQE_MIN_SIZE;
diff --git a/drivers/infiniband/hw/i40iw/i40iw_utils.c b/drivers/infiniband/hw/i40iw/i40iw_utils.c
index e311ec559f4e..62f1f45b8737 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_utils.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_utils.c
@@ -445,23 +445,29 @@ static int i40iw_wait_event(struct i40iw_device *iwdev,
 {
 	struct cqp_commands_info *info = &cqp_request->info;
 	struct i40iw_cqp *iwcqp = &iwdev->cqp;
+	struct i40iw_cqp_timeout cqp_timeout;
 	bool cqp_error = false;
 	int err_code = 0;
-	int timeout_ret = 0;
+	memset(&cqp_timeout, 0, sizeof(cqp_timeout));
+	cqp_timeout.compl_cqp_cmds = iwdev->sc_dev.cqp_cmd_stats[OP_COMPLETED_COMMANDS];
+	do {
+		if (wait_event_timeout(cqp_request->waitq,
+				       cqp_request->request_done, CQP_COMPL_WAIT_TIME))
+			break;
 
-	timeout_ret = wait_event_timeout(cqp_request->waitq,
-					 cqp_request->request_done,
-					 I40IW_EVENT_TIMEOUT);
-	if (!timeout_ret) {
-		i40iw_pr_err("error cqp command 0x%x timed out ret = %d\n",
-			     info->cqp_cmd, timeout_ret);
+		i40iw_check_cqp_progress(&cqp_timeout, &iwdev->sc_dev);
+
+		if (cqp_timeout.count < CQP_TIMEOUT_THRESHOLD)
+			continue;
+
+		i40iw_pr_err("error cqp command 0x%x timed out", info->cqp_cmd);
 		err_code = -ETIME;
 		if (!iwdev->reset) {
 			iwdev->reset = true;
 			i40iw_request_reset(iwdev);
 		}
 		goto done;
-	}
+	} while (1);
 	cqp_error = cqp_request->compl_info.error;
 	if (cqp_error) {
 		i40iw_pr_err("error cqp command 0x%x completion maj = 0x%x min=0x%x\n",
diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
index 02d871db7ca5..1aa411034a27 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
@@ -2584,13 +2584,12 @@ static const char * const i40iw_hw_stat_names[] = {
 		"iwRdmaInv"
 };
 
-static void i40iw_get_dev_fw_str(struct ib_device *dev, char *str,
-				 size_t str_len)
+static void i40iw_get_dev_fw_str(struct ib_device *dev, char *str)
 {
 	u32 firmware_version = I40IW_FW_VERSION;
 
-	snprintf(str, str_len, "%u.%u", firmware_version,
-		       (firmware_version & 0x000000ff));
+	snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u", firmware_version,
+		 (firmware_version & 0x000000ff));
 }
 
 /**
diff --git a/drivers/infiniband/hw/mlx4/alias_GUID.c b/drivers/infiniband/hw/mlx4/alias_GUID.c
index ea24230ea0d4..155b4dfc0ae8 100644
--- a/drivers/infiniband/hw/mlx4/alias_GUID.c
+++ b/drivers/infiniband/hw/mlx4/alias_GUID.c
@@ -528,7 +528,7 @@ static int set_guid_rec(struct ib_device *ibdev,
 
 	memset(&guid_info_rec, 0, sizeof (struct ib_sa_guidinfo_rec));
 
-	guid_info_rec.lid = cpu_to_be16(attr.lid);
+	guid_info_rec.lid = ib_lid_be16(attr.lid);
 	guid_info_rec.block_num = index;
 
 	memcpy(guid_info_rec.guid_info_list, rec_det->all_recs,
@@ -781,7 +781,7 @@ void mlx4_ib_init_alias_guid_work(struct mlx4_ib_dev *dev, int port)
 	spin_lock_irqsave(&dev->sriov.going_down_lock, flags);
 	spin_lock_irqsave(&dev->sriov.alias_guid.ag_work_lock, flags1);
 	if (!dev->sriov.is_going_down) {
-		/* If there is pending one should cancell then run, otherwise
+		/* If there is pending one should cancel then run, otherwise
 		  * won't run till previous one is ended as same work
 		  * struct is used.
 		  */
diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c
index ff931c580557..cab796341697 100644
--- a/drivers/infiniband/hw/mlx4/cq.c
+++ b/drivers/infiniband/hw/mlx4/cq.c
@@ -218,6 +218,7 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,
 			goto err_mtt;
 
 		uar = &to_mucontext(context)->uar;
+		cq->mcq.usage = MLX4_RES_USAGE_USER_VERBS;
 	} else {
 		err = mlx4_db_alloc(dev->dev, &cq->db, 1);
 		if (err)
@@ -233,6 +234,7 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,
 			goto err_db;
 
 		uar = &dev->priv_uar;
+		cq->mcq.usage = MLX4_RES_USAGE_DRIVER;
 	}
 
 	if (dev->eq_table)
@@ -635,7 +637,7 @@ static void mlx4_ib_poll_sw_comp(struct mlx4_ib_cq *cq, int num_entries,
 	struct mlx4_ib_qp *qp;
 
 	*npolled = 0;
-	/* Find uncompleted WQEs belonging to that cq and retrun
+	/* Find uncompleted WQEs belonging to that cq and return
 	 * simulated FLUSH_ERR completions
 	 */
 	list_for_each_entry(qp, &cq->send_qp_list, cq_send_list) {
diff --git a/drivers/infiniband/hw/mlx4/mad.c b/drivers/infiniband/hw/mlx4/mad.c
index 21d31cb1325f..0793a21d76f4 100644
--- a/drivers/infiniband/hw/mlx4/mad.c
+++ b/drivers/infiniband/hw/mlx4/mad.c
@@ -169,7 +169,7 @@ int mlx4_MAD_IFC(struct mlx4_ib_dev *dev, int mad_ifc_flags,
 
 		op_modifier |= 0x4;
 
-		in_modifier |= in_wc->slid << 16;
+		in_modifier |= ib_lid_cpu16(in_wc->slid) << 16;
 	}
 
 	err = mlx4_cmd_box(dev->dev, inmailbox->dma, outmailbox->dma, in_modifier,
@@ -625,7 +625,7 @@ int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port,
 		memcpy((char *)&tun_mad->hdr.slid_mac_47_32, &(wc->smac[4]), 2);
 	} else {
 		tun_mad->hdr.sl_vid = cpu_to_be16(((u16)(wc->sl)) << 12);
-		tun_mad->hdr.slid_mac_47_32 = cpu_to_be16(wc->slid);
+		tun_mad->hdr.slid_mac_47_32 = ib_lid_be16(wc->slid);
 	}
 
 	ib_dma_sync_single_for_device(&dev->ib_dev,
@@ -826,7 +826,7 @@ static int ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
 		}
 	}
 
-	slid = in_wc ? in_wc->slid : be16_to_cpu(IB_LID_PERMISSIVE);
+	slid = in_wc ? ib_lid_cpu16(in_wc->slid) : be16_to_cpu(IB_LID_PERMISSIVE);
 
 	if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP && slid == 0) {
 		forward_trap(to_mdev(ibdev), port_num, in_mad);
@@ -860,7 +860,7 @@ static int ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
 	    in_mad->mad_hdr.method == IB_MGMT_METHOD_SET &&
 	    in_mad->mad_hdr.attr_id == IB_SMP_ATTR_PORT_INFO &&
 	    !ib_query_port(ibdev, port_num, &pattr))
-		prev_lid = pattr.lid;
+		prev_lid = ib_lid_cpu16(pattr.lid);
 
 	err = mlx4_MAD_IFC(to_mdev(ibdev),
 			   (mad_flags & IB_MAD_IGNORE_MKEY ? MLX4_MAD_IFC_IGNORE_MKEY : 0) |
diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
index d1b43cbbfea7..c636842c5be0 100644
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -70,7 +70,6 @@
 MODULE_AUTHOR("Roland Dreier");
 MODULE_DESCRIPTION("Mellanox ConnectX HCA InfiniBand driver");
 MODULE_LICENSE("Dual BSD/GPL");
-MODULE_VERSION(DRV_VERSION);
 
 int mlx4_ib_sm_guid_assign = 0;
 module_param_named(sm_guid_assign, mlx4_ib_sm_guid_assign, int, 0444);
@@ -81,6 +80,8 @@ static const char mlx4_ib_version[] =
 	DRV_VERSION "\n";
 
 static void do_slave_init(struct mlx4_ib_dev *ibdev, int slave, int do_init);
+static enum rdma_link_layer mlx4_ib_port_link_layer(struct ib_device *device,
+						    u8 port_num);
 
 static struct workqueue_struct *wq;
 
@@ -552,6 +553,16 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,
 	props->timestamp_mask = 0xFFFFFFFFFFFFULL;
 	props->max_ah = INT_MAX;
 
+	if ((dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS) &&
+	    (mlx4_ib_port_link_layer(ibdev, 1) == IB_LINK_LAYER_ETHERNET ||
+	     mlx4_ib_port_link_layer(ibdev, 2) == IB_LINK_LAYER_ETHERNET)) {
+		props->rss_caps.max_rwq_indirection_tables = props->max_qp;
+		props->rss_caps.max_rwq_indirection_table_size =
+			dev->dev->caps.max_rss_tbl_sz;
+		props->rss_caps.supported_qpts = 1 << IB_QPT_RAW_PACKET;
+		props->max_wq_type_rq = props->max_qp;
+	}
+
 	if (!mlx4_is_slave(dev->dev))
 		err = mlx4_get_internal_clock_params(dev->dev, &clock_params);
 
@@ -563,6 +574,13 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,
 		}
 	}
 
+	if (uhw->outlen >= resp.response_length +
+	    sizeof(resp.max_inl_recv_sz)) {
+		resp.response_length += sizeof(resp.max_inl_recv_sz);
+		resp.max_inl_recv_sz  = dev->dev->caps.max_rq_sg *
+			sizeof(struct mlx4_wqe_data_seg);
+	}
+
 	if (uhw->outlen) {
 		err = ib_copy_to_udata(uhw, &resp, resp.response_length);
 		if (err)
@@ -1069,6 +1087,9 @@ static struct ib_ucontext *mlx4_ib_alloc_ucontext(struct ib_device *ibdev,
 	INIT_LIST_HEAD(&context->db_page_list);
 	mutex_init(&context->db_page_mutex);
 
+	INIT_LIST_HEAD(&context->wqn_ranges_list);
+	mutex_init(&context->wqn_ranges_mutex);
+
 	if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION)
 		err = ib_copy_to_udata(udata, &resp_v3, sizeof(resp_v3));
 	else
@@ -2566,12 +2587,11 @@ static int mlx4_port_immutable(struct ib_device *ibdev, u8 port_num,
 	return 0;
 }
 
-static void get_fw_ver_str(struct ib_device *device, char *str,
-			   size_t str_len)
+static void get_fw_ver_str(struct ib_device *device, char *str)
 {
 	struct mlx4_ib_dev *dev =
 		container_of(device, struct mlx4_ib_dev, ib_dev);
-	snprintf(str, str_len, "%d.%d.%d",
+	snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%d",
 		 (int) (dev->dev->caps.fw_ver >> 32),
 		 (int) (dev->dev->caps.fw_ver >> 16) & 0xffff,
 		 (int) dev->dev->caps.fw_ver & 0xffff);
@@ -2713,6 +2733,26 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
 	ibdev->ib_dev.get_dev_fw_str    = get_fw_ver_str;
 	ibdev->ib_dev.disassociate_ucontext = mlx4_ib_disassociate_ucontext;
 
+	if ((dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS) &&
+	    ((mlx4_ib_port_link_layer(&ibdev->ib_dev, 1) ==
+	    IB_LINK_LAYER_ETHERNET) ||
+	    (mlx4_ib_port_link_layer(&ibdev->ib_dev, 2) ==
+	    IB_LINK_LAYER_ETHERNET))) {
+		ibdev->ib_dev.create_wq		= mlx4_ib_create_wq;
+		ibdev->ib_dev.modify_wq		= mlx4_ib_modify_wq;
+		ibdev->ib_dev.destroy_wq	= mlx4_ib_destroy_wq;
+		ibdev->ib_dev.create_rwq_ind_table  =
+			mlx4_ib_create_rwq_ind_table;
+		ibdev->ib_dev.destroy_rwq_ind_table =
+			mlx4_ib_destroy_rwq_ind_table;
+		ibdev->ib_dev.uverbs_ex_cmd_mask |=
+			(1ull << IB_USER_VERBS_EX_CMD_CREATE_WQ)	  |
+			(1ull << IB_USER_VERBS_EX_CMD_MODIFY_WQ)	  |
+			(1ull << IB_USER_VERBS_EX_CMD_DESTROY_WQ)	  |
+			(1ull << IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL) |
+			(1ull << IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL);
+	}
+
 	if (!mlx4_is_slave(ibdev->dev)) {
 		ibdev->ib_dev.alloc_fmr		= mlx4_ib_fmr_alloc;
 		ibdev->ib_dev.map_phys_fmr	= mlx4_ib_map_phys_fmr;
@@ -2772,7 +2812,8 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
 		allocated = 0;
 		if (mlx4_ib_port_link_layer(&ibdev->ib_dev, i + 1) ==
 						IB_LINK_LAYER_ETHERNET) {
-			err = mlx4_counter_alloc(ibdev->dev, &counter_index);
+			err = mlx4_counter_alloc(ibdev->dev, &counter_index,
+						 MLX4_RES_USAGE_DRIVER);
 			/* if failed to allocate a new counter, use default */
 			if (err)
 				counter_index =
@@ -2827,7 +2868,8 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
 		ibdev->steer_qpn_count = MLX4_IB_UC_MAX_NUM_QPS;
 		err = mlx4_qp_reserve_range(dev, ibdev->steer_qpn_count,
 					    MLX4_IB_UC_STEER_QPN_ALIGN,
-					    &ibdev->steer_qpn_base, 0);
+					    &ibdev->steer_qpn_base, 0,
+					    MLX4_RES_USAGE_DRIVER);
 		if (err)
 			goto err_counter;
 
diff --git a/drivers/infiniband/hw/mlx4/mcg.c b/drivers/infiniband/hw/mlx4/mcg.c
index b73f89700ef9..70eb9f917303 100644
--- a/drivers/infiniband/hw/mlx4/mcg.c
+++ b/drivers/infiniband/hw/mlx4/mcg.c
@@ -808,8 +808,7 @@ static ssize_t sysfs_show_group(struct device *dev,
 		struct device_attribute *attr, char *buf);
 
 static struct mcast_group *acquire_group(struct mlx4_ib_demux_ctx *ctx,
-					 union ib_gid *mgid, int create,
-					 gfp_t gfp_mask)
+					 union ib_gid *mgid, int create)
 {
 	struct mcast_group *group, *cur_group;
 	int is_mgid0;
@@ -825,7 +824,7 @@ static struct mcast_group *acquire_group(struct mlx4_ib_demux_ctx *ctx,
 	if (!create)
 		return ERR_PTR(-ENOENT);
 
-	group = kzalloc(sizeof *group, gfp_mask);
+	group = kzalloc(sizeof(*group), GFP_KERNEL);
 	if (!group)
 		return ERR_PTR(-ENOMEM);
 
@@ -892,7 +891,7 @@ int mlx4_ib_mcg_demux_handler(struct ib_device *ibdev, int port, int slave,
 	case IB_MGMT_METHOD_GET_RESP:
 	case IB_SA_METHOD_DELETE_RESP:
 		mutex_lock(&ctx->mcg_table_lock);
-		group = acquire_group(ctx, &rec->mgid, 0, GFP_KERNEL);
+		group = acquire_group(ctx, &rec->mgid, 0);
 		mutex_unlock(&ctx->mcg_table_lock);
 		if (IS_ERR(group)) {
 			if (mad->mad_hdr.method == IB_MGMT_METHOD_GET_RESP) {
@@ -954,7 +953,7 @@ int mlx4_ib_mcg_multiplex_handler(struct ib_device *ibdev, int port,
 		req->sa_mad = *sa_mad;
 
 		mutex_lock(&ctx->mcg_table_lock);
-		group = acquire_group(ctx, &rec->mgid, may_create, GFP_KERNEL);
+		group = acquire_group(ctx, &rec->mgid, may_create);
 		mutex_unlock(&ctx->mcg_table_lock);
 		if (IS_ERR(group)) {
 			kfree(req);
diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h
index 9db82e67e959..1fa19820355a 100644
--- a/drivers/infiniband/hw/mlx4/mlx4_ib.h
+++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h
@@ -46,6 +46,7 @@
 
 #include <linux/mlx4/device.h>
 #include <linux/mlx4/doorbell.h>
+#include <linux/mlx4/qp.h>
 
 #define MLX4_IB_DRV_NAME	"mlx4_ib"
 
@@ -88,6 +89,8 @@ struct mlx4_ib_ucontext {
 	struct list_head	db_page_list;
 	struct mutex		db_page_mutex;
 	struct mlx4_ib_vma_private_data hw_bar_info[HW_BAR_COUNT];
+	struct list_head	wqn_ranges_list;
+	struct mutex		wqn_ranges_mutex; /* protect wqn_ranges_list */
 };
 
 struct mlx4_ib_pd {
@@ -289,8 +292,25 @@ struct mlx4_roce_smac_vlan_info {
 	int update_vid;
 };
 
+struct mlx4_wqn_range {
+	int			base_wqn;
+	int			size;
+	int			refcount;
+	bool			dirty;
+	struct list_head	list;
+};
+
+struct mlx4_ib_rss {
+	unsigned int		base_qpn_tbl_sz;
+	u8			flags;
+	u8			rss_key[MLX4_EN_RSS_KEY_SIZE];
+};
+
 struct mlx4_ib_qp {
-	struct ib_qp		ibqp;
+	union {
+		struct ib_qp	ibqp;
+		struct ib_wq	ibwq;
+	};
 	struct mlx4_qp		mqp;
 	struct mlx4_buf		buf;
 
@@ -318,6 +338,7 @@ struct mlx4_ib_qp {
 	u8			sq_no_prefetch;
 	u8			state;
 	int			mlx_type;
+	u32			inl_recv_sz;
 	struct list_head	gid_list;
 	struct list_head	steering_rules;
 	struct mlx4_ib_buf	*sqp_proxy_rcv;
@@ -328,6 +349,10 @@ struct mlx4_ib_qp {
 	struct list_head	cq_recv_list;
 	struct list_head	cq_send_list;
 	struct counter_index	*counter_index;
+	struct mlx4_wqn_range	*wqn_range;
+	/* Number of RSS QP parents that uses this WQ */
+	u32			rss_usecnt;
+	struct mlx4_ib_rss	*rss_ctx;
 };
 
 struct mlx4_ib_srq {
@@ -623,6 +648,8 @@ struct mlx4_uverbs_ex_query_device_resp {
 	__u32 comp_mask;
 	__u32 response_length;
 	__u64 hca_core_clock_offset;
+	__u32 max_inl_recv_sz;
+	__u32 reserved;
 };
 
 static inline struct mlx4_ib_dev *to_mdev(struct ib_device *ibdev)
@@ -890,4 +917,17 @@ void mlx4_sched_ib_sl2vl_update_work(struct mlx4_ib_dev *ibdev,
 
 void mlx4_ib_sl2vl_update(struct mlx4_ib_dev *mdev, int port);
 
+struct ib_wq *mlx4_ib_create_wq(struct ib_pd *pd,
+				struct ib_wq_init_attr *init_attr,
+				struct ib_udata *udata);
+int mlx4_ib_destroy_wq(struct ib_wq *wq);
+int mlx4_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
+		      u32 wq_attr_mask, struct ib_udata *udata);
+
+struct ib_rwq_ind_table
+*mlx4_ib_create_rwq_ind_table(struct ib_device *device,
+			      struct ib_rwq_ind_table_init_attr *init_attr,
+			      struct ib_udata *udata);
+int mlx4_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table);
+
 #endif /* MLX4_IB_H */
diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c
index 75c0e6c5dd56..b6b33d99b0b4 100644
--- a/drivers/infiniband/hw/mlx4/qp.c
+++ b/drivers/infiniband/hw/mlx4/qp.c
@@ -36,7 +36,6 @@
 #include <net/ip.h>
 #include <linux/slab.h>
 #include <linux/netdevice.h>
-#include <linux/vmalloc.h>
 
 #include <rdma/ib_cache.h>
 #include <rdma/ib_pack.h>
@@ -53,6 +52,7 @@ static void mlx4_ib_lock_cqs(struct mlx4_ib_cq *send_cq,
 			     struct mlx4_ib_cq *recv_cq);
 static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq,
 			       struct mlx4_ib_cq *recv_cq);
+static int _mlx4_ib_modify_wq(struct ib_wq *ibwq, enum ib_wq_state new_state);
 
 enum {
 	MLX4_IB_ACK_REQ_FREQ	= 8,
@@ -116,6 +116,11 @@ static const __be32 mlx4_ib_opcode[] = {
 	[IB_WR_MASKED_ATOMIC_FETCH_AND_ADD]	= cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_FA),
 };
 
+enum mlx4_ib_source_type {
+	MLX4_IB_QP_SRC	= 0,
+	MLX4_IB_RWQ_SRC	= 1,
+};
+
 static struct mlx4_ib_sqp *to_msqp(struct mlx4_ib_qp *mqp)
 {
 	return container_of(mqp, struct mlx4_ib_sqp, qp);
@@ -145,8 +150,8 @@ static int is_sqp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
 	/* VF or PF -- proxy SQP */
 	if (mlx4_is_mfunc(dev->dev)) {
 		for (i = 0; i < dev->dev->caps.num_ports; i++) {
-			if (qp->mqp.qpn == dev->dev->caps.qp0_proxy[i] ||
-			    qp->mqp.qpn == dev->dev->caps.qp1_proxy[i]) {
+			if (qp->mqp.qpn == dev->dev->caps.spec_qps[i].qp0_proxy ||
+			    qp->mqp.qpn == dev->dev->caps.spec_qps[i].qp1_proxy) {
 				proxy_sqp = 1;
 				break;
 			}
@@ -173,7 +178,7 @@ static int is_qp0(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
 	/* VF or PF -- proxy QP0 */
 	if (mlx4_is_mfunc(dev->dev)) {
 		for (i = 0; i < dev->dev->caps.num_ports; i++) {
-			if (qp->mqp.qpn == dev->dev->caps.qp0_proxy[i]) {
+			if (qp->mqp.qpn == dev->dev->caps.spec_qps[i].qp0_proxy) {
 				proxy_qp0 = 1;
 				break;
 			}
@@ -330,6 +335,12 @@ static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type)
 	}
 }
 
+static void mlx4_ib_wq_event(struct mlx4_qp *qp, enum mlx4_event type)
+{
+	pr_warn_ratelimited("Unexpected event type %d on WQ 0x%06x. Events are not supported for WQs\n",
+			    type, qp->qpn);
+}
+
 static int send_wqe_overhead(enum mlx4_ib_qp_type type, u32 flags)
 {
 	/*
@@ -377,7 +388,8 @@ static int send_wqe_overhead(enum mlx4_ib_qp_type type, u32 flags)
 }
 
 static int set_rq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
-		       int is_user, int has_rq, struct mlx4_ib_qp *qp)
+		       int is_user, int has_rq, struct mlx4_ib_qp *qp,
+		       u32 inl_recv_sz)
 {
 	/* Sanity check RQ size before proceeding */
 	if (cap->max_recv_wr > dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE ||
@@ -385,18 +397,24 @@ static int set_rq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
 		return -EINVAL;
 
 	if (!has_rq) {
-		if (cap->max_recv_wr)
+		if (cap->max_recv_wr || inl_recv_sz)
 			return -EINVAL;
 
 		qp->rq.wqe_cnt = qp->rq.max_gs = 0;
 	} else {
+		u32 max_inl_recv_sz = dev->dev->caps.max_rq_sg *
+			sizeof(struct mlx4_wqe_data_seg);
+		u32 wqe_size;
+
 		/* HW requires >= 1 RQ entry with >= 1 gather entry */
-		if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge))
+		if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge ||
+				inl_recv_sz > max_inl_recv_sz))
 			return -EINVAL;
 
 		qp->rq.wqe_cnt	 = roundup_pow_of_two(max(1U, cap->max_recv_wr));
 		qp->rq.max_gs	 = roundup_pow_of_two(max(1U, cap->max_recv_sge));
-		qp->rq.wqe_shift = ilog2(qp->rq.max_gs * sizeof (struct mlx4_wqe_data_seg));
+		wqe_size = qp->rq.max_gs * sizeof(struct mlx4_wqe_data_seg);
+		qp->rq.wqe_shift = ilog2(max_t(u32, wqe_size, inl_recv_sz));
 	}
 
 	/* leave userspace return values as they were, so as not to break ABI */
@@ -614,8 +632,8 @@ static int qp0_enabled_vf(struct mlx4_dev *dev, int qpn)
 {
 	int i;
 	for (i = 0; i < dev->caps.num_ports; i++) {
-		if (qpn == dev->caps.qp0_proxy[i])
-			return !!dev->caps.qp0_qkey[i];
+		if (qpn == dev->caps.spec_qps[i].qp0_proxy)
+			return !!dev->caps.spec_qps[i].qp0_qkey;
 	}
 	return 0;
 }
@@ -632,7 +650,300 @@ static void mlx4_ib_free_qp_counter(struct mlx4_ib_dev *dev,
 	qp->counter_index = NULL;
 }
 
+static int set_qp_rss(struct mlx4_ib_dev *dev, struct mlx4_ib_rss *rss_ctx,
+		      struct ib_qp_init_attr *init_attr,
+		      struct mlx4_ib_create_qp_rss *ucmd)
+{
+	rss_ctx->base_qpn_tbl_sz = init_attr->rwq_ind_tbl->ind_tbl[0]->wq_num |
+		(init_attr->rwq_ind_tbl->log_ind_tbl_size << 24);
+
+	if ((ucmd->rx_hash_function == MLX4_IB_RX_HASH_FUNC_TOEPLITZ) &&
+	    (dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS_TOP)) {
+		memcpy(rss_ctx->rss_key, ucmd->rx_hash_key,
+		       MLX4_EN_RSS_KEY_SIZE);
+	} else {
+		pr_debug("RX Hash function is not supported\n");
+		return (-EOPNOTSUPP);
+	}
+
+	if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_IPV4) &&
+	    (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_IPV4)) {
+		rss_ctx->flags = MLX4_RSS_IPV4;
+	} else if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_IPV4) ||
+		   (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_IPV4)) {
+		pr_debug("RX Hash fields_mask is not supported - both IPv4 SRC and DST must be set\n");
+		return (-EOPNOTSUPP);
+	}
+
+	if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_IPV6) &&
+	    (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_IPV6)) {
+		rss_ctx->flags |= MLX4_RSS_IPV6;
+	} else if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_IPV6) ||
+		   (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_IPV6)) {
+		pr_debug("RX Hash fields_mask is not supported - both IPv6 SRC and DST must be set\n");
+		return (-EOPNOTSUPP);
+	}
+
+	if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_PORT_UDP) &&
+	    (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_PORT_UDP)) {
+		if (!(dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_UDP_RSS)) {
+			pr_debug("RX Hash fields_mask for UDP is not supported\n");
+			return (-EOPNOTSUPP);
+		}
+
+		if (rss_ctx->flags & MLX4_RSS_IPV4) {
+			rss_ctx->flags |= MLX4_RSS_UDP_IPV4;
+		} else if (rss_ctx->flags & MLX4_RSS_IPV6) {
+			rss_ctx->flags |= MLX4_RSS_UDP_IPV6;
+		} else {
+			pr_debug("RX Hash fields_mask is not supported - UDP must be set with IPv4 or IPv6\n");
+			return (-EOPNOTSUPP);
+		}
+	} else if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_PORT_UDP) ||
+		   (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_PORT_UDP)) {
+		pr_debug("RX Hash fields_mask is not supported - both UDP SRC and DST must be set\n");
+		return (-EOPNOTSUPP);
+	}
+
+	if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_PORT_TCP) &&
+	    (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_PORT_TCP)) {
+		if (rss_ctx->flags & MLX4_RSS_IPV4) {
+			rss_ctx->flags |= MLX4_RSS_TCP_IPV4;
+		} else if (rss_ctx->flags & MLX4_RSS_IPV6) {
+			rss_ctx->flags |= MLX4_RSS_TCP_IPV6;
+		} else {
+			pr_debug("RX Hash fields_mask is not supported - TCP must be set with IPv4 or IPv6\n");
+			return (-EOPNOTSUPP);
+		}
+
+	} else if ((ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_SRC_PORT_TCP) ||
+		   (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_DST_PORT_TCP)) {
+		pr_debug("RX Hash fields_mask is not supported - both TCP SRC and DST must be set\n");
+		return (-EOPNOTSUPP);
+	}
+
+	return 0;
+}
+
+static int create_qp_rss(struct mlx4_ib_dev *dev, struct ib_pd *ibpd,
+			 struct ib_qp_init_attr *init_attr,
+			 struct mlx4_ib_create_qp_rss *ucmd,
+			 struct mlx4_ib_qp *qp)
+{
+	int qpn;
+	int err;
+
+	qp->mqp.usage = MLX4_RES_USAGE_USER_VERBS;
+
+	err = mlx4_qp_reserve_range(dev->dev, 1, 1, &qpn, 0, qp->mqp.usage);
+	if (err)
+		return err;
+
+	err = mlx4_qp_alloc(dev->dev, qpn, &qp->mqp);
+	if (err)
+		goto err_qpn;
+
+	mutex_init(&qp->mutex);
+
+	INIT_LIST_HEAD(&qp->gid_list);
+	INIT_LIST_HEAD(&qp->steering_rules);
+
+	qp->mlx4_ib_qp_type = MLX4_IB_QPT_RAW_PACKET;
+	qp->state = IB_QPS_RESET;
+
+	/* Set dummy send resources to be compatible with HV and PRM */
+	qp->sq_no_prefetch = 1;
+	qp->sq.wqe_cnt = 1;
+	qp->sq.wqe_shift = MLX4_IB_MIN_SQ_STRIDE;
+	qp->buf_size = qp->sq.wqe_cnt << MLX4_IB_MIN_SQ_STRIDE;
+	qp->mtt = (to_mqp(
+		   (struct ib_qp *)init_attr->rwq_ind_tbl->ind_tbl[0]))->mtt;
+
+	qp->rss_ctx = kzalloc(sizeof(*qp->rss_ctx), GFP_KERNEL);
+	if (!qp->rss_ctx) {
+		err = -ENOMEM;
+		goto err_qp_alloc;
+	}
+
+	err = set_qp_rss(dev, qp->rss_ctx, init_attr, ucmd);
+	if (err)
+		goto err;
+
+	return 0;
+
+err:
+	kfree(qp->rss_ctx);
+
+err_qp_alloc:
+	mlx4_qp_remove(dev->dev, &qp->mqp);
+	mlx4_qp_free(dev->dev, &qp->mqp);
+
+err_qpn:
+	mlx4_qp_release_range(dev->dev, qpn, 1);
+	return err;
+}
+
+static struct ib_qp *_mlx4_ib_create_qp_rss(struct ib_pd *pd,
+					    struct ib_qp_init_attr *init_attr,
+					    struct ib_udata *udata)
+{
+	struct mlx4_ib_qp *qp;
+	struct mlx4_ib_create_qp_rss ucmd = {};
+	size_t required_cmd_sz;
+	int err;
+
+	if (!udata) {
+		pr_debug("RSS QP with NULL udata\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	if (udata->outlen)
+		return ERR_PTR(-EOPNOTSUPP);
+
+	required_cmd_sz = offsetof(typeof(ucmd), reserved1) +
+					sizeof(ucmd.reserved1);
+	if (udata->inlen < required_cmd_sz) {
+		pr_debug("invalid inlen\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) {
+		pr_debug("copy failed\n");
+		return ERR_PTR(-EFAULT);
+	}
+
+	if (memchr_inv(ucmd.reserved, 0, sizeof(ucmd.reserved)))
+		return ERR_PTR(-EOPNOTSUPP);
+
+	if (ucmd.comp_mask || ucmd.reserved1)
+		return ERR_PTR(-EOPNOTSUPP);
+
+	if (udata->inlen > sizeof(ucmd) &&
+	    !ib_is_udata_cleared(udata, sizeof(ucmd),
+				 udata->inlen - sizeof(ucmd))) {
+		pr_debug("inlen is not supported\n");
+		return ERR_PTR(-EOPNOTSUPP);
+	}
+
+	if (init_attr->qp_type != IB_QPT_RAW_PACKET) {
+		pr_debug("RSS QP with unsupported QP type %d\n",
+			 init_attr->qp_type);
+		return ERR_PTR(-EOPNOTSUPP);
+	}
+
+	if (init_attr->create_flags) {
+		pr_debug("RSS QP doesn't support create flags\n");
+		return ERR_PTR(-EOPNOTSUPP);
+	}
+
+	if (init_attr->send_cq || init_attr->cap.max_send_wr) {
+		pr_debug("RSS QP with unsupported send attributes\n");
+		return ERR_PTR(-EOPNOTSUPP);
+	}
+
+	qp = kzalloc(sizeof(*qp), GFP_KERNEL);
+	if (!qp)
+		return ERR_PTR(-ENOMEM);
+
+	qp->pri.vid = 0xFFFF;
+	qp->alt.vid = 0xFFFF;
+
+	err = create_qp_rss(to_mdev(pd->device), pd, init_attr, &ucmd, qp);
+	if (err) {
+		kfree(qp);
+		return ERR_PTR(err);
+	}
+
+	qp->ibqp.qp_num = qp->mqp.qpn;
+
+	return &qp->ibqp;
+}
+
+/*
+ * This function allocates a WQN from a range which is consecutive and aligned
+ * to its size. In case the range is full, then it creates a new range and
+ * allocates WQN from it. The new range will be used for following allocations.
+ */
+static int mlx4_ib_alloc_wqn(struct mlx4_ib_ucontext *context,
+			     struct mlx4_ib_qp *qp, int range_size, int *wqn)
+{
+	struct mlx4_ib_dev *dev = to_mdev(context->ibucontext.device);
+	struct mlx4_wqn_range *range;
+	int err = 0;
+
+	mutex_lock(&context->wqn_ranges_mutex);
+
+	range = list_first_entry_or_null(&context->wqn_ranges_list,
+					 struct mlx4_wqn_range, list);
+
+	if (!range || (range->refcount == range->size) || range->dirty) {
+		range = kzalloc(sizeof(*range), GFP_KERNEL);
+		if (!range) {
+			err = -ENOMEM;
+			goto out;
+		}
+
+		err = mlx4_qp_reserve_range(dev->dev, range_size,
+					    range_size, &range->base_wqn, 0,
+					    qp->mqp.usage);
+		if (err) {
+			kfree(range);
+			goto out;
+		}
+
+		range->size = range_size;
+		list_add(&range->list, &context->wqn_ranges_list);
+	} else if (range_size != 1) {
+		/*
+		 * Requesting a new range (>1) when last range is still open, is
+		 * not valid.
+		 */
+		err = -EINVAL;
+		goto out;
+	}
+
+	qp->wqn_range = range;
+
+	*wqn = range->base_wqn + range->refcount;
+
+	range->refcount++;
+
+out:
+	mutex_unlock(&context->wqn_ranges_mutex);
+
+	return err;
+}
+
+static void mlx4_ib_release_wqn(struct mlx4_ib_ucontext *context,
+				struct mlx4_ib_qp *qp, bool dirty_release)
+{
+	struct mlx4_ib_dev *dev = to_mdev(context->ibucontext.device);
+	struct mlx4_wqn_range *range;
+
+	mutex_lock(&context->wqn_ranges_mutex);
+
+	range = qp->wqn_range;
+
+	range->refcount--;
+	if (!range->refcount) {
+		mlx4_qp_release_range(dev->dev, range->base_wqn,
+				      range->size);
+		list_del(&range->list);
+		kfree(range);
+	} else if (dirty_release) {
+	/*
+	 * A range which one of its WQNs is destroyed, won't be able to be
+	 * reused for further WQN allocations.
+	 * The next created WQ will allocate a new range.
+	 */
+		range->dirty = 1;
+	}
+
+	mutex_unlock(&context->wqn_ranges_mutex);
+}
+
 static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
+			    enum mlx4_ib_source_type src,
 			    struct ib_qp_init_attr *init_attr,
 			    struct ib_udata *udata, int sqpn,
 			    struct mlx4_ib_qp **caller_qp)
@@ -645,6 +956,7 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
 	enum mlx4_ib_qp_type qp_type = (enum mlx4_ib_qp_type) init_attr->qp_type;
 	struct mlx4_ib_cq *mcq;
 	unsigned long flags;
+	int range_size = 0;
 
 	/* When tunneling special qps, we use a plain UD qp */
 	if (sqpn) {
@@ -719,26 +1031,70 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
 	if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
 		qp->sq_signal_bits = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
 
-	err = set_rq_size(dev, &init_attr->cap, !!pd->uobject, qp_has_rq(init_attr), qp);
-	if (err)
-		goto err;
 
 	if (pd->uobject) {
-		struct mlx4_ib_create_qp ucmd;
+		union {
+			struct mlx4_ib_create_qp qp;
+			struct mlx4_ib_create_wq wq;
+		} ucmd;
+		size_t copy_len;
 
-		if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
+		copy_len = (src == MLX4_IB_QP_SRC) ?
+			   sizeof(struct mlx4_ib_create_qp) :
+			   min(sizeof(struct mlx4_ib_create_wq), udata->inlen);
+
+		if (ib_copy_from_udata(&ucmd, udata, copy_len)) {
 			err = -EFAULT;
 			goto err;
 		}
 
-		qp->sq_no_prefetch = ucmd.sq_no_prefetch;
+		if (src == MLX4_IB_RWQ_SRC) {
+			if (ucmd.wq.comp_mask || ucmd.wq.reserved[0] ||
+			    ucmd.wq.reserved[1] || ucmd.wq.reserved[2]) {
+				pr_debug("user command isn't supported\n");
+				err = -EOPNOTSUPP;
+				goto err;
+			}
+
+			if (ucmd.wq.log_range_size >
+			    ilog2(dev->dev->caps.max_rss_tbl_sz)) {
+				pr_debug("WQN range size must be equal or smaller than %d\n",
+					 dev->dev->caps.max_rss_tbl_sz);
+				err = -EOPNOTSUPP;
+				goto err;
+			}
+			range_size = 1 << ucmd.wq.log_range_size;
+		} else {
+			qp->inl_recv_sz = ucmd.qp.inl_recv_sz;
+		}
 
-		err = set_user_sq_size(dev, qp, &ucmd);
+		err = set_rq_size(dev, &init_attr->cap, !!pd->uobject,
+				  qp_has_rq(init_attr), qp, qp->inl_recv_sz);
 		if (err)
 			goto err;
 
-		qp->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr,
-				       qp->buf_size, 0, 0);
+		if (src == MLX4_IB_QP_SRC) {
+			qp->sq_no_prefetch = ucmd.qp.sq_no_prefetch;
+
+			err = set_user_sq_size(dev, qp,
+					       (struct mlx4_ib_create_qp *)
+					       &ucmd);
+			if (err)
+				goto err;
+		} else {
+			qp->sq_no_prefetch = 1;
+			qp->sq.wqe_cnt = 1;
+			qp->sq.wqe_shift = MLX4_IB_MIN_SQ_STRIDE;
+			/* Allocated buffer expects to have at least that SQ
+			 * size.
+			 */
+			qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
+				(qp->sq.wqe_cnt << qp->sq.wqe_shift);
+		}
+
+		qp->umem = ib_umem_get(pd->uobject->context,
+				(src == MLX4_IB_QP_SRC) ? ucmd.qp.buf_addr :
+				ucmd.wq.buf_addr, qp->buf_size, 0, 0);
 		if (IS_ERR(qp->umem)) {
 			err = PTR_ERR(qp->umem);
 			goto err;
@@ -755,11 +1111,18 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
 
 		if (qp_has_rq(init_attr)) {
 			err = mlx4_ib_db_map_user(to_mucontext(pd->uobject->context),
-						  ucmd.db_addr, &qp->db);
+				(src == MLX4_IB_QP_SRC) ? ucmd.qp.db_addr :
+				ucmd.wq.db_addr, &qp->db);
 			if (err)
 				goto err_mtt;
 		}
+		qp->mqp.usage = MLX4_RES_USAGE_USER_VERBS;
 	} else {
+		err = set_rq_size(dev, &init_attr->cap, !!pd->uobject,
+				  qp_has_rq(init_attr), qp, 0);
+		if (err)
+			goto err;
+
 		qp->sq_no_prefetch = 0;
 
 		if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)
@@ -812,20 +1175,15 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
 		if (err)
 			goto err_mtt;
 
-		qp->sq.wrid = kmalloc_array(qp->sq.wqe_cnt, sizeof(u64),
-					GFP_KERNEL | __GFP_NOWARN);
-		if (!qp->sq.wrid)
-			qp->sq.wrid = __vmalloc(qp->sq.wqe_cnt * sizeof(u64),
-						GFP_KERNEL, PAGE_KERNEL);
-		qp->rq.wrid = kmalloc_array(qp->rq.wqe_cnt, sizeof(u64),
-					GFP_KERNEL | __GFP_NOWARN);
-		if (!qp->rq.wrid)
-			qp->rq.wrid = __vmalloc(qp->rq.wqe_cnt * sizeof(u64),
-						GFP_KERNEL, PAGE_KERNEL);
+		qp->sq.wrid = kvmalloc_array(qp->sq.wqe_cnt,
+					     sizeof(u64), GFP_KERNEL);
+		qp->rq.wrid = kvmalloc_array(qp->rq.wqe_cnt,
+					     sizeof(u64), GFP_KERNEL);
 		if (!qp->sq.wrid || !qp->rq.wrid) {
 			err = -ENOMEM;
 			goto err_wrid;
 		}
+		qp->mqp.usage = MLX4_RES_USAGE_DRIVER;
 	}
 
 	if (sqpn) {
@@ -836,6 +1194,11 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
 				goto err_wrid;
 			}
 		}
+	} else if (src == MLX4_IB_RWQ_SRC) {
+		err = mlx4_ib_alloc_wqn(to_mucontext(pd->uobject->context), qp,
+					range_size, &qpn);
+		if (err)
+			goto err_wrid;
 	} else {
 		/* Raw packet QPNs may not have bits 6,7 set in their qp_num;
 		 * otherwise, the WQE BlueFlame setup flow wrongly causes
@@ -845,13 +1208,14 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
 						    (init_attr->cap.max_send_wr ?
 						     MLX4_RESERVE_ETH_BF_QP : 0) |
 						    (init_attr->cap.max_recv_wr ?
-						     MLX4_RESERVE_A0_QP : 0));
+						     MLX4_RESERVE_A0_QP : 0),
+						    qp->mqp.usage);
 		else
 			if (qp->flags & MLX4_IB_QP_NETIF)
 				err = mlx4_ib_steer_qp_alloc(dev, 1, &qpn);
 			else
 				err = mlx4_qp_reserve_range(dev->dev, 1, 1,
-							    &qpn, 0);
+							    &qpn, 0, qp->mqp.usage);
 		if (err)
 			goto err_proxy;
 	}
@@ -873,7 +1237,9 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
 	 */
 	qp->doorbell_qpn = swab32(qp->mqp.qpn << 8);
 
-	qp->mqp.event = mlx4_ib_qp_event;
+	qp->mqp.event = (src == MLX4_IB_QP_SRC) ? mlx4_ib_qp_event :
+						  mlx4_ib_wq_event;
+
 	if (!*caller_qp)
 		*caller_qp = qp;
 
@@ -900,6 +1266,9 @@ err_qpn:
 	if (!sqpn) {
 		if (qp->flags & MLX4_IB_QP_NETIF)
 			mlx4_ib_steer_qp_free(dev, qpn, 1);
+		else if (src == MLX4_IB_RWQ_SRC)
+			mlx4_ib_release_wqn(to_mucontext(pd->uobject->context),
+					    qp, 0);
 		else
 			mlx4_qp_release_range(dev->dev, qpn, 1);
 	}
@@ -998,7 +1367,7 @@ static struct mlx4_ib_pd *get_pd(struct mlx4_ib_qp *qp)
 		return to_mpd(qp->ibqp.pd);
 }
 
-static void get_cqs(struct mlx4_ib_qp *qp,
+static void get_cqs(struct mlx4_ib_qp *qp, enum mlx4_ib_source_type src,
 		    struct mlx4_ib_cq **send_cq, struct mlx4_ib_cq **recv_cq)
 {
 	switch (qp->ibqp.qp_type) {
@@ -1011,14 +1380,46 @@ static void get_cqs(struct mlx4_ib_qp *qp,
 		*recv_cq = *send_cq;
 		break;
 	default:
-		*send_cq = to_mcq(qp->ibqp.send_cq);
-		*recv_cq = to_mcq(qp->ibqp.recv_cq);
+		*recv_cq = (src == MLX4_IB_QP_SRC) ? to_mcq(qp->ibqp.recv_cq) :
+						     to_mcq(qp->ibwq.cq);
+		*send_cq = (src == MLX4_IB_QP_SRC) ? to_mcq(qp->ibqp.send_cq) :
+						     *recv_cq;
 		break;
 	}
 }
 
+static void destroy_qp_rss(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
+{
+	if (qp->state != IB_QPS_RESET) {
+		int i;
+
+		for (i = 0; i < (1 << qp->ibqp.rwq_ind_tbl->log_ind_tbl_size);
+		     i++) {
+			struct ib_wq *ibwq = qp->ibqp.rwq_ind_tbl->ind_tbl[i];
+			struct mlx4_ib_qp *wq =	to_mqp((struct ib_qp *)ibwq);
+
+			mutex_lock(&wq->mutex);
+
+			wq->rss_usecnt--;
+
+			mutex_unlock(&wq->mutex);
+		}
+
+		if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state),
+				   MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp))
+			pr_warn("modify QP %06x to RESET failed.\n",
+				qp->mqp.qpn);
+	}
+
+	mlx4_qp_remove(dev->dev, &qp->mqp);
+	mlx4_qp_free(dev->dev, &qp->mqp);
+	mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1);
+	del_gid_entries(qp);
+	kfree(qp->rss_ctx);
+}
+
 static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
-			      int is_user)
+			      enum mlx4_ib_source_type src, int is_user)
 {
 	struct mlx4_ib_cq *send_cq, *recv_cq;
 	unsigned long flags;
@@ -1051,7 +1452,7 @@ static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
 		}
 	}
 
-	get_cqs(qp, &send_cq, &recv_cq);
+	get_cqs(qp, src, &send_cq, &recv_cq);
 
 	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
 	mlx4_ib_lock_cqs(send_cq, recv_cq);
@@ -1077,6 +1478,9 @@ static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
 	if (!is_sqp(dev, qp) && !is_tunnel_qp(dev, qp)) {
 		if (qp->flags & MLX4_IB_QP_NETIF)
 			mlx4_ib_steer_qp_free(dev, qp->mqp.qpn, 1);
+		else if (src == MLX4_IB_RWQ_SRC)
+			mlx4_ib_release_wqn(to_mucontext(
+					    qp->ibwq.uobject->context), qp, 1);
 		else
 			mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1);
 	}
@@ -1084,9 +1488,12 @@ static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
 	mlx4_mtt_cleanup(dev->dev, &qp->mtt);
 
 	if (is_user) {
-		if (qp->rq.wqe_cnt)
-			mlx4_ib_db_unmap_user(to_mucontext(qp->ibqp.uobject->context),
-					      &qp->db);
+		if (qp->rq.wqe_cnt) {
+			struct mlx4_ib_ucontext *mcontext = !src ?
+				to_mucontext(qp->ibqp.uobject->context) :
+				to_mucontext(qp->ibwq.uobject->context);
+			mlx4_ib_db_unmap_user(mcontext, &qp->db);
+		}
 		ib_umem_release(qp->umem);
 	} else {
 		kvfree(qp->sq.wrid);
@@ -1114,9 +1521,9 @@ static u32 get_sqp_num(struct mlx4_ib_dev *dev, struct ib_qp_init_attr *attr)
 	}
 	/* PF or VF -- creating proxies */
 	if (attr->qp_type == IB_QPT_SMI)
-		return dev->dev->caps.qp0_proxy[attr->port_num - 1];
+		return dev->dev->caps.spec_qps[attr->port_num - 1].qp0_proxy;
 	else
-		return dev->dev->caps.qp1_proxy[attr->port_num - 1];
+		return dev->dev->caps.spec_qps[attr->port_num - 1].qp1_proxy;
 }
 
 static struct ib_qp *_mlx4_ib_create_qp(struct ib_pd *pd,
@@ -1128,6 +1535,9 @@ static struct ib_qp *_mlx4_ib_create_qp(struct ib_pd *pd,
 	int sup_u_create_flags = MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;
 	u16 xrcdn = 0;
 
+	if (init_attr->rwq_ind_tbl)
+		return _mlx4_ib_create_qp_rss(pd, init_attr, udata);
+
 	/*
 	 * We only support LSO, vendor flag1, and multicast loopback blocking,
 	 * and only for kernel UD QPs.
@@ -1182,8 +1592,8 @@ static struct ib_qp *_mlx4_ib_create_qp(struct ib_pd *pd,
 		/* fall through */
 	case IB_QPT_UD:
 	{
-		err = create_qp_common(to_mdev(pd->device), pd, init_attr,
-				       udata, 0, &qp);
+		err = create_qp_common(to_mdev(pd->device), pd,	MLX4_IB_QP_SRC,
+				       init_attr, udata, 0, &qp);
 		if (err) {
 			kfree(qp);
 			return ERR_PTR(err);
@@ -1203,7 +1613,9 @@ static struct ib_qp *_mlx4_ib_create_qp(struct ib_pd *pd,
 		if (udata)
 			return ERR_PTR(-EINVAL);
 		if (init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI) {
-			int res = mlx4_qp_reserve_range(to_mdev(pd->device)->dev, 1, 1, &sqpn, 0);
+			int res = mlx4_qp_reserve_range(to_mdev(pd->device)->dev,
+							1, 1, &sqpn, 0,
+							MLX4_RES_USAGE_DRIVER);
 
 			if (res)
 				return ERR_PTR(res);
@@ -1211,8 +1623,8 @@ static struct ib_qp *_mlx4_ib_create_qp(struct ib_pd *pd,
 			sqpn = get_sqp_num(to_mdev(pd->device), init_attr);
 		}
 
-		err = create_qp_common(to_mdev(pd->device), pd, init_attr, udata,
-				       sqpn, &qp);
+		err = create_qp_common(to_mdev(pd->device), pd, MLX4_IB_QP_SRC,
+				       init_attr, udata, sqpn, &qp);
 		if (err)
 			return ERR_PTR(err);
 
@@ -1267,7 +1679,6 @@ static int _mlx4_ib_destroy_qp(struct ib_qp *qp)
 {
 	struct mlx4_ib_dev *dev = to_mdev(qp->device);
 	struct mlx4_ib_qp *mqp = to_mqp(qp);
-	struct mlx4_ib_pd *pd;
 
 	if (is_qp0(dev, mqp))
 		mlx4_CLOSE_PORT(dev->dev, mqp->port);
@@ -1282,8 +1693,14 @@ static int _mlx4_ib_destroy_qp(struct ib_qp *qp)
 	if (mqp->counter_index)
 		mlx4_ib_free_qp_counter(dev, mqp);
 
-	pd = get_pd(mqp);
-	destroy_qp_common(dev, mqp, !!pd->ibpd.uobject);
+	if (qp->rwq_ind_tbl) {
+		destroy_qp_rss(dev, mqp);
+	} else {
+		struct mlx4_ib_pd *pd;
+
+		pd = get_pd(mqp);
+		destroy_qp_common(dev, mqp, MLX4_IB_QP_SRC, !!pd->ibpd.uobject);
+	}
 
 	if (is_sqp(dev, mqp))
 		kfree(to_msqp(mqp));
@@ -1566,7 +1983,7 @@ static int create_qp_lb_counter(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
 	    !(dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_LB_SRC_CHK))
 		return 0;
 
-	err = mlx4_counter_alloc(dev->dev, &tmp_idx);
+	err = mlx4_counter_alloc(dev->dev, &tmp_idx, MLX4_RES_USAGE_DRIVER);
 	if (err)
 		return err;
 
@@ -1606,12 +2023,119 @@ static u8 gid_type_to_qpc(enum ib_gid_type gid_type)
 	}
 }
 
-static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
+/*
+ * Go over all RSS QP's childes (WQs) and apply their HW state according to
+ * their logic state if the RSS QP is the first RSS QP associated for the WQ.
+ */
+static int bringup_rss_rwqs(struct ib_rwq_ind_table *ind_tbl, u8 port_num)
+{
+	int err = 0;
+	int i;
+
+	for (i = 0; i < (1 << ind_tbl->log_ind_tbl_size); i++) {
+		struct ib_wq *ibwq = ind_tbl->ind_tbl[i];
+		struct mlx4_ib_qp *wq = to_mqp((struct ib_qp *)ibwq);
+
+		mutex_lock(&wq->mutex);
+
+		/* Mlx4_ib restrictions:
+		 * WQ's is associated to a port according to the RSS QP it is
+		 * associates to.
+		 * In case the WQ is associated to a different port by another
+		 * RSS QP, return a failure.
+		 */
+		if ((wq->rss_usecnt > 0) && (wq->port != port_num)) {
+			err = -EINVAL;
+			mutex_unlock(&wq->mutex);
+			break;
+		}
+		wq->port = port_num;
+		if ((wq->rss_usecnt == 0) && (ibwq->state == IB_WQS_RDY)) {
+			err = _mlx4_ib_modify_wq(ibwq, IB_WQS_RDY);
+			if (err) {
+				mutex_unlock(&wq->mutex);
+				break;
+			}
+		}
+		wq->rss_usecnt++;
+
+		mutex_unlock(&wq->mutex);
+	}
+
+	if (i && err) {
+		int j;
+
+		for (j = (i - 1); j >= 0; j--) {
+			struct ib_wq *ibwq = ind_tbl->ind_tbl[j];
+			struct mlx4_ib_qp *wq = to_mqp((struct ib_qp *)ibwq);
+
+			mutex_lock(&wq->mutex);
+
+			if ((wq->rss_usecnt == 1) &&
+			    (ibwq->state == IB_WQS_RDY))
+				if (_mlx4_ib_modify_wq(ibwq, IB_WQS_RESET))
+					pr_warn("failed to reverse WQN=0x%06x\n",
+						ibwq->wq_num);
+			wq->rss_usecnt--;
+
+			mutex_unlock(&wq->mutex);
+		}
+	}
+
+	return err;
+}
+
+static void bring_down_rss_rwqs(struct ib_rwq_ind_table *ind_tbl)
+{
+	int i;
+
+	for (i = 0; i < (1 << ind_tbl->log_ind_tbl_size); i++) {
+		struct ib_wq *ibwq = ind_tbl->ind_tbl[i];
+		struct mlx4_ib_qp *wq = to_mqp((struct ib_qp *)ibwq);
+
+		mutex_lock(&wq->mutex);
+
+		if ((wq->rss_usecnt == 1) && (ibwq->state == IB_WQS_RDY))
+			if (_mlx4_ib_modify_wq(ibwq, IB_WQS_RESET))
+				pr_warn("failed to reverse WQN=%x\n",
+					ibwq->wq_num);
+		wq->rss_usecnt--;
+
+		mutex_unlock(&wq->mutex);
+	}
+}
+
+static void fill_qp_rss_context(struct mlx4_qp_context *context,
+				struct mlx4_ib_qp *qp)
+{
+	struct mlx4_rss_context *rss_context;
+
+	rss_context = (void *)context + offsetof(struct mlx4_qp_context,
+			pri_path) + MLX4_RSS_OFFSET_IN_QPC_PRI_PATH;
+
+	rss_context->base_qpn = cpu_to_be32(qp->rss_ctx->base_qpn_tbl_sz);
+	rss_context->default_qpn =
+		cpu_to_be32(qp->rss_ctx->base_qpn_tbl_sz & 0xffffff);
+	if (qp->rss_ctx->flags & (MLX4_RSS_UDP_IPV4 | MLX4_RSS_UDP_IPV6))
+		rss_context->base_qpn_udp = rss_context->default_qpn;
+	rss_context->flags = qp->rss_ctx->flags;
+	/* Currently support just toeplitz */
+	rss_context->hash_fn = MLX4_RSS_HASH_TOP;
+
+	memcpy(rss_context->rss_key, qp->rss_ctx->rss_key,
+	       MLX4_EN_RSS_KEY_SIZE);
+}
+
+static int __mlx4_ib_modify_qp(void *src, enum mlx4_ib_source_type src_type,
 			       const struct ib_qp_attr *attr, int attr_mask,
 			       enum ib_qp_state cur_state, enum ib_qp_state new_state)
 {
-	struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
-	struct mlx4_ib_qp *qp = to_mqp(ibqp);
+	struct ib_uobject *ibuobject;
+	struct ib_srq  *ibsrq;
+	struct ib_rwq_ind_table *rwq_ind_tbl;
+	enum ib_qp_type qp_type;
+	struct mlx4_ib_dev *dev;
+	struct mlx4_ib_qp *qp;
 	struct mlx4_ib_pd *pd;
 	struct mlx4_ib_cq *send_cq, *recv_cq;
 	struct mlx4_qp_context *context;
@@ -1621,6 +2145,30 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
 	int err = -EINVAL;
 	int counter_index;
 
+	if (src_type == MLX4_IB_RWQ_SRC) {
+		struct ib_wq *ibwq;
+
+		ibwq	    = (struct ib_wq *)src;
+		ibuobject   = ibwq->uobject;
+		ibsrq	    = NULL;
+		rwq_ind_tbl = NULL;
+		qp_type     = IB_QPT_RAW_PACKET;
+		qp	    = to_mqp((struct ib_qp *)ibwq);
+		dev	    = to_mdev(ibwq->device);
+		pd	    = to_mpd(ibwq->pd);
+	} else {
+		struct ib_qp *ibqp;
+
+		ibqp	    = (struct ib_qp *)src;
+		ibuobject   = ibqp->uobject;
+		ibsrq	    = ibqp->srq;
+		rwq_ind_tbl = ibqp->rwq_ind_tbl;
+		qp_type     = ibqp->qp_type;
+		qp	    = to_mqp(ibqp);
+		dev	    = to_mdev(ibqp->device);
+		pd	    = get_pd(qp);
+	}
+
 	/* APM is not supported under RoCE */
 	if (attr_mask & IB_QP_ALT_PATH &&
 	    rdma_port_get_link_layer(&dev->ib_dev, qp->port) ==
@@ -1634,6 +2182,11 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
 	context->flags = cpu_to_be32((to_mlx4_state(new_state) << 28) |
 				     (to_mlx4_st(dev, qp->mlx4_ib_qp_type) << 16));
 
+	if (rwq_ind_tbl) {
+		fill_qp_rss_context(context, qp);
+		context->flags |= cpu_to_be32(1 << MLX4_RSS_QPC_FLAG_OFFSET);
+	}
+
 	if (!(attr_mask & IB_QP_PATH_MIG_STATE))
 		context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11);
 	else {
@@ -1651,11 +2204,14 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
 		}
 	}
 
-	if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI)
+	if (qp->inl_recv_sz)
+		context->param3 |= cpu_to_be32(1 << 25);
+
+	if (qp_type == IB_QPT_GSI || qp_type == IB_QPT_SMI)
 		context->mtu_msgmax = (IB_MTU_4096 << 5) | 11;
-	else if (ibqp->qp_type == IB_QPT_RAW_PACKET)
+	else if (qp_type == IB_QPT_RAW_PACKET)
 		context->mtu_msgmax = (MLX4_RAW_QP_MTU << 5) | MLX4_RAW_QP_MSGMAX;
-	else if (ibqp->qp_type == IB_QPT_UD) {
+	else if (qp_type == IB_QPT_UD) {
 		if (qp->flags & MLX4_IB_QP_LSO)
 			context->mtu_msgmax = (IB_MTU_4096 << 5) |
 					      ilog2(dev->dev->caps.max_gso_sz);
@@ -1671,9 +2227,11 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
 			ilog2(dev->dev->caps.max_msg_sz);
 	}
 
-	if (qp->rq.wqe_cnt)
-		context->rq_size_stride = ilog2(qp->rq.wqe_cnt) << 3;
-	context->rq_size_stride |= qp->rq.wqe_shift - 4;
+	if (!rwq_ind_tbl) { /* PRM RSS receive side should be left zeros */
+		if (qp->rq.wqe_cnt)
+			context->rq_size_stride = ilog2(qp->rq.wqe_cnt) << 3;
+		context->rq_size_stride |= qp->rq.wqe_shift - 4;
+	}
 
 	if (qp->sq.wqe_cnt)
 		context->sq_size_stride = ilog2(qp->sq.wqe_cnt) << 3;
@@ -1685,14 +2243,15 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
 	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
 		context->sq_size_stride |= !!qp->sq_no_prefetch << 7;
 		context->xrcd = cpu_to_be32((u32) qp->xrcdn);
-		if (ibqp->qp_type == IB_QPT_RAW_PACKET)
+		if (qp_type == IB_QPT_RAW_PACKET)
 			context->param3 |= cpu_to_be32(1 << 30);
 	}
 
-	if (qp->ibqp.uobject)
+	if (ibuobject)
 		context->usr_page = cpu_to_be32(
 			mlx4_to_hw_uar_index(dev->dev,
-					     to_mucontext(ibqp->uobject->context)->uar.index));
+					     to_mucontext(ibuobject->context)
+					     ->uar.index));
 	else
 		context->usr_page = cpu_to_be32(
 			mlx4_to_hw_uar_index(dev->dev, dev->priv_uar.index));
@@ -1736,7 +2295,7 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
 			steer_qp = 1;
 		}
 
-		if (ibqp->qp_type == IB_QPT_GSI) {
+		if (qp_type == IB_QPT_GSI) {
 			enum ib_gid_type gid_type = qp->flags & MLX4_IB_ROCE_V2_GSI_QP ?
 				IB_GID_TYPE_ROCE_UDP_ENCAP : IB_GID_TYPE_ROCE;
 			u8 qpc_roce_mode = gid_type_to_qpc(gid_type);
@@ -1753,7 +2312,7 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
 	}
 
 	if (attr_mask & IB_QP_AV) {
-		u8 port_num = mlx4_is_bonded(to_mdev(ibqp->device)->dev) ? 1 :
+		u8 port_num = mlx4_is_bonded(dev->dev) ? 1 :
 			attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
 		union ib_gid gid;
 		struct ib_gid_attr gid_attr = {.gid_type = IB_GID_TYPE_IB};
@@ -1768,7 +2327,7 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
 			int index =
 				rdma_ah_read_grh(&attr->ah_attr)->sgid_index;
 
-			status = ib_get_cached_gid(ibqp->device, port_num,
+			status = ib_get_cached_gid(&dev->ib_dev, port_num,
 						   index, &gid, &gid_attr);
 			if (!status && !memcmp(&gid, &zgid, sizeof(gid)))
 				status = -ENOENT;
@@ -1825,15 +2384,20 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
 		optpar |= MLX4_QP_OPTPAR_ALT_ADDR_PATH;
 	}
 
-	pd = get_pd(qp);
-	get_cqs(qp, &send_cq, &recv_cq);
-	context->pd       = cpu_to_be32(pd->pdn);
+	context->pd = cpu_to_be32(pd->pdn);
+
+	if (!rwq_ind_tbl) {
+		get_cqs(qp, src_type, &send_cq, &recv_cq);
+	} else { /* Set dummy CQs to be compatible with HV and PRM */
+		send_cq = to_mcq(rwq_ind_tbl->ind_tbl[0]->cq);
+		recv_cq = send_cq;
+	}
 	context->cqn_send = cpu_to_be32(send_cq->mcq.cqn);
 	context->cqn_recv = cpu_to_be32(recv_cq->mcq.cqn);
 	context->params1  = cpu_to_be32(MLX4_IB_ACK_REQ_FREQ << 28);
 
 	/* Set "fast registration enabled" for all kernel QPs */
-	if (!qp->ibqp.uobject)
+	if (!ibuobject)
 		context->params1 |= cpu_to_be32(1 << 11);
 
 	if (attr_mask & IB_QP_RNR_RETRY) {
@@ -1868,7 +2432,7 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
 		optpar |= MLX4_QP_OPTPAR_RWE | MLX4_QP_OPTPAR_RRE | MLX4_QP_OPTPAR_RAE;
 	}
 
-	if (ibqp->srq)
+	if (ibsrq)
 		context->params2 |= cpu_to_be32(MLX4_QP_BIT_RIC);
 
 	if (attr_mask & IB_QP_MIN_RNR_TIMER) {
@@ -1899,17 +2463,19 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
 		optpar |= MLX4_QP_OPTPAR_Q_KEY;
 	}
 
-	if (ibqp->srq)
-		context->srqn = cpu_to_be32(1 << 24 | to_msrq(ibqp->srq)->msrq.srqn);
+	if (ibsrq)
+		context->srqn = cpu_to_be32(1 << 24 |
+					    to_msrq(ibsrq)->msrq.srqn);
 
-	if (qp->rq.wqe_cnt && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
+	if (qp->rq.wqe_cnt &&
+	    cur_state == IB_QPS_RESET &&
+	    new_state == IB_QPS_INIT)
 		context->db_rec_addr = cpu_to_be64(qp->db.dma);
 
 	if (cur_state == IB_QPS_INIT &&
 	    new_state == IB_QPS_RTR  &&
-	    (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI ||
-	     ibqp->qp_type == IB_QPT_UD ||
-	     ibqp->qp_type == IB_QPT_RAW_PACKET)) {
+	    (qp_type == IB_QPT_GSI || qp_type == IB_QPT_SMI ||
+	     qp_type == IB_QPT_UD || qp_type == IB_QPT_RAW_PACKET)) {
 		context->pri_path.sched_queue = (qp->port - 1) << 6;
 		if (qp->mlx4_ib_qp_type == MLX4_IB_QPT_SMI ||
 		    qp->mlx4_ib_qp_type &
@@ -1942,7 +2508,7 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
 		}
 	}
 
-	if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET) {
+	if (qp_type == IB_QPT_RAW_PACKET) {
 		context->pri_path.ackto = (context->pri_path.ackto & 0xf8) |
 					MLX4_IB_LINK_TYPE_ETH;
 		if (dev->dev->caps.tunnel_offload_mode ==  MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) {
@@ -1952,7 +2518,7 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
 		}
 	}
 
-	if (ibqp->qp_type == IB_QPT_UD && (new_state == IB_QPS_RTR)) {
+	if (qp_type == IB_QPT_UD && (new_state == IB_QPS_RTR)) {
 		int is_eth = rdma_port_get_link_layer(
 				&dev->ib_dev, qp->port) ==
 				IB_LINK_LAYER_ETHERNET;
@@ -1962,14 +2528,15 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
 		}
 	}
 
-
 	if (cur_state == IB_QPS_RTS && new_state == IB_QPS_SQD	&&
 	    attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY && attr->en_sqd_async_notify)
 		sqd_event = 1;
 	else
 		sqd_event = 0;
 
-	if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
+	if (!ibuobject &&
+	    cur_state == IB_QPS_RESET &&
+	    new_state == IB_QPS_INIT)
 		context->rlkey_roce_mode |= (1 << 4);
 
 	/*
@@ -1978,7 +2545,9 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
 	 * headroom is stamped so that the hardware doesn't start
 	 * processing stale work requests.
 	 */
-	if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
+	if (!ibuobject &&
+	    cur_state == IB_QPS_RESET &&
+	    new_state == IB_QPS_INIT) {
 		struct mlx4_wqe_ctrl_seg *ctrl;
 		int i;
 
@@ -2035,9 +2604,9 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
 	 * entries and reinitialize the QP.
 	 */
 	if (new_state == IB_QPS_RESET) {
-		if (!ibqp->uobject) {
+		if (!ibuobject) {
 			mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn,
-					 ibqp->srq ? to_msrq(ibqp->srq) : NULL);
+					 ibsrq ? to_msrq(ibsrq) : NULL);
 			if (send_cq != recv_cq)
 				mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL);
 
@@ -2148,22 +2717,25 @@ out:
 	return err;
 }
 
+enum {
+	MLX4_IB_MODIFY_QP_RSS_SUP_ATTR_MSK = (IB_QP_STATE	|
+					      IB_QP_PORT),
+};
+
 static int _mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 			      int attr_mask, struct ib_udata *udata)
 {
+	enum rdma_link_layer ll = IB_LINK_LAYER_UNSPECIFIED;
 	struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
 	struct mlx4_ib_qp *qp = to_mqp(ibqp);
 	enum ib_qp_state cur_state, new_state;
 	int err = -EINVAL;
-	int ll;
 	mutex_lock(&qp->mutex);
 
 	cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;
 	new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
 
-	if (cur_state == new_state && cur_state == IB_QPS_RESET) {
-		ll = IB_LINK_LAYER_UNSPECIFIED;
-	} else {
+	if (cur_state != new_state || cur_state != IB_QPS_RESET) {
 		int port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
 		ll = rdma_port_get_link_layer(&dev->ib_dev, port);
 	}
@@ -2178,6 +2750,27 @@ static int _mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 		goto out;
 	}
 
+	if (ibqp->rwq_ind_tbl) {
+		if (!(((cur_state == IB_QPS_RESET) &&
+		       (new_state == IB_QPS_INIT)) ||
+		      ((cur_state == IB_QPS_INIT)  &&
+		       (new_state == IB_QPS_RTR)))) {
+			pr_debug("qpn 0x%x: RSS QP unsupported transition %d to %d\n",
+				 ibqp->qp_num, cur_state, new_state);
+
+			err = -EOPNOTSUPP;
+			goto out;
+		}
+
+		if (attr_mask & ~MLX4_IB_MODIFY_QP_RSS_SUP_ATTR_MSK) {
+			pr_debug("qpn 0x%x: RSS QP unsupported attribute mask 0x%x for transition %d to %d\n",
+				 ibqp->qp_num, attr_mask, cur_state, new_state);
+
+			err = -EOPNOTSUPP;
+			goto out;
+		}
+	}
+
 	if (mlx4_is_bonded(dev->dev) && (attr_mask & IB_QP_PORT)) {
 		if ((cur_state == IB_QPS_RESET) && (new_state == IB_QPS_INIT)) {
 			if ((ibqp->qp_type == IB_QPT_RC) ||
@@ -2242,7 +2835,17 @@ static int _mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 		goto out;
 	}
 
-	err = __mlx4_ib_modify_qp(ibqp, attr, attr_mask, cur_state, new_state);
+	if (ibqp->rwq_ind_tbl && (new_state == IB_QPS_INIT)) {
+		err = bringup_rss_rwqs(ibqp->rwq_ind_tbl, attr->port_num);
+		if (err)
+			goto out;
+	}
+
+	err = __mlx4_ib_modify_qp(ibqp, MLX4_IB_QP_SRC, attr, attr_mask,
+				  cur_state, new_state);
+
+	if (ibqp->rwq_ind_tbl && err)
+		bring_down_rss_rwqs(ibqp->rwq_ind_tbl);
 
 	if (mlx4_is_bonded(dev->dev) && (attr_mask & IB_QP_PORT))
 		attr->port_num = 1;
@@ -2277,9 +2880,9 @@ static int vf_get_qp0_qkey(struct mlx4_dev *dev, int qpn, u32 *qkey)
 {
 	int i;
 	for (i = 0; i < dev->caps.num_ports; i++) {
-		if (qpn == dev->caps.qp0_proxy[i] ||
-		    qpn == dev->caps.qp0_tunnel[i]) {
-			*qkey = dev->caps.qp0_qkey[i];
+		if (qpn == dev->caps.spec_qps[i].qp0_proxy ||
+		    qpn == dev->caps.spec_qps[i].qp0_tunnel) {
+			*qkey = dev->caps.spec_qps[i].qp0_qkey;
 			return 0;
 		}
 	}
@@ -2340,7 +2943,7 @@ static int build_sriov_qp0_header(struct mlx4_ib_sqp *sqp,
 		sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->remote_qpn);
 	else
 		sqp->ud_header.bth.destination_qpn =
-			cpu_to_be32(mdev->dev->caps.qp0_tunnel[sqp->qp.port - 1]);
+			cpu_to_be32(mdev->dev->caps.spec_qps[sqp->qp.port - 1].qp0_tunnel);
 
 	sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1));
 	if (mlx4_is_master(mdev->dev)) {
@@ -2800,9 +3403,9 @@ static void set_tunnel_datagram_seg(struct mlx4_ib_dev *dev,
 
 	memcpy(dseg->av, &sqp_av, sizeof (struct mlx4_av));
 	if (qpt == MLX4_IB_QPT_PROXY_GSI)
-		dseg->dqpn = cpu_to_be32(dev->dev->caps.qp1_tunnel[port - 1]);
+		dseg->dqpn = cpu_to_be32(dev->dev->caps.spec_qps[port - 1].qp1_tunnel);
 	else
-		dseg->dqpn = cpu_to_be32(dev->dev->caps.qp0_tunnel[port - 1]);
+		dseg->dqpn = cpu_to_be32(dev->dev->caps.spec_qps[port - 1].qp0_tunnel);
 	/* Use QKEY from the QP context, which is set by master */
 	dseg->qkey = cpu_to_be32(IB_QP_SET_QKEY);
 }
@@ -3432,6 +4035,9 @@ int mlx4_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr
 	int mlx4_state;
 	int err = 0;
 
+	if (ibqp->rwq_ind_tbl)
+		return -EOPNOTSUPP;
+
 	mutex_lock(&qp->mutex);
 
 	if (qp->state == IB_QPS_RESET) {
@@ -3527,3 +4133,285 @@ out:
 	return err;
 }
 
+struct ib_wq *mlx4_ib_create_wq(struct ib_pd *pd,
+				struct ib_wq_init_attr *init_attr,
+				struct ib_udata *udata)
+{
+	struct mlx4_ib_dev *dev;
+	struct ib_qp_init_attr ib_qp_init_attr;
+	struct mlx4_ib_qp *qp;
+	struct mlx4_ib_create_wq ucmd;
+	int err, required_cmd_sz;
+
+	if (!(udata && pd->uobject))
+		return ERR_PTR(-EINVAL);
+
+	required_cmd_sz = offsetof(typeof(ucmd), comp_mask) +
+			  sizeof(ucmd.comp_mask);
+	if (udata->inlen < required_cmd_sz) {
+		pr_debug("invalid inlen\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	if (udata->inlen > sizeof(ucmd) &&
+	    !ib_is_udata_cleared(udata, sizeof(ucmd),
+				 udata->inlen - sizeof(ucmd))) {
+		pr_debug("inlen is not supported\n");
+		return ERR_PTR(-EOPNOTSUPP);
+	}
+
+	if (udata->outlen)
+		return ERR_PTR(-EOPNOTSUPP);
+
+	dev = to_mdev(pd->device);
+
+	if (init_attr->wq_type != IB_WQT_RQ) {
+		pr_debug("unsupported wq type %d\n", init_attr->wq_type);
+		return ERR_PTR(-EOPNOTSUPP);
+	}
+
+	if (init_attr->create_flags) {
+		pr_debug("unsupported create_flags %u\n",
+			 init_attr->create_flags);
+		return ERR_PTR(-EOPNOTSUPP);
+	}
+
+	qp = kzalloc(sizeof(*qp), GFP_KERNEL);
+	if (!qp)
+		return ERR_PTR(-ENOMEM);
+
+	qp->pri.vid = 0xFFFF;
+	qp->alt.vid = 0xFFFF;
+
+	memset(&ib_qp_init_attr, 0, sizeof(ib_qp_init_attr));
+	ib_qp_init_attr.qp_context = init_attr->wq_context;
+	ib_qp_init_attr.qp_type = IB_QPT_RAW_PACKET;
+	ib_qp_init_attr.cap.max_recv_wr = init_attr->max_wr;
+	ib_qp_init_attr.cap.max_recv_sge = init_attr->max_sge;
+	ib_qp_init_attr.recv_cq = init_attr->cq;
+	ib_qp_init_attr.send_cq = ib_qp_init_attr.recv_cq; /* Dummy CQ */
+
+	err = create_qp_common(dev, pd, MLX4_IB_RWQ_SRC, &ib_qp_init_attr,
+			       udata, 0, &qp);
+	if (err) {
+		kfree(qp);
+		return ERR_PTR(err);
+	}
+
+	qp->ibwq.event_handler = init_attr->event_handler;
+	qp->ibwq.wq_num = qp->mqp.qpn;
+	qp->ibwq.state = IB_WQS_RESET;
+
+	return &qp->ibwq;
+}
+
+static int ib_wq2qp_state(enum ib_wq_state state)
+{
+	switch (state) {
+	case IB_WQS_RESET:
+		return IB_QPS_RESET;
+	case IB_WQS_RDY:
+		return IB_QPS_RTR;
+	default:
+		return IB_QPS_ERR;
+	}
+}
+
+static int _mlx4_ib_modify_wq(struct ib_wq *ibwq, enum ib_wq_state new_state)
+{
+	struct mlx4_ib_qp *qp = to_mqp((struct ib_qp *)ibwq);
+	enum ib_qp_state qp_cur_state;
+	enum ib_qp_state qp_new_state;
+	int attr_mask;
+	int err;
+
+	/* ib_qp.state represents the WQ HW state while ib_wq.state represents
+	 * the WQ logic state.
+	 */
+	qp_cur_state = qp->state;
+	qp_new_state = ib_wq2qp_state(new_state);
+
+	if (ib_wq2qp_state(new_state) == qp_cur_state)
+		return 0;
+
+	if (new_state == IB_WQS_RDY) {
+		struct ib_qp_attr attr = {};
+
+		attr.port_num = qp->port;
+		attr_mask = IB_QP_PORT;
+
+		err = __mlx4_ib_modify_qp(ibwq, MLX4_IB_RWQ_SRC, &attr,
+					  attr_mask, IB_QPS_RESET, IB_QPS_INIT);
+		if (err) {
+			pr_debug("WQN=0x%06x failed to apply RST->INIT on the HW QP\n",
+				 ibwq->wq_num);
+			return err;
+		}
+
+		qp_cur_state = IB_QPS_INIT;
+	}
+
+	attr_mask = 0;
+	err = __mlx4_ib_modify_qp(ibwq, MLX4_IB_RWQ_SRC, NULL, attr_mask,
+				  qp_cur_state,  qp_new_state);
+
+	if (err && (qp_cur_state == IB_QPS_INIT)) {
+		qp_new_state = IB_QPS_RESET;
+		if (__mlx4_ib_modify_qp(ibwq, MLX4_IB_RWQ_SRC, NULL,
+					attr_mask, IB_QPS_INIT, IB_QPS_RESET)) {
+			pr_warn("WQN=0x%06x failed with reverting HW's resources failure\n",
+				ibwq->wq_num);
+			qp_new_state = IB_QPS_INIT;
+		}
+	}
+
+	qp->state = qp_new_state;
+
+	return err;
+}
+
+int mlx4_ib_modify_wq(struct ib_wq *ibwq, struct ib_wq_attr *wq_attr,
+		      u32 wq_attr_mask, struct ib_udata *udata)
+{
+	struct mlx4_ib_qp *qp = to_mqp((struct ib_qp *)ibwq);
+	struct mlx4_ib_modify_wq ucmd = {};
+	size_t required_cmd_sz;
+	enum ib_wq_state cur_state, new_state;
+	int err = 0;
+
+	required_cmd_sz = offsetof(typeof(ucmd), reserved) +
+				   sizeof(ucmd.reserved);
+	if (udata->inlen < required_cmd_sz)
+		return -EINVAL;
+
+	if (udata->inlen > sizeof(ucmd) &&
+	    !ib_is_udata_cleared(udata, sizeof(ucmd),
+				 udata->inlen - sizeof(ucmd)))
+		return -EOPNOTSUPP;
+
+	if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen)))
+		return -EFAULT;
+
+	if (ucmd.comp_mask || ucmd.reserved)
+		return -EOPNOTSUPP;
+
+	if (wq_attr_mask & IB_WQ_FLAGS)
+		return -EOPNOTSUPP;
+
+	cur_state = wq_attr_mask & IB_WQ_CUR_STATE ? wq_attr->curr_wq_state :
+						     ibwq->state;
+	new_state = wq_attr_mask & IB_WQ_STATE ? wq_attr->wq_state : cur_state;
+
+	if (cur_state  < IB_WQS_RESET || cur_state  > IB_WQS_ERR ||
+	    new_state < IB_WQS_RESET || new_state > IB_WQS_ERR)
+		return -EINVAL;
+
+	if ((new_state == IB_WQS_RDY) && (cur_state == IB_WQS_ERR))
+		return -EINVAL;
+
+	if ((new_state == IB_WQS_ERR) && (cur_state == IB_WQS_RESET))
+		return -EINVAL;
+
+	/* Need to protect against the parent RSS which also may modify WQ
+	 * state.
+	 */
+	mutex_lock(&qp->mutex);
+
+	/* Can update HW state only if a RSS QP has already associated to this
+	 * WQ, so we can apply its port on the WQ.
+	 */
+	if (qp->rss_usecnt)
+		err = _mlx4_ib_modify_wq(ibwq, new_state);
+
+	if (!err)
+		ibwq->state = new_state;
+
+	mutex_unlock(&qp->mutex);
+
+	return err;
+}
+
+int mlx4_ib_destroy_wq(struct ib_wq *ibwq)
+{
+	struct mlx4_ib_dev *dev = to_mdev(ibwq->device);
+	struct mlx4_ib_qp *qp = to_mqp((struct ib_qp *)ibwq);
+
+	if (qp->counter_index)
+		mlx4_ib_free_qp_counter(dev, qp);
+
+	destroy_qp_common(dev, qp, MLX4_IB_RWQ_SRC, 1);
+
+	kfree(qp);
+
+	return 0;
+}
+
+struct ib_rwq_ind_table
+*mlx4_ib_create_rwq_ind_table(struct ib_device *device,
+			      struct ib_rwq_ind_table_init_attr *init_attr,
+			      struct ib_udata *udata)
+{
+	struct ib_rwq_ind_table *rwq_ind_table;
+	struct mlx4_ib_create_rwq_ind_tbl_resp resp = {};
+	unsigned int ind_tbl_size = 1 << init_attr->log_ind_tbl_size;
+	unsigned int base_wqn;
+	size_t min_resp_len;
+	int i;
+	int err;
+
+	if (udata->inlen > 0 &&
+	    !ib_is_udata_cleared(udata, 0,
+				 udata->inlen))
+		return ERR_PTR(-EOPNOTSUPP);
+
+	min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);
+	if (udata->outlen && udata->outlen < min_resp_len)
+		return ERR_PTR(-EINVAL);
+
+	if (ind_tbl_size >
+	    device->attrs.rss_caps.max_rwq_indirection_table_size) {
+		pr_debug("log_ind_tbl_size = %d is bigger than supported = %d\n",
+			 ind_tbl_size,
+			 device->attrs.rss_caps.max_rwq_indirection_table_size);
+		return ERR_PTR(-EINVAL);
+	}
+
+	base_wqn = init_attr->ind_tbl[0]->wq_num;
+
+	if (base_wqn % ind_tbl_size) {
+		pr_debug("WQN=0x%x isn't aligned with indirection table size\n",
+			 base_wqn);
+		return ERR_PTR(-EINVAL);
+	}
+
+	for (i = 1; i < ind_tbl_size; i++) {
+		if (++base_wqn != init_attr->ind_tbl[i]->wq_num) {
+			pr_debug("indirection table's WQNs aren't consecutive\n");
+			return ERR_PTR(-EINVAL);
+		}
+	}
+
+	rwq_ind_table = kzalloc(sizeof(*rwq_ind_table), GFP_KERNEL);
+	if (!rwq_ind_table)
+		return ERR_PTR(-ENOMEM);
+
+	if (udata->outlen) {
+		resp.response_length = offsetof(typeof(resp), response_length) +
+					sizeof(resp.response_length);
+		err = ib_copy_to_udata(udata, &resp, resp.response_length);
+		if (err)
+			goto err;
+	}
+
+	return rwq_ind_table;
+
+err:
+	kfree(rwq_ind_table);
+	return ERR_PTR(err);
+}
+
+int mlx4_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)
+{
+	kfree(ib_rwq_ind_tbl);
+	return 0;
+}
diff --git a/drivers/infiniband/hw/mlx4/srq.c b/drivers/infiniband/hw/mlx4/srq.c
index 0facaf5f6d23..ebee56cbc0e2 100644
--- a/drivers/infiniband/hw/mlx4/srq.c
+++ b/drivers/infiniband/hw/mlx4/srq.c
@@ -34,7 +34,6 @@
 #include <linux/mlx4/qp.h>
 #include <linux/mlx4/srq.h>
 #include <linux/slab.h>
-#include <linux/vmalloc.h>
 
 #include "mlx4_ib.h"
 #include <rdma/mlx4-abi.h>
@@ -171,20 +170,16 @@ struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd,
 		if (err)
 			goto err_mtt;
 
-		srq->wrid = kmalloc_array(srq->msrq.max, sizeof(u64),
-					GFP_KERNEL | __GFP_NOWARN);
+		srq->wrid = kvmalloc_array(srq->msrq.max,
+					   sizeof(u64), GFP_KERNEL);
 		if (!srq->wrid) {
-			srq->wrid = __vmalloc(srq->msrq.max * sizeof(u64),
-					      GFP_KERNEL, PAGE_KERNEL);
-			if (!srq->wrid) {
-				err = -ENOMEM;
-				goto err_mtt;
-			}
+			err = -ENOMEM;
+			goto err_mtt;
 		}
 	}
 
-	cqn = (init_attr->srq_type == IB_SRQT_XRC) ?
-		to_mcq(init_attr->ext.xrc.cq)->mcq.cqn : 0;
+	cqn = ib_srq_has_cq(init_attr->srq_type) ?
+		to_mcq(init_attr->ext.cq)->mcq.cqn : 0;
 	xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ?
 		to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn :
 		(u16) dev->dev->caps.reserved_xrcds;
diff --git a/drivers/infiniband/hw/mlx4/sysfs.c b/drivers/infiniband/hw/mlx4/sysfs.c
index 0ba5ba7540c8..e219093d2764 100644
--- a/drivers/infiniband/hw/mlx4/sysfs.c
+++ b/drivers/infiniband/hw/mlx4/sysfs.c
@@ -221,7 +221,7 @@ void del_sysfs_port_mcg_attr(struct mlx4_ib_dev *device, int port_num,
 static int add_port_entries(struct mlx4_ib_dev *device, int port_num)
 {
 	int i;
-	char buff[10];
+	char buff[11];
 	struct mlx4_ib_iov_port *port = NULL;
 	int ret = 0 ;
 	struct ib_port_attr attr;
diff --git a/drivers/infiniband/hw/mlx5/Makefile b/drivers/infiniband/hw/mlx5/Makefile
index 90ad2adc752f..bc6299697dda 100644
--- a/drivers/infiniband/hw/mlx5/Makefile
+++ b/drivers/infiniband/hw/mlx5/Makefile
@@ -1,4 +1,4 @@
 obj-$(CONFIG_MLX5_INFINIBAND)	+= mlx5_ib.o
 
-mlx5_ib-y :=	main.o cq.o doorbell.o qp.o mem.o srq.o mr.o ah.o mad.o gsi.o ib_virt.o cmd.o
+mlx5_ib-y :=	main.o cq.o doorbell.o qp.o mem.o srq.o mr.o ah.o mad.o gsi.o ib_virt.o cmd.o cong.o
 mlx5_ib-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += odp.o
diff --git a/drivers/infiniband/hw/mlx5/cmd.c b/drivers/infiniband/hw/mlx5/cmd.c
index 18d5e1db93ed..470995fa38d2 100644
--- a/drivers/infiniband/hw/mlx5/cmd.c
+++ b/drivers/infiniband/hw/mlx5/cmd.c
@@ -57,3 +57,23 @@ int mlx5_cmd_query_cong_counter(struct mlx5_core_dev *dev,
 	MLX5_SET(query_cong_statistics_in, in, clear, reset);
 	return mlx5_cmd_exec(dev, in, sizeof(in), out, out_size);
 }
+
+int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point,
+			       void *out, int out_size)
+{
+	u32 in[MLX5_ST_SZ_DW(query_cong_params_in)] = { };
+
+	MLX5_SET(query_cong_params_in, in, opcode,
+		 MLX5_CMD_OP_QUERY_CONG_PARAMS);
+	MLX5_SET(query_cong_params_in, in, cong_protocol, cong_point);
+
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, out_size);
+}
+
+int mlx5_cmd_modify_cong_params(struct mlx5_core_dev *dev,
+				void *in, int in_size)
+{
+	u32 out[MLX5_ST_SZ_DW(modify_cong_params_out)] = { };
+
+	return mlx5_cmd_exec(dev, in, in_size, out, sizeof(out));
+}
diff --git a/drivers/infiniband/hw/mlx5/cmd.h b/drivers/infiniband/hw/mlx5/cmd.h
index fa09228193a6..af4c24596274 100644
--- a/drivers/infiniband/hw/mlx5/cmd.h
+++ b/drivers/infiniband/hw/mlx5/cmd.h
@@ -39,4 +39,8 @@
 int mlx5_cmd_null_mkey(struct mlx5_core_dev *dev, u32 *null_mkey);
 int mlx5_cmd_query_cong_counter(struct mlx5_core_dev *dev,
 				bool reset, void *out, int out_size);
+int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point,
+			       void *out, int out_size);
+int mlx5_cmd_modify_cong_params(struct mlx5_core_dev *mdev,
+				void *in, int in_size);
 #endif /* MLX5_IB_CMD_H */
diff --git a/drivers/infiniband/hw/mlx5/cong.c b/drivers/infiniband/hw/mlx5/cong.c
new file mode 100644
index 000000000000..2d32b519bb61
--- /dev/null
+++ b/drivers/infiniband/hw/mlx5/cong.c
@@ -0,0 +1,421 @@
+/*
+ * Copyright (c) 2013-2017, Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/debugfs.h>
+
+#include "mlx5_ib.h"
+#include "cmd.h"
+
+enum mlx5_ib_cong_node_type {
+	MLX5_IB_RROCE_ECN_RP = 1,
+	MLX5_IB_RROCE_ECN_NP = 2,
+};
+
+static const char * const mlx5_ib_dbg_cc_name[] = {
+	"rp_clamp_tgt_rate",
+	"rp_clamp_tgt_rate_ati",
+	"rp_time_reset",
+	"rp_byte_reset",
+	"rp_threshold",
+	"rp_ai_rate",
+	"rp_hai_rate",
+	"rp_min_dec_fac",
+	"rp_min_rate",
+	"rp_rate_to_set_on_first_cnp",
+	"rp_dce_tcp_g",
+	"rp_dce_tcp_rtt",
+	"rp_rate_reduce_monitor_period",
+	"rp_initial_alpha_value",
+	"rp_gd",
+	"np_cnp_dscp",
+	"np_cnp_prio_mode",
+	"np_cnp_prio",
+};
+
+#define MLX5_IB_RP_CLAMP_TGT_RATE_ATTR			BIT(1)
+#define MLX5_IB_RP_CLAMP_TGT_RATE_ATI_ATTR		BIT(2)
+#define MLX5_IB_RP_TIME_RESET_ATTR			BIT(3)
+#define MLX5_IB_RP_BYTE_RESET_ATTR			BIT(4)
+#define MLX5_IB_RP_THRESHOLD_ATTR			BIT(5)
+#define MLX5_IB_RP_AI_RATE_ATTR				BIT(7)
+#define MLX5_IB_RP_HAI_RATE_ATTR			BIT(8)
+#define MLX5_IB_RP_MIN_DEC_FAC_ATTR			BIT(9)
+#define MLX5_IB_RP_MIN_RATE_ATTR			BIT(10)
+#define MLX5_IB_RP_RATE_TO_SET_ON_FIRST_CNP_ATTR	BIT(11)
+#define MLX5_IB_RP_DCE_TCP_G_ATTR			BIT(12)
+#define MLX5_IB_RP_DCE_TCP_RTT_ATTR			BIT(13)
+#define MLX5_IB_RP_RATE_REDUCE_MONITOR_PERIOD_ATTR	BIT(14)
+#define MLX5_IB_RP_INITIAL_ALPHA_VALUE_ATTR		BIT(15)
+#define MLX5_IB_RP_GD_ATTR				BIT(16)
+
+#define MLX5_IB_NP_CNP_DSCP_ATTR			BIT(3)
+#define MLX5_IB_NP_CNP_PRIO_MODE_ATTR			BIT(4)
+
+static enum mlx5_ib_cong_node_type
+mlx5_ib_param_to_node(enum mlx5_ib_dbg_cc_types param_offset)
+{
+	if (param_offset >= MLX5_IB_DBG_CC_RP_CLAMP_TGT_RATE &&
+	    param_offset <= MLX5_IB_DBG_CC_RP_GD)
+		return MLX5_IB_RROCE_ECN_RP;
+	else
+		return MLX5_IB_RROCE_ECN_NP;
+}
+
+static u32 mlx5_get_cc_param_val(void *field, int offset)
+{
+	switch (offset) {
+	case MLX5_IB_DBG_CC_RP_CLAMP_TGT_RATE:
+		return MLX5_GET(cong_control_r_roce_ecn_rp, field,
+				clamp_tgt_rate);
+	case MLX5_IB_DBG_CC_RP_CLAMP_TGT_RATE_ATI:
+		return MLX5_GET(cong_control_r_roce_ecn_rp, field,
+				clamp_tgt_rate_after_time_inc);
+	case MLX5_IB_DBG_CC_RP_TIME_RESET:
+		return MLX5_GET(cong_control_r_roce_ecn_rp, field,
+				rpg_time_reset);
+	case MLX5_IB_DBG_CC_RP_BYTE_RESET:
+		return MLX5_GET(cong_control_r_roce_ecn_rp, field,
+				rpg_byte_reset);
+	case MLX5_IB_DBG_CC_RP_THRESHOLD:
+		return MLX5_GET(cong_control_r_roce_ecn_rp, field,
+				rpg_threshold);
+	case MLX5_IB_DBG_CC_RP_AI_RATE:
+		return MLX5_GET(cong_control_r_roce_ecn_rp, field,
+				rpg_ai_rate);
+	case MLX5_IB_DBG_CC_RP_HAI_RATE:
+		return MLX5_GET(cong_control_r_roce_ecn_rp, field,
+				rpg_hai_rate);
+	case MLX5_IB_DBG_CC_RP_MIN_DEC_FAC:
+		return MLX5_GET(cong_control_r_roce_ecn_rp, field,
+				rpg_min_dec_fac);
+	case MLX5_IB_DBG_CC_RP_MIN_RATE:
+		return MLX5_GET(cong_control_r_roce_ecn_rp, field,
+				rpg_min_rate);
+	case MLX5_IB_DBG_CC_RP_RATE_TO_SET_ON_FIRST_CNP:
+		return MLX5_GET(cong_control_r_roce_ecn_rp, field,
+				rate_to_set_on_first_cnp);
+	case MLX5_IB_DBG_CC_RP_DCE_TCP_G:
+		return MLX5_GET(cong_control_r_roce_ecn_rp, field,
+				dce_tcp_g);
+	case MLX5_IB_DBG_CC_RP_DCE_TCP_RTT:
+		return MLX5_GET(cong_control_r_roce_ecn_rp, field,
+				dce_tcp_rtt);
+	case MLX5_IB_DBG_CC_RP_RATE_REDUCE_MONITOR_PERIOD:
+		return MLX5_GET(cong_control_r_roce_ecn_rp, field,
+				rate_reduce_monitor_period);
+	case MLX5_IB_DBG_CC_RP_INITIAL_ALPHA_VALUE:
+		return MLX5_GET(cong_control_r_roce_ecn_rp, field,
+				initial_alpha_value);
+	case MLX5_IB_DBG_CC_RP_GD:
+		return MLX5_GET(cong_control_r_roce_ecn_rp, field,
+				rpg_gd);
+	case MLX5_IB_DBG_CC_NP_CNP_DSCP:
+		return MLX5_GET(cong_control_r_roce_ecn_np, field,
+				cnp_dscp);
+	case MLX5_IB_DBG_CC_NP_CNP_PRIO_MODE:
+		return MLX5_GET(cong_control_r_roce_ecn_np, field,
+				cnp_prio_mode);
+	case MLX5_IB_DBG_CC_NP_CNP_PRIO:
+		return MLX5_GET(cong_control_r_roce_ecn_np, field,
+				cnp_802p_prio);
+	default:
+		return 0;
+	}
+}
+
+static void mlx5_ib_set_cc_param_mask_val(void *field, int offset,
+					  u32 var, u32 *attr_mask)
+{
+	switch (offset) {
+	case MLX5_IB_DBG_CC_RP_CLAMP_TGT_RATE:
+		*attr_mask |= MLX5_IB_RP_CLAMP_TGT_RATE_ATTR;
+		MLX5_SET(cong_control_r_roce_ecn_rp, field,
+			 clamp_tgt_rate, var);
+		break;
+	case MLX5_IB_DBG_CC_RP_CLAMP_TGT_RATE_ATI:
+		*attr_mask |= MLX5_IB_RP_CLAMP_TGT_RATE_ATI_ATTR;
+		MLX5_SET(cong_control_r_roce_ecn_rp, field,
+			 clamp_tgt_rate_after_time_inc, var);
+		break;
+	case MLX5_IB_DBG_CC_RP_TIME_RESET:
+		*attr_mask |= MLX5_IB_RP_TIME_RESET_ATTR;
+		MLX5_SET(cong_control_r_roce_ecn_rp, field,
+			 rpg_time_reset, var);
+		break;
+	case MLX5_IB_DBG_CC_RP_BYTE_RESET:
+		*attr_mask |= MLX5_IB_RP_BYTE_RESET_ATTR;
+		MLX5_SET(cong_control_r_roce_ecn_rp, field,
+			 rpg_byte_reset, var);
+		break;
+	case MLX5_IB_DBG_CC_RP_THRESHOLD:
+		*attr_mask |= MLX5_IB_RP_THRESHOLD_ATTR;
+		MLX5_SET(cong_control_r_roce_ecn_rp, field,
+			 rpg_threshold, var);
+		break;
+	case MLX5_IB_DBG_CC_RP_AI_RATE:
+		*attr_mask |= MLX5_IB_RP_AI_RATE_ATTR;
+		MLX5_SET(cong_control_r_roce_ecn_rp, field,
+			 rpg_ai_rate, var);
+		break;
+	case MLX5_IB_DBG_CC_RP_HAI_RATE:
+		*attr_mask |= MLX5_IB_RP_HAI_RATE_ATTR;
+		MLX5_SET(cong_control_r_roce_ecn_rp, field,
+			 rpg_hai_rate, var);
+		break;
+	case MLX5_IB_DBG_CC_RP_MIN_DEC_FAC:
+		*attr_mask |= MLX5_IB_RP_MIN_DEC_FAC_ATTR;
+		MLX5_SET(cong_control_r_roce_ecn_rp, field,
+			 rpg_min_dec_fac, var);
+		break;
+	case MLX5_IB_DBG_CC_RP_MIN_RATE:
+		*attr_mask |= MLX5_IB_RP_MIN_RATE_ATTR;
+		MLX5_SET(cong_control_r_roce_ecn_rp, field,
+			 rpg_min_rate, var);
+		break;
+	case MLX5_IB_DBG_CC_RP_RATE_TO_SET_ON_FIRST_CNP:
+		*attr_mask |= MLX5_IB_RP_RATE_TO_SET_ON_FIRST_CNP_ATTR;
+		MLX5_SET(cong_control_r_roce_ecn_rp, field,
+			 rate_to_set_on_first_cnp, var);
+		break;
+	case MLX5_IB_DBG_CC_RP_DCE_TCP_G:
+		*attr_mask |= MLX5_IB_RP_DCE_TCP_G_ATTR;
+		MLX5_SET(cong_control_r_roce_ecn_rp, field,
+			 dce_tcp_g, var);
+		break;
+	case MLX5_IB_DBG_CC_RP_DCE_TCP_RTT:
+		*attr_mask |= MLX5_IB_RP_DCE_TCP_RTT_ATTR;
+		MLX5_SET(cong_control_r_roce_ecn_rp, field,
+			 dce_tcp_rtt, var);
+		break;
+	case MLX5_IB_DBG_CC_RP_RATE_REDUCE_MONITOR_PERIOD:
+		*attr_mask |= MLX5_IB_RP_RATE_REDUCE_MONITOR_PERIOD_ATTR;
+		MLX5_SET(cong_control_r_roce_ecn_rp, field,
+			 rate_reduce_monitor_period, var);
+		break;
+	case MLX5_IB_DBG_CC_RP_INITIAL_ALPHA_VALUE:
+		*attr_mask |= MLX5_IB_RP_INITIAL_ALPHA_VALUE_ATTR;
+		MLX5_SET(cong_control_r_roce_ecn_rp, field,
+			 initial_alpha_value, var);
+		break;
+	case MLX5_IB_DBG_CC_RP_GD:
+		*attr_mask |= MLX5_IB_RP_GD_ATTR;
+		MLX5_SET(cong_control_r_roce_ecn_rp, field,
+			 rpg_gd, var);
+		break;
+	case MLX5_IB_DBG_CC_NP_CNP_DSCP:
+		*attr_mask |= MLX5_IB_NP_CNP_DSCP_ATTR;
+		MLX5_SET(cong_control_r_roce_ecn_np, field, cnp_dscp, var);
+		break;
+	case MLX5_IB_DBG_CC_NP_CNP_PRIO_MODE:
+		*attr_mask |= MLX5_IB_NP_CNP_PRIO_MODE_ATTR;
+		MLX5_SET(cong_control_r_roce_ecn_np, field, cnp_prio_mode, var);
+		break;
+	case MLX5_IB_DBG_CC_NP_CNP_PRIO:
+		*attr_mask |= MLX5_IB_NP_CNP_PRIO_MODE_ATTR;
+		MLX5_SET(cong_control_r_roce_ecn_np, field, cnp_prio_mode, 0);
+		MLX5_SET(cong_control_r_roce_ecn_np, field, cnp_802p_prio, var);
+		break;
+	}
+}
+
+static int mlx5_ib_get_cc_params(struct mlx5_ib_dev *dev, int offset, u32 *var)
+{
+	int outlen = MLX5_ST_SZ_BYTES(query_cong_params_out);
+	void *out;
+	void *field;
+	int err;
+	enum mlx5_ib_cong_node_type node;
+
+	out = kvzalloc(outlen, GFP_KERNEL);
+	if (!out)
+		return -ENOMEM;
+
+	node = mlx5_ib_param_to_node(offset);
+
+	err = mlx5_cmd_query_cong_params(dev->mdev, node, out, outlen);
+	if (err)
+		goto free;
+
+	field = MLX5_ADDR_OF(query_cong_params_out, out, congestion_parameters);
+	*var = mlx5_get_cc_param_val(field, offset);
+
+free:
+	kvfree(out);
+	return err;
+}
+
+static int mlx5_ib_set_cc_params(struct mlx5_ib_dev *dev, int offset, u32 var)
+{
+	int inlen = MLX5_ST_SZ_BYTES(modify_cong_params_in);
+	void *in;
+	void *field;
+	enum mlx5_ib_cong_node_type node;
+	u32 attr_mask = 0;
+	int err;
+
+	in = kvzalloc(inlen, GFP_KERNEL);
+	if (!in)
+		return -ENOMEM;
+
+	MLX5_SET(modify_cong_params_in, in, opcode,
+		 MLX5_CMD_OP_MODIFY_CONG_PARAMS);
+
+	node = mlx5_ib_param_to_node(offset);
+	MLX5_SET(modify_cong_params_in, in, cong_protocol, node);
+
+	field = MLX5_ADDR_OF(modify_cong_params_in, in, congestion_parameters);
+	mlx5_ib_set_cc_param_mask_val(field, offset, var, &attr_mask);
+
+	field = MLX5_ADDR_OF(modify_cong_params_in, in, field_select);
+	MLX5_SET(field_select_r_roce_rp, field, field_select_r_roce_rp,
+		 attr_mask);
+
+	err = mlx5_cmd_modify_cong_params(dev->mdev, in, inlen);
+	kvfree(in);
+	return err;
+}
+
+static ssize_t set_param(struct file *filp, const char __user *buf,
+			 size_t count, loff_t *pos)
+{
+	struct mlx5_ib_dbg_param *param = filp->private_data;
+	int offset = param->offset;
+	char lbuf[11] = { };
+	u32 var;
+	int ret;
+
+	if (count > sizeof(lbuf))
+		return -EINVAL;
+
+	if (copy_from_user(lbuf, buf, count))
+		return -EFAULT;
+
+	lbuf[sizeof(lbuf) - 1] = '\0';
+
+	if (kstrtou32(lbuf, 0, &var))
+		return -EINVAL;
+
+	ret = mlx5_ib_set_cc_params(param->dev, offset, var);
+	return ret ? ret : count;
+}
+
+static ssize_t get_param(struct file *filp, char __user *buf, size_t count,
+			 loff_t *pos)
+{
+	struct mlx5_ib_dbg_param *param = filp->private_data;
+	int offset = param->offset;
+	u32 var = 0;
+	int ret;
+	char lbuf[11];
+
+	if (*pos)
+		return 0;
+
+	ret = mlx5_ib_get_cc_params(param->dev, offset, &var);
+	if (ret)
+		return ret;
+
+	ret = snprintf(lbuf, sizeof(lbuf), "%d\n", var);
+	if (ret < 0)
+		return ret;
+
+	if (copy_to_user(buf, lbuf, ret))
+		return -EFAULT;
+
+	*pos += ret;
+	return ret;
+}
+
+static const struct file_operations dbg_cc_fops = {
+	.owner	= THIS_MODULE,
+	.open	= simple_open,
+	.write	= set_param,
+	.read	= get_param,
+};
+
+void mlx5_ib_cleanup_cong_debugfs(struct mlx5_ib_dev *dev)
+{
+	if (!mlx5_debugfs_root ||
+	    !dev->dbg_cc_params ||
+	    !dev->dbg_cc_params->root)
+		return;
+
+	debugfs_remove_recursive(dev->dbg_cc_params->root);
+	kfree(dev->dbg_cc_params);
+	dev->dbg_cc_params = NULL;
+}
+
+int mlx5_ib_init_cong_debugfs(struct mlx5_ib_dev *dev)
+{
+	struct mlx5_ib_dbg_cc_params *dbg_cc_params;
+	int i;
+
+	if (!mlx5_debugfs_root)
+		goto out;
+
+	if (!MLX5_CAP_GEN(dev->mdev, cc_query_allowed) ||
+	    !MLX5_CAP_GEN(dev->mdev, cc_modify_allowed))
+		goto out;
+
+	dbg_cc_params = kzalloc(sizeof(*dbg_cc_params), GFP_KERNEL);
+	if (!dbg_cc_params)
+		goto out;
+
+	dev->dbg_cc_params = dbg_cc_params;
+
+	dbg_cc_params->root = debugfs_create_dir("cc_params",
+						 dev->mdev->priv.dbg_root);
+	if (!dbg_cc_params->root)
+		goto err;
+
+	for (i = 0; i < MLX5_IB_DBG_CC_MAX; i++) {
+		dbg_cc_params->params[i].offset = i;
+		dbg_cc_params->params[i].dev = dev;
+		dbg_cc_params->params[i].dentry =
+			debugfs_create_file(mlx5_ib_dbg_cc_name[i],
+					    0600, dbg_cc_params->root,
+					    &dbg_cc_params->params[i],
+					    &dbg_cc_fops);
+		if (!dbg_cc_params->params[i].dentry)
+			goto err;
+	}
+out:	return 0;
+
+err:
+	mlx5_ib_warn(dev, "cong debugfs failure\n");
+	mlx5_ib_cleanup_cong_debugfs(dev);
+	/*
+	 * We don't want to fail driver if debugfs failed to initialize,
+	 * so we are not forwarding error to the user.
+	 */
+	return 0;
+}
diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c
index a384d72ea3cd..2aa53f427685 100644
--- a/drivers/infiniband/hw/mlx5/cq.c
+++ b/drivers/infiniband/hw/mlx5/cq.c
@@ -499,7 +499,7 @@ static void mlx5_ib_poll_sw_comp(struct mlx5_ib_cq *cq, int num_entries,
 	struct mlx5_ib_qp *qp;
 
 	*npolled = 0;
-	/* Find uncompleted WQEs belonging to that cq and retrun mmics ones */
+	/* Find uncompleted WQEs belonging to that cq and return mmics ones */
 	list_for_each_entry(qp, &cq->list_send_qp, cq_send_list) {
 		sw_send_comp(qp, num_entries, wc + *npolled, npolled);
 		if (*npolled >= num_entries)
@@ -751,10 +751,8 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
 	void *cqc;
 	int err;
 
-	ucmdlen =
-		(udata->inlen - sizeof(struct ib_uverbs_cmd_hdr) <
-		 sizeof(ucmd)) ? (sizeof(ucmd) -
-				  sizeof(ucmd.reserved)) : sizeof(ucmd);
+	ucmdlen = udata->inlen < sizeof(ucmd) ?
+		  (sizeof(ucmd) - sizeof(ucmd.reserved)) : sizeof(ucmd);
 
 	if (ib_copy_from_udata(&ucmd, udata, ucmdlen))
 		return -EFAULT;
diff --git a/drivers/infiniband/hw/mlx5/ib_virt.c b/drivers/infiniband/hw/mlx5/ib_virt.c
index c1b9de800fe5..649a3364f838 100644
--- a/drivers/infiniband/hw/mlx5/ib_virt.c
+++ b/drivers/infiniband/hw/mlx5/ib_virt.c
@@ -96,6 +96,7 @@ int mlx5_ib_set_vf_link_state(struct ib_device *device, int vf,
 	struct mlx5_ib_dev *dev = to_mdev(device);
 	struct mlx5_core_dev *mdev = dev->mdev;
 	struct mlx5_hca_vport_context *in;
+	struct mlx5_vf_context *vfs_ctx = mdev->priv.sriov.vfs_ctx;
 	int err;
 
 	in = kzalloc(sizeof(*in), GFP_KERNEL);
@@ -109,6 +110,8 @@ int mlx5_ib_set_vf_link_state(struct ib_device *device, int vf,
 	}
 	in->field_select = MLX5_HCA_VPORT_SEL_STATE_POLICY;
 	err = mlx5_core_modify_hca_vport_context(mdev, 1, 1, vf + 1, in);
+	if (!err)
+		vfs_ctx[vf].policy = in->policy;
 
 out:
 	kfree(in);
@@ -151,6 +154,7 @@ static int set_vf_node_guid(struct ib_device *device, int vf, u8 port, u64 guid)
 	struct mlx5_ib_dev *dev = to_mdev(device);
 	struct mlx5_core_dev *mdev = dev->mdev;
 	struct mlx5_hca_vport_context *in;
+	struct mlx5_vf_context *vfs_ctx = mdev->priv.sriov.vfs_ctx;
 	int err;
 
 	in = kzalloc(sizeof(*in), GFP_KERNEL);
@@ -160,6 +164,8 @@ static int set_vf_node_guid(struct ib_device *device, int vf, u8 port, u64 guid)
 	in->field_select = MLX5_HCA_VPORT_SEL_NODE_GUID;
 	in->node_guid = guid;
 	err = mlx5_core_modify_hca_vport_context(mdev, 1, 1, vf + 1, in);
+	if (!err)
+		vfs_ctx[vf].node_guid = guid;
 	kfree(in);
 	return err;
 }
@@ -169,6 +175,7 @@ static int set_vf_port_guid(struct ib_device *device, int vf, u8 port, u64 guid)
 	struct mlx5_ib_dev *dev = to_mdev(device);
 	struct mlx5_core_dev *mdev = dev->mdev;
 	struct mlx5_hca_vport_context *in;
+	struct mlx5_vf_context *vfs_ctx = mdev->priv.sriov.vfs_ctx;
 	int err;
 
 	in = kzalloc(sizeof(*in), GFP_KERNEL);
@@ -178,6 +185,8 @@ static int set_vf_port_guid(struct ib_device *device, int vf, u8 port, u64 guid)
 	in->field_select = MLX5_HCA_VPORT_SEL_PORT_GUID;
 	in->port_guid = guid;
 	err = mlx5_core_modify_hca_vport_context(mdev, 1, 1, vf + 1, in);
+	if (!err)
+		vfs_ctx[vf].port_guid = guid;
 	kfree(in);
 	return err;
 }
diff --git a/drivers/infiniband/hw/mlx5/mad.c b/drivers/infiniband/hw/mlx5/mad.c
index 95db929bdc34..1003b0133a49 100644
--- a/drivers/infiniband/hw/mlx5/mad.c
+++ b/drivers/infiniband/hw/mlx5/mad.c
@@ -78,7 +78,7 @@ static int process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
 	u16 slid;
 	int err;
 
-	slid = in_wc ? in_wc->slid : be16_to_cpu(IB_LID_PERMISSIVE);
+	slid = in_wc ? ib_lid_cpu16(in_wc->slid) : be16_to_cpu(IB_LID_PERMISSIVE);
 
 	if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP && slid == 0)
 		return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED;
@@ -204,7 +204,7 @@ static int process_pma_cmd(struct ib_device *ibdev, u8 port_num,
 	int err;
 	void *out_cnt;
 
-	/* Decalring support of extended counters */
+	/* Declaring support of extended counters */
 	if (in_mad->mad_hdr.attr_id == IB_PMA_CLASS_PORT_INFO) {
 		struct ib_class_port_info cpi = {};
 
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index a7f2e60085c4..ab3c562d5ba7 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -30,6 +30,7 @@
  * SOFTWARE.
  */
 
+#include <linux/debugfs.h>
 #include <linux/highmem.h>
 #include <linux/module.h>
 #include <linux/init.h>
@@ -58,6 +59,7 @@
 #include <linux/mlx5/vport.h>
 #include "mlx5_ib.h"
 #include "cmd.h"
+#include <linux/mlx5/vport.h>
 
 #define DRIVER_NAME "mlx5_ib"
 #define DRIVER_VERSION "5.0-0"
@@ -65,7 +67,6 @@
 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
 MODULE_DESCRIPTION("Mellanox Connect-IB HCA IB driver");
 MODULE_LICENSE("Dual BSD/GPL");
-MODULE_VERSION(DRIVER_VERSION);
 
 static char mlx5_version[] =
 	DRIVER_NAME ": Mellanox Connect-IB Infiniband driver v"
@@ -97,6 +98,20 @@ mlx5_ib_port_link_layer(struct ib_device *device, u8 port_num)
 	return mlx5_port_type_cap_to_rdma_ll(port_type_cap);
 }
 
+static int get_port_state(struct ib_device *ibdev,
+			  u8 port_num,
+			  enum ib_port_state *state)
+{
+	struct ib_port_attr attr;
+	int ret;
+
+	memset(&attr, 0, sizeof(attr));
+	ret = mlx5_ib_query_port(ibdev, port_num, &attr);
+	if (!ret)
+		*state = attr.state;
+	return ret;
+}
+
 static int mlx5_netdev_event(struct notifier_block *this,
 			     unsigned long event, void *ptr)
 {
@@ -114,6 +129,7 @@ static int mlx5_netdev_event(struct notifier_block *this,
 		write_unlock(&ibdev->roce.netdev_lock);
 		break;
 
+	case NETDEV_CHANGE:
 	case NETDEV_UP:
 	case NETDEV_DOWN: {
 		struct net_device *lag_ndev = mlx5_lag_get_roce_netdev(ibdev->mdev);
@@ -127,10 +143,23 @@ static int mlx5_netdev_event(struct notifier_block *this,
 		if ((upper == ndev || (!upper && ndev == ibdev->roce.netdev))
 		    && ibdev->ib_active) {
 			struct ib_event ibev = { };
+			enum ib_port_state port_state;
 
+			if (get_port_state(&ibdev->ib_dev, 1, &port_state))
+				return NOTIFY_DONE;
+
+			if (ibdev->roce.last_port_state == port_state)
+				return NOTIFY_DONE;
+
+			ibdev->roce.last_port_state = port_state;
 			ibev.device = &ibdev->ib_dev;
-			ibev.event = (event == NETDEV_UP) ?
-				     IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
+			if (port_state == IB_PORT_DOWN)
+				ibev.event = IB_EVENT_PORT_ERR;
+			else if (port_state == IB_PORT_ACTIVE)
+				ibev.event = IB_EVENT_PORT_ACTIVE;
+			else
+				return NOTIFY_DONE;
+
 			ibev.element.port_num = 1;
 			ib_dispatch_event(&ibev);
 		}
@@ -668,6 +697,14 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 		props->device_cap_flags |= IB_DEVICE_UD_TSO;
 	}
 
+	if (MLX5_CAP_GEN(dev->mdev, rq_delay_drop) &&
+	    MLX5_CAP_GEN(dev->mdev, general_notification_event))
+		props->raw_packet_caps |= IB_RAW_PACKET_CAP_DELAY_DROP;
+
+	if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads) &&
+	    MLX5_CAP_IPOIB_ENHANCED(mdev, csum_cap))
+		props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
+
 	if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
 	    MLX5_CAP_ETH(dev->mdev, scatter_fcs)) {
 		/* Legacy bit to support old userspace libraries */
@@ -740,6 +777,16 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 			1 << MLX5_CAP_GEN(dev->mdev, log_max_rq);
 	}
 
+	if (MLX5_CAP_GEN(mdev, tag_matching)) {
+		props->xrq_caps.max_rndv_hdr_size = MLX5_TM_MAX_RNDV_MSG_SIZE;
+		props->xrq_caps.max_num_tags =
+			(1 << MLX5_CAP_GEN(mdev, log_tag_matching_list_sz)) - 1;
+		props->xrq_caps.flags = IB_TM_CAP_RC;
+		props->xrq_caps.max_ops =
+			1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
+		props->xrq_caps.max_sge = MLX5_TM_MAX_SGE;
+	}
+
 	if (field_avail(typeof(resp), cqe_comp_caps, uhw->outlen)) {
 		resp.cqe_comp_caps.max_num =
 			MLX5_CAP_GEN(dev->mdev, cqe_compression) ?
@@ -765,8 +812,14 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 
 	if (field_avail(typeof(resp), mlx5_ib_support_multi_pkt_send_wqes,
 			uhw->outlen)) {
-		resp.mlx5_ib_support_multi_pkt_send_wqes =
-			MLX5_CAP_ETH(mdev, multi_pkt_send_wqe);
+		if (MLX5_CAP_ETH(mdev, multi_pkt_send_wqe))
+			resp.mlx5_ib_support_multi_pkt_send_wqes =
+				MLX5_IB_ALLOW_MPW;
+
+		if (MLX5_CAP_ETH(mdev, enhanced_multi_pkt_send_wqe))
+			resp.mlx5_ib_support_multi_pkt_send_wqes |=
+				MLX5_IB_SUPPORT_EMPW;
+
 		resp.response_length +=
 			sizeof(resp.mlx5_ib_support_multi_pkt_send_wqes);
 	}
@@ -774,6 +827,27 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 	if (field_avail(typeof(resp), reserved, uhw->outlen))
 		resp.response_length += sizeof(resp.reserved);
 
+	if (field_avail(typeof(resp), sw_parsing_caps,
+			uhw->outlen)) {
+		resp.response_length += sizeof(resp.sw_parsing_caps);
+		if (MLX5_CAP_ETH(mdev, swp)) {
+			resp.sw_parsing_caps.sw_parsing_offloads |=
+				MLX5_IB_SW_PARSING;
+
+			if (MLX5_CAP_ETH(mdev, swp_csum))
+				resp.sw_parsing_caps.sw_parsing_offloads |=
+					MLX5_IB_SW_PARSING_CSUM;
+
+			if (MLX5_CAP_ETH(mdev, swp_lso))
+				resp.sw_parsing_caps.sw_parsing_offloads |=
+					MLX5_IB_SW_PARSING_LSO;
+
+			if (resp.sw_parsing_caps.sw_parsing_offloads)
+				resp.sw_parsing_caps.supported_qpts =
+					BIT(IB_QPT_RAW_PACKET);
+		}
+	}
+
 	if (uhw->outlen) {
 		err = ib_copy_to_udata(uhw, &resp, resp.response_length);
 
@@ -1085,6 +1159,12 @@ static int mlx5_ib_modify_port(struct ib_device *ibdev, u8 port, int mask,
 	bool is_ib = (mlx5_ib_port_link_layer(ibdev, port) ==
 		      IB_LINK_LAYER_INFINIBAND);
 
+	/* CM layer calls ib_modify_port() regardless of the link layer. For
+	 * Ethernet ports, qkey violation and Port capabilities are meaningless.
+	 */
+	if (!is_ib)
+		return 0;
+
 	if (MLX5_CAP_GEN(dev->mdev, ib_virt) && is_ib) {
 		change_mask = props->clr_port_cap_mask | props->set_port_cap_mask;
 		value = ~props->clr_port_cap_mask | props->set_port_cap_mask;
@@ -1138,7 +1218,7 @@ static int calc_total_bfregs(struct mlx5_ib_dev *dev, bool lib_uar_4k,
 	if (req->num_low_latency_bfregs > req->total_num_bfregs - 1)
 		return -EINVAL;
 
-	mlx5_ib_dbg(dev, "uar_4k: fw support %s, lib support %s, user requested %d bfregs, alloated %d, using %d sys pages\n",
+	mlx5_ib_dbg(dev, "uar_4k: fw support %s, lib support %s, user requested %d bfregs, allocated %d, using %d sys pages\n",
 		    MLX5_CAP_GEN(dev->mdev, uar_4k) ? "yes" : "no",
 		    lib_uar_4k ? "yes" : "no", ref_bfregs,
 		    req->total_num_bfregs, *num_sys_pages);
@@ -1187,6 +1267,45 @@ static int deallocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *con
 	return 0;
 }
 
+static int mlx5_ib_alloc_transport_domain(struct mlx5_ib_dev *dev, u32 *tdn)
+{
+	int err;
+
+	err = mlx5_core_alloc_transport_domain(dev->mdev, tdn);
+	if (err)
+		return err;
+
+	if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
+	    !MLX5_CAP_GEN(dev->mdev, disable_local_lb))
+		return err;
+
+	mutex_lock(&dev->lb_mutex);
+	dev->user_td++;
+
+	if (dev->user_td == 2)
+		err = mlx5_nic_vport_update_local_lb(dev->mdev, true);
+
+	mutex_unlock(&dev->lb_mutex);
+	return err;
+}
+
+static void mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev *dev, u32 tdn)
+{
+	mlx5_core_dealloc_transport_domain(dev->mdev, tdn);
+
+	if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
+	    !MLX5_CAP_GEN(dev->mdev, disable_local_lb))
+		return;
+
+	mutex_lock(&dev->lb_mutex);
+	dev->user_td--;
+
+	if (dev->user_td < 2)
+		mlx5_nic_vport_update_local_lb(dev->mdev, false);
+
+	mutex_unlock(&dev->lb_mutex);
+}
+
 static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
 						  struct ib_udata *udata)
 {
@@ -1197,7 +1316,6 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
 	struct mlx5_bfreg_info *bfregi;
 	int ver;
 	int err;
-	size_t reqlen;
 	size_t min_req_v2 = offsetof(struct mlx5_ib_alloc_ucontext_req_v2,
 				     max_cqe_version);
 	bool lib_uar_4k;
@@ -1205,18 +1323,14 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
 	if (!dev->ib_active)
 		return ERR_PTR(-EAGAIN);
 
-	if (udata->inlen < sizeof(struct ib_uverbs_cmd_hdr))
-		return ERR_PTR(-EINVAL);
-
-	reqlen = udata->inlen - sizeof(struct ib_uverbs_cmd_hdr);
-	if (reqlen == sizeof(struct mlx5_ib_alloc_ucontext_req))
+	if (udata->inlen == sizeof(struct mlx5_ib_alloc_ucontext_req))
 		ver = 0;
-	else if (reqlen >= min_req_v2)
+	else if (udata->inlen >= min_req_v2)
 		ver = 2;
 	else
 		return ERR_PTR(-EINVAL);
 
-	err = ib_copy_from_udata(&req, udata, min(reqlen, sizeof(req)));
+	err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req)));
 	if (err)
 		return ERR_PTR(err);
 
@@ -1295,8 +1409,7 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
 	mutex_init(&context->upd_xlt_page_mutex);
 
 	if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain)) {
-		err = mlx5_core_alloc_transport_domain(dev->mdev,
-						       &context->tdn);
+		err = mlx5_ib_alloc_transport_domain(dev, &context->tdn);
 		if (err)
 			goto out_page;
 	}
@@ -1362,7 +1475,7 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
 
 out_td:
 	if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
-		mlx5_core_dealloc_transport_domain(dev->mdev, context->tdn);
+		mlx5_ib_dealloc_transport_domain(dev, context->tdn);
 
 out_page:
 	free_page(context->upd_xlt_page);
@@ -1390,7 +1503,7 @@ static int mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
 
 	bfregi = &context->bfregi;
 	if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
-		mlx5_core_dealloc_transport_domain(dev->mdev, context->tdn);
+		mlx5_ib_dealloc_transport_domain(dev, context->tdn);
 
 	free_page(context->upd_xlt_page);
 	deallocate_uars(dev, context);
@@ -2028,23 +2141,34 @@ static int parse_flow_attr(struct mlx5_core_dev *mdev, u32 *match_c,
  * it won't fall into the multicast flow steering table and this rule
  * could steal other multicast packets.
  */
-static bool flow_is_multicast_only(struct ib_flow_attr *ib_attr)
+static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr)
 {
-	struct ib_flow_spec_eth *eth_spec;
+	union ib_flow_spec *flow_spec;
 
 	if (ib_attr->type != IB_FLOW_ATTR_NORMAL ||
-	    ib_attr->size < sizeof(struct ib_flow_attr) +
-	    sizeof(struct ib_flow_spec_eth) ||
 	    ib_attr->num_of_specs < 1)
 		return false;
 
-	eth_spec = (struct ib_flow_spec_eth *)(ib_attr + 1);
-	if (eth_spec->type != IB_FLOW_SPEC_ETH ||
-	    eth_spec->size != sizeof(*eth_spec))
+	flow_spec = (union ib_flow_spec *)(ib_attr + 1);
+	if (flow_spec->type == IB_FLOW_SPEC_IPV4) {
+		struct ib_flow_spec_ipv4 *ipv4_spec;
+
+		ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec;
+		if (ipv4_is_multicast(ipv4_spec->val.dst_ip))
+			return true;
+
 		return false;
+	}
+
+	if (flow_spec->type == IB_FLOW_SPEC_ETH) {
+		struct ib_flow_spec_eth *eth_spec;
+
+		eth_spec = (struct ib_flow_spec_eth *)flow_spec;
+		return is_multicast_ether_addr(eth_spec->mask.dst_mac) &&
+		       is_multicast_ether_addr(eth_spec->val.dst_mac);
+	}
 
-	return is_multicast_ether_addr(eth_spec->mask.dst_mac) &&
-	       is_multicast_ether_addr(eth_spec->val.dst_mac);
+	return false;
 }
 
 static bool is_valid_ethertype(struct mlx5_core_dev *mdev,
@@ -2229,10 +2353,31 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
 	return err ? ERR_PTR(err) : prio;
 }
 
-static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,
-						     struct mlx5_ib_flow_prio *ft_prio,
-						     const struct ib_flow_attr *flow_attr,
-						     struct mlx5_flow_destination *dst)
+static void set_underlay_qp(struct mlx5_ib_dev *dev,
+			    struct mlx5_flow_spec *spec,
+			    u32 underlay_qpn)
+{
+	void *misc_params_c = MLX5_ADDR_OF(fte_match_param,
+					   spec->match_criteria,
+					   misc_parameters);
+	void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
+					   misc_parameters);
+
+	if (underlay_qpn &&
+	    MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
+				      ft_field_support.bth_dst_qp)) {
+		MLX5_SET(fte_match_set_misc,
+			 misc_params_v, bth_dst_qp, underlay_qpn);
+		MLX5_SET(fte_match_set_misc,
+			 misc_params_c, bth_dst_qp, 0xffffff);
+	}
+}
+
+static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev,
+						      struct mlx5_ib_flow_prio *ft_prio,
+						      const struct ib_flow_attr *flow_attr,
+						      struct mlx5_flow_destination *dst,
+						      u32 underlay_qpn)
 {
 	struct mlx5_flow_table	*ft = ft_prio->flow_table;
 	struct mlx5_ib_flow_handler *handler;
@@ -2268,6 +2413,9 @@ static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,
 		ib_flow += ((union ib_flow_spec *)ib_flow)->size;
 	}
 
+	if (!flow_is_multicast_only(flow_attr))
+		set_underlay_qp(dev, spec, underlay_qpn);
+
 	spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria);
 	if (is_drop) {
 		flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP;
@@ -2307,6 +2455,14 @@ free:
 	return err ? ERR_PTR(err) : handler;
 }
 
+static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,
+						     struct mlx5_ib_flow_prio *ft_prio,
+						     const struct ib_flow_attr *flow_attr,
+						     struct mlx5_flow_destination *dst)
+{
+	return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0);
+}
+
 static struct mlx5_ib_flow_handler *create_dont_trap_rule(struct mlx5_ib_dev *dev,
 							  struct mlx5_ib_flow_prio *ft_prio,
 							  struct ib_flow_attr *flow_attr,
@@ -2443,6 +2599,7 @@ static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
 	struct mlx5_ib_flow_prio *ft_prio_tx = NULL;
 	struct mlx5_ib_flow_prio *ft_prio;
 	int err;
+	int underlay_qpn;
 
 	if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO)
 		return ERR_PTR(-ENOMEM);
@@ -2483,8 +2640,10 @@ static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
 			handler = create_dont_trap_rule(dev, ft_prio,
 							flow_attr, dst);
 		} else {
-			handler = create_flow_rule(dev, ft_prio, flow_attr,
-						   dst);
+			underlay_qpn = (mqp->flags & MLX5_IB_QP_UNDERLAY) ?
+					mqp->underlay_qpn : 0;
+			handler = _create_flow_rule(dev, ft_prio, flow_attr,
+						    dst, underlay_qpn);
 		}
 	} else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
 		   flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) {
@@ -2522,8 +2681,14 @@ unlock:
 static int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
 {
 	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
+	struct mlx5_ib_qp *mqp = to_mqp(ibqp);
 	int err;
 
+	if (mqp->flags & MLX5_IB_QP_UNDERLAY) {
+		mlx5_ib_dbg(dev, "Attaching a multi cast group to underlay QP is not supported\n");
+		return -EOPNOTSUPP;
+	}
+
 	err = mlx5_core_attach_mcg(dev->mdev, gid, ibqp->qp_num);
 	if (err)
 		mlx5_ib_warn(dev, "failed attaching QPN 0x%x, MGID %pI6\n",
@@ -2685,6 +2850,26 @@ static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev)
 	spin_unlock_irqrestore(&ibdev->reset_flow_resource_lock, flags);
 }
 
+static void delay_drop_handler(struct work_struct *work)
+{
+	int err;
+	struct mlx5_ib_delay_drop *delay_drop =
+		container_of(work, struct mlx5_ib_delay_drop,
+			     delay_drop_work);
+
+	atomic_inc(&delay_drop->events_cnt);
+
+	mutex_lock(&delay_drop->lock);
+	err = mlx5_core_set_delay_drop(delay_drop->dev->mdev,
+				       delay_drop->timeout);
+	if (err) {
+		mlx5_ib_warn(delay_drop->dev, "Failed to set delay drop, timeout=%u\n",
+			     delay_drop->timeout);
+		delay_drop->activate = false;
+	}
+	mutex_unlock(&delay_drop->lock);
+}
+
 static void mlx5_ib_event(struct mlx5_core_dev *dev, void *context,
 			  enum mlx5_dev_event event, unsigned long param)
 {
@@ -2737,8 +2922,11 @@ static void mlx5_ib_event(struct mlx5_core_dev *dev, void *context,
 		ibev.event = IB_EVENT_CLIENT_REREGISTER;
 		port = (u8)param;
 		break;
+	case MLX5_DEV_EVENT_DELAY_DROP_TIMEOUT:
+		schedule_work(&ibdev->delay_drop.delay_drop_work);
+		goto out;
 	default:
-		return;
+		goto out;
 	}
 
 	ibev.device	      = &ibdev->ib_dev;
@@ -2746,7 +2934,7 @@ static void mlx5_ib_event(struct mlx5_core_dev *dev, void *context,
 
 	if (port < 1 || port > ibdev->num_ports) {
 		mlx5_ib_warn(ibdev, "warning: event on port %d\n", port);
-		return;
+		goto out;
 	}
 
 	if (ibdev->ib_active)
@@ -2754,6 +2942,9 @@ static void mlx5_ib_event(struct mlx5_core_dev *dev, void *context,
 
 	if (fatal)
 		ibdev->ib_active = false;
+
+out:
+	return;
 }
 
 static int set_has_smi_cap(struct mlx5_ib_dev *dev)
@@ -3036,7 +3227,7 @@ static int create_dev_resources(struct mlx5_ib_resources *devr)
 	attr.attr.max_sge = 1;
 	attr.attr.max_wr = 1;
 	attr.srq_type = IB_SRQT_XRC;
-	attr.ext.xrc.cq = devr->c0;
+	attr.ext.cq = devr->c0;
 	attr.ext.xrc.xrcd = devr->x0;
 
 	devr->s0 = mlx5_ib_create_srq(devr->p0, &attr, NULL);
@@ -3051,9 +3242,9 @@ static int create_dev_resources(struct mlx5_ib_resources *devr)
 	devr->s0->srq_context   = NULL;
 	devr->s0->srq_type      = IB_SRQT_XRC;
 	devr->s0->ext.xrc.xrcd	= devr->x0;
-	devr->s0->ext.xrc.cq	= devr->c0;
+	devr->s0->ext.cq	= devr->c0;
 	atomic_inc(&devr->s0->ext.xrc.xrcd->usecnt);
-	atomic_inc(&devr->s0->ext.xrc.cq->usecnt);
+	atomic_inc(&devr->s0->ext.cq->usecnt);
 	atomic_inc(&devr->p0->usecnt);
 	atomic_set(&devr->s0->usecnt, 0);
 
@@ -3072,9 +3263,9 @@ static int create_dev_resources(struct mlx5_ib_resources *devr)
 	devr->s1->event_handler = NULL;
 	devr->s1->srq_context   = NULL;
 	devr->s1->srq_type      = IB_SRQT_BASIC;
-	devr->s1->ext.xrc.cq	= devr->c0;
+	devr->s1->ext.cq	= devr->c0;
 	atomic_inc(&devr->p0->usecnt);
-	atomic_set(&devr->s0->usecnt, 0);
+	atomic_set(&devr->s1->usecnt, 0);
 
 	for (port = 0; port < ARRAY_SIZE(devr->ports); ++port) {
 		INIT_WORK(&devr->ports[port].pkey_change_work,
@@ -3167,13 +3358,13 @@ static int mlx5_port_immutable(struct ib_device *ibdev, u8 port_num,
 	return 0;
 }
 
-static void get_dev_fw_str(struct ib_device *ibdev, char *str,
-			   size_t str_len)
+static void get_dev_fw_str(struct ib_device *ibdev, char *str)
 {
 	struct mlx5_ib_dev *dev =
 		container_of(ibdev, struct mlx5_ib_dev, ib_dev);
-	snprintf(str, str_len, "%d.%d.%04d", fw_rev_maj(dev->mdev),
-		       fw_rev_min(dev->mdev), fw_rev_sub(dev->mdev));
+	snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%04d",
+		 fw_rev_maj(dev->mdev), fw_rev_min(dev->mdev),
+		 fw_rev_sub(dev->mdev));
 }
 
 static int mlx5_eth_lag_init(struct mlx5_ib_dev *dev)
@@ -3313,6 +3504,17 @@ static const struct mlx5_ib_counter cong_cnts[] = {
 	INIT_CONG_COUNTER(np_cnp_sent),
 };
 
+static const struct mlx5_ib_counter extended_err_cnts[] = {
+	INIT_Q_COUNTER(resp_local_length_error),
+	INIT_Q_COUNTER(resp_cqe_error),
+	INIT_Q_COUNTER(req_cqe_error),
+	INIT_Q_COUNTER(req_remote_invalid_request),
+	INIT_Q_COUNTER(req_remote_access_errors),
+	INIT_Q_COUNTER(resp_remote_access_errors),
+	INIT_Q_COUNTER(resp_cqe_flush_error),
+	INIT_Q_COUNTER(req_cqe_flush_error),
+};
+
 static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev)
 {
 	unsigned int i;
@@ -3337,6 +3539,10 @@ static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev,
 
 	if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters))
 		num_counters += ARRAY_SIZE(retrans_q_cnts);
+
+	if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters))
+		num_counters += ARRAY_SIZE(extended_err_cnts);
+
 	cnts->num_q_counters = num_counters;
 
 	if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
@@ -3386,6 +3592,13 @@ static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev,
 		}
 	}
 
+	if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) {
+		for (i = 0; i < ARRAY_SIZE(extended_err_cnts); i++, j++) {
+			names[j] = extended_err_cnts[i].name;
+			offsets[j] = extended_err_cnts[i].offset;
+		}
+	}
+
 	if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
 		for (i = 0; i < ARRAY_SIZE(cong_cnts); i++, j++) {
 			names[j] = cong_cnts[i].name;
@@ -3556,6 +3769,136 @@ mlx5_ib_alloc_rdma_netdev(struct ib_device *hca,
 	return netdev;
 }
 
+static void delay_drop_debugfs_cleanup(struct mlx5_ib_dev *dev)
+{
+	if (!dev->delay_drop.dbg)
+		return;
+	debugfs_remove_recursive(dev->delay_drop.dbg->dir_debugfs);
+	kfree(dev->delay_drop.dbg);
+	dev->delay_drop.dbg = NULL;
+}
+
+static void cancel_delay_drop(struct mlx5_ib_dev *dev)
+{
+	if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
+		return;
+
+	cancel_work_sync(&dev->delay_drop.delay_drop_work);
+	delay_drop_debugfs_cleanup(dev);
+}
+
+static ssize_t delay_drop_timeout_read(struct file *filp, char __user *buf,
+				       size_t count, loff_t *pos)
+{
+	struct mlx5_ib_delay_drop *delay_drop = filp->private_data;
+	char lbuf[20];
+	int len;
+
+	len = snprintf(lbuf, sizeof(lbuf), "%u\n", delay_drop->timeout);
+	return simple_read_from_buffer(buf, count, pos, lbuf, len);
+}
+
+static ssize_t delay_drop_timeout_write(struct file *filp, const char __user *buf,
+					size_t count, loff_t *pos)
+{
+	struct mlx5_ib_delay_drop *delay_drop = filp->private_data;
+	u32 timeout;
+	u32 var;
+
+	if (kstrtouint_from_user(buf, count, 0, &var))
+		return -EFAULT;
+
+	timeout = min_t(u32, roundup(var, 100), MLX5_MAX_DELAY_DROP_TIMEOUT_MS *
+			1000);
+	if (timeout != var)
+		mlx5_ib_dbg(delay_drop->dev, "Round delay drop timeout to %u usec\n",
+			    timeout);
+
+	delay_drop->timeout = timeout;
+
+	return count;
+}
+
+static const struct file_operations fops_delay_drop_timeout = {
+	.owner	= THIS_MODULE,
+	.open	= simple_open,
+	.write	= delay_drop_timeout_write,
+	.read	= delay_drop_timeout_read,
+};
+
+static int delay_drop_debugfs_init(struct mlx5_ib_dev *dev)
+{
+	struct mlx5_ib_dbg_delay_drop *dbg;
+
+	if (!mlx5_debugfs_root)
+		return 0;
+
+	dbg = kzalloc(sizeof(*dbg), GFP_KERNEL);
+	if (!dbg)
+		return -ENOMEM;
+
+	dbg->dir_debugfs =
+		debugfs_create_dir("delay_drop",
+				   dev->mdev->priv.dbg_root);
+	if (!dbg->dir_debugfs)
+		return -ENOMEM;
+
+	dbg->events_cnt_debugfs =
+		debugfs_create_atomic_t("num_timeout_events", 0400,
+					dbg->dir_debugfs,
+					&dev->delay_drop.events_cnt);
+	if (!dbg->events_cnt_debugfs)
+		goto out_debugfs;
+
+	dbg->rqs_cnt_debugfs =
+		debugfs_create_atomic_t("num_rqs", 0400,
+					dbg->dir_debugfs,
+					&dev->delay_drop.rqs_cnt);
+	if (!dbg->rqs_cnt_debugfs)
+		goto out_debugfs;
+
+	dbg->timeout_debugfs =
+		debugfs_create_file("timeout", 0600,
+				    dbg->dir_debugfs,
+				    &dev->delay_drop,
+				    &fops_delay_drop_timeout);
+	if (!dbg->timeout_debugfs)
+		goto out_debugfs;
+
+	dev->delay_drop.dbg = dbg;
+
+	return 0;
+
+out_debugfs:
+	delay_drop_debugfs_cleanup(dev);
+	return -ENOMEM;
+}
+
+static void init_delay_drop(struct mlx5_ib_dev *dev)
+{
+	if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
+		return;
+
+	mutex_init(&dev->delay_drop.lock);
+	dev->delay_drop.dev = dev;
+	dev->delay_drop.activate = false;
+	dev->delay_drop.timeout = MLX5_MAX_DELAY_DROP_TIMEOUT_MS * 1000;
+	INIT_WORK(&dev->delay_drop.delay_drop_work, delay_drop_handler);
+	atomic_set(&dev->delay_drop.rqs_cnt, 0);
+	atomic_set(&dev->delay_drop.events_cnt, 0);
+
+	if (delay_drop_debugfs_init(dev))
+		mlx5_ib_warn(dev, "Failed to init delay drop debugfs\n");
+}
+
+static const struct cpumask *
+mlx5_ib_get_vector_affinity(struct ib_device *ibdev, int comp_vector)
+{
+	struct mlx5_ib_dev *dev = to_mdev(ibdev);
+
+	return mlx5_get_vector_affinity(dev->mdev, comp_vector);
+}
+
 static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 {
 	struct mlx5_ib_dev *dev;
@@ -3686,6 +4029,7 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 	dev->ib_dev.check_mr_status	= mlx5_ib_check_mr_status;
 	dev->ib_dev.get_port_immutable  = mlx5_port_immutable;
 	dev->ib_dev.get_dev_fw_str      = get_dev_fw_str;
+	dev->ib_dev.get_vector_affinity	= mlx5_ib_get_vector_affinity;
 	if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads))
 		dev->ib_dev.alloc_rdma_netdev	= mlx5_ib_alloc_rdma_netdev;
 
@@ -3723,18 +4067,20 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 			(1ull << IB_USER_VERBS_CMD_CLOSE_XRCD);
 	}
 
+	dev->ib_dev.create_flow	= mlx5_ib_create_flow;
+	dev->ib_dev.destroy_flow = mlx5_ib_destroy_flow;
+	dev->ib_dev.uverbs_ex_cmd_mask |=
+			(1ull << IB_USER_VERBS_EX_CMD_CREATE_FLOW) |
+			(1ull << IB_USER_VERBS_EX_CMD_DESTROY_FLOW);
+
 	if (mlx5_ib_port_link_layer(&dev->ib_dev, 1) ==
 	    IB_LINK_LAYER_ETHERNET) {
-		dev->ib_dev.create_flow	= mlx5_ib_create_flow;
-		dev->ib_dev.destroy_flow = mlx5_ib_destroy_flow;
 		dev->ib_dev.create_wq	 = mlx5_ib_create_wq;
 		dev->ib_dev.modify_wq	 = mlx5_ib_modify_wq;
 		dev->ib_dev.destroy_wq	 = mlx5_ib_destroy_wq;
 		dev->ib_dev.create_rwq_ind_table = mlx5_ib_create_rwq_ind_table;
 		dev->ib_dev.destroy_rwq_ind_table = mlx5_ib_destroy_rwq_ind_table;
 		dev->ib_dev.uverbs_ex_cmd_mask |=
-			(1ull << IB_USER_VERBS_EX_CMD_CREATE_FLOW) |
-			(1ull << IB_USER_VERBS_EX_CMD_DESTROY_FLOW) |
 			(1ull << IB_USER_VERBS_EX_CMD_CREATE_WQ) |
 			(1ull << IB_USER_VERBS_EX_CMD_MODIFY_WQ) |
 			(1ull << IB_USER_VERBS_EX_CMD_DESTROY_WQ) |
@@ -3754,6 +4100,7 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 		err = mlx5_enable_eth(dev);
 		if (err)
 			goto err_free_port;
+		dev->roce.last_port_state = IB_PORT_DOWN;
 	}
 
 	err = create_dev_resources(&dev->devr);
@@ -3770,9 +4117,13 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 			goto err_odp;
 	}
 
+	err = mlx5_ib_init_cong_debugfs(dev);
+	if (err)
+		goto err_cnt;
+
 	dev->mdev->priv.uar = mlx5_get_uars_page(dev->mdev);
 	if (!dev->mdev->priv.uar)
-		goto err_cnt;
+		goto err_cong;
 
 	err = mlx5_alloc_bfreg(dev->mdev, &dev->bfreg, false, false);
 	if (err)
@@ -3790,18 +4141,25 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 	if (err)
 		goto err_dev;
 
+	init_delay_drop(dev);
+
 	for (i = 0; i < ARRAY_SIZE(mlx5_class_attributes); i++) {
 		err = device_create_file(&dev->ib_dev.dev,
 					 mlx5_class_attributes[i]);
 		if (err)
-			goto err_umrc;
+			goto err_delay_drop;
 	}
 
+	if ((MLX5_CAP_GEN(mdev, port_type) == MLX5_CAP_PORT_TYPE_ETH) &&
+	    MLX5_CAP_GEN(mdev, disable_local_lb))
+		mutex_init(&dev->lb_mutex);
+
 	dev->ib_active = true;
 
 	return dev;
 
-err_umrc:
+err_delay_drop:
+	cancel_delay_drop(dev);
 	destroy_umrc_res(dev);
 
 err_dev:
@@ -3817,6 +4175,8 @@ err_uar_page:
 	mlx5_put_uars_page(dev->mdev, dev->mdev->priv.uar);
 
 err_cnt:
+	mlx5_ib_cleanup_cong_debugfs(dev);
+err_cong:
 	if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
 		mlx5_ib_dealloc_counters(dev);
 
@@ -3846,11 +4206,13 @@ static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context)
 	struct mlx5_ib_dev *dev = context;
 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev, 1);
 
+	cancel_delay_drop(dev);
 	mlx5_remove_netdev_notifier(dev);
 	ib_unregister_device(&dev->ib_dev);
 	mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
 	mlx5_free_bfreg(dev->mdev, &dev->bfreg);
 	mlx5_put_uars_page(dev->mdev, mdev->priv.uar);
+	mlx5_ib_cleanup_cong_debugfs(dev);
 	if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
 		mlx5_ib_dealloc_counters(dev);
 	destroy_umrc_res(dev);
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index bdcf25410c99..189e80cd6b2f 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -107,6 +107,11 @@ enum {
 	MLX5_CQE_VERSION_V1,
 };
 
+enum {
+	MLX5_TM_MAX_RNDV_MSG_SIZE	= 64,
+	MLX5_TM_MAX_SGE			= 1,
+};
+
 struct mlx5_ib_vma_private_data {
 	struct list_head list;
 	struct vm_area_struct *vma;
@@ -247,6 +252,10 @@ struct mlx5_ib_wq {
 	void		       *qend;
 };
 
+enum mlx5_ib_wq_flags {
+	MLX5_IB_WQ_FLAGS_DELAY_DROP = 0x1,
+};
+
 struct mlx5_ib_rwq {
 	struct ib_wq		ibwq;
 	struct mlx5_core_qp	core_qp;
@@ -264,6 +273,7 @@ struct mlx5_ib_rwq {
 	u32			wqe_count;
 	u32			wqe_shift;
 	int			wq_sig;
+	u32			create_flags; /* Use enum mlx5_ib_wq_flags */
 };
 
 enum {
@@ -378,6 +388,7 @@ struct mlx5_ib_qp {
 	struct list_head	cq_recv_list;
 	struct list_head	cq_send_list;
 	u32			rate_limit;
+	u32                     underlay_qpn;
 };
 
 struct mlx5_ib_cq_buf {
@@ -399,6 +410,7 @@ enum mlx5_ib_qp_flags {
 	MLX5_IB_QP_CAP_SCATTER_FCS		= 1 << 7,
 	MLX5_IB_QP_RSS				= 1 << 8,
 	MLX5_IB_QP_CVLAN_STRIPPING		= 1 << 9,
+	MLX5_IB_QP_UNDERLAY			= 1 << 10,
 };
 
 struct mlx5_umr_wr {
@@ -496,7 +508,7 @@ struct mlx5_ib_mr {
 	struct mlx5_shared_mr_info	*smr_info;
 	struct list_head	list;
 	int			order;
-	int			umred;
+	bool			allocated_from_cache;
 	int			npages;
 	struct mlx5_ib_dev     *dev;
 	u32 out[MLX5_ST_SZ_DW(create_mkey_out)];
@@ -616,6 +628,63 @@ struct mlx5_roce {
 	struct net_device	*netdev;
 	struct notifier_block	nb;
 	atomic_t		next_port;
+	enum ib_port_state last_port_state;
+};
+
+struct mlx5_ib_dbg_param {
+	int			offset;
+	struct mlx5_ib_dev	*dev;
+	struct dentry		*dentry;
+};
+
+enum mlx5_ib_dbg_cc_types {
+	MLX5_IB_DBG_CC_RP_CLAMP_TGT_RATE,
+	MLX5_IB_DBG_CC_RP_CLAMP_TGT_RATE_ATI,
+	MLX5_IB_DBG_CC_RP_TIME_RESET,
+	MLX5_IB_DBG_CC_RP_BYTE_RESET,
+	MLX5_IB_DBG_CC_RP_THRESHOLD,
+	MLX5_IB_DBG_CC_RP_AI_RATE,
+	MLX5_IB_DBG_CC_RP_HAI_RATE,
+	MLX5_IB_DBG_CC_RP_MIN_DEC_FAC,
+	MLX5_IB_DBG_CC_RP_MIN_RATE,
+	MLX5_IB_DBG_CC_RP_RATE_TO_SET_ON_FIRST_CNP,
+	MLX5_IB_DBG_CC_RP_DCE_TCP_G,
+	MLX5_IB_DBG_CC_RP_DCE_TCP_RTT,
+	MLX5_IB_DBG_CC_RP_RATE_REDUCE_MONITOR_PERIOD,
+	MLX5_IB_DBG_CC_RP_INITIAL_ALPHA_VALUE,
+	MLX5_IB_DBG_CC_RP_GD,
+	MLX5_IB_DBG_CC_NP_CNP_DSCP,
+	MLX5_IB_DBG_CC_NP_CNP_PRIO_MODE,
+	MLX5_IB_DBG_CC_NP_CNP_PRIO,
+	MLX5_IB_DBG_CC_MAX,
+};
+
+struct mlx5_ib_dbg_cc_params {
+	struct dentry			*root;
+	struct mlx5_ib_dbg_param	params[MLX5_IB_DBG_CC_MAX];
+};
+
+enum {
+	MLX5_MAX_DELAY_DROP_TIMEOUT_MS = 100,
+};
+
+struct mlx5_ib_dbg_delay_drop {
+	struct dentry		*dir_debugfs;
+	struct dentry		*rqs_cnt_debugfs;
+	struct dentry		*events_cnt_debugfs;
+	struct dentry		*timeout_debugfs;
+};
+
+struct mlx5_ib_delay_drop {
+	struct mlx5_ib_dev     *dev;
+	struct work_struct	delay_drop_work;
+	/* serialize setting of delay drop */
+	struct mutex		lock;
+	u32			timeout;
+	bool			activate;
+	atomic_t		events_cnt;
+	atomic_t		rqs_cnt;
+	struct mlx5_ib_dbg_delay_drop *dbg;
 };
 
 struct mlx5_ib_dev {
@@ -652,9 +721,15 @@ struct mlx5_ib_dev {
 	struct list_head	qp_list;
 	/* Array with num_ports elements */
 	struct mlx5_ib_port	*port;
-	struct mlx5_sq_bfreg     bfreg;
-	struct mlx5_sq_bfreg     fp_bfreg;
-	u8				umr_fence;
+	struct mlx5_sq_bfreg	bfreg;
+	struct mlx5_sq_bfreg	fp_bfreg;
+	struct mlx5_ib_delay_drop	delay_drop;
+	struct mlx5_ib_dbg_cc_params	*dbg_cc_params;
+
+	/* protect the user_td */
+	struct mutex		lb_mutex;
+	u32			user_td;
+	u8			umr_fence;
 };
 
 static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)
@@ -904,6 +979,9 @@ __be16 mlx5_get_roce_udp_sport(struct mlx5_ib_dev *dev, u8 port_num,
 int mlx5_get_roce_gid_type(struct mlx5_ib_dev *dev, u8 port_num,
 			   int index, enum ib_gid_type *gid_type);
 
+void mlx5_ib_cleanup_cong_debugfs(struct mlx5_ib_dev *dev);
+int mlx5_ib_init_cong_debugfs(struct mlx5_ib_dev *dev);
+
 /* GSI QP helper functions */
 struct ib_qp *mlx5_ib_gsi_create_qp(struct ib_pd *pd,
 				    struct ib_qp_init_attr *init_attr);
diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index 2c40a2e989d2..0e2789d9bb4d 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -48,7 +48,7 @@ enum {
 #define MLX5_UMR_ALIGN 2048
 
 static int clean_mr(struct mlx5_ib_mr *mr);
-static int use_umr(struct mlx5_ib_dev *dev, int order);
+static int mr_cache_max_order(struct mlx5_ib_dev *dev);
 static int unreg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
 
 static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
@@ -183,7 +183,7 @@ static int add_keys(struct mlx5_ib_dev *dev, int c, int num)
 			break;
 		}
 		mr->order = ent->order;
-		mr->umred = 1;
+		mr->allocated_from_cache = 1;
 		mr->dev = dev;
 
 		MLX5_SET(mkc, mkc, free, 1);
@@ -491,16 +491,18 @@ static struct mlx5_ib_mr *alloc_cached_mr(struct mlx5_ib_dev *dev, int order)
 	struct mlx5_mr_cache *cache = &dev->cache;
 	struct mlx5_ib_mr *mr = NULL;
 	struct mlx5_cache_ent *ent;
+	int last_umr_cache_entry;
 	int c;
 	int i;
 
 	c = order2idx(dev, order);
-	if (c < 0 || c > MAX_UMR_CACHE_ENTRY) {
+	last_umr_cache_entry = order2idx(dev, mr_cache_max_order(dev));
+	if (c < 0 || c > last_umr_cache_entry) {
 		mlx5_ib_warn(dev, "order %d, cache index %d\n", order, c);
 		return NULL;
 	}
 
-	for (i = c; i < MAX_UMR_CACHE_ENTRY; i++) {
+	for (i = c; i <= last_umr_cache_entry; i++) {
 		ent = &cache->ent[i];
 
 		mlx5_ib_dbg(dev, "order %d, cache index %d\n", ent->order, i);
@@ -674,12 +676,12 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)
 		INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func);
 		queue_work(cache->wq, &ent->work);
 
-		if (i > MAX_UMR_CACHE_ENTRY) {
+		if (i > MR_CACHE_LAST_STD_ENTRY) {
 			mlx5_odp_init_mr_cache_entry(ent);
 			continue;
 		}
 
-		if (!use_umr(dev, ent->order))
+		if (ent->order > mr_cache_max_order(dev))
 			continue;
 
 		ent->page = PAGE_SHIFT;
@@ -806,21 +808,22 @@ err_free:
 	return ERR_PTR(err);
 }
 
-static int get_octo_len(u64 addr, u64 len, int page_size)
+static int get_octo_len(u64 addr, u64 len, int page_shift)
 {
+	u64 page_size = 1ULL << page_shift;
 	u64 offset;
 	int npages;
 
 	offset = addr & (page_size - 1);
-	npages = ALIGN(len + offset, page_size) >> ilog2(page_size);
+	npages = ALIGN(len + offset, page_size) >> page_shift;
 	return (npages + 1) / 2;
 }
 
-static int use_umr(struct mlx5_ib_dev *dev, int order)
+static int mr_cache_max_order(struct mlx5_ib_dev *dev)
 {
 	if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset))
-		return order <= MAX_UMR_CACHE_ENTRY + 2;
-	return order <= MLX5_MAX_UMR_SHIFT;
+		return MR_CACHE_LAST_STD_ENTRY + 2;
+	return MLX5_MAX_UMR_SHIFT;
 }
 
 static int mr_umem_get(struct ib_pd *pd, u64 start, u64 length,
@@ -896,7 +899,8 @@ static int mlx5_ib_post_send_wait(struct mlx5_ib_dev *dev,
 	return err;
 }
 
-static struct mlx5_ib_mr *reg_umr(struct ib_pd *pd, struct ib_umem *umem,
+static struct mlx5_ib_mr *alloc_mr_from_cache(
+				  struct ib_pd *pd, struct ib_umem *umem,
 				  u64 virt_addr, u64 len, int npages,
 				  int page_shift, int order, int access_flags)
 {
@@ -928,16 +932,6 @@ static struct mlx5_ib_mr *reg_umr(struct ib_pd *pd, struct ib_umem *umem,
 	mr->mmkey.size = len;
 	mr->mmkey.pd = to_mpd(pd)->pdn;
 
-	err = mlx5_ib_update_xlt(mr, 0, npages, page_shift,
-				 MLX5_IB_UPD_XLT_ENABLE);
-
-	if (err) {
-		mlx5_mr_cache_free(dev, mr);
-		return ERR_PTR(err);
-	}
-
-	mr->live = 1;
-
 	return mr;
 }
 
@@ -1103,7 +1097,8 @@ free_xlt:
 static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,
 				     u64 virt_addr, u64 length,
 				     struct ib_umem *umem, int npages,
-				     int page_shift, int access_flags)
+				     int page_shift, int access_flags,
+				     bool populate)
 {
 	struct mlx5_ib_dev *dev = to_mdev(pd->device);
 	struct mlx5_ib_mr *mr;
@@ -1118,15 +1113,19 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,
 	if (!mr)
 		return ERR_PTR(-ENOMEM);
 
-	inlen = MLX5_ST_SZ_BYTES(create_mkey_in) +
-		sizeof(*pas) * ((npages + 1) / 2) * 2;
+	mr->ibmr.pd = pd;
+	mr->access_flags = access_flags;
+
+	inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
+	if (populate)
+		inlen += sizeof(*pas) * roundup(npages, 2);
 	in = kvzalloc(inlen, GFP_KERNEL);
 	if (!in) {
 		err = -ENOMEM;
 		goto err_1;
 	}
 	pas = (__be64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
-	if (!(access_flags & IB_ACCESS_ON_DEMAND))
+	if (populate && !(access_flags & IB_ACCESS_ON_DEMAND))
 		mlx5_ib_populate_pas(dev, umem, page_shift, pas,
 				     pg_cap ? MLX5_IB_MTT_PRESENT : 0);
 
@@ -1135,23 +1134,27 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,
 	MLX5_SET(create_mkey_in, in, pg_access, !!(pg_cap));
 
 	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
+	MLX5_SET(mkc, mkc, free, !populate);
 	MLX5_SET(mkc, mkc, access_mode, MLX5_MKC_ACCESS_MODE_MTT);
 	MLX5_SET(mkc, mkc, a, !!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
 	MLX5_SET(mkc, mkc, rw, !!(access_flags & IB_ACCESS_REMOTE_WRITE));
 	MLX5_SET(mkc, mkc, rr, !!(access_flags & IB_ACCESS_REMOTE_READ));
 	MLX5_SET(mkc, mkc, lw, !!(access_flags & IB_ACCESS_LOCAL_WRITE));
 	MLX5_SET(mkc, mkc, lr, 1);
+	MLX5_SET(mkc, mkc, umr_en, 1);
 
 	MLX5_SET64(mkc, mkc, start_addr, virt_addr);
 	MLX5_SET64(mkc, mkc, len, length);
 	MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
 	MLX5_SET(mkc, mkc, bsf_octword_size, 0);
 	MLX5_SET(mkc, mkc, translations_octword_size,
-		 get_octo_len(virt_addr, length, 1 << page_shift));
+		 get_octo_len(virt_addr, length, page_shift));
 	MLX5_SET(mkc, mkc, log_page_size, page_shift);
 	MLX5_SET(mkc, mkc, qpn, 0xffffff);
-	MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
-		 get_octo_len(virt_addr, length, 1 << page_shift));
+	if (populate) {
+		MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
+			 get_octo_len(virt_addr, length, page_shift));
+	}
 
 	err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen);
 	if (err) {
@@ -1160,9 +1163,7 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,
 	}
 	mr->mmkey.type = MLX5_MKEY_MR;
 	mr->desc_size = sizeof(struct mlx5_mtt);
-	mr->umem = umem;
 	mr->dev = dev;
-	mr->live = 1;
 	kvfree(in);
 
 	mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr->mmkey.key);
@@ -1202,6 +1203,7 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 	int ncont;
 	int order;
 	int err;
+	bool use_umr = true;
 
 	mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",
 		    start, virt_addr, length, access_flags);
@@ -1220,27 +1222,29 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 	err = mr_umem_get(pd, start, length, access_flags, &umem, &npages,
 			   &page_shift, &ncont, &order);
 
-        if (err < 0)
+	if (err < 0)
 		return ERR_PTR(err);
 
-	if (use_umr(dev, order)) {
-		mr = reg_umr(pd, umem, virt_addr, length, ncont, page_shift,
-			     order, access_flags);
+	if (order <= mr_cache_max_order(dev)) {
+		mr = alloc_mr_from_cache(pd, umem, virt_addr, length, ncont,
+					 page_shift, order, access_flags);
 		if (PTR_ERR(mr) == -EAGAIN) {
 			mlx5_ib_dbg(dev, "cache empty for order %d", order);
 			mr = NULL;
 		}
-	} else if (access_flags & IB_ACCESS_ON_DEMAND &&
-		   !MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) {
-		err = -EINVAL;
-		pr_err("Got MR registration for ODP MR > 512MB, not supported for Connect-IB");
-		goto error;
+	} else if (!MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) {
+		if (access_flags & IB_ACCESS_ON_DEMAND) {
+			err = -EINVAL;
+			pr_err("Got MR registration for ODP MR > 512MB, not supported for Connect-IB");
+			goto error;
+		}
+		use_umr = false;
 	}
 
 	if (!mr) {
 		mutex_lock(&dev->slow_path_mutex);
 		mr = reg_create(NULL, pd, virt_addr, length, umem, ncont,
-				page_shift, access_flags);
+				page_shift, access_flags, !use_umr);
 		mutex_unlock(&dev->slow_path_mutex);
 	}
 
@@ -1258,8 +1262,22 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 	update_odp_mr(mr);
 #endif
 
-	return &mr->ibmr;
+	if (use_umr) {
+		int update_xlt_flags = MLX5_IB_UPD_XLT_ENABLE;
 
+		if (access_flags & IB_ACCESS_ON_DEMAND)
+			update_xlt_flags |= MLX5_IB_UPD_XLT_ZAP;
+
+		err = mlx5_ib_update_xlt(mr, 0, ncont, page_shift,
+					 update_xlt_flags);
+		if (err) {
+			mlx5_ib_dereg_mr(&mr->ibmr);
+			return ERR_PTR(err);
+		}
+	}
+
+	mr->live = 1;
+	return &mr->ibmr;
 error:
 	ib_umem_release(umem);
 	return ERR_PTR(err);
@@ -1347,7 +1365,7 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
 		/*
 		 * UMR can't be used - MKey needs to be replaced.
 		 */
-		if (mr->umred) {
+		if (mr->allocated_from_cache) {
 			err = unreg_umr(dev, mr);
 			if (err)
 				mlx5_ib_warn(dev, "Failed to unregister MR\n");
@@ -1360,12 +1378,13 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
 			return err;
 
 		mr = reg_create(ib_mr, pd, addr, len, mr->umem, ncont,
-				page_shift, access_flags);
+				page_shift, access_flags, true);
 
 		if (IS_ERR(mr))
 			return PTR_ERR(mr);
 
-		mr->umred = 0;
+		mr->allocated_from_cache = 0;
+		mr->live = 1;
 	} else {
 		/*
 		 * Send a UMR WQE
@@ -1453,7 +1472,7 @@ mlx5_free_priv_descs(struct mlx5_ib_mr *mr)
 static int clean_mr(struct mlx5_ib_mr *mr)
 {
 	struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.device);
-	int umred = mr->umred;
+	int allocated_from_cache = mr->allocated_from_cache;
 	int err;
 
 	if (mr->sig) {
@@ -1471,20 +1490,20 @@ static int clean_mr(struct mlx5_ib_mr *mr)
 
 	mlx5_free_priv_descs(mr);
 
-	if (!umred) {
+	if (!allocated_from_cache) {
+		u32 key = mr->mmkey.key;
+
 		err = destroy_mkey(dev, mr);
+		kfree(mr);
 		if (err) {
 			mlx5_ib_warn(dev, "failed to destroy mkey 0x%x (%d)\n",
-				     mr->mmkey.key, err);
+				     key, err);
 			return err;
 		}
 	} else {
 		mlx5_mr_cache_free(dev, mr);
 	}
 
-	if (!umred)
-		kfree(mr);
-
 	return 0;
 }
 
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index 0889ff367c86..acb79d3a4f1d 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -34,6 +34,7 @@
 #include <rdma/ib_umem.h>
 #include <rdma/ib_cache.h>
 #include <rdma/ib_user_verbs.h>
+#include <linux/mlx5/fs.h>
 #include "mlx5_ib.h"
 
 /* not supported currently */
@@ -453,7 +454,8 @@ static int set_user_buf_size(struct mlx5_ib_dev *dev,
 		return -EINVAL;
 	}
 
-	if (attr->qp_type == IB_QPT_RAW_PACKET) {
+	if (attr->qp_type == IB_QPT_RAW_PACKET ||
+	    qp->flags & MLX5_IB_QP_UNDERLAY) {
 		base->ubuffer.buf_size = qp->rq.wqe_cnt << qp->rq.wqe_shift;
 		qp->raw_packet_qp.sq.ubuffer.buf_size = qp->sq.wqe_cnt << 6;
 	} else {
@@ -675,10 +677,14 @@ err_umem:
 	return err;
 }
 
-static void destroy_user_rq(struct ib_pd *pd, struct mlx5_ib_rwq *rwq)
+static void destroy_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,
+			    struct mlx5_ib_rwq *rwq)
 {
 	struct mlx5_ib_ucontext *context;
 
+	if (rwq->create_flags & MLX5_IB_WQ_FLAGS_DELAY_DROP)
+		atomic_dec(&dev->delay_drop.rqs_cnt);
+
 	context = to_mucontext(pd->uobject->context);
 	mlx5_ib_db_unmap_user(context, &rwq->db);
 	if (rwq->umem)
@@ -959,11 +965,16 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev,
 		goto err_free;
 	}
 
-	qp->sq.wrid = kmalloc(qp->sq.wqe_cnt * sizeof(*qp->sq.wrid), GFP_KERNEL);
-	qp->sq.wr_data = kmalloc(qp->sq.wqe_cnt * sizeof(*qp->sq.wr_data), GFP_KERNEL);
-	qp->rq.wrid = kmalloc(qp->rq.wqe_cnt * sizeof(*qp->rq.wrid), GFP_KERNEL);
-	qp->sq.w_list = kmalloc(qp->sq.wqe_cnt * sizeof(*qp->sq.w_list), GFP_KERNEL);
-	qp->sq.wqe_head = kmalloc(qp->sq.wqe_cnt * sizeof(*qp->sq.wqe_head), GFP_KERNEL);
+	qp->sq.wrid = kvmalloc_array(qp->sq.wqe_cnt,
+				     sizeof(*qp->sq.wrid), GFP_KERNEL);
+	qp->sq.wr_data = kvmalloc_array(qp->sq.wqe_cnt,
+					sizeof(*qp->sq.wr_data), GFP_KERNEL);
+	qp->rq.wrid = kvmalloc_array(qp->rq.wqe_cnt,
+				     sizeof(*qp->rq.wrid), GFP_KERNEL);
+	qp->sq.w_list = kvmalloc_array(qp->sq.wqe_cnt,
+				       sizeof(*qp->sq.w_list), GFP_KERNEL);
+	qp->sq.wqe_head = kvmalloc_array(qp->sq.wqe_cnt,
+					 sizeof(*qp->sq.wqe_head), GFP_KERNEL);
 
 	if (!qp->sq.wrid || !qp->sq.wr_data || !qp->rq.wrid ||
 	    !qp->sq.w_list || !qp->sq.wqe_head) {
@@ -975,11 +986,11 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev,
 	return 0;
 
 err_wrid:
-	kfree(qp->sq.wqe_head);
-	kfree(qp->sq.w_list);
-	kfree(qp->sq.wrid);
-	kfree(qp->sq.wr_data);
-	kfree(qp->rq.wrid);
+	kvfree(qp->sq.wqe_head);
+	kvfree(qp->sq.w_list);
+	kvfree(qp->sq.wrid);
+	kvfree(qp->sq.wr_data);
+	kvfree(qp->rq.wrid);
 	mlx5_db_free(dev->mdev, &qp->db);
 
 err_free:
@@ -992,11 +1003,11 @@ err_buf:
 
 static void destroy_qp_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
 {
-	kfree(qp->sq.wqe_head);
-	kfree(qp->sq.w_list);
-	kfree(qp->sq.wrid);
-	kfree(qp->sq.wr_data);
-	kfree(qp->rq.wrid);
+	kvfree(qp->sq.wqe_head);
+	kvfree(qp->sq.w_list);
+	kvfree(qp->sq.wrid);
+	kvfree(qp->sq.wr_data);
+	kvfree(qp->rq.wrid);
 	mlx5_db_free(dev->mdev, &qp->db);
 	mlx5_buf_free(dev->mdev, &qp->buf);
 }
@@ -1021,12 +1032,16 @@ static int is_connected(enum ib_qp_type qp_type)
 }
 
 static int create_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
+				    struct mlx5_ib_qp *qp,
 				    struct mlx5_ib_sq *sq, u32 tdn)
 {
 	u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {0};
 	void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
 
 	MLX5_SET(tisc, tisc, transport_domain, tdn);
+	if (qp->flags & MLX5_IB_QP_UNDERLAY)
+		MLX5_SET(tisc, tisc, underlay_qpn, qp->underlay_qpn);
+
 	return mlx5_core_create_tis(dev->mdev, in, sizeof(in), &sq->tisn);
 }
 
@@ -1068,11 +1083,16 @@ static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
 
 	sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
 	MLX5_SET(sqc, sqc, flush_in_error_en, 1);
+	if (MLX5_CAP_ETH(dev->mdev, multi_pkt_send_wqe))
+		MLX5_SET(sqc, sqc, allow_multi_pkt_send_wqe, 1);
 	MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
 	MLX5_SET(sqc, sqc, user_index, MLX5_GET(qpc, qpc, user_index));
 	MLX5_SET(sqc, sqc, cqn, MLX5_GET(qpc, qpc, cqn_snd));
 	MLX5_SET(sqc, sqc, tis_lst_sz, 1);
 	MLX5_SET(sqc, sqc, tis_num_0, sq->tisn);
+	if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
+	    MLX5_CAP_ETH(dev->mdev, swp))
+		MLX5_SET(sqc, sqc, allow_swp, 1);
 
 	wq = MLX5_ADDR_OF(sqc, sqc, wq);
 	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
@@ -1229,7 +1249,7 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 	u32 tdn = mucontext->tdn;
 
 	if (qp->sq.wqe_cnt) {
-		err = create_raw_packet_qp_tis(dev, sq, tdn);
+		err = create_raw_packet_qp_tis(dev, qp, sq, tdn);
 		if (err)
 			return err;
 
@@ -1238,6 +1258,7 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 			goto err_destroy_tis;
 
 		sq->base.container_mibqp = qp;
+		sq->base.mqp.event = mlx5_ib_qp_event;
 	}
 
 	if (qp->rq.wqe_cnt) {
@@ -1502,10 +1523,6 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 	u32 *in;
 	int err;
 
-	base = init_attr->qp_type == IB_QPT_RAW_PACKET ?
-	       &qp->raw_packet_qp.rq.base :
-	       &qp->trans_qp.base;
-
 	mutex_init(&qp->mutex);
 	spin_lock_init(&qp->sq.lock);
 	spin_lock_init(&qp->rq.lock);
@@ -1587,10 +1604,28 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 
 		qp->wq_sig = !!(ucmd.flags & MLX5_QP_FLAG_SIGNATURE);
 		qp->scat_cqe = !!(ucmd.flags & MLX5_QP_FLAG_SCATTER_CQE);
+
+		if (init_attr->create_flags & IB_QP_CREATE_SOURCE_QPN) {
+			if (init_attr->qp_type != IB_QPT_UD ||
+			    (MLX5_CAP_GEN(dev->mdev, port_type) !=
+			     MLX5_CAP_PORT_TYPE_IB) ||
+			    !mlx5_get_flow_namespace(dev->mdev, MLX5_FLOW_NAMESPACE_BYPASS)) {
+				mlx5_ib_dbg(dev, "Source QP option isn't supported\n");
+				return -EOPNOTSUPP;
+			}
+
+			qp->flags |= MLX5_IB_QP_UNDERLAY;
+			qp->underlay_qpn = init_attr->source_qpn;
+		}
 	} else {
 		qp->wq_sig = !!wq_signature;
 	}
 
+	base = (init_attr->qp_type == IB_QPT_RAW_PACKET ||
+		qp->flags & MLX5_IB_QP_UNDERLAY) ?
+	       &qp->raw_packet_qp.rq.base :
+	       &qp->trans_qp.base;
+
 	qp->has_rq = qp_has_rq(init_attr);
 	err = set_rq_size(dev, &init_attr->cap, qp->has_rq,
 			  qp, (pd && pd->uobject) ? &ucmd : NULL);
@@ -1694,10 +1729,15 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 
 	MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, init_attr));
 
-	if (qp->sq.wqe_cnt)
+	if (qp->sq.wqe_cnt) {
 		MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt));
-	else
+	} else {
 		MLX5_SET(qpc, qpc, no_sq, 1);
+		if (init_attr->srq &&
+		    init_attr->srq->srq_type == IB_SRQT_TM)
+			MLX5_SET(qpc, qpc, offload_type,
+				 MLX5_QPC_OFFLOAD_TYPE_RNDV);
+	}
 
 	/* Set default resources */
 	switch (init_attr->qp_type) {
@@ -1741,7 +1781,8 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 		qp->flags |= MLX5_IB_QP_LSO;
 	}
 
-	if (init_attr->qp_type == IB_QPT_RAW_PACKET) {
+	if (init_attr->qp_type == IB_QPT_RAW_PACKET ||
+	    qp->flags & MLX5_IB_QP_UNDERLAY) {
 		qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd.sq_buf_addr;
 		raw_packet_qp_copy_info(qp, &qp->raw_packet_qp);
 		err = create_raw_packet_qp(dev, qp, in, pd);
@@ -1893,7 +1934,7 @@ static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
 {
 	struct mlx5_ib_cq *send_cq, *recv_cq;
-	struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
+	struct mlx5_ib_qp_base *base;
 	unsigned long flags;
 	int err;
 
@@ -1902,12 +1943,14 @@ static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
 		return;
 	}
 
-	base = qp->ibqp.qp_type == IB_QPT_RAW_PACKET ?
+	base = (qp->ibqp.qp_type == IB_QPT_RAW_PACKET ||
+		qp->flags & MLX5_IB_QP_UNDERLAY) ?
 	       &qp->raw_packet_qp.rq.base :
 	       &qp->trans_qp.base;
 
 	if (qp->state != IB_QPS_RESET) {
-		if (qp->ibqp.qp_type != IB_QPT_RAW_PACKET) {
+		if (qp->ibqp.qp_type != IB_QPT_RAW_PACKET &&
+		    !(qp->flags & MLX5_IB_QP_UNDERLAY)) {
 			err = mlx5_core_qp_modify(dev->mdev,
 						  MLX5_CMD_OP_2RST_QP, 0,
 						  NULL, &base->mqp);
@@ -1946,7 +1989,8 @@ static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
 	mlx5_ib_unlock_cqs(send_cq, recv_cq);
 	spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
 
-	if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET) {
+	if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET ||
+	    qp->flags & MLX5_IB_QP_UNDERLAY) {
 		destroy_raw_packet_qp(dev, qp);
 	} else {
 		err = mlx5_core_destroy_qp(dev->mdev, &base->mqp);
@@ -2702,7 +2746,8 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 
 	if (is_sqp(ibqp->qp_type)) {
 		context->mtu_msgmax = (IB_MTU_256 << 5) | 8;
-	} else if (ibqp->qp_type == IB_QPT_UD ||
+	} else if ((ibqp->qp_type == IB_QPT_UD &&
+		    !(qp->flags & MLX5_IB_QP_UNDERLAY)) ||
 		   ibqp->qp_type == MLX5_IB_QPT_REG_UMR) {
 		context->mtu_msgmax = (IB_MTU_4096 << 5) | 12;
 	} else if (attr_mask & IB_QP_PATH_MTU) {
@@ -2799,6 +2844,11 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
 		u8 port_num = (attr_mask & IB_QP_PORT ? attr->port_num :
 			       qp->port) - 1;
+
+		/* Underlay port should be used - index 0 function per port */
+		if (qp->flags & MLX5_IB_QP_UNDERLAY)
+			port_num = 0;
+
 		mibport = &dev->port[port_num];
 		context->qp_counter_set_usr_page |=
 			cpu_to_be32((u32)(mibport->cnts.set_id) << 24);
@@ -2824,7 +2874,8 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 	optpar = ib_mask_to_mlx5_opt(attr_mask);
 	optpar &= opt_mask[mlx5_cur][mlx5_new][mlx5_st];
 
-	if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET) {
+	if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET ||
+	    qp->flags & MLX5_IB_QP_UNDERLAY) {
 		struct mlx5_modify_raw_qp_param raw_qp_param = {};
 
 		raw_qp_param.operation = op;
@@ -2913,7 +2964,13 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 		ll = dev->ib_dev.get_link_layer(&dev->ib_dev, port);
 	}
 
-	if (qp_type != MLX5_IB_QPT_REG_UMR &&
+	if (qp->flags & MLX5_IB_QP_UNDERLAY) {
+		if (attr_mask & ~(IB_QP_STATE | IB_QP_CUR_STATE)) {
+			mlx5_ib_dbg(dev, "invalid attr_mask 0x%x when underlay QP is used\n",
+				    attr_mask);
+			goto out;
+		}
+	} else if (qp_type != MLX5_IB_QPT_REG_UMR &&
 	    !ib_modify_qp_is_ok(cur_state, new_state, qp_type, attr_mask, ll)) {
 		mlx5_ib_dbg(dev, "invalid QP state transition from %d to %d, qp_type %d, attr_mask 0x%x\n",
 			    cur_state, new_state, ibqp->qp_type, attr_mask);
@@ -4477,9 +4534,14 @@ int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
 		return mlx5_ib_gsi_query_qp(ibqp, qp_attr, qp_attr_mask,
 					    qp_init_attr);
 
+	/* Not all of output fields are applicable, make sure to zero them */
+	memset(qp_init_attr, 0, sizeof(*qp_init_attr));
+	memset(qp_attr, 0, sizeof(*qp_attr));
+
 	mutex_lock(&qp->mutex);
 
-	if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET) {
+	if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET ||
+	    qp->flags & MLX5_IB_QP_UNDERLAY) {
 		err = query_raw_packet_qp_state(dev, qp, &raw_packet_qp_state);
 		if (err)
 			goto out;
@@ -4597,6 +4659,27 @@ static void mlx5_ib_wq_event(struct mlx5_core_qp *core_qp, int type)
 	}
 }
 
+static int set_delay_drop(struct mlx5_ib_dev *dev)
+{
+	int err = 0;
+
+	mutex_lock(&dev->delay_drop.lock);
+	if (dev->delay_drop.activate)
+		goto out;
+
+	err = mlx5_core_set_delay_drop(dev->mdev, dev->delay_drop.timeout);
+	if (err)
+		goto out;
+
+	dev->delay_drop.activate = true;
+out:
+	mutex_unlock(&dev->delay_drop.lock);
+
+	if (!err)
+		atomic_inc(&dev->delay_drop.rqs_cnt);
+	return err;
+}
+
 static int  create_rq(struct mlx5_ib_rwq *rwq, struct ib_pd *pd,
 		      struct ib_wq_init_attr *init_attr)
 {
@@ -4651,9 +4734,28 @@ static int  create_rq(struct mlx5_ib_rwq *rwq, struct ib_pd *pd,
 		}
 		MLX5_SET(rqc, rqc, scatter_fcs, 1);
 	}
+	if (init_attr->create_flags & IB_WQ_FLAGS_DELAY_DROP) {
+		if (!(dev->ib_dev.attrs.raw_packet_caps &
+		      IB_RAW_PACKET_CAP_DELAY_DROP)) {
+			mlx5_ib_dbg(dev, "Delay drop is not supported\n");
+			err = -EOPNOTSUPP;
+			goto out;
+		}
+		MLX5_SET(rqc, rqc, delay_drop_en, 1);
+	}
 	rq_pas0 = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
 	mlx5_ib_populate_pas(dev, rwq->umem, rwq->page_shift, rq_pas0, 0);
 	err = mlx5_core_create_rq_tracked(dev->mdev, in, inlen, &rwq->core_qp);
+	if (!err && init_attr->create_flags & IB_WQ_FLAGS_DELAY_DROP) {
+		err = set_delay_drop(dev);
+		if (err) {
+			mlx5_ib_warn(dev, "Failed to enable delay drop err=%d\n",
+				     err);
+			mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp);
+		} else {
+			rwq->create_flags |= MLX5_IB_WQ_FLAGS_DELAY_DROP;
+		}
+	}
 out:
 	kvfree(in);
 	return err;
@@ -4787,7 +4889,7 @@ struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
 err_copy:
 	mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp);
 err_user_rq:
-	destroy_user_rq(pd, rwq);
+	destroy_user_rq(dev, pd, rwq);
 err:
 	kfree(rwq);
 	return ERR_PTR(err);
@@ -4799,7 +4901,7 @@ int mlx5_ib_destroy_wq(struct ib_wq *wq)
 	struct mlx5_ib_rwq *rwq = to_mrwq(wq);
 
 	mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp);
-	destroy_user_rq(wq->pd, rwq);
+	destroy_user_rq(dev, wq->pd, rwq);
 	kfree(rwq);
 
 	return 0;
diff --git a/drivers/infiniband/hw/mlx5/srq.c b/drivers/infiniband/hw/mlx5/srq.c
index 43707b101f47..6d5fadad9090 100644
--- a/drivers/infiniband/hw/mlx5/srq.c
+++ b/drivers/infiniband/hw/mlx5/srq.c
@@ -101,7 +101,7 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq,
 				 udata->inlen - sizeof(ucmd)))
 		return -EINVAL;
 
-	if (in->type == IB_SRQT_XRC) {
+	if (in->type != IB_SRQT_BASIC) {
 		err = get_srq_user_index(to_mucontext(pd->uobject->context),
 					 &ucmd, udata->inlen, &uidx);
 		if (err)
@@ -145,7 +145,7 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq,
 	in->log_page_size = page_shift - MLX5_ADAPTER_PAGE_SHIFT;
 	in->page_offset = offset;
 	if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 &&
-	    in->type == IB_SRQT_XRC)
+	    in->type != IB_SRQT_BASIC)
 		in->user_index = uidx;
 
 	return 0;
@@ -196,7 +196,7 @@ static int create_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq,
 	}
 	mlx5_fill_page_array(&srq->buf, in->pas);
 
-	srq->wrid = kmalloc(srq->msrq.max * sizeof(u64), GFP_KERNEL);
+	srq->wrid = kvmalloc_array(srq->msrq.max, sizeof(u64), GFP_KERNEL);
 	if (!srq->wrid) {
 		err = -ENOMEM;
 		goto err_in;
@@ -205,7 +205,7 @@ static int create_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq,
 
 	in->log_page_size = srq->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT;
 	if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 &&
-	    in->type == IB_SRQT_XRC)
+	    in->type != IB_SRQT_BASIC)
 		in->user_index = MLX5_IB_DEFAULT_UIDX;
 
 	return 0;
@@ -230,7 +230,7 @@ static void destroy_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq)
 
 static void destroy_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq)
 {
-	kfree(srq->wrid);
+	kvfree(srq->wrid);
 	mlx5_buf_free(dev->mdev, &srq->buf);
 	mlx5_db_free(dev->mdev, &srq->db);
 }
@@ -292,14 +292,29 @@ struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
 	in.wqe_shift = srq->msrq.wqe_shift - 4;
 	if (srq->wq_sig)
 		in.flags |= MLX5_SRQ_FLAG_WQ_SIG;
-	if (init_attr->srq_type == IB_SRQT_XRC) {
+
+	if (init_attr->srq_type == IB_SRQT_XRC)
 		in.xrcd = to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn;
-		in.cqn = to_mcq(init_attr->ext.xrc.cq)->mcq.cqn;
-	} else if (init_attr->srq_type == IB_SRQT_BASIC) {
+	else
 		in.xrcd = to_mxrcd(dev->devr.x0)->xrcdn;
-		in.cqn = to_mcq(dev->devr.c0)->mcq.cqn;
+
+	if (init_attr->srq_type == IB_SRQT_TM) {
+		in.tm_log_list_size =
+			ilog2(init_attr->ext.tag_matching.max_num_tags) + 1;
+		if (in.tm_log_list_size >
+		    MLX5_CAP_GEN(dev->mdev, log_tag_matching_list_sz)) {
+			mlx5_ib_dbg(dev, "TM SRQ max_num_tags exceeding limit\n");
+			err = -EINVAL;
+			goto err_usr_kern_srq;
+		}
+		in.flags |= MLX5_SRQ_FLAG_RNDV;
 	}
 
+	if (ib_srq_has_cq(init_attr->srq_type))
+		in.cqn = to_mcq(init_attr->ext.cq)->mcq.cqn;
+	else
+		in.cqn = to_mcq(dev->devr.c0)->mcq.cqn;
+
 	in.pd = to_mpd(pd)->pdn;
 	in.db_record = srq->db.dma;
 	err = mlx5_core_create_srq(dev->mdev, &srq->msrq, &in);
diff --git a/drivers/infiniband/hw/mthca/mthca_av.c b/drivers/infiniband/hw/mthca/mthca_av.c
index 2aec9908c40a..e7f6223e9c60 100644
--- a/drivers/infiniband/hw/mthca/mthca_av.c
+++ b/drivers/infiniband/hw/mthca/mthca_av.c
@@ -186,7 +186,7 @@ int mthca_create_ah(struct mthca_dev *dev,
 
 on_hca_fail:
 	if (ah->type == MTHCA_AH_PCI_POOL) {
-		ah->av = pci_pool_zalloc(dev->av_table.pool,
+		ah->av = dma_pool_zalloc(dev->av_table.pool,
 					 GFP_ATOMIC, &ah->avdma);
 		if (!ah->av)
 			return -ENOMEM;
@@ -250,7 +250,7 @@ int mthca_destroy_ah(struct mthca_dev *dev, struct mthca_ah *ah)
 		break;
 
 	case MTHCA_AH_PCI_POOL:
-		pci_pool_free(dev->av_table.pool, ah->av, ah->avdma);
+		dma_pool_free(dev->av_table.pool, ah->av, ah->avdma);
 		break;
 
 	case MTHCA_AH_KMALLOC:
@@ -340,7 +340,7 @@ int mthca_init_av_table(struct mthca_dev *dev)
 	if (err)
 		return err;
 
-	dev->av_table.pool = pci_pool_create("mthca_av", dev->pdev,
+	dev->av_table.pool = dma_pool_create("mthca_av", &dev->pdev->dev,
 					     MTHCA_AV_SIZE,
 					     MTHCA_AV_SIZE, 0);
 	if (!dev->av_table.pool)
@@ -360,7 +360,7 @@ int mthca_init_av_table(struct mthca_dev *dev)
 	return 0;
 
  out_free_pool:
-	pci_pool_destroy(dev->av_table.pool);
+	dma_pool_destroy(dev->av_table.pool);
 
  out_free_alloc:
 	mthca_alloc_cleanup(&dev->av_table.alloc);
@@ -374,6 +374,6 @@ void mthca_cleanup_av_table(struct mthca_dev *dev)
 
 	if (dev->av_table.av_map)
 		iounmap(dev->av_table.av_map);
-	pci_pool_destroy(dev->av_table.pool);
+	dma_pool_destroy(dev->av_table.pool);
 	mthca_alloc_cleanup(&dev->av_table.alloc);
 }
diff --git a/drivers/infiniband/hw/mthca/mthca_cmd.c b/drivers/infiniband/hw/mthca/mthca_cmd.c
index 9d83a53c0c67..419a2a20c047 100644
--- a/drivers/infiniband/hw/mthca/mthca_cmd.c
+++ b/drivers/infiniband/hw/mthca/mthca_cmd.c
@@ -538,7 +538,7 @@ int mthca_cmd_init(struct mthca_dev *dev)
 		return -ENOMEM;
 	}
 
-	dev->cmd.pool = pci_pool_create("mthca_cmd", dev->pdev,
+	dev->cmd.pool = dma_pool_create("mthca_cmd", &dev->pdev->dev,
 					MTHCA_MAILBOX_SIZE,
 					MTHCA_MAILBOX_SIZE, 0);
 	if (!dev->cmd.pool) {
@@ -551,7 +551,7 @@ int mthca_cmd_init(struct mthca_dev *dev)
 
 void mthca_cmd_cleanup(struct mthca_dev *dev)
 {
-	pci_pool_destroy(dev->cmd.pool);
+	dma_pool_destroy(dev->cmd.pool);
 	iounmap(dev->hcr);
 	if (dev->cmd.flags & MTHCA_CMD_POST_DOORBELLS)
 		iounmap(dev->cmd.dbell_map);
@@ -621,7 +621,7 @@ struct mthca_mailbox *mthca_alloc_mailbox(struct mthca_dev *dev,
 	if (!mailbox)
 		return ERR_PTR(-ENOMEM);
 
-	mailbox->buf = pci_pool_alloc(dev->cmd.pool, gfp_mask, &mailbox->dma);
+	mailbox->buf = dma_pool_alloc(dev->cmd.pool, gfp_mask, &mailbox->dma);
 	if (!mailbox->buf) {
 		kfree(mailbox);
 		return ERR_PTR(-ENOMEM);
@@ -635,7 +635,7 @@ void mthca_free_mailbox(struct mthca_dev *dev, struct mthca_mailbox *mailbox)
 	if (!mailbox)
 		return;
 
-	pci_pool_free(dev->cmd.pool, mailbox->buf, mailbox->dma);
+	dma_pool_free(dev->cmd.pool, mailbox->buf, mailbox->dma);
 	kfree(mailbox);
 }
 
@@ -698,7 +698,7 @@ static int mthca_map_cmd(struct mthca_dev *dev, u16 op, struct mthca_icm *icm,
 		for (i = 0; i < mthca_icm_size(&iter) >> lg; ++i) {
 			if (virt != -1) {
 				pages[nent * 2] = cpu_to_be64(virt);
-				virt += 1 << lg;
+				virt += 1ULL << lg;
 			}
 
 			pages[nent * 2 + 1] =
@@ -1921,7 +1921,7 @@ int mthca_MAD_IFC(struct mthca_dev *dev, int ignore_mkey, int ignore_bkey,
 			(in_wc->wc_flags & IB_WC_GRH ? 0x80 : 0);
 		MTHCA_PUT(inbox, val,               MAD_IFC_G_PATH_OFFSET);
 
-		MTHCA_PUT(inbox, in_wc->slid,       MAD_IFC_RLID_OFFSET);
+		MTHCA_PUT(inbox, ib_lid_cpu16(in_wc->slid), MAD_IFC_RLID_OFFSET);
 		MTHCA_PUT(inbox, in_wc->pkey_index, MAD_IFC_PKEY_OFFSET);
 
 		if (in_grh)
@@ -1929,7 +1929,7 @@ int mthca_MAD_IFC(struct mthca_dev *dev, int ignore_mkey, int ignore_bkey,
 
 		op_modifier |= 0x4;
 
-		in_modifier |= in_wc->slid << 16;
+		in_modifier |= ib_lid_cpu16(in_wc->slid) << 16;
 	}
 
 	err = mthca_cmd_box(dev, inmailbox->dma, outmailbox->dma,
diff --git a/drivers/infiniband/hw/mthca/mthca_dev.h b/drivers/infiniband/hw/mthca/mthca_dev.h
index ec7da9a474cd..5508afbf1c67 100644
--- a/drivers/infiniband/hw/mthca/mthca_dev.h
+++ b/drivers/infiniband/hw/mthca/mthca_dev.h
@@ -118,7 +118,7 @@ enum {
 };
 
 struct mthca_cmd {
-	struct pci_pool          *pool;
+	struct dma_pool          *pool;
 	struct mutex              hcr_mutex;
 	struct semaphore 	  poll_sem;
 	struct semaphore 	  event_sem;
@@ -263,7 +263,7 @@ struct mthca_qp_table {
 };
 
 struct mthca_av_table {
-	struct pci_pool   *pool;
+	struct dma_pool   *pool;
 	int                num_ddr_avs;
 	u64                ddr_av_base;
 	void __iomem      *av_map;
diff --git a/drivers/infiniband/hw/mthca/mthca_mad.c b/drivers/infiniband/hw/mthca/mthca_mad.c
index 7df3db71777a..093f7755c843 100644
--- a/drivers/infiniband/hw/mthca/mthca_mad.c
+++ b/drivers/infiniband/hw/mthca/mthca_mad.c
@@ -205,7 +205,7 @@ int mthca_process_mad(struct ib_device *ibdev,
 		      u16 *out_mad_pkey_index)
 {
 	int err;
-	u16 slid = in_wc ? in_wc->slid : be16_to_cpu(IB_LID_PERMISSIVE);
+	u16 slid = in_wc ? ib_lid_cpu16(in_wc->slid) : be16_to_cpu(IB_LID_PERMISSIVE);
 	u16 prev_lid = 0;
 	struct ib_port_attr pattr;
 	const struct ib_mad *in_mad = (const struct ib_mad *)in;
@@ -256,7 +256,7 @@ int mthca_process_mad(struct ib_device *ibdev,
 	    in_mad->mad_hdr.method == IB_MGMT_METHOD_SET &&
 	    in_mad->mad_hdr.attr_id == IB_SMP_ATTR_PORT_INFO &&
 	    !ib_query_port(ibdev, port_num, &pattr))
-		prev_lid = pattr.lid;
+		prev_lid = ib_lid_cpu16(pattr.lid);
 
 	err = mthca_MAD_IFC(to_mdev(ibdev),
 			    mad_flags & IB_MAD_IGNORE_MKEY,
diff --git a/drivers/infiniband/hw/mthca/mthca_main.c b/drivers/infiniband/hw/mthca/mthca_main.c
index c309e5c96383..e36a9bc52268 100644
--- a/drivers/infiniband/hw/mthca/mthca_main.c
+++ b/drivers/infiniband/hw/mthca/mthca_main.c
@@ -49,7 +49,6 @@
 MODULE_AUTHOR("Roland Dreier");
 MODULE_DESCRIPTION("Mellanox InfiniBand HCA low-level driver");
 MODULE_LICENSE("Dual BSD/GPL");
-MODULE_VERSION(DRV_VERSION);
 
 #ifdef CONFIG_INFINIBAND_MTHCA_DEBUG
 
@@ -1162,7 +1161,7 @@ static void mthca_remove_one(struct pci_dev *pdev)
 	mutex_unlock(&mthca_device_mutex);
 }
 
-static struct pci_device_id mthca_pci_table[] = {
+static const struct pci_device_id mthca_pci_table[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_TAVOR),
 	  .driver_data = TAVOR },
 	{ PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_TAVOR),
diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c
index c197cd9b193f..6fee7795d1c8 100644
--- a/drivers/infiniband/hw/mthca/mthca_provider.c
+++ b/drivers/infiniband/hw/mthca/mthca_provider.c
@@ -914,7 +914,7 @@ static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 	int err = 0;
 	int write_mtt_size;
 
-	if (udata->inlen - sizeof (struct ib_uverbs_cmd_hdr) < sizeof ucmd) {
+	if (udata->inlen < sizeof ucmd) {
 		if (!to_mucontext(pd->uobject->context)->reg_mr_warned) {
 			mthca_warn(dev, "Process '%s' did not pass in MR attrs.\n",
 				   current->comm);
@@ -1178,12 +1178,11 @@ static int mthca_port_immutable(struct ib_device *ibdev, u8 port_num,
 	return 0;
 }
 
-static void get_dev_fw_str(struct ib_device *device, char *str,
-			   size_t str_len)
+static void get_dev_fw_str(struct ib_device *device, char *str)
 {
 	struct mthca_dev *dev =
 		container_of(device, struct mthca_dev, ib_dev);
-	snprintf(str, str_len, "%d.%d.%d",
+	snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%d",
 		 (int) (dev->fw_ver >> 32),
 		 (int) (dev->fw_ver >> 16) & 0xffff,
 		 (int) dev->fw_ver & 0xffff);
diff --git a/drivers/infiniband/hw/nes/nes.c b/drivers/infiniband/hw/nes/nes.c
index a30aa6527f7e..942ca84713c9 100644
--- a/drivers/infiniband/hw/nes/nes.c
+++ b/drivers/infiniband/hw/nes/nes.c
@@ -63,7 +63,6 @@
 MODULE_AUTHOR("NetEffect");
 MODULE_DESCRIPTION("NetEffect RNIC Low-level iWARP Driver");
 MODULE_LICENSE("Dual BSD/GPL");
-MODULE_VERSION(DRV_VERSION);
 
 int interrupt_mod_interval = 0;
 
@@ -102,7 +101,7 @@ static unsigned int ee_flsh_adapter;
 static unsigned int sysfs_nonidx_addr;
 static unsigned int sysfs_idx_addr;
 
-static struct pci_device_id nes_pci_table[] = {
+static const struct pci_device_id nes_pci_table[] = {
 	{ PCI_VDEVICE(NETEFFECT, PCI_DEVICE_ID_NETEFFECT_NE020), },
 	{ PCI_VDEVICE(NETEFFECT, PCI_DEVICE_ID_NETEFFECT_NE020_KR), },
 	{0}
@@ -808,13 +807,6 @@ static void nes_remove(struct pci_dev *pcidev)
 }
 
 
-static struct pci_driver nes_pci_driver = {
-	.name = DRV_NAME,
-	.id_table = nes_pci_table,
-	.probe = nes_probe,
-	.remove = nes_remove,
-};
-
 static ssize_t adapter_show(struct device_driver *ddp, char *buf)
 {
 	unsigned int  devfn = 0xffffffff;
@@ -1156,35 +1148,29 @@ static DRIVER_ATTR_RW(idx_addr);
 static DRIVER_ATTR_RW(idx_data);
 static DRIVER_ATTR_RW(wqm_quanta);
 
-static int nes_create_driver_sysfs(struct pci_driver *drv)
-{
-	int error;
-	error  = driver_create_file(&drv->driver, &driver_attr_adapter);
-	error |= driver_create_file(&drv->driver, &driver_attr_eeprom_cmd);
-	error |= driver_create_file(&drv->driver, &driver_attr_eeprom_data);
-	error |= driver_create_file(&drv->driver, &driver_attr_flash_cmd);
-	error |= driver_create_file(&drv->driver, &driver_attr_flash_data);
-	error |= driver_create_file(&drv->driver, &driver_attr_nonidx_addr);
-	error |= driver_create_file(&drv->driver, &driver_attr_nonidx_data);
-	error |= driver_create_file(&drv->driver, &driver_attr_idx_addr);
-	error |= driver_create_file(&drv->driver, &driver_attr_idx_data);
-	error |= driver_create_file(&drv->driver, &driver_attr_wqm_quanta);
-	return error;
-}
+static struct attribute *nes_attrs[] = {
+	&driver_attr_adapter.attr,
+	&driver_attr_eeprom_cmd.attr,
+	&driver_attr_eeprom_data.attr,
+	&driver_attr_flash_cmd.attr,
+	&driver_attr_flash_data.attr,
+	&driver_attr_nonidx_addr.attr,
+	&driver_attr_nonidx_data.attr,
+	&driver_attr_idx_addr.attr,
+	&driver_attr_idx_data.attr,
+	&driver_attr_wqm_quanta.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(nes);
+
+static struct pci_driver nes_pci_driver = {
+	.name = DRV_NAME,
+	.id_table = nes_pci_table,
+	.probe = nes_probe,
+	.remove = nes_remove,
+	.groups = nes_groups,
+};
 
-static void nes_remove_driver_sysfs(struct pci_driver *drv)
-{
-	driver_remove_file(&drv->driver, &driver_attr_adapter);
-	driver_remove_file(&drv->driver, &driver_attr_eeprom_cmd);
-	driver_remove_file(&drv->driver, &driver_attr_eeprom_data);
-	driver_remove_file(&drv->driver, &driver_attr_flash_cmd);
-	driver_remove_file(&drv->driver, &driver_attr_flash_data);
-	driver_remove_file(&drv->driver, &driver_attr_nonidx_addr);
-	driver_remove_file(&drv->driver, &driver_attr_nonidx_data);
-	driver_remove_file(&drv->driver, &driver_attr_idx_addr);
-	driver_remove_file(&drv->driver, &driver_attr_idx_data);
-	driver_remove_file(&drv->driver, &driver_attr_wqm_quanta);
-}
 
 /**
  * nes_init_module - module initialization entry point
@@ -1192,20 +1178,13 @@ static void nes_remove_driver_sysfs(struct pci_driver *drv)
 static int __init nes_init_module(void)
 {
 	int retval;
-	int retval1;
 
 	retval = nes_cm_start();
 	if (retval) {
 		printk(KERN_ERR PFX "Unable to start NetEffect iWARP CM.\n");
 		return retval;
 	}
-	retval = pci_register_driver(&nes_pci_driver);
-	if (retval >= 0) {
-		retval1 = nes_create_driver_sysfs(&nes_pci_driver);
-		if (retval1 < 0)
-			printk(KERN_ERR PFX "Unable to create NetEffect sys files.\n");
-	}
-	return retval;
+	return pci_register_driver(&nes_pci_driver);
 }
 
 
@@ -1215,7 +1194,6 @@ static int __init nes_init_module(void)
 static void __exit nes_exit_module(void)
 {
 	nes_cm_stop();
-	nes_remove_driver_sysfs(&nes_pci_driver);
 
 	pci_unregister_driver(&nes_pci_driver);
 }
diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c
index 25dcd7573df9..f0dc5f4aa177 100644
--- a/drivers/infiniband/hw/nes/nes_verbs.c
+++ b/drivers/infiniband/hw/nes/nes_verbs.c
@@ -481,21 +481,16 @@ static int nes_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr
 	props->active_mtu = ib_mtu_int_to_enum(netdev->mtu);
 
 	props->lid = 1;
-	props->lmc = 0;
-	props->sm_lid = 0;
-	props->sm_sl = 0;
 	if (netif_queue_stopped(netdev))
 		props->state = IB_PORT_DOWN;
 	else if (nesvnic->linkup)
 		props->state = IB_PORT_ACTIVE;
 	else
 		props->state = IB_PORT_DOWN;
-	props->phys_state = 0;
 	props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
 			IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;
 	props->gid_tbl_len = 1;
 	props->pkey_tbl_len = 1;
-	props->qkey_viol_cntr = 0;
 	props->active_width = IB_WIDTH_4X;
 	props->active_speed = IB_SPEED_SDR;
 	props->max_msg_sz = 0x80000000;
@@ -3672,15 +3667,14 @@ static int nes_port_immutable(struct ib_device *ibdev, u8 port_num,
 	return 0;
 }
 
-static void get_dev_fw_str(struct ib_device *dev, char *str,
-			   size_t str_len)
+static void get_dev_fw_str(struct ib_device *dev, char *str)
 {
 	struct nes_ib_device *nesibdev =
 			container_of(dev, struct nes_ib_device, ibdev);
 	struct nes_vnic *nesvnic = nesibdev->nesvnic;
 
 	nes_debug(NES_DBG_INIT, "\n");
-	snprintf(str, str_len, "%u.%u",
+	snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u",
 		 (nesvnic->nesdev->nesadapter->firmware_version >> 16),
 		 (nesvnic->nesdev->nesadapter->firmware_version & 0x000000ff));
 }
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_main.c b/drivers/infiniband/hw/ocrdma/ocrdma_main.c
index 57c9a2ad0260..fbfbd9e96147 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_main.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_main.c
@@ -58,7 +58,6 @@
 #include "ocrdma_stats.h"
 #include <rdma/ocrdma-abi.h>
 
-MODULE_VERSION(OCRDMA_ROCE_DRV_VERSION);
 MODULE_DESCRIPTION(OCRDMA_ROCE_DRV_DESC " " OCRDMA_ROCE_DRV_VERSION);
 MODULE_AUTHOR("Emulex Corporation");
 MODULE_LICENSE("Dual BSD/GPL");
@@ -108,12 +107,11 @@ static int ocrdma_port_immutable(struct ib_device *ibdev, u8 port_num,
 	return 0;
 }
 
-static void get_dev_fw_str(struct ib_device *device, char *str,
-			   size_t str_len)
+static void get_dev_fw_str(struct ib_device *device, char *str)
 {
 	struct ocrdma_dev *dev = get_ocrdma_dev(device);
 
-	snprintf(str, str_len, "%s", &dev->attr.fw_ver[0]);
+	snprintf(str, IB_FW_VERSION_NAME_MAX, "%s", &dev->attr.fw_ver[0]);
 }
 
 static int ocrdma_register_device(struct ocrdma_dev *dev)
diff --git a/drivers/infiniband/hw/qedr/main.c b/drivers/infiniband/hw/qedr/main.c
index b5851fd67d4f..97d033f51dc9 100644
--- a/drivers/infiniband/hw/qedr/main.c
+++ b/drivers/infiniband/hw/qedr/main.c
@@ -47,7 +47,6 @@
 MODULE_DESCRIPTION("QLogic 40G/100G ROCE Driver");
 MODULE_AUTHOR("QLogic Corporation");
 MODULE_LICENSE("Dual BSD/GPL");
-MODULE_VERSION(QEDR_MODULE_VERSION);
 
 #define QEDR_WQ_MULTIPLIER_DFT	(3)
 
@@ -69,13 +68,12 @@ static enum rdma_link_layer qedr_link_layer(struct ib_device *device,
 	return IB_LINK_LAYER_ETHERNET;
 }
 
-static void qedr_get_dev_fw_str(struct ib_device *ibdev, char *str,
-				size_t str_len)
+static void qedr_get_dev_fw_str(struct ib_device *ibdev, char *str)
 {
 	struct qedr_dev *qedr = get_qedr_dev(ibdev);
 	u32 fw_ver = (u32)qedr->attr.fw_ver;
 
-	snprintf(str, str_len, "%d. %d. %d. %d",
+	snprintf(str, IB_FW_VERSION_NAME_MAX, "%d. %d. %d. %d",
 		 (fw_ver >> 24) & 0xFF, (fw_ver >> 16) & 0xFF,
 		 (fw_ver >> 8) & 0xFF, fw_ver & 0xFF);
 }
@@ -778,6 +776,7 @@ static struct qedr_dev *qedr_add(struct qed_dev *cdev, struct pci_dev *pdev,
 	if (rc)
 		goto init_err;
 
+	dev->user_dpm_enabled = dev_info.user_dpm_enabled;
 	dev->num_hwfns = dev_info.common.num_hwfns;
 	dev->rdma_ctx = dev->ops->rdma_get_rdma_ctx(cdev);
 
diff --git a/drivers/infiniband/hw/qedr/qedr.h b/drivers/infiniband/hw/qedr/qedr.h
index ab7784bfdac6..b2bb42e2805d 100644
--- a/drivers/infiniband/hw/qedr/qedr.h
+++ b/drivers/infiniband/hw/qedr/qedr.h
@@ -41,7 +41,6 @@
 #include <linux/qed/roce_common.h>
 #include "qedr_hsi_rdma.h"
 
-#define QEDR_MODULE_VERSION	"8.10.10.0"
 #define QEDR_NODE_DESC "QLogic 579xx RoCE HCA"
 #define DP_NAME(dev) ((dev)->ibdev.name)
 
@@ -163,6 +162,8 @@ struct qedr_dev {
 	struct qedr_qp		*gsi_qp;
 
 	unsigned long enet_state;
+
+	u8 user_dpm_enabled;
 };
 
 #define QEDR_MAX_SQ_PBL			(0x8000)
diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c
index 2ae71b8f1ba8..769ac07c3c8e 100644
--- a/drivers/infiniband/hw/qedr/verbs.c
+++ b/drivers/infiniband/hw/qedr/verbs.c
@@ -376,6 +376,9 @@ struct ib_ucontext *qedr_alloc_ucontext(struct ib_device *ibdev,
 
 	memset(&uresp, 0, sizeof(uresp));
 
+	uresp.dpm_enabled = dev->user_dpm_enabled;
+	uresp.wids_enabled = 1;
+	uresp.wid_count = oparams.wid_count;
 	uresp.db_pa = ctx->dpi_phys_addr;
 	uresp.db_size = ctx->dpi_size;
 	uresp.max_send_wr = dev->attr.max_sqe;
@@ -488,7 +491,7 @@ struct ib_pd *qedr_alloc_pd(struct ib_device *ibdev,
 		 (udata && context) ? "User Lib" : "Kernel");
 
 	if (!dev->rdma_ctx) {
-		DP_ERR(dev, "invlaid RDMA context\n");
+		DP_ERR(dev, "invalid RDMA context\n");
 		return ERR_PTR(-EINVAL);
 	}
 
diff --git a/drivers/infiniband/hw/qib/qib.h b/drivers/infiniband/hw/qib/qib.h
index a3e21a25cea5..f9e1c69603a5 100644
--- a/drivers/infiniband/hw/qib/qib.h
+++ b/drivers/infiniband/hw/qib/qib.h
@@ -1,7 +1,7 @@
 #ifndef _QIB_KERNEL_H
 #define _QIB_KERNEL_H
 /*
- * Copyright (c) 2012, 2013 Intel Corporation.  All rights reserved.
+ * Copyright (c) 2012 - 2017 Intel Corporation.  All rights reserved.
  * Copyright (c) 2006 - 2012 QLogic Corporation. All rights reserved.
  * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
  *
@@ -443,7 +443,7 @@ struct qib_irq_notify;
 #endif
 
 struct qib_msix_entry {
-	struct msix_entry msix;
+	int irq;
 	void *arg;
 #ifdef CONFIG_INFINIBAND_QIB_DCA
 	int dca;
@@ -1433,9 +1433,9 @@ int qib_pcie_init(struct pci_dev *, const struct pci_device_id *);
 int qib_pcie_ddinit(struct qib_devdata *, struct pci_dev *,
 		    const struct pci_device_id *);
 void qib_pcie_ddcleanup(struct qib_devdata *);
-int qib_pcie_params(struct qib_devdata *, u32, u32 *, struct qib_msix_entry *);
+int qib_pcie_params(struct qib_devdata *dd, u32 minw, u32 *nent);
 int qib_reinit_intr(struct qib_devdata *);
-void qib_enable_intx(struct pci_dev *);
+void qib_enable_intx(struct qib_devdata *dd);
 void qib_nomsi(struct qib_devdata *);
 void qib_nomsix(struct qib_devdata *);
 void qib_pcie_getcmd(struct qib_devdata *, u16 *, u8 *, u8 *);
diff --git a/drivers/infiniband/hw/qib/qib_debugfs.c b/drivers/infiniband/hw/qib/qib_debugfs.c
index 5bad8e3b40bb..5ed1ed93380f 100644
--- a/drivers/infiniband/hw/qib/qib_debugfs.c
+++ b/drivers/infiniband/hw/qib/qib_debugfs.c
@@ -1,6 +1,5 @@
-#ifdef CONFIG_DEBUG_FS
 /*
- * Copyright (c) 2013 Intel Corporation.  All rights reserved.
+ * Copyright (c) 2013 - 2017 Intel Corporation.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -191,10 +190,10 @@ DEBUGFS_FILE(ctx_stats)
 static void *_qp_stats_seq_start(struct seq_file *s, loff_t *pos)
 	__acquires(RCU)
 {
-	struct qib_qp_iter *iter;
+	struct rvt_qp_iter *iter;
 	loff_t n = *pos;
 
-	iter = qib_qp_iter_init(s->private);
+	iter = rvt_qp_iter_init(s->private, 0, NULL);
 
 	/* stop calls rcu_read_unlock */
 	rcu_read_lock();
@@ -203,7 +202,7 @@ static void *_qp_stats_seq_start(struct seq_file *s, loff_t *pos)
 		return NULL;
 
 	do {
-		if (qib_qp_iter_next(iter)) {
+		if (rvt_qp_iter_next(iter)) {
 			kfree(iter);
 			return NULL;
 		}
@@ -216,11 +215,11 @@ static void *_qp_stats_seq_next(struct seq_file *s, void *iter_ptr,
 				   loff_t *pos)
 	__must_hold(RCU)
 {
-	struct qib_qp_iter *iter = iter_ptr;
+	struct rvt_qp_iter *iter = iter_ptr;
 
 	(*pos)++;
 
-	if (qib_qp_iter_next(iter)) {
+	if (rvt_qp_iter_next(iter)) {
 		kfree(iter);
 		return NULL;
 	}
@@ -236,7 +235,7 @@ static void _qp_stats_seq_stop(struct seq_file *s, void *iter_ptr)
 
 static int _qp_stats_seq_show(struct seq_file *s, void *iter_ptr)
 {
-	struct qib_qp_iter *iter = iter_ptr;
+	struct rvt_qp_iter *iter = iter_ptr;
 
 	if (!iter)
 		return 0;
@@ -284,6 +283,3 @@ void qib_dbg_exit(void)
 	debugfs_remove_recursive(qib_dbg_root);
 	qib_dbg_root = NULL;
 }
-
-#endif
-
diff --git a/drivers/infiniband/hw/qib/qib_driver.c b/drivers/infiniband/hw/qib/qib_driver.c
index 2b5982f743ef..719906a9fd51 100644
--- a/drivers/infiniband/hw/qib/qib_driver.c
+++ b/drivers/infiniband/hw/qib/qib_driver.c
@@ -66,7 +66,6 @@ MODULE_PARM_DESC(compat_ddr_negotiate,
 MODULE_LICENSE("Dual BSD/GPL");
 MODULE_AUTHOR("Intel <ibsupport@intel.com>");
 MODULE_DESCRIPTION("Intel IB driver");
-MODULE_VERSION(QIB_DRIVER_VERSION);
 
 /*
  * QIB_PIO_MAXIBHDR is the max IB header size allowed for in our
diff --git a/drivers/infiniband/hw/qib/qib_iba6120.c b/drivers/infiniband/hw/qib/qib_iba6120.c
index e423b71e6ea0..3259a60e4f4f 100644
--- a/drivers/infiniband/hw/qib/qib_iba6120.c
+++ b/drivers/infiniband/hw/qib/qib_iba6120.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013 Intel Corporation. All rights reserved.
+ * Copyright (c) 2013 - 2017 Intel Corporation. All rights reserved.
  * Copyright (c) 2006, 2007, 2008, 2009, 2010 QLogic Corporation.
  * All rights reserved.
  * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
@@ -1742,38 +1742,32 @@ static void qib_setup_6120_interrupt(struct qib_devdata *dd)
  */
 static void pe_boardname(struct qib_devdata *dd)
 {
-	char *n;
-	u32 boardid, namelen;
+	u32 boardid;
 
 	boardid = SYM_FIELD(dd->revision, Revision,
 			    BoardID);
 
 	switch (boardid) {
 	case 2:
-		n = "InfiniPath_QLE7140";
+		dd->boardname = "InfiniPath_QLE7140";
 		break;
 	default:
 		qib_dev_err(dd, "Unknown 6120 board with ID %u\n", boardid);
-		n = "Unknown_InfiniPath_6120";
+		dd->boardname = "Unknown_InfiniPath_6120";
 		break;
 	}
-	namelen = strlen(n) + 1;
-	dd->boardname = kmalloc(namelen, GFP_KERNEL);
-	if (dd->boardname)
-		snprintf(dd->boardname, namelen, "%s", n);
 
 	if (dd->majrev != 4 || !dd->minrev || dd->minrev > 2)
 		qib_dev_err(dd,
-			"Unsupported InfiniPath hardware revision %u.%u!\n",
-			dd->majrev, dd->minrev);
+			    "Unsupported InfiniPath hardware revision %u.%u!\n",
+			    dd->majrev, dd->minrev);
 
 	snprintf(dd->boardversion, sizeof(dd->boardversion),
 		 "ChipABI %u.%u, %s, InfiniPath%u %u.%u, SW Compat %u\n",
 		 QIB_CHIP_VERS_MAJ, QIB_CHIP_VERS_MIN, dd->boardname,
-		 (unsigned)SYM_FIELD(dd->revision, Revision_R, Arch),
+		 (unsigned int)SYM_FIELD(dd->revision, Revision_R, Arch),
 		 dd->majrev, dd->minrev,
-		 (unsigned)SYM_FIELD(dd->revision, Revision_R, SW));
-
+		 (unsigned int)SYM_FIELD(dd->revision, Revision_R, SW));
 }
 
 /*
@@ -1838,7 +1832,7 @@ static int qib_6120_setup_reset(struct qib_devdata *dd)
 
 bail:
 	if (ret) {
-		if (qib_pcie_params(dd, dd->lbus_width, NULL, NULL))
+		if (qib_pcie_params(dd, dd->lbus_width, NULL))
 			qib_dev_err(dd,
 				"Reset failed to setup PCIe or interrupts; continuing anyway\n");
 		/* clear the reset error, init error/hwerror mask */
@@ -3562,7 +3556,7 @@ struct qib_devdata *qib_init_iba6120_funcs(struct pci_dev *pdev,
 	if (qib_mini_init)
 		goto bail;
 
-	if (qib_pcie_params(dd, 8, NULL, NULL))
+	if (qib_pcie_params(dd, 8, NULL))
 		qib_dev_err(dd,
 			"Failed to setup PCIe or interrupts; continuing anyway\n");
 	dd->cspec->irq = pdev->irq; /* save IRQ */
diff --git a/drivers/infiniband/hw/qib/qib_iba7220.c b/drivers/infiniband/hw/qib/qib_iba7220.c
index c3679c48e61c..04bdd3d487b1 100644
--- a/drivers/infiniband/hw/qib/qib_iba7220.c
+++ b/drivers/infiniband/hw/qib/qib_iba7220.c
@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2011 - 2017 Intel Corporation.  All rights reserved.
  * Copyright (c) 2006, 2007, 2008, 2009, 2010 QLogic Corporation.
  * All rights reserved.
  * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
@@ -2049,41 +2050,35 @@ static void qib_setup_7220_interrupt(struct qib_devdata *dd)
  */
 static void qib_7220_boardname(struct qib_devdata *dd)
 {
-	char *n;
-	u32 boardid, namelen;
+	u32 boardid;
 
 	boardid = SYM_FIELD(dd->revision, Revision,
 			    BoardID);
 
 	switch (boardid) {
 	case 1:
-		n = "InfiniPath_QLE7240";
+		dd->boardname = "InfiniPath_QLE7240";
 		break;
 	case 2:
-		n = "InfiniPath_QLE7280";
+		dd->boardname = "InfiniPath_QLE7280";
 		break;
 	default:
 		qib_dev_err(dd, "Unknown 7220 board with ID %u\n", boardid);
-		n = "Unknown_InfiniPath_7220";
+		dd->boardname = "Unknown_InfiniPath_7220";
 		break;
 	}
 
-	namelen = strlen(n) + 1;
-	dd->boardname = kmalloc(namelen, GFP_KERNEL);
-	if (dd->boardname)
-		snprintf(dd->boardname, namelen, "%s", n);
-
 	if (dd->majrev != 5 || !dd->minrev || dd->minrev > 2)
 		qib_dev_err(dd,
-			"Unsupported InfiniPath hardware revision %u.%u!\n",
-			dd->majrev, dd->minrev);
+			    "Unsupported InfiniPath hardware revision %u.%u!\n",
+			    dd->majrev, dd->minrev);
 
 	snprintf(dd->boardversion, sizeof(dd->boardversion),
 		 "ChipABI %u.%u, %s, InfiniPath%u %u.%u, SW Compat %u\n",
 		 QIB_CHIP_VERS_MAJ, QIB_CHIP_VERS_MIN, dd->boardname,
-		 (unsigned)SYM_FIELD(dd->revision, Revision_R, Arch),
+		 (unsigned int)SYM_FIELD(dd->revision, Revision_R, Arch),
 		 dd->majrev, dd->minrev,
-		 (unsigned)SYM_FIELD(dd->revision, Revision_R, SW));
+		 (unsigned int)SYM_FIELD(dd->revision, Revision_R, SW));
 }
 
 /*
@@ -2148,7 +2143,7 @@ static int qib_setup_7220_reset(struct qib_devdata *dd)
 
 bail:
 	if (ret) {
-		if (qib_pcie_params(dd, dd->lbus_width, NULL, NULL))
+		if (qib_pcie_params(dd, dd->lbus_width, NULL))
 			qib_dev_err(dd,
 				"Reset failed to setup PCIe or interrupts; continuing anyway\n");
 
@@ -3309,7 +3304,7 @@ static int qib_7220_intr_fallback(struct qib_devdata *dd)
 	qib_devinfo(dd->pcidev,
 		"MSI interrupt not detected, trying INTx interrupts\n");
 	qib_7220_free_irq(dd);
-	qib_enable_intx(dd->pcidev);
+	qib_enable_intx(dd);
 	/*
 	 * Some newer kernels require free_irq before disable_msi,
 	 * and irq can be changed during disable and INTx enable
@@ -4619,7 +4614,7 @@ struct qib_devdata *qib_init_iba7220_funcs(struct pci_dev *pdev,
 		minwidth = 8; /* x8 capable boards */
 		break;
 	}
-	if (qib_pcie_params(dd, minwidth, NULL, NULL))
+	if (qib_pcie_params(dd, minwidth, NULL))
 		qib_dev_err(dd,
 			"Failed to setup PCIe or interrupts; continuing anyway\n");
 
diff --git a/drivers/infiniband/hw/qib/qib_iba7322.c b/drivers/infiniband/hw/qib/qib_iba7322.c
index bb2439fff8fa..14cadf6d6214 100644
--- a/drivers/infiniband/hw/qib/qib_iba7322.c
+++ b/drivers/infiniband/hw/qib/qib_iba7322.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012 Intel Corporation.  All rights reserved.
+ * Copyright (c) 2012 - 2017 Intel Corporation.  All rights reserved.
  * Copyright (c) 2008 - 2012 QLogic Corporation. All rights reserved.
  *
  * This software is available to you under a choice of one of two
@@ -2841,10 +2841,10 @@ static void qib_7322_nomsix(struct qib_devdata *dd)
 			reset_dca_notifier(dd, &dd->cspec->msix_entries[i]);
 #endif
 			irq_set_affinity_hint(
-			  dd->cspec->msix_entries[i].msix.vector, NULL);
+				dd->cspec->msix_entries[i].irq, NULL);
 			free_cpumask_var(dd->cspec->msix_entries[i].mask);
-			free_irq(dd->cspec->msix_entries[i].msix.vector,
-			   dd->cspec->msix_entries[i].arg);
+			free_irq(dd->cspec->msix_entries[i].irq,
+				 dd->cspec->msix_entries[i].arg);
 		}
 		qib_nomsix(dd);
 	}
@@ -3336,9 +3336,9 @@ static void reset_dca_notifier(struct qib_devdata *dd, struct qib_msix_entry *m)
 	qib_devinfo(dd->pcidev,
 		"Disabling notifier on HCA %d irq %d\n",
 		dd->unit,
-		m->msix.vector);
+		m->irq);
 	irq_set_affinity_notifier(
-		m->msix.vector,
+		m->irq,
 		NULL);
 	m->notifier = NULL;
 }
@@ -3354,7 +3354,7 @@ static void setup_dca_notifier(struct qib_devdata *dd, struct qib_msix_entry *m)
 		int ret;
 
 		m->notifier = n;
-		n->notify.irq = m->msix.vector;
+		n->notify.irq = m->irq;
 		n->notify.notify = qib_irq_notifier_notify;
 		n->notify.release = qib_irq_notifier_release;
 		n->arg = m->arg;
@@ -3500,10 +3500,21 @@ try_intx:
 				 - 1,
 				QIB_DRV_NAME "%d (kctx)", dd->unit);
 		}
-		ret = request_irq(
-			dd->cspec->msix_entries[msixnum].msix.vector,
-			handler, 0, dd->cspec->msix_entries[msixnum].name,
-			arg);
+
+		dd->cspec->msix_entries[msixnum].irq = pci_irq_vector(
+			dd->pcidev, msixnum);
+		if (dd->cspec->msix_entries[msixnum].irq < 0) {
+			qib_dev_err(dd,
+				    "Couldn't get MSIx irq (vec=%d): %d\n",
+				    msixnum,
+				    dd->cspec->msix_entries[msixnum].irq);
+			qib_7322_nomsix(dd);
+			goto try_intx;
+		}
+		ret = request_irq(dd->cspec->msix_entries[msixnum].irq,
+				  handler, 0,
+				  dd->cspec->msix_entries[msixnum].name,
+				  arg);
 		if (ret) {
 			/*
 			 * Shouldn't happen since the enable said we could
@@ -3512,7 +3523,7 @@ try_intx:
 			qib_dev_err(dd,
 				"Couldn't setup MSIx interrupt (vec=%d, irq=%d): %d\n",
 				msixnum,
-				dd->cspec->msix_entries[msixnum].msix.vector,
+				dd->cspec->msix_entries[msixnum].irq,
 				ret);
 			qib_7322_nomsix(dd);
 			goto try_intx;
@@ -3548,7 +3559,7 @@ try_intx:
 					dd->cspec->msix_entries[msixnum].mask);
 			}
 			irq_set_affinity_hint(
-				dd->cspec->msix_entries[msixnum].msix.vector,
+				dd->cspec->msix_entries[msixnum].irq,
 				dd->cspec->msix_entries[msixnum].mask);
 		}
 		msixnum++;
@@ -3571,75 +3582,69 @@ bail:;
 static unsigned qib_7322_boardname(struct qib_devdata *dd)
 {
 	/* Will need enumeration of board-types here */
-	char *n;
-	u32 boardid, namelen;
-	unsigned features = DUAL_PORT_CAP;
+	u32 boardid;
+	unsigned int features = DUAL_PORT_CAP;
 
 	boardid = SYM_FIELD(dd->revision, Revision, BoardID);
 
 	switch (boardid) {
 	case 0:
-		n = "InfiniPath_QLE7342_Emulation";
+		dd->boardname = "InfiniPath_QLE7342_Emulation";
 		break;
 	case 1:
-		n = "InfiniPath_QLE7340";
+		dd->boardname = "InfiniPath_QLE7340";
 		dd->flags |= QIB_HAS_QSFP;
 		features = PORT_SPD_CAP;
 		break;
 	case 2:
-		n = "InfiniPath_QLE7342";
+		dd->boardname = "InfiniPath_QLE7342";
 		dd->flags |= QIB_HAS_QSFP;
 		break;
 	case 3:
-		n = "InfiniPath_QMI7342";
+		dd->boardname = "InfiniPath_QMI7342";
 		break;
 	case 4:
-		n = "InfiniPath_Unsupported7342";
+		dd->boardname = "InfiniPath_Unsupported7342";
 		qib_dev_err(dd, "Unsupported version of QMH7342\n");
 		features = 0;
 		break;
 	case BOARD_QMH7342:
-		n = "InfiniPath_QMH7342";
+		dd->boardname = "InfiniPath_QMH7342";
 		features = 0x24;
 		break;
 	case BOARD_QME7342:
-		n = "InfiniPath_QME7342";
+		dd->boardname = "InfiniPath_QME7342";
 		break;
 	case 8:
-		n = "InfiniPath_QME7362";
+		dd->boardname = "InfiniPath_QME7362";
 		dd->flags |= QIB_HAS_QSFP;
 		break;
 	case BOARD_QMH7360:
-		n = "Intel IB QDR 1P FLR-QSFP Adptr";
+		dd->boardname = "Intel IB QDR 1P FLR-QSFP Adptr";
 		dd->flags |= QIB_HAS_QSFP;
 		break;
 	case 15:
-		n = "InfiniPath_QLE7342_TEST";
+		dd->boardname = "InfiniPath_QLE7342_TEST";
 		dd->flags |= QIB_HAS_QSFP;
 		break;
 	default:
-		n = "InfiniPath_QLE73xy_UNKNOWN";
+		dd->boardname = "InfiniPath_QLE73xy_UNKNOWN";
 		qib_dev_err(dd, "Unknown 7322 board type %u\n", boardid);
 		break;
 	}
 	dd->board_atten = 1; /* index into txdds_Xdr */
 
-	namelen = strlen(n) + 1;
-	dd->boardname = kmalloc(namelen, GFP_KERNEL);
-	if (dd->boardname)
-		snprintf(dd->boardname, namelen, "%s", n);
-
 	snprintf(dd->boardversion, sizeof(dd->boardversion),
 		 "ChipABI %u.%u, %s, InfiniPath%u %u.%u, SW Compat %u\n",
 		 QIB_CHIP_VERS_MAJ, QIB_CHIP_VERS_MIN, dd->boardname,
-		 (unsigned)SYM_FIELD(dd->revision, Revision_R, Arch),
+		 (unsigned int)SYM_FIELD(dd->revision, Revision_R, Arch),
 		 dd->majrev, dd->minrev,
-		 (unsigned)SYM_FIELD(dd->revision, Revision_R, SW));
+		 (unsigned int)SYM_FIELD(dd->revision, Revision_R, SW));
 
 	if (qib_singleport && (features >> PORT_SPD_CAP_SHIFT) & PORT_SPD_CAP) {
 		qib_devinfo(dd->pcidev,
-			"IB%u: Forced to single port mode by module parameter\n",
-			dd->unit);
+			    "IB%u: Forced to single port mode by module parameter\n",
+			    dd->unit);
 		features &= PORT_SPD_CAP;
 	}
 
@@ -3744,7 +3749,6 @@ static int qib_do_7322_reset(struct qib_devdata *dd)
 	if (msix_entries) {
 		/* restore the MSIx vector address and data if saved above */
 		for (i = 0; i < msix_entries; i++) {
-			dd->cspec->msix_entries[i].msix.entry = i;
 			if (!msix_vecsave || !msix_vecsave[2 * i])
 				continue;
 			qib_write_kreg(dd, 2 * i +
@@ -3762,8 +3766,7 @@ static int qib_do_7322_reset(struct qib_devdata *dd)
 	write_7322_initregs(dd);
 
 	if (qib_pcie_params(dd, dd->lbus_width,
-			    &dd->cspec->num_msix_entries,
-			    dd->cspec->msix_entries))
+			    &dd->cspec->num_msix_entries))
 		qib_dev_err(dd,
 			"Reset failed to setup PCIe or interrupts; continuing anyway\n");
 
@@ -5195,7 +5198,7 @@ static int qib_7322_intr_fallback(struct qib_devdata *dd)
 	qib_devinfo(dd->pcidev,
 		"MSIx interrupt not detected, trying INTx interrupts\n");
 	qib_7322_nomsix(dd);
-	qib_enable_intx(dd->pcidev);
+	qib_enable_intx(dd);
 	qib_setup_7322_interrupt(dd, 0);
 	return 1;
 }
@@ -6172,7 +6175,7 @@ static int setup_txselect(const char *str, struct kernel_param *kp)
 	unsigned long val;
 	char *n;
 
-	if (strlen(str) >= MAX_ATTEN_LEN) {
+	if (strlen(str) >= ARRAY_SIZE(txselect_list)) {
 		pr_info("txselect_values string too long\n");
 		return -ENOSPC;
 	}
@@ -6183,7 +6186,7 @@ static int setup_txselect(const char *str, struct kernel_param *kp)
 			TXDDS_TABLE_SZ + TXDDS_EXTRA_SZ + TXDDS_MFG_SZ);
 		return -EINVAL;
 	}
-	strcpy(txselect_list, str);
+	strncpy(txselect_list, str, ARRAY_SIZE(txselect_list) - 1);
 
 	list_for_each_entry(dd, &qib_dev_list, list)
 		if (dd->deviceid == PCI_DEVICE_ID_QLOGIC_IB_7322)
@@ -7327,10 +7330,7 @@ struct qib_devdata *qib_init_iba7322_funcs(struct pci_dev *pdev,
 	if (!dd->cspec->msix_entries)
 		tabsize = 0;
 
-	for (i = 0; i < tabsize; i++)
-		dd->cspec->msix_entries[i].msix.entry = i;
-
-	if (qib_pcie_params(dd, 8, &tabsize, dd->cspec->msix_entries))
+	if (qib_pcie_params(dd, 8, &tabsize))
 		qib_dev_err(dd,
 			"Failed to setup PCIe or interrupts; continuing anyway\n");
 	/* may be less than we wanted, if not enough available */
diff --git a/drivers/infiniband/hw/qib/qib_init.c b/drivers/infiniband/hw/qib/qib_init.c
index 6c16ba1107ba..c5a4c65636d6 100644
--- a/drivers/infiniband/hw/qib/qib_init.c
+++ b/drivers/infiniband/hw/qib/qib_init.c
@@ -399,7 +399,7 @@ static int loadtime_init(struct qib_devdata *dd)
 	if (((dd->revision >> QLOGIC_IB_R_SOFTWARE_SHIFT) &
 	     QLOGIC_IB_R_SOFTWARE_MASK) != QIB_CHIP_SWVERSION) {
 		qib_dev_err(dd,
-			"Driver only handles version %d, chip swversion is %d (%llx), failng\n",
+			"Driver only handles version %d, chip swversion is %d (%llx), failing\n",
 			QIB_CHIP_SWVERSION,
 			(int)(dd->revision >>
 				QLOGIC_IB_R_SOFTWARE_SHIFT) &
@@ -1398,7 +1398,6 @@ static void cleanup_device_data(struct qib_devdata *dd)
 		qib_free_ctxtdata(dd, rcd);
 	}
 	kfree(tmp);
-	kfree(dd->boardname);
 }
 
 /*
diff --git a/drivers/infiniband/hw/qib/qib_mad.c b/drivers/infiniband/hw/qib/qib_mad.c
index da295e0392ed..82d9da9b6997 100644
--- a/drivers/infiniband/hw/qib/qib_mad.c
+++ b/drivers/infiniband/hw/qib/qib_mad.c
@@ -105,7 +105,7 @@ static void qib_send_trap(struct qib_ibport *ibp, void *data, unsigned len)
 		if (ibp->rvp.sm_lid != be16_to_cpu(IB_LID_PERMISSIVE)) {
 			struct ib_ah *ah;
 
-			ah = qib_create_qp0_ah(ibp, ibp->rvp.sm_lid);
+			ah = qib_create_qp0_ah(ibp, (u16)ibp->rvp.sm_lid);
 			if (IS_ERR(ah))
 				ret = PTR_ERR(ah);
 			else {
@@ -134,24 +134,21 @@ static void qib_send_trap(struct qib_ibport *ibp, void *data, unsigned len)
 }
 
 /*
- * Send a bad [PQ]_Key trap (ch. 14.3.8).
+ * Send a bad P_Key trap (ch. 14.3.8).
  */
-void qib_bad_pqkey(struct qib_ibport *ibp, __be16 trap_num, u32 key, u32 sl,
-		   u32 qp1, u32 qp2, __be16 lid1, __be16 lid2)
+void qib_bad_pkey(struct qib_ibport *ibp, u32 key, u32 sl,
+		  u32 qp1, u32 qp2, __be16 lid1, __be16 lid2)
 {
 	struct ib_mad_notice_attr data;
 
-	if (trap_num == IB_NOTICE_TRAP_BAD_PKEY)
-		ibp->rvp.pkey_violations++;
-	else
-		ibp->rvp.qkey_violations++;
 	ibp->rvp.n_pkt_drops++;
+	ibp->rvp.pkey_violations++;
 
 	/* Send violation trap */
 	data.generic_type = IB_NOTICE_TYPE_SECURITY;
 	data.prod_type_msb = 0;
 	data.prod_type_lsb = IB_NOTICE_PROD_CA;
-	data.trap_num = trap_num;
+	data.trap_num = IB_NOTICE_TRAP_BAD_PKEY;
 	data.issuer_lid = cpu_to_be16(ppd_from_ibp(ibp)->lid);
 	data.toggle_count = 0;
 	memset(&data.details, 0, sizeof(data.details));
@@ -499,7 +496,7 @@ static int subn_get_portinfo(struct ib_smp *smp, struct ib_device *ibdev,
 		pip->mkey = ibp->rvp.mkey;
 	pip->gid_prefix = ibp->rvp.gid_prefix;
 	pip->lid = cpu_to_be16(ppd->lid);
-	pip->sm_lid = cpu_to_be16(ibp->rvp.sm_lid);
+	pip->sm_lid = cpu_to_be16((u16)ibp->rvp.sm_lid);
 	pip->cap_mask = cpu_to_be32(ibp->rvp.port_cap_flags);
 	/* pip->diag_code; */
 	pip->mkey_lease_period = cpu_to_be16(ibp->rvp.mkey_lease_period);
@@ -874,8 +871,6 @@ static int subn_set_portinfo(struct ib_smp *smp, struct ib_device *ibdev,
 		ib_dispatch_event(&event);
 	}
 
-	ret = subn_get_portinfo(smp, ibdev, port);
-
 	/* restore re-reg bit per o14-12.2.1 */
 	pip->clientrereg_resv_subnetto |= clientrereg;
 
@@ -1578,8 +1573,8 @@ static int pma_get_portcounters_cong(struct ib_pma_mad *pmp,
 	cntrs.port_xmit_packets -= ibp->z_port_xmit_packets;
 	cntrs.port_rcv_packets -= ibp->z_port_rcv_packets;
 
-	memset(pmp->reserved, 0, sizeof(pmp->reserved) +
-	       sizeof(pmp->data));
+	memset(pmp->reserved, 0, sizeof(pmp->reserved));
+	memset(pmp->data, 0, sizeof(pmp->data));
 
 	/*
 	 * Set top 3 bits to indicate interval in picoseconds in
diff --git a/drivers/infiniband/hw/qib/qib_pcie.c b/drivers/infiniband/hw/qib/qib_pcie.c
index c379b8342a09..d90403e31a9d 100644
--- a/drivers/infiniband/hw/qib/qib_pcie.c
+++ b/drivers/infiniband/hw/qib/qib_pcie.c
@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2010 - 2017 Intel Corporation.  All rights reserved.
  * Copyright (c) 2008, 2009 QLogic Corporation. All rights reserved.
  *
  * This software is available to you under a choice of one of two
@@ -187,112 +188,84 @@ void qib_pcie_ddcleanup(struct qib_devdata *dd)
 	pci_set_drvdata(dd->pcidev, NULL);
 }
 
-static void qib_msix_setup(struct qib_devdata *dd, int pos, u32 *msixcnt,
-			   struct qib_msix_entry *qib_msix_entry)
-{
-	int ret;
-	int nvec = *msixcnt;
-	struct msix_entry *msix_entry;
-	int i;
-
-	ret = pci_msix_vec_count(dd->pcidev);
-	if (ret < 0)
-		goto do_intx;
-
-	nvec = min(nvec, ret);
-
-	/* We can't pass qib_msix_entry array to qib_msix_setup
-	 * so use a dummy msix_entry array and copy the allocated
-	 * irq back to the qib_msix_entry array. */
-	msix_entry = kcalloc(nvec, sizeof(*msix_entry), GFP_KERNEL);
-	if (!msix_entry)
-		goto do_intx;
-
-	for (i = 0; i < nvec; i++)
-		msix_entry[i] = qib_msix_entry[i].msix;
-
-	ret = pci_enable_msix_range(dd->pcidev, msix_entry, 1, nvec);
-	if (ret < 0)
-		goto free_msix_entry;
-	else
-		nvec = ret;
-
-	for (i = 0; i < nvec; i++)
-		qib_msix_entry[i].msix = msix_entry[i];
-
-	kfree(msix_entry);
-	*msixcnt = nvec;
-	return;
-
-free_msix_entry:
-	kfree(msix_entry);
-
-do_intx:
-	qib_dev_err(
-		dd,
-		"pci_enable_msix_range %d vectors failed: %d, falling back to INTx\n",
-		nvec, ret);
-	*msixcnt = 0;
-	qib_enable_intx(dd->pcidev);
-}
-
 /**
  * We save the msi lo and hi values, so we can restore them after
  * chip reset (the kernel PCI infrastructure doesn't yet handle that
  * correctly.
  */
-static int qib_msi_setup(struct qib_devdata *dd, int pos)
+static void qib_msi_setup(struct qib_devdata *dd, int pos)
 {
 	struct pci_dev *pdev = dd->pcidev;
 	u16 control;
-	int ret;
 
-	ret = pci_enable_msi(pdev);
-	if (ret)
-		qib_dev_err(dd,
-			"pci_enable_msi failed: %d, interrupts may not work\n",
-			ret);
-	/* continue even if it fails, we may still be OK... */
-
-	pci_read_config_dword(pdev, pos + PCI_MSI_ADDRESS_LO,
-			      &dd->msi_lo);
-	pci_read_config_dword(pdev, pos + PCI_MSI_ADDRESS_HI,
-			      &dd->msi_hi);
+	pci_read_config_dword(pdev, pos + PCI_MSI_ADDRESS_LO, &dd->msi_lo);
+	pci_read_config_dword(pdev, pos + PCI_MSI_ADDRESS_HI, &dd->msi_hi);
 	pci_read_config_word(pdev, pos + PCI_MSI_FLAGS, &control);
+
 	/* now save the data (vector) info */
-	pci_read_config_word(pdev, pos + ((control & PCI_MSI_FLAGS_64BIT)
-				    ? 12 : 8),
+	pci_read_config_word(pdev,
+			     pos + ((control & PCI_MSI_FLAGS_64BIT) ? 12 : 8),
 			     &dd->msi_data);
-	return ret;
 }
 
-int qib_pcie_params(struct qib_devdata *dd, u32 minw, u32 *nent,
-		    struct qib_msix_entry *entry)
+static int qib_allocate_irqs(struct qib_devdata *dd, u32 maxvec)
+{
+	unsigned int flags = PCI_IRQ_LEGACY;
+
+	/* Check our capabilities */
+	if (dd->pcidev->msix_cap) {
+		flags |= PCI_IRQ_MSIX;
+	} else {
+		if (dd->pcidev->msi_cap) {
+			flags |= PCI_IRQ_MSI;
+			/* Get msi_lo and msi_hi */
+			qib_msi_setup(dd, dd->pcidev->msi_cap);
+		}
+	}
+
+	if (!(flags & (PCI_IRQ_MSIX | PCI_IRQ_MSI)))
+		qib_dev_err(dd, "No PCI MSI or MSIx capability!\n");
+
+	return pci_alloc_irq_vectors(dd->pcidev, 1, maxvec, flags);
+}
+
+int qib_pcie_params(struct qib_devdata *dd, u32 minw, u32 *nent)
 {
 	u16 linkstat, speed;
-	int pos = 0, ret = 1;
+	int nvec;
+	int maxvec;
+	int ret = 0;
 
 	if (!pci_is_pcie(dd->pcidev)) {
 		qib_dev_err(dd, "Can't find PCI Express capability!\n");
 		/* set up something... */
 		dd->lbus_width = 1;
 		dd->lbus_speed = 2500; /* Gen1, 2.5GHz */
+		ret = -1;
 		goto bail;
 	}
 
-	pos = dd->pcidev->msix_cap;
-	if (nent && *nent && pos) {
-		qib_msix_setup(dd, pos, nent, entry);
-		ret = 0; /* did it, either MSIx or INTx */
-	} else {
-		pos = dd->pcidev->msi_cap;
-		if (pos)
-			ret = qib_msi_setup(dd, pos);
-		else
-			qib_dev_err(dd, "No PCI MSI or MSIx capability!\n");
+	maxvec = (nent && *nent) ? *nent : 1;
+	nvec = qib_allocate_irqs(dd, maxvec);
+	if (nvec < 0) {
+		ret = nvec;
+		goto bail;
+	}
+
+	/*
+	 * If nent exists, make sure to record how many vectors were allocated
+	 */
+	if (nent) {
+		*nent = nvec;
+
+		/*
+		 * If we requested (nent) MSIX, but msix_enabled is not set,
+		 * pci_alloc_irq_vectors() enabled INTx.
+		 */
+		if (!dd->pcidev->msix_enabled)
+			qib_dev_err(dd,
+				    "no msix vectors allocated, using INTx\n");
 	}
-	if (!pos)
-		qib_enable_intx(dd->pcidev);
 
 	pcie_capability_read_word(dd->pcidev, PCI_EXP_LNKSTA, &linkstat);
 	/*
@@ -379,7 +352,7 @@ int qib_reinit_intr(struct qib_devdata *dd)
 	ret = 1;
 bail:
 	if (!ret && (dd->flags & QIB_HAS_INTX)) {
-		qib_enable_intx(dd->pcidev);
+		qib_enable_intx(dd);
 		ret = 1;
 	}
 
@@ -397,7 +370,7 @@ bail:
 void qib_nomsi(struct qib_devdata *dd)
 {
 	dd->msi_lo = 0;
-	pci_disable_msi(dd->pcidev);
+	pci_free_irq_vectors(dd->pcidev);
 }
 
 /*
@@ -405,23 +378,21 @@ void qib_nomsi(struct qib_devdata *dd)
  */
 void qib_nomsix(struct qib_devdata *dd)
 {
-	pci_disable_msix(dd->pcidev);
+	pci_free_irq_vectors(dd->pcidev);
 }
 
 /*
  * Similar to pci_intx(pdev, 1), except that we make sure
  * msi(x) is off.
  */
-void qib_enable_intx(struct pci_dev *pdev)
+void qib_enable_intx(struct qib_devdata *dd)
 {
 	u16 cw, new;
 	int pos;
+	struct pci_dev *pdev = dd->pcidev;
 
-	/* first, turn on INTx */
-	pci_read_config_word(pdev, PCI_COMMAND, &cw);
-	new = cw & ~PCI_COMMAND_INTX_DISABLE;
-	if (new != cw)
-		pci_write_config_word(pdev, PCI_COMMAND, new);
+	if (pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_LEGACY) < 0)
+		qib_dev_err(dd,	"Failed to enable INTx\n");
 
 	pos = pdev->msi_cap;
 	if (pos) {
diff --git a/drivers/infiniband/hw/qib/qib_qp.c b/drivers/infiniband/hw/qib/qib_qp.c
index a343e3b5d4cb..344e401915f7 100644
--- a/drivers/infiniband/hw/qib/qib_qp.c
+++ b/drivers/infiniband/hw/qib/qib_qp.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012, 2013 Intel Corporation.  All rights reserved.
+ * Copyright (c) 2012 - 2017 Intel Corporation.  All rights reserved.
  * Copyright (c) 2006 - 2012 QLogic Corporation.  * All rights reserved.
  * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
  *
@@ -415,53 +415,16 @@ int qib_check_send_wqe(struct rvt_qp *qp,
 
 #ifdef CONFIG_DEBUG_FS
 
-struct qib_qp_iter {
-	struct qib_ibdev *dev;
-	struct rvt_qp *qp;
-	int n;
-};
-
-struct qib_qp_iter *qib_qp_iter_init(struct qib_ibdev *dev)
-{
-	struct qib_qp_iter *iter;
-
-	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
-	if (!iter)
-		return NULL;
-
-	iter->dev = dev;
-
-	return iter;
-}
-
-int qib_qp_iter_next(struct qib_qp_iter *iter)
-{
-	struct qib_ibdev *dev = iter->dev;
-	int n = iter->n;
-	int ret = 1;
-	struct rvt_qp *pqp = iter->qp;
-	struct rvt_qp *qp;
-
-	for (; n < dev->rdi.qp_dev->qp_table_size; n++) {
-		if (pqp)
-			qp = rcu_dereference(pqp->next);
-		else
-			qp = rcu_dereference(dev->rdi.qp_dev->qp_table[n]);
-		pqp = qp;
-		if (qp) {
-			iter->qp = qp;
-			iter->n = n;
-			return 0;
-		}
-	}
-	return ret;
-}
-
 static const char * const qp_type_str[] = {
 	"SMI", "GSI", "RC", "UC", "UD",
 };
 
-void qib_qp_iter_print(struct seq_file *s, struct qib_qp_iter *iter)
+/**
+ * qib_qp_iter_print - print information to seq_file
+ * @s - the seq_file
+ * @iter - the iterator
+ */
+void qib_qp_iter_print(struct seq_file *s, struct rvt_qp_iter *iter)
 {
 	struct rvt_swqe *wqe;
 	struct rvt_qp *qp = iter->qp;
diff --git a/drivers/infiniband/hw/qib/qib_rc.c b/drivers/infiniband/hw/qib/qib_rc.c
index 4ddbcac5eabe..e9a91736b12d 100644
--- a/drivers/infiniband/hw/qib/qib_rc.c
+++ b/drivers/infiniband/hw/qib/qib_rc.c
@@ -348,7 +348,7 @@ int qib_make_rc_req(struct rvt_qp *qp, unsigned long *flags)
 		case IB_WR_RDMA_WRITE:
 			if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
 				qp->s_lsn++;
-			/* FALLTHROUGH */
+			goto no_flow_control;
 		case IB_WR_RDMA_WRITE_WITH_IMM:
 			/* If no credit, return. */
 			if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT) &&
@@ -356,7 +356,7 @@ int qib_make_rc_req(struct rvt_qp *qp, unsigned long *flags)
 				qp->s_flags |= RVT_S_WAIT_SSN_CREDIT;
 				goto bail;
 			}
-
+no_flow_control:
 			ohdr->u.rc.reth.vaddr =
 				cpu_to_be64(wqe->rdma_wr.remote_addr);
 			ohdr->u.rc.reth.rkey =
diff --git a/drivers/infiniband/hw/qib/qib_ruc.c b/drivers/infiniband/hw/qib/qib_ruc.c
index bd09de7c6e56..53efbb0b40c4 100644
--- a/drivers/infiniband/hw/qib/qib_ruc.c
+++ b/drivers/infiniband/hw/qib/qib_ruc.c
@@ -58,8 +58,10 @@ static int qib_init_sge(struct rvt_qp *qp, struct rvt_rwqe *wqe)
 		if (wqe->sg_list[i].length == 0)
 			continue;
 		/* Check LKEY */
-		if (!rvt_lkey_ok(rkt, pd, j ? &ss->sg_list[j - 1] : &ss->sge,
-				 &wqe->sg_list[i], IB_ACCESS_LOCAL_WRITE))
+		ret = rvt_lkey_ok(rkt, pd, j ? &ss->sg_list[j - 1] : &ss->sge,
+				  NULL, &wqe->sg_list[i],
+				  IB_ACCESS_LOCAL_WRITE);
+		if (unlikely(ret <= 0))
 			goto bad_lkey;
 		qp->r_len += wqe->sg_list[i].length;
 		j++;
@@ -256,11 +258,11 @@ int qib_ruc_check_hdr(struct qib_ibport *ibp, struct ib_header *hdr,
 		}
 		if (!qib_pkey_ok((u16)bth0,
 				 qib_get_pkey(ibp, qp->s_alt_pkey_index))) {
-			qib_bad_pqkey(ibp, IB_NOTICE_TRAP_BAD_PKEY,
-				      (u16)bth0,
-				      (be16_to_cpu(hdr->lrh[0]) >> 4) & 0xF,
-				      0, qp->ibqp.qp_num,
-				      hdr->lrh[3], hdr->lrh[1]);
+			qib_bad_pkey(ibp,
+				     (u16)bth0,
+				     (be16_to_cpu(hdr->lrh[0]) >> 4) & 0xF,
+				     0, qp->ibqp.qp_num,
+				     hdr->lrh[3], hdr->lrh[1]);
 			goto err;
 		}
 		/* Validate the SLID. See Ch. 9.6.1.5 and 17.2.8 */
@@ -295,11 +297,11 @@ int qib_ruc_check_hdr(struct qib_ibport *ibp, struct ib_header *hdr,
 		}
 		if (!qib_pkey_ok((u16)bth0,
 				 qib_get_pkey(ibp, qp->s_pkey_index))) {
-			qib_bad_pqkey(ibp, IB_NOTICE_TRAP_BAD_PKEY,
-				      (u16)bth0,
-				      (be16_to_cpu(hdr->lrh[0]) >> 4) & 0xF,
-				      0, qp->ibqp.qp_num,
-				      hdr->lrh[3], hdr->lrh[1]);
+			qib_bad_pkey(ibp,
+				     (u16)bth0,
+				     (be16_to_cpu(hdr->lrh[0]) >> 4) & 0xF,
+				     0, qp->ibqp.qp_num,
+				     hdr->lrh[3], hdr->lrh[1]);
 			goto err;
 		}
 		/* Validate the SLID. See Ch. 9.6.1.5 */
@@ -643,8 +645,10 @@ u32 qib_make_grh(struct qib_ibport *ibp, struct ib_grh *hdr,
 	hdr->hop_limit = grh->hop_limit;
 	/* The SGID is 32-bit aligned. */
 	hdr->sgid.global.subnet_prefix = ibp->rvp.gid_prefix;
-	hdr->sgid.global.interface_id = grh->sgid_index ?
-		ibp->guids[grh->sgid_index - 1] : ppd_from_ibp(ibp)->guid;
+	if (!grh->sgid_index)
+		hdr->sgid.global.interface_id = ppd_from_ibp(ibp)->guid;
+	else if (grh->sgid_index < QIB_GUIDS_PER_PORT)
+		hdr->sgid.global.interface_id = ibp->guids[grh->sgid_index - 1];
 	hdr->dgid = grh->dgid;
 
 	/* GRH header size in 32-bit words. */
diff --git a/drivers/infiniband/hw/qib/qib_sysfs.c b/drivers/infiniband/hw/qib/qib_sysfs.c
index fe4cf5e4acec..ca2638d8f35e 100644
--- a/drivers/infiniband/hw/qib/qib_sysfs.c
+++ b/drivers/infiniband/hw/qib/qib_sysfs.c
@@ -247,7 +247,7 @@ static struct kobj_type qib_port_cc_ktype = {
 	.release = qib_port_release,
 };
 
-static struct bin_attribute cc_table_bin_attr = {
+static const struct bin_attribute cc_table_bin_attr = {
 	.attr = {.name = "cc_table_bin", .mode = 0444},
 	.read = read_cc_table_bin,
 	.size = PAGE_SIZE,
@@ -286,7 +286,7 @@ static ssize_t read_cc_setting_bin(struct file *filp, struct kobject *kobj,
 	return count;
 }
 
-static struct bin_attribute cc_setting_bin_attr = {
+static const struct bin_attribute cc_setting_bin_attr = {
 	.attr = {.name = "cc_settings_bin", .mode = 0444},
 	.read = read_cc_setting_bin,
 	.size = PAGE_SIZE,
diff --git a/drivers/infiniband/hw/qib/qib_ud.c b/drivers/infiniband/hw/qib/qib_ud.c
index 341a123ee95c..be4907453ac4 100644
--- a/drivers/infiniband/hw/qib/qib_ud.c
+++ b/drivers/infiniband/hw/qib/qib_ud.c
@@ -66,8 +66,7 @@ static void qib_ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe)
 	qp = rvt_lookup_qpn(rdi, &ibp->rvp, swqe->ud_wr.remote_qpn);
 	if (!qp) {
 		ibp->rvp.n_pkt_drops++;
-		rcu_read_unlock();
-		return;
+		goto drop;
 	}
 
 	sqptype = sqp->ibqp.qp_type == IB_QPT_GSI ?
@@ -94,11 +93,11 @@ static void qib_ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe)
 		if (unlikely(!qib_pkey_ok(pkey1, pkey2))) {
 			lid = ppd->lid | (rdma_ah_get_path_bits(ah_attr) &
 					  ((1 << ppd->lmc) - 1));
-			qib_bad_pqkey(ibp, IB_NOTICE_TRAP_BAD_PKEY, pkey1,
-				      rdma_ah_get_sl(ah_attr),
-				      sqp->ibqp.qp_num, qp->ibqp.qp_num,
-				      cpu_to_be16(lid),
-				      cpu_to_be16(rdma_ah_get_dlid(ah_attr)));
+			qib_bad_pkey(ibp, pkey1,
+				     rdma_ah_get_sl(ah_attr),
+				     sqp->ibqp.qp_num, qp->ibqp.qp_num,
+				     cpu_to_be16(lid),
+				     cpu_to_be16(rdma_ah_get_dlid(ah_attr)));
 			goto drop;
 		}
 	}
@@ -113,18 +112,8 @@ static void qib_ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe)
 
 		qkey = (int)swqe->ud_wr.remote_qkey < 0 ?
 			sqp->qkey : swqe->ud_wr.remote_qkey;
-		if (unlikely(qkey != qp->qkey)) {
-			u16 lid;
-
-			lid = ppd->lid | (rdma_ah_get_path_bits(ah_attr) &
-					  ((1 << ppd->lmc) - 1));
-			qib_bad_pqkey(ibp, IB_NOTICE_TRAP_BAD_QKEY, qkey,
-				      rdma_ah_get_sl(ah_attr),
-				      sqp->ibqp.qp_num, qp->ibqp.qp_num,
-				      cpu_to_be16(lid),
-				      cpu_to_be16(rdma_ah_get_dlid(ah_attr)));
+		if (unlikely(qkey != qp->qkey))
 			goto drop;
-		}
 	}
 
 	/*
@@ -487,22 +476,18 @@ void qib_ud_rcv(struct qib_ibport *ibp, struct ib_header *hdr,
 			pkey1 = be32_to_cpu(ohdr->bth[0]);
 			pkey2 = qib_get_pkey(ibp, qp->s_pkey_index);
 			if (unlikely(!qib_pkey_ok(pkey1, pkey2))) {
-				qib_bad_pqkey(ibp, IB_NOTICE_TRAP_BAD_PKEY,
-					      pkey1,
-					      (be16_to_cpu(hdr->lrh[0]) >> 4) &
+				qib_bad_pkey(ibp,
+					     pkey1,
+					     (be16_to_cpu(hdr->lrh[0]) >> 4) &
 						0xF,
-					      src_qp, qp->ibqp.qp_num,
-					      hdr->lrh[3], hdr->lrh[1]);
+					     src_qp, qp->ibqp.qp_num,
+					     hdr->lrh[3], hdr->lrh[1]);
 				return;
 			}
 		}
-		if (unlikely(qkey != qp->qkey)) {
-			qib_bad_pqkey(ibp, IB_NOTICE_TRAP_BAD_QKEY, qkey,
-				      (be16_to_cpu(hdr->lrh[0]) >> 4) & 0xF,
-				      src_qp, qp->ibqp.qp_num,
-				      hdr->lrh[3], hdr->lrh[1]);
+		if (unlikely(qkey != qp->qkey))
 			return;
-		}
+
 		/* Drop invalid MAD packets (see 13.5.3.1). */
 		if (unlikely(qp->ibqp.qp_num == 1 &&
 			     (tlen != 256 ||
diff --git a/drivers/infiniband/hw/qib/qib_verbs.c b/drivers/infiniband/hw/qib/qib_verbs.c
index ac42dce7e281..9d92aeb8d9a1 100644
--- a/drivers/infiniband/hw/qib/qib_verbs.c
+++ b/drivers/infiniband/hw/qib/qib_verbs.c
@@ -1341,6 +1341,15 @@ int qib_check_ah(struct ib_device *ibdev, struct rdma_ah_attr *ah_attr)
 	if (rdma_ah_get_sl(ah_attr) > 15)
 		return -EINVAL;
 
+	if (rdma_ah_get_dlid(ah_attr) == 0)
+		return -EINVAL;
+	if (rdma_ah_get_dlid(ah_attr) >=
+		be16_to_cpu(IB_MULTICAST_LID_BASE) &&
+	    rdma_ah_get_dlid(ah_attr) !=
+		be16_to_cpu(IB_LID_PERMISSIVE) &&
+	    !(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH))
+		return -EINVAL;
+
 	return 0;
 }
 
diff --git a/drivers/infiniband/hw/qib/qib_verbs.h b/drivers/infiniband/hw/qib/qib_verbs.h
index a52fc67b40d7..f887737ac142 100644
--- a/drivers/infiniband/hw/qib/qib_verbs.h
+++ b/drivers/infiniband/hw/qib/qib_verbs.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012, 2013 Intel Corporation.  All rights reserved.
+ * Copyright (c) 2012 - 2017 Intel Corporation.  All rights reserved.
  * Copyright (c) 2006 - 2012 QLogic Corporation. All rights reserved.
  * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
  *
@@ -241,8 +241,8 @@ static inline int qib_pkey_ok(u16 pkey1, u16 pkey2)
 	return p1 && p1 == p2 && ((__s16)pkey1 < 0 || (__s16)pkey2 < 0);
 }
 
-void qib_bad_pqkey(struct qib_ibport *ibp, __be16 trap_num, u32 key, u32 sl,
-		   u32 qp1, u32 qp2, __be16 lid1, __be16 lid2);
+void qib_bad_pkey(struct qib_ibport *ibp, u32 key, u32 sl,
+		  u32 qp1, u32 qp2, __be16 lid1, __be16 lid2);
 void qib_cap_mask_chg(struct rvt_dev_info *rdi, u8 port_num);
 void qib_sys_guid_chg(struct qib_ibport *ibp);
 void qib_node_desc_chg(struct qib_ibport *ibp);
@@ -282,13 +282,7 @@ int qib_alloc_qpn(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt,
 void qib_restart_rc(struct rvt_qp *qp, u32 psn, int wait);
 #ifdef CONFIG_DEBUG_FS
 
-struct qib_qp_iter;
-
-struct qib_qp_iter *qib_qp_iter_init(struct qib_ibdev *dev);
-
-int qib_qp_iter_next(struct qib_qp_iter *iter);
-
-void qib_qp_iter_print(struct seq_file *s, struct qib_qp_iter *iter);
+void qib_qp_iter_print(struct seq_file *s, struct rvt_qp_iter *iter);
 
 #endif
 
diff --git a/drivers/infiniband/hw/usnic/usnic_fwd.c b/drivers/infiniband/hw/usnic/usnic_fwd.c
index 3c37dd59c04e..995a26b65156 100644
--- a/drivers/infiniband/hw/usnic/usnic_fwd.c
+++ b/drivers/infiniband/hw/usnic/usnic_fwd.c
@@ -110,20 +110,12 @@ void usnic_fwd_set_mac(struct usnic_fwd_dev *ufdev, char mac[ETH_ALEN])
 	spin_unlock(&ufdev->lock);
 }
 
-int usnic_fwd_add_ipaddr(struct usnic_fwd_dev *ufdev, __be32 inaddr)
+void usnic_fwd_add_ipaddr(struct usnic_fwd_dev *ufdev, __be32 inaddr)
 {
-	int status;
-
 	spin_lock(&ufdev->lock);
-	if (ufdev->inaddr == 0) {
+	if (!ufdev->inaddr)
 		ufdev->inaddr = inaddr;
-		status = 0;
-	} else {
-		status = -EFAULT;
-	}
 	spin_unlock(&ufdev->lock);
-
-	return status;
 }
 
 void usnic_fwd_del_ipaddr(struct usnic_fwd_dev *ufdev)
diff --git a/drivers/infiniband/hw/usnic/usnic_fwd.h b/drivers/infiniband/hw/usnic/usnic_fwd.h
index b2ac22be0731..0b2cc4e79707 100644
--- a/drivers/infiniband/hw/usnic/usnic_fwd.h
+++ b/drivers/infiniband/hw/usnic/usnic_fwd.h
@@ -75,7 +75,7 @@ struct usnic_fwd_dev *usnic_fwd_dev_alloc(struct pci_dev *pdev);
 void usnic_fwd_dev_free(struct usnic_fwd_dev *ufdev);
 
 void usnic_fwd_set_mac(struct usnic_fwd_dev *ufdev, char mac[ETH_ALEN]);
-int usnic_fwd_add_ipaddr(struct usnic_fwd_dev *ufdev, __be32 inaddr);
+void usnic_fwd_add_ipaddr(struct usnic_fwd_dev *ufdev, __be32 inaddr);
 void usnic_fwd_del_ipaddr(struct usnic_fwd_dev *ufdev);
 void usnic_fwd_carrier_up(struct usnic_fwd_dev *ufdev);
 void usnic_fwd_carrier_down(struct usnic_fwd_dev *ufdev);
diff --git a/drivers/infiniband/hw/usnic/usnic_ib_main.c b/drivers/infiniband/hw/usnic/usnic_ib_main.c
index c0c1e8b027b1..f45e99a938e0 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib_main.c
+++ b/drivers/infiniband/hw/usnic/usnic_ib_main.c
@@ -333,9 +333,7 @@ static int usnic_port_immutable(struct ib_device *ibdev, u8 port_num,
 	return 0;
 }
 
-static void usnic_get_dev_fw_str(struct ib_device *device,
-				 char *str,
-				 size_t str_len)
+static void usnic_get_dev_fw_str(struct ib_device *device, char *str)
 {
 	struct usnic_ib_dev *us_ibdev =
 		container_of(device, struct usnic_ib_dev, ib_dev);
@@ -345,7 +343,7 @@ static void usnic_get_dev_fw_str(struct ib_device *device,
 	us_ibdev->netdev->ethtool_ops->get_drvinfo(us_ibdev->netdev, &info);
 	mutex_unlock(&us_ibdev->usdev_lock);
 
-	snprintf(str, str_len, "%s", info.fw_version);
+	snprintf(str, IB_FW_VERSION_NAME_MAX, "%s", info.fw_version);
 }
 
 /* Start of PF discovery section */
@@ -353,7 +351,7 @@ static void *usnic_ib_device_add(struct pci_dev *dev)
 {
 	struct usnic_ib_dev *us_ibdev;
 	union ib_gid gid;
-	struct in_ifaddr *in;
+	struct in_device *ind;
 	struct net_device *netdev;
 
 	usnic_dbg("\n");
@@ -409,6 +407,7 @@ static void *usnic_ib_device_add(struct pci_dev *dev)
 	us_ibdev->ib_dev.query_port = usnic_ib_query_port;
 	us_ibdev->ib_dev.query_pkey = usnic_ib_query_pkey;
 	us_ibdev->ib_dev.query_gid = usnic_ib_query_gid;
+	us_ibdev->ib_dev.get_netdev = usnic_get_netdev;
 	us_ibdev->ib_dev.get_link_layer = usnic_ib_port_link_layer;
 	us_ibdev->ib_dev.alloc_pd = usnic_ib_alloc_pd;
 	us_ibdev->ib_dev.dealloc_pd = usnic_ib_dealloc_pd;
@@ -442,9 +441,11 @@ static void *usnic_ib_device_add(struct pci_dev *dev)
 	if (netif_carrier_ok(us_ibdev->netdev))
 		usnic_fwd_carrier_up(us_ibdev->ufdev);
 
-	in = ((struct in_device *)(netdev->ip_ptr))->ifa_list;
-	if (in != NULL)
-		usnic_fwd_add_ipaddr(us_ibdev->ufdev, in->ifa_address);
+	ind = in_dev_get(netdev);
+	if (ind->ifa_list)
+		usnic_fwd_add_ipaddr(us_ibdev->ufdev,
+				     ind->ifa_list->ifa_address);
+	in_dev_put(ind);
 
 	usnic_mac_ip_to_gid(us_ibdev->netdev->perm_addr,
 				us_ibdev->ufdev->inaddr, &gid.raw[0]);
@@ -720,7 +721,6 @@ static void __exit usnic_ib_destroy(void)
 MODULE_DESCRIPTION("Cisco VIC (usNIC) Verbs Driver");
 MODULE_AUTHOR("Upinder Malhi <umalhi@cisco.com>");
 MODULE_LICENSE("Dual BSD/GPL");
-MODULE_VERSION(DRV_VERSION);
 module_param(usnic_log_lvl, uint, S_IRUGO | S_IWUSR);
 module_param(usnic_ib_share_vf, uint, S_IRUGO | S_IWUSR);
 MODULE_PARM_DESC(usnic_log_lvl, " Off=0, Err=1, Info=2, Debug=3");
diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
index 4996984885c2..e4113ef09315 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
+++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
@@ -164,6 +164,8 @@ find_free_vf_and_create_qp_grp(struct usnic_ib_dev *us_ibdev,
 	if (usnic_ib_share_vf) {
 		/* Try to find resouces on a used vf which is in pd */
 		dev_list = usnic_uiom_get_dev_list(pd->umem_pd);
+		if (IS_ERR(dev_list))
+			return ERR_CAST(dev_list);
 		for (i = 0; dev_list[i]; i++) {
 			dev = dev_list[i];
 			vf = pci_get_drvdata(to_pci_dev(dev));
@@ -226,27 +228,6 @@ static void qp_grp_destroy(struct usnic_ib_qp_grp *qp_grp)
 	spin_unlock(&vf->lock);
 }
 
-static void eth_speed_to_ib_speed(int speed, u8 *active_speed,
-					u8 *active_width)
-{
-	if (speed <= 10000) {
-		*active_width = IB_WIDTH_1X;
-		*active_speed = IB_SPEED_FDR10;
-	} else if (speed <= 20000) {
-		*active_width = IB_WIDTH_4X;
-		*active_speed = IB_SPEED_DDR;
-	} else if (speed <= 30000) {
-		*active_width = IB_WIDTH_4X;
-		*active_speed = IB_SPEED_QDR;
-	} else if (speed <= 40000) {
-		*active_width = IB_WIDTH_4X;
-		*active_speed = IB_SPEED_FDR10;
-	} else {
-		*active_width = IB_WIDTH_4X;
-		*active_speed = IB_SPEED_EDR;
-	}
-}
-
 static int create_qp_validate_user_data(struct usnic_ib_create_qp_cmd cmd)
 {
 	if (cmd.spec.trans_type <= USNIC_TRANSPORT_UNKNOWN ||
@@ -326,12 +307,16 @@ int usnic_ib_query_port(struct ib_device *ibdev, u8 port,
 				struct ib_port_attr *props)
 {
 	struct usnic_ib_dev *us_ibdev = to_usdev(ibdev);
-	struct ethtool_link_ksettings cmd;
 
 	usnic_dbg("\n");
 
 	mutex_lock(&us_ibdev->usdev_lock);
-	__ethtool_get_link_ksettings(us_ibdev->netdev, &cmd);
+	if (ib_get_eth_speed(ibdev, port, &props->active_speed,
+			     &props->active_width)) {
+		mutex_unlock(&us_ibdev->usdev_lock);
+		return -EINVAL;
+	}
+
 	/* props being zeroed by the caller, avoid zeroing it here */
 
 	props->lid = 0;
@@ -355,8 +340,6 @@ int usnic_ib_query_port(struct ib_device *ibdev, u8 port,
 	props->pkey_tbl_len = 1;
 	props->bad_pkey_cntr = 0;
 	props->qkey_viol_cntr = 0;
-	eth_speed_to_ib_speed(cmd.base.speed, &props->active_speed,
-			      &props->active_width);
 	props->max_mtu = IB_MTU_4096;
 	props->active_mtu = iboe_get_mtu(us_ibdev->ufdev->mtu);
 	/* Userspace will adjust for hdrs */
@@ -424,6 +407,16 @@ int usnic_ib_query_gid(struct ib_device *ibdev, u8 port, int index,
 	return 0;
 }
 
+struct net_device *usnic_get_netdev(struct ib_device *device, u8 port_num)
+{
+	struct usnic_ib_dev *us_ibdev = to_usdev(device);
+
+	if (us_ibdev->netdev)
+		dev_hold(us_ibdev->netdev);
+
+	return us_ibdev->netdev;
+}
+
 int usnic_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
 				u16 *pkey)
 {
diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.h b/drivers/infiniband/hw/usnic/usnic_ib_verbs.h
index 172e43b6fa95..1fda94425116 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.h
+++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.h
@@ -48,6 +48,7 @@ int usnic_ib_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
 				struct ib_qp_init_attr *qp_init_attr);
 int usnic_ib_query_gid(struct ib_device *ibdev, u8 port, int index,
 				union ib_gid *gid);
+struct net_device *usnic_get_netdev(struct ib_device *device, u8 port_num);
 int usnic_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
 				u16 *pkey);
 struct ib_pd *usnic_ib_alloc_pd(struct ib_device *ibdev,
diff --git a/drivers/infiniband/hw/usnic/usnic_uiom.c b/drivers/infiniband/hw/usnic/usnic_uiom.c
index c49db7c33979..4381c0a9a873 100644
--- a/drivers/infiniband/hw/usnic/usnic_uiom.c
+++ b/drivers/infiniband/hw/usnic/usnic_uiom.c
@@ -227,7 +227,7 @@ static void __usnic_uiom_reg_release(struct usnic_uiom_pd *pd,
 	vpn_last = vpn_start + npages - 1;
 
 	spin_lock(&pd->lock);
-	usnic_uiom_remove_interval(&pd->rb_root, vpn_start,
+	usnic_uiom_remove_interval(&pd->root, vpn_start,
 					vpn_last, &rm_intervals);
 	usnic_uiom_unmap_sorted_intervals(&rm_intervals, pd);
 
@@ -379,7 +379,7 @@ struct usnic_uiom_reg *usnic_uiom_reg_get(struct usnic_uiom_pd *pd,
 	err = usnic_uiom_get_intervals_diff(vpn_start, vpn_last,
 						(writable) ? IOMMU_WRITE : 0,
 						IOMMU_WRITE,
-						&pd->rb_root,
+						&pd->root,
 						&sorted_diff_intervals);
 	if (err) {
 		usnic_err("Failed disjoint interval vpn [0x%lx,0x%lx] err %d\n",
@@ -395,7 +395,7 @@ struct usnic_uiom_reg *usnic_uiom_reg_get(struct usnic_uiom_pd *pd,
 
 	}
 
-	err = usnic_uiom_insert_interval(&pd->rb_root, vpn_start, vpn_last,
+	err = usnic_uiom_insert_interval(&pd->root, vpn_start, vpn_last,
 					(writable) ? IOMMU_WRITE : 0);
 	if (err) {
 		usnic_err("Failed insert interval vpn [0x%lx,0x%lx] err %d\n",
diff --git a/drivers/infiniband/hw/usnic/usnic_uiom.h b/drivers/infiniband/hw/usnic/usnic_uiom.h
index 45ca7c1613a7..431efe4143f4 100644
--- a/drivers/infiniband/hw/usnic/usnic_uiom.h
+++ b/drivers/infiniband/hw/usnic/usnic_uiom.h
@@ -55,7 +55,7 @@ struct usnic_uiom_dev {
 struct usnic_uiom_pd {
 	struct iommu_domain		*domain;
 	spinlock_t			lock;
-	struct rb_root			rb_root;
+	struct rb_root_cached		root;
 	struct list_head		devs;
 	int				dev_cnt;
 };
diff --git a/drivers/infiniband/hw/usnic/usnic_uiom_interval_tree.c b/drivers/infiniband/hw/usnic/usnic_uiom_interval_tree.c
index 42b4b4c4e452..d399523206c7 100644
--- a/drivers/infiniband/hw/usnic/usnic_uiom_interval_tree.c
+++ b/drivers/infiniband/hw/usnic/usnic_uiom_interval_tree.c
@@ -100,9 +100,9 @@ static int interval_cmp(void *priv, struct list_head *a, struct list_head *b)
 }
 
 static void
-find_intervals_intersection_sorted(struct rb_root *root, unsigned long start,
-					unsigned long last,
-					struct list_head *list)
+find_intervals_intersection_sorted(struct rb_root_cached *root,
+				   unsigned long start, unsigned long last,
+				   struct list_head *list)
 {
 	struct usnic_uiom_interval_node *node;
 
@@ -118,7 +118,7 @@ find_intervals_intersection_sorted(struct rb_root *root, unsigned long start,
 
 int usnic_uiom_get_intervals_diff(unsigned long start, unsigned long last,
 					int flags, int flag_mask,
-					struct rb_root *root,
+					struct rb_root_cached *root,
 					struct list_head *diff_set)
 {
 	struct usnic_uiom_interval_node *interval, *tmp;
@@ -175,7 +175,7 @@ void usnic_uiom_put_interval_set(struct list_head *intervals)
 		kfree(interval);
 }
 
-int usnic_uiom_insert_interval(struct rb_root *root, unsigned long start,
+int usnic_uiom_insert_interval(struct rb_root_cached *root, unsigned long start,
 				unsigned long last, int flags)
 {
 	struct usnic_uiom_interval_node *interval, *tmp;
@@ -246,8 +246,9 @@ err_out:
 	return err;
 }
 
-void usnic_uiom_remove_interval(struct rb_root *root, unsigned long start,
-				unsigned long last, struct list_head *removed)
+void usnic_uiom_remove_interval(struct rb_root_cached *root,
+				unsigned long start, unsigned long last,
+				struct list_head *removed)
 {
 	struct usnic_uiom_interval_node *interval;
 
diff --git a/drivers/infiniband/hw/usnic/usnic_uiom_interval_tree.h b/drivers/infiniband/hw/usnic/usnic_uiom_interval_tree.h
index c0b0b876ab90..1d7fc3226bca 100644
--- a/drivers/infiniband/hw/usnic/usnic_uiom_interval_tree.h
+++ b/drivers/infiniband/hw/usnic/usnic_uiom_interval_tree.h
@@ -48,12 +48,12 @@ struct usnic_uiom_interval_node {
 
 extern void
 usnic_uiom_interval_tree_insert(struct usnic_uiom_interval_node *node,
-					struct rb_root *root);
+					struct rb_root_cached *root);
 extern void
 usnic_uiom_interval_tree_remove(struct usnic_uiom_interval_node *node,
-					struct rb_root *root);
+					struct rb_root_cached *root);
 extern struct usnic_uiom_interval_node *
-usnic_uiom_interval_tree_iter_first(struct rb_root *root,
+usnic_uiom_interval_tree_iter_first(struct rb_root_cached *root,
 					unsigned long start,
 					unsigned long last);
 extern struct usnic_uiom_interval_node *
@@ -63,7 +63,7 @@ usnic_uiom_interval_tree_iter_next(struct usnic_uiom_interval_node *node,
  * Inserts {start...last} into {root}.  If there are overlaps,
  * nodes will be broken up and merged
  */
-int usnic_uiom_insert_interval(struct rb_root *root,
+int usnic_uiom_insert_interval(struct rb_root_cached *root,
 				unsigned long start, unsigned long last,
 				int flags);
 /*
@@ -71,7 +71,7 @@ int usnic_uiom_insert_interval(struct rb_root *root,
  * 'removed.' The caller is responsibile for freeing memory of nodes in
  * 'removed.'
  */
-void usnic_uiom_remove_interval(struct rb_root *root,
+void usnic_uiom_remove_interval(struct rb_root_cached *root,
 				unsigned long start, unsigned long last,
 				struct list_head *removed);
 /*
@@ -81,7 +81,7 @@ void usnic_uiom_remove_interval(struct rb_root *root,
 int usnic_uiom_get_intervals_diff(unsigned long start,
 					unsigned long last, int flags,
 					int flag_mask,
-					struct rb_root *root,
+					struct rb_root_cached *root,
 					struct list_head *diff_set);
 /* Call this to free diff_set returned by usnic_uiom_get_intervals_diff */
 void usnic_uiom_put_interval_set(struct list_head *intervals);
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
index 8e2f0a11690f..663a0c301c43 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
@@ -194,6 +194,7 @@ struct pvrdma_dev {
 	void *resp_slot;
 	unsigned long flags;
 	struct list_head device_link;
+	unsigned int dsr_version;
 
 	/* Locking and interrupt information. */
 	spinlock_t cmd_lock; /* Command lock. */
@@ -444,6 +445,7 @@ void pvrdma_ah_attr_to_rdma(struct rdma_ah_attr *dst,
 			    const struct pvrdma_ah_attr *src);
 void rdma_ah_attr_to_pvrdma(struct pvrdma_ah_attr *dst,
 			    const struct rdma_ah_attr *src);
+u8 ib_gid_type_to_pvrdma(enum ib_gid_type gid_type);
 
 int pvrdma_uar_table_init(struct pvrdma_dev *dev);
 void pvrdma_uar_table_cleanup(struct pvrdma_dev *dev);
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c
index 69bda611d313..3562c0c30492 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c
@@ -65,13 +65,28 @@ int pvrdma_req_notify_cq(struct ib_cq *ibcq,
 	struct pvrdma_dev *dev = to_vdev(ibcq->device);
 	struct pvrdma_cq *cq = to_vcq(ibcq);
 	u32 val = cq->cq_handle;
+	unsigned long flags;
+	int has_data = 0;
 
 	val |= (notify_flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
 		PVRDMA_UAR_CQ_ARM_SOL : PVRDMA_UAR_CQ_ARM;
 
+	spin_lock_irqsave(&cq->cq_lock, flags);
+
 	pvrdma_write_uar_cq(dev, val);
 
-	return 0;
+	if (notify_flags & IB_CQ_REPORT_MISSED_EVENTS) {
+		unsigned int head;
+
+		has_data = pvrdma_idx_ring_has_data(&cq->ring_state->rx,
+						    cq->ibcq.cqe, &head);
+		if (unlikely(has_data == PVRDMA_INVALID_IDX))
+			dev_err(&dev->pdev->dev, "CQ ring state invalid\n");
+	}
+
+	spin_unlock_irqrestore(&cq->cq_lock, flags);
+
+	return has_data;
 }
 
 /**
@@ -284,7 +299,7 @@ static inline struct pvrdma_cqe *get_cqe(struct pvrdma_cq *cq, int i)
 
 void _pvrdma_flush_cqe(struct pvrdma_qp *qp, struct pvrdma_cq *cq)
 {
-	int head;
+	unsigned int head;
 	int has_data;
 
 	if (!cq->is_kernel)
@@ -374,6 +389,7 @@ retry:
 	wc->dlid_path_bits = cqe->dlid_path_bits;
 	wc->port_num = cqe->port_num;
 	wc->vendor_err = cqe->vendor_err;
+	wc->network_hdr_type = cqe->network_hdr_type;
 
 	/* Update shared ring state */
 	pvrdma_idx_ring_inc(&cq->ring_state->rx.cons_head, cq->ibcq.cqe);
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
index 09078ccfaec7..df0a6b525021 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
@@ -50,7 +50,15 @@
 
 #include "pvrdma_verbs.h"
 
-#define PVRDMA_VERSION			17
+/*
+ * PVRDMA version macros. Some new features require updates to PVRDMA_VERSION.
+ * These macros allow us to check for different features if necessary.
+ */
+
+#define PVRDMA_ROCEV1_VERSION		17
+#define PVRDMA_ROCEV2_VERSION		18
+#define PVRDMA_VERSION			PVRDMA_ROCEV2_VERSION
+
 #define PVRDMA_BOARD_ID			1
 #define PVRDMA_REV_ID			1
 
@@ -123,6 +131,31 @@
 #define PVRDMA_GID_TYPE_FLAG_ROCE_V1	BIT(0)
 #define PVRDMA_GID_TYPE_FLAG_ROCE_V2	BIT(1)
 
+/*
+ * Version checks. This checks whether each version supports specific
+ * capabilities from the device.
+ */
+
+#define PVRDMA_IS_VERSION17(_dev)					\
+	(_dev->dsr_version == PVRDMA_ROCEV1_VERSION &&			\
+	 _dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V1)
+
+#define PVRDMA_IS_VERSION18(_dev)					\
+	(_dev->dsr_version >= PVRDMA_ROCEV2_VERSION &&			\
+	 (_dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V1 ||  \
+	  _dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V2))	\
+
+#define PVRDMA_SUPPORTED(_dev)						\
+	((_dev->dsr->caps.mode == PVRDMA_DEVICE_MODE_ROCE) &&		\
+	 (PVRDMA_IS_VERSION17(_dev) || PVRDMA_IS_VERSION18(_dev)))
+
+/*
+ * Get capability values based on device version.
+ */
+
+#define PVRDMA_GET_CAP(_dev, _old_val, _val) \
+	((PVRDMA_IS_VERSION18(_dev)) ? _val : _old_val)
+
 enum pvrdma_pci_resource {
 	PVRDMA_PCI_RESOURCE_MSIX,	/* BAR0: MSI-X, MMIO. */
 	PVRDMA_PCI_RESOURCE_REG,	/* BAR1: Registers, MMIO. */
@@ -225,7 +258,7 @@ struct pvrdma_device_caps {
 	u8  atomic_ops;				/* PVRDMA_ATOMIC_OP_* bits */
 	u8  bmme_flags;				/* FRWR Mem Mgmt Extensions */
 	u8  gid_types;				/* PVRDMA_GID_TYPE_FLAG_ */
-	u8  reserved[4];
+	u32 max_fast_reg_page_list_len;
 };
 
 struct pvrdma_ring_page_info {
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
index 34ebc7615411..6ce709a67959 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
@@ -102,12 +102,11 @@ static struct device_attribute *pvrdma_class_attributes[] = {
 	&dev_attr_board_id
 };
 
-static void pvrdma_get_fw_ver_str(struct ib_device *device, char *str,
-				  size_t str_len)
+static void pvrdma_get_fw_ver_str(struct ib_device *device, char *str)
 {
 	struct pvrdma_dev *dev =
 		container_of(device, struct pvrdma_dev, ib_dev);
-	snprintf(str, str_len, "%d.%d.%d\n",
+	snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%d\n",
 		 (int) (dev->dsr->caps.fw_ver >> 32),
 		 (int) (dev->dsr->caps.fw_ver >> 16) & 0xffff,
 		 (int) dev->dsr->caps.fw_ver & 0xffff);
@@ -129,10 +128,14 @@ static int pvrdma_init_device(struct pvrdma_dev *dev)
 static int pvrdma_port_immutable(struct ib_device *ibdev, u8 port_num,
 				 struct ib_port_immutable *immutable)
 {
+	struct pvrdma_dev *dev = to_vdev(ibdev);
 	struct ib_port_attr attr;
 	int err;
 
-	immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
+	if (dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V1)
+		immutable->core_cap_flags |= RDMA_CORE_PORT_IBA_ROCE;
+	else if (dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V2)
+		immutable->core_cap_flags |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
 
 	err = ib_query_port(ibdev, port_num, &attr);
 	if (err)
@@ -570,6 +573,7 @@ static void pvrdma_free_slots(struct pvrdma_dev *dev)
 
 static int pvrdma_add_gid_at_index(struct pvrdma_dev *dev,
 				   const union ib_gid *gid,
+				   u8 gid_type,
 				   int index)
 {
 	int ret;
@@ -587,7 +591,7 @@ static int pvrdma_add_gid_at_index(struct pvrdma_dev *dev,
 	cmd_bind->mtu = ib_mtu_enum_to_int(IB_MTU_1024);
 	cmd_bind->vlan = 0xfff;
 	cmd_bind->index = index;
-	cmd_bind->gid_type = PVRDMA_GID_TYPE_FLAG_ROCE_V1;
+	cmd_bind->gid_type = gid_type;
 
 	ret = pvrdma_cmd_post(dev, &req, NULL, 0);
 	if (ret < 0) {
@@ -608,7 +612,9 @@ static int pvrdma_add_gid(struct ib_device *ibdev,
 {
 	struct pvrdma_dev *dev = to_vdev(ibdev);
 
-	return pvrdma_add_gid_at_index(dev, gid, index);
+	return pvrdma_add_gid_at_index(dev, gid,
+				       ib_gid_type_to_pvrdma(attr->gid_type),
+				       index);
 }
 
 static int pvrdma_del_gid_at_index(struct pvrdma_dev *dev, int index)
@@ -723,7 +729,6 @@ static int pvrdma_pci_probe(struct pci_dev *pdev,
 	int ret;
 	unsigned long start;
 	unsigned long len;
-	unsigned int version;
 	dma_addr_t slot_dma = 0;
 
 	dev_dbg(&pdev->dev, "initializing driver %s\n", pci_name(pdev));
@@ -820,13 +825,9 @@ static int pvrdma_pci_probe(struct pci_dev *pdev,
 		goto err_unmap_regs;
 	}
 
-	version = pvrdma_read_reg(dev, PVRDMA_REG_VERSION);
+	dev->dsr_version = pvrdma_read_reg(dev, PVRDMA_REG_VERSION);
 	dev_info(&pdev->dev, "device version %d, driver version %d\n",
-		 version, PVRDMA_VERSION);
-	if (version < PVRDMA_VERSION) {
-		dev_err(&pdev->dev, "incompatible device version\n");
-		goto err_uar_unmap;
-	}
+		 dev->dsr_version, PVRDMA_VERSION);
 
 	dev->dsr = dma_alloc_coherent(&pdev->dev, sizeof(*dev->dsr),
 				      &dev->dsrbase, GFP_KERNEL);
@@ -897,17 +898,9 @@ static int pvrdma_pci_probe(struct pci_dev *pdev,
 	/* Make sure the write is complete before reading status. */
 	mb();
 
-	/* Currently, the driver only supports RoCE mode. */
-	if (dev->dsr->caps.mode != PVRDMA_DEVICE_MODE_ROCE) {
-		dev_err(&pdev->dev, "unsupported transport %d\n",
-			dev->dsr->caps.mode);
-		ret = -EFAULT;
-		goto err_free_cq_ring;
-	}
-
-	/* Currently, the driver only supports RoCE V1. */
-	if (!(dev->dsr->caps.gid_types & PVRDMA_GID_TYPE_FLAG_ROCE_V1)) {
-		dev_err(&pdev->dev, "driver needs RoCE v1 support\n");
+	/* The driver supports RoCE V1 and V2. */
+	if (!PVRDMA_SUPPORTED(dev)) {
+		dev_err(&pdev->dev, "driver needs RoCE v1 or v2 support\n");
 		ret = -EFAULT;
 		goto err_free_cq_ring;
 	}
@@ -1078,7 +1071,7 @@ static void pvrdma_pci_remove(struct pci_dev *pdev)
 	pci_set_drvdata(pdev, NULL);
 }
 
-static struct pci_device_id pvrdma_pci_table[] = {
+static const struct pci_device_id pvrdma_pci_table[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_VMWARE, PCI_DEVICE_ID_VMWARE_PVRDMA), },
 	{ 0 },
 };
@@ -1119,5 +1112,4 @@ module_exit(pvrdma_cleanup);
 
 MODULE_AUTHOR("VMware, Inc");
 MODULE_DESCRIPTION("VMware Paravirtual RDMA driver");
-MODULE_VERSION(DRV_VERSION);
 MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c
index ec6a4ca1eeb7..fb0c5c0976b3 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c
@@ -303,3 +303,10 @@ void rdma_ah_attr_to_pvrdma(struct pvrdma_ah_attr *dst,
 	dst->port_num = rdma_ah_get_port_num(src);
 	memcpy(&dst->dmac, src->roce.dmac, sizeof(dst->dmac));
 }
+
+u8 ib_gid_type_to_pvrdma(enum ib_gid_type gid_type)
+{
+	return (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ?
+		PVRDMA_GID_TYPE_FLAG_ROCE_V2 :
+		PVRDMA_GID_TYPE_FLAG_ROCE_V1;
+}
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_ring.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_ring.h
index ed9022a91a1d..8b558ae234c8 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_ring.h
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_ring.h
@@ -111,21 +111,4 @@ static inline __s32 pvrdma_idx_ring_has_data(const struct pvrdma_ring *r,
 	return PVRDMA_INVALID_IDX;
 }
 
-static inline bool pvrdma_idx_ring_is_valid_idx(const struct pvrdma_ring *r,
-						__u32 max_elems, __u32 *idx)
-{
-	const __u32 tail = atomic_read(&r->prod_tail);
-	const __u32 head = atomic_read(&r->cons_head);
-
-	if (pvrdma_idx_valid(tail, max_elems) &&
-	    pvrdma_idx_valid(head, max_elems) &&
-	    pvrdma_idx_valid(*idx, max_elems)) {
-		if (tail > head && (*idx < tail && *idx >= head))
-			return true;
-		else if (head > tail && (*idx >= head || *idx < tail))
-			return true;
-	}
-	return false;
-}
-
 #endif /* __PVRDMA_RING_H__ */
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
index 28517042011d..48776f5ffb0e 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
@@ -83,6 +83,8 @@ int pvrdma_query_device(struct ib_device *ibdev,
 	props->max_qp_wr = dev->dsr->caps.max_qp_wr;
 	props->device_cap_flags = dev->dsr->caps.device_cap_flags;
 	props->max_sge = dev->dsr->caps.max_sge;
+	props->max_sge_rd = PVRDMA_GET_CAP(dev, dev->dsr->caps.max_sge,
+					   dev->dsr->caps.max_sge_rd);
 	props->max_cq = dev->dsr->caps.max_cq;
 	props->max_cqe = dev->dsr->caps.max_cqe;
 	props->max_mr = dev->dsr->caps.max_mr;
@@ -101,8 +103,14 @@ int pvrdma_query_device(struct ib_device *ibdev,
 	    (dev->dsr->caps.bmme_flags & PVRDMA_BMME_FLAG_REMOTE_INV) &&
 	    (dev->dsr->caps.bmme_flags & PVRDMA_BMME_FLAG_FAST_REG_WR)) {
 		props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
+		props->max_fast_reg_page_list_len = PVRDMA_GET_CAP(dev,
+				PVRDMA_MAX_FAST_REG_PAGES,
+				dev->dsr->caps.max_fast_reg_page_list_len);
 	}
 
+	props->device_cap_flags |= IB_DEVICE_PORT_ACTIVE_EVENT |
+				   IB_DEVICE_RC_RNR_NAK_GEN;
+
 	return 0;
 }
 
@@ -143,6 +151,7 @@ int pvrdma_query_port(struct ib_device *ibdev, u8 port,
 	props->gid_tbl_len = resp->attrs.gid_tbl_len;
 	props->port_cap_flags =
 		pvrdma_port_cap_flags_to_ib(resp->attrs.port_cap_flags);
+	props->port_cap_flags |= IB_PORT_CM_SUP | IB_PORT_IP_BASED_GIDS;
 	props->max_msg_sz = resp->attrs.max_msg_sz;
 	props->bad_pkey_cntr = resp->attrs.bad_pkey_cntr;
 	props->qkey_viol_cntr = resp->attrs.qkey_viol_cntr;
diff --git a/drivers/infiniband/sw/rdmavt/ah.c b/drivers/infiniband/sw/rdmavt/ah.c
index a96d4aa80ae8..ba3639a0d77c 100644
--- a/drivers/infiniband/sw/rdmavt/ah.c
+++ b/drivers/infiniband/sw/rdmavt/ah.c
@@ -66,8 +66,6 @@ int rvt_check_ah(struct ib_device *ibdev,
 	int port_num = rdma_ah_get_port_num(ah_attr);
 	struct ib_port_attr port_attr;
 	struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
-	enum rdma_link_layer link = rdma_port_get_link_layer(ibdev, port_num);
-	u32 dlid = rdma_ah_get_dlid(ah_attr);
 	u8 ah_flags = rdma_ah_get_ah_flags(ah_attr);
 	u8 static_rate = rdma_ah_get_static_rate(ah_attr);
 
@@ -83,14 +81,6 @@ int rvt_check_ah(struct ib_device *ibdev,
 	if ((ah_flags & IB_AH_GRH) &&
 	    rdma_ah_read_grh(ah_attr)->sgid_index >= port_attr.gid_tbl_len)
 		return -EINVAL;
-	if (link != IB_LINK_LAYER_ETHERNET) {
-		if (dlid == 0)
-			return -EINVAL;
-		if (dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE) &&
-		    dlid != be16_to_cpu(IB_LID_PERMISSIVE) &&
-		    !(ah_flags & IB_AH_GRH))
-			return -EINVAL;
-	}
 	if (rdi->driver_f.check_ah)
 		return rdi->driver_f.check_ah(ibdev, ah_attr);
 	return 0;
diff --git a/drivers/infiniband/sw/rdmavt/cq.c b/drivers/infiniband/sw/rdmavt/cq.c
index 0ae2ff8cf81e..97d71e49c092 100644
--- a/drivers/infiniband/sw/rdmavt/cq.c
+++ b/drivers/infiniband/sw/rdmavt/cq.c
@@ -107,7 +107,7 @@ void rvt_cq_enter(struct rvt_cq *cq, struct ib_wc *entry, bool solicited)
 		wc->uqueue[head].src_qp = entry->src_qp;
 		wc->uqueue[head].wc_flags = entry->wc_flags;
 		wc->uqueue[head].pkey_index = entry->pkey_index;
-		wc->uqueue[head].slid = entry->slid;
+		wc->uqueue[head].slid = ib_lid_cpu16(entry->slid);
 		wc->uqueue[head].sl = entry->sl;
 		wc->uqueue[head].dlid_path_bits = entry->dlid_path_bits;
 		wc->uqueue[head].port_num = entry->port_num;
diff --git a/drivers/infiniband/sw/rdmavt/mr.c b/drivers/infiniband/sw/rdmavt/mr.c
index aa5f9ea318e4..42713511b53b 100644
--- a/drivers/infiniband/sw/rdmavt/mr.c
+++ b/drivers/infiniband/sw/rdmavt/mr.c
@@ -441,6 +441,105 @@ bail_umem:
 }
 
 /**
+ * rvt_dereg_clean_qp_cb - callback from iterator
+ * @qp - the qp
+ * @v - the mregion (as u64)
+ *
+ * This routine fields the callback for all QPs and
+ * for QPs in the same PD as the MR will call the
+ * rvt_qp_mr_clean() to potentially cleanup references.
+ */
+static void rvt_dereg_clean_qp_cb(struct rvt_qp *qp, u64 v)
+{
+	struct rvt_mregion *mr = (struct rvt_mregion *)v;
+
+	/* skip PDs that are not ours */
+	if (mr->pd != qp->ibqp.pd)
+		return;
+	rvt_qp_mr_clean(qp, mr->lkey);
+}
+
+/**
+ * rvt_dereg_clean_qps - find QPs for reference cleanup
+ * @mr - the MR that is being deregistered
+ *
+ * This routine iterates RC QPs looking for references
+ * to the lkey noted in mr.
+ */
+static void rvt_dereg_clean_qps(struct rvt_mregion *mr)
+{
+	struct rvt_dev_info *rdi = ib_to_rvt(mr->pd->device);
+
+	rvt_qp_iter(rdi, (u64)mr, rvt_dereg_clean_qp_cb);
+}
+
+/**
+ * rvt_check_refs - check references
+ * @mr - the megion
+ * @t - the caller identification
+ *
+ * This routine checks MRs holding a reference during
+ * when being de-registered.
+ *
+ * If the count is non-zero, the code calls a clean routine then
+ * waits for the timeout for the count to zero.
+ */
+static int rvt_check_refs(struct rvt_mregion *mr, const char *t)
+{
+	unsigned long timeout;
+	struct rvt_dev_info *rdi = ib_to_rvt(mr->pd->device);
+
+	if (percpu_ref_is_zero(&mr->refcount))
+		return 0;
+	/* avoid dma mr */
+	if (mr->lkey)
+		rvt_dereg_clean_qps(mr);
+	timeout = wait_for_completion_timeout(&mr->comp, 5 * HZ);
+	if (!timeout) {
+		rvt_pr_err(rdi,
+			   "%s timeout mr %p pd %p lkey %x refcount %ld\n",
+			   t, mr, mr->pd, mr->lkey,
+			   atomic_long_read(&mr->refcount.count));
+		rvt_get_mr(mr);
+		return -EBUSY;
+	}
+	return 0;
+}
+
+/**
+ * rvt_mr_has_lkey - is MR
+ * @mr - the mregion
+ * @lkey - the lkey
+ */
+bool rvt_mr_has_lkey(struct rvt_mregion *mr, u32 lkey)
+{
+	return mr && lkey == mr->lkey;
+}
+
+/**
+ * rvt_ss_has_lkey - is mr in sge tests
+ * @ss - the sge state
+ * @lkey
+ *
+ * This code tests for an MR in the indicated
+ * sge state.
+ */
+bool rvt_ss_has_lkey(struct rvt_sge_state *ss, u32 lkey)
+{
+	int i;
+	bool rval = false;
+
+	if (!ss->num_sge)
+		return rval;
+	/* first one */
+	rval = rvt_mr_has_lkey(ss->sge.mr, lkey);
+	/* any others */
+	for (i = 0; !rval && i < ss->num_sge - 1; i++)
+		rval = rvt_mr_has_lkey(ss->sg_list[i].mr, lkey);
+	return rval;
+}
+
+/**
  * rvt_dereg_mr - unregister and free a memory region
  * @ibmr: the memory region to free
  *
@@ -453,22 +552,14 @@ bail_umem:
 int rvt_dereg_mr(struct ib_mr *ibmr)
 {
 	struct rvt_mr *mr = to_imr(ibmr);
-	struct rvt_dev_info *rdi = ib_to_rvt(ibmr->pd->device);
-	int ret = 0;
-	unsigned long timeout;
+	int ret;
 
 	rvt_free_lkey(&mr->mr);
 
 	rvt_put_mr(&mr->mr); /* will set completion if last */
-	timeout = wait_for_completion_timeout(&mr->mr.comp, 5 * HZ);
-	if (!timeout) {
-		rvt_pr_err(rdi,
-			   "rvt_dereg_mr timeout mr %p pd %p\n",
-			   mr, mr->mr.pd);
-		rvt_get_mr(&mr->mr);
-		ret = -EBUSY;
+	ret = rvt_check_refs(&mr->mr, __func__);
+	if (ret)
 		goto out;
-	}
 	rvt_deinit_mregion(&mr->mr);
 	if (mr->umem)
 		ib_umem_release(mr->umem);
@@ -761,16 +852,12 @@ int rvt_dealloc_fmr(struct ib_fmr *ibfmr)
 {
 	struct rvt_fmr *fmr = to_ifmr(ibfmr);
 	int ret = 0;
-	unsigned long timeout;
 
 	rvt_free_lkey(&fmr->mr);
 	rvt_put_mr(&fmr->mr); /* will set completion if last */
-	timeout = wait_for_completion_timeout(&fmr->mr.comp, 5 * HZ);
-	if (!timeout) {
-		rvt_get_mr(&fmr->mr);
-		ret = -EBUSY;
+	ret = rvt_check_refs(&fmr->mr, __func__);
+	if (ret)
 		goto out;
-	}
 	rvt_deinit_mregion(&fmr->mr);
 	kfree(fmr);
 out:
@@ -778,23 +865,52 @@ out:
 }
 
 /**
+ * rvt_sge_adjacent - is isge compressible
+ * @last_sge: last outgoing SGE written
+ * @sge: SGE to check
+ *
+ * If adjacent will update last_sge to add length.
+ *
+ * Return: true if isge is adjacent to last sge
+ */
+static inline bool rvt_sge_adjacent(struct rvt_sge *last_sge,
+				    struct ib_sge *sge)
+{
+	if (last_sge && sge->lkey == last_sge->mr->lkey &&
+	    ((uint64_t)(last_sge->vaddr + last_sge->length) == sge->addr)) {
+		if (sge->lkey) {
+			if (unlikely((sge->addr - last_sge->mr->user_base +
+			      sge->length > last_sge->mr->length)))
+				return false; /* overrun, caller will catch */
+		} else {
+			last_sge->length += sge->length;
+		}
+		last_sge->sge_length += sge->length;
+		trace_rvt_sge_adjacent(last_sge, sge);
+		return true;
+	}
+	return false;
+}
+
+/**
  * rvt_lkey_ok - check IB SGE for validity and initialize
  * @rkt: table containing lkey to check SGE against
  * @pd: protection domain
  * @isge: outgoing internal SGE
+ * @last_sge: last outgoing SGE written
  * @sge: SGE to check
  * @acc: access flags
  *
  * Check the IB SGE for validity and initialize our internal version
  * of it.
  *
- * Return: 1 if valid and successful, otherwise returns 0.
- *
- * increments the reference count upon success
+ * Increments the reference count when a new sge is stored.
  *
+ * Return: 0 if compressed, 1 if added , otherwise returns -errno.
  */
 int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd,
-		struct rvt_sge *isge, struct ib_sge *sge, int acc)
+		struct rvt_sge *isge, struct rvt_sge *last_sge,
+		struct ib_sge *sge, int acc)
 {
 	struct rvt_mregion *mr;
 	unsigned n, m;
@@ -804,12 +920,14 @@ int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd,
 	 * We use LKEY == zero for kernel virtual addresses
 	 * (see rvt_get_dma_mr() and dma_virt_ops).
 	 */
-	rcu_read_lock();
 	if (sge->lkey == 0) {
 		struct rvt_dev_info *dev = ib_to_rvt(pd->ibpd.device);
 
 		if (pd->user)
-			goto bail;
+			return -EINVAL;
+		if (rvt_sge_adjacent(last_sge, sge))
+			return 0;
+		rcu_read_lock();
 		mr = rcu_dereference(dev->dma_mr);
 		if (!mr)
 			goto bail;
@@ -824,6 +942,9 @@ int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd,
 		isge->n = 0;
 		goto ok;
 	}
+	if (rvt_sge_adjacent(last_sge, sge))
+		return 0;
+	rcu_read_lock();
 	mr = rcu_dereference(rkt->table[sge->lkey >> rkt->shift]);
 	if (!mr)
 		goto bail;
@@ -874,12 +995,13 @@ int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd,
 	isge->m = m;
 	isge->n = n;
 ok:
+	trace_rvt_sge_new(isge, sge);
 	return 1;
 bail_unref:
 	rvt_put_mr(mr);
 bail:
 	rcu_read_unlock();
-	return 0;
+	return -EINVAL;
 }
 EXPORT_SYMBOL(rvt_lkey_ok);
 
diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c
index 8876ee7bc326..22df09ae809e 100644
--- a/drivers/infiniband/sw/rdmavt/qp.c
+++ b/drivers/infiniband/sw/rdmavt/qp.c
@@ -52,6 +52,7 @@
 #include <linux/slab.h>
 #include <rdma/ib_verbs.h>
 #include <rdma/ib_hdrs.h>
+#include <rdma/opa_addr.h>
 #include "qp.h"
 #include "vt.h"
 #include "trace.h"
@@ -421,15 +422,6 @@ bail:
 	return ret;
 }
 
-static void free_qpn(struct rvt_qpn_table *qpt, u32 qpn)
-{
-	struct rvt_qpn_map *map;
-
-	map = qpt->map + qpn / RVT_BITS_PER_PAGE;
-	if (map->page)
-		clear_bit(qpn & RVT_BITS_PER_PAGE_MASK, map->page);
-}
-
 /**
  * rvt_clear_mr_refs - Drop help mr refs
  * @qp: rvt qp data structure
@@ -448,13 +440,9 @@ static void rvt_clear_mr_refs(struct rvt_qp *qp, int clr_sends)
 	if (clr_sends) {
 		while (qp->s_last != qp->s_head) {
 			struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_last);
-			unsigned i;
 
-			for (i = 0; i < wqe->wr.num_sge; i++) {
-				struct rvt_sge *sge = &wqe->sg_list[i];
+			rvt_put_swqe(wqe);
 
-				rvt_put_mr(sge->mr);
-			}
 			if (qp->ibqp.qp_type == IB_QPT_UD ||
 			    qp->ibqp.qp_type == IB_QPT_SMI ||
 			    qp->ibqp.qp_type == IB_QPT_GSI)
@@ -470,10 +458,7 @@ static void rvt_clear_mr_refs(struct rvt_qp *qp, int clr_sends)
 		}
 	}
 
-	if (qp->ibqp.qp_type != IB_QPT_RC)
-		return;
-
-	for (n = 0; n < rvt_max_atomic(rdi); n++) {
+	for (n = 0; qp->s_ack_queue && n < rvt_max_atomic(rdi); n++) {
 		struct rvt_ack_entry *e = &qp->s_ack_queue[n];
 
 		if (e->rdma_sge.mr) {
@@ -484,6 +469,113 @@ static void rvt_clear_mr_refs(struct rvt_qp *qp, int clr_sends)
 }
 
 /**
+ * rvt_swqe_has_lkey - return true if lkey is used by swqe
+ * @wqe - the send wqe
+ * @lkey - the lkey
+ *
+ * Test the swqe for using lkey
+ */
+static bool rvt_swqe_has_lkey(struct rvt_swqe *wqe, u32 lkey)
+{
+	int i;
+
+	for (i = 0; i < wqe->wr.num_sge; i++) {
+		struct rvt_sge *sge = &wqe->sg_list[i];
+
+		if (rvt_mr_has_lkey(sge->mr, lkey))
+			return true;
+	}
+	return false;
+}
+
+/**
+ * rvt_qp_sends_has_lkey - return true is qp sends use lkey
+ * @qp - the rvt_qp
+ * @lkey - the lkey
+ */
+static bool rvt_qp_sends_has_lkey(struct rvt_qp *qp, u32 lkey)
+{
+	u32 s_last = qp->s_last;
+
+	while (s_last != qp->s_head) {
+		struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, s_last);
+
+		if (rvt_swqe_has_lkey(wqe, lkey))
+			return true;
+
+		if (++s_last >= qp->s_size)
+			s_last = 0;
+	}
+	if (qp->s_rdma_mr)
+		if (rvt_mr_has_lkey(qp->s_rdma_mr, lkey))
+			return true;
+	return false;
+}
+
+/**
+ * rvt_qp_acks_has_lkey - return true if acks have lkey
+ * @qp - the qp
+ * @lkey - the lkey
+ */
+static bool rvt_qp_acks_has_lkey(struct rvt_qp *qp, u32 lkey)
+{
+	int i;
+	struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
+
+	for (i = 0; qp->s_ack_queue && i < rvt_max_atomic(rdi); i++) {
+		struct rvt_ack_entry *e = &qp->s_ack_queue[i];
+
+		if (rvt_mr_has_lkey(e->rdma_sge.mr, lkey))
+			return true;
+	}
+	return false;
+}
+
+/*
+ * rvt_qp_mr_clean - clean up remote ops for lkey
+ * @qp - the qp
+ * @lkey - the lkey that is being de-registered
+ *
+ * This routine checks if the lkey is being used by
+ * the qp.
+ *
+ * If so, the qp is put into an error state to elminate
+ * any references from the qp.
+ */
+void rvt_qp_mr_clean(struct rvt_qp *qp, u32 lkey)
+{
+	bool lastwqe = false;
+
+	if (qp->ibqp.qp_type == IB_QPT_SMI ||
+	    qp->ibqp.qp_type == IB_QPT_GSI)
+		/* avoid special QPs */
+		return;
+	spin_lock_irq(&qp->r_lock);
+	spin_lock(&qp->s_hlock);
+	spin_lock(&qp->s_lock);
+
+	if (qp->state == IB_QPS_ERR || qp->state == IB_QPS_RESET)
+		goto check_lwqe;
+
+	if (rvt_ss_has_lkey(&qp->r_sge, lkey) ||
+	    rvt_qp_sends_has_lkey(qp, lkey) ||
+	    rvt_qp_acks_has_lkey(qp, lkey))
+		lastwqe = rvt_error_qp(qp, IB_WC_LOC_PROT_ERR);
+check_lwqe:
+	spin_unlock(&qp->s_lock);
+	spin_unlock(&qp->s_hlock);
+	spin_unlock_irq(&qp->r_lock);
+	if (lastwqe) {
+		struct ib_event ev;
+
+		ev.device = qp->ibqp.device;
+		ev.element.qp = &qp->ibqp;
+		ev.event = IB_EVENT_QP_LAST_WQE_REACHED;
+		qp->ibqp.event_handler(&ev, qp->ibqp.qp_context);
+	}
+}
+
+/**
  * rvt_remove_qp - remove qp form table
  * @rdi: rvt dev struct
  * @qp: qp to remove
@@ -645,6 +737,19 @@ static void rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp,
 	lockdep_assert_held(&qp->s_lock);
 }
 
+/** rvt_free_qpn - Free a qpn from the bit map
+ * @qpt: QP table
+ * @qpn: queue pair number to free
+ */
+static void rvt_free_qpn(struct rvt_qpn_table *qpt, u32 qpn)
+{
+	struct rvt_qpn_map *map;
+
+	map = qpt->map + (qpn & RVT_QPN_MASK) / RVT_BITS_PER_PAGE;
+	if (map->page)
+		clear_bit(qpn & RVT_BITS_PER_PAGE_MASK, map->page);
+}
+
 /**
  * rvt_create_qp - create a queue pair for a device
  * @ibpd: the protection domain who's device we create the queue pair for
@@ -914,7 +1019,7 @@ bail_ip:
 		kref_put(&qp->ip->ref, rvt_release_mmap_info);
 
 bail_qpn:
-	free_qpn(&rdi->qp_dev->qpn_table, qp->ibqp.qp_num);
+	rvt_free_qpn(&rdi->qp_dev->qpn_table, qp->ibqp.qp_num);
 
 bail_rq_wq:
 	if (!qp->ip)
@@ -1062,6 +1167,7 @@ int rvt_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 	int mig = 0;
 	int pmtu = 0; /* for gcc warning only */
 	enum rdma_link_layer link;
+	int opa_ah;
 
 	link = rdma_port_get_link_layer(ibqp->device, qp->port_num);
 
@@ -1072,6 +1178,7 @@ int rvt_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 	cur_state = attr_mask & IB_QP_CUR_STATE ?
 		attr->cur_qp_state : qp->state;
 	new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
+	opa_ah = rdma_cap_opa_ah(ibqp->device, qp->port_num);
 
 	if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type,
 				attr_mask, link))
@@ -1082,17 +1189,31 @@ int rvt_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 		goto inval;
 
 	if (attr_mask & IB_QP_AV) {
-		if (rdma_ah_get_dlid(&attr->ah_attr) >=
-		    be16_to_cpu(IB_MULTICAST_LID_BASE))
-			goto inval;
+		if (opa_ah) {
+			if (rdma_ah_get_dlid(&attr->ah_attr) >=
+				opa_get_mcast_base(OPA_MCAST_NR))
+				goto inval;
+		} else {
+			if (rdma_ah_get_dlid(&attr->ah_attr) >=
+				be16_to_cpu(IB_MULTICAST_LID_BASE))
+				goto inval;
+		}
+
 		if (rvt_check_ah(qp->ibqp.device, &attr->ah_attr))
 			goto inval;
 	}
 
 	if (attr_mask & IB_QP_ALT_PATH) {
-		if (rdma_ah_get_dlid(&attr->alt_ah_attr) >=
-		    be16_to_cpu(IB_MULTICAST_LID_BASE))
-			goto inval;
+		if (opa_ah) {
+			if (rdma_ah_get_dlid(&attr->alt_ah_attr) >=
+				opa_get_mcast_base(OPA_MCAST_NR))
+				goto inval;
+		} else {
+			if (rdma_ah_get_dlid(&attr->alt_ah_attr) >=
+				be16_to_cpu(IB_MULTICAST_LID_BASE))
+				goto inval;
+		}
+
 		if (rvt_check_ah(qp->ibqp.device, &attr->alt_ah_attr))
 			goto inval;
 		if (attr->alt_pkey_index >= rvt_get_npkeys(rdi))
@@ -1239,7 +1360,6 @@ int rvt_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 
 	if (attr_mask & IB_QP_PATH_MTU) {
 		qp->pmtu = rdi->driver_f.mtu_from_qp(rdi, qp, pmtu);
-		qp->path_mtu = rdi->driver_f.mtu_to_path_mtu(qp->pmtu);
 		qp->log_pmtu = ilog2(qp->pmtu);
 	}
 
@@ -1301,19 +1421,6 @@ inval:
 	return -EINVAL;
 }
 
-/** rvt_free_qpn - Free a qpn from the bit map
- * @qpt: QP table
- * @qpn: queue pair number to free
- */
-static void rvt_free_qpn(struct rvt_qpn_table *qpt, u32 qpn)
-{
-	struct rvt_qpn_map *map;
-
-	map = qpt->map + qpn / RVT_BITS_PER_PAGE;
-	if (map->page)
-		clear_bit(qpn & RVT_BITS_PER_PAGE_MASK, map->page);
-}
-
 /**
  * rvt_destroy_qp - destroy a queue pair
  * @ibqp: the queue pair to destroy
@@ -1375,7 +1482,7 @@ int rvt_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 
 	attr->qp_state = qp->state;
 	attr->cur_qp_state = attr->qp_state;
-	attr->path_mtu = qp->path_mtu;
+	attr->path_mtu = rdi->driver_f.mtu_to_path_mtu(qp->pmtu);
 	attr->path_mig_state = qp->s_mig_state;
 	attr->qkey = qp->qkey;
 	attr->rq_psn = qp->r_psn & rdi->dparms.psn_mask;
@@ -1695,22 +1802,23 @@ static int rvt_post_one_wr(struct rvt_qp *qp,
 	wqe->length = 0;
 	j = 0;
 	if (wr->num_sge) {
+		struct rvt_sge *last_sge = NULL;
+
 		acc = wr->opcode >= IB_WR_RDMA_READ ?
 			IB_ACCESS_LOCAL_WRITE : 0;
 		for (i = 0; i < wr->num_sge; i++) {
 			u32 length = wr->sg_list[i].length;
-			int ok;
 
 			if (length == 0)
 				continue;
-			ok = rvt_lkey_ok(rkt, pd, &wqe->sg_list[j],
-					 &wr->sg_list[i], acc);
-			if (!ok) {
-				ret = -EINVAL;
+			ret = rvt_lkey_ok(rkt, pd, &wqe->sg_list[j], last_sge,
+					  &wr->sg_list[i], acc);
+			if (unlikely(ret < 0))
 				goto bail_inval_free;
-			}
 			wqe->length += length;
-			j++;
+			if (ret)
+				last_sge = &wqe->sg_list[j];
+			j += ret;
 		}
 		wqe->wr.num_sge = j;
 	}
@@ -1757,7 +1865,7 @@ static int rvt_post_one_wr(struct rvt_qp *qp,
 		wqe->wr.send_flags &= ~RVT_SEND_RESERVE_USED;
 		qp->s_avail--;
 	}
-	trace_rvt_post_one_wr(qp, wqe);
+	trace_rvt_post_one_wr(qp, wqe, wr->num_sge);
 	smp_wmb(); /* see request builders */
 	qp->s_head = next;
 
@@ -2065,3 +2173,147 @@ enum hrtimer_restart rvt_rc_rnr_retry(struct hrtimer *t)
 	return HRTIMER_NORESTART;
 }
 EXPORT_SYMBOL(rvt_rc_rnr_retry);
+
+/**
+ * rvt_qp_iter_init - initial for QP iteration
+ * @rdi - rvt devinfo
+ * @v - u64 value
+ *
+ * This returns an iterator suitable for iterating QPs
+ * in the system.
+ *
+ * The @cb is a user defined callback and @v is a 64
+ * bit value passed to and relevant for processing in the
+ * @cb.  An example use case would be to alter QP processing
+ * based on criteria not part of the rvt_qp.
+ *
+ * Use cases that require memory allocation to succeed
+ * must preallocate appropriately.
+ *
+ * Return: a pointer to an rvt_qp_iter or NULL
+ */
+struct rvt_qp_iter *rvt_qp_iter_init(struct rvt_dev_info *rdi,
+				     u64 v,
+				     void (*cb)(struct rvt_qp *qp, u64 v))
+{
+	struct rvt_qp_iter *i;
+
+	i = kzalloc(sizeof(*i), GFP_KERNEL);
+	if (!i)
+		return NULL;
+
+	i->rdi = rdi;
+	/* number of special QPs (SMI/GSI) for device */
+	i->specials = rdi->ibdev.phys_port_cnt * 2;
+	i->v = v;
+	i->cb = cb;
+
+	return i;
+}
+EXPORT_SYMBOL(rvt_qp_iter_init);
+
+/**
+ * rvt_qp_iter_next - return the next QP in iter
+ * @iter - the iterator
+ *
+ * Fine grained QP iterator suitable for use
+ * with debugfs seq_file mechanisms.
+ *
+ * Updates iter->qp with the current QP when the return
+ * value is 0.
+ *
+ * Return: 0 - iter->qp is valid 1 - no more QPs
+ */
+int rvt_qp_iter_next(struct rvt_qp_iter *iter)
+	__must_hold(RCU)
+{
+	int n = iter->n;
+	int ret = 1;
+	struct rvt_qp *pqp = iter->qp;
+	struct rvt_qp *qp;
+	struct rvt_dev_info *rdi = iter->rdi;
+
+	/*
+	 * The approach is to consider the special qps
+	 * as additional table entries before the
+	 * real hash table.  Since the qp code sets
+	 * the qp->next hash link to NULL, this works just fine.
+	 *
+	 * iter->specials is 2 * # ports
+	 *
+	 * n = 0..iter->specials is the special qp indices
+	 *
+	 * n = iter->specials..rdi->qp_dev->qp_table_size+iter->specials are
+	 * the potential hash bucket entries
+	 *
+	 */
+	for (; n <  rdi->qp_dev->qp_table_size + iter->specials; n++) {
+		if (pqp) {
+			qp = rcu_dereference(pqp->next);
+		} else {
+			if (n < iter->specials) {
+				struct rvt_ibport *rvp;
+				int pidx;
+
+				pidx = n % rdi->ibdev.phys_port_cnt;
+				rvp = rdi->ports[pidx];
+				qp = rcu_dereference(rvp->qp[n & 1]);
+			} else {
+				qp = rcu_dereference(
+					rdi->qp_dev->qp_table[
+						(n - iter->specials)]);
+			}
+		}
+		pqp = qp;
+		if (qp) {
+			iter->qp = qp;
+			iter->n = n;
+			return 0;
+		}
+	}
+	return ret;
+}
+EXPORT_SYMBOL(rvt_qp_iter_next);
+
+/**
+ * rvt_qp_iter - iterate all QPs
+ * @rdi - rvt devinfo
+ * @v - a 64 bit value
+ * @cb - a callback
+ *
+ * This provides a way for iterating all QPs.
+ *
+ * The @cb is a user defined callback and @v is a 64
+ * bit value passed to and relevant for processing in the
+ * cb.  An example use case would be to alter QP processing
+ * based on criteria not part of the rvt_qp.
+ *
+ * The code has an internal iterator to simplify
+ * non seq_file use cases.
+ */
+void rvt_qp_iter(struct rvt_dev_info *rdi,
+		 u64 v,
+		 void (*cb)(struct rvt_qp *qp, u64 v))
+{
+	int ret;
+	struct rvt_qp_iter i = {
+		.rdi = rdi,
+		.specials = rdi->ibdev.phys_port_cnt * 2,
+		.v = v,
+		.cb = cb
+	};
+
+	rcu_read_lock();
+	do {
+		ret = rvt_qp_iter_next(&i);
+		if (!ret) {
+			rvt_get_qp(i.qp);
+			rcu_read_unlock();
+			i.cb(i.qp, i.v);
+			rcu_read_lock();
+			rvt_put_qp(i.qp);
+		}
+	} while (!ret);
+	rcu_read_unlock();
+}
+EXPORT_SYMBOL(rvt_qp_iter);
diff --git a/drivers/infiniband/sw/rdmavt/trace_mr.h b/drivers/infiniband/sw/rdmavt/trace_mr.h
index 3318a6c36373..976e482930a3 100644
--- a/drivers/infiniband/sw/rdmavt/trace_mr.h
+++ b/drivers/infiniband/sw/rdmavt/trace_mr.h
@@ -103,6 +103,68 @@ DEFINE_EVENT(
 	TP_PROTO(struct rvt_mregion *mr, u16 m, u16 n, void *v, size_t len),
 	TP_ARGS(mr, m, n, v, len));
 
+DECLARE_EVENT_CLASS(
+	rvt_sge_template,
+	TP_PROTO(struct rvt_sge *sge, struct ib_sge *isge),
+	TP_ARGS(sge, isge),
+	TP_STRUCT__entry(
+		RDI_DEV_ENTRY(ib_to_rvt(sge->mr->pd->device))
+		__field(struct rvt_mregion *, mr)
+		__field(struct rvt_sge *, sge)
+		__field(struct ib_sge *, isge)
+		__field(void *, vaddr)
+		__field(u64, ivaddr)
+		__field(u32, lkey)
+		__field(u32, sge_length)
+		__field(u32, length)
+		__field(u32, ilength)
+		__field(int, user)
+		__field(u16, m)
+		__field(u16, n)
+	),
+	TP_fast_assign(
+		RDI_DEV_ASSIGN(ib_to_rvt(sge->mr->pd->device));
+		__entry->mr = sge->mr;
+		__entry->sge = sge;
+		__entry->isge = isge;
+		__entry->vaddr = sge->vaddr;
+		__entry->ivaddr = isge->addr;
+		__entry->lkey = sge->mr->lkey;
+		__entry->sge_length = sge->sge_length;
+		__entry->length = sge->length;
+		__entry->ilength = isge->length;
+		__entry->m = sge->m;
+		__entry->n = sge->m;
+		__entry->user = ibpd_to_rvtpd(sge->mr->pd)->user;
+	),
+	TP_printk(
+		"[%s] mr %p sge %p isge %p vaddr %p ivaddr %llx lkey %x sge_length %u length %u ilength %u m %u n %u user %u",
+		__get_str(dev),
+		__entry->mr,
+		__entry->sge,
+		__entry->isge,
+		__entry->vaddr,
+		__entry->ivaddr,
+		__entry->lkey,
+		__entry->sge_length,
+		__entry->length,
+		__entry->ilength,
+		__entry->m,
+		__entry->n,
+		__entry->user
+	)
+);
+
+DEFINE_EVENT(
+	rvt_sge_template, rvt_sge_adjacent,
+	TP_PROTO(struct rvt_sge *sge, struct ib_sge *isge),
+	TP_ARGS(sge, isge));
+
+DEFINE_EVENT(
+	rvt_sge_template, rvt_sge_new,
+	TP_PROTO(struct rvt_sge *sge, struct ib_sge *isge),
+	TP_ARGS(sge, isge));
+
 #endif /* __RVT_TRACE_MR_H */
 
 #undef TRACE_INCLUDE_PATH
diff --git a/drivers/infiniband/sw/rdmavt/trace_tx.h b/drivers/infiniband/sw/rdmavt/trace_tx.h
index a613a2223751..0ef25fc49f25 100644
--- a/drivers/infiniband/sw/rdmavt/trace_tx.h
+++ b/drivers/infiniband/sw/rdmavt/trace_tx.h
@@ -84,12 +84,12 @@ __print_symbolic(opcode,                                   \
 	wr_opcode_name(RESERVED10))
 
 #define POS_PRN \
-"[%s] wqe %p wr_id %llx send_flags %x qpn %x qpt %u psn %x lpsn %x ssn %x length %u opcode 0x%.2x,%s size %u avail %u head %u last %u pid %u num_sge %u"
+"[%s] wqe %p wr_id %llx send_flags %x qpn %x qpt %u psn %x lpsn %x ssn %x length %u opcode 0x%.2x,%s size %u avail %u head %u last %u pid %u num_sge %u wr_num_sge %u"
 
 TRACE_EVENT(
 	rvt_post_one_wr,
-	TP_PROTO(struct rvt_qp *qp, struct rvt_swqe *wqe),
-	TP_ARGS(qp, wqe),
+	TP_PROTO(struct rvt_qp *qp, struct rvt_swqe *wqe, int wr_num_sge),
+	TP_ARGS(qp, wqe, wr_num_sge),
 	TP_STRUCT__entry(
 		RDI_DEV_ENTRY(ib_to_rvt(qp->ibqp.device))
 		__field(u64, wr_id)
@@ -108,6 +108,7 @@ TRACE_EVENT(
 		__field(int, send_flags)
 		__field(pid_t, pid)
 		__field(int, num_sge)
+		__field(int, wr_num_sge)
 	),
 	TP_fast_assign(
 		RDI_DEV_ASSIGN(ib_to_rvt(qp->ibqp.device))
@@ -127,6 +128,7 @@ TRACE_EVENT(
 		__entry->ssn = wqe->ssn;
 		__entry->send_flags = wqe->wr.send_flags;
 		__entry->num_sge = wqe->wr.num_sge;
+		__entry->wr_num_sge = wr_num_sge;
 	),
 	TP_printk(
 		POS_PRN,
@@ -146,7 +148,8 @@ TRACE_EVENT(
 		__entry->head,
 		__entry->last,
 		__entry->pid,
-		__entry->num_sge
+		__entry->num_sge,
+		__entry->wr_num_sge
 	)
 );
 
diff --git a/drivers/infiniband/sw/rdmavt/vt.c b/drivers/infiniband/sw/rdmavt/vt.c
index 0d7c6bb551d9..64bdd442078a 100644
--- a/drivers/infiniband/sw/rdmavt/vt.c
+++ b/drivers/infiniband/sw/rdmavt/vt.c
@@ -202,8 +202,13 @@ static int rvt_modify_port(struct ib_device *ibdev, u8 port_num,
 		return -EINVAL;
 
 	rvp = rdi->ports[port_index];
-	rvp->port_cap_flags |= props->set_port_cap_mask;
-	rvp->port_cap_flags &= ~props->clr_port_cap_mask;
+	if (port_modify_mask & IB_PORT_OPA_MASK_CHG) {
+		rvp->port_cap3_flags |= props->set_port_cap_mask;
+		rvp->port_cap3_flags &= ~props->clr_port_cap_mask;
+	} else {
+		rvp->port_cap_flags |= props->set_port_cap_mask;
+		rvp->port_cap_flags &= ~props->clr_port_cap_mask;
+	}
 
 	if (props->set_port_cap_mask || props->clr_port_cap_mask)
 		rdi->driver_f.cap_mask_chg(rdi, port_num);
diff --git a/drivers/infiniband/sw/rxe/rxe.c b/drivers/infiniband/sw/rxe/rxe.c
index c21c913f911a..8c3d30b3092d 100644
--- a/drivers/infiniband/sw/rxe/rxe.c
+++ b/drivers/infiniband/sw/rxe/rxe.c
@@ -38,7 +38,6 @@
 MODULE_AUTHOR("Bob Pearson, Frank Zago, John Groves, Kamal Heib");
 MODULE_DESCRIPTION("Soft RDMA transport");
 MODULE_LICENSE("Dual BSD/GPL");
-MODULE_VERSION("0.2");
 
 /* free resources for all ports on a device */
 static void rxe_cleanup_ports(struct rxe_dev *rxe)
diff --git a/drivers/infiniband/sw/rxe/rxe.h b/drivers/infiniband/sw/rxe/rxe.h
index 1ac5b8551a4d..6447d736d5a4 100644
--- a/drivers/infiniband/sw/rxe/rxe.h
+++ b/drivers/infiniband/sw/rxe/rxe.h
@@ -97,7 +97,7 @@ int rxe_rcv(struct sk_buff *skb);
 
 void rxe_dev_put(struct rxe_dev *rxe);
 struct rxe_dev *net_to_rxe(struct net_device *ndev);
-struct rxe_dev *get_rxe_by_name(const char* name);
+struct rxe_dev *get_rxe_by_name(const char *name);
 
 void rxe_port_up(struct rxe_dev *rxe);
 void rxe_port_down(struct rxe_dev *rxe);
diff --git a/drivers/infiniband/sw/rxe/rxe_av.c b/drivers/infiniband/sw/rxe/rxe_av.c
index 5bddf469361b..1cc9e2e1365d 100644
--- a/drivers/infiniband/sw/rxe/rxe_av.c
+++ b/drivers/infiniband/sw/rxe/rxe_av.c
@@ -38,18 +38,13 @@ int rxe_av_chk_attr(struct rxe_dev *rxe, struct rdma_ah_attr *attr)
 {
 	struct rxe_port *port;
 
-	if (rdma_ah_get_port_num(attr) != 1) {
-		pr_info("invalid port_num = %d\n", rdma_ah_get_port_num(attr));
-		return -EINVAL;
-	}
-
 	port = &rxe->port;
 
 	if (rdma_ah_get_ah_flags(attr) & IB_AH_GRH) {
 		u8 sgid_index = rdma_ah_read_grh(attr)->sgid_index;
 
 		if (sgid_index > port->attr.gid_tbl_len) {
-			pr_info("invalid sgid index = %d\n", sgid_index);
+			pr_warn("invalid sgid index = %d\n", sgid_index);
 			return -EINVAL;
 		}
 	}
diff --git a/drivers/infiniband/sw/rxe/rxe_cq.c b/drivers/infiniband/sw/rxe/rxe_cq.c
index 49fe42c23f4d..c4aabf78dc90 100644
--- a/drivers/infiniband/sw/rxe/rxe_cq.c
+++ b/drivers/infiniband/sw/rxe/rxe_cq.c
@@ -69,6 +69,14 @@ err1:
 static void rxe_send_complete(unsigned long data)
 {
 	struct rxe_cq *cq = (struct rxe_cq *)data;
+	unsigned long flags;
+
+	spin_lock_irqsave(&cq->cq_lock, flags);
+	if (cq->is_dying) {
+		spin_unlock_irqrestore(&cq->cq_lock, flags);
+		return;
+	}
+	spin_unlock_irqrestore(&cq->cq_lock, flags);
 
 	cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
 }
@@ -97,6 +105,8 @@ int rxe_cq_from_init(struct rxe_dev *rxe, struct rxe_cq *cq, int cqe,
 	if (udata)
 		cq->is_user = 1;
 
+	cq->is_dying = false;
+
 	tasklet_init(&cq->comp_task, rxe_send_complete, (unsigned long)cq);
 
 	spin_lock_init(&cq->cq_lock);
@@ -156,6 +166,15 @@ int rxe_cq_post(struct rxe_cq *cq, struct rxe_cqe *cqe, int solicited)
 	return 0;
 }
 
+void rxe_cq_disable(struct rxe_cq *cq)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&cq->cq_lock, flags);
+	cq->is_dying = true;
+	spin_unlock_irqrestore(&cq->cq_lock, flags);
+}
+
 void rxe_cq_cleanup(struct rxe_pool_entry *arg)
 {
 	struct rxe_cq *cq = container_of(arg, typeof(*cq), pelem);
diff --git a/drivers/infiniband/sw/rxe/rxe_hw_counters.c b/drivers/infiniband/sw/rxe/rxe_hw_counters.c
index 7ef90aad7dfd..6aeb7a165e46 100644
--- a/drivers/infiniband/sw/rxe/rxe_hw_counters.c
+++ b/drivers/infiniband/sw/rxe/rxe_hw_counters.c
@@ -33,7 +33,7 @@
 #include "rxe.h"
 #include "rxe_hw_counters.h"
 
-const char * const rxe_counter_name[] = {
+static const char * const rxe_counter_name[] = {
 	[RXE_CNT_SENT_PKTS]           =  "sent_pkts",
 	[RXE_CNT_RCVD_PKTS]           =  "rcvd_pkts",
 	[RXE_CNT_DUP_REQ]             =  "duplicate_request",
diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h
index d6299edf9a5b..77b3ed0df936 100644
--- a/drivers/infiniband/sw/rxe/rxe_loc.h
+++ b/drivers/infiniband/sw/rxe/rxe_loc.h
@@ -64,6 +64,8 @@ int rxe_cq_resize_queue(struct rxe_cq *cq, int new_cqe, struct ib_udata *udata);
 
 int rxe_cq_post(struct rxe_cq *cq, struct rxe_cqe *cqe, int solicited);
 
+void rxe_cq_disable(struct rxe_cq *cq);
+
 void rxe_cq_cleanup(struct rxe_pool_entry *arg);
 
 /* rxe_mcast.c */
@@ -219,8 +221,6 @@ static inline void rxe_advance_resp_resource(struct rxe_qp *qp)
 void retransmit_timer(unsigned long data);
 void rnr_nak_timer(unsigned long data);
 
-void dump_qp(struct rxe_qp *qp);
-
 /* rxe_srq.c */
 #define IB_SRQ_INIT_MASK (~IB_SRQ_LIMIT)
 
@@ -250,7 +250,7 @@ void rxe_resp_queue_pkt(struct rxe_dev *rxe,
 void rxe_comp_queue_pkt(struct rxe_dev *rxe,
 			struct rxe_qp *qp, struct sk_buff *skb);
 
-static inline unsigned wr_opcode_mask(int opcode, struct rxe_qp *qp)
+static inline unsigned int wr_opcode_mask(int opcode, struct rxe_qp *qp)
 {
 	return rxe_wr_opcode_info[opcode].mask[qp->ibqp.qp_type];
 }
diff --git a/drivers/infiniband/sw/rxe/rxe_mmap.c b/drivers/infiniband/sw/rxe/rxe_mmap.c
index bd812e00988e..d22431e3a908 100644
--- a/drivers/infiniband/sw/rxe/rxe_mmap.c
+++ b/drivers/infiniband/sw/rxe/rxe_mmap.c
@@ -76,7 +76,7 @@ static void rxe_vma_close(struct vm_area_struct *vma)
 	kref_put(&ip->ref, rxe_mmap_release);
 }
 
-static struct vm_operations_struct rxe_vm_ops = {
+static const struct vm_operations_struct rxe_vm_ops = {
 	.open = rxe_vma_open,
 	.close = rxe_vma_close,
 };
diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c
index e37cc89987e1..5c2684bf430f 100644
--- a/drivers/infiniband/sw/rxe/rxe_mr.c
+++ b/drivers/infiniband/sw/rxe/rxe_mr.c
@@ -367,11 +367,11 @@ int rxe_mem_copy(struct rxe_mem *mem, u64 iova, void *addr, int length,
 		dest = (dir == to_mem_obj) ?
 			((void *)(uintptr_t)iova) : addr;
 
+		memcpy(dest, src, length);
+
 		if (crcp)
 			*crcp = rxe_crc32(to_rdev(mem->pd->ibpd.device),
-					*crcp, src, length);
-
-		memcpy(dest, src, length);
+					*crcp, dest, length);
 
 		return 0;
 	}
@@ -401,11 +401,11 @@ int rxe_mem_copy(struct rxe_mem *mem, u64 iova, void *addr, int length,
 		if (bytes > length)
 			bytes = length;
 
+		memcpy(dest, src, bytes);
+
 		if (crcp)
 			crc = rxe_crc32(to_rdev(mem->pd->ibpd.device),
-					crc, src, bytes);
-
-		memcpy(dest, src, bytes);
+					crc, dest, bytes);
 
 		length	-= bytes;
 		addr	+= bytes;
diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c
index 08f3f90d2912..59dee10bebcb 100644
--- a/drivers/infiniband/sw/rxe/rxe_net.c
+++ b/drivers/infiniband/sw/rxe/rxe_net.c
@@ -191,7 +191,7 @@ static struct dst_entry *rxe_find_route(struct rxe_dev *rxe,
 	if (qp_type(qp) == IB_QPT_RC)
 		dst = sk_dst_get(qp->sk->sk);
 
-	if (!dst || !(dst->obsolete && dst->ops->check(dst, 0))) {
+	if (!dst || !dst_check(dst, qp->dst_cookie)) {
 		if (dst)
 			dst_release(dst);
 
@@ -209,6 +209,11 @@ static struct dst_entry *rxe_find_route(struct rxe_dev *rxe,
 			saddr6 = &av->sgid_addr._sockaddr_in6.sin6_addr;
 			daddr6 = &av->dgid_addr._sockaddr_in6.sin6_addr;
 			dst = rxe_find_route6(rxe->ndev, saddr6, daddr6);
+#if IS_ENABLED(CONFIG_IPV6)
+			if (dst)
+				qp->dst_cookie =
+					rt6_get_cookie((struct rt6_info *)dst);
+#endif
 		}
 	}
 
@@ -337,7 +342,7 @@ static void prepare_ipv6_hdr(struct dst_entry *dst, struct sk_buff *skb,
 	memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));
 	IPCB(skb)->flags &= ~(IPSKB_XFRM_TUNNEL_SIZE | IPSKB_XFRM_TRANSFORMED
 			    | IPSKB_REROUTED);
-	skb_dst_set(skb, dst);
+	skb_dst_set(skb, dst_clone(dst));
 
 	__skb_push(skb, sizeof(*ip6h));
 	skb_reset_network_header(skb);
@@ -388,7 +393,7 @@ static int prepare6(struct rxe_dev *rxe, struct rxe_pkt_info *pkt,
 		    struct sk_buff *skb, struct rxe_av *av)
 {
 	struct rxe_qp *qp = pkt->qp;
-	struct dst_entry *dst = NULL;
+	struct dst_entry *dst;
 	struct in6_addr *saddr = &av->sgid_addr._sockaddr_in6.sin6_addr;
 	struct in6_addr *daddr = &av->dgid_addr._sockaddr_in6.sin6_addr;
 
@@ -460,12 +465,17 @@ int rxe_send(struct rxe_dev *rxe, struct rxe_pkt_info *pkt, struct sk_buff *skb)
 	nskb->destructor = rxe_skb_tx_dtor;
 	nskb->sk = pkt->qp->sk->sk;
 
+	rxe_add_ref(pkt->qp);
+	atomic_inc(&pkt->qp->skb_out);
+
 	if (av->network_type == RDMA_NETWORK_IPV4) {
 		err = ip_local_out(dev_net(skb_dst(skb)->dev), nskb->sk, nskb);
 	} else if (av->network_type == RDMA_NETWORK_IPV6) {
 		err = ip6_local_out(dev_net(skb_dst(skb)->dev), nskb->sk, nskb);
 	} else {
 		pr_err("Unknown layer 3 protocol: %d\n", av->network_type);
+		atomic_dec(&pkt->qp->skb_out);
+		rxe_drop_ref(pkt->qp);
 		kfree_skb(nskb);
 		return -EINVAL;
 	}
@@ -475,10 +485,7 @@ int rxe_send(struct rxe_dev *rxe, struct rxe_pkt_info *pkt, struct sk_buff *skb)
 		return -EAGAIN;
 	}
 
-	rxe_add_ref(pkt->qp);
-	atomic_inc(&pkt->qp->skb_out);
 	kfree_skb(skb);
-
 	return 0;
 }
 
@@ -644,8 +651,13 @@ static int rxe_notify(struct notifier_block *not_blk,
 		pr_info("%s changed mtu to %d\n", ndev->name, ndev->mtu);
 		rxe_set_mtu(rxe, ndev->mtu);
 		break;
-	case NETDEV_REBOOT:
 	case NETDEV_CHANGE:
+		if (netif_running(ndev) && netif_carrier_ok(ndev))
+			rxe_port_up(rxe);
+		else
+			rxe_port_down(rxe);
+		break;
+	case NETDEV_REBOOT:
 	case NETDEV_GOING_DOWN:
 	case NETDEV_CHANGEADDR:
 	case NETDEV_CHANGENAME:
diff --git a/drivers/infiniband/sw/rxe/rxe_pool.c b/drivers/infiniband/sw/rxe/rxe_pool.c
index 75d11ee635ec..c1b5f38f31a5 100644
--- a/drivers/infiniband/sw/rxe/rxe_pool.c
+++ b/drivers/infiniband/sw/rxe/rxe_pool.c
@@ -188,7 +188,7 @@ int rxe_pool_init(
 	struct rxe_dev		*rxe,
 	struct rxe_pool		*pool,
 	enum rxe_elem_type	type,
-	unsigned		max_elem)
+	unsigned int		max_elem)
 {
 	int			err = 0;
 	size_t			size = rxe_type_info[type].size;
diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c
index 80ccc7c7c341..00bda9380a2e 100644
--- a/drivers/infiniband/sw/rxe/rxe_qp.c
+++ b/drivers/infiniband/sw/rxe/rxe_qp.c
@@ -851,13 +851,8 @@ void rxe_qp_cleanup(struct rxe_pool_entry *arg)
 		qp->resp.mr = NULL;
 	}
 
-	if (qp_type(qp) == IB_QPT_RC) {
-		struct dst_entry *dst = NULL;
-
-		dst = sk_dst_get(qp->sk->sk);
-		if (dst)
-			dst_release(dst);
-	}
+	if (qp_type(qp) == IB_QPT_RC)
+		sk_dst_reset(qp->sk->sk);
 
 	free_rd_atomic_resources(qp);
 
diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c
index 7ee465d1a1e1..d84222f9d5d2 100644
--- a/drivers/infiniband/sw/rxe/rxe_req.c
+++ b/drivers/infiniband/sw/rxe/rxe_req.c
@@ -43,7 +43,7 @@ static int next_opcode(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
 
 static inline void retry_first_write_send(struct rxe_qp *qp,
 					  struct rxe_send_wqe *wqe,
-					  unsigned mask, int npsn)
+					  unsigned int mask, int npsn)
 {
 	int i;
 
@@ -594,8 +594,10 @@ int rxe_requester(void *arg)
 	rxe_add_ref(qp);
 
 next_wqe:
-	if (unlikely(!qp->valid))
+	if (unlikely(!qp->valid)) {
+		rxe_drain_req_pkts(qp, true);
 		goto exit;
+	}
 
 	if (unlikely(qp->req.state == QP_STATE_ERROR)) {
 		rxe_drain_req_pkts(qp, true);
diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c
index a958ee918a49..4240866a5331 100644
--- a/drivers/infiniband/sw/rxe/rxe_resp.c
+++ b/drivers/infiniband/sw/rxe/rxe_resp.c
@@ -1055,7 +1055,7 @@ static struct resp_res *find_resource(struct rxe_qp *qp, u32 psn)
 {
 	int i;
 
-	for (i = 0; i < qp->attr.max_rd_atomic; i++) {
+	for (i = 0; i < qp->attr.max_dest_rd_atomic; i++) {
 		struct resp_res *res = &qp->resp.resources[i];
 
 		if (res->type == 0)
diff --git a/drivers/infiniband/sw/rxe/rxe_task.c b/drivers/infiniband/sw/rxe/rxe_task.c
index d2a14a1bdc7f..ea3810b29273 100644
--- a/drivers/infiniband/sw/rxe/rxe_task.c
+++ b/drivers/infiniband/sw/rxe/rxe_task.c
@@ -78,7 +78,7 @@ void rxe_do_task(unsigned long data)
 
 	default:
 		spin_unlock_irqrestore(&task->state_lock, flags);
-		pr_warn("bad state = %d in rxe_do_task\n", task->state);
+		pr_warn("%s failed with bad state %d\n", __func__, task->state);
 		return;
 	}
 
@@ -105,7 +105,7 @@ void rxe_do_task(unsigned long data)
 			break;
 
 		default:
-			pr_warn("bad state = %d in rxe_do_task\n",
+			pr_warn("%s failed with bad state %d\n", __func__,
 				task->state);
 		}
 		spin_unlock_irqrestore(&task->state_lock, flags);
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index af90a7d42b96..0b362f49a10a 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -51,40 +51,16 @@ static int rxe_query_device(struct ib_device *dev,
 	return 0;
 }
 
-static void rxe_eth_speed_to_ib_speed(int speed, u8 *active_speed,
-				      u8 *active_width)
-{
-	if (speed <= 1000) {
-		*active_width = IB_WIDTH_1X;
-		*active_speed = IB_SPEED_SDR;
-	} else if (speed <= 10000) {
-		*active_width = IB_WIDTH_1X;
-		*active_speed = IB_SPEED_FDR10;
-	} else if (speed <= 20000) {
-		*active_width = IB_WIDTH_4X;
-		*active_speed = IB_SPEED_DDR;
-	} else if (speed <= 30000) {
-		*active_width = IB_WIDTH_4X;
-		*active_speed = IB_SPEED_QDR;
-	} else if (speed <= 40000) {
-		*active_width = IB_WIDTH_4X;
-		*active_speed = IB_SPEED_FDR10;
-	} else {
-		*active_width = IB_WIDTH_4X;
-		*active_speed = IB_SPEED_EDR;
-	}
-}
-
 static int rxe_query_port(struct ib_device *dev,
 			  u8 port_num, struct ib_port_attr *attr)
 {
 	struct rxe_dev *rxe = to_rdev(dev);
 	struct rxe_port *port;
-	u32 speed;
+	int rc = -EINVAL;
 
 	if (unlikely(port_num != 1)) {
 		pr_warn("invalid port_number %d\n", port_num);
-		goto err1;
+		goto out;
 	}
 
 	port = &rxe->port;
@@ -93,29 +69,12 @@ static int rxe_query_port(struct ib_device *dev,
 	*attr = port->attr;
 
 	mutex_lock(&rxe->usdev_lock);
-	if (rxe->ndev->ethtool_ops->get_link_ksettings) {
-		struct ethtool_link_ksettings ks;
-
-		rxe->ndev->ethtool_ops->get_link_ksettings(rxe->ndev, &ks);
-		speed = ks.base.speed;
-	} else if (rxe->ndev->ethtool_ops->get_settings) {
-		struct ethtool_cmd cmd;
-
-		rxe->ndev->ethtool_ops->get_settings(rxe->ndev, &cmd);
-		speed = cmd.speed;
-	} else {
-		pr_warn("%s speed is unknown, defaulting to 1000\n",
-			rxe->ndev->name);
-		speed = 1000;
-	}
-	rxe_eth_speed_to_ib_speed(speed, &attr->active_speed,
-				  &attr->active_width);
+	rc = ib_get_eth_speed(dev, port_num, &attr->active_speed,
+			      &attr->active_width);
 	mutex_unlock(&rxe->usdev_lock);
 
-	return 0;
-
-err1:
-	return -EINVAL;
+out:
+	return rc;
 }
 
 static int rxe_query_gid(struct ib_device *device,
@@ -960,6 +919,8 @@ static int rxe_destroy_cq(struct ib_cq *ibcq)
 {
 	struct rxe_cq *cq = to_rcq(ibcq);
 
+	rxe_cq_disable(cq);
+
 	rxe_drop_ref(cq);
 	return 0;
 }
@@ -1210,8 +1171,8 @@ static int rxe_detach_mcast(struct ib_qp *ibqp, union ib_gid *mgid, u16 mlid)
 	return rxe_mcast_drop_grp_elem(rxe, qp, mgid);
 }
 
-static ssize_t rxe_show_parent(struct device *device,
-			       struct device_attribute *attr, char *buf)
+static ssize_t parent_show(struct device *device,
+			   struct device_attribute *attr, char *buf)
 {
 	struct rxe_dev *rxe = container_of(device, struct rxe_dev,
 					   ib_dev.dev);
@@ -1219,7 +1180,7 @@ static ssize_t rxe_show_parent(struct device *device,
 	return snprintf(buf, 16, "%s\n", rxe_parent_name(rxe, 1));
 }
 
-static DEVICE_ATTR(parent, S_IRUGO, rxe_show_parent, NULL);
+static DEVICE_ATTR_RO(parent);
 
 static struct device_attribute *rxe_dev_attributes[] = {
 	&dev_attr_parent,
@@ -1336,15 +1297,15 @@ int rxe_register_device(struct rxe_dev *rxe)
 
 	err = ib_register_device(dev, NULL);
 	if (err) {
-		pr_warn("rxe_register_device failed, err = %d\n", err);
+		pr_warn("%s failed with error %d\n", __func__, err);
 		goto err1;
 	}
 
 	for (i = 0; i < ARRAY_SIZE(rxe_dev_attributes); ++i) {
 		err = device_create_file(&dev->dev, rxe_dev_attributes[i]);
 		if (err) {
-			pr_warn("device_create_file failed, i = %d, err = %d\n",
-				i, err);
+			pr_warn("%s failed with error %d for attr number %d\n",
+				__func__, err, i);
 			goto err2;
 		}
 	}
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index 5a180fbe40d9..0c2dbe45c729 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -89,6 +89,7 @@ struct rxe_cq {
 	struct rxe_queue	*queue;
 	spinlock_t		cq_lock;
 	u8			notify;
+	bool			is_dying;
 	int			is_user;
 	struct tasklet_struct	comp_task;
 };
@@ -247,6 +248,7 @@ struct rxe_qp {
 	struct rxe_rq		rq;
 
 	struct socket		*sk;
+	u32			dst_cookie;
 
 	struct rxe_av		pri_av;
 	struct rxe_av		alt_av;
diff --git a/drivers/infiniband/ulp/ipoib/ipoib.h b/drivers/infiniband/ulp/ipoib/ipoib.h
index 7ac25059c40f..4a5c7a07a631 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib.h
+++ b/drivers/infiniband/ulp/ipoib/ipoib.h
@@ -337,6 +337,7 @@ struct ipoib_dev_priv {
 
 	struct rw_semaphore vlan_rwsem;
 	struct mutex mcast_mutex;
+	struct mutex sysfs_mutex;
 
 	struct rb_root  path_tree;
 	struct list_head path_list;
@@ -367,7 +368,7 @@ struct ipoib_dev_priv {
 	u32		  qkey;
 
 	union ib_gid local_gid;
-	u16	     local_lid;
+	u32	     local_lid;
 
 	unsigned int admin_mtu;
 	unsigned int mcast_mtu;
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_cm.c b/drivers/infiniband/ulp/ipoib/ipoib_cm.c
index d69410c2ed97..14b62f7472b4 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_cm.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_cm.c
@@ -1506,9 +1506,14 @@ static ssize_t set_mode(struct device *d, struct device_attribute *attr,
 	if (test_bit(IPOIB_FLAG_GOING_DOWN, &priv->flags))
 		return -EPERM;
 
-	if (!rtnl_trylock())
+	if (!mutex_trylock(&priv->sysfs_mutex))
 		return restart_syscall();
 
+	if (!rtnl_trylock()) {
+		mutex_unlock(&priv->sysfs_mutex);
+		return restart_syscall();
+	}
+
 	ret = ipoib_set_mode(dev, buf);
 
 	/* The assumption is that the function ipoib_set_mode returned
@@ -1517,6 +1522,7 @@ static ssize_t set_mode(struct device *d, struct device_attribute *attr,
 	 */
 	if (ret != -EBUSY)
 		rtnl_unlock();
+	mutex_unlock(&priv->sysfs_mutex);
 
 	return (!ret || ret == -EBUSY) ? count : ret;
 }
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_ethtool.c b/drivers/infiniband/ulp/ipoib/ipoib_ethtool.c
index 184a22f48027..8dc1e6225cc8 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_ethtool.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_ethtool.c
@@ -63,8 +63,7 @@ static void ipoib_get_drvinfo(struct net_device *netdev,
 {
 	struct ipoib_dev_priv *priv = ipoib_priv(netdev);
 
-	ib_get_device_fw_str(priv->ca, drvinfo->fw_version,
-			     sizeof(drvinfo->fw_version));
+	ib_get_device_fw_str(priv->ca, drvinfo->fw_version);
 
 	strlcpy(drvinfo->bus_info, dev_name(priv->ca->dev.parent),
 		sizeof(drvinfo->bus_info));
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c
index 6c77df34869d..bac95b509a9b 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_main.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c
@@ -60,7 +60,6 @@ const char ipoib_driver_version[] = DRV_VERSION;
 MODULE_AUTHOR("Roland Dreier");
 MODULE_DESCRIPTION("IP-over-InfiniBand net driver");
 MODULE_LICENSE("Dual BSD/GPL");
-MODULE_VERSION(DRV_VERSION);
 
 int ipoib_sendq_size __read_mostly = IPOIB_TX_RING_SIZE;
 int ipoib_recvq_size __read_mostly = IPOIB_RX_RING_SIZE;
@@ -100,6 +99,8 @@ static struct net_device *ipoib_get_net_dev_by_params(
 		const union ib_gid *gid, const struct sockaddr *addr,
 		void *client_data);
 static int ipoib_set_mac(struct net_device *dev, void *addr);
+static int ipoib_ioctl(struct net_device *dev, struct ifreq *ifr,
+		       int cmd);
 
 static struct ib_client ipoib_client = {
 	.name   = "ipoib",
@@ -1681,6 +1682,17 @@ out:
 	return -ENOMEM;
 }
 
+static int ipoib_ioctl(struct net_device *dev, struct ifreq *ifr,
+		       int cmd)
+{
+	struct ipoib_dev_priv *priv = ipoib_priv(dev);
+
+	if (!priv->rn_ops->ndo_do_ioctl)
+		return -EOPNOTSUPP;
+
+	return priv->rn_ops->ndo_do_ioctl(dev, ifr, cmd);
+}
+
 int ipoib_dev_init(struct net_device *dev, struct ib_device *ca, int port)
 {
 	struct ipoib_dev_priv *priv = ipoib_priv(dev);
@@ -1835,6 +1847,7 @@ static const struct net_device_ops ipoib_netdev_ops_pf = {
 	.ndo_set_vf_guid	 = ipoib_set_vf_guid,
 	.ndo_set_mac_address	 = ipoib_set_mac,
 	.ndo_get_stats64	 = ipoib_get_stats,
+	.ndo_do_ioctl		 = ipoib_ioctl,
 };
 
 static const struct net_device_ops ipoib_netdev_ops_vf = {
@@ -1848,6 +1861,7 @@ static const struct net_device_ops ipoib_netdev_ops_vf = {
 	.ndo_set_rx_mode	 = ipoib_set_mcast_list,
 	.ndo_get_iflink		 = ipoib_get_iflink,
 	.ndo_get_stats64	 = ipoib_get_stats,
+	.ndo_do_ioctl		 = ipoib_ioctl,
 };
 
 void ipoib_setup_common(struct net_device *dev)
@@ -1879,6 +1893,7 @@ static void ipoib_build_priv(struct net_device *dev)
 	spin_lock_init(&priv->lock);
 	init_rwsem(&priv->vlan_rwsem);
 	mutex_init(&priv->mcast_mutex);
+	mutex_init(&priv->sysfs_mutex);
 
 	INIT_LIST_HEAD(&priv->path_list);
 	INIT_LIST_HEAD(&priv->child_intfs);
@@ -2228,13 +2243,7 @@ static struct net_device *ipoib_add_port(const char *format,
 
 	INIT_IB_EVENT_HANDLER(&priv->event_handler,
 			      priv->ca, ipoib_event);
-	result = ib_register_event_handler(&priv->event_handler);
-	if (result < 0) {
-		printk(KERN_WARNING "%s: ib_register_event_handler failed for "
-		       "port %d (ret = %d)\n",
-		       hca->name, port, result);
-		goto event_failed;
-	}
+	ib_register_event_handler(&priv->event_handler);
 
 	result = register_netdev(priv->dev);
 	if (result) {
@@ -2267,8 +2276,6 @@ register_failed:
 	set_bit(IPOIB_STOP_NEIGH_GC, &priv->flags);
 	cancel_delayed_work(&priv->neigh_reap_task);
 	flush_workqueue(priv->wq);
-
-event_failed:
 	ipoib_dev_cleanup(priv->dev);
 
 device_init_failed:
@@ -2338,7 +2345,11 @@ static void ipoib_remove_one(struct ib_device *device, void *client_data)
 		cancel_delayed_work(&priv->neigh_reap_task);
 		flush_workqueue(priv->wq);
 
+		/* Wrap rtnl_lock/unlock with mutex to protect sysfs calls */
+		mutex_lock(&priv->sysfs_mutex);
 		unregister_netdev(priv->dev);
+		mutex_unlock(&priv->sysfs_mutex);
+
 		rn->free_rdma_netdev(priv->dev);
 
 		list_for_each_entry_safe(cpriv, tcpriv, &priv->child_intfs, list)
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_vlan.c b/drivers/infiniband/ulp/ipoib/ipoib_vlan.c
index 081b33deff1b..9927cd6b7082 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_vlan.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_vlan.c
@@ -133,12 +133,20 @@ int ipoib_vlan_add(struct net_device *pdev, unsigned short pkey)
 	snprintf(intf_name, sizeof intf_name, "%s.%04x",
 		 ppriv->dev->name, pkey);
 
-	if (!rtnl_trylock())
+	if (!mutex_trylock(&ppriv->sysfs_mutex))
 		return restart_syscall();
 
+	if (!rtnl_trylock()) {
+		mutex_unlock(&ppriv->sysfs_mutex);
+		return restart_syscall();
+	}
+
 	priv = ipoib_intf_alloc(ppriv->ca, ppriv->port, intf_name);
-	if (!priv)
+	if (!priv) {
+		rtnl_unlock();
+		mutex_unlock(&ppriv->sysfs_mutex);
 		return -ENOMEM;
+	}
 
 	down_write(&ppriv->vlan_rwsem);
 
@@ -164,8 +172,8 @@ int ipoib_vlan_add(struct net_device *pdev, unsigned short pkey)
 
 out:
 	up_write(&ppriv->vlan_rwsem);
-
 	rtnl_unlock();
+	mutex_unlock(&ppriv->sysfs_mutex);
 
 	if (result) {
 		free_netdev(priv->dev);
@@ -188,8 +196,13 @@ int ipoib_vlan_delete(struct net_device *pdev, unsigned short pkey)
 	if (test_bit(IPOIB_FLAG_GOING_DOWN, &ppriv->flags))
 		return -EPERM;
 
-	if (!rtnl_trylock())
+	if (!mutex_trylock(&ppriv->sysfs_mutex))
+		return restart_syscall();
+
+	if (!rtnl_trylock()) {
+		mutex_unlock(&ppriv->sysfs_mutex);
 		return restart_syscall();
+	}
 
 	down_write(&ppriv->vlan_rwsem);
 	list_for_each_entry_safe(priv, tpriv, &ppriv->child_intfs, list) {
@@ -208,6 +221,7 @@ int ipoib_vlan_delete(struct net_device *pdev, unsigned short pkey)
 	}
 
 	rtnl_unlock();
+	mutex_unlock(&ppriv->sysfs_mutex);
 
 	if (dev) {
 		free_netdev(dev);
diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.c b/drivers/infiniband/ulp/iser/iscsi_iser.c
index 37b33d708c2d..19624e023ebd 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.c
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.c
@@ -77,7 +77,6 @@
 MODULE_DESCRIPTION("iSER (iSCSI Extensions for RDMA) Datamover");
 MODULE_LICENSE("Dual BSD/GPL");
 MODULE_AUTHOR("Alex Nezhinsky, Dan Bar Dov, Or Gerlitz");
-MODULE_VERSION(DRV_VER);
 
 static struct scsi_host_template iscsi_iser_sht;
 static struct iscsi_transport iscsi_iser_transport;
diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c
index 26a004e97ae0..55a73b0ed4c6 100644
--- a/drivers/infiniband/ulp/iser/iser_verbs.c
+++ b/drivers/infiniband/ulp/iser/iser_verbs.c
@@ -106,9 +106,7 @@ static int iser_create_device_ib_res(struct iser_device *device)
 
 	INIT_IB_EVENT_HANDLER(&device->event_handler, ib_dev,
 			      iser_event_handler);
-	if (ib_register_event_handler(&device->event_handler))
-		goto cq_err;
-
+	ib_register_event_handler(&device->event_handler);
 	return 0;
 
 cq_err:
@@ -141,7 +139,7 @@ static void iser_free_device_ib_res(struct iser_device *device)
 		comp->cq = NULL;
 	}
 
-	(void)ib_unregister_event_handler(&device->event_handler);
+	ib_unregister_event_handler(&device->event_handler);
 	ib_dealloc_pd(device->pd);
 
 	kfree(device->comps);
diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c
index 0e662656ef42..ceabdb85df8b 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.c
+++ b/drivers/infiniband/ulp/isert/ib_isert.c
@@ -2710,7 +2710,6 @@ static void __exit isert_exit(void)
 }
 
 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure");
-MODULE_VERSION("1.0");
 MODULE_AUTHOR("nab@Linux-iSCSI.org");
 MODULE_LICENSE("GPL");
 
diff --git a/drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c b/drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c
index cf768dd78d1b..21f0b481edcc 100644
--- a/drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c
+++ b/drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c
@@ -52,7 +52,9 @@
 
 #include <linux/module.h>
 #include <rdma/ib_addr.h>
-#include <rdma/ib_smi.h>
+#include <rdma/ib_verbs.h>
+#include <rdma/opa_smi.h>
+#include <rdma/opa_port_info.h>
 
 #include "opa_vnic_internal.h"
 
@@ -952,12 +954,7 @@ static int vema_register(struct opa_vnic_ctrl_port *cport)
 
 		INIT_IB_EVENT_HANDLER(&port->event_handler,
 				      cport->ibdev, opa_vnic_event);
-		ret = ib_register_event_handler(&port->event_handler);
-		if (ret) {
-			c_err("port %d: event handler register failed\n", i);
-			vema_unregister(cport);
-			return ret;
-		}
+		ib_register_event_handler(&port->event_handler);
 
 		idr_init(&port->vport_idr);
 		mutex_init(&port->lock);
@@ -980,6 +977,27 @@ static int vema_register(struct opa_vnic_ctrl_port *cport)
 }
 
 /**
+ * opa_vnic_ctrl_config_dev -- This function sends a trap to the EM
+ * by way of ib_modify_port to indicate support for ethernet on the
+ * fabric.
+ * @cport: pointer to control port
+ * @en: enable or disable ethernet on fabric support
+ */
+static void opa_vnic_ctrl_config_dev(struct opa_vnic_ctrl_port *cport, bool en)
+{
+	struct ib_port_modify pm = { 0 };
+	int i;
+
+	if (en)
+		pm.set_port_cap_mask = OPA_CAP_MASK3_IsEthOnFabricSupported;
+	else
+		pm.clr_port_cap_mask = OPA_CAP_MASK3_IsEthOnFabricSupported;
+
+	for (i = 1; i <= cport->num_ports; i++)
+		ib_modify_port(cport->ibdev, i, IB_PORT_OPA_MASK_CHG, &pm);
+}
+
+/**
  * opa_vnic_vema_add_one -- Handle new ib device
  * @device: ib device pointer
  *
@@ -1007,6 +1025,7 @@ static void opa_vnic_vema_add_one(struct ib_device *device)
 		c_info("VNIC client initialized\n");
 
 	ib_set_client_data(device, &opa_vnic_client, cport);
+	opa_vnic_ctrl_config_dev(cport, true);
 }
 
 /**
@@ -1025,6 +1044,7 @@ static void opa_vnic_vema_rem_one(struct ib_device *device,
 		return;
 
 	c_info("removing VNIC client\n");
+	opa_vnic_ctrl_config_dev(cport, false);
 	vema_unregister(cport);
 	kfree(cport);
 }
@@ -1053,4 +1073,3 @@ module_exit(opa_vnic_deinit);
 MODULE_LICENSE("Dual BSD/GPL");
 MODULE_AUTHOR("Intel Corporation");
 MODULE_DESCRIPTION("Intel OPA Virtual Network driver");
-MODULE_VERSION(DRV_VERSION);
diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c
index 2354c742caa1..fa5ccdb3bb2a 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.c
+++ b/drivers/infiniband/ulp/srp/ib_srp.c
@@ -62,7 +62,6 @@
 MODULE_AUTHOR("Roland Dreier");
 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator");
 MODULE_LICENSE("Dual BSD/GPL");
-MODULE_VERSION(DRV_VERSION);
 MODULE_INFO(release_date, DRV_RELDATE);
 
 #if !defined(CONFIG_DYNAMIC_DEBUG)
diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c
index 402275be0931..9e8e9220f816 100644
--- a/drivers/infiniband/ulp/srpt/ib_srpt.c
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.c
@@ -2238,7 +2238,7 @@ static int srpt_write_pending(struct se_cmd *se_cmd)
 				cqe, first_wr);
 		cqe = NULL;
 	}
-	
+
 	ret = ib_post_send(ch->qp, first_wr, &bad_wr);
 	if (ret) {
 		pr_err("%s: ib_post_send() returned %d for %d (avail: %d)\n",
@@ -2530,8 +2530,7 @@ static void srpt_add_one(struct ib_device *device)
 
 	INIT_IB_EVENT_HANDLER(&sdev->event_handler, sdev->device,
 			      srpt_event_handler);
-	if (ib_register_event_handler(&sdev->event_handler))
-		goto err_cm;
+	ib_register_event_handler(&sdev->event_handler);
 
 	sdev->ioctx_ring = (struct srpt_recv_ioctx **)
 		srpt_alloc_ioctx_ring(sdev, sdev->srq_size,
diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.h b/drivers/infiniband/ulp/srpt/ib_srpt.h
index cc1183851af5..1b817e51b84b 100644
--- a/drivers/infiniband/ulp/srpt/ib_srpt.h
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.h
@@ -328,8 +328,8 @@ struct srpt_port {
 	u8			port_guid[24];
 	u8			port_gid[64];
 	u8			port;
-	u16			sm_lid;
-	u16			lid;
+	u32			sm_lid;
+	u32			lid;
 	union ib_gid		gid;
 	struct work_struct	work;
 	struct se_portal_group	port_guid_tpg;
diff --git a/drivers/input/input.c b/drivers/input/input.c
index 7e6842bd525c..d268fdc23c64 100644
--- a/drivers/input/input.c
+++ b/drivers/input/input.c
@@ -1398,7 +1398,7 @@ static struct attribute *input_dev_attrs[] = {
 	NULL
 };
 
-static struct attribute_group input_dev_attr_group = {
+static const struct attribute_group input_dev_attr_group = {
 	.attrs	= input_dev_attrs,
 };
 
@@ -1425,7 +1425,7 @@ static struct attribute *input_dev_id_attrs[] = {
 	NULL
 };
 
-static struct attribute_group input_dev_id_attr_group = {
+static const struct attribute_group input_dev_id_attr_group = {
 	.name	= "id",
 	.attrs	= input_dev_id_attrs,
 };
@@ -1495,7 +1495,7 @@ static struct attribute *input_dev_caps_attrs[] = {
 	NULL
 };
 
-static struct attribute_group input_dev_caps_attr_group = {
+static const struct attribute_group input_dev_caps_attr_group = {
 	.name	= "capabilities",
 	.attrs	= input_dev_caps_attrs,
 };
diff --git a/drivers/input/joystick/adi.c b/drivers/input/joystick/adi.c
index d09cefa37931..15a71acb6997 100644
--- a/drivers/input/joystick/adi.c
+++ b/drivers/input/joystick/adi.c
@@ -313,7 +313,7 @@ static void adi_close(struct input_dev *dev)
 
 static void adi_init_digital(struct gameport *gameport)
 {
-	int seq[] = { 4, -2, -3, 10, -6, -11, -7, -9, 11, 0 };
+	static const int seq[] = { 4, -2, -3, 10, -6, -11, -7, -9, 11, 0 };
 	int i;
 
 	for (i = 0; seq[i]; i++) {
diff --git a/drivers/input/joystick/iforce/iforce-serio.c b/drivers/input/joystick/iforce/iforce-serio.c
index 46d5041d2d9d..154e827b559b 100644
--- a/drivers/input/joystick/iforce/iforce-serio.c
+++ b/drivers/input/joystick/iforce/iforce-serio.c
@@ -164,7 +164,7 @@ static void iforce_serio_disconnect(struct serio *serio)
 	kfree(iforce);
 }
 
-static struct serio_device_id iforce_serio_ids[] = {
+static const struct serio_device_id iforce_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_IFORCE,
diff --git a/drivers/input/joystick/iforce/iforce-usb.c b/drivers/input/joystick/iforce/iforce-usb.c
index db64adfbe1af..e8724f1a4a25 100644
--- a/drivers/input/joystick/iforce/iforce-usb.c
+++ b/drivers/input/joystick/iforce/iforce-usb.c
@@ -209,7 +209,7 @@ static void iforce_usb_disconnect(struct usb_interface *intf)
 	kfree(iforce);
 }
 
-static struct usb_device_id iforce_usb_ids [] = {
+static const struct usb_device_id iforce_usb_ids[] = {
 	{ USB_DEVICE(0x044f, 0xa01c) },		/* Thrustmaster Motor Sport GT */
 	{ USB_DEVICE(0x046d, 0xc281) },		/* Logitech WingMan Force */
 	{ USB_DEVICE(0x046d, 0xc291) },		/* Logitech WingMan Formula Force */
diff --git a/drivers/input/joystick/magellan.c b/drivers/input/joystick/magellan.c
index c5358ba1f571..a9d0e3edca94 100644
--- a/drivers/input/joystick/magellan.c
+++ b/drivers/input/joystick/magellan.c
@@ -198,7 +198,7 @@ static int magellan_connect(struct serio *serio, struct serio_driver *drv)
  * The serio driver structure.
  */
 
-static struct serio_device_id magellan_serio_ids[] = {
+static const struct serio_device_id magellan_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_MAGELLAN,
diff --git a/drivers/input/joystick/spaceball.c b/drivers/input/joystick/spaceball.c
index f4445a4e8d6a..e9712a1b7cad 100644
--- a/drivers/input/joystick/spaceball.c
+++ b/drivers/input/joystick/spaceball.c
@@ -272,7 +272,7 @@ static int spaceball_connect(struct serio *serio, struct serio_driver *drv)
  * The serio driver structure.
  */
 
-static struct serio_device_id spaceball_serio_ids[] = {
+static const struct serio_device_id spaceball_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_SPACEBALL,
diff --git a/drivers/input/joystick/spaceorb.c b/drivers/input/joystick/spaceorb.c
index f2667820e8c5..05da0ed514e2 100644
--- a/drivers/input/joystick/spaceorb.c
+++ b/drivers/input/joystick/spaceorb.c
@@ -213,7 +213,7 @@ static int spaceorb_connect(struct serio *serio, struct serio_driver *drv)
  * The serio driver structure.
  */
 
-static struct serio_device_id spaceorb_serio_ids[] = {
+static const struct serio_device_id spaceorb_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_SPACEORB,
diff --git a/drivers/input/joystick/stinger.c b/drivers/input/joystick/stinger.c
index 099c6d7b5e08..cb10e7b097ae 100644
--- a/drivers/input/joystick/stinger.c
+++ b/drivers/input/joystick/stinger.c
@@ -184,7 +184,7 @@ static int stinger_connect(struct serio *serio, struct serio_driver *drv)
  * The serio driver structure.
  */
 
-static struct serio_device_id stinger_serio_ids[] = {
+static const struct serio_device_id stinger_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_STINGER,
diff --git a/drivers/input/joystick/twidjoy.c b/drivers/input/joystick/twidjoy.c
index 7f7e5ab3f9e3..e60cb004cb8c 100644
--- a/drivers/input/joystick/twidjoy.c
+++ b/drivers/input/joystick/twidjoy.c
@@ -233,7 +233,7 @@ static int twidjoy_connect(struct serio *serio, struct serio_driver *drv)
  * The serio driver structure.
  */
 
-static struct serio_device_id twidjoy_serio_ids[] = {
+static const struct serio_device_id twidjoy_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_TWIDJOY,
diff --git a/drivers/input/joystick/warrior.c b/drivers/input/joystick/warrior.c
index e13a9144a25d..ef5391ba4470 100644
--- a/drivers/input/joystick/warrior.c
+++ b/drivers/input/joystick/warrior.c
@@ -193,7 +193,7 @@ static int warrior_connect(struct serio *serio, struct serio_driver *drv)
  * The serio driver structure.
  */
 
-static struct serio_device_id warrior_serio_ids[] = {
+static const struct serio_device_id warrior_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_WARRIOR,
diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c
index 298a6ba51411..d86e59515b9c 100644
--- a/drivers/input/joystick/xpad.c
+++ b/drivers/input/joystick/xpad.c
@@ -408,7 +408,7 @@ static const signed short xpad_abs_triggers[] = {
 #define XPAD_XBOXONE_VENDOR(vend) \
 	{ XPAD_XBOXONE_VENDOR_PROTOCOL(vend, 208) }
 
-static struct usb_device_id xpad_table[] = {
+static const struct usb_device_id xpad_table[] = {
 	{ USB_INTERFACE_INFO('X', 'B', 0) },	/* X-Box USB-IF not approved class */
 	XPAD_XBOX360_VENDOR(0x044f),		/* Thrustmaster X-Box 360 controllers */
 	XPAD_XBOX360_VENDOR(0x045e),		/* Microsoft X-Box 360 controllers */
@@ -476,10 +476,21 @@ static const u8 xboxone_hori_init[] = {
 };
 
 /*
- * A rumble packet is required for some PowerA pads to start
+ * A specific rumble packet is required for some PowerA pads to start
  * sending input reports. One of those pads is (0x24c6:0x543a).
  */
-static const u8 xboxone_zerorumble_init[] = {
+static const u8 xboxone_rumblebegin_init[] = {
+	0x09, 0x00, 0x00, 0x09, 0x00, 0x0F, 0x00, 0x00,
+	0x1D, 0x1D, 0xFF, 0x00, 0x00
+};
+
+/*
+ * A rumble packet with zero FF intensity will immediately
+ * terminate the rumbling required to init PowerA pads.
+ * This should happen fast enough that the motors don't
+ * spin up to enough speed to actually vibrate the gamepad.
+ */
+static const u8 xboxone_rumbleend_init[] = {
 	0x09, 0x00, 0x00, 0x09, 0x00, 0x0F, 0x00, 0x00,
 	0x00, 0x00, 0x00, 0x00, 0x00
 };
@@ -494,9 +505,12 @@ static const struct xboxone_init_packet xboxone_init_packets[] = {
 	XBOXONE_INIT_PKT(0x0e6f, 0x0165, xboxone_hori_init),
 	XBOXONE_INIT_PKT(0x0f0d, 0x0067, xboxone_hori_init),
 	XBOXONE_INIT_PKT(0x0000, 0x0000, xboxone_fw2015_init),
-	XBOXONE_INIT_PKT(0x24c6, 0x541a, xboxone_zerorumble_init),
-	XBOXONE_INIT_PKT(0x24c6, 0x542a, xboxone_zerorumble_init),
-	XBOXONE_INIT_PKT(0x24c6, 0x543a, xboxone_zerorumble_init),
+	XBOXONE_INIT_PKT(0x24c6, 0x541a, xboxone_rumblebegin_init),
+	XBOXONE_INIT_PKT(0x24c6, 0x542a, xboxone_rumblebegin_init),
+	XBOXONE_INIT_PKT(0x24c6, 0x543a, xboxone_rumblebegin_init),
+	XBOXONE_INIT_PKT(0x24c6, 0x541a, xboxone_rumbleend_init),
+	XBOXONE_INIT_PKT(0x24c6, 0x542a, xboxone_rumbleend_init),
+	XBOXONE_INIT_PKT(0x24c6, 0x543a, xboxone_rumbleend_init),
 };
 
 struct xpad_output_packet {
@@ -1750,10 +1764,12 @@ static int xpad_probe(struct usb_interface *intf, const struct usb_device_id *id
 		struct usb_endpoint_descriptor *ep =
 				&intf->cur_altsetting->endpoint[i].desc;
 
-		if (usb_endpoint_dir_in(ep))
-			ep_irq_in = ep;
-		else
-			ep_irq_out = ep;
+		if (usb_endpoint_xfer_int(ep)) {
+			if (usb_endpoint_dir_in(ep))
+				ep_irq_in = ep;
+			else
+				ep_irq_out = ep;
+		}
 	}
 
 	if (!ep_irq_in || !ep_irq_out) {
diff --git a/drivers/input/joystick/zhenhua.c b/drivers/input/joystick/zhenhua.c
index 4a8258bf13fd..5c6d5de743f1 100644
--- a/drivers/input/joystick/zhenhua.c
+++ b/drivers/input/joystick/zhenhua.c
@@ -192,7 +192,7 @@ static int zhenhua_connect(struct serio *serio, struct serio_driver *drv)
  * The serio driver structure.
  */
 
-static struct serio_device_id zhenhua_serio_ids[] = {
+static const struct serio_device_id zhenhua_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_ZHENHUA,
diff --git a/drivers/input/keyboard/atkbd.c b/drivers/input/keyboard/atkbd.c
index ec876b5b1382..7e75835e220f 100644
--- a/drivers/input/keyboard/atkbd.c
+++ b/drivers/input/keyboard/atkbd.c
@@ -1270,7 +1270,7 @@ static int atkbd_reconnect(struct serio *serio)
 	return retval;
 }
 
-static struct serio_device_id atkbd_serio_ids[] = {
+static const struct serio_device_id atkbd_serio_ids[] = {
 	{
 		.type	= SERIO_8042,
 		.proto	= SERIO_ANY,
diff --git a/drivers/input/keyboard/gpio_keys.c b/drivers/input/keyboard/gpio_keys.c
index a047b9af8369..e9f0ebf3267a 100644
--- a/drivers/input/keyboard/gpio_keys.c
+++ b/drivers/input/keyboard/gpio_keys.c
@@ -353,7 +353,7 @@ static struct attribute *gpio_keys_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group gpio_keys_attr_group = {
+static const struct attribute_group gpio_keys_attr_group = {
 	.attrs = gpio_keys_attrs,
 };
 
@@ -827,7 +827,7 @@ static int gpio_keys_probe(struct platform_device *pdev)
 
 	fwnode_handle_put(child);
 
-	error = sysfs_create_group(&dev->kobj, &gpio_keys_attr_group);
+	error = devm_device_add_group(dev, &gpio_keys_attr_group);
 	if (error) {
 		dev_err(dev, "Unable to export keys/switches, error: %d\n",
 			error);
@@ -838,23 +838,12 @@ static int gpio_keys_probe(struct platform_device *pdev)
 	if (error) {
 		dev_err(dev, "Unable to register input device, error: %d\n",
 			error);
-		goto err_remove_group;
+		return error;
 	}
 
 	device_init_wakeup(dev, wakeup);
 
 	return 0;
-
-err_remove_group:
-	sysfs_remove_group(&dev->kobj, &gpio_keys_attr_group);
-	return error;
-}
-
-static int gpio_keys_remove(struct platform_device *pdev)
-{
-	sysfs_remove_group(&pdev->dev.kobj, &gpio_keys_attr_group);
-
-	return 0;
 }
 
 static int __maybe_unused gpio_keys_suspend(struct device *dev)
@@ -912,7 +901,6 @@ static SIMPLE_DEV_PM_OPS(gpio_keys_pm_ops, gpio_keys_suspend, gpio_keys_resume);
 
 static struct platform_driver gpio_keys_device_driver = {
 	.probe		= gpio_keys_probe,
-	.remove		= gpio_keys_remove,
 	.driver		= {
 		.name	= "gpio-keys",
 		.pm	= &gpio_keys_pm_ops,
diff --git a/drivers/input/keyboard/hil_kbd.c b/drivers/input/keyboard/hil_kbd.c
index 5b152f25a8e1..bb29a7c9a1c0 100644
--- a/drivers/input/keyboard/hil_kbd.c
+++ b/drivers/input/keyboard/hil_kbd.c
@@ -559,7 +559,7 @@ static int hil_dev_connect(struct serio *serio, struct serio_driver *drv)
 	return error;
 }
 
-static struct serio_device_id hil_dev_ids[] = {
+static const struct serio_device_id hil_dev_ids[] = {
 	{
 		.type = SERIO_HIL_MLC,
 		.proto = SERIO_HIL,
diff --git a/drivers/input/keyboard/hilkbd.c b/drivers/input/keyboard/hilkbd.c
index 198dc07a1be5..a4e404aaf64b 100644
--- a/drivers/input/keyboard/hilkbd.c
+++ b/drivers/input/keyboard/hilkbd.c
@@ -299,7 +299,7 @@ static void hil_keyb_exit(void)
 }
 
 #if defined(CONFIG_PARISC)
-static int hil_probe_chip(struct parisc_device *dev)
+static int __init hil_probe_chip(struct parisc_device *dev)
 {
 	/* Only allow one HIL keyboard */
 	if (hil_dev.dev)
@@ -320,14 +320,14 @@ static int hil_probe_chip(struct parisc_device *dev)
 	return hil_keyb_init();
 }
 
-static int hil_remove_chip(struct parisc_device *dev)
+static int __exit hil_remove_chip(struct parisc_device *dev)
 {
 	hil_keyb_exit();
 
 	return 0;
 }
 
-static struct parisc_device_id hil_tbl[] = {
+static const struct parisc_device_id hil_tbl[] __initconst = {
 	{ HPHW_FIO, HVERSION_REV_ANY_ID, HVERSION_ANY_ID, 0x00073 },
 	{ 0, }
 };
@@ -337,11 +337,11 @@ static struct parisc_device_id hil_tbl[] = {
 MODULE_DEVICE_TABLE(parisc, hil_tbl);
 #endif
 
-static struct parisc_driver hil_driver = {
+static struct parisc_driver hil_driver __refdata = {
 	.name		= "hil",
 	.id_table	= hil_tbl,
 	.probe		= hil_probe_chip,
-	.remove		= hil_remove_chip,
+	.remove		= __exit_p(hil_remove_chip),
 };
 
 static int __init hil_init(void)
diff --git a/drivers/input/keyboard/lkkbd.c b/drivers/input/keyboard/lkkbd.c
index 9fcd9f1d5dc8..471d53815c6d 100644
--- a/drivers/input/keyboard/lkkbd.c
+++ b/drivers/input/keyboard/lkkbd.c
@@ -707,7 +707,7 @@ static void lkkbd_disconnect(struct serio *serio)
 	kfree(lk);
 }
 
-static struct serio_device_id lkkbd_serio_ids[] = {
+static const struct serio_device_id lkkbd_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_LKKBD,
diff --git a/drivers/input/keyboard/newtonkbd.c b/drivers/input/keyboard/newtonkbd.c
index 20f044377990..fb9b8e23ab93 100644
--- a/drivers/input/keyboard/newtonkbd.c
+++ b/drivers/input/keyboard/newtonkbd.c
@@ -142,7 +142,7 @@ static void nkbd_disconnect(struct serio *serio)
 	kfree(nkbd);
 }
 
-static struct serio_device_id nkbd_serio_ids[] = {
+static const struct serio_device_id nkbd_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_NEWTON,
diff --git a/drivers/input/keyboard/pxa27x_keypad.c b/drivers/input/keyboard/pxa27x_keypad.c
index 3841fa30db33..d0bdaeadf86d 100644
--- a/drivers/input/keyboard/pxa27x_keypad.c
+++ b/drivers/input/keyboard/pxa27x_keypad.c
@@ -644,9 +644,12 @@ static void pxa27x_keypad_config(struct pxa27x_keypad *keypad)
 static int pxa27x_keypad_open(struct input_dev *dev)
 {
 	struct pxa27x_keypad *keypad = input_get_drvdata(dev);
-
+	int ret;
 	/* Enable unit clock */
-	clk_prepare_enable(keypad->clk);
+	ret = clk_prepare_enable(keypad->clk);
+	if (ret)
+		return ret;
+
 	pxa27x_keypad_config(keypad);
 
 	return 0;
@@ -683,6 +686,7 @@ static int pxa27x_keypad_resume(struct device *dev)
 	struct platform_device *pdev = to_platform_device(dev);
 	struct pxa27x_keypad *keypad = platform_get_drvdata(pdev);
 	struct input_dev *input_dev = keypad->input_dev;
+	int ret = 0;
 
 	/*
 	 * If the keypad is used as wake up source, the clock is not turned
@@ -695,14 +699,15 @@ static int pxa27x_keypad_resume(struct device *dev)
 
 		if (input_dev->users) {
 			/* Enable unit clock */
-			clk_prepare_enable(keypad->clk);
-			pxa27x_keypad_config(keypad);
+			ret = clk_prepare_enable(keypad->clk);
+			if (!ret)
+				pxa27x_keypad_config(keypad);
 		}
 
 		mutex_unlock(&input_dev->mutex);
 	}
 
-	return 0;
+	return ret;
 }
 #endif
 
diff --git a/drivers/input/keyboard/stowaway.c b/drivers/input/keyboard/stowaway.c
index a6e0d565e306..8b6de9a692dc 100644
--- a/drivers/input/keyboard/stowaway.c
+++ b/drivers/input/keyboard/stowaway.c
@@ -146,7 +146,7 @@ static void skbd_disconnect(struct serio *serio)
 	kfree(skbd);
 }
 
-static struct serio_device_id skbd_serio_ids[] = {
+static const struct serio_device_id skbd_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_STOWAWAY,
diff --git a/drivers/input/keyboard/sunkbd.c b/drivers/input/keyboard/sunkbd.c
index dc6bb9d5b4f0..c95707ea2656 100644
--- a/drivers/input/keyboard/sunkbd.c
+++ b/drivers/input/keyboard/sunkbd.c
@@ -339,7 +339,7 @@ static void sunkbd_disconnect(struct serio *serio)
 	kfree(sunkbd);
 }
 
-static struct serio_device_id sunkbd_serio_ids[] = {
+static const struct serio_device_id sunkbd_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_SUNKBD,
diff --git a/drivers/input/keyboard/tegra-kbc.c b/drivers/input/keyboard/tegra-kbc.c
index 0c07e1023a46..edc1385ca00b 100644
--- a/drivers/input/keyboard/tegra-kbc.c
+++ b/drivers/input/keyboard/tegra-kbc.c
@@ -370,8 +370,11 @@ static int tegra_kbc_start(struct tegra_kbc *kbc)
 {
 	unsigned int debounce_cnt;
 	u32 val = 0;
+	int ret;
 
-	clk_prepare_enable(kbc->clk);
+	ret = clk_prepare_enable(kbc->clk);
+	if (ret)
+		return ret;
 
 	/* Reset the KBC controller to clear all previous status.*/
 	reset_control_assert(kbc->rst);
diff --git a/drivers/input/keyboard/twl4030_keypad.c b/drivers/input/keyboard/twl4030_keypad.c
index 39e72b3219d8..f9f98ef1d98e 100644
--- a/drivers/input/keyboard/twl4030_keypad.c
+++ b/drivers/input/keyboard/twl4030_keypad.c
@@ -30,7 +30,7 @@
 #include <linux/interrupt.h>
 #include <linux/input.h>
 #include <linux/platform_device.h>
-#include <linux/i2c/twl.h>
+#include <linux/mfd/twl.h>
 #include <linux/slab.h>
 #include <linux/of.h>
 
diff --git a/drivers/input/keyboard/xtkbd.c b/drivers/input/keyboard/xtkbd.c
index 7c2325bd7408..8f64b9ded8d0 100644
--- a/drivers/input/keyboard/xtkbd.c
+++ b/drivers/input/keyboard/xtkbd.c
@@ -145,7 +145,7 @@ static void xtkbd_disconnect(struct serio *serio)
 	kfree(xtkbd);
 }
 
-static struct serio_device_id xtkbd_serio_ids[] = {
+static const struct serio_device_id xtkbd_serio_ids[] = {
 	{
 		.type	= SERIO_XT,
 		.proto	= SERIO_ANY,
diff --git a/drivers/input/misc/Kconfig b/drivers/input/misc/Kconfig
index 3872488c3fd7..9f082a388388 100644
--- a/drivers/input/misc/Kconfig
+++ b/drivers/input/misc/Kconfig
@@ -581,6 +581,29 @@ config INPUT_PWM_BEEPER
 	  To compile this driver as a module, choose M here: the module will be
 	  called pwm-beeper.
 
+config INPUT_PWM_VIBRA
+	tristate "PWM vibrator support"
+	depends on PWM
+	select INPUT_FF_MEMLESS
+	help
+	  Say Y here to get support for PWM based vibrator devices.
+
+	  If unsure, say N.
+
+	  To compile this driver as a module, choose M here: the module will be
+	  called pwm-vibra.
+
+config INPUT_RK805_PWRKEY
+	tristate "Rockchip RK805 PMIC power key support"
+	depends on MFD_RK808
+	help
+	  Select this option to enable power key driver for RK805.
+
+	  If unsure, say N.
+
+	  To compile this driver as a module, choose M here: the module will be
+	  called rk805_pwrkey.
+
 config INPUT_GPIO_ROTARY_ENCODER
 	tristate "Rotary encoders connected to GPIO pins"
 	depends on GPIOLIB || COMPILE_TEST
diff --git a/drivers/input/misc/Makefile b/drivers/input/misc/Makefile
index b923a9828c88..03fd4262ada9 100644
--- a/drivers/input/misc/Makefile
+++ b/drivers/input/misc/Makefile
@@ -59,11 +59,13 @@ obj-$(CONFIG_INPUT_PM8XXX_VIBRATOR)	+= pm8xxx-vibrator.o
 obj-$(CONFIG_INPUT_PMIC8XXX_PWRKEY)	+= pmic8xxx-pwrkey.o
 obj-$(CONFIG_INPUT_POWERMATE)		+= powermate.o
 obj-$(CONFIG_INPUT_PWM_BEEPER)		+= pwm-beeper.o
+obj-$(CONFIG_INPUT_PWM_VIBRA)		+= pwm-vibra.o
 obj-$(CONFIG_INPUT_RB532_BUTTON)	+= rb532_button.o
 obj-$(CONFIG_INPUT_REGULATOR_HAPTIC)	+= regulator-haptic.o
 obj-$(CONFIG_INPUT_RETU_PWRBUTTON)	+= retu-pwrbutton.o
 obj-$(CONFIG_INPUT_AXP20X_PEK)		+= axp20x-pek.o
 obj-$(CONFIG_INPUT_GPIO_ROTARY_ENCODER)	+= rotary_encoder.o
+obj-$(CONFIG_INPUT_RK805_PWRKEY)	+= rk805-pwrkey.o
 obj-$(CONFIG_INPUT_SGI_BTNS)		+= sgi_btns.o
 obj-$(CONFIG_INPUT_SIRFSOC_ONKEY)	+= sirfsoc-onkey.o
 obj-$(CONFIG_INPUT_SOC_BUTTON_ARRAY)	+= soc_button_array.o
diff --git a/drivers/input/misc/ati_remote2.c b/drivers/input/misc/ati_remote2.c
index 1c5914cae853..ebf4448b31b9 100644
--- a/drivers/input/misc/ati_remote2.c
+++ b/drivers/input/misc/ati_remote2.c
@@ -110,7 +110,7 @@ static const struct kernel_param_ops param_ops_mode_mask = {
 module_param(mode_mask, mode_mask, 0644);
 MODULE_PARM_DESC(mode_mask, "Bitmask of modes to accept <4:PC><3:AUX4><2:AUX3><1:AUX2><0:AUX1>");
 
-static struct usb_device_id ati_remote2_id_table[] = {
+static const struct usb_device_id ati_remote2_id_table[] = {
 	{ USB_DEVICE(0x0471, 0x0602) },	/* ATI Remote Wonder II */
 	{ }
 };
diff --git a/drivers/input/misc/axp20x-pek.c b/drivers/input/misc/axp20x-pek.c
index 38c79ebff033..6cee5adc3b5c 100644
--- a/drivers/input/misc/axp20x-pek.c
+++ b/drivers/input/misc/axp20x-pek.c
@@ -29,9 +29,17 @@
 #define AXP20X_PEK_STARTUP_MASK		(0xc0)
 #define AXP20X_PEK_SHUTDOWN_MASK	(0x03)
 
+struct axp20x_info {
+	const struct axp20x_time *startup_time;
+	unsigned int startup_mask;
+	const struct axp20x_time *shutdown_time;
+	unsigned int shutdown_mask;
+};
+
 struct axp20x_pek {
 	struct axp20x_dev *axp20x;
 	struct input_dev *input;
+	struct axp20x_info *info;
 	int irq_dbr;
 	int irq_dbf;
 };
@@ -48,6 +56,13 @@ static const struct axp20x_time startup_time[] = {
 	{ .time = 2000, .idx = 3 },
 };
 
+static const struct axp20x_time axp221_startup_time[] = {
+	{ .time = 128,  .idx = 0 },
+	{ .time = 1000, .idx = 1 },
+	{ .time = 2000, .idx = 2 },
+	{ .time = 3000, .idx = 3 },
+};
+
 static const struct axp20x_time shutdown_time[] = {
 	{ .time = 4000,  .idx = 0 },
 	{ .time = 6000,  .idx = 1 },
@@ -55,31 +70,25 @@ static const struct axp20x_time shutdown_time[] = {
 	{ .time = 10000, .idx = 3 },
 };
 
-struct axp20x_pek_ext_attr {
-	const struct axp20x_time *p_time;
-	unsigned int mask;
-};
-
-static struct axp20x_pek_ext_attr axp20x_pek_startup_ext_attr = {
-	.p_time	= startup_time,
-	.mask	= AXP20X_PEK_STARTUP_MASK,
+static const struct axp20x_info axp20x_info = {
+	.startup_time = startup_time,
+	.startup_mask = AXP20X_PEK_STARTUP_MASK,
+	.shutdown_time = shutdown_time,
+	.shutdown_mask = AXP20X_PEK_SHUTDOWN_MASK,
 };
 
-static struct axp20x_pek_ext_attr axp20x_pek_shutdown_ext_attr = {
-	.p_time	= shutdown_time,
-	.mask	= AXP20X_PEK_SHUTDOWN_MASK,
+static const struct axp20x_info axp221_info = {
+	.startup_time = axp221_startup_time,
+	.startup_mask = AXP20X_PEK_STARTUP_MASK,
+	.shutdown_time = shutdown_time,
+	.shutdown_mask = AXP20X_PEK_SHUTDOWN_MASK,
 };
 
-static struct axp20x_pek_ext_attr *get_axp_ext_attr(struct device_attribute *attr)
-{
-	return container_of(attr, struct dev_ext_attribute, attr)->var;
-}
-
-static ssize_t axp20x_show_ext_attr(struct device *dev,
-				    struct device_attribute *attr, char *buf)
+static ssize_t axp20x_show_attr(struct device *dev,
+				const struct axp20x_time *time,
+				unsigned int mask, char *buf)
 {
 	struct axp20x_pek *axp20x_pek = dev_get_drvdata(dev);
-	struct axp20x_pek_ext_attr *axp20x_ea = get_axp_ext_attr(attr);
 	unsigned int val;
 	int ret, i;
 
@@ -87,22 +96,42 @@ static ssize_t axp20x_show_ext_attr(struct device *dev,
 	if (ret != 0)
 		return ret;
 
-	val &= axp20x_ea->mask;
-	val >>= ffs(axp20x_ea->mask) - 1;
+	val &= mask;
+	val >>= ffs(mask) - 1;
 
 	for (i = 0; i < 4; i++)
-		if (val == axp20x_ea->p_time[i].idx)
-			val = axp20x_ea->p_time[i].time;
+		if (val == time[i].idx)
+			val = time[i].time;
 
 	return sprintf(buf, "%u\n", val);
 }
 
-static ssize_t axp20x_store_ext_attr(struct device *dev,
-				     struct device_attribute *attr,
-				     const char *buf, size_t count)
+static ssize_t axp20x_show_attr_startup(struct device *dev,
+					struct device_attribute *attr,
+					char *buf)
+{
+	struct axp20x_pek *axp20x_pek = dev_get_drvdata(dev);
+
+	return axp20x_show_attr(dev, axp20x_pek->info->startup_time,
+				axp20x_pek->info->startup_mask, buf);
+}
+
+static ssize_t axp20x_show_attr_shutdown(struct device *dev,
+					 struct device_attribute *attr,
+					 char *buf)
+{
+	struct axp20x_pek *axp20x_pek = dev_get_drvdata(dev);
+
+	return axp20x_show_attr(dev, axp20x_pek->info->shutdown_time,
+				axp20x_pek->info->shutdown_mask, buf);
+}
+
+static ssize_t axp20x_store_attr(struct device *dev,
+				 const struct axp20x_time *time,
+				 unsigned int mask, const char *buf,
+				 size_t count)
 {
 	struct axp20x_pek *axp20x_pek = dev_get_drvdata(dev);
-	struct axp20x_pek_ext_attr *axp20x_ea = get_axp_ext_attr(attr);
 	char val_str[20];
 	size_t len;
 	int ret, i;
@@ -123,39 +152,52 @@ static ssize_t axp20x_store_ext_attr(struct device *dev,
 	for (i = 3; i >= 0; i--) {
 		unsigned int err;
 
-		err = abs(axp20x_ea->p_time[i].time - val);
+		err = abs(time[i].time - val);
 		if (err < best_err) {
 			best_err = err;
-			idx = axp20x_ea->p_time[i].idx;
+			idx = time[i].idx;
 		}
 
 		if (!err)
 			break;
 	}
 
-	idx <<= ffs(axp20x_ea->mask) - 1;
-	ret = regmap_update_bits(axp20x_pek->axp20x->regmap,
-				 AXP20X_PEK_KEY,
-				 axp20x_ea->mask, idx);
+	idx <<= ffs(mask) - 1;
+	ret = regmap_update_bits(axp20x_pek->axp20x->regmap, AXP20X_PEK_KEY,
+				 mask, idx);
 	if (ret != 0)
 		return -EINVAL;
 
 	return count;
 }
 
-static struct dev_ext_attribute axp20x_dev_attr_startup = {
-	.attr	= __ATTR(startup, 0644, axp20x_show_ext_attr, axp20x_store_ext_attr),
-	.var	= &axp20x_pek_startup_ext_attr,
-};
+static ssize_t axp20x_store_attr_startup(struct device *dev,
+					 struct device_attribute *attr,
+					 const char *buf, size_t count)
+{
+	struct axp20x_pek *axp20x_pek = dev_get_drvdata(dev);
 
-static struct dev_ext_attribute axp20x_dev_attr_shutdown = {
-	.attr	= __ATTR(shutdown, 0644, axp20x_show_ext_attr, axp20x_store_ext_attr),
-	.var	= &axp20x_pek_shutdown_ext_attr,
-};
+	return axp20x_store_attr(dev, axp20x_pek->info->startup_time,
+				 axp20x_pek->info->startup_mask, buf, count);
+}
+
+static ssize_t axp20x_store_attr_shutdown(struct device *dev,
+					  struct device_attribute *attr,
+					  const char *buf, size_t count)
+{
+	struct axp20x_pek *axp20x_pek = dev_get_drvdata(dev);
+
+	return axp20x_store_attr(dev, axp20x_pek->info->shutdown_time,
+				 axp20x_pek->info->shutdown_mask, buf, count);
+}
+
+DEVICE_ATTR(startup, 0644, axp20x_show_attr_startup, axp20x_store_attr_startup);
+DEVICE_ATTR(shutdown, 0644, axp20x_show_attr_shutdown,
+	    axp20x_store_attr_shutdown);
 
 static struct attribute *axp20x_attributes[] = {
-	&axp20x_dev_attr_startup.attr.attr,
-	&axp20x_dev_attr_shutdown.attr.attr,
+	&dev_attr_startup.attr,
+	&dev_attr_shutdown.attr,
 	NULL,
 };
 
@@ -182,13 +224,6 @@ static irqreturn_t axp20x_pek_irq(int irq, void *pwr)
 	return IRQ_HANDLED;
 }
 
-static void axp20x_remove_sysfs_group(void *_data)
-{
-	struct device *dev = _data;
-
-	sysfs_remove_group(&dev->kobj, &axp20x_attribute_group);
-}
-
 static int axp20x_pek_probe_input_device(struct axp20x_pek *axp20x_pek,
 					 struct platform_device *pdev)
 {
@@ -298,8 +333,14 @@ static bool axp20x_pek_should_register_input(struct axp20x_pek *axp20x_pek,
 static int axp20x_pek_probe(struct platform_device *pdev)
 {
 	struct axp20x_pek *axp20x_pek;
+	const struct platform_device_id *match = platform_get_device_id(pdev);
 	int error;
 
+	if (!match) {
+		dev_err(&pdev->dev, "Failed to get platform_device_id\n");
+		return -EINVAL;
+	}
+
 	axp20x_pek = devm_kzalloc(&pdev->dev, sizeof(struct axp20x_pek),
 				  GFP_KERNEL);
 	if (!axp20x_pek)
@@ -313,18 +354,11 @@ static int axp20x_pek_probe(struct platform_device *pdev)
 			return error;
 	}
 
-	error = sysfs_create_group(&pdev->dev.kobj, &axp20x_attribute_group);
-	if (error) {
-		dev_err(&pdev->dev, "Failed to create sysfs attributes: %d\n",
-			error);
-		return error;
-	}
+	axp20x_pek->info = (struct axp20x_info *)match->driver_data;
 
-	error = devm_add_action(&pdev->dev,
-				axp20x_remove_sysfs_group, &pdev->dev);
+	error = devm_device_add_group(&pdev->dev, &axp20x_attribute_group);
 	if (error) {
-		axp20x_remove_sysfs_group(&pdev->dev);
-		dev_err(&pdev->dev, "Failed to add sysfs cleanup action: %d\n",
+		dev_err(&pdev->dev, "Failed to create sysfs attributes: %d\n",
 			error);
 		return error;
 	}
@@ -358,8 +392,21 @@ static const struct dev_pm_ops axp20x_pek_pm_ops = {
 #endif
 };
 
+static const struct platform_device_id axp_pek_id_match[] = {
+	{
+		.name = "axp20x-pek",
+		.driver_data = (kernel_ulong_t)&axp20x_info,
+	},
+	{
+		.name = "axp221-pek",
+		.driver_data = (kernel_ulong_t)&axp221_info,
+	},
+	{ /* sentinel */ }
+};
+
 static struct platform_driver axp20x_pek_driver = {
 	.probe		= axp20x_pek_probe,
+	.id_table	= axp_pek_id_match,
 	.driver		= {
 		.name		= "axp20x-pek",
 		.pm		= &axp20x_pek_pm_ops,
diff --git a/drivers/input/misc/dm355evm_keys.c b/drivers/input/misc/dm355evm_keys.c
index bab256ef32b9..c803db64a376 100644
--- a/drivers/input/misc/dm355evm_keys.c
+++ b/drivers/input/misc/dm355evm_keys.c
@@ -15,7 +15,7 @@
 #include <linux/platform_device.h>
 #include <linux/interrupt.h>
 
-#include <linux/i2c/dm355evm_msp.h>
+#include <linux/mfd/dm355evm_msp.h>
 #include <linux/module.h>
 
 
diff --git a/drivers/input/misc/ims-pcu.c b/drivers/input/misc/ims-pcu.c
index f4e8fbec6a94..6bf82ea8c918 100644
--- a/drivers/input/misc/ims-pcu.c
+++ b/drivers/input/misc/ims-pcu.c
@@ -1261,7 +1261,7 @@ static umode_t ims_pcu_is_attr_visible(struct kobject *kobj,
 	return mode;
 }
 
-static struct attribute_group ims_pcu_attr_group = {
+static const struct attribute_group ims_pcu_attr_group = {
 	.is_visible	= ims_pcu_is_attr_visible,
 	.attrs		= ims_pcu_attrs,
 };
@@ -1480,7 +1480,7 @@ static struct attribute *ims_pcu_ofn_attrs[] = {
 	NULL
 };
 
-static struct attribute_group ims_pcu_ofn_attr_group = {
+static const struct attribute_group ims_pcu_ofn_attr_group = {
 	.name	= "ofn",
 	.attrs	= ims_pcu_ofn_attrs,
 };
diff --git a/drivers/input/misc/keyspan_remote.c b/drivers/input/misc/keyspan_remote.c
index a3fe4a990cc9..77c47d6325fe 100644
--- a/drivers/input/misc/keyspan_remote.c
+++ b/drivers/input/misc/keyspan_remote.c
@@ -85,7 +85,7 @@ static const unsigned short keyspan_key_table[] = {
 };
 
 /* table of devices that work with this driver */
-static struct usb_device_id keyspan_table[] = {
+static const struct usb_device_id keyspan_table[] = {
 	{ USB_DEVICE(USB_KEYSPAN_VENDOR_ID, USB_KEYSPAN_PRODUCT_UIA11) },
 	{ }					/* Terminating entry */
 };
diff --git a/drivers/input/misc/pcspkr.c b/drivers/input/misc/pcspkr.c
index 72b1fc3ab910..56ddba21de84 100644
--- a/drivers/input/misc/pcspkr.c
+++ b/drivers/input/misc/pcspkr.c
@@ -18,25 +18,30 @@
 #include <linux/input.h>
 #include <linux/platform_device.h>
 #include <linux/timex.h>
-#include <asm/io.h>
+#include <linux/io.h>
 
 MODULE_AUTHOR("Vojtech Pavlik <vojtech@ucw.cz>");
 MODULE_DESCRIPTION("PC Speaker beeper driver");
 MODULE_LICENSE("GPL");
 MODULE_ALIAS("platform:pcspkr");
 
-static int pcspkr_event(struct input_dev *dev, unsigned int type, unsigned int code, int value)
+static int pcspkr_event(struct input_dev *dev, unsigned int type,
+			unsigned int code, int value)
 {
 	unsigned int count = 0;
 	unsigned long flags;
 
 	if (type != EV_SND)
-		return -1;
+		return -EINVAL;
 
 	switch (code) {
-		case SND_BELL: if (value) value = 1000;
-		case SND_TONE: break;
-		default: return -1;
+	case SND_BELL:
+		if (value)
+			value = 1000;
+	case SND_TONE:
+		break;
+	default:
+		return -EINVAL;
 	}
 
 	if (value > 20 && value < 32767)
diff --git a/drivers/input/misc/powermate.c b/drivers/input/misc/powermate.c
index 84909a12ff36..5c8c79623c87 100644
--- a/drivers/input/misc/powermate.c
+++ b/drivers/input/misc/powermate.c
@@ -432,7 +432,7 @@ static void powermate_disconnect(struct usb_interface *intf)
 	}
 }
 
-static struct usb_device_id powermate_devices [] = {
+static const struct usb_device_id powermate_devices[] = {
 	{ USB_DEVICE(POWERMATE_VENDOR, POWERMATE_PRODUCT_NEW) },
 	{ USB_DEVICE(POWERMATE_VENDOR, POWERMATE_PRODUCT_OLD) },
 	{ USB_DEVICE(CONTOUR_VENDOR, CONTOUR_JOG) },
diff --git a/drivers/input/misc/pwm-vibra.c b/drivers/input/misc/pwm-vibra.c
new file mode 100644
index 000000000000..55da191ae550
--- /dev/null
+++ b/drivers/input/misc/pwm-vibra.c
@@ -0,0 +1,267 @@
+/*
+ *  PWM vibrator driver
+ *
+ *  Copyright (C) 2017 Collabora Ltd.
+ *
+ *  Based on previous work from:
+ *  Copyright (C) 2012 Dmitry Torokhov <dmitry.torokhov@gmail.com>
+ *
+ *  Based on PWM beeper driver:
+ *  Copyright (C) 2010, Lars-Peter Clausen <lars@metafoo.de>
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under  the terms of the GNU General  Public License as published by the
+ *  Free Software Foundation;  either version 2 of the License, or (at your
+ *  option) any later version.
+ */
+
+#include <linux/input.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
+#include <linux/pwm.h>
+#include <linux/regulator/consumer.h>
+#include <linux/slab.h>
+
+struct pwm_vibrator {
+	struct input_dev *input;
+	struct pwm_device *pwm;
+	struct pwm_device *pwm_dir;
+	struct regulator *vcc;
+
+	struct work_struct play_work;
+	u16 level;
+	u32 direction_duty_cycle;
+};
+
+static int pwm_vibrator_start(struct pwm_vibrator *vibrator)
+{
+	struct device *pdev = vibrator->input->dev.parent;
+	struct pwm_state state;
+	int err;
+
+	err = regulator_enable(vibrator->vcc);
+	if (err) {
+		dev_err(pdev, "failed to enable regulator: %d", err);
+		return err;
+	}
+
+	pwm_get_state(vibrator->pwm, &state);
+	pwm_set_relative_duty_cycle(&state, vibrator->level, 0xffff);
+	state.enabled = true;
+
+	err = pwm_apply_state(vibrator->pwm, &state);
+	if (err) {
+		dev_err(pdev, "failed to apply pwm state: %d", err);
+		return err;
+	}
+
+	if (vibrator->pwm_dir) {
+		pwm_get_state(vibrator->pwm_dir, &state);
+		state.duty_cycle = vibrator->direction_duty_cycle;
+		state.enabled = true;
+
+		err = pwm_apply_state(vibrator->pwm_dir, &state);
+		if (err) {
+			dev_err(pdev, "failed to apply dir-pwm state: %d", err);
+			pwm_disable(vibrator->pwm);
+			return err;
+		}
+	}
+
+	return 0;
+}
+
+static void pwm_vibrator_stop(struct pwm_vibrator *vibrator)
+{
+	regulator_disable(vibrator->vcc);
+
+	if (vibrator->pwm_dir)
+		pwm_disable(vibrator->pwm_dir);
+	pwm_disable(vibrator->pwm);
+}
+
+static void pwm_vibrator_play_work(struct work_struct *work)
+{
+	struct pwm_vibrator *vibrator = container_of(work,
+					struct pwm_vibrator, play_work);
+
+	if (vibrator->level)
+		pwm_vibrator_start(vibrator);
+	else
+		pwm_vibrator_stop(vibrator);
+}
+
+static int pwm_vibrator_play_effect(struct input_dev *dev, void *data,
+				    struct ff_effect *effect)
+{
+	struct pwm_vibrator *vibrator = input_get_drvdata(dev);
+
+	vibrator->level = effect->u.rumble.strong_magnitude;
+	if (!vibrator->level)
+		vibrator->level = effect->u.rumble.weak_magnitude;
+
+	schedule_work(&vibrator->play_work);
+
+	return 0;
+}
+
+static void pwm_vibrator_close(struct input_dev *input)
+{
+	struct pwm_vibrator *vibrator = input_get_drvdata(input);
+
+	cancel_work_sync(&vibrator->play_work);
+	pwm_vibrator_stop(vibrator);
+}
+
+static int pwm_vibrator_probe(struct platform_device *pdev)
+{
+	struct pwm_vibrator *vibrator;
+	struct pwm_state state;
+	int err;
+
+	vibrator = devm_kzalloc(&pdev->dev, sizeof(*vibrator), GFP_KERNEL);
+	if (!vibrator)
+		return -ENOMEM;
+
+	vibrator->input = devm_input_allocate_device(&pdev->dev);
+	if (!vibrator->input)
+		return -ENOMEM;
+
+	vibrator->vcc = devm_regulator_get(&pdev->dev, "vcc");
+	err = PTR_ERR_OR_ZERO(vibrator->vcc);
+	if (err) {
+		if (err != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "Failed to request regulator: %d",
+				err);
+		return err;
+	}
+
+	vibrator->pwm = devm_pwm_get(&pdev->dev, "enable");
+	err = PTR_ERR_OR_ZERO(vibrator->pwm);
+	if (err) {
+		if (err != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "Failed to request main pwm: %d",
+				err);
+		return err;
+	}
+
+	INIT_WORK(&vibrator->play_work, pwm_vibrator_play_work);
+
+	/* Sync up PWM state and ensure it is off. */
+	pwm_init_state(vibrator->pwm, &state);
+	state.enabled = false;
+	err = pwm_apply_state(vibrator->pwm, &state);
+	if (err) {
+		dev_err(&pdev->dev, "failed to apply initial PWM state: %d",
+			err);
+		return err;
+	}
+
+	vibrator->pwm_dir = devm_pwm_get(&pdev->dev, "direction");
+	err = PTR_ERR_OR_ZERO(vibrator->pwm_dir);
+	switch (err) {
+	case 0:
+		/* Sync up PWM state and ensure it is off. */
+		pwm_init_state(vibrator->pwm_dir, &state);
+		state.enabled = false;
+		err = pwm_apply_state(vibrator->pwm_dir, &state);
+		if (err) {
+			dev_err(&pdev->dev, "failed to apply initial PWM state: %d",
+				err);
+			return err;
+		}
+
+		vibrator->direction_duty_cycle =
+			pwm_get_period(vibrator->pwm_dir) / 2;
+		device_property_read_u32(&pdev->dev, "direction-duty-cycle-ns",
+					 &vibrator->direction_duty_cycle);
+		break;
+
+	case -ENODATA:
+		/* Direction PWM is optional */
+		vibrator->pwm_dir = NULL;
+		break;
+
+	default:
+		dev_err(&pdev->dev, "Failed to request direction pwm: %d", err);
+		/* Fall through */
+
+	case -EPROBE_DEFER:
+		return err;
+	}
+
+	vibrator->input->name = "pwm-vibrator";
+	vibrator->input->id.bustype = BUS_HOST;
+	vibrator->input->dev.parent = &pdev->dev;
+	vibrator->input->close = pwm_vibrator_close;
+
+	input_set_drvdata(vibrator->input, vibrator);
+	input_set_capability(vibrator->input, EV_FF, FF_RUMBLE);
+
+	err = input_ff_create_memless(vibrator->input, NULL,
+				      pwm_vibrator_play_effect);
+	if (err) {
+		dev_err(&pdev->dev, "Couldn't create FF dev: %d", err);
+		return err;
+	}
+
+	err = input_register_device(vibrator->input);
+	if (err) {
+		dev_err(&pdev->dev, "Couldn't register input dev: %d", err);
+		return err;
+	}
+
+	platform_set_drvdata(pdev, vibrator);
+
+	return 0;
+}
+
+static int __maybe_unused pwm_vibrator_suspend(struct device *dev)
+{
+	struct pwm_vibrator *vibrator = dev_get_drvdata(dev);
+
+	cancel_work_sync(&vibrator->play_work);
+	if (vibrator->level)
+		pwm_vibrator_stop(vibrator);
+
+	return 0;
+}
+
+static int __maybe_unused pwm_vibrator_resume(struct device *dev)
+{
+	struct pwm_vibrator *vibrator = dev_get_drvdata(dev);
+
+	if (vibrator->level)
+		pwm_vibrator_start(vibrator);
+
+	return 0;
+}
+
+static SIMPLE_DEV_PM_OPS(pwm_vibrator_pm_ops,
+			 pwm_vibrator_suspend, pwm_vibrator_resume);
+
+#ifdef CONFIG_OF
+static const struct of_device_id pwm_vibra_dt_match_table[] = {
+	{ .compatible = "pwm-vibrator" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, pwm_vibra_dt_match_table);
+#endif
+
+static struct platform_driver pwm_vibrator_driver = {
+	.probe	= pwm_vibrator_probe,
+	.driver	= {
+		.name	= "pwm-vibrator",
+		.pm	= &pwm_vibrator_pm_ops,
+		.of_match_table = of_match_ptr(pwm_vibra_dt_match_table),
+	},
+};
+module_platform_driver(pwm_vibrator_driver);
+
+MODULE_AUTHOR("Sebastian Reichel <sre@kernel.org>");
+MODULE_DESCRIPTION("PWM vibrator driver");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:pwm-vibrator");
diff --git a/drivers/input/misc/rk805-pwrkey.c b/drivers/input/misc/rk805-pwrkey.c
new file mode 100644
index 000000000000..921003963a53
--- /dev/null
+++ b/drivers/input/misc/rk805-pwrkey.c
@@ -0,0 +1,111 @@
+/*
+ * Rockchip RK805 PMIC Power Key driver
+ *
+ * Copyright (c) 2017, Fuzhou Rockchip Electronics Co., Ltd
+ *
+ * Author: Joseph Chen <chenjh@rock-chips.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under  the terms of the GNU General  Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ */
+
+#include <linux/errno.h>
+#include <linux/init.h>
+#include <linux/input.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+static irqreturn_t pwrkey_fall_irq(int irq, void *_pwr)
+{
+	struct input_dev *pwr = _pwr;
+
+	input_report_key(pwr, KEY_POWER, 1);
+	input_sync(pwr);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t pwrkey_rise_irq(int irq, void *_pwr)
+{
+	struct input_dev *pwr = _pwr;
+
+	input_report_key(pwr, KEY_POWER, 0);
+	input_sync(pwr);
+
+	return IRQ_HANDLED;
+}
+
+static int rk805_pwrkey_probe(struct platform_device *pdev)
+{
+	struct input_dev *pwr;
+	int fall_irq, rise_irq;
+	int err;
+
+	pwr = devm_input_allocate_device(&pdev->dev);
+	if (!pwr) {
+		dev_err(&pdev->dev, "Can't allocate power button\n");
+		return -ENOMEM;
+	}
+
+	pwr->name = "rk805 pwrkey";
+	pwr->phys = "rk805_pwrkey/input0";
+	pwr->id.bustype = BUS_HOST;
+	input_set_capability(pwr, EV_KEY, KEY_POWER);
+
+	fall_irq = platform_get_irq(pdev, 0);
+	if (fall_irq < 0) {
+		dev_err(&pdev->dev, "Can't get fall irq: %d\n", fall_irq);
+		return fall_irq;
+	}
+
+	rise_irq = platform_get_irq(pdev, 1);
+	if (rise_irq < 0) {
+		dev_err(&pdev->dev, "Can't get rise irq: %d\n", rise_irq);
+		return rise_irq;
+	}
+
+	err = devm_request_any_context_irq(&pwr->dev, fall_irq,
+					   pwrkey_fall_irq,
+					   IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
+					   "rk805_pwrkey_fall", pwr);
+	if (err < 0) {
+		dev_err(&pdev->dev, "Can't register fall irq: %d\n", err);
+		return err;
+	}
+
+	err = devm_request_any_context_irq(&pwr->dev, rise_irq,
+					   pwrkey_rise_irq,
+					   IRQF_TRIGGER_RISING | IRQF_ONESHOT,
+					   "rk805_pwrkey_rise", pwr);
+	if (err < 0) {
+		dev_err(&pdev->dev, "Can't register rise irq: %d\n", err);
+		return err;
+	}
+
+	err = input_register_device(pwr);
+	if (err) {
+		dev_err(&pdev->dev, "Can't register power button: %d\n", err);
+		return err;
+	}
+
+	platform_set_drvdata(pdev, pwr);
+	device_init_wakeup(&pdev->dev, true);
+
+	return 0;
+}
+
+static struct platform_driver rk805_pwrkey_driver = {
+	.probe	= rk805_pwrkey_probe,
+	.driver	= {
+		.name = "rk805-pwrkey",
+	},
+};
+module_platform_driver(rk805_pwrkey_driver);
+
+MODULE_AUTHOR("Joseph Chen <chenjh@rock-chips.com>");
+MODULE_DESCRIPTION("RK805 PMIC Power Key driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/input/misc/soc_button_array.c b/drivers/input/misc/soc_button_array.c
index f600f3a7a3c6..23520df7650f 100644
--- a/drivers/input/misc/soc_button_array.c
+++ b/drivers/input/misc/soc_button_array.c
@@ -331,7 +331,7 @@ static int soc_button_probe(struct platform_device *pdev)
 	error = gpiod_count(dev, NULL);
 	if (error < 0) {
 		dev_dbg(dev, "no GPIO attached, ignoring...\n");
-		return error;
+		return -ENODEV;
 	}
 
 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
diff --git a/drivers/input/misc/twl4030-pwrbutton.c b/drivers/input/misc/twl4030-pwrbutton.c
index 1c13005b228f..b307cca17022 100644
--- a/drivers/input/misc/twl4030-pwrbutton.c
+++ b/drivers/input/misc/twl4030-pwrbutton.c
@@ -27,7 +27,7 @@
 #include <linux/input.h>
 #include <linux/interrupt.h>
 #include <linux/platform_device.h>
-#include <linux/i2c/twl.h>
+#include <linux/mfd/twl.h>
 
 #define PWR_PWRON_IRQ (1 << 0)
 
diff --git a/drivers/input/misc/twl4030-vibra.c b/drivers/input/misc/twl4030-vibra.c
index caa5a62c42fb..6c51d404874b 100644
--- a/drivers/input/misc/twl4030-vibra.c
+++ b/drivers/input/misc/twl4030-vibra.c
@@ -28,7 +28,7 @@
 #include <linux/platform_device.h>
 #include <linux/of.h>
 #include <linux/workqueue.h>
-#include <linux/i2c/twl.h>
+#include <linux/mfd/twl.h>
 #include <linux/mfd/twl4030-audio.h>
 #include <linux/input.h>
 #include <linux/slab.h>
diff --git a/drivers/input/misc/xen-kbdfront.c b/drivers/input/misc/xen-kbdfront.c
index fa130e7b734c..6bf56bb5f8d9 100644
--- a/drivers/input/misc/xen-kbdfront.c
+++ b/drivers/input/misc/xen-kbdfront.c
@@ -84,17 +84,20 @@ static void xenkbd_handle_key_event(struct xenkbd_info *info,
 				    struct xenkbd_key *key)
 {
 	struct input_dev *dev;
+	int value = key->pressed;
 
 	if (test_bit(key->keycode, info->ptr->keybit)) {
 		dev = info->ptr;
 	} else if (test_bit(key->keycode, info->kbd->keybit)) {
 		dev = info->kbd;
+		if (key->pressed && test_bit(key->keycode, info->kbd->key))
+			value = 2; /* Mark as autorepeat */
 	} else {
 		pr_warn("unhandled keycode 0x%x\n", key->keycode);
 		return;
 	}
 
-	input_report_key(dev, key->keycode, key->pressed);
+	input_event(dev, EV_KEY, key->keycode, value);
 	input_sync(dev);
 }
 
diff --git a/drivers/input/misc/yealink.c b/drivers/input/misc/yealink.c
index 6e7ff9561d92..a1e0ff59d2f2 100644
--- a/drivers/input/misc/yealink.c
+++ b/drivers/input/misc/yealink.c
@@ -798,7 +798,7 @@ static struct attribute *yld_attributes[] = {
 	NULL
 };
 
-static struct attribute_group yld_attr_group = {
+static const struct attribute_group yld_attr_group = {
 	.attrs = yld_attributes
 };
 
diff --git a/drivers/input/mouse/alps.c b/drivers/input/mouse/alps.c
index 262d1057c1da..850b00e3ad8e 100644
--- a/drivers/input/mouse/alps.c
+++ b/drivers/input/mouse/alps.c
@@ -1215,14 +1215,24 @@ static int alps_decode_ss4_v2(struct alps_fields *f,
 
 	case SS4_PACKET_ID_TWO:
 		if (priv->flags & ALPS_BUTTONPAD) {
-			f->mt[0].x = SS4_BTL_MF_X_V2(p, 0);
+			if (IS_SS4PLUS_DEV(priv->dev_id)) {
+				f->mt[0].x = SS4_PLUS_BTL_MF_X_V2(p, 0);
+				f->mt[1].x = SS4_PLUS_BTL_MF_X_V2(p, 1);
+			} else {
+				f->mt[0].x = SS4_BTL_MF_X_V2(p, 0);
+				f->mt[1].x = SS4_BTL_MF_X_V2(p, 1);
+			}
 			f->mt[0].y = SS4_BTL_MF_Y_V2(p, 0);
-			f->mt[1].x = SS4_BTL_MF_X_V2(p, 1);
 			f->mt[1].y = SS4_BTL_MF_Y_V2(p, 1);
 		} else {
-			f->mt[0].x = SS4_STD_MF_X_V2(p, 0);
+			if (IS_SS4PLUS_DEV(priv->dev_id)) {
+				f->mt[0].x = SS4_PLUS_STD_MF_X_V2(p, 0);
+				f->mt[1].x = SS4_PLUS_STD_MF_X_V2(p, 1);
+			} else {
+				f->mt[0].x = SS4_STD_MF_X_V2(p, 0);
+				f->mt[1].x = SS4_STD_MF_X_V2(p, 1);
+			}
 			f->mt[0].y = SS4_STD_MF_Y_V2(p, 0);
-			f->mt[1].x = SS4_STD_MF_X_V2(p, 1);
 			f->mt[1].y = SS4_STD_MF_Y_V2(p, 1);
 		}
 		f->pressure = SS4_MF_Z_V2(p, 0) ? 0x30 : 0;
@@ -1239,16 +1249,27 @@ static int alps_decode_ss4_v2(struct alps_fields *f,
 
 	case SS4_PACKET_ID_MULTI:
 		if (priv->flags & ALPS_BUTTONPAD) {
-			f->mt[2].x = SS4_BTL_MF_X_V2(p, 0);
+			if (IS_SS4PLUS_DEV(priv->dev_id)) {
+				f->mt[0].x = SS4_PLUS_BTL_MF_X_V2(p, 0);
+				f->mt[1].x = SS4_PLUS_BTL_MF_X_V2(p, 1);
+			} else {
+				f->mt[2].x = SS4_BTL_MF_X_V2(p, 0);
+				f->mt[3].x = SS4_BTL_MF_X_V2(p, 1);
+			}
+
 			f->mt[2].y = SS4_BTL_MF_Y_V2(p, 0);
-			f->mt[3].x = SS4_BTL_MF_X_V2(p, 1);
 			f->mt[3].y = SS4_BTL_MF_Y_V2(p, 1);
 			no_data_x = SS4_MFPACKET_NO_AX_BL;
 			no_data_y = SS4_MFPACKET_NO_AY_BL;
 		} else {
-			f->mt[2].x = SS4_STD_MF_X_V2(p, 0);
+			if (IS_SS4PLUS_DEV(priv->dev_id)) {
+				f->mt[0].x = SS4_PLUS_STD_MF_X_V2(p, 0);
+				f->mt[1].x = SS4_PLUS_STD_MF_X_V2(p, 1);
+			} else {
+				f->mt[0].x = SS4_STD_MF_X_V2(p, 0);
+				f->mt[1].x = SS4_STD_MF_X_V2(p, 1);
+			}
 			f->mt[2].y = SS4_STD_MF_Y_V2(p, 0);
-			f->mt[3].x = SS4_STD_MF_X_V2(p, 1);
 			f->mt[3].y = SS4_STD_MF_Y_V2(p, 1);
 			no_data_x = SS4_MFPACKET_NO_AX;
 			no_data_y = SS4_MFPACKET_NO_AY;
@@ -2541,8 +2562,8 @@ static int alps_set_defaults_ss4_v2(struct psmouse *psmouse,
 
 	memset(otp, 0, sizeof(otp));
 
-	if (alps_get_otp_values_ss4_v2(psmouse, 0, &otp[0][0]) ||
-	    alps_get_otp_values_ss4_v2(psmouse, 1, &otp[1][0]))
+	if (alps_get_otp_values_ss4_v2(psmouse, 1, &otp[1][0]) ||
+	    alps_get_otp_values_ss4_v2(psmouse, 0, &otp[0][0]))
 		return -1;
 
 	alps_update_device_area_ss4_v2(otp, priv);
diff --git a/drivers/input/mouse/alps.h b/drivers/input/mouse/alps.h
index ed2d6879fa52..c80a7c76cb76 100644
--- a/drivers/input/mouse/alps.h
+++ b/drivers/input/mouse/alps.h
@@ -100,6 +100,10 @@ enum SS4_PACKET_ID {
 				 ((_b[1 + _i * 3]  << 5) & 0x1F00)	\
 				)
 
+#define SS4_PLUS_STD_MF_X_V2(_b, _i) (((_b[0 + (_i) * 3] << 4) & 0x0070) | \
+				 ((_b[1 + (_i) * 3]  << 4) & 0x0F80)	\
+				)
+
 #define SS4_STD_MF_Y_V2(_b, _i)	(((_b[1 + (_i) * 3] << 3) & 0x0010) |	\
 				 ((_b[2 + (_i) * 3] << 5) & 0x01E0) |	\
 				 ((_b[2 + (_i) * 3] << 4) & 0x0E00)	\
@@ -109,6 +113,10 @@ enum SS4_PACKET_ID {
 				 ((_b[0 + (_i) * 3] >> 3) & 0x0010)	\
 				)
 
+#define SS4_PLUS_BTL_MF_X_V2(_b, _i) (SS4_PLUS_STD_MF_X_V2(_b, _i) |	\
+				 ((_b[0 + (_i) * 3] >> 4) & 0x0008)	\
+				)
+
 #define SS4_BTL_MF_Y_V2(_b, _i)	(SS4_STD_MF_Y_V2(_b, _i) | \
 				 ((_b[0 + (_i) * 3] >> 3) & 0x0008)	\
 				)
diff --git a/drivers/input/mouse/appletouch.c b/drivers/input/mouse/appletouch.c
index ef234c9b2f2f..81a695d0b4e0 100644
--- a/drivers/input/mouse/appletouch.c
+++ b/drivers/input/mouse/appletouch.c
@@ -125,7 +125,7 @@ static const struct atp_info geyser4_info = {
  *  According to Info.plist Geyser IV is the same as Geyser III.)
  */
 
-static struct usb_device_id atp_table[] = {
+static const struct usb_device_id atp_table[] = {
 	/* PowerBooks Feb 2005, iBooks G4 */
 	ATP_DEVICE(0x020e, fountain_info),	/* FOUNTAIN ANSI */
 	ATP_DEVICE(0x020f, fountain_info),	/* FOUNTAIN ISO */
diff --git a/drivers/input/mouse/byd.c b/drivers/input/mouse/byd.c
index b27aa637f877..b64b81599f7e 100644
--- a/drivers/input/mouse/byd.c
+++ b/drivers/input/mouse/byd.c
@@ -344,7 +344,7 @@ static int byd_reset_touchpad(struct psmouse *psmouse)
 	u8 param[4];
 	size_t i;
 
-	const struct {
+	static const struct {
 		u16 command;
 		u8 arg;
 	} seq[] = {
diff --git a/drivers/input/mouse/elan_i2c.h b/drivers/input/mouse/elan_i2c.h
index 61c202436250..599544c1a91c 100644
--- a/drivers/input/mouse/elan_i2c.h
+++ b/drivers/input/mouse/elan_i2c.h
@@ -58,7 +58,7 @@ struct elan_transport_ops {
 
 	int (*get_version)(struct i2c_client *client, bool iap, u8 *version);
 	int (*get_sm_version)(struct i2c_client *client,
-			      u16 *ic_type, u8 *version);
+			      u16 *ic_type, u8 *version, u8 *clickpad);
 	int (*get_checksum)(struct i2c_client *client, bool iap, u16 *csum);
 	int (*get_product_id)(struct i2c_client *client, u16 *id);
 
diff --git a/drivers/input/mouse/elan_i2c_core.c b/drivers/input/mouse/elan_i2c_core.c
index 3b616cb7c67f..0e761d079dc4 100644
--- a/drivers/input/mouse/elan_i2c_core.c
+++ b/drivers/input/mouse/elan_i2c_core.c
@@ -95,6 +95,7 @@ struct elan_tp_data {
 	u8			min_baseline;
 	u8			max_baseline;
 	bool			baseline_ready;
+	u8			clickpad;
 };
 
 static int elan_get_fwinfo(u16 ic_type, u16 *validpage_count,
@@ -213,7 +214,7 @@ static int elan_query_product(struct elan_tp_data *data)
 		return error;
 
 	error = data->ops->get_sm_version(data->client, &data->ic_type,
-					  &data->sm_version);
+					  &data->sm_version, &data->clickpad);
 	if (error)
 		return error;
 
@@ -923,6 +924,7 @@ static void elan_report_absolute(struct elan_tp_data *data, u8 *packet)
 	}
 
 	input_report_key(input, BTN_LEFT, tp_info & 0x01);
+	input_report_key(input, BTN_RIGHT, tp_info & 0x02);
 	input_report_abs(input, ABS_DISTANCE, hover_event != 0);
 	input_mt_report_pointer_emulation(input, true);
 	input_sync(input);
@@ -991,7 +993,10 @@ static int elan_setup_input_device(struct elan_tp_data *data)
 
 	__set_bit(EV_ABS, input->evbit);
 	__set_bit(INPUT_PROP_POINTER, input->propbit);
-	__set_bit(INPUT_PROP_BUTTONPAD, input->propbit);
+	if (data->clickpad)
+		__set_bit(INPUT_PROP_BUTTONPAD, input->propbit);
+	else
+		__set_bit(BTN_RIGHT, input->keybit);
 	__set_bit(BTN_LEFT, input->keybit);
 
 	/* Set up ST parameters */
@@ -1247,7 +1252,12 @@ static const struct acpi_device_id elan_acpi_id[] = {
 	{ "ELAN0000", 0 },
 	{ "ELAN0100", 0 },
 	{ "ELAN0600", 0 },
+	{ "ELAN0602", 0 },
+	{ "ELAN0605", 0 },
+	{ "ELAN0608", 0 },
 	{ "ELAN0605", 0 },
+	{ "ELAN0609", 0 },
+	{ "ELAN060B", 0 },
 	{ "ELAN1000", 0 },
 	{ }
 };
diff --git a/drivers/input/mouse/elan_i2c_i2c.c b/drivers/input/mouse/elan_i2c_i2c.c
index 80172f25974d..15b1330606c1 100644
--- a/drivers/input/mouse/elan_i2c_i2c.c
+++ b/drivers/input/mouse/elan_i2c_i2c.c
@@ -288,7 +288,8 @@ static int elan_i2c_get_version(struct i2c_client *client,
 }
 
 static int elan_i2c_get_sm_version(struct i2c_client *client,
-				   u16 *ic_type, u8 *version)
+				   u16 *ic_type, u8 *version,
+				   u8 *clickpad)
 {
 	int error;
 	u8 pattern_ver;
@@ -317,6 +318,7 @@ static int elan_i2c_get_sm_version(struct i2c_client *client,
 			return error;
 		}
 		*version = val[1];
+		*clickpad = val[0] & 0x10;
 	} else {
 		error = elan_i2c_read_cmd(client, ETP_I2C_OSM_VERSION_CMD, val);
 		if (error) {
@@ -326,6 +328,15 @@ static int elan_i2c_get_sm_version(struct i2c_client *client,
 		}
 		*version = val[0];
 		*ic_type = val[1];
+
+		error = elan_i2c_read_cmd(client, ETP_I2C_NSM_VERSION_CMD,
+					  val);
+		if (error) {
+			dev_err(&client->dev, "failed to get SM version: %d\n",
+				error);
+			return error;
+		}
+		*clickpad = val[0] & 0x10;
 	}
 
 	return 0;
diff --git a/drivers/input/mouse/elan_i2c_smbus.c b/drivers/input/mouse/elan_i2c_smbus.c
index df7a57ca7331..29f99529b187 100644
--- a/drivers/input/mouse/elan_i2c_smbus.c
+++ b/drivers/input/mouse/elan_i2c_smbus.c
@@ -166,7 +166,8 @@ static int elan_smbus_get_version(struct i2c_client *client,
 }
 
 static int elan_smbus_get_sm_version(struct i2c_client *client,
-				     u16 *ic_type, u8 *version)
+				     u16 *ic_type, u8 *version,
+				     u8 *clickpad)
 {
 	int error;
 	u8 val[3];
@@ -180,6 +181,7 @@ static int elan_smbus_get_sm_version(struct i2c_client *client,
 
 	*version = val[0];
 	*ic_type = val[1];
+	*clickpad = val[0] & 0x10;
 	return 0;
 }
 
diff --git a/drivers/input/mouse/elantech.c b/drivers/input/mouse/elantech.c
index 791993215ea3..b84cd978fce2 100644
--- a/drivers/input/mouse/elantech.c
+++ b/drivers/input/mouse/elantech.c
@@ -700,7 +700,9 @@ static int elantech_debounce_check_v2(struct psmouse *psmouse)
          * When we encounter packet that matches this exactly, it means the
          * hardware is in debounce status. Just ignore the whole packet.
          */
-        const u8 debounce_packet[] = { 0x84, 0xff, 0xff, 0x02, 0xff, 0xff };
+	static const u8 debounce_packet[] = {
+		0x84, 0xff, 0xff, 0x02, 0xff, 0xff
+	};
         unsigned char *packet = psmouse->packet;
 
         return !memcmp(packet, debounce_packet, sizeof(debounce_packet));
@@ -741,7 +743,9 @@ static int elantech_packet_check_v2(struct psmouse *psmouse)
 static int elantech_packet_check_v3(struct psmouse *psmouse)
 {
 	struct elantech_data *etd = psmouse->private;
-	const u8 debounce_packet[] = { 0xc4, 0xff, 0xff, 0x02, 0xff, 0xff };
+	static const u8 debounce_packet[] = {
+		0xc4, 0xff, 0xff, 0x02, 0xff, 0xff
+	};
 	unsigned char *packet = psmouse->packet;
 
 	/*
@@ -1377,7 +1381,7 @@ static struct attribute *elantech_attrs[] = {
 	NULL
 };
 
-static struct attribute_group elantech_attr_group = {
+static const struct attribute_group elantech_attr_group = {
 	.attrs = elantech_attrs,
 };
 
diff --git a/drivers/input/mouse/psmouse-base.c b/drivers/input/mouse/psmouse-base.c
index f73b47b8c578..6a5649e52eed 100644
--- a/drivers/input/mouse/psmouse-base.c
+++ b/drivers/input/mouse/psmouse-base.c
@@ -101,7 +101,7 @@ static struct attribute *psmouse_attributes[] = {
 	NULL
 };
 
-static struct attribute_group psmouse_attribute_group = {
+static const struct attribute_group psmouse_attribute_group = {
 	.attrs	= psmouse_attributes,
 };
 
diff --git a/drivers/input/mouse/synaptics.c b/drivers/input/mouse/synaptics.c
index 16c30460ef04..5af0b7d200bc 100644
--- a/drivers/input/mouse/synaptics.c
+++ b/drivers/input/mouse/synaptics.c
@@ -535,16 +535,17 @@ static void synaptics_apply_quirks(struct psmouse *psmouse,
 	}
 }
 
+static bool synaptics_has_agm(struct synaptics_data *priv)
+{
+	return (SYN_CAP_ADV_GESTURE(priv->info.ext_cap_0c) ||
+		SYN_CAP_IMAGE_SENSOR(priv->info.ext_cap_0c));
+}
+
 static int synaptics_set_advanced_gesture_mode(struct psmouse *psmouse)
 {
 	static u8 param = 0xc8;
-	struct synaptics_data *priv = psmouse->private;
 	int error;
 
-	if (!(SYN_CAP_ADV_GESTURE(priv->info.ext_cap_0c) ||
-	      SYN_CAP_IMAGE_SENSOR(priv->info.ext_cap_0c)))
-		return 0;
-
 	error = psmouse_sliced_command(psmouse, SYN_QUE_MODEL);
 	if (error)
 		return error;
@@ -553,9 +554,6 @@ static int synaptics_set_advanced_gesture_mode(struct psmouse *psmouse)
 	if (error)
 		return error;
 
-	/* Advanced gesture mode also sends multi finger data */
-	priv->info.capabilities |= BIT(1);
-
 	return 0;
 }
 
@@ -578,7 +576,7 @@ static int synaptics_set_mode(struct psmouse *psmouse)
 	if (error)
 		return error;
 
-	if (priv->absolute_mode) {
+	if (priv->absolute_mode && synaptics_has_agm(priv)) {
 		error = synaptics_set_advanced_gesture_mode(psmouse);
 		if (error) {
 			psmouse_err(psmouse,
@@ -766,9 +764,7 @@ static int synaptics_parse_hw_state(const u8 buf[],
 			 ((buf[0] & 0x04) >> 1) |
 			 ((buf[3] & 0x04) >> 2));
 
-		if ((SYN_CAP_ADV_GESTURE(priv->info.ext_cap_0c) ||
-			SYN_CAP_IMAGE_SENSOR(priv->info.ext_cap_0c)) &&
-		    hw->w == 2) {
+		if (synaptics_has_agm(priv) && hw->w == 2) {
 			synaptics_parse_agm(buf, priv, hw);
 			return 1;
 		}
@@ -1033,6 +1029,15 @@ static void synaptics_image_sensor_process(struct psmouse *psmouse,
 	synaptics_report_mt_data(psmouse, sgm, num_fingers);
 }
 
+static bool synaptics_has_multifinger(struct synaptics_data *priv)
+{
+	if (SYN_CAP_MULTIFINGER(priv->info.capabilities))
+		return true;
+
+	/* Advanced gesture mode also sends multi finger data */
+	return synaptics_has_agm(priv);
+}
+
 /*
  *  called for each full received packet from the touchpad
  */
@@ -1079,7 +1084,7 @@ static void synaptics_process_packet(struct psmouse *psmouse)
 		if (SYN_CAP_EXTENDED(info->capabilities)) {
 			switch (hw.w) {
 			case 0 ... 1:
-				if (SYN_CAP_MULTIFINGER(info->capabilities))
+				if (synaptics_has_multifinger(priv))
 					num_fingers = hw.w + 2;
 				break;
 			case 2:
@@ -1123,7 +1128,7 @@ static void synaptics_process_packet(struct psmouse *psmouse)
 		input_report_abs(dev, ABS_TOOL_WIDTH, finger_width);
 
 	input_report_key(dev, BTN_TOOL_FINGER, num_fingers == 1);
-	if (SYN_CAP_MULTIFINGER(info->capabilities)) {
+	if (synaptics_has_multifinger(priv)) {
 		input_report_key(dev, BTN_TOOL_DOUBLETAP, num_fingers == 2);
 		input_report_key(dev, BTN_TOOL_TRIPLETAP, num_fingers == 3);
 	}
@@ -1283,7 +1288,7 @@ static void set_input_params(struct psmouse *psmouse,
 	__set_bit(BTN_TOUCH, dev->keybit);
 	__set_bit(BTN_TOOL_FINGER, dev->keybit);
 
-	if (SYN_CAP_MULTIFINGER(info->capabilities)) {
+	if (synaptics_has_multifinger(priv)) {
 		__set_bit(BTN_TOOL_DOUBLETAP, dev->keybit);
 		__set_bit(BTN_TOOL_TRIPLETAP, dev->keybit);
 	}
diff --git a/drivers/input/mouse/synaptics_usb.c b/drivers/input/mouse/synaptics_usb.c
index 6bcc0189c1c9..cb7d15d826d0 100644
--- a/drivers/input/mouse/synaptics_usb.c
+++ b/drivers/input/mouse/synaptics_usb.c
@@ -525,7 +525,7 @@ static int synusb_reset_resume(struct usb_interface *intf)
 	return synusb_resume(intf);
 }
 
-static struct usb_device_id synusb_idtable[] = {
+static const struct usb_device_id synusb_idtable[] = {
 	{ USB_DEVICE_SYNAPTICS(TP, SYNUSB_TOUCHPAD) },
 	{ USB_DEVICE_SYNAPTICS(INT_TP, SYNUSB_TOUCHPAD) },
 	{ USB_DEVICE_SYNAPTICS(CPAD,
diff --git a/drivers/input/mouse/trackpoint.c b/drivers/input/mouse/trackpoint.c
index 922ea02edcc3..0871010f18d5 100644
--- a/drivers/input/mouse/trackpoint.c
+++ b/drivers/input/mouse/trackpoint.c
@@ -265,7 +265,8 @@ static int trackpoint_start_protocol(struct psmouse *psmouse, unsigned char *fir
 	if (ps2_command(&psmouse->ps2dev, param, MAKE_PS2_CMD(0, 2, TP_READ_ID)))
 		return -1;
 
-	if (param[0] != TP_MAGIC_IDENT)
+	/* add new TP ID. */
+	if (!(param[0] & TP_MAGIC_IDENT))
 		return -1;
 
 	if (firmware_id)
@@ -380,8 +381,8 @@ int trackpoint_detect(struct psmouse *psmouse, bool set_properties)
 		return 0;
 
 	if (trackpoint_read(ps2dev, TP_EXT_BTN, &button_info)) {
-		psmouse_warn(psmouse, "failed to get extended button data\n");
-		button_info = 0;
+		psmouse_warn(psmouse, "failed to get extended button data, assuming 3 buttons\n");
+		button_info = 0x33;
 	}
 
 	psmouse->private = kzalloc(sizeof(struct trackpoint_data), GFP_KERNEL);
diff --git a/drivers/input/mouse/trackpoint.h b/drivers/input/mouse/trackpoint.h
index 5617ed3a7d7a..88055755f82e 100644
--- a/drivers/input/mouse/trackpoint.h
+++ b/drivers/input/mouse/trackpoint.h
@@ -21,8 +21,9 @@
 #define TP_COMMAND		0xE2	/* Commands start with this */
 
 #define TP_READ_ID		0xE1	/* Sent for device identification */
-#define TP_MAGIC_IDENT		0x01	/* Sent after a TP_READ_ID followed */
+#define TP_MAGIC_IDENT		0x03	/* Sent after a TP_READ_ID followed */
 					/* by the firmware ID */
+					/* Firmware ID includes 0x1, 0x2, 0x3 */
 
 
 /*
diff --git a/drivers/input/mousedev.c b/drivers/input/mousedev.c
index 0e0ff84088fd..2d7f691ec71c 100644
--- a/drivers/input/mousedev.c
+++ b/drivers/input/mousedev.c
@@ -15,6 +15,7 @@
 #define MOUSEDEV_MINORS		31
 #define MOUSEDEV_MIX		63
 
+#include <linux/bitops.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/poll.h>
@@ -103,7 +104,7 @@ struct mousedev_client {
 	spinlock_t packet_lock;
 	int pos_x, pos_y;
 
-	signed char ps2[6];
+	u8 ps2[6];
 	unsigned char ready, buffer, bufsiz;
 	unsigned char imexseq, impsseq;
 	enum mousedev_emul mode;
@@ -291,11 +292,10 @@ static void mousedev_notify_readers(struct mousedev *mousedev,
 		}
 
 		client->pos_x += packet->dx;
-		client->pos_x = client->pos_x < 0 ?
-			0 : (client->pos_x >= xres ? xres : client->pos_x);
+		client->pos_x = clamp_val(client->pos_x, 0, xres);
+
 		client->pos_y += packet->dy;
-		client->pos_y = client->pos_y < 0 ?
-			0 : (client->pos_y >= yres ? yres : client->pos_y);
+		client->pos_y = clamp_val(client->pos_y, 0, yres);
 
 		p->dx += packet->dx;
 		p->dy += packet->dy;
@@ -571,44 +571,50 @@ static int mousedev_open(struct inode *inode, struct file *file)
 	return error;
 }
 
-static inline int mousedev_limit_delta(int delta, int limit)
-{
-	return delta > limit ? limit : (delta < -limit ? -limit : delta);
-}
-
-static void mousedev_packet(struct mousedev_client *client,
-			    signed char *ps2_data)
+static void mousedev_packet(struct mousedev_client *client, u8 *ps2_data)
 {
 	struct mousedev_motion *p = &client->packets[client->tail];
+	s8 dx, dy, dz;
+
+	dx = clamp_val(p->dx, -127, 127);
+	p->dx -= dx;
+
+	dy = clamp_val(p->dy, -127, 127);
+	p->dy -= dy;
 
-	ps2_data[0] = 0x08 |
-		((p->dx < 0) << 4) | ((p->dy < 0) << 5) | (p->buttons & 0x07);
-	ps2_data[1] = mousedev_limit_delta(p->dx, 127);
-	ps2_data[2] = mousedev_limit_delta(p->dy, 127);
-	p->dx -= ps2_data[1];
-	p->dy -= ps2_data[2];
+	ps2_data[0] = BIT(3);
+	ps2_data[0] |= ((dx & BIT(7)) >> 3) | ((dy & BIT(7)) >> 2);
+	ps2_data[0] |= p->buttons & 0x07;
+	ps2_data[1] = dx;
+	ps2_data[2] = dy;
 
 	switch (client->mode) {
 	case MOUSEDEV_EMUL_EXPS:
-		ps2_data[3] = mousedev_limit_delta(p->dz, 7);
-		p->dz -= ps2_data[3];
-		ps2_data[3] = (ps2_data[3] & 0x0f) | ((p->buttons & 0x18) << 1);
+		dz = clamp_val(p->dz, -7, 7);
+		p->dz -= dz;
+
+		ps2_data[3] = (dz & 0x0f) | ((p->buttons & 0x18) << 1);
 		client->bufsiz = 4;
 		break;
 
 	case MOUSEDEV_EMUL_IMPS:
-		ps2_data[0] |=
-			((p->buttons & 0x10) >> 3) | ((p->buttons & 0x08) >> 1);
-		ps2_data[3] = mousedev_limit_delta(p->dz, 127);
-		p->dz -= ps2_data[3];
+		dz = clamp_val(p->dz, -127, 127);
+		p->dz -= dz;
+
+		ps2_data[0] |= ((p->buttons & 0x10) >> 3) |
+			       ((p->buttons & 0x08) >> 1);
+		ps2_data[3] = dz;
+
 		client->bufsiz = 4;
 		break;
 
 	case MOUSEDEV_EMUL_PS2:
 	default:
-		ps2_data[0] |=
-			((p->buttons & 0x10) >> 3) | ((p->buttons & 0x08) >> 1);
 		p->dz = 0;
+
+		ps2_data[0] |= ((p->buttons & 0x10) >> 3) |
+			       ((p->buttons & 0x08) >> 1);
+
 		client->bufsiz = 3;
 		break;
 	}
@@ -714,7 +720,7 @@ static ssize_t mousedev_read(struct file *file, char __user *buffer,
 {
 	struct mousedev_client *client = file->private_data;
 	struct mousedev *mousedev = client->mousedev;
-	signed char data[sizeof(client->ps2)];
+	u8 data[sizeof(client->ps2)];
 	int retval = 0;
 
 	if (!client->ready && !client->buffer && mousedev->exist &&
diff --git a/drivers/input/rmi4/rmi_f01.c b/drivers/input/rmi4/rmi_f01.c
index 7f7e9176f7ea..ae966e333a2f 100644
--- a/drivers/input/rmi4/rmi_f01.c
+++ b/drivers/input/rmi4/rmi_f01.c
@@ -334,7 +334,7 @@ static struct attribute *rmi_f01_attrs[] = {
 	NULL
 };
 
-static struct attribute_group rmi_f01_attr_group = {
+static const struct attribute_group rmi_f01_attr_group = {
 	.attrs = rmi_f01_attrs,
 };
 
@@ -570,18 +570,14 @@ static int rmi_f01_probe(struct rmi_function *fn)
 
 	dev_set_drvdata(&fn->dev, f01);
 
-	error = sysfs_create_group(&fn->rmi_dev->dev.kobj, &rmi_f01_attr_group);
+	error = devm_device_add_group(&fn->rmi_dev->dev, &rmi_f01_attr_group);
 	if (error)
-		dev_warn(&fn->dev, "Failed to create sysfs group: %d\n", error);
+		dev_warn(&fn->dev,
+			 "Failed to create attribute group: %d\n", error);
 
 	return 0;
 }
 
-static void rmi_f01_remove(struct rmi_function *fn)
-{
-	sysfs_remove_group(&fn->rmi_dev->dev.kobj, &rmi_f01_attr_group);
-}
-
 static int rmi_f01_config(struct rmi_function *fn)
 {
 	struct f01_data *f01 = dev_get_drvdata(&fn->dev);
@@ -721,7 +717,6 @@ struct rmi_function_handler rmi_f01_handler = {
 	},
 	.func		= 0x01,
 	.probe		= rmi_f01_probe,
-	.remove		= rmi_f01_remove,
 	.config		= rmi_f01_config,
 	.attention	= rmi_f01_attention,
 	.suspend	= rmi_f01_suspend,
diff --git a/drivers/input/rmi4/rmi_f34.c b/drivers/input/rmi4/rmi_f34.c
index b8ee78e0d61f..4cfe9703a8e7 100644
--- a/drivers/input/rmi4/rmi_f34.c
+++ b/drivers/input/rmi4/rmi_f34.c
@@ -516,7 +516,7 @@ static struct attribute *rmi_firmware_attrs[] = {
 	NULL
 };
 
-static struct attribute_group rmi_firmware_attr_group = {
+static const struct attribute_group rmi_firmware_attr_group = {
 	.attrs = rmi_firmware_attrs,
 };
 
diff --git a/drivers/input/serio/Kconfig b/drivers/input/serio/Kconfig
index c3d05b4d3118..21488c048fa3 100644
--- a/drivers/input/serio/Kconfig
+++ b/drivers/input/serio/Kconfig
@@ -292,6 +292,17 @@ config SERIO_SUN4I_PS2
 	  To compile this driver as a module, choose M here: the
 	  module will be called sun4i-ps2.
 
+config SERIO_GPIO_PS2
+	tristate "GPIO PS/2 bit banging driver"
+	depends on GPIOLIB
+	help
+	  Say Y here if you want PS/2 bit banging support via GPIO.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called ps2-gpio.
+
+	  If you are unsure, say N.
+
 config USERIO
 	tristate "User space serio port driver support"
 	help
diff --git a/drivers/input/serio/Makefile b/drivers/input/serio/Makefile
index 2374ef9b33d7..767bd9b6e1ed 100644
--- a/drivers/input/serio/Makefile
+++ b/drivers/input/serio/Makefile
@@ -30,4 +30,5 @@ obj-$(CONFIG_SERIO_APBPS2)	+= apbps2.o
 obj-$(CONFIG_SERIO_OLPC_APSP)	+= olpc_apsp.o
 obj-$(CONFIG_HYPERV_KEYBOARD)	+= hyperv-keyboard.o
 obj-$(CONFIG_SERIO_SUN4I_PS2)	+= sun4i-ps2.o
+obj-$(CONFIG_SERIO_GPIO_PS2)	+= ps2-gpio.o
 obj-$(CONFIG_USERIO)		+= userio.o
diff --git a/drivers/input/serio/ambakmi.c b/drivers/input/serio/ambakmi.c
index c6606cacb6a7..ff3875cf3da1 100644
--- a/drivers/input/serio/ambakmi.c
+++ b/drivers/input/serio/ambakmi.c
@@ -187,7 +187,7 @@ static int __maybe_unused amba_kmi_resume(struct device *dev)
 
 static SIMPLE_DEV_PM_OPS(amba_kmi_dev_pm_ops, NULL, amba_kmi_resume);
 
-static struct amba_id amba_kmi_idtable[] = {
+static const struct amba_id amba_kmi_idtable[] = {
 	{
 		.id	= 0x00041050,
 		.mask	= 0x000fffff,
diff --git a/drivers/input/serio/gscps2.c b/drivers/input/serio/gscps2.c
index ecba666afadb..aa9f29b875de 100644
--- a/drivers/input/serio/gscps2.c
+++ b/drivers/input/serio/gscps2.c
@@ -325,7 +325,7 @@ static void gscps2_close(struct serio *port)
  * @return: success/error report
  */
 
-static int gscps2_probe(struct parisc_device *dev)
+static int __init gscps2_probe(struct parisc_device *dev)
 {
 	struct gscps2port *ps2port;
 	struct serio *serio;
@@ -412,7 +412,7 @@ fail_nomem:
  * @return: success/error report
  */
 
-static int gscps2_remove(struct parisc_device *dev)
+static int __exit gscps2_remove(struct parisc_device *dev)
 {
 	struct gscps2port *ps2port = dev_get_drvdata(&dev->dev);
 
@@ -430,7 +430,7 @@ static int gscps2_remove(struct parisc_device *dev)
 }
 
 
-static struct parisc_device_id gscps2_device_tbl[] = {
+static const struct parisc_device_id gscps2_device_tbl[] __initconst = {
 	{ HPHW_FIO, HVERSION_REV_ANY_ID, HVERSION_ANY_ID, 0x00084 }, /* LASI PS/2 */
 #ifdef DINO_TESTED
 	{ HPHW_FIO, HVERSION_REV_ANY_ID, HVERSION_ANY_ID, 0x00096 }, /* DINO PS/2 */
@@ -439,11 +439,11 @@ static struct parisc_device_id gscps2_device_tbl[] = {
 };
 MODULE_DEVICE_TABLE(parisc, gscps2_device_tbl);
 
-static struct parisc_driver parisc_ps2_driver = {
+static struct parisc_driver parisc_ps2_driver __refdata = {
 	.name		= "gsc_ps2",
 	.id_table	= gscps2_device_tbl,
 	.probe		= gscps2_probe,
-	.remove		= gscps2_remove,
+	.remove		= __exit_p(gscps2_remove),
 };
 
 static int __init gscps2_init(void)
diff --git a/drivers/input/serio/hp_sdc.c b/drivers/input/serio/hp_sdc.c
index 1bfdae4b0d99..8eef6849d066 100644
--- a/drivers/input/serio/hp_sdc.c
+++ b/drivers/input/serio/hp_sdc.c
@@ -805,7 +805,7 @@ static void hp_sdc_kicker(unsigned long data)
 
 #if defined(__hppa__)
 
-static const struct parisc_device_id hp_sdc_tbl[] = {
+static const struct parisc_device_id hp_sdc_tbl[] __initconst = {
 	{
 		.hw_type =	HPHW_FIO,
 		.hversion_rev =	HVERSION_REV_ANY_ID,
@@ -820,7 +820,7 @@ MODULE_DEVICE_TABLE(parisc, hp_sdc_tbl);
 static int __init hp_sdc_init_hppa(struct parisc_device *d);
 static struct delayed_work moduleloader_work;
 
-static struct parisc_driver hp_sdc_driver = {
+static struct parisc_driver hp_sdc_driver __refdata = {
 	.name =		"hp_sdc",
 	.id_table =	hp_sdc_tbl,
 	.probe =	hp_sdc_init_hppa,
diff --git a/drivers/input/serio/i8042-x86ia64io.h b/drivers/input/serio/i8042-x86ia64io.h
index f932a83b4990..6cbbdc6e9687 100644
--- a/drivers/input/serio/i8042-x86ia64io.h
+++ b/drivers/input/serio/i8042-x86ia64io.h
@@ -840,6 +840,13 @@ static const struct dmi_system_id __initconst i8042_dmi_kbdreset_table[] = {
 		},
 	},
 	{
+		/* Gigabyte P57 - Elantech touchpad */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "GIGABYTE"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "P57"),
+		},
+	},
+	{
 		/* Schenker XMG C504 - Elantech touchpad */
 		.matches = {
 			DMI_MATCH(DMI_SYS_VENDOR, "XMG"),
@@ -927,7 +934,7 @@ static int i8042_pnp_aux_probe(struct pnp_dev *dev, const struct pnp_device_id *
 	return 0;
 }
 
-static struct pnp_device_id pnp_kbd_devids[] = {
+static const struct pnp_device_id pnp_kbd_devids[] = {
 	{ .id = "PNP0300", .driver_data = 0 },
 	{ .id = "PNP0301", .driver_data = 0 },
 	{ .id = "PNP0302", .driver_data = 0 },
@@ -957,7 +964,7 @@ static struct pnp_driver i8042_pnp_kbd_driver = {
 	},
 };
 
-static struct pnp_device_id pnp_aux_devids[] = {
+static const struct pnp_device_id pnp_aux_devids[] = {
 	{ .id = "AUI0200", .driver_data = 0 },
 	{ .id = "FJC6000", .driver_data = 0 },
 	{ .id = "FJC6001", .driver_data = 0 },
diff --git a/drivers/input/serio/ps2-gpio.c b/drivers/input/serio/ps2-gpio.c
new file mode 100644
index 000000000000..b50e3817f3c4
--- /dev/null
+++ b/drivers/input/serio/ps2-gpio.c
@@ -0,0 +1,453 @@
+/*
+ * GPIO based serio bus driver for bit banging the PS/2 protocol
+ *
+ * Author: Danilo Krummrich <danilokrummrich@dk-develop.de>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/gpio/consumer.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/serio.h>
+#include <linux/slab.h>
+#include <linux/platform_device.h>
+#include <linux/workqueue.h>
+#include <linux/completion.h>
+#include <linux/mutex.h>
+#include <linux/preempt.h>
+#include <linux/property.h>
+#include <linux/of.h>
+#include <linux/jiffies.h>
+#include <linux/delay.h>
+
+#define DRIVER_NAME		"ps2-gpio"
+
+#define PS2_MODE_RX		0
+#define PS2_MODE_TX		1
+
+#define PS2_START_BIT		0
+#define PS2_DATA_BIT0		1
+#define PS2_DATA_BIT1		2
+#define PS2_DATA_BIT2		3
+#define PS2_DATA_BIT3		4
+#define PS2_DATA_BIT4		5
+#define PS2_DATA_BIT5		6
+#define PS2_DATA_BIT6		7
+#define PS2_DATA_BIT7		8
+#define PS2_PARITY_BIT		9
+#define PS2_STOP_BIT		10
+#define PS2_TX_TIMEOUT		11
+#define PS2_ACK_BIT		12
+
+#define PS2_DEV_RET_ACK		0xfa
+#define PS2_DEV_RET_NACK	0xfe
+
+#define PS2_CMD_RESEND		0xfe
+
+struct ps2_gpio_data {
+	struct device *dev;
+	struct serio *serio;
+	unsigned char mode;
+	struct gpio_desc *gpio_clk;
+	struct gpio_desc *gpio_data;
+	bool write_enable;
+	int irq;
+	unsigned char rx_cnt;
+	unsigned char rx_byte;
+	unsigned char tx_cnt;
+	unsigned char tx_byte;
+	struct completion tx_done;
+	struct mutex tx_mutex;
+	struct delayed_work tx_work;
+};
+
+static int ps2_gpio_open(struct serio *serio)
+{
+	struct ps2_gpio_data *drvdata = serio->port_data;
+
+	enable_irq(drvdata->irq);
+	return 0;
+}
+
+static void ps2_gpio_close(struct serio *serio)
+{
+	struct ps2_gpio_data *drvdata = serio->port_data;
+
+	disable_irq(drvdata->irq);
+}
+
+static int __ps2_gpio_write(struct serio *serio, unsigned char val)
+{
+	struct ps2_gpio_data *drvdata = serio->port_data;
+
+	disable_irq_nosync(drvdata->irq);
+	gpiod_direction_output(drvdata->gpio_clk, 0);
+
+	drvdata->mode = PS2_MODE_TX;
+	drvdata->tx_byte = val;
+
+	schedule_delayed_work(&drvdata->tx_work, usecs_to_jiffies(200));
+
+	return 0;
+}
+
+static int ps2_gpio_write(struct serio *serio, unsigned char val)
+{
+	struct ps2_gpio_data *drvdata = serio->port_data;
+	int ret = 0;
+
+	if (in_task()) {
+		mutex_lock(&drvdata->tx_mutex);
+		__ps2_gpio_write(serio, val);
+		if (!wait_for_completion_timeout(&drvdata->tx_done,
+						 msecs_to_jiffies(10000)))
+			ret = SERIO_TIMEOUT;
+		mutex_unlock(&drvdata->tx_mutex);
+	} else {
+		__ps2_gpio_write(serio, val);
+	}
+
+	return ret;
+}
+
+static void ps2_gpio_tx_work_fn(struct work_struct *work)
+{
+	struct delayed_work *dwork = to_delayed_work(work);
+	struct ps2_gpio_data *drvdata = container_of(dwork,
+						    struct ps2_gpio_data,
+						    tx_work);
+
+	enable_irq(drvdata->irq);
+	gpiod_direction_output(drvdata->gpio_data, 0);
+	gpiod_direction_input(drvdata->gpio_clk);
+}
+
+static irqreturn_t ps2_gpio_irq_rx(struct ps2_gpio_data *drvdata)
+{
+	unsigned char byte, cnt;
+	int data;
+	int rxflags = 0;
+	static unsigned long old_jiffies;
+
+	byte = drvdata->rx_byte;
+	cnt = drvdata->rx_cnt;
+
+	if (old_jiffies == 0)
+		old_jiffies = jiffies;
+
+	if ((jiffies - old_jiffies) > usecs_to_jiffies(100)) {
+		dev_err(drvdata->dev,
+			"RX: timeout, probably we missed an interrupt\n");
+		goto err;
+	}
+	old_jiffies = jiffies;
+
+	data = gpiod_get_value(drvdata->gpio_data);
+	if (unlikely(data < 0)) {
+		dev_err(drvdata->dev, "RX: failed to get data gpio val: %d\n",
+			data);
+		goto err;
+	}
+
+	switch (cnt) {
+	case PS2_START_BIT:
+		/* start bit should be low */
+		if (unlikely(data)) {
+			dev_err(drvdata->dev, "RX: start bit should be low\n");
+			goto err;
+		}
+		break;
+	case PS2_DATA_BIT0:
+	case PS2_DATA_BIT1:
+	case PS2_DATA_BIT2:
+	case PS2_DATA_BIT3:
+	case PS2_DATA_BIT4:
+	case PS2_DATA_BIT5:
+	case PS2_DATA_BIT6:
+	case PS2_DATA_BIT7:
+		/* processing data bits */
+		if (data)
+			byte |= (data << (cnt - 1));
+		break;
+	case PS2_PARITY_BIT:
+		/* check odd parity */
+		if (!((hweight8(byte) & 1) ^ data)) {
+			rxflags |= SERIO_PARITY;
+			dev_warn(drvdata->dev, "RX: parity error\n");
+			if (!drvdata->write_enable)
+				goto err;
+		}
+
+		/* Do not send spurious ACK's and NACK's when write fn is
+		 * not provided.
+		 */
+		if (!drvdata->write_enable) {
+			if (byte == PS2_DEV_RET_NACK)
+				goto err;
+			else if (byte == PS2_DEV_RET_ACK)
+				break;
+		}
+
+		/* Let's send the data without waiting for the stop bit to be
+		 * sent. It may happen that we miss the stop bit. When this
+		 * happens we have no way to recover from this, certainly
+		 * missing the parity bit would be recognized when processing
+		 * the stop bit. When missing both, data is lost.
+		 */
+		serio_interrupt(drvdata->serio, byte, rxflags);
+		dev_dbg(drvdata->dev, "RX: sending byte 0x%x\n", byte);
+		break;
+	case PS2_STOP_BIT:
+		/* stop bit should be high */
+		if (unlikely(!data)) {
+			dev_err(drvdata->dev, "RX: stop bit should be high\n");
+			goto err;
+		}
+		cnt = byte = 0;
+		old_jiffies = 0;
+		goto end; /* success */
+	default:
+		dev_err(drvdata->dev, "RX: got out of sync with the device\n");
+		goto err;
+	}
+
+	cnt++;
+	goto end; /* success */
+
+err:
+	cnt = byte = 0;
+	old_jiffies = 0;
+	__ps2_gpio_write(drvdata->serio, PS2_CMD_RESEND);
+end:
+	drvdata->rx_cnt = cnt;
+	drvdata->rx_byte = byte;
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t ps2_gpio_irq_tx(struct ps2_gpio_data *drvdata)
+{
+	unsigned char byte, cnt;
+	int data;
+	static unsigned long old_jiffies;
+
+	cnt = drvdata->tx_cnt;
+	byte = drvdata->tx_byte;
+
+	if (old_jiffies == 0)
+		old_jiffies = jiffies;
+
+	if ((jiffies - old_jiffies) > usecs_to_jiffies(100)) {
+		dev_err(drvdata->dev,
+			"TX: timeout, probably we missed an interrupt\n");
+		goto err;
+	}
+	old_jiffies = jiffies;
+
+	switch (cnt) {
+	case PS2_START_BIT:
+		/* should never happen */
+		dev_err(drvdata->dev,
+			"TX: start bit should have been sent already\n");
+		goto err;
+	case PS2_DATA_BIT0:
+	case PS2_DATA_BIT1:
+	case PS2_DATA_BIT2:
+	case PS2_DATA_BIT3:
+	case PS2_DATA_BIT4:
+	case PS2_DATA_BIT5:
+	case PS2_DATA_BIT6:
+	case PS2_DATA_BIT7:
+		data = byte & BIT(cnt - 1);
+		gpiod_set_value(drvdata->gpio_data, data);
+		break;
+	case PS2_PARITY_BIT:
+		/* do odd parity */
+		data = !(hweight8(byte) & 1);
+		gpiod_set_value(drvdata->gpio_data, data);
+		break;
+	case PS2_STOP_BIT:
+		/* release data line to generate stop bit */
+		gpiod_direction_input(drvdata->gpio_data);
+		break;
+	case PS2_TX_TIMEOUT:
+		/* Devices generate one extra clock pulse before sending the
+		 * acknowledgment.
+		 */
+		break;
+	case PS2_ACK_BIT:
+		gpiod_direction_input(drvdata->gpio_data);
+		data = gpiod_get_value(drvdata->gpio_data);
+		if (data) {
+			dev_warn(drvdata->dev, "TX: received NACK, retry\n");
+			goto err;
+		}
+
+		drvdata->mode = PS2_MODE_RX;
+		complete(&drvdata->tx_done);
+
+		cnt = 1;
+		old_jiffies = 0;
+		goto end; /* success */
+	default:
+		/* Probably we missed the stop bit. Therefore we release data
+		 * line and try again.
+		 */
+		gpiod_direction_input(drvdata->gpio_data);
+		dev_err(drvdata->dev, "TX: got out of sync with the device\n");
+		goto err;
+	}
+
+	cnt++;
+	goto end; /* success */
+
+err:
+	cnt = 1;
+	old_jiffies = 0;
+	gpiod_direction_input(drvdata->gpio_data);
+	__ps2_gpio_write(drvdata->serio, drvdata->tx_byte);
+end:
+	drvdata->tx_cnt = cnt;
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t ps2_gpio_irq(int irq, void *dev_id)
+{
+	struct ps2_gpio_data *drvdata = dev_id;
+
+	return drvdata->mode ? ps2_gpio_irq_tx(drvdata) :
+		ps2_gpio_irq_rx(drvdata);
+}
+
+static int ps2_gpio_get_props(struct device *dev,
+				 struct ps2_gpio_data *drvdata)
+{
+	drvdata->gpio_data = devm_gpiod_get(dev, "data", GPIOD_IN);
+	if (IS_ERR(drvdata->gpio_data)) {
+		dev_err(dev, "failed to request data gpio: %ld",
+			PTR_ERR(drvdata->gpio_data));
+		return PTR_ERR(drvdata->gpio_data);
+	}
+
+	drvdata->gpio_clk = devm_gpiod_get(dev, "clk", GPIOD_IN);
+	if (IS_ERR(drvdata->gpio_clk)) {
+		dev_err(dev, "failed to request clock gpio: %ld",
+			PTR_ERR(drvdata->gpio_clk));
+		return PTR_ERR(drvdata->gpio_clk);
+	}
+
+	drvdata->write_enable = device_property_read_bool(dev,
+				"write-enable");
+
+	return 0;
+}
+
+static int ps2_gpio_probe(struct platform_device *pdev)
+{
+	struct ps2_gpio_data *drvdata;
+	struct serio *serio;
+	struct device *dev = &pdev->dev;
+	int error;
+
+	drvdata = devm_kzalloc(dev, sizeof(struct ps2_gpio_data), GFP_KERNEL);
+	serio = kzalloc(sizeof(struct serio), GFP_KERNEL);
+	if (!drvdata || !serio) {
+		error = -ENOMEM;
+		goto err_free_serio;
+	}
+
+	error = ps2_gpio_get_props(dev, drvdata);
+	if (error)
+		goto err_free_serio;
+
+	if (gpiod_cansleep(drvdata->gpio_data) ||
+	    gpiod_cansleep(drvdata->gpio_clk)) {
+		dev_err(dev, "GPIO data or clk are connected via slow bus\n");
+		error = -EINVAL;
+	}
+
+	drvdata->irq = platform_get_irq(pdev, 0);
+	if (drvdata->irq < 0) {
+		dev_err(dev, "failed to get irq from platform resource: %d\n",
+			drvdata->irq);
+		error = drvdata->irq;
+		goto err_free_serio;
+	}
+
+	error = devm_request_irq(dev, drvdata->irq, ps2_gpio_irq,
+				 IRQF_NO_THREAD, DRIVER_NAME, drvdata);
+	if (error) {
+		dev_err(dev, "failed to request irq %d: %d\n",
+			drvdata->irq, error);
+		goto err_free_serio;
+	}
+
+	/* Keep irq disabled until serio->open is called. */
+	disable_irq(drvdata->irq);
+
+	serio->id.type = SERIO_8042;
+	serio->open = ps2_gpio_open;
+	serio->close = ps2_gpio_close;
+	/* Write can be enabled in platform/dt data, but possibly it will not
+	 * work because of the tough timings.
+	 */
+	serio->write = drvdata->write_enable ? ps2_gpio_write : NULL;
+	serio->port_data = drvdata;
+	serio->dev.parent = dev;
+	strlcpy(serio->name, dev_name(dev), sizeof(serio->name));
+	strlcpy(serio->phys, dev_name(dev), sizeof(serio->phys));
+
+	drvdata->serio = serio;
+	drvdata->dev = dev;
+	drvdata->mode = PS2_MODE_RX;
+
+	/* Tx count always starts at 1, as the start bit is sent implicitly by
+	 * host-to-device communication initialization.
+	 */
+	drvdata->tx_cnt = 1;
+
+	INIT_DELAYED_WORK(&drvdata->tx_work, ps2_gpio_tx_work_fn);
+	init_completion(&drvdata->tx_done);
+	mutex_init(&drvdata->tx_mutex);
+
+	serio_register_port(serio);
+	platform_set_drvdata(pdev, drvdata);
+
+	return 0;	/* success */
+
+err_free_serio:
+	kfree(serio);
+	return error;
+}
+
+static int ps2_gpio_remove(struct platform_device *pdev)
+{
+	struct ps2_gpio_data *drvdata = platform_get_drvdata(pdev);
+
+	serio_unregister_port(drvdata->serio);
+	return 0;
+}
+
+#if defined(CONFIG_OF)
+static const struct of_device_id ps2_gpio_match[] = {
+	{ .compatible = "ps2-gpio", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, ps2_gpio_match);
+#endif
+
+static struct platform_driver ps2_gpio_driver = {
+	.probe		= ps2_gpio_probe,
+	.remove		= ps2_gpio_remove,
+	.driver = {
+		.name = DRIVER_NAME,
+		.of_match_table = of_match_ptr(ps2_gpio_match),
+	},
+};
+module_platform_driver(ps2_gpio_driver);
+
+MODULE_AUTHOR("Danilo Krummrich <danilokrummrich@dk-develop.de>");
+MODULE_DESCRIPTION("GPIO PS2 driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/input/serio/serio.c b/drivers/input/serio/serio.c
index 30d6230d48f7..24a90c8db5b3 100644
--- a/drivers/input/serio/serio.c
+++ b/drivers/input/serio/serio.c
@@ -469,7 +469,7 @@ static struct attribute *serio_device_id_attrs[] = {
 	NULL
 };
 
-static struct attribute_group serio_id_attr_group = {
+static const struct attribute_group serio_id_attr_group = {
 	.name	= "id",
 	.attrs	= serio_device_id_attrs,
 };
@@ -489,7 +489,7 @@ static struct attribute *serio_device_attrs[] = {
 	NULL
 };
 
-static struct attribute_group serio_device_attr_group = {
+static const struct attribute_group serio_device_attr_group = {
 	.attrs	= serio_device_attrs,
 };
 
diff --git a/drivers/input/serio/serio_raw.c b/drivers/input/serio/serio_raw.c
index 71ef5d65a0c6..516f9fe77a17 100644
--- a/drivers/input/serio/serio_raw.c
+++ b/drivers/input/serio/serio_raw.c
@@ -410,7 +410,7 @@ static void serio_raw_disconnect(struct serio *serio)
 	serio_set_drvdata(serio, NULL);
 }
 
-static struct serio_device_id serio_raw_serio_ids[] = {
+static const struct serio_device_id serio_raw_serio_ids[] = {
 	{
 		.type	= SERIO_8042,
 		.proto	= SERIO_ANY,
diff --git a/drivers/input/serio/xilinx_ps2.c b/drivers/input/serio/xilinx_ps2.c
index 14c40892ed82..07de1b49293c 100644
--- a/drivers/input/serio/xilinx_ps2.c
+++ b/drivers/input/serio/xilinx_ps2.c
@@ -45,8 +45,10 @@
 #define XPS2_STATUS_RX_FULL	0x00000001 /* Receive Full  */
 #define XPS2_STATUS_TX_FULL	0x00000002 /* Transmit Full  */
 
-/* Bit definitions for ISR/IER registers. Both the registers have the same bit
- * definitions and are only defined once. */
+/*
+ * Bit definitions for ISR/IER registers. Both the registers have the same bit
+ * definitions and are only defined once.
+ */
 #define XPS2_IPIXR_WDT_TOUT	0x00000001 /* Watchdog Timeout Interrupt */
 #define XPS2_IPIXR_TX_NOACK	0x00000002 /* Transmit No ACK Interrupt */
 #define XPS2_IPIXR_TX_ACK	0x00000004 /* Transmit ACK (Data) Interrupt */
@@ -292,8 +294,10 @@ static int xps2_of_probe(struct platform_device *ofdev)
 	/* Disable all the interrupts, just in case */
 	out_be32(drvdata->base_address + XPS2_IPIER_OFFSET, 0);
 
-	/* Reset the PS2 device and abort any current transaction, to make sure
-	 * we have the PS2 in a good state */
+	/*
+	 * Reset the PS2 device and abort any current transaction,
+	 * to make sure we have the PS2 in a good state.
+	 */
 	out_be32(drvdata->base_address + XPS2_SRST_OFFSET, XPS2_SRST_RESET);
 
 	dev_info(dev, "Xilinx PS2 at 0x%08llX mapped to 0x%p, irq=%d\n",
diff --git a/drivers/input/tablet/acecad.c b/drivers/input/tablet/acecad.c
index e86e377a90f5..aebb3f9090cd 100644
--- a/drivers/input/tablet/acecad.c
+++ b/drivers/input/tablet/acecad.c
@@ -260,7 +260,7 @@ static void usb_acecad_disconnect(struct usb_interface *intf)
 	kfree(acecad);
 }
 
-static struct usb_device_id usb_acecad_id_table [] = {
+static const struct usb_device_id usb_acecad_id_table[] = {
 	{ USB_DEVICE(USB_VENDOR_ID_ACECAD, USB_DEVICE_ID_FLAIR), .driver_info = 0 },
 	{ USB_DEVICE(USB_VENDOR_ID_ACECAD, USB_DEVICE_ID_302),	 .driver_info = 1 },
 	{ }
diff --git a/drivers/input/tablet/aiptek.c b/drivers/input/tablet/aiptek.c
index d67547bded3e..0b55e1f375b3 100644
--- a/drivers/input/tablet/aiptek.c
+++ b/drivers/input/tablet/aiptek.c
@@ -1676,7 +1676,7 @@ static struct attribute *aiptek_attributes[] = {
 	NULL
 };
 
-static struct attribute_group aiptek_attribute_group = {
+static const struct attribute_group aiptek_attribute_group = {
 	.attrs	= aiptek_attributes,
 };
 
diff --git a/drivers/input/tablet/kbtab.c b/drivers/input/tablet/kbtab.c
index 4d9d64908b59..a41c3ff7c9af 100644
--- a/drivers/input/tablet/kbtab.c
+++ b/drivers/input/tablet/kbtab.c
@@ -88,7 +88,7 @@ static void kbtab_irq(struct urb *urb)
 			__func__, retval);
 }
 
-static struct usb_device_id kbtab_ids[] = {
+static const struct usb_device_id kbtab_ids[] = {
 	{ USB_DEVICE(USB_VENDOR_ID_KBGEAR, 0x1001), .driver_info = 0 },
 	{ }
 };
diff --git a/drivers/input/tablet/wacom_serial4.c b/drivers/input/tablet/wacom_serial4.c
index 20ab802461e7..38bfaca48eab 100644
--- a/drivers/input/tablet/wacom_serial4.c
+++ b/drivers/input/tablet/wacom_serial4.c
@@ -594,7 +594,7 @@ free_device:
 	return err;
 }
 
-static struct serio_device_id wacom_serio_ids[] = {
+static const struct serio_device_id wacom_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_WACOM_IV,
diff --git a/drivers/input/touchscreen/ads7846.c b/drivers/input/touchscreen/ads7846.c
index 735a0be1ad95..a2f45aefce08 100644
--- a/drivers/input/touchscreen/ads7846.c
+++ b/drivers/input/touchscreen/ads7846.c
@@ -499,7 +499,7 @@ static struct attribute *ads7846_attributes[] = {
 	NULL,
 };
 
-static struct attribute_group ads7846_attr_group = {
+static const struct attribute_group ads7846_attr_group = {
 	.attrs = ads7846_attributes,
 	.is_visible = ads7846_is_visible,
 };
@@ -599,7 +599,7 @@ static struct attribute *ads784x_attributes[] = {
 	NULL,
 };
 
-static struct attribute_group ads784x_attr_group = {
+static const struct attribute_group ads784x_attr_group = {
 	.attrs = ads784x_attributes,
 };
 
diff --git a/drivers/input/touchscreen/atmel_mxt_ts.c b/drivers/input/touchscreen/atmel_mxt_ts.c
index dd042a9b0aaa..7659bc48f1db 100644
--- a/drivers/input/touchscreen/atmel_mxt_ts.c
+++ b/drivers/input/touchscreen/atmel_mxt_ts.c
@@ -28,6 +28,7 @@
 #include <linux/interrupt.h>
 #include <linux/of.h>
 #include <linux/slab.h>
+#include <linux/gpio/consumer.h>
 #include <asm/unaligned.h>
 #include <media/v4l2-device.h>
 #include <media/v4l2-ioctl.h>
@@ -300,6 +301,7 @@ struct mxt_data {
 	u8 multitouch;
 	struct t7_config t7_cfg;
 	struct mxt_dbg dbg;
+	struct gpio_desc *reset_gpio;
 
 	/* Cached parameters from object table */
 	u16 T5_address;
@@ -3117,11 +3119,9 @@ static int mxt_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	if (IS_ERR(pdata))
 		return PTR_ERR(pdata);
 
-	data = kzalloc(sizeof(struct mxt_data), GFP_KERNEL);
-	if (!data) {
-		dev_err(&client->dev, "Failed to allocate memory\n");
+	data = devm_kzalloc(&client->dev, sizeof(struct mxt_data), GFP_KERNEL);
+	if (!data)
 		return -ENOMEM;
-	}
 
 	snprintf(data->phys, sizeof(data->phys), "i2c-%u-%04x/input0",
 		 client->adapter->nr, client->addr);
@@ -3135,19 +3135,40 @@ static int mxt_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	init_completion(&data->reset_completion);
 	init_completion(&data->crc_completion);
 
-	error = request_threaded_irq(client->irq, NULL, mxt_interrupt,
-				     pdata->irqflags | IRQF_ONESHOT,
-				     client->name, data);
+	data->reset_gpio = devm_gpiod_get_optional(&client->dev,
+						   "reset", GPIOD_OUT_LOW);
+	if (IS_ERR(data->reset_gpio)) {
+		error = PTR_ERR(data->reset_gpio);
+		dev_err(&client->dev, "Failed to get reset gpio: %d\n", error);
+		return error;
+	}
+
+	error = devm_request_threaded_irq(&client->dev, client->irq,
+					  NULL, mxt_interrupt,
+					  pdata->irqflags | IRQF_ONESHOT,
+					  client->name, data);
 	if (error) {
 		dev_err(&client->dev, "Failed to register interrupt\n");
-		goto err_free_mem;
+		return error;
+	}
+
+	if (data->reset_gpio) {
+		data->in_bootloader = true;
+		msleep(MXT_RESET_TIME);
+		reinit_completion(&data->bl_completion);
+		gpiod_set_value(data->reset_gpio, 1);
+		error = mxt_wait_for_completion(data, &data->bl_completion,
+						MXT_RESET_TIMEOUT);
+		if (error)
+			return error;
+		data->in_bootloader = false;
 	}
 
 	disable_irq(client->irq);
 
 	error = mxt_initialize(data);
 	if (error)
-		goto err_free_irq;
+		return error;
 
 	error = sysfs_create_group(&client->dev.kobj, &mxt_attr_group);
 	if (error) {
@@ -3161,10 +3182,6 @@ static int mxt_probe(struct i2c_client *client, const struct i2c_device_id *id)
 err_free_object:
 	mxt_free_input_device(data);
 	mxt_free_object_table(data);
-err_free_irq:
-	free_irq(client->irq, data);
-err_free_mem:
-	kfree(data);
 	return error;
 }
 
@@ -3172,11 +3189,10 @@ static int mxt_remove(struct i2c_client *client)
 {
 	struct mxt_data *data = i2c_get_clientdata(client);
 
+	disable_irq(data->irq);
 	sysfs_remove_group(&client->dev.kobj, &mxt_attr_group);
-	free_irq(data->irq, data);
 	mxt_free_input_device(data);
 	mxt_free_object_table(data);
-	kfree(data);
 
 	return 0;
 }
diff --git a/drivers/input/touchscreen/dynapro.c b/drivers/input/touchscreen/dynapro.c
index 86237a910876..5b1b66fffbe3 100644
--- a/drivers/input/touchscreen/dynapro.c
+++ b/drivers/input/touchscreen/dynapro.c
@@ -164,7 +164,7 @@ static int dynapro_connect(struct serio *serio, struct serio_driver *drv)
  * The serio driver structure.
  */
 
-static struct serio_device_id dynapro_serio_ids[] = {
+static const struct serio_device_id dynapro_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_DYNAPRO,
diff --git a/drivers/input/touchscreen/edt-ft5x06.c b/drivers/input/touchscreen/edt-ft5x06.c
index f872817e81e4..5bf63f76ddda 100644
--- a/drivers/input/touchscreen/edt-ft5x06.c
+++ b/drivers/input/touchscreen/edt-ft5x06.c
@@ -593,7 +593,7 @@ static int edt_ft5x06_work_mode(struct edt_ft5x06_ts_data *tsdata)
 				  tsdata->gain);
 	edt_ft5x06_register_write(tsdata, reg_addr->reg_offset,
 				  tsdata->offset);
-	if (reg_addr->reg_report_rate)
+	if (reg_addr->reg_report_rate != NO_REGISTER)
 		edt_ft5x06_register_write(tsdata, reg_addr->reg_report_rate,
 				  tsdata->report_rate);
 
@@ -874,6 +874,7 @@ edt_ft5x06_ts_set_regs(struct edt_ft5x06_ts_data *tsdata)
 
 	case M09:
 		reg_addr->reg_threshold = M09_REGISTER_THRESHOLD;
+		reg_addr->reg_report_rate = NO_REGISTER;
 		reg_addr->reg_gain = M09_REGISTER_GAIN;
 		reg_addr->reg_offset = M09_REGISTER_OFFSET;
 		reg_addr->reg_num_x = M09_REGISTER_NUM_X;
diff --git a/drivers/input/touchscreen/elants_i2c.c b/drivers/input/touchscreen/elants_i2c.c
index 872750eeca93..0f4cda7282a2 100644
--- a/drivers/input/touchscreen/elants_i2c.c
+++ b/drivers/input/touchscreen/elants_i2c.c
@@ -1066,7 +1066,7 @@ static struct attribute *elants_attributes[] = {
 	NULL
 };
 
-static struct attribute_group elants_attribute_group = {
+static const struct attribute_group elants_attribute_group = {
 	.attrs = elants_attributes,
 };
 
diff --git a/drivers/input/touchscreen/elo.c b/drivers/input/touchscreen/elo.c
index 8051a4b704ea..83433e8efff7 100644
--- a/drivers/input/touchscreen/elo.c
+++ b/drivers/input/touchscreen/elo.c
@@ -381,7 +381,7 @@ static int elo_connect(struct serio *serio, struct serio_driver *drv)
  * The serio driver structure.
  */
 
-static struct serio_device_id elo_serio_ids[] = {
+static const struct serio_device_id elo_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_ELO,
diff --git a/drivers/input/touchscreen/fujitsu_ts.c b/drivers/input/touchscreen/fujitsu_ts.c
index d0e46a7e183b..a0fbb454499d 100644
--- a/drivers/input/touchscreen/fujitsu_ts.c
+++ b/drivers/input/touchscreen/fujitsu_ts.c
@@ -151,7 +151,7 @@ static int fujitsu_connect(struct serio *serio, struct serio_driver *drv)
 /*
  * The serio driver structure.
  */
-static struct serio_device_id fujitsu_serio_ids[] = {
+static const struct serio_device_id fujitsu_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_FUJITSU,
diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c
index 240b16f3ee97..32d2762448aa 100644
--- a/drivers/input/touchscreen/goodix.c
+++ b/drivers/input/touchscreen/goodix.c
@@ -267,6 +267,12 @@ static void goodix_process_events(struct goodix_ts_data *ts)
 	if (touch_num < 0)
 		return;
 
+	/*
+	 * Bit 4 of the first byte reports the status of the capacitive
+	 * Windows/Home button.
+	 */
+	input_report_key(ts->input_dev, KEY_LEFTMETA, point_data[0] & BIT(4));
+
 	for (i = 0; i < touch_num; i++)
 		goodix_ts_report_touch(ts,
 				&point_data[1 + GOODIX_CONTACT_SIZE * i]);
@@ -612,6 +618,9 @@ static int goodix_request_input_dev(struct goodix_ts_data *ts)
 	ts->input_dev->id.product = ts->id;
 	ts->input_dev->id.version = ts->version;
 
+	/* Capacitive Windows/Home button on some devices */
+	input_set_capability(ts->input_dev, EV_KEY, KEY_LEFTMETA);
+
 	error = input_register_device(ts->input_dev);
 	if (error) {
 		dev_err(&ts->client->dev,
diff --git a/drivers/input/touchscreen/gunze.c b/drivers/input/touchscreen/gunze.c
index e2ee62615273..481586909d28 100644
--- a/drivers/input/touchscreen/gunze.c
+++ b/drivers/input/touchscreen/gunze.c
@@ -162,7 +162,7 @@ static int gunze_connect(struct serio *serio, struct serio_driver *drv)
  * The serio driver structure.
  */
 
-static struct serio_device_id gunze_serio_ids[] = {
+static const struct serio_device_id gunze_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_GUNZE,
diff --git a/drivers/input/touchscreen/hampshire.c b/drivers/input/touchscreen/hampshire.c
index ecb1e0e01328..eb052d559e54 100644
--- a/drivers/input/touchscreen/hampshire.c
+++ b/drivers/input/touchscreen/hampshire.c
@@ -163,7 +163,7 @@ static int hampshire_connect(struct serio *serio, struct serio_driver *drv)
  * The serio driver structure.
  */
 
-static struct serio_device_id hampshire_serio_ids[] = {
+static const struct serio_device_id hampshire_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_HAMPSHIRE,
diff --git a/drivers/input/touchscreen/htcpen.c b/drivers/input/touchscreen/htcpen.c
index 92e2243fb77d..8fd909285877 100644
--- a/drivers/input/touchscreen/htcpen.c
+++ b/drivers/input/touchscreen/htcpen.c
@@ -219,7 +219,7 @@ static struct isa_driver htcpen_isa_driver = {
 	}
 };
 
-static struct dmi_system_id htcshift_dmi_table[] __initdata = {
+static const struct dmi_system_id htcshift_dmi_table[] __initconst = {
 	{
 		.ident = "Shift",
 		.matches = {
diff --git a/drivers/input/touchscreen/inexio.c b/drivers/input/touchscreen/inexio.c
index adb80b65a259..b9bc56233ccc 100644
--- a/drivers/input/touchscreen/inexio.c
+++ b/drivers/input/touchscreen/inexio.c
@@ -165,7 +165,7 @@ static int inexio_connect(struct serio *serio, struct serio_driver *drv)
  * The serio driver structure.
  */
 
-static struct serio_device_id inexio_serio_ids[] = {
+static const struct serio_device_id inexio_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_INEXIO,
diff --git a/drivers/input/touchscreen/mtouch.c b/drivers/input/touchscreen/mtouch.c
index 9b5552a26169..a3707fad4d1c 100644
--- a/drivers/input/touchscreen/mtouch.c
+++ b/drivers/input/touchscreen/mtouch.c
@@ -178,7 +178,7 @@ static int mtouch_connect(struct serio *serio, struct serio_driver *drv)
  * The serio driver structure.
  */
 
-static struct serio_device_id mtouch_serio_ids[] = {
+static const struct serio_device_id mtouch_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_MICROTOUCH,
diff --git a/drivers/input/touchscreen/mxs-lradc-ts.c b/drivers/input/touchscreen/mxs-lradc-ts.c
index 58c016cd6809..3707e927f770 100644
--- a/drivers/input/touchscreen/mxs-lradc-ts.c
+++ b/drivers/input/touchscreen/mxs-lradc-ts.c
@@ -30,7 +30,7 @@
 #include <linux/of_irq.h>
 #include <linux/platform_device.h>
 
-const char *mxs_lradc_ts_irq_names[] = {
+static const char * const mxs_lradc_ts_irq_names[] = {
 	"mxs-lradc-touchscreen",
 	"mxs-lradc-channel6",
 	"mxs-lradc-channel7",
@@ -630,9 +630,11 @@ static int mxs_lradc_ts_probe(struct platform_device *pdev)
 	spin_lock_init(&ts->lock);
 
 	iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!iores)
+		return -EINVAL;
 	ts->base = devm_ioremap(dev, iores->start, resource_size(iores));
-	if (IS_ERR(ts->base))
-		return PTR_ERR(ts->base);
+	if (!ts->base)
+		return -ENOMEM;
 
 	ret = of_property_read_u32(node, "fsl,lradc-touchscreen-wires",
 				   &ts_wires);
diff --git a/drivers/input/touchscreen/penmount.c b/drivers/input/touchscreen/penmount.c
index 417d87379265..6e6d7fd98cd2 100644
--- a/drivers/input/touchscreen/penmount.c
+++ b/drivers/input/touchscreen/penmount.c
@@ -293,7 +293,7 @@ static int pm_connect(struct serio *serio, struct serio_driver *drv)
  * The serio driver structure.
  */
 
-static struct serio_device_id pm_serio_ids[] = {
+static const struct serio_device_id pm_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_PENMOUNT,
diff --git a/drivers/input/touchscreen/raydium_i2c_ts.c b/drivers/input/touchscreen/raydium_i2c_ts.c
index 1252e49ccfa1..4f1d3fd5d412 100644
--- a/drivers/input/touchscreen/raydium_i2c_ts.c
+++ b/drivers/input/touchscreen/raydium_i2c_ts.c
@@ -939,7 +939,7 @@ static struct attribute *raydium_i2c_attributes[] = {
 	NULL
 };
 
-static struct attribute_group raydium_i2c_attribute_group = {
+static const struct attribute_group raydium_i2c_attribute_group = {
 	.attrs = raydium_i2c_attributes,
 };
 
diff --git a/drivers/input/touchscreen/sun4i-ts.c b/drivers/input/touchscreen/sun4i-ts.c
index d07dd29d4848..d2e14d9e5975 100644
--- a/drivers/input/touchscreen/sun4i-ts.c
+++ b/drivers/input/touchscreen/sun4i-ts.c
@@ -206,7 +206,7 @@ static int sun4i_get_tz_temp(void *data, int *temp)
 	return sun4i_get_temp(data, temp);
 }
 
-static struct thermal_zone_of_device_ops sun4i_ts_tz_ops = {
+static const struct thermal_zone_of_device_ops sun4i_ts_tz_ops = {
 	.get_temp = sun4i_get_tz_temp,
 };
 
diff --git a/drivers/input/touchscreen/sur40.c b/drivers/input/touchscreen/sur40.c
index 128e5bd74720..f16f8358c70a 100644
--- a/drivers/input/touchscreen/sur40.c
+++ b/drivers/input/touchscreen/sur40.c
@@ -59,7 +59,7 @@ struct sur40_blob {
 	__le16 blob_id;
 
 	u8 action;         /* 0x02 = enter/exit, 0x03 = update (?) */
-	u8 unknown;        /* always 0x01 or 0x02 (no idea what this is?) */
+	u8 type;           /* bitmask (0x01 blob,  0x02 touch, 0x04 tag) */
 
 	__le16 bb_pos_x;   /* upper left corner of bounding box */
 	__le16 bb_pos_y;
@@ -133,12 +133,19 @@ struct sur40_image_header {
 
 /* control commands */
 #define SUR40_GET_VERSION 0xb0 /* 12 bytes string    */
-#define SUR40_UNKNOWN1    0xb3 /*  5 bytes           */
-#define SUR40_UNKNOWN2    0xc1 /* 24 bytes           */
+#define SUR40_ACCEL_CAPS  0xb3 /*  5 bytes           */
+#define SUR40_SENSOR_CAPS 0xc1 /* 24 bytes           */
+
+#define SUR40_POKE        0xc5 /* poke register byte */
+#define SUR40_PEEK        0xc4 /* 48 bytes registers */
 
 #define SUR40_GET_STATE   0xc5 /*  4 bytes state (?) */
 #define SUR40_GET_SENSORS 0xb1 /*  8 bytes sensors   */
 
+#define SUR40_BLOB	0x01
+#define SUR40_TOUCH	0x02
+#define SUR40_TAG	0x04
+
 static const struct v4l2_pix_format sur40_pix_format[] = {
 	{
 		.pixelformat = V4L2_TCH_FMT_TU08,
@@ -238,11 +245,11 @@ static int sur40_init(struct sur40_state *dev)
 	if (result < 0)
 		goto error;
 
-	result = sur40_command(dev, SUR40_UNKNOWN2,    0x00, buffer, 24);
+	result = sur40_command(dev, SUR40_SENSOR_CAPS, 0x00, buffer, 24);
 	if (result < 0)
 		goto error;
 
-	result = sur40_command(dev, SUR40_UNKNOWN1,    0x00, buffer,  5);
+	result = sur40_command(dev, SUR40_ACCEL_CAPS, 0x00, buffer, 5);
 	if (result < 0)
 		goto error;
 
@@ -289,20 +296,24 @@ static void sur40_close(struct input_polled_dev *polldev)
 static void sur40_report_blob(struct sur40_blob *blob, struct input_dev *input)
 {
 	int wide, major, minor;
+	int bb_size_x, bb_size_y, pos_x, pos_y, ctr_x, ctr_y, slotnum;
 
-	int bb_size_x = le16_to_cpu(blob->bb_size_x);
-	int bb_size_y = le16_to_cpu(blob->bb_size_y);
-
-	int pos_x = le16_to_cpu(blob->pos_x);
-	int pos_y = le16_to_cpu(blob->pos_y);
-
-	int ctr_x = le16_to_cpu(blob->ctr_x);
-	int ctr_y = le16_to_cpu(blob->ctr_y);
+	if (blob->type != SUR40_TOUCH)
+		return;
 
-	int slotnum = input_mt_get_slot_by_key(input, blob->blob_id);
+	slotnum = input_mt_get_slot_by_key(input, blob->blob_id);
 	if (slotnum < 0 || slotnum >= MAX_CONTACTS)
 		return;
 
+	bb_size_x = le16_to_cpu(blob->bb_size_x);
+	bb_size_y = le16_to_cpu(blob->bb_size_y);
+
+	pos_x = le16_to_cpu(blob->pos_x);
+	pos_y = le16_to_cpu(blob->pos_y);
+
+	ctr_x = le16_to_cpu(blob->ctr_x);
+	ctr_y = le16_to_cpu(blob->ctr_y);
+
 	input_mt_slot(input, slotnum);
 	input_mt_report_slot_state(input, MT_TOOL_FINGER, 1);
 	wide = (bb_size_x > bb_size_y);
@@ -367,10 +378,13 @@ static void sur40_poll(struct input_polled_dev *polldev)
 		/*
 		 * Sanity check. when video data is also being retrieved, the
 		 * packet ID will usually increase in the middle of a series
-		 * instead of at the end.
-		 */
+		 * instead of at the end. However, the data is still consistent,
+		 * so the packet ID is probably just valid for the first packet
+		 * in a series.
+
 		if (packet_id != le32_to_cpu(header->packet_id))
 			dev_dbg(sur40->dev, "packet ID mismatch\n");
+		 */
 
 		packet_blobs = result / sizeof(struct sur40_blob);
 		dev_dbg(sur40->dev, "received %d blobs\n", packet_blobs);
diff --git a/drivers/input/touchscreen/surface3_spi.c b/drivers/input/touchscreen/surface3_spi.c
index e12fb9b63f31..5db0f1c4ef38 100644
--- a/drivers/input/touchscreen/surface3_spi.c
+++ b/drivers/input/touchscreen/surface3_spi.c
@@ -173,7 +173,7 @@ static void surface3_spi_process_pen(struct surface3_ts_data *ts_data, u8 *data)
 
 static void surface3_spi_process(struct surface3_ts_data *ts_data)
 {
-	const char header[] = {
+	static const char header[] = {
 		0xff, 0xff, 0xff, 0xff, 0xa5, 0x5a, 0xe7, 0x7e, 0x01
 	};
 	u8 *data = ts_data->rd_buf;
diff --git a/drivers/input/touchscreen/touchit213.c b/drivers/input/touchscreen/touchit213.c
index c27cf8f3d1ca..98a16698be8e 100644
--- a/drivers/input/touchscreen/touchit213.c
+++ b/drivers/input/touchscreen/touchit213.c
@@ -192,7 +192,7 @@ static int touchit213_connect(struct serio *serio, struct serio_driver *drv)
  * The serio driver structure.
  */
 
-static struct serio_device_id touchit213_serio_ids[] = {
+static const struct serio_device_id touchit213_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_TOUCHIT213,
diff --git a/drivers/input/touchscreen/touchright.c b/drivers/input/touchscreen/touchright.c
index 4000e5205407..45c325c33f21 100644
--- a/drivers/input/touchscreen/touchright.c
+++ b/drivers/input/touchscreen/touchright.c
@@ -152,7 +152,7 @@ static int tr_connect(struct serio *serio, struct serio_driver *drv)
  * The serio driver structure.
  */
 
-static struct serio_device_id tr_serio_ids[] = {
+static const struct serio_device_id tr_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_TOUCHRIGHT,
diff --git a/drivers/input/touchscreen/touchwin.c b/drivers/input/touchscreen/touchwin.c
index ba90f447df75..2ba6b4ca28cb 100644
--- a/drivers/input/touchscreen/touchwin.c
+++ b/drivers/input/touchscreen/touchwin.c
@@ -159,7 +159,7 @@ static int tw_connect(struct serio *serio, struct serio_driver *drv)
  * The serio driver structure.
  */
 
-static struct serio_device_id tw_serio_ids[] = {
+static const struct serio_device_id tw_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_TOUCHWIN,
diff --git a/drivers/input/touchscreen/tsc40.c b/drivers/input/touchscreen/tsc40.c
index 29687872cb94..d4ae4ba84c1f 100644
--- a/drivers/input/touchscreen/tsc40.c
+++ b/drivers/input/touchscreen/tsc40.c
@@ -141,7 +141,7 @@ static void tsc_disconnect(struct serio *serio)
 	serio_set_drvdata(serio, NULL);
 }
 
-static struct serio_device_id tsc_serio_ids[] = {
+static const struct serio_device_id tsc_serio_ids[] = {
 	{
 		.type   = SERIO_RS232,
 		.proto  = SERIO_TSC40,
diff --git a/drivers/input/touchscreen/ucb1400_ts.c b/drivers/input/touchscreen/ucb1400_ts.c
index c1e23cfc6155..1a86cbd9326f 100644
--- a/drivers/input/touchscreen/ucb1400_ts.c
+++ b/drivers/input/touchscreen/ucb1400_ts.c
@@ -414,7 +414,7 @@ static int __maybe_unused ucb1400_ts_suspend(struct device *dev)
 	mutex_lock(&idev->mutex);
 
 	if (idev->users)
-		ucb1400_ts_start(ucb);
+		ucb1400_ts_stop(ucb);
 
 	mutex_unlock(&idev->mutex);
 	return 0;
@@ -428,7 +428,7 @@ static int __maybe_unused ucb1400_ts_resume(struct device *dev)
 	mutex_lock(&idev->mutex);
 
 	if (idev->users)
-		ucb1400_ts_stop(ucb);
+		ucb1400_ts_start(ucb);
 
 	mutex_unlock(&idev->mutex);
 	return 0;
diff --git a/drivers/input/touchscreen/wacom_w8001.c b/drivers/input/touchscreen/wacom_w8001.c
index 85e95725d0df..3715d1eace92 100644
--- a/drivers/input/touchscreen/wacom_w8001.c
+++ b/drivers/input/touchscreen/wacom_w8001.c
@@ -681,7 +681,7 @@ fail1:
 	return err;
 }
 
-static struct serio_device_id w8001_serio_ids[] = {
+static const struct serio_device_id w8001_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_W8001,
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index f73ff28f77e2..49bd2ab8c507 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -76,6 +76,8 @@ config IOMMU_DMA
 
 config FSL_PAMU
 	bool "Freescale IOMMU support"
+	depends on PCI
+	depends on PHYS_64BIT
 	depends on PPC_E500MC || (COMPILE_TEST && PPC)
 	select IOMMU_API
 	select GENERIC_ALLOCATOR
@@ -253,6 +255,7 @@ config TEGRA_IOMMU_SMMU
 config EXYNOS_IOMMU
 	bool "Exynos IOMMU Support"
 	depends on ARCH_EXYNOS && MMU
+	depends on !CPU_BIG_ENDIAN # revisit driver if we can enable big-endian ptes
 	select IOMMU_API
 	select ARM_DMA_USE_IOMMU
 	help
@@ -367,4 +370,14 @@ config MTK_IOMMU_V1
 
 	  if unsure, say N here.
 
+config QCOM_IOMMU
+	# Note: iommu drivers cannot (yet?) be built as modules
+	bool "Qualcomm IOMMU Support"
+	depends on ARCH_QCOM || COMPILE_TEST
+	select IOMMU_API
+	select IOMMU_IO_PGTABLE_LPAE
+	select ARM_DMA_USE_IOMMU
+	help
+	  Support for IOMMU on certain Qualcomm SoCs.
+
 endif # IOMMU_SUPPORT
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index 195f7b997d8e..b910aea813a1 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -27,3 +27,4 @@ obj-$(CONFIG_TEGRA_IOMMU_SMMU) += tegra-smmu.o
 obj-$(CONFIG_EXYNOS_IOMMU) += exynos-iommu.o
 obj-$(CONFIG_FSL_PAMU) += fsl_pamu.o fsl_pamu_domain.o
 obj-$(CONFIG_S390_IOMMU) += s390-iommu.o
+obj-$(CONFIG_QCOM_IOMMU) += qcom_iommu.o
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 354cbd6392cd..51f8215877f5 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -103,29 +103,6 @@ int amd_iommu_max_glx_val = -1;
 static const struct dma_map_ops amd_iommu_dma_ops;
 
 /*
- * This struct contains device specific data for the IOMMU
- */
-struct iommu_dev_data {
-	struct list_head list;		  /* For domain->dev_list */
-	struct list_head dev_data_list;	  /* For global dev_data_list */
-	struct protection_domain *domain; /* Domain the device is bound to */
-	u16 devid;			  /* PCI Device ID */
-	u16 alias;			  /* Alias Device ID */
-	bool iommu_v2;			  /* Device can make use of IOMMUv2 */
-	bool passthrough;		  /* Device is identity mapped */
-	struct {
-		bool enabled;
-		int qdep;
-	} ats;				  /* ATS state */
-	bool pri_tlp;			  /* PASID TLB required for
-					     PPR completions */
-	u32 errata;			  /* Bitmap for errata to apply */
-	bool use_vapic;			  /* Enable device to use vapic mode */
-
-	struct ratelimit_state rs;	  /* Ratelimit IOPF messages */
-};
-
-/*
  * general struct to manage commands send to an IOMMU
  */
 struct iommu_cmd {
@@ -137,20 +114,7 @@ struct kmem_cache *amd_iommu_irq_cache;
 static void update_domain(struct protection_domain *domain);
 static int protection_domain_init(struct protection_domain *domain);
 static void detach_device(struct device *dev);
-
-#define FLUSH_QUEUE_SIZE 256
-
-struct flush_queue_entry {
-	unsigned long iova_pfn;
-	unsigned long pages;
-	u64 counter; /* Flush counter when this entry was added to the queue */
-};
-
-struct flush_queue {
-	struct flush_queue_entry *entries;
-	unsigned head, tail;
-	spinlock_t lock;
-};
+static void iova_domain_flush_tlb(struct iova_domain *iovad);
 
 /*
  * Data container for a dma_ops specific protection domain
@@ -161,36 +125,6 @@ struct dma_ops_domain {
 
 	/* IOVA RB-Tree */
 	struct iova_domain iovad;
-
-	struct flush_queue __percpu *flush_queue;
-
-	/*
-	 * We need two counter here to be race-free wrt. IOTLB flushing and
-	 * adding entries to the flush queue.
-	 *
-	 * The flush_start_cnt is incremented _before_ the IOTLB flush starts.
-	 * New entries added to the flush ring-buffer get their 'counter' value
-	 * from here. This way we can make sure that entries added to the queue
-	 * (or other per-cpu queues of the same domain) while the TLB is about
-	 * to be flushed are not considered to be flushed already.
-	 */
-	atomic64_t flush_start_cnt;
-
-	/*
-	 * The flush_finish_cnt is incremented when an IOTLB flush is complete.
-	 * This value is always smaller than flush_start_cnt. The queue_add
-	 * function frees all IOVAs that have a counter value smaller than
-	 * flush_finish_cnt. This makes sure that we only free IOVAs that are
-	 * flushed out of the IOTLB of the domain.
-	 */
-	atomic64_t flush_finish_cnt;
-
-	/*
-	 * Timer to make sure we don't keep IOVAs around unflushed
-	 * for too long
-	 */
-	struct timer_list flush_timer;
-	atomic_t flush_timer_on;
 };
 
 static struct iova_domain reserved_iova_ranges;
@@ -371,19 +305,25 @@ static u16 get_alias(struct device *dev)
 static struct iommu_dev_data *find_dev_data(u16 devid)
 {
 	struct iommu_dev_data *dev_data;
+	struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
 
 	dev_data = search_dev_data(devid);
 
-	if (dev_data == NULL)
+	if (dev_data == NULL) {
 		dev_data = alloc_dev_data(devid);
 
+		if (translation_pre_enabled(iommu))
+			dev_data->defer_attach = true;
+	}
+
 	return dev_data;
 }
 
-static struct iommu_dev_data *get_dev_data(struct device *dev)
+struct iommu_dev_data *get_dev_data(struct device *dev)
 {
 	return dev->archdata.iommu;
 }
+EXPORT_SYMBOL(get_dev_data);
 
 /*
 * Find or create an IOMMU group for a acpihid device.
@@ -575,7 +515,7 @@ static void dump_dte_entry(u16 devid)
 
 static void dump_command(unsigned long phys_addr)
 {
-	struct iommu_cmd *cmd = phys_to_virt(phys_addr);
+	struct iommu_cmd *cmd = iommu_phys_to_virt(phys_addr);
 	int i;
 
 	for (i = 0; i < 4; ++i)
@@ -919,11 +859,13 @@ static void copy_cmd_to_buffer(struct amd_iommu *iommu,
 
 static void build_completion_wait(struct iommu_cmd *cmd, u64 address)
 {
+	u64 paddr = iommu_virt_to_phys((void *)address);
+
 	WARN_ON(address & 0x7ULL);
 
 	memset(cmd, 0, sizeof(*cmd));
-	cmd->data[0] = lower_32_bits(__pa(address)) | CMD_COMPL_WAIT_STORE_MASK;
-	cmd->data[1] = upper_32_bits(__pa(address));
+	cmd->data[0] = lower_32_bits(paddr) | CMD_COMPL_WAIT_STORE_MASK;
+	cmd->data[1] = upper_32_bits(paddr);
 	cmd->data[2] = 1;
 	CMD_SET_TYPE(cmd, CMD_COMPL_WAIT);
 }
@@ -1165,7 +1107,7 @@ static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)
 	return iommu_queue_command(iommu, &cmd);
 }
 
-static void iommu_flush_dte_all(struct amd_iommu *iommu)
+static void amd_iommu_flush_dte_all(struct amd_iommu *iommu)
 {
 	u32 devid;
 
@@ -1179,7 +1121,7 @@ static void iommu_flush_dte_all(struct amd_iommu *iommu)
  * This function uses heavy locking and may disable irqs for some time. But
  * this is no issue because it is only called during resume.
  */
-static void iommu_flush_tlb_all(struct amd_iommu *iommu)
+static void amd_iommu_flush_tlb_all(struct amd_iommu *iommu)
 {
 	u32 dom_id;
 
@@ -1193,7 +1135,7 @@ static void iommu_flush_tlb_all(struct amd_iommu *iommu)
 	iommu_completion_wait(iommu);
 }
 
-static void iommu_flush_all(struct amd_iommu *iommu)
+static void amd_iommu_flush_all(struct amd_iommu *iommu)
 {
 	struct iommu_cmd cmd;
 
@@ -1212,7 +1154,7 @@ static void iommu_flush_irt(struct amd_iommu *iommu, u16 devid)
 	iommu_queue_command(iommu, &cmd);
 }
 
-static void iommu_flush_irt_all(struct amd_iommu *iommu)
+static void amd_iommu_flush_irt_all(struct amd_iommu *iommu)
 {
 	u32 devid;
 
@@ -1225,11 +1167,11 @@ static void iommu_flush_irt_all(struct amd_iommu *iommu)
 void iommu_flush_all_caches(struct amd_iommu *iommu)
 {
 	if (iommu_feature(iommu, FEATURE_IA)) {
-		iommu_flush_all(iommu);
+		amd_iommu_flush_all(iommu);
 	} else {
-		iommu_flush_dte_all(iommu);
-		iommu_flush_irt_all(iommu);
-		iommu_flush_tlb_all(iommu);
+		amd_iommu_flush_dte_all(iommu);
+		amd_iommu_flush_irt_all(iommu);
+		amd_iommu_flush_tlb_all(iommu);
 	}
 }
 
@@ -1383,7 +1325,7 @@ static bool increase_address_space(struct protection_domain *domain,
 		return false;
 
 	*pte             = PM_LEVEL_PDE(domain->mode,
-					virt_to_phys(domain->pt_root));
+					iommu_virt_to_phys(domain->pt_root));
 	domain->pt_root  = pte;
 	domain->mode    += 1;
 	domain->updated  = true;
@@ -1420,7 +1362,7 @@ static u64 *alloc_pte(struct protection_domain *domain,
 			if (!page)
 				return NULL;
 
-			__npte = PM_LEVEL_PDE(level, virt_to_phys(page));
+			__npte = PM_LEVEL_PDE(level, iommu_virt_to_phys(page));
 
 			/* pte could have been changed somewhere. */
 			if (cmpxchg64(pte, __pte, __npte) != __pte) {
@@ -1536,10 +1478,10 @@ static int iommu_map_page(struct protection_domain *dom,
 			return -EBUSY;
 
 	if (count > 1) {
-		__pte = PAGE_SIZE_PTE(phys_addr, page_size);
-		__pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_P | IOMMU_PTE_FC;
+		__pte = PAGE_SIZE_PTE(__sme_set(phys_addr), page_size);
+		__pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_PR | IOMMU_PTE_FC;
 	} else
-		__pte = phys_addr | IOMMU_PTE_P | IOMMU_PTE_FC;
+		__pte = __sme_set(phys_addr) | IOMMU_PTE_PR | IOMMU_PTE_FC;
 
 	if (prot & IOMMU_PROT_IR)
 		__pte |= IOMMU_PTE_IR;
@@ -1755,7 +1697,7 @@ static void free_gcr3_tbl_level1(u64 *tbl)
 		if (!(tbl[i] & GCR3_VALID))
 			continue;
 
-		ptr = __va(tbl[i] & PAGE_MASK);
+		ptr = iommu_phys_to_virt(tbl[i] & PAGE_MASK);
 
 		free_page((unsigned long)ptr);
 	}
@@ -1770,7 +1712,7 @@ static void free_gcr3_tbl_level2(u64 *tbl)
 		if (!(tbl[i] & GCR3_VALID))
 			continue;
 
-		ptr = __va(tbl[i] & PAGE_MASK);
+		ptr = iommu_phys_to_virt(tbl[i] & PAGE_MASK);
 
 		free_gcr3_tbl_level1(ptr);
 	}
@@ -1788,178 +1730,19 @@ static void free_gcr3_table(struct protection_domain *domain)
 	free_page((unsigned long)domain->gcr3_tbl);
 }
 
-static void dma_ops_domain_free_flush_queue(struct dma_ops_domain *dom)
-{
-	int cpu;
-
-	for_each_possible_cpu(cpu) {
-		struct flush_queue *queue;
-
-		queue = per_cpu_ptr(dom->flush_queue, cpu);
-		kfree(queue->entries);
-	}
-
-	free_percpu(dom->flush_queue);
-
-	dom->flush_queue = NULL;
-}
-
-static int dma_ops_domain_alloc_flush_queue(struct dma_ops_domain *dom)
-{
-	int cpu;
-
-	atomic64_set(&dom->flush_start_cnt,  0);
-	atomic64_set(&dom->flush_finish_cnt, 0);
-
-	dom->flush_queue = alloc_percpu(struct flush_queue);
-	if (!dom->flush_queue)
-		return -ENOMEM;
-
-	/* First make sure everything is cleared */
-	for_each_possible_cpu(cpu) {
-		struct flush_queue *queue;
-
-		queue = per_cpu_ptr(dom->flush_queue, cpu);
-		queue->head    = 0;
-		queue->tail    = 0;
-		queue->entries = NULL;
-	}
-
-	/* Now start doing the allocation */
-	for_each_possible_cpu(cpu) {
-		struct flush_queue *queue;
-
-		queue = per_cpu_ptr(dom->flush_queue, cpu);
-		queue->entries = kzalloc(FLUSH_QUEUE_SIZE * sizeof(*queue->entries),
-					 GFP_KERNEL);
-		if (!queue->entries) {
-			dma_ops_domain_free_flush_queue(dom);
-			return -ENOMEM;
-		}
-
-		spin_lock_init(&queue->lock);
-	}
-
-	return 0;
-}
-
 static void dma_ops_domain_flush_tlb(struct dma_ops_domain *dom)
 {
-	atomic64_inc(&dom->flush_start_cnt);
 	domain_flush_tlb(&dom->domain);
 	domain_flush_complete(&dom->domain);
-	atomic64_inc(&dom->flush_finish_cnt);
 }
 
-static inline bool queue_ring_full(struct flush_queue *queue)
+static void iova_domain_flush_tlb(struct iova_domain *iovad)
 {
-	assert_spin_locked(&queue->lock);
+	struct dma_ops_domain *dom;
 
-	return (((queue->tail + 1) % FLUSH_QUEUE_SIZE) == queue->head);
-}
-
-#define queue_ring_for_each(i, q) \
-	for (i = (q)->head; i != (q)->tail; i = (i + 1) % FLUSH_QUEUE_SIZE)
-
-static inline unsigned queue_ring_add(struct flush_queue *queue)
-{
-	unsigned idx = queue->tail;
-
-	assert_spin_locked(&queue->lock);
-	queue->tail = (idx + 1) % FLUSH_QUEUE_SIZE;
-
-	return idx;
-}
-
-static inline void queue_ring_remove_head(struct flush_queue *queue)
-{
-	assert_spin_locked(&queue->lock);
-	queue->head = (queue->head + 1) % FLUSH_QUEUE_SIZE;
-}
-
-static void queue_ring_free_flushed(struct dma_ops_domain *dom,
-				    struct flush_queue *queue)
-{
-	u64 counter = atomic64_read(&dom->flush_finish_cnt);
-	int idx;
-
-	queue_ring_for_each(idx, queue) {
-		/*
-		 * This assumes that counter values in the ring-buffer are
-		 * monotonously rising.
-		 */
-		if (queue->entries[idx].counter >= counter)
-			break;
-
-		free_iova_fast(&dom->iovad,
-			       queue->entries[idx].iova_pfn,
-			       queue->entries[idx].pages);
-
-		queue_ring_remove_head(queue);
-	}
-}
-
-static void queue_add(struct dma_ops_domain *dom,
-		      unsigned long address, unsigned long pages)
-{
-	struct flush_queue *queue;
-	unsigned long flags;
-	int idx;
-
-	pages     = __roundup_pow_of_two(pages);
-	address >>= PAGE_SHIFT;
-
-	queue = get_cpu_ptr(dom->flush_queue);
-	spin_lock_irqsave(&queue->lock, flags);
-
-	/*
-	 * First remove the enries from the ring-buffer that are already
-	 * flushed to make the below queue_ring_full() check less likely
-	 */
-	queue_ring_free_flushed(dom, queue);
-
-	/*
-	 * When ring-queue is full, flush the entries from the IOTLB so
-	 * that we can free all entries with queue_ring_free_flushed()
-	 * below.
-	 */
-	if (queue_ring_full(queue)) {
-		dma_ops_domain_flush_tlb(dom);
-		queue_ring_free_flushed(dom, queue);
-	}
-
-	idx = queue_ring_add(queue);
-
-	queue->entries[idx].iova_pfn = address;
-	queue->entries[idx].pages    = pages;
-	queue->entries[idx].counter  = atomic64_read(&dom->flush_start_cnt);
-
-	spin_unlock_irqrestore(&queue->lock, flags);
-
-	if (atomic_cmpxchg(&dom->flush_timer_on, 0, 1) == 0)
-		mod_timer(&dom->flush_timer, jiffies + msecs_to_jiffies(10));
-
-	put_cpu_ptr(dom->flush_queue);
-}
-
-static void queue_flush_timeout(unsigned long data)
-{
-	struct dma_ops_domain *dom = (struct dma_ops_domain *)data;
-	int cpu;
-
-	atomic_set(&dom->flush_timer_on, 0);
+	dom = container_of(iovad, struct dma_ops_domain, iovad);
 
 	dma_ops_domain_flush_tlb(dom);
-
-	for_each_possible_cpu(cpu) {
-		struct flush_queue *queue;
-		unsigned long flags;
-
-		queue = per_cpu_ptr(dom->flush_queue, cpu);
-		spin_lock_irqsave(&queue->lock, flags);
-		queue_ring_free_flushed(dom, queue);
-		spin_unlock_irqrestore(&queue->lock, flags);
-	}
 }
 
 /*
@@ -1973,11 +1756,6 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom)
 
 	del_domain_from_list(&dom->domain);
 
-	if (timer_pending(&dom->flush_timer))
-		del_timer(&dom->flush_timer);
-
-	dma_ops_domain_free_flush_queue(dom);
-
 	put_iova_domain(&dom->iovad);
 
 	free_pagetable(&dom->domain);
@@ -2013,16 +1791,11 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void)
 	init_iova_domain(&dma_dom->iovad, PAGE_SIZE,
 			 IOVA_START_PFN, DMA_32BIT_PFN);
 
-	/* Initialize reserved ranges */
-	copy_reserved_iova(&reserved_iova_ranges, &dma_dom->iovad);
-
-	if (dma_ops_domain_alloc_flush_queue(dma_dom))
+	if (init_iova_flush_queue(&dma_dom->iovad, iova_domain_flush_tlb, NULL))
 		goto free_dma_dom;
 
-	setup_timer(&dma_dom->flush_timer, queue_flush_timeout,
-		    (unsigned long)dma_dom);
-
-	atomic_set(&dma_dom->flush_timer_on, 0);
+	/* Initialize reserved ranges */
+	copy_reserved_iova(&reserved_iova_ranges, &dma_dom->iovad);
 
 	add_domain_to_list(&dma_dom->domain);
 
@@ -2049,11 +1822,11 @@ static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
 	u64 flags = 0;
 
 	if (domain->mode != PAGE_MODE_NONE)
-		pte_root = virt_to_phys(domain->pt_root);
+		pte_root = iommu_virt_to_phys(domain->pt_root);
 
 	pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
 		    << DEV_ENTRY_MODE_SHIFT;
-	pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
+	pte_root |= DTE_FLAG_IR | DTE_FLAG_IW | DTE_FLAG_V | DTE_FLAG_TV;
 
 	flags = amd_iommu_dev_table[devid].data[1];
 
@@ -2061,7 +1834,7 @@ static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
 		flags |= DTE_FLAG_IOTLB;
 
 	if (domain->flags & PD_IOMMUV2_MASK) {
-		u64 gcr3 = __pa(domain->gcr3_tbl);
+		u64 gcr3 = iommu_virt_to_phys(domain->gcr3_tbl);
 		u64 glx  = domain->glx;
 		u64 tmp;
 
@@ -2086,8 +1859,7 @@ static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
 		flags    |= tmp;
 	}
 
-
-	flags &= ~(DTE_FLAG_SA | 0xffffULL);
+	flags &= ~DEV_DOMID_MASK;
 	flags |= domain->id;
 
 	amd_iommu_dev_table[devid].data[1]  = flags;
@@ -2097,7 +1869,7 @@ static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
 static void clear_dte_entry(u16 devid)
 {
 	/* remove entry from the device table seen by the hardware */
-	amd_iommu_dev_table[devid].data[0]  = IOMMU_PTE_P | IOMMU_PTE_TV;
+	amd_iommu_dev_table[devid].data[0]  = DTE_FLAG_V | DTE_FLAG_TV;
 	amd_iommu_dev_table[devid].data[1] &= DTE_FLAG_MASK;
 
 	amd_iommu_apply_erratum_63(devid);
@@ -2478,11 +2250,21 @@ static struct iommu_group *amd_iommu_device_group(struct device *dev)
 static struct protection_domain *get_domain(struct device *dev)
 {
 	struct protection_domain *domain;
+	struct iommu_domain *io_domain;
 
 	if (!check_device(dev))
 		return ERR_PTR(-EINVAL);
 
 	domain = get_dev_data(dev)->domain;
+	if (domain == NULL && get_dev_data(dev)->defer_attach) {
+		get_dev_data(dev)->defer_attach = false;
+		io_domain = iommu_get_domain_for_dev(dev);
+		domain = to_pdomain(io_domain);
+		attach_device(dev, domain);
+	}
+	if (domain == NULL)
+		return ERR_PTR(-EBUSY);
+
 	if (!dma_ops_domain(domain))
 		return ERR_PTR(-EBUSY);
 
@@ -2528,6 +2310,7 @@ static int dir2prot(enum dma_data_direction direction)
 	else
 		return 0;
 }
+
 /*
  * This function contains common code for mapping of a physically
  * contiguous memory region into DMA address space. It is used by all
@@ -2619,7 +2402,8 @@ static void __unmap_single(struct dma_ops_domain *dma_dom,
 		domain_flush_tlb(&dma_dom->domain);
 		domain_flush_complete(&dma_dom->domain);
 	} else {
-		queue_add(dma_dom, dma_addr, pages);
+		pages = __roundup_pow_of_two(pages);
+		queue_iova(&dma_dom->iovad, dma_addr >> PAGE_SHIFT, pages, 0);
 	}
 }
 
@@ -3373,6 +3157,13 @@ static void amd_iommu_apply_resv_region(struct device *dev,
 	WARN_ON_ONCE(reserve_iova(&dma_dom->iovad, start, end) == NULL);
 }
 
+static bool amd_iommu_is_attach_deferred(struct iommu_domain *domain,
+					 struct device *dev)
+{
+	struct iommu_dev_data *dev_data = dev->archdata.iommu;
+	return dev_data->defer_attach;
+}
+
 const struct iommu_ops amd_iommu_ops = {
 	.capable = amd_iommu_capable,
 	.domain_alloc = amd_iommu_domain_alloc,
@@ -3389,6 +3180,7 @@ const struct iommu_ops amd_iommu_ops = {
 	.get_resv_regions = amd_iommu_get_resv_regions,
 	.put_resv_regions = amd_iommu_put_resv_regions,
 	.apply_resv_region = amd_iommu_apply_resv_region,
+	.is_attach_deferred = amd_iommu_is_attach_deferred,
 	.pgsize_bitmap	= AMD_IOMMU_PGSIZES,
 };
 
@@ -3606,10 +3398,10 @@ static u64 *__get_gcr3_pte(u64 *root, int level, int pasid, bool alloc)
 			if (root == NULL)
 				return NULL;
 
-			*pte = __pa(root) | GCR3_VALID;
+			*pte = iommu_virt_to_phys(root) | GCR3_VALID;
 		}
 
-		root = __va(*pte & PAGE_MASK);
+		root = iommu_phys_to_virt(*pte & PAGE_MASK);
 
 		level -= 1;
 	}
@@ -3777,18 +3569,13 @@ EXPORT_SYMBOL(amd_iommu_device_info);
 
 static struct irq_chip amd_ir_chip;
 
-#define DTE_IRQ_PHYS_ADDR_MASK	(((1ULL << 45)-1) << 6)
-#define DTE_IRQ_REMAP_INTCTL    (2ULL << 60)
-#define DTE_IRQ_TABLE_LEN       (8ULL << 1)
-#define DTE_IRQ_REMAP_ENABLE    1ULL
-
 static void set_dte_irq_entry(u16 devid, struct irq_remap_table *table)
 {
 	u64 dte;
 
 	dte	= amd_iommu_dev_table[devid].data[2];
 	dte	&= ~DTE_IRQ_PHYS_ADDR_MASK;
-	dte	|= virt_to_phys(table->table);
+	dte	|= iommu_virt_to_phys(table->table);
 	dte	|= DTE_IRQ_REMAP_INTCTL;
 	dte	|= DTE_IRQ_TABLE_LEN;
 	dte	|= DTE_IRQ_REMAP_ENABLE;
diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c
index 372303700566..382de42b8359 100644
--- a/drivers/iommu/amd_iommu_init.c
+++ b/drivers/iommu/amd_iommu_init.c
@@ -29,7 +29,7 @@
 #include <linux/export.h>
 #include <linux/iommu.h>
 #include <linux/kmemleak.h>
-#include <linux/crash_dump.h>
+#include <linux/mem_encrypt.h>
 #include <asm/pci-direct.h>
 #include <asm/iommu.h>
 #include <asm/gart.h>
@@ -38,6 +38,7 @@
 #include <asm/io_apic.h>
 #include <asm/irq_remapping.h>
 
+#include <linux/crash_dump.h>
 #include "amd_iommu_proto.h"
 #include "amd_iommu_types.h"
 #include "irq_remapping.h"
@@ -196,6 +197,11 @@ spinlock_t amd_iommu_pd_lock;
  * page table root pointer.
  */
 struct dev_table_entry *amd_iommu_dev_table;
+/*
+ * Pointer to a device table which the content of old device table
+ * will be copied to. It's only be used in kdump kernel.
+ */
+static struct dev_table_entry *old_dev_tbl_cpy;
 
 /*
  * The alias table is a driver specific data structure which contains the
@@ -209,6 +215,7 @@ u16 *amd_iommu_alias_table;
  * for a specific device. It is also indexed by the PCI device id.
  */
 struct amd_iommu **amd_iommu_rlookup_table;
+EXPORT_SYMBOL(amd_iommu_rlookup_table);
 
 /*
  * This table is used to find the irq remapping table for a given device id
@@ -258,6 +265,28 @@ static int amd_iommu_enable_interrupts(void);
 static int __init iommu_go_to_state(enum iommu_init_state state);
 static void init_device_table_dma(void);
 
+static bool amd_iommu_pre_enabled = true;
+
+bool translation_pre_enabled(struct amd_iommu *iommu)
+{
+	return (iommu->flags & AMD_IOMMU_FLAG_TRANS_PRE_ENABLED);
+}
+EXPORT_SYMBOL(translation_pre_enabled);
+
+static void clear_translation_pre_enabled(struct amd_iommu *iommu)
+{
+	iommu->flags &= ~AMD_IOMMU_FLAG_TRANS_PRE_ENABLED;
+}
+
+static void init_translation_status(struct amd_iommu *iommu)
+{
+	u32 ctrl;
+
+	ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);
+	if (ctrl & (1<<CONTROL_IOMMU_EN))
+		iommu->flags |= AMD_IOMMU_FLAG_TRANS_PRE_ENABLED;
+}
+
 static inline void update_last_devid(u16 devid)
 {
 	if (devid > amd_iommu_last_bdf)
@@ -348,7 +377,7 @@ static void iommu_set_device_table(struct amd_iommu *iommu)
 
 	BUG_ON(iommu->mmio_base == NULL);
 
-	entry = virt_to_phys(amd_iommu_dev_table);
+	entry = iommu_virt_to_phys(amd_iommu_dev_table);
 	entry |= (dev_table_size >> 12) - 1;
 	memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET,
 			&entry, sizeof(entry));
@@ -606,7 +635,7 @@ static void iommu_enable_command_buffer(struct amd_iommu *iommu)
 
 	BUG_ON(iommu->cmd_buf == NULL);
 
-	entry = (u64)virt_to_phys(iommu->cmd_buf);
+	entry = iommu_virt_to_phys(iommu->cmd_buf);
 	entry |= MMIO_CMD_SIZE_512;
 
 	memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET,
@@ -615,6 +644,14 @@ static void iommu_enable_command_buffer(struct amd_iommu *iommu)
 	amd_iommu_reset_cmd_buffer(iommu);
 }
 
+/*
+ * This function disables the command buffer
+ */
+static void iommu_disable_command_buffer(struct amd_iommu *iommu)
+{
+	iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
+}
+
 static void __init free_command_buffer(struct amd_iommu *iommu)
 {
 	free_pages((unsigned long)iommu->cmd_buf, get_order(CMD_BUFFER_SIZE));
@@ -635,7 +672,7 @@ static void iommu_enable_event_buffer(struct amd_iommu *iommu)
 
 	BUG_ON(iommu->evt_buf == NULL);
 
-	entry = (u64)virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK;
+	entry = iommu_virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK;
 
 	memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET,
 		    &entry, sizeof(entry));
@@ -647,6 +684,14 @@ static void iommu_enable_event_buffer(struct amd_iommu *iommu)
 	iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);
 }
 
+/*
+ * This function disables the event log buffer
+ */
+static void iommu_disable_event_buffer(struct amd_iommu *iommu)
+{
+	iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);
+}
+
 static void __init free_event_buffer(struct amd_iommu *iommu)
 {
 	free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE));
@@ -668,7 +713,7 @@ static void iommu_enable_ppr_log(struct amd_iommu *iommu)
 	if (iommu->ppr_log == NULL)
 		return;
 
-	entry = (u64)virt_to_phys(iommu->ppr_log) | PPR_LOG_SIZE_512;
+	entry = iommu_virt_to_phys(iommu->ppr_log) | PPR_LOG_SIZE_512;
 
 	memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET,
 		    &entry, sizeof(entry));
@@ -748,10 +793,10 @@ static int iommu_init_ga_log(struct amd_iommu *iommu)
 	if (!iommu->ga_log_tail)
 		goto err_out;
 
-	entry = (u64)virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512;
+	entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512;
 	memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET,
 		    &entry, sizeof(entry));
-	entry = ((u64)virt_to_phys(iommu->ga_log) & 0xFFFFFFFFFFFFFULL) & ~7ULL;
+	entry = (iommu_virt_to_phys(iommu->ga_log) & 0xFFFFFFFFFFFFFULL) & ~7ULL;
 	memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_TAIL_OFFSET,
 		    &entry, sizeof(entry));
 	writel(0x00, iommu->mmio_base + MMIO_GA_HEAD_OFFSET);
@@ -808,6 +853,96 @@ static int get_dev_entry_bit(u16 devid, u8 bit)
 }
 
 
+static bool copy_device_table(void)
+{
+	u64 int_ctl, int_tab_len, entry = 0, last_entry = 0;
+	struct dev_table_entry *old_devtb = NULL;
+	u32 lo, hi, devid, old_devtb_size;
+	phys_addr_t old_devtb_phys;
+	struct amd_iommu *iommu;
+	u16 dom_id, dte_v, irq_v;
+	gfp_t gfp_flag;
+	u64 tmp;
+
+	if (!amd_iommu_pre_enabled)
+		return false;
+
+	pr_warn("Translation is already enabled - trying to copy translation structures\n");
+	for_each_iommu(iommu) {
+		/* All IOMMUs should use the same device table with the same size */
+		lo = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET);
+		hi = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET + 4);
+		entry = (((u64) hi) << 32) + lo;
+		if (last_entry && last_entry != entry) {
+			pr_err("IOMMU:%d should use the same dev table as others!/n",
+				iommu->index);
+			return false;
+		}
+		last_entry = entry;
+
+		old_devtb_size = ((entry & ~PAGE_MASK) + 1) << 12;
+		if (old_devtb_size != dev_table_size) {
+			pr_err("The device table size of IOMMU:%d is not expected!/n",
+				iommu->index);
+			return false;
+		}
+	}
+
+	old_devtb_phys = entry & PAGE_MASK;
+	if (old_devtb_phys >= 0x100000000ULL) {
+		pr_err("The address of old device table is above 4G, not trustworthy!/n");
+		return false;
+	}
+	old_devtb = memremap(old_devtb_phys, dev_table_size, MEMREMAP_WB);
+	if (!old_devtb)
+		return false;
+
+	gfp_flag = GFP_KERNEL | __GFP_ZERO | GFP_DMA32;
+	old_dev_tbl_cpy = (void *)__get_free_pages(gfp_flag,
+				get_order(dev_table_size));
+	if (old_dev_tbl_cpy == NULL) {
+		pr_err("Failed to allocate memory for copying old device table!/n");
+		return false;
+	}
+
+	for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) {
+		old_dev_tbl_cpy[devid] = old_devtb[devid];
+		dom_id = old_devtb[devid].data[1] & DEV_DOMID_MASK;
+		dte_v = old_devtb[devid].data[0] & DTE_FLAG_V;
+
+		if (dte_v && dom_id) {
+			old_dev_tbl_cpy[devid].data[0] = old_devtb[devid].data[0];
+			old_dev_tbl_cpy[devid].data[1] = old_devtb[devid].data[1];
+			__set_bit(dom_id, amd_iommu_pd_alloc_bitmap);
+			/* If gcr3 table existed, mask it out */
+			if (old_devtb[devid].data[0] & DTE_FLAG_GV) {
+				tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;
+				tmp |= DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;
+				old_dev_tbl_cpy[devid].data[1] &= ~tmp;
+				tmp = DTE_GCR3_VAL_A(~0ULL) << DTE_GCR3_SHIFT_A;
+				tmp |= DTE_FLAG_GV;
+				old_dev_tbl_cpy[devid].data[0] &= ~tmp;
+			}
+		}
+
+		irq_v = old_devtb[devid].data[2] & DTE_IRQ_REMAP_ENABLE;
+		int_ctl = old_devtb[devid].data[2] & DTE_IRQ_REMAP_INTCTL_MASK;
+		int_tab_len = old_devtb[devid].data[2] & DTE_IRQ_TABLE_LEN_MASK;
+		if (irq_v && (int_ctl || int_tab_len)) {
+			if ((int_ctl != DTE_IRQ_REMAP_INTCTL) ||
+			    (int_tab_len != DTE_IRQ_TABLE_LEN)) {
+				pr_err("Wrong old irq remapping flag: %#x\n", devid);
+				return false;
+			}
+
+		        old_dev_tbl_cpy[devid].data[2] = old_devtb[devid].data[2];
+		}
+	}
+	memunmap(old_devtb);
+
+	return true;
+}
+
 void amd_iommu_apply_erratum_63(u16 devid)
 {
 	int sysmgt;
@@ -1399,6 +1534,16 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h)
 
 	iommu->int_enabled = false;
 
+	init_translation_status(iommu);
+	if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
+		iommu_disable(iommu);
+		clear_translation_pre_enabled(iommu);
+		pr_warn("Translation was enabled for IOMMU:%d but we are not in kdump mode\n",
+			iommu->index);
+	}
+	if (amd_iommu_pre_enabled)
+		amd_iommu_pre_enabled = translation_pre_enabled(iommu);
+
 	ret = init_iommu_from_acpi(iommu, h);
 	if (ret)
 		return ret;
@@ -1892,8 +2037,7 @@ static int __init init_memory_definitions(struct acpi_table_header *table)
 }
 
 /*
- * Init the device table to not allow DMA access for devices and
- * suppress all page faults
+ * Init the device table to not allow DMA access for devices
  */
 static void init_device_table_dma(void)
 {
@@ -1902,14 +2046,6 @@ static void init_device_table_dma(void)
 	for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) {
 		set_dev_entry_bit(devid, DEV_ENTRY_VALID);
 		set_dev_entry_bit(devid, DEV_ENTRY_TRANSLATION);
-		/*
-		 * In kdump kernels in-flight DMA from the old kernel might
-		 * cause IO_PAGE_FAULTs. There are no reports that a kdump
-		 * actually failed because of that, so just disable fault
-		 * reporting in the hardware to get rid of the messages
-		 */
-		if (is_kdump_kernel())
-			set_dev_entry_bit(devid, DEV_ENTRY_NO_PAGE_FAULT);
 	}
 }
 
@@ -2022,24 +2158,62 @@ static void iommu_enable_ga(struct amd_iommu *iommu)
 #endif
 }
 
+static void early_enable_iommu(struct amd_iommu *iommu)
+{
+	iommu_disable(iommu);
+	iommu_init_flags(iommu);
+	iommu_set_device_table(iommu);
+	iommu_enable_command_buffer(iommu);
+	iommu_enable_event_buffer(iommu);
+	iommu_set_exclusion_range(iommu);
+	iommu_enable_ga(iommu);
+	iommu_enable(iommu);
+	iommu_flush_all_caches(iommu);
+}
+
 /*
  * This function finally enables all IOMMUs found in the system after
- * they have been initialized
+ * they have been initialized.
+ *
+ * Or if in kdump kernel and IOMMUs are all pre-enabled, try to copy
+ * the old content of device table entries. Not this case or copy failed,
+ * just continue as normal kernel does.
  */
 static void early_enable_iommus(void)
 {
 	struct amd_iommu *iommu;
 
-	for_each_iommu(iommu) {
-		iommu_disable(iommu);
-		iommu_init_flags(iommu);
-		iommu_set_device_table(iommu);
-		iommu_enable_command_buffer(iommu);
-		iommu_enable_event_buffer(iommu);
-		iommu_set_exclusion_range(iommu);
-		iommu_enable_ga(iommu);
-		iommu_enable(iommu);
-		iommu_flush_all_caches(iommu);
+
+	if (!copy_device_table()) {
+		/*
+		 * If come here because of failure in copying device table from old
+		 * kernel with all IOMMUs enabled, print error message and try to
+		 * free allocated old_dev_tbl_cpy.
+		 */
+		if (amd_iommu_pre_enabled)
+			pr_err("Failed to copy DEV table from previous kernel.\n");
+		if (old_dev_tbl_cpy != NULL)
+			free_pages((unsigned long)old_dev_tbl_cpy,
+					get_order(dev_table_size));
+
+		for_each_iommu(iommu) {
+			clear_translation_pre_enabled(iommu);
+			early_enable_iommu(iommu);
+		}
+	} else {
+		pr_info("Copied DEV table from previous kernel.\n");
+		free_pages((unsigned long)amd_iommu_dev_table,
+				get_order(dev_table_size));
+		amd_iommu_dev_table = old_dev_tbl_cpy;
+		for_each_iommu(iommu) {
+			iommu_disable_command_buffer(iommu);
+			iommu_disable_event_buffer(iommu);
+			iommu_enable_command_buffer(iommu);
+			iommu_enable_event_buffer(iommu);
+			iommu_enable_ga(iommu);
+			iommu_set_device_table(iommu);
+			iommu_flush_all_caches(iommu);
+		}
 	}
 
 #ifdef CONFIG_IRQ_REMAP
@@ -2275,7 +2449,8 @@ static int __init early_amd_iommu_init(void)
 
 	/* Device table - directly used by all IOMMUs */
 	ret = -ENOMEM;
-	amd_iommu_dev_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
+	amd_iommu_dev_table = (void *)__get_free_pages(
+				      GFP_KERNEL | __GFP_ZERO | GFP_DMA32,
 				      get_order(dev_table_size));
 	if (amd_iommu_dev_table == NULL)
 		goto out;
@@ -2325,7 +2500,8 @@ static int __init early_amd_iommu_init(void)
 		goto out;
 
 	/* Disable any previously enabled IOMMUs */
-	disable_iommus();
+	if (!is_kdump_kernel() || amd_iommu_disabled)
+		disable_iommus();
 
 	if (amd_iommu_irq_remap)
 		amd_iommu_irq_remap = check_ioapic_information();
@@ -2564,6 +2740,24 @@ static int __init amd_iommu_init(void)
 	return ret;
 }
 
+static bool amd_iommu_sme_check(void)
+{
+	if (!sme_active() || (boot_cpu_data.x86 != 0x17))
+		return true;
+
+	/* For Fam17h, a specific level of support is required */
+	if (boot_cpu_data.microcode >= 0x08001205)
+		return true;
+
+	if ((boot_cpu_data.microcode >= 0x08001126) &&
+	    (boot_cpu_data.microcode <= 0x080011ff))
+		return true;
+
+	pr_notice("AMD-Vi: IOMMU not currently supported when SME is active\n");
+
+	return false;
+}
+
 /****************************************************************************
  *
  * Early detect code. This code runs at IOMMU detection time in the DMA
@@ -2578,6 +2772,9 @@ int __init amd_iommu_detect(void)
 	if (no_iommu || (iommu_detected && !gart_iommu_aperture))
 		return -ENODEV;
 
+	if (!amd_iommu_sme_check())
+		return -ENODEV;
+
 	ret = iommu_go_to_state(IOMMU_IVRS_DETECTED);
 	if (ret)
 		return ret;
diff --git a/drivers/iommu/amd_iommu_proto.h b/drivers/iommu/amd_iommu_proto.h
index 466260f8a1df..640c286a0ab9 100644
--- a/drivers/iommu/amd_iommu_proto.h
+++ b/drivers/iommu/amd_iommu_proto.h
@@ -87,4 +87,16 @@ static inline bool iommu_feature(struct amd_iommu *iommu, u64 f)
 	return !!(iommu->features & f);
 }
 
+static inline u64 iommu_virt_to_phys(void *vaddr)
+{
+	return (u64)__sme_set(virt_to_phys(vaddr));
+}
+
+static inline void *iommu_phys_to_virt(unsigned long paddr)
+{
+	return phys_to_virt(__sme_clr(paddr));
+}
+
+extern bool translation_pre_enabled(struct amd_iommu *iommu);
+extern struct iommu_dev_data *get_dev_data(struct device *dev);
 #endif /* _ASM_X86_AMD_IOMMU_PROTO_H  */
diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h
index 294a409e283b..f6b24c7d8b70 100644
--- a/drivers/iommu/amd_iommu_types.h
+++ b/drivers/iommu/amd_iommu_types.h
@@ -250,6 +250,14 @@
 
 #define GA_GUEST_NR		0x1
 
+/* Bit value definition for dte irq remapping fields*/
+#define DTE_IRQ_PHYS_ADDR_MASK	(((1ULL << 45)-1) << 6)
+#define DTE_IRQ_REMAP_INTCTL_MASK	(0x3ULL << 60)
+#define DTE_IRQ_TABLE_LEN_MASK	(0xfULL << 1)
+#define DTE_IRQ_REMAP_INTCTL    (2ULL << 60)
+#define DTE_IRQ_TABLE_LEN       (8ULL << 1)
+#define DTE_IRQ_REMAP_ENABLE    1ULL
+
 #define PAGE_MODE_NONE    0x00
 #define PAGE_MODE_1_LEVEL 0x01
 #define PAGE_MODE_2_LEVEL 0x02
@@ -265,7 +273,7 @@
 #define PM_LEVEL_INDEX(x, a)	(((a) >> PM_LEVEL_SHIFT((x))) & 0x1ffULL)
 #define PM_LEVEL_ENC(x)		(((x) << 9) & 0xe00ULL)
 #define PM_LEVEL_PDE(x, a)	((a) | PM_LEVEL_ENC((x)) | \
-				 IOMMU_PTE_P | IOMMU_PTE_IR | IOMMU_PTE_IW)
+				 IOMMU_PTE_PR | IOMMU_PTE_IR | IOMMU_PTE_IW)
 #define PM_PTE_LEVEL(pte)	(((pte) >> 9) & 0x7ULL)
 
 #define PM_MAP_4k		0
@@ -314,19 +322,29 @@
 #define PTE_LEVEL_PAGE_SIZE(level)			\
 	(1ULL << (12 + (9 * (level))))
 
-#define IOMMU_PTE_P  (1ULL << 0)
-#define IOMMU_PTE_TV (1ULL << 1)
+/*
+ * Bit value definition for I/O PTE fields
+ */
+#define IOMMU_PTE_PR (1ULL << 0)
 #define IOMMU_PTE_U  (1ULL << 59)
 #define IOMMU_PTE_FC (1ULL << 60)
 #define IOMMU_PTE_IR (1ULL << 61)
 #define IOMMU_PTE_IW (1ULL << 62)
 
+/*
+ * Bit value definition for DTE fields
+ */
+#define DTE_FLAG_V  (1ULL << 0)
+#define DTE_FLAG_TV (1ULL << 1)
+#define DTE_FLAG_IR (1ULL << 61)
+#define DTE_FLAG_IW (1ULL << 62)
+
 #define DTE_FLAG_IOTLB	(1ULL << 32)
-#define DTE_FLAG_SA	(1ULL << 34)
 #define DTE_FLAG_GV	(1ULL << 55)
 #define DTE_FLAG_MASK	(0x3ffULL << 32)
 #define DTE_GLX_SHIFT	(56)
 #define DTE_GLX_MASK	(3)
+#define DEV_DOMID_MASK	0xffffULL
 
 #define DTE_GCR3_VAL_A(x)	(((x) >> 12) & 0x00007ULL)
 #define DTE_GCR3_VAL_B(x)	(((x) >> 15) & 0x0ffffULL)
@@ -343,8 +361,8 @@
 #define GCR3_VALID		0x01ULL
 
 #define IOMMU_PAGE_MASK (((1ULL << 52) - 1) & ~0xfffULL)
-#define IOMMU_PTE_PRESENT(pte) ((pte) & IOMMU_PTE_P)
-#define IOMMU_PTE_PAGE(pte) (phys_to_virt((pte) & IOMMU_PAGE_MASK))
+#define IOMMU_PTE_PRESENT(pte) ((pte) & IOMMU_PTE_PR)
+#define IOMMU_PTE_PAGE(pte) (iommu_phys_to_virt((pte) & IOMMU_PAGE_MASK))
 #define IOMMU_PTE_MODE(pte) (((pte) >> 9) & 0x07)
 
 #define IOMMU_PROT_MASK 0x03
@@ -435,6 +453,8 @@ struct iommu_domain;
 struct irq_domain;
 struct amd_irte_ops;
 
+#define AMD_IOMMU_FLAG_TRANS_PRE_ENABLED      (1 << 0)
+
 /*
  * This structure contains generic data for  IOMMU protection domains
  * independent of their use.
@@ -569,12 +589,15 @@ struct amd_iommu {
 	struct amd_irte_ops *irte_ops;
 #endif
 
+	u32 flags;
 	volatile u64 __aligned(8) cmd_sem;
 };
 
 static inline struct amd_iommu *dev_to_amd_iommu(struct device *dev)
 {
-	return container_of(dev, struct amd_iommu, iommu.dev);
+	struct iommu_device *iommu = dev_to_iommu_device(dev);
+
+	return container_of(iommu, struct amd_iommu, iommu);
 }
 
 #define ACPIHID_UID_LEN 256
@@ -597,6 +620,30 @@ struct devid_map {
 	bool cmd_line;
 };
 
+/*
+ * This struct contains device specific data for the IOMMU
+ */
+struct iommu_dev_data {
+	struct list_head list;		  /* For domain->dev_list */
+	struct list_head dev_data_list;	  /* For global dev_data_list */
+	struct protection_domain *domain; /* Domain the device is bound to */
+	u16 devid;			  /* PCI Device ID */
+	u16 alias;			  /* Alias Device ID */
+	bool iommu_v2;			  /* Device can make use of IOMMUv2 */
+	bool passthrough;		  /* Device is identity mapped */
+	struct {
+		bool enabled;
+		int qdep;
+	} ats;				  /* ATS state */
+	bool pri_tlp;			  /* PASID TLB required for
+					     PPR completions */
+	u32 errata;			  /* Bitmap for errata to apply */
+	bool use_vapic;			  /* Enable device to use vapic mode */
+	bool defer_attach;
+
+	struct ratelimit_state rs;        /* Ratelimit IOPF messages */
+};
+
 /* Map HPET and IOAPIC ids to the devid used by the IOMMU */
 extern struct list_head ioapic_map;
 extern struct list_head hpet_map;
diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c
index 6629c472eafd..7d94e1d39e5e 100644
--- a/drivers/iommu/amd_iommu_v2.c
+++ b/drivers/iommu/amd_iommu_v2.c
@@ -391,13 +391,6 @@ static int mn_clear_flush_young(struct mmu_notifier *mn,
 	return 0;
 }
 
-static void mn_invalidate_page(struct mmu_notifier *mn,
-			       struct mm_struct *mm,
-			       unsigned long address)
-{
-	__mn_flush_page(mn, address);
-}
-
 static void mn_invalidate_range(struct mmu_notifier *mn,
 				struct mm_struct *mm,
 				unsigned long start, unsigned long end)
@@ -436,7 +429,6 @@ static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm)
 static const struct mmu_notifier_ops iommu_mn = {
 	.release		= mn_release,
 	.clear_flush_young      = mn_clear_flush_young,
-	.invalidate_page        = mn_invalidate_page,
 	.invalidate_range       = mn_invalidate_range,
 };
 
@@ -562,14 +554,30 @@ static int ppr_notifier(struct notifier_block *nb, unsigned long e, void *data)
 	unsigned long flags;
 	struct fault *fault;
 	bool finish;
-	u16 tag;
+	u16 tag, devid;
 	int ret;
+	struct iommu_dev_data *dev_data;
+	struct pci_dev *pdev = NULL;
 
 	iommu_fault = data;
 	tag         = iommu_fault->tag & 0x1ff;
 	finish      = (iommu_fault->tag >> 9) & 1;
 
+	devid = iommu_fault->device_id;
+	pdev = pci_get_bus_and_slot(PCI_BUS_NUM(devid), devid & 0xff);
+	if (!pdev)
+		return -ENODEV;
+	dev_data = get_dev_data(&pdev->dev);
+
+	/* In kdump kernel pci dev is not initialized yet -> send INVALID */
 	ret = NOTIFY_DONE;
+	if (translation_pre_enabled(amd_iommu_rlookup_table[devid])
+		&& dev_data->defer_attach) {
+		amd_iommu_complete_ppr(pdev, iommu_fault->pasid,
+				       PPR_INVALID, tag);
+		goto out;
+	}
+
 	dev_state = get_device_state(iommu_fault->device_id);
 	if (dev_state == NULL)
 		goto out;
diff --git a/drivers/iommu/arm-smmu-regs.h b/drivers/iommu/arm-smmu-regs.h
new file mode 100644
index 000000000000..a1226e4ab5f8
--- /dev/null
+++ b/drivers/iommu/arm-smmu-regs.h
@@ -0,0 +1,220 @@
+/*
+ * IOMMU API for ARM architected SMMU implementations.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Copyright (C) 2013 ARM Limited
+ *
+ * Author: Will Deacon <will.deacon@arm.com>
+ */
+
+#ifndef _ARM_SMMU_REGS_H
+#define _ARM_SMMU_REGS_H
+
+/* Configuration registers */
+#define ARM_SMMU_GR0_sCR0		0x0
+#define sCR0_CLIENTPD			(1 << 0)
+#define sCR0_GFRE			(1 << 1)
+#define sCR0_GFIE			(1 << 2)
+#define sCR0_EXIDENABLE			(1 << 3)
+#define sCR0_GCFGFRE			(1 << 4)
+#define sCR0_GCFGFIE			(1 << 5)
+#define sCR0_USFCFG			(1 << 10)
+#define sCR0_VMIDPNE			(1 << 11)
+#define sCR0_PTM			(1 << 12)
+#define sCR0_FB				(1 << 13)
+#define sCR0_VMID16EN			(1 << 31)
+#define sCR0_BSU_SHIFT			14
+#define sCR0_BSU_MASK			0x3
+
+/* Auxiliary Configuration register */
+#define ARM_SMMU_GR0_sACR		0x10
+
+/* Identification registers */
+#define ARM_SMMU_GR0_ID0		0x20
+#define ARM_SMMU_GR0_ID1		0x24
+#define ARM_SMMU_GR0_ID2		0x28
+#define ARM_SMMU_GR0_ID3		0x2c
+#define ARM_SMMU_GR0_ID4		0x30
+#define ARM_SMMU_GR0_ID5		0x34
+#define ARM_SMMU_GR0_ID6		0x38
+#define ARM_SMMU_GR0_ID7		0x3c
+#define ARM_SMMU_GR0_sGFSR		0x48
+#define ARM_SMMU_GR0_sGFSYNR0		0x50
+#define ARM_SMMU_GR0_sGFSYNR1		0x54
+#define ARM_SMMU_GR0_sGFSYNR2		0x58
+
+#define ID0_S1TS			(1 << 30)
+#define ID0_S2TS			(1 << 29)
+#define ID0_NTS				(1 << 28)
+#define ID0_SMS				(1 << 27)
+#define ID0_ATOSNS			(1 << 26)
+#define ID0_PTFS_NO_AARCH32		(1 << 25)
+#define ID0_PTFS_NO_AARCH32S		(1 << 24)
+#define ID0_CTTW			(1 << 14)
+#define ID0_NUMIRPT_SHIFT		16
+#define ID0_NUMIRPT_MASK		0xff
+#define ID0_NUMSIDB_SHIFT		9
+#define ID0_NUMSIDB_MASK		0xf
+#define ID0_EXIDS			(1 << 8)
+#define ID0_NUMSMRG_SHIFT		0
+#define ID0_NUMSMRG_MASK		0xff
+
+#define ID1_PAGESIZE			(1 << 31)
+#define ID1_NUMPAGENDXB_SHIFT		28
+#define ID1_NUMPAGENDXB_MASK		7
+#define ID1_NUMS2CB_SHIFT		16
+#define ID1_NUMS2CB_MASK		0xff
+#define ID1_NUMCB_SHIFT			0
+#define ID1_NUMCB_MASK			0xff
+
+#define ID2_OAS_SHIFT			4
+#define ID2_OAS_MASK			0xf
+#define ID2_IAS_SHIFT			0
+#define ID2_IAS_MASK			0xf
+#define ID2_UBS_SHIFT			8
+#define ID2_UBS_MASK			0xf
+#define ID2_PTFS_4K			(1 << 12)
+#define ID2_PTFS_16K			(1 << 13)
+#define ID2_PTFS_64K			(1 << 14)
+#define ID2_VMID16			(1 << 15)
+
+#define ID7_MAJOR_SHIFT			4
+#define ID7_MAJOR_MASK			0xf
+
+/* Global TLB invalidation */
+#define ARM_SMMU_GR0_TLBIVMID		0x64
+#define ARM_SMMU_GR0_TLBIALLNSNH	0x68
+#define ARM_SMMU_GR0_TLBIALLH		0x6c
+#define ARM_SMMU_GR0_sTLBGSYNC		0x70
+#define ARM_SMMU_GR0_sTLBGSTATUS	0x74
+#define sTLBGSTATUS_GSACTIVE		(1 << 0)
+
+/* Stream mapping registers */
+#define ARM_SMMU_GR0_SMR(n)		(0x800 + ((n) << 2))
+#define SMR_VALID			(1 << 31)
+#define SMR_MASK_SHIFT			16
+#define SMR_ID_SHIFT			0
+
+#define ARM_SMMU_GR0_S2CR(n)		(0xc00 + ((n) << 2))
+#define S2CR_CBNDX_SHIFT		0
+#define S2CR_CBNDX_MASK			0xff
+#define S2CR_EXIDVALID			(1 << 10)
+#define S2CR_TYPE_SHIFT			16
+#define S2CR_TYPE_MASK			0x3
+enum arm_smmu_s2cr_type {
+	S2CR_TYPE_TRANS,
+	S2CR_TYPE_BYPASS,
+	S2CR_TYPE_FAULT,
+};
+
+#define S2CR_PRIVCFG_SHIFT		24
+#define S2CR_PRIVCFG_MASK		0x3
+enum arm_smmu_s2cr_privcfg {
+	S2CR_PRIVCFG_DEFAULT,
+	S2CR_PRIVCFG_DIPAN,
+	S2CR_PRIVCFG_UNPRIV,
+	S2CR_PRIVCFG_PRIV,
+};
+
+/* Context bank attribute registers */
+#define ARM_SMMU_GR1_CBAR(n)		(0x0 + ((n) << 2))
+#define CBAR_VMID_SHIFT			0
+#define CBAR_VMID_MASK			0xff
+#define CBAR_S1_BPSHCFG_SHIFT		8
+#define CBAR_S1_BPSHCFG_MASK		3
+#define CBAR_S1_BPSHCFG_NSH		3
+#define CBAR_S1_MEMATTR_SHIFT		12
+#define CBAR_S1_MEMATTR_MASK		0xf
+#define CBAR_S1_MEMATTR_WB		0xf
+#define CBAR_TYPE_SHIFT			16
+#define CBAR_TYPE_MASK			0x3
+#define CBAR_TYPE_S2_TRANS		(0 << CBAR_TYPE_SHIFT)
+#define CBAR_TYPE_S1_TRANS_S2_BYPASS	(1 << CBAR_TYPE_SHIFT)
+#define CBAR_TYPE_S1_TRANS_S2_FAULT	(2 << CBAR_TYPE_SHIFT)
+#define CBAR_TYPE_S1_TRANS_S2_TRANS	(3 << CBAR_TYPE_SHIFT)
+#define CBAR_IRPTNDX_SHIFT		24
+#define CBAR_IRPTNDX_MASK		0xff
+
+#define ARM_SMMU_GR1_CBA2R(n)		(0x800 + ((n) << 2))
+#define CBA2R_RW64_32BIT		(0 << 0)
+#define CBA2R_RW64_64BIT		(1 << 0)
+#define CBA2R_VMID_SHIFT		16
+#define CBA2R_VMID_MASK			0xffff
+
+#define ARM_SMMU_CB_SCTLR		0x0
+#define ARM_SMMU_CB_ACTLR		0x4
+#define ARM_SMMU_CB_RESUME		0x8
+#define ARM_SMMU_CB_TTBCR2		0x10
+#define ARM_SMMU_CB_TTBR0		0x20
+#define ARM_SMMU_CB_TTBR1		0x28
+#define ARM_SMMU_CB_TTBCR		0x30
+#define ARM_SMMU_CB_CONTEXTIDR		0x34
+#define ARM_SMMU_CB_S1_MAIR0		0x38
+#define ARM_SMMU_CB_S1_MAIR1		0x3c
+#define ARM_SMMU_CB_PAR			0x50
+#define ARM_SMMU_CB_FSR			0x58
+#define ARM_SMMU_CB_FAR			0x60
+#define ARM_SMMU_CB_FSYNR0		0x68
+#define ARM_SMMU_CB_S1_TLBIVA		0x600
+#define ARM_SMMU_CB_S1_TLBIASID		0x610
+#define ARM_SMMU_CB_S1_TLBIVAL		0x620
+#define ARM_SMMU_CB_S2_TLBIIPAS2	0x630
+#define ARM_SMMU_CB_S2_TLBIIPAS2L	0x638
+#define ARM_SMMU_CB_TLBSYNC		0x7f0
+#define ARM_SMMU_CB_TLBSTATUS		0x7f4
+#define ARM_SMMU_CB_ATS1PR		0x800
+#define ARM_SMMU_CB_ATSR		0x8f0
+
+#define SCTLR_S1_ASIDPNE		(1 << 12)
+#define SCTLR_CFCFG			(1 << 7)
+#define SCTLR_CFIE			(1 << 6)
+#define SCTLR_CFRE			(1 << 5)
+#define SCTLR_E				(1 << 4)
+#define SCTLR_AFE			(1 << 2)
+#define SCTLR_TRE			(1 << 1)
+#define SCTLR_M				(1 << 0)
+
+#define CB_PAR_F			(1 << 0)
+
+#define ATSR_ACTIVE			(1 << 0)
+
+#define RESUME_RETRY			(0 << 0)
+#define RESUME_TERMINATE		(1 << 0)
+
+#define TTBCR2_SEP_SHIFT		15
+#define TTBCR2_SEP_UPSTREAM		(0x7 << TTBCR2_SEP_SHIFT)
+#define TTBCR2_AS			(1 << 4)
+
+#define TTBRn_ASID_SHIFT		48
+
+#define FSR_MULTI			(1 << 31)
+#define FSR_SS				(1 << 30)
+#define FSR_UUT				(1 << 8)
+#define FSR_ASF				(1 << 7)
+#define FSR_TLBLKF			(1 << 6)
+#define FSR_TLBMCF			(1 << 5)
+#define FSR_EF				(1 << 4)
+#define FSR_PF				(1 << 3)
+#define FSR_AFF				(1 << 2)
+#define FSR_TF				(1 << 1)
+
+#define FSR_IGN				(FSR_AFF | FSR_ASF | \
+					 FSR_TLBMCF | FSR_TLBLKF)
+#define FSR_FAULT			(FSR_MULTI | FSR_SS | FSR_UUT | \
+					 FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
+
+#define FSYNR0_WNR			(1 << 4)
+
+#endif /* _ARM_SMMU_REGS_H */
diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
index 568c400eeaed..e67ba6c40faf 100644
--- a/drivers/iommu/arm-smmu-v3.c
+++ b/drivers/iommu/arm-smmu-v3.c
@@ -2852,9 +2852,15 @@ static int arm_smmu_device_remove(struct platform_device *pdev)
 	struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
 
 	arm_smmu_device_disable(smmu);
+
 	return 0;
 }
 
+static void arm_smmu_device_shutdown(struct platform_device *pdev)
+{
+	arm_smmu_device_remove(pdev);
+}
+
 static const struct of_device_id arm_smmu_of_match[] = {
 	{ .compatible = "arm,smmu-v3", },
 	{ },
@@ -2868,6 +2874,7 @@ static struct platform_driver arm_smmu_driver = {
 	},
 	.probe	= arm_smmu_device_probe,
 	.remove	= arm_smmu_device_remove,
+	.shutdown = arm_smmu_device_shutdown,
 };
 module_platform_driver(arm_smmu_driver);
 
diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index 2d80fa8a0634..3bdb799d3b4b 100644
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -54,6 +54,15 @@
 #include <linux/amba/bus.h>
 
 #include "io-pgtable.h"
+#include "arm-smmu-regs.h"
+
+#define ARM_MMU500_ACTLR_CPRE		(1 << 1)
+
+#define ARM_MMU500_ACR_CACHE_LOCK	(1 << 26)
+#define ARM_MMU500_ACR_SMTNMB_TLBEN	(1 << 8)
+
+#define TLB_LOOP_TIMEOUT		1000000	/* 1s! */
+#define TLB_SPIN_COUNT			10
 
 /* Maximum number of context banks per SMMU */
 #define ARM_SMMU_MAX_CBS		128
@@ -83,211 +92,9 @@
 #define smmu_write_atomic_lq		writel_relaxed
 #endif
 
-/* Configuration registers */
-#define ARM_SMMU_GR0_sCR0		0x0
-#define sCR0_CLIENTPD			(1 << 0)
-#define sCR0_GFRE			(1 << 1)
-#define sCR0_GFIE			(1 << 2)
-#define sCR0_EXIDENABLE			(1 << 3)
-#define sCR0_GCFGFRE			(1 << 4)
-#define sCR0_GCFGFIE			(1 << 5)
-#define sCR0_USFCFG			(1 << 10)
-#define sCR0_VMIDPNE			(1 << 11)
-#define sCR0_PTM			(1 << 12)
-#define sCR0_FB				(1 << 13)
-#define sCR0_VMID16EN			(1 << 31)
-#define sCR0_BSU_SHIFT			14
-#define sCR0_BSU_MASK			0x3
-
-/* Auxiliary Configuration register */
-#define ARM_SMMU_GR0_sACR		0x10
-
-/* Identification registers */
-#define ARM_SMMU_GR0_ID0		0x20
-#define ARM_SMMU_GR0_ID1		0x24
-#define ARM_SMMU_GR0_ID2		0x28
-#define ARM_SMMU_GR0_ID3		0x2c
-#define ARM_SMMU_GR0_ID4		0x30
-#define ARM_SMMU_GR0_ID5		0x34
-#define ARM_SMMU_GR0_ID6		0x38
-#define ARM_SMMU_GR0_ID7		0x3c
-#define ARM_SMMU_GR0_sGFSR		0x48
-#define ARM_SMMU_GR0_sGFSYNR0		0x50
-#define ARM_SMMU_GR0_sGFSYNR1		0x54
-#define ARM_SMMU_GR0_sGFSYNR2		0x58
-
-#define ID0_S1TS			(1 << 30)
-#define ID0_S2TS			(1 << 29)
-#define ID0_NTS				(1 << 28)
-#define ID0_SMS				(1 << 27)
-#define ID0_ATOSNS			(1 << 26)
-#define ID0_PTFS_NO_AARCH32		(1 << 25)
-#define ID0_PTFS_NO_AARCH32S		(1 << 24)
-#define ID0_CTTW			(1 << 14)
-#define ID0_NUMIRPT_SHIFT		16
-#define ID0_NUMIRPT_MASK		0xff
-#define ID0_NUMSIDB_SHIFT		9
-#define ID0_NUMSIDB_MASK		0xf
-#define ID0_EXIDS			(1 << 8)
-#define ID0_NUMSMRG_SHIFT		0
-#define ID0_NUMSMRG_MASK		0xff
-
-#define ID1_PAGESIZE			(1 << 31)
-#define ID1_NUMPAGENDXB_SHIFT		28
-#define ID1_NUMPAGENDXB_MASK		7
-#define ID1_NUMS2CB_SHIFT		16
-#define ID1_NUMS2CB_MASK		0xff
-#define ID1_NUMCB_SHIFT			0
-#define ID1_NUMCB_MASK			0xff
-
-#define ID2_OAS_SHIFT			4
-#define ID2_OAS_MASK			0xf
-#define ID2_IAS_SHIFT			0
-#define ID2_IAS_MASK			0xf
-#define ID2_UBS_SHIFT			8
-#define ID2_UBS_MASK			0xf
-#define ID2_PTFS_4K			(1 << 12)
-#define ID2_PTFS_16K			(1 << 13)
-#define ID2_PTFS_64K			(1 << 14)
-#define ID2_VMID16			(1 << 15)
-
-#define ID7_MAJOR_SHIFT			4
-#define ID7_MAJOR_MASK			0xf
-
-/* Global TLB invalidation */
-#define ARM_SMMU_GR0_TLBIVMID		0x64
-#define ARM_SMMU_GR0_TLBIALLNSNH	0x68
-#define ARM_SMMU_GR0_TLBIALLH		0x6c
-#define ARM_SMMU_GR0_sTLBGSYNC		0x70
-#define ARM_SMMU_GR0_sTLBGSTATUS	0x74
-#define sTLBGSTATUS_GSACTIVE		(1 << 0)
-#define TLB_LOOP_TIMEOUT		1000000	/* 1s! */
-#define TLB_SPIN_COUNT			10
-
-/* Stream mapping registers */
-#define ARM_SMMU_GR0_SMR(n)		(0x800 + ((n) << 2))
-#define SMR_VALID			(1 << 31)
-#define SMR_MASK_SHIFT			16
-#define SMR_ID_SHIFT			0
-
-#define ARM_SMMU_GR0_S2CR(n)		(0xc00 + ((n) << 2))
-#define S2CR_CBNDX_SHIFT		0
-#define S2CR_CBNDX_MASK			0xff
-#define S2CR_EXIDVALID			(1 << 10)
-#define S2CR_TYPE_SHIFT			16
-#define S2CR_TYPE_MASK			0x3
-enum arm_smmu_s2cr_type {
-	S2CR_TYPE_TRANS,
-	S2CR_TYPE_BYPASS,
-	S2CR_TYPE_FAULT,
-};
-
-#define S2CR_PRIVCFG_SHIFT		24
-#define S2CR_PRIVCFG_MASK		0x3
-enum arm_smmu_s2cr_privcfg {
-	S2CR_PRIVCFG_DEFAULT,
-	S2CR_PRIVCFG_DIPAN,
-	S2CR_PRIVCFG_UNPRIV,
-	S2CR_PRIVCFG_PRIV,
-};
-
-/* Context bank attribute registers */
-#define ARM_SMMU_GR1_CBAR(n)		(0x0 + ((n) << 2))
-#define CBAR_VMID_SHIFT			0
-#define CBAR_VMID_MASK			0xff
-#define CBAR_S1_BPSHCFG_SHIFT		8
-#define CBAR_S1_BPSHCFG_MASK		3
-#define CBAR_S1_BPSHCFG_NSH		3
-#define CBAR_S1_MEMATTR_SHIFT		12
-#define CBAR_S1_MEMATTR_MASK		0xf
-#define CBAR_S1_MEMATTR_WB		0xf
-#define CBAR_TYPE_SHIFT			16
-#define CBAR_TYPE_MASK			0x3
-#define CBAR_TYPE_S2_TRANS		(0 << CBAR_TYPE_SHIFT)
-#define CBAR_TYPE_S1_TRANS_S2_BYPASS	(1 << CBAR_TYPE_SHIFT)
-#define CBAR_TYPE_S1_TRANS_S2_FAULT	(2 << CBAR_TYPE_SHIFT)
-#define CBAR_TYPE_S1_TRANS_S2_TRANS	(3 << CBAR_TYPE_SHIFT)
-#define CBAR_IRPTNDX_SHIFT		24
-#define CBAR_IRPTNDX_MASK		0xff
-
-#define ARM_SMMU_GR1_CBA2R(n)		(0x800 + ((n) << 2))
-#define CBA2R_RW64_32BIT		(0 << 0)
-#define CBA2R_RW64_64BIT		(1 << 0)
-#define CBA2R_VMID_SHIFT		16
-#define CBA2R_VMID_MASK			0xffff
-
 /* Translation context bank */
 #define ARM_SMMU_CB(smmu, n)	((smmu)->cb_base + ((n) << (smmu)->pgshift))
 
-#define ARM_SMMU_CB_SCTLR		0x0
-#define ARM_SMMU_CB_ACTLR		0x4
-#define ARM_SMMU_CB_RESUME		0x8
-#define ARM_SMMU_CB_TTBCR2		0x10
-#define ARM_SMMU_CB_TTBR0		0x20
-#define ARM_SMMU_CB_TTBR1		0x28
-#define ARM_SMMU_CB_TTBCR		0x30
-#define ARM_SMMU_CB_CONTEXTIDR		0x34
-#define ARM_SMMU_CB_S1_MAIR0		0x38
-#define ARM_SMMU_CB_S1_MAIR1		0x3c
-#define ARM_SMMU_CB_PAR			0x50
-#define ARM_SMMU_CB_FSR			0x58
-#define ARM_SMMU_CB_FAR			0x60
-#define ARM_SMMU_CB_FSYNR0		0x68
-#define ARM_SMMU_CB_S1_TLBIVA		0x600
-#define ARM_SMMU_CB_S1_TLBIASID		0x610
-#define ARM_SMMU_CB_S1_TLBIVAL		0x620
-#define ARM_SMMU_CB_S2_TLBIIPAS2	0x630
-#define ARM_SMMU_CB_S2_TLBIIPAS2L	0x638
-#define ARM_SMMU_CB_TLBSYNC		0x7f0
-#define ARM_SMMU_CB_TLBSTATUS		0x7f4
-#define ARM_SMMU_CB_ATS1PR		0x800
-#define ARM_SMMU_CB_ATSR		0x8f0
-
-#define SCTLR_S1_ASIDPNE		(1 << 12)
-#define SCTLR_CFCFG			(1 << 7)
-#define SCTLR_CFIE			(1 << 6)
-#define SCTLR_CFRE			(1 << 5)
-#define SCTLR_E				(1 << 4)
-#define SCTLR_AFE			(1 << 2)
-#define SCTLR_TRE			(1 << 1)
-#define SCTLR_M				(1 << 0)
-
-#define ARM_MMU500_ACTLR_CPRE		(1 << 1)
-
-#define ARM_MMU500_ACR_CACHE_LOCK	(1 << 26)
-#define ARM_MMU500_ACR_SMTNMB_TLBEN	(1 << 8)
-
-#define CB_PAR_F			(1 << 0)
-
-#define ATSR_ACTIVE			(1 << 0)
-
-#define RESUME_RETRY			(0 << 0)
-#define RESUME_TERMINATE		(1 << 0)
-
-#define TTBCR2_SEP_SHIFT		15
-#define TTBCR2_SEP_UPSTREAM		(0x7 << TTBCR2_SEP_SHIFT)
-#define TTBCR2_AS			(1 << 4)
-
-#define TTBRn_ASID_SHIFT		48
-
-#define FSR_MULTI			(1 << 31)
-#define FSR_SS				(1 << 30)
-#define FSR_UUT				(1 << 8)
-#define FSR_ASF				(1 << 7)
-#define FSR_TLBLKF			(1 << 6)
-#define FSR_TLBMCF			(1 << 5)
-#define FSR_EF				(1 << 4)
-#define FSR_PF				(1 << 3)
-#define FSR_AFF				(1 << 2)
-#define FSR_TF				(1 << 1)
-
-#define FSR_IGN				(FSR_AFF | FSR_ASF | \
-					 FSR_TLBMCF | FSR_TLBLKF)
-#define FSR_FAULT			(FSR_MULTI | FSR_SS | FSR_UUT | \
-					 FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
-
-#define FSYNR0_WNR			(1 << 4)
-
 #define MSI_IOVA_BASE			0x8000000
 #define MSI_IOVA_LENGTH			0x100000
 
@@ -338,6 +145,13 @@ struct arm_smmu_smr {
 	bool				valid;
 };
 
+struct arm_smmu_cb {
+	u64				ttbr[2];
+	u32				tcr[2];
+	u32				mair[2];
+	struct arm_smmu_cfg		*cfg;
+};
+
 struct arm_smmu_master_cfg {
 	struct arm_smmu_device		*smmu;
 	s16				smendx[];
@@ -380,6 +194,7 @@ struct arm_smmu_device {
 	u32				num_context_banks;
 	u32				num_s2_context_banks;
 	DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
+	struct arm_smmu_cb		*cbs;
 	atomic_t			irptndx;
 
 	u32				num_mapping_groups;
@@ -776,17 +591,74 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 				       struct io_pgtable_cfg *pgtbl_cfg)
 {
-	u32 reg, reg2;
-	u64 reg64;
-	bool stage1;
 	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
-	struct arm_smmu_device *smmu = smmu_domain->smmu;
+	struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
+	bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
+
+	cb->cfg = cfg;
+
+	/* TTBCR */
+	if (stage1) {
+		if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
+			cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
+		} else {
+			cb->tcr[0] = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
+			cb->tcr[1] = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
+			cb->tcr[1] |= TTBCR2_SEP_UPSTREAM;
+			if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
+				cb->tcr[1] |= TTBCR2_AS;
+		}
+	} else {
+		cb->tcr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
+	}
+
+	/* TTBRs */
+	if (stage1) {
+		if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
+			cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
+			cb->ttbr[1] = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
+		} else {
+			cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
+			cb->ttbr[0] |= (u64)cfg->asid << TTBRn_ASID_SHIFT;
+			cb->ttbr[1] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
+			cb->ttbr[1] |= (u64)cfg->asid << TTBRn_ASID_SHIFT;
+		}
+	} else {
+		cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
+	}
+
+	/* MAIRs (stage-1 only) */
+	if (stage1) {
+		if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
+			cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
+			cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
+		} else {
+			cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
+			cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
+		}
+	}
+}
+
+static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
+{
+	u32 reg;
+	bool stage1;
+	struct arm_smmu_cb *cb = &smmu->cbs[idx];
+	struct arm_smmu_cfg *cfg = cb->cfg;
 	void __iomem *cb_base, *gr1_base;
 
+	cb_base = ARM_SMMU_CB(smmu, idx);
+
+	/* Unassigned context banks only need disabling */
+	if (!cfg) {
+		writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
+		return;
+	}
+
 	gr1_base = ARM_SMMU_GR1(smmu);
 	stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
-	cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
 
+	/* CBA2R */
 	if (smmu->version > ARM_SMMU_V1) {
 		if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 			reg = CBA2R_RW64_64BIT;
@@ -796,7 +668,7 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 		if (smmu->features & ARM_SMMU_FEAT_VMID16)
 			reg |= cfg->vmid << CBA2R_VMID_SHIFT;
 
-		writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
+		writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(idx));
 	}
 
 	/* CBAR */
@@ -815,72 +687,41 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 		/* 8-bit VMIDs live in CBAR */
 		reg |= cfg->vmid << CBAR_VMID_SHIFT;
 	}
-	writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
+	writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(idx));
 
 	/*
 	 * TTBCR
 	 * We must write this before the TTBRs, since it determines the
 	 * access behaviour of some fields (in particular, ASID[15:8]).
 	 */
-	if (stage1) {
-		if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
-			reg = pgtbl_cfg->arm_v7s_cfg.tcr;
-			reg2 = 0;
-		} else {
-			reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
-			reg2 = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
-			reg2 |= TTBCR2_SEP_UPSTREAM;
-			if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
-				reg2 |= TTBCR2_AS;
-		}
-		if (smmu->version > ARM_SMMU_V1)
-			writel_relaxed(reg2, cb_base + ARM_SMMU_CB_TTBCR2);
-	} else {
-		reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
-	}
-	writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
+	if (stage1 && smmu->version > ARM_SMMU_V1)
+		writel_relaxed(cb->tcr[1], cb_base + ARM_SMMU_CB_TTBCR2);
+	writel_relaxed(cb->tcr[0], cb_base + ARM_SMMU_CB_TTBCR);
 
 	/* TTBRs */
-	if (stage1) {
-		if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
-			reg = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
-			writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0);
-			reg = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
-			writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1);
-			writel_relaxed(cfg->asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
-		} else {
-			reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
-			reg64 |= (u64)cfg->asid << TTBRn_ASID_SHIFT;
-			writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
-			reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
-			reg64 |= (u64)cfg->asid << TTBRn_ASID_SHIFT;
-			writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
-		}
+	if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
+		writel_relaxed(cfg->asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
+		writel_relaxed(cb->ttbr[0], cb_base + ARM_SMMU_CB_TTBR0);
+		writel_relaxed(cb->ttbr[1], cb_base + ARM_SMMU_CB_TTBR1);
 	} else {
-		reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
-		writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
+		writeq_relaxed(cb->ttbr[0], cb_base + ARM_SMMU_CB_TTBR0);
+		if (stage1)
+			writeq_relaxed(cb->ttbr[1], cb_base + ARM_SMMU_CB_TTBR1);
 	}
 
 	/* MAIRs (stage-1 only) */
 	if (stage1) {
-		if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
-			reg = pgtbl_cfg->arm_v7s_cfg.prrr;
-			reg2 = pgtbl_cfg->arm_v7s_cfg.nmrr;
-		} else {
-			reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
-			reg2 = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
-		}
-		writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
-		writel_relaxed(reg2, cb_base + ARM_SMMU_CB_S1_MAIR1);
+		writel_relaxed(cb->mair[0], cb_base + ARM_SMMU_CB_S1_MAIR0);
+		writel_relaxed(cb->mair[1], cb_base + ARM_SMMU_CB_S1_MAIR1);
 	}
 
 	/* SCTLR */
 	reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
 	if (stage1)
 		reg |= SCTLR_S1_ASIDPNE;
-#ifdef __BIG_ENDIAN
-	reg |= SCTLR_E;
-#endif
+	if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
+		reg |= SCTLR_E;
+
 	writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
 }
 
@@ -1043,6 +884,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 
 	/* Initialise the context bank with our page table cfg */
 	arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
+	arm_smmu_write_context_bank(smmu, cfg->cbndx);
 
 	/*
 	 * Request context fault interrupt. Do this last to avoid the
@@ -1075,7 +917,6 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 	struct arm_smmu_device *smmu = smmu_domain->smmu;
 	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
-	void __iomem *cb_base;
 	int irq;
 
 	if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
@@ -1085,8 +926,8 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 	 * Disable the context bank and free the page tables before freeing
 	 * it.
 	 */
-	cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
-	writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
+	smmu->cbs[cfg->cbndx].cfg = NULL;
+	arm_smmu_write_context_bank(smmu, cfg->cbndx);
 
 	if (cfg->irptndx != INVALID_IRPTNDX) {
 		irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
@@ -1736,7 +1577,6 @@ static struct iommu_ops arm_smmu_ops = {
 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
 {
 	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
-	void __iomem *cb_base;
 	int i;
 	u32 reg, major;
 
@@ -1772,8 +1612,9 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
 
 	/* Make sure all context banks are disabled and clear CB_FSR  */
 	for (i = 0; i < smmu->num_context_banks; ++i) {
-		cb_base = ARM_SMMU_CB(smmu, i);
-		writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
+		void __iomem *cb_base = ARM_SMMU_CB(smmu, i);
+
+		arm_smmu_write_context_bank(smmu, i);
 		writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
 		/*
 		 * Disable MMU-500's not-particularly-beneficial next-page
@@ -1979,6 +1820,10 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
 		smmu->cavium_id_base -= smmu->num_context_banks;
 		dev_notice(smmu->dev, "\tenabling workaround for Cavium erratum 27704\n");
 	}
+	smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
+				 sizeof(*smmu->cbs), GFP_KERNEL);
+	if (!smmu->cbs)
+		return -ENOMEM;
 
 	/* ID2 */
 	id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
@@ -2336,13 +2181,30 @@ static int arm_smmu_device_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static void arm_smmu_device_shutdown(struct platform_device *pdev)
+{
+	arm_smmu_device_remove(pdev);
+}
+
+static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
+{
+	struct arm_smmu_device *smmu = dev_get_drvdata(dev);
+
+	arm_smmu_device_reset(smmu);
+	return 0;
+}
+
+static SIMPLE_DEV_PM_OPS(arm_smmu_pm_ops, NULL, arm_smmu_pm_resume);
+
 static struct platform_driver arm_smmu_driver = {
 	.driver	= {
 		.name		= "arm-smmu",
 		.of_match_table	= of_match_ptr(arm_smmu_of_match),
+		.pm		= &arm_smmu_pm_ops,
 	},
 	.probe	= arm_smmu_device_probe,
 	.remove	= arm_smmu_device_remove,
+	.shutdown = arm_smmu_device_shutdown,
 };
 module_platform_driver(arm_smmu_driver);
 
diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c
index c8b0329c85d2..ca5ebaeafd6a 100644
--- a/drivers/iommu/dmar.c
+++ b/drivers/iommu/dmar.c
@@ -1343,7 +1343,7 @@ void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
 
 	if (mask) {
 		BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
-		addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
+		addr |= (1ULL << (VTD_PAGE_SHIFT + mask - 1)) - 1;
 		desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
 	} else
 		desc.high = QI_DEV_IOTLB_ADDR(addr);
diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c
index 2395478dde75..f596fcc32898 100644
--- a/drivers/iommu/exynos-iommu.c
+++ b/drivers/iommu/exynos-iommu.c
@@ -54,10 +54,6 @@ typedef u32 sysmmu_pte_t;
 #define lv2ent_small(pent) ((*(pent) & 2) == 2)
 #define lv2ent_large(pent) ((*(pent) & 3) == 1)
 
-#ifdef CONFIG_BIG_ENDIAN
-#warning "revisit driver if we can enable big-endian ptes"
-#endif
-
 /*
  * v1.x - v3.x SYSMMU supports 32bit physical and 32bit virtual address spaces
  * v5.0 introduced support for 36bit physical address space by shifting
@@ -569,7 +565,7 @@ static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data,
 	spin_unlock_irqrestore(&data->lock, flags);
 }
 
-static struct iommu_ops exynos_iommu_ops;
+static const struct iommu_ops exynos_iommu_ops;
 
 static int __init exynos_sysmmu_probe(struct platform_device *pdev)
 {
@@ -659,6 +655,13 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev)
 		}
 	}
 
+	/*
+	 * use the first registered sysmmu device for performing
+	 * dma mapping operations on iommu page tables (cpu cache flush)
+	 */
+	if (!dma_dev)
+		dma_dev = &pdev->dev;
+
 	pm_runtime_enable(dev);
 
 	return 0;
@@ -1323,7 +1326,7 @@ static int exynos_iommu_of_xlate(struct device *dev,
 	return 0;
 }
 
-static struct iommu_ops exynos_iommu_ops = {
+static const struct iommu_ops exynos_iommu_ops = {
 	.domain_alloc = exynos_iommu_domain_alloc,
 	.domain_free = exynos_iommu_domain_free,
 	.attach_dev = exynos_iommu_attach_device,
@@ -1339,8 +1342,6 @@ static struct iommu_ops exynos_iommu_ops = {
 	.of_xlate = exynos_iommu_of_xlate,
 };
 
-static bool init_done;
-
 static int __init exynos_iommu_init(void)
 {
 	int ret;
@@ -1373,8 +1374,6 @@ static int __init exynos_iommu_init(void)
 		goto err_set_iommu;
 	}
 
-	init_done = true;
-
 	return 0;
 err_set_iommu:
 	kmem_cache_free(lv2table_kmem_cache, zero_lv2_table);
@@ -1384,27 +1383,6 @@ err_reg_driver:
 	kmem_cache_destroy(lv2table_kmem_cache);
 	return ret;
 }
+core_initcall(exynos_iommu_init);
 
-static int __init exynos_iommu_of_setup(struct device_node *np)
-{
-	struct platform_device *pdev;
-
-	if (!init_done)
-		exynos_iommu_init();
-
-	pdev = of_platform_device_create(np, NULL, platform_bus_type.dev_root);
-	if (!pdev)
-		return -ENODEV;
-
-	/*
-	 * use the first registered sysmmu device for performing
-	 * dma mapping operations on iommu page tables (cpu cache flush)
-	 */
-	if (!dma_dev)
-		dma_dev = &pdev->dev;
-
-	return 0;
-}
-
-IOMMU_OF_DECLARE(exynos_iommu_of, "samsung,exynos-sysmmu",
-		 exynos_iommu_of_setup);
+IOMMU_OF_DECLARE(exynos_iommu_of, "samsung,exynos-sysmmu", NULL);
diff --git a/drivers/iommu/fsl_pamu.c b/drivers/iommu/fsl_pamu.c
index a34355fca37a..8540625796a1 100644
--- a/drivers/iommu/fsl_pamu.c
+++ b/drivers/iommu/fsl_pamu.c
@@ -42,6 +42,8 @@ struct pamu_isr_data {
 static struct paace *ppaact;
 static struct paace *spaact;
 
+static bool probed;			/* Has PAMU been probed? */
+
 /*
  * Table for matching compatible strings, for device tree
  * guts node, for QorIQ SOCs.
@@ -530,8 +532,8 @@ u32 get_stash_id(u32 stash_dest_hint, u32 vcpu)
 		if (node) {
 			prop = of_get_property(node, "cache-stash-id", NULL);
 			if (!prop) {
-				pr_debug("missing cache-stash-id at %s\n",
-					 node->full_name);
+				pr_debug("missing cache-stash-id at %pOF\n",
+					 node);
 				of_node_put(node);
 				return ~(u32)0;
 			}
@@ -557,8 +559,8 @@ found_cpu_node:
 		if (stash_dest_hint == cache_level) {
 			prop = of_get_property(node, "cache-stash-id", NULL);
 			if (!prop) {
-				pr_debug("missing cache-stash-id at %s\n",
-					 node->full_name);
+				pr_debug("missing cache-stash-id at %pOF\n",
+					 node);
 				of_node_put(node);
 				return ~(u32)0;
 			}
@@ -568,8 +570,7 @@ found_cpu_node:
 
 		prop = of_get_property(node, "next-level-cache", NULL);
 		if (!prop) {
-			pr_debug("can't find next-level-cache at %s\n",
-				 node->full_name);
+			pr_debug("can't find next-level-cache at %pOF\n", node);
 			of_node_put(node);
 			return ~(u32)0;  /* can't traverse any further */
 		}
@@ -1033,6 +1034,9 @@ static int fsl_pamu_probe(struct platform_device *pdev)
 	 * NOTE : All PAMUs share the same LIODN tables.
 	 */
 
+	if (WARN_ON(probed))
+		return -EBUSY;
+
 	pamu_regs = of_iomap(dev->of_node, 0);
 	if (!pamu_regs) {
 		dev_err(dev, "ioremap of PAMU node failed\n");
@@ -1063,8 +1067,7 @@ static int fsl_pamu_probe(struct platform_device *pdev)
 
 	guts_node = of_find_matching_node(NULL, guts_device_ids);
 	if (!guts_node) {
-		dev_err(dev, "could not find GUTS node %s\n",
-			dev->of_node->full_name);
+		dev_err(dev, "could not find GUTS node %pOF\n", dev->of_node);
 		ret = -ENODEV;
 		goto error;
 	}
@@ -1172,6 +1175,8 @@ static int fsl_pamu_probe(struct platform_device *pdev)
 
 	setup_liodns();
 
+	probed = true;
+
 	return 0;
 
 error_genpool:
@@ -1246,8 +1251,7 @@ static __init int fsl_pamu_init(void)
 
 	pdev = platform_device_alloc("fsl-of-pamu", 0);
 	if (!pdev) {
-		pr_err("could not allocate device %s\n",
-		       np->full_name);
+		pr_err("could not allocate device %pOF\n", np);
 		ret = -ENOMEM;
 		goto error_device_alloc;
 	}
@@ -1259,8 +1263,7 @@ static __init int fsl_pamu_init(void)
 
 	ret = platform_device_add(pdev);
 	if (ret) {
-		pr_err("could not add device %s (err=%i)\n",
-		       np->full_name, ret);
+		pr_err("could not add device %pOF (err=%i)\n", np, ret);
 		goto error_device_add;
 	}
 
diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c
index da0e1e30ef37..f089136e9c3f 100644
--- a/drivers/iommu/fsl_pamu_domain.c
+++ b/drivers/iommu/fsl_pamu_domain.c
@@ -33,6 +33,8 @@ static struct kmem_cache *fsl_pamu_domain_cache;
 static struct kmem_cache *iommu_devinfo_cache;
 static DEFINE_SPINLOCK(device_domain_lock);
 
+struct iommu_device pamu_iommu;	/* IOMMU core code handle */
+
 static struct fsl_dma_domain *to_fsl_dma_domain(struct iommu_domain *dom)
 {
 	return container_of(dom, struct fsl_dma_domain, iommu_domain);
@@ -619,8 +621,8 @@ static int handle_attach_device(struct fsl_dma_domain *dma_domain,
 	for (i = 0; i < num; i++) {
 		/* Ensure that LIODN value is valid */
 		if (liodn[i] >= PAACE_NUMBER_ENTRIES) {
-			pr_debug("Invalid liodn %d, attach device failed for %s\n",
-				 liodn[i], dev->of_node->full_name);
+			pr_debug("Invalid liodn %d, attach device failed for %pOF\n",
+				 liodn[i], dev->of_node);
 			ret = -EINVAL;
 			break;
 		}
@@ -684,8 +686,7 @@ static int fsl_pamu_attach_device(struct iommu_domain *domain,
 		liodn_cnt = len / sizeof(u32);
 		ret = handle_attach_device(dma_domain, dev, liodn, liodn_cnt);
 	} else {
-		pr_debug("missing fsl,liodn property at %s\n",
-			 dev->of_node->full_name);
+		pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);
 		ret = -EINVAL;
 	}
 
@@ -720,8 +721,7 @@ static void fsl_pamu_detach_device(struct iommu_domain *domain,
 	if (prop)
 		detach_device(dev, dma_domain);
 	else
-		pr_debug("missing fsl,liodn property at %s\n",
-			 dev->of_node->full_name);
+		pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);
 }
 
 static  int configure_domain_geometry(struct iommu_domain *domain, void *data)
@@ -983,11 +983,14 @@ static int fsl_pamu_add_device(struct device *dev)
 
 	iommu_group_put(group);
 
+	iommu_device_link(&pamu_iommu, dev);
+
 	return 0;
 }
 
 static void fsl_pamu_remove_device(struct device *dev)
 {
+	iommu_device_unlink(&pamu_iommu, dev);
 	iommu_group_remove_device(dev);
 }
 
@@ -1073,6 +1076,19 @@ int __init pamu_domain_init(void)
 	if (ret)
 		return ret;
 
+	ret = iommu_device_sysfs_add(&pamu_iommu, NULL, NULL, "iommu0");
+	if (ret)
+		return ret;
+
+	iommu_device_set_ops(&pamu_iommu, &fsl_pamu_ops);
+
+	ret = iommu_device_register(&pamu_iommu);
+	if (ret) {
+		iommu_device_sysfs_remove(&pamu_iommu);
+		pr_err("Can't register iommu device\n");
+		return ret;
+	}
+
 	bus_set_iommu(&platform_bus_type, &fsl_pamu_ops);
 	bus_set_iommu(&pci_bus_type, &fsl_pamu_ops);
 
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index 687f18f65cea..6784a05dd6b2 100644
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -458,31 +458,6 @@ static LIST_HEAD(dmar_rmrr_units);
 #define for_each_rmrr_units(rmrr) \
 	list_for_each_entry(rmrr, &dmar_rmrr_units, list)
 
-static void flush_unmaps_timeout(unsigned long data);
-
-struct deferred_flush_entry {
-	unsigned long iova_pfn;
-	unsigned long nrpages;
-	struct dmar_domain *domain;
-	struct page *freelist;
-};
-
-#define HIGH_WATER_MARK 250
-struct deferred_flush_table {
-	int next;
-	struct deferred_flush_entry entries[HIGH_WATER_MARK];
-};
-
-struct deferred_flush_data {
-	spinlock_t lock;
-	int timer_on;
-	struct timer_list timer;
-	long size;
-	struct deferred_flush_table *tables;
-};
-
-static DEFINE_PER_CPU(struct deferred_flush_data, deferred_flush);
-
 /* bitmap for indexing intel_iommus */
 static int g_num_of_iommus;
 
@@ -901,6 +876,13 @@ static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devf
 		struct pci_dev *pf_pdev;
 
 		pdev = to_pci_dev(dev);
+
+#ifdef CONFIG_X86
+		/* VMD child devices currently cannot be handled individually */
+		if (is_vmd(pdev->bus))
+			return NULL;
+#endif
+
 		/* VFs aren't listed in scope tables; we need to look up
 		 * the PF instead to find the IOMMU. */
 		pf_pdev = pci_physfn(pdev);
@@ -974,20 +956,6 @@ static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 	return ret;
 }
 
-static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
-{
-	struct context_entry *context;
-	unsigned long flags;
-
-	spin_lock_irqsave(&iommu->lock, flags);
-	context = iommu_context_addr(iommu, bus, devfn, 0);
-	if (context) {
-		context_clear_entry(context);
-		__iommu_flush_cache(iommu, context, sizeof(*context));
-	}
-	spin_unlock_irqrestore(&iommu->lock, flags);
-}
-
 static void free_context_table(struct intel_iommu *iommu)
 {
 	int i;
@@ -1137,8 +1105,9 @@ static void dma_pte_clear_range(struct dmar_domain *domain,
 }
 
 static void dma_pte_free_level(struct dmar_domain *domain, int level,
-			       struct dma_pte *pte, unsigned long pfn,
-			       unsigned long start_pfn, unsigned long last_pfn)
+			       int retain_level, struct dma_pte *pte,
+			       unsigned long pfn, unsigned long start_pfn,
+			       unsigned long last_pfn)
 {
 	pfn = max(start_pfn, pfn);
 	pte = &pte[pfn_level_offset(pfn, level)];
@@ -1153,12 +1122,17 @@ static void dma_pte_free_level(struct dmar_domain *domain, int level,
 		level_pfn = pfn & level_mask(level);
 		level_pte = phys_to_virt(dma_pte_addr(pte));
 
-		if (level > 2)
-			dma_pte_free_level(domain, level - 1, level_pte,
-					   level_pfn, start_pfn, last_pfn);
+		if (level > 2) {
+			dma_pte_free_level(domain, level - 1, retain_level,
+					   level_pte, level_pfn, start_pfn,
+					   last_pfn);
+		}
 
-		/* If range covers entire pagetable, free it */
-		if (!(start_pfn > level_pfn ||
+		/*
+		 * Free the page table if we're below the level we want to
+		 * retain and the range covers the entire table.
+		 */
+		if (level < retain_level && !(start_pfn > level_pfn ||
 		      last_pfn < level_pfn + level_size(level) - 1)) {
 			dma_clear_pte(pte);
 			domain_flush_cache(domain, pte, sizeof(*pte));
@@ -1169,10 +1143,14 @@ next:
 	} while (!first_pte_in_page(++pte) && pfn <= last_pfn);
 }
 
-/* clear last level (leaf) ptes and free page table pages. */
+/*
+ * clear last level (leaf) ptes and free page table pages below the
+ * level we wish to keep intact.
+ */
 static void dma_pte_free_pagetable(struct dmar_domain *domain,
 				   unsigned long start_pfn,
-				   unsigned long last_pfn)
+				   unsigned long last_pfn,
+				   int retain_level)
 {
 	BUG_ON(!domain_pfn_supported(domain, start_pfn));
 	BUG_ON(!domain_pfn_supported(domain, last_pfn));
@@ -1181,7 +1159,7 @@ static void dma_pte_free_pagetable(struct dmar_domain *domain,
 	dma_pte_clear_range(domain, start_pfn, last_pfn);
 
 	/* We don't need lock here; nobody else touches the iova range */
-	dma_pte_free_level(domain, agaw_to_level(domain->agaw),
+	dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
 			   domain->pgd, 0, start_pfn, last_pfn);
 
 	/* free pgd */
@@ -1309,6 +1287,13 @@ static void dma_free_pagelist(struct page *freelist)
 	}
 }
 
+static void iova_entry_free(unsigned long data)
+{
+	struct page *freelist = (struct page *)data;
+
+	dma_free_pagelist(freelist);
+}
+
 /* iommu handling */
 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
 {
@@ -1622,6 +1607,25 @@ static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
 				      addr, mask);
 }
 
+static void iommu_flush_iova(struct iova_domain *iovad)
+{
+	struct dmar_domain *domain;
+	int idx;
+
+	domain = container_of(iovad, struct dmar_domain, iovad);
+
+	for_each_domain_iommu(idx, domain) {
+		struct intel_iommu *iommu = g_iommus[idx];
+		u16 did = domain->iommu_did[iommu->seq_id];
+
+		iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
+
+		if (!cap_caching_mode(iommu->cap))
+			iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
+					      0, MAX_AGAW_PFN_WIDTH);
+	}
+}
+
 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
 {
 	u32 pmen;
@@ -1932,9 +1936,16 @@ static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
 {
 	int adjust_width, agaw;
 	unsigned long sagaw;
+	int err;
 
 	init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
 			DMA_32BIT_PFN);
+
+	err = init_iova_flush_queue(&domain->iovad,
+				    iommu_flush_iova, iova_entry_free);
+	if (err)
+		return err;
+
 	domain_reserve_special_ranges(domain);
 
 	/* calculate AGAW */
@@ -1986,14 +1997,6 @@ static void domain_exit(struct dmar_domain *domain)
 	if (!domain)
 		return;
 
-	/* Flush any lazy unmaps that may reference this domain */
-	if (!intel_iommu_strict) {
-		int cpu;
-
-		for_each_possible_cpu(cpu)
-			flush_unmaps_timeout(cpu);
-	}
-
 	/* Remove associated devices and clear attached or cached domains */
 	rcu_read_lock();
 	domain_remove_dev_info(domain);
@@ -2277,8 +2280,11 @@ static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
 				/*
 				 * Ensure that old small page tables are
 				 * removed to make room for superpage(s).
+				 * We're adding new large pages, so make sure
+				 * we don't remove their parent tables.
 				 */
-				dma_pte_free_pagetable(domain, iov_pfn, end_pfn);
+				dma_pte_free_pagetable(domain, iov_pfn, end_pfn,
+						       largepage_lvl + 1);
 			} else {
 				pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
 			}
@@ -2351,13 +2357,33 @@ static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long i
 
 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
 {
+	unsigned long flags;
+	struct context_entry *context;
+	u16 did_old;
+
 	if (!iommu)
 		return;
 
-	clear_context_table(iommu, bus, devfn);
-	iommu->flush.flush_context(iommu, 0, 0, 0,
-					   DMA_CCMD_GLOBAL_INVL);
-	iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
+	spin_lock_irqsave(&iommu->lock, flags);
+	context = iommu_context_addr(iommu, bus, devfn, 0);
+	if (!context) {
+		spin_unlock_irqrestore(&iommu->lock, flags);
+		return;
+	}
+	did_old = context_domain_id(context);
+	context_clear_entry(context);
+	__iommu_flush_cache(iommu, context, sizeof(*context));
+	spin_unlock_irqrestore(&iommu->lock, flags);
+	iommu->flush.flush_context(iommu,
+				   did_old,
+				   (((u16)bus) << 8) | devfn,
+				   DMA_CCMD_MASK_NOBIT,
+				   DMA_CCMD_DEVICE_INVL);
+	iommu->flush.flush_iotlb(iommu,
+				 did_old,
+				 0,
+				 0,
+				 DMA_TLB_DSI_FLUSH);
 }
 
 static inline void unlink_domain_info(struct device_domain_info *info)
@@ -3206,7 +3232,7 @@ static int __init init_dmars(void)
 	bool copied_tables = false;
 	struct device *dev;
 	struct intel_iommu *iommu;
-	int i, ret, cpu;
+	int i, ret;
 
 	/*
 	 * for each drhd
@@ -3239,22 +3265,6 @@ static int __init init_dmars(void)
 		goto error;
 	}
 
-	for_each_possible_cpu(cpu) {
-		struct deferred_flush_data *dfd = per_cpu_ptr(&deferred_flush,
-							      cpu);
-
-		dfd->tables = kzalloc(g_num_of_iommus *
-				      sizeof(struct deferred_flush_table),
-				      GFP_KERNEL);
-		if (!dfd->tables) {
-			ret = -ENOMEM;
-			goto free_g_iommus;
-		}
-
-		spin_lock_init(&dfd->lock);
-		setup_timer(&dfd->timer, flush_unmaps_timeout, cpu);
-	}
-
 	for_each_active_iommu(iommu, drhd) {
 		g_iommus[iommu->seq_id] = iommu;
 
@@ -3437,10 +3447,9 @@ free_iommu:
 		disable_dmar_iommu(iommu);
 		free_dmar_iommu(iommu);
 	}
-free_g_iommus:
-	for_each_possible_cpu(cpu)
-		kfree(per_cpu_ptr(&deferred_flush, cpu)->tables);
+
 	kfree(g_iommus);
+
 error:
 	return ret;
 }
@@ -3645,110 +3654,6 @@ static dma_addr_t intel_map_page(struct device *dev, struct page *page,
 				  dir, *dev->dma_mask);
 }
 
-static void flush_unmaps(struct deferred_flush_data *flush_data)
-{
-	int i, j;
-
-	flush_data->timer_on = 0;
-
-	/* just flush them all */
-	for (i = 0; i < g_num_of_iommus; i++) {
-		struct intel_iommu *iommu = g_iommus[i];
-		struct deferred_flush_table *flush_table =
-				&flush_data->tables[i];
-		if (!iommu)
-			continue;
-
-		if (!flush_table->next)
-			continue;
-
-		/* In caching mode, global flushes turn emulation expensive */
-		if (!cap_caching_mode(iommu->cap))
-			iommu->flush.flush_iotlb(iommu, 0, 0, 0,
-					 DMA_TLB_GLOBAL_FLUSH);
-		for (j = 0; j < flush_table->next; j++) {
-			unsigned long mask;
-			struct deferred_flush_entry *entry =
-						&flush_table->entries[j];
-			unsigned long iova_pfn = entry->iova_pfn;
-			unsigned long nrpages = entry->nrpages;
-			struct dmar_domain *domain = entry->domain;
-			struct page *freelist = entry->freelist;
-
-			/* On real hardware multiple invalidations are expensive */
-			if (cap_caching_mode(iommu->cap))
-				iommu_flush_iotlb_psi(iommu, domain,
-					mm_to_dma_pfn(iova_pfn),
-					nrpages, !freelist, 0);
-			else {
-				mask = ilog2(nrpages);
-				iommu_flush_dev_iotlb(domain,
-						(uint64_t)iova_pfn << PAGE_SHIFT, mask);
-			}
-			free_iova_fast(&domain->iovad, iova_pfn, nrpages);
-			if (freelist)
-				dma_free_pagelist(freelist);
-		}
-		flush_table->next = 0;
-	}
-
-	flush_data->size = 0;
-}
-
-static void flush_unmaps_timeout(unsigned long cpuid)
-{
-	struct deferred_flush_data *flush_data = per_cpu_ptr(&deferred_flush, cpuid);
-	unsigned long flags;
-
-	spin_lock_irqsave(&flush_data->lock, flags);
-	flush_unmaps(flush_data);
-	spin_unlock_irqrestore(&flush_data->lock, flags);
-}
-
-static void add_unmap(struct dmar_domain *dom, unsigned long iova_pfn,
-		      unsigned long nrpages, struct page *freelist)
-{
-	unsigned long flags;
-	int entry_id, iommu_id;
-	struct intel_iommu *iommu;
-	struct deferred_flush_entry *entry;
-	struct deferred_flush_data *flush_data;
-
-	flush_data = raw_cpu_ptr(&deferred_flush);
-
-	/* Flush all CPUs' entries to avoid deferring too much.  If
-	 * this becomes a bottleneck, can just flush us, and rely on
-	 * flush timer for the rest.
-	 */
-	if (flush_data->size == HIGH_WATER_MARK) {
-		int cpu;
-
-		for_each_online_cpu(cpu)
-			flush_unmaps_timeout(cpu);
-	}
-
-	spin_lock_irqsave(&flush_data->lock, flags);
-
-	iommu = domain_get_iommu(dom);
-	iommu_id = iommu->seq_id;
-
-	entry_id = flush_data->tables[iommu_id].next;
-	++(flush_data->tables[iommu_id].next);
-
-	entry = &flush_data->tables[iommu_id].entries[entry_id];
-	entry->domain = dom;
-	entry->iova_pfn = iova_pfn;
-	entry->nrpages = nrpages;
-	entry->freelist = freelist;
-
-	if (!flush_data->timer_on) {
-		mod_timer(&flush_data->timer, jiffies + msecs_to_jiffies(10));
-		flush_data->timer_on = 1;
-	}
-	flush_data->size++;
-	spin_unlock_irqrestore(&flush_data->lock, flags);
-}
-
 static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
 {
 	struct dmar_domain *domain;
@@ -3784,7 +3689,8 @@ static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
 		free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
 		dma_free_pagelist(freelist);
 	} else {
-		add_unmap(domain, iova_pfn, nrpages, freelist);
+		queue_iova(&domain->iovad, iova_pfn, nrpages,
+			   (unsigned long)freelist);
 		/*
 		 * queue up the release of the unmap to save the 1/6th of the
 		 * cpu used up by the iotlb flush operation...
@@ -3938,7 +3844,8 @@ static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nele
 	ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
 	if (unlikely(ret)) {
 		dma_pte_free_pagetable(domain, start_vpfn,
-				       start_vpfn + size - 1);
+				       start_vpfn + size - 1,
+				       agaw_to_level(domain->agaw) + 1);
 		free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
 		return 0;
 	}
@@ -4721,7 +4628,6 @@ static void free_all_cpu_cached_iovas(unsigned int cpu)
 static int intel_iommu_cpu_dead(unsigned int cpu)
 {
 	free_all_cpu_cached_iovas(cpu);
-	flush_unmaps_timeout(cpu);
 	return 0;
 }
 
@@ -4736,7 +4642,9 @@ static void intel_disable_iommus(void)
 
 static inline struct intel_iommu *dev_to_intel_iommu(struct device *dev)
 {
-	return container_of(dev, struct intel_iommu, iommu.dev);
+	struct iommu_device *iommu_dev = dev_to_iommu_device(dev);
+
+	return container_of(iommu_dev, struct intel_iommu, iommu);
 }
 
 static ssize_t intel_iommu_show_version(struct device *dev,
@@ -5341,7 +5249,8 @@ int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sd
 	sdev->sid = PCI_DEVID(info->bus, info->devfn);
 
 	if (!(ctx_lo & CONTEXT_PASIDE)) {
-		context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
+		if (iommu->pasid_state_table)
+			context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
 		context[1].lo = (u64)virt_to_phys(iommu->pasid_table) |
 			intel_iommu_get_pts(iommu);
 
diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c
index f167c0d84ebf..f6697e55c2d4 100644
--- a/drivers/iommu/intel-svm.c
+++ b/drivers/iommu/intel-svm.c
@@ -24,6 +24,7 @@
 #include <linux/pci-ats.h>
 #include <linux/dmar.h>
 #include <linux/interrupt.h>
+#include <asm/page.h>
 
 static irqreturn_t prq_event_thread(int irq, void *d);
 
@@ -223,14 +224,6 @@ static void intel_change_pte(struct mmu_notifier *mn, struct mm_struct *mm,
 	intel_flush_svm_range(svm, address, 1, 1, 0);
 }
 
-static void intel_invalidate_page(struct mmu_notifier *mn, struct mm_struct *mm,
-				  unsigned long address)
-{
-	struct intel_svm *svm = container_of(mn, struct intel_svm, notifier);
-
-	intel_flush_svm_range(svm, address, 1, 1, 0);
-}
-
 /* Pages have been freed at this point */
 static void intel_invalidate_range(struct mmu_notifier *mn,
 				   struct mm_struct *mm,
@@ -285,7 +278,6 @@ static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
 static const struct mmu_notifier_ops intel_mmuops = {
 	.release = intel_mm_release,
 	.change_pte = intel_change_pte,
-	.invalidate_page = intel_invalidate_page,
 	.invalidate_range = intel_invalidate_range,
 };
 
@@ -555,6 +547,14 @@ static bool access_error(struct vm_area_struct *vma, struct page_req_dsc *req)
 	return (requested & ~vma->vm_flags) != 0;
 }
 
+static bool is_canonical_address(u64 addr)
+{
+	int shift = 64 - (__VIRTUAL_MASK_SHIFT + 1);
+	long saddr = (long) addr;
+
+	return (((saddr << shift) >> shift) == saddr);
+}
+
 static irqreturn_t prq_event_thread(int irq, void *d)
 {
 	struct intel_iommu *iommu = d;
@@ -612,6 +612,11 @@ static irqreturn_t prq_event_thread(int irq, void *d)
 		/* If the mm is already defunct, don't handle faults. */
 		if (!mmget_not_zero(svm->mm))
 			goto bad_req;
+
+		/* If address is not canonical, return invalid response */
+		if (!is_canonical_address(address))
+			goto bad_req;
+
 		down_read(&svm->mm->mmap_sem);
 		vma = find_extend_vma(svm->mm, address);
 		if (!vma || address < vma->vm_start)
diff --git a/drivers/iommu/iommu-sysfs.c b/drivers/iommu/iommu-sysfs.c
index c58351ed61c1..36d1a7ce7fc4 100644
--- a/drivers/iommu/iommu-sysfs.c
+++ b/drivers/iommu/iommu-sysfs.c
@@ -62,32 +62,40 @@ int iommu_device_sysfs_add(struct iommu_device *iommu,
 	va_list vargs;
 	int ret;
 
-	device_initialize(&iommu->dev);
+	iommu->dev = kzalloc(sizeof(*iommu->dev), GFP_KERNEL);
+	if (!iommu->dev)
+		return -ENOMEM;
 
-	iommu->dev.class = &iommu_class;
-	iommu->dev.parent = parent;
-	iommu->dev.groups = groups;
+	device_initialize(iommu->dev);
+
+	iommu->dev->class = &iommu_class;
+	iommu->dev->parent = parent;
+	iommu->dev->groups = groups;
 
 	va_start(vargs, fmt);
-	ret = kobject_set_name_vargs(&iommu->dev.kobj, fmt, vargs);
+	ret = kobject_set_name_vargs(&iommu->dev->kobj, fmt, vargs);
 	va_end(vargs);
 	if (ret)
 		goto error;
 
-	ret = device_add(&iommu->dev);
+	ret = device_add(iommu->dev);
 	if (ret)
 		goto error;
 
+	dev_set_drvdata(iommu->dev, iommu);
+
 	return 0;
 
 error:
-	put_device(&iommu->dev);
+	put_device(iommu->dev);
 	return ret;
 }
 
 void iommu_device_sysfs_remove(struct iommu_device *iommu)
 {
-	device_unregister(&iommu->dev);
+	dev_set_drvdata(iommu->dev, NULL);
+	device_unregister(iommu->dev);
+	iommu->dev = NULL;
 }
 /*
  * IOMMU drivers can indicate a device is managed by a given IOMMU using
@@ -102,14 +110,14 @@ int iommu_device_link(struct iommu_device *iommu, struct device *link)
 	if (!iommu || IS_ERR(iommu))
 		return -ENODEV;
 
-	ret = sysfs_add_link_to_group(&iommu->dev.kobj, "devices",
+	ret = sysfs_add_link_to_group(&iommu->dev->kobj, "devices",
 				      &link->kobj, dev_name(link));
 	if (ret)
 		return ret;
 
-	ret = sysfs_create_link_nowarn(&link->kobj, &iommu->dev.kobj, "iommu");
+	ret = sysfs_create_link_nowarn(&link->kobj, &iommu->dev->kobj, "iommu");
 	if (ret)
-		sysfs_remove_link_from_group(&iommu->dev.kobj, "devices",
+		sysfs_remove_link_from_group(&iommu->dev->kobj, "devices",
 					     dev_name(link));
 
 	return ret;
@@ -121,5 +129,5 @@ void iommu_device_unlink(struct iommu_device *iommu, struct device *link)
 		return;
 
 	sysfs_remove_link(&link->kobj, "iommu");
-	sysfs_remove_link_from_group(&iommu->dev.kobj, "devices", dev_name(link));
+	sysfs_remove_link_from_group(&iommu->dev->kobj, "devices", dev_name(link));
 }
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index 3f6ea160afed..3de5c0bcb5cc 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -527,6 +527,8 @@ static int iommu_group_create_direct_mappings(struct iommu_group *group,
 
 	}
 
+	iommu_flush_tlb_all(domain);
+
 out:
 	iommu_put_resv_regions(dev, &mappings);
 
@@ -1005,11 +1007,10 @@ struct iommu_group *iommu_group_get_for_dev(struct device *dev)
 	if (group)
 		return group;
 
-	group = ERR_PTR(-EINVAL);
-
-	if (ops && ops->device_group)
-		group = ops->device_group(dev);
+	if (!ops)
+		return ERR_PTR(-EINVAL);
 
+	group = ops->device_group(dev);
 	if (WARN_ON_ONCE(group == NULL))
 		return ERR_PTR(-EINVAL);
 
@@ -1283,6 +1284,10 @@ static int __iommu_attach_device(struct iommu_domain *domain,
 				 struct device *dev)
 {
 	int ret;
+	if ((domain->ops->is_attach_deferred != NULL) &&
+	    domain->ops->is_attach_deferred(domain, dev))
+		return 0;
+
 	if (unlikely(domain->ops->attach_dev == NULL))
 		return -ENODEV;
 
@@ -1298,12 +1303,8 @@ int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
 	int ret;
 
 	group = iommu_group_get(dev);
-	/* FIXME: Remove this when groups a mandatory for iommu drivers */
-	if (group == NULL)
-		return __iommu_attach_device(domain, dev);
-
 	/*
-	 * We have a group - lock it to make sure the device-count doesn't
+	 * Lock the group to make sure the device-count doesn't
 	 * change while we are attaching
 	 */
 	mutex_lock(&group->mutex);
@@ -1324,6 +1325,10 @@ EXPORT_SYMBOL_GPL(iommu_attach_device);
 static void __iommu_detach_device(struct iommu_domain *domain,
 				  struct device *dev)
 {
+	if ((domain->ops->is_attach_deferred != NULL) &&
+	    domain->ops->is_attach_deferred(domain, dev))
+		return;
+
 	if (unlikely(domain->ops->detach_dev == NULL))
 		return;
 
@@ -1336,9 +1341,6 @@ void iommu_detach_device(struct iommu_domain *domain, struct device *dev)
 	struct iommu_group *group;
 
 	group = iommu_group_get(dev);
-	/* FIXME: Remove this when groups a mandatory for iommu drivers */
-	if (group == NULL)
-		return __iommu_detach_device(domain, dev);
 
 	mutex_lock(&group->mutex);
 	if (iommu_group_device_count(group) != 1) {
@@ -1360,8 +1362,7 @@ struct iommu_domain *iommu_get_domain_for_dev(struct device *dev)
 	struct iommu_group *group;
 
 	group = iommu_group_get(dev);
-	/* FIXME: Remove this when groups a mandatory for iommu drivers */
-	if (group == NULL)
+	if (!group)
 		return NULL;
 
 	domain = group->domain;
@@ -1556,13 +1557,16 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova,
 }
 EXPORT_SYMBOL_GPL(iommu_map);
 
-size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
+static size_t __iommu_unmap(struct iommu_domain *domain,
+			    unsigned long iova, size_t size,
+			    bool sync)
 {
+	const struct iommu_ops *ops = domain->ops;
 	size_t unmapped_page, unmapped = 0;
-	unsigned int min_pagesz;
 	unsigned long orig_iova = iova;
+	unsigned int min_pagesz;
 
-	if (unlikely(domain->ops->unmap == NULL ||
+	if (unlikely(ops->unmap == NULL ||
 		     domain->pgsize_bitmap == 0UL))
 		return -ENODEV;
 
@@ -1592,10 +1596,13 @@ size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
 	while (unmapped < size) {
 		size_t pgsize = iommu_pgsize(domain, iova, size - unmapped);
 
-		unmapped_page = domain->ops->unmap(domain, iova, pgsize);
+		unmapped_page = ops->unmap(domain, iova, pgsize);
 		if (!unmapped_page)
 			break;
 
+		if (sync && ops->iotlb_range_add)
+			ops->iotlb_range_add(domain, iova, pgsize);
+
 		pr_debug("unmapped: iova 0x%lx size 0x%zx\n",
 			 iova, unmapped_page);
 
@@ -1603,11 +1610,27 @@ size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
 		unmapped += unmapped_page;
 	}
 
+	if (sync && ops->iotlb_sync)
+		ops->iotlb_sync(domain);
+
 	trace_unmap(orig_iova, size, unmapped);
 	return unmapped;
 }
+
+size_t iommu_unmap(struct iommu_domain *domain,
+		   unsigned long iova, size_t size)
+{
+	return __iommu_unmap(domain, iova, size, true);
+}
 EXPORT_SYMBOL_GPL(iommu_unmap);
 
+size_t iommu_unmap_fast(struct iommu_domain *domain,
+			unsigned long iova, size_t size)
+{
+	return __iommu_unmap(domain, iova, size, false);
+}
+EXPORT_SYMBOL_GPL(iommu_unmap_fast);
+
 size_t default_iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
 			 struct scatterlist *sg, unsigned int nents, int prot)
 {
diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c
index 246f14c83944..33edfa794ae9 100644
--- a/drivers/iommu/iova.c
+++ b/drivers/iommu/iova.c
@@ -32,6 +32,8 @@ static unsigned long iova_rcache_get(struct iova_domain *iovad,
 				     unsigned long limit_pfn);
 static void init_iova_rcaches(struct iova_domain *iovad);
 static void free_iova_rcaches(struct iova_domain *iovad);
+static void fq_destroy_all_entries(struct iova_domain *iovad);
+static void fq_flush_timeout(unsigned long data);
 
 void
 init_iova_domain(struct iova_domain *iovad, unsigned long granule,
@@ -50,10 +52,61 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule,
 	iovad->granule = granule;
 	iovad->start_pfn = start_pfn;
 	iovad->dma_32bit_pfn = pfn_32bit + 1;
+	iovad->flush_cb = NULL;
+	iovad->fq = NULL;
 	init_iova_rcaches(iovad);
 }
 EXPORT_SYMBOL_GPL(init_iova_domain);
 
+static void free_iova_flush_queue(struct iova_domain *iovad)
+{
+	if (!iovad->fq)
+		return;
+
+	if (timer_pending(&iovad->fq_timer))
+		del_timer(&iovad->fq_timer);
+
+	fq_destroy_all_entries(iovad);
+
+	free_percpu(iovad->fq);
+
+	iovad->fq         = NULL;
+	iovad->flush_cb   = NULL;
+	iovad->entry_dtor = NULL;
+}
+
+int init_iova_flush_queue(struct iova_domain *iovad,
+			  iova_flush_cb flush_cb, iova_entry_dtor entry_dtor)
+{
+	int cpu;
+
+	atomic64_set(&iovad->fq_flush_start_cnt,  0);
+	atomic64_set(&iovad->fq_flush_finish_cnt, 0);
+
+	iovad->fq = alloc_percpu(struct iova_fq);
+	if (!iovad->fq)
+		return -ENOMEM;
+
+	iovad->flush_cb   = flush_cb;
+	iovad->entry_dtor = entry_dtor;
+
+	for_each_possible_cpu(cpu) {
+		struct iova_fq *fq;
+
+		fq = per_cpu_ptr(iovad->fq, cpu);
+		fq->head = 0;
+		fq->tail = 0;
+
+		spin_lock_init(&fq->lock);
+	}
+
+	setup_timer(&iovad->fq_timer, fq_flush_timeout, (unsigned long)iovad);
+	atomic_set(&iovad->fq_timer_on, 0);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(init_iova_flush_queue);
+
 static struct rb_node *
 __get_cached_rbnode(struct iova_domain *iovad, unsigned long *limit_pfn)
 {
@@ -423,6 +476,135 @@ free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size)
 }
 EXPORT_SYMBOL_GPL(free_iova_fast);
 
+#define fq_ring_for_each(i, fq) \
+	for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE)
+
+static inline bool fq_full(struct iova_fq *fq)
+{
+	assert_spin_locked(&fq->lock);
+	return (((fq->tail + 1) % IOVA_FQ_SIZE) == fq->head);
+}
+
+static inline unsigned fq_ring_add(struct iova_fq *fq)
+{
+	unsigned idx = fq->tail;
+
+	assert_spin_locked(&fq->lock);
+
+	fq->tail = (idx + 1) % IOVA_FQ_SIZE;
+
+	return idx;
+}
+
+static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq)
+{
+	u64 counter = atomic64_read(&iovad->fq_flush_finish_cnt);
+	unsigned idx;
+
+	assert_spin_locked(&fq->lock);
+
+	fq_ring_for_each(idx, fq) {
+
+		if (fq->entries[idx].counter >= counter)
+			break;
+
+		if (iovad->entry_dtor)
+			iovad->entry_dtor(fq->entries[idx].data);
+
+		free_iova_fast(iovad,
+			       fq->entries[idx].iova_pfn,
+			       fq->entries[idx].pages);
+
+		fq->head = (fq->head + 1) % IOVA_FQ_SIZE;
+	}
+}
+
+static void iova_domain_flush(struct iova_domain *iovad)
+{
+	atomic64_inc(&iovad->fq_flush_start_cnt);
+	iovad->flush_cb(iovad);
+	atomic64_inc(&iovad->fq_flush_finish_cnt);
+}
+
+static void fq_destroy_all_entries(struct iova_domain *iovad)
+{
+	int cpu;
+
+	/*
+	 * This code runs when the iova_domain is being detroyed, so don't
+	 * bother to free iovas, just call the entry_dtor on all remaining
+	 * entries.
+	 */
+	if (!iovad->entry_dtor)
+		return;
+
+	for_each_possible_cpu(cpu) {
+		struct iova_fq *fq = per_cpu_ptr(iovad->fq, cpu);
+		int idx;
+
+		fq_ring_for_each(idx, fq)
+			iovad->entry_dtor(fq->entries[idx].data);
+	}
+}
+
+static void fq_flush_timeout(unsigned long data)
+{
+	struct iova_domain *iovad = (struct iova_domain *)data;
+	int cpu;
+
+	atomic_set(&iovad->fq_timer_on, 0);
+	iova_domain_flush(iovad);
+
+	for_each_possible_cpu(cpu) {
+		unsigned long flags;
+		struct iova_fq *fq;
+
+		fq = per_cpu_ptr(iovad->fq, cpu);
+		spin_lock_irqsave(&fq->lock, flags);
+		fq_ring_free(iovad, fq);
+		spin_unlock_irqrestore(&fq->lock, flags);
+	}
+}
+
+void queue_iova(struct iova_domain *iovad,
+		unsigned long pfn, unsigned long pages,
+		unsigned long data)
+{
+	struct iova_fq *fq = get_cpu_ptr(iovad->fq);
+	unsigned long flags;
+	unsigned idx;
+
+	spin_lock_irqsave(&fq->lock, flags);
+
+	/*
+	 * First remove all entries from the flush queue that have already been
+	 * flushed out on another CPU. This makes the fq_full() check below less
+	 * likely to be true.
+	 */
+	fq_ring_free(iovad, fq);
+
+	if (fq_full(fq)) {
+		iova_domain_flush(iovad);
+		fq_ring_free(iovad, fq);
+	}
+
+	idx = fq_ring_add(fq);
+
+	fq->entries[idx].iova_pfn = pfn;
+	fq->entries[idx].pages    = pages;
+	fq->entries[idx].data     = data;
+	fq->entries[idx].counter  = atomic64_read(&iovad->fq_flush_start_cnt);
+
+	spin_unlock_irqrestore(&fq->lock, flags);
+
+	if (atomic_cmpxchg(&iovad->fq_timer_on, 0, 1) == 0)
+		mod_timer(&iovad->fq_timer,
+			  jiffies + msecs_to_jiffies(IOVA_FQ_TIMEOUT));
+
+	put_cpu_ptr(iovad->fq);
+}
+EXPORT_SYMBOL_GPL(queue_iova);
+
 /**
  * put_iova_domain - destroys the iova doamin
  * @iovad: - iova domain in question.
@@ -433,6 +615,7 @@ void put_iova_domain(struct iova_domain *iovad)
 	struct rb_node *node;
 	unsigned long flags;
 
+	free_iova_flush_queue(iovad);
 	free_iova_rcaches(iovad);
 	spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
 	node = rb_first(&iovad->rbroot);
diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c
index 2a38aa15be17..195d6e93ac71 100644
--- a/drivers/iommu/ipmmu-vmsa.c
+++ b/drivers/iommu/ipmmu-vmsa.c
@@ -19,6 +19,7 @@
 #include <linux/iommu.h>
 #include <linux/module.h>
 #include <linux/of.h>
+#include <linux/of_platform.h>
 #include <linux/platform_device.h>
 #include <linux/sizes.h>
 #include <linux/slab.h>
@@ -35,7 +36,7 @@
 struct ipmmu_vmsa_device {
 	struct device *dev;
 	void __iomem *base;
-	struct list_head list;
+	struct iommu_device iommu;
 
 	unsigned int num_utlbs;
 	spinlock_t lock;			/* Protects ctx and domains[] */
@@ -58,36 +59,18 @@ struct ipmmu_vmsa_domain {
 
 struct ipmmu_vmsa_iommu_priv {
 	struct ipmmu_vmsa_device *mmu;
-	unsigned int *utlbs;
-	unsigned int num_utlbs;
 	struct device *dev;
 	struct list_head list;
 };
 
-static DEFINE_SPINLOCK(ipmmu_devices_lock);
-static LIST_HEAD(ipmmu_devices);
-
 static struct ipmmu_vmsa_domain *to_vmsa_domain(struct iommu_domain *dom)
 {
 	return container_of(dom, struct ipmmu_vmsa_domain, io_domain);
 }
 
-
 static struct ipmmu_vmsa_iommu_priv *to_priv(struct device *dev)
 {
-#if defined(CONFIG_ARM)
-	return dev->archdata.iommu;
-#else
-	return dev->iommu_fwspec->iommu_priv;
-#endif
-}
-static void set_priv(struct device *dev, struct ipmmu_vmsa_iommu_priv *p)
-{
-#if defined(CONFIG_ARM)
-	dev->archdata.iommu = p;
-#else
-	dev->iommu_fwspec->iommu_priv = p;
-#endif
+	return dev->iommu_fwspec ? dev->iommu_fwspec->iommu_priv : NULL;
 }
 
 #define TLB_LOOP_TIMEOUT		100	/* 100us */
@@ -312,7 +295,7 @@ static void ipmmu_tlb_add_flush(unsigned long iova, size_t size,
 	/* The hardware doesn't support selective TLB flush. */
 }
 
-static struct iommu_gather_ops ipmmu_gather_ops = {
+static const struct iommu_gather_ops ipmmu_gather_ops = {
 	.tlb_flush_all = ipmmu_tlb_flush_all,
 	.tlb_add_flush = ipmmu_tlb_add_flush,
 	.tlb_sync = ipmmu_tlb_flush_all,
@@ -341,6 +324,19 @@ static int ipmmu_domain_allocate_context(struct ipmmu_vmsa_device *mmu,
 	return ret;
 }
 
+static void ipmmu_domain_free_context(struct ipmmu_vmsa_device *mmu,
+				      unsigned int context_id)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&mmu->lock, flags);
+
+	clear_bit(context_id, mmu->ctx);
+	mmu->domains[context_id] = NULL;
+
+	spin_unlock_irqrestore(&mmu->lock, flags);
+}
+
 static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain)
 {
 	u64 ttbr;
@@ -370,22 +366,22 @@ static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain)
 	 */
 	domain->cfg.iommu_dev = domain->mmu->dev;
 
-	domain->iop = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &domain->cfg,
-					   domain);
-	if (!domain->iop)
-		return -EINVAL;
-
 	/*
 	 * Find an unused context.
 	 */
 	ret = ipmmu_domain_allocate_context(domain->mmu, domain);
-	if (ret == IPMMU_CTX_MAX) {
-		free_io_pgtable_ops(domain->iop);
+	if (ret == IPMMU_CTX_MAX)
 		return -EBUSY;
-	}
 
 	domain->context_id = ret;
 
+	domain->iop = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &domain->cfg,
+					   domain);
+	if (!domain->iop) {
+		ipmmu_domain_free_context(domain->mmu, domain->context_id);
+		return -EINVAL;
+	}
+
 	/* TTBR0 */
 	ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr[0];
 	ipmmu_ctx_write(domain, IMTTLBR0, ttbr);
@@ -426,19 +422,6 @@ static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain)
 	return 0;
 }
 
-static void ipmmu_domain_free_context(struct ipmmu_vmsa_device *mmu,
-				      unsigned int context_id)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&mmu->lock, flags);
-
-	clear_bit(context_id, mmu->ctx);
-	mmu->domains[context_id] = NULL;
-
-	spin_unlock_irqrestore(&mmu->lock, flags);
-}
-
 static void ipmmu_domain_destroy_context(struct ipmmu_vmsa_domain *domain)
 {
 	/*
@@ -562,13 +545,14 @@ static int ipmmu_attach_device(struct iommu_domain *io_domain,
 			       struct device *dev)
 {
 	struct ipmmu_vmsa_iommu_priv *priv = to_priv(dev);
+	struct iommu_fwspec *fwspec = dev->iommu_fwspec;
 	struct ipmmu_vmsa_device *mmu = priv->mmu;
 	struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
 	unsigned long flags;
 	unsigned int i;
 	int ret = 0;
 
-	if (!mmu) {
+	if (!priv || !priv->mmu) {
 		dev_err(dev, "Cannot attach to IPMMU\n");
 		return -ENXIO;
 	}
@@ -595,8 +579,8 @@ static int ipmmu_attach_device(struct iommu_domain *io_domain,
 	if (ret < 0)
 		return ret;
 
-	for (i = 0; i < priv->num_utlbs; ++i)
-		ipmmu_utlb_enable(domain, priv->utlbs[i]);
+	for (i = 0; i < fwspec->num_ids; ++i)
+		ipmmu_utlb_enable(domain, fwspec->ids[i]);
 
 	return 0;
 }
@@ -604,12 +588,12 @@ static int ipmmu_attach_device(struct iommu_domain *io_domain,
 static void ipmmu_detach_device(struct iommu_domain *io_domain,
 				struct device *dev)
 {
-	struct ipmmu_vmsa_iommu_priv *priv = to_priv(dev);
+	struct iommu_fwspec *fwspec = dev->iommu_fwspec;
 	struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
 	unsigned int i;
 
-	for (i = 0; i < priv->num_utlbs; ++i)
-		ipmmu_utlb_disable(domain, priv->utlbs[i]);
+	for (i = 0; i < fwspec->num_ids; ++i)
+		ipmmu_utlb_disable(domain, fwspec->ids[i]);
 
 	/*
 	 * TODO: Optimize by disabling the context when no device is attached.
@@ -645,92 +629,36 @@ static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain,
 	return domain->iop->iova_to_phys(domain->iop, iova);
 }
 
-static int ipmmu_find_utlbs(struct ipmmu_vmsa_device *mmu, struct device *dev,
-			    unsigned int *utlbs, unsigned int num_utlbs)
-{
-	unsigned int i;
-
-	for (i = 0; i < num_utlbs; ++i) {
-		struct of_phandle_args args;
-		int ret;
-
-		ret = of_parse_phandle_with_args(dev->of_node, "iommus",
-						 "#iommu-cells", i, &args);
-		if (ret < 0)
-			return ret;
-
-		of_node_put(args.np);
-
-		if (args.np != mmu->dev->of_node || args.args_count != 1)
-			return -EINVAL;
-
-		utlbs[i] = args.args[0];
-	}
-
-	return 0;
-}
-
-static int ipmmu_init_platform_device(struct device *dev)
+static int ipmmu_init_platform_device(struct device *dev,
+				      struct of_phandle_args *args)
 {
+	struct platform_device *ipmmu_pdev;
 	struct ipmmu_vmsa_iommu_priv *priv;
-	struct ipmmu_vmsa_device *mmu;
-	unsigned int *utlbs;
-	unsigned int i;
-	int num_utlbs;
-	int ret = -ENODEV;
-
-	/* Find the master corresponding to the device. */
 
-	num_utlbs = of_count_phandle_with_args(dev->of_node, "iommus",
-					       "#iommu-cells");
-	if (num_utlbs < 0)
+	ipmmu_pdev = of_find_device_by_node(args->np);
+	if (!ipmmu_pdev)
 		return -ENODEV;
 
-	utlbs = kcalloc(num_utlbs, sizeof(*utlbs), GFP_KERNEL);
-	if (!utlbs)
-		return -ENOMEM;
-
-	spin_lock(&ipmmu_devices_lock);
-
-	list_for_each_entry(mmu, &ipmmu_devices, list) {
-		ret = ipmmu_find_utlbs(mmu, dev, utlbs, num_utlbs);
-		if (!ret) {
-			/*
-			 * TODO Take a reference to the MMU to protect
-			 * against device removal.
-			 */
-			break;
-		}
-	}
-
-	spin_unlock(&ipmmu_devices_lock);
-
-	if (ret < 0)
-		goto error;
-
-	for (i = 0; i < num_utlbs; ++i) {
-		if (utlbs[i] >= mmu->num_utlbs) {
-			ret = -EINVAL;
-			goto error;
-		}
-	}
-
 	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
-	if (!priv) {
-		ret = -ENOMEM;
-		goto error;
-	}
+	if (!priv)
+		return -ENOMEM;
 
-	priv->mmu = mmu;
-	priv->utlbs = utlbs;
-	priv->num_utlbs = num_utlbs;
+	priv->mmu = platform_get_drvdata(ipmmu_pdev);
 	priv->dev = dev;
-	set_priv(dev, priv);
+	dev->iommu_fwspec->iommu_priv = priv;
 	return 0;
+}
 
-error:
-	kfree(utlbs);
-	return ret;
+static int ipmmu_of_xlate(struct device *dev,
+			  struct of_phandle_args *spec)
+{
+	iommu_fwspec_add_ids(dev, spec->args, 1);
+
+	/* Initialize once - xlate() will call multiple times */
+	if (to_priv(dev))
+		return 0;
+
+	return ipmmu_init_platform_device(dev, spec);
 }
 
 #if defined(CONFIG_ARM) && !defined(CONFIG_IOMMU_DMA)
@@ -749,11 +677,11 @@ static int ipmmu_add_device(struct device *dev)
 	struct iommu_group *group;
 	int ret;
 
-	if (to_priv(dev)) {
-		dev_warn(dev, "IOMMU driver already assigned to device %s\n",
-			 dev_name(dev));
-		return -EINVAL;
-	}
+	/*
+	 * Only let through devices that have been verified in xlate()
+	 */
+	if (!to_priv(dev))
+		return -ENODEV;
 
 	/* Create a device group and add the device to it. */
 	group = iommu_group_alloc();
@@ -772,10 +700,6 @@ static int ipmmu_add_device(struct device *dev)
 		goto error;
 	}
 
-	ret = ipmmu_init_platform_device(dev);
-	if (ret < 0)
-		goto error;
-
 	/*
 	 * Create the ARM mapping, used by the ARM DMA mapping core to allocate
 	 * VAs. This will allocate a corresponding IOMMU domain.
@@ -816,24 +740,13 @@ error:
 	if (!IS_ERR_OR_NULL(group))
 		iommu_group_remove_device(dev);
 
-	kfree(to_priv(dev)->utlbs);
-	kfree(to_priv(dev));
-	set_priv(dev, NULL);
-
 	return ret;
 }
 
 static void ipmmu_remove_device(struct device *dev)
 {
-	struct ipmmu_vmsa_iommu_priv *priv = to_priv(dev);
-
 	arm_iommu_detach_device(dev);
 	iommu_group_remove_device(dev);
-
-	kfree(priv->utlbs);
-	kfree(priv);
-
-	set_priv(dev, NULL);
 }
 
 static const struct iommu_ops ipmmu_ops = {
@@ -848,6 +761,7 @@ static const struct iommu_ops ipmmu_ops = {
 	.add_device = ipmmu_add_device,
 	.remove_device = ipmmu_remove_device,
 	.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K,
+	.of_xlate = ipmmu_of_xlate,
 };
 
 #endif /* !CONFIG_ARM && CONFIG_IOMMU_DMA */
@@ -890,14 +804,12 @@ static void ipmmu_domain_free_dma(struct iommu_domain *io_domain)
 
 static int ipmmu_add_device_dma(struct device *dev)
 {
-	struct iommu_fwspec *fwspec = dev->iommu_fwspec;
 	struct iommu_group *group;
 
 	/*
 	 * Only let through devices that have been verified in xlate()
-	 * We may get called with dev->iommu_fwspec set to NULL.
 	 */
-	if (!fwspec || !fwspec->iommu_priv)
+	if (!to_priv(dev))
 		return -ENODEV;
 
 	group = iommu_group_get_for_dev(dev);
@@ -957,19 +869,6 @@ static struct iommu_group *ipmmu_find_group_dma(struct device *dev)
 	return group;
 }
 
-static int ipmmu_of_xlate_dma(struct device *dev,
-			      struct of_phandle_args *spec)
-{
-	/* If the IPMMU device is disabled in DT then return error
-	 * to make sure the of_iommu code does not install ops
-	 * even though the iommu device is disabled
-	 */
-	if (!of_device_is_available(spec->np))
-		return -ENODEV;
-
-	return ipmmu_init_platform_device(dev);
-}
-
 static const struct iommu_ops ipmmu_ops = {
 	.domain_alloc = ipmmu_domain_alloc_dma,
 	.domain_free = ipmmu_domain_free_dma,
@@ -983,7 +882,7 @@ static const struct iommu_ops ipmmu_ops = {
 	.remove_device = ipmmu_remove_device_dma,
 	.device_group = ipmmu_find_group_dma,
 	.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K,
-	.of_xlate = ipmmu_of_xlate_dma,
+	.of_xlate = ipmmu_of_xlate,
 };
 
 #endif /* CONFIG_IOMMU_DMA */
@@ -1054,16 +953,24 @@ static int ipmmu_probe(struct platform_device *pdev)
 
 	ipmmu_device_reset(mmu);
 
+	ret = iommu_device_sysfs_add(&mmu->iommu, &pdev->dev, NULL,
+				     dev_name(&pdev->dev));
+	if (ret)
+		return ret;
+
+	iommu_device_set_ops(&mmu->iommu, &ipmmu_ops);
+	iommu_device_set_fwnode(&mmu->iommu, &pdev->dev.of_node->fwnode);
+
+	ret = iommu_device_register(&mmu->iommu);
+	if (ret)
+		return ret;
+
 	/*
 	 * We can't create the ARM mapping here as it requires the bus to have
 	 * an IOMMU, which only happens when bus_set_iommu() is called in
 	 * ipmmu_init() after the probe function returns.
 	 */
 
-	spin_lock(&ipmmu_devices_lock);
-	list_add(&mmu->list, &ipmmu_devices);
-	spin_unlock(&ipmmu_devices_lock);
-
 	platform_set_drvdata(pdev, mmu);
 
 	return 0;
@@ -1073,9 +980,8 @@ static int ipmmu_remove(struct platform_device *pdev)
 {
 	struct ipmmu_vmsa_device *mmu = platform_get_drvdata(pdev);
 
-	spin_lock(&ipmmu_devices_lock);
-	list_del(&mmu->list);
-	spin_unlock(&ipmmu_devices_lock);
+	iommu_device_sysfs_remove(&mmu->iommu);
+	iommu_device_unregister(&mmu->iommu);
 
 #if defined(CONFIG_ARM) && !defined(CONFIG_IOMMU_DMA)
 	arm_iommu_release_mapping(mmu->mapping);
diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c
index d0448353d501..04f4d51ffacb 100644
--- a/drivers/iommu/msm_iommu.c
+++ b/drivers/iommu/msm_iommu.c
@@ -393,6 +393,7 @@ static struct msm_iommu_dev *find_iommu_for_dev(struct device *dev)
 static int msm_iommu_add_device(struct device *dev)
 {
 	struct msm_iommu_dev *iommu;
+	struct iommu_group *group;
 	unsigned long flags;
 	int ret = 0;
 
@@ -406,7 +407,16 @@ static int msm_iommu_add_device(struct device *dev)
 
 	spin_unlock_irqrestore(&msm_iommu_lock, flags);
 
-	return ret;
+	if (ret)
+		return ret;
+
+	group = iommu_group_get_for_dev(dev);
+	if (IS_ERR(group))
+		return PTR_ERR(group);
+
+	iommu_group_put(group);
+
+	return 0;
 }
 
 static void msm_iommu_remove_device(struct device *dev)
@@ -421,6 +431,8 @@ static void msm_iommu_remove_device(struct device *dev)
 		iommu_device_unlink(&iommu->iommu, dev);
 
 	spin_unlock_irqrestore(&msm_iommu_lock, flags);
+
+	iommu_group_remove_device(dev);
 }
 
 static int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
@@ -700,6 +712,7 @@ static struct iommu_ops msm_iommu_ops = {
 	.iova_to_phys = msm_iommu_iova_to_phys,
 	.add_device = msm_iommu_add_device,
 	.remove_device = msm_iommu_remove_device,
+	.device_group = generic_device_group,
 	.pgsize_bitmap = MSM_IOMMU_PGSIZES,
 	.of_xlate = qcom_iommu_of_xlate,
 };
diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c
index 91c6d367ab35..bd515be5b380 100644
--- a/drivers/iommu/mtk_iommu.c
+++ b/drivers/iommu/mtk_iommu.c
@@ -31,7 +31,6 @@
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <asm/barrier.h>
-#include <dt-bindings/memory/mt8173-larb-port.h>
 #include <soc/mediatek/smi.h>
 
 #include "mtk_iommu.h"
@@ -54,10 +53,16 @@
 
 #define REG_MMU_CTRL_REG			0x110
 #define F_MMU_PREFETCH_RT_REPLACE_MOD		BIT(4)
-#define F_MMU_TF_PROTECT_SEL(prot)		(((prot) & 0x3) << 5)
+#define F_MMU_TF_PROTECT_SEL_SHIFT(data) \
+	((data)->m4u_plat == M4U_MT2712 ? 4 : 5)
+/* It's named by F_MMU_TF_PROT_SEL in mt2712. */
+#define F_MMU_TF_PROTECT_SEL(prot, data) \
+	(((prot) & 0x3) << F_MMU_TF_PROTECT_SEL_SHIFT(data))
 
 #define REG_MMU_IVRP_PADDR			0x114
 #define F_MMU_IVRP_PA_SET(pa, ext)		(((pa) >> 1) | ((!!(ext)) << 31))
+#define REG_MMU_VLD_PA_RNG			0x118
+#define F_MMU_VLD_PA_RNG(EA, SA)		(((EA) << 8) | (SA))
 
 #define REG_MMU_INT_CONTROL0			0x120
 #define F_L2_MULIT_HIT_EN			BIT(0)
@@ -82,7 +87,6 @@
 #define REG_MMU_FAULT_ST1			0x134
 
 #define REG_MMU_FAULT_VA			0x13c
-#define F_MMU_FAULT_VA_MSK			0xfffff000
 #define F_MMU_FAULT_VA_WRITE_BIT		BIT(1)
 #define F_MMU_FAULT_VA_LAYER_BIT		BIT(0)
 
@@ -93,6 +97,13 @@
 
 #define MTK_PROTECT_PA_ALIGN			128
 
+/*
+ * Get the local arbiter ID and the portid within the larb arbiter
+ * from mtk_m4u_id which is defined by MTK_M4U_ID.
+ */
+#define MTK_M4U_TO_LARB(id)		(((id) >> 5) & 0xf)
+#define MTK_M4U_TO_PORT(id)		((id) & 0x1f)
+
 struct mtk_iommu_domain {
 	spinlock_t			pgtlock; /* lock for page table */
 
@@ -104,6 +115,27 @@ struct mtk_iommu_domain {
 
 static struct iommu_ops mtk_iommu_ops;
 
+static LIST_HEAD(m4ulist);	/* List all the M4U HWs */
+
+#define for_each_m4u(data)	list_for_each_entry(data, &m4ulist, list)
+
+/*
+ * There may be 1 or 2 M4U HWs, But we always expect they are in the same domain
+ * for the performance.
+ *
+ * Here always return the mtk_iommu_data of the first probed M4U where the
+ * iommu domain information is recorded.
+ */
+static struct mtk_iommu_data *mtk_iommu_get_m4u_data(void)
+{
+	struct mtk_iommu_data *data;
+
+	for_each_m4u(data)
+		return data;
+
+	return NULL;
+}
+
 static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom)
 {
 	return container_of(dom, struct mtk_iommu_domain, domain);
@@ -113,9 +145,12 @@ static void mtk_iommu_tlb_flush_all(void *cookie)
 {
 	struct mtk_iommu_data *data = cookie;
 
-	writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, data->base + REG_MMU_INV_SEL);
-	writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE);
-	wmb(); /* Make sure the tlb flush all done */
+	for_each_m4u(data) {
+		writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0,
+			       data->base + REG_MMU_INV_SEL);
+		writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE);
+		wmb(); /* Make sure the tlb flush all done */
+	}
 }
 
 static void mtk_iommu_tlb_add_flush_nosync(unsigned long iova, size_t size,
@@ -124,12 +159,17 @@ static void mtk_iommu_tlb_add_flush_nosync(unsigned long iova, size_t size,
 {
 	struct mtk_iommu_data *data = cookie;
 
-	writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, data->base + REG_MMU_INV_SEL);
+	for_each_m4u(data) {
+		writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0,
+			       data->base + REG_MMU_INV_SEL);
 
-	writel_relaxed(iova, data->base + REG_MMU_INVLD_START_A);
-	writel_relaxed(iova + size - 1, data->base + REG_MMU_INVLD_END_A);
-	writel_relaxed(F_MMU_INV_RANGE, data->base + REG_MMU_INVALIDATE);
-	data->tlb_flush_active = true;
+		writel_relaxed(iova, data->base + REG_MMU_INVLD_START_A);
+		writel_relaxed(iova + size - 1,
+			       data->base + REG_MMU_INVLD_END_A);
+		writel_relaxed(F_MMU_INV_RANGE,
+			       data->base + REG_MMU_INVALIDATE);
+		data->tlb_flush_active = true;
+	}
 }
 
 static void mtk_iommu_tlb_sync(void *cookie)
@@ -138,20 +178,22 @@ static void mtk_iommu_tlb_sync(void *cookie)
 	int ret;
 	u32 tmp;
 
-	/* Avoid timing out if there's nothing to wait for */
-	if (!data->tlb_flush_active)
-		return;
+	for_each_m4u(data) {
+		/* Avoid timing out if there's nothing to wait for */
+		if (!data->tlb_flush_active)
+			return;
 
-	ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE, tmp,
-					tmp != 0, 10, 100000);
-	if (ret) {
-		dev_warn(data->dev,
-			 "Partial TLB flush timed out, falling back to full flush\n");
-		mtk_iommu_tlb_flush_all(cookie);
+		ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE,
+						tmp, tmp != 0, 10, 100000);
+		if (ret) {
+			dev_warn(data->dev,
+				 "Partial TLB flush timed out, falling back to full flush\n");
+			mtk_iommu_tlb_flush_all(cookie);
+		}
+		/* Clear the CPE status */
+		writel_relaxed(0, data->base + REG_MMU_CPE_DONE);
+		data->tlb_flush_active = false;
 	}
-	/* Clear the CPE status */
-	writel_relaxed(0, data->base + REG_MMU_CPE_DONE);
-	data->tlb_flush_active = false;
 }
 
 static const struct iommu_gather_ops mtk_iommu_gather_ops = {
@@ -173,7 +215,6 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id)
 	fault_iova = readl_relaxed(data->base + REG_MMU_FAULT_VA);
 	layer = fault_iova & F_MMU_FAULT_VA_LAYER_BIT;
 	write = fault_iova & F_MMU_FAULT_VA_WRITE_BIT;
-	fault_iova &= F_MMU_FAULT_VA_MSK;
 	fault_pa = readl_relaxed(data->base + REG_MMU_INVLD_PA);
 	regval = readl_relaxed(data->base + REG_MMU_INT_ID);
 	fault_larb = F_MMU0_INT_ID_LARB_ID(regval);
@@ -221,9 +262,9 @@ static void mtk_iommu_config(struct mtk_iommu_data *data,
 	}
 }
 
-static int mtk_iommu_domain_finalise(struct mtk_iommu_data *data)
+static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom)
 {
-	struct mtk_iommu_domain *dom = data->m4u_dom;
+	struct mtk_iommu_data *data = mtk_iommu_get_m4u_data();
 
 	spin_lock_init(&dom->pgtlock);
 
@@ -249,9 +290,6 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_data *data)
 
 	/* Update our support page sizes bitmap */
 	dom->domain.pgsize_bitmap = dom->cfg.pgsize_bitmap;
-
-	writel(data->m4u_dom->cfg.arm_v7s_cfg.ttbr[0],
-	       data->base + REG_MMU_PT_BASE_ADDR);
 	return 0;
 }
 
@@ -266,20 +304,30 @@ static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type)
 	if (!dom)
 		return NULL;
 
-	if (iommu_get_dma_cookie(&dom->domain)) {
-		kfree(dom);
-		return NULL;
-	}
+	if (iommu_get_dma_cookie(&dom->domain))
+		goto  free_dom;
+
+	if (mtk_iommu_domain_finalise(dom))
+		goto  put_dma_cookie;
 
 	dom->domain.geometry.aperture_start = 0;
 	dom->domain.geometry.aperture_end = DMA_BIT_MASK(32);
 	dom->domain.geometry.force_aperture = true;
 
 	return &dom->domain;
+
+put_dma_cookie:
+	iommu_put_dma_cookie(&dom->domain);
+free_dom:
+	kfree(dom);
+	return NULL;
 }
 
 static void mtk_iommu_domain_free(struct iommu_domain *domain)
 {
+	struct mtk_iommu_domain *dom = to_mtk_domain(domain);
+
+	free_io_pgtable_ops(dom->iop);
 	iommu_put_dma_cookie(domain);
 	kfree(to_mtk_domain(domain));
 }
@@ -289,22 +337,15 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain,
 {
 	struct mtk_iommu_domain *dom = to_mtk_domain(domain);
 	struct mtk_iommu_data *data = dev->iommu_fwspec->iommu_priv;
-	int ret;
 
 	if (!data)
 		return -ENODEV;
 
+	/* Update the pgtable base address register of the M4U HW */
 	if (!data->m4u_dom) {
 		data->m4u_dom = dom;
-		ret = mtk_iommu_domain_finalise(data);
-		if (ret) {
-			data->m4u_dom = NULL;
-			return ret;
-		}
-	} else if (data->m4u_dom != dom) {
-		/* All the client devices should be in the same m4u domain */
-		dev_err(dev, "try to attach into the error iommu domain\n");
-		return -EPERM;
+		writel(dom->cfg.arm_v7s_cfg.ttbr[0],
+		       data->base + REG_MMU_PT_BASE_ADDR);
 	}
 
 	mtk_iommu_config(data, dev, true);
@@ -354,6 +395,7 @@ static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain,
 					  dma_addr_t iova)
 {
 	struct mtk_iommu_domain *dom = to_mtk_domain(domain);
+	struct mtk_iommu_data *data = mtk_iommu_get_m4u_data();
 	unsigned long flags;
 	phys_addr_t pa;
 
@@ -361,6 +403,9 @@ static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain,
 	pa = dom->iop->iova_to_phys(dom->iop, iova);
 	spin_unlock_irqrestore(&dom->pgtlock, flags);
 
+	if (data->enable_4GB)
+		pa |= BIT_ULL(32);
+
 	return pa;
 }
 
@@ -399,7 +444,7 @@ static void mtk_iommu_remove_device(struct device *dev)
 
 static struct iommu_group *mtk_iommu_device_group(struct device *dev)
 {
-	struct mtk_iommu_data *data = dev->iommu_fwspec->iommu_priv;
+	struct mtk_iommu_data *data = mtk_iommu_get_m4u_data();
 
 	if (!data)
 		return ERR_PTR(-ENODEV);
@@ -464,8 +509,9 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data)
 		return ret;
 	}
 
-	regval = F_MMU_PREFETCH_RT_REPLACE_MOD |
-		F_MMU_TF_PROTECT_SEL(2);
+	regval = F_MMU_TF_PROTECT_SEL(2, data);
+	if (data->m4u_plat == M4U_MT8173)
+		regval |= F_MMU_PREFETCH_RT_REPLACE_MOD;
 	writel_relaxed(regval, data->base + REG_MMU_CTRL_REG);
 
 	regval = F_L2_MULIT_HIT_EN |
@@ -487,9 +533,19 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data)
 
 	writel_relaxed(F_MMU_IVRP_PA_SET(data->protect_base, data->enable_4GB),
 		       data->base + REG_MMU_IVRP_PADDR);
-
+	if (data->enable_4GB && data->m4u_plat != M4U_MT8173) {
+		/*
+		 * If 4GB mode is enabled, the validate PA range is from
+		 * 0x1_0000_0000 to 0x1_ffff_ffff. here record bit[32:30].
+		 */
+		regval = F_MMU_VLD_PA_RNG(7, 4);
+		writel_relaxed(regval, data->base + REG_MMU_VLD_PA_RNG);
+	}
 	writel_relaxed(0, data->base + REG_MMU_DCM_DIS);
-	writel_relaxed(0, data->base + REG_MMU_STANDARD_AXI_MODE);
+
+	/* It's MISC control register whose default value is ok except mt8173.*/
+	if (data->m4u_plat == M4U_MT8173)
+		writel_relaxed(0, data->base + REG_MMU_STANDARD_AXI_MODE);
 
 	if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0,
 			     dev_name(data->dev), (void *)data)) {
@@ -521,6 +577,7 @@ static int mtk_iommu_probe(struct platform_device *pdev)
 	if (!data)
 		return -ENOMEM;
 	data->dev = dev;
+	data->m4u_plat = (enum mtk_iommu_plat)of_device_get_match_data(dev);
 
 	/* Protect memory. HW will access here while translation fault.*/
 	protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, GFP_KERNEL);
@@ -529,7 +586,7 @@ static int mtk_iommu_probe(struct platform_device *pdev)
 	data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN);
 
 	/* Whether the current dram is over 4GB */
-	data->enable_4GB = !!(max_pfn > (0xffffffffUL >> PAGE_SHIFT));
+	data->enable_4GB = !!(max_pfn > (BIT_ULL(32) >> PAGE_SHIFT));
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	data->base = devm_ioremap_resource(dev, res);
@@ -554,6 +611,7 @@ static int mtk_iommu_probe(struct platform_device *pdev)
 	for (i = 0; i < larb_nr; i++) {
 		struct device_node *larbnode;
 		struct platform_device *plarbdev;
+		u32 id;
 
 		larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i);
 		if (!larbnode)
@@ -562,17 +620,14 @@ static int mtk_iommu_probe(struct platform_device *pdev)
 		if (!of_device_is_available(larbnode))
 			continue;
 
+		ret = of_property_read_u32(larbnode, "mediatek,larb-id", &id);
+		if (ret)/* The id is consecutive if there is no this property */
+			id = i;
+
 		plarbdev = of_find_device_by_node(larbnode);
-		if (!plarbdev) {
-			plarbdev = of_platform_device_create(
-						larbnode, NULL,
-						platform_bus_type.dev_root);
-			if (!plarbdev) {
-				of_node_put(larbnode);
-				return -EPROBE_DEFER;
-			}
-		}
-		data->smi_imu.larb_imu[i].dev = &plarbdev->dev;
+		if (!plarbdev)
+			return -EPROBE_DEFER;
+		data->smi_imu.larb_imu[id].dev = &plarbdev->dev;
 
 		component_match_add_release(dev, &match, release_of,
 					    compare_of, larbnode);
@@ -596,6 +651,8 @@ static int mtk_iommu_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
+	list_add_tail(&data->list, &m4ulist);
+
 	if (!iommu_present(&platform_bus_type))
 		bus_set_iommu(&platform_bus_type, &mtk_iommu_ops);
 
@@ -612,7 +669,6 @@ static int mtk_iommu_remove(struct platform_device *pdev)
 	if (iommu_present(&platform_bus_type))
 		bus_set_iommu(&platform_bus_type, NULL);
 
-	free_io_pgtable_ops(data->m4u_dom->iop);
 	clk_disable_unprepare(data->bclk);
 	devm_free_irq(&pdev->dev, data->irq, data);
 	component_master_del(&pdev->dev, &mtk_iommu_com_ops);
@@ -631,6 +687,7 @@ static int __maybe_unused mtk_iommu_suspend(struct device *dev)
 	reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG);
 	reg->int_control0 = readl_relaxed(base + REG_MMU_INT_CONTROL0);
 	reg->int_main_control = readl_relaxed(base + REG_MMU_INT_MAIN_CONTROL);
+	clk_disable_unprepare(data->bclk);
 	return 0;
 }
 
@@ -639,9 +696,13 @@ static int __maybe_unused mtk_iommu_resume(struct device *dev)
 	struct mtk_iommu_data *data = dev_get_drvdata(dev);
 	struct mtk_iommu_suspend_reg *reg = &data->reg;
 	void __iomem *base = data->base;
+	int ret;
 
-	writel_relaxed(data->m4u_dom->cfg.arm_v7s_cfg.ttbr[0],
-		       base + REG_MMU_PT_BASE_ADDR);
+	ret = clk_prepare_enable(data->bclk);
+	if (ret) {
+		dev_err(data->dev, "Failed to enable clk(%d) in resume\n", ret);
+		return ret;
+	}
 	writel_relaxed(reg->standard_axi_mode,
 		       base + REG_MMU_STANDARD_AXI_MODE);
 	writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM_DIS);
@@ -650,15 +711,19 @@ static int __maybe_unused mtk_iommu_resume(struct device *dev)
 	writel_relaxed(reg->int_main_control, base + REG_MMU_INT_MAIN_CONTROL);
 	writel_relaxed(F_MMU_IVRP_PA_SET(data->protect_base, data->enable_4GB),
 		       base + REG_MMU_IVRP_PADDR);
+	if (data->m4u_dom)
+		writel(data->m4u_dom->cfg.arm_v7s_cfg.ttbr[0],
+		       base + REG_MMU_PT_BASE_ADDR);
 	return 0;
 }
 
-const struct dev_pm_ops mtk_iommu_pm_ops = {
-	SET_SYSTEM_SLEEP_PM_OPS(mtk_iommu_suspend, mtk_iommu_resume)
+static const struct dev_pm_ops mtk_iommu_pm_ops = {
+	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(mtk_iommu_suspend, mtk_iommu_resume)
 };
 
 static const struct of_device_id mtk_iommu_of_ids[] = {
-	{ .compatible = "mediatek,mt8173-m4u", },
+	{ .compatible = "mediatek,mt2712-m4u", .data = (void *)M4U_MT2712},
+	{ .compatible = "mediatek,mt8173-m4u", .data = (void *)M4U_MT8173},
 	{}
 };
 
@@ -667,27 +732,20 @@ static struct platform_driver mtk_iommu_driver = {
 	.remove	= mtk_iommu_remove,
 	.driver	= {
 		.name = "mtk-iommu",
-		.of_match_table = mtk_iommu_of_ids,
+		.of_match_table = of_match_ptr(mtk_iommu_of_ids),
 		.pm = &mtk_iommu_pm_ops,
 	}
 };
 
-static int mtk_iommu_init_fn(struct device_node *np)
+static int __init mtk_iommu_init(void)
 {
 	int ret;
-	struct platform_device *pdev;
-
-	pdev = of_platform_device_create(np, NULL, platform_bus_type.dev_root);
-	if (!pdev)
-		return -ENOMEM;
 
 	ret = platform_driver_register(&mtk_iommu_driver);
-	if (ret) {
-		pr_err("%s: Failed to register driver\n", __func__);
-		return ret;
-	}
+	if (ret != 0)
+		pr_err("Failed to register MTK IOMMU driver\n");
 
-	return 0;
+	return ret;
 }
 
-IOMMU_OF_DECLARE(mtkm4u, "mediatek,mt8173-m4u", mtk_iommu_init_fn);
+subsys_initcall(mtk_iommu_init)
diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h
index c06cc91b5d9a..b4451a1c7c2f 100644
--- a/drivers/iommu/mtk_iommu.h
+++ b/drivers/iommu/mtk_iommu.h
@@ -34,6 +34,12 @@ struct mtk_iommu_suspend_reg {
 	u32				int_main_control;
 };
 
+enum mtk_iommu_plat {
+	M4U_MT2701,
+	M4U_MT2712,
+	M4U_MT8173,
+};
+
 struct mtk_iommu_domain;
 
 struct mtk_iommu_data {
@@ -50,6 +56,9 @@ struct mtk_iommu_data {
 	bool				tlb_flush_active;
 
 	struct iommu_device		iommu;
+	enum mtk_iommu_plat		m4u_plat;
+
+	struct list_head		list;
 };
 
 static inline int compare_of(struct device *dev, void *data)
diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c
index 8cb60829a7a1..e60e3dba85a0 100644
--- a/drivers/iommu/of_iommu.c
+++ b/drivers/iommu/of_iommu.c
@@ -25,6 +25,8 @@
 #include <linux/of_pci.h>
 #include <linux/slab.h>
 
+#define NO_IOMMU	1
+
 static const struct of_device_id __iommu_of_table_sentinel
 	__used __section(__iommu_of_table_end);
 
@@ -109,8 +111,8 @@ static bool of_iommu_driver_present(struct device_node *np)
 	return of_match_node(&__iommu_of_table, np);
 }
 
-static const struct iommu_ops
-*of_iommu_xlate(struct device *dev, struct of_phandle_args *iommu_spec)
+static int of_iommu_xlate(struct device *dev,
+			  struct of_phandle_args *iommu_spec)
 {
 	const struct iommu_ops *ops;
 	struct fwnode_handle *fwnode = &iommu_spec->np->fwnode;
@@ -120,95 +122,53 @@ static const struct iommu_ops
 	if ((ops && !ops->of_xlate) ||
 	    !of_device_is_available(iommu_spec->np) ||
 	    (!ops && !of_iommu_driver_present(iommu_spec->np)))
-		return NULL;
+		return NO_IOMMU;
 
 	err = iommu_fwspec_init(dev, &iommu_spec->np->fwnode, ops);
 	if (err)
-		return ERR_PTR(err);
+		return err;
 	/*
 	 * The otherwise-empty fwspec handily serves to indicate the specific
 	 * IOMMU device we're waiting for, which will be useful if we ever get
 	 * a proper probe-ordering dependency mechanism in future.
 	 */
 	if (!ops)
-		return ERR_PTR(-EPROBE_DEFER);
-
-	err = ops->of_xlate(dev, iommu_spec);
-	if (err)
-		return ERR_PTR(err);
+		return -EPROBE_DEFER;
 
-	return ops;
+	return ops->of_xlate(dev, iommu_spec);
 }
 
-static int __get_pci_rid(struct pci_dev *pdev, u16 alias, void *data)
-{
-	struct of_phandle_args *iommu_spec = data;
-
-	iommu_spec->args[0] = alias;
-	return iommu_spec->np == pdev->bus->dev.of_node;
-}
+struct of_pci_iommu_alias_info {
+	struct device *dev;
+	struct device_node *np;
+};
 
-static const struct iommu_ops
-*of_pci_iommu_init(struct pci_dev *pdev, struct device_node *bridge_np)
+static int of_pci_iommu_init(struct pci_dev *pdev, u16 alias, void *data)
 {
-	const struct iommu_ops *ops;
-	struct of_phandle_args iommu_spec;
+	struct of_pci_iommu_alias_info *info = data;
+	struct of_phandle_args iommu_spec = { .args_count = 1 };
 	int err;
 
-	/*
-	 * Start by tracing the RID alias down the PCI topology as
-	 * far as the host bridge whose OF node we have...
-	 * (we're not even attempting to handle multi-alias devices yet)
-	 */
-	iommu_spec.args_count = 1;
-	iommu_spec.np = bridge_np;
-	pci_for_each_dma_alias(pdev, __get_pci_rid, &iommu_spec);
-	/*
-	 * ...then find out what that becomes once it escapes the PCI
-	 * bus into the system beyond, and which IOMMU it ends up at.
-	 */
-	iommu_spec.np = NULL;
-	err = of_pci_map_rid(bridge_np, iommu_spec.args[0], "iommu-map",
+	err = of_pci_map_rid(info->np, alias, "iommu-map",
 			     "iommu-map-mask", &iommu_spec.np,
 			     iommu_spec.args);
 	if (err)
-		return err == -ENODEV ? NULL : ERR_PTR(err);
-
-	ops = of_iommu_xlate(&pdev->dev, &iommu_spec);
+		return err == -ENODEV ? NO_IOMMU : err;
 
+	err = of_iommu_xlate(info->dev, &iommu_spec);
 	of_node_put(iommu_spec.np);
-	return ops;
-}
-
-static const struct iommu_ops
-*of_platform_iommu_init(struct device *dev, struct device_node *np)
-{
-	struct of_phandle_args iommu_spec;
-	const struct iommu_ops *ops = NULL;
-	int idx = 0;
-
-	/*
-	 * We don't currently walk up the tree looking for a parent IOMMU.
-	 * See the `Notes:' section of
-	 * Documentation/devicetree/bindings/iommu/iommu.txt
-	 */
-	while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells",
-					   idx, &iommu_spec)) {
-		ops = of_iommu_xlate(dev, &iommu_spec);
-		of_node_put(iommu_spec.np);
-		idx++;
-		if (IS_ERR_OR_NULL(ops))
-			break;
-	}
+	if (err)
+		return err;
 
-	return ops;
+	return info->np == pdev->bus->dev.of_node;
 }
 
 const struct iommu_ops *of_iommu_configure(struct device *dev,
 					   struct device_node *master_np)
 {
-	const struct iommu_ops *ops;
+	const struct iommu_ops *ops = NULL;
 	struct iommu_fwspec *fwspec = dev->iommu_fwspec;
+	int err = NO_IOMMU;
 
 	if (!master_np)
 		return NULL;
@@ -221,25 +181,54 @@ const struct iommu_ops *of_iommu_configure(struct device *dev,
 		iommu_fwspec_free(dev);
 	}
 
-	if (dev_is_pci(dev))
-		ops = of_pci_iommu_init(to_pci_dev(dev), master_np);
-	else
-		ops = of_platform_iommu_init(dev, master_np);
+	/*
+	 * We don't currently walk up the tree looking for a parent IOMMU.
+	 * See the `Notes:' section of
+	 * Documentation/devicetree/bindings/iommu/iommu.txt
+	 */
+	if (dev_is_pci(dev)) {
+		struct of_pci_iommu_alias_info info = {
+			.dev = dev,
+			.np = master_np,
+		};
+
+		err = pci_for_each_dma_alias(to_pci_dev(dev),
+					     of_pci_iommu_init, &info);
+	} else {
+		struct of_phandle_args iommu_spec;
+		int idx = 0;
+
+		while (!of_parse_phandle_with_args(master_np, "iommus",
+						   "#iommu-cells",
+						   idx, &iommu_spec)) {
+			err = of_iommu_xlate(dev, &iommu_spec);
+			of_node_put(iommu_spec.np);
+			idx++;
+			if (err)
+				break;
+		}
+	}
+
+	/*
+	 * Two success conditions can be represented by non-negative err here:
+	 * >0 : there is no IOMMU, or one was unavailable for non-fatal reasons
+	 *  0 : we found an IOMMU, and dev->fwspec is initialised appropriately
+	 * <0 : any actual error
+	 */
+	if (!err)
+		ops = dev->iommu_fwspec->ops;
 	/*
 	 * If we have reason to believe the IOMMU driver missed the initial
 	 * add_device callback for dev, replay it to get things in order.
 	 */
-	if (!IS_ERR_OR_NULL(ops) && ops->add_device &&
-	    dev->bus && !dev->iommu_group) {
-		int err = ops->add_device(dev);
-
-		if (err)
-			ops = ERR_PTR(err);
-	}
+	if (ops && ops->add_device && dev->bus && !dev->iommu_group)
+		err = ops->add_device(dev);
 
 	/* Ignore all other errors apart from EPROBE_DEFER */
-	if (IS_ERR(ops) && (PTR_ERR(ops) != -EPROBE_DEFER)) {
-		dev_dbg(dev, "Adding to IOMMU failed: %ld\n", PTR_ERR(ops));
+	if (err == -EPROBE_DEFER) {
+		ops = ERR_PTR(err);
+	} else if (err < 0) {
+		dev_dbg(dev, "Adding to IOMMU failed: %d\n", err);
 		ops = NULL;
 	}
 
@@ -255,8 +244,7 @@ static int __init of_iommu_init(void)
 		const of_iommu_init_fn init_fn = match->data;
 
 		if (init_fn && init_fn(np))
-			pr_err("Failed to initialise IOMMU %s\n",
-				of_node_full_name(np));
+			pr_err("Failed to initialise IOMMU %pOF\n", np);
 	}
 
 	return 0;
diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c
index 641e035cf866..bd67e1b2c64e 100644
--- a/drivers/iommu/omap-iommu.c
+++ b/drivers/iommu/omap-iommu.c
@@ -11,6 +11,7 @@
  * published by the Free Software Foundation.
  */
 
+#include <linux/dma-mapping.h>
 #include <linux/err.h>
 #include <linux/slab.h>
 #include <linux/interrupt.h>
@@ -29,8 +30,6 @@
 #include <linux/regmap.h>
 #include <linux/mfd/syscon.h>
 
-#include <asm/cacheflush.h>
-
 #include <linux/platform_data/iommu-omap.h>
 
 #include "omap-iopgtable.h"
@@ -454,36 +453,35 @@ static void flush_iotlb_all(struct omap_iommu *obj)
 /*
  *	H/W pagetable operations
  */
-static void flush_iopgd_range(u32 *first, u32 *last)
+static void flush_iopte_range(struct device *dev, dma_addr_t dma,
+			      unsigned long offset, int num_entries)
 {
-	/* FIXME: L2 cache should be taken care of if it exists */
-	do {
-		asm("mcr	p15, 0, %0, c7, c10, 1 @ flush_pgd"
-		    : : "r" (first));
-		first += L1_CACHE_BYTES / sizeof(*first);
-	} while (first <= last);
-}
+	size_t size = num_entries * sizeof(u32);
 
-static void flush_iopte_range(u32 *first, u32 *last)
-{
-	/* FIXME: L2 cache should be taken care of if it exists */
-	do {
-		asm("mcr	p15, 0, %0, c7, c10, 1 @ flush_pte"
-		    : : "r" (first));
-		first += L1_CACHE_BYTES / sizeof(*first);
-	} while (first <= last);
+	dma_sync_single_range_for_device(dev, dma, offset, size, DMA_TO_DEVICE);
 }
 
-static void iopte_free(u32 *iopte)
+static void iopte_free(struct omap_iommu *obj, u32 *iopte, bool dma_valid)
 {
+	dma_addr_t pt_dma;
+
 	/* Note: freed iopte's must be clean ready for re-use */
-	if (iopte)
+	if (iopte) {
+		if (dma_valid) {
+			pt_dma = virt_to_phys(iopte);
+			dma_unmap_single(obj->dev, pt_dma, IOPTE_TABLE_SIZE,
+					 DMA_TO_DEVICE);
+		}
+
 		kmem_cache_free(iopte_cachep, iopte);
+	}
 }
 
-static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd, u32 da)
+static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd,
+			dma_addr_t *pt_dma, u32 da)
 {
 	u32 *iopte;
+	unsigned long offset = iopgd_index(da) * sizeof(da);
 
 	/* a table has already existed */
 	if (*iopgd)
@@ -500,18 +498,38 @@ static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd, u32 da)
 		if (!iopte)
 			return ERR_PTR(-ENOMEM);
 
+		*pt_dma = dma_map_single(obj->dev, iopte, IOPTE_TABLE_SIZE,
+					 DMA_TO_DEVICE);
+		if (dma_mapping_error(obj->dev, *pt_dma)) {
+			dev_err(obj->dev, "DMA map error for L2 table\n");
+			iopte_free(obj, iopte, false);
+			return ERR_PTR(-ENOMEM);
+		}
+
+		/*
+		 * we rely on dma address and the physical address to be
+		 * the same for mapping the L2 table
+		 */
+		if (WARN_ON(*pt_dma != virt_to_phys(iopte))) {
+			dev_err(obj->dev, "DMA translation error for L2 table\n");
+			dma_unmap_single(obj->dev, *pt_dma, IOPTE_TABLE_SIZE,
+					 DMA_TO_DEVICE);
+			iopte_free(obj, iopte, false);
+			return ERR_PTR(-ENOMEM);
+		}
+
 		*iopgd = virt_to_phys(iopte) | IOPGD_TABLE;
-		flush_iopgd_range(iopgd, iopgd);
 
+		flush_iopte_range(obj->dev, obj->pd_dma, offset, 1);
 		dev_vdbg(obj->dev, "%s: a new pte:%p\n", __func__, iopte);
 	} else {
 		/* We raced, free the reduniovant table */
-		iopte_free(iopte);
+		iopte_free(obj, iopte, false);
 	}
 
 pte_ready:
 	iopte = iopte_offset(iopgd, da);
-
+	*pt_dma = virt_to_phys(iopte);
 	dev_vdbg(obj->dev,
 		 "%s: da:%08x pgd:%p *pgd:%08x pte:%p *pte:%08x\n",
 		 __func__, da, iopgd, *iopgd, iopte, *iopte);
@@ -522,6 +540,7 @@ pte_ready:
 static int iopgd_alloc_section(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
 {
 	u32 *iopgd = iopgd_offset(obj, da);
+	unsigned long offset = iopgd_index(da) * sizeof(da);
 
 	if ((da | pa) & ~IOSECTION_MASK) {
 		dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
@@ -530,13 +549,14 @@ static int iopgd_alloc_section(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
 	}
 
 	*iopgd = (pa & IOSECTION_MASK) | prot | IOPGD_SECTION;
-	flush_iopgd_range(iopgd, iopgd);
+	flush_iopte_range(obj->dev, obj->pd_dma, offset, 1);
 	return 0;
 }
 
 static int iopgd_alloc_super(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
 {
 	u32 *iopgd = iopgd_offset(obj, da);
+	unsigned long offset = iopgd_index(da) * sizeof(da);
 	int i;
 
 	if ((da | pa) & ~IOSUPER_MASK) {
@@ -547,20 +567,22 @@ static int iopgd_alloc_super(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
 
 	for (i = 0; i < 16; i++)
 		*(iopgd + i) = (pa & IOSUPER_MASK) | prot | IOPGD_SUPER;
-	flush_iopgd_range(iopgd, iopgd + 15);
+	flush_iopte_range(obj->dev, obj->pd_dma, offset, 16);
 	return 0;
 }
 
 static int iopte_alloc_page(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
 {
 	u32 *iopgd = iopgd_offset(obj, da);
-	u32 *iopte = iopte_alloc(obj, iopgd, da);
+	dma_addr_t pt_dma;
+	u32 *iopte = iopte_alloc(obj, iopgd, &pt_dma, da);
+	unsigned long offset = iopte_index(da) * sizeof(da);
 
 	if (IS_ERR(iopte))
 		return PTR_ERR(iopte);
 
 	*iopte = (pa & IOPAGE_MASK) | prot | IOPTE_SMALL;
-	flush_iopte_range(iopte, iopte);
+	flush_iopte_range(obj->dev, pt_dma, offset, 1);
 
 	dev_vdbg(obj->dev, "%s: da:%08x pa:%08x pte:%p *pte:%08x\n",
 		 __func__, da, pa, iopte, *iopte);
@@ -571,7 +593,9 @@ static int iopte_alloc_page(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
 static int iopte_alloc_large(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
 {
 	u32 *iopgd = iopgd_offset(obj, da);
-	u32 *iopte = iopte_alloc(obj, iopgd, da);
+	dma_addr_t pt_dma;
+	u32 *iopte = iopte_alloc(obj, iopgd, &pt_dma, da);
+	unsigned long offset = iopte_index(da) * sizeof(da);
 	int i;
 
 	if ((da | pa) & ~IOLARGE_MASK) {
@@ -585,7 +609,7 @@ static int iopte_alloc_large(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
 
 	for (i = 0; i < 16; i++)
 		*(iopte + i) = (pa & IOLARGE_MASK) | prot | IOPTE_LARGE;
-	flush_iopte_range(iopte, iopte + 15);
+	flush_iopte_range(obj->dev, pt_dma, offset, 16);
 	return 0;
 }
 
@@ -674,6 +698,9 @@ static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da)
 	size_t bytes;
 	u32 *iopgd = iopgd_offset(obj, da);
 	int nent = 1;
+	dma_addr_t pt_dma;
+	unsigned long pd_offset = iopgd_index(da) * sizeof(da);
+	unsigned long pt_offset = iopte_index(da) * sizeof(da);
 
 	if (!*iopgd)
 		return 0;
@@ -690,7 +717,8 @@ static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da)
 		}
 		bytes *= nent;
 		memset(iopte, 0, nent * sizeof(*iopte));
-		flush_iopte_range(iopte, iopte + (nent - 1) * sizeof(*iopte));
+		pt_dma = virt_to_phys(iopte);
+		flush_iopte_range(obj->dev, pt_dma, pt_offset, nent);
 
 		/*
 		 * do table walk to check if this table is necessary or not
@@ -700,7 +728,7 @@ static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da)
 			if (iopte[i])
 				goto out;
 
-		iopte_free(iopte);
+		iopte_free(obj, iopte, true);
 		nent = 1; /* for the next L1 entry */
 	} else {
 		bytes = IOPGD_SIZE;
@@ -712,7 +740,7 @@ static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da)
 		bytes *= nent;
 	}
 	memset(iopgd, 0, nent * sizeof(*iopgd));
-	flush_iopgd_range(iopgd, iopgd + (nent - 1) * sizeof(*iopgd));
+	flush_iopte_range(obj->dev, obj->pd_dma, pd_offset, nent);
 out:
 	return bytes;
 }
@@ -738,6 +766,7 @@ static size_t iopgtable_clear_entry(struct omap_iommu *obj, u32 da)
 
 static void iopgtable_clear_entry_all(struct omap_iommu *obj)
 {
+	unsigned long offset;
 	int i;
 
 	spin_lock(&obj->page_table_lock);
@@ -748,15 +777,16 @@ static void iopgtable_clear_entry_all(struct omap_iommu *obj)
 
 		da = i << IOPGD_SHIFT;
 		iopgd = iopgd_offset(obj, da);
+		offset = iopgd_index(da) * sizeof(da);
 
 		if (!*iopgd)
 			continue;
 
 		if (iopgd_is_table(*iopgd))
-			iopte_free(iopte_offset(iopgd, 0));
+			iopte_free(obj, iopte_offset(iopgd, 0), true);
 
 		*iopgd = 0;
-		flush_iopgd_range(iopgd, iopgd);
+		flush_iopte_range(obj->dev, obj->pd_dma, offset, 1);
 	}
 
 	flush_iotlb_all(obj);
@@ -786,7 +816,7 @@ static irqreturn_t iommu_fault_handler(int irq, void *data)
 	if (!report_iommu_fault(domain, obj->dev, da, 0))
 		return IRQ_HANDLED;
 
-	iommu_disable(obj);
+	iommu_write_reg(obj, 0, MMU_IRQENABLE);
 
 	iopgd = iopgd_offset(obj, da);
 
@@ -815,10 +845,18 @@ static int omap_iommu_attach(struct omap_iommu *obj, u32 *iopgd)
 
 	spin_lock(&obj->iommu_lock);
 
+	obj->pd_dma = dma_map_single(obj->dev, iopgd, IOPGD_TABLE_SIZE,
+				     DMA_TO_DEVICE);
+	if (dma_mapping_error(obj->dev, obj->pd_dma)) {
+		dev_err(obj->dev, "DMA map error for L1 table\n");
+		err = -ENOMEM;
+		goto out_err;
+	}
+
 	obj->iopgd = iopgd;
 	err = iommu_enable(obj);
 	if (err)
-		goto err_enable;
+		goto out_err;
 	flush_iotlb_all(obj);
 
 	spin_unlock(&obj->iommu_lock);
@@ -827,7 +865,7 @@ static int omap_iommu_attach(struct omap_iommu *obj, u32 *iopgd)
 
 	return 0;
 
-err_enable:
+out_err:
 	spin_unlock(&obj->iommu_lock);
 
 	return err;
@@ -844,7 +882,10 @@ static void omap_iommu_detach(struct omap_iommu *obj)
 
 	spin_lock(&obj->iommu_lock);
 
+	dma_unmap_single(obj->dev, obj->pd_dma, IOPGD_TABLE_SIZE,
+			 DMA_TO_DEVICE);
 	iommu_disable(obj);
+	obj->pd_dma = 0;
 	obj->iopgd = NULL;
 
 	spin_unlock(&obj->iommu_lock);
@@ -1008,11 +1049,6 @@ static struct platform_driver omap_iommu_driver = {
 	},
 };
 
-static void iopte_cachep_ctor(void *iopte)
-{
-	clean_dcache_area(iopte, IOPTE_TABLE_SIZE);
-}
-
 static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, int pgsz)
 {
 	memset(e, 0, sizeof(*e));
@@ -1159,7 +1195,6 @@ static struct iommu_domain *omap_iommu_domain_alloc(unsigned type)
 	if (WARN_ON(!IS_ALIGNED((long)omap_domain->pgtable, IOPGD_TABLE_SIZE)))
 		goto fail_align;
 
-	clean_dcache_area(omap_domain->pgtable, IOPGD_TABLE_SIZE);
 	spin_lock_init(&omap_domain->lock);
 
 	omap_domain->domain.geometry.aperture_start = 0;
@@ -1347,7 +1382,7 @@ static int __init omap_iommu_init(void)
 	of_node_put(np);
 
 	p = kmem_cache_create("iopte_cache", IOPTE_TABLE_SIZE, align, flags,
-			      iopte_cachep_ctor);
+			      NULL);
 	if (!p)
 		return -ENOMEM;
 	iopte_cachep = p;
diff --git a/drivers/iommu/omap-iommu.h b/drivers/iommu/omap-iommu.h
index 6e70515e6038..a675af29a6ec 100644
--- a/drivers/iommu/omap-iommu.h
+++ b/drivers/iommu/omap-iommu.h
@@ -61,6 +61,7 @@ struct omap_iommu {
 	 */
 	u32		*iopgd;
 	spinlock_t	page_table_lock; /* protect iopgd */
+	dma_addr_t	pd_dma;
 
 	int		nr_tlb_entries;
 
diff --git a/drivers/iommu/qcom_iommu.c b/drivers/iommu/qcom_iommu.c
new file mode 100644
index 000000000000..c8a587d034b0
--- /dev/null
+++ b/drivers/iommu/qcom_iommu.c
@@ -0,0 +1,930 @@
+/*
+ * IOMMU API for QCOM secure IOMMUs.  Somewhat based on arm-smmu.c
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Copyright (C) 2013 ARM Limited
+ * Copyright (C) 2017 Red Hat
+ */
+
+#include <linux/atomic.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/dma-iommu.h>
+#include <linux/dma-mapping.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/io-64-nonatomic-hi-lo.h>
+#include <linux/iommu.h>
+#include <linux/iopoll.h>
+#include <linux/kconfig.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/of_iommu.h>
+#include <linux/platform_device.h>
+#include <linux/pm.h>
+#include <linux/pm_runtime.h>
+#include <linux/qcom_scm.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+
+#include "io-pgtable.h"
+#include "arm-smmu-regs.h"
+
+#define SMMU_INTR_SEL_NS     0x2000
+
+struct qcom_iommu_ctx;
+
+struct qcom_iommu_dev {
+	/* IOMMU core code handle */
+	struct iommu_device	 iommu;
+	struct device		*dev;
+	struct clk		*iface_clk;
+	struct clk		*bus_clk;
+	void __iomem		*local_base;
+	u32			 sec_id;
+	u8			 num_ctxs;
+	struct qcom_iommu_ctx	*ctxs[0];   /* indexed by asid-1 */
+};
+
+struct qcom_iommu_ctx {
+	struct device		*dev;
+	void __iomem		*base;
+	bool			 secure_init;
+	u8			 asid;      /* asid and ctx bank # are 1:1 */
+};
+
+struct qcom_iommu_domain {
+	struct io_pgtable_ops	*pgtbl_ops;
+	spinlock_t		 pgtbl_lock;
+	struct mutex		 init_mutex; /* Protects iommu pointer */
+	struct iommu_domain	 domain;
+	struct qcom_iommu_dev	*iommu;
+};
+
+static struct qcom_iommu_domain *to_qcom_iommu_domain(struct iommu_domain *dom)
+{
+	return container_of(dom, struct qcom_iommu_domain, domain);
+}
+
+static const struct iommu_ops qcom_iommu_ops;
+
+static struct qcom_iommu_dev * to_iommu(struct iommu_fwspec *fwspec)
+{
+	if (!fwspec || fwspec->ops != &qcom_iommu_ops)
+		return NULL;
+	return fwspec->iommu_priv;
+}
+
+static struct qcom_iommu_ctx * to_ctx(struct iommu_fwspec *fwspec, unsigned asid)
+{
+	struct qcom_iommu_dev *qcom_iommu = to_iommu(fwspec);
+	if (!qcom_iommu)
+		return NULL;
+	return qcom_iommu->ctxs[asid - 1];
+}
+
+static inline void
+iommu_writel(struct qcom_iommu_ctx *ctx, unsigned reg, u32 val)
+{
+	writel_relaxed(val, ctx->base + reg);
+}
+
+static inline void
+iommu_writeq(struct qcom_iommu_ctx *ctx, unsigned reg, u64 val)
+{
+	writeq_relaxed(val, ctx->base + reg);
+}
+
+static inline u32
+iommu_readl(struct qcom_iommu_ctx *ctx, unsigned reg)
+{
+	return readl_relaxed(ctx->base + reg);
+}
+
+static inline u64
+iommu_readq(struct qcom_iommu_ctx *ctx, unsigned reg)
+{
+	return readq_relaxed(ctx->base + reg);
+}
+
+static void qcom_iommu_tlb_sync(void *cookie)
+{
+	struct iommu_fwspec *fwspec = cookie;
+	unsigned i;
+
+	for (i = 0; i < fwspec->num_ids; i++) {
+		struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+		unsigned int val, ret;
+
+		iommu_writel(ctx, ARM_SMMU_CB_TLBSYNC, 0);
+
+		ret = readl_poll_timeout(ctx->base + ARM_SMMU_CB_TLBSTATUS, val,
+					 (val & 0x1) == 0, 0, 5000000);
+		if (ret)
+			dev_err(ctx->dev, "timeout waiting for TLB SYNC\n");
+	}
+}
+
+static void qcom_iommu_tlb_inv_context(void *cookie)
+{
+	struct iommu_fwspec *fwspec = cookie;
+	unsigned i;
+
+	for (i = 0; i < fwspec->num_ids; i++) {
+		struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+		iommu_writel(ctx, ARM_SMMU_CB_S1_TLBIASID, ctx->asid);
+	}
+
+	qcom_iommu_tlb_sync(cookie);
+}
+
+static void qcom_iommu_tlb_inv_range_nosync(unsigned long iova, size_t size,
+					    size_t granule, bool leaf, void *cookie)
+{
+	struct iommu_fwspec *fwspec = cookie;
+	unsigned i, reg;
+
+	reg = leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
+
+	for (i = 0; i < fwspec->num_ids; i++) {
+		struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+		size_t s = size;
+
+		iova &= ~12UL;
+		iova |= ctx->asid;
+		do {
+			iommu_writel(ctx, reg, iova);
+			iova += granule;
+		} while (s -= granule);
+	}
+}
+
+static const struct iommu_gather_ops qcom_gather_ops = {
+	.tlb_flush_all	= qcom_iommu_tlb_inv_context,
+	.tlb_add_flush	= qcom_iommu_tlb_inv_range_nosync,
+	.tlb_sync	= qcom_iommu_tlb_sync,
+};
+
+static irqreturn_t qcom_iommu_fault(int irq, void *dev)
+{
+	struct qcom_iommu_ctx *ctx = dev;
+	u32 fsr, fsynr;
+	u64 iova;
+
+	fsr = iommu_readl(ctx, ARM_SMMU_CB_FSR);
+
+	if (!(fsr & FSR_FAULT))
+		return IRQ_NONE;
+
+	fsynr = iommu_readl(ctx, ARM_SMMU_CB_FSYNR0);
+	iova = iommu_readq(ctx, ARM_SMMU_CB_FAR);
+
+	dev_err_ratelimited(ctx->dev,
+			    "Unhandled context fault: fsr=0x%x, "
+			    "iova=0x%016llx, fsynr=0x%x, cb=%d\n",
+			    fsr, iova, fsynr, ctx->asid);
+
+	iommu_writel(ctx, ARM_SMMU_CB_FSR, fsr);
+
+	return IRQ_HANDLED;
+}
+
+static int qcom_iommu_init_domain(struct iommu_domain *domain,
+				  struct qcom_iommu_dev *qcom_iommu,
+				  struct iommu_fwspec *fwspec)
+{
+	struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+	struct io_pgtable_ops *pgtbl_ops;
+	struct io_pgtable_cfg pgtbl_cfg;
+	int i, ret = 0;
+	u32 reg;
+
+	mutex_lock(&qcom_domain->init_mutex);
+	if (qcom_domain->iommu)
+		goto out_unlock;
+
+	pgtbl_cfg = (struct io_pgtable_cfg) {
+		.pgsize_bitmap	= qcom_iommu_ops.pgsize_bitmap,
+		.ias		= 32,
+		.oas		= 40,
+		.tlb		= &qcom_gather_ops,
+		.iommu_dev	= qcom_iommu->dev,
+	};
+
+	qcom_domain->iommu = qcom_iommu;
+	pgtbl_ops = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &pgtbl_cfg, fwspec);
+	if (!pgtbl_ops) {
+		dev_err(qcom_iommu->dev, "failed to allocate pagetable ops\n");
+		ret = -ENOMEM;
+		goto out_clear_iommu;
+	}
+
+	/* Update the domain's page sizes to reflect the page table format */
+	domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
+	domain->geometry.aperture_end = (1ULL << pgtbl_cfg.ias) - 1;
+	domain->geometry.force_aperture = true;
+
+	for (i = 0; i < fwspec->num_ids; i++) {
+		struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+
+		if (!ctx->secure_init) {
+			ret = qcom_scm_restore_sec_cfg(qcom_iommu->sec_id, ctx->asid);
+			if (ret) {
+				dev_err(qcom_iommu->dev, "secure init failed: %d\n", ret);
+				goto out_clear_iommu;
+			}
+			ctx->secure_init = true;
+		}
+
+		/* TTBRs */
+		iommu_writeq(ctx, ARM_SMMU_CB_TTBR0,
+				pgtbl_cfg.arm_lpae_s1_cfg.ttbr[0] |
+				((u64)ctx->asid << TTBRn_ASID_SHIFT));
+		iommu_writeq(ctx, ARM_SMMU_CB_TTBR1,
+				pgtbl_cfg.arm_lpae_s1_cfg.ttbr[1] |
+				((u64)ctx->asid << TTBRn_ASID_SHIFT));
+
+		/* TTBCR */
+		iommu_writel(ctx, ARM_SMMU_CB_TTBCR2,
+				(pgtbl_cfg.arm_lpae_s1_cfg.tcr >> 32) |
+				TTBCR2_SEP_UPSTREAM);
+		iommu_writel(ctx, ARM_SMMU_CB_TTBCR,
+				pgtbl_cfg.arm_lpae_s1_cfg.tcr);
+
+		/* MAIRs (stage-1 only) */
+		iommu_writel(ctx, ARM_SMMU_CB_S1_MAIR0,
+				pgtbl_cfg.arm_lpae_s1_cfg.mair[0]);
+		iommu_writel(ctx, ARM_SMMU_CB_S1_MAIR1,
+				pgtbl_cfg.arm_lpae_s1_cfg.mair[1]);
+
+		/* SCTLR */
+		reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE |
+			SCTLR_M | SCTLR_S1_ASIDPNE;
+
+		if (IS_ENABLED(CONFIG_BIG_ENDIAN))
+			reg |= SCTLR_E;
+
+		iommu_writel(ctx, ARM_SMMU_CB_SCTLR, reg);
+	}
+
+	mutex_unlock(&qcom_domain->init_mutex);
+
+	/* Publish page table ops for map/unmap */
+	qcom_domain->pgtbl_ops = pgtbl_ops;
+
+	return 0;
+
+out_clear_iommu:
+	qcom_domain->iommu = NULL;
+out_unlock:
+	mutex_unlock(&qcom_domain->init_mutex);
+	return ret;
+}
+
+static struct iommu_domain *qcom_iommu_domain_alloc(unsigned type)
+{
+	struct qcom_iommu_domain *qcom_domain;
+
+	if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
+		return NULL;
+	/*
+	 * Allocate the domain and initialise some of its data structures.
+	 * We can't really do anything meaningful until we've added a
+	 * master.
+	 */
+	qcom_domain = kzalloc(sizeof(*qcom_domain), GFP_KERNEL);
+	if (!qcom_domain)
+		return NULL;
+
+	if (type == IOMMU_DOMAIN_DMA &&
+	    iommu_get_dma_cookie(&qcom_domain->domain)) {
+		kfree(qcom_domain);
+		return NULL;
+	}
+
+	mutex_init(&qcom_domain->init_mutex);
+	spin_lock_init(&qcom_domain->pgtbl_lock);
+
+	return &qcom_domain->domain;
+}
+
+static void qcom_iommu_domain_free(struct iommu_domain *domain)
+{
+	struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+
+	if (WARN_ON(qcom_domain->iommu))    /* forgot to detach? */
+		return;
+
+	iommu_put_dma_cookie(domain);
+
+	/* NOTE: unmap can be called after client device is powered off,
+	 * for example, with GPUs or anything involving dma-buf.  So we
+	 * cannot rely on the device_link.  Make sure the IOMMU is on to
+	 * avoid unclocked accesses in the TLB inv path:
+	 */
+	pm_runtime_get_sync(qcom_domain->iommu->dev);
+
+	free_io_pgtable_ops(qcom_domain->pgtbl_ops);
+
+	pm_runtime_put_sync(qcom_domain->iommu->dev);
+
+	kfree(qcom_domain);
+}
+
+static int qcom_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
+{
+	struct qcom_iommu_dev *qcom_iommu = to_iommu(dev->iommu_fwspec);
+	struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+	int ret;
+
+	if (!qcom_iommu) {
+		dev_err(dev, "cannot attach to IOMMU, is it on the same bus?\n");
+		return -ENXIO;
+	}
+
+	/* Ensure that the domain is finalized */
+	pm_runtime_get_sync(qcom_iommu->dev);
+	ret = qcom_iommu_init_domain(domain, qcom_iommu, dev->iommu_fwspec);
+	pm_runtime_put_sync(qcom_iommu->dev);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * Sanity check the domain. We don't support domains across
+	 * different IOMMUs.
+	 */
+	if (qcom_domain->iommu != qcom_iommu) {
+		dev_err(dev, "cannot attach to IOMMU %s while already "
+			"attached to domain on IOMMU %s\n",
+			dev_name(qcom_domain->iommu->dev),
+			dev_name(qcom_iommu->dev));
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void qcom_iommu_detach_dev(struct iommu_domain *domain, struct device *dev)
+{
+	struct iommu_fwspec *fwspec = dev->iommu_fwspec;
+	struct qcom_iommu_dev *qcom_iommu = to_iommu(fwspec);
+	struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+	unsigned i;
+
+	if (!qcom_domain->iommu)
+		return;
+
+	pm_runtime_get_sync(qcom_iommu->dev);
+	for (i = 0; i < fwspec->num_ids; i++) {
+		struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
+
+		/* Disable the context bank: */
+		iommu_writel(ctx, ARM_SMMU_CB_SCTLR, 0);
+	}
+	pm_runtime_put_sync(qcom_iommu->dev);
+
+	qcom_domain->iommu = NULL;
+}
+
+static int qcom_iommu_map(struct iommu_domain *domain, unsigned long iova,
+			  phys_addr_t paddr, size_t size, int prot)
+{
+	int ret;
+	unsigned long flags;
+	struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+	struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;
+
+	if (!ops)
+		return -ENODEV;
+
+	spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
+	ret = ops->map(ops, iova, paddr, size, prot);
+	spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
+	return ret;
+}
+
+static size_t qcom_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
+			       size_t size)
+{
+	size_t ret;
+	unsigned long flags;
+	struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+	struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;
+
+	if (!ops)
+		return 0;
+
+	/* NOTE: unmap can be called after client device is powered off,
+	 * for example, with GPUs or anything involving dma-buf.  So we
+	 * cannot rely on the device_link.  Make sure the IOMMU is on to
+	 * avoid unclocked accesses in the TLB inv path:
+	 */
+	pm_runtime_get_sync(qcom_domain->iommu->dev);
+	spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
+	ret = ops->unmap(ops, iova, size);
+	spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
+	pm_runtime_put_sync(qcom_domain->iommu->dev);
+
+	return ret;
+}
+
+static phys_addr_t qcom_iommu_iova_to_phys(struct iommu_domain *domain,
+					   dma_addr_t iova)
+{
+	phys_addr_t ret;
+	unsigned long flags;
+	struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
+	struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;
+
+	if (!ops)
+		return 0;
+
+	spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
+	ret = ops->iova_to_phys(ops, iova);
+	spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
+
+	return ret;
+}
+
+static bool qcom_iommu_capable(enum iommu_cap cap)
+{
+	switch (cap) {
+	case IOMMU_CAP_CACHE_COHERENCY:
+		/*
+		 * Return true here as the SMMU can always send out coherent
+		 * requests.
+		 */
+		return true;
+	case IOMMU_CAP_NOEXEC:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static int qcom_iommu_add_device(struct device *dev)
+{
+	struct qcom_iommu_dev *qcom_iommu = to_iommu(dev->iommu_fwspec);
+	struct iommu_group *group;
+	struct device_link *link;
+
+	if (!qcom_iommu)
+		return -ENODEV;
+
+	/*
+	 * Establish the link between iommu and master, so that the
+	 * iommu gets runtime enabled/disabled as per the master's
+	 * needs.
+	 */
+	link = device_link_add(dev, qcom_iommu->dev, DL_FLAG_PM_RUNTIME);
+	if (!link) {
+		dev_err(qcom_iommu->dev, "Unable to create device link between %s and %s\n",
+			dev_name(qcom_iommu->dev), dev_name(dev));
+		return -ENODEV;
+	}
+
+	group = iommu_group_get_for_dev(dev);
+	if (IS_ERR_OR_NULL(group))
+		return PTR_ERR_OR_ZERO(group);
+
+	iommu_group_put(group);
+	iommu_device_link(&qcom_iommu->iommu, dev);
+
+	return 0;
+}
+
+static void qcom_iommu_remove_device(struct device *dev)
+{
+	struct qcom_iommu_dev *qcom_iommu = to_iommu(dev->iommu_fwspec);
+
+	if (!qcom_iommu)
+		return;
+
+	iommu_device_unlink(&qcom_iommu->iommu, dev);
+	iommu_group_remove_device(dev);
+	iommu_fwspec_free(dev);
+}
+
+static int qcom_iommu_of_xlate(struct device *dev, struct of_phandle_args *args)
+{
+	struct qcom_iommu_dev *qcom_iommu;
+	struct platform_device *iommu_pdev;
+	unsigned asid = args->args[0];
+
+	if (args->args_count != 1) {
+		dev_err(dev, "incorrect number of iommu params found for %s "
+			"(found %d, expected 1)\n",
+			args->np->full_name, args->args_count);
+		return -EINVAL;
+	}
+
+	iommu_pdev = of_find_device_by_node(args->np);
+	if (WARN_ON(!iommu_pdev))
+		return -EINVAL;
+
+	qcom_iommu = platform_get_drvdata(iommu_pdev);
+
+	/* make sure the asid specified in dt is valid, so we don't have
+	 * to sanity check this elsewhere, since 'asid - 1' is used to
+	 * index into qcom_iommu->ctxs:
+	 */
+	if (WARN_ON(asid < 1) ||
+	    WARN_ON(asid > qcom_iommu->num_ctxs))
+		return -EINVAL;
+
+	if (!dev->iommu_fwspec->iommu_priv) {
+		dev->iommu_fwspec->iommu_priv = qcom_iommu;
+	} else {
+		/* make sure devices iommus dt node isn't referring to
+		 * multiple different iommu devices.  Multiple context
+		 * banks are ok, but multiple devices are not:
+		 */
+		if (WARN_ON(qcom_iommu != dev->iommu_fwspec->iommu_priv))
+			return -EINVAL;
+	}
+
+	return iommu_fwspec_add_ids(dev, &asid, 1);
+}
+
+static const struct iommu_ops qcom_iommu_ops = {
+	.capable	= qcom_iommu_capable,
+	.domain_alloc	= qcom_iommu_domain_alloc,
+	.domain_free	= qcom_iommu_domain_free,
+	.attach_dev	= qcom_iommu_attach_dev,
+	.detach_dev	= qcom_iommu_detach_dev,
+	.map		= qcom_iommu_map,
+	.unmap		= qcom_iommu_unmap,
+	.map_sg		= default_iommu_map_sg,
+	.iova_to_phys	= qcom_iommu_iova_to_phys,
+	.add_device	= qcom_iommu_add_device,
+	.remove_device	= qcom_iommu_remove_device,
+	.device_group	= generic_device_group,
+	.of_xlate	= qcom_iommu_of_xlate,
+	.pgsize_bitmap	= SZ_4K | SZ_64K | SZ_1M | SZ_16M,
+};
+
+static int qcom_iommu_enable_clocks(struct qcom_iommu_dev *qcom_iommu)
+{
+	int ret;
+
+	ret = clk_prepare_enable(qcom_iommu->iface_clk);
+	if (ret) {
+		dev_err(qcom_iommu->dev, "Couldn't enable iface_clk\n");
+		return ret;
+	}
+
+	ret = clk_prepare_enable(qcom_iommu->bus_clk);
+	if (ret) {
+		dev_err(qcom_iommu->dev, "Couldn't enable bus_clk\n");
+		clk_disable_unprepare(qcom_iommu->iface_clk);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void qcom_iommu_disable_clocks(struct qcom_iommu_dev *qcom_iommu)
+{
+	clk_disable_unprepare(qcom_iommu->bus_clk);
+	clk_disable_unprepare(qcom_iommu->iface_clk);
+}
+
+static int qcom_iommu_sec_ptbl_init(struct device *dev)
+{
+	size_t psize = 0;
+	unsigned int spare = 0;
+	void *cpu_addr;
+	dma_addr_t paddr;
+	unsigned long attrs;
+	static bool allocated = false;
+	int ret;
+
+	if (allocated)
+		return 0;
+
+	ret = qcom_scm_iommu_secure_ptbl_size(spare, &psize);
+	if (ret) {
+		dev_err(dev, "failed to get iommu secure pgtable size (%d)\n",
+			ret);
+		return ret;
+	}
+
+	dev_info(dev, "iommu sec: pgtable size: %zu\n", psize);
+
+	attrs = DMA_ATTR_NO_KERNEL_MAPPING;
+
+	cpu_addr = dma_alloc_attrs(dev, psize, &paddr, GFP_KERNEL, attrs);
+	if (!cpu_addr) {
+		dev_err(dev, "failed to allocate %zu bytes for pgtable\n",
+			psize);
+		return -ENOMEM;
+	}
+
+	ret = qcom_scm_iommu_secure_ptbl_init(paddr, psize, spare);
+	if (ret) {
+		dev_err(dev, "failed to init iommu pgtable (%d)\n", ret);
+		goto free_mem;
+	}
+
+	allocated = true;
+	return 0;
+
+free_mem:
+	dma_free_attrs(dev, psize, cpu_addr, paddr, attrs);
+	return ret;
+}
+
+static int get_asid(const struct device_node *np)
+{
+	u32 reg;
+
+	/* read the "reg" property directly to get the relative address
+	 * of the context bank, and calculate the asid from that:
+	 */
+	if (of_property_read_u32_index(np, "reg", 0, &reg))
+		return -ENODEV;
+
+	return reg / 0x1000;      /* context banks are 0x1000 apart */
+}
+
+static int qcom_iommu_ctx_probe(struct platform_device *pdev)
+{
+	struct qcom_iommu_ctx *ctx;
+	struct device *dev = &pdev->dev;
+	struct qcom_iommu_dev *qcom_iommu = dev_get_drvdata(dev->parent);
+	struct resource *res;
+	int ret, irq;
+
+	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+
+	ctx->dev = dev;
+	platform_set_drvdata(pdev, ctx);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	ctx->base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(ctx->base))
+		return PTR_ERR(ctx->base);
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		dev_err(dev, "failed to get irq\n");
+		return -ENODEV;
+	}
+
+	/* clear IRQs before registering fault handler, just in case the
+	 * boot-loader left us a surprise:
+	 */
+	iommu_writel(ctx, ARM_SMMU_CB_FSR, iommu_readl(ctx, ARM_SMMU_CB_FSR));
+
+	ret = devm_request_irq(dev, irq,
+			       qcom_iommu_fault,
+			       IRQF_SHARED,
+			       "qcom-iommu-fault",
+			       ctx);
+	if (ret) {
+		dev_err(dev, "failed to request IRQ %u\n", irq);
+		return ret;
+	}
+
+	ret = get_asid(dev->of_node);
+	if (ret < 0) {
+		dev_err(dev, "missing reg property\n");
+		return ret;
+	}
+
+	ctx->asid = ret;
+
+	dev_dbg(dev, "found asid %u\n", ctx->asid);
+
+	qcom_iommu->ctxs[ctx->asid - 1] = ctx;
+
+	return 0;
+}
+
+static int qcom_iommu_ctx_remove(struct platform_device *pdev)
+{
+	struct qcom_iommu_dev *qcom_iommu = dev_get_drvdata(pdev->dev.parent);
+	struct qcom_iommu_ctx *ctx = platform_get_drvdata(pdev);
+
+	platform_set_drvdata(pdev, NULL);
+
+	qcom_iommu->ctxs[ctx->asid - 1] = NULL;
+
+	return 0;
+}
+
+static const struct of_device_id ctx_of_match[] = {
+	{ .compatible = "qcom,msm-iommu-v1-ns" },
+	{ .compatible = "qcom,msm-iommu-v1-sec" },
+	{ /* sentinel */ }
+};
+
+static struct platform_driver qcom_iommu_ctx_driver = {
+	.driver	= {
+		.name		= "qcom-iommu-ctx",
+		.of_match_table	= of_match_ptr(ctx_of_match),
+	},
+	.probe	= qcom_iommu_ctx_probe,
+	.remove = qcom_iommu_ctx_remove,
+};
+
+static bool qcom_iommu_has_secure_context(struct qcom_iommu_dev *qcom_iommu)
+{
+	struct device_node *child;
+
+	for_each_child_of_node(qcom_iommu->dev->of_node, child)
+		if (of_device_is_compatible(child, "qcom,msm-iommu-v1-sec"))
+			return true;
+
+	return false;
+}
+
+static int qcom_iommu_device_probe(struct platform_device *pdev)
+{
+	struct device_node *child;
+	struct qcom_iommu_dev *qcom_iommu;
+	struct device *dev = &pdev->dev;
+	struct resource *res;
+	int ret, sz, max_asid = 0;
+
+	/* find the max asid (which is 1:1 to ctx bank idx), so we know how
+	 * many child ctx devices we have:
+	 */
+	for_each_child_of_node(dev->of_node, child)
+		max_asid = max(max_asid, get_asid(child));
+
+	sz = sizeof(*qcom_iommu) + (max_asid * sizeof(qcom_iommu->ctxs[0]));
+
+	qcom_iommu = devm_kzalloc(dev, sz, GFP_KERNEL);
+	if (!qcom_iommu)
+		return -ENOMEM;
+	qcom_iommu->num_ctxs = max_asid;
+	qcom_iommu->dev = dev;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (res)
+		qcom_iommu->local_base = devm_ioremap_resource(dev, res);
+
+	qcom_iommu->iface_clk = devm_clk_get(dev, "iface");
+	if (IS_ERR(qcom_iommu->iface_clk)) {
+		dev_err(dev, "failed to get iface clock\n");
+		return PTR_ERR(qcom_iommu->iface_clk);
+	}
+
+	qcom_iommu->bus_clk = devm_clk_get(dev, "bus");
+	if (IS_ERR(qcom_iommu->bus_clk)) {
+		dev_err(dev, "failed to get bus clock\n");
+		return PTR_ERR(qcom_iommu->bus_clk);
+	}
+
+	if (of_property_read_u32(dev->of_node, "qcom,iommu-secure-id",
+				 &qcom_iommu->sec_id)) {
+		dev_err(dev, "missing qcom,iommu-secure-id property\n");
+		return -ENODEV;
+	}
+
+	if (qcom_iommu_has_secure_context(qcom_iommu)) {
+		ret = qcom_iommu_sec_ptbl_init(dev);
+		if (ret) {
+			dev_err(dev, "cannot init secure pg table(%d)\n", ret);
+			return ret;
+		}
+	}
+
+	platform_set_drvdata(pdev, qcom_iommu);
+
+	pm_runtime_enable(dev);
+
+	/* register context bank devices, which are child nodes: */
+	ret = devm_of_platform_populate(dev);
+	if (ret) {
+		dev_err(dev, "Failed to populate iommu contexts\n");
+		return ret;
+	}
+
+	ret = iommu_device_sysfs_add(&qcom_iommu->iommu, dev, NULL,
+				     dev_name(dev));
+	if (ret) {
+		dev_err(dev, "Failed to register iommu in sysfs\n");
+		return ret;
+	}
+
+	iommu_device_set_ops(&qcom_iommu->iommu, &qcom_iommu_ops);
+	iommu_device_set_fwnode(&qcom_iommu->iommu, dev->fwnode);
+
+	ret = iommu_device_register(&qcom_iommu->iommu);
+	if (ret) {
+		dev_err(dev, "Failed to register iommu\n");
+		return ret;
+	}
+
+	bus_set_iommu(&platform_bus_type, &qcom_iommu_ops);
+
+	if (qcom_iommu->local_base) {
+		pm_runtime_get_sync(dev);
+		writel_relaxed(0xffffffff, qcom_iommu->local_base + SMMU_INTR_SEL_NS);
+		pm_runtime_put_sync(dev);
+	}
+
+	return 0;
+}
+
+static int qcom_iommu_device_remove(struct platform_device *pdev)
+{
+	struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev);
+
+	bus_set_iommu(&platform_bus_type, NULL);
+
+	pm_runtime_force_suspend(&pdev->dev);
+	platform_set_drvdata(pdev, NULL);
+	iommu_device_sysfs_remove(&qcom_iommu->iommu);
+	iommu_device_unregister(&qcom_iommu->iommu);
+
+	return 0;
+}
+
+static int __maybe_unused qcom_iommu_resume(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev);
+
+	return qcom_iommu_enable_clocks(qcom_iommu);
+}
+
+static int __maybe_unused qcom_iommu_suspend(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev);
+
+	qcom_iommu_disable_clocks(qcom_iommu);
+
+	return 0;
+}
+
+static const struct dev_pm_ops qcom_iommu_pm_ops = {
+	SET_RUNTIME_PM_OPS(qcom_iommu_suspend, qcom_iommu_resume, NULL)
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
+};
+
+static const struct of_device_id qcom_iommu_of_match[] = {
+	{ .compatible = "qcom,msm-iommu-v1" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, qcom_iommu_of_match);
+
+static struct platform_driver qcom_iommu_driver = {
+	.driver	= {
+		.name		= "qcom-iommu",
+		.of_match_table	= of_match_ptr(qcom_iommu_of_match),
+		.pm		= &qcom_iommu_pm_ops,
+	},
+	.probe	= qcom_iommu_device_probe,
+	.remove	= qcom_iommu_device_remove,
+};
+
+static int __init qcom_iommu_init(void)
+{
+	int ret;
+
+	ret = platform_driver_register(&qcom_iommu_ctx_driver);
+	if (ret)
+		return ret;
+
+	ret = platform_driver_register(&qcom_iommu_driver);
+	if (ret)
+		platform_driver_unregister(&qcom_iommu_ctx_driver);
+
+	return ret;
+}
+
+static void __exit qcom_iommu_exit(void)
+{
+	platform_driver_unregister(&qcom_iommu_driver);
+	platform_driver_unregister(&qcom_iommu_ctx_driver);
+}
+
+module_init(qcom_iommu_init);
+module_exit(qcom_iommu_exit);
+
+IOMMU_OF_DECLARE(qcom_iommu_dev, "qcom,msm-iommu-v1", NULL);
+
+MODULE_DESCRIPTION("IOMMU API for QCOM IOMMU v1 implementations");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c
index 4ba48a26b389..9d991c2d8767 100644
--- a/drivers/iommu/rockchip-iommu.c
+++ b/drivers/iommu/rockchip-iommu.c
@@ -90,7 +90,9 @@ struct rk_iommu {
 	struct device *dev;
 	void __iomem **bases;
 	int num_mmu;
-	int irq;
+	int *irq;
+	int num_irq;
+	bool reset_disabled;
 	struct iommu_device iommu;
 	struct list_head node; /* entry in rk_iommu_domain.iommus */
 	struct iommu_domain *domain; /* domain to which iommu is attached */
@@ -414,6 +416,9 @@ static int rk_iommu_force_reset(struct rk_iommu *iommu)
 	int ret, i;
 	u32 dte_addr;
 
+	if (iommu->reset_disabled)
+		return 0;
+
 	/*
 	 * Check if register DTE_ADDR is working by writing DTE_ADDR_DUMMY
 	 * and verifying that upper 5 nybbles are read back.
@@ -825,10 +830,12 @@ static int rk_iommu_attach_device(struct iommu_domain *domain,
 
 	iommu->domain = domain;
 
-	ret = devm_request_irq(iommu->dev, iommu->irq, rk_iommu_irq,
-			       IRQF_SHARED, dev_name(dev), iommu);
-	if (ret)
-		return ret;
+	for (i = 0; i < iommu->num_irq; i++) {
+		ret = devm_request_irq(iommu->dev, iommu->irq[i], rk_iommu_irq,
+				       IRQF_SHARED, dev_name(dev), iommu);
+		if (ret)
+			return ret;
+	}
 
 	for (i = 0; i < iommu->num_mmu; i++) {
 		rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR,
@@ -878,7 +885,8 @@ static void rk_iommu_detach_device(struct iommu_domain *domain,
 	}
 	rk_iommu_disable_stall(iommu);
 
-	devm_free_irq(iommu->dev, iommu->irq, iommu);
+	for (i = 0; i < iommu->num_irq; i++)
+		devm_free_irq(iommu->dev, iommu->irq[i], iommu);
 
 	iommu->domain = NULL;
 
@@ -1008,20 +1016,20 @@ static int rk_iommu_group_set_iommudata(struct iommu_group *group,
 	ret = of_parse_phandle_with_args(np, "iommus", "#iommu-cells", 0,
 					 &args);
 	if (ret) {
-		dev_err(dev, "of_parse_phandle_with_args(%s) => %d\n",
-			np->full_name, ret);
+		dev_err(dev, "of_parse_phandle_with_args(%pOF) => %d\n",
+			np, ret);
 		return ret;
 	}
 	if (args.args_count != 0) {
-		dev_err(dev, "incorrect number of iommu params found for %s (found %d, expected 0)\n",
-			args.np->full_name, args.args_count);
+		dev_err(dev, "incorrect number of iommu params found for %pOF (found %d, expected 0)\n",
+			args.np, args.args_count);
 		return -EINVAL;
 	}
 
 	pd = of_find_device_by_node(args.np);
 	of_node_put(args.np);
 	if (!pd) {
-		dev_err(dev, "iommu %s not found\n", args.np->full_name);
+		dev_err(dev, "iommu %pOF not found\n", args.np);
 		return -EPROBE_DEFER;
 	}
 
@@ -1157,12 +1165,28 @@ static int rk_iommu_probe(struct platform_device *pdev)
 	if (iommu->num_mmu == 0)
 		return PTR_ERR(iommu->bases[0]);
 
-	iommu->irq = platform_get_irq(pdev, 0);
-	if (iommu->irq < 0) {
-		dev_err(dev, "Failed to get IRQ, %d\n", iommu->irq);
+	iommu->num_irq = platform_irq_count(pdev);
+	if (iommu->num_irq < 0)
+		return iommu->num_irq;
+	if (iommu->num_irq == 0)
 		return -ENXIO;
+
+	iommu->irq = devm_kcalloc(dev, iommu->num_irq, sizeof(*iommu->irq),
+				  GFP_KERNEL);
+	if (!iommu->irq)
+		return -ENOMEM;
+
+	for (i = 0; i < iommu->num_irq; i++) {
+		iommu->irq[i] = platform_get_irq(pdev, i);
+		if (iommu->irq[i] < 0) {
+			dev_err(dev, "Failed to get IRQ, %d\n", iommu->irq[i]);
+			return -ENXIO;
+		}
 	}
 
+	iommu->reset_disabled = device_property_read_bool(dev,
+					"rockchip,disable-mmu-reset");
+
 	err = iommu_device_sysfs_add(&iommu->iommu, dev, NULL, dev_name(dev));
 	if (err)
 		return err;
diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c
index 8788640756a7..0e2f31f9032b 100644
--- a/drivers/iommu/s390-iommu.c
+++ b/drivers/iommu/s390-iommu.c
@@ -18,6 +18,8 @@
  */
 #define S390_IOMMU_PGSIZES	(~0xFFFUL)
 
+static const struct iommu_ops s390_iommu_ops;
+
 struct s390_domain {
 	struct iommu_domain	domain;
 	struct list_head	devices;
@@ -166,11 +168,13 @@ static void s390_iommu_detach_device(struct iommu_domain *domain,
 static int s390_iommu_add_device(struct device *dev)
 {
 	struct iommu_group *group = iommu_group_get_for_dev(dev);
+	struct zpci_dev *zdev = to_pci_dev(dev)->sysdata;
 
 	if (IS_ERR(group))
 		return PTR_ERR(group);
 
 	iommu_group_put(group);
+	iommu_device_link(&zdev->iommu_dev, dev);
 
 	return 0;
 }
@@ -197,6 +201,7 @@ static void s390_iommu_remove_device(struct device *dev)
 			s390_iommu_detach_device(domain, dev);
 	}
 
+	iommu_device_unlink(&zdev->iommu_dev, dev);
 	iommu_group_remove_device(dev);
 }
 
@@ -327,7 +332,37 @@ static size_t s390_iommu_unmap(struct iommu_domain *domain,
 	return size;
 }
 
-static struct iommu_ops s390_iommu_ops = {
+int zpci_init_iommu(struct zpci_dev *zdev)
+{
+	int rc = 0;
+
+	rc = iommu_device_sysfs_add(&zdev->iommu_dev, NULL, NULL,
+				    "s390-iommu.%08x", zdev->fid);
+	if (rc)
+		goto out_err;
+
+	iommu_device_set_ops(&zdev->iommu_dev, &s390_iommu_ops);
+
+	rc = iommu_device_register(&zdev->iommu_dev);
+	if (rc)
+		goto out_sysfs;
+
+	return 0;
+
+out_sysfs:
+	iommu_device_sysfs_remove(&zdev->iommu_dev);
+
+out_err:
+	return rc;
+}
+
+void zpci_destroy_iommu(struct zpci_dev *zdev)
+{
+	iommu_device_unregister(&zdev->iommu_dev);
+	iommu_device_sysfs_remove(&zdev->iommu_dev);
+}
+
+static const struct iommu_ops s390_iommu_ops = {
 	.capable = s390_iommu_capable,
 	.domain_alloc = s390_domain_alloc,
 	.domain_free = s390_domain_free,
diff --git a/drivers/iommu/tegra-gart.c b/drivers/iommu/tegra-gart.c
index 37e708fdbb5a..b62f790ad1ba 100644
--- a/drivers/iommu/tegra-gart.c
+++ b/drivers/iommu/tegra-gart.c
@@ -61,6 +61,8 @@ struct gart_device {
 	struct list_head	client;
 	spinlock_t		client_lock;	/* for client list */
 	struct device		*dev;
+
+	struct iommu_device	iommu;		/* IOMMU Core handle */
 };
 
 struct gart_domain {
@@ -334,12 +336,35 @@ static bool gart_iommu_capable(enum iommu_cap cap)
 	return false;
 }
 
+static int gart_iommu_add_device(struct device *dev)
+{
+	struct iommu_group *group = iommu_group_get_for_dev(dev);
+
+	if (IS_ERR(group))
+		return PTR_ERR(group);
+
+	iommu_group_put(group);
+
+	iommu_device_link(&gart_handle->iommu, dev);
+
+	return 0;
+}
+
+static void gart_iommu_remove_device(struct device *dev)
+{
+	iommu_group_remove_device(dev);
+	iommu_device_unlink(&gart_handle->iommu, dev);
+}
+
 static const struct iommu_ops gart_iommu_ops = {
 	.capable	= gart_iommu_capable,
 	.domain_alloc	= gart_iommu_domain_alloc,
 	.domain_free	= gart_iommu_domain_free,
 	.attach_dev	= gart_iommu_attach_dev,
 	.detach_dev	= gart_iommu_detach_dev,
+	.add_device	= gart_iommu_add_device,
+	.remove_device	= gart_iommu_remove_device,
+	.device_group	= generic_device_group,
 	.map		= gart_iommu_map,
 	.map_sg		= default_iommu_map_sg,
 	.unmap		= gart_iommu_unmap,
@@ -378,6 +403,7 @@ static int tegra_gart_probe(struct platform_device *pdev)
 	struct resource *res, *res_remap;
 	void __iomem *gart_regs;
 	struct device *dev = &pdev->dev;
+	int ret;
 
 	if (gart_handle)
 		return -EIO;
@@ -404,6 +430,22 @@ static int tegra_gart_probe(struct platform_device *pdev)
 		return -ENXIO;
 	}
 
+	ret = iommu_device_sysfs_add(&gart->iommu, &pdev->dev, NULL,
+				     dev_name(&pdev->dev));
+	if (ret) {
+		dev_err(dev, "Failed to register IOMMU in sysfs\n");
+		return ret;
+	}
+
+	iommu_device_set_ops(&gart->iommu, &gart_iommu_ops);
+
+	ret = iommu_device_register(&gart->iommu);
+	if (ret) {
+		dev_err(dev, "Failed to register IOMMU\n");
+		iommu_device_sysfs_remove(&gart->iommu);
+		return ret;
+	}
+
 	gart->dev = &pdev->dev;
 	spin_lock_init(&gart->pte_lock);
 	spin_lock_init(&gart->client_lock);
@@ -430,6 +472,9 @@ static int tegra_gart_remove(struct platform_device *pdev)
 {
 	struct gart_device *gart = platform_get_drvdata(pdev);
 
+	iommu_device_unregister(&gart->iommu);
+	iommu_device_sysfs_remove(&gart->iommu);
+
 	writel(0, gart->regs + GART_CONFIG);
 	if (gart->savedata)
 		vfree(gart->savedata);
diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c
index eeb19f560a05..3b6449e2cbf1 100644
--- a/drivers/iommu/tegra-smmu.c
+++ b/drivers/iommu/tegra-smmu.c
@@ -36,6 +36,8 @@ struct tegra_smmu {
 	struct list_head list;
 
 	struct dentry *debugfs;
+
+	struct iommu_device iommu;	/* IOMMU Core code handle */
 };
 
 struct tegra_smmu_as {
@@ -704,6 +706,7 @@ static struct tegra_smmu *tegra_smmu_find(struct device_node *np)
 static int tegra_smmu_add_device(struct device *dev)
 {
 	struct device_node *np = dev->of_node;
+	struct iommu_group *group;
 	struct of_phandle_args args;
 	unsigned int index = 0;
 
@@ -719,18 +722,33 @@ static int tegra_smmu_add_device(struct device *dev)
 			 * first match.
 			 */
 			dev->archdata.iommu = smmu;
+
+			iommu_device_link(&smmu->iommu, dev);
+
 			break;
 		}
 
 		index++;
 	}
 
+	group = iommu_group_get_for_dev(dev);
+	if (IS_ERR(group))
+		return PTR_ERR(group);
+
+	iommu_group_put(group);
+
 	return 0;
 }
 
 static void tegra_smmu_remove_device(struct device *dev)
 {
+	struct tegra_smmu *smmu = dev->archdata.iommu;
+
+	if (smmu)
+		iommu_device_unlink(&smmu->iommu, dev);
+
 	dev->archdata.iommu = NULL;
+	iommu_group_remove_device(dev);
 }
 
 static const struct iommu_ops tegra_smmu_ops = {
@@ -741,6 +759,7 @@ static const struct iommu_ops tegra_smmu_ops = {
 	.detach_dev = tegra_smmu_detach_dev,
 	.add_device = tegra_smmu_add_device,
 	.remove_device = tegra_smmu_remove_device,
+	.device_group = generic_device_group,
 	.map = tegra_smmu_map,
 	.unmap = tegra_smmu_unmap,
 	.map_sg = default_iommu_map_sg,
@@ -930,9 +949,24 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev,
 
 	tegra_smmu_ahb_enable();
 
+	err = iommu_device_sysfs_add(&smmu->iommu, dev, NULL, dev_name(dev));
+	if (err)
+		return ERR_PTR(err);
+
+	iommu_device_set_ops(&smmu->iommu, &tegra_smmu_ops);
+
+	err = iommu_device_register(&smmu->iommu);
+	if (err) {
+		iommu_device_sysfs_remove(&smmu->iommu);
+		return ERR_PTR(err);
+	}
+
 	err = bus_set_iommu(&platform_bus_type, &tegra_smmu_ops);
-	if (err < 0)
+	if (err < 0) {
+		iommu_device_unregister(&smmu->iommu);
+		iommu_device_sysfs_remove(&smmu->iommu);
 		return ERR_PTR(err);
+	}
 
 	if (IS_ENABLED(CONFIG_DEBUG_FS))
 		tegra_smmu_debugfs_init(smmu);
@@ -942,6 +976,9 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev,
 
 void tegra_smmu_remove(struct tegra_smmu *smmu)
 {
+	iommu_device_unregister(&smmu->iommu);
+	iommu_device_sysfs_remove(&smmu->iommu);
+
 	if (IS_ENABLED(CONFIG_DEBUG_FS))
 		tegra_smmu_debugfs_exit(smmu);
 }
diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig
index f1fd5f44d1d4..9d8a1dd2e2c2 100644
--- a/drivers/irqchip/Kconfig
+++ b/drivers/irqchip/Kconfig
@@ -7,6 +7,7 @@ config ARM_GIC
 	select IRQ_DOMAIN
 	select IRQ_DOMAIN_HIERARCHY
 	select MULTI_IRQ_HANDLER
+	select GENERIC_IRQ_EFFECTIVE_AFF_MASK
 
 config ARM_GIC_PM
 	bool
@@ -34,6 +35,7 @@ config ARM_GIC_V3
 	select MULTI_IRQ_HANDLER
 	select IRQ_DOMAIN_HIERARCHY
 	select PARTITION_PERCPU
+	select GENERIC_IRQ_EFFECTIVE_AFF_MASK
 
 config ARM_GIC_V3_ITS
 	bool
@@ -64,6 +66,7 @@ config ARMADA_370_XP_IRQ
 	bool
 	select GENERIC_IRQ_CHIP
 	select PCI_MSI if PCI
+	select GENERIC_IRQ_EFFECTIVE_AFF_MASK
 
 config ALPINE_MSI
 	bool
@@ -93,11 +96,13 @@ config BCM6345_L1_IRQ
 	bool
 	select GENERIC_IRQ_CHIP
 	select IRQ_DOMAIN
+	select GENERIC_IRQ_EFFECTIVE_AFF_MASK
 
 config BCM7038_L1_IRQ
 	bool
 	select GENERIC_IRQ_CHIP
 	select IRQ_DOMAIN
+	select GENERIC_IRQ_EFFECTIVE_AFF_MASK
 
 config BCM7120_L2_IRQ
 	bool
@@ -136,6 +141,7 @@ config IRQ_MIPS_CPU
 	select GENERIC_IRQ_IPI if SYS_SUPPORTS_MULTITHREADING
 	select IRQ_DOMAIN
 	select IRQ_DOMAIN_HIERARCHY if GENERIC_IRQ_IPI
+	select GENERIC_IRQ_EFFECTIVE_AFF_MASK
 
 config CLPS711X_IRQCHIP
 	bool
@@ -217,6 +223,7 @@ config VERSATILE_FPGA_IRQ_NR
 config XTENSA_MX
 	bool
 	select IRQ_DOMAIN
+	select GENERIC_IRQ_EFFECTIVE_AFF_MASK
 
 config XILINX_INTC
 	bool
@@ -306,3 +313,11 @@ config QCOM_IRQ_COMBINER
 	help
 	  Say yes here to add support for the IRQ combiner devices embedded
 	  in Qualcomm Technologies chips.
+
+config IRQ_UNIPHIER_AIDET
+	bool "UniPhier AIDET support" if COMPILE_TEST
+	depends on ARCH_UNIPHIER || COMPILE_TEST
+	default ARCH_UNIPHIER
+	select IRQ_DOMAIN_HIERARCHY
+	help
+	  Support for the UniPhier AIDET (ARM Interrupt Detector).
diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile
index e88d856cc09c..845abc107ad5 100644
--- a/drivers/irqchip/Makefile
+++ b/drivers/irqchip/Makefile
@@ -28,7 +28,7 @@ obj-$(CONFIG_ARM_GIC_PM)		+= irq-gic-pm.o
 obj-$(CONFIG_ARCH_REALVIEW)		+= irq-gic-realview.o
 obj-$(CONFIG_ARM_GIC_V2M)		+= irq-gic-v2m.o
 obj-$(CONFIG_ARM_GIC_V3)		+= irq-gic-v3.o irq-gic-common.o
-obj-$(CONFIG_ARM_GIC_V3_ITS)		+= irq-gic-v3-its.o irq-gic-v3-its-pci-msi.o irq-gic-v3-its-platform-msi.o
+obj-$(CONFIG_ARM_GIC_V3_ITS)		+= irq-gic-v3-its.o irq-gic-v3-its-pci-msi.o irq-gic-v3-its-platform-msi.o irq-gic-v4.o
 obj-$(CONFIG_PARTITION_PERCPU)		+= irq-partition-percpu.o
 obj-$(CONFIG_HISILICON_IRQ_MBIGEN)	+= irq-mbigen.o
 obj-$(CONFIG_ARM_NVIC)			+= irq-nvic.o
@@ -78,3 +78,4 @@ obj-$(CONFIG_EZNPS_GIC)			+= irq-eznps.o
 obj-$(CONFIG_ARCH_ASPEED)		+= irq-aspeed-vic.o irq-aspeed-i2c-ic.o
 obj-$(CONFIG_STM32_EXTI) 		+= irq-stm32-exti.o
 obj-$(CONFIG_QCOM_IRQ_COMBINER)		+= qcom-irq-combiner.o
+obj-$(CONFIG_IRQ_UNIPHIER_AIDET)	+= irq-uniphier-aidet.o
diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c
index b207b2c3aa55..c9bdc5221b82 100644
--- a/drivers/irqchip/irq-armada-370-xp.c
+++ b/drivers/irqchip/irq-armada-370-xp.c
@@ -203,7 +203,7 @@ static struct irq_chip armada_370_xp_msi_irq_chip = {
 
 static struct msi_domain_info armada_370_xp_msi_domain_info = {
 	.flags	= (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
-		   MSI_FLAG_MULTI_PCI_MSI),
+		   MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX),
 	.chip	= &armada_370_xp_msi_irq_chip,
 };
 
@@ -330,6 +330,8 @@ static int armada_xp_set_affinity(struct irq_data *d,
 	writel(reg, main_int_base + ARMADA_370_XP_INT_SOURCE_CTL(hwirq));
 	raw_spin_unlock(&irq_controller_lock);
 
+	irq_data_update_effective_affinity(d, cpumask_of(cpu));
+
 	return IRQ_SET_MASK_OK;
 }
 #endif
@@ -363,6 +365,7 @@ static int armada_370_xp_mpic_irq_map(struct irq_domain *h,
 	} else {
 		irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip,
 					handle_level_irq);
+		irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(virq)));
 	}
 	irq_set_probe(virq);
 
diff --git a/drivers/irqchip/irq-atmel-aic-common.c b/drivers/irqchip/irq-atmel-aic-common.c
index 28b26c80f4cf..072bd227b6c6 100644
--- a/drivers/irqchip/irq-atmel-aic-common.c
+++ b/drivers/irqchip/irq-atmel-aic-common.c
@@ -137,14 +137,14 @@ static void __init aic_common_ext_irq_of_init(struct irq_domain *domain)
 #define AT91_RTC_IMR           0x28
 #define AT91_RTC_IRQ_MASK      0x1f
 
-void __init aic_common_rtc_irq_fixup(struct device_node *root)
+void __init aic_common_rtc_irq_fixup(void)
 {
 	struct device_node *np;
 	void __iomem *regs;
 
-	np = of_find_compatible_node(root, NULL, "atmel,at91rm9200-rtc");
+	np = of_find_compatible_node(NULL, NULL, "atmel,at91rm9200-rtc");
 	if (!np)
-		np = of_find_compatible_node(root, NULL,
+		np = of_find_compatible_node(NULL, NULL,
 					     "atmel,at91sam9x5-rtc");
 
 	if (!np)
@@ -165,7 +165,7 @@ void __init aic_common_rtc_irq_fixup(struct device_node *root)
 #define AT91_RTT_ALMIEN		(1 << 16)		/* Alarm Interrupt Enable */
 #define AT91_RTT_RTTINCIEN	(1 << 17)		/* Real Time Timer Increment Interrupt Enable */
 
-void __init aic_common_rtt_irq_fixup(struct device_node *root)
+void __init aic_common_rtt_irq_fixup(void)
 {
 	struct device_node *np;
 	void __iomem *regs;
@@ -196,11 +196,10 @@ static void __init aic_common_irq_fixup(const struct of_device_id *matches)
 		return;
 
 	match = of_match_node(matches, root);
-	of_node_put(root);
 
 	if (match) {
-		void (*fixup)(struct device_node *) = match->data;
-		fixup(root);
+		void (*fixup)(void) = match->data;
+		fixup();
 	}
 
 	of_node_put(root);
diff --git a/drivers/irqchip/irq-atmel-aic-common.h b/drivers/irqchip/irq-atmel-aic-common.h
index af60376d50de..242e62c1851e 100644
--- a/drivers/irqchip/irq-atmel-aic-common.h
+++ b/drivers/irqchip/irq-atmel-aic-common.h
@@ -33,8 +33,8 @@ struct irq_domain *__init aic_common_of_init(struct device_node *node,
 					     const char *name, int nirqs,
 					     const struct of_device_id *matches);
 
-void __init aic_common_rtc_irq_fixup(struct device_node *root);
+void __init aic_common_rtc_irq_fixup(void);
 
-void __init aic_common_rtt_irq_fixup(struct device_node *root);
+void __init aic_common_rtt_irq_fixup(void);
 
 #endif /* __IRQ_ATMEL_AIC_COMMON_H */
diff --git a/drivers/irqchip/irq-atmel-aic.c b/drivers/irqchip/irq-atmel-aic.c
index 37f952dd9fc9..bb1ad451392f 100644
--- a/drivers/irqchip/irq-atmel-aic.c
+++ b/drivers/irqchip/irq-atmel-aic.c
@@ -209,20 +209,20 @@ static const struct irq_domain_ops aic_irq_ops = {
 	.xlate	= aic_irq_domain_xlate,
 };
 
-static void __init at91rm9200_aic_irq_fixup(struct device_node *root)
+static void __init at91rm9200_aic_irq_fixup(void)
 {
-	aic_common_rtc_irq_fixup(root);
+	aic_common_rtc_irq_fixup();
 }
 
-static void __init at91sam9260_aic_irq_fixup(struct device_node *root)
+static void __init at91sam9260_aic_irq_fixup(void)
 {
-	aic_common_rtt_irq_fixup(root);
+	aic_common_rtt_irq_fixup();
 }
 
-static void __init at91sam9g45_aic_irq_fixup(struct device_node *root)
+static void __init at91sam9g45_aic_irq_fixup(void)
 {
-	aic_common_rtc_irq_fixup(root);
-	aic_common_rtt_irq_fixup(root);
+	aic_common_rtc_irq_fixup();
+	aic_common_rtt_irq_fixup();
 }
 
 static const struct of_device_id aic_irq_fixups[] __initconst = {
diff --git a/drivers/irqchip/irq-atmel-aic5.c b/drivers/irqchip/irq-atmel-aic5.c
index c04ee9a23d09..6acad2ea0fb3 100644
--- a/drivers/irqchip/irq-atmel-aic5.c
+++ b/drivers/irqchip/irq-atmel-aic5.c
@@ -305,9 +305,9 @@ static const struct irq_domain_ops aic5_irq_ops = {
 	.xlate	= aic5_irq_domain_xlate,
 };
 
-static void __init sama5d3_aic_irq_fixup(struct device_node *root)
+static void __init sama5d3_aic_irq_fixup(void)
 {
-	aic_common_rtc_irq_fixup(root);
+	aic_common_rtc_irq_fixup();
 }
 
 static const struct of_device_id aic5_irq_fixups[] __initconst = {
diff --git a/drivers/irqchip/irq-bcm2835.c b/drivers/irqchip/irq-bcm2835.c
index 44d7c38dde47..d2da8a1e6b1b 100644
--- a/drivers/irqchip/irq-bcm2835.c
+++ b/drivers/irqchip/irq-bcm2835.c
@@ -147,13 +147,12 @@ static int __init armctrl_of_init(struct device_node *node,
 
 	base = of_iomap(node, 0);
 	if (!base)
-		panic("%s: unable to map IC registers\n",
-			node->full_name);
+		panic("%pOF: unable to map IC registers\n", node);
 
 	intc.domain = irq_domain_add_linear(node, MAKE_HWIRQ(NR_BANKS, 0),
 			&armctrl_ops, NULL);
 	if (!intc.domain)
-		panic("%s: unable to create IRQ domain\n", node->full_name);
+		panic("%pOF: unable to create IRQ domain\n", node);
 
 	for (b = 0; b < NR_BANKS; b++) {
 		intc.pending[b] = base + reg_pending[b];
@@ -173,8 +172,8 @@ static int __init armctrl_of_init(struct device_node *node,
 		int parent_irq = irq_of_parse_and_map(node, 0);
 
 		if (!parent_irq) {
-			panic("%s: unable to get parent interrupt.\n",
-			      node->full_name);
+			panic("%pOF: unable to get parent interrupt.\n",
+			      node);
 		}
 		irq_set_chained_handler(parent_irq, bcm2836_chained_handle_irq);
 	} else {
diff --git a/drivers/irqchip/irq-bcm2836.c b/drivers/irqchip/irq-bcm2836.c
index e7463e3c0814..dc8c1e3eafe7 100644
--- a/drivers/irqchip/irq-bcm2836.c
+++ b/drivers/irqchip/irq-bcm2836.c
@@ -282,8 +282,7 @@ static int __init bcm2836_arm_irqchip_l1_intc_of_init(struct device_node *node,
 {
 	intc.base = of_iomap(node, 0);
 	if (!intc.base) {
-		panic("%s: unable to map local interrupt registers\n",
-			node->full_name);
+		panic("%pOF: unable to map local interrupt registers\n", node);
 	}
 
 	bcm2835_init_local_timer_frequency();
@@ -292,7 +291,7 @@ static int __init bcm2836_arm_irqchip_l1_intc_of_init(struct device_node *node,
 					    &bcm2836_arm_irqchip_intc_ops,
 					    NULL);
 	if (!intc.domain)
-		panic("%s: unable to create IRQ domain\n", node->full_name);
+		panic("%pOF: unable to create IRQ domain\n", node);
 
 	bcm2836_arm_irqchip_register_irq(LOCAL_IRQ_CNTPSIRQ,
 					 &bcm2836_arm_irqchip_timer);
diff --git a/drivers/irqchip/irq-bcm6345-l1.c b/drivers/irqchip/irq-bcm6345-l1.c
index daa4ae89e466..43f8abe40878 100644
--- a/drivers/irqchip/irq-bcm6345-l1.c
+++ b/drivers/irqchip/irq-bcm6345-l1.c
@@ -231,6 +231,8 @@ static int bcm6345_l1_set_affinity(struct irq_data *d,
 	}
 	raw_spin_unlock_irqrestore(&intc->lock, flags);
 
+	irq_data_update_effective_affinity(d, cpumask_of(new_cpu));
+
 	return IRQ_SET_MASK_OK_NOCOPY;
 }
 
@@ -291,6 +293,7 @@ static int bcm6345_l1_map(struct irq_domain *d, unsigned int virq,
 	irq_set_chip_and_handler(virq,
 		&bcm6345_l1_irq_chip, handle_percpu_irq);
 	irq_set_chip_data(virq, d->host_data);
+	irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(virq)));
 	return 0;
 }
 
diff --git a/drivers/irqchip/irq-bcm7038-l1.c b/drivers/irqchip/irq-bcm7038-l1.c
index c2662a1bfdd3..55cfb986225b 100644
--- a/drivers/irqchip/irq-bcm7038-l1.c
+++ b/drivers/irqchip/irq-bcm7038-l1.c
@@ -212,6 +212,8 @@ static int bcm7038_l1_set_affinity(struct irq_data *d,
 		__bcm7038_l1_unmask(d, first_cpu);
 
 	raw_spin_unlock_irqrestore(&intc->lock, flags);
+	irq_data_update_effective_affinity(d, cpumask_of(first_cpu));
+
 	return 0;
 }
 
@@ -299,6 +301,7 @@ static int bcm7038_l1_map(struct irq_domain *d, unsigned int virq,
 {
 	irq_set_chip_and_handler(virq, &bcm7038_l1_irq_chip, handle_level_irq);
 	irq_set_chip_data(virq, d->host_data);
+	irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(virq)));
 	return 0;
 }
 
diff --git a/drivers/irqchip/irq-bcm7120-l2.c b/drivers/irqchip/irq-bcm7120-l2.c
index 64c2692070ef..983640eba418 100644
--- a/drivers/irqchip/irq-bcm7120-l2.c
+++ b/drivers/irqchip/irq-bcm7120-l2.c
@@ -250,12 +250,6 @@ static int __init bcm7120_l2_intc_probe(struct device_node *dn,
 	if (ret < 0)
 		goto out_free_l1_data;
 
-	for (idx = 0; idx < data->n_words; idx++) {
-		__raw_writel(data->irq_fwd_mask[idx],
-			     data->pair_base[idx] +
-			     data->en_offset[idx]);
-	}
-
 	for (irq = 0; irq < data->num_parent_irqs; irq++) {
 		ret = bcm7120_l2_intc_init_one(dn, data, irq, valid_mask);
 		if (ret)
@@ -297,6 +291,10 @@ static int __init bcm7120_l2_intc_probe(struct device_node *dn,
 		gc->reg_base = data->pair_base[idx];
 		ct->regs.mask = data->en_offset[idx];
 
+		/* gc->reg_base is defined and so is gc->writel */
+		irq_reg_writel(gc, data->irq_fwd_mask[idx],
+			       data->en_offset[idx]);
+
 		ct->chip.irq_mask = irq_gc_mask_clr_bit;
 		ct->chip.irq_unmask = irq_gc_mask_set_bit;
 		ct->chip.irq_ack = irq_gc_noop;
diff --git a/drivers/irqchip/irq-brcmstb-l2.c b/drivers/irqchip/irq-brcmstb-l2.c
index bddf169c4b37..b009b916a292 100644
--- a/drivers/irqchip/irq-brcmstb-l2.c
+++ b/drivers/irqchip/irq-brcmstb-l2.c
@@ -189,6 +189,7 @@ static int __init brcmstb_l2_intc_of_init(struct device_node *np,
 
 	ct->chip.irq_suspend = brcmstb_l2_intc_suspend;
 	ct->chip.irq_resume = brcmstb_l2_intc_resume;
+	ct->chip.irq_pm_shutdown = brcmstb_l2_intc_suspend;
 
 	if (data->can_wake) {
 		/* This IRQ chip can wake the system, set all child interrupts
diff --git a/drivers/irqchip/irq-crossbar.c b/drivers/irqchip/irq-crossbar.c
index f96601268f71..99d97d7e3fd7 100644
--- a/drivers/irqchip/irq-crossbar.c
+++ b/drivers/irqchip/irq-crossbar.c
@@ -341,13 +341,13 @@ static int __init irqcrossbar_init(struct device_node *node,
 	int err;
 
 	if (!parent) {
-		pr_err("%s: no parent, giving up\n", node->full_name);
+		pr_err("%pOF: no parent, giving up\n", node);
 		return -ENODEV;
 	}
 
 	parent_domain = irq_find_host(parent);
 	if (!parent_domain) {
-		pr_err("%s: unable to obtain parent domain\n", node->full_name);
+		pr_err("%pOF: unable to obtain parent domain\n", node);
 		return -ENXIO;
 	}
 
@@ -360,7 +360,7 @@ static int __init irqcrossbar_init(struct device_node *node,
 					  node, &crossbar_domain_ops,
 					  NULL);
 	if (!domain) {
-		pr_err("%s: failed to allocated domain\n", node->full_name);
+		pr_err("%pOF: failed to allocated domain\n", node);
 		return -ENOMEM;
 	}
 
diff --git a/drivers/irqchip/irq-digicolor.c b/drivers/irqchip/irq-digicolor.c
index 3aae015469a5..fc38d2da11b9 100644
--- a/drivers/irqchip/irq-digicolor.c
+++ b/drivers/irqchip/irq-digicolor.c
@@ -78,7 +78,7 @@ static int __init digicolor_of_init(struct device_node *node,
 
 	reg_base = of_iomap(node, 0);
 	if (!reg_base) {
-		pr_err("%s: unable to map IC registers\n", node->full_name);
+		pr_err("%pOF: unable to map IC registers\n", node);
 		return -ENXIO;
 	}
 
@@ -88,7 +88,7 @@ static int __init digicolor_of_init(struct device_node *node,
 
 	ucregs = syscon_regmap_lookup_by_phandle(node, "syscon");
 	if (IS_ERR(ucregs)) {
-		pr_err("%s: unable to map UC registers\n", node->full_name);
+		pr_err("%pOF: unable to map UC registers\n", node);
 		return PTR_ERR(ucregs);
 	}
 	/* channel 1, regular IRQs */
@@ -97,7 +97,7 @@ static int __init digicolor_of_init(struct device_node *node,
 	digicolor_irq_domain =
 		irq_domain_add_linear(node, 64, &irq_generic_chip_ops, NULL);
 	if (!digicolor_irq_domain) {
-		pr_err("%s: unable to create IRQ domain\n", node->full_name);
+		pr_err("%pOF: unable to create IRQ domain\n", node);
 		return -ENOMEM;
 	}
 
@@ -105,7 +105,7 @@ static int __init digicolor_of_init(struct device_node *node,
 					     "digicolor_irq", handle_level_irq,
 					     clr, 0, 0);
 	if (ret) {
-		pr_err("%s: unable to allocate IRQ gc\n", node->full_name);
+		pr_err("%pOF: unable to allocate IRQ gc\n", node);
 		return ret;
 	}
 
diff --git a/drivers/irqchip/irq-dw-apb-ictl.c b/drivers/irqchip/irq-dw-apb-ictl.c
index 052f266364c0..0a19618ce2c8 100644
--- a/drivers/irqchip/irq-dw-apb-ictl.c
+++ b/drivers/irqchip/irq-dw-apb-ictl.c
@@ -79,24 +79,24 @@ static int __init dw_apb_ictl_init(struct device_node *np,
 	/* Map the parent interrupt for the chained handler */
 	irq = irq_of_parse_and_map(np, 0);
 	if (irq <= 0) {
-		pr_err("%s: unable to parse irq\n", np->full_name);
+		pr_err("%pOF: unable to parse irq\n", np);
 		return -EINVAL;
 	}
 
 	ret = of_address_to_resource(np, 0, &r);
 	if (ret) {
-		pr_err("%s: unable to get resource\n", np->full_name);
+		pr_err("%pOF: unable to get resource\n", np);
 		return ret;
 	}
 
 	if (!request_mem_region(r.start, resource_size(&r), np->full_name)) {
-		pr_err("%s: unable to request mem region\n", np->full_name);
+		pr_err("%pOF: unable to request mem region\n", np);
 		return -ENOMEM;
 	}
 
 	iobase = ioremap(r.start, resource_size(&r));
 	if (!iobase) {
-		pr_err("%s: unable to map resource\n", np->full_name);
+		pr_err("%pOF: unable to map resource\n", np);
 		ret = -ENOMEM;
 		goto err_release;
 	}
@@ -123,7 +123,7 @@ static int __init dw_apb_ictl_init(struct device_node *np,
 	domain = irq_domain_add_linear(np, nrirqs,
 				       &irq_generic_chip_ops, NULL);
 	if (!domain) {
-		pr_err("%s: unable to add irq domain\n", np->full_name);
+		pr_err("%pOF: unable to add irq domain\n", np);
 		ret = -ENOMEM;
 		goto err_unmap;
 	}
@@ -132,7 +132,7 @@ static int __init dw_apb_ictl_init(struct device_node *np,
 					     handle_level_irq, clr, 0,
 					     IRQ_GC_INIT_MASK_CACHE);
 	if (ret) {
-		pr_err("%s: unable to alloc irq domain gc\n", np->full_name);
+		pr_err("%pOF: unable to alloc irq domain gc\n", np);
 		goto err_unmap;
 	}
 
diff --git a/drivers/irqchip/irq-gic-v3-its-pci-msi.c b/drivers/irqchip/irq-gic-v3-its-pci-msi.c
index 77931214d954..14a8c0a7e095 100644
--- a/drivers/irqchip/irq-gic-v3-its-pci-msi.c
+++ b/drivers/irqchip/irq-gic-v3-its-pci-msi.c
@@ -138,7 +138,7 @@ static int __init its_pci_of_msi_init(void)
 		if (its_pci_msi_init_one(of_node_to_fwnode(np), np->full_name))
 			continue;
 
-		pr_info("PCI/MSI: %s domain created\n", np->full_name);
+		pr_info("PCI/MSI: %pOF domain created\n", np);
 	}
 
 	return 0;
diff --git a/drivers/irqchip/irq-gic-v3-its-platform-msi.c b/drivers/irqchip/irq-gic-v3-its-platform-msi.c
index 249240d9a425..833a90fe33ae 100644
--- a/drivers/irqchip/irq-gic-v3-its-platform-msi.c
+++ b/drivers/irqchip/irq-gic-v3-its-platform-msi.c
@@ -43,6 +43,7 @@ static int of_pmsi_get_dev_id(struct irq_domain *domain, struct device *dev,
 			*dev_id = args.args[0];
 			break;
 		}
+		index++;
 	} while (!ret);
 
 	return ret;
diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c
index 68932873eebc..e8d89343d613 100644
--- a/drivers/irqchip/irq-gic-v3-its.c
+++ b/drivers/irqchip/irq-gic-v3-its.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2013, 2014 ARM Limited, All Rights Reserved.
+ * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
  * Author: Marc Zyngier <marc.zyngier@arm.com>
  *
  * This program is free software; you can redistribute it and/or modify
@@ -36,6 +36,7 @@
 
 #include <linux/irqchip.h>
 #include <linux/irqchip/arm-gic-v3.h>
+#include <linux/irqchip/arm-gic-v4.h>
 
 #include <asm/cputype.h>
 #include <asm/exception.h>
@@ -48,6 +49,19 @@
 
 #define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING	(1 << 0)
 
+static u32 lpi_id_bits;
+
+/*
+ * We allocate memory for PROPBASE to cover 2 ^ lpi_id_bits LPIs to
+ * deal with (one configuration byte per interrupt). PENDBASE has to
+ * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI).
+ */
+#define LPI_NRBITS		lpi_id_bits
+#define LPI_PROPBASE_SZ		ALIGN(BIT(LPI_NRBITS), SZ_64K)
+#define LPI_PENDBASE_SZ		ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K)
+
+#define LPI_PROP_DEFAULT_PRIO	0xa0
+
 /*
  * Collection structure - just an ID, and a redistributor address to
  * ping. We use one per CPU as a bag of interrupts assigned to this
@@ -88,6 +102,7 @@ struct its_node {
 	u32			ite_size;
 	u32			device_ids;
 	int			numa_node;
+	bool			is_v4;
 };
 
 #define ITS_ITT_ALIGN		SZ_256
@@ -100,11 +115,17 @@ struct event_lpi_map {
 	u16			*col_map;
 	irq_hw_number_t		lpi_base;
 	int			nr_lpis;
+	struct mutex		vlpi_lock;
+	struct its_vm		*vm;
+	struct its_vlpi_map	*vlpi_maps;
+	int			nr_vlpis;
 };
 
 /*
- * The ITS view of a device - belongs to an ITS, a collection, owns an
- * interrupt translation table, and a list of interrupts.
+ * The ITS view of a device - belongs to an ITS, owns an interrupt
+ * translation table, and a list of interrupts.  If it some of its
+ * LPIs are injected into a guest (GICv4), the event_map.vm field
+ * indicates which one.
  */
 struct its_device {
 	struct list_head	entry;
@@ -115,13 +136,33 @@ struct its_device {
 	u32			device_id;
 };
 
+static struct {
+	raw_spinlock_t		lock;
+	struct its_device	*dev;
+	struct its_vpe		**vpes;
+	int			next_victim;
+} vpe_proxy;
+
 static LIST_HEAD(its_nodes);
 static DEFINE_SPINLOCK(its_lock);
 static struct rdists *gic_rdists;
 static struct irq_domain *its_parent;
 
+/*
+ * We have a maximum number of 16 ITSs in the whole system if we're
+ * using the ITSList mechanism
+ */
+#define ITS_LIST_MAX		16
+
+static unsigned long its_list_map;
+static u16 vmovp_seq_num;
+static DEFINE_RAW_SPINLOCK(vmovp_lock);
+
+static DEFINE_IDA(its_vpeid_ida);
+
 #define gic_data_rdist()		(raw_cpu_ptr(gic_rdists->rdist))
 #define gic_data_rdist_rd_base()	(gic_data_rdist()->rd_base)
+#define gic_data_rdist_vlpi_base()	(gic_data_rdist_rd_base() + SZ_128K)
 
 static struct its_collection *dev_event_to_col(struct its_device *its_dev,
 					       u32 event)
@@ -145,6 +186,11 @@ struct its_cmd_desc {
 		struct {
 			struct its_device *dev;
 			u32 event_id;
+		} its_clear_cmd;
+
+		struct {
+			struct its_device *dev;
+			u32 event_id;
 		} its_int_cmd;
 
 		struct {
@@ -177,6 +223,38 @@ struct its_cmd_desc {
 		struct {
 			struct its_collection *col;
 		} its_invall_cmd;
+
+		struct {
+			struct its_vpe *vpe;
+		} its_vinvall_cmd;
+
+		struct {
+			struct its_vpe *vpe;
+			struct its_collection *col;
+			bool valid;
+		} its_vmapp_cmd;
+
+		struct {
+			struct its_vpe *vpe;
+			struct its_device *dev;
+			u32 virt_id;
+			u32 event_id;
+			bool db_enabled;
+		} its_vmapti_cmd;
+
+		struct {
+			struct its_vpe *vpe;
+			struct its_device *dev;
+			u32 event_id;
+			bool db_enabled;
+		} its_vmovi_cmd;
+
+		struct {
+			struct its_vpe *vpe;
+			struct its_collection *col;
+			u16 seq_num;
+			u16 its_list;
+		} its_vmovp_cmd;
 	};
 };
 
@@ -193,6 +271,9 @@ struct its_cmd_block {
 typedef struct its_collection *(*its_cmd_builder_t)(struct its_cmd_block *,
 						    struct its_cmd_desc *);
 
+typedef struct its_vpe *(*its_cmd_vbuilder_t)(struct its_cmd_block *,
+					      struct its_cmd_desc *);
+
 static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l)
 {
 	u64 mask = GENMASK_ULL(h, l);
@@ -245,6 +326,46 @@ static void its_encode_collection(struct its_cmd_block *cmd, u16 col)
 	its_mask_encode(&cmd->raw_cmd[2], col, 15, 0);
 }
 
+static void its_encode_vpeid(struct its_cmd_block *cmd, u16 vpeid)
+{
+	its_mask_encode(&cmd->raw_cmd[1], vpeid, 47, 32);
+}
+
+static void its_encode_virt_id(struct its_cmd_block *cmd, u32 virt_id)
+{
+	its_mask_encode(&cmd->raw_cmd[2], virt_id, 31, 0);
+}
+
+static void its_encode_db_phys_id(struct its_cmd_block *cmd, u32 db_phys_id)
+{
+	its_mask_encode(&cmd->raw_cmd[2], db_phys_id, 63, 32);
+}
+
+static void its_encode_db_valid(struct its_cmd_block *cmd, bool db_valid)
+{
+	its_mask_encode(&cmd->raw_cmd[2], db_valid, 0, 0);
+}
+
+static void its_encode_seq_num(struct its_cmd_block *cmd, u16 seq_num)
+{
+	its_mask_encode(&cmd->raw_cmd[0], seq_num, 47, 32);
+}
+
+static void its_encode_its_list(struct its_cmd_block *cmd, u16 its_list)
+{
+	its_mask_encode(&cmd->raw_cmd[1], its_list, 15, 0);
+}
+
+static void its_encode_vpt_addr(struct its_cmd_block *cmd, u64 vpt_pa)
+{
+	its_mask_encode(&cmd->raw_cmd[3], vpt_pa >> 16, 50, 16);
+}
+
+static void its_encode_vpt_size(struct its_cmd_block *cmd, u8 vpt_size)
+{
+	its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0);
+}
+
 static inline void its_fixup_cmd(struct its_cmd_block *cmd)
 {
 	/* Let's fixup BE commands */
@@ -358,6 +479,40 @@ static struct its_collection *its_build_inv_cmd(struct its_cmd_block *cmd,
 	return col;
 }
 
+static struct its_collection *its_build_int_cmd(struct its_cmd_block *cmd,
+						struct its_cmd_desc *desc)
+{
+	struct its_collection *col;
+
+	col = dev_event_to_col(desc->its_int_cmd.dev,
+			       desc->its_int_cmd.event_id);
+
+	its_encode_cmd(cmd, GITS_CMD_INT);
+	its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
+	its_encode_event_id(cmd, desc->its_int_cmd.event_id);
+
+	its_fixup_cmd(cmd);
+
+	return col;
+}
+
+static struct its_collection *its_build_clear_cmd(struct its_cmd_block *cmd,
+						  struct its_cmd_desc *desc)
+{
+	struct its_collection *col;
+
+	col = dev_event_to_col(desc->its_clear_cmd.dev,
+			       desc->its_clear_cmd.event_id);
+
+	its_encode_cmd(cmd, GITS_CMD_CLEAR);
+	its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
+	its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
+
+	its_fixup_cmd(cmd);
+
+	return col;
+}
+
 static struct its_collection *its_build_invall_cmd(struct its_cmd_block *cmd,
 						   struct its_cmd_desc *desc)
 {
@@ -369,6 +524,94 @@ static struct its_collection *its_build_invall_cmd(struct its_cmd_block *cmd,
 	return NULL;
 }
 
+static struct its_vpe *its_build_vinvall_cmd(struct its_cmd_block *cmd,
+					     struct its_cmd_desc *desc)
+{
+	its_encode_cmd(cmd, GITS_CMD_VINVALL);
+	its_encode_vpeid(cmd, desc->its_vinvall_cmd.vpe->vpe_id);
+
+	its_fixup_cmd(cmd);
+
+	return desc->its_vinvall_cmd.vpe;
+}
+
+static struct its_vpe *its_build_vmapp_cmd(struct its_cmd_block *cmd,
+					   struct its_cmd_desc *desc)
+{
+	unsigned long vpt_addr;
+
+	vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page));
+
+	its_encode_cmd(cmd, GITS_CMD_VMAPP);
+	its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id);
+	its_encode_valid(cmd, desc->its_vmapp_cmd.valid);
+	its_encode_target(cmd, desc->its_vmapp_cmd.col->target_address);
+	its_encode_vpt_addr(cmd, vpt_addr);
+	its_encode_vpt_size(cmd, LPI_NRBITS - 1);
+
+	its_fixup_cmd(cmd);
+
+	return desc->its_vmapp_cmd.vpe;
+}
+
+static struct its_vpe *its_build_vmapti_cmd(struct its_cmd_block *cmd,
+					    struct its_cmd_desc *desc)
+{
+	u32 db;
+
+	if (desc->its_vmapti_cmd.db_enabled)
+		db = desc->its_vmapti_cmd.vpe->vpe_db_lpi;
+	else
+		db = 1023;
+
+	its_encode_cmd(cmd, GITS_CMD_VMAPTI);
+	its_encode_devid(cmd, desc->its_vmapti_cmd.dev->device_id);
+	its_encode_vpeid(cmd, desc->its_vmapti_cmd.vpe->vpe_id);
+	its_encode_event_id(cmd, desc->its_vmapti_cmd.event_id);
+	its_encode_db_phys_id(cmd, db);
+	its_encode_virt_id(cmd, desc->its_vmapti_cmd.virt_id);
+
+	its_fixup_cmd(cmd);
+
+	return desc->its_vmapti_cmd.vpe;
+}
+
+static struct its_vpe *its_build_vmovi_cmd(struct its_cmd_block *cmd,
+					   struct its_cmd_desc *desc)
+{
+	u32 db;
+
+	if (desc->its_vmovi_cmd.db_enabled)
+		db = desc->its_vmovi_cmd.vpe->vpe_db_lpi;
+	else
+		db = 1023;
+
+	its_encode_cmd(cmd, GITS_CMD_VMOVI);
+	its_encode_devid(cmd, desc->its_vmovi_cmd.dev->device_id);
+	its_encode_vpeid(cmd, desc->its_vmovi_cmd.vpe->vpe_id);
+	its_encode_event_id(cmd, desc->its_vmovi_cmd.event_id);
+	its_encode_db_phys_id(cmd, db);
+	its_encode_db_valid(cmd, true);
+
+	its_fixup_cmd(cmd);
+
+	return desc->its_vmovi_cmd.vpe;
+}
+
+static struct its_vpe *its_build_vmovp_cmd(struct its_cmd_block *cmd,
+					   struct its_cmd_desc *desc)
+{
+	its_encode_cmd(cmd, GITS_CMD_VMOVP);
+	its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num);
+	its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list);
+	its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id);
+	its_encode_target(cmd, desc->its_vmovp_cmd.col->target_address);
+
+	its_fixup_cmd(cmd);
+
+	return desc->its_vmovp_cmd.vpe;
+}
+
 static u64 its_cmd_ptr_to_offset(struct its_node *its,
 				 struct its_cmd_block *ptr)
 {
@@ -453,7 +696,13 @@ static void its_wait_for_range_completion(struct its_node *its,
 
 	while (1) {
 		rd_idx = readl_relaxed(its->base + GITS_CREADR);
-		if (rd_idx >= to_idx || rd_idx < from_idx)
+
+		/* Direct case */
+		if (from_idx < to_idx && rd_idx >= to_idx)
+			break;
+
+		/* Wrapped case */
+		if (from_idx >= to_idx && rd_idx >= to_idx && rd_idx < from_idx)
 			break;
 
 		count--;
@@ -466,42 +715,84 @@ static void its_wait_for_range_completion(struct its_node *its,
 	}
 }
 
-static void its_send_single_command(struct its_node *its,
-				    its_cmd_builder_t builder,
-				    struct its_cmd_desc *desc)
+/* Warning, macro hell follows */
+#define BUILD_SINGLE_CMD_FUNC(name, buildtype, synctype, buildfn)	\
+void name(struct its_node *its,						\
+	  buildtype builder,						\
+	  struct its_cmd_desc *desc)					\
+{									\
+	struct its_cmd_block *cmd, *sync_cmd, *next_cmd;		\
+	synctype *sync_obj;						\
+	unsigned long flags;						\
+									\
+	raw_spin_lock_irqsave(&its->lock, flags);			\
+									\
+	cmd = its_allocate_entry(its);					\
+	if (!cmd) {		/* We're soooooo screewed... */		\
+		raw_spin_unlock_irqrestore(&its->lock, flags);		\
+		return;							\
+	}								\
+	sync_obj = builder(cmd, desc);					\
+	its_flush_cmd(its, cmd);					\
+									\
+	if (sync_obj) {							\
+		sync_cmd = its_allocate_entry(its);			\
+		if (!sync_cmd)						\
+			goto post;					\
+									\
+		buildfn(sync_cmd, sync_obj);				\
+		its_flush_cmd(its, sync_cmd);				\
+	}								\
+									\
+post:									\
+	next_cmd = its_post_commands(its);				\
+	raw_spin_unlock_irqrestore(&its->lock, flags);			\
+									\
+	its_wait_for_range_completion(its, cmd, next_cmd);		\
+}
+
+static void its_build_sync_cmd(struct its_cmd_block *sync_cmd,
+			       struct its_collection *sync_col)
+{
+	its_encode_cmd(sync_cmd, GITS_CMD_SYNC);
+	its_encode_target(sync_cmd, sync_col->target_address);
+
+	its_fixup_cmd(sync_cmd);
+}
+
+static BUILD_SINGLE_CMD_FUNC(its_send_single_command, its_cmd_builder_t,
+			     struct its_collection, its_build_sync_cmd)
+
+static void its_build_vsync_cmd(struct its_cmd_block *sync_cmd,
+				struct its_vpe *sync_vpe)
+{
+	its_encode_cmd(sync_cmd, GITS_CMD_VSYNC);
+	its_encode_vpeid(sync_cmd, sync_vpe->vpe_id);
+
+	its_fixup_cmd(sync_cmd);
+}
+
+static BUILD_SINGLE_CMD_FUNC(its_send_single_vcommand, its_cmd_vbuilder_t,
+			     struct its_vpe, its_build_vsync_cmd)
+
+static void its_send_int(struct its_device *dev, u32 event_id)
 {
-	struct its_cmd_block *cmd, *sync_cmd, *next_cmd;
-	struct its_collection *sync_col;
-	unsigned long flags;
+	struct its_cmd_desc desc;
 
-	raw_spin_lock_irqsave(&its->lock, flags);
+	desc.its_int_cmd.dev = dev;
+	desc.its_int_cmd.event_id = event_id;
 
-	cmd = its_allocate_entry(its);
-	if (!cmd) {		/* We're soooooo screewed... */
-		pr_err_ratelimited("ITS can't allocate, dropping command\n");
-		raw_spin_unlock_irqrestore(&its->lock, flags);
-		return;
-	}
-	sync_col = builder(cmd, desc);
-	its_flush_cmd(its, cmd);
+	its_send_single_command(dev->its, its_build_int_cmd, &desc);
+}
 
-	if (sync_col) {
-		sync_cmd = its_allocate_entry(its);
-		if (!sync_cmd) {
-			pr_err_ratelimited("ITS can't SYNC, skipping\n");
-			goto post;
-		}
-		its_encode_cmd(sync_cmd, GITS_CMD_SYNC);
-		its_encode_target(sync_cmd, sync_col->target_address);
-		its_fixup_cmd(sync_cmd);
-		its_flush_cmd(its, sync_cmd);
-	}
+static void its_send_clear(struct its_device *dev, u32 event_id)
+{
+	struct its_cmd_desc desc;
 
-post:
-	next_cmd = its_post_commands(its);
-	raw_spin_unlock_irqrestore(&its->lock, flags);
+	desc.its_clear_cmd.dev = dev;
+	desc.its_clear_cmd.event_id = event_id;
 
-	its_wait_for_range_completion(its, cmd, next_cmd);
+	its_send_single_command(dev->its, its_build_clear_cmd, &desc);
 }
 
 static void its_send_inv(struct its_device *dev, u32 event_id)
@@ -577,6 +868,106 @@ static void its_send_invall(struct its_node *its, struct its_collection *col)
 	its_send_single_command(its, its_build_invall_cmd, &desc);
 }
 
+static void its_send_vmapti(struct its_device *dev, u32 id)
+{
+	struct its_vlpi_map *map = &dev->event_map.vlpi_maps[id];
+	struct its_cmd_desc desc;
+
+	desc.its_vmapti_cmd.vpe = map->vpe;
+	desc.its_vmapti_cmd.dev = dev;
+	desc.its_vmapti_cmd.virt_id = map->vintid;
+	desc.its_vmapti_cmd.event_id = id;
+	desc.its_vmapti_cmd.db_enabled = map->db_enabled;
+
+	its_send_single_vcommand(dev->its, its_build_vmapti_cmd, &desc);
+}
+
+static void its_send_vmovi(struct its_device *dev, u32 id)
+{
+	struct its_vlpi_map *map = &dev->event_map.vlpi_maps[id];
+	struct its_cmd_desc desc;
+
+	desc.its_vmovi_cmd.vpe = map->vpe;
+	desc.its_vmovi_cmd.dev = dev;
+	desc.its_vmovi_cmd.event_id = id;
+	desc.its_vmovi_cmd.db_enabled = map->db_enabled;
+
+	its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc);
+}
+
+static void its_send_vmapp(struct its_vpe *vpe, bool valid)
+{
+	struct its_cmd_desc desc;
+	struct its_node *its;
+
+	desc.its_vmapp_cmd.vpe = vpe;
+	desc.its_vmapp_cmd.valid = valid;
+
+	list_for_each_entry(its, &its_nodes, entry) {
+		if (!its->is_v4)
+			continue;
+
+		desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx];
+		its_send_single_vcommand(its, its_build_vmapp_cmd, &desc);
+	}
+}
+
+static void its_send_vmovp(struct its_vpe *vpe)
+{
+	struct its_cmd_desc desc;
+	struct its_node *its;
+	unsigned long flags;
+	int col_id = vpe->col_idx;
+
+	desc.its_vmovp_cmd.vpe = vpe;
+	desc.its_vmovp_cmd.its_list = (u16)its_list_map;
+
+	if (!its_list_map) {
+		its = list_first_entry(&its_nodes, struct its_node, entry);
+		desc.its_vmovp_cmd.seq_num = 0;
+		desc.its_vmovp_cmd.col = &its->collections[col_id];
+		its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
+		return;
+	}
+
+	/*
+	 * Yet another marvel of the architecture. If using the
+	 * its_list "feature", we need to make sure that all ITSs
+	 * receive all VMOVP commands in the same order. The only way
+	 * to guarantee this is to make vmovp a serialization point.
+	 *
+	 * Wall <-- Head.
+	 */
+	raw_spin_lock_irqsave(&vmovp_lock, flags);
+
+	desc.its_vmovp_cmd.seq_num = vmovp_seq_num++;
+
+	/* Emit VMOVPs */
+	list_for_each_entry(its, &its_nodes, entry) {
+		if (!its->is_v4)
+			continue;
+
+		desc.its_vmovp_cmd.col = &its->collections[col_id];
+		its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
+	}
+
+	raw_spin_unlock_irqrestore(&vmovp_lock, flags);
+}
+
+static void its_send_vinvall(struct its_vpe *vpe)
+{
+	struct its_cmd_desc desc;
+	struct its_node *its;
+
+	desc.its_vinvall_cmd.vpe = vpe;
+
+	list_for_each_entry(its, &its_nodes, entry) {
+		if (!its->is_v4)
+			continue;
+		its_send_single_vcommand(its, its_build_vinvall_cmd, &desc);
+	}
+}
+
 /*
  * irqchip functions - assumes MSI, mostly.
  */
@@ -587,17 +978,26 @@ static inline u32 its_get_event_id(struct irq_data *d)
 	return d->hwirq - its_dev->event_map.lpi_base;
 }
 
-static void lpi_set_config(struct irq_data *d, bool enable)
+static void lpi_write_config(struct irq_data *d, u8 clr, u8 set)
 {
-	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
-	irq_hw_number_t hwirq = d->hwirq;
-	u32 id = its_get_event_id(d);
-	u8 *cfg = page_address(gic_rdists->prop_page) + hwirq - 8192;
+	irq_hw_number_t hwirq;
+	struct page *prop_page;
+	u8 *cfg;
 
-	if (enable)
-		*cfg |= LPI_PROP_ENABLED;
-	else
-		*cfg &= ~LPI_PROP_ENABLED;
+	if (irqd_is_forwarded_to_vcpu(d)) {
+		struct its_device *its_dev = irq_data_get_irq_chip_data(d);
+		u32 event = its_get_event_id(d);
+
+		prop_page = its_dev->event_map.vm->vprop_page;
+		hwirq = its_dev->event_map.vlpi_maps[event].vintid;
+	} else {
+		prop_page = gic_rdists->prop_page;
+		hwirq = d->hwirq;
+	}
+
+	cfg = page_address(prop_page) + hwirq - 8192;
+	*cfg &= ~clr;
+	*cfg |= set | LPI_PROP_GROUP1;
 
 	/*
 	 * Make the above write visible to the redistributors.
@@ -608,17 +1008,53 @@ static void lpi_set_config(struct irq_data *d, bool enable)
 		gic_flush_dcache_to_poc(cfg, sizeof(*cfg));
 	else
 		dsb(ishst);
-	its_send_inv(its_dev, id);
+}
+
+static void lpi_update_config(struct irq_data *d, u8 clr, u8 set)
+{
+	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
+
+	lpi_write_config(d, clr, set);
+	its_send_inv(its_dev, its_get_event_id(d));
+}
+
+static void its_vlpi_set_doorbell(struct irq_data *d, bool enable)
+{
+	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
+	u32 event = its_get_event_id(d);
+
+	if (its_dev->event_map.vlpi_maps[event].db_enabled == enable)
+		return;
+
+	its_dev->event_map.vlpi_maps[event].db_enabled = enable;
+
+	/*
+	 * More fun with the architecture:
+	 *
+	 * Ideally, we'd issue a VMAPTI to set the doorbell to its LPI
+	 * value or to 1023, depending on the enable bit. But that
+	 * would be issueing a mapping for an /existing/ DevID+EventID
+	 * pair, which is UNPREDICTABLE. Instead, let's issue a VMOVI
+	 * to the /same/ vPE, using this opportunity to adjust the
+	 * doorbell. Mouahahahaha. We loves it, Precious.
+	 */
+	its_send_vmovi(its_dev, event);
 }
 
 static void its_mask_irq(struct irq_data *d)
 {
-	lpi_set_config(d, false);
+	if (irqd_is_forwarded_to_vcpu(d))
+		its_vlpi_set_doorbell(d, false);
+
+	lpi_update_config(d, LPI_PROP_ENABLED, 0);
 }
 
 static void its_unmask_irq(struct irq_data *d)
 {
-	lpi_set_config(d, true);
+	if (irqd_is_forwarded_to_vcpu(d))
+		its_vlpi_set_doorbell(d, true);
+
+	lpi_update_config(d, 0, LPI_PROP_ENABLED);
 }
 
 static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
@@ -630,6 +1066,10 @@ static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
 	struct its_collection *target_col;
 	u32 id = its_get_event_id(d);
 
+	/* A forwarded interrupt should use irq_set_vcpu_affinity */
+	if (irqd_is_forwarded_to_vcpu(d))
+		return -EINVAL;
+
        /* lpi cannot be routed to a redistributor that is on a foreign node */
 	if (its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {
 		if (its_dev->its->numa_node >= 0) {
@@ -649,6 +1089,7 @@ static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
 		target_col = &its_dev->its->collections[cpu];
 		its_send_movi(its_dev, target_col, id);
 		its_dev->event_map.col_map[id] = cpu;
+		irq_data_update_effective_affinity(d, cpumask_of(cpu));
 	}
 
 	return IRQ_SET_MASK_OK_DONE;
@@ -670,6 +1111,179 @@ static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
 	iommu_dma_map_msi_msg(d->irq, msg);
 }
 
+static int its_irq_set_irqchip_state(struct irq_data *d,
+				     enum irqchip_irq_state which,
+				     bool state)
+{
+	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
+	u32 event = its_get_event_id(d);
+
+	if (which != IRQCHIP_STATE_PENDING)
+		return -EINVAL;
+
+	if (state)
+		its_send_int(its_dev, event);
+	else
+		its_send_clear(its_dev, event);
+
+	return 0;
+}
+
+static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info)
+{
+	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
+	u32 event = its_get_event_id(d);
+	int ret = 0;
+
+	if (!info->map)
+		return -EINVAL;
+
+	mutex_lock(&its_dev->event_map.vlpi_lock);
+
+	if (!its_dev->event_map.vm) {
+		struct its_vlpi_map *maps;
+
+		maps = kzalloc(sizeof(*maps) * its_dev->event_map.nr_lpis,
+			       GFP_KERNEL);
+		if (!maps) {
+			ret = -ENOMEM;
+			goto out;
+		}
+
+		its_dev->event_map.vm = info->map->vm;
+		its_dev->event_map.vlpi_maps = maps;
+	} else if (its_dev->event_map.vm != info->map->vm) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	/* Get our private copy of the mapping information */
+	its_dev->event_map.vlpi_maps[event] = *info->map;
+
+	if (irqd_is_forwarded_to_vcpu(d)) {
+		/* Already mapped, move it around */
+		its_send_vmovi(its_dev, event);
+	} else {
+		/* Drop the physical mapping */
+		its_send_discard(its_dev, event);
+
+		/* and install the virtual one */
+		its_send_vmapti(its_dev, event);
+		irqd_set_forwarded_to_vcpu(d);
+
+		/* Increment the number of VLPIs */
+		its_dev->event_map.nr_vlpis++;
+	}
+
+out:
+	mutex_unlock(&its_dev->event_map.vlpi_lock);
+	return ret;
+}
+
+static int its_vlpi_get(struct irq_data *d, struct its_cmd_info *info)
+{
+	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
+	u32 event = its_get_event_id(d);
+	int ret = 0;
+
+	mutex_lock(&its_dev->event_map.vlpi_lock);
+
+	if (!its_dev->event_map.vm ||
+	    !its_dev->event_map.vlpi_maps[event].vm) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	/* Copy our mapping information to the incoming request */
+	*info->map = its_dev->event_map.vlpi_maps[event];
+
+out:
+	mutex_unlock(&its_dev->event_map.vlpi_lock);
+	return ret;
+}
+
+static int its_vlpi_unmap(struct irq_data *d)
+{
+	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
+	u32 event = its_get_event_id(d);
+	int ret = 0;
+
+	mutex_lock(&its_dev->event_map.vlpi_lock);
+
+	if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	/* Drop the virtual mapping */
+	its_send_discard(its_dev, event);
+
+	/* and restore the physical one */
+	irqd_clr_forwarded_to_vcpu(d);
+	its_send_mapti(its_dev, d->hwirq, event);
+	lpi_update_config(d, 0xff, (LPI_PROP_DEFAULT_PRIO |
+				    LPI_PROP_ENABLED |
+				    LPI_PROP_GROUP1));
+
+	/*
+	 * Drop the refcount and make the device available again if
+	 * this was the last VLPI.
+	 */
+	if (!--its_dev->event_map.nr_vlpis) {
+		its_dev->event_map.vm = NULL;
+		kfree(its_dev->event_map.vlpi_maps);
+	}
+
+out:
+	mutex_unlock(&its_dev->event_map.vlpi_lock);
+	return ret;
+}
+
+static int its_vlpi_prop_update(struct irq_data *d, struct its_cmd_info *info)
+{
+	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
+
+	if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d))
+		return -EINVAL;
+
+	if (info->cmd_type == PROP_UPDATE_AND_INV_VLPI)
+		lpi_update_config(d, 0xff, info->config);
+	else
+		lpi_write_config(d, 0xff, info->config);
+	its_vlpi_set_doorbell(d, !!(info->config & LPI_PROP_ENABLED));
+
+	return 0;
+}
+
+static int its_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
+{
+	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
+	struct its_cmd_info *info = vcpu_info;
+
+	/* Need a v4 ITS */
+	if (!its_dev->its->is_v4)
+		return -EINVAL;
+
+	/* Unmap request? */
+	if (!info)
+		return its_vlpi_unmap(d);
+
+	switch (info->cmd_type) {
+	case MAP_VLPI:
+		return its_vlpi_map(d, info);
+
+	case GET_VLPI:
+		return its_vlpi_get(d, info);
+
+	case PROP_UPDATE_VLPI:
+	case PROP_UPDATE_AND_INV_VLPI:
+		return its_vlpi_prop_update(d, info);
+
+	default:
+		return -EINVAL;
+	}
+}
+
 static struct irq_chip its_irq_chip = {
 	.name			= "ITS",
 	.irq_mask		= its_mask_irq,
@@ -677,6 +1291,8 @@ static struct irq_chip its_irq_chip = {
 	.irq_eoi		= irq_chip_eoi_parent,
 	.irq_set_affinity	= its_set_affinity,
 	.irq_compose_msi_msg	= its_irq_compose_msi_msg,
+	.irq_set_irqchip_state	= its_irq_set_irqchip_state,
+	.irq_set_vcpu_affinity	= its_irq_set_vcpu_affinity,
 };
 
 /*
@@ -695,7 +1311,6 @@ static struct irq_chip its_irq_chip = {
 
 static unsigned long *lpi_bitmap;
 static u32 lpi_chunks;
-static u32 lpi_id_bits;
 static DEFINE_SPINLOCK(lpi_lock);
 
 static int its_lpi_to_chunk(int lpi)
@@ -766,16 +1381,15 @@ out:
 	return bitmap;
 }
 
-static void its_lpi_free(struct event_lpi_map *map)
+static void its_lpi_free_chunks(unsigned long *bitmap, int base, int nr_ids)
 {
-	int base = map->lpi_base;
-	int nr_ids = map->nr_lpis;
 	int lpi;
 
 	spin_lock(&lpi_lock);
 
 	for (lpi = base; lpi < (base + nr_ids); lpi += IRQS_PER_CHUNK) {
 		int chunk = its_lpi_to_chunk(lpi);
+
 		BUG_ON(chunk > lpi_chunks);
 		if (test_bit(chunk, lpi_bitmap)) {
 			clear_bit(chunk, lpi_bitmap);
@@ -786,28 +1400,40 @@ static void its_lpi_free(struct event_lpi_map *map)
 
 	spin_unlock(&lpi_lock);
 
-	kfree(map->lpi_map);
-	kfree(map->col_map);
+	kfree(bitmap);
 }
 
-/*
- * We allocate memory for PROPBASE to cover 2 ^ lpi_id_bits LPIs to
- * deal with (one configuration byte per interrupt). PENDBASE has to
- * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI).
- */
-#define LPI_NRBITS		lpi_id_bits
-#define LPI_PROPBASE_SZ		ALIGN(BIT(LPI_NRBITS), SZ_64K)
-#define LPI_PENDBASE_SZ		ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K)
+static struct page *its_allocate_prop_table(gfp_t gfp_flags)
+{
+	struct page *prop_page;
 
-#define LPI_PROP_DEFAULT_PRIO	0xa0
+	prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ));
+	if (!prop_page)
+		return NULL;
+
+	/* Priority 0xa0, Group-1, disabled */
+	memset(page_address(prop_page),
+	       LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1,
+	       LPI_PROPBASE_SZ);
+
+	/* Make sure the GIC will observe the written configuration */
+	gic_flush_dcache_to_poc(page_address(prop_page), LPI_PROPBASE_SZ);
+
+	return prop_page;
+}
+
+static void its_free_prop_table(struct page *prop_page)
+{
+	free_pages((unsigned long)page_address(prop_page),
+		   get_order(LPI_PROPBASE_SZ));
+}
 
 static int __init its_alloc_lpi_tables(void)
 {
 	phys_addr_t paddr;
 
 	lpi_id_bits = min_t(u32, gic_rdists->id_bits, ITS_MAX_LPI_NRBITS);
-	gic_rdists->prop_page = alloc_pages(GFP_NOWAIT,
-					   get_order(LPI_PROPBASE_SZ));
+	gic_rdists->prop_page = its_allocate_prop_table(GFP_NOWAIT);
 	if (!gic_rdists->prop_page) {
 		pr_err("Failed to allocate PROPBASE\n");
 		return -ENOMEM;
@@ -816,14 +1442,6 @@ static int __init its_alloc_lpi_tables(void)
 	paddr = page_to_phys(gic_rdists->prop_page);
 	pr_info("GIC: using LPI property table @%pa\n", &paddr);
 
-	/* Priority 0xa0, Group-1, disabled */
-	memset(page_address(gic_rdists->prop_page),
-	       LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1,
-	       LPI_PROPBASE_SZ);
-
-	/* Make sure the GIC will observe the written configuration */
-	gic_flush_dcache_to_poc(page_address(gic_rdists->prop_page), LPI_PROPBASE_SZ);
-
 	return its_lpi_init(lpi_id_bits);
 }
 
@@ -962,10 +1580,13 @@ retry_baser:
 	return 0;
 }
 
-static bool its_parse_baser_device(struct its_node *its, struct its_baser *baser,
-				   u32 psz, u32 *order)
+static bool its_parse_indirect_baser(struct its_node *its,
+				     struct its_baser *baser,
+				     u32 psz, u32 *order)
 {
-	u64 esz = GITS_BASER_ENTRY_SIZE(its_read_baser(its, baser));
+	u64 tmp = its_read_baser(its, baser);
+	u64 type = GITS_BASER_TYPE(tmp);
+	u64 esz = GITS_BASER_ENTRY_SIZE(tmp);
 	u64 val = GITS_BASER_InnerShareable | GITS_BASER_RaWaWb;
 	u32 ids = its->device_ids;
 	u32 new_order = *order;
@@ -1004,8 +1625,9 @@ static bool its_parse_baser_device(struct its_node *its, struct its_baser *baser
 	if (new_order >= MAX_ORDER) {
 		new_order = MAX_ORDER - 1;
 		ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / (int)esz);
-		pr_warn("ITS@%pa: Device Table too large, reduce ids %u->%u\n",
-			&its->phys_base, its->device_ids, ids);
+		pr_warn("ITS@%pa: %s Table too large, reduce ids %u->%u\n",
+			&its->phys_base, its_base_type_string[type],
+			its->device_ids, ids);
 	}
 
 	*order = new_order;
@@ -1053,11 +1675,16 @@ static int its_alloc_tables(struct its_node *its)
 		u32 order = get_order(psz);
 		bool indirect = false;
 
-		if (type == GITS_BASER_TYPE_NONE)
+		switch (type) {
+		case GITS_BASER_TYPE_NONE:
 			continue;
 
-		if (type == GITS_BASER_TYPE_DEVICE)
-			indirect = its_parse_baser_device(its, baser, psz, &order);
+		case GITS_BASER_TYPE_DEVICE:
+		case GITS_BASER_TYPE_VCPU:
+			indirect = its_parse_indirect_baser(its, baser,
+							    psz, &order);
+			break;
+		}
 
 		err = its_setup_baser(its, baser, cache, shr, psz, order, indirect);
 		if (err < 0) {
@@ -1084,6 +1711,30 @@ static int its_alloc_collections(struct its_node *its)
 	return 0;
 }
 
+static struct page *its_allocate_pending_table(gfp_t gfp_flags)
+{
+	struct page *pend_page;
+	/*
+	 * The pending pages have to be at least 64kB aligned,
+	 * hence the 'max(LPI_PENDBASE_SZ, SZ_64K)' below.
+	 */
+	pend_page = alloc_pages(gfp_flags | __GFP_ZERO,
+				get_order(max_t(u32, LPI_PENDBASE_SZ, SZ_64K)));
+	if (!pend_page)
+		return NULL;
+
+	/* Make sure the GIC will observe the zero-ed page */
+	gic_flush_dcache_to_poc(page_address(pend_page), LPI_PENDBASE_SZ);
+
+	return pend_page;
+}
+
+static void its_free_pending_table(struct page *pt)
+{
+	free_pages((unsigned long)page_address(pt),
+		   get_order(max_t(u32, LPI_PENDBASE_SZ, SZ_64K)));
+}
+
 static void its_cpu_init_lpis(void)
 {
 	void __iomem *rbase = gic_data_rdist_rd_base();
@@ -1094,21 +1745,14 @@ static void its_cpu_init_lpis(void)
 	pend_page = gic_data_rdist()->pend_page;
 	if (!pend_page) {
 		phys_addr_t paddr;
-		/*
-		 * The pending pages have to be at least 64kB aligned,
-		 * hence the 'max(LPI_PENDBASE_SZ, SZ_64K)' below.
-		 */
-		pend_page = alloc_pages(GFP_NOWAIT | __GFP_ZERO,
-					get_order(max_t(u32, LPI_PENDBASE_SZ, SZ_64K)));
+
+		pend_page = its_allocate_pending_table(GFP_NOWAIT);
 		if (!pend_page) {
 			pr_err("Failed to allocate PENDBASE for CPU%d\n",
 			       smp_processor_id());
 			return;
 		}
 
-		/* Make sure the GIC will observe the zero-ed page */
-		gic_flush_dcache_to_poc(page_address(pend_page), LPI_PENDBASE_SZ);
-
 		paddr = page_to_phys(pend_page);
 		pr_info("CPU%d: using LPI pending table @%pa\n",
 			smp_processor_id(), &paddr);
@@ -1259,26 +1903,19 @@ static struct its_baser *its_get_baser(struct its_node *its, u32 type)
 	return NULL;
 }
 
-static bool its_alloc_device_table(struct its_node *its, u32 dev_id)
+static bool its_alloc_table_entry(struct its_baser *baser, u32 id)
 {
-	struct its_baser *baser;
 	struct page *page;
 	u32 esz, idx;
 	__le64 *table;
 
-	baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE);
-
-	/* Don't allow device id that exceeds ITS hardware limit */
-	if (!baser)
-		return (ilog2(dev_id) < its->device_ids);
-
 	/* Don't allow device id that exceeds single, flat table limit */
 	esz = GITS_BASER_ENTRY_SIZE(baser->val);
 	if (!(baser->val & GITS_BASER_INDIRECT))
-		return (dev_id < (PAGE_ORDER_TO_SIZE(baser->order) / esz));
+		return (id < (PAGE_ORDER_TO_SIZE(baser->order) / esz));
 
 	/* Compute 1st level table index & check if that exceeds table limit */
-	idx = dev_id >> ilog2(baser->psz / esz);
+	idx = id >> ilog2(baser->psz / esz);
 	if (idx >= (PAGE_ORDER_TO_SIZE(baser->order) / GITS_LVL1_ENTRY_SIZE))
 		return false;
 
@@ -1307,11 +1944,52 @@ static bool its_alloc_device_table(struct its_node *its, u32 dev_id)
 	return true;
 }
 
+static bool its_alloc_device_table(struct its_node *its, u32 dev_id)
+{
+	struct its_baser *baser;
+
+	baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE);
+
+	/* Don't allow device id that exceeds ITS hardware limit */
+	if (!baser)
+		return (ilog2(dev_id) < its->device_ids);
+
+	return its_alloc_table_entry(baser, dev_id);
+}
+
+static bool its_alloc_vpe_table(u32 vpe_id)
+{
+	struct its_node *its;
+
+	/*
+	 * Make sure the L2 tables are allocated on *all* v4 ITSs. We
+	 * could try and only do it on ITSs corresponding to devices
+	 * that have interrupts targeted at this VPE, but the
+	 * complexity becomes crazy (and you have tons of memory
+	 * anyway, right?).
+	 */
+	list_for_each_entry(its, &its_nodes, entry) {
+		struct its_baser *baser;
+
+		if (!its->is_v4)
+			continue;
+
+		baser = its_get_baser(its, GITS_BASER_TYPE_VCPU);
+		if (!baser)
+			return false;
+
+		if (!its_alloc_table_entry(baser, vpe_id))
+			return false;
+	}
+
+	return true;
+}
+
 static struct its_device *its_create_device(struct its_node *its, u32 dev_id,
-					    int nvecs)
+					    int nvecs, bool alloc_lpis)
 {
 	struct its_device *dev;
-	unsigned long *lpi_map;
+	unsigned long *lpi_map = NULL;
 	unsigned long flags;
 	u16 *col_map = NULL;
 	void *itt;
@@ -1333,11 +2011,18 @@ static struct its_device *its_create_device(struct its_node *its, u32 dev_id,
 	sz = nr_ites * its->ite_size;
 	sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1;
 	itt = kzalloc(sz, GFP_KERNEL);
-	lpi_map = its_lpi_alloc_chunks(nvecs, &lpi_base, &nr_lpis);
-	if (lpi_map)
-		col_map = kzalloc(sizeof(*col_map) * nr_lpis, GFP_KERNEL);
+	if (alloc_lpis) {
+		lpi_map = its_lpi_alloc_chunks(nvecs, &lpi_base, &nr_lpis);
+		if (lpi_map)
+			col_map = kzalloc(sizeof(*col_map) * nr_lpis,
+					  GFP_KERNEL);
+	} else {
+		col_map = kzalloc(sizeof(*col_map) * nr_ites, GFP_KERNEL);
+		nr_lpis = 0;
+		lpi_base = 0;
+	}
 
-	if (!dev || !itt || !lpi_map || !col_map) {
+	if (!dev || !itt ||  !col_map || (!lpi_map && alloc_lpis)) {
 		kfree(dev);
 		kfree(itt);
 		kfree(lpi_map);
@@ -1354,6 +2039,7 @@ static struct its_device *its_create_device(struct its_node *its, u32 dev_id,
 	dev->event_map.col_map = col_map;
 	dev->event_map.lpi_base = lpi_base;
 	dev->event_map.nr_lpis = nr_lpis;
+	mutex_init(&dev->event_map.vlpi_lock);
 	dev->device_id = dev_id;
 	INIT_LIST_HEAD(&dev->entry);
 
@@ -1412,6 +2098,16 @@ static int its_msi_prepare(struct irq_domain *domain, struct device *dev,
 	msi_info = msi_get_domain_info(domain);
 	its = msi_info->data;
 
+	if (!gic_rdists->has_direct_lpi &&
+	    vpe_proxy.dev &&
+	    vpe_proxy.dev->its == its &&
+	    dev_id == vpe_proxy.dev->device_id) {
+		/* Bad luck. Get yourself a better implementation */
+		WARN_ONCE(1, "DevId %x clashes with GICv4 VPE proxy device\n",
+			  dev_id);
+		return -EINVAL;
+	}
+
 	its_dev = its_find_device(its, dev_id);
 	if (its_dev) {
 		/*
@@ -1423,7 +2119,7 @@ static int its_msi_prepare(struct irq_domain *domain, struct device *dev,
 		goto out;
 	}
 
-	its_dev = its_create_device(its, dev_id, nvec);
+	its_dev = its_create_device(its, dev_id, nvec, true);
 	if (!its_dev)
 		return -ENOMEM;
 
@@ -1481,6 +2177,7 @@ static int its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
 
 		irq_domain_set_hwirq_and_chip(domain, virq + i,
 					      hwirq, &its_irq_chip, its_dev);
+		irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(virq + i)));
 		pr_debug("ID:%d pID:%d vID:%d\n",
 			 (int)(hwirq - its_dev->event_map.lpi_base),
 			 (int) hwirq, virq + i);
@@ -1495,13 +2192,16 @@ static void its_irq_domain_activate(struct irq_domain *domain,
 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
 	u32 event = its_get_event_id(d);
 	const struct cpumask *cpu_mask = cpu_online_mask;
+	int cpu;
 
 	/* get the cpu_mask of local node */
 	if (its_dev->its->numa_node >= 0)
 		cpu_mask = cpumask_of_node(its_dev->its->numa_node);
 
 	/* Bind the LPI to the first possible CPU */
-	its_dev->event_map.col_map[event] = cpumask_first(cpu_mask);
+	cpu = cpumask_first(cpu_mask);
+	its_dev->event_map.col_map[event] = cpu;
+	irq_data_update_effective_affinity(d, cpumask_of(cpu));
 
 	/* Map the GIC IRQ and event to the device */
 	its_send_mapti(its_dev, d->hwirq, event);
@@ -1539,7 +2239,10 @@ static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq,
 	/* If all interrupts have been freed, start mopping the floor */
 	if (bitmap_empty(its_dev->event_map.lpi_map,
 			 its_dev->event_map.nr_lpis)) {
-		its_lpi_free(&its_dev->event_map);
+		its_lpi_free_chunks(its_dev->event_map.lpi_map,
+				    its_dev->event_map.lpi_base,
+				    its_dev->event_map.nr_lpis);
+		kfree(its_dev->event_map.col_map);
 
 		/* Unmap device/itt */
 		its_send_mapd(its_dev, 0);
@@ -1556,6 +2259,451 @@ static const struct irq_domain_ops its_domain_ops = {
 	.deactivate		= its_irq_domain_deactivate,
 };
 
+/*
+ * This is insane.
+ *
+ * If a GICv4 doesn't implement Direct LPIs (which is extremely
+ * likely), the only way to perform an invalidate is to use a fake
+ * device to issue an INV command, implying that the LPI has first
+ * been mapped to some event on that device. Since this is not exactly
+ * cheap, we try to keep that mapping around as long as possible, and
+ * only issue an UNMAP if we're short on available slots.
+ *
+ * Broken by design(tm).
+ */
+static void its_vpe_db_proxy_unmap_locked(struct its_vpe *vpe)
+{
+	/* Already unmapped? */
+	if (vpe->vpe_proxy_event == -1)
+		return;
+
+	its_send_discard(vpe_proxy.dev, vpe->vpe_proxy_event);
+	vpe_proxy.vpes[vpe->vpe_proxy_event] = NULL;
+
+	/*
+	 * We don't track empty slots at all, so let's move the
+	 * next_victim pointer if we can quickly reuse that slot
+	 * instead of nuking an existing entry. Not clear that this is
+	 * always a win though, and this might just generate a ripple
+	 * effect... Let's just hope VPEs don't migrate too often.
+	 */
+	if (vpe_proxy.vpes[vpe_proxy.next_victim])
+		vpe_proxy.next_victim = vpe->vpe_proxy_event;
+
+	vpe->vpe_proxy_event = -1;
+}
+
+static void its_vpe_db_proxy_unmap(struct its_vpe *vpe)
+{
+	if (!gic_rdists->has_direct_lpi) {
+		unsigned long flags;
+
+		raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
+		its_vpe_db_proxy_unmap_locked(vpe);
+		raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
+	}
+}
+
+static void its_vpe_db_proxy_map_locked(struct its_vpe *vpe)
+{
+	/* Already mapped? */
+	if (vpe->vpe_proxy_event != -1)
+		return;
+
+	/* This slot was already allocated. Kick the other VPE out. */
+	if (vpe_proxy.vpes[vpe_proxy.next_victim])
+		its_vpe_db_proxy_unmap_locked(vpe_proxy.vpes[vpe_proxy.next_victim]);
+
+	/* Map the new VPE instead */
+	vpe_proxy.vpes[vpe_proxy.next_victim] = vpe;
+	vpe->vpe_proxy_event = vpe_proxy.next_victim;
+	vpe_proxy.next_victim = (vpe_proxy.next_victim + 1) % vpe_proxy.dev->nr_ites;
+
+	vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = vpe->col_idx;
+	its_send_mapti(vpe_proxy.dev, vpe->vpe_db_lpi, vpe->vpe_proxy_event);
+}
+
+static void its_vpe_db_proxy_move(struct its_vpe *vpe, int from, int to)
+{
+	unsigned long flags;
+	struct its_collection *target_col;
+
+	if (gic_rdists->has_direct_lpi) {
+		void __iomem *rdbase;
+
+		rdbase = per_cpu_ptr(gic_rdists->rdist, from)->rd_base;
+		gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
+		while (gic_read_lpir(rdbase + GICR_SYNCR) & 1)
+			cpu_relax();
+
+		return;
+	}
+
+	raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
+
+	its_vpe_db_proxy_map_locked(vpe);
+
+	target_col = &vpe_proxy.dev->its->collections[to];
+	its_send_movi(vpe_proxy.dev, target_col, vpe->vpe_proxy_event);
+	vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = to;
+
+	raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
+}
+
+static int its_vpe_set_affinity(struct irq_data *d,
+				const struct cpumask *mask_val,
+				bool force)
+{
+	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
+	int cpu = cpumask_first(mask_val);
+
+	/*
+	 * Changing affinity is mega expensive, so let's be as lazy as
+	 * we can and only do it if we really have to. Also, if mapped
+	 * into the proxy device, we need to move the doorbell
+	 * interrupt to its new location.
+	 */
+	if (vpe->col_idx != cpu) {
+		int from = vpe->col_idx;
+
+		vpe->col_idx = cpu;
+		its_send_vmovp(vpe);
+		its_vpe_db_proxy_move(vpe, from, cpu);
+	}
+
+	return IRQ_SET_MASK_OK_DONE;
+}
+
+static void its_vpe_schedule(struct its_vpe *vpe)
+{
+	void * __iomem vlpi_base = gic_data_rdist_vlpi_base();
+	u64 val;
+
+	/* Schedule the VPE */
+	val  = virt_to_phys(page_address(vpe->its_vm->vprop_page)) &
+		GENMASK_ULL(51, 12);
+	val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
+	val |= GICR_VPROPBASER_RaWb;
+	val |= GICR_VPROPBASER_InnerShareable;
+	gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
+
+	val  = virt_to_phys(page_address(vpe->vpt_page)) &
+		GENMASK_ULL(51, 16);
+	val |= GICR_VPENDBASER_RaWaWb;
+	val |= GICR_VPENDBASER_NonShareable;
+	/*
+	 * There is no good way of finding out if the pending table is
+	 * empty as we can race against the doorbell interrupt very
+	 * easily. So in the end, vpe->pending_last is only an
+	 * indication that the vcpu has something pending, not one
+	 * that the pending table is empty. A good implementation
+	 * would be able to read its coarse map pretty quickly anyway,
+	 * making this a tolerable issue.
+	 */
+	val |= GICR_VPENDBASER_PendingLast;
+	val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0;
+	val |= GICR_VPENDBASER_Valid;
+	gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
+}
+
+static void its_vpe_deschedule(struct its_vpe *vpe)
+{
+	void * __iomem vlpi_base = gic_data_rdist_vlpi_base();
+	u32 count = 1000000;	/* 1s! */
+	bool clean;
+	u64 val;
+
+	/* We're being scheduled out */
+	val = gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
+	val &= ~GICR_VPENDBASER_Valid;
+	gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
+
+	do {
+		val = gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
+		clean = !(val & GICR_VPENDBASER_Dirty);
+		if (!clean) {
+			count--;
+			cpu_relax();
+			udelay(1);
+		}
+	} while (!clean && count);
+
+	if (unlikely(!clean && !count)) {
+		pr_err_ratelimited("ITS virtual pending table not cleaning\n");
+		vpe->idai = false;
+		vpe->pending_last = true;
+	} else {
+		vpe->idai = !!(val & GICR_VPENDBASER_IDAI);
+		vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
+	}
+}
+
+static int its_vpe_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
+{
+	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
+	struct its_cmd_info *info = vcpu_info;
+
+	switch (info->cmd_type) {
+	case SCHEDULE_VPE:
+		its_vpe_schedule(vpe);
+		return 0;
+
+	case DESCHEDULE_VPE:
+		its_vpe_deschedule(vpe);
+		return 0;
+
+	case INVALL_VPE:
+		its_send_vinvall(vpe);
+		return 0;
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static void its_vpe_send_cmd(struct its_vpe *vpe,
+			     void (*cmd)(struct its_device *, u32))
+{
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
+
+	its_vpe_db_proxy_map_locked(vpe);
+	cmd(vpe_proxy.dev, vpe->vpe_proxy_event);
+
+	raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
+}
+
+static void its_vpe_send_inv(struct irq_data *d)
+{
+	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
+
+	if (gic_rdists->has_direct_lpi) {
+		void __iomem *rdbase;
+
+		rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;
+		gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_INVLPIR);
+		while (gic_read_lpir(rdbase + GICR_SYNCR) & 1)
+			cpu_relax();
+	} else {
+		its_vpe_send_cmd(vpe, its_send_inv);
+	}
+}
+
+static void its_vpe_mask_irq(struct irq_data *d)
+{
+	/*
+	 * We need to unmask the LPI, which is described by the parent
+	 * irq_data. Instead of calling into the parent (which won't
+	 * exactly do the right thing, let's simply use the
+	 * parent_data pointer. Yes, I'm naughty.
+	 */
+	lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
+	its_vpe_send_inv(d);
+}
+
+static void its_vpe_unmask_irq(struct irq_data *d)
+{
+	/* Same hack as above... */
+	lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
+	its_vpe_send_inv(d);
+}
+
+static int its_vpe_set_irqchip_state(struct irq_data *d,
+				     enum irqchip_irq_state which,
+				     bool state)
+{
+	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
+
+	if (which != IRQCHIP_STATE_PENDING)
+		return -EINVAL;
+
+	if (gic_rdists->has_direct_lpi) {
+		void __iomem *rdbase;
+
+		rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;
+		if (state) {
+			gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR);
+		} else {
+			gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
+			while (gic_read_lpir(rdbase + GICR_SYNCR) & 1)
+				cpu_relax();
+		}
+	} else {
+		if (state)
+			its_vpe_send_cmd(vpe, its_send_int);
+		else
+			its_vpe_send_cmd(vpe, its_send_clear);
+	}
+
+	return 0;
+}
+
+static struct irq_chip its_vpe_irq_chip = {
+	.name			= "GICv4-vpe",
+	.irq_mask		= its_vpe_mask_irq,
+	.irq_unmask		= its_vpe_unmask_irq,
+	.irq_eoi		= irq_chip_eoi_parent,
+	.irq_set_affinity	= its_vpe_set_affinity,
+	.irq_set_irqchip_state	= its_vpe_set_irqchip_state,
+	.irq_set_vcpu_affinity	= its_vpe_set_vcpu_affinity,
+};
+
+static int its_vpe_id_alloc(void)
+{
+	return ida_simple_get(&its_vpeid_ida, 0, 1 << 16, GFP_KERNEL);
+}
+
+static void its_vpe_id_free(u16 id)
+{
+	ida_simple_remove(&its_vpeid_ida, id);
+}
+
+static int its_vpe_init(struct its_vpe *vpe)
+{
+	struct page *vpt_page;
+	int vpe_id;
+
+	/* Allocate vpe_id */
+	vpe_id = its_vpe_id_alloc();
+	if (vpe_id < 0)
+		return vpe_id;
+
+	/* Allocate VPT */
+	vpt_page = its_allocate_pending_table(GFP_KERNEL);
+	if (!vpt_page) {
+		its_vpe_id_free(vpe_id);
+		return -ENOMEM;
+	}
+
+	if (!its_alloc_vpe_table(vpe_id)) {
+		its_vpe_id_free(vpe_id);
+		its_free_pending_table(vpe->vpt_page);
+		return -ENOMEM;
+	}
+
+	vpe->vpe_id = vpe_id;
+	vpe->vpt_page = vpt_page;
+	vpe->vpe_proxy_event = -1;
+
+	return 0;
+}
+
+static void its_vpe_teardown(struct its_vpe *vpe)
+{
+	its_vpe_db_proxy_unmap(vpe);
+	its_vpe_id_free(vpe->vpe_id);
+	its_free_pending_table(vpe->vpt_page);
+}
+
+static void its_vpe_irq_domain_free(struct irq_domain *domain,
+				    unsigned int virq,
+				    unsigned int nr_irqs)
+{
+	struct its_vm *vm = domain->host_data;
+	int i;
+
+	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
+
+	for (i = 0; i < nr_irqs; i++) {
+		struct irq_data *data = irq_domain_get_irq_data(domain,
+								virq + i);
+		struct its_vpe *vpe = irq_data_get_irq_chip_data(data);
+
+		BUG_ON(vm != vpe->its_vm);
+
+		clear_bit(data->hwirq, vm->db_bitmap);
+		its_vpe_teardown(vpe);
+		irq_domain_reset_irq_data(data);
+	}
+
+	if (bitmap_empty(vm->db_bitmap, vm->nr_db_lpis)) {
+		its_lpi_free_chunks(vm->db_bitmap, vm->db_lpi_base, vm->nr_db_lpis);
+		its_free_prop_table(vm->vprop_page);
+	}
+}
+
+static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
+				    unsigned int nr_irqs, void *args)
+{
+	struct its_vm *vm = args;
+	unsigned long *bitmap;
+	struct page *vprop_page;
+	int base, nr_ids, i, err = 0;
+
+	BUG_ON(!vm);
+
+	bitmap = its_lpi_alloc_chunks(nr_irqs, &base, &nr_ids);
+	if (!bitmap)
+		return -ENOMEM;
+
+	if (nr_ids < nr_irqs) {
+		its_lpi_free_chunks(bitmap, base, nr_ids);
+		return -ENOMEM;
+	}
+
+	vprop_page = its_allocate_prop_table(GFP_KERNEL);
+	if (!vprop_page) {
+		its_lpi_free_chunks(bitmap, base, nr_ids);
+		return -ENOMEM;
+	}
+
+	vm->db_bitmap = bitmap;
+	vm->db_lpi_base = base;
+	vm->nr_db_lpis = nr_ids;
+	vm->vprop_page = vprop_page;
+
+	for (i = 0; i < nr_irqs; i++) {
+		vm->vpes[i]->vpe_db_lpi = base + i;
+		err = its_vpe_init(vm->vpes[i]);
+		if (err)
+			break;
+		err = its_irq_gic_domain_alloc(domain, virq + i,
+					       vm->vpes[i]->vpe_db_lpi);
+		if (err)
+			break;
+		irq_domain_set_hwirq_and_chip(domain, virq + i, i,
+					      &its_vpe_irq_chip, vm->vpes[i]);
+		set_bit(i, bitmap);
+	}
+
+	if (err) {
+		if (i > 0)
+			its_vpe_irq_domain_free(domain, virq, i - 1);
+
+		its_lpi_free_chunks(bitmap, base, nr_ids);
+		its_free_prop_table(vprop_page);
+	}
+
+	return err;
+}
+
+static void its_vpe_irq_domain_activate(struct irq_domain *domain,
+					struct irq_data *d)
+{
+	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
+
+	/* Map the VPE to the first possible CPU */
+	vpe->col_idx = cpumask_first(cpu_online_mask);
+	its_send_vmapp(vpe, true);
+	its_send_vinvall(vpe);
+}
+
+static void its_vpe_irq_domain_deactivate(struct irq_domain *domain,
+					  struct irq_data *d)
+{
+	struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
+
+	its_send_vmapp(vpe, false);
+}
+
+static const struct irq_domain_ops its_vpe_domain_ops = {
+	.alloc			= its_vpe_irq_domain_alloc,
+	.free			= its_vpe_irq_domain_free,
+	.activate		= its_vpe_irq_domain_activate,
+	.deactivate		= its_vpe_irq_domain_deactivate,
+};
+
 static int its_force_quiescent(void __iomem *base)
 {
 	u32 count = 1000000;	/* 1s */
@@ -1571,7 +2719,7 @@ static int its_force_quiescent(void __iomem *base)
 		return 0;
 
 	/* Disable the generation of all interrupts to this ITS */
-	val &= ~GITS_CTLR_ENABLE;
+	val &= ~(GITS_CTLR_ENABLE | GITS_CTLR_ImDe);
 	writel_relaxed(val, base + GITS_CTLR);
 
 	/* Poll GITS_CTLR and wait until ITS becomes quiescent */
@@ -1672,13 +2820,92 @@ static int its_init_domain(struct fwnode_handle *handle, struct its_node *its)
 	return 0;
 }
 
+static int its_init_vpe_domain(void)
+{
+	struct its_node *its;
+	u32 devid;
+	int entries;
+
+	if (gic_rdists->has_direct_lpi) {
+		pr_info("ITS: Using DirectLPI for VPE invalidation\n");
+		return 0;
+	}
+
+	/* Any ITS will do, even if not v4 */
+	its = list_first_entry(&its_nodes, struct its_node, entry);
+
+	entries = roundup_pow_of_two(nr_cpu_ids);
+	vpe_proxy.vpes = kzalloc(sizeof(*vpe_proxy.vpes) * entries,
+				 GFP_KERNEL);
+	if (!vpe_proxy.vpes) {
+		pr_err("ITS: Can't allocate GICv4 proxy device array\n");
+		return -ENOMEM;
+	}
+
+	/* Use the last possible DevID */
+	devid = GENMASK(its->device_ids - 1, 0);
+	vpe_proxy.dev = its_create_device(its, devid, entries, false);
+	if (!vpe_proxy.dev) {
+		kfree(vpe_proxy.vpes);
+		pr_err("ITS: Can't allocate GICv4 proxy device\n");
+		return -ENOMEM;
+	}
+
+	BUG_ON(entries != vpe_proxy.dev->nr_ites);
+
+	raw_spin_lock_init(&vpe_proxy.lock);
+	vpe_proxy.next_victim = 0;
+	pr_info("ITS: Allocated DevID %x as GICv4 proxy device (%d slots)\n",
+		devid, vpe_proxy.dev->nr_ites);
+
+	return 0;
+}
+
+static int __init its_compute_its_list_map(struct resource *res,
+					   void __iomem *its_base)
+{
+	int its_number;
+	u32 ctlr;
+
+	/*
+	 * This is assumed to be done early enough that we're
+	 * guaranteed to be single-threaded, hence no
+	 * locking. Should this change, we should address
+	 * this.
+	 */
+	its_number = find_first_zero_bit(&its_list_map, ITS_LIST_MAX);
+	if (its_number >= ITS_LIST_MAX) {
+		pr_err("ITS@%pa: No ITSList entry available!\n",
+		       &res->start);
+		return -EINVAL;
+	}
+
+	ctlr = readl_relaxed(its_base + GITS_CTLR);
+	ctlr &= ~GITS_CTLR_ITS_NUMBER;
+	ctlr |= its_number << GITS_CTLR_ITS_NUMBER_SHIFT;
+	writel_relaxed(ctlr, its_base + GITS_CTLR);
+	ctlr = readl_relaxed(its_base + GITS_CTLR);
+	if ((ctlr & GITS_CTLR_ITS_NUMBER) != (its_number << GITS_CTLR_ITS_NUMBER_SHIFT)) {
+		its_number = ctlr & GITS_CTLR_ITS_NUMBER;
+		its_number >>= GITS_CTLR_ITS_NUMBER_SHIFT;
+	}
+
+	if (test_and_set_bit(its_number, &its_list_map)) {
+		pr_err("ITS@%pa: Duplicate ITSList entry %d\n",
+		       &res->start, its_number);
+		return -EINVAL;
+	}
+
+	return its_number;
+}
+
 static int __init its_probe_one(struct resource *res,
 				struct fwnode_handle *handle, int numa_node)
 {
 	struct its_node *its;
 	void __iomem *its_base;
-	u32 val;
-	u64 baser, tmp;
+	u32 val, ctlr;
+	u64 baser, tmp, typer;
 	int err;
 
 	its_base = ioremap(res->start, resource_size(res));
@@ -1711,9 +2938,24 @@ static int __init its_probe_one(struct resource *res,
 	raw_spin_lock_init(&its->lock);
 	INIT_LIST_HEAD(&its->entry);
 	INIT_LIST_HEAD(&its->its_device_list);
+	typer = gic_read_typer(its_base + GITS_TYPER);
 	its->base = its_base;
 	its->phys_base = res->start;
-	its->ite_size = ((gic_read_typer(its_base + GITS_TYPER) >> 4) & 0xf) + 1;
+	its->ite_size = GITS_TYPER_ITT_ENTRY_SIZE(typer);
+	its->is_v4 = !!(typer & GITS_TYPER_VLPIS);
+	if (its->is_v4) {
+		if (!(typer & GITS_TYPER_VMOVP)) {
+			err = its_compute_its_list_map(res, its_base);
+			if (err < 0)
+				goto out_free_its;
+
+			pr_info("ITS@%pa: Using ITS number %d\n",
+				&res->start, err);
+		} else {
+			pr_info("ITS@%pa: Single VMOVP capable\n", &res->start);
+		}
+	}
+
 	its->numa_node = numa_node;
 
 	its->cmd_base = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
@@ -1760,7 +3002,11 @@ static int __init its_probe_one(struct resource *res,
 	}
 
 	gits_write_cwriter(0, its->base + GITS_CWRITER);
-	writel_relaxed(GITS_CTLR_ENABLE, its->base + GITS_CTLR);
+	ctlr = readl_relaxed(its->base + GITS_CTLR);
+	ctlr |= GITS_CTLR_ENABLE;
+	if (its->is_v4)
+		ctlr |= GITS_CTLR_ImDe;
+	writel_relaxed(ctlr, its->base + GITS_CTLR);
 
 	err = its_init_domain(handle, its);
 	if (err)
@@ -1816,13 +3062,13 @@ static int __init its_of_probe(struct device_node *node)
 	for (np = of_find_matching_node(node, its_device_id); np;
 	     np = of_find_matching_node(np, its_device_id)) {
 		if (!of_property_read_bool(np, "msi-controller")) {
-			pr_warn("%s: no msi-controller property, ITS ignored\n",
-				np->full_name);
+			pr_warn("%pOF: no msi-controller property, ITS ignored\n",
+				np);
 			continue;
 		}
 
 		if (of_address_to_resource(np, 0, &res)) {
-			pr_warn("%s: no regs?\n", np->full_name);
+			pr_warn("%pOF: no regs?\n", np);
 			continue;
 		}
 
@@ -1835,7 +3081,7 @@ static int __init its_of_probe(struct device_node *node)
 
 #define ACPI_GICV3_ITS_MEM_SIZE (SZ_128K)
 
-#if defined(CONFIG_ACPI_NUMA) && (ACPI_CA_VERSION >= 0x20170531)
+#ifdef CONFIG_ACPI_NUMA
 struct its_srat_map {
 	/* numa node id */
 	u32	numa_node;
@@ -1843,7 +3089,7 @@ struct its_srat_map {
 	u32	its_id;
 };
 
-static struct its_srat_map its_srat_maps[MAX_NUMNODES] __initdata;
+static struct its_srat_map *its_srat_maps __initdata;
 static int its_in_srat __initdata;
 
 static int __init acpi_get_its_numa_node(u32 its_id)
@@ -1857,6 +3103,12 @@ static int __init acpi_get_its_numa_node(u32 its_id)
 	return NUMA_NO_NODE;
 }
 
+static int __init gic_acpi_match_srat_its(struct acpi_subtable_header *header,
+					  const unsigned long end)
+{
+	return 0;
+}
+
 static int __init gic_acpi_parse_srat_its(struct acpi_subtable_header *header,
 			 const unsigned long end)
 {
@@ -1873,12 +3125,6 @@ static int __init gic_acpi_parse_srat_its(struct acpi_subtable_header *header,
 		return -EINVAL;
 	}
 
-	if (its_in_srat >= MAX_NUMNODES) {
-		pr_err("SRAT: ITS affinity exceeding max count[%d]\n",
-				MAX_NUMNODES);
-		return -EINVAL;
-	}
-
 	node = acpi_map_pxm_to_node(its_affinity->proximity_domain);
 
 	if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) {
@@ -1897,14 +3143,37 @@ static int __init gic_acpi_parse_srat_its(struct acpi_subtable_header *header,
 
 static void __init acpi_table_parse_srat_its(void)
 {
+	int count;
+
+	count = acpi_table_parse_entries(ACPI_SIG_SRAT,
+			sizeof(struct acpi_table_srat),
+			ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
+			gic_acpi_match_srat_its, 0);
+	if (count <= 0)
+		return;
+
+	its_srat_maps = kmalloc(count * sizeof(struct its_srat_map),
+				GFP_KERNEL);
+	if (!its_srat_maps) {
+		pr_warn("SRAT: Failed to allocate memory for its_srat_maps!\n");
+		return;
+	}
+
 	acpi_table_parse_entries(ACPI_SIG_SRAT,
 			sizeof(struct acpi_table_srat),
 			ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
 			gic_acpi_parse_srat_its, 0);
 }
+
+/* free the its_srat_maps after ITS probing */
+static void __init acpi_its_srat_maps_free(void)
+{
+	kfree(its_srat_maps);
+}
 #else
 static void __init acpi_table_parse_srat_its(void)	{ }
 static int __init acpi_get_its_numa_node(u32 its_id) { return NUMA_NO_NODE; }
+static void __init acpi_its_srat_maps_free(void) { }
 #endif
 
 static int __init gic_acpi_parse_madt_its(struct acpi_subtable_header *header,
@@ -1951,6 +3220,7 @@ static void __init its_acpi_probe(void)
 	acpi_table_parse_srat_its();
 	acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
 			      gic_acpi_parse_madt_its, 0);
+	acpi_its_srat_maps_free();
 }
 #else
 static void __init its_acpi_probe(void) { }
@@ -1960,6 +3230,9 @@ int __init its_init(struct fwnode_handle *handle, struct rdists *rdists,
 		    struct irq_domain *parent_domain)
 {
 	struct device_node *of_node;
+	struct its_node *its;
+	bool has_v4 = false;
+	int err;
 
 	its_parent = parent_domain;
 	of_node = to_of_node(handle);
@@ -1974,5 +3247,20 @@ int __init its_init(struct fwnode_handle *handle, struct rdists *rdists,
 	}
 
 	gic_rdists = rdists;
-	return its_alloc_lpi_tables();
+	err = its_alloc_lpi_tables();
+	if (err)
+		return err;
+
+	list_for_each_entry(its, &its_nodes, entry)
+		has_v4 |= its->is_v4;
+
+	if (has_v4 & rdists->has_vlpis) {
+		if (its_init_vpe_domain() ||
+		    its_init_v4(parent_domain, &its_vpe_domain_ops)) {
+			rdists->has_vlpis = false;
+			pr_err("ITS: Disabling GICv4 support\n");
+		}
+	}
+
+	return 0;
 }
diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
index dbffb7ab6203..519149ec9053 100644
--- a/drivers/irqchip/irq-gic-v3.c
+++ b/drivers/irqchip/irq-gic-v3.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2013, 2014 ARM Limited, All Rights Reserved.
+ * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
  * Author: Marc Zyngier <marc.zyngier@arm.com>
  *
  * This program is free software; you can redistribute it and/or modify
@@ -353,6 +353,8 @@ static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs
 
 			if (static_key_true(&supports_deactivate))
 				gic_write_eoir(irqnr);
+			else
+				isb();
 
 			err = handle_domain_irq(gic_data.domain, irqnr, regs);
 			if (err) {
@@ -421,24 +423,14 @@ static void __init gic_dist_init(void)
 		gic_write_irouter(affinity, base + GICD_IROUTER + i * 8);
 }
 
-static int gic_populate_rdist(void)
+static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *))
 {
-	unsigned long mpidr = cpu_logical_map(smp_processor_id());
-	u64 typer;
-	u32 aff;
+	int ret = -ENODEV;
 	int i;
 
-	/*
-	 * Convert affinity to a 32bit value that can be matched to
-	 * GICR_TYPER bits [63:32].
-	 */
-	aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
-	       MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
-	       MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
-	       MPIDR_AFFINITY_LEVEL(mpidr, 0));
-
 	for (i = 0; i < gic_data.nr_redist_regions; i++) {
 		void __iomem *ptr = gic_data.redist_regions[i].redist_base;
+		u64 typer;
 		u32 reg;
 
 		reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK;
@@ -450,15 +442,9 @@ static int gic_populate_rdist(void)
 
 		do {
 			typer = gic_read_typer(ptr + GICR_TYPER);
-			if ((typer >> 32) == aff) {
-				u64 offset = ptr - gic_data.redist_regions[i].redist_base;
-				gic_data_rdist_rd_base() = ptr;
-				gic_data_rdist()->phys_base = gic_data.redist_regions[i].phys_base + offset;
-				pr_info("CPU%d: found redistributor %lx region %d:%pa\n",
-					smp_processor_id(), mpidr, i,
-					&gic_data_rdist()->phys_base);
+			ret = fn(gic_data.redist_regions + i, ptr);
+			if (!ret)
 				return 0;
-			}
 
 			if (gic_data.redist_regions[i].single_redist)
 				break;
@@ -473,12 +459,71 @@ static int gic_populate_rdist(void)
 		} while (!(typer & GICR_TYPER_LAST));
 	}
 
+	return ret ? -ENODEV : 0;
+}
+
+static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr)
+{
+	unsigned long mpidr = cpu_logical_map(smp_processor_id());
+	u64 typer;
+	u32 aff;
+
+	/*
+	 * Convert affinity to a 32bit value that can be matched to
+	 * GICR_TYPER bits [63:32].
+	 */
+	aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
+	       MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
+	       MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
+	       MPIDR_AFFINITY_LEVEL(mpidr, 0));
+
+	typer = gic_read_typer(ptr + GICR_TYPER);
+	if ((typer >> 32) == aff) {
+		u64 offset = ptr - region->redist_base;
+		gic_data_rdist_rd_base() = ptr;
+		gic_data_rdist()->phys_base = region->phys_base + offset;
+
+		pr_info("CPU%d: found redistributor %lx region %d:%pa\n",
+			smp_processor_id(), mpidr,
+			(int)(region - gic_data.redist_regions),
+			&gic_data_rdist()->phys_base);
+		return 0;
+	}
+
+	/* Try next one */
+	return 1;
+}
+
+static int gic_populate_rdist(void)
+{
+	if (gic_iterate_rdists(__gic_populate_rdist) == 0)
+		return 0;
+
 	/* We couldn't even deal with ourselves... */
 	WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n",
-	     smp_processor_id(), mpidr);
+	     smp_processor_id(),
+	     (unsigned long)cpu_logical_map(smp_processor_id()));
 	return -ENODEV;
 }
 
+static int __gic_update_vlpi_properties(struct redist_region *region,
+					void __iomem *ptr)
+{
+	u64 typer = gic_read_typer(ptr + GICR_TYPER);
+	gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS);
+	gic_data.rdists.has_direct_lpi &= !!(typer & GICR_TYPER_DirectLPIS);
+
+	return 1;
+}
+
+static void gic_update_vlpi_properties(void)
+{
+	gic_iterate_rdists(__gic_update_vlpi_properties);
+	pr_info("%sVLPI support, %sdirect LPI support\n",
+		!gic_data.rdists.has_vlpis ? "no " : "",
+		!gic_data.rdists.has_direct_lpi ? "no " : "");
+}
+
 static void gic_cpu_sys_reg_init(void)
 {
 	/*
@@ -640,11 +685,16 @@ static void gic_smp_init(void)
 static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
 			    bool force)
 {
-	unsigned int cpu = cpumask_any_and(mask_val, cpu_online_mask);
+	unsigned int cpu;
 	void __iomem *reg;
 	int enabled;
 	u64 val;
 
+	if (force)
+		cpu = cpumask_first(mask_val);
+	else
+		cpu = cpumask_any_and(mask_val, cpu_online_mask);
+
 	if (cpu >= nr_cpu_ids)
 		return -EINVAL;
 
@@ -670,6 +720,8 @@ static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
 	else
 		gic_dist_wait_for_rwp();
 
+	irq_data_update_effective_affinity(d, cpumask_of(cpu));
+
 	return IRQ_SET_MASK_OK_DONE;
 }
 #else
@@ -768,6 +820,7 @@ static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
 		irq_domain_set_info(d, irq, hw, chip, d->host_data,
 				    handle_fasteoi_irq, NULL, NULL);
 		irq_set_probe(irq);
+		irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq)));
 	}
 	/* LPIs */
 	if (hw >= 8192 && hw < GIC_ID_NR) {
@@ -831,8 +884,11 @@ static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
 	if (ret)
 		return ret;
 
-	for (i = 0; i < nr_irqs; i++)
-		gic_irq_domain_map(domain, virq + i, hwirq + i);
+	for (i = 0; i < nr_irqs; i++) {
+		ret = gic_irq_domain_map(domain, virq + i, hwirq + i);
+		if (ret)
+			return ret;
+	}
 
 	return 0;
 }
@@ -943,6 +999,8 @@ static int __init gic_init_bases(void __iomem *dist_base,
 	gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops,
 						 &gic_data);
 	gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist));
+	gic_data.rdists.has_vlpis = true;
+	gic_data.rdists.has_direct_lpi = true;
 
 	if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) {
 		err = -ENOMEM;
@@ -951,6 +1009,8 @@ static int __init gic_init_bases(void __iomem *dist_base,
 
 	set_handle_irq(gic_handle_irq);
 
+	gic_update_vlpi_properties();
+
 	if (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && gic_dist_supports_lpis())
 		its_init(handle, &gic_data.rdists, gic_data.domain);
 
@@ -1057,7 +1117,7 @@ static void __init gic_populate_ppi_partitions(struct device_node *gic_node)
 			if (WARN_ON(cpu == -1))
 				continue;
 
-			pr_cont("%s[%d] ", cpu_node->full_name, cpu);
+			pr_cont("%pOF[%d] ", cpu_node, cpu);
 
 			cpumask_set_cpu(cpu, &part->mask);
 		}
@@ -1112,6 +1172,7 @@ static void __init gic_of_setup_kvm_info(struct device_node *node)
 	if (!ret)
 		gic_v3_kvm_info.vcpu = r;
 
+	gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis;
 	gic_set_kvm_info(&gic_v3_kvm_info);
 }
 
@@ -1125,15 +1186,13 @@ static int __init gic_of_init(struct device_node *node, struct device_node *pare
 
 	dist_base = of_iomap(node, 0);
 	if (!dist_base) {
-		pr_err("%s: unable to map gic dist registers\n",
-			node->full_name);
+		pr_err("%pOF: unable to map gic dist registers\n", node);
 		return -ENXIO;
 	}
 
 	err = gic_validate_dist_version(dist_base);
 	if (err) {
-		pr_err("%s: no distributor detected, giving up\n",
-			node->full_name);
+		pr_err("%pOF: no distributor detected, giving up\n", node);
 		goto out_unmap_dist;
 	}
 
@@ -1153,8 +1212,7 @@ static int __init gic_of_init(struct device_node *node, struct device_node *pare
 		ret = of_address_to_resource(node, 1 + i, &res);
 		rdist_regs[i].redist_base = of_iomap(node, 1 + i);
 		if (ret || !rdist_regs[i].redist_base) {
-			pr_err("%s: couldn't map region %d\n",
-			       node->full_name, i);
+			pr_err("%pOF: couldn't map region %d\n", node, i);
 			err = -ENODEV;
 			goto out_unmap_rdist;
 		}
@@ -1408,6 +1466,7 @@ static void __init gic_acpi_setup_kvm_info(void)
 		vcpu->end = vcpu->start + ACPI_GICV2_VCPU_MEM_SIZE - 1;
 	}
 
+	gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis;
 	gic_set_kvm_info(&gic_v3_kvm_info);
 }
 
diff --git a/drivers/irqchip/irq-gic-v4.c b/drivers/irqchip/irq-gic-v4.c
new file mode 100644
index 000000000000..2370e6d9e603
--- /dev/null
+++ b/drivers/irqchip/irq-gic-v4.c
@@ -0,0 +1,225 @@
+/*
+ * Copyright (C) 2016,2017 ARM Limited, All Rights Reserved.
+ * Author: Marc Zyngier <marc.zyngier@arm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/irqdomain.h>
+#include <linux/msi.h>
+#include <linux/sched.h>
+
+#include <linux/irqchip/arm-gic-v4.h>
+
+/*
+ * WARNING: The blurb below assumes that you understand the
+ * intricacies of GICv3, GICv4, and how a guest's view of a GICv3 gets
+ * translated into GICv4 commands. So it effectively targets at most
+ * two individuals. You know who you are.
+ *
+ * The core GICv4 code is designed to *avoid* exposing too much of the
+ * core GIC code (that would in turn leak into the hypervisor code),
+ * and instead provide a hypervisor agnostic interface to the HW (of
+ * course, the astute reader will quickly realize that hypervisor
+ * agnostic actually means KVM-specific - what were you thinking?).
+ *
+ * In order to achieve a modicum of isolation, we try to hide most of
+ * the GICv4 "stuff" behind normal irqchip operations:
+ *
+ * - Any guest-visible VLPI is backed by a Linux interrupt (and a
+ *   physical LPI which gets unmapped when the guest maps the
+ *   VLPI). This allows the same DevID/EventID pair to be either
+ *   mapped to the LPI (host) or the VLPI (guest). Note that this is
+ *   exclusive, and you cannot have both.
+ *
+ * - Enabling/disabling a VLPI is done by issuing mask/unmask calls.
+ *
+ * - Guest INT/CLEAR commands are implemented through
+ *   irq_set_irqchip_state().
+ *
+ * - The *bizarre* stuff (mapping/unmapping an interrupt to a VLPI, or
+ *   issuing an INV after changing a priority) gets shoved into the
+ *   irq_set_vcpu_affinity() method. While this is quite horrible
+ *   (let's face it, this is the irqchip version of an ioctl), it
+ *   confines the crap to a single location. And map/unmap really is
+ *   about setting the affinity of a VLPI to a vcpu, so only INV is
+ *   majorly out of place. So there.
+ *
+ * A number of commands are simply not provided by this interface, as
+ * they do not make direct sense. For example, MAPD is purely local to
+ * the virtual ITS (because it references a virtual device, and the
+ * physical ITS is still very much in charge of the physical
+ * device). Same goes for things like MAPC (the physical ITS deals
+ * with the actual vPE affinity, and not the braindead concept of
+ * collection). SYNC is not provided either, as each and every command
+ * is followed by a VSYNC. This could be relaxed in the future, should
+ * this be seen as a bottleneck (yes, this means *never*).
+ *
+ * But handling VLPIs is only one side of the job of the GICv4
+ * code. The other (darker) side is to take care of the doorbell
+ * interrupts which are delivered when a VLPI targeting a non-running
+ * vcpu is being made pending.
+ *
+ * The choice made here is that each vcpu (VPE in old northern GICv4
+ * dialect) gets a single doorbell LPI, no matter how many interrupts
+ * are targeting it. This has a nice property, which is that the
+ * interrupt becomes a handle for the VPE, and that the hypervisor
+ * code can manipulate it through the normal interrupt API:
+ *
+ * - VMs (or rather the VM abstraction that matters to the GIC)
+ *   contain an irq domain where each interrupt maps to a VPE. In
+ *   turn, this domain sits on top of the normal LPI allocator, and a
+ *   specially crafted irq_chip implementation.
+ *
+ * - mask/unmask do what is expected on the doorbell interrupt.
+ *
+ * - irq_set_affinity is used to move a VPE from one redistributor to
+ *   another.
+ *
+ * - irq_set_vcpu_affinity once again gets hijacked for the purpose of
+ *   creating a new sub-API, namely scheduling/descheduling a VPE
+ *   (which involves programming GICR_V{PROP,PEND}BASER) and
+ *   performing INVALL operations.
+ */
+
+static struct irq_domain *gic_domain;
+static const struct irq_domain_ops *vpe_domain_ops;
+
+int its_alloc_vcpu_irqs(struct its_vm *vm)
+{
+	int vpe_base_irq, i;
+
+	vm->fwnode = irq_domain_alloc_named_id_fwnode("GICv4-vpe",
+						      task_pid_nr(current));
+	if (!vm->fwnode)
+		goto err;
+
+	vm->domain = irq_domain_create_hierarchy(gic_domain, 0, vm->nr_vpes,
+						 vm->fwnode, vpe_domain_ops,
+						 vm);
+	if (!vm->domain)
+		goto err;
+
+	for (i = 0; i < vm->nr_vpes; i++) {
+		vm->vpes[i]->its_vm = vm;
+		vm->vpes[i]->idai = true;
+	}
+
+	vpe_base_irq = __irq_domain_alloc_irqs(vm->domain, -1, vm->nr_vpes,
+					       NUMA_NO_NODE, vm,
+					       false, NULL);
+	if (vpe_base_irq <= 0)
+		goto err;
+
+	for (i = 0; i < vm->nr_vpes; i++)
+		vm->vpes[i]->irq = vpe_base_irq + i;
+
+	return 0;
+
+err:
+	if (vm->domain)
+		irq_domain_remove(vm->domain);
+	if (vm->fwnode)
+		irq_domain_free_fwnode(vm->fwnode);
+
+	return -ENOMEM;
+}
+
+void its_free_vcpu_irqs(struct its_vm *vm)
+{
+	irq_domain_free_irqs(vm->vpes[0]->irq, vm->nr_vpes);
+	irq_domain_remove(vm->domain);
+	irq_domain_free_fwnode(vm->fwnode);
+}
+
+static int its_send_vpe_cmd(struct its_vpe *vpe, struct its_cmd_info *info)
+{
+	return irq_set_vcpu_affinity(vpe->irq, info);
+}
+
+int its_schedule_vpe(struct its_vpe *vpe, bool on)
+{
+	struct its_cmd_info info;
+
+	WARN_ON(preemptible());
+
+	info.cmd_type = on ? SCHEDULE_VPE : DESCHEDULE_VPE;
+
+	return its_send_vpe_cmd(vpe, &info);
+}
+
+int its_invall_vpe(struct its_vpe *vpe)
+{
+	struct its_cmd_info info = {
+		.cmd_type = INVALL_VPE,
+	};
+
+	return its_send_vpe_cmd(vpe, &info);
+}
+
+int its_map_vlpi(int irq, struct its_vlpi_map *map)
+{
+	struct its_cmd_info info = {
+		.cmd_type = MAP_VLPI,
+		.map      = map,
+	};
+
+	/*
+	 * The host will never see that interrupt firing again, so it
+	 * is vital that we don't do any lazy masking.
+	 */
+	irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY);
+
+	return irq_set_vcpu_affinity(irq, &info);
+}
+
+int its_get_vlpi(int irq, struct its_vlpi_map *map)
+{
+	struct its_cmd_info info = {
+		.cmd_type = GET_VLPI,
+		.map      = map,
+	};
+
+	return irq_set_vcpu_affinity(irq, &info);
+}
+
+int its_unmap_vlpi(int irq)
+{
+	irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY);
+	return irq_set_vcpu_affinity(irq, NULL);
+}
+
+int its_prop_update_vlpi(int irq, u8 config, bool inv)
+{
+	struct its_cmd_info info = {
+		.cmd_type = inv ? PROP_UPDATE_AND_INV_VLPI : PROP_UPDATE_VLPI,
+		.config   = config,
+	};
+
+	return irq_set_vcpu_affinity(irq, &info);
+}
+
+int its_init_v4(struct irq_domain *domain, const struct irq_domain_ops *ops)
+{
+	if (domain) {
+		pr_info("ITS: Enabling GICv4 support\n");
+		gic_domain = domain;
+		vpe_domain_ops = ops;
+		return 0;
+	}
+
+	pr_err("ITS: No GICv4 VPE domain allocated\n");
+	return -ENODEV;
+}
diff --git a/drivers/irqchip/irq-gic.c b/drivers/irqchip/irq-gic.c
index 1b1df4f770bd..651d726e8b12 100644
--- a/drivers/irqchip/irq-gic.c
+++ b/drivers/irqchip/irq-gic.c
@@ -344,6 +344,8 @@ static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
 	writel_relaxed(val | bit, reg);
 	gic_unlock_irqrestore(flags);
 
+	irq_data_update_effective_affinity(d, cpumask_of(cpu));
+
 	return IRQ_SET_MASK_OK_DONE;
 }
 #endif
@@ -361,6 +363,7 @@ static void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)
 		if (likely(irqnr > 15 && irqnr < 1020)) {
 			if (static_key_true(&supports_deactivate))
 				writel_relaxed(irqstat, cpu_base + GIC_CPU_EOI);
+			isb();
 			handle_domain_irq(gic->domain, irqnr, regs);
 			continue;
 		}
@@ -401,16 +404,18 @@ static void gic_handle_cascade_irq(struct irq_desc *desc)
 		goto out;
 
 	cascade_irq = irq_find_mapping(chip_data->domain, gic_irq);
-	if (unlikely(gic_irq < 32 || gic_irq > 1020))
+	if (unlikely(gic_irq < 32 || gic_irq > 1020)) {
 		handle_bad_irq(desc);
-	else
+	} else {
+		isb();
 		generic_handle_irq(cascade_irq);
+	}
 
  out:
 	chained_irq_exit(chip, desc);
 }
 
-static struct irq_chip gic_chip = {
+static const struct irq_chip gic_chip = {
 	.irq_mask		= gic_mask_irq,
 	.irq_unmask		= gic_unmask_irq,
 	.irq_eoi		= gic_eoi_irq,
@@ -966,6 +971,7 @@ static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
 		irq_domain_set_info(d, irq, hw, &gic->chip, d->host_data,
 				    handle_fasteoi_irq, NULL, NULL);
 		irq_set_probe(irq);
+		irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq)));
 	}
 	return 0;
 }
@@ -1027,8 +1033,11 @@ static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
 	if (ret)
 		return ret;
 
-	for (i = 0; i < nr_irqs; i++)
-		gic_irq_domain_map(domain, virq + i, hwirq + i);
+	for (i = 0; i < nr_irqs; i++) {
+		ret = gic_irq_domain_map(domain, virq + i, hwirq + i);
+		if (ret)
+			return ret;
+	}
 
 	return 0;
 }
diff --git a/drivers/irqchip/irq-hip04.c b/drivers/irqchip/irq-hip04.c
index c1b4ee955dbe..5b4fd2f4e5f8 100644
--- a/drivers/irqchip/irq-hip04.c
+++ b/drivers/irqchip/irq-hip04.c
@@ -165,6 +165,8 @@ static int hip04_irq_set_affinity(struct irq_data *d,
 	writel_relaxed(val | bit, reg);
 	raw_spin_unlock(&irq_controller_lock);
 
+	irq_data_update_effective_affinity(d, cpumask_of(cpu));
+
 	return IRQ_SET_MASK_OK;
 }
 #endif
@@ -312,6 +314,7 @@ static int hip04_irq_domain_map(struct irq_domain *d, unsigned int irq,
 		irq_set_chip_and_handler(irq, &hip04_irq_chip,
 					 handle_fasteoi_irq);
 		irq_set_probe(irq);
+		irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq)));
 	}
 	irq_set_chip_data(irq, d->host_data);
 	return 0;
diff --git a/drivers/irqchip/irq-imx-gpcv2.c b/drivers/irqchip/irq-imx-gpcv2.c
index bb36f572e322..675eda5ff2b8 100644
--- a/drivers/irqchip/irq-imx-gpcv2.c
+++ b/drivers/irqchip/irq-imx-gpcv2.c
@@ -214,13 +214,13 @@ static int __init imx_gpcv2_irqchip_init(struct device_node *node,
 	int i;
 
 	if (!parent) {
-		pr_err("%s: no parent, giving up\n", node->full_name);
+		pr_err("%pOF: no parent, giving up\n", node);
 		return -ENODEV;
 	}
 
 	parent_domain = irq_find_host(parent);
 	if (!parent_domain) {
-		pr_err("%s: unable to get parent domain\n", node->full_name);
+		pr_err("%pOF: unable to get parent domain\n", node);
 		return -ENXIO;
 	}
 
diff --git a/drivers/irqchip/irq-lpc32xx.c b/drivers/irqchip/irq-lpc32xx.c
index 1034aeb2e98a..a48357d369b5 100644
--- a/drivers/irqchip/irq-lpc32xx.c
+++ b/drivers/irqchip/irq-lpc32xx.c
@@ -191,7 +191,7 @@ static int __init lpc32xx_of_ic_init(struct device_node *node,
 
 	irqc->base = of_iomap(node, 0);
 	if (!irqc->base) {
-		pr_err("%s: unable to map registers\n", node->full_name);
+		pr_err("%pOF: unable to map registers\n", node);
 		kfree(irqc);
 		return -EINVAL;
 	}
diff --git a/drivers/irqchip/irq-ls-scfg-msi.c b/drivers/irqchip/irq-ls-scfg-msi.c
index 02cca74cab94..119f4ef0d421 100644
--- a/drivers/irqchip/irq-ls-scfg-msi.c
+++ b/drivers/irqchip/irq-ls-scfg-msi.c
@@ -17,13 +17,32 @@
 #include <linux/irq.h>
 #include <linux/irqchip/chained_irq.h>
 #include <linux/irqdomain.h>
+#include <linux/of_irq.h>
 #include <linux/of_pci.h>
 #include <linux/of_platform.h>
 #include <linux/spinlock.h>
 
-#define MSI_MAX_IRQS	32
-#define MSI_IBS_SHIFT	3
-#define MSIR		4
+#define MSI_IRQS_PER_MSIR	32
+#define MSI_MSIR_OFFSET		4
+
+#define MSI_LS1043V1_1_IRQS_PER_MSIR	8
+#define MSI_LS1043V1_1_MSIR_OFFSET	0x10
+
+struct ls_scfg_msi_cfg {
+	u32 ibs_shift; /* Shift of interrupt bit select */
+	u32 msir_irqs; /* The irq number per MSIR */
+	u32 msir_base; /* The base address of MSIR */
+};
+
+struct ls_scfg_msir {
+	struct ls_scfg_msi *msi_data;
+	unsigned int index;
+	unsigned int gic_irq;
+	unsigned int bit_start;
+	unsigned int bit_end;
+	unsigned int srs; /* Shared interrupt register select */
+	void __iomem *reg;
+};
 
 struct ls_scfg_msi {
 	spinlock_t		lock;
@@ -32,8 +51,11 @@ struct ls_scfg_msi {
 	struct irq_domain	*msi_domain;
 	void __iomem		*regs;
 	phys_addr_t		msiir_addr;
-	int			irq;
-	DECLARE_BITMAP(used, MSI_MAX_IRQS);
+	struct ls_scfg_msi_cfg	*cfg;
+	u32			msir_num;
+	struct ls_scfg_msir	*msir;
+	u32			irqs_num;
+	unsigned long		*used;
 };
 
 static struct irq_chip ls_scfg_msi_irq_chip = {
@@ -49,19 +71,56 @@ static struct msi_domain_info ls_scfg_msi_domain_info = {
 	.chip	= &ls_scfg_msi_irq_chip,
 };
 
+static int msi_affinity_flag = 1;
+
+static int __init early_parse_ls_scfg_msi(char *p)
+{
+	if (p && strncmp(p, "no-affinity", 11) == 0)
+		msi_affinity_flag = 0;
+	else
+		msi_affinity_flag = 1;
+
+	return 0;
+}
+early_param("lsmsi", early_parse_ls_scfg_msi);
+
 static void ls_scfg_msi_compose_msg(struct irq_data *data, struct msi_msg *msg)
 {
 	struct ls_scfg_msi *msi_data = irq_data_get_irq_chip_data(data);
 
 	msg->address_hi = upper_32_bits(msi_data->msiir_addr);
 	msg->address_lo = lower_32_bits(msi_data->msiir_addr);
-	msg->data = data->hwirq << MSI_IBS_SHIFT;
+	msg->data = data->hwirq;
+
+	if (msi_affinity_flag)
+		msg->data |= cpumask_first(data->common->affinity);
 }
 
 static int ls_scfg_msi_set_affinity(struct irq_data *irq_data,
 				    const struct cpumask *mask, bool force)
 {
-	return -EINVAL;
+	struct ls_scfg_msi *msi_data = irq_data_get_irq_chip_data(irq_data);
+	u32 cpu;
+
+	if (!msi_affinity_flag)
+		return -EINVAL;
+
+	if (!force)
+		cpu = cpumask_any_and(mask, cpu_online_mask);
+	else
+		cpu = cpumask_first(mask);
+
+	if (cpu >= msi_data->msir_num)
+		return -EINVAL;
+
+	if (msi_data->msir[cpu].gic_irq <= 0) {
+		pr_warn("cannot bind the irq to cpu%d\n", cpu);
+		return -EINVAL;
+	}
+
+	cpumask_copy(irq_data->common->affinity, mask);
+
+	return IRQ_SET_MASK_OK;
 }
 
 static struct irq_chip ls_scfg_msi_parent_chip = {
@@ -81,8 +140,8 @@ static int ls_scfg_msi_domain_irq_alloc(struct irq_domain *domain,
 	WARN_ON(nr_irqs != 1);
 
 	spin_lock(&msi_data->lock);
-	pos = find_first_zero_bit(msi_data->used, MSI_MAX_IRQS);
-	if (pos < MSI_MAX_IRQS)
+	pos = find_first_zero_bit(msi_data->used, msi_data->irqs_num);
+	if (pos < msi_data->irqs_num)
 		__set_bit(pos, msi_data->used);
 	else
 		err = -ENOSPC;
@@ -106,7 +165,7 @@ static void ls_scfg_msi_domain_irq_free(struct irq_domain *domain,
 	int pos;
 
 	pos = d->hwirq;
-	if (pos < 0 || pos >= MSI_MAX_IRQS) {
+	if (pos < 0 || pos >= msi_data->irqs_num) {
 		pr_err("failed to teardown msi. Invalid hwirq %d\n", pos);
 		return;
 	}
@@ -123,15 +182,22 @@ static const struct irq_domain_ops ls_scfg_msi_domain_ops = {
 
 static void ls_scfg_msi_irq_handler(struct irq_desc *desc)
 {
-	struct ls_scfg_msi *msi_data = irq_desc_get_handler_data(desc);
+	struct ls_scfg_msir *msir = irq_desc_get_handler_data(desc);
+	struct ls_scfg_msi *msi_data = msir->msi_data;
 	unsigned long val;
-	int pos, virq;
+	int pos, size, virq, hwirq;
 
 	chained_irq_enter(irq_desc_get_chip(desc), desc);
 
-	val = ioread32be(msi_data->regs + MSIR);
-	for_each_set_bit(pos, &val, MSI_MAX_IRQS) {
-		virq = irq_find_mapping(msi_data->parent, (31 - pos));
+	val = ioread32be(msir->reg);
+
+	pos = msir->bit_start;
+	size = msir->bit_end + 1;
+
+	for_each_set_bit_from(pos, &val, size) {
+		hwirq = ((msir->bit_end - pos) << msi_data->cfg->ibs_shift) |
+			msir->srs;
+		virq = irq_find_mapping(msi_data->parent, hwirq);
 		if (virq)
 			generic_handle_irq(virq);
 	}
@@ -143,7 +209,7 @@ static int ls_scfg_msi_domains_init(struct ls_scfg_msi *msi_data)
 {
 	/* Initialize MSI domain parent */
 	msi_data->parent = irq_domain_add_linear(NULL,
-						 MSI_MAX_IRQS,
+						 msi_data->irqs_num,
 						 &ls_scfg_msi_domain_ops,
 						 msi_data);
 	if (!msi_data->parent) {
@@ -164,16 +230,117 @@ static int ls_scfg_msi_domains_init(struct ls_scfg_msi *msi_data)
 	return 0;
 }
 
+static int ls_scfg_msi_setup_hwirq(struct ls_scfg_msi *msi_data, int index)
+{
+	struct ls_scfg_msir *msir;
+	int virq, i, hwirq;
+
+	virq = platform_get_irq(msi_data->pdev, index);
+	if (virq <= 0)
+		return -ENODEV;
+
+	msir = &msi_data->msir[index];
+	msir->index = index;
+	msir->msi_data = msi_data;
+	msir->gic_irq = virq;
+	msir->reg = msi_data->regs + msi_data->cfg->msir_base + 4 * index;
+
+	if (msi_data->cfg->msir_irqs == MSI_LS1043V1_1_IRQS_PER_MSIR) {
+		msir->bit_start = 32 - ((msir->index + 1) *
+				  MSI_LS1043V1_1_IRQS_PER_MSIR);
+		msir->bit_end = msir->bit_start +
+				MSI_LS1043V1_1_IRQS_PER_MSIR - 1;
+	} else {
+		msir->bit_start = 0;
+		msir->bit_end = msi_data->cfg->msir_irqs - 1;
+	}
+
+	irq_set_chained_handler_and_data(msir->gic_irq,
+					 ls_scfg_msi_irq_handler,
+					 msir);
+
+	if (msi_affinity_flag) {
+		/* Associate MSIR interrupt to the cpu */
+		irq_set_affinity(msir->gic_irq, get_cpu_mask(index));
+		msir->srs = 0; /* This value is determined by the CPU */
+	} else
+		msir->srs = index;
+
+	/* Release the hwirqs corresponding to this MSIR */
+	if (!msi_affinity_flag || msir->index == 0) {
+		for (i = 0; i < msi_data->cfg->msir_irqs; i++) {
+			hwirq = i << msi_data->cfg->ibs_shift | msir->index;
+			bitmap_clear(msi_data->used, hwirq, 1);
+		}
+	}
+
+	return 0;
+}
+
+static int ls_scfg_msi_teardown_hwirq(struct ls_scfg_msir *msir)
+{
+	struct ls_scfg_msi *msi_data = msir->msi_data;
+	int i, hwirq;
+
+	if (msir->gic_irq > 0)
+		irq_set_chained_handler_and_data(msir->gic_irq, NULL, NULL);
+
+	for (i = 0; i < msi_data->cfg->msir_irqs; i++) {
+		hwirq = i << msi_data->cfg->ibs_shift | msir->index;
+		bitmap_set(msi_data->used, hwirq, 1);
+	}
+
+	return 0;
+}
+
+static struct ls_scfg_msi_cfg ls1021_msi_cfg = {
+	.ibs_shift = 3,
+	.msir_irqs = MSI_IRQS_PER_MSIR,
+	.msir_base = MSI_MSIR_OFFSET,
+};
+
+static struct ls_scfg_msi_cfg ls1046_msi_cfg = {
+	.ibs_shift = 2,
+	.msir_irqs = MSI_IRQS_PER_MSIR,
+	.msir_base = MSI_MSIR_OFFSET,
+};
+
+static struct ls_scfg_msi_cfg ls1043_v1_1_msi_cfg = {
+	.ibs_shift = 2,
+	.msir_irqs = MSI_LS1043V1_1_IRQS_PER_MSIR,
+	.msir_base = MSI_LS1043V1_1_MSIR_OFFSET,
+};
+
+static const struct of_device_id ls_scfg_msi_id[] = {
+	/* The following two misspelled compatibles are obsolete */
+	{ .compatible = "fsl,1s1021a-msi", .data = &ls1021_msi_cfg},
+	{ .compatible = "fsl,1s1043a-msi", .data = &ls1021_msi_cfg},
+
+	{ .compatible = "fsl,ls1021a-msi", .data = &ls1021_msi_cfg },
+	{ .compatible = "fsl,ls1043a-msi", .data = &ls1021_msi_cfg },
+	{ .compatible = "fsl,ls1043a-v1.1-msi", .data = &ls1043_v1_1_msi_cfg },
+	{ .compatible = "fsl,ls1046a-msi", .data = &ls1046_msi_cfg },
+	{},
+};
+MODULE_DEVICE_TABLE(of, ls_scfg_msi_id);
+
 static int ls_scfg_msi_probe(struct platform_device *pdev)
 {
+	const struct of_device_id *match;
 	struct ls_scfg_msi *msi_data;
 	struct resource *res;
-	int ret;
+	int i, ret;
+
+	match = of_match_device(ls_scfg_msi_id, &pdev->dev);
+	if (!match)
+		return -ENODEV;
 
 	msi_data = devm_kzalloc(&pdev->dev, sizeof(*msi_data), GFP_KERNEL);
 	if (!msi_data)
 		return -ENOMEM;
 
+	msi_data->cfg = (struct ls_scfg_msi_cfg *) match->data;
+
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	msi_data->regs = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(msi_data->regs)) {
@@ -182,23 +349,48 @@ static int ls_scfg_msi_probe(struct platform_device *pdev)
 	}
 	msi_data->msiir_addr = res->start;
 
-	msi_data->irq = platform_get_irq(pdev, 0);
-	if (msi_data->irq <= 0) {
-		dev_err(&pdev->dev, "failed to get MSI irq\n");
-		return -ENODEV;
-	}
-
 	msi_data->pdev = pdev;
 	spin_lock_init(&msi_data->lock);
 
+	msi_data->irqs_num = MSI_IRQS_PER_MSIR *
+			     (1 << msi_data->cfg->ibs_shift);
+	msi_data->used = devm_kcalloc(&pdev->dev,
+				    BITS_TO_LONGS(msi_data->irqs_num),
+				    sizeof(*msi_data->used),
+				    GFP_KERNEL);
+	if (!msi_data->used)
+		return -ENOMEM;
+	/*
+	 * Reserve all the hwirqs
+	 * The available hwirqs will be released in ls1_msi_setup_hwirq()
+	 */
+	bitmap_set(msi_data->used, 0, msi_data->irqs_num);
+
+	msi_data->msir_num = of_irq_count(pdev->dev.of_node);
+
+	if (msi_affinity_flag) {
+		u32 cpu_num;
+
+		cpu_num = num_possible_cpus();
+		if (msi_data->msir_num >= cpu_num)
+			msi_data->msir_num = cpu_num;
+		else
+			msi_affinity_flag = 0;
+	}
+
+	msi_data->msir = devm_kcalloc(&pdev->dev, msi_data->msir_num,
+				      sizeof(*msi_data->msir),
+				      GFP_KERNEL);
+	if (!msi_data->msir)
+		return -ENOMEM;
+
+	for (i = 0; i < msi_data->msir_num; i++)
+		ls_scfg_msi_setup_hwirq(msi_data, i);
+
 	ret = ls_scfg_msi_domains_init(msi_data);
 	if (ret)
 		return ret;
 
-	irq_set_chained_handler_and_data(msi_data->irq,
-					 ls_scfg_msi_irq_handler,
-					 msi_data);
-
 	platform_set_drvdata(pdev, msi_data);
 
 	return 0;
@@ -207,8 +399,10 @@ static int ls_scfg_msi_probe(struct platform_device *pdev)
 static int ls_scfg_msi_remove(struct platform_device *pdev)
 {
 	struct ls_scfg_msi *msi_data = platform_get_drvdata(pdev);
+	int i;
 
-	irq_set_chained_handler_and_data(msi_data->irq, NULL, NULL);
+	for (i = 0; i < msi_data->msir_num; i++)
+		ls_scfg_msi_teardown_hwirq(&msi_data->msir[i]);
 
 	irq_domain_remove(msi_data->msi_domain);
 	irq_domain_remove(msi_data->parent);
@@ -218,12 +412,6 @@ static int ls_scfg_msi_remove(struct platform_device *pdev)
 	return 0;
 }
 
-static const struct of_device_id ls_scfg_msi_id[] = {
-	{ .compatible = "fsl,1s1021a-msi", },
-	{ .compatible = "fsl,1s1043a-msi", },
-	{},
-};
-
 static struct platform_driver ls_scfg_msi_driver = {
 	.driver = {
 		.name = "ls-scfg-msi",
diff --git a/drivers/irqchip/irq-metag-ext.c b/drivers/irqchip/irq-metag-ext.c
index 0cdd923d1535..be7216bfb8dd 100644
--- a/drivers/irqchip/irq-metag-ext.c
+++ b/drivers/irqchip/irq-metag-ext.c
@@ -518,6 +518,8 @@ static int meta_intc_set_affinity(struct irq_data *data,
 
 	metag_out32(TBI_TRIG_VEC(TBID_SIGNUM_TR2(thread)), vec_addr);
 
+	irq_data_update_effective_affinity(data, cpumask_of(cpu));
+
 	return 0;
 }
 #else
@@ -578,6 +580,8 @@ static int meta_intc_map(struct irq_domain *d, unsigned int irq,
 	else
 		irq_set_chip_and_handler(irq, &meta_intc_edge_chip,
 					 handle_edge_irq);
+
+	irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq)));
 	return 0;
 }
 
diff --git a/drivers/irqchip/irq-mips-cpu.c b/drivers/irqchip/irq-mips-cpu.c
index 14461cbfab2f..66f97fde13d8 100644
--- a/drivers/irqchip/irq-mips-cpu.c
+++ b/drivers/irqchip/irq-mips-cpu.c
@@ -101,7 +101,7 @@ static void mips_mt_send_ipi(struct irq_data *d, unsigned int cpu)
 	local_irq_save(flags);
 
 	/* We can only send IPIs to VPEs within the local core */
-	WARN_ON(cpu_data[cpu].core != current_cpu_data.core);
+	WARN_ON(!cpus_are_siblings(smp_processor_id(), cpu));
 
 	vpflags = dvpe();
 	settc(cpu_vpe_id(&cpu_data[cpu]));
diff --git a/drivers/irqchip/irq-mips-gic.c b/drivers/irqchip/irq-mips-gic.c
index 6ab1d3afec02..6e52a88bbd9e 100644
--- a/drivers/irqchip/irq-mips-gic.c
+++ b/drivers/irqchip/irq-mips-gic.c
@@ -12,27 +12,38 @@
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/irqchip.h>
-#include <linux/irqchip/mips-gic.h>
 #include <linux/of_address.h>
+#include <linux/percpu.h>
 #include <linux/sched.h>
 #include <linux/smp.h>
 
-#include <asm/mips-cm.h>
+#include <asm/mips-cps.h>
 #include <asm/setup.h>
 #include <asm/traps.h>
 
 #include <dt-bindings/interrupt-controller/mips-gic.h>
 
-unsigned int gic_present;
+#define GIC_MAX_INTRS		256
+#define GIC_MAX_LONGS		BITS_TO_LONGS(GIC_MAX_INTRS)
 
-struct gic_pcpu_mask {
-	DECLARE_BITMAP(pcpu_mask, GIC_MAX_INTRS);
-};
+/* Add 2 to convert GIC CPU pin to core interrupt */
+#define GIC_CPU_PIN_OFFSET	2
+
+/* Mapped interrupt to pin X, then GIC will generate the vector (X+1). */
+#define GIC_PIN_TO_VEC_OFFSET	1
+
+/* Convert between local/shared IRQ number and GIC HW IRQ number. */
+#define GIC_LOCAL_HWIRQ_BASE	0
+#define GIC_LOCAL_TO_HWIRQ(x)	(GIC_LOCAL_HWIRQ_BASE + (x))
+#define GIC_HWIRQ_TO_LOCAL(x)	((x) - GIC_LOCAL_HWIRQ_BASE)
+#define GIC_SHARED_HWIRQ_BASE	GIC_NUM_LOCAL_INTRS
+#define GIC_SHARED_TO_HWIRQ(x)	(GIC_SHARED_HWIRQ_BASE + (x))
+#define GIC_HWIRQ_TO_SHARED(x)	((x) - GIC_SHARED_HWIRQ_BASE)
+
+void __iomem *mips_gic_base;
 
-static unsigned long __gic_base_addr;
+DEFINE_PER_CPU_READ_MOSTLY(unsigned long[GIC_MAX_LONGS], pcpu_masks);
 
-static void __iomem *gic_base;
-static struct gic_pcpu_mask pcpu_masks[NR_CPUS];
 static DEFINE_SPINLOCK(gic_lock);
 static struct irq_domain *gic_irq_domain;
 static struct irq_domain *gic_ipi_domain;
@@ -44,202 +55,13 @@ static struct irq_chip gic_level_irq_controller, gic_edge_irq_controller;
 DECLARE_BITMAP(ipi_resrv, GIC_MAX_INTRS);
 DECLARE_BITMAP(ipi_available, GIC_MAX_INTRS);
 
-static void __gic_irq_dispatch(void);
-
-static inline u32 gic_read32(unsigned int reg)
-{
-	return __raw_readl(gic_base + reg);
-}
-
-static inline u64 gic_read64(unsigned int reg)
-{
-	return __raw_readq(gic_base + reg);
-}
-
-static inline unsigned long gic_read(unsigned int reg)
-{
-	if (!mips_cm_is64)
-		return gic_read32(reg);
-	else
-		return gic_read64(reg);
-}
-
-static inline void gic_write32(unsigned int reg, u32 val)
-{
-	return __raw_writel(val, gic_base + reg);
-}
-
-static inline void gic_write64(unsigned int reg, u64 val)
-{
-	return __raw_writeq(val, gic_base + reg);
-}
-
-static inline void gic_write(unsigned int reg, unsigned long val)
-{
-	if (!mips_cm_is64)
-		return gic_write32(reg, (u32)val);
-	else
-		return gic_write64(reg, (u64)val);
-}
-
-static inline void gic_update_bits(unsigned int reg, unsigned long mask,
-				   unsigned long val)
-{
-	unsigned long regval;
-
-	regval = gic_read(reg);
-	regval &= ~mask;
-	regval |= val;
-	gic_write(reg, regval);
-}
-
-static inline void gic_reset_mask(unsigned int intr)
-{
-	gic_write(GIC_REG(SHARED, GIC_SH_RMASK) + GIC_INTR_OFS(intr),
-		  1ul << GIC_INTR_BIT(intr));
-}
-
-static inline void gic_set_mask(unsigned int intr)
-{
-	gic_write(GIC_REG(SHARED, GIC_SH_SMASK) + GIC_INTR_OFS(intr),
-		  1ul << GIC_INTR_BIT(intr));
-}
-
-static inline void gic_set_polarity(unsigned int intr, unsigned int pol)
-{
-	gic_update_bits(GIC_REG(SHARED, GIC_SH_SET_POLARITY) +
-			GIC_INTR_OFS(intr), 1ul << GIC_INTR_BIT(intr),
-			(unsigned long)pol << GIC_INTR_BIT(intr));
-}
-
-static inline void gic_set_trigger(unsigned int intr, unsigned int trig)
-{
-	gic_update_bits(GIC_REG(SHARED, GIC_SH_SET_TRIGGER) +
-			GIC_INTR_OFS(intr), 1ul << GIC_INTR_BIT(intr),
-			(unsigned long)trig << GIC_INTR_BIT(intr));
-}
-
-static inline void gic_set_dual_edge(unsigned int intr, unsigned int dual)
-{
-	gic_update_bits(GIC_REG(SHARED, GIC_SH_SET_DUAL) + GIC_INTR_OFS(intr),
-			1ul << GIC_INTR_BIT(intr),
-			(unsigned long)dual << GIC_INTR_BIT(intr));
-}
-
-static inline void gic_map_to_pin(unsigned int intr, unsigned int pin)
-{
-	gic_write32(GIC_REG(SHARED, GIC_SH_INTR_MAP_TO_PIN_BASE) +
-		    GIC_SH_MAP_TO_PIN(intr), GIC_MAP_TO_PIN_MSK | pin);
-}
-
-static inline void gic_map_to_vpe(unsigned int intr, unsigned int vpe)
-{
-	gic_write(GIC_REG(SHARED, GIC_SH_INTR_MAP_TO_VPE_BASE) +
-		  GIC_SH_MAP_TO_VPE_REG_OFF(intr, vpe),
-		  GIC_SH_MAP_TO_VPE_REG_BIT(vpe));
-}
-
-#ifdef CONFIG_CLKSRC_MIPS_GIC
-u64 notrace gic_read_count(void)
-{
-	unsigned int hi, hi2, lo;
-
-	if (mips_cm_is64)
-		return (u64)gic_read(GIC_REG(SHARED, GIC_SH_COUNTER));
-
-	do {
-		hi = gic_read32(GIC_REG(SHARED, GIC_SH_COUNTER_63_32));
-		lo = gic_read32(GIC_REG(SHARED, GIC_SH_COUNTER_31_00));
-		hi2 = gic_read32(GIC_REG(SHARED, GIC_SH_COUNTER_63_32));
-	} while (hi2 != hi);
-
-	return (((u64) hi) << 32) + lo;
-}
-
-unsigned int gic_get_count_width(void)
-{
-	unsigned int bits, config;
-
-	config = gic_read(GIC_REG(SHARED, GIC_SH_CONFIG));
-	bits = 32 + 4 * ((config & GIC_SH_CONFIG_COUNTBITS_MSK) >>
-			 GIC_SH_CONFIG_COUNTBITS_SHF);
-
-	return bits;
-}
-
-void notrace gic_write_compare(u64 cnt)
-{
-	if (mips_cm_is64) {
-		gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE), cnt);
-	} else {
-		gic_write32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_HI),
-					(int)(cnt >> 32));
-		gic_write32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_LO),
-					(int)(cnt & 0xffffffff));
-	}
-}
-
-void notrace gic_write_cpu_compare(u64 cnt, int cpu)
-{
-	unsigned long flags;
-
-	local_irq_save(flags);
-
-	gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR), mips_cm_vp_id(cpu));
-
-	if (mips_cm_is64) {
-		gic_write(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE), cnt);
-	} else {
-		gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE_HI),
-					(int)(cnt >> 32));
-		gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE_LO),
-					(int)(cnt & 0xffffffff));
-	}
-
-	local_irq_restore(flags);
-}
-
-u64 gic_read_compare(void)
-{
-	unsigned int hi, lo;
-
-	if (mips_cm_is64)
-		return (u64)gic_read(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE));
-
-	hi = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_HI));
-	lo = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_LO));
-
-	return (((u64) hi) << 32) + lo;
-}
-
-void gic_start_count(void)
-{
-	u32 gicconfig;
-
-	/* Start the counter */
-	gicconfig = gic_read(GIC_REG(SHARED, GIC_SH_CONFIG));
-	gicconfig &= ~(1 << GIC_SH_CONFIG_COUNTSTOP_SHF);
-	gic_write(GIC_REG(SHARED, GIC_SH_CONFIG), gicconfig);
-}
-
-void gic_stop_count(void)
+static void gic_clear_pcpu_masks(unsigned int intr)
 {
-	u32 gicconfig;
-
-	/* Stop the counter */
-	gicconfig = gic_read(GIC_REG(SHARED, GIC_SH_CONFIG));
-	gicconfig |= 1 << GIC_SH_CONFIG_COUNTSTOP_SHF;
-	gic_write(GIC_REG(SHARED, GIC_SH_CONFIG), gicconfig);
-}
-
-#endif
-
-unsigned gic_read_local_vp_id(void)
-{
-	unsigned long ident;
+	unsigned int i;
 
-	ident = gic_read(GIC_REG(VPE_LOCAL, GIC_VP_IDENT));
-	return ident & GIC_VP_IDENT_VCNUM_MSK;
+	/* Clear the interrupt's bit in all pcpu_masks */
+	for_each_possible_cpu(i)
+		clear_bit(intr, per_cpu_ptr(pcpu_masks, i));
 }
 
 static bool gic_local_irq_is_routable(int intr)
@@ -250,17 +72,17 @@ static bool gic_local_irq_is_routable(int intr)
 	if (cpu_has_veic)
 		return true;
 
-	vpe_ctl = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_CTL));
+	vpe_ctl = read_gic_vl_ctl();
 	switch (intr) {
 	case GIC_LOCAL_INT_TIMER:
-		return vpe_ctl & GIC_VPE_CTL_TIMER_RTBL_MSK;
+		return vpe_ctl & GIC_VX_CTL_TIMER_ROUTABLE;
 	case GIC_LOCAL_INT_PERFCTR:
-		return vpe_ctl & GIC_VPE_CTL_PERFCNT_RTBL_MSK;
+		return vpe_ctl & GIC_VX_CTL_PERFCNT_ROUTABLE;
 	case GIC_LOCAL_INT_FDC:
-		return vpe_ctl & GIC_VPE_CTL_FDC_RTBL_MSK;
+		return vpe_ctl & GIC_VX_CTL_FDC_ROUTABLE;
 	case GIC_LOCAL_INT_SWINT0:
 	case GIC_LOCAL_INT_SWINT1:
-		return vpe_ctl & GIC_VPE_CTL_SWINT_RTBL_MSK;
+		return vpe_ctl & GIC_VX_CTL_SWINT_ROUTABLE;
 	default:
 		return true;
 	}
@@ -272,15 +94,14 @@ static void gic_bind_eic_interrupt(int irq, int set)
 	irq -= GIC_PIN_TO_VEC_OFFSET;
 
 	/* Set irq to use shadow set */
-	gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_EIC_SHADOW_SET_BASE) +
-		  GIC_VPE_EIC_SS(irq), set);
+	write_gic_vl_eic_shadow_set(irq, set);
 }
 
 static void gic_send_ipi(struct irq_data *d, unsigned int cpu)
 {
 	irq_hw_number_t hwirq = GIC_HWIRQ_TO_SHARED(irqd_to_hwirq(d));
 
-	gic_write(GIC_REG(SHARED, GIC_SH_WEDGE), GIC_SH_WEDGE_SET(hwirq));
+	write_gic_wedge(GIC_WEDGE_RW | hwirq);
 }
 
 int gic_get_c0_compare_int(void)
@@ -316,47 +137,22 @@ int gic_get_c0_fdc_int(void)
 				  GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_FDC));
 }
 
-int gic_get_usm_range(struct resource *gic_usm_res)
-{
-	if (!gic_present)
-		return -1;
-
-	gic_usm_res->start = __gic_base_addr + USM_VISIBLE_SECTION_OFS;
-	gic_usm_res->end = gic_usm_res->start + (USM_VISIBLE_SECTION_SIZE - 1);
-
-	return 0;
-}
-
 static void gic_handle_shared_int(bool chained)
 {
-	unsigned int i, intr, virq, gic_reg_step = mips_cm_is64 ? 8 : 4;
+	unsigned int intr, virq;
 	unsigned long *pcpu_mask;
-	unsigned long pending_reg, intrmask_reg;
 	DECLARE_BITMAP(pending, GIC_MAX_INTRS);
-	DECLARE_BITMAP(intrmask, GIC_MAX_INTRS);
 
 	/* Get per-cpu bitmaps */
-	pcpu_mask = pcpu_masks[smp_processor_id()].pcpu_mask;
+	pcpu_mask = this_cpu_ptr(pcpu_masks);
 
-	pending_reg = GIC_REG(SHARED, GIC_SH_PEND);
-	intrmask_reg = GIC_REG(SHARED, GIC_SH_MASK);
-
-	for (i = 0; i < BITS_TO_LONGS(gic_shared_intrs); i++) {
-		pending[i] = gic_read(pending_reg);
-		intrmask[i] = gic_read(intrmask_reg);
-		pending_reg += gic_reg_step;
-		intrmask_reg += gic_reg_step;
-
-		if (!IS_ENABLED(CONFIG_64BIT) || mips_cm_is64)
-			continue;
-
-		pending[i] |= (u64)gic_read(pending_reg) << 32;
-		intrmask[i] |= (u64)gic_read(intrmask_reg) << 32;
-		pending_reg += gic_reg_step;
-		intrmask_reg += gic_reg_step;
-	}
+	if (mips_cm_is64)
+		__ioread64_copy(pending, addr_gic_pend(),
+				DIV_ROUND_UP(gic_shared_intrs, 64));
+	else
+		__ioread32_copy(pending, addr_gic_pend(),
+				DIV_ROUND_UP(gic_shared_intrs, 32));
 
-	bitmap_and(pending, pending, intrmask, gic_shared_intrs);
 	bitmap_and(pending, pending, pcpu_mask, gic_shared_intrs);
 
 	for_each_set_bit(intr, pending, gic_shared_intrs) {
@@ -371,19 +167,30 @@ static void gic_handle_shared_int(bool chained)
 
 static void gic_mask_irq(struct irq_data *d)
 {
-	gic_reset_mask(GIC_HWIRQ_TO_SHARED(d->hwirq));
+	unsigned int intr = GIC_HWIRQ_TO_SHARED(d->hwirq);
+
+	write_gic_rmask(BIT(intr));
+	gic_clear_pcpu_masks(intr);
 }
 
 static void gic_unmask_irq(struct irq_data *d)
 {
-	gic_set_mask(GIC_HWIRQ_TO_SHARED(d->hwirq));
+	struct cpumask *affinity = irq_data_get_affinity_mask(d);
+	unsigned int intr = GIC_HWIRQ_TO_SHARED(d->hwirq);
+	unsigned int cpu;
+
+	write_gic_smask(BIT(intr));
+
+	gic_clear_pcpu_masks(intr);
+	cpu = cpumask_first_and(affinity, cpu_online_mask);
+	set_bit(intr, per_cpu_ptr(pcpu_masks, cpu));
 }
 
 static void gic_ack_irq(struct irq_data *d)
 {
 	unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
 
-	gic_write(GIC_REG(SHARED, GIC_SH_WEDGE), GIC_SH_WEDGE_CLR(irq));
+	write_gic_wedge(irq);
 }
 
 static int gic_set_type(struct irq_data *d, unsigned int type)
@@ -395,34 +202,34 @@ static int gic_set_type(struct irq_data *d, unsigned int type)
 	spin_lock_irqsave(&gic_lock, flags);
 	switch (type & IRQ_TYPE_SENSE_MASK) {
 	case IRQ_TYPE_EDGE_FALLING:
-		gic_set_polarity(irq, GIC_POL_NEG);
-		gic_set_trigger(irq, GIC_TRIG_EDGE);
-		gic_set_dual_edge(irq, GIC_TRIG_DUAL_DISABLE);
+		change_gic_pol(irq, GIC_POL_FALLING_EDGE);
+		change_gic_trig(irq, GIC_TRIG_EDGE);
+		change_gic_dual(irq, GIC_DUAL_SINGLE);
 		is_edge = true;
 		break;
 	case IRQ_TYPE_EDGE_RISING:
-		gic_set_polarity(irq, GIC_POL_POS);
-		gic_set_trigger(irq, GIC_TRIG_EDGE);
-		gic_set_dual_edge(irq, GIC_TRIG_DUAL_DISABLE);
+		change_gic_pol(irq, GIC_POL_RISING_EDGE);
+		change_gic_trig(irq, GIC_TRIG_EDGE);
+		change_gic_dual(irq, GIC_DUAL_SINGLE);
 		is_edge = true;
 		break;
 	case IRQ_TYPE_EDGE_BOTH:
 		/* polarity is irrelevant in this case */
-		gic_set_trigger(irq, GIC_TRIG_EDGE);
-		gic_set_dual_edge(irq, GIC_TRIG_DUAL_ENABLE);
+		change_gic_trig(irq, GIC_TRIG_EDGE);
+		change_gic_dual(irq, GIC_DUAL_DUAL);
 		is_edge = true;
 		break;
 	case IRQ_TYPE_LEVEL_LOW:
-		gic_set_polarity(irq, GIC_POL_NEG);
-		gic_set_trigger(irq, GIC_TRIG_LEVEL);
-		gic_set_dual_edge(irq, GIC_TRIG_DUAL_DISABLE);
+		change_gic_pol(irq, GIC_POL_ACTIVE_LOW);
+		change_gic_trig(irq, GIC_TRIG_LEVEL);
+		change_gic_dual(irq, GIC_DUAL_SINGLE);
 		is_edge = false;
 		break;
 	case IRQ_TYPE_LEVEL_HIGH:
 	default:
-		gic_set_polarity(irq, GIC_POL_POS);
-		gic_set_trigger(irq, GIC_TRIG_LEVEL);
-		gic_set_dual_edge(irq, GIC_TRIG_DUAL_DISABLE);
+		change_gic_pol(irq, GIC_POL_ACTIVE_HIGH);
+		change_gic_trig(irq, GIC_TRIG_LEVEL);
+		change_gic_dual(irq, GIC_DUAL_SINGLE);
 		is_edge = false;
 		break;
 	}
@@ -443,29 +250,28 @@ static int gic_set_affinity(struct irq_data *d, const struct cpumask *cpumask,
 			    bool force)
 {
 	unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
-	cpumask_t	tmp = CPU_MASK_NONE;
-	unsigned long	flags;
-	int		i;
+	unsigned long flags;
+	unsigned int cpu;
 
-	cpumask_and(&tmp, cpumask, cpu_online_mask);
-	if (cpumask_empty(&tmp))
+	cpu = cpumask_first_and(cpumask, cpu_online_mask);
+	if (cpu >= NR_CPUS)
 		return -EINVAL;
 
 	/* Assumption : cpumask refers to a single CPU */
 	spin_lock_irqsave(&gic_lock, flags);
 
 	/* Re-route this IRQ */
-	gic_map_to_vpe(irq, mips_cm_vp_id(cpumask_first(&tmp)));
+	write_gic_map_vp(irq, BIT(mips_cm_vp_id(cpu)));
 
 	/* Update the pcpu_masks */
-	for (i = 0; i < min(gic_vpes, NR_CPUS); i++)
-		clear_bit(irq, pcpu_masks[i].pcpu_mask);
-	set_bit(irq, pcpu_masks[cpumask_first(&tmp)].pcpu_mask);
+	gic_clear_pcpu_masks(irq);
+	if (read_gic_mask(irq))
+		set_bit(irq, per_cpu_ptr(pcpu_masks, cpu));
 
-	cpumask_copy(irq_data_get_affinity_mask(d), cpumask);
+	irq_data_update_effective_affinity(d, cpumask_of(cpu));
 	spin_unlock_irqrestore(&gic_lock, flags);
 
-	return IRQ_SET_MASK_OK_NOCOPY;
+	return IRQ_SET_MASK_OK;
 }
 #endif
 
@@ -496,8 +302,8 @@ static void gic_handle_local_int(bool chained)
 	unsigned long pending, masked;
 	unsigned int intr, virq;
 
-	pending = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_PEND));
-	masked = gic_read32(GIC_REG(VPE_LOCAL, GIC_VPE_MASK));
+	pending = read_gic_vl_pend();
+	masked = read_gic_vl_mask();
 
 	bitmap_and(&pending, &pending, &masked, GIC_NUM_LOCAL_INTRS);
 
@@ -515,14 +321,14 @@ static void gic_mask_local_irq(struct irq_data *d)
 {
 	int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
 
-	gic_write32(GIC_REG(VPE_LOCAL, GIC_VPE_RMASK), 1 << intr);
+	write_gic_vl_rmask(BIT(intr));
 }
 
 static void gic_unmask_local_irq(struct irq_data *d)
 {
 	int intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
 
-	gic_write32(GIC_REG(VPE_LOCAL, GIC_VPE_SMASK), 1 << intr);
+	write_gic_vl_smask(BIT(intr));
 }
 
 static struct irq_chip gic_local_irq_controller = {
@@ -539,9 +345,8 @@ static void gic_mask_local_irq_all_vpes(struct irq_data *d)
 
 	spin_lock_irqsave(&gic_lock, flags);
 	for (i = 0; i < gic_vpes; i++) {
-		gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR),
-			  mips_cm_vp_id(i));
-		gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_RMASK), 1 << intr);
+		write_gic_vl_other(mips_cm_vp_id(i));
+		write_gic_vo_rmask(BIT(intr));
 	}
 	spin_unlock_irqrestore(&gic_lock, flags);
 }
@@ -554,9 +359,8 @@ static void gic_unmask_local_irq_all_vpes(struct irq_data *d)
 
 	spin_lock_irqsave(&gic_lock, flags);
 	for (i = 0; i < gic_vpes; i++) {
-		gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR),
-			  mips_cm_vp_id(i));
-		gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_SMASK), 1 << intr);
+		write_gic_vl_other(mips_cm_vp_id(i));
+		write_gic_vo_smask(BIT(intr));
 	}
 	spin_unlock_irqrestore(&gic_lock, flags);
 }
@@ -579,103 +383,50 @@ static void gic_irq_dispatch(struct irq_desc *desc)
 	gic_handle_shared_int(true);
 }
 
-static void __init gic_basic_init(void)
-{
-	unsigned int i;
-
-	board_bind_eic_interrupt = &gic_bind_eic_interrupt;
-
-	/* Setup defaults */
-	for (i = 0; i < gic_shared_intrs; i++) {
-		gic_set_polarity(i, GIC_POL_POS);
-		gic_set_trigger(i, GIC_TRIG_LEVEL);
-		gic_reset_mask(i);
-	}
-
-	for (i = 0; i < gic_vpes; i++) {
-		unsigned int j;
-
-		gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR),
-			  mips_cm_vp_id(i));
-		for (j = 0; j < GIC_NUM_LOCAL_INTRS; j++) {
-			if (!gic_local_irq_is_routable(j))
-				continue;
-			gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_RMASK), 1 << j);
-		}
-	}
-}
-
 static int gic_local_irq_domain_map(struct irq_domain *d, unsigned int virq,
 				    irq_hw_number_t hw)
 {
 	int intr = GIC_HWIRQ_TO_LOCAL(hw);
-	int ret = 0;
 	int i;
 	unsigned long flags;
+	u32 val;
 
 	if (!gic_local_irq_is_routable(intr))
 		return -EPERM;
 
+	if (intr > GIC_LOCAL_INT_FDC) {
+		pr_err("Invalid local IRQ %d\n", intr);
+		return -EINVAL;
+	}
+
+	if (intr == GIC_LOCAL_INT_TIMER) {
+		/* CONFIG_MIPS_CMP workaround (see __gic_init) */
+		val = GIC_MAP_PIN_MAP_TO_PIN | timer_cpu_pin;
+	} else {
+		val = GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin;
+	}
+
 	spin_lock_irqsave(&gic_lock, flags);
 	for (i = 0; i < gic_vpes; i++) {
-		u32 val = GIC_MAP_TO_PIN_MSK | gic_cpu_pin;
-
-		gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR),
-			  mips_cm_vp_id(i));
-
-		switch (intr) {
-		case GIC_LOCAL_INT_WD:
-			gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_WD_MAP), val);
-			break;
-		case GIC_LOCAL_INT_COMPARE:
-			gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE_MAP),
-				    val);
-			break;
-		case GIC_LOCAL_INT_TIMER:
-			/* CONFIG_MIPS_CMP workaround (see __gic_init) */
-			val = GIC_MAP_TO_PIN_MSK | timer_cpu_pin;
-			gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_TIMER_MAP),
-				    val);
-			break;
-		case GIC_LOCAL_INT_PERFCTR:
-			gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_PERFCTR_MAP),
-				    val);
-			break;
-		case GIC_LOCAL_INT_SWINT0:
-			gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_SWINT0_MAP),
-				    val);
-			break;
-		case GIC_LOCAL_INT_SWINT1:
-			gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_SWINT1_MAP),
-				    val);
-			break;
-		case GIC_LOCAL_INT_FDC:
-			gic_write32(GIC_REG(VPE_OTHER, GIC_VPE_FDC_MAP), val);
-			break;
-		default:
-			pr_err("Invalid local IRQ %d\n", intr);
-			ret = -EINVAL;
-			break;
-		}
+		write_gic_vl_other(mips_cm_vp_id(i));
+		write_gic_vo_map(intr, val);
 	}
 	spin_unlock_irqrestore(&gic_lock, flags);
 
-	return ret;
+	return 0;
 }
 
 static int gic_shared_irq_domain_map(struct irq_domain *d, unsigned int virq,
-				     irq_hw_number_t hw, unsigned int vpe)
+				     irq_hw_number_t hw, unsigned int cpu)
 {
 	int intr = GIC_HWIRQ_TO_SHARED(hw);
 	unsigned long flags;
-	int i;
 
 	spin_lock_irqsave(&gic_lock, flags);
-	gic_map_to_pin(intr, gic_cpu_pin);
-	gic_map_to_vpe(intr, mips_cm_vp_id(vpe));
-	for (i = 0; i < min(gic_vpes, NR_CPUS); i++)
-		clear_bit(intr, pcpu_masks[i].pcpu_mask);
-	set_bit(intr, pcpu_masks[vpe].pcpu_mask);
+	write_gic_map_pin(intr, GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin);
+	write_gic_map_vp(intr, BIT(mips_cm_vp_id(cpu)));
+	gic_clear_pcpu_masks(intr);
+	set_bit(intr, per_cpu_ptr(pcpu_masks, cpu));
 	spin_unlock_irqrestore(&gic_lock, flags);
 
 	return 0;
@@ -716,6 +467,7 @@ static int gic_irq_domain_map(struct irq_domain *d, unsigned int virq,
 		if (err)
 			return err;
 
+		irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(virq)));
 		return gic_shared_irq_domain_map(d, virq, hwirq, 0);
 	}
 
@@ -881,34 +633,69 @@ static const struct irq_domain_ops gic_ipi_domain_ops = {
 	.match = gic_ipi_domain_match,
 };
 
-static void __init __gic_init(unsigned long gic_base_addr,
-			      unsigned long gic_addrspace_size,
-			      unsigned int cpu_vec, unsigned int irqbase,
-			      struct device_node *node)
+
+static int __init gic_of_init(struct device_node *node,
+			      struct device_node *parent)
 {
-	unsigned int gicconfig, cpu;
-	unsigned int v[2];
+	unsigned int cpu_vec, i, j, gicconfig, cpu, v[2];
+	unsigned long reserved;
+	phys_addr_t gic_base;
+	struct resource res;
+	size_t gic_len;
+
+	/* Find the first available CPU vector. */
+	i = 0;
+	reserved = (C_SW0 | C_SW1) >> __fls(C_SW0);
+	while (!of_property_read_u32_index(node, "mti,reserved-cpu-vectors",
+					   i++, &cpu_vec))
+		reserved |= BIT(cpu_vec);
+
+	cpu_vec = find_first_zero_bit(&reserved, hweight_long(ST0_IM));
+	if (cpu_vec == hweight_long(ST0_IM)) {
+		pr_err("No CPU vectors available for GIC\n");
+		return -ENODEV;
+	}
+
+	if (of_address_to_resource(node, 0, &res)) {
+		/*
+		 * Probe the CM for the GIC base address if not specified
+		 * in the device-tree.
+		 */
+		if (mips_cm_present()) {
+			gic_base = read_gcr_gic_base() &
+				~CM_GCR_GIC_BASE_GICEN;
+			gic_len = 0x20000;
+		} else {
+			pr_err("Failed to get GIC memory range\n");
+			return -ENODEV;
+		}
+	} else {
+		gic_base = res.start;
+		gic_len = resource_size(&res);
+	}
 
-	__gic_base_addr = gic_base_addr;
+	if (mips_cm_present()) {
+		write_gcr_gic_base(gic_base | CM_GCR_GIC_BASE_GICEN);
+		/* Ensure GIC region is enabled before trying to access it */
+		__sync();
+	}
 
-	gic_base = ioremap_nocache(gic_base_addr, gic_addrspace_size);
+	mips_gic_base = ioremap_nocache(gic_base, gic_len);
 
-	gicconfig = gic_read(GIC_REG(SHARED, GIC_SH_CONFIG));
-	gic_shared_intrs = (gicconfig & GIC_SH_CONFIG_NUMINTRS_MSK) >>
-		   GIC_SH_CONFIG_NUMINTRS_SHF;
-	gic_shared_intrs = ((gic_shared_intrs + 1) * 8);
+	gicconfig = read_gic_config();
+	gic_shared_intrs = gicconfig & GIC_CONFIG_NUMINTERRUPTS;
+	gic_shared_intrs >>= __fls(GIC_CONFIG_NUMINTERRUPTS);
+	gic_shared_intrs = (gic_shared_intrs + 1) * 8;
 
-	gic_vpes = (gicconfig & GIC_SH_CONFIG_NUMVPES_MSK) >>
-		  GIC_SH_CONFIG_NUMVPES_SHF;
+	gic_vpes = gicconfig & GIC_CONFIG_PVPS;
+	gic_vpes >>= __fls(GIC_CONFIG_PVPS);
 	gic_vpes = gic_vpes + 1;
 
 	if (cpu_has_veic) {
 		/* Set EIC mode for all VPEs */
 		for_each_present_cpu(cpu) {
-			gic_write(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR),
-				  mips_cm_vp_id(cpu));
-			gic_write(GIC_REG(VPE_OTHER, GIC_VPE_CTL),
-				  GIC_VPE_CTL_EIC_MODE_MSK);
+			write_gic_vl_other(mips_cm_vp_id(cpu));
+			write_gic_vo_ctl(GIC_VX_CTL_EIC);
 		}
 
 		/* Always use vector 1 in EIC mode */
@@ -933,9 +720,7 @@ static void __init __gic_init(unsigned long gic_base_addr,
 		 */
 		if (IS_ENABLED(CONFIG_MIPS_CMP) &&
 		    gic_local_irq_is_routable(GIC_LOCAL_INT_TIMER)) {
-			timer_cpu_pin = gic_read32(GIC_REG(VPE_LOCAL,
-							 GIC_VPE_TIMER_MAP)) &
-					GIC_MAP_MSK;
+			timer_cpu_pin = read_gic_vl_timer_map() & GIC_MAP_PIN_MAP;
 			irq_set_chained_handler(MIPS_CPU_IRQ_BASE +
 						GIC_CPU_PIN_OFFSET +
 						timer_cpu_pin,
@@ -946,17 +731,21 @@ static void __init __gic_init(unsigned long gic_base_addr,
 	}
 
 	gic_irq_domain = irq_domain_add_simple(node, GIC_NUM_LOCAL_INTRS +
-					       gic_shared_intrs, irqbase,
+					       gic_shared_intrs, 0,
 					       &gic_irq_domain_ops, NULL);
-	if (!gic_irq_domain)
-		panic("Failed to add GIC IRQ domain");
+	if (!gic_irq_domain) {
+		pr_err("Failed to add GIC IRQ domain");
+		return -ENXIO;
+	}
 
 	gic_ipi_domain = irq_domain_add_hierarchy(gic_irq_domain,
 						  IRQ_DOMAIN_FLAG_IPI_PER_CPU,
 						  GIC_NUM_LOCAL_INTRS + gic_shared_intrs,
 						  node, &gic_ipi_domain_ops, NULL);
-	if (!gic_ipi_domain)
-		panic("Failed to add GIC IPI domain");
+	if (!gic_ipi_domain) {
+		pr_err("Failed to add GIC IPI domain");
+		return -ENXIO;
+	}
 
 	irq_domain_update_bus_token(gic_ipi_domain, DOMAIN_BUS_IPI);
 
@@ -971,61 +760,25 @@ static void __init __gic_init(unsigned long gic_base_addr,
 	}
 
 	bitmap_copy(ipi_available, ipi_resrv, GIC_MAX_INTRS);
-	gic_basic_init();
-}
-
-void __init gic_init(unsigned long gic_base_addr,
-		     unsigned long gic_addrspace_size,
-		     unsigned int cpu_vec, unsigned int irqbase)
-{
-	__gic_init(gic_base_addr, gic_addrspace_size, cpu_vec, irqbase, NULL);
-}
 
-static int __init gic_of_init(struct device_node *node,
-			      struct device_node *parent)
-{
-	struct resource res;
-	unsigned int cpu_vec, i = 0, reserved = 0;
-	phys_addr_t gic_base;
-	size_t gic_len;
+	board_bind_eic_interrupt = &gic_bind_eic_interrupt;
 
-	/* Find the first available CPU vector. */
-	while (!of_property_read_u32_index(node, "mti,reserved-cpu-vectors",
-					   i++, &cpu_vec))
-		reserved |= BIT(cpu_vec);
-	for (cpu_vec = 2; cpu_vec < 8; cpu_vec++) {
-		if (!(reserved & BIT(cpu_vec)))
-			break;
-	}
-	if (cpu_vec == 8) {
-		pr_err("No CPU vectors available for GIC\n");
-		return -ENODEV;
+	/* Setup defaults */
+	for (i = 0; i < gic_shared_intrs; i++) {
+		change_gic_pol(i, GIC_POL_ACTIVE_HIGH);
+		change_gic_trig(i, GIC_TRIG_LEVEL);
+		write_gic_rmask(BIT(i));
 	}
 
-	if (of_address_to_resource(node, 0, &res)) {
-		/*
-		 * Probe the CM for the GIC base address if not specified
-		 * in the device-tree.
-		 */
-		if (mips_cm_present()) {
-			gic_base = read_gcr_gic_base() &
-				~CM_GCR_GIC_BASE_GICEN_MSK;
-			gic_len = 0x20000;
-		} else {
-			pr_err("Failed to get GIC memory range\n");
-			return -ENODEV;
+	for (i = 0; i < gic_vpes; i++) {
+		write_gic_vl_other(mips_cm_vp_id(i));
+		for (j = 0; j < GIC_NUM_LOCAL_INTRS; j++) {
+			if (!gic_local_irq_is_routable(j))
+				continue;
+			write_gic_vo_rmask(BIT(j));
 		}
-	} else {
-		gic_base = res.start;
-		gic_len = resource_size(&res);
 	}
 
-	if (mips_cm_present())
-		write_gcr_gic_base(gic_base | CM_GCR_GIC_BASE_GICEN_MSK);
-	gic_present = true;
-
-	__gic_init(gic_base, gic_len, cpu_vec, 0, node);
-
 	return 0;
 }
 IRQCHIP_DECLARE(mips_gic, "mti,gic", gic_of_init);
diff --git a/drivers/irqchip/irq-mmp.c b/drivers/irqchip/irq-mmp.c
index 013fc9659a84..25f32e1d7764 100644
--- a/drivers/irqchip/irq-mmp.c
+++ b/drivers/irqchip/irq-mmp.c
@@ -181,13 +181,13 @@ const struct irq_domain_ops mmp_irq_domain_ops = {
 	.xlate		= mmp_irq_domain_xlate,
 };
 
-static struct mmp_intc_conf mmp_conf = {
+static const struct mmp_intc_conf mmp_conf = {
 	.conf_enable	= 0x51,
 	.conf_disable	= 0x0,
 	.conf_mask	= 0x7f,
 };
 
-static struct mmp_intc_conf mmp2_conf = {
+static const struct mmp_intc_conf mmp2_conf = {
 	.conf_enable	= 0x20,
 	.conf_disable	= 0x0,
 	.conf_mask	= 0x7f,
diff --git a/drivers/irqchip/irq-mtk-sysirq.c b/drivers/irqchip/irq-mtk-sysirq.c
index eeac512ec5a8..90aaf190157f 100644
--- a/drivers/irqchip/irq-mtk-sysirq.c
+++ b/drivers/irqchip/irq-mtk-sysirq.c
@@ -178,8 +178,7 @@ static int __init mtk_sysirq_of_init(struct device_node *node,
 		chip_data->intpol_words[i] = size / 4;
 		chip_data->intpol_bases[i] = of_iomap(node, i);
 		if (ret || !chip_data->intpol_bases[i]) {
-			pr_err("%s: couldn't map region %d\n",
-			       node->full_name, i);
+			pr_err("%pOF: couldn't map region %d\n", node, i);
 			ret = -ENODEV;
 			goto out_free_intpol;
 		}
diff --git a/drivers/irqchip/irq-mxs.c b/drivers/irqchip/irq-mxs.c
index 05fa9f7af53c..e8b31f52e071 100644
--- a/drivers/irqchip/irq-mxs.c
+++ b/drivers/irqchip/irq-mxs.c
@@ -179,7 +179,7 @@ static void __init icoll_add_domain(struct device_node *np,
 					     &icoll_irq_domain_ops, NULL);
 
 	if (!icoll_domain)
-		panic("%s: unable to create irq domain", np->full_name);
+		panic("%pOF: unable to create irq domain", np);
 }
 
 static void __iomem * __init icoll_init_iobase(struct device_node *np)
@@ -188,7 +188,7 @@ static void __iomem * __init icoll_init_iobase(struct device_node *np)
 
 	icoll_base = of_io_request_and_map(np, 0, np->name);
 	if (IS_ERR(icoll_base))
-		panic("%s: unable to map resource", np->full_name);
+		panic("%pOF: unable to map resource", np);
 	return icoll_base;
 }
 
diff --git a/drivers/irqchip/irq-stm32-exti.c b/drivers/irqchip/irq-stm32-exti.c
index 491568c95aa5..45363ff8d06f 100644
--- a/drivers/irqchip/irq-stm32-exti.c
+++ b/drivers/irqchip/irq-stm32-exti.c
@@ -140,7 +140,7 @@ static int __init stm32_exti_init(struct device_node *node,
 
 	base = of_iomap(node, 0);
 	if (!base) {
-		pr_err("%s: Unable to map registers\n", node->full_name);
+		pr_err("%pOF: Unable to map registers\n", node);
 		return -ENOMEM;
 	}
 
@@ -149,7 +149,7 @@ static int __init stm32_exti_init(struct device_node *node,
 	nr_exti = fls(readl_relaxed(base + EXTI_RTSR));
 	writel_relaxed(0, base + EXTI_RTSR);
 
-	pr_info("%s: %d External IRQs detected\n", node->full_name, nr_exti);
+	pr_info("%pOF: %d External IRQs detected\n", node, nr_exti);
 
 	domain = irq_domain_add_linear(node, nr_exti,
 				       &irq_exti_domain_ops, NULL);
@@ -163,8 +163,8 @@ static int __init stm32_exti_init(struct device_node *node,
 	ret = irq_alloc_domain_generic_chips(domain, nr_exti, 1, "exti",
 					     handle_edge_irq, clr, 0, 0);
 	if (ret) {
-		pr_err("%s: Could not allocate generic interrupt chip.\n",
-			node->full_name);
+		pr_err("%pOF: Could not allocate generic interrupt chip.\n",
+			node);
 		goto out_free_domain;
 	}
 
diff --git a/drivers/irqchip/irq-sun4i.c b/drivers/irqchip/irq-sun4i.c
index 376b28074e0d..e3e5b9132b75 100644
--- a/drivers/irqchip/irq-sun4i.c
+++ b/drivers/irqchip/irq-sun4i.c
@@ -97,8 +97,8 @@ static int __init sun4i_of_init(struct device_node *node,
 {
 	sun4i_irq_base = of_iomap(node, 0);
 	if (!sun4i_irq_base)
-		panic("%s: unable to map IC registers\n",
-			node->full_name);
+		panic("%pOF: unable to map IC registers\n",
+			node);
 
 	/* Disable all interrupts */
 	writel(0, sun4i_irq_base + SUN4I_IRQ_ENABLE_REG(0));
@@ -124,7 +124,7 @@ static int __init sun4i_of_init(struct device_node *node,
 	sun4i_irq_domain = irq_domain_add_linear(node, 3 * 32,
 						 &sun4i_irq_ops, NULL);
 	if (!sun4i_irq_domain)
-		panic("%s: unable to create IRQ domain\n", node->full_name);
+		panic("%pOF: unable to create IRQ domain\n", node);
 
 	set_handle_irq(sun4i_handle_irq);
 
diff --git a/drivers/irqchip/irq-tegra.c b/drivers/irqchip/irq-tegra.c
index 3973a14bb15b..0abc0cd1c32e 100644
--- a/drivers/irqchip/irq-tegra.c
+++ b/drivers/irqchip/irq-tegra.c
@@ -291,13 +291,13 @@ static int __init tegra_ictlr_init(struct device_node *node,
 	int err;
 
 	if (!parent) {
-		pr_err("%s: no parent, giving up\n", node->full_name);
+		pr_err("%pOF: no parent, giving up\n", node);
 		return -ENODEV;
 	}
 
 	parent_domain = irq_find_host(parent);
 	if (!parent_domain) {
-		pr_err("%s: unable to obtain parent domain\n", node->full_name);
+		pr_err("%pOF: unable to obtain parent domain\n", node);
 		return -ENXIO;
 	}
 
@@ -329,29 +329,29 @@ static int __init tegra_ictlr_init(struct device_node *node,
 	}
 
 	if (!num_ictlrs) {
-		pr_err("%s: no valid regions, giving up\n", node->full_name);
+		pr_err("%pOF: no valid regions, giving up\n", node);
 		err = -ENOMEM;
 		goto out_free;
 	}
 
 	WARN(num_ictlrs != soc->num_ictlrs,
-	     "%s: Found %u interrupt controllers in DT; expected %u.\n",
-	     node->full_name, num_ictlrs, soc->num_ictlrs);
+	     "%pOF: Found %u interrupt controllers in DT; expected %u.\n",
+	     node, num_ictlrs, soc->num_ictlrs);
 
 
 	domain = irq_domain_add_hierarchy(parent_domain, 0, num_ictlrs * 32,
 					  node, &tegra_ictlr_domain_ops,
 					  lic);
 	if (!domain) {
-		pr_err("%s: failed to allocated domain\n", node->full_name);
+		pr_err("%pOF: failed to allocated domain\n", node);
 		err = -ENOMEM;
 		goto out_unmap;
 	}
 
 	tegra_ictlr_syscore_init();
 
-	pr_info("%s: %d interrupts forwarded to %s\n",
-		node->full_name, num_ictlrs * 32, parent->full_name);
+	pr_info("%pOF: %d interrupts forwarded to %pOF\n",
+		node, num_ictlrs * 32, parent);
 
 	return 0;
 
diff --git a/drivers/irqchip/irq-uniphier-aidet.c b/drivers/irqchip/irq-uniphier-aidet.c
new file mode 100644
index 000000000000..7ba7f253470e
--- /dev/null
+++ b/drivers/irqchip/irq-uniphier-aidet.c
@@ -0,0 +1,261 @@
+/*
+ * Driver for UniPhier AIDET (ARM Interrupt Detector)
+ *
+ * Copyright (C) 2017 Socionext Inc.
+ *   Author: Masahiro Yamada <yamada.masahiro@socionext.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/bitops.h>
+#include <linux/init.h>
+#include <linux/irq.h>
+#include <linux/irqdomain.h>
+#include <linux/kernel.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+#include <linux/spinlock.h>
+
+#define UNIPHIER_AIDET_NR_IRQS		256
+
+#define UNIPHIER_AIDET_DETCONF		0x04	/* inverter register base */
+
+struct uniphier_aidet_priv {
+	struct irq_domain *domain;
+	void __iomem *reg_base;
+	spinlock_t lock;
+	u32 saved_vals[UNIPHIER_AIDET_NR_IRQS / 32];
+};
+
+static void uniphier_aidet_reg_update(struct uniphier_aidet_priv *priv,
+				      unsigned int reg, u32 mask, u32 val)
+{
+	unsigned long flags;
+	u32 tmp;
+
+	spin_lock_irqsave(&priv->lock, flags);
+	tmp = readl_relaxed(priv->reg_base + reg);
+	tmp &= ~mask;
+	tmp |= mask & val;
+	writel_relaxed(tmp, priv->reg_base + reg);
+	spin_unlock_irqrestore(&priv->lock, flags);
+}
+
+static void uniphier_aidet_detconf_update(struct uniphier_aidet_priv *priv,
+					  unsigned long index, unsigned int val)
+{
+	unsigned int reg;
+	u32 mask;
+
+	reg = UNIPHIER_AIDET_DETCONF + index / 32 * 4;
+	mask = BIT(index % 32);
+
+	uniphier_aidet_reg_update(priv, reg, mask, val ? mask : 0);
+}
+
+static int uniphier_aidet_irq_set_type(struct irq_data *data, unsigned int type)
+{
+	struct uniphier_aidet_priv *priv = data->chip_data;
+	unsigned int val;
+
+	/* enable inverter for active low triggers */
+	switch (type) {
+	case IRQ_TYPE_EDGE_RISING:
+	case IRQ_TYPE_LEVEL_HIGH:
+		val = 0;
+		break;
+	case IRQ_TYPE_EDGE_FALLING:
+		val = 1;
+		type = IRQ_TYPE_EDGE_RISING;
+		break;
+	case IRQ_TYPE_LEVEL_LOW:
+		val = 1;
+		type = IRQ_TYPE_LEVEL_HIGH;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	uniphier_aidet_detconf_update(priv, data->hwirq, val);
+
+	return irq_chip_set_type_parent(data, type);
+}
+
+static struct irq_chip uniphier_aidet_irq_chip = {
+	.name = "AIDET",
+	.irq_mask = irq_chip_mask_parent,
+	.irq_unmask = irq_chip_unmask_parent,
+	.irq_eoi = irq_chip_eoi_parent,
+	.irq_set_affinity = irq_chip_set_affinity_parent,
+	.irq_set_type = uniphier_aidet_irq_set_type,
+};
+
+static int uniphier_aidet_domain_translate(struct irq_domain *domain,
+					   struct irq_fwspec *fwspec,
+					   unsigned long *out_hwirq,
+					   unsigned int *out_type)
+{
+	if (WARN_ON(fwspec->param_count < 2))
+		return -EINVAL;
+
+	*out_hwirq = fwspec->param[0];
+	*out_type = fwspec->param[1] & IRQ_TYPE_SENSE_MASK;
+
+	return 0;
+}
+
+static int uniphier_aidet_domain_alloc(struct irq_domain *domain,
+				       unsigned int virq, unsigned int nr_irqs,
+				       void *arg)
+{
+	struct irq_fwspec parent_fwspec;
+	irq_hw_number_t hwirq;
+	unsigned int type;
+	int ret;
+
+	if (nr_irqs != 1)
+		return -EINVAL;
+
+	ret = uniphier_aidet_domain_translate(domain, arg, &hwirq, &type);
+	if (ret)
+		return ret;
+
+	switch (type) {
+	case IRQ_TYPE_EDGE_RISING:
+	case IRQ_TYPE_LEVEL_HIGH:
+		break;
+	case IRQ_TYPE_EDGE_FALLING:
+		type = IRQ_TYPE_EDGE_RISING;
+		break;
+	case IRQ_TYPE_LEVEL_LOW:
+		type = IRQ_TYPE_LEVEL_HIGH;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (hwirq >= UNIPHIER_AIDET_NR_IRQS)
+		return -ENXIO;
+
+	ret = irq_domain_set_hwirq_and_chip(domain, virq, hwirq,
+					    &uniphier_aidet_irq_chip,
+					    domain->host_data);
+	if (ret)
+		return ret;
+
+	/* parent is GIC */
+	parent_fwspec.fwnode = domain->parent->fwnode;
+	parent_fwspec.param_count = 3;
+	parent_fwspec.param[0] = 0;		/* SPI */
+	parent_fwspec.param[1] = hwirq;
+	parent_fwspec.param[2] = type;
+
+	return irq_domain_alloc_irqs_parent(domain, virq, 1, &parent_fwspec);
+}
+
+static const struct irq_domain_ops uniphier_aidet_domain_ops = {
+	.alloc = uniphier_aidet_domain_alloc,
+	.free = irq_domain_free_irqs_common,
+	.translate = uniphier_aidet_domain_translate,
+};
+
+static int uniphier_aidet_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *parent_np;
+	struct irq_domain *parent_domain;
+	struct uniphier_aidet_priv *priv;
+	struct resource *res;
+
+	parent_np = of_irq_find_parent(dev->of_node);
+	if (!parent_np)
+		return -ENXIO;
+
+	parent_domain = irq_find_host(parent_np);
+	of_node_put(parent_np);
+	if (!parent_domain)
+		return -EPROBE_DEFER;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	priv->reg_base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(priv->reg_base))
+		return PTR_ERR(priv->reg_base);
+
+	spin_lock_init(&priv->lock);
+
+	priv->domain = irq_domain_create_hierarchy(
+					parent_domain, 0,
+					UNIPHIER_AIDET_NR_IRQS,
+					of_node_to_fwnode(dev->of_node),
+					&uniphier_aidet_domain_ops, priv);
+	if (!priv->domain)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, priv);
+
+	return 0;
+}
+
+static int __maybe_unused uniphier_aidet_suspend(struct device *dev)
+{
+	struct uniphier_aidet_priv *priv = dev_get_drvdata(dev);
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(priv->saved_vals); i++)
+		priv->saved_vals[i] = readl_relaxed(
+			priv->reg_base + UNIPHIER_AIDET_DETCONF + i * 4);
+
+	return 0;
+}
+
+static int __maybe_unused uniphier_aidet_resume(struct device *dev)
+{
+	struct uniphier_aidet_priv *priv = dev_get_drvdata(dev);
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(priv->saved_vals); i++)
+		writel_relaxed(priv->saved_vals[i],
+			       priv->reg_base + UNIPHIER_AIDET_DETCONF + i * 4);
+
+	return 0;
+}
+
+static const struct dev_pm_ops uniphier_aidet_pm_ops = {
+	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(uniphier_aidet_suspend,
+				      uniphier_aidet_resume)
+};
+
+static const struct of_device_id uniphier_aidet_match[] = {
+	{ .compatible = "socionext,uniphier-ld4-aidet" },
+	{ .compatible = "socionext,uniphier-pro4-aidet" },
+	{ .compatible = "socionext,uniphier-sld8-aidet" },
+	{ .compatible = "socionext,uniphier-pro5-aidet" },
+	{ .compatible = "socionext,uniphier-pxs2-aidet" },
+	{ .compatible = "socionext,uniphier-ld11-aidet" },
+	{ .compatible = "socionext,uniphier-ld20-aidet" },
+	{ .compatible = "socionext,uniphier-pxs3-aidet" },
+	{ /* sentinel */ }
+};
+
+static struct platform_driver uniphier_aidet_driver = {
+	.probe = uniphier_aidet_probe,
+	.driver = {
+		.name = "uniphier-aidet",
+		.of_match_table = uniphier_aidet_match,
+		.pm = &uniphier_aidet_pm_ops,
+	},
+};
+builtin_platform_driver(uniphier_aidet_driver);
diff --git a/drivers/irqchip/irq-xilinx-intc.c b/drivers/irqchip/irq-xilinx-intc.c
index 3db7ab1c9741..e3043ded8973 100644
--- a/drivers/irqchip/irq-xilinx-intc.c
+++ b/drivers/irqchip/irq-xilinx-intc.c
@@ -186,8 +186,8 @@ static int __init xilinx_intc_of_init(struct device_node *intc,
 	if (irqc->intr_mask >> nr_irq)
 		pr_warn("irq-xilinx: mismatch in kind-of-intr param\n");
 
-	pr_info("irq-xilinx: %s: num_irq=%d, edge=0x%x\n",
-		intc->full_name, nr_irq, irqc->intr_mask);
+	pr_info("irq-xilinx: %pOF: num_irq=%d, edge=0x%x\n",
+		intc, nr_irq, irqc->intr_mask);
 
 
 	/*
diff --git a/drivers/irqchip/irq-xtensa-mx.c b/drivers/irqchip/irq-xtensa-mx.c
index 72a391e01011..a15a9510c904 100644
--- a/drivers/irqchip/irq-xtensa-mx.c
+++ b/drivers/irqchip/irq-xtensa-mx.c
@@ -32,6 +32,7 @@ static int xtensa_mx_irq_map(struct irq_domain *d, unsigned int irq,
 		irq_set_status_flags(irq, IRQ_LEVEL);
 		return 0;
 	}
+	irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq)));
 	return xtensa_irq_map(d, irq, hw);
 }
 
@@ -121,9 +122,12 @@ static int xtensa_mx_irq_retrigger(struct irq_data *d)
 static int xtensa_mx_irq_set_affinity(struct irq_data *d,
 		const struct cpumask *dest, bool force)
 {
-	unsigned mask = 1u << cpumask_any_and(dest, cpu_online_mask);
+	int cpu = cpumask_any_and(dest, cpu_online_mask);
+	unsigned mask = 1u << cpu;
 
 	set_er(mask, MIROUT(d->hwirq - HW_IRQ_MX_BASE));
+	irq_data_update_effective_affinity(d, cpumask_of(cpu));
+
 	return 0;
 
 }
diff --git a/drivers/isdn/capi/kcapi.c b/drivers/isdn/capi/kcapi.c
index 9ca691d6c13b..46c189ad8d94 100644
--- a/drivers/isdn/capi/kcapi.c
+++ b/drivers/isdn/capi/kcapi.c
@@ -55,7 +55,7 @@ struct capictr_event {
 
 /* ------------------------------------------------------------- */
 
-static struct capi_version driver_version = {2, 0, 1, 1 << 4};
+static const struct capi_version driver_version = {2, 0, 1, 1 << 4};
 static char driver_serial[CAPI_SERIAL_LEN] = "0004711";
 static char capi_manufakturer[64] = "AVM Berlin";
 
diff --git a/drivers/isdn/hardware/eicon/divacapi.h b/drivers/isdn/hardware/eicon/divacapi.h
index a315a2914d70..c4868a0d82f4 100644
--- a/drivers/isdn/hardware/eicon/divacapi.h
+++ b/drivers/isdn/hardware/eicon/divacapi.h
@@ -26,15 +26,7 @@
 
 /*#define DEBUG */
 
-
-
-
-
-
-
-
-
-
+#include <linux/types.h>
 
 #define IMPLEMENT_DTMF 1
 #define IMPLEMENT_LINE_INTERCONNECT2 1
@@ -82,8 +74,6 @@
 #define CODEC_PERMANENT    0x02
 #define ADV_VOICE          0x03
 #define MAX_CIP_TYPES      5  /* kind of CIP types for group optimization */
-#define C_IND_MASK_DWORDS  ((MAX_APPL + 32) >> 5)
-
 
 #define FAX_CONNECT_INFO_BUFFER_SIZE  256
 #define NCPI_BUFFER_SIZE              256
@@ -265,8 +255,8 @@ struct _PLCI {
 	word          ncci_ring_list;
 	byte          inc_dis_ncci_table[MAX_CHANNELS_PER_PLCI];
 	t_std_internal_command internal_command_queue[MAX_INTERNAL_COMMAND_LEVELS];
-	dword         c_ind_mask_table[C_IND_MASK_DWORDS];
-	dword         group_optimization_mask_table[C_IND_MASK_DWORDS];
+	DECLARE_BITMAP(c_ind_mask_table, MAX_APPL);
+	DECLARE_BITMAP(group_optimization_mask_table, MAX_APPL);
 	byte          RBuffer[200];
 	dword         msg_in_queue[MSG_IN_QUEUE_SIZE/sizeof(dword)];
 	API_SAVE      saved_msg;
diff --git a/drivers/isdn/hardware/eicon/message.c b/drivers/isdn/hardware/eicon/message.c
index 3b11422b1cce..eadd1ed1e014 100644
--- a/drivers/isdn/hardware/eicon/message.c
+++ b/drivers/isdn/hardware/eicon/message.c
@@ -23,9 +23,7 @@
  *
  */
 
-
-
-
+#include <linux/bitmap.h>
 
 #include "platform.h"
 #include "di_defs.h"
@@ -35,19 +33,9 @@
 #include "mdm_msg.h"
 #include "divasync.h"
 
-
-
 #define FILE_ "MESSAGE.C"
 #define dprintf
 
-
-
-
-
-
-
-
-
 /*------------------------------------------------------------------*/
 /* This is options supported for all adapters that are server by    */
 /* XDI driver. Allo it is not necessary to ask it from every adapter*/
@@ -72,9 +60,6 @@ static dword diva_xdi_extended_features = 0;
 /*------------------------------------------------------------------*/
 
 static void group_optimization(DIVA_CAPI_ADAPTER *a, PLCI *plci);
-static void set_group_ind_mask(PLCI *plci);
-static void clear_group_ind_mask_bit(PLCI *plci, word b);
-static byte test_group_ind_mask_bit(PLCI *plci, word b);
 void AutomaticLaw(DIVA_CAPI_ADAPTER *);
 word CapiRelease(word);
 word CapiRegister(word);
@@ -1087,106 +1072,6 @@ static void plci_remove(PLCI *plci)
 }
 
 /*------------------------------------------------------------------*/
-/* Application Group function helpers                               */
-/*------------------------------------------------------------------*/
-
-static void set_group_ind_mask(PLCI *plci)
-{
-	word i;
-
-	for (i = 0; i < C_IND_MASK_DWORDS; i++)
-		plci->group_optimization_mask_table[i] = 0xffffffffL;
-}
-
-static void clear_group_ind_mask_bit(PLCI *plci, word b)
-{
-	plci->group_optimization_mask_table[b >> 5] &= ~(1L << (b & 0x1f));
-}
-
-static byte test_group_ind_mask_bit(PLCI *plci, word b)
-{
-	return ((plci->group_optimization_mask_table[b >> 5] & (1L << (b & 0x1f))) != 0);
-}
-
-/*------------------------------------------------------------------*/
-/* c_ind_mask operations for arbitrary MAX_APPL                     */
-/*------------------------------------------------------------------*/
-
-static void clear_c_ind_mask(PLCI *plci)
-{
-	word i;
-
-	for (i = 0; i < C_IND_MASK_DWORDS; i++)
-		plci->c_ind_mask_table[i] = 0;
-}
-
-static byte c_ind_mask_empty(PLCI *plci)
-{
-	word i;
-
-	i = 0;
-	while ((i < C_IND_MASK_DWORDS) && (plci->c_ind_mask_table[i] == 0))
-		i++;
-	return (i == C_IND_MASK_DWORDS);
-}
-
-static void set_c_ind_mask_bit(PLCI *plci, word b)
-{
-	plci->c_ind_mask_table[b >> 5] |= (1L << (b & 0x1f));
-}
-
-static void clear_c_ind_mask_bit(PLCI *plci, word b)
-{
-	plci->c_ind_mask_table[b >> 5] &= ~(1L << (b & 0x1f));
-}
-
-static byte test_c_ind_mask_bit(PLCI *plci, word b)
-{
-	return ((plci->c_ind_mask_table[b >> 5] & (1L << (b & 0x1f))) != 0);
-}
-
-static void dump_c_ind_mask(PLCI *plci)
-{
-	word i, j, k;
-	dword d;
-	char *p;
-	char buf[40];
-
-	for (i = 0; i < C_IND_MASK_DWORDS; i += 4)
-	{
-		p = buf + 36;
-		*p = '\0';
-		for (j = 0; j < 4; j++)
-		{
-			if (i + j < C_IND_MASK_DWORDS)
-			{
-				d = plci->c_ind_mask_table[i + j];
-				for (k = 0; k < 8; k++)
-				{
-					*(--p) = hex_asc_lo(d);
-					d >>= 4;
-				}
-			}
-			else if (i != 0)
-			{
-				for (k = 0; k < 8; k++)
-					*(--p) = ' ';
-			}
-			*(--p) = ' ';
-		}
-		dbug(1, dprintf("c_ind_mask =%s", (char *) p));
-	}
-}
-
-
-
-
-
-#define dump_plcis(a)
-
-
-
-/*------------------------------------------------------------------*/
 /* translation function for each message                            */
 /*------------------------------------------------------------------*/
 
@@ -1457,13 +1342,13 @@ static byte connect_res(dword Id, word Number, DIVA_CAPI_ADAPTER *a,
 		return 1;
 	}
 	else if (plci->State == INC_CON_PENDING || plci->State == INC_CON_ALERT) {
-		clear_c_ind_mask_bit(plci, (word)(appl->Id - 1));
-		dump_c_ind_mask(plci);
+		__clear_bit(appl->Id - 1, plci->c_ind_mask_table);
+		dbug(1, dprintf("c_ind_mask =%*pb", MAX_APPL, plci->c_ind_mask_table));
 		Reject = GET_WORD(parms[0].info);
 		dbug(1, dprintf("Reject=0x%x", Reject));
 		if (Reject)
 		{
-			if (c_ind_mask_empty(plci))
+			if (bitmap_empty(plci->c_ind_mask_table, MAX_APPL))
 			{
 				if ((Reject & 0xff00) == 0x3400)
 				{
@@ -1553,11 +1438,8 @@ static byte connect_res(dword Id, word Number, DIVA_CAPI_ADAPTER *a,
 				sig_req(plci, CALL_RES, 0);
 			}
 
-			for (i = 0; i < max_appl; i++) {
-				if (test_c_ind_mask_bit(plci, i)) {
-					sendf(&application[i], _DISCONNECT_I, Id, 0, "w", _OTHER_APPL_CONNECTED);
-				}
-			}
+			for_each_set_bit(i, plci->c_ind_mask_table, max_appl)
+				sendf(&application[i], _DISCONNECT_I, Id, 0, "w", _OTHER_APPL_CONNECTED);
 		}
 	}
 	return 1;
@@ -1584,13 +1466,10 @@ static byte disconnect_req(dword Id, word Number, DIVA_CAPI_ADAPTER *a,
 	{
 		if (plci->State == INC_CON_PENDING || plci->State == INC_CON_ALERT)
 		{
-			clear_c_ind_mask_bit(plci, (word)(appl->Id - 1));
+			__clear_bit(appl->Id - 1, plci->c_ind_mask_table);
 			plci->appl = appl;
-			for (i = 0; i < max_appl; i++)
-			{
-				if (test_c_ind_mask_bit(plci, i))
-					sendf(&application[i], _DISCONNECT_I, Id, 0, "w", 0);
-			}
+			for_each_set_bit(i, plci->c_ind_mask_table, max_appl)
+				sendf(&application[i], _DISCONNECT_I, Id, 0, "w", 0);
 			plci->State = OUTG_DIS_PENDING;
 		}
 		if (plci->Sig.Id && plci->appl)
@@ -1634,7 +1513,7 @@ static byte disconnect_res(dword Id, word Number, DIVA_CAPI_ADAPTER *a,
 	{
 		/* clear ind mask bit, just in case of collsion of          */
 		/* DISCONNECT_IND and CONNECT_RES                           */
-		clear_c_ind_mask_bit(plci, (word)(appl->Id - 1));
+		__clear_bit(appl->Id - 1, plci->c_ind_mask_table);
 		ncci_free_receive_buffers(plci, 0);
 		if (plci_remove_check(plci))
 		{
@@ -1642,7 +1521,7 @@ static byte disconnect_res(dword Id, word Number, DIVA_CAPI_ADAPTER *a,
 		}
 		if (plci->State == INC_DIS_PENDING
 		    || plci->State == SUSPENDING) {
-			if (c_ind_mask_empty(plci)) {
+			if (bitmap_empty(plci->c_ind_mask_table, MAX_APPL)) {
 				if (plci->State != SUSPENDING) plci->State = IDLE;
 				dbug(1, dprintf("chs=%d", plci->channels));
 				if (!plci->channels) {
@@ -3351,13 +3230,11 @@ static byte select_b_req(dword Id, word Number, DIVA_CAPI_ADAPTER *a,
 				}
 				plci->State = INC_CON_CONNECTED_ALERT;
 				plci->appl = appl;
-				clear_c_ind_mask_bit(plci, (word)(appl->Id - 1));
-				dump_c_ind_mask(plci);
-				for (i = 0; i < max_appl; i++) /* disconnect the other appls */
-				{                         /* its quasi a connect        */
-					if (test_c_ind_mask_bit(plci, i))
-						sendf(&application[i], _DISCONNECT_I, Id, 0, "w", _OTHER_APPL_CONNECTED);
-				}
+				__clear_bit(appl->Id - 1, plci->c_ind_mask_table);
+				dbug(1, dprintf("c_ind_mask =%*pb", MAX_APPL, plci->c_ind_mask_table));
+				/* disconnect the other appls its quasi a connect */
+				for_each_set_bit(i, plci->c_ind_mask_table, max_appl)
+					sendf(&application[i], _DISCONNECT_I, Id, 0, "w", _OTHER_APPL_CONNECTED);
 			}
 
 			api_save_msg(msg, "s", &plci->saved_msg);
@@ -5692,19 +5569,17 @@ static void sig_ind(PLCI *plci)
 		cip = find_cip(a, parms[4], parms[6]);
 		cip_mask = 1L << cip;
 		dbug(1, dprintf("cip=%d,cip_mask=%lx", cip, cip_mask));
-		clear_c_ind_mask(plci);
+		bitmap_zero(plci->c_ind_mask_table, MAX_APPL);
 		if (!remove_started && !a->adapter_disabled)
 		{
-			set_c_ind_mask_bit(plci, MAX_APPL);
 			group_optimization(a, plci);
-			for (i = 0; i < max_appl; i++) {
+			for_each_set_bit(i, plci->group_optimization_mask_table, max_appl) {
 				if (application[i].Id
 				    && (a->CIP_Mask[i] & 1 || a->CIP_Mask[i] & cip_mask)
-				    && CPN_filter_ok(parms[0], a, i)
-				    && test_group_ind_mask_bit(plci, i)) {
+				    && CPN_filter_ok(parms[0], a, i)) {
 					dbug(1, dprintf("storedcip_mask[%d]=0x%lx", i, a->CIP_Mask[i]));
-					set_c_ind_mask_bit(plci, i);
-					dump_c_ind_mask(plci);
+					__set_bit(i, plci->c_ind_mask_table);
+					dbug(1, dprintf("c_ind_mask =%*pb", MAX_APPL, plci->c_ind_mask_table));
 					plci->State = INC_CON_PENDING;
 					plci->call_dir = (plci->call_dir & ~(CALL_DIR_OUT | CALL_DIR_ORIGINATE)) |
 						CALL_DIR_IN | CALL_DIR_ANSWER;
@@ -5750,10 +5625,9 @@ static void sig_ind(PLCI *plci)
 						      SendMultiIE(plci, Id, multi_pi_parms, PI, 0x210, true));
 				}
 			}
-			clear_c_ind_mask_bit(plci, MAX_APPL);
-			dump_c_ind_mask(plci);
+			dbug(1, dprintf("c_ind_mask =%*pb", MAX_APPL, plci->c_ind_mask_table));
 		}
-		if (c_ind_mask_empty(plci)) {
+		if (bitmap_empty(plci->c_ind_mask_table, MAX_APPL)) {
 			sig_req(plci, HANGUP, 0);
 			send_req(plci);
 			plci->State = IDLE;
@@ -5994,13 +5868,13 @@ static void sig_ind(PLCI *plci)
 		break;
 
 	case RESUME:
-		clear_c_ind_mask_bit(plci, (word)(plci->appl->Id - 1));
+		__clear_bit(plci->appl->Id - 1, plci->c_ind_mask_table);
 		PUT_WORD(&resume_cau[4], GOOD);
 		sendf(plci->appl, _FACILITY_I, Id, 0, "ws", (word)3, resume_cau);
 		break;
 
 	case SUSPEND:
-		clear_c_ind_mask(plci);
+		bitmap_zero(plci->c_ind_mask_table, MAX_APPL);
 
 		if (plci->NL.Id && !plci->nl_remove_id) {
 			mixer_remove(plci);
@@ -6037,15 +5911,12 @@ static void sig_ind(PLCI *plci)
 
 		if (plci->State == INC_CON_PENDING || plci->State == INC_CON_ALERT)
 		{
-			for (i = 0; i < max_appl; i++)
-			{
-				if (test_c_ind_mask_bit(plci, i))
-					sendf(&application[i], _DISCONNECT_I, Id, 0, "w", 0);
-			}
+			for_each_set_bit(i, plci->c_ind_mask_table, max_appl)
+				sendf(&application[i], _DISCONNECT_I, Id, 0, "w", 0);
 		}
 		else
 		{
-			clear_c_ind_mask(plci);
+			bitmap_zero(plci->c_ind_mask_table, MAX_APPL);
 		}
 		if (!plci->appl)
 		{
@@ -6055,7 +5926,7 @@ static void sig_ind(PLCI *plci)
 				a->listen_active--;
 			}
 			plci->State = INC_DIS_PENDING;
-			if (c_ind_mask_empty(plci))
+			if (bitmap_empty(plci->c_ind_mask_table, MAX_APPL))
 			{
 				plci->State = IDLE;
 				if (plci->NL.Id && !plci->nl_remove_id)
@@ -6341,14 +6212,10 @@ static void SendInfo(PLCI *plci, dword Id, byte **parms, byte iesent)
 			    || Info_Number == DSP
 			    || Info_Number == UUI)
 			{
-				for (j = 0; j < max_appl; j++)
-				{
-					if (test_c_ind_mask_bit(plci, j))
-					{
-						dbug(1, dprintf("Ovl_Ind"));
-						iesent = true;
-						sendf(&application[j], _INFO_I, Id, 0, "wS", Info_Number, Info_Element);
-					}
+				for_each_set_bit(j, plci->c_ind_mask_table, max_appl) {
+					dbug(1, dprintf("Ovl_Ind"));
+					iesent = true;
+					sendf(&application[j], _INFO_I, Id, 0, "wS", Info_Number, Info_Element);
 				}
 			}
 		}               /* all other signalling states */
@@ -6416,14 +6283,10 @@ static byte SendMultiIE(PLCI *plci, dword Id, byte **parms, byte ie_type,
 		}
 		else if (!plci->appl && Info_Number)
 		{                                        /* overlap receiving broadcast */
-			for (j = 0; j < max_appl; j++)
-			{
-				if (test_c_ind_mask_bit(plci, j))
-				{
-					iesent = true;
-					dbug(1, dprintf("Mlt_Ovl_Ind"));
-					sendf(&application[j] , _INFO_I, Id, 0, "wS", Info_Number, Info_Element);
-				}
+			for_each_set_bit(j, plci->c_ind_mask_table, max_appl) {
+				iesent = true;
+				dbug(1, dprintf("Mlt_Ovl_Ind"));
+				sendf(&application[j] , _INFO_I, Id, 0, "wS", Info_Number, Info_Element);
 			}
 		}                                        /* all other signalling states */
 		else if (Info_Number
@@ -7270,7 +7133,6 @@ static word get_plci(DIVA_CAPI_ADAPTER *a)
 	word i, j;
 	PLCI *plci;
 
-	dump_plcis(a);
 	for (i = 0; i < a->max_plci && a->plci[i].Id; i++);
 	if (i == a->max_plci) {
 		dbug(1, dprintf("get_plci: out of PLCIs"));
@@ -7321,8 +7183,8 @@ static word get_plci(DIVA_CAPI_ADAPTER *a)
 
 	plci->ncci_ring_list = 0;
 	for (j = 0; j < MAX_CHANNELS_PER_PLCI; j++) plci->inc_dis_ncci_table[j] = 0;
-	clear_c_ind_mask(plci);
-	set_group_ind_mask(plci);
+	bitmap_zero(plci->c_ind_mask_table, MAX_APPL);
+	bitmap_fill(plci->group_optimization_mask_table, MAX_APPL);
 	plci->fax_connect_info_length = 0;
 	plci->nsf_control_bits = 0;
 	plci->ncpi_state = 0x00;
@@ -9373,10 +9235,10 @@ word CapiRelease(word Id)
 					if (plci->State == INC_CON_PENDING
 					    || plci->State == INC_CON_ALERT)
 					{
-						if (test_c_ind_mask_bit(plci, (word)(Id - 1)))
+						if (test_bit(Id - 1, plci->c_ind_mask_table))
 						{
-							clear_c_ind_mask_bit(plci, (word)(Id - 1));
-							if (c_ind_mask_empty(plci))
+							__clear_bit(Id - 1, plci->c_ind_mask_table);
+							if (bitmap_empty(plci->c_ind_mask_table, MAX_APPL))
 							{
 								sig_req(plci, HANGUP, 0);
 								send_req(plci);
@@ -9384,10 +9246,10 @@ word CapiRelease(word Id)
 							}
 						}
 					}
-					if (test_c_ind_mask_bit(plci, (word)(Id - 1)))
+					if (test_bit(Id - 1, plci->c_ind_mask_table))
 					{
-						clear_c_ind_mask_bit(plci, (word)(Id - 1));
-						if (c_ind_mask_empty(plci))
+						__clear_bit(Id - 1, plci->c_ind_mask_table);
+						if (bitmap_empty(plci->c_ind_mask_table, MAX_APPL))
 						{
 							if (!plci->appl)
 							{
@@ -9452,7 +9314,7 @@ word CapiRelease(word Id)
 static word plci_remove_check(PLCI *plci)
 {
 	if (!plci) return true;
-	if (!plci->NL.Id && c_ind_mask_empty(plci))
+	if (!plci->NL.Id && bitmap_empty(plci->c_ind_mask_table, MAX_APPL))
 	{
 		if (plci->Sig.Id == 0xff)
 			plci->Sig.Id = 0;
@@ -14735,7 +14597,8 @@ static void group_optimization(DIVA_CAPI_ADAPTER *a, PLCI *plci)
 	word appl_number_group_type[MAX_APPL];
 	PLCI *auxplci;
 
-	set_group_ind_mask(plci); /* all APPLs within this inc. call are allowed to dial in */
+	/* all APPLs within this inc. call are allowed to dial in */
+	bitmap_fill(plci->group_optimization_mask_table, MAX_APPL);
 
 	if (!a->group_optimization_enabled)
 	{
@@ -14771,13 +14634,12 @@ static void group_optimization(DIVA_CAPI_ADAPTER *a, PLCI *plci)
 				if (a->plci[k].Id)
 				{
 					auxplci = &a->plci[k];
-					if (auxplci->appl == &application[i]) /* application has a busy PLCI */
-					{
+					if (auxplci->appl == &application[i]) {
+						/* application has a busy PLCI */
 						busy = true;
 						dbug(1, dprintf("Appl 0x%x is busy", i + 1));
-					}
-					else if (test_c_ind_mask_bit(auxplci, i)) /* application has an incoming call pending */
-					{
+					} else if (test_bit(i, plci->c_ind_mask_table)) {
+						/* application has an incoming call pending */
 						busy = true;
 						dbug(1, dprintf("Appl 0x%x has inc. call pending", i + 1));
 					}
@@ -14826,7 +14688,8 @@ static void group_optimization(DIVA_CAPI_ADAPTER *a, PLCI *plci)
 					if (appl_number_group_type[i] == appl_number_group_type[j])
 					{
 						dbug(1, dprintf("Appl 0x%x is member of group 0x%x, no call", j + 1, appl_number_group_type[j]));
-						clear_group_ind_mask_bit(plci, j);           /* disable call on other group members */
+						/* disable call on other group members */
+						__clear_bit(j, plci->group_optimization_mask_table);
 						appl_number_group_type[j] = 0;       /* remove disabled group member from group list */
 					}
 				}
@@ -14834,7 +14697,7 @@ static void group_optimization(DIVA_CAPI_ADAPTER *a, PLCI *plci)
 		}
 		else                                                 /* application should not get a call */
 		{
-			clear_group_ind_mask_bit(plci, i);
+			__clear_bit(i, plci->group_optimization_mask_table);
 		}
 	}
 
diff --git a/drivers/isdn/hardware/mISDN/hfcsusb.h b/drivers/isdn/hardware/mISDN/hfcsusb.h
index 4157311d569d..5f8f1d9cac11 100644
--- a/drivers/isdn/hardware/mISDN/hfcsusb.h
+++ b/drivers/isdn/hardware/mISDN/hfcsusb.h
@@ -337,7 +337,7 @@ static const char *HFC_NT_LAYER1_STATES[HFC_MAX_NT_LAYER1_STATE + 1] = {
 };
 
 /* supported devices */
-static struct usb_device_id hfcsusb_idtab[] = {
+static const struct usb_device_id hfcsusb_idtab[] = {
 	{
 		USB_DEVICE(0x0959, 0x2bd0),
 		.driver_info = (unsigned long) &((struct hfcsusb_vdata)
diff --git a/drivers/isdn/hisax/hfc_usb.c b/drivers/isdn/hisax/hfc_usb.c
index ef4748083efd..e8212185d386 100644
--- a/drivers/isdn/hisax/hfc_usb.c
+++ b/drivers/isdn/hisax/hfc_usb.c
@@ -65,7 +65,7 @@ typedef struct {
 } hfcsusb_vdata;
 
 /* VID/PID device list */
-static struct usb_device_id hfcusb_idtab[] = {
+static const struct usb_device_id hfcusb_idtab[] = {
 	{
 		USB_DEVICE(0x0959, 0x2bd0),
 		.driver_info = (unsigned long) &((hfcsusb_vdata)
diff --git a/drivers/isdn/isdnloop/isdnloop.c b/drivers/isdn/isdnloop/isdnloop.c
index 6ffd13466b8c..e97232646ba1 100644
--- a/drivers/isdn/isdnloop/isdnloop.c
+++ b/drivers/isdn/isdnloop/isdnloop.c
@@ -409,7 +409,7 @@ isdnloop_sendbuf(int channel, struct sk_buff *skb, isdnloop_card *card)
 		return -EINVAL;
 	}
 	if (len) {
-		if (!(card->flags & (channel) ? ISDNLOOP_FLAGS_B2ACTIVE : ISDNLOOP_FLAGS_B1ACTIVE))
+		if (!(card->flags & (channel ? ISDNLOOP_FLAGS_B2ACTIVE : ISDNLOOP_FLAGS_B1ACTIVE)))
 			return 0;
 		if (card->sndcount[channel] > ISDNLOOP_MAX_SQUEUE)
 			return 0;
diff --git a/drivers/isdn/mISDN/fsm.c b/drivers/isdn/mISDN/fsm.c
index 78fc5d5e9051..92e6570b1143 100644
--- a/drivers/isdn/mISDN/fsm.c
+++ b/drivers/isdn/mISDN/fsm.c
@@ -26,7 +26,7 @@
 
 #define FSM_TIMER_DEBUG 0
 
-void
+int
 mISDN_FsmNew(struct Fsm *fsm,
 	     struct FsmNode *fnlist, int fncount)
 {
@@ -34,6 +34,8 @@ mISDN_FsmNew(struct Fsm *fsm,
 
 	fsm->jumpmatrix = kzalloc(sizeof(FSMFNPTR) * fsm->state_count *
 				  fsm->event_count, GFP_KERNEL);
+	if (fsm->jumpmatrix == NULL)
+		return -ENOMEM;
 
 	for (i = 0; i < fncount; i++)
 		if ((fnlist[i].state >= fsm->state_count) ||
@@ -45,6 +47,7 @@ mISDN_FsmNew(struct Fsm *fsm,
 		} else
 			fsm->jumpmatrix[fsm->state_count * fnlist[i].event +
 					fnlist[i].state] = (FSMFNPTR) fnlist[i].routine;
+	return 0;
 }
 EXPORT_SYMBOL(mISDN_FsmNew);
 
diff --git a/drivers/isdn/mISDN/fsm.h b/drivers/isdn/mISDN/fsm.h
index 928f5be192c1..e1def8490221 100644
--- a/drivers/isdn/mISDN/fsm.h
+++ b/drivers/isdn/mISDN/fsm.h
@@ -55,7 +55,7 @@ struct FsmTimer {
 	void *arg;
 };
 
-extern void mISDN_FsmNew(struct Fsm *, struct FsmNode *, int);
+extern int mISDN_FsmNew(struct Fsm *, struct FsmNode *, int);
 extern void mISDN_FsmFree(struct Fsm *);
 extern int mISDN_FsmEvent(struct FsmInst *, int , void *);
 extern void mISDN_FsmChangeState(struct FsmInst *, int);
diff --git a/drivers/isdn/mISDN/layer1.c b/drivers/isdn/mISDN/layer1.c
index bebc57b72138..3192b0eb3944 100644
--- a/drivers/isdn/mISDN/layer1.c
+++ b/drivers/isdn/mISDN/layer1.c
@@ -414,8 +414,7 @@ l1_init(u_int *deb)
 	l1fsm_s.event_count = L1_EVENT_COUNT;
 	l1fsm_s.strEvent = strL1Event;
 	l1fsm_s.strState = strL1SState;
-	mISDN_FsmNew(&l1fsm_s, L1SFnList, ARRAY_SIZE(L1SFnList));
-	return 0;
+	return mISDN_FsmNew(&l1fsm_s, L1SFnList, ARRAY_SIZE(L1SFnList));
 }
 
 void
diff --git a/drivers/isdn/mISDN/layer2.c b/drivers/isdn/mISDN/layer2.c
index 7243a6746f8b..9ff0903a0e89 100644
--- a/drivers/isdn/mISDN/layer2.c
+++ b/drivers/isdn/mISDN/layer2.c
@@ -2247,15 +2247,26 @@ static struct Bprotocol X75SLP = {
 int
 Isdnl2_Init(u_int *deb)
 {
+	int res;
 	debug = deb;
 	mISDN_register_Bprotocol(&X75SLP);
 	l2fsm.state_count = L2_STATE_COUNT;
 	l2fsm.event_count = L2_EVENT_COUNT;
 	l2fsm.strEvent = strL2Event;
 	l2fsm.strState = strL2State;
-	mISDN_FsmNew(&l2fsm, L2FnList, ARRAY_SIZE(L2FnList));
-	TEIInit(deb);
+	res = mISDN_FsmNew(&l2fsm, L2FnList, ARRAY_SIZE(L2FnList));
+	if (res)
+		goto error;
+	res = TEIInit(deb);
+	if (res)
+		goto error_fsm;
 	return 0;
+
+error_fsm:
+	mISDN_FsmFree(&l2fsm);
+error:
+	mISDN_unregister_Bprotocol(&X75SLP);
+	return res;
 }
 
 void
diff --git a/drivers/isdn/mISDN/tei.c b/drivers/isdn/mISDN/tei.c
index 908127efccf8..12d9e5f4beb1 100644
--- a/drivers/isdn/mISDN/tei.c
+++ b/drivers/isdn/mISDN/tei.c
@@ -1387,23 +1387,37 @@ create_teimanager(struct mISDNdevice *dev)
 
 int TEIInit(u_int *deb)
 {
+	int res;
 	debug = deb;
 	teifsmu.state_count = TEI_STATE_COUNT;
 	teifsmu.event_count = TEI_EVENT_COUNT;
 	teifsmu.strEvent = strTeiEvent;
 	teifsmu.strState = strTeiState;
-	mISDN_FsmNew(&teifsmu, TeiFnListUser, ARRAY_SIZE(TeiFnListUser));
+	res = mISDN_FsmNew(&teifsmu, TeiFnListUser, ARRAY_SIZE(TeiFnListUser));
+	if (res)
+		goto error;
 	teifsmn.state_count = TEI_STATE_COUNT;
 	teifsmn.event_count = TEI_EVENT_COUNT;
 	teifsmn.strEvent = strTeiEvent;
 	teifsmn.strState = strTeiState;
-	mISDN_FsmNew(&teifsmn, TeiFnListNet, ARRAY_SIZE(TeiFnListNet));
+	res = mISDN_FsmNew(&teifsmn, TeiFnListNet, ARRAY_SIZE(TeiFnListNet));
+	if (res)
+		goto error_smn;
 	deactfsm.state_count =  DEACT_STATE_COUNT;
 	deactfsm.event_count = DEACT_EVENT_COUNT;
 	deactfsm.strEvent = strDeactEvent;
 	deactfsm.strState = strDeactState;
-	mISDN_FsmNew(&deactfsm, DeactFnList, ARRAY_SIZE(DeactFnList));
+	res = mISDN_FsmNew(&deactfsm, DeactFnList, ARRAY_SIZE(DeactFnList));
+	if (res)
+		goto error_deact;
 	return 0;
+
+error_deact:
+	mISDN_FsmFree(&teifsmn);
+error_smn:
+	mISDN_FsmFree(&teifsmu);
+error:
+	return res;
 }
 
 void TEIFree(void)
diff --git a/drivers/leds/Kconfig b/drivers/leds/Kconfig
index 594b24d410c3..52ea34e337cd 100644
--- a/drivers/leds/Kconfig
+++ b/drivers/leds/Kconfig
@@ -58,6 +58,15 @@ config LEDS_AAT1290
 	help
 	 This option enables support for the LEDs on the AAT1290.
 
+config LEDS_AS3645A
+	tristate "AS3645A LED flash controller support"
+	depends on I2C && LEDS_CLASS_FLASH
+	depends on V4L2_FLASH_LED_CLASS || !V4L2_FLASH_LED_CLASS
+	help
+	  Enable LED flash class support for AS3645A LED flash
+	  controller. V4L2 flash API is provided as well if
+	  CONFIG_V4L2_FLASH_API is enabled.
+
 config LEDS_BCM6328
 	tristate "LED Support for Broadcom BCM6328"
 	depends on LEDS_CLASS
@@ -377,6 +386,17 @@ config LEDS_PCA955X
 	  LED driver chips accessed via the I2C bus.  Supported
 	  devices include PCA9550, PCA9551, PCA9552, and PCA9553.
 
+config LEDS_PCA955X_GPIO
+	bool "Enable GPIO support for PCA955X"
+	depends on LEDS_PCA955X
+	depends on GPIOLIB
+	help
+	  Allow unused pins on PCA955X to be used as gpio.
+
+	  To use a pin as gpio the pin type should be set to
+	  PCA955X_TYPE_GPIO in the device tree.
+
+
 config LEDS_PCA963X
 	tristate "LED support for PCA963x I2C chip"
 	depends on LEDS_CLASS
diff --git a/drivers/leds/Makefile b/drivers/leds/Makefile
index 909dae62ba05..7d7b26552923 100644
--- a/drivers/leds/Makefile
+++ b/drivers/leds/Makefile
@@ -8,6 +8,7 @@ obj-$(CONFIG_LEDS_TRIGGERS)		+= led-triggers.o
 # LED Platform Drivers
 obj-$(CONFIG_LEDS_88PM860X)		+= leds-88pm860x.o
 obj-$(CONFIG_LEDS_AAT1290)		+= leds-aat1290.o
+obj-$(CONFIG_LEDS_AS3645A)		+= leds-as3645a.o
 obj-$(CONFIG_LEDS_BCM6328)		+= leds-bcm6328.o
 obj-$(CONFIG_LEDS_BCM6358)		+= leds-bcm6358.o
 obj-$(CONFIG_LEDS_BD2802)		+= leds-bd2802.o
diff --git a/drivers/leds/leds-aat1290.c b/drivers/leds/leds-aat1290.c
index a21e19297745..43bd8a43f36c 100644
--- a/drivers/leds/leds-aat1290.c
+++ b/drivers/leds/leds-aat1290.c
@@ -314,8 +314,10 @@ static void aat1290_led_validate_mm_current(struct aat1290_led *led,
 static int init_mm_current_scale(struct aat1290_led *led,
 			struct aat1290_led_config_data *cfg)
 {
-	int max_mm_current_percent[] = { 20, 22, 25, 28, 32, 36, 40, 45, 50, 56,
-						63, 71, 79, 89, 100 };
+	static const int max_mm_current_percent[] = {
+		20, 22, 25, 28, 32, 36, 40, 45, 50, 56,
+		63, 71, 79, 89, 100
+	};
 	int i, max_mm_current =
 			AAT1290_MAX_MM_CURRENT(cfg->max_flash_current);
 
@@ -432,7 +434,7 @@ static void aat1290_init_v4l2_flash_config(struct aat1290_led *led,
 	strlcpy(v4l2_sd_cfg->dev_name, led_cdev->name,
 		sizeof(v4l2_sd_cfg->dev_name));
 
-	s = &v4l2_sd_cfg->torch_intensity;
+	s = &v4l2_sd_cfg->intensity;
 	s->min = led->mm_current_scale[0];
 	s->max = led_cfg->max_mm_current;
 	s->step = 1;
@@ -504,7 +506,7 @@ static int aat1290_led_probe(struct platform_device *pdev)
 
 	/* Create V4L2 Flash subdev. */
 	led->v4l2_flash = v4l2_flash_init(dev, of_fwnode_handle(sub_node),
-					  fled_cdev, NULL, &v4l2_flash_ops,
+					  fled_cdev, &v4l2_flash_ops,
 					  &v4l2_sd_cfg);
 	if (IS_ERR(led->v4l2_flash)) {
 		ret = PTR_ERR(led->v4l2_flash);
diff --git a/drivers/leds/leds-as3645a.c b/drivers/leds/leds-as3645a.c
new file mode 100644
index 000000000000..bbbbe0898233
--- /dev/null
+++ b/drivers/leds/leds-as3645a.c
@@ -0,0 +1,763 @@
+/*
+ * drivers/leds/leds-as3645a.c - AS3645A and LM3555 flash controllers driver
+ *
+ * Copyright (C) 2008-2011 Nokia Corporation
+ * Copyright (c) 2011, 2017 Intel Corporation.
+ *
+ * Based on drivers/media/i2c/as3645a.c.
+ *
+ * Contact: Sakari Ailus <sakari.ailus@iki.fi>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/led-class-flash.h>
+#include <linux/leds.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/slab.h>
+
+#include <media/v4l2-flash-led-class.h>
+
+#define AS_TIMER_US_TO_CODE(t)			(((t) / 1000 - 100) / 50)
+#define AS_TIMER_CODE_TO_US(c)			((50 * (c) + 100) * 1000)
+
+/* Register definitions */
+
+/* Read-only Design info register: Reset state: xxxx 0001 */
+#define AS_DESIGN_INFO_REG			0x00
+#define AS_DESIGN_INFO_FACTORY(x)		(((x) >> 4))
+#define AS_DESIGN_INFO_MODEL(x)			((x) & 0x0f)
+
+/* Read-only Version control register: Reset state: 0000 0000
+ * for first engineering samples
+ */
+#define AS_VERSION_CONTROL_REG			0x01
+#define AS_VERSION_CONTROL_RFU(x)		(((x) >> 4))
+#define AS_VERSION_CONTROL_VERSION(x)		((x) & 0x0f)
+
+/* Read / Write	(Indicator and timer register): Reset state: 0000 1111 */
+#define AS_INDICATOR_AND_TIMER_REG		0x02
+#define AS_INDICATOR_AND_TIMER_TIMEOUT_SHIFT	0
+#define AS_INDICATOR_AND_TIMER_VREF_SHIFT	4
+#define AS_INDICATOR_AND_TIMER_INDICATOR_SHIFT	6
+
+/* Read / Write	(Current set register): Reset state: 0110 1001 */
+#define AS_CURRENT_SET_REG			0x03
+#define AS_CURRENT_ASSIST_LIGHT_SHIFT		0
+#define AS_CURRENT_LED_DET_ON			(1 << 3)
+#define AS_CURRENT_FLASH_CURRENT_SHIFT		4
+
+/* Read / Write	(Control register): Reset state: 1011 0100 */
+#define AS_CONTROL_REG				0x04
+#define AS_CONTROL_MODE_SETTING_SHIFT		0
+#define AS_CONTROL_STROBE_ON			(1 << 2)
+#define AS_CONTROL_OUT_ON			(1 << 3)
+#define AS_CONTROL_EXT_TORCH_ON			(1 << 4)
+#define AS_CONTROL_STROBE_TYPE_EDGE		(0 << 5)
+#define AS_CONTROL_STROBE_TYPE_LEVEL		(1 << 5)
+#define AS_CONTROL_COIL_PEAK_SHIFT		6
+
+/* Read only (D3 is read / write) (Fault and info): Reset state: 0000 x000 */
+#define AS_FAULT_INFO_REG			0x05
+#define AS_FAULT_INFO_INDUCTOR_PEAK_LIMIT	(1 << 1)
+#define AS_FAULT_INFO_INDICATOR_LED		(1 << 2)
+#define AS_FAULT_INFO_LED_AMOUNT		(1 << 3)
+#define AS_FAULT_INFO_TIMEOUT			(1 << 4)
+#define AS_FAULT_INFO_OVER_TEMPERATURE		(1 << 5)
+#define AS_FAULT_INFO_SHORT_CIRCUIT		(1 << 6)
+#define AS_FAULT_INFO_OVER_VOLTAGE		(1 << 7)
+
+/* Boost register */
+#define AS_BOOST_REG				0x0d
+#define AS_BOOST_CURRENT_DISABLE		(0 << 0)
+#define AS_BOOST_CURRENT_ENABLE			(1 << 0)
+
+/* Password register is used to unlock boost register writing */
+#define AS_PASSWORD_REG				0x0f
+#define AS_PASSWORD_UNLOCK_VALUE		0x55
+
+#define AS_NAME					"as3645a"
+#define AS_I2C_ADDR				(0x60 >> 1) /* W:0x60, R:0x61 */
+
+#define AS_FLASH_TIMEOUT_MIN			100000	/* us */
+#define AS_FLASH_TIMEOUT_MAX			850000
+#define AS_FLASH_TIMEOUT_STEP			50000
+
+#define AS_FLASH_INTENSITY_MIN			200000	/* uA */
+#define AS_FLASH_INTENSITY_MAX_1LED		500000
+#define AS_FLASH_INTENSITY_MAX_2LEDS		400000
+#define AS_FLASH_INTENSITY_STEP			20000
+
+#define AS_TORCH_INTENSITY_MIN			20000	/* uA */
+#define AS_TORCH_INTENSITY_MAX			160000
+#define AS_TORCH_INTENSITY_STEP			20000
+
+#define AS_INDICATOR_INTENSITY_MIN		0	/* uA */
+#define AS_INDICATOR_INTENSITY_MAX		10000
+#define AS_INDICATOR_INTENSITY_STEP		2500
+
+#define AS_PEAK_mA_MAX				2000
+#define AS_PEAK_mA_TO_REG(a) \
+	((min_t(u32, AS_PEAK_mA_MAX, a) - 1250) / 250)
+
+enum as_mode {
+	AS_MODE_EXT_TORCH = 0 << AS_CONTROL_MODE_SETTING_SHIFT,
+	AS_MODE_INDICATOR = 1 << AS_CONTROL_MODE_SETTING_SHIFT,
+	AS_MODE_ASSIST = 2 << AS_CONTROL_MODE_SETTING_SHIFT,
+	AS_MODE_FLASH = 3 << AS_CONTROL_MODE_SETTING_SHIFT,
+};
+
+struct as3645a_config {
+	u32 flash_timeout_us;
+	u32 flash_max_ua;
+	u32 assist_max_ua;
+	u32 indicator_max_ua;
+	u32 voltage_reference;
+	u32 peak;
+};
+
+struct as3645a_names {
+	char flash[32];
+	char indicator[32];
+};
+
+struct as3645a {
+	struct i2c_client *client;
+
+	struct mutex mutex;
+
+	struct led_classdev_flash fled;
+	struct led_classdev iled_cdev;
+
+	struct v4l2_flash *vf;
+	struct v4l2_flash *vfind;
+
+	struct device_node *flash_node;
+	struct device_node *indicator_node;
+
+	struct as3645a_config cfg;
+
+	enum as_mode mode;
+	unsigned int timeout;
+	unsigned int flash_current;
+	unsigned int assist_current;
+	unsigned int indicator_current;
+	enum v4l2_flash_strobe_source strobe_source;
+};
+
+#define fled_to_as3645a(__fled) container_of(__fled, struct as3645a, fled)
+#define iled_cdev_to_as3645a(__iled_cdev) \
+	container_of(__iled_cdev, struct as3645a, iled_cdev)
+
+/* Return negative errno else zero on success */
+static int as3645a_write(struct as3645a *flash, u8 addr, u8 val)
+{
+	struct i2c_client *client = flash->client;
+	int rval;
+
+	rval = i2c_smbus_write_byte_data(client, addr, val);
+
+	dev_dbg(&client->dev, "Write Addr:%02X Val:%02X %s\n", addr, val,
+		rval < 0 ? "fail" : "ok");
+
+	return rval;
+}
+
+/* Return negative errno else a data byte received from the device. */
+static int as3645a_read(struct as3645a *flash, u8 addr)
+{
+	struct i2c_client *client = flash->client;
+	int rval;
+
+	rval = i2c_smbus_read_byte_data(client, addr);
+
+	dev_dbg(&client->dev, "Read Addr:%02X Val:%02X %s\n", addr, rval,
+		rval < 0 ? "fail" : "ok");
+
+	return rval;
+}
+
+/* -----------------------------------------------------------------------------
+ * Hardware configuration and trigger
+ */
+
+/**
+ * as3645a_set_config - Set flash configuration registers
+ * @flash: The flash
+ *
+ * Configure the hardware with flash, assist and indicator currents, as well as
+ * flash timeout.
+ *
+ * Return 0 on success, or a negative error code if an I2C communication error
+ * occurred.
+ */
+static int as3645a_set_current(struct as3645a *flash)
+{
+	u8 val;
+
+	val = (flash->flash_current << AS_CURRENT_FLASH_CURRENT_SHIFT)
+	    | (flash->assist_current << AS_CURRENT_ASSIST_LIGHT_SHIFT)
+	    | AS_CURRENT_LED_DET_ON;
+
+	return as3645a_write(flash, AS_CURRENT_SET_REG, val);
+}
+
+static int as3645a_set_timeout(struct as3645a *flash)
+{
+	u8 val;
+
+	val = flash->timeout << AS_INDICATOR_AND_TIMER_TIMEOUT_SHIFT;
+
+	val |= (flash->cfg.voltage_reference
+		<< AS_INDICATOR_AND_TIMER_VREF_SHIFT)
+	    |  ((flash->indicator_current ? flash->indicator_current - 1 : 0)
+		 << AS_INDICATOR_AND_TIMER_INDICATOR_SHIFT);
+
+	return as3645a_write(flash, AS_INDICATOR_AND_TIMER_REG, val);
+}
+
+/**
+ * as3645a_set_control - Set flash control register
+ * @flash: The flash
+ * @mode: Desired output mode
+ * @on: Desired output state
+ *
+ * Configure the hardware with output mode and state.
+ *
+ * Return 0 on success, or a negative error code if an I2C communication error
+ * occurred.
+ */
+static int
+as3645a_set_control(struct as3645a *flash, enum as_mode mode, bool on)
+{
+	u8 reg;
+
+	/* Configure output parameters and operation mode. */
+	reg = (flash->cfg.peak << AS_CONTROL_COIL_PEAK_SHIFT)
+	    | (on ? AS_CONTROL_OUT_ON : 0)
+	    | mode;
+
+	if (mode == AS_MODE_FLASH &&
+	    flash->strobe_source == V4L2_FLASH_STROBE_SOURCE_EXTERNAL)
+		reg |= AS_CONTROL_STROBE_TYPE_LEVEL
+		    |  AS_CONTROL_STROBE_ON;
+
+	return as3645a_write(flash, AS_CONTROL_REG, reg);
+}
+
+static int as3645a_get_fault(struct led_classdev_flash *fled, u32 *fault)
+{
+	struct as3645a *flash = fled_to_as3645a(fled);
+	int rval;
+
+	/* NOTE: reading register clears fault status */
+	rval = as3645a_read(flash, AS_FAULT_INFO_REG);
+	if (rval < 0)
+		return rval;
+
+	if (rval & AS_FAULT_INFO_INDUCTOR_PEAK_LIMIT)
+		*fault |= LED_FAULT_OVER_CURRENT;
+
+	if (rval & AS_FAULT_INFO_INDICATOR_LED)
+		*fault |= LED_FAULT_INDICATOR;
+
+	dev_dbg(&flash->client->dev, "%u connected LEDs\n",
+		rval & AS_FAULT_INFO_LED_AMOUNT ? 2 : 1);
+
+	if (rval & AS_FAULT_INFO_TIMEOUT)
+		*fault |= LED_FAULT_TIMEOUT;
+
+	if (rval & AS_FAULT_INFO_OVER_TEMPERATURE)
+		*fault |= LED_FAULT_OVER_TEMPERATURE;
+
+	if (rval & AS_FAULT_INFO_SHORT_CIRCUIT)
+		*fault |= LED_FAULT_OVER_CURRENT;
+
+	if (rval & AS_FAULT_INFO_OVER_VOLTAGE)
+		*fault |= LED_FAULT_INPUT_VOLTAGE;
+
+	return rval;
+}
+
+static unsigned int __as3645a_current_to_reg(unsigned int min, unsigned int max,
+					     unsigned int step,
+					     unsigned int val)
+{
+	if (val < min)
+		val = min;
+
+	if (val > max)
+		val = max;
+
+	return (val - min) / step;
+}
+
+static unsigned int as3645a_current_to_reg(struct as3645a *flash, bool is_flash,
+					   unsigned int ua)
+{
+	if (is_flash)
+		return __as3645a_current_to_reg(AS_TORCH_INTENSITY_MIN,
+						flash->cfg.assist_max_ua,
+						AS_TORCH_INTENSITY_STEP, ua);
+	else
+		return __as3645a_current_to_reg(AS_FLASH_INTENSITY_MIN,
+						flash->cfg.flash_max_ua,
+						AS_FLASH_INTENSITY_STEP, ua);
+}
+
+static int as3645a_set_indicator_brightness(struct led_classdev *iled_cdev,
+					    enum led_brightness brightness)
+{
+	struct as3645a *flash = iled_cdev_to_as3645a(iled_cdev);
+	int rval;
+
+	flash->indicator_current = brightness;
+
+	rval = as3645a_set_timeout(flash);
+	if (rval)
+		return rval;
+
+	return as3645a_set_control(flash, AS_MODE_INDICATOR, brightness);
+}
+
+static int as3645a_set_assist_brightness(struct led_classdev *fled_cdev,
+					 enum led_brightness brightness)
+{
+	struct led_classdev_flash *fled = lcdev_to_flcdev(fled_cdev);
+	struct as3645a *flash = fled_to_as3645a(fled);
+	int rval;
+
+	if (brightness) {
+		/* Register value 0 is 20 mA. */
+		flash->assist_current = brightness - 1;
+
+		rval = as3645a_set_current(flash);
+		if (rval)
+			return rval;
+	}
+
+	return as3645a_set_control(flash, AS_MODE_ASSIST, brightness);
+}
+
+static int as3645a_set_flash_brightness(struct led_classdev_flash *fled,
+					u32 brightness_ua)
+{
+	struct as3645a *flash = fled_to_as3645a(fled);
+
+	flash->flash_current = as3645a_current_to_reg(flash, true, brightness_ua);
+
+	return as3645a_set_current(flash);
+}
+
+static int as3645a_set_flash_timeout(struct led_classdev_flash *fled,
+				     u32 timeout_us)
+{
+	struct as3645a *flash = fled_to_as3645a(fled);
+
+	flash->timeout = AS_TIMER_US_TO_CODE(timeout_us);
+
+	return as3645a_set_timeout(flash);
+}
+
+static int as3645a_set_strobe(struct led_classdev_flash *fled, bool state)
+{
+	struct as3645a *flash = fled_to_as3645a(fled);
+
+	return as3645a_set_control(flash, AS_MODE_FLASH, state);
+}
+
+static const struct led_flash_ops as3645a_led_flash_ops = {
+	.flash_brightness_set = as3645a_set_flash_brightness,
+	.timeout_set = as3645a_set_flash_timeout,
+	.strobe_set = as3645a_set_strobe,
+	.fault_get = as3645a_get_fault,
+};
+
+static int as3645a_setup(struct as3645a *flash)
+{
+	struct device *dev = &flash->client->dev;
+	u32 fault = 0;
+	int rval;
+
+	/* clear errors */
+	rval = as3645a_read(flash, AS_FAULT_INFO_REG);
+	if (rval < 0)
+		return rval;
+
+	dev_dbg(dev, "Fault info: %02x\n", rval);
+
+	rval = as3645a_set_current(flash);
+	if (rval < 0)
+		return rval;
+
+	rval = as3645a_set_timeout(flash);
+	if (rval < 0)
+		return rval;
+
+	rval = as3645a_set_control(flash, AS_MODE_INDICATOR, false);
+	if (rval < 0)
+		return rval;
+
+	/* read status */
+	rval = as3645a_get_fault(&flash->fled, &fault);
+	if (rval < 0)
+		return rval;
+
+	dev_dbg(dev, "AS_INDICATOR_AND_TIMER_REG: %02x\n",
+		as3645a_read(flash, AS_INDICATOR_AND_TIMER_REG));
+	dev_dbg(dev, "AS_CURRENT_SET_REG: %02x\n",
+		as3645a_read(flash, AS_CURRENT_SET_REG));
+	dev_dbg(dev, "AS_CONTROL_REG: %02x\n",
+		as3645a_read(flash, AS_CONTROL_REG));
+
+	return rval & ~AS_FAULT_INFO_LED_AMOUNT ? -EIO : 0;
+}
+
+static int as3645a_detect(struct as3645a *flash)
+{
+	struct device *dev = &flash->client->dev;
+	int rval, man, model, rfu, version;
+	const char *vendor;
+
+	rval = as3645a_read(flash, AS_DESIGN_INFO_REG);
+	if (rval < 0) {
+		dev_err(dev, "can't read design info reg\n");
+		return rval;
+	}
+
+	man = AS_DESIGN_INFO_FACTORY(rval);
+	model = AS_DESIGN_INFO_MODEL(rval);
+
+	rval = as3645a_read(flash, AS_VERSION_CONTROL_REG);
+	if (rval < 0) {
+		dev_err(dev, "can't read version control reg\n");
+		return rval;
+	}
+
+	rfu = AS_VERSION_CONTROL_RFU(rval);
+	version = AS_VERSION_CONTROL_VERSION(rval);
+
+	/* Verify the chip model and version. */
+	if (model != 0x01 || rfu != 0x00) {
+		dev_err(dev, "AS3645A not detected "
+			"(model %d rfu %d)\n", model, rfu);
+		return -ENODEV;
+	}
+
+	switch (man) {
+	case 1:
+		vendor = "AMS, Austria Micro Systems";
+		break;
+	case 2:
+		vendor = "ADI, Analog Devices Inc.";
+		break;
+	case 3:
+		vendor = "NSC, National Semiconductor";
+		break;
+	case 4:
+		vendor = "NXP";
+		break;
+	case 5:
+		vendor = "TI, Texas Instrument";
+		break;
+	default:
+		vendor = "Unknown";
+	}
+
+	dev_info(dev, "Chip vendor: %s (%d) Version: %d\n", vendor,
+		 man, version);
+
+	rval = as3645a_write(flash, AS_PASSWORD_REG, AS_PASSWORD_UNLOCK_VALUE);
+	if (rval < 0)
+		return rval;
+
+	return as3645a_write(flash, AS_BOOST_REG, AS_BOOST_CURRENT_DISABLE);
+}
+
+static int as3645a_parse_node(struct as3645a *flash,
+			      struct as3645a_names *names,
+			      struct device_node *node)
+{
+	struct as3645a_config *cfg = &flash->cfg;
+	const char *name;
+	int rval;
+
+	flash->flash_node = of_get_child_by_name(node, "flash");
+	if (!flash->flash_node) {
+		dev_err(&flash->client->dev, "can't find flash node\n");
+		return -ENODEV;
+	}
+
+	rval = of_property_read_string(flash->flash_node, "label", &name);
+	if (!rval)
+		strlcpy(names->flash, name, sizeof(names->flash));
+	else
+		snprintf(names->flash, sizeof(names->flash),
+			 "%s:flash", node->name);
+
+	rval = of_property_read_u32(flash->flash_node, "flash-timeout-us",
+				    &cfg->flash_timeout_us);
+	if (rval < 0) {
+		dev_err(&flash->client->dev,
+			"can't read flash-timeout-us property for flash\n");
+		goto out_err;
+	}
+
+	rval = of_property_read_u32(flash->flash_node, "flash-max-microamp",
+				    &cfg->flash_max_ua);
+	if (rval < 0) {
+		dev_err(&flash->client->dev,
+			"can't read flash-max-microamp property for flash\n");
+		goto out_err;
+	}
+
+	rval = of_property_read_u32(flash->flash_node, "led-max-microamp",
+				    &cfg->assist_max_ua);
+	if (rval < 0) {
+		dev_err(&flash->client->dev,
+			"can't read led-max-microamp property for flash\n");
+		goto out_err;
+	}
+
+	of_property_read_u32(flash->flash_node, "voltage-reference",
+			     &cfg->voltage_reference);
+
+	of_property_read_u32(flash->flash_node, "peak-current-limit",
+			     &cfg->peak);
+	cfg->peak = AS_PEAK_mA_TO_REG(cfg->peak);
+
+	flash->indicator_node = of_get_child_by_name(node, "indicator");
+	if (!flash->indicator_node) {
+		dev_warn(&flash->client->dev,
+			 "can't find indicator node\n");
+		goto out_err;
+	}
+
+	rval = of_property_read_string(flash->indicator_node, "label", &name);
+	if (!rval)
+		strlcpy(names->indicator, name, sizeof(names->indicator));
+	else
+		snprintf(names->indicator, sizeof(names->indicator),
+			 "%s:indicator", node->name);
+
+	rval = of_property_read_u32(flash->indicator_node, "led-max-microamp",
+				    &cfg->indicator_max_ua);
+	if (rval < 0) {
+		dev_err(&flash->client->dev,
+			"can't read led-max-microamp property for indicator\n");
+		goto out_err;
+	}
+
+	return 0;
+
+out_err:
+	of_node_put(flash->flash_node);
+	of_node_put(flash->indicator_node);
+
+	return rval;
+}
+
+static int as3645a_led_class_setup(struct as3645a *flash,
+				   struct as3645a_names *names)
+{
+	struct led_classdev *fled_cdev = &flash->fled.led_cdev;
+	struct led_classdev *iled_cdev = &flash->iled_cdev;
+	struct led_flash_setting *cfg;
+	int rval;
+
+	iled_cdev->name = names->indicator;
+	iled_cdev->brightness_set_blocking = as3645a_set_indicator_brightness;
+	iled_cdev->max_brightness =
+		flash->cfg.indicator_max_ua / AS_INDICATOR_INTENSITY_STEP;
+	iled_cdev->flags = LED_CORE_SUSPENDRESUME;
+
+	rval = led_classdev_register(&flash->client->dev, iled_cdev);
+	if (rval < 0)
+		return rval;
+
+	cfg = &flash->fled.brightness;
+	cfg->min = AS_FLASH_INTENSITY_MIN;
+	cfg->max = flash->cfg.flash_max_ua;
+	cfg->step = AS_FLASH_INTENSITY_STEP;
+	cfg->val = flash->cfg.flash_max_ua;
+
+	cfg = &flash->fled.timeout;
+	cfg->min = AS_FLASH_TIMEOUT_MIN;
+	cfg->max = flash->cfg.flash_timeout_us;
+	cfg->step = AS_FLASH_TIMEOUT_STEP;
+	cfg->val = flash->cfg.flash_timeout_us;
+
+	flash->fled.ops = &as3645a_led_flash_ops;
+
+	fled_cdev->name = names->flash;
+	fled_cdev->brightness_set_blocking = as3645a_set_assist_brightness;
+	/* Value 0 is off in LED class. */
+	fled_cdev->max_brightness =
+		as3645a_current_to_reg(flash, false,
+				       flash->cfg.assist_max_ua) + 1;
+	fled_cdev->flags = LED_DEV_CAP_FLASH | LED_CORE_SUSPENDRESUME;
+
+	rval = led_classdev_flash_register(&flash->client->dev, &flash->fled);
+	if (rval) {
+		led_classdev_unregister(iled_cdev);
+		dev_err(&flash->client->dev,
+			"led_classdev_flash_register() failed, error %d\n",
+			rval);
+	}
+
+	return rval;
+}
+
+static int as3645a_v4l2_setup(struct as3645a *flash)
+{
+	struct led_classdev_flash *fled = &flash->fled;
+	struct led_classdev *led = &fled->led_cdev;
+	struct v4l2_flash_config cfg = {
+		.intensity = {
+			.min = AS_TORCH_INTENSITY_MIN,
+			.max = flash->cfg.assist_max_ua,
+			.step = AS_TORCH_INTENSITY_STEP,
+			.val = flash->cfg.assist_max_ua,
+		},
+	};
+	struct v4l2_flash_config cfgind = {
+		.intensity = {
+			.min = AS_INDICATOR_INTENSITY_MIN,
+			.max = flash->cfg.indicator_max_ua,
+			.step = AS_INDICATOR_INTENSITY_STEP,
+			.val = flash->cfg.indicator_max_ua,
+		},
+	};
+
+	strlcpy(cfg.dev_name, led->name, sizeof(cfg.dev_name));
+	strlcpy(cfgind.dev_name, flash->iled_cdev.name, sizeof(cfg.dev_name));
+
+	flash->vf = v4l2_flash_init(
+		&flash->client->dev, of_fwnode_handle(flash->flash_node),
+		&flash->fled, NULL, &cfg);
+	if (IS_ERR(flash->vf))
+		return PTR_ERR(flash->vf);
+
+	flash->vfind = v4l2_flash_indicator_init(
+		&flash->client->dev, of_fwnode_handle(flash->indicator_node),
+		&flash->iled_cdev, &cfgind);
+	if (IS_ERR(flash->vfind)) {
+		v4l2_flash_release(flash->vf);
+		return PTR_ERR(flash->vfind);
+	}
+
+	return 0;
+}
+
+static int as3645a_probe(struct i2c_client *client)
+{
+	struct as3645a_names names;
+	struct as3645a *flash;
+	int rval;
+
+	if (client->dev.of_node == NULL)
+		return -ENODEV;
+
+	flash = devm_kzalloc(&client->dev, sizeof(*flash), GFP_KERNEL);
+	if (flash == NULL)
+		return -ENOMEM;
+
+	flash->client = client;
+
+	rval = as3645a_parse_node(flash, &names, client->dev.of_node);
+	if (rval < 0)
+		return rval;
+
+	rval = as3645a_detect(flash);
+	if (rval < 0)
+		goto out_put_nodes;
+
+	mutex_init(&flash->mutex);
+	i2c_set_clientdata(client, flash);
+
+	rval = as3645a_setup(flash);
+	if (rval)
+		goto out_mutex_destroy;
+
+	rval = as3645a_led_class_setup(flash, &names);
+	if (rval)
+		goto out_mutex_destroy;
+
+	rval = as3645a_v4l2_setup(flash);
+	if (rval)
+		goto out_led_classdev_flash_unregister;
+
+	return 0;
+
+out_led_classdev_flash_unregister:
+	led_classdev_flash_unregister(&flash->fled);
+
+out_mutex_destroy:
+	mutex_destroy(&flash->mutex);
+
+out_put_nodes:
+	of_node_put(flash->flash_node);
+	of_node_put(flash->indicator_node);
+
+	return rval;
+}
+
+static int as3645a_remove(struct i2c_client *client)
+{
+	struct as3645a *flash = i2c_get_clientdata(client);
+
+	as3645a_set_control(flash, AS_MODE_EXT_TORCH, false);
+
+	v4l2_flash_release(flash->vf);
+
+	led_classdev_flash_unregister(&flash->fled);
+	led_classdev_unregister(&flash->iled_cdev);
+
+	mutex_destroy(&flash->mutex);
+
+	of_node_put(flash->flash_node);
+	of_node_put(flash->indicator_node);
+
+	return 0;
+}
+
+static const struct i2c_device_id as3645a_id_table[] = {
+	{ AS_NAME, 0 },
+	{ },
+};
+MODULE_DEVICE_TABLE(i2c, as3645a_id_table);
+
+static const struct of_device_id as3645a_of_table[] = {
+	{ .compatible = "ams,as3645a" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, as3645a_of_table);
+
+static struct i2c_driver as3645a_i2c_driver = {
+	.driver	= {
+		.of_match_table = as3645a_of_table,
+		.name = AS_NAME,
+	},
+	.probe_new	= as3645a_probe,
+	.remove	= as3645a_remove,
+	.id_table = as3645a_id_table,
+};
+
+module_i2c_driver(as3645a_i2c_driver);
+
+MODULE_AUTHOR("Laurent Pinchart <laurent.pinchart@ideasonboard.com>");
+MODULE_AUTHOR("Sakari Ailus <sakari.ailus@iki.fi>");
+MODULE_DESCRIPTION("LED flash driver for AS3645A, LM3555 and their clones");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/leds/leds-blinkm.c b/drivers/leds/leds-blinkm.c
index 617fe975bf6e..d03ed6b4176b 100644
--- a/drivers/leds/leds-blinkm.c
+++ b/drivers/leds/leds-blinkm.c
@@ -298,7 +298,7 @@ static struct attribute *blinkm_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group blinkm_group = {
+static const struct attribute_group blinkm_group = {
 	.name = "blinkm",
 	.attrs = blinkm_attrs,
 };
diff --git a/drivers/leds/leds-clevo-mail.c b/drivers/leds/leds-clevo-mail.c
index 0f9ed1ea0e89..492789f56896 100644
--- a/drivers/leds/leds-clevo-mail.c
+++ b/drivers/leds/leds-clevo-mail.c
@@ -40,7 +40,7 @@ static int __init clevo_mail_led_dmi_callback(const struct dmi_system_id *id)
  * detected as working, but in reality it is not) as low as
  * possible.
  */
-static struct dmi_system_id clevo_mail_led_dmi_table[] __initdata = {
+static const struct dmi_system_id clevo_mail_led_dmi_table[] __initconst = {
 	{
 		.callback = clevo_mail_led_dmi_callback,
 		.ident = "Clevo D410J",
diff --git a/drivers/leds/leds-gpio.c b/drivers/leds/leds-gpio.c
index e753ba93ba1e..764c31301f90 100644
--- a/drivers/leds/leds-gpio.c
+++ b/drivers/leds/leds-gpio.c
@@ -134,6 +134,8 @@ static int create_gpio_led(const struct gpio_led *template,
 		led_dat->cdev.flags |= LED_CORE_SUSPENDRESUME;
 	if (template->panic_indicator)
 		led_dat->cdev.flags |= LED_PANIC_INDICATOR;
+	if (template->retain_state_shutdown)
+		led_dat->cdev.flags |= LED_RETAIN_AT_SHUTDOWN;
 
 	ret = gpiod_direction_output(led_dat->gpiod, state);
 	if (ret < 0)
@@ -205,6 +207,8 @@ static struct gpio_leds_priv *gpio_leds_create(struct platform_device *pdev)
 
 		if (fwnode_property_present(child, "retain-state-suspended"))
 			led.retain_state_suspended = 1;
+		if (fwnode_property_present(child, "retain-state-shutdown"))
+			led.retain_state_shutdown = 1;
 		if (fwnode_property_present(child, "panic-indicator"))
 			led.panic_indicator = 1;
 
@@ -267,7 +271,8 @@ static void gpio_led_shutdown(struct platform_device *pdev)
 	for (i = 0; i < priv->num_leds; i++) {
 		struct gpio_led_data *led = &priv->leds[i];
 
-		gpio_led_set(&led->cdev, LED_OFF);
+		if (!(led->cdev.flags & LED_RETAIN_AT_SHUTDOWN))
+			gpio_led_set(&led->cdev, LED_OFF);
 	}
 }
 
diff --git a/drivers/leds/leds-is31fl32xx.c b/drivers/leds/leds-is31fl32xx.c
index 478844c5cead..31a9d749c8be 100644
--- a/drivers/leds/leds-is31fl32xx.c
+++ b/drivers/leds/leds-is31fl32xx.c
@@ -348,8 +348,8 @@ static int is31fl32xx_parse_child_dt(const struct device *dev,
 	ret = of_property_read_u32(child, "reg", &reg);
 	if (ret || reg < 1 || reg > led_data->priv->cdef->channels) {
 		dev_err(dev,
-			"Child node %s does not have a valid reg property\n",
-			child->full_name);
+			"Child node %pOF does not have a valid reg property\n",
+			child);
 		return -EINVAL;
 	}
 	led_data->channel = reg;
diff --git a/drivers/leds/leds-lm3533.c b/drivers/leds/leds-lm3533.c
index 5b529dc013d2..72224b599ffc 100644
--- a/drivers/leds/leds-lm3533.c
+++ b/drivers/leds/leds-lm3533.c
@@ -626,7 +626,7 @@ static umode_t lm3533_led_attr_is_visible(struct kobject *kobj,
 	return mode;
 };
 
-static struct attribute_group lm3533_led_attribute_group = {
+static const struct attribute_group lm3533_led_attribute_group = {
 	.is_visible	= lm3533_led_attr_is_visible,
 	.attrs		= lm3533_led_attributes
 };
diff --git a/drivers/leds/leds-lp5521.c b/drivers/leds/leds-lp5521.c
index f53c8cda1bde..55c0517fbe03 100644
--- a/drivers/leds/leds-lp5521.c
+++ b/drivers/leds/leds-lp5521.c
@@ -134,13 +134,13 @@ static void lp5521_set_led_current(struct lp55xx_led *led, u8 led_current)
 static void lp5521_load_engine(struct lp55xx_chip *chip)
 {
 	enum lp55xx_engine_index idx = chip->engine_idx;
-	u8 mask[] = {
+	static const u8 mask[] = {
 		[LP55XX_ENGINE_1] = LP5521_MODE_R_M,
 		[LP55XX_ENGINE_2] = LP5521_MODE_G_M,
 		[LP55XX_ENGINE_3] = LP5521_MODE_B_M,
 	};
 
-	u8 val[] = {
+	static const u8 val[] = {
 		[LP55XX_ENGINE_1] = LP5521_LOAD_R,
 		[LP55XX_ENGINE_2] = LP5521_LOAD_G,
 		[LP55XX_ENGINE_3] = LP5521_LOAD_B,
@@ -160,7 +160,7 @@ static void lp5521_stop_all_engines(struct lp55xx_chip *chip)
 static void lp5521_stop_engine(struct lp55xx_chip *chip)
 {
 	enum lp55xx_engine_index idx = chip->engine_idx;
-	u8 mask[] = {
+	static const u8 mask[] = {
 		[LP55XX_ENGINE_1] = LP5521_MODE_R_M,
 		[LP55XX_ENGINE_2] = LP5521_MODE_G_M,
 		[LP55XX_ENGINE_3] = LP5521_MODE_B_M,
@@ -226,7 +226,7 @@ static int lp5521_update_program_memory(struct lp55xx_chip *chip,
 {
 	enum lp55xx_engine_index idx = chip->engine_idx;
 	u8 pattern[LP5521_PROGRAM_LENGTH] = {0};
-	u8 addr[] = {
+	static const u8 addr[] = {
 		[LP55XX_ENGINE_1] = LP5521_REG_R_PROG_MEM,
 		[LP55XX_ENGINE_2] = LP5521_REG_G_PROG_MEM,
 		[LP55XX_ENGINE_3] = LP5521_REG_B_PROG_MEM,
diff --git a/drivers/leds/leds-lp5562.c b/drivers/leds/leds-lp5562.c
index 90892585bcb5..05ffa34fb6ad 100644
--- a/drivers/leds/leds-lp5562.c
+++ b/drivers/leds/leds-lp5562.c
@@ -116,7 +116,7 @@ static inline void lp5562_wait_enable_done(void)
 
 static void lp5562_set_led_current(struct lp55xx_led *led, u8 led_current)
 {
-	u8 addr[] = {
+	static const u8 addr[] = {
 		LP5562_REG_R_CURRENT,
 		LP5562_REG_G_CURRENT,
 		LP5562_REG_B_CURRENT,
@@ -130,13 +130,13 @@ static void lp5562_set_led_current(struct lp55xx_led *led, u8 led_current)
 static void lp5562_load_engine(struct lp55xx_chip *chip)
 {
 	enum lp55xx_engine_index idx = chip->engine_idx;
-	u8 mask[] = {
+	static const u8 mask[] = {
 		[LP55XX_ENGINE_1] = LP5562_MODE_ENG1_M,
 		[LP55XX_ENGINE_2] = LP5562_MODE_ENG2_M,
 		[LP55XX_ENGINE_3] = LP5562_MODE_ENG3_M,
 	};
 
-	u8 val[] = {
+	static const u8 val[] = {
 		[LP55XX_ENGINE_1] = LP5562_LOAD_ENG1,
 		[LP55XX_ENGINE_2] = LP5562_LOAD_ENG2,
 		[LP55XX_ENGINE_3] = LP5562_LOAD_ENG3,
@@ -211,7 +211,7 @@ static int lp5562_update_firmware(struct lp55xx_chip *chip,
 {
 	enum lp55xx_engine_index idx = chip->engine_idx;
 	u8 pattern[LP5562_PROGRAM_LENGTH] = {0};
-	u8 addr[] = {
+	static const u8 addr[] = {
 		[LP55XX_ENGINE_1] = LP5562_REG_PROG_MEM_ENG1,
 		[LP55XX_ENGINE_2] = LP5562_REG_PROG_MEM_ENG2,
 		[LP55XX_ENGINE_3] = LP5562_REG_PROG_MEM_ENG3,
@@ -314,7 +314,7 @@ static int lp5562_post_init_device(struct lp55xx_chip *chip)
 static int lp5562_led_brightness(struct lp55xx_led *led)
 {
 	struct lp55xx_chip *chip = led->chip;
-	u8 addr[] = {
+	static const u8 addr[] = {
 		LP5562_REG_R_PWM,
 		LP5562_REG_G_PWM,
 		LP5562_REG_B_PWM,
diff --git a/drivers/leds/leds-lp8501.c b/drivers/leds/leds-lp8501.c
index 3f9675bd214a..3adb113cf02e 100644
--- a/drivers/leds/leds-lp8501.c
+++ b/drivers/leds/leds-lp8501.c
@@ -118,19 +118,19 @@ static int lp8501_post_init_device(struct lp55xx_chip *chip)
 static void lp8501_load_engine(struct lp55xx_chip *chip)
 {
 	enum lp55xx_engine_index idx = chip->engine_idx;
-	u8 mask[] = {
+	static const u8 mask[] = {
 		[LP55XX_ENGINE_1] = LP8501_MODE_ENG1_M,
 		[LP55XX_ENGINE_2] = LP8501_MODE_ENG2_M,
 		[LP55XX_ENGINE_3] = LP8501_MODE_ENG3_M,
 	};
 
-	u8 val[] = {
+	static const u8 val[] = {
 		[LP55XX_ENGINE_1] = LP8501_LOAD_ENG1,
 		[LP55XX_ENGINE_2] = LP8501_LOAD_ENG2,
 		[LP55XX_ENGINE_3] = LP8501_LOAD_ENG3,
 	};
 
-	u8 page_sel[] = {
+	static const u8 page_sel[] = {
 		[LP55XX_ENGINE_1] = LP8501_PAGE_ENG1,
 		[LP55XX_ENGINE_2] = LP8501_PAGE_ENG2,
 		[LP55XX_ENGINE_3] = LP8501_PAGE_ENG3,
diff --git a/drivers/leds/leds-max77693.c b/drivers/leds/leds-max77693.c
index 2d3062d53325..adf0f191f794 100644
--- a/drivers/leds/leds-max77693.c
+++ b/drivers/leds/leds-max77693.c
@@ -856,7 +856,7 @@ static void max77693_init_v4l2_flash_config(struct max77693_sub_led *sub_led,
 		 "%s %d-%04x", sub_led->fled_cdev.led_cdev.name,
 		 i2c_adapter_id(i2c->adapter), i2c->addr);
 
-	s = &v4l2_sd_cfg->torch_intensity;
+	s = &v4l2_sd_cfg->intensity;
 	s->min = TORCH_IOUT_MIN;
 	s->max = sub_led->fled_cdev.led_cdev.max_brightness * TORCH_IOUT_STEP;
 	s->step = TORCH_IOUT_STEP;
@@ -931,7 +931,7 @@ static int max77693_register_led(struct max77693_sub_led *sub_led,
 
 	/* Register in the V4L2 subsystem. */
 	sub_led->v4l2_flash = v4l2_flash_init(dev, of_fwnode_handle(sub_node),
-					      fled_cdev, NULL, &v4l2_flash_ops,
+					      fled_cdev, &v4l2_flash_ops,
 					      &v4l2_sd_cfg);
 	if (IS_ERR(sub_led->v4l2_flash)) {
 		ret = PTR_ERR(sub_led->v4l2_flash);
diff --git a/drivers/leds/leds-pca955x.c b/drivers/leds/leds-pca955x.c
index 9a873118ea5f..905729191d3e 100644
--- a/drivers/leds/leds-pca955x.c
+++ b/drivers/leds/leds-pca955x.c
@@ -41,14 +41,19 @@
  */
 
 #include <linux/acpi.h>
-#include <linux/module.h>
-#include <linux/delay.h>
-#include <linux/string.h>
 #include <linux/ctype.h>
-#include <linux/leds.h>
+#include <linux/delay.h>
 #include <linux/err.h>
+#include <linux/gpio.h>
 #include <linux/i2c.h>
+#include <linux/leds.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/of.h>
 #include <linux/slab.h>
+#include <linux/string.h>
+
+#include <dt-bindings/leds/leds-pca955x.h>
 
 /* LED select registers determine the source that drives LED outputs */
 #define PCA955X_LS_LED_ON	0x0	/* Output LOW */
@@ -115,6 +120,9 @@ struct pca955x {
 	struct pca955x_led *leds;
 	struct pca955x_chipdef	*chipdef;
 	struct i2c_client	*client;
+#ifdef CONFIG_LEDS_PCA955X_GPIO
+	struct gpio_chip gpio;
+#endif
 };
 
 struct pca955x_led {
@@ -122,6 +130,13 @@ struct pca955x_led {
 	struct led_classdev	led_cdev;
 	int			led_num;	/* 0 .. 15 potentially */
 	char			name[32];
+	u32			type;
+	const char		*default_trigger;
+};
+
+struct pca955x_platform_data {
+	struct pca955x_led	*leds;
+	int			num_leds;
 };
 
 /* 8 bits per input register */
@@ -150,13 +165,18 @@ static inline u8 pca955x_ledsel(u8 oldval, int led_num, int state)
  * Write to frequency prescaler register, used to program the
  * period of the PWM output.  period = (PSCx + 1) / 38
  */
-static void pca955x_write_psc(struct i2c_client *client, int n, u8 val)
+static int pca955x_write_psc(struct i2c_client *client, int n, u8 val)
 {
 	struct pca955x *pca955x = i2c_get_clientdata(client);
+	int ret;
 
-	i2c_smbus_write_byte_data(client,
+	ret = i2c_smbus_write_byte_data(client,
 		pca95xx_num_input_regs(pca955x->chipdef->bits) + 2*n,
 		val);
+	if (ret < 0)
+		dev_err(&client->dev, "%s: reg 0x%x, val 0x%x, err %d\n",
+			__func__, n, val, ret);
+	return ret;
 }
 
 /*
@@ -166,38 +186,56 @@ static void pca955x_write_psc(struct i2c_client *client, int n, u8 val)
  *
  * Duty cycle is (256 - PWMx) / 256
  */
-static void pca955x_write_pwm(struct i2c_client *client, int n, u8 val)
+static int pca955x_write_pwm(struct i2c_client *client, int n, u8 val)
 {
 	struct pca955x *pca955x = i2c_get_clientdata(client);
+	int ret;
 
-	i2c_smbus_write_byte_data(client,
+	ret = i2c_smbus_write_byte_data(client,
 		pca95xx_num_input_regs(pca955x->chipdef->bits) + 1 + 2*n,
 		val);
+	if (ret < 0)
+		dev_err(&client->dev, "%s: reg 0x%x, val 0x%x, err %d\n",
+			__func__, n, val, ret);
+	return ret;
 }
 
 /*
  * Write to LED selector register, which determines the source that
  * drives the LED output.
  */
-static void pca955x_write_ls(struct i2c_client *client, int n, u8 val)
+static int pca955x_write_ls(struct i2c_client *client, int n, u8 val)
 {
 	struct pca955x *pca955x = i2c_get_clientdata(client);
+	int ret;
 
-	i2c_smbus_write_byte_data(client,
+	ret = i2c_smbus_write_byte_data(client,
 		pca95xx_num_input_regs(pca955x->chipdef->bits) + 4 + n,
 		val);
+	if (ret < 0)
+		dev_err(&client->dev, "%s: reg 0x%x, val 0x%x, err %d\n",
+			__func__, n, val, ret);
+	return ret;
 }
 
 /*
  * Read the LED selector register, which determines the source that
  * drives the LED output.
  */
-static u8 pca955x_read_ls(struct i2c_client *client, int n)
+static int pca955x_read_ls(struct i2c_client *client, int n, u8 *val)
 {
 	struct pca955x *pca955x = i2c_get_clientdata(client);
+	int ret;
 
-	return (u8) i2c_smbus_read_byte_data(client,
+	ret = i2c_smbus_read_byte_data(client,
 		pca95xx_num_input_regs(pca955x->chipdef->bits) + 4 + n);
+	if (ret < 0) {
+		dev_err(&client->dev, "%s: reg 0x%x, err %d\n",
+			__func__, n, ret);
+		return ret;
+	}
+	*val = (u8)ret;
+	return 0;
 }
 
 static int pca955x_led_set(struct led_classdev *led_cdev,
@@ -208,6 +246,7 @@ static int pca955x_led_set(struct led_classdev *led_cdev,
 	u8 ls;
 	int chip_ls;	/* which LSx to use (0-3 potentially) */
 	int ls_led;	/* which set of bits within LSx to use (0-3) */
+	int ret;
 
 	pca955x_led = container_of(led_cdev, struct pca955x_led, led_cdev);
 	pca955x = pca955x_led->pca955x;
@@ -217,7 +256,9 @@ static int pca955x_led_set(struct led_classdev *led_cdev,
 
 	mutex_lock(&pca955x->lock);
 
-	ls = pca955x_read_ls(pca955x->client, chip_ls);
+	ret = pca955x_read_ls(pca955x->client, chip_ls, &ls);
+	if (ret)
+		goto out;
 
 	switch (value) {
 	case LED_FULL:
@@ -237,18 +278,159 @@ static int pca955x_led_set(struct led_classdev *led_cdev,
 		 * OFF, HALF, or FULL.  But, this is probably better than
 		 * just turning off for all other values.
 		 */
-		pca955x_write_pwm(pca955x->client, 1,
-				255 - value);
+		ret = pca955x_write_pwm(pca955x->client, 1, 255 - value);
+		if (ret)
+			goto out;
 		ls = pca955x_ledsel(ls, ls_led, PCA955X_LS_BLINK1);
 		break;
 	}
 
-	pca955x_write_ls(pca955x->client, chip_ls, ls);
+	ret = pca955x_write_ls(pca955x->client, chip_ls, ls);
 
+out:
 	mutex_unlock(&pca955x->lock);
 
+	return ret;
+}
+
+#ifdef CONFIG_LEDS_PCA955X_GPIO
+/*
+ * Read the INPUT register, which contains the state of LEDs.
+ */
+static int pca955x_read_input(struct i2c_client *client, int n, u8 *val)
+{
+	int ret = i2c_smbus_read_byte_data(client, n);
+
+	if (ret < 0) {
+		dev_err(&client->dev, "%s: reg 0x%x, err %d\n",
+			__func__, n, ret);
+		return ret;
+	}
+	*val = (u8)ret;
 	return 0;
+
+}
+
+static int pca955x_gpio_request_pin(struct gpio_chip *gc, unsigned int offset)
+{
+	struct pca955x *pca955x = gpiochip_get_data(gc);
+	struct pca955x_led *led = &pca955x->leds[offset];
+
+	if (led->type == PCA955X_TYPE_GPIO)
+		return 0;
+
+	return -EBUSY;
+}
+
+static int pca955x_set_value(struct gpio_chip *gc, unsigned int offset,
+			     int val)
+{
+	struct pca955x *pca955x = gpiochip_get_data(gc);
+	struct pca955x_led *led = &pca955x->leds[offset];
+
+	if (val)
+		return pca955x_led_set(&led->led_cdev, LED_FULL);
+	else
+		return pca955x_led_set(&led->led_cdev, LED_OFF);
+}
+
+static void pca955x_gpio_set_value(struct gpio_chip *gc, unsigned int offset,
+				   int val)
+{
+	pca955x_set_value(gc, offset, val);
+}
+
+static int pca955x_gpio_get_value(struct gpio_chip *gc, unsigned int offset)
+{
+	struct pca955x *pca955x = gpiochip_get_data(gc);
+	struct pca955x_led *led = &pca955x->leds[offset];
+	u8 reg = 0;
+
+	/* There is nothing we can do about errors */
+	pca955x_read_input(pca955x->client, led->led_num / 8, &reg);
+
+	return !!(reg & (1 << (led->led_num % 8)));
+}
+
+static int pca955x_gpio_direction_input(struct gpio_chip *gc,
+					unsigned int offset)
+{
+	/* To use as input ensure pin is not driven */
+	return pca955x_set_value(gc, offset, 0);
+}
+
+static int pca955x_gpio_direction_output(struct gpio_chip *gc,
+					 unsigned int offset, int val)
+{
+	return pca955x_set_value(gc, offset, val);
 }
+#endif /* CONFIG_LEDS_PCA955X_GPIO */
+
+#if IS_ENABLED(CONFIG_OF)
+static struct pca955x_platform_data *
+pca955x_pdata_of_init(struct i2c_client *client, struct pca955x_chipdef *chip)
+{
+	struct device_node *np = client->dev.of_node;
+	struct device_node *child;
+	struct pca955x_platform_data *pdata;
+	int count;
+
+	count = of_get_child_count(np);
+	if (!count || count > chip->bits)
+		return ERR_PTR(-ENODEV);
+
+	pdata = devm_kzalloc(&client->dev, sizeof(*pdata), GFP_KERNEL);
+	if (!pdata)
+		return ERR_PTR(-ENOMEM);
+
+	pdata->leds = devm_kzalloc(&client->dev,
+				   sizeof(struct pca955x_led) * chip->bits,
+				   GFP_KERNEL);
+	if (!pdata->leds)
+		return ERR_PTR(-ENOMEM);
+
+	for_each_child_of_node(np, child) {
+		const char *name;
+		u32 reg;
+		int res;
+
+		res = of_property_read_u32(child, "reg", &reg);
+		if ((res != 0) || (reg >= chip->bits))
+			continue;
+
+		if (of_property_read_string(child, "label", &name))
+			name = child->name;
+
+		snprintf(pdata->leds[reg].name, sizeof(pdata->leds[reg].name),
+			 "%s", name);
+
+		pdata->leds[reg].type = PCA955X_TYPE_LED;
+		of_property_read_u32(child, "type", &pdata->leds[reg].type);
+		of_property_read_string(child, "linux,default-trigger",
+					&pdata->leds[reg].default_trigger);
+	}
+
+	pdata->num_leds = chip->bits;
+
+	return pdata;
+}
+
+static const struct of_device_id of_pca955x_match[] = {
+	{ .compatible = "nxp,pca9550", .data = (void *)pca9550 },
+	{ .compatible = "nxp,pca9551", .data = (void *)pca9551 },
+	{ .compatible = "nxp,pca9552", .data = (void *)pca9552 },
+	{ .compatible = "nxp,pca9553", .data = (void *)pca9553 },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, of_pca955x_match);
+#else
+static struct pca955x_platform_data *
+pca955x_pdata_of_init(struct i2c_client *client, struct pca955x_chipdef *chip)
+{
+	return ERR_PTR(-ENODEV);
+}
+#endif
 
 static int pca955x_probe(struct i2c_client *client,
 					const struct i2c_device_id *id)
@@ -257,8 +439,9 @@ static int pca955x_probe(struct i2c_client *client,
 	struct pca955x_led *pca955x_led;
 	struct pca955x_chipdef *chip;
 	struct i2c_adapter *adapter;
-	struct led_platform_data *pdata;
 	int i, err;
+	struct pca955x_platform_data *pdata;
+	int ngpios = 0;
 
 	if (id) {
 		chip = &pca955x_chipdefs[id->driver_data];
@@ -272,6 +455,11 @@ static int pca955x_probe(struct i2c_client *client,
 	}
 	adapter = to_i2c_adapter(client->dev.parent);
 	pdata = dev_get_platdata(&client->dev);
+	if (!pdata) {
+		pdata =	pca955x_pdata_of_init(client, chip);
+		if (IS_ERR(pdata))
+			return PTR_ERR(pdata);
+	}
 
 	/* Make sure the slave address / chip type combo given is possible */
 	if ((client->addr & ~((1 << chip->slv_addr_shift) - 1)) !=
@@ -288,13 +476,11 @@ static int pca955x_probe(struct i2c_client *client,
 	if (!i2c_check_functionality(adapter, I2C_FUNC_SMBUS_BYTE_DATA))
 		return -EIO;
 
-	if (pdata) {
-		if (pdata->num_leds != chip->bits) {
-			dev_err(&client->dev, "board info claims %d LEDs"
-					" on a %d-bit chip\n",
-					pdata->num_leds, chip->bits);
-			return -ENODEV;
-		}
+	if (pdata->num_leds != chip->bits) {
+		dev_err(&client->dev,
+			"board info claims %d LEDs on a %d-bit chip\n",
+			pdata->num_leds, chip->bits);
+		return -ENODEV;
 	}
 
 	pca955x = devm_kzalloc(&client->dev, sizeof(*pca955x), GFP_KERNEL);
@@ -316,60 +502,92 @@ static int pca955x_probe(struct i2c_client *client,
 		pca955x_led = &pca955x->leds[i];
 		pca955x_led->led_num = i;
 		pca955x_led->pca955x = pca955x;
+		pca955x_led->type = pdata->leds[i].type;
+
+		switch (pca955x_led->type) {
+		case PCA955X_TYPE_NONE:
+			break;
+		case PCA955X_TYPE_GPIO:
+			ngpios++;
+			break;
+		case PCA955X_TYPE_LED:
+			/*
+			 * Platform data can specify LED names and
+			 * default triggers
+			 */
+			if (pdata->leds[i].name[0] == '\0')
+				snprintf(pdata->leds[i].name,
+					sizeof(pdata->leds[i].name), "%d", i);
+
+			snprintf(pca955x_led->name,
+				sizeof(pca955x_led->name), "pca955x:%s",
+				pdata->leds[i].name);
 
-		/* Platform data can specify LED names and default triggers */
-		if (pdata) {
-			if (pdata->leds[i].name)
-				snprintf(pca955x_led->name,
-					sizeof(pca955x_led->name), "pca955x:%s",
-					pdata->leds[i].name);
 			if (pdata->leds[i].default_trigger)
 				pca955x_led->led_cdev.default_trigger =
 					pdata->leds[i].default_trigger;
-		} else {
-			snprintf(pca955x_led->name, sizeof(pca955x_led->name),
-				 "pca955x:%d", i);
-		}
 
-		pca955x_led->led_cdev.name = pca955x_led->name;
-		pca955x_led->led_cdev.brightness_set_blocking = pca955x_led_set;
+			pca955x_led->led_cdev.name = pca955x_led->name;
+			pca955x_led->led_cdev.brightness_set_blocking =
+				pca955x_led_set;
 
-		err = led_classdev_register(&client->dev,
-					&pca955x_led->led_cdev);
-		if (err < 0)
-			goto exit;
-	}
+			err = devm_led_classdev_register(&client->dev,
+							&pca955x_led->led_cdev);
+			if (err)
+				return err;
 
-	/* Turn off LEDs */
-	for (i = 0; i < pca95xx_num_led_regs(chip->bits); i++)
-		pca955x_write_ls(client, i, 0x55);
+			/* Turn off LED */
+			err = pca955x_led_set(&pca955x_led->led_cdev, LED_OFF);
+			if (err)
+				return err;
+		}
+	}
 
 	/* PWM0 is used for half brightness or 50% duty cycle */
-	pca955x_write_pwm(client, 0, 255-LED_HALF);
+	err = pca955x_write_pwm(client, 0, 255 - LED_HALF);
+	if (err)
+		return err;
 
 	/* PWM1 is used for variable brightness, default to OFF */
-	pca955x_write_pwm(client, 1, 0);
+	err = pca955x_write_pwm(client, 1, 0);
+	if (err)
+		return err;
 
 	/* Set to fast frequency so we do not see flashing */
-	pca955x_write_psc(client, 0, 0);
-	pca955x_write_psc(client, 1, 0);
-
-	return 0;
-
-exit:
-	while (i--)
-		led_classdev_unregister(&pca955x->leds[i].led_cdev);
-
-	return err;
-}
-
-static int pca955x_remove(struct i2c_client *client)
-{
-	struct pca955x *pca955x = i2c_get_clientdata(client);
-	int i;
-
-	for (i = 0; i < pca955x->chipdef->bits; i++)
-		led_classdev_unregister(&pca955x->leds[i].led_cdev);
+	err = pca955x_write_psc(client, 0, 0);
+	if (err)
+		return err;
+	err = pca955x_write_psc(client, 1, 0);
+	if (err)
+		return err;
+
+#ifdef CONFIG_LEDS_PCA955X_GPIO
+	if (ngpios) {
+		pca955x->gpio.label = "gpio-pca955x";
+		pca955x->gpio.direction_input = pca955x_gpio_direction_input;
+		pca955x->gpio.direction_output = pca955x_gpio_direction_output;
+		pca955x->gpio.set = pca955x_gpio_set_value;
+		pca955x->gpio.get = pca955x_gpio_get_value;
+		pca955x->gpio.request = pca955x_gpio_request_pin;
+		pca955x->gpio.can_sleep = 1;
+		pca955x->gpio.base = -1;
+		pca955x->gpio.ngpio = ngpios;
+		pca955x->gpio.parent = &client->dev;
+		pca955x->gpio.owner = THIS_MODULE;
+
+		err = devm_gpiochip_add_data(&client->dev, &pca955x->gpio,
+					     pca955x);
+		if (err) {
+			/* Use data->gpio.dev as a flag for freeing gpiochip */
+			pca955x->gpio.parent = NULL;
+			dev_warn(&client->dev, "could not add gpiochip\n");
+			return err;
+		}
+		dev_info(&client->dev, "gpios %i...%i\n",
+			 pca955x->gpio.base, pca955x->gpio.base +
+			 pca955x->gpio.ngpio - 1);
+	}
+#endif
 
 	return 0;
 }
@@ -378,9 +596,9 @@ static struct i2c_driver pca955x_driver = {
 	.driver = {
 		.name	= "leds-pca955x",
 		.acpi_match_table = ACPI_PTR(pca955x_acpi_ids),
+		.of_match_table = of_match_ptr(of_pca955x_match),
 	},
 	.probe	= pca955x_probe,
-	.remove	= pca955x_remove,
 	.id_table = pca955x_id,
 };
 
diff --git a/drivers/leds/leds-powernv.c b/drivers/leds/leds-powernv.c
index b2a98c7b521b..b1adbd70ce2e 100644
--- a/drivers/leds/leds-powernv.c
+++ b/drivers/leds/leds-powernv.c
@@ -224,12 +224,8 @@ static int powernv_led_create(struct device *dev,
 	powernv_led->cdev.name = devm_kasprintf(dev, GFP_KERNEL, "%s:%s",
 						powernv_led->loc_code,
 						led_type_desc);
-	if (!powernv_led->cdev.name) {
-		dev_err(dev,
-			"%s: Memory allocation failed for classdev name\n",
-			__func__);
+	if (!powernv_led->cdev.name)
 		return -ENOMEM;
-	}
 
 	powernv_led->cdev.brightness_set_blocking = powernv_brightness_set;
 	powernv_led->cdev.brightness_get = powernv_brightness_get;
diff --git a/drivers/leds/leds-ss4200.c b/drivers/leds/leds-ss4200.c
index 732eb86bc1a5..a9db8674cd02 100644
--- a/drivers/leds/leds-ss4200.c
+++ b/drivers/leds/leds-ss4200.c
@@ -91,7 +91,7 @@ MODULE_PARM_DESC(nodetect, "Skip DMI-based hardware detection");
  * detected as working, but in reality it is not) as low as
  * possible.
  */
-static struct dmi_system_id nas_led_whitelist[] __initdata = {
+static const struct dmi_system_id nas_led_whitelist[] __initconst = {
 	{
 		.callback = ss4200_led_dmi_callback,
 		.ident = "Intel SS4200-E",
diff --git a/drivers/leds/leds-tlc591xx.c b/drivers/leds/leds-tlc591xx.c
index 304531644938..f5357f6d9e58 100644
--- a/drivers/leds/leds-tlc591xx.c
+++ b/drivers/leds/leds-tlc591xx.c
@@ -230,12 +230,15 @@ tlc591xx_probe(struct i2c_client *client,
 
 	for_each_child_of_node(np, child) {
 		err = of_property_read_u32(child, "reg", &reg);
-		if (err)
+		if (err) {
+			of_node_put(child);
 			return err;
-		if (reg < 0 || reg >= tlc591xx->max_leds)
-			return -EINVAL;
-		if (priv->leds[reg].active)
+		}
+		if (reg < 0 || reg >= tlc591xx->max_leds ||
+		    priv->leds[reg].active) {
+			of_node_put(child);
 			return -EINVAL;
+		}
 		priv->leds[reg].active = true;
 		priv->leds[reg].ldev.name =
 			of_get_property(child, "label", NULL) ? : child->name;
diff --git a/drivers/lguest/Kconfig b/drivers/lguest/Kconfig
deleted file mode 100644
index 169172d2ba05..000000000000
--- a/drivers/lguest/Kconfig
+++ /dev/null
@@ -1,13 +0,0 @@
-config LGUEST
-	tristate "Linux hypervisor example code"
-	depends on X86_32 && EVENTFD && TTY && PCI_DIRECT
-	select HVC_DRIVER
-	---help---
-	  This is a very simple module which allows you to run
-	  multiple instances of the same Linux kernel, using the
-	  "lguest" command found in the tools/lguest directory.
-
-	  Note that "lguest" is pronounced to rhyme with "fell quest",
-	  not "rustyvisor". See tools/lguest/lguest.txt.
-
-	  If unsure, say N.  If curious, say M.  If masochistic, say Y.
diff --git a/drivers/lguest/Makefile b/drivers/lguest/Makefile
deleted file mode 100644
index 16f52ee73994..000000000000
--- a/drivers/lguest/Makefile
+++ /dev/null
@@ -1,26 +0,0 @@
-# Host requires the other files, which can be a module.
-obj-$(CONFIG_LGUEST)	+= lg.o
-lg-y = core.o hypercalls.o page_tables.o interrupts_and_traps.o \
-	segments.o lguest_user.o
-
-lg-$(CONFIG_X86_32) += x86/switcher_32.o x86/core.o
-
-Preparation Preparation!: PREFIX=P
-Guest: PREFIX=G
-Drivers: PREFIX=D
-Launcher: PREFIX=L
-Host: PREFIX=H
-Switcher: PREFIX=S
-Mastery: PREFIX=M
-Beer:
-	@for f in Preparation Guest Drivers Launcher Host Switcher Mastery; do echo "{==- $$f -==}"; make -s $$f; done; echo "{==-==}"
-Preparation Preparation! Guest Drivers Launcher Host Switcher Mastery:
-	@sh ../../tools/lguest/extract $(PREFIX) `find ../../* -name '*.[chS]' -wholename '*lguest*'`
-Puppy:
-	@clear
-	@printf "      __  \n (___()'\`;\n /,    /\`\n \\\\\\\"--\\\\\\   \n"
-	@sleep 2; clear; printf "\n\n   Sit!\n\n"; sleep 1; clear
-	@printf "    __    \n   ()'\`;  \n   /\\|\` \n  /  |  \n(/_)_|_   \n"
-	@sleep 2; clear; printf "\n\n  Stand!\n\n"; sleep 1; clear
-	@printf "    __    \n   ()'\`;  \n   /\\|\` \n  /._.= \n /| /     \n(_\_)_    \n"
-	@sleep 2; clear; printf "\n\n  Good puppy!\n\n"; sleep 1; clear
diff --git a/drivers/lguest/README b/drivers/lguest/README
deleted file mode 100644
index b7db39a64c66..000000000000
--- a/drivers/lguest/README
+++ /dev/null
@@ -1,47 +0,0 @@
-Welcome, friend reader, to lguest.
-
-Lguest is an adventure, with you, the reader, as Hero.  I can't think of many
-5000-line projects which offer both such capability and glimpses of future
-potential; it is an exciting time to be delving into the source!
-
-But be warned; this is an arduous journey of several hours or more!  And as we
-know, all true Heroes are driven by a Noble Goal.  Thus I offer a Beer (or
-equivalent) to anyone I meet who has completed this documentation.
-
-So get comfortable and keep your wits about you (both quick and humorous).
-Along your way to the Noble Goal, you will also gain masterly insight into
-lguest, and hypervisors and x86 virtualization in general.
-
-Our Quest is in seven parts: (best read with C highlighting turned on)
-
-I) Preparation
-	- In which our potential hero is flown quickly over the landscape for a
-	  taste of its scope.  Suitable for the armchair coders and other such
-	  persons of faint constitution.
-
-II) Guest
-	- Where we encounter the first tantalising wisps of code, and come to
-	  understand the details of the life of a Guest kernel.
-
-III) Drivers
-	- Whereby the Guest finds its voice and become useful, and our
-	  understanding of the Guest is completed.
-
-IV) Launcher
-	- Where we trace back to the creation of the Guest, and thus begin our
-	  understanding of the Host.
-
-V) Host
-	- Where we master the Host code, through a long and tortuous journey.
-	  Indeed, it is here that our hero is tested in the Bit of Despair.
-
-VI) Switcher
-	- Where our understanding of the intertwined nature of Guests and Hosts
-	  is completed.
-
-VII) Mastery
-	- Where our fully fledged hero grapples with the Great Question:
-	  "What next?"
-
-make Preparation!
-Rusty Russell.
diff --git a/drivers/lguest/core.c b/drivers/lguest/core.c
deleted file mode 100644
index 395ed1961dbf..000000000000
--- a/drivers/lguest/core.c
+++ /dev/null
@@ -1,398 +0,0 @@
-/*P:400
- * This contains run_guest() which actually calls into the Host<->Guest
- * Switcher and analyzes the return, such as determining if the Guest wants the
- * Host to do something.  This file also contains useful helper routines.
-:*/
-#include <linux/module.h>
-#include <linux/stringify.h>
-#include <linux/stddef.h>
-#include <linux/io.h>
-#include <linux/mm.h>
-#include <linux/sched/signal.h>
-#include <linux/vmalloc.h>
-#include <linux/cpu.h>
-#include <linux/freezer.h>
-#include <linux/highmem.h>
-#include <linux/slab.h>
-#include <asm/paravirt.h>
-#include <asm/pgtable.h>
-#include <linux/uaccess.h>
-#include <asm/poll.h>
-#include <asm/asm-offsets.h>
-#include "lg.h"
-
-unsigned long switcher_addr;
-struct page **lg_switcher_pages;
-static struct vm_struct *switcher_text_vma;
-static struct vm_struct *switcher_stacks_vma;
-
-/* This One Big lock protects all inter-guest data structures. */
-DEFINE_MUTEX(lguest_lock);
-
-/*H:010
- * We need to set up the Switcher at a high virtual address.  Remember the
- * Switcher is a few hundred bytes of assembler code which actually changes the
- * CPU to run the Guest, and then changes back to the Host when a trap or
- * interrupt happens.
- *
- * The Switcher code must be at the same virtual address in the Guest as the
- * Host since it will be running as the switchover occurs.
- *
- * Trying to map memory at a particular address is an unusual thing to do, so
- * it's not a simple one-liner.
- */
-static __init int map_switcher(void)
-{
-	int i, err;
-
-	/*
-	 * Map the Switcher in to high memory.
-	 *
-	 * It turns out that if we choose the address 0xFFC00000 (4MB under the
-	 * top virtual address), it makes setting up the page tables really
-	 * easy.
-	 */
-
-	/* We assume Switcher text fits into a single page. */
-	if (end_switcher_text - start_switcher_text > PAGE_SIZE) {
-		printk(KERN_ERR "lguest: switcher text too large (%zu)\n",
-		       end_switcher_text - start_switcher_text);
-		return -EINVAL;
-	}
-
-	/*
-	 * We allocate an array of struct page pointers.  map_vm_area() wants
-	 * this, rather than just an array of pages.
-	 */
-	lg_switcher_pages = kmalloc(sizeof(lg_switcher_pages[0])
-				    * TOTAL_SWITCHER_PAGES,
-				    GFP_KERNEL);
-	if (!lg_switcher_pages) {
-		err = -ENOMEM;
-		goto out;
-	}
-
-	/*
-	 * Now we actually allocate the pages.  The Guest will see these pages,
-	 * so we make sure they're zeroed.
-	 */
-	for (i = 0; i < TOTAL_SWITCHER_PAGES; i++) {
-		lg_switcher_pages[i] = alloc_page(GFP_KERNEL|__GFP_ZERO);
-		if (!lg_switcher_pages[i]) {
-			err = -ENOMEM;
-			goto free_some_pages;
-		}
-	}
-
-	/*
-	 * Copy in the compiled-in Switcher code (from x86/switcher_32.S).
-	 * It goes in the first page, which we map in momentarily.
-	 */
-	memcpy(kmap(lg_switcher_pages[0]), start_switcher_text,
-	       end_switcher_text - start_switcher_text);
-	kunmap(lg_switcher_pages[0]);
-
-	/*
-	 * We place the Switcher underneath the fixmap area, which is the
-	 * highest virtual address we can get.  This is important, since we
-	 * tell the Guest it can't access this memory, so we want its ceiling
-	 * as high as possible.
-	 */
-	switcher_addr = FIXADDR_START - TOTAL_SWITCHER_PAGES*PAGE_SIZE;
-
-	/*
-	 * Now we reserve the "virtual memory area"s we want.  We might
-	 * not get them in theory, but in practice it's worked so far.
-	 *
-	 * We want the switcher text to be read-only and executable, and
-	 * the stacks to be read-write and non-executable.
-	 */
-	switcher_text_vma = __get_vm_area(PAGE_SIZE, VM_ALLOC|VM_NO_GUARD,
-					  switcher_addr,
-					  switcher_addr + PAGE_SIZE);
-
-	if (!switcher_text_vma) {
-		err = -ENOMEM;
-		printk("lguest: could not map switcher pages high\n");
-		goto free_pages;
-	}
-
-	switcher_stacks_vma = __get_vm_area(SWITCHER_STACK_PAGES * PAGE_SIZE,
-					    VM_ALLOC|VM_NO_GUARD,
-					    switcher_addr + PAGE_SIZE,
-					    switcher_addr + TOTAL_SWITCHER_PAGES * PAGE_SIZE);
-	if (!switcher_stacks_vma) {
-		err = -ENOMEM;
-		printk("lguest: could not map switcher pages high\n");
-		goto free_text_vma;
-	}
-
-	/*
-	 * This code actually sets up the pages we've allocated to appear at
-	 * switcher_addr.  map_vm_area() takes the vma we allocated above, the
-	 * kind of pages we're mapping (kernel text pages and kernel writable
-	 * pages respectively), and a pointer to our array of struct pages.
-	 */
-	err = map_vm_area(switcher_text_vma, PAGE_KERNEL_RX, lg_switcher_pages);
-	if (err) {
-		printk("lguest: text map_vm_area failed: %i\n", err);
-		goto free_vmas;
-	}
-
-	err = map_vm_area(switcher_stacks_vma, PAGE_KERNEL,
-			  lg_switcher_pages + SWITCHER_TEXT_PAGES);
-	if (err) {
-		printk("lguest: stacks map_vm_area failed: %i\n", err);
-		goto free_vmas;
-	}
-
-	/*
-	 * Now the Switcher is mapped at the right address, we can't fail!
-	 */
-	printk(KERN_INFO "lguest: mapped switcher at %p\n",
-	       switcher_text_vma->addr);
-	/* And we succeeded... */
-	return 0;
-
-free_vmas:
-	/* Undoes map_vm_area and __get_vm_area */
-	vunmap(switcher_stacks_vma->addr);
-free_text_vma:
-	vunmap(switcher_text_vma->addr);
-free_pages:
-	i = TOTAL_SWITCHER_PAGES;
-free_some_pages:
-	for (--i; i >= 0; i--)
-		__free_pages(lg_switcher_pages[i], 0);
-	kfree(lg_switcher_pages);
-out:
-	return err;
-}
-/*:*/
-
-/* Cleaning up the mapping when the module is unloaded is almost... too easy. */
-static void unmap_switcher(void)
-{
-	unsigned int i;
-
-	/* vunmap() undoes *both* map_vm_area() and __get_vm_area(). */
-	vunmap(switcher_text_vma->addr);
-	vunmap(switcher_stacks_vma->addr);
-	/* Now we just need to free the pages we copied the switcher into */
-	for (i = 0; i < TOTAL_SWITCHER_PAGES; i++)
-		__free_pages(lg_switcher_pages[i], 0);
-	kfree(lg_switcher_pages);
-}
-
-/*H:032
- * Dealing With Guest Memory.
- *
- * Before we go too much further into the Host, we need to grok the routines
- * we use to deal with Guest memory.
- *
- * When the Guest gives us (what it thinks is) a physical address, we can use
- * the normal copy_from_user() & copy_to_user() on the corresponding place in
- * the memory region allocated by the Launcher.
- *
- * But we can't trust the Guest: it might be trying to access the Launcher
- * code.  We have to check that the range is below the pfn_limit the Launcher
- * gave us.  We have to make sure that addr + len doesn't give us a false
- * positive by overflowing, too.
- */
-bool lguest_address_ok(const struct lguest *lg,
-		       unsigned long addr, unsigned long len)
-{
-	return addr+len <= lg->pfn_limit * PAGE_SIZE && (addr+len >= addr);
-}
-
-/*
- * This routine copies memory from the Guest.  Here we can see how useful the
- * kill_lguest() routine we met in the Launcher can be: we return a random
- * value (all zeroes) instead of needing to return an error.
- */
-void __lgread(struct lg_cpu *cpu, void *b, unsigned long addr, unsigned bytes)
-{
-	if (!lguest_address_ok(cpu->lg, addr, bytes)
-	    || copy_from_user(b, cpu->lg->mem_base + addr, bytes) != 0) {
-		/* copy_from_user should do this, but as we rely on it... */
-		memset(b, 0, bytes);
-		kill_guest(cpu, "bad read address %#lx len %u", addr, bytes);
-	}
-}
-
-/* This is the write (copy into Guest) version. */
-void __lgwrite(struct lg_cpu *cpu, unsigned long addr, const void *b,
-	       unsigned bytes)
-{
-	if (!lguest_address_ok(cpu->lg, addr, bytes)
-	    || copy_to_user(cpu->lg->mem_base + addr, b, bytes) != 0)
-		kill_guest(cpu, "bad write address %#lx len %u", addr, bytes);
-}
-/*:*/
-
-/*H:030
- * Let's jump straight to the the main loop which runs the Guest.
- * Remember, this is called by the Launcher reading /dev/lguest, and we keep
- * going around and around until something interesting happens.
- */
-int run_guest(struct lg_cpu *cpu, unsigned long __user *user)
-{
-	/* If the launcher asked for a register with LHREQ_GETREG */
-	if (cpu->reg_read) {
-		if (put_user(*cpu->reg_read, user))
-			return -EFAULT;
-		cpu->reg_read = NULL;
-		return sizeof(*cpu->reg_read);
-	}
-
-	/* We stop running once the Guest is dead. */
-	while (!cpu->lg->dead) {
-		unsigned int irq;
-		bool more;
-
-		/* First we run any hypercalls the Guest wants done. */
-		if (cpu->hcall)
-			do_hypercalls(cpu);
-
-		/* Do we have to tell the Launcher about a trap? */
-		if (cpu->pending.trap) {
-			if (copy_to_user(user, &cpu->pending,
-					 sizeof(cpu->pending)))
-				return -EFAULT;
-			return sizeof(cpu->pending);
-		}
-
-		/*
-		 * All long-lived kernel loops need to check with this horrible
-		 * thing called the freezer.  If the Host is trying to suspend,
-		 * it stops us.
-		 */
-		try_to_freeze();
-
-		/* Check for signals */
-		if (signal_pending(current))
-			return -ERESTARTSYS;
-
-		/*
-		 * Check if there are any interrupts which can be delivered now:
-		 * if so, this sets up the hander to be executed when we next
-		 * run the Guest.
-		 */
-		irq = interrupt_pending(cpu, &more);
-		if (irq < LGUEST_IRQS)
-			try_deliver_interrupt(cpu, irq, more);
-
-		/*
-		 * Just make absolutely sure the Guest is still alive.  One of
-		 * those hypercalls could have been fatal, for example.
-		 */
-		if (cpu->lg->dead)
-			break;
-
-		/*
-		 * If the Guest asked to be stopped, we sleep.  The Guest's
-		 * clock timer will wake us.
-		 */
-		if (cpu->halted) {
-			set_current_state(TASK_INTERRUPTIBLE);
-			/*
-			 * Just before we sleep, make sure no interrupt snuck in
-			 * which we should be doing.
-			 */
-			if (interrupt_pending(cpu, &more) < LGUEST_IRQS)
-				set_current_state(TASK_RUNNING);
-			else
-				schedule();
-			continue;
-		}
-
-		/*
-		 * OK, now we're ready to jump into the Guest.  First we put up
-		 * the "Do Not Disturb" sign:
-		 */
-		local_irq_disable();
-
-		/* Actually run the Guest until something happens. */
-		lguest_arch_run_guest(cpu);
-
-		/* Now we're ready to be interrupted or moved to other CPUs */
-		local_irq_enable();
-
-		/* Now we deal with whatever happened to the Guest. */
-		lguest_arch_handle_trap(cpu);
-	}
-
-	/* Special case: Guest is 'dead' but wants a reboot. */
-	if (cpu->lg->dead == ERR_PTR(-ERESTART))
-		return -ERESTART;
-
-	/* The Guest is dead => "No such file or directory" */
-	return -ENOENT;
-}
-
-/*H:000
- * Welcome to the Host!
- *
- * By this point your brain has been tickled by the Guest code and numbed by
- * the Launcher code; prepare for it to be stretched by the Host code.  This is
- * the heart.  Let's begin at the initialization routine for the Host's lg
- * module.
- */
-static int __init init(void)
-{
-	int err;
-
-	/* Lguest can't run under Xen, VMI or itself.  It does Tricky Stuff. */
-	if (get_kernel_rpl() != 0) {
-		printk("lguest is afraid of being a guest\n");
-		return -EPERM;
-	}
-
-	/* First we put the Switcher up in very high virtual memory. */
-	err = map_switcher();
-	if (err)
-		goto out;
-
-	/* We might need to reserve an interrupt vector. */
-	err = init_interrupts();
-	if (err)
-		goto unmap;
-
-	/* /dev/lguest needs to be registered. */
-	err = lguest_device_init();
-	if (err)
-		goto free_interrupts;
-
-	/* Finally we do some architecture-specific setup. */
-	lguest_arch_host_init();
-
-	/* All good! */
-	return 0;
-
-free_interrupts:
-	free_interrupts();
-unmap:
-	unmap_switcher();
-out:
-	return err;
-}
-
-/* Cleaning up is just the same code, backwards.  With a little French. */
-static void __exit fini(void)
-{
-	lguest_device_remove();
-	free_interrupts();
-	unmap_switcher();
-
-	lguest_arch_host_fini();
-}
-/*:*/
-
-/*
- * The Host side of lguest can be a module.  This is a nice way for people to
- * play with it.
- */
-module_init(init);
-module_exit(fini);
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Rusty Russell <rusty@rustcorp.com.au>");
diff --git a/drivers/lguest/hypercalls.c b/drivers/lguest/hypercalls.c
deleted file mode 100644
index 601f81c04873..000000000000
--- a/drivers/lguest/hypercalls.c
+++ /dev/null
@@ -1,304 +0,0 @@
-/*P:500
- * Just as userspace programs request kernel operations through a system
- * call, the Guest requests Host operations through a "hypercall".  You might
- * notice this nomenclature doesn't really follow any logic, but the name has
- * been around for long enough that we're stuck with it.  As you'd expect, this
- * code is basically a one big switch statement.
-:*/
-
-/*  Copyright (C) 2006 Rusty Russell IBM Corporation
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program; if not, write to the Free Software
-    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
-*/
-#include <linux/uaccess.h>
-#include <linux/syscalls.h>
-#include <linux/mm.h>
-#include <linux/ktime.h>
-#include <asm/page.h>
-#include <asm/pgtable.h>
-#include "lg.h"
-
-/*H:120
- * This is the core hypercall routine: where the Guest gets what it wants.
- * Or gets killed.  Or, in the case of LHCALL_SHUTDOWN, both.
- */
-static void do_hcall(struct lg_cpu *cpu, struct hcall_args *args)
-{
-	switch (args->arg0) {
-	case LHCALL_FLUSH_ASYNC:
-		/*
-		 * This call does nothing, except by breaking out of the Guest
-		 * it makes us process all the asynchronous hypercalls.
-		 */
-		break;
-	case LHCALL_SEND_INTERRUPTS:
-		/*
-		 * This call does nothing too, but by breaking out of the Guest
-		 * it makes us process any pending interrupts.
-		 */
-		break;
-	case LHCALL_LGUEST_INIT:
-		/*
-		 * You can't get here unless you're already initialized.  Don't
-		 * do that.
-		 */
-		kill_guest(cpu, "already have lguest_data");
-		break;
-	case LHCALL_SHUTDOWN: {
-		char msg[128];
-		/*
-		 * Shutdown is such a trivial hypercall that we do it in five
-		 * lines right here.
-		 *
-		 * If the lgread fails, it will call kill_guest() itself; the
-		 * kill_guest() with the message will be ignored.
-		 */
-		__lgread(cpu, msg, args->arg1, sizeof(msg));
-		msg[sizeof(msg)-1] = '\0';
-		kill_guest(cpu, "CRASH: %s", msg);
-		if (args->arg2 == LGUEST_SHUTDOWN_RESTART)
-			cpu->lg->dead = ERR_PTR(-ERESTART);
-		break;
-	}
-	case LHCALL_FLUSH_TLB:
-		/* FLUSH_TLB comes in two flavors, depending on the argument: */
-		if (args->arg1)
-			guest_pagetable_clear_all(cpu);
-		else
-			guest_pagetable_flush_user(cpu);
-		break;
-
-	/*
-	 * All these calls simply pass the arguments through to the right
-	 * routines.
-	 */
-	case LHCALL_NEW_PGTABLE:
-		guest_new_pagetable(cpu, args->arg1);
-		break;
-	case LHCALL_SET_STACK:
-		guest_set_stack(cpu, args->arg1, args->arg2, args->arg3);
-		break;
-	case LHCALL_SET_PTE:
-#ifdef CONFIG_X86_PAE
-		guest_set_pte(cpu, args->arg1, args->arg2,
-				__pte(args->arg3 | (u64)args->arg4 << 32));
-#else
-		guest_set_pte(cpu, args->arg1, args->arg2, __pte(args->arg3));
-#endif
-		break;
-	case LHCALL_SET_PGD:
-		guest_set_pgd(cpu->lg, args->arg1, args->arg2);
-		break;
-#ifdef CONFIG_X86_PAE
-	case LHCALL_SET_PMD:
-		guest_set_pmd(cpu->lg, args->arg1, args->arg2);
-		break;
-#endif
-	case LHCALL_SET_CLOCKEVENT:
-		guest_set_clockevent(cpu, args->arg1);
-		break;
-	case LHCALL_HALT:
-		/* Similarly, this sets the halted flag for run_guest(). */
-		cpu->halted = 1;
-		break;
-	default:
-		/* It should be an architecture-specific hypercall. */
-		if (lguest_arch_do_hcall(cpu, args))
-			kill_guest(cpu, "Bad hypercall %li\n", args->arg0);
-	}
-}
-
-/*H:124
- * Asynchronous hypercalls are easy: we just look in the array in the
- * Guest's "struct lguest_data" to see if any new ones are marked "ready".
- *
- * We are careful to do these in order: obviously we respect the order the
- * Guest put them in the ring, but we also promise the Guest that they will
- * happen before any normal hypercall (which is why we check this before
- * checking for a normal hcall).
- */
-static void do_async_hcalls(struct lg_cpu *cpu)
-{
-	unsigned int i;
-	u8 st[LHCALL_RING_SIZE];
-
-	/* For simplicity, we copy the entire call status array in at once. */
-	if (copy_from_user(&st, &cpu->lg->lguest_data->hcall_status, sizeof(st)))
-		return;
-
-	/* We process "struct lguest_data"s hcalls[] ring once. */
-	for (i = 0; i < ARRAY_SIZE(st); i++) {
-		struct hcall_args args;
-		/*
-		 * We remember where we were up to from last time.  This makes
-		 * sure that the hypercalls are done in the order the Guest
-		 * places them in the ring.
-		 */
-		unsigned int n = cpu->next_hcall;
-
-		/* 0xFF means there's no call here (yet). */
-		if (st[n] == 0xFF)
-			break;
-
-		/*
-		 * OK, we have hypercall.  Increment the "next_hcall" cursor,
-		 * and wrap back to 0 if we reach the end.
-		 */
-		if (++cpu->next_hcall == LHCALL_RING_SIZE)
-			cpu->next_hcall = 0;
-
-		/*
-		 * Copy the hypercall arguments into a local copy of the
-		 * hcall_args struct.
-		 */
-		if (copy_from_user(&args, &cpu->lg->lguest_data->hcalls[n],
-				   sizeof(struct hcall_args))) {
-			kill_guest(cpu, "Fetching async hypercalls");
-			break;
-		}
-
-		/* Do the hypercall, same as a normal one. */
-		do_hcall(cpu, &args);
-
-		/* Mark the hypercall done. */
-		if (put_user(0xFF, &cpu->lg->lguest_data->hcall_status[n])) {
-			kill_guest(cpu, "Writing result for async hypercall");
-			break;
-		}
-
-		/*
-		 * Stop doing hypercalls if they want to notify the Launcher:
-		 * it needs to service this first.
-		 */
-		if (cpu->pending.trap)
-			break;
-	}
-}
-
-/*
- * Last of all, we look at what happens first of all.  The very first time the
- * Guest makes a hypercall, we end up here to set things up:
- */
-static void initialize(struct lg_cpu *cpu)
-{
-	/*
-	 * You can't do anything until you're initialized.  The Guest knows the
-	 * rules, so we're unforgiving here.
-	 */
-	if (cpu->hcall->arg0 != LHCALL_LGUEST_INIT) {
-		kill_guest(cpu, "hypercall %li before INIT", cpu->hcall->arg0);
-		return;
-	}
-
-	if (lguest_arch_init_hypercalls(cpu))
-		kill_guest(cpu, "bad guest page %p", cpu->lg->lguest_data);
-
-	/*
-	 * The Guest tells us where we're not to deliver interrupts by putting
-	 * the instruction address into "struct lguest_data".
-	 */
-	if (get_user(cpu->lg->noirq_iret, &cpu->lg->lguest_data->noirq_iret))
-		kill_guest(cpu, "bad guest page %p", cpu->lg->lguest_data);
-
-	/*
-	 * We write the current time into the Guest's data page once so it can
-	 * set its clock.
-	 */
-	write_timestamp(cpu);
-
-	/* page_tables.c will also do some setup. */
-	page_table_guest_data_init(cpu);
-
-	/*
-	 * This is the one case where the above accesses might have been the
-	 * first write to a Guest page.  This may have caused a copy-on-write
-	 * fault, but the old page might be (read-only) in the Guest
-	 * pagetable.
-	 */
-	guest_pagetable_clear_all(cpu);
-}
-/*:*/
-
-/*M:013
- * If a Guest reads from a page (so creates a mapping) that it has never
- * written to, and then the Launcher writes to it (ie. the output of a virtual
- * device), the Guest will still see the old page.  In practice, this never
- * happens: why would the Guest read a page which it has never written to?  But
- * a similar scenario might one day bite us, so it's worth mentioning.
- *
- * Note that if we used a shared anonymous mapping in the Launcher instead of
- * mapping /dev/zero private, we wouldn't worry about cop-on-write.  And we
- * need that to switch the Launcher to processes (away from threads) anyway.
-:*/
-
-/*H:100
- * Hypercalls
- *
- * Remember from the Guest, hypercalls come in two flavors: normal and
- * asynchronous.  This file handles both of types.
- */
-void do_hypercalls(struct lg_cpu *cpu)
-{
-	/* Not initialized yet?  This hypercall must do it. */
-	if (unlikely(!cpu->lg->lguest_data)) {
-		/* Set up the "struct lguest_data" */
-		initialize(cpu);
-		/* Hcall is done. */
-		cpu->hcall = NULL;
-		return;
-	}
-
-	/*
-	 * The Guest has initialized.
-	 *
-	 * Look in the hypercall ring for the async hypercalls:
-	 */
-	do_async_hcalls(cpu);
-
-	/*
-	 * If we stopped reading the hypercall ring because the Guest did a
-	 * NOTIFY to the Launcher, we want to return now.  Otherwise we do
-	 * the hypercall.
-	 */
-	if (!cpu->pending.trap) {
-		do_hcall(cpu, cpu->hcall);
-		/*
-		 * Tricky point: we reset the hcall pointer to mark the
-		 * hypercall as "done".  We use the hcall pointer rather than
-		 * the trap number to indicate a hypercall is pending.
-		 * Normally it doesn't matter: the Guest will run again and
-		 * update the trap number before we come back here.
-		 *
-		 * However, if we are signalled or the Guest sends I/O to the
-		 * Launcher, the run_guest() loop will exit without running the
-		 * Guest.  When it comes back it would try to re-run the
-		 * hypercall.  Finding that bug sucked.
-		 */
-		cpu->hcall = NULL;
-	}
-}
-
-/*
- * This routine supplies the Guest with time: it's used for wallclock time at
- * initial boot and as a rough time source if the TSC isn't available.
- */
-void write_timestamp(struct lg_cpu *cpu)
-{
-	struct timespec now;
-	ktime_get_real_ts(&now);
-	if (copy_to_user(&cpu->lg->lguest_data->time,
-			 &now, sizeof(struct timespec)))
-		kill_guest(cpu, "Writing timestamp");
-}
diff --git a/drivers/lguest/interrupts_and_traps.c b/drivers/lguest/interrupts_and_traps.c
deleted file mode 100644
index 67392b6ab845..000000000000
--- a/drivers/lguest/interrupts_and_traps.c
+++ /dev/null
@@ -1,706 +0,0 @@
-/*P:800
- * Interrupts (traps) are complicated enough to earn their own file.
- * There are three classes of interrupts:
- *
- * 1) Real hardware interrupts which occur while we're running the Guest,
- * 2) Interrupts for virtual devices attached to the Guest, and
- * 3) Traps and faults from the Guest.
- *
- * Real hardware interrupts must be delivered to the Host, not the Guest.
- * Virtual interrupts must be delivered to the Guest, but we make them look
- * just like real hardware would deliver them.  Traps from the Guest can be set
- * up to go directly back into the Guest, but sometimes the Host wants to see
- * them first, so we also have a way of "reflecting" them into the Guest as if
- * they had been delivered to it directly.
-:*/
-#include <linux/uaccess.h>
-#include <linux/interrupt.h>
-#include <linux/module.h>
-#include <linux/sched.h>
-#include "lg.h"
-
-/* Allow Guests to use a non-128 (ie. non-Linux) syscall trap. */
-static unsigned int syscall_vector = IA32_SYSCALL_VECTOR;
-module_param(syscall_vector, uint, 0444);
-
-/* The address of the interrupt handler is split into two bits: */
-static unsigned long idt_address(u32 lo, u32 hi)
-{
-	return (lo & 0x0000FFFF) | (hi & 0xFFFF0000);
-}
-
-/*
- * The "type" of the interrupt handler is a 4 bit field: we only support a
- * couple of types.
- */
-static int idt_type(u32 lo, u32 hi)
-{
-	return (hi >> 8) & 0xF;
-}
-
-/* An IDT entry can't be used unless the "present" bit is set. */
-static bool idt_present(u32 lo, u32 hi)
-{
-	return (hi & 0x8000);
-}
-
-/*
- * We need a helper to "push" a value onto the Guest's stack, since that's a
- * big part of what delivering an interrupt does.
- */
-static void push_guest_stack(struct lg_cpu *cpu, unsigned long *gstack, u32 val)
-{
-	/* Stack grows upwards: move stack then write value. */
-	*gstack -= 4;
-	lgwrite(cpu, *gstack, u32, val);
-}
-
-/*H:210
- * The push_guest_interrupt_stack() routine saves Guest state on the stack for
- * an interrupt or trap.  The mechanics of delivering traps and interrupts to
- * the Guest are the same, except some traps have an "error code" which gets
- * pushed onto the stack as well: the caller tells us if this is one.
- *
- * We set up the stack just like the CPU does for a real interrupt, so it's
- * identical for the Guest (and the standard "iret" instruction will undo
- * it).
- */
-static void push_guest_interrupt_stack(struct lg_cpu *cpu, bool has_err)
-{
-	unsigned long gstack, origstack;
-	u32 eflags, ss, irq_enable;
-	unsigned long virtstack;
-
-	/*
-	 * There are two cases for interrupts: one where the Guest is already
-	 * in the kernel, and a more complex one where the Guest is in
-	 * userspace.  We check the privilege level to find out.
-	 */
-	if ((cpu->regs->ss&0x3) != GUEST_PL) {
-		/*
-		 * The Guest told us their kernel stack with the SET_STACK
-		 * hypercall: both the virtual address and the segment.
-		 */
-		virtstack = cpu->esp1;
-		ss = cpu->ss1;
-
-		origstack = gstack = guest_pa(cpu, virtstack);
-		/*
-		 * We push the old stack segment and pointer onto the new
-		 * stack: when the Guest does an "iret" back from the interrupt
-		 * handler the CPU will notice they're dropping privilege
-		 * levels and expect these here.
-		 */
-		push_guest_stack(cpu, &gstack, cpu->regs->ss);
-		push_guest_stack(cpu, &gstack, cpu->regs->esp);
-	} else {
-		/* We're staying on the same Guest (kernel) stack. */
-		virtstack = cpu->regs->esp;
-		ss = cpu->regs->ss;
-
-		origstack = gstack = guest_pa(cpu, virtstack);
-	}
-
-	/*
-	 * Remember that we never let the Guest actually disable interrupts, so
-	 * the "Interrupt Flag" bit is always set.  We copy that bit from the
-	 * Guest's "irq_enabled" field into the eflags word: we saw the Guest
-	 * copy it back in "lguest_iret".
-	 */
-	eflags = cpu->regs->eflags;
-	if (get_user(irq_enable, &cpu->lg->lguest_data->irq_enabled) == 0
-	    && !(irq_enable & X86_EFLAGS_IF))
-		eflags &= ~X86_EFLAGS_IF;
-
-	/*
-	 * An interrupt is expected to push three things on the stack: the old
-	 * "eflags" word, the old code segment, and the old instruction
-	 * pointer.
-	 */
-	push_guest_stack(cpu, &gstack, eflags);
-	push_guest_stack(cpu, &gstack, cpu->regs->cs);
-	push_guest_stack(cpu, &gstack, cpu->regs->eip);
-
-	/* For the six traps which supply an error code, we push that, too. */
-	if (has_err)
-		push_guest_stack(cpu, &gstack, cpu->regs->errcode);
-
-	/* Adjust the stack pointer and stack segment. */
-	cpu->regs->ss = ss;
-	cpu->regs->esp = virtstack + (gstack - origstack);
-}
-
-/*
- * This actually makes the Guest start executing the given interrupt/trap
- * handler.
- *
- * "lo" and "hi" are the two parts of the Interrupt Descriptor Table for this
- * interrupt or trap.  It's split into two parts for traditional reasons: gcc
- * on i386 used to be frightened by 64 bit numbers.
- */
-static void guest_run_interrupt(struct lg_cpu *cpu, u32 lo, u32 hi)
-{
-	/* If we're already in the kernel, we don't change stacks. */
-	if ((cpu->regs->ss&0x3) != GUEST_PL)
-		cpu->regs->ss = cpu->esp1;
-
-	/*
-	 * Set the code segment and the address to execute.
-	 */
-	cpu->regs->cs = (__KERNEL_CS|GUEST_PL);
-	cpu->regs->eip = idt_address(lo, hi);
-
-	/*
-	 * Trapping always clears these flags:
-	 * TF: Trap flag
-	 * VM: Virtual 8086 mode
-	 * RF: Resume
-	 * NT: Nested task.
-	 */
-	cpu->regs->eflags &=
-		~(X86_EFLAGS_TF|X86_EFLAGS_VM|X86_EFLAGS_RF|X86_EFLAGS_NT);
-
-	/*
-	 * There are two kinds of interrupt handlers: 0xE is an "interrupt
-	 * gate" which expects interrupts to be disabled on entry.
-	 */
-	if (idt_type(lo, hi) == 0xE)
-		if (put_user(0, &cpu->lg->lguest_data->irq_enabled))
-			kill_guest(cpu, "Disabling interrupts");
-}
-
-/* This restores the eflags word which was pushed on the stack by a trap */
-static void restore_eflags(struct lg_cpu *cpu)
-{
-	/* This is the physical address of the stack. */
-	unsigned long stack_pa = guest_pa(cpu, cpu->regs->esp);
-
-	/*
-	 * Stack looks like this:
-	 * Address	Contents
-	 * esp		EIP
-	 * esp + 4	CS
-	 * esp + 8	EFLAGS
-	 */
-	cpu->regs->eflags = lgread(cpu, stack_pa + 8, u32);
-	cpu->regs->eflags &=
-		~(X86_EFLAGS_TF|X86_EFLAGS_VM|X86_EFLAGS_RF|X86_EFLAGS_NT);
-}
-
-/*H:205
- * Virtual Interrupts.
- *
- * interrupt_pending() returns the first pending interrupt which isn't blocked
- * by the Guest.  It is called before every entry to the Guest, and just before
- * we go to sleep when the Guest has halted itself.
- */
-unsigned int interrupt_pending(struct lg_cpu *cpu, bool *more)
-{
-	unsigned int irq;
-	DECLARE_BITMAP(blk, LGUEST_IRQS);
-
-	/* If the Guest hasn't even initialized yet, we can do nothing. */
-	if (!cpu->lg->lguest_data)
-		return LGUEST_IRQS;
-
-	/*
-	 * Take our "irqs_pending" array and remove any interrupts the Guest
-	 * wants blocked: the result ends up in "blk".
-	 */
-	if (copy_from_user(&blk, cpu->lg->lguest_data->blocked_interrupts,
-			   sizeof(blk)))
-		return LGUEST_IRQS;
-	bitmap_andnot(blk, cpu->irqs_pending, blk, LGUEST_IRQS);
-
-	/* Find the first interrupt. */
-	irq = find_first_bit(blk, LGUEST_IRQS);
-	*more = find_next_bit(blk, LGUEST_IRQS, irq+1);
-
-	return irq;
-}
-
-/*
- * This actually diverts the Guest to running an interrupt handler, once an
- * interrupt has been identified by interrupt_pending().
- */
-void try_deliver_interrupt(struct lg_cpu *cpu, unsigned int irq, bool more)
-{
-	struct desc_struct *idt;
-
-	BUG_ON(irq >= LGUEST_IRQS);
-
-	/* If they're halted, interrupts restart them. */
-	if (cpu->halted) {
-		/* Re-enable interrupts. */
-		if (put_user(X86_EFLAGS_IF, &cpu->lg->lguest_data->irq_enabled))
-			kill_guest(cpu, "Re-enabling interrupts");
-		cpu->halted = 0;
-	} else {
-		/* Otherwise we check if they have interrupts disabled. */
-		u32 irq_enabled;
-		if (get_user(irq_enabled, &cpu->lg->lguest_data->irq_enabled))
-			irq_enabled = 0;
-		if (!irq_enabled) {
-			/* Make sure they know an IRQ is pending. */
-			put_user(X86_EFLAGS_IF,
-				 &cpu->lg->lguest_data->irq_pending);
-			return;
-		}
-	}
-
-	/*
-	 * Look at the IDT entry the Guest gave us for this interrupt.  The
-	 * first 32 (FIRST_EXTERNAL_VECTOR) entries are for traps, so we skip
-	 * over them.
-	 */
-	idt = &cpu->arch.idt[FIRST_EXTERNAL_VECTOR+irq];
-	/* If they don't have a handler (yet?), we just ignore it */
-	if (idt_present(idt->a, idt->b)) {
-		/* OK, mark it no longer pending and deliver it. */
-		clear_bit(irq, cpu->irqs_pending);
-
-		/*
-		 * They may be about to iret, where they asked us never to
-		 * deliver interrupts.  In this case, we can emulate that iret
-		 * then immediately deliver the interrupt.  This is basically
-		 * a noop: the iret would pop the interrupt frame and restore
-		 * eflags, and then we'd set it up again.  So just restore the
-		 * eflags word and jump straight to the handler in this case.
-		 *
-		 * Denys Vlasenko points out that this isn't quite right: if
-		 * the iret was returning to userspace, then that interrupt
-		 * would reset the stack pointer (which the Guest told us
-		 * about via LHCALL_SET_STACK).  But unless the Guest is being
-		 * *really* weird, that will be the same as the current stack
-		 * anyway.
-		 */
-		if (cpu->regs->eip == cpu->lg->noirq_iret) {
-			restore_eflags(cpu);
-		} else {
-			/*
-			 * set_guest_interrupt() takes a flag to say whether
-			 * this interrupt pushes an error code onto the stack
-			 * as well: virtual interrupts never do.
-			 */
-			push_guest_interrupt_stack(cpu, false);
-		}
-		/* Actually make Guest cpu jump to handler. */
-		guest_run_interrupt(cpu, idt->a, idt->b);
-	}
-
-	/*
-	 * Every time we deliver an interrupt, we update the timestamp in the
-	 * Guest's lguest_data struct.  It would be better for the Guest if we
-	 * did this more often, but it can actually be quite slow: doing it
-	 * here is a compromise which means at least it gets updated every
-	 * timer interrupt.
-	 */
-	write_timestamp(cpu);
-
-	/*
-	 * If there are no other interrupts we want to deliver, clear
-	 * the pending flag.
-	 */
-	if (!more)
-		put_user(0, &cpu->lg->lguest_data->irq_pending);
-}
-
-/* And this is the routine when we want to set an interrupt for the Guest. */
-void set_interrupt(struct lg_cpu *cpu, unsigned int irq)
-{
-	/*
-	 * Next time the Guest runs, the core code will see if it can deliver
-	 * this interrupt.
-	 */
-	set_bit(irq, cpu->irqs_pending);
-
-	/*
-	 * Make sure it sees it; it might be asleep (eg. halted), or running
-	 * the Guest right now, in which case kick_process() will knock it out.
-	 */
-	if (!wake_up_process(cpu->tsk))
-		kick_process(cpu->tsk);
-}
-/*:*/
-
-/*
- * Linux uses trap 128 for system calls.  Plan9 uses 64, and Ron Minnich sent
- * me a patch, so we support that too.  It'd be a big step for lguest if half
- * the Plan 9 user base were to start using it.
- *
- * Actually now I think of it, it's possible that Ron *is* half the Plan 9
- * userbase.  Oh well.
- */
-bool could_be_syscall(unsigned int num)
-{
-	/* Normal Linux IA32_SYSCALL_VECTOR or reserved vector? */
-	return num == IA32_SYSCALL_VECTOR || num == syscall_vector;
-}
-
-/* The syscall vector it wants must be unused by Host. */
-bool check_syscall_vector(struct lguest *lg)
-{
-	u32 vector;
-
-	if (get_user(vector, &lg->lguest_data->syscall_vec))
-		return false;
-
-	return could_be_syscall(vector);
-}
-
-int init_interrupts(void)
-{
-	/* If they want some strange system call vector, reserve it now */
-	if (syscall_vector != IA32_SYSCALL_VECTOR) {
-		if (test_bit(syscall_vector, used_vectors) ||
-		    vector_used_by_percpu_irq(syscall_vector)) {
-			printk(KERN_ERR "lg: couldn't reserve syscall %u\n",
-				 syscall_vector);
-			return -EBUSY;
-		}
-		set_bit(syscall_vector, used_vectors);
-	}
-
-	return 0;
-}
-
-void free_interrupts(void)
-{
-	if (syscall_vector != IA32_SYSCALL_VECTOR)
-		clear_bit(syscall_vector, used_vectors);
-}
-
-/*H:220
- * Now we've got the routines to deliver interrupts, delivering traps like
- * page fault is easy.  The only trick is that Intel decided that some traps
- * should have error codes:
- */
-static bool has_err(unsigned int trap)
-{
-	return (trap == 8 || (trap >= 10 && trap <= 14) || trap == 17);
-}
-
-/* deliver_trap() returns true if it could deliver the trap. */
-bool deliver_trap(struct lg_cpu *cpu, unsigned int num)
-{
-	/*
-	 * Trap numbers are always 8 bit, but we set an impossible trap number
-	 * for traps inside the Switcher, so check that here.
-	 */
-	if (num >= ARRAY_SIZE(cpu->arch.idt))
-		return false;
-
-	/*
-	 * Early on the Guest hasn't set the IDT entries (or maybe it put a
-	 * bogus one in): if we fail here, the Guest will be killed.
-	 */
-	if (!idt_present(cpu->arch.idt[num].a, cpu->arch.idt[num].b))
-		return false;
-	push_guest_interrupt_stack(cpu, has_err(num));
-	guest_run_interrupt(cpu, cpu->arch.idt[num].a,
-			    cpu->arch.idt[num].b);
-	return true;
-}
-
-/*H:250
- * Here's the hard part: returning to the Host every time a trap happens
- * and then calling deliver_trap() and re-entering the Guest is slow.
- * Particularly because Guest userspace system calls are traps (usually trap
- * 128).
- *
- * So we'd like to set up the IDT to tell the CPU to deliver traps directly
- * into the Guest.  This is possible, but the complexities cause the size of
- * this file to double!  However, 150 lines of code is worth writing for taking
- * system calls down from 1750ns to 270ns.  Plus, if lguest didn't do it, all
- * the other hypervisors would beat it up at lunchtime.
- *
- * This routine indicates if a particular trap number could be delivered
- * directly.
- *
- * Unfortunately, Linux 4.6 started using an interrupt gate instead of a
- * trap gate for syscalls, so this trick is ineffective.  See Mastery for
- * how we could do this anyway...
- */
-static bool direct_trap(unsigned int num)
-{
-	/*
-	 * Hardware interrupts don't go to the Guest at all (except system
-	 * call).
-	 */
-	if (num >= FIRST_EXTERNAL_VECTOR && !could_be_syscall(num))
-		return false;
-
-	/*
-	 * The Host needs to see page faults (for shadow paging and to save the
-	 * fault address), general protection faults (in/out emulation) and
-	 * device not available (TS handling) and of course, the hypercall trap.
-	 */
-	return num != 14 && num != 13 && num != 7 && num != LGUEST_TRAP_ENTRY;
-}
-/*:*/
-
-/*M:005
- * The Guest has the ability to turn its interrupt gates into trap gates,
- * if it is careful.  The Host will let trap gates can go directly to the
- * Guest, but the Guest needs the interrupts atomically disabled for an
- * interrupt gate.  The Host could provide a mechanism to register more
- * "no-interrupt" regions, and the Guest could point the trap gate at
- * instructions within that region, where it can safely disable interrupts.
- */
-
-/*M:006
- * The Guests do not use the sysenter (fast system call) instruction,
- * because it's hardcoded to enter privilege level 0 and so can't go direct.
- * It's about twice as fast as the older "int 0x80" system call, so it might
- * still be worthwhile to handle it in the Switcher and lcall down to the
- * Guest.  The sysenter semantics are hairy tho: search for that keyword in
- * entry.S
-:*/
-
-/*H:260
- * When we make traps go directly into the Guest, we need to make sure
- * the kernel stack is valid (ie. mapped in the page tables).  Otherwise, the
- * CPU trying to deliver the trap will fault while trying to push the interrupt
- * words on the stack: this is called a double fault, and it forces us to kill
- * the Guest.
- *
- * Which is deeply unfair, because (literally!) it wasn't the Guests' fault.
- */
-void pin_stack_pages(struct lg_cpu *cpu)
-{
-	unsigned int i;
-
-	/*
-	 * Depending on the CONFIG_4KSTACKS option, the Guest can have one or
-	 * two pages of stack space.
-	 */
-	for (i = 0; i < cpu->lg->stack_pages; i++)
-		/*
-		 * The stack grows *upwards*, so the address we're given is the
-		 * start of the page after the kernel stack.  Subtract one to
-		 * get back onto the first stack page, and keep subtracting to
-		 * get to the rest of the stack pages.
-		 */
-		pin_page(cpu, cpu->esp1 - 1 - i * PAGE_SIZE);
-}
-
-/*
- * Direct traps also mean that we need to know whenever the Guest wants to use
- * a different kernel stack, so we can change the guest TSS to use that
- * stack.  The TSS entries expect a virtual address, so unlike most addresses
- * the Guest gives us, the "esp" (stack pointer) value here is virtual, not
- * physical.
- *
- * In Linux each process has its own kernel stack, so this happens a lot: we
- * change stacks on each context switch.
- */
-void guest_set_stack(struct lg_cpu *cpu, u32 seg, u32 esp, unsigned int pages)
-{
-	/*
-	 * You're not allowed a stack segment with privilege level 0: bad Guest!
-	 */
-	if ((seg & 0x3) != GUEST_PL)
-		kill_guest(cpu, "bad stack segment %i", seg);
-	/* We only expect one or two stack pages. */
-	if (pages > 2)
-		kill_guest(cpu, "bad stack pages %u", pages);
-	/* Save where the stack is, and how many pages */
-	cpu->ss1 = seg;
-	cpu->esp1 = esp;
-	cpu->lg->stack_pages = pages;
-	/* Make sure the new stack pages are mapped */
-	pin_stack_pages(cpu);
-}
-
-/*
- * All this reference to mapping stacks leads us neatly into the other complex
- * part of the Host: page table handling.
- */
-
-/*H:235
- * This is the routine which actually checks the Guest's IDT entry and
- * transfers it into the entry in "struct lguest":
- */
-static void set_trap(struct lg_cpu *cpu, struct desc_struct *trap,
-		     unsigned int num, u32 lo, u32 hi)
-{
-	u8 type = idt_type(lo, hi);
-
-	/* We zero-out a not-present entry */
-	if (!idt_present(lo, hi)) {
-		trap->a = trap->b = 0;
-		return;
-	}
-
-	/* We only support interrupt and trap gates. */
-	if (type != 0xE && type != 0xF)
-		kill_guest(cpu, "bad IDT type %i", type);
-
-	/*
-	 * We only copy the handler address, present bit, privilege level and
-	 * type.  The privilege level controls where the trap can be triggered
-	 * manually with an "int" instruction.  This is usually GUEST_PL,
-	 * except for system calls which userspace can use.
-	 */
-	trap->a = ((__KERNEL_CS|GUEST_PL)<<16) | (lo&0x0000FFFF);
-	trap->b = (hi&0xFFFFEF00);
-}
-
-/*H:230
- * While we're here, dealing with delivering traps and interrupts to the
- * Guest, we might as well complete the picture: how the Guest tells us where
- * it wants them to go.  This would be simple, except making traps fast
- * requires some tricks.
- *
- * We saw the Guest setting Interrupt Descriptor Table (IDT) entries with the
- * LHCALL_LOAD_IDT_ENTRY hypercall before: that comes here.
- */
-void load_guest_idt_entry(struct lg_cpu *cpu, unsigned int num, u32 lo, u32 hi)
-{
-	/*
-	 * Guest never handles: NMI, doublefault, spurious interrupt or
-	 * hypercall.  We ignore when it tries to set them.
-	 */
-	if (num == 2 || num == 8 || num == 15 || num == LGUEST_TRAP_ENTRY)
-		return;
-
-	/*
-	 * Mark the IDT as changed: next time the Guest runs we'll know we have
-	 * to copy this again.
-	 */
-	cpu->changed |= CHANGED_IDT;
-
-	/* Check that the Guest doesn't try to step outside the bounds. */
-	if (num >= ARRAY_SIZE(cpu->arch.idt))
-		kill_guest(cpu, "Setting idt entry %u", num);
-	else
-		set_trap(cpu, &cpu->arch.idt[num], num, lo, hi);
-}
-
-/*
- * The default entry for each interrupt points into the Switcher routines which
- * simply return to the Host.  The run_guest() loop will then call
- * deliver_trap() to bounce it back into the Guest.
- */
-static void default_idt_entry(struct desc_struct *idt,
-			      int trap,
-			      const unsigned long handler,
-			      const struct desc_struct *base)
-{
-	/* A present interrupt gate. */
-	u32 flags = 0x8e00;
-
-	/*
-	 * Set the privilege level on the entry for the hypercall: this allows
-	 * the Guest to use the "int" instruction to trigger it.
-	 */
-	if (trap == LGUEST_TRAP_ENTRY)
-		flags |= (GUEST_PL << 13);
-	else if (base)
-		/*
-		 * Copy privilege level from what Guest asked for.  This allows
-		 * debug (int 3) traps from Guest userspace, for example.
-		 */
-		flags |= (base->b & 0x6000);
-
-	/* Now pack it into the IDT entry in its weird format. */
-	idt->a = (LGUEST_CS<<16) | (handler&0x0000FFFF);
-	idt->b = (handler&0xFFFF0000) | flags;
-}
-
-/* When the Guest first starts, we put default entries into the IDT. */
-void setup_default_idt_entries(struct lguest_ro_state *state,
-			       const unsigned long *def)
-{
-	unsigned int i;
-
-	for (i = 0; i < ARRAY_SIZE(state->guest_idt); i++)
-		default_idt_entry(&state->guest_idt[i], i, def[i], NULL);
-}
-
-/*H:240
- * We don't use the IDT entries in the "struct lguest" directly, instead
- * we copy them into the IDT which we've set up for Guests on this CPU, just
- * before we run the Guest.  This routine does that copy.
- */
-void copy_traps(const struct lg_cpu *cpu, struct desc_struct *idt,
-		const unsigned long *def)
-{
-	unsigned int i;
-
-	/*
-	 * We can simply copy the direct traps, otherwise we use the default
-	 * ones in the Switcher: they will return to the Host.
-	 */
-	for (i = 0; i < ARRAY_SIZE(cpu->arch.idt); i++) {
-		const struct desc_struct *gidt = &cpu->arch.idt[i];
-
-		/* If no Guest can ever override this trap, leave it alone. */
-		if (!direct_trap(i))
-			continue;
-
-		/*
-		 * Only trap gates (type 15) can go direct to the Guest.
-		 * Interrupt gates (type 14) disable interrupts as they are
-		 * entered, which we never let the Guest do.  Not present
-		 * entries (type 0x0) also can't go direct, of course.
-		 *
-		 * If it can't go direct, we still need to copy the priv. level:
-		 * they might want to give userspace access to a software
-		 * interrupt.
-		 */
-		if (idt_type(gidt->a, gidt->b) == 0xF)
-			idt[i] = *gidt;
-		else
-			default_idt_entry(&idt[i], i, def[i], gidt);
-	}
-}
-
-/*H:200
- * The Guest Clock.
- *
- * There are two sources of virtual interrupts.  We saw one in lguest_user.c:
- * the Launcher sending interrupts for virtual devices.  The other is the Guest
- * timer interrupt.
- *
- * The Guest uses the LHCALL_SET_CLOCKEVENT hypercall to tell us how long to
- * the next timer interrupt (in nanoseconds).  We use the high-resolution timer
- * infrastructure to set a callback at that time.
- *
- * 0 means "turn off the clock".
- */
-void guest_set_clockevent(struct lg_cpu *cpu, unsigned long delta)
-{
-	ktime_t expires;
-
-	if (unlikely(delta == 0)) {
-		/* Clock event device is shutting down. */
-		hrtimer_cancel(&cpu->hrt);
-		return;
-	}
-
-	/*
-	 * We use wallclock time here, so the Guest might not be running for
-	 * all the time between now and the timer interrupt it asked for.  This
-	 * is almost always the right thing to do.
-	 */
-	expires = ktime_add_ns(ktime_get_real(), delta);
-	hrtimer_start(&cpu->hrt, expires, HRTIMER_MODE_ABS);
-}
-
-/* This is the function called when the Guest's timer expires. */
-static enum hrtimer_restart clockdev_fn(struct hrtimer *timer)
-{
-	struct lg_cpu *cpu = container_of(timer, struct lg_cpu, hrt);
-
-	/* Remember the first interrupt is the timer interrupt. */
-	set_interrupt(cpu, 0);
-	return HRTIMER_NORESTART;
-}
-
-/* This sets up the timer for this Guest. */
-void init_clockdev(struct lg_cpu *cpu)
-{
-	hrtimer_init(&cpu->hrt, CLOCK_REALTIME, HRTIMER_MODE_ABS);
-	cpu->hrt.function = clockdev_fn;
-}
diff --git a/drivers/lguest/lg.h b/drivers/lguest/lg.h
deleted file mode 100644
index 2356a2318034..000000000000
--- a/drivers/lguest/lg.h
+++ /dev/null
@@ -1,258 +0,0 @@
-#ifndef _LGUEST_H
-#define _LGUEST_H
-
-#ifndef __ASSEMBLY__
-#include <linux/types.h>
-#include <linux/init.h>
-#include <linux/stringify.h>
-#include <linux/lguest.h>
-#include <linux/lguest_launcher.h>
-#include <linux/wait.h>
-#include <linux/hrtimer.h>
-#include <linux/err.h>
-#include <linux/slab.h>
-
-#include <asm/lguest.h>
-
-struct pgdir {
-	unsigned long gpgdir;
-	bool switcher_mapped;
-	int last_host_cpu;
-	pgd_t *pgdir;
-};
-
-/* We have two pages shared with guests, per cpu.  */
-struct lguest_pages {
-	/* This is the stack page mapped rw in guest */
-	char spare[PAGE_SIZE - sizeof(struct lguest_regs)];
-	struct lguest_regs regs;
-
-	/* This is the host state & guest descriptor page, ro in guest */
-	struct lguest_ro_state state;
-} __attribute__((aligned(PAGE_SIZE)));
-
-#define CHANGED_IDT		1
-#define CHANGED_GDT		2
-#define CHANGED_GDT_TLS		4 /* Actually a subset of CHANGED_GDT */
-#define CHANGED_ALL	        3
-
-struct lg_cpu {
-	unsigned int id;
-	struct lguest *lg;
-	struct task_struct *tsk;
-	struct mm_struct *mm; 	/* == tsk->mm, but that becomes NULL on exit */
-
-	u32 cr2;
-	u32 esp1;
-	u16 ss1;
-
-	/* Bitmap of what has changed: see CHANGED_* above. */
-	int changed;
-
-	/* Pending operation. */
-	struct lguest_pending pending;
-
-	unsigned long *reg_read; /* register from LHREQ_GETREG */
-
-	/* At end of a page shared mapped over lguest_pages in guest. */
-	unsigned long regs_page;
-	struct lguest_regs *regs;
-
-	struct lguest_pages *last_pages;
-
-	/* Initialization mode: linear map everything. */
-	bool linear_pages;
-	int cpu_pgd; /* Which pgd this cpu is currently using */
-
-	/* If a hypercall was asked for, this points to the arguments. */
-	struct hcall_args *hcall;
-	u32 next_hcall;
-
-	/* Virtual clock device */
-	struct hrtimer hrt;
-
-	/* Did the Guest tell us to halt? */
-	int halted;
-
-	/* Pending virtual interrupts */
-	DECLARE_BITMAP(irqs_pending, LGUEST_IRQS);
-
-	struct lg_cpu_arch arch;
-};
-
-/* The private info the thread maintains about the guest. */
-struct lguest {
-	struct lguest_data __user *lguest_data;
-	struct lg_cpu cpus[NR_CPUS];
-	unsigned int nr_cpus;
-
-	/* Valid guest memory pages must be < this. */
-	u32 pfn_limit;
-
-	/* Device memory is >= pfn_limit and < device_limit. */
-	u32 device_limit;
-
-	/*
-	 * This provides the offset to the base of guest-physical memory in the
-	 * Launcher.
-	 */
-	void __user *mem_base;
-	unsigned long kernel_address;
-
-	struct pgdir pgdirs[4];
-
-	unsigned long noirq_iret;
-
-	unsigned int stack_pages;
-	u32 tsc_khz;
-
-	/* Dead? */
-	const char *dead;
-};
-
-extern struct mutex lguest_lock;
-
-/* core.c: */
-bool lguest_address_ok(const struct lguest *lg,
-		       unsigned long addr, unsigned long len);
-void __lgread(struct lg_cpu *, void *, unsigned long, unsigned);
-void __lgwrite(struct lg_cpu *, unsigned long, const void *, unsigned);
-extern struct page **lg_switcher_pages;
-
-/*H:035
- * Using memory-copy operations like that is usually inconvient, so we
- * have the following helper macros which read and write a specific type (often
- * an unsigned long).
- *
- * This reads into a variable of the given type then returns that.
- */
-#define lgread(cpu, addr, type)						\
-	({ type _v; __lgread((cpu), &_v, (addr), sizeof(_v)); _v; })
-
-/* This checks that the variable is of the given type, then writes it out. */
-#define lgwrite(cpu, addr, type, val)				\
-	do {							\
-		typecheck(type, val);				\
-		__lgwrite((cpu), (addr), &(val), sizeof(val));	\
-	} while(0)
-/* (end of memory access helper routines) :*/
-
-int run_guest(struct lg_cpu *cpu, unsigned long __user *user);
-
-/*
- * Helper macros to obtain the first 12 or the last 20 bits, this is only the
- * first step in the migration to the kernel types.  pte_pfn is already defined
- * in the kernel.
- */
-#define pgd_flags(x)	(pgd_val(x) & ~PAGE_MASK)
-#define pgd_pfn(x)	(pgd_val(x) >> PAGE_SHIFT)
-#define pmd_flags(x)    (pmd_val(x) & ~PAGE_MASK)
-#define pmd_pfn(x)	(pmd_val(x) >> PAGE_SHIFT)
-
-/* interrupts_and_traps.c: */
-unsigned int interrupt_pending(struct lg_cpu *cpu, bool *more);
-void try_deliver_interrupt(struct lg_cpu *cpu, unsigned int irq, bool more);
-void set_interrupt(struct lg_cpu *cpu, unsigned int irq);
-bool deliver_trap(struct lg_cpu *cpu, unsigned int num);
-void load_guest_idt_entry(struct lg_cpu *cpu, unsigned int i,
-			  u32 low, u32 hi);
-void guest_set_stack(struct lg_cpu *cpu, u32 seg, u32 esp, unsigned int pages);
-void pin_stack_pages(struct lg_cpu *cpu);
-void setup_default_idt_entries(struct lguest_ro_state *state,
-			       const unsigned long *def);
-void copy_traps(const struct lg_cpu *cpu, struct desc_struct *idt,
-		const unsigned long *def);
-void guest_set_clockevent(struct lg_cpu *cpu, unsigned long delta);
-bool send_notify_to_eventfd(struct lg_cpu *cpu);
-void init_clockdev(struct lg_cpu *cpu);
-bool check_syscall_vector(struct lguest *lg);
-bool could_be_syscall(unsigned int num);
-int init_interrupts(void);
-void free_interrupts(void);
-
-/* segments.c: */
-void setup_default_gdt_entries(struct lguest_ro_state *state);
-void setup_guest_gdt(struct lg_cpu *cpu);
-void load_guest_gdt_entry(struct lg_cpu *cpu, unsigned int i,
-			  u32 low, u32 hi);
-void guest_load_tls(struct lg_cpu *cpu, unsigned long tls_array);
-void copy_gdt(const struct lg_cpu *cpu, struct desc_struct *gdt);
-void copy_gdt_tls(const struct lg_cpu *cpu, struct desc_struct *gdt);
-
-/* page_tables.c: */
-int init_guest_pagetable(struct lguest *lg);
-void free_guest_pagetable(struct lguest *lg);
-void guest_new_pagetable(struct lg_cpu *cpu, unsigned long pgtable);
-void guest_set_pgd(struct lguest *lg, unsigned long gpgdir, u32 i);
-#ifdef CONFIG_X86_PAE
-void guest_set_pmd(struct lguest *lg, unsigned long gpgdir, u32 i);
-#endif
-void guest_pagetable_clear_all(struct lg_cpu *cpu);
-void guest_pagetable_flush_user(struct lg_cpu *cpu);
-void guest_set_pte(struct lg_cpu *cpu, unsigned long gpgdir,
-		   unsigned long vaddr, pte_t val);
-void map_switcher_in_guest(struct lg_cpu *cpu, struct lguest_pages *pages);
-bool demand_page(struct lg_cpu *cpu, unsigned long cr2, int errcode,
-		 unsigned long *iomem);
-void pin_page(struct lg_cpu *cpu, unsigned long vaddr);
-bool __guest_pa(struct lg_cpu *cpu, unsigned long vaddr, unsigned long *paddr);
-unsigned long guest_pa(struct lg_cpu *cpu, unsigned long vaddr);
-void page_table_guest_data_init(struct lg_cpu *cpu);
-
-/* <arch>/core.c: */
-void lguest_arch_host_init(void);
-void lguest_arch_host_fini(void);
-void lguest_arch_run_guest(struct lg_cpu *cpu);
-void lguest_arch_handle_trap(struct lg_cpu *cpu);
-int lguest_arch_init_hypercalls(struct lg_cpu *cpu);
-int lguest_arch_do_hcall(struct lg_cpu *cpu, struct hcall_args *args);
-void lguest_arch_setup_regs(struct lg_cpu *cpu, unsigned long start);
-unsigned long *lguest_arch_regptr(struct lg_cpu *cpu, size_t reg_off, bool any);
-
-/* <arch>/switcher.S: */
-extern char start_switcher_text[], end_switcher_text[], switch_to_guest[];
-
-/* lguest_user.c: */
-int lguest_device_init(void);
-void lguest_device_remove(void);
-
-/* hypercalls.c: */
-void do_hypercalls(struct lg_cpu *cpu);
-void write_timestamp(struct lg_cpu *cpu);
-
-/*L:035
- * Let's step aside for the moment, to study one important routine that's used
- * widely in the Host code.
- *
- * There are many cases where the Guest can do something invalid, like pass crap
- * to a hypercall.  Since only the Guest kernel can make hypercalls, it's quite
- * acceptable to simply terminate the Guest and give the Launcher a nicely
- * formatted reason.  It's also simpler for the Guest itself, which doesn't
- * need to check most hypercalls for "success"; if you're still running, it
- * succeeded.
- *
- * Once this is called, the Guest will never run again, so most Host code can
- * call this then continue as if nothing had happened.  This means many
- * functions don't have to explicitly return an error code, which keeps the
- * code simple.
- *
- * It also means that this can be called more than once: only the first one is
- * remembered.  The only trick is that we still need to kill the Guest even if
- * we can't allocate memory to store the reason.  Linux has a neat way of
- * packing error codes into invalid pointers, so we use that here.
- *
- * Like any macro which uses an "if", it is safely wrapped in a run-once "do {
- * } while(0)".
- */
-#define kill_guest(cpu, fmt...)					\
-do {								\
-	if (!(cpu)->lg->dead) {					\
-		(cpu)->lg->dead = kasprintf(GFP_ATOMIC, fmt);	\
-		if (!(cpu)->lg->dead)				\
-			(cpu)->lg->dead = ERR_PTR(-ENOMEM);	\
-	}							\
-} while(0)
-/* (End of aside) :*/
-
-#endif	/* __ASSEMBLY__ */
-#endif	/* _LGUEST_H */
diff --git a/drivers/lguest/lguest_user.c b/drivers/lguest/lguest_user.c
deleted file mode 100644
index 1a6787bc9386..000000000000
--- a/drivers/lguest/lguest_user.c
+++ /dev/null
@@ -1,446 +0,0 @@
-/*P:200 This contains all the /dev/lguest code, whereby the userspace
- * launcher controls and communicates with the Guest.  For example,
- * the first write will tell us the Guest's memory layout and entry
- * point.  A read will run the Guest until something happens, such as
- * a signal or the Guest accessing a device.
-:*/
-#include <linux/uaccess.h>
-#include <linux/miscdevice.h>
-#include <linux/fs.h>
-#include <linux/sched.h>
-#include <linux/sched/mm.h>
-#include <linux/file.h>
-#include <linux/slab.h>
-#include <linux/export.h>
-#include "lg.h"
-
-/*L:052
-  The Launcher can get the registers, and also set some of them.
-*/
-static int getreg_setup(struct lg_cpu *cpu, const unsigned long __user *input)
-{
-	unsigned long which;
-
-	/* We re-use the ptrace structure to specify which register to read. */
-	if (get_user(which, input) != 0)
-		return -EFAULT;
-
-	/*
-	 * We set up the cpu register pointer, and their next read will
-	 * actually get the value (instead of running the guest).
-	 *
-	 * The last argument 'true' says we can access any register.
-	 */
-	cpu->reg_read = lguest_arch_regptr(cpu, which, true);
-	if (!cpu->reg_read)
-		return -ENOENT;
-
-	/* And because this is a write() call, we return the length used. */
-	return sizeof(unsigned long) * 2;
-}
-
-static int setreg(struct lg_cpu *cpu, const unsigned long __user *input)
-{
-	unsigned long which, value, *reg;
-
-	/* We re-use the ptrace structure to specify which register to read. */
-	if (get_user(which, input) != 0)
-		return -EFAULT;
-	input++;
-	if (get_user(value, input) != 0)
-		return -EFAULT;
-
-	/* The last argument 'false' means we can't access all registers. */
-	reg = lguest_arch_regptr(cpu, which, false);
-	if (!reg)
-		return -ENOENT;
-
-	*reg = value;
-
-	/* And because this is a write() call, we return the length used. */
-	return sizeof(unsigned long) * 3;
-}
-
-/*L:050
- * Sending an interrupt is done by writing LHREQ_IRQ and an interrupt
- * number to /dev/lguest.
- */
-static int user_send_irq(struct lg_cpu *cpu, const unsigned long __user *input)
-{
-	unsigned long irq;
-
-	if (get_user(irq, input) != 0)
-		return -EFAULT;
-	if (irq >= LGUEST_IRQS)
-		return -EINVAL;
-
-	/*
-	 * Next time the Guest runs, the core code will see if it can deliver
-	 * this interrupt.
-	 */
-	set_interrupt(cpu, irq);
-	return 0;
-}
-
-/*L:053
- * Deliver a trap: this is used by the Launcher if it can't emulate
- * an instruction.
- */
-static int trap(struct lg_cpu *cpu, const unsigned long __user *input)
-{
-	unsigned long trapnum;
-
-	if (get_user(trapnum, input) != 0)
-		return -EFAULT;
-
-	if (!deliver_trap(cpu, trapnum))
-		return -EINVAL;
-
-	return 0;
-}
-
-/*L:040
- * Once our Guest is initialized, the Launcher makes it run by reading
- * from /dev/lguest.
- */
-static ssize_t read(struct file *file, char __user *user, size_t size,loff_t*o)
-{
-	struct lguest *lg = file->private_data;
-	struct lg_cpu *cpu;
-	unsigned int cpu_id = *o;
-
-	/* You must write LHREQ_INITIALIZE first! */
-	if (!lg)
-		return -EINVAL;
-
-	/* Watch out for arbitrary vcpu indexes! */
-	if (cpu_id >= lg->nr_cpus)
-		return -EINVAL;
-
-	cpu = &lg->cpus[cpu_id];
-
-	/* If you're not the task which owns the Guest, go away. */
-	if (current != cpu->tsk)
-		return -EPERM;
-
-	/* If the Guest is already dead, we indicate why */
-	if (lg->dead) {
-		size_t len;
-
-		/* lg->dead either contains an error code, or a string. */
-		if (IS_ERR(lg->dead))
-			return PTR_ERR(lg->dead);
-
-		/* We can only return as much as the buffer they read with. */
-		len = min(size, strlen(lg->dead)+1);
-		if (copy_to_user(user, lg->dead, len) != 0)
-			return -EFAULT;
-		return len;
-	}
-
-	/*
-	 * If we returned from read() last time because the Guest sent I/O,
-	 * clear the flag.
-	 */
-	if (cpu->pending.trap)
-		cpu->pending.trap = 0;
-
-	/* Run the Guest until something interesting happens. */
-	return run_guest(cpu, (unsigned long __user *)user);
-}
-
-/*L:025
- * This actually initializes a CPU.  For the moment, a Guest is only
- * uniprocessor, so "id" is always 0.
- */
-static int lg_cpu_start(struct lg_cpu *cpu, unsigned id, unsigned long start_ip)
-{
-	/* We have a limited number of CPUs in the lguest struct. */
-	if (id >= ARRAY_SIZE(cpu->lg->cpus))
-		return -EINVAL;
-
-	/* Set up this CPU's id, and pointer back to the lguest struct. */
-	cpu->id = id;
-	cpu->lg = container_of(cpu, struct lguest, cpus[id]);
-	cpu->lg->nr_cpus++;
-
-	/* Each CPU has a timer it can set. */
-	init_clockdev(cpu);
-
-	/*
-	 * We need a complete page for the Guest registers: they are accessible
-	 * to the Guest and we can only grant it access to whole pages.
-	 */
-	cpu->regs_page = get_zeroed_page(GFP_KERNEL);
-	if (!cpu->regs_page)
-		return -ENOMEM;
-
-	/* We actually put the registers at the end of the page. */
-	cpu->regs = (void *)cpu->regs_page + PAGE_SIZE - sizeof(*cpu->regs);
-
-	/*
-	 * Now we initialize the Guest's registers, handing it the start
-	 * address.
-	 */
-	lguest_arch_setup_regs(cpu, start_ip);
-
-	/*
-	 * We keep a pointer to the Launcher task (ie. current task) for when
-	 * other Guests want to wake this one (eg. console input).
-	 */
-	cpu->tsk = current;
-
-	/*
-	 * We need to keep a pointer to the Launcher's memory map, because if
-	 * the Launcher dies we need to clean it up.  If we don't keep a
-	 * reference, it is destroyed before close() is called.
-	 */
-	cpu->mm = get_task_mm(cpu->tsk);
-
-	/*
-	 * We remember which CPU's pages this Guest used last, for optimization
-	 * when the same Guest runs on the same CPU twice.
-	 */
-	cpu->last_pages = NULL;
-
-	/* No error == success. */
-	return 0;
-}
-
-/*L:020
- * The initialization write supplies 3 pointer sized (32 or 64 bit) values (in
- * addition to the LHREQ_INITIALIZE value).  These are:
- *
- * base: The start of the Guest-physical memory inside the Launcher memory.
- *
- * pfnlimit: The highest (Guest-physical) page number the Guest should be
- * allowed to access.  The Guest memory lives inside the Launcher, so it sets
- * this to ensure the Guest can only reach its own memory.
- *
- * start: The first instruction to execute ("eip" in x86-speak).
- */
-static int initialize(struct file *file, const unsigned long __user *input)
-{
-	/* "struct lguest" contains all we (the Host) know about a Guest. */
-	struct lguest *lg;
-	int err;
-	unsigned long args[4];
-
-	/*
-	 * We grab the Big Lguest lock, which protects against multiple
-	 * simultaneous initializations.
-	 */
-	mutex_lock(&lguest_lock);
-	/* You can't initialize twice!  Close the device and start again... */
-	if (file->private_data) {
-		err = -EBUSY;
-		goto unlock;
-	}
-
-	if (copy_from_user(args, input, sizeof(args)) != 0) {
-		err = -EFAULT;
-		goto unlock;
-	}
-
-	lg = kzalloc(sizeof(*lg), GFP_KERNEL);
-	if (!lg) {
-		err = -ENOMEM;
-		goto unlock;
-	}
-
-	/* Populate the easy fields of our "struct lguest" */
-	lg->mem_base = (void __user *)args[0];
-	lg->pfn_limit = args[1];
-	lg->device_limit = args[3];
-
-	/* This is the first cpu (cpu 0) and it will start booting at args[2] */
-	err = lg_cpu_start(&lg->cpus[0], 0, args[2]);
-	if (err)
-		goto free_lg;
-
-	/*
-	 * Initialize the Guest's shadow page tables.  This allocates
-	 * memory, so can fail.
-	 */
-	err = init_guest_pagetable(lg);
-	if (err)
-		goto free_regs;
-
-	/* We keep our "struct lguest" in the file's private_data. */
-	file->private_data = lg;
-
-	mutex_unlock(&lguest_lock);
-
-	/* And because this is a write() call, we return the length used. */
-	return sizeof(args);
-
-free_regs:
-	/* FIXME: This should be in free_vcpu */
-	free_page(lg->cpus[0].regs_page);
-free_lg:
-	kfree(lg);
-unlock:
-	mutex_unlock(&lguest_lock);
-	return err;
-}
-
-/*L:010
- * The first operation the Launcher does must be a write.  All writes
- * start with an unsigned long number: for the first write this must be
- * LHREQ_INITIALIZE to set up the Guest.  After that the Launcher can use
- * writes of other values to send interrupts or set up receipt of notifications.
- *
- * Note that we overload the "offset" in the /dev/lguest file to indicate what
- * CPU number we're dealing with.  Currently this is always 0 since we only
- * support uniprocessor Guests, but you can see the beginnings of SMP support
- * here.
- */
-static ssize_t write(struct file *file, const char __user *in,
-		     size_t size, loff_t *off)
-{
-	/*
-	 * Once the Guest is initialized, we hold the "struct lguest" in the
-	 * file private data.
-	 */
-	struct lguest *lg = file->private_data;
-	const unsigned long __user *input = (const unsigned long __user *)in;
-	unsigned long req;
-	struct lg_cpu *uninitialized_var(cpu);
-	unsigned int cpu_id = *off;
-
-	/* The first value tells us what this request is. */
-	if (get_user(req, input) != 0)
-		return -EFAULT;
-	input++;
-
-	/* If you haven't initialized, you must do that first. */
-	if (req != LHREQ_INITIALIZE) {
-		if (!lg || (cpu_id >= lg->nr_cpus))
-			return -EINVAL;
-		cpu = &lg->cpus[cpu_id];
-
-		/* Once the Guest is dead, you can only read() why it died. */
-		if (lg->dead)
-			return -ENOENT;
-	}
-
-	switch (req) {
-	case LHREQ_INITIALIZE:
-		return initialize(file, input);
-	case LHREQ_IRQ:
-		return user_send_irq(cpu, input);
-	case LHREQ_GETREG:
-		return getreg_setup(cpu, input);
-	case LHREQ_SETREG:
-		return setreg(cpu, input);
-	case LHREQ_TRAP:
-		return trap(cpu, input);
-	default:
-		return -EINVAL;
-	}
-}
-
-static int open(struct inode *inode, struct file *file)
-{
-	file->private_data = NULL;
-
-	return 0;
-}
-
-/*L:060
- * The final piece of interface code is the close() routine.  It reverses
- * everything done in initialize().  This is usually called because the
- * Launcher exited.
- *
- * Note that the close routine returns 0 or a negative error number: it can't
- * really fail, but it can whine.  I blame Sun for this wart, and K&R C for
- * letting them do it.
-:*/
-static int close(struct inode *inode, struct file *file)
-{
-	struct lguest *lg = file->private_data;
-	unsigned int i;
-
-	/* If we never successfully initialized, there's nothing to clean up */
-	if (!lg)
-		return 0;
-
-	/*
-	 * We need the big lock, to protect from inter-guest I/O and other
-	 * Launchers initializing guests.
-	 */
-	mutex_lock(&lguest_lock);
-
-	/* Free up the shadow page tables for the Guest. */
-	free_guest_pagetable(lg);
-
-	for (i = 0; i < lg->nr_cpus; i++) {
-		/* Cancels the hrtimer set via LHCALL_SET_CLOCKEVENT. */
-		hrtimer_cancel(&lg->cpus[i].hrt);
-		/* We can free up the register page we allocated. */
-		free_page(lg->cpus[i].regs_page);
-		/*
-		 * Now all the memory cleanups are done, it's safe to release
-		 * the Launcher's memory management structure.
-		 */
-		mmput(lg->cpus[i].mm);
-	}
-
-	/*
-	 * If lg->dead doesn't contain an error code it will be NULL or a
-	 * kmalloc()ed string, either of which is ok to hand to kfree().
-	 */
-	if (!IS_ERR(lg->dead))
-		kfree(lg->dead);
-	/* Free the memory allocated to the lguest_struct */
-	kfree(lg);
-	/* Release lock and exit. */
-	mutex_unlock(&lguest_lock);
-
-	return 0;
-}
-
-/*L:000
- * Welcome to our journey through the Launcher!
- *
- * The Launcher is the Host userspace program which sets up, runs and services
- * the Guest.  In fact, many comments in the Drivers which refer to "the Host"
- * doing things are inaccurate: the Launcher does all the device handling for
- * the Guest, but the Guest can't know that.
- *
- * Just to confuse you: to the Host kernel, the Launcher *is* the Guest and we
- * shall see more of that later.
- *
- * We begin our understanding with the Host kernel interface which the Launcher
- * uses: reading and writing a character device called /dev/lguest.  All the
- * work happens in the read(), write() and close() routines:
- */
-static const struct file_operations lguest_fops = {
-	.owner	 = THIS_MODULE,
-	.open	 = open,
-	.release = close,
-	.write	 = write,
-	.read	 = read,
-	.llseek  = default_llseek,
-};
-/*:*/
-
-/*
- * This is a textbook example of a "misc" character device.  Populate a "struct
- * miscdevice" and register it with misc_register().
- */
-static struct miscdevice lguest_dev = {
-	.minor	= MISC_DYNAMIC_MINOR,
-	.name	= "lguest",
-	.fops	= &lguest_fops,
-};
-
-int __init lguest_device_init(void)
-{
-	return misc_register(&lguest_dev);
-}
-
-void __exit lguest_device_remove(void)
-{
-	misc_deregister(&lguest_dev);
-}
diff --git a/drivers/lguest/page_tables.c b/drivers/lguest/page_tables.c
deleted file mode 100644
index 0bc127e9f16a..000000000000
--- a/drivers/lguest/page_tables.c
+++ /dev/null
@@ -1,1239 +0,0 @@
-/*P:700
- * The pagetable code, on the other hand, still shows the scars of
- * previous encounters.  It's functional, and as neat as it can be in the
- * circumstances, but be wary, for these things are subtle and break easily.
- * The Guest provides a virtual to physical mapping, but we can neither trust
- * it nor use it: we verify and convert it here then point the CPU to the
- * converted Guest pages when running the Guest.
-:*/
-
-/* Copyright (C) Rusty Russell IBM Corporation 2013.
- * GPL v2 and any later version */
-#include <linux/mm.h>
-#include <linux/gfp.h>
-#include <linux/types.h>
-#include <linux/spinlock.h>
-#include <linux/random.h>
-#include <linux/percpu.h>
-#include <asm/tlbflush.h>
-#include <linux/uaccess.h>
-#include "lg.h"
-
-/*M:008
- * We hold reference to pages, which prevents them from being swapped.
- * It'd be nice to have a callback in the "struct mm_struct" when Linux wants
- * to swap out.  If we had this, and a shrinker callback to trim PTE pages, we
- * could probably consider launching Guests as non-root.
-:*/
-
-/*H:300
- * The Page Table Code
- *
- * We use two-level page tables for the Guest, or three-level with PAE.  If
- * you're not entirely comfortable with virtual addresses, physical addresses
- * and page tables then I recommend you review arch/x86/lguest/boot.c's "Page
- * Table Handling" (with diagrams!).
- *
- * The Guest keeps page tables, but we maintain the actual ones here: these are
- * called "shadow" page tables.  Which is a very Guest-centric name: these are
- * the real page tables the CPU uses, although we keep them up to date to
- * reflect the Guest's.  (See what I mean about weird naming?  Since when do
- * shadows reflect anything?)
- *
- * Anyway, this is the most complicated part of the Host code.  There are seven
- * parts to this:
- *  (i) Looking up a page table entry when the Guest faults,
- *  (ii) Making sure the Guest stack is mapped,
- *  (iii) Setting up a page table entry when the Guest tells us one has changed,
- *  (iv) Switching page tables,
- *  (v) Flushing (throwing away) page tables,
- *  (vi) Mapping the Switcher when the Guest is about to run,
- *  (vii) Setting up the page tables initially.
-:*/
-
-/*
- * The Switcher uses the complete top PTE page.  That's 1024 PTE entries (4MB)
- * or 512 PTE entries with PAE (2MB).
- */
-#define SWITCHER_PGD_INDEX (PTRS_PER_PGD - 1)
-
-/*
- * For PAE we need the PMD index as well. We use the last 2MB, so we
- * will need the last pmd entry of the last pmd page.
- */
-#ifdef CONFIG_X86_PAE
-#define CHECK_GPGD_MASK		_PAGE_PRESENT
-#else
-#define CHECK_GPGD_MASK		_PAGE_TABLE
-#endif
-
-/*H:320
- * The page table code is curly enough to need helper functions to keep it
- * clear and clean.  The kernel itself provides many of them; one advantage
- * of insisting that the Guest and Host use the same CONFIG_X86_PAE setting.
- *
- * There are two functions which return pointers to the shadow (aka "real")
- * page tables.
- *
- * spgd_addr() takes the virtual address and returns a pointer to the top-level
- * page directory entry (PGD) for that address.  Since we keep track of several
- * page tables, the "i" argument tells us which one we're interested in (it's
- * usually the current one).
- */
-static pgd_t *spgd_addr(struct lg_cpu *cpu, u32 i, unsigned long vaddr)
-{
-	unsigned int index = pgd_index(vaddr);
-
-	/* Return a pointer index'th pgd entry for the i'th page table. */
-	return &cpu->lg->pgdirs[i].pgdir[index];
-}
-
-#ifdef CONFIG_X86_PAE
-/*
- * This routine then takes the PGD entry given above, which contains the
- * address of the PMD page.  It then returns a pointer to the PMD entry for the
- * given address.
- */
-static pmd_t *spmd_addr(struct lg_cpu *cpu, pgd_t spgd, unsigned long vaddr)
-{
-	unsigned int index = pmd_index(vaddr);
-	pmd_t *page;
-
-	/* You should never call this if the PGD entry wasn't valid */
-	BUG_ON(!(pgd_flags(spgd) & _PAGE_PRESENT));
-	page = __va(pgd_pfn(spgd) << PAGE_SHIFT);
-
-	return &page[index];
-}
-#endif
-
-/*
- * This routine then takes the page directory entry returned above, which
- * contains the address of the page table entry (PTE) page.  It then returns a
- * pointer to the PTE entry for the given address.
- */
-static pte_t *spte_addr(struct lg_cpu *cpu, pgd_t spgd, unsigned long vaddr)
-{
-#ifdef CONFIG_X86_PAE
-	pmd_t *pmd = spmd_addr(cpu, spgd, vaddr);
-	pte_t *page = __va(pmd_pfn(*pmd) << PAGE_SHIFT);
-
-	/* You should never call this if the PMD entry wasn't valid */
-	BUG_ON(!(pmd_flags(*pmd) & _PAGE_PRESENT));
-#else
-	pte_t *page = __va(pgd_pfn(spgd) << PAGE_SHIFT);
-	/* You should never call this if the PGD entry wasn't valid */
-	BUG_ON(!(pgd_flags(spgd) & _PAGE_PRESENT));
-#endif
-
-	return &page[pte_index(vaddr)];
-}
-
-/*
- * These functions are just like the above, except they access the Guest
- * page tables.  Hence they return a Guest address.
- */
-static unsigned long gpgd_addr(struct lg_cpu *cpu, unsigned long vaddr)
-{
-	unsigned int index = vaddr >> (PGDIR_SHIFT);
-	return cpu->lg->pgdirs[cpu->cpu_pgd].gpgdir + index * sizeof(pgd_t);
-}
-
-#ifdef CONFIG_X86_PAE
-/* Follow the PGD to the PMD. */
-static unsigned long gpmd_addr(pgd_t gpgd, unsigned long vaddr)
-{
-	unsigned long gpage = pgd_pfn(gpgd) << PAGE_SHIFT;
-	BUG_ON(!(pgd_flags(gpgd) & _PAGE_PRESENT));
-	return gpage + pmd_index(vaddr) * sizeof(pmd_t);
-}
-
-/* Follow the PMD to the PTE. */
-static unsigned long gpte_addr(struct lg_cpu *cpu,
-			       pmd_t gpmd, unsigned long vaddr)
-{
-	unsigned long gpage = pmd_pfn(gpmd) << PAGE_SHIFT;
-
-	BUG_ON(!(pmd_flags(gpmd) & _PAGE_PRESENT));
-	return gpage + pte_index(vaddr) * sizeof(pte_t);
-}
-#else
-/* Follow the PGD to the PTE (no mid-level for !PAE). */
-static unsigned long gpte_addr(struct lg_cpu *cpu,
-				pgd_t gpgd, unsigned long vaddr)
-{
-	unsigned long gpage = pgd_pfn(gpgd) << PAGE_SHIFT;
-
-	BUG_ON(!(pgd_flags(gpgd) & _PAGE_PRESENT));
-	return gpage + pte_index(vaddr) * sizeof(pte_t);
-}
-#endif
-/*:*/
-
-/*M:007
- * get_pfn is slow: we could probably try to grab batches of pages here as
- * an optimization (ie. pre-faulting).
-:*/
-
-/*H:350
- * This routine takes a page number given by the Guest and converts it to
- * an actual, physical page number.  It can fail for several reasons: the
- * virtual address might not be mapped by the Launcher, the write flag is set
- * and the page is read-only, or the write flag was set and the page was
- * shared so had to be copied, but we ran out of memory.
- *
- * This holds a reference to the page, so release_pte() is careful to put that
- * back.
- */
-static unsigned long get_pfn(unsigned long virtpfn, int write)
-{
-	struct page *page;
-
-	/* gup me one page at this address please! */
-	if (get_user_pages_fast(virtpfn << PAGE_SHIFT, 1, write, &page) == 1)
-		return page_to_pfn(page);
-
-	/* This value indicates failure. */
-	return -1UL;
-}
-
-/*H:340
- * Converting a Guest page table entry to a shadow (ie. real) page table
- * entry can be a little tricky.  The flags are (almost) the same, but the
- * Guest PTE contains a virtual page number: the CPU needs the real page
- * number.
- */
-static pte_t gpte_to_spte(struct lg_cpu *cpu, pte_t gpte, int write)
-{
-	unsigned long pfn, base, flags;
-
-	/*
-	 * The Guest sets the global flag, because it thinks that it is using
-	 * PGE.  We only told it to use PGE so it would tell us whether it was
-	 * flushing a kernel mapping or a userspace mapping.  We don't actually
-	 * use the global bit, so throw it away.
-	 */
-	flags = (pte_flags(gpte) & ~_PAGE_GLOBAL);
-
-	/* The Guest's pages are offset inside the Launcher. */
-	base = (unsigned long)cpu->lg->mem_base / PAGE_SIZE;
-
-	/*
-	 * We need a temporary "unsigned long" variable to hold the answer from
-	 * get_pfn(), because it returns 0xFFFFFFFF on failure, which wouldn't
-	 * fit in spte.pfn.  get_pfn() finds the real physical number of the
-	 * page, given the virtual number.
-	 */
-	pfn = get_pfn(base + pte_pfn(gpte), write);
-	if (pfn == -1UL) {
-		kill_guest(cpu, "failed to get page %lu", pte_pfn(gpte));
-		/*
-		 * When we destroy the Guest, we'll go through the shadow page
-		 * tables and release_pte() them.  Make sure we don't think
-		 * this one is valid!
-		 */
-		flags = 0;
-	}
-	/* Now we assemble our shadow PTE from the page number and flags. */
-	return pfn_pte(pfn, __pgprot(flags));
-}
-
-/*H:460 And to complete the chain, release_pte() looks like this: */
-static void release_pte(pte_t pte)
-{
-	/*
-	 * Remember that get_user_pages_fast() took a reference to the page, in
-	 * get_pfn()?  We have to put it back now.
-	 */
-	if (pte_flags(pte) & _PAGE_PRESENT)
-		put_page(pte_page(pte));
-}
-/*:*/
-
-static bool gpte_in_iomem(struct lg_cpu *cpu, pte_t gpte)
-{
-	/* We don't handle large pages. */
-	if (pte_flags(gpte) & _PAGE_PSE)
-		return false;
-
-	return (pte_pfn(gpte) >= cpu->lg->pfn_limit
-		&& pte_pfn(gpte) < cpu->lg->device_limit);
-}
-
-static bool check_gpte(struct lg_cpu *cpu, pte_t gpte)
-{
-	if ((pte_flags(gpte) & _PAGE_PSE) ||
-	    pte_pfn(gpte) >= cpu->lg->pfn_limit) {
-		kill_guest(cpu, "bad page table entry");
-		return false;
-	}
-	return true;
-}
-
-static bool check_gpgd(struct lg_cpu *cpu, pgd_t gpgd)
-{
-	if ((pgd_flags(gpgd) & ~CHECK_GPGD_MASK) ||
-	    (pgd_pfn(gpgd) >= cpu->lg->pfn_limit)) {
-		kill_guest(cpu, "bad page directory entry");
-		return false;
-	}
-	return true;
-}
-
-#ifdef CONFIG_X86_PAE
-static bool check_gpmd(struct lg_cpu *cpu, pmd_t gpmd)
-{
-	if ((pmd_flags(gpmd) & ~_PAGE_TABLE) ||
-	    (pmd_pfn(gpmd) >= cpu->lg->pfn_limit)) {
-		kill_guest(cpu, "bad page middle directory entry");
-		return false;
-	}
-	return true;
-}
-#endif
-
-/*H:331
- * This is the core routine to walk the shadow page tables and find the page
- * table entry for a specific address.
- *
- * If allocate is set, then we allocate any missing levels, setting the flags
- * on the new page directory and mid-level directories using the arguments
- * (which are copied from the Guest's page table entries).
- */
-static pte_t *find_spte(struct lg_cpu *cpu, unsigned long vaddr, bool allocate,
-			int pgd_flags, int pmd_flags)
-{
-	pgd_t *spgd;
-	/* Mid level for PAE. */
-#ifdef CONFIG_X86_PAE
-	pmd_t *spmd;
-#endif
-
-	/* Get top level entry. */
-	spgd = spgd_addr(cpu, cpu->cpu_pgd, vaddr);
-	if (!(pgd_flags(*spgd) & _PAGE_PRESENT)) {
-		/* No shadow entry: allocate a new shadow PTE page. */
-		unsigned long ptepage;
-
-		/* If they didn't want us to allocate anything, stop. */
-		if (!allocate)
-			return NULL;
-
-		ptepage = get_zeroed_page(GFP_KERNEL);
-		/*
-		 * This is not really the Guest's fault, but killing it is
-		 * simple for this corner case.
-		 */
-		if (!ptepage) {
-			kill_guest(cpu, "out of memory allocating pte page");
-			return NULL;
-		}
-		/*
-		 * And we copy the flags to the shadow PGD entry.  The page
-		 * number in the shadow PGD is the page we just allocated.
-		 */
-		set_pgd(spgd, __pgd(__pa(ptepage) | pgd_flags));
-	}
-
-	/*
-	 * Intel's Physical Address Extension actually uses three levels of
-	 * page tables, so we need to look in the mid-level.
-	 */
-#ifdef CONFIG_X86_PAE
-	/* Now look at the mid-level shadow entry. */
-	spmd = spmd_addr(cpu, *spgd, vaddr);
-
-	if (!(pmd_flags(*spmd) & _PAGE_PRESENT)) {
-		/* No shadow entry: allocate a new shadow PTE page. */
-		unsigned long ptepage;
-
-		/* If they didn't want us to allocate anything, stop. */
-		if (!allocate)
-			return NULL;
-
-		ptepage = get_zeroed_page(GFP_KERNEL);
-
-		/*
-		 * This is not really the Guest's fault, but killing it is
-		 * simple for this corner case.
-		 */
-		if (!ptepage) {
-			kill_guest(cpu, "out of memory allocating pmd page");
-			return NULL;
-		}
-
-		/*
-		 * And we copy the flags to the shadow PMD entry.  The page
-		 * number in the shadow PMD is the page we just allocated.
-		 */
-		set_pmd(spmd, __pmd(__pa(ptepage) | pmd_flags));
-	}
-#endif
-
-	/* Get the pointer to the shadow PTE entry we're going to set. */
-	return spte_addr(cpu, *spgd, vaddr);
-}
-
-/*H:330
- * (i) Looking up a page table entry when the Guest faults.
- *
- * We saw this call in run_guest(): when we see a page fault in the Guest, we
- * come here.  That's because we only set up the shadow page tables lazily as
- * they're needed, so we get page faults all the time and quietly fix them up
- * and return to the Guest without it knowing.
- *
- * If we fixed up the fault (ie. we mapped the address), this routine returns
- * true.  Otherwise, it was a real fault and we need to tell the Guest.
- *
- * There's a corner case: they're trying to access memory between
- * pfn_limit and device_limit, which is I/O memory.  In this case, we
- * return false and set @iomem to the physical address, so the the
- * Launcher can handle the instruction manually.
- */
-bool demand_page(struct lg_cpu *cpu, unsigned long vaddr, int errcode,
-		 unsigned long *iomem)
-{
-	unsigned long gpte_ptr;
-	pte_t gpte;
-	pte_t *spte;
-	pmd_t gpmd;
-	pgd_t gpgd;
-
-	*iomem = 0;
-
-	/* We never demand page the Switcher, so trying is a mistake. */
-	if (vaddr >= switcher_addr)
-		return false;
-
-	/* First step: get the top-level Guest page table entry. */
-	if (unlikely(cpu->linear_pages)) {
-		/* Faking up a linear mapping. */
-		gpgd = __pgd(CHECK_GPGD_MASK);
-	} else {
-		gpgd = lgread(cpu, gpgd_addr(cpu, vaddr), pgd_t);
-		/* Toplevel not present?  We can't map it in. */
-		if (!(pgd_flags(gpgd) & _PAGE_PRESENT))
-			return false;
-
-		/* 
-		 * This kills the Guest if it has weird flags or tries to
-		 * refer to a "physical" address outside the bounds.
-		 */
-		if (!check_gpgd(cpu, gpgd))
-			return false;
-	}
-
-	/* This "mid-level" entry is only used for non-linear, PAE mode. */
-	gpmd = __pmd(_PAGE_TABLE);
-
-#ifdef CONFIG_X86_PAE
-	if (likely(!cpu->linear_pages)) {
-		gpmd = lgread(cpu, gpmd_addr(gpgd, vaddr), pmd_t);
-		/* Middle level not present?  We can't map it in. */
-		if (!(pmd_flags(gpmd) & _PAGE_PRESENT))
-			return false;
-
-		/* 
-		 * This kills the Guest if it has weird flags or tries to
-		 * refer to a "physical" address outside the bounds.
-		 */
-		if (!check_gpmd(cpu, gpmd))
-			return false;
-	}
-
-	/*
-	 * OK, now we look at the lower level in the Guest page table: keep its
-	 * address, because we might update it later.
-	 */
-	gpte_ptr = gpte_addr(cpu, gpmd, vaddr);
-#else
-	/*
-	 * OK, now we look at the lower level in the Guest page table: keep its
-	 * address, because we might update it later.
-	 */
-	gpte_ptr = gpte_addr(cpu, gpgd, vaddr);
-#endif
-
-	if (unlikely(cpu->linear_pages)) {
-		/* Linear?  Make up a PTE which points to same page. */
-		gpte = __pte((vaddr & PAGE_MASK) | _PAGE_RW | _PAGE_PRESENT);
-	} else {
-		/* Read the actual PTE value. */
-		gpte = lgread(cpu, gpte_ptr, pte_t);
-	}
-
-	/* If this page isn't in the Guest page tables, we can't page it in. */
-	if (!(pte_flags(gpte) & _PAGE_PRESENT))
-		return false;
-
-	/*
-	 * Check they're not trying to write to a page the Guest wants
-	 * read-only (bit 2 of errcode == write).
-	 */
-	if ((errcode & 2) && !(pte_flags(gpte) & _PAGE_RW))
-		return false;
-
-	/* User access to a kernel-only page? (bit 3 == user access) */
-	if ((errcode & 4) && !(pte_flags(gpte) & _PAGE_USER))
-		return false;
-
-	/* If they're accessing io memory, we expect a fault. */
-	if (gpte_in_iomem(cpu, gpte)) {
-		*iomem = (pte_pfn(gpte) << PAGE_SHIFT) | (vaddr & ~PAGE_MASK);
-		return false;
-	}
-
-	/*
-	 * Check that the Guest PTE flags are OK, and the page number is below
-	 * the pfn_limit (ie. not mapping the Launcher binary).
-	 */
-	if (!check_gpte(cpu, gpte))
-		return false;
-
-	/* Add the _PAGE_ACCESSED and (for a write) _PAGE_DIRTY flag */
-	gpte = pte_mkyoung(gpte);
-	if (errcode & 2)
-		gpte = pte_mkdirty(gpte);
-
-	/* Get the pointer to the shadow PTE entry we're going to set. */
-	spte = find_spte(cpu, vaddr, true, pgd_flags(gpgd), pmd_flags(gpmd));
-	if (!spte)
-		return false;
-
-	/*
-	 * If there was a valid shadow PTE entry here before, we release it.
-	 * This can happen with a write to a previously read-only entry.
-	 */
-	release_pte(*spte);
-
-	/*
-	 * If this is a write, we insist that the Guest page is writable (the
-	 * final arg to gpte_to_spte()).
-	 */
-	if (pte_dirty(gpte))
-		*spte = gpte_to_spte(cpu, gpte, 1);
-	else
-		/*
-		 * If this is a read, don't set the "writable" bit in the page
-		 * table entry, even if the Guest says it's writable.  That way
-		 * we will come back here when a write does actually occur, so
-		 * we can update the Guest's _PAGE_DIRTY flag.
-		 */
-		set_pte(spte, gpte_to_spte(cpu, pte_wrprotect(gpte), 0));
-
-	/*
-	 * Finally, we write the Guest PTE entry back: we've set the
-	 * _PAGE_ACCESSED and maybe the _PAGE_DIRTY flags.
-	 */
-	if (likely(!cpu->linear_pages))
-		lgwrite(cpu, gpte_ptr, pte_t, gpte);
-
-	/*
-	 * The fault is fixed, the page table is populated, the mapping
-	 * manipulated, the result returned and the code complete.  A small
-	 * delay and a trace of alliteration are the only indications the Guest
-	 * has that a page fault occurred at all.
-	 */
-	return true;
-}
-
-/*H:360
- * (ii) Making sure the Guest stack is mapped.
- *
- * Remember that direct traps into the Guest need a mapped Guest kernel stack.
- * pin_stack_pages() calls us here: we could simply call demand_page(), but as
- * we've seen that logic is quite long, and usually the stack pages are already
- * mapped, so it's overkill.
- *
- * This is a quick version which answers the question: is this virtual address
- * mapped by the shadow page tables, and is it writable?
- */
-static bool page_writable(struct lg_cpu *cpu, unsigned long vaddr)
-{
-	pte_t *spte;
-	unsigned long flags;
-
-	/* You can't put your stack in the Switcher! */
-	if (vaddr >= switcher_addr)
-		return false;
-
-	/* If there's no shadow PTE, it's not writable. */
-	spte = find_spte(cpu, vaddr, false, 0, 0);
-	if (!spte)
-		return false;
-
-	/*
-	 * Check the flags on the pte entry itself: it must be present and
-	 * writable.
-	 */
-	flags = pte_flags(*spte);
-	return (flags & (_PAGE_PRESENT|_PAGE_RW)) == (_PAGE_PRESENT|_PAGE_RW);
-}
-
-/*
- * So, when pin_stack_pages() asks us to pin a page, we check if it's already
- * in the page tables, and if not, we call demand_page() with error code 2
- * (meaning "write").
- */
-void pin_page(struct lg_cpu *cpu, unsigned long vaddr)
-{
-	unsigned long iomem;
-
-	if (!page_writable(cpu, vaddr) && !demand_page(cpu, vaddr, 2, &iomem))
-		kill_guest(cpu, "bad stack page %#lx", vaddr);
-}
-/*:*/
-
-#ifdef CONFIG_X86_PAE
-static void release_pmd(pmd_t *spmd)
-{
-	/* If the entry's not present, there's nothing to release. */
-	if (pmd_flags(*spmd) & _PAGE_PRESENT) {
-		unsigned int i;
-		pte_t *ptepage = __va(pmd_pfn(*spmd) << PAGE_SHIFT);
-		/* For each entry in the page, we might need to release it. */
-		for (i = 0; i < PTRS_PER_PTE; i++)
-			release_pte(ptepage[i]);
-		/* Now we can free the page of PTEs */
-		free_page((long)ptepage);
-		/* And zero out the PMD entry so we never release it twice. */
-		set_pmd(spmd, __pmd(0));
-	}
-}
-
-static void release_pgd(pgd_t *spgd)
-{
-	/* If the entry's not present, there's nothing to release. */
-	if (pgd_flags(*spgd) & _PAGE_PRESENT) {
-		unsigned int i;
-		pmd_t *pmdpage = __va(pgd_pfn(*spgd) << PAGE_SHIFT);
-
-		for (i = 0; i < PTRS_PER_PMD; i++)
-			release_pmd(&pmdpage[i]);
-
-		/* Now we can free the page of PMDs */
-		free_page((long)pmdpage);
-		/* And zero out the PGD entry so we never release it twice. */
-		set_pgd(spgd, __pgd(0));
-	}
-}
-
-#else /* !CONFIG_X86_PAE */
-/*H:450
- * If we chase down the release_pgd() code, the non-PAE version looks like
- * this.  The PAE version is almost identical, but instead of calling
- * release_pte it calls release_pmd(), which looks much like this.
- */
-static void release_pgd(pgd_t *spgd)
-{
-	/* If the entry's not present, there's nothing to release. */
-	if (pgd_flags(*spgd) & _PAGE_PRESENT) {
-		unsigned int i;
-		/*
-		 * Converting the pfn to find the actual PTE page is easy: turn
-		 * the page number into a physical address, then convert to a
-		 * virtual address (easy for kernel pages like this one).
-		 */
-		pte_t *ptepage = __va(pgd_pfn(*spgd) << PAGE_SHIFT);
-		/* For each entry in the page, we might need to release it. */
-		for (i = 0; i < PTRS_PER_PTE; i++)
-			release_pte(ptepage[i]);
-		/* Now we can free the page of PTEs */
-		free_page((long)ptepage);
-		/* And zero out the PGD entry so we never release it twice. */
-		*spgd = __pgd(0);
-	}
-}
-#endif
-
-/*H:445
- * We saw flush_user_mappings() twice: once from the flush_user_mappings()
- * hypercall and once in new_pgdir() when we re-used a top-level pgdir page.
- * It simply releases every PTE page from 0 up to the Guest's kernel address.
- */
-static void flush_user_mappings(struct lguest *lg, int idx)
-{
-	unsigned int i;
-	/* Release every pgd entry up to the kernel's address. */
-	for (i = 0; i < pgd_index(lg->kernel_address); i++)
-		release_pgd(lg->pgdirs[idx].pgdir + i);
-}
-
-/*H:440
- * (v) Flushing (throwing away) page tables,
- *
- * The Guest has a hypercall to throw away the page tables: it's used when a
- * large number of mappings have been changed.
- */
-void guest_pagetable_flush_user(struct lg_cpu *cpu)
-{
-	/* Drop the userspace part of the current page table. */
-	flush_user_mappings(cpu->lg, cpu->cpu_pgd);
-}
-/*:*/
-
-/* We walk down the guest page tables to get a guest-physical address */
-bool __guest_pa(struct lg_cpu *cpu, unsigned long vaddr, unsigned long *paddr)
-{
-	pgd_t gpgd;
-	pte_t gpte;
-#ifdef CONFIG_X86_PAE
-	pmd_t gpmd;
-#endif
-
-	/* Still not set up?  Just map 1:1. */
-	if (unlikely(cpu->linear_pages)) {
-		*paddr = vaddr;
-		return true;
-	}
-
-	/* First step: get the top-level Guest page table entry. */
-	gpgd = lgread(cpu, gpgd_addr(cpu, vaddr), pgd_t);
-	/* Toplevel not present?  We can't map it in. */
-	if (!(pgd_flags(gpgd) & _PAGE_PRESENT))
-		goto fail;
-
-#ifdef CONFIG_X86_PAE
-	gpmd = lgread(cpu, gpmd_addr(gpgd, vaddr), pmd_t);
-	if (!(pmd_flags(gpmd) & _PAGE_PRESENT))
-		goto fail;
-	gpte = lgread(cpu, gpte_addr(cpu, gpmd, vaddr), pte_t);
-#else
-	gpte = lgread(cpu, gpte_addr(cpu, gpgd, vaddr), pte_t);
-#endif
-	if (!(pte_flags(gpte) & _PAGE_PRESENT))
-		goto fail;
-
-	*paddr = pte_pfn(gpte) * PAGE_SIZE | (vaddr & ~PAGE_MASK);
-	return true;
-
-fail:
-	*paddr = -1UL;
-	return false;
-}
-
-/*
- * This is the version we normally use: kills the Guest if it uses a
- * bad address
- */
-unsigned long guest_pa(struct lg_cpu *cpu, unsigned long vaddr)
-{
-	unsigned long paddr;
-
-	if (!__guest_pa(cpu, vaddr, &paddr))
-		kill_guest(cpu, "Bad address %#lx", vaddr);
-	return paddr;
-}
-
-/*
- * We keep several page tables.  This is a simple routine to find the page
- * table (if any) corresponding to this top-level address the Guest has given
- * us.
- */
-static unsigned int find_pgdir(struct lguest *lg, unsigned long pgtable)
-{
-	unsigned int i;
-	for (i = 0; i < ARRAY_SIZE(lg->pgdirs); i++)
-		if (lg->pgdirs[i].pgdir && lg->pgdirs[i].gpgdir == pgtable)
-			break;
-	return i;
-}
-
-/*H:435
- * And this is us, creating the new page directory.  If we really do
- * allocate a new one (and so the kernel parts are not there), we set
- * blank_pgdir.
- */
-static unsigned int new_pgdir(struct lg_cpu *cpu,
-			      unsigned long gpgdir,
-			      int *blank_pgdir)
-{
-	unsigned int next;
-
-	/*
-	 * We pick one entry at random to throw out.  Choosing the Least
-	 * Recently Used might be better, but this is easy.
-	 */
-	next = prandom_u32() % ARRAY_SIZE(cpu->lg->pgdirs);
-	/* If it's never been allocated at all before, try now. */
-	if (!cpu->lg->pgdirs[next].pgdir) {
-		cpu->lg->pgdirs[next].pgdir =
-					(pgd_t *)get_zeroed_page(GFP_KERNEL);
-		/* If the allocation fails, just keep using the one we have */
-		if (!cpu->lg->pgdirs[next].pgdir)
-			next = cpu->cpu_pgd;
-		else {
-			/*
-			 * This is a blank page, so there are no kernel
-			 * mappings: caller must map the stack!
-			 */
-			*blank_pgdir = 1;
-		}
-	}
-	/* Record which Guest toplevel this shadows. */
-	cpu->lg->pgdirs[next].gpgdir = gpgdir;
-	/* Release all the non-kernel mappings. */
-	flush_user_mappings(cpu->lg, next);
-
-	/* This hasn't run on any CPU at all. */
-	cpu->lg->pgdirs[next].last_host_cpu = -1;
-
-	return next;
-}
-
-/*H:501
- * We do need the Switcher code mapped at all times, so we allocate that
- * part of the Guest page table here.  We map the Switcher code immediately,
- * but defer mapping of the guest register page and IDT/LDT etc page until
- * just before we run the guest in map_switcher_in_guest().
- *
- * We *could* do this setup in map_switcher_in_guest(), but at that point
- * we've interrupts disabled, and allocating pages like that is fraught: we
- * can't sleep if we need to free up some memory.
- */
-static bool allocate_switcher_mapping(struct lg_cpu *cpu)
-{
-	int i;
-
-	for (i = 0; i < TOTAL_SWITCHER_PAGES; i++) {
-		pte_t *pte = find_spte(cpu, switcher_addr + i * PAGE_SIZE, true,
-				       CHECK_GPGD_MASK, _PAGE_TABLE);
-		if (!pte)
-			return false;
-
-		/*
-		 * Map the switcher page if not already there.  It might
-		 * already be there because we call allocate_switcher_mapping()
-		 * in guest_set_pgd() just in case it did discard our Switcher
-		 * mapping, but it probably didn't.
-		 */
-		if (i == 0 && !(pte_flags(*pte) & _PAGE_PRESENT)) {
-			/* Get a reference to the Switcher page. */
-			get_page(lg_switcher_pages[0]);
-			/* Create a read-only, exectuable, kernel-style PTE */
-			set_pte(pte,
-				mk_pte(lg_switcher_pages[0], PAGE_KERNEL_RX));
-		}
-	}
-	cpu->lg->pgdirs[cpu->cpu_pgd].switcher_mapped = true;
-	return true;
-}
-
-/*H:470
- * Finally, a routine which throws away everything: all PGD entries in all
- * the shadow page tables, including the Guest's kernel mappings.  This is used
- * when we destroy the Guest.
- */
-static void release_all_pagetables(struct lguest *lg)
-{
-	unsigned int i, j;
-
-	/* Every shadow pagetable this Guest has */
-	for (i = 0; i < ARRAY_SIZE(lg->pgdirs); i++) {
-		if (!lg->pgdirs[i].pgdir)
-			continue;
-
-		/* Every PGD entry. */
-		for (j = 0; j < PTRS_PER_PGD; j++)
-			release_pgd(lg->pgdirs[i].pgdir + j);
-		lg->pgdirs[i].switcher_mapped = false;
-		lg->pgdirs[i].last_host_cpu = -1;
-	}
-}
-
-/*
- * We also throw away everything when a Guest tells us it's changed a kernel
- * mapping.  Since kernel mappings are in every page table, it's easiest to
- * throw them all away.  This traps the Guest in amber for a while as
- * everything faults back in, but it's rare.
- */
-void guest_pagetable_clear_all(struct lg_cpu *cpu)
-{
-	release_all_pagetables(cpu->lg);
-	/* We need the Guest kernel stack mapped again. */
-	pin_stack_pages(cpu);
-	/* And we need Switcher allocated. */
-	if (!allocate_switcher_mapping(cpu))
-		kill_guest(cpu, "Cannot populate switcher mapping");
-}
-
-/*H:430
- * (iv) Switching page tables
- *
- * Now we've seen all the page table setting and manipulation, let's see
- * what happens when the Guest changes page tables (ie. changes the top-level
- * pgdir).  This occurs on almost every context switch.
- */
-void guest_new_pagetable(struct lg_cpu *cpu, unsigned long pgtable)
-{
-	int newpgdir, repin = 0;
-
-	/*
-	 * The very first time they call this, we're actually running without
-	 * any page tables; we've been making it up.  Throw them away now.
-	 */
-	if (unlikely(cpu->linear_pages)) {
-		release_all_pagetables(cpu->lg);
-		cpu->linear_pages = false;
-		/* Force allocation of a new pgdir. */
-		newpgdir = ARRAY_SIZE(cpu->lg->pgdirs);
-	} else {
-		/* Look to see if we have this one already. */
-		newpgdir = find_pgdir(cpu->lg, pgtable);
-	}
-
-	/*
-	 * If not, we allocate or mug an existing one: if it's a fresh one,
-	 * repin gets set to 1.
-	 */
-	if (newpgdir == ARRAY_SIZE(cpu->lg->pgdirs))
-		newpgdir = new_pgdir(cpu, pgtable, &repin);
-	/* Change the current pgd index to the new one. */
-	cpu->cpu_pgd = newpgdir;
-	/*
-	 * If it was completely blank, we map in the Guest kernel stack and
-	 * the Switcher.
-	 */
-	if (repin)
-		pin_stack_pages(cpu);
-
-	if (!cpu->lg->pgdirs[cpu->cpu_pgd].switcher_mapped) {
-		if (!allocate_switcher_mapping(cpu))
-			kill_guest(cpu, "Cannot populate switcher mapping");
-	}
-}
-/*:*/
-
-/*M:009
- * Since we throw away all mappings when a kernel mapping changes, our
- * performance sucks for guests using highmem.  In fact, a guest with
- * PAGE_OFFSET 0xc0000000 (the default) and more than about 700MB of RAM is
- * usually slower than a Guest with less memory.
- *
- * This, of course, cannot be fixed.  It would take some kind of... well, I
- * don't know, but the term "puissant code-fu" comes to mind.
-:*/
-
-/*H:420
- * This is the routine which actually sets the page table entry for then
- * "idx"'th shadow page table.
- *
- * Normally, we can just throw out the old entry and replace it with 0: if they
- * use it demand_page() will put the new entry in.  We need to do this anyway:
- * The Guest expects _PAGE_ACCESSED to be set on its PTE the first time a page
- * is read from, and _PAGE_DIRTY when it's written to.
- *
- * But Avi Kivity pointed out that most Operating Systems (Linux included) set
- * these bits on PTEs immediately anyway.  This is done to save the CPU from
- * having to update them, but it helps us the same way: if they set
- * _PAGE_ACCESSED then we can put a read-only PTE entry in immediately, and if
- * they set _PAGE_DIRTY then we can put a writable PTE entry in immediately.
- */
-static void __guest_set_pte(struct lg_cpu *cpu, int idx,
-		       unsigned long vaddr, pte_t gpte)
-{
-	/* Look up the matching shadow page directory entry. */
-	pgd_t *spgd = spgd_addr(cpu, idx, vaddr);
-#ifdef CONFIG_X86_PAE
-	pmd_t *spmd;
-#endif
-
-	/* If the top level isn't present, there's no entry to update. */
-	if (pgd_flags(*spgd) & _PAGE_PRESENT) {
-#ifdef CONFIG_X86_PAE
-		spmd = spmd_addr(cpu, *spgd, vaddr);
-		if (pmd_flags(*spmd) & _PAGE_PRESENT) {
-#endif
-			/* Otherwise, start by releasing the existing entry. */
-			pte_t *spte = spte_addr(cpu, *spgd, vaddr);
-			release_pte(*spte);
-
-			/*
-			 * If they're setting this entry as dirty or accessed,
-			 * we might as well put that entry they've given us in
-			 * now.  This shaves 10% off a copy-on-write
-			 * micro-benchmark.
-			 */
-			if ((pte_flags(gpte) & (_PAGE_DIRTY | _PAGE_ACCESSED))
-			    && !gpte_in_iomem(cpu, gpte)) {
-				if (!check_gpte(cpu, gpte))
-					return;
-				set_pte(spte,
-					gpte_to_spte(cpu, gpte,
-						pte_flags(gpte) & _PAGE_DIRTY));
-			} else {
-				/*
-				 * Otherwise kill it and we can demand_page()
-				 * it in later.
-				 */
-				set_pte(spte, __pte(0));
-			}
-#ifdef CONFIG_X86_PAE
-		}
-#endif
-	}
-}
-
-/*H:410
- * Updating a PTE entry is a little trickier.
- *
- * We keep track of several different page tables (the Guest uses one for each
- * process, so it makes sense to cache at least a few).  Each of these have
- * identical kernel parts: ie. every mapping above PAGE_OFFSET is the same for
- * all processes.  So when the page table above that address changes, we update
- * all the page tables, not just the current one.  This is rare.
- *
- * The benefit is that when we have to track a new page table, we can keep all
- * the kernel mappings.  This speeds up context switch immensely.
- */
-void guest_set_pte(struct lg_cpu *cpu,
-		   unsigned long gpgdir, unsigned long vaddr, pte_t gpte)
-{
-	/* We don't let you remap the Switcher; we need it to get back! */
-	if (vaddr >= switcher_addr) {
-		kill_guest(cpu, "attempt to set pte into Switcher pages");
-		return;
-	}
-
-	/*
-	 * Kernel mappings must be changed on all top levels.  Slow, but doesn't
-	 * happen often.
-	 */
-	if (vaddr >= cpu->lg->kernel_address) {
-		unsigned int i;
-		for (i = 0; i < ARRAY_SIZE(cpu->lg->pgdirs); i++)
-			if (cpu->lg->pgdirs[i].pgdir)
-				__guest_set_pte(cpu, i, vaddr, gpte);
-	} else {
-		/* Is this page table one we have a shadow for? */
-		int pgdir = find_pgdir(cpu->lg, gpgdir);
-		if (pgdir != ARRAY_SIZE(cpu->lg->pgdirs))
-			/* If so, do the update. */
-			__guest_set_pte(cpu, pgdir, vaddr, gpte);
-	}
-}
-
-/*H:400
- * (iii) Setting up a page table entry when the Guest tells us one has changed.
- *
- * Just like we did in interrupts_and_traps.c, it makes sense for us to deal
- * with the other side of page tables while we're here: what happens when the
- * Guest asks for a page table to be updated?
- *
- * We already saw that demand_page() will fill in the shadow page tables when
- * needed, so we can simply remove shadow page table entries whenever the Guest
- * tells us they've changed.  When the Guest tries to use the new entry it will
- * fault and demand_page() will fix it up.
- *
- * So with that in mind here's our code to update a (top-level) PGD entry:
- */
-void guest_set_pgd(struct lguest *lg, unsigned long gpgdir, u32 idx)
-{
-	int pgdir;
-
-	if (idx > PTRS_PER_PGD) {
-		kill_guest(&lg->cpus[0], "Attempt to set pgd %u/%u",
-			   idx, PTRS_PER_PGD);
-		return;
-	}
-
-	/* If they're talking about a page table we have a shadow for... */
-	pgdir = find_pgdir(lg, gpgdir);
-	if (pgdir < ARRAY_SIZE(lg->pgdirs)) {
-		/* ... throw it away. */
-		release_pgd(lg->pgdirs[pgdir].pgdir + idx);
-		/* That might have been the Switcher mapping, remap it. */
-		if (!allocate_switcher_mapping(&lg->cpus[0])) {
-			kill_guest(&lg->cpus[0],
-				   "Cannot populate switcher mapping");
-		}
-		lg->pgdirs[pgdir].last_host_cpu = -1;
-	}
-}
-
-#ifdef CONFIG_X86_PAE
-/* For setting a mid-level, we just throw everything away.  It's easy. */
-void guest_set_pmd(struct lguest *lg, unsigned long pmdp, u32 idx)
-{
-	guest_pagetable_clear_all(&lg->cpus[0]);
-}
-#endif
-
-/*H:500
- * (vii) Setting up the page tables initially.
- *
- * When a Guest is first created, set initialize a shadow page table which
- * we will populate on future faults.  The Guest doesn't have any actual
- * pagetables yet, so we set linear_pages to tell demand_page() to fake it
- * for the moment.
- *
- * We do need the Switcher to be mapped at all times, so we allocate that
- * part of the Guest page table here.
- */
-int init_guest_pagetable(struct lguest *lg)
-{
-	struct lg_cpu *cpu = &lg->cpus[0];
-	int allocated = 0;
-
-	/* lg (and lg->cpus[]) starts zeroed: this allocates a new pgdir */
-	cpu->cpu_pgd = new_pgdir(cpu, 0, &allocated);
-	if (!allocated)
-		return -ENOMEM;
-
-	/* We start with a linear mapping until the initialize. */
-	cpu->linear_pages = true;
-
-	/* Allocate the page tables for the Switcher. */
-	if (!allocate_switcher_mapping(cpu)) {
-		release_all_pagetables(lg);
-		return -ENOMEM;
-	}
-
-	return 0;
-}
-
-/*H:508 When the Guest calls LHCALL_LGUEST_INIT we do more setup. */
-void page_table_guest_data_init(struct lg_cpu *cpu)
-{
-	/*
-	 * We tell the Guest that it can't use the virtual addresses
-	 * used by the Switcher.  This trick is equivalent to 4GB -
-	 * switcher_addr.
-	 */
-	u32 top = ~switcher_addr + 1;
-
-	/* We get the kernel address: above this is all kernel memory. */
-	if (get_user(cpu->lg->kernel_address,
-		     &cpu->lg->lguest_data->kernel_address)
-		/*
-		 * We tell the Guest that it can't use the top virtual
-		 * addresses (used by the Switcher).
-		 */
-	    || put_user(top, &cpu->lg->lguest_data->reserve_mem)) {
-		kill_guest(cpu, "bad guest page %p", cpu->lg->lguest_data);
-		return;
-	}
-
-	/*
-	 * In flush_user_mappings() we loop from 0 to
-	 * "pgd_index(lg->kernel_address)".  This assumes it won't hit the
-	 * Switcher mappings, so check that now.
-	 */
-	if (cpu->lg->kernel_address >= switcher_addr)
-		kill_guest(cpu, "bad kernel address %#lx",
-				 cpu->lg->kernel_address);
-}
-
-/* When a Guest dies, our cleanup is fairly simple. */
-void free_guest_pagetable(struct lguest *lg)
-{
-	unsigned int i;
-
-	/* Throw away all page table pages. */
-	release_all_pagetables(lg);
-	/* Now free the top levels: free_page() can handle 0 just fine. */
-	for (i = 0; i < ARRAY_SIZE(lg->pgdirs); i++)
-		free_page((long)lg->pgdirs[i].pgdir);
-}
-
-/*H:481
- * This clears the Switcher mappings for cpu #i.
- */
-static void remove_switcher_percpu_map(struct lg_cpu *cpu, unsigned int i)
-{
-	unsigned long base = switcher_addr + PAGE_SIZE + i * PAGE_SIZE*2;
-	pte_t *pte;
-
-	/* Clear the mappings for both pages. */
-	pte = find_spte(cpu, base, false, 0, 0);
-	release_pte(*pte);
-	set_pte(pte, __pte(0));
-
-	pte = find_spte(cpu, base + PAGE_SIZE, false, 0, 0);
-	release_pte(*pte);
-	set_pte(pte, __pte(0));
-}
-
-/*H:480
- * (vi) Mapping the Switcher when the Guest is about to run.
- *
- * The Switcher and the two pages for this CPU need to be visible in the Guest
- * (and not the pages for other CPUs).
- *
- * The pages for the pagetables have all been allocated before: we just need
- * to make sure the actual PTEs are up-to-date for the CPU we're about to run
- * on.
- */
-void map_switcher_in_guest(struct lg_cpu *cpu, struct lguest_pages *pages)
-{
-	unsigned long base;
-	struct page *percpu_switcher_page, *regs_page;
-	pte_t *pte;
-	struct pgdir *pgdir = &cpu->lg->pgdirs[cpu->cpu_pgd];
-
-	/* Switcher page should always be mapped by now! */
-	BUG_ON(!pgdir->switcher_mapped);
-
-	/* 
-	 * Remember that we have two pages for each Host CPU, so we can run a
-	 * Guest on each CPU without them interfering.  We need to make sure
-	 * those pages are mapped correctly in the Guest, but since we usually
-	 * run on the same CPU, we cache that, and only update the mappings
-	 * when we move.
-	 */
-	if (pgdir->last_host_cpu == raw_smp_processor_id())
-		return;
-
-	/* -1 means unknown so we remove everything. */
-	if (pgdir->last_host_cpu == -1) {
-		unsigned int i;
-		for_each_possible_cpu(i)
-			remove_switcher_percpu_map(cpu, i);
-	} else {
-		/* We know exactly what CPU mapping to remove. */
-		remove_switcher_percpu_map(cpu, pgdir->last_host_cpu);
-	}
-
-	/*
-	 * When we're running the Guest, we want the Guest's "regs" page to
-	 * appear where the first Switcher page for this CPU is.  This is an
-	 * optimization: when the Switcher saves the Guest registers, it saves
-	 * them into the first page of this CPU's "struct lguest_pages": if we
-	 * make sure the Guest's register page is already mapped there, we
-	 * don't have to copy them out again.
-	 */
-	/* Find the shadow PTE for this regs page. */
-	base = switcher_addr + PAGE_SIZE
-		+ raw_smp_processor_id() * sizeof(struct lguest_pages);
-	pte = find_spte(cpu, base, false, 0, 0);
-	regs_page = pfn_to_page(__pa(cpu->regs_page) >> PAGE_SHIFT);
-	get_page(regs_page);
-	set_pte(pte, mk_pte(regs_page, __pgprot(__PAGE_KERNEL & ~_PAGE_GLOBAL)));
-
-	/*
-	 * We map the second page of the struct lguest_pages read-only in
-	 * the Guest: the IDT, GDT and other things it's not supposed to
-	 * change.
-	 */
-	pte = find_spte(cpu, base + PAGE_SIZE, false, 0, 0);
-	percpu_switcher_page
-		= lg_switcher_pages[1 + raw_smp_processor_id()*2 + 1];
-	get_page(percpu_switcher_page);
-	set_pte(pte, mk_pte(percpu_switcher_page,
-			    __pgprot(__PAGE_KERNEL_RO & ~_PAGE_GLOBAL)));
-
-	pgdir->last_host_cpu = raw_smp_processor_id();
-}
-
-/*H:490
- * We've made it through the page table code.  Perhaps our tired brains are
- * still processing the details, or perhaps we're simply glad it's over.
- *
- * If nothing else, note that all this complexity in juggling shadow page tables
- * in sync with the Guest's page tables is for one reason: for most Guests this
- * page table dance determines how bad performance will be.  This is why Xen
- * uses exotic direct Guest pagetable manipulation, and why both Intel and AMD
- * have implemented shadow page table support directly into hardware.
- *
- * There is just one file remaining in the Host.
- */
diff --git a/drivers/lguest/segments.c b/drivers/lguest/segments.c
deleted file mode 100644
index c4fb424dfddb..000000000000
--- a/drivers/lguest/segments.c
+++ /dev/null
@@ -1,228 +0,0 @@
-/*P:600
- * The x86 architecture has segments, which involve a table of descriptors
- * which can be used to do funky things with virtual address interpretation.
- * We originally used to use segments so the Guest couldn't alter the
- * Guest<->Host Switcher, and then we had to trim Guest segments, and restore
- * for userspace per-thread segments, but trim again for on userspace->kernel
- * transitions...  This nightmarish creation was contained within this file,
- * where we knew not to tread without heavy armament and a change of underwear.
- *
- * In these modern times, the segment handling code consists of simple sanity
- * checks, and the worst you'll experience reading this code is butterfly-rash
- * from frolicking through its parklike serenity.
-:*/
-#include "lg.h"
-
-/*H:600
- * Segments & The Global Descriptor Table
- *
- * (That title sounds like a bad Nerdcore group.  Not to suggest that there are
- * any good Nerdcore groups, but in high school a friend of mine had a band
- * called Joe Fish and the Chips, so there are definitely worse band names).
- *
- * To refresh: the GDT is a table of 8-byte values describing segments.  Once
- * set up, these segments can be loaded into one of the 6 "segment registers".
- *
- * GDT entries are passed around as "struct desc_struct"s, which like IDT
- * entries are split into two 32-bit members, "a" and "b".  One day, someone
- * will clean that up, and be declared a Hero.  (No pressure, I'm just saying).
- *
- * Anyway, the GDT entry contains a base (the start address of the segment), a
- * limit (the size of the segment - 1), and some flags.  Sounds simple, and it
- * would be, except those zany Intel engineers decided that it was too boring
- * to put the base at one end, the limit at the other, and the flags in
- * between.  They decided to shotgun the bits at random throughout the 8 bytes,
- * like so:
- *
- * 0               16                     40       48  52  56     63
- * [ limit part 1 ][     base part 1     ][ flags ][li][fl][base ]
- *                                                  mit ags part 2
- *                                                part 2
- *
- * As a result, this file contains a certain amount of magic numeracy.  Let's
- * begin.
- */
-
-/*
- * There are several entries we don't let the Guest set.  The TSS entry is the
- * "Task State Segment" which controls all kinds of delicate things.  The
- * LGUEST_CS and LGUEST_DS entries are reserved for the Switcher, and the
- * the Guest can't be trusted to deal with double faults.
- */
-static bool ignored_gdt(unsigned int num)
-{
-	return (num == GDT_ENTRY_TSS
-		|| num == GDT_ENTRY_LGUEST_CS
-		|| num == GDT_ENTRY_LGUEST_DS
-		|| num == GDT_ENTRY_DOUBLEFAULT_TSS);
-}
-
-/*H:630
- * Once the Guest gave us new GDT entries, we fix them up a little.  We
- * don't care if they're invalid: the worst that can happen is a General
- * Protection Fault in the Switcher when it restores a Guest segment register
- * which tries to use that entry.  Then we kill the Guest for causing such a
- * mess: the message will be "unhandled trap 256".
- */
-static void fixup_gdt_table(struct lg_cpu *cpu, unsigned start, unsigned end)
-{
-	unsigned int i;
-
-	for (i = start; i < end; i++) {
-		/*
-		 * We never copy these ones to real GDT, so we don't care what
-		 * they say
-		 */
-		if (ignored_gdt(i))
-			continue;
-
-		/*
-		 * Segment descriptors contain a privilege level: the Guest is
-		 * sometimes careless and leaves this as 0, even though it's
-		 * running at privilege level 1.  If so, we fix it here.
-		 */
-		if (cpu->arch.gdt[i].dpl == 0)
-			cpu->arch.gdt[i].dpl |= GUEST_PL;
-
-		/*
-		 * Each descriptor has an "accessed" bit.  If we don't set it
-		 * now, the CPU will try to set it when the Guest first loads
-		 * that entry into a segment register.  But the GDT isn't
-		 * writable by the Guest, so bad things can happen.
-		 */
-		cpu->arch.gdt[i].type |= 0x1;
-	}
-}
-
-/*H:610
- * Like the IDT, we never simply use the GDT the Guest gives us.  We keep
- * a GDT for each CPU, and copy across the Guest's entries each time we want to
- * run the Guest on that CPU.
- *
- * This routine is called at boot or modprobe time for each CPU to set up the
- * constant GDT entries: the ones which are the same no matter what Guest we're
- * running.
- */
-void setup_default_gdt_entries(struct lguest_ro_state *state)
-{
-	struct desc_struct *gdt = state->guest_gdt;
-	unsigned long tss = (unsigned long)&state->guest_tss;
-
-	/* The Switcher segments are full 0-4G segments, privilege level 0 */
-	gdt[GDT_ENTRY_LGUEST_CS] = FULL_EXEC_SEGMENT;
-	gdt[GDT_ENTRY_LGUEST_DS] = FULL_SEGMENT;
-
-	/*
-	 * The TSS segment refers to the TSS entry for this particular CPU.
-	 */
-	gdt[GDT_ENTRY_TSS].a = 0;
-	gdt[GDT_ENTRY_TSS].b = 0;
-
-	gdt[GDT_ENTRY_TSS].limit0 = 0x67;
-	gdt[GDT_ENTRY_TSS].base0  = tss & 0xFFFF;
-	gdt[GDT_ENTRY_TSS].base1  = (tss >> 16) & 0xFF;
-	gdt[GDT_ENTRY_TSS].base2  = tss >> 24;
-	gdt[GDT_ENTRY_TSS].type   = 0x9; /* 32-bit TSS (available) */
-	gdt[GDT_ENTRY_TSS].p      = 0x1; /* Entry is present */
-	gdt[GDT_ENTRY_TSS].dpl    = 0x0; /* Privilege level 0 */
-	gdt[GDT_ENTRY_TSS].s      = 0x0; /* system segment */
-
-}
-
-/*
- * This routine sets up the initial Guest GDT for booting.  All entries start
- * as 0 (unusable).
- */
-void setup_guest_gdt(struct lg_cpu *cpu)
-{
-	/*
-	 * Start with full 0-4G segments...except the Guest is allowed to use
-	 * them, so set the privilege level appropriately in the flags.
-	 */
-	cpu->arch.gdt[GDT_ENTRY_KERNEL_CS] = FULL_EXEC_SEGMENT;
-	cpu->arch.gdt[GDT_ENTRY_KERNEL_DS] = FULL_SEGMENT;
-	cpu->arch.gdt[GDT_ENTRY_KERNEL_CS].dpl |= GUEST_PL;
-	cpu->arch.gdt[GDT_ENTRY_KERNEL_DS].dpl |= GUEST_PL;
-}
-
-/*H:650
- * An optimization of copy_gdt(), for just the three "thead-local storage"
- * entries.
- */
-void copy_gdt_tls(const struct lg_cpu *cpu, struct desc_struct *gdt)
-{
-	unsigned int i;
-
-	for (i = GDT_ENTRY_TLS_MIN; i <= GDT_ENTRY_TLS_MAX; i++)
-		gdt[i] = cpu->arch.gdt[i];
-}
-
-/*H:640
- * When the Guest is run on a different CPU, or the GDT entries have changed,
- * copy_gdt() is called to copy the Guest's GDT entries across to this CPU's
- * GDT.
- */
-void copy_gdt(const struct lg_cpu *cpu, struct desc_struct *gdt)
-{
-	unsigned int i;
-
-	/*
-	 * The default entries from setup_default_gdt_entries() are not
-	 * replaced.  See ignored_gdt() above.
-	 */
-	for (i = 0; i < GDT_ENTRIES; i++)
-		if (!ignored_gdt(i))
-			gdt[i] = cpu->arch.gdt[i];
-}
-
-/*H:620
- * This is where the Guest asks us to load a new GDT entry
- * (LHCALL_LOAD_GDT_ENTRY).  We tweak the entry and copy it in.
- */
-void load_guest_gdt_entry(struct lg_cpu *cpu, u32 num, u32 lo, u32 hi)
-{
-	/*
-	 * We assume the Guest has the same number of GDT entries as the
-	 * Host, otherwise we'd have to dynamically allocate the Guest GDT.
-	 */
-	if (num >= ARRAY_SIZE(cpu->arch.gdt)) {
-		kill_guest(cpu, "too many gdt entries %i", num);
-		return;
-	}
-
-	/* Set it up, then fix it. */
-	cpu->arch.gdt[num].a = lo;
-	cpu->arch.gdt[num].b = hi;
-	fixup_gdt_table(cpu, num, num+1);
-	/*
-	 * Mark that the GDT changed so the core knows it has to copy it again,
-	 * even if the Guest is run on the same CPU.
-	 */
-	cpu->changed |= CHANGED_GDT;
-}
-
-/*
- * This is the fast-track version for just changing the three TLS entries.
- * Remember that this happens on every context switch, so it's worth
- * optimizing.  But wouldn't it be neater to have a single hypercall to cover
- * both cases?
- */
-void guest_load_tls(struct lg_cpu *cpu, unsigned long gtls)
-{
-	struct desc_struct *tls = &cpu->arch.gdt[GDT_ENTRY_TLS_MIN];
-
-	__lgread(cpu, tls, gtls, sizeof(*tls)*GDT_ENTRY_TLS_ENTRIES);
-	fixup_gdt_table(cpu, GDT_ENTRY_TLS_MIN, GDT_ENTRY_TLS_MAX+1);
-	/* Note that just the TLS entries have changed. */
-	cpu->changed |= CHANGED_GDT_TLS;
-}
-
-/*H:660
- * With this, we have finished the Host.
- *
- * Five of the seven parts of our task are complete.  You have made it through
- * the Bit of Despair (I think that's somewhere in the page table code,
- * myself).
- *
- * Next, we examine "make Switcher".  It's short, but intense.
- */
diff --git a/drivers/lguest/x86/core.c b/drivers/lguest/x86/core.c
deleted file mode 100644
index b4f79b923aea..000000000000
--- a/drivers/lguest/x86/core.c
+++ /dev/null
@@ -1,724 +0,0 @@
-/*
- * Copyright (C) 2006, Rusty Russell <rusty@rustcorp.com.au> IBM Corporation.
- * Copyright (C) 2007, Jes Sorensen <jes@sgi.com> SGI.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- * NON INFRINGEMENT.  See the GNU General Public License for more
- * details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- */
-/*P:450
- * This file contains the x86-specific lguest code.  It used to be all
- * mixed in with drivers/lguest/core.c but several foolhardy code slashers
- * wrestled most of the dependencies out to here in preparation for porting
- * lguest to other architectures (see what I mean by foolhardy?).
- *
- * This also contains a couple of non-obvious setup and teardown pieces which
- * were implemented after days of debugging pain.
-:*/
-#include <linux/kernel.h>
-#include <linux/start_kernel.h>
-#include <linux/string.h>
-#include <linux/console.h>
-#include <linux/screen_info.h>
-#include <linux/irq.h>
-#include <linux/interrupt.h>
-#include <linux/clocksource.h>
-#include <linux/clockchips.h>
-#include <linux/cpu.h>
-#include <linux/lguest.h>
-#include <linux/lguest_launcher.h>
-#include <asm/paravirt.h>
-#include <asm/param.h>
-#include <asm/page.h>
-#include <asm/pgtable.h>
-#include <asm/desc.h>
-#include <asm/setup.h>
-#include <asm/lguest.h>
-#include <linux/uaccess.h>
-#include <asm/fpu/internal.h>
-#include <asm/tlbflush.h>
-#include "../lg.h"
-
-static int cpu_had_pge;
-
-static struct {
-	unsigned long offset;
-	unsigned short segment;
-} lguest_entry;
-
-/* Offset from where switcher.S was compiled to where we've copied it */
-static unsigned long switcher_offset(void)
-{
-	return switcher_addr - (unsigned long)start_switcher_text;
-}
-
-/* This cpu's struct lguest_pages (after the Switcher text page) */
-static struct lguest_pages *lguest_pages(unsigned int cpu)
-{
-	return &(((struct lguest_pages *)(switcher_addr + PAGE_SIZE))[cpu]);
-}
-
-static DEFINE_PER_CPU(struct lg_cpu *, lg_last_cpu);
-
-/*S:010
- * We approach the Switcher.
- *
- * Remember that each CPU has two pages which are visible to the Guest when it
- * runs on that CPU.  This has to contain the state for that Guest: we copy the
- * state in just before we run the Guest.
- *
- * Each Guest has "changed" flags which indicate what has changed in the Guest
- * since it last ran.  We saw this set in interrupts_and_traps.c and
- * segments.c.
- */
-static void copy_in_guest_info(struct lg_cpu *cpu, struct lguest_pages *pages)
-{
-	/*
-	 * Copying all this data can be quite expensive.  We usually run the
-	 * same Guest we ran last time (and that Guest hasn't run anywhere else
-	 * meanwhile).  If that's not the case, we pretend everything in the
-	 * Guest has changed.
-	 */
-	if (__this_cpu_read(lg_last_cpu) != cpu || cpu->last_pages != pages) {
-		__this_cpu_write(lg_last_cpu, cpu);
-		cpu->last_pages = pages;
-		cpu->changed = CHANGED_ALL;
-	}
-
-	/*
-	 * These copies are pretty cheap, so we do them unconditionally: */
-	/* Save the current Host top-level page directory.
-	 */
-	pages->state.host_cr3 = __pa(current->mm->pgd);
-	/*
-	 * Set up the Guest's page tables to see this CPU's pages (and no
-	 * other CPU's pages).
-	 */
-	map_switcher_in_guest(cpu, pages);
-	/*
-	 * Set up the two "TSS" members which tell the CPU what stack to use
-	 * for traps which do directly into the Guest (ie. traps at privilege
-	 * level 1).
-	 */
-	pages->state.guest_tss.sp1 = cpu->esp1;
-	pages->state.guest_tss.ss1 = cpu->ss1;
-
-	/* Copy direct-to-Guest trap entries. */
-	if (cpu->changed & CHANGED_IDT)
-		copy_traps(cpu, pages->state.guest_idt, default_idt_entries);
-
-	/* Copy all GDT entries which the Guest can change. */
-	if (cpu->changed & CHANGED_GDT)
-		copy_gdt(cpu, pages->state.guest_gdt);
-	/* If only the TLS entries have changed, copy them. */
-	else if (cpu->changed & CHANGED_GDT_TLS)
-		copy_gdt_tls(cpu, pages->state.guest_gdt);
-
-	/* Mark the Guest as unchanged for next time. */
-	cpu->changed = 0;
-}
-
-/* Finally: the code to actually call into the Switcher to run the Guest. */
-static void run_guest_once(struct lg_cpu *cpu, struct lguest_pages *pages)
-{
-	/* This is a dummy value we need for GCC's sake. */
-	unsigned int clobber;
-
-	/*
-	 * Copy the guest-specific information into this CPU's "struct
-	 * lguest_pages".
-	 */
-	copy_in_guest_info(cpu, pages);
-
-	/*
-	 * Set the trap number to 256 (impossible value).  If we fault while
-	 * switching to the Guest (bad segment registers or bug), this will
-	 * cause us to abort the Guest.
-	 */
-	cpu->regs->trapnum = 256;
-
-	/*
-	 * Now: we push the "eflags" register on the stack, then do an "lcall".
-	 * This is how we change from using the kernel code segment to using
-	 * the dedicated lguest code segment, as well as jumping into the
-	 * Switcher.
-	 *
-	 * The lcall also pushes the old code segment (KERNEL_CS) onto the
-	 * stack, then the address of this call.  This stack layout happens to
-	 * exactly match the stack layout created by an interrupt...
-	 */
-	asm volatile("pushf; lcall *%4"
-		     /*
-		      * This is how we tell GCC that %eax ("a") and %ebx ("b")
-		      * are changed by this routine.  The "=" means output.
-		      */
-		     : "=a"(clobber), "=b"(clobber)
-		     /*
-		      * %eax contains the pages pointer.  ("0" refers to the
-		      * 0-th argument above, ie "a").  %ebx contains the
-		      * physical address of the Guest's top-level page
-		      * directory.
-		      */
-		     : "0"(pages), 
-		       "1"(__pa(cpu->lg->pgdirs[cpu->cpu_pgd].pgdir)),
-		       "m"(lguest_entry)
-		     /*
-		      * We tell gcc that all these registers could change,
-		      * which means we don't have to save and restore them in
-		      * the Switcher.
-		      */
-		     : "memory", "%edx", "%ecx", "%edi", "%esi");
-}
-/*:*/
-
-unsigned long *lguest_arch_regptr(struct lg_cpu *cpu, size_t reg_off, bool any)
-{
-	switch (reg_off) {
-	case offsetof(struct pt_regs, bx):
-		return &cpu->regs->ebx;
-	case offsetof(struct pt_regs, cx):
-		return &cpu->regs->ecx;
-	case offsetof(struct pt_regs, dx):
-		return &cpu->regs->edx;
-	case offsetof(struct pt_regs, si):
-		return &cpu->regs->esi;
-	case offsetof(struct pt_regs, di):
-		return &cpu->regs->edi;
-	case offsetof(struct pt_regs, bp):
-		return &cpu->regs->ebp;
-	case offsetof(struct pt_regs, ax):
-		return &cpu->regs->eax;
-	case offsetof(struct pt_regs, ip):
-		return &cpu->regs->eip;
-	case offsetof(struct pt_regs, sp):
-		return &cpu->regs->esp;
-	}
-
-	/* Launcher can read these, but we don't allow any setting. */
-	if (any) {
-		switch (reg_off) {
-		case offsetof(struct pt_regs, ds):
-			return &cpu->regs->ds;
-		case offsetof(struct pt_regs, es):
-			return &cpu->regs->es;
-		case offsetof(struct pt_regs, fs):
-			return &cpu->regs->fs;
-		case offsetof(struct pt_regs, gs):
-			return &cpu->regs->gs;
-		case offsetof(struct pt_regs, cs):
-			return &cpu->regs->cs;
-		case offsetof(struct pt_regs, flags):
-			return &cpu->regs->eflags;
-		case offsetof(struct pt_regs, ss):
-			return &cpu->regs->ss;
-		}
-	}
-
-	return NULL;
-}
-
-/*M:002
- * There are hooks in the scheduler which we can register to tell when we
- * get kicked off the CPU (preempt_notifier_register()).  This would allow us
- * to lazily disable SYSENTER which would regain some performance, and should
- * also simplify copy_in_guest_info().  Note that we'd still need to restore
- * things when we exit to Launcher userspace, but that's fairly easy.
- *
- * We could also try using these hooks for PGE, but that might be too expensive.
- *
- * The hooks were designed for KVM, but we can also put them to good use.
-:*/
-
-/*H:040
- * This is the i386-specific code to setup and run the Guest.  Interrupts
- * are disabled: we own the CPU.
- */
-void lguest_arch_run_guest(struct lg_cpu *cpu)
-{
-	/*
-	 * SYSENTER is an optimized way of doing system calls.  We can't allow
-	 * it because it always jumps to privilege level 0.  A normal Guest
-	 * won't try it because we don't advertise it in CPUID, but a malicious
-	 * Guest (or malicious Guest userspace program) could, so we tell the
-	 * CPU to disable it before running the Guest.
-	 */
-	if (boot_cpu_has(X86_FEATURE_SEP))
-		wrmsr(MSR_IA32_SYSENTER_CS, 0, 0);
-
-	/*
-	 * Now we actually run the Guest.  It will return when something
-	 * interesting happens, and we can examine its registers to see what it
-	 * was doing.
-	 */
-	run_guest_once(cpu, lguest_pages(raw_smp_processor_id()));
-
-	/*
-	 * Note that the "regs" structure contains two extra entries which are
-	 * not really registers: a trap number which says what interrupt or
-	 * trap made the switcher code come back, and an error code which some
-	 * traps set.
-	 */
-
-	 /* Restore SYSENTER if it's supposed to be on. */
-	 if (boot_cpu_has(X86_FEATURE_SEP))
-		wrmsr(MSR_IA32_SYSENTER_CS, __KERNEL_CS, 0);
-
-	/*
-	 * If the Guest page faulted, then the cr2 register will tell us the
-	 * bad virtual address.  We have to grab this now, because once we
-	 * re-enable interrupts an interrupt could fault and thus overwrite
-	 * cr2, or we could even move off to a different CPU.
-	 */
-	if (cpu->regs->trapnum == 14)
-		cpu->arch.last_pagefault = read_cr2();
-	/*
-	 * Similarly, if we took a trap because the Guest used the FPU,
-	 * we have to restore the FPU it expects to see.
-	 * fpu__restore() may sleep and we may even move off to
-	 * a different CPU. So all the critical stuff should be done
-	 * before this.
-	 */
-	else if (cpu->regs->trapnum == 7 && !fpregs_active())
-		fpu__restore(&current->thread.fpu);
-}
-
-/*H:130
- * Now we've examined the hypercall code; our Guest can make requests.
- * Our Guest is usually so well behaved; it never tries to do things it isn't
- * allowed to, and uses hypercalls instead.  Unfortunately, Linux's paravirtual
- * infrastructure isn't quite complete, because it doesn't contain replacements
- * for the Intel I/O instructions.  As a result, the Guest sometimes fumbles
- * across one during the boot process as it probes for various things which are
- * usually attached to a PC.
- *
- * When the Guest uses one of these instructions, we get a trap (General
- * Protection Fault) and come here.  We queue this to be sent out to the
- * Launcher to handle.
- */
-
-/*
- * The eip contains the *virtual* address of the Guest's instruction:
- * we copy the instruction here so the Launcher doesn't have to walk
- * the page tables to decode it.  We handle the case (eg. in a kernel
- * module) where the instruction is over two pages, and the pages are
- * virtually but not physically contiguous.
- *
- * The longest possible x86 instruction is 15 bytes, but we don't handle
- * anything that strange.
- */
-static void copy_from_guest(struct lg_cpu *cpu,
-			    void *dst, unsigned long vaddr, size_t len)
-{
-	size_t to_page_end = PAGE_SIZE - (vaddr % PAGE_SIZE);
-	unsigned long paddr;
-
-	BUG_ON(len > PAGE_SIZE);
-
-	/* If it goes over a page, copy in two parts. */
-	if (len > to_page_end) {
-		/* But make sure the next page is mapped! */
-		if (__guest_pa(cpu, vaddr + to_page_end, &paddr))
-			copy_from_guest(cpu, dst + to_page_end,
-					vaddr + to_page_end,
-					len - to_page_end);
-		else
-			/* Otherwise fill with zeroes. */
-			memset(dst + to_page_end, 0, len - to_page_end);
-		len = to_page_end;
-	}
-
-	/* This will kill the guest if it isn't mapped, but that
-	 * shouldn't happen. */
-	__lgread(cpu, dst, guest_pa(cpu, vaddr), len);
-}
-
-
-static void setup_emulate_insn(struct lg_cpu *cpu)
-{
-	cpu->pending.trap = 13;
-	copy_from_guest(cpu, cpu->pending.insn, cpu->regs->eip,
-			sizeof(cpu->pending.insn));
-}
-
-static void setup_iomem_insn(struct lg_cpu *cpu, unsigned long iomem_addr)
-{
-	cpu->pending.trap = 14;
-	cpu->pending.addr = iomem_addr;
-	copy_from_guest(cpu, cpu->pending.insn, cpu->regs->eip,
-			sizeof(cpu->pending.insn));
-}
-
-/*H:050 Once we've re-enabled interrupts, we look at why the Guest exited. */
-void lguest_arch_handle_trap(struct lg_cpu *cpu)
-{
-	unsigned long iomem_addr;
-
-	switch (cpu->regs->trapnum) {
-	case 13: /* We've intercepted a General Protection Fault. */
-		/* Hand to Launcher to emulate those pesky IN and OUT insns */
-		if (cpu->regs->errcode == 0) {
-			setup_emulate_insn(cpu);
-			return;
-		}
-		break;
-	case 14: /* We've intercepted a Page Fault. */
-		/*
-		 * The Guest accessed a virtual address that wasn't mapped.
-		 * This happens a lot: we don't actually set up most of the page
-		 * tables for the Guest at all when we start: as it runs it asks
-		 * for more and more, and we set them up as required. In this
-		 * case, we don't even tell the Guest that the fault happened.
-		 *
-		 * The errcode tells whether this was a read or a write, and
-		 * whether kernel or userspace code.
-		 */
-		if (demand_page(cpu, cpu->arch.last_pagefault,
-				cpu->regs->errcode, &iomem_addr))
-			return;
-
-		/* Was this an access to memory mapped IO? */
-		if (iomem_addr) {
-			/* Tell Launcher, let it handle it. */
-			setup_iomem_insn(cpu, iomem_addr);
-			return;
-		}
-
-		/*
-		 * OK, it's really not there (or not OK): the Guest needs to
-		 * know.  We write out the cr2 value so it knows where the
-		 * fault occurred.
-		 *
-		 * Note that if the Guest were really messed up, this could
-		 * happen before it's done the LHCALL_LGUEST_INIT hypercall, so
-		 * lg->lguest_data could be NULL
-		 */
-		if (cpu->lg->lguest_data &&
-		    put_user(cpu->arch.last_pagefault,
-			     &cpu->lg->lguest_data->cr2))
-			kill_guest(cpu, "Writing cr2");
-		break;
-	case 7: /* We've intercepted a Device Not Available fault. */
-		/* No special handling is needed here. */
-		break;
-	case 32 ... 255:
-		/* This might be a syscall. */
-		if (could_be_syscall(cpu->regs->trapnum))
-			break;
-
-		/*
-		 * Other values mean a real interrupt occurred, in which case
-		 * the Host handler has already been run. We just do a
-		 * friendly check if another process should now be run, then
-		 * return to run the Guest again.
-		 */
-		cond_resched();
-		return;
-	case LGUEST_TRAP_ENTRY:
-		/*
-		 * Our 'struct hcall_args' maps directly over our regs: we set
-		 * up the pointer now to indicate a hypercall is pending.
-		 */
-		cpu->hcall = (struct hcall_args *)cpu->regs;
-		return;
-	}
-
-	/* We didn't handle the trap, so it needs to go to the Guest. */
-	if (!deliver_trap(cpu, cpu->regs->trapnum))
-		/*
-		 * If the Guest doesn't have a handler (either it hasn't
-		 * registered any yet, or it's one of the faults we don't let
-		 * it handle), it dies with this cryptic error message.
-		 */
-		kill_guest(cpu, "unhandled trap %li at %#lx (%#lx)",
-			   cpu->regs->trapnum, cpu->regs->eip,
-			   cpu->regs->trapnum == 14 ? cpu->arch.last_pagefault
-			   : cpu->regs->errcode);
-}
-
-/*
- * Now we can look at each of the routines this calls, in increasing order of
- * complexity: do_hypercalls(), emulate_insn(), maybe_do_interrupt(),
- * deliver_trap() and demand_page().  After all those, we'll be ready to
- * examine the Switcher, and our philosophical understanding of the Host/Guest
- * duality will be complete.
-:*/
-static void adjust_pge(void *on)
-{
-	if (on)
-		cr4_set_bits(X86_CR4_PGE);
-	else
-		cr4_clear_bits(X86_CR4_PGE);
-}
-
-/*H:020
- * Now the Switcher is mapped and every thing else is ready, we need to do
- * some more i386-specific initialization.
- */
-void __init lguest_arch_host_init(void)
-{
-	int i;
-
-	/*
-	 * Most of the x86/switcher_32.S doesn't care that it's been moved; on
-	 * Intel, jumps are relative, and it doesn't access any references to
-	 * external code or data.
-	 *
-	 * The only exception is the interrupt handlers in switcher.S: their
-	 * addresses are placed in a table (default_idt_entries), so we need to
-	 * update the table with the new addresses.  switcher_offset() is a
-	 * convenience function which returns the distance between the
-	 * compiled-in switcher code and the high-mapped copy we just made.
-	 */
-	for (i = 0; i < IDT_ENTRIES; i++)
-		default_idt_entries[i] += switcher_offset();
-
-	/*
-	 * Set up the Switcher's per-cpu areas.
-	 *
-	 * Each CPU gets two pages of its own within the high-mapped region
-	 * (aka. "struct lguest_pages").  Much of this can be initialized now,
-	 * but some depends on what Guest we are running (which is set up in
-	 * copy_in_guest_info()).
-	 */
-	for_each_possible_cpu(i) {
-		/* lguest_pages() returns this CPU's two pages. */
-		struct lguest_pages *pages = lguest_pages(i);
-		/* This is a convenience pointer to make the code neater. */
-		struct lguest_ro_state *state = &pages->state;
-
-		/*
-		 * The Global Descriptor Table: the Host has a different one
-		 * for each CPU.  We keep a descriptor for the GDT which says
-		 * where it is and how big it is (the size is actually the last
-		 * byte, not the size, hence the "-1").
-		 */
-		state->host_gdt_desc.size = GDT_SIZE-1;
-		state->host_gdt_desc.address = (long)get_cpu_gdt_rw(i);
-
-		/*
-		 * All CPUs on the Host use the same Interrupt Descriptor
-		 * Table, so we just use store_idt(), which gets this CPU's IDT
-		 * descriptor.
-		 */
-		store_idt(&state->host_idt_desc);
-
-		/*
-		 * The descriptors for the Guest's GDT and IDT can be filled
-		 * out now, too.  We copy the GDT & IDT into ->guest_gdt and
-		 * ->guest_idt before actually running the Guest.
-		 */
-		state->guest_idt_desc.size = sizeof(state->guest_idt)-1;
-		state->guest_idt_desc.address = (long)&state->guest_idt;
-		state->guest_gdt_desc.size = sizeof(state->guest_gdt)-1;
-		state->guest_gdt_desc.address = (long)&state->guest_gdt;
-
-		/*
-		 * We know where we want the stack to be when the Guest enters
-		 * the Switcher: in pages->regs.  The stack grows upwards, so
-		 * we start it at the end of that structure.
-		 */
-		state->guest_tss.sp0 = (long)(&pages->regs + 1);
-		/*
-		 * And this is the GDT entry to use for the stack: we keep a
-		 * couple of special LGUEST entries.
-		 */
-		state->guest_tss.ss0 = LGUEST_DS;
-
-		/*
-		 * x86 can have a finegrained bitmap which indicates what I/O
-		 * ports the process can use.  We set it to the end of our
-		 * structure, meaning "none".
-		 */
-		state->guest_tss.io_bitmap_base = sizeof(state->guest_tss);
-
-		/*
-		 * Some GDT entries are the same across all Guests, so we can
-		 * set them up now.
-		 */
-		setup_default_gdt_entries(state);
-		/* Most IDT entries are the same for all Guests, too.*/
-		setup_default_idt_entries(state, default_idt_entries);
-
-		/*
-		 * The Host needs to be able to use the LGUEST segments on this
-		 * CPU, too, so put them in the Host GDT.
-		 */
-		get_cpu_gdt_rw(i)[GDT_ENTRY_LGUEST_CS] = FULL_EXEC_SEGMENT;
-		get_cpu_gdt_rw(i)[GDT_ENTRY_LGUEST_DS] = FULL_SEGMENT;
-	}
-
-	/*
-	 * In the Switcher, we want the %cs segment register to use the
-	 * LGUEST_CS GDT entry: we've put that in the Host and Guest GDTs, so
-	 * it will be undisturbed when we switch.  To change %cs and jump we
-	 * need this structure to feed to Intel's "lcall" instruction.
-	 */
-	lguest_entry.offset = (long)switch_to_guest + switcher_offset();
-	lguest_entry.segment = LGUEST_CS;
-
-	/*
-	 * Finally, we need to turn off "Page Global Enable".  PGE is an
-	 * optimization where page table entries are specially marked to show
-	 * they never change.  The Host kernel marks all the kernel pages this
-	 * way because it's always present, even when userspace is running.
-	 *
-	 * Lguest breaks this: unbeknownst to the rest of the Host kernel, we
-	 * switch to the Guest kernel.  If you don't disable this on all CPUs,
-	 * you'll get really weird bugs that you'll chase for two days.
-	 *
-	 * I used to turn PGE off every time we switched to the Guest and back
-	 * on when we return, but that slowed the Switcher down noticibly.
-	 */
-
-	/*
-	 * We don't need the complexity of CPUs coming and going while we're
-	 * doing this.
-	 */
-	get_online_cpus();
-	if (boot_cpu_has(X86_FEATURE_PGE)) { /* We have a broader idea of "global". */
-		/* Remember that this was originally set (for cleanup). */
-		cpu_had_pge = 1;
-		/*
-		 * adjust_pge is a helper function which sets or unsets the PGE
-		 * bit on its CPU, depending on the argument (0 == unset).
-		 */
-		on_each_cpu(adjust_pge, (void *)0, 1);
-		/* Turn off the feature in the global feature set. */
-		clear_cpu_cap(&boot_cpu_data, X86_FEATURE_PGE);
-	}
-	put_online_cpus();
-}
-/*:*/
-
-void __exit lguest_arch_host_fini(void)
-{
-	/* If we had PGE before we started, turn it back on now. */
-	get_online_cpus();
-	if (cpu_had_pge) {
-		set_cpu_cap(&boot_cpu_data, X86_FEATURE_PGE);
-		/* adjust_pge's argument "1" means set PGE. */
-		on_each_cpu(adjust_pge, (void *)1, 1);
-	}
-	put_online_cpus();
-}
-
-
-/*H:122 The i386-specific hypercalls simply farm out to the right functions. */
-int lguest_arch_do_hcall(struct lg_cpu *cpu, struct hcall_args *args)
-{
-	switch (args->arg0) {
-	case LHCALL_LOAD_GDT_ENTRY:
-		load_guest_gdt_entry(cpu, args->arg1, args->arg2, args->arg3);
-		break;
-	case LHCALL_LOAD_IDT_ENTRY:
-		load_guest_idt_entry(cpu, args->arg1, args->arg2, args->arg3);
-		break;
-	case LHCALL_LOAD_TLS:
-		guest_load_tls(cpu, args->arg1);
-		break;
-	default:
-		/* Bad Guest.  Bad! */
-		return -EIO;
-	}
-	return 0;
-}
-
-/*H:126 i386-specific hypercall initialization: */
-int lguest_arch_init_hypercalls(struct lg_cpu *cpu)
-{
-	u32 tsc_speed;
-
-	/*
-	 * The pointer to the Guest's "struct lguest_data" is the only argument.
-	 * We check that address now.
-	 */
-	if (!lguest_address_ok(cpu->lg, cpu->hcall->arg1,
-			       sizeof(*cpu->lg->lguest_data)))
-		return -EFAULT;
-
-	/*
-	 * Having checked it, we simply set lg->lguest_data to point straight
-	 * into the Launcher's memory at the right place and then use
-	 * copy_to_user/from_user from now on, instead of lgread/write.  I put
-	 * this in to show that I'm not immune to writing stupid
-	 * optimizations.
-	 */
-	cpu->lg->lguest_data = cpu->lg->mem_base + cpu->hcall->arg1;
-
-	/*
-	 * We insist that the Time Stamp Counter exist and doesn't change with
-	 * cpu frequency.  Some devious chip manufacturers decided that TSC
-	 * changes could be handled in software.  I decided that time going
-	 * backwards might be good for benchmarks, but it's bad for users.
-	 *
-	 * We also insist that the TSC be stable: the kernel detects unreliable
-	 * TSCs for its own purposes, and we use that here.
-	 */
-	if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC) && !check_tsc_unstable())
-		tsc_speed = tsc_khz;
-	else
-		tsc_speed = 0;
-	if (put_user(tsc_speed, &cpu->lg->lguest_data->tsc_khz))
-		return -EFAULT;
-
-	/* The interrupt code might not like the system call vector. */
-	if (!check_syscall_vector(cpu->lg))
-		kill_guest(cpu, "bad syscall vector");
-
-	return 0;
-}
-/*:*/
-
-/*L:030
- * Most of the Guest's registers are left alone: we used get_zeroed_page() to
- * allocate the structure, so they will be 0.
- */
-void lguest_arch_setup_regs(struct lg_cpu *cpu, unsigned long start)
-{
-	struct lguest_regs *regs = cpu->regs;
-
-	/*
-	 * There are four "segment" registers which the Guest needs to boot:
-	 * The "code segment" register (cs) refers to the kernel code segment
-	 * __KERNEL_CS, and the "data", "extra" and "stack" segment registers
-	 * refer to the kernel data segment __KERNEL_DS.
-	 *
-	 * The privilege level is packed into the lower bits.  The Guest runs
-	 * at privilege level 1 (GUEST_PL).
-	 */
-	regs->ds = regs->es = regs->ss = __KERNEL_DS|GUEST_PL;
-	regs->cs = __KERNEL_CS|GUEST_PL;
-
-	/*
-	 * The "eflags" register contains miscellaneous flags.  Bit 1 (0x002)
-	 * is supposed to always be "1".  Bit 9 (0x200) controls whether
-	 * interrupts are enabled.  We always leave interrupts enabled while
-	 * running the Guest.
-	 */
-	regs->eflags = X86_EFLAGS_IF | X86_EFLAGS_FIXED;
-
-	/*
-	 * The "Extended Instruction Pointer" register says where the Guest is
-	 * running.
-	 */
-	regs->eip = start;
-
-	/*
-	 * %esi points to our boot information, at physical address 0, so don't
-	 * touch it.
-	 */
-
-	/* There are a couple of GDT entries the Guest expects at boot. */
-	setup_guest_gdt(cpu);
-}
diff --git a/drivers/lguest/x86/switcher_32.S b/drivers/lguest/x86/switcher_32.S
deleted file mode 100644
index 40634b0db9f7..000000000000
--- a/drivers/lguest/x86/switcher_32.S
+++ /dev/null
@@ -1,388 +0,0 @@
-/*P:900
- * This is the Switcher: code which sits at 0xFFC00000 (or 0xFFE00000) astride
- * both the Host and Guest to do the low-level Guest<->Host switch.  It is as
- * simple as it can be made, but it's naturally very specific to x86.
- *
- * You have now completed Preparation.  If this has whet your appetite; if you
- * are feeling invigorated and refreshed then the next, more challenging stage
- * can be found in "make Guest".
- :*/
-
-/*M:012
- * Lguest is meant to be simple: my rule of thumb is that 1% more LOC must
- * gain at least 1% more performance.  Since neither LOC nor performance can be
- * measured beforehand, it generally means implementing a feature then deciding
- * if it's worth it.  And once it's implemented, who can say no?
- *
- * This is why I haven't implemented this idea myself.  I want to, but I
- * haven't.  You could, though.
- *
- * The main place where lguest performance sucks is Guest page faulting.  When
- * a Guest userspace process hits an unmapped page we switch back to the Host,
- * walk the page tables, find it's not mapped, switch back to the Guest page
- * fault handler, which calls a hypercall to set the page table entry, then
- * finally returns to userspace.  That's two round-trips.
- *
- * If we had a small walker in the Switcher, we could quickly check the Guest
- * page table and if the page isn't mapped, immediately reflect the fault back
- * into the Guest.  This means the Switcher would have to know the top of the
- * Guest page table and the page fault handler address.
- *
- * For simplicity, the Guest should only handle the case where the privilege
- * level of the fault is 3 and probably only not present or write faults.  It
- * should also detect recursive faults, and hand the original fault to the
- * Host (which is actually really easy).
- *
- * Two questions remain.  Would the performance gain outweigh the complexity?
- * And who would write the verse documenting it?
-:*/
-
-/*M:011
- * Lguest64 handles NMI.  This gave me NMI envy (until I looked at their
- * code).  It's worth doing though, since it would let us use oprofile in the
- * Host when a Guest is running.
-:*/
-
-/*S:100
- * Welcome to the Switcher itself!
- *
- * This file contains the low-level code which changes the CPU to run the Guest
- * code, and returns to the Host when something happens.  Understand this, and
- * you understand the heart of our journey.
- *
- * Because this is in assembler rather than C, our tale switches from prose to
- * verse.  First I tried limericks:
- *
- *	There once was an eax reg,
- *	To which our pointer was fed,
- *	It needed an add,
- *	Which asm-offsets.h had
- *	But this limerick is hurting my head.
- *
- * Next I tried haikus, but fitting the required reference to the seasons in
- * every stanza was quickly becoming tiresome:
- *
- *	The %eax reg
- *	Holds "struct lguest_pages" now:
- *	Cherry blossoms fall.
- *
- * Then I started with Heroic Verse, but the rhyming requirement leeched away
- * the content density and led to some uniquely awful oblique rhymes:
- *
- *	These constants are coming from struct offsets
- *	For use within the asm switcher text.
- *
- * Finally, I settled for something between heroic hexameter, and normal prose
- * with inappropriate linebreaks.  Anyway, it aint no Shakespeare.
- */
-
-// Not all kernel headers work from assembler
-// But these ones are needed: the ENTRY() define
-// And constants extracted from struct offsets
-// To avoid magic numbers and breakage:
-// Should they change the compiler can't save us
-// Down here in the depths of assembler code.
-#include <linux/linkage.h>
-#include <asm/asm-offsets.h>
-#include <asm/page.h>
-#include <asm/segment.h>
-#include <asm/lguest.h>
-
-// We mark the start of the code to copy
-// It's placed in .text tho it's never run here
-// You'll see the trick macro at the end
-// Which interleaves data and text to effect.
-.text
-ENTRY(start_switcher_text)
-
-// When we reach switch_to_guest we have just left
-// The safe and comforting shores of C code
-// %eax has the "struct lguest_pages" to use
-// Where we save state and still see it from the Guest
-// And %ebx holds the Guest shadow pagetable:
-// Once set we have truly left Host behind.
-ENTRY(switch_to_guest)
-	// We told gcc all its regs could fade,
-	// Clobbered by our journey into the Guest
-	// We could have saved them, if we tried
-	// But time is our master and cycles count.
-
-	// Segment registers must be saved for the Host
-	// We push them on the Host stack for later
-	pushl	%es
-	pushl	%ds
-	pushl	%gs
-	pushl	%fs
-	// But the compiler is fickle, and heeds
-	// No warning of %ebp clobbers
-	// When frame pointers are used.  That register
-	// Must be saved and restored or chaos strikes.
-	pushl	%ebp
-	// The Host's stack is done, now save it away
-	// In our "struct lguest_pages" at offset
-	// Distilled into asm-offsets.h
-	movl	%esp, LGUEST_PAGES_host_sp(%eax)
-
-	// All saved and there's now five steps before us:
-	// Stack, GDT, IDT, TSS
-	// Then last of all the page tables are flipped.
-
-	// Yet beware that our stack pointer must be
-	// Always valid lest an NMI hits
-	// %edx does the duty here as we juggle
-	// %eax is lguest_pages: our stack lies within.
-	movl	%eax, %edx
-	addl	$LGUEST_PAGES_regs, %edx
-	movl	%edx, %esp
-
-	// The Guest's GDT we so carefully
-	// Placed in the "struct lguest_pages" before
-	lgdt	LGUEST_PAGES_guest_gdt_desc(%eax)
-
-	// The Guest's IDT we did partially
-	// Copy to "struct lguest_pages" as well.
-	lidt	LGUEST_PAGES_guest_idt_desc(%eax)
-
-	// The TSS entry which controls traps
-	// Must be loaded up with "ltr" now:
-	// The GDT entry that TSS uses 
-	// Changes type when we load it: damn Intel!
-	// For after we switch over our page tables
-	// That entry will be read-only: we'd crash.
-	movl	$(GDT_ENTRY_TSS*8), %edx
-	ltr	%dx
-
-	// Look back now, before we take this last step!
-	// The Host's TSS entry was also marked used;
-	// Let's clear it again for our return.
-	// The GDT descriptor of the Host
-	// Points to the table after two "size" bytes
-	movl	(LGUEST_PAGES_host_gdt_desc+2)(%eax), %edx
-	// Clear "used" from type field (byte 5, bit 2)
-	andb	$0xFD, (GDT_ENTRY_TSS*8 + 5)(%edx)
-
-	// Once our page table's switched, the Guest is live!
-	// The Host fades as we run this final step.
-	// Our "struct lguest_pages" is now read-only.
-	movl	%ebx, %cr3
-
-	// The page table change did one tricky thing:
-	// The Guest's register page has been mapped
-	// Writable under our %esp (stack) --
-	// We can simply pop off all Guest regs.
-	popl	%eax
-	popl	%ebx
-	popl	%ecx
-	popl	%edx
-	popl	%esi
-	popl	%edi
-	popl	%ebp
-	popl	%gs
-	popl	%fs
-	popl	%ds
-	popl	%es
-
-	// Near the base of the stack lurk two strange fields
-	// Which we fill as we exit the Guest
-	// These are the trap number and its error
-	// We can simply step past them on our way.
-	addl	$8, %esp
-
-	// The last five stack slots hold return address
-	// And everything needed to switch privilege
-	// From Switcher's level 0 to Guest's 1,
-	// And the stack where the Guest had last left it.
-	// Interrupts are turned back on: we are Guest.
-	iret
-
-// We tread two paths to switch back to the Host
-// Yet both must save Guest state and restore Host
-// So we put the routine in a macro.
-#define SWITCH_TO_HOST							\
-	/* We save the Guest state: all registers first			\
-	 * Laid out just as "struct lguest_regs" defines */		\
-	pushl	%es;							\
-	pushl	%ds;							\
-	pushl	%fs;							\
-	pushl	%gs;							\
-	pushl	%ebp;							\
-	pushl	%edi;							\
-	pushl	%esi;							\
-	pushl	%edx;							\
-	pushl	%ecx;							\
-	pushl	%ebx;							\
-	pushl	%eax;							\
-	/* Our stack and our code are using segments			\
-	 * Set in the TSS and IDT					\
-	 * Yet if we were to touch data we'd use			\
-	 * Whatever data segment the Guest had.				\
-	 * Load the lguest ds segment for now. */			\
-	movl	$(LGUEST_DS), %eax;					\
-	movl	%eax, %ds;						\
-	/* So where are we?  Which CPU, which struct?			\
-	 * The stack is our clue: our TSS starts			\
-	 * It at the end of "struct lguest_pages".			\
-	 * Or we may have stumbled while restoring			\
-	 * Our Guest segment regs while in switch_to_guest,		\
-	 * The fault pushed atop that part-unwound stack.		\
-	 * If we round the stack down to the page start			\
-	 * We're at the start of "struct lguest_pages". */		\
-	movl	%esp, %eax;						\
-	andl	$(~(1 << PAGE_SHIFT - 1)), %eax;			\
-	/* Save our trap number: the switch will obscure it		\
-	 * (In the Host the Guest regs are not mapped here)		\
-	 * %ebx holds it safe for deliver_to_host */			\
-	movl	LGUEST_PAGES_regs_trapnum(%eax), %ebx;			\
-	/* The Host GDT, IDT and stack!					\
-	 * All these lie safely hidden from the Guest:			\
-	 * We must return to the Host page tables			\
-	 * (Hence that was saved in struct lguest_pages) */		\
-	movl	LGUEST_PAGES_host_cr3(%eax), %edx;			\
-	movl	%edx, %cr3;						\
-	/* As before, when we looked back at the Host			\
-	 * As we left and marked TSS unused				\
-	 * So must we now for the Guest left behind. */			\
-	andb	$0xFD, (LGUEST_PAGES_guest_gdt+GDT_ENTRY_TSS*8+5)(%eax); \
-	/* Switch to Host's GDT, IDT. */				\
-	lgdt	LGUEST_PAGES_host_gdt_desc(%eax);			\
-	lidt	LGUEST_PAGES_host_idt_desc(%eax);			\
-	/* Restore the Host's stack where its saved regs lie */		\
-	movl	LGUEST_PAGES_host_sp(%eax), %esp;			\
-	/* Last the TSS: our Host is returned */			\
-	movl	$(GDT_ENTRY_TSS*8), %edx;				\
-	ltr	%dx;							\
-	/* Restore now the regs saved right at the first. */		\
-	popl	%ebp;							\
-	popl	%fs;							\
-	popl	%gs;							\
-	popl	%ds;							\
-	popl	%es
-
-// The first path is trod when the Guest has trapped:
-// (Which trap it was has been pushed on the stack).
-// We need only switch back, and the Host will decode
-// Why we came home, and what needs to be done.
-return_to_host:
-	SWITCH_TO_HOST
-	iret
-
-// We are lead to the second path like so:
-// An interrupt, with some cause external
-// Has ajerked us rudely from the Guest's code
-// Again we must return home to the Host
-deliver_to_host:
-	SWITCH_TO_HOST
-	// But now we must go home via that place
-	// Where that interrupt was supposed to go
-	// Had we not been ensconced, running the Guest.
-	// Here we see the trickness of run_guest_once():
-	// The Host stack is formed like an interrupt
-	// With EIP, CS and EFLAGS layered.
-	// Interrupt handlers end with "iret"
-	// And that will take us home at long long last.
-
-	// But first we must find the handler to call!
-	// The IDT descriptor for the Host
-	// Has two bytes for size, and four for address:
-	// %edx will hold it for us for now.
-	movl	(LGUEST_PAGES_host_idt_desc+2)(%eax), %edx
-	// We now know the table address we need,
-	// And saved the trap's number inside %ebx.
-	// Yet the pointer to the handler is smeared
-	// Across the bits of the table entry.
-	// What oracle can tell us how to extract
-	// From such a convoluted encoding?
-	// I consulted gcc, and it gave
-	// These instructions, which I gladly credit:
-	leal	(%edx,%ebx,8), %eax
-	movzwl	(%eax),%edx
-	movl	4(%eax), %eax
-	xorw	%ax, %ax
-	orl	%eax, %edx
-	// Now the address of the handler's in %edx
-	// We call it now: its "iret" drops us home.
-	jmp	*%edx
-
-// Every interrupt can come to us here
-// But we must truly tell each apart.
-// They number two hundred and fifty six
-// And each must land in a different spot,
-// Push its number on stack, and join the stream.
-
-// And worse, a mere six of the traps stand apart
-// And push on their stack an addition:
-// An error number, thirty two bits long
-// So we punish the other two fifty
-// And make them push a zero so they match.
-
-// Yet two fifty six entries is long
-// And all will look most the same as the last
-// So we create a macro which can make
-// As many entries as we need to fill.
-
-// Note the change to .data then .text:
-// We plant the address of each entry
-// Into a (data) table for the Host
-// To know where each Guest interrupt should go.
-.macro IRQ_STUB N TARGET
-	.data; .long 1f; .text; 1:
- // Trap eight, ten through fourteen and seventeen
- // Supply an error number.  Else zero.
- .if (\N <> 8) && (\N < 10 || \N > 14) && (\N <> 17)
-	pushl	$0
- .endif
-	pushl	$\N
-	jmp	\TARGET
-	ALIGN
-.endm
-
-// This macro creates numerous entries
-// Using GAS macros which out-power C's.
-.macro IRQ_STUBS FIRST LAST TARGET
- irq=\FIRST
- .rept \LAST-\FIRST+1
-	IRQ_STUB irq \TARGET
-  irq=irq+1
- .endr
-.endm
-
-// Here's the marker for our pointer table
-// Laid in the data section just before
-// Each macro places the address of code
-// Forming an array: each one points to text
-// Which handles interrupt in its turn.
-.data
-.global default_idt_entries
-default_idt_entries:
-.text
-	// The first two traps go straight back to the Host
-	IRQ_STUBS 0 1 return_to_host
-	// We'll say nothing, yet, about NMI
-	IRQ_STUB 2 handle_nmi
-	// Other traps also return to the Host
-	IRQ_STUBS 3 31 return_to_host
-	// All interrupts go via their handlers
-	IRQ_STUBS 32 127 deliver_to_host
-	// 'Cept system calls coming from userspace
-	// Are to go to the Guest, never the Host.
-	IRQ_STUB 128 return_to_host
-	IRQ_STUBS 129 255 deliver_to_host
-
-// The NMI, what a fabulous beast
-// Which swoops in and stops us no matter that
-// We're suspended between heaven and hell,
-// (Or more likely between the Host and Guest)
-// When in it comes!  We are dazed and confused
-// So we do the simplest thing which one can.
-// Though we've pushed the trap number and zero
-// We discard them, return, and hope we live.
-handle_nmi:
-	addl	$8, %esp
-	iret
-
-// We are done; all that's left is Mastery
-// And "make Mastery" is a journey long
-// Designed to make your fingers itch to code.
-
-// Here ends the text, the file and poem.
-ENTRY(end_switcher_text)
diff --git a/drivers/macintosh/macio_sysfs.c b/drivers/macintosh/macio_sysfs.c
index 2445274f7e4b..281f5345661e 100644
--- a/drivers/macintosh/macio_sysfs.c
+++ b/drivers/macintosh/macio_sysfs.c
@@ -52,7 +52,7 @@ static ssize_t devspec_show(struct device *dev,
 	struct platform_device *ofdev;
 
 	ofdev = to_platform_device(dev);
-	return sprintf(buf, "%s\n", ofdev->dev.of_node->full_name);
+	return sprintf(buf, "%pOF\n", ofdev->dev.of_node);
 }
 static DEVICE_ATTR_RO(modalias);
 static DEVICE_ATTR_RO(devspec);
diff --git a/drivers/macintosh/rack-meter.c b/drivers/macintosh/rack-meter.c
index e199fd6c71ce..910b5b6f96b1 100644
--- a/drivers/macintosh/rack-meter.c
+++ b/drivers/macintosh/rack-meter.c
@@ -411,16 +411,16 @@ static int rackmeter_probe(struct macio_dev* mdev,
 #if 0 /* Use that when i2s-a is finally an mdev per-se */
 	if (macio_resource_count(mdev) < 2 || macio_irq_count(mdev) < 2) {
 		printk(KERN_ERR
-		       "rackmeter: found match but lacks resources: %s"
+		       "rackmeter: found match but lacks resources: %pOF"
 		       " (%d resources, %d interrupts)\n",
-		       mdev->ofdev.node->full_name);
+		       mdev->ofdev.dev.of_node);
 		rc = -ENXIO;
 		goto bail_free;
 	}
 	if (macio_request_resources(mdev, "rackmeter")) {
 		printk(KERN_ERR
-		       "rackmeter: failed to request resources: %s\n",
-		       mdev->ofdev.node->full_name);
+		       "rackmeter: failed to request resources: %pOF\n",
+		       mdev->ofdev.dev.of_node);
 		rc = -EBUSY;
 		goto bail_free;
 	}
@@ -431,8 +431,8 @@ static int rackmeter_probe(struct macio_dev* mdev,
 	    of_address_to_resource(i2s, 0, &ri2s) ||
 	    of_address_to_resource(i2s, 1, &rdma)) {
 		printk(KERN_ERR
-		       "rackmeter: found match but lacks resources: %s",
-		       mdev->ofdev.dev.of_node->full_name);
+		       "rackmeter: found match but lacks resources: %pOF",
+		       mdev->ofdev.dev.of_node);
 		rc = -ENXIO;
 		goto bail_free;
 	}
@@ -579,7 +579,7 @@ static int rackmeter_shutdown(struct macio_dev* mdev)
 	return 0;
 }
 
-static struct of_device_id rackmeter_match[] = {
+static const struct of_device_id rackmeter_match[] = {
 	{ .name = "i2s" },
 	{ }
 };
diff --git a/drivers/macintosh/smu.c b/drivers/macintosh/smu.c
index 1ac66421877a..ea9bdc85a21d 100644
--- a/drivers/macintosh/smu.c
+++ b/drivers/macintosh/smu.c
@@ -589,14 +589,14 @@ static int smu_late_init(void)
 	if (smu->db_node) {
 		smu->db_irq = irq_of_parse_and_map(smu->db_node, 0);
 		if (!smu->db_irq)
-			printk(KERN_ERR "smu: failed to map irq for node %s\n",
-			       smu->db_node->full_name);
+			printk(KERN_ERR "smu: failed to map irq for node %pOF\n",
+			       smu->db_node);
 	}
 	if (smu->msg_node) {
 		smu->msg_irq = irq_of_parse_and_map(smu->msg_node, 0);
 		if (!smu->msg_irq)
-			printk(KERN_ERR "smu: failed to map irq for node %s\n",
-			       smu->msg_node->full_name);
+			printk(KERN_ERR "smu: failed to map irq for node %pOF\n",
+			       smu->msg_node);
 	}
 
 	/*
diff --git a/drivers/macintosh/via-cuda.c b/drivers/macintosh/via-cuda.c
index c60415958dfe..147da4edd021 100644
--- a/drivers/macintosh/via-cuda.c
+++ b/drivers/macintosh/via-cuda.c
@@ -297,8 +297,8 @@ static int __init via_cuda_start(void)
 #else
     cuda_irq = irq_of_parse_and_map(vias, 0);
     if (!cuda_irq) {
-	printk(KERN_ERR "via-cuda: can't map interrupts for %s\n",
-	       vias->full_name);
+	printk(KERN_ERR "via-cuda: can't map interrupts for %pOF\n",
+	       vias);
 	return -ENODEV;
     }
 #endif
diff --git a/drivers/macintosh/windfarm_cpufreq_clamp.c b/drivers/macintosh/windfarm_cpufreq_clamp.c
index 72d1fdfe02a5..2626990331dc 100644
--- a/drivers/macintosh/windfarm_cpufreq_clamp.c
+++ b/drivers/macintosh/windfarm_cpufreq_clamp.c
@@ -63,7 +63,7 @@ static s32 clamp_max(struct wf_control *ct)
 	return 1;
 }
 
-static struct wf_control_ops clamp_ops = {
+static const struct wf_control_ops clamp_ops = {
 	.set_value	= clamp_set,
 	.get_value	= clamp_get,
 	.get_min	= clamp_min,
diff --git a/drivers/macintosh/windfarm_fcu_controls.c b/drivers/macintosh/windfarm_fcu_controls.c
index 0226b796a21c..fab7a21e9577 100644
--- a/drivers/macintosh/windfarm_fcu_controls.c
+++ b/drivers/macintosh/windfarm_fcu_controls.c
@@ -470,8 +470,8 @@ static void wf_fcu_lookup_fans(struct wf_fcu_priv *pv)
 				id = ((*reg) - 0x30) / 2;
 			if (id > 7) {
 				pr_warning("wf_fcu: Can't parse "
-				       "fan ID in device-tree for %s\n",
-					   np->full_name);
+				       "fan ID in device-tree for %pOF\n",
+					   np);
 				break;
 			}
 			wf_fcu_add_fan(pv, name, type, id);
diff --git a/drivers/macintosh/windfarm_lm75_sensor.c b/drivers/macintosh/windfarm_lm75_sensor.c
index 590214ba736c..6cdfe714901d 100644
--- a/drivers/macintosh/windfarm_lm75_sensor.c
+++ b/drivers/macintosh/windfarm_lm75_sensor.c
@@ -82,7 +82,7 @@ static void wf_lm75_release(struct wf_sensor *sr)
 	kfree(lm);
 }
 
-static struct wf_sensor_ops wf_lm75_ops = {
+static const struct wf_sensor_ops wf_lm75_ops = {
 	.get_value	= wf_lm75_get,
 	.release	= wf_lm75_release,
 	.owner		= THIS_MODULE,
diff --git a/drivers/macintosh/windfarm_lm87_sensor.c b/drivers/macintosh/windfarm_lm87_sensor.c
index c071aab79dd1..35aa571d498a 100644
--- a/drivers/macintosh/windfarm_lm87_sensor.c
+++ b/drivers/macintosh/windfarm_lm87_sensor.c
@@ -91,7 +91,7 @@ static void wf_lm87_release(struct wf_sensor *sr)
 	kfree(lm);
 }
 
-static struct wf_sensor_ops wf_lm87_ops = {
+static const struct wf_sensor_ops wf_lm87_ops = {
 	.get_value	= wf_lm87_get,
 	.release	= wf_lm87_release,
 	.owner		= THIS_MODULE,
@@ -126,8 +126,8 @@ static int wf_lm87_probe(struct i2c_client *client,
 		}
 	}
 	if (!name) {
-		pr_warning("wf_lm87: Unsupported sensor %s\n",
-			   client->dev.of_node->full_name);
+		pr_warning("wf_lm87: Unsupported sensor %pOF\n",
+			   client->dev.of_node);
 		return -ENODEV;
 	}
 
diff --git a/drivers/macintosh/windfarm_max6690_sensor.c b/drivers/macintosh/windfarm_max6690_sensor.c
index 87e439b10318..6ad035e13c08 100644
--- a/drivers/macintosh/windfarm_max6690_sensor.c
+++ b/drivers/macintosh/windfarm_max6690_sensor.c
@@ -55,7 +55,7 @@ static void wf_max6690_release(struct wf_sensor *sr)
 	kfree(max);
 }
 
-static struct wf_sensor_ops wf_max6690_ops = {
+static const struct wf_sensor_ops wf_max6690_ops = {
 	.get_value	= wf_max6690_get,
 	.release	= wf_max6690_release,
 	.owner		= THIS_MODULE,
diff --git a/drivers/macintosh/windfarm_rm31.c b/drivers/macintosh/windfarm_rm31.c
index bdfcb8a8bfbb..a0cd9c7f9835 100644
--- a/drivers/macintosh/windfarm_rm31.c
+++ b/drivers/macintosh/windfarm_rm31.c
@@ -338,7 +338,7 @@ static int cpu_setup_pid(int cpu)
 }
 
 /* Backside/U3 fan */
-static struct wf_pid_param backside_param = {
+static const struct wf_pid_param backside_param = {
 	.interval	= 1,
 	.history_len	= 2,
 	.gd		= 0x00500000,
@@ -351,7 +351,7 @@ static struct wf_pid_param backside_param = {
 };
 
 /* DIMMs temperature (clamp the backside fan) */
-static struct wf_pid_param dimms_param = {
+static const struct wf_pid_param dimms_param = {
 	.interval	= 1,
 	.history_len	= 20,
 	.gd		= 0,
diff --git a/drivers/macintosh/windfarm_smu_controls.c b/drivers/macintosh/windfarm_smu_controls.c
index c155a54e8638..d174c7437337 100644
--- a/drivers/macintosh/windfarm_smu_controls.c
+++ b/drivers/macintosh/windfarm_smu_controls.c
@@ -145,7 +145,7 @@ static s32 smu_fan_max(struct wf_control *ct)
 	return fct->max;
 }
 
-static struct wf_control_ops smu_fan_ops = {
+static const struct wf_control_ops smu_fan_ops = {
 	.set_value	= smu_fan_set,
 	.get_value	= smu_fan_get,
 	.get_min	= smu_fan_min,
diff --git a/drivers/macintosh/windfarm_smu_sat.c b/drivers/macintosh/windfarm_smu_sat.c
index ad6223e88340..da7f4fc1a51d 100644
--- a/drivers/macintosh/windfarm_smu_sat.c
+++ b/drivers/macintosh/windfarm_smu_sat.c
@@ -195,7 +195,7 @@ static void wf_sat_sensor_release(struct wf_sensor *sr)
 	kref_put(&sat->ref, wf_sat_release);
 }
 
-static struct wf_sensor_ops wf_sat_ops = {
+static const struct wf_sensor_ops wf_sat_ops = {
 	.get_value	= wf_sat_sensor_get,
 	.release	= wf_sat_sensor_release,
 	.owner		= THIS_MODULE,
@@ -248,7 +248,7 @@ static int wf_sat_probe(struct i2c_client *client,
 		core = loc[5] - '0';
 		if (chip > 1 || core > 1) {
 			printk(KERN_ERR "wf_sat_create: don't understand "
-			       "location %s for %s\n", loc, child->full_name);
+			       "location %s for %pOF\n", loc, child);
 			continue;
 		}
 		cpu = 2 * chip + core;
diff --git a/drivers/macintosh/windfarm_smu_sensors.c b/drivers/macintosh/windfarm_smu_sensors.c
index 1cc4e4953d89..172fd267dcf6 100644
--- a/drivers/macintosh/windfarm_smu_sensors.c
+++ b/drivers/macintosh/windfarm_smu_sensors.c
@@ -172,22 +172,22 @@ static int smu_slotspow_get(struct wf_sensor *sr, s32 *value)
 }
 
 
-static struct wf_sensor_ops smu_cputemp_ops = {
+static const struct wf_sensor_ops smu_cputemp_ops = {
 	.get_value	= smu_cputemp_get,
 	.release	= smu_ads_release,
 	.owner		= THIS_MODULE,
 };
-static struct wf_sensor_ops smu_cpuamp_ops = {
+static const struct wf_sensor_ops smu_cpuamp_ops = {
 	.get_value	= smu_cpuamp_get,
 	.release	= smu_ads_release,
 	.owner		= THIS_MODULE,
 };
-static struct wf_sensor_ops smu_cpuvolt_ops = {
+static const struct wf_sensor_ops smu_cpuvolt_ops = {
 	.get_value	= smu_cpuvolt_get,
 	.release	= smu_ads_release,
 	.owner		= THIS_MODULE,
 };
-static struct wf_sensor_ops smu_slotspow_ops = {
+static const struct wf_sensor_ops smu_slotspow_ops = {
 	.get_value	= smu_slotspow_get,
 	.release	= smu_ads_release,
 	.owner		= THIS_MODULE,
@@ -327,7 +327,7 @@ static int smu_cpu_power_get(struct wf_sensor *sr, s32 *value)
 	return 0;
 }
 
-static struct wf_sensor_ops smu_cpu_power_ops = {
+static const struct wf_sensor_ops smu_cpu_power_ops = {
 	.get_value	= smu_cpu_power_get,
 	.release	= smu_cpu_power_release,
 	.owner		= THIS_MODULE,
diff --git a/drivers/mailbox/bcm-flexrm-mailbox.c b/drivers/mailbox/bcm-flexrm-mailbox.c
index da67882caa7b..ae6146311934 100644
--- a/drivers/mailbox/bcm-flexrm-mailbox.c
+++ b/drivers/mailbox/bcm-flexrm-mailbox.c
@@ -17,12 +17,14 @@
 
 #include <asm/barrier.h>
 #include <asm/byteorder.h>
+#include <linux/atomic.h>
+#include <linux/bitmap.h>
+#include <linux/debugfs.h>
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/dma-mapping.h>
 #include <linux/dmapool.h>
 #include <linux/err.h>
-#include <linux/idr.h>
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
 #include <linux/mailbox_controller.h>
@@ -95,7 +97,7 @@
 
 /* Register RING_CMPL_START_ADDR fields */
 #define CMPL_START_ADDR_VALUE(pa)			\
-	((u32)((((u64)(pa)) >> RING_CMPL_ALIGN_ORDER) & 0x03ffffff))
+	((u32)((((u64)(pa)) >> RING_CMPL_ALIGN_ORDER) & 0x07ffffff))
 
 /* Register RING_CONTROL fields */
 #define CONTROL_MASK_DISABLE_CONTROL			12
@@ -260,18 +262,21 @@ struct flexrm_ring {
 	void __iomem *regs;
 	bool irq_requested;
 	unsigned int irq;
+	cpumask_t irq_aff_hint;
 	unsigned int msi_timer_val;
 	unsigned int msi_count_threshold;
-	struct ida requests_ida;
 	struct brcm_message *requests[RING_MAX_REQ_COUNT];
 	void *bd_base;
 	dma_addr_t bd_dma_base;
 	u32 bd_write_offset;
 	void *cmpl_base;
 	dma_addr_t cmpl_dma_base;
+	/* Atomic stats */
+	atomic_t msg_send_count;
+	atomic_t msg_cmpl_count;
 	/* Protected members */
 	spinlock_t lock;
-	struct brcm_message *last_pending_msg;
+	DECLARE_BITMAP(requests_bmap, RING_MAX_REQ_COUNT);
 	u32 cmpl_read_offset;
 };
 
@@ -282,6 +287,9 @@ struct flexrm_mbox {
 	struct flexrm_ring *rings;
 	struct dma_pool *bd_pool;
 	struct dma_pool *cmpl_pool;
+	struct dentry *root;
+	struct dentry *config;
+	struct dentry *stats;
 	struct mbox_controller controller;
 };
 
@@ -912,6 +920,62 @@ static void *flexrm_write_descs(struct brcm_message *msg, u32 nhcnt,
 
 /* ====== FlexRM driver helper routines ===== */
 
+static void flexrm_write_config_in_seqfile(struct flexrm_mbox *mbox,
+					   struct seq_file *file)
+{
+	int i;
+	const char *state;
+	struct flexrm_ring *ring;
+
+	seq_printf(file, "%-5s %-9s %-18s %-10s %-18s %-10s\n",
+		   "Ring#", "State", "BD_Addr", "BD_Size",
+		   "Cmpl_Addr", "Cmpl_Size");
+
+	for (i = 0; i < mbox->num_rings; i++) {
+		ring = &mbox->rings[i];
+		if (readl(ring->regs + RING_CONTROL) &
+		    BIT(CONTROL_ACTIVE_SHIFT))
+			state = "active";
+		else
+			state = "inactive";
+		seq_printf(file,
+			   "%-5d %-9s 0x%016llx 0x%08x 0x%016llx 0x%08x\n",
+			   ring->num, state,
+			   (unsigned long long)ring->bd_dma_base,
+			   (u32)RING_BD_SIZE,
+			   (unsigned long long)ring->cmpl_dma_base,
+			   (u32)RING_CMPL_SIZE);
+	}
+}
+
+static void flexrm_write_stats_in_seqfile(struct flexrm_mbox *mbox,
+					  struct seq_file *file)
+{
+	int i;
+	u32 val, bd_read_offset;
+	struct flexrm_ring *ring;
+
+	seq_printf(file, "%-5s %-10s %-10s %-10s %-11s %-11s\n",
+		   "Ring#", "BD_Read", "BD_Write",
+		   "Cmpl_Read", "Submitted", "Completed");
+
+	for (i = 0; i < mbox->num_rings; i++) {
+		ring = &mbox->rings[i];
+		bd_read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR);
+		val = readl_relaxed(ring->regs + RING_BD_START_ADDR);
+		bd_read_offset *= RING_DESC_SIZE;
+		bd_read_offset += (u32)(BD_START_ADDR_DECODE(val) -
+					ring->bd_dma_base);
+		seq_printf(file, "%-5d 0x%08x 0x%08x 0x%08x %-11d %-11d\n",
+			   ring->num,
+			   (u32)bd_read_offset,
+			   (u32)ring->bd_write_offset,
+			   (u32)ring->cmpl_read_offset,
+			   (u32)atomic_read(&ring->msg_send_count),
+			   (u32)atomic_read(&ring->msg_cmpl_count));
+	}
+}
+
 static int flexrm_new_request(struct flexrm_ring *ring,
 				struct brcm_message *batch_msg,
 				struct brcm_message *msg)
@@ -929,38 +993,24 @@ static int flexrm_new_request(struct flexrm_ring *ring,
 	msg->error = 0;
 
 	/* If no requests possible then save data pointer and goto done. */
-	reqid = ida_simple_get(&ring->requests_ida, 0,
-				RING_MAX_REQ_COUNT, GFP_KERNEL);
-	if (reqid < 0) {
-		spin_lock_irqsave(&ring->lock, flags);
-		if (batch_msg)
-			ring->last_pending_msg = batch_msg;
-		else
-			ring->last_pending_msg = msg;
-		spin_unlock_irqrestore(&ring->lock, flags);
-		return 0;
-	}
+	spin_lock_irqsave(&ring->lock, flags);
+	reqid = bitmap_find_free_region(ring->requests_bmap,
+					RING_MAX_REQ_COUNT, 0);
+	spin_unlock_irqrestore(&ring->lock, flags);
+	if (reqid < 0)
+		return -ENOSPC;
 	ring->requests[reqid] = msg;
 
 	/* Do DMA mappings for the message */
 	ret = flexrm_dma_map(ring->mbox->dev, msg);
 	if (ret < 0) {
 		ring->requests[reqid] = NULL;
-		ida_simple_remove(&ring->requests_ida, reqid);
+		spin_lock_irqsave(&ring->lock, flags);
+		bitmap_release_region(ring->requests_bmap, reqid, 0);
+		spin_unlock_irqrestore(&ring->lock, flags);
 		return ret;
 	}
 
-	/* If last_pending_msg is already set then goto done with error */
-	spin_lock_irqsave(&ring->lock, flags);
-	if (ring->last_pending_msg)
-		ret = -ENOSPC;
-	spin_unlock_irqrestore(&ring->lock, flags);
-	if (ret < 0) {
-		dev_warn(ring->mbox->dev, "no space in ring %d\n", ring->num);
-		exit_cleanup = true;
-		goto exit;
-	}
-
 	/* Determine current HW BD read offset */
 	read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR);
 	val = readl_relaxed(ring->regs + RING_BD_START_ADDR);
@@ -987,13 +1037,7 @@ static int flexrm_new_request(struct flexrm_ring *ring,
 			break;
 	}
 	if (count) {
-		spin_lock_irqsave(&ring->lock, flags);
-		if (batch_msg)
-			ring->last_pending_msg = batch_msg;
-		else
-			ring->last_pending_msg = msg;
-		spin_unlock_irqrestore(&ring->lock, flags);
-		ret = 0;
+		ret = -ENOSPC;
 		exit_cleanup = true;
 		goto exit;
 	}
@@ -1012,6 +1056,9 @@ static int flexrm_new_request(struct flexrm_ring *ring,
 	/* Save ring BD write offset */
 	ring->bd_write_offset = (unsigned long)(next - ring->bd_base);
 
+	/* Increment number of messages sent */
+	atomic_inc_return(&ring->msg_send_count);
+
 exit:
 	/* Update error status in message */
 	msg->error = ret;
@@ -1020,7 +1067,9 @@ exit:
 	if (exit_cleanup) {
 		flexrm_dma_unmap(ring->mbox->dev, msg);
 		ring->requests[reqid] = NULL;
-		ida_simple_remove(&ring->requests_ida, reqid);
+		spin_lock_irqsave(&ring->lock, flags);
+		bitmap_release_region(ring->requests_bmap, reqid, 0);
+		spin_unlock_irqrestore(&ring->lock, flags);
 	}
 
 	return ret;
@@ -1037,12 +1086,6 @@ static int flexrm_process_completions(struct flexrm_ring *ring)
 
 	spin_lock_irqsave(&ring->lock, flags);
 
-	/* Check last_pending_msg */
-	if (ring->last_pending_msg) {
-		msg = ring->last_pending_msg;
-		ring->last_pending_msg = NULL;
-	}
-
 	/*
 	 * Get current completion read and write offset
 	 *
@@ -1058,10 +1101,6 @@ static int flexrm_process_completions(struct flexrm_ring *ring)
 
 	spin_unlock_irqrestore(&ring->lock, flags);
 
-	/* If last_pending_msg was set then queue it back */
-	if (msg)
-		mbox_send_message(chan, msg);
-
 	/* For each completed request notify mailbox clients */
 	reqid = 0;
 	while (cmpl_read_offset != cmpl_write_offset) {
@@ -1095,7 +1134,9 @@ static int flexrm_process_completions(struct flexrm_ring *ring)
 
 		/* Release reqid for recycling */
 		ring->requests[reqid] = NULL;
-		ida_simple_remove(&ring->requests_ida, reqid);
+		spin_lock_irqsave(&ring->lock, flags);
+		bitmap_release_region(ring->requests_bmap, reqid, 0);
+		spin_unlock_irqrestore(&ring->lock, flags);
 
 		/* Unmap DMA mappings */
 		flexrm_dma_unmap(ring->mbox->dev, msg);
@@ -1105,12 +1146,37 @@ static int flexrm_process_completions(struct flexrm_ring *ring)
 		mbox_chan_received_data(chan, msg);
 
 		/* Increment number of completions processed */
+		atomic_inc_return(&ring->msg_cmpl_count);
 		count++;
 	}
 
 	return count;
 }
 
+/* ====== FlexRM Debugfs callbacks ====== */
+
+static int flexrm_debugfs_conf_show(struct seq_file *file, void *offset)
+{
+	struct platform_device *pdev = to_platform_device(file->private);
+	struct flexrm_mbox *mbox = platform_get_drvdata(pdev);
+
+	/* Write config in file */
+	flexrm_write_config_in_seqfile(mbox, file);
+
+	return 0;
+}
+
+static int flexrm_debugfs_stats_show(struct seq_file *file, void *offset)
+{
+	struct platform_device *pdev = to_platform_device(file->private);
+	struct flexrm_mbox *mbox = platform_get_drvdata(pdev);
+
+	/* Write stats in file */
+	flexrm_write_stats_in_seqfile(mbox, file);
+
+	return 0;
+}
+
 /* ====== FlexRM interrupt handler ===== */
 
 static irqreturn_t flexrm_irq_event(int irq, void *dev_id)
@@ -1217,6 +1283,18 @@ static int flexrm_startup(struct mbox_chan *chan)
 	}
 	ring->irq_requested = true;
 
+	/* Set IRQ affinity hint */
+	ring->irq_aff_hint = CPU_MASK_NONE;
+	val = ring->mbox->num_rings;
+	val = (num_online_cpus() < val) ? val / num_online_cpus() : 1;
+	cpumask_set_cpu((ring->num / val) % num_online_cpus(),
+			&ring->irq_aff_hint);
+	ret = irq_set_affinity_hint(ring->irq, &ring->irq_aff_hint);
+	if (ret) {
+		dev_err(ring->mbox->dev, "failed to set IRQ affinity hint\n");
+		goto fail_free_irq;
+	}
+
 	/* Disable/inactivate ring */
 	writel_relaxed(0x0, ring->regs + RING_CONTROL);
 
@@ -1233,9 +1311,6 @@ static int flexrm_startup(struct mbox_chan *chan)
 	val = CMPL_START_ADDR_VALUE(ring->cmpl_dma_base);
 	writel_relaxed(val, ring->regs + RING_CMPL_START_ADDR);
 
-	/* Ensure last pending message is cleared */
-	ring->last_pending_msg = NULL;
-
 	/* Completion read pointer will be same as HW write pointer */
 	ring->cmpl_read_offset =
 			readl_relaxed(ring->regs + RING_CMPL_WRITE_PTR);
@@ -1259,8 +1334,15 @@ static int flexrm_startup(struct mbox_chan *chan)
 	val = BIT(CONTROL_ACTIVE_SHIFT);
 	writel_relaxed(val, ring->regs + RING_CONTROL);
 
+	/* Reset stats to zero */
+	atomic_set(&ring->msg_send_count, 0);
+	atomic_set(&ring->msg_cmpl_count, 0);
+
 	return 0;
 
+fail_free_irq:
+	free_irq(ring->irq, ring);
+	ring->irq_requested = false;
 fail_free_cmpl_memory:
 	dma_pool_free(ring->mbox->cmpl_pool,
 		      ring->cmpl_base, ring->cmpl_dma_base);
@@ -1302,7 +1384,6 @@ static void flexrm_shutdown(struct mbox_chan *chan)
 
 		/* Release reqid for recycling */
 		ring->requests[reqid] = NULL;
-		ida_simple_remove(&ring->requests_ida, reqid);
 
 		/* Unmap DMA mappings */
 		flexrm_dma_unmap(ring->mbox->dev, msg);
@@ -1312,8 +1393,12 @@ static void flexrm_shutdown(struct mbox_chan *chan)
 		mbox_chan_received_data(chan, msg);
 	}
 
+	/* Clear requests bitmap */
+	bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT);
+
 	/* Release IRQ */
 	if (ring->irq_requested) {
+		irq_set_affinity_hint(ring->irq, NULL);
 		free_irq(ring->irq, ring);
 		ring->irq_requested = false;
 	}
@@ -1333,24 +1418,10 @@ static void flexrm_shutdown(struct mbox_chan *chan)
 	}
 }
 
-static bool flexrm_last_tx_done(struct mbox_chan *chan)
-{
-	bool ret;
-	unsigned long flags;
-	struct flexrm_ring *ring = chan->con_priv;
-
-	spin_lock_irqsave(&ring->lock, flags);
-	ret = (ring->last_pending_msg) ? false : true;
-	spin_unlock_irqrestore(&ring->lock, flags);
-
-	return ret;
-}
-
 static const struct mbox_chan_ops flexrm_mbox_chan_ops = {
 	.send_data	= flexrm_send_data,
 	.startup	= flexrm_startup,
 	.shutdown	= flexrm_shutdown,
-	.last_tx_done	= flexrm_last_tx_done,
 	.peek_data	= flexrm_peek_data,
 };
 
@@ -1468,14 +1539,15 @@ static int flexrm_mbox_probe(struct platform_device *pdev)
 		ring->irq_requested = false;
 		ring->msi_timer_val = MSI_TIMER_VAL_MASK;
 		ring->msi_count_threshold = 0x1;
-		ida_init(&ring->requests_ida);
 		memset(ring->requests, 0, sizeof(ring->requests));
 		ring->bd_base = NULL;
 		ring->bd_dma_base = 0;
 		ring->cmpl_base = NULL;
 		ring->cmpl_dma_base = 0;
+		atomic_set(&ring->msg_send_count, 0);
+		atomic_set(&ring->msg_cmpl_count, 0);
 		spin_lock_init(&ring->lock);
-		ring->last_pending_msg = NULL;
+		bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT);
 		ring->cmpl_read_offset = 0;
 	}
 
@@ -1515,10 +1587,39 @@ static int flexrm_mbox_probe(struct platform_device *pdev)
 		ring->irq = desc->irq;
 	}
 
+	/* Check availability of debugfs */
+	if (!debugfs_initialized())
+		goto skip_debugfs;
+
+	/* Create debugfs root entry */
+	mbox->root = debugfs_create_dir(dev_name(mbox->dev), NULL);
+	if (IS_ERR_OR_NULL(mbox->root)) {
+		ret = PTR_ERR_OR_ZERO(mbox->root);
+		goto fail_free_msis;
+	}
+
+	/* Create debugfs config entry */
+	mbox->config = debugfs_create_devm_seqfile(mbox->dev,
+						   "config", mbox->root,
+						   flexrm_debugfs_conf_show);
+	if (IS_ERR_OR_NULL(mbox->config)) {
+		ret = PTR_ERR_OR_ZERO(mbox->config);
+		goto fail_free_debugfs_root;
+	}
+
+	/* Create debugfs stats entry */
+	mbox->stats = debugfs_create_devm_seqfile(mbox->dev,
+						  "stats", mbox->root,
+						  flexrm_debugfs_stats_show);
+	if (IS_ERR_OR_NULL(mbox->stats)) {
+		ret = PTR_ERR_OR_ZERO(mbox->stats);
+		goto fail_free_debugfs_root;
+	}
+skip_debugfs:
+
 	/* Initialize mailbox controller */
 	mbox->controller.txdone_irq = false;
-	mbox->controller.txdone_poll = true;
-	mbox->controller.txpoll_period = 1;
+	mbox->controller.txdone_poll = false;
 	mbox->controller.ops = &flexrm_mbox_chan_ops;
 	mbox->controller.dev = dev;
 	mbox->controller.num_chans = mbox->num_rings;
@@ -1527,7 +1628,7 @@ static int flexrm_mbox_probe(struct platform_device *pdev)
 				sizeof(*mbox->controller.chans), GFP_KERNEL);
 	if (!mbox->controller.chans) {
 		ret = -ENOMEM;
-		goto fail_free_msis;
+		goto fail_free_debugfs_root;
 	}
 	for (index = 0; index < mbox->num_rings; index++)
 		mbox->controller.chans[index].con_priv = &mbox->rings[index];
@@ -1535,13 +1636,15 @@ static int flexrm_mbox_probe(struct platform_device *pdev)
 	/* Register mailbox controller */
 	ret = mbox_controller_register(&mbox->controller);
 	if (ret)
-		goto fail_free_msis;
+		goto fail_free_debugfs_root;
 
 	dev_info(dev, "registered flexrm mailbox with %d channels\n",
 			mbox->controller.num_chans);
 
 	return 0;
 
+fail_free_debugfs_root:
+	debugfs_remove_recursive(mbox->root);
 fail_free_msis:
 	platform_msi_domain_free_irqs(dev);
 fail_destroy_cmpl_pool:
@@ -1554,23 +1657,18 @@ fail:
 
 static int flexrm_mbox_remove(struct platform_device *pdev)
 {
-	int index;
 	struct device *dev = &pdev->dev;
-	struct flexrm_ring *ring;
 	struct flexrm_mbox *mbox = platform_get_drvdata(pdev);
 
 	mbox_controller_unregister(&mbox->controller);
 
+	debugfs_remove_recursive(mbox->root);
+
 	platform_msi_domain_free_irqs(dev);
 
 	dma_pool_destroy(mbox->cmpl_pool);
 	dma_pool_destroy(mbox->bd_pool);
 
-	for (index = 0; index < mbox->num_rings; index++) {
-		ring = &mbox->rings[index];
-		ida_destroy(&ring->requests_ida);
-	}
-
 	return 0;
 }
 
diff --git a/drivers/mailbox/pcc.c b/drivers/mailbox/pcc.c
index cbca5e51b975..9b7005e1345e 100644
--- a/drivers/mailbox/pcc.c
+++ b/drivers/mailbox/pcc.c
@@ -457,10 +457,8 @@ static int __init acpi_pcc_probe(void)
 	/* Search for PCCT */
 	status = acpi_get_table(ACPI_SIG_PCCT, 0, &pcct_tbl);
 
-	if (ACPI_FAILURE(status) || !pcct_tbl) {
-		pr_warn("PCCT header not found.\n");
+	if (ACPI_FAILURE(status) || !pcct_tbl)
 		return -ENODEV;
-	}
 
 	count = acpi_table_parse_entries(ACPI_SIG_PCCT,
 			sizeof(struct acpi_table_pcct),
diff --git a/drivers/mcb/mcb-core.c b/drivers/mcb/mcb-core.c
index 921a5d2a802b..bb5c5692dedc 100644
--- a/drivers/mcb/mcb-core.c
+++ b/drivers/mcb/mcb-core.c
@@ -418,6 +418,22 @@ void mcb_bus_add_devices(const struct mcb_bus *bus)
 EXPORT_SYMBOL_GPL(mcb_bus_add_devices);
 
 /**
+ * mcb_get_resource() - get a resource for a mcb device
+ * @dev: the mcb device
+ * @type: the type of resource
+ */
+struct resource *mcb_get_resource(struct mcb_device *dev, unsigned int type)
+{
+	if (type == IORESOURCE_MEM)
+		return &dev->mem;
+	else if (type == IORESOURCE_IRQ)
+		return &dev->irq;
+	else
+		return NULL;
+}
+EXPORT_SYMBOL_GPL(mcb_get_resource);
+
+/**
  * mcb_request_mem() - Request memory
  * @dev: The @mcb_device the memory is for
  * @name: The name for the memory reference.
@@ -460,7 +476,9 @@ EXPORT_SYMBOL_GPL(mcb_release_mem);
 
 static int __mcb_get_irq(struct mcb_device *dev)
 {
-	struct resource *irq = &dev->irq;
+	struct resource *irq;
+
+	irq = mcb_get_resource(dev, IORESOURCE_IRQ);
 
 	return irq->start;
 }
diff --git a/drivers/mcb/mcb-lpc.c b/drivers/mcb/mcb-lpc.c
index d072c088ce73..945091a88354 100644
--- a/drivers/mcb/mcb-lpc.c
+++ b/drivers/mcb/mcb-lpc.c
@@ -114,6 +114,12 @@ static struct resource sc24_fpga_resource = {
 	.flags = IORESOURCE_MEM,
 };
 
+static struct resource sc31_fpga_resource = {
+	.start = 0xf000e000,
+	.end = 0xf000e000 + CHAM_HEADER_SIZE,
+	.flags = IORESOURCE_MEM,
+};
+
 static struct platform_driver mcb_lpc_driver = {
 	.driver		= {
 		.name = "mcb-lpc",
@@ -132,6 +138,15 @@ static const struct dmi_system_id mcb_lpc_dmi_table[] = {
 		.driver_data = (void *)&sc24_fpga_resource,
 		.callback = mcb_lpc_create_platform_device,
 	},
+	{
+		.ident = "SC31",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "MEN"),
+			DMI_MATCH(DMI_PRODUCT_VERSION, "14SC31"),
+		},
+		.driver_data = (void *)&sc31_fpga_resource,
+		.callback = mcb_lpc_create_platform_device,
+	},
 	{}
 };
 MODULE_DEVICE_TABLE(dmi, mcb_lpc_dmi_table);
diff --git a/drivers/mcb/mcb-parse.c b/drivers/mcb/mcb-parse.c
index ee7fb6ec96bd..7369bda3442f 100644
--- a/drivers/mcb/mcb-parse.c
+++ b/drivers/mcb/mcb-parse.c
@@ -182,7 +182,7 @@ int chameleon_parse_cells(struct mcb_bus *bus, phys_addr_t mapbase,
 	int num_cells = 0;
 	uint32_t dtype;
 	int bar_count;
-	int ret = 0;
+	int ret;
 	u32 hsize;
 
 	hsize = sizeof(struct chameleon_fpga_header);
@@ -210,8 +210,10 @@ int chameleon_parse_cells(struct mcb_bus *bus, phys_addr_t mapbase,
 		 header->filename);
 
 	bar_count = chameleon_get_bar(&p, mapbase, &cb);
-	if (bar_count < 0)
+	if (bar_count < 0) {
+		ret = bar_count;
 		goto free_header;
+	}
 
 	for_each_chameleon_cell(dtype, p) {
 		switch (dtype) {
diff --git a/drivers/md/bcache/alloc.c b/drivers/md/bcache/alloc.c
index ca4abe1ccd8d..cacbe2dbd5c3 100644
--- a/drivers/md/bcache/alloc.c
+++ b/drivers/md/bcache/alloc.c
@@ -68,6 +68,8 @@
 #include <linux/random.h>
 #include <trace/events/bcache.h>
 
+#define MAX_OPEN_BUCKETS 128
+
 /* Bucket heap / gen */
 
 uint8_t bch_inc_gen(struct cache *ca, struct bucket *b)
@@ -671,7 +673,7 @@ int bch_open_buckets_alloc(struct cache_set *c)
 
 	spin_lock_init(&c->data_bucket_lock);
 
-	for (i = 0; i < 6; i++) {
+	for (i = 0; i < MAX_OPEN_BUCKETS; i++) {
 		struct open_bucket *b = kzalloc(sizeof(*b), GFP_KERNEL);
 		if (!b)
 			return -ENOMEM;
diff --git a/drivers/md/bcache/bcache.h b/drivers/md/bcache/bcache.h
index dee542fff68e..2ed9bd231d84 100644
--- a/drivers/md/bcache/bcache.h
+++ b/drivers/md/bcache/bcache.h
@@ -333,6 +333,7 @@ struct cached_dev {
 	/* Limit number of writeback bios in flight */
 	struct semaphore	in_flight;
 	struct task_struct	*writeback_thread;
+	struct workqueue_struct	*writeback_write_wq;
 
 	struct keybuf		writeback_keys;
 
diff --git a/drivers/md/bcache/closure.c b/drivers/md/bcache/closure.c
index 864e673aec39..7d5286b05036 100644
--- a/drivers/md/bcache/closure.c
+++ b/drivers/md/bcache/closure.c
@@ -70,21 +70,10 @@ void __closure_wake_up(struct closure_waitlist *wait_list)
 	list = llist_del_all(&wait_list->list);
 
 	/* We first reverse the list to preserve FIFO ordering and fairness */
-
-	while (list) {
-		struct llist_node *t = list;
-		list = llist_next(list);
-
-		t->next = reverse;
-		reverse = t;
-	}
+	reverse = llist_reverse_order(list);
 
 	/* Then do the wakeups */
-
-	while (reverse) {
-		cl = container_of(reverse, struct closure, list);
-		reverse = llist_next(reverse);
-
+	llist_for_each_entry(cl, reverse, list) {
 		closure_set_waiting(cl, 0);
 		closure_sub(cl, CLOSURE_WAITING + 1);
 	}
diff --git a/drivers/md/bcache/closure.h b/drivers/md/bcache/closure.h
index 1ec84ca81146..295b7e43f92c 100644
--- a/drivers/md/bcache/closure.h
+++ b/drivers/md/bcache/closure.h
@@ -312,8 +312,6 @@ static inline void closure_wake_up(struct closure_waitlist *list)
  * been dropped with closure_put()), it will resume execution at @fn running out
  * of @wq (or, if @wq is NULL, @fn will be called by closure_put() directly).
  *
- * NOTE: This macro expands to a return in the calling function!
- *
  * This is because after calling continue_at() you no longer have a ref on @cl,
  * and whatever @cl owns may be freed out from under you - a running closure fn
  * has a ref on its own closure which continue_at() drops.
@@ -340,8 +338,6 @@ do {									\
  * Causes @fn to be executed out of @cl, in @wq context (or called directly if
  * @wq is NULL).
  *
- * NOTE: like continue_at(), this macro expands to a return in the caller!
- *
  * The ref the caller of continue_at_nobarrier() had on @cl is now owned by @fn,
  * thus it's not safe to touch anything protected by @cl after a
  * continue_at_nobarrier().
diff --git a/drivers/md/bcache/debug.c b/drivers/md/bcache/debug.c
index 35a5a7210e51..61076eda2e6d 100644
--- a/drivers/md/bcache/debug.c
+++ b/drivers/md/bcache/debug.c
@@ -49,7 +49,7 @@ void bch_btree_verify(struct btree *b)
 	v->keys.ops = b->keys.ops;
 
 	bio = bch_bbio_alloc(b->c);
-	bio->bi_bdev		= PTR_CACHE(b->c, &b->key, 0)->bdev;
+	bio_set_dev(bio, PTR_CACHE(b->c, &b->key, 0)->bdev);
 	bio->bi_iter.bi_sector	= PTR_OFFSET(&b->key, 0);
 	bio->bi_iter.bi_size	= KEY_SIZE(&v->key) << 9;
 	bio->bi_opf		= REQ_OP_READ | REQ_META;
diff --git a/drivers/md/bcache/io.c b/drivers/md/bcache/io.c
index 6a9b85095e7b..7e871bdc0097 100644
--- a/drivers/md/bcache/io.c
+++ b/drivers/md/bcache/io.c
@@ -34,7 +34,7 @@ void __bch_submit_bbio(struct bio *bio, struct cache_set *c)
 	struct bbio *b = container_of(bio, struct bbio, bio);
 
 	bio->bi_iter.bi_sector	= PTR_OFFSET(&b->key, 0);
-	bio->bi_bdev		= PTR_CACHE(c, &b->key, 0)->bdev;
+	bio_set_dev(bio, PTR_CACHE(c, &b->key, 0)->bdev);
 
 	b->submit_time_us = local_clock_us();
 	closure_bio_submit(bio, bio->bi_private);
diff --git a/drivers/md/bcache/journal.c b/drivers/md/bcache/journal.c
index 0352d05e495c..7e1d1c3ba33a 100644
--- a/drivers/md/bcache/journal.c
+++ b/drivers/md/bcache/journal.c
@@ -53,7 +53,7 @@ reread:		left = ca->sb.bucket_size - offset;
 
 		bio_reset(bio);
 		bio->bi_iter.bi_sector	= bucket + offset;
-		bio->bi_bdev	= ca->bdev;
+		bio_set_dev(bio, ca->bdev);
 		bio->bi_iter.bi_size	= len << 9;
 
 		bio->bi_end_io	= journal_read_endio;
@@ -452,7 +452,7 @@ static void do_journal_discard(struct cache *ca)
 		bio_set_op_attrs(bio, REQ_OP_DISCARD, 0);
 		bio->bi_iter.bi_sector	= bucket_to_sector(ca->set,
 						ca->sb.d[ja->discard_idx]);
-		bio->bi_bdev		= ca->bdev;
+		bio_set_dev(bio, ca->bdev);
 		bio->bi_iter.bi_size	= bucket_bytes(ca);
 		bio->bi_end_io		= journal_discard_endio;
 
@@ -623,7 +623,7 @@ static void journal_write_unlocked(struct closure *cl)
 
 		bio_reset(bio);
 		bio->bi_iter.bi_sector	= PTR_OFFSET(k, i);
-		bio->bi_bdev	= ca->bdev;
+		bio_set_dev(bio, ca->bdev);
 		bio->bi_iter.bi_size = sectors << 9;
 
 		bio->bi_end_io	= journal_write_endio;
diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c
index 019b3df9f1c6..681b4f12b05a 100644
--- a/drivers/md/bcache/request.c
+++ b/drivers/md/bcache/request.c
@@ -196,12 +196,12 @@ static void bch_data_insert_start(struct closure *cl)
 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
 	struct bio *bio = op->bio, *n;
 
-	if (atomic_sub_return(bio_sectors(bio), &op->c->sectors_to_gc) < 0)
-		wake_up_gc(op->c);
-
 	if (op->bypass)
 		return bch_data_invalidate(cl);
 
+	if (atomic_sub_return(bio_sectors(bio), &op->c->sectors_to_gc) < 0)
+		wake_up_gc(op->c);
+
 	/*
 	 * Journal writes are marked REQ_PREFLUSH; if the original write was a
 	 * flush, it'll wait on the journal write.
@@ -400,12 +400,6 @@ static bool check_should_bypass(struct cached_dev *dc, struct bio *bio)
 	if (!congested && !dc->sequential_cutoff)
 		goto rescale;
 
-	if (!congested &&
-	    mode == CACHE_MODE_WRITEBACK &&
-	    op_is_write(bio->bi_opf) &&
-	    op_is_sync(bio->bi_opf))
-		goto rescale;
-
 	spin_lock(&dc->io_lock);
 
 	hlist_for_each_entry(i, iohash(dc, bio->bi_iter.bi_sector), hash)
@@ -607,7 +601,8 @@ static void request_endio(struct bio *bio)
 static void bio_complete(struct search *s)
 {
 	if (s->orig_bio) {
-		generic_end_io_acct(bio_data_dir(s->orig_bio),
+		struct request_queue *q = s->orig_bio->bi_disk->queue;
+		generic_end_io_acct(q, bio_data_dir(s->orig_bio),
 				    &s->d->disk->part0, s->start_time);
 
 		trace_bcache_request_end(s->d, s->orig_bio);
@@ -734,7 +729,7 @@ static void cached_dev_read_done(struct closure *cl)
 	if (s->iop.bio) {
 		bio_reset(s->iop.bio);
 		s->iop.bio->bi_iter.bi_sector = s->cache_miss->bi_iter.bi_sector;
-		s->iop.bio->bi_bdev = s->cache_miss->bi_bdev;
+		bio_copy_dev(s->iop.bio, s->cache_miss);
 		s->iop.bio->bi_iter.bi_size = s->insert_bio_sectors << 9;
 		bch_bio_map(s->iop.bio, NULL);
 
@@ -793,7 +788,7 @@ static int cached_dev_cache_miss(struct btree *b, struct search *s,
 	    !(bio->bi_opf & REQ_META) &&
 	    s->iop.c->gc_stats.in_use < CUTOFF_CACHE_READA)
 		reada = min_t(sector_t, dc->readahead >> 9,
-			      bdev_sectors(bio->bi_bdev) - bio_end_sector(bio));
+			      get_capacity(bio->bi_disk) - bio_end_sector(bio));
 
 	s->insert_bio_sectors = min(sectors, bio_sectors(bio) + reada);
 
@@ -819,7 +814,7 @@ static int cached_dev_cache_miss(struct btree *b, struct search *s,
 		goto out_submit;
 
 	cache_bio->bi_iter.bi_sector	= miss->bi_iter.bi_sector;
-	cache_bio->bi_bdev		= miss->bi_bdev;
+	bio_copy_dev(cache_bio, miss);
 	cache_bio->bi_iter.bi_size	= s->insert_bio_sectors << 9;
 
 	cache_bio->bi_end_io	= request_endio;
@@ -918,7 +913,7 @@ static void cached_dev_write(struct cached_dev *dc, struct search *s)
 			struct bio *flush = bio_alloc_bioset(GFP_NOIO, 0,
 							     dc->disk.bio_split);
 
-			flush->bi_bdev	= bio->bi_bdev;
+			bio_copy_dev(flush, bio);
 			flush->bi_end_io = request_endio;
 			flush->bi_private = cl;
 			flush->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
@@ -955,13 +950,13 @@ static blk_qc_t cached_dev_make_request(struct request_queue *q,
 					struct bio *bio)
 {
 	struct search *s;
-	struct bcache_device *d = bio->bi_bdev->bd_disk->private_data;
+	struct bcache_device *d = bio->bi_disk->private_data;
 	struct cached_dev *dc = container_of(d, struct cached_dev, disk);
 	int rw = bio_data_dir(bio);
 
-	generic_start_io_acct(rw, bio_sectors(bio), &d->disk->part0);
+	generic_start_io_acct(q, rw, bio_sectors(bio), &d->disk->part0);
 
-	bio->bi_bdev = dc->bdev;
+	bio_set_dev(bio, dc->bdev);
 	bio->bi_iter.bi_sector += dc->sb.data_offset;
 
 	if (cached_dev_get(dc)) {
@@ -1071,10 +1066,10 @@ static blk_qc_t flash_dev_make_request(struct request_queue *q,
 {
 	struct search *s;
 	struct closure *cl;
-	struct bcache_device *d = bio->bi_bdev->bd_disk->private_data;
+	struct bcache_device *d = bio->bi_disk->private_data;
 	int rw = bio_data_dir(bio);
 
-	generic_start_io_acct(rw, bio_sectors(bio), &d->disk->part0);
+	generic_start_io_acct(q, rw, bio_sectors(bio), &d->disk->part0);
 
 	s = search_alloc(bio, d);
 	cl = &s->cl;
diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c
index 8352fad765f6..fc0a31b13ac4 100644
--- a/drivers/md/bcache/super.c
+++ b/drivers/md/bcache/super.c
@@ -257,7 +257,7 @@ void bch_write_bdev_super(struct cached_dev *dc, struct closure *parent)
 	closure_init(cl, parent);
 
 	bio_reset(bio);
-	bio->bi_bdev	= dc->bdev;
+	bio_set_dev(bio, dc->bdev);
 	bio->bi_end_io	= write_bdev_super_endio;
 	bio->bi_private = dc;
 
@@ -303,7 +303,7 @@ void bcache_write_super(struct cache_set *c)
 		SET_CACHE_SYNC(&ca->sb, CACHE_SYNC(&c->sb));
 
 		bio_reset(bio);
-		bio->bi_bdev	= ca->bdev;
+		bio_set_dev(bio, ca->bdev);
 		bio->bi_end_io	= write_super_endio;
 		bio->bi_private = ca;
 
@@ -508,7 +508,7 @@ static void prio_io(struct cache *ca, uint64_t bucket, int op,
 	closure_init_stack(cl);
 
 	bio->bi_iter.bi_sector	= bucket * ca->sb.bucket_size;
-	bio->bi_bdev		= ca->bdev;
+	bio_set_dev(bio, ca->bdev);
 	bio->bi_iter.bi_size	= bucket_bytes(ca);
 
 	bio->bi_end_io	= prio_endio;
@@ -1026,7 +1026,7 @@ int bch_cached_dev_attach(struct cached_dev *dc, struct cache_set *c)
 	}
 
 	if (BDEV_STATE(&dc->sb) == BDEV_STATE_DIRTY) {
-		bch_sectors_dirty_init(dc);
+		bch_sectors_dirty_init(&dc->disk);
 		atomic_set(&dc->has_dirty, 1);
 		atomic_inc(&dc->count);
 		bch_writeback_queue(dc);
@@ -1059,6 +1059,8 @@ static void cached_dev_free(struct closure *cl)
 	cancel_delayed_work_sync(&dc->writeback_rate_update);
 	if (!IS_ERR_OR_NULL(dc->writeback_thread))
 		kthread_stop(dc->writeback_thread);
+	if (dc->writeback_write_wq)
+		destroy_workqueue(dc->writeback_write_wq);
 
 	mutex_lock(&bch_register_lock);
 
@@ -1228,6 +1230,7 @@ static int flash_dev_run(struct cache_set *c, struct uuid_entry *u)
 		goto err;
 
 	bcache_device_attach(d, c, u - c->uuids);
+	bch_sectors_dirty_init(d);
 	bch_flash_dev_request_init(d);
 	add_disk(d->disk);
 
@@ -1374,9 +1377,6 @@ static void cache_set_flush(struct closure *cl)
 	struct btree *b;
 	unsigned i;
 
-	if (!c)
-		closure_return(cl);
-
 	bch_cache_accounting_destroy(&c->accounting);
 
 	kobject_put(&c->internal);
@@ -1964,6 +1964,8 @@ static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr,
 			else
 				err = "device busy";
 			mutex_unlock(&bch_register_lock);
+			if (!IS_ERR(bdev))
+				bdput(bdev);
 			if (attr == &ksysfs_register_quiet)
 				goto out;
 		}
diff --git a/drivers/md/bcache/sysfs.c b/drivers/md/bcache/sysfs.c
index f90f13616980..104c57cd666c 100644
--- a/drivers/md/bcache/sysfs.c
+++ b/drivers/md/bcache/sysfs.c
@@ -192,7 +192,7 @@ STORE(__cached_dev)
 {
 	struct cached_dev *dc = container_of(kobj, struct cached_dev,
 					     disk.kobj);
-	unsigned v = size;
+	ssize_t v = size;
 	struct cache_set *c;
 	struct kobj_uevent_env *env;
 
@@ -227,7 +227,7 @@ STORE(__cached_dev)
 		bch_cached_dev_run(dc);
 
 	if (attr == &sysfs_cache_mode) {
-		ssize_t v = bch_read_string_list(buf, bch_cache_modes + 1);
+		v = bch_read_string_list(buf, bch_cache_modes + 1);
 
 		if (v < 0)
 			return v;
@@ -615,8 +615,21 @@ STORE(__bch_cache_set)
 		bch_cache_accounting_clear(&c->accounting);
 	}
 
-	if (attr == &sysfs_trigger_gc)
+	if (attr == &sysfs_trigger_gc) {
+		/*
+		 * Garbage collection thread only works when sectors_to_gc < 0,
+		 * when users write to sysfs entry trigger_gc, most of time
+		 * they want to forcibly triger gargage collection. Here -1 is
+		 * set to c->sectors_to_gc, to make gc_should_run() give a
+		 * chance to permit gc thread to run. "give a chance" means
+		 * before going into gc_should_run(), there is still chance
+		 * that c->sectors_to_gc being set to other positive value. So
+		 * writing sysfs entry trigger_gc won't always make sure gc
+		 * thread takes effect.
+		 */
+		atomic_set(&c->sectors_to_gc, -1);
 		wake_up_gc(c);
+	}
 
 	if (attr == &sysfs_prune_cache) {
 		struct shrink_control sc;
diff --git a/drivers/md/bcache/util.c b/drivers/md/bcache/util.c
index 8c3a938f4bf0..176d3c2ef5f5 100644
--- a/drivers/md/bcache/util.c
+++ b/drivers/md/bcache/util.c
@@ -74,24 +74,44 @@ STRTO_H(strtouint, unsigned int)
 STRTO_H(strtoll, long long)
 STRTO_H(strtoull, unsigned long long)
 
+/**
+ * bch_hprint() - formats @v to human readable string for sysfs.
+ *
+ * @v - signed 64 bit integer
+ * @buf - the (at least 8 byte) buffer to format the result into.
+ *
+ * Returns the number of bytes used by format.
+ */
 ssize_t bch_hprint(char *buf, int64_t v)
 {
 	static const char units[] = "?kMGTPEZY";
-	char dec[4] = "";
-	int u, t = 0;
-
-	for (u = 0; v >= 1024 || v <= -1024; u++) {
-		t = v & ~(~0 << 10);
-		v >>= 10;
-	}
-
-	if (!u)
-		return sprintf(buf, "%llu", v);
-
-	if (v < 100 && v > -100)
-		snprintf(dec, sizeof(dec), ".%i", t / 100);
-
-	return sprintf(buf, "%lli%s%c", v, dec, units[u]);
+	int u = 0, t;
+
+	uint64_t q;
+
+	if (v < 0)
+		q = -v;
+	else
+		q = v;
+
+	/* For as long as the number is more than 3 digits, but at least
+	 * once, shift right / divide by 1024.  Keep the remainder for
+	 * a digit after the decimal point.
+	 */
+	do {
+		u++;
+
+		t = q & ~(~0 << 10);
+		q >>= 10;
+	} while (q >= 1000);
+
+	if (v < 0)
+		/* '-', up to 3 digits, '.', 1 digit, 1 character, null;
+		 * yields 8 bytes.
+		 */
+		return sprintf(buf, "-%llu.%i%c", q, t * 10 / 1024, units[u]);
+	else
+		return sprintf(buf, "%llu.%i%c", q, t * 10 / 1024, units[u]);
 }
 
 ssize_t bch_snprint_string_list(char *buf, size_t size, const char * const list[],
diff --git a/drivers/md/bcache/writeback.c b/drivers/md/bcache/writeback.c
index 42c66e76f05e..e663ca082183 100644
--- a/drivers/md/bcache/writeback.c
+++ b/drivers/md/bcache/writeback.c
@@ -21,7 +21,8 @@
 static void __update_writeback_rate(struct cached_dev *dc)
 {
 	struct cache_set *c = dc->disk.c;
-	uint64_t cache_sectors = c->nbuckets * c->sb.bucket_size;
+	uint64_t cache_sectors = c->nbuckets * c->sb.bucket_size -
+				bcache_flash_devs_sectors_dirty(c);
 	uint64_t cache_dirty_target =
 		div_u64(cache_sectors * dc->writeback_percent, 100);
 
@@ -181,12 +182,12 @@ static void write_dirty(struct closure *cl)
 	dirty_init(w);
 	bio_set_op_attrs(&io->bio, REQ_OP_WRITE, 0);
 	io->bio.bi_iter.bi_sector = KEY_START(&w->key);
-	io->bio.bi_bdev		= io->dc->bdev;
+	bio_set_dev(&io->bio, io->dc->bdev);
 	io->bio.bi_end_io	= dirty_endio;
 
 	closure_bio_submit(&io->bio, cl);
 
-	continue_at(cl, write_dirty_finish, system_wq);
+	continue_at(cl, write_dirty_finish, io->dc->writeback_write_wq);
 }
 
 static void read_dirty_endio(struct bio *bio)
@@ -206,7 +207,7 @@ static void read_dirty_submit(struct closure *cl)
 
 	closure_bio_submit(&io->bio, cl);
 
-	continue_at(cl, write_dirty, system_wq);
+	continue_at(cl, write_dirty, io->dc->writeback_write_wq);
 }
 
 static void read_dirty(struct cached_dev *dc)
@@ -250,8 +251,7 @@ static void read_dirty(struct cached_dev *dc)
 		dirty_init(w);
 		bio_set_op_attrs(&io->bio, REQ_OP_READ, 0);
 		io->bio.bi_iter.bi_sector = PTR_OFFSET(&w->key, 0);
-		io->bio.bi_bdev		= PTR_CACHE(dc->disk.c,
-						    &w->key, 0)->bdev;
+		bio_set_dev(&io->bio, PTR_CACHE(dc->disk.c, &w->key, 0)->bdev);
 		io->bio.bi_end_io	= read_dirty_endio;
 
 		if (bio_alloc_pages(&io->bio, GFP_KERNEL))
@@ -482,17 +482,17 @@ static int sectors_dirty_init_fn(struct btree_op *_op, struct btree *b,
 	return MAP_CONTINUE;
 }
 
-void bch_sectors_dirty_init(struct cached_dev *dc)
+void bch_sectors_dirty_init(struct bcache_device *d)
 {
 	struct sectors_dirty_init op;
 
 	bch_btree_op_init(&op.op, -1);
-	op.inode = dc->disk.id;
+	op.inode = d->id;
 
-	bch_btree_map_keys(&op.op, dc->disk.c, &KEY(op.inode, 0, 0),
+	bch_btree_map_keys(&op.op, d->c, &KEY(op.inode, 0, 0),
 			   sectors_dirty_init_fn, 0);
 
-	dc->disk.sectors_dirty_last = bcache_dev_sectors_dirty(&dc->disk);
+	d->sectors_dirty_last = bcache_dev_sectors_dirty(d);
 }
 
 void bch_cached_dev_writeback_init(struct cached_dev *dc)
@@ -516,6 +516,11 @@ void bch_cached_dev_writeback_init(struct cached_dev *dc)
 
 int bch_cached_dev_writeback_start(struct cached_dev *dc)
 {
+	dc->writeback_write_wq = alloc_workqueue("bcache_writeback_wq",
+						WQ_MEM_RECLAIM, 0);
+	if (!dc->writeback_write_wq)
+		return -ENOMEM;
+
 	dc->writeback_thread = kthread_create(bch_writeback_thread, dc,
 					      "bcache_writeback");
 	if (IS_ERR(dc->writeback_thread))
diff --git a/drivers/md/bcache/writeback.h b/drivers/md/bcache/writeback.h
index 629bd1a502fd..e35421d20d2e 100644
--- a/drivers/md/bcache/writeback.h
+++ b/drivers/md/bcache/writeback.h
@@ -14,6 +14,25 @@ static inline uint64_t bcache_dev_sectors_dirty(struct bcache_device *d)
 	return ret;
 }
 
+static inline uint64_t  bcache_flash_devs_sectors_dirty(struct cache_set *c)
+{
+	uint64_t i, ret = 0;
+
+	mutex_lock(&bch_register_lock);
+
+	for (i = 0; i < c->nr_uuids; i++) {
+		struct bcache_device *d = c->devices[i];
+
+		if (!d || !UUID_FLASH_ONLY(&c->uuids[i]))
+			continue;
+	   ret += bcache_dev_sectors_dirty(d);
+	}
+
+	mutex_unlock(&bch_register_lock);
+
+	return ret;
+}
+
 static inline unsigned offset_to_stripe(struct bcache_device *d,
 					uint64_t offset)
 {
@@ -84,7 +103,7 @@ static inline void bch_writeback_add(struct cached_dev *dc)
 
 void bcache_dev_sectors_dirty_add(struct cache_set *, unsigned, uint64_t, int);
 
-void bch_sectors_dirty_init(struct cached_dev *dc);
+void bch_sectors_dirty_init(struct bcache_device *);
 void bch_cached_dev_writeback_init(struct cached_dev *);
 int bch_cached_dev_writeback_start(struct cached_dev *);
 
diff --git a/drivers/md/bitmap.c b/drivers/md/bitmap.c
index 40f3cd7eab0f..d2121637b4ab 100644
--- a/drivers/md/bitmap.c
+++ b/drivers/md/bitmap.c
@@ -625,7 +625,7 @@ re_read:
 		err = read_sb_page(bitmap->mddev,
 				   offset,
 				   sb_page,
-				   0, sizeof(bitmap_super_t));
+				   0, PAGE_SIZE);
 	}
 	if (err)
 		return err;
@@ -2058,6 +2058,11 @@ int bitmap_resize(struct bitmap *bitmap, sector_t blocks,
 	long pages;
 	struct bitmap_page *new_bp;
 
+	if (bitmap->storage.file && !init) {
+		pr_info("md: cannot resize file-based bitmap\n");
+		return -EINVAL;
+	}
+
 	if (chunksize == 0) {
 		/* If there is enough space, leave the chunk size unchanged,
 		 * else increase by factor of two until there is enough space.
@@ -2118,7 +2123,7 @@ int bitmap_resize(struct bitmap *bitmap, sector_t blocks,
 	if (store.sb_page && bitmap->storage.sb_page)
 		memcpy(page_address(store.sb_page),
 		       page_address(bitmap->storage.sb_page),
-		       sizeof(bitmap_super_t));
+		       PAGE_SIZE);
 	bitmap_file_unmap(&bitmap->storage);
 	bitmap->storage = store;
 
diff --git a/drivers/md/dm-bio-record.h b/drivers/md/dm-bio-record.h
index dd3646111561..c82578af56a5 100644
--- a/drivers/md/dm-bio-record.h
+++ b/drivers/md/dm-bio-record.h
@@ -18,21 +18,24 @@
  */
 
 struct dm_bio_details {
-	struct block_device *bi_bdev;
+	struct gendisk *bi_disk;
+	u8 bi_partno;
 	unsigned long bi_flags;
 	struct bvec_iter bi_iter;
 };
 
 static inline void dm_bio_record(struct dm_bio_details *bd, struct bio *bio)
 {
-	bd->bi_bdev = bio->bi_bdev;
+	bd->bi_disk = bio->bi_disk;
+	bd->bi_partno = bio->bi_partno;
 	bd->bi_flags = bio->bi_flags;
 	bd->bi_iter = bio->bi_iter;
 }
 
 static inline void dm_bio_restore(struct dm_bio_details *bd, struct bio *bio)
 {
-	bio->bi_bdev = bd->bi_bdev;
+	bio->bi_disk = bd->bi_disk;
+	bio->bi_partno = bd->bi_partno;
 	bio->bi_flags = bd->bi_flags;
 	bio->bi_iter = bd->bi_iter;
 }
diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c
index 44f4a8ac95bd..d216a8f7bc22 100644
--- a/drivers/md/dm-bufio.c
+++ b/drivers/md/dm-bufio.c
@@ -64,6 +64,12 @@
 #define DM_BUFIO_BLOCK_SIZE_GFP_LIMIT	(PAGE_SIZE << (MAX_ORDER - 1))
 
 /*
+ * Align buffer writes to this boundary.
+ * Tests show that SSDs have the highest IOPS when using 4k writes.
+ */
+#define DM_BUFIO_WRITE_ALIGN		4096
+
+/*
  * dm_buffer->list_mode
  */
 #define LIST_CLEAN	0
@@ -149,6 +155,10 @@ struct dm_buffer {
 	blk_status_t write_error;
 	unsigned long state;
 	unsigned long last_accessed;
+	unsigned dirty_start;
+	unsigned dirty_end;
+	unsigned write_start;
+	unsigned write_end;
 	struct dm_bufio_client *c;
 	struct list_head write_list;
 	struct bio bio;
@@ -560,7 +570,7 @@ static void dmio_complete(unsigned long error, void *context)
 }
 
 static void use_dmio(struct dm_buffer *b, int rw, sector_t sector,
-		     unsigned n_sectors, bio_end_io_t *end_io)
+		     unsigned n_sectors, unsigned offset, bio_end_io_t *end_io)
 {
 	int r;
 	struct dm_io_request io_req = {
@@ -578,10 +588,10 @@ static void use_dmio(struct dm_buffer *b, int rw, sector_t sector,
 
 	if (b->data_mode != DATA_MODE_VMALLOC) {
 		io_req.mem.type = DM_IO_KMEM;
-		io_req.mem.ptr.addr = b->data;
+		io_req.mem.ptr.addr = (char *)b->data + offset;
 	} else {
 		io_req.mem.type = DM_IO_VMA;
-		io_req.mem.ptr.vma = b->data;
+		io_req.mem.ptr.vma = (char *)b->data + offset;
 	}
 
 	b->bio.bi_end_io = end_io;
@@ -609,14 +619,14 @@ static void inline_endio(struct bio *bio)
 }
 
 static void use_inline_bio(struct dm_buffer *b, int rw, sector_t sector,
-			   unsigned n_sectors, bio_end_io_t *end_io)
+			   unsigned n_sectors, unsigned offset, bio_end_io_t *end_io)
 {
 	char *ptr;
-	int len;
+	unsigned len;
 
 	bio_init(&b->bio, b->bio_vec, DM_BUFIO_INLINE_VECS);
 	b->bio.bi_iter.bi_sector = sector;
-	b->bio.bi_bdev = b->c->bdev;
+	bio_set_dev(&b->bio, b->c->bdev);
 	b->bio.bi_end_io = inline_endio;
 	/*
 	 * Use of .bi_private isn't a problem here because
@@ -625,29 +635,20 @@ static void use_inline_bio(struct dm_buffer *b, int rw, sector_t sector,
 	b->bio.bi_private = end_io;
 	bio_set_op_attrs(&b->bio, rw, 0);
 
-	/*
-	 * We assume that if len >= PAGE_SIZE ptr is page-aligned.
-	 * If len < PAGE_SIZE the buffer doesn't cross page boundary.
-	 */
-	ptr = b->data;
+	ptr = (char *)b->data + offset;
 	len = n_sectors << SECTOR_SHIFT;
 
-	if (len >= PAGE_SIZE)
-		BUG_ON((unsigned long)ptr & (PAGE_SIZE - 1));
-	else
-		BUG_ON((unsigned long)ptr & (len - 1));
-
 	do {
-		if (!bio_add_page(&b->bio, virt_to_page(ptr),
-				  len < PAGE_SIZE ? len : PAGE_SIZE,
+		unsigned this_step = min((unsigned)(PAGE_SIZE - offset_in_page(ptr)), len);
+		if (!bio_add_page(&b->bio, virt_to_page(ptr), this_step,
 				  offset_in_page(ptr))) {
 			BUG_ON(b->c->block_size <= PAGE_SIZE);
-			use_dmio(b, rw, sector, n_sectors, end_io);
+			use_dmio(b, rw, sector, n_sectors, offset, end_io);
 			return;
 		}
 
-		len -= PAGE_SIZE;
-		ptr += PAGE_SIZE;
+		len -= this_step;
+		ptr += this_step;
 	} while (len > 0);
 
 	submit_bio(&b->bio);
@@ -657,18 +658,33 @@ static void submit_io(struct dm_buffer *b, int rw, bio_end_io_t *end_io)
 {
 	unsigned n_sectors;
 	sector_t sector;
-
-	if (rw == WRITE && b->c->write_callback)
-		b->c->write_callback(b);
+	unsigned offset, end;
 
 	sector = (b->block << b->c->sectors_per_block_bits) + b->c->start;
-	n_sectors = 1 << b->c->sectors_per_block_bits;
+
+	if (rw != WRITE) {
+		n_sectors = 1 << b->c->sectors_per_block_bits;
+		offset = 0;
+	} else {
+		if (b->c->write_callback)
+			b->c->write_callback(b);
+		offset = b->write_start;
+		end = b->write_end;
+		offset &= -DM_BUFIO_WRITE_ALIGN;
+		end += DM_BUFIO_WRITE_ALIGN - 1;
+		end &= -DM_BUFIO_WRITE_ALIGN;
+		if (unlikely(end > b->c->block_size))
+			end = b->c->block_size;
+
+		sector += offset >> SECTOR_SHIFT;
+		n_sectors = (end - offset) >> SECTOR_SHIFT;
+	}
 
 	if (n_sectors <= ((DM_BUFIO_INLINE_VECS * PAGE_SIZE) >> SECTOR_SHIFT) &&
 	    b->data_mode != DATA_MODE_VMALLOC)
-		use_inline_bio(b, rw, sector, n_sectors, end_io);
+		use_inline_bio(b, rw, sector, n_sectors, offset, end_io);
 	else
-		use_dmio(b, rw, sector, n_sectors, end_io);
+		use_dmio(b, rw, sector, n_sectors, offset, end_io);
 }
 
 /*----------------------------------------------------------------
@@ -720,6 +736,9 @@ static void __write_dirty_buffer(struct dm_buffer *b,
 	clear_bit(B_DIRTY, &b->state);
 	wait_on_bit_lock_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE);
 
+	b->write_start = b->dirty_start;
+	b->write_end = b->dirty_end;
+
 	if (!write_list)
 		submit_io(b, WRITE, write_endio);
 	else
@@ -1221,19 +1240,37 @@ void dm_bufio_release(struct dm_buffer *b)
 }
 EXPORT_SYMBOL_GPL(dm_bufio_release);
 
-void dm_bufio_mark_buffer_dirty(struct dm_buffer *b)
+void dm_bufio_mark_partial_buffer_dirty(struct dm_buffer *b,
+					unsigned start, unsigned end)
 {
 	struct dm_bufio_client *c = b->c;
 
+	BUG_ON(start >= end);
+	BUG_ON(end > b->c->block_size);
+
 	dm_bufio_lock(c);
 
 	BUG_ON(test_bit(B_READING, &b->state));
 
-	if (!test_and_set_bit(B_DIRTY, &b->state))
+	if (!test_and_set_bit(B_DIRTY, &b->state)) {
+		b->dirty_start = start;
+		b->dirty_end = end;
 		__relink_lru(b, LIST_DIRTY);
+	} else {
+		if (start < b->dirty_start)
+			b->dirty_start = start;
+		if (end > b->dirty_end)
+			b->dirty_end = end;
+	}
 
 	dm_bufio_unlock(c);
 }
+EXPORT_SYMBOL_GPL(dm_bufio_mark_partial_buffer_dirty);
+
+void dm_bufio_mark_buffer_dirty(struct dm_buffer *b)
+{
+	dm_bufio_mark_partial_buffer_dirty(b, 0, b->c->block_size);
+}
 EXPORT_SYMBOL_GPL(dm_bufio_mark_buffer_dirty);
 
 void dm_bufio_write_dirty_buffers_async(struct dm_bufio_client *c)
@@ -1398,6 +1435,8 @@ retry:
 		wait_on_bit_io(&b->state, B_WRITING,
 			       TASK_UNINTERRUPTIBLE);
 		set_bit(B_DIRTY, &b->state);
+		b->dirty_start = 0;
+		b->dirty_end = c->block_size;
 		__unlink_buffer(b);
 		__link_buffer(b, new_block, LIST_DIRTY);
 	} else {
diff --git a/drivers/md/dm-bufio.h b/drivers/md/dm-bufio.h
index b6d8f53ec15b..be732d3f8611 100644
--- a/drivers/md/dm-bufio.h
+++ b/drivers/md/dm-bufio.h
@@ -94,6 +94,15 @@ void dm_bufio_release(struct dm_buffer *b);
 void dm_bufio_mark_buffer_dirty(struct dm_buffer *b);
 
 /*
+ * Mark a part of the buffer dirty.
+ *
+ * The specified part of the buffer is scheduled to be written. dm-bufio may
+ * write the specified part of the buffer or it may write a larger superset.
+ */
+void dm_bufio_mark_partial_buffer_dirty(struct dm_buffer *b,
+					unsigned start, unsigned end);
+
+/*
  * Initiate writing of dirty buffers, without waiting for completion.
  */
 void dm_bufio_write_dirty_buffers_async(struct dm_bufio_client *c);
diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c
index c5ea03fc7ee1..8785134c9f1f 100644
--- a/drivers/md/dm-cache-target.c
+++ b/drivers/md/dm-cache-target.c
@@ -833,7 +833,7 @@ static bool is_discarded_oblock(struct cache *cache, dm_oblock_t b)
  *--------------------------------------------------------------*/
 static void remap_to_origin(struct cache *cache, struct bio *bio)
 {
-	bio->bi_bdev = cache->origin_dev->bdev;
+	bio_set_dev(bio, cache->origin_dev->bdev);
 }
 
 static void remap_to_cache(struct cache *cache, struct bio *bio,
@@ -842,7 +842,7 @@ static void remap_to_cache(struct cache *cache, struct bio *bio,
 	sector_t bi_sector = bio->bi_iter.bi_sector;
 	sector_t block = from_cblock(cblock);
 
-	bio->bi_bdev = cache->cache_dev->bdev;
+	bio_set_dev(bio, cache->cache_dev->bdev);
 	if (!block_size_is_power_of_two(cache))
 		bio->bi_iter.bi_sector =
 			(block * cache->sectors_per_block) +
@@ -2306,7 +2306,7 @@ static void init_features(struct cache_features *cf)
 static int parse_features(struct cache_args *ca, struct dm_arg_set *as,
 			  char **error)
 {
-	static struct dm_arg _args[] = {
+	static const struct dm_arg _args[] = {
 		{0, 2, "Invalid number of cache feature arguments"},
 	};
 
@@ -2348,7 +2348,7 @@ static int parse_features(struct cache_args *ca, struct dm_arg_set *as,
 static int parse_policy(struct cache_args *ca, struct dm_arg_set *as,
 			char **error)
 {
-	static struct dm_arg _args[] = {
+	static const struct dm_arg _args[] = {
 		{0, 1024, "Invalid number of policy arguments"},
 	};
 
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index cdf6b1e12460..a55ffd4f5933 100644
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -758,9 +758,8 @@ static int crypt_iv_tcw_whitening(struct crypt_config *cc,
 	int i, r;
 
 	/* xor whitening with sector number */
-	memcpy(buf, tcw->whitening, TCW_WHITENING_SIZE);
-	crypto_xor(buf, (u8 *)&sector, 8);
-	crypto_xor(&buf[8], (u8 *)&sector, 8);
+	crypto_xor_cpy(buf, tcw->whitening, (u8 *)&sector, 8);
+	crypto_xor_cpy(&buf[8], tcw->whitening + 8, (u8 *)&sector, 8);
 
 	/* calculate crc32 for every 32bit part and xor it */
 	desc->tfm = tcw->crc32_tfm;
@@ -805,10 +804,10 @@ static int crypt_iv_tcw_gen(struct crypt_config *cc, u8 *iv,
 	}
 
 	/* Calculate IV */
-	memcpy(iv, tcw->iv_seed, cc->iv_size);
-	crypto_xor(iv, (u8 *)&sector, 8);
+	crypto_xor_cpy(iv, tcw->iv_seed, (u8 *)&sector, 8);
 	if (cc->iv_size > 8)
-		crypto_xor(&iv[8], (u8 *)&sector, cc->iv_size - 8);
+		crypto_xor_cpy(&iv[8], tcw->iv_seed + 8, (u8 *)&sector,
+			       cc->iv_size - 8);
 
 	return r;
 }
@@ -933,9 +932,6 @@ static int dm_crypt_integrity_io_alloc(struct dm_crypt_io *io, struct bio *bio)
 	bip->bip_iter.bi_size = tag_len;
 	bip->bip_iter.bi_sector = io->cc->start + io->sector;
 
-	/* We own the metadata, do not let bio_free to release it */
-	bip->bip_flags &= ~BIP_BLOCK_INTEGRITY;
-
 	ret = bio_integrity_add_page(bio, virt_to_page(io->integrity_metadata),
 				     tag_len, offset_in_page(io->integrity_metadata));
 	if (unlikely(ret != tag_len))
@@ -1547,7 +1543,7 @@ static void clone_init(struct dm_crypt_io *io, struct bio *clone)
 
 	clone->bi_private = io;
 	clone->bi_end_io  = crypt_endio;
-	clone->bi_bdev    = cc->dev->bdev;
+	bio_set_dev(clone, cc->dev->bdev);
 	clone->bi_opf	  = io->base_bio->bi_opf;
 }
 
@@ -2533,7 +2529,7 @@ static int crypt_ctr_optional(struct dm_target *ti, unsigned int argc, char **ar
 {
 	struct crypt_config *cc = ti->private;
 	struct dm_arg_set as;
-	static struct dm_arg _args[] = {
+	static const struct dm_arg _args[] = {
 		{0, 6, "Invalid number of feature args"},
 	};
 	unsigned int opt_params, val;
@@ -2796,7 +2792,7 @@ static int crypt_map(struct dm_target *ti, struct bio *bio)
 	 */
 	if (unlikely(bio->bi_opf & REQ_PREFLUSH ||
 	    bio_op(bio) == REQ_OP_DISCARD)) {
-		bio->bi_bdev = cc->dev->bdev;
+		bio_set_dev(bio, cc->dev->bdev);
 		if (bio_sectors(bio))
 			bio->bi_iter.bi_sector = cc->start +
 				dm_target_offset(ti, bio->bi_iter.bi_sector);
diff --git a/drivers/md/dm-delay.c b/drivers/md/dm-delay.c
index ae3158795d26..2209a9700acd 100644
--- a/drivers/md/dm-delay.c
+++ b/drivers/md/dm-delay.c
@@ -282,7 +282,7 @@ static int delay_map(struct dm_target *ti, struct bio *bio)
 	struct delay_c *dc = ti->private;
 
 	if ((bio_data_dir(bio) == WRITE) && (dc->dev_write)) {
-		bio->bi_bdev = dc->dev_write->bdev;
+		bio_set_dev(bio, dc->dev_write->bdev);
 		if (bio_sectors(bio))
 			bio->bi_iter.bi_sector = dc->start_write +
 				dm_target_offset(ti, bio->bi_iter.bi_sector);
@@ -290,7 +290,7 @@ static int delay_map(struct dm_target *ti, struct bio *bio)
 		return delay_bio(dc, dc->write_delay, bio);
 	}
 
-	bio->bi_bdev = dc->dev_read->bdev;
+	bio_set_dev(bio, dc->dev_read->bdev);
 	bio->bi_iter.bi_sector = dc->start_read +
 		dm_target_offset(ti, bio->bi_iter.bi_sector);
 
diff --git a/drivers/md/dm-era-target.c b/drivers/md/dm-era-target.c
index e7ba89f98d8d..ba84b8d62cd0 100644
--- a/drivers/md/dm-era-target.c
+++ b/drivers/md/dm-era-target.c
@@ -1192,7 +1192,7 @@ static dm_block_t get_block(struct era *era, struct bio *bio)
 
 static void remap_to_origin(struct era *era, struct bio *bio)
 {
-	bio->bi_bdev = era->origin_dev->bdev;
+	bio_set_dev(bio, era->origin_dev->bdev);
 }
 
 /*----------------------------------------------------------------
diff --git a/drivers/md/dm-flakey.c b/drivers/md/dm-flakey.c
index e2c7234931bc..b82cb1ab1eaa 100644
--- a/drivers/md/dm-flakey.c
+++ b/drivers/md/dm-flakey.c
@@ -51,7 +51,7 @@ static int parse_features(struct dm_arg_set *as, struct flakey_c *fc,
 	unsigned argc;
 	const char *arg_name;
 
-	static struct dm_arg _args[] = {
+	static const struct dm_arg _args[] = {
 		{0, 6, "Invalid number of feature args"},
 		{1, UINT_MAX, "Invalid corrupt bio byte"},
 		{0, 255, "Invalid corrupt value to write into bio byte (0-255)"},
@@ -178,7 +178,7 @@ static int parse_features(struct dm_arg_set *as, struct flakey_c *fc,
  */
 static int flakey_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 {
-	static struct dm_arg _args[] = {
+	static const struct dm_arg _args[] = {
 		{0, UINT_MAX, "Invalid up interval"},
 		{0, UINT_MAX, "Invalid down interval"},
 	};
@@ -274,7 +274,7 @@ static void flakey_map_bio(struct dm_target *ti, struct bio *bio)
 {
 	struct flakey_c *fc = ti->private;
 
-	bio->bi_bdev = fc->dev->bdev;
+	bio_set_dev(bio, fc->dev->bdev);
 	if (bio_sectors(bio) || bio_op(bio) == REQ_OP_ZONE_RESET)
 		bio->bi_iter.bi_sector =
 			flakey_map_sector(ti, bio->bi_iter.bi_sector);
diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c
index 3acce09bba35..096fe9b66c50 100644
--- a/drivers/md/dm-integrity.c
+++ b/drivers/md/dm-integrity.c
@@ -225,6 +225,8 @@ struct dm_integrity_c {
 	struct alg_spec internal_hash_alg;
 	struct alg_spec journal_crypt_alg;
 	struct alg_spec journal_mac_alg;
+
+	atomic64_t number_of_mismatches;
 };
 
 struct dm_integrity_range {
@@ -250,7 +252,8 @@ struct dm_integrity_io {
 
 	struct completion *completion;
 
-	struct block_device *orig_bi_bdev;
+	struct gendisk *orig_bi_disk;
+	u8 orig_bi_partno;
 	bio_end_io_t *orig_bi_end_io;
 	struct bio_integrity_payload *orig_bi_integrity;
 	struct bvec_iter orig_bi_iter;
@@ -297,7 +300,7 @@ static void __DEBUG_bytes(__u8 *bytes, size_t len, const char *msg, ...)
 /*
  * DM Integrity profile, protection is performed layer above (dm-crypt)
  */
-static struct blk_integrity_profile dm_integrity_profile = {
+static const struct blk_integrity_profile dm_integrity_profile = {
 	.name			= "DM-DIF-EXT-TAG",
 	.generate_fn		= NULL,
 	.verify_fn		= NULL,
@@ -309,6 +312,8 @@ static void dm_integrity_dtr(struct dm_target *ti);
 
 static void dm_integrity_io_error(struct dm_integrity_c *ic, const char *msg, int err)
 {
+	if (err == -EILSEQ)
+		atomic64_inc(&ic->number_of_mismatches);
 	if (!cmpxchg(&ic->failed, 0, err))
 		DMERR("Error on %s: %d", msg, err);
 }
@@ -769,13 +774,13 @@ static void write_journal(struct dm_integrity_c *ic, unsigned commit_start, unsi
 	unsigned i;
 
 	io_comp.ic = ic;
-	io_comp.comp = COMPLETION_INITIALIZER_ONSTACK(io_comp.comp);
+	init_completion(&io_comp.comp);
 
 	if (commit_start + commit_sections <= ic->journal_sections) {
 		io_comp.in_flight = (atomic_t)ATOMIC_INIT(1);
 		if (ic->journal_io) {
 			crypt_comp_1.ic = ic;
-			crypt_comp_1.comp = COMPLETION_INITIALIZER_ONSTACK(crypt_comp_1.comp);
+			init_completion(&crypt_comp_1.comp);
 			crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
 			encrypt_journal(ic, true, commit_start, commit_sections, &crypt_comp_1);
 			wait_for_completion_io(&crypt_comp_1.comp);
@@ -791,18 +796,18 @@ static void write_journal(struct dm_integrity_c *ic, unsigned commit_start, unsi
 		to_end = ic->journal_sections - commit_start;
 		if (ic->journal_io) {
 			crypt_comp_1.ic = ic;
-			crypt_comp_1.comp = COMPLETION_INITIALIZER_ONSTACK(crypt_comp_1.comp);
+			init_completion(&crypt_comp_1.comp);
 			crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
 			encrypt_journal(ic, true, commit_start, to_end, &crypt_comp_1);
 			if (try_wait_for_completion(&crypt_comp_1.comp)) {
 				rw_journal(ic, REQ_OP_WRITE, REQ_FUA, commit_start, to_end, &io_comp);
-				crypt_comp_1.comp = COMPLETION_INITIALIZER_ONSTACK(crypt_comp_1.comp);
+				reinit_completion(&crypt_comp_1.comp);
 				crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
 				encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_1);
 				wait_for_completion_io(&crypt_comp_1.comp);
 			} else {
 				crypt_comp_2.ic = ic;
-				crypt_comp_2.comp = COMPLETION_INITIALIZER_ONSTACK(crypt_comp_2.comp);
+				init_completion(&crypt_comp_2.comp);
 				crypt_comp_2.in_flight = (atomic_t)ATOMIC_INIT(0);
 				encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_2);
 				wait_for_completion_io(&crypt_comp_1.comp);
@@ -1040,7 +1045,7 @@ static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, se
 			memcpy(tag, dp, to_copy);
 		} else if (op == TAG_WRITE) {
 			memcpy(dp, tag, to_copy);
-			dm_bufio_mark_buffer_dirty(b);
+			dm_bufio_mark_partial_buffer_dirty(b, *metadata_offset, *metadata_offset + to_copy);
 		} else  {
 			/* e.g.: op == TAG_CMP */
 			if (unlikely(memcmp(dp, tag, to_copy))) {
@@ -1164,7 +1169,8 @@ static void integrity_end_io(struct bio *bio)
 	struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
 
 	bio->bi_iter = dio->orig_bi_iter;
-	bio->bi_bdev = dio->orig_bi_bdev;
+	bio->bi_disk = dio->orig_bi_disk;
+	bio->bi_partno = dio->orig_bi_partno;
 	if (dio->orig_bi_integrity) {
 		bio->bi_integrity = dio->orig_bi_integrity;
 		bio->bi_opf |= REQ_INTEGRITY;
@@ -1273,6 +1279,7 @@ again:
 					DMERR("Checksum failed at sector 0x%llx",
 					      (unsigned long long)(sector - ((r + ic->tag_size - 1) / ic->tag_size)));
 					r = -EILSEQ;
+					atomic64_inc(&ic->number_of_mismatches);
 				}
 				if (likely(checksums != checksums_onstack))
 					kfree(checksums);
@@ -1674,15 +1681,16 @@ sleep:
 	dio->in_flight = (atomic_t)ATOMIC_INIT(2);
 
 	if (need_sync_io) {
-		read_comp = COMPLETION_INITIALIZER_ONSTACK(read_comp);
+		init_completion(&read_comp);
 		dio->completion = &read_comp;
 	} else
 		dio->completion = NULL;
 
 	dio->orig_bi_iter = bio->bi_iter;
 
-	dio->orig_bi_bdev = bio->bi_bdev;
-	bio->bi_bdev = ic->dev->bdev;
+	dio->orig_bi_disk = bio->bi_disk;
+	dio->orig_bi_partno = bio->bi_partno;
+	bio_set_dev(bio, ic->dev->bdev);
 
 	dio->orig_bi_integrity = bio_integrity(bio);
 	bio->bi_integrity = NULL;
@@ -1697,7 +1705,11 @@ sleep:
 
 	if (need_sync_io) {
 		wait_for_completion_io(&read_comp);
-		integrity_metadata(&dio->work);
+		if (likely(!bio->bi_status))
+			integrity_metadata(&dio->work);
+		else
+			dec_in_flight(dio);
+
 	} else {
 		INIT_WORK(&dio->work, integrity_metadata);
 		queue_work(ic->metadata_wq, &dio->work);
@@ -1831,7 +1843,7 @@ static void do_journal_write(struct dm_integrity_c *ic, unsigned write_start,
 
 	comp.ic = ic;
 	comp.in_flight = (atomic_t)ATOMIC_INIT(1);
-	comp.comp = COMPLETION_INITIALIZER_ONSTACK(comp.comp);
+	init_completion(&comp.comp);
 
 	i = write_start;
 	for (n = 0; n < write_sections; n++, i++, wraparound_section(ic, &i)) {
@@ -2058,7 +2070,7 @@ static void replay_journal(struct dm_integrity_c *ic)
 		if (ic->journal_io) {
 			struct journal_completion crypt_comp;
 			crypt_comp.ic = ic;
-			crypt_comp.comp = COMPLETION_INITIALIZER_ONSTACK(crypt_comp.comp);
+			init_completion(&crypt_comp.comp);
 			crypt_comp.in_flight = (atomic_t)ATOMIC_INIT(0);
 			encrypt_journal(ic, false, 0, ic->journal_sections, &crypt_comp);
 			wait_for_completion(&crypt_comp.comp);
@@ -2230,7 +2242,7 @@ static void dm_integrity_status(struct dm_target *ti, status_type_t type,
 
 	switch (type) {
 	case STATUSTYPE_INFO:
-		result[0] = '\0';
+		DMEMIT("%llu", (unsigned long long)atomic64_read(&ic->number_of_mismatches));
 		break;
 
 	case STATUSTYPE_TABLE: {
@@ -2631,7 +2643,7 @@ static int create_journal(struct dm_integrity_c *ic, char **error)
 			memset(iv, 0x00, ivsize);
 
 			skcipher_request_set_crypt(req, sg, sg, PAGE_SIZE * ic->journal_pages + sizeof ic->commit_ids, iv);
-			comp.comp = COMPLETION_INITIALIZER_ONSTACK(comp.comp);
+			init_completion(&comp.comp);
 			comp.in_flight = (atomic_t)ATOMIC_INIT(1);
 			if (do_crypt(true, req, &comp))
 				wait_for_completion(&comp.comp);
@@ -2688,7 +2700,7 @@ static int create_journal(struct dm_integrity_c *ic, char **error)
 
 				sg_init_one(&sg, crypt_data, crypt_len);
 				skcipher_request_set_crypt(req, &sg, &sg, crypt_len, iv);
-				comp.comp = COMPLETION_INITIALIZER_ONSTACK(comp.comp);
+				init_completion(&comp.comp);
 				comp.in_flight = (atomic_t)ATOMIC_INIT(1);
 				if (do_crypt(true, req, &comp))
 					wait_for_completion(&comp.comp);
@@ -2775,7 +2787,7 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv)
 	int r;
 	unsigned extra_args;
 	struct dm_arg_set as;
-	static struct dm_arg _args[] = {
+	static const struct dm_arg _args[] = {
 		{0, 9, "Invalid number of feature args"},
 	};
 	unsigned journal_sectors, interleave_sectors, buffer_sectors, journal_watermark, sync_msec;
@@ -2803,6 +2815,7 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv)
 	bio_list_init(&ic->flush_bio_list);
 	init_waitqueue_head(&ic->copy_to_journal_wait);
 	init_completion(&ic->crypto_backoff);
+	atomic64_set(&ic->number_of_mismatches, 0);
 
 	r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &ic->dev);
 	if (r) {
@@ -3199,7 +3212,7 @@ static void dm_integrity_dtr(struct dm_target *ti)
 
 static struct target_type integrity_target = {
 	.name			= "integrity",
-	.version		= {1, 0, 0},
+	.version		= {1, 1, 0},
 	.module			= THIS_MODULE,
 	.features		= DM_TARGET_SINGLETON | DM_TARGET_INTEGRITY,
 	.ctr			= dm_integrity_ctr,
diff --git a/drivers/md/dm-io.c b/drivers/md/dm-io.c
index 25039607f3cb..b4357ed4d541 100644
--- a/drivers/md/dm-io.c
+++ b/drivers/md/dm-io.c
@@ -347,7 +347,7 @@ static void do_region(int op, int op_flags, unsigned region,
 
 		bio = bio_alloc_bioset(GFP_NOIO, num_bvecs, io->client->bios);
 		bio->bi_iter.bi_sector = where->sector + (where->count - remaining);
-		bio->bi_bdev = where->bdev;
+		bio_set_dev(bio, where->bdev);
 		bio->bi_end_io = endio;
 		bio_set_op_attrs(bio, op, op_flags);
 		store_io_and_region_in_bio(bio, io, region);
diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c
index e06f0ef7d2ec..8756a6850431 100644
--- a/drivers/md/dm-ioctl.c
+++ b/drivers/md/dm-ioctl.c
@@ -1629,7 +1629,7 @@ static int target_message(struct file *filp, struct dm_ioctl *param, size_t para
  *---------------------------------------------------------------*/
 static ioctl_fn lookup_ioctl(unsigned int cmd, int *ioctl_flags)
 {
-	static struct {
+	static const struct {
 		int cmd;
 		int flags;
 		ioctl_fn fn;
diff --git a/drivers/md/dm-linear.c b/drivers/md/dm-linear.c
index 41971a090e34..d5f8eff7c11d 100644
--- a/drivers/md/dm-linear.c
+++ b/drivers/md/dm-linear.c
@@ -88,7 +88,7 @@ static void linear_map_bio(struct dm_target *ti, struct bio *bio)
 {
 	struct linear_c *lc = ti->private;
 
-	bio->bi_bdev = lc->dev->bdev;
+	bio_set_dev(bio, lc->dev->bdev);
 	if (bio_sectors(bio) || bio_op(bio) == REQ_OP_ZONE_RESET)
 		bio->bi_iter.bi_sector =
 			linear_map_sector(ti, bio->bi_iter.bi_sector);
@@ -184,20 +184,6 @@ static size_t linear_dax_copy_from_iter(struct dm_target *ti, pgoff_t pgoff,
 	return dax_copy_from_iter(dax_dev, pgoff, addr, bytes, i);
 }
 
-static void linear_dax_flush(struct dm_target *ti, pgoff_t pgoff, void *addr,
-		size_t size)
-{
-	struct linear_c *lc = ti->private;
-	struct block_device *bdev = lc->dev->bdev;
-	struct dax_device *dax_dev = lc->dev->dax_dev;
-	sector_t dev_sector, sector = pgoff * PAGE_SECTORS;
-
-	dev_sector = linear_map_sector(ti, sector);
-	if (bdev_dax_pgoff(bdev, dev_sector, ALIGN(size, PAGE_SIZE), &pgoff))
-		return;
-	dax_flush(dax_dev, pgoff, addr, size);
-}
-
 static struct target_type linear_target = {
 	.name   = "linear",
 	.version = {1, 4, 0},
@@ -212,7 +198,6 @@ static struct target_type linear_target = {
 	.iterate_devices = linear_iterate_devices,
 	.direct_access = linear_dax_direct_access,
 	.dax_copy_from_iter = linear_dax_copy_from_iter,
-	.dax_flush = linear_dax_flush,
 };
 
 int __init dm_linear_init(void)
diff --git a/drivers/md/dm-log-writes.c b/drivers/md/dm-log-writes.c
index a1da0eb58a93..8b80a9ce9ea9 100644
--- a/drivers/md/dm-log-writes.c
+++ b/drivers/md/dm-log-writes.c
@@ -100,6 +100,7 @@ struct log_writes_c {
 	struct dm_dev *logdev;
 	u64 logged_entries;
 	u32 sectorsize;
+	u32 sectorshift;
 	atomic_t io_blocks;
 	atomic_t pending_blocks;
 	sector_t next_sector;
@@ -128,6 +129,18 @@ struct per_bio_data {
 	struct pending_block *block;
 };
 
+static inline sector_t bio_to_dev_sectors(struct log_writes_c *lc,
+					  sector_t sectors)
+{
+	return sectors >> (lc->sectorshift - SECTOR_SHIFT);
+}
+
+static inline sector_t dev_to_bio_sectors(struct log_writes_c *lc,
+					  sector_t sectors)
+{
+	return sectors << (lc->sectorshift - SECTOR_SHIFT);
+}
+
 static void put_pending_block(struct log_writes_c *lc)
 {
 	if (atomic_dec_and_test(&lc->pending_blocks)) {
@@ -198,7 +211,7 @@ static int write_metadata(struct log_writes_c *lc, void *entry,
 	}
 	bio->bi_iter.bi_size = 0;
 	bio->bi_iter.bi_sector = sector;
-	bio->bi_bdev = lc->logdev->bdev;
+	bio_set_dev(bio, lc->logdev->bdev);
 	bio->bi_end_io = log_end_io;
 	bio->bi_private = lc;
 	bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
@@ -253,7 +266,7 @@ static int log_one_block(struct log_writes_c *lc,
 
 	if (!block->vec_cnt)
 		goto out;
-	sector++;
+	sector += dev_to_bio_sectors(lc, 1);
 
 	atomic_inc(&lc->io_blocks);
 	bio = bio_alloc(GFP_KERNEL, min(block->vec_cnt, BIO_MAX_PAGES));
@@ -263,7 +276,7 @@ static int log_one_block(struct log_writes_c *lc,
 	}
 	bio->bi_iter.bi_size = 0;
 	bio->bi_iter.bi_sector = sector;
-	bio->bi_bdev = lc->logdev->bdev;
+	bio_set_dev(bio, lc->logdev->bdev);
 	bio->bi_end_io = log_end_io;
 	bio->bi_private = lc;
 	bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
@@ -285,7 +298,7 @@ static int log_one_block(struct log_writes_c *lc,
 			}
 			bio->bi_iter.bi_size = 0;
 			bio->bi_iter.bi_sector = sector;
-			bio->bi_bdev = lc->logdev->bdev;
+			bio_set_dev(bio, lc->logdev->bdev);
 			bio->bi_end_io = log_end_io;
 			bio->bi_private = lc;
 			bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
@@ -354,10 +367,9 @@ static int log_writes_kthread(void *arg)
 				goto next;
 
 			sector = lc->next_sector;
-			if (block->flags & LOG_DISCARD_FLAG)
-				lc->next_sector++;
-			else
-				lc->next_sector += block->nr_sectors + 1;
+			if (!(block->flags & LOG_DISCARD_FLAG))
+				lc->next_sector += dev_to_bio_sectors(lc, block->nr_sectors);
+			lc->next_sector += dev_to_bio_sectors(lc, 1);
 
 			/*
 			 * Apparently the size of the device may not be known
@@ -399,7 +411,7 @@ next:
 		if (!try_to_freeze()) {
 			set_current_state(TASK_INTERRUPTIBLE);
 			if (!kthread_should_stop() &&
-			    !atomic_read(&lc->pending_blocks))
+			    list_empty(&lc->logging_blocks))
 				schedule();
 			__set_current_state(TASK_RUNNING);
 		}
@@ -435,7 +447,6 @@ static int log_writes_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 	INIT_LIST_HEAD(&lc->unflushed_blocks);
 	INIT_LIST_HEAD(&lc->logging_blocks);
 	init_waitqueue_head(&lc->wait);
-	lc->sectorsize = 1 << SECTOR_SHIFT;
 	atomic_set(&lc->io_blocks, 0);
 	atomic_set(&lc->pending_blocks, 0);
 
@@ -455,6 +466,8 @@ static int log_writes_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 		goto bad;
 	}
 
+	lc->sectorsize = bdev_logical_block_size(lc->dev->bdev);
+	lc->sectorshift = ilog2(lc->sectorsize);
 	lc->log_kthread = kthread_run(log_writes_kthread, lc, "log-write");
 	if (IS_ERR(lc->log_kthread)) {
 		ret = PTR_ERR(lc->log_kthread);
@@ -464,8 +477,12 @@ static int log_writes_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 		goto bad;
 	}
 
-	/* We put the super at sector 0, start logging at sector 1 */
-	lc->next_sector = 1;
+	/*
+	 * next_sector is in 512b sectors to correspond to what bi_sector expects.
+	 * The super starts at sector 0, and the next_sector is the next logical
+	 * one based on the sectorsize of the device.
+	 */
+	lc->next_sector = lc->sectorsize >> SECTOR_SHIFT;
 	lc->logging_enabled = true;
 	lc->end_sector = logdev_last_sector(lc);
 	lc->device_supports_discard = true;
@@ -539,7 +556,7 @@ static void normal_map_bio(struct dm_target *ti, struct bio *bio)
 {
 	struct log_writes_c *lc = ti->private;
 
-	bio->bi_bdev = lc->dev->bdev;
+	bio_set_dev(bio, lc->dev->bdev);
 }
 
 static int log_writes_map(struct dm_target *ti, struct bio *bio)
@@ -599,8 +616,8 @@ static int log_writes_map(struct dm_target *ti, struct bio *bio)
 	if (discard_bio)
 		block->flags |= LOG_DISCARD_FLAG;
 
-	block->sector = bio->bi_iter.bi_sector;
-	block->nr_sectors = bio_sectors(bio);
+	block->sector = bio_to_dev_sectors(lc, bio->bi_iter.bi_sector);
+	block->nr_sectors = bio_to_dev_sectors(lc, bio_sectors(bio));
 
 	/* We don't need the data, just submit */
 	if (discard_bio) {
@@ -767,9 +784,12 @@ static void log_writes_io_hints(struct dm_target *ti, struct queue_limits *limit
 
 	if (!q || !blk_queue_discard(q)) {
 		lc->device_supports_discard = false;
-		limits->discard_granularity = 1 << SECTOR_SHIFT;
+		limits->discard_granularity = lc->sectorsize;
 		limits->max_discard_sectors = (UINT_MAX >> SECTOR_SHIFT);
 	}
+	limits->logical_block_size = bdev_logical_block_size(lc->dev->bdev);
+	limits->physical_block_size = bdev_physical_block_size(lc->dev->bdev);
+	limits->io_min = limits->physical_block_size;
 }
 
 static struct target_type log_writes_target = {
diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c
index 0e8ab5bb3575..11f273d2f018 100644
--- a/drivers/md/dm-mpath.c
+++ b/drivers/md/dm-mpath.c
@@ -504,7 +504,6 @@ static int multipath_clone_and_map(struct dm_target *ti, struct request *rq,
 		if (queue_dying) {
 			atomic_inc(&m->pg_init_in_progress);
 			activate_or_offline_path(pgpath);
-			return DM_MAPIO_REQUEUE;
 		}
 		return DM_MAPIO_DELAY_REQUEUE;
 	}
@@ -566,7 +565,7 @@ static int __multipath_map_bio(struct multipath *m, struct bio *bio, struct dm_m
 	mpio->nr_bytes = nr_bytes;
 
 	bio->bi_status = 0;
-	bio->bi_bdev = pgpath->path.dev->bdev;
+	bio_set_dev(bio, pgpath->path.dev->bdev);
 	bio->bi_opf |= REQ_FAILFAST_TRANSPORT;
 
 	if (pgpath->pg->ps.type->start_io)
@@ -633,6 +632,10 @@ static void process_queued_bios(struct work_struct *work)
 		case DM_MAPIO_REMAPPED:
 			generic_make_request(bio);
 			break;
+		case 0:
+			break;
+		default:
+			WARN_ONCE(true, "__multipath_map_bio() returned %d\n", r);
 		}
 	}
 	blk_finish_plug(&plug);
@@ -699,7 +702,7 @@ static int parse_path_selector(struct dm_arg_set *as, struct priority_group *pg,
 	struct path_selector_type *pst;
 	unsigned ps_argc;
 
-	static struct dm_arg _args[] = {
+	static const struct dm_arg _args[] = {
 		{0, 1024, "invalid number of path selector args"},
 	};
 
@@ -823,7 +826,7 @@ retain:
 static struct priority_group *parse_priority_group(struct dm_arg_set *as,
 						   struct multipath *m)
 {
-	static struct dm_arg _args[] = {
+	static const struct dm_arg _args[] = {
 		{1, 1024, "invalid number of paths"},
 		{0, 1024, "invalid number of selector args"}
 	};
@@ -899,7 +902,7 @@ static int parse_hw_handler(struct dm_arg_set *as, struct multipath *m)
 	int ret;
 	struct dm_target *ti = m->ti;
 
-	static struct dm_arg _args[] = {
+	static const struct dm_arg _args[] = {
 		{0, 1024, "invalid number of hardware handler args"},
 	};
 
@@ -951,7 +954,7 @@ static int parse_features(struct dm_arg_set *as, struct multipath *m)
 	struct dm_target *ti = m->ti;
 	const char *arg_name;
 
-	static struct dm_arg _args[] = {
+	static const struct dm_arg _args[] = {
 		{0, 8, "invalid number of feature args"},
 		{1, 50, "pg_init_retries must be between 1 and 50"},
 		{0, 60000, "pg_init_delay_msecs must be between 0 and 60000"},
@@ -1020,7 +1023,7 @@ static int parse_features(struct dm_arg_set *as, struct multipath *m)
 static int multipath_ctr(struct dm_target *ti, unsigned argc, char **argv)
 {
 	/* target arguments */
-	static struct dm_arg _args[] = {
+	static const struct dm_arg _args[] = {
 		{0, 1024, "invalid number of priority groups"},
 		{0, 1024, "invalid initial priority group number"},
 	};
@@ -1380,6 +1383,7 @@ static void pg_init_done(void *data, int errors)
 	case SCSI_DH_RETRY:
 		/* Wait before retrying. */
 		delay_retry = 1;
+		/* fall through */
 	case SCSI_DH_IMM_RETRY:
 	case SCSI_DH_RES_TEMP_UNAVAIL:
 		if (pg_init_limit_reached(m, pgpath))
@@ -1458,7 +1462,6 @@ static int noretry_error(blk_status_t error)
 	case BLK_STS_TARGET:
 	case BLK_STS_NEXUS:
 	case BLK_STS_MEDIUM:
-	case BLK_STS_RESOURCE:
 		return 1;
 	}
 
diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c
index a4fbd911d566..c0b82136b2d1 100644
--- a/drivers/md/dm-raid1.c
+++ b/drivers/md/dm-raid1.c
@@ -145,7 +145,7 @@ static void dispatch_bios(void *context, struct bio_list *bio_list)
 
 struct dm_raid1_bio_record {
 	struct mirror *m;
-	/* if details->bi_bdev == NULL, details were not saved */
+	/* if details->bi_disk == NULL, details were not saved */
 	struct dm_bio_details details;
 	region_t write_region;
 };
@@ -464,7 +464,7 @@ static sector_t map_sector(struct mirror *m, struct bio *bio)
 
 static void map_bio(struct mirror *m, struct bio *bio)
 {
-	bio->bi_bdev = m->dev->bdev;
+	bio_set_dev(bio, m->dev->bdev);
 	bio->bi_iter.bi_sector = map_sector(m, bio);
 }
 
@@ -1199,7 +1199,7 @@ static int mirror_map(struct dm_target *ti, struct bio *bio)
 	struct dm_raid1_bio_record *bio_record =
 	  dm_per_bio_data(bio, sizeof(struct dm_raid1_bio_record));
 
-	bio_record->details.bi_bdev = NULL;
+	bio_record->details.bi_disk = NULL;
 
 	if (rw == WRITE) {
 		/* Save region for mirror_end_io() handler */
@@ -1266,7 +1266,7 @@ static int mirror_end_io(struct dm_target *ti, struct bio *bio,
 		goto out;
 
 	if (unlikely(*error)) {
-		if (!bio_record->details.bi_bdev) {
+		if (!bio_record->details.bi_disk) {
 			/*
 			 * There wasn't enough memory to record necessary
 			 * information for a retry or there was no other
@@ -1291,7 +1291,7 @@ static int mirror_end_io(struct dm_target *ti, struct bio *bio,
 			bd = &bio_record->details;
 
 			dm_bio_restore(bd, bio);
-			bio_record->details.bi_bdev = NULL;
+			bio_record->details.bi_disk = NULL;
 			bio->bi_status = 0;
 
 			queue_bio(ms, bio, rw);
@@ -1301,7 +1301,7 @@ static int mirror_end_io(struct dm_target *ti, struct bio *bio,
 	}
 
 out:
-	bio_record->details.bi_bdev = NULL;
+	bio_record->details.bi_disk = NULL;
 
 	return DM_ENDIO_DONE;
 }
diff --git a/drivers/md/dm-rq.c b/drivers/md/dm-rq.c
index c6ebc5b1e00e..eadfcfd106ff 100644
--- a/drivers/md/dm-rq.c
+++ b/drivers/md/dm-rq.c
@@ -117,9 +117,9 @@ static void end_clone_bio(struct bio *clone)
 	struct dm_rq_clone_bio_info *info =
 		container_of(clone, struct dm_rq_clone_bio_info, clone);
 	struct dm_rq_target_io *tio = info->tio;
-	struct bio *bio = info->orig;
 	unsigned int nr_bytes = info->orig->bi_iter.bi_size;
 	blk_status_t error = clone->bi_status;
+	bool is_last = !clone->bi_next;
 
 	bio_put(clone);
 
@@ -137,28 +137,23 @@ static void end_clone_bio(struct bio *clone)
 		 * when the request is completed.
 		 */
 		tio->error = error;
-		return;
+		goto exit;
 	}
 
 	/*
 	 * I/O for the bio successfully completed.
 	 * Notice the data completion to the upper layer.
 	 */
-
-	/*
-	 * bios are processed from the head of the list.
-	 * So the completing bio should always be rq->bio.
-	 * If it's not, something wrong is happening.
-	 */
-	if (tio->orig->bio != bio)
-		DMERR("bio completion is going in the middle of the request");
+	tio->completed += nr_bytes;
 
 	/*
 	 * Update the original request.
 	 * Do not use blk_end_request() here, because it may complete
 	 * the original request before the clone, and break the ordering.
 	 */
-	blk_update_request(tio->orig, BLK_STS_OK, nr_bytes);
+	if (is_last)
+ exit:
+		blk_update_request(tio->orig, BLK_STS_OK, tio->completed);
 }
 
 static struct dm_rq_target_io *tio_from_request(struct request *rq)
@@ -237,14 +232,14 @@ static void dm_end_request(struct request *clone, blk_status_t error)
 /*
  * Requeue the original request of a clone.
  */
-static void dm_old_requeue_request(struct request *rq)
+static void dm_old_requeue_request(struct request *rq, unsigned long delay_ms)
 {
 	struct request_queue *q = rq->q;
 	unsigned long flags;
 
 	spin_lock_irqsave(q->queue_lock, flags);
 	blk_requeue_request(q, rq);
-	blk_run_queue_async(q);
+	blk_delay_queue(q, delay_ms);
 	spin_unlock_irqrestore(q->queue_lock, flags);
 }
 
@@ -270,6 +265,7 @@ static void dm_requeue_original_request(struct dm_rq_target_io *tio, bool delay_
 	struct mapped_device *md = tio->md;
 	struct request *rq = tio->orig;
 	int rw = rq_data_dir(rq);
+	unsigned long delay_ms = delay_requeue ? 100 : 0;
 
 	rq_end_stats(md, rq);
 	if (tio->clone) {
@@ -278,9 +274,9 @@ static void dm_requeue_original_request(struct dm_rq_target_io *tio, bool delay_
 	}
 
 	if (!rq->q->mq_ops)
-		dm_old_requeue_request(rq);
+		dm_old_requeue_request(rq, delay_ms);
 	else
-		dm_mq_delay_requeue_request(rq, delay_requeue ? 100/*ms*/ : 0);
+		dm_mq_delay_requeue_request(rq, delay_ms);
 
 	rq_completed(md, rw, false);
 }
@@ -455,6 +451,7 @@ static void init_tio(struct dm_rq_target_io *tio, struct request *rq,
 	tio->clone = NULL;
 	tio->orig = rq;
 	tio->error = 0;
+	tio->completed = 0;
 	/*
 	 * Avoid initializing info for blk-mq; it passes
 	 * target-specific data through info.ptr
diff --git a/drivers/md/dm-rq.h b/drivers/md/dm-rq.h
index 9813922e4fe5..f43c45460aac 100644
--- a/drivers/md/dm-rq.h
+++ b/drivers/md/dm-rq.h
@@ -29,6 +29,7 @@ struct dm_rq_target_io {
 	struct dm_stats_aux stats_aux;
 	unsigned long duration_jiffies;
 	unsigned n_sectors;
+	unsigned completed;
 };
 
 /*
diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c
index 1ba41048b438..1113b42e1eda 100644
--- a/drivers/md/dm-snap.c
+++ b/drivers/md/dm-snap.c
@@ -1663,7 +1663,7 @@ __find_pending_exception(struct dm_snapshot *s,
 static void remap_exception(struct dm_snapshot *s, struct dm_exception *e,
 			    struct bio *bio, chunk_t chunk)
 {
-	bio->bi_bdev = s->cow->bdev;
+	bio_set_dev(bio, s->cow->bdev);
 	bio->bi_iter.bi_sector =
 		chunk_to_sector(s->store, dm_chunk_number(e->new_chunk) +
 				(chunk - e->old_chunk)) +
@@ -1681,7 +1681,7 @@ static int snapshot_map(struct dm_target *ti, struct bio *bio)
 	init_tracked_chunk(bio);
 
 	if (bio->bi_opf & REQ_PREFLUSH) {
-		bio->bi_bdev = s->cow->bdev;
+		bio_set_dev(bio, s->cow->bdev);
 		return DM_MAPIO_REMAPPED;
 	}
 
@@ -1769,7 +1769,7 @@ static int snapshot_map(struct dm_target *ti, struct bio *bio)
 			goto out;
 		}
 	} else {
-		bio->bi_bdev = s->origin->bdev;
+		bio_set_dev(bio, s->origin->bdev);
 		track_chunk(s, bio, chunk);
 	}
 
@@ -1802,9 +1802,9 @@ static int snapshot_merge_map(struct dm_target *ti, struct bio *bio)
 
 	if (bio->bi_opf & REQ_PREFLUSH) {
 		if (!dm_bio_get_target_bio_nr(bio))
-			bio->bi_bdev = s->origin->bdev;
+			bio_set_dev(bio, s->origin->bdev);
 		else
-			bio->bi_bdev = s->cow->bdev;
+			bio_set_dev(bio, s->cow->bdev);
 		return DM_MAPIO_REMAPPED;
 	}
 
@@ -1824,7 +1824,7 @@ static int snapshot_merge_map(struct dm_target *ti, struct bio *bio)
 		    chunk >= s->first_merging_chunk &&
 		    chunk < (s->first_merging_chunk +
 			     s->num_merging_chunks)) {
-			bio->bi_bdev = s->origin->bdev;
+			bio_set_dev(bio, s->origin->bdev);
 			bio_list_add(&s->bios_queued_during_merge, bio);
 			r = DM_MAPIO_SUBMITTED;
 			goto out_unlock;
@@ -1838,7 +1838,7 @@ static int snapshot_merge_map(struct dm_target *ti, struct bio *bio)
 	}
 
 redirect_to_origin:
-	bio->bi_bdev = s->origin->bdev;
+	bio_set_dev(bio, s->origin->bdev);
 
 	if (bio_data_dir(bio) == WRITE) {
 		up_write(&s->lock);
@@ -2285,7 +2285,7 @@ static int origin_map(struct dm_target *ti, struct bio *bio)
 	struct dm_origin *o = ti->private;
 	unsigned available_sectors;
 
-	bio->bi_bdev = o->dev->bdev;
+	bio_set_dev(bio, o->dev->bdev);
 
 	if (unlikely(bio->bi_opf & REQ_PREFLUSH))
 		return DM_MAPIO_REMAPPED;
diff --git a/drivers/md/dm-stripe.c b/drivers/md/dm-stripe.c
index a0375530b07f..b5e892149c54 100644
--- a/drivers/md/dm-stripe.c
+++ b/drivers/md/dm-stripe.c
@@ -270,7 +270,7 @@ static int stripe_map_range(struct stripe_c *sc, struct bio *bio,
 	stripe_map_range_sector(sc, bio_end_sector(bio),
 				target_stripe, &end);
 	if (begin < end) {
-		bio->bi_bdev = sc->stripe[target_stripe].dev->bdev;
+		bio_set_dev(bio, sc->stripe[target_stripe].dev->bdev);
 		bio->bi_iter.bi_sector = begin +
 			sc->stripe[target_stripe].physical_start;
 		bio->bi_iter.bi_size = to_bytes(end - begin);
@@ -291,7 +291,7 @@ static int stripe_map(struct dm_target *ti, struct bio *bio)
 	if (bio->bi_opf & REQ_PREFLUSH) {
 		target_bio_nr = dm_bio_get_target_bio_nr(bio);
 		BUG_ON(target_bio_nr >= sc->stripes);
-		bio->bi_bdev = sc->stripe[target_bio_nr].dev->bdev;
+		bio_set_dev(bio, sc->stripe[target_bio_nr].dev->bdev);
 		return DM_MAPIO_REMAPPED;
 	}
 	if (unlikely(bio_op(bio) == REQ_OP_DISCARD) ||
@@ -306,7 +306,7 @@ static int stripe_map(struct dm_target *ti, struct bio *bio)
 			  &stripe, &bio->bi_iter.bi_sector);
 
 	bio->bi_iter.bi_sector += sc->stripe[stripe].physical_start;
-	bio->bi_bdev = sc->stripe[stripe].dev->bdev;
+	bio_set_dev(bio, sc->stripe[stripe].dev->bdev);
 
 	return DM_MAPIO_REMAPPED;
 }
@@ -351,25 +351,6 @@ static size_t stripe_dax_copy_from_iter(struct dm_target *ti, pgoff_t pgoff,
 	return dax_copy_from_iter(dax_dev, pgoff, addr, bytes, i);
 }
 
-static void stripe_dax_flush(struct dm_target *ti, pgoff_t pgoff, void *addr,
-		size_t size)
-{
-	sector_t dev_sector, sector = pgoff * PAGE_SECTORS;
-	struct stripe_c *sc = ti->private;
-	struct dax_device *dax_dev;
-	struct block_device *bdev;
-	uint32_t stripe;
-
-	stripe_map_sector(sc, sector, &stripe, &dev_sector);
-	dev_sector += sc->stripe[stripe].physical_start;
-	dax_dev = sc->stripe[stripe].dev->dax_dev;
-	bdev = sc->stripe[stripe].dev->bdev;
-
-	if (bdev_dax_pgoff(bdev, dev_sector, ALIGN(size, PAGE_SIZE), &pgoff))
-		return;
-	dax_flush(dax_dev, pgoff, addr, size);
-}
-
 /*
  * Stripe status:
  *
@@ -430,9 +411,7 @@ static int stripe_end_io(struct dm_target *ti, struct bio *bio,
 		return DM_ENDIO_DONE;
 
 	memset(major_minor, 0, sizeof(major_minor));
-	sprintf(major_minor, "%d:%d",
-		MAJOR(disk_devt(bio->bi_bdev->bd_disk)),
-		MINOR(disk_devt(bio->bi_bdev->bd_disk)));
+	sprintf(major_minor, "%d:%d", MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)));
 
 	/*
 	 * Test to see which stripe drive triggered the event
@@ -491,7 +470,6 @@ static struct target_type stripe_target = {
 	.io_hints = stripe_io_hints,
 	.direct_access = stripe_dax_direct_access,
 	.dax_copy_from_iter = stripe_dax_copy_from_iter,
-	.dax_flush = stripe_dax_flush,
 };
 
 int __init dm_stripe_init(void)
diff --git a/drivers/md/dm-switch.c b/drivers/md/dm-switch.c
index 871c18fe000d..4c8de1ff78ca 100644
--- a/drivers/md/dm-switch.c
+++ b/drivers/md/dm-switch.c
@@ -251,7 +251,7 @@ static void switch_dtr(struct dm_target *ti)
  */
 static int switch_ctr(struct dm_target *ti, unsigned argc, char **argv)
 {
-	static struct dm_arg _args[] = {
+	static const struct dm_arg _args[] = {
 		{1, (KMALLOC_MAX_SIZE - sizeof(struct switch_ctx)) / sizeof(struct switch_path), "Invalid number of paths"},
 		{1, UINT_MAX, "Invalid region size"},
 		{0, 0, "Invalid number of optional args"},
@@ -322,7 +322,7 @@ static int switch_map(struct dm_target *ti, struct bio *bio)
 	sector_t offset = dm_target_offset(ti, bio->bi_iter.bi_sector);
 	unsigned path_nr = switch_get_path_nr(sctx, offset);
 
-	bio->bi_bdev = sctx->path_list[path_nr].dmdev->bdev;
+	bio_set_dev(bio, sctx->path_list[path_nr].dmdev->bdev);
 	bio->bi_iter.bi_sector = sctx->path_list[path_nr].start + offset;
 
 	return DM_MAPIO_REMAPPED;
diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c
index 28a4071cdf85..ef7b8f201f73 100644
--- a/drivers/md/dm-table.c
+++ b/drivers/md/dm-table.c
@@ -806,7 +806,8 @@ int dm_table_add_target(struct dm_table *t, const char *type,
 /*
  * Target argument parsing helpers.
  */
-static int validate_next_arg(struct dm_arg *arg, struct dm_arg_set *arg_set,
+static int validate_next_arg(const struct dm_arg *arg,
+			     struct dm_arg_set *arg_set,
 			     unsigned *value, char **error, unsigned grouped)
 {
 	const char *arg_str = dm_shift_arg(arg_set);
@@ -824,14 +825,14 @@ static int validate_next_arg(struct dm_arg *arg, struct dm_arg_set *arg_set,
 	return 0;
 }
 
-int dm_read_arg(struct dm_arg *arg, struct dm_arg_set *arg_set,
+int dm_read_arg(const struct dm_arg *arg, struct dm_arg_set *arg_set,
 		unsigned *value, char **error)
 {
 	return validate_next_arg(arg, arg_set, value, error, 0);
 }
 EXPORT_SYMBOL(dm_read_arg);
 
-int dm_read_arg_group(struct dm_arg *arg, struct dm_arg_set *arg_set,
+int dm_read_arg_group(const struct dm_arg *arg, struct dm_arg_set *arg_set,
 		      unsigned *value, char **error)
 {
 	return validate_next_arg(arg, arg_set, value, error, 1);
diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c
index 9dec2f8cc739..1e25705209c2 100644
--- a/drivers/md/dm-thin.c
+++ b/drivers/md/dm-thin.c
@@ -679,7 +679,7 @@ static void remap(struct thin_c *tc, struct bio *bio, dm_block_t block)
 	struct pool *pool = tc->pool;
 	sector_t bi_sector = bio->bi_iter.bi_sector;
 
-	bio->bi_bdev = tc->pool_dev->bdev;
+	bio_set_dev(bio, tc->pool_dev->bdev);
 	if (block_size_is_power_of_two(pool))
 		bio->bi_iter.bi_sector =
 			(block << pool->sectors_per_block_shift) |
@@ -691,7 +691,7 @@ static void remap(struct thin_c *tc, struct bio *bio, dm_block_t block)
 
 static void remap_to_origin(struct thin_c *tc, struct bio *bio)
 {
-	bio->bi_bdev = tc->origin_dev->bdev;
+	bio_set_dev(bio, tc->origin_dev->bdev);
 }
 
 static int bio_triggers_commit(struct thin_c *tc, struct bio *bio)
@@ -3041,7 +3041,7 @@ static int parse_pool_features(struct dm_arg_set *as, struct pool_features *pf,
 	unsigned argc;
 	const char *arg_name;
 
-	static struct dm_arg _args[] = {
+	static const struct dm_arg _args[] = {
 		{0, 4, "Invalid number of pool feature arguments"},
 	};
 
@@ -3313,7 +3313,7 @@ static int pool_map(struct dm_target *ti, struct bio *bio)
 	 * As this is a singleton target, ti->begin is always zero.
 	 */
 	spin_lock_irqsave(&pool->lock, flags);
-	bio->bi_bdev = pt->data_dev->bdev;
+	bio_set_dev(bio, pt->data_dev->bdev);
 	r = DM_MAPIO_REMAPPED;
 	spin_unlock_irqrestore(&pool->lock, flags);
 
diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c
index b46705ebf01f..bda3caca23ca 100644
--- a/drivers/md/dm-verity-target.c
+++ b/drivers/md/dm-verity-target.c
@@ -637,7 +637,7 @@ static int verity_map(struct dm_target *ti, struct bio *bio)
 	struct dm_verity *v = ti->private;
 	struct dm_verity_io *io;
 
-	bio->bi_bdev = v->data_dev->bdev;
+	bio_set_dev(bio, v->data_dev->bdev);
 	bio->bi_iter.bi_sector = verity_map_sector(v, bio->bi_iter.bi_sector);
 
 	if (((unsigned)bio->bi_iter.bi_sector | bio_sectors(bio)) &
@@ -839,7 +839,7 @@ static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v)
 	struct dm_target *ti = v->ti;
 	const char *arg_name;
 
-	static struct dm_arg _args[] = {
+	static const struct dm_arg _args[] = {
 		{0, DM_VERITY_OPTS_MAX, "Invalid number of feature args"},
 	};
 
diff --git a/drivers/md/dm-zoned-metadata.c b/drivers/md/dm-zoned-metadata.c
index a4fa2ada6883..70485de37b66 100644
--- a/drivers/md/dm-zoned-metadata.c
+++ b/drivers/md/dm-zoned-metadata.c
@@ -409,7 +409,7 @@ static struct dmz_mblock *dmz_fetch_mblock(struct dmz_metadata *zmd,
 	}
 
 	bio->bi_iter.bi_sector = dmz_blk2sect(block);
-	bio->bi_bdev = zmd->dev->bdev;
+	bio_set_dev(bio, zmd->dev->bdev);
 	bio->bi_private = mblk;
 	bio->bi_end_io = dmz_mblock_bio_end_io;
 	bio_set_op_attrs(bio, REQ_OP_READ, REQ_META | REQ_PRIO);
@@ -564,7 +564,7 @@ static void dmz_write_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk,
 	set_bit(DMZ_META_WRITING, &mblk->state);
 
 	bio->bi_iter.bi_sector = dmz_blk2sect(block);
-	bio->bi_bdev = zmd->dev->bdev;
+	bio_set_dev(bio, zmd->dev->bdev);
 	bio->bi_private = mblk;
 	bio->bi_end_io = dmz_mblock_bio_end_io;
 	bio_set_op_attrs(bio, REQ_OP_WRITE, REQ_META | REQ_PRIO);
@@ -586,7 +586,7 @@ static int dmz_rdwr_block(struct dmz_metadata *zmd, int op, sector_t block,
 		return -ENOMEM;
 
 	bio->bi_iter.bi_sector = dmz_blk2sect(block);
-	bio->bi_bdev = zmd->dev->bdev;
+	bio_set_dev(bio, zmd->dev->bdev);
 	bio_set_op_attrs(bio, op, REQ_SYNC | REQ_META | REQ_PRIO);
 	bio_add_page(bio, page, DMZ_BLOCK_SIZE, 0);
 	ret = submit_bio_wait(bio);
diff --git a/drivers/md/dm-zoned-target.c b/drivers/md/dm-zoned-target.c
index b08bbbd4d902..b87c1741da4b 100644
--- a/drivers/md/dm-zoned-target.c
+++ b/drivers/md/dm-zoned-target.c
@@ -238,7 +238,7 @@ static void dmz_submit_write_bio(struct dmz_target *dmz, struct dm_zone *zone,
 	struct dmz_bioctx *bioctx = dm_per_bio_data(bio, sizeof(struct dmz_bioctx));
 
 	/* Setup and submit the BIO */
-	bio->bi_bdev = dmz->dev->bdev;
+	bio_set_dev(bio, dmz->dev->bdev);
 	bio->bi_iter.bi_sector = dmz_start_sect(dmz->metadata, zone) + dmz_blk2sect(chunk_block);
 	atomic_inc(&bioctx->ref);
 	generic_make_request(bio);
@@ -586,7 +586,7 @@ static int dmz_map(struct dm_target *ti, struct bio *bio)
 		      (unsigned long long)dmz_chunk_block(dmz->dev, dmz_bio_block(bio)),
 		      (unsigned int)dmz_bio_blocks(bio));
 
-	bio->bi_bdev = dev->bdev;
+	bio_set_dev(bio, dev->bdev);
 
 	if (!nr_sectors && bio_op(bio) != REQ_OP_WRITE)
 		return DM_MAPIO_REMAPPED;
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index 2edbcc2d7d3f..6e54145969c5 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -27,16 +27,6 @@
 
 #define DM_MSG_PREFIX "core"
 
-#ifdef CONFIG_PRINTK
-/*
- * ratelimit state to be used in DMXXX_LIMIT().
- */
-DEFINE_RATELIMIT_STATE(dm_ratelimit_state,
-		       DEFAULT_RATELIMIT_INTERVAL,
-		       DEFAULT_RATELIMIT_BURST);
-EXPORT_SYMBOL(dm_ratelimit_state);
-#endif
-
 /*
  * Cookies are numeric values sent with CHANGE and REMOVE
  * uevents while resuming, removing or renaming the device.
@@ -520,7 +510,7 @@ static void start_io_acct(struct dm_io *io)
 	io->start_time = jiffies;
 
 	cpu = part_stat_lock();
-	part_round_stats(cpu, &dm_disk(md)->part0);
+	part_round_stats(md->queue, cpu, &dm_disk(md)->part0);
 	part_stat_unlock();
 	atomic_set(&dm_disk(md)->part0.in_flight[rw],
 		atomic_inc_return(&md->pending[rw]));
@@ -539,7 +529,7 @@ static void end_io_acct(struct dm_io *io)
 	int pending;
 	int rw = bio_data_dir(bio);
 
-	generic_end_io_acct(rw, &dm_disk(md)->part0, io->start_time);
+	generic_end_io_acct(md->queue, rw, &dm_disk(md)->part0, io->start_time);
 
 	if (unlikely(dm_stats_used(&md->stats)))
 		dm_stats_account_io(&md->stats, bio_data_dir(bio),
@@ -851,10 +841,10 @@ static void clone_endio(struct bio *bio)
 
 	if (unlikely(error == BLK_STS_TARGET)) {
 		if (bio_op(bio) == REQ_OP_WRITE_SAME &&
-		    !bdev_get_queue(bio->bi_bdev)->limits.max_write_same_sectors)
+		    !bio->bi_disk->queue->limits.max_write_same_sectors)
 			disable_write_same(md);
 		if (bio_op(bio) == REQ_OP_WRITE_ZEROES &&
-		    !bdev_get_queue(bio->bi_bdev)->limits.max_write_zeroes_sectors)
+		    !bio->bi_disk->queue->limits.max_write_zeroes_sectors)
 			disable_write_zeroes(md);
 	}
 
@@ -997,24 +987,6 @@ static size_t dm_dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff,
 	return ret;
 }
 
-static void dm_dax_flush(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
-		size_t size)
-{
-	struct mapped_device *md = dax_get_private(dax_dev);
-	sector_t sector = pgoff * PAGE_SECTORS;
-	struct dm_target *ti;
-	int srcu_idx;
-
-	ti = dm_dax_get_live_target(md, sector, &srcu_idx);
-
-	if (!ti)
-		goto out;
-	if (ti->type->dax_flush)
-		ti->type->dax_flush(ti, pgoff, addr, size);
- out:
-	dm_put_live_table(md, srcu_idx);
-}
-
 /*
  * A target may call dm_accept_partial_bio only from the map routine.  It is
  * allowed for all bio types except REQ_PREFLUSH.
@@ -1215,8 +1187,8 @@ static void __map_bio(struct dm_target_io *tio)
 		break;
 	case DM_MAPIO_REMAPPED:
 		/* the bio has been remapped so dispatch it */
-		trace_block_bio_remap(bdev_get_queue(clone->bi_bdev), clone,
-				      tio->io->bio->bi_bdev->bd_dev, sector);
+		trace_block_bio_remap(clone->bi_disk->queue, clone,
+				      bio_dev(tio->io->bio), sector);
 		generic_make_request(clone);
 		break;
 	case DM_MAPIO_KILL:
@@ -1523,7 +1495,7 @@ static void __split_and_process_bio(struct mapped_device *md,
 	}
 
 	/* drop the extra reference count */
-	dec_pending(ci.io, error);
+	dec_pending(ci.io, errno_to_blk_status(error));
 }
 /*-----------------------------------------------------------------
  * CRUD END
@@ -1542,7 +1514,7 @@ static blk_qc_t dm_make_request(struct request_queue *q, struct bio *bio)
 
 	map = dm_get_live_table(md, &srcu_idx);
 
-	generic_start_io_acct(rw, bio_sectors(bio), &dm_disk(md)->part0);
+	generic_start_io_acct(q, rw, bio_sectors(bio), &dm_disk(md)->part0);
 
 	/* if we're suspended, we have to queue this io for later */
 	if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) {
@@ -1796,7 +1768,7 @@ static struct mapped_device *alloc_dev(int minor)
 		goto bad;
 
 	bio_init(&md->flush_bio, NULL, 0);
-	md->flush_bio.bi_bdev = md->bdev;
+	bio_set_dev(&md->flush_bio, md->bdev);
 	md->flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC;
 
 	dm_stats_init(&md->stats);
@@ -3002,7 +2974,6 @@ static const struct block_device_operations dm_blk_dops = {
 static const struct dax_operations dm_dax_ops = {
 	.direct_access = dm_dax_direct_access,
 	.copy_from_iter = dm_dax_copy_from_iter,
-	.flush = dm_dax_flush,
 };
 
 /*
diff --git a/drivers/md/faulty.c b/drivers/md/faulty.c
index 06a64d5d8c6c..38264b38420f 100644
--- a/drivers/md/faulty.c
+++ b/drivers/md/faulty.c
@@ -216,12 +216,12 @@ static bool faulty_make_request(struct mddev *mddev, struct bio *bio)
 	if (failit) {
 		struct bio *b = bio_clone_fast(bio, GFP_NOIO, mddev->bio_set);
 
-		b->bi_bdev = conf->rdev->bdev;
+		bio_set_dev(b, conf->rdev->bdev);
 		b->bi_private = bio;
 		b->bi_end_io = faulty_fail;
 		bio = b;
 	} else
-		bio->bi_bdev = conf->rdev->bdev;
+		bio_set_dev(bio, conf->rdev->bdev);
 
 	generic_make_request(bio);
 	return true;
diff --git a/drivers/md/linear.c b/drivers/md/linear.c
index 5f1eb9189542..c464fb48039a 100644
--- a/drivers/md/linear.c
+++ b/drivers/md/linear.c
@@ -275,17 +275,17 @@ static bool linear_make_request(struct mddev *mddev, struct bio *bio)
 		bio = split;
 	}
 
-	bio->bi_bdev = tmp_dev->rdev->bdev;
+	bio_set_dev(bio, tmp_dev->rdev->bdev);
 	bio->bi_iter.bi_sector = bio->bi_iter.bi_sector -
 		start_sector + data_offset;
 
 	if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
-		     !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) {
+		     !blk_queue_discard(bio->bi_disk->queue))) {
 		/* Just ignore it */
 		bio_endio(bio);
 	} else {
 		if (mddev->gendisk)
-			trace_block_bio_remap(bdev_get_queue(bio->bi_bdev),
+			trace_block_bio_remap(bio->bi_disk->queue,
 					      bio, disk_devt(mddev->gendisk),
 					      bio_sector);
 		mddev_check_writesame(mddev, bio);
diff --git a/drivers/md/md.c b/drivers/md/md.c
index c99634612fc4..08fcaebc61bd 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -422,7 +422,7 @@ static void submit_flushes(struct work_struct *ws)
 			bi = bio_alloc_mddev(GFP_NOIO, 0, mddev);
 			bi->bi_end_io = md_end_flush;
 			bi->bi_private = rdev;
-			bi->bi_bdev = rdev->bdev;
+			bio_set_dev(bi, rdev->bdev);
 			bi->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
 			atomic_inc(&mddev->flush_pending);
 			submit_bio(bi);
@@ -772,7 +772,7 @@ void md_super_write(struct mddev *mddev, struct md_rdev *rdev,
 
 	atomic_inc(&rdev->nr_pending);
 
-	bio->bi_bdev = rdev->meta_bdev ? rdev->meta_bdev : rdev->bdev;
+	bio_set_dev(bio, rdev->meta_bdev ? rdev->meta_bdev : rdev->bdev);
 	bio->bi_iter.bi_sector = sector;
 	bio_add_page(bio, page, size, 0);
 	bio->bi_private = rdev;
@@ -803,8 +803,10 @@ int sync_page_io(struct md_rdev *rdev, sector_t sector, int size,
 	struct bio *bio = md_bio_alloc_sync(rdev->mddev);
 	int ret;
 
-	bio->bi_bdev = (metadata_op && rdev->meta_bdev) ?
-		rdev->meta_bdev : rdev->bdev;
+	if (metadata_op && rdev->meta_bdev)
+		bio_set_dev(bio, rdev->meta_bdev);
+	else
+		bio_set_dev(bio, rdev->bdev);
 	bio_set_op_attrs(bio, op, op_flags);
 	if (metadata_op)
 		bio->bi_iter.bi_sector = sector + rdev->sb_start;
@@ -1536,7 +1538,8 @@ static int super_1_load(struct md_rdev *rdev, struct md_rdev *refdev, int minor_
 	} else if (sb->bblog_offset != 0)
 		rdev->badblocks.shift = 0;
 
-	if (le32_to_cpu(sb->feature_map) & MD_FEATURE_PPL) {
+	if ((le32_to_cpu(sb->feature_map) &
+	    (MD_FEATURE_PPL | MD_FEATURE_MULTIPLE_PPLS))) {
 		rdev->ppl.offset = (__s16)le16_to_cpu(sb->ppl.offset);
 		rdev->ppl.size = le16_to_cpu(sb->ppl.size);
 		rdev->ppl.sector = rdev->sb_start + rdev->ppl.offset;
@@ -1655,10 +1658,15 @@ static int super_1_validate(struct mddev *mddev, struct md_rdev *rdev)
 		if (le32_to_cpu(sb->feature_map) & MD_FEATURE_JOURNAL)
 			set_bit(MD_HAS_JOURNAL, &mddev->flags);
 
-		if (le32_to_cpu(sb->feature_map) & MD_FEATURE_PPL) {
+		if (le32_to_cpu(sb->feature_map) &
+		    (MD_FEATURE_PPL | MD_FEATURE_MULTIPLE_PPLS)) {
 			if (le32_to_cpu(sb->feature_map) &
 			    (MD_FEATURE_BITMAP_OFFSET | MD_FEATURE_JOURNAL))
 				return -EINVAL;
+			if ((le32_to_cpu(sb->feature_map) & MD_FEATURE_PPL) &&
+			    (le32_to_cpu(sb->feature_map) &
+					    MD_FEATURE_MULTIPLE_PPLS))
+				return -EINVAL;
 			set_bit(MD_HAS_PPL, &mddev->flags);
 		}
 	} else if (mddev->pers == NULL) {
@@ -1875,7 +1883,11 @@ retry:
 		sb->feature_map |= cpu_to_le32(MD_FEATURE_JOURNAL);
 
 	if (test_bit(MD_HAS_PPL, &mddev->flags)) {
-		sb->feature_map |= cpu_to_le32(MD_FEATURE_PPL);
+		if (test_bit(MD_HAS_MULTIPLE_PPLS, &mddev->flags))
+			sb->feature_map |=
+			    cpu_to_le32(MD_FEATURE_MULTIPLE_PPLS);
+		else
+			sb->feature_map |= cpu_to_le32(MD_FEATURE_PPL);
 		sb->ppl.offset = cpu_to_le16(rdev->ppl.offset);
 		sb->ppl.size = cpu_to_le16(rdev->ppl.size);
 	}
@@ -4283,6 +4295,8 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len)
 	if (err)
 		export_rdev(rdev);
 	mddev_unlock(mddev);
+	if (!err)
+		md_new_event(mddev);
 	return err ? err : len;
 }
 
@@ -7836,7 +7850,7 @@ static const struct file_operations md_seq_fops = {
 	.open           = md_seq_open,
 	.read           = seq_read,
 	.llseek         = seq_lseek,
-	.release	= seq_release_private,
+	.release	= seq_release,
 	.poll		= mdstat_poll,
 };
 
@@ -7996,7 +8010,7 @@ bool md_write_start(struct mddev *mddev, struct bio *bi)
 	if (mddev->safemode == 1)
 		mddev->safemode = 0;
 	/* sync_checkers is always 0 when writes_pending is in per-cpu mode */
-	if (mddev->in_sync || !mddev->sync_checkers) {
+	if (mddev->in_sync || mddev->sync_checkers) {
 		spin_lock(&mddev->lock);
 		if (mddev->in_sync) {
 			mddev->in_sync = 0;
@@ -8656,6 +8670,9 @@ void md_check_recovery(struct mddev *mddev)
 	if (mddev_trylock(mddev)) {
 		int spares = 0;
 
+		if (!mddev->external && mddev->safemode == 1)
+			mddev->safemode = 0;
+
 		if (mddev->ro) {
 			struct md_rdev *rdev;
 			if (!mddev->external && mddev->in_sync)
diff --git a/drivers/md/md.h b/drivers/md/md.h
index 09db03455801..561d22b9a9a8 100644
--- a/drivers/md/md.h
+++ b/drivers/md/md.h
@@ -236,6 +236,7 @@ enum mddev_flags {
 				 * never cause the array to become failed.
 				 */
 	MD_HAS_PPL,		/* The raid array has PPL feature set */
+	MD_HAS_MULTIPLE_PPLS,	/* The raid array has multiple PPLs feature set */
 };
 
 enum mddev_sb_flags {
@@ -509,6 +510,11 @@ static inline void md_sync_acct(struct block_device *bdev, unsigned long nr_sect
 	atomic_add(nr_sectors, &bdev->bd_contains->bd_disk->sync_io);
 }
 
+static inline void md_sync_acct_bio(struct bio *bio, unsigned long nr_sectors)
+{
+	atomic_add(nr_sectors, &bio->bi_disk->sync_io);
+}
+
 struct md_personality
 {
 	char *name;
@@ -721,14 +727,14 @@ static inline void mddev_clear_unsupported_flags(struct mddev *mddev,
 static inline void mddev_check_writesame(struct mddev *mddev, struct bio *bio)
 {
 	if (bio_op(bio) == REQ_OP_WRITE_SAME &&
-	    !bdev_get_queue(bio->bi_bdev)->limits.max_write_same_sectors)
+	    !bio->bi_disk->queue->limits.max_write_same_sectors)
 		mddev->queue->limits.max_write_same_sectors = 0;
 }
 
 static inline void mddev_check_write_zeroes(struct mddev *mddev, struct bio *bio)
 {
 	if (bio_op(bio) == REQ_OP_WRITE_ZEROES &&
-	    !bdev_get_queue(bio->bi_bdev)->limits.max_write_zeroes_sectors)
+	    !bio->bi_disk->queue->limits.max_write_zeroes_sectors)
 		mddev->queue->limits.max_write_zeroes_sectors = 0;
 }
 #endif /* _MD_MD_H */
diff --git a/drivers/md/multipath.c b/drivers/md/multipath.c
index 23a162ba6c56..b68e0666b9b0 100644
--- a/drivers/md/multipath.c
+++ b/drivers/md/multipath.c
@@ -134,7 +134,7 @@ static bool multipath_make_request(struct mddev *mddev, struct bio * bio)
 	__bio_clone_fast(&mp_bh->bio, bio);
 
 	mp_bh->bio.bi_iter.bi_sector += multipath->rdev->data_offset;
-	mp_bh->bio.bi_bdev = multipath->rdev->bdev;
+	bio_set_dev(&mp_bh->bio, multipath->rdev->bdev);
 	mp_bh->bio.bi_opf |= REQ_FAILFAST_TRANSPORT;
 	mp_bh->bio.bi_end_io = multipath_end_request;
 	mp_bh->bio.bi_private = mp_bh;
@@ -345,17 +345,17 @@ static void multipathd(struct md_thread *thread)
 
 		if ((mp_bh->path = multipath_map (conf))<0) {
 			pr_err("multipath: %s: unrecoverable IO read error for block %llu\n",
-			       bdevname(bio->bi_bdev,b),
+			       bio_devname(bio, b),
 			       (unsigned long long)bio->bi_iter.bi_sector);
 			multipath_end_bh_io(mp_bh, BLK_STS_IOERR);
 		} else {
 			pr_err("multipath: %s: redirecting sector %llu to another IO path\n",
-			       bdevname(bio->bi_bdev,b),
+			       bio_devname(bio, b),
 			       (unsigned long long)bio->bi_iter.bi_sector);
 			*bio = *(mp_bh->master_bio);
 			bio->bi_iter.bi_sector +=
 				conf->multipaths[mp_bh->path].rdev->data_offset;
-			bio->bi_bdev = conf->multipaths[mp_bh->path].rdev->bdev;
+			bio_set_dev(bio, conf->multipaths[mp_bh->path].rdev->bdev);
 			bio->bi_opf |= REQ_FAILFAST_TRANSPORT;
 			bio->bi_end_io = multipath_end_request;
 			bio->bi_private = mp_bh;
diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c
index 94d9ae9b0fd0..5a00fc118470 100644
--- a/drivers/md/raid0.c
+++ b/drivers/md/raid0.c
@@ -30,7 +30,8 @@
 	((1L << MD_HAS_JOURNAL) |	\
 	 (1L << MD_JOURNAL_CLEAN) |	\
 	 (1L << MD_FAILFAST_SUPPORTED) |\
-	 (1L << MD_HAS_PPL))
+	 (1L << MD_HAS_PPL) |		\
+	 (1L << MD_HAS_MULTIPLE_PPLS))
 
 static int raid0_congested(struct mddev *mddev, int bits)
 {
@@ -539,6 +540,7 @@ static void raid0_handle_discard(struct mddev *mddev, struct bio *bio)
 		    !discard_bio)
 			continue;
 		bio_chain(discard_bio, bio);
+		bio_clone_blkcg_association(discard_bio, bio);
 		if (mddev->gendisk)
 			trace_block_bio_remap(bdev_get_queue(rdev->bdev),
 				discard_bio, disk_devt(mddev->gendisk),
@@ -588,14 +590,13 @@ static bool raid0_make_request(struct mddev *mddev, struct bio *bio)
 
 	zone = find_zone(mddev->private, &sector);
 	tmp_dev = map_sector(mddev, zone, sector, &sector);
-	bio->bi_bdev = tmp_dev->bdev;
+	bio_set_dev(bio, tmp_dev->bdev);
 	bio->bi_iter.bi_sector = sector + zone->dev_start +
 		tmp_dev->data_offset;
 
 	if (mddev->gendisk)
-		trace_block_bio_remap(bdev_get_queue(bio->bi_bdev),
-				      bio, disk_devt(mddev->gendisk),
-				      bio_sector);
+		trace_block_bio_remap(bio->bi_disk->queue, bio,
+				disk_devt(mddev->gendisk), bio_sector);
 	mddev_check_writesame(mddev, bio);
 	mddev_check_write_zeroes(mddev, bio);
 	generic_make_request(bio);
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
index f50958ded9f0..f3f3e40dc9d8 100644
--- a/drivers/md/raid1.c
+++ b/drivers/md/raid1.c
@@ -48,7 +48,8 @@
 #define UNSUPPORTED_MDDEV_FLAGS		\
 	((1L << MD_HAS_JOURNAL) |	\
 	 (1L << MD_JOURNAL_CLEAN) |	\
-	 (1L << MD_HAS_PPL))
+	 (1L << MD_HAS_PPL) |		\
+	 (1L << MD_HAS_MULTIPLE_PPLS))
 
 /*
  * Number of guaranteed r1bios in case of extreme VM load:
@@ -786,13 +787,13 @@ static void flush_bio_list(struct r1conf *conf, struct bio *bio)
 
 	while (bio) { /* submit pending writes */
 		struct bio *next = bio->bi_next;
-		struct md_rdev *rdev = (void*)bio->bi_bdev;
+		struct md_rdev *rdev = (void *)bio->bi_disk;
 		bio->bi_next = NULL;
-		bio->bi_bdev = rdev->bdev;
+		bio_set_dev(bio, rdev->bdev);
 		if (test_bit(Faulty, &rdev->flags)) {
 			bio_io_error(bio);
 		} else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) &&
-				    !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
+				    !blk_queue_discard(bio->bi_disk->queue)))
 			/* Just ignore it */
 			bio_endio(bio);
 		else
@@ -1273,7 +1274,7 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio,
 
 	read_bio->bi_iter.bi_sector = r1_bio->sector +
 		mirror->rdev->data_offset;
-	read_bio->bi_bdev = mirror->rdev->bdev;
+	bio_set_dev(read_bio, mirror->rdev->bdev);
 	read_bio->bi_end_io = raid1_end_read_request;
 	bio_set_op_attrs(read_bio, op, do_sync);
 	if (test_bit(FailFast, &mirror->rdev->flags) &&
@@ -1282,9 +1283,8 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio,
 	read_bio->bi_private = r1_bio;
 
 	if (mddev->gendisk)
-	        trace_block_bio_remap(bdev_get_queue(read_bio->bi_bdev),
-	                              read_bio, disk_devt(mddev->gendisk),
-	                              r1_bio->sector);
+	        trace_block_bio_remap(read_bio->bi_disk->queue, read_bio,
+				disk_devt(mddev->gendisk), r1_bio->sector);
 
 	generic_make_request(read_bio);
 }
@@ -1496,7 +1496,7 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio,
 
 		mbio->bi_iter.bi_sector	= (r1_bio->sector +
 				   conf->mirrors[i].rdev->data_offset);
-		mbio->bi_bdev = conf->mirrors[i].rdev->bdev;
+		bio_set_dev(mbio, conf->mirrors[i].rdev->bdev);
 		mbio->bi_end_io	= raid1_end_write_request;
 		mbio->bi_opf = bio_op(bio) | (bio->bi_opf & (REQ_SYNC | REQ_FUA));
 		if (test_bit(FailFast, &conf->mirrors[i].rdev->flags) &&
@@ -1508,11 +1508,11 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio,
 		atomic_inc(&r1_bio->remaining);
 
 		if (mddev->gendisk)
-			trace_block_bio_remap(bdev_get_queue(mbio->bi_bdev),
+			trace_block_bio_remap(mbio->bi_disk->queue,
 					      mbio, disk_devt(mddev->gendisk),
 					      r1_bio->sector);
 		/* flush_pending_writes() needs access to the rdev so...*/
-		mbio->bi_bdev = (void*)conf->mirrors[i].rdev;
+		mbio->bi_disk = (void *)conf->mirrors[i].rdev;
 
 		cb = blk_check_plugged(raid1_unplug, mddev, sizeof(*plug));
 		if (cb)
@@ -1990,8 +1990,7 @@ static int fix_sync_read_error(struct r1bio *r1_bio)
 			 * Don't fail devices as that won't really help.
 			 */
 			pr_crit_ratelimited("md/raid1:%s: %s: unrecoverable I/O read error for block %llu\n",
-					    mdname(mddev),
-					    bdevname(bio->bi_bdev, b),
+					    mdname(mddev), bio_devname(bio, b),
 					    (unsigned long long)r1_bio->sector);
 			for (d = 0; d < conf->raid_disks * 2; d++) {
 				rdev = conf->mirrors[d].rdev;
@@ -2082,7 +2081,7 @@ static void process_checks(struct r1bio *r1_bio)
 		b->bi_status = status;
 		b->bi_iter.bi_sector = r1_bio->sector +
 			conf->mirrors[i].rdev->data_offset;
-		b->bi_bdev = conf->mirrors[i].rdev->bdev;
+		bio_set_dev(b, conf->mirrors[i].rdev->bdev);
 		b->bi_end_io = end_sync_read;
 		rp->raid_bio = r1_bio;
 		b->bi_private = rp;
@@ -2350,7 +2349,7 @@ static int narrow_write_error(struct r1bio *r1_bio, int i)
 
 		bio_trim(wbio, sector - r1_bio->sector, sectors);
 		wbio->bi_iter.bi_sector += rdev->data_offset;
-		wbio->bi_bdev = rdev->bdev;
+		bio_set_dev(wbio, rdev->bdev);
 
 		if (submit_bio_wait(wbio) < 0)
 			/* failure! */
@@ -2440,7 +2439,6 @@ static void handle_read_error(struct r1conf *conf, struct r1bio *r1_bio)
 	struct mddev *mddev = conf->mddev;
 	struct bio *bio;
 	struct md_rdev *rdev;
-	dev_t bio_dev;
 	sector_t bio_sector;
 
 	clear_bit(R1BIO_ReadError, &r1_bio->state);
@@ -2454,7 +2452,6 @@ static void handle_read_error(struct r1conf *conf, struct r1bio *r1_bio)
 	 */
 
 	bio = r1_bio->bios[r1_bio->read_disk];
-	bio_dev = bio->bi_bdev->bd_dev;
 	bio_sector = conf->mirrors[r1_bio->read_disk].rdev->data_offset + r1_bio->sector;
 	bio_put(bio);
 	r1_bio->bios[r1_bio->read_disk] = NULL;
@@ -2564,6 +2561,23 @@ static int init_resync(struct r1conf *conf)
 	return 0;
 }
 
+static struct r1bio *raid1_alloc_init_r1buf(struct r1conf *conf)
+{
+	struct r1bio *r1bio = mempool_alloc(conf->r1buf_pool, GFP_NOIO);
+	struct resync_pages *rps;
+	struct bio *bio;
+	int i;
+
+	for (i = conf->poolinfo->raid_disks; i--; ) {
+		bio = r1bio->bios[i];
+		rps = bio->bi_private;
+		bio_reset(bio);
+		bio->bi_private = rps;
+	}
+	r1bio->master_bio = NULL;
+	return r1bio;
+}
+
 /*
  * perform a "sync" on one "block"
  *
@@ -2649,7 +2663,7 @@ static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr,
 
 	bitmap_cond_end_sync(mddev->bitmap, sector_nr,
 		mddev_is_clustered(mddev) && (sector_nr + 2 * RESYNC_SECTORS > conf->cluster_sync_high));
-	r1_bio = mempool_alloc(conf->r1buf_pool, GFP_NOIO);
+	r1_bio = raid1_alloc_init_r1buf(conf);
 
 	raise_barrier(conf, sector_nr);
 
@@ -2727,7 +2741,7 @@ static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr,
 		if (bio->bi_end_io) {
 			atomic_inc(&rdev->nr_pending);
 			bio->bi_iter.bi_sector = sector_nr + rdev->data_offset;
-			bio->bi_bdev = rdev->bdev;
+			bio_set_dev(bio, rdev->bdev);
 			if (test_bit(FailFast, &rdev->flags))
 				bio->bi_opf |= MD_FAILFAST;
 		}
@@ -2853,7 +2867,7 @@ static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr,
 			bio = r1_bio->bios[i];
 			if (bio->bi_end_io == end_sync_read) {
 				read_targets--;
-				md_sync_acct(bio->bi_bdev, nr_sectors);
+				md_sync_acct_bio(bio, nr_sectors);
 				if (read_targets == 1)
 					bio->bi_opf &= ~MD_FAILFAST;
 				generic_make_request(bio);
@@ -2862,7 +2876,7 @@ static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr,
 	} else {
 		atomic_set(&r1_bio->remaining, 1);
 		bio = r1_bio->bios[r1_bio->read_disk];
-		md_sync_acct(bio->bi_bdev, nr_sectors);
+		md_sync_acct_bio(bio, nr_sectors);
 		if (read_targets == 1)
 			bio->bi_opf &= ~MD_FAILFAST;
 		generic_make_request(bio);
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c
index f55d4cc085f6..374df5796649 100644
--- a/drivers/md/raid10.c
+++ b/drivers/md/raid10.c
@@ -901,13 +901,13 @@ static void flush_pending_writes(struct r10conf *conf)
 
 		while (bio) { /* submit pending writes */
 			struct bio *next = bio->bi_next;
-			struct md_rdev *rdev = (void*)bio->bi_bdev;
+			struct md_rdev *rdev = (void*)bio->bi_disk;
 			bio->bi_next = NULL;
-			bio->bi_bdev = rdev->bdev;
+			bio_set_dev(bio, rdev->bdev);
 			if (test_bit(Faulty, &rdev->flags)) {
 				bio_io_error(bio);
 			} else if (unlikely((bio_op(bio) ==  REQ_OP_DISCARD) &&
-					    !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
+					    !blk_queue_discard(bio->bi_disk->queue)))
 				/* Just ignore it */
 				bio_endio(bio);
 			else
@@ -1085,13 +1085,13 @@ static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule)
 
 	while (bio) { /* submit pending writes */
 		struct bio *next = bio->bi_next;
-		struct md_rdev *rdev = (void*)bio->bi_bdev;
+		struct md_rdev *rdev = (void*)bio->bi_disk;
 		bio->bi_next = NULL;
-		bio->bi_bdev = rdev->bdev;
+		bio_set_dev(bio, rdev->bdev);
 		if (test_bit(Faulty, &rdev->flags)) {
 			bio_io_error(bio);
 		} else if (unlikely((bio_op(bio) ==  REQ_OP_DISCARD) &&
-				    !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
+				    !blk_queue_discard(bio->bi_disk->queue)))
 			/* Just ignore it */
 			bio_endio(bio);
 		else
@@ -1200,7 +1200,7 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio,
 
 	read_bio->bi_iter.bi_sector = r10_bio->devs[slot].addr +
 		choose_data_offset(r10_bio, rdev);
-	read_bio->bi_bdev = rdev->bdev;
+	bio_set_dev(read_bio, rdev->bdev);
 	read_bio->bi_end_io = raid10_end_read_request;
 	bio_set_op_attrs(read_bio, op, do_sync);
 	if (test_bit(FailFast, &rdev->flags) &&
@@ -1209,7 +1209,7 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio,
 	read_bio->bi_private = r10_bio;
 
 	if (mddev->gendisk)
-	        trace_block_bio_remap(bdev_get_queue(read_bio->bi_bdev),
+	        trace_block_bio_remap(read_bio->bi_disk->queue,
 	                              read_bio, disk_devt(mddev->gendisk),
 	                              r10_bio->sector);
 	generic_make_request(read_bio);
@@ -1249,7 +1249,7 @@ static void raid10_write_one_disk(struct mddev *mddev, struct r10bio *r10_bio,
 
 	mbio->bi_iter.bi_sector	= (r10_bio->devs[n_copy].addr +
 				   choose_data_offset(r10_bio, rdev));
-	mbio->bi_bdev = rdev->bdev;
+	bio_set_dev(mbio, rdev->bdev);
 	mbio->bi_end_io	= raid10_end_write_request;
 	bio_set_op_attrs(mbio, op, do_sync | do_fua);
 	if (!replacement && test_bit(FailFast,
@@ -1259,11 +1259,11 @@ static void raid10_write_one_disk(struct mddev *mddev, struct r10bio *r10_bio,
 	mbio->bi_private = r10_bio;
 
 	if (conf->mddev->gendisk)
-		trace_block_bio_remap(bdev_get_queue(mbio->bi_bdev),
+		trace_block_bio_remap(mbio->bi_disk->queue,
 				      mbio, disk_devt(conf->mddev->gendisk),
 				      r10_bio->sector);
 	/* flush_pending_writes() needs access to the rdev so...*/
-	mbio->bi_bdev = (void *)rdev;
+	mbio->bi_disk = (void *)rdev;
 
 	atomic_inc(&r10_bio->remaining);
 
@@ -2094,7 +2094,7 @@ static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio)
 		if (test_bit(FailFast, &conf->mirrors[d].rdev->flags))
 			tbio->bi_opf |= MD_FAILFAST;
 		tbio->bi_iter.bi_sector += conf->mirrors[d].rdev->data_offset;
-		tbio->bi_bdev = conf->mirrors[d].rdev->bdev;
+		bio_set_dev(tbio, conf->mirrors[d].rdev->bdev);
 		generic_make_request(tbio);
 	}
 
@@ -2552,7 +2552,7 @@ static int narrow_write_error(struct r10bio *r10_bio, int i)
 		wsector = r10_bio->devs[i].addr + (sector - r10_bio->sector);
 		wbio->bi_iter.bi_sector = wsector +
 				   choose_data_offset(r10_bio, rdev);
-		wbio->bi_bdev = rdev->bdev;
+		bio_set_dev(wbio, rdev->bdev);
 		bio_set_op_attrs(wbio, REQ_OP_WRITE, 0);
 
 		if (submit_bio_wait(wbio) < 0)
@@ -2575,7 +2575,6 @@ static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio)
 	struct bio *bio;
 	struct r10conf *conf = mddev->private;
 	struct md_rdev *rdev = r10_bio->devs[slot].rdev;
-	dev_t bio_dev;
 	sector_t bio_last_sector;
 
 	/* we got a read error. Maybe the drive is bad.  Maybe just
@@ -2587,7 +2586,6 @@ static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio)
 	 * frozen.
 	 */
 	bio = r10_bio->devs[slot].bio;
-	bio_dev = bio->bi_bdev->bd_dev;
 	bio_last_sector = r10_bio->devs[slot].addr + rdev->data_offset + r10_bio->sectors;
 	bio_put(bio);
 	r10_bio->devs[slot].bio = NULL;
@@ -2798,6 +2796,35 @@ static int init_resync(struct r10conf *conf)
 	return 0;
 }
 
+static struct r10bio *raid10_alloc_init_r10buf(struct r10conf *conf)
+{
+	struct r10bio *r10bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
+	struct rsync_pages *rp;
+	struct bio *bio;
+	int nalloc;
+	int i;
+
+	if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) ||
+	    test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery))
+		nalloc = conf->copies; /* resync */
+	else
+		nalloc = 2; /* recovery */
+
+	for (i = 0; i < nalloc; i++) {
+		bio = r10bio->devs[i].bio;
+		rp = bio->bi_private;
+		bio_reset(bio);
+		bio->bi_private = rp;
+		bio = r10bio->devs[i].repl_bio;
+		if (bio) {
+			rp = bio->bi_private;
+			bio_reset(bio);
+			bio->bi_private = rp;
+		}
+	}
+	return r10bio;
+}
+
 /*
  * perform a "sync" on one "block"
  *
@@ -2950,7 +2977,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
 
 	/* Again, very different code for resync and recovery.
 	 * Both must result in an r10bio with a list of bios that
-	 * have bi_end_io, bi_sector, bi_bdev set,
+	 * have bi_end_io, bi_sector, bi_disk set,
 	 * and bi_private set to the r10bio.
 	 * For recovery, we may actually create several r10bios
 	 * with 2 bios in each, that correspond to the bios in the main one.
@@ -3027,7 +3054,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
 				atomic_inc(&mreplace->nr_pending);
 			rcu_read_unlock();
 
-			r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
+			r10_bio = raid10_alloc_init_r10buf(conf);
 			r10_bio->state = 0;
 			raise_barrier(conf, rb2 != NULL);
 			atomic_set(&r10_bio->remaining, 0);
@@ -3095,7 +3122,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
 				from_addr = r10_bio->devs[j].addr;
 				bio->bi_iter.bi_sector = from_addr +
 					rdev->data_offset;
-				bio->bi_bdev = rdev->bdev;
+				bio_set_dev(bio, rdev->bdev);
 				atomic_inc(&rdev->nr_pending);
 				/* and we write to 'i' (if not in_sync) */
 
@@ -3117,7 +3144,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
 					bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
 					bio->bi_iter.bi_sector = to_addr
 						+ mrdev->data_offset;
-					bio->bi_bdev = mrdev->bdev;
+					bio_set_dev(bio, mrdev->bdev);
 					atomic_inc(&r10_bio->remaining);
 				} else
 					r10_bio->devs[1].bio->bi_end_io = NULL;
@@ -3143,7 +3170,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
 				bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
 				bio->bi_iter.bi_sector = to_addr +
 					mreplace->data_offset;
-				bio->bi_bdev = mreplace->bdev;
+				bio_set_dev(bio, mreplace->bdev);
 				atomic_inc(&r10_bio->remaining);
 				break;
 			}
@@ -3236,7 +3263,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
 		}
 		if (sync_blocks < max_sync)
 			max_sync = sync_blocks;
-		r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
+		r10_bio = raid10_alloc_init_r10buf(conf);
 		r10_bio->state = 0;
 
 		r10_bio->mddev = mddev;
@@ -3289,7 +3316,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
 			if (test_bit(FailFast, &rdev->flags))
 				bio->bi_opf |= MD_FAILFAST;
 			bio->bi_iter.bi_sector = sector + rdev->data_offset;
-			bio->bi_bdev = rdev->bdev;
+			bio_set_dev(bio, rdev->bdev);
 			count++;
 
 			rdev = rcu_dereference(conf->mirrors[d].replacement);
@@ -3311,7 +3338,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
 			if (test_bit(FailFast, &rdev->flags))
 				bio->bi_opf |= MD_FAILFAST;
 			bio->bi_iter.bi_sector = sector + rdev->data_offset;
-			bio->bi_bdev = rdev->bdev;
+			bio_set_dev(bio, rdev->bdev);
 			count++;
 			rcu_read_unlock();
 		}
@@ -3367,7 +3394,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
 		r10_bio->sectors = nr_sectors;
 
 		if (bio->bi_end_io == end_sync_read) {
-			md_sync_acct(bio->bi_bdev, nr_sectors);
+			md_sync_acct_bio(bio, nr_sectors);
 			bio->bi_status = 0;
 			generic_make_request(bio);
 		}
@@ -4360,7 +4387,7 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
 
 read_more:
 	/* Now schedule reads for blocks from sector_nr to last */
-	r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
+	r10_bio = raid10_alloc_init_r10buf(conf);
 	r10_bio->state = 0;
 	raise_barrier(conf, sectors_done != 0);
 	atomic_set(&r10_bio->remaining, 0);
@@ -4383,7 +4410,7 @@ read_more:
 
 	read_bio = bio_alloc_mddev(GFP_KERNEL, RESYNC_PAGES, mddev);
 
-	read_bio->bi_bdev = rdev->bdev;
+	bio_set_dev(read_bio, rdev->bdev);
 	read_bio->bi_iter.bi_sector = (r10_bio->devs[r10_bio->read_slot].addr
 			       + rdev->data_offset);
 	read_bio->bi_private = r10_bio;
@@ -4417,7 +4444,7 @@ read_more:
 		if (!rdev2 || test_bit(Faulty, &rdev2->flags))
 			continue;
 
-		b->bi_bdev = rdev2->bdev;
+		bio_set_dev(b, rdev2->bdev);
 		b->bi_iter.bi_sector = r10_bio->devs[s/2].addr +
 			rdev2->new_data_offset;
 		b->bi_end_io = end_reshape_write;
@@ -4449,7 +4476,7 @@ read_more:
 	r10_bio->sectors = nr_sectors;
 
 	/* Now submit the read */
-	md_sync_acct(read_bio->bi_bdev, r10_bio->sectors);
+	md_sync_acct_bio(read_bio, r10_bio->sectors);
 	atomic_inc(&r10_bio->remaining);
 	read_bio->bi_next = NULL;
 	generic_make_request(read_bio);
@@ -4511,7 +4538,7 @@ static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio)
 		}
 		atomic_inc(&rdev->nr_pending);
 		rcu_read_unlock();
-		md_sync_acct(b->bi_bdev, r10_bio->sectors);
+		md_sync_acct_bio(b, r10_bio->sectors);
 		atomic_inc(&r10_bio->remaining);
 		b->bi_next = NULL;
 		generic_make_request(b);
diff --git a/drivers/md/raid5-cache.c b/drivers/md/raid5-cache.c
index bfa1e907c472..0b7406ac8ce1 100644
--- a/drivers/md/raid5-cache.c
+++ b/drivers/md/raid5-cache.c
@@ -236,9 +236,10 @@ struct r5l_io_unit {
 	bool need_split_bio;
 	struct bio *split_bio;
 
-	unsigned int has_flush:1;      /* include flush request */
-	unsigned int has_fua:1;        /* include fua request */
-	unsigned int has_null_flush:1; /* include empty flush request */
+	unsigned int has_flush:1;		/* include flush request */
+	unsigned int has_fua:1;			/* include fua request */
+	unsigned int has_null_flush:1;		/* include null flush request */
+	unsigned int has_flush_payload:1;	/* include flush payload  */
 	/*
 	 * io isn't sent yet, flush/fua request can only be submitted till it's
 	 * the first IO in running_ios list
@@ -571,6 +572,8 @@ static void r5l_log_endio(struct bio *bio)
 	struct r5l_io_unit *io_deferred;
 	struct r5l_log *log = io->log;
 	unsigned long flags;
+	bool has_null_flush;
+	bool has_flush_payload;
 
 	if (bio->bi_status)
 		md_error(log->rdev->mddev, log->rdev);
@@ -580,6 +583,16 @@ static void r5l_log_endio(struct bio *bio)
 
 	spin_lock_irqsave(&log->io_list_lock, flags);
 	__r5l_set_io_unit_state(io, IO_UNIT_IO_END);
+
+	/*
+	 * if the io doesn't not have null_flush or flush payload,
+	 * it is not safe to access it after releasing io_list_lock.
+	 * Therefore, it is necessary to check the condition with
+	 * the lock held.
+	 */
+	has_null_flush = io->has_null_flush;
+	has_flush_payload = io->has_flush_payload;
+
 	if (log->need_cache_flush && !list_empty(&io->stripe_list))
 		r5l_move_to_end_ios(log);
 	else
@@ -600,19 +613,23 @@ static void r5l_log_endio(struct bio *bio)
 	if (log->need_cache_flush)
 		md_wakeup_thread(log->rdev->mddev->thread);
 
-	if (io->has_null_flush) {
+	/* finish flush only io_unit and PAYLOAD_FLUSH only io_unit */
+	if (has_null_flush) {
 		struct bio *bi;
 
 		WARN_ON(bio_list_empty(&io->flush_barriers));
 		while ((bi = bio_list_pop(&io->flush_barriers)) != NULL) {
 			bio_endio(bi);
-			atomic_dec(&io->pending_stripe);
+			if (atomic_dec_and_test(&io->pending_stripe)) {
+				__r5l_stripe_write_finished(io);
+				return;
+			}
 		}
 	}
-
-	/* finish flush only io_unit and PAYLOAD_FLUSH only io_unit */
-	if (atomic_read(&io->pending_stripe) == 0)
-		__r5l_stripe_write_finished(io);
+	/* decrease pending_stripe for flush payload */
+	if (has_flush_payload)
+		if (atomic_dec_and_test(&io->pending_stripe))
+			__r5l_stripe_write_finished(io);
 }
 
 static void r5l_do_submit_io(struct r5l_log *log, struct r5l_io_unit *io)
@@ -728,7 +745,7 @@ static struct bio *r5l_bio_alloc(struct r5l_log *log)
 	struct bio *bio = bio_alloc_bioset(GFP_NOIO, BIO_MAX_PAGES, log->bs);
 
 	bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
-	bio->bi_bdev = log->rdev->bdev;
+	bio_set_dev(bio, log->rdev->bdev);
 	bio->bi_iter.bi_sector = log->rdev->data_offset + log->log_start;
 
 	return bio;
@@ -881,6 +898,11 @@ static void r5l_append_flush_payload(struct r5l_log *log, sector_t sect)
 	payload->size = cpu_to_le32(sizeof(__le64));
 	payload->flush_stripes[0] = cpu_to_le64(sect);
 	io->meta_offset += meta_size;
+	/* multiple flush payloads count as one pending_stripe */
+	if (!io->has_flush_payload) {
+		io->has_flush_payload = 1;
+		atomic_inc(&io->pending_stripe);
+	}
 	mutex_unlock(&log->io_mutex);
 }
 
@@ -1291,7 +1313,7 @@ void r5l_flush_stripe_to_raid(struct r5l_log *log)
 	if (!do_flush)
 		return;
 	bio_reset(&log->flush_bio);
-	log->flush_bio.bi_bdev = log->rdev->bdev;
+	bio_set_dev(&log->flush_bio, log->rdev->bdev);
 	log->flush_bio.bi_end_io = r5l_log_flush_endio;
 	log->flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
 	submit_bio(&log->flush_bio);
@@ -1669,7 +1691,7 @@ static int r5l_recovery_fetch_ra_pool(struct r5l_log *log,
 				      sector_t offset)
 {
 	bio_reset(ctx->ra_bio);
-	ctx->ra_bio->bi_bdev = log->rdev->bdev;
+	bio_set_dev(ctx->ra_bio, log->rdev->bdev);
 	bio_set_op_attrs(ctx->ra_bio, REQ_OP_READ, 0);
 	ctx->ra_bio->bi_iter.bi_sector = log->rdev->data_offset + offset;
 
@@ -2507,11 +2529,18 @@ static void r5l_write_super(struct r5l_log *log, sector_t cp)
 
 static ssize_t r5c_journal_mode_show(struct mddev *mddev, char *page)
 {
-	struct r5conf *conf = mddev->private;
+	struct r5conf *conf;
 	int ret;
 
-	if (!conf->log)
+	ret = mddev_lock(mddev);
+	if (ret)
+		return ret;
+
+	conf = mddev->private;
+	if (!conf || !conf->log) {
+		mddev_unlock(mddev);
 		return 0;
+	}
 
 	switch (conf->log->r5c_journal_mode) {
 	case R5C_JOURNAL_MODE_WRITE_THROUGH:
@@ -2529,6 +2558,7 @@ static ssize_t r5c_journal_mode_show(struct mddev *mddev, char *page)
 	default:
 		ret = 0;
 	}
+	mddev_unlock(mddev);
 	return ret;
 }
 
@@ -2540,23 +2570,32 @@ static ssize_t r5c_journal_mode_show(struct mddev *mddev, char *page)
  */
 int r5c_journal_mode_set(struct mddev *mddev, int mode)
 {
-	struct r5conf *conf = mddev->private;
-	struct r5l_log *log = conf->log;
-
-	if (!log)
-		return -ENODEV;
+	struct r5conf *conf;
+	int err;
 
 	if (mode < R5C_JOURNAL_MODE_WRITE_THROUGH ||
 	    mode > R5C_JOURNAL_MODE_WRITE_BACK)
 		return -EINVAL;
 
+	err = mddev_lock(mddev);
+	if (err)
+		return err;
+	conf = mddev->private;
+	if (!conf || !conf->log) {
+		mddev_unlock(mddev);
+		return -ENODEV;
+	}
+
 	if (raid5_calc_degraded(conf) > 0 &&
-	    mode == R5C_JOURNAL_MODE_WRITE_BACK)
+	    mode == R5C_JOURNAL_MODE_WRITE_BACK) {
+		mddev_unlock(mddev);
 		return -EINVAL;
+	}
 
 	mddev_suspend(mddev);
 	conf->log->r5c_journal_mode = mode;
 	mddev_resume(mddev);
+	mddev_unlock(mddev);
 
 	pr_debug("md/raid:%s: setting r5c cache mode to %d: %s\n",
 		 mdname(mddev), mode, r5c_journal_mode_str[mode]);
diff --git a/drivers/md/raid5-ppl.c b/drivers/md/raid5-ppl.c
index 44ad5baf3206..cd026c88f7ef 100644
--- a/drivers/md/raid5-ppl.c
+++ b/drivers/md/raid5-ppl.c
@@ -87,6 +87,8 @@
  * The current io_unit accepting new stripes is always at the end of the list.
  */
 
+#define PPL_SPACE_SIZE (128 * 1024)
+
 struct ppl_conf {
 	struct mddev *mddev;
 
@@ -122,6 +124,10 @@ struct ppl_log {
 					 * always at the end of io_list */
 	spinlock_t io_list_lock;
 	struct list_head io_list;	/* all io_units of this log */
+
+	sector_t next_io_sector;
+	unsigned int entry_space;
+	bool use_multippl;
 };
 
 #define PPL_IO_INLINE_BVECS 32
@@ -264,13 +270,12 @@ static int ppl_log_stripe(struct ppl_log *log, struct stripe_head *sh)
 	int i;
 	sector_t data_sector = 0;
 	int data_disks = 0;
-	unsigned int entry_space = (log->rdev->ppl.size << 9) - PPL_HEADER_SIZE;
 	struct r5conf *conf = sh->raid_conf;
 
 	pr_debug("%s: stripe: %llu\n", __func__, (unsigned long long)sh->sector);
 
 	/* check if current io_unit is full */
-	if (io && (io->pp_size == entry_space ||
+	if (io && (io->pp_size == log->entry_space ||
 		   io->entries_count == PPL_HDR_MAX_ENTRIES)) {
 		pr_debug("%s: add io_unit blocked by seq: %llu\n",
 			 __func__, io->seq);
@@ -415,7 +420,7 @@ static void ppl_submit_iounit_bio(struct ppl_io_unit *io, struct bio *bio)
 	pr_debug("%s: seq: %llu size: %u sector: %llu dev: %s\n",
 		 __func__, io->seq, bio->bi_iter.bi_size,
 		 (unsigned long long)bio->bi_iter.bi_sector,
-		 bdevname(bio->bi_bdev, b));
+		 bio_devname(bio, b));
 
 	submit_bio(bio);
 }
@@ -451,12 +456,25 @@ static void ppl_submit_iounit(struct ppl_io_unit *io)
 	pplhdr->entries_count = cpu_to_le32(io->entries_count);
 	pplhdr->checksum = cpu_to_le32(~crc32c_le(~0, pplhdr, PPL_HEADER_SIZE));
 
+	/* Rewind the buffer if current PPL is larger then remaining space */
+	if (log->use_multippl &&
+	    log->rdev->ppl.sector + log->rdev->ppl.size - log->next_io_sector <
+	    (PPL_HEADER_SIZE + io->pp_size) >> 9)
+		log->next_io_sector = log->rdev->ppl.sector;
+
+
 	bio->bi_end_io = ppl_log_endio;
 	bio->bi_opf = REQ_OP_WRITE | REQ_FUA;
-	bio->bi_bdev = log->rdev->bdev;
-	bio->bi_iter.bi_sector = log->rdev->ppl.sector;
+	bio_set_dev(bio, log->rdev->bdev);
+	bio->bi_iter.bi_sector = log->next_io_sector;
 	bio_add_page(bio, io->header_page, PAGE_SIZE, 0);
 
+	pr_debug("%s: log->current_io_sector: %llu\n", __func__,
+	    (unsigned long long)log->next_io_sector);
+
+	if (log->use_multippl)
+		log->next_io_sector += (PPL_HEADER_SIZE + io->pp_size) >> 9;
+
 	list_for_each_entry(sh, &io->stripe_list, log_list) {
 		/* entries for full stripe writes have no partial parity */
 		if (test_bit(STRIPE_FULL_WRITE, &sh->state))
@@ -468,7 +486,7 @@ static void ppl_submit_iounit(struct ppl_io_unit *io)
 			bio = bio_alloc_bioset(GFP_NOIO, BIO_MAX_PAGES,
 					       ppl_conf->bs);
 			bio->bi_opf = prev->bi_opf;
-			bio->bi_bdev = prev->bi_bdev;
+			bio_copy_dev(bio, prev);
 			bio->bi_iter.bi_sector = bio_end_sector(prev);
 			bio_add_page(bio, sh->ppl_page, PAGE_SIZE, 0);
 
@@ -813,12 +831,14 @@ out:
 	return ret;
 }
 
-static int ppl_recover(struct ppl_log *log, struct ppl_header *pplhdr)
+static int ppl_recover(struct ppl_log *log, struct ppl_header *pplhdr,
+		       sector_t offset)
 {
 	struct ppl_conf *ppl_conf = log->ppl_conf;
 	struct md_rdev *rdev = log->rdev;
 	struct mddev *mddev = rdev->mddev;
-	sector_t ppl_sector = rdev->ppl.sector + (PPL_HEADER_SIZE >> 9);
+	sector_t ppl_sector = rdev->ppl.sector + offset +
+			      (PPL_HEADER_SIZE >> 9);
 	struct page *page;
 	int i;
 	int ret = 0;
@@ -902,6 +922,9 @@ static int ppl_write_empty_header(struct ppl_log *log)
 		return -ENOMEM;
 
 	pplhdr = page_address(page);
+	/* zero out PPL space to avoid collision with old PPLs */
+	blkdev_issue_zeroout(rdev->bdev, rdev->ppl.sector,
+			    log->rdev->ppl.size, GFP_NOIO, 0);
 	memset(pplhdr->reserved, 0xff, PPL_HDR_RESERVED);
 	pplhdr->signature = cpu_to_le32(log->ppl_conf->signature);
 	pplhdr->checksum = cpu_to_le32(~crc32c_le(~0, pplhdr, PAGE_SIZE));
@@ -922,63 +945,110 @@ static int ppl_load_distributed(struct ppl_log *log)
 	struct ppl_conf *ppl_conf = log->ppl_conf;
 	struct md_rdev *rdev = log->rdev;
 	struct mddev *mddev = rdev->mddev;
-	struct page *page;
-	struct ppl_header *pplhdr;
+	struct page *page, *page2, *tmp;
+	struct ppl_header *pplhdr = NULL, *prev_pplhdr = NULL;
 	u32 crc, crc_stored;
 	u32 signature;
-	int ret = 0;
+	int ret = 0, i;
+	sector_t pplhdr_offset = 0, prev_pplhdr_offset = 0;
 
 	pr_debug("%s: disk: %d\n", __func__, rdev->raid_disk);
-
-	/* read PPL header */
+	/* read PPL headers, find the recent one */
 	page = alloc_page(GFP_KERNEL);
 	if (!page)
 		return -ENOMEM;
 
-	if (!sync_page_io(rdev, rdev->ppl.sector - rdev->data_offset,
-			  PAGE_SIZE, page, REQ_OP_READ, 0, false)) {
-		md_error(mddev, rdev);
-		ret = -EIO;
-		goto out;
+	page2 = alloc_page(GFP_KERNEL);
+	if (!page2) {
+		__free_page(page);
+		return -ENOMEM;
 	}
-	pplhdr = page_address(page);
 
-	/* check header validity */
-	crc_stored = le32_to_cpu(pplhdr->checksum);
-	pplhdr->checksum = 0;
-	crc = ~crc32c_le(~0, pplhdr, PAGE_SIZE);
+	/* searching ppl area for latest ppl */
+	while (pplhdr_offset < rdev->ppl.size - (PPL_HEADER_SIZE >> 9)) {
+		if (!sync_page_io(rdev,
+				  rdev->ppl.sector - rdev->data_offset +
+				  pplhdr_offset, PAGE_SIZE, page, REQ_OP_READ,
+				  0, false)) {
+			md_error(mddev, rdev);
+			ret = -EIO;
+			/* if not able to read - don't recover any PPL */
+			pplhdr = NULL;
+			break;
+		}
+		pplhdr = page_address(page);
+
+		/* check header validity */
+		crc_stored = le32_to_cpu(pplhdr->checksum);
+		pplhdr->checksum = 0;
+		crc = ~crc32c_le(~0, pplhdr, PAGE_SIZE);
+
+		if (crc_stored != crc) {
+			pr_debug("%s: ppl header crc does not match: stored: 0x%x calculated: 0x%x (offset: %llu)\n",
+				 __func__, crc_stored, crc,
+				 (unsigned long long)pplhdr_offset);
+			pplhdr = prev_pplhdr;
+			pplhdr_offset = prev_pplhdr_offset;
+			break;
+		}
 
-	if (crc_stored != crc) {
-		pr_debug("%s: ppl header crc does not match: stored: 0x%x calculated: 0x%x\n",
-			 __func__, crc_stored, crc);
-		ppl_conf->mismatch_count++;
-		goto out;
-	}
+		signature = le32_to_cpu(pplhdr->signature);
 
-	signature = le32_to_cpu(pplhdr->signature);
+		if (mddev->external) {
+			/*
+			 * For external metadata the header signature is set and
+			 * validated in userspace.
+			 */
+			ppl_conf->signature = signature;
+		} else if (ppl_conf->signature != signature) {
+			pr_debug("%s: ppl header signature does not match: stored: 0x%x configured: 0x%x (offset: %llu)\n",
+				 __func__, signature, ppl_conf->signature,
+				 (unsigned long long)pplhdr_offset);
+			pplhdr = prev_pplhdr;
+			pplhdr_offset = prev_pplhdr_offset;
+			break;
+		}
 
-	if (mddev->external) {
-		/*
-		 * For external metadata the header signature is set and
-		 * validated in userspace.
-		 */
-		ppl_conf->signature = signature;
-	} else if (ppl_conf->signature != signature) {
-		pr_debug("%s: ppl header signature does not match: stored: 0x%x configured: 0x%x\n",
-			 __func__, signature, ppl_conf->signature);
-		ppl_conf->mismatch_count++;
-		goto out;
+		if (prev_pplhdr && le64_to_cpu(prev_pplhdr->generation) >
+		    le64_to_cpu(pplhdr->generation)) {
+			/* previous was newest */
+			pplhdr = prev_pplhdr;
+			pplhdr_offset = prev_pplhdr_offset;
+			break;
+		}
+
+		prev_pplhdr_offset = pplhdr_offset;
+		prev_pplhdr = pplhdr;
+
+		tmp = page;
+		page = page2;
+		page2 = tmp;
+
+		/* calculate next potential ppl offset */
+		for (i = 0; i < le32_to_cpu(pplhdr->entries_count); i++)
+			pplhdr_offset +=
+			    le32_to_cpu(pplhdr->entries[i].pp_size) >> 9;
+		pplhdr_offset += PPL_HEADER_SIZE >> 9;
 	}
 
+	/* no valid ppl found */
+	if (!pplhdr)
+		ppl_conf->mismatch_count++;
+	else
+		pr_debug("%s: latest PPL found at offset: %llu, with generation: %llu\n",
+		    __func__, (unsigned long long)pplhdr_offset,
+		    le64_to_cpu(pplhdr->generation));
+
 	/* attempt to recover from log if we are starting a dirty array */
-	if (!mddev->pers && mddev->recovery_cp != MaxSector)
-		ret = ppl_recover(log, pplhdr);
-out:
+	if (pplhdr && !mddev->pers && mddev->recovery_cp != MaxSector)
+		ret = ppl_recover(log, pplhdr, pplhdr_offset);
+
 	/* write empty header if we are starting the array */
 	if (!ret && !mddev->pers)
 		ret = ppl_write_empty_header(log);
 
 	__free_page(page);
+	__free_page(page2);
 
 	pr_debug("%s: return: %d mismatch_count: %d recovered_entries: %d\n",
 		 __func__, ret, ppl_conf->mismatch_count,
@@ -1031,6 +1101,7 @@ static int ppl_load(struct ppl_conf *ppl_conf)
 static void __ppl_exit_log(struct ppl_conf *ppl_conf)
 {
 	clear_bit(MD_HAS_PPL, &ppl_conf->mddev->flags);
+	clear_bit(MD_HAS_MULTIPLE_PPLS, &ppl_conf->mddev->flags);
 
 	kfree(ppl_conf->child_logs);
 
@@ -1099,6 +1170,22 @@ static int ppl_validate_rdev(struct md_rdev *rdev)
 	return 0;
 }
 
+static void ppl_init_child_log(struct ppl_log *log, struct md_rdev *rdev)
+{
+	if ((rdev->ppl.size << 9) >= (PPL_SPACE_SIZE +
+				      PPL_HEADER_SIZE) * 2) {
+		log->use_multippl = true;
+		set_bit(MD_HAS_MULTIPLE_PPLS,
+			&log->ppl_conf->mddev->flags);
+		log->entry_space = PPL_SPACE_SIZE;
+	} else {
+		log->use_multippl = false;
+		log->entry_space = (log->rdev->ppl.size << 9) -
+				   PPL_HEADER_SIZE;
+	}
+	log->next_io_sector = rdev->ppl.sector;
+}
+
 int ppl_init_log(struct r5conf *conf)
 {
 	struct ppl_conf *ppl_conf;
@@ -1196,6 +1283,7 @@ int ppl_init_log(struct r5conf *conf)
 			q = bdev_get_queue(rdev->bdev);
 			if (test_bit(QUEUE_FLAG_WC, &q->queue_flags))
 				need_cache_flush = true;
+			ppl_init_child_log(log, rdev);
 		}
 	}
 
@@ -1261,6 +1349,7 @@ int ppl_modify_log(struct r5conf *conf, struct md_rdev *rdev, bool add)
 		if (!ret) {
 			log->rdev = rdev;
 			ret = ppl_write_empty_header(log);
+			ppl_init_child_log(log, rdev);
 		}
 	} else {
 		log->rdev = NULL;
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index 0fc2748aaf95..076409455b60 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -494,7 +494,6 @@ static int grow_buffers(struct stripe_head *sh, gfp_t gfp)
 	return 0;
 }
 
-static void raid5_build_block(struct stripe_head *sh, int i, int previous);
 static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
 			    struct stripe_head *sh);
 
@@ -530,7 +529,7 @@ retry:
 			WARN_ON(1);
 		}
 		dev->flags = 0;
-		raid5_build_block(sh, i, previous);
+		dev->sector = raid5_compute_blocknr(sh, i, previous);
 	}
 	if (read_seqcount_retry(&conf->gen_lock, seq))
 		goto retry;
@@ -812,6 +811,14 @@ static void stripe_add_to_batch_list(struct r5conf *conf, struct stripe_head *sh
 			spin_unlock(&head->batch_head->batch_lock);
 			goto unlock_out;
 		}
+		/*
+		 * We must assign batch_head of this stripe within the
+		 * batch_lock, otherwise clear_batch_ready of batch head
+		 * stripe could clear BATCH_READY bit of this stripe and
+		 * this stripe->batch_head doesn't get assigned, which
+		 * could confuse clear_batch_ready for this stripe
+		 */
+		sh->batch_head = head->batch_head;
 
 		/*
 		 * at this point, head's BATCH_READY could be cleared, but we
@@ -819,8 +826,6 @@ static void stripe_add_to_batch_list(struct r5conf *conf, struct stripe_head *sh
 		 */
 		list_add(&sh->batch_list, &head->batch_list);
 		spin_unlock(&head->batch_head->batch_lock);
-
-		sh->batch_head = head->batch_head;
 	} else {
 		head->batch_head = head;
 		sh->batch_head = head->batch_head;
@@ -1096,7 +1101,7 @@ again:
 
 			set_bit(STRIPE_IO_STARTED, &sh->state);
 
-			bi->bi_bdev = rdev->bdev;
+			bio_set_dev(bi, rdev->bdev);
 			bio_set_op_attrs(bi, op, op_flags);
 			bi->bi_end_io = op_is_write(op)
 				? raid5_end_write_request
@@ -1145,7 +1150,7 @@ again:
 				set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags);
 
 			if (conf->mddev->gendisk)
-				trace_block_bio_remap(bdev_get_queue(bi->bi_bdev),
+				trace_block_bio_remap(bi->bi_disk->queue,
 						      bi, disk_devt(conf->mddev->gendisk),
 						      sh->dev[i].sector);
 			if (should_defer && op_is_write(op))
@@ -1160,7 +1165,7 @@ again:
 
 			set_bit(STRIPE_IO_STARTED, &sh->state);
 
-			rbi->bi_bdev = rrdev->bdev;
+			bio_set_dev(rbi, rrdev->bdev);
 			bio_set_op_attrs(rbi, op, op_flags);
 			BUG_ON(!op_is_write(op));
 			rbi->bi_end_io = raid5_end_write_request;
@@ -1193,7 +1198,7 @@ again:
 			if (op == REQ_OP_DISCARD)
 				rbi->bi_vcnt = 0;
 			if (conf->mddev->gendisk)
-				trace_block_bio_remap(bdev_get_queue(rbi->bi_bdev),
+				trace_block_bio_remap(rbi->bi_disk->queue,
 						      rbi, disk_devt(conf->mddev->gendisk),
 						      sh->dev[i].sector);
 			if (should_defer && op_is_write(op))
@@ -2662,14 +2667,6 @@ static void raid5_end_write_request(struct bio *bi)
 		raid5_release_stripe(sh->batch_head);
 }
 
-static void raid5_build_block(struct stripe_head *sh, int i, int previous)
-{
-	struct r5dev *dev = &sh->dev[i];
-
-	dev->flags = 0;
-	dev->sector = raid5_compute_blocknr(sh, i, previous);
-}
-
 static void raid5_error(struct mddev *mddev, struct md_rdev *rdev)
 {
 	char b[BDEVNAME_SIZE];
@@ -4608,7 +4605,8 @@ static void break_stripe_batch_list(struct stripe_head *head_sh,
 
 		set_mask_bits(&sh->state, ~(STRIPE_EXPAND_SYNC_FLAGS |
 					    (1 << STRIPE_PREREAD_ACTIVE) |
-					    (1 << STRIPE_DEGRADED)),
+					    (1 << STRIPE_DEGRADED) |
+					    (1 << STRIPE_ON_UNPLUG_LIST)),
 			      head_sh->state & (1 << STRIPE_INSYNC));
 
 		sh->check_state = head_sh->check_state;
@@ -5092,10 +5090,12 @@ static int raid5_congested(struct mddev *mddev, int bits)
 static int in_chunk_boundary(struct mddev *mddev, struct bio *bio)
 {
 	struct r5conf *conf = mddev->private;
-	sector_t sector = bio->bi_iter.bi_sector + get_start_sect(bio->bi_bdev);
+	sector_t sector = bio->bi_iter.bi_sector;
 	unsigned int chunk_sectors;
 	unsigned int bio_sectors = bio_sectors(bio);
 
+	WARN_ON_ONCE(bio->bi_partno);
+
 	chunk_sectors = min(conf->chunk_sectors, conf->prev_chunk_sectors);
 	return  chunk_sectors >=
 		((sector & (chunk_sectors - 1)) + bio_sectors);
@@ -5231,7 +5231,7 @@ static int raid5_read_one_chunk(struct mddev *mddev, struct bio *raid_bio)
 		atomic_inc(&rdev->nr_pending);
 		rcu_read_unlock();
 		raid_bio->bi_next = (void*)rdev;
-		align_bi->bi_bdev =  rdev->bdev;
+		bio_set_dev(align_bi, rdev->bdev);
 		bio_clear_flag(align_bi, BIO_SEG_VALID);
 
 		if (is_badblock(rdev, align_bi->bi_iter.bi_sector,
@@ -5253,7 +5253,7 @@ static int raid5_read_one_chunk(struct mddev *mddev, struct bio *raid_bio)
 		spin_unlock_irq(&conf->device_lock);
 
 		if (mddev->gendisk)
-			trace_block_bio_remap(bdev_get_queue(align_bi->bi_bdev),
+			trace_block_bio_remap(align_bi->bi_disk->queue,
 					      align_bi, disk_devt(mddev->gendisk),
 					      raid_bio->bi_iter.bi_sector);
 		generic_make_request(align_bi);
@@ -6235,6 +6235,10 @@ static void raid5_do_work(struct work_struct *work)
 
 	spin_unlock_irq(&conf->device_lock);
 
+	flush_deferred_bios(conf);
+
+	r5l_flush_stripe_to_raid(conf->log);
+
 	async_tx_issue_pending_all();
 	blk_finish_plug(&plug);
 
@@ -7241,6 +7245,7 @@ static int raid5_run(struct mddev *mddev)
 		pr_warn("md/raid:%s: using journal device and PPL not allowed - disabling PPL\n",
 			mdname(mddev));
 		clear_bit(MD_HAS_PPL, &mddev->flags);
+		clear_bit(MD_HAS_MULTIPLE_PPLS, &mddev->flags);
 	}
 
 	if (mddev->private == NULL)
diff --git a/drivers/media/Kconfig b/drivers/media/Kconfig
index 55d9c2b82b7e..edfe99b22d56 100644
--- a/drivers/media/Kconfig
+++ b/drivers/media/Kconfig
@@ -8,6 +8,11 @@ config CEC_CORE
 config CEC_NOTIFIER
 	bool
 
+config CEC_PIN
+	bool
+
+source "drivers/media/rc/Kconfig"
+
 menuconfig MEDIA_SUPPORT
 	tristate "Multimedia support"
 	depends on HAS_IOMEM
@@ -72,20 +77,6 @@ config MEDIA_SDR_SUPPORT
 
 	  Say Y when you have a software defined radio device.
 
-config MEDIA_RC_SUPPORT
-	bool "Remote Controller support"
-	depends on INPUT
-	---help---
-	  Enable support for Remote Controllers on Linux. This is
-	  needed in order to support several video capture adapters,
-	  standalone IR receivers/transmitters, and RF receivers.
-
-	  Enable this option if you have a video capture board even
-	  if you don't need IR, as otherwise, you may not be able to
-	  compile the driver for your adapter.
-
-	  Say Y when you have a TV or an IR device.
-
 config MEDIA_CEC_SUPPORT
        bool "HDMI CEC support"
        ---help---
@@ -175,7 +166,6 @@ config TTPCI_EEPROM
 source "drivers/media/dvb-core/Kconfig"
 
 comment "Media drivers"
-source "drivers/media/rc/Kconfig"
 
 #
 # V4L platform/mem2mem drivers
diff --git a/drivers/media/cec/Makefile b/drivers/media/cec/Makefile
index eaf408e64669..3353c1741961 100644
--- a/drivers/media/cec/Makefile
+++ b/drivers/media/cec/Makefile
@@ -4,4 +4,8 @@ ifeq ($(CONFIG_CEC_NOTIFIER),y)
   cec-objs += cec-notifier.o
 endif
 
+ifeq ($(CONFIG_CEC_PIN),y)
+  cec-objs += cec-pin.o
+endif
+
 obj-$(CONFIG_CEC_CORE) += cec.o
diff --git a/drivers/media/cec/cec-adap.c b/drivers/media/cec/cec-adap.c
index d596b601ff42..eed6c397d840 100644
--- a/drivers/media/cec/cec-adap.c
+++ b/drivers/media/cec/cec-adap.c
@@ -78,42 +78,62 @@ static unsigned int cec_log_addr2dev(const struct cec_adapter *adap, u8 log_addr
  * Queue a new event for this filehandle. If ts == 0, then set it
  * to the current time.
  *
- * The two events that are currently defined do not need to keep track
- * of intermediate events, so no actual queue of events is needed,
- * instead just store the latest state and the total number of lost
- * messages.
- *
- * Should new events be added in the future that require intermediate
- * results to be queued as well, then a proper queue data structure is
- * required. But until then, just keep it simple.
+ * We keep a queue of at most max_event events where max_event differs
+ * per event. If the queue becomes full, then drop the oldest event and
+ * keep track of how many events we've dropped.
  */
 void cec_queue_event_fh(struct cec_fh *fh,
 			const struct cec_event *new_ev, u64 ts)
 {
-	struct cec_event *ev = &fh->events[new_ev->event - 1];
+	static const u8 max_events[CEC_NUM_EVENTS] = {
+		1, 1, 64, 64,
+	};
+	struct cec_event_entry *entry;
+	unsigned int ev_idx = new_ev->event - 1;
+
+	if (WARN_ON(ev_idx >= ARRAY_SIZE(fh->events)))
+		return;
 
 	if (ts == 0)
 		ts = ktime_get_ns();
 
 	mutex_lock(&fh->lock);
-	if (new_ev->event == CEC_EVENT_LOST_MSGS &&
-	    fh->pending_events & (1 << new_ev->event)) {
-		/*
-		 * If there is already a lost_msgs event, then just
-		 * update the lost_msgs count. This effectively
-		 * merges the old and new events into one.
-		 */
-		ev->lost_msgs.lost_msgs += new_ev->lost_msgs.lost_msgs;
-		goto unlock;
-	}
+	if (ev_idx < CEC_NUM_CORE_EVENTS)
+		entry = &fh->core_events[ev_idx];
+	else
+		entry = kmalloc(sizeof(*entry), GFP_KERNEL);
+	if (entry) {
+		if (new_ev->event == CEC_EVENT_LOST_MSGS &&
+		    fh->queued_events[ev_idx]) {
+			entry->ev.lost_msgs.lost_msgs +=
+				new_ev->lost_msgs.lost_msgs;
+			goto unlock;
+		}
+		entry->ev = *new_ev;
+		entry->ev.ts = ts;
+
+		if (fh->queued_events[ev_idx] < max_events[ev_idx]) {
+			/* Add new msg at the end of the queue */
+			list_add_tail(&entry->list, &fh->events[ev_idx]);
+			fh->queued_events[ev_idx]++;
+			fh->total_queued_events++;
+			goto unlock;
+		}
 
-	/*
-	 * Intermediate states are not interesting, so just
-	 * overwrite any older event.
-	 */
-	*ev = *new_ev;
-	ev->ts = ts;
-	fh->pending_events |= 1 << new_ev->event;
+		if (ev_idx >= CEC_NUM_CORE_EVENTS) {
+			list_add_tail(&entry->list, &fh->events[ev_idx]);
+			/* drop the oldest event */
+			entry = list_first_entry(&fh->events[ev_idx],
+						 struct cec_event_entry, list);
+			list_del(&entry->list);
+			kfree(entry);
+		}
+	}
+	/* Mark that events were lost */
+	entry = list_first_entry_or_null(&fh->events[ev_idx],
+					 struct cec_event_entry, list);
+	if (entry)
+		entry->ev.flags |= CEC_EVENT_FL_DROPPED_EVENTS;
 
 unlock:
 	mutex_unlock(&fh->lock);
@@ -133,47 +153,71 @@ static void cec_queue_event(struct cec_adapter *adap,
 	mutex_unlock(&adap->devnode.lock);
 }
 
+/* Notify userspace that the CEC pin changed state at the given time. */
+void cec_queue_pin_cec_event(struct cec_adapter *adap, bool is_high, ktime_t ts)
+{
+	struct cec_event ev = {
+		.event = is_high ? CEC_EVENT_PIN_CEC_HIGH :
+				   CEC_EVENT_PIN_CEC_LOW,
+	};
+	struct cec_fh *fh;
+
+	mutex_lock(&adap->devnode.lock);
+	list_for_each_entry(fh, &adap->devnode.fhs, list)
+		if (fh->mode_follower == CEC_MODE_MONITOR_PIN)
+			cec_queue_event_fh(fh, &ev, ktime_to_ns(ts));
+	mutex_unlock(&adap->devnode.lock);
+}
+EXPORT_SYMBOL_GPL(cec_queue_pin_cec_event);
+
 /*
- * Queue a new message for this filehandle. If there is no more room
- * in the queue, then send the LOST_MSGS event instead.
+ * Queue a new message for this filehandle.
+ *
+ * We keep a queue of at most CEC_MAX_MSG_RX_QUEUE_SZ messages. If the
+ * queue becomes full, then drop the oldest message and keep track
+ * of how many messages we've dropped.
  */
 static void cec_queue_msg_fh(struct cec_fh *fh, const struct cec_msg *msg)
 {
-	static const struct cec_event ev_lost_msg = {
-		.ts = 0,
+	static const struct cec_event ev_lost_msgs = {
 		.event = CEC_EVENT_LOST_MSGS,
 		.flags = 0,
 		{
-			.lost_msgs.lost_msgs = 1,
+			.lost_msgs = { 1 },
 		},
 	};
 	struct cec_msg_entry *entry;
 
 	mutex_lock(&fh->lock);
 	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
-	if (!entry)
-		goto lost_msgs;
-
-	entry->msg = *msg;
-	/* Add new msg at the end of the queue */
-	list_add_tail(&entry->list, &fh->msgs);
+	if (entry) {
+		entry->msg = *msg;
+		/* Add new msg at the end of the queue */
+		list_add_tail(&entry->list, &fh->msgs);
+
+		if (fh->queued_msgs < CEC_MAX_MSG_RX_QUEUE_SZ) {
+			/* All is fine if there is enough room */
+			fh->queued_msgs++;
+			mutex_unlock(&fh->lock);
+			wake_up_interruptible(&fh->wait);
+			return;
+		}
 
-	/*
-	 * if the queue now has more than CEC_MAX_MSG_RX_QUEUE_SZ
-	 * messages, drop the oldest one and send a lost message event.
-	 */
-	if (fh->queued_msgs == CEC_MAX_MSG_RX_QUEUE_SZ) {
+		/*
+		 * if the message queue is full, then drop the oldest one and
+		 * send a lost message event.
+		 */
+		entry = list_first_entry(&fh->msgs, struct cec_msg_entry, list);
 		list_del(&entry->list);
-		goto lost_msgs;
+		kfree(entry);
 	}
-	fh->queued_msgs++;
 	mutex_unlock(&fh->lock);
-	wake_up_interruptible(&fh->wait);
-	return;
 
-lost_msgs:
-	mutex_unlock(&fh->lock);
-	cec_queue_event_fh(fh, &ev_lost_msg, 0);
+	/*
+	 * We lost a message, either because kmalloc failed or the queue
+	 * was full.
+	 */
+	cec_queue_event_fh(fh, &ev_lost_msgs, ktime_get_ns());
 }
 
 /*
@@ -394,13 +438,17 @@ int cec_thread_func(void *_adap)
 
 		if (adap->transmitting && timeout) {
 			/*
-			 * If we timeout, then log that. This really shouldn't
-			 * happen and is an indication of a faulty CEC adapter
-			 * driver, or the CEC bus is in some weird state.
+			 * If we timeout, then log that. Normally this does
+			 * not happen and it is an indication of a faulty CEC
+			 * adapter driver, or the CEC bus is in some weird
+			 * state. On rare occasions it can happen if there is
+			 * so much traffic on the bus that the adapter was
+			 * unable to transmit for CEC_XFER_TIMEOUT_MS (2.1s).
 			 */
-			dprintk(0, "%s: message %*ph timed out!\n", __func__,
+			dprintk(1, "%s: message %*ph timed out\n", __func__,
 				adap->transmitting->msg.len,
 				adap->transmitting->msg.msg);
+			adap->tx_timeouts++;
 			/* Just give up on this. */
 			cec_data_cancel(adap->transmitting);
 			goto unlock;
@@ -467,14 +515,19 @@ unlock:
 /*
  * Called by the CEC adapter if a transmit finished.
  */
-void cec_transmit_done(struct cec_adapter *adap, u8 status, u8 arb_lost_cnt,
-		       u8 nack_cnt, u8 low_drive_cnt, u8 error_cnt)
+void cec_transmit_done_ts(struct cec_adapter *adap, u8 status,
+			  u8 arb_lost_cnt, u8 nack_cnt, u8 low_drive_cnt,
+			  u8 error_cnt, ktime_t ts)
 {
 	struct cec_data *data;
 	struct cec_msg *msg;
-	u64 ts = ktime_get_ns();
+	unsigned int attempts_made = arb_lost_cnt + nack_cnt +
+				     low_drive_cnt + error_cnt;
 
 	dprintk(2, "%s: status %02x\n", __func__, status);
+	if (attempts_made < 1)
+		attempts_made = 1;
+
 	mutex_lock(&adap->lock);
 	data = adap->transmitting;
 	if (!data) {
@@ -492,7 +545,7 @@ void cec_transmit_done(struct cec_adapter *adap, u8 status, u8 arb_lost_cnt,
 
 	/* Drivers must fill in the status! */
 	WARN_ON(status == 0);
-	msg->tx_ts = ts;
+	msg->tx_ts = ktime_to_ns(ts);
 	msg->tx_status |= status;
 	msg->tx_arb_lost_cnt += arb_lost_cnt;
 	msg->tx_nack_cnt += nack_cnt;
@@ -507,10 +560,10 @@ void cec_transmit_done(struct cec_adapter *adap, u8 status, u8 arb_lost_cnt,
 	 * the hardware didn't signal that it retried itself (by setting
 	 * CEC_TX_STATUS_MAX_RETRIES), then we will retry ourselves.
 	 */
-	if (data->attempts > 1 &&
+	if (data->attempts > attempts_made &&
 	    !(status & (CEC_TX_STATUS_MAX_RETRIES | CEC_TX_STATUS_OK))) {
 		/* Retry this message */
-		data->attempts--;
+		data->attempts -= attempts_made;
 		if (msg->timeout)
 			dprintk(2, "retransmit: %*ph (attempts: %d, wait for 0x%02x)\n",
 				msg->len, msg->msg, data->attempts, msg->reply);
@@ -555,25 +608,26 @@ wake_thread:
 unlock:
 	mutex_unlock(&adap->lock);
 }
-EXPORT_SYMBOL_GPL(cec_transmit_done);
+EXPORT_SYMBOL_GPL(cec_transmit_done_ts);
 
-void cec_transmit_attempt_done(struct cec_adapter *adap, u8 status)
+void cec_transmit_attempt_done_ts(struct cec_adapter *adap,
+				  u8 status, ktime_t ts)
 {
 	switch (status & ~CEC_TX_STATUS_MAX_RETRIES) {
 	case CEC_TX_STATUS_OK:
-		cec_transmit_done(adap, status, 0, 0, 0, 0);
+		cec_transmit_done_ts(adap, status, 0, 0, 0, 0, ts);
 		return;
 	case CEC_TX_STATUS_ARB_LOST:
-		cec_transmit_done(adap, status, 1, 0, 0, 0);
+		cec_transmit_done_ts(adap, status, 1, 0, 0, 0, ts);
 		return;
 	case CEC_TX_STATUS_NACK:
-		cec_transmit_done(adap, status, 0, 1, 0, 0);
+		cec_transmit_done_ts(adap, status, 0, 1, 0, 0, ts);
 		return;
 	case CEC_TX_STATUS_LOW_DRIVE:
-		cec_transmit_done(adap, status, 0, 0, 1, 0);
+		cec_transmit_done_ts(adap, status, 0, 0, 1, 0, ts);
 		return;
 	case CEC_TX_STATUS_ERROR:
-		cec_transmit_done(adap, status, 0, 0, 0, 1);
+		cec_transmit_done_ts(adap, status, 0, 0, 0, 1, ts);
 		return;
 	default:
 		/* Should never happen */
@@ -581,7 +635,7 @@ void cec_transmit_attempt_done(struct cec_adapter *adap, u8 status)
 		return;
 	}
 }
-EXPORT_SYMBOL_GPL(cec_transmit_attempt_done);
+EXPORT_SYMBOL_GPL(cec_transmit_attempt_done_ts);
 
 /*
  * Called when waiting for a reply times out.
@@ -630,9 +684,7 @@ int cec_transmit_msg_fh(struct cec_adapter *adap, struct cec_msg *msg,
 	msg->tx_nack_cnt = 0;
 	msg->tx_low_drive_cnt = 0;
 	msg->tx_error_cnt = 0;
-	msg->sequence = ++adap->sequence;
-	if (!msg->sequence)
-		msg->sequence = ++adap->sequence;
+	msg->sequence = 0;
 
 	if (msg->reply && msg->timeout == 0) {
 		/* Make sure the timeout isn't 0. */
@@ -671,6 +723,9 @@ int cec_transmit_msg_fh(struct cec_adapter *adap, struct cec_msg *msg,
 			msg->tx_status = CEC_TX_STATUS_NACK |
 					 CEC_TX_STATUS_MAX_RETRIES;
 			msg->tx_nack_cnt = 1;
+			msg->sequence = ++adap->sequence;
+			if (!msg->sequence)
+				msg->sequence = ++adap->sequence;
 			return 0;
 		}
 	}
@@ -705,6 +760,10 @@ int cec_transmit_msg_fh(struct cec_adapter *adap, struct cec_msg *msg,
 	if (!data)
 		return -ENOMEM;
 
+	msg->sequence = ++adap->sequence;
+	if (!msg->sequence)
+		msg->sequence = ++adap->sequence;
+
 	if (msg->len > 1 && msg->msg[1] == CEC_MSG_CDC_MESSAGE) {
 		msg->msg[2] = adap->phys_addr >> 8;
 		msg->msg[3] = adap->phys_addr & 0xff;
@@ -712,7 +771,8 @@ int cec_transmit_msg_fh(struct cec_adapter *adap, struct cec_msg *msg,
 
 	if (msg->timeout)
 		dprintk(2, "%s: %*ph (wait for 0x%02x%s)\n",
-			__func__, msg->len, msg->msg, msg->reply, !block ? ", nb" : "");
+			__func__, msg->len, msg->msg, msg->reply,
+			!block ? ", nb" : "");
 	else
 		dprintk(2, "%s: %*ph%s\n",
 			__func__, msg->len, msg->msg, !block ? " (nb)" : "");
@@ -909,7 +969,8 @@ static const u8 cec_msg_size[256] = {
 };
 
 /* Called by the CEC adapter if a message is received */
-void cec_received_msg(struct cec_adapter *adap, struct cec_msg *msg)
+void cec_received_msg_ts(struct cec_adapter *adap,
+			 struct cec_msg *msg, ktime_t ts)
 {
 	struct cec_data *data;
 	u8 msg_init = cec_msg_initiator(msg);
@@ -937,7 +998,7 @@ void cec_received_msg(struct cec_adapter *adap, struct cec_msg *msg)
 	    cec_has_log_addr(adap, msg_init))
 		return;
 
-	msg->rx_ts = ktime_get_ns();
+	msg->rx_ts = ktime_to_ns(ts);
 	msg->rx_status = CEC_RX_STATUS_OK;
 	msg->sequence = msg->reply = msg->timeout = 0;
 	msg->tx_status = 0;
@@ -1102,7 +1163,7 @@ void cec_received_msg(struct cec_adapter *adap, struct cec_msg *msg)
 	 */
 	cec_receive_notify(adap, msg, is_reply);
 }
-EXPORT_SYMBOL_GPL(cec_received_msg);
+EXPORT_SYMBOL_GPL(cec_received_msg_ts);
 
 /* Logical Address Handling */
 
@@ -1390,7 +1451,9 @@ static void cec_claim_log_addrs(struct cec_adapter *adap, bool block)
  */
 void __cec_s_phys_addr(struct cec_adapter *adap, u16 phys_addr, bool block)
 {
-	if (phys_addr == adap->phys_addr || adap->devnode.unregistered)
+	if (phys_addr == adap->phys_addr)
+		return;
+	if (phys_addr != CEC_PHYS_ADDR_INVALID && adap->devnode.unregistered)
 		return;
 
 	dprintk(1, "new physical address %x.%x.%x.%x\n",
@@ -1471,8 +1534,13 @@ int __cec_s_log_addrs(struct cec_adapter *adap,
 		return -ENODEV;
 
 	if (!log_addrs || log_addrs->num_log_addrs == 0) {
-		adap->log_addrs.num_log_addrs = 0;
 		cec_adap_unconfigure(adap);
+		adap->log_addrs.num_log_addrs = 0;
+		for (i = 0; i < CEC_MAX_LOG_ADDRS; i++)
+			adap->log_addrs.log_addr[i] = CEC_LOG_ADDR_INVALID;
+		adap->log_addrs.osd_name[0] = '\0';
+		adap->log_addrs.vendor_id = CEC_VENDOR_ID_NONE;
+		adap->log_addrs.cec_version = CEC_OP_CEC_VERSION_2_0;
 		return 0;
 	}
 
@@ -1704,6 +1772,9 @@ static int cec_receive_notify(struct cec_adapter *adap, struct cec_msg *msg,
 	int la_idx = cec_log_addr2idx(adap, dest_laddr);
 	bool from_unregistered = init_laddr == 0xf;
 	struct cec_msg tx_cec_msg = { };
+#ifdef CONFIG_MEDIA_CEC_RC
+	int scancode;
+#endif
 
 	dprintk(2, "%s: %*ph\n", __func__, msg->len, msg->msg);
 
@@ -1792,11 +1863,9 @@ static int cec_receive_notify(struct cec_adapter *adap, struct cec_msg *msg,
 		 */
 		case 0x60:
 			if (msg->len == 2)
-				rc_keydown(adap->rc, RC_TYPE_CEC,
-					   msg->msg[2], 0);
+				scancode = msg->msg[2];
 			else
-				rc_keydown(adap->rc, RC_TYPE_CEC,
-					   msg->msg[2] << 8 | msg->msg[3], 0);
+				scancode = msg->msg[2] << 8 | msg->msg[3];
 			break;
 		/*
 		 * Other function messages that are not handled.
@@ -1809,11 +1878,54 @@ static int cec_receive_notify(struct cec_adapter *adap, struct cec_msg *msg,
 		 */
 		case 0x56: case 0x57:
 		case 0x67: case 0x68: case 0x69: case 0x6a:
+			scancode = -1;
 			break;
 		default:
-			rc_keydown(adap->rc, RC_TYPE_CEC, msg->msg[2], 0);
+			scancode = msg->msg[2];
 			break;
 		}
+
+		/* Was repeating, but keypress timed out */
+		if (adap->rc_repeating && !adap->rc->keypressed) {
+			adap->rc_repeating = false;
+			adap->rc_last_scancode = -1;
+		}
+		/* Different keypress from last time, ends repeat mode */
+		if (adap->rc_last_scancode != scancode) {
+			rc_keyup(adap->rc);
+			adap->rc_repeating = false;
+		}
+		/* We can't handle this scancode */
+		if (scancode < 0) {
+			adap->rc_last_scancode = scancode;
+			break;
+		}
+
+		/* Send key press */
+		rc_keydown(adap->rc, RC_PROTO_CEC, scancode, 0);
+
+		/* When in repeating mode, we're done */
+		if (adap->rc_repeating)
+			break;
+
+		/*
+		 * We are not repeating, but the new scancode is
+		 * the same as the last one, and this second key press is
+		 * within 550 ms (the 'Follower Safety Timeout') from the
+		 * previous key press, so we now enable the repeating mode.
+		 */
+		if (adap->rc_last_scancode == scancode &&
+		    msg->rx_ts - adap->rc_last_keypress < 550 * NSEC_PER_MSEC) {
+			adap->rc_repeating = true;
+			break;
+		}
+		/*
+		 * Not in repeating mode, so avoid triggering repeat mode
+		 * by calling keyup.
+		 */
+		rc_keyup(adap->rc);
+		adap->rc_last_scancode = scancode;
+		adap->rc_last_keypress = msg->rx_ts;
 #endif
 		break;
 
@@ -1823,6 +1935,8 @@ static int cec_receive_notify(struct cec_adapter *adap, struct cec_msg *msg,
 			break;
 #ifdef CONFIG_MEDIA_CEC_RC
 		rc_keyup(adap->rc);
+		adap->rc_repeating = false;
+		adap->rc_last_scancode = -1;
 #endif
 		break;
 
@@ -1941,6 +2055,11 @@ int cec_adap_status(struct seq_file *file, void *priv)
 	if (adap->monitor_all_cnt)
 		seq_printf(file, "file handles in Monitor All mode: %u\n",
 			   adap->monitor_all_cnt);
+	if (adap->tx_timeouts) {
+		seq_printf(file, "transmit timeouts: %u\n",
+			   adap->tx_timeouts);
+		adap->tx_timeouts = 0;
+	}
 	data = adap->transmitting;
 	if (data)
 		seq_printf(file, "transmitting message: %*ph (reply: %02x, timeout: %ums)\n",
diff --git a/drivers/media/cec/cec-api.c b/drivers/media/cec/cec-api.c
index f7eb4c54a354..a079f7fe018c 100644
--- a/drivers/media/cec/cec-api.c
+++ b/drivers/media/cec/cec-api.c
@@ -30,6 +30,7 @@
 #include <linux/uaccess.h>
 #include <linux/version.h>
 
+#include <media/cec-pin.h>
 #include "cec-priv.h"
 
 static inline struct cec_devnode *cec_devnode_data(struct file *filp)
@@ -57,7 +58,7 @@ static unsigned int cec_poll(struct file *filp,
 		res |= POLLOUT | POLLWRNORM;
 	if (fh->queued_msgs)
 		res |= POLLIN | POLLRDNORM;
-	if (fh->pending_events)
+	if (fh->total_queued_events)
 		res |= POLLPRI;
 	poll_wait(filp, &fh->wait, poll);
 	mutex_unlock(&adap->lock);
@@ -289,15 +290,17 @@ static long cec_receive(struct cec_adapter *adap, struct cec_fh *fh,
 static long cec_dqevent(struct cec_adapter *adap, struct cec_fh *fh,
 			bool block, struct cec_event __user *parg)
 {
-	struct cec_event *ev = NULL;
+	struct cec_event_entry *ev = NULL;
 	u64 ts = ~0ULL;
 	unsigned int i;
+	unsigned int ev_idx;
 	long err = 0;
 
 	mutex_lock(&fh->lock);
-	while (!fh->pending_events && block) {
+	while (!fh->total_queued_events && block) {
 		mutex_unlock(&fh->lock);
-		err = wait_event_interruptible(fh->wait, fh->pending_events);
+		err = wait_event_interruptible(fh->wait,
+					       fh->total_queued_events);
 		if (err)
 			return err;
 		mutex_lock(&fh->lock);
@@ -305,23 +308,29 @@ static long cec_dqevent(struct cec_adapter *adap, struct cec_fh *fh,
 
 	/* Find the oldest event */
 	for (i = 0; i < CEC_NUM_EVENTS; i++) {
-		if (fh->pending_events & (1 << (i + 1)) &&
-		    fh->events[i].ts <= ts) {
-			ev = &fh->events[i];
-			ts = ev->ts;
+		struct cec_event_entry *entry =
+			list_first_entry_or_null(&fh->events[i],
+						 struct cec_event_entry, list);
+
+		if (entry && entry->ev.ts <= ts) {
+			ev = entry;
+			ev_idx = i;
+			ts = ev->ev.ts;
 		}
 	}
+
 	if (!ev) {
 		err = -EAGAIN;
 		goto unlock;
 	}
+	list_del(&ev->list);
 
-	if (copy_to_user(parg, ev, sizeof(*ev))) {
+	if (copy_to_user(parg, &ev->ev, sizeof(ev->ev)))
 		err = -EFAULT;
-		goto unlock;
-	}
-
-	fh->pending_events &= ~(1 << ev->event);
+	if (ev_idx >= CEC_NUM_CORE_EVENTS)
+		kfree(ev);
+	fh->queued_events[ev_idx]--;
+	fh->total_queued_events--;
 
 unlock:
 	mutex_unlock(&fh->lock);
@@ -348,33 +357,50 @@ static long cec_s_mode(struct cec_adapter *adap, struct cec_fh *fh,
 
 	if (copy_from_user(&mode, parg, sizeof(mode)))
 		return -EFAULT;
-	if (mode & ~(CEC_MODE_INITIATOR_MSK | CEC_MODE_FOLLOWER_MSK))
+	if (mode & ~(CEC_MODE_INITIATOR_MSK | CEC_MODE_FOLLOWER_MSK)) {
+		dprintk(1, "%s: invalid mode bits set\n", __func__);
 		return -EINVAL;
+	}
 
 	mode_initiator = mode & CEC_MODE_INITIATOR_MSK;
 	mode_follower = mode & CEC_MODE_FOLLOWER_MSK;
 
 	if (mode_initiator > CEC_MODE_EXCL_INITIATOR ||
-	    mode_follower > CEC_MODE_MONITOR_ALL)
+	    mode_follower > CEC_MODE_MONITOR_ALL) {
+		dprintk(1, "%s: unknown mode\n", __func__);
 		return -EINVAL;
+	}
 
 	if (mode_follower == CEC_MODE_MONITOR_ALL &&
-	    !(adap->capabilities & CEC_CAP_MONITOR_ALL))
+	    !(adap->capabilities & CEC_CAP_MONITOR_ALL)) {
+		dprintk(1, "%s: MONITOR_ALL not supported\n", __func__);
 		return -EINVAL;
+	}
+
+	if (mode_follower == CEC_MODE_MONITOR_PIN &&
+	    !(adap->capabilities & CEC_CAP_MONITOR_PIN)) {
+		dprintk(1, "%s: MONITOR_PIN not supported\n", __func__);
+		return -EINVAL;
+	}
 
 	/* Follower modes should always be able to send CEC messages */
 	if ((mode_initiator == CEC_MODE_NO_INITIATOR ||
 	     !(adap->capabilities & CEC_CAP_TRANSMIT)) &&
 	    mode_follower >= CEC_MODE_FOLLOWER &&
-	    mode_follower <= CEC_MODE_EXCL_FOLLOWER_PASSTHRU)
+	    mode_follower <= CEC_MODE_EXCL_FOLLOWER_PASSTHRU) {
+		dprintk(1, "%s: cannot transmit\n", __func__);
 		return -EINVAL;
+	}
 
 	/* Monitor modes require CEC_MODE_NO_INITIATOR */
-	if (mode_initiator && mode_follower >= CEC_MODE_MONITOR)
+	if (mode_initiator && mode_follower >= CEC_MODE_MONITOR_PIN) {
+		dprintk(1, "%s: monitor modes require NO_INITIATOR\n",
+			__func__);
 		return -EINVAL;
+	}
 
 	/* Monitor modes require CAP_NET_ADMIN */
-	if (mode_follower >= CEC_MODE_MONITOR && !capable(CAP_NET_ADMIN))
+	if (mode_follower >= CEC_MODE_MONITOR_PIN && !capable(CAP_NET_ADMIN))
 		return -EPERM;
 
 	mutex_lock(&adap->lock);
@@ -413,8 +439,20 @@ static long cec_s_mode(struct cec_adapter *adap, struct cec_fh *fh,
 
 	if (fh->mode_follower == CEC_MODE_FOLLOWER)
 		adap->follower_cnt--;
+	if (fh->mode_follower == CEC_MODE_MONITOR_PIN)
+		adap->monitor_pin_cnt--;
 	if (mode_follower == CEC_MODE_FOLLOWER)
 		adap->follower_cnt++;
+	if (mode_follower == CEC_MODE_MONITOR_PIN) {
+		struct cec_event ev = {
+			.flags = CEC_EVENT_FL_INITIAL_STATE,
+		};
+
+		ev.event = adap->cec_pin_is_high ? CEC_EVENT_PIN_CEC_HIGH :
+						   CEC_EVENT_PIN_CEC_LOW;
+		cec_queue_event_fh(fh, &ev, 0);
+		adap->monitor_pin_cnt++;
+	}
 	if (mode_follower == CEC_MODE_EXCL_FOLLOWER ||
 	    mode_follower == CEC_MODE_EXCL_FOLLOWER_PASSTHRU) {
 		adap->passthrough =
@@ -495,6 +533,7 @@ static int cec_open(struct inode *inode, struct file *filp)
 		.event = CEC_EVENT_STATE_CHANGE,
 		.flags = CEC_EVENT_FL_INITIAL_STATE,
 	};
+	unsigned int i;
 	int err;
 
 	if (!fh)
@@ -502,6 +541,8 @@ static int cec_open(struct inode *inode, struct file *filp)
 
 	INIT_LIST_HEAD(&fh->msgs);
 	INIT_LIST_HEAD(&fh->xfer_list);
+	for (i = 0; i < CEC_NUM_EVENTS; i++)
+		INIT_LIST_HEAD(&fh->events[i]);
 	mutex_init(&fh->lock);
 	init_waitqueue_head(&fh->wait);
 
@@ -544,6 +585,7 @@ static int cec_release(struct inode *inode, struct file *filp)
 	struct cec_devnode *devnode = cec_devnode_data(filp);
 	struct cec_adapter *adap = to_cec_adapter(devnode);
 	struct cec_fh *fh = filp->private_data;
+	unsigned int i;
 
 	mutex_lock(&adap->lock);
 	if (adap->cec_initiator == fh)
@@ -554,6 +596,8 @@ static int cec_release(struct inode *inode, struct file *filp)
 	}
 	if (fh->mode_follower == CEC_MODE_FOLLOWER)
 		adap->follower_cnt--;
+	if (fh->mode_follower == CEC_MODE_MONITOR_PIN)
+		adap->monitor_pin_cnt--;
 	if (fh->mode_follower == CEC_MODE_MONITOR_ALL)
 		cec_monitor_all_cnt_dec(adap);
 	mutex_unlock(&adap->lock);
@@ -585,6 +629,16 @@ static int cec_release(struct inode *inode, struct file *filp)
 		list_del(&entry->list);
 		kfree(entry);
 	}
+	for (i = CEC_NUM_CORE_EVENTS; i < CEC_NUM_EVENTS; i++) {
+		while (!list_empty(&fh->events[i])) {
+			struct cec_event_entry *entry =
+				list_first_entry(&fh->events[i],
+						 struct cec_event_entry, list);
+
+			list_del(&entry->list);
+			kfree(entry);
+		}
+	}
 	kfree(fh);
 
 	cec_put_device(devnode);
diff --git a/drivers/media/cec/cec-core.c b/drivers/media/cec/cec-core.c
index b516d599d6c4..648136e552d5 100644
--- a/drivers/media/cec/cec-core.c
+++ b/drivers/media/cec/cec-core.c
@@ -227,6 +227,7 @@ struct cec_adapter *cec_allocate_adapter(const struct cec_adap_ops *ops,
 		return ERR_PTR(-ENOMEM);
 	strlcpy(adap->name, name, sizeof(adap->name));
 	adap->phys_addr = CEC_PHYS_ADDR_INVALID;
+	adap->cec_pin_is_high = true;
 	adap->log_addrs.cec_version = CEC_OP_CEC_VERSION_2_0;
 	adap->log_addrs.vendor_id = CEC_VENDOR_ID_NONE;
 	adap->capabilities = caps;
@@ -263,22 +264,24 @@ struct cec_adapter *cec_allocate_adapter(const struct cec_adap_ops *ops,
 		return ERR_PTR(-ENOMEM);
 	}
 
-	snprintf(adap->input_name, sizeof(adap->input_name),
+	snprintf(adap->device_name, sizeof(adap->device_name),
 		 "RC for %s", name);
 	snprintf(adap->input_phys, sizeof(adap->input_phys),
 		 "%s/input0", name);
 
-	adap->rc->input_name = adap->input_name;
+	adap->rc->device_name = adap->device_name;
 	adap->rc->input_phys = adap->input_phys;
 	adap->rc->input_id.bustype = BUS_CEC;
 	adap->rc->input_id.vendor = 0;
 	adap->rc->input_id.product = 0;
 	adap->rc->input_id.version = 1;
 	adap->rc->driver_name = CEC_NAME;
-	adap->rc->allowed_protocols = RC_BIT_CEC;
+	adap->rc->allowed_protocols = RC_PROTO_BIT_CEC;
+	adap->rc->enabled_protocols = RC_PROTO_BIT_CEC;
 	adap->rc->priv = adap;
 	adap->rc->map_name = RC_MAP_CEC;
 	adap->rc->timeout = MS_TO_NS(100);
+	adap->rc_last_scancode = -1;
 #endif
 	return adap;
 }
@@ -310,6 +313,17 @@ int cec_register_adapter(struct cec_adapter *adap,
 			adap->rc = NULL;
 			return res;
 		}
+		/*
+		 * The REP_DELAY for CEC is really the time between the initial
+		 * 'User Control Pressed' message and the second. The first
+		 * keypress is always seen as non-repeating, the second
+		 * (provided it has the same UI Command) will start the 'Press
+		 * and Hold' (aka repeat) behavior. By setting REP_DELAY to the
+		 * same value as REP_PERIOD the expected CEC behavior is
+		 * reproduced.
+		 */
+		adap->rc->input_dev->rep[REP_DELAY] =
+			adap->rc->input_dev->rep[REP_PERIOD];
 	}
 #endif
 
@@ -374,6 +388,8 @@ void cec_delete_adapter(struct cec_adapter *adap)
 	kthread_stop(adap->kthread);
 	if (adap->kthread_config)
 		kthread_stop(adap->kthread_config);
+	if (adap->ops->adap_free)
+		adap->ops->adap_free(adap);
 #ifdef CONFIG_MEDIA_CEC_RC
 	rc_free_device(adap->rc);
 #endif
@@ -386,11 +402,8 @@ EXPORT_SYMBOL_GPL(cec_delete_adapter);
  */
 static int __init cec_devnode_init(void)
 {
-	int ret;
+	int ret = alloc_chrdev_region(&cec_dev_t, 0, CEC_NUM_DEVICES, CEC_NAME);
 
-	pr_info("Linux cec interface: v0.10\n");
-	ret = alloc_chrdev_region(&cec_dev_t, 0, CEC_NUM_DEVICES,
-				  CEC_NAME);
 	if (ret < 0) {
 		pr_warn("cec: unable to allocate major\n");
 		return ret;
diff --git a/drivers/media/cec/cec-pin.c b/drivers/media/cec/cec-pin.c
new file mode 100644
index 000000000000..c003b8eac617
--- /dev/null
+++ b/drivers/media/cec/cec-pin.c
@@ -0,0 +1,802 @@
+/*
+ * Copyright 2017 Cisco Systems, Inc. and/or its affiliates. All rights reserved.
+ *
+ * This program is free software; you may redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/delay.h>
+#include <linux/slab.h>
+#include <linux/sched/types.h>
+
+#include <media/cec-pin.h>
+
+/* All timings are in microseconds */
+
+/* start bit timings */
+#define CEC_TIM_START_BIT_LOW		3700
+#define CEC_TIM_START_BIT_LOW_MIN	3500
+#define CEC_TIM_START_BIT_LOW_MAX	3900
+#define CEC_TIM_START_BIT_TOTAL		4500
+#define CEC_TIM_START_BIT_TOTAL_MIN	4300
+#define CEC_TIM_START_BIT_TOTAL_MAX	4700
+
+/* data bit timings */
+#define CEC_TIM_DATA_BIT_0_LOW		1500
+#define CEC_TIM_DATA_BIT_0_LOW_MIN	1300
+#define CEC_TIM_DATA_BIT_0_LOW_MAX	1700
+#define CEC_TIM_DATA_BIT_1_LOW		600
+#define CEC_TIM_DATA_BIT_1_LOW_MIN	400
+#define CEC_TIM_DATA_BIT_1_LOW_MAX	800
+#define CEC_TIM_DATA_BIT_TOTAL		2400
+#define CEC_TIM_DATA_BIT_TOTAL_MIN	2050
+#define CEC_TIM_DATA_BIT_TOTAL_MAX	2750
+/* earliest safe time to sample the bit state */
+#define CEC_TIM_DATA_BIT_SAMPLE		850
+/* earliest time the bit is back to 1 (T7 + 50) */
+#define CEC_TIM_DATA_BIT_HIGH		1750
+
+/* when idle, sample once per millisecond */
+#define CEC_TIM_IDLE_SAMPLE		1000
+/* when processing the start bit, sample twice per millisecond */
+#define CEC_TIM_START_BIT_SAMPLE	500
+/* when polling for a state change, sample once every 50 micoseconds */
+#define CEC_TIM_SAMPLE			50
+
+#define CEC_TIM_LOW_DRIVE_ERROR		(1.5 * CEC_TIM_DATA_BIT_TOTAL)
+
+struct cec_state {
+	const char * const name;
+	unsigned int usecs;
+};
+
+static const struct cec_state states[CEC_PIN_STATES] = {
+	{ "Off",		   0 },
+	{ "Idle",		   CEC_TIM_IDLE_SAMPLE },
+	{ "Tx Wait",		   CEC_TIM_SAMPLE },
+	{ "Tx Wait for High",	   CEC_TIM_IDLE_SAMPLE },
+	{ "Tx Start Bit Low",	   CEC_TIM_START_BIT_LOW },
+	{ "Tx Start Bit High",	   CEC_TIM_START_BIT_TOTAL - CEC_TIM_START_BIT_LOW },
+	{ "Tx Data 0 Low",	   CEC_TIM_DATA_BIT_0_LOW },
+	{ "Tx Data 0 High",	   CEC_TIM_DATA_BIT_TOTAL - CEC_TIM_DATA_BIT_0_LOW },
+	{ "Tx Data 1 Low",	   CEC_TIM_DATA_BIT_1_LOW },
+	{ "Tx Data 1 High",	   CEC_TIM_DATA_BIT_TOTAL - CEC_TIM_DATA_BIT_1_LOW },
+	{ "Tx Data 1 Pre Sample",  CEC_TIM_DATA_BIT_SAMPLE - CEC_TIM_DATA_BIT_1_LOW },
+	{ "Tx Data 1 Post Sample", CEC_TIM_DATA_BIT_TOTAL - CEC_TIM_DATA_BIT_SAMPLE },
+	{ "Rx Start Bit Low",	   CEC_TIM_SAMPLE },
+	{ "Rx Start Bit High",	   CEC_TIM_SAMPLE },
+	{ "Rx Data Sample",	   CEC_TIM_DATA_BIT_SAMPLE },
+	{ "Rx Data Post Sample",   CEC_TIM_DATA_BIT_HIGH - CEC_TIM_DATA_BIT_SAMPLE },
+	{ "Rx Data High",	   CEC_TIM_SAMPLE },
+	{ "Rx Ack Low",		   CEC_TIM_DATA_BIT_0_LOW },
+	{ "Rx Ack Low Post",	   CEC_TIM_DATA_BIT_HIGH - CEC_TIM_DATA_BIT_0_LOW },
+	{ "Rx Ack High Post",	   CEC_TIM_DATA_BIT_HIGH },
+	{ "Rx Ack Finish",	   CEC_TIM_DATA_BIT_TOTAL_MIN - CEC_TIM_DATA_BIT_HIGH },
+	{ "Rx Low Drive",	   CEC_TIM_LOW_DRIVE_ERROR },
+	{ "Rx Irq",		   0 },
+};
+
+static void cec_pin_update(struct cec_pin *pin, bool v, bool force)
+{
+	if (!force && v == pin->adap->cec_pin_is_high)
+		return;
+
+	pin->adap->cec_pin_is_high = v;
+	if (atomic_read(&pin->work_pin_events) < CEC_NUM_PIN_EVENTS) {
+		pin->work_pin_is_high[pin->work_pin_events_wr] = v;
+		pin->work_pin_ts[pin->work_pin_events_wr] = ktime_get();
+		pin->work_pin_events_wr =
+			(pin->work_pin_events_wr + 1) % CEC_NUM_PIN_EVENTS;
+		atomic_inc(&pin->work_pin_events);
+	}
+	wake_up_interruptible(&pin->kthread_waitq);
+}
+
+static bool cec_pin_read(struct cec_pin *pin)
+{
+	bool v = pin->ops->read(pin->adap);
+
+	cec_pin_update(pin, v, false);
+	return v;
+}
+
+static void cec_pin_low(struct cec_pin *pin)
+{
+	pin->ops->low(pin->adap);
+	cec_pin_update(pin, false, false);
+}
+
+static bool cec_pin_high(struct cec_pin *pin)
+{
+	pin->ops->high(pin->adap);
+	return cec_pin_read(pin);
+}
+
+static void cec_pin_to_idle(struct cec_pin *pin)
+{
+	/*
+	 * Reset all status fields, release the bus and
+	 * go to idle state.
+	 */
+	pin->rx_bit = pin->tx_bit = 0;
+	pin->rx_msg.len = 0;
+	memset(pin->rx_msg.msg, 0, sizeof(pin->rx_msg.msg));
+	pin->state = CEC_ST_IDLE;
+	pin->ts = 0;
+}
+
+/*
+ * Handle Transmit-related states
+ *
+ * Basic state changes when transmitting:
+ *
+ * Idle -> Tx Wait (waiting for the end of signal free time) ->
+ *	Tx Start Bit Low -> Tx Start Bit High ->
+ *
+ *   Regular data bits + EOM:
+ *	Tx Data 0 Low -> Tx Data 0 High ->
+ *   or:
+ *	Tx Data 1 Low -> Tx Data 1 High ->
+ *
+ *   First 4 data bits or Ack bit:
+ *	Tx Data 0 Low -> Tx Data 0 High ->
+ *   or:
+ *	Tx Data 1 Low -> Tx Data 1 High -> Tx Data 1 Pre Sample ->
+ *		Tx Data 1 Post Sample ->
+ *
+ *   After the last Ack go to Idle.
+ *
+ * If it detects a Low Drive condition then:
+ *	Tx Wait For High -> Idle
+ *
+ * If it loses arbitration, then it switches to state Rx Data Post Sample.
+ */
+static void cec_pin_tx_states(struct cec_pin *pin, ktime_t ts)
+{
+	bool v;
+	bool is_ack_bit, ack;
+
+	switch (pin->state) {
+	case CEC_ST_TX_WAIT_FOR_HIGH:
+		if (cec_pin_read(pin))
+			cec_pin_to_idle(pin);
+		break;
+
+	case CEC_ST_TX_START_BIT_LOW:
+		pin->state = CEC_ST_TX_START_BIT_HIGH;
+		/* Generate start bit */
+		cec_pin_high(pin);
+		break;
+
+	case CEC_ST_TX_DATA_BIT_1_HIGH_POST_SAMPLE:
+		/* If the read value is 1, then all is OK */
+		if (!cec_pin_read(pin)) {
+			/*
+			 * It's 0, so someone detected an error and pulled the
+			 * line low for 1.5 times the nominal bit period.
+			 */
+			pin->tx_msg.len = 0;
+			pin->work_tx_ts = ts;
+			pin->work_tx_status = CEC_TX_STATUS_LOW_DRIVE;
+			pin->state = CEC_ST_TX_WAIT_FOR_HIGH;
+			wake_up_interruptible(&pin->kthread_waitq);
+			break;
+		}
+		if (pin->tx_nacked) {
+			cec_pin_to_idle(pin);
+			pin->tx_msg.len = 0;
+			pin->work_tx_ts = ts;
+			pin->work_tx_status = CEC_TX_STATUS_NACK;
+			wake_up_interruptible(&pin->kthread_waitq);
+			break;
+		}
+		/* fall through */
+	case CEC_ST_TX_DATA_BIT_0_HIGH:
+	case CEC_ST_TX_DATA_BIT_1_HIGH:
+		pin->tx_bit++;
+		/* fall through */
+	case CEC_ST_TX_START_BIT_HIGH:
+		if (pin->tx_bit / 10 >= pin->tx_msg.len) {
+			cec_pin_to_idle(pin);
+			pin->tx_msg.len = 0;
+			pin->work_tx_ts = ts;
+			pin->work_tx_status = CEC_TX_STATUS_OK;
+			wake_up_interruptible(&pin->kthread_waitq);
+			break;
+		}
+
+		switch (pin->tx_bit % 10) {
+		default:
+			v = pin->tx_msg.msg[pin->tx_bit / 10] &
+				(1 << (7 - (pin->tx_bit % 10)));
+			pin->state = v ? CEC_ST_TX_DATA_BIT_1_LOW :
+				CEC_ST_TX_DATA_BIT_0_LOW;
+			break;
+		case 8:
+			v = pin->tx_bit / 10 == pin->tx_msg.len - 1;
+			pin->state = v ? CEC_ST_TX_DATA_BIT_1_LOW :
+				CEC_ST_TX_DATA_BIT_0_LOW;
+			break;
+		case 9:
+			pin->state = CEC_ST_TX_DATA_BIT_1_LOW;
+			break;
+		}
+		cec_pin_low(pin);
+		break;
+
+	case CEC_ST_TX_DATA_BIT_0_LOW:
+	case CEC_ST_TX_DATA_BIT_1_LOW:
+		v = pin->state == CEC_ST_TX_DATA_BIT_1_LOW;
+		pin->state = v ? CEC_ST_TX_DATA_BIT_1_HIGH :
+			CEC_ST_TX_DATA_BIT_0_HIGH;
+		is_ack_bit = pin->tx_bit % 10 == 9;
+		if (v && (pin->tx_bit < 4 || is_ack_bit))
+			pin->state = CEC_ST_TX_DATA_BIT_1_HIGH_PRE_SAMPLE;
+		cec_pin_high(pin);
+		break;
+
+	case CEC_ST_TX_DATA_BIT_1_HIGH_PRE_SAMPLE:
+		/* Read the CEC value at the sample time */
+		v = cec_pin_read(pin);
+		is_ack_bit = pin->tx_bit % 10 == 9;
+		/*
+		 * If v == 0 and we're within the first 4 bits
+		 * of the initiator, then someone else started
+		 * transmitting and we lost the arbitration
+		 * (i.e. the logical address of the other
+		 * transmitter has more leading 0 bits in the
+		 * initiator).
+		 */
+		if (!v && !is_ack_bit) {
+			pin->tx_msg.len = 0;
+			pin->work_tx_ts = ts;
+			pin->work_tx_status = CEC_TX_STATUS_ARB_LOST;
+			wake_up_interruptible(&pin->kthread_waitq);
+			pin->rx_bit = pin->tx_bit;
+			pin->tx_bit = 0;
+			memset(pin->rx_msg.msg, 0, sizeof(pin->rx_msg.msg));
+			pin->rx_msg.msg[0] = pin->tx_msg.msg[0];
+			pin->rx_msg.msg[0] &= ~(1 << (7 - pin->rx_bit));
+			pin->rx_msg.len = 0;
+			pin->state = CEC_ST_RX_DATA_POST_SAMPLE;
+			pin->rx_bit++;
+			break;
+		}
+		pin->state = CEC_ST_TX_DATA_BIT_1_HIGH_POST_SAMPLE;
+		if (!is_ack_bit)
+			break;
+		/* Was the message ACKed? */
+		ack = cec_msg_is_broadcast(&pin->tx_msg) ? v : !v;
+		if (!ack) {
+			/*
+			 * Note: the CEC spec is ambiguous regarding
+			 * what action to take when a NACK appears
+			 * before the last byte of the payload was
+			 * transmitted: either stop transmitting
+			 * immediately, or wait until the last byte
+			 * was transmitted.
+			 *
+			 * Most CEC implementations appear to stop
+			 * immediately, and that's what we do here
+			 * as well.
+			 */
+			pin->tx_nacked = true;
+		}
+		break;
+
+	default:
+		break;
+	}
+}
+
+/*
+ * Handle Receive-related states
+ *
+ * Basic state changes when receiving:
+ *
+ *	Rx Start Bit Low -> Rx Start Bit High ->
+ *   Regular data bits + EOM:
+ *	Rx Data Sample -> Rx Data Post Sample -> Rx Data High ->
+ *   Ack bit 0:
+ *	Rx Ack Low -> Rx Ack Low Post -> Rx Data High ->
+ *   Ack bit 1:
+ *	Rx Ack High Post -> Rx Data High ->
+ *   Ack bit 0 && EOM:
+ *	Rx Ack Low -> Rx Ack Low Post -> Rx Ack Finish -> Idle
+ */
+static void cec_pin_rx_states(struct cec_pin *pin, ktime_t ts)
+{
+	s32 delta;
+	bool v;
+	bool ack;
+	bool bcast, for_us;
+	u8 dest;
+
+	switch (pin->state) {
+	/* Receive states */
+	case CEC_ST_RX_START_BIT_LOW:
+		v = cec_pin_read(pin);
+		if (!v)
+			break;
+		pin->state = CEC_ST_RX_START_BIT_HIGH;
+		delta = ktime_us_delta(ts, pin->ts);
+		pin->ts = ts;
+		/* Start bit low is too short, go back to idle */
+		if (delta < CEC_TIM_START_BIT_LOW_MIN -
+			    CEC_TIM_IDLE_SAMPLE) {
+			cec_pin_to_idle(pin);
+		}
+		break;
+
+	case CEC_ST_RX_START_BIT_HIGH:
+		v = cec_pin_read(pin);
+		delta = ktime_us_delta(ts, pin->ts);
+		if (v && delta > CEC_TIM_START_BIT_TOTAL_MAX -
+				 CEC_TIM_START_BIT_LOW_MIN) {
+			cec_pin_to_idle(pin);
+			break;
+		}
+		if (v)
+			break;
+		pin->state = CEC_ST_RX_DATA_SAMPLE;
+		pin->ts = ts;
+		pin->rx_eom = false;
+		break;
+
+	case CEC_ST_RX_DATA_SAMPLE:
+		v = cec_pin_read(pin);
+		pin->state = CEC_ST_RX_DATA_POST_SAMPLE;
+		switch (pin->rx_bit % 10) {
+		default:
+			if (pin->rx_bit / 10 < CEC_MAX_MSG_SIZE)
+				pin->rx_msg.msg[pin->rx_bit / 10] |=
+					v << (7 - (pin->rx_bit % 10));
+			break;
+		case 8:
+			pin->rx_eom = v;
+			pin->rx_msg.len = pin->rx_bit / 10 + 1;
+			break;
+		case 9:
+			break;
+		}
+		pin->rx_bit++;
+		break;
+
+	case CEC_ST_RX_DATA_POST_SAMPLE:
+		pin->state = CEC_ST_RX_DATA_HIGH;
+		break;
+
+	case CEC_ST_RX_DATA_HIGH:
+		v = cec_pin_read(pin);
+		delta = ktime_us_delta(ts, pin->ts);
+		if (v && delta > CEC_TIM_DATA_BIT_TOTAL_MAX) {
+			cec_pin_to_idle(pin);
+			break;
+		}
+		if (v)
+			break;
+		/*
+		 * Go to low drive state when the total bit time is
+		 * too short.
+		 */
+		if (delta < CEC_TIM_DATA_BIT_TOTAL_MIN) {
+			cec_pin_low(pin);
+			pin->state = CEC_ST_LOW_DRIVE;
+			break;
+		}
+		pin->ts = ts;
+		if (pin->rx_bit % 10 != 9) {
+			pin->state = CEC_ST_RX_DATA_SAMPLE;
+			break;
+		}
+
+		dest = cec_msg_destination(&pin->rx_msg);
+		bcast = dest == CEC_LOG_ADDR_BROADCAST;
+		/* for_us == broadcast or directed to us */
+		for_us = bcast || (pin->la_mask & (1 << dest));
+		/* ACK bit value */
+		ack = bcast ? 1 : !for_us;
+
+		if (ack) {
+			/* No need to write to the bus, just wait */
+			pin->state = CEC_ST_RX_ACK_HIGH_POST;
+			break;
+		}
+		cec_pin_low(pin);
+		pin->state = CEC_ST_RX_ACK_LOW;
+		break;
+
+	case CEC_ST_RX_ACK_LOW:
+		cec_pin_high(pin);
+		pin->state = CEC_ST_RX_ACK_LOW_POST;
+		break;
+
+	case CEC_ST_RX_ACK_LOW_POST:
+	case CEC_ST_RX_ACK_HIGH_POST:
+		v = cec_pin_read(pin);
+		if (v && pin->rx_eom) {
+			pin->work_rx_msg = pin->rx_msg;
+			pin->work_rx_msg.rx_ts = ts;
+			wake_up_interruptible(&pin->kthread_waitq);
+			pin->ts = ts;
+			pin->state = CEC_ST_RX_ACK_FINISH;
+			break;
+		}
+		pin->rx_bit++;
+		pin->state = CEC_ST_RX_DATA_HIGH;
+		break;
+
+	case CEC_ST_RX_ACK_FINISH:
+		cec_pin_to_idle(pin);
+		break;
+
+	default:
+		break;
+	}
+}
+
+/*
+ * Main timer function
+ *
+ */
+static enum hrtimer_restart cec_pin_timer(struct hrtimer *timer)
+{
+	struct cec_pin *pin = container_of(timer, struct cec_pin, timer);
+	struct cec_adapter *adap = pin->adap;
+	ktime_t ts;
+	s32 delta;
+
+	ts = ktime_get();
+	if (pin->timer_ts) {
+		delta = ktime_us_delta(ts, pin->timer_ts);
+		pin->timer_cnt++;
+		if (delta > 100 && pin->state != CEC_ST_IDLE) {
+			/* Keep track of timer overruns */
+			pin->timer_sum_overrun += delta;
+			pin->timer_100ms_overruns++;
+			if (delta > 300)
+				pin->timer_300ms_overruns++;
+			if (delta > pin->timer_max_overrun)
+				pin->timer_max_overrun = delta;
+		}
+	}
+	if (adap->monitor_pin_cnt)
+		cec_pin_read(pin);
+
+	if (pin->wait_usecs) {
+		/*
+		 * If we are monitoring the pin, then we have to
+		 * sample at regular intervals.
+		 */
+		if (pin->wait_usecs > 150) {
+			pin->wait_usecs -= 100;
+			pin->timer_ts = ktime_add_us(ts, 100);
+			hrtimer_forward_now(timer, 100000);
+			return HRTIMER_RESTART;
+		}
+		if (pin->wait_usecs > 100) {
+			pin->wait_usecs /= 2;
+			pin->timer_ts = ktime_add_us(ts, pin->wait_usecs);
+			hrtimer_forward_now(timer, pin->wait_usecs * 1000);
+			return HRTIMER_RESTART;
+		}
+		pin->timer_ts = ktime_add_us(ts, pin->wait_usecs);
+		hrtimer_forward_now(timer, pin->wait_usecs * 1000);
+		pin->wait_usecs = 0;
+		return HRTIMER_RESTART;
+	}
+
+	switch (pin->state) {
+	/* Transmit states */
+	case CEC_ST_TX_WAIT_FOR_HIGH:
+	case CEC_ST_TX_START_BIT_LOW:
+	case CEC_ST_TX_DATA_BIT_1_HIGH_POST_SAMPLE:
+	case CEC_ST_TX_DATA_BIT_0_HIGH:
+	case CEC_ST_TX_DATA_BIT_1_HIGH:
+	case CEC_ST_TX_START_BIT_HIGH:
+	case CEC_ST_TX_DATA_BIT_0_LOW:
+	case CEC_ST_TX_DATA_BIT_1_LOW:
+	case CEC_ST_TX_DATA_BIT_1_HIGH_PRE_SAMPLE:
+		cec_pin_tx_states(pin, ts);
+		break;
+
+	/* Receive states */
+	case CEC_ST_RX_START_BIT_LOW:
+	case CEC_ST_RX_START_BIT_HIGH:
+	case CEC_ST_RX_DATA_SAMPLE:
+	case CEC_ST_RX_DATA_POST_SAMPLE:
+	case CEC_ST_RX_DATA_HIGH:
+	case CEC_ST_RX_ACK_LOW:
+	case CEC_ST_RX_ACK_LOW_POST:
+	case CEC_ST_RX_ACK_HIGH_POST:
+	case CEC_ST_RX_ACK_FINISH:
+		cec_pin_rx_states(pin, ts);
+		break;
+
+	case CEC_ST_IDLE:
+	case CEC_ST_TX_WAIT:
+		if (!cec_pin_high(pin)) {
+			/* Start bit, switch to receive state */
+			pin->ts = ts;
+			pin->state = CEC_ST_RX_START_BIT_LOW;
+			break;
+		}
+		if (pin->ts == 0)
+			pin->ts = ts;
+		if (pin->tx_msg.len) {
+			/*
+			 * Check if the bus has been free for long enough
+			 * so we can kick off the pending transmit.
+			 */
+			delta = ktime_us_delta(ts, pin->ts);
+			if (delta / CEC_TIM_DATA_BIT_TOTAL >
+			    pin->tx_signal_free_time) {
+				pin->tx_nacked = false;
+				pin->state = CEC_ST_TX_START_BIT_LOW;
+				/* Generate start bit */
+				cec_pin_low(pin);
+				break;
+			}
+			if (delta / CEC_TIM_DATA_BIT_TOTAL >
+			    pin->tx_signal_free_time - 1)
+				pin->state = CEC_ST_TX_WAIT;
+			break;
+		}
+		if (pin->state != CEC_ST_IDLE || pin->ops->enable_irq == NULL ||
+		    pin->enable_irq_failed || adap->is_configuring ||
+		    adap->is_configured || adap->monitor_all_cnt)
+			break;
+		/* Switch to interrupt mode */
+		atomic_set(&pin->work_irq_change, CEC_PIN_IRQ_ENABLE);
+		pin->state = CEC_ST_RX_IRQ;
+		wake_up_interruptible(&pin->kthread_waitq);
+		return HRTIMER_NORESTART;
+
+	case CEC_ST_LOW_DRIVE:
+		cec_pin_to_idle(pin);
+		break;
+
+	default:
+		break;
+	}
+	if (!adap->monitor_pin_cnt || states[pin->state].usecs <= 150) {
+		pin->wait_usecs = 0;
+		pin->timer_ts = ktime_add_us(ts, states[pin->state].usecs);
+		hrtimer_forward_now(timer, states[pin->state].usecs * 1000);
+		return HRTIMER_RESTART;
+	}
+	pin->wait_usecs = states[pin->state].usecs - 100;
+	pin->timer_ts = ktime_add_us(ts, 100);
+	hrtimer_forward_now(timer, 100000);
+	return HRTIMER_RESTART;
+}
+
+static int cec_pin_thread_func(void *_adap)
+{
+	struct cec_adapter *adap = _adap;
+	struct cec_pin *pin = adap->pin;
+
+	for (;;) {
+		wait_event_interruptible(pin->kthread_waitq,
+			kthread_should_stop() ||
+			pin->work_rx_msg.len ||
+			pin->work_tx_status ||
+			atomic_read(&pin->work_irq_change) ||
+			atomic_read(&pin->work_pin_events));
+
+		if (pin->work_rx_msg.len) {
+			cec_received_msg_ts(adap, &pin->work_rx_msg,
+					    pin->work_rx_msg.rx_ts);
+			pin->work_rx_msg.len = 0;
+		}
+		if (pin->work_tx_status) {
+			unsigned int tx_status = pin->work_tx_status;
+
+			pin->work_tx_status = 0;
+			cec_transmit_attempt_done_ts(adap, tx_status,
+						     pin->work_tx_ts);
+		}
+
+		while (atomic_read(&pin->work_pin_events)) {
+			unsigned int idx = pin->work_pin_events_rd;
+
+			cec_queue_pin_cec_event(adap,
+						pin->work_pin_is_high[idx],
+						pin->work_pin_ts[idx]);
+			pin->work_pin_events_rd = (idx + 1) % CEC_NUM_PIN_EVENTS;
+			atomic_dec(&pin->work_pin_events);
+		}
+
+		switch (atomic_xchg(&pin->work_irq_change,
+				    CEC_PIN_IRQ_UNCHANGED)) {
+		case CEC_PIN_IRQ_DISABLE:
+			pin->ops->disable_irq(adap);
+			cec_pin_high(pin);
+			cec_pin_to_idle(pin);
+			hrtimer_start(&pin->timer, 0, HRTIMER_MODE_REL);
+			break;
+		case CEC_PIN_IRQ_ENABLE:
+			pin->enable_irq_failed = !pin->ops->enable_irq(adap);
+			if (pin->enable_irq_failed) {
+				cec_pin_to_idle(pin);
+				hrtimer_start(&pin->timer, 0, HRTIMER_MODE_REL);
+			}
+			break;
+		default:
+			break;
+		}
+
+		if (kthread_should_stop())
+			break;
+	}
+	return 0;
+}
+
+static int cec_pin_adap_enable(struct cec_adapter *adap, bool enable)
+{
+	struct cec_pin *pin = adap->pin;
+
+	pin->enabled = enable;
+	if (enable) {
+		atomic_set(&pin->work_pin_events, 0);
+		pin->work_pin_events_rd = pin->work_pin_events_wr = 0;
+		cec_pin_read(pin);
+		cec_pin_to_idle(pin);
+		pin->tx_msg.len = 0;
+		pin->timer_ts = 0;
+		atomic_set(&pin->work_irq_change, CEC_PIN_IRQ_UNCHANGED);
+		pin->kthread = kthread_run(cec_pin_thread_func, adap,
+					   "cec-pin");
+		if (IS_ERR(pin->kthread)) {
+			pr_err("cec-pin: kernel_thread() failed\n");
+			return PTR_ERR(pin->kthread);
+		}
+		hrtimer_start(&pin->timer, 0, HRTIMER_MODE_REL);
+	} else {
+		if (pin->ops->disable_irq)
+			pin->ops->disable_irq(adap);
+		hrtimer_cancel(&pin->timer);
+		kthread_stop(pin->kthread);
+		cec_pin_read(pin);
+		cec_pin_to_idle(pin);
+		pin->state = CEC_ST_OFF;
+	}
+	return 0;
+}
+
+static int cec_pin_adap_log_addr(struct cec_adapter *adap, u8 log_addr)
+{
+	struct cec_pin *pin = adap->pin;
+
+	if (log_addr == CEC_LOG_ADDR_INVALID)
+		pin->la_mask = 0;
+	else
+		pin->la_mask |= (1 << log_addr);
+	return 0;
+}
+
+static int cec_pin_adap_transmit(struct cec_adapter *adap, u8 attempts,
+				      u32 signal_free_time, struct cec_msg *msg)
+{
+	struct cec_pin *pin = adap->pin;
+
+	pin->tx_signal_free_time = signal_free_time;
+	pin->tx_msg = *msg;
+	pin->work_tx_status = 0;
+	pin->tx_bit = 0;
+	if (pin->state == CEC_ST_RX_IRQ) {
+		atomic_set(&pin->work_irq_change, CEC_PIN_IRQ_UNCHANGED);
+		pin->ops->disable_irq(adap);
+		cec_pin_high(pin);
+		cec_pin_to_idle(pin);
+		hrtimer_start(&pin->timer, 0, HRTIMER_MODE_REL);
+	}
+	return 0;
+}
+
+static void cec_pin_adap_status(struct cec_adapter *adap,
+				       struct seq_file *file)
+{
+	struct cec_pin *pin = adap->pin;
+
+	seq_printf(file, "state:   %s\n", states[pin->state].name);
+	seq_printf(file, "tx_bit:  %d\n", pin->tx_bit);
+	seq_printf(file, "rx_bit:  %d\n", pin->rx_bit);
+	seq_printf(file, "cec pin: %d\n", pin->ops->read(adap));
+	seq_printf(file, "irq failed: %d\n", pin->enable_irq_failed);
+	if (pin->timer_100ms_overruns) {
+		seq_printf(file, "timer overruns > 100ms: %u of %u\n",
+			   pin->timer_100ms_overruns, pin->timer_cnt);
+		seq_printf(file, "timer overruns > 300ms: %u of %u\n",
+			   pin->timer_300ms_overruns, pin->timer_cnt);
+		seq_printf(file, "max timer overrun: %u usecs\n",
+			   pin->timer_max_overrun);
+		seq_printf(file, "avg timer overrun: %u usecs\n",
+			   pin->timer_sum_overrun / pin->timer_100ms_overruns);
+	}
+	pin->timer_cnt = 0;
+	pin->timer_100ms_overruns = 0;
+	pin->timer_300ms_overruns = 0;
+	pin->timer_max_overrun = 0;
+	pin->timer_sum_overrun = 0;
+	if (pin->ops->status)
+		pin->ops->status(adap, file);
+}
+
+static int cec_pin_adap_monitor_all_enable(struct cec_adapter *adap,
+						  bool enable)
+{
+	struct cec_pin *pin = adap->pin;
+
+	pin->monitor_all = enable;
+	return 0;
+}
+
+static void cec_pin_adap_free(struct cec_adapter *adap)
+{
+	struct cec_pin *pin = adap->pin;
+
+	if (pin->ops->free)
+		pin->ops->free(adap);
+	adap->pin = NULL;
+	kfree(pin);
+}
+
+void cec_pin_changed(struct cec_adapter *adap, bool value)
+{
+	struct cec_pin *pin = adap->pin;
+
+	cec_pin_update(pin, value, false);
+	if (!value && (adap->is_configuring || adap->is_configured ||
+		       adap->monitor_all_cnt))
+		atomic_set(&pin->work_irq_change, CEC_PIN_IRQ_DISABLE);
+}
+EXPORT_SYMBOL_GPL(cec_pin_changed);
+
+static const struct cec_adap_ops cec_pin_adap_ops = {
+	.adap_enable = cec_pin_adap_enable,
+	.adap_monitor_all_enable = cec_pin_adap_monitor_all_enable,
+	.adap_log_addr = cec_pin_adap_log_addr,
+	.adap_transmit = cec_pin_adap_transmit,
+	.adap_status = cec_pin_adap_status,
+	.adap_free = cec_pin_adap_free,
+};
+
+struct cec_adapter *cec_pin_allocate_adapter(const struct cec_pin_ops *pin_ops,
+					void *priv, const char *name, u32 caps)
+{
+	struct cec_adapter *adap;
+	struct cec_pin *pin = kzalloc(sizeof(*pin), GFP_KERNEL);
+
+	if (pin == NULL)
+		return ERR_PTR(-ENOMEM);
+	pin->ops = pin_ops;
+	hrtimer_init(&pin->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+	pin->timer.function = cec_pin_timer;
+	init_waitqueue_head(&pin->kthread_waitq);
+
+	adap = cec_allocate_adapter(&cec_pin_adap_ops, priv, name,
+			    caps | CEC_CAP_MONITOR_ALL | CEC_CAP_MONITOR_PIN,
+			    CEC_MAX_LOG_ADDRS);
+
+	if (PTR_ERR_OR_ZERO(adap)) {
+		kfree(pin);
+		return adap;
+	}
+
+	adap->pin = pin;
+	pin->adap = adap;
+	cec_pin_update(pin, cec_pin_high(pin), true);
+	return adap;
+}
+EXPORT_SYMBOL_GPL(cec_pin_allocate_adapter);
diff --git a/drivers/media/common/saa7146/saa7146_i2c.c b/drivers/media/common/saa7146/saa7146_i2c.c
index 239a2db35068..75897f95e4b4 100644
--- a/drivers/media/common/saa7146/saa7146_i2c.c
+++ b/drivers/media/common/saa7146/saa7146_i2c.c
@@ -395,7 +395,7 @@ static int saa7146_i2c_xfer(struct i2c_adapter* adapter, struct i2c_msg *msg, in
 /* i2c-adapter helper functions                                              */
 
 /* exported algorithm data */
-static struct i2c_algorithm saa7146_algo = {
+static const struct i2c_algorithm saa7146_algo = {
 	.master_xfer	= saa7146_i2c_xfer,
 	.functionality	= saa7146_i2c_func,
 };
diff --git a/drivers/media/common/saa7146/saa7146_vbi.c b/drivers/media/common/saa7146/saa7146_vbi.c
index 3553ac4cba5c..d79e4d7ecd9f 100644
--- a/drivers/media/common/saa7146/saa7146_vbi.c
+++ b/drivers/media/common/saa7146/saa7146_vbi.c
@@ -308,7 +308,7 @@ static void buffer_release(struct videobuf_queue *q, struct videobuf_buffer *vb)
 	saa7146_dma_free(dev,q,buf);
 }
 
-static struct videobuf_queue_ops vbi_qops = {
+static const struct videobuf_queue_ops vbi_qops = {
 	.buf_setup    = buffer_setup,
 	.buf_prepare  = buffer_prepare,
 	.buf_queue    = buffer_queue,
diff --git a/drivers/media/common/saa7146/saa7146_video.c b/drivers/media/common/saa7146/saa7146_video.c
index b3b29d4f36ed..37b4654dc21c 100644
--- a/drivers/media/common/saa7146/saa7146_video.c
+++ b/drivers/media/common/saa7146/saa7146_video.c
@@ -1187,7 +1187,7 @@ static void buffer_release(struct videobuf_queue *q, struct videobuf_buffer *vb)
 	release_all_pagetables(dev, buf);
 }
 
-static struct videobuf_queue_ops video_qops = {
+static const struct videobuf_queue_ops video_qops = {
 	.buf_setup    = buffer_setup,
 	.buf_prepare  = buffer_prepare,
 	.buf_queue    = buffer_queue,
diff --git a/drivers/media/common/siano/smsir.c b/drivers/media/common/siano/smsir.c
index 7c898b06d85c..e77bb0c95e69 100644
--- a/drivers/media/common/siano/smsir.c
+++ b/drivers/media/common/siano/smsir.c
@@ -73,7 +73,7 @@ int sms_ir_init(struct smscore_device_t *coredev)
 	strlcpy(coredev->ir.phys, coredev->devpath, sizeof(coredev->ir.phys));
 	strlcat(coredev->ir.phys, "/ir0", sizeof(coredev->ir.phys));
 
-	dev->input_name = coredev->ir.name;
+	dev->device_name = coredev->ir.name;
 	dev->input_phys = coredev->ir.phys;
 	dev->dev.parent = coredev->device;
 
@@ -86,12 +86,12 @@ int sms_ir_init(struct smscore_device_t *coredev)
 #endif
 
 	dev->priv = coredev;
-	dev->allowed_protocols = RC_BIT_ALL_IR_DECODER;
+	dev->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
 	dev->map_name = sms_get_board(board_id)->rc_codes;
 	dev->driver_name = MODULE_NAME;
 
 	pr_debug("Input device (IR) %s is set for key events\n",
-		 dev->input_name);
+		 dev->device_name);
 
 	err = rc_register_device(dev);
 	if (err < 0) {
diff --git a/drivers/media/common/v4l2-tpg/v4l2-tpg-colors.c b/drivers/media/common/v4l2-tpg/v4l2-tpg-colors.c
index 9bcbd318489b..5b5f95c38fe1 100644
--- a/drivers/media/common/v4l2-tpg/v4l2-tpg-colors.c
+++ b/drivers/media/common/v4l2-tpg/v4l2-tpg-colors.c
@@ -646,14 +646,14 @@ const struct color16 tpg_csc_colors[V4L2_COLORSPACE_DCI_P3 + 1][V4L2_XFER_FUNC_S
 	[V4L2_COLORSPACE_SMPTE170M][V4L2_XFER_FUNC_DCI_P3][5] = { 3248, 944, 1094 },
 	[V4L2_COLORSPACE_SMPTE170M][V4L2_XFER_FUNC_DCI_P3][6] = { 1017, 967, 3168 },
 	[V4L2_COLORSPACE_SMPTE170M][V4L2_XFER_FUNC_DCI_P3][7] = { 1084, 1084, 1084 },
-	[V4L2_COLORSPACE_SMPTE170M][V4L2_XFER_FUNC_SMPTE2084][0] = { 3798, 3798, 3798 },
-	[V4L2_COLORSPACE_SMPTE170M][V4L2_XFER_FUNC_SMPTE2084][1] = { 3802, 3805, 2602 },
-	[V4L2_COLORSPACE_SMPTE170M][V4L2_XFER_FUNC_SMPTE2084][2] = { 0, 3806, 3797 },
-	[V4L2_COLORSPACE_SMPTE170M][V4L2_XFER_FUNC_SMPTE2084][3] = { 1780, 3812, 2592 },
-	[V4L2_COLORSPACE_SMPTE170M][V4L2_XFER_FUNC_SMPTE2084][4] = { 3820, 2215, 3796 },
-	[V4L2_COLORSPACE_SMPTE170M][V4L2_XFER_FUNC_SMPTE2084][5] = { 3824, 2409, 2574 },
-	[V4L2_COLORSPACE_SMPTE170M][V4L2_XFER_FUNC_SMPTE2084][6] = { 2491, 2435, 3795 },
-	[V4L2_COLORSPACE_SMPTE170M][V4L2_XFER_FUNC_SMPTE2084][7] = { 2563, 2563, 2563 },
+	[V4L2_COLORSPACE_SMPTE170M][V4L2_XFER_FUNC_SMPTE2084][0] = { 1812, 1812, 1812 },
+	[V4L2_COLORSPACE_SMPTE170M][V4L2_XFER_FUNC_SMPTE2084][1] = { 1815, 1818, 910 },
+	[V4L2_COLORSPACE_SMPTE170M][V4L2_XFER_FUNC_SMPTE2084][2] = { 0, 1819, 1811 },
+	[V4L2_COLORSPACE_SMPTE170M][V4L2_XFER_FUNC_SMPTE2084][3] = { 472, 1825, 904 },
+	[V4L2_COLORSPACE_SMPTE170M][V4L2_XFER_FUNC_SMPTE2084][4] = { 1832, 686, 1810 },
+	[V4L2_COLORSPACE_SMPTE170M][V4L2_XFER_FUNC_SMPTE2084][5] = { 1835, 794, 893 },
+	[V4L2_COLORSPACE_SMPTE170M][V4L2_XFER_FUNC_SMPTE2084][6] = { 843, 809, 1810 },
+	[V4L2_COLORSPACE_SMPTE170M][V4L2_XFER_FUNC_SMPTE2084][7] = { 886, 886, 886 },
 	[V4L2_COLORSPACE_SMPTE240M][V4L2_XFER_FUNC_709][0] = { 2939, 2939, 2939 },
 	[V4L2_COLORSPACE_SMPTE240M][V4L2_XFER_FUNC_709][1] = { 2953, 2963, 586 },
 	[V4L2_COLORSPACE_SMPTE240M][V4L2_XFER_FUNC_709][2] = { 0, 2967, 2937 },
@@ -702,14 +702,14 @@ const struct color16 tpg_csc_colors[V4L2_COLORSPACE_DCI_P3 + 1][V4L2_XFER_FUNC_S
 	[V4L2_COLORSPACE_SMPTE240M][V4L2_XFER_FUNC_DCI_P3][5] = { 3248, 944, 1094 },
 	[V4L2_COLORSPACE_SMPTE240M][V4L2_XFER_FUNC_DCI_P3][6] = { 1017, 967, 3168 },
 	[V4L2_COLORSPACE_SMPTE240M][V4L2_XFER_FUNC_DCI_P3][7] = { 1084, 1084, 1084 },
-	[V4L2_COLORSPACE_SMPTE240M][V4L2_XFER_FUNC_SMPTE2084][0] = { 3798, 3798, 3798 },
-	[V4L2_COLORSPACE_SMPTE240M][V4L2_XFER_FUNC_SMPTE2084][1] = { 3802, 3805, 2602 },
-	[V4L2_COLORSPACE_SMPTE240M][V4L2_XFER_FUNC_SMPTE2084][2] = { 0, 3806, 3797 },
-	[V4L2_COLORSPACE_SMPTE240M][V4L2_XFER_FUNC_SMPTE2084][3] = { 1780, 3812, 2592 },
-	[V4L2_COLORSPACE_SMPTE240M][V4L2_XFER_FUNC_SMPTE2084][4] = { 3820, 2215, 3796 },
-	[V4L2_COLORSPACE_SMPTE240M][V4L2_XFER_FUNC_SMPTE2084][5] = { 3824, 2409, 2574 },
-	[V4L2_COLORSPACE_SMPTE240M][V4L2_XFER_FUNC_SMPTE2084][6] = { 2491, 2435, 3795 },
-	[V4L2_COLORSPACE_SMPTE240M][V4L2_XFER_FUNC_SMPTE2084][7] = { 2563, 2563, 2563 },
+	[V4L2_COLORSPACE_SMPTE240M][V4L2_XFER_FUNC_SMPTE2084][0] = { 1812, 1812, 1812 },
+	[V4L2_COLORSPACE_SMPTE240M][V4L2_XFER_FUNC_SMPTE2084][1] = { 1815, 1818, 910 },
+	[V4L2_COLORSPACE_SMPTE240M][V4L2_XFER_FUNC_SMPTE2084][2] = { 0, 1819, 1811 },
+	[V4L2_COLORSPACE_SMPTE240M][V4L2_XFER_FUNC_SMPTE2084][3] = { 472, 1825, 904 },
+	[V4L2_COLORSPACE_SMPTE240M][V4L2_XFER_FUNC_SMPTE2084][4] = { 1832, 686, 1810 },
+	[V4L2_COLORSPACE_SMPTE240M][V4L2_XFER_FUNC_SMPTE2084][5] = { 1835, 794, 893 },
+	[V4L2_COLORSPACE_SMPTE240M][V4L2_XFER_FUNC_SMPTE2084][6] = { 843, 809, 1810 },
+	[V4L2_COLORSPACE_SMPTE240M][V4L2_XFER_FUNC_SMPTE2084][7] = { 886, 886, 886 },
 	[V4L2_COLORSPACE_REC709][V4L2_XFER_FUNC_709][0] = { 2939, 2939, 2939 },
 	[V4L2_COLORSPACE_REC709][V4L2_XFER_FUNC_709][1] = { 2939, 2939, 547 },
 	[V4L2_COLORSPACE_REC709][V4L2_XFER_FUNC_709][2] = { 547, 2939, 2939 },
@@ -758,14 +758,14 @@ const struct color16 tpg_csc_colors[V4L2_COLORSPACE_DCI_P3 + 1][V4L2_XFER_FUNC_S
 	[V4L2_COLORSPACE_REC709][V4L2_XFER_FUNC_DCI_P3][5] = { 3175, 1084, 1084 },
 	[V4L2_COLORSPACE_REC709][V4L2_XFER_FUNC_DCI_P3][6] = { 1084, 1084, 3175 },
 	[V4L2_COLORSPACE_REC709][V4L2_XFER_FUNC_DCI_P3][7] = { 1084, 1084, 1084 },
-	[V4L2_COLORSPACE_REC709][V4L2_XFER_FUNC_SMPTE2084][0] = { 3798, 3798, 3798 },
-	[V4L2_COLORSPACE_REC709][V4L2_XFER_FUNC_SMPTE2084][1] = { 3798, 3798, 2563 },
-	[V4L2_COLORSPACE_REC709][V4L2_XFER_FUNC_SMPTE2084][2] = { 2563, 3798, 3798 },
-	[V4L2_COLORSPACE_REC709][V4L2_XFER_FUNC_SMPTE2084][3] = { 2563, 3798, 2563 },
-	[V4L2_COLORSPACE_REC709][V4L2_XFER_FUNC_SMPTE2084][4] = { 3798, 2563, 3798 },
-	[V4L2_COLORSPACE_REC709][V4L2_XFER_FUNC_SMPTE2084][5] = { 3798, 2563, 2563 },
-	[V4L2_COLORSPACE_REC709][V4L2_XFER_FUNC_SMPTE2084][6] = { 2563, 2563, 3798 },
-	[V4L2_COLORSPACE_REC709][V4L2_XFER_FUNC_SMPTE2084][7] = { 2563, 2563, 2563 },
+	[V4L2_COLORSPACE_REC709][V4L2_XFER_FUNC_SMPTE2084][0] = { 1812, 1812, 1812 },
+	[V4L2_COLORSPACE_REC709][V4L2_XFER_FUNC_SMPTE2084][1] = { 1812, 1812, 886 },
+	[V4L2_COLORSPACE_REC709][V4L2_XFER_FUNC_SMPTE2084][2] = { 886, 1812, 1812 },
+	[V4L2_COLORSPACE_REC709][V4L2_XFER_FUNC_SMPTE2084][3] = { 886, 1812, 886 },
+	[V4L2_COLORSPACE_REC709][V4L2_XFER_FUNC_SMPTE2084][4] = { 1812, 886, 1812 },
+	[V4L2_COLORSPACE_REC709][V4L2_XFER_FUNC_SMPTE2084][5] = { 1812, 886, 886 },
+	[V4L2_COLORSPACE_REC709][V4L2_XFER_FUNC_SMPTE2084][6] = { 886, 886, 1812 },
+	[V4L2_COLORSPACE_REC709][V4L2_XFER_FUNC_SMPTE2084][7] = { 886, 886, 886 },
 	[V4L2_COLORSPACE_470_SYSTEM_M][V4L2_XFER_FUNC_709][0] = { 2939, 2939, 2939 },
 	[V4L2_COLORSPACE_470_SYSTEM_M][V4L2_XFER_FUNC_709][1] = { 2892, 3034, 910 },
 	[V4L2_COLORSPACE_470_SYSTEM_M][V4L2_XFER_FUNC_709][2] = { 1715, 2916, 2914 },
@@ -814,14 +814,14 @@ const struct color16 tpg_csc_colors[V4L2_COLORSPACE_DCI_P3 + 1][V4L2_XFER_FUNC_S
 	[V4L2_COLORSPACE_470_SYSTEM_M][V4L2_XFER_FUNC_DCI_P3][5] = { 2765, 1182, 1190 },
 	[V4L2_COLORSPACE_470_SYSTEM_M][V4L2_XFER_FUNC_DCI_P3][6] = { 1270, 0, 3094 },
 	[V4L2_COLORSPACE_470_SYSTEM_M][V4L2_XFER_FUNC_DCI_P3][7] = { 1084, 1084, 1084 },
-	[V4L2_COLORSPACE_470_SYSTEM_M][V4L2_XFER_FUNC_SMPTE2084][0] = { 3798, 3798, 3798 },
-	[V4L2_COLORSPACE_470_SYSTEM_M][V4L2_XFER_FUNC_SMPTE2084][1] = { 3784, 3825, 2879 },
-	[V4L2_COLORSPACE_470_SYSTEM_M][V4L2_XFER_FUNC_SMPTE2084][2] = { 3351, 3791, 3790 },
-	[V4L2_COLORSPACE_470_SYSTEM_M][V4L2_XFER_FUNC_SMPTE2084][3] = { 3311, 3819, 2815 },
-	[V4L2_COLORSPACE_470_SYSTEM_M][V4L2_XFER_FUNC_SMPTE2084][4] = { 3659, 1900, 3777 },
-	[V4L2_COLORSPACE_470_SYSTEM_M][V4L2_XFER_FUNC_SMPTE2084][5] = { 3640, 2662, 2669 },
-	[V4L2_COLORSPACE_470_SYSTEM_M][V4L2_XFER_FUNC_SMPTE2084][6] = { 2743, 0, 3769 },
-	[V4L2_COLORSPACE_470_SYSTEM_M][V4L2_XFER_FUNC_SMPTE2084][7] = { 2563, 2563, 2563 },
+	[V4L2_COLORSPACE_470_SYSTEM_M][V4L2_XFER_FUNC_SMPTE2084][0] = { 1812, 1812, 1812 },
+	[V4L2_COLORSPACE_470_SYSTEM_M][V4L2_XFER_FUNC_SMPTE2084][1] = { 1800, 1836, 1090 },
+	[V4L2_COLORSPACE_470_SYSTEM_M][V4L2_XFER_FUNC_SMPTE2084][2] = { 1436, 1806, 1805 },
+	[V4L2_COLORSPACE_470_SYSTEM_M][V4L2_XFER_FUNC_SMPTE2084][3] = { 1405, 1830, 1047 },
+	[V4L2_COLORSPACE_470_SYSTEM_M][V4L2_XFER_FUNC_SMPTE2084][4] = { 1691, 527, 1793 },
+	[V4L2_COLORSPACE_470_SYSTEM_M][V4L2_XFER_FUNC_SMPTE2084][5] = { 1674, 947, 952 },
+	[V4L2_COLORSPACE_470_SYSTEM_M][V4L2_XFER_FUNC_SMPTE2084][6] = { 1000, 0, 1786 },
+	[V4L2_COLORSPACE_470_SYSTEM_M][V4L2_XFER_FUNC_SMPTE2084][7] = { 886, 886, 886 },
 	[V4L2_COLORSPACE_470_SYSTEM_BG][V4L2_XFER_FUNC_709][0] = { 2939, 2939, 2939 },
 	[V4L2_COLORSPACE_470_SYSTEM_BG][V4L2_XFER_FUNC_709][1] = { 2939, 2939, 464 },
 	[V4L2_COLORSPACE_470_SYSTEM_BG][V4L2_XFER_FUNC_709][2] = { 786, 2939, 2939 },
@@ -870,14 +870,14 @@ const struct color16 tpg_csc_colors[V4L2_COLORSPACE_DCI_P3 + 1][V4L2_XFER_FUNC_S
 	[V4L2_COLORSPACE_470_SYSTEM_BG][V4L2_XFER_FUNC_DCI_P3][5] = { 3126, 1084, 1084 },
 	[V4L2_COLORSPACE_470_SYSTEM_BG][V4L2_XFER_FUNC_DCI_P3][6] = { 1084, 1084, 3188 },
 	[V4L2_COLORSPACE_470_SYSTEM_BG][V4L2_XFER_FUNC_DCI_P3][7] = { 1084, 1084, 1084 },
-	[V4L2_COLORSPACE_470_SYSTEM_BG][V4L2_XFER_FUNC_SMPTE2084][0] = { 3798, 3798, 3798 },
-	[V4L2_COLORSPACE_470_SYSTEM_BG][V4L2_XFER_FUNC_SMPTE2084][1] = { 3798, 3798, 2476 },
-	[V4L2_COLORSPACE_470_SYSTEM_BG][V4L2_XFER_FUNC_SMPTE2084][2] = { 2782, 3798, 3798 },
-	[V4L2_COLORSPACE_470_SYSTEM_BG][V4L2_XFER_FUNC_SMPTE2084][3] = { 2782, 3798, 2476 },
-	[V4L2_COLORSPACE_470_SYSTEM_BG][V4L2_XFER_FUNC_SMPTE2084][4] = { 3780, 2563, 3803 },
-	[V4L2_COLORSPACE_470_SYSTEM_BG][V4L2_XFER_FUNC_SMPTE2084][5] = { 3780, 2563, 2563 },
-	[V4L2_COLORSPACE_470_SYSTEM_BG][V4L2_XFER_FUNC_SMPTE2084][6] = { 2563, 2563, 3803 },
-	[V4L2_COLORSPACE_470_SYSTEM_BG][V4L2_XFER_FUNC_SMPTE2084][7] = { 2563, 2563, 2563 },
+	[V4L2_COLORSPACE_470_SYSTEM_BG][V4L2_XFER_FUNC_SMPTE2084][0] = { 1812, 1812, 1812 },
+	[V4L2_COLORSPACE_470_SYSTEM_BG][V4L2_XFER_FUNC_SMPTE2084][1] = { 1812, 1812, 833 },
+	[V4L2_COLORSPACE_470_SYSTEM_BG][V4L2_XFER_FUNC_SMPTE2084][2] = { 1025, 1812, 1812 },
+	[V4L2_COLORSPACE_470_SYSTEM_BG][V4L2_XFER_FUNC_SMPTE2084][3] = { 1025, 1812, 833 },
+	[V4L2_COLORSPACE_470_SYSTEM_BG][V4L2_XFER_FUNC_SMPTE2084][4] = { 1796, 886, 1816 },
+	[V4L2_COLORSPACE_470_SYSTEM_BG][V4L2_XFER_FUNC_SMPTE2084][5] = { 1796, 886, 886 },
+	[V4L2_COLORSPACE_470_SYSTEM_BG][V4L2_XFER_FUNC_SMPTE2084][6] = { 886, 886, 1816 },
+	[V4L2_COLORSPACE_470_SYSTEM_BG][V4L2_XFER_FUNC_SMPTE2084][7] = { 886, 886, 886 },
 	[V4L2_COLORSPACE_SRGB][V4L2_XFER_FUNC_709][0] = { 2939, 2939, 2939 },
 	[V4L2_COLORSPACE_SRGB][V4L2_XFER_FUNC_709][1] = { 2939, 2939, 547 },
 	[V4L2_COLORSPACE_SRGB][V4L2_XFER_FUNC_709][2] = { 547, 2939, 2939 },
@@ -926,14 +926,14 @@ const struct color16 tpg_csc_colors[V4L2_COLORSPACE_DCI_P3 + 1][V4L2_XFER_FUNC_S
 	[V4L2_COLORSPACE_SRGB][V4L2_XFER_FUNC_DCI_P3][5] = { 3175, 1084, 1084 },
 	[V4L2_COLORSPACE_SRGB][V4L2_XFER_FUNC_DCI_P3][6] = { 1084, 1084, 3175 },
 	[V4L2_COLORSPACE_SRGB][V4L2_XFER_FUNC_DCI_P3][7] = { 1084, 1084, 1084 },
-	[V4L2_COLORSPACE_SRGB][V4L2_XFER_FUNC_SMPTE2084][0] = { 3798, 3798, 3798 },
-	[V4L2_COLORSPACE_SRGB][V4L2_XFER_FUNC_SMPTE2084][1] = { 3798, 3798, 2563 },
-	[V4L2_COLORSPACE_SRGB][V4L2_XFER_FUNC_SMPTE2084][2] = { 2563, 3798, 3798 },
-	[V4L2_COLORSPACE_SRGB][V4L2_XFER_FUNC_SMPTE2084][3] = { 2563, 3798, 2563 },
-	[V4L2_COLORSPACE_SRGB][V4L2_XFER_FUNC_SMPTE2084][4] = { 3798, 2563, 3798 },
-	[V4L2_COLORSPACE_SRGB][V4L2_XFER_FUNC_SMPTE2084][5] = { 3798, 2563, 2563 },
-	[V4L2_COLORSPACE_SRGB][V4L2_XFER_FUNC_SMPTE2084][6] = { 2563, 2563, 3798 },
-	[V4L2_COLORSPACE_SRGB][V4L2_XFER_FUNC_SMPTE2084][7] = { 2563, 2563, 2563 },
+	[V4L2_COLORSPACE_SRGB][V4L2_XFER_FUNC_SMPTE2084][0] = { 1812, 1812, 1812 },
+	[V4L2_COLORSPACE_SRGB][V4L2_XFER_FUNC_SMPTE2084][1] = { 1812, 1812, 886 },
+	[V4L2_COLORSPACE_SRGB][V4L2_XFER_FUNC_SMPTE2084][2] = { 886, 1812, 1812 },
+	[V4L2_COLORSPACE_SRGB][V4L2_XFER_FUNC_SMPTE2084][3] = { 886, 1812, 886 },
+	[V4L2_COLORSPACE_SRGB][V4L2_XFER_FUNC_SMPTE2084][4] = { 1812, 886, 1812 },
+	[V4L2_COLORSPACE_SRGB][V4L2_XFER_FUNC_SMPTE2084][5] = { 1812, 886, 886 },
+	[V4L2_COLORSPACE_SRGB][V4L2_XFER_FUNC_SMPTE2084][6] = { 886, 886, 1812 },
+	[V4L2_COLORSPACE_SRGB][V4L2_XFER_FUNC_SMPTE2084][7] = { 886, 886, 886 },
 	[V4L2_COLORSPACE_ADOBERGB][V4L2_XFER_FUNC_709][0] = { 2939, 2939, 2939 },
 	[V4L2_COLORSPACE_ADOBERGB][V4L2_XFER_FUNC_709][1] = { 2939, 2939, 781 },
 	[V4L2_COLORSPACE_ADOBERGB][V4L2_XFER_FUNC_709][2] = { 1622, 2939, 2939 },
@@ -982,14 +982,14 @@ const struct color16 tpg_csc_colors[V4L2_COLORSPACE_DCI_P3 + 1][V4L2_XFER_FUNC_S
 	[V4L2_COLORSPACE_ADOBERGB][V4L2_XFER_FUNC_DCI_P3][5] = { 2816, 1084, 1084 },
 	[V4L2_COLORSPACE_ADOBERGB][V4L2_XFER_FUNC_DCI_P3][6] = { 1084, 1084, 3127 },
 	[V4L2_COLORSPACE_ADOBERGB][V4L2_XFER_FUNC_DCI_P3][7] = { 1084, 1084, 1084 },
-	[V4L2_COLORSPACE_ADOBERGB][V4L2_XFER_FUNC_SMPTE2084][0] = { 3798, 3798, 3798 },
-	[V4L2_COLORSPACE_ADOBERGB][V4L2_XFER_FUNC_SMPTE2084][1] = { 3798, 3798, 2778 },
-	[V4L2_COLORSPACE_ADOBERGB][V4L2_XFER_FUNC_SMPTE2084][2] = { 3306, 3798, 3798 },
-	[V4L2_COLORSPACE_ADOBERGB][V4L2_XFER_FUNC_SMPTE2084][3] = { 3306, 3798, 2778 },
-	[V4L2_COLORSPACE_ADOBERGB][V4L2_XFER_FUNC_SMPTE2084][4] = { 3661, 2563, 3781 },
-	[V4L2_COLORSPACE_ADOBERGB][V4L2_XFER_FUNC_SMPTE2084][5] = { 3661, 2563, 2563 },
-	[V4L2_COLORSPACE_ADOBERGB][V4L2_XFER_FUNC_SMPTE2084][6] = { 2563, 2563, 3781 },
-	[V4L2_COLORSPACE_ADOBERGB][V4L2_XFER_FUNC_SMPTE2084][7] = { 2563, 2563, 2563 },
+	[V4L2_COLORSPACE_ADOBERGB][V4L2_XFER_FUNC_SMPTE2084][0] = { 1812, 1812, 1812 },
+	[V4L2_COLORSPACE_ADOBERGB][V4L2_XFER_FUNC_SMPTE2084][1] = { 1812, 1812, 1022 },
+	[V4L2_COLORSPACE_ADOBERGB][V4L2_XFER_FUNC_SMPTE2084][2] = { 1402, 1812, 1812 },
+	[V4L2_COLORSPACE_ADOBERGB][V4L2_XFER_FUNC_SMPTE2084][3] = { 1402, 1812, 1022 },
+	[V4L2_COLORSPACE_ADOBERGB][V4L2_XFER_FUNC_SMPTE2084][4] = { 1692, 886, 1797 },
+	[V4L2_COLORSPACE_ADOBERGB][V4L2_XFER_FUNC_SMPTE2084][5] = { 1692, 886, 886 },
+	[V4L2_COLORSPACE_ADOBERGB][V4L2_XFER_FUNC_SMPTE2084][6] = { 886, 886, 1797 },
+	[V4L2_COLORSPACE_ADOBERGB][V4L2_XFER_FUNC_SMPTE2084][7] = { 886, 886, 886 },
 	[V4L2_COLORSPACE_BT2020][V4L2_XFER_FUNC_709][0] = { 2939, 2939, 2939 },
 	[V4L2_COLORSPACE_BT2020][V4L2_XFER_FUNC_709][1] = { 2877, 2923, 1058 },
 	[V4L2_COLORSPACE_BT2020][V4L2_XFER_FUNC_709][2] = { 1837, 2840, 2916 },
@@ -1038,14 +1038,14 @@ const struct color16 tpg_csc_colors[V4L2_COLORSPACE_DCI_P3 + 1][V4L2_XFER_FUNC_S
 	[V4L2_COLORSPACE_BT2020][V4L2_XFER_FUNC_DCI_P3][5] = { 2690, 1431, 1182 },
 	[V4L2_COLORSPACE_BT2020][V4L2_XFER_FUNC_DCI_P3][6] = { 1318, 1153, 3051 },
 	[V4L2_COLORSPACE_BT2020][V4L2_XFER_FUNC_DCI_P3][7] = { 1084, 1084, 1084 },
-	[V4L2_COLORSPACE_BT2020][V4L2_XFER_FUNC_SMPTE2084][0] = { 3798, 3798, 3798 },
-	[V4L2_COLORSPACE_BT2020][V4L2_XFER_FUNC_SMPTE2084][1] = { 3780, 3793, 2984 },
-	[V4L2_COLORSPACE_BT2020][V4L2_XFER_FUNC_SMPTE2084][2] = { 3406, 3768, 3791 },
-	[V4L2_COLORSPACE_BT2020][V4L2_XFER_FUNC_SMPTE2084][3] = { 3359, 3763, 2939 },
-	[V4L2_COLORSPACE_BT2020][V4L2_XFER_FUNC_SMPTE2084][4] = { 3636, 2916, 3760 },
-	[V4L2_COLORSPACE_BT2020][V4L2_XFER_FUNC_SMPTE2084][5] = { 3609, 2880, 2661 },
-	[V4L2_COLORSPACE_BT2020][V4L2_XFER_FUNC_SMPTE2084][6] = { 2786, 2633, 3753 },
-	[V4L2_COLORSPACE_BT2020][V4L2_XFER_FUNC_SMPTE2084][7] = { 2563, 2563, 2563 },
+	[V4L2_COLORSPACE_BT2020][V4L2_XFER_FUNC_SMPTE2084][0] = { 1812, 1812, 1812 },
+	[V4L2_COLORSPACE_BT2020][V4L2_XFER_FUNC_SMPTE2084][1] = { 1796, 1808, 1163 },
+	[V4L2_COLORSPACE_BT2020][V4L2_XFER_FUNC_SMPTE2084][2] = { 1480, 1786, 1806 },
+	[V4L2_COLORSPACE_BT2020][V4L2_XFER_FUNC_SMPTE2084][3] = { 1443, 1781, 1131 },
+	[V4L2_COLORSPACE_BT2020][V4L2_XFER_FUNC_SMPTE2084][4] = { 1670, 1116, 1778 },
+	[V4L2_COLORSPACE_BT2020][V4L2_XFER_FUNC_SMPTE2084][5] = { 1648, 1091, 947 },
+	[V4L2_COLORSPACE_BT2020][V4L2_XFER_FUNC_SMPTE2084][6] = { 1028, 929, 1772 },
+	[V4L2_COLORSPACE_BT2020][V4L2_XFER_FUNC_SMPTE2084][7] = { 886, 886, 886 },
 	[V4L2_COLORSPACE_DCI_P3][V4L2_XFER_FUNC_709][0] = { 2939, 2939, 2939 },
 	[V4L2_COLORSPACE_DCI_P3][V4L2_XFER_FUNC_709][1] = { 2936, 2934, 992 },
 	[V4L2_COLORSPACE_DCI_P3][V4L2_XFER_FUNC_709][2] = { 1159, 2890, 2916 },
@@ -1094,14 +1094,14 @@ const struct color16 tpg_csc_colors[V4L2_COLORSPACE_DCI_P3 + 1][V4L2_XFER_FUNC_S
 	[V4L2_COLORSPACE_DCI_P3][V4L2_XFER_FUNC_DCI_P3][5] = { 3018, 1276, 1184 },
 	[V4L2_COLORSPACE_DCI_P3][V4L2_XFER_FUNC_DCI_P3][6] = { 1100, 1107, 3071 },
 	[V4L2_COLORSPACE_DCI_P3][V4L2_XFER_FUNC_DCI_P3][7] = { 1084, 1084, 1084 },
-	[V4L2_COLORSPACE_DCI_P3][V4L2_XFER_FUNC_SMPTE2084][0] = { 3798, 3798, 3798 },
-	[V4L2_COLORSPACE_DCI_P3][V4L2_XFER_FUNC_SMPTE2084][1] = { 3797, 3796, 2938 },
-	[V4L2_COLORSPACE_DCI_P3][V4L2_XFER_FUNC_SMPTE2084][2] = { 3049, 3783, 3791 },
-	[V4L2_COLORSPACE_DCI_P3][V4L2_XFER_FUNC_SMPTE2084][3] = { 3044, 3782, 2887 },
-	[V4L2_COLORSPACE_DCI_P3][V4L2_XFER_FUNC_SMPTE2084][4] = { 3741, 2765, 3768 },
-	[V4L2_COLORSPACE_DCI_P3][V4L2_XFER_FUNC_SMPTE2084][5] = { 3740, 2749, 2663 },
-	[V4L2_COLORSPACE_DCI_P3][V4L2_XFER_FUNC_SMPTE2084][6] = { 2580, 2587, 3760 },
-	[V4L2_COLORSPACE_DCI_P3][V4L2_XFER_FUNC_SMPTE2084][7] = { 2563, 2563, 2563 },
+	[V4L2_COLORSPACE_DCI_P3][V4L2_XFER_FUNC_SMPTE2084][0] = { 1812, 1812, 1812 },
+	[V4L2_COLORSPACE_DCI_P3][V4L2_XFER_FUNC_SMPTE2084][1] = { 1811, 1810, 1131 },
+	[V4L2_COLORSPACE_DCI_P3][V4L2_XFER_FUNC_SMPTE2084][2] = { 1210, 1799, 1806 },
+	[V4L2_COLORSPACE_DCI_P3][V4L2_XFER_FUNC_SMPTE2084][3] = { 1206, 1798, 1096 },
+	[V4L2_COLORSPACE_DCI_P3][V4L2_XFER_FUNC_SMPTE2084][4] = { 1762, 1014, 1785 },
+	[V4L2_COLORSPACE_DCI_P3][V4L2_XFER_FUNC_SMPTE2084][5] = { 1761, 1004, 948 },
+	[V4L2_COLORSPACE_DCI_P3][V4L2_XFER_FUNC_SMPTE2084][6] = { 896, 901, 1778 },
+	[V4L2_COLORSPACE_DCI_P3][V4L2_XFER_FUNC_SMPTE2084][7] = { 886, 886, 886 },
 };
 
 #else
@@ -1225,6 +1225,12 @@ static double transfer_rgb_to_smpte2084(double v)
 	const double c2 = 32.0 * 2413.0 / 4096.0;
 	const double c3 = 32.0 * 2392.0 / 4096.0;
 
+	/*
+	 * The RGB input maps to the luminance range 0-100 cd/m^2, while
+	 * SMPTE-2084 maps values to the luminance range of 0-10000 cd/m^2.
+	 * Hence the factor 100.
+	 */
+	v /= 100.0;
 	v = pow(v, m1);
 	return pow((c1 + c2 * v) / (1 + c3 * v), m2);
 }
diff --git a/drivers/media/common/v4l2-tpg/v4l2-tpg-core.c b/drivers/media/common/v4l2-tpg/v4l2-tpg-core.c
index 3dd22da7e17d..a772976cfe26 100644
--- a/drivers/media/common/v4l2-tpg/v4l2-tpg-core.c
+++ b/drivers/media/common/v4l2-tpg/v4l2-tpg-core.c
@@ -615,7 +615,7 @@ static void color_to_ycbcr(struct tpg_data *tpg, int r, int g, int b,
 	static const int bt2020_full[3][3] = {
 		{ COEFF(0.2627, 255),  COEFF(0.6780, 255),  COEFF(0.0593, 255)  },
 		{ COEFF(-0.1396, 255), COEFF(-0.3604, 255), COEFF(0.5, 255)     },
-		{ COEFF(0.5, 255),     COEFF(-0.4698, 255), COEFF(-0.0402, 255) },
+		{ COEFF(0.5, 255),     COEFF(-0.4598, 255), COEFF(-0.0402, 255) },
 	};
 	static const int bt2020c[4] = {
 		COEFF(1.0 / 1.9404, 224), COEFF(1.0 / 1.5816, 224),
diff --git a/drivers/media/dvb-core/demux.h b/drivers/media/dvb-core/demux.h
index f854309ba8a5..c4df6cee48e6 100644
--- a/drivers/media/dvb-core/demux.h
+++ b/drivers/media/dvb-core/demux.h
@@ -210,7 +210,7 @@ struct dmx_section_feed {
  * the start of the first undelivered TS packet within a circular buffer.
  * The @buffer2 buffer parameter is normally NULL, except when the received
  * TS packets have crossed the last address of the circular buffer and
- * �wrapped� to the beginning of the buffer. In the latter case the @buffer1
+ * "wrapped" to the beginning of the buffer. In the latter case the @buffer1
  * parameter would contain an address within the circular buffer, while the
  * @buffer2 parameter would contain the first address of the circular buffer.
  * The number of bytes delivered with this function (i.e. @buffer1_length +
diff --git a/drivers/media/dvb-core/dmxdev.c b/drivers/media/dvb-core/dmxdev.c
index 45e91add73ba..18e4230865be 100644
--- a/drivers/media/dvb-core/dmxdev.c
+++ b/drivers/media/dvb-core/dmxdev.c
@@ -562,7 +562,7 @@ static int dvb_dmxdev_start_feed(struct dmxdev *dmxdev,
 {
 	ktime_t timeout = 0;
 	struct dmx_pes_filter_params *para = &filter->params.pes;
-	dmx_output_t otype;
+	enum dmx_output otype;
 	int ret;
 	int ts_type;
 	enum dmx_ts_pes ts_pes;
@@ -787,7 +787,7 @@ static int dvb_dmxdev_filter_free(struct dmxdev *dmxdev,
 	return 0;
 }
 
-static inline void invert_mode(dmx_filter_t *filter)
+static inline void invert_mode(struct dmx_filter *filter)
 {
 	int i;
 
@@ -1025,26 +1025,6 @@ static int dvb_demux_do_ioctl(struct file *file,
 		dmxdev->demux->get_pes_pids(dmxdev->demux, parg);
 		break;
 
-#if 0
-	/* Not used upstream and never documented */
-
-	case DMX_GET_CAPS:
-		if (!dmxdev->demux->get_caps) {
-			ret = -EINVAL;
-			break;
-		}
-		ret = dmxdev->demux->get_caps(dmxdev->demux, parg);
-		break;
-
-	case DMX_SET_SOURCE:
-		if (!dmxdev->demux->set_source) {
-			ret = -EINVAL;
-			break;
-		}
-		ret = dmxdev->demux->set_source(dmxdev->demux, parg);
-		break;
-#endif
-
 	case DMX_GET_STC:
 		if (!dmxdev->demux->get_stc) {
 			ret = -EINVAL;
diff --git a/drivers/media/dvb-core/dvb-usb-ids.h b/drivers/media/dvb-core/dvb-usb-ids.h
index e200aa6f2d2f..5b6041d462bc 100644
--- a/drivers/media/dvb-core/dvb-usb-ids.h
+++ b/drivers/media/dvb-core/dvb-usb-ids.h
@@ -279,6 +279,7 @@
 #define USB_PID_TERRATEC_H7				0x10b4
 #define USB_PID_TERRATEC_H7_2				0x10a3
 #define USB_PID_TERRATEC_H7_3				0x10a5
+#define USB_PID_TERRATEC_T1				0x10ae
 #define USB_PID_TERRATEC_T3				0x10a0
 #define USB_PID_TERRATEC_T5				0x10a1
 #define USB_PID_NOXON_DAB_STICK				0x00b3
diff --git a/drivers/media/dvb-core/dvb_ca_en50221.c b/drivers/media/dvb-core/dvb_ca_en50221.c
index 17970cdd55fa..95b3723282f4 100644
--- a/drivers/media/dvb-core/dvb_ca_en50221.c
+++ b/drivers/media/dvb-core/dvb_ca_en50221.c
@@ -76,8 +76,6 @@ MODULE_PARM_DESC(cam_debug, "enable verbose debug messages");
 #define STATUSREG_WE     2	/* write error */
 #define STATUSREG_FR  0x40	/* module free */
 #define STATUSREG_DA  0x80	/* data available */
-#define STATUSREG_TXERR (STATUSREG_RE|STATUSREG_WE)	/* general transfer error */
-
 
 #define DVB_CA_SLOTSTATE_NONE           0
 #define DVB_CA_SLOTSTATE_UNINITIALISED  1
@@ -88,10 +86,8 @@ MODULE_PARM_DESC(cam_debug, "enable verbose debug messages");
 #define DVB_CA_SLOTSTATE_WAITFR         6
 #define DVB_CA_SLOTSTATE_LINKINIT       7
 
-
 /* Information on a CA slot */
 struct dvb_ca_slot {
-
 	/* current state of the CAM */
 	int slot_state;
 
@@ -157,7 +153,10 @@ struct dvb_ca_private {
 	/* Delay the main thread should use */
 	unsigned long delay;
 
-	/* Slot to start looking for data to read from in the next user-space read operation */
+	/*
+	 * Slot to start looking for data to read from in the next user-space
+	 * read operation
+	 */
 	int next_read_slot;
 
 	/* mutex serializing ioctls */
@@ -178,7 +177,9 @@ static void dvb_ca_private_free(struct dvb_ca_private *ca)
 
 static void dvb_ca_private_release(struct kref *ref)
 {
-	struct dvb_ca_private *ca = container_of(ref, struct dvb_ca_private, refcount);
+	struct dvb_ca_private *ca;
+
+	ca = container_of(ref, struct dvb_ca_private, refcount);
 	dvb_ca_private_free(ca);
 }
 
@@ -198,7 +199,6 @@ static int dvb_ca_en50221_read_data(struct dvb_ca_private *ca, int slot,
 static int dvb_ca_en50221_write_data(struct dvb_ca_private *ca, int slot,
 				     u8 *ebuf, int ecount);
 
-
 /**
  * Safely find needle in haystack.
  *
@@ -223,25 +223,22 @@ static char *findstr(char *haystack, int hlen, char *needle, int nlen)
 	return NULL;
 }
 
-
-
-/* ******************************************************************************** */
+/* ************************************************************************** */
 /* EN50221 physical interface functions */
 
-
 /**
  * dvb_ca_en50221_check_camstatus - Check CAM status.
  */
 static int dvb_ca_en50221_check_camstatus(struct dvb_ca_private *ca, int slot)
 {
+	struct dvb_ca_slot *sl = &ca->slot_info[slot];
 	int slot_status;
 	int cam_present_now;
 	int cam_changed;
 
 	/* IRQ mode */
-	if (ca->flags & DVB_CA_EN50221_FLAG_IRQ_CAMCHANGE) {
-		return (atomic_read(&ca->slot_info[slot].camchange_count) != 0);
-	}
+	if (ca->flags & DVB_CA_EN50221_FLAG_IRQ_CAMCHANGE)
+		return (atomic_read(&sl->camchange_count) != 0);
 
 	/* poll mode */
 	slot_status = ca->pub->poll_slot_status(ca->pub, slot, ca->open);
@@ -249,29 +246,28 @@ static int dvb_ca_en50221_check_camstatus(struct dvb_ca_private *ca, int slot)
 	cam_present_now = (slot_status & DVB_CA_EN50221_POLL_CAM_PRESENT) ? 1 : 0;
 	cam_changed = (slot_status & DVB_CA_EN50221_POLL_CAM_CHANGED) ? 1 : 0;
 	if (!cam_changed) {
-		int cam_present_old = (ca->slot_info[slot].slot_state != DVB_CA_SLOTSTATE_NONE);
+		int cam_present_old = (sl->slot_state != DVB_CA_SLOTSTATE_NONE);
+
 		cam_changed = (cam_present_now != cam_present_old);
 	}
 
 	if (cam_changed) {
-		if (!cam_present_now) {
-			ca->slot_info[slot].camchange_type = DVB_CA_EN50221_CAMCHANGE_REMOVED;
-		} else {
-			ca->slot_info[slot].camchange_type = DVB_CA_EN50221_CAMCHANGE_INSERTED;
-		}
-		atomic_set(&ca->slot_info[slot].camchange_count, 1);
+		if (!cam_present_now)
+			sl->camchange_type = DVB_CA_EN50221_CAMCHANGE_REMOVED;
+		else
+			sl->camchange_type = DVB_CA_EN50221_CAMCHANGE_INSERTED;
+		atomic_set(&sl->camchange_count, 1);
 	} else {
-		if ((ca->slot_info[slot].slot_state == DVB_CA_SLOTSTATE_WAITREADY) &&
+		if ((sl->slot_state == DVB_CA_SLOTSTATE_WAITREADY) &&
 		    (slot_status & DVB_CA_EN50221_POLL_CAM_READY)) {
-			// move to validate state if reset is completed
-			ca->slot_info[slot].slot_state = DVB_CA_SLOTSTATE_VALIDATE;
+			/* move to validate state if reset is completed */
+			sl->slot_state = DVB_CA_SLOTSTATE_VALIDATE;
 		}
 	}
 
 	return cam_changed;
 }
 
-
 /**
  * dvb_ca_en50221_wait_if_status - Wait for flags to become set on the STATUS
  *	 register on a CAM interface, checking for errors and timeout.
@@ -295,8 +291,10 @@ static int dvb_ca_en50221_wait_if_status(struct dvb_ca_private *ca, int slot,
 	start = jiffies;
 	timeout = jiffies + timeout_hz;
 	while (1) {
+		int res;
+
 		/* read the status and check for error */
-		int res = ca->pub->read_cam_control(ca->pub, slot, CTRLIF_STATUS);
+		res = ca->pub->read_cam_control(ca->pub, slot, CTRLIF_STATUS);
 		if (res < 0)
 			return -EIO;
 
@@ -308,12 +306,11 @@ static int dvb_ca_en50221_wait_if_status(struct dvb_ca_private *ca, int slot,
 		}
 
 		/* check for timeout */
-		if (time_after(jiffies, timeout)) {
+		if (time_after(jiffies, timeout))
 			break;
-		}
 
 		/* wait for a bit */
-		msleep(1);
+		usleep_range(1000, 1100);
 	}
 
 	dprintk("%s failed timeout:%lu\n", __func__, jiffies - start);
@@ -322,7 +319,6 @@ static int dvb_ca_en50221_wait_if_status(struct dvb_ca_private *ca, int slot,
 	return -ETIMEDOUT;
 }
 
-
 /**
  * dvb_ca_en50221_link_init - Initialise the link layer connection to a CAM.
  *
@@ -333,6 +329,7 @@ static int dvb_ca_en50221_wait_if_status(struct dvb_ca_private *ca, int slot,
  */
 static int dvb_ca_en50221_link_init(struct dvb_ca_private *ca, int slot)
 {
+	struct dvb_ca_slot *sl = &ca->slot_info[slot];
 	int ret;
 	int buf_size;
 	u8 buf[2];
@@ -340,40 +337,54 @@ static int dvb_ca_en50221_link_init(struct dvb_ca_private *ca, int slot)
 	dprintk("%s\n", __func__);
 
 	/* we'll be determining these during this function */
-	ca->slot_info[slot].da_irq_supported = 0;
+	sl->da_irq_supported = 0;
 
-	/* set the host link buffer size temporarily. it will be overwritten with the
-	 * real negotiated size later. */
-	ca->slot_info[slot].link_buf_size = 2;
+	/*
+	 * set the host link buffer size temporarily. it will be overwritten
+	 * with the real negotiated size later.
+	 */
+	sl->link_buf_size = 2;
 
 	/* read the buffer size from the CAM */
-	if ((ret = ca->pub->write_cam_control(ca->pub, slot, CTRLIF_COMMAND, IRQEN | CMDREG_SR)) != 0)
+	ret = ca->pub->write_cam_control(ca->pub, slot, CTRLIF_COMMAND,
+					 IRQEN | CMDREG_SR);
+	if (ret)
 		return ret;
 	ret = dvb_ca_en50221_wait_if_status(ca, slot, STATUSREG_DA, HZ);
-	if (ret != 0)
+	if (ret)
 		return ret;
-	if ((ret = dvb_ca_en50221_read_data(ca, slot, buf, 2)) != 2)
+	ret = dvb_ca_en50221_read_data(ca, slot, buf, 2);
+	if (ret != 2)
 		return -EIO;
-	if ((ret = ca->pub->write_cam_control(ca->pub, slot, CTRLIF_COMMAND, IRQEN)) != 0)
+	ret = ca->pub->write_cam_control(ca->pub, slot, CTRLIF_COMMAND, IRQEN);
+	if (ret)
 		return ret;
 
-	/* store it, and choose the minimum of our buffer and the CAM's buffer size */
+	/*
+	 * store it, and choose the minimum of our buffer and the CAM's buffer
+	 * size
+	 */
 	buf_size = (buf[0] << 8) | buf[1];
 	if (buf_size > HOST_LINK_BUF_SIZE)
 		buf_size = HOST_LINK_BUF_SIZE;
-	ca->slot_info[slot].link_buf_size = buf_size;
+	sl->link_buf_size = buf_size;
 	buf[0] = buf_size >> 8;
 	buf[1] = buf_size & 0xff;
 	dprintk("Chosen link buffer size of %i\n", buf_size);
 
 	/* write the buffer size to the CAM */
-	if ((ret = ca->pub->write_cam_control(ca->pub, slot, CTRLIF_COMMAND, IRQEN | CMDREG_SW)) != 0)
+	ret = ca->pub->write_cam_control(ca->pub, slot, CTRLIF_COMMAND,
+					 IRQEN | CMDREG_SW);
+	if (ret)
 		return ret;
-	if ((ret = dvb_ca_en50221_wait_if_status(ca, slot, STATUSREG_FR, HZ / 10)) != 0)
+	ret = dvb_ca_en50221_wait_if_status(ca, slot, STATUSREG_FR, HZ / 10);
+	if (ret)
 		return ret;
-	if ((ret = dvb_ca_en50221_write_data(ca, slot, buf, 2)) != 2)
+	ret = dvb_ca_en50221_write_data(ca, slot, buf, 2);
+	if (ret != 2)
 		return -EIO;
-	if ((ret = ca->pub->write_cam_control(ca->pub, slot, CTRLIF_COMMAND, IRQEN)) != 0)
+	ret = ca->pub->write_cam_control(ca->pub, slot, CTRLIF_COMMAND, IRQEN);
+	if (ret)
 		return ret;
 
 	/* success */
@@ -393,47 +404,50 @@ static int dvb_ca_en50221_link_init(struct dvb_ca_private *ca, int slot)
  * @return 0 on success, nonzero on error.
  */
 static int dvb_ca_en50221_read_tuple(struct dvb_ca_private *ca, int slot,
-				     int *address, int *tupleType,
-				     int *tupleLength, u8 *tuple)
+				     int *address, int *tuple_type,
+				     int *tuple_length, u8 *tuple)
 {
 	int i;
-	int _tupleType;
-	int _tupleLength;
+	int _tuple_type;
+	int _tuple_length;
 	int _address = *address;
 
 	/* grab the next tuple length and type */
-	if ((_tupleType = ca->pub->read_attribute_mem(ca->pub, slot, _address)) < 0)
-		return _tupleType;
-	if (_tupleType == 0xff) {
-		dprintk("END OF CHAIN TUPLE type:0x%x\n", _tupleType);
+	_tuple_type = ca->pub->read_attribute_mem(ca->pub, slot, _address);
+	if (_tuple_type < 0)
+		return _tuple_type;
+	if (_tuple_type == 0xff) {
+		dprintk("END OF CHAIN TUPLE type:0x%x\n", _tuple_type);
 		*address += 2;
-		*tupleType = _tupleType;
-		*tupleLength = 0;
+		*tuple_type = _tuple_type;
+		*tuple_length = 0;
 		return 0;
 	}
-	if ((_tupleLength = ca->pub->read_attribute_mem(ca->pub, slot, _address + 2)) < 0)
-		return _tupleLength;
+	_tuple_length = ca->pub->read_attribute_mem(ca->pub, slot,
+						    _address + 2);
+	if (_tuple_length < 0)
+		return _tuple_length;
 	_address += 4;
 
-	dprintk("TUPLE type:0x%x length:%i\n", _tupleType, _tupleLength);
+	dprintk("TUPLE type:0x%x length:%i\n", _tuple_type, _tuple_length);
 
 	/* read in the whole tuple */
-	for (i = 0; i < _tupleLength; i++) {
-		tuple[i] = ca->pub->read_attribute_mem(ca->pub, slot, _address + (i * 2));
+	for (i = 0; i < _tuple_length; i++) {
+		tuple[i] = ca->pub->read_attribute_mem(ca->pub, slot,
+						       _address + (i * 2));
 		dprintk("  0x%02x: 0x%02x %c\n",
 			i, tuple[i] & 0xff,
 			((tuple[i] > 31) && (tuple[i] < 127)) ? tuple[i] : '.');
 	}
-	_address += (_tupleLength * 2);
+	_address += (_tuple_length * 2);
 
-	// success
-	*tupleType = _tupleType;
-	*tupleLength = _tupleLength;
+	/* success */
+	*tuple_type = _tuple_type;
+	*tuple_length = _tuple_length;
 	*address = _address;
 	return 0;
 }
 
-
 /**
  * dvb_ca_en50221_parse_attributes - Parse attribute memory of a CAM module,
  *	extracting Config register, and checking it is a DVB CAM module.
@@ -445,9 +459,10 @@ static int dvb_ca_en50221_read_tuple(struct dvb_ca_private *ca, int slot,
  */
 static int dvb_ca_en50221_parse_attributes(struct dvb_ca_private *ca, int slot)
 {
+	struct dvb_ca_slot *sl;
 	int address = 0;
-	int tupleLength;
-	int tupleType;
+	int tuple_length;
+	int tuple_type;
 	u8 tuple[257];
 	char *dvb_str;
 	int rasz;
@@ -458,70 +473,66 @@ static int dvb_ca_en50221_parse_attributes(struct dvb_ca_private *ca, int slot)
 	u16 manfid = 0;
 	u16 devid = 0;
 
-
-	// CISTPL_DEVICE_0A
-	if ((status =
-	     dvb_ca_en50221_read_tuple(ca, slot, &address, &tupleType, &tupleLength, tuple)) < 0)
+	/* CISTPL_DEVICE_0A */
+	status = dvb_ca_en50221_read_tuple(ca, slot, &address, &tuple_type,
+					   &tuple_length, tuple);
+	if (status < 0)
 		return status;
-	if (tupleType != 0x1D)
+	if (tuple_type != 0x1D)
 		return -EINVAL;
 
-
-
-	// CISTPL_DEVICE_0C
-	if ((status =
-	     dvb_ca_en50221_read_tuple(ca, slot, &address, &tupleType, &tupleLength, tuple)) < 0)
+	/* CISTPL_DEVICE_0C */
+	status = dvb_ca_en50221_read_tuple(ca, slot, &address, &tuple_type,
+					   &tuple_length, tuple);
+	if (status < 0)
 		return status;
-	if (tupleType != 0x1C)
+	if (tuple_type != 0x1C)
 		return -EINVAL;
 
-
-
-	// CISTPL_VERS_1
-	if ((status =
-	     dvb_ca_en50221_read_tuple(ca, slot, &address, &tupleType, &tupleLength, tuple)) < 0)
+	/* CISTPL_VERS_1 */
+	status = dvb_ca_en50221_read_tuple(ca, slot, &address, &tuple_type,
+					   &tuple_length, tuple);
+	if (status < 0)
 		return status;
-	if (tupleType != 0x15)
+	if (tuple_type != 0x15)
 		return -EINVAL;
 
-
-
-	// CISTPL_MANFID
-	if ((status = dvb_ca_en50221_read_tuple(ca, slot, &address, &tupleType,
-						&tupleLength, tuple)) < 0)
+	/* CISTPL_MANFID */
+	status = dvb_ca_en50221_read_tuple(ca, slot, &address, &tuple_type,
+					   &tuple_length, tuple);
+	if (status < 0)
 		return status;
-	if (tupleType != 0x20)
+	if (tuple_type != 0x20)
 		return -EINVAL;
-	if (tupleLength != 4)
+	if (tuple_length != 4)
 		return -EINVAL;
 	manfid = (tuple[1] << 8) | tuple[0];
 	devid = (tuple[3] << 8) | tuple[2];
 
-
-
-	// CISTPL_CONFIG
-	if ((status = dvb_ca_en50221_read_tuple(ca, slot, &address, &tupleType,
-						&tupleLength, tuple)) < 0)
+	/* CISTPL_CONFIG */
+	status = dvb_ca_en50221_read_tuple(ca, slot, &address, &tuple_type,
+					   &tuple_length, tuple);
+	if (status < 0)
 		return status;
-	if (tupleType != 0x1A)
+	if (tuple_type != 0x1A)
 		return -EINVAL;
-	if (tupleLength < 3)
+	if (tuple_length < 3)
 		return -EINVAL;
 
 	/* extract the configbase */
 	rasz = tuple[0] & 3;
-	if (tupleLength < (3 + rasz + 14))
+	if (tuple_length < (3 + rasz + 14))
 		return -EINVAL;
-	ca->slot_info[slot].config_base = 0;
-	for (i = 0; i < rasz + 1; i++) {
-		ca->slot_info[slot].config_base |= (tuple[2 + i] << (8 * i));
-	}
+	sl = &ca->slot_info[slot];
+	sl->config_base = 0;
+	for (i = 0; i < rasz + 1; i++)
+		sl->config_base |= (tuple[2 + i] << (8 * i));
 
 	/* check it contains the correct DVB string */
-	dvb_str = findstr((char *)tuple, tupleLength, "DVB_CI_V", 8);
-	if (dvb_str == NULL)
+	dvb_str = findstr((char *)tuple, tuple_length, "DVB_CI_V", 8);
+	if (!dvb_str)
 		return -EINVAL;
-	if (tupleLength < ((dvb_str - (char *) tuple) + 12))
+	if (tuple_length < ((dvb_str - (char *)tuple) + 12))
 		return -EINVAL;
 
 	/* is it a version we support? */
@@ -534,12 +545,14 @@ static int dvb_ca_en50221_parse_attributes(struct dvb_ca_private *ca, int slot)
 
 	/* process the CFTABLE_ENTRY tuples, and any after those */
 	while ((!end_chain) && (address < 0x1000)) {
-		if ((status = dvb_ca_en50221_read_tuple(ca, slot, &address, &tupleType,
-							&tupleLength, tuple)) < 0)
+		status = dvb_ca_en50221_read_tuple(ca, slot, &address,
+						   &tuple_type, &tuple_length,
+						   tuple);
+		if (status < 0)
 			return status;
-		switch (tupleType) {
-		case 0x1B:	// CISTPL_CFTABLE_ENTRY
-			if (tupleLength < (2 + 11 + 17))
+		switch (tuple_type) {
+		case 0x1B:	/* CISTPL_CFTABLE_ENTRY */
+			if (tuple_length < (2 + 11 + 17))
 				break;
 
 			/* if we've already parsed one, just use it */
@@ -547,26 +560,28 @@ static int dvb_ca_en50221_parse_attributes(struct dvb_ca_private *ca, int slot)
 				break;
 
 			/* get the config option */
-			ca->slot_info[slot].config_option = tuple[0] & 0x3f;
+			sl->config_option = tuple[0] & 0x3f;
 
 			/* OK, check it contains the correct strings */
-			if ((findstr((char *)tuple, tupleLength, "DVB_HOST", 8) == NULL) ||
-			    (findstr((char *)tuple, tupleLength, "DVB_CI_MODULE", 13) == NULL))
+			if (!findstr((char *)tuple, tuple_length,
+				     "DVB_HOST", 8) ||
+			    !findstr((char *)tuple, tuple_length,
+				     "DVB_CI_MODULE", 13))
 				break;
 
 			got_cftableentry = 1;
 			break;
 
-		case 0x14:	// CISTPL_NO_LINK
+		case 0x14:	/* CISTPL_NO_LINK */
 			break;
 
-		case 0xFF:	// CISTPL_END
+		case 0xFF:	/* CISTPL_END */
 			end_chain = 1;
 			break;
 
-		default:	/* Unknown tuple type - just skip this tuple and move to the next one */
+		default:	/* Unknown tuple type - just skip this tuple */
 			dprintk("dvb_ca: Skipping unknown tuple type:0x%x length:0x%x\n",
-				tupleType, tupleLength);
+				tuple_type, tuple_length);
 			break;
 		}
 	}
@@ -575,14 +590,12 @@ static int dvb_ca_en50221_parse_attributes(struct dvb_ca_private *ca, int slot)
 		return -EINVAL;
 
 	dprintk("Valid DVB CAM detected MANID:%x DEVID:%x CONFIGBASE:0x%x CONFIGOPTION:0x%x\n",
-		manfid, devid, ca->slot_info[slot].config_base,
-		ca->slot_info[slot].config_option);
+		manfid, devid, sl->config_base, sl->config_option);
 
-	// success!
+	/* success! */
 	return 0;
 }
 
-
 /**
  * dvb_ca_en50221_set_configoption - Set CAM's configoption correctly.
  *
@@ -591,26 +604,25 @@ static int dvb_ca_en50221_parse_attributes(struct dvb_ca_private *ca, int slot)
  */
 static int dvb_ca_en50221_set_configoption(struct dvb_ca_private *ca, int slot)
 {
+	struct dvb_ca_slot *sl = &ca->slot_info[slot];
 	int configoption;
 
 	dprintk("%s\n", __func__);
 
 	/* set the config option */
-	ca->pub->write_attribute_mem(ca->pub, slot,
-				     ca->slot_info[slot].config_base,
-				     ca->slot_info[slot].config_option);
+	ca->pub->write_attribute_mem(ca->pub, slot, sl->config_base,
+				     sl->config_option);
 
 	/* check it */
-	configoption = ca->pub->read_attribute_mem(ca->pub, slot, ca->slot_info[slot].config_base);
+	configoption = ca->pub->read_attribute_mem(ca->pub, slot,
+						   sl->config_base);
 	dprintk("Set configoption 0x%x, read configoption 0x%x\n",
-		ca->slot_info[slot].config_option, configoption & 0x3f);
+		sl->config_option, configoption & 0x3f);
 
 	/* fine! */
 	return 0;
-
 }
 
-
 /**
  * dvb_ca_en50221_read_data - This function talks to an EN50221 CAM control
  *	interface. It reads a buffer of data from the CAM. The data can either
@@ -628,6 +640,7 @@ static int dvb_ca_en50221_set_configoption(struct dvb_ca_private *ca, int slot)
 static int dvb_ca_en50221_read_data(struct dvb_ca_private *ca, int slot,
 				    u8 *ebuf, int ecount)
 {
+	struct dvb_ca_slot *sl = &ca->slot_info[slot];
 	int bytes_read;
 	int status;
 	u8 buf[HOST_LINK_BUF_SIZE];
@@ -636,16 +649,16 @@ static int dvb_ca_en50221_read_data(struct dvb_ca_private *ca, int slot,
 	dprintk("%s\n", __func__);
 
 	/* check if we have space for a link buf in the rx_buffer */
-	if (ebuf == NULL) {
+	if (!ebuf) {
 		int buf_free;
 
-		if (ca->slot_info[slot].rx_buffer.data == NULL) {
+		if (!sl->rx_buffer.data) {
 			status = -EIO;
 			goto exit;
 		}
-		buf_free = dvb_ringbuffer_free(&ca->slot_info[slot].rx_buffer);
+		buf_free = dvb_ringbuffer_free(&sl->rx_buffer);
 
-		if (buf_free < (ca->slot_info[slot].link_buf_size +
+		if (buf_free < (sl->link_buf_size +
 				DVB_RINGBUFFER_PKTHDRSIZE)) {
 			status = -EAGAIN;
 			goto exit;
@@ -653,8 +666,8 @@ static int dvb_ca_en50221_read_data(struct dvb_ca_private *ca, int slot,
 	}
 
 	if (ca->pub->read_data &&
-	    (ca->slot_info[slot].slot_state != DVB_CA_SLOTSTATE_LINKINIT)) {
-		if (ebuf == NULL)
+	    (sl->slot_state != DVB_CA_SLOTSTATE_LINKINIT)) {
+		if (!ebuf)
 			status = ca->pub->read_data(ca->pub, slot, buf,
 						    sizeof(buf));
 		else
@@ -665,7 +678,6 @@ static int dvb_ca_en50221_read_data(struct dvb_ca_private *ca, int slot,
 		if (status == 0)
 			goto exit;
 	} else {
-
 		/* check if there is data available */
 		status = ca->pub->read_cam_control(ca->pub, slot,
 						   CTRLIF_STATUS);
@@ -690,21 +702,19 @@ static int dvb_ca_en50221_read_data(struct dvb_ca_private *ca, int slot,
 		bytes_read |= status;
 
 		/* check it will fit */
-		if (ebuf == NULL) {
-			if (bytes_read > ca->slot_info[slot].link_buf_size) {
+		if (!ebuf) {
+			if (bytes_read > sl->link_buf_size) {
 				pr_err("dvb_ca adapter %d: CAM tried to send a buffer larger than the link buffer size (%i > %i)!\n",
 				       ca->dvbdev->adapter->num, bytes_read,
-				       ca->slot_info[slot].link_buf_size);
-				ca->slot_info[slot].slot_state =
-						     DVB_CA_SLOTSTATE_LINKINIT;
+				       sl->link_buf_size);
+				sl->slot_state = DVB_CA_SLOTSTATE_LINKINIT;
 				status = -EIO;
 				goto exit;
 			}
 			if (bytes_read < 2) {
 				pr_err("dvb_ca adapter %d: CAM sent a buffer that was less than 2 bytes!\n",
 				       ca->dvbdev->adapter->num);
-				ca->slot_info[slot].slot_state =
-						     DVB_CA_SLOTSTATE_LINKINIT;
+				sl->slot_state = DVB_CA_SLOTSTATE_LINKINIT;
 				status = -EIO;
 				goto exit;
 			}
@@ -735,20 +745,22 @@ static int dvb_ca_en50221_read_data(struct dvb_ca_private *ca, int slot,
 		if (status < 0)
 			goto exit;
 		if (status & STATUSREG_RE) {
-			ca->slot_info[slot].slot_state =
-						     DVB_CA_SLOTSTATE_LINKINIT;
+			sl->slot_state = DVB_CA_SLOTSTATE_LINKINIT;
 			status = -EIO;
 			goto exit;
 		}
 	}
 
-	/* OK, add it to the receive buffer, or copy into external buffer if supplied */
-	if (ebuf == NULL) {
-		if (ca->slot_info[slot].rx_buffer.data == NULL) {
+	/*
+	 * OK, add it to the receive buffer, or copy into external buffer if
+	 * supplied
+	 */
+	if (!ebuf) {
+		if (!sl->rx_buffer.data) {
 			status = -EIO;
 			goto exit;
 		}
-		dvb_ringbuffer_pkt_write(&ca->slot_info[slot].rx_buffer, buf, bytes_read);
+		dvb_ringbuffer_pkt_write(&sl->rx_buffer, buf, bytes_read);
 	} else {
 		memcpy(ebuf, buf, bytes_read);
 	}
@@ -757,16 +769,15 @@ static int dvb_ca_en50221_read_data(struct dvb_ca_private *ca, int slot,
 		buf[0], (buf[1] & 0x80) == 0, bytes_read);
 
 	/* wake up readers when a last_fragment is received */
-	if ((buf[1] & 0x80) == 0x00) {
+	if ((buf[1] & 0x80) == 0x00)
 		wake_up_interruptible(&ca->wait_queue);
-	}
+
 	status = bytes_read;
 
 exit:
 	return status;
 }
 
-
 /**
  * dvb_ca_en50221_write_data - This function talks to an EN50221 CAM control
  *				interface. It writes a buffer of data to a CAM.
@@ -782,25 +793,28 @@ exit:
 static int dvb_ca_en50221_write_data(struct dvb_ca_private *ca, int slot,
 				     u8 *buf, int bytes_write)
 {
+	struct dvb_ca_slot *sl = &ca->slot_info[slot];
 	int status;
 	int i;
 
 	dprintk("%s\n", __func__);
 
-
 	/* sanity check */
-	if (bytes_write > ca->slot_info[slot].link_buf_size)
+	if (bytes_write > sl->link_buf_size)
 		return -EINVAL;
 
 	if (ca->pub->write_data &&
-	    (ca->slot_info[slot].slot_state != DVB_CA_SLOTSTATE_LINKINIT))
+	    (sl->slot_state != DVB_CA_SLOTSTATE_LINKINIT))
 		return ca->pub->write_data(ca->pub, slot, buf, bytes_write);
 
-	/* it is possible we are dealing with a single buffer implementation,
-	   thus if there is data available for read or if there is even a read
-	   already in progress, we do nothing but awake the kernel thread to
-	   process the data if necessary. */
-	if ((status = ca->pub->read_cam_control(ca->pub, slot, CTRLIF_STATUS)) < 0)
+	/*
+	 * it is possible we are dealing with a single buffer implementation,
+	 * thus if there is data available for read or if there is even a read
+	 * already in progress, we do nothing but awake the kernel thread to
+	 * process the data if necessary.
+	 */
+	status = ca->pub->read_cam_control(ca->pub, slot, CTRLIF_STATUS);
+	if (status < 0)
 		goto exitnowrite;
 	if (status & (STATUSREG_DA | STATUSREG_RE)) {
 		if (status & STATUSREG_DA)
@@ -811,12 +825,14 @@ static int dvb_ca_en50221_write_data(struct dvb_ca_private *ca, int slot,
 	}
 
 	/* OK, set HC bit */
-	if ((status = ca->pub->write_cam_control(ca->pub, slot, CTRLIF_COMMAND,
-						 IRQEN | CMDREG_HC)) != 0)
+	status = ca->pub->write_cam_control(ca->pub, slot, CTRLIF_COMMAND,
+					    IRQEN | CMDREG_HC);
+	if (status)
 		goto exit;
 
 	/* check if interface is still free */
-	if ((status = ca->pub->read_cam_control(ca->pub, slot, CTRLIF_STATUS)) < 0)
+	status = ca->pub->read_cam_control(ca->pub, slot, CTRLIF_STATUS);
+	if (status < 0)
 		goto exit;
 	if (!(status & STATUSREG_FR)) {
 		/* it wasn't free => try again later */
@@ -848,23 +864,29 @@ static int dvb_ca_en50221_write_data(struct dvb_ca_private *ca, int slot,
 	}
 
 	/* send the amount of data */
-	if ((status = ca->pub->write_cam_control(ca->pub, slot, CTRLIF_SIZE_HIGH, bytes_write >> 8)) != 0)
+	status = ca->pub->write_cam_control(ca->pub, slot, CTRLIF_SIZE_HIGH,
+					    bytes_write >> 8);
+	if (status)
 		goto exit;
-	if ((status = ca->pub->write_cam_control(ca->pub, slot, CTRLIF_SIZE_LOW,
-						 bytes_write & 0xff)) != 0)
+	status = ca->pub->write_cam_control(ca->pub, slot, CTRLIF_SIZE_LOW,
+					    bytes_write & 0xff);
+	if (status)
 		goto exit;
 
 	/* send the buffer */
 	for (i = 0; i < bytes_write; i++) {
-		if ((status = ca->pub->write_cam_control(ca->pub, slot, CTRLIF_DATA, buf[i])) != 0)
+		status = ca->pub->write_cam_control(ca->pub, slot, CTRLIF_DATA,
+						    buf[i]);
+		if (status)
 			goto exit;
 	}
 
 	/* check for write error (WE should now be 0) */
-	if ((status = ca->pub->read_cam_control(ca->pub, slot, CTRLIF_STATUS)) < 0)
+	status = ca->pub->read_cam_control(ca->pub, slot, CTRLIF_STATUS);
+	if (status < 0)
 		goto exit;
 	if (status & STATUSREG_WE) {
-		ca->slot_info[slot].slot_state = DVB_CA_SLOTSTATE_LINKINIT;
+		sl->slot_state = DVB_CA_SLOTSTATE_LINKINIT;
 		status = -EIO;
 		goto exit;
 	}
@@ -880,12 +902,9 @@ exitnowrite:
 	return status;
 }
 
-
-
-/* ******************************************************************************** */
+/* ************************************************************************** */
 /* EN50221 higher level functions */
 
-
 /**
  * dvb_ca_en50221_slot_shutdown - A CAM has been removed => shut it down.
  *
@@ -899,8 +918,10 @@ static int dvb_ca_en50221_slot_shutdown(struct dvb_ca_private *ca, int slot)
 	ca->pub->slot_shutdown(ca->pub, slot);
 	ca->slot_info[slot].slot_state = DVB_CA_SLOTSTATE_NONE;
 
-	/* need to wake up all processes to check if they're now
-	   trying to write to a defunct CAM */
+	/*
+	 * need to wake up all processes to check if they're now trying to
+	 * write to a defunct CAM
+	 */
 	wake_up_interruptible(&ca->wait_queue);
 
 	dprintk("Slot %i shutdown\n", slot);
@@ -909,7 +930,6 @@ static int dvb_ca_en50221_slot_shutdown(struct dvb_ca_private *ca, int slot)
 	return 0;
 }
 
-
 /**
  * dvb_ca_en50221_camchange_irq - A CAMCHANGE IRQ has occurred.
  *
@@ -917,9 +937,11 @@ static int dvb_ca_en50221_slot_shutdown(struct dvb_ca_private *ca, int slot)
  * @slot: Slot concerned.
  * @change_type: One of the DVB_CA_CAMCHANGE_* values.
  */
-void dvb_ca_en50221_camchange_irq(struct dvb_ca_en50221 *pubca, int slot, int change_type)
+void dvb_ca_en50221_camchange_irq(struct dvb_ca_en50221 *pubca, int slot,
+				  int change_type)
 {
 	struct dvb_ca_private *ca = pubca->private;
+	struct dvb_ca_slot *sl = &ca->slot_info[slot];
 
 	dprintk("CAMCHANGE IRQ slot:%i change_type:%i\n", slot, change_type);
 
@@ -932,13 +954,12 @@ void dvb_ca_en50221_camchange_irq(struct dvb_ca_en50221 *pubca, int slot, int ch
 		return;
 	}
 
-	ca->slot_info[slot].camchange_type = change_type;
-	atomic_inc(&ca->slot_info[slot].camchange_count);
+	sl->camchange_type = change_type;
+	atomic_inc(&sl->camchange_count);
 	dvb_ca_en50221_thread_wakeup(ca);
 }
 EXPORT_SYMBOL(dvb_ca_en50221_camchange_irq);
 
-
 /**
  * dvb_ca_en50221_camready_irq - A CAMREADY IRQ has occurred.
  *
@@ -948,17 +969,17 @@ EXPORT_SYMBOL(dvb_ca_en50221_camchange_irq);
 void dvb_ca_en50221_camready_irq(struct dvb_ca_en50221 *pubca, int slot)
 {
 	struct dvb_ca_private *ca = pubca->private;
+	struct dvb_ca_slot *sl = &ca->slot_info[slot];
 
 	dprintk("CAMREADY IRQ slot:%i\n", slot);
 
-	if (ca->slot_info[slot].slot_state == DVB_CA_SLOTSTATE_WAITREADY) {
-		ca->slot_info[slot].slot_state = DVB_CA_SLOTSTATE_VALIDATE;
+	if (sl->slot_state == DVB_CA_SLOTSTATE_WAITREADY) {
+		sl->slot_state = DVB_CA_SLOTSTATE_VALIDATE;
 		dvb_ca_en50221_thread_wakeup(ca);
 	}
 }
 EXPORT_SYMBOL(dvb_ca_en50221_camready_irq);
 
-
 /**
  * dvb_ca_en50221_frda_irq - An FR or DA IRQ has occurred.
  *
@@ -968,16 +989,17 @@ EXPORT_SYMBOL(dvb_ca_en50221_camready_irq);
 void dvb_ca_en50221_frda_irq(struct dvb_ca_en50221 *pubca, int slot)
 {
 	struct dvb_ca_private *ca = pubca->private;
+	struct dvb_ca_slot *sl = &ca->slot_info[slot];
 	int flags;
 
 	dprintk("FR/DA IRQ slot:%i\n", slot);
 
-	switch (ca->slot_info[slot].slot_state) {
+	switch (sl->slot_state) {
 	case DVB_CA_SLOTSTATE_LINKINIT:
 		flags = ca->pub->read_cam_control(pubca, slot, CTRLIF_STATUS);
 		if (flags & STATUSREG_DA) {
 			dprintk("CAM supports DA IRQ\n");
-			ca->slot_info[slot].da_irq_supported = 1;
+			sl->da_irq_supported = 1;
 		}
 		break;
 
@@ -989,8 +1011,7 @@ void dvb_ca_en50221_frda_irq(struct dvb_ca_en50221 *pubca, int slot)
 }
 EXPORT_SYMBOL(dvb_ca_en50221_frda_irq);
 
-
-/* ******************************************************************************** */
+/* ************************************************************************** */
 /* EN50221 thread functions */
 
 /**
@@ -1000,7 +1021,6 @@ EXPORT_SYMBOL(dvb_ca_en50221_frda_irq);
  */
 static void dvb_ca_en50221_thread_wakeup(struct dvb_ca_private *ca)
 {
-
 	dprintk("%s\n", __func__);
 
 	ca->wakeup = 1;
@@ -1019,11 +1039,14 @@ static void dvb_ca_en50221_thread_update_delay(struct dvb_ca_private *ca)
 	int curdelay = 100000000;
 	int slot;
 
-	/* Beware of too high polling frequency, because one polling
+	/*
+	 * Beware of too high polling frequency, because one polling
 	 * call might take several hundred milliseconds until timeout!
 	 */
 	for (slot = 0; slot < ca->slot_count; slot++) {
-		switch (ca->slot_info[slot].slot_state) {
+		struct dvb_ca_slot *sl = &ca->slot_info[slot];
+
+		switch (sl->slot_state) {
 		default:
 		case DVB_CA_SLOTSTATE_NONE:
 			delay = HZ * 60;  /* 60s */
@@ -1049,7 +1072,7 @@ static void dvb_ca_en50221_thread_update_delay(struct dvb_ca_private *ca)
 			if (!(ca->flags & DVB_CA_EN50221_FLAG_IRQ_CAMCHANGE))
 				delay = HZ / 10;  /* 100ms */
 			if (ca->open) {
-				if ((!ca->slot_info[slot].da_irq_supported) ||
+				if ((!sl->da_irq_supported) ||
 				    (!(ca->flags & DVB_CA_EN50221_FLAG_IRQ_DA)))
 					delay = HZ / 10;  /* 100ms */
 			}
@@ -1063,215 +1086,250 @@ static void dvb_ca_en50221_thread_update_delay(struct dvb_ca_private *ca)
 	ca->delay = curdelay;
 }
 
+/**
+ * Poll if the CAM is gone.
+ *
+ * @ca: CA instance.
+ * @slot: Slot to process.
+ * @return: 0 .. no change
+ *          1 .. CAM state changed
+ */
 
+static int dvb_ca_en50221_poll_cam_gone(struct dvb_ca_private *ca, int slot)
+{
+	int changed = 0;
+	int status;
+
+	/*
+	 * we need this extra check for annoying interfaces like the
+	 * budget-av
+	 */
+	if ((!(ca->flags & DVB_CA_EN50221_FLAG_IRQ_CAMCHANGE)) &&
+	    (ca->pub->poll_slot_status)) {
+		status = ca->pub->poll_slot_status(ca->pub, slot, 0);
+		if (!(status &
+			DVB_CA_EN50221_POLL_CAM_PRESENT)) {
+			ca->slot_info[slot].slot_state = DVB_CA_SLOTSTATE_NONE;
+			dvb_ca_en50221_thread_update_delay(ca);
+			changed = 1;
+		}
+	}
+	return changed;
+}
 
 /**
- * Kernel thread which monitors CA slots for CAM changes, and performs data transfers.
+ * Thread state machine for one CA slot to perform the data transfer.
+ *
+ * @ca: CA instance.
+ * @slot: Slot to process.
  */
-static int dvb_ca_en50221_thread(void *data)
+static void dvb_ca_en50221_thread_state_machine(struct dvb_ca_private *ca,
+						int slot)
 {
-	struct dvb_ca_private *ca = data;
-	int slot;
+	struct dvb_ca_slot *sl = &ca->slot_info[slot];
 	int flags;
-	int status;
 	int pktcount;
 	void *rxbuf;
 
-	dprintk("%s\n", __func__);
+	mutex_lock(&sl->slot_lock);
 
-	/* choose the correct initial delay */
-	dvb_ca_en50221_thread_update_delay(ca);
+	/* check the cam status + deal with CAMCHANGEs */
+	while (dvb_ca_en50221_check_camstatus(ca, slot)) {
+		/* clear down an old CI slot if necessary */
+		if (sl->slot_state != DVB_CA_SLOTSTATE_NONE)
+			dvb_ca_en50221_slot_shutdown(ca, slot);
 
-	/* main loop */
-	while (!kthread_should_stop()) {
-		/* sleep for a bit */
-		if (!ca->wakeup) {
-			set_current_state(TASK_INTERRUPTIBLE);
-			schedule_timeout(ca->delay);
-			if (kthread_should_stop())
-				return 0;
+		/* if a CAM is NOW present, initialise it */
+		if (sl->camchange_type == DVB_CA_EN50221_CAMCHANGE_INSERTED)
+			sl->slot_state = DVB_CA_SLOTSTATE_UNINITIALISED;
+
+		/* we've handled one CAMCHANGE */
+		dvb_ca_en50221_thread_update_delay(ca);
+		atomic_dec(&sl->camchange_count);
+	}
+
+	/* CAM state machine */
+	switch (sl->slot_state) {
+	case DVB_CA_SLOTSTATE_NONE:
+	case DVB_CA_SLOTSTATE_INVALID:
+		/* no action needed */
+		break;
+
+	case DVB_CA_SLOTSTATE_UNINITIALISED:
+		sl->slot_state = DVB_CA_SLOTSTATE_WAITREADY;
+		ca->pub->slot_reset(ca->pub, slot);
+		sl->timeout = jiffies + (INIT_TIMEOUT_SECS * HZ);
+		break;
+
+	case DVB_CA_SLOTSTATE_WAITREADY:
+		if (time_after(jiffies, sl->timeout)) {
+			pr_err("dvb_ca adaptor %d: PC card did not respond :(\n",
+			       ca->dvbdev->adapter->num);
+			sl->slot_state = DVB_CA_SLOTSTATE_INVALID;
+			dvb_ca_en50221_thread_update_delay(ca);
+			break;
 		}
-		ca->wakeup = 0;
+		/*
+		 * no other action needed; will automatically change state when
+		 * ready
+		 */
+		break;
 
-		/* go through all the slots processing them */
-		for (slot = 0; slot < ca->slot_count; slot++) {
+	case DVB_CA_SLOTSTATE_VALIDATE:
+		if (dvb_ca_en50221_parse_attributes(ca, slot) != 0) {
+			if (dvb_ca_en50221_poll_cam_gone(ca, slot))
+				break;
 
-			mutex_lock(&ca->slot_info[slot].slot_lock);
+			pr_err("dvb_ca adapter %d: Invalid PC card inserted :(\n",
+			       ca->dvbdev->adapter->num);
+			sl->slot_state = DVB_CA_SLOTSTATE_INVALID;
+			dvb_ca_en50221_thread_update_delay(ca);
+			break;
+		}
+		if (dvb_ca_en50221_set_configoption(ca, slot) != 0) {
+			pr_err("dvb_ca adapter %d: Unable to initialise CAM :(\n",
+			       ca->dvbdev->adapter->num);
+			sl->slot_state = DVB_CA_SLOTSTATE_INVALID;
+			dvb_ca_en50221_thread_update_delay(ca);
+			break;
+		}
+		if (ca->pub->write_cam_control(ca->pub, slot,
+					       CTRLIF_COMMAND,
+					       CMDREG_RS) != 0) {
+			pr_err("dvb_ca adapter %d: Unable to reset CAM IF\n",
+			       ca->dvbdev->adapter->num);
+			sl->slot_state = DVB_CA_SLOTSTATE_INVALID;
+			dvb_ca_en50221_thread_update_delay(ca);
+			break;
+		}
+		dprintk("DVB CAM validated successfully\n");
 
-			// check the cam status + deal with CAMCHANGEs
-			while (dvb_ca_en50221_check_camstatus(ca, slot)) {
-				/* clear down an old CI slot if necessary */
-				if (ca->slot_info[slot].slot_state != DVB_CA_SLOTSTATE_NONE)
-					dvb_ca_en50221_slot_shutdown(ca, slot);
+		sl->timeout = jiffies + (INIT_TIMEOUT_SECS * HZ);
+		sl->slot_state = DVB_CA_SLOTSTATE_WAITFR;
+		ca->wakeup = 1;
+		break;
 
-				/* if a CAM is NOW present, initialise it */
-				if (ca->slot_info[slot].camchange_type == DVB_CA_EN50221_CAMCHANGE_INSERTED) {
-					ca->slot_info[slot].slot_state = DVB_CA_SLOTSTATE_UNINITIALISED;
-				}
+	case DVB_CA_SLOTSTATE_WAITFR:
+		if (time_after(jiffies, sl->timeout)) {
+			pr_err("dvb_ca adapter %d: DVB CAM did not respond :(\n",
+			       ca->dvbdev->adapter->num);
+			sl->slot_state = DVB_CA_SLOTSTATE_INVALID;
+			dvb_ca_en50221_thread_update_delay(ca);
+			break;
+		}
 
-				/* we've handled one CAMCHANGE */
-				dvb_ca_en50221_thread_update_delay(ca);
-				atomic_dec(&ca->slot_info[slot].camchange_count);
-			}
+		flags = ca->pub->read_cam_control(ca->pub, slot, CTRLIF_STATUS);
+		if (flags & STATUSREG_FR) {
+			sl->slot_state = DVB_CA_SLOTSTATE_LINKINIT;
+			ca->wakeup = 1;
+		}
+		break;
 
-			// CAM state machine
-			switch (ca->slot_info[slot].slot_state) {
-			case DVB_CA_SLOTSTATE_NONE:
-			case DVB_CA_SLOTSTATE_INVALID:
-				// no action needed
+	case DVB_CA_SLOTSTATE_LINKINIT:
+		if (dvb_ca_en50221_link_init(ca, slot) != 0) {
+			if (dvb_ca_en50221_poll_cam_gone(ca, slot))
 				break;
 
-			case DVB_CA_SLOTSTATE_UNINITIALISED:
-				ca->slot_info[slot].slot_state = DVB_CA_SLOTSTATE_WAITREADY;
-				ca->pub->slot_reset(ca->pub, slot);
-				ca->slot_info[slot].timeout = jiffies + (INIT_TIMEOUT_SECS * HZ);
-				break;
+			pr_err("dvb_ca adapter %d: DVB CAM link initialisation failed :(\n",
+			       ca->dvbdev->adapter->num);
+			sl->slot_state = DVB_CA_SLOTSTATE_UNINITIALISED;
+			dvb_ca_en50221_thread_update_delay(ca);
+			break;
+		}
 
-			case DVB_CA_SLOTSTATE_WAITREADY:
-				if (time_after(jiffies, ca->slot_info[slot].timeout)) {
-					pr_err("dvb_ca adaptor %d: PC card did not respond :(\n",
-					       ca->dvbdev->adapter->num);
-					ca->slot_info[slot].slot_state = DVB_CA_SLOTSTATE_INVALID;
-					dvb_ca_en50221_thread_update_delay(ca);
-					break;
-				}
-				// no other action needed; will automatically change state when ready
+		if (!sl->rx_buffer.data) {
+			rxbuf = vmalloc(RX_BUFFER_SIZE);
+			if (!rxbuf) {
+				pr_err("dvb_ca adapter %d: Unable to allocate CAM rx buffer :(\n",
+				       ca->dvbdev->adapter->num);
+				sl->slot_state = DVB_CA_SLOTSTATE_INVALID;
+				dvb_ca_en50221_thread_update_delay(ca);
 				break;
+			}
+			dvb_ringbuffer_init(&sl->rx_buffer, rxbuf,
+					    RX_BUFFER_SIZE);
+		}
 
-			case DVB_CA_SLOTSTATE_VALIDATE:
-				if (dvb_ca_en50221_parse_attributes(ca, slot) != 0) {
-					/* we need this extra check for annoying interfaces like the budget-av */
-					if ((!(ca->flags & DVB_CA_EN50221_FLAG_IRQ_CAMCHANGE)) &&
-					    (ca->pub->poll_slot_status)) {
-						status = ca->pub->poll_slot_status(ca->pub, slot, 0);
-						if (!(status & DVB_CA_EN50221_POLL_CAM_PRESENT)) {
-							ca->slot_info[slot].slot_state = DVB_CA_SLOTSTATE_NONE;
-							dvb_ca_en50221_thread_update_delay(ca);
-							break;
-						}
-					}
-
-					pr_err("dvb_ca adapter %d: Invalid PC card inserted :(\n",
-					       ca->dvbdev->adapter->num);
-					ca->slot_info[slot].slot_state = DVB_CA_SLOTSTATE_INVALID;
-					dvb_ca_en50221_thread_update_delay(ca);
-					break;
-				}
-				if (dvb_ca_en50221_set_configoption(ca, slot) != 0) {
-					pr_err("dvb_ca adapter %d: Unable to initialise CAM :(\n",
-					       ca->dvbdev->adapter->num);
-					ca->slot_info[slot].slot_state = DVB_CA_SLOTSTATE_INVALID;
-					dvb_ca_en50221_thread_update_delay(ca);
-					break;
-				}
-				if (ca->pub->write_cam_control(ca->pub, slot,
-							       CTRLIF_COMMAND, CMDREG_RS) != 0) {
-					pr_err("dvb_ca adapter %d: Unable to reset CAM IF\n",
-					       ca->dvbdev->adapter->num);
-					ca->slot_info[slot].slot_state = DVB_CA_SLOTSTATE_INVALID;
-					dvb_ca_en50221_thread_update_delay(ca);
-					break;
-				}
-				dprintk("DVB CAM validated successfully\n");
-
-				ca->slot_info[slot].timeout = jiffies + (INIT_TIMEOUT_SECS * HZ);
-				ca->slot_info[slot].slot_state = DVB_CA_SLOTSTATE_WAITFR;
-				ca->wakeup = 1;
-				break;
+		ca->pub->slot_ts_enable(ca->pub, slot);
+		sl->slot_state = DVB_CA_SLOTSTATE_RUNNING;
+		dvb_ca_en50221_thread_update_delay(ca);
+		pr_err("dvb_ca adapter %d: DVB CAM detected and initialised successfully\n",
+		       ca->dvbdev->adapter->num);
+		break;
 
-			case DVB_CA_SLOTSTATE_WAITFR:
-				if (time_after(jiffies, ca->slot_info[slot].timeout)) {
-					pr_err("dvb_ca adapter %d: DVB CAM did not respond :(\n",
-					       ca->dvbdev->adapter->num);
-					ca->slot_info[slot].slot_state = DVB_CA_SLOTSTATE_INVALID;
-					dvb_ca_en50221_thread_update_delay(ca);
-					break;
-				}
-
-				flags = ca->pub->read_cam_control(ca->pub, slot, CTRLIF_STATUS);
-				if (flags & STATUSREG_FR) {
-					ca->slot_info[slot].slot_state = DVB_CA_SLOTSTATE_LINKINIT;
-					ca->wakeup = 1;
-				}
-				break;
+	case DVB_CA_SLOTSTATE_RUNNING:
+		if (!ca->open)
+			break;
 
-			case DVB_CA_SLOTSTATE_LINKINIT:
-				if (dvb_ca_en50221_link_init(ca, slot) != 0) {
-					/* we need this extra check for annoying interfaces like the budget-av */
-					if ((!(ca->flags & DVB_CA_EN50221_FLAG_IRQ_CAMCHANGE)) &&
-					    (ca->pub->poll_slot_status)) {
-						status = ca->pub->poll_slot_status(ca->pub, slot, 0);
-						if (!(status & DVB_CA_EN50221_POLL_CAM_PRESENT)) {
-							ca->slot_info[slot].slot_state = DVB_CA_SLOTSTATE_NONE;
-							dvb_ca_en50221_thread_update_delay(ca);
-							break;
-						}
-					}
-
-					pr_err("dvb_ca adapter %d: DVB CAM link initialisation failed :(\n",
-					       ca->dvbdev->adapter->num);
-					ca->slot_info[slot].slot_state =
-						DVB_CA_SLOTSTATE_UNINITIALISED;
-					dvb_ca_en50221_thread_update_delay(ca);
-					break;
-				}
-
-				if (ca->slot_info[slot].rx_buffer.data == NULL) {
-					rxbuf = vmalloc(RX_BUFFER_SIZE);
-					if (rxbuf == NULL) {
-						pr_err("dvb_ca adapter %d: Unable to allocate CAM rx buffer :(\n",
-						       ca->dvbdev->adapter->num);
-						ca->slot_info[slot].slot_state = DVB_CA_SLOTSTATE_INVALID;
-						dvb_ca_en50221_thread_update_delay(ca);
-						break;
-					}
-					dvb_ringbuffer_init(&ca->slot_info[slot].rx_buffer, rxbuf, RX_BUFFER_SIZE);
-				}
-
-				ca->pub->slot_ts_enable(ca->pub, slot);
-				ca->slot_info[slot].slot_state = DVB_CA_SLOTSTATE_RUNNING;
-				dvb_ca_en50221_thread_update_delay(ca);
-				pr_err("dvb_ca adapter %d: DVB CAM detected and initialised successfully\n",
-				       ca->dvbdev->adapter->num);
+		/* poll slots for data */
+		pktcount = 0;
+		while (dvb_ca_en50221_read_data(ca, slot, NULL, 0) > 0) {
+			if (!ca->open)
 				break;
 
-			case DVB_CA_SLOTSTATE_RUNNING:
-				if (!ca->open)
-					break;
-
-				// poll slots for data
-				pktcount = 0;
-				while ((status = dvb_ca_en50221_read_data(ca, slot, NULL, 0)) > 0) {
-					if (!ca->open)
-						break;
-
-					/* if a CAMCHANGE occurred at some point, do not do any more processing of this slot */
-					if (dvb_ca_en50221_check_camstatus(ca, slot)) {
-						// we dont want to sleep on the next iteration so we can handle the cam change
-						ca->wakeup = 1;
-						break;
-					}
-
-					/* check if we've hit our limit this time */
-					if (++pktcount >= MAX_RX_PACKETS_PER_ITERATION) {
-						// dont sleep; there is likely to be more data to read
-						ca->wakeup = 1;
-						break;
-					}
-				}
+			/*
+			 * if a CAMCHANGE occurred at some point, do not do any
+			 * more processing of this slot
+			 */
+			if (dvb_ca_en50221_check_camstatus(ca, slot)) {
+				/*
+				 * we don't want to sleep on the next iteration
+				 * so we can handle the cam change
+				 */
+				ca->wakeup = 1;
 				break;
 			}
 
-			mutex_unlock(&ca->slot_info[slot].slot_lock);
+			/* check if we've hit our limit this time */
+			if (++pktcount >= MAX_RX_PACKETS_PER_ITERATION) {
+				/*
+				 * don't sleep; there is likely to be more data
+				 * to read
+				 */
+				ca->wakeup = 1;
+				break;
+			}
 		}
+		break;
 	}
 
-	return 0;
+	mutex_unlock(&sl->slot_lock);
 }
 
+/**
+ * Kernel thread which monitors CA slots for CAM changes, and performs data
+ * transfers.
+ */
+static int dvb_ca_en50221_thread(void *data)
+{
+	struct dvb_ca_private *ca = data;
+	int slot;
+
+	dprintk("%s\n", __func__);
+
+	/* choose the correct initial delay */
+	dvb_ca_en50221_thread_update_delay(ca);
+
+	/* main loop */
+	while (!kthread_should_stop()) {
+		/* sleep for a bit */
+		if (!ca->wakeup) {
+			set_current_state(TASK_INTERRUPTIBLE);
+			schedule_timeout(ca->delay);
+			if (kthread_should_stop())
+				return 0;
+		}
+		ca->wakeup = 0;
+
+		/* go through all the slots processing them */
+		for (slot = 0; slot < ca->slot_count; slot++)
+			dvb_ca_en50221_thread_state_machine(ca, slot);
+	}
 
+	return 0;
+}
 
-/* ******************************************************************************** */
+/* ************************************************************************** */
 /* EN50221 IO interface functions */
 
 /**
@@ -1301,15 +1359,17 @@ static int dvb_ca_en50221_io_do_ioctl(struct file *file,
 	switch (cmd) {
 	case CA_RESET:
 		for (slot = 0; slot < ca->slot_count; slot++) {
-			mutex_lock(&ca->slot_info[slot].slot_lock);
-			if (ca->slot_info[slot].slot_state != DVB_CA_SLOTSTATE_NONE) {
+			struct dvb_ca_slot *sl = &ca->slot_info[slot];
+
+			mutex_lock(&sl->slot_lock);
+			if (sl->slot_state != DVB_CA_SLOTSTATE_NONE) {
 				dvb_ca_en50221_slot_shutdown(ca, slot);
 				if (ca->flags & DVB_CA_EN50221_FLAG_IRQ_CAMCHANGE)
 					dvb_ca_en50221_camchange_irq(ca->pub,
 								     slot,
 								     DVB_CA_EN50221_CAMCHANGE_INSERTED);
 			}
-			mutex_unlock(&ca->slot_info[slot].slot_lock);
+			mutex_unlock(&sl->slot_lock);
 		}
 		ca->next_read_slot = 0;
 		dvb_ca_en50221_thread_wakeup(ca);
@@ -1327,21 +1387,23 @@ static int dvb_ca_en50221_io_do_ioctl(struct file *file,
 
 	case CA_GET_SLOT_INFO: {
 		struct ca_slot_info *info = parg;
+		struct dvb_ca_slot *sl;
 
-		if ((info->num > ca->slot_count) || (info->num < 0)) {
+		slot = info->num;
+		if ((slot > ca->slot_count) || (slot < 0)) {
 			err = -EINVAL;
 			goto out_unlock;
 		}
 
 		info->type = CA_CI_LINK;
 		info->flags = 0;
-		if ((ca->slot_info[info->num].slot_state != DVB_CA_SLOTSTATE_NONE)
-			&& (ca->slot_info[info->num].slot_state != DVB_CA_SLOTSTATE_INVALID)) {
+		sl = &ca->slot_info[slot];
+		if ((sl->slot_state != DVB_CA_SLOTSTATE_NONE) &&
+		    (sl->slot_state != DVB_CA_SLOTSTATE_INVALID)) {
 			info->flags = CA_CI_MODULE_PRESENT;
 		}
-		if (ca->slot_info[info->num].slot_state == DVB_CA_SLOTSTATE_RUNNING) {
+		if (sl->slot_state == DVB_CA_SLOTSTATE_RUNNING)
 			info->flags |= CA_CI_MODULE_READY;
-		}
 		break;
 	}
 
@@ -1355,7 +1417,6 @@ out_unlock:
 	return err;
 }
 
-
 /**
  * Wrapper for ioctl implementation.
  *
@@ -1372,7 +1433,6 @@ static long dvb_ca_en50221_io_ioctl(struct file *file,
 	return dvb_usercopy(file, cmd, arg, dvb_ca_en50221_io_do_ioctl);
 }
 
-
 /**
  * Implementation of write() syscall.
  *
@@ -1389,6 +1449,7 @@ static ssize_t dvb_ca_en50221_io_write(struct file *file,
 {
 	struct dvb_device *dvbdev = file->private_data;
 	struct dvb_ca_private *ca = dvbdev->priv;
+	struct dvb_ca_slot *sl;
 	u8 slot, connection_id;
 	int status;
 	u8 fragbuf[HOST_LINK_BUF_SIZE];
@@ -1399,7 +1460,10 @@ static ssize_t dvb_ca_en50221_io_write(struct file *file,
 
 	dprintk("%s\n", __func__);
 
-	/* Incoming packet has a 2 byte header. hdr[0] = slot_id, hdr[1] = connection_id */
+	/*
+	 * Incoming packet has a 2 byte header.
+	 * hdr[0] = slot_id, hdr[1] = connection_id
+	 */
 	if (count < 2)
 		return -EINVAL;
 
@@ -1410,14 +1474,15 @@ static ssize_t dvb_ca_en50221_io_write(struct file *file,
 		return -EFAULT;
 	buf += 2;
 	count -= 2;
+	sl = &ca->slot_info[slot];
 
 	/* check if the slot is actually running */
-	if (ca->slot_info[slot].slot_state != DVB_CA_SLOTSTATE_RUNNING)
+	if (sl->slot_state != DVB_CA_SLOTSTATE_RUNNING)
 		return -EINVAL;
 
 	/* fragment the packets & store in the buffer */
 	while (fragpos < count) {
-		fraglen = ca->slot_info[slot].link_buf_size - 2;
+		fraglen = sl->link_buf_size - 2;
 		if (fraglen < 0)
 			break;
 		if (fraglen > HOST_LINK_BUF_SIZE - 2)
@@ -1436,15 +1501,19 @@ static ssize_t dvb_ca_en50221_io_write(struct file *file,
 		timeout = jiffies + HZ / 2;
 		written = 0;
 		while (!time_after(jiffies, timeout)) {
-			/* check the CAM hasn't been removed/reset in the meantime */
-			if (ca->slot_info[slot].slot_state != DVB_CA_SLOTSTATE_RUNNING) {
+			/*
+			 * check the CAM hasn't been removed/reset in the
+			 * meantime
+			 */
+			if (sl->slot_state != DVB_CA_SLOTSTATE_RUNNING) {
 				status = -EIO;
 				goto exit;
 			}
 
-			mutex_lock(&ca->slot_info[slot].slot_lock);
-			status = dvb_ca_en50221_write_data(ca, slot, fragbuf, fraglen + 2);
-			mutex_unlock(&ca->slot_info[slot].slot_lock);
+			mutex_lock(&sl->slot_lock);
+			status = dvb_ca_en50221_write_data(ca, slot, fragbuf,
+							   fraglen + 2);
+			mutex_unlock(&sl->slot_lock);
 			if (status == (fraglen + 2)) {
 				written = 1;
 				break;
@@ -1452,7 +1521,7 @@ static ssize_t dvb_ca_en50221_io_write(struct file *file,
 			if (status != -EAGAIN)
 				goto exit;
 
-			msleep(1);
+			usleep_range(1000, 1100);
 		}
 		if (!written) {
 			status = -EIO;
@@ -1467,7 +1536,6 @@ exit:
 	return status;
 }
 
-
 /**
  * Condition for waking up in dvb_ca_en50221_io_read_condition
  */
@@ -1484,25 +1552,28 @@ static int dvb_ca_en50221_io_read_condition(struct dvb_ca_private *ca,
 
 	slot = ca->next_read_slot;
 	while ((slot_count < ca->slot_count) && (!found)) {
-		if (ca->slot_info[slot].slot_state != DVB_CA_SLOTSTATE_RUNNING)
+		struct dvb_ca_slot *sl = &ca->slot_info[slot];
+
+		if (sl->slot_state != DVB_CA_SLOTSTATE_RUNNING)
 			goto nextslot;
 
-		if (ca->slot_info[slot].rx_buffer.data == NULL) {
+		if (!sl->rx_buffer.data)
 			return 0;
-		}
 
-		idx = dvb_ringbuffer_pkt_next(&ca->slot_info[slot].rx_buffer, -1, &fraglen);
+		idx = dvb_ringbuffer_pkt_next(&sl->rx_buffer, -1, &fraglen);
 		while (idx != -1) {
-			dvb_ringbuffer_pkt_read(&ca->slot_info[slot].rx_buffer, idx, 0, hdr, 2);
+			dvb_ringbuffer_pkt_read(&sl->rx_buffer, idx, 0, hdr, 2);
 			if (connection_id == -1)
 				connection_id = hdr[0];
-			if ((hdr[0] == connection_id) && ((hdr[1] & 0x80) == 0)) {
+			if ((hdr[0] == connection_id) &&
+			    ((hdr[1] & 0x80) == 0)) {
 				*_slot = slot;
 				found = 1;
 				break;
 			}
 
-			idx = dvb_ringbuffer_pkt_next(&ca->slot_info[slot].rx_buffer, idx, &fraglen);
+			idx = dvb_ringbuffer_pkt_next(&sl->rx_buffer, idx,
+						      &fraglen);
 		}
 
 nextslot:
@@ -1514,7 +1585,6 @@ nextslot:
 	return found;
 }
 
-
 /**
  * Implementation of read() syscall.
  *
@@ -1530,6 +1600,7 @@ static ssize_t dvb_ca_en50221_io_read(struct file *file, char __user *buf,
 {
 	struct dvb_device *dvbdev = file->private_data;
 	struct dvb_ca_private *ca = dvbdev->priv;
+	struct dvb_ca_slot *sl;
 	int status;
 	int result = 0;
 	u8 hdr[2];
@@ -1543,13 +1614,16 @@ static ssize_t dvb_ca_en50221_io_read(struct file *file, char __user *buf,
 
 	dprintk("%s\n", __func__);
 
-	/* Outgoing packet has a 2 byte header. hdr[0] = slot_id, hdr[1] = connection_id */
+	/*
+	 * Outgoing packet has a 2 byte header.
+	 * hdr[0] = slot_id, hdr[1] = connection_id
+	 */
 	if (count < 2)
 		return -EINVAL;
 
 	/* wait for some data */
-	if ((status = dvb_ca_en50221_io_read_condition(ca, &result, &slot)) == 0) {
-
+	status = dvb_ca_en50221_io_read_condition(ca, &result, &slot);
+	if (status == 0) {
 		/* if we're in nonblocking mode, exit immediately */
 		if (file->f_flags & O_NONBLOCK)
 			return -EWOULDBLOCK;
@@ -1565,7 +1639,8 @@ static ssize_t dvb_ca_en50221_io_read(struct file *file, char __user *buf,
 		return status;
 	}
 
-	idx = dvb_ringbuffer_pkt_next(&ca->slot_info[slot].rx_buffer, -1, &fraglen);
+	sl = &ca->slot_info[slot];
+	idx = dvb_ringbuffer_pkt_next(&sl->rx_buffer, -1, &fraglen);
 	pktlen = 2;
 	do {
 		if (idx == -1) {
@@ -1575,21 +1650,24 @@ static ssize_t dvb_ca_en50221_io_read(struct file *file, char __user *buf,
 			goto exit;
 		}
 
-		dvb_ringbuffer_pkt_read(&ca->slot_info[slot].rx_buffer, idx, 0, hdr, 2);
+		dvb_ringbuffer_pkt_read(&sl->rx_buffer, idx, 0, hdr, 2);
 		if (connection_id == -1)
 			connection_id = hdr[0];
 		if (hdr[0] == connection_id) {
 			if (pktlen < count) {
-				if ((pktlen + fraglen - 2) > count) {
+				if ((pktlen + fraglen - 2) > count)
 					fraglen = count - pktlen;
-				} else {
+				else
 					fraglen -= 2;
-				}
 
-				if ((status = dvb_ringbuffer_pkt_read_user(&ca->slot_info[slot].rx_buffer, idx, 2,
-								      buf + pktlen, fraglen)) < 0) {
+				status =
+				   dvb_ringbuffer_pkt_read_user(&sl->rx_buffer,
+								idx, 2,
+								buf + pktlen,
+								fraglen);
+				if (status < 0)
 					goto exit;
-				}
+
 				pktlen += fraglen;
 			}
 
@@ -1598,9 +1676,9 @@ static ssize_t dvb_ca_en50221_io_read(struct file *file, char __user *buf,
 			dispose = 1;
 		}
 
-		idx2 = dvb_ringbuffer_pkt_next(&ca->slot_info[slot].rx_buffer, idx, &fraglen);
+		idx2 = dvb_ringbuffer_pkt_next(&sl->rx_buffer, idx, &fraglen);
 		if (dispose)
-			dvb_ringbuffer_pkt_dispose(&ca->slot_info[slot].rx_buffer, idx);
+			dvb_ringbuffer_pkt_dispose(&sl->rx_buffer, idx);
 		idx = idx2;
 		dispose = 0;
 	} while (!last_fragment);
@@ -1618,7 +1696,6 @@ exit:
 	return status;
 }
 
-
 /**
  * Implementation of file open syscall.
  *
@@ -1646,12 +1723,16 @@ static int dvb_ca_en50221_io_open(struct inode *inode, struct file *file)
 	}
 
 	for (i = 0; i < ca->slot_count; i++) {
-
-		if (ca->slot_info[i].slot_state == DVB_CA_SLOTSTATE_RUNNING) {
-			if (ca->slot_info[i].rx_buffer.data != NULL) {
-				/* it is safe to call this here without locks because
-				 * ca->open == 0. Data is not read in this case */
-				dvb_ringbuffer_flush(&ca->slot_info[i].rx_buffer);
+		struct dvb_ca_slot *sl = &ca->slot_info[i];
+
+		if (sl->slot_state == DVB_CA_SLOTSTATE_RUNNING) {
+			if (!sl->rx_buffer.data) {
+				/*
+				 * it is safe to call this here without locks
+				 * because ca->open == 0. Data is not read in
+				 * this case
+				 */
+				dvb_ringbuffer_flush(&sl->rx_buffer);
 			}
 		}
 	}
@@ -1665,7 +1746,6 @@ static int dvb_ca_en50221_io_open(struct inode *inode, struct file *file)
 	return 0;
 }
 
-
 /**
  * Implementation of file close syscall.
  *
@@ -1695,7 +1775,6 @@ static int dvb_ca_en50221_io_release(struct inode *inode, struct file *file)
 	return err;
 }
 
-
 /**
  * Implementation of poll() syscall.
  *
@@ -1714,9 +1793,8 @@ static unsigned int dvb_ca_en50221_io_poll(struct file *file, poll_table *wait)
 
 	dprintk("%s\n", __func__);
 
-	if (dvb_ca_en50221_io_read_condition(ca, &result, &slot) == 1) {
+	if (dvb_ca_en50221_io_read_condition(ca, &result, &slot) == 1)
 		mask |= POLLIN;
-	}
 
 	/* if there is something, return now */
 	if (mask)
@@ -1725,14 +1803,11 @@ static unsigned int dvb_ca_en50221_io_poll(struct file *file, poll_table *wait)
 	/* wait for something to happen */
 	poll_wait(file, &ca->wait_queue, wait);
 
-	if (dvb_ca_en50221_io_read_condition(ca, &result, &slot) == 1) {
+	if (dvb_ca_en50221_io_read_condition(ca, &result, &slot) == 1)
 		mask |= POLLIN;
-	}
 
 	return mask;
 }
-EXPORT_SYMBOL(dvb_ca_en50221_init);
-
 
 static const struct file_operations dvb_ca_fops = {
 	.owner = THIS_MODULE,
@@ -1756,10 +1831,9 @@ static const struct dvb_device dvbdev_ca = {
 	.fops = &dvb_ca_fops,
 };
 
-/* ******************************************************************************** */
+/* ************************************************************************** */
 /* Initialisation/shutdown functions */
 
-
 /**
  * Initialise a new DVB CA EN50221 interface device.
  *
@@ -1783,7 +1857,8 @@ int dvb_ca_en50221_init(struct dvb_adapter *dvb_adapter,
 		return -EINVAL;
 
 	/* initialise the system data */
-	if ((ca = kzalloc(sizeof(struct dvb_ca_private), GFP_KERNEL)) == NULL) {
+	ca = kzalloc(sizeof(*ca), GFP_KERNEL);
+	if (!ca) {
 		ret = -ENOMEM;
 		goto exit;
 	}
@@ -1791,7 +1866,9 @@ int dvb_ca_en50221_init(struct dvb_adapter *dvb_adapter,
 	ca->pub = pubca;
 	ca->flags = flags;
 	ca->slot_count = slot_count;
-	if ((ca->slot_info = kcalloc(slot_count, sizeof(struct dvb_ca_slot), GFP_KERNEL)) == NULL) {
+	ca->slot_info = kcalloc(slot_count, sizeof(struct dvb_ca_slot),
+				GFP_KERNEL);
+	if (!ca->slot_info) {
 		ret = -ENOMEM;
 		goto free_ca;
 	}
@@ -1802,17 +1879,20 @@ int dvb_ca_en50221_init(struct dvb_adapter *dvb_adapter,
 	pubca->private = ca;
 
 	/* register the DVB device */
-	ret = dvb_register_device(dvb_adapter, &ca->dvbdev, &dvbdev_ca, ca, DVB_DEVICE_CA, 0);
+	ret = dvb_register_device(dvb_adapter, &ca->dvbdev, &dvbdev_ca, ca,
+				  DVB_DEVICE_CA, 0);
 	if (ret)
 		goto free_slot_info;
 
 	/* now initialise each slot */
 	for (i = 0; i < slot_count; i++) {
-		memset(&ca->slot_info[i], 0, sizeof(struct dvb_ca_slot));
-		ca->slot_info[i].slot_state = DVB_CA_SLOTSTATE_NONE;
-		atomic_set(&ca->slot_info[i].camchange_count, 0);
-		ca->slot_info[i].camchange_type = DVB_CA_EN50221_CAMCHANGE_REMOVED;
-		mutex_init(&ca->slot_info[i].slot_lock);
+		struct dvb_ca_slot *sl = &ca->slot_info[i];
+
+		memset(sl, 0, sizeof(struct dvb_ca_slot));
+		sl->slot_state = DVB_CA_SLOTSTATE_NONE;
+		atomic_set(&sl->camchange_count, 0);
+		sl->camchange_type = DVB_CA_EN50221_CAMCHANGE_REMOVED;
+		mutex_init(&sl->slot_lock);
 	}
 
 	mutex_init(&ca->ioctl_mutex);
@@ -1844,9 +1924,7 @@ exit:
 	pubca->private = NULL;
 	return ret;
 }
-EXPORT_SYMBOL(dvb_ca_en50221_release);
-
-
+EXPORT_SYMBOL(dvb_ca_en50221_init);
 
 /**
  * Release a DVB CA EN50221 interface device.
@@ -1864,10 +1942,11 @@ void dvb_ca_en50221_release(struct dvb_ca_en50221 *pubca)
 	/* shutdown the thread if there was one */
 	kthread_stop(ca->thread);
 
-	for (i = 0; i < ca->slot_count; i++) {
+	for (i = 0; i < ca->slot_count; i++)
 		dvb_ca_en50221_slot_shutdown(ca, i);
-	}
+
 	dvb_remove_device(ca->dvbdev);
 	dvb_ca_private_put(ca);
 	pubca->private = NULL;
 }
+EXPORT_SYMBOL(dvb_ca_en50221_release);
diff --git a/drivers/media/dvb-core/dvb_ca_en50221.h b/drivers/media/dvb-core/dvb_ca_en50221.h
index 82617bac0875..367687d2b41a 100644
--- a/drivers/media/dvb-core/dvb_ca_en50221.h
+++ b/drivers/media/dvb-core/dvb_ca_en50221.h
@@ -69,9 +69,9 @@ struct dvb_ca_en50221 {
 				 int slot, u8 address, u8 value);
 
 	int (*read_data)(struct dvb_ca_en50221 *ca,
-				int slot, u8 *ebuf, int ecount);
+			 int slot, u8 *ebuf, int ecount);
 	int (*write_data)(struct dvb_ca_en50221 *ca,
-				int slot, u8 *ebuf, int ecount);
+			  int slot, u8 *ebuf, int ecount);
 
 	int (*slot_reset)(struct dvb_ca_en50221 *ca, int slot);
 	int (*slot_shutdown)(struct dvb_ca_en50221 *ca, int slot);
@@ -128,8 +128,8 @@ void dvb_ca_en50221_frda_irq(struct dvb_ca_en50221 *ca, int slot);
  *
  * @return 0 on success, nonzero on failure
  */
-extern int dvb_ca_en50221_init(struct dvb_adapter *dvb_adapter,
-			       struct dvb_ca_en50221 *ca, int flags,
+int dvb_ca_en50221_init(struct dvb_adapter *dvb_adapter,
+			struct dvb_ca_en50221 *ca, int flags,
 			       int slot_count);
 
 /**
@@ -137,6 +137,6 @@ extern int dvb_ca_en50221_init(struct dvb_adapter *dvb_adapter,
  *
  * @ca: The associated dvb_ca instance.
  */
-extern void dvb_ca_en50221_release(struct dvb_ca_en50221 *ca);
+void dvb_ca_en50221_release(struct dvb_ca_en50221 *ca);
 
 #endif
diff --git a/drivers/media/dvb-core/dvb_frontend.c b/drivers/media/dvb-core/dvb_frontend.c
index e3fff8f64d37..2fcba1616168 100644
--- a/drivers/media/dvb-core/dvb_frontend.c
+++ b/drivers/media/dvb-core/dvb_frontend.c
@@ -460,7 +460,7 @@ static int dvb_frontend_swzigzag_autotune(struct dvb_frontend *fe, int check_wra
 
 static void dvb_frontend_swzigzag(struct dvb_frontend *fe)
 {
-	enum fe_status s = 0;
+	enum fe_status s = FE_NONE;
 	int retval = 0;
 	struct dvb_frontend_private *fepriv = fe->frontend_priv;
 	struct dtv_frontend_properties *c = &fe->dtv_property_cache, tmp;
@@ -631,7 +631,7 @@ static int dvb_frontend_thread(void *data)
 	struct dvb_frontend *fe = data;
 	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
 	struct dvb_frontend_private *fepriv = fe->frontend_priv;
-	enum fe_status s;
+	enum fe_status s = FE_NONE;
 	enum dvbfe_algo algo;
 	bool re_tune = false;
 	bool semheld = false;
@@ -1000,6 +1000,17 @@ static int dvb_frontend_clear_cache(struct dvb_frontend *fe)
 	.buffer = b \
 }
 
+struct dtv_cmds_h {
+	char	*name;		/* A display name for debugging purposes */
+
+	__u32	cmd;		/* A unique ID */
+
+	/* Flags */
+	__u32	set:1;		/* Either a set or get property */
+	__u32	buffer:1;	/* Does this property use the buffer? */
+	__u32	reserved:30;	/* Align */
+};
+
 static struct dtv_cmds_h dtv_cmds[DTV_MAX_COMMAND + 1] = {
 	_DTV_CMD(DTV_TUNE, 1, 0),
 	_DTV_CMD(DTV_CLEAR, 1, 0),
diff --git a/drivers/media/dvb-frontends/Kconfig b/drivers/media/dvb-frontends/Kconfig
index 3a260b82b3e8..2631d0e0a024 100644
--- a/drivers/media/dvb-frontends/Kconfig
+++ b/drivers/media/dvb-frontends/Kconfig
@@ -28,6 +28,15 @@ config DVB_STV090x
 	  DVB-S/S2/DSS Multistandard Professional/Broadcast demodulators.
 	  Say Y when you want to support these frontends.
 
+config DVB_STV0910
+	tristate "STV0910 based"
+	depends on DVB_CORE && I2C
+	default m if !MEDIA_SUBDRV_AUTOSELECT
+	help
+	  ST STV0910 DVB-S/S2 demodulator driver.
+
+	  Say Y when you want to support these frontends.
+
 config DVB_STV6110x
 	tristate "STV6110/(A) based tuners"
 	depends on DVB_CORE && I2C
@@ -35,6 +44,24 @@ config DVB_STV6110x
 	help
 	  A Silicon tuner that supports DVB-S and DVB-S2 modes
 
+config DVB_STV6111
+	tristate "STV6111 based tuners"
+	depends on DVB_CORE && I2C
+	default m if !MEDIA_SUBDRV_AUTOSELECT
+	help
+	  A Silicon tuner that supports DVB-S and DVB-S2 modes
+
+	  Say Y when you want to support these frontends.
+
+config DVB_MXL5XX
+	tristate "MaxLinear MxL5xx based tuner-demodulators"
+	depends on DVB_CORE && I2C
+	default m if !MEDIA_SUBDRV_AUTOSELECT
+	help
+	  MaxLinear MxL5xx family of DVB-S/S2 tuners/demodulators.
+
+	  Say Y when you want to support these frontends.
+
 config DVB_M88DS3103
 	tristate "Montage Technology M88DS3103"
 	depends on DVB_CORE && I2C && I2C_MUX
diff --git a/drivers/media/dvb-frontends/Makefile b/drivers/media/dvb-frontends/Makefile
index 3fccaf34ef52..f45f6a4a4371 100644
--- a/drivers/media/dvb-frontends/Makefile
+++ b/drivers/media/dvb-frontends/Makefile
@@ -110,6 +110,9 @@ obj-$(CONFIG_DVB_CXD2820R) += cxd2820r.o
 obj-$(CONFIG_DVB_CXD2841ER) += cxd2841er.o
 obj-$(CONFIG_DVB_DRXK) += drxk.o
 obj-$(CONFIG_DVB_TDA18271C2DD) += tda18271c2dd.o
+obj-$(CONFIG_DVB_STV0910) += stv0910.o
+obj-$(CONFIG_DVB_STV6111) += stv6111.o
+obj-$(CONFIG_DVB_MXL5XX) += mxl5xx.o
 obj-$(CONFIG_DVB_SI2165) += si2165.o
 obj-$(CONFIG_DVB_A8293) += a8293.o
 obj-$(CONFIG_DVB_SP2) += sp2.o
diff --git a/drivers/media/dvb-frontends/cx24123.c b/drivers/media/dvb-frontends/cx24123.c
index 4ae3d922a8e8..1d59d1d3bd82 100644
--- a/drivers/media/dvb-frontends/cx24123.c
+++ b/drivers/media/dvb-frontends/cx24123.c
@@ -1032,7 +1032,7 @@ static u32 cx24123_tuner_i2c_func(struct i2c_adapter *adapter)
 	return I2C_FUNC_I2C;
 }
 
-static struct i2c_algorithm cx24123_tuner_i2c_algo = {
+static const struct i2c_algorithm cx24123_tuner_i2c_algo = {
 	.master_xfer   = cx24123_tuner_i2c_tuner_xfer,
 	.functionality = cx24123_tuner_i2c_func,
 };
diff --git a/drivers/media/dvb-frontends/cxd2841er.c b/drivers/media/dvb-frontends/cxd2841er.c
index 12bff778c97f..48ee9bc00c06 100644
--- a/drivers/media/dvb-frontends/cxd2841er.c
+++ b/drivers/media/dvb-frontends/cxd2841er.c
@@ -487,6 +487,8 @@ static int cxd2841er_sleep_tc_to_shutdown(struct cxd2841er_priv *priv);
 
 static int cxd2841er_shutdown_to_sleep_tc(struct cxd2841er_priv *priv);
 
+static int cxd2841er_sleep_tc(struct dvb_frontend *fe);
+
 static int cxd2841er_retune_active(struct cxd2841er_priv *priv,
 				   struct dtv_frontend_properties *p)
 {
@@ -2178,42 +2180,42 @@ static int cxd2841er_sleep_tc_to_active_t2_band(struct cxd2841er_priv *priv,
 	u32 iffreq, ifhz;
 	u8 data[MAX_WRITE_REGSIZE];
 
-	const uint8_t nominalRate8bw[3][5] = {
+	static const uint8_t nominalRate8bw[3][5] = {
 		/* TRCG Nominal Rate [37:0] */
 		{0x11, 0xF0, 0x00, 0x00, 0x00}, /* 20.5MHz XTal */
 		{0x15, 0x00, 0x00, 0x00, 0x00}, /* 24MHz XTal */
 		{0x11, 0xF0, 0x00, 0x00, 0x00}  /* 41MHz XTal */
 	};
 
-	const uint8_t nominalRate7bw[3][5] = {
+	static const uint8_t nominalRate7bw[3][5] = {
 		/* TRCG Nominal Rate [37:0] */
 		{0x14, 0x80, 0x00, 0x00, 0x00}, /* 20.5MHz XTal */
 		{0x18, 0x00, 0x00, 0x00, 0x00}, /* 24MHz XTal */
 		{0x14, 0x80, 0x00, 0x00, 0x00}  /* 41MHz XTal */
 	};
 
-	const uint8_t nominalRate6bw[3][5] = {
+	static const uint8_t nominalRate6bw[3][5] = {
 		/* TRCG Nominal Rate [37:0] */
 		{0x17, 0xEA, 0xAA, 0xAA, 0xAA}, /* 20.5MHz XTal */
 		{0x1C, 0x00, 0x00, 0x00, 0x00}, /* 24MHz XTal */
 		{0x17, 0xEA, 0xAA, 0xAA, 0xAA}  /* 41MHz XTal */
 	};
 
-	const uint8_t nominalRate5bw[3][5] = {
+	static const uint8_t nominalRate5bw[3][5] = {
 		/* TRCG Nominal Rate [37:0] */
 		{0x1C, 0xB3, 0x33, 0x33, 0x33}, /* 20.5MHz XTal */
 		{0x21, 0x99, 0x99, 0x99, 0x99}, /* 24MHz XTal */
 		{0x1C, 0xB3, 0x33, 0x33, 0x33}  /* 41MHz XTal */
 	};
 
-	const uint8_t nominalRate17bw[3][5] = {
+	static const uint8_t nominalRate17bw[3][5] = {
 		/* TRCG Nominal Rate [37:0] */
 		{0x58, 0xE2, 0xAF, 0xE0, 0xBC}, /* 20.5MHz XTal */
 		{0x68, 0x0F, 0xA2, 0x32, 0xD0}, /* 24MHz XTal */
 		{0x58, 0xE2, 0xAF, 0xE0, 0xBC}  /* 41MHz XTal */
 	};
 
-	const uint8_t itbCoef8bw[3][14] = {
+	static const uint8_t itbCoef8bw[3][14] = {
 		{0x26, 0xAF, 0x06, 0xCD, 0x13, 0xBB, 0x28, 0xBA,
 			0x23, 0xA9, 0x1F, 0xA8, 0x2C, 0xC8}, /* 20.5MHz XTal */
 		{0x2F, 0xBA, 0x28, 0x9B, 0x28, 0x9D, 0x28, 0xA1,
@@ -2222,7 +2224,7 @@ static int cxd2841er_sleep_tc_to_active_t2_band(struct cxd2841er_priv *priv,
 			0x23, 0xA9, 0x1F, 0xA8, 0x2C, 0xC8}  /* 41MHz XTal   */
 	};
 
-	const uint8_t itbCoef7bw[3][14] = {
+	static const uint8_t itbCoef7bw[3][14] = {
 		{0x2C, 0xBD, 0x02, 0xCF, 0x04, 0xF8, 0x23, 0xA6,
 			0x29, 0xB0, 0x26, 0xA9, 0x21, 0xA5}, /* 20.5MHz XTal */
 		{0x30, 0xB1, 0x29, 0x9A, 0x28, 0x9C, 0x28, 0xA0,
@@ -2231,7 +2233,7 @@ static int cxd2841er_sleep_tc_to_active_t2_band(struct cxd2841er_priv *priv,
 			0x29, 0xB0, 0x26, 0xA9, 0x21, 0xA5}  /* 41MHz XTal   */
 	};
 
-	const uint8_t itbCoef6bw[3][14] = {
+	static const uint8_t itbCoef6bw[3][14] = {
 		{0x27, 0xA7, 0x28, 0xB3, 0x02, 0xF0, 0x01, 0xE8,
 			0x00, 0xCF, 0x00, 0xE6, 0x23, 0xA4}, /* 20.5MHz XTal */
 		{0x31, 0xA8, 0x29, 0x9B, 0x27, 0x9C, 0x28, 0x9E,
@@ -2240,7 +2242,7 @@ static int cxd2841er_sleep_tc_to_active_t2_band(struct cxd2841er_priv *priv,
 			0x00, 0xCF, 0x00, 0xE6, 0x23, 0xA4}  /* 41MHz XTal   */
 	};
 
-	const uint8_t itbCoef5bw[3][14] = {
+	static const uint8_t itbCoef5bw[3][14] = {
 		{0x27, 0xA7, 0x28, 0xB3, 0x02, 0xF0, 0x01, 0xE8,
 			0x00, 0xCF, 0x00, 0xE6, 0x23, 0xA4}, /* 20.5MHz XTal */
 		{0x31, 0xA8, 0x29, 0x9B, 0x27, 0x9C, 0x28, 0x9E,
@@ -2249,7 +2251,7 @@ static int cxd2841er_sleep_tc_to_active_t2_band(struct cxd2841er_priv *priv,
 			0x00, 0xCF, 0x00, 0xE6, 0x23, 0xA4}  /* 41MHz XTal   */
 	};
 
-	const uint8_t itbCoef17bw[3][14] = {
+	static const uint8_t itbCoef17bw[3][14] = {
 		{0x25, 0xA0, 0x36, 0x8D, 0x2E, 0x94, 0x28, 0x9B,
 			0x32, 0x90, 0x2C, 0x9D, 0x29, 0x99}, /* 20.5MHz XTal */
 		{0x33, 0x8E, 0x2B, 0x97, 0x2D, 0x95, 0x37, 0x8B,
@@ -2423,32 +2425,32 @@ static int cxd2841er_sleep_tc_to_active_t_band(
 {
 	u8 data[MAX_WRITE_REGSIZE];
 	u32 iffreq, ifhz;
-	u8 nominalRate8bw[3][5] = {
+	static const u8 nominalRate8bw[3][5] = {
 		/* TRCG Nominal Rate [37:0] */
 		{0x11, 0xF0, 0x00, 0x00, 0x00}, /* 20.5MHz XTal */
 		{0x15, 0x00, 0x00, 0x00, 0x00}, /* 24MHz XTal */
 		{0x11, 0xF0, 0x00, 0x00, 0x00}  /* 41MHz XTal */
 	};
-	u8 nominalRate7bw[3][5] = {
+	static const u8 nominalRate7bw[3][5] = {
 		/* TRCG Nominal Rate [37:0] */
 		{0x14, 0x80, 0x00, 0x00, 0x00}, /* 20.5MHz XTal */
 		{0x18, 0x00, 0x00, 0x00, 0x00}, /* 24MHz XTal */
 		{0x14, 0x80, 0x00, 0x00, 0x00}  /* 41MHz XTal */
 	};
-	u8 nominalRate6bw[3][5] = {
+	static const u8 nominalRate6bw[3][5] = {
 		/* TRCG Nominal Rate [37:0] */
 		{0x17, 0xEA, 0xAA, 0xAA, 0xAA}, /* 20.5MHz XTal */
 		{0x1C, 0x00, 0x00, 0x00, 0x00}, /* 24MHz XTal */
 		{0x17, 0xEA, 0xAA, 0xAA, 0xAA}  /* 41MHz XTal */
 	};
-	u8 nominalRate5bw[3][5] = {
+	static const u8 nominalRate5bw[3][5] = {
 		/* TRCG Nominal Rate [37:0] */
 		{0x1C, 0xB3, 0x33, 0x33, 0x33}, /* 20.5MHz XTal */
 		{0x21, 0x99, 0x99, 0x99, 0x99}, /* 24MHz XTal */
 		{0x1C, 0xB3, 0x33, 0x33, 0x33}  /* 41MHz XTal */
 	};
 
-	u8 itbCoef8bw[3][14] = {
+	static const u8 itbCoef8bw[3][14] = {
 		{0x26, 0xAF, 0x06, 0xCD, 0x13, 0xBB, 0x28, 0xBA, 0x23, 0xA9,
 			0x1F, 0xA8, 0x2C, 0xC8}, /* 20.5MHz XTal */
 		{0x2F, 0xBA, 0x28, 0x9B, 0x28, 0x9D, 0x28, 0xA1, 0x29, 0xA5,
@@ -2456,7 +2458,7 @@ static int cxd2841er_sleep_tc_to_active_t_band(
 		{0x26, 0xAF, 0x06, 0xCD, 0x13, 0xBB, 0x28, 0xBA, 0x23, 0xA9,
 			0x1F, 0xA8, 0x2C, 0xC8}  /* 41MHz XTal   */
 	};
-	u8 itbCoef7bw[3][14] = {
+	static const u8 itbCoef7bw[3][14] = {
 		{0x2C, 0xBD, 0x02, 0xCF, 0x04, 0xF8, 0x23, 0xA6, 0x29, 0xB0,
 			0x26, 0xA9, 0x21, 0xA5}, /* 20.5MHz XTal */
 		{0x30, 0xB1, 0x29, 0x9A, 0x28, 0x9C, 0x28, 0xA0, 0x29, 0xA2,
@@ -2464,7 +2466,7 @@ static int cxd2841er_sleep_tc_to_active_t_band(
 		{0x2C, 0xBD, 0x02, 0xCF, 0x04, 0xF8, 0x23, 0xA6, 0x29, 0xB0,
 			0x26, 0xA9, 0x21, 0xA5}  /* 41MHz XTal   */
 	};
-	u8 itbCoef6bw[3][14] = {
+	static const u8 itbCoef6bw[3][14] = {
 		{0x27, 0xA7, 0x28, 0xB3, 0x02, 0xF0, 0x01, 0xE8, 0x00, 0xCF,
 			0x00, 0xE6, 0x23, 0xA4}, /* 20.5MHz XTal */
 		{0x31, 0xA8, 0x29, 0x9B, 0x27, 0x9C, 0x28, 0x9E, 0x29, 0xA4,
@@ -2472,7 +2474,7 @@ static int cxd2841er_sleep_tc_to_active_t_band(
 		{0x27, 0xA7, 0x28, 0xB3, 0x02, 0xF0, 0x01, 0xE8, 0x00, 0xCF,
 			0x00, 0xE6, 0x23, 0xA4}  /* 41MHz XTal   */
 	};
-	u8 itbCoef5bw[3][14] = {
+	static const u8 itbCoef5bw[3][14] = {
 		{0x27, 0xA7, 0x28, 0xB3, 0x02, 0xF0, 0x01, 0xE8, 0x00, 0xCF,
 			0x00, 0xE6, 0x23, 0xA4}, /* 20.5MHz XTal */
 		{0x31, 0xA8, 0x29, 0x9B, 0x27, 0x9C, 0x28, 0x9E, 0x29, 0xA4,
@@ -2652,39 +2654,39 @@ static int cxd2841er_sleep_tc_to_active_i_band(
 	u8 data[3];
 
 	/* TRCG Nominal Rate */
-	u8 nominalRate8bw[3][5] = {
+	static const u8 nominalRate8bw[3][5] = {
 		{0x00, 0x00, 0x00, 0x00, 0x00}, /* 20.5MHz XTal */
 		{0x11, 0xB8, 0x00, 0x00, 0x00}, /* 24MHz XTal */
 		{0x00, 0x00, 0x00, 0x00, 0x00}  /* 41MHz XTal */
 	};
 
-	u8 nominalRate7bw[3][5] = {
+	static const u8 nominalRate7bw[3][5] = {
 		{0x00, 0x00, 0x00, 0x00, 0x00}, /* 20.5MHz XTal */
 		{0x14, 0x40, 0x00, 0x00, 0x00}, /* 24MHz XTal */
 		{0x00, 0x00, 0x00, 0x00, 0x00}  /* 41MHz XTal */
 	};
 
-	u8 nominalRate6bw[3][5] = {
+	static const u8 nominalRate6bw[3][5] = {
 		{0x14, 0x2E, 0x00, 0x00, 0x00}, /* 20.5MHz XTal */
 		{0x17, 0xA0, 0x00, 0x00, 0x00}, /* 24MHz XTal */
 		{0x14, 0x2E, 0x00, 0x00, 0x00}  /* 41MHz XTal */
 	};
 
-	u8 itbCoef8bw[3][14] = {
+	static const u8 itbCoef8bw[3][14] = {
 		{0x00}, /* 20.5MHz XTal */
 		{0x2F, 0xBA, 0x28, 0x9B, 0x28, 0x9D, 0x28, 0xA1, 0x29,
 			0xA5, 0x2A, 0xAC, 0x29, 0xB5}, /* 24MHz Xtal */
 		{0x0}, /* 41MHz XTal   */
 	};
 
-	u8 itbCoef7bw[3][14] = {
+	static const u8 itbCoef7bw[3][14] = {
 		{0x00}, /* 20.5MHz XTal */
 		{0x30, 0xB1, 0x29, 0x9A, 0x28, 0x9C, 0x28, 0xA0, 0x29,
 			0xA2, 0x2B, 0xA6, 0x2B, 0xAD}, /* 24MHz Xtal */
 		{0x00}, /* 41MHz XTal   */
 	};
 
-	u8 itbCoef6bw[3][14] = {
+	static const u8 itbCoef6bw[3][14] = {
 		{0x27, 0xA7, 0x28, 0xB3, 0x02, 0xF0, 0x01, 0xE8, 0x00,
 			0xCF, 0x00, 0xE6, 0x23, 0xA4}, /* 20.5MHz XTal */
 		{0x31, 0xA8, 0x29, 0x9B, 0x27, 0x9C, 0x28, 0x9E, 0x29,
@@ -3378,6 +3380,14 @@ static int cxd2841er_set_frontend_tc(struct dvb_frontend *fe)
 	if (priv->flags & CXD2841ER_EARLY_TUNE)
 		cxd2841er_tuner_set(fe);
 
+	/* deconfigure/put demod to sleep on delsys switch if active */
+	if (priv->state == STATE_ACTIVE_TC &&
+	    priv->system != p->delivery_system) {
+		dev_dbg(&priv->i2c->dev, "%s(): old_delsys=%d, new_delsys=%d -> sleep\n",
+			 __func__, priv->system, p->delivery_system);
+		cxd2841er_sleep_tc(fe);
+	}
+
 	if (p->delivery_system == SYS_DVBT) {
 		priv->system = SYS_DVBT;
 		switch (priv->state) {
@@ -3594,6 +3604,7 @@ static int cxd2841er_sleep_tc(struct dvb_frontend *fe)
 	struct cxd2841er_priv *priv = fe->demodulator_priv;
 
 	dev_dbg(&priv->i2c->dev, "%s()\n", __func__);
+
 	if (priv->state == STATE_ACTIVE_TC) {
 		switch (priv->system) {
 		case SYS_DVBT:
@@ -3619,7 +3630,17 @@ static int cxd2841er_sleep_tc(struct dvb_frontend *fe)
 			__func__, priv->state);
 		return -EINVAL;
 	}
-	cxd2841er_sleep_tc_to_shutdown(priv);
+	return 0;
+}
+
+static int cxd2841er_shutdown_tc(struct dvb_frontend *fe)
+{
+	struct cxd2841er_priv *priv = fe->demodulator_priv;
+
+	dev_dbg(&priv->i2c->dev, "%s()\n", __func__);
+
+	if (!cxd2841er_sleep_tc(fe))
+		cxd2841er_sleep_tc_to_shutdown(priv);
 	return 0;
 }
 
@@ -3968,7 +3989,7 @@ static struct dvb_frontend_ops cxd2841er_t_c_ops = {
 		.symbol_rate_max = 11700000
 	},
 	.init = cxd2841er_init_tc,
-	.sleep = cxd2841er_sleep_tc,
+	.sleep = cxd2841er_shutdown_tc,
 	.release = cxd2841er_release,
 	.set_frontend = cxd2841er_set_frontend_tc,
 	.get_frontend = cxd2841er_get_frontend,
@@ -3978,6 +3999,6 @@ static struct dvb_frontend_ops cxd2841er_t_c_ops = {
 	.get_frontend_algo = cxd2841er_get_algo
 };
 
-MODULE_DESCRIPTION("Sony CXD2841ER/CXD2854ER DVB-C/C2/T/T2/S/S2 demodulator driver");
+MODULE_DESCRIPTION("Sony CXD2837/38/41/43/54ER DVB-C/C2/T/T2/S/S2 demodulator driver");
 MODULE_AUTHOR("Sergey Kozlov <serjk@netup.ru>, Abylay Ospan <aospan@netup.ru>");
 MODULE_LICENSE("GPL");
diff --git a/drivers/media/dvb-frontends/dib0090.c b/drivers/media/dvb-frontends/dib0090.c
index 33af14df27bd..d9d730dfe0b1 100644
--- a/drivers/media/dvb-frontends/dib0090.c
+++ b/drivers/media/dvb-frontends/dib0090.c
@@ -2052,7 +2052,7 @@ int dib0090_update_tuning_table_7090(struct dvb_frontend *fe,
 	struct dib0090_state *state = fe->tuner_priv;
 	const struct dib0090_tuning *tune =
 		dib0090_tuning_table_cband_7090e_sensitivity;
-	const struct dib0090_tuning dib0090_tuning_table_cband_7090e_aci[] = {
+	static const struct dib0090_tuning dib0090_tuning_table_cband_7090e_aci[] = {
 		{ 300000,  0 ,  3,  0x8165, 0x2c0, 0x2d12, 0xb84e, EN_CAB },
 		{ 650000,  0 ,  4,  0x815B, 0x280, 0x2d12, 0xb84e, EN_CAB },
 		{ 860000,  0 ,  5,  0x84EF, 0x280, 0x2d12, 0xb84e, EN_CAB },
@@ -2435,14 +2435,7 @@ static int dib0090_tune(struct dvb_frontend *fe)
 			Den = 1;
 
 			if (Rest > 0) {
-				if (state->config->analog_output)
-					lo6 |= (1 << 2) | 2;
-				else {
-					if (state->identity.in_soc)
-						lo6 |= (1 << 2) | 2;
-					else
-						lo6 |= (1 << 2) | 2;
-				}
+				lo6 |= (1 << 2) | 2;
 				Den = 255;
 			}
 			dib0090_write_reg(state, 0x15, (u16) FBDiv);
diff --git a/drivers/media/dvb-frontends/dib7000p.c b/drivers/media/dvb-frontends/dib7000p.c
index 1caa04d8f60f..0fbaabe43682 100644
--- a/drivers/media/dvb-frontends/dib7000p.c
+++ b/drivers/media/dvb-frontends/dib7000p.c
@@ -2388,7 +2388,7 @@ static u32 dib7000p_i2c_func(struct i2c_adapter *adapter)
 	return I2C_FUNC_I2C;
 }
 
-static struct i2c_algorithm dib7090_tuner_xfer_algo = {
+static const struct i2c_algorithm dib7090_tuner_xfer_algo = {
 	.master_xfer = dib7090_tuner_xfer,
 	.functionality = dib7000p_i2c_func,
 };
diff --git a/drivers/media/dvb-frontends/dib8000.c b/drivers/media/dvb-frontends/dib8000.c
index e501ec964df1..5d9381509b07 100644
--- a/drivers/media/dvb-frontends/dib8000.c
+++ b/drivers/media/dvb-frontends/dib8000.c
@@ -1880,7 +1880,7 @@ static u32 dib8096p_i2c_func(struct i2c_adapter *adapter)
 	return I2C_FUNC_I2C;
 }
 
-static struct i2c_algorithm dib8096p_tuner_xfer_algo = {
+static const struct i2c_algorithm dib8096p_tuner_xfer_algo = {
 	.master_xfer = dib8096p_tuner_xfer,
 	.functionality = dib8096p_i2c_func,
 };
@@ -4255,23 +4255,6 @@ static int dib8000_set_slave_frontend(struct dvb_frontend *fe, struct dvb_fronte
 	return -ENOMEM;
 }
 
-static int dib8000_remove_slave_frontend(struct dvb_frontend *fe)
-{
-	struct dib8000_state *state = fe->demodulator_priv;
-	u8 index_frontend = 1;
-
-	while ((index_frontend < MAX_NUMBER_OF_FRONTENDS) && (state->fe[index_frontend] != NULL))
-		index_frontend++;
-	if (index_frontend != 1) {
-		dprintk("remove slave fe %p (index %i)\n", state->fe[index_frontend-1], index_frontend-1);
-		state->fe[index_frontend] = NULL;
-		return 0;
-	}
-
-	dprintk("no frontend to be removed\n");
-	return -ENODEV;
-}
-
 static struct dvb_frontend *dib8000_get_slave_frontend(struct dvb_frontend *fe, int slave_index)
 {
 	struct dib8000_state *state = fe->demodulator_priv;
@@ -4506,7 +4489,6 @@ void *dib8000_attach(struct dib8000_ops *ops)
 	ops->get_slave_frontend = dib8000_get_slave_frontend;
 	ops->set_tune_state = dib8000_set_tune_state;
 	ops->pid_filter_ctrl = dib8000_pid_filter_ctrl;
-	ops->remove_slave_frontend = dib8000_remove_slave_frontend;
 	ops->get_adc_power = dib8000_get_adc_power;
 	ops->update_pll = dib8000_update_pll;
 	ops->tuner_sleep = dib8096p_tuner_sleep;
diff --git a/drivers/media/dvb-frontends/dib8000.h b/drivers/media/dvb-frontends/dib8000.h
index 2b8b4b1656a2..75cc8e47ec8f 100644
--- a/drivers/media/dvb-frontends/dib8000.h
+++ b/drivers/media/dvb-frontends/dib8000.h
@@ -53,7 +53,6 @@ struct dib8000_ops {
 	enum frontend_tune_state (*get_tune_state)(struct dvb_frontend *fe);
 	int (*set_tune_state)(struct dvb_frontend *fe, enum frontend_tune_state tune_state);
 	int (*set_slave_frontend)(struct dvb_frontend *fe, struct dvb_frontend *fe_slave);
-	int (*remove_slave_frontend)(struct dvb_frontend *fe);
 	struct dvb_frontend *(*get_slave_frontend)(struct dvb_frontend *fe, int slave_index);
 	int (*i2c_enumeration)(struct i2c_adapter *host, int no_of_demods,
 		u8 default_addr, u8 first_addr, u8 is_dib8096p);
diff --git a/drivers/media/dvb-frontends/dib9000.c b/drivers/media/dvb-frontends/dib9000.c
index c95fff4f9582..1b7a4331af05 100644
--- a/drivers/media/dvb-frontends/dib9000.c
+++ b/drivers/media/dvb-frontends/dib9000.c
@@ -1714,12 +1714,12 @@ static u32 dib9000_i2c_func(struct i2c_adapter *adapter)
 	return I2C_FUNC_I2C;
 }
 
-static struct i2c_algorithm dib9000_tuner_algo = {
+static const struct i2c_algorithm dib9000_tuner_algo = {
 	.master_xfer = dib9000_tuner_xfer,
 	.functionality = dib9000_i2c_func,
 };
 
-static struct i2c_algorithm dib9000_component_bus_algo = {
+static const struct i2c_algorithm dib9000_component_bus_algo = {
 	.master_xfer = dib9000_fw_component_bus_xfer,
 	.functionality = dib9000_i2c_func,
 };
@@ -2462,24 +2462,6 @@ int dib9000_set_slave_frontend(struct dvb_frontend *fe, struct dvb_frontend *fe_
 }
 EXPORT_SYMBOL(dib9000_set_slave_frontend);
 
-int dib9000_remove_slave_frontend(struct dvb_frontend *fe)
-{
-	struct dib9000_state *state = fe->demodulator_priv;
-	u8 index_frontend = 1;
-
-	while ((index_frontend < MAX_NUMBER_OF_FRONTENDS) && (state->fe[index_frontend] != NULL))
-		index_frontend++;
-	if (index_frontend != 1) {
-		dprintk("remove slave fe %p (index %i)\n", state->fe[index_frontend - 1], index_frontend - 1);
-		state->fe[index_frontend] = NULL;
-		return 0;
-	}
-
-	dprintk("no frontend to be removed\n");
-	return -ENODEV;
-}
-EXPORT_SYMBOL(dib9000_remove_slave_frontend);
-
 struct dvb_frontend *dib9000_get_slave_frontend(struct dvb_frontend *fe, int slave_index)
 {
 	struct dib9000_state *state = fe->demodulator_priv;
diff --git a/drivers/media/dvb-frontends/dib9000.h b/drivers/media/dvb-frontends/dib9000.h
index b10a70aa7c9f..40883b41e66b 100644
--- a/drivers/media/dvb-frontends/dib9000.h
+++ b/drivers/media/dvb-frontends/dib9000.h
@@ -37,7 +37,6 @@ extern int dib9000_fw_pid_filter_ctrl(struct dvb_frontend *fe, u8 onoff);
 extern int dib9000_fw_pid_filter(struct dvb_frontend *fe, u8 id, u16 pid, u8 onoff);
 extern int dib9000_firmware_post_pll_init(struct dvb_frontend *fe);
 extern int dib9000_set_slave_frontend(struct dvb_frontend *fe, struct dvb_frontend *fe_slave);
-extern int dib9000_remove_slave_frontend(struct dvb_frontend *fe);
 extern struct dvb_frontend *dib9000_get_slave_frontend(struct dvb_frontend *fe, int slave_index);
 extern struct i2c_adapter *dib9000_get_component_bus_interface(struct dvb_frontend *fe);
 extern int dib9000_set_i2c_adapter(struct dvb_frontend *fe, struct i2c_adapter *i2c);
@@ -97,12 +96,6 @@ static inline int dib9000_set_slave_frontend(struct dvb_frontend *fe, struct dvb
 	return -ENODEV;
 }
 
-static inline int dib9000_remove_slave_frontend(struct dvb_frontend *fe)
-{
-	printk(KERN_WARNING "%s: driver disabled by Kconfig\n", __func__);
-	return -ENODEV;
-}
-
 static inline struct dvb_frontend *dib9000_get_slave_frontend(struct dvb_frontend *fe, int slave_index)
 {
 	printk(KERN_WARNING "%s: driver disabled by Kconfig\n", __func__);
diff --git a/drivers/media/dvb-frontends/drx39xyj/drxj.c b/drivers/media/dvb-frontends/drx39xyj/drxj.c
index 14040c915dbb..499ccff557bf 100644
--- a/drivers/media/dvb-frontends/drx39xyj/drxj.c
+++ b/drivers/media/dvb-frontends/drx39xyj/drxj.c
@@ -5489,7 +5489,7 @@ static int set_vsb_leak_n_gain(struct drx_demod_instance *demod)
 	struct i2c_device_addr *dev_addr = NULL;
 	int rc;
 
-	const u8 vsb_ffe_leak_gain_ram0[] = {
+	static const u8 vsb_ffe_leak_gain_ram0[] = {
 		DRXJ_16TO8(0x8),	/* FFETRAINLKRATIO1  */
 		DRXJ_16TO8(0x8),	/* FFETRAINLKRATIO2  */
 		DRXJ_16TO8(0x8),	/* FFETRAINLKRATIO3  */
@@ -5620,7 +5620,7 @@ static int set_vsb_leak_n_gain(struct drx_demod_instance *demod)
 		DRXJ_16TO8(0x1010)	/* FIRRCA1GAIN8 */
 	};
 
-	const u8 vsb_ffe_leak_gain_ram1[] = {
+	static const u8 vsb_ffe_leak_gain_ram1[] = {
 		DRXJ_16TO8(0x1010),	/* FIRRCA1GAIN9 */
 		DRXJ_16TO8(0x0808),	/* FIRRCA1GAIN10 */
 		DRXJ_16TO8(0x0808),	/* FIRRCA1GAIN11 */
@@ -5710,7 +5710,7 @@ static int set_vsb(struct drx_demod_instance *demod)
 	struct drxj_data *ext_attr = NULL;
 	u16 cmd_result = 0;
 	u16 cmd_param = 0;
-	const u8 vsb_taps_re[] = {
+	static const u8 vsb_taps_re[] = {
 		DRXJ_16TO8(-2),	/* re0  */
 		DRXJ_16TO8(4),	/* re1  */
 		DRXJ_16TO8(1),	/* re2  */
@@ -6666,7 +6666,7 @@ static int set_qam16(struct drx_demod_instance *demod)
 {
 	struct i2c_device_addr *dev_addr = demod->my_i2c_dev_addr;
 	int rc;
-	const u8 qam_dq_qual_fun[] = {
+	static const u8 qam_dq_qual_fun[] = {
 		DRXJ_16TO8(2),	/* fun0  */
 		DRXJ_16TO8(2),	/* fun1  */
 		DRXJ_16TO8(2),	/* fun2  */
@@ -6674,7 +6674,7 @@ static int set_qam16(struct drx_demod_instance *demod)
 		DRXJ_16TO8(3),	/* fun4  */
 		DRXJ_16TO8(3),	/* fun5  */
 	};
-	const u8 qam_eq_cma_rad[] = {
+	static const u8 qam_eq_cma_rad[] = {
 		DRXJ_16TO8(13517),	/* RAD0  */
 		DRXJ_16TO8(13517),	/* RAD1  */
 		DRXJ_16TO8(13517),	/* RAD2  */
@@ -6901,7 +6901,7 @@ static int set_qam32(struct drx_demod_instance *demod)
 {
 	struct i2c_device_addr *dev_addr = demod->my_i2c_dev_addr;
 	int rc;
-	const u8 qam_dq_qual_fun[] = {
+	static const u8 qam_dq_qual_fun[] = {
 		DRXJ_16TO8(3),	/* fun0  */
 		DRXJ_16TO8(3),	/* fun1  */
 		DRXJ_16TO8(3),	/* fun2  */
@@ -6909,7 +6909,7 @@ static int set_qam32(struct drx_demod_instance *demod)
 		DRXJ_16TO8(4),	/* fun4  */
 		DRXJ_16TO8(4),	/* fun5  */
 	};
-	const u8 qam_eq_cma_rad[] = {
+	static const u8 qam_eq_cma_rad[] = {
 		DRXJ_16TO8(6707),	/* RAD0  */
 		DRXJ_16TO8(6707),	/* RAD1  */
 		DRXJ_16TO8(6707),	/* RAD2  */
@@ -7136,7 +7136,8 @@ static int set_qam64(struct drx_demod_instance *demod)
 {
 	struct i2c_device_addr *dev_addr = demod->my_i2c_dev_addr;
 	int rc;
-	const u8 qam_dq_qual_fun[] = {	/* this is hw reset value. no necessary to re-write */
+	static const u8 qam_dq_qual_fun[] = {
+		/* this is hw reset value. no necessary to re-write */
 		DRXJ_16TO8(4),	/* fun0  */
 		DRXJ_16TO8(4),	/* fun1  */
 		DRXJ_16TO8(4),	/* fun2  */
@@ -7144,7 +7145,7 @@ static int set_qam64(struct drx_demod_instance *demod)
 		DRXJ_16TO8(6),	/* fun4  */
 		DRXJ_16TO8(6),	/* fun5  */
 	};
-	const u8 qam_eq_cma_rad[] = {
+	static const u8 qam_eq_cma_rad[] = {
 		DRXJ_16TO8(13336),	/* RAD0  */
 		DRXJ_16TO8(12618),	/* RAD1  */
 		DRXJ_16TO8(11988),	/* RAD2  */
@@ -7371,7 +7372,7 @@ static int set_qam128(struct drx_demod_instance *demod)
 {
 	struct i2c_device_addr *dev_addr = demod->my_i2c_dev_addr;
 	int rc;
-	const u8 qam_dq_qual_fun[] = {
+	static const u8 qam_dq_qual_fun[] = {
 		DRXJ_16TO8(6),	/* fun0  */
 		DRXJ_16TO8(6),	/* fun1  */
 		DRXJ_16TO8(6),	/* fun2  */
@@ -7379,7 +7380,7 @@ static int set_qam128(struct drx_demod_instance *demod)
 		DRXJ_16TO8(9),	/* fun4  */
 		DRXJ_16TO8(9),	/* fun5  */
 	};
-	const u8 qam_eq_cma_rad[] = {
+	static const u8 qam_eq_cma_rad[] = {
 		DRXJ_16TO8(6164),	/* RAD0  */
 		DRXJ_16TO8(6598),	/* RAD1  */
 		DRXJ_16TO8(6394),	/* RAD2  */
@@ -7606,7 +7607,7 @@ static int set_qam256(struct drx_demod_instance *demod)
 {
 	struct i2c_device_addr *dev_addr = demod->my_i2c_dev_addr;
 	int rc;
-	const u8 qam_dq_qual_fun[] = {
+	static const u8 qam_dq_qual_fun[] = {
 		DRXJ_16TO8(8),	/* fun0  */
 		DRXJ_16TO8(8),	/* fun1  */
 		DRXJ_16TO8(8),	/* fun2  */
@@ -7614,7 +7615,7 @@ static int set_qam256(struct drx_demod_instance *demod)
 		DRXJ_16TO8(12),	/* fun4  */
 		DRXJ_16TO8(12),	/* fun5  */
 	};
-	const u8 qam_eq_cma_rad[] = {
+	static const u8 qam_eq_cma_rad[] = {
 		DRXJ_16TO8(12345),	/* RAD0  */
 		DRXJ_16TO8(12345),	/* RAD1  */
 		DRXJ_16TO8(13626),	/* RAD2  */
@@ -7862,7 +7863,7 @@ set_qam(struct drx_demod_instance *demod,
 		/* parameter    */ NULL,
 		/* result       */ NULL
 	};
-	const u8 qam_a_taps[] = {
+	static const u8 qam_a_taps[] = {
 		DRXJ_16TO8(-1),	/* re0  */
 		DRXJ_16TO8(1),	/* re1  */
 		DRXJ_16TO8(1),	/* re2  */
@@ -7892,7 +7893,7 @@ set_qam(struct drx_demod_instance *demod,
 		DRXJ_16TO8(-40),	/* re26 */
 		DRXJ_16TO8(619)	/* re27 */
 	};
-	const u8 qam_b64_taps[] = {
+	static const u8 qam_b64_taps[] = {
 		DRXJ_16TO8(0),	/* re0  */
 		DRXJ_16TO8(-2),	/* re1  */
 		DRXJ_16TO8(1),	/* re2  */
@@ -7922,7 +7923,7 @@ set_qam(struct drx_demod_instance *demod,
 		DRXJ_16TO8(-46),	/* re26 */
 		DRXJ_16TO8(614)	/* re27 */
 	};
-	const u8 qam_b256_taps[] = {
+	static const u8 qam_b256_taps[] = {
 		DRXJ_16TO8(-2),	/* re0  */
 		DRXJ_16TO8(4),	/* re1  */
 		DRXJ_16TO8(1),	/* re2  */
@@ -7952,7 +7953,7 @@ set_qam(struct drx_demod_instance *demod,
 		DRXJ_16TO8(-32),	/* re26 */
 		DRXJ_16TO8(628)	/* re27 */
 	};
-	const u8 qam_c_taps[] = {
+	static const u8 qam_c_taps[] = {
 		DRXJ_16TO8(-3),	/* re0  */
 		DRXJ_16TO8(3),	/* re1  */
 		DRXJ_16TO8(2),	/* re2  */
diff --git a/drivers/media/dvb-frontends/drxd_hard.c b/drivers/media/dvb-frontends/drxd_hard.c
index 17638e08835a..7d04400b18dd 100644
--- a/drivers/media/dvb-frontends/drxd_hard.c
+++ b/drivers/media/dvb-frontends/drxd_hard.c
@@ -638,8 +638,10 @@ static int SetCfgIfAgc(struct drxd_state *state, struct SCfgAgc *cfg)
 			/* == Speed == */
 			{
 				const u16 maxRur = 8;
-				const u16 slowIncrDecLUT[] = { 3, 4, 4, 5, 6 };
-				const u16 fastIncrDecLUT[] = { 14, 15, 15, 16,
+				static const u16 slowIncrDecLUT[] = {
+					3, 4, 4, 5, 6 };
+				const u16 fastIncrDecLUT[] = {
+					14, 15, 15, 16,
 					17, 18, 18, 19,
 					20, 21, 22, 23,
 					24, 26, 27, 28,
diff --git a/drivers/media/dvb-frontends/isl6421.c b/drivers/media/dvb-frontends/isl6421.c
index 838b42771a05..3f3487887672 100644
--- a/drivers/media/dvb-frontends/isl6421.c
+++ b/drivers/media/dvb-frontends/isl6421.c
@@ -38,35 +38,101 @@ struct isl6421 {
 	u8			override_and;
 	struct i2c_adapter	*i2c;
 	u8			i2c_addr;
+	bool			is_off;
 };
 
 static int isl6421_set_voltage(struct dvb_frontend *fe,
 			       enum fe_sec_voltage voltage)
 {
+	int ret;
+	u8 buf;
+	bool is_off;
 	struct isl6421 *isl6421 = (struct isl6421 *) fe->sec_priv;
-	struct i2c_msg msg = {	.addr = isl6421->i2c_addr, .flags = 0,
-				.buf = &isl6421->config,
-				.len = sizeof(isl6421->config) };
+	struct i2c_msg msg[2] = {
+		{
+		  .addr = isl6421->i2c_addr,
+		  .flags = 0,
+		  .buf = &isl6421->config,
+		  .len = 1,
+		}, {
+		  .addr = isl6421->i2c_addr,
+		  .flags = I2C_M_RD,
+		  .buf = &buf,
+		  .len = 1,
+		}
+
+	};
 
 	isl6421->config &= ~(ISL6421_VSEL1 | ISL6421_EN1);
 
 	switch(voltage) {
 	case SEC_VOLTAGE_OFF:
+		is_off = true;
 		break;
 	case SEC_VOLTAGE_13:
+		is_off = false;
 		isl6421->config |= ISL6421_EN1;
 		break;
 	case SEC_VOLTAGE_18:
+		is_off = false;
 		isl6421->config |= (ISL6421_EN1 | ISL6421_VSEL1);
 		break;
 	default:
 		return -EINVAL;
 	}
 
+	/*
+	 * If LNBf were not powered on, disable dynamic current limit, as,
+	 * according with datasheet, highly capacitive load on the output may
+	 * cause a difficult start-up.
+	 */
+	if (isl6421->is_off && !is_off)
+		isl6421->config |= ISL6421_DCL;
+
 	isl6421->config |= isl6421->override_or;
 	isl6421->config &= isl6421->override_and;
 
-	return (i2c_transfer(isl6421->i2c, &msg, 1) == 1) ? 0 : -EIO;
+	ret = i2c_transfer(isl6421->i2c, msg, 2);
+	if (ret < 0)
+		return ret;
+	if (ret != 2)
+		return -EIO;
+
+	/* Store off status now incase future commands fail */
+	isl6421->is_off = is_off;
+
+	/* On overflow, the device will try again after 900 ms (typically) */
+	if (!is_off && (buf & ISL6421_OLF1))
+		msleep(1000);
+
+	/* Re-enable dynamic current limit */
+	if ((isl6421->config & ISL6421_DCL) &&
+	    !(isl6421->override_or & ISL6421_DCL)) {
+		isl6421->config &= ~ISL6421_DCL;
+
+		ret = i2c_transfer(isl6421->i2c, msg, 2);
+		if (ret < 0)
+			return ret;
+		if (ret != 2)
+			return -EIO;
+	}
+
+	/* Check if overload flag is active. If so, disable power */
+	if (!is_off && (buf & ISL6421_OLF1)) {
+		isl6421->config &= ~(ISL6421_VSEL1 | ISL6421_EN1);
+		ret = i2c_transfer(isl6421->i2c, msg, 1);
+		if (ret < 0)
+			return ret;
+		if (ret != 1)
+			return -EIO;
+		isl6421->is_off = true;
+
+		dev_warn(&isl6421->i2c->dev,
+			 "Overload current detected. disabling LNBf power\n");
+		return -EINVAL;
+	}
+
+	return 0;
 }
 
 static int isl6421_enable_high_lnb_voltage(struct dvb_frontend *fe, long arg)
@@ -148,6 +214,8 @@ struct dvb_frontend *isl6421_attach(struct dvb_frontend *fe, struct i2c_adapter
 		return NULL;
 	}
 
+	isl6421->is_off = true;
+
 	/* install release callback */
 	fe->ops.release_sec = isl6421_release;
 
diff --git a/drivers/media/dvb-frontends/mb86a16.c b/drivers/media/dvb-frontends/mb86a16.c
index 9bb122c39c1b..dfe322eccaa1 100644
--- a/drivers/media/dvb-frontends/mb86a16.c
+++ b/drivers/media/dvb-frontends/mb86a16.c
@@ -415,27 +415,21 @@ static int signal_det(struct mb86a16_state *state,
 		      int smrt,
 		      unsigned char *SIG)
 {
-
-	int ret ;
-	int smrtd ;
-	int wait_sym ;
-
-	u32 wait_t;
-	unsigned char S[3] ;
-	int i ;
+	int ret;
+	int smrtd;
+	unsigned char S[3];
+	int i;
 
 	if (*SIG > 45) {
 		if (CNTM_set(state, 2, 1, 2) < 0) {
 			dprintk(verbose, MB86A16_ERROR, 1, "CNTM set Error");
 			return -1;
 		}
-		wait_sym = 40000;
 	} else {
 		if (CNTM_set(state, 3, 1, 2) < 0) {
 			dprintk(verbose, MB86A16_ERROR, 1, "CNTM set Error");
 			return -1;
 		}
-		wait_sym = 80000;
 	}
 	for (i = 0; i < 3; i++) {
 		if (i == 0)
@@ -447,22 +441,17 @@ static int signal_det(struct mb86a16_state *state,
 		smrt_info_get(state, smrtd);
 		smrt_set(state, smrtd);
 		srst(state);
-		wait_t = (wait_sym + 99 * smrtd / 100) / smrtd;
-		if (wait_t == 0)
-			wait_t = 1;
 		msleep_interruptible(10);
 		if (mb86a16_read(state, 0x37, &(S[i])) != 2) {
 			dprintk(verbose, MB86A16_ERROR, 1, "I2C transfer error");
 			return -EREMOTEIO;
 		}
 	}
-	if ((S[1] > S[0] * 112 / 100) &&
-	    (S[1] > S[2] * 112 / 100)) {
-
+	if ((S[1] > S[0] * 112 / 100) && (S[1] > S[2] * 112 / 100))
 		ret = 1;
-	} else {
+	else
 		ret = 0;
-	}
+
 	*SIG = S[1];
 
 	if (CNTM_set(state, 0, 1, 2) < 0) {
diff --git a/drivers/media/dvb-frontends/mn88472.c b/drivers/media/dvb-frontends/mn88472.c
index f6938f9607ac..5e8fd63832e9 100644
--- a/drivers/media/dvb-frontends/mn88472.c
+++ b/drivers/media/dvb-frontends/mn88472.c
@@ -377,7 +377,9 @@ static int mn88472_set_frontend(struct dvb_frontend *fe)
 		ret = regmap_write(dev->regmap[1], 0xf6, 0x05);
 		if (ret)
 			goto err;
-		ret = regmap_write(dev->regmap[2], 0x32, c->stream_id);
+		ret = regmap_write(dev->regmap[2], 0x32,
+				(c->stream_id == NO_STREAM_ID_FILTER) ? 0 :
+				c->stream_id );
 		if (ret)
 			goto err;
 		break;
diff --git a/drivers/media/dvb-frontends/mn88473.c b/drivers/media/dvb-frontends/mn88473.c
index 15874244fd8b..58247432a628 100644
--- a/drivers/media/dvb-frontends/mn88473.c
+++ b/drivers/media/dvb-frontends/mn88473.c
@@ -225,7 +225,9 @@ static int mn88473_set_frontend(struct dvb_frontend *fe)
 
 	/* PLP */
 	if (c->delivery_system == SYS_DVBT2) {
-		ret = regmap_write(dev->regmap[2], 0x36, c->stream_id);
+		ret = regmap_write(dev->regmap[2], 0x36,
+				(c->stream_id == NO_STREAM_ID_FILTER) ? 0 :
+				c->stream_id );
 		if (ret)
 			goto err;
 	}
diff --git a/drivers/media/dvb-frontends/mxl5xx.c b/drivers/media/dvb-frontends/mxl5xx.c
new file mode 100644
index 000000000000..676c96c216c3
--- /dev/null
+++ b/drivers/media/dvb-frontends/mxl5xx.c
@@ -0,0 +1,1873 @@
+/*
+ * Driver for the MaxLinear MxL5xx family of tuners/demods
+ *
+ * Copyright (C) 2014-2015 Ralph Metzler <rjkm@metzlerbros.de>
+ *                         Marcus Metzler <mocm@metzlerbros.de>
+ *                         developed for Digital Devices GmbH
+ *
+ * based on code:
+ * Copyright (c) 2011-2013 MaxLinear, Inc. All rights reserved
+ * which was released under GPL V2
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/init.h>
+#include <linux/delay.h>
+#include <linux/firmware.h>
+#include <linux/i2c.h>
+#include <linux/version.h>
+#include <linux/mutex.h>
+#include <linux/vmalloc.h>
+#include <asm/div64.h>
+#include <asm/unaligned.h>
+
+#include "dvb_frontend.h"
+#include "mxl5xx.h"
+#include "mxl5xx_regs.h"
+#include "mxl5xx_defs.h"
+
+#define BYTE0(v) ((v >>  0) & 0xff)
+#define BYTE1(v) ((v >>  8) & 0xff)
+#define BYTE2(v) ((v >> 16) & 0xff)
+#define BYTE3(v) ((v >> 24) & 0xff)
+
+LIST_HEAD(mxllist);
+
+struct mxl_base {
+	struct list_head     mxllist;
+	struct list_head     mxls;
+
+	u8                   adr;
+	struct i2c_adapter  *i2c;
+
+	u32                  count;
+	u32                  type;
+	u32                  sku_type;
+	u32                  chipversion;
+	u32                  clock;
+	u32                  fwversion;
+
+	u8                  *ts_map;
+	u8                   can_clkout;
+	u8                   chan_bond;
+	u8                   demod_num;
+	u8                   tuner_num;
+
+	unsigned long        next_tune;
+
+	struct mutex         i2c_lock;
+	struct mutex         status_lock;
+	struct mutex         tune_lock;
+
+	u8                   buf[MXL_HYDRA_OEM_MAX_CMD_BUFF_LEN];
+
+	u32                  cmd_size;
+	u8                   cmd_data[MAX_CMD_DATA];
+};
+
+struct mxl {
+	struct list_head     mxl;
+
+	struct mxl_base     *base;
+	struct dvb_frontend  fe;
+	struct device       *i2cdev;
+	u32                  demod;
+	u32                  tuner;
+	u32                  tuner_in_use;
+	u8                   xbar[3];
+
+	unsigned long        tune_time;
+};
+
+static void convert_endian(u8 flag, u32 size, u8 *d)
+{
+	u32 i;
+
+	if (!flag)
+		return;
+	for (i = 0; i < (size & ~3); i += 4) {
+		d[i + 0] ^= d[i + 3];
+		d[i + 3] ^= d[i + 0];
+		d[i + 0] ^= d[i + 3];
+
+		d[i + 1] ^= d[i + 2];
+		d[i + 2] ^= d[i + 1];
+		d[i + 1] ^= d[i + 2];
+	}
+
+	switch (size & 3) {
+	case 0:
+	case 1:
+		/* do nothing */
+		break;
+	case 2:
+		d[i + 0] ^= d[i + 1];
+		d[i + 1] ^= d[i + 0];
+		d[i + 0] ^= d[i + 1];
+		break;
+
+	case 3:
+		d[i + 0] ^= d[i + 2];
+		d[i + 2] ^= d[i + 0];
+		d[i + 0] ^= d[i + 2];
+		break;
+	}
+
+}
+
+static int i2c_write(struct i2c_adapter *adap, u8 adr,
+			    u8 *data, u32 len)
+{
+	struct i2c_msg msg = {.addr = adr, .flags = 0,
+			      .buf = data, .len = len};
+
+	return (i2c_transfer(adap, &msg, 1) == 1) ? 0 : -1;
+}
+
+static int i2c_read(struct i2c_adapter *adap, u8 adr,
+			   u8 *data, u32 len)
+{
+	struct i2c_msg msg = {.addr = adr, .flags = I2C_M_RD,
+			      .buf = data, .len = len};
+
+	return (i2c_transfer(adap, &msg, 1) == 1) ? 0 : -1;
+}
+
+static int i2cread(struct mxl *state, u8 *data, int len)
+{
+	return i2c_read(state->base->i2c, state->base->adr, data, len);
+}
+
+static int i2cwrite(struct mxl *state, u8 *data, int len)
+{
+	return i2c_write(state->base->i2c, state->base->adr, data, len);
+}
+
+static int read_register_unlocked(struct mxl *state, u32 reg, u32 *val)
+{
+	int stat;
+	u8 data[MXL_HYDRA_REG_SIZE_IN_BYTES + MXL_HYDRA_I2C_HDR_SIZE] = {
+		MXL_HYDRA_PLID_REG_READ, 0x04,
+		GET_BYTE(reg, 0), GET_BYTE(reg, 1),
+		GET_BYTE(reg, 2), GET_BYTE(reg, 3),
+	};
+
+	stat = i2cwrite(state, data,
+			MXL_HYDRA_REG_SIZE_IN_BYTES + MXL_HYDRA_I2C_HDR_SIZE);
+	if (stat)
+		dev_err(state->i2cdev, "i2c read error 1\n");
+	if (!stat)
+		stat = i2cread(state, (u8 *) val,
+			       MXL_HYDRA_REG_SIZE_IN_BYTES);
+	le32_to_cpus(val);
+	if (stat)
+		dev_err(state->i2cdev, "i2c read error 2\n");
+	return stat;
+}
+
+#define DMA_I2C_INTERRUPT_ADDR 0x8000011C
+#define DMA_INTR_PROT_WR_CMP 0x08
+
+static int send_command(struct mxl *state, u32 size, u8 *buf)
+{
+	int stat;
+	u32 val, count = 10;
+
+	mutex_lock(&state->base->i2c_lock);
+	if (state->base->fwversion > 0x02010109)  {
+		read_register_unlocked(state, DMA_I2C_INTERRUPT_ADDR, &val);
+		if (DMA_INTR_PROT_WR_CMP & val)
+			dev_info(state->i2cdev, "%s busy\n", __func__);
+		while ((DMA_INTR_PROT_WR_CMP & val) && --count) {
+			mutex_unlock(&state->base->i2c_lock);
+			usleep_range(1000, 2000);
+			mutex_lock(&state->base->i2c_lock);
+			read_register_unlocked(state, DMA_I2C_INTERRUPT_ADDR,
+					       &val);
+		}
+		if (!count) {
+			dev_info(state->i2cdev, "%s busy\n", __func__);
+			mutex_unlock(&state->base->i2c_lock);
+			return -EBUSY;
+		}
+	}
+	stat = i2cwrite(state, buf, size);
+	mutex_unlock(&state->base->i2c_lock);
+	return stat;
+}
+
+static int write_register(struct mxl *state, u32 reg, u32 val)
+{
+	int stat;
+	u8 data[MXL_HYDRA_REG_WRITE_LEN] = {
+		MXL_HYDRA_PLID_REG_WRITE, 0x08,
+		BYTE0(reg), BYTE1(reg), BYTE2(reg), BYTE3(reg),
+		BYTE0(val), BYTE1(val), BYTE2(val), BYTE3(val),
+	};
+	mutex_lock(&state->base->i2c_lock);
+	stat = i2cwrite(state, data, sizeof(data));
+	mutex_unlock(&state->base->i2c_lock);
+	if (stat)
+		dev_err(state->i2cdev, "i2c write error\n");
+	return stat;
+}
+
+static int write_firmware_block(struct mxl *state,
+				u32 reg, u32 size, u8 *reg_data_ptr)
+{
+	int stat;
+	u8 *buf = state->base->buf;
+
+	mutex_lock(&state->base->i2c_lock);
+	buf[0] = MXL_HYDRA_PLID_REG_WRITE;
+	buf[1] = size + 4;
+	buf[2] = GET_BYTE(reg, 0);
+	buf[3] = GET_BYTE(reg, 1);
+	buf[4] = GET_BYTE(reg, 2);
+	buf[5] = GET_BYTE(reg, 3);
+	memcpy(&buf[6], reg_data_ptr, size);
+	stat = i2cwrite(state, buf,
+			MXL_HYDRA_I2C_HDR_SIZE +
+			MXL_HYDRA_REG_SIZE_IN_BYTES + size);
+	mutex_unlock(&state->base->i2c_lock);
+	if (stat)
+		dev_err(state->i2cdev, "fw block write failed\n");
+	return stat;
+}
+
+static int read_register(struct mxl *state, u32 reg, u32 *val)
+{
+	int stat;
+	u8 data[MXL_HYDRA_REG_SIZE_IN_BYTES + MXL_HYDRA_I2C_HDR_SIZE] = {
+		MXL_HYDRA_PLID_REG_READ, 0x04,
+		GET_BYTE(reg, 0), GET_BYTE(reg, 1),
+		GET_BYTE(reg, 2), GET_BYTE(reg, 3),
+	};
+
+	mutex_lock(&state->base->i2c_lock);
+	stat = i2cwrite(state, data,
+			MXL_HYDRA_REG_SIZE_IN_BYTES + MXL_HYDRA_I2C_HDR_SIZE);
+	if (stat)
+		dev_err(state->i2cdev, "i2c read error 1\n");
+	if (!stat)
+		stat = i2cread(state, (u8 *) val,
+			       MXL_HYDRA_REG_SIZE_IN_BYTES);
+	mutex_unlock(&state->base->i2c_lock);
+	le32_to_cpus(val);
+	if (stat)
+		dev_err(state->i2cdev, "i2c read error 2\n");
+	return stat;
+}
+
+static int read_register_block(struct mxl *state, u32 reg, u32 size, u8 *data)
+{
+	int stat;
+	u8 *buf = state->base->buf;
+
+	mutex_lock(&state->base->i2c_lock);
+
+	buf[0] = MXL_HYDRA_PLID_REG_READ;
+	buf[1] = size + 4;
+	buf[2] = GET_BYTE(reg, 0);
+	buf[3] = GET_BYTE(reg, 1);
+	buf[4] = GET_BYTE(reg, 2);
+	buf[5] = GET_BYTE(reg, 3);
+	stat = i2cwrite(state, buf,
+			MXL_HYDRA_I2C_HDR_SIZE + MXL_HYDRA_REG_SIZE_IN_BYTES);
+	if (!stat) {
+		stat = i2cread(state, data, size);
+		convert_endian(MXL_ENABLE_BIG_ENDIAN, size, data);
+	}
+	mutex_unlock(&state->base->i2c_lock);
+	return stat;
+}
+
+static int read_by_mnemonic(struct mxl *state,
+			    u32 reg, u8 lsbloc, u8 numofbits, u32 *val)
+{
+	u32 data = 0, mask = 0;
+	int stat;
+
+	stat = read_register(state, reg, &data);
+	if (stat)
+		return stat;
+	mask = MXL_GET_REG_MASK_32(lsbloc, numofbits);
+	data &= mask;
+	data >>= lsbloc;
+	*val = data;
+	return 0;
+}
+
+
+static int update_by_mnemonic(struct mxl *state,
+			      u32 reg, u8 lsbloc, u8 numofbits, u32 val)
+{
+	u32 data, mask;
+	int stat;
+
+	stat = read_register(state, reg, &data);
+	if (stat)
+		return stat;
+	mask = MXL_GET_REG_MASK_32(lsbloc, numofbits);
+	data = (data & ~mask) | ((val << lsbloc) & mask);
+	stat = write_register(state, reg, data);
+	return stat;
+}
+
+static int firmware_is_alive(struct mxl *state)
+{
+	u32 hb0, hb1;
+
+	if (read_register(state, HYDRA_HEAR_BEAT, &hb0))
+		return 0;
+	msleep(20);
+	if (read_register(state, HYDRA_HEAR_BEAT, &hb1))
+		return 0;
+	if (hb1 == hb0)
+		return 0;
+	return 1;
+}
+
+static int init(struct dvb_frontend *fe)
+{
+	struct dtv_frontend_properties *p = &fe->dtv_property_cache;
+
+	/* init fe stats */
+	p->strength.len = 1;
+	p->strength.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+	p->cnr.len = 1;
+	p->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+	p->pre_bit_error.len = 1;
+	p->pre_bit_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+	p->pre_bit_count.len = 1;
+	p->pre_bit_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+	p->post_bit_error.len = 1;
+	p->post_bit_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+	p->post_bit_count.len = 1;
+	p->post_bit_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+
+	return 0;
+}
+
+static void release(struct dvb_frontend *fe)
+{
+	struct mxl *state = fe->demodulator_priv;
+
+	list_del(&state->mxl);
+	/* Release one frontend, two more shall take its place! */
+	state->base->count--;
+	if (state->base->count == 0) {
+		list_del(&state->base->mxllist);
+		kfree(state->base);
+	}
+	kfree(state);
+}
+
+static int get_algo(struct dvb_frontend *fe)
+{
+	return DVBFE_ALGO_HW;
+}
+
+static int cfg_demod_abort_tune(struct mxl *state)
+{
+	struct MXL_HYDRA_DEMOD_ABORT_TUNE_T abort_tune_cmd;
+	u8 cmd_size = sizeof(abort_tune_cmd);
+	u8 cmd_buff[MXL_HYDRA_OEM_MAX_CMD_BUFF_LEN];
+
+	abort_tune_cmd.demod_id = state->demod;
+	BUILD_HYDRA_CMD(MXL_HYDRA_ABORT_TUNE_CMD, MXL_CMD_WRITE,
+			cmd_size, &abort_tune_cmd, cmd_buff);
+	return send_command(state, cmd_size + MXL_HYDRA_CMD_HEADER_SIZE,
+			    &cmd_buff[0]);
+}
+
+static int send_master_cmd(struct dvb_frontend *fe,
+			   struct dvb_diseqc_master_cmd *cmd)
+{
+	/*struct mxl *state = fe->demodulator_priv;*/
+
+	return 0; /*CfgDemodAbortTune(state);*/
+}
+
+static int set_parameters(struct dvb_frontend *fe)
+{
+	struct mxl *state = fe->demodulator_priv;
+	struct dtv_frontend_properties *p = &fe->dtv_property_cache;
+	struct MXL_HYDRA_DEMOD_PARAM_T demod_chan_cfg;
+	u8 cmd_size = sizeof(demod_chan_cfg);
+	u8 cmd_buff[MXL_HYDRA_OEM_MAX_CMD_BUFF_LEN];
+	u32 srange = 10;
+	int stat;
+
+	if (p->frequency < 950000 || p->frequency > 2150000)
+		return -EINVAL;
+	if (p->symbol_rate < 1000000 || p->symbol_rate > 45000000)
+		return -EINVAL;
+
+	/* CfgDemodAbortTune(state); */
+
+	switch (p->delivery_system) {
+	case SYS_DSS:
+		demod_chan_cfg.standard = MXL_HYDRA_DSS;
+		demod_chan_cfg.roll_off = MXL_HYDRA_ROLLOFF_AUTO;
+		break;
+	case SYS_DVBS:
+		srange = p->symbol_rate / 1000000;
+		if (srange > 10)
+			srange = 10;
+		demod_chan_cfg.standard = MXL_HYDRA_DVBS;
+		demod_chan_cfg.roll_off = MXL_HYDRA_ROLLOFF_0_35;
+		demod_chan_cfg.modulation_scheme = MXL_HYDRA_MOD_QPSK;
+		demod_chan_cfg.pilots = MXL_HYDRA_PILOTS_OFF;
+		break;
+	case SYS_DVBS2:
+		demod_chan_cfg.standard = MXL_HYDRA_DVBS2;
+		demod_chan_cfg.roll_off = MXL_HYDRA_ROLLOFF_AUTO;
+		demod_chan_cfg.modulation_scheme = MXL_HYDRA_MOD_AUTO;
+		demod_chan_cfg.pilots = MXL_HYDRA_PILOTS_AUTO;
+		/* cfg_scrambler(state); */
+		break;
+	default:
+		return -EINVAL;
+	}
+	demod_chan_cfg.tuner_index = state->tuner;
+	demod_chan_cfg.demod_index = state->demod;
+	demod_chan_cfg.frequency_in_hz = p->frequency * 1000;
+	demod_chan_cfg.symbol_rate_in_hz = p->symbol_rate;
+	demod_chan_cfg.max_carrier_offset_in_mhz = srange;
+	demod_chan_cfg.spectrum_inversion = MXL_HYDRA_SPECTRUM_AUTO;
+	demod_chan_cfg.fec_code_rate = MXL_HYDRA_FEC_AUTO;
+
+	mutex_lock(&state->base->tune_lock);
+	if (time_after(jiffies + msecs_to_jiffies(200),
+		       state->base->next_tune))
+		while (time_before(jiffies, state->base->next_tune))
+			usleep_range(10000, 11000);
+	state->base->next_tune = jiffies + msecs_to_jiffies(100);
+	state->tuner_in_use = state->tuner;
+	BUILD_HYDRA_CMD(MXL_HYDRA_DEMOD_SET_PARAM_CMD, MXL_CMD_WRITE,
+			cmd_size, &demod_chan_cfg, cmd_buff);
+	stat = send_command(state, cmd_size + MXL_HYDRA_CMD_HEADER_SIZE,
+			    &cmd_buff[0]);
+	mutex_unlock(&state->base->tune_lock);
+	return stat;
+}
+
+static int enable_tuner(struct mxl *state, u32 tuner, u32 enable);
+
+static int sleep(struct dvb_frontend *fe)
+{
+	struct mxl *state = fe->demodulator_priv;
+	struct mxl *p;
+
+	cfg_demod_abort_tune(state);
+	if (state->tuner_in_use != 0xffffffff) {
+		mutex_lock(&state->base->tune_lock);
+		state->tuner_in_use = 0xffffffff;
+		list_for_each_entry(p, &state->base->mxls, mxl) {
+			if (p->tuner_in_use == state->tuner)
+				break;
+		}
+		if (&p->mxl == &state->base->mxls)
+			enable_tuner(state, state->tuner, 0);
+		mutex_unlock(&state->base->tune_lock);
+	}
+	return 0;
+}
+
+static int read_snr(struct dvb_frontend *fe)
+{
+	struct mxl *state = fe->demodulator_priv;
+	int stat;
+	u32 reg_data = 0;
+	struct dtv_frontend_properties *p = &fe->dtv_property_cache;
+
+	mutex_lock(&state->base->status_lock);
+	HYDRA_DEMOD_STATUS_LOCK(state, state->demod);
+	stat = read_register(state, (HYDRA_DMD_SNR_ADDR_OFFSET +
+				     HYDRA_DMD_STATUS_OFFSET(state->demod)),
+			     &reg_data);
+	HYDRA_DEMOD_STATUS_UNLOCK(state, state->demod);
+	mutex_unlock(&state->base->status_lock);
+
+	p->cnr.stat[0].scale = FE_SCALE_DECIBEL;
+	p->cnr.stat[0].svalue = (s16)reg_data * 10;
+
+	return stat;
+}
+
+static int read_ber(struct dvb_frontend *fe)
+{
+	struct mxl *state = fe->demodulator_priv;
+	struct dtv_frontend_properties *p = &fe->dtv_property_cache;
+	u32 reg[8];
+
+	mutex_lock(&state->base->status_lock);
+	HYDRA_DEMOD_STATUS_LOCK(state, state->demod);
+	read_register_block(state,
+		(HYDRA_DMD_DVBS_1ST_CORR_RS_ERRORS_ADDR_OFFSET +
+		 HYDRA_DMD_STATUS_OFFSET(state->demod)),
+		(4 * sizeof(u32)),
+		(u8 *) &reg[0]);
+	HYDRA_DEMOD_STATUS_UNLOCK(state, state->demod);
+
+	switch (p->delivery_system) {
+	case SYS_DSS:
+	case SYS_DVBS:
+		p->pre_bit_error.stat[0].scale = FE_SCALE_COUNTER;
+		p->pre_bit_error.stat[0].uvalue = reg[2];
+		p->pre_bit_count.stat[0].scale = FE_SCALE_COUNTER;
+		p->pre_bit_count.stat[0].uvalue = reg[3];
+		break;
+	default:
+		break;
+	}
+
+	read_register_block(state,
+		(HYDRA_DMD_DVBS2_CRC_ERRORS_ADDR_OFFSET +
+		 HYDRA_DMD_STATUS_OFFSET(state->demod)),
+		(7 * sizeof(u32)),
+		(u8 *) &reg[0]);
+
+	switch (p->delivery_system) {
+	case SYS_DSS:
+	case SYS_DVBS:
+		p->post_bit_error.stat[0].scale = FE_SCALE_COUNTER;
+		p->post_bit_error.stat[0].uvalue = reg[5];
+		p->post_bit_count.stat[0].scale = FE_SCALE_COUNTER;
+		p->post_bit_count.stat[0].uvalue = reg[6];
+		break;
+	case SYS_DVBS2:
+		p->post_bit_error.stat[0].scale = FE_SCALE_COUNTER;
+		p->post_bit_error.stat[0].uvalue = reg[1];
+		p->post_bit_count.stat[0].scale = FE_SCALE_COUNTER;
+		p->post_bit_count.stat[0].uvalue = reg[2];
+		break;
+	default:
+		break;
+	}
+
+	mutex_unlock(&state->base->status_lock);
+
+	return 0;
+}
+
+static int read_signal_strength(struct dvb_frontend *fe)
+{
+	struct mxl *state = fe->demodulator_priv;
+	struct dtv_frontend_properties *p = &fe->dtv_property_cache;
+	int stat;
+	u32 reg_data = 0;
+
+	mutex_lock(&state->base->status_lock);
+	HYDRA_DEMOD_STATUS_LOCK(state, state->demod);
+	stat = read_register(state, (HYDRA_DMD_STATUS_INPUT_POWER_ADDR +
+				     HYDRA_DMD_STATUS_OFFSET(state->demod)),
+			     &reg_data);
+	HYDRA_DEMOD_STATUS_UNLOCK(state, state->demod);
+	mutex_unlock(&state->base->status_lock);
+
+	p->strength.stat[0].scale = FE_SCALE_DECIBEL;
+	p->strength.stat[0].svalue = (s16) reg_data * 10; /* fix scale */
+
+	return stat;
+}
+
+static int read_status(struct dvb_frontend *fe, enum fe_status *status)
+{
+	struct mxl *state = fe->demodulator_priv;
+	struct dtv_frontend_properties *p = &fe->dtv_property_cache;
+	u32 reg_data = 0;
+
+	mutex_lock(&state->base->status_lock);
+	HYDRA_DEMOD_STATUS_LOCK(state, state->demod);
+	read_register(state, (HYDRA_DMD_LOCK_STATUS_ADDR_OFFSET +
+			     HYDRA_DMD_STATUS_OFFSET(state->demod)),
+			     &reg_data);
+	HYDRA_DEMOD_STATUS_UNLOCK(state, state->demod);
+	mutex_unlock(&state->base->status_lock);
+
+	*status = (reg_data == 1) ? 0x1f : 0;
+
+	/* signal statistics */
+
+	/* signal strength is always available */
+	read_signal_strength(fe);
+
+	if (*status & FE_HAS_CARRIER)
+		read_snr(fe);
+	else
+		p->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+
+	if (*status & FE_HAS_SYNC)
+		read_ber(fe);
+	else {
+		p->pre_bit_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+		p->pre_bit_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+		p->post_bit_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+		p->post_bit_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+	}
+
+	return 0;
+}
+
+static int tune(struct dvb_frontend *fe, bool re_tune,
+		unsigned int mode_flags,
+		unsigned int *delay, enum fe_status *status)
+{
+	struct mxl *state = fe->demodulator_priv;
+	int r = 0;
+
+	*delay = HZ / 2;
+	if (re_tune) {
+		r = set_parameters(fe);
+		if (r)
+			return r;
+		state->tune_time = jiffies;
+		return 0;
+	}
+	if (*status & FE_HAS_LOCK)
+		return 0;
+
+	r = read_status(fe, status);
+	if (r)
+		return r;
+
+	return 0;
+}
+
+static enum fe_code_rate conv_fec(enum MXL_HYDRA_FEC_E fec)
+{
+	enum fe_code_rate fec2fec[11] = {
+		FEC_NONE, FEC_1_2, FEC_3_5, FEC_2_3,
+		FEC_3_4, FEC_4_5, FEC_5_6, FEC_6_7,
+		FEC_7_8, FEC_8_9, FEC_9_10
+	};
+
+	if (fec > MXL_HYDRA_FEC_9_10)
+		return FEC_NONE;
+	return fec2fec[fec];
+}
+
+static int get_frontend(struct dvb_frontend *fe,
+			struct dtv_frontend_properties *p)
+{
+	struct mxl *state = fe->demodulator_priv;
+	u32 reg_data[MXL_DEMOD_CHAN_PARAMS_BUFF_SIZE];
+	u32 freq;
+
+	mutex_lock(&state->base->status_lock);
+	HYDRA_DEMOD_STATUS_LOCK(state, state->demod);
+	read_register_block(state,
+		(HYDRA_DMD_STANDARD_ADDR_OFFSET +
+		HYDRA_DMD_STATUS_OFFSET(state->demod)),
+		(MXL_DEMOD_CHAN_PARAMS_BUFF_SIZE * 4), /* 25 * 4 bytes */
+		(u8 *) &reg_data[0]);
+	/* read demod channel parameters */
+	read_register_block(state,
+		(HYDRA_DMD_STATUS_CENTER_FREQ_IN_KHZ_ADDR +
+		HYDRA_DMD_STATUS_OFFSET(state->demod)),
+		(4), /* 4 bytes */
+		(u8 *) &freq);
+	HYDRA_DEMOD_STATUS_UNLOCK(state, state->demod);
+	mutex_unlock(&state->base->status_lock);
+
+	dev_dbg(state->i2cdev, "freq=%u delsys=%u srate=%u\n",
+		freq * 1000, reg_data[DMD_STANDARD_ADDR],
+		reg_data[DMD_SYMBOL_RATE_ADDR]);
+	p->symbol_rate = reg_data[DMD_SYMBOL_RATE_ADDR];
+	p->frequency = freq;
+	/*
+	 * p->delivery_system =
+	 *	(MXL_HYDRA_BCAST_STD_E) regData[DMD_STANDARD_ADDR];
+	 * p->inversion =
+	 *	(MXL_HYDRA_SPECTRUM_E) regData[DMD_SPECTRUM_INVERSION_ADDR];
+	 * freqSearchRangeKHz =
+	 *	(regData[DMD_FREQ_SEARCH_RANGE_IN_KHZ_ADDR]);
+	 */
+
+	p->fec_inner = conv_fec(reg_data[DMD_FEC_CODE_RATE_ADDR]);
+	switch (p->delivery_system) {
+	case SYS_DSS:
+		break;
+	case SYS_DVBS2:
+		switch ((enum MXL_HYDRA_PILOTS_E)
+			reg_data[DMD_DVBS2_PILOT_ON_OFF_ADDR]) {
+		case MXL_HYDRA_PILOTS_OFF:
+			p->pilot = PILOT_OFF;
+			break;
+		case MXL_HYDRA_PILOTS_ON:
+			p->pilot = PILOT_ON;
+			break;
+		default:
+			break;
+		}
+	case SYS_DVBS:
+		switch ((enum MXL_HYDRA_MODULATION_E)
+			reg_data[DMD_MODULATION_SCHEME_ADDR]) {
+		case MXL_HYDRA_MOD_QPSK:
+			p->modulation = QPSK;
+			break;
+		case MXL_HYDRA_MOD_8PSK:
+			p->modulation = PSK_8;
+			break;
+		default:
+			break;
+		}
+		switch ((enum MXL_HYDRA_ROLLOFF_E)
+			reg_data[DMD_SPECTRUM_ROLL_OFF_ADDR]) {
+		case MXL_HYDRA_ROLLOFF_0_20:
+			p->rolloff = ROLLOFF_20;
+			break;
+		case MXL_HYDRA_ROLLOFF_0_35:
+			p->rolloff = ROLLOFF_35;
+			break;
+		case MXL_HYDRA_ROLLOFF_0_25:
+			p->rolloff = ROLLOFF_25;
+			break;
+		default:
+			break;
+		}
+		break;
+	default:
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static int set_input(struct dvb_frontend *fe, int input)
+{
+	struct mxl *state = fe->demodulator_priv;
+
+	state->tuner = input;
+	return 0;
+}
+
+static struct dvb_frontend_ops mxl_ops = {
+	.delsys = { SYS_DVBS, SYS_DVBS2, SYS_DSS },
+	.info = {
+		.name			= "MaxLinear MxL5xx DVB-S/S2 tuner-demodulator",
+		.frequency_min		= 300000,
+		.frequency_max		= 2350000,
+		.frequency_stepsize	= 0,
+		.frequency_tolerance	= 0,
+		.symbol_rate_min	= 1000000,
+		.symbol_rate_max	= 45000000,
+		.caps			= FE_CAN_INVERSION_AUTO |
+					  FE_CAN_FEC_AUTO       |
+					  FE_CAN_QPSK           |
+					  FE_CAN_2G_MODULATION
+	},
+	.init				= init,
+	.release                        = release,
+	.get_frontend_algo              = get_algo,
+	.tune                           = tune,
+	.read_status			= read_status,
+	.sleep				= sleep,
+	.get_frontend                   = get_frontend,
+	.diseqc_send_master_cmd		= send_master_cmd,
+};
+
+static struct mxl_base *match_base(struct i2c_adapter  *i2c, u8 adr)
+{
+	struct mxl_base *p;
+
+	list_for_each_entry(p, &mxllist, mxllist)
+		if (p->i2c == i2c && p->adr == adr)
+			return p;
+	return NULL;
+}
+
+static void cfg_dev_xtal(struct mxl *state, u32 freq, u32 cap, u32 enable)
+{
+	if (state->base->can_clkout || !enable)
+		update_by_mnemonic(state, 0x90200054, 23, 1, enable);
+
+	if (freq == 24000000)
+		write_register(state, HYDRA_CRYSTAL_SETTING, 0);
+	else
+		write_register(state, HYDRA_CRYSTAL_SETTING, 1);
+
+	write_register(state, HYDRA_CRYSTAL_CAP, cap);
+}
+
+static u32 get_big_endian(u8 num_of_bits, const u8 buf[])
+{
+	u32 ret_value = 0;
+
+	switch (num_of_bits) {
+	case 24:
+		ret_value = (((u32) buf[0]) << 16) |
+			(((u32) buf[1]) << 8) | buf[2];
+		break;
+	case 32:
+		ret_value = (((u32) buf[0]) << 24) |
+			(((u32) buf[1]) << 16) |
+			(((u32) buf[2]) << 8) | buf[3];
+		break;
+	default:
+		break;
+	}
+
+	return ret_value;
+}
+
+static int write_fw_segment(struct mxl *state,
+			    u32 mem_addr, u32 total_size, u8 *data_ptr)
+{
+	int status;
+	u32 data_count = 0;
+	u32 size = 0;
+	u32 orig_size = 0;
+	u8 *w_buf_ptr = NULL;
+	u32 block_size = ((MXL_HYDRA_OEM_MAX_BLOCK_WRITE_LENGTH -
+			 (MXL_HYDRA_I2C_HDR_SIZE +
+			  MXL_HYDRA_REG_SIZE_IN_BYTES)) / 4) * 4;
+	u8 w_msg_buffer[MXL_HYDRA_OEM_MAX_BLOCK_WRITE_LENGTH -
+		      (MXL_HYDRA_I2C_HDR_SIZE + MXL_HYDRA_REG_SIZE_IN_BYTES)];
+
+	do {
+		size = orig_size = (((u32)(data_count + block_size)) > total_size) ?
+			(total_size - data_count) : block_size;
+
+		if (orig_size & 3)
+			size = (orig_size + 4) & ~3;
+		w_buf_ptr = &w_msg_buffer[0];
+		memset((void *) w_buf_ptr, 0, size);
+		memcpy((void *) w_buf_ptr, (void *) data_ptr, orig_size);
+		convert_endian(1, size, w_buf_ptr);
+		status  = write_firmware_block(state, mem_addr, size, w_buf_ptr);
+		if (status)
+			return status;
+		data_count += size;
+		mem_addr   += size;
+		data_ptr   += size;
+	} while (data_count < total_size);
+
+	return status;
+}
+
+static int do_firmware_download(struct mxl *state, u8 *mbin_buffer_ptr,
+				u32 mbin_buffer_size)
+
+{
+	int status;
+	u32 index = 0;
+	u32 seg_length = 0;
+	u32 seg_address = 0;
+	struct MBIN_FILE_T *mbin_ptr  = (struct MBIN_FILE_T *)mbin_buffer_ptr;
+	struct MBIN_SEGMENT_T *segment_ptr;
+	enum MXL_BOOL_E xcpu_fw_flag = MXL_FALSE;
+
+	if (mbin_ptr->header.id != MBIN_FILE_HEADER_ID) {
+		dev_err(state->i2cdev, "%s: Invalid file header ID (%c)\n",
+		       __func__, mbin_ptr->header.id);
+		return -EINVAL;
+	}
+	status = write_register(state, FW_DL_SIGN_ADDR, 0);
+	if (status)
+		return status;
+	segment_ptr = (struct MBIN_SEGMENT_T *) (&mbin_ptr->data[0]);
+	for (index = 0; index < mbin_ptr->header.num_segments; index++) {
+		if (segment_ptr->header.id != MBIN_SEGMENT_HEADER_ID) {
+			dev_err(state->i2cdev, "%s: Invalid segment header ID (%c)\n",
+			       __func__, segment_ptr->header.id);
+			return -EINVAL;
+		}
+		seg_length  = get_big_endian(24,
+					    &(segment_ptr->header.len24[0]));
+		seg_address = get_big_endian(32,
+					    &(segment_ptr->header.address[0]));
+
+		if (state->base->type == MXL_HYDRA_DEVICE_568) {
+			if ((((seg_address & 0x90760000) == 0x90760000) ||
+			     ((seg_address & 0x90740000) == 0x90740000)) &&
+			    (xcpu_fw_flag == MXL_FALSE)) {
+				update_by_mnemonic(state, 0x8003003C, 0, 1, 1);
+				msleep(200);
+				write_register(state, 0x90720000, 0);
+				usleep_range(10000, 11000);
+				xcpu_fw_flag = MXL_TRUE;
+			}
+			status = write_fw_segment(state, seg_address,
+						  seg_length,
+						  (u8 *) segment_ptr->data);
+		} else {
+			if (((seg_address & 0x90760000) != 0x90760000) &&
+			    ((seg_address & 0x90740000) != 0x90740000))
+				status = write_fw_segment(state, seg_address,
+					seg_length, (u8 *) segment_ptr->data);
+		}
+		if (status)
+			return status;
+		segment_ptr = (struct MBIN_SEGMENT_T *)
+			&(segment_ptr->data[((seg_length + 3) / 4) * 4]);
+	}
+	return status;
+}
+
+static int check_fw(struct mxl *state, u8 *mbin, u32 mbin_len)
+{
+	struct MBIN_FILE_HEADER_T *fh = (struct MBIN_FILE_HEADER_T *) mbin;
+	u32 flen = (fh->image_size24[0] << 16) |
+		(fh->image_size24[1] <<  8) | fh->image_size24[2];
+	u8 *fw, cs = 0;
+	u32 i;
+
+	if (fh->id != 'M' || fh->fmt_version != '1' || flen > 0x3FFF0) {
+		dev_info(state->i2cdev, "Invalid FW Header\n");
+		return -1;
+	}
+	fw = mbin + sizeof(struct MBIN_FILE_HEADER_T);
+	for (i = 0; i < flen; i += 1)
+		cs += fw[i];
+	if (cs != fh->image_checksum) {
+		dev_info(state->i2cdev, "Invalid FW Checksum\n");
+		return -1;
+	}
+	return 0;
+}
+
+static int firmware_download(struct mxl *state, u8 *mbin, u32 mbin_len)
+{
+	int status;
+	u32 reg_data = 0;
+	struct MXL_HYDRA_SKU_COMMAND_T dev_sku_cfg;
+	u8 cmd_size = sizeof(struct MXL_HYDRA_SKU_COMMAND_T);
+	u8 cmd_buff[sizeof(struct MXL_HYDRA_SKU_COMMAND_T) + 6];
+
+	if (check_fw(state, mbin, mbin_len))
+		return -1;
+
+	/* put CPU into reset */
+	status = update_by_mnemonic(state, 0x8003003C, 0, 1, 0);
+	if (status)
+		return status;
+	usleep_range(1000, 2000);
+
+	/* Reset TX FIFO's, BBAND, XBAR */
+	status = write_register(state, HYDRA_RESET_TRANSPORT_FIFO_REG,
+				HYDRA_RESET_TRANSPORT_FIFO_DATA);
+	if (status)
+		return status;
+	status = write_register(state, HYDRA_RESET_BBAND_REG,
+				HYDRA_RESET_BBAND_DATA);
+	if (status)
+		return status;
+	status = write_register(state, HYDRA_RESET_XBAR_REG,
+				HYDRA_RESET_XBAR_DATA);
+	if (status)
+		return status;
+
+	/* Disable clock to Baseband, Wideband, SerDes,
+	 * Alias ext & Transport modules
+	 */
+	status = write_register(state, HYDRA_MODULES_CLK_2_REG,
+				HYDRA_DISABLE_CLK_2);
+	if (status)
+		return status;
+	/* Clear Software & Host interrupt status - (Clear on read) */
+	status = read_register(state, HYDRA_PRCM_ROOT_CLK_REG, &reg_data);
+	if (status)
+		return status;
+	status = do_firmware_download(state, mbin, mbin_len);
+	if (status)
+		return status;
+
+	if (state->base->type == MXL_HYDRA_DEVICE_568) {
+		usleep_range(10000, 11000);
+
+		/* bring XCPU out of reset */
+		status = write_register(state, 0x90720000, 1);
+		if (status)
+			return status;
+		msleep(500);
+
+		/* Enable XCPU UART message processing in MCPU */
+		status = write_register(state, 0x9076B510, 1);
+		if (status)
+			return status;
+	} else {
+		/* Bring CPU out of reset */
+		status = update_by_mnemonic(state, 0x8003003C, 0, 1, 1);
+		if (status)
+			return status;
+		/* Wait until FW boots */
+		msleep(150);
+	}
+
+	/* Initialize XPT XBAR */
+	status = write_register(state, XPT_DMD0_BASEADDR, 0x76543210);
+	if (status)
+		return status;
+
+	if (!firmware_is_alive(state))
+		return -1;
+
+	dev_info(state->i2cdev, "Hydra FW alive. Hail!\n");
+
+	/* sometimes register values are wrong shortly
+	 * after first heart beats
+	 */
+	msleep(50);
+
+	dev_sku_cfg.sku_type = state->base->sku_type;
+	BUILD_HYDRA_CMD(MXL_HYDRA_DEV_CFG_SKU_CMD, MXL_CMD_WRITE,
+			cmd_size, &dev_sku_cfg, cmd_buff);
+	status = send_command(state, cmd_size + MXL_HYDRA_CMD_HEADER_SIZE,
+			      &cmd_buff[0]);
+
+	return status;
+}
+
+static int cfg_ts_pad_mux(struct mxl *state, enum MXL_BOOL_E enable_serial_ts)
+{
+	int status = 0;
+	u32 pad_mux_value = 0;
+
+	if (enable_serial_ts == MXL_TRUE) {
+		pad_mux_value = 0;
+		if ((state->base->type == MXL_HYDRA_DEVICE_541) ||
+		    (state->base->type == MXL_HYDRA_DEVICE_541S))
+			pad_mux_value = 2;
+	} else {
+		if ((state->base->type == MXL_HYDRA_DEVICE_581) ||
+		    (state->base->type == MXL_HYDRA_DEVICE_581S))
+			pad_mux_value = 2;
+		else
+			pad_mux_value = 3;
+	}
+
+	switch (state->base->type) {
+	case MXL_HYDRA_DEVICE_561:
+	case MXL_HYDRA_DEVICE_581:
+	case MXL_HYDRA_DEVICE_541:
+	case MXL_HYDRA_DEVICE_541S:
+	case MXL_HYDRA_DEVICE_561S:
+	case MXL_HYDRA_DEVICE_581S:
+		status |= update_by_mnemonic(state, 0x90000170, 24, 3,
+					     pad_mux_value);
+		status |= update_by_mnemonic(state, 0x90000170, 28, 3,
+					     pad_mux_value);
+		status |= update_by_mnemonic(state, 0x90000174, 0, 3,
+					     pad_mux_value);
+		status |= update_by_mnemonic(state, 0x90000174, 4, 3,
+					     pad_mux_value);
+		status |= update_by_mnemonic(state, 0x90000174, 8, 3,
+					     pad_mux_value);
+		status |= update_by_mnemonic(state, 0x90000174, 12, 3,
+					     pad_mux_value);
+		status |= update_by_mnemonic(state, 0x90000174, 16, 3,
+					     pad_mux_value);
+		status |= update_by_mnemonic(state, 0x90000174, 20, 3,
+					     pad_mux_value);
+		status |= update_by_mnemonic(state, 0x90000174, 24, 3,
+					     pad_mux_value);
+		status |= update_by_mnemonic(state, 0x90000174, 28, 3,
+					     pad_mux_value);
+		status |= update_by_mnemonic(state, 0x90000178, 0, 3,
+					     pad_mux_value);
+		status |= update_by_mnemonic(state, 0x90000178, 4, 3,
+					     pad_mux_value);
+		status |= update_by_mnemonic(state, 0x90000178, 8, 3,
+					     pad_mux_value);
+		break;
+
+	case MXL_HYDRA_DEVICE_544:
+	case MXL_HYDRA_DEVICE_542:
+		status |= update_by_mnemonic(state, 0x9000016C, 4, 3, 1);
+		status |= update_by_mnemonic(state, 0x9000016C, 8, 3, 0);
+		status |= update_by_mnemonic(state, 0x9000016C, 12, 3, 0);
+		status |= update_by_mnemonic(state, 0x9000016C, 16, 3, 0);
+		status |= update_by_mnemonic(state, 0x90000170, 0, 3, 0);
+		status |= update_by_mnemonic(state, 0x90000178, 12, 3, 1);
+		status |= update_by_mnemonic(state, 0x90000178, 16, 3, 1);
+		status |= update_by_mnemonic(state, 0x90000178, 20, 3, 1);
+		status |= update_by_mnemonic(state, 0x90000178, 24, 3, 1);
+		status |= update_by_mnemonic(state, 0x9000017C, 0, 3, 1);
+		status |= update_by_mnemonic(state, 0x9000017C, 4, 3, 1);
+		if (enable_serial_ts == MXL_ENABLE) {
+			status |= update_by_mnemonic(state,
+				0x90000170, 4, 3, 0);
+			status |= update_by_mnemonic(state,
+				0x90000170, 8, 3, 0);
+			status |= update_by_mnemonic(state,
+				0x90000170, 12, 3, 0);
+			status |= update_by_mnemonic(state,
+				0x90000170, 16, 3, 0);
+			status |= update_by_mnemonic(state,
+				0x90000170, 20, 3, 1);
+			status |= update_by_mnemonic(state,
+				0x90000170, 24, 3, 1);
+			status |= update_by_mnemonic(state,
+				0x90000170, 28, 3, 2);
+			status |= update_by_mnemonic(state,
+				0x90000174, 0, 3, 2);
+			status |= update_by_mnemonic(state,
+				0x90000174, 4, 3, 2);
+			status |= update_by_mnemonic(state,
+				0x90000174, 8, 3, 2);
+			status |= update_by_mnemonic(state,
+				0x90000174, 12, 3, 2);
+			status |= update_by_mnemonic(state,
+				0x90000174, 16, 3, 2);
+			status |= update_by_mnemonic(state,
+				0x90000174, 20, 3, 2);
+			status |= update_by_mnemonic(state,
+				0x90000174, 24, 3, 2);
+			status |= update_by_mnemonic(state,
+				0x90000174, 28, 3, 2);
+			status |= update_by_mnemonic(state,
+				0x90000178, 0, 3, 2);
+			status |= update_by_mnemonic(state,
+				0x90000178, 4, 3, 2);
+			status |= update_by_mnemonic(state,
+				0x90000178, 8, 3, 2);
+		} else {
+			status |= update_by_mnemonic(state,
+				0x90000170, 4, 3, 3);
+			status |= update_by_mnemonic(state,
+				0x90000170, 8, 3, 3);
+			status |= update_by_mnemonic(state,
+				0x90000170, 12, 3, 3);
+			status |= update_by_mnemonic(state,
+				0x90000170, 16, 3, 3);
+			status |= update_by_mnemonic(state,
+				0x90000170, 20, 3, 3);
+			status |= update_by_mnemonic(state,
+				0x90000170, 24, 3, 3);
+			status |= update_by_mnemonic(state,
+				0x90000170, 28, 3, 3);
+			status |= update_by_mnemonic(state,
+				0x90000174, 0, 3, 3);
+			status |= update_by_mnemonic(state,
+				0x90000174, 4, 3, 3);
+			status |= update_by_mnemonic(state,
+				0x90000174, 8, 3, 3);
+			status |= update_by_mnemonic(state,
+				0x90000174, 12, 3, 3);
+			status |= update_by_mnemonic(state,
+				0x90000174, 16, 3, 3);
+			status |= update_by_mnemonic(state,
+				0x90000174, 20, 3, 1);
+			status |= update_by_mnemonic(state,
+				0x90000174, 24, 3, 1);
+			status |= update_by_mnemonic(state,
+				0x90000174, 28, 3, 1);
+			status |= update_by_mnemonic(state,
+				0x90000178, 0, 3, 1);
+			status |= update_by_mnemonic(state,
+				0x90000178, 4, 3, 1);
+			status |= update_by_mnemonic(state,
+				0x90000178, 8, 3, 1);
+		}
+		break;
+
+	case MXL_HYDRA_DEVICE_568:
+		if (enable_serial_ts == MXL_FALSE) {
+			status |= update_by_mnemonic(state,
+				0x9000016C, 8, 3, 5);
+			status |= update_by_mnemonic(state,
+				0x9000016C, 12, 3, 5);
+			status |= update_by_mnemonic(state,
+				0x9000016C, 16, 3, 5);
+			status |= update_by_mnemonic(state,
+				0x9000016C, 20, 3, 5);
+			status |= update_by_mnemonic(state,
+				0x9000016C, 24, 3, 5);
+			status |= update_by_mnemonic(state,
+				0x9000016C, 28, 3, 5);
+			status |= update_by_mnemonic(state,
+				0x90000170, 0, 3, 5);
+			status |= update_by_mnemonic(state,
+				0x90000170, 4, 3, 5);
+			status |= update_by_mnemonic(state,
+				0x90000170, 8, 3, 5);
+			status |= update_by_mnemonic(state,
+				0x90000170, 12, 3, 5);
+			status |= update_by_mnemonic(state,
+				0x90000170, 16, 3, 5);
+			status |= update_by_mnemonic(state,
+				0x90000170, 20, 3, 5);
+
+			status |= update_by_mnemonic(state,
+				0x90000170, 24, 3, pad_mux_value);
+			status |= update_by_mnemonic(state,
+				0x90000174, 0, 3, pad_mux_value);
+			status |= update_by_mnemonic(state,
+				0x90000174, 4, 3, pad_mux_value);
+			status |= update_by_mnemonic(state,
+				0x90000174, 8, 3, pad_mux_value);
+			status |= update_by_mnemonic(state,
+				0x90000174, 12, 3, pad_mux_value);
+			status |= update_by_mnemonic(state,
+				0x90000174, 16, 3, pad_mux_value);
+			status |= update_by_mnemonic(state,
+				0x90000174, 20, 3, pad_mux_value);
+			status |= update_by_mnemonic(state,
+				0x90000174, 24, 3, pad_mux_value);
+			status |= update_by_mnemonic(state,
+				0x90000174, 28, 3, pad_mux_value);
+			status |= update_by_mnemonic(state,
+				0x90000178, 0, 3, pad_mux_value);
+			status |= update_by_mnemonic(state,
+				0x90000178, 4, 3, pad_mux_value);
+
+			status |= update_by_mnemonic(state,
+				0x90000178, 8, 3, 5);
+			status |= update_by_mnemonic(state,
+				0x90000178, 12, 3, 5);
+			status |= update_by_mnemonic(state,
+				0x90000178, 16, 3, 5);
+			status |= update_by_mnemonic(state,
+				0x90000178, 20, 3, 5);
+			status |= update_by_mnemonic(state,
+				0x90000178, 24, 3, 5);
+			status |= update_by_mnemonic(state,
+				0x90000178, 28, 3, 5);
+			status |= update_by_mnemonic(state,
+				0x9000017C, 0, 3, 5);
+			status |= update_by_mnemonic(state,
+				0x9000017C, 4, 3, 5);
+		} else {
+			status |= update_by_mnemonic(state,
+				0x90000170, 4, 3, pad_mux_value);
+			status |= update_by_mnemonic(state,
+				0x90000170, 8, 3, pad_mux_value);
+			status |= update_by_mnemonic(state,
+				0x90000170, 12, 3, pad_mux_value);
+			status |= update_by_mnemonic(state,
+				0x90000170, 16, 3, pad_mux_value);
+			status |= update_by_mnemonic(state,
+				0x90000170, 20, 3, pad_mux_value);
+			status |= update_by_mnemonic(state,
+				0x90000170, 24, 3, pad_mux_value);
+			status |= update_by_mnemonic(state,
+				0x90000170, 28, 3, pad_mux_value);
+			status |= update_by_mnemonic(state,
+				0x90000174, 0, 3, pad_mux_value);
+			status |= update_by_mnemonic(state,
+				0x90000174, 4, 3, pad_mux_value);
+			status |= update_by_mnemonic(state,
+				0x90000174, 8, 3, pad_mux_value);
+			status |= update_by_mnemonic(state,
+				0x90000174, 12, 3, pad_mux_value);
+		}
+		break;
+
+
+	case MXL_HYDRA_DEVICE_584:
+	default:
+		status |= update_by_mnemonic(state,
+			0x90000170, 4, 3, pad_mux_value);
+		status |= update_by_mnemonic(state,
+			0x90000170, 8, 3, pad_mux_value);
+		status |= update_by_mnemonic(state,
+			0x90000170, 12, 3, pad_mux_value);
+		status |= update_by_mnemonic(state,
+			0x90000170, 16, 3, pad_mux_value);
+		status |= update_by_mnemonic(state,
+			0x90000170, 20, 3, pad_mux_value);
+		status |= update_by_mnemonic(state,
+			0x90000170, 24, 3, pad_mux_value);
+		status |= update_by_mnemonic(state,
+			0x90000170, 28, 3, pad_mux_value);
+		status |= update_by_mnemonic(state,
+			0x90000174, 0, 3, pad_mux_value);
+		status |= update_by_mnemonic(state,
+			0x90000174, 4, 3, pad_mux_value);
+		status |= update_by_mnemonic(state,
+			0x90000174, 8, 3, pad_mux_value);
+		status |= update_by_mnemonic(state,
+			0x90000174, 12, 3, pad_mux_value);
+		break;
+	}
+	return status;
+}
+
+static int set_drive_strength(struct mxl *state,
+		enum MXL_HYDRA_TS_DRIVE_STRENGTH_E ts_drive_strength)
+{
+	int stat = 0;
+	u32 val;
+
+	read_register(state, 0x90000194, &val);
+	dev_info(state->i2cdev, "DIGIO = %08x\n", val);
+	dev_info(state->i2cdev, "set drive_strength = %u\n", ts_drive_strength);
+
+
+	stat |= update_by_mnemonic(state, 0x90000194, 0, 3, ts_drive_strength);
+	stat |= update_by_mnemonic(state, 0x90000194, 20, 3, ts_drive_strength);
+	stat |= update_by_mnemonic(state, 0x90000194, 24, 3, ts_drive_strength);
+	stat |= update_by_mnemonic(state, 0x90000198, 12, 3, ts_drive_strength);
+	stat |= update_by_mnemonic(state, 0x90000198, 16, 3, ts_drive_strength);
+	stat |= update_by_mnemonic(state, 0x90000198, 20, 3, ts_drive_strength);
+	stat |= update_by_mnemonic(state, 0x90000198, 24, 3, ts_drive_strength);
+	stat |= update_by_mnemonic(state, 0x9000019C, 0, 3, ts_drive_strength);
+	stat |= update_by_mnemonic(state, 0x9000019C, 4, 3, ts_drive_strength);
+	stat |= update_by_mnemonic(state, 0x9000019C, 8, 3, ts_drive_strength);
+	stat |= update_by_mnemonic(state, 0x9000019C, 24, 3, ts_drive_strength);
+	stat |= update_by_mnemonic(state, 0x9000019C, 28, 3, ts_drive_strength);
+	stat |= update_by_mnemonic(state, 0x900001A0, 0, 3, ts_drive_strength);
+	stat |= update_by_mnemonic(state, 0x900001A0, 4, 3, ts_drive_strength);
+	stat |= update_by_mnemonic(state, 0x900001A0, 20, 3, ts_drive_strength);
+	stat |= update_by_mnemonic(state, 0x900001A0, 24, 3, ts_drive_strength);
+	stat |= update_by_mnemonic(state, 0x900001A0, 28, 3, ts_drive_strength);
+
+	return stat;
+}
+
+static int enable_tuner(struct mxl *state, u32 tuner, u32 enable)
+{
+	int stat = 0;
+	struct MXL_HYDRA_TUNER_CMD ctrl_tuner_cmd;
+	u8 cmd_size = sizeof(ctrl_tuner_cmd);
+	u8 cmd_buff[MXL_HYDRA_OEM_MAX_CMD_BUFF_LEN];
+	u32 val, count = 10;
+
+	ctrl_tuner_cmd.tuner_id = tuner;
+	ctrl_tuner_cmd.enable = enable;
+	BUILD_HYDRA_CMD(MXL_HYDRA_TUNER_ACTIVATE_CMD, MXL_CMD_WRITE,
+			cmd_size, &ctrl_tuner_cmd, cmd_buff);
+	stat = send_command(state, cmd_size + MXL_HYDRA_CMD_HEADER_SIZE,
+			    &cmd_buff[0]);
+	if (stat)
+		return stat;
+	read_register(state, HYDRA_TUNER_ENABLE_COMPLETE, &val);
+	while (--count && ((val >> tuner) & 1) != enable) {
+		msleep(20);
+		read_register(state, HYDRA_TUNER_ENABLE_COMPLETE, &val);
+	}
+	if (!count)
+		return -1;
+	read_register(state, HYDRA_TUNER_ENABLE_COMPLETE, &val);
+	dev_dbg(state->i2cdev, "tuner %u ready = %u\n",
+		tuner, (val >> tuner) & 1);
+
+	return 0;
+}
+
+
+static int config_ts(struct mxl *state, enum MXL_HYDRA_DEMOD_ID_E demod_id,
+		     struct MXL_HYDRA_MPEGOUT_PARAM_T *mpeg_out_param_ptr)
+{
+	int status = 0;
+	u32 nco_count_min = 0;
+	u32 clk_type = 0;
+
+	struct MXL_REG_FIELD_T xpt_sync_polarity[MXL_HYDRA_DEMOD_MAX] = {
+		{0x90700010, 8, 1}, {0x90700010, 9, 1},
+		{0x90700010, 10, 1}, {0x90700010, 11, 1},
+		{0x90700010, 12, 1}, {0x90700010, 13, 1},
+		{0x90700010, 14, 1}, {0x90700010, 15, 1} };
+	struct MXL_REG_FIELD_T xpt_clock_polarity[MXL_HYDRA_DEMOD_MAX] = {
+		{0x90700010, 16, 1}, {0x90700010, 17, 1},
+		{0x90700010, 18, 1}, {0x90700010, 19, 1},
+		{0x90700010, 20, 1}, {0x90700010, 21, 1},
+		{0x90700010, 22, 1}, {0x90700010, 23, 1} };
+	struct MXL_REG_FIELD_T xpt_valid_polarity[MXL_HYDRA_DEMOD_MAX] = {
+		{0x90700014, 0, 1}, {0x90700014, 1, 1},
+		{0x90700014, 2, 1}, {0x90700014, 3, 1},
+		{0x90700014, 4, 1}, {0x90700014, 5, 1},
+		{0x90700014, 6, 1}, {0x90700014, 7, 1} };
+	struct MXL_REG_FIELD_T xpt_ts_clock_phase[MXL_HYDRA_DEMOD_MAX] = {
+		{0x90700018, 0, 3}, {0x90700018, 4, 3},
+		{0x90700018, 8, 3}, {0x90700018, 12, 3},
+		{0x90700018, 16, 3}, {0x90700018, 20, 3},
+		{0x90700018, 24, 3}, {0x90700018, 28, 3} };
+	struct MXL_REG_FIELD_T xpt_lsb_first[MXL_HYDRA_DEMOD_MAX] = {
+		{0x9070000C, 16, 1}, {0x9070000C, 17, 1},
+		{0x9070000C, 18, 1}, {0x9070000C, 19, 1},
+		{0x9070000C, 20, 1}, {0x9070000C, 21, 1},
+		{0x9070000C, 22, 1}, {0x9070000C, 23, 1} };
+	struct MXL_REG_FIELD_T xpt_sync_byte[MXL_HYDRA_DEMOD_MAX] = {
+		{0x90700010, 0, 1}, {0x90700010, 1, 1},
+		{0x90700010, 2, 1}, {0x90700010, 3, 1},
+		{0x90700010, 4, 1}, {0x90700010, 5, 1},
+		{0x90700010, 6, 1}, {0x90700010, 7, 1} };
+	struct MXL_REG_FIELD_T xpt_enable_output[MXL_HYDRA_DEMOD_MAX] = {
+		{0x9070000C, 0, 1}, {0x9070000C, 1, 1},
+		{0x9070000C, 2, 1}, {0x9070000C, 3, 1},
+		{0x9070000C, 4, 1}, {0x9070000C, 5, 1},
+		{0x9070000C, 6, 1}, {0x9070000C, 7, 1} };
+	struct MXL_REG_FIELD_T xpt_err_replace_sync[MXL_HYDRA_DEMOD_MAX] = {
+		{0x9070000C, 24, 1}, {0x9070000C, 25, 1},
+		{0x9070000C, 26, 1}, {0x9070000C, 27, 1},
+		{0x9070000C, 28, 1}, {0x9070000C, 29, 1},
+		{0x9070000C, 30, 1}, {0x9070000C, 31, 1} };
+	struct MXL_REG_FIELD_T xpt_err_replace_valid[MXL_HYDRA_DEMOD_MAX] = {
+		{0x90700014, 8, 1}, {0x90700014, 9, 1},
+		{0x90700014, 10, 1}, {0x90700014, 11, 1},
+		{0x90700014, 12, 1}, {0x90700014, 13, 1},
+		{0x90700014, 14, 1}, {0x90700014, 15, 1} };
+	struct MXL_REG_FIELD_T xpt_continuous_clock[MXL_HYDRA_DEMOD_MAX] = {
+		{0x907001D4, 0, 1}, {0x907001D4, 1, 1},
+		{0x907001D4, 2, 1}, {0x907001D4, 3, 1},
+		{0x907001D4, 4, 1}, {0x907001D4, 5, 1},
+		{0x907001D4, 6, 1}, {0x907001D4, 7, 1} };
+	struct MXL_REG_FIELD_T xpt_nco_clock_rate[MXL_HYDRA_DEMOD_MAX] = {
+		{0x90700044, 16, 80}, {0x90700044, 16, 81},
+		{0x90700044, 16, 82}, {0x90700044, 16, 83},
+		{0x90700044, 16, 84}, {0x90700044, 16, 85},
+		{0x90700044, 16, 86}, {0x90700044, 16, 87} };
+
+	demod_id = state->base->ts_map[demod_id];
+
+	if (mpeg_out_param_ptr->enable == MXL_ENABLE) {
+		if (mpeg_out_param_ptr->mpeg_mode ==
+		    MXL_HYDRA_MPEG_MODE_PARALLEL) {
+		} else {
+			cfg_ts_pad_mux(state, MXL_TRUE);
+			update_by_mnemonic(state,
+				0x90700010, 27, 1, MXL_FALSE);
+		}
+	}
+
+	nco_count_min =
+		(u32)(MXL_HYDRA_NCO_CLK / mpeg_out_param_ptr->max_mpeg_clk_rate);
+
+	if (state->base->chipversion >= 2) {
+		status |= update_by_mnemonic(state,
+			xpt_nco_clock_rate[demod_id].reg_addr, /* Reg Addr */
+			xpt_nco_clock_rate[demod_id].lsb_pos, /* LSB pos */
+			xpt_nco_clock_rate[demod_id].num_of_bits, /* Num of bits */
+			nco_count_min); /* Data */
+	} else
+		update_by_mnemonic(state, 0x90700044, 16, 8, nco_count_min);
+
+	if (mpeg_out_param_ptr->mpeg_clk_type == MXL_HYDRA_MPEG_CLK_CONTINUOUS)
+		clk_type = 1;
+
+	if (mpeg_out_param_ptr->mpeg_mode < MXL_HYDRA_MPEG_MODE_PARALLEL) {
+		status |= update_by_mnemonic(state,
+			xpt_continuous_clock[demod_id].reg_addr,
+			xpt_continuous_clock[demod_id].lsb_pos,
+			xpt_continuous_clock[demod_id].num_of_bits,
+			clk_type);
+	} else
+		update_by_mnemonic(state, 0x907001D4, 8, 1, clk_type);
+
+	status |= update_by_mnemonic(state,
+		xpt_sync_polarity[demod_id].reg_addr,
+		xpt_sync_polarity[demod_id].lsb_pos,
+		xpt_sync_polarity[demod_id].num_of_bits,
+		mpeg_out_param_ptr->mpeg_sync_pol);
+
+	status |= update_by_mnemonic(state,
+		xpt_valid_polarity[demod_id].reg_addr,
+		xpt_valid_polarity[demod_id].lsb_pos,
+		xpt_valid_polarity[demod_id].num_of_bits,
+		mpeg_out_param_ptr->mpeg_valid_pol);
+
+	status |= update_by_mnemonic(state,
+		xpt_clock_polarity[demod_id].reg_addr,
+		xpt_clock_polarity[demod_id].lsb_pos,
+		xpt_clock_polarity[demod_id].num_of_bits,
+		mpeg_out_param_ptr->mpeg_clk_pol);
+
+	status |= update_by_mnemonic(state,
+		xpt_sync_byte[demod_id].reg_addr,
+		xpt_sync_byte[demod_id].lsb_pos,
+		xpt_sync_byte[demod_id].num_of_bits,
+		mpeg_out_param_ptr->mpeg_sync_pulse_width);
+
+	status |= update_by_mnemonic(state,
+		xpt_ts_clock_phase[demod_id].reg_addr,
+		xpt_ts_clock_phase[demod_id].lsb_pos,
+		xpt_ts_clock_phase[demod_id].num_of_bits,
+		mpeg_out_param_ptr->mpeg_clk_phase);
+
+	status |= update_by_mnemonic(state,
+		xpt_lsb_first[demod_id].reg_addr,
+		xpt_lsb_first[demod_id].lsb_pos,
+		xpt_lsb_first[demod_id].num_of_bits,
+		mpeg_out_param_ptr->lsb_or_msb_first);
+
+	switch (mpeg_out_param_ptr->mpeg_error_indication) {
+	case MXL_HYDRA_MPEG_ERR_REPLACE_SYNC:
+		status |= update_by_mnemonic(state,
+			xpt_err_replace_sync[demod_id].reg_addr,
+			xpt_err_replace_sync[demod_id].lsb_pos,
+			xpt_err_replace_sync[demod_id].num_of_bits,
+			MXL_TRUE);
+		status |= update_by_mnemonic(state,
+			xpt_err_replace_valid[demod_id].reg_addr,
+			xpt_err_replace_valid[demod_id].lsb_pos,
+			xpt_err_replace_valid[demod_id].num_of_bits,
+			MXL_FALSE);
+		break;
+
+	case MXL_HYDRA_MPEG_ERR_REPLACE_VALID:
+		status |= update_by_mnemonic(state,
+			xpt_err_replace_sync[demod_id].reg_addr,
+			xpt_err_replace_sync[demod_id].lsb_pos,
+			xpt_err_replace_sync[demod_id].num_of_bits,
+			MXL_FALSE);
+
+		status |= update_by_mnemonic(state,
+			xpt_err_replace_valid[demod_id].reg_addr,
+			xpt_err_replace_valid[demod_id].lsb_pos,
+			xpt_err_replace_valid[demod_id].num_of_bits,
+			MXL_TRUE);
+		break;
+
+	case MXL_HYDRA_MPEG_ERR_INDICATION_DISABLED:
+	default:
+		status |= update_by_mnemonic(state,
+			xpt_err_replace_sync[demod_id].reg_addr,
+			xpt_err_replace_sync[demod_id].lsb_pos,
+			xpt_err_replace_sync[demod_id].num_of_bits,
+			MXL_FALSE);
+
+		status |= update_by_mnemonic(state,
+			xpt_err_replace_valid[demod_id].reg_addr,
+			xpt_err_replace_valid[demod_id].lsb_pos,
+			xpt_err_replace_valid[demod_id].num_of_bits,
+			MXL_FALSE);
+
+		break;
+
+	}
+
+	if (mpeg_out_param_ptr->mpeg_mode != MXL_HYDRA_MPEG_MODE_PARALLEL) {
+		status |= update_by_mnemonic(state,
+			xpt_enable_output[demod_id].reg_addr,
+			xpt_enable_output[demod_id].lsb_pos,
+			xpt_enable_output[demod_id].num_of_bits,
+			mpeg_out_param_ptr->enable);
+	}
+	return status;
+}
+
+static int config_mux(struct mxl *state)
+{
+	update_by_mnemonic(state, 0x9070000C, 0, 1, 0);
+	update_by_mnemonic(state, 0x9070000C, 1, 1, 0);
+	update_by_mnemonic(state, 0x9070000C, 2, 1, 0);
+	update_by_mnemonic(state, 0x9070000C, 3, 1, 0);
+	update_by_mnemonic(state, 0x9070000C, 4, 1, 0);
+	update_by_mnemonic(state, 0x9070000C, 5, 1, 0);
+	update_by_mnemonic(state, 0x9070000C, 6, 1, 0);
+	update_by_mnemonic(state, 0x9070000C, 7, 1, 0);
+	update_by_mnemonic(state, 0x90700008, 0, 2, 1);
+	update_by_mnemonic(state, 0x90700008, 2, 2, 1);
+	return 0;
+}
+
+static int load_fw(struct mxl *state, struct mxl5xx_cfg *cfg)
+{
+	int stat = 0;
+	u8 *buf;
+
+	if (cfg->fw)
+		return firmware_download(state, cfg->fw, cfg->fw_len);
+
+	if (!cfg->fw_read)
+		return -1;
+
+	buf = vmalloc(0x40000);
+	if (!buf)
+		return -ENOMEM;
+
+	cfg->fw_read(cfg->fw_priv, buf, 0x40000);
+	stat = firmware_download(state, buf, 0x40000);
+	vfree(buf);
+
+	return stat;
+}
+
+static int validate_sku(struct mxl *state)
+{
+	u32 pad_mux_bond = 0, prcm_chip_id = 0, prcm_so_cid = 0;
+	int status;
+	u32 type = state->base->type;
+
+	status = read_by_mnemonic(state, 0x90000190, 0, 3, &pad_mux_bond);
+	status |= read_by_mnemonic(state, 0x80030000, 0, 12, &prcm_chip_id);
+	status |= read_by_mnemonic(state, 0x80030004, 24, 8, &prcm_so_cid);
+	if (status)
+		return -1;
+
+	dev_info(state->i2cdev, "padMuxBond=%08x, prcmChipId=%08x, prcmSoCId=%08x\n",
+		pad_mux_bond, prcm_chip_id, prcm_so_cid);
+
+	if (prcm_chip_id != 0x560) {
+		switch (pad_mux_bond) {
+		case MXL_HYDRA_SKU_ID_581:
+			if (type == MXL_HYDRA_DEVICE_581)
+				return 0;
+			if (type == MXL_HYDRA_DEVICE_581S) {
+				state->base->type = MXL_HYDRA_DEVICE_581;
+				return 0;
+			}
+			break;
+		case MXL_HYDRA_SKU_ID_584:
+			if (type == MXL_HYDRA_DEVICE_584)
+				return 0;
+			break;
+		case MXL_HYDRA_SKU_ID_544:
+			if (type == MXL_HYDRA_DEVICE_544)
+				return 0;
+			if (type == MXL_HYDRA_DEVICE_542)
+				return 0;
+			break;
+		case MXL_HYDRA_SKU_ID_582:
+			if (type == MXL_HYDRA_DEVICE_582)
+				return 0;
+			break;
+		default:
+			return -1;
+		}
+	} else {
+
+	}
+	return -1;
+}
+
+static int get_fwinfo(struct mxl *state)
+{
+	int status;
+	u32 val = 0;
+
+	status = read_by_mnemonic(state, 0x90000190, 0, 3, &val);
+	if (status)
+		return status;
+	dev_info(state->i2cdev, "chipID=%08x\n", val);
+
+	status = read_by_mnemonic(state, 0x80030004, 8, 8, &val);
+	if (status)
+		return status;
+	dev_info(state->i2cdev, "chipVer=%08x\n", val);
+
+	status = read_register(state, HYDRA_FIRMWARE_VERSION, &val);
+	if (status)
+		return status;
+	dev_info(state->i2cdev, "FWVer=%08x\n", val);
+
+	state->base->fwversion = val;
+	return status;
+}
+
+
+static u8 ts_map1_to_1[MXL_HYDRA_DEMOD_MAX] = {
+	MXL_HYDRA_DEMOD_ID_0,
+	MXL_HYDRA_DEMOD_ID_1,
+	MXL_HYDRA_DEMOD_ID_2,
+	MXL_HYDRA_DEMOD_ID_3,
+	MXL_HYDRA_DEMOD_ID_4,
+	MXL_HYDRA_DEMOD_ID_5,
+	MXL_HYDRA_DEMOD_ID_6,
+	MXL_HYDRA_DEMOD_ID_7,
+};
+
+static u8 ts_map54x[MXL_HYDRA_DEMOD_MAX] = {
+	MXL_HYDRA_DEMOD_ID_2,
+	MXL_HYDRA_DEMOD_ID_3,
+	MXL_HYDRA_DEMOD_ID_4,
+	MXL_HYDRA_DEMOD_ID_5,
+	MXL_HYDRA_DEMOD_MAX,
+	MXL_HYDRA_DEMOD_MAX,
+	MXL_HYDRA_DEMOD_MAX,
+	MXL_HYDRA_DEMOD_MAX,
+};
+
+static int probe(struct mxl *state, struct mxl5xx_cfg *cfg)
+{
+	u32 chipver;
+	int fw, status, j;
+	struct MXL_HYDRA_MPEGOUT_PARAM_T mpeg_interface_cfg;
+
+	state->base->ts_map = ts_map1_to_1;
+
+	switch (state->base->type) {
+	case MXL_HYDRA_DEVICE_581:
+	case MXL_HYDRA_DEVICE_581S:
+		state->base->can_clkout = 1;
+		state->base->demod_num = 8;
+		state->base->tuner_num = 1;
+		state->base->sku_type = MXL_HYDRA_SKU_TYPE_581;
+		break;
+	case MXL_HYDRA_DEVICE_582:
+		state->base->can_clkout = 1;
+		state->base->demod_num = 8;
+		state->base->tuner_num = 3;
+		state->base->sku_type = MXL_HYDRA_SKU_TYPE_582;
+		break;
+	case MXL_HYDRA_DEVICE_585:
+		state->base->can_clkout = 0;
+		state->base->demod_num = 8;
+		state->base->tuner_num = 4;
+		state->base->sku_type = MXL_HYDRA_SKU_TYPE_585;
+		break;
+	case MXL_HYDRA_DEVICE_544:
+		state->base->can_clkout = 0;
+		state->base->demod_num = 4;
+		state->base->tuner_num = 4;
+		state->base->sku_type = MXL_HYDRA_SKU_TYPE_544;
+		state->base->ts_map = ts_map54x;
+		break;
+	case MXL_HYDRA_DEVICE_541:
+	case MXL_HYDRA_DEVICE_541S:
+		state->base->can_clkout = 0;
+		state->base->demod_num = 4;
+		state->base->tuner_num = 1;
+		state->base->sku_type = MXL_HYDRA_SKU_TYPE_541;
+		state->base->ts_map = ts_map54x;
+		break;
+	case MXL_HYDRA_DEVICE_561:
+	case MXL_HYDRA_DEVICE_561S:
+		state->base->can_clkout = 0;
+		state->base->demod_num = 6;
+		state->base->tuner_num = 1;
+		state->base->sku_type = MXL_HYDRA_SKU_TYPE_561;
+		break;
+	case MXL_HYDRA_DEVICE_568:
+		state->base->can_clkout = 0;
+		state->base->demod_num = 8;
+		state->base->tuner_num = 1;
+		state->base->chan_bond = 1;
+		state->base->sku_type = MXL_HYDRA_SKU_TYPE_568;
+		break;
+	case MXL_HYDRA_DEVICE_542:
+		state->base->can_clkout = 1;
+		state->base->demod_num = 4;
+		state->base->tuner_num = 3;
+		state->base->sku_type = MXL_HYDRA_SKU_TYPE_542;
+		state->base->ts_map = ts_map54x;
+		break;
+	case MXL_HYDRA_DEVICE_TEST:
+	case MXL_HYDRA_DEVICE_584:
+	default:
+		state->base->can_clkout = 0;
+		state->base->demod_num = 8;
+		state->base->tuner_num = 4;
+		state->base->sku_type = MXL_HYDRA_SKU_TYPE_584;
+		break;
+	}
+
+	status = validate_sku(state);
+	if (status)
+		return status;
+
+	update_by_mnemonic(state, 0x80030014, 9, 1, 1);
+	update_by_mnemonic(state, 0x8003003C, 12, 1, 1);
+	status = read_by_mnemonic(state, 0x80030000, 12, 4, &chipver);
+	if (status)
+		state->base->chipversion = 0;
+	else
+		state->base->chipversion = (chipver == 2) ? 2 : 1;
+	dev_info(state->i2cdev, "Hydra chip version %u\n",
+		state->base->chipversion);
+
+	cfg_dev_xtal(state, cfg->clk, cfg->cap, 0);
+
+	fw = firmware_is_alive(state);
+	if (!fw) {
+		status = load_fw(state, cfg);
+		if (status)
+			return status;
+	}
+	get_fwinfo(state);
+
+	config_mux(state);
+	mpeg_interface_cfg.enable = MXL_ENABLE;
+	mpeg_interface_cfg.lsb_or_msb_first = MXL_HYDRA_MPEG_SERIAL_MSB_1ST;
+	/*  supports only (0-104&139)MHz */
+	if (cfg->ts_clk)
+		mpeg_interface_cfg.max_mpeg_clk_rate = cfg->ts_clk;
+	else
+		mpeg_interface_cfg.max_mpeg_clk_rate = 69; /* 139; */
+	mpeg_interface_cfg.mpeg_clk_phase = MXL_HYDRA_MPEG_CLK_PHASE_SHIFT_0_DEG;
+	mpeg_interface_cfg.mpeg_clk_pol = MXL_HYDRA_MPEG_CLK_IN_PHASE;
+	/* MXL_HYDRA_MPEG_CLK_GAPPED; */
+	mpeg_interface_cfg.mpeg_clk_type = MXL_HYDRA_MPEG_CLK_CONTINUOUS;
+	mpeg_interface_cfg.mpeg_error_indication =
+		MXL_HYDRA_MPEG_ERR_INDICATION_DISABLED;
+	mpeg_interface_cfg.mpeg_mode = MXL_HYDRA_MPEG_MODE_SERIAL_3_WIRE;
+	mpeg_interface_cfg.mpeg_sync_pol  = MXL_HYDRA_MPEG_ACTIVE_HIGH;
+	mpeg_interface_cfg.mpeg_sync_pulse_width  = MXL_HYDRA_MPEG_SYNC_WIDTH_BIT;
+	mpeg_interface_cfg.mpeg_valid_pol  = MXL_HYDRA_MPEG_ACTIVE_HIGH;
+
+	for (j = 0; j < state->base->demod_num; j++) {
+		status = config_ts(state, (enum MXL_HYDRA_DEMOD_ID_E) j,
+				   &mpeg_interface_cfg);
+		if (status)
+			return status;
+	}
+	set_drive_strength(state, 1);
+	return 0;
+}
+
+struct dvb_frontend *mxl5xx_attach(struct i2c_adapter *i2c,
+	struct mxl5xx_cfg *cfg, u32 demod, u32 tuner,
+	int (**fn_set_input)(struct dvb_frontend *, int))
+{
+	struct mxl *state;
+	struct mxl_base *base;
+
+	state = kzalloc(sizeof(struct mxl), GFP_KERNEL);
+	if (!state)
+		return NULL;
+
+	state->demod = demod;
+	state->tuner = tuner;
+	state->tuner_in_use = 0xffffffff;
+	state->i2cdev = &i2c->dev;
+
+	base = match_base(i2c, cfg->adr);
+	if (base) {
+		base->count++;
+		if (base->count > base->demod_num)
+			goto fail;
+		state->base = base;
+	} else {
+		base = kzalloc(sizeof(struct mxl_base), GFP_KERNEL);
+		if (!base)
+			goto fail;
+		base->i2c = i2c;
+		base->adr = cfg->adr;
+		base->type = cfg->type;
+		base->count = 1;
+		mutex_init(&base->i2c_lock);
+		mutex_init(&base->status_lock);
+		mutex_init(&base->tune_lock);
+		INIT_LIST_HEAD(&base->mxls);
+
+		state->base = base;
+		if (probe(state, cfg) < 0) {
+			kfree(base);
+			goto fail;
+		}
+		list_add(&base->mxllist, &mxllist);
+	}
+	state->fe.ops               = mxl_ops;
+	state->xbar[0]              = 4;
+	state->xbar[1]              = demod;
+	state->xbar[2]              = 8;
+	state->fe.demodulator_priv  = state;
+	*fn_set_input               = set_input;
+
+	list_add(&state->mxl, &base->mxls);
+	return &state->fe;
+
+fail:
+	kfree(state);
+	return NULL;
+}
+EXPORT_SYMBOL_GPL(mxl5xx_attach);
+
+MODULE_DESCRIPTION("MaxLinear MxL5xx DVB-S/S2 tuner-demodulator driver");
+MODULE_AUTHOR("Ralph and Marcus Metzler, Metzler Brothers Systementwicklung GbR");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/dvb-frontends/mxl5xx.h b/drivers/media/dvb-frontends/mxl5xx.h
new file mode 100644
index 000000000000..532e08111537
--- /dev/null
+++ b/drivers/media/dvb-frontends/mxl5xx.h
@@ -0,0 +1,41 @@
+#ifndef _MXL5XX_H_
+#define _MXL5XX_H_
+
+#include <linux/types.h>
+#include <linux/i2c.h>
+
+#include "dvb_frontend.h"
+
+struct mxl5xx_cfg {
+	u8   adr;
+	u8   type;
+	u32  cap;
+	u32  clk;
+	u32  ts_clk;
+
+	u8  *fw;
+	u32  fw_len;
+
+	int (*fw_read)(void *priv, u8 *buf, u32 len);
+	void *fw_priv;
+};
+
+#if IS_REACHABLE(CONFIG_DVB_MXL5XX)
+
+extern struct dvb_frontend *mxl5xx_attach(struct i2c_adapter *i2c,
+	struct mxl5xx_cfg *cfg, u32 demod, u32 tuner,
+	int (**fn_set_input)(struct dvb_frontend *, int));
+
+#else
+
+static inline struct dvb_frontend *mxl5xx_attach(struct i2c_adapter *i2c,
+	struct mxl5xx_cfg *cfg, u32 demod, u32 tuner,
+	int (**fn_set_input)(struct dvb_frontend *, int))
+{
+	pr_warn("%s: driver disabled by Kconfig\n", __func__);
+	return NULL;
+}
+
+#endif /* CONFIG_DVB_MXL5XX */
+
+#endif /* _MXL5XX_H_ */
diff --git a/drivers/media/dvb-frontends/mxl5xx_defs.h b/drivers/media/dvb-frontends/mxl5xx_defs.h
new file mode 100644
index 000000000000..fd9e61e0188f
--- /dev/null
+++ b/drivers/media/dvb-frontends/mxl5xx_defs.h
@@ -0,0 +1,731 @@
+/*
+ * Defines for the Maxlinear MX58x family of tuners/demods
+ *
+ * Copyright (C) 2014 Digital Devices GmbH
+ *
+ * based on code:
+ * Copyright (c) 2011-2013 MaxLinear, Inc. All rights reserved
+ * which was released under GPL V2
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2, as published by the Free Software Foundation.
+ */
+
+enum MXL_BOOL_E {
+	MXL_DISABLE = 0,
+	MXL_ENABLE  = 1,
+
+	MXL_FALSE = 0,
+	MXL_TRUE  = 1,
+
+	MXL_INVALID = 0,
+	MXL_VALID   = 1,
+
+	MXL_NO      = 0,
+	MXL_YES     = 1,
+
+	MXL_OFF     = 0,
+	MXL_ON      = 1
+};
+
+/* Firmware-Host Command IDs */
+enum MXL_HYDRA_HOST_CMD_ID_E {
+	/* --Device command IDs-- */
+	MXL_HYDRA_DEV_NO_OP_CMD = 0, /* No OP */
+
+	MXL_HYDRA_DEV_SET_POWER_MODE_CMD = 1,
+	MXL_HYDRA_DEV_SET_OVERWRITE_DEF_CMD = 2,
+
+	/* Host-used CMD, not used by firmware */
+	MXL_HYDRA_DEV_FIRMWARE_DOWNLOAD_CMD = 3,
+
+	/* Additional CONTROL types from DTV */
+	MXL_HYDRA_DEV_SET_BROADCAST_PID_STB_ID_CMD = 4,
+	MXL_HYDRA_DEV_GET_PMM_SLEEP_CMD = 5,
+
+	/* --Tuner command IDs-- */
+	MXL_HYDRA_TUNER_TUNE_CMD = 6,
+	MXL_HYDRA_TUNER_GET_STATUS_CMD = 7,
+
+	/* --Demod command IDs-- */
+	MXL_HYDRA_DEMOD_SET_PARAM_CMD = 8,
+	MXL_HYDRA_DEMOD_GET_STATUS_CMD = 9,
+
+	MXL_HYDRA_DEMOD_RESET_FEC_COUNTER_CMD = 10,
+
+	MXL_HYDRA_DEMOD_SET_PKT_NUM_CMD = 11,
+
+	MXL_HYDRA_DEMOD_SET_IQ_SOURCE_CMD = 12,
+	MXL_HYDRA_DEMOD_GET_IQ_DATA_CMD = 13,
+
+	MXL_HYDRA_DEMOD_GET_M68HC05_VER_CMD = 14,
+
+	MXL_HYDRA_DEMOD_SET_ERROR_COUNTER_MODE_CMD = 15,
+
+	/* --- ABORT channel tune */
+	MXL_HYDRA_ABORT_TUNE_CMD = 16, /* Abort current tune command. */
+
+	/* --SWM/FSK command IDs-- */
+	MXL_HYDRA_FSK_RESET_CMD = 17,
+	MXL_HYDRA_FSK_MSG_CMD = 18,
+	MXL_HYDRA_FSK_SET_OP_MODE_CMD = 19,
+
+	/* --DiSeqC command IDs-- */
+	MXL_HYDRA_DISEQC_MSG_CMD = 20,
+	MXL_HYDRA_DISEQC_COPY_MSG_TO_MAILBOX = 21,
+	MXL_HYDRA_DISEQC_CFG_MSG_CMD = 22,
+
+	/* --- FFT Debug Command IDs-- */
+	MXL_HYDRA_REQ_FFT_SPECTRUM_CMD = 23,
+
+	/* -- Demod scramblle code */
+	MXL_HYDRA_DEMOD_SCRAMBLE_CODE_CMD = 24,
+
+	/* ---For host to know how many commands in total */
+	MXL_HYDRA_LAST_HOST_CMD = 25,
+
+	MXL_HYDRA_DEMOD_INTR_TYPE_CMD = 47,
+	MXL_HYDRA_DEV_INTR_CLEAR_CMD = 48,
+	MXL_HYDRA_TUNER_SPECTRUM_REQ_CMD = 53,
+	MXL_HYDRA_TUNER_ACTIVATE_CMD = 55,
+	MXL_HYDRA_DEV_CFG_POWER_MODE_CMD = 56,
+	MXL_HYDRA_DEV_XTAL_CAP_CMD = 57,
+	MXL_HYDRA_DEV_CFG_SKU_CMD = 58,
+	MXL_HYDRA_TUNER_SPECTRUM_MIN_GAIN_CMD = 59,
+	MXL_HYDRA_DISEQC_CONT_TONE_CFG = 60,
+	MXL_HYDRA_DEV_RF_WAKE_UP_CMD = 61,
+	MXL_HYDRA_DEMOD_CFG_EQ_CTRL_PARAM_CMD = 62,
+	MXL_HYDRA_DEMOD_FREQ_OFFSET_SEARCH_RANGE_CMD = 63,
+	MXL_HYDRA_DEV_REQ_PWR_FROM_ADCRSSI_CMD = 64,
+
+	MXL_XCPU_PID_FLT_CFG_CMD = 65,
+	MXL_XCPU_SHMEM_TEST_CMD = 66,
+	MXL_XCPU_ABORT_TUNE_CMD = 67,
+	MXL_XCPU_CHAN_TUNE_CMD = 68,
+	MXL_XCPU_FLT_BOND_HDRS_CMD = 69,
+
+	MXL_HYDRA_DEV_BROADCAST_WAKE_UP_CMD = 70,
+	MXL_HYDRA_FSK_CFG_FSK_FREQ_CMD = 71,
+	MXL_HYDRA_FSK_POWER_DOWN_CMD = 72,
+	MXL_XCPU_CLEAR_CB_STATS_CMD = 73,
+	MXL_XCPU_CHAN_BOND_RESTART_CMD = 74
+};
+
+#define MXL_ENABLE_BIG_ENDIAN        (0)
+
+#define MXL_HYDRA_OEM_MAX_BLOCK_WRITE_LENGTH   248
+
+#define MXL_HYDRA_OEM_MAX_CMD_BUFF_LEN        (248)
+
+#define MXL_HYDRA_CAP_MIN     10
+#define MXL_HYDRA_CAP_MAX     33
+
+#define MXL_HYDRA_PLID_REG_READ       0xFB   /* Read register PLID */
+#define MXL_HYDRA_PLID_REG_WRITE      0xFC   /* Write register PLID */
+
+#define MXL_HYDRA_PLID_CMD_READ       0xFD   /* Command Read PLID */
+#define MXL_HYDRA_PLID_CMD_WRITE      0xFE   /* Command Write PLID */
+
+#define MXL_HYDRA_REG_SIZE_IN_BYTES   4      /* Hydra register size in bytes */
+#define MXL_HYDRA_I2C_HDR_SIZE        (2 * sizeof(u8)) /* PLID + LEN(0xFF) */
+#define MXL_HYDRA_CMD_HEADER_SIZE     (MXL_HYDRA_REG_SIZE_IN_BYTES + MXL_HYDRA_I2C_HDR_SIZE)
+
+#define MXL_HYDRA_SKU_ID_581 0
+#define MXL_HYDRA_SKU_ID_584 1
+#define MXL_HYDRA_SKU_ID_585 2
+#define MXL_HYDRA_SKU_ID_544 3
+#define MXL_HYDRA_SKU_ID_561 4
+#define MXL_HYDRA_SKU_ID_582 5
+#define MXL_HYDRA_SKU_ID_568 6
+
+/* macro for register write data buffer size
+ * (PLID + LEN (0xFF) + RegAddr + RegData)
+ */
+#define MXL_HYDRA_REG_WRITE_LEN       (MXL_HYDRA_I2C_HDR_SIZE + (2 * MXL_HYDRA_REG_SIZE_IN_BYTES))
+
+/* macro to extract a single byte from 4-byte(32-bit) data */
+#define GET_BYTE(x, n)  (((x) >> (8*(n))) & 0xFF)
+
+#define MAX_CMD_DATA 512
+
+#define MXL_GET_REG_MASK_32(lsb_loc, num_of_bits) ((0xFFFFFFFF >> (32 - (num_of_bits))) << (lsb_loc))
+
+#define FW_DL_SIGN (0xDEADBEEF)
+
+#define MBIN_FORMAT_VERSION               '1'
+#define MBIN_FILE_HEADER_ID               'M'
+#define MBIN_SEGMENT_HEADER_ID            'S'
+#define MBIN_MAX_FILE_LENGTH              (1<<23)
+
+struct MBIN_FILE_HEADER_T {
+	u8 id;
+	u8 fmt_version;
+	u8 header_len;
+	u8 num_segments;
+	u8 entry_address[4];
+	u8 image_size24[3];
+	u8 image_checksum;
+	u8 reserved[4];
+};
+
+struct MBIN_FILE_T {
+	struct MBIN_FILE_HEADER_T header;
+	u8 data[1];
+};
+
+struct MBIN_SEGMENT_HEADER_T {
+	u8 id;
+	u8 len24[3];
+	u8 address[4];
+};
+
+struct MBIN_SEGMENT_T {
+	struct MBIN_SEGMENT_HEADER_T header;
+	u8 data[1];
+};
+
+enum MXL_CMD_TYPE_E { MXL_CMD_WRITE = 0, MXL_CMD_READ };
+
+#define BUILD_HYDRA_CMD(cmd_id, req_type, size, data_ptr, cmd_buff)		\
+	do {								\
+		cmd_buff[0] = ((req_type == MXL_CMD_WRITE) ? MXL_HYDRA_PLID_CMD_WRITE : MXL_HYDRA_PLID_CMD_READ); \
+		cmd_buff[1] = (size > 251) ? 0xff : (u8) (size + 4);	\
+		cmd_buff[2] = size;					\
+		cmd_buff[3] = cmd_id;					\
+		cmd_buff[4] = 0x00;					\
+		cmd_buff[5] = 0x00;					\
+		convert_endian(MXL_ENABLE_BIG_ENDIAN, size, (u8 *)data_ptr); \
+		memcpy((void *)&cmd_buff[6], data_ptr, size);		\
+	} while (0)
+
+struct MXL_REG_FIELD_T {
+	u32 reg_addr;
+	u8 lsb_pos;
+	u8 num_of_bits;
+};
+
+struct MXL_DEV_CMD_DATA_T {
+	u32 data_size;
+	u8 data[MAX_CMD_DATA];
+};
+
+enum MXL_HYDRA_SKU_TYPE_E {
+	MXL_HYDRA_SKU_TYPE_MIN = 0x00,
+	MXL_HYDRA_SKU_TYPE_581 = 0x00,
+	MXL_HYDRA_SKU_TYPE_584 = 0x01,
+	MXL_HYDRA_SKU_TYPE_585 = 0x02,
+	MXL_HYDRA_SKU_TYPE_544 = 0x03,
+	MXL_HYDRA_SKU_TYPE_561 = 0x04,
+	MXL_HYDRA_SKU_TYPE_5XX = 0x05,
+	MXL_HYDRA_SKU_TYPE_5YY = 0x06,
+	MXL_HYDRA_SKU_TYPE_511 = 0x07,
+	MXL_HYDRA_SKU_TYPE_561_DE = 0x08,
+	MXL_HYDRA_SKU_TYPE_582 = 0x09,
+	MXL_HYDRA_SKU_TYPE_541 = 0x0A,
+	MXL_HYDRA_SKU_TYPE_568 = 0x0B,
+	MXL_HYDRA_SKU_TYPE_542 = 0x0C,
+	MXL_HYDRA_SKU_TYPE_MAX = 0x0D,
+};
+
+struct MXL_HYDRA_SKU_COMMAND_T {
+	enum MXL_HYDRA_SKU_TYPE_E sku_type;
+};
+
+enum MXL_HYDRA_DEMOD_ID_E {
+	MXL_HYDRA_DEMOD_ID_0 = 0,
+	MXL_HYDRA_DEMOD_ID_1,
+	MXL_HYDRA_DEMOD_ID_2,
+	MXL_HYDRA_DEMOD_ID_3,
+	MXL_HYDRA_DEMOD_ID_4,
+	MXL_HYDRA_DEMOD_ID_5,
+	MXL_HYDRA_DEMOD_ID_6,
+	MXL_HYDRA_DEMOD_ID_7,
+	MXL_HYDRA_DEMOD_MAX
+};
+
+#define MXL_DEMOD_SCRAMBLE_SEQ_LEN  12
+
+#define MAX_STEP_SIZE_24_XTAL_102_05_KHZ  195
+#define MAX_STEP_SIZE_24_XTAL_204_10_KHZ  215
+#define MAX_STEP_SIZE_24_XTAL_306_15_KHZ  203
+#define MAX_STEP_SIZE_24_XTAL_408_20_KHZ  177
+
+#define MAX_STEP_SIZE_27_XTAL_102_05_KHZ  195
+#define MAX_STEP_SIZE_27_XTAL_204_10_KHZ  215
+#define MAX_STEP_SIZE_27_XTAL_306_15_KHZ  203
+#define MAX_STEP_SIZE_27_XTAL_408_20_KHZ  177
+
+#define MXL_HYDRA_SPECTRUM_MIN_FREQ_KHZ  300000
+#define MXL_HYDRA_SPECTRUM_MAX_FREQ_KHZ 2350000
+
+enum MXL_DEMOD_CHAN_PARAMS_OFFSET_E {
+	DMD_STANDARD_ADDR = 0,
+	DMD_SPECTRUM_INVERSION_ADDR,
+	DMD_SPECTRUM_ROLL_OFF_ADDR,
+	DMD_SYMBOL_RATE_ADDR,
+	DMD_MODULATION_SCHEME_ADDR,
+	DMD_FEC_CODE_RATE_ADDR,
+	DMD_SNR_ADDR,
+	DMD_FREQ_OFFSET_ADDR,
+	DMD_CTL_FREQ_OFFSET_ADDR,
+	DMD_STR_FREQ_OFFSET_ADDR,
+	DMD_FTL_FREQ_OFFSET_ADDR,
+	DMD_STR_NBC_SYNC_LOCK_ADDR,
+	DMD_CYCLE_SLIP_COUNT_ADDR,
+	DMD_DISPLAY_IQ_ADDR,
+	DMD_DVBS2_CRC_ERRORS_ADDR,
+	DMD_DVBS2_PER_COUNT_ADDR,
+	DMD_DVBS2_PER_WINDOW_ADDR,
+	DMD_DVBS_CORR_RS_ERRORS_ADDR,
+	DMD_DVBS_UNCORR_RS_ERRORS_ADDR,
+	DMD_DVBS_BER_COUNT_ADDR,
+	DMD_DVBS_BER_WINDOW_ADDR,
+	DMD_TUNER_ID_ADDR,
+	DMD_DVBS2_PILOT_ON_OFF_ADDR,
+	DMD_FREQ_SEARCH_RANGE_IN_KHZ_ADDR,
+
+	MXL_DEMOD_CHAN_PARAMS_BUFF_SIZE,
+};
+
+enum MXL_HYDRA_TUNER_ID_E {
+	MXL_HYDRA_TUNER_ID_0 = 0,
+	MXL_HYDRA_TUNER_ID_1,
+	MXL_HYDRA_TUNER_ID_2,
+	MXL_HYDRA_TUNER_ID_3,
+	MXL_HYDRA_TUNER_MAX
+};
+
+enum MXL_HYDRA_BCAST_STD_E {
+	MXL_HYDRA_DSS = 0,
+	MXL_HYDRA_DVBS,
+	MXL_HYDRA_DVBS2,
+};
+
+enum MXL_HYDRA_FEC_E {
+	MXL_HYDRA_FEC_AUTO = 0,
+	MXL_HYDRA_FEC_1_2,
+	MXL_HYDRA_FEC_3_5,
+	MXL_HYDRA_FEC_2_3,
+	MXL_HYDRA_FEC_3_4,
+	MXL_HYDRA_FEC_4_5,
+	MXL_HYDRA_FEC_5_6,
+	MXL_HYDRA_FEC_6_7,
+	MXL_HYDRA_FEC_7_8,
+	MXL_HYDRA_FEC_8_9,
+	MXL_HYDRA_FEC_9_10,
+};
+
+enum MXL_HYDRA_MODULATION_E {
+	MXL_HYDRA_MOD_AUTO = 0,
+	MXL_HYDRA_MOD_QPSK,
+	MXL_HYDRA_MOD_8PSK
+};
+
+enum MXL_HYDRA_SPECTRUM_E {
+	MXL_HYDRA_SPECTRUM_AUTO = 0,
+	MXL_HYDRA_SPECTRUM_INVERTED,
+	MXL_HYDRA_SPECTRUM_NON_INVERTED,
+};
+
+enum MXL_HYDRA_ROLLOFF_E {
+	MXL_HYDRA_ROLLOFF_AUTO  = 0,
+	MXL_HYDRA_ROLLOFF_0_20,
+	MXL_HYDRA_ROLLOFF_0_25,
+	MXL_HYDRA_ROLLOFF_0_35
+};
+
+enum MXL_HYDRA_PILOTS_E {
+	MXL_HYDRA_PILOTS_OFF  = 0,
+	MXL_HYDRA_PILOTS_ON,
+	MXL_HYDRA_PILOTS_AUTO
+};
+
+enum MXL_HYDRA_CONSTELLATION_SRC_E {
+	MXL_HYDRA_FORMATTER = 0,
+	MXL_HYDRA_LEGACY_FEC,
+	MXL_HYDRA_FREQ_RECOVERY,
+	MXL_HYDRA_NBC,
+	MXL_HYDRA_CTL,
+	MXL_HYDRA_EQ,
+};
+
+struct MXL_HYDRA_DEMOD_LOCK_T {
+	int agc_lock; /* AGC lock info */
+	int fec_lock; /* Demod FEC block lock info */
+};
+
+struct MXL_HYDRA_DEMOD_STATUS_DVBS_T {
+	u32 rs_errors;        /* RS decoder err counter */
+	u32 ber_window;       /* Ber Windows */
+	u32 ber_count;        /* BER count */
+	u32 ber_window_iter1; /* Ber Windows - post viterbi */
+	u32 ber_count_iter1;  /* BER count - post viterbi */
+};
+
+struct MXL_HYDRA_DEMOD_STATUS_DSS_T {
+	u32 rs_errors;  /* RS decoder err counter */
+	u32 ber_window; /* Ber Windows */
+	u32 ber_count;  /* BER count */
+};
+
+struct MXL_HYDRA_DEMOD_STATUS_DVBS2_T {
+	u32 crc_errors;        /* CRC error counter */
+	u32 packet_error_count; /* Number of packet errors */
+	u32 total_packets;     /* Total packets */
+};
+
+struct MXL_HYDRA_DEMOD_STATUS_T {
+	enum MXL_HYDRA_BCAST_STD_E standard_mask; /* Standard DVB-S, DVB-S2 or DSS */
+
+	union {
+		struct MXL_HYDRA_DEMOD_STATUS_DVBS_T demod_status_dvbs;   /* DVB-S demod status */
+		struct MXL_HYDRA_DEMOD_STATUS_DVBS2_T demod_status_dvbs2; /* DVB-S2 demod status */
+		struct MXL_HYDRA_DEMOD_STATUS_DSS_T demod_status_dss;     /* DSS demod status */
+	} u;
+};
+
+struct MXL_HYDRA_DEMOD_SIG_OFFSET_INFO_T {
+	s32 carrier_offset_in_hz; /* CRL offset info */
+	s32 symbol_offset_in_symbol; /* SRL offset info */
+};
+
+struct MXL_HYDRA_DEMOD_SCRAMBLE_INFO_T {
+	u8 scramble_sequence[MXL_DEMOD_SCRAMBLE_SEQ_LEN]; /* scramble sequence */
+	u32 scramble_code; /* scramble gold code */
+};
+
+enum MXL_HYDRA_SPECTRUM_STEP_SIZE_E {
+	MXL_HYDRA_STEP_SIZE_24_XTAL_102_05KHZ, /* 102.05 KHz for 24 MHz XTAL */
+	MXL_HYDRA_STEP_SIZE_24_XTAL_204_10KHZ, /* 204.10 KHz for 24 MHz XTAL */
+	MXL_HYDRA_STEP_SIZE_24_XTAL_306_15KHZ, /* 306.15 KHz for 24 MHz XTAL */
+	MXL_HYDRA_STEP_SIZE_24_XTAL_408_20KHZ, /* 408.20 KHz for 24 MHz XTAL */
+
+	MXL_HYDRA_STEP_SIZE_27_XTAL_102_05KHZ, /* 102.05 KHz for 27 MHz XTAL */
+	MXL_HYDRA_STEP_SIZE_27_XTAL_204_35KHZ, /* 204.35 KHz for 27 MHz XTAL */
+	MXL_HYDRA_STEP_SIZE_27_XTAL_306_52KHZ, /* 306.52 KHz for 27 MHz XTAL */
+	MXL_HYDRA_STEP_SIZE_27_XTAL_408_69KHZ, /* 408.69 KHz for 27 MHz XTAL */
+};
+
+enum MXL_HYDRA_SPECTRUM_RESOLUTION_E {
+	MXL_HYDRA_SPECTRUM_RESOLUTION_00_1_DB, /* 0.1 dB */
+	MXL_HYDRA_SPECTRUM_RESOLUTION_01_0_DB, /* 1.0 dB */
+	MXL_HYDRA_SPECTRUM_RESOLUTION_05_0_DB, /* 5.0 dB */
+	MXL_HYDRA_SPECTRUM_RESOLUTION_10_0_DB, /* 10 dB */
+};
+
+enum MXL_HYDRA_SPECTRUM_ERROR_CODE_E {
+	MXL_SPECTRUM_NO_ERROR,
+	MXL_SPECTRUM_INVALID_PARAMETER,
+	MXL_SPECTRUM_INVALID_STEP_SIZE,
+	MXL_SPECTRUM_BW_CANNOT_BE_COVERED,
+	MXL_SPECTRUM_DEMOD_BUSY,
+	MXL_SPECTRUM_TUNER_NOT_ENABLED,
+};
+
+struct MXL_HYDRA_SPECTRUM_REQ_T {
+	u32 tuner_index; /* TUNER Ctrl: one of MXL58x_TUNER_ID_E */
+	u32 demod_index; /* DEMOD Ctrl: one of MXL58x_DEMOD_ID_E */
+	enum MXL_HYDRA_SPECTRUM_STEP_SIZE_E step_size_in_khz;
+	u32 starting_freq_ink_hz;
+	u32 total_steps;
+	enum MXL_HYDRA_SPECTRUM_RESOLUTION_E spectrum_division;
+};
+
+enum MXL_HYDRA_SEARCH_FREQ_OFFSET_TYPE_E {
+	MXL_HYDRA_SEARCH_MAX_OFFSET = 0, /* DMD searches for max freq offset (i.e. 5MHz) */
+	MXL_HYDRA_SEARCH_BW_PLUS_ROLLOFF, /* DMD searches for BW + ROLLOFF/2 */
+};
+
+struct MXL58X_CFG_FREQ_OFF_SEARCH_RANGE_T {
+	u32 demod_index;
+	enum MXL_HYDRA_SEARCH_FREQ_OFFSET_TYPE_E search_type;
+};
+
+/* there are two slices
+ * slice0 - TS0, TS1, TS2 & TS3
+ * slice1 - TS4, TS5, TS6 & TS7
+ */
+#define MXL_HYDRA_TS_SLICE_MAX  2
+
+#define MAX_FIXED_PID_NUM   32
+
+#define MXL_HYDRA_NCO_CLK   418 /* 418 MHz */
+
+#define MXL_HYDRA_MAX_TS_CLOCK  139 /* 139 MHz */
+
+#define MXL_HYDRA_TS_FIXED_PID_FILT_SIZE          32
+
+#define MXL_HYDRA_SHARED_PID_FILT_SIZE_DEFAULT    33   /* Shared PID filter size in 1-1 mux mode */
+#define MXL_HYDRA_SHARED_PID_FILT_SIZE_2_TO_1     66   /* Shared PID filter size in 2-1 mux mode */
+#define MXL_HYDRA_SHARED_PID_FILT_SIZE_4_TO_1     132  /* Shared PID filter size in 4-1 mux mode */
+
+enum MXL_HYDRA_PID_BANK_TYPE_E {
+	MXL_HYDRA_SOFTWARE_PID_BANK = 0,
+	MXL_HYDRA_HARDWARE_PID_BANK,
+};
+
+enum MXL_HYDRA_TS_MUX_MODE_E {
+	MXL_HYDRA_TS_MUX_PID_REMAP = 0,
+	MXL_HYDRA_TS_MUX_PREFIX_EXTRA_HEADER = 1,
+};
+
+enum MXL_HYDRA_TS_MUX_TYPE_E {
+	MXL_HYDRA_TS_MUX_DISABLE = 0, /* No Mux ( 1 TSIF to 1 TSIF) */
+	MXL_HYDRA_TS_MUX_2_TO_1, /* Mux 2 TSIF to 1 TSIF */
+	MXL_HYDRA_TS_MUX_4_TO_1, /* Mux 4 TSIF to 1 TSIF */
+};
+
+enum MXL_HYDRA_TS_GROUP_E {
+	MXL_HYDRA_TS_GROUP_0_3 = 0, /* TS group 0 to 3 (TS0, TS1, TS2 & TS3) */
+	MXL_HYDRA_TS_GROUP_4_7,     /* TS group 0 to 3 (TS4, TS5, TS6 & TS7) */
+};
+
+enum MXL_HYDRA_TS_PID_FLT_CTRL_E {
+	MXL_HYDRA_TS_PIDS_ALLOW_ALL = 0, /* Allow all pids */
+	MXL_HYDRA_TS_PIDS_DROP_ALL,	 /* Drop all pids */
+	MXL_HYDRA_TS_INVALIDATE_PID_FILTER, /* Delete current PD filter in the device */
+};
+
+enum MXL_HYDRA_TS_PID_TYPE_E {
+	MXL_HYDRA_TS_PID_FIXED = 0,
+	MXL_HYDRA_TS_PID_REGULAR,
+};
+
+struct MXL_HYDRA_TS_PID_T {
+	u16 original_pid;           /* pid from TS */
+	u16 remapped_pid;           /* remapped pid */
+	enum MXL_BOOL_E enable;         /* enable or disable pid */
+	enum MXL_BOOL_E allow_or_drop;    /* allow or drop pid */
+	enum MXL_BOOL_E enable_pid_remap; /* enable or disable pid remap */
+	u8 bond_id;                 /* Bond ID in A0 always 0 - Only for 568 Sku */
+	u8 dest_id;                 /* Output port ID for the PID - Only for 568 Sku */
+};
+
+struct MXL_HYDRA_TS_MUX_PREFIX_HEADER_T {
+	enum MXL_BOOL_E enable;
+	u8 num_byte;
+	u8 header[12];
+};
+
+enum MXL_HYDRA_PID_FILTER_BANK_E {
+	MXL_HYDRA_PID_BANK_A = 0,
+	MXL_HYDRA_PID_BANK_B,
+};
+
+enum MXL_HYDRA_MPEG_DATA_FMT_E {
+	MXL_HYDRA_MPEG_SERIAL_MSB_1ST = 0,
+	MXL_HYDRA_MPEG_SERIAL_LSB_1ST,
+
+	MXL_HYDRA_MPEG_SYNC_WIDTH_BIT = 0,
+	MXL_HYDRA_MPEG_SYNC_WIDTH_BYTE
+};
+
+enum MXL_HYDRA_MPEG_MODE_E {
+	MXL_HYDRA_MPEG_MODE_SERIAL_4_WIRE = 0, /* MPEG 4 Wire serial mode */
+	MXL_HYDRA_MPEG_MODE_SERIAL_3_WIRE,     /* MPEG 3 Wire serial mode */
+	MXL_HYDRA_MPEG_MODE_SERIAL_2_WIRE,     /* MPEG 2 Wire serial mode */
+	MXL_HYDRA_MPEG_MODE_PARALLEL           /* MPEG parallel mode - valid only for MxL581 */
+};
+
+enum MXL_HYDRA_MPEG_CLK_TYPE_E {
+	MXL_HYDRA_MPEG_CLK_CONTINUOUS = 0, /* Continuous MPEG clock */
+	MXL_HYDRA_MPEG_CLK_GAPPED,         /* Gapped (gated) MPEG clock */
+};
+
+enum MXL_HYDRA_MPEG_CLK_FMT_E {
+	MXL_HYDRA_MPEG_ACTIVE_LOW = 0,
+	MXL_HYDRA_MPEG_ACTIVE_HIGH,
+
+	MXL_HYDRA_MPEG_CLK_NEGATIVE = 0,
+	MXL_HYDRA_MPEG_CLK_POSITIVE,
+
+	MXL_HYDRA_MPEG_CLK_IN_PHASE = 0,
+	MXL_HYDRA_MPEG_CLK_INVERTED,
+};
+
+enum MXL_HYDRA_MPEG_CLK_PHASE_E {
+	MXL_HYDRA_MPEG_CLK_PHASE_SHIFT_0_DEG = 0,
+	MXL_HYDRA_MPEG_CLK_PHASE_SHIFT_90_DEG,
+	MXL_HYDRA_MPEG_CLK_PHASE_SHIFT_180_DEG,
+	MXL_HYDRA_MPEG_CLK_PHASE_SHIFT_270_DEG
+};
+
+enum MXL_HYDRA_MPEG_ERR_INDICATION_E {
+	MXL_HYDRA_MPEG_ERR_REPLACE_SYNC = 0,
+	MXL_HYDRA_MPEG_ERR_REPLACE_VALID,
+	MXL_HYDRA_MPEG_ERR_INDICATION_DISABLED
+};
+
+struct MXL_HYDRA_MPEGOUT_PARAM_T {
+	int                                  enable;               /* Enable or Disable MPEG OUT */
+	enum MXL_HYDRA_MPEG_CLK_TYPE_E       mpeg_clk_type;          /* Continuous or gapped */
+	enum MXL_HYDRA_MPEG_CLK_FMT_E        mpeg_clk_pol;           /* MPEG Clk polarity */
+	u8                                   max_mpeg_clk_rate;       /* Max MPEG Clk rate (0 - 104 MHz, 139 MHz) */
+	enum MXL_HYDRA_MPEG_CLK_PHASE_E      mpeg_clk_phase;         /* MPEG Clk phase */
+	enum MXL_HYDRA_MPEG_DATA_FMT_E       lsb_or_msb_first;        /* LSB first or MSB first in TS transmission */
+	enum MXL_HYDRA_MPEG_DATA_FMT_E       mpeg_sync_pulse_width;   /* MPEG SYNC pulse width (1-bit or 1-byte) */
+	enum MXL_HYDRA_MPEG_CLK_FMT_E        mpeg_valid_pol;         /* MPEG VALID polarity */
+	enum MXL_HYDRA_MPEG_CLK_FMT_E        mpeg_sync_pol;          /* MPEG SYNC polarity */
+	enum MXL_HYDRA_MPEG_MODE_E           mpeg_mode;             /* config 4/3/2-wire serial or parallel TS out */
+	enum MXL_HYDRA_MPEG_ERR_INDICATION_E mpeg_error_indication;  /* Enable or Disable MPEG error indication */
+};
+
+enum MXL_HYDRA_EXT_TS_IN_ID_E {
+	MXL_HYDRA_EXT_TS_IN_0 = 0,
+	MXL_HYDRA_EXT_TS_IN_1,
+	MXL_HYDRA_EXT_TS_IN_2,
+	MXL_HYDRA_EXT_TS_IN_3,
+	MXL_HYDRA_EXT_TS_IN_MAX
+};
+
+enum MXL_HYDRA_TS_OUT_ID_E {
+	MXL_HYDRA_TS_OUT_0 = 0,
+	MXL_HYDRA_TS_OUT_1,
+	MXL_HYDRA_TS_OUT_2,
+	MXL_HYDRA_TS_OUT_3,
+	MXL_HYDRA_TS_OUT_4,
+	MXL_HYDRA_TS_OUT_5,
+	MXL_HYDRA_TS_OUT_6,
+	MXL_HYDRA_TS_OUT_7,
+	MXL_HYDRA_TS_OUT_MAX
+};
+
+enum MXL_HYDRA_TS_DRIVE_STRENGTH_E {
+	MXL_HYDRA_TS_DRIVE_STRENGTH_1X = 0,
+	MXL_HYDRA_TS_DRIVE_STRENGTH_2X,
+	MXL_HYDRA_TS_DRIVE_STRENGTH_3X,
+	MXL_HYDRA_TS_DRIVE_STRENGTH_4X,
+	MXL_HYDRA_TS_DRIVE_STRENGTH_5X,
+	MXL_HYDRA_TS_DRIVE_STRENGTH_6X,
+	MXL_HYDRA_TS_DRIVE_STRENGTH_7X,
+	MXL_HYDRA_TS_DRIVE_STRENGTH_8X
+};
+
+enum MXL_HYDRA_DEVICE_E {
+	MXL_HYDRA_DEVICE_581 = 0,
+	MXL_HYDRA_DEVICE_584,
+	MXL_HYDRA_DEVICE_585,
+	MXL_HYDRA_DEVICE_544,
+	MXL_HYDRA_DEVICE_561,
+	MXL_HYDRA_DEVICE_TEST,
+	MXL_HYDRA_DEVICE_582,
+	MXL_HYDRA_DEVICE_541,
+	MXL_HYDRA_DEVICE_568,
+	MXL_HYDRA_DEVICE_542,
+	MXL_HYDRA_DEVICE_541S,
+	MXL_HYDRA_DEVICE_561S,
+	MXL_HYDRA_DEVICE_581S,
+	MXL_HYDRA_DEVICE_MAX
+};
+
+/* Demod IQ data */
+struct MXL_HYDRA_DEMOD_IQ_SRC_T {
+	u32 demod_id;
+	u32 source_of_iq; /* == 0, it means I/Q comes from Formatter
+			 * == 1, Legacy FEC
+			 * == 2, Frequency Recovery
+			 * == 3, NBC
+			 * == 4, CTL
+			 * == 5, EQ
+			 * == 6, FPGA
+			 */
+};
+
+struct MXL_HYDRA_DEMOD_ABORT_TUNE_T {
+	u32 demod_id;
+};
+
+struct MXL_HYDRA_TUNER_CMD {
+	u8 tuner_id;
+	u8 enable;
+};
+
+/* Demod Para for Channel Tune */
+struct MXL_HYDRA_DEMOD_PARAM_T {
+	u32 tuner_index;
+	u32 demod_index;
+	u32 frequency_in_hz;     /* Frequency */
+	u32 standard;          /* one of MXL_HYDRA_BCAST_STD_E */
+	u32 spectrum_inversion; /* Input : Spectrum inversion. */
+	u32 roll_off;           /* rollOff (alpha) factor */
+	u32 symbol_rate_in_hz;    /* Symbol rate */
+	u32 pilots;            /* TRUE = pilots enabled */
+	u32 modulation_scheme;  /* Input : Modulation Scheme is one of MXL_HYDRA_MODULATION_E */
+	u32 fec_code_rate;       /* Input : Forward error correction rate. Is one of MXL_HYDRA_FEC_E */
+	u32 max_carrier_offset_in_mhz; /* Maximum carrier freq offset in MHz. Same as freqSearchRangeKHz, but in unit of MHz. */
+};
+
+struct MXL_HYDRA_DEMOD_SCRAMBLE_CODE_T {
+	u32 demod_index;
+	u8 scramble_sequence[12]; /* scramble sequence */
+	u32 scramble_code; /* scramble gold code */
+};
+
+struct MXL_INTR_CFG_T {
+	u32 intr_type;
+	u32 intr_duration_in_nano_secs;
+	u32 intr_mask;
+};
+
+struct MXL_HYDRA_POWER_MODE_CMD {
+	u8 power_mode; /* enumeration values are defined in MXL_HYDRA_PWR_MODE_E (device API.h) */
+};
+
+struct MXL_HYDRA_RF_WAKEUP_PARAM_T {
+	u32 time_interval_in_seconds; /* in seconds */
+	u32 tuner_index;
+	s32 rssi_threshold;
+};
+
+struct MXL_HYDRA_RF_WAKEUP_CFG_T {
+	u32 tuner_count;
+	struct MXL_HYDRA_RF_WAKEUP_PARAM_T params;
+};
+
+enum MXL_HYDRA_AUX_CTRL_MODE_E {
+	MXL_HYDRA_AUX_CTRL_MODE_FSK = 0, /* Select FSK controller */
+	MXL_HYDRA_AUX_CTRL_MODE_DISEQC,  /* Select DiSEqC controller */
+};
+
+enum MXL_HYDRA_DISEQC_OPMODE_E {
+	MXL_HYDRA_DISEQC_ENVELOPE_MODE = 0,
+	MXL_HYDRA_DISEQC_TONE_MODE,
+};
+
+enum MXL_HYDRA_DISEQC_VER_E {
+	MXL_HYDRA_DISEQC_1_X = 0, /* Config DiSEqC 1.x mode */
+	MXL_HYDRA_DISEQC_2_X, /* Config DiSEqC 2.x mode */
+	MXL_HYDRA_DISEQC_DISABLE /* Disable DiSEqC */
+};
+
+enum MXL_HYDRA_DISEQC_CARRIER_FREQ_E {
+	MXL_HYDRA_DISEQC_CARRIER_FREQ_22KHZ = 0, /* DiSEqC signal frequency of 22 KHz */
+	MXL_HYDRA_DISEQC_CARRIER_FREQ_33KHZ,     /* DiSEqC signal frequency of 33 KHz */
+	MXL_HYDRA_DISEQC_CARRIER_FREQ_44KHZ      /* DiSEqC signal frequency of 44 KHz */
+};
+
+enum MXL_HYDRA_DISEQC_ID_E {
+	MXL_HYDRA_DISEQC_ID_0 = 0,
+	MXL_HYDRA_DISEQC_ID_1,
+	MXL_HYDRA_DISEQC_ID_2,
+	MXL_HYDRA_DISEQC_ID_3
+};
+
+enum MXL_HYDRA_FSK_OP_MODE_E {
+	MXL_HYDRA_FSK_CFG_TYPE_39KPBS = 0, /* 39.0kbps */
+	MXL_HYDRA_FSK_CFG_TYPE_39_017KPBS, /* 39.017kbps */
+	MXL_HYDRA_FSK_CFG_TYPE_115_2KPBS   /* 115.2kbps */
+};
+
+struct MXL58X_DSQ_OP_MODE_T {
+	u32 diseqc_id; /* DSQ 0, 1, 2 or 3 */
+	u32 op_mode; /* Envelope mode (0) or internal tone mode (1) */
+	u32 version; /* 0: 1.0, 1: 1.1, 2: Disable */
+	u32 center_freq; /* 0: 22KHz, 1: 33KHz and 2: 44 KHz */
+};
+
+struct MXL_HYDRA_DISEQC_CFG_CONT_TONE_T {
+	u32 diseqc_id;
+	u32 cont_tone_flag; /* 1: Enable , 0: Disable */
+};
diff --git a/drivers/media/dvb-frontends/mxl5xx_regs.h b/drivers/media/dvb-frontends/mxl5xx_regs.h
new file mode 100644
index 000000000000..5001dafe1ba8
--- /dev/null
+++ b/drivers/media/dvb-frontends/mxl5xx_regs.h
@@ -0,0 +1,367 @@
+/*
+ * Copyright (c) 2011-2013 MaxLinear, Inc. All rights reserved
+ *
+ * License type: GPLv2
+ *
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free Software
+ * Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+ *
+ * This program may alternatively be licensed under a proprietary license from
+ * MaxLinear, Inc.
+ *
+ */
+
+#ifndef __MXL58X_REGISTERS_H__
+#define __MXL58X_REGISTERS_H__
+
+#define HYDRA_INTR_STATUS_REG               0x80030008
+#define HYDRA_INTR_MASK_REG                 0x8003000C
+
+#define HYDRA_CRYSTAL_SETTING               0x3FFFC5F0 /* 0 - 24 MHz & 1 - 27 MHz */
+#define HYDRA_CRYSTAL_CAP                   0x3FFFEDA4 /* 0 - 24 MHz & 1 - 27 MHz */
+
+#define HYDRA_CPU_RESET_REG                 0x8003003C
+#define HYDRA_CPU_RESET_DATA                0x00000400
+
+#define HYDRA_RESET_TRANSPORT_FIFO_REG      0x80030028
+#define HYDRA_RESET_TRANSPORT_FIFO_DATA     0x00000000
+
+#define HYDRA_RESET_BBAND_REG               0x80030024
+#define HYDRA_RESET_BBAND_DATA              0x00000000
+
+#define HYDRA_RESET_XBAR_REG                0x80030020
+#define HYDRA_RESET_XBAR_DATA               0x00000000
+
+#define HYDRA_MODULES_CLK_1_REG             0x80030014
+#define HYDRA_DISABLE_CLK_1                 0x00000000
+
+#define HYDRA_MODULES_CLK_2_REG             0x8003001C
+#define HYDRA_DISABLE_CLK_2                 0x0000000B
+
+#define HYDRA_PRCM_ROOT_CLK_REG             0x80030018
+#define HYDRA_PRCM_ROOT_CLK_DISABLE         0x00000000
+
+#define HYDRA_CPU_RESET_CHECK_REG           0x80030008
+#define HYDRA_CPU_RESET_CHECK_OFFSET        0x40000000  /* <bit 30> */
+
+#define HYDRA_SKU_ID_REG                    0x90000190
+
+#define FW_DL_SIGN_ADDR                     0x3FFFEAE0
+
+/* Register to check if FW is running or not */
+#define HYDRA_HEAR_BEAT                     0x3FFFEDDC
+
+/* Firmware version */
+#define HYDRA_FIRMWARE_VERSION              0x3FFFEDB8
+#define HYDRA_FW_RC_VERSION                 0x3FFFCFAC
+
+/* Firmware patch version */
+#define HYDRA_FIRMWARE_PATCH_VERSION        0x3FFFEDC2
+
+/* SOC operating temperature in C */
+#define HYDRA_TEMPARATURE                   0x3FFFEDB4
+
+/* Demod & Tuner status registers */
+/* Demod 0 status base address */
+#define HYDRA_DEMOD_0_BASE_ADDR             0x3FFFC64C
+
+/* Tuner 0 status base address */
+#define HYDRA_TUNER_0_BASE_ADDR             0x3FFFCE4C
+
+#define POWER_FROM_ADCRSSI_READBACK         0x3FFFEB6C
+
+/* Macros to determine base address of respective demod or tuner */
+#define HYDRA_DMD_STATUS_OFFSET(demodID)        ((demodID) * 0x100)
+#define HYDRA_TUNER_STATUS_OFFSET(tunerID)      ((tunerID) * 0x40)
+
+/* Demod status address offset from respective demod's base address */
+#define HYDRA_DMD_AGC_DIG_LEVEL_ADDR_OFFSET               0x3FFFC64C
+#define HYDRA_DMD_LOCK_STATUS_ADDR_OFFSET                 0x3FFFC650
+#define HYDRA_DMD_ACQ_STATUS_ADDR_OFFSET                  0x3FFFC654
+
+#define HYDRA_DMD_STANDARD_ADDR_OFFSET                    0x3FFFC658
+#define HYDRA_DMD_SPECTRUM_INVERSION_ADDR_OFFSET          0x3FFFC65C
+#define HYDRA_DMD_SPECTRUM_ROLL_OFF_ADDR_OFFSET           0x3FFFC660
+#define HYDRA_DMD_SYMBOL_RATE_ADDR_OFFSET                 0x3FFFC664
+#define HYDRA_DMD_MODULATION_SCHEME_ADDR_OFFSET           0x3FFFC668
+#define HYDRA_DMD_FEC_CODE_RATE_ADDR_OFFSET               0x3FFFC66C
+
+#define HYDRA_DMD_SNR_ADDR_OFFSET                         0x3FFFC670
+#define HYDRA_DMD_FREQ_OFFSET_ADDR_OFFSET                 0x3FFFC674
+#define HYDRA_DMD_CTL_FREQ_OFFSET_ADDR_OFFSET             0x3FFFC678
+#define HYDRA_DMD_STR_FREQ_OFFSET_ADDR_OFFSET             0x3FFFC67C
+#define HYDRA_DMD_FTL_FREQ_OFFSET_ADDR_OFFSET             0x3FFFC680
+#define HYDRA_DMD_STR_NBC_SYNC_LOCK_ADDR_OFFSET           0x3FFFC684
+#define HYDRA_DMD_CYCLE_SLIP_COUNT_ADDR_OFFSET            0x3FFFC688
+
+#define HYDRA_DMD_DISPLAY_I_ADDR_OFFSET                   0x3FFFC68C
+#define HYDRA_DMD_DISPLAY_Q_ADDR_OFFSET                   0x3FFFC68E
+
+#define HYDRA_DMD_DVBS2_CRC_ERRORS_ADDR_OFFSET            0x3FFFC690
+#define HYDRA_DMD_DVBS2_PER_COUNT_ADDR_OFFSET             0x3FFFC694
+#define HYDRA_DMD_DVBS2_PER_WINDOW_ADDR_OFFSET            0x3FFFC698
+
+#define HYDRA_DMD_DVBS_CORR_RS_ERRORS_ADDR_OFFSET         0x3FFFC69C
+#define HYDRA_DMD_DVBS_UNCORR_RS_ERRORS_ADDR_OFFSET       0x3FFFC6A0
+#define HYDRA_DMD_DVBS_BER_COUNT_ADDR_OFFSET              0x3FFFC6A4
+#define HYDRA_DMD_DVBS_BER_WINDOW_ADDR_OFFSET             0x3FFFC6A8
+
+/* Debug-purpose DVB-S DMD 0 */
+#define HYDRA_DMD_DVBS_1ST_CORR_RS_ERRORS_ADDR_OFFSET     0x3FFFC6C8  /* corrected RS Errors: 1st iteration */
+#define HYDRA_DMD_DVBS_1ST_UNCORR_RS_ERRORS_ADDR_OFFSET   0x3FFFC6CC  /* uncorrected RS Errors: 1st iteration */
+#define HYDRA_DMD_DVBS_BER_COUNT_1ST_ADDR_OFFSET          0x3FFFC6D0
+#define HYDRA_DMD_DVBS_BER_WINDOW_1ST_ADDR_OFFSET         0x3FFFC6D4
+
+#define HYDRA_DMD_TUNER_ID_ADDR_OFFSET                    0x3FFFC6AC
+#define HYDRA_DMD_DVBS2_PILOT_ON_OFF_ADDR_OFFSET          0x3FFFC6B0
+#define HYDRA_DMD_FREQ_SEARCH_RANGE_KHZ_ADDR_OFFSET       0x3FFFC6B4
+#define HYDRA_DMD_STATUS_LOCK_ADDR_OFFSET                 0x3FFFC6B8
+#define HYDRA_DMD_STATUS_CENTER_FREQ_IN_KHZ_ADDR          0x3FFFC704
+#define HYDRA_DMD_STATUS_INPUT_POWER_ADDR                 0x3FFFC708
+
+/* DVB-S new scaled_BER_count for a new BER API, see HYDRA-1343 "DVB-S post viterbi information" */
+#define DMD0_STATUS_DVBS_1ST_SCALED_BER_COUNT_ADDR        0x3FFFC710 /* DMD 0: 1st iteration BER count scaled by HYDRA_BER_COUNT_SCALING_FACTOR */
+#define DMD0_STATUS_DVBS_SCALED_BER_COUNT_ADDR            0x3FFFC714 /* DMD 0: 2nd iteration BER count scaled by HYDRA_BER_COUNT_SCALING_FACTOR */
+
+#define DMD0_SPECTRUM_MIN_GAIN_STATUS                     0x3FFFC73C
+#define DMD0_SPECTRUM_MIN_GAIN_WB_SAGC_VALUE              0x3FFFC740
+#define DMD0_SPECTRUM_MIN_GAIN_NB_SAGC_VALUE              0x3FFFC744
+
+#define HYDRA_DMD_STATUS_END_ADDR_OFFSET                  0x3FFFC748
+
+/* Tuner status address offset from respective tuners's base address */
+#define HYDRA_TUNER_DEMOD_ID_ADDR_OFFSET                  0x3FFFCE4C
+#define HYDRA_TUNER_AGC_LOCK_OFFSET                       0x3FFFCE50
+#define HYDRA_TUNER_SPECTRUM_STATUS_OFFSET                0x3FFFCE54
+#define HYDRA_TUNER_SPECTRUM_BIN_SIZE_OFFSET              0x3FFFCE58
+#define HYDRA_TUNER_SPECTRUM_ADDRESS_OFFSET               0x3FFFCE5C
+#define HYDRA_TUNER_ENABLE_COMPLETE                       0x3FFFEB78
+
+#define HYDRA_DEMOD_STATUS_LOCK(devId, demodId)   write_register(devId, (HYDRA_DMD_STATUS_LOCK_ADDR_OFFSET + HYDRA_DMD_STATUS_OFFSET(demodId)), MXL_YES)
+#define HYDRA_DEMOD_STATUS_UNLOCK(devId, demodId) write_register(devId, (HYDRA_DMD_STATUS_LOCK_ADDR_OFFSET + HYDRA_DMD_STATUS_OFFSET(demodId)), MXL_NO)
+
+#define HYDRA_VERSION                                     0x3FFFEDB8
+#define HYDRA_DEMOD0_VERSION                              0x3FFFEDBC
+#define HYDRA_DEMOD1_VERSION                              0x3FFFEDC0
+#define HYDRA_DEMOD2_VERSION                              0x3FFFEDC4
+#define HYDRA_DEMOD3_VERSION                              0x3FFFEDC8
+#define HYDRA_DEMOD4_VERSION                              0x3FFFEDCC
+#define HYDRA_DEMOD5_VERSION                              0x3FFFEDD0
+#define HYDRA_DEMOD6_VERSION                              0x3FFFEDD4
+#define HYDRA_DEMOD7_VERSION                              0x3FFFEDD8
+#define HYDRA_HEAR_BEAT                                   0x3FFFEDDC
+#define HYDRA_SKU_MGMT                                    0x3FFFEBC0
+
+#define MXL_HYDRA_FPGA_A_ADDRESS                          0x91C00000
+#define MXL_HYDRA_FPGA_B_ADDRESS                          0x91D00000
+
+/* TS control base address */
+#define HYDRA_TS_CTRL_BASE_ADDR                           0x90700000
+
+#define MPEG_MUX_MODE_SLICE0_REG            (HYDRA_TS_CTRL_BASE_ADDR + 0x08)
+
+#define MPEG_MUX_MODE_SLICE1_REG            (HYDRA_TS_CTRL_BASE_ADDR + 0x08)
+
+#define PID_BANK_SEL_SLICE0_REG             (HYDRA_TS_CTRL_BASE_ADDR + 0x190)
+#define PID_BANK_SEL_SLICE1_REG             (HYDRA_TS_CTRL_BASE_ADDR + 0x1B0)
+
+#define MPEG_CLK_GATED_REG                  (HYDRA_TS_CTRL_BASE_ADDR + 0x20)
+
+#define MPEG_CLK_ALWAYS_ON_REG              (HYDRA_TS_CTRL_BASE_ADDR + 0x1D4)
+
+#define HYDRA_REGULAR_PID_BANK_A_REG        (HYDRA_TS_CTRL_BASE_ADDR + 0x190)
+
+#define HYDRA_FIXED_PID_BANK_A_REG          (HYDRA_TS_CTRL_BASE_ADDR + 0x190)
+
+#define HYDRA_REGULAR_PID_BANK_B_REG        (HYDRA_TS_CTRL_BASE_ADDR + 0x1B0)
+
+#define HYDRA_FIXED_PID_BANK_B_REG          (HYDRA_TS_CTRL_BASE_ADDR + 0x1B0)
+
+#define FIXED_PID_TBL_REG_ADDRESS_0         (HYDRA_TS_CTRL_BASE_ADDR + 0x9000)
+#define FIXED_PID_TBL_REG_ADDRESS_1         (HYDRA_TS_CTRL_BASE_ADDR + 0x9100)
+#define FIXED_PID_TBL_REG_ADDRESS_2         (HYDRA_TS_CTRL_BASE_ADDR + 0x9200)
+#define FIXED_PID_TBL_REG_ADDRESS_3         (HYDRA_TS_CTRL_BASE_ADDR + 0x9300)
+
+#define FIXED_PID_TBL_REG_ADDRESS_4         (HYDRA_TS_CTRL_BASE_ADDR + 0xB000)
+#define FIXED_PID_TBL_REG_ADDRESS_5         (HYDRA_TS_CTRL_BASE_ADDR + 0xB100)
+#define FIXED_PID_TBL_REG_ADDRESS_6         (HYDRA_TS_CTRL_BASE_ADDR + 0xB200)
+#define FIXED_PID_TBL_REG_ADDRESS_7         (HYDRA_TS_CTRL_BASE_ADDR + 0xB300)
+
+#define REGULAR_PID_TBL_REG_ADDRESS_0       (HYDRA_TS_CTRL_BASE_ADDR + 0x8000)
+#define REGULAR_PID_TBL_REG_ADDRESS_1       (HYDRA_TS_CTRL_BASE_ADDR + 0x8200)
+#define REGULAR_PID_TBL_REG_ADDRESS_2       (HYDRA_TS_CTRL_BASE_ADDR + 0x8400)
+#define REGULAR_PID_TBL_REG_ADDRESS_3       (HYDRA_TS_CTRL_BASE_ADDR + 0x8600)
+
+#define REGULAR_PID_TBL_REG_ADDRESS_4       (HYDRA_TS_CTRL_BASE_ADDR + 0xA000)
+#define REGULAR_PID_TBL_REG_ADDRESS_5       (HYDRA_TS_CTRL_BASE_ADDR + 0xA200)
+#define REGULAR_PID_TBL_REG_ADDRESS_6       (HYDRA_TS_CTRL_BASE_ADDR + 0xA400)
+#define REGULAR_PID_TBL_REG_ADDRESS_7       (HYDRA_TS_CTRL_BASE_ADDR + 0xA600)
+
+/***************************************************************************/
+
+#define PAD_MUX_GPIO_00_SYNC_BASEADDR                          0x90000188
+
+
+#define PAD_MUX_UART_RX_C_PINMUX_BASEADDR 0x9000001C
+
+#define   XPT_PACKET_GAP_MIN_BASEADDR                            0x90700044
+#define   XPT_NCO_COUNT_BASEADDR                                 0x90700238
+
+#define   XPT_NCO_COUNT_BASEADDR1                                0x9070023C
+
+/* V2 DigRF status register */
+
+#define   XPT_PID_BASEADDR                                       0x90708000
+
+#define   XPT_PID_REMAP_BASEADDR                                 0x90708004
+
+#define   XPT_KNOWN_PID_BASEADDR                                 0x90709000
+
+#define   XPT_PID_BASEADDR1                                      0x9070A000
+
+#define   XPT_PID_REMAP_BASEADDR1                                0x9070A004
+
+#define   XPT_KNOWN_PID_BASEADDR1                                0x9070B000
+
+#define   XPT_BERT_LOCK_BASEADDR                                 0x907000B8
+
+#define   XPT_BERT_BASEADDR                                      0x907000BC
+
+#define   XPT_BERT_INVERT_BASEADDR                               0x907000C0
+
+#define   XPT_BERT_HEADER_BASEADDR                               0x907000C4
+
+#define   XPT_BERT_BASEADDR1                                     0x907000C8
+
+#define   XPT_BERT_BIT_COUNT0_BASEADDR                           0x907000CC
+
+#define   XPT_BERT_BIT_COUNT0_BASEADDR1                          0x907000D0
+
+#define   XPT_BERT_BIT_COUNT1_BASEADDR                           0x907000D4
+
+#define   XPT_BERT_BIT_COUNT1_BASEADDR1                          0x907000D8
+
+#define   XPT_BERT_BIT_COUNT2_BASEADDR                           0x907000DC
+
+#define   XPT_BERT_BIT_COUNT2_BASEADDR1                          0x907000E0
+
+#define   XPT_BERT_BIT_COUNT3_BASEADDR                           0x907000E4
+
+#define   XPT_BERT_BIT_COUNT3_BASEADDR1                          0x907000E8
+
+#define   XPT_BERT_BIT_COUNT4_BASEADDR                           0x907000EC
+
+#define   XPT_BERT_BIT_COUNT4_BASEADDR1                          0x907000F0
+
+#define   XPT_BERT_BIT_COUNT5_BASEADDR                           0x907000F4
+
+#define   XPT_BERT_BIT_COUNT5_BASEADDR1                          0x907000F8
+
+#define   XPT_BERT_BIT_COUNT6_BASEADDR                           0x907000FC
+
+#define   XPT_BERT_BIT_COUNT6_BASEADDR1                          0x90700100
+
+#define   XPT_BERT_BIT_COUNT7_BASEADDR                           0x90700104
+
+#define   XPT_BERT_BIT_COUNT7_BASEADDR1                          0x90700108
+
+#define   XPT_BERT_ERR_COUNT0_BASEADDR                           0x9070010C
+
+#define   XPT_BERT_ERR_COUNT0_BASEADDR1                          0x90700110
+
+#define   XPT_BERT_ERR_COUNT1_BASEADDR                           0x90700114
+
+#define   XPT_BERT_ERR_COUNT1_BASEADDR1                          0x90700118
+
+#define   XPT_BERT_ERR_COUNT2_BASEADDR                           0x9070011C
+
+#define   XPT_BERT_ERR_COUNT2_BASEADDR1                          0x90700120
+
+#define   XPT_BERT_ERR_COUNT3_BASEADDR                           0x90700124
+
+#define   XPT_BERT_ERR_COUNT3_BASEADDR1                          0x90700128
+
+#define   XPT_BERT_ERR_COUNT4_BASEADDR                           0x9070012C
+
+#define   XPT_BERT_ERR_COUNT4_BASEADDR1                          0x90700130
+
+#define   XPT_BERT_ERR_COUNT5_BASEADDR                           0x90700134
+
+#define   XPT_BERT_ERR_COUNT5_BASEADDR1                          0x90700138
+
+#define   XPT_BERT_ERR_COUNT6_BASEADDR                           0x9070013C
+
+#define   XPT_BERT_ERR_COUNT6_BASEADDR1                          0x90700140
+
+#define   XPT_BERT_ERR_COUNT7_BASEADDR                           0x90700144
+
+#define   XPT_BERT_ERR_COUNT7_BASEADDR1                          0x90700148
+
+#define   XPT_BERT_ERROR_BASEADDR                                0x9070014C
+
+#define   XPT_BERT_ANALYZER_BASEADDR                             0x90700150
+
+#define   XPT_BERT_ANALYZER_BASEADDR1                            0x90700154
+
+#define   XPT_BERT_ANALYZER_BASEADDR2                            0x90700158
+
+#define   XPT_BERT_ANALYZER_BASEADDR3                            0x9070015C
+
+#define   XPT_BERT_ANALYZER_BASEADDR4                            0x90700160
+
+#define   XPT_BERT_ANALYZER_BASEADDR5                            0x90700164
+
+#define   XPT_BERT_ANALYZER_BASEADDR6                            0x90700168
+
+#define   XPT_BERT_ANALYZER_BASEADDR7                            0x9070016C
+
+#define   XPT_BERT_ANALYZER_BASEADDR8                            0x90700170
+
+#define   XPT_BERT_ANALYZER_BASEADDR9                            0x90700174
+
+#define   XPT_DMD0_BASEADDR                                      0x9070024C
+
+/* V2 AGC Gain Freeze & step */
+#define   DBG_ENABLE_DISABLE_AGC                                 (0x3FFFCF60) /* 1: DISABLE, 0:ENABLE */
+#define   WB_DFE0_DFE_FB_RF1_BASEADDR                            0x903004A4
+
+#define   WB_DFE1_DFE_FB_RF1_BASEADDR                            0x904004A4
+
+#define   WB_DFE2_DFE_FB_RF1_BASEADDR                            0x905004A4
+
+#define   WB_DFE3_DFE_FB_RF1_BASEADDR                            0x906004A4
+
+#define   AFE_REG_D2A_TA_RFFE_LNA_BO_1P8_BASEADDR                0x90200104
+
+#define   AFE_REG_AFE_REG_SPARE_BASEADDR                         0x902000A0
+
+#define   AFE_REG_AFE_REG_SPARE_BASEADDR1                        0x902000B4
+
+#define   AFE_REG_AFE_REG_SPARE_BASEADDR2                        0x902000C4
+
+#define   AFE_REG_AFE_REG_SPARE_BASEADDR3                        0x902000D4
+
+#define   WB_DFE0_DFE_FB_AGC_BASEADDR                            0x90300498
+
+#define   WB_DFE1_DFE_FB_AGC_BASEADDR                            0x90400498
+
+#define   WB_DFE2_DFE_FB_AGC_BASEADDR                            0x90500498
+
+#define   WB_DFE3_DFE_FB_AGC_BASEADDR                            0x90600498
+
+#define   WDT_WD_INT_BASEADDR                                    0x8002000C
+
+#define   FSK_TX_FTM_BASEADDR                                    0x80090000
+
+#define   FSK_TX_FTM_TX_CNT_BASEADDR                             0x80090018
+
+#define   AFE_REG_D2A_FSK_BIAS_BASEADDR                          0x90200040
+
+#define   DMD_TEI_BASEADDR                                       0x3FFFEBE0
+
+#endif /* __MXL58X_REGISTERS_H__ */
diff --git a/drivers/media/dvb-frontends/s5h1420.c b/drivers/media/dvb-frontends/s5h1420.c
index cba9bff05b12..fd427a29c001 100644
--- a/drivers/media/dvb-frontends/s5h1420.c
+++ b/drivers/media/dvb-frontends/s5h1420.c
@@ -864,7 +864,7 @@ static int s5h1420_tuner_i2c_tuner_xfer(struct i2c_adapter *i2c_adap, struct i2c
 	return i2c_transfer(state->i2c, m, 1 + num) == 1 + num ? num : -EIO;
 }
 
-static struct i2c_algorithm s5h1420_tuner_i2c_algo = {
+static const struct i2c_algorithm s5h1420_tuner_i2c_algo = {
 	.master_xfer   = s5h1420_tuner_i2c_tuner_xfer,
 	.functionality = s5h1420_tuner_i2c_func,
 };
diff --git a/drivers/media/dvb-frontends/stv0367.c b/drivers/media/dvb-frontends/stv0367.c
index 8ac0f598978d..f3529df8211d 100644
--- a/drivers/media/dvb-frontends/stv0367.c
+++ b/drivers/media/dvb-frontends/stv0367.c
@@ -2149,6 +2149,71 @@ static u32 stv0367cab_GetSymbolRate(struct stv0367_state *state, u32 mclk_hz)
 	return regsym;
 }
 
+static u32 stv0367cab_fsm_status(struct stv0367_state *state)
+{
+	return stv0367_readbits(state, F367CAB_FSM_STATUS);
+}
+
+static u32 stv0367cab_qamfec_lock(struct stv0367_state *state)
+{
+	return stv0367_readbits(state,
+		(state->cab_state->qamfec_status_reg ?
+		 state->cab_state->qamfec_status_reg :
+		 F367CAB_QAMFEC_LOCK));
+}
+
+static
+enum stv0367_cab_signal_type stv0367cab_fsm_signaltype(u32 qam_fsm_status)
+{
+	enum stv0367_cab_signal_type signaltype = FE_CAB_NOAGC;
+
+	switch (qam_fsm_status) {
+	case 1:
+		signaltype = FE_CAB_NOAGC;
+		break;
+	case 2:
+		signaltype = FE_CAB_NOTIMING;
+		break;
+	case 3:
+		signaltype = FE_CAB_TIMINGOK;
+		break;
+	case 4:
+		signaltype = FE_CAB_NOCARRIER;
+		break;
+	case 5:
+		signaltype = FE_CAB_CARRIEROK;
+		break;
+	case 7:
+		signaltype = FE_CAB_NOBLIND;
+		break;
+	case 8:
+		signaltype = FE_CAB_BLINDOK;
+		break;
+	case 10:
+		signaltype = FE_CAB_NODEMOD;
+		break;
+	case 11:
+		signaltype = FE_CAB_DEMODOK;
+		break;
+	case 12:
+		signaltype = FE_CAB_DEMODOK;
+		break;
+	case 13:
+		signaltype = FE_CAB_NODEMOD;
+		break;
+	case 14:
+		signaltype = FE_CAB_NOBLIND;
+		break;
+	case 15:
+		signaltype = FE_CAB_NOSIGNAL;
+		break;
+	default:
+		break;
+	}
+
+	return signaltype;
+}
+
 static int stv0367cab_read_status(struct dvb_frontend *fe,
 				  enum fe_status *status)
 {
@@ -2158,22 +2223,26 @@ static int stv0367cab_read_status(struct dvb_frontend *fe,
 
 	*status = 0;
 
-	if (state->cab_state->state > FE_CAB_NOSIGNAL)
-		*status |= FE_HAS_SIGNAL;
+	/* update cab_state->state from QAM_FSM_STATUS */
+	state->cab_state->state = stv0367cab_fsm_signaltype(
+		stv0367cab_fsm_status(state));
 
-	if (state->cab_state->state > FE_CAB_NOCARRIER)
-		*status |= FE_HAS_CARRIER;
+	if (stv0367cab_qamfec_lock(state)) {
+		*status = FE_HAS_SIGNAL | FE_HAS_CARRIER | FE_HAS_VITERBI
+			  | FE_HAS_SYNC | FE_HAS_LOCK;
+		dprintk("%s: stv0367 has locked\n", __func__);
+	} else {
+		if (state->cab_state->state > FE_CAB_NOSIGNAL)
+			*status |= FE_HAS_SIGNAL;
 
-	if (state->cab_state->state >= FE_CAB_DEMODOK)
-		*status |= FE_HAS_VITERBI;
+		if (state->cab_state->state > FE_CAB_NOCARRIER)
+			*status |= FE_HAS_CARRIER;
 
-	if (state->cab_state->state >= FE_CAB_DATAOK)
-		*status |= FE_HAS_SYNC;
+		if (state->cab_state->state >= FE_CAB_DEMODOK)
+			*status |= FE_HAS_VITERBI;
 
-	if (stv0367_readbits(state, (state->cab_state->qamfec_status_reg ?
-		state->cab_state->qamfec_status_reg : F367CAB_QAMFEC_LOCK))) {
-		*status |= FE_HAS_LOCK;
-		dprintk("%s: stv0367 has locked\n", __func__);
+		if (state->cab_state->state >= FE_CAB_DATAOK)
+			*status |= FE_HAS_SYNC;
 	}
 
 	return 0;
@@ -2374,7 +2443,7 @@ enum stv0367_cab_signal_type stv0367cab_algo(struct stv0367_state *state,
 	LockTime = 0;
 	stv0367_writereg(state, R367CAB_CTRL_1, 0x00);
 	do {
-		QAM_Lock = stv0367_readbits(state, F367CAB_FSM_STATUS);
+		QAM_Lock = stv0367cab_fsm_status(state);
 		if ((LockTime >= (DemodTimeOut - EQLTimeOut)) &&
 							(QAM_Lock == 0x04))
 			/*
@@ -2435,10 +2504,7 @@ enum stv0367_cab_signal_type stv0367cab_algo(struct stv0367_state *state,
 		do {
 			usleep_range(5000, 7000);
 			LockTime += 5;
-			QAMFEC_Lock = stv0367_readbits(state,
-				(state->cab_state->qamfec_status_reg ?
-				state->cab_state->qamfec_status_reg :
-				F367CAB_QAMFEC_LOCK));
+			QAMFEC_Lock = stv0367cab_qamfec_lock(state);
 		} while (!QAMFEC_Lock && (LockTime < FECTimeOut));
 	} else
 		QAMFEC_Lock = 0;
@@ -2474,52 +2540,8 @@ enum stv0367_cab_signal_type stv0367cab_algo(struct stv0367_state *state,
 		cab_state->locked = 1;
 
 		/* stv0367_setbits(state, F367CAB_AGC_ACCUMRSTSEL,7);*/
-	} else {
-		switch (QAM_Lock) {
-		case 1:
-			signalType = FE_CAB_NOAGC;
-			break;
-		case 2:
-			signalType = FE_CAB_NOTIMING;
-			break;
-		case 3:
-			signalType = FE_CAB_TIMINGOK;
-			break;
-		case 4:
-			signalType = FE_CAB_NOCARRIER;
-			break;
-		case 5:
-			signalType = FE_CAB_CARRIEROK;
-			break;
-		case 7:
-			signalType = FE_CAB_NOBLIND;
-			break;
-		case 8:
-			signalType = FE_CAB_BLINDOK;
-			break;
-		case 10:
-			signalType = FE_CAB_NODEMOD;
-			break;
-		case 11:
-			signalType = FE_CAB_DEMODOK;
-			break;
-		case 12:
-			signalType = FE_CAB_DEMODOK;
-			break;
-		case 13:
-			signalType = FE_CAB_NODEMOD;
-			break;
-		case 14:
-			signalType = FE_CAB_NOBLIND;
-			break;
-		case 15:
-			signalType = FE_CAB_NOSIGNAL;
-			break;
-		default:
-			break;
-		}
-
-	}
+	} else
+		signalType = stv0367cab_fsm_signaltype(QAM_Lock);
 
 	/* Set the AGC control values to tracking values */
 	stv0367_writebits(state, F367CAB_AGC_ACCUMRSTSEL, TrackAGCAccum);
@@ -3090,7 +3112,7 @@ static int stv0367ddb_read_status(struct dvb_frontend *fe,
 {
 	struct stv0367_state *state = fe->demodulator_priv;
 	struct dtv_frontend_properties *p = &fe->dtv_property_cache;
-	int ret;
+	int ret = 0;
 
 	switch (state->activedemod) {
 	case demod_ter:
@@ -3100,7 +3122,7 @@ static int stv0367ddb_read_status(struct dvb_frontend *fe,
 		ret = stv0367cab_read_status(fe, status);
 		break;
 	default:
-		return 0;
+		break;
 	}
 
 	/* stop and report on *_read_status failure */
@@ -3138,7 +3160,7 @@ static int stv0367ddb_get_frontend(struct dvb_frontend *fe,
 		break;
 	}
 
-	return -EINVAL;
+	return 0;
 }
 
 static int stv0367ddb_sleep(struct dvb_frontend *fe)
@@ -3261,7 +3283,7 @@ static const struct dvb_frontend_ops stv0367ddb_ops = {
 			0x400 |/* FE_CAN_QAM_4 */
 			FE_CAN_QAM_16 | FE_CAN_QAM_32  |
 			FE_CAN_QAM_64 | FE_CAN_QAM_128 |
-			FE_CAN_QAM_256 | FE_CAN_QAM_AUTO |
+			FE_CAN_QAM_256 |
 			/* DVB-T */
 			FE_CAN_FEC_1_2 | FE_CAN_FEC_2_3 | FE_CAN_FEC_3_4 |
 			FE_CAN_FEC_5_6 | FE_CAN_FEC_7_8 | FE_CAN_FEC_AUTO |
diff --git a/drivers/media/dvb-frontends/stv0910.c b/drivers/media/dvb-frontends/stv0910.c
new file mode 100644
index 000000000000..8bf855c301f5
--- /dev/null
+++ b/drivers/media/dvb-frontends/stv0910.c
@@ -0,0 +1,1813 @@
+/*
+ * Driver for the ST STV0910 DVB-S/S2 demodulator.
+ *
+ * Copyright (C) 2014-2015 Ralph Metzler <rjkm@metzlerbros.de>
+ *                         Marcus Metzler <mocm@metzlerbros.de>
+ *                         developed for Digital Devices GmbH
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 only, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/init.h>
+#include <linux/delay.h>
+#include <linux/firmware.h>
+#include <linux/i2c.h>
+#include <asm/div64.h>
+
+#include "dvb_math.h"
+#include "dvb_frontend.h"
+#include "stv0910.h"
+#include "stv0910_regs.h"
+
+#define EXT_CLOCK    30000000
+#define TUNING_DELAY 200
+#define BER_SRC_S    0x20
+#define BER_SRC_S2   0x20
+
+static LIST_HEAD(stvlist);
+
+enum receive_mode { RCVMODE_NONE, RCVMODE_DVBS, RCVMODE_DVBS2, RCVMODE_AUTO };
+
+enum dvbs2_fectype { DVBS2_64K, DVBS2_16K };
+
+enum dvbs2_mod_cod {
+	DVBS2_DUMMY_PLF, DVBS2_QPSK_1_4, DVBS2_QPSK_1_3, DVBS2_QPSK_2_5,
+	DVBS2_QPSK_1_2, DVBS2_QPSK_3_5, DVBS2_QPSK_2_3,	DVBS2_QPSK_3_4,
+	DVBS2_QPSK_4_5,	DVBS2_QPSK_5_6,	DVBS2_QPSK_8_9,	DVBS2_QPSK_9_10,
+	DVBS2_8PSK_3_5,	DVBS2_8PSK_2_3,	DVBS2_8PSK_3_4,	DVBS2_8PSK_5_6,
+	DVBS2_8PSK_8_9,	DVBS2_8PSK_9_10, DVBS2_16APSK_2_3, DVBS2_16APSK_3_4,
+	DVBS2_16APSK_4_5, DVBS2_16APSK_5_6, DVBS2_16APSK_8_9, DVBS2_16APSK_9_10,
+	DVBS2_32APSK_3_4, DVBS2_32APSK_4_5, DVBS2_32APSK_5_6, DVBS2_32APSK_8_9,
+	DVBS2_32APSK_9_10
+};
+
+enum fe_stv0910_mod_cod {
+	FE_DUMMY_PLF, FE_QPSK_14, FE_QPSK_13, FE_QPSK_25,
+	FE_QPSK_12, FE_QPSK_35, FE_QPSK_23, FE_QPSK_34,
+	FE_QPSK_45, FE_QPSK_56, FE_QPSK_89, FE_QPSK_910,
+	FE_8PSK_35, FE_8PSK_23, FE_8PSK_34, FE_8PSK_56,
+	FE_8PSK_89, FE_8PSK_910, FE_16APSK_23, FE_16APSK_34,
+	FE_16APSK_45, FE_16APSK_56, FE_16APSK_89, FE_16APSK_910,
+	FE_32APSK_34, FE_32APSK_45, FE_32APSK_56, FE_32APSK_89,
+	FE_32APSK_910
+};
+
+enum fe_stv0910_roll_off { FE_SAT_35, FE_SAT_25, FE_SAT_20, FE_SAT_15 };
+
+static inline u32 muldiv32(u32 a, u32 b, u32 c)
+{
+	u64 tmp64;
+
+	tmp64 = (u64)a * (u64)b;
+	do_div(tmp64, c);
+
+	return (u32)tmp64;
+}
+
+struct stv_base {
+	struct list_head     stvlist;
+
+	u8                   adr;
+	struct i2c_adapter  *i2c;
+	struct mutex         i2c_lock; /* shared I2C access protect */
+	struct mutex         reg_lock; /* shared register write protect */
+	int                  count;
+
+	u32                  extclk;
+	u32                  mclk;
+};
+
+struct stv {
+	struct stv_base     *base;
+	struct dvb_frontend  fe;
+	int                  nr;
+	u16                  regoff;
+	u8                   i2crpt;
+	u8                   tscfgh;
+	u8                   tsgeneral;
+	u8                   tsspeed;
+	u8                   single;
+	unsigned long        tune_time;
+
+	s32                  search_range;
+	u32                  started;
+	u32                  demod_lock_time;
+	enum receive_mode    receive_mode;
+	u32                  demod_timeout;
+	u32                  fec_timeout;
+	u32                  first_time_lock;
+	u8                   demod_bits;
+	u32                  symbol_rate;
+
+	u8                       last_viterbi_rate;
+	enum fe_code_rate        puncture_rate;
+	enum fe_stv0910_mod_cod  mod_cod;
+	enum dvbs2_fectype       fectype;
+	u32                      pilots;
+	enum fe_stv0910_roll_off feroll_off;
+
+	int   is_standard_broadcast;
+	int   is_vcm;
+
+	u32   cur_scrambling_code;
+
+	u32   last_bernumerator;
+	u32   last_berdenominator;
+	u8    berscale;
+
+	u8    vth[6];
+};
+
+struct sinit_table {
+	u16  address;
+	u8   data;
+};
+
+struct slookup {
+	s16  value;
+	u32  reg_value;
+};
+
+static inline int i2c_write(struct i2c_adapter *adap, u8 adr,
+			    u8 *data, int len)
+{
+	struct i2c_msg msg = {.addr = adr, .flags = 0,
+			      .buf = data, .len = len};
+
+	if (i2c_transfer(adap, &msg, 1) != 1) {
+		dev_warn(&adap->dev, "i2c write error ([%02x] %04x: %02x)\n",
+			 adr, (data[0] << 8) | data[1],
+			 (len > 2 ? data[2] : 0));
+		return -EREMOTEIO;
+	}
+	return 0;
+}
+
+static int i2c_write_reg16(struct i2c_adapter *adap, u8 adr, u16 reg, u8 val)
+{
+	u8 msg[3] = {reg >> 8, reg & 0xff, val};
+
+	return i2c_write(adap, adr, msg, 3);
+}
+
+static int write_reg(struct stv *state, u16 reg, u8 val)
+{
+	return i2c_write_reg16(state->base->i2c, state->base->adr, reg, val);
+}
+
+static inline int i2c_read_regs16(struct i2c_adapter *adapter, u8 adr,
+				  u16 reg, u8 *val, int count)
+{
+	u8 msg[2] = {reg >> 8, reg & 0xff};
+	struct i2c_msg msgs[2] = {{.addr = adr, .flags = 0,
+				   .buf  = msg, .len   = 2},
+				  {.addr = adr, .flags = I2C_M_RD,
+				   .buf  = val, .len   = count } };
+
+	if (i2c_transfer(adapter, msgs, 2) != 2) {
+		dev_warn(&adapter->dev, "i2c read error ([%02x] %04x)\n",
+			 adr, reg);
+		return -EREMOTEIO;
+	}
+	return 0;
+}
+
+static int read_reg(struct stv *state, u16 reg, u8 *val)
+{
+	return i2c_read_regs16(state->base->i2c, state->base->adr,
+			       reg, val, 1);
+}
+
+static int read_regs(struct stv *state, u16 reg, u8 *val, int len)
+{
+	return i2c_read_regs16(state->base->i2c, state->base->adr,
+			       reg, val, len);
+}
+
+static int write_shared_reg(struct stv *state, u16 reg, u8 mask, u8 val)
+{
+	int status;
+	u8 tmp;
+
+	mutex_lock(&state->base->reg_lock);
+	status = read_reg(state, reg, &tmp);
+	if (!status)
+		status = write_reg(state, reg, (tmp & ~mask) | (val & mask));
+	mutex_unlock(&state->base->reg_lock);
+	return status;
+}
+
+static const struct slookup s1_sn_lookup[] = {
+	{   0,    9242  }, /* C/N=   0dB */
+	{   5,    9105  }, /* C/N= 0.5dB */
+	{  10,    8950  }, /* C/N= 1.0dB */
+	{  15,    8780  }, /* C/N= 1.5dB */
+	{  20,    8566  }, /* C/N= 2.0dB */
+	{  25,    8366  }, /* C/N= 2.5dB */
+	{  30,    8146  }, /* C/N= 3.0dB */
+	{  35,    7908  }, /* C/N= 3.5dB */
+	{  40,    7666  }, /* C/N= 4.0dB */
+	{  45,    7405  }, /* C/N= 4.5dB */
+	{  50,    7136  }, /* C/N= 5.0dB */
+	{  55,    6861  }, /* C/N= 5.5dB */
+	{  60,    6576  }, /* C/N= 6.0dB */
+	{  65,    6330  }, /* C/N= 6.5dB */
+	{  70,    6048  }, /* C/N= 7.0dB */
+	{  75,    5768  }, /* C/N= 7.5dB */
+	{  80,    5492  }, /* C/N= 8.0dB */
+	{  85,    5224  }, /* C/N= 8.5dB */
+	{  90,    4959  }, /* C/N= 9.0dB */
+	{  95,    4709  }, /* C/N= 9.5dB */
+	{  100,   4467  }, /* C/N=10.0dB */
+	{  105,   4236  }, /* C/N=10.5dB */
+	{  110,   4013  }, /* C/N=11.0dB */
+	{  115,   3800  }, /* C/N=11.5dB */
+	{  120,   3598  }, /* C/N=12.0dB */
+	{  125,   3406  }, /* C/N=12.5dB */
+	{  130,   3225  }, /* C/N=13.0dB */
+	{  135,   3052  }, /* C/N=13.5dB */
+	{  140,   2889  }, /* C/N=14.0dB */
+	{  145,   2733  }, /* C/N=14.5dB */
+	{  150,   2587  }, /* C/N=15.0dB */
+	{  160,   2318  }, /* C/N=16.0dB */
+	{  170,   2077  }, /* C/N=17.0dB */
+	{  180,   1862  }, /* C/N=18.0dB */
+	{  190,   1670  }, /* C/N=19.0dB */
+	{  200,   1499  }, /* C/N=20.0dB */
+	{  210,   1347  }, /* C/N=21.0dB */
+	{  220,   1213  }, /* C/N=22.0dB */
+	{  230,   1095  }, /* C/N=23.0dB */
+	{  240,    992  }, /* C/N=24.0dB */
+	{  250,    900  }, /* C/N=25.0dB */
+	{  260,    826  }, /* C/N=26.0dB */
+	{  270,    758  }, /* C/N=27.0dB */
+	{  280,    702  }, /* C/N=28.0dB */
+	{  290,    653  }, /* C/N=29.0dB */
+	{  300,    613  }, /* C/N=30.0dB */
+	{  310,    579  }, /* C/N=31.0dB */
+	{  320,    550  }, /* C/N=32.0dB */
+	{  330,    526  }, /* C/N=33.0dB */
+	{  350,    490  }, /* C/N=33.0dB */
+	{  400,    445  }, /* C/N=40.0dB */
+	{  450,    430  }, /* C/N=45.0dB */
+	{  500,    426  }, /* C/N=50.0dB */
+	{  510,    425  }  /* C/N=51.0dB */
+};
+
+static const struct slookup s2_sn_lookup[] = {
+	{  -30,  13950  }, /* C/N=-2.5dB */
+	{  -25,  13580  }, /* C/N=-2.5dB */
+	{  -20,  13150  }, /* C/N=-2.0dB */
+	{  -15,  12760  }, /* C/N=-1.5dB */
+	{  -10,  12345  }, /* C/N=-1.0dB */
+	{   -5,  11900  }, /* C/N=-0.5dB */
+	{    0,  11520  }, /* C/N=   0dB */
+	{    5,  11080  }, /* C/N= 0.5dB */
+	{   10,  10630  }, /* C/N= 1.0dB */
+	{   15,  10210  }, /* C/N= 1.5dB */
+	{   20,   9790  }, /* C/N= 2.0dB */
+	{   25,   9390  }, /* C/N= 2.5dB */
+	{   30,   8970  }, /* C/N= 3.0dB */
+	{   35,   8575  }, /* C/N= 3.5dB */
+	{   40,   8180  }, /* C/N= 4.0dB */
+	{   45,   7800  }, /* C/N= 4.5dB */
+	{   50,   7430  }, /* C/N= 5.0dB */
+	{   55,   7080  }, /* C/N= 5.5dB */
+	{   60,   6720  }, /* C/N= 6.0dB */
+	{   65,   6320  }, /* C/N= 6.5dB */
+	{   70,   6060  }, /* C/N= 7.0dB */
+	{   75,   5760  }, /* C/N= 7.5dB */
+	{   80,   5480  }, /* C/N= 8.0dB */
+	{   85,   5200  }, /* C/N= 8.5dB */
+	{   90,   4930  }, /* C/N= 9.0dB */
+	{   95,   4680  }, /* C/N= 9.5dB */
+	{  100,   4425  }, /* C/N=10.0dB */
+	{  105,   4210  }, /* C/N=10.5dB */
+	{  110,   3980  }, /* C/N=11.0dB */
+	{  115,   3765  }, /* C/N=11.5dB */
+	{  120,   3570  }, /* C/N=12.0dB */
+	{  125,   3315  }, /* C/N=12.5dB */
+	{  130,   3140  }, /* C/N=13.0dB */
+	{  135,   2980  }, /* C/N=13.5dB */
+	{  140,   2820  }, /* C/N=14.0dB */
+	{  145,   2670  }, /* C/N=14.5dB */
+	{  150,   2535  }, /* C/N=15.0dB */
+	{  160,   2270  }, /* C/N=16.0dB */
+	{  170,   2035  }, /* C/N=17.0dB */
+	{  180,   1825  }, /* C/N=18.0dB */
+	{  190,   1650  }, /* C/N=19.0dB */
+	{  200,   1485  }, /* C/N=20.0dB */
+	{  210,   1340  }, /* C/N=21.0dB */
+	{  220,   1212  }, /* C/N=22.0dB */
+	{  230,   1100  }, /* C/N=23.0dB */
+	{  240,   1000  }, /* C/N=24.0dB */
+	{  250,    910  }, /* C/N=25.0dB */
+	{  260,    836  }, /* C/N=26.0dB */
+	{  270,    772  }, /* C/N=27.0dB */
+	{  280,    718  }, /* C/N=28.0dB */
+	{  290,    671  }, /* C/N=29.0dB */
+	{  300,    635  }, /* C/N=30.0dB */
+	{  310,    602  }, /* C/N=31.0dB */
+	{  320,    575  }, /* C/N=32.0dB */
+	{  330,    550  }, /* C/N=33.0dB */
+	{  350,    517  }, /* C/N=35.0dB */
+	{  400,    480  }, /* C/N=40.0dB */
+	{  450,    466  }, /* C/N=45.0dB */
+	{  500,    464  }, /* C/N=50.0dB */
+	{  510,    463  }, /* C/N=51.0dB */
+};
+
+static const struct slookup padc_lookup[] = {
+	{    0,  118000 }, /* PADC= +0dBm */
+	{ -100,  93600  }, /* PADC= -1dBm */
+	{ -200,  74500  }, /* PADC= -2dBm */
+	{ -300,  59100  }, /* PADC= -3dBm */
+	{ -400,  47000  }, /* PADC= -4dBm */
+	{ -500,  37300  }, /* PADC= -5dBm */
+	{ -600,  29650  }, /* PADC= -6dBm */
+	{ -700,  23520  }, /* PADC= -7dBm */
+	{ -900,  14850  }, /* PADC= -9dBm */
+	{ -1100, 9380   }, /* PADC=-11dBm */
+	{ -1300, 5910   }, /* PADC=-13dBm */
+	{ -1500, 3730   }, /* PADC=-15dBm */
+	{ -1700, 2354   }, /* PADC=-17dBm */
+	{ -1900, 1485   }, /* PADC=-19dBm */
+	{ -2000, 1179   }, /* PADC=-20dBm */
+	{ -2100, 1000   }, /* PADC=-21dBm */
+};
+
+/*********************************************************************
+ * Tracking carrier loop carrier QPSK 1/4 to 8PSK 9/10 long Frame
+ *********************************************************************/
+static const u8 s2car_loop[] =	{
+	/*
+	 * Modcod  2MPon 2MPoff 5MPon 5MPoff 10MPon 10MPoff
+	 * 20MPon 20MPoff 30MPon 30MPoff
+	 */
+
+	/* FE_QPSK_14  */
+	0x0C,  0x3C,  0x0B,  0x3C,  0x2A,  0x2C,  0x2A,  0x1C,  0x3A,  0x3B,
+	/* FE_QPSK_13  */
+	0x0C,  0x3C,  0x0B,  0x3C,  0x2A,  0x2C,  0x3A,  0x0C,  0x3A,  0x2B,
+	/* FE_QPSK_25  */
+	0x1C,  0x3C,  0x1B,  0x3C,  0x3A,  0x1C,  0x3A,  0x3B,  0x3A,  0x2B,
+	/* FE_QPSK_12  */
+	0x0C,  0x1C,  0x2B,  0x1C,  0x0B,  0x2C,  0x0B,  0x0C,  0x2A,  0x2B,
+	/* FE_QPSK_35  */
+	0x1C,  0x1C,  0x2B,  0x1C,  0x0B,  0x2C,  0x0B,  0x0C,  0x2A,  0x2B,
+	/* FE_QPSK_23  */
+	0x2C,  0x2C,  0x2B,  0x1C,  0x0B,  0x2C,  0x0B,  0x0C,  0x2A,  0x2B,
+	/* FE_QPSK_34  */
+	0x3C,  0x2C,  0x3B,  0x2C,  0x1B,  0x1C,  0x1B,  0x3B,  0x3A,  0x1B,
+	/* FE_QPSK_45  */
+	0x0D,  0x3C,  0x3B,  0x2C,  0x1B,  0x1C,  0x1B,  0x3B,  0x3A,  0x1B,
+	/* FE_QPSK_56  */
+	0x1D,  0x3C,  0x0C,  0x2C,  0x2B,  0x1C,  0x1B,  0x3B,  0x0B,  0x1B,
+	/* FE_QPSK_89  */
+	0x3D,  0x0D,  0x0C,  0x2C,  0x2B,  0x0C,  0x2B,  0x2B,  0x0B,  0x0B,
+	/* FE_QPSK_910 */
+	0x1E,  0x0D,  0x1C,  0x2C,  0x3B,  0x0C,  0x2B,  0x2B,  0x1B,  0x0B,
+	/* FE_8PSK_35  */
+	0x28,  0x09,  0x28,  0x09,  0x28,  0x09,  0x28,  0x08,  0x28,  0x27,
+	/* FE_8PSK_23  */
+	0x19,  0x29,  0x19,  0x29,  0x19,  0x29,  0x38,  0x19,  0x28,  0x09,
+	/* FE_8PSK_34  */
+	0x1A,  0x0B,  0x1A,  0x3A,  0x0A,  0x2A,  0x39,  0x2A,  0x39,  0x1A,
+	/* FE_8PSK_56  */
+	0x2B,  0x2B,  0x1B,  0x1B,  0x0B,  0x1B,  0x1A,  0x0B,  0x1A,  0x1A,
+	/* FE_8PSK_89  */
+	0x0C,  0x0C,  0x3B,  0x3B,  0x1B,  0x1B,  0x2A,  0x0B,  0x2A,  0x2A,
+	/* FE_8PSK_910 */
+	0x0C,  0x1C,  0x0C,  0x3B,  0x2B,  0x1B,  0x3A,  0x0B,  0x2A,  0x2A,
+
+	/**********************************************************************
+	 * Tracking carrier loop carrier 16APSK 2/3 to 32APSK 9/10 long Frame
+	 **********************************************************************/
+
+	/*
+	 * Modcod 2MPon  2MPoff 5MPon 5MPoff 10MPon 10MPoff 20MPon
+	 * 20MPoff 30MPon 30MPoff
+	 */
+
+	/* FE_16APSK_23  */
+	0x0A,  0x0A,  0x0A,  0x0A,  0x1A,  0x0A,  0x39,  0x0A,  0x29,  0x0A,
+	/* FE_16APSK_34  */
+	0x0A,  0x0A,  0x0A,  0x0A,  0x0B,  0x0A,  0x2A,  0x0A,  0x1A,  0x0A,
+	/* FE_16APSK_45  */
+	0x0A,  0x0A,  0x0A,  0x0A,  0x1B,  0x0A,  0x3A,  0x0A,  0x2A,  0x0A,
+	/* FE_16APSK_56  */
+	0x0A,  0x0A,  0x0A,  0x0A,  0x1B,  0x0A,  0x3A,  0x0A,  0x2A,  0x0A,
+	/* FE_16APSK_89  */
+	0x0A,  0x0A,  0x0A,  0x0A,  0x2B,  0x0A,  0x0B,  0x0A,  0x3A,  0x0A,
+	/* FE_16APSK_910 */
+	0x0A,  0x0A,  0x0A,  0x0A,  0x2B,  0x0A,  0x0B,  0x0A,  0x3A,  0x0A,
+	/* FE_32APSK_34  */
+	0x09,  0x09,  0x09,  0x09,  0x09,  0x09,  0x09,  0x09,  0x09,  0x09,
+	/* FE_32APSK_45  */
+	0x09,  0x09,  0x09,  0x09,  0x09,  0x09,  0x09,  0x09,  0x09,  0x09,
+	/* FE_32APSK_56  */
+	0x09,  0x09,  0x09,  0x09,  0x09,  0x09,  0x09,  0x09,  0x09,  0x09,
+	/* FE_32APSK_89  */
+	0x09,  0x09,  0x09,  0x09,  0x09,  0x09,  0x09,  0x09,  0x09,  0x09,
+	/* FE_32APSK_910 */
+	0x09,  0x09,  0x09,  0x09,  0x09,  0x09,  0x09,  0x09,  0x09,  0x09,
+};
+
+static u8 get_optim_cloop(struct stv *state,
+			  enum fe_stv0910_mod_cod mod_cod, u32 pilots)
+{
+	int i = 0;
+
+	if (mod_cod >= FE_32APSK_910)
+		i = ((int)FE_32APSK_910 - (int)FE_QPSK_14) * 10;
+	else if (mod_cod >= FE_QPSK_14)
+		i = ((int)mod_cod - (int)FE_QPSK_14) * 10;
+
+	if (state->symbol_rate <= 3000000)
+		i += 0;
+	else if (state->symbol_rate <=  7000000)
+		i += 2;
+	else if (state->symbol_rate <= 15000000)
+		i += 4;
+	else if (state->symbol_rate <= 25000000)
+		i += 6;
+	else
+		i += 8;
+
+	if (!pilots)
+		i += 1;
+
+	return s2car_loop[i];
+}
+
+static int get_cur_symbol_rate(struct stv *state, u32 *p_symbol_rate)
+{
+	int status = 0;
+	u8 symb_freq0;
+	u8 symb_freq1;
+	u8 symb_freq2;
+	u8 symb_freq3;
+	u8 tim_offs0;
+	u8 tim_offs1;
+	u8 tim_offs2;
+	u32 symbol_rate;
+	s32 timing_offset;
+
+	*p_symbol_rate = 0;
+	if (!state->started)
+		return status;
+
+	read_reg(state, RSTV0910_P2_SFR3 + state->regoff, &symb_freq3);
+	read_reg(state, RSTV0910_P2_SFR2 + state->regoff, &symb_freq2);
+	read_reg(state, RSTV0910_P2_SFR1 + state->regoff, &symb_freq1);
+	read_reg(state, RSTV0910_P2_SFR0 + state->regoff, &symb_freq0);
+	read_reg(state, RSTV0910_P2_TMGREG2 + state->regoff, &tim_offs2);
+	read_reg(state, RSTV0910_P2_TMGREG1 + state->regoff, &tim_offs1);
+	read_reg(state, RSTV0910_P2_TMGREG0 + state->regoff, &tim_offs0);
+
+	symbol_rate = ((u32)symb_freq3 << 24) | ((u32)symb_freq2 << 16) |
+		((u32)symb_freq1 << 8) | (u32)symb_freq0;
+	timing_offset = ((u32)tim_offs2 << 16) | ((u32)tim_offs1 << 8) |
+		(u32)tim_offs0;
+
+	if ((timing_offset & (1 << 23)) != 0)
+		timing_offset |= 0xFF000000; /* Sign extent */
+
+	symbol_rate = (u32)(((u64)symbol_rate * state->base->mclk) >> 32);
+	timing_offset = (s32)(((s64)symbol_rate * (s64)timing_offset) >> 29);
+
+	*p_symbol_rate = symbol_rate + timing_offset;
+
+	return 0;
+}
+
+static int get_signal_parameters(struct stv *state)
+{
+	u8 tmp;
+
+	if (!state->started)
+		return -EINVAL;
+
+	if (state->receive_mode == RCVMODE_DVBS2) {
+		read_reg(state, RSTV0910_P2_DMDMODCOD + state->regoff, &tmp);
+		state->mod_cod = (enum fe_stv0910_mod_cod)((tmp & 0x7c) >> 2);
+		state->pilots = (tmp & 0x01) != 0;
+		state->fectype = (enum dvbs2_fectype)((tmp & 0x02) >> 1);
+
+	} else if (state->receive_mode == RCVMODE_DVBS) {
+		read_reg(state, RSTV0910_P2_VITCURPUN + state->regoff, &tmp);
+		state->puncture_rate = FEC_NONE;
+		switch (tmp & 0x1F) {
+		case 0x0d:
+			state->puncture_rate = FEC_1_2;
+			break;
+		case 0x12:
+			state->puncture_rate = FEC_2_3;
+			break;
+		case 0x15:
+			state->puncture_rate = FEC_3_4;
+			break;
+		case 0x18:
+			state->puncture_rate = FEC_5_6;
+			break;
+		case 0x1a:
+			state->puncture_rate = FEC_7_8;
+			break;
+		}
+		state->is_vcm = 0;
+		state->is_standard_broadcast = 1;
+		state->feroll_off = FE_SAT_35;
+	}
+	return 0;
+}
+
+static int tracking_optimization(struct stv *state)
+{
+	u32 symbol_rate = 0;
+	u8 tmp;
+
+	get_cur_symbol_rate(state, &symbol_rate);
+	read_reg(state, RSTV0910_P2_DMDCFGMD + state->regoff, &tmp);
+	tmp &= ~0xC0;
+
+	switch (state->receive_mode) {
+	case RCVMODE_DVBS:
+		tmp |= 0x40;
+		break;
+	case RCVMODE_DVBS2:
+		tmp |= 0x80;
+		break;
+	default:
+		tmp |= 0xC0;
+		break;
+	}
+	write_reg(state, RSTV0910_P2_DMDCFGMD + state->regoff, tmp);
+
+	if (state->receive_mode == RCVMODE_DVBS2) {
+		/* Disable Reed-Solomon */
+		write_shared_reg(state,
+				 RSTV0910_TSTTSRS, state->nr ? 0x02 : 0x01,
+				 0x03);
+
+		if (state->fectype == DVBS2_64K) {
+			u8 aclc = get_optim_cloop(state, state->mod_cod,
+						  state->pilots);
+
+			if (state->mod_cod <= FE_QPSK_910) {
+				write_reg(state, RSTV0910_P2_ACLC2S2Q +
+					  state->regoff, aclc);
+			} else if (state->mod_cod <= FE_8PSK_910) {
+				write_reg(state, RSTV0910_P2_ACLC2S2Q +
+					  state->regoff, 0x2a);
+				write_reg(state, RSTV0910_P2_ACLC2S28 +
+					  state->regoff, aclc);
+			} else if (state->mod_cod <= FE_16APSK_910) {
+				write_reg(state, RSTV0910_P2_ACLC2S2Q +
+					  state->regoff, 0x2a);
+				write_reg(state, RSTV0910_P2_ACLC2S216A +
+					  state->regoff, aclc);
+			} else if (state->mod_cod <= FE_32APSK_910) {
+				write_reg(state, RSTV0910_P2_ACLC2S2Q +
+					  state->regoff, 0x2a);
+				write_reg(state, RSTV0910_P2_ACLC2S232A +
+					  state->regoff, aclc);
+			}
+		}
+	}
+	return 0;
+}
+
+static s32 table_lookup(const struct slookup *table,
+			int table_size, u32 reg_value)
+{
+	s32 value;
+	int imin = 0;
+	int imax = table_size - 1;
+	int i;
+	s32 reg_diff;
+
+	/* Assumes Table[0].RegValue > Table[imax].RegValue */
+	if (reg_value >= table[0].reg_value) {
+		value = table[0].value;
+	} else if (reg_value <= table[imax].reg_value) {
+		value = table[imax].value;
+	} else {
+		while ((imax - imin) > 1) {
+			i = (imax + imin) / 2;
+			if ((table[imin].reg_value >= reg_value) &&
+			    (reg_value >= table[i].reg_value))
+				imax = i;
+			else
+				imin = i;
+		}
+
+		reg_diff = table[imax].reg_value - table[imin].reg_value;
+		value = table[imin].value;
+		if (reg_diff != 0)
+			value += ((s32)(reg_value - table[imin].reg_value) *
+				  (s32)(table[imax].value
+					- table[imin].value))
+					/ (reg_diff);
+	}
+
+	return value;
+}
+
+static int get_signal_to_noise(struct stv *state, s32 *signal_to_noise)
+{
+	u8 data0;
+	u8 data1;
+	u16 data;
+	int n_lookup;
+	const struct slookup *lookup;
+
+	*signal_to_noise = 0;
+
+	if (!state->started)
+		return -EINVAL;
+
+	if (state->receive_mode == RCVMODE_DVBS2) {
+		read_reg(state, RSTV0910_P2_NNOSPLHT1 + state->regoff,
+			 &data1);
+		read_reg(state, RSTV0910_P2_NNOSPLHT0 + state->regoff,
+			 &data0);
+		n_lookup = ARRAY_SIZE(s2_sn_lookup);
+		lookup = s2_sn_lookup;
+	} else {
+		read_reg(state, RSTV0910_P2_NNOSDATAT1 + state->regoff,
+			 &data1);
+		read_reg(state, RSTV0910_P2_NNOSDATAT0 + state->regoff,
+			 &data0);
+		n_lookup = ARRAY_SIZE(s1_sn_lookup);
+		lookup = s1_sn_lookup;
+	}
+	data = (((u16)data1) << 8) | (u16)data0;
+	*signal_to_noise = table_lookup(lookup, n_lookup, data);
+	return 0;
+}
+
+static int get_bit_error_rate_s(struct stv *state, u32 *bernumerator,
+				u32 *berdenominator)
+{
+	u8 regs[3];
+
+	int status = read_regs(state,
+			       RSTV0910_P2_ERRCNT12 + state->regoff,
+			       regs, 3);
+
+	if (status)
+		return -EINVAL;
+
+	if ((regs[0] & 0x80) == 0) {
+		state->last_berdenominator = 1 << ((state->berscale * 2) +
+						  10 + 3);
+		state->last_bernumerator = ((u32)(regs[0] & 0x7F) << 16) |
+			((u32)regs[1] << 8) | regs[2];
+		if (state->last_bernumerator < 256 && state->berscale < 6) {
+			state->berscale += 1;
+			status = write_reg(state, RSTV0910_P2_ERRCTRL1 +
+					   state->regoff,
+					   0x20 | state->berscale);
+		} else if (state->last_bernumerator > 1024 &&
+			   state->berscale > 2) {
+			state->berscale -= 1;
+			status = write_reg(state, RSTV0910_P2_ERRCTRL1 +
+					   state->regoff, 0x20 |
+					   state->berscale);
+		}
+	}
+	*bernumerator = state->last_bernumerator;
+	*berdenominator = state->last_berdenominator;
+	return 0;
+}
+
+static u32 dvbs2_nbch(enum dvbs2_mod_cod mod_cod, enum dvbs2_fectype fectype)
+{
+	static const u32 nbch[][2] = {
+		{    0,     0}, /* DUMMY_PLF   */
+		{16200,  3240}, /* QPSK_1_4,   */
+		{21600,  5400}, /* QPSK_1_3,   */
+		{25920,  6480}, /* QPSK_2_5,   */
+		{32400,  7200}, /* QPSK_1_2,   */
+		{38880,  9720}, /* QPSK_3_5,   */
+		{43200, 10800}, /* QPSK_2_3,   */
+		{48600, 11880}, /* QPSK_3_4,   */
+		{51840, 12600}, /* QPSK_4_5,   */
+		{54000, 13320}, /* QPSK_5_6,   */
+		{57600, 14400}, /* QPSK_8_9,   */
+		{58320, 16000}, /* QPSK_9_10,  */
+		{43200,  9720}, /* 8PSK_3_5,   */
+		{48600, 10800}, /* 8PSK_2_3,   */
+		{51840, 11880}, /* 8PSK_3_4,   */
+		{54000, 13320}, /* 8PSK_5_6,   */
+		{57600, 14400}, /* 8PSK_8_9,   */
+		{58320, 16000}, /* 8PSK_9_10,  */
+		{43200, 10800}, /* 16APSK_2_3, */
+		{48600, 11880}, /* 16APSK_3_4, */
+		{51840, 12600}, /* 16APSK_4_5, */
+		{54000, 13320}, /* 16APSK_5_6, */
+		{57600, 14400}, /* 16APSK_8_9, */
+		{58320, 16000}, /* 16APSK_9_10 */
+		{48600, 11880}, /* 32APSK_3_4, */
+		{51840, 12600}, /* 32APSK_4_5, */
+		{54000, 13320}, /* 32APSK_5_6, */
+		{57600, 14400}, /* 32APSK_8_9, */
+		{58320, 16000}, /* 32APSK_9_10 */
+	};
+
+	if (mod_cod >= DVBS2_QPSK_1_4 &&
+	    mod_cod <= DVBS2_32APSK_9_10 && fectype <= DVBS2_16K)
+		return nbch[mod_cod][fectype];
+	return 64800;
+}
+
+static int get_bit_error_rate_s2(struct stv *state, u32 *bernumerator,
+				 u32 *berdenominator)
+{
+	u8 regs[3];
+
+	int status = read_regs(state, RSTV0910_P2_ERRCNT12 + state->regoff,
+			       regs, 3);
+
+	if (status)
+		return -EINVAL;
+
+	if ((regs[0] & 0x80) == 0) {
+		state->last_berdenominator =
+			dvbs2_nbch((enum dvbs2_mod_cod)state->mod_cod,
+				   state->fectype) <<
+			(state->berscale * 2);
+		state->last_bernumerator = (((u32)regs[0] & 0x7F) << 16) |
+			((u32)regs[1] << 8) | regs[2];
+		if (state->last_bernumerator < 256 && state->berscale < 6) {
+			state->berscale += 1;
+			write_reg(state, RSTV0910_P2_ERRCTRL1 + state->regoff,
+				  0x20 | state->berscale);
+		} else if (state->last_bernumerator > 1024 &&
+			   state->berscale > 2) {
+			state->berscale -= 1;
+			write_reg(state, RSTV0910_P2_ERRCTRL1 + state->regoff,
+				  0x20 | state->berscale);
+		}
+	}
+	*bernumerator = state->last_bernumerator;
+	*berdenominator = state->last_berdenominator;
+	return status;
+}
+
+static int get_bit_error_rate(struct stv *state, u32 *bernumerator,
+			      u32 *berdenominator)
+{
+	*bernumerator = 0;
+	*berdenominator = 1;
+
+	switch (state->receive_mode) {
+	case RCVMODE_DVBS:
+		return get_bit_error_rate_s(state,
+					    bernumerator, berdenominator);
+	case RCVMODE_DVBS2:
+		return get_bit_error_rate_s2(state,
+					     bernumerator, berdenominator);
+	default:
+		break;
+	}
+	return 0;
+}
+
+static int set_mclock(struct stv *state, u32 master_clock)
+{
+	u32 idf = 1;
+	u32 odf = 4;
+	u32 quartz = state->base->extclk / 1000000;
+	u32 fphi = master_clock / 1000000;
+	u32 ndiv = (fphi * odf * idf) / quartz;
+	u32 cp = 7;
+	u32 fvco;
+
+	if (ndiv >= 7 && ndiv <= 71)
+		cp = 7;
+	else if (ndiv >=  72 && ndiv <=  79)
+		cp = 8;
+	else if (ndiv >=  80 && ndiv <=  87)
+		cp = 9;
+	else if (ndiv >=  88 && ndiv <=  95)
+		cp = 10;
+	else if (ndiv >=  96 && ndiv <= 103)
+		cp = 11;
+	else if (ndiv >= 104 && ndiv <= 111)
+		cp = 12;
+	else if (ndiv >= 112 && ndiv <= 119)
+		cp = 13;
+	else if (ndiv >= 120 && ndiv <= 127)
+		cp = 14;
+	else if (ndiv >= 128 && ndiv <= 135)
+		cp = 15;
+	else if (ndiv >= 136 && ndiv <= 143)
+		cp = 16;
+	else if (ndiv >= 144 && ndiv <= 151)
+		cp = 17;
+	else if (ndiv >= 152 && ndiv <= 159)
+		cp = 18;
+	else if (ndiv >= 160 && ndiv <= 167)
+		cp = 19;
+	else if (ndiv >= 168 && ndiv <= 175)
+		cp = 20;
+	else if (ndiv >= 176 && ndiv <= 183)
+		cp = 21;
+	else if (ndiv >= 184 && ndiv <= 191)
+		cp = 22;
+	else if (ndiv >= 192 && ndiv <= 199)
+		cp = 23;
+	else if (ndiv >= 200 && ndiv <= 207)
+		cp = 24;
+	else if (ndiv >= 208 && ndiv <= 215)
+		cp = 25;
+	else if (ndiv >= 216 && ndiv <= 223)
+		cp = 26;
+	else if (ndiv >= 224 && ndiv <= 225)
+		cp = 27;
+
+	write_reg(state, RSTV0910_NCOARSE, (cp << 3) | idf);
+	write_reg(state, RSTV0910_NCOARSE2, odf);
+	write_reg(state, RSTV0910_NCOARSE1, ndiv);
+
+	fvco = (quartz * 2 * ndiv) / idf;
+	state->base->mclk = fvco / (2 * odf) * 1000000;
+
+	return 0;
+}
+
+static int stop(struct stv *state)
+{
+	if (state->started) {
+		u8 tmp;
+
+		write_reg(state, RSTV0910_P2_TSCFGH + state->regoff,
+			  state->tscfgh | 0x01);
+		read_reg(state, RSTV0910_P2_PDELCTRL1 + state->regoff, &tmp);
+		tmp &= ~0x01; /* release reset DVBS2 packet delin */
+		write_reg(state, RSTV0910_P2_PDELCTRL1 + state->regoff, tmp);
+		/* Blind optim*/
+		write_reg(state, RSTV0910_P2_AGC2O + state->regoff, 0x5B);
+		/* Stop the demod */
+		write_reg(state, RSTV0910_P2_DMDISTATE + state->regoff, 0x5c);
+		state->started = 0;
+	}
+	state->receive_mode = RCVMODE_NONE;
+	return 0;
+}
+
+static int init_search_param(struct stv *state)
+{
+	u8 tmp;
+
+	read_reg(state, RSTV0910_P2_PDELCTRL1 + state->regoff, &tmp);
+	tmp |= 0x20; /* Filter_en (no effect if SIS=non-MIS */
+	write_reg(state, RSTV0910_P2_PDELCTRL1 + state->regoff, tmp);
+
+	read_reg(state, RSTV0910_P2_PDELCTRL2 + state->regoff, &tmp);
+	tmp &= ~0x02; /* frame mode = 0 */
+	write_reg(state, RSTV0910_P2_PDELCTRL2 + state->regoff, tmp);
+
+	write_reg(state, RSTV0910_P2_UPLCCST0 + state->regoff, 0xe0);
+	write_reg(state, RSTV0910_P2_ISIBITENA + state->regoff, 0x00);
+
+	read_reg(state, RSTV0910_P2_TSSTATEM + state->regoff, &tmp);
+	tmp &= ~0x01; /* nosync = 0, in case next signal is standard TS */
+	write_reg(state, RSTV0910_P2_TSSTATEM + state->regoff, tmp);
+
+	read_reg(state, RSTV0910_P2_TSCFGL + state->regoff, &tmp);
+	tmp &= ~0x04; /* embindvb = 0 */
+	write_reg(state, RSTV0910_P2_TSCFGL + state->regoff, tmp);
+
+	read_reg(state, RSTV0910_P2_TSINSDELH + state->regoff, &tmp);
+	tmp &= ~0x80; /* syncbyte = 0 */
+	write_reg(state, RSTV0910_P2_TSINSDELH + state->regoff, tmp);
+
+	read_reg(state, RSTV0910_P2_TSINSDELM + state->regoff, &tmp);
+	tmp &= ~0x08; /* token = 0 */
+	write_reg(state, RSTV0910_P2_TSINSDELM + state->regoff, tmp);
+
+	read_reg(state, RSTV0910_P2_TSDLYSET2 + state->regoff, &tmp);
+	tmp &= ~0x30; /* hysteresis threshold = 0 */
+	write_reg(state, RSTV0910_P2_TSDLYSET2 + state->regoff, tmp);
+
+	read_reg(state, RSTV0910_P2_PDELCTRL0 + state->regoff, &tmp);
+	tmp = (tmp & ~0x30) | 0x10; /* isi obs mode = 1, observe min ISI */
+	write_reg(state, RSTV0910_P2_PDELCTRL0 + state->regoff, tmp);
+
+	return 0;
+}
+
+static int enable_puncture_rate(struct stv *state, enum fe_code_rate rate)
+{
+	switch (rate) {
+	case FEC_1_2:
+		return write_reg(state,
+				 RSTV0910_P2_PRVIT + state->regoff, 0x01);
+	case FEC_2_3:
+		return write_reg(state,
+				 RSTV0910_P2_PRVIT + state->regoff, 0x02);
+	case FEC_3_4:
+		return write_reg(state,
+				 RSTV0910_P2_PRVIT + state->regoff, 0x04);
+	case FEC_5_6:
+		return write_reg(state,
+				 RSTV0910_P2_PRVIT + state->regoff, 0x08);
+	case FEC_7_8:
+		return write_reg(state,
+				 RSTV0910_P2_PRVIT + state->regoff, 0x20);
+	case FEC_NONE:
+	default:
+		return write_reg(state,
+				 RSTV0910_P2_PRVIT + state->regoff, 0x2f);
+	}
+}
+
+static int set_vth_default(struct stv *state)
+{
+	state->vth[0] = 0xd7;
+	state->vth[1] = 0x85;
+	state->vth[2] = 0x58;
+	state->vth[3] = 0x3a;
+	state->vth[4] = 0x34;
+	state->vth[5] = 0x28;
+	write_reg(state, RSTV0910_P2_VTH12 + state->regoff + 0, state->vth[0]);
+	write_reg(state, RSTV0910_P2_VTH12 + state->regoff + 1, state->vth[1]);
+	write_reg(state, RSTV0910_P2_VTH12 + state->regoff + 2, state->vth[2]);
+	write_reg(state, RSTV0910_P2_VTH12 + state->regoff + 3, state->vth[3]);
+	write_reg(state, RSTV0910_P2_VTH12 + state->regoff + 4, state->vth[4]);
+	write_reg(state, RSTV0910_P2_VTH12 + state->regoff + 5, state->vth[5]);
+	return 0;
+}
+
+static int set_vth(struct stv *state)
+{
+	static const struct slookup vthlookup_table[] = {
+		{250,	8780}, /* C/N= 1.5dB */
+		{100,	7405}, /* C/N= 4.5dB */
+		{40,	6330}, /* C/N= 6.5dB */
+		{12,	5224}, /* C/N= 8.5dB */
+		{5,	4236}  /* C/N=10.5dB */
+	};
+
+	int i;
+	u8 tmp[2];
+	int status = read_regs(state,
+			       RSTV0910_P2_NNOSDATAT1 + state->regoff,
+			       tmp, 2);
+	u16 reg_value = (tmp[0] << 8) | tmp[1];
+	s32 vth = table_lookup(vthlookup_table, ARRAY_SIZE(vthlookup_table),
+			      reg_value);
+
+	for (i = 0; i < 6; i += 1)
+		if (state->vth[i] > vth)
+			state->vth[i] = vth;
+
+	write_reg(state, RSTV0910_P2_VTH12 + state->regoff + 0, state->vth[0]);
+	write_reg(state, RSTV0910_P2_VTH12 + state->regoff + 1, state->vth[1]);
+	write_reg(state, RSTV0910_P2_VTH12 + state->regoff + 2, state->vth[2]);
+	write_reg(state, RSTV0910_P2_VTH12 + state->regoff + 3, state->vth[3]);
+	write_reg(state, RSTV0910_P2_VTH12 + state->regoff + 4, state->vth[4]);
+	write_reg(state, RSTV0910_P2_VTH12 + state->regoff + 5, state->vth[5]);
+	return status;
+}
+
+static int start(struct stv *state, struct dtv_frontend_properties *p)
+{
+	s32 freq;
+	u8  reg_dmdcfgmd;
+	u16 symb;
+	u32 scrambling_code = 1;
+
+	if (p->symbol_rate < 100000 || p->symbol_rate > 70000000)
+		return -EINVAL;
+
+	state->receive_mode = RCVMODE_NONE;
+	state->demod_lock_time = 0;
+
+	/* Demod Stop */
+	if (state->started)
+		write_reg(state, RSTV0910_P2_DMDISTATE + state->regoff, 0x5C);
+
+	init_search_param(state);
+
+	if (p->stream_id != NO_STREAM_ID_FILTER) {
+		/*
+		 * Backwards compatibility to "crazy" API.
+		 * PRBS X root cannot be 0, so this should always work.
+		 */
+		if (p->stream_id & 0xffffff00)
+			scrambling_code = p->stream_id >> 8;
+		write_reg(state, RSTV0910_P2_ISIENTRY + state->regoff,
+			  p->stream_id & 0xff);
+		write_reg(state, RSTV0910_P2_ISIBITENA + state->regoff,
+			  0xff);
+	}
+
+	if (scrambling_code != state->cur_scrambling_code) {
+		write_reg(state, RSTV0910_P2_PLROOT0 + state->regoff,
+			  scrambling_code & 0xff);
+		write_reg(state, RSTV0910_P2_PLROOT1 + state->regoff,
+			  (scrambling_code >> 8) & 0xff);
+		write_reg(state, RSTV0910_P2_PLROOT2 + state->regoff,
+			  (scrambling_code >> 16) & 0x0f);
+		state->cur_scrambling_code = scrambling_code;
+	}
+
+	if (p->symbol_rate <= 1000000) { /* SR <=1Msps */
+		state->demod_timeout = 3000;
+		state->fec_timeout = 2000;
+	} else if (p->symbol_rate <= 2000000) { /* 1Msps < SR <=2Msps */
+		state->demod_timeout = 2500;
+		state->fec_timeout = 1300;
+	} else if (p->symbol_rate <= 5000000) { /* 2Msps< SR <=5Msps */
+		state->demod_timeout = 1000;
+		state->fec_timeout = 650;
+	} else if (p->symbol_rate <= 10000000) { /* 5Msps< SR <=10Msps */
+		state->demod_timeout = 700;
+		state->fec_timeout = 350;
+	} else if (p->symbol_rate < 20000000) { /* 10Msps< SR <=20Msps */
+		state->demod_timeout = 400;
+		state->fec_timeout = 200;
+	} else { /* SR >=20Msps */
+		state->demod_timeout = 300;
+		state->fec_timeout = 200;
+	}
+
+	/* Set the Init Symbol rate */
+	symb = muldiv32(p->symbol_rate, 65536, state->base->mclk);
+	write_reg(state, RSTV0910_P2_SFRINIT1 + state->regoff,
+		  ((symb >> 8) & 0x7F));
+	write_reg(state, RSTV0910_P2_SFRINIT0 + state->regoff, (symb & 0xFF));
+
+	state->demod_bits |= 0x80;
+	write_reg(state, RSTV0910_P2_DEMOD + state->regoff, state->demod_bits);
+
+	/* FE_STV0910_SetSearchStandard */
+	read_reg(state, RSTV0910_P2_DMDCFGMD + state->regoff, &reg_dmdcfgmd);
+	write_reg(state, RSTV0910_P2_DMDCFGMD + state->regoff,
+		  reg_dmdcfgmd |= 0xC0);
+
+	write_shared_reg(state,
+			 RSTV0910_TSTTSRS, state->nr ? 0x02 : 0x01, 0x00);
+
+	/* Disable DSS */
+	write_reg(state, RSTV0910_P2_FECM  + state->regoff, 0x00);
+	write_reg(state, RSTV0910_P2_PRVIT + state->regoff, 0x2F);
+
+	enable_puncture_rate(state, FEC_NONE);
+
+	/* 8PSK 3/5, 8PSK 2/3 Poff tracking optimization WA */
+	write_reg(state, RSTV0910_P2_ACLC2S2Q + state->regoff, 0x0B);
+	write_reg(state, RSTV0910_P2_ACLC2S28 + state->regoff, 0x0A);
+	write_reg(state, RSTV0910_P2_BCLC2S2Q + state->regoff, 0x84);
+	write_reg(state, RSTV0910_P2_BCLC2S28 + state->regoff, 0x84);
+	write_reg(state, RSTV0910_P2_CARHDR + state->regoff, 0x1C);
+	write_reg(state, RSTV0910_P2_CARFREQ + state->regoff, 0x79);
+
+	write_reg(state, RSTV0910_P2_ACLC2S216A + state->regoff, 0x29);
+	write_reg(state, RSTV0910_P2_ACLC2S232A + state->regoff, 0x09);
+	write_reg(state, RSTV0910_P2_BCLC2S216A + state->regoff, 0x84);
+	write_reg(state, RSTV0910_P2_BCLC2S232A + state->regoff, 0x84);
+
+	/*
+	 * Reset CAR3, bug DVBS2->DVBS1 lock
+	 * Note: The bit is only pulsed -> no lock on shared register needed
+	 */
+	write_reg(state, RSTV0910_TSTRES0, state->nr ? 0x04 : 0x08);
+	write_reg(state, RSTV0910_TSTRES0, 0);
+
+	set_vth_default(state);
+	/* Reset demod */
+	write_reg(state, RSTV0910_P2_DMDISTATE + state->regoff, 0x1F);
+
+	write_reg(state, RSTV0910_P2_CARCFG + state->regoff, 0x46);
+
+	if (p->symbol_rate <= 5000000)
+		freq = (state->search_range / 2000) + 80;
+	else
+		freq = (state->search_range / 2000) + 1600;
+	freq = (freq << 16) / (state->base->mclk / 1000);
+
+	write_reg(state, RSTV0910_P2_CFRUP1 + state->regoff,
+		  (freq >> 8) & 0xff);
+	write_reg(state, RSTV0910_P2_CFRUP0 + state->regoff, (freq & 0xff));
+	/* CFR Low Setting */
+	freq = -freq;
+	write_reg(state, RSTV0910_P2_CFRLOW1 + state->regoff,
+		  (freq >> 8) & 0xff);
+	write_reg(state, RSTV0910_P2_CFRLOW0 + state->regoff, (freq & 0xff));
+
+	/* init the demod frequency offset to 0 */
+	write_reg(state, RSTV0910_P2_CFRINIT1 + state->regoff, 0);
+	write_reg(state, RSTV0910_P2_CFRINIT0 + state->regoff, 0);
+
+	write_reg(state, RSTV0910_P2_DMDISTATE + state->regoff, 0x1F);
+	/* Trigger acq */
+	write_reg(state, RSTV0910_P2_DMDISTATE + state->regoff, 0x15);
+
+	state->demod_lock_time += TUNING_DELAY;
+	state->started = 1;
+
+	return 0;
+}
+
+static int init_diseqc(struct stv *state)
+{
+	u16 offs = state->nr ? 0x40 : 0; /* Address offset */
+	u8 freq = ((state->base->mclk + 11000 * 32) / (22000 * 32));
+
+	/* Disable receiver */
+	write_reg(state, RSTV0910_P1_DISRXCFG + offs, 0x00);
+	write_reg(state, RSTV0910_P1_DISTXCFG + offs, 0xBA); /* Reset = 1 */
+	write_reg(state, RSTV0910_P1_DISTXCFG + offs, 0x3A); /* Reset = 0 */
+	write_reg(state, RSTV0910_P1_DISTXF22 + offs, freq);
+	return 0;
+}
+
+static int probe(struct stv *state)
+{
+	u8 id;
+
+	state->receive_mode = RCVMODE_NONE;
+	state->started = 0;
+
+	if (read_reg(state, RSTV0910_MID, &id) < 0)
+		return -ENODEV;
+
+	if (id != 0x51)
+		return -EINVAL;
+
+	/* Configure the I2C repeater to off */
+	write_reg(state, RSTV0910_P1_I2CRPT, 0x24);
+	/* Configure the I2C repeater to off */
+	write_reg(state, RSTV0910_P2_I2CRPT, 0x24);
+	/* Set the I2C to oversampling ratio */
+	write_reg(state, RSTV0910_I2CCFG, 0x88); /* state->i2ccfg */
+
+	write_reg(state, RSTV0910_OUTCFG,    0x00); /* OUTCFG */
+	write_reg(state, RSTV0910_PADCFG,    0x05); /* RFAGC Pads Dev = 05 */
+	write_reg(state, RSTV0910_SYNTCTRL,  0x02); /* SYNTCTRL */
+	write_reg(state, RSTV0910_TSGENERAL, state->tsgeneral); /* TSGENERAL */
+	write_reg(state, RSTV0910_CFGEXT,    0x02); /* CFGEXT */
+
+	if (state->single)
+		write_reg(state, RSTV0910_GENCFG, 0x14); /* GENCFG */
+	else
+		write_reg(state, RSTV0910_GENCFG, 0x15); /* GENCFG */
+
+	write_reg(state, RSTV0910_P1_TNRCFG2, 0x02); /* IQSWAP = 0 */
+	write_reg(state, RSTV0910_P2_TNRCFG2, 0x82); /* IQSWAP = 1 */
+
+	write_reg(state, RSTV0910_P1_CAR3CFG, 0x02);
+	write_reg(state, RSTV0910_P2_CAR3CFG, 0x02);
+	write_reg(state, RSTV0910_P1_DMDCFG4, 0x04);
+	write_reg(state, RSTV0910_P2_DMDCFG4, 0x04);
+
+	write_reg(state, RSTV0910_TSTRES0, 0x80); /* LDPC Reset */
+	write_reg(state, RSTV0910_TSTRES0, 0x00);
+
+	write_reg(state, RSTV0910_P1_TSPIDFLT1, 0x00);
+	write_reg(state, RSTV0910_P2_TSPIDFLT1, 0x00);
+
+	write_reg(state, RSTV0910_P1_TMGCFG2, 0x80);
+	write_reg(state, RSTV0910_P2_TMGCFG2, 0x80);
+
+	set_mclock(state, 135000000);
+
+	/* TS output */
+	write_reg(state, RSTV0910_P1_TSCFGH, state->tscfgh | 0x01);
+	write_reg(state, RSTV0910_P1_TSCFGH, state->tscfgh);
+	write_reg(state, RSTV0910_P1_TSCFGM, 0xC0); /* Manual speed */
+	write_reg(state, RSTV0910_P1_TSCFGL, 0x20);
+
+	/* Speed = 67.5 MHz */
+	write_reg(state, RSTV0910_P1_TSSPEED, state->tsspeed);
+
+	write_reg(state, RSTV0910_P2_TSCFGH, state->tscfgh | 0x01);
+	write_reg(state, RSTV0910_P2_TSCFGH, state->tscfgh);
+	write_reg(state, RSTV0910_P2_TSCFGM, 0xC0); /* Manual speed */
+	write_reg(state, RSTV0910_P2_TSCFGL, 0x20);
+
+	/* Speed = 67.5 MHz */
+	write_reg(state, RSTV0910_P2_TSSPEED, state->tsspeed);
+
+	/* Reset stream merger */
+	write_reg(state, RSTV0910_P1_TSCFGH, state->tscfgh | 0x01);
+	write_reg(state, RSTV0910_P2_TSCFGH, state->tscfgh | 0x01);
+	write_reg(state, RSTV0910_P1_TSCFGH, state->tscfgh);
+	write_reg(state, RSTV0910_P2_TSCFGH, state->tscfgh);
+
+	write_reg(state, RSTV0910_P1_I2CRPT, state->i2crpt);
+	write_reg(state, RSTV0910_P2_I2CRPT, state->i2crpt);
+
+	init_diseqc(state);
+	return 0;
+}
+
+static int gate_ctrl(struct dvb_frontend *fe, int enable)
+{
+	struct stv *state = fe->demodulator_priv;
+	u8 i2crpt = state->i2crpt & ~0x86;
+
+	/*
+	 * mutex_lock note: Concurrent I2C gate bus accesses must be
+	 * prevented (STV0910 = dual demod on a single IC with a single I2C
+	 * gate/bus, and two tuners attached), similar to most (if not all)
+	 * other I2C host interfaces/busses.
+	 *
+	 * enable=1 (open I2C gate) will grab the lock
+	 * enable=0 (close I2C gate) releases the lock
+	 */
+
+	if (enable) {
+		mutex_lock(&state->base->i2c_lock);
+		i2crpt |= 0x80;
+	} else {
+		i2crpt |= 0x02;
+	}
+
+	if (write_reg(state, state->nr ? RSTV0910_P2_I2CRPT :
+		      RSTV0910_P1_I2CRPT, i2crpt) < 0) {
+		/* don't hold the I2C bus lock on failure */
+		mutex_unlock(&state->base->i2c_lock);
+		dev_err(&state->base->i2c->dev,
+			"%s() write_reg failure (enable=%d)\n",
+			__func__, enable);
+		return -EIO;
+	}
+
+	state->i2crpt = i2crpt;
+
+	if (!enable)
+		mutex_unlock(&state->base->i2c_lock);
+	return 0;
+}
+
+static void release(struct dvb_frontend *fe)
+{
+	struct stv *state = fe->demodulator_priv;
+
+	state->base->count--;
+	if (state->base->count == 0) {
+		list_del(&state->base->stvlist);
+		kfree(state->base);
+	}
+	kfree(state);
+}
+
+static int set_parameters(struct dvb_frontend *fe)
+{
+	int stat = 0;
+	struct stv *state = fe->demodulator_priv;
+	u32 iffreq;
+	struct dtv_frontend_properties *p = &fe->dtv_property_cache;
+
+	stop(state);
+	if (fe->ops.tuner_ops.set_params)
+		fe->ops.tuner_ops.set_params(fe);
+	if (fe->ops.tuner_ops.get_if_frequency)
+		fe->ops.tuner_ops.get_if_frequency(fe, &iffreq);
+	state->symbol_rate = p->symbol_rate;
+	stat = start(state, p);
+	return stat;
+}
+
+static int manage_matype_info(struct stv *state)
+{
+	if (!state->started)
+		return -EINVAL;
+	if (state->receive_mode == RCVMODE_DVBS2) {
+		u8 bbheader[2];
+
+		read_regs(state, RSTV0910_P2_MATSTR1 + state->regoff,
+			  bbheader, 2);
+		state->feroll_off =
+			(enum fe_stv0910_roll_off)(bbheader[0] & 0x03);
+		state->is_vcm = (bbheader[0] & 0x10) == 0;
+		state->is_standard_broadcast = (bbheader[0] & 0xFC) == 0xF0;
+	} else if (state->receive_mode == RCVMODE_DVBS) {
+		state->is_vcm = 0;
+		state->is_standard_broadcast = 1;
+		state->feroll_off = FE_SAT_35;
+	}
+	return 0;
+}
+
+static int read_snr(struct dvb_frontend *fe)
+{
+	struct stv *state = fe->demodulator_priv;
+	struct dtv_frontend_properties *p = &fe->dtv_property_cache;
+	s32 snrval;
+
+	if (!get_signal_to_noise(state, &snrval)) {
+		p->cnr.stat[0].scale = FE_SCALE_DECIBEL;
+		p->cnr.stat[0].uvalue = 100 * snrval; /* fix scale */
+	} else {
+		p->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+	}
+
+	return 0;
+}
+
+static int read_ber(struct dvb_frontend *fe)
+{
+	struct stv *state = fe->demodulator_priv;
+	struct dtv_frontend_properties *p = &fe->dtv_property_cache;
+	u32 n, d;
+
+	get_bit_error_rate(state, &n, &d);
+
+	p->pre_bit_error.stat[0].scale = FE_SCALE_COUNTER;
+	p->pre_bit_error.stat[0].uvalue = n;
+	p->pre_bit_count.stat[0].scale = FE_SCALE_COUNTER;
+	p->pre_bit_count.stat[0].uvalue = d;
+
+	return 0;
+}
+
+static void read_signal_strength(struct dvb_frontend *fe)
+{
+	struct stv *state = fe->demodulator_priv;
+	struct dtv_frontend_properties *p = &state->fe.dtv_property_cache;
+	u8 reg[2];
+	u16 agc;
+	s32 padc, power = 0;
+	int i;
+
+	read_regs(state, RSTV0910_P2_AGCIQIN1 + state->regoff, reg, 2);
+
+	agc = (((u32)reg[0]) << 8) | reg[1];
+
+	for (i = 0; i < 5; i += 1) {
+		read_regs(state, RSTV0910_P2_POWERI + state->regoff, reg, 2);
+		power += (u32)reg[0] * (u32)reg[0]
+			+ (u32)reg[1] * (u32)reg[1];
+		usleep_range(3000, 4000);
+	}
+	power /= 5;
+
+	padc = table_lookup(padc_lookup, ARRAY_SIZE(padc_lookup), power) + 352;
+
+	p->strength.stat[0].scale = FE_SCALE_DECIBEL;
+	p->strength.stat[0].svalue = (padc - agc);
+}
+
+static int read_status(struct dvb_frontend *fe, enum fe_status *status)
+{
+	struct stv *state = fe->demodulator_priv;
+	struct dtv_frontend_properties *p = &fe->dtv_property_cache;
+	u8 dmd_state = 0;
+	u8 dstatus  = 0;
+	enum receive_mode cur_receive_mode = RCVMODE_NONE;
+	u32 feclock = 0;
+
+	*status = 0;
+
+	read_reg(state, RSTV0910_P2_DMDSTATE + state->regoff, &dmd_state);
+
+	if (dmd_state & 0x40) {
+		read_reg(state, RSTV0910_P2_DSTATUS + state->regoff, &dstatus);
+		if (dstatus & 0x08)
+			cur_receive_mode = (dmd_state & 0x20) ?
+				RCVMODE_DVBS : RCVMODE_DVBS2;
+	}
+	if (cur_receive_mode == RCVMODE_NONE) {
+		set_vth(state);
+
+		/* reset signal statistics */
+		p->strength.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+		p->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+		p->pre_bit_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+		p->pre_bit_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+
+		return 0;
+	}
+
+	*status |= (FE_HAS_SIGNAL
+		| FE_HAS_CARRIER
+		| FE_HAS_VITERBI
+		| FE_HAS_SYNC);
+
+	if (state->receive_mode == RCVMODE_NONE) {
+		state->receive_mode = cur_receive_mode;
+		state->demod_lock_time = jiffies;
+		state->first_time_lock = 1;
+
+		get_signal_parameters(state);
+		tracking_optimization(state);
+
+		write_reg(state, RSTV0910_P2_TSCFGH + state->regoff,
+			  state->tscfgh);
+		usleep_range(3000, 4000);
+		write_reg(state, RSTV0910_P2_TSCFGH + state->regoff,
+			  state->tscfgh | 0x01);
+		write_reg(state, RSTV0910_P2_TSCFGH + state->regoff,
+			  state->tscfgh);
+	}
+	if (dmd_state & 0x40) {
+		if (state->receive_mode == RCVMODE_DVBS2) {
+			u8 pdelstatus;
+
+			read_reg(state,
+				 RSTV0910_P2_PDELSTATUS1 + state->regoff,
+				 &pdelstatus);
+			feclock = (pdelstatus & 0x02) != 0;
+		} else {
+			u8 vstatus;
+
+			read_reg(state,
+				 RSTV0910_P2_VSTATUSVIT + state->regoff,
+				 &vstatus);
+			feclock = (vstatus & 0x08) != 0;
+		}
+	}
+
+	if (feclock) {
+		*status |= FE_HAS_LOCK;
+
+		if (state->first_time_lock) {
+			u8 tmp;
+
+			state->first_time_lock = 0;
+
+			manage_matype_info(state);
+
+			if (state->receive_mode == RCVMODE_DVBS2) {
+				/*
+				 * FSTV0910_P2_MANUALSX_ROLLOFF,
+				 * FSTV0910_P2_MANUALS2_ROLLOFF = 0
+				 */
+				state->demod_bits &= ~0x84;
+				write_reg(state,
+					  RSTV0910_P2_DEMOD + state->regoff,
+					  state->demod_bits);
+				read_reg(state,
+					 RSTV0910_P2_PDELCTRL2 + state->regoff,
+					 &tmp);
+				/* reset DVBS2 packet delinator error counter */
+				tmp |= 0x40;
+				write_reg(state,
+					  RSTV0910_P2_PDELCTRL2 + state->regoff,
+					  tmp);
+				/* reset DVBS2 packet delinator error counter */
+				tmp &= ~0x40;
+				write_reg(state,
+					  RSTV0910_P2_PDELCTRL2 + state->regoff,
+					  tmp);
+
+				state->berscale = 2;
+				state->last_bernumerator = 0;
+				state->last_berdenominator = 1;
+				/* force to PRE BCH Rate */
+				write_reg(state,
+					  RSTV0910_P2_ERRCTRL1 + state->regoff,
+					  BER_SRC_S2 | state->berscale);
+			} else {
+				state->berscale = 2;
+				state->last_bernumerator = 0;
+				state->last_berdenominator = 1;
+				/* force to PRE RS Rate */
+				write_reg(state,
+					  RSTV0910_P2_ERRCTRL1 + state->regoff,
+					  BER_SRC_S | state->berscale);
+			}
+			/* Reset the Total packet counter */
+			write_reg(state,
+				  RSTV0910_P2_FBERCPT4 + state->regoff, 0x00);
+			/*
+			 * Reset the packet Error counter2 (and Set it to
+			 * infinit error count mode)
+			 */
+			write_reg(state,
+				  RSTV0910_P2_ERRCTRL2 + state->regoff, 0xc1);
+
+			set_vth_default(state);
+			if (state->receive_mode == RCVMODE_DVBS)
+				enable_puncture_rate(state,
+						     state->puncture_rate);
+		}
+	}
+
+	/* read signal statistics */
+
+	/* read signal strength */
+	read_signal_strength(fe);
+
+	/* read carrier/noise on FE_HAS_CARRIER */
+	if (*status & FE_HAS_CARRIER)
+		read_snr(fe);
+	else
+		p->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+
+	/* read ber */
+	if (*status & FE_HAS_VITERBI) {
+		read_ber(fe);
+	} else {
+		p->pre_bit_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+		p->pre_bit_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+	}
+
+	return 0;
+}
+
+static int get_frontend(struct dvb_frontend *fe,
+			struct dtv_frontend_properties *p)
+{
+	struct stv *state = fe->demodulator_priv;
+	u8 tmp;
+
+	if (state->receive_mode == RCVMODE_DVBS2) {
+		u32 mc;
+		const enum fe_modulation modcod2mod[0x20] = {
+			QPSK, QPSK, QPSK, QPSK,
+			QPSK, QPSK, QPSK, QPSK,
+			QPSK, QPSK, QPSK, QPSK,
+			PSK_8, PSK_8, PSK_8, PSK_8,
+			PSK_8, PSK_8, APSK_16, APSK_16,
+			APSK_16, APSK_16, APSK_16, APSK_16,
+			APSK_32, APSK_32, APSK_32, APSK_32,
+			APSK_32,
+		};
+		const enum fe_code_rate modcod2fec[0x20] = {
+			FEC_NONE, FEC_NONE, FEC_NONE, FEC_2_5,
+			FEC_1_2, FEC_3_5, FEC_2_3, FEC_3_4,
+			FEC_4_5, FEC_5_6, FEC_8_9, FEC_9_10,
+			FEC_3_5, FEC_2_3, FEC_3_4, FEC_5_6,
+			FEC_8_9, FEC_9_10, FEC_2_3, FEC_3_4,
+			FEC_4_5, FEC_5_6, FEC_8_9, FEC_9_10,
+			FEC_3_4, FEC_4_5, FEC_5_6, FEC_8_9,
+			FEC_9_10
+		};
+		read_reg(state, RSTV0910_P2_DMDMODCOD + state->regoff, &tmp);
+		mc = ((tmp & 0x7c) >> 2);
+		p->pilot = (tmp & 0x01) ? PILOT_ON : PILOT_OFF;
+		p->modulation = modcod2mod[mc];
+		p->fec_inner = modcod2fec[mc];
+	} else if (state->receive_mode == RCVMODE_DVBS) {
+		read_reg(state, RSTV0910_P2_VITCURPUN + state->regoff, &tmp);
+		switch (tmp & 0x1F) {
+		case 0x0d:
+			p->fec_inner = FEC_1_2;
+			break;
+		case 0x12:
+			p->fec_inner = FEC_2_3;
+			break;
+		case 0x15:
+			p->fec_inner = FEC_3_4;
+			break;
+		case 0x18:
+			p->fec_inner = FEC_5_6;
+			break;
+		case 0x1a:
+			p->fec_inner = FEC_7_8;
+			break;
+		default:
+			p->fec_inner = FEC_NONE;
+			break;
+		}
+		p->rolloff = ROLLOFF_35;
+	}
+
+	return 0;
+}
+
+static int tune(struct dvb_frontend *fe, bool re_tune,
+		unsigned int mode_flags,
+		unsigned int *delay, enum fe_status *status)
+{
+	struct stv *state = fe->demodulator_priv;
+	int r;
+
+	if (re_tune) {
+		r = set_parameters(fe);
+		if (r)
+			return r;
+		state->tune_time = jiffies;
+	}
+
+	r = read_status(fe, status);
+	if (r)
+		return r;
+
+	if (*status & FE_HAS_LOCK)
+		return 0;
+	*delay = HZ;
+
+	return 0;
+}
+
+static int get_algo(struct dvb_frontend *fe)
+{
+	return DVBFE_ALGO_HW;
+}
+
+static int set_tone(struct dvb_frontend *fe, enum fe_sec_tone_mode tone)
+{
+	struct stv *state = fe->demodulator_priv;
+	u16 offs = state->nr ? 0x40 : 0;
+
+	switch (tone) {
+	case SEC_TONE_ON:
+		return write_reg(state, RSTV0910_P1_DISTXCFG + offs, 0x38);
+	case SEC_TONE_OFF:
+		return write_reg(state, RSTV0910_P1_DISTXCFG + offs, 0x3a);
+	default:
+		break;
+	}
+	return -EINVAL;
+}
+
+static int wait_dis(struct stv *state, u8 flag, u8 val)
+{
+	int i;
+	u8 stat;
+	u16 offs = state->nr ? 0x40 : 0;
+
+	for (i = 0; i < 10; i++) {
+		read_reg(state, RSTV0910_P1_DISTXSTATUS + offs, &stat);
+		if ((stat & flag) == val)
+			return 0;
+		usleep_range(10000, 11000);
+	}
+	return -ETIMEDOUT;
+}
+
+static int send_master_cmd(struct dvb_frontend *fe,
+			   struct dvb_diseqc_master_cmd *cmd)
+{
+	struct stv *state = fe->demodulator_priv;
+	u16 offs = state->nr ? 0x40 : 0;
+	int i;
+
+	write_reg(state, RSTV0910_P1_DISTXCFG + offs, 0x3E);
+	for (i = 0; i < cmd->msg_len; i++) {
+		wait_dis(state, 0x40, 0x00);
+		write_reg(state, RSTV0910_P1_DISTXFIFO + offs, cmd->msg[i]);
+	}
+	write_reg(state, RSTV0910_P1_DISTXCFG + offs, 0x3A);
+	wait_dis(state, 0x20, 0x20);
+	return 0;
+}
+
+static int send_burst(struct dvb_frontend *fe, enum fe_sec_mini_cmd burst)
+{
+	struct stv *state = fe->demodulator_priv;
+	u16 offs = state->nr ? 0x40 : 0;
+	u8 value;
+
+	if (burst == SEC_MINI_A) {
+		write_reg(state, RSTV0910_P1_DISTXCFG + offs, 0x3F);
+		value = 0x00;
+	} else {
+		write_reg(state, RSTV0910_P1_DISTXCFG + offs, 0x3E);
+		value = 0xFF;
+	}
+	wait_dis(state, 0x40, 0x00);
+	write_reg(state, RSTV0910_P1_DISTXFIFO + offs, value);
+	write_reg(state, RSTV0910_P1_DISTXCFG + offs, 0x3A);
+	wait_dis(state, 0x20, 0x20);
+
+	return 0;
+}
+
+static int sleep(struct dvb_frontend *fe)
+{
+	struct stv *state = fe->demodulator_priv;
+
+	stop(state);
+	return 0;
+}
+
+static const struct dvb_frontend_ops stv0910_ops = {
+	.delsys = { SYS_DVBS, SYS_DVBS2, SYS_DSS },
+	.info = {
+		.name			= "ST STV0910",
+		.frequency_min		= 950000,
+		.frequency_max		= 2150000,
+		.frequency_stepsize	= 0,
+		.frequency_tolerance	= 0,
+		.symbol_rate_min	= 100000,
+		.symbol_rate_max	= 70000000,
+		.caps			= FE_CAN_INVERSION_AUTO |
+					  FE_CAN_FEC_AUTO       |
+					  FE_CAN_QPSK           |
+					  FE_CAN_2G_MODULATION  |
+					  FE_CAN_MULTISTREAM
+	},
+	.sleep				= sleep,
+	.release			= release,
+	.i2c_gate_ctrl			= gate_ctrl,
+	.set_frontend			= set_parameters,
+	.get_frontend_algo		= get_algo,
+	.get_frontend			= get_frontend,
+	.tune				= tune,
+	.read_status			= read_status,
+	.set_tone			= set_tone,
+
+	.diseqc_send_master_cmd		= send_master_cmd,
+	.diseqc_send_burst		= send_burst,
+};
+
+static struct stv_base *match_base(struct i2c_adapter *i2c, u8 adr)
+{
+	struct stv_base *p;
+
+	list_for_each_entry(p, &stvlist, stvlist)
+		if (p->i2c == i2c && p->adr == adr)
+			return p;
+	return NULL;
+}
+
+static void stv0910_init_stats(struct stv *state)
+{
+	struct dtv_frontend_properties *p = &state->fe.dtv_property_cache;
+
+	p->strength.len = 1;
+	p->strength.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+	p->cnr.len = 1;
+	p->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+	p->pre_bit_error.len = 1;
+	p->pre_bit_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+	p->pre_bit_count.len = 1;
+	p->pre_bit_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+}
+
+struct dvb_frontend *stv0910_attach(struct i2c_adapter *i2c,
+				    struct stv0910_cfg *cfg,
+				    int nr)
+{
+	struct stv *state;
+	struct stv_base *base;
+
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (!state)
+		return NULL;
+
+	state->tscfgh = 0x20 | (cfg->parallel ? 0 : 0x40);
+	state->tsgeneral = (cfg->parallel == 2) ? 0x02 : 0x00;
+	state->i2crpt = 0x0A | ((cfg->rptlvl & 0x07) << 4);
+	state->tsspeed = 0x28;
+	state->nr = nr;
+	state->regoff = state->nr ? 0 : 0x200;
+	state->search_range = 16000000;
+	state->demod_bits = 0x10; /* Inversion : Auto with reset to 0 */
+	state->receive_mode = RCVMODE_NONE;
+	state->cur_scrambling_code = (~0U);
+	state->single = cfg->single ? 1 : 0;
+
+	base = match_base(i2c, cfg->adr);
+	if (base) {
+		base->count++;
+		state->base = base;
+	} else {
+		base = kzalloc(sizeof(*base), GFP_KERNEL);
+		if (!base)
+			goto fail;
+		base->i2c = i2c;
+		base->adr = cfg->adr;
+		base->count = 1;
+		base->extclk = cfg->clk ? cfg->clk : 30000000;
+
+		mutex_init(&base->i2c_lock);
+		mutex_init(&base->reg_lock);
+		state->base = base;
+		if (probe(state) < 0) {
+			dev_info(&i2c->dev, "No demod found at adr %02X on %s\n",
+				 cfg->adr, dev_name(&i2c->dev));
+			kfree(base);
+			goto fail;
+		}
+		list_add(&base->stvlist, &stvlist);
+	}
+	state->fe.ops = stv0910_ops;
+	state->fe.demodulator_priv = state;
+	state->nr = nr;
+
+	dev_info(&i2c->dev, "%s demod found at adr %02X on %s\n",
+		 state->fe.ops.info.name, cfg->adr, dev_name(&i2c->dev));
+
+	stv0910_init_stats(state);
+
+	return &state->fe;
+
+fail:
+	kfree(state);
+	return NULL;
+}
+EXPORT_SYMBOL_GPL(stv0910_attach);
+
+MODULE_DESCRIPTION("ST STV0910 multistandard frontend driver");
+MODULE_AUTHOR("Ralph and Marcus Metzler, Manfred Voelkel");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/dvb-frontends/stv0910.h b/drivers/media/dvb-frontends/stv0910.h
new file mode 100644
index 000000000000..fccd8d9b665f
--- /dev/null
+++ b/drivers/media/dvb-frontends/stv0910.h
@@ -0,0 +1,32 @@
+#ifndef _STV0910_H_
+#define _STV0910_H_
+
+#include <linux/types.h>
+#include <linux/i2c.h>
+
+struct stv0910_cfg {
+	u32 clk;
+	u8  adr;
+	u8  parallel;
+	u8  rptlvl;
+	u8  single;
+};
+
+#if IS_REACHABLE(CONFIG_DVB_STV0910)
+
+struct dvb_frontend *stv0910_attach(struct i2c_adapter *i2c,
+				    struct stv0910_cfg *cfg, int nr);
+
+#else
+
+static inline struct dvb_frontend *stv0910_attach(struct i2c_adapter *i2c,
+						  struct stv0910_cfg *cfg,
+						  int nr)
+{
+	pr_warn("%s: driver disabled by Kconfig\n", __func__);
+	return NULL;
+}
+
+#endif /* CONFIG_DVB_STV0910 */
+
+#endif /* _STV0910_H_ */
diff --git a/drivers/media/dvb-frontends/stv0910_regs.h b/drivers/media/dvb-frontends/stv0910_regs.h
new file mode 100644
index 000000000000..32ced4eaf296
--- /dev/null
+++ b/drivers/media/dvb-frontends/stv0910_regs.h
@@ -0,0 +1,4760 @@
+/*
+ * @DVB-S/DVB-S2 STMicroelectronics STV0900 register definitions
+ * Author Manfred Voelkel, August 2013
+ * (c) 2013 Digital Devices GmbH Germany.  All rights reserved
+ *
+ * =======================================================================
+ * Registers Declaration (Internal ST, All Applications )
+ * -------------------------
+ * Each register (RSTV0910__XXXXX) is defined by its address (2 bytes).
+ *
+ * Each field (FSTV0910__XXXXX)is defined as follow:
+ * [register address -- 2bytes][field sign -- 1byte][field mask -- 1byte]
+ *  ======================================================================
+ */
+
+/* MID */
+#define RSTV0910_MID  0xf100
+#define FSTV0910_MCHIP_IDENT  0xf10000f0
+#define FSTV0910_MRELEASE  0xf100000f
+
+/* DID */
+#define RSTV0910_DID  0xf101
+#define FSTV0910_DEVICE_ID  0xf10100ff
+
+/* DACR1 */
+#define RSTV0910_DACR1  0xf113
+#define FSTV0910_DAC_MODE  0xf11300e0
+#define FSTV0910_DAC_VALUE1  0xf113000f
+
+/* DACR2 */
+#define RSTV0910_DACR2  0xf114
+#define FSTV0910_DAC_VALUE0  0xf11400ff
+
+/* PADCFG */
+#define RSTV0910_PADCFG  0xf11a
+#define FSTV0910_AGCRF2_OPD  0xf11a0008
+#define FSTV0910_AGCRF2_XOR  0xf11a0004
+#define FSTV0910_AGCRF1_OPD  0xf11a0002
+#define FSTV0910_AGCRF1_XOR  0xf11a0001
+
+/* OUTCFG2 */
+#define RSTV0910_OUTCFG2  0xf11b
+#define FSTV0910_TS2_ERROR_XOR  0xf11b0080
+#define FSTV0910_TS2_DPN_XOR  0xf11b0040
+#define FSTV0910_TS2_STROUT_XOR  0xf11b0020
+#define FSTV0910_TS2_CLOCKOUT_XOR  0xf11b0010
+#define FSTV0910_TS1_ERROR_XOR  0xf11b0008
+#define FSTV0910_TS1_DPN_XOR  0xf11b0004
+#define FSTV0910_TS1_STROUT_XOR  0xf11b0002
+#define FSTV0910_TS1_CLOCKOUT_XOR  0xf11b0001
+
+/* OUTCFG */
+#define RSTV0910_OUTCFG  0xf11c
+#define FSTV0910_TS2_OUTSER_HZ  0xf11c0020
+#define FSTV0910_TS1_OUTSER_HZ  0xf11c0010
+#define FSTV0910_TS2_OUTPAR_HZ  0xf11c0008
+#define FSTV0910_TS1_OUTPAR_HZ  0xf11c0004
+#define FSTV0910_TS_SERDATA0  0xf11c0002
+
+/* IRQSTATUS3 */
+#define RSTV0910_IRQSTATUS3  0xf120
+#define FSTV0910_SPLL_LOCK  0xf1200020
+#define FSTV0910_SSTREAM_LCK_1  0xf1200010
+#define FSTV0910_SSTREAM_LCK_2  0xf1200008
+#define FSTV0910_SDVBS1_PRF_2  0xf1200002
+#define FSTV0910_SDVBS1_PRF_1  0xf1200001
+
+/* IRQSTATUS2 */
+#define RSTV0910_IRQSTATUS2  0xf121
+#define FSTV0910_SSPY_ENDSIM_1  0xf1210080
+#define FSTV0910_SSPY_ENDSIM_2  0xf1210040
+#define FSTV0910_SPKTDEL_ERROR_2  0xf1210010
+#define FSTV0910_SPKTDEL_LOCKB_2  0xf1210008
+#define FSTV0910_SPKTDEL_LOCK_2  0xf1210004
+#define FSTV0910_SPKTDEL_ERROR_1  0xf1210002
+#define FSTV0910_SPKTDEL_LOCKB_1  0xf1210001
+
+/* IRQSTATUS1 */
+#define RSTV0910_IRQSTATUS1  0xf122
+#define FSTV0910_SPKTDEL_LOCK_1  0xf1220080
+#define FSTV0910_SFEC_LOCKB_2  0xf1220040
+#define FSTV0910_SFEC_LOCK_2  0xf1220020
+#define FSTV0910_SFEC_LOCKB_1  0xf1220010
+#define FSTV0910_SFEC_LOCK_1  0xf1220008
+#define FSTV0910_SDEMOD_LOCKB_2  0xf1220004
+#define FSTV0910_SDEMOD_LOCK_2  0xf1220002
+#define FSTV0910_SDEMOD_IRQ_2  0xf1220001
+
+/* IRQSTATUS0 */
+#define RSTV0910_IRQSTATUS0  0xf123
+#define FSTV0910_SDEMOD_LOCKB_1  0xf1230080
+#define FSTV0910_SDEMOD_LOCK_1  0xf1230040
+#define FSTV0910_SDEMOD_IRQ_1  0xf1230020
+#define FSTV0910_SBCH_ERRFLAG  0xf1230010
+#define FSTV0910_SDISEQC2_IRQ  0xf1230004
+#define FSTV0910_SDISEQC1_IRQ  0xf1230001
+
+/* IRQMASK3 */
+#define RSTV0910_IRQMASK3  0xf124
+#define FSTV0910_MPLL_LOCK  0xf1240020
+#define FSTV0910_MSTREAM_LCK_1  0xf1240010
+#define FSTV0910_MSTREAM_LCK_2  0xf1240008
+#define FSTV0910_MDVBS1_PRF_2  0xf1240002
+#define FSTV0910_MDVBS1_PRF_1  0xf1240001
+
+/* IRQMASK2 */
+#define RSTV0910_IRQMASK2  0xf125
+#define FSTV0910_MSPY_ENDSIM_1  0xf1250080
+#define FSTV0910_MSPY_ENDSIM_2  0xf1250040
+#define FSTV0910_MPKTDEL_ERROR_2  0xf1250010
+#define FSTV0910_MPKTDEL_LOCKB_2  0xf1250008
+#define FSTV0910_MPKTDEL_LOCK_2  0xf1250004
+#define FSTV0910_MPKTDEL_ERROR_1  0xf1250002
+#define FSTV0910_MPKTDEL_LOCKB_1  0xf1250001
+
+/* IRQMASK1 */
+#define RSTV0910_IRQMASK1  0xf126
+#define FSTV0910_MPKTDEL_LOCK_1  0xf1260080
+#define FSTV0910_MFEC_LOCKB_2  0xf1260040
+#define FSTV0910_MFEC_LOCK_2  0xf1260020
+#define FSTV0910_MFEC_LOCKB_1  0xf1260010
+#define FSTV0910_MFEC_LOCK_1  0xf1260008
+#define FSTV0910_MDEMOD_LOCKB_2  0xf1260004
+#define FSTV0910_MDEMOD_LOCK_2  0xf1260002
+#define FSTV0910_MDEMOD_IRQ_2  0xf1260001
+
+/* IRQMASK0 */
+#define RSTV0910_IRQMASK0  0xf127
+#define FSTV0910_MDEMOD_LOCKB_1  0xf1270080
+#define FSTV0910_MDEMOD_LOCK_1  0xf1270040
+#define FSTV0910_MDEMOD_IRQ_1  0xf1270020
+#define FSTV0910_MBCH_ERRFLAG  0xf1270010
+#define FSTV0910_MDISEQC2_IRQ  0xf1270004
+#define FSTV0910_MDISEQC1_IRQ  0xf1270001
+
+/* I2CCFG */
+#define RSTV0910_I2CCFG  0xf129
+#define FSTV0910_I2C_FASTMODE  0xf1290008
+#define FSTV0910_I2CADDR_INC  0xf1290003
+
+/* P1_I2CRPT */
+#define RSTV0910_P1_I2CRPT  0xf12a
+#define FSTV0910_P1_I2CT_ON  0xf12a0080
+#define FSTV0910_P1_ENARPT_LEVEL  0xf12a0070
+#define FSTV0910_P1_SCLT_DELAY  0xf12a0008
+#define FSTV0910_P1_STOP_ENABLE  0xf12a0004
+#define FSTV0910_P1_STOP_SDAT2SDA  0xf12a0002
+
+/* P2_I2CRPT */
+#define RSTV0910_P2_I2CRPT  0xf12b
+#define FSTV0910_P2_I2CT_ON  0xf12b0080
+#define FSTV0910_P2_ENARPT_LEVEL  0xf12b0070
+#define FSTV0910_P2_SCLT_DELAY  0xf12b0008
+#define FSTV0910_P2_STOP_ENABLE  0xf12b0004
+#define FSTV0910_P2_STOP_SDAT2SDA  0xf12b0002
+
+/* GPIO0CFG */
+#define RSTV0910_GPIO0CFG  0xf140
+#define FSTV0910_GPIO0_OPD  0xf1400080
+#define FSTV0910_GPIO0_CONFIG  0xf140007e
+#define FSTV0910_GPIO0_XOR  0xf1400001
+
+/* GPIO1CFG */
+#define RSTV0910_GPIO1CFG  0xf141
+#define FSTV0910_GPIO1_OPD  0xf1410080
+#define FSTV0910_GPIO1_CONFIG  0xf141007e
+#define FSTV0910_GPIO1_XOR  0xf1410001
+
+/* GPIO2CFG */
+#define RSTV0910_GPIO2CFG  0xf142
+#define FSTV0910_GPIO2_OPD  0xf1420080
+#define FSTV0910_GPIO2_CONFIG  0xf142007e
+#define FSTV0910_GPIO2_XOR  0xf1420001
+
+/* GPIO3CFG */
+#define RSTV0910_GPIO3CFG  0xf143
+#define FSTV0910_GPIO3_OPD  0xf1430080
+#define FSTV0910_GPIO3_CONFIG  0xf143007e
+#define FSTV0910_GPIO3_XOR  0xf1430001
+
+/* GPIO4CFG */
+#define RSTV0910_GPIO4CFG  0xf144
+#define FSTV0910_GPIO4_OPD  0xf1440080
+#define FSTV0910_GPIO4_CONFIG  0xf144007e
+#define FSTV0910_GPIO4_XOR  0xf1440001
+
+/* GPIO5CFG */
+#define RSTV0910_GPIO5CFG  0xf145
+#define FSTV0910_GPIO5_OPD  0xf1450080
+#define FSTV0910_GPIO5_CONFIG  0xf145007e
+#define FSTV0910_GPIO5_XOR  0xf1450001
+
+/* GPIO6CFG */
+#define RSTV0910_GPIO6CFG  0xf146
+#define FSTV0910_GPIO6_OPD  0xf1460080
+#define FSTV0910_GPIO6_CONFIG  0xf146007e
+#define FSTV0910_GPIO6_XOR  0xf1460001
+
+/* GPIO7CFG */
+#define RSTV0910_GPIO7CFG  0xf147
+#define FSTV0910_GPIO7_OPD  0xf1470080
+#define FSTV0910_GPIO7_CONFIG  0xf147007e
+#define FSTV0910_GPIO7_XOR  0xf1470001
+
+/* GPIO8CFG */
+#define RSTV0910_GPIO8CFG  0xf148
+#define FSTV0910_GPIO8_OPD  0xf1480080
+#define FSTV0910_GPIO8_CONFIG  0xf148007e
+#define FSTV0910_GPIO8_XOR  0xf1480001
+
+/* GPIO9CFG */
+#define RSTV0910_GPIO9CFG  0xf149
+#define FSTV0910_GPIO9_OPD  0xf1490080
+#define FSTV0910_GPIO9_CONFIG  0xf149007e
+#define FSTV0910_GPIO9_XOR  0xf1490001
+
+/* GPIO10CFG */
+#define RSTV0910_GPIO10CFG  0xf14a
+#define FSTV0910_GPIO10_OPD  0xf14a0080
+#define FSTV0910_GPIO10_CONFIG  0xf14a007e
+#define FSTV0910_GPIO10_XOR  0xf14a0001
+
+/* GPIO11CFG */
+#define RSTV0910_GPIO11CFG  0xf14b
+#define FSTV0910_GPIO11_OPD  0xf14b0080
+#define FSTV0910_GPIO11_CONFIG  0xf14b007e
+#define FSTV0910_GPIO11_XOR  0xf14b0001
+
+/* GPIO12CFG */
+#define RSTV0910_GPIO12CFG  0xf14c
+#define FSTV0910_GPIO12_OPD  0xf14c0080
+#define FSTV0910_GPIO12_CONFIG  0xf14c007e
+#define FSTV0910_GPIO12_XOR  0xf14c0001
+
+/* GPIO13CFG */
+#define RSTV0910_GPIO13CFG  0xf14d
+#define FSTV0910_GPIO13_OPD  0xf14d0080
+#define FSTV0910_GPIO13_CONFIG  0xf14d007e
+#define FSTV0910_GPIO13_XOR  0xf14d0001
+
+/* GPIO14CFG */
+#define RSTV0910_GPIO14CFG  0xf14e
+#define FSTV0910_GPIO14_OPD  0xf14e0080
+#define FSTV0910_GPIO14_CONFIG  0xf14e007e
+#define FSTV0910_GPIO14_XOR  0xf14e0001
+
+/* GPIO15CFG */
+#define RSTV0910_GPIO15CFG  0xf14f
+#define FSTV0910_GPIO15_OPD  0xf14f0080
+#define FSTV0910_GPIO15_CONFIG  0xf14f007e
+#define FSTV0910_GPIO15_XOR  0xf14f0001
+
+/* GPIO16CFG */
+#define RSTV0910_GPIO16CFG  0xf150
+#define FSTV0910_GPIO16_OPD  0xf1500080
+#define FSTV0910_GPIO16_CONFIG  0xf150007e
+#define FSTV0910_GPIO16_XOR  0xf1500001
+
+/* GPIO17CFG */
+#define RSTV0910_GPIO17CFG  0xf151
+#define FSTV0910_GPIO17_OPD  0xf1510080
+#define FSTV0910_GPIO17_CONFIG  0xf151007e
+#define FSTV0910_GPIO17_XOR  0xf1510001
+
+/* GPIO18CFG */
+#define RSTV0910_GPIO18CFG  0xf152
+#define FSTV0910_GPIO18_OPD  0xf1520080
+#define FSTV0910_GPIO18_CONFIG  0xf152007e
+#define FSTV0910_GPIO18_XOR  0xf1520001
+
+/* GPIO19CFG */
+#define RSTV0910_GPIO19CFG  0xf153
+#define FSTV0910_GPIO19_OPD  0xf1530080
+#define FSTV0910_GPIO19_CONFIG  0xf153007e
+#define FSTV0910_GPIO19_XOR  0xf1530001
+
+/* GPIO20CFG */
+#define RSTV0910_GPIO20CFG  0xf154
+#define FSTV0910_GPIO20_OPD  0xf1540080
+#define FSTV0910_GPIO20_CONFIG  0xf154007e
+#define FSTV0910_GPIO20_XOR  0xf1540001
+
+/* GPIO21CFG */
+#define RSTV0910_GPIO21CFG  0xf155
+#define FSTV0910_GPIO21_OPD  0xf1550080
+#define FSTV0910_GPIO21_CONFIG  0xf155007e
+#define FSTV0910_GPIO21_XOR  0xf1550001
+
+/* GPIO22CFG */
+#define RSTV0910_GPIO22CFG  0xf156
+#define FSTV0910_GPIO22_OPD  0xf1560080
+#define FSTV0910_GPIO22_CONFIG  0xf156007e
+#define FSTV0910_GPIO22_XOR  0xf1560001
+
+/* STRSTATUS1 */
+#define RSTV0910_STRSTATUS1  0xf16a
+#define FSTV0910_STRSTATUS_SEL2  0xf16a00f0
+#define FSTV0910_STRSTATUS_SEL1  0xf16a000f
+
+/* STRSTATUS2 */
+#define RSTV0910_STRSTATUS2  0xf16b
+#define FSTV0910_STRSTATUS_SEL4  0xf16b00f0
+#define FSTV0910_STRSTATUS_SEL3  0xf16b000f
+
+/* STRSTATUS3 */
+#define RSTV0910_STRSTATUS3  0xf16c
+#define FSTV0910_STRSTATUS_SEL6  0xf16c00f0
+#define FSTV0910_STRSTATUS_SEL5  0xf16c000f
+
+/* FSKTFC2 */
+#define RSTV0910_FSKTFC2  0xf170
+#define FSTV0910_FSKT_KMOD  0xf17000fc
+#define FSTV0910_FSKT_CAR2  0xf1700003
+
+/* FSKTFC1 */
+#define RSTV0910_FSKTFC1  0xf171
+#define FSTV0910_FSKT_CAR1  0xf17100ff
+
+/* FSKTFC0 */
+#define RSTV0910_FSKTFC0  0xf172
+#define FSTV0910_FSKT_CAR0  0xf17200ff
+
+/* FSKTDELTAF1 */
+#define RSTV0910_FSKTDELTAF1  0xf173
+#define FSTV0910_FSKT_DELTAF1  0xf173000f
+
+/* FSKTDELTAF0 */
+#define RSTV0910_FSKTDELTAF0  0xf174
+#define FSTV0910_FSKT_DELTAF0  0xf17400ff
+
+/* FSKTCTRL */
+#define RSTV0910_FSKTCTRL  0xf175
+#define FSTV0910_FSKT_PINSEL  0xf1750080
+#define FSTV0910_FSKT_EN_SGN  0xf1750040
+#define FSTV0910_FSKT_MOD_SGN  0xf1750020
+#define FSTV0910_FSKT_MOD_EN  0xf175001c
+#define FSTV0910_FSKT_DACMODE  0xf1750003
+
+/* FSKRFC2 */
+#define RSTV0910_FSKRFC2  0xf176
+#define FSTV0910_FSKR_DETSGN  0xf1760040
+#define FSTV0910_FSKR_OUTSGN  0xf1760020
+#define FSTV0910_FSKR_KAGC  0xf176001c
+#define FSTV0910_FSKR_CAR2  0xf1760003
+
+/* FSKRFC1 */
+#define RSTV0910_FSKRFC1  0xf177
+#define FSTV0910_FSKR_CAR1  0xf17700ff
+
+/* FSKRFC0 */
+#define RSTV0910_FSKRFC0  0xf178
+#define FSTV0910_FSKR_CAR0  0xf17800ff
+
+/* FSKRK1 */
+#define RSTV0910_FSKRK1  0xf179
+#define FSTV0910_FSKR_K1_EXP  0xf17900e0
+#define FSTV0910_FSKR_K1_MANT  0xf179001f
+
+/* FSKRK2 */
+#define RSTV0910_FSKRK2  0xf17a
+#define FSTV0910_FSKR_K2_EXP  0xf17a00e0
+#define FSTV0910_FSKR_K2_MANT  0xf17a001f
+
+/* FSKRAGCR */
+#define RSTV0910_FSKRAGCR  0xf17b
+#define FSTV0910_FSKR_OUTCTL  0xf17b00c0
+#define FSTV0910_FSKR_AGC_REF  0xf17b003f
+
+/* FSKRAGC */
+#define RSTV0910_FSKRAGC  0xf17c
+#define FSTV0910_FSKR_AGC_ACCU  0xf17c00ff
+
+/* FSKRALPHA */
+#define RSTV0910_FSKRALPHA  0xf17d
+#define FSTV0910_FSKR_ALPHA_EXP  0xf17d001c
+#define FSTV0910_FSKR_ALPHA_M  0xf17d0003
+
+/* FSKRPLTH1 */
+#define RSTV0910_FSKRPLTH1  0xf17e
+#define FSTV0910_FSKR_BETA  0xf17e00f0
+#define FSTV0910_FSKR_PLL_TRESH1  0xf17e000f
+
+/* FSKRPLTH0 */
+#define RSTV0910_FSKRPLTH0  0xf17f
+#define FSTV0910_FSKR_PLL_TRESH0  0xf17f00ff
+
+/* FSKRDF1 */
+#define RSTV0910_FSKRDF1  0xf180
+#define FSTV0910_FSKR_OUT  0xf1800080
+#define FSTV0910_FSKR_STATE  0xf1800060
+#define FSTV0910_FSKR_DELTAF1  0xf180001f
+
+/* FSKRDF0 */
+#define RSTV0910_FSKRDF0  0xf181
+#define FSTV0910_FSKR_DELTAF0  0xf18100ff
+
+/* FSKRSTEPP */
+#define RSTV0910_FSKRSTEPP  0xf182
+#define FSTV0910_FSKR_STEP_PLUS  0xf18200ff
+
+/* FSKRSTEPM */
+#define RSTV0910_FSKRSTEPM  0xf183
+#define FSTV0910_FSKR_STEP_MINUS  0xf18300ff
+
+/* FSKRDET1 */
+#define RSTV0910_FSKRDET1  0xf184
+#define FSTV0910_FSKR_DETECT  0xf1840080
+#define FSTV0910_FSKR_CARDET_ACCU1  0xf184000f
+
+/* FSKRDET0 */
+#define RSTV0910_FSKRDET0  0xf185
+#define FSTV0910_FSKR_CARDET_ACCU0  0xf18500ff
+
+/* FSKRDTH1 */
+#define RSTV0910_FSKRDTH1  0xf186
+#define FSTV0910_FSKR_CARLOSS_THRESH1  0xf18600f0
+#define FSTV0910_FSKR_CARDET_THRESH1  0xf186000f
+
+/* FSKRDTH0 */
+#define RSTV0910_FSKRDTH0  0xf187
+#define FSTV0910_FSKR_CARDET_THRESH0  0xf18700ff
+
+/* FSKRLOSS */
+#define RSTV0910_FSKRLOSS  0xf188
+#define FSTV0910_FSKR_CARLOSS_THRESH0  0xf18800ff
+
+/* NCOARSE */
+#define RSTV0910_NCOARSE  0xf1b3
+#define FSTV0910_CP  0xf1b300f8
+#define FSTV0910_IDF  0xf1b30007
+
+/* NCOARSE1 */
+#define RSTV0910_NCOARSE1  0xf1b4
+#define FSTV0910_N_DIV  0xf1b400ff
+
+/* NCOARSE2 */
+#define RSTV0910_NCOARSE2  0xf1b5
+#define FSTV0910_ODF  0xf1b5003f
+
+/* SYNTCTRL */
+#define RSTV0910_SYNTCTRL  0xf1b6
+#define FSTV0910_STANDBY  0xf1b60080
+#define FSTV0910_BYPASSPLLCORE  0xf1b60040
+#define FSTV0910_STOP_PLL  0xf1b60008
+#define FSTV0910_OSCI_E  0xf1b60002
+
+/* FILTCTRL */
+#define RSTV0910_FILTCTRL  0xf1b7
+#define FSTV0910_INV_CLKFSK  0xf1b70002
+#define FSTV0910_BYPASS_APPLI  0xf1b70001
+
+/* PLLSTAT */
+#define RSTV0910_PLLSTAT  0xf1b8
+#define FSTV0910_PLLLOCK  0xf1b80001
+
+/* STOPCLK1 */
+#define RSTV0910_STOPCLK1  0xf1c2
+#define FSTV0910_INV_CLKADCI2  0xf1c20004
+#define FSTV0910_INV_CLKADCI1  0xf1c20001
+
+/* STOPCLK2 */
+#define RSTV0910_STOPCLK2  0xf1c3
+#define FSTV0910_STOP_DVBS2FEC2  0xf1c30020
+#define FSTV0910_STOP_DVBS2FEC  0xf1c30010
+#define FSTV0910_STOP_DVBS1FEC2  0xf1c30008
+#define FSTV0910_STOP_DVBS1FEC  0xf1c30004
+#define FSTV0910_STOP_DEMOD2  0xf1c30002
+#define FSTV0910_STOP_DEMOD  0xf1c30001
+
+/* PREGCTL */
+#define RSTV0910_PREGCTL  0xf1c8
+#define FSTV0910_REG3V3TO2V5_POFF  0xf1c80080
+
+/* TSTTNR0 */
+#define RSTV0910_TSTTNR0  0xf1df
+#define FSTV0910_FSK_PON  0xf1df0004
+
+/* TSTTNR1 */
+#define RSTV0910_TSTTNR1  0xf1e0
+#define FSTV0910_ADC1_PON  0xf1e00002
+
+/* TSTTNR2 */
+#define RSTV0910_TSTTNR2  0xf1e1
+#define FSTV0910_I2C_DISEQC_PON  0xf1e10020
+#define FSTV0910_DISEQC_CLKDIV  0xf1e1000f
+
+/* TSTTNR3 */
+#define RSTV0910_TSTTNR3  0xf1e2
+#define FSTV0910_ADC2_PON  0xf1e20002
+
+/* P2_IQCONST */
+#define RSTV0910_P2_IQCONST  0xf200
+#define FSTV0910_P2_CONSTEL_SELECT  0xf2000060
+#define FSTV0910_P2_IQSYMB_SEL  0xf200001f
+
+/* P2_NOSCFG */
+#define RSTV0910_P2_NOSCFG  0xf201
+#define FSTV0910_P2_DUMMYPL_NOSDATA  0xf2010020
+#define FSTV0910_P2_NOSPLH_BETA  0xf2010018
+#define FSTV0910_P2_NOSDATA_BETA  0xf2010007
+
+/* P2_ISYMB */
+#define RSTV0910_P2_ISYMB  0xf202
+#define FSTV0910_P2_I_SYMBOL  0xf20201ff
+
+/* P2_QSYMB */
+#define RSTV0910_P2_QSYMB  0xf203
+#define FSTV0910_P2_Q_SYMBOL  0xf20301ff
+
+/* P2_AGC1CFG */
+#define RSTV0910_P2_AGC1CFG  0xf204
+#define FSTV0910_P2_DC_FROZEN  0xf2040080
+#define FSTV0910_P2_DC_CORRECT  0xf2040040
+#define FSTV0910_P2_AMM_FROZEN  0xf2040020
+#define FSTV0910_P2_AMM_CORRECT  0xf2040010
+#define FSTV0910_P2_QUAD_FROZEN  0xf2040008
+#define FSTV0910_P2_QUAD_CORRECT  0xf2040004
+
+/* P2_AGC1CN */
+#define RSTV0910_P2_AGC1CN  0xf206
+#define FSTV0910_P2_AGC1_LOCKED  0xf2060080
+#define FSTV0910_P2_AGC1_MINPOWER  0xf2060010
+#define FSTV0910_P2_AGCOUT_FAST  0xf2060008
+#define FSTV0910_P2_AGCIQ_BETA  0xf2060007
+
+/* P2_AGC1REF */
+#define RSTV0910_P2_AGC1REF  0xf207
+#define FSTV0910_P2_AGCIQ_REF  0xf20700ff
+
+/* P2_IDCCOMP */
+#define RSTV0910_P2_IDCCOMP  0xf208
+#define FSTV0910_P2_IAVERAGE_ADJ  0xf20801ff
+
+/* P2_QDCCOMP */
+#define RSTV0910_P2_QDCCOMP  0xf209
+#define FSTV0910_P2_QAVERAGE_ADJ  0xf20901ff
+
+/* P2_POWERI */
+#define RSTV0910_P2_POWERI  0xf20a
+#define FSTV0910_P2_POWER_I  0xf20a00ff
+
+/* P2_POWERQ */
+#define RSTV0910_P2_POWERQ  0xf20b
+#define FSTV0910_P2_POWER_Q  0xf20b00ff
+
+/* P2_AGC1AMM */
+#define RSTV0910_P2_AGC1AMM  0xf20c
+#define FSTV0910_P2_AMM_VALUE  0xf20c00ff
+
+/* P2_AGC1QUAD */
+#define RSTV0910_P2_AGC1QUAD  0xf20d
+#define FSTV0910_P2_QUAD_VALUE  0xf20d01ff
+
+/* P2_AGCIQIN1 */
+#define RSTV0910_P2_AGCIQIN1  0xf20e
+#define FSTV0910_P2_AGCIQ_VALUE1  0xf20e00ff
+
+/* P2_AGCIQIN0 */
+#define RSTV0910_P2_AGCIQIN0  0xf20f
+#define FSTV0910_P2_AGCIQ_VALUE0  0xf20f00ff
+
+/* P2_DEMOD */
+#define RSTV0910_P2_DEMOD  0xf210
+#define FSTV0910_P2_MANUALS2_ROLLOFF  0xf2100080
+#define FSTV0910_P2_SPECINV_CONTROL  0xf2100030
+#define FSTV0910_P2_MANUALSX_ROLLOFF  0xf2100004
+#define FSTV0910_P2_ROLLOFF_CONTROL  0xf2100003
+
+/* P2_DMDMODCOD */
+#define RSTV0910_P2_DMDMODCOD  0xf211
+#define FSTV0910_P2_MANUAL_MODCOD  0xf2110080
+#define FSTV0910_P2_DEMOD_MODCOD  0xf211007c
+#define FSTV0910_P2_DEMOD_TYPE  0xf2110003
+
+/* P2_DSTATUS */
+#define RSTV0910_P2_DSTATUS  0xf212
+#define FSTV0910_P2_CAR_LOCK  0xf2120080
+#define FSTV0910_P2_TMGLOCK_QUALITY  0xf2120060
+#define FSTV0910_P2_LOCK_DEFINITIF  0xf2120008
+#define FSTV0910_P2_OVADC_DETECT  0xf2120001
+
+/* P2_DSTATUS2 */
+#define RSTV0910_P2_DSTATUS2  0xf213
+#define FSTV0910_P2_DEMOD_DELOCK  0xf2130080
+#define FSTV0910_P2_MODCODRQ_SYNCTAG  0xf2130020
+#define FSTV0910_P2_POLYPH_SATEVENT  0xf2130010
+#define FSTV0910_P2_AGC1_NOSIGNALACK  0xf2130008
+#define FSTV0910_P2_AGC2_OVERFLOW  0xf2130004
+#define FSTV0910_P2_CFR_OVERFLOW  0xf2130002
+#define FSTV0910_P2_GAMMA_OVERUNDER  0xf2130001
+
+/* P2_DMDCFGMD */
+#define RSTV0910_P2_DMDCFGMD  0xf214
+#define FSTV0910_P2_DVBS2_ENABLE  0xf2140080
+#define FSTV0910_P2_DVBS1_ENABLE  0xf2140040
+#define FSTV0910_P2_SCAN_ENABLE  0xf2140010
+#define FSTV0910_P2_CFR_AUTOSCAN  0xf2140008
+#define FSTV0910_P2_TUN_RNG  0xf2140003
+
+/* P2_DMDCFG2 */
+#define RSTV0910_P2_DMDCFG2  0xf215
+#define FSTV0910_P2_S1S2_SEQUENTIAL  0xf2150040
+#define FSTV0910_P2_INFINITE_RELOCK  0xf2150010
+
+/* P2_DMDISTATE */
+#define RSTV0910_P2_DMDISTATE  0xf216
+#define FSTV0910_P2_I2C_NORESETDMODE  0xf2160080
+#define FSTV0910_P2_I2C_DEMOD_MODE  0xf216001f
+
+/* P2_DMDT0M */
+#define RSTV0910_P2_DMDT0M  0xf217
+#define FSTV0910_P2_DMDT0_MIN  0xf21700ff
+
+/* P2_DMDSTATE */
+#define RSTV0910_P2_DMDSTATE  0xf21b
+#define FSTV0910_P2_HEADER_MODE  0xf21b0060
+
+/* P2_DMDFLYW */
+#define RSTV0910_P2_DMDFLYW  0xf21c
+#define FSTV0910_P2_I2C_IRQVAL  0xf21c00f0
+#define FSTV0910_P2_FLYWHEEL_CPT  0xf21c000f
+
+/* P2_DSTATUS3 */
+#define RSTV0910_P2_DSTATUS3  0xf21d
+#define FSTV0910_P2_CFR_ZIGZAG  0xf21d0080
+#define FSTV0910_P2_DEMOD_CFGMODE  0xf21d0060
+#define FSTV0910_P2_GAMMA_LOWBAUDRATE  0xf21d0010
+
+/* P2_DMDCFG3 */
+#define RSTV0910_P2_DMDCFG3  0xf21e
+#define FSTV0910_P2_NOSTOP_FIFOFULL  0xf21e0008
+
+/* P2_DMDCFG4 */
+#define RSTV0910_P2_DMDCFG4  0xf21f
+#define FSTV0910_P2_DIS_VITLOCK  0xf21f0080
+#define FSTV0910_P2_DIS_CLKENABLE  0xf21f0004
+
+/* P2_CORRELMANT */
+#define RSTV0910_P2_CORRELMANT  0xf220
+#define FSTV0910_P2_CORREL_MANT  0xf22000ff
+
+/* P2_CORRELABS */
+#define RSTV0910_P2_CORRELABS  0xf221
+#define FSTV0910_P2_CORREL_ABS  0xf22100ff
+
+/* P2_CORRELEXP */
+#define RSTV0910_P2_CORRELEXP  0xf222
+#define FSTV0910_P2_CORREL_ABSEXP  0xf22200f0
+#define FSTV0910_P2_CORREL_EXP  0xf222000f
+
+/* P2_PLHMODCOD */
+#define RSTV0910_P2_PLHMODCOD  0xf224
+#define FSTV0910_P2_SPECINV_DEMOD  0xf2240080
+#define FSTV0910_P2_PLH_MODCOD  0xf224007c
+#define FSTV0910_P2_PLH_TYPE  0xf2240003
+
+/* P2_DMDREG */
+#define RSTV0910_P2_DMDREG  0xf225
+#define FSTV0910_P2_DECIM_PLFRAMES  0xf2250001
+
+/* P2_AGCNADJ */
+#define RSTV0910_P2_AGCNADJ  0xf226
+#define FSTV0910_P2_RADJOFF_AGC2  0xf2260080
+#define FSTV0910_P2_RADJOFF_AGC1  0xf2260040
+#define FSTV0910_P2_AGC_NADJ  0xf226013f
+
+/* P2_AGCKS */
+#define RSTV0910_P2_AGCKS  0xf227
+#define FSTV0910_P2_RSADJ_MANUALCFG  0xf2270080
+#define FSTV0910_P2_RSADJ_CCMMODE  0xf2270040
+#define FSTV0910_P2_RADJ_SPSK  0xf227013f
+
+/* P2_AGCKQ */
+#define RSTV0910_P2_AGCKQ  0xf228
+#define FSTV0910_P2_RADJON_DVBS1  0xf2280040
+#define FSTV0910_P2_RADJ_QPSK  0xf228013f
+
+/* P2_AGCK8 */
+#define RSTV0910_P2_AGCK8  0xf229
+#define FSTV0910_P2_RADJ_8PSK  0xf229013f
+
+/* P2_AGCK16 */
+#define RSTV0910_P2_AGCK16  0xf22a
+#define FSTV0910_P2_R2ADJOFF_16APSK  0xf22a0040
+#define FSTV0910_P2_R1ADJOFF_16APSK  0xf22a0020
+#define FSTV0910_P2_RADJ_16APSK  0xf22a011f
+
+/* P2_AGCK32 */
+#define RSTV0910_P2_AGCK32  0xf22b
+#define FSTV0910_P2_R3ADJOFF_32APSK  0xf22b0080
+#define FSTV0910_P2_R2ADJOFF_32APSK  0xf22b0040
+#define FSTV0910_P2_R1ADJOFF_32APSK  0xf22b0020
+#define FSTV0910_P2_RADJ_32APSK  0xf22b011f
+
+/* P2_AGC2O */
+#define RSTV0910_P2_AGC2O  0xf22c
+#define FSTV0910_P2_CSTENV_MODE  0xf22c00c0
+#define FSTV0910_P2_AGC2_COEF  0xf22c0007
+
+/* P2_AGC2REF */
+#define RSTV0910_P2_AGC2REF  0xf22d
+#define FSTV0910_P2_AGC2_REF  0xf22d00ff
+
+/* P2_AGC1ADJ */
+#define RSTV0910_P2_AGC1ADJ  0xf22e
+#define FSTV0910_P2_AGC1_ADJUSTED  0xf22e007f
+
+/* P2_AGCRSADJ */
+#define RSTV0910_P2_AGCRSADJ  0xf22f
+#define FSTV0910_P2_RS_ADJUSTED  0xf22f007f
+
+/* P2_AGCRQADJ */
+#define RSTV0910_P2_AGCRQADJ  0xf230
+#define FSTV0910_P2_RQ_ADJUSTED  0xf230007f
+
+/* P2_AGCR8ADJ */
+#define RSTV0910_P2_AGCR8ADJ  0xf231
+#define FSTV0910_P2_R8_ADJUSTED  0xf231007f
+
+/* P2_AGCR1ADJ */
+#define RSTV0910_P2_AGCR1ADJ  0xf232
+#define FSTV0910_P2_R1_ADJUSTED  0xf232007f
+
+/* P2_AGCR2ADJ */
+#define RSTV0910_P2_AGCR2ADJ  0xf233
+#define FSTV0910_P2_R2_ADJUSTED  0xf233007f
+
+/* P2_AGCR3ADJ */
+#define RSTV0910_P2_AGCR3ADJ  0xf234
+#define FSTV0910_P2_R3_ADJUSTED  0xf234007f
+
+/* P2_AGCREFADJ */
+#define RSTV0910_P2_AGCREFADJ  0xf235
+#define FSTV0910_P2_AGC2REF_ADJUSTED  0xf235007f
+
+/* P2_AGC2I1 */
+#define RSTV0910_P2_AGC2I1  0xf236
+#define FSTV0910_P2_AGC2_INTEGRATOR1  0xf23600ff
+
+/* P2_AGC2I0 */
+#define RSTV0910_P2_AGC2I0  0xf237
+#define FSTV0910_P2_AGC2_INTEGRATOR0  0xf23700ff
+
+/* P2_CARCFG */
+#define RSTV0910_P2_CARCFG  0xf238
+#define FSTV0910_P2_ROTAON  0xf2380004
+#define FSTV0910_P2_PH_DET_ALGO  0xf2380003
+
+/* P2_ACLC */
+#define RSTV0910_P2_ACLC  0xf239
+#define FSTV0910_P2_CAR_ALPHA_MANT  0xf2390030
+#define FSTV0910_P2_CAR_ALPHA_EXP  0xf239000f
+
+/* P2_BCLC */
+#define RSTV0910_P2_BCLC  0xf23a
+#define FSTV0910_P2_CAR_BETA_MANT  0xf23a0030
+#define FSTV0910_P2_CAR_BETA_EXP  0xf23a000f
+
+/* P2_ACLCS2 */
+#define RSTV0910_P2_ACLCS2  0xf23b
+#define FSTV0910_P2_CARS2_APLHA_MANTISSE  0xf23b0030
+#define FSTV0910_P2_CARS2_ALPHA_EXP  0xf23b000f
+
+/* P2_BCLCS2 */
+#define RSTV0910_P2_BCLCS2  0xf23c
+#define FSTV0910_P2_CARS2_BETA_MANTISSE  0xf23c0030
+#define FSTV0910_P2_CARS2_BETA_EXP  0xf23c000f
+
+/* P2_CARFREQ */
+#define RSTV0910_P2_CARFREQ  0xf23d
+#define FSTV0910_P2_KC_COARSE_EXP  0xf23d00f0
+#define FSTV0910_P2_BETA_FREQ  0xf23d000f
+
+/* P2_CARHDR */
+#define RSTV0910_P2_CARHDR  0xf23e
+#define FSTV0910_P2_K_FREQ_HDR  0xf23e00ff
+
+/* P2_LDT */
+#define RSTV0910_P2_LDT  0xf23f
+#define FSTV0910_P2_CARLOCK_THRES  0xf23f01ff
+
+/* P2_LDT2 */
+#define RSTV0910_P2_LDT2  0xf240
+#define FSTV0910_P2_CARLOCK_THRES2  0xf24001ff
+
+/* P2_CFRICFG */
+#define RSTV0910_P2_CFRICFG  0xf241
+#define FSTV0910_P2_NEG_CFRSTEP  0xf2410001
+
+/* P2_CFRUP1 */
+#define RSTV0910_P2_CFRUP1  0xf242
+#define FSTV0910_P2_CFR_UP1  0xf24201ff
+
+/* P2_CFRUP0 */
+#define RSTV0910_P2_CFRUP0  0xf243
+#define FSTV0910_P2_CFR_UP0  0xf24300ff
+
+/* P2_CFRIBASE1 */
+#define RSTV0910_P2_CFRIBASE1  0xf244
+#define FSTV0910_P2_CFRINIT_BASE1  0xf24400ff
+
+/* P2_CFRIBASE0 */
+#define RSTV0910_P2_CFRIBASE0  0xf245
+#define FSTV0910_P2_CFRINIT_BASE0  0xf24500ff
+
+/* P2_CFRLOW1 */
+#define RSTV0910_P2_CFRLOW1  0xf246
+#define FSTV0910_P2_CFR_LOW1  0xf24601ff
+
+/* P2_CFRLOW0 */
+#define RSTV0910_P2_CFRLOW0  0xf247
+#define FSTV0910_P2_CFR_LOW0  0xf24700ff
+
+/* P2_CFRINIT1 */
+#define RSTV0910_P2_CFRINIT1  0xf248
+#define FSTV0910_P2_CFR_INIT1  0xf24801ff
+
+/* P2_CFRINIT0 */
+#define RSTV0910_P2_CFRINIT0  0xf249
+#define FSTV0910_P2_CFR_INIT0  0xf24900ff
+
+/* P2_CFRINC1 */
+#define RSTV0910_P2_CFRINC1  0xf24a
+#define FSTV0910_P2_MANUAL_CFRINC  0xf24a0080
+#define FSTV0910_P2_CFR_INC1  0xf24a003f
+
+/* P2_CFRINC0 */
+#define RSTV0910_P2_CFRINC0  0xf24b
+#define FSTV0910_P2_CFR_INC0  0xf24b00ff
+
+/* P2_CFR2 */
+#define RSTV0910_P2_CFR2  0xf24c
+#define FSTV0910_P2_CAR_FREQ2  0xf24c01ff
+
+/* P2_CFR1 */
+#define RSTV0910_P2_CFR1  0xf24d
+#define FSTV0910_P2_CAR_FREQ1  0xf24d00ff
+
+/* P2_CFR0 */
+#define RSTV0910_P2_CFR0  0xf24e
+#define FSTV0910_P2_CAR_FREQ0  0xf24e00ff
+
+/* P2_LDI */
+#define RSTV0910_P2_LDI  0xf24f
+#define FSTV0910_P2_LOCK_DET_INTEGR  0xf24f01ff
+
+/* P2_TMGCFG */
+#define RSTV0910_P2_TMGCFG  0xf250
+#define FSTV0910_P2_TMGLOCK_BETA  0xf25000c0
+#define FSTV0910_P2_DO_TIMING_CORR  0xf2500010
+#define FSTV0910_P2_TMG_MINFREQ  0xf2500003
+
+/* P2_RTC */
+#define RSTV0910_P2_RTC  0xf251
+#define FSTV0910_P2_TMGALPHA_EXP  0xf25100f0
+#define FSTV0910_P2_TMGBETA_EXP  0xf251000f
+
+/* P2_RTCS2 */
+#define RSTV0910_P2_RTCS2  0xf252
+#define FSTV0910_P2_TMGALPHAS2_EXP  0xf25200f0
+#define FSTV0910_P2_TMGBETAS2_EXP  0xf252000f
+
+/* P2_TMGTHRISE */
+#define RSTV0910_P2_TMGTHRISE  0xf253
+#define FSTV0910_P2_TMGLOCK_THRISE  0xf25300ff
+
+/* P2_TMGTHFALL */
+#define RSTV0910_P2_TMGTHFALL  0xf254
+#define FSTV0910_P2_TMGLOCK_THFALL  0xf25400ff
+
+/* P2_SFRUPRATIO */
+#define RSTV0910_P2_SFRUPRATIO  0xf255
+#define FSTV0910_P2_SFR_UPRATIO  0xf25500ff
+
+/* P2_SFRLOWRATIO */
+#define RSTV0910_P2_SFRLOWRATIO  0xf256
+#define FSTV0910_P2_SFR_LOWRATIO  0xf25600ff
+
+/* P2_KTTMG */
+#define RSTV0910_P2_KTTMG  0xf257
+#define FSTV0910_P2_KT_TMG_EXP  0xf25700f0
+
+/* P2_KREFTMG */
+#define RSTV0910_P2_KREFTMG  0xf258
+#define FSTV0910_P2_KREF_TMG  0xf25800ff
+
+/* P2_SFRSTEP */
+#define RSTV0910_P2_SFRSTEP  0xf259
+#define FSTV0910_P2_SFR_SCANSTEP  0xf25900f0
+#define FSTV0910_P2_SFR_CENTERSTEP  0xf259000f
+
+/* P2_TMGCFG2 */
+#define RSTV0910_P2_TMGCFG2  0xf25a
+#define FSTV0910_P2_DIS_AUTOSAMP  0xf25a0008
+#define FSTV0910_P2_SFRRATIO_FINE  0xf25a0001
+
+/* P2_KREFTMG2 */
+#define RSTV0910_P2_KREFTMG2  0xf25b
+#define FSTV0910_P2_KREF_TMG2  0xf25b00ff
+
+/* P2_TMGCFG3 */
+#define RSTV0910_P2_TMGCFG3  0xf25d
+#define FSTV0910_P2_CONT_TMGCENTER  0xf25d0008
+#define FSTV0910_P2_AUTO_GUP  0xf25d0004
+#define FSTV0910_P2_AUTO_GLOW  0xf25d0002
+
+/* P2_SFRINIT1 */
+#define RSTV0910_P2_SFRINIT1  0xf25e
+#define FSTV0910_P2_SFR_INIT1  0xf25e00ff
+
+/* P2_SFRINIT0 */
+#define RSTV0910_P2_SFRINIT0  0xf25f
+#define FSTV0910_P2_SFR_INIT0  0xf25f00ff
+
+/* P2_SFRUP1 */
+#define RSTV0910_P2_SFRUP1  0xf260
+#define FSTV0910_P2_SYMB_FREQ_UP1  0xf26000ff
+
+/* P2_SFRUP0 */
+#define RSTV0910_P2_SFRUP0  0xf261
+#define FSTV0910_P2_SYMB_FREQ_UP0  0xf26100ff
+
+/* P2_SFRLOW1 */
+#define RSTV0910_P2_SFRLOW1  0xf262
+#define FSTV0910_P2_SYMB_FREQ_LOW1  0xf26200ff
+
+/* P2_SFRLOW0 */
+#define RSTV0910_P2_SFRLOW0  0xf263
+#define FSTV0910_P2_SYMB_FREQ_LOW0  0xf26300ff
+
+/* P2_SFR3 */
+#define RSTV0910_P2_SFR3  0xf264
+#define FSTV0910_P2_SYMB_FREQ3  0xf26400ff
+
+/* P2_SFR2 */
+#define RSTV0910_P2_SFR2  0xf265
+#define FSTV0910_P2_SYMB_FREQ2  0xf26500ff
+
+/* P2_SFR1 */
+#define RSTV0910_P2_SFR1  0xf266
+#define FSTV0910_P2_SYMB_FREQ1  0xf26600ff
+
+/* P2_SFR0 */
+#define RSTV0910_P2_SFR0  0xf267
+#define FSTV0910_P2_SYMB_FREQ0  0xf26700ff
+
+/* P2_TMGREG2 */
+#define RSTV0910_P2_TMGREG2  0xf268
+#define FSTV0910_P2_TMGREG2  0xf26800ff
+
+/* P2_TMGREG1 */
+#define RSTV0910_P2_TMGREG1  0xf269
+#define FSTV0910_P2_TMGREG1  0xf26900ff
+
+/* P2_TMGREG0 */
+#define RSTV0910_P2_TMGREG0  0xf26a
+#define FSTV0910_P2_TMGREG0  0xf26a00ff
+
+/* P2_TMGLOCK1 */
+#define RSTV0910_P2_TMGLOCK1  0xf26b
+#define FSTV0910_P2_TMGLOCK_LEVEL1  0xf26b01ff
+
+/* P2_TMGLOCK0 */
+#define RSTV0910_P2_TMGLOCK0  0xf26c
+#define FSTV0910_P2_TMGLOCK_LEVEL0  0xf26c00ff
+
+/* P2_TMGOBS */
+#define RSTV0910_P2_TMGOBS  0xf26d
+#define FSTV0910_P2_ROLLOFF_STATUS  0xf26d00c0
+
+/* P2_EQUALCFG */
+#define RSTV0910_P2_EQUALCFG  0xf26f
+#define FSTV0910_P2_EQUAL_ON  0xf26f0040
+#define FSTV0910_P2_MU_EQUALDFE  0xf26f0007
+
+/* P2_EQUAI1 */
+#define RSTV0910_P2_EQUAI1  0xf270
+#define FSTV0910_P2_EQUA_ACCI1  0xf27001ff
+
+/* P2_EQUAQ1 */
+#define RSTV0910_P2_EQUAQ1  0xf271
+#define FSTV0910_P2_EQUA_ACCQ1  0xf27101ff
+
+/* P2_EQUAI2 */
+#define RSTV0910_P2_EQUAI2  0xf272
+#define FSTV0910_P2_EQUA_ACCI2  0xf27201ff
+
+/* P2_EQUAQ2 */
+#define RSTV0910_P2_EQUAQ2  0xf273
+#define FSTV0910_P2_EQUA_ACCQ2  0xf27301ff
+
+/* P2_EQUAI3 */
+#define RSTV0910_P2_EQUAI3  0xf274
+#define FSTV0910_P2_EQUA_ACCI3  0xf27401ff
+
+/* P2_EQUAQ3 */
+#define RSTV0910_P2_EQUAQ3  0xf275
+#define FSTV0910_P2_EQUA_ACCQ3  0xf27501ff
+
+/* P2_EQUAI4 */
+#define RSTV0910_P2_EQUAI4  0xf276
+#define FSTV0910_P2_EQUA_ACCI4  0xf27601ff
+
+/* P2_EQUAQ4 */
+#define RSTV0910_P2_EQUAQ4  0xf277
+#define FSTV0910_P2_EQUA_ACCQ4  0xf27701ff
+
+/* P2_EQUAI5 */
+#define RSTV0910_P2_EQUAI5  0xf278
+#define FSTV0910_P2_EQUA_ACCI5  0xf27801ff
+
+/* P2_EQUAQ5 */
+#define RSTV0910_P2_EQUAQ5  0xf279
+#define FSTV0910_P2_EQUA_ACCQ5  0xf27901ff
+
+/* P2_EQUAI6 */
+#define RSTV0910_P2_EQUAI6  0xf27a
+#define FSTV0910_P2_EQUA_ACCI6  0xf27a01ff
+
+/* P2_EQUAQ6 */
+#define RSTV0910_P2_EQUAQ6  0xf27b
+#define FSTV0910_P2_EQUA_ACCQ6  0xf27b01ff
+
+/* P2_EQUAI7 */
+#define RSTV0910_P2_EQUAI7  0xf27c
+#define FSTV0910_P2_EQUA_ACCI7  0xf27c01ff
+
+/* P2_EQUAQ7 */
+#define RSTV0910_P2_EQUAQ7  0xf27d
+#define FSTV0910_P2_EQUA_ACCQ7  0xf27d01ff
+
+/* P2_EQUAI8 */
+#define RSTV0910_P2_EQUAI8  0xf27e
+#define FSTV0910_P2_EQUA_ACCI8  0xf27e01ff
+
+/* P2_EQUAQ8 */
+#define RSTV0910_P2_EQUAQ8  0xf27f
+#define FSTV0910_P2_EQUA_ACCQ8  0xf27f01ff
+
+/* P2_NNOSDATAT1 */
+#define RSTV0910_P2_NNOSDATAT1  0xf280
+#define FSTV0910_P2_NOSDATAT_NORMED1  0xf28000ff
+
+/* P2_NNOSDATAT0 */
+#define RSTV0910_P2_NNOSDATAT0  0xf281
+#define FSTV0910_P2_NOSDATAT_NORMED0  0xf28100ff
+
+/* P2_NNOSDATA1 */
+#define RSTV0910_P2_NNOSDATA1  0xf282
+#define FSTV0910_P2_NOSDATA_NORMED1  0xf28200ff
+
+/* P2_NNOSDATA0 */
+#define RSTV0910_P2_NNOSDATA0  0xf283
+#define FSTV0910_P2_NOSDATA_NORMED0  0xf28300ff
+
+/* P2_NNOSPLHT1 */
+#define RSTV0910_P2_NNOSPLHT1  0xf284
+#define FSTV0910_P2_NOSPLHT_NORMED1  0xf28400ff
+
+/* P2_NNOSPLHT0 */
+#define RSTV0910_P2_NNOSPLHT0  0xf285
+#define FSTV0910_P2_NOSPLHT_NORMED0  0xf28500ff
+
+/* P2_NNOSPLH1 */
+#define RSTV0910_P2_NNOSPLH1  0xf286
+#define FSTV0910_P2_NOSPLH_NORMED1  0xf28600ff
+
+/* P2_NNOSPLH0 */
+#define RSTV0910_P2_NNOSPLH0  0xf287
+#define FSTV0910_P2_NOSPLH_NORMED0  0xf28700ff
+
+/* P2_NOSDATAT1 */
+#define RSTV0910_P2_NOSDATAT1  0xf288
+#define FSTV0910_P2_NOSDATAT_UNNORMED1  0xf28800ff
+
+/* P2_NOSDATAT0 */
+#define RSTV0910_P2_NOSDATAT0  0xf289
+#define FSTV0910_P2_NOSDATAT_UNNORMED0  0xf28900ff
+
+/* P2_NNOSFRAME1 */
+#define RSTV0910_P2_NNOSFRAME1  0xf28a
+#define FSTV0910_P2_NOSFRAME_NORMED1  0xf28a00ff
+
+/* P2_NNOSFRAME0 */
+#define RSTV0910_P2_NNOSFRAME0  0xf28b
+#define FSTV0910_P2_NOSFRAME_NORMED0  0xf28b00ff
+
+/* P2_NNOSRAD1 */
+#define RSTV0910_P2_NNOSRAD1  0xf28c
+#define FSTV0910_P2_NOSRADIAL_NORMED1  0xf28c00ff
+
+/* P2_NNOSRAD0 */
+#define RSTV0910_P2_NNOSRAD0  0xf28d
+#define FSTV0910_P2_NOSRADIAL_NORMED0  0xf28d00ff
+
+/* P2_NOSCFGF1 */
+#define RSTV0910_P2_NOSCFGF1  0xf28e
+#define FSTV0910_P2_LOWNOISE_MESURE  0xf28e0080
+#define FSTV0910_P2_NOS_DELFRAME  0xf28e0040
+#define FSTV0910_P2_NOSDATA_MODE  0xf28e0030
+#define FSTV0910_P2_FRAMESEL_TYPESEL  0xf28e000c
+#define FSTV0910_P2_FRAMESEL_TYPE  0xf28e0003
+
+/* P2_NOSCFGF2 */
+#define RSTV0910_P2_NOSCFGF2  0xf28f
+#define FSTV0910_P2_DIS_NOSPILOTS  0xf28f0080
+#define FSTV0910_P2_FRAMESEL_MODCODSEL  0xf28f0060
+#define FSTV0910_P2_FRAMESEL_MODCOD  0xf28f001f
+
+/* P2_CAR2CFG */
+#define RSTV0910_P2_CAR2CFG  0xf290
+#define FSTV0910_P2_ROTA2ON  0xf2900004
+#define FSTV0910_P2_PH_DET_ALGO2  0xf2900003
+
+/* P2_CFR2CFR1 */
+#define RSTV0910_P2_CFR2CFR1  0xf291
+#define FSTV0910_P2_EN_S2CAR2CENTER  0xf2910020
+#define FSTV0910_P2_CFR2TOCFR1_BETA  0xf2910007
+
+/* P2_CAR3CFG */
+#define RSTV0910_P2_CAR3CFG  0xf292
+#define FSTV0910_P2_CARRIER23_MODE  0xf29200c0
+#define FSTV0910_P2_CAR3INTERM_DVBS1  0xf2920020
+#define FSTV0910_P2_ABAMPLIF_MODE  0xf2920018
+#define FSTV0910_P2_CARRIER3_ALPHA3DL  0xf2920007
+
+/* P2_CFR22 */
+#define RSTV0910_P2_CFR22  0xf293
+#define FSTV0910_P2_CAR2_FREQ2  0xf29301ff
+
+/* P2_CFR21 */
+#define RSTV0910_P2_CFR21  0xf294
+#define FSTV0910_P2_CAR2_FREQ1  0xf29400ff
+
+/* P2_CFR20 */
+#define RSTV0910_P2_CFR20  0xf295
+#define FSTV0910_P2_CAR2_FREQ0  0xf29500ff
+
+/* P2_ACLC2S2Q */
+#define RSTV0910_P2_ACLC2S2Q  0xf297
+#define FSTV0910_P2_ENAB_SPSKSYMB  0xf2970080
+#define FSTV0910_P2_CAR2S2_Q_ALPH_M  0xf2970030
+#define FSTV0910_P2_CAR2S2_Q_ALPH_E  0xf297000f
+
+/* P2_ACLC2S28 */
+#define RSTV0910_P2_ACLC2S28  0xf298
+#define FSTV0910_P2_CAR2S2_8_ALPH_M  0xf2980030
+#define FSTV0910_P2_CAR2S2_8_ALPH_E  0xf298000f
+
+/* P2_ACLC2S216A */
+#define RSTV0910_P2_ACLC2S216A  0xf299
+#define FSTV0910_P2_CAR2S2_16A_ALPH_M  0xf2990030
+#define FSTV0910_P2_CAR2S2_16A_ALPH_E  0xf299000f
+
+/* P2_ACLC2S232A */
+#define RSTV0910_P2_ACLC2S232A  0xf29a
+#define FSTV0910_P2_CAR2S2_32A_ALPH_M  0xf29a0030
+#define FSTV0910_P2_CAR2S2_32A_ALPH_E  0xf29a000f
+
+/* P2_BCLC2S2Q */
+#define RSTV0910_P2_BCLC2S2Q  0xf29c
+#define FSTV0910_P2_CAR2S2_Q_BETA_M  0xf29c0030
+#define FSTV0910_P2_CAR2S2_Q_BETA_E  0xf29c000f
+
+/* P2_BCLC2S28 */
+#define RSTV0910_P2_BCLC2S28  0xf29d
+#define FSTV0910_P2_CAR2S2_8_BETA_M  0xf29d0030
+#define FSTV0910_P2_CAR2S2_8_BETA_E  0xf29d000f
+
+/* P2_BCLC2S216A */
+#define RSTV0910_P2_BCLC2S216A  0xf29e
+#define FSTV0910_P2_DVBS2S216A_NIP  0xf29e0080
+#define FSTV0910_P2_CAR2S2_16A_BETA_M  0xf29e0030
+#define FSTV0910_P2_CAR2S2_16A_BETA_E  0xf29e000f
+
+/* P2_BCLC2S232A */
+#define RSTV0910_P2_BCLC2S232A  0xf29f
+#define FSTV0910_P2_DVBS2S232A_NIP  0xf29f0080
+#define FSTV0910_P2_CAR2S2_32A_BETA_M  0xf29f0030
+#define FSTV0910_P2_CAR2S2_32A_BETA_E  0xf29f000f
+
+/* P2_PLROOT2 */
+#define RSTV0910_P2_PLROOT2  0xf2ac
+#define FSTV0910_P2_PLSCRAMB_MODE  0xf2ac000c
+#define FSTV0910_P2_PLSCRAMB_ROOT2  0xf2ac0003
+
+/* P2_PLROOT1 */
+#define RSTV0910_P2_PLROOT1  0xf2ad
+#define FSTV0910_P2_PLSCRAMB_ROOT1  0xf2ad00ff
+
+/* P2_PLROOT0 */
+#define RSTV0910_P2_PLROOT0  0xf2ae
+#define FSTV0910_P2_PLSCRAMB_ROOT0  0xf2ae00ff
+
+/* P2_MODCODLST0 */
+#define RSTV0910_P2_MODCODLST0  0xf2b0
+#define FSTV0910_P2_NACCES_MODCODCH  0xf2b00001
+
+/* P2_MODCODLST1 */
+#define RSTV0910_P2_MODCODLST1  0xf2b1
+#define FSTV0910_P2_SYMBRATE_FILTER  0xf2b10008
+#define FSTV0910_P2_NRESET_MODCODLST  0xf2b10004
+#define FSTV0910_P2_DIS_32PSK_9_10  0xf2b10003
+
+/* P2_MODCODLST2 */
+#define RSTV0910_P2_MODCODLST2  0xf2b2
+#define FSTV0910_P2_DIS_32PSK_8_9  0xf2b200f0
+#define FSTV0910_P2_DIS_32PSK_5_6  0xf2b2000f
+
+/* P2_MODCODLST3 */
+#define RSTV0910_P2_MODCODLST3  0xf2b3
+#define FSTV0910_P2_DIS_32PSK_4_5  0xf2b300f0
+#define FSTV0910_P2_DIS_32PSK_3_4  0xf2b3000f
+
+/* P2_MODCODLST4 */
+#define RSTV0910_P2_MODCODLST4  0xf2b4
+#define FSTV0910_P2_DUMMYPL_PILOT  0xf2b40080
+#define FSTV0910_P2_DUMMYPL_NOPILOT  0xf2b40040
+#define FSTV0910_P2_DIS_16PSK_9_10  0xf2b40030
+#define FSTV0910_P2_DIS_16PSK_8_9  0xf2b4000f
+
+/* P2_MODCODLST5 */
+#define RSTV0910_P2_MODCODLST5  0xf2b5
+#define FSTV0910_P2_DIS_16PSK_5_6  0xf2b500f0
+#define FSTV0910_P2_DIS_16PSK_4_5  0xf2b5000f
+
+/* P2_MODCODLST6 */
+#define RSTV0910_P2_MODCODLST6  0xf2b6
+#define FSTV0910_P2_DIS_16PSK_3_4  0xf2b600f0
+#define FSTV0910_P2_DIS_16PSK_2_3  0xf2b6000f
+
+/* P2_MODCODLST7 */
+#define RSTV0910_P2_MODCODLST7  0xf2b7
+#define FSTV0910_P2_MODCOD_NNOSFILTER  0xf2b70080
+#define FSTV0910_P2_DIS_8PSK_9_10  0xf2b70030
+#define FSTV0910_P2_DIS_8PSK_8_9  0xf2b7000f
+
+/* P2_MODCODLST8 */
+#define RSTV0910_P2_MODCODLST8  0xf2b8
+#define FSTV0910_P2_DIS_8PSK_5_6  0xf2b800f0
+#define FSTV0910_P2_DIS_8PSK_3_4  0xf2b8000f
+
+/* P2_MODCODLST9 */
+#define RSTV0910_P2_MODCODLST9  0xf2b9
+#define FSTV0910_P2_DIS_8PSK_2_3  0xf2b900f0
+#define FSTV0910_P2_DIS_8PSK_3_5  0xf2b9000f
+
+/* P2_MODCODLSTA */
+#define RSTV0910_P2_MODCODLSTA  0xf2ba
+#define FSTV0910_P2_NOSFILTER_LIMITE  0xf2ba0080
+#define FSTV0910_P2_DIS_QPSK_9_10  0xf2ba0030
+#define FSTV0910_P2_DIS_QPSK_8_9  0xf2ba000f
+
+/* P2_MODCODLSTB */
+#define RSTV0910_P2_MODCODLSTB  0xf2bb
+#define FSTV0910_P2_DIS_QPSK_5_6  0xf2bb00f0
+#define FSTV0910_P2_DIS_QPSK_4_5  0xf2bb000f
+
+/* P2_MODCODLSTC */
+#define RSTV0910_P2_MODCODLSTC  0xf2bc
+#define FSTV0910_P2_DIS_QPSK_3_4  0xf2bc00f0
+#define FSTV0910_P2_DIS_QPSK_2_3  0xf2bc000f
+
+/* P2_MODCODLSTD */
+#define RSTV0910_P2_MODCODLSTD  0xf2bd
+#define FSTV0910_P2_DIS_QPSK_3_5  0xf2bd00f0
+#define FSTV0910_P2_DIS_QPSK_1_2  0xf2bd000f
+
+/* P2_MODCODLSTE */
+#define RSTV0910_P2_MODCODLSTE  0xf2be
+#define FSTV0910_P2_DIS_QPSK_2_5  0xf2be00f0
+#define FSTV0910_P2_DIS_QPSK_1_3  0xf2be000f
+
+/* P2_MODCODLSTF */
+#define RSTV0910_P2_MODCODLSTF  0xf2bf
+#define FSTV0910_P2_DIS_QPSK_1_4  0xf2bf00f0
+#define FSTV0910_P2_DEMOD_INVMODLST  0xf2bf0008
+#define FSTV0910_P2_DEMODOUT_ENABLE  0xf2bf0004
+#define FSTV0910_P2_DDEMOD_NSET  0xf2bf0002
+#define FSTV0910_P2_MODCOD_NSTOCK  0xf2bf0001
+
+/* P2_GAUSSR0 */
+#define RSTV0910_P2_GAUSSR0  0xf2c0
+#define FSTV0910_P2_EN_CCIMODE  0xf2c00080
+#define FSTV0910_P2_R0_GAUSSIEN  0xf2c0007f
+
+/* P2_CCIR0 */
+#define RSTV0910_P2_CCIR0  0xf2c1
+#define FSTV0910_P2_CCIDETECT_PLHONLY  0xf2c10080
+#define FSTV0910_P2_R0_CCI  0xf2c1007f
+
+/* P2_CCIQUANT */
+#define RSTV0910_P2_CCIQUANT  0xf2c2
+#define FSTV0910_P2_CCI_BETA  0xf2c200e0
+#define FSTV0910_P2_CCI_QUANT  0xf2c2001f
+
+/* P2_CCITHRES */
+#define RSTV0910_P2_CCITHRES  0xf2c3
+#define FSTV0910_P2_CCI_THRESHOLD  0xf2c300ff
+
+/* P2_CCIACC */
+#define RSTV0910_P2_CCIACC  0xf2c4
+#define FSTV0910_P2_CCI_VALUE  0xf2c400ff
+
+/* P2_DSTATUS4 */
+#define RSTV0910_P2_DSTATUS4  0xf2c5
+#define FSTV0910_P2_RAINFADE_DETECT  0xf2c50080
+#define FSTV0910_P2_NOTHRES2_FAIL  0xf2c50040
+#define FSTV0910_P2_NOTHRES1_FAIL  0xf2c50020
+#define FSTV0910_P2_DMDPROG_ERROR  0xf2c50004
+#define FSTV0910_P2_CSTENV_DETECT  0xf2c50002
+#define FSTV0910_P2_DETECTION_TRIAX  0xf2c50001
+
+/* P2_DMDRESCFG */
+#define RSTV0910_P2_DMDRESCFG  0xf2c6
+#define FSTV0910_P2_DMDRES_RESET  0xf2c60080
+#define FSTV0910_P2_DMDRES_STRALL  0xf2c60008
+#define FSTV0910_P2_DMDRES_NEWONLY  0xf2c60004
+#define FSTV0910_P2_DMDRES_NOSTORE  0xf2c60002
+
+/* P2_DMDRESADR */
+#define RSTV0910_P2_DMDRESADR  0xf2c7
+#define FSTV0910_P2_DMDRES_VALIDCFR  0xf2c70040
+#define FSTV0910_P2_DMDRES_MEMFULL  0xf2c70030
+#define FSTV0910_P2_DMDRES_RESNBR  0xf2c7000f
+
+/* P2_DMDRESDATA7 */
+#define RSTV0910_P2_DMDRESDATA7  0xf2c8
+#define FSTV0910_P2_DMDRES_DATA7  0xf2c800ff
+
+/* P2_DMDRESDATA6 */
+#define RSTV0910_P2_DMDRESDATA6  0xf2c9
+#define FSTV0910_P2_DMDRES_DATA6  0xf2c900ff
+
+/* P2_DMDRESDATA5 */
+#define RSTV0910_P2_DMDRESDATA5  0xf2ca
+#define FSTV0910_P2_DMDRES_DATA5  0xf2ca00ff
+
+/* P2_DMDRESDATA4 */
+#define RSTV0910_P2_DMDRESDATA4  0xf2cb
+#define FSTV0910_P2_DMDRES_DATA4  0xf2cb00ff
+
+/* P2_DMDRESDATA3 */
+#define RSTV0910_P2_DMDRESDATA3  0xf2cc
+#define FSTV0910_P2_DMDRES_DATA3  0xf2cc00ff
+
+/* P2_DMDRESDATA2 */
+#define RSTV0910_P2_DMDRESDATA2  0xf2cd
+#define FSTV0910_P2_DMDRES_DATA2  0xf2cd00ff
+
+/* P2_DMDRESDATA1 */
+#define RSTV0910_P2_DMDRESDATA1  0xf2ce
+#define FSTV0910_P2_DMDRES_DATA1  0xf2ce00ff
+
+/* P2_DMDRESDATA0 */
+#define RSTV0910_P2_DMDRESDATA0  0xf2cf
+#define FSTV0910_P2_DMDRES_DATA0  0xf2cf00ff
+
+/* P2_FFEI1 */
+#define RSTV0910_P2_FFEI1  0xf2d0
+#define FSTV0910_P2_FFE_ACCI1  0xf2d001ff
+
+/* P2_FFEQ1 */
+#define RSTV0910_P2_FFEQ1  0xf2d1
+#define FSTV0910_P2_FFE_ACCQ1  0xf2d101ff
+
+/* P2_FFEI2 */
+#define RSTV0910_P2_FFEI2  0xf2d2
+#define FSTV0910_P2_FFE_ACCI2  0xf2d201ff
+
+/* P2_FFEQ2 */
+#define RSTV0910_P2_FFEQ2  0xf2d3
+#define FSTV0910_P2_FFE_ACCQ2  0xf2d301ff
+
+/* P2_FFEI3 */
+#define RSTV0910_P2_FFEI3  0xf2d4
+#define FSTV0910_P2_FFE_ACCI3  0xf2d401ff
+
+/* P2_FFEQ3 */
+#define RSTV0910_P2_FFEQ3  0xf2d5
+#define FSTV0910_P2_FFE_ACCQ3  0xf2d501ff
+
+/* P2_FFEI4 */
+#define RSTV0910_P2_FFEI4  0xf2d6
+#define FSTV0910_P2_FFE_ACCI4  0xf2d601ff
+
+/* P2_FFEQ4 */
+#define RSTV0910_P2_FFEQ4  0xf2d7
+#define FSTV0910_P2_FFE_ACCQ4  0xf2d701ff
+
+/* P2_FFECFG */
+#define RSTV0910_P2_FFECFG  0xf2d8
+#define FSTV0910_P2_EQUALFFE_ON  0xf2d80040
+#define FSTV0910_P2_EQUAL_USEDSYMB  0xf2d80030
+#define FSTV0910_P2_MU_EQUALFFE  0xf2d80007
+
+/* P2_TNRCFG2 */
+#define RSTV0910_P2_TNRCFG2  0xf2e1
+#define FSTV0910_P2_TUN_IQSWAP  0xf2e10080
+
+/* P2_SMAPCOEF7 */
+#define RSTV0910_P2_SMAPCOEF7  0xf300
+#define FSTV0910_P2_DIS_QSCALE  0xf3000080
+#define FSTV0910_P2_SMAPCOEF_Q_LLR12  0xf300017f
+
+/* P2_SMAPCOEF6 */
+#define RSTV0910_P2_SMAPCOEF6  0xf301
+#define FSTV0910_P2_DIS_AGC2SCALE  0xf3010080
+#define FSTV0910_P2_ADJ_8PSKLLR1  0xf3010004
+#define FSTV0910_P2_OLD_8PSKLLR1  0xf3010002
+#define FSTV0910_P2_DIS_AB8PSK  0xf3010001
+
+/* P2_SMAPCOEF5 */
+#define RSTV0910_P2_SMAPCOEF5  0xf302
+#define FSTV0910_P2_DIS_8SCALE  0xf3020080
+#define FSTV0910_P2_SMAPCOEF_8P_LLR23  0xf302017f
+
+/* P2_SMAPCOEF4 */
+#define RSTV0910_P2_SMAPCOEF4  0xf303
+#define FSTV0910_P2_SMAPCOEF_16APSK_LLR12  0xf303017f
+
+/* P2_SMAPCOEF3 */
+#define RSTV0910_P2_SMAPCOEF3  0xf304
+#define FSTV0910_P2_SMAPCOEF_16APSK_LLR34  0xf304017f
+
+/* P2_SMAPCOEF2 */
+#define RSTV0910_P2_SMAPCOEF2  0xf305
+#define FSTV0910_P2_SMAPCOEF_32APSK_R2R3  0xf30501f0
+#define FSTV0910_P2_SMAPCOEF_32APSK_LLR2  0xf305010f
+
+/* P2_SMAPCOEF1 */
+#define RSTV0910_P2_SMAPCOEF1  0xf306
+#define FSTV0910_P2_DIS_16SCALE  0xf3060080
+#define FSTV0910_P2_SMAPCOEF_32_LLR34  0xf306017f
+
+/* P2_SMAPCOEF0 */
+#define RSTV0910_P2_SMAPCOEF0  0xf307
+#define FSTV0910_P2_DIS_32SCALE  0xf3070080
+#define FSTV0910_P2_SMAPCOEF_32_LLR15  0xf307017f
+
+/* P2_NOSTHRES1 */
+#define RSTV0910_P2_NOSTHRES1  0xf309
+#define FSTV0910_P2_NOS_THRESHOLD1  0xf30900ff
+
+/* P2_NOSTHRES2 */
+#define RSTV0910_P2_NOSTHRES2  0xf30a
+#define FSTV0910_P2_NOS_THRESHOLD2  0xf30a00ff
+
+/* P2_NOSDIFF1 */
+#define RSTV0910_P2_NOSDIFF1  0xf30b
+#define FSTV0910_P2_NOSTHRES1_DIFF  0xf30b00ff
+
+/* P2_RAINFADE */
+#define RSTV0910_P2_RAINFADE  0xf30c
+#define FSTV0910_P2_NOSTHRES_DATAT  0xf30c0080
+#define FSTV0910_P2_RAINFADE_CNLIMIT  0xf30c0070
+#define FSTV0910_P2_RAINFADE_TIMEOUT  0xf30c0007
+
+/* P2_NOSRAMCFG */
+#define RSTV0910_P2_NOSRAMCFG  0xf30d
+#define FSTV0910_P2_NOSRAM_ACTIVATION  0xf30d0030
+#define FSTV0910_P2_NOSRAM_CNRONLY  0xf30d0008
+#define FSTV0910_P2_NOSRAM_LGNCNR1  0xf30d0007
+
+/* P2_NOSRAMPOS */
+#define RSTV0910_P2_NOSRAMPOS  0xf30e
+#define FSTV0910_P2_NOSRAM_LGNCNR0  0xf30e00f0
+#define FSTV0910_P2_NOSRAM_VALIDE  0xf30e0004
+#define FSTV0910_P2_NOSRAM_CNRVAL1  0xf30e0003
+
+/* P2_NOSRAMVAL */
+#define RSTV0910_P2_NOSRAMVAL  0xf30f
+#define FSTV0910_P2_NOSRAM_CNRVAL0  0xf30f00ff
+
+/* P2_DMDPLHSTAT */
+#define RSTV0910_P2_DMDPLHSTAT  0xf320
+#define FSTV0910_P2_PLH_STATISTIC  0xf32000ff
+
+/* P2_LOCKTIME3 */
+#define RSTV0910_P2_LOCKTIME3  0xf322
+#define FSTV0910_P2_DEMOD_LOCKTIME3  0xf32200ff
+
+/* P2_LOCKTIME2 */
+#define RSTV0910_P2_LOCKTIME2  0xf323
+#define FSTV0910_P2_DEMOD_LOCKTIME2  0xf32300ff
+
+/* P2_LOCKTIME1 */
+#define RSTV0910_P2_LOCKTIME1  0xf324
+#define FSTV0910_P2_DEMOD_LOCKTIME1  0xf32400ff
+
+/* P2_LOCKTIME0 */
+#define RSTV0910_P2_LOCKTIME0  0xf325
+#define FSTV0910_P2_DEMOD_LOCKTIME0  0xf32500ff
+
+/* P2_VITSCALE */
+#define RSTV0910_P2_VITSCALE  0xf332
+#define FSTV0910_P2_NVTH_NOSRANGE  0xf3320080
+#define FSTV0910_P2_VERROR_MAXMODE  0xf3320040
+#define FSTV0910_P2_NSLOWSN_LOCKED  0xf3320008
+#define FSTV0910_P2_DIS_RSFLOCK  0xf3320002
+
+/* P2_FECM */
+#define RSTV0910_P2_FECM  0xf333
+#define FSTV0910_P2_DSS_DVB  0xf3330080
+#define FSTV0910_P2_DSS_SRCH  0xf3330010
+#define FSTV0910_P2_SYNCVIT  0xf3330002
+#define FSTV0910_P2_IQINV  0xf3330001
+
+/* P2_VTH12 */
+#define RSTV0910_P2_VTH12  0xf334
+#define FSTV0910_P2_VTH12  0xf33400ff
+
+/* P2_VTH23 */
+#define RSTV0910_P2_VTH23  0xf335
+#define FSTV0910_P2_VTH23  0xf33500ff
+
+/* P2_VTH34 */
+#define RSTV0910_P2_VTH34  0xf336
+#define FSTV0910_P2_VTH34  0xf33600ff
+
+/* P2_VTH56 */
+#define RSTV0910_P2_VTH56  0xf337
+#define FSTV0910_P2_VTH56  0xf33700ff
+
+/* P2_VTH67 */
+#define RSTV0910_P2_VTH67  0xf338
+#define FSTV0910_P2_VTH67  0xf33800ff
+
+/* P2_VTH78 */
+#define RSTV0910_P2_VTH78  0xf339
+#define FSTV0910_P2_VTH78  0xf33900ff
+
+/* P2_VITCURPUN */
+#define RSTV0910_P2_VITCURPUN  0xf33a
+#define FSTV0910_P2_VIT_CURPUN  0xf33a001f
+
+/* P2_VERROR */
+#define RSTV0910_P2_VERROR  0xf33b
+#define FSTV0910_P2_REGERR_VIT  0xf33b00ff
+
+/* P2_PRVIT */
+#define RSTV0910_P2_PRVIT  0xf33c
+#define FSTV0910_P2_DIS_VTHLOCK  0xf33c0040
+#define FSTV0910_P2_E7_8VIT  0xf33c0020
+#define FSTV0910_P2_E6_7VIT  0xf33c0010
+#define FSTV0910_P2_E5_6VIT  0xf33c0008
+#define FSTV0910_P2_E3_4VIT  0xf33c0004
+#define FSTV0910_P2_E2_3VIT  0xf33c0002
+#define FSTV0910_P2_E1_2VIT  0xf33c0001
+
+/* P2_VAVSRVIT */
+#define RSTV0910_P2_VAVSRVIT  0xf33d
+#define FSTV0910_P2_AMVIT  0xf33d0080
+#define FSTV0910_P2_FROZENVIT  0xf33d0040
+#define FSTV0910_P2_SNVIT  0xf33d0030
+#define FSTV0910_P2_TOVVIT  0xf33d000c
+#define FSTV0910_P2_HYPVIT  0xf33d0003
+
+/* P2_VSTATUSVIT */
+#define RSTV0910_P2_VSTATUSVIT  0xf33e
+#define FSTV0910_P2_PRFVIT  0xf33e0010
+#define FSTV0910_P2_LOCKEDVIT  0xf33e0008
+
+/* P2_VTHINUSE */
+#define RSTV0910_P2_VTHINUSE  0xf33f
+#define FSTV0910_P2_VIT_INUSE  0xf33f00ff
+
+/* P2_KDIV12 */
+#define RSTV0910_P2_KDIV12  0xf340
+#define FSTV0910_P2_K_DIVIDER_12  0xf340007f
+
+/* P2_KDIV23 */
+#define RSTV0910_P2_KDIV23  0xf341
+#define FSTV0910_P2_K_DIVIDER_23  0xf341007f
+
+/* P2_KDIV34 */
+#define RSTV0910_P2_KDIV34  0xf342
+#define FSTV0910_P2_K_DIVIDER_34  0xf342007f
+
+/* P2_KDIV56 */
+#define RSTV0910_P2_KDIV56  0xf343
+#define FSTV0910_P2_K_DIVIDER_56  0xf343007f
+
+/* P2_KDIV67 */
+#define RSTV0910_P2_KDIV67  0xf344
+#define FSTV0910_P2_K_DIVIDER_67  0xf344007f
+
+/* P2_KDIV78 */
+#define RSTV0910_P2_KDIV78  0xf345
+#define FSTV0910_P2_K_DIVIDER_78  0xf345007f
+
+/* P2_TSPIDFLT1 */
+#define RSTV0910_P2_TSPIDFLT1  0xf346
+#define FSTV0910_P2_PIDFLT_ADDR  0xf34600ff
+
+/* P2_TSPIDFLT0 */
+#define RSTV0910_P2_TSPIDFLT0  0xf347
+#define FSTV0910_P2_PIDFLT_DATA  0xf34700ff
+
+/* P2_PDELCTRL0 */
+#define RSTV0910_P2_PDELCTRL0  0xf34f
+#define FSTV0910_P2_ISIOBS_MODE  0xf34f0030
+
+/* P2_PDELCTRL1 */
+#define RSTV0910_P2_PDELCTRL1  0xf350
+#define FSTV0910_P2_INV_MISMASK  0xf3500080
+#define FSTV0910_P2_FILTER_EN  0xf3500020
+#define FSTV0910_P2_HYSTEN  0xf3500008
+#define FSTV0910_P2_HYSTSWRST  0xf3500004
+#define FSTV0910_P2_EN_MIS00  0xf3500002
+#define FSTV0910_P2_ALGOSWRST  0xf3500001
+
+/* P2_PDELCTRL2 */
+#define RSTV0910_P2_PDELCTRL2  0xf351
+#define FSTV0910_P2_FORCE_CONTINUOUS  0xf3510080
+#define FSTV0910_P2_RESET_UPKO_COUNT  0xf3510040
+#define FSTV0910_P2_USER_PKTDELIN_NB  0xf3510020
+#define FSTV0910_P2_FRAME_MODE  0xf3510002
+
+/* P2_HYSTTHRESH */
+#define RSTV0910_P2_HYSTTHRESH  0xf354
+#define FSTV0910_P2_DELIN_LOCKTHRES  0xf35400f0
+#define FSTV0910_P2_DELIN_UNLOCKTHRES  0xf354000f
+
+/* P2_UPLCCST0 */
+#define RSTV0910_P2_UPLCCST0  0xf358
+#define FSTV0910_P2_UPL_CST0  0xf35800f8
+#define FSTV0910_P2_UPL_MODE  0xf3580007
+
+/* P2_ISIENTRY */
+#define RSTV0910_P2_ISIENTRY  0xf35e
+#define FSTV0910_P2_ISI_ENTRY  0xf35e00ff
+
+/* P2_ISIBITENA */
+#define RSTV0910_P2_ISIBITENA  0xf35f
+#define FSTV0910_P2_ISI_BIT_EN  0xf35f00ff
+
+/* P2_MATSTR1 */
+#define RSTV0910_P2_MATSTR1  0xf360
+#define FSTV0910_P2_MATYPE_CURRENT1  0xf36000ff
+
+/* P2_MATSTR0 */
+#define RSTV0910_P2_MATSTR0  0xf361
+#define FSTV0910_P2_MATYPE_CURRENT0  0xf36100ff
+
+/* P2_UPLSTR1 */
+#define RSTV0910_P2_UPLSTR1  0xf362
+#define FSTV0910_P2_UPL_CURRENT1  0xf36200ff
+
+/* P2_UPLSTR0 */
+#define RSTV0910_P2_UPLSTR0  0xf363
+#define FSTV0910_P2_UPL_CURRENT0  0xf36300ff
+
+/* P2_DFLSTR1 */
+#define RSTV0910_P2_DFLSTR1  0xf364
+#define FSTV0910_P2_DFL_CURRENT1  0xf36400ff
+
+/* P2_DFLSTR0 */
+#define RSTV0910_P2_DFLSTR0  0xf365
+#define FSTV0910_P2_DFL_CURRENT0  0xf36500ff
+
+/* P2_SYNCSTR */
+#define RSTV0910_P2_SYNCSTR  0xf366
+#define FSTV0910_P2_SYNC_CURRENT  0xf36600ff
+
+/* P2_SYNCDSTR1 */
+#define RSTV0910_P2_SYNCDSTR1  0xf367
+#define FSTV0910_P2_SYNCD_CURRENT1  0xf36700ff
+
+/* P2_SYNCDSTR0 */
+#define RSTV0910_P2_SYNCDSTR0  0xf368
+#define FSTV0910_P2_SYNCD_CURRENT0  0xf36800ff
+
+/* P2_PDELSTATUS1 */
+#define RSTV0910_P2_PDELSTATUS1  0xf369
+#define FSTV0910_P2_PKTDELIN_DELOCK  0xf3690080
+#define FSTV0910_P2_SYNCDUPDFL_BADDFL  0xf3690040
+#define FSTV0910_P2_UNACCEPTED_STREAM  0xf3690010
+#define FSTV0910_P2_BCH_ERROR_FLAG  0xf3690008
+#define FSTV0910_P2_PKTDELIN_LOCK  0xf3690002
+#define FSTV0910_P2_FIRST_LOCK  0xf3690001
+
+/* P2_PDELSTATUS2 */
+#define RSTV0910_P2_PDELSTATUS2  0xf36a
+#define FSTV0910_P2_FRAME_MODCOD  0xf36a007c
+#define FSTV0910_P2_FRAME_TYPE  0xf36a0003
+
+/* P2_BBFCRCKO1 */
+#define RSTV0910_P2_BBFCRCKO1  0xf36b
+#define FSTV0910_P2_BBHCRC_KOCNT1  0xf36b00ff
+
+/* P2_BBFCRCKO0 */
+#define RSTV0910_P2_BBFCRCKO0  0xf36c
+#define FSTV0910_P2_BBHCRC_KOCNT0  0xf36c00ff
+
+/* P2_UPCRCKO1 */
+#define RSTV0910_P2_UPCRCKO1  0xf36d
+#define FSTV0910_P2_PKTCRC_KOCNT1  0xf36d00ff
+
+/* P2_UPCRCKO0 */
+#define RSTV0910_P2_UPCRCKO0  0xf36e
+#define FSTV0910_P2_PKTCRC_KOCNT0  0xf36e00ff
+
+/* P2_PDELCTRL3 */
+#define RSTV0910_P2_PDELCTRL3  0xf36f
+#define FSTV0910_P2_NOFIFO_BCHERR  0xf36f0020
+#define FSTV0910_P2_PKTDELIN_DELACMERR  0xf36f0010
+
+/* P2_TSSTATEM */
+#define RSTV0910_P2_TSSTATEM  0xf370
+#define FSTV0910_P2_TSDIL_ON  0xf3700080
+#define FSTV0910_P2_TSRS_ON  0xf3700020
+#define FSTV0910_P2_TSDESCRAMB_ON  0xf3700010
+#define FSTV0910_P2_TSFRAME_MODE  0xf3700008
+#define FSTV0910_P2_TS_DISABLE  0xf3700004
+#define FSTV0910_P2_TSACM_MODE  0xf3700002
+#define FSTV0910_P2_TSOUT_NOSYNC  0xf3700001
+
+/* P2_TSSTATEL */
+#define RSTV0910_P2_TSSTATEL  0xf371
+#define FSTV0910_P2_TSNOSYNCBYTE  0xf3710080
+#define FSTV0910_P2_TSPARITY_ON  0xf3710040
+#define FSTV0910_P2_TSISSYI_ON  0xf3710008
+#define FSTV0910_P2_TSNPD_ON  0xf3710004
+#define FSTV0910_P2_TSCRC8_ON  0xf3710002
+#define FSTV0910_P2_TSDSS_PACKET  0xf3710001
+
+/* P2_TSCFGH */
+#define RSTV0910_P2_TSCFGH  0xf372
+#define FSTV0910_P2_TSFIFO_DVBCI  0xf3720080
+#define FSTV0910_P2_TSFIFO_SERIAL  0xf3720040
+#define FSTV0910_P2_TSFIFO_TEIUPDATE  0xf3720020
+#define FSTV0910_P2_TSFIFO_DUTY50  0xf3720010
+#define FSTV0910_P2_TSFIFO_HSGNLOUT  0xf3720008
+#define FSTV0910_P2_TSFIFO_ERRMODE  0xf3720006
+#define FSTV0910_P2_RST_HWARE  0xf3720001
+
+/* P2_TSCFGM */
+#define RSTV0910_P2_TSCFGM  0xf373
+#define FSTV0910_P2_TSFIFO_MANSPEED  0xf37300c0
+#define FSTV0910_P2_TSFIFO_PERMDATA  0xf3730020
+#define FSTV0910_P2_TSFIFO_NONEWSGNL  0xf3730010
+#define FSTV0910_P2_TSFIFO_INVDATA  0xf3730001
+
+/* P2_TSCFGL */
+#define RSTV0910_P2_TSCFGL  0xf374
+#define FSTV0910_P2_TSFIFO_BCLKDEL1CK  0xf37400c0
+#define FSTV0910_P2_BCHERROR_MODE  0xf3740030
+#define FSTV0910_P2_TSFIFO_NSGNL2DATA  0xf3740008
+#define FSTV0910_P2_TSFIFO_EMBINDVB  0xf3740004
+#define FSTV0910_P2_TSFIFO_BITSPEED  0xf3740003
+
+/* P2_TSSYNC */
+#define RSTV0910_P2_TSSYNC  0xf375
+#define FSTV0910_P2_TSFIFO_SYNCMODE  0xf3750018
+
+/* P2_TSINSDELH */
+#define RSTV0910_P2_TSINSDELH  0xf376
+#define FSTV0910_P2_TSDEL_SYNCBYTE  0xf3760080
+#define FSTV0910_P2_TSDEL_XXHEADER  0xf3760040
+#define FSTV0910_P2_TSDEL_DATAFIELD  0xf3760010
+#define FSTV0910_P2_TSINSDEL_RSPARITY  0xf3760002
+#define FSTV0910_P2_TSINSDEL_CRC8  0xf3760001
+
+/* P2_TSINSDELM */
+#define RSTV0910_P2_TSINSDELM  0xf377
+#define FSTV0910_P2_TSINS_EMODCOD  0xf3770010
+#define FSTV0910_P2_TSINS_TOKEN  0xf3770008
+#define FSTV0910_P2_TSINS_XXXERR  0xf3770004
+#define FSTV0910_P2_TSINS_MATYPE  0xf3770002
+#define FSTV0910_P2_TSINS_UPL  0xf3770001
+
+/* P2_TSINSDELL */
+#define RSTV0910_P2_TSINSDELL  0xf378
+#define FSTV0910_P2_TSINS_DFL  0xf3780080
+#define FSTV0910_P2_TSINS_SYNCD  0xf3780040
+#define FSTV0910_P2_TSINS_BLOCLEN  0xf3780020
+#define FSTV0910_P2_TSINS_SIGPCOUNT  0xf3780010
+#define FSTV0910_P2_TSINS_FIFO  0xf3780008
+#define FSTV0910_P2_TSINS_REALPACK  0xf3780004
+#define FSTV0910_P2_TSINS_TSCONFIG  0xf3780002
+#define FSTV0910_P2_TSINS_LATENCY  0xf3780001
+
+/* P2_TSDIVN */
+#define RSTV0910_P2_TSDIVN  0xf379
+#define FSTV0910_P2_TSFIFO_SPEEDMODE  0xf37900c0
+#define FSTV0910_P2_TSFIFO_RISEOK  0xf3790007
+
+/* P2_TSCFG4 */
+#define RSTV0910_P2_TSCFG4  0xf37a
+#define FSTV0910_P2_TSFIFO_TSSPEEDMODE  0xf37a00c0
+
+/* P2_TSSPEED */
+#define RSTV0910_P2_TSSPEED  0xf380
+#define FSTV0910_P2_TSFIFO_OUTSPEED  0xf38000ff
+
+/* P2_TSSTATUS */
+#define RSTV0910_P2_TSSTATUS  0xf381
+#define FSTV0910_P2_TSFIFO_LINEOK  0xf3810080
+#define FSTV0910_P2_TSFIFO_ERROR  0xf3810040
+#define FSTV0910_P2_TSFIFO_NOSYNC  0xf3810010
+#define FSTV0910_P2_TSREGUL_ERROR  0xf3810004
+#define FSTV0910_P2_DIL_READY  0xf3810001
+
+/* P2_TSSTATUS2 */
+#define RSTV0910_P2_TSSTATUS2  0xf382
+#define FSTV0910_P2_TSFIFO_DEMODSEL  0xf3820080
+#define FSTV0910_P2_TSFIFOSPEED_STORE  0xf3820040
+#define FSTV0910_P2_DILXX_RESET  0xf3820020
+#define FSTV0910_P2_SCRAMBDETECT  0xf3820002
+
+/* P2_TSBITRATE1 */
+#define RSTV0910_P2_TSBITRATE1  0xf383
+#define FSTV0910_P2_TSFIFO_BITRATE1  0xf38300ff
+
+/* P2_TSBITRATE0 */
+#define RSTV0910_P2_TSBITRATE0  0xf384
+#define FSTV0910_P2_TSFIFO_BITRATE0  0xf38400ff
+
+/* P2_TSPACKLEN1 */
+#define RSTV0910_P2_TSPACKLEN1  0xf385
+#define FSTV0910_P2_TSFIFO_PACKCPT  0xf38500e0
+
+/* P2_TSDLY2 */
+#define RSTV0910_P2_TSDLY2  0xf389
+#define FSTV0910_P2_SOFFIFO_LATENCY2  0xf389000f
+
+/* P2_TSDLY1 */
+#define RSTV0910_P2_TSDLY1  0xf38a
+#define FSTV0910_P2_SOFFIFO_LATENCY1  0xf38a00ff
+
+/* P2_TSDLY0 */
+#define RSTV0910_P2_TSDLY0  0xf38b
+#define FSTV0910_P2_SOFFIFO_LATENCY0  0xf38b00ff
+
+/* P2_TSNPDAV */
+#define RSTV0910_P2_TSNPDAV  0xf38c
+#define FSTV0910_P2_TSNPD_AVERAGE  0xf38c00ff
+
+/* P2_TSBUFSTAT2 */
+#define RSTV0910_P2_TSBUFSTAT2  0xf38d
+#define FSTV0910_P2_TSISCR_3BYTES  0xf38d0080
+#define FSTV0910_P2_TSISCR_NEWDATA  0xf38d0040
+#define FSTV0910_P2_TSISCR_BUFSTAT2  0xf38d003f
+
+/* P2_TSBUFSTAT1 */
+#define RSTV0910_P2_TSBUFSTAT1  0xf38e
+#define FSTV0910_P2_TSISCR_BUFSTAT1  0xf38e00ff
+
+/* P2_TSBUFSTAT0 */
+#define RSTV0910_P2_TSBUFSTAT0  0xf38f
+#define FSTV0910_P2_TSISCR_BUFSTAT0  0xf38f00ff
+
+/* P2_TSDEBUGL */
+#define RSTV0910_P2_TSDEBUGL  0xf391
+#define FSTV0910_P2_TSFIFO_ERROR_EVNT  0xf3910004
+#define FSTV0910_P2_TSFIFO_OVERFLOWM  0xf3910001
+
+/* P2_TSDLYSET2 */
+#define RSTV0910_P2_TSDLYSET2  0xf392
+#define FSTV0910_P2_SOFFIFO_OFFSET  0xf39200c0
+#define FSTV0910_P2_HYSTERESIS_THRESHOLD  0xf3920030
+#define FSTV0910_P2_SOFFIFO_SYMBOFFS2  0xf392000f
+
+/* P2_TSDLYSET1 */
+#define RSTV0910_P2_TSDLYSET1  0xf393
+#define FSTV0910_P2_SOFFIFO_SYMBOFFS1  0xf39300ff
+
+/* P2_TSDLYSET0 */
+#define RSTV0910_P2_TSDLYSET0  0xf394
+#define FSTV0910_P2_SOFFIFO_SYMBOFFS0  0xf39400ff
+
+/* P2_ERRCTRL1 */
+#define RSTV0910_P2_ERRCTRL1  0xf398
+#define FSTV0910_P2_ERR_SOURCE1  0xf39800f0
+#define FSTV0910_P2_NUM_EVENT1  0xf3980007
+
+/* P2_ERRCNT12 */
+#define RSTV0910_P2_ERRCNT12  0xf399
+#define FSTV0910_P2_ERRCNT1_OLDVALUE  0xf3990080
+#define FSTV0910_P2_ERR_CNT12  0xf399007f
+
+/* P2_ERRCNT11 */
+#define RSTV0910_P2_ERRCNT11  0xf39a
+#define FSTV0910_P2_ERR_CNT11  0xf39a00ff
+
+/* P2_ERRCNT10 */
+#define RSTV0910_P2_ERRCNT10  0xf39b
+#define FSTV0910_P2_ERR_CNT10  0xf39b00ff
+
+/* P2_ERRCTRL2 */
+#define RSTV0910_P2_ERRCTRL2  0xf39c
+#define FSTV0910_P2_ERR_SOURCE2  0xf39c00f0
+#define FSTV0910_P2_NUM_EVENT2  0xf39c0007
+
+/* P2_ERRCNT22 */
+#define RSTV0910_P2_ERRCNT22  0xf39d
+#define FSTV0910_P2_ERRCNT2_OLDVALUE  0xf39d0080
+#define FSTV0910_P2_ERR_CNT22  0xf39d007f
+
+/* P2_ERRCNT21 */
+#define RSTV0910_P2_ERRCNT21  0xf39e
+#define FSTV0910_P2_ERR_CNT21  0xf39e00ff
+
+/* P2_ERRCNT20 */
+#define RSTV0910_P2_ERRCNT20  0xf39f
+#define FSTV0910_P2_ERR_CNT20  0xf39f00ff
+
+/* P2_FECSPY */
+#define RSTV0910_P2_FECSPY  0xf3a0
+#define FSTV0910_P2_SPY_ENABLE  0xf3a00080
+#define FSTV0910_P2_NO_SYNCBYTE  0xf3a00040
+#define FSTV0910_P2_SERIAL_MODE  0xf3a00020
+#define FSTV0910_P2_UNUSUAL_PACKET  0xf3a00010
+#define FSTV0910_P2_BERMETER_DATAMODE  0xf3a0000c
+#define FSTV0910_P2_BERMETER_LMODE  0xf3a00002
+#define FSTV0910_P2_BERMETER_RESET  0xf3a00001
+
+/* P2_FSPYCFG */
+#define RSTV0910_P2_FSPYCFG  0xf3a1
+#define FSTV0910_P2_FECSPY_INPUT  0xf3a100c0
+#define FSTV0910_P2_RST_ON_ERROR  0xf3a10020
+#define FSTV0910_P2_ONE_SHOT  0xf3a10010
+#define FSTV0910_P2_I2C_MODE  0xf3a1000c
+#define FSTV0910_P2_SPY_HYSTERESIS  0xf3a10003
+
+/* P2_FSPYDATA */
+#define RSTV0910_P2_FSPYDATA  0xf3a2
+#define FSTV0910_P2_SPY_STUFFING  0xf3a20080
+#define FSTV0910_P2_SPY_CNULLPKT  0xf3a20020
+#define FSTV0910_P2_SPY_OUTDATA_MODE  0xf3a2001f
+
+/* P2_FSPYOUT */
+#define RSTV0910_P2_FSPYOUT  0xf3a3
+#define FSTV0910_P2_FSPY_DIRECT  0xf3a30080
+#define FSTV0910_P2_STUFF_MODE  0xf3a30007
+
+/* P2_FSTATUS */
+#define RSTV0910_P2_FSTATUS  0xf3a4
+#define FSTV0910_P2_SPY_ENDSIM  0xf3a40080
+#define FSTV0910_P2_VALID_SIM  0xf3a40040
+#define FSTV0910_P2_FOUND_SIGNAL  0xf3a40020
+#define FSTV0910_P2_DSS_SYNCBYTE  0xf3a40010
+#define FSTV0910_P2_RESULT_STATE  0xf3a4000f
+
+/* P2_FBERCPT4 */
+#define RSTV0910_P2_FBERCPT4  0xf3a8
+#define FSTV0910_P2_FBERMETER_CPT4  0xf3a800ff
+
+/* P2_FBERCPT3 */
+#define RSTV0910_P2_FBERCPT3  0xf3a9
+#define FSTV0910_P2_FBERMETER_CPT3  0xf3a900ff
+
+/* P2_FBERCPT2 */
+#define RSTV0910_P2_FBERCPT2  0xf3aa
+#define FSTV0910_P2_FBERMETER_CPT2  0xf3aa00ff
+
+/* P2_FBERCPT1 */
+#define RSTV0910_P2_FBERCPT1  0xf3ab
+#define FSTV0910_P2_FBERMETER_CPT1  0xf3ab00ff
+
+/* P2_FBERCPT0 */
+#define RSTV0910_P2_FBERCPT0  0xf3ac
+#define FSTV0910_P2_FBERMETER_CPT0  0xf3ac00ff
+
+/* P2_FBERERR2 */
+#define RSTV0910_P2_FBERERR2  0xf3ad
+#define FSTV0910_P2_FBERMETER_ERR2  0xf3ad00ff
+
+/* P2_FBERERR1 */
+#define RSTV0910_P2_FBERERR1  0xf3ae
+#define FSTV0910_P2_FBERMETER_ERR1  0xf3ae00ff
+
+/* P2_FBERERR0 */
+#define RSTV0910_P2_FBERERR0  0xf3af
+#define FSTV0910_P2_FBERMETER_ERR0  0xf3af00ff
+
+/* P2_FSPYBER */
+#define RSTV0910_P2_FSPYBER  0xf3b2
+#define FSTV0910_P2_FSPYBER_SYNCBYTE  0xf3b20010
+#define FSTV0910_P2_FSPYBER_UNSYNC  0xf3b20008
+#define FSTV0910_P2_FSPYBER_CTIME  0xf3b20007
+
+/* P2_SFERROR */
+#define RSTV0910_P2_SFERROR  0xf3c1
+#define FSTV0910_P2_SFEC_REGERR_VIT  0xf3c100ff
+
+/* P2_SFECSTATUS */
+#define RSTV0910_P2_SFECSTATUS  0xf3c3
+#define FSTV0910_P2_SFEC_ON  0xf3c30080
+#define FSTV0910_P2_SFEC_OFF  0xf3c30040
+#define FSTV0910_P2_LOCKEDSFEC  0xf3c30008
+#define FSTV0910_P2_SFEC_DELOCK  0xf3c30004
+#define FSTV0910_P2_SFEC_DEMODSEL  0xf3c30002
+#define FSTV0910_P2_SFEC_OVFON  0xf3c30001
+
+/* P2_SFKDIV12 */
+#define RSTV0910_P2_SFKDIV12  0xf3c4
+#define FSTV0910_P2_SFECKDIV12_MAN  0xf3c40080
+
+/* P2_SFKDIV23 */
+#define RSTV0910_P2_SFKDIV23  0xf3c5
+#define FSTV0910_P2_SFECKDIV23_MAN  0xf3c50080
+
+/* P2_SFKDIV34 */
+#define RSTV0910_P2_SFKDIV34  0xf3c6
+#define FSTV0910_P2_SFECKDIV34_MAN  0xf3c60080
+
+/* P2_SFKDIV56 */
+#define RSTV0910_P2_SFKDIV56  0xf3c7
+#define FSTV0910_P2_SFECKDIV56_MAN  0xf3c70080
+
+/* P2_SFKDIV67 */
+#define RSTV0910_P2_SFKDIV67  0xf3c8
+#define FSTV0910_P2_SFECKDIV67_MAN  0xf3c80080
+
+/* P2_SFKDIV78 */
+#define RSTV0910_P2_SFKDIV78  0xf3c9
+#define FSTV0910_P2_SFECKDIV78_MAN  0xf3c90080
+
+/* P2_SFSTATUS */
+#define RSTV0910_P2_SFSTATUS  0xf3cc
+#define FSTV0910_P2_SFEC_LINEOK  0xf3cc0080
+#define FSTV0910_P2_SFEC_ERROR  0xf3cc0040
+#define FSTV0910_P2_SFEC_DATA7  0xf3cc0020
+#define FSTV0910_P2_SFEC_PKTDNBRFAIL  0xf3cc0010
+#define FSTV0910_P2_TSSFEC_DEMODSEL  0xf3cc0008
+#define FSTV0910_P2_SFEC_NOSYNC  0xf3cc0004
+#define FSTV0910_P2_SFEC_UNREGULA  0xf3cc0002
+#define FSTV0910_P2_SFEC_READY  0xf3cc0001
+
+/* P2_SFDLYSET2 */
+#define RSTV0910_P2_SFDLYSET2  0xf3d0
+#define FSTV0910_P2_SFEC_DISABLE  0xf3d00002
+
+/* P2_SFERRCTRL */
+#define RSTV0910_P2_SFERRCTRL  0xf3d8
+#define FSTV0910_P2_SFEC_ERR_SOURCE  0xf3d800f0
+#define FSTV0910_P2_SFEC_NUM_EVENT  0xf3d80007
+
+/* P2_SFERRCNT2 */
+#define RSTV0910_P2_SFERRCNT2  0xf3d9
+#define FSTV0910_P2_SFERRC_OLDVALUE  0xf3d90080
+#define FSTV0910_P2_SFEC_ERR_CNT2  0xf3d9007f
+
+/* P2_SFERRCNT1 */
+#define RSTV0910_P2_SFERRCNT1  0xf3da
+#define FSTV0910_P2_SFEC_ERR_CNT1  0xf3da00ff
+
+/* P2_SFERRCNT0 */
+#define RSTV0910_P2_SFERRCNT0  0xf3db
+#define FSTV0910_P2_SFEC_ERR_CNT0  0xf3db00ff
+
+/* P1_IQCONST */
+#define RSTV0910_P1_IQCONST  0xf400
+#define FSTV0910_P1_CONSTEL_SELECT  0xf4000060
+#define FSTV0910_P1_IQSYMB_SEL  0xf400001f
+
+/* P1_NOSCFG */
+#define RSTV0910_P1_NOSCFG  0xf401
+#define FSTV0910_P1_DUMMYPL_NOSDATA  0xf4010020
+#define FSTV0910_P1_NOSPLH_BETA  0xf4010018
+#define FSTV0910_P1_NOSDATA_BETA  0xf4010007
+
+/* P1_ISYMB */
+#define RSTV0910_P1_ISYMB  0xf402
+#define FSTV0910_P1_I_SYMBOL  0xf40201ff
+
+/* P1_QSYMB */
+#define RSTV0910_P1_QSYMB  0xf403
+#define FSTV0910_P1_Q_SYMBOL  0xf40301ff
+
+/* P1_AGC1CFG */
+#define RSTV0910_P1_AGC1CFG  0xf404
+#define FSTV0910_P1_DC_FROZEN  0xf4040080
+#define FSTV0910_P1_DC_CORRECT  0xf4040040
+#define FSTV0910_P1_AMM_FROZEN  0xf4040020
+#define FSTV0910_P1_AMM_CORRECT  0xf4040010
+#define FSTV0910_P1_QUAD_FROZEN  0xf4040008
+#define FSTV0910_P1_QUAD_CORRECT  0xf4040004
+
+/* P1_AGC1CN */
+#define RSTV0910_P1_AGC1CN  0xf406
+#define FSTV0910_P1_AGC1_LOCKED  0xf4060080
+#define FSTV0910_P1_AGC1_MINPOWER  0xf4060010
+#define FSTV0910_P1_AGCOUT_FAST  0xf4060008
+#define FSTV0910_P1_AGCIQ_BETA  0xf4060007
+
+/* P1_AGC1REF */
+#define RSTV0910_P1_AGC1REF  0xf407
+#define FSTV0910_P1_AGCIQ_REF  0xf40700ff
+
+/* P1_IDCCOMP */
+#define RSTV0910_P1_IDCCOMP  0xf408
+#define FSTV0910_P1_IAVERAGE_ADJ  0xf40801ff
+
+/* P1_QDCCOMP */
+#define RSTV0910_P1_QDCCOMP  0xf409
+#define FSTV0910_P1_QAVERAGE_ADJ  0xf40901ff
+
+/* P1_POWERI */
+#define RSTV0910_P1_POWERI  0xf40a
+#define FSTV0910_P1_POWER_I  0xf40a00ff
+
+/* P1_POWERQ */
+#define RSTV0910_P1_POWERQ  0xf40b
+#define FSTV0910_P1_POWER_Q  0xf40b00ff
+
+/* P1_AGC1AMM */
+#define RSTV0910_P1_AGC1AMM  0xf40c
+#define FSTV0910_P1_AMM_VALUE  0xf40c00ff
+
+/* P1_AGC1QUAD */
+#define RSTV0910_P1_AGC1QUAD  0xf40d
+#define FSTV0910_P1_QUAD_VALUE  0xf40d01ff
+
+/* P1_AGCIQIN1 */
+#define RSTV0910_P1_AGCIQIN1  0xf40e
+#define FSTV0910_P1_AGCIQ_VALUE1  0xf40e00ff
+
+/* P1_AGCIQIN0 */
+#define RSTV0910_P1_AGCIQIN0  0xf40f
+#define FSTV0910_P1_AGCIQ_VALUE0  0xf40f00ff
+
+/* P1_DEMOD */
+#define RSTV0910_P1_DEMOD  0xf410
+#define FSTV0910_P1_MANUALS2_ROLLOFF  0xf4100080
+#define FSTV0910_P1_SPECINV_CONTROL  0xf4100030
+#define FSTV0910_P1_MANUALSX_ROLLOFF  0xf4100004
+#define FSTV0910_P1_ROLLOFF_CONTROL  0xf4100003
+
+/* P1_DMDMODCOD */
+#define RSTV0910_P1_DMDMODCOD  0xf411
+#define FSTV0910_P1_MANUAL_MODCOD  0xf4110080
+#define FSTV0910_P1_DEMOD_MODCOD  0xf411007c
+#define FSTV0910_P1_DEMOD_TYPE  0xf4110003
+
+/* P1_DSTATUS */
+#define RSTV0910_P1_DSTATUS  0xf412
+#define FSTV0910_P1_CAR_LOCK  0xf4120080
+#define FSTV0910_P1_TMGLOCK_QUALITY  0xf4120060
+#define FSTV0910_P1_LOCK_DEFINITIF  0xf4120008
+#define FSTV0910_P1_OVADC_DETECT  0xf4120001
+
+/* P1_DSTATUS2 */
+#define RSTV0910_P1_DSTATUS2  0xf413
+#define FSTV0910_P1_DEMOD_DELOCK  0xf4130080
+#define FSTV0910_P1_MODCODRQ_SYNCTAG  0xf4130020
+#define FSTV0910_P1_POLYPH_SATEVENT  0xf4130010
+#define FSTV0910_P1_AGC1_NOSIGNALACK  0xf4130008
+#define FSTV0910_P1_AGC2_OVERFLOW  0xf4130004
+#define FSTV0910_P1_CFR_OVERFLOW  0xf4130002
+#define FSTV0910_P1_GAMMA_OVERUNDER  0xf4130001
+
+/* P1_DMDCFGMD */
+#define RSTV0910_P1_DMDCFGMD  0xf414
+#define FSTV0910_P1_DVBS2_ENABLE  0xf4140080
+#define FSTV0910_P1_DVBS1_ENABLE  0xf4140040
+#define FSTV0910_P1_SCAN_ENABLE  0xf4140010
+#define FSTV0910_P1_CFR_AUTOSCAN  0xf4140008
+#define FSTV0910_P1_TUN_RNG  0xf4140003
+
+/* P1_DMDCFG2 */
+#define RSTV0910_P1_DMDCFG2  0xf415
+#define FSTV0910_P1_S1S2_SEQUENTIAL  0xf4150040
+#define FSTV0910_P1_INFINITE_RELOCK  0xf4150010
+
+/* P1_DMDISTATE */
+#define RSTV0910_P1_DMDISTATE  0xf416
+#define FSTV0910_P1_I2C_NORESETDMODE  0xf4160080
+#define FSTV0910_P1_I2C_DEMOD_MODE  0xf416001f
+
+/* P1_DMDT0M */
+#define RSTV0910_P1_DMDT0M  0xf417
+#define FSTV0910_P1_DMDT0_MIN  0xf41700ff
+
+/* P1_DMDSTATE */
+#define RSTV0910_P1_DMDSTATE  0xf41b
+#define FSTV0910_P1_HEADER_MODE  0xf41b0060
+
+/* P1_DMDFLYW */
+#define RSTV0910_P1_DMDFLYW  0xf41c
+#define FSTV0910_P1_I2C_IRQVAL  0xf41c00f0
+#define FSTV0910_P1_FLYWHEEL_CPT  0xf41c000f
+
+/* P1_DSTATUS3 */
+#define RSTV0910_P1_DSTATUS3  0xf41d
+#define FSTV0910_P1_CFR_ZIGZAG  0xf41d0080
+#define FSTV0910_P1_DEMOD_CFGMODE  0xf41d0060
+#define FSTV0910_P1_GAMMA_LOWBAUDRATE  0xf41d0010
+
+/* P1_DMDCFG3 */
+#define RSTV0910_P1_DMDCFG3  0xf41e
+#define FSTV0910_P1_NOSTOP_FIFOFULL  0xf41e0008
+
+/* P1_DMDCFG4 */
+#define RSTV0910_P1_DMDCFG4  0xf41f
+#define FSTV0910_P1_DIS_VITLOCK  0xf41f0080
+#define FSTV0910_P1_DIS_CLKENABLE  0xf41f0004
+
+/* P1_CORRELMANT */
+#define RSTV0910_P1_CORRELMANT  0xf420
+#define FSTV0910_P1_CORREL_MANT  0xf42000ff
+
+/* P1_CORRELABS */
+#define RSTV0910_P1_CORRELABS  0xf421
+#define FSTV0910_P1_CORREL_ABS  0xf42100ff
+
+/* P1_CORRELEXP */
+#define RSTV0910_P1_CORRELEXP  0xf422
+#define FSTV0910_P1_CORREL_ABSEXP  0xf42200f0
+#define FSTV0910_P1_CORREL_EXP  0xf422000f
+
+/* P1_PLHMODCOD */
+#define RSTV0910_P1_PLHMODCOD  0xf424
+#define FSTV0910_P1_SPECINV_DEMOD  0xf4240080
+#define FSTV0910_P1_PLH_MODCOD  0xf424007c
+#define FSTV0910_P1_PLH_TYPE  0xf4240003
+
+/* P1_DMDREG */
+#define RSTV0910_P1_DMDREG  0xf425
+#define FSTV0910_P1_DECIM_PLFRAMES  0xf4250001
+
+/* P1_AGCNADJ */
+#define RSTV0910_P1_AGCNADJ  0xf426
+#define FSTV0910_P1_RADJOFF_AGC2  0xf4260080
+#define FSTV0910_P1_RADJOFF_AGC1  0xf4260040
+#define FSTV0910_P1_AGC_NADJ  0xf426013f
+
+/* P1_AGCKS */
+#define RSTV0910_P1_AGCKS  0xf427
+#define FSTV0910_P1_RSADJ_MANUALCFG  0xf4270080
+#define FSTV0910_P1_RSADJ_CCMMODE  0xf4270040
+#define FSTV0910_P1_RADJ_SPSK  0xf427013f
+
+/* P1_AGCKQ */
+#define RSTV0910_P1_AGCKQ  0xf428
+#define FSTV0910_P1_RADJON_DVBS1  0xf4280040
+#define FSTV0910_P1_RADJ_QPSK  0xf428013f
+
+/* P1_AGCK8 */
+#define RSTV0910_P1_AGCK8  0xf429
+#define FSTV0910_P1_RADJ_8PSK  0xf429013f
+
+/* P1_AGCK16 */
+#define RSTV0910_P1_AGCK16  0xf42a
+#define FSTV0910_P1_R2ADJOFF_16APSK  0xf42a0040
+#define FSTV0910_P1_R1ADJOFF_16APSK  0xf42a0020
+#define FSTV0910_P1_RADJ_16APSK  0xf42a011f
+
+/* P1_AGCK32 */
+#define RSTV0910_P1_AGCK32  0xf42b
+#define FSTV0910_P1_R3ADJOFF_32APSK  0xf42b0080
+#define FSTV0910_P1_R2ADJOFF_32APSK  0xf42b0040
+#define FSTV0910_P1_R1ADJOFF_32APSK  0xf42b0020
+#define FSTV0910_P1_RADJ_32APSK  0xf42b011f
+
+/* P1_AGC2O */
+#define RSTV0910_P1_AGC2O  0xf42c
+#define FSTV0910_P1_CSTENV_MODE  0xf42c00c0
+#define FSTV0910_P1_AGC2_COEF  0xf42c0007
+
+/* P1_AGC2REF */
+#define RSTV0910_P1_AGC2REF  0xf42d
+#define FSTV0910_P1_AGC2_REF  0xf42d00ff
+
+/* P1_AGC1ADJ */
+#define RSTV0910_P1_AGC1ADJ  0xf42e
+#define FSTV0910_P1_AGC1_ADJUSTED  0xf42e007f
+
+/* P1_AGCRSADJ */
+#define RSTV0910_P1_AGCRSADJ  0xf42f
+#define FSTV0910_P1_RS_ADJUSTED  0xf42f007f
+
+/* P1_AGCRQADJ */
+#define RSTV0910_P1_AGCRQADJ  0xf430
+#define FSTV0910_P1_RQ_ADJUSTED  0xf430007f
+
+/* P1_AGCR8ADJ */
+#define RSTV0910_P1_AGCR8ADJ  0xf431
+#define FSTV0910_P1_R8_ADJUSTED  0xf431007f
+
+/* P1_AGCR1ADJ */
+#define RSTV0910_P1_AGCR1ADJ  0xf432
+#define FSTV0910_P1_R1_ADJUSTED  0xf432007f
+
+/* P1_AGCR2ADJ */
+#define RSTV0910_P1_AGCR2ADJ  0xf433
+#define FSTV0910_P1_R2_ADJUSTED  0xf433007f
+
+/* P1_AGCR3ADJ */
+#define RSTV0910_P1_AGCR3ADJ  0xf434
+#define FSTV0910_P1_R3_ADJUSTED  0xf434007f
+
+/* P1_AGCREFADJ */
+#define RSTV0910_P1_AGCREFADJ  0xf435
+#define FSTV0910_P1_AGC2REF_ADJUSTED  0xf435007f
+
+/* P1_AGC2I1 */
+#define RSTV0910_P1_AGC2I1  0xf436
+#define FSTV0910_P1_AGC2_INTEGRATOR1  0xf43600ff
+
+/* P1_AGC2I0 */
+#define RSTV0910_P1_AGC2I0  0xf437
+#define FSTV0910_P1_AGC2_INTEGRATOR0  0xf43700ff
+
+/* P1_CARCFG */
+#define RSTV0910_P1_CARCFG  0xf438
+#define FSTV0910_P1_ROTAON  0xf4380004
+#define FSTV0910_P1_PH_DET_ALGO  0xf4380003
+
+/* P1_ACLC */
+#define RSTV0910_P1_ACLC  0xf439
+#define FSTV0910_P1_CAR_ALPHA_MANT  0xf4390030
+#define FSTV0910_P1_CAR_ALPHA_EXP  0xf439000f
+
+/* P1_BCLC */
+#define RSTV0910_P1_BCLC  0xf43a
+#define FSTV0910_P1_CAR_BETA_MANT  0xf43a0030
+#define FSTV0910_P1_CAR_BETA_EXP  0xf43a000f
+
+/* P1_ACLCS2 */
+#define RSTV0910_P1_ACLCS2  0xf43b
+#define FSTV0910_P1_CARS2_APLHA_MANTISSE  0xf43b0030
+#define FSTV0910_P1_CARS2_ALPHA_EXP  0xf43b000f
+
+/* P1_BCLCS2 */
+#define RSTV0910_P1_BCLCS2  0xf43c
+#define FSTV0910_P1_CARS2_BETA_MANTISSE  0xf43c0030
+#define FSTV0910_P1_CARS2_BETA_EXP  0xf43c000f
+
+/* P1_CARFREQ */
+#define RSTV0910_P1_CARFREQ  0xf43d
+#define FSTV0910_P1_KC_COARSE_EXP  0xf43d00f0
+#define FSTV0910_P1_BETA_FREQ  0xf43d000f
+
+/* P1_CARHDR */
+#define RSTV0910_P1_CARHDR  0xf43e
+#define FSTV0910_P1_K_FREQ_HDR  0xf43e00ff
+
+/* P1_LDT */
+#define RSTV0910_P1_LDT  0xf43f
+#define FSTV0910_P1_CARLOCK_THRES  0xf43f01ff
+
+/* P1_LDT2 */
+#define RSTV0910_P1_LDT2  0xf440
+#define FSTV0910_P1_CARLOCK_THRES2  0xf44001ff
+
+/* P1_CFRICFG */
+#define RSTV0910_P1_CFRICFG  0xf441
+#define FSTV0910_P1_NEG_CFRSTEP  0xf4410001
+
+/* P1_CFRUP1 */
+#define RSTV0910_P1_CFRUP1  0xf442
+#define FSTV0910_P1_CFR_UP1  0xf44201ff
+
+/* P1_CFRUP0 */
+#define RSTV0910_P1_CFRUP0  0xf443
+#define FSTV0910_P1_CFR_UP0  0xf44300ff
+
+/* P1_CFRIBASE1 */
+#define RSTV0910_P1_CFRIBASE1  0xf444
+#define FSTV0910_P1_CFRINIT_BASE1  0xf44400ff
+
+/* P1_CFRIBASE0 */
+#define RSTV0910_P1_CFRIBASE0  0xf445
+#define FSTV0910_P1_CFRINIT_BASE0  0xf44500ff
+
+/* P1_CFRLOW1 */
+#define RSTV0910_P1_CFRLOW1  0xf446
+#define FSTV0910_P1_CFR_LOW1  0xf44601ff
+
+/* P1_CFRLOW0 */
+#define RSTV0910_P1_CFRLOW0  0xf447
+#define FSTV0910_P1_CFR_LOW0  0xf44700ff
+
+/* P1_CFRINIT1 */
+#define RSTV0910_P1_CFRINIT1  0xf448
+#define FSTV0910_P1_CFR_INIT1  0xf44801ff
+
+/* P1_CFRINIT0 */
+#define RSTV0910_P1_CFRINIT0  0xf449
+#define FSTV0910_P1_CFR_INIT0  0xf44900ff
+
+/* P1_CFRINC1 */
+#define RSTV0910_P1_CFRINC1  0xf44a
+#define FSTV0910_P1_MANUAL_CFRINC  0xf44a0080
+#define FSTV0910_P1_CFR_INC1  0xf44a003f
+
+/* P1_CFRINC0 */
+#define RSTV0910_P1_CFRINC0  0xf44b
+#define FSTV0910_P1_CFR_INC0  0xf44b00ff
+
+/* P1_CFR2 */
+#define RSTV0910_P1_CFR2  0xf44c
+#define FSTV0910_P1_CAR_FREQ2  0xf44c01ff
+
+/* P1_CFR1 */
+#define RSTV0910_P1_CFR1  0xf44d
+#define FSTV0910_P1_CAR_FREQ1  0xf44d00ff
+
+/* P1_CFR0 */
+#define RSTV0910_P1_CFR0  0xf44e
+#define FSTV0910_P1_CAR_FREQ0  0xf44e00ff
+
+/* P1_LDI */
+#define RSTV0910_P1_LDI  0xf44f
+#define FSTV0910_P1_LOCK_DET_INTEGR  0xf44f01ff
+
+/* P1_TMGCFG */
+#define RSTV0910_P1_TMGCFG  0xf450
+#define FSTV0910_P1_TMGLOCK_BETA  0xf45000c0
+#define FSTV0910_P1_DO_TIMING_CORR  0xf4500010
+#define FSTV0910_P1_TMG_MINFREQ  0xf4500003
+
+/* P1_RTC */
+#define RSTV0910_P1_RTC  0xf451
+#define FSTV0910_P1_TMGALPHA_EXP  0xf45100f0
+#define FSTV0910_P1_TMGBETA_EXP  0xf451000f
+
+/* P1_RTCS2 */
+#define RSTV0910_P1_RTCS2  0xf452
+#define FSTV0910_P1_TMGALPHAS2_EXP  0xf45200f0
+#define FSTV0910_P1_TMGBETAS2_EXP  0xf452000f
+
+/* P1_TMGTHRISE */
+#define RSTV0910_P1_TMGTHRISE  0xf453
+#define FSTV0910_P1_TMGLOCK_THRISE  0xf45300ff
+
+/* P1_TMGTHFALL */
+#define RSTV0910_P1_TMGTHFALL  0xf454
+#define FSTV0910_P1_TMGLOCK_THFALL  0xf45400ff
+
+/* P1_SFRUPRATIO */
+#define RSTV0910_P1_SFRUPRATIO  0xf455
+#define FSTV0910_P1_SFR_UPRATIO  0xf45500ff
+
+/* P1_SFRLOWRATIO */
+#define RSTV0910_P1_SFRLOWRATIO  0xf456
+#define FSTV0910_P1_SFR_LOWRATIO  0xf45600ff
+
+/* P1_KTTMG */
+#define RSTV0910_P1_KTTMG  0xf457
+#define FSTV0910_P1_KT_TMG_EXP  0xf45700f0
+
+/* P1_KREFTMG */
+#define RSTV0910_P1_KREFTMG  0xf458
+#define FSTV0910_P1_KREF_TMG  0xf45800ff
+
+/* P1_SFRSTEP */
+#define RSTV0910_P1_SFRSTEP  0xf459
+#define FSTV0910_P1_SFR_SCANSTEP  0xf45900f0
+#define FSTV0910_P1_SFR_CENTERSTEP  0xf459000f
+
+/* P1_TMGCFG2 */
+#define RSTV0910_P1_TMGCFG2  0xf45a
+#define FSTV0910_P1_DIS_AUTOSAMP  0xf45a0008
+#define FSTV0910_P1_SFRRATIO_FINE  0xf45a0001
+
+/* P1_KREFTMG2 */
+#define RSTV0910_P1_KREFTMG2  0xf45b
+#define FSTV0910_P1_KREF_TMG2  0xf45b00ff
+
+/* P1_TMGCFG3 */
+#define RSTV0910_P1_TMGCFG3  0xf45d
+#define FSTV0910_P1_CONT_TMGCENTER  0xf45d0008
+#define FSTV0910_P1_AUTO_GUP  0xf45d0004
+#define FSTV0910_P1_AUTO_GLOW  0xf45d0002
+
+/* P1_SFRINIT1 */
+#define RSTV0910_P1_SFRINIT1  0xf45e
+#define FSTV0910_P1_SFR_INIT1  0xf45e00ff
+
+/* P1_SFRINIT0 */
+#define RSTV0910_P1_SFRINIT0  0xf45f
+#define FSTV0910_P1_SFR_INIT0  0xf45f00ff
+
+/* P1_SFRUP1 */
+#define RSTV0910_P1_SFRUP1  0xf460
+#define FSTV0910_P1_SYMB_FREQ_UP1  0xf46000ff
+
+/* P1_SFRUP0 */
+#define RSTV0910_P1_SFRUP0  0xf461
+#define FSTV0910_P1_SYMB_FREQ_UP0  0xf46100ff
+
+/* P1_SFRLOW1 */
+#define RSTV0910_P1_SFRLOW1  0xf462
+#define FSTV0910_P1_SYMB_FREQ_LOW1  0xf46200ff
+
+/* P1_SFRLOW0 */
+#define RSTV0910_P1_SFRLOW0  0xf463
+#define FSTV0910_P1_SYMB_FREQ_LOW0  0xf46300ff
+
+/* P1_SFR3 */
+#define RSTV0910_P1_SFR3  0xf464
+#define FSTV0910_P1_SYMB_FREQ3  0xf46400ff
+
+/* P1_SFR2 */
+#define RSTV0910_P1_SFR2  0xf465
+#define FSTV0910_P1_SYMB_FREQ2  0xf46500ff
+
+/* P1_SFR1 */
+#define RSTV0910_P1_SFR1  0xf466
+#define FSTV0910_P1_SYMB_FREQ1  0xf46600ff
+
+/* P1_SFR0 */
+#define RSTV0910_P1_SFR0  0xf467
+#define FSTV0910_P1_SYMB_FREQ0  0xf46700ff
+
+/* P1_TMGREG2 */
+#define RSTV0910_P1_TMGREG2  0xf468
+#define FSTV0910_P1_TMGREG2  0xf46800ff
+
+/* P1_TMGREG1 */
+#define RSTV0910_P1_TMGREG1  0xf469
+#define FSTV0910_P1_TMGREG1  0xf46900ff
+
+/* P1_TMGREG0 */
+#define RSTV0910_P1_TMGREG0  0xf46a
+#define FSTV0910_P1_TMGREG0  0xf46a00ff
+
+/* P1_TMGLOCK1 */
+#define RSTV0910_P1_TMGLOCK1  0xf46b
+#define FSTV0910_P1_TMGLOCK_LEVEL1  0xf46b01ff
+
+/* P1_TMGLOCK0 */
+#define RSTV0910_P1_TMGLOCK0  0xf46c
+#define FSTV0910_P1_TMGLOCK_LEVEL0  0xf46c00ff
+
+/* P1_TMGOBS */
+#define RSTV0910_P1_TMGOBS  0xf46d
+#define FSTV0910_P1_ROLLOFF_STATUS  0xf46d00c0
+
+/* P1_EQUALCFG */
+#define RSTV0910_P1_EQUALCFG  0xf46f
+#define FSTV0910_P1_EQUAL_ON  0xf46f0040
+#define FSTV0910_P1_MU_EQUALDFE  0xf46f0007
+
+/* P1_EQUAI1 */
+#define RSTV0910_P1_EQUAI1  0xf470
+#define FSTV0910_P1_EQUA_ACCI1  0xf47001ff
+
+/* P1_EQUAQ1 */
+#define RSTV0910_P1_EQUAQ1  0xf471
+#define FSTV0910_P1_EQUA_ACCQ1  0xf47101ff
+
+/* P1_EQUAI2 */
+#define RSTV0910_P1_EQUAI2  0xf472
+#define FSTV0910_P1_EQUA_ACCI2  0xf47201ff
+
+/* P1_EQUAQ2 */
+#define RSTV0910_P1_EQUAQ2  0xf473
+#define FSTV0910_P1_EQUA_ACCQ2  0xf47301ff
+
+/* P1_EQUAI3 */
+#define RSTV0910_P1_EQUAI3  0xf474
+#define FSTV0910_P1_EQUA_ACCI3  0xf47401ff
+
+/* P1_EQUAQ3 */
+#define RSTV0910_P1_EQUAQ3  0xf475
+#define FSTV0910_P1_EQUA_ACCQ3  0xf47501ff
+
+/* P1_EQUAI4 */
+#define RSTV0910_P1_EQUAI4  0xf476
+#define FSTV0910_P1_EQUA_ACCI4  0xf47601ff
+
+/* P1_EQUAQ4 */
+#define RSTV0910_P1_EQUAQ4  0xf477
+#define FSTV0910_P1_EQUA_ACCQ4  0xf47701ff
+
+/* P1_EQUAI5 */
+#define RSTV0910_P1_EQUAI5  0xf478
+#define FSTV0910_P1_EQUA_ACCI5  0xf47801ff
+
+/* P1_EQUAQ5 */
+#define RSTV0910_P1_EQUAQ5  0xf479
+#define FSTV0910_P1_EQUA_ACCQ5  0xf47901ff
+
+/* P1_EQUAI6 */
+#define RSTV0910_P1_EQUAI6  0xf47a
+#define FSTV0910_P1_EQUA_ACCI6  0xf47a01ff
+
+/* P1_EQUAQ6 */
+#define RSTV0910_P1_EQUAQ6  0xf47b
+#define FSTV0910_P1_EQUA_ACCQ6  0xf47b01ff
+
+/* P1_EQUAI7 */
+#define RSTV0910_P1_EQUAI7  0xf47c
+#define FSTV0910_P1_EQUA_ACCI7  0xf47c01ff
+
+/* P1_EQUAQ7 */
+#define RSTV0910_P1_EQUAQ7  0xf47d
+#define FSTV0910_P1_EQUA_ACCQ7  0xf47d01ff
+
+/* P1_EQUAI8 */
+#define RSTV0910_P1_EQUAI8  0xf47e
+#define FSTV0910_P1_EQUA_ACCI8  0xf47e01ff
+
+/* P1_EQUAQ8 */
+#define RSTV0910_P1_EQUAQ8  0xf47f
+#define FSTV0910_P1_EQUA_ACCQ8  0xf47f01ff
+
+/* P1_NNOSDATAT1 */
+#define RSTV0910_P1_NNOSDATAT1  0xf480
+#define FSTV0910_P1_NOSDATAT_NORMED1  0xf48000ff
+
+/* P1_NNOSDATAT0 */
+#define RSTV0910_P1_NNOSDATAT0  0xf481
+#define FSTV0910_P1_NOSDATAT_NORMED0  0xf48100ff
+
+/* P1_NNOSDATA1 */
+#define RSTV0910_P1_NNOSDATA1  0xf482
+#define FSTV0910_P1_NOSDATA_NORMED1  0xf48200ff
+
+/* P1_NNOSDATA0 */
+#define RSTV0910_P1_NNOSDATA0  0xf483
+#define FSTV0910_P1_NOSDATA_NORMED0  0xf48300ff
+
+/* P1_NNOSPLHT1 */
+#define RSTV0910_P1_NNOSPLHT1  0xf484
+#define FSTV0910_P1_NOSPLHT_NORMED1  0xf48400ff
+
+/* P1_NNOSPLHT0 */
+#define RSTV0910_P1_NNOSPLHT0  0xf485
+#define FSTV0910_P1_NOSPLHT_NORMED0  0xf48500ff
+
+/* P1_NNOSPLH1 */
+#define RSTV0910_P1_NNOSPLH1  0xf486
+#define FSTV0910_P1_NOSPLH_NORMED1  0xf48600ff
+
+/* P1_NNOSPLH0 */
+#define RSTV0910_P1_NNOSPLH0  0xf487
+#define FSTV0910_P1_NOSPLH_NORMED0  0xf48700ff
+
+/* P1_NOSDATAT1 */
+#define RSTV0910_P1_NOSDATAT1  0xf488
+#define FSTV0910_P1_NOSDATAT_UNNORMED1  0xf48800ff
+
+/* P1_NOSDATAT0 */
+#define RSTV0910_P1_NOSDATAT0  0xf489
+#define FSTV0910_P1_NOSDATAT_UNNORMED0  0xf48900ff
+
+/* P1_NNOSFRAME1 */
+#define RSTV0910_P1_NNOSFRAME1  0xf48a
+#define FSTV0910_P1_NOSFRAME_NORMED1  0xf48a00ff
+
+/* P1_NNOSFRAME0 */
+#define RSTV0910_P1_NNOSFRAME0  0xf48b
+#define FSTV0910_P1_NOSFRAME_NORMED0  0xf48b00ff
+
+/* P1_NNOSRAD1 */
+#define RSTV0910_P1_NNOSRAD1  0xf48c
+#define FSTV0910_P1_NOSRADIAL_NORMED1  0xf48c00ff
+
+/* P1_NNOSRAD0 */
+#define RSTV0910_P1_NNOSRAD0  0xf48d
+#define FSTV0910_P1_NOSRADIAL_NORMED0  0xf48d00ff
+
+/* P1_NOSCFGF1 */
+#define RSTV0910_P1_NOSCFGF1  0xf48e
+#define FSTV0910_P1_LOWNOISE_MESURE  0xf48e0080
+#define FSTV0910_P1_NOS_DELFRAME  0xf48e0040
+#define FSTV0910_P1_NOSDATA_MODE  0xf48e0030
+#define FSTV0910_P1_FRAMESEL_TYPESEL  0xf48e000c
+#define FSTV0910_P1_FRAMESEL_TYPE  0xf48e0003
+
+/* P1_NOSCFGF2 */
+#define RSTV0910_P1_NOSCFGF2  0xf48f
+#define FSTV0910_P1_DIS_NOSPILOTS  0xf48f0080
+#define FSTV0910_P1_FRAMESEL_MODCODSEL  0xf48f0060
+#define FSTV0910_P1_FRAMESEL_MODCOD  0xf48f001f
+
+/* P1_CAR2CFG */
+#define RSTV0910_P1_CAR2CFG  0xf490
+#define FSTV0910_P1_ROTA2ON  0xf4900004
+#define FSTV0910_P1_PH_DET_ALGO2  0xf4900003
+
+/* P1_CFR2CFR1 */
+#define RSTV0910_P1_CFR2CFR1  0xf491
+#define FSTV0910_P1_EN_S2CAR2CENTER  0xf4910020
+#define FSTV0910_P1_CFR2TOCFR1_BETA  0xf4910007
+
+/* P1_CAR3CFG */
+#define RSTV0910_P1_CAR3CFG  0xf492
+#define FSTV0910_P1_CARRIER23_MODE  0xf49200c0
+#define FSTV0910_P1_CAR3INTERM_DVBS1  0xf4920020
+#define FSTV0910_P1_ABAMPLIF_MODE  0xf4920018
+#define FSTV0910_P1_CARRIER3_ALPHA3DL  0xf4920007
+
+/* P1_CFR22 */
+#define RSTV0910_P1_CFR22  0xf493
+#define FSTV0910_P1_CAR2_FREQ2  0xf49301ff
+
+/* P1_CFR21 */
+#define RSTV0910_P1_CFR21  0xf494
+#define FSTV0910_P1_CAR2_FREQ1  0xf49400ff
+
+/* P1_CFR20 */
+#define RSTV0910_P1_CFR20  0xf495
+#define FSTV0910_P1_CAR2_FREQ0  0xf49500ff
+
+/* P1_ACLC2S2Q */
+#define RSTV0910_P1_ACLC2S2Q  0xf497
+#define FSTV0910_P1_ENAB_SPSKSYMB  0xf4970080
+#define FSTV0910_P1_CAR2S2_Q_ALPH_M  0xf4970030
+#define FSTV0910_P1_CAR2S2_Q_ALPH_E  0xf497000f
+
+/* P1_ACLC2S28 */
+#define RSTV0910_P1_ACLC2S28  0xf498
+#define FSTV0910_P1_CAR2S2_8_ALPH_M  0xf4980030
+#define FSTV0910_P1_CAR2S2_8_ALPH_E  0xf498000f
+
+/* P1_ACLC2S216A */
+#define RSTV0910_P1_ACLC2S216A  0xf499
+#define FSTV0910_P1_CAR2S2_16A_ALPH_M  0xf4990030
+#define FSTV0910_P1_CAR2S2_16A_ALPH_E  0xf499000f
+
+/* P1_ACLC2S232A */
+#define RSTV0910_P1_ACLC2S232A  0xf49a
+#define FSTV0910_P1_CAR2S2_32A_ALPH_M  0xf49a0030
+#define FSTV0910_P1_CAR2S2_32A_ALPH_E  0xf49a000f
+
+/* P1_BCLC2S2Q */
+#define RSTV0910_P1_BCLC2S2Q  0xf49c
+#define FSTV0910_P1_CAR2S2_Q_BETA_M  0xf49c0030
+#define FSTV0910_P1_CAR2S2_Q_BETA_E  0xf49c000f
+
+/* P1_BCLC2S28 */
+#define RSTV0910_P1_BCLC2S28  0xf49d
+#define FSTV0910_P1_CAR2S2_8_BETA_M  0xf49d0030
+#define FSTV0910_P1_CAR2S2_8_BETA_E  0xf49d000f
+
+/* P1_BCLC2S216A */
+#define RSTV0910_P1_BCLC2S216A  0xf49e
+#define FSTV0910_P1_DVBS2S216A_NIP  0xf49e0080
+#define FSTV0910_P1_CAR2S2_16A_BETA_M  0xf49e0030
+#define FSTV0910_P1_CAR2S2_16A_BETA_E  0xf49e000f
+
+/* P1_BCLC2S232A */
+#define RSTV0910_P1_BCLC2S232A  0xf49f
+#define FSTV0910_P1_DVBS2S232A_NIP  0xf49f0080
+#define FSTV0910_P1_CAR2S2_32A_BETA_M  0xf49f0030
+#define FSTV0910_P1_CAR2S2_32A_BETA_E  0xf49f000f
+
+/* P1_PLROOT2 */
+#define RSTV0910_P1_PLROOT2  0xf4ac
+#define FSTV0910_P1_PLSCRAMB_MODE  0xf4ac000c
+#define FSTV0910_P1_PLSCRAMB_ROOT2  0xf4ac0003
+
+/* P1_PLROOT1 */
+#define RSTV0910_P1_PLROOT1  0xf4ad
+#define FSTV0910_P1_PLSCRAMB_ROOT1  0xf4ad00ff
+
+/* P1_PLROOT0 */
+#define RSTV0910_P1_PLROOT0  0xf4ae
+#define FSTV0910_P1_PLSCRAMB_ROOT0  0xf4ae00ff
+
+/* P1_MODCODLST0 */
+#define RSTV0910_P1_MODCODLST0  0xf4b0
+#define FSTV0910_P1_NACCES_MODCODCH  0xf4b00001
+
+/* P1_MODCODLST1 */
+#define RSTV0910_P1_MODCODLST1  0xf4b1
+#define FSTV0910_P1_SYMBRATE_FILTER  0xf4b10008
+#define FSTV0910_P1_NRESET_MODCODLST  0xf4b10004
+#define FSTV0910_P1_DIS_32PSK_9_10  0xf4b10003
+
+/* P1_MODCODLST2 */
+#define RSTV0910_P1_MODCODLST2  0xf4b2
+#define FSTV0910_P1_DIS_32PSK_8_9  0xf4b200f0
+#define FSTV0910_P1_DIS_32PSK_5_6  0xf4b2000f
+
+/* P1_MODCODLST3 */
+#define RSTV0910_P1_MODCODLST3  0xf4b3
+#define FSTV0910_P1_DIS_32PSK_4_5  0xf4b300f0
+#define FSTV0910_P1_DIS_32PSK_3_4  0xf4b3000f
+
+/* P1_MODCODLST4 */
+#define RSTV0910_P1_MODCODLST4  0xf4b4
+#define FSTV0910_P1_DUMMYPL_PILOT  0xf4b40080
+#define FSTV0910_P1_DUMMYPL_NOPILOT  0xf4b40040
+#define FSTV0910_P1_DIS_16PSK_9_10  0xf4b40030
+#define FSTV0910_P1_DIS_16PSK_8_9  0xf4b4000f
+
+/* P1_MODCODLST5 */
+#define RSTV0910_P1_MODCODLST5  0xf4b5
+#define FSTV0910_P1_DIS_16PSK_5_6  0xf4b500f0
+#define FSTV0910_P1_DIS_16PSK_4_5  0xf4b5000f
+
+/* P1_MODCODLST6 */
+#define RSTV0910_P1_MODCODLST6  0xf4b6
+#define FSTV0910_P1_DIS_16PSK_3_4  0xf4b600f0
+#define FSTV0910_P1_DIS_16PSK_2_3  0xf4b6000f
+
+/* P1_MODCODLST7 */
+#define RSTV0910_P1_MODCODLST7  0xf4b7
+#define FSTV0910_P1_MODCOD_NNOSFILTER  0xf4b70080
+#define FSTV0910_P1_DIS_8PSK_9_10  0xf4b70030
+#define FSTV0910_P1_DIS_8PSK_8_9  0xf4b7000f
+
+/* P1_MODCODLST8 */
+#define RSTV0910_P1_MODCODLST8  0xf4b8
+#define FSTV0910_P1_DIS_8PSK_5_6  0xf4b800f0
+#define FSTV0910_P1_DIS_8PSK_3_4  0xf4b8000f
+
+/* P1_MODCODLST9 */
+#define RSTV0910_P1_MODCODLST9  0xf4b9
+#define FSTV0910_P1_DIS_8PSK_2_3  0xf4b900f0
+#define FSTV0910_P1_DIS_8PSK_3_5  0xf4b9000f
+
+/* P1_MODCODLSTA */
+#define RSTV0910_P1_MODCODLSTA  0xf4ba
+#define FSTV0910_P1_NOSFILTER_LIMITE  0xf4ba0080
+#define FSTV0910_P1_DIS_QPSK_9_10  0xf4ba0030
+#define FSTV0910_P1_DIS_QPSK_8_9  0xf4ba000f
+
+/* P1_MODCODLSTB */
+#define RSTV0910_P1_MODCODLSTB  0xf4bb
+#define FSTV0910_P1_DIS_QPSK_5_6  0xf4bb00f0
+#define FSTV0910_P1_DIS_QPSK_4_5  0xf4bb000f
+
+/* P1_MODCODLSTC */
+#define RSTV0910_P1_MODCODLSTC  0xf4bc
+#define FSTV0910_P1_DIS_QPSK_3_4  0xf4bc00f0
+#define FSTV0910_P1_DIS_QPSK_2_3  0xf4bc000f
+
+/* P1_MODCODLSTD */
+#define RSTV0910_P1_MODCODLSTD  0xf4bd
+#define FSTV0910_P1_DIS_QPSK_3_5  0xf4bd00f0
+#define FSTV0910_P1_DIS_QPSK_1_2  0xf4bd000f
+
+/* P1_MODCODLSTE */
+#define RSTV0910_P1_MODCODLSTE  0xf4be
+#define FSTV0910_P1_DIS_QPSK_2_5  0xf4be00f0
+#define FSTV0910_P1_DIS_QPSK_1_3  0xf4be000f
+
+/* P1_MODCODLSTF */
+#define RSTV0910_P1_MODCODLSTF  0xf4bf
+#define FSTV0910_P1_DIS_QPSK_1_4  0xf4bf00f0
+#define FSTV0910_P1_DEMOD_INVMODLST  0xf4bf0008
+#define FSTV0910_P1_DEMODOUT_ENABLE  0xf4bf0004
+#define FSTV0910_P1_DDEMOD_NSET  0xf4bf0002
+#define FSTV0910_P1_MODCOD_NSTOCK  0xf4bf0001
+
+/* P1_GAUSSR0 */
+#define RSTV0910_P1_GAUSSR0  0xf4c0
+#define FSTV0910_P1_EN_CCIMODE  0xf4c00080
+#define FSTV0910_P1_R0_GAUSSIEN  0xf4c0007f
+
+/* P1_CCIR0 */
+#define RSTV0910_P1_CCIR0  0xf4c1
+#define FSTV0910_P1_CCIDETECT_PLHONLY  0xf4c10080
+#define FSTV0910_P1_R0_CCI  0xf4c1007f
+
+/* P1_CCIQUANT */
+#define RSTV0910_P1_CCIQUANT  0xf4c2
+#define FSTV0910_P1_CCI_BETA  0xf4c200e0
+#define FSTV0910_P1_CCI_QUANT  0xf4c2001f
+
+/* P1_CCITHRES */
+#define RSTV0910_P1_CCITHRES  0xf4c3
+#define FSTV0910_P1_CCI_THRESHOLD  0xf4c300ff
+
+/* P1_CCIACC */
+#define RSTV0910_P1_CCIACC  0xf4c4
+#define FSTV0910_P1_CCI_VALUE  0xf4c400ff
+
+/* P1_DSTATUS4 */
+#define RSTV0910_P1_DSTATUS4  0xf4c5
+#define FSTV0910_P1_RAINFADE_DETECT  0xf4c50080
+#define FSTV0910_P1_NOTHRES2_FAIL  0xf4c50040
+#define FSTV0910_P1_NOTHRES1_FAIL  0xf4c50020
+#define FSTV0910_P1_DMDPROG_ERROR  0xf4c50004
+#define FSTV0910_P1_CSTENV_DETECT  0xf4c50002
+#define FSTV0910_P1_DETECTION_TRIAX  0xf4c50001
+
+/* P1_DMDRESCFG */
+#define RSTV0910_P1_DMDRESCFG  0xf4c6
+#define FSTV0910_P1_DMDRES_RESET  0xf4c60080
+#define FSTV0910_P1_DMDRES_STRALL  0xf4c60008
+#define FSTV0910_P1_DMDRES_NEWONLY  0xf4c60004
+#define FSTV0910_P1_DMDRES_NOSTORE  0xf4c60002
+
+/* P1_DMDRESADR */
+#define RSTV0910_P1_DMDRESADR  0xf4c7
+#define FSTV0910_P1_DMDRES_VALIDCFR  0xf4c70040
+#define FSTV0910_P1_DMDRES_MEMFULL  0xf4c70030
+#define FSTV0910_P1_DMDRES_RESNBR  0xf4c7000f
+
+/* P1_DMDRESDATA7 */
+#define RSTV0910_P1_DMDRESDATA7  0xf4c8
+#define FSTV0910_P1_DMDRES_DATA7  0xf4c800ff
+
+/* P1_DMDRESDATA6 */
+#define RSTV0910_P1_DMDRESDATA6  0xf4c9
+#define FSTV0910_P1_DMDRES_DATA6  0xf4c900ff
+
+/* P1_DMDRESDATA5 */
+#define RSTV0910_P1_DMDRESDATA5  0xf4ca
+#define FSTV0910_P1_DMDRES_DATA5  0xf4ca00ff
+
+/* P1_DMDRESDATA4 */
+#define RSTV0910_P1_DMDRESDATA4  0xf4cb
+#define FSTV0910_P1_DMDRES_DATA4  0xf4cb00ff
+
+/* P1_DMDRESDATA3 */
+#define RSTV0910_P1_DMDRESDATA3  0xf4cc
+#define FSTV0910_P1_DMDRES_DATA3  0xf4cc00ff
+
+/* P1_DMDRESDATA2 */
+#define RSTV0910_P1_DMDRESDATA2  0xf4cd
+#define FSTV0910_P1_DMDRES_DATA2  0xf4cd00ff
+
+/* P1_DMDRESDATA1 */
+#define RSTV0910_P1_DMDRESDATA1  0xf4ce
+#define FSTV0910_P1_DMDRES_DATA1  0xf4ce00ff
+
+/* P1_DMDRESDATA0 */
+#define RSTV0910_P1_DMDRESDATA0  0xf4cf
+#define FSTV0910_P1_DMDRES_DATA0  0xf4cf00ff
+
+/* P1_FFEI1 */
+#define RSTV0910_P1_FFEI1  0xf4d0
+#define FSTV0910_P1_FFE_ACCI1  0xf4d001ff
+
+/* P1_FFEQ1 */
+#define RSTV0910_P1_FFEQ1  0xf4d1
+#define FSTV0910_P1_FFE_ACCQ1  0xf4d101ff
+
+/* P1_FFEI2 */
+#define RSTV0910_P1_FFEI2  0xf4d2
+#define FSTV0910_P1_FFE_ACCI2  0xf4d201ff
+
+/* P1_FFEQ2 */
+#define RSTV0910_P1_FFEQ2  0xf4d3
+#define FSTV0910_P1_FFE_ACCQ2  0xf4d301ff
+
+/* P1_FFEI3 */
+#define RSTV0910_P1_FFEI3  0xf4d4
+#define FSTV0910_P1_FFE_ACCI3  0xf4d401ff
+
+/* P1_FFEQ3 */
+#define RSTV0910_P1_FFEQ3  0xf4d5
+#define FSTV0910_P1_FFE_ACCQ3  0xf4d501ff
+
+/* P1_FFEI4 */
+#define RSTV0910_P1_FFEI4  0xf4d6
+#define FSTV0910_P1_FFE_ACCI4  0xf4d601ff
+
+/* P1_FFEQ4 */
+#define RSTV0910_P1_FFEQ4  0xf4d7
+#define FSTV0910_P1_FFE_ACCQ4  0xf4d701ff
+
+/* P1_FFECFG */
+#define RSTV0910_P1_FFECFG  0xf4d8
+#define FSTV0910_P1_EQUALFFE_ON  0xf4d80040
+#define FSTV0910_P1_EQUAL_USEDSYMB  0xf4d80030
+#define FSTV0910_P1_MU_EQUALFFE  0xf4d80007
+
+/* P1_TNRCFG2 */
+#define RSTV0910_P1_TNRCFG2  0xf4e1
+#define FSTV0910_P1_TUN_IQSWAP  0xf4e10080
+
+/* P1_SMAPCOEF7 */
+#define RSTV0910_P1_SMAPCOEF7  0xf500
+#define FSTV0910_P1_DIS_QSCALE  0xf5000080
+#define FSTV0910_P1_SMAPCOEF_Q_LLR12  0xf500017f
+
+/* P1_SMAPCOEF6 */
+#define RSTV0910_P1_SMAPCOEF6  0xf501
+#define FSTV0910_P1_DIS_AGC2SCALE  0xf5010080
+#define FSTV0910_P1_ADJ_8PSKLLR1  0xf5010004
+#define FSTV0910_P1_OLD_8PSKLLR1  0xf5010002
+#define FSTV0910_P1_DIS_AB8PSK  0xf5010001
+
+/* P1_SMAPCOEF5 */
+#define RSTV0910_P1_SMAPCOEF5  0xf502
+#define FSTV0910_P1_DIS_8SCALE  0xf5020080
+#define FSTV0910_P1_SMAPCOEF_8P_LLR23  0xf502017f
+
+/* P1_SMAPCOEF4 */
+#define RSTV0910_P1_SMAPCOEF4  0xf503
+#define FSTV0910_P1_SMAPCOEF_16APSK_LLR12  0xf503017f
+
+/* P1_SMAPCOEF3 */
+#define RSTV0910_P1_SMAPCOEF3  0xf504
+#define FSTV0910_P1_SMAPCOEF_16APSK_LLR34  0xf504017f
+
+/* P1_SMAPCOEF2 */
+#define RSTV0910_P1_SMAPCOEF2  0xf505
+#define FSTV0910_P1_SMAPCOEF_32APSK_R2R3  0xf50501f0
+#define FSTV0910_P1_SMAPCOEF_32APSK_LLR2  0xf505010f
+
+/* P1_SMAPCOEF1 */
+#define RSTV0910_P1_SMAPCOEF1  0xf506
+#define FSTV0910_P1_DIS_16SCALE  0xf5060080
+#define FSTV0910_P1_SMAPCOEF_32_LLR34  0xf506017f
+
+/* P1_SMAPCOEF0 */
+#define RSTV0910_P1_SMAPCOEF0  0xf507
+#define FSTV0910_P1_DIS_32SCALE  0xf5070080
+#define FSTV0910_P1_SMAPCOEF_32_LLR15  0xf507017f
+
+/* P1_NOSTHRES1 */
+#define RSTV0910_P1_NOSTHRES1  0xf509
+#define FSTV0910_P1_NOS_THRESHOLD1  0xf50900ff
+
+/* P1_NOSTHRES2 */
+#define RSTV0910_P1_NOSTHRES2  0xf50a
+#define FSTV0910_P1_NOS_THRESHOLD2  0xf50a00ff
+
+/* P1_NOSDIFF1 */
+#define RSTV0910_P1_NOSDIFF1  0xf50b
+#define FSTV0910_P1_NOSTHRES1_DIFF  0xf50b00ff
+
+/* P1_RAINFADE */
+#define RSTV0910_P1_RAINFADE  0xf50c
+#define FSTV0910_P1_NOSTHRES_DATAT  0xf50c0080
+#define FSTV0910_P1_RAINFADE_CNLIMIT  0xf50c0070
+#define FSTV0910_P1_RAINFADE_TIMEOUT  0xf50c0007
+
+/* P1_NOSRAMCFG */
+#define RSTV0910_P1_NOSRAMCFG  0xf50d
+#define FSTV0910_P1_NOSRAM_ACTIVATION  0xf50d0030
+#define FSTV0910_P1_NOSRAM_CNRONLY  0xf50d0008
+#define FSTV0910_P1_NOSRAM_LGNCNR1  0xf50d0007
+
+/* P1_NOSRAMPOS */
+#define RSTV0910_P1_NOSRAMPOS  0xf50e
+#define FSTV0910_P1_NOSRAM_LGNCNR0  0xf50e00f0
+#define FSTV0910_P1_NOSRAM_VALIDE  0xf50e0004
+#define FSTV0910_P1_NOSRAM_CNRVAL1  0xf50e0003
+
+/* P1_NOSRAMVAL */
+#define RSTV0910_P1_NOSRAMVAL  0xf50f
+#define FSTV0910_P1_NOSRAM_CNRVAL0  0xf50f00ff
+
+/* P1_DMDPLHSTAT */
+#define RSTV0910_P1_DMDPLHSTAT  0xf520
+#define FSTV0910_P1_PLH_STATISTIC  0xf52000ff
+
+/* P1_LOCKTIME3 */
+#define RSTV0910_P1_LOCKTIME3  0xf522
+#define FSTV0910_P1_DEMOD_LOCKTIME3  0xf52200ff
+
+/* P1_LOCKTIME2 */
+#define RSTV0910_P1_LOCKTIME2  0xf523
+#define FSTV0910_P1_DEMOD_LOCKTIME2  0xf52300ff
+
+/* P1_LOCKTIME1 */
+#define RSTV0910_P1_LOCKTIME1  0xf524
+#define FSTV0910_P1_DEMOD_LOCKTIME1  0xf52400ff
+
+/* P1_LOCKTIME0 */
+#define RSTV0910_P1_LOCKTIME0  0xf525
+#define FSTV0910_P1_DEMOD_LOCKTIME0  0xf52500ff
+
+/* P1_VITSCALE */
+#define RSTV0910_P1_VITSCALE  0xf532
+#define FSTV0910_P1_NVTH_NOSRANGE  0xf5320080
+#define FSTV0910_P1_VERROR_MAXMODE  0xf5320040
+#define FSTV0910_P1_NSLOWSN_LOCKED  0xf5320008
+#define FSTV0910_P1_DIS_RSFLOCK  0xf5320002
+
+/* P1_FECM */
+#define RSTV0910_P1_FECM  0xf533
+#define FSTV0910_P1_DSS_DVB  0xf5330080
+#define FSTV0910_P1_DSS_SRCH  0xf5330010
+#define FSTV0910_P1_SYNCVIT  0xf5330002
+#define FSTV0910_P1_IQINV  0xf5330001
+
+/* P1_VTH12 */
+#define RSTV0910_P1_VTH12  0xf534
+#define FSTV0910_P1_VTH12  0xf53400ff
+
+/* P1_VTH23 */
+#define RSTV0910_P1_VTH23  0xf535
+#define FSTV0910_P1_VTH23  0xf53500ff
+
+/* P1_VTH34 */
+#define RSTV0910_P1_VTH34  0xf536
+#define FSTV0910_P1_VTH34  0xf53600ff
+
+/* P1_VTH56 */
+#define RSTV0910_P1_VTH56  0xf537
+#define FSTV0910_P1_VTH56  0xf53700ff
+
+/* P1_VTH67 */
+#define RSTV0910_P1_VTH67  0xf538
+#define FSTV0910_P1_VTH67  0xf53800ff
+
+/* P1_VTH78 */
+#define RSTV0910_P1_VTH78  0xf539
+#define FSTV0910_P1_VTH78  0xf53900ff
+
+/* P1_VITCURPUN */
+#define RSTV0910_P1_VITCURPUN  0xf53a
+#define FSTV0910_P1_VIT_CURPUN  0xf53a001f
+
+/* P1_VERROR */
+#define RSTV0910_P1_VERROR  0xf53b
+#define FSTV0910_P1_REGERR_VIT  0xf53b00ff
+
+/* P1_PRVIT */
+#define RSTV0910_P1_PRVIT  0xf53c
+#define FSTV0910_P1_DIS_VTHLOCK  0xf53c0040
+#define FSTV0910_P1_E7_8VIT  0xf53c0020
+#define FSTV0910_P1_E6_7VIT  0xf53c0010
+#define FSTV0910_P1_E5_6VIT  0xf53c0008
+#define FSTV0910_P1_E3_4VIT  0xf53c0004
+#define FSTV0910_P1_E2_3VIT  0xf53c0002
+#define FSTV0910_P1_E1_2VIT  0xf53c0001
+
+/* P1_VAVSRVIT */
+#define RSTV0910_P1_VAVSRVIT  0xf53d
+#define FSTV0910_P1_AMVIT  0xf53d0080
+#define FSTV0910_P1_FROZENVIT  0xf53d0040
+#define FSTV0910_P1_SNVIT  0xf53d0030
+#define FSTV0910_P1_TOVVIT  0xf53d000c
+#define FSTV0910_P1_HYPVIT  0xf53d0003
+
+/* P1_VSTATUSVIT */
+#define RSTV0910_P1_VSTATUSVIT  0xf53e
+#define FSTV0910_P1_PRFVIT  0xf53e0010
+#define FSTV0910_P1_LOCKEDVIT  0xf53e0008
+
+/* P1_VTHINUSE */
+#define RSTV0910_P1_VTHINUSE  0xf53f
+#define FSTV0910_P1_VIT_INUSE  0xf53f00ff
+
+/* P1_KDIV12 */
+#define RSTV0910_P1_KDIV12  0xf540
+#define FSTV0910_P1_K_DIVIDER_12  0xf540007f
+
+/* P1_KDIV23 */
+#define RSTV0910_P1_KDIV23  0xf541
+#define FSTV0910_P1_K_DIVIDER_23  0xf541007f
+
+/* P1_KDIV34 */
+#define RSTV0910_P1_KDIV34  0xf542
+#define FSTV0910_P1_K_DIVIDER_34  0xf542007f
+
+/* P1_KDIV56 */
+#define RSTV0910_P1_KDIV56  0xf543
+#define FSTV0910_P1_K_DIVIDER_56  0xf543007f
+
+/* P1_KDIV67 */
+#define RSTV0910_P1_KDIV67  0xf544
+#define FSTV0910_P1_K_DIVIDER_67  0xf544007f
+
+/* P1_KDIV78 */
+#define RSTV0910_P1_KDIV78  0xf545
+#define FSTV0910_P1_K_DIVIDER_78  0xf545007f
+
+/* P1_TSPIDFLT1 */
+#define RSTV0910_P1_TSPIDFLT1  0xf546
+#define FSTV0910_P1_PIDFLT_ADDR  0xf54600ff
+
+/* P1_TSPIDFLT0 */
+#define RSTV0910_P1_TSPIDFLT0  0xf547
+#define FSTV0910_P1_PIDFLT_DATA  0xf54700ff
+
+/* P1_PDELCTRL0 */
+#define RSTV0910_P1_PDELCTRL0  0xf54f
+#define FSTV0910_P1_ISIOBS_MODE  0xf54f0030
+
+/* P1_PDELCTRL1 */
+#define RSTV0910_P1_PDELCTRL1  0xf550
+#define FSTV0910_P1_INV_MISMASK  0xf5500080
+#define FSTV0910_P1_FILTER_EN  0xf5500020
+#define FSTV0910_P1_HYSTEN  0xf5500008
+#define FSTV0910_P1_HYSTSWRST  0xf5500004
+#define FSTV0910_P1_EN_MIS00  0xf5500002
+#define FSTV0910_P1_ALGOSWRST  0xf5500001
+
+/* P1_PDELCTRL2 */
+#define RSTV0910_P1_PDELCTRL2  0xf551
+#define FSTV0910_P1_FORCE_CONTINUOUS  0xf5510080
+#define FSTV0910_P1_RESET_UPKO_COUNT  0xf5510040
+#define FSTV0910_P1_USER_PKTDELIN_NB  0xf5510020
+#define FSTV0910_P1_FRAME_MODE  0xf5510002
+
+/* P1_HYSTTHRESH */
+#define RSTV0910_P1_HYSTTHRESH  0xf554
+#define FSTV0910_P1_DELIN_LOCKTHRES  0xf55400f0
+#define FSTV0910_P1_DELIN_UNLOCKTHRES  0xf554000f
+
+/* P1_UPLCCST0 */
+#define RSTV0910_P1_UPLCCST0  0xf558
+#define FSTV0910_P1_UPL_CST0  0xf55800f8
+#define FSTV0910_P1_UPL_MODE  0xf5580007
+
+/* P1_ISIENTRY */
+#define RSTV0910_P1_ISIENTRY  0xf55e
+#define FSTV0910_P1_ISI_ENTRY  0xf55e00ff
+
+/* P1_ISIBITENA */
+#define RSTV0910_P1_ISIBITENA  0xf55f
+#define FSTV0910_P1_ISI_BIT_EN  0xf55f00ff
+
+/* P1_MATSTR1 */
+#define RSTV0910_P1_MATSTR1  0xf560
+#define FSTV0910_P1_MATYPE_CURRENT1  0xf56000ff
+
+/* P1_MATSTR0 */
+#define RSTV0910_P1_MATSTR0  0xf561
+#define FSTV0910_P1_MATYPE_CURRENT0  0xf56100ff
+
+/* P1_UPLSTR1 */
+#define RSTV0910_P1_UPLSTR1  0xf562
+#define FSTV0910_P1_UPL_CURRENT1  0xf56200ff
+
+/* P1_UPLSTR0 */
+#define RSTV0910_P1_UPLSTR0  0xf563
+#define FSTV0910_P1_UPL_CURRENT0  0xf56300ff
+
+/* P1_DFLSTR1 */
+#define RSTV0910_P1_DFLSTR1  0xf564
+#define FSTV0910_P1_DFL_CURRENT1  0xf56400ff
+
+/* P1_DFLSTR0 */
+#define RSTV0910_P1_DFLSTR0  0xf565
+#define FSTV0910_P1_DFL_CURRENT0  0xf56500ff
+
+/* P1_SYNCSTR */
+#define RSTV0910_P1_SYNCSTR  0xf566
+#define FSTV0910_P1_SYNC_CURRENT  0xf56600ff
+
+/* P1_SYNCDSTR1 */
+#define RSTV0910_P1_SYNCDSTR1  0xf567
+#define FSTV0910_P1_SYNCD_CURRENT1  0xf56700ff
+
+/* P1_SYNCDSTR0 */
+#define RSTV0910_P1_SYNCDSTR0  0xf568
+#define FSTV0910_P1_SYNCD_CURRENT0  0xf56800ff
+
+/* P1_PDELSTATUS1 */
+#define RSTV0910_P1_PDELSTATUS1  0xf569
+#define FSTV0910_P1_PKTDELIN_DELOCK  0xf5690080
+#define FSTV0910_P1_SYNCDUPDFL_BADDFL  0xf5690040
+#define FSTV0910_P1_UNACCEPTED_STREAM  0xf5690010
+#define FSTV0910_P1_BCH_ERROR_FLAG  0xf5690008
+#define FSTV0910_P1_PKTDELIN_LOCK  0xf5690002
+#define FSTV0910_P1_FIRST_LOCK  0xf5690001
+
+/* P1_PDELSTATUS2 */
+#define RSTV0910_P1_PDELSTATUS2  0xf56a
+#define FSTV0910_P1_FRAME_MODCOD  0xf56a007c
+#define FSTV0910_P1_FRAME_TYPE  0xf56a0003
+
+/* P1_BBFCRCKO1 */
+#define RSTV0910_P1_BBFCRCKO1  0xf56b
+#define FSTV0910_P1_BBHCRC_KOCNT1  0xf56b00ff
+
+/* P1_BBFCRCKO0 */
+#define RSTV0910_P1_BBFCRCKO0  0xf56c
+#define FSTV0910_P1_BBHCRC_KOCNT0  0xf56c00ff
+
+/* P1_UPCRCKO1 */
+#define RSTV0910_P1_UPCRCKO1  0xf56d
+#define FSTV0910_P1_PKTCRC_KOCNT1  0xf56d00ff
+
+/* P1_UPCRCKO0 */
+#define RSTV0910_P1_UPCRCKO0  0xf56e
+#define FSTV0910_P1_PKTCRC_KOCNT0  0xf56e00ff
+
+/* P1_PDELCTRL3 */
+#define RSTV0910_P1_PDELCTRL3  0xf56f
+#define FSTV0910_P1_NOFIFO_BCHERR  0xf56f0020
+#define FSTV0910_P1_PKTDELIN_DELACMERR  0xf56f0010
+
+/* P1_TSSTATEM */
+#define RSTV0910_P1_TSSTATEM  0xf570
+#define FSTV0910_P1_TSDIL_ON  0xf5700080
+#define FSTV0910_P1_TSRS_ON  0xf5700020
+#define FSTV0910_P1_TSDESCRAMB_ON  0xf5700010
+#define FSTV0910_P1_TSFRAME_MODE  0xf5700008
+#define FSTV0910_P1_TS_DISABLE  0xf5700004
+#define FSTV0910_P1_TSACM_MODE  0xf5700002
+#define FSTV0910_P1_TSOUT_NOSYNC  0xf5700001
+
+/* P1_TSSTATEL */
+#define RSTV0910_P1_TSSTATEL  0xf571
+#define FSTV0910_P1_TSNOSYNCBYTE  0xf5710080
+#define FSTV0910_P1_TSPARITY_ON  0xf5710040
+#define FSTV0910_P1_TSISSYI_ON  0xf5710008
+#define FSTV0910_P1_TSNPD_ON  0xf5710004
+#define FSTV0910_P1_TSCRC8_ON  0xf5710002
+#define FSTV0910_P1_TSDSS_PACKET  0xf5710001
+
+/* P1_TSCFGH */
+#define RSTV0910_P1_TSCFGH  0xf572
+#define FSTV0910_P1_TSFIFO_DVBCI  0xf5720080
+#define FSTV0910_P1_TSFIFO_SERIAL  0xf5720040
+#define FSTV0910_P1_TSFIFO_TEIUPDATE  0xf5720020
+#define FSTV0910_P1_TSFIFO_DUTY50  0xf5720010
+#define FSTV0910_P1_TSFIFO_HSGNLOUT  0xf5720008
+#define FSTV0910_P1_TSFIFO_ERRMODE  0xf5720006
+#define FSTV0910_P1_RST_HWARE  0xf5720001
+
+/* P1_TSCFGM */
+#define RSTV0910_P1_TSCFGM  0xf573
+#define FSTV0910_P1_TSFIFO_MANSPEED  0xf57300c0
+#define FSTV0910_P1_TSFIFO_PERMDATA  0xf5730020
+#define FSTV0910_P1_TSFIFO_NONEWSGNL  0xf5730010
+#define FSTV0910_P1_TSFIFO_INVDATA  0xf5730001
+
+/* P1_TSCFGL */
+#define RSTV0910_P1_TSCFGL  0xf574
+#define FSTV0910_P1_TSFIFO_BCLKDEL1CK  0xf57400c0
+#define FSTV0910_P1_BCHERROR_MODE  0xf5740030
+#define FSTV0910_P1_TSFIFO_NSGNL2DATA  0xf5740008
+#define FSTV0910_P1_TSFIFO_EMBINDVB  0xf5740004
+#define FSTV0910_P1_TSFIFO_BITSPEED  0xf5740003
+
+/* P1_TSSYNC */
+#define RSTV0910_P1_TSSYNC  0xf575
+#define FSTV0910_P1_TSFIFO_SYNCMODE  0xf5750018
+
+/* P1_TSINSDELH */
+#define RSTV0910_P1_TSINSDELH  0xf576
+#define FSTV0910_P1_TSDEL_SYNCBYTE  0xf5760080
+#define FSTV0910_P1_TSDEL_XXHEADER  0xf5760040
+#define FSTV0910_P1_TSDEL_DATAFIELD  0xf5760010
+#define FSTV0910_P1_TSINSDEL_RSPARITY  0xf5760002
+#define FSTV0910_P1_TSINSDEL_CRC8  0xf5760001
+
+/* P1_TSINSDELM */
+#define RSTV0910_P1_TSINSDELM  0xf577
+#define FSTV0910_P1_TSINS_EMODCOD  0xf5770010
+#define FSTV0910_P1_TSINS_TOKEN  0xf5770008
+#define FSTV0910_P1_TSINS_XXXERR  0xf5770004
+#define FSTV0910_P1_TSINS_MATYPE  0xf5770002
+#define FSTV0910_P1_TSINS_UPL  0xf5770001
+
+/* P1_TSINSDELL */
+#define RSTV0910_P1_TSINSDELL  0xf578
+#define FSTV0910_P1_TSINS_DFL  0xf5780080
+#define FSTV0910_P1_TSINS_SYNCD  0xf5780040
+#define FSTV0910_P1_TSINS_BLOCLEN  0xf5780020
+#define FSTV0910_P1_TSINS_SIGPCOUNT  0xf5780010
+#define FSTV0910_P1_TSINS_FIFO  0xf5780008
+#define FSTV0910_P1_TSINS_REALPACK  0xf5780004
+#define FSTV0910_P1_TSINS_TSCONFIG  0xf5780002
+#define FSTV0910_P1_TSINS_LATENCY  0xf5780001
+
+/* P1_TSDIVN */
+#define RSTV0910_P1_TSDIVN  0xf579
+#define FSTV0910_P1_TSFIFO_SPEEDMODE  0xf57900c0
+#define FSTV0910_P1_TSFIFO_RISEOK  0xf5790007
+
+/* P1_TSCFG4 */
+#define RSTV0910_P1_TSCFG4  0xf57a
+#define FSTV0910_P1_TSFIFO_TSSPEEDMODE  0xf57a00c0
+
+/* P1_TSSPEED */
+#define RSTV0910_P1_TSSPEED  0xf580
+#define FSTV0910_P1_TSFIFO_OUTSPEED  0xf58000ff
+
+/* P1_TSSTATUS */
+#define RSTV0910_P1_TSSTATUS  0xf581
+#define FSTV0910_P1_TSFIFO_LINEOK  0xf5810080
+#define FSTV0910_P1_TSFIFO_ERROR  0xf5810040
+#define FSTV0910_P1_TSFIFO_NOSYNC  0xf5810010
+#define FSTV0910_P1_TSREGUL_ERROR  0xf5810004
+#define FSTV0910_P1_DIL_READY  0xf5810001
+
+/* P1_TSSTATUS2 */
+#define RSTV0910_P1_TSSTATUS2  0xf582
+#define FSTV0910_P1_TSFIFO_DEMODSEL  0xf5820080
+#define FSTV0910_P1_TSFIFOSPEED_STORE  0xf5820040
+#define FSTV0910_P1_DILXX_RESET  0xf5820020
+#define FSTV0910_P1_SCRAMBDETECT  0xf5820002
+
+/* P1_TSBITRATE1 */
+#define RSTV0910_P1_TSBITRATE1  0xf583
+#define FSTV0910_P1_TSFIFO_BITRATE1  0xf58300ff
+
+/* P1_TSBITRATE0 */
+#define RSTV0910_P1_TSBITRATE0  0xf584
+#define FSTV0910_P1_TSFIFO_BITRATE0  0xf58400ff
+
+/* P1_TSPACKLEN1 */
+#define RSTV0910_P1_TSPACKLEN1  0xf585
+#define FSTV0910_P1_TSFIFO_PACKCPT  0xf58500e0
+
+/* P1_TSDLY2 */
+#define RSTV0910_P1_TSDLY2  0xf589
+#define FSTV0910_P1_SOFFIFO_LATENCY2  0xf589000f
+
+/* P1_TSDLY1 */
+#define RSTV0910_P1_TSDLY1  0xf58a
+#define FSTV0910_P1_SOFFIFO_LATENCY1  0xf58a00ff
+
+/* P1_TSDLY0 */
+#define RSTV0910_P1_TSDLY0  0xf58b
+#define FSTV0910_P1_SOFFIFO_LATENCY0  0xf58b00ff
+
+/* P1_TSNPDAV */
+#define RSTV0910_P1_TSNPDAV  0xf58c
+#define FSTV0910_P1_TSNPD_AVERAGE  0xf58c00ff
+
+/* P1_TSBUFSTAT2 */
+#define RSTV0910_P1_TSBUFSTAT2  0xf58d
+#define FSTV0910_P1_TSISCR_3BYTES  0xf58d0080
+#define FSTV0910_P1_TSISCR_NEWDATA  0xf58d0040
+#define FSTV0910_P1_TSISCR_BUFSTAT2  0xf58d003f
+
+/* P1_TSBUFSTAT1 */
+#define RSTV0910_P1_TSBUFSTAT1  0xf58e
+#define FSTV0910_P1_TSISCR_BUFSTAT1  0xf58e00ff
+
+/* P1_TSBUFSTAT0 */
+#define RSTV0910_P1_TSBUFSTAT0  0xf58f
+#define FSTV0910_P1_TSISCR_BUFSTAT0  0xf58f00ff
+
+/* P1_TSDEBUGL */
+#define RSTV0910_P1_TSDEBUGL  0xf591
+#define FSTV0910_P1_TSFIFO_ERROR_EVNT  0xf5910004
+#define FSTV0910_P1_TSFIFO_OVERFLOWM  0xf5910001
+
+/* P1_TSDLYSET2 */
+#define RSTV0910_P1_TSDLYSET2  0xf592
+#define FSTV0910_P1_SOFFIFO_OFFSET  0xf59200c0
+#define FSTV0910_P1_HYSTERESIS_THRESHOLD  0xf5920030
+#define FSTV0910_P1_SOFFIFO_SYMBOFFS2  0xf592000f
+
+/* P1_TSDLYSET1 */
+#define RSTV0910_P1_TSDLYSET1  0xf593
+#define FSTV0910_P1_SOFFIFO_SYMBOFFS1  0xf59300ff
+
+/* P1_TSDLYSET0 */
+#define RSTV0910_P1_TSDLYSET0  0xf594
+#define FSTV0910_P1_SOFFIFO_SYMBOFFS0  0xf59400ff
+
+/* P1_ERRCTRL1 */
+#define RSTV0910_P1_ERRCTRL1  0xf598
+#define FSTV0910_P1_ERR_SOURCE1  0xf59800f0
+#define FSTV0910_P1_NUM_EVENT1  0xf5980007
+
+/* P1_ERRCNT12 */
+#define RSTV0910_P1_ERRCNT12  0xf599
+#define FSTV0910_P1_ERRCNT1_OLDVALUE  0xf5990080
+#define FSTV0910_P1_ERR_CNT12  0xf599007f
+
+/* P1_ERRCNT11 */
+#define RSTV0910_P1_ERRCNT11  0xf59a
+#define FSTV0910_P1_ERR_CNT11  0xf59a00ff
+
+/* P1_ERRCNT10 */
+#define RSTV0910_P1_ERRCNT10  0xf59b
+#define FSTV0910_P1_ERR_CNT10  0xf59b00ff
+
+/* P1_ERRCTRL2 */
+#define RSTV0910_P1_ERRCTRL2  0xf59c
+#define FSTV0910_P1_ERR_SOURCE2  0xf59c00f0
+#define FSTV0910_P1_NUM_EVENT2  0xf59c0007
+
+/* P1_ERRCNT22 */
+#define RSTV0910_P1_ERRCNT22  0xf59d
+#define FSTV0910_P1_ERRCNT2_OLDVALUE  0xf59d0080
+#define FSTV0910_P1_ERR_CNT22  0xf59d007f
+
+/* P1_ERRCNT21 */
+#define RSTV0910_P1_ERRCNT21  0xf59e
+#define FSTV0910_P1_ERR_CNT21  0xf59e00ff
+
+/* P1_ERRCNT20 */
+#define RSTV0910_P1_ERRCNT20  0xf59f
+#define FSTV0910_P1_ERR_CNT20  0xf59f00ff
+
+/* P1_FECSPY */
+#define RSTV0910_P1_FECSPY  0xf5a0
+#define FSTV0910_P1_SPY_ENABLE  0xf5a00080
+#define FSTV0910_P1_NO_SYNCBYTE  0xf5a00040
+#define FSTV0910_P1_SERIAL_MODE  0xf5a00020
+#define FSTV0910_P1_UNUSUAL_PACKET  0xf5a00010
+#define FSTV0910_P1_BERMETER_DATAMODE  0xf5a0000c
+#define FSTV0910_P1_BERMETER_LMODE  0xf5a00002
+#define FSTV0910_P1_BERMETER_RESET  0xf5a00001
+
+/* P1_FSPYCFG */
+#define RSTV0910_P1_FSPYCFG  0xf5a1
+#define FSTV0910_P1_FECSPY_INPUT  0xf5a100c0
+#define FSTV0910_P1_RST_ON_ERROR  0xf5a10020
+#define FSTV0910_P1_ONE_SHOT  0xf5a10010
+#define FSTV0910_P1_I2C_MODE  0xf5a1000c
+#define FSTV0910_P1_SPY_HYSTERESIS  0xf5a10003
+
+/* P1_FSPYDATA */
+#define RSTV0910_P1_FSPYDATA  0xf5a2
+#define FSTV0910_P1_SPY_STUFFING  0xf5a20080
+#define FSTV0910_P1_SPY_CNULLPKT  0xf5a20020
+#define FSTV0910_P1_SPY_OUTDATA_MODE  0xf5a2001f
+
+/* P1_FSPYOUT */
+#define RSTV0910_P1_FSPYOUT  0xf5a3
+#define FSTV0910_P1_FSPY_DIRECT  0xf5a30080
+#define FSTV0910_P1_STUFF_MODE  0xf5a30007
+
+/* P1_FSTATUS */
+#define RSTV0910_P1_FSTATUS  0xf5a4
+#define FSTV0910_P1_SPY_ENDSIM  0xf5a40080
+#define FSTV0910_P1_VALID_SIM  0xf5a40040
+#define FSTV0910_P1_FOUND_SIGNAL  0xf5a40020
+#define FSTV0910_P1_DSS_SYNCBYTE  0xf5a40010
+#define FSTV0910_P1_RESULT_STATE  0xf5a4000f
+
+/* P1_FBERCPT4 */
+#define RSTV0910_P1_FBERCPT4  0xf5a8
+#define FSTV0910_P1_FBERMETER_CPT4  0xf5a800ff
+
+/* P1_FBERCPT3 */
+#define RSTV0910_P1_FBERCPT3  0xf5a9
+#define FSTV0910_P1_FBERMETER_CPT3  0xf5a900ff
+
+/* P1_FBERCPT2 */
+#define RSTV0910_P1_FBERCPT2  0xf5aa
+#define FSTV0910_P1_FBERMETER_CPT2  0xf5aa00ff
+
+/* P1_FBERCPT1 */
+#define RSTV0910_P1_FBERCPT1  0xf5ab
+#define FSTV0910_P1_FBERMETER_CPT1  0xf5ab00ff
+
+/* P1_FBERCPT0 */
+#define RSTV0910_P1_FBERCPT0  0xf5ac
+#define FSTV0910_P1_FBERMETER_CPT0  0xf5ac00ff
+
+/* P1_FBERERR2 */
+#define RSTV0910_P1_FBERERR2  0xf5ad
+#define FSTV0910_P1_FBERMETER_ERR2  0xf5ad00ff
+
+/* P1_FBERERR1 */
+#define RSTV0910_P1_FBERERR1  0xf5ae
+#define FSTV0910_P1_FBERMETER_ERR1  0xf5ae00ff
+
+/* P1_FBERERR0 */
+#define RSTV0910_P1_FBERERR0  0xf5af
+#define FSTV0910_P1_FBERMETER_ERR0  0xf5af00ff
+
+/* P1_FSPYBER */
+#define RSTV0910_P1_FSPYBER  0xf5b2
+#define FSTV0910_P1_FSPYBER_SYNCBYTE  0xf5b20010
+#define FSTV0910_P1_FSPYBER_UNSYNC  0xf5b20008
+#define FSTV0910_P1_FSPYBER_CTIME  0xf5b20007
+
+/* P1_SFERROR */
+#define RSTV0910_P1_SFERROR  0xf5c1
+#define FSTV0910_P1_SFEC_REGERR_VIT  0xf5c100ff
+
+/* P1_SFECSTATUS */
+#define RSTV0910_P1_SFECSTATUS  0xf5c3
+#define FSTV0910_P1_SFEC_ON  0xf5c30080
+#define FSTV0910_P1_SFEC_OFF  0xf5c30040
+#define FSTV0910_P1_LOCKEDSFEC  0xf5c30008
+#define FSTV0910_P1_SFEC_DELOCK  0xf5c30004
+#define FSTV0910_P1_SFEC_DEMODSEL  0xf5c30002
+#define FSTV0910_P1_SFEC_OVFON  0xf5c30001
+
+/* P1_SFKDIV12 */
+#define RSTV0910_P1_SFKDIV12  0xf5c4
+#define FSTV0910_P1_SFECKDIV12_MAN  0xf5c40080
+
+/* P1_SFKDIV23 */
+#define RSTV0910_P1_SFKDIV23  0xf5c5
+#define FSTV0910_P1_SFECKDIV23_MAN  0xf5c50080
+
+/* P1_SFKDIV34 */
+#define RSTV0910_P1_SFKDIV34  0xf5c6
+#define FSTV0910_P1_SFECKDIV34_MAN  0xf5c60080
+
+/* P1_SFKDIV56 */
+#define RSTV0910_P1_SFKDIV56  0xf5c7
+#define FSTV0910_P1_SFECKDIV56_MAN  0xf5c70080
+
+/* P1_SFKDIV67 */
+#define RSTV0910_P1_SFKDIV67  0xf5c8
+#define FSTV0910_P1_SFECKDIV67_MAN  0xf5c80080
+
+/* P1_SFKDIV78 */
+#define RSTV0910_P1_SFKDIV78  0xf5c9
+#define FSTV0910_P1_SFECKDIV78_MAN  0xf5c90080
+
+/* P1_SFSTATUS */
+#define RSTV0910_P1_SFSTATUS  0xf5cc
+#define FSTV0910_P1_SFEC_LINEOK  0xf5cc0080
+#define FSTV0910_P1_SFEC_ERROR  0xf5cc0040
+#define FSTV0910_P1_SFEC_DATA7  0xf5cc0020
+#define FSTV0910_P1_SFEC_PKTDNBRFAIL  0xf5cc0010
+#define FSTV0910_P1_TSSFEC_DEMODSEL  0xf5cc0008
+#define FSTV0910_P1_SFEC_NOSYNC  0xf5cc0004
+#define FSTV0910_P1_SFEC_UNREGULA  0xf5cc0002
+#define FSTV0910_P1_SFEC_READY  0xf5cc0001
+
+/* P1_SFDLYSET2 */
+#define RSTV0910_P1_SFDLYSET2  0xf5d0
+#define FSTV0910_P1_SFEC_DISABLE  0xf5d00002
+
+/* P1_SFERRCTRL */
+#define RSTV0910_P1_SFERRCTRL  0xf5d8
+#define FSTV0910_P1_SFEC_ERR_SOURCE  0xf5d800f0
+#define FSTV0910_P1_SFEC_NUM_EVENT  0xf5d80007
+
+/* P1_SFERRCNT2 */
+#define RSTV0910_P1_SFERRCNT2  0xf5d9
+#define FSTV0910_P1_SFERRC_OLDVALUE  0xf5d90080
+#define FSTV0910_P1_SFEC_ERR_CNT2  0xf5d9007f
+
+/* P1_SFERRCNT1 */
+#define RSTV0910_P1_SFERRCNT1  0xf5da
+#define FSTV0910_P1_SFEC_ERR_CNT1  0xf5da00ff
+
+/* P1_SFERRCNT0 */
+#define RSTV0910_P1_SFERRCNT0  0xf5db
+#define FSTV0910_P1_SFEC_ERR_CNT0  0xf5db00ff
+
+/* RCCFG2 */
+#define RSTV0910_RCCFG2  0xf600
+#define FSTV0910_TSRCFIFO_DVBCI  0xf6000080
+#define FSTV0910_TSRCFIFO_SERIAL  0xf6000040
+#define FSTV0910_TSRCFIFO_DISABLE  0xf6000020
+#define FSTV0910_TSFIFO_2TORC  0xf6000010
+#define FSTV0910_TSRCFIFO_HSGNLOUT  0xf6000008
+#define FSTV0910_TSRCFIFO_ERRMODE  0xf6000006
+
+/* RCCFG1 */
+#define RSTV0910_RCCFG1  0xf601
+#define FSTV0910_TSRCFIFO_MANSPEED  0xf60100c0
+#define FSTV0910_TSRCFIFO_PERMDATA  0xf6010020
+#define FSTV0910_TSRCFIFO_NONEWSGNL  0xf6010010
+#define FSTV0910_TSRCFIFO_INVDATA  0xf6010001
+
+/* RCCFG0 */
+#define RSTV0910_RCCFG0  0xf602
+#define FSTV0910_TSRCFIFO_BCLKDEL1CK  0xf60200c0
+#define FSTV0910_TSRCFIFO_DUTY50  0xf6020010
+#define FSTV0910_TSRCFIFO_NSGNL2DATA  0xf6020008
+#define FSTV0910_TSRCFIFO_NPDSGNL  0xf6020004
+
+/* RCINSDEL2 */
+#define RSTV0910_RCINSDEL2  0xf603
+#define FSTV0910_TSRCDEL_SYNCBYTE  0xf6030080
+#define FSTV0910_TSRCDEL_XXHEADER  0xf6030040
+#define FSTV0910_TSRCDEL_BBHEADER  0xf6030020
+#define FSTV0910_TSRCDEL_DATAFIELD  0xf6030010
+#define FSTV0910_TSRCINSDEL_ISCR  0xf6030008
+#define FSTV0910_TSRCINSDEL_NPD  0xf6030004
+#define FSTV0910_TSRCINSDEL_RSPARITY  0xf6030002
+#define FSTV0910_TSRCINSDEL_CRC8  0xf6030001
+
+/* RCINSDEL1 */
+#define RSTV0910_RCINSDEL1  0xf604
+#define FSTV0910_TSRCINS_BBPADDING  0xf6040080
+#define FSTV0910_TSRCINS_BCHFEC  0xf6040040
+#define FSTV0910_TSRCINS_EMODCOD  0xf6040010
+#define FSTV0910_TSRCINS_TOKEN  0xf6040008
+#define FSTV0910_TSRCINS_XXXERR  0xf6040004
+#define FSTV0910_TSRCINS_MATYPE  0xf6040002
+#define FSTV0910_TSRCINS_UPL  0xf6040001
+
+/* RCINSDEL0 */
+#define RSTV0910_RCINSDEL0  0xf605
+#define FSTV0910_TSRCINS_DFL  0xf6050080
+#define FSTV0910_TSRCINS_SYNCD  0xf6050040
+#define FSTV0910_TSRCINS_BLOCLEN  0xf6050020
+#define FSTV0910_TSRCINS_SIGPCOUNT  0xf6050010
+#define FSTV0910_TSRCINS_FIFO  0xf6050008
+#define FSTV0910_TSRCINS_REALPACK  0xf6050004
+#define FSTV0910_TSRCINS_TSCONFIG  0xf6050002
+#define FSTV0910_TSRCINS_LATENCY  0xf6050001
+
+/* RCSTATUS */
+#define RSTV0910_RCSTATUS  0xf606
+#define FSTV0910_TSRCFIFO_LINEOK  0xf6060080
+#define FSTV0910_TSRCFIFO_ERROR  0xf6060040
+#define FSTV0910_TSRCREGUL_ERROR  0xf6060010
+#define FSTV0910_TSRCFIFO_DEMODSEL  0xf6060008
+#define FSTV0910_TSRCFIFOSPEED_STORE  0xf6060004
+#define FSTV0910_TSRCSPEED_IMPOSSIBLE  0xf6060001
+
+/* RCSPEED */
+#define RSTV0910_RCSPEED  0xf607
+#define FSTV0910_TSRCFIFO_OUTSPEED  0xf60700ff
+
+/* TSGENERAL */
+#define RSTV0910_TSGENERAL  0xf630
+#define FSTV0910_TSFIFO_DISTS2PAR  0xf6300040
+#define FSTV0910_MUXSTREAM_OUTMODE  0xf6300008
+#define FSTV0910_TSFIFO_PERMPARAL  0xf6300006
+
+/* P1_DISIRQCFG */
+#define RSTV0910_P1_DISIRQCFG  0xf700
+#define FSTV0910_P1_ENRXEND  0xf7000040
+#define FSTV0910_P1_ENRXFIFO8B  0xf7000020
+#define FSTV0910_P1_ENTRFINISH  0xf7000010
+#define FSTV0910_P1_ENTIMEOUT  0xf7000008
+#define FSTV0910_P1_ENTXEND  0xf7000004
+#define FSTV0910_P1_ENTXFIFO64B  0xf7000002
+#define FSTV0910_P1_ENGAPBURST  0xf7000001
+
+/* P1_DISIRQSTAT */
+#define RSTV0910_P1_DISIRQSTAT  0xf701
+#define FSTV0910_P1_IRQRXEND  0xf7010040
+#define FSTV0910_P1_IRQRXFIFO8B  0xf7010020
+#define FSTV0910_P1_IRQTRFINISH  0xf7010010
+#define FSTV0910_P1_IRQTIMEOUT  0xf7010008
+#define FSTV0910_P1_IRQTXEND  0xf7010004
+#define FSTV0910_P1_IRQTXFIFO64B  0xf7010002
+#define FSTV0910_P1_IRQGAPBURST  0xf7010001
+
+/* P1_DISTXCFG */
+#define RSTV0910_P1_DISTXCFG  0xf702
+#define FSTV0910_P1_DISTX_RESET  0xf7020080
+#define FSTV0910_P1_TIM_OFF  0xf7020040
+#define FSTV0910_P1_TIM_CMD  0xf7020030
+#define FSTV0910_P1_ENVELOP  0xf7020008
+#define FSTV0910_P1_DIS_PRECHARGE  0xf7020004
+#define FSTV0910_P1_DISEQC_MODE  0xf7020003
+
+/* P1_DISTXSTATUS */
+#define RSTV0910_P1_DISTXSTATUS  0xf703
+#define FSTV0910_P1_TX_FIFO_FULL  0xf7030040
+#define FSTV0910_P1_TX_IDLE  0xf7030020
+#define FSTV0910_P1_GAP_BURST  0xf7030010
+#define FSTV0910_P1_TX_FIFO64B  0xf7030008
+#define FSTV0910_P1_TX_END  0xf7030004
+#define FSTV0910_P1_TR_TIMEOUT  0xf7030002
+#define FSTV0910_P1_TR_FINISH  0xf7030001
+
+/* P1_DISTXBYTES */
+#define RSTV0910_P1_DISTXBYTES  0xf704
+#define FSTV0910_P1_TXFIFO_BYTES  0xf70400ff
+
+/* P1_DISTXFIFO */
+#define RSTV0910_P1_DISTXFIFO  0xf705
+#define FSTV0910_P1_DISEQC_TX_FIFO  0xf70500ff
+
+/* P1_DISTXF22 */
+#define RSTV0910_P1_DISTXF22  0xf706
+#define FSTV0910_P1_F22TX  0xf70600ff
+
+/* P1_DISTIMEOCFG */
+#define RSTV0910_P1_DISTIMEOCFG  0xf708
+#define FSTV0910_P1_RXCHOICE  0xf7080006
+#define FSTV0910_P1_TIMEOUT_OFF  0xf7080001
+
+/* P1_DISTIMEOUT */
+#define RSTV0910_P1_DISTIMEOUT  0xf709
+#define FSTV0910_P1_TIMEOUT_COUNT  0xf70900ff
+
+/* P1_DISRXCFG */
+#define RSTV0910_P1_DISRXCFG  0xf70a
+#define FSTV0910_P1_DISRX_RESET  0xf70a0080
+#define FSTV0910_P1_EXTENVELOP  0xf70a0040
+#define FSTV0910_P1_PINSELECT  0xf70a0038
+#define FSTV0910_P1_IGNORE_SHORT22K  0xf70a0004
+#define FSTV0910_P1_SIGNED_RXIN  0xf70a0002
+#define FSTV0910_P1_DISRX_ON  0xf70a0001
+
+/* P1_DISRXSTAT1 */
+#define RSTV0910_P1_DISRXSTAT1  0xf70b
+#define FSTV0910_P1_RXEND  0xf70b0080
+#define FSTV0910_P1_RXACTIVE  0xf70b0040
+#define FSTV0910_P1_RXDETECT  0xf70b0020
+#define FSTV0910_P1_CONTTONE  0xf70b0010
+#define FSTV0910_P1_8BFIFOREADY  0xf70b0008
+#define FSTV0910_P1_FIFOEMPTY  0xf70b0004
+
+/* P1_DISRXSTAT0 */
+#define RSTV0910_P1_DISRXSTAT0  0xf70c
+#define FSTV0910_P1_RXFAIL  0xf70c0080
+#define FSTV0910_P1_FIFOPFAIL  0xf70c0040
+#define FSTV0910_P1_RXNONBYTE  0xf70c0020
+#define FSTV0910_P1_FIFOOVF  0xf70c0010
+#define FSTV0910_P1_SHORT22K  0xf70c0008
+#define FSTV0910_P1_RXMSGLOST  0xf70c0004
+
+/* P1_DISRXBYTES */
+#define RSTV0910_P1_DISRXBYTES  0xf70d
+#define FSTV0910_P1_RXFIFO_BYTES  0xf70d001f
+
+/* P1_DISRXPARITY1 */
+#define RSTV0910_P1_DISRXPARITY1  0xf70e
+#define FSTV0910_P1_DISRX_PARITY1  0xf70e00ff
+
+/* P1_DISRXPARITY0 */
+#define RSTV0910_P1_DISRXPARITY0  0xf70f
+#define FSTV0910_P1_DISRX_PARITY0  0xf70f00ff
+
+/* P1_DISRXFIFO */
+#define RSTV0910_P1_DISRXFIFO  0xf710
+#define FSTV0910_P1_DISEQC_RX_FIFO  0xf71000ff
+
+/* P1_DISRXDC1 */
+#define RSTV0910_P1_DISRXDC1  0xf711
+#define FSTV0910_P1_DC_VALUE1  0xf7110103
+
+/* P1_DISRXDC0 */
+#define RSTV0910_P1_DISRXDC0  0xf712
+#define FSTV0910_P1_DC_VALUE0  0xf71200ff
+
+/* P1_DISRXF221 */
+#define RSTV0910_P1_DISRXF221  0xf714
+#define FSTV0910_P1_F22RX1  0xf714000f
+
+/* P1_DISRXF220 */
+#define RSTV0910_P1_DISRXF220  0xf715
+#define FSTV0910_P1_F22RX0  0xf71500ff
+
+/* P1_DISRXF100 */
+#define RSTV0910_P1_DISRXF100  0xf716
+#define FSTV0910_P1_F100RX  0xf71600ff
+
+/* P1_DISRXSHORT22K */
+#define RSTV0910_P1_DISRXSHORT22K  0xf71c
+#define FSTV0910_P1_SHORT22K_LENGTH  0xf71c001f
+
+/* P1_ACRPRESC */
+#define RSTV0910_P1_ACRPRESC  0xf71e
+#define FSTV0910_P1_ACR_PRESC  0xf71e0007
+
+/* P1_ACRDIV */
+#define RSTV0910_P1_ACRDIV  0xf71f
+#define FSTV0910_P1_ACR_DIV  0xf71f00ff
+
+/* P2_DISIRQCFG */
+#define RSTV0910_P2_DISIRQCFG  0xf740
+#define FSTV0910_P2_ENRXEND  0xf7400040
+#define FSTV0910_P2_ENRXFIFO8B  0xf7400020
+#define FSTV0910_P2_ENTRFINISH  0xf7400010
+#define FSTV0910_P2_ENTIMEOUT  0xf7400008
+#define FSTV0910_P2_ENTXEND  0xf7400004
+#define FSTV0910_P2_ENTXFIFO64B  0xf7400002
+#define FSTV0910_P2_ENGAPBURST  0xf7400001
+
+/* P2_DISIRQSTAT */
+#define RSTV0910_P2_DISIRQSTAT  0xf741
+#define FSTV0910_P2_IRQRXEND  0xf7410040
+#define FSTV0910_P2_IRQRXFIFO8B  0xf7410020
+#define FSTV0910_P2_IRQTRFINISH  0xf7410010
+#define FSTV0910_P2_IRQTIMEOUT  0xf7410008
+#define FSTV0910_P2_IRQTXEND  0xf7410004
+#define FSTV0910_P2_IRQTXFIFO64B  0xf7410002
+#define FSTV0910_P2_IRQGAPBURST  0xf7410001
+
+/* P2_DISTXCFG */
+#define RSTV0910_P2_DISTXCFG  0xf742
+#define FSTV0910_P2_DISTX_RESET  0xf7420080
+#define FSTV0910_P2_TIM_OFF  0xf7420040
+#define FSTV0910_P2_TIM_CMD  0xf7420030
+#define FSTV0910_P2_ENVELOP  0xf7420008
+#define FSTV0910_P2_DIS_PRECHARGE  0xf7420004
+#define FSTV0910_P2_DISEQC_MODE  0xf7420003
+
+/* P2_DISTXSTATUS */
+#define RSTV0910_P2_DISTXSTATUS  0xf743
+#define FSTV0910_P2_TX_FIFO_FULL  0xf7430040
+#define FSTV0910_P2_TX_IDLE  0xf7430020
+#define FSTV0910_P2_GAP_BURST  0xf7430010
+#define FSTV0910_P2_TX_FIFO64B  0xf7430008
+#define FSTV0910_P2_TX_END  0xf7430004
+#define FSTV0910_P2_TR_TIMEOUT  0xf7430002
+#define FSTV0910_P2_TR_FINISH  0xf7430001
+
+/* P2_DISTXBYTES */
+#define RSTV0910_P2_DISTXBYTES  0xf744
+#define FSTV0910_P2_TXFIFO_BYTES  0xf74400ff
+
+/* P2_DISTXFIFO */
+#define RSTV0910_P2_DISTXFIFO  0xf745
+#define FSTV0910_P2_DISEQC_TX_FIFO  0xf74500ff
+
+/* P2_DISTXF22 */
+#define RSTV0910_P2_DISTXF22  0xf746
+#define FSTV0910_P2_F22TX  0xf74600ff
+
+/* P2_DISTIMEOCFG */
+#define RSTV0910_P2_DISTIMEOCFG  0xf748
+#define FSTV0910_P2_RXCHOICE  0xf7480006
+#define FSTV0910_P2_TIMEOUT_OFF  0xf7480001
+
+/* P2_DISTIMEOUT */
+#define RSTV0910_P2_DISTIMEOUT  0xf749
+#define FSTV0910_P2_TIMEOUT_COUNT  0xf74900ff
+
+/* P2_DISRXCFG */
+#define RSTV0910_P2_DISRXCFG  0xf74a
+#define FSTV0910_P2_DISRX_RESET  0xf74a0080
+#define FSTV0910_P2_EXTENVELOP  0xf74a0040
+#define FSTV0910_P2_PINSELECT  0xf74a0038
+#define FSTV0910_P2_IGNORE_SHORT22K  0xf74a0004
+#define FSTV0910_P2_SIGNED_RXIN  0xf74a0002
+#define FSTV0910_P2_DISRX_ON  0xf74a0001
+
+/* P2_DISRXSTAT1 */
+#define RSTV0910_P2_DISRXSTAT1  0xf74b
+#define FSTV0910_P2_RXEND  0xf74b0080
+#define FSTV0910_P2_RXACTIVE  0xf74b0040
+#define FSTV0910_P2_RXDETECT  0xf74b0020
+#define FSTV0910_P2_CONTTONE  0xf74b0010
+#define FSTV0910_P2_8BFIFOREADY  0xf74b0008
+#define FSTV0910_P2_FIFOEMPTY  0xf74b0004
+
+/* P2_DISRXSTAT0 */
+#define RSTV0910_P2_DISRXSTAT0  0xf74c
+#define FSTV0910_P2_RXFAIL  0xf74c0080
+#define FSTV0910_P2_FIFOPFAIL  0xf74c0040
+#define FSTV0910_P2_RXNONBYTE  0xf74c0020
+#define FSTV0910_P2_FIFOOVF  0xf74c0010
+#define FSTV0910_P2_SHORT22K  0xf74c0008
+#define FSTV0910_P2_RXMSGLOST  0xf74c0004
+
+/* P2_DISRXBYTES */
+#define RSTV0910_P2_DISRXBYTES  0xf74d
+#define FSTV0910_P2_RXFIFO_BYTES  0xf74d001f
+
+/* P2_DISRXPARITY1 */
+#define RSTV0910_P2_DISRXPARITY1  0xf74e
+#define FSTV0910_P2_DISRX_PARITY1  0xf74e00ff
+
+/* P2_DISRXPARITY0 */
+#define RSTV0910_P2_DISRXPARITY0  0xf74f
+#define FSTV0910_P2_DISRX_PARITY0  0xf74f00ff
+
+/* P2_DISRXFIFO */
+#define RSTV0910_P2_DISRXFIFO  0xf750
+#define FSTV0910_P2_DISEQC_RX_FIFO  0xf75000ff
+
+/* P2_DISRXDC1 */
+#define RSTV0910_P2_DISRXDC1  0xf751
+#define FSTV0910_P2_DC_VALUE1  0xf7510103
+
+/* P2_DISRXDC0 */
+#define RSTV0910_P2_DISRXDC0  0xf752
+#define FSTV0910_P2_DC_VALUE0  0xf75200ff
+
+/* P2_DISRXF221 */
+#define RSTV0910_P2_DISRXF221  0xf754
+#define FSTV0910_P2_F22RX1  0xf754000f
+
+/* P2_DISRXF220 */
+#define RSTV0910_P2_DISRXF220  0xf755
+#define FSTV0910_P2_F22RX0  0xf75500ff
+
+/* P2_DISRXF100 */
+#define RSTV0910_P2_DISRXF100  0xf756
+#define FSTV0910_P2_F100RX  0xf75600ff
+
+/* P2_DISRXSHORT22K */
+#define RSTV0910_P2_DISRXSHORT22K  0xf75c
+#define FSTV0910_P2_SHORT22K_LENGTH  0xf75c001f
+
+/* P2_ACRPRESC */
+#define RSTV0910_P2_ACRPRESC  0xf75e
+#define FSTV0910_P2_ACR_PRESC  0xf75e0007
+
+/* P2_ACRDIV */
+#define RSTV0910_P2_ACRDIV  0xf75f
+#define FSTV0910_P2_ACR_DIV  0xf75f00ff
+
+/* P1_NBITER_NF1 */
+#define RSTV0910_P1_NBITER_NF1  0xfa00
+#define FSTV0910_P1_NBITER_NF_QPSK_1_4  0xfa0000ff
+
+/* P1_NBITER_NF2 */
+#define RSTV0910_P1_NBITER_NF2  0xfa01
+#define FSTV0910_P1_NBITER_NF_QPSK_1_3  0xfa0100ff
+
+/* P1_NBITER_NF3 */
+#define RSTV0910_P1_NBITER_NF3  0xfa02
+#define FSTV0910_P1_NBITER_NF_QPSK_2_5  0xfa0200ff
+
+/* P1_NBITER_NF4 */
+#define RSTV0910_P1_NBITER_NF4  0xfa03
+#define FSTV0910_P1_NBITER_NF_QPSK_1_2  0xfa0300ff
+
+/* P1_NBITER_NF5 */
+#define RSTV0910_P1_NBITER_NF5  0xfa04
+#define FSTV0910_P1_NBITER_NF_QPSK_3_5  0xfa0400ff
+
+/* P1_NBITER_NF6 */
+#define RSTV0910_P1_NBITER_NF6  0xfa05
+#define FSTV0910_P1_NBITER_NF_QPSK_2_3  0xfa0500ff
+
+/* P1_NBITER_NF7 */
+#define RSTV0910_P1_NBITER_NF7  0xfa06
+#define FSTV0910_P1_NBITER_NF_QPSK_3_4  0xfa0600ff
+
+/* P1_NBITER_NF8 */
+#define RSTV0910_P1_NBITER_NF8  0xfa07
+#define FSTV0910_P1_NBITER_NF_QPSK_4_5  0xfa0700ff
+
+/* P1_NBITER_NF9 */
+#define RSTV0910_P1_NBITER_NF9  0xfa08
+#define FSTV0910_P1_NBITER_NF_QPSK_5_6  0xfa0800ff
+
+/* P1_NBITER_NF10 */
+#define RSTV0910_P1_NBITER_NF10  0xfa09
+#define FSTV0910_P1_NBITER_NF_QPSK_8_9  0xfa0900ff
+
+/* P1_NBITER_NF11 */
+#define RSTV0910_P1_NBITER_NF11  0xfa0a
+#define FSTV0910_P1_NBITER_NF_QPSK_9_10  0xfa0a00ff
+
+/* P1_NBITER_NF12 */
+#define RSTV0910_P1_NBITER_NF12  0xfa0b
+#define FSTV0910_P1_NBITER_NF_8PSK_3_5  0xfa0b00ff
+
+/* P1_NBITER_NF13 */
+#define RSTV0910_P1_NBITER_NF13  0xfa0c
+#define FSTV0910_P1_NBITER_NF_8PSK_2_3  0xfa0c00ff
+
+/* P1_NBITER_NF14 */
+#define RSTV0910_P1_NBITER_NF14  0xfa0d
+#define FSTV0910_P1_NBITER_NF_8PSK_3_4  0xfa0d00ff
+
+/* P1_NBITER_NF15 */
+#define RSTV0910_P1_NBITER_NF15  0xfa0e
+#define FSTV0910_P1_NBITER_NF_8PSK_5_6  0xfa0e00ff
+
+/* P1_NBITER_NF16 */
+#define RSTV0910_P1_NBITER_NF16  0xfa0f
+#define FSTV0910_P1_NBITER_NF_8PSK_8_9  0xfa0f00ff
+
+/* P1_NBITER_NF17 */
+#define RSTV0910_P1_NBITER_NF17  0xfa10
+#define FSTV0910_P1_NBITER_NF_8PSK_9_10  0xfa1000ff
+
+/* P1_NBITER_NF18 */
+#define RSTV0910_P1_NBITER_NF18  0xfa11
+#define FSTV0910_P1_NBITER_NF_16APSK_2_3  0xfa1100ff
+
+/* P1_NBITER_NF19 */
+#define RSTV0910_P1_NBITER_NF19  0xfa12
+#define FSTV0910_P1_NBITER_NF_16APSK_3_4  0xfa1200ff
+
+/* P1_NBITER_NF20 */
+#define RSTV0910_P1_NBITER_NF20  0xfa13
+#define FSTV0910_P1_NBITER_NF_16APSK_4_5  0xfa1300ff
+
+/* P1_NBITER_NF21 */
+#define RSTV0910_P1_NBITER_NF21  0xfa14
+#define FSTV0910_P1_NBITER_NF_16APSK_5_6  0xfa1400ff
+
+/* P1_NBITER_NF22 */
+#define RSTV0910_P1_NBITER_NF22  0xfa15
+#define FSTV0910_P1_NBITER_NF_16APSK_8_9  0xfa1500ff
+
+/* P1_NBITER_NF23 */
+#define RSTV0910_P1_NBITER_NF23  0xfa16
+#define FSTV0910_P1_NBITER_NF_16APSK_9_10  0xfa1600ff
+
+/* P1_NBITER_NF24 */
+#define RSTV0910_P1_NBITER_NF24  0xfa17
+#define FSTV0910_P1_NBITER_NF_32APSK_3_4  0xfa1700ff
+
+/* P1_NBITER_NF25 */
+#define RSTV0910_P1_NBITER_NF25  0xfa18
+#define FSTV0910_P1_NBITER_NF_32APSK_4_5  0xfa1800ff
+
+/* P1_NBITER_NF26 */
+#define RSTV0910_P1_NBITER_NF26  0xfa19
+#define FSTV0910_P1_NBITER_NF_32APSK_5_6  0xfa1900ff
+
+/* P1_NBITER_NF27 */
+#define RSTV0910_P1_NBITER_NF27  0xfa1a
+#define FSTV0910_P1_NBITER_NF_32APSK_8_9  0xfa1a00ff
+
+/* P1_NBITER_NF28 */
+#define RSTV0910_P1_NBITER_NF28  0xfa1b
+#define FSTV0910_P1_NBITER_NF_32APSK_9_10  0xfa1b00ff
+
+/* P1_NBITER_SF1 */
+#define RSTV0910_P1_NBITER_SF1  0xfa1c
+#define FSTV0910_P1_NBITER_SF_QPSK_1_4  0xfa1c00ff
+
+/* P1_NBITER_SF2 */
+#define RSTV0910_P1_NBITER_SF2  0xfa1d
+#define FSTV0910_P1_NBITER_SF_QPSK_1_3  0xfa1d00ff
+
+/* P1_NBITER_SF3 */
+#define RSTV0910_P1_NBITER_SF3  0xfa1e
+#define FSTV0910_P1_NBITER_SF_QPSK_2_5  0xfa1e00ff
+
+/* P1_NBITER_SF4 */
+#define RSTV0910_P1_NBITER_SF4  0xfa1f
+#define FSTV0910_P1_NBITER_SF_QPSK_1_2  0xfa1f00ff
+
+/* P1_NBITER_SF5 */
+#define RSTV0910_P1_NBITER_SF5  0xfa20
+#define FSTV0910_P1_NBITER_SF_QPSK_3_5  0xfa2000ff
+
+/* P1_NBITER_SF6 */
+#define RSTV0910_P1_NBITER_SF6  0xfa21
+#define FSTV0910_P1_NBITER_SF_QPSK_2_3  0xfa2100ff
+
+/* P1_NBITER_SF7 */
+#define RSTV0910_P1_NBITER_SF7  0xfa22
+#define FSTV0910_P1_NBITER_SF_QPSK_3_4  0xfa2200ff
+
+/* P1_NBITER_SF8 */
+#define RSTV0910_P1_NBITER_SF8  0xfa23
+#define FSTV0910_P1_NBITER_SF_QPSK_4_5  0xfa2300ff
+
+/* P1_NBITER_SF9 */
+#define RSTV0910_P1_NBITER_SF9  0xfa24
+#define FSTV0910_P1_NBITER_SF_QPSK_5_6  0xfa2400ff
+
+/* P1_NBITER_SF10 */
+#define RSTV0910_P1_NBITER_SF10  0xfa25
+#define FSTV0910_P1_NBITER_SF_QPSK_8_9  0xfa2500ff
+
+/* P1_NBITER_SF12 */
+#define RSTV0910_P1_NBITER_SF12  0xfa26
+#define FSTV0910_P1_NBITER_SF_8PSK_3_5  0xfa2600ff
+
+/* P1_NBITER_SF13 */
+#define RSTV0910_P1_NBITER_SF13  0xfa27
+#define FSTV0910_P1_NBITER_SF_8PSK_2_3  0xfa2700ff
+
+/* P1_NBITER_SF14 */
+#define RSTV0910_P1_NBITER_SF14  0xfa28
+#define FSTV0910_P1_NBITER_SF_8PSK_3_4  0xfa2800ff
+
+/* P1_NBITER_SF15 */
+#define RSTV0910_P1_NBITER_SF15  0xfa29
+#define FSTV0910_P1_NBITER_SF_8PSK_5_6  0xfa2900ff
+
+/* P1_NBITER_SF16 */
+#define RSTV0910_P1_NBITER_SF16  0xfa2a
+#define FSTV0910_P1_NBITER_SF_8PSK_8_9  0xfa2a00ff
+
+/* P1_NBITER_SF18 */
+#define RSTV0910_P1_NBITER_SF18  0xfa2b
+#define FSTV0910_P1_NBITER_SF_16APSK_2_3  0xfa2b00ff
+
+/* P1_NBITER_SF19 */
+#define RSTV0910_P1_NBITER_SF19  0xfa2c
+#define FSTV0910_P1_NBITER_SF_16APSK_3_4  0xfa2c00ff
+
+/* P1_NBITER_SF20 */
+#define RSTV0910_P1_NBITER_SF20  0xfa2d
+#define FSTV0910_P1_NBITER_SF_16APSK_4_5  0xfa2d00ff
+
+/* P1_NBITER_SF21 */
+#define RSTV0910_P1_NBITER_SF21  0xfa2e
+#define FSTV0910_P1_NBITER_SF_16APSK_5_6  0xfa2e00ff
+
+/* P1_NBITER_SF22 */
+#define RSTV0910_P1_NBITER_SF22  0xfa2f
+#define FSTV0910_P1_NBITER_SF_16APSK_8_9  0xfa2f00ff
+
+/* P1_NBITER_SF24 */
+#define RSTV0910_P1_NBITER_SF24  0xfa30
+#define FSTV0910_P1_NBITER_SF_32APSK_3_4  0xfa3000ff
+
+/* P1_NBITER_SF25 */
+#define RSTV0910_P1_NBITER_SF25  0xfa31
+#define FSTV0910_P1_NBITER_SF_32APSK_4_5  0xfa3100ff
+
+/* P1_NBITER_SF26 */
+#define RSTV0910_P1_NBITER_SF26  0xfa32
+#define FSTV0910_P1_NBITER_SF_32APSK_5_6  0xfa3200ff
+
+/* P1_NBITER_SF27 */
+#define RSTV0910_P1_NBITER_SF27  0xfa33
+#define FSTV0910_P1_NBITER_SF_32APSK_8_9  0xfa3300ff
+
+/* SELSATUR6 */
+#define RSTV0910_SELSATUR6  0xfa34
+#define FSTV0910_SSAT_SF27  0xfa340008
+#define FSTV0910_SSAT_SF26  0xfa340004
+#define FSTV0910_SSAT_SF25  0xfa340002
+#define FSTV0910_SSAT_SF24  0xfa340001
+
+/* SELSATUR5 */
+#define RSTV0910_SELSATUR5  0xfa35
+#define FSTV0910_SSAT_SF22  0xfa350080
+#define FSTV0910_SSAT_SF21  0xfa350040
+#define FSTV0910_SSAT_SF20  0xfa350020
+#define FSTV0910_SSAT_SF19  0xfa350010
+#define FSTV0910_SSAT_SF18  0xfa350008
+#define FSTV0910_SSAT_SF16  0xfa350004
+#define FSTV0910_SSAT_SF15  0xfa350002
+#define FSTV0910_SSAT_SF14  0xfa350001
+
+/* SELSATUR4 */
+#define RSTV0910_SELSATUR4  0xfa36
+#define FSTV0910_SSAT_SF13  0xfa360080
+#define FSTV0910_SSAT_SF12  0xfa360040
+#define FSTV0910_SSAT_SF10  0xfa360020
+#define FSTV0910_SSAT_SF9  0xfa360010
+#define FSTV0910_SSAT_SF8  0xfa360008
+#define FSTV0910_SSAT_SF7  0xfa360004
+#define FSTV0910_SSAT_SF6  0xfa360002
+#define FSTV0910_SSAT_SF5  0xfa360001
+
+/* SELSATUR3 */
+#define RSTV0910_SELSATUR3  0xfa37
+#define FSTV0910_SSAT_SF4  0xfa370080
+#define FSTV0910_SSAT_SF3  0xfa370040
+#define FSTV0910_SSAT_SF2  0xfa370020
+#define FSTV0910_SSAT_SF1  0xfa370010
+#define FSTV0910_SSAT_NF28  0xfa370008
+#define FSTV0910_SSAT_NF27  0xfa370004
+#define FSTV0910_SSAT_NF26  0xfa370002
+#define FSTV0910_SSAT_NF25  0xfa370001
+
+/* SELSATUR2 */
+#define RSTV0910_SELSATUR2  0xfa38
+#define FSTV0910_SSAT_NF24  0xfa380080
+#define FSTV0910_SSAT_NF23  0xfa380040
+#define FSTV0910_SSAT_NF22  0xfa380020
+#define FSTV0910_SSAT_NF21  0xfa380010
+#define FSTV0910_SSAT_NF20  0xfa380008
+#define FSTV0910_SSAT_NF19  0xfa380004
+#define FSTV0910_SSAT_NF18  0xfa380002
+#define FSTV0910_SSAT_NF17  0xfa380001
+
+/* SELSATUR1 */
+#define RSTV0910_SELSATUR1  0xfa39
+#define FSTV0910_SSAT_NF16  0xfa390080
+#define FSTV0910_SSAT_NF15  0xfa390040
+#define FSTV0910_SSAT_NF14  0xfa390020
+#define FSTV0910_SSAT_NF13  0xfa390010
+#define FSTV0910_SSAT_NF12  0xfa390008
+#define FSTV0910_SSAT_NF11  0xfa390004
+#define FSTV0910_SSAT_NF10  0xfa390002
+#define FSTV0910_SSAT_NF9  0xfa390001
+
+/* SELSATUR0 */
+#define RSTV0910_SELSATUR0  0xfa3a
+#define FSTV0910_SSAT_NF8  0xfa3a0080
+#define FSTV0910_SSAT_NF7  0xfa3a0040
+#define FSTV0910_SSAT_NF6  0xfa3a0020
+#define FSTV0910_SSAT_NF5  0xfa3a0010
+#define FSTV0910_SSAT_NF4  0xfa3a0008
+#define FSTV0910_SSAT_NF3  0xfa3a0004
+#define FSTV0910_SSAT_NF2  0xfa3a0002
+#define FSTV0910_SSAT_NF1  0xfa3a0001
+
+/* GAINLLR_NF1 */
+#define RSTV0910_GAINLLR_NF1  0xfa40
+#define FSTV0910_GAINLLR_NF_QPSK_1_4  0xfa40007f
+
+/* GAINLLR_NF2 */
+#define RSTV0910_GAINLLR_NF2  0xfa41
+#define FSTV0910_GAINLLR_NF_QPSK_1_3  0xfa41007f
+
+/* GAINLLR_NF3 */
+#define RSTV0910_GAINLLR_NF3  0xfa42
+#define FSTV0910_GAINLLR_NF_QPSK_2_5  0xfa42007f
+
+/* GAINLLR_NF4 */
+#define RSTV0910_GAINLLR_NF4  0xfa43
+#define FSTV0910_GAINLLR_NF_QPSK_1_2  0xfa43007f
+
+/* GAINLLR_NF5 */
+#define RSTV0910_GAINLLR_NF5  0xfa44
+#define FSTV0910_GAINLLR_NF_QPSK_3_5  0xfa44007f
+
+/* GAINLLR_NF6 */
+#define RSTV0910_GAINLLR_NF6  0xfa45
+#define FSTV0910_GAINLLR_NF_QPSK_2_3  0xfa45007f
+
+/* GAINLLR_NF7 */
+#define RSTV0910_GAINLLR_NF7  0xfa46
+#define FSTV0910_GAINLLR_NF_QPSK_3_4  0xfa46007f
+
+/* GAINLLR_NF8 */
+#define RSTV0910_GAINLLR_NF8  0xfa47
+#define FSTV0910_GAINLLR_NF_QPSK_4_5  0xfa47007f
+
+/* GAINLLR_NF9 */
+#define RSTV0910_GAINLLR_NF9  0xfa48
+#define FSTV0910_GAINLLR_NF_QPSK_5_6  0xfa48007f
+
+/* GAINLLR_NF10 */
+#define RSTV0910_GAINLLR_NF10  0xfa49
+#define FSTV0910_GAINLLR_NF_QPSK_8_9  0xfa49007f
+
+/* GAINLLR_NF11 */
+#define RSTV0910_GAINLLR_NF11  0xfa4a
+#define FSTV0910_GAINLLR_NF_QPSK_9_10  0xfa4a007f
+
+/* GAINLLR_NF12 */
+#define RSTV0910_GAINLLR_NF12  0xfa4b
+#define FSTV0910_GAINLLR_NF_8PSK_3_5  0xfa4b007f
+
+/* GAINLLR_NF13 */
+#define RSTV0910_GAINLLR_NF13  0xfa4c
+#define FSTV0910_GAINLLR_NF_8PSK_2_3  0xfa4c007f
+
+/* GAINLLR_NF14 */
+#define RSTV0910_GAINLLR_NF14  0xfa4d
+#define FSTV0910_GAINLLR_NF_8PSK_3_4  0xfa4d007f
+
+/* GAINLLR_NF15 */
+#define RSTV0910_GAINLLR_NF15  0xfa4e
+#define FSTV0910_GAINLLR_NF_8PSK_5_6  0xfa4e007f
+
+/* GAINLLR_NF16 */
+#define RSTV0910_GAINLLR_NF16  0xfa4f
+#define FSTV0910_GAINLLR_NF_8PSK_8_9  0xfa4f007f
+
+/* GAINLLR_NF17 */
+#define RSTV0910_GAINLLR_NF17  0xfa50
+#define FSTV0910_GAINLLR_NF_8PSK_9_10  0xfa50007f
+
+/* GAINLLR_NF18 */
+#define RSTV0910_GAINLLR_NF18  0xfa51
+#define FSTV0910_GAINLLR_NF_16APSK_2_3  0xfa51007f
+
+/* GAINLLR_NF19 */
+#define RSTV0910_GAINLLR_NF19  0xfa52
+#define FSTV0910_GAINLLR_NF_16APSK_3_4  0xfa52007f
+
+/* GAINLLR_NF20 */
+#define RSTV0910_GAINLLR_NF20  0xfa53
+#define FSTV0910_GAINLLR_NF_16APSK_4_5  0xfa53007f
+
+/* GAINLLR_NF21 */
+#define RSTV0910_GAINLLR_NF21  0xfa54
+#define FSTV0910_GAINLLR_NF_16APSK_5_6  0xfa54007f
+
+/* GAINLLR_NF22 */
+#define RSTV0910_GAINLLR_NF22  0xfa55
+#define FSTV0910_GAINLLR_NF_16APSK_8_9  0xfa55007f
+
+/* GAINLLR_NF23 */
+#define RSTV0910_GAINLLR_NF23  0xfa56
+#define FSTV0910_GAINLLR_NF_16APSK_9_10  0xfa56007f
+
+/* GAINLLR_NF24 */
+#define RSTV0910_GAINLLR_NF24  0xfa57
+#define FSTV0910_GAINLLR_NF_32APSK_3_4  0xfa57007f
+
+/* GAINLLR_NF25 */
+#define RSTV0910_GAINLLR_NF25  0xfa58
+#define FSTV0910_GAINLLR_NF_32APSK_4_5  0xfa58007f
+
+/* GAINLLR_NF26 */
+#define RSTV0910_GAINLLR_NF26  0xfa59
+#define FSTV0910_GAINLLR_NF_32APSK_5_6  0xfa59007f
+
+/* GAINLLR_NF27 */
+#define RSTV0910_GAINLLR_NF27  0xfa5a
+#define FSTV0910_GAINLLR_NF_32APSK_8_9  0xfa5a007f
+
+/* GAINLLR_NF28 */
+#define RSTV0910_GAINLLR_NF28  0xfa5b
+#define FSTV0910_GAINLLR_NF_32APSK_9_10  0xfa5b007f
+
+/* GAINLLR_SF1 */
+#define RSTV0910_GAINLLR_SF1  0xfa5c
+#define FSTV0910_GAINLLR_SF_QPSK_1_4  0xfa5c007f
+
+/* GAINLLR_SF2 */
+#define RSTV0910_GAINLLR_SF2  0xfa5d
+#define FSTV0910_GAINLLR_SF_QPSK_1_3  0xfa5d007f
+
+/* GAINLLR_SF3 */
+#define RSTV0910_GAINLLR_SF3  0xfa5e
+#define FSTV0910_GAINLLR_SF_QPSK_2_5  0xfa5e007f
+
+/* GAINLLR_SF4 */
+#define RSTV0910_GAINLLR_SF4  0xfa5f
+#define FSTV0910_GAINLLR_SF_QPSK_1_2  0xfa5f007f
+
+/* GAINLLR_SF5 */
+#define RSTV0910_GAINLLR_SF5  0xfa60
+#define FSTV0910_GAINLLR_SF_QPSK_3_5  0xfa60007f
+
+/* GAINLLR_SF6 */
+#define RSTV0910_GAINLLR_SF6  0xfa61
+#define FSTV0910_GAINLLR_SF_QPSK_2_3  0xfa61007f
+
+/* GAINLLR_SF7 */
+#define RSTV0910_GAINLLR_SF7  0xfa62
+#define FSTV0910_GAINLLR_SF_QPSK_3_4  0xfa62007f
+
+/* GAINLLR_SF8 */
+#define RSTV0910_GAINLLR_SF8  0xfa63
+#define FSTV0910_GAINLLR_SF_QPSK_4_5  0xfa63007f
+
+/* GAINLLR_SF9 */
+#define RSTV0910_GAINLLR_SF9  0xfa64
+#define FSTV0910_GAINLLR_SF_QPSK_5_6  0xfa64007f
+
+/* GAINLLR_SF10 */
+#define RSTV0910_GAINLLR_SF10  0xfa65
+#define FSTV0910_GAINLLR_SF_QPSK_8_9  0xfa65007f
+
+/* GAINLLR_SF12 */
+#define RSTV0910_GAINLLR_SF12  0xfa66
+#define FSTV0910_GAINLLR_SF_8PSK_3_5  0xfa66007f
+
+/* GAINLLR_SF13 */
+#define RSTV0910_GAINLLR_SF13  0xfa67
+#define FSTV0910_GAINLLR_SF_8PSK_2_3  0xfa67007f
+
+/* GAINLLR_SF14 */
+#define RSTV0910_GAINLLR_SF14  0xfa68
+#define FSTV0910_GAINLLR_SF_8PSK_3_4  0xfa68007f
+
+/* GAINLLR_SF15 */
+#define RSTV0910_GAINLLR_SF15  0xfa69
+#define FSTV0910_GAINLLR_SF_8PSK_5_6  0xfa69007f
+
+/* GAINLLR_SF16 */
+#define RSTV0910_GAINLLR_SF16  0xfa6a
+#define FSTV0910_GAINLLR_SF_8PSK_8_9  0xfa6a007f
+
+/* GAINLLR_SF18 */
+#define RSTV0910_GAINLLR_SF18  0xfa6b
+#define FSTV0910_GAINLLR_SF_16APSK_2_3  0xfa6b007f
+
+/* GAINLLR_SF19 */
+#define RSTV0910_GAINLLR_SF19  0xfa6c
+#define FSTV0910_GAINLLR_SF_16APSK_3_4  0xfa6c007f
+
+/* GAINLLR_SF20 */
+#define RSTV0910_GAINLLR_SF20  0xfa6d
+#define FSTV0910_GAINLLR_SF_16APSK_4_5  0xfa6d007f
+
+/* GAINLLR_SF21 */
+#define RSTV0910_GAINLLR_SF21  0xfa6e
+#define FSTV0910_GAINLLR_SF_16APSK_5_6  0xfa6e007f
+
+/* GAINLLR_SF22 */
+#define RSTV0910_GAINLLR_SF22  0xfa6f
+#define FSTV0910_GAINLLR_SF_16APSK_8_9  0xfa6f007f
+
+/* GAINLLR_SF24 */
+#define RSTV0910_GAINLLR_SF24  0xfa70
+#define FSTV0910_GAINLLR_SF_32APSK_3_4  0xfa70007f
+
+/* GAINLLR_SF25 */
+#define RSTV0910_GAINLLR_SF25  0xfa71
+#define FSTV0910_GAINLLR_SF_32APSK_4_5  0xfa71007f
+
+/* GAINLLR_SF26 */
+#define RSTV0910_GAINLLR_SF26  0xfa72
+#define FSTV0910_GAINLLR_SF_32APSK_5_6  0xfa72007f
+
+/* GAINLLR_SF27 */
+#define RSTV0910_GAINLLR_SF27  0xfa73
+#define FSTV0910_GAINLLR_SF_32APSK_8_9  0xfa73007f
+
+/* CFGEXT */
+#define RSTV0910_CFGEXT  0xfa80
+#define FSTV0910_BYPBCH  0xfa800040
+#define FSTV0910_BYPLDPC  0xfa800020
+#define FSTV0910_SHORTMULT  0xfa800004
+
+/* GENCFG */
+#define RSTV0910_GENCFG  0xfa86
+#define FSTV0910_BROADCAST  0xfa860010
+#define FSTV0910_CROSSINPUT  0xfa860002
+#define FSTV0910_DDEMOD  0xfa860001
+
+/* LDPCERR1 */
+#define RSTV0910_LDPCERR1  0xfa96
+#define FSTV0910_LDPC_ERRORS1  0xfa9600ff
+
+/* LDPCERR0 */
+#define RSTV0910_LDPCERR0  0xfa97
+#define FSTV0910_LDPC_ERRORS0  0xfa9700ff
+
+/* BCHERR */
+#define RSTV0910_BCHERR  0xfa98
+#define FSTV0910_ERRORFLAG  0xfa980010
+#define FSTV0910_BCH_ERRORS_COUNTER  0xfa98000f
+
+/* P1_MAXEXTRAITER */
+#define RSTV0910_P1_MAXEXTRAITER  0xfab1
+#define FSTV0910_P1_MAX_EXTRA_ITER  0xfab100ff
+
+/* P2_MAXEXTRAITER */
+#define RSTV0910_P2_MAXEXTRAITER  0xfab6
+#define FSTV0910_P2_MAX_EXTRA_ITER  0xfab600ff
+
+/* P1_STATUSITER */
+#define RSTV0910_P1_STATUSITER  0xfabc
+#define FSTV0910_P1_STATUS_ITER  0xfabc00ff
+
+/* P1_STATUSMAXITER */
+#define RSTV0910_P1_STATUSMAXITER  0xfabd
+#define FSTV0910_P1_STATUS_MAX_ITER  0xfabd00ff
+
+/* P2_STATUSITER */
+#define RSTV0910_P2_STATUSITER  0xfabe
+#define FSTV0910_P2_STATUS_ITER  0xfabe00ff
+
+/* P2_STATUSMAXITER */
+#define RSTV0910_P2_STATUSMAXITER  0xfabf
+#define FSTV0910_P2_STATUS_MAX_ITER  0xfabf00ff
+
+/* P2_NBITER_NF1 */
+#define RSTV0910_P2_NBITER_NF1  0xfac0
+#define FSTV0910_P2_NBITER_NF_QPSK_1_4  0xfac000ff
+
+/* P2_NBITER_NF2 */
+#define RSTV0910_P2_NBITER_NF2  0xfac1
+#define FSTV0910_P2_NBITER_NF_QPSK_1_3  0xfac100ff
+
+/* P2_NBITER_NF3 */
+#define RSTV0910_P2_NBITER_NF3  0xfac2
+#define FSTV0910_P2_NBITER_NF_QPSK_2_5  0xfac200ff
+
+/* P2_NBITER_NF4 */
+#define RSTV0910_P2_NBITER_NF4  0xfac3
+#define FSTV0910_P2_NBITER_NF_QPSK_1_2  0xfac300ff
+
+/* P2_NBITER_NF5 */
+#define RSTV0910_P2_NBITER_NF5  0xfac4
+#define FSTV0910_P2_NBITER_NF_QPSK_3_5  0xfac400ff
+
+/* P2_NBITER_NF6 */
+#define RSTV0910_P2_NBITER_NF6  0xfac5
+#define FSTV0910_P2_NBITER_NF_QPSK_2_3  0xfac500ff
+
+/* P2_NBITER_NF7 */
+#define RSTV0910_P2_NBITER_NF7  0xfac6
+#define FSTV0910_P2_NBITER_NF_QPSK_3_4  0xfac600ff
+
+/* P2_NBITER_NF8 */
+#define RSTV0910_P2_NBITER_NF8  0xfac7
+#define FSTV0910_P2_NBITER_NF_QPSK_4_5  0xfac700ff
+
+/* P2_NBITER_NF9 */
+#define RSTV0910_P2_NBITER_NF9  0xfac8
+#define FSTV0910_P2_NBITER_NF_QPSK_5_6  0xfac800ff
+
+/* P2_NBITER_NF10 */
+#define RSTV0910_P2_NBITER_NF10  0xfac9
+#define FSTV0910_P2_NBITER_NF_QPSK_8_9  0xfac900ff
+
+/* P2_NBITER_NF11 */
+#define RSTV0910_P2_NBITER_NF11  0xfaca
+#define FSTV0910_P2_NBITER_NF_QPSK_9_10  0xfaca00ff
+
+/* P2_NBITER_NF12 */
+#define RSTV0910_P2_NBITER_NF12  0xfacb
+#define FSTV0910_P2_NBITER_NF_8PSK_3_5  0xfacb00ff
+
+/* P2_NBITER_NF13 */
+#define RSTV0910_P2_NBITER_NF13  0xfacc
+#define FSTV0910_P2_NBITER_NF_8PSK_2_3  0xfacc00ff
+
+/* P2_NBITER_NF14 */
+#define RSTV0910_P2_NBITER_NF14  0xfacd
+#define FSTV0910_P2_NBITER_NF_8PSK_3_4  0xfacd00ff
+
+/* P2_NBITER_NF15 */
+#define RSTV0910_P2_NBITER_NF15  0xface
+#define FSTV0910_P2_NBITER_NF_8PSK_5_6  0xface00ff
+
+/* P2_NBITER_NF16 */
+#define RSTV0910_P2_NBITER_NF16  0xfacf
+#define FSTV0910_P2_NBITER_NF_8PSK_8_9  0xfacf00ff
+
+/* P2_NBITER_NF17 */
+#define RSTV0910_P2_NBITER_NF17  0xfad0
+#define FSTV0910_P2_NBITER_NF_8PSK_9_10  0xfad000ff
+
+/* P2_NBITER_NF18 */
+#define RSTV0910_P2_NBITER_NF18  0xfad1
+#define FSTV0910_P2_NBITER_NF_16APSK_2_3  0xfad100ff
+
+/* P2_NBITER_NF19 */
+#define RSTV0910_P2_NBITER_NF19  0xfad2
+#define FSTV0910_P2_NBITER_NF_16APSK_3_4  0xfad200ff
+
+/* P2_NBITER_NF20 */
+#define RSTV0910_P2_NBITER_NF20  0xfad3
+#define FSTV0910_P2_NBITER_NF_16APSK_4_5  0xfad300ff
+
+/* P2_NBITER_NF21 */
+#define RSTV0910_P2_NBITER_NF21  0xfad4
+#define FSTV0910_P2_NBITER_NF_16APSK_5_6  0xfad400ff
+
+/* P2_NBITER_NF22 */
+#define RSTV0910_P2_NBITER_NF22  0xfad5
+#define FSTV0910_P2_NBITER_NF_16APSK_8_9  0xfad500ff
+
+/* P2_NBITER_NF23 */
+#define RSTV0910_P2_NBITER_NF23  0xfad6
+#define FSTV0910_P2_NBITER_NF_16APSK_9_10  0xfad600ff
+
+/* P2_NBITER_NF24 */
+#define RSTV0910_P2_NBITER_NF24  0xfad7
+#define FSTV0910_P2_NBITER_NF_32APSK_3_4  0xfad700ff
+
+/* P2_NBITER_NF25 */
+#define RSTV0910_P2_NBITER_NF25  0xfad8
+#define FSTV0910_P2_NBITER_NF_32APSK_4_5  0xfad800ff
+
+/* P2_NBITER_NF26 */
+#define RSTV0910_P2_NBITER_NF26  0xfad9
+#define FSTV0910_P2_NBITER_NF_32APSK_5_6  0xfad900ff
+
+/* P2_NBITER_NF27 */
+#define RSTV0910_P2_NBITER_NF27  0xfada
+#define FSTV0910_P2_NBITER_NF_32APSK_8_9  0xfada00ff
+
+/* P2_NBITER_NF28 */
+#define RSTV0910_P2_NBITER_NF28  0xfadb
+#define FSTV0910_P2_NBITER_NF_32APSK_9_10  0xfadb00ff
+
+/* P2_NBITER_SF1 */
+#define RSTV0910_P2_NBITER_SF1  0xfadc
+#define FSTV0910_P2_NBITER_SF_QPSK_1_4  0xfadc00ff
+
+/* P2_NBITER_SF2 */
+#define RSTV0910_P2_NBITER_SF2  0xfadd
+#define FSTV0910_P2_NBITER_SF_QPSK_1_3  0xfadd00ff
+
+/* P2_NBITER_SF3 */
+#define RSTV0910_P2_NBITER_SF3  0xfade
+#define FSTV0910_P2_NBITER_SF_QPSK_2_5  0xfade00ff
+
+/* P2_NBITER_SF4 */
+#define RSTV0910_P2_NBITER_SF4  0xfadf
+#define FSTV0910_P2_NBITER_SF_QPSK_1_2  0xfadf00ff
+
+/* P2_NBITER_SF5 */
+#define RSTV0910_P2_NBITER_SF5  0xfae0
+#define FSTV0910_P2_NBITER_SF_QPSK_3_5  0xfae000ff
+
+/* P2_NBITER_SF6 */
+#define RSTV0910_P2_NBITER_SF6  0xfae1
+#define FSTV0910_P2_NBITER_SF_QPSK_2_3  0xfae100ff
+
+/* P2_NBITER_SF7 */
+#define RSTV0910_P2_NBITER_SF7  0xfae2
+#define FSTV0910_P2_NBITER_SF_QPSK_3_4  0xfae200ff
+
+/* P2_NBITER_SF8 */
+#define RSTV0910_P2_NBITER_SF8  0xfae3
+#define FSTV0910_P2_NBITER_SF_QPSK_4_5  0xfae300ff
+
+/* P2_NBITER_SF9 */
+#define RSTV0910_P2_NBITER_SF9  0xfae4
+#define FSTV0910_P2_NBITER_SF_QPSK_5_6  0xfae400ff
+
+/* P2_NBITER_SF10 */
+#define RSTV0910_P2_NBITER_SF10  0xfae5
+#define FSTV0910_P2_NBITER_SF_QPSK_8_9  0xfae500ff
+
+/* P2_NBITER_SF12 */
+#define RSTV0910_P2_NBITER_SF12  0xfae6
+#define FSTV0910_P2_NBITER_SF_8PSK_3_5  0xfae600ff
+
+/* P2_NBITER_SF13 */
+#define RSTV0910_P2_NBITER_SF13  0xfae7
+#define FSTV0910_P2_NBITER_SF_8PSK_2_3  0xfae700ff
+
+/* P2_NBITER_SF14 */
+#define RSTV0910_P2_NBITER_SF14  0xfae8
+#define FSTV0910_P2_NBITER_SF_8PSK_3_4  0xfae800ff
+
+/* P2_NBITER_SF15 */
+#define RSTV0910_P2_NBITER_SF15  0xfae9
+#define FSTV0910_P2_NBITER_SF_8PSK_5_6  0xfae900ff
+
+/* P2_NBITER_SF16 */
+#define RSTV0910_P2_NBITER_SF16  0xfaea
+#define FSTV0910_P2_NBITER_SF_8PSK_8_9  0xfaea00ff
+
+/* P2_NBITER_SF18 */
+#define RSTV0910_P2_NBITER_SF18  0xfaeb
+#define FSTV0910_P2_NBITER_SF_16APSK_2_3  0xfaeb00ff
+
+/* P2_NBITER_SF19 */
+#define RSTV0910_P2_NBITER_SF19  0xfaec
+#define FSTV0910_P2_NBITER_SF_16APSK_3_4  0xfaec00ff
+
+/* P2_NBITER_SF20 */
+#define RSTV0910_P2_NBITER_SF20  0xfaed
+#define FSTV0910_P2_NBITER_SF_16APSK_4_5  0xfaed00ff
+
+/* P2_NBITER_SF21 */
+#define RSTV0910_P2_NBITER_SF21  0xfaee
+#define FSTV0910_P2_NBITER_SF_16APSK_5_6  0xfaee00ff
+
+/* P2_NBITER_SF22 */
+#define RSTV0910_P2_NBITER_SF22  0xfaef
+#define FSTV0910_P2_NBITER_SF_16APSK_8_9  0xfaef00ff
+
+/* P2_NBITER_SF24 */
+#define RSTV0910_P2_NBITER_SF24  0xfaf0
+#define FSTV0910_P2_NBITER_SF_32APSK_3_4  0xfaf000ff
+
+/* P2_NBITER_SF25 */
+#define RSTV0910_P2_NBITER_SF25  0xfaf1
+#define FSTV0910_P2_NBITER_SF_32APSK_4_5  0xfaf100ff
+
+/* P2_NBITER_SF26 */
+#define RSTV0910_P2_NBITER_SF26  0xfaf2
+#define FSTV0910_P2_NBITER_SF_32APSK_5_6  0xfaf200ff
+
+/* P2_NBITER_SF27 */
+#define RSTV0910_P2_NBITER_SF27  0xfaf3
+#define FSTV0910_P2_NBITER_SF_32APSK_8_9  0xfaf300ff
+
+/* TSTRES0 */
+#define RSTV0910_TSTRES0  0xff11
+#define FSTV0910_FRESFEC  0xff110080
+#define FSTV0910_FRESSYM1  0xff110008
+#define FSTV0910_FRESSYM2  0xff110004
+
+/* TSTOUT */
+#define RSTV0910_TSTOUT  0xff12
+#define FSTV0910_TS  0xff12003e
+#define FSTV0910_TEST_OUT  0xff120001
+
+/* TSTIN */
+#define RSTV0910_TSTIN  0xff13
+#define FSTV0910_TEST_IN  0xff130080
+
+/* P2_TSTDMD */
+#define RSTV0910_P2_TSTDMD  0xff20
+#define FSTV0910_P2_CFRINIT_INVZIGZAG  0xff200008
+
+/* P2_TCTL1 */
+#define RSTV0910_P2_TCTL1  0xff24
+#define FSTV0910_P2_TST_IQSYMBSEL  0xff24001f
+
+/* P2_TCTL4 */
+#define RSTV0910_P2_TCTL4  0xff28
+#define FSTV0910_P2_CFR2TOCFR1_DVBS1  0xff2800c0
+
+/* P2_TPKTDELIN */
+#define RSTV0910_P2_TPKTDELIN  0xff37
+#define FSTV0910_P2_CFG_RSPARITYON  0xff370080
+
+/* P1_TSTDMD */
+#define RSTV0910_P1_TSTDMD  0xff40
+#define FSTV0910_P1_CFRINIT_INVZIGZAG  0xff400008
+
+/* P1_TCTL1 */
+#define RSTV0910_P1_TCTL1  0xff44
+#define FSTV0910_P1_TST_IQSYMBSEL  0xff44001f
+
+/* P1_TCTL4 */
+#define RSTV0910_P1_TCTL4  0xff48
+#define FSTV0910_P1_CFR2TOCFR1_DVBS1  0xff4800c0
+
+/* P1_TPKTDELIN */
+#define RSTV0910_P1_TPKTDELIN  0xff57
+#define FSTV0910_P1_CFG_RSPARITYON  0xff570080
+
+/* TSTTSRS */
+#define RSTV0910_TSTTSRS  0xff6d
+#define FSTV0910_TSTRS_DISRS2  0xff6d0002
+#define FSTV0910_TSTRS_DISRS1  0xff6d0001
+
+#define STV0910_NBREGS		975
+#define STV0910_NBFIELDS		1818
diff --git a/drivers/media/dvb-frontends/stv6111.c b/drivers/media/dvb-frontends/stv6111.c
new file mode 100644
index 000000000000..e3e90070e293
--- /dev/null
+++ b/drivers/media/dvb-frontends/stv6111.c
@@ -0,0 +1,681 @@
+/*
+ * Driver for the ST STV6111 tuner
+ *
+ * Copyright (C) 2014 Digital Devices GmbH
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 only, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/init.h>
+#include <linux/delay.h>
+#include <linux/firmware.h>
+#include <linux/i2c.h>
+#include <asm/div64.h>
+
+#include "stv6111.h"
+
+#include "dvb_frontend.h"
+
+struct stv {
+	struct i2c_adapter *i2c;
+	u8 adr;
+
+	u8 reg[11];
+	u32 ref_freq;
+	u32 frequency;
+};
+
+struct slookup {
+	s16 value;
+	u16 reg_value;
+};
+
+static const struct slookup lnagain_nf_lookup[] = {
+	/* Gain *100dB // Reg */
+	{ 2572,	0 },
+	{ 2575, 1 },
+	{ 2580, 2 },
+	{ 2588, 3 },
+	{ 2596, 4 },
+	{ 2611, 5 },
+	{ 2633, 6 },
+	{ 2664, 7 },
+	{ 2701, 8 },
+	{ 2753, 9 },
+	{ 2816, 10 },
+	{ 2902, 11 },
+	{ 2995, 12 },
+	{ 3104, 13 },
+	{ 3215, 14 },
+	{ 3337, 15 },
+	{ 3492, 16 },
+	{ 3614, 17 },
+	{ 3731, 18 },
+	{ 3861, 19 },
+	{ 3988, 20 },
+	{ 4124, 21 },
+	{ 4253, 22 },
+	{ 4386,	23 },
+	{ 4505,	24 },
+	{ 4623,	25 },
+	{ 4726,	26 },
+	{ 4821,	27 },
+	{ 4903,	28 },
+	{ 4979,	29 },
+	{ 5045,	30 },
+	{ 5102,	31 }
+};
+
+static const struct slookup lnagain_iip3_lookup[] = {
+	/* Gain *100dB // reg */
+	{ 1548,	0 },
+	{ 1552,	1 },
+	{ 1569,	2 },
+	{ 1565,	3 },
+	{ 1577,	4 },
+	{ 1594,	5 },
+	{ 1627,	6 },
+	{ 1656,	7 },
+	{ 1700,	8 },
+	{ 1748,	9 },
+	{ 1805,	10 },
+	{ 1896,	11 },
+	{ 1995,	12 },
+	{ 2113,	13 },
+	{ 2233,	14 },
+	{ 2366,	15 },
+	{ 2543,	16 },
+	{ 2687,	17 },
+	{ 2842,	18 },
+	{ 2999,	19 },
+	{ 3167,	20 },
+	{ 3342,	21 },
+	{ 3507,	22 },
+	{ 3679,	23 },
+	{ 3827,	24 },
+	{ 3970,	25 },
+	{ 4094,	26 },
+	{ 4210,	27 },
+	{ 4308,	28 },
+	{ 4396,	29 },
+	{ 4468,	30 },
+	{ 4535,	31 }
+};
+
+static const struct slookup gain_rfagc_lookup[] = {
+	/* Gain *100dB // reg */
+	{ 4870,	0x3000 },
+	{ 4850,	0x3C00 },
+	{ 4800,	0x4500 },
+	{ 4750,	0x4800 },
+	{ 4700,	0x4B00 },
+	{ 4650,	0x4D00 },
+	{ 4600,	0x4F00 },
+	{ 4550,	0x5100 },
+	{ 4500,	0x5200 },
+	{ 4420,	0x5500 },
+	{ 4316,	0x5800 },
+	{ 4200,	0x5B00 },
+	{ 4119,	0x5D00 },
+	{ 3999,	0x6000 },
+	{ 3950,	0x6100 },
+	{ 3876,	0x6300 },
+	{ 3755,	0x6600 },
+	{ 3641,	0x6900 },
+	{ 3567,	0x6B00 },
+	{ 3425,	0x6F00 },
+	{ 3350,	0x7100 },
+	{ 3236,	0x7400 },
+	{ 3118,	0x7700 },
+	{ 3004,	0x7A00 },
+	{ 2917,	0x7C00 },
+	{ 2776,	0x7F00 },
+	{ 2635,	0x8200 },
+	{ 2516,	0x8500 },
+	{ 2406,	0x8800 },
+	{ 2290,	0x8B00 },
+	{ 2170,	0x8E00 },
+	{ 2073,	0x9100 },
+	{ 1949,	0x9400 },
+	{ 1836,	0x9700 },
+	{ 1712,	0x9A00 },
+	{ 1631,	0x9C00 },
+	{ 1515,	0x9F00 },
+	{ 1400,	0xA200 },
+	{ 1323,	0xA400 },
+	{ 1203,	0xA700 },
+	{ 1091,	0xAA00 },
+	{ 1011,	0xAC00 },
+	{ 904,	0xAF00 },
+	{ 787,	0xB200 },
+	{ 685,	0xB500 },
+	{ 571,	0xB800 },
+	{ 464,	0xBB00 },
+	{ 374,	0xBE00 },
+	{ 275,	0xC200 },
+	{ 181,	0xC600 },
+	{ 102,	0xCC00 },
+	{ 49,	0xD900 }
+};
+
+/*
+ * This table is 6 dB too low comapred to the others (probably created with
+ * a different BB_MAG setting)
+ */
+static const struct slookup gain_channel_agc_nf_lookup[] = {
+	/* Gain *100dB // reg */
+	{ 7082,	0x3000 },
+	{ 7052,	0x4000 },
+	{ 7007,	0x4600 },
+	{ 6954,	0x4A00 },
+	{ 6909,	0x4D00 },
+	{ 6833,	0x5100 },
+	{ 6753,	0x5400 },
+	{ 6659,	0x5700 },
+	{ 6561,	0x5A00 },
+	{ 6472,	0x5C00 },
+	{ 6366,	0x5F00 },
+	{ 6259,	0x6100 },
+	{ 6151,	0x6400 },
+	{ 6026,	0x6700 },
+	{ 5920,	0x6900 },
+	{ 5835,	0x6B00 },
+	{ 5770,	0x6C00 },
+	{ 5681,	0x6E00 },
+	{ 5596,	0x7000 },
+	{ 5503,	0x7200 },
+	{ 5429,	0x7300 },
+	{ 5319,	0x7500 },
+	{ 5220,	0x7700 },
+	{ 5111,	0x7900 },
+	{ 4983,	0x7B00 },
+	{ 4876,	0x7D00 },
+	{ 4755,	0x7F00 },
+	{ 4635,	0x8100 },
+	{ 4499,	0x8300 },
+	{ 4405,	0x8500 },
+	{ 4323,	0x8600 },
+	{ 4233,	0x8800 },
+	{ 4156,	0x8A00 },
+	{ 4038,	0x8C00 },
+	{ 3935,	0x8E00 },
+	{ 3823,	0x9000 },
+	{ 3712,	0x9200 },
+	{ 3601,	0x9500 },
+	{ 3511,	0x9700 },
+	{ 3413,	0x9900 },
+	{ 3309,	0x9B00 },
+	{ 3213,	0x9D00 },
+	{ 3088,	0x9F00 },
+	{ 2992,	0xA100 },
+	{ 2878,	0xA400 },
+	{ 2769,	0xA700 },
+	{ 2645,	0xAA00 },
+	{ 2538,	0xAD00 },
+	{ 2441,	0xB000 },
+	{ 2350,	0xB600 },
+	{ 2237,	0xBA00 },
+	{ 2137,	0xBF00 },
+	{ 2039,	0xC500 },
+	{ 1938,	0xDF00 },
+	{ 1927,	0xFF00 }
+};
+
+static const struct slookup gain_channel_agc_iip3_lookup[] = {
+	/* Gain *100dB // reg */
+	{ 7070,	0x3000 },
+	{ 7028,	0x4000 },
+	{ 7019,	0x4600 },
+	{ 6900,	0x4A00 },
+	{ 6811,	0x4D00 },
+	{ 6763,	0x5100 },
+	{ 6690,	0x5400 },
+	{ 6644,	0x5700 },
+	{ 6617,	0x5A00 },
+	{ 6598,	0x5C00 },
+	{ 6462,	0x5F00 },
+	{ 6348,	0x6100 },
+	{ 6197,	0x6400 },
+	{ 6154,	0x6700 },
+	{ 6098,	0x6900 },
+	{ 5893,	0x6B00 },
+	{ 5812,	0x6C00 },
+	{ 5773,	0x6E00 },
+	{ 5723,	0x7000 },
+	{ 5661,	0x7200 },
+	{ 5579,	0x7300 },
+	{ 5460,	0x7500 },
+	{ 5308,	0x7700 },
+	{ 5099,	0x7900 },
+	{ 4910,	0x7B00 },
+	{ 4800,	0x7D00 },
+	{ 4785,	0x7F00 },
+	{ 4635,	0x8100 },
+	{ 4466,	0x8300 },
+	{ 4314,	0x8500 },
+	{ 4295,	0x8600 },
+	{ 4144,	0x8800 },
+	{ 3920,	0x8A00 },
+	{ 3889,	0x8C00 },
+	{ 3771,	0x8E00 },
+	{ 3655,	0x9000 },
+	{ 3446,	0x9200 },
+	{ 3298,	0x9500 },
+	{ 3083,	0x9700 },
+	{ 3015,	0x9900 },
+	{ 2833,	0x9B00 },
+	{ 2746,	0x9D00 },
+	{ 2632,	0x9F00 },
+	{ 2598,	0xA100 },
+	{ 2480,	0xA400 },
+	{ 2236,	0xA700 },
+	{ 2171,	0xAA00 },
+	{ 2060,	0xAD00 },
+	{ 1999,	0xB000 },
+	{ 1974,	0xB600 },
+	{ 1820,	0xBA00 },
+	{ 1741,	0xBF00 },
+	{ 1655,	0xC500 },
+	{ 1444,	0xDF00 },
+	{ 1325,	0xFF00 },
+};
+
+static inline u32 muldiv32(u32 a, u32 b, u32 c)
+{
+	u64 tmp64;
+
+	tmp64 = (u64)a * (u64)b;
+	do_div(tmp64, c);
+
+	return (u32)tmp64;
+}
+
+static int i2c_read(struct i2c_adapter *adap,
+		    u8 adr, u8 *msg, int len, u8 *answ, int alen)
+{
+	struct i2c_msg msgs[2] = { { .addr = adr, .flags = 0,
+				     .buf = msg, .len = len},
+				   { .addr = adr, .flags = I2C_M_RD,
+				     .buf = answ, .len = alen } };
+	if (i2c_transfer(adap, msgs, 2) != 2) {
+		dev_err(&adap->dev, "i2c read error\n");
+		return -EIO;
+	}
+	return 0;
+}
+
+static int i2c_write(struct i2c_adapter *adap, u8 adr, u8 *data, int len)
+{
+	struct i2c_msg msg = {.addr = adr, .flags = 0,
+			      .buf = data, .len = len};
+
+	if (i2c_transfer(adap, &msg, 1) != 1) {
+		dev_err(&adap->dev, "i2c write error\n");
+		return -EIO;
+	}
+	return 0;
+}
+
+static int write_regs(struct stv *state, int reg, int len)
+{
+	u8 d[12];
+
+	memcpy(&d[1], &state->reg[reg], len);
+	d[0] = reg;
+	return i2c_write(state->i2c, state->adr, d, len + 1);
+}
+
+static int write_reg(struct stv *state, u8 reg, u8 val)
+{
+	u8 d[2] = {reg, val};
+
+	return i2c_write(state->i2c, state->adr, d, 2);
+}
+
+static int read_reg(struct stv *state, u8 reg, u8 *val)
+{
+	return i2c_read(state->i2c, state->adr, &reg, 1, val, 1);
+}
+
+static int wait_for_call_done(struct stv *state, u8 mask)
+{
+	int status = 0;
+	u32 lock_retry_count = 10;
+
+	while (lock_retry_count > 0) {
+		u8 regval;
+
+		status = read_reg(state, 9, &regval);
+		if (status < 0)
+			return status;
+
+		if ((regval & mask) == 0)
+			break;
+		usleep_range(4000, 6000);
+		lock_retry_count -= 1;
+
+		status = -EIO;
+	}
+	return status;
+}
+
+static void init_state(struct stv *state)
+{
+	u32 clkdiv = 0;
+	u32 agcmode = 0;
+	u32 agcref = 2;
+	u32 agcset = 0xffffffff;
+	u32 bbmode = 0xffffffff;
+
+	state->reg[0] = 0x08;
+	state->reg[1] = 0x41;
+	state->reg[2] = 0x8f;
+	state->reg[3] = 0x00;
+	state->reg[4] = 0xce;
+	state->reg[5] = 0x54;
+	state->reg[6] = 0x55;
+	state->reg[7] = 0x45;
+	state->reg[8] = 0x46;
+	state->reg[9] = 0xbd;
+	state->reg[10] = 0x11;
+
+	state->ref_freq = 16000;
+
+	if (clkdiv <= 3)
+		state->reg[0x00] |= (clkdiv & 0x03);
+	if (agcmode <= 3) {
+		state->reg[0x03] |= (agcmode << 5);
+		if (agcmode == 0x01)
+			state->reg[0x01] |= 0x30;
+	}
+	if (bbmode <= 3)
+		state->reg[0x01] = (state->reg[0x01] & ~0x30) | (bbmode << 4);
+	if (agcref <= 7)
+		state->reg[0x03] |= agcref;
+	if (agcset <= 31)
+		state->reg[0x02] = (state->reg[0x02] & ~0x1F) | agcset | 0x40;
+}
+
+static int attach_init(struct stv *state)
+{
+	if (write_regs(state, 0, 11))
+		return -ENODEV;
+	return 0;
+}
+
+static void release(struct dvb_frontend *fe)
+{
+	kfree(fe->tuner_priv);
+	fe->tuner_priv = NULL;
+}
+
+static int set_bandwidth(struct dvb_frontend *fe, u32 cutoff_frequency)
+{
+	struct stv *state = fe->tuner_priv;
+	u32 index = (cutoff_frequency + 999999) / 1000000;
+
+	if (index < 6)
+		index = 6;
+	if (index > 50)
+		index = 50;
+	if ((state->reg[0x08] & ~0xFC) == ((index - 6) << 2))
+		return 0;
+
+	state->reg[0x08] = (state->reg[0x08] & ~0xFC) | ((index - 6) << 2);
+	state->reg[0x09] = (state->reg[0x09] & ~0x0C) | 0x08;
+	if (fe->ops.i2c_gate_ctrl)
+		fe->ops.i2c_gate_ctrl(fe, 1);
+	write_regs(state, 0x08, 2);
+	wait_for_call_done(state, 0x08);
+	if (fe->ops.i2c_gate_ctrl)
+		fe->ops.i2c_gate_ctrl(fe, 0);
+	return 0;
+}
+
+static int set_lof(struct stv *state, u32 local_frequency, u32 cutoff_frequency)
+{
+	u32 index = (cutoff_frequency + 999999) / 1000000;
+	u32 frequency = (local_frequency + 500) / 1000;
+	u32 p = 1, psel = 0, fvco, div, frac;
+	u8 icp, tmp;
+
+	if (index < 6)
+		index = 6;
+	if (index > 50)
+		index = 50;
+
+	if (frequency <= 1300000) {
+		p =  4;
+		psel = 1;
+	} else {
+		p =  2;
+		psel = 0;
+	}
+	fvco = frequency * p;
+	div = fvco / state->ref_freq;
+	frac = fvco % state->ref_freq;
+	frac = muldiv32(frac, 0x40000, state->ref_freq);
+
+	icp = 0;
+	if (fvco < 2700000)
+		icp = 0;
+	else if (fvco < 2950000)
+		icp = 1;
+	else if (fvco < 3300000)
+		icp = 2;
+	else if (fvco < 3700000)
+		icp = 3;
+	else if (fvco < 4200000)
+		icp = 5;
+	else if (fvco < 4800000)
+		icp = 6;
+	else
+		icp = 7;
+
+	state->reg[0x02] |= 0x80; /* LNA IIP3 Mode */
+
+	state->reg[0x03] = (state->reg[0x03] & ~0x80) | (psel << 7);
+	state->reg[0x04] = (div & 0xFF);
+	state->reg[0x05] = (((div >> 8) & 0x01) | ((frac & 0x7F) << 1)) & 0xff;
+	state->reg[0x06] = ((frac >> 7) & 0xFF);
+	state->reg[0x07] = (state->reg[0x07] & ~0x07) | ((frac >> 15) & 0x07);
+	state->reg[0x07] = (state->reg[0x07] & ~0xE0) | (icp << 5);
+
+	state->reg[0x08] = (state->reg[0x08] & ~0xFC) | ((index - 6) << 2);
+	/* Start cal vco,CF */
+	state->reg[0x09] = (state->reg[0x09] & ~0x0C) | 0x0C;
+	write_regs(state, 2, 8);
+
+	wait_for_call_done(state, 0x0C);
+
+	usleep_range(10000, 12000);
+
+	read_reg(state, 0x03, &tmp);
+	if (tmp & 0x10)	{
+		state->reg[0x02] &= ~0x80; /* LNA NF Mode */
+		write_regs(state, 2, 1);
+	}
+	read_reg(state, 0x08, &tmp);
+
+	state->frequency = frequency;
+
+	return 0;
+}
+
+static int set_params(struct dvb_frontend *fe)
+{
+	struct stv *state = fe->tuner_priv;
+	struct dtv_frontend_properties *p = &fe->dtv_property_cache;
+	u32 freq, cutoff;
+
+	if (p->delivery_system != SYS_DVBS && p->delivery_system != SYS_DVBS2)
+		return -EINVAL;
+
+	freq = p->frequency * 1000;
+	cutoff = 5000000 + muldiv32(p->symbol_rate, 135, 200);
+
+	if (fe->ops.i2c_gate_ctrl)
+		fe->ops.i2c_gate_ctrl(fe, 1);
+	set_lof(state, freq, cutoff);
+	if (fe->ops.i2c_gate_ctrl)
+		fe->ops.i2c_gate_ctrl(fe, 0);
+	return 0;
+}
+
+static s32 table_lookup(const struct slookup *table,
+			int table_size, u16 reg_value)
+{
+	s32 gain;
+	s32 reg_diff;
+	int imin = 0;
+	int imax = table_size - 1;
+	int i;
+
+	/* Assumes Table[0].RegValue < Table[imax].RegValue */
+	if (reg_value <= table[0].reg_value) {
+		gain = table[0].value;
+	} else if (reg_value >= table[imax].reg_value) {
+		gain = table[imax].value;
+	} else {
+		while ((imax - imin) > 1) {
+			i = (imax + imin) / 2;
+			if ((table[imin].reg_value <= reg_value) &&
+			    (reg_value <= table[i].reg_value))
+				imax = i;
+			else
+				imin = i;
+		}
+		reg_diff = table[imax].reg_value - table[imin].reg_value;
+		gain = table[imin].value;
+		if (reg_diff != 0)
+			gain += ((s32)(reg_value - table[imin].reg_value) *
+				(s32)(table[imax].value
+				- table[imin].value)) / reg_diff;
+	}
+	return gain;
+}
+
+static int get_rf_strength(struct dvb_frontend *fe, u16 *st)
+{
+	struct stv *state = fe->tuner_priv;
+	u16 rfagc = *st;
+	s32 gain;
+
+	if ((state->reg[0x03] & 0x60) == 0) {
+		/* RF Mode, Read AGC ADC */
+		u8 reg = 0;
+
+		if (fe->ops.i2c_gate_ctrl)
+			fe->ops.i2c_gate_ctrl(fe, 1);
+		write_reg(state, 0x02, state->reg[0x02] | 0x20);
+		read_reg(state, 2, &reg);
+		if (reg & 0x20)
+			read_reg(state, 2, &reg);
+		if (fe->ops.i2c_gate_ctrl)
+			fe->ops.i2c_gate_ctrl(fe, 0);
+
+		if ((state->reg[0x02] & 0x80) == 0)
+			/* NF */
+			gain = table_lookup(lnagain_nf_lookup,
+					    ARRAY_SIZE(lnagain_nf_lookup),
+					    reg & 0x1F);
+		else
+			/* IIP3 */
+			gain = table_lookup(lnagain_iip3_lookup,
+					    ARRAY_SIZE(lnagain_iip3_lookup),
+					    reg & 0x1F);
+
+		gain += table_lookup(gain_rfagc_lookup,
+				     ARRAY_SIZE(gain_rfagc_lookup), rfagc);
+
+		gain -= 2400;
+	} else {
+		/* Channel Mode */
+		if ((state->reg[0x02] & 0x80) == 0) {
+			/* NF */
+			gain = table_lookup(
+				gain_channel_agc_nf_lookup,
+				ARRAY_SIZE(gain_channel_agc_nf_lookup), rfagc);
+
+			gain += 600;
+		} else {
+			/* IIP3 */
+			gain = table_lookup(
+				gain_channel_agc_iip3_lookup,
+				ARRAY_SIZE(gain_channel_agc_iip3_lookup),
+				rfagc);
+		}
+	}
+
+	if (state->frequency > 0)
+		/* Tilt correction ( 0.00016 dB/MHz ) */
+		gain -= ((((s32)(state->frequency / 1000) - 1550) * 2) / 12);
+
+	/* + (BBGain * 10); */
+	gain +=  (s32)((state->reg[0x01] & 0xC0) >> 6) * 600 - 1300;
+
+	if (gain < 0)
+		gain = 0;
+	else if (gain > 10000)
+		gain = 10000;
+
+	*st = 10000 - gain;
+
+	return 0;
+}
+
+static const struct dvb_tuner_ops tuner_ops = {
+	.info = {
+		.name		= "ST STV6111",
+		.frequency_min	= 950000,
+		.frequency_max	= 2150000,
+		.frequency_step	= 0
+	},
+	.set_params		= set_params,
+	.release		= release,
+	.get_rf_strength	= get_rf_strength,
+	.set_bandwidth		= set_bandwidth,
+};
+
+struct dvb_frontend *stv6111_attach(struct dvb_frontend *fe,
+				    struct i2c_adapter *i2c, u8 adr)
+{
+	struct stv *state;
+	int stat;
+
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (!state)
+		return NULL;
+	state->adr = adr;
+	state->i2c = i2c;
+	memcpy(&fe->ops.tuner_ops, &tuner_ops, sizeof(struct dvb_tuner_ops));
+	init_state(state);
+
+	if (fe->ops.i2c_gate_ctrl)
+		fe->ops.i2c_gate_ctrl(fe, 1);
+	stat = attach_init(state);
+	if (fe->ops.i2c_gate_ctrl)
+		fe->ops.i2c_gate_ctrl(fe, 0);
+	if (stat < 0) {
+		kfree(state);
+		return NULL;
+	}
+	fe->tuner_priv = state;
+	return fe;
+}
+EXPORT_SYMBOL_GPL(stv6111_attach);
+
+MODULE_DESCRIPTION("ST STV6111 satellite tuner driver");
+MODULE_AUTHOR("Ralph Metzler, Manfred Voelkel");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/dvb-frontends/stv6111.h b/drivers/media/dvb-frontends/stv6111.h
new file mode 100644
index 000000000000..5bc1228dc9bd
--- /dev/null
+++ b/drivers/media/dvb-frontends/stv6111.h
@@ -0,0 +1,21 @@
+#ifndef _STV6111_H_
+#define _STV6111_H_
+
+#if IS_REACHABLE(CONFIG_DVB_STV6111)
+
+struct dvb_frontend *stv6111_attach(struct dvb_frontend *fe,
+				    struct i2c_adapter *i2c, u8 adr);
+
+#else
+
+static inline struct dvb_frontend *stv6111_attach(struct dvb_frontend *fe,
+						  struct i2c_adapter *i2c,
+						  u8 adr)
+{
+	pr_warn("%s: Driver disabled by Kconfig\n", __func__);
+	return NULL;
+}
+
+#endif /* CONFIG_DVB_STV6111 */
+
+#endif /* _STV6111_H_ */
diff --git a/drivers/media/dvb-frontends/zd1301_demod.c b/drivers/media/dvb-frontends/zd1301_demod.c
index fcf5f69de0c5..84a2b25a574a 100644
--- a/drivers/media/dvb-frontends/zd1301_demod.c
+++ b/drivers/media/dvb-frontends/zd1301_demod.c
@@ -445,7 +445,7 @@ static u32 zd1301_demod_i2c_functionality(struct i2c_adapter *adapter)
 	return I2C_FUNC_I2C;
 }
 
-static struct i2c_algorithm zd1301_demod_i2c_algorithm = {
+static const struct i2c_algorithm zd1301_demod_i2c_algorithm = {
 	.master_xfer   = zd1301_demod_i2c_master_xfer,
 	.functionality = zd1301_demod_i2c_functionality,
 };
diff --git a/drivers/media/i2c/Kconfig b/drivers/media/i2c/Kconfig
index 121b3b5394cb..94153895fcd4 100644
--- a/drivers/media/i2c/Kconfig
+++ b/drivers/media/i2c/Kconfig
@@ -204,6 +204,18 @@ config VIDEO_ADV7183
 	  To compile this driver as a module, choose M here: the
 	  module will be called adv7183.
 
+config VIDEO_ADV748X
+	tristate "Analog Devices ADV748x decoder"
+	depends on VIDEO_V4L2 && I2C && VIDEO_V4L2_SUBDEV_API
+	depends on OF
+	select REGMAP_I2C
+	---help---
+	  V4L2 subdevice driver for the Analog Devices
+	  ADV7481 and ADV7482 HDMI/Analog video decoders.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called adv748x.
+
 config VIDEO_ADV7604
 	tristate "Analog Devices ADV7604 decoder"
 	depends on VIDEO_V4L2 && I2C && VIDEO_V4L2_SUBDEV_API
@@ -593,6 +605,30 @@ config VIDEO_OV5647
 	  To compile this driver as a module, choose M here: the
 	  module will be called ov5647.
 
+config VIDEO_OV6650
+	tristate "OmniVision OV6650 sensor support"
+	depends on I2C && VIDEO_V4L2
+	depends on MEDIA_CAMERA_SUPPORT
+	---help---
+	  This is a Video4Linux2 sensor-level driver for the OmniVision
+	  OV6650 camera.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called ov6650.
+
+config VIDEO_OV5670
+	tristate "OmniVision OV5670 sensor support"
+	depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
+	depends on MEDIA_CAMERA_SUPPORT
+	depends on MEDIA_CONTROLLER
+	select V4L2_FWNODE
+	---help---
+	  This is a Video4Linux2 sensor-level driver for the OmniVision
+	  OV5670 camera.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called ov5670.
+
 config VIDEO_OV7640
 	tristate "OmniVision OV7640 sensor support"
 	depends on I2C && VIDEO_V4L2
diff --git a/drivers/media/i2c/Makefile b/drivers/media/i2c/Makefile
index 2c0868fa6034..c843c181dfb9 100644
--- a/drivers/media/i2c/Makefile
+++ b/drivers/media/i2c/Makefile
@@ -28,6 +28,7 @@ obj-$(CONFIG_VIDEO_ADV7180) += adv7180.o
 obj-$(CONFIG_VIDEO_ADV7183) += adv7183.o
 obj-$(CONFIG_VIDEO_ADV7343) += adv7343.o
 obj-$(CONFIG_VIDEO_ADV7393) += adv7393.o
+obj-$(CONFIG_VIDEO_ADV748X) += adv748x/
 obj-$(CONFIG_VIDEO_ADV7604) += adv7604.o
 obj-$(CONFIG_VIDEO_ADV7842) += adv7842.o
 obj-$(CONFIG_VIDEO_AD9389B) += ad9389b.o
@@ -62,6 +63,8 @@ obj-$(CONFIG_VIDEO_OV2640) += ov2640.o
 obj-$(CONFIG_VIDEO_OV5640) += ov5640.o
 obj-$(CONFIG_VIDEO_OV5645) += ov5645.o
 obj-$(CONFIG_VIDEO_OV5647) += ov5647.o
+obj-$(CONFIG_VIDEO_OV5670) += ov5670.o
+obj-$(CONFIG_VIDEO_OV6650) += ov6650.o
 obj-$(CONFIG_VIDEO_OV7640) += ov7640.o
 obj-$(CONFIG_VIDEO_OV7670) += ov7670.o
 obj-$(CONFIG_VIDEO_OV9650) += ov9650.o
diff --git a/drivers/media/i2c/ad9389b.c b/drivers/media/i2c/ad9389b.c
index 50f354144ee7..a056d6cdaaaa 100644
--- a/drivers/media/i2c/ad9389b.c
+++ b/drivers/media/i2c/ad9389b.c
@@ -1208,7 +1208,7 @@ static int ad9389b_remove(struct i2c_client *client)
 
 /* ----------------------------------------------------------------------- */
 
-static struct i2c_device_id ad9389b_id[] = {
+static const struct i2c_device_id ad9389b_id[] = {
 	{ "ad9389b", 0 },
 	{ "ad9889b", 0 },
 	{ }
diff --git a/drivers/media/i2c/adv7180.c b/drivers/media/i2c/adv7180.c
index 78de7ddf5081..3df28f2f9b38 100644
--- a/drivers/media/i2c/adv7180.c
+++ b/drivers/media/i2c/adv7180.c
@@ -1402,6 +1402,8 @@ static int adv7180_remove(struct i2c_client *client)
 
 static const struct i2c_device_id adv7180_id[] = {
 	{ "adv7180", (kernel_ulong_t)&adv7180_info },
+	{ "adv7180cp", (kernel_ulong_t)&adv7180_info },
+	{ "adv7180st", (kernel_ulong_t)&adv7180_info },
 	{ "adv7182", (kernel_ulong_t)&adv7182_info },
 	{ "adv7280", (kernel_ulong_t)&adv7280_info },
 	{ "adv7280-m", (kernel_ulong_t)&adv7280_m_info },
diff --git a/drivers/media/i2c/adv748x/Makefile b/drivers/media/i2c/adv748x/Makefile
new file mode 100644
index 000000000000..c0711e076f1d
--- /dev/null
+++ b/drivers/media/i2c/adv748x/Makefile
@@ -0,0 +1,7 @@
+adv748x-objs	:= \
+		adv748x-afe.o \
+		adv748x-core.o \
+		adv748x-csi2.o \
+		adv748x-hdmi.o
+
+obj-$(CONFIG_VIDEO_ADV748X)	+= adv748x.o
diff --git a/drivers/media/i2c/adv748x/adv748x-afe.c b/drivers/media/i2c/adv748x/adv748x-afe.c
new file mode 100644
index 000000000000..b33ccfc08708
--- /dev/null
+++ b/drivers/media/i2c/adv748x/adv748x-afe.c
@@ -0,0 +1,552 @@
+/*
+ * Driver for Analog Devices ADV748X 8 channel analog front end (AFE) receiver
+ * with standard definition processor (SDP)
+ *
+ * Copyright (C) 2017 Renesas Electronics Corp.
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+
+#include <linux/delay.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/v4l2-dv-timings.h>
+
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-dv-timings.h>
+#include <media/v4l2-ioctl.h>
+
+#include "adv748x.h"
+
+/* -----------------------------------------------------------------------------
+ * SDP
+ */
+
+#define ADV748X_AFE_STD_AD_PAL_BG_NTSC_J_SECAM		0x0
+#define ADV748X_AFE_STD_AD_PAL_BG_NTSC_J_SECAM_PED	0x1
+#define ADV748X_AFE_STD_AD_PAL_N_NTSC_J_SECAM		0x2
+#define ADV748X_AFE_STD_AD_PAL_N_NTSC_M_SECAM		0x3
+#define ADV748X_AFE_STD_NTSC_J				0x4
+#define ADV748X_AFE_STD_NTSC_M				0x5
+#define ADV748X_AFE_STD_PAL60				0x6
+#define ADV748X_AFE_STD_NTSC_443			0x7
+#define ADV748X_AFE_STD_PAL_BG				0x8
+#define ADV748X_AFE_STD_PAL_N				0x9
+#define ADV748X_AFE_STD_PAL_M				0xa
+#define ADV748X_AFE_STD_PAL_M_PED			0xb
+#define ADV748X_AFE_STD_PAL_COMB_N			0xc
+#define ADV748X_AFE_STD_PAL_COMB_N_PED			0xd
+#define ADV748X_AFE_STD_PAL_SECAM			0xe
+#define ADV748X_AFE_STD_PAL_SECAM_PED			0xf
+
+static int adv748x_afe_read_ro_map(struct adv748x_state *state, u8 reg)
+{
+	int ret;
+
+	/* Select SDP Read-Only Main Map */
+	ret = sdp_write(state, ADV748X_SDP_MAP_SEL,
+			ADV748X_SDP_MAP_SEL_RO_MAIN);
+	if (ret < 0)
+		return ret;
+
+	return sdp_read(state, reg);
+}
+
+static int adv748x_afe_status(struct adv748x_afe *afe, u32 *signal,
+			      v4l2_std_id *std)
+{
+	struct adv748x_state *state = adv748x_afe_to_state(afe);
+	int info;
+
+	/* Read status from reg 0x10 of SDP RO Map */
+	info = adv748x_afe_read_ro_map(state, ADV748X_SDP_RO_10);
+	if (info < 0)
+		return info;
+
+	if (signal)
+		*signal = info & ADV748X_SDP_RO_10_IN_LOCK ?
+				0 : V4L2_IN_ST_NO_SIGNAL;
+
+	if (!std)
+		return 0;
+
+	/* Standard not valid if there is no signal */
+	if (!(info & ADV748X_SDP_RO_10_IN_LOCK)) {
+		*std = V4L2_STD_UNKNOWN;
+		return 0;
+	}
+
+	switch (info & 0x70) {
+	case 0x00:
+		*std = V4L2_STD_NTSC;
+		break;
+	case 0x10:
+		*std = V4L2_STD_NTSC_443;
+		break;
+	case 0x20:
+		*std = V4L2_STD_PAL_M;
+		break;
+	case 0x30:
+		*std = V4L2_STD_PAL_60;
+		break;
+	case 0x40:
+		*std = V4L2_STD_PAL;
+		break;
+	case 0x50:
+		*std = V4L2_STD_SECAM;
+		break;
+	case 0x60:
+		*std = V4L2_STD_PAL_Nc | V4L2_STD_PAL_N;
+		break;
+	case 0x70:
+		*std = V4L2_STD_SECAM;
+		break;
+	default:
+		*std = V4L2_STD_UNKNOWN;
+		break;
+	}
+
+	return 0;
+}
+
+static void adv748x_afe_fill_format(struct adv748x_afe *afe,
+				    struct v4l2_mbus_framefmt *fmt)
+{
+	memset(fmt, 0, sizeof(*fmt));
+
+	fmt->code = MEDIA_BUS_FMT_UYVY8_2X8;
+	fmt->colorspace = V4L2_COLORSPACE_SMPTE170M;
+	fmt->field = V4L2_FIELD_ALTERNATE;
+
+	fmt->width = 720;
+	fmt->height = afe->curr_norm & V4L2_STD_525_60 ? 480 : 576;
+
+	/* Field height */
+	fmt->height /= 2;
+}
+
+static int adv748x_afe_std(v4l2_std_id std)
+{
+	if (std == V4L2_STD_PAL_60)
+		return ADV748X_AFE_STD_PAL60;
+	if (std == V4L2_STD_NTSC_443)
+		return ADV748X_AFE_STD_NTSC_443;
+	if (std == V4L2_STD_PAL_N)
+		return ADV748X_AFE_STD_PAL_N;
+	if (std == V4L2_STD_PAL_M)
+		return ADV748X_AFE_STD_PAL_M;
+	if (std == V4L2_STD_PAL_Nc)
+		return ADV748X_AFE_STD_PAL_COMB_N;
+	if (std & V4L2_STD_NTSC)
+		return ADV748X_AFE_STD_NTSC_M;
+	if (std & V4L2_STD_PAL)
+		return ADV748X_AFE_STD_PAL_BG;
+	if (std & V4L2_STD_SECAM)
+		return ADV748X_AFE_STD_PAL_SECAM;
+
+	return -EINVAL;
+}
+
+static void adv748x_afe_set_video_standard(struct adv748x_state *state,
+					  int sdpstd)
+{
+	sdp_clrset(state, ADV748X_SDP_VID_SEL, ADV748X_SDP_VID_SEL_MASK,
+		   (sdpstd & 0xf) << ADV748X_SDP_VID_SEL_SHIFT);
+}
+
+static int adv748x_afe_s_input(struct adv748x_afe *afe, unsigned int input)
+{
+	struct adv748x_state *state = adv748x_afe_to_state(afe);
+
+	return sdp_write(state, ADV748X_SDP_INSEL, input);
+}
+
+static int adv748x_afe_g_pixelaspect(struct v4l2_subdev *sd,
+				     struct v4l2_fract *aspect)
+{
+	struct adv748x_afe *afe = adv748x_sd_to_afe(sd);
+
+	if (afe->curr_norm & V4L2_STD_525_60) {
+		aspect->numerator = 11;
+		aspect->denominator = 10;
+	} else {
+		aspect->numerator = 54;
+		aspect->denominator = 59;
+	}
+
+	return 0;
+}
+
+/* -----------------------------------------------------------------------------
+ * v4l2_subdev_video_ops
+ */
+
+static int adv748x_afe_g_std(struct v4l2_subdev *sd, v4l2_std_id *norm)
+{
+	struct adv748x_afe *afe = adv748x_sd_to_afe(sd);
+
+	*norm = afe->curr_norm;
+
+	return 0;
+}
+
+static int adv748x_afe_s_std(struct v4l2_subdev *sd, v4l2_std_id std)
+{
+	struct adv748x_afe *afe = adv748x_sd_to_afe(sd);
+	struct adv748x_state *state = adv748x_afe_to_state(afe);
+	int afe_std = adv748x_afe_std(std);
+
+	if (afe_std < 0)
+		return afe_std;
+
+	mutex_lock(&state->mutex);
+
+	adv748x_afe_set_video_standard(state, afe_std);
+	afe->curr_norm = std;
+
+	mutex_unlock(&state->mutex);
+
+	return 0;
+}
+
+static int adv748x_afe_querystd(struct v4l2_subdev *sd, v4l2_std_id *std)
+{
+	struct adv748x_afe *afe = adv748x_sd_to_afe(sd);
+	struct adv748x_state *state = adv748x_afe_to_state(afe);
+	int ret;
+
+	mutex_lock(&state->mutex);
+
+	if (afe->streaming) {
+		ret = -EBUSY;
+		goto unlock;
+	}
+
+	/* Set auto detect mode */
+	adv748x_afe_set_video_standard(state,
+				       ADV748X_AFE_STD_AD_PAL_BG_NTSC_J_SECAM);
+
+	msleep(100);
+
+	/* Read detected standard */
+	ret = adv748x_afe_status(afe, NULL, std);
+
+	/* Restore original state */
+	adv748x_afe_set_video_standard(state, afe->curr_norm);
+
+unlock:
+	mutex_unlock(&state->mutex);
+
+	return ret;
+}
+
+static int adv748x_afe_g_tvnorms(struct v4l2_subdev *sd, v4l2_std_id *norm)
+{
+	*norm = V4L2_STD_ALL;
+
+	return 0;
+}
+
+static int adv748x_afe_g_input_status(struct v4l2_subdev *sd, u32 *status)
+{
+	struct adv748x_afe *afe = adv748x_sd_to_afe(sd);
+	struct adv748x_state *state = adv748x_afe_to_state(afe);
+	int ret;
+
+	mutex_lock(&state->mutex);
+
+	ret = adv748x_afe_status(afe, status, NULL);
+
+	mutex_unlock(&state->mutex);
+	return ret;
+}
+
+static int adv748x_afe_s_stream(struct v4l2_subdev *sd, int enable)
+{
+	struct adv748x_afe *afe = adv748x_sd_to_afe(sd);
+	struct adv748x_state *state = adv748x_afe_to_state(afe);
+	int ret, signal = V4L2_IN_ST_NO_SIGNAL;
+
+	mutex_lock(&state->mutex);
+
+	if (enable) {
+		ret = adv748x_afe_s_input(afe, afe->input);
+		if (ret)
+			goto unlock;
+	}
+
+	ret = adv748x_txb_power(state, enable);
+	if (ret)
+		goto unlock;
+
+	afe->streaming = enable;
+
+	adv748x_afe_status(afe, &signal, NULL);
+	if (signal != V4L2_IN_ST_NO_SIGNAL)
+		adv_dbg(state, "Detected SDP signal\n");
+	else
+		adv_dbg(state, "Couldn't detect SDP video signal\n");
+
+unlock:
+	mutex_unlock(&state->mutex);
+
+	return ret;
+}
+
+static const struct v4l2_subdev_video_ops adv748x_afe_video_ops = {
+	.g_std = adv748x_afe_g_std,
+	.s_std = adv748x_afe_s_std,
+	.querystd = adv748x_afe_querystd,
+	.g_tvnorms = adv748x_afe_g_tvnorms,
+	.g_input_status = adv748x_afe_g_input_status,
+	.s_stream = adv748x_afe_s_stream,
+	.g_pixelaspect = adv748x_afe_g_pixelaspect,
+};
+
+/* -----------------------------------------------------------------------------
+ * v4l2_subdev_pad_ops
+ */
+
+static int adv748x_afe_propagate_pixelrate(struct adv748x_afe *afe)
+{
+	struct v4l2_subdev *tx;
+	unsigned int width, height, fps;
+
+	tx = adv748x_get_remote_sd(&afe->pads[ADV748X_AFE_SOURCE]);
+	if (!tx)
+		return -ENOLINK;
+
+	width = 720;
+	height = afe->curr_norm & V4L2_STD_525_60 ? 480 : 576;
+	fps = afe->curr_norm & V4L2_STD_525_60 ? 30 : 25;
+
+	return adv748x_csi2_set_pixelrate(tx, width * height * fps);
+}
+
+static int adv748x_afe_enum_mbus_code(struct v4l2_subdev *sd,
+				      struct v4l2_subdev_pad_config *cfg,
+				      struct v4l2_subdev_mbus_code_enum *code)
+{
+	if (code->index != 0)
+		return -EINVAL;
+
+	code->code = MEDIA_BUS_FMT_UYVY8_2X8;
+
+	return 0;
+}
+
+static int adv748x_afe_get_format(struct v4l2_subdev *sd,
+				      struct v4l2_subdev_pad_config *cfg,
+				      struct v4l2_subdev_format *sdformat)
+{
+	struct adv748x_afe *afe = adv748x_sd_to_afe(sd);
+	struct v4l2_mbus_framefmt *mbusformat;
+
+	/* It makes no sense to get the format of the analog sink pads */
+	if (sdformat->pad != ADV748X_AFE_SOURCE)
+		return -EINVAL;
+
+	if (sdformat->which == V4L2_SUBDEV_FORMAT_TRY) {
+		mbusformat = v4l2_subdev_get_try_format(sd, cfg, sdformat->pad);
+		sdformat->format = *mbusformat;
+	} else {
+		adv748x_afe_fill_format(afe, &sdformat->format);
+		adv748x_afe_propagate_pixelrate(afe);
+	}
+
+	return 0;
+}
+
+static int adv748x_afe_set_format(struct v4l2_subdev *sd,
+				      struct v4l2_subdev_pad_config *cfg,
+				      struct v4l2_subdev_format *sdformat)
+{
+	struct v4l2_mbus_framefmt *mbusformat;
+
+	/* It makes no sense to get the format of the analog sink pads */
+	if (sdformat->pad != ADV748X_AFE_SOURCE)
+		return -EINVAL;
+
+	if (sdformat->which == V4L2_SUBDEV_FORMAT_ACTIVE)
+		return adv748x_afe_get_format(sd, cfg, sdformat);
+
+	mbusformat = v4l2_subdev_get_try_format(sd, cfg, sdformat->pad);
+	*mbusformat = sdformat->format;
+
+	return 0;
+}
+
+static const struct v4l2_subdev_pad_ops adv748x_afe_pad_ops = {
+	.enum_mbus_code = adv748x_afe_enum_mbus_code,
+	.set_fmt = adv748x_afe_set_format,
+	.get_fmt = adv748x_afe_get_format,
+};
+
+/* -----------------------------------------------------------------------------
+ * v4l2_subdev_ops
+ */
+
+static const struct v4l2_subdev_ops adv748x_afe_ops = {
+	.video = &adv748x_afe_video_ops,
+	.pad = &adv748x_afe_pad_ops,
+};
+
+/* -----------------------------------------------------------------------------
+ * Controls
+ */
+
+static const char * const afe_ctrl_frp_menu[] = {
+	"Disabled",
+	"Solid Blue",
+	"Color Bars",
+	"Grey Ramp",
+	"Cb Ramp",
+	"Cr Ramp",
+	"Boundary"
+};
+
+static int adv748x_afe_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct adv748x_afe *afe = adv748x_ctrl_to_afe(ctrl);
+	struct adv748x_state *state = adv748x_afe_to_state(afe);
+	bool enable;
+	int ret;
+
+	ret = sdp_write(state, 0x0e, 0x00);
+	if (ret < 0)
+		return ret;
+
+	switch (ctrl->id) {
+	case V4L2_CID_BRIGHTNESS:
+		ret = sdp_write(state, ADV748X_SDP_BRI, ctrl->val);
+		break;
+	case V4L2_CID_HUE:
+		/* Hue is inverted according to HSL chart */
+		ret = sdp_write(state, ADV748X_SDP_HUE, -ctrl->val);
+		break;
+	case V4L2_CID_CONTRAST:
+		ret = sdp_write(state, ADV748X_SDP_CON, ctrl->val);
+		break;
+	case V4L2_CID_SATURATION:
+		ret = sdp_write(state, ADV748X_SDP_SD_SAT_U, ctrl->val);
+		if (ret)
+			break;
+		ret = sdp_write(state, ADV748X_SDP_SD_SAT_V, ctrl->val);
+		break;
+	case V4L2_CID_TEST_PATTERN:
+		enable = !!ctrl->val;
+
+		/* Enable/Disable Color bar test patterns */
+		ret = sdp_clrset(state, ADV748X_SDP_DEF, ADV748X_SDP_DEF_VAL_EN,
+				enable);
+		if (ret)
+			break;
+		ret = sdp_clrset(state, ADV748X_SDP_FRP, ADV748X_SDP_FRP_MASK,
+				enable ? ctrl->val - 1 : 0);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return ret;
+}
+
+static const struct v4l2_ctrl_ops adv748x_afe_ctrl_ops = {
+	.s_ctrl = adv748x_afe_s_ctrl,
+};
+
+static int adv748x_afe_init_controls(struct adv748x_afe *afe)
+{
+	struct adv748x_state *state = adv748x_afe_to_state(afe);
+
+	v4l2_ctrl_handler_init(&afe->ctrl_hdl, 5);
+
+	/* Use our mutex for the controls */
+	afe->ctrl_hdl.lock = &state->mutex;
+
+	v4l2_ctrl_new_std(&afe->ctrl_hdl, &adv748x_afe_ctrl_ops,
+			  V4L2_CID_BRIGHTNESS, ADV748X_SDP_BRI_MIN,
+			  ADV748X_SDP_BRI_MAX, 1, ADV748X_SDP_BRI_DEF);
+	v4l2_ctrl_new_std(&afe->ctrl_hdl, &adv748x_afe_ctrl_ops,
+			  V4L2_CID_CONTRAST, ADV748X_SDP_CON_MIN,
+			  ADV748X_SDP_CON_MAX, 1, ADV748X_SDP_CON_DEF);
+	v4l2_ctrl_new_std(&afe->ctrl_hdl, &adv748x_afe_ctrl_ops,
+			  V4L2_CID_SATURATION, ADV748X_SDP_SAT_MIN,
+			  ADV748X_SDP_SAT_MAX, 1, ADV748X_SDP_SAT_DEF);
+	v4l2_ctrl_new_std(&afe->ctrl_hdl, &adv748x_afe_ctrl_ops,
+			  V4L2_CID_HUE, ADV748X_SDP_HUE_MIN,
+			  ADV748X_SDP_HUE_MAX, 1, ADV748X_SDP_HUE_DEF);
+
+	v4l2_ctrl_new_std_menu_items(&afe->ctrl_hdl, &adv748x_afe_ctrl_ops,
+				     V4L2_CID_TEST_PATTERN,
+				     ARRAY_SIZE(afe_ctrl_frp_menu) - 1,
+				     0, 0, afe_ctrl_frp_menu);
+
+	afe->sd.ctrl_handler = &afe->ctrl_hdl;
+	if (afe->ctrl_hdl.error) {
+		v4l2_ctrl_handler_free(&afe->ctrl_hdl);
+		return afe->ctrl_hdl.error;
+	}
+
+	return v4l2_ctrl_handler_setup(&afe->ctrl_hdl);
+}
+
+int adv748x_afe_init(struct adv748x_afe *afe)
+{
+	struct adv748x_state *state = adv748x_afe_to_state(afe);
+	int ret;
+	unsigned int i;
+
+	afe->input = 0;
+	afe->streaming = false;
+	afe->curr_norm = V4L2_STD_NTSC_M;
+
+	adv748x_subdev_init(&afe->sd, state, &adv748x_afe_ops,
+			    MEDIA_ENT_F_ATV_DECODER, "afe");
+
+	/* Identify the first connector found as a default input if set */
+	for (i = ADV748X_PORT_AIN0; i <= ADV748X_PORT_AIN7; i++) {
+		/* Inputs and ports are 1-indexed to match the data sheet */
+		if (state->endpoints[i]) {
+			afe->input = i;
+			break;
+		}
+	}
+
+	adv748x_afe_s_input(afe, afe->input);
+
+	adv_dbg(state, "AFE Default input set to %d\n", afe->input);
+
+	/* Entity pads and sinks are 0-indexed to match the pads */
+	for (i = ADV748X_AFE_SINK_AIN0; i <= ADV748X_AFE_SINK_AIN7; i++)
+		afe->pads[i].flags = MEDIA_PAD_FL_SINK;
+
+	afe->pads[ADV748X_AFE_SOURCE].flags = MEDIA_PAD_FL_SOURCE;
+
+	ret = media_entity_pads_init(&afe->sd.entity, ADV748X_AFE_NR_PADS,
+			afe->pads);
+	if (ret)
+		return ret;
+
+	ret = adv748x_afe_init_controls(afe);
+	if (ret)
+		goto error;
+
+	return 0;
+
+error:
+	media_entity_cleanup(&afe->sd.entity);
+
+	return ret;
+}
+
+void adv748x_afe_cleanup(struct adv748x_afe *afe)
+{
+	v4l2_device_unregister_subdev(&afe->sd);
+	media_entity_cleanup(&afe->sd.entity);
+	v4l2_ctrl_handler_free(&afe->ctrl_hdl);
+}
diff --git a/drivers/media/i2c/adv748x/adv748x-core.c b/drivers/media/i2c/adv748x/adv748x-core.c
new file mode 100644
index 000000000000..5ee14f2c2747
--- /dev/null
+++ b/drivers/media/i2c/adv748x/adv748x-core.c
@@ -0,0 +1,833 @@
+/*
+ * Driver for Analog Devices ADV748X HDMI receiver with AFE
+ *
+ * Copyright (C) 2017 Renesas Electronics Corp.
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ *
+ * Authors:
+ *	Koji Matsuoka <koji.matsuoka.xm@renesas.com>
+ *	Niklas Söderlund <niklas.soderlund@ragnatech.se>
+ *	Kieran Bingham <kieran.bingham@ideasonboard.com>
+ */
+
+#include <linux/delay.h>
+#include <linux/errno.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of_graph.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+#include <linux/v4l2-dv-timings.h>
+
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-dv-timings.h>
+#include <media/v4l2-ioctl.h>
+
+#include "adv748x.h"
+
+/* -----------------------------------------------------------------------------
+ * Register manipulation
+ */
+
+static const struct regmap_config adv748x_regmap_cnf[] = {
+	{
+		.name			= "io",
+		.reg_bits		= 8,
+		.val_bits		= 8,
+
+		.max_register		= 0xff,
+		.cache_type		= REGCACHE_NONE,
+	},
+	{
+		.name			= "dpll",
+		.reg_bits		= 8,
+		.val_bits		= 8,
+
+		.max_register		= 0xff,
+		.cache_type		= REGCACHE_NONE,
+	},
+	{
+		.name			= "cp",
+		.reg_bits		= 8,
+		.val_bits		= 8,
+
+		.max_register		= 0xff,
+		.cache_type		= REGCACHE_NONE,
+	},
+	{
+		.name			= "hdmi",
+		.reg_bits		= 8,
+		.val_bits		= 8,
+
+		.max_register		= 0xff,
+		.cache_type		= REGCACHE_NONE,
+	},
+	{
+		.name			= "edid",
+		.reg_bits		= 8,
+		.val_bits		= 8,
+
+		.max_register		= 0xff,
+		.cache_type		= REGCACHE_NONE,
+	},
+	{
+		.name			= "repeater",
+		.reg_bits		= 8,
+		.val_bits		= 8,
+
+		.max_register		= 0xff,
+		.cache_type		= REGCACHE_NONE,
+	},
+	{
+		.name			= "infoframe",
+		.reg_bits		= 8,
+		.val_bits		= 8,
+
+		.max_register		= 0xff,
+		.cache_type		= REGCACHE_NONE,
+	},
+	{
+		.name			= "cec",
+		.reg_bits		= 8,
+		.val_bits		= 8,
+
+		.max_register		= 0xff,
+		.cache_type		= REGCACHE_NONE,
+	},
+	{
+		.name			= "sdp",
+		.reg_bits		= 8,
+		.val_bits		= 8,
+
+		.max_register		= 0xff,
+		.cache_type		= REGCACHE_NONE,
+	},
+
+	{
+		.name			= "txb",
+		.reg_bits		= 8,
+		.val_bits		= 8,
+
+		.max_register		= 0xff,
+		.cache_type		= REGCACHE_NONE,
+	},
+	{
+		.name			= "txa",
+		.reg_bits		= 8,
+		.val_bits		= 8,
+
+		.max_register		= 0xff,
+		.cache_type		= REGCACHE_NONE,
+	},
+};
+
+static int adv748x_configure_regmap(struct adv748x_state *state, int region)
+{
+	int err;
+
+	if (!state->i2c_clients[region])
+		return -ENODEV;
+
+	state->regmap[region] =
+		devm_regmap_init_i2c(state->i2c_clients[region],
+				     &adv748x_regmap_cnf[region]);
+
+	if (IS_ERR(state->regmap[region])) {
+		err = PTR_ERR(state->regmap[region]);
+		adv_err(state,
+			"Error initializing regmap %d with error %d\n",
+			region, err);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/* Default addresses for the I2C pages */
+static int adv748x_i2c_addresses[ADV748X_PAGE_MAX] = {
+	ADV748X_I2C_IO,
+	ADV748X_I2C_DPLL,
+	ADV748X_I2C_CP,
+	ADV748X_I2C_HDMI,
+	ADV748X_I2C_EDID,
+	ADV748X_I2C_REPEATER,
+	ADV748X_I2C_INFOFRAME,
+	ADV748X_I2C_CEC,
+	ADV748X_I2C_SDP,
+	ADV748X_I2C_TXB,
+	ADV748X_I2C_TXA,
+};
+
+static int adv748x_read_check(struct adv748x_state *state,
+			      int client_page, u8 reg)
+{
+	struct i2c_client *client = state->i2c_clients[client_page];
+	int err;
+	unsigned int val;
+
+	err = regmap_read(state->regmap[client_page], reg, &val);
+
+	if (err) {
+		adv_err(state, "error reading %02x, %02x\n",
+				client->addr, reg);
+		return err;
+	}
+
+	return val;
+}
+
+int adv748x_read(struct adv748x_state *state, u8 page, u8 reg)
+{
+	return adv748x_read_check(state, page, reg);
+}
+
+int adv748x_write(struct adv748x_state *state, u8 page, u8 reg, u8 value)
+{
+	return regmap_write(state->regmap[page], reg, value);
+}
+
+/* adv748x_write_block(): Write raw data with a maximum of I2C_SMBUS_BLOCK_MAX
+ * size to one or more registers.
+ *
+ * A value of zero will be returned on success, a negative errno will
+ * be returned in error cases.
+ */
+int adv748x_write_block(struct adv748x_state *state, int client_page,
+			unsigned int init_reg, const void *val,
+			size_t val_len)
+{
+	struct regmap *regmap = state->regmap[client_page];
+
+	if (val_len > I2C_SMBUS_BLOCK_MAX)
+		val_len = I2C_SMBUS_BLOCK_MAX;
+
+	return regmap_raw_write(regmap, init_reg, val, val_len);
+}
+
+static struct i2c_client *adv748x_dummy_client(struct adv748x_state *state,
+					       u8 addr, u8 io_reg)
+{
+	struct i2c_client *client = state->client;
+
+	if (addr)
+		io_write(state, io_reg, addr << 1);
+
+	return i2c_new_dummy(client->adapter, io_read(state, io_reg) >> 1);
+}
+
+static void adv748x_unregister_clients(struct adv748x_state *state)
+{
+	unsigned int i;
+
+	for (i = 1; i < ARRAY_SIZE(state->i2c_clients); ++i) {
+		if (state->i2c_clients[i])
+			i2c_unregister_device(state->i2c_clients[i]);
+	}
+}
+
+static int adv748x_initialise_clients(struct adv748x_state *state)
+{
+	int i;
+	int ret;
+
+	for (i = ADV748X_PAGE_DPLL; i < ADV748X_PAGE_MAX; ++i) {
+		state->i2c_clients[i] =
+			adv748x_dummy_client(state, adv748x_i2c_addresses[i],
+					     ADV748X_IO_SLAVE_ADDR_BASE + i);
+		if (state->i2c_clients[i] == NULL) {
+			adv_err(state, "failed to create i2c client %u\n", i);
+			return -ENOMEM;
+		}
+
+		ret = adv748x_configure_regmap(state, i);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+/**
+ * struct adv748x_reg_value - Register write instruction
+ * @page:		Regmap page identifier
+ * @reg:		I2C register
+ * @value:		value to write to @page at @reg
+ */
+struct adv748x_reg_value {
+	u8 page;
+	u8 reg;
+	u8 value;
+};
+
+static int adv748x_write_regs(struct adv748x_state *state,
+			      const struct adv748x_reg_value *regs)
+{
+	int ret;
+
+	while (regs->page != ADV748X_PAGE_EOR) {
+		if (regs->page == ADV748X_PAGE_WAIT) {
+			msleep(regs->value);
+		} else {
+			ret = adv748x_write(state, regs->page, regs->reg,
+				      regs->value);
+			if (ret < 0) {
+				adv_err(state,
+					"Error regs page: 0x%02x reg: 0x%02x\n",
+					regs->page, regs->reg);
+				return ret;
+			}
+		}
+		regs++;
+	}
+
+	return 0;
+}
+
+/* -----------------------------------------------------------------------------
+ * TXA and TXB
+ */
+
+static const struct adv748x_reg_value adv748x_power_up_txa_4lane[] = {
+
+	{ADV748X_PAGE_TXA, 0x00, 0x84},	/* Enable 4-lane MIPI */
+	{ADV748X_PAGE_TXA, 0x00, 0xa4},	/* Set Auto DPHY Timing */
+
+	{ADV748X_PAGE_TXA, 0x31, 0x82},	/* ADI Required Write */
+	{ADV748X_PAGE_TXA, 0x1e, 0x40},	/* ADI Required Write */
+	{ADV748X_PAGE_TXA, 0xda, 0x01},	/* i2c_mipi_pll_en - 1'b1 */
+	{ADV748X_PAGE_WAIT, 0x00, 0x02},/* delay 2 */
+	{ADV748X_PAGE_TXA, 0x00, 0x24 },/* Power-up CSI-TX */
+	{ADV748X_PAGE_WAIT, 0x00, 0x01},/* delay 1 */
+	{ADV748X_PAGE_TXA, 0xc1, 0x2b},	/* ADI Required Write */
+	{ADV748X_PAGE_WAIT, 0x00, 0x01},/* delay 1 */
+	{ADV748X_PAGE_TXA, 0x31, 0x80},	/* ADI Required Write */
+
+	{ADV748X_PAGE_EOR, 0xff, 0xff}	/* End of register table */
+};
+
+static const struct adv748x_reg_value adv748x_power_down_txa_4lane[] = {
+
+	{ADV748X_PAGE_TXA, 0x31, 0x82},	/* ADI Required Write */
+	{ADV748X_PAGE_TXA, 0x1e, 0x00},	/* ADI Required Write */
+	{ADV748X_PAGE_TXA, 0x00, 0x84},	/* Enable 4-lane MIPI */
+	{ADV748X_PAGE_TXA, 0xda, 0x01},	/* i2c_mipi_pll_en - 1'b1 */
+	{ADV748X_PAGE_TXA, 0xc1, 0x3b},	/* ADI Required Write */
+
+	{ADV748X_PAGE_EOR, 0xff, 0xff}	/* End of register table */
+};
+
+static const struct adv748x_reg_value adv748x_power_up_txb_1lane[] = {
+
+	{ADV748X_PAGE_TXB, 0x00, 0x81},	/* Enable 1-lane MIPI */
+	{ADV748X_PAGE_TXB, 0x00, 0xa1},	/* Set Auto DPHY Timing */
+
+	{ADV748X_PAGE_TXB, 0x31, 0x82},	/* ADI Required Write */
+	{ADV748X_PAGE_TXB, 0x1e, 0x40},	/* ADI Required Write */
+	{ADV748X_PAGE_TXB, 0xda, 0x01},	/* i2c_mipi_pll_en - 1'b1 */
+	{ADV748X_PAGE_WAIT, 0x00, 0x02},/* delay 2 */
+	{ADV748X_PAGE_TXB, 0x00, 0x21 },/* Power-up CSI-TX */
+	{ADV748X_PAGE_WAIT, 0x00, 0x01},/* delay 1 */
+	{ADV748X_PAGE_TXB, 0xc1, 0x2b},	/* ADI Required Write */
+	{ADV748X_PAGE_WAIT, 0x00, 0x01},/* delay 1 */
+	{ADV748X_PAGE_TXB, 0x31, 0x80},	/* ADI Required Write */
+
+	{ADV748X_PAGE_EOR, 0xff, 0xff}	/* End of register table */
+};
+
+static const struct adv748x_reg_value adv748x_power_down_txb_1lane[] = {
+
+	{ADV748X_PAGE_TXB, 0x31, 0x82},	/* ADI Required Write */
+	{ADV748X_PAGE_TXB, 0x1e, 0x00},	/* ADI Required Write */
+	{ADV748X_PAGE_TXB, 0x00, 0x81},	/* Enable 4-lane MIPI */
+	{ADV748X_PAGE_TXB, 0xda, 0x01},	/* i2c_mipi_pll_en - 1'b1 */
+	{ADV748X_PAGE_TXB, 0xc1, 0x3b},	/* ADI Required Write */
+
+	{ADV748X_PAGE_EOR, 0xff, 0xff}	/* End of register table */
+};
+
+int adv748x_txa_power(struct adv748x_state *state, bool on)
+{
+	int val;
+
+	val = txa_read(state, ADV748X_CSI_FS_AS_LS);
+	if (val < 0)
+		return val;
+
+	/*
+	 * This test against BIT(6) is not documented by the datasheet, but was
+	 * specified in the downstream driver.
+	 * Track with a WARN_ONCE to determine if it is ever set by HW.
+	 */
+	WARN_ONCE((on && val & ADV748X_CSI_FS_AS_LS_UNKNOWN),
+			"Enabling with unknown bit set");
+
+	if (on)
+		return adv748x_write_regs(state, adv748x_power_up_txa_4lane);
+
+	return adv748x_write_regs(state, adv748x_power_down_txa_4lane);
+}
+
+int adv748x_txb_power(struct adv748x_state *state, bool on)
+{
+	int val;
+
+	val = txb_read(state, ADV748X_CSI_FS_AS_LS);
+	if (val < 0)
+		return val;
+
+	/*
+	 * This test against BIT(6) is not documented by the datasheet, but was
+	 * specified in the downstream driver.
+	 * Track with a WARN_ONCE to determine if it is ever set by HW.
+	 */
+	WARN_ONCE((on && val & ADV748X_CSI_FS_AS_LS_UNKNOWN),
+			"Enabling with unknown bit set");
+
+	if (on)
+		return adv748x_write_regs(state, adv748x_power_up_txb_1lane);
+
+	return adv748x_write_regs(state, adv748x_power_down_txb_1lane);
+}
+
+/* -----------------------------------------------------------------------------
+ * Media Operations
+ */
+
+static const struct media_entity_operations adv748x_media_ops = {
+	.link_validate = v4l2_subdev_link_validate,
+};
+
+/* -----------------------------------------------------------------------------
+ * HW setup
+ */
+
+static const struct adv748x_reg_value adv748x_sw_reset[] = {
+
+	{ADV748X_PAGE_IO, 0xff, 0xff},	/* SW reset */
+	{ADV748X_PAGE_WAIT, 0x00, 0x05},/* delay 5 */
+	{ADV748X_PAGE_IO, 0x01, 0x76},	/* ADI Required Write */
+	{ADV748X_PAGE_IO, 0xf2, 0x01},	/* Enable I2C Read Auto-Increment */
+	{ADV748X_PAGE_EOR, 0xff, 0xff}	/* End of register table */
+};
+
+static const struct adv748x_reg_value adv748x_set_slave_address[] = {
+	{ADV748X_PAGE_IO, 0xf3, ADV748X_I2C_DPLL << 1},
+	{ADV748X_PAGE_IO, 0xf4, ADV748X_I2C_CP << 1},
+	{ADV748X_PAGE_IO, 0xf5, ADV748X_I2C_HDMI << 1},
+	{ADV748X_PAGE_IO, 0xf6, ADV748X_I2C_EDID << 1},
+	{ADV748X_PAGE_IO, 0xf7, ADV748X_I2C_REPEATER << 1},
+	{ADV748X_PAGE_IO, 0xf8, ADV748X_I2C_INFOFRAME << 1},
+	{ADV748X_PAGE_IO, 0xfa, ADV748X_I2C_CEC << 1},
+	{ADV748X_PAGE_IO, 0xfb, ADV748X_I2C_SDP << 1},
+	{ADV748X_PAGE_IO, 0xfc, ADV748X_I2C_TXB << 1},
+	{ADV748X_PAGE_IO, 0xfd, ADV748X_I2C_TXA << 1},
+	{ADV748X_PAGE_EOR, 0xff, 0xff}	/* End of register table */
+};
+
+/* Supported Formats For Script Below */
+/* - 01-29 HDMI to MIPI TxA CSI 4-Lane - RGB888: */
+static const struct adv748x_reg_value adv748x_init_txa_4lane[] = {
+	/* Disable chip powerdown & Enable HDMI Rx block */
+	{ADV748X_PAGE_IO, 0x00, 0x40},
+
+	{ADV748X_PAGE_REPEATER, 0x40, 0x83}, /* Enable HDCP 1.1 */
+
+	{ADV748X_PAGE_HDMI, 0x00, 0x08},/* Foreground Channel = A */
+	{ADV748X_PAGE_HDMI, 0x98, 0xff},/* ADI Required Write */
+	{ADV748X_PAGE_HDMI, 0x99, 0xa3},/* ADI Required Write */
+	{ADV748X_PAGE_HDMI, 0x9a, 0x00},/* ADI Required Write */
+	{ADV748X_PAGE_HDMI, 0x9b, 0x0a},/* ADI Required Write */
+	{ADV748X_PAGE_HDMI, 0x9d, 0x40},/* ADI Required Write */
+	{ADV748X_PAGE_HDMI, 0xcb, 0x09},/* ADI Required Write */
+	{ADV748X_PAGE_HDMI, 0x3d, 0x10},/* ADI Required Write */
+	{ADV748X_PAGE_HDMI, 0x3e, 0x7b},/* ADI Required Write */
+	{ADV748X_PAGE_HDMI, 0x3f, 0x5e},/* ADI Required Write */
+	{ADV748X_PAGE_HDMI, 0x4e, 0xfe},/* ADI Required Write */
+	{ADV748X_PAGE_HDMI, 0x4f, 0x18},/* ADI Required Write */
+	{ADV748X_PAGE_HDMI, 0x57, 0xa3},/* ADI Required Write */
+	{ADV748X_PAGE_HDMI, 0x58, 0x04},/* ADI Required Write */
+	{ADV748X_PAGE_HDMI, 0x85, 0x10},/* ADI Required Write */
+
+	{ADV748X_PAGE_HDMI, 0x83, 0x00},/* Enable All Terminations */
+	{ADV748X_PAGE_HDMI, 0xa3, 0x01},/* ADI Required Write */
+	{ADV748X_PAGE_HDMI, 0xbe, 0x00},/* ADI Required Write */
+
+	{ADV748X_PAGE_HDMI, 0x6c, 0x01},/* HPA Manual Enable */
+	{ADV748X_PAGE_HDMI, 0xf8, 0x01},/* HPA Asserted */
+	{ADV748X_PAGE_HDMI, 0x0f, 0x00},/* Audio Mute Speed Set to Fastest */
+	/* (Smallest Step Size) */
+
+	{ADV748X_PAGE_IO, 0x04, 0x02},	/* RGB Out of CP */
+	{ADV748X_PAGE_IO, 0x12, 0xf0},	/* CSC Depends on ip Packets, SDR 444 */
+	{ADV748X_PAGE_IO, 0x17, 0x80},	/* Luma & Chroma can reach 254d */
+	{ADV748X_PAGE_IO, 0x03, 0x86},	/* CP-Insert_AV_Code */
+
+	{ADV748X_PAGE_CP, 0x7c, 0x00},	/* ADI Required Write */
+
+	{ADV748X_PAGE_IO, 0x0c, 0xe0},	/* Enable LLC_DLL & Double LLC Timing */
+	{ADV748X_PAGE_IO, 0x0e, 0xdd},	/* LLC/PIX/SPI PINS TRISTATED AUD */
+	/* Outputs Enabled */
+	{ADV748X_PAGE_IO, 0x10, 0xa0},	/* Enable 4-lane CSI Tx & Pixel Port */
+
+	{ADV748X_PAGE_TXA, 0x00, 0x84},	/* Enable 4-lane MIPI */
+	{ADV748X_PAGE_TXA, 0x00, 0xa4},	/* Set Auto DPHY Timing */
+	{ADV748X_PAGE_TXA, 0xdb, 0x10},	/* ADI Required Write */
+	{ADV748X_PAGE_TXA, 0xd6, 0x07},	/* ADI Required Write */
+	{ADV748X_PAGE_TXA, 0xc4, 0x0a},	/* ADI Required Write */
+	{ADV748X_PAGE_TXA, 0x71, 0x33},	/* ADI Required Write */
+	{ADV748X_PAGE_TXA, 0x72, 0x11},	/* ADI Required Write */
+	{ADV748X_PAGE_TXA, 0xf0, 0x00},	/* i2c_dphy_pwdn - 1'b0 */
+
+	{ADV748X_PAGE_TXA, 0x31, 0x82},	/* ADI Required Write */
+	{ADV748X_PAGE_TXA, 0x1e, 0x40},	/* ADI Required Write */
+	{ADV748X_PAGE_TXA, 0xda, 0x01},	/* i2c_mipi_pll_en - 1'b1 */
+	{ADV748X_PAGE_WAIT, 0x00, 0x02},/* delay 2 */
+	{ADV748X_PAGE_TXA, 0x00, 0x24 },/* Power-up CSI-TX */
+	{ADV748X_PAGE_WAIT, 0x00, 0x01},/* delay 1 */
+	{ADV748X_PAGE_TXA, 0xc1, 0x2b},	/* ADI Required Write */
+	{ADV748X_PAGE_WAIT, 0x00, 0x01},/* delay 1 */
+	{ADV748X_PAGE_TXA, 0x31, 0x80},	/* ADI Required Write */
+
+	{ADV748X_PAGE_EOR, 0xff, 0xff}	/* End of register table */
+};
+
+/* 02-01 Analog CVBS to MIPI TX-B CSI 1-Lane - */
+/* Autodetect CVBS Single Ended In Ain 1 - MIPI Out */
+static const struct adv748x_reg_value adv748x_init_txb_1lane[] = {
+
+	{ADV748X_PAGE_IO, 0x00, 0x30},	/* Disable chip powerdown Rx */
+	{ADV748X_PAGE_IO, 0xf2, 0x01},	/* Enable I2C Read Auto-Increment */
+
+	{ADV748X_PAGE_IO, 0x0e, 0xff},	/* LLC/PIX/AUD/SPI PINS TRISTATED */
+
+	{ADV748X_PAGE_SDP, 0x0f, 0x00},	/* Exit Power Down Mode */
+	{ADV748X_PAGE_SDP, 0x52, 0xcd},	/* ADI Required Write */
+
+	{ADV748X_PAGE_SDP, 0x0e, 0x80},	/* ADI Required Write */
+	{ADV748X_PAGE_SDP, 0x9c, 0x00},	/* ADI Required Write */
+	{ADV748X_PAGE_SDP, 0x9c, 0xff},	/* ADI Required Write */
+	{ADV748X_PAGE_SDP, 0x0e, 0x00},	/* ADI Required Write */
+
+	/* ADI recommended writes for improved video quality */
+	{ADV748X_PAGE_SDP, 0x80, 0x51},	/* ADI Required Write */
+	{ADV748X_PAGE_SDP, 0x81, 0x51},	/* ADI Required Write */
+	{ADV748X_PAGE_SDP, 0x82, 0x68},	/* ADI Required Write */
+
+	{ADV748X_PAGE_SDP, 0x03, 0x42},	/* Tri-S Output , PwrDwn 656 pads */
+	{ADV748X_PAGE_SDP, 0x04, 0xb5},	/* ITU-R BT.656-4 compatible */
+	{ADV748X_PAGE_SDP, 0x13, 0x00},	/* ADI Required Write */
+
+	{ADV748X_PAGE_SDP, 0x17, 0x41},	/* Select SH1 */
+	{ADV748X_PAGE_SDP, 0x31, 0x12},	/* ADI Required Write */
+	{ADV748X_PAGE_SDP, 0xe6, 0x4f},  /* V bit end pos manually in NTSC */
+
+	/* Enable 1-Lane MIPI Tx, */
+	/* enable pixel output and route SD through Pixel port */
+	{ADV748X_PAGE_IO, 0x10, 0x70},
+
+	{ADV748X_PAGE_TXB, 0x00, 0x81},	/* Enable 1-lane MIPI */
+	{ADV748X_PAGE_TXB, 0x00, 0xa1},	/* Set Auto DPHY Timing */
+	{ADV748X_PAGE_TXB, 0xd2, 0x40},	/* ADI Required Write */
+	{ADV748X_PAGE_TXB, 0xc4, 0x0a},	/* ADI Required Write */
+	{ADV748X_PAGE_TXB, 0x71, 0x33},	/* ADI Required Write */
+	{ADV748X_PAGE_TXB, 0x72, 0x11},	/* ADI Required Write */
+	{ADV748X_PAGE_TXB, 0xf0, 0x00},	/* i2c_dphy_pwdn - 1'b0 */
+	{ADV748X_PAGE_TXB, 0x31, 0x82},	/* ADI Required Write */
+	{ADV748X_PAGE_TXB, 0x1e, 0x40},	/* ADI Required Write */
+	{ADV748X_PAGE_TXB, 0xda, 0x01},	/* i2c_mipi_pll_en - 1'b1 */
+
+	{ADV748X_PAGE_WAIT, 0x00, 0x02},/* delay 2 */
+	{ADV748X_PAGE_TXB, 0x00, 0x21 },/* Power-up CSI-TX */
+	{ADV748X_PAGE_WAIT, 0x00, 0x01},/* delay 1 */
+	{ADV748X_PAGE_TXB, 0xc1, 0x2b},	/* ADI Required Write */
+	{ADV748X_PAGE_WAIT, 0x00, 0x01},/* delay 1 */
+	{ADV748X_PAGE_TXB, 0x31, 0x80},	/* ADI Required Write */
+
+	{ADV748X_PAGE_EOR, 0xff, 0xff}	/* End of register table */
+};
+
+static int adv748x_reset(struct adv748x_state *state)
+{
+	int ret;
+
+	ret = adv748x_write_regs(state, adv748x_sw_reset);
+	if (ret < 0)
+		return ret;
+
+	ret = adv748x_write_regs(state, adv748x_set_slave_address);
+	if (ret < 0)
+		return ret;
+
+	/* Init and power down TXA */
+	ret = adv748x_write_regs(state, adv748x_init_txa_4lane);
+	if (ret)
+		return ret;
+
+	adv748x_txa_power(state, 0);
+
+	/* Init and power down TXB */
+	ret = adv748x_write_regs(state, adv748x_init_txb_1lane);
+	if (ret)
+		return ret;
+
+	adv748x_txb_power(state, 0);
+
+	/* Disable chip powerdown & Enable HDMI Rx block */
+	io_write(state, ADV748X_IO_PD, ADV748X_IO_PD_RX_EN);
+
+	/* Enable 4-lane CSI Tx & Pixel Port */
+	io_write(state, ADV748X_IO_10, ADV748X_IO_10_CSI4_EN |
+				       ADV748X_IO_10_CSI1_EN |
+				       ADV748X_IO_10_PIX_OUT_EN);
+
+	/* Use vid_std and v_freq as freerun resolution for CP */
+	cp_clrset(state, ADV748X_CP_CLMP_POS, ADV748X_CP_CLMP_POS_DIS_AUTO,
+					      ADV748X_CP_CLMP_POS_DIS_AUTO);
+
+	return 0;
+}
+
+static int adv748x_identify_chip(struct adv748x_state *state)
+{
+	int msb, lsb;
+
+	lsb = io_read(state, ADV748X_IO_CHIP_REV_ID_1);
+	msb = io_read(state, ADV748X_IO_CHIP_REV_ID_2);
+
+	if (lsb < 0 || msb < 0) {
+		adv_err(state, "Failed to read chip revision\n");
+		return -EIO;
+	}
+
+	adv_info(state, "chip found @ 0x%02x revision %02x%02x\n",
+		 state->client->addr << 1, lsb, msb);
+
+	return 0;
+}
+
+/* -----------------------------------------------------------------------------
+ * i2c driver
+ */
+
+void adv748x_subdev_init(struct v4l2_subdev *sd, struct adv748x_state *state,
+			 const struct v4l2_subdev_ops *ops, u32 function,
+			 const char *ident)
+{
+	v4l2_subdev_init(sd, ops);
+	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+
+	/* the owner is the same as the i2c_client's driver owner */
+	sd->owner = state->dev->driver->owner;
+	sd->dev = state->dev;
+
+	v4l2_set_subdevdata(sd, state);
+
+	/* initialize name */
+	snprintf(sd->name, sizeof(sd->name), "%s %d-%04x %s",
+		state->dev->driver->name,
+		i2c_adapter_id(state->client->adapter),
+		state->client->addr, ident);
+
+	sd->entity.function = function;
+	sd->entity.ops = &adv748x_media_ops;
+}
+
+static int adv748x_parse_dt(struct adv748x_state *state)
+{
+	struct device_node *ep_np = NULL;
+	struct of_endpoint ep;
+	bool found = false;
+
+	for_each_endpoint_of_node(state->dev->of_node, ep_np) {
+		of_graph_parse_endpoint(ep_np, &ep);
+		adv_info(state, "Endpoint %s on port %d",
+				of_node_full_name(ep.local_node),
+				ep.port);
+
+		if (ep.port >= ADV748X_PORT_MAX) {
+			adv_err(state, "Invalid endpoint %s on port %d",
+				of_node_full_name(ep.local_node),
+				ep.port);
+
+			continue;
+		}
+
+		if (state->endpoints[ep.port]) {
+			adv_err(state,
+				"Multiple port endpoints are not supported");
+			continue;
+		}
+
+		of_node_get(ep_np);
+		state->endpoints[ep.port] = ep_np;
+
+		found = true;
+	}
+
+	return found ? 0 : -ENODEV;
+}
+
+static void adv748x_dt_cleanup(struct adv748x_state *state)
+{
+	unsigned int i;
+
+	for (i = 0; i < ADV748X_PORT_MAX; i++)
+		of_node_put(state->endpoints[i]);
+}
+
+static int adv748x_probe(struct i2c_client *client,
+			 const struct i2c_device_id *id)
+{
+	struct adv748x_state *state;
+	int ret;
+
+	/* Check if the adapter supports the needed features */
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_BYTE_DATA))
+		return -EIO;
+
+	state = kzalloc(sizeof(struct adv748x_state), GFP_KERNEL);
+	if (!state)
+		return -ENOMEM;
+
+	mutex_init(&state->mutex);
+
+	state->dev = &client->dev;
+	state->client = client;
+	state->i2c_clients[ADV748X_PAGE_IO] = client;
+	i2c_set_clientdata(client, state);
+
+	/* Discover and process ports declared by the Device tree endpoints */
+	ret = adv748x_parse_dt(state);
+	if (ret) {
+		adv_err(state, "Failed to parse device tree");
+		goto err_free_mutex;
+	}
+
+	/* Configure IO Regmap region */
+	ret = adv748x_configure_regmap(state, ADV748X_PAGE_IO);
+	if (ret) {
+		adv_err(state, "Error configuring IO regmap region");
+		goto err_cleanup_dt;
+	}
+
+	ret = adv748x_identify_chip(state);
+	if (ret) {
+		adv_err(state, "Failed to identify chip");
+		goto err_cleanup_clients;
+	}
+
+	/* Configure remaining pages as I2C clients with regmap access */
+	ret = adv748x_initialise_clients(state);
+	if (ret) {
+		adv_err(state, "Failed to setup client regmap pages");
+		goto err_cleanup_clients;
+	}
+
+	/* SW reset ADV748X to its default values */
+	ret = adv748x_reset(state);
+	if (ret) {
+		adv_err(state, "Failed to reset hardware");
+		goto err_cleanup_clients;
+	}
+
+	/* Initialise HDMI */
+	ret = adv748x_hdmi_init(&state->hdmi);
+	if (ret) {
+		adv_err(state, "Failed to probe HDMI");
+		goto err_cleanup_clients;
+	}
+
+	/* Initialise AFE */
+	ret = adv748x_afe_init(&state->afe);
+	if (ret) {
+		adv_err(state, "Failed to probe AFE");
+		goto err_cleanup_hdmi;
+	}
+
+	/* Initialise TXA */
+	ret = adv748x_csi2_init(state, &state->txa);
+	if (ret) {
+		adv_err(state, "Failed to probe TXA");
+		goto err_cleanup_afe;
+	}
+
+	/* Initialise TXB */
+	ret = adv748x_csi2_init(state, &state->txb);
+	if (ret) {
+		adv_err(state, "Failed to probe TXB");
+		goto err_cleanup_txa;
+	}
+
+	return 0;
+
+err_cleanup_txa:
+	adv748x_csi2_cleanup(&state->txa);
+err_cleanup_afe:
+	adv748x_afe_cleanup(&state->afe);
+err_cleanup_hdmi:
+	adv748x_hdmi_cleanup(&state->hdmi);
+err_cleanup_clients:
+	adv748x_unregister_clients(state);
+err_cleanup_dt:
+	adv748x_dt_cleanup(state);
+err_free_mutex:
+	mutex_destroy(&state->mutex);
+	kfree(state);
+
+	return ret;
+}
+
+static int adv748x_remove(struct i2c_client *client)
+{
+	struct adv748x_state *state = i2c_get_clientdata(client);
+
+	adv748x_afe_cleanup(&state->afe);
+	adv748x_hdmi_cleanup(&state->hdmi);
+
+	adv748x_csi2_cleanup(&state->txa);
+	adv748x_csi2_cleanup(&state->txb);
+
+	adv748x_unregister_clients(state);
+	adv748x_dt_cleanup(state);
+	mutex_destroy(&state->mutex);
+
+	kfree(state);
+
+	return 0;
+}
+
+static const struct i2c_device_id adv748x_id[] = {
+	{ "adv7481", 0 },
+	{ "adv7482", 0 },
+	{ },
+};
+MODULE_DEVICE_TABLE(i2c, adv748x_id);
+
+static const struct of_device_id adv748x_of_table[] = {
+	{ .compatible = "adi,adv7481", },
+	{ .compatible = "adi,adv7482", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, adv748x_of_table);
+
+static struct i2c_driver adv748x_driver = {
+	.driver = {
+		.name = "adv748x",
+		.of_match_table = adv748x_of_table,
+	},
+	.probe = adv748x_probe,
+	.remove = adv748x_remove,
+	.id_table = adv748x_id,
+};
+
+module_i2c_driver(adv748x_driver);
+
+MODULE_AUTHOR("Kieran Bingham <kieran.bingham@ideasonboard.com>");
+MODULE_DESCRIPTION("ADV748X video decoder");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/i2c/adv748x/adv748x-csi2.c b/drivers/media/i2c/adv748x/adv748x-csi2.c
new file mode 100644
index 000000000000..979825d4a419
--- /dev/null
+++ b/drivers/media/i2c/adv748x/adv748x-csi2.c
@@ -0,0 +1,326 @@
+/*
+ * Driver for Analog Devices ADV748X CSI-2 Transmitter
+ *
+ * Copyright (C) 2017 Renesas Electronics Corp.
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+
+#include <linux/module.h>
+#include <linux/mutex.h>
+
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-ioctl.h>
+
+#include "adv748x.h"
+
+static bool is_txa(struct adv748x_csi2 *tx)
+{
+	return tx == &tx->state->txa;
+}
+
+static int adv748x_csi2_set_virtual_channel(struct adv748x_csi2 *tx,
+					    unsigned int vc)
+{
+	return tx_write(tx, ADV748X_CSI_VC_REF, vc << ADV748X_CSI_VC_REF_SHIFT);
+}
+
+/**
+ * adv748x_csi2_register_link : Register and link internal entities
+ *
+ * @tx: CSI2 private entity
+ * @v4l2_dev: Video registration device
+ * @src: Source subdevice to establish link
+ * @src_pad: Pad number of source to link to this @tx
+ *
+ * Ensure that the subdevice is registered against the v4l2_device, and link the
+ * source pad to the sink pad of the CSI2 bus entity.
+ */
+static int adv748x_csi2_register_link(struct adv748x_csi2 *tx,
+				      struct v4l2_device *v4l2_dev,
+				      struct v4l2_subdev *src,
+				      unsigned int src_pad)
+{
+	int enabled = MEDIA_LNK_FL_ENABLED;
+	int ret;
+
+	/*
+	 * Dynamic linking of the AFE is not supported.
+	 * Register the links as immutable.
+	 */
+	enabled |= MEDIA_LNK_FL_IMMUTABLE;
+
+	if (!src->v4l2_dev) {
+		ret = v4l2_device_register_subdev(v4l2_dev, src);
+		if (ret)
+			return ret;
+	}
+
+	return media_create_pad_link(&src->entity, src_pad,
+				     &tx->sd.entity, ADV748X_CSI2_SINK,
+				     enabled);
+}
+
+/* -----------------------------------------------------------------------------
+ * v4l2_subdev_internal_ops
+ *
+ * We use the internal registered operation to be able to ensure that our
+ * incremental subdevices (not connected in the forward path) can be registered
+ * against the resulting video path and media device.
+ */
+
+static int adv748x_csi2_registered(struct v4l2_subdev *sd)
+{
+	struct adv748x_csi2 *tx = adv748x_sd_to_csi2(sd);
+	struct adv748x_state *state = tx->state;
+
+	adv_dbg(state, "Registered %s (%s)", is_txa(tx) ? "TXA":"TXB",
+			sd->name);
+
+	/*
+	 * The adv748x hardware allows the AFE to route through the TXA, however
+	 * this is not currently supported in this driver.
+	 *
+	 * Link HDMI->TXA, and AFE->TXB directly.
+	 */
+	if (is_txa(tx)) {
+		return adv748x_csi2_register_link(tx, sd->v4l2_dev,
+						  &state->hdmi.sd,
+						  ADV748X_HDMI_SOURCE);
+	} else {
+		return adv748x_csi2_register_link(tx, sd->v4l2_dev,
+						  &state->afe.sd,
+						  ADV748X_AFE_SOURCE);
+	}
+}
+
+static const struct v4l2_subdev_internal_ops adv748x_csi2_internal_ops = {
+	.registered = adv748x_csi2_registered,
+};
+
+/* -----------------------------------------------------------------------------
+ * v4l2_subdev_video_ops
+ */
+
+static int adv748x_csi2_s_stream(struct v4l2_subdev *sd, int enable)
+{
+	struct adv748x_csi2 *tx = adv748x_sd_to_csi2(sd);
+	struct v4l2_subdev *src;
+
+	src = adv748x_get_remote_sd(&tx->pads[ADV748X_CSI2_SINK]);
+	if (!src)
+		return -EPIPE;
+
+	return v4l2_subdev_call(src, video, s_stream, enable);
+}
+
+static const struct v4l2_subdev_video_ops adv748x_csi2_video_ops = {
+	.s_stream = adv748x_csi2_s_stream,
+};
+
+/* -----------------------------------------------------------------------------
+ * v4l2_subdev_pad_ops
+ *
+ * The CSI2 bus pads are ignorant to the data sizes or formats.
+ * But we must support setting the pad formats for format propagation.
+ */
+
+static struct v4l2_mbus_framefmt *
+adv748x_csi2_get_pad_format(struct v4l2_subdev *sd,
+			    struct v4l2_subdev_pad_config *cfg,
+			    unsigned int pad, u32 which)
+{
+	struct adv748x_csi2 *tx = adv748x_sd_to_csi2(sd);
+
+	if (which == V4L2_SUBDEV_FORMAT_TRY)
+		return v4l2_subdev_get_try_format(sd, cfg, pad);
+
+	return &tx->format;
+}
+
+static int adv748x_csi2_get_format(struct v4l2_subdev *sd,
+				   struct v4l2_subdev_pad_config *cfg,
+				   struct v4l2_subdev_format *sdformat)
+{
+	struct adv748x_csi2 *tx = adv748x_sd_to_csi2(sd);
+	struct adv748x_state *state = tx->state;
+	struct v4l2_mbus_framefmt *mbusformat;
+
+	mbusformat = adv748x_csi2_get_pad_format(sd, cfg, sdformat->pad,
+						 sdformat->which);
+	if (!mbusformat)
+		return -EINVAL;
+
+	mutex_lock(&state->mutex);
+
+	sdformat->format = *mbusformat;
+
+	mutex_unlock(&state->mutex);
+
+	return 0;
+}
+
+static int adv748x_csi2_set_format(struct v4l2_subdev *sd,
+				   struct v4l2_subdev_pad_config *cfg,
+				   struct v4l2_subdev_format *sdformat)
+{
+	struct adv748x_csi2 *tx = adv748x_sd_to_csi2(sd);
+	struct adv748x_state *state = tx->state;
+	struct v4l2_mbus_framefmt *mbusformat;
+	int ret = 0;
+
+	mbusformat = adv748x_csi2_get_pad_format(sd, cfg, sdformat->pad,
+						 sdformat->which);
+	if (!mbusformat)
+		return -EINVAL;
+
+	mutex_lock(&state->mutex);
+
+	if (sdformat->pad == ADV748X_CSI2_SOURCE) {
+		const struct v4l2_mbus_framefmt *sink_fmt;
+
+		sink_fmt = adv748x_csi2_get_pad_format(sd, cfg,
+						       ADV748X_CSI2_SINK,
+						       sdformat->which);
+
+		if (!sink_fmt) {
+			ret = -EINVAL;
+			goto unlock;
+		}
+
+		sdformat->format = *sink_fmt;
+	}
+
+	*mbusformat = sdformat->format;
+
+unlock:
+	mutex_unlock(&state->mutex);
+
+	return ret;
+}
+
+static const struct v4l2_subdev_pad_ops adv748x_csi2_pad_ops = {
+	.get_fmt = adv748x_csi2_get_format,
+	.set_fmt = adv748x_csi2_set_format,
+};
+
+/* -----------------------------------------------------------------------------
+ * v4l2_subdev_ops
+ */
+
+static const struct v4l2_subdev_ops adv748x_csi2_ops = {
+	.video = &adv748x_csi2_video_ops,
+	.pad = &adv748x_csi2_pad_ops,
+};
+
+/* -----------------------------------------------------------------------------
+ * Subdev module and controls
+ */
+
+int adv748x_csi2_set_pixelrate(struct v4l2_subdev *sd, s64 rate)
+{
+	struct v4l2_ctrl *ctrl;
+
+	ctrl = v4l2_ctrl_find(sd->ctrl_handler, V4L2_CID_PIXEL_RATE);
+	if (!ctrl)
+		return -EINVAL;
+
+	return v4l2_ctrl_s_ctrl_int64(ctrl, rate);
+}
+
+static int adv748x_csi2_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+	switch (ctrl->id) {
+	case V4L2_CID_PIXEL_RATE:
+		return 0;
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct v4l2_ctrl_ops adv748x_csi2_ctrl_ops = {
+	.s_ctrl = adv748x_csi2_s_ctrl,
+};
+
+static int adv748x_csi2_init_controls(struct adv748x_csi2 *tx)
+{
+
+	v4l2_ctrl_handler_init(&tx->ctrl_hdl, 1);
+
+	v4l2_ctrl_new_std(&tx->ctrl_hdl, &adv748x_csi2_ctrl_ops,
+			  V4L2_CID_PIXEL_RATE, 1, INT_MAX, 1, 1);
+
+	tx->sd.ctrl_handler = &tx->ctrl_hdl;
+	if (tx->ctrl_hdl.error) {
+		v4l2_ctrl_handler_free(&tx->ctrl_hdl);
+		return tx->ctrl_hdl.error;
+	}
+
+	return v4l2_ctrl_handler_setup(&tx->ctrl_hdl);
+}
+
+int adv748x_csi2_init(struct adv748x_state *state, struct adv748x_csi2 *tx)
+{
+	struct device_node *ep;
+	int ret;
+
+	/* We can not use container_of to get back to the state with two TXs */
+	tx->state = state;
+	tx->page = is_txa(tx) ? ADV748X_PAGE_TXA : ADV748X_PAGE_TXB;
+
+	ep = state->endpoints[is_txa(tx) ? ADV748X_PORT_TXA : ADV748X_PORT_TXB];
+	if (!ep) {
+		adv_err(state, "No endpoint found for %s\n",
+				is_txa(tx) ? "txa" : "txb");
+		return -ENODEV;
+	}
+
+	/* Initialise the virtual channel */
+	adv748x_csi2_set_virtual_channel(tx, 0);
+
+	adv748x_subdev_init(&tx->sd, state, &adv748x_csi2_ops,
+			    MEDIA_ENT_F_UNKNOWN,
+			    is_txa(tx) ? "txa" : "txb");
+
+	/* Ensure that matching is based upon the endpoint fwnodes */
+	tx->sd.fwnode = of_fwnode_handle(ep);
+
+	/* Register internal ops for incremental subdev registration */
+	tx->sd.internal_ops = &adv748x_csi2_internal_ops;
+
+	tx->pads[ADV748X_CSI2_SINK].flags = MEDIA_PAD_FL_SINK;
+	tx->pads[ADV748X_CSI2_SOURCE].flags = MEDIA_PAD_FL_SOURCE;
+
+	ret = media_entity_pads_init(&tx->sd.entity, ADV748X_CSI2_NR_PADS,
+				     tx->pads);
+	if (ret)
+		return ret;
+
+	ret = adv748x_csi2_init_controls(tx);
+	if (ret)
+		goto err_free_media;
+
+	ret = v4l2_async_register_subdev(&tx->sd);
+	if (ret)
+		goto err_free_ctrl;
+
+	return 0;
+
+err_free_ctrl:
+	v4l2_ctrl_handler_free(&tx->ctrl_hdl);
+err_free_media:
+	media_entity_cleanup(&tx->sd.entity);
+
+	return ret;
+}
+
+void adv748x_csi2_cleanup(struct adv748x_csi2 *tx)
+{
+	v4l2_async_unregister_subdev(&tx->sd);
+	media_entity_cleanup(&tx->sd.entity);
+	v4l2_ctrl_handler_free(&tx->ctrl_hdl);
+}
diff --git a/drivers/media/i2c/adv748x/adv748x-hdmi.c b/drivers/media/i2c/adv748x/adv748x-hdmi.c
new file mode 100644
index 000000000000..4da4253553fc
--- /dev/null
+++ b/drivers/media/i2c/adv748x/adv748x-hdmi.c
@@ -0,0 +1,768 @@
+/*
+ * Driver for Analog Devices ADV748X HDMI receiver and Component Processor (CP)
+ *
+ * Copyright (C) 2017 Renesas Electronics Corp.
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+
+#include <linux/module.h>
+#include <linux/mutex.h>
+
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-dv-timings.h>
+#include <media/v4l2-ioctl.h>
+
+#include <uapi/linux/v4l2-dv-timings.h>
+
+#include "adv748x.h"
+
+/* -----------------------------------------------------------------------------
+ * HDMI and CP
+ */
+
+#define ADV748X_HDMI_MIN_WIDTH		640
+#define ADV748X_HDMI_MAX_WIDTH		1920
+#define ADV748X_HDMI_MIN_HEIGHT		480
+#define ADV748X_HDMI_MAX_HEIGHT		1200
+
+/* V4L2_DV_BT_CEA_720X480I59_94 - 0.5 MHz */
+#define ADV748X_HDMI_MIN_PIXELCLOCK	13000000
+/* V4L2_DV_BT_DMT_1600X1200P60 */
+#define ADV748X_HDMI_MAX_PIXELCLOCK	162000000
+
+static const struct v4l2_dv_timings_cap adv748x_hdmi_timings_cap = {
+	.type = V4L2_DV_BT_656_1120,
+	/* keep this initialization for compatibility with GCC < 4.4.6 */
+	.reserved = { 0 },
+
+	V4L2_INIT_BT_TIMINGS(ADV748X_HDMI_MIN_WIDTH, ADV748X_HDMI_MAX_WIDTH,
+			     ADV748X_HDMI_MIN_HEIGHT, ADV748X_HDMI_MAX_HEIGHT,
+			     ADV748X_HDMI_MIN_PIXELCLOCK,
+			     ADV748X_HDMI_MAX_PIXELCLOCK,
+			     V4L2_DV_BT_STD_CEA861 | V4L2_DV_BT_STD_DMT,
+			     V4L2_DV_BT_CAP_PROGRESSIVE)
+};
+
+struct adv748x_hdmi_video_standards {
+	struct v4l2_dv_timings timings;
+	u8 vid_std;
+	u8 v_freq;
+};
+
+static const struct adv748x_hdmi_video_standards
+adv748x_hdmi_video_standards[] = {
+	{ V4L2_DV_BT_CEA_720X480P59_94, 0x4a, 0x00 },
+	{ V4L2_DV_BT_CEA_720X576P50, 0x4b, 0x00 },
+	{ V4L2_DV_BT_CEA_1280X720P60, 0x53, 0x00 },
+	{ V4L2_DV_BT_CEA_1280X720P50, 0x53, 0x01 },
+	{ V4L2_DV_BT_CEA_1280X720P30, 0x53, 0x02 },
+	{ V4L2_DV_BT_CEA_1280X720P25, 0x53, 0x03 },
+	{ V4L2_DV_BT_CEA_1280X720P24, 0x53, 0x04 },
+	{ V4L2_DV_BT_CEA_1920X1080P60, 0x5e, 0x00 },
+	{ V4L2_DV_BT_CEA_1920X1080P50, 0x5e, 0x01 },
+	{ V4L2_DV_BT_CEA_1920X1080P30, 0x5e, 0x02 },
+	{ V4L2_DV_BT_CEA_1920X1080P25, 0x5e, 0x03 },
+	{ V4L2_DV_BT_CEA_1920X1080P24, 0x5e, 0x04 },
+	/* SVGA */
+	{ V4L2_DV_BT_DMT_800X600P56, 0x80, 0x00 },
+	{ V4L2_DV_BT_DMT_800X600P60, 0x81, 0x00 },
+	{ V4L2_DV_BT_DMT_800X600P72, 0x82, 0x00 },
+	{ V4L2_DV_BT_DMT_800X600P75, 0x83, 0x00 },
+	{ V4L2_DV_BT_DMT_800X600P85, 0x84, 0x00 },
+	/* SXGA */
+	{ V4L2_DV_BT_DMT_1280X1024P60, 0x85, 0x00 },
+	{ V4L2_DV_BT_DMT_1280X1024P75, 0x86, 0x00 },
+	/* VGA */
+	{ V4L2_DV_BT_DMT_640X480P60, 0x88, 0x00 },
+	{ V4L2_DV_BT_DMT_640X480P72, 0x89, 0x00 },
+	{ V4L2_DV_BT_DMT_640X480P75, 0x8a, 0x00 },
+	{ V4L2_DV_BT_DMT_640X480P85, 0x8b, 0x00 },
+	/* XGA */
+	{ V4L2_DV_BT_DMT_1024X768P60, 0x8c, 0x00 },
+	{ V4L2_DV_BT_DMT_1024X768P70, 0x8d, 0x00 },
+	{ V4L2_DV_BT_DMT_1024X768P75, 0x8e, 0x00 },
+	{ V4L2_DV_BT_DMT_1024X768P85, 0x8f, 0x00 },
+	/* UXGA */
+	{ V4L2_DV_BT_DMT_1600X1200P60, 0x96, 0x00 },
+};
+
+static void adv748x_hdmi_fill_format(struct adv748x_hdmi *hdmi,
+				     struct v4l2_mbus_framefmt *fmt)
+{
+	memset(fmt, 0, sizeof(*fmt));
+
+	fmt->code = MEDIA_BUS_FMT_RGB888_1X24;
+	fmt->field = hdmi->timings.bt.interlaced ?
+			V4L2_FIELD_ALTERNATE : V4L2_FIELD_NONE;
+
+	/* TODO: The colorspace depends on the AVI InfoFrame contents */
+	fmt->colorspace = V4L2_COLORSPACE_SRGB;
+
+	fmt->width = hdmi->timings.bt.width;
+	fmt->height = hdmi->timings.bt.height;
+}
+
+static void adv748x_fill_optional_dv_timings(struct v4l2_dv_timings *timings)
+{
+	v4l2_find_dv_timings_cap(timings, &adv748x_hdmi_timings_cap,
+				 250000, NULL, NULL);
+}
+
+static bool adv748x_hdmi_has_signal(struct adv748x_state *state)
+{
+	int val;
+
+	/* Check that VERT_FILTER and DE_REGEN is locked */
+	val = hdmi_read(state, ADV748X_HDMI_LW1);
+	return (val & ADV748X_HDMI_LW1_VERT_FILTER) &&
+	       (val & ADV748X_HDMI_LW1_DE_REGEN);
+}
+
+static int adv748x_hdmi_read_pixelclock(struct adv748x_state *state)
+{
+	int a, b;
+
+	a = hdmi_read(state, ADV748X_HDMI_TMDS_1);
+	b = hdmi_read(state, ADV748X_HDMI_TMDS_2);
+	if (a < 0 || b < 0)
+		return -ENODATA;
+
+	/*
+	 * The high 9 bits store TMDS frequency measurement in MHz
+	 * The low 7 bits of TMDS_2 store the 7-bit TMDS fractional frequency
+	 * measurement in 1/128 MHz
+	 */
+	return ((a << 1) | (b >> 7)) * 1000000 + (b & 0x7f) * 1000000 / 128;
+}
+
+/*
+ * adv748x_hdmi_set_de_timings: Adjust horizontal picture offset through DE
+ *
+ * HDMI CP uses a Data Enable synchronisation timing reference
+ *
+ * Vary the leading and trailing edge position of the DE signal output by the CP
+ * core. Values are stored as signed-twos-complement in one-pixel-clock units
+ *
+ * The start and end are shifted equally by the 10-bit shift value.
+ */
+static void adv748x_hdmi_set_de_timings(struct adv748x_state *state, int shift)
+{
+	u8 high, low;
+
+	/* POS_HIGH stores bits 8 and 9 of both the start and end */
+	high = ADV748X_CP_DE_POS_HIGH_SET;
+	high |= (shift & 0x300) >> 8;
+	low = shift & 0xff;
+
+	/* The sequence of the writes is important and must be followed */
+	cp_write(state, ADV748X_CP_DE_POS_HIGH, high);
+	cp_write(state, ADV748X_CP_DE_POS_END_LOW, low);
+
+	high |= (shift & 0x300) >> 6;
+
+	cp_write(state, ADV748X_CP_DE_POS_HIGH, high);
+	cp_write(state, ADV748X_CP_DE_POS_START_LOW, low);
+}
+
+static int adv748x_hdmi_set_video_timings(struct adv748x_state *state,
+					  const struct v4l2_dv_timings *timings)
+{
+	const struct adv748x_hdmi_video_standards *stds =
+		adv748x_hdmi_video_standards;
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(adv748x_hdmi_video_standards); i++) {
+		if (!v4l2_match_dv_timings(timings, &stds[i].timings, 250000,
+					   false))
+			continue;
+	}
+
+	if (i >= ARRAY_SIZE(adv748x_hdmi_video_standards))
+		return -EINVAL;
+
+	/*
+	 * When setting cp_vid_std to either 720p, 1080i, or 1080p, the video
+	 * will get shifted horizontally to the left in active video mode.
+	 * The de_h_start and de_h_end controls are used to centre the picture
+	 * correctly
+	 */
+	switch (stds[i].vid_std) {
+	case 0x53: /* 720p */
+		adv748x_hdmi_set_de_timings(state, -40);
+		break;
+	case 0x54: /* 1080i */
+	case 0x5e: /* 1080p */
+		adv748x_hdmi_set_de_timings(state, -44);
+		break;
+	default:
+		adv748x_hdmi_set_de_timings(state, 0);
+		break;
+	}
+
+	io_write(state, ADV748X_IO_VID_STD, stds[i].vid_std);
+	io_clrset(state, ADV748X_IO_DATAPATH, ADV748X_IO_DATAPATH_VFREQ_M,
+		  stds[i].v_freq << ADV748X_IO_DATAPATH_VFREQ_SHIFT);
+
+	return 0;
+}
+
+/* -----------------------------------------------------------------------------
+ * v4l2_subdev_video_ops
+ */
+
+static int adv748x_hdmi_s_dv_timings(struct v4l2_subdev *sd,
+				     struct v4l2_dv_timings *timings)
+{
+	struct adv748x_hdmi *hdmi = adv748x_sd_to_hdmi(sd);
+	struct adv748x_state *state = adv748x_hdmi_to_state(hdmi);
+	int ret;
+
+	if (!timings)
+		return -EINVAL;
+
+	if (v4l2_match_dv_timings(&hdmi->timings, timings, 0, false))
+		return 0;
+
+	if (!v4l2_valid_dv_timings(timings, &adv748x_hdmi_timings_cap,
+				   NULL, NULL))
+		return -ERANGE;
+
+	adv748x_fill_optional_dv_timings(timings);
+
+	mutex_lock(&state->mutex);
+
+	ret = adv748x_hdmi_set_video_timings(state, timings);
+	if (ret)
+		goto error;
+
+	hdmi->timings = *timings;
+
+	cp_clrset(state, ADV748X_CP_VID_ADJ_2, ADV748X_CP_VID_ADJ_2_INTERLACED,
+		  timings->bt.interlaced ?
+				  ADV748X_CP_VID_ADJ_2_INTERLACED : 0);
+
+	mutex_unlock(&state->mutex);
+
+	return 0;
+
+error:
+	mutex_unlock(&state->mutex);
+	return ret;
+}
+
+static int adv748x_hdmi_g_dv_timings(struct v4l2_subdev *sd,
+				     struct v4l2_dv_timings *timings)
+{
+	struct adv748x_hdmi *hdmi = adv748x_sd_to_hdmi(sd);
+	struct adv748x_state *state = adv748x_hdmi_to_state(hdmi);
+
+	mutex_lock(&state->mutex);
+
+	*timings = hdmi->timings;
+
+	mutex_unlock(&state->mutex);
+
+	return 0;
+}
+
+static int adv748x_hdmi_query_dv_timings(struct v4l2_subdev *sd,
+					 struct v4l2_dv_timings *timings)
+{
+	struct adv748x_hdmi *hdmi = adv748x_sd_to_hdmi(sd);
+	struct adv748x_state *state = adv748x_hdmi_to_state(hdmi);
+	struct v4l2_bt_timings *bt = &timings->bt;
+	int pixelclock;
+	int polarity;
+
+	if (!timings)
+		return -EINVAL;
+
+	memset(timings, 0, sizeof(struct v4l2_dv_timings));
+
+	if (!adv748x_hdmi_has_signal(state))
+		return -ENOLINK;
+
+	pixelclock = adv748x_hdmi_read_pixelclock(state);
+	if (pixelclock < 0)
+		return -ENODATA;
+
+	timings->type = V4L2_DV_BT_656_1120;
+
+	bt->pixelclock = pixelclock;
+	bt->interlaced = hdmi_read(state, ADV748X_HDMI_F1H1) &
+				ADV748X_HDMI_F1H1_INTERLACED ?
+				V4L2_DV_INTERLACED : V4L2_DV_PROGRESSIVE;
+	bt->width = hdmi_read16(state, ADV748X_HDMI_LW1,
+				ADV748X_HDMI_LW1_WIDTH_MASK);
+	bt->height = hdmi_read16(state, ADV748X_HDMI_F0H1,
+				 ADV748X_HDMI_F0H1_HEIGHT_MASK);
+	bt->hfrontporch = hdmi_read16(state, ADV748X_HDMI_HFRONT_PORCH,
+				      ADV748X_HDMI_HFRONT_PORCH_MASK);
+	bt->hsync = hdmi_read16(state, ADV748X_HDMI_HSYNC_WIDTH,
+				ADV748X_HDMI_HSYNC_WIDTH_MASK);
+	bt->hbackporch = hdmi_read16(state, ADV748X_HDMI_HBACK_PORCH,
+				     ADV748X_HDMI_HBACK_PORCH_MASK);
+	bt->vfrontporch = hdmi_read16(state, ADV748X_HDMI_VFRONT_PORCH,
+				      ADV748X_HDMI_VFRONT_PORCH_MASK) / 2;
+	bt->vsync = hdmi_read16(state, ADV748X_HDMI_VSYNC_WIDTH,
+				ADV748X_HDMI_VSYNC_WIDTH_MASK) / 2;
+	bt->vbackporch = hdmi_read16(state, ADV748X_HDMI_VBACK_PORCH,
+				     ADV748X_HDMI_VBACK_PORCH_MASK) / 2;
+
+	polarity = hdmi_read(state, 0x05);
+	bt->polarities = (polarity & BIT(4) ? V4L2_DV_VSYNC_POS_POL : 0) |
+		(polarity & BIT(5) ? V4L2_DV_HSYNC_POS_POL : 0);
+
+	if (bt->interlaced == V4L2_DV_INTERLACED) {
+		bt->height += hdmi_read16(state, 0x0b, 0x1fff);
+		bt->il_vfrontporch = hdmi_read16(state, 0x2c, 0x3fff) / 2;
+		bt->il_vsync = hdmi_read16(state, 0x30, 0x3fff) / 2;
+		bt->il_vbackporch = hdmi_read16(state, 0x34, 0x3fff) / 2;
+	}
+
+	adv748x_fill_optional_dv_timings(timings);
+
+	/*
+	 * No interrupt handling is implemented yet.
+	 * There should be an IRQ when a cable is plugged and the new timings
+	 * should be figured out and stored to state.
+	 */
+	hdmi->timings = *timings;
+
+	return 0;
+}
+
+static int adv748x_hdmi_g_input_status(struct v4l2_subdev *sd, u32 *status)
+{
+	struct adv748x_hdmi *hdmi = adv748x_sd_to_hdmi(sd);
+	struct adv748x_state *state = adv748x_hdmi_to_state(hdmi);
+
+	mutex_lock(&state->mutex);
+
+	*status = adv748x_hdmi_has_signal(state) ? 0 : V4L2_IN_ST_NO_SIGNAL;
+
+	mutex_unlock(&state->mutex);
+
+	return 0;
+}
+
+static int adv748x_hdmi_s_stream(struct v4l2_subdev *sd, int enable)
+{
+	struct adv748x_hdmi *hdmi = adv748x_sd_to_hdmi(sd);
+	struct adv748x_state *state = adv748x_hdmi_to_state(hdmi);
+	int ret;
+
+	mutex_lock(&state->mutex);
+
+	ret = adv748x_txa_power(state, enable);
+	if (ret)
+		goto done;
+
+	if (adv748x_hdmi_has_signal(state))
+		adv_dbg(state, "Detected HDMI signal\n");
+	else
+		adv_dbg(state, "Couldn't detect HDMI video signal\n");
+
+done:
+	mutex_unlock(&state->mutex);
+	return ret;
+}
+
+static int adv748x_hdmi_g_pixelaspect(struct v4l2_subdev *sd,
+				      struct v4l2_fract *aspect)
+{
+	aspect->numerator = 1;
+	aspect->denominator = 1;
+
+	return 0;
+}
+
+static const struct v4l2_subdev_video_ops adv748x_video_ops_hdmi = {
+	.s_dv_timings = adv748x_hdmi_s_dv_timings,
+	.g_dv_timings = adv748x_hdmi_g_dv_timings,
+	.query_dv_timings = adv748x_hdmi_query_dv_timings,
+	.g_input_status = adv748x_hdmi_g_input_status,
+	.s_stream = adv748x_hdmi_s_stream,
+	.g_pixelaspect = adv748x_hdmi_g_pixelaspect,
+};
+
+/* -----------------------------------------------------------------------------
+ * v4l2_subdev_pad_ops
+ */
+
+static int adv748x_hdmi_propagate_pixelrate(struct adv748x_hdmi *hdmi)
+{
+	struct v4l2_subdev *tx;
+	struct v4l2_dv_timings timings;
+	struct v4l2_bt_timings *bt = &timings.bt;
+	unsigned int fps;
+
+	tx = adv748x_get_remote_sd(&hdmi->pads[ADV748X_HDMI_SOURCE]);
+	if (!tx)
+		return -ENOLINK;
+
+	adv748x_hdmi_query_dv_timings(&hdmi->sd, &timings);
+
+	fps = DIV_ROUND_CLOSEST_ULL(bt->pixelclock,
+				    V4L2_DV_BT_FRAME_WIDTH(bt) *
+				    V4L2_DV_BT_FRAME_HEIGHT(bt));
+
+	return adv748x_csi2_set_pixelrate(tx, bt->width * bt->height * fps);
+}
+
+static int adv748x_hdmi_enum_mbus_code(struct v4l2_subdev *sd,
+				  struct v4l2_subdev_pad_config *cfg,
+				  struct v4l2_subdev_mbus_code_enum *code)
+{
+	if (code->index != 0)
+		return -EINVAL;
+
+	code->code = MEDIA_BUS_FMT_RGB888_1X24;
+
+	return 0;
+}
+
+static int adv748x_hdmi_get_format(struct v4l2_subdev *sd,
+				   struct v4l2_subdev_pad_config *cfg,
+				   struct v4l2_subdev_format *sdformat)
+{
+	struct adv748x_hdmi *hdmi = adv748x_sd_to_hdmi(sd);
+	struct v4l2_mbus_framefmt *mbusformat;
+
+	if (sdformat->pad != ADV748X_HDMI_SOURCE)
+		return -EINVAL;
+
+	if (sdformat->which == V4L2_SUBDEV_FORMAT_TRY) {
+		mbusformat = v4l2_subdev_get_try_format(sd, cfg, sdformat->pad);
+		sdformat->format = *mbusformat;
+	} else {
+		adv748x_hdmi_fill_format(hdmi, &sdformat->format);
+		adv748x_hdmi_propagate_pixelrate(hdmi);
+	}
+
+	return 0;
+}
+
+static int adv748x_hdmi_set_format(struct v4l2_subdev *sd,
+				   struct v4l2_subdev_pad_config *cfg,
+				   struct v4l2_subdev_format *sdformat)
+{
+	struct v4l2_mbus_framefmt *mbusformat;
+
+	if (sdformat->pad != ADV748X_HDMI_SOURCE)
+		return -EINVAL;
+
+	if (sdformat->which == V4L2_SUBDEV_FORMAT_ACTIVE)
+		return adv748x_hdmi_get_format(sd, cfg, sdformat);
+
+	mbusformat = v4l2_subdev_get_try_format(sd, cfg, sdformat->pad);
+	*mbusformat = sdformat->format;
+
+	return 0;
+}
+
+static int adv748x_hdmi_get_edid(struct v4l2_subdev *sd, struct v4l2_edid *edid)
+{
+	struct adv748x_hdmi *hdmi = adv748x_sd_to_hdmi(sd);
+
+	memset(edid->reserved, 0, sizeof(edid->reserved));
+
+	if (!hdmi->edid.present)
+		return -ENODATA;
+
+	if (edid->start_block == 0 && edid->blocks == 0) {
+		edid->blocks = hdmi->edid.blocks;
+		return 0;
+	}
+
+	if (edid->start_block >= hdmi->edid.blocks)
+		return -EINVAL;
+
+	if (edid->start_block + edid->blocks > hdmi->edid.blocks)
+		edid->blocks = hdmi->edid.blocks - edid->start_block;
+
+	memcpy(edid->edid, hdmi->edid.edid + edid->start_block * 128,
+			edid->blocks * 128);
+
+	return 0;
+}
+
+static inline int adv748x_hdmi_edid_write_block(struct adv748x_hdmi *hdmi,
+					unsigned int total_len, const u8 *val)
+{
+	struct adv748x_state *state = adv748x_hdmi_to_state(hdmi);
+	int err = 0;
+	int i = 0;
+	int len = 0;
+
+	adv_dbg(state, "%s: write EDID block (%d byte)\n",
+				__func__, total_len);
+
+	while (!err && i < total_len) {
+		len = (total_len - i) > I2C_SMBUS_BLOCK_MAX ?
+				I2C_SMBUS_BLOCK_MAX :
+				(total_len - i);
+
+		err = adv748x_write_block(state, ADV748X_PAGE_EDID,
+				i, val + i, len);
+		i += len;
+	}
+
+	return err;
+}
+
+static int adv748x_hdmi_set_edid(struct v4l2_subdev *sd, struct v4l2_edid *edid)
+{
+	struct adv748x_hdmi *hdmi = adv748x_sd_to_hdmi(sd);
+	struct adv748x_state *state = adv748x_hdmi_to_state(hdmi);
+	int err;
+
+	memset(edid->reserved, 0, sizeof(edid->reserved));
+
+	if (edid->start_block != 0)
+		return -EINVAL;
+
+	if (edid->blocks == 0) {
+		hdmi->edid.blocks = 0;
+		hdmi->edid.present = 0;
+
+		/* Fall back to a 16:9 aspect ratio */
+		hdmi->aspect_ratio.numerator = 16;
+		hdmi->aspect_ratio.denominator = 9;
+
+		/* Disable the EDID */
+		repeater_write(state, ADV748X_REPEATER_EDID_SZ,
+			       edid->blocks << ADV748X_REPEATER_EDID_SZ_SHIFT);
+
+		repeater_write(state, ADV748X_REPEATER_EDID_CTL, 0);
+
+		return 0;
+	}
+
+	if (edid->blocks > 4) {
+		edid->blocks = 4;
+		return -E2BIG;
+	}
+
+	memcpy(hdmi->edid.edid, edid->edid, 128 * edid->blocks);
+	hdmi->edid.blocks = edid->blocks;
+	hdmi->edid.present = true;
+
+	hdmi->aspect_ratio = v4l2_calc_aspect_ratio(edid->edid[0x15],
+			edid->edid[0x16]);
+
+	err = adv748x_hdmi_edid_write_block(hdmi, 128 * edid->blocks,
+			hdmi->edid.edid);
+	if (err < 0) {
+		v4l2_err(sd, "error %d writing edid pad %d\n", err, edid->pad);
+		return err;
+	}
+
+	repeater_write(state, ADV748X_REPEATER_EDID_SZ,
+		       edid->blocks << ADV748X_REPEATER_EDID_SZ_SHIFT);
+
+	repeater_write(state, ADV748X_REPEATER_EDID_CTL,
+		       ADV748X_REPEATER_EDID_CTL_EN);
+
+	return 0;
+}
+
+static bool adv748x_hdmi_check_dv_timings(const struct v4l2_dv_timings *timings,
+					  void *hdl)
+{
+	const struct adv748x_hdmi_video_standards *stds =
+		adv748x_hdmi_video_standards;
+	unsigned int i;
+
+	for (i = 0; stds[i].timings.bt.width; i++)
+		if (v4l2_match_dv_timings(timings, &stds[i].timings, 0, false))
+			return true;
+
+	return false;
+}
+
+static int adv748x_hdmi_enum_dv_timings(struct v4l2_subdev *sd,
+					struct v4l2_enum_dv_timings *timings)
+{
+	return v4l2_enum_dv_timings_cap(timings, &adv748x_hdmi_timings_cap,
+					adv748x_hdmi_check_dv_timings, NULL);
+}
+
+static int adv748x_hdmi_dv_timings_cap(struct v4l2_subdev *sd,
+				       struct v4l2_dv_timings_cap *cap)
+{
+	*cap = adv748x_hdmi_timings_cap;
+	return 0;
+}
+
+static const struct v4l2_subdev_pad_ops adv748x_pad_ops_hdmi = {
+	.enum_mbus_code = adv748x_hdmi_enum_mbus_code,
+	.set_fmt = adv748x_hdmi_set_format,
+	.get_fmt = adv748x_hdmi_get_format,
+	.get_edid = adv748x_hdmi_get_edid,
+	.set_edid = adv748x_hdmi_set_edid,
+	.dv_timings_cap = adv748x_hdmi_dv_timings_cap,
+	.enum_dv_timings = adv748x_hdmi_enum_dv_timings,
+};
+
+/* -----------------------------------------------------------------------------
+ * v4l2_subdev_ops
+ */
+
+static const struct v4l2_subdev_ops adv748x_ops_hdmi = {
+	.video = &adv748x_video_ops_hdmi,
+	.pad = &adv748x_pad_ops_hdmi,
+};
+
+/* -----------------------------------------------------------------------------
+ * Controls
+ */
+
+static const char * const hdmi_ctrl_patgen_menu[] = {
+	"Disabled",
+	"Solid Color",
+	"Color Bars",
+	"Ramp Grey",
+	"Ramp Blue",
+	"Ramp Red",
+	"Checkered"
+};
+
+static int adv748x_hdmi_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct adv748x_hdmi *hdmi = adv748x_ctrl_to_hdmi(ctrl);
+	struct adv748x_state *state = adv748x_hdmi_to_state(hdmi);
+	int ret;
+	u8 pattern;
+
+	/* Enable video adjustment first */
+	ret = cp_clrset(state, ADV748X_CP_VID_ADJ,
+			ADV748X_CP_VID_ADJ_ENABLE,
+			ADV748X_CP_VID_ADJ_ENABLE);
+	if (ret < 0)
+		return ret;
+
+	switch (ctrl->id) {
+	case V4L2_CID_BRIGHTNESS:
+		ret = cp_write(state, ADV748X_CP_BRI, ctrl->val);
+		break;
+	case V4L2_CID_HUE:
+		ret = cp_write(state, ADV748X_CP_HUE, ctrl->val);
+		break;
+	case V4L2_CID_CONTRAST:
+		ret = cp_write(state, ADV748X_CP_CON, ctrl->val);
+		break;
+	case V4L2_CID_SATURATION:
+		ret = cp_write(state, ADV748X_CP_SAT, ctrl->val);
+		break;
+	case V4L2_CID_TEST_PATTERN:
+		pattern = ctrl->val;
+
+		/* Pattern is 0-indexed. Ctrl Menu is 1-indexed */
+		if (pattern) {
+			pattern--;
+			pattern |= ADV748X_CP_PAT_GEN_EN;
+		}
+
+		ret = cp_write(state, ADV748X_CP_PAT_GEN, pattern);
+
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return ret;
+}
+
+static const struct v4l2_ctrl_ops adv748x_hdmi_ctrl_ops = {
+	.s_ctrl = adv748x_hdmi_s_ctrl,
+};
+
+static int adv748x_hdmi_init_controls(struct adv748x_hdmi *hdmi)
+{
+	struct adv748x_state *state = adv748x_hdmi_to_state(hdmi);
+
+	v4l2_ctrl_handler_init(&hdmi->ctrl_hdl, 5);
+
+	/* Use our mutex for the controls */
+	hdmi->ctrl_hdl.lock = &state->mutex;
+
+	v4l2_ctrl_new_std(&hdmi->ctrl_hdl, &adv748x_hdmi_ctrl_ops,
+			  V4L2_CID_BRIGHTNESS, ADV748X_CP_BRI_MIN,
+			  ADV748X_CP_BRI_MAX, 1, ADV748X_CP_BRI_DEF);
+	v4l2_ctrl_new_std(&hdmi->ctrl_hdl, &adv748x_hdmi_ctrl_ops,
+			  V4L2_CID_CONTRAST, ADV748X_CP_CON_MIN,
+			  ADV748X_CP_CON_MAX, 1, ADV748X_CP_CON_DEF);
+	v4l2_ctrl_new_std(&hdmi->ctrl_hdl, &adv748x_hdmi_ctrl_ops,
+			  V4L2_CID_SATURATION, ADV748X_CP_SAT_MIN,
+			  ADV748X_CP_SAT_MAX, 1, ADV748X_CP_SAT_DEF);
+	v4l2_ctrl_new_std(&hdmi->ctrl_hdl, &adv748x_hdmi_ctrl_ops,
+			  V4L2_CID_HUE, ADV748X_CP_HUE_MIN,
+			  ADV748X_CP_HUE_MAX, 1, ADV748X_CP_HUE_DEF);
+
+	/*
+	 * Todo: V4L2_CID_DV_RX_POWER_PRESENT should also be supported when
+	 * interrupts are handled correctly
+	 */
+
+	v4l2_ctrl_new_std_menu_items(&hdmi->ctrl_hdl, &adv748x_hdmi_ctrl_ops,
+				     V4L2_CID_TEST_PATTERN,
+				     ARRAY_SIZE(hdmi_ctrl_patgen_menu) - 1,
+				     0, 0, hdmi_ctrl_patgen_menu);
+
+	hdmi->sd.ctrl_handler = &hdmi->ctrl_hdl;
+	if (hdmi->ctrl_hdl.error) {
+		v4l2_ctrl_handler_free(&hdmi->ctrl_hdl);
+		return hdmi->ctrl_hdl.error;
+	}
+
+	return v4l2_ctrl_handler_setup(&hdmi->ctrl_hdl);
+}
+
+int adv748x_hdmi_init(struct adv748x_hdmi *hdmi)
+{
+	struct adv748x_state *state = adv748x_hdmi_to_state(hdmi);
+	static const struct v4l2_dv_timings cea1280x720 =
+		V4L2_DV_BT_CEA_1280X720P30;
+	int ret;
+
+	hdmi->timings = cea1280x720;
+
+	/* Initialise a default 16:9 aspect ratio */
+	hdmi->aspect_ratio.numerator = 16;
+	hdmi->aspect_ratio.denominator = 9;
+
+	adv748x_subdev_init(&hdmi->sd, state, &adv748x_ops_hdmi,
+			    MEDIA_ENT_F_IO_DTV, "hdmi");
+
+	hdmi->pads[ADV748X_HDMI_SINK].flags = MEDIA_PAD_FL_SINK;
+	hdmi->pads[ADV748X_HDMI_SOURCE].flags = MEDIA_PAD_FL_SOURCE;
+
+	ret = media_entity_pads_init(&hdmi->sd.entity,
+				     ADV748X_HDMI_NR_PADS, hdmi->pads);
+	if (ret)
+		return ret;
+
+	ret = adv748x_hdmi_init_controls(hdmi);
+	if (ret)
+		goto err_free_media;
+
+	return 0;
+
+err_free_media:
+	media_entity_cleanup(&hdmi->sd.entity);
+
+	return ret;
+}
+
+void adv748x_hdmi_cleanup(struct adv748x_hdmi *hdmi)
+{
+	v4l2_device_unregister_subdev(&hdmi->sd);
+	media_entity_cleanup(&hdmi->sd.entity);
+	v4l2_ctrl_handler_free(&hdmi->ctrl_hdl);
+}
diff --git a/drivers/media/i2c/adv748x/adv748x.h b/drivers/media/i2c/adv748x/adv748x.h
new file mode 100644
index 000000000000..cc4151b5b31e
--- /dev/null
+++ b/drivers/media/i2c/adv748x/adv748x.h
@@ -0,0 +1,425 @@
+/*
+ * Driver for Analog Devices ADV748X video decoder and HDMI receiver
+ *
+ * Copyright (C) 2017 Renesas Electronics Corp.
+ *
+ * This program is free software; you can redistribute  it and/or modify it
+ * under  the terms of  the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ *
+ * Authors:
+ *	Koji Matsuoka <koji.matsuoka.xm@renesas.com>
+ *	Niklas Söderlund <niklas.soderlund@ragnatech.se>
+ *	Kieran Bingham <kieran.bingham@ideasonboard.com>
+ *
+ * The ADV748x range of receivers have the following configurations:
+ *
+ *                  Analog   HDMI  MHL  4-Lane  1-Lane
+ *                    In      In         CSI     CSI
+ *       ADV7480               X    X     X
+ *       ADV7481      X        X    X     X       X
+ *       ADV7482      X        X          X       X
+ */
+
+#include <linux/i2c.h>
+
+#ifndef _ADV748X_H_
+#define _ADV748X_H_
+
+/* I2C slave addresses */
+#define ADV748X_I2C_IO			0x70	/* IO Map */
+#define ADV748X_I2C_DPLL		0x26	/* DPLL Map */
+#define ADV748X_I2C_CP			0x22	/* CP Map */
+#define ADV748X_I2C_HDMI		0x34	/* HDMI Map */
+#define ADV748X_I2C_EDID		0x36	/* EDID Map */
+#define ADV748X_I2C_REPEATER		0x32	/* HDMI RX Repeater Map */
+#define ADV748X_I2C_INFOFRAME		0x31	/* HDMI RX InfoFrame Map */
+#define ADV748X_I2C_CEC			0x41	/* CEC Map */
+#define ADV748X_I2C_SDP			0x79	/* SDP Map */
+#define ADV748X_I2C_TXB			0x48	/* CSI-TXB Map */
+#define ADV748X_I2C_TXA			0x4a	/* CSI-TXA Map */
+
+enum adv748x_page {
+	ADV748X_PAGE_IO,
+	ADV748X_PAGE_DPLL,
+	ADV748X_PAGE_CP,
+	ADV748X_PAGE_HDMI,
+	ADV748X_PAGE_EDID,
+	ADV748X_PAGE_REPEATER,
+	ADV748X_PAGE_INFOFRAME,
+	ADV748X_PAGE_CEC,
+	ADV748X_PAGE_SDP,
+	ADV748X_PAGE_TXB,
+	ADV748X_PAGE_TXA,
+	ADV748X_PAGE_MAX,
+
+	/* Fake pages for register sequences */
+	ADV748X_PAGE_WAIT,		/* Wait x msec */
+	ADV748X_PAGE_EOR,		/* End Mark */
+};
+
+/**
+ * enum adv748x_ports - Device tree port number definitions
+ *
+ * The ADV748X ports define the mapping between subdevices
+ * and the device tree specification
+ */
+enum adv748x_ports {
+	ADV748X_PORT_AIN0 = 0,
+	ADV748X_PORT_AIN1 = 1,
+	ADV748X_PORT_AIN2 = 2,
+	ADV748X_PORT_AIN3 = 3,
+	ADV748X_PORT_AIN4 = 4,
+	ADV748X_PORT_AIN5 = 5,
+	ADV748X_PORT_AIN6 = 6,
+	ADV748X_PORT_AIN7 = 7,
+	ADV748X_PORT_HDMI = 8,
+	ADV748X_PORT_TTL = 9,
+	ADV748X_PORT_TXA = 10,
+	ADV748X_PORT_TXB = 11,
+	ADV748X_PORT_MAX = 12,
+};
+
+enum adv748x_csi2_pads {
+	ADV748X_CSI2_SINK,
+	ADV748X_CSI2_SOURCE,
+	ADV748X_CSI2_NR_PADS,
+};
+
+/* CSI2 transmitters can have 2 internal connections, HDMI/AFE */
+#define ADV748X_CSI2_MAX_SUBDEVS 2
+
+struct adv748x_csi2 {
+	struct adv748x_state *state;
+	struct v4l2_mbus_framefmt format;
+	unsigned int page;
+
+	struct media_pad pads[ADV748X_CSI2_NR_PADS];
+	struct v4l2_ctrl_handler ctrl_hdl;
+	struct v4l2_subdev sd;
+};
+
+#define notifier_to_csi2(n) container_of(n, struct adv748x_csi2, notifier)
+#define adv748x_sd_to_csi2(sd) container_of(sd, struct adv748x_csi2, sd)
+
+enum adv748x_hdmi_pads {
+	ADV748X_HDMI_SINK,
+	ADV748X_HDMI_SOURCE,
+	ADV748X_HDMI_NR_PADS,
+};
+
+struct adv748x_hdmi {
+	struct media_pad pads[ADV748X_HDMI_NR_PADS];
+	struct v4l2_ctrl_handler ctrl_hdl;
+	struct v4l2_subdev sd;
+	struct v4l2_mbus_framefmt format;
+
+	struct v4l2_dv_timings timings;
+	struct v4l2_fract aspect_ratio;
+
+	struct {
+		u8 edid[512];
+		u32 present;
+		unsigned int blocks;
+	} edid;
+};
+
+#define adv748x_ctrl_to_hdmi(ctrl) \
+	container_of(ctrl->handler, struct adv748x_hdmi, ctrl_hdl)
+#define adv748x_sd_to_hdmi(sd) container_of(sd, struct adv748x_hdmi, sd)
+
+enum adv748x_afe_pads {
+	ADV748X_AFE_SINK_AIN0,
+	ADV748X_AFE_SINK_AIN1,
+	ADV748X_AFE_SINK_AIN2,
+	ADV748X_AFE_SINK_AIN3,
+	ADV748X_AFE_SINK_AIN4,
+	ADV748X_AFE_SINK_AIN5,
+	ADV748X_AFE_SINK_AIN6,
+	ADV748X_AFE_SINK_AIN7,
+	ADV748X_AFE_SOURCE,
+	ADV748X_AFE_NR_PADS,
+};
+
+struct adv748x_afe {
+	struct media_pad pads[ADV748X_AFE_NR_PADS];
+	struct v4l2_ctrl_handler ctrl_hdl;
+	struct v4l2_subdev sd;
+	struct v4l2_mbus_framefmt format;
+
+	bool streaming;
+	v4l2_std_id curr_norm;
+	unsigned int input;
+};
+
+#define adv748x_ctrl_to_afe(ctrl) \
+	container_of(ctrl->handler, struct adv748x_afe, ctrl_hdl)
+#define adv748x_sd_to_afe(sd) container_of(sd, struct adv748x_afe, sd)
+
+/**
+ * struct adv748x_state - State of ADV748X
+ * @dev:		(OF) device
+ * @client:		I2C client
+ * @mutex:		protect global state
+ *
+ * @endpoints:		parsed device node endpoints for each port
+ *
+ * @i2c_addresses	I2C Page addresses
+ * @i2c_clients		I2C clients for the page accesses
+ * @regmap		regmap configuration pages.
+ *
+ * @hdmi:		state of HDMI receiver context
+ * @afe:		state of AFE receiver context
+ * @txa:		state of TXA transmitter context
+ * @txb:		state of TXB transmitter context
+ */
+struct adv748x_state {
+	struct device *dev;
+	struct i2c_client *client;
+	struct mutex mutex;
+
+	struct device_node *endpoints[ADV748X_PORT_MAX];
+
+	struct i2c_client *i2c_clients[ADV748X_PAGE_MAX];
+	struct regmap *regmap[ADV748X_PAGE_MAX];
+
+	struct adv748x_hdmi hdmi;
+	struct adv748x_afe afe;
+	struct adv748x_csi2 txa;
+	struct adv748x_csi2 txb;
+};
+
+#define adv748x_hdmi_to_state(h) container_of(h, struct adv748x_state, hdmi)
+#define adv748x_afe_to_state(a) container_of(a, struct adv748x_state, afe)
+
+#define adv_err(a, fmt, arg...)	dev_err(a->dev, fmt, ##arg)
+#define adv_info(a, fmt, arg...) dev_info(a->dev, fmt, ##arg)
+#define adv_dbg(a, fmt, arg...)	dev_dbg(a->dev, fmt, ##arg)
+
+/* Register Mappings */
+
+/* IO Map */
+#define ADV748X_IO_PD			0x00	/* power down controls */
+#define ADV748X_IO_PD_RX_EN		BIT(6)
+
+#define ADV748X_IO_REG_04		0x04
+#define ADV748X_IO_REG_04_FORCE_FR	BIT(0)	/* Force CP free-run */
+
+#define ADV748X_IO_DATAPATH		0x03	/* datapath cntrl */
+#define ADV748X_IO_DATAPATH_VFREQ_M	0x70
+#define ADV748X_IO_DATAPATH_VFREQ_SHIFT	4
+
+#define ADV748X_IO_VID_STD		0x05
+
+#define ADV748X_IO_10			0x10	/* io_reg_10 */
+#define ADV748X_IO_10_CSI4_EN		BIT(7)
+#define ADV748X_IO_10_CSI1_EN		BIT(6)
+#define ADV748X_IO_10_PIX_OUT_EN	BIT(5)
+
+#define ADV748X_IO_CHIP_REV_ID_1	0xdf
+#define ADV748X_IO_CHIP_REV_ID_2	0xe0
+
+#define ADV748X_IO_SLAVE_ADDR_BASE	0xf2
+
+/* HDMI RX Map */
+#define ADV748X_HDMI_LW1		0x07	/* line width_1 */
+#define ADV748X_HDMI_LW1_VERT_FILTER	BIT(7)
+#define ADV748X_HDMI_LW1_DE_REGEN	BIT(5)
+#define ADV748X_HDMI_LW1_WIDTH_MASK	0x1fff
+
+#define ADV748X_HDMI_F0H1		0x09	/* field0 height_1 */
+#define ADV748X_HDMI_F0H1_HEIGHT_MASK	0x1fff
+
+#define ADV748X_HDMI_F1H1		0x0b	/* field1 height_1 */
+#define ADV748X_HDMI_F1H1_INTERLACED	BIT(5)
+
+#define ADV748X_HDMI_HFRONT_PORCH	0x20	/* hsync_front_porch_1 */
+#define ADV748X_HDMI_HFRONT_PORCH_MASK	0x1fff
+
+#define ADV748X_HDMI_HSYNC_WIDTH	0x22	/* hsync_pulse_width_1 */
+#define ADV748X_HDMI_HSYNC_WIDTH_MASK	0x1fff
+
+#define ADV748X_HDMI_HBACK_PORCH	0x24	/* hsync_back_porch_1 */
+#define ADV748X_HDMI_HBACK_PORCH_MASK	0x1fff
+
+#define ADV748X_HDMI_VFRONT_PORCH	0x2a	/* field0_vs_front_porch_1 */
+#define ADV748X_HDMI_VFRONT_PORCH_MASK	0x3fff
+
+#define ADV748X_HDMI_VSYNC_WIDTH	0x2e	/* field0_vs_pulse_width_1 */
+#define ADV748X_HDMI_VSYNC_WIDTH_MASK	0x3fff
+
+#define ADV748X_HDMI_VBACK_PORCH	0x32	/* field0_vs_back_porch_1 */
+#define ADV748X_HDMI_VBACK_PORCH_MASK	0x3fff
+
+#define ADV748X_HDMI_TMDS_1		0x51	/* hdmi_reg_51 */
+#define ADV748X_HDMI_TMDS_2		0x52	/* hdmi_reg_52 */
+
+/* HDMI RX Repeater Map */
+#define ADV748X_REPEATER_EDID_SZ	0x70	/* primary_edid_size */
+#define ADV748X_REPEATER_EDID_SZ_SHIFT	4
+
+#define ADV748X_REPEATER_EDID_CTL	0x74	/* hdcp edid controls */
+#define ADV748X_REPEATER_EDID_CTL_EN	BIT(0)	/* man_edid_a_enable */
+
+/* SDP Main Map */
+#define ADV748X_SDP_INSEL		0x00	/* user_map_rw_reg_00 */
+
+#define ADV748X_SDP_VID_SEL		0x02	/* user_map_rw_reg_02 */
+#define ADV748X_SDP_VID_SEL_MASK	0xf0
+#define ADV748X_SDP_VID_SEL_SHIFT	4
+
+/* Contrast - Unsigned*/
+#define ADV748X_SDP_CON			0x08	/* user_map_rw_reg_08 */
+#define ADV748X_SDP_CON_MIN		0
+#define ADV748X_SDP_CON_DEF		128
+#define ADV748X_SDP_CON_MAX		255
+
+/* Brightness - Signed */
+#define ADV748X_SDP_BRI			0x0a	/* user_map_rw_reg_0a */
+#define ADV748X_SDP_BRI_MIN		-128
+#define ADV748X_SDP_BRI_DEF		0
+#define ADV748X_SDP_BRI_MAX		127
+
+/* Hue - Signed, inverted*/
+#define ADV748X_SDP_HUE			0x0b	/* user_map_rw_reg_0b */
+#define ADV748X_SDP_HUE_MIN		-127
+#define ADV748X_SDP_HUE_DEF		0
+#define ADV748X_SDP_HUE_MAX		128
+
+/* Test Patterns / Default Values */
+#define ADV748X_SDP_DEF			0x0c	/* user_map_rw_reg_0c */
+#define ADV748X_SDP_DEF_VAL_EN		BIT(0)	/* Force free run mode */
+#define ADV748X_SDP_DEF_VAL_AUTO_EN	BIT(1)	/* Free run when no signal */
+
+#define ADV748X_SDP_MAP_SEL		0x0e	/* user_map_rw_reg_0e */
+#define ADV748X_SDP_MAP_SEL_RO_MAIN	1
+
+/* Free run pattern select */
+#define ADV748X_SDP_FRP			0x14
+#define ADV748X_SDP_FRP_MASK		GENMASK(3, 1)
+
+/* Saturation */
+#define ADV748X_SDP_SD_SAT_U		0xe3	/* user_map_rw_reg_e3 */
+#define ADV748X_SDP_SD_SAT_V		0xe4	/* user_map_rw_reg_e4 */
+#define ADV748X_SDP_SAT_MIN		0
+#define ADV748X_SDP_SAT_DEF		128
+#define ADV748X_SDP_SAT_MAX		255
+
+/* SDP RO Main Map */
+#define ADV748X_SDP_RO_10		0x10
+#define ADV748X_SDP_RO_10_IN_LOCK	BIT(0)
+
+/* CP Map */
+#define ADV748X_CP_PAT_GEN		0x37	/* int_pat_gen_1 */
+#define ADV748X_CP_PAT_GEN_EN		BIT(7)
+
+/* Contrast Control - Unsigned */
+#define ADV748X_CP_CON			0x3a	/* contrast_cntrl */
+#define ADV748X_CP_CON_MIN		0	/* Minimum contrast */
+#define ADV748X_CP_CON_DEF		128	/* Default */
+#define ADV748X_CP_CON_MAX		255	/* Maximum contrast */
+
+/* Saturation Control - Unsigned */
+#define ADV748X_CP_SAT			0x3b	/* saturation_cntrl */
+#define ADV748X_CP_SAT_MIN		0	/* Minimum saturation */
+#define ADV748X_CP_SAT_DEF		128	/* Default */
+#define ADV748X_CP_SAT_MAX		255	/* Maximum saturation */
+
+/* Brightness Control - Signed */
+#define ADV748X_CP_BRI			0x3c	/* brightness_cntrl */
+#define ADV748X_CP_BRI_MIN		-128	/* Luma is -512d */
+#define ADV748X_CP_BRI_DEF		0	/* Luma is 0 */
+#define ADV748X_CP_BRI_MAX		127	/* Luma is 508d */
+
+/* Hue Control */
+#define ADV748X_CP_HUE			0x3d	/* hue_cntrl */
+#define ADV748X_CP_HUE_MIN		0	/* -90 degree */
+#define ADV748X_CP_HUE_DEF		0	/* -90 degree */
+#define ADV748X_CP_HUE_MAX		255	/* +90 degree */
+
+#define ADV748X_CP_VID_ADJ		0x3e	/* vid_adj_0 */
+#define ADV748X_CP_VID_ADJ_ENABLE	BIT(7)	/* Enable colour controls */
+
+#define ADV748X_CP_DE_POS_HIGH		0x8b	/* de_pos_adj_6 */
+#define ADV748X_CP_DE_POS_HIGH_SET	BIT(6)
+#define ADV748X_CP_DE_POS_END_LOW	0x8c	/* de_pos_adj_7 */
+#define ADV748X_CP_DE_POS_START_LOW	0x8d	/* de_pos_adj_8 */
+
+#define ADV748X_CP_VID_ADJ_2			0x91
+#define ADV748X_CP_VID_ADJ_2_INTERLACED		BIT(6)
+#define ADV748X_CP_VID_ADJ_2_INTERLACED_3D	BIT(4)
+
+#define ADV748X_CP_CLMP_POS		0xc9	/* clmp_pos_cntrl_4 */
+#define ADV748X_CP_CLMP_POS_DIS_AUTO	BIT(0)	/* dis_auto_param_buff */
+
+/* CSI : TXA/TXB Maps */
+#define ADV748X_CSI_VC_REF		0x0d	/* csi_tx_top_reg_0d */
+#define ADV748X_CSI_VC_REF_SHIFT	6
+
+#define ADV748X_CSI_FS_AS_LS		0x1e	/* csi_tx_top_reg_1e */
+#define ADV748X_CSI_FS_AS_LS_UNKNOWN	BIT(6)	/* Undocumented bit */
+
+/* Register handling */
+
+int adv748x_read(struct adv748x_state *state, u8 addr, u8 reg);
+int adv748x_write(struct adv748x_state *state, u8 page, u8 reg, u8 value);
+int adv748x_write_block(struct adv748x_state *state, int client_page,
+			unsigned int init_reg, const void *val,
+			size_t val_len);
+
+#define io_read(s, r) adv748x_read(s, ADV748X_PAGE_IO, r)
+#define io_write(s, r, v) adv748x_write(s, ADV748X_PAGE_IO, r, v)
+#define io_clrset(s, r, m, v) io_write(s, r, (io_read(s, r) & ~m) | v)
+
+#define hdmi_read(s, r) adv748x_read(s, ADV748X_PAGE_HDMI, r)
+#define hdmi_read16(s, r, m) (((hdmi_read(s, r) << 8) | hdmi_read(s, r+1)) & m)
+#define hdmi_write(s, r, v) adv748x_write(s, ADV748X_PAGE_HDMI, r, v)
+
+#define repeater_read(s, r) adv748x_read(s, ADV748X_PAGE_REPEATER, r)
+#define repeater_write(s, r, v) adv748x_write(s, ADV748X_PAGE_REPEATER, r, v)
+
+#define sdp_read(s, r) adv748x_read(s, ADV748X_PAGE_SDP, r)
+#define sdp_write(s, r, v) adv748x_write(s, ADV748X_PAGE_SDP, r, v)
+#define sdp_clrset(s, r, m, v) sdp_write(s, r, (sdp_read(s, r) & ~m) | v)
+
+#define cp_read(s, r) adv748x_read(s, ADV748X_PAGE_CP, r)
+#define cp_write(s, r, v) adv748x_write(s, ADV748X_PAGE_CP, r, v)
+#define cp_clrset(s, r, m, v) cp_write(s, r, (cp_read(s, r) & ~m) | v)
+
+#define txa_read(s, r) adv748x_read(s, ADV748X_PAGE_TXA, r)
+#define txb_read(s, r) adv748x_read(s, ADV748X_PAGE_TXB, r)
+
+#define tx_read(t, r) adv748x_read(t->state, t->page, r)
+#define tx_write(t, r, v) adv748x_write(t->state, t->page, r, v)
+
+static inline struct v4l2_subdev *adv748x_get_remote_sd(struct media_pad *pad)
+{
+	pad = media_entity_remote_pad(pad);
+	if (!pad)
+		return NULL;
+
+	return media_entity_to_v4l2_subdev(pad->entity);
+}
+
+void adv748x_subdev_init(struct v4l2_subdev *sd, struct adv748x_state *state,
+			 const struct v4l2_subdev_ops *ops, u32 function,
+			 const char *ident);
+
+int adv748x_register_subdevs(struct adv748x_state *state,
+			     struct v4l2_device *v4l2_dev);
+
+int adv748x_txa_power(struct adv748x_state *state, bool on);
+int adv748x_txb_power(struct adv748x_state *state, bool on);
+
+int adv748x_afe_init(struct adv748x_afe *afe);
+void adv748x_afe_cleanup(struct adv748x_afe *afe);
+
+int adv748x_csi2_init(struct adv748x_state *state, struct adv748x_csi2 *tx);
+void adv748x_csi2_cleanup(struct adv748x_csi2 *tx);
+int adv748x_csi2_set_pixelrate(struct v4l2_subdev *sd, s64 rate);
+
+int adv748x_hdmi_init(struct adv748x_hdmi *hdmi);
+void adv748x_hdmi_cleanup(struct adv748x_hdmi *hdmi);
+
+#endif /* _ADV748X_H_ */
diff --git a/drivers/media/i2c/adv7511.c b/drivers/media/i2c/adv7511.c
index ccc478605643..2817bafc67bf 100644
--- a/drivers/media/i2c/adv7511.c
+++ b/drivers/media/i2c/adv7511.c
@@ -1927,8 +1927,7 @@ static int adv7511_probe(struct i2c_client *client, const struct i2c_device_id *
 
 #if IS_ENABLED(CONFIG_VIDEO_ADV7511_CEC)
 	state->cec_adap = cec_allocate_adapter(&adv7511_cec_adap_ops,
-		state, dev_name(&client->dev), CEC_CAP_TRANSMIT |
-		CEC_CAP_LOG_ADDRS | CEC_CAP_PASSTHROUGH | CEC_CAP_RC,
+		state, dev_name(&client->dev), CEC_CAP_DEFAULTS,
 		ADV7511_MAX_ADDRS);
 	err = PTR_ERR_OR_ZERO(state->cec_adap);
 	if (err) {
@@ -1986,7 +1985,7 @@ static int adv7511_remove(struct i2c_client *client)
 
 /* ----------------------------------------------------------------------- */
 
-static struct i2c_device_id adv7511_id[] = {
+static const struct i2c_device_id adv7511_id[] = {
 	{ "adv7511", 0 },
 	{ }
 };
diff --git a/drivers/media/i2c/adv7604.c b/drivers/media/i2c/adv7604.c
index 660bacb8f7d9..f289b8aca1da 100644
--- a/drivers/media/i2c/adv7604.c
+++ b/drivers/media/i2c/adv7604.c
@@ -618,7 +618,7 @@ static int adv76xx_read_reg(struct v4l2_subdev *sd, unsigned int reg)
 	unsigned int val;
 	int err;
 
-	if (!(BIT(page) & state->info->page_mask))
+	if (page >= ADV76XX_PAGE_MAX || !(BIT(page) & state->info->page_mask))
 		return -EINVAL;
 
 	reg &= 0xff;
@@ -633,7 +633,7 @@ static int adv76xx_write_reg(struct v4l2_subdev *sd, unsigned int reg, u8 val)
 	struct adv76xx_state *state = to_state(sd);
 	unsigned int page = reg >> 8;
 
-	if (!(BIT(page) & state->info->page_mask))
+	if (page >= ADV76XX_PAGE_MAX || !(BIT(page) & state->info->page_mask))
 		return -EINVAL;
 
 	reg &= 0xff;
@@ -3515,8 +3515,7 @@ static int adv76xx_probe(struct i2c_client *client,
 #if IS_ENABLED(CONFIG_VIDEO_ADV7604_CEC)
 	state->cec_adap = cec_allocate_adapter(&adv76xx_cec_adap_ops,
 		state, dev_name(&client->dev),
-		CEC_CAP_TRANSMIT | CEC_CAP_LOG_ADDRS |
-		CEC_CAP_PASSTHROUGH | CEC_CAP_RC, ADV76XX_MAX_ADDRS);
+		CEC_CAP_DEFAULTS, ADV76XX_MAX_ADDRS);
 	err = PTR_ERR_OR_ZERO(state->cec_adap);
 	if (err)
 		goto err_entity;
diff --git a/drivers/media/i2c/adv7842.c b/drivers/media/i2c/adv7842.c
index 303effda1a2e..65f34e7e146f 100644
--- a/drivers/media/i2c/adv7842.c
+++ b/drivers/media/i2c/adv7842.c
@@ -3568,8 +3568,7 @@ static int adv7842_probe(struct i2c_client *client,
 #if IS_ENABLED(CONFIG_VIDEO_ADV7842_CEC)
 	state->cec_adap = cec_allocate_adapter(&adv7842_cec_adap_ops,
 		state, dev_name(&client->dev),
-		CEC_CAP_TRANSMIT | CEC_CAP_LOG_ADDRS |
-		CEC_CAP_PASSTHROUGH | CEC_CAP_RC, ADV7842_MAX_ADDRS);
+		CEC_CAP_DEFAULTS, ADV7842_MAX_ADDRS);
 	err = PTR_ERR_OR_ZERO(state->cec_adap);
 	if (err)
 		goto err_entity;
@@ -3608,7 +3607,7 @@ static int adv7842_remove(struct i2c_client *client)
 
 /* ----------------------------------------------------------------------- */
 
-static struct i2c_device_id adv7842_id[] = {
+static const struct i2c_device_id adv7842_id[] = {
 	{ "adv7842", 0 },
 	{ }
 };
diff --git a/drivers/media/i2c/dw9714.c b/drivers/media/i2c/dw9714.c
index 6a607d7f82de..95af4fc99cd0 100644
--- a/drivers/media/i2c/dw9714.c
+++ b/drivers/media/i2c/dw9714.c
@@ -11,7 +11,6 @@
  * GNU General Public License for more details.
  */
 
-#include <linux/acpi.h>
 #include <linux/delay.h>
 #include <linux/i2c.h>
 #include <linux/module.h>
@@ -147,8 +146,7 @@ static int dw9714_init_controls(struct dw9714_device *dev_vcm)
 	return hdl->error;
 }
 
-static int dw9714_probe(struct i2c_client *client,
-			const struct i2c_device_id *devid)
+static int dw9714_probe(struct i2c_client *client)
 {
 	struct dw9714_device *dw9714_dev;
 	int rval;
@@ -250,20 +248,18 @@ static int  __maybe_unused dw9714_vcm_resume(struct device *dev)
 	return 0;
 }
 
-#ifdef CONFIG_ACPI
-static const struct acpi_device_id dw9714_acpi_match[] = {
-	{},
-};
-MODULE_DEVICE_TABLE(acpi, dw9714_acpi_match);
-#endif
-
 static const struct i2c_device_id dw9714_id_table[] = {
-	{DW9714_NAME, 0},
-	{}
+	{ DW9714_NAME, 0 },
+	{ { 0 } }
 };
-
 MODULE_DEVICE_TABLE(i2c, dw9714_id_table);
 
+static const struct of_device_id dw9714_of_table[] = {
+	{ .compatible = "dongwoon,dw9714" },
+	{ { 0 } }
+};
+MODULE_DEVICE_TABLE(of, dw9714_of_table);
+
 static const struct dev_pm_ops dw9714_pm_ops = {
 	SET_SYSTEM_SLEEP_PM_OPS(dw9714_vcm_suspend, dw9714_vcm_resume)
 	SET_RUNTIME_PM_OPS(dw9714_vcm_suspend, dw9714_vcm_resume, NULL)
@@ -273,9 +269,9 @@ static struct i2c_driver dw9714_i2c_driver = {
 	.driver = {
 		.name = DW9714_NAME,
 		.pm = &dw9714_pm_ops,
-		.acpi_match_table = ACPI_PTR(dw9714_acpi_match),
+		.of_match_table = dw9714_of_table,
 	},
-	.probe = dw9714_probe,
+	.probe_new = dw9714_probe,
 	.remove = dw9714_remove,
 	.id_table = dw9714_id_table,
 };
diff --git a/drivers/media/i2c/et8ek8/et8ek8_driver.c b/drivers/media/i2c/et8ek8/et8ek8_driver.c
index f39f5179dd95..c14f0fd6ded3 100644
--- a/drivers/media/i2c/et8ek8/et8ek8_driver.c
+++ b/drivers/media/i2c/et8ek8/et8ek8_driver.c
@@ -43,7 +43,7 @@
 
 #define ET8EK8_NAME		"et8ek8"
 #define ET8EK8_PRIV_MEM_SIZE	128
-#define ET8EK8_MAX_MSG		48
+#define ET8EK8_MAX_MSG		8
 
 struct et8ek8_sensor {
 	struct v4l2_subdev subdev;
@@ -220,7 +220,8 @@ static void et8ek8_i2c_create_msg(struct i2c_client *client, u16 len, u16 reg,
 
 /*
  * A buffered write method that puts the wanted register write
- * commands in a message list and passes the list to the i2c framework
+ * commands in smaller number of message lists and passes the lists to
+ * the i2c framework
  */
 static int et8ek8_i2c_buffered_write_regs(struct i2c_client *client,
 					  const struct et8ek8_reg *wnext,
@@ -231,11 +232,7 @@ static int et8ek8_i2c_buffered_write_regs(struct i2c_client *client,
 	int wcnt = 0;
 	u16 reg, data_length;
 	u32 val;
-
-	if (WARN_ONCE(cnt > ET8EK8_MAX_MSG,
-		      ET8EK8_NAME ": %s: too many messages.\n", __func__)) {
-		return -EINVAL;
-	}
+	int rval;
 
 	/* Create new write messages for all writes */
 	while (wcnt < cnt) {
@@ -249,10 +246,21 @@ static int et8ek8_i2c_buffered_write_regs(struct i2c_client *client,
 
 		/* Update write count */
 		wcnt++;
+
+		if (wcnt < ET8EK8_MAX_MSG)
+			continue;
+
+		rval = i2c_transfer(client->adapter, msg, wcnt);
+		if (rval < 0)
+			return rval;
+
+		cnt -= wcnt;
+		wcnt = 0;
 	}
 
-	/* Now we send everything ... */
-	return i2c_transfer(client->adapter, msg, wcnt);
+	rval = i2c_transfer(client->adapter, msg, wcnt);
+
+	return rval < 0 ? rval : 0;
 }
 
 /*
diff --git a/drivers/media/i2c/ir-kbd-i2c.c b/drivers/media/i2c/ir-kbd-i2c.c
index cee7fd9cf08b..a374e2a0ac3d 100644
--- a/drivers/media/i2c/ir-kbd-i2c.c
+++ b/drivers/media/i2c/ir-kbd-i2c.c
@@ -59,8 +59,8 @@ module_param(debug, int, 0644);    /* debug level (0,1,2) */
 
 /* ----------------------------------------------------------------------- */
 
-static int get_key_haup_common(struct IR_i2c *ir, enum rc_type *protocol,
-					u32 *scancode, u8 *ptoggle, int size)
+static int get_key_haup_common(struct IR_i2c *ir, enum rc_proto *protocol,
+			       u32 *scancode, u8 *ptoggle, int size)
 {
 	unsigned char buf[6];
 	int start, range, toggle, dev, code, ircode, vendor;
@@ -99,7 +99,7 @@ static int get_key_haup_common(struct IR_i2c *ir, enum rc_type *protocol,
 		dprintk(1, "ir hauppauge (rc5): s%d r%d t%d dev=%d code=%d\n",
 			start, range, toggle, dev, code);
 
-		*protocol = RC_TYPE_RC5;
+		*protocol = RC_PROTO_RC5;
 		*scancode = RC_SCANCODE_RC5(dev, code);
 		*ptoggle = toggle;
 
@@ -111,13 +111,13 @@ static int get_key_haup_common(struct IR_i2c *ir, enum rc_type *protocol,
 
 		if (vendor == 0x800f) {
 			*ptoggle = (dev & 0x80) != 0;
-			*protocol = RC_TYPE_RC6_MCE;
+			*protocol = RC_PROTO_RC6_MCE;
 			dev &= 0x7f;
 			dprintk(1, "ir hauppauge (rc6-mce): t%d vendor=%d dev=%d code=%d\n",
 						*ptoggle, vendor, dev, code);
 		} else {
 			*ptoggle = 0;
-			*protocol = RC_TYPE_RC6_6A_32;
+			*protocol = RC_PROTO_RC6_6A_32;
 			dprintk(1, "ir hauppauge (rc6-6a-32): vendor=%d dev=%d code=%d\n",
 							vendor, dev, code);
 		}
@@ -130,13 +130,13 @@ static int get_key_haup_common(struct IR_i2c *ir, enum rc_type *protocol,
 	return 0;
 }
 
-static int get_key_haup(struct IR_i2c *ir, enum rc_type *protocol,
+static int get_key_haup(struct IR_i2c *ir, enum rc_proto *protocol,
 			u32 *scancode, u8 *toggle)
 {
 	return get_key_haup_common(ir, protocol, scancode, toggle, 3);
 }
 
-static int get_key_haup_xvr(struct IR_i2c *ir, enum rc_type *protocol,
+static int get_key_haup_xvr(struct IR_i2c *ir, enum rc_proto *protocol,
 			    u32 *scancode, u8 *toggle)
 {
 	int ret;
@@ -155,7 +155,7 @@ static int get_key_haup_xvr(struct IR_i2c *ir, enum rc_type *protocol,
 	return get_key_haup_common(ir, protocol, scancode, toggle, 6);
 }
 
-static int get_key_pixelview(struct IR_i2c *ir, enum rc_type *protocol,
+static int get_key_pixelview(struct IR_i2c *ir, enum rc_proto *protocol,
 			     u32 *scancode, u8 *toggle)
 {
 	unsigned char b;
@@ -166,13 +166,13 @@ static int get_key_pixelview(struct IR_i2c *ir, enum rc_type *protocol,
 		return -EIO;
 	}
 
-	*protocol = RC_TYPE_OTHER;
+	*protocol = RC_PROTO_OTHER;
 	*scancode = b;
 	*toggle = 0;
 	return 1;
 }
 
-static int get_key_fusionhdtv(struct IR_i2c *ir, enum rc_type *protocol,
+static int get_key_fusionhdtv(struct IR_i2c *ir, enum rc_proto *protocol,
 			      u32 *scancode, u8 *toggle)
 {
 	unsigned char buf[4];
@@ -191,13 +191,13 @@ static int get_key_fusionhdtv(struct IR_i2c *ir, enum rc_type *protocol,
 	if(buf[0] != 0x1 ||  buf[1] != 0xfe)
 		return 0;
 
-	*protocol = RC_TYPE_UNKNOWN;
+	*protocol = RC_PROTO_UNKNOWN;
 	*scancode = buf[2];
 	*toggle = 0;
 	return 1;
 }
 
-static int get_key_knc1(struct IR_i2c *ir, enum rc_type *protocol,
+static int get_key_knc1(struct IR_i2c *ir, enum rc_proto *protocol,
 			u32 *scancode, u8 *toggle)
 {
 	unsigned char b;
@@ -221,13 +221,13 @@ static int get_key_knc1(struct IR_i2c *ir, enum rc_type *protocol,
 		/* keep old data */
 		return 1;
 
-	*protocol = RC_TYPE_UNKNOWN;
+	*protocol = RC_PROTO_UNKNOWN;
 	*scancode = b;
 	*toggle = 0;
 	return 1;
 }
 
-static int get_key_avermedia_cardbus(struct IR_i2c *ir, enum rc_type *protocol,
+static int get_key_avermedia_cardbus(struct IR_i2c *ir, enum rc_proto *protocol,
 				     u32 *scancode, u8 *toggle)
 {
 	unsigned char subaddr, key, keygroup;
@@ -262,7 +262,7 @@ static int get_key_avermedia_cardbus(struct IR_i2c *ir, enum rc_type *protocol,
 	}
 	key |= (keygroup & 1) << 6;
 
-	*protocol = RC_TYPE_UNKNOWN;
+	*protocol = RC_PROTO_UNKNOWN;
 	*scancode = key;
 	if (ir->c->addr == 0x41) /* AVerMedia EM78P153 */
 		*scancode |= keygroup << 8;
@@ -274,7 +274,7 @@ static int get_key_avermedia_cardbus(struct IR_i2c *ir, enum rc_type *protocol,
 
 static int ir_key_poll(struct IR_i2c *ir)
 {
-	enum rc_type protocol;
+	enum rc_proto protocol;
 	u32 scancode;
 	u8 toggle;
 	int rc;
@@ -315,7 +315,7 @@ static int ir_probe(struct i2c_client *client, const struct i2c_device_id *id)
 {
 	char *ir_codes = NULL;
 	const char *name = NULL;
-	u64 rc_type = RC_BIT_UNKNOWN;
+	u64 rc_proto = RC_PROTO_BIT_UNKNOWN;
 	struct IR_i2c *ir;
 	struct rc_dev *rc = NULL;
 	struct i2c_adapter *adap = client->adapter;
@@ -334,7 +334,7 @@ static int ir_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	case 0x64:
 		name        = "Pixelview";
 		ir->get_key = get_key_pixelview;
-		rc_type     = RC_BIT_OTHER;
+		rc_proto    = RC_PROTO_BIT_OTHER;
 		ir_codes    = RC_MAP_EMPTY;
 		break;
 	case 0x18:
@@ -342,38 +342,39 @@ static int ir_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	case 0x1a:
 		name        = "Hauppauge";
 		ir->get_key = get_key_haup;
-		rc_type     = RC_BIT_RC5;
+		rc_proto    = RC_PROTO_BIT_RC5;
 		ir_codes    = RC_MAP_HAUPPAUGE;
 		break;
 	case 0x30:
 		name        = "KNC One";
 		ir->get_key = get_key_knc1;
-		rc_type     = RC_BIT_OTHER;
+		rc_proto    = RC_PROTO_BIT_OTHER;
 		ir_codes    = RC_MAP_EMPTY;
 		break;
 	case 0x6b:
 		name        = "FusionHDTV";
 		ir->get_key = get_key_fusionhdtv;
-		rc_type     = RC_BIT_UNKNOWN;
+		rc_proto    = RC_PROTO_BIT_UNKNOWN;
 		ir_codes    = RC_MAP_FUSIONHDTV_MCE;
 		break;
 	case 0x40:
 		name        = "AVerMedia Cardbus remote";
 		ir->get_key = get_key_avermedia_cardbus;
-		rc_type     = RC_BIT_OTHER;
+		rc_proto    = RC_PROTO_BIT_OTHER;
 		ir_codes    = RC_MAP_AVERMEDIA_CARDBUS;
 		break;
 	case 0x41:
 		name        = "AVerMedia EM78P153";
 		ir->get_key = get_key_avermedia_cardbus;
-		rc_type     = RC_BIT_OTHER;
+		rc_proto    = RC_PROTO_BIT_OTHER;
 		/* RM-KV remote, seems to be same as RM-K6 */
 		ir_codes    = RC_MAP_AVERMEDIA_M733A_RM_K6;
 		break;
 	case 0x71:
 		name        = "Hauppauge/Zilog Z8";
 		ir->get_key = get_key_haup_xvr;
-		rc_type     = RC_BIT_RC5 | RC_BIT_RC6_MCE | RC_BIT_RC6_6A_32;
+		rc_proto    = RC_PROTO_BIT_RC5 | RC_PROTO_BIT_RC6_MCE |
+							RC_PROTO_BIT_RC6_6A_32;
 		ir_codes    = RC_MAP_HAUPPAUGE;
 		break;
 	}
@@ -388,7 +389,7 @@ static int ir_probe(struct i2c_client *client, const struct i2c_device_id *id)
 
 		name = init_data->name;
 		if (init_data->type)
-			rc_type = init_data->type;
+			rc_proto = init_data->type;
 
 		if (init_data->polling_interval)
 			ir->polling_interval = init_data->polling_interval;
@@ -431,7 +432,7 @@ static int ir_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	ir->rc = rc;
 
 	/* Make sure we are all setup before going on */
-	if (!name || !ir->get_key || !rc_type || !ir_codes) {
+	if (!name || !ir->get_key || !rc_proto || !ir_codes) {
 		dprintk(1, ": Unsupported device at address 0x%02x\n",
 			addr);
 		err = -ENODEV;
@@ -452,14 +453,14 @@ static int ir_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	 */
 	rc->input_id.bustype = BUS_I2C;
 	rc->input_phys       = ir->phys;
-	rc->input_name	     = ir->name;
+	rc->device_name	     = ir->name;
 
 	/*
 	 * Initialize the other fields of rc_dev
 	 */
 	rc->map_name       = ir->ir_codes;
-	rc->allowed_protocols = rc_type;
-	rc->enabled_protocols = rc_type;
+	rc->allowed_protocols = rc_proto;
+	rc->enabled_protocols = rc_proto;
 	if (!rc->driver_name)
 		rc->driver_name = MODULE_NAME;
 
diff --git a/drivers/media/i2c/m5mols/m5mols_core.c b/drivers/media/i2c/m5mols/m5mols_core.c
index 9ccb5ee55fa9..463534d44756 100644
--- a/drivers/media/i2c/m5mols/m5mols_core.c
+++ b/drivers/media/i2c/m5mols/m5mols_core.c
@@ -457,7 +457,7 @@ static int m5mols_get_version(struct v4l2_subdev *sd)
 
 	v4l2_info(sd, "Manufacturer\t[%s]\n",
 			is_manufacturer(info, REG_SAMSUNG_ELECTRO) ?
-			"Samsung Electro-Machanics" :
+			"Samsung Electro-Mechanics" :
 			is_manufacturer(info, REG_SAMSUNG_OPTICS) ?
 			"Samsung Fiber-Optics" :
 			is_manufacturer(info, REG_SAMSUNG_TECHWIN) ?
diff --git a/drivers/media/i2c/max2175.c b/drivers/media/i2c/max2175.c
index a4736a8a7792..bf0e821a2b93 100644
--- a/drivers/media/i2c/max2175.c
+++ b/drivers/media/i2c/max2175.c
@@ -1319,7 +1319,7 @@ static int max2175_probe(struct i2c_client *client,
 	if (IS_ERR(clk)) {
 		ret = PTR_ERR(clk);
 		dev_err(&client->dev, "cannot get clock %d\n", ret);
-		return -ENODEV;
+		return ret;
 	}
 
 	regmap = devm_regmap_init_i2c(client, &max2175_regmap_config);
diff --git a/drivers/media/i2c/mt9m111.c b/drivers/media/i2c/mt9m111.c
index 72e71b762827..99b992e46702 100644
--- a/drivers/media/i2c/mt9m111.c
+++ b/drivers/media/i2c/mt9m111.c
@@ -835,7 +835,7 @@ static const struct v4l2_ctrl_ops mt9m111_ctrl_ops = {
 	.s_ctrl = mt9m111_s_ctrl,
 };
 
-static struct v4l2_subdev_core_ops mt9m111_subdev_core_ops = {
+static const struct v4l2_subdev_core_ops mt9m111_subdev_core_ops = {
 	.s_power	= mt9m111_s_power,
 #ifdef CONFIG_VIDEO_ADV_DEBUG
 	.g_register	= mt9m111_g_register,
@@ -865,7 +865,7 @@ static int mt9m111_g_mbus_config(struct v4l2_subdev *sd,
 	return 0;
 }
 
-static struct v4l2_subdev_video_ops mt9m111_subdev_video_ops = {
+static const struct v4l2_subdev_video_ops mt9m111_subdev_video_ops = {
 	.g_mbus_config	= mt9m111_g_mbus_config,
 };
 
@@ -877,7 +877,7 @@ static const struct v4l2_subdev_pad_ops mt9m111_subdev_pad_ops = {
 	.set_fmt	= mt9m111_set_fmt,
 };
 
-static struct v4l2_subdev_ops mt9m111_subdev_ops = {
+static const struct v4l2_subdev_ops mt9m111_subdev_ops = {
 	.core	= &mt9m111_subdev_core_ops,
 	.video	= &mt9m111_subdev_video_ops,
 	.pad	= &mt9m111_subdev_pad_ops,
diff --git a/drivers/media/i2c/mt9t001.c b/drivers/media/i2c/mt9t001.c
index 842017fa4aab..9d981d9f5686 100644
--- a/drivers/media/i2c/mt9t001.c
+++ b/drivers/media/i2c/mt9t001.c
@@ -822,15 +822,15 @@ static int mt9t001_close(struct v4l2_subdev *subdev, struct v4l2_subdev_fh *fh)
 	return mt9t001_set_power(subdev, 0);
 }
 
-static struct v4l2_subdev_core_ops mt9t001_subdev_core_ops = {
+static const struct v4l2_subdev_core_ops mt9t001_subdev_core_ops = {
 	.s_power = mt9t001_set_power,
 };
 
-static struct v4l2_subdev_video_ops mt9t001_subdev_video_ops = {
+static const struct v4l2_subdev_video_ops mt9t001_subdev_video_ops = {
 	.s_stream = mt9t001_s_stream,
 };
 
-static struct v4l2_subdev_pad_ops mt9t001_subdev_pad_ops = {
+static const struct v4l2_subdev_pad_ops mt9t001_subdev_pad_ops = {
 	.enum_mbus_code = mt9t001_enum_mbus_code,
 	.enum_frame_size = mt9t001_enum_frame_size,
 	.get_fmt = mt9t001_get_format,
@@ -839,7 +839,7 @@ static struct v4l2_subdev_pad_ops mt9t001_subdev_pad_ops = {
 	.set_selection = mt9t001_set_selection,
 };
 
-static struct v4l2_subdev_ops mt9t001_subdev_ops = {
+static const struct v4l2_subdev_ops mt9t001_subdev_ops = {
 	.core = &mt9t001_subdev_core_ops,
 	.video = &mt9t001_subdev_video_ops,
 	.pad = &mt9t001_subdev_pad_ops,
diff --git a/drivers/media/i2c/ov13858.c b/drivers/media/i2c/ov13858.c
index 86550d8ddfee..af7af0d14c69 100644
--- a/drivers/media/i2c/ov13858.c
+++ b/drivers/media/i2c/ov13858.c
@@ -57,16 +57,14 @@
 #define OV13858_VTS_30FPS		0x0c8e /* 30 fps */
 #define OV13858_VTS_60FPS		0x0648 /* 60 fps */
 #define OV13858_VTS_MAX			0x7fff
-#define OV13858_VBLANK_MIN		56
 
 /* HBLANK control - read only */
-#define OV13858_PPL_540MHZ		2244
-#define OV13858_PPL_1080MHZ		4488
+#define OV13858_PPL_270MHZ		2244
+#define OV13858_PPL_540MHZ		4488
 
 /* Exposure control */
 #define OV13858_REG_EXPOSURE		0x3500
 #define OV13858_EXPOSURE_MIN		4
-#define OV13858_EXPOSURE_MAX		(OV13858_VTS_MAX - 8)
 #define OV13858_EXPOSURE_STEP		1
 #define OV13858_EXPOSURE_DEFAULT	0x640
 
@@ -78,13 +76,13 @@
 #define OV13858_ANA_GAIN_DEFAULT	0x80
 
 /* Digital gain control */
-#define OV13858_REG_DIGITAL_GAIN	0x350a
-#define OV13858_DGTL_GAIN_MASK		0xf3
-#define OV13858_DGTL_GAIN_SHIFT		2
-#define OV13858_DGTL_GAIN_MIN		1
-#define OV13858_DGTL_GAIN_MAX		4
-#define OV13858_DGTL_GAIN_STEP		1
-#define OV13858_DGTL_GAIN_DEFAULT	1
+#define OV13858_REG_B_MWB_GAIN		0x5100
+#define OV13858_REG_G_MWB_GAIN		0x5102
+#define OV13858_REG_R_MWB_GAIN		0x5104
+#define OV13858_DGTL_GAIN_MIN		0
+#define OV13858_DGTL_GAIN_MAX		16384	/* Max = 16 X */
+#define OV13858_DGTL_GAIN_DEFAULT	1024	/* Default gain = 1 X */
+#define OV13858_DGTL_GAIN_STEP		1	/* Each step = 1/1024 */
 
 /* Test Pattern Control */
 #define OV13858_REG_TEST_PATTERN	0x4503
@@ -121,7 +119,8 @@ struct ov13858_mode {
 	u32 height;
 
 	/* V-timing */
-	u32 vts;
+	u32 vts_def;
+	u32 vts_min;
 
 	/* Index of Link frequency config to be used */
 	u32 link_freq_index;
@@ -944,31 +943,33 @@ static const char * const ov13858_test_pattern_menu[] = {
 
 /* Configurations for supported link frequencies */
 #define OV13858_NUM_OF_LINK_FREQS	2
-#define OV13858_LINK_FREQ_1080MBPS	1080000000
-#define OV13858_LINK_FREQ_540MBPS	540000000
+#define OV13858_LINK_FREQ_540MHZ	540000000ULL
+#define OV13858_LINK_FREQ_270MHZ	270000000ULL
 #define OV13858_LINK_FREQ_INDEX_0	0
 #define OV13858_LINK_FREQ_INDEX_1	1
 
 /* Menu items for LINK_FREQ V4L2 control */
 static const s64 link_freq_menu_items[OV13858_NUM_OF_LINK_FREQS] = {
-	OV13858_LINK_FREQ_1080MBPS,
-	OV13858_LINK_FREQ_540MBPS
+	OV13858_LINK_FREQ_540MHZ,
+	OV13858_LINK_FREQ_270MHZ
 };
 
 /* Link frequency configs */
 static const struct ov13858_link_freq_config
 			link_freq_configs[OV13858_NUM_OF_LINK_FREQS] = {
 	{
-		.pixel_rate = 864000000,
-		.pixels_per_line = OV13858_PPL_1080MHZ,
+		/* pixel_rate = link_freq * 2 * nr_of_lanes / bits_per_sample */
+		.pixel_rate = (OV13858_LINK_FREQ_540MHZ * 2 * 4) / 10,
+		.pixels_per_line = OV13858_PPL_540MHZ,
 		.reg_list = {
 			.num_of_regs = ARRAY_SIZE(mipi_data_rate_1080mbps),
 			.regs = mipi_data_rate_1080mbps,
 		}
 	},
 	{
-		.pixel_rate = 432000000,
-		.pixels_per_line = OV13858_PPL_540MHZ,
+		/* pixel_rate = link_freq * 2 * nr_of_lanes / bits_per_sample */
+		.pixel_rate = (OV13858_LINK_FREQ_270MHZ * 2 * 4) / 10,
+		.pixels_per_line = OV13858_PPL_270MHZ,
 		.reg_list = {
 			.num_of_regs = ARRAY_SIZE(mipi_data_rate_540mbps),
 			.regs = mipi_data_rate_540mbps,
@@ -981,7 +982,8 @@ static const struct ov13858_mode supported_modes[] = {
 	{
 		.width = 4224,
 		.height = 3136,
-		.vts = OV13858_VTS_30FPS,
+		.vts_def = OV13858_VTS_30FPS,
+		.vts_min = OV13858_VTS_30FPS,
 		.reg_list = {
 			.num_of_regs = ARRAY_SIZE(mode_4224x3136_regs),
 			.regs = mode_4224x3136_regs,
@@ -991,7 +993,8 @@ static const struct ov13858_mode supported_modes[] = {
 	{
 		.width = 2112,
 		.height = 1568,
-		.vts = OV13858_VTS_30FPS,
+		.vts_def = OV13858_VTS_30FPS,
+		.vts_min = 1608,
 		.reg_list = {
 			.num_of_regs = ARRAY_SIZE(mode_2112x1568_regs),
 			.regs = mode_2112x1568_regs,
@@ -1001,7 +1004,8 @@ static const struct ov13858_mode supported_modes[] = {
 	{
 		.width = 2112,
 		.height = 1188,
-		.vts = OV13858_VTS_30FPS,
+		.vts_def = OV13858_VTS_30FPS,
+		.vts_min = 1608,
 		.reg_list = {
 			.num_of_regs = ARRAY_SIZE(mode_2112x1188_regs),
 			.regs = mode_2112x1188_regs,
@@ -1011,7 +1015,8 @@ static const struct ov13858_mode supported_modes[] = {
 	{
 		.width = 1056,
 		.height = 784,
-		.vts = OV13858_VTS_30FPS,
+		.vts_def = OV13858_VTS_30FPS,
+		.vts_min = 804,
 		.reg_list = {
 			.num_of_regs = ARRAY_SIZE(mode_1056x784_regs),
 			.regs = mode_1056x784_regs,
@@ -1161,21 +1166,21 @@ static int ov13858_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
 static int ov13858_update_digital_gain(struct ov13858 *ov13858, u32 d_gain)
 {
 	int ret;
-	u32 val;
 
-	if (d_gain == 3)
-		return -EINVAL;
+	ret = ov13858_write_reg(ov13858, OV13858_REG_B_MWB_GAIN,
+				OV13858_REG_VALUE_16BIT, d_gain);
+	if (ret)
+		return ret;
 
-	ret = ov13858_read_reg(ov13858, OV13858_REG_DIGITAL_GAIN,
-			       OV13858_REG_VALUE_08BIT, &val);
+	ret = ov13858_write_reg(ov13858, OV13858_REG_G_MWB_GAIN,
+				OV13858_REG_VALUE_16BIT, d_gain);
 	if (ret)
 		return ret;
 
-	val &= OV13858_DGTL_GAIN_MASK;
-	val |= (d_gain - 1) << OV13858_DGTL_GAIN_SHIFT;
+	ret = ov13858_write_reg(ov13858, OV13858_REG_R_MWB_GAIN,
+				OV13858_REG_VALUE_16BIT, d_gain);
 
-	return ov13858_write_reg(ov13858, OV13858_REG_DIGITAL_GAIN,
-				 OV13858_REG_VALUE_08BIT, val);
+	return ret;
 }
 
 static int ov13858_enable_test_pattern(struct ov13858 *ov13858, u32 pattern)
@@ -1377,6 +1382,8 @@ ov13858_set_pad_format(struct v4l2_subdev *sd,
 	struct ov13858 *ov13858 = to_ov13858(sd);
 	const struct ov13858_mode *mode;
 	struct v4l2_mbus_framefmt *framefmt;
+	s32 vblank_def;
+	s32 vblank_min;
 	s64 h_blank;
 
 	mutex_lock(&ov13858->mutex);
@@ -1397,10 +1404,15 @@ ov13858_set_pad_format(struct v4l2_subdev *sd,
 			ov13858->pixel_rate,
 			link_freq_configs[mode->link_freq_index].pixel_rate);
 		/* Update limits and set FPS to default */
+		vblank_def = ov13858->cur_mode->vts_def -
+			     ov13858->cur_mode->height;
+		vblank_min = ov13858->cur_mode->vts_min -
+			     ov13858->cur_mode->height;
 		__v4l2_ctrl_modify_range(
-			ov13858->vblank, OV13858_VBLANK_MIN,
+			ov13858->vblank, vblank_min,
 			OV13858_VTS_MAX - ov13858->cur_mode->height, 1,
-			ov13858->cur_mode->vts - ov13858->cur_mode->height);
+			vblank_def);
+		__v4l2_ctrl_s_ctrl(ov13858->vblank, vblank_def);
 		h_blank =
 			link_freq_configs[mode->link_freq_index].pixels_per_line
 			 - ov13858->cur_mode->width;
@@ -1602,6 +1614,9 @@ static int ov13858_init_controls(struct ov13858 *ov13858)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(&ov13858->sd);
 	struct v4l2_ctrl_handler *ctrl_hdlr;
+	s64 exposure_max;
+	s64 vblank_def;
+	s64 vblank_min;
 	int ret;
 
 	ctrl_hdlr = &ov13858->ctrl_handler;
@@ -1625,25 +1640,27 @@ static int ov13858_init_controls(struct ov13858 *ov13858)
 					link_freq_configs[0].pixel_rate, 1,
 					link_freq_configs[0].pixel_rate);
 
+	vblank_def = ov13858->cur_mode->vts_def - ov13858->cur_mode->height;
+	vblank_min = ov13858->cur_mode->vts_min - ov13858->cur_mode->height;
 	ov13858->vblank = v4l2_ctrl_new_std(
 				ctrl_hdlr, &ov13858_ctrl_ops, V4L2_CID_VBLANK,
-				OV13858_VBLANK_MIN,
+				vblank_min,
 				OV13858_VTS_MAX - ov13858->cur_mode->height, 1,
-				ov13858->cur_mode->vts
-				  - ov13858->cur_mode->height);
+				vblank_def);
 
 	ov13858->hblank = v4l2_ctrl_new_std(
 				ctrl_hdlr, &ov13858_ctrl_ops, V4L2_CID_HBLANK,
-				OV13858_PPL_1080MHZ - ov13858->cur_mode->width,
-				OV13858_PPL_1080MHZ - ov13858->cur_mode->width,
+				OV13858_PPL_540MHZ - ov13858->cur_mode->width,
+				OV13858_PPL_540MHZ - ov13858->cur_mode->width,
 				1,
-				OV13858_PPL_1080MHZ - ov13858->cur_mode->width);
+				OV13858_PPL_540MHZ - ov13858->cur_mode->width);
 	ov13858->hblank->flags |= V4L2_CTRL_FLAG_READ_ONLY;
 
+	exposure_max = ov13858->cur_mode->vts_def - 8;
 	ov13858->exposure = v4l2_ctrl_new_std(
 				ctrl_hdlr, &ov13858_ctrl_ops,
 				V4L2_CID_EXPOSURE, OV13858_EXPOSURE_MIN,
-				OV13858_EXPOSURE_MAX, OV13858_EXPOSURE_STEP,
+				exposure_max, OV13858_EXPOSURE_STEP,
 				OV13858_EXPOSURE_DEFAULT);
 
 	v4l2_ctrl_new_std(ctrl_hdlr, &ov13858_ctrl_ops, V4L2_CID_ANALOGUE_GAIN,
diff --git a/drivers/media/i2c/ov5640.c b/drivers/media/i2c/ov5640.c
index 1f5b483cf334..39a2269c0bee 100644
--- a/drivers/media/i2c/ov5640.c
+++ b/drivers/media/i2c/ov5640.c
@@ -1524,8 +1524,7 @@ static int ov5640_restore_mode(struct ov5640_dev *sensor)
 
 static void ov5640_power(struct ov5640_dev *sensor, bool enable)
 {
-	if (sensor->pwdn_gpio)
-		gpiod_set_value(sensor->pwdn_gpio, enable ? 0 : 1);
+	gpiod_set_value(sensor->pwdn_gpio, enable ? 0 : 1);
 }
 
 static void ov5640_reset(struct ov5640_dev *sensor)
diff --git a/drivers/media/i2c/ov5645.c b/drivers/media/i2c/ov5645.c
index d1e844f7f03f..d28845f7356f 100644
--- a/drivers/media/i2c/ov5645.c
+++ b/drivers/media/i2c/ov5645.c
@@ -80,6 +80,8 @@ struct ov5645_mode_info {
 	u32 height;
 	const struct reg_value *data;
 	u32 data_size;
+	u32 pixel_clock;
+	u32 link_freq;
 };
 
 struct ov5645 {
@@ -99,6 +101,8 @@ struct ov5645 {
 	const struct ov5645_mode_info *current_mode;
 
 	struct v4l2_ctrl_handler ctrls;
+	struct v4l2_ctrl *pixel_clock;
+	struct v4l2_ctrl *link_freq;
 
 	/* Cached register values */
 	u8 aec_pk_manual;
@@ -505,24 +509,35 @@ static const struct reg_value ov5645_setting_full[] = {
 	{ 0x4202, 0x00 }
 };
 
+static const s64 link_freq[] = {
+	222880000,
+	334320000
+};
+
 static const struct ov5645_mode_info ov5645_mode_info_data[] = {
 	{
 		.width = 1280,
 		.height = 960,
 		.data = ov5645_setting_sxga,
-		.data_size = ARRAY_SIZE(ov5645_setting_sxga)
+		.data_size = ARRAY_SIZE(ov5645_setting_sxga),
+		.pixel_clock = 111440000,
+		.link_freq = 0 /* an index in link_freq[] */
 	},
 	{
 		.width = 1920,
 		.height = 1080,
 		.data = ov5645_setting_1080p,
-		.data_size = ARRAY_SIZE(ov5645_setting_1080p)
+		.data_size = ARRAY_SIZE(ov5645_setting_1080p),
+		.pixel_clock = 167160000,
+		.link_freq = 1 /* an index in link_freq[] */
 	},
 	{
 		.width = 2592,
 		.height = 1944,
 		.data = ov5645_setting_full,
-		.data_size = ARRAY_SIZE(ov5645_setting_full)
+		.data_size = ARRAY_SIZE(ov5645_setting_full),
+		.pixel_clock = 167160000,
+		.link_freq = 1 /* an index in link_freq[] */
 	},
 };
 
@@ -969,6 +984,7 @@ static int ov5645_set_format(struct v4l2_subdev *sd,
 	struct v4l2_mbus_framefmt *__format;
 	struct v4l2_rect *__crop;
 	const struct ov5645_mode_info *new_mode;
+	int ret;
 
 	__crop = __ov5645_get_pad_crop(ov5645, cfg, format->pad,
 			format->which);
@@ -978,8 +994,19 @@ static int ov5645_set_format(struct v4l2_subdev *sd,
 	__crop->width = new_mode->width;
 	__crop->height = new_mode->height;
 
-	if (format->which == V4L2_SUBDEV_FORMAT_ACTIVE)
+	if (format->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
+		ret = v4l2_ctrl_s_ctrl_int64(ov5645->pixel_clock,
+					     new_mode->pixel_clock);
+		if (ret < 0)
+			return ret;
+
+		ret = v4l2_ctrl_s_ctrl(ov5645->link_freq,
+				       new_mode->link_freq);
+		if (ret < 0)
+			return ret;
+
 		ov5645->current_mode = new_mode;
+	}
 
 	__format = __ov5645_get_pad_format(ov5645, cfg, format->pad,
 			format->which);
@@ -1197,7 +1224,7 @@ static int ov5645_probe(struct i2c_client *client,
 
 	mutex_init(&ov5645->power_lock);
 
-	v4l2_ctrl_handler_init(&ov5645->ctrls, 7);
+	v4l2_ctrl_handler_init(&ov5645->ctrls, 9);
 	v4l2_ctrl_new_std(&ov5645->ctrls, &ov5645_ctrl_ops,
 			  V4L2_CID_SATURATION, -4, 4, 1, 0);
 	v4l2_ctrl_new_std(&ov5645->ctrls, &ov5645_ctrl_ops,
@@ -1215,6 +1242,17 @@ static int ov5645_probe(struct i2c_client *client,
 				     V4L2_CID_TEST_PATTERN,
 				     ARRAY_SIZE(ov5645_test_pattern_menu) - 1,
 				     0, 0, ov5645_test_pattern_menu);
+	ov5645->pixel_clock = v4l2_ctrl_new_std(&ov5645->ctrls,
+						&ov5645_ctrl_ops,
+						V4L2_CID_PIXEL_RATE,
+						1, INT_MAX, 1, 1);
+	ov5645->link_freq = v4l2_ctrl_new_int_menu(&ov5645->ctrls,
+						   &ov5645_ctrl_ops,
+						   V4L2_CID_LINK_FREQ,
+						   ARRAY_SIZE(link_freq) - 1,
+						   0, link_freq);
+	if (ov5645->link_freq)
+		ov5645->link_freq->flags |= V4L2_CTRL_FLAG_READ_ONLY;
 
 	ov5645->sd.ctrl_handler = &ov5645->ctrls;
 
@@ -1229,6 +1267,7 @@ static int ov5645_probe(struct i2c_client *client,
 	ov5645->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
 	ov5645->pad.flags = MEDIA_PAD_FL_SOURCE;
 	ov5645->sd.dev = &client->dev;
+	ov5645->sd.entity.function = MEDIA_ENT_F_CAM_SENSOR;
 
 	ret = media_entity_pads_init(&ov5645->sd.entity, 1, &ov5645->pad);
 	if (ret < 0) {
diff --git a/drivers/media/i2c/ov5670.c b/drivers/media/i2c/ov5670.c
new file mode 100644
index 000000000000..6f7a1d6d2200
--- /dev/null
+++ b/drivers/media/i2c/ov5670.c
@@ -0,0 +1,2601 @@
+/*
+ * Copyright (c) 2017 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version
+ * 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/acpi.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/pm_runtime.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+
+#define OV5670_REG_CHIP_ID		0x300a
+#define OV5670_CHIP_ID			0x005670
+
+#define OV5670_REG_MODE_SELECT		0x0100
+#define OV5670_MODE_STANDBY		0x00
+#define OV5670_MODE_STREAMING		0x01
+
+#define OV5670_REG_SOFTWARE_RST		0x0103
+#define OV5670_SOFTWARE_RST		0x01
+
+/* vertical-timings from sensor */
+#define OV5670_REG_VTS			0x380e
+#define OV5670_VTS_30FPS		0x0808 /* default for 30 fps */
+#define OV5670_VTS_MAX			0xffff
+
+/* horizontal-timings from sensor */
+#define OV5670_REG_HTS			0x380c
+
+/*
+ * Pixels-per-line(PPL) = Time-per-line * pixel-rate
+ * In OV5670, Time-per-line = HTS/SCLK.
+ * HTS is fixed for all resolutions, not recommended to change.
+ */
+#define OV5670_FIXED_PPL		2724	/* Pixels per line */
+
+/* Exposure controls from sensor */
+#define OV5670_REG_EXPOSURE		0x3500
+#define	OV5670_EXPOSURE_MIN		4
+#define	OV5670_EXPOSURE_STEP		1
+
+/* Analog gain controls from sensor */
+#define OV5670_REG_ANALOG_GAIN		0x3508
+#define	ANALOG_GAIN_MIN			0
+#define	ANALOG_GAIN_MAX			8191
+#define	ANALOG_GAIN_STEP		1
+#define	ANALOG_GAIN_DEFAULT		128
+
+/* Digital gain controls from sensor */
+#define OV5670_REG_R_DGTL_GAIN		0x5032
+#define OV5670_REG_G_DGTL_GAIN		0x5034
+#define OV5670_REG_B_DGTL_GAIN		0x5036
+#define OV5670_DGTL_GAIN_MIN		0
+#define OV5670_DGTL_GAIN_MAX		4095
+#define OV5670_DGTL_GAIN_STEP		1
+#define OV5670_DGTL_GAIN_DEFAULT	1024
+
+/* Test Pattern Control */
+#define OV5670_REG_TEST_PATTERN		0x4303
+#define OV5670_TEST_PATTERN_ENABLE	BIT(3)
+#define OV5670_REG_TEST_PATTERN_CTRL	0x4320
+
+#define OV5670_REG_VALUE_08BIT		1
+#define OV5670_REG_VALUE_16BIT		2
+#define OV5670_REG_VALUE_24BIT		3
+
+/* Initial number of frames to skip to avoid possible garbage */
+#define OV5670_NUM_OF_SKIP_FRAMES	2
+
+struct ov5670_reg {
+	u16 address;
+	u8 val;
+};
+
+struct ov5670_reg_list {
+	u32 num_of_regs;
+	const struct ov5670_reg *regs;
+};
+
+struct ov5670_link_freq_config {
+	u32 pixel_rate;
+	const struct ov5670_reg_list reg_list;
+};
+
+struct ov5670_mode {
+	/* Frame width in pixels */
+	u32 width;
+
+	/* Frame height in pixels */
+	u32 height;
+
+	/* Default vertical timining size */
+	u32 vts_def;
+
+	/* Min vertical timining size */
+	u32 vts_min;
+
+	/* Link frequency needed for this resolution */
+	u32 link_freq_index;
+
+	/* Sensor register settings for this resolution */
+	const struct ov5670_reg_list reg_list;
+};
+
+static const struct ov5670_reg mipi_data_rate_840mbps[] = {
+	{0x0300, 0x04},
+	{0x0301, 0x00},
+	{0x0302, 0x84},
+	{0x0303, 0x00},
+	{0x0304, 0x03},
+	{0x0305, 0x01},
+	{0x0306, 0x01},
+	{0x030a, 0x00},
+	{0x030b, 0x00},
+	{0x030c, 0x00},
+	{0x030d, 0x26},
+	{0x030e, 0x00},
+	{0x030f, 0x06},
+	{0x0312, 0x01},
+	{0x3031, 0x0a},
+};
+
+static const struct ov5670_reg mode_2592x1944_regs[] = {
+	{0x3000, 0x00},
+	{0x3002, 0x21},
+	{0x3005, 0xf0},
+	{0x3007, 0x00},
+	{0x3015, 0x0f},
+	{0x3018, 0x32},
+	{0x301a, 0xf0},
+	{0x301b, 0xf0},
+	{0x301c, 0xf0},
+	{0x301d, 0xf0},
+	{0x301e, 0xf0},
+	{0x3030, 0x00},
+	{0x3031, 0x0a},
+	{0x303c, 0xff},
+	{0x303e, 0xff},
+	{0x3040, 0xf0},
+	{0x3041, 0x00},
+	{0x3042, 0xf0},
+	{0x3106, 0x11},
+	{0x3500, 0x00},
+	{0x3501, 0x80},
+	{0x3502, 0x00},
+	{0x3503, 0x04},
+	{0x3504, 0x03},
+	{0x3505, 0x83},
+	{0x3508, 0x04},
+	{0x3509, 0x00},
+	{0x350e, 0x04},
+	{0x350f, 0x00},
+	{0x3510, 0x00},
+	{0x3511, 0x02},
+	{0x3512, 0x00},
+	{0x3601, 0xc8},
+	{0x3610, 0x88},
+	{0x3612, 0x48},
+	{0x3614, 0x5b},
+	{0x3615, 0x96},
+	{0x3621, 0xd0},
+	{0x3622, 0x00},
+	{0x3623, 0x00},
+	{0x3633, 0x13},
+	{0x3634, 0x13},
+	{0x3635, 0x13},
+	{0x3636, 0x13},
+	{0x3645, 0x13},
+	{0x3646, 0x82},
+	{0x3650, 0x00},
+	{0x3652, 0xff},
+	{0x3655, 0x20},
+	{0x3656, 0xff},
+	{0x365a, 0xff},
+	{0x365e, 0xff},
+	{0x3668, 0x00},
+	{0x366a, 0x07},
+	{0x366e, 0x10},
+	{0x366d, 0x00},
+	{0x366f, 0x80},
+	{0x3700, 0x28},
+	{0x3701, 0x10},
+	{0x3702, 0x3a},
+	{0x3703, 0x19},
+	{0x3704, 0x10},
+	{0x3705, 0x00},
+	{0x3706, 0x66},
+	{0x3707, 0x08},
+	{0x3708, 0x34},
+	{0x3709, 0x40},
+	{0x370a, 0x01},
+	{0x370b, 0x1b},
+	{0x3714, 0x24},
+	{0x371a, 0x3e},
+	{0x3733, 0x00},
+	{0x3734, 0x00},
+	{0x373a, 0x05},
+	{0x373b, 0x06},
+	{0x373c, 0x0a},
+	{0x373f, 0xa0},
+	{0x3755, 0x00},
+	{0x3758, 0x00},
+	{0x375b, 0x0e},
+	{0x3766, 0x5f},
+	{0x3768, 0x00},
+	{0x3769, 0x22},
+	{0x3773, 0x08},
+	{0x3774, 0x1f},
+	{0x3776, 0x06},
+	{0x37a0, 0x88},
+	{0x37a1, 0x5c},
+	{0x37a7, 0x88},
+	{0x37a8, 0x70},
+	{0x37aa, 0x88},
+	{0x37ab, 0x48},
+	{0x37b3, 0x66},
+	{0x37c2, 0x04},
+	{0x37c5, 0x00},
+	{0x37c8, 0x00},
+	{0x3800, 0x00},
+	{0x3801, 0x0c},
+	{0x3802, 0x00},
+	{0x3803, 0x04},
+	{0x3804, 0x0a},
+	{0x3805, 0x33},
+	{0x3806, 0x07},
+	{0x3807, 0xa3},
+	{0x3808, 0x0a},
+	{0x3809, 0x20},
+	{0x380a, 0x07},
+	{0x380b, 0x98},
+	{0x380c, 0x06},
+	{0x380d, 0x90},
+	{0x380e, 0x08},
+	{0x380f, 0x08},
+	{0x3811, 0x04},
+	{0x3813, 0x02},
+	{0x3814, 0x01},
+	{0x3815, 0x01},
+	{0x3816, 0x00},
+	{0x3817, 0x00},
+	{0x3818, 0x00},
+	{0x3819, 0x00},
+	{0x3820, 0x84},
+	{0x3821, 0x46},
+	{0x3822, 0x48},
+	{0x3826, 0x00},
+	{0x3827, 0x08},
+	{0x382a, 0x01},
+	{0x382b, 0x01},
+	{0x3830, 0x08},
+	{0x3836, 0x02},
+	{0x3837, 0x00},
+	{0x3838, 0x10},
+	{0x3841, 0xff},
+	{0x3846, 0x48},
+	{0x3861, 0x00},
+	{0x3862, 0x04},
+	{0x3863, 0x06},
+	{0x3a11, 0x01},
+	{0x3a12, 0x78},
+	{0x3b00, 0x00},
+	{0x3b02, 0x00},
+	{0x3b03, 0x00},
+	{0x3b04, 0x00},
+	{0x3b05, 0x00},
+	{0x3c00, 0x89},
+	{0x3c01, 0xab},
+	{0x3c02, 0x01},
+	{0x3c03, 0x00},
+	{0x3c04, 0x00},
+	{0x3c05, 0x03},
+	{0x3c06, 0x00},
+	{0x3c07, 0x05},
+	{0x3c0c, 0x00},
+	{0x3c0d, 0x00},
+	{0x3c0e, 0x00},
+	{0x3c0f, 0x00},
+	{0x3c40, 0x00},
+	{0x3c41, 0xa3},
+	{0x3c43, 0x7d},
+	{0x3c45, 0xd7},
+	{0x3c47, 0xfc},
+	{0x3c50, 0x05},
+	{0x3c52, 0xaa},
+	{0x3c54, 0x71},
+	{0x3c56, 0x80},
+	{0x3d85, 0x17},
+	{0x3f03, 0x00},
+	{0x3f0a, 0x00},
+	{0x3f0b, 0x00},
+	{0x4001, 0x60},
+	{0x4009, 0x0d},
+	{0x4020, 0x00},
+	{0x4021, 0x00},
+	{0x4022, 0x00},
+	{0x4023, 0x00},
+	{0x4024, 0x00},
+	{0x4025, 0x00},
+	{0x4026, 0x00},
+	{0x4027, 0x00},
+	{0x4028, 0x00},
+	{0x4029, 0x00},
+	{0x402a, 0x00},
+	{0x402b, 0x00},
+	{0x402c, 0x00},
+	{0x402d, 0x00},
+	{0x402e, 0x00},
+	{0x402f, 0x00},
+	{0x4040, 0x00},
+	{0x4041, 0x03},
+	{0x4042, 0x00},
+	{0x4043, 0x7A},
+	{0x4044, 0x00},
+	{0x4045, 0x7A},
+	{0x4046, 0x00},
+	{0x4047, 0x7A},
+	{0x4048, 0x00},
+	{0x4049, 0x7A},
+	{0x4307, 0x30},
+	{0x4500, 0x58},
+	{0x4501, 0x04},
+	{0x4502, 0x40},
+	{0x4503, 0x10},
+	{0x4508, 0xaa},
+	{0x4509, 0xaa},
+	{0x450a, 0x00},
+	{0x450b, 0x00},
+	{0x4600, 0x01},
+	{0x4601, 0x03},
+	{0x4700, 0xa4},
+	{0x4800, 0x4c},
+	{0x4816, 0x53},
+	{0x481f, 0x40},
+	{0x4837, 0x13},
+	{0x5000, 0x56},
+	{0x5001, 0x01},
+	{0x5002, 0x28},
+	{0x5004, 0x0c},
+	{0x5006, 0x0c},
+	{0x5007, 0xe0},
+	{0x5008, 0x01},
+	{0x5009, 0xb0},
+	{0x5901, 0x00},
+	{0x5a01, 0x00},
+	{0x5a03, 0x00},
+	{0x5a04, 0x0c},
+	{0x5a05, 0xe0},
+	{0x5a06, 0x09},
+	{0x5a07, 0xb0},
+	{0x5a08, 0x06},
+	{0x5e00, 0x00},
+	{0x3734, 0x40},
+	{0x5b00, 0x01},
+	{0x5b01, 0x10},
+	{0x5b02, 0x01},
+	{0x5b03, 0xdb},
+	{0x3d8c, 0x71},
+	{0x3d8d, 0xea},
+	{0x4017, 0x08},
+	{0x3618, 0x2a},
+	{0x5780, 0x3e},
+	{0x5781, 0x0f},
+	{0x5782, 0x44},
+	{0x5783, 0x02},
+	{0x5784, 0x01},
+	{0x5785, 0x01},
+	{0x5786, 0x00},
+	{0x5787, 0x04},
+	{0x5788, 0x02},
+	{0x5789, 0x0f},
+	{0x578a, 0xfd},
+	{0x578b, 0xf5},
+	{0x578c, 0xf5},
+	{0x578d, 0x03},
+	{0x578e, 0x08},
+	{0x578f, 0x0c},
+	{0x5790, 0x08},
+	{0x5791, 0x06},
+	{0x5792, 0x00},
+	{0x5793, 0x52},
+	{0x5794, 0xa3},
+	{0x3503, 0x00}
+};
+
+static const struct ov5670_reg mode_1296x972_regs[] = {
+	{0x3000, 0x00},
+	{0x3002, 0x21},
+	{0x3005, 0xf0},
+	{0x3007, 0x00},
+	{0x3015, 0x0f},
+	{0x3018, 0x32},
+	{0x301a, 0xf0},
+	{0x301b, 0xf0},
+	{0x301c, 0xf0},
+	{0x301d, 0xf0},
+	{0x301e, 0xf0},
+	{0x3030, 0x00},
+	{0x3031, 0x0a},
+	{0x303c, 0xff},
+	{0x303e, 0xff},
+	{0x3040, 0xf0},
+	{0x3041, 0x00},
+	{0x3042, 0xf0},
+	{0x3106, 0x11},
+	{0x3500, 0x00},
+	{0x3501, 0x80},
+	{0x3502, 0x00},
+	{0x3503, 0x04},
+	{0x3504, 0x03},
+	{0x3505, 0x83},
+	{0x3508, 0x07},
+	{0x3509, 0x80},
+	{0x350e, 0x04},
+	{0x350f, 0x00},
+	{0x3510, 0x00},
+	{0x3511, 0x02},
+	{0x3512, 0x00},
+	{0x3601, 0xc8},
+	{0x3610, 0x88},
+	{0x3612, 0x48},
+	{0x3614, 0x5b},
+	{0x3615, 0x96},
+	{0x3621, 0xd0},
+	{0x3622, 0x00},
+	{0x3623, 0x00},
+	{0x3633, 0x13},
+	{0x3634, 0x13},
+	{0x3635, 0x13},
+	{0x3636, 0x13},
+	{0x3645, 0x13},
+	{0x3646, 0x82},
+	{0x3650, 0x00},
+	{0x3652, 0xff},
+	{0x3655, 0x20},
+	{0x3656, 0xff},
+	{0x365a, 0xff},
+	{0x365e, 0xff},
+	{0x3668, 0x00},
+	{0x366a, 0x07},
+	{0x366e, 0x08},
+	{0x366d, 0x00},
+	{0x366f, 0x80},
+	{0x3700, 0x28},
+	{0x3701, 0x10},
+	{0x3702, 0x3a},
+	{0x3703, 0x19},
+	{0x3704, 0x10},
+	{0x3705, 0x00},
+	{0x3706, 0x66},
+	{0x3707, 0x08},
+	{0x3708, 0x34},
+	{0x3709, 0x40},
+	{0x370a, 0x01},
+	{0x370b, 0x1b},
+	{0x3714, 0x24},
+	{0x371a, 0x3e},
+	{0x3733, 0x00},
+	{0x3734, 0x00},
+	{0x373a, 0x05},
+	{0x373b, 0x06},
+	{0x373c, 0x0a},
+	{0x373f, 0xa0},
+	{0x3755, 0x00},
+	{0x3758, 0x00},
+	{0x375b, 0x0e},
+	{0x3766, 0x5f},
+	{0x3768, 0x00},
+	{0x3769, 0x22},
+	{0x3773, 0x08},
+	{0x3774, 0x1f},
+	{0x3776, 0x06},
+	{0x37a0, 0x88},
+	{0x37a1, 0x5c},
+	{0x37a7, 0x88},
+	{0x37a8, 0x70},
+	{0x37aa, 0x88},
+	{0x37ab, 0x48},
+	{0x37b3, 0x66},
+	{0x37c2, 0x04},
+	{0x37c5, 0x00},
+	{0x37c8, 0x00},
+	{0x3800, 0x00},
+	{0x3801, 0x0c},
+	{0x3802, 0x00},
+	{0x3803, 0x04},
+	{0x3804, 0x0a},
+	{0x3805, 0x33},
+	{0x3806, 0x07},
+	{0x3807, 0xa3},
+	{0x3808, 0x05},
+	{0x3809, 0x10},
+	{0x380a, 0x03},
+	{0x380b, 0xcc},
+	{0x380c, 0x06},
+	{0x380d, 0x90},
+	{0x380e, 0x08},
+	{0x380f, 0x08},
+	{0x3811, 0x04},
+	{0x3813, 0x04},
+	{0x3814, 0x03},
+	{0x3815, 0x01},
+	{0x3816, 0x00},
+	{0x3817, 0x00},
+	{0x3818, 0x00},
+	{0x3819, 0x00},
+	{0x3820, 0x94},
+	{0x3821, 0x47},
+	{0x3822, 0x48},
+	{0x3826, 0x00},
+	{0x3827, 0x08},
+	{0x382a, 0x03},
+	{0x382b, 0x01},
+	{0x3830, 0x08},
+	{0x3836, 0x02},
+	{0x3837, 0x00},
+	{0x3838, 0x10},
+	{0x3841, 0xff},
+	{0x3846, 0x48},
+	{0x3861, 0x00},
+	{0x3862, 0x04},
+	{0x3863, 0x06},
+	{0x3a11, 0x01},
+	{0x3a12, 0x78},
+	{0x3b00, 0x00},
+	{0x3b02, 0x00},
+	{0x3b03, 0x00},
+	{0x3b04, 0x00},
+	{0x3b05, 0x00},
+	{0x3c00, 0x89},
+	{0x3c01, 0xab},
+	{0x3c02, 0x01},
+	{0x3c03, 0x00},
+	{0x3c04, 0x00},
+	{0x3c05, 0x03},
+	{0x3c06, 0x00},
+	{0x3c07, 0x05},
+	{0x3c0c, 0x00},
+	{0x3c0d, 0x00},
+	{0x3c0e, 0x00},
+	{0x3c0f, 0x00},
+	{0x3c40, 0x00},
+	{0x3c41, 0xa3},
+	{0x3c43, 0x7d},
+	{0x3c45, 0xd7},
+	{0x3c47, 0xfc},
+	{0x3c50, 0x05},
+	{0x3c52, 0xaa},
+	{0x3c54, 0x71},
+	{0x3c56, 0x80},
+	{0x3d85, 0x17},
+	{0x3f03, 0x00},
+	{0x3f0a, 0x00},
+	{0x3f0b, 0x00},
+	{0x4001, 0x60},
+	{0x4009, 0x05},
+	{0x4020, 0x00},
+	{0x4021, 0x00},
+	{0x4022, 0x00},
+	{0x4023, 0x00},
+	{0x4024, 0x00},
+	{0x4025, 0x00},
+	{0x4026, 0x00},
+	{0x4027, 0x00},
+	{0x4028, 0x00},
+	{0x4029, 0x00},
+	{0x402a, 0x00},
+	{0x402b, 0x00},
+	{0x402c, 0x00},
+	{0x402d, 0x00},
+	{0x402e, 0x00},
+	{0x402f, 0x00},
+	{0x4040, 0x00},
+	{0x4041, 0x03},
+	{0x4042, 0x00},
+	{0x4043, 0x7A},
+	{0x4044, 0x00},
+	{0x4045, 0x7A},
+	{0x4046, 0x00},
+	{0x4047, 0x7A},
+	{0x4048, 0x00},
+	{0x4049, 0x7A},
+	{0x4307, 0x30},
+	{0x4500, 0x58},
+	{0x4501, 0x04},
+	{0x4502, 0x48},
+	{0x4503, 0x10},
+	{0x4508, 0x55},
+	{0x4509, 0x55},
+	{0x450a, 0x00},
+	{0x450b, 0x00},
+	{0x4600, 0x00},
+	{0x4601, 0x81},
+	{0x4700, 0xa4},
+	{0x4800, 0x4c},
+	{0x4816, 0x53},
+	{0x481f, 0x40},
+	{0x4837, 0x13},
+	{0x5000, 0x56},
+	{0x5001, 0x01},
+	{0x5002, 0x28},
+	{0x5004, 0x0c},
+	{0x5006, 0x0c},
+	{0x5007, 0xe0},
+	{0x5008, 0x01},
+	{0x5009, 0xb0},
+	{0x5901, 0x00},
+	{0x5a01, 0x00},
+	{0x5a03, 0x00},
+	{0x5a04, 0x0c},
+	{0x5a05, 0xe0},
+	{0x5a06, 0x09},
+	{0x5a07, 0xb0},
+	{0x5a08, 0x06},
+	{0x5e00, 0x00},
+	{0x3734, 0x40},
+	{0x5b00, 0x01},
+	{0x5b01, 0x10},
+	{0x5b02, 0x01},
+	{0x5b03, 0xdb},
+	{0x3d8c, 0x71},
+	{0x3d8d, 0xea},
+	{0x4017, 0x10},
+	{0x3618, 0x2a},
+	{0x5780, 0x3e},
+	{0x5781, 0x0f},
+	{0x5782, 0x44},
+	{0x5783, 0x02},
+	{0x5784, 0x01},
+	{0x5785, 0x01},
+	{0x5786, 0x00},
+	{0x5787, 0x04},
+	{0x5788, 0x02},
+	{0x5789, 0x0f},
+	{0x578a, 0xfd},
+	{0x578b, 0xf5},
+	{0x578c, 0xf5},
+	{0x578d, 0x03},
+	{0x578e, 0x08},
+	{0x578f, 0x0c},
+	{0x5790, 0x08},
+	{0x5791, 0x04},
+	{0x5792, 0x00},
+	{0x5793, 0x52},
+	{0x5794, 0xa3},
+	{0x3503, 0x00}
+};
+
+static const struct ov5670_reg mode_648x486_regs[] = {
+	{0x3000, 0x00},
+	{0x3002, 0x21},
+	{0x3005, 0xf0},
+	{0x3007, 0x00},
+	{0x3015, 0x0f},
+	{0x3018, 0x32},
+	{0x301a, 0xf0},
+	{0x301b, 0xf0},
+	{0x301c, 0xf0},
+	{0x301d, 0xf0},
+	{0x301e, 0xf0},
+	{0x3030, 0x00},
+	{0x3031, 0x0a},
+	{0x303c, 0xff},
+	{0x303e, 0xff},
+	{0x3040, 0xf0},
+	{0x3041, 0x00},
+	{0x3042, 0xf0},
+	{0x3106, 0x11},
+	{0x3500, 0x00},
+	{0x3501, 0x80},
+	{0x3502, 0x00},
+	{0x3503, 0x04},
+	{0x3504, 0x03},
+	{0x3505, 0x83},
+	{0x3508, 0x04},
+	{0x3509, 0x00},
+	{0x350e, 0x04},
+	{0x350f, 0x00},
+	{0x3510, 0x00},
+	{0x3511, 0x02},
+	{0x3512, 0x00},
+	{0x3601, 0xc8},
+	{0x3610, 0x88},
+	{0x3612, 0x48},
+	{0x3614, 0x5b},
+	{0x3615, 0x96},
+	{0x3621, 0xd0},
+	{0x3622, 0x00},
+	{0x3623, 0x04},
+	{0x3633, 0x13},
+	{0x3634, 0x13},
+	{0x3635, 0x13},
+	{0x3636, 0x13},
+	{0x3645, 0x13},
+	{0x3646, 0x82},
+	{0x3650, 0x00},
+	{0x3652, 0xff},
+	{0x3655, 0x20},
+	{0x3656, 0xff},
+	{0x365a, 0xff},
+	{0x365e, 0xff},
+	{0x3668, 0x00},
+	{0x366a, 0x07},
+	{0x366e, 0x08},
+	{0x366d, 0x00},
+	{0x366f, 0x80},
+	{0x3700, 0x28},
+	{0x3701, 0x10},
+	{0x3702, 0x3a},
+	{0x3703, 0x19},
+	{0x3704, 0x10},
+	{0x3705, 0x00},
+	{0x3706, 0x66},
+	{0x3707, 0x08},
+	{0x3708, 0x34},
+	{0x3709, 0x40},
+	{0x370a, 0x01},
+	{0x370b, 0x1b},
+	{0x3714, 0x24},
+	{0x371a, 0x3e},
+	{0x3733, 0x00},
+	{0x3734, 0x00},
+	{0x373a, 0x05},
+	{0x373b, 0x06},
+	{0x373c, 0x0a},
+	{0x373f, 0xa0},
+	{0x3755, 0x00},
+	{0x3758, 0x00},
+	{0x375b, 0x0e},
+	{0x3766, 0x5f},
+	{0x3768, 0x00},
+	{0x3769, 0x22},
+	{0x3773, 0x08},
+	{0x3774, 0x1f},
+	{0x3776, 0x06},
+	{0x37a0, 0x88},
+	{0x37a1, 0x5c},
+	{0x37a7, 0x88},
+	{0x37a8, 0x70},
+	{0x37aa, 0x88},
+	{0x37ab, 0x48},
+	{0x37b3, 0x66},
+	{0x37c2, 0x04},
+	{0x37c5, 0x00},
+	{0x37c8, 0x00},
+	{0x3800, 0x00},
+	{0x3801, 0x0c},
+	{0x3802, 0x00},
+	{0x3803, 0x04},
+	{0x3804, 0x0a},
+	{0x3805, 0x33},
+	{0x3806, 0x07},
+	{0x3807, 0xa3},
+	{0x3808, 0x02},
+	{0x3809, 0x88},
+	{0x380a, 0x01},
+	{0x380b, 0xe6},
+	{0x380c, 0x06},
+	{0x380d, 0x90},
+	{0x380e, 0x08},
+	{0x380f, 0x08},
+	{0x3811, 0x04},
+	{0x3813, 0x02},
+	{0x3814, 0x07},
+	{0x3815, 0x01},
+	{0x3816, 0x00},
+	{0x3817, 0x00},
+	{0x3818, 0x00},
+	{0x3819, 0x00},
+	{0x3820, 0x94},
+	{0x3821, 0xc6},
+	{0x3822, 0x48},
+	{0x3826, 0x00},
+	{0x3827, 0x08},
+	{0x382a, 0x07},
+	{0x382b, 0x01},
+	{0x3830, 0x08},
+	{0x3836, 0x02},
+	{0x3837, 0x00},
+	{0x3838, 0x10},
+	{0x3841, 0xff},
+	{0x3846, 0x48},
+	{0x3861, 0x00},
+	{0x3862, 0x04},
+	{0x3863, 0x06},
+	{0x3a11, 0x01},
+	{0x3a12, 0x78},
+	{0x3b00, 0x00},
+	{0x3b02, 0x00},
+	{0x3b03, 0x00},
+	{0x3b04, 0x00},
+	{0x3b05, 0x00},
+	{0x3c00, 0x89},
+	{0x3c01, 0xab},
+	{0x3c02, 0x01},
+	{0x3c03, 0x00},
+	{0x3c04, 0x00},
+	{0x3c05, 0x03},
+	{0x3c06, 0x00},
+	{0x3c07, 0x05},
+	{0x3c0c, 0x00},
+	{0x3c0d, 0x00},
+	{0x3c0e, 0x00},
+	{0x3c0f, 0x00},
+	{0x3c40, 0x00},
+	{0x3c41, 0xa3},
+	{0x3c43, 0x7d},
+	{0x3c45, 0xd7},
+	{0x3c47, 0xfc},
+	{0x3c50, 0x05},
+	{0x3c52, 0xaa},
+	{0x3c54, 0x71},
+	{0x3c56, 0x80},
+	{0x3d85, 0x17},
+	{0x3f03, 0x00},
+	{0x3f0a, 0x00},
+	{0x3f0b, 0x00},
+	{0x4001, 0x60},
+	{0x4009, 0x05},
+	{0x4020, 0x00},
+	{0x4021, 0x00},
+	{0x4022, 0x00},
+	{0x4023, 0x00},
+	{0x4024, 0x00},
+	{0x4025, 0x00},
+	{0x4026, 0x00},
+	{0x4027, 0x00},
+	{0x4028, 0x00},
+	{0x4029, 0x00},
+	{0x402a, 0x00},
+	{0x402b, 0x00},
+	{0x402c, 0x00},
+	{0x402d, 0x00},
+	{0x402e, 0x00},
+	{0x402f, 0x00},
+	{0x4040, 0x00},
+	{0x4041, 0x03},
+	{0x4042, 0x00},
+	{0x4043, 0x7A},
+	{0x4044, 0x00},
+	{0x4045, 0x7A},
+	{0x4046, 0x00},
+	{0x4047, 0x7A},
+	{0x4048, 0x00},
+	{0x4049, 0x7A},
+	{0x4307, 0x30},
+	{0x4500, 0x58},
+	{0x4501, 0x04},
+	{0x4502, 0x40},
+	{0x4503, 0x10},
+	{0x4508, 0x55},
+	{0x4509, 0x55},
+	{0x450a, 0x02},
+	{0x450b, 0x00},
+	{0x4600, 0x00},
+	{0x4601, 0x40},
+	{0x4700, 0xa4},
+	{0x4800, 0x4c},
+	{0x4816, 0x53},
+	{0x481f, 0x40},
+	{0x4837, 0x13},
+	{0x5000, 0x56},
+	{0x5001, 0x01},
+	{0x5002, 0x28},
+	{0x5004, 0x0c},
+	{0x5006, 0x0c},
+	{0x5007, 0xe0},
+	{0x5008, 0x01},
+	{0x5009, 0xb0},
+	{0x5901, 0x00},
+	{0x5a01, 0x00},
+	{0x5a03, 0x00},
+	{0x5a04, 0x0c},
+	{0x5a05, 0xe0},
+	{0x5a06, 0x09},
+	{0x5a07, 0xb0},
+	{0x5a08, 0x06},
+	{0x5e00, 0x00},
+	{0x3734, 0x40},
+	{0x5b00, 0x01},
+	{0x5b01, 0x10},
+	{0x5b02, 0x01},
+	{0x5b03, 0xdb},
+	{0x3d8c, 0x71},
+	{0x3d8d, 0xea},
+	{0x4017, 0x10},
+	{0x3618, 0x2a},
+	{0x5780, 0x3e},
+	{0x5781, 0x0f},
+	{0x5782, 0x44},
+	{0x5783, 0x02},
+	{0x5784, 0x01},
+	{0x5785, 0x01},
+	{0x5786, 0x00},
+	{0x5787, 0x04},
+	{0x5788, 0x02},
+	{0x5789, 0x0f},
+	{0x578a, 0xfd},
+	{0x578b, 0xf5},
+	{0x578c, 0xf5},
+	{0x578d, 0x03},
+	{0x578e, 0x08},
+	{0x578f, 0x0c},
+	{0x5790, 0x08},
+	{0x5791, 0x06},
+	{0x5792, 0x00},
+	{0x5793, 0x52},
+	{0x5794, 0xa3},
+	{0x3503, 0x00}
+};
+
+static const struct ov5670_reg mode_2560x1440_regs[] = {
+	{0x3000, 0x00},
+	{0x3002, 0x21},
+	{0x3005, 0xf0},
+	{0x3007, 0x00},
+	{0x3015, 0x0f},
+	{0x3018, 0x32},
+	{0x301a, 0xf0},
+	{0x301b, 0xf0},
+	{0x301c, 0xf0},
+	{0x301d, 0xf0},
+	{0x301e, 0xf0},
+	{0x3030, 0x00},
+	{0x3031, 0x0a},
+	{0x303c, 0xff},
+	{0x303e, 0xff},
+	{0x3040, 0xf0},
+	{0x3041, 0x00},
+	{0x3042, 0xf0},
+	{0x3106, 0x11},
+	{0x3500, 0x00},
+	{0x3501, 0x80},
+	{0x3502, 0x00},
+	{0x3503, 0x04},
+	{0x3504, 0x03},
+	{0x3505, 0x83},
+	{0x3508, 0x04},
+	{0x3509, 0x00},
+	{0x350e, 0x04},
+	{0x350f, 0x00},
+	{0x3510, 0x00},
+	{0x3511, 0x02},
+	{0x3512, 0x00},
+	{0x3601, 0xc8},
+	{0x3610, 0x88},
+	{0x3612, 0x48},
+	{0x3614, 0x5b},
+	{0x3615, 0x96},
+	{0x3621, 0xd0},
+	{0x3622, 0x00},
+	{0x3623, 0x00},
+	{0x3633, 0x13},
+	{0x3634, 0x13},
+	{0x3635, 0x13},
+	{0x3636, 0x13},
+	{0x3645, 0x13},
+	{0x3646, 0x82},
+	{0x3650, 0x00},
+	{0x3652, 0xff},
+	{0x3655, 0x20},
+	{0x3656, 0xff},
+	{0x365a, 0xff},
+	{0x365e, 0xff},
+	{0x3668, 0x00},
+	{0x366a, 0x07},
+	{0x366e, 0x10},
+	{0x366d, 0x00},
+	{0x366f, 0x80},
+	{0x3700, 0x28},
+	{0x3701, 0x10},
+	{0x3702, 0x3a},
+	{0x3703, 0x19},
+	{0x3704, 0x10},
+	{0x3705, 0x00},
+	{0x3706, 0x66},
+	{0x3707, 0x08},
+	{0x3708, 0x34},
+	{0x3709, 0x40},
+	{0x370a, 0x01},
+	{0x370b, 0x1b},
+	{0x3714, 0x24},
+	{0x371a, 0x3e},
+	{0x3733, 0x00},
+	{0x3734, 0x00},
+	{0x373a, 0x05},
+	{0x373b, 0x06},
+	{0x373c, 0x0a},
+	{0x373f, 0xa0},
+	{0x3755, 0x00},
+	{0x3758, 0x00},
+	{0x375b, 0x0e},
+	{0x3766, 0x5f},
+	{0x3768, 0x00},
+	{0x3769, 0x22},
+	{0x3773, 0x08},
+	{0x3774, 0x1f},
+	{0x3776, 0x06},
+	{0x37a0, 0x88},
+	{0x37a1, 0x5c},
+	{0x37a7, 0x88},
+	{0x37a8, 0x70},
+	{0x37aa, 0x88},
+	{0x37ab, 0x48},
+	{0x37b3, 0x66},
+	{0x37c2, 0x04},
+	{0x37c5, 0x00},
+	{0x37c8, 0x00},
+	{0x3800, 0x00},
+	{0x3801, 0x0c},
+	{0x3802, 0x00},
+	{0x3803, 0x04},
+	{0x3804, 0x0a},
+	{0x3805, 0x33},
+	{0x3806, 0x07},
+	{0x3807, 0xa3},
+	{0x3808, 0x0a},
+	{0x3809, 0x00},
+	{0x380a, 0x05},
+	{0x380b, 0xa0},
+	{0x380c, 0x06},
+	{0x380d, 0x90},
+	{0x380e, 0x08},
+	{0x380f, 0x08},
+	{0x3811, 0x04},
+	{0x3813, 0x02},
+	{0x3814, 0x01},
+	{0x3815, 0x01},
+	{0x3816, 0x00},
+	{0x3817, 0x00},
+	{0x3818, 0x00},
+	{0x3819, 0x00},
+	{0x3820, 0x84},
+	{0x3821, 0x46},
+	{0x3822, 0x48},
+	{0x3826, 0x00},
+	{0x3827, 0x08},
+	{0x382a, 0x01},
+	{0x382b, 0x01},
+	{0x3830, 0x08},
+	{0x3836, 0x02},
+	{0x3837, 0x00},
+	{0x3838, 0x10},
+	{0x3841, 0xff},
+	{0x3846, 0x48},
+	{0x3861, 0x00},
+	{0x3862, 0x04},
+	{0x3863, 0x06},
+	{0x3a11, 0x01},
+	{0x3a12, 0x78},
+	{0x3b00, 0x00},
+	{0x3b02, 0x00},
+	{0x3b03, 0x00},
+	{0x3b04, 0x00},
+	{0x3b05, 0x00},
+	{0x3c00, 0x89},
+	{0x3c01, 0xab},
+	{0x3c02, 0x01},
+	{0x3c03, 0x00},
+	{0x3c04, 0x00},
+	{0x3c05, 0x03},
+	{0x3c06, 0x00},
+	{0x3c07, 0x05},
+	{0x3c0c, 0x00},
+	{0x3c0d, 0x00},
+	{0x3c0e, 0x00},
+	{0x3c0f, 0x00},
+	{0x3c40, 0x00},
+	{0x3c41, 0xa3},
+	{0x3c43, 0x7d},
+	{0x3c45, 0xd7},
+	{0x3c47, 0xfc},
+	{0x3c50, 0x05},
+	{0x3c52, 0xaa},
+	{0x3c54, 0x71},
+	{0x3c56, 0x80},
+	{0x3d85, 0x17},
+	{0x3f03, 0x00},
+	{0x3f0a, 0x00},
+	{0x3f0b, 0x00},
+	{0x4001, 0x60},
+	{0x4009, 0x0d},
+	{0x4020, 0x00},
+	{0x4021, 0x00},
+	{0x4022, 0x00},
+	{0x4023, 0x00},
+	{0x4024, 0x00},
+	{0x4025, 0x00},
+	{0x4026, 0x00},
+	{0x4027, 0x00},
+	{0x4028, 0x00},
+	{0x4029, 0x00},
+	{0x402a, 0x00},
+	{0x402b, 0x00},
+	{0x402c, 0x00},
+	{0x402d, 0x00},
+	{0x402e, 0x00},
+	{0x402f, 0x00},
+	{0x4040, 0x00},
+	{0x4041, 0x03},
+	{0x4042, 0x00},
+	{0x4043, 0x7A},
+	{0x4044, 0x00},
+	{0x4045, 0x7A},
+	{0x4046, 0x00},
+	{0x4047, 0x7A},
+	{0x4048, 0x00},
+	{0x4049, 0x7A},
+	{0x4307, 0x30},
+	{0x4500, 0x58},
+	{0x4501, 0x04},
+	{0x4502, 0x40},
+	{0x4503, 0x10},
+	{0x4508, 0xaa},
+	{0x4509, 0xaa},
+	{0x450a, 0x00},
+	{0x450b, 0x00},
+	{0x4600, 0x01},
+	{0x4601, 0x00},
+	{0x4700, 0xa4},
+	{0x4800, 0x4c},
+	{0x4816, 0x53},
+	{0x481f, 0x40},
+	{0x4837, 0x13},
+	{0x5000, 0x56},
+	{0x5001, 0x01},
+	{0x5002, 0x28},
+	{0x5004, 0x0c},
+	{0x5006, 0x0c},
+	{0x5007, 0xe0},
+	{0x5008, 0x01},
+	{0x5009, 0xb0},
+	{0x5901, 0x00},
+	{0x5a01, 0x00},
+	{0x5a03, 0x00},
+	{0x5a04, 0x0c},
+	{0x5a05, 0xe0},
+	{0x5a06, 0x09},
+	{0x5a07, 0xb0},
+	{0x5a08, 0x06},
+	{0x5e00, 0x00},
+	{0x3734, 0x40},
+	{0x5b00, 0x01},
+	{0x5b01, 0x10},
+	{0x5b02, 0x01},
+	{0x5b03, 0xdb},
+	{0x3d8c, 0x71},
+	{0x3d8d, 0xea},
+	{0x4017, 0x08},
+	{0x3618, 0x2a},
+	{0x5780, 0x3e},
+	{0x5781, 0x0f},
+	{0x5782, 0x44},
+	{0x5783, 0x02},
+	{0x5784, 0x01},
+	{0x5785, 0x01},
+	{0x5786, 0x00},
+	{0x5787, 0x04},
+	{0x5788, 0x02},
+	{0x5789, 0x0f},
+	{0x578a, 0xfd},
+	{0x578b, 0xf5},
+	{0x578c, 0xf5},
+	{0x578d, 0x03},
+	{0x578e, 0x08},
+	{0x578f, 0x0c},
+	{0x5790, 0x08},
+	{0x5791, 0x06},
+	{0x5792, 0x00},
+	{0x5793, 0x52},
+	{0x5794, 0xa3}
+};
+
+static const struct ov5670_reg mode_1280x720_regs[] = {
+	{0x3000, 0x00},
+	{0x3002, 0x21},
+	{0x3005, 0xf0},
+	{0x3007, 0x00},
+	{0x3015, 0x0f},
+	{0x3018, 0x32},
+	{0x301a, 0xf0},
+	{0x301b, 0xf0},
+	{0x301c, 0xf0},
+	{0x301d, 0xf0},
+	{0x301e, 0xf0},
+	{0x3030, 0x00},
+	{0x3031, 0x0a},
+	{0x303c, 0xff},
+	{0x303e, 0xff},
+	{0x3040, 0xf0},
+	{0x3041, 0x00},
+	{0x3042, 0xf0},
+	{0x3106, 0x11},
+	{0x3500, 0x00},
+	{0x3501, 0x80},
+	{0x3502, 0x00},
+	{0x3503, 0x04},
+	{0x3504, 0x03},
+	{0x3505, 0x83},
+	{0x3508, 0x04},
+	{0x3509, 0x00},
+	{0x350e, 0x04},
+	{0x350f, 0x00},
+	{0x3510, 0x00},
+	{0x3511, 0x02},
+	{0x3512, 0x00},
+	{0x3601, 0xc8},
+	{0x3610, 0x88},
+	{0x3612, 0x48},
+	{0x3614, 0x5b},
+	{0x3615, 0x96},
+	{0x3621, 0xd0},
+	{0x3622, 0x00},
+	{0x3623, 0x00},
+	{0x3633, 0x13},
+	{0x3634, 0x13},
+	{0x3635, 0x13},
+	{0x3636, 0x13},
+	{0x3645, 0x13},
+	{0x3646, 0x82},
+	{0x3650, 0x00},
+	{0x3652, 0xff},
+	{0x3655, 0x20},
+	{0x3656, 0xff},
+	{0x365a, 0xff},
+	{0x365e, 0xff},
+	{0x3668, 0x00},
+	{0x366a, 0x07},
+	{0x366e, 0x08},
+	{0x366d, 0x00},
+	{0x366f, 0x80},
+	{0x3700, 0x28},
+	{0x3701, 0x10},
+	{0x3702, 0x3a},
+	{0x3703, 0x19},
+	{0x3704, 0x10},
+	{0x3705, 0x00},
+	{0x3706, 0x66},
+	{0x3707, 0x08},
+	{0x3708, 0x34},
+	{0x3709, 0x40},
+	{0x370a, 0x01},
+	{0x370b, 0x1b},
+	{0x3714, 0x24},
+	{0x371a, 0x3e},
+	{0x3733, 0x00},
+	{0x3734, 0x00},
+	{0x373a, 0x05},
+	{0x373b, 0x06},
+	{0x373c, 0x0a},
+	{0x373f, 0xa0},
+	{0x3755, 0x00},
+	{0x3758, 0x00},
+	{0x375b, 0x0e},
+	{0x3766, 0x5f},
+	{0x3768, 0x00},
+	{0x3769, 0x22},
+	{0x3773, 0x08},
+	{0x3774, 0x1f},
+	{0x3776, 0x06},
+	{0x37a0, 0x88},
+	{0x37a1, 0x5c},
+	{0x37a7, 0x88},
+	{0x37a8, 0x70},
+	{0x37aa, 0x88},
+	{0x37ab, 0x48},
+	{0x37b3, 0x66},
+	{0x37c2, 0x04},
+	{0x37c5, 0x00},
+	{0x37c8, 0x00},
+	{0x3800, 0x00},
+	{0x3801, 0x0c},
+	{0x3802, 0x00},
+	{0x3803, 0x04},
+	{0x3804, 0x0a},
+	{0x3805, 0x33},
+	{0x3806, 0x07},
+	{0x3807, 0xa3},
+	{0x3808, 0x05},
+	{0x3809, 0x00},
+	{0x380a, 0x02},
+	{0x380b, 0xd0},
+	{0x380c, 0x06},
+	{0x380d, 0x90},
+	{0x380e, 0x08},
+	{0x380f, 0x08},
+	{0x3811, 0x04},
+	{0x3813, 0x02},
+	{0x3814, 0x03},
+	{0x3815, 0x01},
+	{0x3816, 0x00},
+	{0x3817, 0x00},
+	{0x3818, 0x00},
+	{0x3819, 0x00},
+	{0x3820, 0x94},
+	{0x3821, 0x47},
+	{0x3822, 0x48},
+	{0x3826, 0x00},
+	{0x3827, 0x08},
+	{0x382a, 0x03},
+	{0x382b, 0x01},
+	{0x3830, 0x08},
+	{0x3836, 0x02},
+	{0x3837, 0x00},
+	{0x3838, 0x10},
+	{0x3841, 0xff},
+	{0x3846, 0x48},
+	{0x3861, 0x00},
+	{0x3862, 0x04},
+	{0x3863, 0x06},
+	{0x3a11, 0x01},
+	{0x3a12, 0x78},
+	{0x3b00, 0x00},
+	{0x3b02, 0x00},
+	{0x3b03, 0x00},
+	{0x3b04, 0x00},
+	{0x3b05, 0x00},
+	{0x3c00, 0x89},
+	{0x3c01, 0xab},
+	{0x3c02, 0x01},
+	{0x3c03, 0x00},
+	{0x3c04, 0x00},
+	{0x3c05, 0x03},
+	{0x3c06, 0x00},
+	{0x3c07, 0x05},
+	{0x3c0c, 0x00},
+	{0x3c0d, 0x00},
+	{0x3c0e, 0x00},
+	{0x3c0f, 0x00},
+	{0x3c40, 0x00},
+	{0x3c41, 0xa3},
+	{0x3c43, 0x7d},
+	{0x3c45, 0xd7},
+	{0x3c47, 0xfc},
+	{0x3c50, 0x05},
+	{0x3c52, 0xaa},
+	{0x3c54, 0x71},
+	{0x3c56, 0x80},
+	{0x3d85, 0x17},
+	{0x3f03, 0x00},
+	{0x3f0a, 0x00},
+	{0x3f0b, 0x00},
+	{0x4001, 0x60},
+	{0x4009, 0x05},
+	{0x4020, 0x00},
+	{0x4021, 0x00},
+	{0x4022, 0x00},
+	{0x4023, 0x00},
+	{0x4024, 0x00},
+	{0x4025, 0x00},
+	{0x4026, 0x00},
+	{0x4027, 0x00},
+	{0x4028, 0x00},
+	{0x4029, 0x00},
+	{0x402a, 0x00},
+	{0x402b, 0x00},
+	{0x402c, 0x00},
+	{0x402d, 0x00},
+	{0x402e, 0x00},
+	{0x402f, 0x00},
+	{0x4040, 0x00},
+	{0x4041, 0x03},
+	{0x4042, 0x00},
+	{0x4043, 0x7A},
+	{0x4044, 0x00},
+	{0x4045, 0x7A},
+	{0x4046, 0x00},
+	{0x4047, 0x7A},
+	{0x4048, 0x00},
+	{0x4049, 0x7A},
+	{0x4307, 0x30},
+	{0x4500, 0x58},
+	{0x4501, 0x04},
+	{0x4502, 0x48},
+	{0x4503, 0x10},
+	{0x4508, 0x55},
+	{0x4509, 0x55},
+	{0x450a, 0x00},
+	{0x450b, 0x00},
+	{0x4600, 0x00},
+	{0x4601, 0x80},
+	{0x4700, 0xa4},
+	{0x4800, 0x4c},
+	{0x4816, 0x53},
+	{0x481f, 0x40},
+	{0x4837, 0x13},
+	{0x5000, 0x56},
+	{0x5001, 0x01},
+	{0x5002, 0x28},
+	{0x5004, 0x0c},
+	{0x5006, 0x0c},
+	{0x5007, 0xe0},
+	{0x5008, 0x01},
+	{0x5009, 0xb0},
+	{0x5901, 0x00},
+	{0x5a01, 0x00},
+	{0x5a03, 0x00},
+	{0x5a04, 0x0c},
+	{0x5a05, 0xe0},
+	{0x5a06, 0x09},
+	{0x5a07, 0xb0},
+	{0x5a08, 0x06},
+	{0x5e00, 0x00},
+	{0x3734, 0x40},
+	{0x5b00, 0x01},
+	{0x5b01, 0x10},
+	{0x5b02, 0x01},
+	{0x5b03, 0xdb},
+	{0x3d8c, 0x71},
+	{0x3d8d, 0xea},
+	{0x4017, 0x10},
+	{0x3618, 0x2a},
+	{0x5780, 0x3e},
+	{0x5781, 0x0f},
+	{0x5782, 0x44},
+	{0x5783, 0x02},
+	{0x5784, 0x01},
+	{0x5785, 0x01},
+	{0x5786, 0x00},
+	{0x5787, 0x04},
+	{0x5788, 0x02},
+	{0x5789, 0x0f},
+	{0x578a, 0xfd},
+	{0x578b, 0xf5},
+	{0x578c, 0xf5},
+	{0x578d, 0x03},
+	{0x578e, 0x08},
+	{0x578f, 0x0c},
+	{0x5790, 0x08},
+	{0x5791, 0x06},
+	{0x5792, 0x00},
+	{0x5793, 0x52},
+	{0x5794, 0xa3},
+	{0x3503, 0x00}
+};
+
+static const struct ov5670_reg mode_640x360_regs[] = {
+	{0x3000, 0x00},
+	{0x3002, 0x21},
+	{0x3005, 0xf0},
+	{0x3007, 0x00},
+	{0x3015, 0x0f},
+	{0x3018, 0x32},
+	{0x301a, 0xf0},
+	{0x301b, 0xf0},
+	{0x301c, 0xf0},
+	{0x301d, 0xf0},
+	{0x301e, 0xf0},
+	{0x3030, 0x00},
+	{0x3031, 0x0a},
+	{0x303c, 0xff},
+	{0x303e, 0xff},
+	{0x3040, 0xf0},
+	{0x3041, 0x00},
+	{0x3042, 0xf0},
+	{0x3106, 0x11},
+	{0x3500, 0x00},
+	{0x3501, 0x80},
+	{0x3502, 0x00},
+	{0x3503, 0x04},
+	{0x3504, 0x03},
+	{0x3505, 0x83},
+	{0x3508, 0x04},
+	{0x3509, 0x00},
+	{0x350e, 0x04},
+	{0x350f, 0x00},
+	{0x3510, 0x00},
+	{0x3511, 0x02},
+	{0x3512, 0x00},
+	{0x3601, 0xc8},
+	{0x3610, 0x88},
+	{0x3612, 0x48},
+	{0x3614, 0x5b},
+	{0x3615, 0x96},
+	{0x3621, 0xd0},
+	{0x3622, 0x00},
+	{0x3623, 0x04},
+	{0x3633, 0x13},
+	{0x3634, 0x13},
+	{0x3635, 0x13},
+	{0x3636, 0x13},
+	{0x3645, 0x13},
+	{0x3646, 0x82},
+	{0x3650, 0x00},
+	{0x3652, 0xff},
+	{0x3655, 0x20},
+	{0x3656, 0xff},
+	{0x365a, 0xff},
+	{0x365e, 0xff},
+	{0x3668, 0x00},
+	{0x366a, 0x07},
+	{0x366e, 0x08},
+	{0x366d, 0x00},
+	{0x366f, 0x80},
+	{0x3700, 0x28},
+	{0x3701, 0x10},
+	{0x3702, 0x3a},
+	{0x3703, 0x19},
+	{0x3704, 0x10},
+	{0x3705, 0x00},
+	{0x3706, 0x66},
+	{0x3707, 0x08},
+	{0x3708, 0x34},
+	{0x3709, 0x40},
+	{0x370a, 0x01},
+	{0x370b, 0x1b},
+	{0x3714, 0x24},
+	{0x371a, 0x3e},
+	{0x3733, 0x00},
+	{0x3734, 0x00},
+	{0x373a, 0x05},
+	{0x373b, 0x06},
+	{0x373c, 0x0a},
+	{0x373f, 0xa0},
+	{0x3755, 0x00},
+	{0x3758, 0x00},
+	{0x375b, 0x0e},
+	{0x3766, 0x5f},
+	{0x3768, 0x00},
+	{0x3769, 0x22},
+	{0x3773, 0x08},
+	{0x3774, 0x1f},
+	{0x3776, 0x06},
+	{0x37a0, 0x88},
+	{0x37a1, 0x5c},
+	{0x37a7, 0x88},
+	{0x37a8, 0x70},
+	{0x37aa, 0x88},
+	{0x37ab, 0x48},
+	{0x37b3, 0x66},
+	{0x37c2, 0x04},
+	{0x37c5, 0x00},
+	{0x37c8, 0x00},
+	{0x3800, 0x00},
+	{0x3801, 0x0c},
+	{0x3802, 0x00},
+	{0x3803, 0x04},
+	{0x3804, 0x0a},
+	{0x3805, 0x33},
+	{0x3806, 0x07},
+	{0x3807, 0xa3},
+	{0x3808, 0x02},
+	{0x3809, 0x80},
+	{0x380a, 0x01},
+	{0x380b, 0x68},
+	{0x380c, 0x06},
+	{0x380d, 0x90},
+	{0x380e, 0x08},
+	{0x380f, 0x08},
+	{0x3811, 0x04},
+	{0x3813, 0x02},
+	{0x3814, 0x07},
+	{0x3815, 0x01},
+	{0x3816, 0x00},
+	{0x3817, 0x00},
+	{0x3818, 0x00},
+	{0x3819, 0x00},
+	{0x3820, 0x94},
+	{0x3821, 0xc6},
+	{0x3822, 0x48},
+	{0x3826, 0x00},
+	{0x3827, 0x08},
+	{0x382a, 0x07},
+	{0x382b, 0x01},
+	{0x3830, 0x08},
+	{0x3836, 0x02},
+	{0x3837, 0x00},
+	{0x3838, 0x10},
+	{0x3841, 0xff},
+	{0x3846, 0x48},
+	{0x3861, 0x00},
+	{0x3862, 0x04},
+	{0x3863, 0x06},
+	{0x3a11, 0x01},
+	{0x3a12, 0x78},
+	{0x3b00, 0x00},
+	{0x3b02, 0x00},
+	{0x3b03, 0x00},
+	{0x3b04, 0x00},
+	{0x3b05, 0x00},
+	{0x3c00, 0x89},
+	{0x3c01, 0xab},
+	{0x3c02, 0x01},
+	{0x3c03, 0x00},
+	{0x3c04, 0x00},
+	{0x3c05, 0x03},
+	{0x3c06, 0x00},
+	{0x3c07, 0x05},
+	{0x3c0c, 0x00},
+	{0x3c0d, 0x00},
+	{0x3c0e, 0x00},
+	{0x3c0f, 0x00},
+	{0x3c40, 0x00},
+	{0x3c41, 0xa3},
+	{0x3c43, 0x7d},
+	{0x3c45, 0xd7},
+	{0x3c47, 0xfc},
+	{0x3c50, 0x05},
+	{0x3c52, 0xaa},
+	{0x3c54, 0x71},
+	{0x3c56, 0x80},
+	{0x3d85, 0x17},
+	{0x3f03, 0x00},
+	{0x3f0a, 0x00},
+	{0x3f0b, 0x00},
+	{0x4001, 0x60},
+	{0x4009, 0x05},
+	{0x4020, 0x00},
+	{0x4021, 0x00},
+	{0x4022, 0x00},
+	{0x4023, 0x00},
+	{0x4024, 0x00},
+	{0x4025, 0x00},
+	{0x4026, 0x00},
+	{0x4027, 0x00},
+	{0x4028, 0x00},
+	{0x4029, 0x00},
+	{0x402a, 0x00},
+	{0x402b, 0x00},
+	{0x402c, 0x00},
+	{0x402d, 0x00},
+	{0x402e, 0x00},
+	{0x402f, 0x00},
+	{0x4040, 0x00},
+	{0x4041, 0x03},
+	{0x4042, 0x00},
+	{0x4043, 0x7A},
+	{0x4044, 0x00},
+	{0x4045, 0x7A},
+	{0x4046, 0x00},
+	{0x4047, 0x7A},
+	{0x4048, 0x00},
+	{0x4049, 0x7A},
+	{0x4307, 0x30},
+	{0x4500, 0x58},
+	{0x4501, 0x04},
+	{0x4502, 0x40},
+	{0x4503, 0x10},
+	{0x4508, 0x55},
+	{0x4509, 0x55},
+	{0x450a, 0x02},
+	{0x450b, 0x00},
+	{0x4600, 0x00},
+	{0x4601, 0x40},
+	{0x4700, 0xa4},
+	{0x4800, 0x4c},
+	{0x4816, 0x53},
+	{0x481f, 0x40},
+	{0x4837, 0x13},
+	{0x5000, 0x56},
+	{0x5001, 0x01},
+	{0x5002, 0x28},
+	{0x5004, 0x0c},
+	{0x5006, 0x0c},
+	{0x5007, 0xe0},
+	{0x5008, 0x01},
+	{0x5009, 0xb0},
+	{0x5901, 0x00},
+	{0x5a01, 0x00},
+	{0x5a03, 0x00},
+	{0x5a04, 0x0c},
+	{0x5a05, 0xe0},
+	{0x5a06, 0x09},
+	{0x5a07, 0xb0},
+	{0x5a08, 0x06},
+	{0x5e00, 0x00},
+	{0x3734, 0x40},
+	{0x5b00, 0x01},
+	{0x5b01, 0x10},
+	{0x5b02, 0x01},
+	{0x5b03, 0xdb},
+	{0x3d8c, 0x71},
+	{0x3d8d, 0xea},
+	{0x4017, 0x10},
+	{0x3618, 0x2a},
+	{0x5780, 0x3e},
+	{0x5781, 0x0f},
+	{0x5782, 0x44},
+	{0x5783, 0x02},
+	{0x5784, 0x01},
+	{0x5785, 0x01},
+	{0x5786, 0x00},
+	{0x5787, 0x04},
+	{0x5788, 0x02},
+	{0x5789, 0x0f},
+	{0x578a, 0xfd},
+	{0x578b, 0xf5},
+	{0x578c, 0xf5},
+	{0x578d, 0x03},
+	{0x578e, 0x08},
+	{0x578f, 0x0c},
+	{0x5790, 0x08},
+	{0x5791, 0x06},
+	{0x5792, 0x00},
+	{0x5793, 0x52},
+	{0x5794, 0xa3},
+	{0x3503, 0x00}
+};
+
+static const char * const ov5670_test_pattern_menu[] = {
+	"Disabled",
+	"Vertical Color Bar Type 1",
+};
+
+/* Supported link frequencies */
+#define OV5670_LINK_FREQ_422MHZ		422400000
+#define OV5670_LINK_FREQ_422MHZ_INDEX	0
+static const struct ov5670_link_freq_config link_freq_configs[] = {
+	{
+		/* pixel_rate = link_freq * 2 * nr_of_lanes / bits_per_sample */
+		.pixel_rate = (OV5670_LINK_FREQ_422MHZ * 2 * 2) / 10,
+		.reg_list = {
+			.num_of_regs = ARRAY_SIZE(mipi_data_rate_840mbps),
+			.regs = mipi_data_rate_840mbps,
+		}
+	}
+};
+
+static const s64 link_freq_menu_items[] = {
+	OV5670_LINK_FREQ_422MHZ
+};
+
+/*
+ * OV5670 sensor supports following resolutions with full FOV:
+ * 4:3  ==> {2592x1944, 1296x972, 648x486}
+ * 16:9 ==> {2560x1440, 1280x720, 640x360}
+ */
+static const struct ov5670_mode supported_modes[] = {
+	{
+		.width = 2592,
+		.height = 1944,
+		.vts_def = OV5670_VTS_30FPS,
+		.vts_min = OV5670_VTS_30FPS,
+		.reg_list = {
+			.num_of_regs = ARRAY_SIZE(mode_2592x1944_regs),
+			.regs = mode_2592x1944_regs,
+		},
+		.link_freq_index = OV5670_LINK_FREQ_422MHZ_INDEX,
+	},
+	{
+		.width = 1296,
+		.height = 972,
+		.vts_def = OV5670_VTS_30FPS,
+		.vts_min = 996,
+		.reg_list = {
+			.num_of_regs = ARRAY_SIZE(mode_1296x972_regs),
+			.regs = mode_1296x972_regs,
+		},
+		.link_freq_index = OV5670_LINK_FREQ_422MHZ_INDEX,
+	},
+	{
+		.width = 648,
+		.height = 486,
+		.vts_def = OV5670_VTS_30FPS,
+		.vts_min = 516,
+		.reg_list = {
+			.num_of_regs = ARRAY_SIZE(mode_648x486_regs),
+			.regs = mode_648x486_regs,
+		},
+		.link_freq_index = OV5670_LINK_FREQ_422MHZ_INDEX,
+	},
+	{
+		.width = 2560,
+		.height = 1440,
+		.vts_def = OV5670_VTS_30FPS,
+		.vts_min = OV5670_VTS_30FPS,
+		.reg_list = {
+			.num_of_regs = ARRAY_SIZE(mode_2560x1440_regs),
+			.regs = mode_2560x1440_regs,
+		},
+		.link_freq_index = OV5670_LINK_FREQ_422MHZ_INDEX,
+	},
+	{
+		.width = 1280,
+		.height = 720,
+		.vts_def = OV5670_VTS_30FPS,
+		.vts_min = 1020,
+		.reg_list = {
+			.num_of_regs = ARRAY_SIZE(mode_1280x720_regs),
+			.regs = mode_1280x720_regs,
+		},
+		.link_freq_index = OV5670_LINK_FREQ_422MHZ_INDEX,
+	},
+	{
+		.width = 640,
+		.height = 360,
+		.vts_def = OV5670_VTS_30FPS,
+		.vts_min = 510,
+		.reg_list = {
+			.num_of_regs = ARRAY_SIZE(mode_640x360_regs),
+			.regs = mode_640x360_regs,
+		},
+		.link_freq_index = OV5670_LINK_FREQ_422MHZ_INDEX,
+	}
+};
+
+struct ov5670 {
+	struct v4l2_subdev sd;
+	struct media_pad pad;
+
+	struct v4l2_ctrl_handler ctrl_handler;
+	/* V4L2 Controls */
+	struct v4l2_ctrl *link_freq;
+	struct v4l2_ctrl *pixel_rate;
+	struct v4l2_ctrl *vblank;
+	struct v4l2_ctrl *hblank;
+	struct v4l2_ctrl *exposure;
+
+	/* Current mode */
+	const struct ov5670_mode *cur_mode;
+
+	/* To serialize asynchronus callbacks */
+	struct mutex mutex;
+
+	/* Streaming on/off */
+	bool streaming;
+};
+
+#define to_ov5670(_sd)	container_of(_sd, struct ov5670, sd)
+
+/* Read registers up to 4 at a time */
+static int ov5670_read_reg(struct ov5670 *ov5670, u16 reg, unsigned int len,
+			   u32 *val)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(&ov5670->sd);
+	struct i2c_msg msgs[2];
+	u8 *data_be_p;
+	u32 data_be = 0;
+	u16 reg_addr_be = cpu_to_be16(reg);
+	int ret;
+
+	if (len > 4)
+		return -EINVAL;
+
+	data_be_p = (u8 *)&data_be;
+	/* Write register address */
+	msgs[0].addr = client->addr;
+	msgs[0].flags = 0;
+	msgs[0].len = 2;
+	msgs[0].buf = (u8 *)&reg_addr_be;
+
+	/* Read data from register */
+	msgs[1].addr = client->addr;
+	msgs[1].flags = I2C_M_RD;
+	msgs[1].len = len;
+	msgs[1].buf = &data_be_p[4 - len];
+
+	ret = i2c_transfer(client->adapter, msgs, ARRAY_SIZE(msgs));
+	if (ret != ARRAY_SIZE(msgs))
+		return -EIO;
+
+	*val = be32_to_cpu(data_be);
+
+	return 0;
+}
+
+/* Write registers up to 4 at a time */
+static int ov5670_write_reg(struct ov5670 *ov5670, u16 reg, unsigned int len,
+			    u32 val)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(&ov5670->sd);
+	int buf_i;
+	int val_i;
+	u8 buf[6];
+	u8 *val_p;
+
+	if (len > 4)
+		return -EINVAL;
+
+	buf[0] = reg >> 8;
+	buf[1] = reg & 0xff;
+
+	val = cpu_to_be32(val);
+	val_p = (u8 *)&val;
+	buf_i = 2;
+	val_i = 4 - len;
+
+	while (val_i < 4)
+		buf[buf_i++] = val_p[val_i++];
+
+	if (i2c_master_send(client, buf, len + 2) != len + 2)
+		return -EIO;
+
+	return 0;
+}
+
+/* Write a list of registers */
+static int ov5670_write_regs(struct ov5670 *ov5670,
+			     const struct ov5670_reg *regs, unsigned int len)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(&ov5670->sd);
+	unsigned int i;
+	int ret;
+
+	for (i = 0; i < len; i++) {
+		ret = ov5670_write_reg(ov5670, regs[i].address, 1, regs[i].val);
+		if (ret) {
+			dev_err_ratelimited(
+				&client->dev,
+				"Failed to write reg 0x%4.4x. error = %d\n",
+				regs[i].address, ret);
+
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int ov5670_write_reg_list(struct ov5670 *ov5670,
+				 const struct ov5670_reg_list *r_list)
+{
+	return ov5670_write_regs(ov5670, r_list->regs, r_list->num_of_regs);
+}
+
+/* Open sub-device */
+static int ov5670_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
+{
+	struct ov5670 *ov5670 = to_ov5670(sd);
+	struct v4l2_mbus_framefmt *try_fmt =
+				v4l2_subdev_get_try_format(sd, fh->pad, 0);
+
+	mutex_lock(&ov5670->mutex);
+
+	/* Initialize try_fmt */
+	try_fmt->width = ov5670->cur_mode->width;
+	try_fmt->height = ov5670->cur_mode->height;
+	try_fmt->code = MEDIA_BUS_FMT_SGRBG10_1X10;
+	try_fmt->field = V4L2_FIELD_NONE;
+
+	/* No crop or compose */
+	mutex_unlock(&ov5670->mutex);
+
+	return 0;
+}
+
+static int ov5670_update_digital_gain(struct ov5670 *ov5670, u32 d_gain)
+{
+	int ret;
+
+	ret = ov5670_write_reg(ov5670, OV5670_REG_R_DGTL_GAIN,
+			       OV5670_REG_VALUE_16BIT, d_gain);
+	if (ret)
+		return ret;
+
+	ret = ov5670_write_reg(ov5670, OV5670_REG_G_DGTL_GAIN,
+			       OV5670_REG_VALUE_16BIT, d_gain);
+	if (ret)
+		return ret;
+
+	return ov5670_write_reg(ov5670, OV5670_REG_B_DGTL_GAIN,
+				OV5670_REG_VALUE_16BIT, d_gain);
+}
+
+static int ov5670_enable_test_pattern(struct ov5670 *ov5670, u32 pattern)
+{
+	u32 val;
+	int ret;
+
+	/* Set the bayer order that we support */
+	ret = ov5670_write_reg(ov5670, OV5670_REG_TEST_PATTERN_CTRL,
+			       OV5670_REG_VALUE_08BIT, 0);
+	if (ret)
+		return ret;
+
+	ret = ov5670_read_reg(ov5670, OV5670_REG_TEST_PATTERN,
+			      OV5670_REG_VALUE_08BIT, &val);
+	if (ret)
+		return ret;
+
+	if (pattern)
+		val |= OV5670_TEST_PATTERN_ENABLE;
+	else
+		val &= ~OV5670_TEST_PATTERN_ENABLE;
+
+	return ov5670_write_reg(ov5670, OV5670_REG_TEST_PATTERN,
+				OV5670_REG_VALUE_08BIT, val);
+}
+
+/* Initialize control handlers */
+static int ov5670_set_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct ov5670 *ov5670 = container_of(ctrl->handler,
+					     struct ov5670, ctrl_handler);
+	struct i2c_client *client = v4l2_get_subdevdata(&ov5670->sd);
+	s64 max;
+	int ret = 0;
+
+	/* Propagate change of current control to all related controls */
+	switch (ctrl->id) {
+	case V4L2_CID_VBLANK:
+		/* Update max exposure while meeting expected vblanking */
+		max = ov5670->cur_mode->height + ctrl->val - 8;
+		__v4l2_ctrl_modify_range(ov5670->exposure,
+					 ov5670->exposure->minimum, max,
+					 ov5670->exposure->step, max);
+		break;
+	}
+
+	/* V4L2 controls values will be applied only when power is already up */
+	if (pm_runtime_get_if_in_use(&client->dev) <= 0)
+		return 0;
+
+	switch (ctrl->id) {
+	case V4L2_CID_ANALOGUE_GAIN:
+		ret = ov5670_write_reg(ov5670, OV5670_REG_ANALOG_GAIN,
+				       OV5670_REG_VALUE_16BIT, ctrl->val);
+		break;
+	case V4L2_CID_DIGITAL_GAIN:
+		ret = ov5670_update_digital_gain(ov5670, ctrl->val);
+		break;
+	case V4L2_CID_EXPOSURE:
+		/* 4 least significant bits of expsoure are fractional part */
+		ret = ov5670_write_reg(ov5670, OV5670_REG_EXPOSURE,
+				       OV5670_REG_VALUE_24BIT, ctrl->val << 4);
+		break;
+	case V4L2_CID_VBLANK:
+		/* Update VTS that meets expected vertical blanking */
+		ret = ov5670_write_reg(ov5670, OV5670_REG_VTS,
+				       OV5670_REG_VALUE_16BIT,
+				       ov5670->cur_mode->height + ctrl->val);
+		break;
+	case V4L2_CID_TEST_PATTERN:
+		ret = ov5670_enable_test_pattern(ov5670, ctrl->val);
+		break;
+	default:
+		dev_info(&client->dev, "%s Unhandled id:0x%x, val:0x%x\n",
+			 __func__, ctrl->id, ctrl->val);
+		break;
+	}
+
+	pm_runtime_put(&client->dev);
+
+	return ret;
+}
+
+static const struct v4l2_ctrl_ops ov5670_ctrl_ops = {
+	.s_ctrl = ov5670_set_ctrl,
+};
+
+/* Initialize control handlers */
+static int ov5670_init_controls(struct ov5670 *ov5670)
+{
+	struct v4l2_ctrl_handler *ctrl_hdlr;
+	s64 vblank_max;
+	s64 vblank_def;
+	s64 vblank_min;
+	s64 exposure_max;
+	int ret;
+
+	ctrl_hdlr = &ov5670->ctrl_handler;
+	ret = v4l2_ctrl_handler_init(ctrl_hdlr, 8);
+	if (ret)
+		return ret;
+
+	ctrl_hdlr->lock = &ov5670->mutex;
+	ov5670->link_freq = v4l2_ctrl_new_int_menu(ctrl_hdlr,
+						   &ov5670_ctrl_ops,
+						   V4L2_CID_LINK_FREQ,
+						   0, 0, link_freq_menu_items);
+	if (ov5670->link_freq)
+		ov5670->link_freq->flags |= V4L2_CTRL_FLAG_READ_ONLY;
+
+	/* By default, V4L2_CID_PIXEL_RATE is read only */
+	ov5670->pixel_rate = v4l2_ctrl_new_std(ctrl_hdlr, &ov5670_ctrl_ops,
+					       V4L2_CID_PIXEL_RATE, 0,
+					       link_freq_configs[0].pixel_rate,
+					       1,
+					       link_freq_configs[0].pixel_rate);
+
+	vblank_max = OV5670_VTS_MAX - ov5670->cur_mode->height;
+	vblank_def = ov5670->cur_mode->vts_def - ov5670->cur_mode->height;
+	vblank_min = ov5670->cur_mode->vts_min - ov5670->cur_mode->height;
+	ov5670->vblank = v4l2_ctrl_new_std(ctrl_hdlr, &ov5670_ctrl_ops,
+					   V4L2_CID_VBLANK, vblank_min,
+					   vblank_max, 1, vblank_def);
+
+	ov5670->hblank = v4l2_ctrl_new_std(
+				ctrl_hdlr, &ov5670_ctrl_ops, V4L2_CID_HBLANK,
+				OV5670_FIXED_PPL - ov5670->cur_mode->width,
+				OV5670_FIXED_PPL - ov5670->cur_mode->width, 1,
+				OV5670_FIXED_PPL - ov5670->cur_mode->width);
+	if (ov5670->hblank)
+		ov5670->hblank->flags |= V4L2_CTRL_FLAG_READ_ONLY;
+
+	/* Get min, max, step, default from sensor */
+	v4l2_ctrl_new_std(ctrl_hdlr, &ov5670_ctrl_ops, V4L2_CID_ANALOGUE_GAIN,
+			  ANALOG_GAIN_MIN, ANALOG_GAIN_MAX, ANALOG_GAIN_STEP,
+			  ANALOG_GAIN_DEFAULT);
+
+	/* Digital gain */
+	v4l2_ctrl_new_std(ctrl_hdlr, &ov5670_ctrl_ops, V4L2_CID_DIGITAL_GAIN,
+			  OV5670_DGTL_GAIN_MIN, OV5670_DGTL_GAIN_MAX,
+			  OV5670_DGTL_GAIN_STEP, OV5670_DGTL_GAIN_DEFAULT);
+
+	/* Get min, max, step, default from sensor */
+	exposure_max = ov5670->cur_mode->vts_def - 8;
+	ov5670->exposure = v4l2_ctrl_new_std(ctrl_hdlr, &ov5670_ctrl_ops,
+					     V4L2_CID_EXPOSURE,
+					     OV5670_EXPOSURE_MIN,
+					     exposure_max, OV5670_EXPOSURE_STEP,
+					     exposure_max);
+
+	v4l2_ctrl_new_std_menu_items(ctrl_hdlr, &ov5670_ctrl_ops,
+				     V4L2_CID_TEST_PATTERN,
+				     ARRAY_SIZE(ov5670_test_pattern_menu) - 1,
+				     0, 0, ov5670_test_pattern_menu);
+
+	if (ctrl_hdlr->error) {
+		ret = ctrl_hdlr->error;
+		goto error;
+	}
+
+	ov5670->sd.ctrl_handler = ctrl_hdlr;
+
+	return 0;
+
+error:
+	v4l2_ctrl_handler_free(ctrl_hdlr);
+
+	return ret;
+}
+
+static int ov5670_enum_mbus_code(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_pad_config *cfg,
+				 struct v4l2_subdev_mbus_code_enum *code)
+{
+	/* Only one bayer order GRBG is supported */
+	if (code->index > 0)
+		return -EINVAL;
+
+	code->code = MEDIA_BUS_FMT_SGRBG10_1X10;
+
+	return 0;
+}
+
+static int ov5670_enum_frame_size(struct v4l2_subdev *sd,
+				  struct v4l2_subdev_pad_config *cfg,
+				  struct v4l2_subdev_frame_size_enum *fse)
+{
+	if (fse->index >= ARRAY_SIZE(supported_modes))
+		return -EINVAL;
+
+	if (fse->code != MEDIA_BUS_FMT_SGRBG10_1X10)
+		return -EINVAL;
+
+	fse->min_width = supported_modes[fse->index].width;
+	fse->max_width = fse->min_width;
+	fse->min_height = supported_modes[fse->index].height;
+	fse->max_height = fse->min_height;
+
+	return 0;
+}
+
+/* Calculate resolution distance */
+static int ov5670_get_reso_dist(const struct ov5670_mode *mode,
+				struct v4l2_mbus_framefmt *framefmt)
+{
+	return abs(mode->width - framefmt->width) +
+	       abs(mode->height - framefmt->height);
+}
+
+/* Find the closest supported resolution to the requested resolution */
+static const struct ov5670_mode *ov5670_find_best_fit(
+						struct ov5670 *ov5670,
+						struct v4l2_subdev_format *fmt)
+{
+	struct v4l2_mbus_framefmt *framefmt = &fmt->format;
+	int dist;
+	int cur_best_fit = 0;
+	int cur_best_fit_dist = -1;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(supported_modes); i++) {
+		dist = ov5670_get_reso_dist(&supported_modes[i], framefmt);
+		if (cur_best_fit_dist == -1 || dist < cur_best_fit_dist) {
+			cur_best_fit_dist = dist;
+			cur_best_fit = i;
+		}
+	}
+
+	return &supported_modes[cur_best_fit];
+}
+
+static void ov5670_update_pad_format(const struct ov5670_mode *mode,
+				     struct v4l2_subdev_format *fmt)
+{
+	fmt->format.width = mode->width;
+	fmt->format.height = mode->height;
+	fmt->format.code = MEDIA_BUS_FMT_SGRBG10_1X10;
+	fmt->format.field = V4L2_FIELD_NONE;
+}
+
+static int ov5670_do_get_pad_format(struct ov5670 *ov5670,
+				    struct v4l2_subdev_pad_config *cfg,
+				    struct v4l2_subdev_format *fmt)
+{
+	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY)
+		fmt->format = *v4l2_subdev_get_try_format(&ov5670->sd, cfg,
+							  fmt->pad);
+	else
+		ov5670_update_pad_format(ov5670->cur_mode, fmt);
+
+	return 0;
+}
+
+static int ov5670_get_pad_format(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_pad_config *cfg,
+				 struct v4l2_subdev_format *fmt)
+{
+	struct ov5670 *ov5670 = to_ov5670(sd);
+	int ret;
+
+	mutex_lock(&ov5670->mutex);
+	ret = ov5670_do_get_pad_format(ov5670, cfg, fmt);
+	mutex_unlock(&ov5670->mutex);
+
+	return ret;
+}
+
+static int ov5670_set_pad_format(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_pad_config *cfg,
+				 struct v4l2_subdev_format *fmt)
+{
+	struct ov5670 *ov5670 = to_ov5670(sd);
+	const struct ov5670_mode *mode;
+	s32 vblank_def;
+	s32 h_blank;
+
+	mutex_lock(&ov5670->mutex);
+
+	fmt->format.code = MEDIA_BUS_FMT_SGRBG10_1X10;
+
+	mode = ov5670_find_best_fit(ov5670, fmt);
+	ov5670_update_pad_format(mode, fmt);
+	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) {
+		*v4l2_subdev_get_try_format(sd, cfg, fmt->pad) = fmt->format;
+	} else {
+		ov5670->cur_mode = mode;
+		__v4l2_ctrl_s_ctrl(ov5670->link_freq, mode->link_freq_index);
+		__v4l2_ctrl_s_ctrl_int64(
+			ov5670->pixel_rate,
+			link_freq_configs[mode->link_freq_index].pixel_rate);
+		/* Update limits and set FPS to default */
+		vblank_def = ov5670->cur_mode->vts_def -
+			     ov5670->cur_mode->height;
+		__v4l2_ctrl_modify_range(
+			ov5670->vblank,
+			ov5670->cur_mode->vts_min - ov5670->cur_mode->height,
+			OV5670_VTS_MAX - ov5670->cur_mode->height, 1,
+			vblank_def);
+		__v4l2_ctrl_s_ctrl(ov5670->vblank, vblank_def);
+		h_blank = OV5670_FIXED_PPL - ov5670->cur_mode->width;
+		__v4l2_ctrl_modify_range(ov5670->hblank, h_blank, h_blank, 1,
+					 h_blank);
+	}
+
+	mutex_unlock(&ov5670->mutex);
+
+	return 0;
+}
+
+static int ov5670_get_skip_frames(struct v4l2_subdev *sd, u32 *frames)
+{
+	*frames = OV5670_NUM_OF_SKIP_FRAMES;
+
+	return 0;
+}
+
+/* Prepare streaming by writing default values and customized values */
+static int ov5670_start_streaming(struct ov5670 *ov5670)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(&ov5670->sd);
+	const struct ov5670_reg_list *reg_list;
+	int link_freq_index;
+	int ret;
+
+	/* Get out of from software reset */
+	ret = ov5670_write_reg(ov5670, OV5670_REG_SOFTWARE_RST,
+			       OV5670_REG_VALUE_08BIT, OV5670_SOFTWARE_RST);
+	if (ret) {
+		dev_err(&client->dev, "%s failed to set powerup registers\n",
+			__func__);
+		return ret;
+	}
+
+	/* Setup PLL */
+	link_freq_index = ov5670->cur_mode->link_freq_index;
+	reg_list = &link_freq_configs[link_freq_index].reg_list;
+	ret = ov5670_write_reg_list(ov5670, reg_list);
+	if (ret) {
+		dev_err(&client->dev, "%s failed to set plls\n", __func__);
+		return ret;
+	}
+
+	/* Apply default values of current mode */
+	reg_list = &ov5670->cur_mode->reg_list;
+	ret = ov5670_write_reg_list(ov5670, reg_list);
+	if (ret) {
+		dev_err(&client->dev, "%s failed to set mode\n", __func__);
+		return ret;
+	}
+
+	ret = __v4l2_ctrl_handler_setup(ov5670->sd.ctrl_handler);
+	if (ret)
+		return ret;
+
+	/* Write stream on list */
+	ret = ov5670_write_reg(ov5670, OV5670_REG_MODE_SELECT,
+			       OV5670_REG_VALUE_08BIT, OV5670_MODE_STREAMING);
+	if (ret) {
+		dev_err(&client->dev, "%s failed to set stream\n", __func__);
+		return ret;
+	}
+
+	ov5670->streaming = true;
+
+	return 0;
+}
+
+static int ov5670_stop_streaming(struct ov5670 *ov5670)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(&ov5670->sd);
+	int ret;
+
+	ret = ov5670_write_reg(ov5670, OV5670_REG_MODE_SELECT,
+			       OV5670_REG_VALUE_08BIT, OV5670_MODE_STANDBY);
+	if (ret)
+		dev_err(&client->dev, "%s failed to set stream\n", __func__);
+
+	ov5670->streaming = false;
+
+	/* Return success even if it was an error, as there is nothing the
+	 * caller can do about it.
+	 */
+	return 0;
+}
+
+static int ov5670_set_stream(struct v4l2_subdev *sd, int enable)
+{
+	struct ov5670 *ov5670 = to_ov5670(sd);
+	struct i2c_client *client = v4l2_get_subdevdata(sd);
+	int ret = 0;
+
+	mutex_lock(&ov5670->mutex);
+	if (ov5670->streaming == enable)
+		goto unlock_and_return;
+
+	if (enable) {
+		ret = pm_runtime_get_sync(&client->dev);
+		if (ret < 0) {
+			pm_runtime_put_noidle(&client->dev);
+			goto unlock_and_return;
+		}
+
+		ret = ov5670_start_streaming(ov5670);
+		if (ret)
+			goto error;
+	} else {
+		ret = ov5670_stop_streaming(ov5670);
+		pm_runtime_put(&client->dev);
+	}
+	goto unlock_and_return;
+
+error:
+	pm_runtime_put(&client->dev);
+
+unlock_and_return:
+	mutex_unlock(&ov5670->mutex);
+
+	return ret;
+}
+
+static int __maybe_unused ov5670_suspend(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct v4l2_subdev *sd = i2c_get_clientdata(client);
+	struct ov5670 *ov5670 = to_ov5670(sd);
+
+	if (ov5670->streaming)
+		ov5670_stop_streaming(ov5670);
+
+	return 0;
+}
+
+static int __maybe_unused ov5670_resume(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct v4l2_subdev *sd = i2c_get_clientdata(client);
+	struct ov5670 *ov5670 = to_ov5670(sd);
+	int ret;
+
+	if (ov5670->streaming) {
+		ret = ov5670_start_streaming(ov5670);
+		if (ret) {
+			ov5670_stop_streaming(ov5670);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+/* Verify chip ID */
+static int ov5670_identify_module(struct ov5670 *ov5670)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(&ov5670->sd);
+	int ret;
+	u32 val;
+
+	ret = ov5670_read_reg(ov5670, OV5670_REG_CHIP_ID,
+			      OV5670_REG_VALUE_24BIT, &val);
+	if (ret)
+		return ret;
+
+	if (val != OV5670_CHIP_ID) {
+		dev_err(&client->dev, "chip id mismatch: %x!=%x\n",
+			OV5670_CHIP_ID, val);
+		return -ENXIO;
+	}
+
+	return 0;
+}
+
+static const struct v4l2_subdev_video_ops ov5670_video_ops = {
+	.s_stream = ov5670_set_stream,
+};
+
+static const struct v4l2_subdev_pad_ops ov5670_pad_ops = {
+	.enum_mbus_code = ov5670_enum_mbus_code,
+	.get_fmt = ov5670_get_pad_format,
+	.set_fmt = ov5670_set_pad_format,
+	.enum_frame_size = ov5670_enum_frame_size,
+};
+
+static const struct v4l2_subdev_sensor_ops ov5670_sensor_ops = {
+	.g_skip_frames = ov5670_get_skip_frames,
+};
+
+static const struct v4l2_subdev_ops ov5670_subdev_ops = {
+	.video = &ov5670_video_ops,
+	.pad = &ov5670_pad_ops,
+	.sensor = &ov5670_sensor_ops,
+};
+
+static const struct media_entity_operations ov5670_subdev_entity_ops = {
+	.link_validate = v4l2_subdev_link_validate,
+};
+
+static const struct v4l2_subdev_internal_ops ov5670_internal_ops = {
+	.open = ov5670_open,
+};
+
+static int ov5670_probe(struct i2c_client *client)
+{
+	struct ov5670 *ov5670;
+	const char *err_msg;
+	u32 input_clk = 0;
+	int ret;
+
+	device_property_read_u32(&client->dev, "clock-frequency", &input_clk);
+	if (input_clk != 19200000)
+		return -EINVAL;
+
+	ov5670 = devm_kzalloc(&client->dev, sizeof(*ov5670), GFP_KERNEL);
+	if (!ov5670) {
+		ret = -ENOMEM;
+		err_msg = "devm_kzalloc() error";
+		goto error_print;
+	}
+
+	/* Initialize subdev */
+	v4l2_i2c_subdev_init(&ov5670->sd, client, &ov5670_subdev_ops);
+
+	/* Check module identity */
+	ret = ov5670_identify_module(ov5670);
+	if (ret) {
+		err_msg = "ov5670_identify_module() error";
+		goto error_print;
+	}
+
+	mutex_init(&ov5670->mutex);
+
+	/* Set default mode to max resolution */
+	ov5670->cur_mode = &supported_modes[0];
+
+	ret = ov5670_init_controls(ov5670);
+	if (ret) {
+		err_msg = "ov5670_init_controls() error";
+		goto error_mutex_destroy;
+	}
+
+	ov5670->sd.internal_ops = &ov5670_internal_ops;
+	ov5670->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+	ov5670->sd.entity.ops = &ov5670_subdev_entity_ops;
+	ov5670->sd.entity.function = MEDIA_ENT_F_CAM_SENSOR;
+
+	/* Source pad initialization */
+	ov5670->pad.flags = MEDIA_PAD_FL_SOURCE;
+	ret = media_entity_pads_init(&ov5670->sd.entity, 1, &ov5670->pad);
+	if (ret) {
+		err_msg = "media_entity_pads_init() error";
+		goto error_handler_free;
+	}
+
+	/* Async register for subdev */
+	ret = v4l2_async_register_subdev(&ov5670->sd);
+	if (ret < 0) {
+		err_msg = "v4l2_async_register_subdev() error";
+		goto error_entity_cleanup;
+	}
+
+	ov5670->streaming = false;
+
+	/*
+	 * Device is already turned on by i2c-core with ACPI domain PM.
+	 * Enable runtime PM and turn off the device.
+	 */
+	pm_runtime_get_noresume(&client->dev);
+	pm_runtime_set_active(&client->dev);
+	pm_runtime_enable(&client->dev);
+	pm_runtime_put(&client->dev);
+
+	return 0;
+
+error_entity_cleanup:
+	media_entity_cleanup(&ov5670->sd.entity);
+
+error_handler_free:
+	v4l2_ctrl_handler_free(ov5670->sd.ctrl_handler);
+
+error_mutex_destroy:
+	mutex_destroy(&ov5670->mutex);
+
+error_print:
+	dev_err(&client->dev, "%s: %s %d\n", __func__, err_msg, ret);
+
+	return ret;
+}
+
+static int ov5670_remove(struct i2c_client *client)
+{
+	struct v4l2_subdev *sd = i2c_get_clientdata(client);
+	struct ov5670 *ov5670 = to_ov5670(sd);
+
+	v4l2_async_unregister_subdev(sd);
+	media_entity_cleanup(&sd->entity);
+	v4l2_ctrl_handler_free(sd->ctrl_handler);
+	mutex_destroy(&ov5670->mutex);
+
+	/*
+	 * Disable runtime PM but keep the device turned on.
+	 * i2c-core with ACPI domain PM will turn off the device.
+	 */
+	pm_runtime_get_sync(&client->dev);
+	pm_runtime_disable(&client->dev);
+	pm_runtime_set_suspended(&client->dev);
+	pm_runtime_put_noidle(&client->dev);
+
+	return 0;
+}
+
+static const struct dev_pm_ops ov5670_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(ov5670_suspend, ov5670_resume)
+};
+
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id ov5670_acpi_ids[] = {
+	{"INT3479"},
+	{ /* sentinel */ }
+};
+
+MODULE_DEVICE_TABLE(acpi, ov5670_acpi_ids);
+#endif
+
+static struct i2c_driver ov5670_i2c_driver = {
+	.driver = {
+		.name = "ov5670",
+		.pm = &ov5670_pm_ops,
+		.acpi_match_table = ACPI_PTR(ov5670_acpi_ids),
+	},
+	.probe_new = ov5670_probe,
+	.remove = ov5670_remove,
+};
+
+module_i2c_driver(ov5670_i2c_driver);
+
+MODULE_AUTHOR("Rapolu, Chiranjeevi <chiranjeevi.rapolu@intel.com>");
+MODULE_AUTHOR("Yang, Hyungwoo <hyungwoo.yang@intel.com>");
+MODULE_DESCRIPTION("Omnivision ov5670 sensor driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/i2c/soc_camera/ov6650.c b/drivers/media/i2c/ov6650.c
index d2be64d54b22..768f2950ea36 100644
--- a/drivers/media/i2c/soc_camera/ov6650.c
+++ b/drivers/media/i2c/ov6650.c
@@ -1,5 +1,5 @@
 /*
- * V4L2 SoC Camera driver for OmniVision OV6650 Camera Sensor
+ * V4L2 subdevice driver for OmniVision OV6650 Camera Sensor
  *
  * Copyright (C) 2010 Janusz Krzysztofik <jkrzyszt@tis.icnet.pl>
  *
@@ -31,9 +31,9 @@
 #include <linux/v4l2-mediabus.h>
 #include <linux/module.h>
 
-#include <media/soc_camera.h>
 #include <media/v4l2-clk.h>
 #include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
 
 /* Register definitions */
 #define REG_GAIN		0x00	/* range 00 - 3F */
@@ -426,10 +426,15 @@ static int ov6650_set_register(struct v4l2_subdev *sd,
 static int ov6650_s_power(struct v4l2_subdev *sd, int on)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct soc_camera_subdev_desc *ssdd = soc_camera_i2c_to_desc(client);
 	struct ov6650 *priv = to_ov6650(client);
+	int ret = 0;
 
-	return soc_camera_set_power(&client->dev, ssdd, priv->clk, on);
+	if (on)
+		ret = v4l2_clk_enable(priv->clk);
+	else
+		v4l2_clk_disable(priv->clk);
+
+	return ret;
 }
 
 static int ov6650_get_selection(struct v4l2_subdev *sd,
@@ -471,14 +476,13 @@ static int ov6650_set_selection(struct v4l2_subdev *sd,
 	    sel->target != V4L2_SEL_TGT_CROP)
 		return -EINVAL;
 
-	rect.left   = ALIGN(rect.left,   2);
-	rect.width  = ALIGN(rect.width,  2);
-	rect.top    = ALIGN(rect.top,    2);
-	rect.height = ALIGN(rect.height, 2);
-	soc_camera_limit_side(&rect.left, &rect.width,
-			DEF_HSTRT << 1, 2, W_CIF);
-	soc_camera_limit_side(&rect.top, &rect.height,
-			DEF_VSTRT << 1, 2, H_CIF);
+	v4l_bound_align_image(&rect.width, 2, W_CIF, 1,
+			      &rect.height, 2, H_CIF, 1, 0);
+	v4l_bound_align_image(&rect.left, DEF_HSTRT << 1,
+			      (DEF_HSTRT << 1) + W_CIF - (__s32)rect.width, 1,
+			      &rect.top, DEF_VSTRT << 1,
+			      (DEF_VSTRT << 1) + H_CIF - (__s32)rect.height, 1,
+			      0);
 
 	ret = ov6650_reg_write(client, REG_HSTRT, rect.left >> 1);
 	if (!ret) {
@@ -547,8 +551,6 @@ static u8 to_clkrc(struct v4l2_fract *timeperframe,
 static int ov6650_s_fmt(struct v4l2_subdev *sd, struct v4l2_mbus_framefmt *mf)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct soc_camera_device *icd = v4l2_get_subdev_hostdata(sd);
-	struct soc_camera_sense *sense = icd->sense;
 	struct ov6650 *priv = to_ov6650(client);
 	bool half_scale = !is_unscaled_ok(mf->width, mf->height, &priv->rect);
 	struct v4l2_subdev_selection sel = {
@@ -640,32 +642,10 @@ static int ov6650_s_fmt(struct v4l2_subdev *sd, struct v4l2_mbus_framefmt *mf)
 	}
 	priv->half_scale = half_scale;
 
-	if (sense) {
-		if (sense->master_clock == 8000000) {
-			dev_dbg(&client->dev, "8MHz input clock\n");
-			clkrc = CLKRC_6MHz;
-		} else if (sense->master_clock == 12000000) {
-			dev_dbg(&client->dev, "12MHz input clock\n");
-			clkrc = CLKRC_12MHz;
-		} else if (sense->master_clock == 16000000) {
-			dev_dbg(&client->dev, "16MHz input clock\n");
-			clkrc = CLKRC_16MHz;
-		} else if (sense->master_clock == 24000000) {
-			dev_dbg(&client->dev, "24MHz input clock\n");
-			clkrc = CLKRC_24MHz;
-		} else {
-			dev_err(&client->dev,
-				"unsupported input clock, check platform data\n");
-			return -EINVAL;
-		}
-		mclk = sense->master_clock;
-		priv->pclk_limit = sense->pixel_clock_max;
-	} else {
-		clkrc = CLKRC_24MHz;
-		mclk = 24000000;
-		priv->pclk_limit = 0;
-		dev_dbg(&client->dev, "using default 24MHz input clock\n");
-	}
+	clkrc = CLKRC_12MHz;
+	mclk = 12000000;
+	priv->pclk_limit = 1334000;
+	dev_dbg(&client->dev, "using 12MHz input clock\n");
 
 	clkrc |= to_clkrc(&priv->tpf, priv->pclk_limit, priv->pclk_max);
 
@@ -899,8 +879,6 @@ static const struct v4l2_subdev_core_ops ov6650_core_ops = {
 static int ov6650_g_mbus_config(struct v4l2_subdev *sd,
 				struct v4l2_mbus_config *cfg)
 {
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct soc_camera_subdev_desc *ssdd = soc_camera_i2c_to_desc(client);
 
 	cfg->flags = V4L2_MBUS_MASTER |
 		V4L2_MBUS_PCLK_SAMPLE_RISING | V4L2_MBUS_PCLK_SAMPLE_FALLING |
@@ -908,7 +886,6 @@ static int ov6650_g_mbus_config(struct v4l2_subdev *sd,
 		V4L2_MBUS_VSYNC_ACTIVE_HIGH | V4L2_MBUS_VSYNC_ACTIVE_LOW |
 		V4L2_MBUS_DATA_ACTIVE_HIGH;
 	cfg->type = V4L2_MBUS_PARALLEL;
-	cfg->flags = soc_camera_apply_board_flags(ssdd, cfg);
 
 	return 0;
 }
@@ -918,25 +895,23 @@ static int ov6650_s_mbus_config(struct v4l2_subdev *sd,
 				const struct v4l2_mbus_config *cfg)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	struct soc_camera_subdev_desc *ssdd = soc_camera_i2c_to_desc(client);
-	unsigned long flags = soc_camera_apply_board_flags(ssdd, cfg);
 	int ret;
 
-	if (flags & V4L2_MBUS_PCLK_SAMPLE_RISING)
+	if (cfg->flags & V4L2_MBUS_PCLK_SAMPLE_RISING)
 		ret = ov6650_reg_rmw(client, REG_COMJ, COMJ_PCLK_RISING, 0);
 	else
 		ret = ov6650_reg_rmw(client, REG_COMJ, 0, COMJ_PCLK_RISING);
 	if (ret)
 		return ret;
 
-	if (flags & V4L2_MBUS_HSYNC_ACTIVE_LOW)
+	if (cfg->flags & V4L2_MBUS_HSYNC_ACTIVE_LOW)
 		ret = ov6650_reg_rmw(client, REG_COMF, COMF_HREF_LOW, 0);
 	else
 		ret = ov6650_reg_rmw(client, REG_COMF, 0, COMF_HREF_LOW);
 	if (ret)
 		return ret;
 
-	if (flags & V4L2_MBUS_VSYNC_ACTIVE_HIGH)
+	if (cfg->flags & V4L2_MBUS_VSYNC_ACTIVE_HIGH)
 		ret = ov6650_reg_rmw(client, REG_COMJ, COMJ_VSYNC_HIGH, 0);
 	else
 		ret = ov6650_reg_rmw(client, REG_COMJ, 0, COMJ_VSYNC_HIGH);
@@ -973,14 +948,8 @@ static int ov6650_probe(struct i2c_client *client,
 			const struct i2c_device_id *did)
 {
 	struct ov6650 *priv;
-	struct soc_camera_subdev_desc *ssdd = soc_camera_i2c_to_desc(client);
 	int ret;
 
-	if (!ssdd) {
-		dev_err(&client->dev, "Missing platform_data for driver\n");
-		return -EINVAL;
-	}
-
 	priv = devm_kzalloc(&client->dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv) {
 		dev_err(&client->dev,
diff --git a/drivers/media/i2c/ov7670.c b/drivers/media/i2c/ov7670.c
index 7270c68ed18a..e88549f0e704 100644
--- a/drivers/media/i2c/ov7670.c
+++ b/drivers/media/i2c/ov7670.c
@@ -1614,8 +1614,10 @@ static int ov7670_probe(struct i2c_client *client,
 
 	info->clk = devm_clk_get(&client->dev, "xclk");
 	if (IS_ERR(info->clk))
-		return -EPROBE_DEFER;
-	clk_prepare_enable(info->clk);
+		return PTR_ERR(info->clk);
+	ret = clk_prepare_enable(info->clk);
+	if (ret)
+		return ret;
 
 	ret = ov7670_init_gpio(client, info);
 	if (ret)
diff --git a/drivers/media/i2c/ov9650.c b/drivers/media/i2c/ov9650.c
index 2de2fbb13b85..6ffb460e8589 100644
--- a/drivers/media/i2c/ov9650.c
+++ b/drivers/media/i2c/ov9650.c
@@ -484,6 +484,7 @@ static int ov965x_set_default_gamma_curve(struct ov965x *ov965x)
 
 	for (i = 0; i < ARRAY_SIZE(gamma_curve); i++) {
 		int ret = ov965x_write(ov965x->client, addr, gamma_curve[i]);
+
 		if (ret < 0)
 			return ret;
 		addr++;
@@ -503,6 +504,7 @@ static int ov965x_set_color_matrix(struct ov965x *ov965x)
 
 	for (i = 0; i < ARRAY_SIZE(mtx); i++) {
 		int ret = ov965x_write(ov965x->client, addr, mtx[i]);
+
 		if (ret < 0)
 			return ret;
 		addr++;
@@ -611,7 +613,7 @@ static int ov965x_set_banding_filter(struct ov965x *ov965x, int value)
 	}
 	if (value == V4L2_CID_POWER_LINE_FREQUENCY_DISABLED)
 		return 0;
-	if (WARN_ON(ov965x->fiv == NULL))
+	if (WARN_ON(!ov965x->fiv))
 		return -EINVAL;
 	/* Set minimal exposure time for 50/60 HZ lighting */
 	if (value == V4L2_CID_POWER_LINE_FREQUENCY_50HZ)
@@ -999,44 +1001,47 @@ static int ov965x_initialize_controls(struct ov965x *ov965x)
 
 	/* Auto/manual white balance */
 	ctrls->auto_wb = v4l2_ctrl_new_std(hdl, ops,
-				V4L2_CID_AUTO_WHITE_BALANCE,
-				0, 1, 1, 1);
+					   V4L2_CID_AUTO_WHITE_BALANCE,
+					   0, 1, 1, 1);
 	ctrls->blue_balance = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_BLUE_BALANCE,
 						0, 0xff, 1, 0x80);
 	ctrls->red_balance = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_RED_BALANCE,
-						0, 0xff, 1, 0x80);
+					       0, 0xff, 1, 0x80);
 	/* Auto/manual exposure */
-	ctrls->auto_exp = v4l2_ctrl_new_std_menu(hdl, ops,
-				V4L2_CID_EXPOSURE_AUTO,
-				V4L2_EXPOSURE_MANUAL, 0, V4L2_EXPOSURE_AUTO);
+	ctrls->auto_exp =
+		v4l2_ctrl_new_std_menu(hdl, ops,
+				       V4L2_CID_EXPOSURE_AUTO,
+				       V4L2_EXPOSURE_MANUAL, 0,
+				       V4L2_EXPOSURE_AUTO);
 	/* Exposure time, in 100 us units. min/max is updated dynamically. */
 	ctrls->exposure = v4l2_ctrl_new_std(hdl, ops,
-				V4L2_CID_EXPOSURE_ABSOLUTE,
-				2, 1500, 1, 500);
+					    V4L2_CID_EXPOSURE_ABSOLUTE,
+					    2, 1500, 1, 500);
 	/* Auto/manual gain */
 	ctrls->auto_gain = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_AUTOGAIN,
-						0, 1, 1, 1);
+					     0, 1, 1, 1);
 	ctrls->gain = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_GAIN,
-						16, 64 * (16 + 15), 1, 64 * 16);
+					16, 64 * (16 + 15), 1, 64 * 16);
 
 	ctrls->saturation = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_SATURATION,
-						-2, 2, 1, 0);
+					      -2, 2, 1, 0);
 	ctrls->brightness = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_BRIGHTNESS,
-						-3, 3, 1, 0);
+					      -3, 3, 1, 0);
 	ctrls->sharpness = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_SHARPNESS,
-						0, 32, 1, 6);
+					     0, 32, 1, 6);
 
 	ctrls->hflip = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_HFLIP, 0, 1, 1, 0);
 	ctrls->vflip = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_VFLIP, 0, 1, 1, 0);
 
-	ctrls->light_freq = v4l2_ctrl_new_std_menu(hdl, ops,
-				V4L2_CID_POWER_LINE_FREQUENCY,
-				V4L2_CID_POWER_LINE_FREQUENCY_60HZ, ~0x7,
-				V4L2_CID_POWER_LINE_FREQUENCY_50HZ);
+	ctrls->light_freq =
+		v4l2_ctrl_new_std_menu(hdl, ops,
+				       V4L2_CID_POWER_LINE_FREQUENCY,
+				       V4L2_CID_POWER_LINE_FREQUENCY_60HZ, ~0x7,
+				       V4L2_CID_POWER_LINE_FREQUENCY_50HZ);
 
 	v4l2_ctrl_new_std_menu_items(hdl, ops, V4L2_CID_TEST_PATTERN,
-				ARRAY_SIZE(test_pattern_menu) - 1, 0, 0,
-				test_pattern_menu);
+				     ARRAY_SIZE(test_pattern_menu) - 1, 0, 0,
+				     test_pattern_menu);
 	if (hdl->error) {
 		ret = hdl->error;
 		v4l2_ctrl_handler_free(hdl);
@@ -1121,7 +1126,6 @@ static int __ov965x_set_frame_interval(struct ov965x *ov965x,
 	u64 req_int, err, min_err = ~0ULL;
 	unsigned int i;
 
-
 	if (fi->interval.denominator == 0)
 		return -EINVAL;
 
@@ -1165,7 +1169,8 @@ static int ov965x_s_frame_interval(struct v4l2_subdev *sd,
 	return ret;
 }
 
-static int ov965x_get_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg,
+static int ov965x_get_fmt(struct v4l2_subdev *sd,
+			  struct v4l2_subdev_pad_config *cfg,
 			  struct v4l2_subdev_format *fmt)
 {
 	struct ov965x *ov965x = to_ov965x(sd);
@@ -1209,7 +1214,8 @@ static void __ov965x_try_frame_size(struct v4l2_mbus_framefmt *mf,
 		*size = match;
 }
 
-static int ov965x_set_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg,
+static int ov965x_set_fmt(struct v4l2_subdev *sd,
+			  struct v4l2_subdev_pad_config *cfg,
 			  struct v4l2_subdev_format *fmt)
 {
 	unsigned int index = ARRAY_SIZE(ov965x_formats);
@@ -1231,7 +1237,7 @@ static int ov965x_set_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config
 	mutex_lock(&ov965x->lock);
 
 	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) {
-		if (cfg != NULL) {
+		if (cfg) {
 			mf = v4l2_subdev_get_try_format(sd, cfg, fmt->pad);
 			*mf = fmt->format;
 		}
@@ -1362,7 +1368,8 @@ static int ov965x_s_stream(struct v4l2_subdev *sd, int on)
  */
 static int ov965x_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
 {
-	struct v4l2_mbus_framefmt *mf = v4l2_subdev_get_try_format(sd, fh->pad, 0);
+	struct v4l2_mbus_framefmt *mf =
+		v4l2_subdev_get_try_format(sd, fh->pad, 0);
 
 	ov965x_get_default_format(mf);
 	return 0;
@@ -1470,7 +1477,7 @@ static int ov965x_probe(struct i2c_client *client,
 	struct ov965x *ov965x;
 	int ret;
 
-	if (pdata == NULL) {
+	if (!pdata) {
 		dev_err(&client->dev, "platform data not specified\n");
 		return -EINVAL;
 	}
@@ -1498,13 +1505,13 @@ static int ov965x_probe(struct i2c_client *client,
 
 	ret = ov965x_configure_gpios(ov965x, pdata);
 	if (ret < 0)
-		return ret;
+		goto err_mutex;
 
 	ov965x->pad.flags = MEDIA_PAD_FL_SOURCE;
 	sd->entity.function = MEDIA_ENT_F_CAM_SENSOR;
 	ret = media_entity_pads_init(&sd->entity, 1, &ov965x->pad);
 	if (ret < 0)
-		return ret;
+		goto err_mutex;
 
 	ret = ov965x_initialize_controls(ov965x);
 	if (ret < 0)
@@ -1530,16 +1537,20 @@ err_ctrls:
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
 err_me:
 	media_entity_cleanup(&sd->entity);
+err_mutex:
+	mutex_destroy(&ov965x->lock);
 	return ret;
 }
 
 static int ov965x_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
+	struct ov965x *ov965x = to_ov965x(sd);
 
 	v4l2_async_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
 	media_entity_cleanup(&sd->entity);
+	mutex_destroy(&ov965x->lock);
 
 	return 0;
 }
diff --git a/drivers/media/i2c/s5c73m3/s5c73m3-core.c b/drivers/media/i2c/s5c73m3/s5c73m3-core.c
index f434fb2ee6fc..cdc4f2392ef9 100644
--- a/drivers/media/i2c/s5c73m3/s5c73m3-core.c
+++ b/drivers/media/i2c/s5c73m3/s5c73m3-core.c
@@ -1635,8 +1635,7 @@ static int s5c73m3_get_platform_data(struct s5c73m3 *state)
 
 	node_ep = of_graph_get_next_endpoint(node, NULL);
 	if (!node_ep) {
-		dev_warn(dev, "no endpoint defined for node: %s\n",
-						node->full_name);
+		dev_warn(dev, "no endpoint defined for node: %pOF\n", node);
 		return 0;
 	}
 
diff --git a/drivers/media/i2c/s5k5baf.c b/drivers/media/i2c/s5k5baf.c
index 962051b9939d..ff46d2c96cea 100644
--- a/drivers/media/i2c/s5k5baf.c
+++ b/drivers/media/i2c/s5k5baf.c
@@ -1374,7 +1374,7 @@ static int s5k5baf_get_selection(struct v4l2_subdev *sd,
 				 struct v4l2_subdev_pad_config *cfg,
 				 struct v4l2_subdev_selection *sel)
 {
-	static enum selection_rect rtype;
+	enum selection_rect rtype;
 	struct s5k5baf *state = to_s5k5baf(sd);
 
 	rtype = s5k5baf_get_sel_rect(sel->pad, sel->target);
@@ -1863,8 +1863,7 @@ static int s5k5baf_parse_device_node(struct s5k5baf *state, struct device *dev)
 
 	node_ep = of_graph_get_next_endpoint(node, NULL);
 	if (!node_ep) {
-		dev_err(dev, "no endpoint defined at node %s\n",
-			node->full_name);
+		dev_err(dev, "no endpoint defined at node %pOF\n", node);
 		return -EINVAL;
 	}
 
@@ -1882,8 +1881,8 @@ static int s5k5baf_parse_device_node(struct s5k5baf *state, struct device *dev)
 	case V4L2_MBUS_PARALLEL:
 		break;
 	default:
-		dev_err(dev, "unsupported bus in endpoint defined at node %s\n",
-			node->full_name);
+		dev_err(dev, "unsupported bus in endpoint defined at node %pOF\n",
+			node);
 		return -EINVAL;
 	}
 
diff --git a/drivers/media/i2c/saa7127.c b/drivers/media/i2c/saa7127.c
index 99c303002e90..01784d441ae6 100644
--- a/drivers/media/i2c/saa7127.c
+++ b/drivers/media/i2c/saa7127.c
@@ -806,7 +806,7 @@ static int saa7127_remove(struct i2c_client *client)
 
 /* ----------------------------------------------------------------------- */
 
-static struct i2c_device_id saa7127_id[] = {
+static const struct i2c_device_id saa7127_id[] = {
 	{ "saa7127_auto", 0 },	/* auto-detection */
 	{ "saa7126", SAA7127 },
 	{ "saa7127", SAA7127 },
diff --git a/drivers/media/i2c/saa717x.c b/drivers/media/i2c/saa717x.c
index e1f6bc219c64..102467e00fb3 100644
--- a/drivers/media/i2c/saa717x.c
+++ b/drivers/media/i2c/saa717x.c
@@ -1069,7 +1069,7 @@ static int saa717x_s_std(struct v4l2_subdev *sd, v4l2_std_id std)
 	struct saa717x_state *decoder = to_state(sd);
 
 	v4l2_dbg(1, debug, sd, "decoder set norm ");
-	v4l2_dbg(1, debug, sd, "(not yet implementd)\n");
+	v4l2_dbg(1, debug, sd, "(not yet implemented)\n");
 
 	decoder->radio = 0;
 	decoder->std = std;
diff --git a/drivers/media/i2c/smiapp/smiapp-core.c b/drivers/media/i2c/smiapp/smiapp-core.c
index e0b0c032c4ac..700f433261d0 100644
--- a/drivers/media/i2c/smiapp/smiapp-core.c
+++ b/drivers/media/i2c/smiapp/smiapp-core.c
@@ -841,6 +841,8 @@ static int smiapp_get_mbus_formats(struct smiapp_sensor *sensor)
 		&client->dev,
 		compressed_max_bpp - sensor->compressed_min_bpp + 1,
 		sizeof(*sensor->valid_link_freqs), GFP_KERNEL);
+	if (!sensor->valid_link_freqs)
+		return -ENOMEM;
 
 	for (i = 0; i < ARRAY_SIZE(smiapp_csi_data_formats); i++) {
 		const struct smiapp_csi_data_format *f =
@@ -2809,13 +2811,19 @@ static struct smiapp_hwconfig *smiapp_get_hwconfig(struct device *dev)
 	switch (bus_cfg->bus_type) {
 	case V4L2_MBUS_CSI2:
 		hwcfg->csi_signalling_mode = SMIAPP_CSI_SIGNALLING_MODE_CSI2;
+		hwcfg->lanes = bus_cfg->bus.mipi_csi2.num_data_lanes;
+		break;
+	case V4L2_MBUS_CCP2:
+		hwcfg->csi_signalling_mode = (bus_cfg->bus.mipi_csi1.strobe) ?
+		SMIAPP_CSI_SIGNALLING_MODE_CCP2_DATA_STROBE :
+		SMIAPP_CSI_SIGNALLING_MODE_CCP2_DATA_CLOCK;
+		hwcfg->lanes = 1;
 		break;
-		/* FIXME: add CCP2 support. */
 	default:
+		dev_err(dev, "unsupported bus %u\n", bus_cfg->bus_type);
 		goto out_err;
 	}
 
-	hwcfg->lanes = bus_cfg->bus.mipi_csi2.num_data_lanes;
 	dev_dbg(dev, "lanes %u\n", hwcfg->lanes);
 
 	/* NVM size is not mandatory */
@@ -2828,8 +2836,8 @@ static struct smiapp_hwconfig *smiapp_get_hwconfig(struct device *dev)
 		goto out_err;
 	}
 
-	dev_dbg(dev, "nvm %d, clk %d, csi %d\n", hwcfg->nvm_size,
-		hwcfg->ext_clk, hwcfg->csi_signalling_mode);
+	dev_dbg(dev, "nvm %d, clk %d, mode %d\n",
+		hwcfg->nvm_size, hwcfg->ext_clk, hwcfg->csi_signalling_mode);
 
 	if (!bus_cfg->nr_of_link_frequencies) {
 		dev_warn(dev, "no link frequencies defined\n");
diff --git a/drivers/media/i2c/smiapp/smiapp-quirk.c b/drivers/media/i2c/smiapp/smiapp-quirk.c
index cb128eae9c54..95c0272bb014 100644
--- a/drivers/media/i2c/smiapp/smiapp-quirk.c
+++ b/drivers/media/i2c/smiapp/smiapp-quirk.c
@@ -71,7 +71,7 @@ static int jt8ew9_limits(struct smiapp_sensor *sensor)
 
 static int jt8ew9_post_poweron(struct smiapp_sensor *sensor)
 {
-	const struct smiapp_reg_8 regs[] = {
+	static const struct smiapp_reg_8 regs[] = {
 		{ 0x30a3, 0xd8 }, /* Output port control : LVDS ports only */
 		{ 0x30ae, 0x00 }, /* 0x0307 pll_multiplier maximum value on PLL input 9.6MHz ( 19.2MHz is divided on pre_pll_div) */
 		{ 0x30af, 0xd0 }, /* 0x0307 pll_multiplier maximum value on PLL input 9.6MHz ( 19.2MHz is divided on pre_pll_div) */
@@ -115,7 +115,7 @@ const struct smiapp_quirk smiapp_jt8ew9_quirk = {
 static int imx125es_post_poweron(struct smiapp_sensor *sensor)
 {
 	/* Taken from v02. No idea what the other two are. */
-	const struct smiapp_reg_8 regs[] = {
+	static const struct smiapp_reg_8 regs[] = {
 		/*
 		 * 0x3302: clk during frame blanking:
 		 * 0x00 - HS mode, 0x01 - LP11
@@ -145,7 +145,7 @@ static int jt8ev1_post_poweron(struct smiapp_sensor *sensor)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(&sensor->src->sd);
 	int rval;
-	const struct smiapp_reg_8 regs[] = {
+	static const struct smiapp_reg_8 regs[] = {
 		{ 0x3031, 0xcd }, /* For digital binning (EQ_MONI) */
 		{ 0x30a3, 0xd0 }, /* FLASH STROBE enable */
 		{ 0x3237, 0x00 }, /* For control of pulse timing for ADC */
@@ -166,7 +166,7 @@ static int jt8ev1_post_poweron(struct smiapp_sensor *sensor)
 		{ 0x33cf, 0xec }, /* For Black sun */
 		{ 0x3328, 0x80 }, /* Ugh. No idea what's this. */
 	};
-	const struct smiapp_reg_8 regs_96[] = {
+	static const struct smiapp_reg_8 regs_96[] = {
 		{ 0x30ae, 0x00 }, /* For control of ADC clock */
 		{ 0x30af, 0xd0 },
 		{ 0x30b0, 0x01 },
diff --git a/drivers/media/i2c/soc_camera/Kconfig b/drivers/media/i2c/soc_camera/Kconfig
index 96859f37cb1c..72b369895b37 100644
--- a/drivers/media/i2c/soc_camera/Kconfig
+++ b/drivers/media/i2c/soc_camera/Kconfig
@@ -47,12 +47,6 @@ config SOC_CAMERA_OV5642
 	help
 	  This is a V4L2 camera driver for the OmniVision OV5642 sensor
 
-config SOC_CAMERA_OV6650
-	tristate "ov6650 sensor support"
-	depends on SOC_CAMERA && I2C
-	---help---
-	  This is a V4L2 SoC camera driver for the OmniVision OV6650 sensor
-
 config SOC_CAMERA_OV772X
 	tristate "ov772x camera support"
 	depends on SOC_CAMERA && I2C
diff --git a/drivers/media/i2c/soc_camera/Makefile b/drivers/media/i2c/soc_camera/Makefile
index 974bdb721dbe..78532a7fb8e2 100644
--- a/drivers/media/i2c/soc_camera/Makefile
+++ b/drivers/media/i2c/soc_camera/Makefile
@@ -4,7 +4,6 @@ obj-$(CONFIG_SOC_CAMERA_MT9T031)	+= mt9t031.o
 obj-$(CONFIG_SOC_CAMERA_MT9T112)	+= mt9t112.o
 obj-$(CONFIG_SOC_CAMERA_MT9V022)	+= mt9v022.o
 obj-$(CONFIG_SOC_CAMERA_OV5642)		+= ov5642.o
-obj-$(CONFIG_SOC_CAMERA_OV6650)		+= ov6650.o
 obj-$(CONFIG_SOC_CAMERA_OV772X)		+= ov772x.o
 obj-$(CONFIG_SOC_CAMERA_OV9640)		+= ov9640.o
 obj-$(CONFIG_SOC_CAMERA_OV9740)		+= ov9740.o
diff --git a/drivers/media/i2c/soc_camera/mt9t031.c b/drivers/media/i2c/soc_camera/mt9t031.c
index 714fb3555b34..4802d30e47de 100644
--- a/drivers/media/i2c/soc_camera/mt9t031.c
+++ b/drivers/media/i2c/soc_camera/mt9t031.c
@@ -592,7 +592,7 @@ static const struct dev_pm_ops mt9t031_dev_pm_ops = {
 	.runtime_resume		= mt9t031_runtime_resume,
 };
 
-static struct device_type mt9t031_dev_type = {
+static const struct device_type mt9t031_dev_type = {
 	.name	= "MT9T031",
 	.pm	= &mt9t031_dev_pm_ops,
 };
diff --git a/drivers/media/i2c/tc358743.c b/drivers/media/i2c/tc358743.c
index 5788af238b86..e6f5c363ccab 100644
--- a/drivers/media/i2c/tc358743.c
+++ b/drivers/media/i2c/tc358743.c
@@ -2013,7 +2013,7 @@ static int tc358743_remove(struct i2c_client *client)
 	return 0;
 }
 
-static struct i2c_device_id tc358743_id[] = {
+static const struct i2c_device_id tc358743_id[] = {
 	{"tc358743", 0},
 	{}
 };
diff --git a/drivers/media/i2c/ths8200.c b/drivers/media/i2c/ths8200.c
index 42340e364cea..498ad2368cbc 100644
--- a/drivers/media/i2c/ths8200.c
+++ b/drivers/media/i2c/ths8200.c
@@ -483,7 +483,7 @@ static int ths8200_remove(struct i2c_client *client)
 	return 0;
 }
 
-static struct i2c_device_id ths8200_id[] = {
+static const struct i2c_device_id ths8200_id[] = {
 	{ "ths8200", 0 },
 	{},
 };
diff --git a/drivers/media/i2c/vs6624.c b/drivers/media/i2c/vs6624.c
index f0741ab338df..560738213c00 100644
--- a/drivers/media/i2c/vs6624.c
+++ b/drivers/media/i2c/vs6624.c
@@ -58,7 +58,7 @@ static const struct vs6624_format {
 	},
 };
 
-static struct v4l2_mbus_framefmt vs6624_default_fmt = {
+static const struct v4l2_mbus_framefmt vs6624_default_fmt = {
 	.width = VGA_WIDTH,
 	.height = VGA_HEIGHT,
 	.code = MEDIA_BUS_FMT_UYVY8_2X8,
diff --git a/drivers/media/media-device.c b/drivers/media/media-device.c
index 760e3e424e23..e79f72b8b858 100644
--- a/drivers/media/media-device.c
+++ b/drivers/media/media-device.c
@@ -69,9 +69,9 @@ static int media_device_get_info(struct media_device *dev,
 	strlcpy(info->serial, dev->serial, sizeof(info->serial));
 	strlcpy(info->bus_info, dev->bus_info, sizeof(info->bus_info));
 
-	info->media_version = MEDIA_API_VERSION;
+	info->media_version = LINUX_VERSION_CODE;
+	info->driver_version = info->media_version;
 	info->hw_revision = dev->hw_revision;
-	info->driver_version = dev->driver_version;
 
 	return 0;
 }
@@ -537,9 +537,9 @@ static DEVICE_ATTR(model, S_IRUGO, show_model, NULL);
  * Registration/unregistration
  */
 
-static void media_device_release(struct media_devnode *mdev)
+static void media_device_release(struct media_devnode *devnode)
 {
-	dev_dbg(mdev->parent, "Media device released\n");
+	dev_dbg(devnode->parent, "Media device released\n");
 }
 
 /**
@@ -591,9 +591,8 @@ int __must_check media_device_register_entity(struct media_device *mdev,
 			       &entity->pads[i].graph_obj);
 
 	/* invoke entity_notify callbacks */
-	list_for_each_entry_safe(notify, next, &mdev->entity_notify, list) {
-		(notify)->notify(entity, notify->notify_data);
-	}
+	list_for_each_entry_safe(notify, next, &mdev->entity_notify, list)
+		notify->notify(entity, notify->notify_data);
 
 	if (mdev->entity_internal_idx_max
 	    >= mdev->pm_count_walk.ent_enum.idx_max) {
@@ -834,8 +833,6 @@ void media_device_pci_init(struct media_device *mdev,
 	mdev->hw_revision = (pci_dev->subsystem_vendor << 16)
 			    | pci_dev->subsystem_device;
 
-	mdev->driver_version = LINUX_VERSION_CODE;
-
 	media_device_init(mdev);
 }
 EXPORT_SYMBOL_GPL(media_device_pci_init);
@@ -863,7 +860,6 @@ void __media_device_usb_init(struct media_device *mdev,
 		strlcpy(mdev->serial, udev->serial, sizeof(mdev->serial));
 	usb_make_path(udev, mdev->bus_info, sizeof(mdev->bus_info));
 	mdev->hw_revision = le16_to_cpu(udev->descriptor.bcdDevice);
-	mdev->driver_version = LINUX_VERSION_CODE;
 
 	media_device_init(mdev);
 }
diff --git a/drivers/media/media-entity.c b/drivers/media/media-entity.c
index dd0f0ead9516..2ace0410d277 100644
--- a/drivers/media/media-entity.c
+++ b/drivers/media/media-entity.c
@@ -917,7 +917,7 @@ media_entity_find_link(struct media_pad *source, struct media_pad *sink)
 }
 EXPORT_SYMBOL_GPL(media_entity_find_link);
 
-struct media_pad *media_entity_remote_pad(struct media_pad *pad)
+struct media_pad *media_entity_remote_pad(const struct media_pad *pad)
 {
 	struct media_link *link;
 
diff --git a/drivers/media/pci/b2c2/flexcop-pci.c b/drivers/media/pci/b2c2/flexcop-pci.c
index 6e60decb2198..cc6527e35537 100644
--- a/drivers/media/pci/b2c2/flexcop-pci.c
+++ b/drivers/media/pci/b2c2/flexcop-pci.c
@@ -415,7 +415,7 @@ static void flexcop_pci_remove(struct pci_dev *pdev)
 	flexcop_device_kfree(fc_pci->fc_dev);
 }
 
-static struct pci_device_id flexcop_pci_tbl[] = {
+static const struct pci_device_id flexcop_pci_tbl[] = {
 	{ PCI_DEVICE(0x13d0, 0x2103) },
 	{ },
 };
diff --git a/drivers/media/pci/bt8xx/bt878.c b/drivers/media/pci/bt8xx/bt878.c
index 8aa726651630..a5f52137d306 100644
--- a/drivers/media/pci/bt8xx/bt878.c
+++ b/drivers/media/pci/bt8xx/bt878.c
@@ -383,7 +383,7 @@ EXPORT_SYMBOL(bt878_device_control);
 		.driver_data = (unsigned long) name \
 	}
 
-static struct pci_device_id bt878_pci_tbl[] = {
+static const struct pci_device_id bt878_pci_tbl[] = {
 	BROOKTREE_878_DEVICE(0x0071, 0x0101, "Nebula Electronics DigiTV"),
 	BROOKTREE_878_DEVICE(0x1461, 0x0761, "AverMedia AverTV DVB-T 761"),
 	BROOKTREE_878_DEVICE(0x11bd, 0x001c, "Pinnacle PCTV Sat"),
diff --git a/drivers/media/pci/bt8xx/bttv-driver.c b/drivers/media/pci/bt8xx/bttv-driver.c
index ed319f18ba48..227086a2e99c 100644
--- a/drivers/media/pci/bt8xx/bttv-driver.c
+++ b/drivers/media/pci/bt8xx/bttv-driver.c
@@ -1702,7 +1702,7 @@ static void buffer_release(struct videobuf_queue *q, struct videobuf_buffer *vb)
 	bttv_dma_free(q,fh->btv,buf);
 }
 
-static struct videobuf_queue_ops bttv_video_qops = {
+static const struct videobuf_queue_ops bttv_video_qops = {
 	.buf_setup    = buffer_setup,
 	.buf_prepare  = buffer_prepare,
 	.buf_queue    = buffer_queue,
@@ -4388,7 +4388,7 @@ static int bttv_resume(struct pci_dev *pci_dev)
 }
 #endif
 
-static struct pci_device_id bttv_pci_tbl[] = {
+static const struct pci_device_id bttv_pci_tbl[] = {
 	{PCI_VDEVICE(BROOKTREE, PCI_DEVICE_ID_BT848), 0},
 	{PCI_VDEVICE(BROOKTREE, PCI_DEVICE_ID_BT849), 0},
 	{PCI_VDEVICE(BROOKTREE, PCI_DEVICE_ID_BT878), 0},
diff --git a/drivers/media/pci/bt8xx/bttv-i2c.c b/drivers/media/pci/bt8xx/bttv-i2c.c
index 274fd036b306..eccd1e3d717a 100644
--- a/drivers/media/pci/bt8xx/bttv-i2c.c
+++ b/drivers/media/pci/bt8xx/bttv-i2c.c
@@ -97,7 +97,7 @@ static int bttv_bit_getsda(void *data)
 	return state;
 }
 
-static struct i2c_algo_bit_data bttv_i2c_algo_bit_template = {
+static const struct i2c_algo_bit_data bttv_i2c_algo_bit_template = {
 	.setsda  = bttv_bit_setsda,
 	.setscl  = bttv_bit_setscl,
 	.getsda  = bttv_bit_getsda,
diff --git a/drivers/media/pci/bt8xx/bttv-input.c b/drivers/media/pci/bt8xx/bttv-input.c
index 2fd07a8afcd2..73d655d073d6 100644
--- a/drivers/media/pci/bt8xx/bttv-input.c
+++ b/drivers/media/pci/bt8xx/bttv-input.c
@@ -69,12 +69,13 @@ static void ir_handle_key(struct bttv *btv)
 
 	if ((ir->mask_keydown && (gpio & ir->mask_keydown)) ||
 	    (ir->mask_keyup   && !(gpio & ir->mask_keyup))) {
-		rc_keydown_notimeout(ir->dev, RC_TYPE_UNKNOWN, data, 0);
+		rc_keydown_notimeout(ir->dev, RC_PROTO_UNKNOWN, data, 0);
 	} else {
 		/* HACK: Probably, ir->mask_keydown is missing
 		   for this board */
 		if (btv->c.type == BTTV_BOARD_WINFAST2000)
-			rc_keydown_notimeout(ir->dev, RC_TYPE_UNKNOWN, data, 0);
+			rc_keydown_notimeout(ir->dev, RC_PROTO_UNKNOWN, data,
+					     0);
 
 		rc_keyup(ir->dev);
 	}
@@ -99,7 +100,7 @@ static void ir_enltv_handle_key(struct bttv *btv)
 			gpio, data,
 			(gpio & ir->mask_keyup) ? " up" : "up/down");
 
-		rc_keydown_notimeout(ir->dev, RC_TYPE_UNKNOWN, data, 0);
+		rc_keydown_notimeout(ir->dev, RC_PROTO_UNKNOWN, data, 0);
 		if (keyup)
 			rc_keyup(ir->dev);
 	} else {
@@ -113,7 +114,8 @@ static void ir_enltv_handle_key(struct bttv *btv)
 		if (keyup)
 			rc_keyup(ir->dev);
 		else
-			rc_keydown_notimeout(ir->dev, RC_TYPE_UNKNOWN, data, 0);
+			rc_keydown_notimeout(ir->dev, RC_PROTO_UNKNOWN, data,
+					     0);
 	}
 
 	ir->last_gpio = data | keyup;
@@ -235,7 +237,7 @@ static void bttv_rc5_timer_end(unsigned long data)
 	}
 
 	scancode = RC_SCANCODE_RC5(system, command);
-	rc_keydown(ir->dev, RC_TYPE_RC5, scancode, toggle);
+	rc_keydown(ir->dev, RC_PROTO_RC5, scancode, toggle);
 	dprintk("scancode %x, toggle %x\n", scancode, toggle);
 }
 
@@ -327,7 +329,7 @@ static void bttv_ir_stop(struct bttv *btv)
  * Get_key functions used by I2C remotes
  */
 
-static int get_key_pv951(struct IR_i2c *ir, enum rc_type *protocol,
+static int get_key_pv951(struct IR_i2c *ir, enum rc_proto *protocol,
 			 u32 *scancode, u8 *toggle)
 {
 	unsigned char b;
@@ -355,7 +357,7 @@ static int get_key_pv951(struct IR_i2c *ir, enum rc_type *protocol,
 	 * 	   the device is bound to the vendor-provided RC.
 	 */
 
-	*protocol = RC_TYPE_UNKNOWN;
+	*protocol = RC_PROTO_UNKNOWN;
 	*scancode = b;
 	*toggle = 0;
 	return 1;
@@ -535,7 +537,7 @@ int bttv_input_init(struct bttv *btv)
 	snprintf(ir->phys, sizeof(ir->phys), "pci-%s/ir0",
 		 pci_name(btv->c.pci));
 
-	rc->input_name = ir->name;
+	rc->device_name = ir->name;
 	rc->input_phys = ir->phys;
 	rc->input_id.bustype = BUS_PCI;
 	rc->input_id.version = 1;
diff --git a/drivers/media/pci/bt8xx/dst_ca.c b/drivers/media/pci/bt8xx/dst_ca.c
index 90f4263452d3..530b3e9764ce 100644
--- a/drivers/media/pci/bt8xx/dst_ca.c
+++ b/drivers/media/pci/bt8xx/dst_ca.c
@@ -57,20 +57,6 @@ static unsigned int verbose = 5;
 module_param(verbose, int, 0644);
 MODULE_PARM_DESC(verbose, "verbose startup messages, default is 1 (yes)");
 
-/*	Need some more work	*/
-static int ca_set_slot_descr(void)
-{
-	/*	We could make this more graceful ?	*/
-	return -EOPNOTSUPP;
-}
-
-/*	Need some more work	*/
-static int ca_set_pid(void)
-{
-	/*	We could make this more graceful ?	*/
-	return -EOPNOTSUPP;
-}
-
 static void put_command_and_length(u8 *data, int command, int length)
 {
 	data[0] = (command >> 16) & 0xff;
@@ -144,7 +130,7 @@ static int dst_put_ci(struct dst_state *state, u8 *data, int len, u8 *ca_string,
 	}
 
 	if(dst_ca_comm_err == RETRIES)
-		return -1;
+		return -EIO;
 
 	return 0;
 }
@@ -159,7 +145,7 @@ static int ca_get_app_info(struct dst_state *state)
 	put_checksum(&command[0], command[0]);
 	if ((dst_put_ci(state, command, sizeof(command), state->messages, GET_REPLY)) < 0) {
 		dprintk(verbose, DST_CA_ERROR, 1, " -->dst_put_ci FAILED !");
-		return -1;
+		return -EIO;
 	}
 	dprintk(verbose, DST_CA_INFO, 1, " -->dst_put_ci SUCCESS !");
 	dprintk(verbose, DST_CA_INFO, 1, " ================================ CI Module Application Info ======================================");
@@ -198,7 +184,7 @@ static int ca_get_ca_info(struct dst_state *state)
 	put_checksum(&slot_command[0], slot_command[0]);
 	if ((dst_put_ci(state, slot_command, sizeof (slot_command), state->messages, GET_REPLY)) < 0) {
 		dprintk(verbose, DST_CA_ERROR, 1, " -->dst_put_ci FAILED !");
-		return -1;
+		return -EIO;
 	}
 	dprintk(verbose, DST_CA_INFO, 1, " -->dst_put_ci SUCCESS !");
 
@@ -242,7 +228,7 @@ static int ca_get_slot_caps(struct dst_state *state, struct ca_caps *p_ca_caps,
 	put_checksum(&slot_command[0], slot_command[0]);
 	if ((dst_put_ci(state, slot_command, sizeof (slot_command), slot_cap, GET_REPLY)) < 0) {
 		dprintk(verbose, DST_CA_ERROR, 1, " -->dst_put_ci FAILED !");
-		return -1;
+		return -EIO;
 	}
 	dprintk(verbose, DST_CA_NOTICE, 1, " -->dst_put_ci SUCCESS !");
 
@@ -282,7 +268,7 @@ static int ca_get_slot_info(struct dst_state *state, struct ca_slot_info *p_ca_s
 	put_checksum(&slot_command[0], 7);
 	if ((dst_put_ci(state, slot_command, sizeof (slot_command), slot_info, GET_REPLY)) < 0) {
 		dprintk(verbose, DST_CA_ERROR, 1, " -->dst_put_ci FAILED !");
-		return -1;
+		return -EIO;
 	}
 	dprintk(verbose, DST_CA_INFO, 1, " -->dst_put_ci SUCCESS !");
 
@@ -354,7 +340,7 @@ static int handle_dst_tag(struct dst_state *state, struct ca_msg *p_ca_message,
 	} else {
 		if (length > 247) {
 			dprintk(verbose, DST_CA_ERROR, 1, " Message too long ! *** Bailing Out *** !");
-			return -1;
+			return -EIO;
 		}
 		hw_buffer->msg[0] = (length & 0xff) + 7;
 		hw_buffer->msg[1] = 0x40;
@@ -380,7 +366,7 @@ static int write_to_8820(struct dst_state *state, struct ca_msg *hw_buffer, u8 l
 		dprintk(verbose, DST_CA_ERROR, 1, " DST-CI Command failed.");
 		dprintk(verbose, DST_CA_NOTICE, 1, " Resetting DST.");
 		rdc_reset_state(state);
-		return -1;
+		return -EIO;
 	}
 	dprintk(verbose, DST_CA_NOTICE, 1, " DST-CI Command success.");
 
@@ -453,7 +439,7 @@ static int dst_check_ca_pmt(struct dst_state *state, struct ca_msg *p_ca_message
 	if (ca_pmt_reply_test) {
 		if ((ca_set_pmt(state, p_ca_message, hw_buffer, 1, GET_REPLY)) < 0) {
 			dprintk(verbose, DST_CA_ERROR, 1, " ca_set_pmt.. failed !");
-			return -1;
+			return -EIO;
 		}
 
 	/*	Process CA PMT Reply		*/
@@ -464,7 +450,7 @@ static int dst_check_ca_pmt(struct dst_state *state, struct ca_msg *p_ca_message
 	if (!ca_pmt_reply_test) {
 		if ((ca_set_pmt(state, p_ca_message, hw_buffer, 0, NO_REPLY)) < 0) {
 			dprintk(verbose, DST_CA_ERROR, 1, " ca_set_pmt.. failed !");
-			return -1;
+			return -EIO;
 		}
 		dprintk(verbose, DST_CA_NOTICE, 1, " ca_set_pmt.. success !");
 	/*	put a dummy message		*/
@@ -573,17 +559,18 @@ static long dst_ca_ioctl(struct file *file, unsigned int cmd, unsigned long ioct
 	switch (cmd) {
 	case CA_SEND_MSG:
 		dprintk(verbose, DST_CA_INFO, 1, " Sending message");
-		if ((ca_send_message(state, p_ca_message, arg)) < 0) {
+		result = ca_send_message(state, p_ca_message, arg);
+
+		if (result < 0) {
 			dprintk(verbose, DST_CA_ERROR, 1, " -->CA_SEND_MSG Failed !");
-			result = -1;
 			goto free_mem_and_exit;
 		}
 		break;
 	case CA_GET_MSG:
 		dprintk(verbose, DST_CA_INFO, 1, " Getting message");
-		if ((ca_get_message(state, p_ca_message, arg)) < 0) {
+		result = ca_get_message(state, p_ca_message, arg);
+		if (result < 0) {
 			dprintk(verbose, DST_CA_ERROR, 1, " -->CA_GET_MSG Failed !");
-			result = -1;
 			goto free_mem_and_exit;
 		}
 		dprintk(verbose, DST_CA_INFO, 1, " -->CA_GET_MSG Success !");
@@ -595,7 +582,8 @@ static long dst_ca_ioctl(struct file *file, unsigned int cmd, unsigned long ioct
 		break;
 	case CA_GET_SLOT_INFO:
 		dprintk(verbose, DST_CA_INFO, 1, " Getting Slot info");
-		if ((ca_get_slot_info(state, p_ca_slot_info, arg)) < 0) {
+		result = ca_get_slot_info(state, p_ca_slot_info, arg);
+		if (result < 0) {
 			dprintk(verbose, DST_CA_ERROR, 1, " -->CA_GET_SLOT_INFO Failed !");
 			result = -1;
 			goto free_mem_and_exit;
@@ -604,40 +592,22 @@ static long dst_ca_ioctl(struct file *file, unsigned int cmd, unsigned long ioct
 		break;
 	case CA_GET_CAP:
 		dprintk(verbose, DST_CA_INFO, 1, " Getting Slot capabilities");
-		if ((ca_get_slot_caps(state, p_ca_caps, arg)) < 0) {
+		result = ca_get_slot_caps(state, p_ca_caps, arg);
+		if (result < 0) {
 			dprintk(verbose, DST_CA_ERROR, 1, " -->CA_GET_CAP Failed !");
-			result = -1;
 			goto free_mem_and_exit;
 		}
 		dprintk(verbose, DST_CA_INFO, 1, " -->CA_GET_CAP Success !");
 		break;
 	case CA_GET_DESCR_INFO:
 		dprintk(verbose, DST_CA_INFO, 1, " Getting descrambler description");
-		if ((ca_get_slot_descr(state, p_ca_message, arg)) < 0) {
+		result = ca_get_slot_descr(state, p_ca_message, arg);
+		if (result < 0) {
 			dprintk(verbose, DST_CA_ERROR, 1, " -->CA_GET_DESCR_INFO Failed !");
-			result = -1;
 			goto free_mem_and_exit;
 		}
 		dprintk(verbose, DST_CA_INFO, 1, " -->CA_GET_DESCR_INFO Success !");
 		break;
-	case CA_SET_DESCR:
-		dprintk(verbose, DST_CA_INFO, 1, " Setting descrambler");
-		if ((ca_set_slot_descr()) < 0) {
-			dprintk(verbose, DST_CA_ERROR, 1, " -->CA_SET_DESCR Failed !");
-			result = -1;
-			goto free_mem_and_exit;
-		}
-		dprintk(verbose, DST_CA_INFO, 1, " -->CA_SET_DESCR Success !");
-		break;
-	case CA_SET_PID:
-		dprintk(verbose, DST_CA_INFO, 1, " Setting PID");
-		if ((ca_set_pid()) < 0) {
-			dprintk(verbose, DST_CA_ERROR, 1, " -->CA_SET_PID Failed !");
-			result = -1;
-			goto free_mem_and_exit;
-		}
-		dprintk(verbose, DST_CA_INFO, 1, " -->CA_SET_PID Success !");
-		break;
 	default:
 		result = -EOPNOTSUPP;
 	}
diff --git a/drivers/media/pci/cobalt/cobalt-alsa-pcm.c b/drivers/media/pci/cobalt/cobalt-alsa-pcm.c
index 49013c6b8646..b69b258d39b9 100644
--- a/drivers/media/pci/cobalt/cobalt-alsa-pcm.c
+++ b/drivers/media/pci/cobalt/cobalt-alsa-pcm.c
@@ -43,7 +43,7 @@ MODULE_PARM_DESC(pcm_debug, "enable debug messages for pcm");
 			pr_info("cobalt-alsa-pcm %s: " fmt, __func__, ##arg); \
 	} while (0)
 
-static struct snd_pcm_hardware snd_cobalt_hdmi_capture = {
+static const struct snd_pcm_hardware snd_cobalt_hdmi_capture = {
 	.info = SNDRV_PCM_INFO_BLOCK_TRANSFER |
 		SNDRV_PCM_INFO_MMAP           |
 		SNDRV_PCM_INFO_INTERLEAVED    |
@@ -64,7 +64,7 @@ static struct snd_pcm_hardware snd_cobalt_hdmi_capture = {
 	.periods_max = 4,
 };
 
-static struct snd_pcm_hardware snd_cobalt_playback = {
+static const struct snd_pcm_hardware snd_cobalt_playback = {
 	.info = SNDRV_PCM_INFO_BLOCK_TRANSFER |
 		SNDRV_PCM_INFO_MMAP           |
 		SNDRV_PCM_INFO_INTERLEAVED    |
diff --git a/drivers/media/pci/cobalt/cobalt-driver.c b/drivers/media/pci/cobalt/cobalt-driver.c
index f8e173f3e9e2..98b6cb9505d1 100644
--- a/drivers/media/pci/cobalt/cobalt-driver.c
+++ b/drivers/media/pci/cobalt/cobalt-driver.c
@@ -36,7 +36,7 @@
 #include "cobalt-omnitek.h"
 
 /* add your revision and whatnot here */
-static struct pci_device_id cobalt_pci_tbl[] = {
+static const struct pci_device_id cobalt_pci_tbl[] = {
 	{PCI_VENDOR_ID_CISCO, PCI_DEVICE_ID_COBALT,
 	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
 	{0,}
diff --git a/drivers/media/pci/cobalt/cobalt-i2c.c b/drivers/media/pci/cobalt/cobalt-i2c.c
index ad16b89b8d0c..1a5c55673ea8 100644
--- a/drivers/media/pci/cobalt/cobalt-i2c.c
+++ b/drivers/media/pci/cobalt/cobalt-i2c.c
@@ -301,7 +301,7 @@ static u32 cobalt_func(struct i2c_adapter *adap)
 }
 
 /* template for i2c-bit-algo */
-static struct i2c_adapter cobalt_i2c_adap_template = {
+static const struct i2c_adapter cobalt_i2c_adap_template = {
 	.name = "cobalt i2c driver",
 	.algo = NULL,                   /* set by i2c-algo-bit */
 	.algo_data = NULL,              /* filled from template */
diff --git a/drivers/media/pci/cx18/cx18-alsa-mixer.c b/drivers/media/pci/cx18/cx18-alsa-mixer.c
index 06b066bc9301..cb04c3d820e2 100644
--- a/drivers/media/pci/cx18/cx18-alsa-mixer.c
+++ b/drivers/media/pci/cx18/cx18-alsa-mixer.c
@@ -161,7 +161,7 @@ int __init snd_cx18_mixer_create(struct snd_cx18_card *cxsc)
 
 	strlcpy(sc->mixername, "CX23418 Mixer", sizeof(sc->mixername));
 
-	ret = snd_ctl_add(sc, snd_ctl_new1(snd_cx18_mixer_tv_vol, cxsc));
+	ret = snd_ctl_add(sc, snd_ctl_new1(&snd_cx18_mixer_tv_vol, cxsc));
 	if (ret) {
 		CX18_ALSA_WARN("%s: failed to add %s control, err %d\n",
 				__func__, snd_cx18_mixer_tv_vol.name, ret);
diff --git a/drivers/media/pci/cx18/cx18-alsa-pcm.c b/drivers/media/pci/cx18/cx18-alsa-pcm.c
index f68ee57a9ae2..aadd76466aec 100644
--- a/drivers/media/pci/cx18/cx18-alsa-pcm.c
+++ b/drivers/media/pci/cx18/cx18-alsa-pcm.c
@@ -44,7 +44,7 @@ MODULE_PARM_DESC(pcm_debug, "enable debug messages for pcm");
 				  __func__, ##arg); 			\
 	} while (0)
 
-static struct snd_pcm_hardware snd_cx18_hw_capture = {
+static const struct snd_pcm_hardware snd_cx18_hw_capture = {
 	.info = SNDRV_PCM_INFO_BLOCK_TRANSFER |
 		SNDRV_PCM_INFO_MMAP           |
 		SNDRV_PCM_INFO_INTERLEAVED    |
diff --git a/drivers/media/pci/cx18/cx18-driver.c b/drivers/media/pci/cx18/cx18-driver.c
index 8bce49cdad46..8654710464cc 100644
--- a/drivers/media/pci/cx18/cx18-driver.c
+++ b/drivers/media/pci/cx18/cx18-driver.c
@@ -48,7 +48,7 @@ int (*cx18_ext_init)(struct cx18 *);
 EXPORT_SYMBOL(cx18_ext_init);
 
 /* add your revision and whatnot here */
-static struct pci_device_id cx18_pci_tbl[] = {
+static const struct pci_device_id cx18_pci_tbl[] = {
 	{PCI_VENDOR_ID_CX, PCI_DEVICE_ID_CX23418,
 	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
 	{0,}
diff --git a/drivers/media/pci/cx18/cx18-i2c.c b/drivers/media/pci/cx18/cx18-i2c.c
index eabdd4c5520a..7f588eeac60f 100644
--- a/drivers/media/pci/cx18/cx18-i2c.c
+++ b/drivers/media/pci/cx18/cx18-i2c.c
@@ -93,8 +93,8 @@ static int cx18_i2c_new_ir(struct cx18 *cx, struct i2c_adapter *adap, u32 hw,
 	case CX18_HW_Z8F0811_IR_RX_HAUP:
 		init_data->ir_codes = RC_MAP_HAUPPAUGE;
 		init_data->internal_get_key_func = IR_KBD_GET_KEY_HAUP_XVR;
-		init_data->type = RC_BIT_RC5 | RC_BIT_RC6_MCE |
-							RC_BIT_RC6_6A_32;
+		init_data->type = RC_PROTO_BIT_RC5 | RC_PROTO_BIT_RC6_MCE |
+							RC_PROTO_BIT_RC6_6A_32;
 		init_data->name = cx->card_name;
 		info.platform_data = init_data;
 		break;
@@ -206,7 +206,7 @@ static int cx18_getsda(void *data)
 }
 
 /* template for i2c-bit-algo */
-static struct i2c_adapter cx18_i2c_adap_template = {
+static const struct i2c_adapter cx18_i2c_adap_template = {
 	.name = "cx18 i2c driver",
 	.algo = NULL,                   /* set by i2c-algo-bit */
 	.algo_data = NULL,              /* filled from template */
@@ -216,7 +216,7 @@ static struct i2c_adapter cx18_i2c_adap_template = {
 #define CX18_SCL_PERIOD (10) /* usecs. 10 usec is period for a 100 KHz clock */
 #define CX18_ALGO_BIT_TIMEOUT (2) /* seconds */
 
-static struct i2c_algo_bit_data cx18_i2c_algo_template = {
+static const struct i2c_algo_bit_data cx18_i2c_algo_template = {
 	.setsda		= cx18_setsda,
 	.setscl		= cx18_setscl,
 	.getsda		= cx18_getsda,
diff --git a/drivers/media/pci/cx18/cx18-streams.c b/drivers/media/pci/cx18/cx18-streams.c
index 3c45e0071530..8385411af641 100644
--- a/drivers/media/pci/cx18/cx18-streams.c
+++ b/drivers/media/pci/cx18/cx18-streams.c
@@ -31,7 +31,7 @@
 
 #define CX18_DSP0_INTERRUPT_MASK     	0xd0004C
 
-static struct v4l2_file_operations cx18_v4l2_enc_fops = {
+static const struct v4l2_file_operations cx18_v4l2_enc_fops = {
 	.owner = THIS_MODULE,
 	.read = cx18_v4l2_read,
 	.open = cx18_v4l2_open,
@@ -240,7 +240,7 @@ static void buffer_queue(struct videobuf_queue *q, struct videobuf_buffer *vb)
 	list_add_tail(&buf->vb.queue, &s->vb_capture);
 }
 
-static struct videobuf_queue_ops cx18_videobuf_qops = {
+static const struct videobuf_queue_ops cx18_videobuf_qops = {
 	.buf_setup    = buffer_setup,
 	.buf_prepare  = buffer_prepare,
 	.buf_queue    = buffer_queue,
diff --git a/drivers/media/pci/cx23885/cx23885-417.c b/drivers/media/pci/cx23885/cx23885-417.c
index 2ff1d1e274be..a71f3c7569ce 100644
--- a/drivers/media/pci/cx23885/cx23885-417.c
+++ b/drivers/media/pci/cx23885/cx23885-417.c
@@ -1416,7 +1416,7 @@ static int vidioc_log_status(struct file *file, void *priv)
 	return 0;
 }
 
-static struct v4l2_file_operations mpeg_fops = {
+static const struct v4l2_file_operations mpeg_fops = {
 	.owner	       = THIS_MODULE,
 	.open           = v4l2_fh_open,
 	.release        = vb2_fop_release,
diff --git a/drivers/media/pci/cx23885/cx23885-alsa.c b/drivers/media/pci/cx23885/cx23885-alsa.c
index c148f9a4a9ac..d8c3637e492e 100644
--- a/drivers/media/pci/cx23885/cx23885-alsa.c
+++ b/drivers/media/pci/cx23885/cx23885-alsa.c
@@ -293,7 +293,7 @@ static int dsp_buffer_free(struct cx23885_audio_dev *chip)
  */
 #define DEFAULT_FIFO_SIZE	4096
 
-static struct snd_pcm_hardware snd_cx23885_digital_hw = {
+static const struct snd_pcm_hardware snd_cx23885_digital_hw = {
 	.info = SNDRV_PCM_INFO_MMAP |
 		SNDRV_PCM_INFO_INTERLEAVED |
 		SNDRV_PCM_INFO_BLOCK_TRANSFER |
diff --git a/drivers/media/pci/cx23885/cx23885-cards.c b/drivers/media/pci/cx23885/cx23885-cards.c
index c48fa8e25a70..78a8836d03e4 100644
--- a/drivers/media/pci/cx23885/cx23885-cards.c
+++ b/drivers/media/pci/cx23885/cx23885-cards.c
@@ -1278,6 +1278,12 @@ static void hauppauge_eeprom(struct cx23885_dev *dev, u8 *eeprom_data)
 	case 85721:
 		/* WinTV-HVR1290 (PCIe, OEM, RCA in, IR,
 			Dual channel ATSC and Basic analog */
+	case 121019:
+		/* WinTV-HVR4400 (PCIe, DVB-S2, DVB-C/T) */
+		break;
+	case 121029:
+		/* WinTV-HVR5500 (PCIe, DVB-S2, DVB-C/T) */
+		break;
 	case 150329:
 		/* WinTV-HVR5525 (PCIe, DVB-S/S2, DVB-T/T2/C) */
 		break;
diff --git a/drivers/media/pci/cx23885/cx23885-core.c b/drivers/media/pci/cx23885/cx23885-core.c
index 02b5ec549369..8f63df1cb418 100644
--- a/drivers/media/pci/cx23885/cx23885-core.c
+++ b/drivers/media/pci/cx23885/cx23885-core.c
@@ -2056,7 +2056,7 @@ static void cx23885_finidev(struct pci_dev *pci_dev)
 	kfree(dev);
 }
 
-static struct pci_device_id cx23885_pci_tbl[] = {
+static const struct pci_device_id cx23885_pci_tbl[] = {
 	{
 		/* CX23885 */
 		.vendor       = 0x14f1,
diff --git a/drivers/media/pci/cx23885/cx23885-dvb.c b/drivers/media/pci/cx23885/cx23885-dvb.c
index 979b66627f60..e795ddeb7fe2 100644
--- a/drivers/media/pci/cx23885/cx23885-dvb.c
+++ b/drivers/media/pci/cx23885/cx23885-dvb.c
@@ -2637,6 +2637,11 @@ int cx23885_dvb_unregister(struct cx23885_tsport *port)
 	struct vb2_dvb_frontend *fe0;
 	struct i2c_client *client;
 
+	fe0 = vb2_dvb_get_frontend(&port->frontends, 1);
+
+	if (fe0 && fe0->dvb.frontend)
+		vb2_dvb_unregister_bus(&port->frontends);
+
 	/* remove I2C client for CI */
 	client = port->i2c_client_ci;
 	if (client) {
@@ -2665,11 +2670,6 @@ int cx23885_dvb_unregister(struct cx23885_tsport *port)
 		i2c_unregister_device(client);
 	}
 
-	fe0 = vb2_dvb_get_frontend(&port->frontends, 1);
-
-	if (fe0 && fe0->dvb.frontend)
-		vb2_dvb_unregister_bus(&port->frontends);
-
 	switch (port->dev->board) {
 	case CX23885_BOARD_NETUP_DUAL_DVBS2_CI:
 		netup_ci_exit(port);
diff --git a/drivers/media/pci/cx23885/cx23885-i2c.c b/drivers/media/pci/cx23885/cx23885-i2c.c
index 8528032090f2..0f21467ae88e 100644
--- a/drivers/media/pci/cx23885/cx23885-i2c.c
+++ b/drivers/media/pci/cx23885/cx23885-i2c.c
@@ -264,7 +264,7 @@ static const struct i2c_algorithm cx23885_i2c_algo_template = {
 
 /* ----------------------------------------------------------------------- */
 
-static struct i2c_adapter cx23885_i2c_adap_template = {
+static const struct i2c_adapter cx23885_i2c_adap_template = {
 	.name              = "cx23885",
 	.owner             = THIS_MODULE,
 	.algo              = &cx23885_i2c_algo_template,
diff --git a/drivers/media/pci/cx23885/cx23885-input.c b/drivers/media/pci/cx23885/cx23885-input.c
index 4367cb3162b6..944b70831f12 100644
--- a/drivers/media/pci/cx23885/cx23885-input.c
+++ b/drivers/media/pci/cx23885/cx23885-input.c
@@ -284,32 +284,32 @@ int cx23885_input_init(struct cx23885_dev *dev)
 	case CX23885_BOARD_HAUPPAUGE_HVR1290:
 	case CX23885_BOARD_HAUPPAUGE_HVR1250:
 		/* Integrated CX2388[58] IR controller */
-		allowed_protos = RC_BIT_ALL_IR_DECODER;
+		allowed_protos = RC_PROTO_BIT_ALL_IR_DECODER;
 		/* The grey Hauppauge RC-5 remote */
 		rc_map = RC_MAP_HAUPPAUGE;
 		break;
 	case CX23885_BOARD_TERRATEC_CINERGY_T_PCIE_DUAL:
 		/* Integrated CX23885 IR controller */
-		allowed_protos = RC_BIT_ALL_IR_DECODER;
+		allowed_protos = RC_PROTO_BIT_ALL_IR_DECODER;
 		/* The grey Terratec remote with orange buttons */
 		rc_map = RC_MAP_NEC_TERRATEC_CINERGY_XS;
 		break;
 	case CX23885_BOARD_TEVII_S470:
 		/* Integrated CX23885 IR controller */
-		allowed_protos = RC_BIT_ALL_IR_DECODER;
+		allowed_protos = RC_PROTO_BIT_ALL_IR_DECODER;
 		/* A guess at the remote */
 		rc_map = RC_MAP_TEVII_NEC;
 		break;
 	case CX23885_BOARD_MYGICA_X8507:
 		/* Integrated CX23885 IR controller */
-		allowed_protos = RC_BIT_ALL_IR_DECODER;
+		allowed_protos = RC_PROTO_BIT_ALL_IR_DECODER;
 		/* A guess at the remote */
 		rc_map = RC_MAP_TOTAL_MEDIA_IN_HAND_02;
 		break;
 	case CX23885_BOARD_TBS_6980:
 	case CX23885_BOARD_TBS_6981:
 		/* Integrated CX23885 IR controller */
-		allowed_protos = RC_BIT_ALL_IR_DECODER;
+		allowed_protos = RC_PROTO_BIT_ALL_IR_DECODER;
 		/* A guess at the remote */
 		rc_map = RC_MAP_TBS_NEC;
 		break;
@@ -320,12 +320,12 @@ int cx23885_input_init(struct cx23885_dev *dev)
 	case CX23885_BOARD_DVBSKY_S952:
 	case CX23885_BOARD_DVBSKY_T982:
 		/* Integrated CX23885 IR controller */
-		allowed_protos = RC_BIT_ALL_IR_DECODER;
+		allowed_protos = RC_PROTO_BIT_ALL_IR_DECODER;
 		rc_map = RC_MAP_DVBSKY;
 		break;
 	case CX23885_BOARD_TT_CT2_4500_CI:
 		/* Integrated CX23885 IR controller */
-		allowed_protos = RC_BIT_ALL_IR_DECODER;
+		allowed_protos = RC_PROTO_BIT_ALL_IR_DECODER;
 		rc_map = RC_MAP_TT_1500;
 		break;
 	default:
@@ -351,7 +351,7 @@ int cx23885_input_init(struct cx23885_dev *dev)
 	}
 
 	kernel_ir->rc = rc;
-	rc->input_name = kernel_ir->name;
+	rc->device_name = kernel_ir->name;
 	rc->input_phys = kernel_ir->phys;
 	rc->input_id.bustype = BUS_PCI;
 	rc->input_id.version = 1;
diff --git a/drivers/media/pci/cx25821/cx25821-alsa.c b/drivers/media/pci/cx25821/cx25821-alsa.c
index 519b81c0c837..2b34990e86f2 100644
--- a/drivers/media/pci/cx25821/cx25821-alsa.c
+++ b/drivers/media/pci/cx25821/cx25821-alsa.c
@@ -428,7 +428,7 @@ static int dsp_buffer_free(struct cx25821_audio_dev *chip)
  * Digital hardware definition
  */
 #define DEFAULT_FIFO_SIZE	384
-static struct snd_pcm_hardware snd_cx25821_digital_hw = {
+static const struct snd_pcm_hardware snd_cx25821_digital_hw = {
 	.info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_INTERLEAVED |
 		SNDRV_PCM_INFO_BLOCK_TRANSFER | SNDRV_PCM_INFO_MMAP_VALID,
 	.formats = SNDRV_PCM_FMTBIT_S16_LE,
diff --git a/drivers/media/pci/cx25821/cx25821-audio-upstream.c b/drivers/media/pci/cx25821/cx25821-audio-upstream.c
index b94eb1c0023d..ada26d4acfb4 100644
--- a/drivers/media/pci/cx25821/cx25821-audio-upstream.c
+++ b/drivers/media/pci/cx25821/cx25821-audio-upstream.c
@@ -277,7 +277,7 @@ static int cx25821_get_audio_data(struct cx25821_dev *dev,
 		p = (char *)dev->_audiodata_buf_virt_addr + frame_offset;
 
 	for (i = 0; i < dev->_audio_lines_count; i++) {
-		int n = kernel_read(file, file_offset, mybuf, AUDIO_LINE_SIZE);
+		int n = kernel_read(file, mybuf, AUDIO_LINE_SIZE, &file_offset);
 		if (n < AUDIO_LINE_SIZE) {
 			pr_info("Done: exit %s() since no more bytes to read from Audio file\n",
 				__func__);
@@ -290,7 +290,6 @@ static int cx25821_get_audio_data(struct cx25821_dev *dev,
 			memcpy(p, mybuf, n);
 			p += n;
 		}
-		file_offset += n;
 	}
 	dev->_audioframe_count++;
 	fput(file);
@@ -318,7 +317,7 @@ static int cx25821_openfile_audio(struct cx25821_dev *dev,
 {
 	char *p = (void *)dev->_audiodata_buf_virt_addr;
 	struct file *file;
-	loff_t offset;
+	loff_t file_offset = 0;
 	int i, j;
 
 	file = filp_open(dev->_audiofilename, O_RDONLY | O_LARGEFILE, 0);
@@ -328,11 +327,11 @@ static int cx25821_openfile_audio(struct cx25821_dev *dev,
 		return PTR_ERR(file);
 	}
 
-	for (j = 0, offset = 0; j < NUM_AUDIO_FRAMES; j++) {
+	for (j = 0; j < NUM_AUDIO_FRAMES; j++) {
 		for (i = 0; i < dev->_audio_lines_count; i++) {
 			char buf[AUDIO_LINE_SIZE];
-			int n = kernel_read(file, offset, buf,
-						AUDIO_LINE_SIZE);
+			loff_t offset = file_offset;
+			int n = kernel_read(file, buf, AUDIO_LINE_SIZE, &file_offset);
 
 			if (n < AUDIO_LINE_SIZE) {
 				pr_info("Done: exit %s() since no more bytes to read from Audio file\n",
@@ -344,8 +343,6 @@ static int cx25821_openfile_audio(struct cx25821_dev *dev,
 
 			if (p)
 				memcpy(p + offset, buf, n);
-
-			offset += n;
 		}
 		dev->_audioframe_count++;
 	}
diff --git a/drivers/media/pci/cx25821/cx25821-core.c b/drivers/media/pci/cx25821/cx25821-core.c
index fbc0229183bd..04aa4a68a0ae 100644
--- a/drivers/media/pci/cx25821/cx25821-core.c
+++ b/drivers/media/pci/cx25821/cx25821-core.c
@@ -1390,10 +1390,7 @@ static struct pci_driver cx25821_pci_driver = {
 
 static int __init cx25821_init(void)
 {
-	pr_info("driver version %d.%d.%d loaded\n",
-		(CX25821_VERSION_CODE >> 16) & 0xff,
-		(CX25821_VERSION_CODE >> 8) & 0xff,
-		CX25821_VERSION_CODE & 0xff);
+	pr_info("driver loaded\n");
 	return pci_register_driver(&cx25821_pci_driver);
 }
 
diff --git a/drivers/media/pci/cx25821/cx25821-i2c.c b/drivers/media/pci/cx25821/cx25821-i2c.c
index 263a1cf36ef1..000049d3c71b 100644
--- a/drivers/media/pci/cx25821/cx25821-i2c.c
+++ b/drivers/media/pci/cx25821/cx25821-i2c.c
@@ -285,7 +285,7 @@ static const struct i2c_algorithm cx25821_i2c_algo_template = {
 #endif
 };
 
-static struct i2c_adapter cx25821_i2c_adap_template = {
+static const struct i2c_adapter cx25821_i2c_adap_template = {
 	.name = "cx25821",
 	.owner = THIS_MODULE,
 	.algo = &cx25821_i2c_algo_template,
diff --git a/drivers/media/pci/cx25821/cx25821.h b/drivers/media/pci/cx25821/cx25821.h
index 0f20e89b0cde..b3eb2dabb30b 100644
--- a/drivers/media/pci/cx25821/cx25821.h
+++ b/drivers/media/pci/cx25821/cx25821.h
@@ -41,8 +41,6 @@
 #include <linux/version.h>
 #include <linux/mutex.h>
 
-#define CX25821_VERSION_CODE KERNEL_VERSION(0, 0, 106)
-
 #define UNSET (-1U)
 #define NO_SYNC_LINE (-1U)
 
diff --git a/drivers/media/pci/cx88/cx88-alsa.c b/drivers/media/pci/cx88/cx88-alsa.c
index c81fe4681d14..9740326bc93f 100644
--- a/drivers/media/pci/cx88/cx88-alsa.c
+++ b/drivers/media/pci/cx88/cx88-alsa.c
@@ -799,7 +799,7 @@ static int snd_cx88_alc_put(struct snd_kcontrol *kcontrol,
 	return 0;
 }
 
-static struct snd_kcontrol_new snd_cx88_alc_switch = {
+static const struct snd_kcontrol_new snd_cx88_alc_switch = {
 	.iface = SNDRV_CTL_ELEM_IFACE_MIXER,
 	.name = "Line-In ALC Switch",
 	.info = snd_ctl_boolean_mono_info,
diff --git a/drivers/media/pci/cx88/cx88-blackbird.c b/drivers/media/pci/cx88/cx88-blackbird.c
index aa49c9597d9c..e3101f04941c 100644
--- a/drivers/media/pci/cx88/cx88-blackbird.c
+++ b/drivers/media/pci/cx88/cx88-blackbird.c
@@ -1075,7 +1075,7 @@ static const struct v4l2_ioctl_ops mpeg_ioctl_ops = {
 	.vidioc_unsubscribe_event    = v4l2_event_unsubscribe,
 };
 
-static struct video_device cx8802_mpeg_template = {
+static const struct video_device cx8802_mpeg_template = {
 	.name                 = "cx8802",
 	.fops                 = &mpeg_fops,
 	.ioctl_ops	      = &mpeg_ioctl_ops,
diff --git a/drivers/media/pci/cx88/cx88-input.c b/drivers/media/pci/cx88/cx88-input.c
index 01f2e472a2a0..e02449bf2041 100644
--- a/drivers/media/pci/cx88/cx88-input.c
+++ b/drivers/media/pci/cx88/cx88-input.c
@@ -132,7 +132,7 @@ static void cx88_ir_handle_key(struct cx88_IR *ir)
 
 		data = (data << 4) | ((gpio_key & 0xf0) >> 4);
 
-		rc_keydown(ir->dev, RC_TYPE_UNKNOWN, data, 0);
+		rc_keydown(ir->dev, RC_PROTO_UNKNOWN, data, 0);
 
 	} else if (ir->core->boardnr == CX88_BOARD_PROLINK_PLAYTVPVR ||
 		   ir->core->boardnr == CX88_BOARD_PIXELVIEW_PLAYTV_ULTRA_PRO) {
@@ -146,7 +146,7 @@ static void cx88_ir_handle_key(struct cx88_IR *ir)
 		scancode = RC_SCANCODE_NECX(addr, cmd);
 
 		if (0 == (gpio & ir->mask_keyup))
-			rc_keydown_notimeout(ir->dev, RC_TYPE_NECX, scancode,
+			rc_keydown_notimeout(ir->dev, RC_PROTO_NECX, scancode,
 					     0);
 		else
 			rc_keyup(ir->dev);
@@ -154,20 +154,22 @@ static void cx88_ir_handle_key(struct cx88_IR *ir)
 	} else if (ir->mask_keydown) {
 		/* bit set on keydown */
 		if (gpio & ir->mask_keydown)
-			rc_keydown_notimeout(ir->dev, RC_TYPE_UNKNOWN, data, 0);
+			rc_keydown_notimeout(ir->dev, RC_PROTO_UNKNOWN, data,
+					     0);
 		else
 			rc_keyup(ir->dev);
 
 	} else if (ir->mask_keyup) {
 		/* bit cleared on keydown */
 		if (0 == (gpio & ir->mask_keyup))
-			rc_keydown_notimeout(ir->dev, RC_TYPE_UNKNOWN, data, 0);
+			rc_keydown_notimeout(ir->dev, RC_PROTO_UNKNOWN, data,
+					     0);
 		else
 			rc_keyup(ir->dev);
 
 	} else {
 		/* can't distinguish keydown/up :-/ */
-		rc_keydown_notimeout(ir->dev, RC_TYPE_UNKNOWN, data, 0);
+		rc_keydown_notimeout(ir->dev, RC_PROTO_UNKNOWN, data, 0);
 		rc_keyup(ir->dev);
 	}
 }
@@ -267,7 +269,7 @@ int cx88_ir_init(struct cx88_core *core, struct pci_dev *pci)
 	struct cx88_IR *ir;
 	struct rc_dev *dev;
 	char *ir_codes = NULL;
-	u64 rc_type = RC_BIT_OTHER;
+	u64 rc_proto = RC_PROTO_BIT_OTHER;
 	int err = -ENOMEM;
 	u32 hardware_mask = 0;	/* For devices with a hardware mask, when
 				 * used with a full-code IR table
@@ -348,7 +350,7 @@ int cx88_ir_init(struct cx88_core *core, struct pci_dev *pci)
 		 * 002-T mini RC, provided with newer PV hardware
 		 */
 		ir_codes = RC_MAP_PIXELVIEW_MK12;
-		rc_type = RC_BIT_NECX;
+		rc_proto = RC_PROTO_BIT_NECX;
 		ir->gpio_addr = MO_GP1_IO;
 		ir->mask_keyup = 0x80;
 		ir->polling = 10; /* ms */
@@ -464,7 +466,7 @@ int cx88_ir_init(struct cx88_core *core, struct pci_dev *pci)
 	snprintf(ir->name, sizeof(ir->name), "cx88 IR (%s)", core->board.name);
 	snprintf(ir->phys, sizeof(ir->phys), "pci-%s/ir0", pci_name(pci));
 
-	dev->input_name = ir->name;
+	dev->device_name = ir->name;
 	dev->input_phys = ir->phys;
 	dev->input_id.bustype = BUS_PCI;
 	dev->input_id.version = 1;
@@ -487,7 +489,7 @@ int cx88_ir_init(struct cx88_core *core, struct pci_dev *pci)
 		dev->timeout = 10 * 1000 * 1000; /* 10 ms */
 	} else {
 		dev->driver_type = RC_DRIVER_SCANCODE;
-		dev->allowed_protocols = rc_type;
+		dev->allowed_protocols = rc_proto;
 	}
 
 	ir->core = core;
@@ -557,7 +559,7 @@ void cx88_ir_irq(struct cx88_core *core)
 	ir_raw_event_handle(ir->dev);
 }
 
-static int get_key_pvr2000(struct IR_i2c *ir, enum rc_type *protocol,
+static int get_key_pvr2000(struct IR_i2c *ir, enum rc_proto *protocol,
 			   u32 *scancode, u8 *toggle)
 {
 	int flags, code;
@@ -582,7 +584,7 @@ static int get_key_pvr2000(struct IR_i2c *ir, enum rc_type *protocol,
 	dprintk("IR Key/Flags: (0x%02x/0x%02x)\n",
 		code & 0xff, flags & 0xff);
 
-	*protocol = RC_TYPE_UNKNOWN;
+	*protocol = RC_PROTO_UNKNOWN;
 	*scancode = code & 0xff;
 	*toggle = 0;
 	return 1;
@@ -612,7 +614,7 @@ void cx88_i2c_init_ir(struct cx88_core *core)
 	case CX88_BOARD_LEADTEK_PVR2000:
 		addr_list = pvr2000_addr_list;
 		core->init_data.name = "cx88 Leadtek PVR 2000 remote";
-		core->init_data.type = RC_BIT_UNKNOWN;
+		core->init_data.type = RC_PROTO_BIT_UNKNOWN;
 		core->init_data.get_key = get_key_pvr2000;
 		core->init_data.ir_codes = RC_MAP_EMPTY;
 		break;
@@ -633,8 +635,8 @@ void cx88_i2c_init_ir(struct cx88_core *core)
 			/* Hauppauge XVR */
 			core->init_data.name = "cx88 Hauppauge XVR remote";
 			core->init_data.ir_codes = RC_MAP_HAUPPAUGE;
-			core->init_data.type = RC_BIT_RC5 | RC_BIT_RC6_MCE |
-							RC_BIT_RC6_6A_32;
+			core->init_data.type = RC_PROTO_BIT_RC5 |
+				RC_PROTO_BIT_RC6_MCE | RC_PROTO_BIT_RC6_6A_32;
 			core->init_data.internal_get_key_func = IR_KBD_GET_KEY_HAUP_XVR;
 
 			info.platform_data = &core->init_data;
diff --git a/drivers/media/pci/ddbridge/Kconfig b/drivers/media/pci/ddbridge/Kconfig
index ffed78c2ffb4..f43d0b83fc0c 100644
--- a/drivers/media/pci/ddbridge/Kconfig
+++ b/drivers/media/pci/ddbridge/Kconfig
@@ -8,7 +8,11 @@ config DVB_DDBRIDGE
 	select DVB_TDA18271C2DD if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_STV0367 if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_CXD2841ER if MEDIA_SUBDRV_AUTOSELECT
+	select DVB_STV0910 if MEDIA_SUBDRV_AUTOSELECT
+	select DVB_STV6111 if MEDIA_SUBDRV_AUTOSELECT
+	select DVB_LNBH25 if MEDIA_SUBDRV_AUTOSELECT
 	select MEDIA_TUNER_TDA18212 if MEDIA_SUBDRV_AUTOSELECT
+	select DVB_MXL5XX if MEDIA_SUBDRV_AUTOSELECT
 	---help---
 	  Support for cards with the Digital Devices PCI express bridge:
 	  - Octopus PCIe Bridge
@@ -20,5 +24,22 @@ config DVB_DDBRIDGE
 	  - CineCTv6 and DuoFlex CT (STV0367-based)
 	  - CineCTv7 and DuoFlex CT2/C2T2/C2T2I (Sony CXD28xx-based)
 	  - MaxA8 series
+	  - CineS2 V7/V7A and DuoFlex S2 V4 (ST STV0910-based)
+	  - Max S4/8
 
 	  Say Y if you own such a card and want to use it.
+
+config DVB_DDBRIDGE_MSIENABLE
+	bool "Enable Message Signaled Interrupts (MSI) per default (EXPERIMENTAL)"
+	depends on DVB_DDBRIDGE
+	depends on PCI_MSI
+	default n
+	---help---
+	  Use PCI MSI (Message Signaled Interrupts) per default. Enabling this
+	  might lead to I2C errors originating from the bridge in conjunction
+	  with certain SATA controllers, requiring a reload of the ddbridge
+	  module. MSI can still be disabled by passing msi=0 as option, as
+	  this will just change the msi option default value.
+
+	  If you're unsure, concerned about stability and don't want to pass
+	  module options in case of troubles, say N.
diff --git a/drivers/media/pci/ddbridge/Makefile b/drivers/media/pci/ddbridge/Makefile
index 7446c8b677b5..09703312a3f1 100644
--- a/drivers/media/pci/ddbridge/Makefile
+++ b/drivers/media/pci/ddbridge/Makefile
@@ -2,7 +2,8 @@
 # Makefile for the ddbridge device driver
 #
 
-ddbridge-objs := ddbridge-core.o
+ddbridge-objs := ddbridge-main.o ddbridge-core.o ddbridge-hw.o \
+		ddbridge-i2c.o ddbridge-maxs8.o
 
 obj-$(CONFIG_DVB_DDBRIDGE) += ddbridge.o
 
diff --git a/drivers/media/pci/ddbridge/ddbridge-core.c b/drivers/media/pci/ddbridge/ddbridge-core.c
index cd1723e79a07..f4bd4908acdd 100644
--- a/drivers/media/pci/ddbridge/ddbridge-core.c
+++ b/drivers/media/pci/ddbridge/ddbridge-core.c
@@ -1,7 +1,10 @@
 /*
- * ddbridge.c: Digital Devices PCIe bridge driver
+ * ddbridge-core.c: Digital Devices bridge core functions
+ *
+ * Copyright (C) 2010-2017 Digital Devices GmbH
+ *                         Marcus Metzler <mocm@metzlerbros.de>
+ *                         Ralph Metzler <rjkm@metzlerbros.de>
  *
- * Copyright (C) 2010-2011 Digital Devices GmbH
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,8 +20,6 @@
  * http://www.gnu.org/copyleft/gpl.html
  */
 
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
@@ -32,9 +33,12 @@
 #include <linux/i2c.h>
 #include <linux/swab.h>
 #include <linux/vmalloc.h>
-#include "ddbridge.h"
 
+#include "ddbridge.h"
+#include "ddbridge-i2c.h"
 #include "ddbridge-regs.h"
+#include "ddbridge-maxs8.h"
+#include "ddbridge-io.h"
 
 #include "tda18271c2dd.h"
 #include "stv6110x.h"
@@ -45,446 +49,511 @@
 #include "stv0367_priv.h"
 #include "cxd2841er.h"
 #include "tda18212.h"
+#include "stv0910.h"
+#include "stv6111.h"
+#include "lnbh25.h"
+#include "cxd2099.h"
 
-static int xo2_speed = 2;
-module_param(xo2_speed, int, 0444);
-MODULE_PARM_DESC(xo2_speed, "default transfer speed for xo2 based duoflex, 0=55,1=75,2=90,3=104 MBit/s, default=2, use attribute to change for individual cards");
-
-DVB_DEFINE_MOD_OPT_ADAPTER_NR(adapter_nr);
+/****************************************************************************/
 
-/* MSI had problems with lost interrupts, fixed but needs testing */
-#undef CONFIG_PCI_MSI
+#define DDB_MAX_ADAPTER 64
 
-/******************************************************************************/
+/****************************************************************************/
 
-static int i2c_io(struct i2c_adapter *adapter, u8 adr,
-		  u8 *wbuf, u32 wlen, u8 *rbuf, u32 rlen)
-{
-	struct i2c_msg msgs[2] = {{.addr = adr,  .flags = 0,
-				   .buf  = wbuf, .len   = wlen },
-				  {.addr = adr,  .flags = I2C_M_RD,
-				   .buf  = rbuf,  .len   = rlen } };
-	return (i2c_transfer(adapter, msgs, 2) == 2) ? 0 : -1;
-}
+DVB_DEFINE_MOD_OPT_ADAPTER_NR(adapter_nr);
 
-static int i2c_write(struct i2c_adapter *adap, u8 adr, u8 *data, int len)
-{
-	struct i2c_msg msg = {.addr = adr, .flags = 0,
-			      .buf = data, .len = len};
+static int adapter_alloc;
+module_param(adapter_alloc, int, 0444);
+MODULE_PARM_DESC(adapter_alloc,
+		 "0-one adapter per io, 1-one per tab with io, 2-one per tab, 3-one for all");
 
-	return (i2c_transfer(adap, &msg, 1) == 1) ? 0 : -1;
-}
+/****************************************************************************/
 
-static int i2c_read(struct i2c_adapter *adapter, u8 adr, u8 *val)
-{
-	struct i2c_msg msgs[1] = {{.addr = adr,  .flags = I2C_M_RD,
-				   .buf  = val,  .len   = 1 } };
-	return (i2c_transfer(adapter, msgs, 1) == 1) ? 0 : -1;
-}
+static DEFINE_MUTEX(redirect_lock);
 
-static int i2c_read_regs(struct i2c_adapter *adapter,
-			 u8 adr, u8 reg, u8 *val, u8 len)
-{
-	struct i2c_msg msgs[2] = {{.addr = adr,  .flags = 0,
-				   .buf  = &reg, .len   = 1 },
-				  {.addr = adr,  .flags = I2C_M_RD,
-				   .buf  = val,  .len   = len } };
-	return (i2c_transfer(adapter, msgs, 2) == 2) ? 0 : -1;
-}
+struct workqueue_struct *ddb_wq;
 
-static int i2c_read_reg(struct i2c_adapter *adapter, u8 adr, u8 reg, u8 *val)
-{
-	return i2c_read_regs(adapter, adr, reg, val, 1);
-}
+static struct ddb *ddbs[DDB_MAX_ADAPTER];
 
-static int i2c_read_reg16(struct i2c_adapter *adapter, u8 adr,
-			  u16 reg, u8 *val)
-{
-	u8 msg[2] = {reg>>8, reg&0xff};
-	struct i2c_msg msgs[2] = {{.addr = adr, .flags = 0,
-				   .buf  = msg, .len   = 2},
-				  {.addr = adr, .flags = I2C_M_RD,
-				   .buf  = val, .len   = 1} };
-	return (i2c_transfer(adapter, msgs, 2) == 2) ? 0 : -1;
-}
+/****************************************************************************/
+/****************************************************************************/
+/****************************************************************************/
 
-static int i2c_write_reg(struct i2c_adapter *adap, u8 adr,
-			 u8 reg, u8 val)
+static void ddb_set_dma_table(struct ddb_io *io)
 {
-	u8 msg[2] = {reg, val};
+	struct ddb *dev = io->port->dev;
+	struct ddb_dma *dma = io->dma;
+	u32 i;
+	u64 mem;
 
-	return i2c_write(adap, adr, msg, 2);
+	if (!dma)
+		return;
+	for (i = 0; i < dma->num; i++) {
+		mem = dma->pbuf[i];
+		ddbwritel(dev, mem & 0xffffffff, dma->bufregs + i * 8);
+		ddbwritel(dev, mem >> 32, dma->bufregs + i * 8 + 4);
+	}
+	dma->bufval = ((dma->div & 0x0f) << 16) |
+		((dma->num & 0x1f) << 11) |
+		((dma->size >> 7) & 0x7ff);
 }
 
-static inline u32 safe_ddbreadl(struct ddb *dev, u32 adr)
+static void ddb_set_dma_tables(struct ddb *dev)
 {
-	u32 val = ddbreadl(adr);
+	u32 i;
 
-	/* (ddb)readl returns (uint)-1 (all bits set) on failure, catch that */
-	if (val == ~0) {
-		dev_err(&dev->pdev->dev, "ddbreadl failure, adr=%08x\n", adr);
-		return 0;
+	for (i = 0; i < DDB_MAX_PORT; i++) {
+		if (dev->port[i].input[0])
+			ddb_set_dma_table(dev->port[i].input[0]);
+		if (dev->port[i].input[1])
+			ddb_set_dma_table(dev->port[i].input[1]);
+		if (dev->port[i].output)
+			ddb_set_dma_table(dev->port[i].output);
 	}
-
-	return val;
 }
 
-static int ddb_i2c_cmd(struct ddb_i2c *i2c, u32 adr, u32 cmd)
+
+/****************************************************************************/
+/****************************************************************************/
+/****************************************************************************/
+
+static void ddb_redirect_dma(struct ddb *dev,
+			     struct ddb_dma *sdma,
+			     struct ddb_dma *ddma)
 {
-	struct ddb *dev = i2c->dev;
-	long stat;
-	u32 val;
+	u32 i, base;
+	u64 mem;
 
-	i2c->done = 0;
-	ddbwritel((adr << 9) | cmd, i2c->regs + I2C_COMMAND);
-	stat = wait_event_timeout(i2c->wq, i2c->done == 1, HZ);
-	if (stat == 0) {
-		dev_err(&dev->pdev->dev, "I2C timeout\n");
-		{ /* MSI debugging*/
-			u32 istat = ddbreadl(INTERRUPT_STATUS);
-			dev_err(&dev->pdev->dev, "IRS %08x\n", istat);
-			ddbwritel(istat, INTERRUPT_ACK);
-		}
-		return -EIO;
+	sdma->bufval = ddma->bufval;
+	base = sdma->bufregs;
+	for (i = 0; i < ddma->num; i++) {
+		mem = ddma->pbuf[i];
+		ddbwritel(dev, mem & 0xffffffff, base + i * 8);
+		ddbwritel(dev, mem >> 32, base + i * 8 + 4);
 	}
-	val = ddbreadl(i2c->regs+I2C_COMMAND);
-	if (val & 0x70000)
-		return -EIO;
-	return 0;
 }
 
-static int ddb_i2c_master_xfer(struct i2c_adapter *adapter,
-			       struct i2c_msg msg[], int num)
+static int ddb_unredirect(struct ddb_port *port)
 {
-	struct ddb_i2c *i2c = (struct ddb_i2c *)i2c_get_adapdata(adapter);
-	struct ddb *dev = i2c->dev;
-	u8 addr = 0;
-
-	if (num)
-		addr = msg[0].addr;
-
-	if (num == 2 && msg[1].flags & I2C_M_RD &&
-	    !(msg[0].flags & I2C_M_RD)) {
-		memcpy_toio(dev->regs + I2C_TASKMEM_BASE + i2c->wbuf,
-			    msg[0].buf, msg[0].len);
-		ddbwritel(msg[0].len|(msg[1].len << 16),
-			  i2c->regs+I2C_TASKLENGTH);
-		if (!ddb_i2c_cmd(i2c, addr, 1)) {
-			memcpy_fromio(msg[1].buf,
-				      dev->regs + I2C_TASKMEM_BASE + i2c->rbuf,
-				      msg[1].len);
-			return num;
-		}
-	}
+	struct ddb_input *oredi, *iredi = NULL;
+	struct ddb_output *iredo = NULL;
 
-	if (num == 1 && !(msg[0].flags & I2C_M_RD)) {
-		ddbcpyto(I2C_TASKMEM_BASE + i2c->wbuf, msg[0].buf, msg[0].len);
-		ddbwritel(msg[0].len, i2c->regs + I2C_TASKLENGTH);
-		if (!ddb_i2c_cmd(i2c, addr, 2))
-			return num;
+	/* dev_info(port->dev->dev,
+	 * "unredirect %d.%d\n", port->dev->nr, port->nr);
+	 */
+	mutex_lock(&redirect_lock);
+	if (port->output->dma->running) {
+		mutex_unlock(&redirect_lock);
+		return -EBUSY;
 	}
-	if (num == 1 && (msg[0].flags & I2C_M_RD)) {
-		ddbwritel(msg[0].len << 16, i2c->regs + I2C_TASKLENGTH);
-		if (!ddb_i2c_cmd(i2c, addr, 3)) {
-			ddbcpyfrom(msg[0].buf,
-				   I2C_TASKMEM_BASE + i2c->rbuf, msg[0].len);
-			return num;
+	oredi = port->output->redi;
+	if (!oredi)
+		goto done;
+	if (port->input[0]) {
+		iredi = port->input[0]->redi;
+		iredo = port->input[0]->redo;
+
+		if (iredo) {
+			iredo->port->output->redi = oredi;
+			if (iredo->port->input[0]) {
+				iredo->port->input[0]->redi = iredi;
+				ddb_redirect_dma(oredi->port->dev,
+						 oredi->dma, iredo->dma);
+			}
+			port->input[0]->redo = NULL;
+			ddb_set_dma_table(port->input[0]);
 		}
+		oredi->redi = iredi;
+		port->input[0]->redi = NULL;
 	}
-	return -EIO;
-}
-
-
-static u32 ddb_i2c_functionality(struct i2c_adapter *adap)
-{
-	return I2C_FUNC_SMBUS_EMUL;
-}
-
-static struct i2c_algorithm ddb_i2c_algo = {
-	.master_xfer   = ddb_i2c_master_xfer,
-	.functionality = ddb_i2c_functionality,
-};
-
-static void ddb_i2c_release(struct ddb *dev)
-{
-	int i;
-	struct ddb_i2c *i2c;
-	struct i2c_adapter *adap;
+	oredi->redo = NULL;
+	port->output->redi = NULL;
 
-	for (i = 0; i < dev->info->port_num; i++) {
-		i2c = &dev->i2c[i];
-		adap = &i2c->adap;
-		i2c_del_adapter(adap);
-	}
+	ddb_set_dma_table(oredi);
+done:
+	mutex_unlock(&redirect_lock);
+	return 0;
 }
 
-static int ddb_i2c_init(struct ddb *dev)
+static int ddb_redirect(u32 i, u32 p)
 {
-	int i, j, stat = 0;
-	struct ddb_i2c *i2c;
-	struct i2c_adapter *adap;
-
-	for (i = 0; i < dev->info->port_num; i++) {
-		i2c = &dev->i2c[i];
-		i2c->dev = dev;
-		i2c->nr = i;
-		i2c->wbuf = i * (I2C_TASKMEM_SIZE / 4);
-		i2c->rbuf = i2c->wbuf + (I2C_TASKMEM_SIZE / 8);
-		i2c->regs = 0x80 + i * 0x20;
-		ddbwritel(I2C_SPEED_100, i2c->regs + I2C_TIMING);
-		ddbwritel((i2c->rbuf << 16) | i2c->wbuf,
-			  i2c->regs + I2C_TASKADDRESS);
-		init_waitqueue_head(&i2c->wq);
-
-		adap = &i2c->adap;
-		i2c_set_adapdata(adap, i2c);
-#ifdef I2C_ADAP_CLASS_TV_DIGITAL
-		adap->class = I2C_ADAP_CLASS_TV_DIGITAL|I2C_CLASS_TV_ANALOG;
-#else
-#ifdef I2C_CLASS_TV_ANALOG
-		adap->class = I2C_CLASS_TV_ANALOG;
-#endif
-#endif
-		strcpy(adap->name, "ddbridge");
-		adap->algo = &ddb_i2c_algo;
-		adap->algo_data = (void *)i2c;
-		adap->dev.parent = &dev->pdev->dev;
-		stat = i2c_add_adapter(adap);
-		if (stat)
-			break;
-	}
-	if (stat)
-		for (j = 0; j < i; j++) {
-			i2c = &dev->i2c[j];
-			adap = &i2c->adap;
-			i2c_del_adapter(adap);
-		}
-	return stat;
-}
+	struct ddb *idev = ddbs[(i >> 4) & 0x3f];
+	struct ddb_input *input, *input2;
+	struct ddb *pdev = ddbs[(p >> 4) & 0x3f];
+	struct ddb_port *port;
 
+	if (!idev || !pdev)
+		return -EINVAL;
+	if (!idev->has_dma || !pdev->has_dma)
+		return -EINVAL;
 
-/******************************************************************************/
-/******************************************************************************/
-/******************************************************************************/
+	port = &pdev->port[p & 0x0f];
+	if (!port->output)
+		return -EINVAL;
+	if (ddb_unredirect(port))
+		return -EBUSY;
 
-#if 0
-static void set_table(struct ddb *dev, u32 off,
-		      dma_addr_t *pbuf, u32 num)
-{
-	u32 i, base;
-	u64 mem;
+	if (i == 8)
+		return 0;
 
-	base = DMA_BASE_ADDRESS_TABLE + off;
-	for (i = 0; i < num; i++) {
-		mem = pbuf[i];
-		ddbwritel(mem & 0xffffffff, base + i * 8);
-		ddbwritel(mem >> 32, base + i * 8 + 4);
-	}
-}
-#endif
+	input = &idev->input[i & 7];
+	if (!input)
+		return -EINVAL;
 
-static void ddb_address_table(struct ddb *dev)
-{
-	u32 i, j, base;
-	u64 mem;
-	dma_addr_t *pbuf;
-
-	for (i = 0; i < dev->info->port_num * 2; i++) {
-		base = DMA_BASE_ADDRESS_TABLE + i * 0x100;
-		pbuf = dev->input[i].pbuf;
-		for (j = 0; j < dev->input[i].dma_buf_num; j++) {
-			mem = pbuf[j];
-			ddbwritel(mem & 0xffffffff, base + j * 8);
-			ddbwritel(mem >> 32, base + j * 8 + 4);
-		}
+	mutex_lock(&redirect_lock);
+	if (port->output->dma->running || input->dma->running) {
+		mutex_unlock(&redirect_lock);
+		return -EBUSY;
 	}
-	for (i = 0; i < dev->info->port_num; i++) {
-		base = DMA_BASE_ADDRESS_TABLE + 0x800 + i * 0x100;
-		pbuf = dev->output[i].pbuf;
-		for (j = 0; j < dev->output[i].dma_buf_num; j++) {
-			mem = pbuf[j];
-			ddbwritel(mem & 0xffffffff, base + j * 8);
-			ddbwritel(mem >> 32, base + j * 8 + 4);
-		}
+	input2 = port->input[0];
+	if (input2) {
+		if (input->redi) {
+			input2->redi = input->redi;
+			input->redi = NULL;
+		} else
+			input2->redi = input;
 	}
+	input->redo = port->output;
+	port->output->redi = input;
+
+	ddb_redirect_dma(input->port->dev, input->dma, port->output->dma);
+	mutex_unlock(&redirect_lock);
+	return 0;
 }
 
-static void io_free(struct pci_dev *pdev, u8 **vbuf,
-		    dma_addr_t *pbuf, u32 size, int num)
+/****************************************************************************/
+/****************************************************************************/
+/****************************************************************************/
+
+static void dma_free(struct pci_dev *pdev, struct ddb_dma *dma, int dir)
 {
 	int i;
 
-	for (i = 0; i < num; i++) {
-		if (vbuf[i]) {
-			pci_free_consistent(pdev, size, vbuf[i], pbuf[i]);
-			vbuf[i] = NULL;
+	if (!dma)
+		return;
+	for (i = 0; i < dma->num; i++) {
+		if (dma->vbuf[i]) {
+			if (alt_dma) {
+				dma_unmap_single(&pdev->dev, dma->pbuf[i],
+						 dma->size,
+						 dir ? DMA_TO_DEVICE :
+						 DMA_FROM_DEVICE);
+				kfree(dma->vbuf[i]);
+				dma->vbuf[i] = NULL;
+			} else {
+				dma_free_coherent(&pdev->dev, dma->size,
+						  dma->vbuf[i], dma->pbuf[i]);
+			}
+
+			dma->vbuf[i] = NULL;
 		}
 	}
 }
 
-static int io_alloc(struct pci_dev *pdev, u8 **vbuf,
-		    dma_addr_t *pbuf, u32 size, int num)
+static int dma_alloc(struct pci_dev *pdev, struct ddb_dma *dma, int dir)
 {
 	int i;
 
-	for (i = 0; i < num; i++) {
-		vbuf[i] = pci_alloc_consistent(pdev, size, &pbuf[i]);
-		if (!vbuf[i])
-			return -ENOMEM;
+	if (!dma)
+		return 0;
+	for (i = 0; i < dma->num; i++) {
+		if (alt_dma) {
+			dma->vbuf[i] = kmalloc(dma->size, __GFP_RETRY_MAYFAIL);
+			if (!dma->vbuf[i])
+				return -ENOMEM;
+			dma->pbuf[i] = dma_map_single(&pdev->dev,
+						      dma->vbuf[i],
+						      dma->size,
+						      dir ? DMA_TO_DEVICE :
+						      DMA_FROM_DEVICE);
+			if (dma_mapping_error(&pdev->dev, dma->pbuf[i])) {
+				kfree(dma->vbuf[i]);
+				dma->vbuf[i] = NULL;
+				return -ENOMEM;
+			}
+		} else {
+			dma->vbuf[i] = dma_alloc_coherent(&pdev->dev,
+							  dma->size,
+							  &dma->pbuf[i],
+							  GFP_KERNEL);
+			if (!dma->vbuf[i])
+				return -ENOMEM;
+		}
 	}
 	return 0;
 }
 
-static int ddb_buffers_alloc(struct ddb *dev)
+int ddb_buffers_alloc(struct ddb *dev)
 {
 	int i;
 	struct ddb_port *port;
 
-	for (i = 0; i < dev->info->port_num; i++) {
+	for (i = 0; i < dev->port_num; i++) {
 		port = &dev->port[i];
 		switch (port->class) {
 		case DDB_PORT_TUNER:
-			if (io_alloc(dev->pdev, port->input[0]->vbuf,
-				     port->input[0]->pbuf,
-				     port->input[0]->dma_buf_size,
-				     port->input[0]->dma_buf_num) < 0)
-				return -1;
-			if (io_alloc(dev->pdev, port->input[1]->vbuf,
-				     port->input[1]->pbuf,
-				     port->input[1]->dma_buf_size,
-				     port->input[1]->dma_buf_num) < 0)
-				return -1;
+			if (port->input[0]->dma)
+				if (dma_alloc(dev->pdev, port->input[0]->dma, 0)
+					< 0)
+					return -1;
+			if (port->input[1]->dma)
+				if (dma_alloc(dev->pdev, port->input[1]->dma, 0)
+					< 0)
+					return -1;
 			break;
 		case DDB_PORT_CI:
-			if (io_alloc(dev->pdev, port->input[0]->vbuf,
-				     port->input[0]->pbuf,
-				     port->input[0]->dma_buf_size,
-				     port->input[0]->dma_buf_num) < 0)
-				return -1;
-			if (io_alloc(dev->pdev, port->output->vbuf,
-				     port->output->pbuf,
-				     port->output->dma_buf_size,
-				     port->output->dma_buf_num) < 0)
-				return -1;
+		case DDB_PORT_LOOP:
+			if (port->input[0]->dma)
+				if (dma_alloc(dev->pdev, port->input[0]->dma, 0)
+					< 0)
+					return -1;
+			if (port->output->dma)
+				if (dma_alloc(dev->pdev, port->output->dma, 1)
+					< 0)
+					return -1;
 			break;
 		default:
 			break;
 		}
 	}
-	ddb_address_table(dev);
+	ddb_set_dma_tables(dev);
 	return 0;
 }
 
-static void ddb_buffers_free(struct ddb *dev)
+void ddb_buffers_free(struct ddb *dev)
 {
 	int i;
 	struct ddb_port *port;
 
-	for (i = 0; i < dev->info->port_num; i++) {
+	for (i = 0; i < dev->port_num; i++) {
 		port = &dev->port[i];
-		io_free(dev->pdev, port->input[0]->vbuf,
-			port->input[0]->pbuf,
-			port->input[0]->dma_buf_size,
-			port->input[0]->dma_buf_num);
-		io_free(dev->pdev, port->input[1]->vbuf,
-			port->input[1]->pbuf,
-			port->input[1]->dma_buf_size,
-			port->input[1]->dma_buf_num);
-		io_free(dev->pdev, port->output->vbuf,
-			port->output->pbuf,
-			port->output->dma_buf_size,
-			port->output->dma_buf_num);
+
+		if (port->input[0] && port->input[0]->dma)
+			dma_free(dev->pdev, port->input[0]->dma, 0);
+		if (port->input[1] && port->input[1]->dma)
+			dma_free(dev->pdev, port->input[1]->dma, 0);
+		if (port->output && port->output->dma)
+			dma_free(dev->pdev, port->output->dma, 1);
 	}
 }
 
-static void ddb_input_start(struct ddb_input *input)
+static void calc_con(struct ddb_output *output, u32 *con, u32 *con2, u32 flags)
 {
-	struct ddb *dev = input->port->dev;
+	struct ddb *dev = output->port->dev;
+	u32 bitrate = output->port->obr, max_bitrate = 72000;
+	u32 gap = 4, nco = 0;
+
+	*con = 0x1c;
+	if (output->port->gap != 0xffffffff) {
+		flags |= 1;
+		gap = output->port->gap;
+		max_bitrate = 0;
+	}
+	if (dev->link[0].info->type == DDB_OCTOPUS_CI && output->port->nr > 1) {
+		*con = 0x10c;
+		if (dev->link[0].ids.regmapid >= 0x10003 && !(flags & 1)) {
+			if (!(flags & 2)) {
+				/* NCO */
+				max_bitrate = 0;
+				gap = 0;
+				if (bitrate != 72000) {
+					if (bitrate >= 96000)
+						*con |= 0x800;
+					else {
+						*con |= 0x1000;
+						nco = (bitrate * 8192 + 71999)
+							/ 72000;
+					}
+				}
+			} else {
+				/* Divider and gap */
+				*con |= 0x1810;
+				if (bitrate <= 64000) {
+					max_bitrate = 64000;
+					nco = 8;
+				} else if (bitrate <= 72000) {
+					max_bitrate = 72000;
+					nco = 7;
+				} else {
+					max_bitrate = 96000;
+					nco = 5;
+				}
+			}
+		} else {
+			if (bitrate > 72000) {
+				*con |= 0x810; /* 96 MBit/s and gap */
+				max_bitrate = 96000;
+			}
+			*con |= 0x10; /* enable gap */
+		}
+	}
+	if (max_bitrate > 0) {
+		if (bitrate > max_bitrate)
+			bitrate = max_bitrate;
+		if (bitrate < 31000)
+			bitrate = 31000;
+		gap = ((max_bitrate - bitrate) * 94) / bitrate;
+		if (gap < 2)
+			*con &= ~0x10; /* Disable gap */
+		else
+			gap -= 2;
+		if (gap > 127)
+			gap = 127;
+	}
+
+	*con2 = (nco << 16) | gap;
+}
 
-	spin_lock_irq(&input->lock);
-	input->cbuf = 0;
-	input->coff = 0;
+static void ddb_output_start(struct ddb_output *output)
+{
+	struct ddb *dev = output->port->dev;
+	u32 con = 0x11c, con2 = 0;
+
+	if (output->dma) {
+		spin_lock_irq(&output->dma->lock);
+		output->dma->cbuf = 0;
+		output->dma->coff = 0;
+		output->dma->stat = 0;
+		ddbwritel(dev, 0, DMA_BUFFER_CONTROL(output->dma));
+	}
 
-	/* reset */
-	ddbwritel(0, TS_INPUT_CONTROL(input->nr));
-	ddbwritel(2, TS_INPUT_CONTROL(input->nr));
-	ddbwritel(0, TS_INPUT_CONTROL(input->nr));
+	if (output->port->input[0]->port->class == DDB_PORT_LOOP)
+		con = (1UL << 13) | 0x14;
+	else
+		calc_con(output, &con, &con2, 0);
+
+	ddbwritel(dev, 0, TS_CONTROL(output));
+	ddbwritel(dev, 2, TS_CONTROL(output));
+	ddbwritel(dev, 0, TS_CONTROL(output));
+	ddbwritel(dev, con, TS_CONTROL(output));
+	ddbwritel(dev, con2, TS_CONTROL2(output));
+
+	if (output->dma) {
+		ddbwritel(dev, output->dma->bufval,
+			  DMA_BUFFER_SIZE(output->dma));
+		ddbwritel(dev, 0, DMA_BUFFER_ACK(output->dma));
+		ddbwritel(dev, 1, DMA_BASE_READ);
+		ddbwritel(dev, 7, DMA_BUFFER_CONTROL(output->dma));
+	}
 
-	ddbwritel((1 << 16) |
-		  (input->dma_buf_num << 11) |
-		  (input->dma_buf_size >> 7),
-		  DMA_BUFFER_SIZE(input->nr));
-	ddbwritel(0, DMA_BUFFER_ACK(input->nr));
+	ddbwritel(dev, con | 1, TS_CONTROL(output));
 
-	ddbwritel(1, DMA_BASE_WRITE);
-	ddbwritel(3, DMA_BUFFER_CONTROL(input->nr));
-	ddbwritel(9, TS_INPUT_CONTROL(input->nr));
-	input->running = 1;
-	spin_unlock_irq(&input->lock);
+	if (output->dma) {
+		output->dma->running = 1;
+		spin_unlock_irq(&output->dma->lock);
+	}
+}
+
+static void ddb_output_stop(struct ddb_output *output)
+{
+	struct ddb *dev = output->port->dev;
+
+	if (output->dma)
+		spin_lock_irq(&output->dma->lock);
+
+	ddbwritel(dev, 0, TS_CONTROL(output));
+
+	if (output->dma) {
+		ddbwritel(dev, 0, DMA_BUFFER_CONTROL(output->dma));
+		output->dma->running = 0;
+		spin_unlock_irq(&output->dma->lock);
+	}
 }
 
 static void ddb_input_stop(struct ddb_input *input)
 {
 	struct ddb *dev = input->port->dev;
+	u32 tag = DDB_LINK_TAG(input->port->lnr);
+
+	if (input->dma)
+		spin_lock_irq(&input->dma->lock);
+	ddbwritel(dev, 0, tag | TS_CONTROL(input));
+	if (input->dma) {
+		ddbwritel(dev, 0, DMA_BUFFER_CONTROL(input->dma));
+		input->dma->running = 0;
+		spin_unlock_irq(&input->dma->lock);
+	}
+}
+
+static void ddb_input_start(struct ddb_input *input)
+{
+	struct ddb *dev = input->port->dev;
+
+	if (input->dma) {
+		spin_lock_irq(&input->dma->lock);
+		input->dma->cbuf = 0;
+		input->dma->coff = 0;
+		input->dma->stat = 0;
+		ddbwritel(dev, 0, DMA_BUFFER_CONTROL(input->dma));
+	}
+	ddbwritel(dev, 0, TS_CONTROL(input));
+	ddbwritel(dev, 2, TS_CONTROL(input));
+	ddbwritel(dev, 0, TS_CONTROL(input));
+
+	if (input->dma) {
+		ddbwritel(dev, input->dma->bufval,
+			  DMA_BUFFER_SIZE(input->dma));
+		ddbwritel(dev, 0, DMA_BUFFER_ACK(input->dma));
+		ddbwritel(dev, 1, DMA_BASE_WRITE);
+		ddbwritel(dev, 3, DMA_BUFFER_CONTROL(input->dma));
+	}
 
-	spin_lock_irq(&input->lock);
-	ddbwritel(0, TS_INPUT_CONTROL(input->nr));
-	ddbwritel(0, DMA_BUFFER_CONTROL(input->nr));
-	input->running = 0;
-	spin_unlock_irq(&input->lock);
+	ddbwritel(dev, 0x09, TS_CONTROL(input));
+
+	if (input->dma) {
+		input->dma->running = 1;
+		spin_unlock_irq(&input->dma->lock);
+	}
 }
 
-static void ddb_output_start(struct ddb_output *output)
+
+static void ddb_input_start_all(struct ddb_input *input)
 {
-	struct ddb *dev = output->port->dev;
+	struct ddb_input *i = input;
+	struct ddb_output *o;
 
-	spin_lock_irq(&output->lock);
-	output->cbuf = 0;
-	output->coff = 0;
-	ddbwritel(0, TS_OUTPUT_CONTROL(output->nr));
-	ddbwritel(2, TS_OUTPUT_CONTROL(output->nr));
-	ddbwritel(0, TS_OUTPUT_CONTROL(output->nr));
-	ddbwritel(0x3c, TS_OUTPUT_CONTROL(output->nr));
-	ddbwritel((1 << 16) |
-		  (output->dma_buf_num << 11) |
-		  (output->dma_buf_size >> 7),
-		  DMA_BUFFER_SIZE(output->nr + 8));
-	ddbwritel(0, DMA_BUFFER_ACK(output->nr + 8));
-
-	ddbwritel(1, DMA_BASE_READ);
-	ddbwritel(3, DMA_BUFFER_CONTROL(output->nr + 8));
-	/* ddbwritel(0xbd, TS_OUTPUT_CONTROL(output->nr)); */
-	ddbwritel(0x1d, TS_OUTPUT_CONTROL(output->nr));
-	output->running = 1;
-	spin_unlock_irq(&output->lock);
+	mutex_lock(&redirect_lock);
+	while (i && (o = i->redo)) {
+		ddb_output_start(o);
+		i = o->port->input[0];
+		if (i)
+			ddb_input_start(i);
+	}
+	ddb_input_start(input);
+	mutex_unlock(&redirect_lock);
 }
 
-static void ddb_output_stop(struct ddb_output *output)
+static void ddb_input_stop_all(struct ddb_input *input)
 {
-	struct ddb *dev = output->port->dev;
+	struct ddb_input *i = input;
+	struct ddb_output *o;
 
-	spin_lock_irq(&output->lock);
-	ddbwritel(0, TS_OUTPUT_CONTROL(output->nr));
-	ddbwritel(0, DMA_BUFFER_CONTROL(output->nr + 8));
-	output->running = 0;
-	spin_unlock_irq(&output->lock);
+	mutex_lock(&redirect_lock);
+	ddb_input_stop(input);
+	while (i && (o = i->redo)) {
+		ddb_output_stop(o);
+		i = o->port->input[0];
+		if (i)
+			ddb_input_stop(i);
+	}
+	mutex_unlock(&redirect_lock);
 }
 
 static u32 ddb_output_free(struct ddb_output *output)
 {
-	u32 idx, off, stat = output->stat;
+	u32 idx, off, stat = output->dma->stat;
 	s32 diff;
 
 	idx = (stat >> 11) & 0x1f;
 	off = (stat & 0x7ff) << 7;
 
-	if (output->cbuf != idx) {
-		if ((((output->cbuf + 1) % output->dma_buf_num) == idx) &&
-		    (output->dma_buf_size - output->coff <= 188))
+	if (output->dma->cbuf != idx) {
+		if ((((output->dma->cbuf + 1) % output->dma->num) == idx) &&
+		    (output->dma->size - output->dma->coff <= 188))
 			return 0;
 		return 188;
 	}
-	diff = off - output->coff;
+	diff = off - output->dma->coff;
 	if (diff <= 0 || diff > 188)
 		return 188;
 	return 0;
@@ -494,46 +563,51 @@ static ssize_t ddb_output_write(struct ddb_output *output,
 				const __user u8 *buf, size_t count)
 {
 	struct ddb *dev = output->port->dev;
-	u32 idx, off, stat = output->stat;
+	u32 idx, off, stat = output->dma->stat;
 	u32 left = count, len;
 
 	idx = (stat >> 11) & 0x1f;
 	off = (stat & 0x7ff) << 7;
 
 	while (left) {
-		len = output->dma_buf_size - output->coff;
-		if ((((output->cbuf + 1) % output->dma_buf_num) == idx) &&
+		len = output->dma->size - output->dma->coff;
+		if ((((output->dma->cbuf + 1) % output->dma->num) == idx) &&
 		    (off == 0)) {
 			if (len <= 188)
 				break;
 			len -= 188;
 		}
-		if (output->cbuf == idx) {
-			if (off > output->coff) {
-#if 1
-				len = off - output->coff;
+		if (output->dma->cbuf == idx) {
+			if (off > output->dma->coff) {
+				len = off - output->dma->coff;
 				len -= (len % 188);
 				if (len <= 188)
-
-#endif
 					break;
 				len -= 188;
 			}
 		}
 		if (len > left)
 			len = left;
-		if (copy_from_user(output->vbuf[output->cbuf] + output->coff,
+		if (copy_from_user(output->dma->vbuf[output->dma->cbuf] +
+				   output->dma->coff,
 				   buf, len))
 			return -EIO;
+		if (alt_dma)
+			dma_sync_single_for_device(dev->dev,
+				output->dma->pbuf[output->dma->cbuf],
+				output->dma->size, DMA_TO_DEVICE);
 		left -= len;
 		buf += len;
-		output->coff += len;
-		if (output->coff == output->dma_buf_size) {
-			output->coff = 0;
-			output->cbuf = ((output->cbuf + 1) % output->dma_buf_num);
+		output->dma->coff += len;
+		if (output->dma->coff == output->dma->size) {
+			output->dma->coff = 0;
+			output->dma->cbuf = ((output->dma->cbuf + 1) %
+					     output->dma->num);
 		}
-		ddbwritel((output->cbuf << 11) | (output->coff >> 7),
-			  DMA_BUFFER_ACK(output->nr + 8));
+		ddbwritel(dev,
+			  (output->dma->cbuf << 11) |
+			  (output->dma->coff >> 7),
+			  DMA_BUFFER_ACK(output->dma));
 	}
 	return count - left;
 }
@@ -541,81 +615,229 @@ static ssize_t ddb_output_write(struct ddb_output *output,
 static u32 ddb_input_avail(struct ddb_input *input)
 {
 	struct ddb *dev = input->port->dev;
-	u32 idx, off, stat = input->stat;
-	u32 ctrl = ddbreadl(DMA_BUFFER_CONTROL(input->nr));
+	u32 idx, off, stat = input->dma->stat;
+	u32 ctrl = ddbreadl(dev, DMA_BUFFER_CONTROL(input->dma));
 
 	idx = (stat >> 11) & 0x1f;
 	off = (stat & 0x7ff) << 7;
 
 	if (ctrl & 4) {
-		dev_err(&dev->pdev->dev, "IA %d %d %08x\n", idx, off, ctrl);
-		ddbwritel(input->stat, DMA_BUFFER_ACK(input->nr));
+		dev_err(dev->dev, "IA %d %d %08x\n", idx, off, ctrl);
+		ddbwritel(dev, stat, DMA_BUFFER_ACK(input->dma));
 		return 0;
 	}
-	if (input->cbuf != idx)
+	if (input->dma->cbuf != idx)
 		return 188;
 	return 0;
 }
 
-static ssize_t ddb_input_read(struct ddb_input *input, __user u8 *buf, size_t count)
+static ssize_t ddb_input_read(struct ddb_input *input,
+		__user u8 *buf, size_t count)
 {
 	struct ddb *dev = input->port->dev;
 	u32 left = count;
-	u32 idx, free, stat = input->stat;
+	u32 idx, free, stat = input->dma->stat;
 	int ret;
 
 	idx = (stat >> 11) & 0x1f;
 
 	while (left) {
-		if (input->cbuf == idx)
+		if (input->dma->cbuf == idx)
 			return count - left;
-		free = input->dma_buf_size - input->coff;
+		free = input->dma->size - input->dma->coff;
 		if (free > left)
 			free = left;
-		ret = copy_to_user(buf, input->vbuf[input->cbuf] +
-				   input->coff, free);
+		if (alt_dma)
+			dma_sync_single_for_cpu(dev->dev,
+				input->dma->pbuf[input->dma->cbuf],
+				input->dma->size, DMA_FROM_DEVICE);
+		ret = copy_to_user(buf, input->dma->vbuf[input->dma->cbuf] +
+				   input->dma->coff, free);
 		if (ret)
 			return -EFAULT;
-		input->coff += free;
-		if (input->coff == input->dma_buf_size) {
-			input->coff = 0;
-			input->cbuf = (input->cbuf+1) % input->dma_buf_num;
+		input->dma->coff += free;
+		if (input->dma->coff == input->dma->size) {
+			input->dma->coff = 0;
+			input->dma->cbuf = (input->dma->cbuf + 1) %
+				input->dma->num;
 		}
 		left -= free;
-		ddbwritel((input->cbuf << 11) | (input->coff >> 7),
-			  DMA_BUFFER_ACK(input->nr));
+		buf += free;
+		ddbwritel(dev,
+			  (input->dma->cbuf << 11) | (input->dma->coff >> 7),
+			  DMA_BUFFER_ACK(input->dma));
 	}
 	return count;
 }
 
-/******************************************************************************/
-/******************************************************************************/
-/******************************************************************************/
+/****************************************************************************/
+/****************************************************************************/
 
-#if 0
-static struct ddb_input *fe2input(struct ddb *dev, struct dvb_frontend *fe)
+static ssize_t ts_write(struct file *file, const __user char *buf,
+			size_t count, loff_t *ppos)
 {
-	int i;
+	struct dvb_device *dvbdev = file->private_data;
+	struct ddb_output *output = dvbdev->priv;
+	struct ddb *dev = output->port->dev;
+	size_t left = count;
+	int stat;
 
-	for (i = 0; i < dev->info->port_num * 2; i++) {
-		if (dev->input[i].fe == fe)
-			return &dev->input[i];
+	if (!dev->has_dma)
+		return -EINVAL;
+	while (left) {
+		if (ddb_output_free(output) < 188) {
+			if (file->f_flags & O_NONBLOCK)
+				break;
+			if (wait_event_interruptible(
+				    output->dma->wq,
+				    ddb_output_free(output) >= 188) < 0)
+				break;
+		}
+		stat = ddb_output_write(output, buf, left);
+		if (stat < 0)
+			return stat;
+		buf += stat;
+		left -= stat;
 	}
-	return NULL;
+	return (left == count) ? -EAGAIN : (count - left);
 }
-#endif
 
-static int drxk_gate_ctrl(struct dvb_frontend *fe, int enable)
+static ssize_t ts_read(struct file *file, __user char *buf,
+		       size_t count, loff_t *ppos)
+{
+	struct dvb_device *dvbdev = file->private_data;
+	struct ddb_output *output = dvbdev->priv;
+	struct ddb_input *input = output->port->input[0];
+	struct ddb *dev = output->port->dev;
+	size_t left = count;
+	int stat;
+
+	if (!dev->has_dma)
+		return -EINVAL;
+	while (left) {
+		if (ddb_input_avail(input) < 188) {
+			if (file->f_flags & O_NONBLOCK)
+				break;
+			if (wait_event_interruptible(
+				    input->dma->wq,
+				    ddb_input_avail(input) >= 188) < 0)
+				break;
+		}
+		stat = ddb_input_read(input, buf, left);
+		if (stat < 0)
+			return stat;
+		left -= stat;
+		buf += stat;
+	}
+	return (count && (left == count)) ? -EAGAIN : (count - left);
+}
+
+static unsigned int ts_poll(struct file *file, poll_table *wait)
+{
+	struct dvb_device *dvbdev = file->private_data;
+	struct ddb_output *output = dvbdev->priv;
+	struct ddb_input *input = output->port->input[0];
+
+	unsigned int mask = 0;
+
+	poll_wait(file, &input->dma->wq, wait);
+	poll_wait(file, &output->dma->wq, wait);
+	if (ddb_input_avail(input) >= 188)
+		mask |= POLLIN | POLLRDNORM;
+	if (ddb_output_free(output) >= 188)
+		mask |= POLLOUT | POLLWRNORM;
+	return mask;
+}
+
+static int ts_release(struct inode *inode, struct file *file)
+{
+	struct dvb_device *dvbdev = file->private_data;
+	struct ddb_output *output = NULL;
+	struct ddb_input *input = NULL;
+
+	if (dvbdev) {
+		output = dvbdev->priv;
+		input = output->port->input[0];
+	}
+
+	if ((file->f_flags & O_ACCMODE) == O_RDONLY) {
+		if (!input)
+			return -EINVAL;
+		ddb_input_stop(input);
+	} else if ((file->f_flags & O_ACCMODE) == O_WRONLY) {
+		if (!output)
+			return -EINVAL;
+		ddb_output_stop(output);
+	}
+	return dvb_generic_release(inode, file);
+}
+
+static int ts_open(struct inode *inode, struct file *file)
+{
+	int err;
+	struct dvb_device *dvbdev = file->private_data;
+	struct ddb_output *output = NULL;
+	struct ddb_input *input = NULL;
+
+	if (dvbdev) {
+		output = dvbdev->priv;
+		input = output->port->input[0];
+	}
+
+	if ((file->f_flags & O_ACCMODE) == O_RDONLY) {
+		if (!input)
+			return -EINVAL;
+		if (input->redo || input->redi)
+			return -EBUSY;
+	} else if ((file->f_flags & O_ACCMODE) == O_WRONLY) {
+		if (!output)
+			return -EINVAL;
+	} else
+		return -EINVAL;
+	err = dvb_generic_open(inode, file);
+	if (err < 0)
+		return err;
+	if ((file->f_flags & O_ACCMODE) == O_RDONLY)
+		ddb_input_start(input);
+	else if ((file->f_flags & O_ACCMODE) == O_WRONLY)
+		ddb_output_start(output);
+	return err;
+}
+
+static const struct file_operations ci_fops = {
+	.owner   = THIS_MODULE,
+	.read    = ts_read,
+	.write   = ts_write,
+	.open    = ts_open,
+	.release = ts_release,
+	.poll    = ts_poll,
+	.mmap    = NULL,
+};
+
+static struct dvb_device dvbdev_ci = {
+	.priv    = NULL,
+	.readers = 1,
+	.writers = 1,
+	.users   = 2,
+	.fops    = &ci_fops,
+};
+
+
+/****************************************************************************/
+/****************************************************************************/
+
+static int locked_gate_ctrl(struct dvb_frontend *fe, int enable)
 {
 	struct ddb_input *input = fe->sec_priv;
 	struct ddb_port *port = input->port;
+	struct ddb_dvb *dvb = &port->dvb[input->nr & 1];
 	int status;
 
 	if (enable) {
 		mutex_lock(&port->i2c_gate_lock);
-		status = input->gate_ctrl(fe, 1);
+		status = dvb->i2c_gate_ctrl(fe, 1);
 	} else {
-		status = input->gate_ctrl(fe, 0);
+		status = dvb->i2c_gate_ctrl(fe, 0);
 		mutex_unlock(&port->i2c_gate_lock);
 	}
 	return status;
@@ -624,41 +846,42 @@ static int drxk_gate_ctrl(struct dvb_frontend *fe, int enable)
 static int demod_attach_drxk(struct ddb_input *input)
 {
 	struct i2c_adapter *i2c = &input->port->i2c->adap;
+	struct ddb_dvb *dvb = &input->port->dvb[input->nr & 1];
+	struct device *dev = input->port->dev->dev;
 	struct dvb_frontend *fe;
 	struct drxk_config config;
-	struct device *dev = &input->port->dev->pdev->dev;
 
 	memset(&config, 0, sizeof(config));
-	config.microcode_name = "drxk_a3.mc";
-	config.qam_demod_parameter_count = 4;
 	config.adr = 0x29 + (input->nr & 1);
+	config.microcode_name = "drxk_a3.mc";
 
-	fe = input->fe = dvb_attach(drxk_attach, &config, i2c);
-	if (!input->fe) {
+	fe = dvb->fe = dvb_attach(drxk_attach, &config, i2c);
+	if (!fe) {
 		dev_err(dev, "No DRXK found!\n");
 		return -ENODEV;
 	}
 	fe->sec_priv = input;
-	input->gate_ctrl = fe->ops.i2c_gate_ctrl;
-	fe->ops.i2c_gate_ctrl = drxk_gate_ctrl;
+	dvb->i2c_gate_ctrl = fe->ops.i2c_gate_ctrl;
+	fe->ops.i2c_gate_ctrl = locked_gate_ctrl;
 	return 0;
 }
 
 static int tuner_attach_tda18271(struct ddb_input *input)
 {
 	struct i2c_adapter *i2c = &input->port->i2c->adap;
+	struct ddb_dvb *dvb = &input->port->dvb[input->nr & 1];
+	struct device *dev = input->port->dev->dev;
 	struct dvb_frontend *fe;
-	struct device *dev = &input->port->dev->pdev->dev;
 
-	if (input->fe->ops.i2c_gate_ctrl)
-		input->fe->ops.i2c_gate_ctrl(input->fe, 1);
-	fe = dvb_attach(tda18271c2dd_attach, input->fe, i2c, 0x60);
+	if (dvb->fe->ops.i2c_gate_ctrl)
+		dvb->fe->ops.i2c_gate_ctrl(dvb->fe, 1);
+	fe = dvb_attach(tda18271c2dd_attach, dvb->fe, i2c, 0x60);
+	if (dvb->fe->ops.i2c_gate_ctrl)
+		dvb->fe->ops.i2c_gate_ctrl(dvb->fe, 0);
 	if (!fe) {
 		dev_err(dev, "No TDA18271 found!\n");
 		return -ENODEV;
 	}
-	if (input->fe->ops.i2c_gate_ctrl)
-		input->fe->ops.i2c_gate_ctrl(input->fe, 0);
 	return 0;
 }
 
@@ -687,43 +910,43 @@ static struct stv0367_config ddb_stv0367_config[] = {
 static int demod_attach_stv0367(struct ddb_input *input)
 {
 	struct i2c_adapter *i2c = &input->port->i2c->adap;
-	struct device *dev = &input->port->dev->pdev->dev;
+	struct ddb_dvb *dvb = &input->port->dvb[input->nr & 1];
+	struct device *dev = input->port->dev->dev;
+	struct dvb_frontend *fe;
 
 	/* attach frontend */
-	input->fe = dvb_attach(stv0367ddb_attach,
+	fe = dvb->fe = dvb_attach(stv0367ddb_attach,
 		&ddb_stv0367_config[(input->nr & 1)], i2c);
 
-	if (!input->fe) {
-		dev_err(dev, "stv0367ddb_attach failed (not found?)\n");
+	if (!dvb->fe) {
+		dev_err(dev, "No stv0367 found!\n");
 		return -ENODEV;
 	}
-
-	input->fe->sec_priv = input;
-	input->gate_ctrl = input->fe->ops.i2c_gate_ctrl;
-	input->fe->ops.i2c_gate_ctrl = drxk_gate_ctrl;
-
+	fe->sec_priv = input;
+	dvb->i2c_gate_ctrl = fe->ops.i2c_gate_ctrl;
+	fe->ops.i2c_gate_ctrl = locked_gate_ctrl;
 	return 0;
 }
 
 static int tuner_tda18212_ping(struct ddb_input *input, unsigned short adr)
 {
 	struct i2c_adapter *adapter = &input->port->i2c->adap;
-	struct device *dev = &input->port->dev->pdev->dev;
-
+	struct ddb_dvb *dvb = &input->port->dvb[input->nr & 1];
+	struct device *dev = input->port->dev->dev;
 	u8 tda_id[2];
 	u8 subaddr = 0x00;
 
 	dev_dbg(dev, "stv0367-tda18212 tuner ping\n");
-	if (input->fe->ops.i2c_gate_ctrl)
-		input->fe->ops.i2c_gate_ctrl(input->fe, 1);
+	if (dvb->fe->ops.i2c_gate_ctrl)
+		dvb->fe->ops.i2c_gate_ctrl(dvb->fe, 1);
 
 	if (i2c_read_regs(adapter, adr, subaddr, tda_id, sizeof(tda_id)) < 0)
 		dev_dbg(dev, "tda18212 ping 1 fail\n");
 	if (i2c_read_regs(adapter, adr, subaddr, tda_id, sizeof(tda_id)) < 0)
 		dev_warn(dev, "tda18212 ping failed, expect problems\n");
 
-	if (input->fe->ops.i2c_gate_ctrl)
-		input->fe->ops.i2c_gate_ctrl(input->fe, 0);
+	if (dvb->fe->ops.i2c_gate_ctrl)
+		dvb->fe->ops.i2c_gate_ctrl(dvb->fe, 0);
 
 	return 0;
 }
@@ -731,7 +954,9 @@ static int tuner_tda18212_ping(struct ddb_input *input, unsigned short adr)
 static int demod_attach_cxd28xx(struct ddb_input *input, int par, int osc24)
 {
 	struct i2c_adapter *i2c = &input->port->i2c->adap;
-	struct device *dev = &input->port->dev->pdev->dev;
+	struct ddb_dvb *dvb = &input->port->dvb[input->nr & 1];
+	struct device *dev = input->port->dev->dev;
+	struct dvb_frontend *fe;
 	struct cxd2841er_config cfg;
 
 	/* the cxd2841er driver expects 8bit/shifted I2C addresses */
@@ -746,27 +971,26 @@ static int demod_attach_cxd28xx(struct ddb_input *input, int par, int osc24)
 		cfg.flags |= CXD2841ER_TS_SERIAL;
 
 	/* attach frontend */
-	input->fe = dvb_attach(cxd2841er_attach_t_c, &cfg, i2c);
+	fe = dvb->fe = dvb_attach(cxd2841er_attach_t_c, &cfg, i2c);
 
-	if (!input->fe) {
-		dev_err(dev, "No Sony CXD28xx found!\n");
+	if (!dvb->fe) {
+		dev_err(dev, "No cxd2837/38/43/54 found!\n");
 		return -ENODEV;
 	}
-
-	input->fe->sec_priv = input;
-	input->gate_ctrl = input->fe->ops.i2c_gate_ctrl;
-	input->fe->ops.i2c_gate_ctrl = drxk_gate_ctrl;
-
+	fe->sec_priv = input;
+	dvb->i2c_gate_ctrl = fe->ops.i2c_gate_ctrl;
+	fe->ops.i2c_gate_ctrl = locked_gate_ctrl;
 	return 0;
 }
 
 static int tuner_attach_tda18212(struct ddb_input *input, u32 porttype)
 {
 	struct i2c_adapter *adapter = &input->port->i2c->adap;
-	struct device *dev = &input->port->dev->pdev->dev;
+	struct ddb_dvb *dvb = &input->port->dvb[input->nr & 1];
+	struct device *dev = input->port->dev->dev;
 	struct i2c_client *client;
 	struct tda18212_config config = {
-		.fe = input->fe,
+		.fe = dvb->fe,
 		.if_dvbt_6 = 3550,
 		.if_dvbt_7 = 3700,
 		.if_dvbt_8 = 4150,
@@ -804,17 +1028,17 @@ static int tuner_attach_tda18212(struct ddb_input *input, u32 porttype)
 		goto err;
 	}
 
-	input->i2c_client[0] = client;
+	dvb->i2c_client[0] = client;
 
 	return 0;
 err:
-	dev_warn(dev, "TDA18212 tuner not found. Device is not fully operational.\n");
+	dev_notice(dev, "TDA18212 tuner not found. Device is not fully operational.\n");
 	return -ENODEV;
 }
 
-/******************************************************************************/
-/******************************************************************************/
-/******************************************************************************/
+/****************************************************************************/
+/****************************************************************************/
+/****************************************************************************/
 
 static struct stv090x_config stv0900 = {
 	.device         = STV0900,
@@ -827,6 +1051,9 @@ static struct stv090x_config stv0900 = {
 	.ts1_mode       = STV090x_TSMODE_SERIAL_PUNCTURED,
 	.ts2_mode       = STV090x_TSMODE_SERIAL_PUNCTURED,
 
+	.ts1_tei        = 1,
+	.ts2_tei        = 1,
+
 	.repeater_level = STV090x_RPTLEVEL_16,
 
 	.adc1_range	= STV090x_ADC_1Vpp,
@@ -846,6 +1073,9 @@ static struct stv090x_config stv0900_aa = {
 	.ts1_mode       = STV090x_TSMODE_SERIAL_PUNCTURED,
 	.ts2_mode       = STV090x_TSMODE_SERIAL_PUNCTURED,
 
+	.ts1_tei        = 1,
+	.ts2_tei        = 1,
+
 	.repeater_level = STV090x_RPTLEVEL_16,
 
 	.adc1_range	= STV090x_ADC_1Vpp,
@@ -869,17 +1099,18 @@ static struct stv6110x_config stv6110b = {
 static int demod_attach_stv0900(struct ddb_input *input, int type)
 {
 	struct i2c_adapter *i2c = &input->port->i2c->adap;
-	struct device *dev = &input->port->dev->pdev->dev;
 	struct stv090x_config *feconf = type ? &stv0900_aa : &stv0900;
+	struct ddb_dvb *dvb = &input->port->dvb[input->nr & 1];
+	struct device *dev = input->port->dev->dev;
 
-	input->fe = dvb_attach(stv090x_attach, feconf, i2c,
-			       (input->nr & 1) ? STV090x_DEMODULATOR_1
-			       : STV090x_DEMODULATOR_0);
-	if (!input->fe) {
+	dvb->fe = dvb_attach(stv090x_attach, feconf, i2c,
+			     (input->nr & 1) ? STV090x_DEMODULATOR_1
+			     : STV090x_DEMODULATOR_0);
+	if (!dvb->fe) {
 		dev_err(dev, "No STV0900 found!\n");
 		return -ENODEV;
 	}
-	if (!dvb_attach(lnbh24_attach, input->fe, i2c, 0,
+	if (!dvb_attach(lnbh24_attach, dvb->fe, i2c, 0,
 			0, (input->nr & 1) ?
 			(0x09 - type) : (0x0b - type))) {
 		dev_err(dev, "No LNBH24 found!\n");
@@ -891,19 +1122,20 @@ static int demod_attach_stv0900(struct ddb_input *input, int type)
 static int tuner_attach_stv6110(struct ddb_input *input, int type)
 {
 	struct i2c_adapter *i2c = &input->port->i2c->adap;
-	struct device *dev = &input->port->dev->pdev->dev;
+	struct ddb_dvb *dvb = &input->port->dvb[input->nr & 1];
+	struct device *dev = input->port->dev->dev;
 	struct stv090x_config *feconf = type ? &stv0900_aa : &stv0900;
 	struct stv6110x_config *tunerconf = (input->nr & 1) ?
 		&stv6110b : &stv6110a;
 	const struct stv6110x_devctl *ctl;
 
-	ctl = dvb_attach(stv6110x_attach, input->fe, tunerconf, i2c);
+	ctl = dvb_attach(stv6110x_attach, dvb->fe, tunerconf, i2c);
 	if (!ctl) {
 		dev_err(dev, "No STV6110X found!\n");
 		return -ENODEV;
 	}
 	dev_info(dev, "attach tuner input %d adr %02x\n",
-			 input->nr, tunerconf->addr);
+		input->nr, tunerconf->addr);
 
 	feconf->tuner_init          = ctl->tuner_init;
 	feconf->tuner_sleep         = ctl->tuner_sleep;
@@ -920,388 +1152,1062 @@ static int tuner_attach_stv6110(struct ddb_input *input, int type)
 	return 0;
 }
 
-static int my_dvb_dmx_ts_card_init(struct dvb_demux *dvbdemux, char *id,
-			    int (*start_feed)(struct dvb_demux_feed *),
-			    int (*stop_feed)(struct dvb_demux_feed *),
-			    void *priv)
+static const struct stv0910_cfg stv0910_p = {
+	.adr      = 0x68,
+	.parallel = 1,
+	.rptlvl   = 4,
+	.clk      = 30000000,
+};
+
+static const struct lnbh25_config lnbh25_cfg = {
+	.i2c_address = 0x0c << 1,
+	.data2_config = LNBH25_TEN
+};
+
+static int demod_attach_stv0910(struct ddb_input *input, int type)
 {
-	dvbdemux->priv = priv;
+	struct i2c_adapter *i2c = &input->port->i2c->adap;
+	struct ddb_dvb *dvb = &input->port->dvb[input->nr & 1];
+	struct device *dev = input->port->dev->dev;
+	struct stv0910_cfg cfg = stv0910_p;
+	struct lnbh25_config lnbcfg = lnbh25_cfg;
+
+	if (stv0910_single)
+		cfg.single = 1;
+
+	if (type)
+		cfg.parallel = 2;
+	dvb->fe = dvb_attach(stv0910_attach, i2c, &cfg, (input->nr & 1));
+	if (!dvb->fe) {
+		cfg.adr = 0x6c;
+		dvb->fe = dvb_attach(stv0910_attach, i2c,
+				     &cfg, (input->nr & 1));
+	}
+	if (!dvb->fe) {
+		dev_err(dev, "No STV0910 found!\n");
+		return -ENODEV;
+	}
 
-	dvbdemux->filternum = 256;
-	dvbdemux->feednum = 256;
-	dvbdemux->start_feed = start_feed;
-	dvbdemux->stop_feed = stop_feed;
-	dvbdemux->write_to_decoder = NULL;
-	dvbdemux->dmx.capabilities = (DMX_TS_FILTERING |
-				      DMX_SECTION_FILTERING |
-				      DMX_MEMORY_BASED_FILTERING);
-	return dvb_dmx_init(dvbdemux);
+	/* attach lnbh25 - leftshift by one as the lnbh25 driver expects 8bit
+	 * i2c addresses
+	 */
+	lnbcfg.i2c_address = (((input->nr & 1) ? 0x0d : 0x0c) << 1);
+	if (!dvb_attach(lnbh25_attach, dvb->fe, &lnbcfg, i2c)) {
+		lnbcfg.i2c_address = (((input->nr & 1) ? 0x09 : 0x08) << 1);
+		if (!dvb_attach(lnbh25_attach, dvb->fe, &lnbcfg, i2c)) {
+			dev_err(dev, "No LNBH25 found!\n");
+			return -ENODEV;
+		}
+	}
+
+	return 0;
 }
 
-static int my_dvb_dmxdev_ts_card_init(struct dmxdev *dmxdev,
-			       struct dvb_demux *dvbdemux,
-			       struct dmx_frontend *hw_frontend,
-			       struct dmx_frontend *mem_frontend,
-			       struct dvb_adapter *dvb_adapter)
+static int tuner_attach_stv6111(struct ddb_input *input, int type)
 {
-	int ret;
-
-	dmxdev->filternum = 256;
-	dmxdev->demux = &dvbdemux->dmx;
-	dmxdev->capabilities = 0;
-	ret = dvb_dmxdev_init(dmxdev, dvb_adapter);
-	if (ret < 0)
-		return ret;
+	struct i2c_adapter *i2c = &input->port->i2c->adap;
+	struct ddb_dvb *dvb = &input->port->dvb[input->nr & 1];
+	struct device *dev = input->port->dev->dev;
+	struct dvb_frontend *fe;
+	u8 adr = (type ? 0 : 4) + ((input->nr & 1) ? 0x63 : 0x60);
 
-	hw_frontend->source = DMX_FRONTEND_0;
-	dvbdemux->dmx.add_frontend(&dvbdemux->dmx, hw_frontend);
-	mem_frontend->source = DMX_MEMORY_FE;
-	dvbdemux->dmx.add_frontend(&dvbdemux->dmx, mem_frontend);
-	return dvbdemux->dmx.connect_frontend(&dvbdemux->dmx, hw_frontend);
+	fe = dvb_attach(stv6111_attach, dvb->fe, i2c, adr);
+	if (!fe) {
+		fe = dvb_attach(stv6111_attach, dvb->fe, i2c, adr & ~4);
+		if (!fe) {
+			dev_err(dev, "No STV6111 found at 0x%02x!\n", adr);
+			return -ENODEV;
+		}
+	}
+	return 0;
 }
 
 static int start_feed(struct dvb_demux_feed *dvbdmxfeed)
 {
 	struct dvb_demux *dvbdmx = dvbdmxfeed->demux;
 	struct ddb_input *input = dvbdmx->priv;
+	struct ddb_dvb *dvb = &input->port->dvb[input->nr & 1];
 
-	if (!input->users)
-		ddb_input_start(input);
+	if (!dvb->users)
+		ddb_input_start_all(input);
 
-	return ++input->users;
+	return ++dvb->users;
 }
 
 static int stop_feed(struct dvb_demux_feed *dvbdmxfeed)
 {
 	struct dvb_demux *dvbdmx = dvbdmxfeed->demux;
 	struct ddb_input *input = dvbdmx->priv;
+	struct ddb_dvb *dvb = &input->port->dvb[input->nr & 1];
 
-	if (--input->users)
-		return input->users;
+	if (--dvb->users)
+		return dvb->users;
 
-	ddb_input_stop(input);
+	ddb_input_stop_all(input);
 	return 0;
 }
 
-
 static void dvb_input_detach(struct ddb_input *input)
 {
-	struct dvb_adapter *adap = &input->adap;
-	struct dvb_demux *dvbdemux = &input->demux;
+	struct ddb_dvb *dvb = &input->port->dvb[input->nr & 1];
+	struct dvb_demux *dvbdemux = &dvb->demux;
 	struct i2c_client *client;
 
-	switch (input->attached) {
-	case 5:
-		client = input->i2c_client[0];
+	switch (dvb->attached) {
+	case 0x31:
+		if (dvb->fe2)
+			dvb_unregister_frontend(dvb->fe2);
+		if (dvb->fe)
+			dvb_unregister_frontend(dvb->fe);
+		/* fallthrough */
+	case 0x30:
+		if (dvb->fe2)
+			dvb_frontend_detach(dvb->fe2);
+		if (dvb->fe)
+			dvb_frontend_detach(dvb->fe);
+		dvb->fe = dvb->fe2 = NULL;
+		/* fallthrough */
+	case 0x20:
+		client = dvb->i2c_client[0];
 		if (client) {
 			module_put(client->dev.driver->owner);
 			i2c_unregister_device(client);
 		}
-		if (input->fe2) {
-			dvb_unregister_frontend(input->fe2);
-			input->fe2 = NULL;
-		}
-		if (input->fe) {
-			dvb_unregister_frontend(input->fe);
-			dvb_frontend_detach(input->fe);
-			input->fe = NULL;
-		}
-		/* fall-through */
-	case 4:
-		dvb_net_release(&input->dvbnet);
-		/* fall-through */
-	case 3:
-		dvbdemux->dmx.close(&dvbdemux->dmx);
+
+		dvb_net_release(&dvb->dvbnet);
+		/* fallthrough */
+	case 0x12:
 		dvbdemux->dmx.remove_frontend(&dvbdemux->dmx,
-					      &input->hw_frontend);
+					      &dvb->hw_frontend);
 		dvbdemux->dmx.remove_frontend(&dvbdemux->dmx,
-					      &input->mem_frontend);
-		dvb_dmxdev_release(&input->dmxdev);
-		/* fall-through */
-	case 2:
-		dvb_dmx_release(&input->demux);
-		/* fall-through */
-	case 1:
-		dvb_unregister_adapter(adap);
+					      &dvb->mem_frontend);
+		/* fallthrough */
+	case 0x11:
+		dvb_dmxdev_release(&dvb->dmxdev);
+		/* fallthrough */
+	case 0x10:
+		dvb_dmx_release(&dvb->demux);
+		/* fallthrough */
+	case 0x01:
+		break;
+	}
+	dvb->attached = 0x00;
+}
+
+static int dvb_register_adapters(struct ddb *dev)
+{
+	int i, ret = 0;
+	struct ddb_port *port;
+	struct dvb_adapter *adap;
+
+	if (adapter_alloc == 3) {
+		port = &dev->port[0];
+		adap = port->dvb[0].adap;
+		ret = dvb_register_adapter(adap, "DDBridge", THIS_MODULE,
+					   port->dev->dev,
+					   adapter_nr);
+		if (ret < 0)
+			return ret;
+		port->dvb[0].adap_registered = 1;
+		for (i = 0; i < dev->port_num; i++) {
+			port = &dev->port[i];
+			port->dvb[0].adap = adap;
+			port->dvb[1].adap = adap;
+		}
+		return 0;
+	}
+
+	for (i = 0; i < dev->port_num; i++) {
+		port = &dev->port[i];
+		switch (port->class) {
+		case DDB_PORT_TUNER:
+			adap = port->dvb[0].adap;
+			ret = dvb_register_adapter(adap, "DDBridge",
+						   THIS_MODULE,
+						   port->dev->dev,
+						   adapter_nr);
+			if (ret < 0)
+				return ret;
+			port->dvb[0].adap_registered = 1;
+
+			if (adapter_alloc > 0) {
+				port->dvb[1].adap = port->dvb[0].adap;
+				break;
+			}
+			adap = port->dvb[1].adap;
+			ret = dvb_register_adapter(adap, "DDBridge",
+						   THIS_MODULE,
+						   port->dev->dev,
+						   adapter_nr);
+			if (ret < 0)
+				return ret;
+			port->dvb[1].adap_registered = 1;
+			break;
+
+		case DDB_PORT_CI:
+		case DDB_PORT_LOOP:
+			adap = port->dvb[0].adap;
+			ret = dvb_register_adapter(adap, "DDBridge",
+						   THIS_MODULE,
+						   port->dev->dev,
+						   adapter_nr);
+			if (ret < 0)
+				return ret;
+			port->dvb[0].adap_registered = 1;
+			break;
+		default:
+			if (adapter_alloc < 2)
+				break;
+			adap = port->dvb[0].adap;
+			ret = dvb_register_adapter(adap, "DDBridge",
+						   THIS_MODULE,
+						   port->dev->dev,
+						   adapter_nr);
+			if (ret < 0)
+				return ret;
+			port->dvb[0].adap_registered = 1;
+			break;
+		}
+	}
+	return ret;
+}
+
+static void dvb_unregister_adapters(struct ddb *dev)
+{
+	int i;
+	struct ddb_port *port;
+	struct ddb_dvb *dvb;
+
+	for (i = 0; i < dev->link[0].info->port_num; i++) {
+		port = &dev->port[i];
+
+		dvb = &port->dvb[0];
+		if (dvb->adap_registered)
+			dvb_unregister_adapter(dvb->adap);
+		dvb->adap_registered = 0;
+
+		dvb = &port->dvb[1];
+		if (dvb->adap_registered)
+			dvb_unregister_adapter(dvb->adap);
+		dvb->adap_registered = 0;
 	}
-	input->attached = 0;
 }
 
 static int dvb_input_attach(struct ddb_input *input)
 {
-	int ret;
+	int ret = 0;
+	struct ddb_dvb *dvb = &input->port->dvb[input->nr & 1];
 	struct ddb_port *port = input->port;
-	struct dvb_adapter *adap = &input->adap;
-	struct dvb_demux *dvbdemux = &input->demux;
-	struct device *dev = &input->port->dev->pdev->dev;
-	int sony_osc24 = 0, sony_tspar = 0;
-
-	ret = dvb_register_adapter(adap, "DDBridge", THIS_MODULE,
-				   &input->port->dev->pdev->dev,
-				   adapter_nr);
-	if (ret < 0) {
-		dev_err(dev, "Could not register adapter. Check if you enabled enough adapters in dvb-core!\n");
+	struct dvb_adapter *adap = dvb->adap;
+	struct dvb_demux *dvbdemux = &dvb->demux;
+	int par = 0, osc24 = 0;
+
+	dvb->attached = 0x01;
+
+	dvbdemux->priv = input;
+	dvbdemux->dmx.capabilities = DMX_TS_FILTERING |
+		DMX_SECTION_FILTERING | DMX_MEMORY_BASED_FILTERING;
+	dvbdemux->start_feed = start_feed;
+	dvbdemux->stop_feed = stop_feed;
+	dvbdemux->filternum = dvbdemux->feednum = 256;
+	ret = dvb_dmx_init(dvbdemux);
+	if (ret < 0)
 		return ret;
-	}
-	input->attached = 1;
+	dvb->attached = 0x10;
 
-	ret = my_dvb_dmx_ts_card_init(dvbdemux, "SW demux",
-				      start_feed,
-				      stop_feed, input);
+	dvb->dmxdev.filternum = 256;
+	dvb->dmxdev.demux = &dvbdemux->dmx;
+	ret = dvb_dmxdev_init(&dvb->dmxdev, adap);
 	if (ret < 0)
 		return ret;
-	input->attached = 2;
+	dvb->attached = 0x11;
 
-	ret = my_dvb_dmxdev_ts_card_init(&input->dmxdev, &input->demux,
-					 &input->hw_frontend,
-					 &input->mem_frontend, adap);
+	dvb->mem_frontend.source = DMX_MEMORY_FE;
+	dvb->demux.dmx.add_frontend(&dvb->demux.dmx, &dvb->mem_frontend);
+	dvb->hw_frontend.source = DMX_FRONTEND_0;
+	dvb->demux.dmx.add_frontend(&dvb->demux.dmx, &dvb->hw_frontend);
+	ret = dvbdemux->dmx.connect_frontend(&dvbdemux->dmx, &dvb->hw_frontend);
 	if (ret < 0)
 		return ret;
-	input->attached = 3;
+	dvb->attached = 0x12;
 
-	ret = dvb_net_init(adap, &input->dvbnet, input->dmxdev.demux);
+	ret = dvb_net_init(adap, &dvb->dvbnet, dvb->dmxdev.demux);
 	if (ret < 0)
 		return ret;
-	input->attached = 4;
+	dvb->attached = 0x20;
 
-	input->fe = NULL;
+	dvb->fe = dvb->fe2 = NULL;
 	switch (port->type) {
+	case DDB_TUNER_MXL5XX:
+		if (fe_attach_mxl5xx(input) < 0)
+			return -ENODEV;
+		break;
 	case DDB_TUNER_DVBS_ST:
 		if (demod_attach_stv0900(input, 0) < 0)
 			return -ENODEV;
 		if (tuner_attach_stv6110(input, 0) < 0)
 			return -ENODEV;
-		if (input->fe) {
-			if (dvb_register_frontend(adap, input->fe) < 0)
-				return -ENODEV;
-		}
 		break;
 	case DDB_TUNER_DVBS_ST_AA:
 		if (demod_attach_stv0900(input, 1) < 0)
 			return -ENODEV;
 		if (tuner_attach_stv6110(input, 1) < 0)
 			return -ENODEV;
-		if (input->fe) {
-			if (dvb_register_frontend(adap, input->fe) < 0)
-				return -ENODEV;
-		}
+		break;
+	case DDB_TUNER_DVBS_STV0910:
+		if (demod_attach_stv0910(input, 0) < 0)
+			return -ENODEV;
+		if (tuner_attach_stv6111(input, 0) < 0)
+			return -ENODEV;
+		break;
+	case DDB_TUNER_DVBS_STV0910_PR:
+		if (demod_attach_stv0910(input, 1) < 0)
+			return -ENODEV;
+		if (tuner_attach_stv6111(input, 1) < 0)
+			return -ENODEV;
+		break;
+	case DDB_TUNER_DVBS_STV0910_P:
+		if (demod_attach_stv0910(input, 0) < 0)
+			return -ENODEV;
+		if (tuner_attach_stv6111(input, 1) < 0)
+			return -ENODEV;
 		break;
 	case DDB_TUNER_DVBCT_TR:
 		if (demod_attach_drxk(input) < 0)
 			return -ENODEV;
 		if (tuner_attach_tda18271(input) < 0)
 			return -ENODEV;
-		if (dvb_register_frontend(adap, input->fe) < 0)
-			return -ENODEV;
-		if (input->fe2) {
-			if (dvb_register_frontend(adap, input->fe2) < 0)
-				return -ENODEV;
-			input->fe2->tuner_priv = input->fe->tuner_priv;
-			memcpy(&input->fe2->ops.tuner_ops,
-			       &input->fe->ops.tuner_ops,
-			       sizeof(struct dvb_tuner_ops));
-		}
 		break;
 	case DDB_TUNER_DVBCT_ST:
 		if (demod_attach_stv0367(input) < 0)
 			return -ENODEV;
-		if (tuner_attach_tda18212(input, port->type) < 0)
+		if (tuner_attach_tda18212(input, port->type) < 0) {
+			if (dvb->fe2)
+				dvb_frontend_detach(dvb->fe2);
+			if (dvb->fe)
+				dvb_frontend_detach(dvb->fe);
 			return -ENODEV;
-		if (input->fe) {
-			if (dvb_register_frontend(adap, input->fe) < 0)
-				return -ENODEV;
 		}
 		break;
 	case DDB_TUNER_DVBC2T2I_SONY_P:
+		if (input->port->dev->link[input->port->lnr].info->ts_quirks &
+		    TS_QUIRK_ALT_OSC)
+			osc24 = 0;
+		else
+			osc24 = 1;
+		/* fall-through */
 	case DDB_TUNER_DVBCT2_SONY_P:
 	case DDB_TUNER_DVBC2T2_SONY_P:
 	case DDB_TUNER_ISDBT_SONY_P:
-		if (port->type == DDB_TUNER_DVBC2T2I_SONY_P)
-			sony_osc24 = 1;
-		if (input->port->dev->info->ts_quirks & TS_QUIRK_ALT_OSC)
-			sony_osc24 = 0;
-		if (input->port->dev->info->ts_quirks & TS_QUIRK_SERIAL)
-			sony_tspar = 0;
+		if (input->port->dev->link[input->port->lnr].info->ts_quirks
+			& TS_QUIRK_SERIAL)
+			par = 0;
 		else
-			sony_tspar = 1;
-
-		if (demod_attach_cxd28xx(input, sony_tspar, sony_osc24) < 0)
+			par = 1;
+		if (demod_attach_cxd28xx(input, par, osc24) < 0)
 			return -ENODEV;
-		if (tuner_attach_tda18212(input, port->type) < 0)
+		if (tuner_attach_tda18212(input, port->type) < 0) {
+			if (dvb->fe2)
+				dvb_frontend_detach(dvb->fe2);
+			if (dvb->fe)
+				dvb_frontend_detach(dvb->fe);
 			return -ENODEV;
-		if (input->fe) {
-			if (dvb_register_frontend(adap, input->fe) < 0)
-				return -ENODEV;
 		}
 		break;
-	case DDB_TUNER_XO2_DVBC2T2I_SONY:
-	case DDB_TUNER_XO2_DVBCT2_SONY:
-	case DDB_TUNER_XO2_DVBC2T2_SONY:
-	case DDB_TUNER_XO2_ISDBT_SONY:
-		if (port->type == DDB_TUNER_XO2_DVBC2T2I_SONY)
-			sony_osc24 = 1;
-
-		if (demod_attach_cxd28xx(input, 0, sony_osc24) < 0)
+	case DDB_TUNER_DVBC2T2I_SONY:
+		osc24 = 1;
+		/* fall-through */
+	case DDB_TUNER_DVBCT2_SONY:
+	case DDB_TUNER_DVBC2T2_SONY:
+	case DDB_TUNER_ISDBT_SONY:
+		if (demod_attach_cxd28xx(input, 0, osc24) < 0)
 			return -ENODEV;
-		if (tuner_attach_tda18212(input, port->type) < 0)
+		if (tuner_attach_tda18212(input, port->type) < 0) {
+			if (dvb->fe2)
+				dvb_frontend_detach(dvb->fe2);
+			if (dvb->fe)
+				dvb_frontend_detach(dvb->fe);
 			return -ENODEV;
-		if (input->fe) {
-			if (dvb_register_frontend(adap, input->fe) < 0)
-				return -ENODEV;
 		}
 		break;
+	default:
+		return 0;
+	}
+	dvb->attached = 0x30;
+
+	if (dvb->fe) {
+		if (dvb_register_frontend(adap, dvb->fe) < 0)
+			return -ENODEV;
+
+		if (dvb->fe2) {
+			if (dvb_register_frontend(adap, dvb->fe2) < 0)
+				return -ENODEV;
+			dvb->fe2->tuner_priv = dvb->fe->tuner_priv;
+			memcpy(&dvb->fe2->ops.tuner_ops,
+			       &dvb->fe->ops.tuner_ops,
+			       sizeof(struct dvb_tuner_ops));
+		}
 	}
 
-	input->attached = 5;
+	dvb->attached = 0x31;
 	return 0;
 }
 
-/****************************************************************************/
-/****************************************************************************/
+static int port_has_encti(struct ddb_port *port)
+{
+	struct device *dev = port->dev->dev;
+	u8 val;
+	int ret = i2c_read_reg(&port->i2c->adap, 0x20, 0, &val);
 
-static ssize_t ts_write(struct file *file, const __user char *buf,
-			size_t count, loff_t *ppos)
+	if (!ret)
+		dev_info(dev, "[0x20]=0x%02x\n", val);
+	return ret ? 0 : 1;
+}
+
+static int port_has_cxd(struct ddb_port *port, u8 *type)
 {
-	struct dvb_device *dvbdev = file->private_data;
-	struct ddb_output *output = dvbdev->priv;
-	size_t left = count;
-	int stat;
+	u8 val;
+	u8 probe[4] = { 0xe0, 0x00, 0x00, 0x00 }, data[4];
+	struct i2c_msg msgs[2] = {{ .addr = 0x40,  .flags = 0,
+				    .buf  = probe, .len   = 4 },
+				  { .addr = 0x40,  .flags = I2C_M_RD,
+				    .buf  = data,  .len   = 4 } };
+	val = i2c_transfer(&port->i2c->adap, msgs, 2);
+	if (val != 2)
+		return 0;
 
-	while (left) {
-		if (ddb_output_free(output) < 188) {
-			if (file->f_flags & O_NONBLOCK)
-				break;
-			if (wait_event_interruptible(
-				    output->wq, ddb_output_free(output) >= 188) < 0)
-				break;
-		}
-		stat = ddb_output_write(output, buf, left);
-		if (stat < 0)
-			break;
-		buf += stat;
-		left -= stat;
+	if (data[0] == 0x02 && data[1] == 0x2b && data[3] == 0x43)
+		*type = 2;
+	else
+		*type = 1;
+	return 1;
+}
+
+static int port_has_xo2(struct ddb_port *port, u8 *type, u8 *id)
+{
+	u8 probe[1] = { 0x00 }, data[4];
+
+	if (i2c_io(&port->i2c->adap, 0x10, probe, 1, data, 4))
+		return 0;
+	if (data[0] == 'D' && data[1] == 'F') {
+		*id = data[2];
+		*type = 1;
+		return 1;
 	}
-	return (left == count) ? -EAGAIN : (count - left);
+	if (data[0] == 'C' && data[1] == 'I') {
+		*id = data[2];
+		*type = 2;
+		return 1;
+	}
+	return 0;
 }
 
-static ssize_t ts_read(struct file *file, __user char *buf,
-		       size_t count, loff_t *ppos)
+static int port_has_stv0900(struct ddb_port *port)
 {
-	struct dvb_device *dvbdev = file->private_data;
-	struct ddb_output *output = dvbdev->priv;
-	struct ddb_input *input = output->port->input[0];
-	int left, read;
+	u8 val;
 
-	count -= count % 188;
-	left = count;
-	while (left) {
-		if (ddb_input_avail(input) < 188) {
-			if (file->f_flags & O_NONBLOCK)
-				break;
-			if (wait_event_interruptible(
-				    input->wq, ddb_input_avail(input) >= 188) < 0)
-				break;
-		}
-		read = ddb_input_read(input, buf, left);
-		if (read < 0)
-			return read;
-		left -= read;
-		buf += read;
+	if (i2c_read_reg16(&port->i2c->adap, 0x69, 0xf100, &val) < 0)
+		return 0;
+	return 1;
+}
+
+static int port_has_stv0900_aa(struct ddb_port *port, u8 *id)
+{
+	if (i2c_read_reg16(&port->i2c->adap, 0x68, 0xf100, id) < 0)
+		return 0;
+	return 1;
+}
+
+static int port_has_drxks(struct ddb_port *port)
+{
+	u8 val;
+
+	if (i2c_read(&port->i2c->adap, 0x29, &val) < 0)
+		return 0;
+	if (i2c_read(&port->i2c->adap, 0x2a, &val) < 0)
+		return 0;
+	return 1;
+}
+
+static int port_has_stv0367(struct ddb_port *port)
+{
+	u8 val;
+
+	if (i2c_read_reg16(&port->i2c->adap, 0x1e, 0xf000, &val) < 0)
+		return 0;
+	if (val != 0x60)
+		return 0;
+	if (i2c_read_reg16(&port->i2c->adap, 0x1f, 0xf000, &val) < 0)
+		return 0;
+	if (val != 0x60)
+		return 0;
+	return 1;
+}
+
+static int init_xo2(struct ddb_port *port)
+{
+	struct i2c_adapter *i2c = &port->i2c->adap;
+	struct ddb *dev = port->dev;
+	u8 val, data[2];
+	int res;
+
+	res = i2c_read_regs(i2c, 0x10, 0x04, data, 2);
+	if (res < 0)
+		return res;
+
+	if (data[0] != 0x01)  {
+		dev_info(dev->dev, "Port %d: invalid XO2\n", port->nr);
+		return -1;
 	}
-	return (left == count) ? -EAGAIN : (count - left);
+
+	i2c_read_reg(i2c, 0x10, 0x08, &val);
+	if (val != 0) {
+		i2c_write_reg(i2c, 0x10, 0x08, 0x00);
+		msleep(100);
+	}
+	/* Enable tuner power, disable pll, reset demods */
+	i2c_write_reg(i2c, 0x10, 0x08, 0x04);
+	usleep_range(2000, 3000);
+	/* Release demod resets */
+	i2c_write_reg(i2c, 0x10, 0x08, 0x07);
+
+	/* speed: 0=55,1=75,2=90,3=104 MBit/s */
+	i2c_write_reg(i2c, 0x10, 0x09, xo2_speed);
+
+	if (dev->link[port->lnr].info->con_clock) {
+		dev_info(dev->dev, "Setting continuous clock for XO2\n");
+		i2c_write_reg(i2c, 0x10, 0x0a, 0x03);
+		i2c_write_reg(i2c, 0x10, 0x0b, 0x03);
+	} else {
+		i2c_write_reg(i2c, 0x10, 0x0a, 0x01);
+		i2c_write_reg(i2c, 0x10, 0x0b, 0x01);
+	}
+
+	usleep_range(2000, 3000);
+	/* Start XO2 PLL */
+	i2c_write_reg(i2c, 0x10, 0x08, 0x87);
+
+	return 0;
 }
 
-static unsigned int ts_poll(struct file *file, poll_table *wait)
+static int init_xo2_ci(struct ddb_port *port)
 {
-	/*
-	struct dvb_device *dvbdev = file->private_data;
-	struct ddb_output *output = dvbdev->priv;
-	struct ddb_input *input = output->port->input[0];
-	*/
-	unsigned int mask = 0;
+	struct i2c_adapter *i2c = &port->i2c->adap;
+	struct ddb *dev = port->dev;
+	u8 val, data[2];
+	int res;
 
-#if 0
-	if (data_avail_to_read)
-		mask |= POLLIN | POLLRDNORM;
-	if (data_avail_to_write)
-		mask |= POLLOUT | POLLWRNORM;
+	res = i2c_read_regs(i2c, 0x10, 0x04, data, 2);
+	if (res < 0)
+		return res;
 
-	poll_wait(file, &read_queue, wait);
-	poll_wait(file, &write_queue, wait);
-#endif
-	return mask;
+	if (data[0] > 1)  {
+		dev_info(dev->dev, "Port %d: invalid XO2 CI %02x\n",
+			port->nr, data[0]);
+		return -1;
+	}
+	dev_info(dev->dev, "Port %d: DuoFlex CI %u.%u\n",
+		port->nr, data[0], data[1]);
+
+	i2c_read_reg(i2c, 0x10, 0x08, &val);
+	if (val != 0) {
+		i2c_write_reg(i2c, 0x10, 0x08, 0x00);
+		msleep(100);
+	}
+	/* Enable both CI */
+	i2c_write_reg(i2c, 0x10, 0x08, 3);
+	usleep_range(2000, 3000);
+
+
+	/* speed: 0=55,1=75,2=90,3=104 MBit/s */
+	i2c_write_reg(i2c, 0x10, 0x09, 1);
+
+	i2c_write_reg(i2c, 0x10, 0x08, 0x83);
+	usleep_range(2000, 3000);
+
+	if (dev->link[port->lnr].info->con_clock) {
+		dev_info(dev->dev, "Setting continuous clock for DuoFlex CI\n");
+		i2c_write_reg(i2c, 0x10, 0x0a, 0x03);
+		i2c_write_reg(i2c, 0x10, 0x0b, 0x03);
+	} else {
+		i2c_write_reg(i2c, 0x10, 0x0a, 0x01);
+		i2c_write_reg(i2c, 0x10, 0x0b, 0x01);
+	}
+	return 0;
 }
 
-static const struct file_operations ci_fops = {
-	.owner   = THIS_MODULE,
-	.read    = ts_read,
-	.write   = ts_write,
-	.open    = dvb_generic_open,
-	.release = dvb_generic_release,
-	.poll    = ts_poll,
+static int port_has_cxd28xx(struct ddb_port *port, u8 *id)
+{
+	struct i2c_adapter *i2c = &port->i2c->adap;
+	int status;
+
+	status = i2c_write_reg(&port->i2c->adap, 0x6e, 0, 0);
+	if (status)
+		return 0;
+	status = i2c_read_reg(i2c, 0x6e, 0xfd, id);
+	if (status)
+		return 0;
+	return 1;
+}
+
+static char *xo2names[] = {
+	"DUAL DVB-S2", "DUAL DVB-C/T/T2",
+	"DUAL DVB-ISDBT", "DUAL DVB-C/C2/T/T2",
+	"DUAL ATSC", "DUAL DVB-C/C2/T/T2,ISDB-T",
+	"", ""
 };
 
-static struct dvb_device dvbdev_ci = {
-	.readers = -1,
-	.writers = -1,
-	.users   = -1,
-	.fops    = &ci_fops,
+static char *xo2types[] = {
+	"DVBS_ST", "DVBCT2_SONY",
+	"ISDBT_SONY", "DVBC2T2_SONY",
+	"ATSC_ST", "DVBC2T2I_SONY"
 };
 
+static void ddb_port_probe(struct ddb_port *port)
+{
+	struct ddb *dev = port->dev;
+	u32 l = port->lnr;
+	u8 id, type;
+
+	port->name = "NO MODULE";
+	port->type_name = "NONE";
+	port->class = DDB_PORT_NONE;
+
+	/* Handle missing ports and ports without I2C */
+
+	if (port->nr == ts_loop) {
+		port->name = "TS LOOP";
+		port->class = DDB_PORT_LOOP;
+		return;
+	}
+
+	if (port->nr == 1 && dev->link[l].info->type == DDB_OCTOPUS_CI &&
+	    dev->link[l].info->i2c_mask == 1) {
+		port->name = "NO TAB";
+		port->class = DDB_PORT_NONE;
+		return;
+	}
+
+	if (dev->link[l].info->type == DDB_OCTOPUS_MAX) {
+		port->name = "DUAL DVB-S2 MAX";
+		port->type_name = "MXL5XX";
+		port->class = DDB_PORT_TUNER;
+		port->type = DDB_TUNER_MXL5XX;
+		if (port->i2c)
+			ddbwritel(dev, I2C_SPEED_400,
+				  port->i2c->regs + I2C_TIMING);
+		return;
+	}
+
+	if (port->nr > 1 && dev->link[l].info->type == DDB_OCTOPUS_CI) {
+		port->name = "CI internal";
+		port->type_name = "INTERNAL";
+		port->class = DDB_PORT_CI;
+		port->type = DDB_CI_INTERNAL;
+	}
+
+	if (!port->i2c)
+		return;
+
+	/* Probe ports with I2C */
+
+	if (port_has_cxd(port, &id)) {
+		if (id == 1) {
+			port->name = "CI";
+			port->type_name = "CXD2099";
+			port->class = DDB_PORT_CI;
+			port->type = DDB_CI_EXTERNAL_SONY;
+			ddbwritel(dev, I2C_SPEED_400,
+				  port->i2c->regs + I2C_TIMING);
+		} else {
+			dev_info(dev->dev, "Port %d: Uninitialized DuoFlex\n",
+			       port->nr);
+			return;
+		}
+	} else if (port_has_xo2(port, &type, &id)) {
+		ddbwritel(dev, I2C_SPEED_400, port->i2c->regs + I2C_TIMING);
+		/*dev_info(dev->dev, "XO2 ID %02x\n", id);*/
+		if (type == 2) {
+			port->name = "DuoFlex CI";
+			port->class = DDB_PORT_CI;
+			port->type = DDB_CI_EXTERNAL_XO2;
+			port->type_name = "CI_XO2";
+			init_xo2_ci(port);
+			return;
+		}
+		id >>= 2;
+		if (id > 5) {
+			port->name = "unknown XO2 DuoFlex";
+			port->type_name = "UNKNOWN";
+		} else {
+			port->name = xo2names[id];
+			port->class = DDB_PORT_TUNER;
+			port->type = DDB_TUNER_XO2 + id;
+			port->type_name = xo2types[id];
+			init_xo2(port);
+		}
+	} else if (port_has_cxd28xx(port, &id)) {
+		switch (id) {
+		case 0xa4:
+			port->name = "DUAL DVB-C2T2 CXD2843";
+			port->type = DDB_TUNER_DVBC2T2_SONY_P;
+			port->type_name = "DVBC2T2_SONY";
+			break;
+		case 0xb1:
+			port->name = "DUAL DVB-CT2 CXD2837";
+			port->type = DDB_TUNER_DVBCT2_SONY_P;
+			port->type_name = "DVBCT2_SONY";
+			break;
+		case 0xb0:
+			port->name = "DUAL ISDB-T CXD2838";
+			port->type = DDB_TUNER_ISDBT_SONY_P;
+			port->type_name = "ISDBT_SONY";
+			break;
+		case 0xc1:
+			port->name = "DUAL DVB-C2T2 ISDB-T CXD2854";
+			port->type = DDB_TUNER_DVBC2T2I_SONY_P;
+			port->type_name = "DVBC2T2I_ISDBT_SONY";
+			break;
+		default:
+			return;
+		}
+		port->class = DDB_PORT_TUNER;
+		ddbwritel(dev, I2C_SPEED_400, port->i2c->regs + I2C_TIMING);
+	} else if (port_has_stv0900(port)) {
+		port->name = "DUAL DVB-S2";
+		port->class = DDB_PORT_TUNER;
+		port->type = DDB_TUNER_DVBS_ST;
+		port->type_name = "DVBS_ST";
+		ddbwritel(dev, I2C_SPEED_100, port->i2c->regs + I2C_TIMING);
+	} else if (port_has_stv0900_aa(port, &id)) {
+		port->name = "DUAL DVB-S2";
+		port->class = DDB_PORT_TUNER;
+		if (id == 0x51) {
+			if (port->nr == 0 &&
+			    dev->link[l].info->ts_quirks & TS_QUIRK_REVERSED)
+				port->type = DDB_TUNER_DVBS_STV0910_PR;
+			else
+				port->type = DDB_TUNER_DVBS_STV0910_P;
+			port->type_name = "DVBS_ST_0910";
+		} else {
+			port->type = DDB_TUNER_DVBS_ST_AA;
+			port->type_name = "DVBS_ST_AA";
+		}
+		ddbwritel(dev, I2C_SPEED_100, port->i2c->regs + I2C_TIMING);
+	} else if (port_has_drxks(port)) {
+		port->name = "DUAL DVB-C/T";
+		port->class = DDB_PORT_TUNER;
+		port->type = DDB_TUNER_DVBCT_TR;
+		port->type_name = "DVBCT_TR";
+		ddbwritel(dev, I2C_SPEED_400, port->i2c->regs + I2C_TIMING);
+	} else if (port_has_stv0367(port)) {
+		port->name = "DUAL DVB-C/T";
+		port->class = DDB_PORT_TUNER;
+		port->type = DDB_TUNER_DVBCT_ST;
+		port->type_name = "DVBCT_ST";
+		ddbwritel(dev, I2C_SPEED_100, port->i2c->regs + I2C_TIMING);
+	} else if (port_has_encti(port)) {
+		port->name = "ENCTI";
+		port->class = DDB_PORT_LOOP;
+	}
+}
+
+
 /****************************************************************************/
 /****************************************************************************/
 /****************************************************************************/
 
-static void input_tasklet(unsigned long data)
+static int wait_ci_ready(struct ddb_ci *ci)
 {
-	struct ddb_input *input = (struct ddb_input *) data;
-	struct ddb *dev = input->port->dev;
+	u32 count = 10;
+
+	ndelay(500);
+	do {
+		if (ddbreadl(ci->port->dev,
+			     CI_CONTROL(ci->nr)) & CI_READY)
+			break;
+		usleep_range(1, 2);
+		if ((--count) == 0)
+			return -1;
+	} while (1);
+	return 0;
+}
+
+static int read_attribute_mem(struct dvb_ca_en50221 *ca,
+			      int slot, int address)
+{
+	struct ddb_ci *ci = ca->data;
+	u32 val, off = (address >> 1) & (CI_BUFFER_SIZE - 1);
+
+	if (address > CI_BUFFER_SIZE)
+		return -1;
+	ddbwritel(ci->port->dev, CI_READ_CMD | (1 << 16) | address,
+		  CI_DO_READ_ATTRIBUTES(ci->nr));
+	wait_ci_ready(ci);
+	val = 0xff & ddbreadl(ci->port->dev, CI_BUFFER(ci->nr) + off);
+	return val;
+}
+
+static int write_attribute_mem(struct dvb_ca_en50221 *ca, int slot,
+			       int address, u8 value)
+{
+	struct ddb_ci *ci = ca->data;
+
+	ddbwritel(ci->port->dev, CI_WRITE_CMD | (value << 16) | address,
+		  CI_DO_ATTRIBUTE_RW(ci->nr));
+	wait_ci_ready(ci);
+	return 0;
+}
 
-	spin_lock(&input->lock);
-	if (!input->running) {
-		spin_unlock(&input->lock);
+static int read_cam_control(struct dvb_ca_en50221 *ca,
+			    int slot, u8 address)
+{
+	u32 count = 100;
+	struct ddb_ci *ci = ca->data;
+	u32 res;
+
+	ddbwritel(ci->port->dev, CI_READ_CMD | address,
+		  CI_DO_IO_RW(ci->nr));
+	ndelay(500);
+	do {
+		res = ddbreadl(ci->port->dev, CI_READDATA(ci->nr));
+		if (res & CI_READY)
+			break;
+		usleep_range(1, 2);
+		if ((--count) == 0)
+			return -1;
+	} while (1);
+	return 0xff & res;
+}
+
+static int write_cam_control(struct dvb_ca_en50221 *ca, int slot,
+			     u8 address, u8 value)
+{
+	struct ddb_ci *ci = ca->data;
+
+	ddbwritel(ci->port->dev, CI_WRITE_CMD | (value << 16) | address,
+		  CI_DO_IO_RW(ci->nr));
+	wait_ci_ready(ci);
+	return 0;
+}
+
+static int slot_reset(struct dvb_ca_en50221 *ca, int slot)
+{
+	struct ddb_ci *ci = ca->data;
+
+	ddbwritel(ci->port->dev, CI_POWER_ON,
+		  CI_CONTROL(ci->nr));
+	msleep(100);
+	ddbwritel(ci->port->dev, CI_POWER_ON | CI_RESET_CAM,
+		  CI_CONTROL(ci->nr));
+	ddbwritel(ci->port->dev, CI_ENABLE | CI_POWER_ON | CI_RESET_CAM,
+		  CI_CONTROL(ci->nr));
+	udelay(20);
+	ddbwritel(ci->port->dev, CI_ENABLE | CI_POWER_ON,
+		  CI_CONTROL(ci->nr));
+	return 0;
+}
+
+static int slot_shutdown(struct dvb_ca_en50221 *ca, int slot)
+{
+	struct ddb_ci *ci = ca->data;
+
+	ddbwritel(ci->port->dev, 0, CI_CONTROL(ci->nr));
+	msleep(300);
+	return 0;
+}
+
+static int slot_ts_enable(struct dvb_ca_en50221 *ca, int slot)
+{
+	struct ddb_ci *ci = ca->data;
+	u32 val = ddbreadl(ci->port->dev, CI_CONTROL(ci->nr));
+
+	ddbwritel(ci->port->dev, val | CI_BYPASS_DISABLE,
+		  CI_CONTROL(ci->nr));
+	return 0;
+}
+
+static int poll_slot_status(struct dvb_ca_en50221 *ca, int slot, int open)
+{
+	struct ddb_ci *ci = ca->data;
+	u32 val = ddbreadl(ci->port->dev, CI_CONTROL(ci->nr));
+	int stat = 0;
+
+	if (val & CI_CAM_DETECT)
+		stat |= DVB_CA_EN50221_POLL_CAM_PRESENT;
+	if (val & CI_CAM_READY)
+		stat |= DVB_CA_EN50221_POLL_CAM_READY;
+	return stat;
+}
+
+static struct dvb_ca_en50221 en_templ = {
+	.read_attribute_mem  = read_attribute_mem,
+	.write_attribute_mem = write_attribute_mem,
+	.read_cam_control    = read_cam_control,
+	.write_cam_control   = write_cam_control,
+	.slot_reset          = slot_reset,
+	.slot_shutdown       = slot_shutdown,
+	.slot_ts_enable      = slot_ts_enable,
+	.poll_slot_status    = poll_slot_status,
+};
+
+static void ci_attach(struct ddb_port *port)
+{
+	struct ddb_ci *ci = NULL;
+
+	ci = kzalloc(sizeof(*ci), GFP_KERNEL);
+	if (!ci)
 		return;
-	}
-	input->stat = ddbreadl(DMA_BUFFER_CURRENT(input->nr));
+	memcpy(&ci->en, &en_templ, sizeof(en_templ));
+	ci->en.data = ci;
+	port->en = &ci->en;
+	ci->port = port;
+	ci->nr = port->nr - 2;
+}
+
+/****************************************************************************/
+/****************************************************************************/
+/****************************************************************************/
 
-	if (input->port->class == DDB_PORT_TUNER) {
-		if (4&ddbreadl(DMA_BUFFER_CONTROL(input->nr)))
-			dev_err(&dev->pdev->dev, "Overflow input %d\n", input->nr);
-		while (input->cbuf != ((input->stat >> 11) & 0x1f)
-		       || (4 & safe_ddbreadl(dev, DMA_BUFFER_CONTROL(input->nr)))) {
-			dvb_dmx_swfilter_packets(&input->demux,
-						 input->vbuf[input->cbuf],
-						 input->dma_buf_size / 188);
+static int write_creg(struct ddb_ci *ci, u8 data, u8 mask)
+{
+	struct i2c_adapter *i2c = &ci->port->i2c->adap;
+	u8 adr = (ci->port->type == DDB_CI_EXTERNAL_XO2) ? 0x12 : 0x13;
 
-			input->cbuf = (input->cbuf + 1) % input->dma_buf_num;
-			ddbwritel((input->cbuf << 11),
-				  DMA_BUFFER_ACK(input->nr));
-			input->stat = ddbreadl(DMA_BUFFER_CURRENT(input->nr));
-		       }
-	}
-	if (input->port->class == DDB_PORT_CI)
-		wake_up(&input->wq);
-	spin_unlock(&input->lock);
+	ci->port->creg = (ci->port->creg & ~mask) | data;
+	return i2c_write_reg(i2c, adr, 0x02, ci->port->creg);
 }
 
-static void output_tasklet(unsigned long data)
+static int read_attribute_mem_xo2(struct dvb_ca_en50221 *ca,
+				  int slot, int address)
 {
-	struct ddb_output *output = (struct ddb_output *) data;
-	struct ddb *dev = output->port->dev;
+	struct ddb_ci *ci = ca->data;
+	struct i2c_adapter *i2c = &ci->port->i2c->adap;
+	u8 adr = (ci->port->type == DDB_CI_EXTERNAL_XO2) ? 0x12 : 0x13;
+	int res;
+	u8 val;
 
-	spin_lock(&output->lock);
-	if (!output->running) {
-		spin_unlock(&output->lock);
+	res = i2c_read_reg16(i2c, adr, 0x8000 | address, &val);
+	return res ? res : val;
+}
+
+static int write_attribute_mem_xo2(struct dvb_ca_en50221 *ca, int slot,
+				   int address, u8 value)
+{
+	struct ddb_ci *ci = ca->data;
+	struct i2c_adapter *i2c = &ci->port->i2c->adap;
+	u8 adr = (ci->port->type == DDB_CI_EXTERNAL_XO2) ? 0x12 : 0x13;
+
+	return i2c_write_reg16(i2c, adr, 0x8000 | address, value);
+}
+
+static int read_cam_control_xo2(struct dvb_ca_en50221 *ca,
+				int slot, u8 address)
+{
+	struct ddb_ci *ci = ca->data;
+	struct i2c_adapter *i2c = &ci->port->i2c->adap;
+	u8 adr = (ci->port->type == DDB_CI_EXTERNAL_XO2) ? 0x12 : 0x13;
+	u8 val;
+	int res;
+
+	res = i2c_read_reg(i2c, adr, 0x20 | (address & 3), &val);
+	return res ? res : val;
+}
+
+static int write_cam_control_xo2(struct dvb_ca_en50221 *ca, int slot,
+				 u8 address, u8 value)
+{
+	struct ddb_ci *ci = ca->data;
+	struct i2c_adapter *i2c = &ci->port->i2c->adap;
+	u8 adr = (ci->port->type == DDB_CI_EXTERNAL_XO2) ? 0x12 : 0x13;
+
+	return i2c_write_reg(i2c, adr, 0x20 | (address & 3), value);
+}
+
+static int slot_reset_xo2(struct dvb_ca_en50221 *ca, int slot)
+{
+	struct ddb_ci *ci = ca->data;
+
+	dev_dbg(ci->port->dev->dev, "%s\n", __func__);
+	write_creg(ci, 0x01, 0x01);
+	write_creg(ci, 0x04, 0x04);
+	msleep(20);
+	write_creg(ci, 0x02, 0x02);
+	write_creg(ci, 0x00, 0x04);
+	write_creg(ci, 0x18, 0x18);
+	return 0;
+}
+
+static int slot_shutdown_xo2(struct dvb_ca_en50221 *ca, int slot)
+{
+	struct ddb_ci *ci = ca->data;
+
+	dev_dbg(ci->port->dev->dev, "%s\n", __func__);
+	write_creg(ci, 0x10, 0xff);
+	write_creg(ci, 0x08, 0x08);
+	return 0;
+}
+
+static int slot_ts_enable_xo2(struct dvb_ca_en50221 *ca, int slot)
+{
+	struct ddb_ci *ci = ca->data;
+
+	dev_info(ci->port->dev->dev, "%s\n", __func__);
+	write_creg(ci, 0x00, 0x10);
+	return 0;
+}
+
+static int poll_slot_status_xo2(struct dvb_ca_en50221 *ca, int slot, int open)
+{
+	struct ddb_ci *ci = ca->data;
+	struct i2c_adapter *i2c = &ci->port->i2c->adap;
+	u8 adr = (ci->port->type == DDB_CI_EXTERNAL_XO2) ? 0x12 : 0x13;
+	u8 val = 0;
+	int stat = 0;
+
+	i2c_read_reg(i2c, adr, 0x01, &val);
+
+	if (val & 2)
+		stat |= DVB_CA_EN50221_POLL_CAM_PRESENT;
+	if (val & 1)
+		stat |= DVB_CA_EN50221_POLL_CAM_READY;
+	return stat;
+}
+
+static struct dvb_ca_en50221 en_xo2_templ = {
+	.read_attribute_mem  = read_attribute_mem_xo2,
+	.write_attribute_mem = write_attribute_mem_xo2,
+	.read_cam_control    = read_cam_control_xo2,
+	.write_cam_control   = write_cam_control_xo2,
+	.slot_reset          = slot_reset_xo2,
+	.slot_shutdown       = slot_shutdown_xo2,
+	.slot_ts_enable      = slot_ts_enable_xo2,
+	.poll_slot_status    = poll_slot_status_xo2,
+};
+
+static void ci_xo2_attach(struct ddb_port *port)
+{
+	struct ddb_ci *ci;
+
+	ci = kzalloc(sizeof(*ci), GFP_KERNEL);
+	if (!ci)
 		return;
-	}
-	output->stat = ddbreadl(DMA_BUFFER_CURRENT(output->nr + 8));
-	wake_up(&output->wq);
-	spin_unlock(&output->lock);
+	memcpy(&ci->en, &en_xo2_templ, sizeof(en_xo2_templ));
+	ci->en.data = ci;
+	port->en = &ci->en;
+	ci->port = port;
+	ci->nr = port->nr - 2;
+	ci->port->creg = 0;
+	write_creg(ci, 0x10, 0xff);
+	write_creg(ci, 0x08, 0x08);
 }
 
+/****************************************************************************/
+/****************************************************************************/
+/****************************************************************************/
 
 static struct cxd2099_cfg cxd_cfg = {
-	.bitrate =  62000,
+	.bitrate =  72000,
 	.adr     =  0x40,
 	.polarity = 1,
 	.clock_mode = 1,
@@ -1310,33 +2216,36 @@ static struct cxd2099_cfg cxd_cfg = {
 
 static int ddb_ci_attach(struct ddb_port *port)
 {
-	int ret;
+	switch (port->type) {
+	case DDB_CI_EXTERNAL_SONY:
+		cxd_cfg.bitrate = ci_bitrate;
+		port->en = cxd2099_attach(&cxd_cfg, port, &port->i2c->adap);
+		if (!port->en)
+			return -ENODEV;
+		dvb_ca_en50221_init(port->dvb[0].adap,
+				    port->en, 0, 1);
+		break;
 
-	ret = dvb_register_adapter(&port->output->adap,
-				   "DDBridge",
-				   THIS_MODULE,
-				   &port->dev->pdev->dev,
-				   adapter_nr);
-	if (ret < 0)
-		return ret;
-	port->en = cxd2099_attach(&cxd_cfg, port, &port->i2c->adap);
-	if (!port->en) {
-		dvb_unregister_adapter(&port->output->adap);
-		return -ENODEV;
+	case DDB_CI_EXTERNAL_XO2:
+	case DDB_CI_EXTERNAL_XO2_B:
+		ci_xo2_attach(port);
+		if (!port->en)
+			return -ENODEV;
+		dvb_ca_en50221_init(port->dvb[0].adap, port->en, 0, 1);
+		break;
+
+	case DDB_CI_INTERNAL:
+		ci_attach(port);
+		if (!port->en)
+			return -ENODEV;
+		dvb_ca_en50221_init(port->dvb[0].adap, port->en, 0, 1);
+		break;
 	}
-	ddb_input_start(port->input[0]);
-	ddb_output_start(port->output);
-	dvb_ca_en50221_init(&port->output->adap,
-			    port->en, 0, 1);
-	ret = dvb_register_device(&port->output->adap, &port->output->dev,
-				  &dvbdev_ci, (void *) port->output,
-				  DVB_DEVICE_SEC, 0);
-	return ret;
+	return 0;
 }
 
 static int ddb_port_attach(struct ddb_port *port)
 {
-	struct device *dev = &port->dev->pdev->dev;
 	int ret = 0;
 
 	switch (port->class) {
@@ -1345,371 +2254,429 @@ static int ddb_port_attach(struct ddb_port *port)
 		if (ret < 0)
 			break;
 		ret = dvb_input_attach(port->input[1]);
+		if (ret < 0)
+			break;
+		port->input[0]->redi = port->input[0];
+		port->input[1]->redi = port->input[1];
 		break;
 	case DDB_PORT_CI:
 		ret = ddb_ci_attach(port);
+		if (ret < 0)
+			break;
+		/* fall-through */
+	case DDB_PORT_LOOP:
+		ret = dvb_register_device(port->dvb[0].adap,
+					  &port->dvb[0].dev,
+					  &dvbdev_ci, (void *) port->output,
+					  DVB_DEVICE_SEC, 0);
 		break;
 	default:
 		break;
 	}
 	if (ret < 0)
-		dev_err(dev, "port_attach on port %d failed\n", port->nr);
+		dev_err(port->dev->dev, "port_attach on port %d failed\n",
+			port->nr);
 	return ret;
 }
 
-static int ddb_ports_attach(struct ddb *dev)
+int ddb_ports_attach(struct ddb *dev)
 {
 	int i, ret = 0;
 	struct ddb_port *port;
 
-	for (i = 0; i < dev->info->port_num; i++) {
+	if (dev->port_num) {
+		ret = dvb_register_adapters(dev);
+		if (ret < 0) {
+			dev_err(dev->dev, "Registering adapters failed. Check DVB_MAX_ADAPTERS in config.\n");
+			return ret;
+		}
+	}
+	for (i = 0; i < dev->port_num; i++) {
 		port = &dev->port[i];
 		ret = ddb_port_attach(port);
-		if (ret < 0)
-			break;
 	}
 	return ret;
 }
 
-static void ddb_ports_detach(struct ddb *dev)
+void ddb_ports_detach(struct ddb *dev)
 {
 	int i;
 	struct ddb_port *port;
 
-	for (i = 0; i < dev->info->port_num; i++) {
+	for (i = 0; i < dev->port_num; i++) {
 		port = &dev->port[i];
+
 		switch (port->class) {
 		case DDB_PORT_TUNER:
 			dvb_input_detach(port->input[0]);
 			dvb_input_detach(port->input[1]);
 			break;
 		case DDB_PORT_CI:
-			dvb_unregister_device(port->output->dev);
+		case DDB_PORT_LOOP:
+			if (port->dvb[0].dev)
+				dvb_unregister_device(port->dvb[0].dev);
 			if (port->en) {
-				ddb_input_stop(port->input[0]);
-				ddb_output_stop(port->output);
 				dvb_ca_en50221_release(port->en);
 				kfree(port->en);
 				port->en = NULL;
-				dvb_unregister_adapter(&port->output->adap);
 			}
 			break;
 		}
 	}
+	dvb_unregister_adapters(dev);
 }
 
-/****************************************************************************/
-/****************************************************************************/
 
-static int init_xo2(struct ddb_port *port)
+/* Copy input DMA pointers to output DMA and ACK. */
+
+static void input_write_output(struct ddb_input *input,
+			       struct ddb_output *output)
 {
-	struct i2c_adapter *i2c = &port->i2c->adap;
-	struct device *dev = &port->dev->pdev->dev;
-	u8 val, data[2];
-	int res;
+	ddbwritel(output->port->dev,
+		  input->dma->stat, DMA_BUFFER_ACK(output->dma));
+	output->dma->cbuf = (input->dma->stat >> 11) & 0x1f;
+	output->dma->coff = (input->dma->stat & 0x7ff) << 7;
+}
 
-	res = i2c_read_regs(i2c, 0x10, 0x04, data, 2);
-	if (res < 0)
-		return res;
+static void output_ack_input(struct ddb_output *output,
+			     struct ddb_input *input)
+{
+	ddbwritel(input->port->dev,
+		  output->dma->stat, DMA_BUFFER_ACK(input->dma));
+}
 
-	if (data[0] != 0x01)  {
-		dev_info(dev, "Port %d: invalid XO2\n", port->nr);
-		return -1;
+static void input_write_dvb(struct ddb_input *input,
+			    struct ddb_input *input2)
+{
+	struct ddb_dvb *dvb = &input2->port->dvb[input2->nr & 1];
+	struct ddb_dma *dma, *dma2;
+	struct ddb *dev = input->port->dev;
+	int ack = 1;
+
+	dma = dma2 = input->dma;
+	/* if there also is an output connected, do not ACK.
+	 * input_write_output will ACK.
+	 */
+	if (input->redo) {
+		dma2 = input->redo->dma;
+		ack = 0;
+	}
+	while (dma->cbuf != ((dma->stat >> 11) & 0x1f)
+	       || (4 & dma->ctrl)) {
+		if (4 & dma->ctrl) {
+			/* dev_err(dev->dev, "Overflow dma %d\n", dma->nr); */
+			ack = 1;
+		}
+		if (alt_dma)
+			dma_sync_single_for_cpu(dev->dev, dma2->pbuf[dma->cbuf],
+						dma2->size, DMA_FROM_DEVICE);
+		dvb_dmx_swfilter_packets(&dvb->demux,
+					 dma2->vbuf[dma->cbuf],
+					 dma2->size / 188);
+		dma->cbuf = (dma->cbuf + 1) % dma2->num;
+		if (ack)
+			ddbwritel(dev, (dma->cbuf << 11),
+				  DMA_BUFFER_ACK(dma));
+		dma->stat = safe_ddbreadl(dev, DMA_BUFFER_CURRENT(dma));
+		dma->ctrl = safe_ddbreadl(dev, DMA_BUFFER_CONTROL(dma));
 	}
+}
 
-	i2c_read_reg(i2c, 0x10, 0x08, &val);
-	if (val != 0) {
-		i2c_write_reg(i2c, 0x10, 0x08, 0x00);
-		msleep(100);
+static void input_work(struct work_struct *work)
+{
+	struct ddb_dma *dma = container_of(work, struct ddb_dma, work);
+	struct ddb_input *input = (struct ddb_input *) dma->io;
+	struct ddb *dev = input->port->dev;
+	unsigned long flags;
+
+	spin_lock_irqsave(&dma->lock, flags);
+	if (!dma->running) {
+		spin_unlock_irqrestore(&dma->lock, flags);
+		return;
 	}
-	/* Enable tuner power, disable pll, reset demods */
-	i2c_write_reg(i2c, 0x10, 0x08, 0x04);
-	usleep_range(2000, 3000);
-	/* Release demod resets */
-	i2c_write_reg(i2c, 0x10, 0x08, 0x07);
+	dma->stat = ddbreadl(dev, DMA_BUFFER_CURRENT(dma));
+	dma->ctrl = ddbreadl(dev, DMA_BUFFER_CONTROL(dma));
 
-	/* speed: 0=55,1=75,2=90,3=104 MBit/s */
-	i2c_write_reg(i2c, 0x10, 0x09,
-		((xo2_speed >= 0 && xo2_speed <= 3) ? xo2_speed : 2));
+	if (input->redi)
+		input_write_dvb(input, input->redi);
+	if (input->redo)
+		input_write_output(input, input->redo);
+	wake_up(&dma->wq);
+	spin_unlock_irqrestore(&dma->lock, flags);
+}
 
-	i2c_write_reg(i2c, 0x10, 0x0a, 0x01);
-	i2c_write_reg(i2c, 0x10, 0x0b, 0x01);
+static void input_handler(unsigned long data)
+{
+	struct ddb_input *input = (struct ddb_input *) data;
+	struct ddb_dma *dma = input->dma;
 
-	usleep_range(2000, 3000);
-	/* Start XO2 PLL */
-	i2c_write_reg(i2c, 0x10, 0x08, 0x87);
 
-	return 0;
+	/* If there is no input connected, input_tasklet() will
+	 * just copy pointers and ACK. So, there is no need to go
+	 * through the tasklet scheduler.
+	 */
+	if (input->redi)
+		queue_work(ddb_wq, &dma->work);
+	else
+		input_work(&dma->work);
 }
 
-static int port_has_xo2(struct ddb_port *port, u8 *type, u8 *id)
+static void output_handler(unsigned long data)
 {
-	u8 probe[1] = { 0x00 }, data[4];
-
-	*type = DDB_XO2_TYPE_NONE;
+	struct ddb_output *output = (struct ddb_output *) data;
+	struct ddb_dma *dma = output->dma;
+	struct ddb *dev = output->port->dev;
 
-	if (i2c_io(&port->i2c->adap, 0x10, probe, 1, data, 4))
-		return 0;
-	if (data[0] == 'D' && data[1] == 'F') {
-		*id = data[2];
-		*type = DDB_XO2_TYPE_DUOFLEX;
-		return 1;
-	}
-	if (data[0] == 'C' && data[1] == 'I') {
-		*id = data[2];
-		*type = DDB_XO2_TYPE_CI;
-		return 1;
+	spin_lock(&dma->lock);
+	if (!dma->running) {
+		spin_unlock(&dma->lock);
+		return;
 	}
-	return 0;
+	dma->stat = ddbreadl(dev, DMA_BUFFER_CURRENT(dma));
+	dma->ctrl = ddbreadl(dev, DMA_BUFFER_CONTROL(dma));
+	if (output->redi)
+		output_ack_input(output, output->redi);
+	wake_up(&dma->wq);
+	spin_unlock(&dma->lock);
 }
 
 /****************************************************************************/
 /****************************************************************************/
 
-static int port_has_ci(struct ddb_port *port)
+static const struct ddb_regmap *io_regmap(struct ddb_io *io, int link)
 {
-	u8 val;
-	return i2c_read_reg(&port->i2c->adap, 0x40, 0, &val) ? 0 : 1;
-}
+	const struct ddb_info *info;
 
-static int port_has_stv0900(struct ddb_port *port)
-{
-	u8 val;
-	if (i2c_read_reg16(&port->i2c->adap, 0x69, 0xf100, &val) < 0)
-		return 0;
-	return 1;
-}
+	if (link)
+		info = io->port->dev->link[io->port->lnr].info;
+	else
+		info = io->port->dev->link[0].info;
 
-static int port_has_stv0900_aa(struct ddb_port *port)
-{
-	u8 val;
-	if (i2c_read_reg16(&port->i2c->adap, 0x68, 0xf100, &val) < 0)
-		return 0;
-	return 1;
-}
+	if (!info)
+		return NULL;
 
-static int port_has_drxks(struct ddb_port *port)
-{
-	u8 val;
-	if (i2c_read(&port->i2c->adap, 0x29, &val) < 0)
-		return 0;
-	if (i2c_read(&port->i2c->adap, 0x2a, &val) < 0)
-		return 0;
-	return 1;
+	return info->regmap;
 }
 
-static int port_has_stv0367(struct ddb_port *port)
+static void ddb_dma_init(struct ddb_io *io, int nr, int out)
 {
-	u8 val;
-	if (i2c_read_reg16(&port->i2c->adap, 0x1e, 0xf000, &val) < 0)
-		return 0;
-	if (val != 0x60)
-		return 0;
-	if (i2c_read_reg16(&port->i2c->adap, 0x1f, 0xf000, &val) < 0)
-		return 0;
-	if (val != 0x60)
-		return 0;
-	return 1;
+	struct ddb_dma *dma;
+	const struct ddb_regmap *rm = io_regmap(io, 0);
+
+	dma = out ? &io->port->dev->odma[nr] : &io->port->dev->idma[nr];
+	io->dma = dma;
+	dma->io = io;
+
+	spin_lock_init(&dma->lock);
+	init_waitqueue_head(&dma->wq);
+	if (out) {
+		dma->regs = rm->odma->base + rm->odma->size * nr;
+		dma->bufregs = rm->odma_buf->base + rm->odma_buf->size * nr;
+		dma->num = OUTPUT_DMA_BUFS;
+		dma->size = OUTPUT_DMA_SIZE;
+		dma->div = OUTPUT_DMA_IRQ_DIV;
+	} else {
+		INIT_WORK(&dma->work, input_work);
+		dma->regs = rm->idma->base + rm->idma->size * nr;
+		dma->bufregs = rm->idma_buf->base + rm->idma_buf->size * nr;
+		dma->num = INPUT_DMA_BUFS;
+		dma->size = INPUT_DMA_SIZE;
+		dma->div = INPUT_DMA_IRQ_DIV;
+	}
+	ddbwritel(io->port->dev, 0, DMA_BUFFER_ACK(dma));
+	dev_dbg(io->port->dev->dev, "init link %u, io %u, dma %u, dmaregs %08x bufregs %08x\n",
+		io->port->lnr, io->nr, nr, dma->regs, dma->bufregs);
 }
 
-static int port_has_cxd28xx(struct ddb_port *port, u8 *id)
+static void ddb_input_init(struct ddb_port *port, int nr, int pnr, int anr)
 {
-	struct i2c_adapter *i2c = &port->i2c->adap;
-	int status;
+	struct ddb *dev = port->dev;
+	struct ddb_input *input = &dev->input[anr];
+	const struct ddb_regmap *rm;
 
-	status = i2c_write_reg(&port->i2c->adap, 0x6e, 0, 0);
-	if (status)
-		return 0;
-	status = i2c_read_reg(i2c, 0x6e, 0xfd, id);
-	if (status)
-		return 0;
-	return 1;
+	port->input[pnr] = input;
+	input->nr = nr;
+	input->port = port;
+	rm = io_regmap(input, 1);
+	input->regs = DDB_LINK_TAG(port->lnr) |
+		(rm->input->base + rm->input->size * nr);
+	dev_dbg(dev->dev, "init link %u, input %u, regs %08x\n",
+		port->lnr, nr, input->regs);
+
+	if (dev->has_dma) {
+		const struct ddb_regmap *rm0 = io_regmap(input, 0);
+		u32 base = rm0->irq_base_idma;
+		u32 dma_nr = nr;
+
+		if (port->lnr)
+			dma_nr += 32 + (port->lnr - 1) * 8;
+
+		dev_dbg(dev->dev, "init link %u, input %u, handler %u\n",
+			 port->lnr, nr, dma_nr + base);
+
+		dev->handler[0][dma_nr + base] = input_handler;
+		dev->handler_data[0][dma_nr + base] = (unsigned long) input;
+		ddb_dma_init(input, dma_nr, 0);
+	}
 }
 
-static void ddb_port_probe(struct ddb_port *port)
+static void ddb_output_init(struct ddb_port *port, int nr)
 {
 	struct ddb *dev = port->dev;
-	char *modname = "NO MODULE";
-	u8 xo2_type, xo2_id, cxd_id;
+	struct ddb_output *output = &dev->output[nr];
+	const struct ddb_regmap *rm;
 
-	port->class = DDB_PORT_NONE;
+	port->output = output;
+	output->nr = nr;
+	output->port = port;
+	rm = io_regmap(output, 1);
+	output->regs = DDB_LINK_TAG(port->lnr) |
+		(rm->output->base + rm->output->size * nr);
 
-	if (port_has_ci(port)) {
-		modname = "CI";
-		port->class = DDB_PORT_CI;
-		ddbwritel(I2C_SPEED_400, port->i2c->regs + I2C_TIMING);
-	} else if (port_has_xo2(port, &xo2_type, &xo2_id)) {
-		dev_dbg(&dev->pdev->dev, "Port %d (TAB %d): XO2 type: %d, id: %d\n",
-			port->nr, port->nr+1, xo2_type, xo2_id);
+	dev_dbg(dev->dev, "init link %u, output %u, regs %08x\n",
+		 port->lnr, nr, output->regs);
 
-		ddbwritel(I2C_SPEED_400, port->i2c->regs + I2C_TIMING);
+	if (dev->has_dma) {
+		const struct ddb_regmap *rm0 = io_regmap(output, 0);
+		u32 base = rm0->irq_base_odma;
 
-		switch (xo2_type) {
-		case DDB_XO2_TYPE_DUOFLEX:
-			init_xo2(port);
-			switch (xo2_id >> 2) {
-			case 0:
-				modname = "DUAL DVB-S2 (unsupported)";
-				port->class = DDB_PORT_NONE;
-				port->type = DDB_TUNER_XO2_DVBS_STV0910;
-				break;
-			case 1:
-				modname = "DUAL DVB-C/T/T2";
-				port->class = DDB_PORT_TUNER;
-				port->type = DDB_TUNER_XO2_DVBCT2_SONY;
-				break;
-			case 2:
-				modname = "DUAL DVB-ISDBT";
-				port->class = DDB_PORT_TUNER;
-				port->type = DDB_TUNER_XO2_ISDBT_SONY;
-				break;
-			case 3:
-				modname = "DUAL DVB-C/C2/T/T2";
-				port->class = DDB_PORT_TUNER;
-				port->type = DDB_TUNER_XO2_DVBC2T2_SONY;
-				break;
-			case 4:
-				modname = "DUAL ATSC (unsupported)";
-				port->class = DDB_PORT_NONE;
-				port->type = DDB_TUNER_XO2_ATSC_ST;
-				break;
-			case 5:
-				modname = "DUAL DVB-C/C2/T/T2/ISDBT";
-				port->class = DDB_PORT_TUNER;
-				port->type = DDB_TUNER_XO2_DVBC2T2I_SONY;
-				break;
-			default:
-				modname = "Unknown XO2 DuoFlex module\n";
-				break;
-			}
-			break;
-		case DDB_XO2_TYPE_CI:
-			dev_info(&dev->pdev->dev, "DuoFlex CI modules not supported\n");
-			break;
-		default:
-			dev_info(&dev->pdev->dev, "Unknown XO2 DuoFlex module\n");
-			break;
-		}
-	} else if (port_has_cxd28xx(port, &cxd_id)) {
-		switch (cxd_id) {
-		case 0xa4:
-			modname = "DUAL DVB-C2T2 CXD2843";
-			port->class = DDB_PORT_TUNER;
-			port->type = DDB_TUNER_DVBC2T2_SONY_P;
-			break;
-		case 0xb1:
-			modname = "DUAL DVB-CT2 CXD2837";
-			port->class = DDB_PORT_TUNER;
-			port->type = DDB_TUNER_DVBCT2_SONY_P;
-			break;
-		case 0xb0:
-			modname = "DUAL ISDB-T CXD2838";
-			port->class = DDB_PORT_TUNER;
-			port->type = DDB_TUNER_ISDBT_SONY_P;
-			break;
-		case 0xc1:
-			modname = "DUAL DVB-C2T2 ISDB-T CXD2854";
-			port->class = DDB_PORT_TUNER;
-			port->type = DDB_TUNER_DVBC2T2I_SONY_P;
-			break;
-		default:
-			modname = "Unknown CXD28xx tuner";
-			break;
-		}
-		ddbwritel(I2C_SPEED_400, port->i2c->regs + I2C_TIMING);
-	} else if (port_has_stv0900(port)) {
-		modname = "DUAL DVB-S2";
-		port->class = DDB_PORT_TUNER;
-		port->type = DDB_TUNER_DVBS_ST;
-		ddbwritel(I2C_SPEED_100, port->i2c->regs + I2C_TIMING);
-	} else if (port_has_stv0900_aa(port)) {
-		modname = "DUAL DVB-S2";
-		port->class = DDB_PORT_TUNER;
-		port->type = DDB_TUNER_DVBS_ST_AA;
-		ddbwritel(I2C_SPEED_100, port->i2c->regs + I2C_TIMING);
-	} else if (port_has_drxks(port)) {
-		modname = "DUAL DVB-C/T";
-		port->class = DDB_PORT_TUNER;
-		port->type = DDB_TUNER_DVBCT_TR;
-		ddbwritel(I2C_SPEED_400, port->i2c->regs + I2C_TIMING);
-	} else if (port_has_stv0367(port)) {
-		modname = "DUAL DVB-C/T";
-		port->class = DDB_PORT_TUNER;
-		port->type = DDB_TUNER_DVBCT_ST;
-		ddbwritel(I2C_SPEED_100, port->i2c->regs + I2C_TIMING);
+		dev->handler[0][nr + base] = output_handler;
+		dev->handler_data[0][nr + base] = (unsigned long) output;
+		ddb_dma_init(output, nr, 1);
 	}
-
-	dev_info(&dev->pdev->dev, "Port %d (TAB %d): %s\n",
-			 port->nr, port->nr+1, modname);
 }
 
-static void ddb_input_init(struct ddb_port *port, int nr)
+static int ddb_port_match_i2c(struct ddb_port *port)
 {
 	struct ddb *dev = port->dev;
-	struct ddb_input *input = &dev->input[nr];
+	u32 i;
 
-	input->nr = nr;
-	input->port = port;
-	input->dma_buf_num = INPUT_DMA_BUFS;
-	input->dma_buf_size = INPUT_DMA_SIZE;
-	ddbwritel(0, TS_INPUT_CONTROL(nr));
-	ddbwritel(2, TS_INPUT_CONTROL(nr));
-	ddbwritel(0, TS_INPUT_CONTROL(nr));
-	ddbwritel(0, DMA_BUFFER_ACK(nr));
-	tasklet_init(&input->tasklet, input_tasklet, (unsigned long) input);
-	spin_lock_init(&input->lock);
-	init_waitqueue_head(&input->wq);
+	for (i = 0; i < dev->i2c_num; i++) {
+		if (dev->i2c[i].link == port->lnr &&
+		    dev->i2c[i].nr == port->nr) {
+			port->i2c = &dev->i2c[i];
+			return 1;
+		}
+	}
+	return 0;
 }
 
-static void ddb_output_init(struct ddb_port *port, int nr)
+static int ddb_port_match_link_i2c(struct ddb_port *port)
 {
 	struct ddb *dev = port->dev;
-	struct ddb_output *output = &dev->output[nr];
-	output->nr = nr;
-	output->port = port;
-	output->dma_buf_num = OUTPUT_DMA_BUFS;
-	output->dma_buf_size = OUTPUT_DMA_SIZE;
+	u32 i;
 
-	ddbwritel(0, TS_OUTPUT_CONTROL(nr));
-	ddbwritel(2, TS_OUTPUT_CONTROL(nr));
-	ddbwritel(0, TS_OUTPUT_CONTROL(nr));
-	tasklet_init(&output->tasklet, output_tasklet, (unsigned long) output);
-	init_waitqueue_head(&output->wq);
+	for (i = 0; i < dev->i2c_num; i++) {
+		if (dev->i2c[i].link == port->lnr) {
+			port->i2c = &dev->i2c[i];
+			return 1;
+		}
+	}
+	return 0;
 }
 
-static void ddb_ports_init(struct ddb *dev)
+void ddb_ports_init(struct ddb *dev)
 {
-	int i;
+	u32 i, l, p;
 	struct ddb_port *port;
+	const struct ddb_info *info;
+	const struct ddb_regmap *rm;
+
+	for (p = l = 0; l < DDB_MAX_LINK; l++) {
+		info = dev->link[l].info;
+		if (!info)
+			continue;
+		rm = info->regmap;
+		if (!rm)
+			continue;
+		for (i = 0; i < info->port_num; i++, p++) {
+			port = &dev->port[p];
+			port->dev = dev;
+			port->nr = i;
+			port->lnr = l;
+			port->pnr = p;
+			port->gap = 0xffffffff;
+			port->obr = ci_bitrate;
+			mutex_init(&port->i2c_gate_lock);
+
+			if (!ddb_port_match_i2c(port)) {
+				if (info->type == DDB_OCTOPUS_MAX)
+					ddb_port_match_link_i2c(port);
+			}
 
-	for (i = 0; i < dev->info->port_num; i++) {
-		port = &dev->port[i];
-		port->dev = dev;
-		port->nr = i;
-		port->i2c = &dev->i2c[i];
-		port->input[0] = &dev->input[2 * i];
-		port->input[1] = &dev->input[2 * i + 1];
-		port->output = &dev->output[i];
+			ddb_port_probe(port);
+
+			port->dvb[0].adap = &dev->adap[2 * p];
+			port->dvb[1].adap = &dev->adap[2 * p + 1];
+
+			if ((port->class == DDB_PORT_NONE) && i && p &&
+			    dev->port[p - 1].type == DDB_CI_EXTERNAL_XO2) {
+				port->class = DDB_PORT_CI;
+				port->type = DDB_CI_EXTERNAL_XO2_B;
+				port->name = "DuoFlex CI_B";
+				port->i2c = dev->port[p - 1].i2c;
+			}
+
+			dev_info(dev->dev, "Port %u: Link %u, Link Port %u (TAB %u): %s\n",
+				port->pnr, port->lnr, port->nr, port->nr + 1,
+				port->name);
 
-		mutex_init(&port->i2c_gate_lock);
-		ddb_port_probe(port);
-		ddb_input_init(port, 2 * i);
-		ddb_input_init(port, 2 * i + 1);
-		ddb_output_init(port, i);
+			if (port->class == DDB_PORT_CI &&
+			    port->type == DDB_CI_EXTERNAL_XO2) {
+				ddb_input_init(port, 2 * i, 0, 2 * i);
+				ddb_output_init(port, i);
+				continue;
+			}
+
+			if (port->class == DDB_PORT_CI &&
+			    port->type == DDB_CI_EXTERNAL_XO2_B) {
+				ddb_input_init(port, 2 * i - 1, 0, 2 * i - 1);
+				ddb_output_init(port, i);
+				continue;
+			}
+
+			if (port->class == DDB_PORT_NONE)
+				continue;
+
+			switch (dev->link[l].info->type) {
+			case DDB_OCTOPUS_CI:
+				if (i >= 2) {
+					ddb_input_init(port, 2 + i, 0, 2 + i);
+					ddb_input_init(port, 4 + i, 1, 4 + i);
+					ddb_output_init(port, i);
+					break;
+				} /* fallthrough */
+			case DDB_OCTOPUS:
+				ddb_input_init(port, 2 * i, 0, 2 * i);
+				ddb_input_init(port, 2 * i + 1, 1, 2 * i + 1);
+				ddb_output_init(port, i);
+				break;
+			case DDB_OCTOPUS_MAX:
+			case DDB_OCTOPUS_MAX_CT:
+				ddb_input_init(port, 2 * i, 0, 2 * p);
+				ddb_input_init(port, 2 * i + 1, 1, 2 * p + 1);
+				break;
+			default:
+				break;
+			}
+		}
 	}
+	dev->port_num = p;
 }
 
-static void ddb_ports_release(struct ddb *dev)
+void ddb_ports_release(struct ddb *dev)
 {
 	int i;
 	struct ddb_port *port;
 
-	for (i = 0; i < dev->info->port_num; i++) {
+	for (i = 0; i < dev->port_num; i++) {
 		port = &dev->port[i];
-		port->dev = dev;
-		tasklet_kill(&port->input[0]->tasklet);
-		tasklet_kill(&port->input[1]->tasklet);
-		tasklet_kill(&port->output->tasklet);
+		if (port->input[0] && port->input[0]->dma)
+			cancel_work_sync(&port->input[0]->dma->work);
+		if (port->input[1] && port->input[1]->dma)
+			cancel_work_sync(&port->input[1]->dma->work);
+		if (port->output && port->output->dma)
+			cancel_work_sync(&port->output->dma->work);
 	}
 }
 
@@ -1717,90 +2684,158 @@ static void ddb_ports_release(struct ddb *dev)
 /****************************************************************************/
 /****************************************************************************/
 
-static void irq_handle_i2c(struct ddb *dev, int n)
+#define IRQ_HANDLE(_nr) \
+	do { if ((s & (1UL << ((_nr) & 0x1f))) && dev->handler[0][_nr]) \
+		dev->handler[0][_nr](dev->handler_data[0][_nr]); } \
+	while (0)
+
+static void irq_handle_msg(struct ddb *dev, u32 s)
 {
-	struct ddb_i2c *i2c = &dev->i2c[n];
+	dev->i2c_irq++;
+	IRQ_HANDLE(0);
+	IRQ_HANDLE(1);
+	IRQ_HANDLE(2);
+	IRQ_HANDLE(3);
+}
 
-	i2c->done = 1;
-	wake_up(&i2c->wq);
+static void irq_handle_io(struct ddb *dev, u32 s)
+{
+	dev->ts_irq++;
+	if ((s & 0x000000f0)) {
+		IRQ_HANDLE(4);
+		IRQ_HANDLE(5);
+		IRQ_HANDLE(6);
+		IRQ_HANDLE(7);
+	}
+	if ((s & 0x0000ff00)) {
+		IRQ_HANDLE(8);
+		IRQ_HANDLE(9);
+		IRQ_HANDLE(10);
+		IRQ_HANDLE(11);
+		IRQ_HANDLE(12);
+		IRQ_HANDLE(13);
+		IRQ_HANDLE(14);
+		IRQ_HANDLE(15);
+	}
+	if ((s & 0x00ff0000)) {
+		IRQ_HANDLE(16);
+		IRQ_HANDLE(17);
+		IRQ_HANDLE(18);
+		IRQ_HANDLE(19);
+		IRQ_HANDLE(20);
+		IRQ_HANDLE(21);
+		IRQ_HANDLE(22);
+		IRQ_HANDLE(23);
+	}
+	if ((s & 0xff000000)) {
+		IRQ_HANDLE(24);
+		IRQ_HANDLE(25);
+		IRQ_HANDLE(26);
+		IRQ_HANDLE(27);
+		IRQ_HANDLE(28);
+		IRQ_HANDLE(29);
+		IRQ_HANDLE(30);
+		IRQ_HANDLE(31);
+	}
 }
 
-static irqreturn_t irq_handler(int irq, void *dev_id)
+irqreturn_t ddb_irq_handler0(int irq, void *dev_id)
 {
 	struct ddb *dev = (struct ddb *) dev_id;
-	u32 s = ddbreadl(INTERRUPT_STATUS);
+	u32 s = ddbreadl(dev, INTERRUPT_STATUS);
 
-	if (!s)
-		return IRQ_NONE;
+	do {
+		if (s & 0x80000000)
+			return IRQ_NONE;
+		if (!(s & 0xfffff00))
+			return IRQ_NONE;
+		ddbwritel(dev, s & 0xfffff00, INTERRUPT_ACK);
+		irq_handle_io(dev, s);
+	} while ((s = ddbreadl(dev, INTERRUPT_STATUS)));
+
+	return IRQ_HANDLED;
+}
+
+irqreturn_t ddb_irq_handler1(int irq, void *dev_id)
+{
+	struct ddb *dev = (struct ddb *) dev_id;
+	u32 s = ddbreadl(dev, INTERRUPT_STATUS);
 
 	do {
-		ddbwritel(s, INTERRUPT_ACK);
-
-		if (s & 0x00000001)
-			irq_handle_i2c(dev, 0);
-		if (s & 0x00000002)
-			irq_handle_i2c(dev, 1);
-		if (s & 0x00000004)
-			irq_handle_i2c(dev, 2);
-		if (s & 0x00000008)
-			irq_handle_i2c(dev, 3);
-
-		if (s & 0x00000100)
-			tasklet_schedule(&dev->input[0].tasklet);
-		if (s & 0x00000200)
-			tasklet_schedule(&dev->input[1].tasklet);
-		if (s & 0x00000400)
-			tasklet_schedule(&dev->input[2].tasklet);
-		if (s & 0x00000800)
-			tasklet_schedule(&dev->input[3].tasklet);
-		if (s & 0x00001000)
-			tasklet_schedule(&dev->input[4].tasklet);
-		if (s & 0x00002000)
-			tasklet_schedule(&dev->input[5].tasklet);
-		if (s & 0x00004000)
-			tasklet_schedule(&dev->input[6].tasklet);
-		if (s & 0x00008000)
-			tasklet_schedule(&dev->input[7].tasklet);
-
-		if (s & 0x00010000)
-			tasklet_schedule(&dev->output[0].tasklet);
-		if (s & 0x00020000)
-			tasklet_schedule(&dev->output[1].tasklet);
-		if (s & 0x00040000)
-			tasklet_schedule(&dev->output[2].tasklet);
-		if (s & 0x00080000)
-			tasklet_schedule(&dev->output[3].tasklet);
-
-		/* if (s & 0x000f0000)	printk(KERN_DEBUG "%08x\n", istat); */
-	} while ((s = ddbreadl(INTERRUPT_STATUS)));
+		if (s & 0x80000000)
+			return IRQ_NONE;
+		if (!(s & 0x0000f))
+			return IRQ_NONE;
+		ddbwritel(dev, s & 0x0000f, INTERRUPT_ACK);
+		irq_handle_msg(dev, s);
+	} while ((s = ddbreadl(dev, INTERRUPT_STATUS)));
 
 	return IRQ_HANDLED;
 }
 
-/******************************************************************************/
-/******************************************************************************/
-/******************************************************************************/
+irqreturn_t ddb_irq_handler(int irq, void *dev_id)
+{
+	struct ddb *dev = (struct ddb *) dev_id;
+	u32 s = ddbreadl(dev, INTERRUPT_STATUS);
+	int ret = IRQ_HANDLED;
+
+	if (!s)
+		return IRQ_NONE;
+	do {
+		if (s & 0x80000000)
+			return IRQ_NONE;
+		ddbwritel(dev, s, INTERRUPT_ACK);
+
+		if (s & 0x0000000f)
+			irq_handle_msg(dev, s);
+		if (s & 0x0fffff00)
+			irq_handle_io(dev, s);
+	} while ((s = ddbreadl(dev, INTERRUPT_STATUS)));
+
+	return ret;
+}
+
+/****************************************************************************/
+/****************************************************************************/
+/****************************************************************************/
+
+static int reg_wait(struct ddb *dev, u32 reg, u32 bit)
+{
+	u32 count = 0;
+
+	while (safe_ddbreadl(dev, reg) & bit) {
+		ndelay(10);
+		if (++count == 100)
+			return -1;
+	}
+	return 0;
+}
 
-static int flashio(struct ddb *dev, u8 *wbuf, u32 wlen, u8 *rbuf, u32 rlen)
+static int flashio(struct ddb *dev, u32 lnr, u8 *wbuf, u32 wlen, u8 *rbuf,
+	u32 rlen)
 {
 	u32 data, shift;
+	u32 tag = DDB_LINK_TAG(lnr);
+	struct ddb_link *link = &dev->link[lnr];
 
+	mutex_lock(&link->flash_mutex);
 	if (wlen > 4)
-		ddbwritel(1, SPI_CONTROL);
+		ddbwritel(dev, 1, tag | SPI_CONTROL);
 	while (wlen > 4) {
 		/* FIXME: check for big-endian */
-		data = swab32(*(u32 *)wbuf);
+		data = swab32(*(u32 *) wbuf);
 		wbuf += 4;
 		wlen -= 4;
-		ddbwritel(data, SPI_DATA);
-		while (safe_ddbreadl(dev, SPI_CONTROL) & 0x0004)
-			;
+		ddbwritel(dev, data, tag | SPI_DATA);
+		if (reg_wait(dev, tag | SPI_CONTROL, 4))
+			goto fail;
 	}
-
 	if (rlen)
-		ddbwritel(0x0001 | ((wlen << (8 + 3)) & 0x1f00), SPI_CONTROL);
+		ddbwritel(dev, 0x0001 | ((wlen << (8 + 3)) & 0x1f00),
+			  tag | SPI_CONTROL);
 	else
-		ddbwritel(0x0003 | ((wlen << (8 + 3)) & 0x1f00), SPI_CONTROL);
+		ddbwritel(dev, 0x0003 | ((wlen << (8 + 3)) & 0x1f00),
+			  tag | SPI_CONTROL);
 
 	data = 0;
 	shift = ((4 - wlen) * 8);
@@ -1812,33 +2847,34 @@ static int flashio(struct ddb *dev, u8 *wbuf, u32 wlen, u8 *rbuf, u32 rlen)
 	}
 	if (shift)
 		data <<= shift;
-	ddbwritel(data, SPI_DATA);
-	while (safe_ddbreadl(dev, SPI_CONTROL) & 0x0004)
-		;
+	ddbwritel(dev, data, tag | SPI_DATA);
+	if (reg_wait(dev, tag | SPI_CONTROL, 4))
+		goto fail;
 
 	if (!rlen) {
-		ddbwritel(0, SPI_CONTROL);
-		return 0;
+		ddbwritel(dev, 0, tag | SPI_CONTROL);
+		goto exit;
 	}
 	if (rlen > 4)
-		ddbwritel(1, SPI_CONTROL);
+		ddbwritel(dev, 1, tag | SPI_CONTROL);
 
 	while (rlen > 4) {
-		ddbwritel(0xffffffff, SPI_DATA);
-		while (safe_ddbreadl(dev, SPI_CONTROL) & 0x0004)
-			;
-		data = ddbreadl(SPI_DATA);
+		ddbwritel(dev, 0xffffffff, tag | SPI_DATA);
+		if (reg_wait(dev, tag | SPI_CONTROL, 4))
+			goto fail;
+		data = ddbreadl(dev, tag | SPI_DATA);
 		*(u32 *) rbuf = swab32(data);
 		rbuf += 4;
 		rlen -= 4;
 	}
-	ddbwritel(0x0003 | ((rlen << (8 + 3)) & 0x1F00), SPI_CONTROL);
-	ddbwritel(0xffffffff, SPI_DATA);
-	while (safe_ddbreadl(dev, SPI_CONTROL) & 0x0004)
-		;
+	ddbwritel(dev, 0x0003 | ((rlen << (8 + 3)) & 0x1F00),
+		tag | SPI_CONTROL);
+	ddbwritel(dev, 0xffffffff, tag | SPI_DATA);
+	if (reg_wait(dev, tag | SPI_CONTROL, 4))
+		goto fail;
 
-	data = ddbreadl(SPI_DATA);
-	ddbwritel(0, SPI_CONTROL);
+	data = ddbreadl(dev, tag | SPI_DATA);
+	ddbwritel(dev, 0, tag | SPI_CONTROL);
 
 	if (rlen < 4)
 		data <<= ((4 - rlen) * 8);
@@ -1849,31 +2885,47 @@ static int flashio(struct ddb *dev, u8 *wbuf, u32 wlen, u8 *rbuf, u32 rlen)
 		rbuf++;
 		rlen--;
 	}
+exit:
+	mutex_unlock(&link->flash_mutex);
 	return 0;
+fail:
+	mutex_unlock(&link->flash_mutex);
+	return -1;
 }
 
-#define DDB_MAGIC 'd'
+int ddbridge_flashread(struct ddb *dev, u32 link, u8 *buf, u32 addr, u32 len)
+{
+	u8 cmd[4] = {0x03, (addr >> 16) & 0xff,
+		     (addr >> 8) & 0xff, addr & 0xff};
 
-struct ddb_flashio {
-	__user __u8 *write_buf;
-	__u32 write_len;
-	__user __u8 *read_buf;
-	__u32 read_len;
-};
+	return flashio(dev, link, cmd, 4, buf, len);
+}
 
-#define IOCTL_DDB_FLASHIO  _IOWR(DDB_MAGIC, 0x00, struct ddb_flashio)
+/*
+ * TODO/FIXME: add/implement IOCTLs from upstream driver
+ */
 
 #define DDB_NAME "ddbridge"
 
 static u32 ddb_num;
-static struct ddb *ddbs[32];
-static struct class *ddb_class;
 static int ddb_major;
+static DEFINE_MUTEX(ddb_mutex);
+
+static int ddb_release(struct inode *inode, struct file *file)
+{
+	struct ddb *dev = file->private_data;
+
+	dev->ddb_dev_users--;
+	return 0;
+}
 
 static int ddb_open(struct inode *inode, struct file *file)
 {
 	struct ddb *dev = ddbs[iminor(inode)];
 
+	if (dev->ddb_dev_users)
+		return -EBUSY;
+	dev->ddb_dev_users++;
 	file->private_data = dev;
 	return 0;
 }
@@ -1881,44 +2933,17 @@ static int ddb_open(struct inode *inode, struct file *file)
 static long ddb_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 {
 	struct ddb *dev = file->private_data;
-	__user void *parg = (__user void *)arg;
-	int res;
 
-	switch (cmd) {
-	case IOCTL_DDB_FLASHIO:
-	{
-		struct ddb_flashio fio;
-		u8 *rbuf, *wbuf;
+	dev_warn(dev->dev, "DDB IOCTLs unsupported (cmd: %d, arg: %lu)\n",
+		 cmd, arg);
 
-		if (copy_from_user(&fio, parg, sizeof(fio)))
-			return -EFAULT;
-
-		if (fio.write_len > 1028 || fio.read_len > 1028)
-			return -EINVAL;
-		if (fio.write_len + fio.read_len > 1028)
-			return -EINVAL;
-
-		wbuf = &dev->iobuf[0];
-		rbuf = wbuf + fio.write_len;
-
-		if (copy_from_user(wbuf, fio.write_buf, fio.write_len))
-			return -EFAULT;
-		res = flashio(dev, wbuf, fio.write_len, rbuf, fio.read_len);
-		if (res)
-			return res;
-		if (copy_to_user(fio.read_buf, rbuf, fio.read_len))
-			return -EFAULT;
-		break;
-	}
-	default:
-		return -ENOTTY;
-	}
-	return 0;
+	return -ENOTTY;
 }
 
 static const struct file_operations ddb_fops = {
 	.unlocked_ioctl = ddb_ioctl,
 	.open           = ddb_open,
+	.release        = ddb_release,
 };
 
 static char *ddb_devnode(struct device *device, umode_t *mode)
@@ -1928,369 +2953,684 @@ static char *ddb_devnode(struct device *device, umode_t *mode)
 	return kasprintf(GFP_KERNEL, "ddbridge/card%d", dev->nr);
 }
 
-static int ddb_class_create(void)
+#define __ATTR_MRO(_name, _show) {				\
+	.attr	= { .name = __stringify(_name), .mode = 0444 },	\
+	.show	= _show,					\
+}
+
+#define __ATTR_MWO(_name, _store) {				\
+	.attr	= { .name = __stringify(_name), .mode = 0222 },	\
+	.store	= _store,					\
+}
+
+static ssize_t ports_show(struct device *device,
+			  struct device_attribute *attr, char *buf)
 {
-	ddb_major = register_chrdev(0, DDB_NAME, &ddb_fops);
-	if (ddb_major < 0)
-		return ddb_major;
+	struct ddb *dev = dev_get_drvdata(device);
 
-	ddb_class = class_create(THIS_MODULE, DDB_NAME);
-	if (IS_ERR(ddb_class)) {
-		unregister_chrdev(ddb_major, DDB_NAME);
-		return PTR_ERR(ddb_class);
-	}
-	ddb_class->devnode = ddb_devnode;
-	return 0;
+	return sprintf(buf, "%d\n", dev->port_num);
 }
 
-static void ddb_class_destroy(void)
+static ssize_t ts_irq_show(struct device *device,
+			   struct device_attribute *attr, char *buf)
 {
-	class_destroy(ddb_class);
-	unregister_chrdev(ddb_major, DDB_NAME);
+	struct ddb *dev = dev_get_drvdata(device);
+
+	return sprintf(buf, "%d\n", dev->ts_irq);
 }
 
-static int ddb_device_create(struct ddb *dev)
+static ssize_t i2c_irq_show(struct device *device,
+			    struct device_attribute *attr, char *buf)
 {
-	dev->nr = ddb_num++;
-	dev->ddb_dev = device_create(ddb_class, NULL,
-				     MKDEV(ddb_major, dev->nr),
-				     dev, "ddbridge%d", dev->nr);
-	ddbs[dev->nr] = dev;
-	if (IS_ERR(dev->ddb_dev))
-		return -1;
-	return 0;
+	struct ddb *dev = dev_get_drvdata(device);
+
+	return sprintf(buf, "%d\n", dev->i2c_irq);
 }
 
-static void ddb_device_destroy(struct ddb *dev)
+static ssize_t fan_show(struct device *device,
+			struct device_attribute *attr, char *buf)
 {
-	ddb_num--;
-	if (IS_ERR(dev->ddb_dev))
-		return;
-	device_destroy(ddb_class, MKDEV(ddb_major, 0));
+	struct ddb *dev = dev_get_drvdata(device);
+	u32 val;
+
+	val = ddbreadl(dev, GPIO_OUTPUT) & 1;
+	return sprintf(buf, "%d\n", val);
 }
 
+static ssize_t fan_store(struct device *device, struct device_attribute *d,
+			 const char *buf, size_t count)
+{
+	struct ddb *dev = dev_get_drvdata(device);
+	u32 val;
 
-/****************************************************************************/
-/****************************************************************************/
-/****************************************************************************/
+	if (sscanf(buf, "%u\n", &val) != 1)
+		return -EINVAL;
+	ddbwritel(dev, 1, GPIO_DIRECTION);
+	ddbwritel(dev, val & 1, GPIO_OUTPUT);
+	return count;
+}
 
-static void ddb_unmap(struct ddb *dev)
+static ssize_t fanspeed_show(struct device *device,
+			struct device_attribute *attr, char *buf)
 {
-	if (dev->regs)
-		iounmap(dev->regs);
-	vfree(dev);
+	struct ddb *dev = dev_get_drvdata(device);
+	int num = attr->attr.name[8] - 0x30;
+	struct ddb_link *link = &dev->link[num];
+	u32 spd;
+
+	spd = ddblreadl(link, TEMPMON_FANCONTROL) & 0xff;
+	return sprintf(buf, "%u\n", spd * 100);
 }
 
+static ssize_t temp_show(struct device *device,
+			 struct device_attribute *attr, char *buf)
+{
+	struct ddb *dev = dev_get_drvdata(device);
+	struct ddb_link *link = &dev->link[0];
+	struct i2c_adapter *adap;
+	int temp, temp2;
+	u8 tmp[2];
+
+	if (!link->info->temp_num)
+		return sprintf(buf, "no sensor\n");
+	adap = &dev->i2c[link->info->temp_bus].adap;
+	if (i2c_read_regs(adap, 0x48, 0, tmp, 2) < 0)
+		return sprintf(buf, "read_error\n");
+	temp = (tmp[0] << 3) | (tmp[1] >> 5);
+	temp *= 125;
+	if (link->info->temp_num == 2) {
+		if (i2c_read_regs(adap, 0x49, 0, tmp, 2) < 0)
+			return sprintf(buf, "read_error\n");
+		temp2 = (tmp[0] << 3) | (tmp[1] >> 5);
+		temp2 *= 125;
+		return sprintf(buf, "%d %d\n", temp, temp2);
+	}
+	return sprintf(buf, "%d\n", temp);
+}
 
-static void ddb_remove(struct pci_dev *pdev)
+static ssize_t ctemp_show(struct device *device,
+		struct device_attribute *attr, char *buf)
 {
-	struct ddb *dev = pci_get_drvdata(pdev);
+	struct ddb *dev = dev_get_drvdata(device);
+	struct i2c_adapter *adap;
+	int temp;
+	u8 tmp[2];
+	int num = attr->attr.name[4] - 0x30;
 
-	ddb_ports_detach(dev);
-	ddb_i2c_release(dev);
+	adap = &dev->i2c[num].adap;
+	if (!adap)
+		return 0;
+	if (i2c_read_regs(adap, 0x49, 0, tmp, 2) < 0)
+		if (i2c_read_regs(adap, 0x4d, 0, tmp, 2) < 0)
+			return sprintf(buf, "no sensor\n");
+	temp = tmp[0] * 1000;
+	return sprintf(buf, "%d\n", temp);
+}
 
-	ddbwritel(0, INTERRUPT_ENABLE);
-	free_irq(dev->pdev->irq, dev);
-#ifdef CONFIG_PCI_MSI
-	if (dev->msi)
-		pci_disable_msi(dev->pdev);
-#endif
-	ddb_ports_release(dev);
-	ddb_buffers_free(dev);
-	ddb_device_destroy(dev);
+static ssize_t led_show(struct device *device,
+			struct device_attribute *attr, char *buf)
+{
+	struct ddb *dev = dev_get_drvdata(device);
+	int num = attr->attr.name[3] - 0x30;
 
-	ddb_unmap(dev);
-	pci_set_drvdata(pdev, NULL);
-	pci_disable_device(pdev);
+	return sprintf(buf, "%d\n", dev->leds & (1 << num) ? 1 : 0);
 }
 
 
-static int ddb_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+static void ddb_set_led(struct ddb *dev, int num, int val)
 {
-	struct ddb *dev;
-	int stat = 0;
-	int irq_flag = IRQF_SHARED;
+	if (!dev->link[0].info->led_num)
+		return;
+	switch (dev->port[num].class) {
+	case DDB_PORT_TUNER:
+		switch (dev->port[num].type) {
+		case DDB_TUNER_DVBS_ST:
+			i2c_write_reg16(&dev->i2c[num].adap,
+					0x69, 0xf14c, val ? 2 : 0);
+			break;
+		case DDB_TUNER_DVBCT_ST:
+			i2c_write_reg16(&dev->i2c[num].adap,
+					0x1f, 0xf00e, 0);
+			i2c_write_reg16(&dev->i2c[num].adap,
+					0x1f, 0xf00f, val ? 1 : 0);
+			break;
+		case DDB_TUNER_XO2 ... DDB_TUNER_DVBC2T2I_SONY:
+		{
+			u8 v;
 
-	if (pci_enable_device(pdev) < 0)
-		return -ENODEV;
+			i2c_read_reg(&dev->i2c[num].adap, 0x10, 0x08, &v);
+			v = (v & ~0x10) | (val ? 0x10 : 0);
+			i2c_write_reg(&dev->i2c[num].adap, 0x10, 0x08, v);
+			break;
+		}
+		default:
+			break;
+		}
+		break;
+	}
+}
 
-	dev = vzalloc(sizeof(struct ddb));
-	if (dev == NULL)
-		return -ENOMEM;
+static ssize_t led_store(struct device *device,
+			 struct device_attribute *attr,
+			 const char *buf, size_t count)
+{
+	struct ddb *dev = dev_get_drvdata(device);
+	int num = attr->attr.name[3] - 0x30;
+	u32 val;
 
-	dev->pdev = pdev;
-	pci_set_drvdata(pdev, dev);
-	dev->info = (struct ddb_info *) id->driver_data;
-	dev_info(&pdev->dev, "Detected %s\n", dev->info->name);
+	if (sscanf(buf, "%u\n", &val) != 1)
+		return -EINVAL;
+	if (val)
+		dev->leds |= (1 << num);
+	else
+		dev->leds &= ~(1 << num);
+	ddb_set_led(dev, num, val);
+	return count;
+}
 
-	dev->regs = ioremap(pci_resource_start(dev->pdev, 0),
-			    pci_resource_len(dev->pdev, 0));
-	if (!dev->regs) {
-		stat = -ENOMEM;
-		goto fail;
-	}
-	dev_info(&pdev->dev, "HW %08x FW %08x\n", ddbreadl(0), ddbreadl(4));
+static ssize_t snr_show(struct device *device,
+			struct device_attribute *attr, char *buf)
+{
+	struct ddb *dev = dev_get_drvdata(device);
+	char snr[32];
+	int num = attr->attr.name[3] - 0x30;
 
-#ifdef CONFIG_PCI_MSI
-	if (pci_msi_enabled())
-		stat = pci_enable_msi(dev->pdev);
-	if (stat) {
-		dev_info(&pdev->dev, "MSI not available.\n");
+	if (dev->port[num].type >= DDB_TUNER_XO2) {
+		if (i2c_read_regs(&dev->i2c[num].adap, 0x10, 0x10, snr, 16) < 0)
+			return sprintf(buf, "NO SNR\n");
+		snr[16] = 0;
 	} else {
-		irq_flag = 0;
-		dev->msi = 1;
-	}
-#endif
-	stat = request_irq(dev->pdev->irq, irq_handler,
-			   irq_flag, "DDBridge", (void *) dev);
-	if (stat < 0)
-		goto fail1;
-	ddbwritel(0, DMA_BASE_WRITE);
-	ddbwritel(0, DMA_BASE_READ);
-	ddbwritel(0xffffffff, INTERRUPT_ACK);
-	ddbwritel(0xfff0f, INTERRUPT_ENABLE);
-	ddbwritel(0, MSI1_ENABLE);
-
-	/* board control */
-	if (dev->info->board_control) {
-		ddbwritel(0, DDB_LINK_TAG(0) | BOARD_CONTROL);
-		msleep(100);
-		ddbwritel(dev->info->board_control_2,
-			DDB_LINK_TAG(0) | BOARD_CONTROL);
-		usleep_range(2000, 3000);
-		ddbwritel(dev->info->board_control_2
-			| dev->info->board_control,
-			DDB_LINK_TAG(0) | BOARD_CONTROL);
-		usleep_range(2000, 3000);
+		/* serial number at 0x100-0x11f */
+		if (i2c_read_regs16(&dev->i2c[num].adap,
+				    0x57, 0x100, snr, 32) < 0)
+			if (i2c_read_regs16(&dev->i2c[num].adap,
+					    0x50, 0x100, snr, 32) < 0)
+				return sprintf(buf, "NO SNR\n");
+		snr[31] = 0; /* in case it is not terminated on EEPROM */
 	}
+	return sprintf(buf, "%s\n", snr);
+}
 
-	if (ddb_i2c_init(dev) < 0)
-		goto fail1;
-	ddb_ports_init(dev);
-	if (ddb_buffers_alloc(dev) < 0) {
-		dev_err(&pdev->dev, "Could not allocate buffer memory\n");
-		goto fail2;
-	}
-	if (ddb_ports_attach(dev) < 0)
-		goto fail3;
-	ddb_device_create(dev);
+static ssize_t bsnr_show(struct device *device,
+			 struct device_attribute *attr, char *buf)
+{
+	struct ddb *dev = dev_get_drvdata(device);
+	char snr[16];
+
+	ddbridge_flashread(dev, 0, snr, 0x10, 15);
+	snr[15] = 0; /* in case it is not terminated on EEPROM */
+	return sprintf(buf, "%s\n", snr);
+}
+
+static ssize_t bpsnr_show(struct device *device,
+			 struct device_attribute *attr, char *buf)
+{
+	struct ddb *dev = dev_get_drvdata(device);
+	unsigned char snr[32];
+
+	if (!dev->i2c_num)
+		return 0;
+
+	if (i2c_read_regs16(&dev->i2c[0].adap,
+			    0x50, 0x0000, snr, 32) < 0 ||
+	    snr[0] == 0xff)
+		return sprintf(buf, "NO SNR\n");
+	snr[31] = 0; /* in case it is not terminated on EEPROM */
+	return sprintf(buf, "%s\n", snr);
+}
+
+static ssize_t redirect_show(struct device *device,
+			     struct device_attribute *attr, char *buf)
+{
 	return 0;
+}
 
-fail3:
-	ddb_ports_detach(dev);
-	dev_err(&pdev->dev, "fail3\n");
-	ddb_ports_release(dev);
-fail2:
-	dev_err(&pdev->dev, "fail2\n");
-	ddb_buffers_free(dev);
-fail1:
-	dev_err(&pdev->dev, "fail1\n");
-	if (dev->msi)
-		pci_disable_msi(dev->pdev);
-	if (stat == 0)
-		free_irq(dev->pdev->irq, dev);
-fail:
-	dev_err(&pdev->dev, "fail\n");
-	ddb_unmap(dev);
-	pci_set_drvdata(pdev, NULL);
-	pci_disable_device(pdev);
-	return -1;
+static ssize_t redirect_store(struct device *device,
+			      struct device_attribute *attr,
+			      const char *buf, size_t count)
+{
+	unsigned int i, p;
+	int res;
+
+	if (sscanf(buf, "%x %x\n", &i, &p) != 2)
+		return -EINVAL;
+	res = ddb_redirect(i, p);
+	if (res < 0)
+		return res;
+	dev_info(device, "redirect: %02x, %02x\n", i, p);
+	return count;
 }
 
-/******************************************************************************/
-/******************************************************************************/
-/******************************************************************************/
+static ssize_t gap_show(struct device *device,
+			struct device_attribute *attr, char *buf)
+{
+	struct ddb *dev = dev_get_drvdata(device);
+	int num = attr->attr.name[3] - 0x30;
 
-static const struct ddb_info ddb_none = {
-	.type     = DDB_NONE,
-	.name     = "Digital Devices PCIe bridge",
-};
+	return sprintf(buf, "%d\n", dev->port[num].gap);
 
-static const struct ddb_info ddb_octopus = {
-	.type     = DDB_OCTOPUS,
-	.name     = "Digital Devices Octopus DVB adapter",
-	.port_num = 4,
-};
+}
 
-static const struct ddb_info ddb_octopus_le = {
-	.type     = DDB_OCTOPUS,
-	.name     = "Digital Devices Octopus LE DVB adapter",
-	.port_num = 2,
-};
+static ssize_t gap_store(struct device *device, struct device_attribute *attr,
+			 const char *buf, size_t count)
+{
+	struct ddb *dev = dev_get_drvdata(device);
+	int num = attr->attr.name[3] - 0x30;
+	unsigned int val;
+
+	if (sscanf(buf, "%u\n", &val) != 1)
+		return -EINVAL;
+	if (val > 128)
+		return -EINVAL;
+	if (val == 128)
+		val = 0xffffffff;
+	dev->port[num].gap = val;
+	return count;
+}
+
+static ssize_t version_show(struct device *device,
+			    struct device_attribute *attr, char *buf)
+{
+	struct ddb *dev = dev_get_drvdata(device);
+
+	return sprintf(buf, "%08x %08x\n",
+		       dev->link[0].ids.hwid, dev->link[0].ids.regmapid);
+}
+
+static ssize_t hwid_show(struct device *device,
+			 struct device_attribute *attr, char *buf)
+{
+	struct ddb *dev = dev_get_drvdata(device);
+
+	return sprintf(buf, "0x%08X\n", dev->link[0].ids.hwid);
+}
+
+static ssize_t regmap_show(struct device *device,
+			   struct device_attribute *attr, char *buf)
+{
+	struct ddb *dev = dev_get_drvdata(device);
+
+	return sprintf(buf, "0x%08X\n", dev->link[0].ids.regmapid);
+}
+
+static ssize_t fmode_show(struct device *device,
+			 struct device_attribute *attr, char *buf)
+{
+	int num = attr->attr.name[5] - 0x30;
+	struct ddb *dev = dev_get_drvdata(device);
+
+	return sprintf(buf, "%u\n", dev->link[num].lnb.fmode);
+}
 
-static const struct ddb_info ddb_octopus_oem = {
-	.type     = DDB_OCTOPUS,
-	.name     = "Digital Devices Octopus OEM",
-	.port_num = 4,
+static ssize_t devid_show(struct device *device,
+			  struct device_attribute *attr, char *buf)
+{
+	int num = attr->attr.name[5] - 0x30;
+	struct ddb *dev = dev_get_drvdata(device);
+
+	return sprintf(buf, "%08x\n", dev->link[num].ids.devid);
+}
+
+static ssize_t fmode_store(struct device *device, struct device_attribute *attr,
+			  const char *buf, size_t count)
+{
+	struct ddb *dev = dev_get_drvdata(device);
+	int num = attr->attr.name[5] - 0x30;
+	unsigned int val;
+
+	if (sscanf(buf, "%u\n", &val) != 1)
+		return -EINVAL;
+	if (val > 3)
+		return -EINVAL;
+	lnb_init_fmode(dev, &dev->link[num], val);
+	return count;
+}
+
+static struct device_attribute ddb_attrs[] = {
+	__ATTR_RO(version),
+	__ATTR_RO(ports),
+	__ATTR_RO(ts_irq),
+	__ATTR_RO(i2c_irq),
+	__ATTR(gap0, 0664, gap_show, gap_store),
+	__ATTR(gap1, 0664, gap_show, gap_store),
+	__ATTR(gap2, 0664, gap_show, gap_store),
+	__ATTR(gap3, 0664, gap_show, gap_store),
+	__ATTR(fmode0, 0664, fmode_show, fmode_store),
+	__ATTR(fmode1, 0664, fmode_show, fmode_store),
+	__ATTR(fmode2, 0664, fmode_show, fmode_store),
+	__ATTR(fmode3, 0664, fmode_show, fmode_store),
+	__ATTR_MRO(devid0, devid_show),
+	__ATTR_MRO(devid1, devid_show),
+	__ATTR_MRO(devid2, devid_show),
+	__ATTR_MRO(devid3, devid_show),
+	__ATTR_RO(hwid),
+	__ATTR_RO(regmap),
+	__ATTR(redirect, 0664, redirect_show, redirect_store),
+	__ATTR_MRO(snr,  bsnr_show),
+	__ATTR_RO(bpsnr),
+	__ATTR_NULL,
 };
 
-static const struct ddb_info ddb_octopus_mini = {
-	.type     = DDB_OCTOPUS,
-	.name     = "Digital Devices Octopus Mini",
-	.port_num = 4,
+static struct device_attribute ddb_attrs_temp[] = {
+	__ATTR_RO(temp),
 };
 
-static const struct ddb_info ddb_v6 = {
-	.type     = DDB_OCTOPUS,
-	.name     = "Digital Devices Cine S2 V6 DVB adapter",
-	.port_num = 3,
+static struct device_attribute ddb_attrs_fan[] = {
+	__ATTR(fan, 0664, fan_show, fan_store),
 };
-static const struct ddb_info ddb_v6_5 = {
-	.type     = DDB_OCTOPUS,
-	.name     = "Digital Devices Cine S2 V6.5 DVB adapter",
-	.port_num = 4,
+
+static struct device_attribute ddb_attrs_snr[] = {
+	__ATTR_MRO(snr0, snr_show),
+	__ATTR_MRO(snr1, snr_show),
+	__ATTR_MRO(snr2, snr_show),
+	__ATTR_MRO(snr3, snr_show),
 };
 
-static const struct ddb_info ddb_dvbct = {
-	.type     = DDB_OCTOPUS,
-	.name     = "Digital Devices DVBCT V6.1 DVB adapter",
-	.port_num = 3,
+static struct device_attribute ddb_attrs_ctemp[] = {
+	__ATTR_MRO(temp0, ctemp_show),
+	__ATTR_MRO(temp1, ctemp_show),
+	__ATTR_MRO(temp2, ctemp_show),
+	__ATTR_MRO(temp3, ctemp_show),
 };
 
-static const struct ddb_info ddb_ctv7 = {
-	.type     = DDB_OCTOPUS,
-	.name     = "Digital Devices Cine CT V7 DVB adapter",
-	.port_num = 4,
-	.board_control   = 3,
-	.board_control_2 = 4,
+static struct device_attribute ddb_attrs_led[] = {
+	__ATTR(led0, 0664, led_show, led_store),
+	__ATTR(led1, 0664, led_show, led_store),
+	__ATTR(led2, 0664, led_show, led_store),
+	__ATTR(led3, 0664, led_show, led_store),
 };
 
-static const struct ddb_info ddb_satixS2v3 = {
-	.type     = DDB_OCTOPUS,
-	.name     = "Mystique SaTiX-S2 V3 DVB adapter",
-	.port_num = 3,
+static struct device_attribute ddb_attrs_fanspeed[] = {
+	__ATTR_MRO(fanspeed0, fanspeed_show),
+	__ATTR_MRO(fanspeed1, fanspeed_show),
+	__ATTR_MRO(fanspeed2, fanspeed_show),
+	__ATTR_MRO(fanspeed3, fanspeed_show),
 };
 
-static const struct ddb_info ddb_octopusv3 = {
-	.type     = DDB_OCTOPUS,
-	.name     = "Digital Devices Octopus V3 DVB adapter",
-	.port_num = 4,
+static struct class ddb_class = {
+	.name		= "ddbridge",
+	.owner          = THIS_MODULE,
+	.devnode        = ddb_devnode,
 };
 
-/*** MaxA8 adapters ***********************************************************/
+int ddb_class_create(void)
+{
+	ddb_major = register_chrdev(0, DDB_NAME, &ddb_fops);
+	if (ddb_major < 0)
+		return ddb_major;
+	if (class_register(&ddb_class) < 0)
+		return -1;
+	return 0;
+}
 
-static struct ddb_info ddb_ct2_8 = {
-	.type     = DDB_OCTOPUS_MAX_CT,
-	.name     = "Digital Devices MAX A8 CT2",
-	.port_num = 4,
-	.board_control   = 0x0ff,
-	.board_control_2 = 0xf00,
-	.ts_quirks = TS_QUIRK_SERIAL,
-};
+void ddb_class_destroy(void)
+{
+	class_unregister(&ddb_class);
+	unregister_chrdev(ddb_major, DDB_NAME);
+}
 
-static struct ddb_info ddb_c2t2_8 = {
-	.type     = DDB_OCTOPUS_MAX_CT,
-	.name     = "Digital Devices MAX A8 C2T2",
-	.port_num = 4,
-	.board_control   = 0x0ff,
-	.board_control_2 = 0xf00,
-	.ts_quirks = TS_QUIRK_SERIAL,
-};
+static void ddb_device_attrs_del(struct ddb *dev)
+{
+	int i;
 
-static struct ddb_info ddb_isdbt_8 = {
-	.type     = DDB_OCTOPUS_MAX_CT,
-	.name     = "Digital Devices MAX A8 ISDBT",
-	.port_num = 4,
-	.board_control   = 0x0ff,
-	.board_control_2 = 0xf00,
-	.ts_quirks = TS_QUIRK_SERIAL,
-};
+	for (i = 0; i < 4; i++)
+		if (dev->link[i].info && dev->link[i].info->tempmon_irq)
+			device_remove_file(dev->ddb_dev,
+					   &ddb_attrs_fanspeed[i]);
+	for (i = 0; i < dev->link[0].info->temp_num; i++)
+		device_remove_file(dev->ddb_dev, &ddb_attrs_temp[i]);
+	for (i = 0; i < dev->link[0].info->fan_num; i++)
+		device_remove_file(dev->ddb_dev, &ddb_attrs_fan[i]);
+	for (i = 0; i < dev->i2c_num && i < 4; i++) {
+		if (dev->link[0].info->led_num)
+			device_remove_file(dev->ddb_dev, &ddb_attrs_led[i]);
+		device_remove_file(dev->ddb_dev, &ddb_attrs_snr[i]);
+		device_remove_file(dev->ddb_dev, &ddb_attrs_ctemp[i]);
+	}
+	for (i = 0; ddb_attrs[i].attr.name != NULL; i++)
+		device_remove_file(dev->ddb_dev, &ddb_attrs[i]);
+}
 
-static struct ddb_info ddb_c2t2i_v0_8 = {
-	.type     = DDB_OCTOPUS_MAX_CT,
-	.name     = "Digital Devices MAX A8 C2T2I V0",
-	.port_num = 4,
-	.board_control   = 0x0ff,
-	.board_control_2 = 0xf00,
-	.ts_quirks = TS_QUIRK_SERIAL | TS_QUIRK_ALT_OSC,
-};
+static int ddb_device_attrs_add(struct ddb *dev)
+{
+	int i;
 
-static struct ddb_info ddb_c2t2i_8 = {
-	.type     = DDB_OCTOPUS_MAX_CT,
-	.name     = "Digital Devices MAX A8 C2T2I",
-	.port_num = 4,
-	.board_control   = 0x0ff,
-	.board_control_2 = 0xf00,
-	.ts_quirks = TS_QUIRK_SERIAL,
-};
+	for (i = 0; ddb_attrs[i].attr.name != NULL; i++)
+		if (device_create_file(dev->ddb_dev, &ddb_attrs[i]))
+			goto fail;
+	for (i = 0; i < dev->link[0].info->temp_num; i++)
+		if (device_create_file(dev->ddb_dev, &ddb_attrs_temp[i]))
+			goto fail;
+	for (i = 0; i < dev->link[0].info->fan_num; i++)
+		if (device_create_file(dev->ddb_dev, &ddb_attrs_fan[i]))
+			goto fail;
+	for (i = 0; (i < dev->i2c_num) && (i < 4); i++) {
+		if (device_create_file(dev->ddb_dev, &ddb_attrs_snr[i]))
+			goto fail;
+		if (device_create_file(dev->ddb_dev, &ddb_attrs_ctemp[i]))
+			goto fail;
+		if (dev->link[0].info->led_num)
+			if (device_create_file(dev->ddb_dev,
+					       &ddb_attrs_led[i]))
+				goto fail;
+	}
+	for (i = 0; i < 4; i++)
+		if (dev->link[i].info && dev->link[i].info->tempmon_irq)
+			if (device_create_file(dev->ddb_dev,
+					       &ddb_attrs_fanspeed[i]))
+				goto fail;
+	return 0;
+fail:
+	return -1;
+}
 
-/******************************************************************************/
+int ddb_device_create(struct ddb *dev)
+{
+	int res = 0;
 
-#define DDVID 0xdd01 /* Digital Devices Vendor ID */
-
-#define DDB_ID(_vend, _dev, _subvend, _subdev, _driverdata) {	\
-	.vendor      = _vend,    .device    = _dev, \
-	.subvendor   = _subvend, .subdevice = _subdev, \
-	.driver_data = (unsigned long)&_driverdata }
-
-static const struct pci_device_id ddb_id_tbl[] = {
-	DDB_ID(DDVID, 0x0002, DDVID, 0x0001, ddb_octopus),
-	DDB_ID(DDVID, 0x0003, DDVID, 0x0001, ddb_octopus),
-	DDB_ID(DDVID, 0x0005, DDVID, 0x0004, ddb_octopusv3),
-	DDB_ID(DDVID, 0x0003, DDVID, 0x0002, ddb_octopus_le),
-	DDB_ID(DDVID, 0x0003, DDVID, 0x0003, ddb_octopus_oem),
-	DDB_ID(DDVID, 0x0003, DDVID, 0x0010, ddb_octopus_mini),
-	DDB_ID(DDVID, 0x0005, DDVID, 0x0011, ddb_octopus_mini),
-	DDB_ID(DDVID, 0x0003, DDVID, 0x0020, ddb_v6),
-	DDB_ID(DDVID, 0x0003, DDVID, 0x0021, ddb_v6_5),
-	DDB_ID(DDVID, 0x0003, DDVID, 0x0030, ddb_dvbct),
-	DDB_ID(DDVID, 0x0003, DDVID, 0xdb03, ddb_satixS2v3),
-	DDB_ID(DDVID, 0x0006, DDVID, 0x0031, ddb_ctv7),
-	DDB_ID(DDVID, 0x0006, DDVID, 0x0032, ddb_ctv7),
-	DDB_ID(DDVID, 0x0006, DDVID, 0x0033, ddb_ctv7),
-	DDB_ID(DDVID, 0x0008, DDVID, 0x0034, ddb_ct2_8),
-	DDB_ID(DDVID, 0x0008, DDVID, 0x0035, ddb_c2t2_8),
-	DDB_ID(DDVID, 0x0008, DDVID, 0x0036, ddb_isdbt_8),
-	DDB_ID(DDVID, 0x0008, DDVID, 0x0037, ddb_c2t2i_v0_8),
-	DDB_ID(DDVID, 0x0008, DDVID, 0x0038, ddb_c2t2i_8),
-	DDB_ID(DDVID, 0x0006, DDVID, 0x0039, ddb_ctv7),
-	/* in case sub-ids got deleted in flash */
-	DDB_ID(DDVID, 0x0003, PCI_ANY_ID, PCI_ANY_ID, ddb_none),
-	DDB_ID(DDVID, 0x0005, PCI_ANY_ID, PCI_ANY_ID, ddb_none),
-	DDB_ID(DDVID, 0x0006, PCI_ANY_ID, PCI_ANY_ID, ddb_none),
-	DDB_ID(DDVID, 0x0007, PCI_ANY_ID, PCI_ANY_ID, ddb_none),
-	DDB_ID(DDVID, 0x0008, PCI_ANY_ID, PCI_ANY_ID, ddb_none),
-	DDB_ID(DDVID, 0x0011, PCI_ANY_ID, PCI_ANY_ID, ddb_none),
-	DDB_ID(DDVID, 0x0013, PCI_ANY_ID, PCI_ANY_ID, ddb_none),
-	DDB_ID(DDVID, 0x0201, PCI_ANY_ID, PCI_ANY_ID, ddb_none),
-	DDB_ID(DDVID, 0x0320, PCI_ANY_ID, PCI_ANY_ID, ddb_none),
-	{0}
-};
-MODULE_DEVICE_TABLE(pci, ddb_id_tbl);
+	if (ddb_num == DDB_MAX_ADAPTER)
+		return -ENOMEM;
+	mutex_lock(&ddb_mutex);
+	dev->nr = ddb_num;
+	ddbs[dev->nr] = dev;
+	dev->ddb_dev = device_create(&ddb_class, dev->dev,
+				     MKDEV(ddb_major, dev->nr),
+				     dev, "ddbridge%d", dev->nr);
+	if (IS_ERR(dev->ddb_dev)) {
+		res = PTR_ERR(dev->ddb_dev);
+		dev_info(dev->dev, "Could not create ddbridge%d\n", dev->nr);
+		goto fail;
+	}
+	res = ddb_device_attrs_add(dev);
+	if (res) {
+		ddb_device_attrs_del(dev);
+		device_destroy(&ddb_class, MKDEV(ddb_major, dev->nr));
+		ddbs[dev->nr] = NULL;
+		dev->ddb_dev = ERR_PTR(-ENODEV);
+	} else
+		ddb_num++;
+fail:
+	mutex_unlock(&ddb_mutex);
+	return res;
+}
 
+void ddb_device_destroy(struct ddb *dev)
+{
+	if (IS_ERR(dev->ddb_dev))
+		return;
+	ddb_device_attrs_del(dev);
+	device_destroy(&ddb_class, MKDEV(ddb_major, dev->nr));
+}
 
-static struct pci_driver ddb_pci_driver = {
-	.name        = "DDBridge",
-	.id_table    = ddb_id_tbl,
-	.probe       = ddb_probe,
-	.remove      = ddb_remove,
-};
+/****************************************************************************/
+/****************************************************************************/
+/****************************************************************************/
 
-static __init int module_init_ddbridge(void)
+static void tempmon_setfan(struct ddb_link *link)
 {
-	int ret;
+	u32 temp, temp2, pwm;
+
+	if ((ddblreadl(link, TEMPMON_CONTROL) &
+	    TEMPMON_CONTROL_OVERTEMP) != 0) {
+		dev_info(link->dev->dev, "Over temperature condition\n");
+		link->overtemperature_error = 1;
+	}
+	temp  = (ddblreadl(link, TEMPMON_SENSOR0) >> 8) & 0xFF;
+	if (temp & 0x80)
+		temp = 0;
+	temp2  = (ddblreadl(link, TEMPMON_SENSOR1) >> 8) & 0xFF;
+	if (temp2 & 0x80)
+		temp2 = 0;
+	if (temp2 > temp)
+		temp = temp2;
+
+	pwm = (ddblreadl(link, TEMPMON_FANCONTROL) >> 8) & 0x0F;
+	if (pwm > 10)
+		pwm = 10;
+
+	if (temp >= link->temp_tab[pwm]) {
+		while (pwm < 10 && temp >= link->temp_tab[pwm + 1])
+			pwm += 1;
+	} else {
+		while (pwm > 1 && temp < link->temp_tab[pwm - 2])
+			pwm -= 1;
+	}
+	ddblwritel(link, (pwm << 8), TEMPMON_FANCONTROL);
+}
 
-	pr_info("Digital Devices PCIE bridge driver, Copyright (C) 2010-11 Digital Devices GmbH\n");
+static void temp_handler(unsigned long data)
+{
+	struct ddb_link *link = (struct ddb_link *) data;
 
-	ret = ddb_class_create();
-	if (ret < 0)
-		return ret;
-	ret = pci_register_driver(&ddb_pci_driver);
-	if (ret < 0)
-		ddb_class_destroy();
-	return ret;
+	spin_lock(&link->temp_lock);
+	tempmon_setfan(link);
+	spin_unlock(&link->temp_lock);
+}
+
+static int tempmon_init(struct ddb_link *link, int first_time)
+{
+	struct ddb *dev = link->dev;
+	int status = 0;
+	u32 l = link->nr;
+
+	spin_lock_irq(&link->temp_lock);
+	if (first_time) {
+		static u8 temperature_table[11] = {
+			30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80 };
+
+		memcpy(link->temp_tab, temperature_table,
+			sizeof(temperature_table));
+	}
+	dev->handler[l][link->info->tempmon_irq] = temp_handler;
+	dev->handler_data[l][link->info->tempmon_irq] = (unsigned long) link;
+	ddblwritel(link, (TEMPMON_CONTROL_OVERTEMP | TEMPMON_CONTROL_AUTOSCAN |
+			  TEMPMON_CONTROL_INTENABLE),
+		   TEMPMON_CONTROL);
+	ddblwritel(link, (3 << 8), TEMPMON_FANCONTROL);
+
+	link->overtemperature_error =
+		((ddblreadl(link, TEMPMON_CONTROL) &
+			TEMPMON_CONTROL_OVERTEMP) != 0);
+	if (link->overtemperature_error) {
+		dev_info(link->dev->dev, "Over temperature condition\n");
+		status = -1;
+	}
+	tempmon_setfan(link);
+	spin_unlock_irq(&link->temp_lock);
+	return status;
 }
 
-static __exit void module_exit_ddbridge(void)
+static int ddb_init_tempmon(struct ddb_link *link)
 {
-	pci_unregister_driver(&ddb_pci_driver);
-	ddb_class_destroy();
+	const struct ddb_info *info = link->info;
+
+	if (!info->tempmon_irq)
+		return 0;
+	if (info->type == DDB_OCTOPUS_MAX_CT)
+		if (link->ids.regmapid < 0x00010002)
+			return 0;
+	spin_lock_init(&link->temp_lock);
+	dev_dbg(link->dev->dev, "init_tempmon\n");
+	return tempmon_init(link, 1);
+}
+
+/****************************************************************************/
+/****************************************************************************/
+/****************************************************************************/
+
+static int ddb_init_boards(struct ddb *dev)
+{
+	const struct ddb_info *info;
+	struct ddb_link *link;
+	u32 l;
+
+	for (l = 0; l < DDB_MAX_LINK; l++) {
+		link = &dev->link[l];
+		info = link->info;
+
+		if (!info)
+			continue;
+		if (info->board_control) {
+			ddbwritel(dev, 0, DDB_LINK_TAG(l) | BOARD_CONTROL);
+			msleep(100);
+			ddbwritel(dev, info->board_control_2,
+				DDB_LINK_TAG(l) | BOARD_CONTROL);
+			usleep_range(2000, 3000);
+			ddbwritel(dev,
+				info->board_control_2 | info->board_control,
+				DDB_LINK_TAG(l) | BOARD_CONTROL);
+			usleep_range(2000, 3000);
+		}
+		ddb_init_tempmon(link);
+	}
+	return 0;
 }
 
-module_init(module_init_ddbridge);
-module_exit(module_exit_ddbridge);
+int ddb_init(struct ddb *dev)
+{
+	mutex_init(&dev->link[0].lnb.lock);
+	mutex_init(&dev->link[0].flash_mutex);
+	if (no_init) {
+		ddb_device_create(dev);
+		return 0;
+	}
+
+	ddb_init_boards(dev);
+
+	if (ddb_i2c_init(dev) < 0)
+		goto fail;
+	ddb_ports_init(dev);
+	if (ddb_buffers_alloc(dev) < 0) {
+		dev_info(dev->dev, "Could not allocate buffer memory\n");
+		goto fail2;
+	}
+	if (ddb_ports_attach(dev) < 0)
+		goto fail3;
+
+	ddb_device_create(dev);
+
+	if (dev->link[0].info->fan_num)	{
+		ddbwritel(dev, 1, GPIO_DIRECTION);
+		ddbwritel(dev, 1, GPIO_OUTPUT);
+	}
+	return 0;
+
+fail3:
+	ddb_ports_detach(dev);
+	dev_err(dev->dev, "fail3\n");
+	ddb_ports_release(dev);
+fail2:
+	dev_err(dev->dev, "fail2\n");
+	ddb_buffers_free(dev);
+	ddb_i2c_release(dev);
+fail:
+	dev_err(dev->dev, "fail1\n");
+	return -1;
+}
 
-MODULE_DESCRIPTION("Digital Devices PCIe Bridge");
-MODULE_AUTHOR("Ralph Metzler");
-MODULE_LICENSE("GPL");
-MODULE_VERSION("0.5");
+void ddb_unmap(struct ddb *dev)
+{
+	if (dev->regs)
+		iounmap(dev->regs);
+	vfree(dev);
+}
diff --git a/drivers/media/pci/ddbridge/ddbridge-hw.c b/drivers/media/pci/ddbridge/ddbridge-hw.c
new file mode 100644
index 000000000000..48248bcd59c2
--- /dev/null
+++ b/drivers/media/pci/ddbridge/ddbridge-hw.c
@@ -0,0 +1,376 @@
+/*
+ * ddbridge-hw.c: Digital Devices bridge hardware maps
+ *
+ * Copyright (C) 2010-2017 Digital Devices GmbH
+ *                         Ralph Metzler <rjkm@metzlerbros.de>
+ *                         Marcus Metzler <mocm@metzlerbros.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 only, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include "ddbridge.h"
+#include "ddbridge-hw.h"
+
+/******************************************************************************/
+
+static const struct ddb_regset octopus_input = {
+	.base = 0x200,
+	.num  = 0x08,
+	.size = 0x10,
+};
+
+static const struct ddb_regset octopus_output = {
+	.base = 0x280,
+	.num  = 0x08,
+	.size = 0x10,
+};
+
+static const struct ddb_regset octopus_idma = {
+	.base = 0x300,
+	.num  = 0x08,
+	.size = 0x10,
+};
+
+static const struct ddb_regset octopus_idma_buf = {
+	.base = 0x2000,
+	.num  = 0x08,
+	.size = 0x100,
+};
+
+static const struct ddb_regset octopus_odma = {
+	.base = 0x380,
+	.num  = 0x04,
+	.size = 0x10,
+};
+
+static const struct ddb_regset octopus_odma_buf = {
+	.base = 0x2800,
+	.num  = 0x04,
+	.size = 0x100,
+};
+
+static const struct ddb_regset octopus_i2c = {
+	.base = 0x80,
+	.num  = 0x04,
+	.size = 0x20,
+};
+
+static const struct ddb_regset octopus_i2c_buf = {
+	.base = 0x1000,
+	.num  = 0x04,
+	.size = 0x200,
+};
+
+/****************************************************************************/
+
+static const struct ddb_regmap octopus_map = {
+	.irq_base_i2c = 0,
+	.irq_base_idma = 8,
+	.irq_base_odma = 16,
+	.i2c = &octopus_i2c,
+	.i2c_buf = &octopus_i2c_buf,
+	.idma = &octopus_idma,
+	.idma_buf = &octopus_idma_buf,
+	.odma = &octopus_odma,
+	.odma_buf = &octopus_odma_buf,
+	.input = &octopus_input,
+	.output = &octopus_output,
+};
+
+/****************************************************************************/
+
+static const struct ddb_info ddb_none = {
+	.type     = DDB_NONE,
+	.name     = "unknown Digital Devices PCIe card, install newer driver",
+	.regmap   = &octopus_map,
+};
+
+static const struct ddb_info ddb_octopus = {
+	.type     = DDB_OCTOPUS,
+	.name     = "Digital Devices Octopus DVB adapter",
+	.regmap   = &octopus_map,
+	.port_num = 4,
+	.i2c_mask = 0x0f,
+};
+
+static const struct ddb_info ddb_octopusv3 = {
+	.type     = DDB_OCTOPUS,
+	.name     = "Digital Devices Octopus V3 DVB adapter",
+	.regmap   = &octopus_map,
+	.port_num = 4,
+	.i2c_mask = 0x0f,
+};
+
+static const struct ddb_info ddb_octopus_le = {
+	.type     = DDB_OCTOPUS,
+	.name     = "Digital Devices Octopus LE DVB adapter",
+	.regmap   = &octopus_map,
+	.port_num = 2,
+	.i2c_mask = 0x03,
+};
+
+static const struct ddb_info ddb_octopus_oem = {
+	.type     = DDB_OCTOPUS,
+	.name     = "Digital Devices Octopus OEM",
+	.regmap   = &octopus_map,
+	.port_num = 4,
+	.i2c_mask = 0x0f,
+	.led_num  = 1,
+	.fan_num  = 1,
+	.temp_num = 1,
+	.temp_bus = 0,
+};
+
+static const struct ddb_info ddb_octopus_mini = {
+	.type     = DDB_OCTOPUS,
+	.name     = "Digital Devices Octopus Mini",
+	.regmap   = &octopus_map,
+	.port_num = 4,
+	.i2c_mask = 0x0f,
+};
+
+static const struct ddb_info ddb_v6 = {
+	.type     = DDB_OCTOPUS,
+	.name     = "Digital Devices Cine S2 V6 DVB adapter",
+	.regmap   = &octopus_map,
+	.port_num = 3,
+	.i2c_mask = 0x07,
+};
+
+static const struct ddb_info ddb_v6_5 = {
+	.type     = DDB_OCTOPUS,
+	.name     = "Digital Devices Cine S2 V6.5 DVB adapter",
+	.regmap   = &octopus_map,
+	.port_num = 4,
+	.i2c_mask = 0x0f,
+};
+
+static const struct ddb_info ddb_v7 = {
+	.type     = DDB_OCTOPUS,
+	.name     = "Digital Devices Cine S2 V7 DVB adapter",
+	.regmap   = &octopus_map,
+	.port_num = 4,
+	.i2c_mask = 0x0f,
+	.board_control   = 2,
+	.board_control_2 = 4,
+	.ts_quirks = TS_QUIRK_REVERSED,
+};
+
+static const struct ddb_info ddb_v7a = {
+	.type     = DDB_OCTOPUS,
+	.name     = "Digital Devices Cine S2 V7 Advanced DVB adapter",
+	.regmap   = &octopus_map,
+	.port_num = 4,
+	.i2c_mask = 0x0f,
+	.board_control   = 2,
+	.board_control_2 = 4,
+	.ts_quirks = TS_QUIRK_REVERSED,
+};
+
+static const struct ddb_info ddb_ctv7 = {
+	.type     = DDB_OCTOPUS,
+	.name     = "Digital Devices Cine CT V7 DVB adapter",
+	.regmap   = &octopus_map,
+	.port_num = 4,
+	.i2c_mask = 0x0f,
+	.board_control   = 3,
+	.board_control_2 = 4,
+};
+
+static const struct ddb_info ddb_satixS2v3 = {
+	.type     = DDB_OCTOPUS,
+	.name     = "Mystique SaTiX-S2 V3 DVB adapter",
+	.regmap   = &octopus_map,
+	.port_num = 3,
+	.i2c_mask = 0x07,
+};
+
+static const struct ddb_info ddb_ci = {
+	.type     = DDB_OCTOPUS_CI,
+	.name     = "Digital Devices Octopus CI",
+	.regmap   = &octopus_map,
+	.port_num = 4,
+	.i2c_mask = 0x03,
+};
+
+static const struct ddb_info ddb_cis = {
+	.type     = DDB_OCTOPUS_CI,
+	.name     = "Digital Devices Octopus CI single",
+	.regmap   = &octopus_map,
+	.port_num = 3,
+	.i2c_mask = 0x03,
+};
+
+static const struct ddb_info ddb_ci_s2_pro = {
+	.type     = DDB_OCTOPUS_CI,
+	.name     = "Digital Devices Octopus CI S2 Pro",
+	.regmap   = &octopus_map,
+	.port_num = 4,
+	.i2c_mask = 0x01,
+	.board_control   = 2,
+	.board_control_2 = 4,
+};
+
+static const struct ddb_info ddb_ci_s2_pro_a = {
+	.type     = DDB_OCTOPUS_CI,
+	.name     = "Digital Devices Octopus CI S2 Pro Advanced",
+	.regmap   = &octopus_map,
+	.port_num = 4,
+	.i2c_mask = 0x01,
+	.board_control   = 2,
+	.board_control_2 = 4,
+};
+
+static const struct ddb_info ddb_dvbct = {
+	.type     = DDB_OCTOPUS,
+	.name     = "Digital Devices DVBCT V6.1 DVB adapter",
+	.regmap   = &octopus_map,
+	.port_num = 3,
+	.i2c_mask = 0x07,
+};
+
+/****************************************************************************/
+
+static const struct ddb_info ddb_ct2_8 = {
+	.type     = DDB_OCTOPUS_MAX_CT,
+	.name     = "Digital Devices MAX A8 CT2",
+	.regmap   = &octopus_map,
+	.port_num = 4,
+	.i2c_mask = 0x0f,
+	.board_control   = 0x0ff,
+	.board_control_2 = 0xf00,
+	.ts_quirks = TS_QUIRK_SERIAL,
+	.tempmon_irq = 24,
+};
+
+static const struct ddb_info ddb_c2t2_8 = {
+	.type     = DDB_OCTOPUS_MAX_CT,
+	.name     = "Digital Devices MAX A8 C2T2",
+	.regmap   = &octopus_map,
+	.port_num = 4,
+	.i2c_mask = 0x0f,
+	.board_control   = 0x0ff,
+	.board_control_2 = 0xf00,
+	.ts_quirks = TS_QUIRK_SERIAL,
+	.tempmon_irq = 24,
+};
+
+static const struct ddb_info ddb_isdbt_8 = {
+	.type     = DDB_OCTOPUS_MAX_CT,
+	.name     = "Digital Devices MAX A8 ISDBT",
+	.regmap   = &octopus_map,
+	.port_num = 4,
+	.i2c_mask = 0x0f,
+	.board_control   = 0x0ff,
+	.board_control_2 = 0xf00,
+	.ts_quirks = TS_QUIRK_SERIAL,
+	.tempmon_irq = 24,
+};
+
+static const struct ddb_info ddb_c2t2i_v0_8 = {
+	.type     = DDB_OCTOPUS_MAX_CT,
+	.name     = "Digital Devices MAX A8 C2T2I V0",
+	.regmap   = &octopus_map,
+	.port_num = 4,
+	.i2c_mask = 0x0f,
+	.board_control   = 0x0ff,
+	.board_control_2 = 0xf00,
+	.ts_quirks = TS_QUIRK_SERIAL | TS_QUIRK_ALT_OSC,
+	.tempmon_irq = 24,
+};
+
+static const struct ddb_info ddb_c2t2i_8 = {
+	.type     = DDB_OCTOPUS_MAX_CT,
+	.name     = "Digital Devices MAX A8 C2T2I",
+	.regmap   = &octopus_map,
+	.port_num = 4,
+	.i2c_mask = 0x0f,
+	.board_control   = 0x0ff,
+	.board_control_2 = 0xf00,
+	.ts_quirks = TS_QUIRK_SERIAL,
+	.tempmon_irq = 24,
+};
+
+/****************************************************************************/
+
+static const struct ddb_info ddb_s2_48 = {
+	.type     = DDB_OCTOPUS_MAX,
+	.name     = "Digital Devices MAX S8 4/8",
+	.regmap   = &octopus_map,
+	.port_num = 4,
+	.i2c_mask = 0x01,
+	.board_control = 1,
+	.tempmon_irq = 24,
+};
+
+/****************************************************************************/
+/****************************************************************************/
+/****************************************************************************/
+
+#define DDB_DEVID(_device, _subdevice, _info) { \
+	.vendor = DDVID, \
+	.device = _device, \
+	.subvendor = DDVID, \
+	.subdevice = _subdevice, \
+	.info = &_info }
+
+static const struct ddb_device_id ddb_device_ids[] = {
+	/* PCIe devices */
+	DDB_DEVID(0x0002, 0x0001, ddb_octopus),
+	DDB_DEVID(0x0003, 0x0001, ddb_octopus),
+	DDB_DEVID(0x0005, 0x0004, ddb_octopusv3),
+	DDB_DEVID(0x0003, 0x0002, ddb_octopus_le),
+	DDB_DEVID(0x0003, 0x0003, ddb_octopus_oem),
+	DDB_DEVID(0x0003, 0x0010, ddb_octopus_mini),
+	DDB_DEVID(0x0005, 0x0011, ddb_octopus_mini),
+	DDB_DEVID(0x0003, 0x0020, ddb_v6),
+	DDB_DEVID(0x0003, 0x0021, ddb_v6_5),
+	DDB_DEVID(0x0006, 0x0022, ddb_v7),
+	DDB_DEVID(0x0006, 0x0024, ddb_v7a),
+	DDB_DEVID(0x0003, 0x0030, ddb_dvbct),
+	DDB_DEVID(0x0003, 0xdb03, ddb_satixS2v3),
+	DDB_DEVID(0x0006, 0x0031, ddb_ctv7),
+	DDB_DEVID(0x0006, 0x0032, ddb_ctv7),
+	DDB_DEVID(0x0006, 0x0033, ddb_ctv7),
+	DDB_DEVID(0x0007, 0x0023, ddb_s2_48),
+	DDB_DEVID(0x0008, 0x0034, ddb_ct2_8),
+	DDB_DEVID(0x0008, 0x0035, ddb_c2t2_8),
+	DDB_DEVID(0x0008, 0x0036, ddb_isdbt_8),
+	DDB_DEVID(0x0008, 0x0037, ddb_c2t2i_v0_8),
+	DDB_DEVID(0x0008, 0x0038, ddb_c2t2i_8),
+	DDB_DEVID(0x0006, 0x0039, ddb_ctv7),
+	DDB_DEVID(0x0011, 0x0040, ddb_ci),
+	DDB_DEVID(0x0011, 0x0041, ddb_cis),
+	DDB_DEVID(0x0012, 0x0042, ddb_ci),
+	DDB_DEVID(0x0013, 0x0043, ddb_ci_s2_pro),
+	DDB_DEVID(0x0013, 0x0044, ddb_ci_s2_pro_a),
+};
+
+/****************************************************************************/
+
+const struct ddb_info *get_ddb_info(u16 vendor, u16 device,
+				    u16 subvendor, u16 subdevice)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(ddb_device_ids); i++) {
+		const struct ddb_device_id *id = &ddb_device_ids[i];
+
+		if (vendor == id->vendor &&
+		    device == id->device &&
+		    subvendor == id->subvendor &&
+		    ((subdevice == id->subdevice) ||
+		     (id->subdevice == 0xffff)))
+			return id->info;
+	}
+
+	return &ddb_none;
+}
diff --git a/drivers/media/pci/ddbridge/ddbridge-hw.h b/drivers/media/pci/ddbridge/ddbridge-hw.h
new file mode 100644
index 000000000000..7c142419419c
--- /dev/null
+++ b/drivers/media/pci/ddbridge/ddbridge-hw.h
@@ -0,0 +1,43 @@
+/*
+ * ddbridge-hw.h: Digital Devices bridge hardware maps
+ *
+ * Copyright (C) 2010-2017 Digital Devices GmbH
+ *                         Ralph Metzler <rjkm@metzlerbros.de>
+ *                         Marcus Metzler <mocm@metzlerbros.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 only, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#ifndef _DDBRIDGE_HW_H_
+#define _DDBRIDGE_HW_H_
+
+#include "ddbridge.h"
+
+/******************************************************************************/
+
+#define DDVID 0xdd01 /* Digital Devices Vendor ID */
+
+/******************************************************************************/
+
+struct ddb_device_id {
+	u16 vendor;
+	u16 device;
+	u16 subvendor;
+	u16 subdevice;
+	const struct ddb_info *info;
+};
+
+/******************************************************************************/
+
+const struct ddb_info *get_ddb_info(u16 vendor, u16 device,
+				    u16 subvendor, u16 subdevice);
+
+#endif /* _DDBRIDGE_HW_H */
diff --git a/drivers/media/pci/ddbridge/ddbridge-i2c.c b/drivers/media/pci/ddbridge/ddbridge-i2c.c
new file mode 100644
index 000000000000..e4d39c3270ae
--- /dev/null
+++ b/drivers/media/pci/ddbridge/ddbridge-i2c.c
@@ -0,0 +1,230 @@
+/*
+ * ddbridge-i2c.c: Digital Devices bridge i2c driver
+ *
+ * Copyright (C) 2010-2017 Digital Devices GmbH
+ *                         Ralph Metzler <rjkm@metzlerbros.de>
+ *                         Marcus Metzler <mocm@metzlerbros.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 only, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/delay.h>
+#include <linux/slab.h>
+#include <linux/poll.h>
+#include <linux/io.h>
+#include <linux/pci.h>
+#include <linux/pci_ids.h>
+#include <linux/timer.h>
+#include <linux/i2c.h>
+#include <linux/swab.h>
+#include <linux/vmalloc.h>
+
+#include "ddbridge.h"
+#include "ddbridge-i2c.h"
+#include "ddbridge-regs.h"
+#include "ddbridge-io.h"
+
+/******************************************************************************/
+
+static int ddb_i2c_cmd(struct ddb_i2c *i2c, u32 adr, u32 cmd)
+{
+	struct ddb *dev = i2c->dev;
+	unsigned long stat;
+	u32 val;
+
+	ddbwritel(dev, (adr << 9) | cmd, i2c->regs + I2C_COMMAND);
+	stat = wait_for_completion_timeout(&i2c->completion, HZ);
+	val = ddbreadl(dev, i2c->regs + I2C_COMMAND);
+	if (stat == 0) {
+		dev_err(dev->dev, "I2C timeout, card %d, port %d, link %u\n",
+			dev->nr, i2c->nr, i2c->link);
+		{
+			u32 istat = ddbreadl(dev, INTERRUPT_STATUS);
+
+			dev_err(dev->dev, "DDBridge IRS %08x\n", istat);
+			if (i2c->link) {
+				u32 listat = ddbreadl(dev,
+					DDB_LINK_TAG(i2c->link) |
+					INTERRUPT_STATUS);
+
+				dev_err(dev->dev, "DDBridge link %u IRS %08x\n",
+					i2c->link, listat);
+			}
+			if (istat & 1) {
+				ddbwritel(dev, istat & 1, INTERRUPT_ACK);
+			} else {
+				u32 mon = ddbreadl(dev,
+					i2c->regs + I2C_MONITOR);
+
+				dev_err(dev->dev, "I2C cmd=%08x mon=%08x\n",
+					val, mon);
+			}
+		}
+		return -EIO;
+	}
+	if (val & 0x70000)
+		return -EIO;
+	return 0;
+}
+
+static int ddb_i2c_master_xfer(struct i2c_adapter *adapter,
+			       struct i2c_msg msg[], int num)
+{
+	struct ddb_i2c *i2c = (struct ddb_i2c *) i2c_get_adapdata(adapter);
+	struct ddb *dev = i2c->dev;
+	u8 addr = 0;
+
+	addr = msg[0].addr;
+	if (msg[0].len > i2c->bsize)
+		return -EIO;
+	switch (num) {
+	case 1:
+		if (msg[0].flags & I2C_M_RD) {
+			ddbwritel(dev, msg[0].len << 16,
+				  i2c->regs + I2C_TASKLENGTH);
+			if (ddb_i2c_cmd(i2c, addr, 3))
+				break;
+			ddbcpyfrom(dev, msg[0].buf,
+				   i2c->rbuf, msg[0].len);
+			return num;
+		}
+		ddbcpyto(dev, i2c->wbuf, msg[0].buf, msg[0].len);
+		ddbwritel(dev, msg[0].len, i2c->regs + I2C_TASKLENGTH);
+		if (ddb_i2c_cmd(i2c, addr, 2))
+			break;
+		return num;
+	case 2:
+		if ((msg[0].flags & I2C_M_RD) == I2C_M_RD)
+			break;
+		if ((msg[1].flags & I2C_M_RD) != I2C_M_RD)
+			break;
+		if (msg[1].len > i2c->bsize)
+			break;
+		ddbcpyto(dev, i2c->wbuf, msg[0].buf, msg[0].len);
+		ddbwritel(dev, msg[0].len | (msg[1].len << 16),
+			  i2c->regs + I2C_TASKLENGTH);
+		if (ddb_i2c_cmd(i2c, addr, 1))
+			break;
+		ddbcpyfrom(dev, msg[1].buf,
+			   i2c->rbuf,
+			   msg[1].len);
+		return num;
+	default:
+		break;
+	}
+	return -EIO;
+}
+
+static u32 ddb_i2c_functionality(struct i2c_adapter *adap)
+{
+	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
+}
+
+static const struct i2c_algorithm ddb_i2c_algo = {
+	.master_xfer   = ddb_i2c_master_xfer,
+	.functionality = ddb_i2c_functionality,
+};
+
+void ddb_i2c_release(struct ddb *dev)
+{
+	int i;
+	struct ddb_i2c *i2c;
+
+	for (i = 0; i < dev->i2c_num; i++) {
+		i2c = &dev->i2c[i];
+		i2c_del_adapter(&i2c->adap);
+	}
+}
+
+static void i2c_handler(unsigned long priv)
+{
+	struct ddb_i2c *i2c = (struct ddb_i2c *) priv;
+
+	complete(&i2c->completion);
+}
+
+static int ddb_i2c_add(struct ddb *dev, struct ddb_i2c *i2c,
+		       const struct ddb_regmap *regmap, int link,
+		       int i, int num)
+{
+	struct i2c_adapter *adap;
+
+	i2c->nr = i;
+	i2c->dev = dev;
+	i2c->link = link;
+	i2c->bsize = regmap->i2c_buf->size;
+	i2c->wbuf = DDB_LINK_TAG(link) |
+		(regmap->i2c_buf->base + i2c->bsize * i);
+	i2c->rbuf = i2c->wbuf; /* + i2c->bsize / 2 */
+	i2c->regs = DDB_LINK_TAG(link) |
+		(regmap->i2c->base + regmap->i2c->size * i);
+	ddbwritel(dev, I2C_SPEED_100, i2c->regs + I2C_TIMING);
+	ddbwritel(dev, ((i2c->rbuf & 0xffff) << 16) | (i2c->wbuf & 0xffff),
+		i2c->regs + I2C_TASKADDRESS);
+	init_completion(&i2c->completion);
+
+	adap = &i2c->adap;
+	i2c_set_adapdata(adap, i2c);
+#ifdef I2C_ADAP_CLASS_TV_DIGITAL
+	adap->class = I2C_ADAP_CLASS_TV_DIGITAL|I2C_CLASS_TV_ANALOG;
+#else
+#ifdef I2C_CLASS_TV_ANALOG
+	adap->class = I2C_CLASS_TV_ANALOG;
+#endif
+#endif
+	snprintf(adap->name, I2C_NAME_SIZE, "ddbridge_%02x.%x.%x",
+		dev->nr, i2c->link, i);
+	adap->algo = &ddb_i2c_algo;
+	adap->algo_data = (void *)i2c;
+	adap->dev.parent = dev->dev;
+	return i2c_add_adapter(adap);
+}
+
+int ddb_i2c_init(struct ddb *dev)
+{
+	int stat = 0;
+	u32 i, j, num = 0, l, base;
+	struct ddb_i2c *i2c;
+	struct i2c_adapter *adap;
+	const struct ddb_regmap *regmap;
+
+	for (l = 0; l < DDB_MAX_LINK; l++) {
+		if (!dev->link[l].info)
+			continue;
+		regmap = dev->link[l].info->regmap;
+		if (!regmap || !regmap->i2c)
+			continue;
+		base = regmap->irq_base_i2c;
+		for (i = 0; i < regmap->i2c->num; i++) {
+			if (!(dev->link[l].info->i2c_mask & (1 << i)))
+				continue;
+			i2c = &dev->i2c[num];
+			dev->handler_data[l][i + base] = (unsigned long) i2c;
+			dev->handler[l][i + base] = i2c_handler;
+			stat = ddb_i2c_add(dev, i2c, regmap, l, i, num);
+			if (stat)
+				break;
+			num++;
+		}
+	}
+	if (stat) {
+		for (j = 0; j < num; j++) {
+			i2c = &dev->i2c[j];
+			adap = &i2c->adap;
+			i2c_del_adapter(adap);
+		}
+	} else
+		dev->i2c_num = num;
+	return stat;
+}
diff --git a/drivers/media/pci/ddbridge/ddbridge-i2c.h b/drivers/media/pci/ddbridge/ddbridge-i2c.h
new file mode 100644
index 000000000000..7ed220506c05
--- /dev/null
+++ b/drivers/media/pci/ddbridge/ddbridge-i2c.h
@@ -0,0 +1,112 @@
+/*
+ * ddbridge-i2c.c: Digital Devices bridge i2c driver
+ *
+ * Copyright (C) 2010-2017 Digital Devices GmbH
+ *                         Ralph Metzler <rjkm@metzlerbros.de>
+ *                         Marcus Metzler <mocm@metzlerbros.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 only, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#ifndef __DDBRIDGE_I2C_H__
+#define __DDBRIDGE_I2C_H__
+
+#include <linux/i2c.h>
+
+#include "ddbridge.h"
+
+/******************************************************************************/
+
+void ddb_i2c_release(struct ddb *dev);
+int ddb_i2c_init(struct ddb *dev);
+
+/******************************************************************************/
+
+static int __maybe_unused i2c_io(struct i2c_adapter *adapter, u8 adr,
+				 u8 *wbuf, u32 wlen, u8 *rbuf, u32 rlen)
+{
+	struct i2c_msg msgs[2] = { { .addr = adr,  .flags = 0,
+				     .buf  = wbuf, .len   = wlen },
+				   { .addr = adr,  .flags = I2C_M_RD,
+				     .buf  = rbuf, .len   = rlen } };
+
+	return (i2c_transfer(adapter, msgs, 2) == 2) ? 0 : -1;
+}
+
+static int __maybe_unused i2c_write(struct i2c_adapter *adap, u8 adr,
+				    u8 *data, int len)
+{
+	struct i2c_msg msg = { .addr = adr, .flags = 0,
+			       .buf = data, .len = len };
+
+	return (i2c_transfer(adap, &msg, 1) == 1) ? 0 : -1;
+}
+
+static int __maybe_unused i2c_read(struct i2c_adapter *adapter, u8 adr, u8 *val)
+{
+	struct i2c_msg msgs[1] = { { .addr = adr, .flags = I2C_M_RD,
+				     .buf  = val, .len   = 1 } };
+
+	return (i2c_transfer(adapter, msgs, 1) == 1) ? 0 : -1;
+}
+
+static int __maybe_unused i2c_read_regs(struct i2c_adapter *adapter,
+					u8 adr, u8 reg, u8 *val, u8 len)
+{
+	struct i2c_msg msgs[2] = { { .addr = adr,  .flags = 0,
+				     .buf  = &reg, .len   = 1 },
+				   { .addr = adr,  .flags = I2C_M_RD,
+				     .buf  = val,  .len   = len } };
+
+	return (i2c_transfer(adapter, msgs, 2) == 2) ? 0 : -1;
+}
+
+static int __maybe_unused i2c_read_regs16(struct i2c_adapter *adapter,
+					  u8 adr, u16 reg, u8 *val, u8 len)
+{
+	u8 msg[2] = { reg >> 8, reg & 0xff };
+	struct i2c_msg msgs[2] = { { .addr = adr, .flags = 0,
+				     .buf  = msg, .len   = 2 },
+				   { .addr = adr, .flags = I2C_M_RD,
+				     .buf  = val, .len   = len } };
+
+	return (i2c_transfer(adapter, msgs, 2) == 2) ? 0 : -1;
+}
+
+static int __maybe_unused i2c_write_reg16(struct i2c_adapter *adap,
+					  u8 adr, u16 reg, u8 val)
+{
+	u8 msg[3] = { reg >> 8, reg & 0xff, val };
+
+	return i2c_write(adap, adr, msg, 3);
+}
+
+static int __maybe_unused i2c_write_reg(struct i2c_adapter *adap,
+					u8 adr, u8 reg, u8 val)
+{
+	u8 msg[2] = { reg, val };
+
+	return i2c_write(adap, adr, msg, 2);
+}
+
+static int __maybe_unused i2c_read_reg16(struct i2c_adapter *adapter,
+					 u8 adr, u16 reg, u8 *val)
+{
+	return i2c_read_regs16(adapter, adr, reg, val, 1);
+}
+
+static int __maybe_unused i2c_read_reg(struct i2c_adapter *adapter,
+				       u8 adr, u8 reg, u8 *val)
+{
+	return i2c_read_regs(adapter, adr, reg, val, 1);
+}
+
+#endif /* __DDBRIDGE_I2C_H__ */
diff --git a/drivers/media/pci/ddbridge/ddbridge-io.h b/drivers/media/pci/ddbridge/ddbridge-io.h
new file mode 100644
index 000000000000..a4c6bbe09168
--- /dev/null
+++ b/drivers/media/pci/ddbridge/ddbridge-io.h
@@ -0,0 +1,71 @@
+/*
+ * ddbridge-io.h: Digital Devices bridge I/O inline functions
+ *
+ * Copyright (C) 2010-2017 Digital Devices GmbH
+ *                         Ralph Metzler <rjkm@metzlerbros.de>
+ *                         Marcus Metzler <mocm@metzlerbros.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 only, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#ifndef __DDBRIDGE_IO_H__
+#define __DDBRIDGE_IO_H__
+
+#include <linux/io.h>
+
+#include "ddbridge.h"
+
+/******************************************************************************/
+
+static inline u32 ddblreadl(struct ddb_link *link, u32 adr)
+{
+	return readl(link->dev->regs + adr);
+}
+
+static inline void ddblwritel(struct ddb_link *link, u32 val, u32 adr)
+{
+	writel(val, link->dev->regs + adr);
+}
+
+static inline u32 ddbreadl(struct ddb *dev, u32 adr)
+{
+	return readl(dev->regs + adr);
+}
+
+static inline void ddbwritel(struct ddb *dev, u32 val, u32 adr)
+{
+	writel(val, dev->regs + adr);
+}
+
+static inline void ddbcpyto(struct ddb *dev, u32 adr, void *src, long count)
+{
+	return memcpy_toio(dev->regs + adr, src, count);
+}
+
+static inline void ddbcpyfrom(struct ddb *dev, void *dst, u32 adr, long count)
+{
+	return memcpy_fromio(dst, dev->regs + adr, count);
+}
+
+static inline u32 safe_ddbreadl(struct ddb *dev, u32 adr)
+{
+	u32 val = ddbreadl(dev, adr);
+
+	/* (ddb)readl returns (uint)-1 (all bits set) on failure, catch that */
+	if (val == ~0) {
+		dev_err(&dev->pdev->dev, "ddbreadl failure, adr=%08x\n", adr);
+		return 0;
+	}
+
+	return val;
+}
+
+#endif /* __DDBRIDGE_IO_H__ */
diff --git a/drivers/media/pci/ddbridge/ddbridge-main.c b/drivers/media/pci/ddbridge/ddbridge-main.c
new file mode 100644
index 000000000000..ccac7fe31336
--- /dev/null
+++ b/drivers/media/pci/ddbridge/ddbridge-main.c
@@ -0,0 +1,346 @@
+/*
+ * ddbridge.c: Digital Devices PCIe bridge driver
+ *
+ * Copyright (C) 2010-2017 Digital Devices GmbH
+ *                         Ralph Metzler <rjkm@metzlerbros.de>
+ *                         Marcus Metzler <mocm@metzlerbros.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 only, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/delay.h>
+#include <linux/slab.h>
+#include <linux/poll.h>
+#include <linux/io.h>
+#include <linux/pci.h>
+#include <linux/pci_ids.h>
+#include <linux/timer.h>
+#include <linux/i2c.h>
+#include <linux/swab.h>
+#include <linux/vmalloc.h>
+
+#include "ddbridge.h"
+#include "ddbridge-i2c.h"
+#include "ddbridge-regs.h"
+#include "ddbridge-hw.h"
+#include "ddbridge-io.h"
+
+/****************************************************************************/
+/* module parameters */
+
+#ifdef CONFIG_PCI_MSI
+#ifdef CONFIG_DVB_DDBRIDGE_MSIENABLE
+static int msi = 1;
+#else
+static int msi;
+#endif
+module_param(msi, int, 0444);
+#ifdef CONFIG_DVB_DDBRIDGE_MSIENABLE
+MODULE_PARM_DESC(msi, "Control MSI interrupts: 0-disable, 1-enable (default)");
+#else
+MODULE_PARM_DESC(msi, "Control MSI interrupts: 0-disable (default), 1-enable");
+#endif
+#endif
+
+int ci_bitrate = 70000;
+module_param(ci_bitrate, int, 0444);
+MODULE_PARM_DESC(ci_bitrate, " Bitrate in KHz for output to CI.");
+
+int ts_loop = -1;
+module_param(ts_loop, int, 0444);
+MODULE_PARM_DESC(ts_loop, "TS in/out test loop on port ts_loop");
+
+int xo2_speed = 2;
+module_param(xo2_speed, int, 0444);
+MODULE_PARM_DESC(xo2_speed, "default transfer speed for xo2 based duoflex, 0=55,1=75,2=90,3=104 MBit/s, default=2, use attribute to change for individual cards");
+
+#ifdef __arm__
+int alt_dma = 1;
+#else
+int alt_dma;
+#endif
+module_param(alt_dma, int, 0444);
+MODULE_PARM_DESC(alt_dma, "use alternative DMA buffer handling");
+
+int no_init;
+module_param(no_init, int, 0444);
+MODULE_PARM_DESC(no_init, "do not initialize most devices");
+
+int stv0910_single;
+module_param(stv0910_single, int, 0444);
+MODULE_PARM_DESC(stv0910_single, "use stv0910 cards as single demods");
+
+/****************************************************************************/
+/****************************************************************************/
+/****************************************************************************/
+
+static void ddb_irq_disable(struct ddb *dev)
+{
+	ddbwritel(dev, 0, INTERRUPT_ENABLE);
+	ddbwritel(dev, 0, MSI1_ENABLE);
+}
+
+static void ddb_irq_exit(struct ddb *dev)
+{
+	ddb_irq_disable(dev);
+	if (dev->msi == 2)
+		free_irq(dev->pdev->irq + 1, dev);
+	free_irq(dev->pdev->irq, dev);
+#ifdef CONFIG_PCI_MSI
+	if (dev->msi)
+		pci_disable_msi(dev->pdev);
+#endif
+}
+
+static void ddb_remove(struct pci_dev *pdev)
+{
+	struct ddb *dev = (struct ddb *) pci_get_drvdata(pdev);
+
+	ddb_device_destroy(dev);
+	ddb_ports_detach(dev);
+	ddb_i2c_release(dev);
+
+	ddb_irq_exit(dev);
+	ddb_ports_release(dev);
+	ddb_buffers_free(dev);
+
+	ddb_unmap(dev);
+	pci_set_drvdata(pdev, NULL);
+	pci_disable_device(pdev);
+}
+
+#ifdef CONFIG_PCI_MSI
+static void ddb_irq_msi(struct ddb *dev, int nr)
+{
+	int stat;
+
+	if (msi && pci_msi_enabled()) {
+		stat = pci_alloc_irq_vectors(dev->pdev, 1, nr, PCI_IRQ_MSI);
+		if (stat >= 1) {
+			dev->msi = stat;
+			dev_info(dev->dev, "using %d MSI interrupt(s)\n",
+				dev->msi);
+		} else
+			dev_info(dev->dev, "MSI not available.\n");
+	}
+}
+#endif
+
+static int ddb_irq_init(struct ddb *dev)
+{
+	int stat;
+	int irq_flag = IRQF_SHARED;
+
+	ddbwritel(dev, 0x00000000, INTERRUPT_ENABLE);
+	ddbwritel(dev, 0x00000000, MSI1_ENABLE);
+	ddbwritel(dev, 0x00000000, MSI2_ENABLE);
+	ddbwritel(dev, 0x00000000, MSI3_ENABLE);
+	ddbwritel(dev, 0x00000000, MSI4_ENABLE);
+	ddbwritel(dev, 0x00000000, MSI5_ENABLE);
+	ddbwritel(dev, 0x00000000, MSI6_ENABLE);
+	ddbwritel(dev, 0x00000000, MSI7_ENABLE);
+
+#ifdef CONFIG_PCI_MSI
+	ddb_irq_msi(dev, 2);
+
+	if (dev->msi)
+		irq_flag = 0;
+	if (dev->msi == 2) {
+		stat = request_irq(dev->pdev->irq, ddb_irq_handler0,
+				   irq_flag, "ddbridge", (void *) dev);
+		if (stat < 0)
+			return stat;
+		stat = request_irq(dev->pdev->irq + 1, ddb_irq_handler1,
+				   irq_flag, "ddbridge", (void *) dev);
+		if (stat < 0) {
+			free_irq(dev->pdev->irq, dev);
+			return stat;
+		}
+	} else
+#endif
+	{
+		stat = request_irq(dev->pdev->irq, ddb_irq_handler,
+				   irq_flag, "ddbridge", (void *) dev);
+		if (stat < 0)
+			return stat;
+	}
+	if (dev->msi == 2) {
+		ddbwritel(dev, 0x0fffff00, INTERRUPT_ENABLE);
+		ddbwritel(dev, 0x0000000f, MSI1_ENABLE);
+	} else {
+		ddbwritel(dev, 0x0fffff0f, INTERRUPT_ENABLE);
+		ddbwritel(dev, 0x00000000, MSI1_ENABLE);
+	}
+	return stat;
+}
+
+static int ddb_probe(struct pci_dev *pdev,
+			       const struct pci_device_id *id)
+{
+	struct ddb *dev;
+	int stat = 0;
+
+	if (pci_enable_device(pdev) < 0)
+		return -ENODEV;
+
+	pci_set_master(pdev);
+
+	if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64)))
+		if (pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))
+			return -ENODEV;
+
+	dev = vzalloc(sizeof(struct ddb));
+	if (dev == NULL)
+		return -ENOMEM;
+
+	mutex_init(&dev->mutex);
+	dev->has_dma = 1;
+	dev->pdev = pdev;
+	dev->dev = &pdev->dev;
+	pci_set_drvdata(pdev, dev);
+
+	dev->link[0].ids.vendor = id->vendor;
+	dev->link[0].ids.device = id->device;
+	dev->link[0].ids.subvendor = id->subvendor;
+	dev->link[0].ids.subdevice = pdev->subsystem_device;
+
+	dev->link[0].dev = dev;
+	dev->link[0].info = get_ddb_info(id->vendor, id->device,
+					 id->subvendor, pdev->subsystem_device);
+
+	dev_info(&pdev->dev, "detected %s\n", dev->link[0].info->name);
+
+	dev->regs_len = pci_resource_len(dev->pdev, 0);
+	dev->regs = ioremap(pci_resource_start(dev->pdev, 0),
+			    pci_resource_len(dev->pdev, 0));
+
+	if (!dev->regs) {
+		dev_err(&pdev->dev, "not enough memory for register map\n");
+		stat = -ENOMEM;
+		goto fail;
+	}
+	if (ddbreadl(dev, 0) == 0xffffffff) {
+		dev_err(&pdev->dev, "cannot read registers\n");
+		stat = -ENODEV;
+		goto fail;
+	}
+
+	dev->link[0].ids.hwid = ddbreadl(dev, 0);
+	dev->link[0].ids.regmapid = ddbreadl(dev, 4);
+
+	dev_info(&pdev->dev, "HW %08x REGMAP %08x\n",
+		dev->link[0].ids.hwid, dev->link[0].ids.regmapid);
+
+	ddbwritel(dev, 0, DMA_BASE_READ);
+	ddbwritel(dev, 0, DMA_BASE_WRITE);
+
+	stat = ddb_irq_init(dev);
+	if (stat < 0)
+		goto fail0;
+
+	if (ddb_init(dev) == 0)
+		return 0;
+
+	ddb_irq_exit(dev);
+fail0:
+	dev_err(&pdev->dev, "fail0\n");
+	if (dev->msi)
+		pci_disable_msi(dev->pdev);
+fail:
+	dev_err(&pdev->dev, "fail\n");
+
+	ddb_unmap(dev);
+	pci_set_drvdata(pdev, NULL);
+	pci_disable_device(pdev);
+	return -1;
+}
+
+/****************************************************************************/
+/****************************************************************************/
+/****************************************************************************/
+
+#define DDB_DEVICE_ANY(_device) \
+		{ PCI_DEVICE_SUB(DDVID, _device, DDVID, PCI_ANY_ID) }
+
+static const struct pci_device_id ddb_id_table[] = {
+	DDB_DEVICE_ANY(0x0002),
+	DDB_DEVICE_ANY(0x0003),
+	DDB_DEVICE_ANY(0x0005),
+	DDB_DEVICE_ANY(0x0006),
+	DDB_DEVICE_ANY(0x0007),
+	DDB_DEVICE_ANY(0x0008),
+	DDB_DEVICE_ANY(0x0011),
+	DDB_DEVICE_ANY(0x0012),
+	DDB_DEVICE_ANY(0x0013),
+	DDB_DEVICE_ANY(0x0201),
+	DDB_DEVICE_ANY(0x0203),
+	DDB_DEVICE_ANY(0x0210),
+	DDB_DEVICE_ANY(0x0220),
+	DDB_DEVICE_ANY(0x0320),
+	DDB_DEVICE_ANY(0x0321),
+	DDB_DEVICE_ANY(0x0322),
+	DDB_DEVICE_ANY(0x0323),
+	DDB_DEVICE_ANY(0x0328),
+	DDB_DEVICE_ANY(0x0329),
+	{0}
+};
+
+MODULE_DEVICE_TABLE(pci, ddb_id_table);
+
+static struct pci_driver ddb_pci_driver = {
+	.name        = "ddbridge",
+	.id_table    = ddb_id_table,
+	.probe       = ddb_probe,
+	.remove      = ddb_remove,
+};
+
+static __init int module_init_ddbridge(void)
+{
+	int stat = -1;
+
+	pr_info("Digital Devices PCIE bridge driver "
+		DDBRIDGE_VERSION
+		", Copyright (C) 2010-17 Digital Devices GmbH\n");
+	if (ddb_class_create() < 0)
+		return -1;
+	ddb_wq = create_workqueue("ddbridge");
+	if (ddb_wq == NULL)
+		goto exit1;
+	stat = pci_register_driver(&ddb_pci_driver);
+	if (stat < 0)
+		goto exit2;
+	return stat;
+exit2:
+	destroy_workqueue(ddb_wq);
+exit1:
+	ddb_class_destroy();
+	return stat;
+}
+
+static __exit void module_exit_ddbridge(void)
+{
+	pci_unregister_driver(&ddb_pci_driver);
+	destroy_workqueue(ddb_wq);
+	ddb_class_destroy();
+}
+
+module_init(module_init_ddbridge);
+module_exit(module_exit_ddbridge);
+
+MODULE_DESCRIPTION("Digital Devices PCIe Bridge");
+MODULE_AUTHOR("Ralph and Marcus Metzler, Metzler Brothers Systementwicklung GbR");
+MODULE_LICENSE("GPL");
+MODULE_VERSION(DDBRIDGE_VERSION);
diff --git a/drivers/media/pci/ddbridge/ddbridge-maxs8.c b/drivers/media/pci/ddbridge/ddbridge-maxs8.c
new file mode 100644
index 000000000000..f8a53bc7c86c
--- /dev/null
+++ b/drivers/media/pci/ddbridge/ddbridge-maxs8.c
@@ -0,0 +1,444 @@
+/*
+ * ddbridge-maxs8.c: Digital Devices bridge MaxS4/8 support
+ *
+ * Copyright (C) 2010-2017 Digital Devices GmbH
+ *                         Ralph Metzler <rjkm@metzlerbros.de>
+ *                         Marcus Metzler <mocm@metzlerbros.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 only, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/delay.h>
+#include <linux/slab.h>
+#include <linux/poll.h>
+#include <linux/io.h>
+#include <linux/pci.h>
+#include <linux/pci_ids.h>
+#include <linux/timer.h>
+#include <linux/i2c.h>
+#include <linux/swab.h>
+#include <linux/vmalloc.h>
+
+#include "ddbridge.h"
+#include "ddbridge-regs.h"
+#include "ddbridge-io.h"
+
+#include "ddbridge-maxs8.h"
+#include "mxl5xx.h"
+
+/******************************************************************************/
+
+/* MaxS4/8 related modparams */
+static int fmode;
+module_param(fmode, int, 0444);
+MODULE_PARM_DESC(fmode, "frontend emulation mode");
+
+static int fmode_sat = -1;
+module_param(fmode_sat, int, 0444);
+MODULE_PARM_DESC(fmode_sat, "set frontend emulation mode sat");
+
+static int old_quattro;
+module_param(old_quattro, int, 0444);
+MODULE_PARM_DESC(old_quattro, "old quattro LNB input order ");
+
+/******************************************************************************/
+
+static int lnb_command(struct ddb *dev, u32 link, u32 lnb, u32 cmd)
+{
+	u32 c, v = 0, tag = DDB_LINK_TAG(link);
+
+	v = LNB_TONE & (dev->link[link].lnb.tone << (15 - lnb));
+	ddbwritel(dev, cmd | v, tag | LNB_CONTROL(lnb));
+	for (c = 0; c < 10; c++) {
+		v = ddbreadl(dev, tag | LNB_CONTROL(lnb));
+		if ((v & LNB_BUSY) == 0)
+			break;
+		msleep(20);
+	}
+	if (c == 10)
+		dev_info(dev->dev, "%s lnb = %08x  cmd = %08x\n",
+			__func__, lnb, cmd);
+	return 0;
+}
+
+static int max_send_master_cmd(struct dvb_frontend *fe,
+			       struct dvb_diseqc_master_cmd *cmd)
+{
+	struct ddb_input *input = fe->sec_priv;
+	struct ddb_port *port = input->port;
+	struct ddb *dev = port->dev;
+	struct ddb_dvb *dvb = &port->dvb[input->nr & 1];
+	u32 tag = DDB_LINK_TAG(port->lnr);
+	int i;
+	u32 fmode = dev->link[port->lnr].lnb.fmode;
+
+	if (fmode == 2 || fmode == 1)
+		return 0;
+	if (dvb->diseqc_send_master_cmd)
+		dvb->diseqc_send_master_cmd(fe, cmd);
+
+	mutex_lock(&dev->link[port->lnr].lnb.lock);
+	ddbwritel(dev, 0, tag | LNB_BUF_LEVEL(dvb->input));
+	for (i = 0; i < cmd->msg_len; i++)
+		ddbwritel(dev, cmd->msg[i], tag | LNB_BUF_WRITE(dvb->input));
+	lnb_command(dev, port->lnr, dvb->input, LNB_CMD_DISEQC);
+	mutex_unlock(&dev->link[port->lnr].lnb.lock);
+	return 0;
+}
+
+static int lnb_send_diseqc(struct ddb *dev, u32 link, u32 input,
+			   struct dvb_diseqc_master_cmd *cmd)
+{
+	u32 tag = DDB_LINK_TAG(link);
+	int i;
+
+	ddbwritel(dev, 0, tag | LNB_BUF_LEVEL(input));
+	for (i = 0; i < cmd->msg_len; i++)
+		ddbwritel(dev, cmd->msg[i], tag | LNB_BUF_WRITE(input));
+	lnb_command(dev, link, input, LNB_CMD_DISEQC);
+	return 0;
+}
+
+static int lnb_set_sat(struct ddb *dev, u32 link, u32 input, u32 sat, u32 band,
+		       u32 hor)
+{
+	struct dvb_diseqc_master_cmd cmd = {
+		.msg = {0xe0, 0x10, 0x38, 0xf0, 0x00, 0x00},
+		.msg_len = 4
+	};
+	cmd.msg[3] = 0xf0 | (((sat << 2) & 0x0c) | (band ? 1 : 0) |
+		(hor ? 2 : 0));
+	return lnb_send_diseqc(dev, link, input, &cmd);
+}
+
+static int lnb_set_tone(struct ddb *dev, u32 link, u32 input,
+	enum fe_sec_tone_mode tone)
+{
+	int s = 0;
+	u32 mask = (1ULL << input);
+
+	switch (tone) {
+	case SEC_TONE_OFF:
+		if (!(dev->link[link].lnb.tone & mask))
+			return 0;
+		dev->link[link].lnb.tone &= ~(1ULL << input);
+		break;
+	case SEC_TONE_ON:
+		if (dev->link[link].lnb.tone & mask)
+			return 0;
+		dev->link[link].lnb.tone |= (1ULL << input);
+		break;
+	default:
+		s = -EINVAL;
+		break;
+	}
+	if (!s)
+		s = lnb_command(dev, link, input, LNB_CMD_NOP);
+	return s;
+}
+
+static int lnb_set_voltage(struct ddb *dev, u32 link, u32 input,
+	enum fe_sec_voltage voltage)
+{
+	int s = 0;
+
+	if (dev->link[link].lnb.oldvoltage[input] == voltage)
+		return 0;
+	switch (voltage) {
+	case SEC_VOLTAGE_OFF:
+		if (dev->link[link].lnb.voltage[input])
+			return 0;
+		lnb_command(dev, link, input, LNB_CMD_OFF);
+		break;
+	case SEC_VOLTAGE_13:
+		lnb_command(dev, link, input, LNB_CMD_LOW);
+		break;
+	case SEC_VOLTAGE_18:
+		lnb_command(dev, link, input, LNB_CMD_HIGH);
+		break;
+	default:
+		s = -EINVAL;
+		break;
+	}
+	dev->link[link].lnb.oldvoltage[input] = voltage;
+	return s;
+}
+
+static int max_set_input_unlocked(struct dvb_frontend *fe, int in)
+{
+	struct ddb_input *input = fe->sec_priv;
+	struct ddb_port *port = input->port;
+	struct ddb *dev = port->dev;
+	struct ddb_dvb *dvb = &port->dvb[input->nr & 1];
+	int res = 0;
+
+	if (in > 3)
+		return -EINVAL;
+	if (dvb->input != in) {
+		u32 bit = (1ULL << input->nr);
+		u32 obit =
+			dev->link[port->lnr].lnb.voltage[dvb->input & 3] & bit;
+
+		dev->link[port->lnr].lnb.voltage[dvb->input & 3] &= ~bit;
+		dvb->input = in;
+		dev->link[port->lnr].lnb.voltage[dvb->input & 3] |= obit;
+	}
+	res = dvb->set_input(fe, in);
+	return res;
+}
+
+static int max_set_tone(struct dvb_frontend *fe, enum fe_sec_tone_mode tone)
+{
+	struct ddb_input *input = fe->sec_priv;
+	struct ddb_port *port = input->port;
+	struct ddb *dev = port->dev;
+	struct ddb_dvb *dvb = &port->dvb[input->nr & 1];
+	int tuner = 0;
+	int res = 0;
+	u32 fmode = dev->link[port->lnr].lnb.fmode;
+
+	mutex_lock(&dev->link[port->lnr].lnb.lock);
+	dvb->tone = tone;
+	switch (fmode) {
+	default:
+	case 0:
+	case 3:
+		res = lnb_set_tone(dev, port->lnr, dvb->input, tone);
+		break;
+	case 1:
+	case 2:
+		if (old_quattro) {
+			if (dvb->tone == SEC_TONE_ON)
+				tuner |= 2;
+			if (dvb->voltage == SEC_VOLTAGE_18)
+				tuner |= 1;
+		} else {
+			if (dvb->tone == SEC_TONE_ON)
+				tuner |= 1;
+			if (dvb->voltage == SEC_VOLTAGE_18)
+				tuner |= 2;
+		}
+		res = max_set_input_unlocked(fe, tuner);
+		break;
+	}
+	mutex_unlock(&dev->link[port->lnr].lnb.lock);
+	return res;
+}
+
+static int max_set_voltage(struct dvb_frontend *fe, enum fe_sec_voltage voltage)
+{
+	struct ddb_input *input = fe->sec_priv;
+	struct ddb_port *port = input->port;
+	struct ddb *dev = port->dev;
+	struct ddb_dvb *dvb = &port->dvb[input->nr & 1];
+	int tuner = 0;
+	u32 nv, ov = dev->link[port->lnr].lnb.voltages;
+	int res = 0;
+	u32 fmode = dev->link[port->lnr].lnb.fmode;
+
+	mutex_lock(&dev->link[port->lnr].lnb.lock);
+	dvb->voltage = voltage;
+
+	switch (fmode) {
+	case 3:
+	default:
+	case 0:
+		if (fmode == 3)
+			max_set_input_unlocked(fe, 0);
+		if (voltage == SEC_VOLTAGE_OFF)
+			dev->link[port->lnr].lnb.voltage[dvb->input] &=
+				~(1ULL << input->nr);
+		else
+			dev->link[port->lnr].lnb.voltage[dvb->input] |=
+				(1ULL << input->nr);
+
+		res = lnb_set_voltage(dev, port->lnr, dvb->input, voltage);
+		break;
+	case 1:
+	case 2:
+		if (voltage == SEC_VOLTAGE_OFF)
+			dev->link[port->lnr].lnb.voltages &=
+				~(1ULL << input->nr);
+		else
+			dev->link[port->lnr].lnb.voltages |=
+				(1ULL << input->nr);
+
+		nv = dev->link[port->lnr].lnb.voltages;
+
+		if (old_quattro) {
+			if (dvb->tone == SEC_TONE_ON)
+				tuner |= 2;
+			if (dvb->voltage == SEC_VOLTAGE_18)
+				tuner |= 1;
+		} else {
+			if (dvb->tone == SEC_TONE_ON)
+				tuner |= 1;
+			if (dvb->voltage == SEC_VOLTAGE_18)
+				tuner |= 2;
+		}
+		res = max_set_input_unlocked(fe, tuner);
+
+		if (nv != ov) {
+			if (nv) {
+				lnb_set_voltage(dev,
+					port->lnr, 0, SEC_VOLTAGE_13);
+				if (fmode == 1) {
+					lnb_set_voltage(dev, port->lnr,
+						0, SEC_VOLTAGE_13);
+					if (old_quattro) {
+						lnb_set_voltage(dev, port->lnr,
+							1, SEC_VOLTAGE_18);
+						lnb_set_voltage(dev, port->lnr,
+							2, SEC_VOLTAGE_13);
+					} else {
+						lnb_set_voltage(dev, port->lnr,
+							1, SEC_VOLTAGE_13);
+						lnb_set_voltage(dev, port->lnr,
+							2, SEC_VOLTAGE_18);
+					}
+					lnb_set_voltage(dev, port->lnr,
+						3, SEC_VOLTAGE_18);
+				}
+			} else {
+				lnb_set_voltage(dev, port->lnr,
+					0, SEC_VOLTAGE_OFF);
+				if (fmode == 1) {
+					lnb_set_voltage(dev, port->lnr,
+						1, SEC_VOLTAGE_OFF);
+					lnb_set_voltage(dev, port->lnr,
+						2, SEC_VOLTAGE_OFF);
+					lnb_set_voltage(dev, port->lnr,
+						3, SEC_VOLTAGE_OFF);
+				}
+			}
+		}
+		break;
+	}
+	mutex_unlock(&dev->link[port->lnr].lnb.lock);
+	return res;
+}
+
+static int max_enable_high_lnb_voltage(struct dvb_frontend *fe, long arg)
+{
+
+	return 0;
+}
+
+static int max_send_burst(struct dvb_frontend *fe, enum fe_sec_mini_cmd burst)
+{
+	return 0;
+}
+
+static int mxl_fw_read(void *priv, u8 *buf, u32 len)
+{
+	struct ddb_link *link = priv;
+	struct ddb *dev = link->dev;
+
+	dev_info(dev->dev, "Read mxl_fw from link %u\n", link->nr);
+
+	return ddbridge_flashread(dev, link->nr, buf, 0xc0000, len);
+}
+
+int lnb_init_fmode(struct ddb *dev, struct ddb_link *link, u32 fm)
+{
+	u32 l = link->nr;
+
+	if (link->lnb.fmode == fm)
+		return 0;
+	dev_info(dev->dev, "Set fmode link %u = %u\n", l, fm);
+	mutex_lock(&link->lnb.lock);
+	if (fm == 2 || fm == 1) {
+		if (fmode_sat >= 0) {
+			lnb_set_sat(dev, l, 0, fmode_sat, 0, 0);
+			if (old_quattro) {
+				lnb_set_sat(dev, l, 1, fmode_sat, 0, 1);
+				lnb_set_sat(dev, l, 2, fmode_sat, 1, 0);
+			} else {
+				lnb_set_sat(dev, l, 1, fmode_sat, 1, 0);
+				lnb_set_sat(dev, l, 2, fmode_sat, 0, 1);
+			}
+			lnb_set_sat(dev, l, 3, fmode_sat, 1, 1);
+		}
+		lnb_set_tone(dev, l, 0, SEC_TONE_OFF);
+		if (old_quattro) {
+			lnb_set_tone(dev, l, 1, SEC_TONE_OFF);
+			lnb_set_tone(dev, l, 2, SEC_TONE_ON);
+		} else {
+			lnb_set_tone(dev, l, 1, SEC_TONE_ON);
+			lnb_set_tone(dev, l, 2, SEC_TONE_OFF);
+		}
+		lnb_set_tone(dev, l, 3, SEC_TONE_ON);
+	}
+	link->lnb.fmode = fm;
+	mutex_unlock(&link->lnb.lock);
+	return 0;
+}
+
+static struct mxl5xx_cfg mxl5xx = {
+	.adr      = 0x60,
+	.type     = 0x01,
+	.clk      = 27000000,
+	.ts_clk   = 139,
+	.cap      = 12,
+	.fw_read  = mxl_fw_read,
+};
+
+int fe_attach_mxl5xx(struct ddb_input *input)
+{
+	struct ddb *dev = input->port->dev;
+	struct i2c_adapter *i2c = &input->port->i2c->adap;
+	struct ddb_dvb *dvb = &input->port->dvb[input->nr & 1];
+	struct ddb_port *port = input->port;
+	struct ddb_link *link = &dev->link[port->lnr];
+	struct mxl5xx_cfg cfg;
+	int demod, tuner;
+
+	cfg = mxl5xx;
+	cfg.fw_priv = link;
+	dvb->set_input = NULL;
+
+	demod = input->nr;
+	tuner = demod & 3;
+	if (fmode == 3)
+		tuner = 0;
+
+	dvb->fe = dvb_attach(mxl5xx_attach, i2c, &cfg,
+		demod, tuner, &dvb->set_input);
+
+	if (!dvb->fe) {
+		dev_err(dev->dev, "No MXL5XX found!\n");
+		return -ENODEV;
+	}
+
+	if (!dvb->set_input) {
+		dev_err(dev->dev, "No mxl5xx_set_input function pointer!\n");
+		return -ENODEV;
+	}
+
+	if (input->nr < 4) {
+		lnb_command(dev, port->lnr, input->nr, LNB_CMD_INIT);
+		lnb_set_voltage(dev, port->lnr, input->nr, SEC_VOLTAGE_OFF);
+	}
+	lnb_init_fmode(dev, link, fmode);
+
+	dvb->fe->ops.set_voltage = max_set_voltage;
+	dvb->fe->ops.enable_high_lnb_voltage = max_enable_high_lnb_voltage;
+	dvb->fe->ops.set_tone = max_set_tone;
+	dvb->diseqc_send_master_cmd = dvb->fe->ops.diseqc_send_master_cmd;
+	dvb->fe->ops.diseqc_send_master_cmd = max_send_master_cmd;
+	dvb->fe->ops.diseqc_send_burst = max_send_burst;
+	dvb->fe->sec_priv = input;
+	dvb->input = tuner;
+	return 0;
+}
diff --git a/drivers/media/pci/ddbridge/ddbridge-maxs8.h b/drivers/media/pci/ddbridge/ddbridge-maxs8.h
new file mode 100644
index 000000000000..bb8884811a46
--- /dev/null
+++ b/drivers/media/pci/ddbridge/ddbridge-maxs8.h
@@ -0,0 +1,29 @@
+/*
+ * ddbridge-maxs8.h: Digital Devices bridge MaxS4/8 support
+ *
+ * Copyright (C) 2010-2017 Digital Devices GmbH
+ *                         Ralph Metzler <rjkm@metzlerbros.de>
+ *                         Marcus Metzler <mocm@metzlerbros.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 only, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#ifndef _DDBRIDGE_MAXS8_H_
+#define _DDBRIDGE_MAXS8_H_
+
+#include "ddbridge.h"
+
+/******************************************************************************/
+
+int lnb_init_fmode(struct ddb *dev, struct ddb_link *link, u32 fm);
+int fe_attach_mxl5xx(struct ddb_input *input);
+
+#endif /* _DDBRIDGE_MAXS8_H */
diff --git a/drivers/media/pci/ddbridge/ddbridge-regs.h b/drivers/media/pci/ddbridge/ddbridge-regs.h
index 98cebb97d64f..9d44f8d3af75 100644
--- a/drivers/media/pci/ddbridge/ddbridge-regs.h
+++ b/drivers/media/pci/ddbridge/ddbridge-regs.h
@@ -1,7 +1,7 @@
 /*
  * ddbridge-regs.h: Digital Devices PCIe bridge driver
  *
- * Copyright (C) 2010-2011 Digital Devices GmbH
+ * Copyright (C) 2010-2017 Digital Devices GmbH
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,20 +17,26 @@
  * http://www.gnu.org/copyleft/gpl.html
  */
 
-/* DD-DVBBridgeV1.h 273 2010-09-17 05:03:16Z manfred */
+/* ------------------------------------------------------------------------- */
+/* SPI Controller */
 
-/* Register Definitions */
+#define SPI_CONTROL     0x10
+#define SPI_DATA        0x14
 
-#define CUR_REGISTERMAP_VERSION 0x10000
+/* ------------------------------------------------------------------------- */
+/* GPIO */
 
-#define HARDWARE_VERSION       0x00
-#define REGISTERMAP_VERSION    0x04
+#define GPIO_OUTPUT      0x20
+#define GPIO_INPUT       0x24
+#define GPIO_DIRECTION   0x28
 
 /* ------------------------------------------------------------------------- */
-/* SPI Controller */
+/* MDIO */
 
-#define SPI_CONTROL     0x10
-#define SPI_DATA        0x14
+#define MDIO_CTRL        0x20
+#define MDIO_ADR         0x24
+#define MDIO_REG         0x28
+#define MDIO_VAL         0x2C
 
 /* ------------------------------------------------------------------------- */
 
@@ -38,14 +44,14 @@
 
 /* ------------------------------------------------------------------------- */
 
-/* Interrupt controller                                     */
-/* How many MSI's are available depends on HW (Min 2 max 8) */
-/* How many are usable also depends on Host platform        */
+/* Interrupt controller
+ * How many MSI's are available depends on HW (Min 2 max 8)
+ * How many are usable also depends on Host platform
+ */
 
 #define INTERRUPT_BASE   (0x40)
 
 #define INTERRUPT_ENABLE (INTERRUPT_BASE + 0x00)
-#define MSI0_ENABLE      (INTERRUPT_BASE + 0x00)
 #define MSI1_ENABLE      (INTERRUPT_BASE + 0x04)
 #define MSI2_ENABLE      (INTERRUPT_BASE + 0x08)
 #define MSI3_ENABLE      (INTERRUPT_BASE + 0x0C)
@@ -57,59 +63,31 @@
 #define INTERRUPT_STATUS (INTERRUPT_BASE + 0x20)
 #define INTERRUPT_ACK    (INTERRUPT_BASE + 0x20)
 
-#define INTMASK_I2C1        (0x00000001)
-#define INTMASK_I2C2        (0x00000002)
-#define INTMASK_I2C3        (0x00000004)
-#define INTMASK_I2C4        (0x00000008)
-
-#define INTMASK_CIRQ1       (0x00000010)
-#define INTMASK_CIRQ2       (0x00000020)
-#define INTMASK_CIRQ3       (0x00000040)
-#define INTMASK_CIRQ4       (0x00000080)
-
-#define INTMASK_TSINPUT1    (0x00000100)
-#define INTMASK_TSINPUT2    (0x00000200)
-#define INTMASK_TSINPUT3    (0x00000400)
-#define INTMASK_TSINPUT4    (0x00000800)
-#define INTMASK_TSINPUT5    (0x00001000)
-#define INTMASK_TSINPUT6    (0x00002000)
-#define INTMASK_TSINPUT7    (0x00004000)
-#define INTMASK_TSINPUT8    (0x00008000)
-
-#define INTMASK_TSOUTPUT1   (0x00010000)
-#define INTMASK_TSOUTPUT2   (0x00020000)
-#define INTMASK_TSOUTPUT3   (0x00040000)
-#define INTMASK_TSOUTPUT4   (0x00080000)
+/* Temperature Monitor ( 2x LM75A @ 0x90,0x92 I2c ) */
+#define TEMPMON_BASE			(0x1c0)
+#define TEMPMON_CONTROL			(TEMPMON_BASE + 0x00)
+
+#define TEMPMON_CONTROL_AUTOSCAN	(0x00000002)
+#define TEMPMON_CONTROL_INTENABLE	(0x00000004)
+#define TEMPMON_CONTROL_OVERTEMP	(0x00008000)
+
+/* SHORT Temperature in Celsius x 256 */
+#define TEMPMON_SENSOR0			(TEMPMON_BASE + 0x04)
+#define TEMPMON_SENSOR1			(TEMPMON_BASE + 0x08)
+
+#define TEMPMON_FANCONTROL		(TEMPMON_BASE + 0x10)
 
 /* ------------------------------------------------------------------------- */
 /* I2C Master Controller */
 
-#define I2C_BASE        (0x80)  /* Byte offset */
-
 #define I2C_COMMAND     (0x00)
 #define I2C_TIMING      (0x04)
 #define I2C_TASKLENGTH  (0x08)     /* High read, low write */
 #define I2C_TASKADDRESS (0x0C)     /* High read, low write */
-
 #define I2C_MONITOR     (0x1C)
 
-#define I2C_BASE_1      (I2C_BASE + 0x00)
-#define I2C_BASE_2      (I2C_BASE + 0x20)
-#define I2C_BASE_3      (I2C_BASE + 0x40)
-#define I2C_BASE_4      (I2C_BASE + 0x60)
-
-#define I2C_BASE_N(i)   (I2C_BASE + (i) * 0x20)
-
-#define I2C_TASKMEM_BASE    (0x1000)    /* Byte offset */
-#define I2C_TASKMEM_SIZE    (0x1000)
-
 #define I2C_SPEED_400   (0x04030404)
-#define I2C_SPEED_200   (0x09080909)
-#define I2C_SPEED_154   (0x0C0B0C0C)
 #define I2C_SPEED_100   (0x13121313)
-#define I2C_SPEED_77    (0x19181919)
-#define I2C_SPEED_50    (0x27262727)
-
 
 /* ------------------------------------------------------------------------- */
 /* DMA  Controller */
@@ -117,35 +95,62 @@
 #define DMA_BASE_WRITE        (0x100)
 #define DMA_BASE_READ         (0x140)
 
-#define DMA_CONTROL     (0x00)                  /* 64 */
-#define DMA_ERROR       (0x04)                  /* 65 ( only read instance ) */
-
-#define DMA_DIAG_CONTROL                (0x1C)  /* 71 */
-#define DMA_DIAG_PACKETCOUNTER_LOW      (0x20)  /* 72 */
-#define DMA_DIAG_PACKETCOUNTER_HIGH     (0x24)  /* 73 */
-#define DMA_DIAG_TIMECOUNTER_LOW        (0x28)  /* 74 */
-#define DMA_DIAG_TIMECOUNTER_HIGH       (0x2C)  /* 75 */
-#define DMA_DIAG_RECHECKCOUNTER         (0x30)  /* 76  ( Split completions on read ) */
-#define DMA_DIAG_WAITTIMEOUTINIT        (0x34)  /* 77 */
-#define DMA_DIAG_WAITOVERFLOWCOUNTER    (0x38)  /* 78 */
-#define DMA_DIAG_WAITCOUNTER            (0x3C)  /* 79 */
+#define TS_CONTROL(_io)         (_io->regs + 0x00)
+#define TS_CONTROL2(_io)        (_io->regs + 0x04)
 
 /* ------------------------------------------------------------------------- */
 /* DMA  Buffer */
 
-#define TS_INPUT_BASE       (0x200)
-#define TS_INPUT_CONTROL(i)         (TS_INPUT_BASE + (i) * 16 + 0x00)
+#define DMA_BUFFER_CONTROL(_dma)       (_dma->regs + 0x00)
+#define DMA_BUFFER_ACK(_dma)           (_dma->regs + 0x04)
+#define DMA_BUFFER_CURRENT(_dma)       (_dma->regs + 0x08)
+#define DMA_BUFFER_SIZE(_dma)          (_dma->regs + 0x0c)
+
+/* ------------------------------------------------------------------------- */
+/* CI Interface (only CI-Bridge) */
+
+#define CI_BASE                     (0x400)
+#define CI_CONTROL(i)               (CI_BASE + (i) * 32 + 0x00)
+
+#define CI_DO_ATTRIBUTE_RW(i)       (CI_BASE + (i) * 32 + 0x04)
+#define CI_DO_IO_RW(i)              (CI_BASE + (i) * 32 + 0x08)
+#define CI_READDATA(i)              (CI_BASE + (i) * 32 + 0x0c)
+#define CI_DO_READ_ATTRIBUTES(i)    (CI_BASE + (i) * 32 + 0x10)
+
+#define CI_RESET_CAM                    (0x00000001)
+#define CI_POWER_ON                     (0x00000002)
+#define CI_ENABLE                       (0x00000004)
+#define CI_BYPASS_DISABLE               (0x00000010)
+
+#define CI_CAM_READY                    (0x00010000)
+#define CI_CAM_DETECT                   (0x00020000)
+#define CI_READY                        (0x80000000)
+
+#define CI_READ_CMD                     (0x40000000)
+#define CI_WRITE_CMD                    (0x80000000)
+
+#define CI_BUFFER_BASE                  (0x3000)
+#define CI_BUFFER_SIZE                  (0x0800)
+
+#define CI_BUFFER(i)                  (CI_BUFFER_BASE + (i) * CI_BUFFER_SIZE)
+
+/* ------------------------------------------------------------------------- */
+/* LNB commands (mxl5xx / Max S8) */
 
-#define TS_OUTPUT_BASE       (0x280)
-#define TS_OUTPUT_CONTROL(i)         (TS_OUTPUT_BASE + (i) * 16 + 0x00)
+#define LNB_BASE			(0x400)
+#define LNB_CONTROL(i)			(LNB_BASE + (i) * 0x20 + 0x00)
 
-#define DMA_BUFFER_BASE     (0x300)
+#define LNB_CMD				(7ULL <<  0)
+#define LNB_CMD_NOP			0
+#define LNB_CMD_INIT			1
+#define LNB_CMD_LOW			3
+#define LNB_CMD_HIGH			4
+#define LNB_CMD_OFF			5
+#define LNB_CMD_DISEQC			6
 
-#define DMA_BUFFER_CONTROL(i)       (DMA_BUFFER_BASE + (i) * 16 + 0x00)
-#define DMA_BUFFER_ACK(i)           (DMA_BUFFER_BASE + (i) * 16 + 0x04)
-#define DMA_BUFFER_CURRENT(i)       (DMA_BUFFER_BASE + (i) * 16 + 0x08)
-#define DMA_BUFFER_SIZE(i)          (DMA_BUFFER_BASE + (i) * 16 + 0x0c)
+#define LNB_BUSY			(1ULL <<  4)
+#define LNB_TONE			(1ULL << 15)
 
-#define DMA_BASE_ADDRESS_TABLE  (0x2000)
-#define DMA_BASE_ADDRESS_TABLE_ENTRIES (512)
+#define LNB_BUF_LEVEL(i)		(LNB_BASE + (i) * 0x20 + 0x10)
+#define LNB_BUF_WRITE(i)		(LNB_BASE + (i) * 0x20 + 0x14)
 
diff --git a/drivers/media/pci/ddbridge/ddbridge.h b/drivers/media/pci/ddbridge/ddbridge.h
index 4a0e3283d646..e9afa96bd9df 100644
--- a/drivers/media/pci/ddbridge/ddbridge.h
+++ b/drivers/media/pci/ddbridge/ddbridge.h
@@ -1,7 +1,8 @@
 /*
  * ddbridge.h: Digital Devices PCIe bridge driver
  *
- * Copyright (C) 2010-2011 Digital Devices GmbH
+ * Copyright (C) 2010-2017 Digital Devices GmbH
+ *                         Ralph Metzler <rmetzler@digitaldevices.de>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -20,15 +21,39 @@
 #ifndef _DDBRIDGE_H_
 #define _DDBRIDGE_H_
 
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/delay.h>
+#include <linux/slab.h>
+#include <linux/poll.h>
+#include <linux/io.h>
+#include <linux/pci.h>
+#include <linux/timer.h>
+#include <linux/i2c.h>
+#include <linux/swab.h>
+#include <linux/vmalloc.h>
+#include <linux/workqueue.h>
+#include <linux/kthread.h>
+#include <linux/platform_device.h>
+#include <linux/clk.h>
+#include <linux/spi/spi.h>
+#include <linux/gpio.h>
+#include <linux/completion.h>
+
 #include <linux/types.h>
 #include <linux/sched.h>
 #include <linux/interrupt.h>
-#include <linux/i2c.h>
 #include <linux/mutex.h>
 #include <asm/dma.h>
-#include <linux/dvb/frontend.h>
+#include <asm/irq.h>
+#include <linux/io.h>
+#include <linux/uaccess.h>
+
 #include <linux/dvb/ca.h>
 #include <linux/socket.h>
+#include <linux/device.h>
+#include <linux/io.h>
 
 #include "dmxdev.h"
 #include "dvbdev.h"
@@ -37,69 +62,122 @@
 #include "dvb_ringbuffer.h"
 #include "dvb_ca_en50221.h"
 #include "dvb_net.h"
-#include "cxd2099.h"
 
-#define DDB_MAX_I2C     4
-#define DDB_MAX_PORT    4
-#define DDB_MAX_INPUT   8
-#define DDB_MAX_OUTPUT  4
+#define DDBRIDGE_VERSION "0.9.31intermediate-integrated"
+
+#define DDB_MAX_I2C    32
+#define DDB_MAX_PORT   32
+#define DDB_MAX_INPUT  64
+#define DDB_MAX_OUTPUT 32
 #define DDB_MAX_LINK    4
 #define DDB_LINK_SHIFT 28
 
 #define DDB_LINK_TAG(_x) (_x << DDB_LINK_SHIFT)
 
-#define DDB_XO2_TYPE_NONE	0
-#define DDB_XO2_TYPE_DUOFLEX	1
-#define DDB_XO2_TYPE_CI		2
+struct ddb_regset {
+	u32 base;
+	u32 num;
+	u32 size;
+};
+
+struct ddb_regmap {
+	u32 irq_base_i2c;
+	u32 irq_base_idma;
+	u32 irq_base_odma;
+
+	const struct ddb_regset *i2c;
+	const struct ddb_regset *i2c_buf;
+	const struct ddb_regset *idma;
+	const struct ddb_regset *idma_buf;
+	const struct ddb_regset *odma;
+	const struct ddb_regset *odma_buf;
+
+	const struct ddb_regset *input;
+	const struct ddb_regset *output;
+
+	const struct ddb_regset *channel;
+};
+
+struct ddb_ids {
+	u16 vendor;
+	u16 device;
+	u16 subvendor;
+	u16 subdevice;
+
+	u32 hwid;
+	u32 regmapid;
+	u32 devid;
+	u32 mac;
+};
 
 struct ddb_info {
 	int   type;
-#define DDB_NONE		0
-#define DDB_OCTOPUS		1
-#define DDB_OCTOPUS_MAX_CT	6
+#define DDB_NONE         0
+#define DDB_OCTOPUS      1
+#define DDB_OCTOPUS_CI   2
+#define DDB_OCTOPUS_MAX  5
+#define DDB_OCTOPUS_MAX_CT  6
 	char *name;
-	int   port_num;
-	u32   port_type[DDB_MAX_PORT];
+	u32   i2c_mask;
+	u8    port_num;
+	u8    led_num;
+	u8    fan_num;
+	u8    temp_num;
+	u8    temp_bus;
 	u32   board_control;
 	u32   board_control_2;
+	u8    mdio_num;
+	u8    con_clock; /* use a continuous clock */
 	u8    ts_quirks;
 #define TS_QUIRK_SERIAL   1
 #define TS_QUIRK_REVERSED 2
 #define TS_QUIRK_ALT_OSC  8
+	u32   tempmon_irq;
+	const struct ddb_regmap *regmap;
 };
 
-/* DMA_SIZE MUST be divisible by 188 and 128 !!! */
+/* DMA_SIZE MUST be smaller than 256k and
+ * MUST be divisible by 188 and 128 !!!
+ */
+
+#define DMA_MAX_BUFS 32      /* hardware table limit */
 
-#define INPUT_DMA_MAX_BUFS 32      /* hardware table limit */
 #define INPUT_DMA_BUFS 8
 #define INPUT_DMA_SIZE (128*47*21)
+#define INPUT_DMA_IRQ_DIV 1
 
-#define OUTPUT_DMA_MAX_BUFS 32
 #define OUTPUT_DMA_BUFS 8
 #define OUTPUT_DMA_SIZE (128*47*21)
+#define OUTPUT_DMA_IRQ_DIV 1
 
 struct ddb;
 struct ddb_port;
 
-struct ddb_input {
-	struct ddb_port       *port;
-	u32                    nr;
-	int                    attached;
+struct ddb_dma {
+	void                  *io;
+	u32                    regs;
+	u32                    bufregs;
 
-	dma_addr_t             pbuf[INPUT_DMA_MAX_BUFS];
-	u8                    *vbuf[INPUT_DMA_MAX_BUFS];
-	u32                    dma_buf_num;
-	u32                    dma_buf_size;
+	dma_addr_t             pbuf[DMA_MAX_BUFS];
+	u8                    *vbuf[DMA_MAX_BUFS];
+	u32                    num;
+	u32                    size;
+	u32                    div;
+	u32                    bufval;
 
-	struct tasklet_struct  tasklet;
+	struct work_struct     work;
 	spinlock_t             lock;
 	wait_queue_head_t      wq;
 	int                    running;
 	u32                    stat;
+	u32                    ctrl;
 	u32                    cbuf;
 	u32                    coff;
+};
 
-	struct dvb_adapter     adap;
+struct ddb_dvb {
+	struct dvb_adapter    *adap;
+	int                    adap_registered;
 	struct dvb_device     *dev;
 	struct i2c_client     *i2c_client[1];
 	struct dvb_frontend   *fe;
@@ -110,97 +188,210 @@ struct ddb_input {
 	struct dmx_frontend    hw_frontend;
 	struct dmx_frontend    mem_frontend;
 	int                    users;
-	int (*gate_ctrl)(struct dvb_frontend *, int);
+	u32                    attached;
+	u8                     input;
+
+	enum fe_sec_tone_mode  tone;
+	enum fe_sec_voltage    voltage;
+
+	int (*i2c_gate_ctrl)(struct dvb_frontend *, int);
+	int (*set_voltage)(struct dvb_frontend *fe,
+		enum fe_sec_voltage voltage);
+	int (*set_input)(struct dvb_frontend *fe, int input);
+	int (*diseqc_send_master_cmd)(struct dvb_frontend *fe,
+		struct dvb_diseqc_master_cmd *cmd);
 };
 
-struct ddb_output {
+struct ddb_ci {
+	struct dvb_ca_en50221  en;
 	struct ddb_port       *port;
 	u32                    nr;
-	dma_addr_t             pbuf[OUTPUT_DMA_MAX_BUFS];
-	u8                    *vbuf[OUTPUT_DMA_MAX_BUFS];
-	u32                    dma_buf_num;
-	u32                    dma_buf_size;
-	struct tasklet_struct  tasklet;
-	spinlock_t             lock;
-	wait_queue_head_t      wq;
-	int                    running;
-	u32                    stat;
-	u32                    cbuf;
-	u32                    coff;
+	struct mutex           lock;
+};
 
-	struct dvb_adapter     adap;
-	struct dvb_device     *dev;
+struct ddb_io {
+	struct ddb_port       *port;
+	u32                    nr;
+	u32                    regs;
+	struct ddb_dma        *dma;
+	struct ddb_io         *redo;
+	struct ddb_io         *redi;
 };
 
+#define ddb_output ddb_io
+#define ddb_input ddb_io
+
 struct ddb_i2c {
 	struct ddb            *dev;
 	u32                    nr;
-	struct i2c_adapter     adap;
-	struct i2c_adapter     adap2;
 	u32                    regs;
+	u32                    link;
+	struct i2c_adapter     adap;
 	u32                    rbuf;
 	u32                    wbuf;
-	int                    done;
-	wait_queue_head_t      wq;
+	u32                    bsize;
+	struct completion      completion;
 };
 
 struct ddb_port {
 	struct ddb            *dev;
 	u32                    nr;
+	u32                    pnr;
+	u32                    regs;
+	u32                    lnr;
 	struct ddb_i2c        *i2c;
 	struct mutex           i2c_gate_lock;
 	u32                    class;
 #define DDB_PORT_NONE           0
 #define DDB_PORT_CI             1
 #define DDB_PORT_TUNER          2
-	u32                    type;
-#define DDB_TUNER_NONE			0
-#define DDB_TUNER_DVBS_ST		1
-#define DDB_TUNER_DVBS_ST_AA		2
-#define DDB_TUNER_DVBCT2_SONY_P		7
-#define DDB_TUNER_DVBC2T2_SONY_P	8
-#define DDB_TUNER_ISDBT_SONY_P		9
-#define DDB_TUNER_DVBC2T2I_SONY_P	15
-#define DDB_TUNER_DVBCT_TR		16
-#define DDB_TUNER_DVBCT_ST		17
-#define DDB_TUNER_XO2_DVBS_STV0910	32
-#define DDB_TUNER_XO2_DVBCT2_SONY	33
-#define DDB_TUNER_XO2_ISDBT_SONY	34
-#define DDB_TUNER_XO2_DVBC2T2_SONY	35
-#define DDB_TUNER_XO2_ATSC_ST		36
-#define DDB_TUNER_XO2_DVBC2T2I_SONY	37
-
-	u32                    adr;
+#define DDB_PORT_LOOP           3
+	char                   *name;
+	char                   *type_name;
+	u32                     type;
+#define DDB_TUNER_NONE           0
+#define DDB_TUNER_DVBS_ST        1
+#define DDB_TUNER_DVBS_ST_AA     2
+#define DDB_TUNER_DVBCT_TR       3
+#define DDB_TUNER_DVBCT_ST       4
+#define DDB_CI_INTERNAL          5
+#define DDB_CI_EXTERNAL_SONY     6
+#define DDB_TUNER_DVBCT2_SONY_P  7
+#define DDB_TUNER_DVBC2T2_SONY_P 8
+#define DDB_TUNER_ISDBT_SONY_P   9
+#define DDB_TUNER_DVBS_STV0910_P 10
+#define DDB_TUNER_MXL5XX         11
+#define DDB_CI_EXTERNAL_XO2      12
+#define DDB_CI_EXTERNAL_XO2_B    13
+#define DDB_TUNER_DVBS_STV0910_PR 14
+#define DDB_TUNER_DVBC2T2I_SONY_P 15
+
+#define DDB_TUNER_XO2            32
+#define DDB_TUNER_DVBS_STV0910   (DDB_TUNER_XO2 + 0)
+#define DDB_TUNER_DVBCT2_SONY    (DDB_TUNER_XO2 + 1)
+#define DDB_TUNER_ISDBT_SONY     (DDB_TUNER_XO2 + 2)
+#define DDB_TUNER_DVBC2T2_SONY   (DDB_TUNER_XO2 + 3)
+#define DDB_TUNER_ATSC_ST        (DDB_TUNER_XO2 + 4)
+#define DDB_TUNER_DVBC2T2I_SONY  (DDB_TUNER_XO2 + 5)
 
 	struct ddb_input      *input[2];
 	struct ddb_output     *output;
 	struct dvb_ca_en50221 *en;
+	struct ddb_dvb         dvb[2];
+	u32                    gap;
+	u32                    obr;
+	u8                     creg;
+};
+
+#define CM_STARTUP_DELAY 2
+#define CM_AVERAGE  20
+#define CM_GAIN     10
+
+#define HW_LSB_SHIFT    12
+#define HW_LSB_MASK     0x1000
+
+#define CM_IDLE    0
+#define CM_STARTUP 1
+#define CM_ADJUST  2
+
+#define TS_CAPTURE_LEN  (4096)
+
+struct ddb_lnb {
+	struct mutex           lock;
+	u32                    tone;
+	enum fe_sec_voltage    oldvoltage[4];
+	u32                    voltage[4];
+	u32                    voltages;
+	u32                    fmode;
+};
+
+struct ddb_link {
+	struct ddb            *dev;
+	const struct ddb_info *info;
+	u32                    nr;
+	u32                    regs;
+	spinlock_t             lock;
+	struct mutex           flash_mutex;
+	struct ddb_lnb         lnb;
+	struct tasklet_struct  tasklet;
+	struct ddb_ids         ids;
+
+	spinlock_t             temp_lock;
+	int                    overtemperature_error;
+	u8                     temp_tab[11];
 };
 
 struct ddb {
 	struct pci_dev        *pdev;
+	struct platform_device *pfdev;
+	struct device         *dev;
+
+	int                    msi;
+	struct workqueue_struct *wq;
+	u32                    has_dma;
+
+	struct ddb_link        link[DDB_MAX_LINK];
 	unsigned char __iomem *regs;
+	u32                    regs_len;
+	u32                    port_num;
 	struct ddb_port        port[DDB_MAX_PORT];
+	u32                    i2c_num;
 	struct ddb_i2c         i2c[DDB_MAX_I2C];
 	struct ddb_input       input[DDB_MAX_INPUT];
 	struct ddb_output      output[DDB_MAX_OUTPUT];
+	struct dvb_adapter     adap[DDB_MAX_INPUT];
+	struct ddb_dma         idma[DDB_MAX_INPUT];
+	struct ddb_dma         odma[DDB_MAX_OUTPUT];
+
+	void                   (*handler[4][256])(unsigned long);
+	unsigned long          handler_data[4][256];
 
 	struct device         *ddb_dev;
-	int                    nr;
+	u32                    ddb_dev_users;
+	u32                    nr;
 	u8                     iobuf[1028];
 
-	struct ddb_info       *info;
-	int                    msi;
+	u8                     leds;
+	u32                    ts_irq;
+	u32                    i2c_irq;
+
+	struct mutex           mutex;
+
+	u8                     tsbuf[TS_CAPTURE_LEN];
 };
 
 /****************************************************************************/
+/****************************************************************************/
+/****************************************************************************/
 
-#define ddbwritel(_val, _adr)        writel((_val), \
-				     dev->regs+(_adr))
-#define ddbreadl(_adr)               readl(dev->regs+(_adr))
-#define ddbcpyto(_adr, _src, _count) memcpy_toio(dev->regs+(_adr), (_src), (_count))
-#define ddbcpyfrom(_dst, _adr, _count) memcpy_fromio((_dst), dev->regs+(_adr), (_count))
+int ddbridge_flashread(struct ddb *dev, u32 link, u8 *buf, u32 addr, u32 len);
 
 /****************************************************************************/
 
-#endif
+/* ddbridge-main.c (modparams) */
+extern int ci_bitrate;
+extern int ts_loop;
+extern int xo2_speed;
+extern int alt_dma;
+extern int no_init;
+extern int stv0910_single;
+extern struct workqueue_struct *ddb_wq;
+
+/* ddbridge-core.c */
+void ddb_ports_detach(struct ddb *dev);
+void ddb_ports_release(struct ddb *dev);
+void ddb_buffers_free(struct ddb *dev);
+void ddb_device_destroy(struct ddb *dev);
+irqreturn_t ddb_irq_handler0(int irq, void *dev_id);
+irqreturn_t ddb_irq_handler1(int irq, void *dev_id);
+irqreturn_t ddb_irq_handler(int irq, void *dev_id);
+void ddb_ports_init(struct ddb *dev);
+int ddb_buffers_alloc(struct ddb *dev);
+int ddb_ports_attach(struct ddb *dev);
+int ddb_device_create(struct ddb *dev);
+int ddb_class_create(void);
+void ddb_class_destroy(void);
+int ddb_init(struct ddb *dev);
+void ddb_unmap(struct ddb *dev);
+
+#endif /* DDBRIDGE_H */
diff --git a/drivers/media/pci/dm1105/dm1105.c b/drivers/media/pci/dm1105/dm1105.c
index 1d41934cfaf5..7c3900dec368 100644
--- a/drivers/media/pci/dm1105/dm1105.c
+++ b/drivers/media/pci/dm1105/dm1105.c
@@ -571,7 +571,7 @@ static u32 functionality(struct i2c_adapter *adap)
 	return I2C_FUNC_I2C;
 }
 
-static struct i2c_algorithm dm1105_algo = {
+static const struct i2c_algorithm dm1105_algo = {
 	.master_xfer   = dm1105_i2c_xfer,
 	.functionality = functionality,
 };
@@ -675,7 +675,7 @@ static void dm1105_emit_key(struct work_struct *work)
 	data = (ircom >> 8) & 0x7f;
 
 	/* FIXME: UNKNOWN because we don't generate a full NEC scancode (yet?) */
-	rc_keydown(ir->dev, RC_TYPE_UNKNOWN, data, 0);
+	rc_keydown(ir->dev, RC_PROTO_UNKNOWN, data, 0);
 }
 
 /* work handler */
@@ -748,7 +748,7 @@ static int dm1105_ir_init(struct dm1105_dev *dm1105)
 
 	dev->driver_name = MODULE_NAME;
 	dev->map_name = RC_MAP_DM1105_NEC;
-	dev->input_name = "DVB on-card IR receiver";
+	dev->device_name = "DVB on-card IR receiver";
 	dev->input_phys = dm1105->ir.input_phys;
 	dev->input_id.bustype = BUS_PCI;
 	dev->input_id.version = 1;
@@ -1208,7 +1208,7 @@ static void dm1105_remove(struct pci_dev *pdev)
 	kfree(dev);
 }
 
-static struct pci_device_id dm1105_id_table[] = {
+static const struct pci_device_id dm1105_id_table[] = {
 	{
 		.vendor = PCI_VENDOR_ID_TRIGEM,
 		.device = PCI_DEVICE_ID_DM1105,
diff --git a/drivers/media/pci/dt3155/dt3155.c b/drivers/media/pci/dt3155/dt3155.c
index 6a219694b225..1775c36891ae 100644
--- a/drivers/media/pci/dt3155/dt3155.c
+++ b/drivers/media/pci/dt3155/dt3155.c
@@ -499,7 +499,7 @@ static int dt3155_init_board(struct dt3155_priv *pd)
 	return 0;
 }
 
-static struct video_device dt3155_vdev = {
+static const struct video_device dt3155_vdev = {
 	.name = DT3155_NAME,
 	.fops = &dt3155_fops,
 	.ioctl_ops = &dt3155_ioctl_ops,
diff --git a/drivers/media/pci/ivtv/ivtv-alsa-mixer.c b/drivers/media/pci/ivtv/ivtv-alsa-mixer.c
index ba372a23eb5c..aee453fcff37 100644
--- a/drivers/media/pci/ivtv/ivtv-alsa-mixer.c
+++ b/drivers/media/pci/ivtv/ivtv-alsa-mixer.c
@@ -156,7 +156,7 @@ int __init snd_ivtv_mixer_create(struct snd_ivtv_card *itvsc)
 
 	strlcpy(sc->mixername, "CX2341[56] Mixer", sizeof(sc->mixername));
 
-	ret = snd_ctl_add(sc, snd_ctl_new1(snd_ivtv_mixer_tv_vol, itvsc));
+	ret = snd_ctl_add(sc, snd_ctl_new1(&snd_ivtv_mixer_tv_vol, itvsc));
 	if (ret) {
 		IVTV_ALSA_WARN("%s: failed to add %s control, err %d\n",
 			       __func__, snd_ivtv_mixer_tv_vol.name, ret);
diff --git a/drivers/media/pci/ivtv/ivtv-alsa-pcm.c b/drivers/media/pci/ivtv/ivtv-alsa-pcm.c
index 417d03da01f0..5326d86fa375 100644
--- a/drivers/media/pci/ivtv/ivtv-alsa-pcm.c
+++ b/drivers/media/pci/ivtv/ivtv-alsa-pcm.c
@@ -41,7 +41,7 @@ MODULE_PARM_DESC(pcm_debug, "enable debug messages for pcm");
 			pr_info("ivtv-alsa-pcm %s: " fmt, __func__, ##arg); \
 	} while (0)
 
-static struct snd_pcm_hardware snd_ivtv_hw_capture = {
+static const struct snd_pcm_hardware snd_ivtv_hw_capture = {
 	.info = SNDRV_PCM_INFO_BLOCK_TRANSFER |
 		SNDRV_PCM_INFO_MMAP           |
 		SNDRV_PCM_INFO_INTERLEAVED    |
diff --git a/drivers/media/pci/ivtv/ivtv-driver.c b/drivers/media/pci/ivtv/ivtv-driver.c
index e8fa99b6c7b4..54dcac4b2229 100644
--- a/drivers/media/pci/ivtv/ivtv-driver.c
+++ b/drivers/media/pci/ivtv/ivtv-driver.c
@@ -73,7 +73,7 @@ int (*ivtv_ext_init)(struct ivtv *);
 EXPORT_SYMBOL(ivtv_ext_init);
 
 /* add your revision and whatnot here */
-static struct pci_device_id ivtv_pci_tbl[] = {
+static const struct pci_device_id ivtv_pci_tbl[] = {
 	{PCI_VENDOR_ID_ICOMP, PCI_DEVICE_ID_IVTV15,
 	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
 	{PCI_VENDOR_ID_ICOMP, PCI_DEVICE_ID_IVTV16,
diff --git a/drivers/media/pci/ivtv/ivtv-i2c.c b/drivers/media/pci/ivtv/ivtv-i2c.c
index dea80efd5836..5a35e366f4c0 100644
--- a/drivers/media/pci/ivtv/ivtv-i2c.c
+++ b/drivers/media/pci/ivtv/ivtv-i2c.c
@@ -148,7 +148,7 @@ static const char * const hw_devicenames[] = {
 	"ir_video",		/* IVTV_HW_I2C_IR_RX_ADAPTEC */
 };
 
-static int get_key_adaptec(struct IR_i2c *ir, enum rc_type *protocol,
+static int get_key_adaptec(struct IR_i2c *ir, enum rc_proto *protocol,
 			   u32 *scancode, u8 *toggle)
 {
 	unsigned char keybuf[4];
@@ -168,7 +168,7 @@ static int get_key_adaptec(struct IR_i2c *ir, enum rc_type *protocol,
 	keybuf[2] &= 0x7f;
 	keybuf[3] |= 0x80;
 
-	*protocol = RC_TYPE_UNKNOWN;
+	*protocol = RC_PROTO_UNKNOWN;
 	*scancode = keybuf[3] | keybuf[2] << 8 | keybuf[1] << 16 |keybuf[0] << 24;
 	*toggle = 0;
 	return 1;
@@ -201,22 +201,22 @@ static int ivtv_i2c_new_ir(struct ivtv *itv, u32 hw, const char *type, u8 addr)
 		init_data->ir_codes = RC_MAP_AVERMEDIA_CARDBUS;
 		init_data->internal_get_key_func =
 					IR_KBD_GET_KEY_AVERMEDIA_CARDBUS;
-		init_data->type = RC_BIT_OTHER;
+		init_data->type = RC_PROTO_BIT_OTHER;
 		init_data->name = "AVerMedia AVerTV card";
 		break;
 	case IVTV_HW_I2C_IR_RX_HAUP_EXT:
 	case IVTV_HW_I2C_IR_RX_HAUP_INT:
 		init_data->ir_codes = RC_MAP_HAUPPAUGE;
 		init_data->internal_get_key_func = IR_KBD_GET_KEY_HAUP;
-		init_data->type = RC_BIT_RC5;
+		init_data->type = RC_PROTO_BIT_RC5;
 		init_data->name = itv->card_name;
 		break;
 	case IVTV_HW_Z8F0811_IR_RX_HAUP:
 		/* Default to grey remote */
 		init_data->ir_codes = RC_MAP_HAUPPAUGE;
 		init_data->internal_get_key_func = IR_KBD_GET_KEY_HAUP_XVR;
-		init_data->type = RC_BIT_RC5 | RC_BIT_RC6_MCE |
-							RC_BIT_RC6_6A_32;
+		init_data->type = RC_PROTO_BIT_RC5 | RC_PROTO_BIT_RC6_MCE |
+							RC_PROTO_BIT_RC6_6A_32;
 		init_data->name = itv->card_name;
 		break;
 	case IVTV_HW_I2C_IR_RX_ADAPTEC:
@@ -224,7 +224,7 @@ static int ivtv_i2c_new_ir(struct ivtv *itv, u32 hw, const char *type, u8 addr)
 		init_data->name = itv->card_name;
 		/* FIXME: The protocol and RC_MAP needs to be corrected */
 		init_data->ir_codes = RC_MAP_EMPTY;
-		init_data->type = RC_BIT_UNKNOWN;
+		init_data->type = RC_PROTO_BIT_UNKNOWN;
 		break;
 	}
 
@@ -632,7 +632,7 @@ static const struct i2c_algorithm ivtv_algo = {
 };
 
 /* template for our-bit banger */
-static struct i2c_adapter ivtv_i2c_adap_hw_template = {
+static const struct i2c_adapter ivtv_i2c_adap_hw_template = {
 	.name = "ivtv i2c driver",
 	.algo = &ivtv_algo,
 	.algo_data = NULL,			/* filled from template */
@@ -682,7 +682,7 @@ static int ivtv_getsda_old(void *data)
 }
 
 /* template for i2c-bit-algo */
-static struct i2c_adapter ivtv_i2c_adap_template = {
+static const struct i2c_adapter ivtv_i2c_adap_template = {
 	.name = "ivtv i2c driver",
 	.algo = NULL,                   /* set by i2c-algo-bit */
 	.algo_data = NULL,              /* filled from template */
diff --git a/drivers/media/pci/mantis/hopper_cards.c b/drivers/media/pci/mantis/hopper_cards.c
index 68b5800030b7..11e987860b23 100644
--- a/drivers/media/pci/mantis/hopper_cards.c
+++ b/drivers/media/pci/mantis/hopper_cards.c
@@ -255,7 +255,7 @@ static void hopper_pci_remove(struct pci_dev *pdev)
 
 }
 
-static struct pci_device_id hopper_pci_table[] = {
+static const struct pci_device_id hopper_pci_table[] = {
 	MAKE_ENTRY(TWINHAN_TECHNOLOGIES, MANTIS_VP_3028_DVB_T, &vp3028_config,
 		   NULL),
 	{ }
diff --git a/drivers/media/pci/mantis/mantis_cards.c b/drivers/media/pci/mantis/mantis_cards.c
index cdefffc16d9e..adc980d33711 100644
--- a/drivers/media/pci/mantis/mantis_cards.c
+++ b/drivers/media/pci/mantis/mantis_cards.c
@@ -281,7 +281,7 @@ static void mantis_pci_remove(struct pci_dev *pdev)
 	return;
 }
 
-static struct pci_device_id mantis_pci_table[] = {
+static const struct pci_device_id mantis_pci_table[] = {
 	MAKE_ENTRY(TECHNISAT, CABLESTAR_HD2, &vp2040_config,
 		   RC_MAP_TECHNISAT_TS35),
 	MAKE_ENTRY(TECHNISAT, SKYSTAR_HD2_10, &vp1041_config,
diff --git a/drivers/media/pci/mantis/mantis_common.h b/drivers/media/pci/mantis/mantis_common.h
index d48778a366a9..a664c319ef0a 100644
--- a/drivers/media/pci/mantis/mantis_common.h
+++ b/drivers/media/pci/mantis/mantis_common.h
@@ -176,7 +176,7 @@ struct mantis_pci {
 	struct work_struct	uart_work;
 
 	struct rc_dev		*rc;
-	char			input_name[80];
+	char			device_name[80];
 	char			input_phys[80];
 	char			*rc_map_name;
 };
diff --git a/drivers/media/pci/mantis/mantis_i2c.c b/drivers/media/pci/mantis/mantis_i2c.c
index d72ee47dc6e4..496c10dfc4df 100644
--- a/drivers/media/pci/mantis/mantis_i2c.c
+++ b/drivers/media/pci/mantis/mantis_i2c.c
@@ -212,7 +212,7 @@ static u32 mantis_i2c_func(struct i2c_adapter *adapter)
 	return I2C_FUNC_SMBUS_EMUL;
 }
 
-static struct i2c_algorithm mantis_algo = {
+static const struct i2c_algorithm mantis_algo = {
 	.master_xfer		= mantis_i2c_xfer,
 	.functionality		= mantis_i2c_func,
 };
diff --git a/drivers/media/pci/mantis/mantis_input.c b/drivers/media/pci/mantis/mantis_input.c
index 50d10cb7d49d..7519dcc934dd 100644
--- a/drivers/media/pci/mantis/mantis_input.c
+++ b/drivers/media/pci/mantis/mantis_input.c
@@ -31,7 +31,7 @@
 void mantis_input_process(struct mantis_pci *mantis, int scancode)
 {
 	if (mantis->rc)
-		rc_keydown(mantis->rc, RC_TYPE_UNKNOWN, scancode, 0);
+		rc_keydown(mantis->rc, RC_PROTO_UNKNOWN, scancode, 0);
 }
 
 int mantis_input_init(struct mantis_pci *mantis)
@@ -46,12 +46,12 @@ int mantis_input_init(struct mantis_pci *mantis)
 		goto out;
 	}
 
-	snprintf(mantis->input_name, sizeof(mantis->input_name),
+	snprintf(mantis->device_name, sizeof(mantis->device_name),
 		 "Mantis %s IR receiver", mantis->hwconfig->model_name);
 	snprintf(mantis->input_phys, sizeof(mantis->input_phys),
 		 "pci-%s/ir0", pci_name(mantis->pdev));
 
-	dev->input_name         = mantis->input_name;
+	dev->device_name        = mantis->device_name;
 	dev->input_phys         = mantis->input_phys;
 	dev->input_id.bustype   = BUS_PCI;
 	dev->input_id.vendor    = mantis->vendor_id;
diff --git a/drivers/media/pci/meye/meye.c b/drivers/media/pci/meye/meye.c
index 9c4a024745de..49e047e4a81e 100644
--- a/drivers/media/pci/meye/meye.c
+++ b/drivers/media/pci/meye/meye.c
@@ -1533,7 +1533,7 @@ static const struct v4l2_ioctl_ops meye_ioctl_ops = {
 	.vidioc_default		= vidioc_default,
 };
 
-static struct video_device meye_template = {
+static const struct video_device meye_template = {
 	.name		= "meye",
 	.fops		= &meye_fops,
 	.ioctl_ops 	= &meye_ioctl_ops,
@@ -1801,7 +1801,7 @@ static void meye_remove(struct pci_dev *pcidev)
 	printk(KERN_INFO "meye: removed\n");
 }
 
-static struct pci_device_id meye_pci_tbl[] = {
+static const struct pci_device_id meye_pci_tbl[] = {
 	{ PCI_VDEVICE(KAWASAKI, PCI_DEVICE_ID_MCHIP_KL5A72002), 0 },
 	{ }
 };
diff --git a/drivers/media/pci/netup_unidvb/netup_unidvb_core.c b/drivers/media/pci/netup_unidvb/netup_unidvb_core.c
index 5c0a4e614413..60e6cd5b3a03 100644
--- a/drivers/media/pci/netup_unidvb/netup_unidvb_core.c
+++ b/drivers/media/pci/netup_unidvb/netup_unidvb_core.c
@@ -1014,7 +1014,7 @@ static void netup_unidvb_finidev(struct pci_dev *pci_dev)
 }
 
 
-static struct pci_device_id netup_unidvb_pci_tbl[] = {
+static const struct pci_device_id netup_unidvb_pci_tbl[] = {
 	{ PCI_DEVICE(0x1b55, 0x18f6) }, /* hw rev. 1.3 */
 	{ PCI_DEVICE(0x1b55, 0x18f7) }, /* hw rev. 1.4 */
 	{ 0, }
diff --git a/drivers/media/pci/netup_unidvb/netup_unidvb_i2c.c b/drivers/media/pci/netup_unidvb/netup_unidvb_i2c.c
index b49e4f9788e8..b13e319d24b7 100644
--- a/drivers/media/pci/netup_unidvb/netup_unidvb_i2c.c
+++ b/drivers/media/pci/netup_unidvb/netup_unidvb_i2c.c
@@ -300,7 +300,7 @@ static const struct i2c_algorithm netup_i2c_algorithm = {
 	.functionality	= netup_i2c_func,
 };
 
-static struct i2c_adapter netup_i2c_adapter = {
+static const struct i2c_adapter netup_i2c_adapter = {
 	.owner		= THIS_MODULE,
 	.name		= NETUP_UNIDVB_NAME,
 	.class		= I2C_CLASS_HWMON | I2C_CLASS_SPD,
diff --git a/drivers/media/pci/ngene/ngene-i2c.c b/drivers/media/pci/ngene/ngene-i2c.c
index fbf36353c701..3004947f300b 100644
--- a/drivers/media/pci/ngene/ngene-i2c.c
+++ b/drivers/media/pci/ngene/ngene-i2c.c
@@ -150,7 +150,7 @@ static u32 ngene_i2c_functionality(struct i2c_adapter *adap)
 	return I2C_FUNC_SMBUS_EMUL;
 }
 
-static struct i2c_algorithm ngene_i2c_algo = {
+static const struct i2c_algorithm ngene_i2c_algo = {
 	.master_xfer = ngene_i2c_master_xfer,
 	.functionality = ngene_i2c_functionality,
 };
diff --git a/drivers/media/pci/pluto2/pluto2.c b/drivers/media/pci/pluto2/pluto2.c
index 74838109afe5..39dcba2b620c 100644
--- a/drivers/media/pci/pluto2/pluto2.c
+++ b/drivers/media/pci/pluto2/pluto2.c
@@ -770,7 +770,7 @@ static void pluto2_remove(struct pci_dev *pdev)
 #define PCI_DEVICE_ID_PLUTO2	0x0001
 #endif
 
-static struct pci_device_id pluto2_id_table[] = {
+static const struct pci_device_id pluto2_id_table[] = {
 	{
 		.vendor = PCI_VENDOR_ID_SCM,
 		.device = PCI_DEVICE_ID_PLUTO2,
diff --git a/drivers/media/pci/pt1/pt1.c b/drivers/media/pci/pt1/pt1.c
index 3219d2f3271e..b6b1a8d20d86 100644
--- a/drivers/media/pci/pt1/pt1.c
+++ b/drivers/media/pci/pt1/pt1.c
@@ -1202,7 +1202,7 @@ err:
 
 }
 
-static struct pci_device_id pt1_id_table[] = {
+static const struct pci_device_id pt1_id_table[] = {
 	{ PCI_DEVICE(0x10ee, 0x211a) },
 	{ PCI_DEVICE(0x10ee, 0x222a) },
 	{ },
diff --git a/drivers/media/pci/pt3/pt3.c b/drivers/media/pci/pt3/pt3.c
index e8b5d0992157..34044a45fecc 100644
--- a/drivers/media/pci/pt3/pt3.c
+++ b/drivers/media/pci/pt3/pt3.c
@@ -472,7 +472,6 @@ static int pt3_fetch_thread(void *data)
 	}
 	dev_dbg(adap->dvb_adap.device, "PT3: [%s] exited\n",
 		adap->thread->comm);
-	adap->thread = NULL;
 	return 0;
 }
 
@@ -486,6 +485,7 @@ static int pt3_start_streaming(struct pt3_adapter *adap)
 	if (IS_ERR(thread)) {
 		int ret = PTR_ERR(thread);
 
+		adap->thread = NULL;
 		dev_warn(adap->dvb_adap.device,
 			 "PT3 (adap:%d, dmx:%d): failed to start kthread\n",
 			 adap->dvb_adap.num, adap->dmxdev.dvbdev->id);
@@ -508,6 +508,7 @@ static int pt3_stop_streaming(struct pt3_adapter *adap)
 
 	/* kill the fetching thread */
 	ret = kthread_stop(adap->thread);
+	adap->thread = NULL;
 	return ret;
 }
 
@@ -520,14 +521,8 @@ static int pt3_start_feed(struct dvb_demux_feed *feed)
 
 	adap = container_of(feed->demux, struct pt3_adapter, demux);
 	adap->num_feeds++;
-	if (adap->thread)
+	if (adap->num_feeds > 1)
 		return 0;
-	if (adap->num_feeds != 1) {
-		dev_warn(adap->dvb_adap.device,
-			 "%s: unmatched start/stop_feed in adap:%i/dmx:%i\n",
-			 __func__, adap->dvb_adap.num, adap->dmxdev.dvbdev->id);
-		adap->num_feeds = 1;
-	}
 
 	return pt3_start_streaming(adap);
 
diff --git a/drivers/media/pci/saa7134/saa7134-alsa.c b/drivers/media/pci/saa7134/saa7134-alsa.c
index bf358ec7aca5..c59b69f1af9d 100644
--- a/drivers/media/pci/saa7134/saa7134-alsa.c
+++ b/drivers/media/pci/saa7134/saa7134-alsa.c
@@ -627,7 +627,7 @@ snd_card_saa7134_capture_pointer(struct snd_pcm_substream * substream)
  *    switching to 32kHz without any frequency translation
  */
 
-static struct snd_pcm_hardware snd_card_saa7134_capture =
+static const struct snd_pcm_hardware snd_card_saa7134_capture =
 {
 	.info =                 (SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_INTERLEAVED |
 				 SNDRV_PCM_INFO_BLOCK_TRANSFER |
diff --git a/drivers/media/pci/saa7134/saa7134-empress.c b/drivers/media/pci/saa7134/saa7134-empress.c
index b1d3648dcba1..66acfd35ffc6 100644
--- a/drivers/media/pci/saa7134/saa7134-empress.c
+++ b/drivers/media/pci/saa7134/saa7134-empress.c
@@ -205,7 +205,7 @@ static const struct v4l2_ioctl_ops ts_ioctl_ops = {
 
 /* ----------------------------------------------------------- */
 
-static struct video_device saa7134_empress_template = {
+static const struct video_device saa7134_empress_template = {
 	.name          = "saa7134-empress",
 	.fops          = &ts_fops,
 	.ioctl_ops     = &ts_ioctl_ops,
diff --git a/drivers/media/pci/saa7134/saa7134-i2c.c b/drivers/media/pci/saa7134/saa7134-i2c.c
index 9d0e69eae036..8f2ed632840f 100644
--- a/drivers/media/pci/saa7134/saa7134-i2c.c
+++ b/drivers/media/pci/saa7134/saa7134-i2c.c
@@ -339,7 +339,7 @@ static const struct i2c_algorithm saa7134_algo = {
 	.functionality = functionality,
 };
 
-static struct i2c_adapter saa7134_adap_template = {
+static const struct i2c_adapter saa7134_adap_template = {
 	.owner         = THIS_MODULE,
 	.name          = "saa7134",
 	.algo          = &saa7134_algo,
diff --git a/drivers/media/pci/saa7134/saa7134-input.c b/drivers/media/pci/saa7134/saa7134-input.c
index 78849c19f68a..9337e4615519 100644
--- a/drivers/media/pci/saa7134/saa7134-input.c
+++ b/drivers/media/pci/saa7134/saa7134-input.c
@@ -83,14 +83,16 @@ static int build_key(struct saa7134_dev *dev)
 		if (data == ir->mask_keycode)
 			rc_keyup(ir->dev);
 		else
-			rc_keydown_notimeout(ir->dev, RC_TYPE_UNKNOWN, data, 0);
+			rc_keydown_notimeout(ir->dev, RC_PROTO_UNKNOWN, data,
+					     0);
 		return 0;
 	}
 
 	if (ir->polling) {
 		if ((ir->mask_keydown  &&  (0 != (gpio & ir->mask_keydown))) ||
 		    (ir->mask_keyup    &&  (0 == (gpio & ir->mask_keyup)))) {
-			rc_keydown_notimeout(ir->dev, RC_TYPE_UNKNOWN, data, 0);
+			rc_keydown_notimeout(ir->dev, RC_PROTO_UNKNOWN, data,
+					     0);
 		} else {
 			rc_keyup(ir->dev);
 		}
@@ -98,7 +100,8 @@ static int build_key(struct saa7134_dev *dev)
 	else {	/* IRQ driven mode - handle key press and release in one go */
 		if ((ir->mask_keydown  &&  (0 != (gpio & ir->mask_keydown))) ||
 		    (ir->mask_keyup    &&  (0 == (gpio & ir->mask_keyup)))) {
-			rc_keydown_notimeout(ir->dev, RC_TYPE_UNKNOWN, data, 0);
+			rc_keydown_notimeout(ir->dev, RC_PROTO_UNKNOWN, data,
+					     0);
 			rc_keyup(ir->dev);
 		}
 	}
@@ -108,7 +111,7 @@ static int build_key(struct saa7134_dev *dev)
 
 /* --------------------- Chip specific I2C key builders ----------------- */
 
-static int get_key_flydvb_trio(struct IR_i2c *ir, enum rc_type *protocol,
+static int get_key_flydvb_trio(struct IR_i2c *ir, enum rc_proto *protocol,
 			       u32 *scancode, u8 *toggle)
 {
 	int gpio;
@@ -154,13 +157,14 @@ static int get_key_flydvb_trio(struct IR_i2c *ir, enum rc_type *protocol,
 		return -EIO;
 	}
 
-	*protocol = RC_TYPE_UNKNOWN;
+	*protocol = RC_PROTO_UNKNOWN;
 	*scancode = b;
 	*toggle = 0;
 	return 1;
 }
 
-static int get_key_msi_tvanywhere_plus(struct IR_i2c *ir, enum rc_type *protocol,
+static int get_key_msi_tvanywhere_plus(struct IR_i2c *ir,
+				       enum rc_proto *protocol,
 				       u32 *scancode, u8 *toggle)
 {
 	unsigned char b;
@@ -201,14 +205,14 @@ static int get_key_msi_tvanywhere_plus(struct IR_i2c *ir, enum rc_type *protocol
 	/* Button pressed */
 
 	input_dbg("get_key_msi_tvanywhere_plus: Key = 0x%02X\n", b);
-	*protocol = RC_TYPE_UNKNOWN;
+	*protocol = RC_PROTO_UNKNOWN;
 	*scancode = b;
 	*toggle = 0;
 	return 1;
 }
 
 /* copied and modified from get_key_msi_tvanywhere_plus() */
-static int get_key_kworld_pc150u(struct IR_i2c *ir, enum rc_type *protocol,
+static int get_key_kworld_pc150u(struct IR_i2c *ir, enum rc_proto *protocol,
 				 u32 *scancode, u8 *toggle)
 {
 	unsigned char b;
@@ -249,13 +253,13 @@ static int get_key_kworld_pc150u(struct IR_i2c *ir, enum rc_type *protocol,
 	/* Button pressed */
 
 	input_dbg("get_key_kworld_pc150u: Key = 0x%02X\n", b);
-	*protocol = RC_TYPE_UNKNOWN;
+	*protocol = RC_PROTO_UNKNOWN;
 	*scancode = b;
 	*toggle = 0;
 	return 1;
 }
 
-static int get_key_purpletv(struct IR_i2c *ir, enum rc_type *protocol,
+static int get_key_purpletv(struct IR_i2c *ir, enum rc_proto *protocol,
 			    u32 *scancode, u8 *toggle)
 {
 	unsigned char b;
@@ -274,13 +278,13 @@ static int get_key_purpletv(struct IR_i2c *ir, enum rc_type *protocol,
 	if (b & 0x80)
 		return 1;
 
-	*protocol = RC_TYPE_UNKNOWN;
+	*protocol = RC_PROTO_UNKNOWN;
 	*scancode = b;
 	*toggle = 0;
 	return 1;
 }
 
-static int get_key_hvr1110(struct IR_i2c *ir, enum rc_type *protocol,
+static int get_key_hvr1110(struct IR_i2c *ir, enum rc_proto *protocol,
 			   u32 *scancode, u8 *toggle)
 {
 	unsigned char buf[5];
@@ -304,14 +308,14 @@ static int get_key_hvr1110(struct IR_i2c *ir, enum rc_type *protocol,
 	 *
 	 * FIXME: start bits could maybe be used...?
 	 */
-	*protocol = RC_TYPE_RC5;
+	*protocol = RC_PROTO_RC5;
 	*scancode = RC_SCANCODE_RC5(buf[3] & 0x1f, buf[4] >> 2);
 	*toggle = !!(buf[3] & 0x40);
 	return 1;
 }
 
 
-static int get_key_beholdm6xx(struct IR_i2c *ir, enum rc_type *protocol,
+static int get_key_beholdm6xx(struct IR_i2c *ir, enum rc_proto *protocol,
 			      u32 *scancode, u8 *toggle)
 {
 	unsigned char data[12];
@@ -338,7 +342,7 @@ static int get_key_beholdm6xx(struct IR_i2c *ir, enum rc_type *protocol,
 	if (data[9] != (unsigned char)(~data[8]))
 		return 0;
 
-	*protocol = RC_TYPE_NECX;
+	*protocol = RC_PROTO_NECX;
 	*scancode = RC_SCANCODE_NECX(data[11] << 8 | data[10], data[9]);
 	*toggle = 0;
 	return 1;
@@ -347,7 +351,7 @@ static int get_key_beholdm6xx(struct IR_i2c *ir, enum rc_type *protocol,
 /* Common (grey or coloured) pinnacle PCTV remote handling
  *
  */
-static int get_key_pinnacle(struct IR_i2c *ir, enum rc_type *protocol,
+static int get_key_pinnacle(struct IR_i2c *ir, enum rc_proto *protocol,
 			    u32 *scancode, u8 *toggle, int parity_offset,
 			    int marker, int code_modulo)
 {
@@ -384,7 +388,7 @@ static int get_key_pinnacle(struct IR_i2c *ir, enum rc_type *protocol,
 
 	code %= code_modulo;
 
-	*protocol = RC_TYPE_UNKNOWN;
+	*protocol = RC_PROTO_UNKNOWN;
 	*scancode = code;
 	*toggle = 0;
 
@@ -401,7 +405,7 @@ static int get_key_pinnacle(struct IR_i2c *ir, enum rc_type *protocol,
  *
  * Sylvain Pasche <sylvain.pasche@gmail.com>
  */
-static int get_key_pinnacle_grey(struct IR_i2c *ir, enum rc_type *protocol,
+static int get_key_pinnacle_grey(struct IR_i2c *ir, enum rc_proto *protocol,
 				 u32 *scancode, u8 *toggle)
 {
 
@@ -413,7 +417,7 @@ static int get_key_pinnacle_grey(struct IR_i2c *ir, enum rc_type *protocol,
  *
  * Ricardo Cerqueira <v4l@cerqueira.org>
  */
-static int get_key_pinnacle_color(struct IR_i2c *ir, enum rc_type *protocol,
+static int get_key_pinnacle_color(struct IR_i2c *ir, enum rc_proto *protocol,
 				  u32 *scancode, u8 *toggle)
 {
 	/* code_modulo parameter (0x88) is used to reduce code value to fit inside IR_KEYTAB_SIZE
@@ -452,13 +456,6 @@ static void saa7134_input_timer(unsigned long data)
 	mod_timer(&ir->timer, jiffies + msecs_to_jiffies(ir->polling));
 }
 
-static void ir_raw_decode_timer_end(unsigned long data)
-{
-	struct saa7134_dev *dev = (struct saa7134_dev *)data;
-
-	ir_raw_event_handle(dev->remote->dev);
-}
-
 static int __saa7134_ir_start(void *priv)
 {
 	struct saa7134_dev *dev = priv;
@@ -514,10 +511,6 @@ static int __saa7134_ir_start(void *priv)
 			    (unsigned long)dev);
 		ir->timer.expires = jiffies + HZ;
 		add_timer(&ir->timer);
-	} else if (ir->raw_decode) {
-		/* set timer_end for code completion */
-		setup_timer(&ir->timer, ir_raw_decode_timer_end,
-			    (unsigned long)dev);
 	}
 
 	return 0;
@@ -535,7 +528,7 @@ static void __saa7134_ir_stop(void *priv)
 	if (!ir->running)
 		return;
 
-	if (ir->polling || ir->raw_decode)
+	if (ir->polling)
 		del_timer_sync(&ir->timer);
 
 	ir->running = false;
@@ -867,10 +860,12 @@ int saa7134_input_init1(struct saa7134_dev *dev)
 	rc->priv = dev;
 	rc->open = saa7134_ir_open;
 	rc->close = saa7134_ir_close;
-	if (raw_decode)
+	if (raw_decode) {
 		rc->driver_type = RC_DRIVER_IR_RAW;
+		rc->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
+	}
 
-	rc->input_name = ir->name;
+	rc->device_name = ir->name;
 	rc->input_phys = ir->phys;
 	rc->input_id.bustype = BUS_PCI;
 	rc->input_id.version = 1;
@@ -884,6 +879,9 @@ int saa7134_input_init1(struct saa7134_dev *dev)
 	rc->dev.parent = &dev->pci->dev;
 	rc->map_name = ir_codes;
 	rc->driver_name = MODULE_NAME;
+	rc->min_timeout = 1;
+	rc->timeout = IR_DEFAULT_TIMEOUT;
+	rc->max_timeout = 10 * IR_DEFAULT_TIMEOUT;
 
 	err = rc_register_device(rc);
 	if (err)
@@ -1028,7 +1026,7 @@ void saa7134_probe_i2c_ir(struct saa7134_dev *dev)
 		dev->init_data.name = "BeholdTV";
 		dev->init_data.get_key = get_key_beholdm6xx;
 		dev->init_data.ir_codes = RC_MAP_BEHOLD;
-		dev->init_data.type = RC_BIT_NECX;
+		dev->init_data.type = RC_PROTO_BIT_NECX;
 		info.addr = 0x2d;
 		break;
 	case SAA7134_BOARD_AVERMEDIA_CARDBUS_501:
@@ -1057,26 +1055,13 @@ void saa7134_probe_i2c_ir(struct saa7134_dev *dev)
 static int saa7134_raw_decode_irq(struct saa7134_dev *dev)
 {
 	struct saa7134_card_ir *ir = dev->remote;
-	unsigned long timeout;
 	int space;
 
 	/* Generate initial event */
 	saa_clearb(SAA7134_GPIO_GPMODE3, SAA7134_GPIO_GPRESCAN);
 	saa_setb(SAA7134_GPIO_GPMODE3, SAA7134_GPIO_GPRESCAN);
 	space = saa_readl(SAA7134_GPIO_GPSTATUS0 >> 2) & ir->mask_keydown;
-	ir_raw_event_store_edge(dev->remote->dev, space ? IR_SPACE : IR_PULSE);
-
-	/*
-	 * Wait 15 ms from the start of the first IR event before processing
-	 * the event. This time is enough for NEC protocol. May need adjustments
-	 * to work with other protocols.
-	 */
-	smp_mb();
-
-	if (!timer_pending(&ir->timer)) {
-		timeout = jiffies + msecs_to_jiffies(15);
-		mod_timer(&ir->timer, timeout);
-	}
+	ir_raw_event_store_edge(dev->remote->dev, !space);
 
 	return 1;
 }
diff --git a/drivers/media/pci/saa7146/hexium_gemini.c b/drivers/media/pci/saa7146/hexium_gemini.c
index c889ec9f8a5a..f708cab01fef 100644
--- a/drivers/media/pci/saa7146/hexium_gemini.c
+++ b/drivers/media/pci/saa7146/hexium_gemini.c
@@ -363,7 +363,7 @@ static struct saa7146_pci_extension_data hexium_gemini_dual_4bnc = {
 	.ext = &hexium_extension,
 };
 
-static struct pci_device_id pci_tbl[] = {
+static const struct pci_device_id pci_tbl[] = {
 	{
 	 .vendor = PCI_VENDOR_ID_PHILIPS,
 	 .device = PCI_DEVICE_ID_PHILIPS_SAA7146,
diff --git a/drivers/media/pci/saa7146/hexium_orion.c b/drivers/media/pci/saa7146/hexium_orion.c
index c306a92e8909..01f01580c7ca 100644
--- a/drivers/media/pci/saa7146/hexium_orion.c
+++ b/drivers/media/pci/saa7146/hexium_orion.c
@@ -427,7 +427,7 @@ static struct saa7146_pci_extension_data hexium_orion_4bnc = {
 	.ext = &extension,
 };
 
-static struct pci_device_id pci_tbl[] = {
+static const struct pci_device_id pci_tbl[] = {
 	{
 	 .vendor = PCI_VENDOR_ID_PHILIPS,
 	 .device = PCI_DEVICE_ID_PHILIPS_SAA7146,
diff --git a/drivers/media/pci/saa7146/mxb.c b/drivers/media/pci/saa7146/mxb.c
index 504d78807639..930218cc2de1 100644
--- a/drivers/media/pci/saa7146/mxb.c
+++ b/drivers/media/pci/saa7146/mxb.c
@@ -819,7 +819,7 @@ static struct saa7146_pci_extension_data mxb = {
 	.ext = &extension,
 };
 
-static struct pci_device_id pci_tbl[] = {
+static const struct pci_device_id pci_tbl[] = {
 	{
 		.vendor    = PCI_VENDOR_ID_PHILIPS,
 		.device	   = PCI_DEVICE_ID_PHILIPS_SAA7146,
diff --git a/drivers/media/pci/saa7164/saa7164-core.c b/drivers/media/pci/saa7164/saa7164-core.c
index 75eed4cc4823..fca36a4910c2 100644
--- a/drivers/media/pci/saa7164/saa7164-core.c
+++ b/drivers/media/pci/saa7164/saa7164-core.c
@@ -1490,7 +1490,7 @@ static void saa7164_finidev(struct pci_dev *pci_dev)
 	kfree(dev);
 }
 
-static struct pci_device_id saa7164_pci_tbl[] = {
+static const struct pci_device_id saa7164_pci_tbl[] = {
 	{
 		/* SAA7164 */
 		.vendor       = 0x1131,
diff --git a/drivers/media/pci/saa7164/saa7164-i2c.c b/drivers/media/pci/saa7164/saa7164-i2c.c
index 430f6789f222..4bcde7c79dc3 100644
--- a/drivers/media/pci/saa7164/saa7164-i2c.c
+++ b/drivers/media/pci/saa7164/saa7164-i2c.c
@@ -78,7 +78,7 @@ static const struct i2c_algorithm saa7164_i2c_algo_template = {
 
 /* ----------------------------------------------------------------------- */
 
-static struct i2c_adapter saa7164_i2c_adap_template = {
+static const struct i2c_adapter saa7164_i2c_adap_template = {
 	.name              = "saa7164",
 	.owner             = THIS_MODULE,
 	.algo              = &saa7164_i2c_algo_template,
diff --git a/drivers/media/pci/smipcie/smipcie-ir.c b/drivers/media/pci/smipcie/smipcie-ir.c
index d2730c3fdbae..c5595af6b976 100644
--- a/drivers/media/pci/smipcie/smipcie-ir.c
+++ b/drivers/media/pci/smipcie/smipcie-ir.c
@@ -144,7 +144,7 @@ static void smi_ir_decode(struct work_struct *work)
 			rc5_system = (dwIRCode & 0x7C0) >> 6;
 			toggle = (dwIRCode & 0x800) ? 1 : 0;
 			scancode = rc5_system << 8 | rc5_command;
-			rc_keydown(rc_dev, RC_TYPE_RC5, scancode, toggle);
+			rc_keydown(rc_dev, RC_PROTO_RC5, scancode, toggle);
 		}
 	}
 end_ir_decode:
@@ -188,14 +188,14 @@ int smi_ir_init(struct smi_dev *dev)
 		return -ENOMEM;
 
 	/* init input device */
-	snprintf(ir->input_name, sizeof(ir->input_name), "IR (%s)",
+	snprintf(ir->device_name, sizeof(ir->device_name), "IR (%s)",
 		 dev->info->name);
 	snprintf(ir->input_phys, sizeof(ir->input_phys), "pci-%s/ir0",
 		 pci_name(dev->pci_dev));
 
 	rc_dev->driver_name = "SMI_PCIe";
 	rc_dev->input_phys = ir->input_phys;
-	rc_dev->input_name = ir->input_name;
+	rc_dev->device_name = ir->device_name;
 	rc_dev->input_id.bustype = BUS_PCI;
 	rc_dev->input_id.version = 1;
 	rc_dev->input_id.vendor = dev->pci_dev->subsystem_vendor;
diff --git a/drivers/media/pci/smipcie/smipcie.h b/drivers/media/pci/smipcie/smipcie.h
index 611e4f02cadd..c8368c78ddd5 100644
--- a/drivers/media/pci/smipcie/smipcie.h
+++ b/drivers/media/pci/smipcie/smipcie.h
@@ -240,7 +240,7 @@ struct smi_rc {
 	struct smi_dev *dev;
 	struct rc_dev *rc_dev;
 	char input_phys[64];
-	char input_name[64];
+	char device_name[64];
 	struct work_struct work;
 	u8 irData[256];
 
diff --git a/drivers/media/pci/solo6x10/solo6x10-g723.c b/drivers/media/pci/solo6x10/solo6x10-g723.c
index 3ca947092775..81be1b8df758 100644
--- a/drivers/media/pci/solo6x10/solo6x10-g723.c
+++ b/drivers/media/pci/solo6x10/solo6x10-g723.c
@@ -319,7 +319,7 @@ static int snd_solo_capture_volume_put(struct snd_kcontrol *kcontrol,
 	return 1;
 }
 
-static struct snd_kcontrol_new snd_solo_capture_volume = {
+static const struct snd_kcontrol_new snd_solo_capture_volume = {
 	.iface = SNDRV_CTL_ELEM_IFACE_MIXER,
 	.name = "Capture Volume",
 	.info = snd_solo_capture_volume_info,
diff --git a/drivers/media/pci/solo6x10/solo6x10-gpio.c b/drivers/media/pci/solo6x10/solo6x10-gpio.c
index 6d3b4a36bc11..3d0d1aa2f6a8 100644
--- a/drivers/media/pci/solo6x10/solo6x10-gpio.c
+++ b/drivers/media/pci/solo6x10/solo6x10-gpio.c
@@ -57,6 +57,9 @@ static void solo_gpio_mode(struct solo_dev *solo_dev,
 			ret |= 1 << port;
 	}
 
+	/* Enable GPIO[31:16] */
+	ret |= 0xffff0000;
+
 	solo_reg_write(solo_dev, SOLO_GPIO_CONFIG_1, ret);
 }
 
@@ -90,16 +93,110 @@ static void solo_gpio_config(struct solo_dev *solo_dev)
 
 	/* Initially set relay status to 0 */
 	solo_gpio_clear(solo_dev, 0xff00);
+
+	/* Set input pins direction */
+	solo_gpio_mode(solo_dev, 0xffff0000, 0);
+}
+
+#ifdef CONFIG_GPIOLIB
+/* Pins 0-7 are not exported, because it seems from code above they are
+ * used for internal purposes. So offset 0 corresponds to pin 8, therefore
+ * offsets 0-7 are relay GPIOs, 8-23 - input GPIOs.
+ */
+static int solo_gpiochip_get_direction(struct gpio_chip *chip,
+				       unsigned int offset)
+{
+	int ret, mode;
+	struct solo_dev *solo_dev = gpiochip_get_data(chip);
+
+	if (offset < 8) {
+		ret = solo_reg_read(solo_dev, SOLO_GPIO_CONFIG_0);
+		mode = 3 & (ret >> ((offset + 8) * 2));
+	} else {
+		ret = solo_reg_read(solo_dev, SOLO_GPIO_CONFIG_1);
+		mode =  1 & (ret >> (offset - 8));
+	}
+
+	if (!mode)
+		return 1;
+	else if (mode == 1)
+		return 0;
+
+	return -1;
 }
 
+static int solo_gpiochip_direction_input(struct gpio_chip *chip,
+					 unsigned int offset)
+{
+	return -1;
+}
+
+static int solo_gpiochip_direction_output(struct gpio_chip *chip,
+					  unsigned int offset, int value)
+{
+	return -1;
+}
+
+static int solo_gpiochip_get(struct gpio_chip *chip,
+						unsigned int offset)
+{
+	int ret;
+	struct solo_dev *solo_dev = gpiochip_get_data(chip);
+
+	ret = solo_reg_read(solo_dev, SOLO_GPIO_DATA_IN);
+
+	return 1 & (ret >> (offset + 8));
+}
+
+static void solo_gpiochip_set(struct gpio_chip *chip,
+						unsigned int offset, int value)
+{
+	struct solo_dev *solo_dev = gpiochip_get_data(chip);
+
+	if (value)
+		solo_gpio_set(solo_dev, 1 << (offset + 8));
+	else
+		solo_gpio_clear(solo_dev, 1 << (offset + 8));
+}
+#endif
+
 int solo_gpio_init(struct solo_dev *solo_dev)
 {
+	int ret;
+
 	solo_gpio_config(solo_dev);
+#ifdef CONFIG_GPIOLIB
+	solo_dev->gpio_dev.label = SOLO6X10_NAME"_gpio";
+	solo_dev->gpio_dev.parent = &solo_dev->pdev->dev;
+	solo_dev->gpio_dev.owner = THIS_MODULE;
+	solo_dev->gpio_dev.base = -1;
+	solo_dev->gpio_dev.ngpio = 24;
+	solo_dev->gpio_dev.can_sleep = 0;
+
+	solo_dev->gpio_dev.get_direction = solo_gpiochip_get_direction;
+	solo_dev->gpio_dev.direction_input = solo_gpiochip_direction_input;
+	solo_dev->gpio_dev.direction_output = solo_gpiochip_direction_output;
+	solo_dev->gpio_dev.get = solo_gpiochip_get;
+	solo_dev->gpio_dev.set = solo_gpiochip_set;
+
+	ret = gpiochip_add_data(&solo_dev->gpio_dev, solo_dev);
+
+	if (ret) {
+		solo_dev->gpio_dev.label = NULL;
+		return -1;
+	}
+#endif
 	return 0;
 }
 
 void solo_gpio_exit(struct solo_dev *solo_dev)
 {
+#ifdef CONFIG_GPIOLIB
+	if (solo_dev->gpio_dev.label) {
+		gpiochip_remove(&solo_dev->gpio_dev);
+		solo_dev->gpio_dev.label = NULL;
+	}
+#endif
 	solo_gpio_clear(solo_dev, 0x30);
 	solo_gpio_config(solo_dev);
 }
diff --git a/drivers/media/pci/solo6x10/solo6x10-tw28.c b/drivers/media/pci/solo6x10/solo6x10-tw28.c
index 0632d3f7c73c..7ecb725b6dd2 100644
--- a/drivers/media/pci/solo6x10/solo6x10-tw28.c
+++ b/drivers/media/pci/solo6x10/solo6x10-tw28.c
@@ -532,7 +532,7 @@ static void saa712x_write_regs(struct solo_dev *dev, const u8 *vals,
 static void saa712x_setup(struct solo_dev *dev)
 {
 	const int reg_start = 0x26;
-	const u8 saa7128_regs_ntsc[] = {
+	static const u8 saa7128_regs_ntsc[] = {
 	/* :0x26 */
 		0x0d, 0x00,
 	/* :0x28 */
@@ -606,6 +606,7 @@ int solo_tw28_init(struct solo_dev *solo_dev)
 			solo_dev->tw28_cnt++;
 			break;
 		case 0x0c:
+		case 0x0d:
 			solo_dev->tw2864 |= 1 << i;
 			solo_dev->tw28_cnt++;
 			break;
diff --git a/drivers/media/pci/solo6x10/solo6x10-v4l2.c b/drivers/media/pci/solo6x10/solo6x10-v4l2.c
index 3266fc21825f..99ffd1ed4a73 100644
--- a/drivers/media/pci/solo6x10/solo6x10-v4l2.c
+++ b/drivers/media/pci/solo6x10/solo6x10-v4l2.c
@@ -630,7 +630,7 @@ static const struct v4l2_ioctl_ops solo_v4l2_ioctl_ops = {
 	.vidioc_unsubscribe_event	= v4l2_event_unsubscribe,
 };
 
-static struct video_device solo_v4l2_template = {
+static const struct video_device solo_v4l2_template = {
 	.name			= SOLO6X10_NAME,
 	.fops			= &solo_v4l2_fops,
 	.ioctl_ops		= &solo_v4l2_ioctl_ops,
diff --git a/drivers/media/pci/solo6x10/solo6x10.h b/drivers/media/pci/solo6x10/solo6x10.h
index 3f8da5e8c430..3a1893ae2dad 100644
--- a/drivers/media/pci/solo6x10/solo6x10.h
+++ b/drivers/media/pci/solo6x10/solo6x10.h
@@ -31,6 +31,7 @@
 #include <linux/atomic.h>
 #include <linux/slab.h>
 #include <linux/videodev2.h>
+#include <linux/gpio/driver.h>
 
 #include <media/v4l2-dev.h>
 #include <media/v4l2-device.h>
@@ -199,6 +200,10 @@ struct solo_dev {
 	u32			irq_mask;
 	u32			motion_mask;
 	struct v4l2_device	v4l2_dev;
+#ifdef CONFIG_GPIOLIB
+	/* GPIO */
+	struct gpio_chip	gpio_dev;
+#endif
 
 	/* tw28xx accounting */
 	u8			tw2865, tw2864, tw2815;
diff --git a/drivers/media/pci/sta2x11/sta2x11_vip.c b/drivers/media/pci/sta2x11/sta2x11_vip.c
index 6343d24eb1d5..eb5a9eae7c8e 100644
--- a/drivers/media/pci/sta2x11/sta2x11_vip.c
+++ b/drivers/media/pci/sta2x11/sta2x11_vip.c
@@ -754,7 +754,7 @@ static const struct v4l2_ioctl_ops vip_ioctl_ops = {
 	.vidioc_unsubscribe_event = v4l2_event_unsubscribe,
 };
 
-static struct video_device video_dev_template = {
+static const struct video_device video_dev_template = {
 	.name = KBUILD_MODNAME,
 	.release = video_device_release_empty,
 	.fops = &vip_fops,
diff --git a/drivers/media/pci/ttpci/av7110.c b/drivers/media/pci/ttpci/av7110.c
index f2905bd80366..f46947d8adf8 100644
--- a/drivers/media/pci/ttpci/av7110.c
+++ b/drivers/media/pci/ttpci/av7110.c
@@ -2872,7 +2872,7 @@ MAKE_AV7110_INFO(fsc,        "Fujitsu Siemens DVB-C");
 MAKE_AV7110_INFO(fss,        "Fujitsu Siemens DVB-S rev1.6");
 MAKE_AV7110_INFO(gxs_1_3,    "Galaxis DVB-S rev1.3");
 
-static struct pci_device_id pci_tbl[] = {
+static const struct pci_device_id pci_tbl[] = {
 	MAKE_EXTENSION_PCI(fsc,         0x110a, 0x0000),
 	MAKE_EXTENSION_PCI(tts_1_X_fsc, 0x13c2, 0x0000),
 	MAKE_EXTENSION_PCI(ttt_1_X,     0x13c2, 0x0001),
diff --git a/drivers/media/pci/ttpci/av7110.h b/drivers/media/pci/ttpci/av7110.h
index 824c1e262fbb..347827925c14 100644
--- a/drivers/media/pci/ttpci/av7110.h
+++ b/drivers/media/pci/ttpci/av7110.h
@@ -177,7 +177,7 @@ struct av7110 {
 
 	/* CA */
 
-	ca_slot_info_t		ci_slot[2];
+	struct ca_slot_info	ci_slot[2];
 
 	enum av7110_video_mode	vidmode;
 	struct dmxdev		dmxdev;
diff --git a/drivers/media/pci/ttpci/av7110_ca.c b/drivers/media/pci/ttpci/av7110_ca.c
index f64723aea56b..1fe49171d823 100644
--- a/drivers/media/pci/ttpci/av7110_ca.c
+++ b/drivers/media/pci/ttpci/av7110_ca.c
@@ -119,7 +119,7 @@ static void ci_ll_release(struct dvb_ringbuffer *cirbuf, struct dvb_ringbuffer *
 }
 
 static int ci_ll_reset(struct dvb_ringbuffer *cibuf, struct file *file,
-		       int slots, ca_slot_info_t *slot)
+		       int slots, struct ca_slot_info *slot)
 {
 	int i;
 	int len = 0;
@@ -264,7 +264,7 @@ static int dvb_ca_ioctl(struct file *file, unsigned int cmd, void *parg)
 		break;
 	case CA_GET_CAP:
 	{
-		ca_caps_t cap;
+		struct ca_caps cap;
 
 		cap.slot_num = 2;
 		cap.slot_type = (FW_CI_LL_SUPPORT(av7110->arm_app) ?
@@ -277,7 +277,7 @@ static int dvb_ca_ioctl(struct file *file, unsigned int cmd, void *parg)
 
 	case CA_GET_SLOT_INFO:
 	{
-		ca_slot_info_t *info=(ca_slot_info_t *)parg;
+		struct ca_slot_info *info=(struct ca_slot_info *)parg;
 
 		if (info->num < 0 || info->num > 1) {
 			mutex_unlock(&av7110->ioctl_mutex);
@@ -286,7 +286,7 @@ static int dvb_ca_ioctl(struct file *file, unsigned int cmd, void *parg)
 		av7110->ci_slot[info->num].num = info->num;
 		av7110->ci_slot[info->num].type = FW_CI_LL_SUPPORT(av7110->arm_app) ?
 							CA_CI_LINK : CA_CI;
-		memcpy(info, &av7110->ci_slot[info->num], sizeof(ca_slot_info_t));
+		memcpy(info, &av7110->ci_slot[info->num], sizeof(struct ca_slot_info));
 		break;
 	}
 
@@ -298,7 +298,7 @@ static int dvb_ca_ioctl(struct file *file, unsigned int cmd, void *parg)
 
 	case CA_GET_DESCR_INFO:
 	{
-		ca_descr_info_t info;
+		struct ca_descr_info info;
 
 		info.num = 16;
 		info.type = CA_ECD;
@@ -308,7 +308,7 @@ static int dvb_ca_ioctl(struct file *file, unsigned int cmd, void *parg)
 
 	case CA_SET_DESCR:
 	{
-		ca_descr_t *descr = (ca_descr_t*) parg;
+		struct ca_descr *descr = (struct ca_descr*) parg;
 
 		if (descr->index >= 16 || descr->parity > 1) {
 			mutex_unlock(&av7110->ioctl_mutex);
diff --git a/drivers/media/pci/ttpci/av7110_v4l.c b/drivers/media/pci/ttpci/av7110_v4l.c
index 397fe146dedd..e4cf42c32284 100644
--- a/drivers/media/pci/ttpci/av7110_v4l.c
+++ b/drivers/media/pci/ttpci/av7110_v4l.c
@@ -218,7 +218,7 @@ static struct saa7146_standard analog_standard[];
 static struct saa7146_standard dvb_standard[];
 static struct saa7146_standard standard[];
 
-static struct v4l2_audio msp3400_v4l2_audio = {
+static const struct v4l2_audio msp3400_v4l2_audio = {
 	.index = 0,
 	.name = "Television",
 	.capability = V4L2_AUDCAP_STEREO
diff --git a/drivers/media/pci/ttpci/budget-av.c b/drivers/media/pci/ttpci/budget-av.c
index dc7be8fac9a3..ac83fff9fe0b 100644
--- a/drivers/media/pci/ttpci/budget-av.c
+++ b/drivers/media/pci/ttpci/budget-av.c
@@ -1567,7 +1567,7 @@ MAKE_BUDGET_INFO(cin1200c, "Terratec Cinergy 1200 DVB-C", BUDGET_CIN1200C);
 MAKE_BUDGET_INFO(cin1200cmk3, "Terratec Cinergy 1200 DVB-C MK3", BUDGET_CIN1200C_MK3);
 MAKE_BUDGET_INFO(cin1200t, "Terratec Cinergy 1200 DVB-T", BUDGET_CIN1200T);
 
-static struct pci_device_id pci_tbl[] = {
+static const struct pci_device_id pci_tbl[] = {
 	MAKE_EXTENSION_PCI(knc1s, 0x1131, 0x4f56),
 	MAKE_EXTENSION_PCI(knc1s, 0x1131, 0x0010),
 	MAKE_EXTENSION_PCI(knc1s, 0x1894, 0x0010),
diff --git a/drivers/media/pci/ttpci/budget-ci.c b/drivers/media/pci/ttpci/budget-ci.c
index 11b9227307bf..57af11804fd6 100644
--- a/drivers/media/pci/ttpci/budget-ci.c
+++ b/drivers/media/pci/ttpci/budget-ci.c
@@ -158,14 +158,15 @@ static void msp430_ir_interrupt(unsigned long data)
 		return;
 
 	if (budget_ci->ir.full_rc5) {
-		rc_keydown(dev, RC_TYPE_RC5,
+		rc_keydown(dev, RC_PROTO_RC5,
 			   RC_SCANCODE_RC5(budget_ci->ir.rc5_device, budget_ci->ir.ir_key),
 			   !!(command & 0x20));
 		return;
 	}
 
 	/* FIXME: We should generate complete scancodes for all devices */
-	rc_keydown(dev, RC_TYPE_UNKNOWN, budget_ci->ir.ir_key, !!(command & 0x20));
+	rc_keydown(dev, RC_PROTO_UNKNOWN, budget_ci->ir.ir_key,
+		   !!(command & 0x20));
 }
 
 static int msp430_ir_init(struct budget_ci *budget_ci)
@@ -186,7 +187,7 @@ static int msp430_ir_init(struct budget_ci *budget_ci)
 		 "pci-%s/ir0", pci_name(saa->pci));
 
 	dev->driver_name = MODULE_NAME;
-	dev->input_name = budget_ci->ir.name;
+	dev->device_name = budget_ci->ir.name;
 	dev->input_phys = budget_ci->ir.phys;
 	dev->input_id.bustype = BUS_PCI;
 	dev->input_id.version = 1;
@@ -1538,7 +1539,7 @@ MAKE_BUDGET_INFO(ttc1501, "TT-Budget C-1501 PCI", BUDGET_TT);
 MAKE_BUDGET_INFO(tt3200, "TT-Budget S2-3200 PCI", BUDGET_TT);
 MAKE_BUDGET_INFO(ttbs1500b, "TT-Budget S-1500B PCI", BUDGET_TT);
 
-static struct pci_device_id pci_tbl[] = {
+static const struct pci_device_id pci_tbl[] = {
 	MAKE_EXTENSION_PCI(ttbci, 0x13c2, 0x100c),
 	MAKE_EXTENSION_PCI(ttbci, 0x13c2, 0x100f),
 	MAKE_EXTENSION_PCI(ttbcci, 0x13c2, 0x1010),
diff --git a/drivers/media/pci/ttpci/budget-patch.c b/drivers/media/pci/ttpci/budget-patch.c
index 442992372008..a738018cdca8 100644
--- a/drivers/media/pci/ttpci/budget-patch.c
+++ b/drivers/media/pci/ttpci/budget-patch.c
@@ -45,7 +45,7 @@ static struct saa7146_extension budget_extension;
 MAKE_BUDGET_INFO(ttbp, "TT-Budget/Patch DVB-S 1.x PCI", BUDGET_PATCH);
 //MAKE_BUDGET_INFO(satel,"TT-Budget/Patch SATELCO PCI", BUDGET_TT_HW_DISEQC);
 
-static struct pci_device_id pci_tbl[] = {
+static const struct pci_device_id pci_tbl[] = {
 	MAKE_EXTENSION_PCI(ttbp,0x13c2, 0x0000),
 //        MAKE_EXTENSION_PCI(satel, 0x13c2, 0x1013),
 	{
diff --git a/drivers/media/pci/ttpci/budget.c b/drivers/media/pci/ttpci/budget.c
index 81fe35cedd10..f59eadb7a5eb 100644
--- a/drivers/media/pci/ttpci/budget.c
+++ b/drivers/media/pci/ttpci/budget.c
@@ -845,7 +845,7 @@ MAKE_BUDGET_INFO(fsact1, "Fujitsu Siemens Activy Budget-T PCI (rev AL/ALPS TDHD1
 MAKE_BUDGET_INFO(omicom, "Omicom S2 PCI", BUDGET_TT);
 MAKE_BUDGET_INFO(sylt,   "Philips Semi Sylt PCI", BUDGET_TT_HW_DISEQC);
 
-static struct pci_device_id pci_tbl[] = {
+static const struct pci_device_id pci_tbl[] = {
 	MAKE_EXTENSION_PCI(ttbs,  0x13c2, 0x1003),
 	MAKE_EXTENSION_PCI(ttbc,  0x13c2, 0x1004),
 	MAKE_EXTENSION_PCI(ttbt,  0x13c2, 0x1005),
diff --git a/drivers/media/pci/tw68/tw68-video.c b/drivers/media/pci/tw68/tw68-video.c
index 58c4dd75bfa1..8c1f4a049764 100644
--- a/drivers/media/pci/tw68/tw68-video.c
+++ b/drivers/media/pci/tw68/tw68-video.c
@@ -916,7 +916,7 @@ static const struct v4l2_ioctl_ops video_ioctl_ops = {
 #endif
 };
 
-static struct video_device tw68_video_template = {
+static const struct video_device tw68_video_template = {
 	.name			= "tw68_video",
 	.fops			= &video_fops,
 	.ioctl_ops		= &video_ioctl_ops,
diff --git a/drivers/media/pci/zoran/zoran_card.c b/drivers/media/pci/zoran/zoran_card.c
index 4680f001653a..a6b9ebd20263 100644
--- a/drivers/media/pci/zoran/zoran_card.c
+++ b/drivers/media/pci/zoran/zoran_card.c
@@ -130,7 +130,7 @@ MODULE_VERSION(ZORAN_VERSION);
 	.vendor = PCI_VENDOR_ID_ZORAN, .device = PCI_DEVICE_ID_ZORAN_36057, \
 	.subvendor = (subven), .subdevice = (subdev), .driver_data = (data) }
 
-static struct pci_device_id zr36067_pci_tbl[] = {
+static const struct pci_device_id zr36067_pci_tbl[] = {
 	ZR_DEVICE(PCI_VENDOR_ID_MIRO, PCI_DEVICE_ID_MIRO_DC10PLUS, DC10plus),
 	ZR_DEVICE(PCI_VENDOR_ID_MIRO, PCI_DEVICE_ID_MIRO_DC30PLUS, DC30plus),
 	ZR_DEVICE(PCI_VENDOR_ID_ELECTRONICDESIGNGMBH, PCI_DEVICE_ID_LML_33R10, LML33R10),
diff --git a/drivers/media/platform/Kconfig b/drivers/media/platform/Kconfig
index fb1fa0b82077..7e7cc49b8674 100644
--- a/drivers/media/platform/Kconfig
+++ b/drivers/media/platform/Kconfig
@@ -76,7 +76,8 @@ config VIDEO_M32R_AR_M64278
 
 config VIDEO_MUX
 	tristate "Video Multiplexer"
-	depends on OF && VIDEO_V4L2_SUBDEV_API && MEDIA_CONTROLLER
+	select MULTIPLEXER
+	depends on VIDEO_V4L2 && OF && VIDEO_V4L2_SUBDEV_API && MEDIA_CONTROLLER
 	select REGMAP
 	help
 	  This driver provides support for N:1 video bus multiplexers.
@@ -109,6 +110,13 @@ config VIDEO_PXA27x
 	---help---
 	  This is a v4l2 driver for the PXA27x Quick Capture Interface
 
+config VIDEO_QCOM_CAMSS
+	tristate "Qualcomm 8x16 V4L2 Camera Subsystem driver"
+	depends on VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
+	depends on (ARCH_QCOM && IOMMU_DMA) || COMPILE_TEST
+	select VIDEOBUF2_DMA_SG
+	select V4L2_FWNODE
+
 config VIDEO_S3C_CAMIF
 	tristate "Samsung S3C24XX/S3C64XX SoC Camera Interface driver"
 	depends on VIDEO_V4L2 && I2C && VIDEO_V4L2_SUBDEV_API
@@ -536,6 +544,17 @@ menuconfig CEC_PLATFORM_DRIVERS
 
 if CEC_PLATFORM_DRIVERS
 
+config VIDEO_MESON_AO_CEC
+	tristate "Amlogic Meson AO CEC driver"
+	depends on ARCH_MESON || COMPILE_TEST
+	select CEC_CORE
+	select CEC_NOTIFIER
+	---help---
+	  This is a driver for Amlogic Meson SoCs AO CEC interface. It uses the
+	  generic CEC framework interface.
+	  CEC bus is present in the HDMI connector and enables communication
+	  between compatible devices.
+
 config VIDEO_SAMSUNG_S5P_CEC
        tristate "Samsung S5P CEC driver"
        depends on PLAT_S5P || ARCH_EXYNOS || COMPILE_TEST
diff --git a/drivers/media/platform/Makefile b/drivers/media/platform/Makefile
index 9beadc760467..c1ef946bf032 100644
--- a/drivers/media/platform/Makefile
+++ b/drivers/media/platform/Makefile
@@ -85,4 +85,8 @@ obj-$(CONFIG_VIDEO_MEDIATEK_MDP)	+= mtk-mdp/
 
 obj-$(CONFIG_VIDEO_MEDIATEK_JPEG)	+= mtk-jpeg/
 
+obj-$(CONFIG_VIDEO_QCOM_CAMSS)		+= qcom/camss-8x16/
+
 obj-$(CONFIG_VIDEO_QCOM_VENUS)		+= qcom/venus/
+
+obj-y					+= meson/
diff --git a/drivers/media/platform/am437x/am437x-vpfe.c b/drivers/media/platform/am437x/am437x-vpfe.c
index 466aba8b0e00..dfcc484cab89 100644
--- a/drivers/media/platform/am437x/am437x-vpfe.c
+++ b/drivers/media/platform/am437x/am437x-vpfe.c
@@ -2490,8 +2490,8 @@ vpfe_get_pdata(struct platform_device *pdev)
 
 		rem = of_graph_get_remote_port_parent(endpoint);
 		if (!rem) {
-			dev_err(&pdev->dev, "Remote device at %s not found\n",
-				endpoint->full_name);
+			dev_err(&pdev->dev, "Remote device at %pOF not found\n",
+				endpoint);
 			goto done;
 		}
 
diff --git a/drivers/media/platform/atmel/atmel-isc.c b/drivers/media/platform/atmel/atmel-isc.c
index d6534252cdcd..d7103c5f92c3 100644
--- a/drivers/media/platform/atmel/atmel-isc.c
+++ b/drivers/media/platform/atmel/atmel-isc.c
@@ -873,7 +873,7 @@ static void isc_buffer_queue(struct vb2_buffer *vb)
 	spin_unlock_irqrestore(&isc->dma_queue_lock, flags);
 }
 
-static struct vb2_ops isc_vb2_ops = {
+static const struct vb2_ops isc_vb2_ops = {
 	.queue_setup		= isc_queue_setup,
 	.wait_prepare		= vb2_ops_wait_prepare,
 	.wait_finish		= vb2_ops_wait_finish,
@@ -1700,8 +1700,8 @@ static int isc_parse_dt(struct device *dev, struct isc_device *isc)
 
 		rem = of_graph_get_remote_port_parent(epn);
 		if (!rem) {
-			dev_notice(dev, "Remote device at %s not found\n",
-				   of_node_full_name(epn));
+			dev_notice(dev, "Remote device at %pOF not found\n",
+				   epn);
 			continue;
 		}
 
diff --git a/drivers/media/platform/blackfin/bfin_capture.c b/drivers/media/platform/blackfin/bfin_capture.c
index 1c5166df46f5..41f179117fb0 100644
--- a/drivers/media/platform/blackfin/bfin_capture.c
+++ b/drivers/media/platform/blackfin/bfin_capture.c
@@ -375,7 +375,7 @@ static void bcap_stop_streaming(struct vb2_queue *vq)
 	}
 }
 
-static struct vb2_ops bcap_video_qops = {
+static const struct vb2_ops bcap_video_qops = {
 	.queue_setup            = bcap_queue_setup,
 	.buf_prepare            = bcap_buffer_prepare,
 	.buf_cleanup            = bcap_buffer_cleanup,
@@ -769,7 +769,7 @@ static const struct v4l2_ioctl_ops bcap_ioctl_ops = {
 	.vidioc_log_status       = bcap_log_status,
 };
 
-static struct v4l2_file_operations bcap_fops = {
+static const struct v4l2_file_operations bcap_fops = {
 	.owner = THIS_MODULE,
 	.open = v4l2_fh_open,
 	.release = vb2_fop_release,
diff --git a/drivers/media/platform/coda/coda-bit.c b/drivers/media/platform/coda/coda-bit.c
index bba1eb43b5d8..291c40933935 100644
--- a/drivers/media/platform/coda/coda-bit.c
+++ b/drivers/media/platform/coda/coda-bit.c
@@ -394,7 +394,8 @@ static int coda_alloc_framebuffers(struct coda_ctx *ctx,
 	int i;
 
 	if (ctx->codec->src_fourcc == V4L2_PIX_FMT_H264 ||
-	    ctx->codec->dst_fourcc == V4L2_PIX_FMT_H264) {
+	    ctx->codec->dst_fourcc == V4L2_PIX_FMT_H264 ||
+	    ctx->codec->dst_fourcc == V4L2_PIX_FMT_MPEG4) {
 		width = round_up(q_data->width, 16);
 		height = round_up(q_data->height, 16);
 	} else {
@@ -702,6 +703,8 @@ static u32 coda_supported_firmwares[] = {
 	CODA_FIRMWARE_VERNUM(CODA_DX6, 2, 2, 5),
 	CODA_FIRMWARE_VERNUM(CODA_7541, 1, 4, 50),
 	CODA_FIRMWARE_VERNUM(CODA_960, 2, 1, 5),
+	CODA_FIRMWARE_VERNUM(CODA_960, 2, 3, 10),
+	CODA_FIRMWARE_VERNUM(CODA_960, 3, 1, 1),
 };
 
 static bool coda_firmware_supported(u32 vernum)
@@ -1006,7 +1009,7 @@ static int coda_start_encoding(struct coda_ctx *ctx)
 			break;
 		}
 		coda_write(dev, value, CODA_CMD_ENC_SEQ_SLICE_MODE);
-		value = ctx->params.gop_size & CODA_GOP_SIZE_MASK;
+		value = ctx->params.gop_size;
 		coda_write(dev, value, CODA_CMD_ENC_SEQ_GOP_SIZE);
 	}
 
@@ -1250,7 +1253,8 @@ static int coda_prepare_encode(struct coda_ctx *ctx)
 	force_ipicture = ctx->params.force_ipicture;
 	if (force_ipicture)
 		ctx->params.force_ipicture = false;
-	else if ((src_buf->sequence % ctx->params.gop_size) == 0)
+	else if (ctx->params.gop_size != 0 &&
+		 (src_buf->sequence % ctx->params.gop_size) == 0)
 		force_ipicture = 1;
 
 	/*
@@ -1411,6 +1415,7 @@ static void coda_finish_encode(struct coda_ctx *ctx)
 	}
 
 	dst_buf->vb2_buf.timestamp = src_buf->vb2_buf.timestamp;
+	dst_buf->field = src_buf->field;
 	dst_buf->flags &= ~V4L2_BUF_FLAG_TSTAMP_SRC_MASK;
 	dst_buf->flags |=
 		src_buf->flags & V4L2_BUF_FLAG_TSTAMP_SRC_MASK;
@@ -1634,9 +1639,6 @@ static int __coda_start_decoding(struct coda_ctx *ctx)
 	ctx->frm_dis_flg = 0;
 	coda_write(dev, 0, CODA_REG_BIT_FRM_DIS_FLG(ctx->reg_idx));
 
-	coda_write(dev, CODA_BIT_DEC_SEQ_INIT_ESCAPE,
-			CODA_REG_BIT_BIT_STREAM_PARAM);
-
 	coda_write(dev, bitstream_buf, CODA_CMD_DEC_SEQ_BB_START);
 	coda_write(dev, bitstream_size / 1024, CODA_CMD_DEC_SEQ_BB_SIZE);
 	val = 0;
@@ -1652,6 +1654,10 @@ static int __coda_start_decoding(struct coda_ctx *ctx)
 		ctx->params.codec_mode_aux = CODA_MP4_AUX_MPEG4;
 	else
 		ctx->params.codec_mode_aux = 0;
+	if (src_fourcc == V4L2_PIX_FMT_MPEG4) {
+		coda_write(dev, CODA_MP4_CLASS_MPEG4,
+			   CODA_CMD_DEC_SEQ_MP4_ASP_CLASS);
+	}
 	if (src_fourcc == V4L2_PIX_FMT_H264) {
 		if (dev->devtype->product == CODA_7541) {
 			coda_write(dev, ctx->psbuf.paddr,
@@ -1667,18 +1673,18 @@ static int __coda_start_decoding(struct coda_ctx *ctx)
 	if (dev->devtype->product != CODA_960)
 		coda_write(dev, 0, CODA_CMD_DEC_SEQ_SRC_SIZE);
 
-	if (coda_command_sync(ctx, CODA_COMMAND_SEQ_INIT)) {
+	ctx->bit_stream_param = CODA_BIT_DEC_SEQ_INIT_ESCAPE;
+	ret = coda_command_sync(ctx, CODA_COMMAND_SEQ_INIT);
+	ctx->bit_stream_param = 0;
+	if (ret) {
 		v4l2_err(&dev->v4l2_dev, "CODA_COMMAND_SEQ_INIT timeout\n");
-		coda_write(dev, 0, CODA_REG_BIT_BIT_STREAM_PARAM);
-		return -ETIMEDOUT;
+		return ret;
 	}
 	ctx->initialized = 1;
 
 	/* Update kfifo out pointer from coda bitstream read pointer */
 	coda_kfifo_sync_from_device(ctx);
 
-	coda_write(dev, 0, CODA_REG_BIT_BIT_STREAM_PARAM);
-
 	if (coda_read(dev, CODA_RET_DEC_SEQ_SUCCESS) == 0) {
 		v4l2_err(&dev->v4l2_dev,
 			"CODA_COMMAND_SEQ_INIT failed, error code = %d\n",
@@ -2153,6 +2159,7 @@ static void coda_finish_decode(struct coda_ctx *ctx)
 		dst_buf = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
 		dst_buf->sequence = ctx->osequence++;
 
+		dst_buf->field = V4L2_FIELD_NONE;
 		dst_buf->flags &= ~(V4L2_BUF_FLAG_KEYFRAME |
 					     V4L2_BUF_FLAG_PFRAME |
 					     V4L2_BUF_FLAG_BFRAME);
diff --git a/drivers/media/platform/coda/coda-common.c b/drivers/media/platform/coda/coda-common.c
index 829c7895a98a..15eb5dc4dff9 100644
--- a/drivers/media/platform/coda/coda-common.c
+++ b/drivers/media/platform/coda/coda-common.c
@@ -73,7 +73,7 @@ MODULE_PARM_DESC(disable_vdoa, "Disable Video Data Order Adapter tiled to raster
 
 static int enable_bwb = 0;
 module_param(enable_bwb, int, 0644);
-MODULE_PARM_DESC(enable_bwb, "Enable BWB unit, may crash on certain streams");
+MODULE_PARM_DESC(enable_bwb, "Enable BWB unit for decoding, may crash on certain streams");
 
 void coda_write(struct coda_dev *dev, u32 data, u32 reg)
 {
@@ -714,9 +714,10 @@ static int coda_s_fmt(struct coda_ctx *ctx, struct v4l2_format *f,
 		ctx->tiled_map_type = GDI_TILED_FRAME_MB_RASTER_MAP;
 		break;
 	case V4L2_PIX_FMT_NV12:
-		ctx->tiled_map_type = GDI_TILED_FRAME_MB_RASTER_MAP;
-		if (!disable_tiling)
+		if (!disable_tiling) {
+			ctx->tiled_map_type = GDI_TILED_FRAME_MB_RASTER_MAP;
 			break;
+		}
 		/* else fall through */
 	case V4L2_PIX_FMT_YUV420:
 	case V4L2_PIX_FMT_YVU420:
@@ -932,7 +933,7 @@ static int coda_encoder_cmd(struct file *file, void *fh,
 	ctx->bit_stream_param |= CODA_BIT_STREAM_END_FLAG;
 
 	/* If there is no buffer in flight, wake up */
-	if (ctx->qsequence == ctx->osequence) {
+	if (!ctx->streamon_out || ctx->qsequence == ctx->osequence) {
 		dst_vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx,
 					 V4L2_BUF_TYPE_VIDEO_CAPTURE);
 		dst_vq->last_buffer_dequeued = true;
@@ -1683,12 +1684,23 @@ static int coda_s_ctrl(struct v4l2_ctrl *ctrl)
 		ctx->params.h264_deblk_enabled = (ctrl->val ==
 				V4L2_MPEG_VIDEO_H264_LOOP_FILTER_MODE_ENABLED);
 		break;
+	case V4L2_CID_MPEG_VIDEO_H264_PROFILE:
+		/* TODO: switch between baseline and constrained baseline */
+		ctx->params.h264_profile_idc = 66;
+		break;
+	case V4L2_CID_MPEG_VIDEO_H264_LEVEL:
+		/* nothing to do, this is set by the encoder */
+		break;
 	case V4L2_CID_MPEG_VIDEO_MPEG4_I_FRAME_QP:
 		ctx->params.mpeg4_intra_qp = ctrl->val;
 		break;
 	case V4L2_CID_MPEG_VIDEO_MPEG4_P_FRAME_QP:
 		ctx->params.mpeg4_inter_qp = ctrl->val;
 		break;
+	case V4L2_CID_MPEG_VIDEO_MPEG4_PROFILE:
+	case V4L2_CID_MPEG_VIDEO_MPEG4_LEVEL:
+		/* nothing to do, these are fixed */
+		break;
 	case V4L2_CID_MPEG_VIDEO_MULTI_SLICE_MODE:
 		ctx->params.slice_mode = ctrl->val;
 		break;
@@ -1734,10 +1746,12 @@ static const struct v4l2_ctrl_ops coda_ctrl_ops = {
 
 static void coda_encode_ctrls(struct coda_ctx *ctx)
 {
+	int max_gop_size = (ctx->dev->devtype->product == CODA_DX6) ? 60 : 99;
+
 	v4l2_ctrl_new_std(&ctx->ctrls, &coda_ctrl_ops,
 		V4L2_CID_MPEG_VIDEO_BITRATE, 0, 32767000, 1000, 0);
 	v4l2_ctrl_new_std(&ctx->ctrls, &coda_ctrl_ops,
-		V4L2_CID_MPEG_VIDEO_GOP_SIZE, 1, 60, 1, 16);
+		V4L2_CID_MPEG_VIDEO_GOP_SIZE, 0, max_gop_size, 1, 16);
 	v4l2_ctrl_new_std(&ctx->ctrls, &coda_ctrl_ops,
 		V4L2_CID_MPEG_VIDEO_H264_I_FRAME_QP, 0, 51, 1, 25);
 	v4l2_ctrl_new_std(&ctx->ctrls, &coda_ctrl_ops,
@@ -1756,11 +1770,47 @@ static void coda_encode_ctrls(struct coda_ctx *ctx)
 		V4L2_CID_MPEG_VIDEO_H264_LOOP_FILTER_MODE,
 		V4L2_MPEG_VIDEO_H264_LOOP_FILTER_MODE_DISABLED, 0x0,
 		V4L2_MPEG_VIDEO_H264_LOOP_FILTER_MODE_ENABLED);
+	v4l2_ctrl_new_std_menu(&ctx->ctrls, &coda_ctrl_ops,
+		V4L2_CID_MPEG_VIDEO_H264_PROFILE,
+		V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE, 0x0,
+		V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE);
+	if (ctx->dev->devtype->product == CODA_7541) {
+		v4l2_ctrl_new_std_menu(&ctx->ctrls, &coda_ctrl_ops,
+			V4L2_CID_MPEG_VIDEO_H264_LEVEL,
+			V4L2_MPEG_VIDEO_H264_LEVEL_3_1,
+			~((1 << V4L2_MPEG_VIDEO_H264_LEVEL_2_0) |
+			  (1 << V4L2_MPEG_VIDEO_H264_LEVEL_3_0) |
+			  (1 << V4L2_MPEG_VIDEO_H264_LEVEL_3_1)),
+			V4L2_MPEG_VIDEO_H264_LEVEL_3_1);
+	}
+	if (ctx->dev->devtype->product == CODA_960) {
+		v4l2_ctrl_new_std_menu(&ctx->ctrls, &coda_ctrl_ops,
+			V4L2_CID_MPEG_VIDEO_H264_LEVEL,
+			V4L2_MPEG_VIDEO_H264_LEVEL_4_0,
+			~((1 << V4L2_MPEG_VIDEO_H264_LEVEL_2_0) |
+			  (1 << V4L2_MPEG_VIDEO_H264_LEVEL_3_0) |
+			  (1 << V4L2_MPEG_VIDEO_H264_LEVEL_3_1) |
+			  (1 << V4L2_MPEG_VIDEO_H264_LEVEL_3_2) |
+			  (1 << V4L2_MPEG_VIDEO_H264_LEVEL_4_0)),
+			V4L2_MPEG_VIDEO_H264_LEVEL_4_0);
+	}
 	v4l2_ctrl_new_std(&ctx->ctrls, &coda_ctrl_ops,
 		V4L2_CID_MPEG_VIDEO_MPEG4_I_FRAME_QP, 1, 31, 1, 2);
 	v4l2_ctrl_new_std(&ctx->ctrls, &coda_ctrl_ops,
 		V4L2_CID_MPEG_VIDEO_MPEG4_P_FRAME_QP, 1, 31, 1, 2);
 	v4l2_ctrl_new_std_menu(&ctx->ctrls, &coda_ctrl_ops,
+		V4L2_CID_MPEG_VIDEO_MPEG4_PROFILE,
+		V4L2_MPEG_VIDEO_MPEG4_PROFILE_SIMPLE, 0x0,
+		V4L2_MPEG_VIDEO_MPEG4_PROFILE_SIMPLE);
+	if (ctx->dev->devtype->product == CODA_7541 ||
+	    ctx->dev->devtype->product == CODA_960) {
+		v4l2_ctrl_new_std_menu(&ctx->ctrls, &coda_ctrl_ops,
+			V4L2_CID_MPEG_VIDEO_MPEG4_LEVEL,
+			V4L2_MPEG_VIDEO_MPEG4_LEVEL_5,
+			~(1 << V4L2_MPEG_VIDEO_MPEG4_LEVEL_5),
+			V4L2_MPEG_VIDEO_MPEG4_LEVEL_5);
+	}
+	v4l2_ctrl_new_std_menu(&ctx->ctrls, &coda_ctrl_ops,
 		V4L2_CID_MPEG_VIDEO_MULTI_SLICE_MODE,
 		V4L2_MPEG_VIDEO_MULTI_SICE_MODE_MAX_BYTES, 0x0,
 		V4L2_MPEG_VIDEO_MULTI_SLICE_MODE_SINGLE);
@@ -1938,7 +1988,13 @@ static int coda_open(struct file *file)
 	ctx->idx = idx;
 	switch (dev->devtype->product) {
 	case CODA_960:
-		if (enable_bwb)
+		/*
+		 * Enabling the BWB when decoding can hang the firmware with
+		 * certain streams. The issue was tracked as ENGR00293425 by
+		 * Freescale. As a workaround, disable BWB for all decoders.
+		 * The enable_bwb module parameter allows to override this.
+		 */
+		if (enable_bwb || ctx->inst_type == CODA_INST_ENCODER)
 			ctx->frame_mem_ctrl = CODA9_FRAME_ENABLE_BWB;
 		/* fallthrough */
 	case CODA_7541:
@@ -2142,7 +2198,8 @@ static int coda_hw_init(struct coda_dev *dev)
 			   CODA_REG_BIT_STREAM_CTRL);
 	}
 	if (dev->devtype->product == CODA_960)
-		coda_write(dev, 1 << 12, CODA_REG_BIT_FRAME_MEM_CTRL);
+		coda_write(dev, CODA9_FRAME_ENABLE_BWB,
+				CODA_REG_BIT_FRAME_MEM_CTRL);
 	else
 		coda_write(dev, 0, CODA_REG_BIT_FRAME_MEM_CTRL);
 
@@ -2386,11 +2443,11 @@ static const struct coda_devtype coda_devdata[] = {
 		.num_vdevs    = ARRAY_SIZE(coda9_video_devices),
 		.workbuf_size = 80 * 1024,
 		.tempbuf_size = 204 * 1024,
-		.iram_size    = 0x20000,
+		.iram_size    = 0x1f000, /* leave 4k for suspend code */
 	},
 };
 
-static struct platform_device_id coda_platform_ids[] = {
+static const struct platform_device_id coda_platform_ids[] = {
 	{ .name = "coda-imx27", .driver_data = CODA_IMX27 },
 	{ /* sentinel */ }
 };
@@ -2470,7 +2527,8 @@ static int coda_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	dev->rstc = devm_reset_control_get_optional(&pdev->dev, NULL);
+	dev->rstc = devm_reset_control_get_optional_exclusive(&pdev->dev,
+							      NULL);
 	if (IS_ERR(dev->rstc)) {
 		ret = PTR_ERR(dev->rstc);
 		dev_err(&pdev->dev, "failed get reset control: %d\n", ret);
diff --git a/drivers/media/platform/coda/coda_regs.h b/drivers/media/platform/coda/coda_regs.h
index 77ee46a93427..38df5fd9a2fa 100644
--- a/drivers/media/platform/coda/coda_regs.h
+++ b/drivers/media/platform/coda/coda_regs.h
@@ -158,6 +158,7 @@
 #define CODA_CMD_DEC_SEQ_PS_BB_START		0x194
 #define CODA_CMD_DEC_SEQ_PS_BB_SIZE		0x198
 #define CODA_CMD_DEC_SEQ_MP4_ASP_CLASS		0x19c
+#define		CODA_MP4_CLASS_MPEG4			0
 #define CODA_CMD_DEC_SEQ_X264_MV_EN		0x19c
 #define CODA_CMD_DEC_SEQ_SPP_CHUNK_SIZE		0x1a0
 
diff --git a/drivers/media/platform/coda/imx-vdoa.c b/drivers/media/platform/coda/imx-vdoa.c
index df9b71621420..8eb3e0c05473 100644
--- a/drivers/media/platform/coda/imx-vdoa.c
+++ b/drivers/media/platform/coda/imx-vdoa.c
@@ -314,6 +314,8 @@ static int vdoa_probe(struct platform_device *pdev)
 		return PTR_ERR(vdoa->regs);
 
 	res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
+	if (!res)
+		return -EINVAL;
 	vdoa->irq = devm_request_threaded_irq(&pdev->dev, res->start, NULL,
 					vdoa_irq_handler, IRQF_ONESHOT,
 					"vdoa", vdoa);
diff --git a/drivers/media/platform/davinci/vpbe.c b/drivers/media/platform/davinci/vpbe.c
index 3679b1e7b39e..7f6462562579 100644
--- a/drivers/media/platform/davinci/vpbe.c
+++ b/drivers/media/platform/davinci/vpbe.c
@@ -790,7 +790,7 @@ static void vpbe_deinitialize(struct device *dev, struct vpbe_device *vpbe_dev)
 	vpss_enable_clock(VPSS_VPBE_CLOCK, 0);
 }
 
-static struct vpbe_device_ops vpbe_dev_ops = {
+static const struct vpbe_device_ops vpbe_dev_ops = {
 	.g_cropcap = vpbe_g_cropcap,
 	.enum_outputs = vpbe_enum_outputs,
 	.set_output = vpbe_set_output,
diff --git a/drivers/media/platform/davinci/vpbe_display.c b/drivers/media/platform/davinci/vpbe_display.c
index a9bc0175e4d3..13d027031ff0 100644
--- a/drivers/media/platform/davinci/vpbe_display.c
+++ b/drivers/media/platform/davinci/vpbe_display.c
@@ -355,7 +355,7 @@ static void vpbe_stop_streaming(struct vb2_queue *vq)
 	spin_unlock_irqrestore(&disp->dma_queue_lock, flags);
 }
 
-static struct vb2_ops video_qops = {
+static const struct vb2_ops video_qops = {
 	.queue_setup = vpbe_buffer_queue_setup,
 	.wait_prepare = vb2_ops_wait_prepare,
 	.wait_finish = vb2_ops_wait_finish,
@@ -1275,7 +1275,7 @@ static const struct v4l2_ioctl_ops vpbe_ioctl_ops = {
 	.vidioc_enum_dv_timings	 = vpbe_display_enum_dv_timings,
 };
 
-static struct v4l2_file_operations vpbe_fops = {
+static const struct v4l2_file_operations vpbe_fops = {
 	.owner = THIS_MODULE,
 	.open = vpbe_display_open,
 	.release = vpbe_display_release,
diff --git a/drivers/media/platform/davinci/vpbe_osd.c b/drivers/media/platform/davinci/vpbe_osd.c
index df042e84a678..66449791c70c 100644
--- a/drivers/media/platform/davinci/vpbe_osd.c
+++ b/drivers/media/platform/davinci/vpbe_osd.c
@@ -37,7 +37,7 @@
 
 #define MODULE_NAME	"davinci-vpbe-osd"
 
-static struct platform_device_id vpbe_osd_devtype[] = {
+static const struct platform_device_id vpbe_osd_devtype[] = {
 	{
 		.name = DM644X_VPBE_OSD_SUBDEV_NAME,
 		.driver_data = VPBE_VERSION_1,
diff --git a/drivers/media/platform/davinci/vpbe_venc.c b/drivers/media/platform/davinci/vpbe_venc.c
index 8bfe90a24681..3a4e78595149 100644
--- a/drivers/media/platform/davinci/vpbe_venc.c
+++ b/drivers/media/platform/davinci/vpbe_venc.c
@@ -36,7 +36,7 @@
 
 #define MODULE_NAME	"davinci-vpbe-venc"
 
-static struct platform_device_id vpbe_venc_devtype[] = {
+static const struct platform_device_id vpbe_venc_devtype[] = {
 	{
 		.name = DM644X_VPBE_VENC_SUBDEV_NAME,
 		.driver_data = VPBE_VERSION_1,
diff --git a/drivers/media/platform/davinci/vpfe_capture.c b/drivers/media/platform/davinci/vpfe_capture.c
index b1bf4a7e8eb7..6792da16d9c7 100644
--- a/drivers/media/platform/davinci/vpfe_capture.c
+++ b/drivers/media/platform/davinci/vpfe_capture.c
@@ -1288,7 +1288,7 @@ static void vpfe_videobuf_release(struct videobuf_queue *vq,
 	vb->state = VIDEOBUF_NEEDS_INIT;
 }
 
-static struct videobuf_queue_ops vpfe_videobuf_qops = {
+static const struct videobuf_queue_ops vpfe_videobuf_qops = {
 	.buf_setup      = vpfe_videobuf_setup,
 	.buf_prepare    = vpfe_videobuf_prepare,
 	.buf_queue      = vpfe_videobuf_queue,
diff --git a/drivers/media/platform/davinci/vpif_capture.c b/drivers/media/platform/davinci/vpif_capture.c
index 4be6554c56c5..0ef36cec21d1 100644
--- a/drivers/media/platform/davinci/vpif_capture.c
+++ b/drivers/media/platform/davinci/vpif_capture.c
@@ -312,7 +312,7 @@ static void vpif_stop_streaming(struct vb2_queue *vq)
 	spin_unlock_irqrestore(&common->irqlock, flags);
 }
 
-static struct vb2_ops video_qops = {
+static const struct vb2_ops video_qops = {
 	.queue_setup		= vpif_buffer_queue_setup,
 	.buf_prepare		= vpif_buffer_prepare,
 	.start_streaming	= vpif_start_streaming,
@@ -1344,7 +1344,7 @@ static const struct v4l2_ioctl_ops vpif_ioctl_ops = {
 };
 
 /* vpif file operations */
-static struct v4l2_file_operations vpif_fops = {
+static const struct v4l2_file_operations vpif_fops = {
 	.owner = THIS_MODULE,
 	.open = v4l2_fh_open,
 	.release = vb2_fop_release,
@@ -1397,9 +1397,9 @@ static int vpif_async_bound(struct v4l2_async_notifier *notifier,
 			vpif_obj.config->chan_config->inputs[i].subdev_name =
 				(char *)to_of_node(subdev->fwnode)->full_name;
 			vpif_dbg(2, debug,
-				 "%s: setting input %d subdev_name = %s\n",
+				 "%s: setting input %d subdev_name = %pOF\n",
 				 __func__, i,
-				 to_of_node(subdev->fwnode)->full_name);
+				 to_of_node(subdev->fwnode));
 			return 0;
 		}
 	}
@@ -1557,8 +1557,8 @@ vpif_capture_get_pdata(struct platform_device *pdev)
 			dev_err(&pdev->dev, "Could not parse the endpoint\n");
 			goto done;
 		}
-		dev_dbg(&pdev->dev, "Endpoint %s, bus_width = %d\n",
-			endpoint->full_name, bus_cfg.bus.parallel.bus_width);
+		dev_dbg(&pdev->dev, "Endpoint %pOF, bus_width = %d\n",
+			endpoint, bus_cfg.bus.parallel.bus_width);
 		flags = bus_cfg.bus.parallel.flags;
 
 		if (flags & V4L2_MBUS_HSYNC_ACTIVE_HIGH)
@@ -1569,13 +1569,13 @@ vpif_capture_get_pdata(struct platform_device *pdev)
 
 		rem = of_graph_get_remote_port_parent(endpoint);
 		if (!rem) {
-			dev_dbg(&pdev->dev, "Remote device at %s not found\n",
-				endpoint->full_name);
+			dev_dbg(&pdev->dev, "Remote device at %pOF not found\n",
+				endpoint);
 			goto done;
 		}
 
-		dev_dbg(&pdev->dev, "Remote device %s, %s found\n",
-			rem->name, rem->full_name);
+		dev_dbg(&pdev->dev, "Remote device %s, %pOF found\n",
+			rem->name, rem);
 		sdinfo->name = rem->full_name;
 
 		pdata->asd[i] = devm_kzalloc(&pdev->dev,
@@ -1593,9 +1593,11 @@ vpif_capture_get_pdata(struct platform_device *pdev)
 	}
 
 done:
-	pdata->asd_sizes[0] = i;
-	pdata->subdev_count = i;
-	pdata->card_name = "DA850/OMAP-L138 Video Capture";
+	if (pdata) {
+		pdata->asd_sizes[0] = i;
+		pdata->subdev_count = i;
+		pdata->card_name = "DA850/OMAP-L138 Video Capture";
+	}
 
 	return pdata;
 }
diff --git a/drivers/media/platform/davinci/vpif_display.c b/drivers/media/platform/davinci/vpif_display.c
index bf982bf86542..56fe4e5b396e 100644
--- a/drivers/media/platform/davinci/vpif_display.c
+++ b/drivers/media/platform/davinci/vpif_display.c
@@ -290,7 +290,7 @@ static void vpif_stop_streaming(struct vb2_queue *vq)
 	spin_unlock_irqrestore(&common->irqlock, flags);
 }
 
-static struct vb2_ops video_qops = {
+static const struct vb2_ops video_qops = {
 	.queue_setup		= vpif_buffer_queue_setup,
 	.wait_prepare		= vb2_ops_wait_prepare,
 	.wait_finish		= vb2_ops_wait_finish,
diff --git a/drivers/media/platform/exynos-gsc/gsc-m2m.c b/drivers/media/platform/exynos-gsc/gsc-m2m.c
index 33611a46ce35..2a2994ef15d5 100644
--- a/drivers/media/platform/exynos-gsc/gsc-m2m.c
+++ b/drivers/media/platform/exynos-gsc/gsc-m2m.c
@@ -747,7 +747,7 @@ static const struct v4l2_file_operations gsc_m2m_fops = {
 	.mmap		= gsc_m2m_mmap,
 };
 
-static struct v4l2_m2m_ops gsc_m2m_ops = {
+static const struct v4l2_m2m_ops gsc_m2m_ops = {
 	.device_run	= gsc_m2m_device_run,
 	.job_abort	= gsc_m2m_job_abort,
 };
diff --git a/drivers/media/platform/exynos4-is/fimc-is-i2c.c b/drivers/media/platform/exynos4-is/fimc-is-i2c.c
index 2f559663e51e..70dd4852b2b9 100644
--- a/drivers/media/platform/exynos4-is/fimc-is-i2c.c
+++ b/drivers/media/platform/exynos4-is/fimc-is-i2c.c
@@ -130,7 +130,7 @@ static int fimc_is_i2c_resume(struct device *dev)
 }
 #endif
 
-static struct dev_pm_ops fimc_is_i2c_pm_ops = {
+static const struct dev_pm_ops fimc_is_i2c_pm_ops = {
 	SET_RUNTIME_PM_OPS(fimc_is_i2c_runtime_suspend,
 					fimc_is_i2c_runtime_resume, NULL)
 	SET_SYSTEM_SLEEP_PM_OPS(fimc_is_i2c_suspend, fimc_is_i2c_resume)
diff --git a/drivers/media/platform/exynos4-is/fimc-is.c b/drivers/media/platform/exynos4-is/fimc-is.c
index 340d906db370..5ddb2321e9e4 100644
--- a/drivers/media/platform/exynos4-is/fimc-is.c
+++ b/drivers/media/platform/exynos4-is/fimc-is.c
@@ -174,8 +174,8 @@ static int fimc_is_parse_sensor_config(struct fimc_is *is, unsigned int index,
 
 	sensor->drvdata = fimc_is_sensor_get_drvdata(node);
 	if (!sensor->drvdata) {
-		dev_err(&is->pdev->dev, "no driver data found for: %s\n",
-							 node->full_name);
+		dev_err(&is->pdev->dev, "no driver data found for: %pOF\n",
+							 node);
 		return -EINVAL;
 	}
 
@@ -191,8 +191,8 @@ static int fimc_is_parse_sensor_config(struct fimc_is *is, unsigned int index,
 	/* Use MIPI-CSIS channel id to determine the ISP I2C bus index. */
 	ret = of_property_read_u32(port, "reg", &tmp);
 	if (ret < 0) {
-		dev_err(&is->pdev->dev, "reg property not found at: %s\n",
-							 port->full_name);
+		dev_err(&is->pdev->dev, "reg property not found at: %pOF\n",
+							 port);
 		of_node_put(port);
 		return ret;
 	}
diff --git a/drivers/media/platform/exynos4-is/fimc-isp.c b/drivers/media/platform/exynos4-is/fimc-isp.c
index 8efe9160ab34..fd793d3ac072 100644
--- a/drivers/media/platform/exynos4-is/fimc-isp.c
+++ b/drivers/media/platform/exynos4-is/fimc-isp.c
@@ -433,7 +433,7 @@ static const struct v4l2_subdev_core_ops fimc_is_core_ops = {
 	.s_power = fimc_isp_subdev_s_power,
 };
 
-static struct v4l2_subdev_ops fimc_is_subdev_ops = {
+static const struct v4l2_subdev_ops fimc_is_subdev_ops = {
 	.core = &fimc_is_core_ops,
 	.video = &fimc_is_subdev_video_ops,
 	.pad = &fimc_is_subdev_pad_ops,
diff --git a/drivers/media/platform/exynos4-is/fimc-lite.c b/drivers/media/platform/exynos4-is/fimc-lite.c
index 7d3ec5cc6608..4a3c9948ca54 100644
--- a/drivers/media/platform/exynos4-is/fimc-lite.c
+++ b/drivers/media/platform/exynos4-is/fimc-lite.c
@@ -1361,7 +1361,7 @@ static const struct v4l2_subdev_core_ops fimc_lite_core_ops = {
 	.log_status = fimc_lite_log_status,
 };
 
-static struct v4l2_subdev_ops fimc_lite_subdev_ops = {
+static const struct v4l2_subdev_ops fimc_lite_subdev_ops = {
 	.core = &fimc_lite_core_ops,
 	.video = &fimc_lite_subdev_video_ops,
 	.pad = &fimc_lite_subdev_pad_ops,
@@ -1493,8 +1493,7 @@ static int fimc_lite_probe(struct platform_device *pdev)
 
 	if (!drv_data || fimc->index >= drv_data->num_instances ||
 						fimc->index < 0) {
-		dev_err(dev, "Wrong %s node alias\n",
-					dev->of_node->full_name);
+		dev_err(dev, "Wrong %pOF node alias\n", dev->of_node);
 		return -EINVAL;
 	}
 
diff --git a/drivers/media/platform/exynos4-is/fimc-m2m.c b/drivers/media/platform/exynos4-is/fimc-m2m.c
index d8724fe9e9da..9027d0b0d2bd 100644
--- a/drivers/media/platform/exynos4-is/fimc-m2m.c
+++ b/drivers/media/platform/exynos4-is/fimc-m2m.c
@@ -704,7 +704,7 @@ static const struct v4l2_file_operations fimc_m2m_fops = {
 	.mmap		= v4l2_m2m_fop_mmap,
 };
 
-static struct v4l2_m2m_ops m2m_ops = {
+static const struct v4l2_m2m_ops m2m_ops = {
 	.device_run	= fimc_device_run,
 	.job_abort	= fimc_job_abort,
 };
diff --git a/drivers/media/platform/exynos4-is/media-dev.c b/drivers/media/platform/exynos4-is/media-dev.c
index 7d1cf78846c4..d4656d5175d7 100644
--- a/drivers/media/platform/exynos4-is/media-dev.c
+++ b/drivers/media/platform/exynos4-is/media-dev.c
@@ -412,8 +412,8 @@ static int fimc_md_parse_port_node(struct fimc_md *fmd,
 	rem = of_graph_get_remote_port_parent(ep);
 	of_node_put(ep);
 	if (rem == NULL) {
-		v4l2_info(&fmd->v4l2_dev, "Remote device at %s not found\n",
-							ep->full_name);
+		v4l2_info(&fmd->v4l2_dev, "Remote device at %pOF not found\n",
+							ep);
 		return 0;
 	}
 
@@ -430,8 +430,8 @@ static int fimc_md_parse_port_node(struct fimc_md *fmd,
 		 */
 		pd->sensor_bus_type = FIMC_BUS_TYPE_MIPI_CSI2;
 	} else {
-		v4l2_err(&fmd->v4l2_dev, "Wrong port id (%u) at node %s\n",
-			 endpoint.base.port, rem->full_name);
+		v4l2_err(&fmd->v4l2_dev, "Wrong port id (%u) at node %pOF\n",
+			 endpoint.base.port, rem);
 	}
 	/*
 	 * For FIMC-IS handled sensors, that are placed under i2c-isp device
diff --git a/drivers/media/platform/exynos4-is/mipi-csis.c b/drivers/media/platform/exynos4-is/mipi-csis.c
index 98c89873c2dc..560aadabcb11 100644
--- a/drivers/media/platform/exynos4-is/mipi-csis.c
+++ b/drivers/media/platform/exynos4-is/mipi-csis.c
@@ -730,8 +730,8 @@ static int s5pcsis_parse_dt(struct platform_device *pdev,
 
 	node = of_graph_get_next_endpoint(node, NULL);
 	if (!node) {
-		dev_err(&pdev->dev, "No port node at %s\n",
-				pdev->dev.of_node->full_name);
+		dev_err(&pdev->dev, "No port node at %pOF\n",
+				pdev->dev.of_node);
 		return -EINVAL;
 	}
 	/* Get port node and validate MIPI-CSI channel id. */
diff --git a/drivers/media/platform/fsl-viu.c b/drivers/media/platform/fsl-viu.c
index 97e164b2075a..fb43025df573 100644
--- a/drivers/media/platform/fsl-viu.c
+++ b/drivers/media/platform/fsl-viu.c
@@ -549,7 +549,7 @@ static void buffer_release(struct videobuf_queue *vq,
 	free_buffer(vq, buf);
 }
 
-static struct videobuf_queue_ops viu_video_qops = {
+static const struct videobuf_queue_ops viu_video_qops = {
 	.buf_setup      = buffer_setup,
 	.buf_prepare    = buffer_prepare,
 	.buf_queue      = buffer_queue,
@@ -1340,7 +1340,7 @@ static int viu_mmap(struct file *file, struct vm_area_struct *vma)
 	return ret;
 }
 
-static struct v4l2_file_operations viu_fops = {
+static const struct v4l2_file_operations viu_fops = {
 	.owner		= THIS_MODULE,
 	.open		= viu_open,
 	.release	= viu_release,
@@ -1380,7 +1380,7 @@ static const struct v4l2_ioctl_ops viu_ioctl_ops = {
 	.vidioc_unsubscribe_event = v4l2_event_unsubscribe,
 };
 
-static struct video_device viu_template = {
+static const struct video_device viu_template = {
 	.name		= "FSL viu",
 	.fops		= &viu_fops,
 	.minor		= -1,
diff --git a/drivers/media/platform/m2m-deinterlace.c b/drivers/media/platform/m2m-deinterlace.c
index 980066b8d32a..c8a12493f395 100644
--- a/drivers/media/platform/m2m-deinterlace.c
+++ b/drivers/media/platform/m2m-deinterlace.c
@@ -979,7 +979,7 @@ static const struct v4l2_file_operations deinterlace_fops = {
 	.mmap		= deinterlace_mmap,
 };
 
-static struct video_device deinterlace_videodev = {
+static const struct video_device deinterlace_videodev = {
 	.name		= MEM2MEM_NAME,
 	.fops		= &deinterlace_fops,
 	.ioctl_ops	= &deinterlace_ioctl_ops,
@@ -988,7 +988,7 @@ static struct video_device deinterlace_videodev = {
 	.vfl_dir	= VFL_DIR_M2M,
 };
 
-static struct v4l2_m2m_ops m2m_ops = {
+static const struct v4l2_m2m_ops m2m_ops = {
 	.device_run	= deinterlace_device_run,
 	.job_ready	= deinterlace_job_ready,
 	.job_abort	= deinterlace_job_abort,
diff --git a/drivers/media/platform/marvell-ccic/cafe-driver.c b/drivers/media/platform/marvell-ccic/cafe-driver.c
index 77890bd0deab..57d2c483ad09 100644
--- a/drivers/media/platform/marvell-ccic/cafe-driver.c
+++ b/drivers/media/platform/marvell-ccic/cafe-driver.c
@@ -326,7 +326,7 @@ static u32 cafe_smbus_func(struct i2c_adapter *adapter)
 	       I2C_FUNC_SMBUS_WRITE_BYTE_DATA;
 }
 
-static struct i2c_algorithm cafe_smbus_algo = {
+static const struct i2c_algorithm cafe_smbus_algo = {
 	.smbus_xfer = cafe_smbus_xfer,
 	.functionality = cafe_smbus_func
 };
@@ -612,7 +612,7 @@ static int cafe_pci_resume(struct pci_dev *pdev)
 
 #endif  /* CONFIG_PM */
 
-static struct pci_device_id cafe_ids[] = {
+static const struct pci_device_id cafe_ids[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_MARVELL,
 		     PCI_DEVICE_ID_MARVELL_88ALP01_CCIC) },
 	{ 0, }
diff --git a/drivers/media/platform/marvell-ccic/mcam-core.c b/drivers/media/platform/marvell-ccic/mcam-core.c
index 8cac2f202099..b07a251e8857 100644
--- a/drivers/media/platform/marvell-ccic/mcam-core.c
+++ b/drivers/media/platform/marvell-ccic/mcam-core.c
@@ -1639,7 +1639,7 @@ static const struct v4l2_file_operations mcam_v4l_fops = {
  * This template device holds all of those v4l2 methods; we
  * clone it for specific real devices.
  */
-static struct video_device mcam_v4l_template = {
+static const struct video_device mcam_v4l_template = {
 	.name = "mcam",
 	.fops = &mcam_v4l_fops,
 	.ioctl_ops = &mcam_v4l_ioctl_ops,
diff --git a/drivers/media/platform/meson/Makefile b/drivers/media/platform/meson/Makefile
new file mode 100644
index 000000000000..597beb8f34d1
--- /dev/null
+++ b/drivers/media/platform/meson/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_VIDEO_MESON_AO_CEC)	+= ao-cec.o
diff --git a/drivers/media/platform/meson/ao-cec.c b/drivers/media/platform/meson/ao-cec.c
new file mode 100644
index 000000000000..8040a6285c3f
--- /dev/null
+++ b/drivers/media/platform/meson/ao-cec.c
@@ -0,0 +1,744 @@
+/*
+ * Driver for Amlogic Meson AO CEC Controller
+ *
+ * Copyright (C) 2015 Amlogic, Inc. All rights reserved
+ * Copyright (C) 2017 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ */
+
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/delay.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/types.h>
+#include <linux/interrupt.h>
+#include <linux/reset.h>
+#include <media/cec.h>
+#include <media/cec-notifier.h>
+
+/* CEC Registers */
+
+/*
+ * [2:1] cntl_clk
+ *  - 0 = Disable clk (Power-off mode)
+ *  - 1 = Enable gated clock (Normal mode)
+ *  - 2 = Enable free-run clk (Debug mode)
+ */
+#define CEC_GEN_CNTL_REG		0x00
+
+#define CEC_GEN_CNTL_RESET		BIT(0)
+#define CEC_GEN_CNTL_CLK_DISABLE	0
+#define CEC_GEN_CNTL_CLK_ENABLE		1
+#define CEC_GEN_CNTL_CLK_ENABLE_DBG	2
+#define CEC_GEN_CNTL_CLK_CTRL_MASK	GENMASK(2, 1)
+
+/*
+ * [7:0] cec_reg_addr
+ * [15:8] cec_reg_wrdata
+ * [16] cec_reg_wr
+ *  - 0 = Read
+ *  - 1 = Write
+ * [23] bus free
+ * [31:24] cec_reg_rddata
+ */
+#define CEC_RW_REG			0x04
+
+#define CEC_RW_ADDR			GENMASK(7, 0)
+#define CEC_RW_WR_DATA			GENMASK(15, 8)
+#define CEC_RW_WRITE_EN			BIT(16)
+#define CEC_RW_BUS_BUSY			BIT(23)
+#define CEC_RW_RD_DATA			GENMASK(31, 24)
+
+/*
+ * [1] tx intr
+ * [2] rx intr
+ */
+#define CEC_INTR_MASKN_REG		0x08
+#define CEC_INTR_CLR_REG		0x0c
+#define CEC_INTR_STAT_REG		0x10
+
+#define CEC_INTR_TX			BIT(1)
+#define CEC_INTR_RX			BIT(2)
+
+/* CEC Commands */
+
+#define CEC_TX_MSG_0_HEADER		0x00
+#define CEC_TX_MSG_1_OPCODE		0x01
+#define CEC_TX_MSG_2_OP1		0x02
+#define CEC_TX_MSG_3_OP2		0x03
+#define CEC_TX_MSG_4_OP3		0x04
+#define CEC_TX_MSG_5_OP4		0x05
+#define CEC_TX_MSG_6_OP5		0x06
+#define CEC_TX_MSG_7_OP6		0x07
+#define CEC_TX_MSG_8_OP7		0x08
+#define CEC_TX_MSG_9_OP8		0x09
+#define CEC_TX_MSG_A_OP9		0x0A
+#define CEC_TX_MSG_B_OP10		0x0B
+#define CEC_TX_MSG_C_OP11		0x0C
+#define CEC_TX_MSG_D_OP12		0x0D
+#define CEC_TX_MSG_E_OP13		0x0E
+#define CEC_TX_MSG_F_OP14		0x0F
+#define CEC_TX_MSG_LENGTH		0x10
+#define CEC_TX_MSG_CMD			0x11
+#define CEC_TX_WRITE_BUF		0x12
+#define CEC_TX_CLEAR_BUF		0x13
+#define CEC_RX_MSG_CMD			0x14
+#define CEC_RX_CLEAR_BUF		0x15
+#define CEC_LOGICAL_ADDR0		0x16
+#define CEC_LOGICAL_ADDR1		0x17
+#define CEC_LOGICAL_ADDR2		0x18
+#define CEC_LOGICAL_ADDR3		0x19
+#define CEC_LOGICAL_ADDR4		0x1A
+#define CEC_CLOCK_DIV_H			0x1B
+#define CEC_CLOCK_DIV_L			0x1C
+#define CEC_QUIESCENT_25MS_BIT7_0	0x20
+#define CEC_QUIESCENT_25MS_BIT11_8	0x21
+#define CEC_STARTBITMINL2H_3MS5_BIT7_0	0x22
+#define CEC_STARTBITMINL2H_3MS5_BIT8	0x23
+#define CEC_STARTBITMAXL2H_3MS9_BIT7_0	0x24
+#define CEC_STARTBITMAXL2H_3MS9_BIT8	0x25
+#define CEC_STARTBITMINH_0MS6_BIT7_0	0x26
+#define CEC_STARTBITMINH_0MS6_BIT8	0x27
+#define CEC_STARTBITMAXH_1MS0_BIT7_0	0x28
+#define CEC_STARTBITMAXH_1MS0_BIT8	0x29
+#define CEC_STARTBITMINTOT_4MS3_BIT7_0	0x2A
+#define CEC_STARTBITMINTOT_4MS3_BIT9_8	0x2B
+#define CEC_STARTBITMAXTOT_4MS7_BIT7_0	0x2C
+#define CEC_STARTBITMAXTOT_4MS7_BIT9_8	0x2D
+#define CEC_LOGIC1MINL2H_0MS4_BIT7_0	0x2E
+#define CEC_LOGIC1MINL2H_0MS4_BIT8	0x2F
+#define CEC_LOGIC1MAXL2H_0MS8_BIT7_0	0x30
+#define CEC_LOGIC1MAXL2H_0MS8_BIT8	0x31
+#define CEC_LOGIC0MINL2H_1MS3_BIT7_0	0x32
+#define CEC_LOGIC0MINL2H_1MS3_BIT8	0x33
+#define CEC_LOGIC0MAXL2H_1MS7_BIT7_0	0x34
+#define CEC_LOGIC0MAXL2H_1MS7_BIT8	0x35
+#define CEC_LOGICMINTOTAL_2MS05_BIT7_0	0x36
+#define CEC_LOGICMINTOTAL_2MS05_BIT9_8	0x37
+#define CEC_LOGICMAXHIGH_2MS8_BIT7_0	0x38
+#define CEC_LOGICMAXHIGH_2MS8_BIT8	0x39
+#define CEC_LOGICERRLOW_3MS4_BIT7_0	0x3A
+#define CEC_LOGICERRLOW_3MS4_BIT8	0x3B
+#define CEC_NOMSMPPOINT_1MS05		0x3C
+#define CEC_DELCNTR_LOGICERR		0x3E
+#define CEC_TXTIME_17MS_BIT7_0		0x40
+#define CEC_TXTIME_17MS_BIT10_8		0x41
+#define CEC_TXTIME_2BIT_BIT7_0		0x42
+#define CEC_TXTIME_2BIT_BIT10_8		0x43
+#define CEC_TXTIME_4BIT_BIT7_0		0x44
+#define CEC_TXTIME_4BIT_BIT10_8		0x45
+#define CEC_STARTBITNOML2H_3MS7_BIT7_0	0x46
+#define CEC_STARTBITNOML2H_3MS7_BIT8	0x47
+#define CEC_STARTBITNOMH_0MS8_BIT7_0	0x48
+#define CEC_STARTBITNOMH_0MS8_BIT8	0x49
+#define CEC_LOGIC1NOML2H_0MS6_BIT7_0	0x4A
+#define CEC_LOGIC1NOML2H_0MS6_BIT8	0x4B
+#define CEC_LOGIC0NOML2H_1MS5_BIT7_0	0x4C
+#define CEC_LOGIC0NOML2H_1MS5_BIT8	0x4D
+#define CEC_LOGIC1NOMH_1MS8_BIT7_0	0x4E
+#define CEC_LOGIC1NOMH_1MS8_BIT8	0x4F
+#define CEC_LOGIC0NOMH_0MS9_BIT7_0	0x50
+#define CEC_LOGIC0NOMH_0MS9_BIT8	0x51
+#define CEC_LOGICERRLOW_3MS6_BIT7_0	0x52
+#define CEC_LOGICERRLOW_3MS6_BIT8	0x53
+#define CEC_CHKCONTENTION_0MS1		0x54
+#define CEC_PREPARENXTBIT_0MS05_BIT7_0	0x56
+#define CEC_PREPARENXTBIT_0MS05_BIT8	0x57
+#define CEC_NOMSMPACKPOINT_0MS45	0x58
+#define CEC_ACK0NOML2H_1MS5_BIT7_0	0x5A
+#define CEC_ACK0NOML2H_1MS5_BIT8	0x5B
+#define CEC_BUGFIX_DISABLE_0		0x60
+#define CEC_BUGFIX_DISABLE_1		0x61
+#define CEC_RX_MSG_0_HEADER		0x80
+#define CEC_RX_MSG_1_OPCODE		0x81
+#define CEC_RX_MSG_2_OP1		0x82
+#define CEC_RX_MSG_3_OP2		0x83
+#define CEC_RX_MSG_4_OP3		0x84
+#define CEC_RX_MSG_5_OP4		0x85
+#define CEC_RX_MSG_6_OP5		0x86
+#define CEC_RX_MSG_7_OP6		0x87
+#define CEC_RX_MSG_8_OP7		0x88
+#define CEC_RX_MSG_9_OP8		0x89
+#define CEC_RX_MSG_A_OP9		0x8A
+#define CEC_RX_MSG_B_OP10		0x8B
+#define CEC_RX_MSG_C_OP11		0x8C
+#define CEC_RX_MSG_D_OP12		0x8D
+#define CEC_RX_MSG_E_OP13		0x8E
+#define CEC_RX_MSG_F_OP14		0x8F
+#define CEC_RX_MSG_LENGTH		0x90
+#define CEC_RX_MSG_STATUS		0x91
+#define CEC_RX_NUM_MSG			0x92
+#define CEC_TX_MSG_STATUS		0x93
+#define CEC_TX_NUM_MSG			0x94
+
+
+/* CEC_TX_MSG_CMD definition */
+#define TX_NO_OP	0  /* No transaction */
+#define TX_REQ_CURRENT	1  /* Transmit earliest message in buffer */
+#define TX_ABORT	2  /* Abort transmitting earliest message */
+#define TX_REQ_NEXT	3  /* Overwrite earliest msg, transmit next */
+
+/* tx_msg_status definition */
+#define TX_IDLE		0  /* No transaction */
+#define TX_BUSY		1  /* Transmitter is busy */
+#define TX_DONE		2  /* Message successfully transmitted */
+#define TX_ERROR	3  /* Message transmitted with error */
+
+/* rx_msg_cmd */
+#define RX_NO_OP	0  /* No transaction */
+#define RX_ACK_CURRENT	1  /* Read earliest message in buffer */
+#define RX_DISABLE	2  /* Disable receiving latest message */
+#define RX_ACK_NEXT	3  /* Clear earliest msg, read next */
+
+/* rx_msg_status */
+#define RX_IDLE		0  /* No transaction */
+#define RX_BUSY		1  /* Receiver is busy */
+#define RX_DONE		2  /* Message has been received successfully */
+#define RX_ERROR	3  /* Message has been received with error */
+
+/* RX_CLEAR_BUF options */
+#define CLEAR_START	1
+#define CLEAR_STOP	0
+
+/* CEC_LOGICAL_ADDRx options */
+#define LOGICAL_ADDR_MASK	0xf
+#define LOGICAL_ADDR_VALID	BIT(4)
+#define LOGICAL_ADDR_DISABLE	0
+
+#define CEC_CLK_RATE		32768
+
+struct meson_ao_cec_device {
+	struct platform_device		*pdev;
+	void __iomem			*base;
+	struct clk			*core;
+	spinlock_t			cec_reg_lock;
+	struct cec_notifier		*notify;
+	struct cec_adapter		*adap;
+	struct cec_msg			rx_msg;
+};
+
+#define writel_bits_relaxed(mask, val, addr) \
+	writel_relaxed((readl_relaxed(addr) & ~(mask)) | (val), addr)
+
+static inline int meson_ao_cec_wait_busy(struct meson_ao_cec_device *ao_cec)
+{
+	ktime_t timeout = ktime_add_us(ktime_get(), 5000);
+
+	while (readl_relaxed(ao_cec->base + CEC_RW_REG) & CEC_RW_BUS_BUSY) {
+		if (ktime_compare(ktime_get(), timeout) > 0)
+			return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static void meson_ao_cec_read(struct meson_ao_cec_device *ao_cec,
+			     unsigned long address, u8 *data,
+			     int *res)
+{
+	unsigned long flags;
+	u32 reg = FIELD_PREP(CEC_RW_ADDR, address);
+	int ret = 0;
+
+	if (res && *res)
+		return;
+
+	spin_lock_irqsave(&ao_cec->cec_reg_lock, flags);
+
+	ret = meson_ao_cec_wait_busy(ao_cec);
+	if (ret)
+		goto read_out;
+
+	writel_relaxed(reg, ao_cec->base + CEC_RW_REG);
+
+	ret = meson_ao_cec_wait_busy(ao_cec);
+	if (ret)
+		goto read_out;
+
+	*data = FIELD_GET(CEC_RW_RD_DATA,
+			  readl_relaxed(ao_cec->base + CEC_RW_REG));
+
+read_out:
+	spin_unlock_irqrestore(&ao_cec->cec_reg_lock, flags);
+
+	if (res)
+		*res = ret;
+}
+
+static void meson_ao_cec_write(struct meson_ao_cec_device *ao_cec,
+			       unsigned long address, u8 data,
+			       int *res)
+{
+	unsigned long flags;
+	u32 reg = FIELD_PREP(CEC_RW_ADDR, address) |
+		  FIELD_PREP(CEC_RW_WR_DATA, data) |
+		  CEC_RW_WRITE_EN;
+	int ret = 0;
+
+	if (res && *res)
+		return;
+
+	spin_lock_irqsave(&ao_cec->cec_reg_lock, flags);
+
+	ret = meson_ao_cec_wait_busy(ao_cec);
+	if (ret)
+		goto write_out;
+
+	writel_relaxed(reg, ao_cec->base + CEC_RW_REG);
+
+write_out:
+	spin_unlock_irqrestore(&ao_cec->cec_reg_lock, flags);
+
+	if (res)
+		*res = ret;
+}
+
+static inline void meson_ao_cec_irq_setup(struct meson_ao_cec_device *ao_cec,
+				      bool enable)
+{
+	u32 cfg = CEC_INTR_TX | CEC_INTR_RX;
+
+	writel_bits_relaxed(cfg, enable ? cfg : 0,
+			    ao_cec->base + CEC_INTR_MASKN_REG);
+}
+
+static inline int meson_ao_cec_clear(struct meson_ao_cec_device *ao_cec)
+{
+	int ret = 0;
+
+	meson_ao_cec_write(ao_cec, CEC_RX_MSG_CMD, RX_DISABLE, &ret);
+	meson_ao_cec_write(ao_cec, CEC_TX_MSG_CMD, TX_ABORT, &ret);
+	meson_ao_cec_write(ao_cec, CEC_RX_CLEAR_BUF, 1, &ret);
+	meson_ao_cec_write(ao_cec, CEC_TX_CLEAR_BUF, 1, &ret);
+	if (ret)
+		return ret;
+
+	udelay(100);
+
+	meson_ao_cec_write(ao_cec, CEC_RX_CLEAR_BUF, 0, &ret);
+	meson_ao_cec_write(ao_cec, CEC_TX_CLEAR_BUF, 0, &ret);
+	if (ret)
+		return ret;
+
+	udelay(100);
+
+	meson_ao_cec_write(ao_cec, CEC_RX_MSG_CMD, RX_NO_OP, &ret);
+	meson_ao_cec_write(ao_cec, CEC_TX_MSG_CMD, TX_NO_OP, &ret);
+
+	return ret;
+}
+
+static int meson_ao_cec_arbit_bit_time_set(struct meson_ao_cec_device *ao_cec,
+					   unsigned int bit_set,
+					   unsigned int time_set)
+{
+	int ret = 0;
+
+	switch (bit_set) {
+	case CEC_SIGNAL_FREE_TIME_RETRY:
+		meson_ao_cec_write(ao_cec, CEC_TXTIME_4BIT_BIT7_0,
+				   time_set & 0xff, &ret);
+		meson_ao_cec_write(ao_cec, CEC_TXTIME_4BIT_BIT10_8,
+				   (time_set >> 8) & 0x7, &ret);
+		break;
+
+	case CEC_SIGNAL_FREE_TIME_NEW_INITIATOR:
+		meson_ao_cec_write(ao_cec, CEC_TXTIME_2BIT_BIT7_0,
+				   time_set & 0xff, &ret);
+		meson_ao_cec_write(ao_cec, CEC_TXTIME_2BIT_BIT10_8,
+				   (time_set >> 8) & 0x7, &ret);
+		break;
+
+	case CEC_SIGNAL_FREE_TIME_NEXT_XFER:
+		meson_ao_cec_write(ao_cec, CEC_TXTIME_17MS_BIT7_0,
+				   time_set & 0xff, &ret);
+		meson_ao_cec_write(ao_cec, CEC_TXTIME_17MS_BIT10_8,
+				   (time_set >> 8) & 0x7, &ret);
+		break;
+	}
+
+	return ret;
+}
+
+static irqreturn_t meson_ao_cec_irq(int irq, void *data)
+{
+	struct meson_ao_cec_device *ao_cec = data;
+	u32 stat = readl_relaxed(ao_cec->base + CEC_INTR_STAT_REG);
+
+	if (stat)
+		return IRQ_WAKE_THREAD;
+
+	return IRQ_NONE;
+}
+
+static void meson_ao_cec_irq_tx(struct meson_ao_cec_device *ao_cec)
+{
+	unsigned long tx_status = 0;
+	u8 stat;
+	int ret = 0;
+
+	meson_ao_cec_read(ao_cec, CEC_TX_MSG_STATUS, &stat, &ret);
+	if (ret)
+		goto tx_reg_err;
+
+	switch (stat) {
+	case TX_DONE:
+		tx_status = CEC_TX_STATUS_OK;
+		break;
+
+	case TX_BUSY:
+		tx_status = CEC_TX_STATUS_ARB_LOST;
+		break;
+
+	case TX_IDLE:
+		tx_status = CEC_TX_STATUS_LOW_DRIVE;
+		break;
+
+	case TX_ERROR:
+	default:
+		tx_status = CEC_TX_STATUS_NACK;
+		break;
+	}
+
+	/* Clear Interruption */
+	writel_relaxed(CEC_INTR_TX, ao_cec->base + CEC_INTR_CLR_REG);
+
+	/* Stop TX */
+	meson_ao_cec_write(ao_cec, CEC_TX_MSG_CMD, TX_NO_OP, &ret);
+	if (ret)
+		goto tx_reg_err;
+
+	cec_transmit_attempt_done(ao_cec->adap, tx_status);
+	return;
+
+tx_reg_err:
+	cec_transmit_attempt_done(ao_cec->adap, CEC_TX_STATUS_ERROR);
+}
+
+static void meson_ao_cec_irq_rx(struct meson_ao_cec_device *ao_cec)
+{
+	int i, ret = 0;
+	u8 reg;
+
+	meson_ao_cec_read(ao_cec, CEC_RX_MSG_STATUS, &reg, &ret);
+	if (reg != RX_DONE)
+		goto rx_out;
+
+	meson_ao_cec_read(ao_cec, CEC_RX_NUM_MSG, &reg, &ret);
+	if (reg != 1)
+		goto rx_out;
+
+	meson_ao_cec_read(ao_cec, CEC_RX_MSG_LENGTH, &reg, &ret);
+
+	ao_cec->rx_msg.len = reg + 1;
+	if (ao_cec->rx_msg.len > CEC_MAX_MSG_SIZE)
+		ao_cec->rx_msg.len = CEC_MAX_MSG_SIZE;
+
+	for (i = 0; i < ao_cec->rx_msg.len; i++) {
+		u8 byte;
+
+		meson_ao_cec_read(ao_cec, CEC_RX_MSG_0_HEADER + i, &byte, &ret);
+
+		ao_cec->rx_msg.msg[i] = byte;
+	}
+
+	if (ret)
+		goto rx_out;
+
+	cec_received_msg(ao_cec->adap, &ao_cec->rx_msg);
+
+rx_out:
+	/* Clear Interruption */
+	writel_relaxed(CEC_INTR_RX, ao_cec->base + CEC_INTR_CLR_REG);
+
+	/* Ack RX message */
+	meson_ao_cec_write(ao_cec, CEC_RX_MSG_CMD, RX_ACK_CURRENT, &ret);
+	meson_ao_cec_write(ao_cec, CEC_RX_MSG_CMD, RX_NO_OP, &ret);
+
+	/* Clear RX buffer */
+	meson_ao_cec_write(ao_cec, CEC_RX_CLEAR_BUF, CLEAR_START, &ret);
+	meson_ao_cec_write(ao_cec, CEC_RX_CLEAR_BUF, CLEAR_STOP, &ret);
+}
+
+static irqreturn_t meson_ao_cec_irq_thread(int irq, void *data)
+{
+	struct meson_ao_cec_device *ao_cec = data;
+	u32 stat = readl_relaxed(ao_cec->base + CEC_INTR_STAT_REG);
+
+	if (stat & CEC_INTR_TX)
+		meson_ao_cec_irq_tx(ao_cec);
+
+	meson_ao_cec_irq_rx(ao_cec);
+
+	return IRQ_HANDLED;
+}
+
+static int meson_ao_cec_set_log_addr(struct cec_adapter *adap, u8 logical_addr)
+{
+	struct meson_ao_cec_device *ao_cec = adap->priv;
+	int ret = 0;
+
+	meson_ao_cec_write(ao_cec, CEC_LOGICAL_ADDR0,
+			   LOGICAL_ADDR_DISABLE, &ret);
+	if (ret)
+		return ret;
+
+	ret = meson_ao_cec_clear(ao_cec);
+	if (ret)
+		return ret;
+
+	if (logical_addr == CEC_LOG_ADDR_INVALID)
+		return 0;
+
+	meson_ao_cec_write(ao_cec, CEC_LOGICAL_ADDR0,
+			   logical_addr & LOGICAL_ADDR_MASK, &ret);
+	if (ret)
+		return ret;
+
+	udelay(100);
+
+	meson_ao_cec_write(ao_cec, CEC_LOGICAL_ADDR0,
+			   (logical_addr & LOGICAL_ADDR_MASK) |
+			   LOGICAL_ADDR_VALID, &ret);
+
+	return ret;
+}
+
+static int meson_ao_cec_transmit(struct cec_adapter *adap, u8 attempts,
+				 u32 signal_free_time, struct cec_msg *msg)
+{
+	struct meson_ao_cec_device *ao_cec = adap->priv;
+	int i, ret = 0;
+	u8 reg;
+
+	meson_ao_cec_read(ao_cec, CEC_TX_MSG_STATUS, &reg, &ret);
+	if (ret)
+		return ret;
+
+	if (reg == TX_BUSY) {
+		dev_err(&ao_cec->pdev->dev, "%s: busy TX: aborting\n",
+			__func__);
+		meson_ao_cec_write(ao_cec, CEC_TX_MSG_CMD, TX_ABORT, &ret);
+	}
+
+	for (i = 0; i < msg->len; i++) {
+		meson_ao_cec_write(ao_cec, CEC_TX_MSG_0_HEADER + i,
+				   msg->msg[i], &ret);
+	}
+
+	meson_ao_cec_write(ao_cec, CEC_TX_MSG_LENGTH, msg->len - 1, &ret);
+	meson_ao_cec_write(ao_cec, CEC_TX_MSG_CMD, TX_REQ_CURRENT, &ret);
+
+	return ret;
+}
+
+static int meson_ao_cec_adap_enable(struct cec_adapter *adap, bool enable)
+{
+	struct meson_ao_cec_device *ao_cec = adap->priv;
+	int ret;
+
+	meson_ao_cec_irq_setup(ao_cec, false);
+
+	writel_bits_relaxed(CEC_GEN_CNTL_RESET, CEC_GEN_CNTL_RESET,
+			    ao_cec->base + CEC_GEN_CNTL_REG);
+
+	if (!enable)
+		return 0;
+
+	/* Enable gated clock (Normal mode). */
+	writel_bits_relaxed(CEC_GEN_CNTL_CLK_CTRL_MASK,
+			    FIELD_PREP(CEC_GEN_CNTL_CLK_CTRL_MASK,
+				       CEC_GEN_CNTL_CLK_ENABLE),
+			    ao_cec->base + CEC_GEN_CNTL_REG);
+
+	udelay(100);
+
+	/* Release Reset */
+	writel_bits_relaxed(CEC_GEN_CNTL_RESET, 0,
+			    ao_cec->base + CEC_GEN_CNTL_REG);
+
+	/* Clear buffers */
+	ret = meson_ao_cec_clear(ao_cec);
+	if (ret)
+		return ret;
+
+	/* CEC arbitration 3/5/7 bit time set. */
+	ret = meson_ao_cec_arbit_bit_time_set(ao_cec,
+					CEC_SIGNAL_FREE_TIME_RETRY,
+					0x118);
+	if (ret)
+		return ret;
+	ret = meson_ao_cec_arbit_bit_time_set(ao_cec,
+					CEC_SIGNAL_FREE_TIME_NEW_INITIATOR,
+					0x000);
+	if (ret)
+		return ret;
+	ret = meson_ao_cec_arbit_bit_time_set(ao_cec,
+					CEC_SIGNAL_FREE_TIME_NEXT_XFER,
+					0x2aa);
+	if (ret)
+		return ret;
+
+	meson_ao_cec_irq_setup(ao_cec, true);
+
+	return 0;
+}
+
+static const struct cec_adap_ops meson_ao_cec_ops = {
+	.adap_enable = meson_ao_cec_adap_enable,
+	.adap_log_addr = meson_ao_cec_set_log_addr,
+	.adap_transmit = meson_ao_cec_transmit,
+};
+
+static int meson_ao_cec_probe(struct platform_device *pdev)
+{
+	struct meson_ao_cec_device *ao_cec;
+	struct platform_device *hdmi_dev;
+	struct device_node *np;
+	struct resource *res;
+	int ret, irq;
+
+	np = of_parse_phandle(pdev->dev.of_node, "hdmi-phandle", 0);
+	if (!np) {
+		dev_err(&pdev->dev, "Failed to find hdmi node\n");
+		return -ENODEV;
+	}
+
+	hdmi_dev = of_find_device_by_node(np);
+	if (hdmi_dev == NULL)
+		return -EPROBE_DEFER;
+
+	ao_cec = devm_kzalloc(&pdev->dev, sizeof(*ao_cec), GFP_KERNEL);
+	if (!ao_cec)
+		return -ENOMEM;
+
+	spin_lock_init(&ao_cec->cec_reg_lock);
+
+	ao_cec->notify = cec_notifier_get(&hdmi_dev->dev);
+	if (!ao_cec->notify)
+		return -ENOMEM;
+
+	ao_cec->adap = cec_allocate_adapter(&meson_ao_cec_ops, ao_cec,
+					    "meson_ao_cec",
+					    CEC_CAP_LOG_ADDRS |
+					    CEC_CAP_TRANSMIT |
+					    CEC_CAP_RC |
+					    CEC_CAP_PASSTHROUGH,
+					    1); /* Use 1 for now */
+	if (IS_ERR(ao_cec->adap)) {
+		ret = PTR_ERR(ao_cec->adap);
+		goto out_probe_notify;
+	}
+
+	ao_cec->adap->owner = THIS_MODULE;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	ao_cec->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(ao_cec->base)) {
+		ret = PTR_ERR(ao_cec->base);
+		goto out_probe_adapter;
+	}
+
+	irq = platform_get_irq(pdev, 0);
+	ret = devm_request_threaded_irq(&pdev->dev, irq,
+					meson_ao_cec_irq,
+					meson_ao_cec_irq_thread,
+					0, NULL, ao_cec);
+	if (ret) {
+		dev_err(&pdev->dev, "irq request failed\n");
+		goto out_probe_adapter;
+	}
+
+	ao_cec->core = devm_clk_get(&pdev->dev, "core");
+	if (IS_ERR(ao_cec->core)) {
+		dev_err(&pdev->dev, "core clock request failed\n");
+		ret = PTR_ERR(ao_cec->core);
+		goto out_probe_adapter;
+	}
+
+	ret = clk_prepare_enable(ao_cec->core);
+	if (ret) {
+		dev_err(&pdev->dev, "core clock enable failed\n");
+		goto out_probe_adapter;
+	}
+
+	ret = clk_set_rate(ao_cec->core, CEC_CLK_RATE);
+	if (ret) {
+		dev_err(&pdev->dev, "core clock set rate failed\n");
+		goto out_probe_clk;
+	}
+
+	device_reset_optional(&pdev->dev);
+
+	ao_cec->pdev = pdev;
+	platform_set_drvdata(pdev, ao_cec);
+
+	ret = cec_register_adapter(ao_cec->adap, &pdev->dev);
+	if (ret < 0) {
+		cec_notifier_put(ao_cec->notify);
+		goto out_probe_clk;
+	}
+
+	/* Setup Hardware */
+	writel_relaxed(CEC_GEN_CNTL_RESET,
+		       ao_cec->base + CEC_GEN_CNTL_REG);
+
+	cec_register_cec_notifier(ao_cec->adap, ao_cec->notify);
+
+	return 0;
+
+out_probe_clk:
+	clk_disable_unprepare(ao_cec->core);
+
+out_probe_adapter:
+	cec_delete_adapter(ao_cec->adap);
+
+out_probe_notify:
+	cec_notifier_put(ao_cec->notify);
+
+	dev_err(&pdev->dev, "CEC controller registration failed\n");
+
+	return ret;
+}
+
+static int meson_ao_cec_remove(struct platform_device *pdev)
+{
+	struct meson_ao_cec_device *ao_cec = platform_get_drvdata(pdev);
+
+	clk_disable_unprepare(ao_cec->core);
+
+	cec_unregister_adapter(ao_cec->adap);
+
+	cec_notifier_put(ao_cec->notify);
+
+	return 0;
+}
+
+static const struct of_device_id meson_ao_cec_of_match[] = {
+	{ .compatible = "amlogic,meson-gx-ao-cec", },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, meson_ao_cec_of_match);
+
+static struct platform_driver meson_ao_cec_driver = {
+	.probe   = meson_ao_cec_probe,
+	.remove  = meson_ao_cec_remove,
+	.driver  = {
+		.name = "meson-ao-cec",
+		.of_match_table = of_match_ptr(meson_ao_cec_of_match),
+	},
+};
+
+module_platform_driver(meson_ao_cec_driver);
+
+MODULE_DESCRIPTION("Meson AO CEC Controller driver");
+MODULE_AUTHOR("Neil Armstrong <narmstrong@baylibre.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/platform/mtk-jpeg/mtk_jpeg_core.c b/drivers/media/platform/mtk-jpeg/mtk_jpeg_core.c
index 451a54039e65..226f90886484 100644
--- a/drivers/media/platform/mtk-jpeg/mtk_jpeg_core.c
+++ b/drivers/media/platform/mtk-jpeg/mtk_jpeg_core.c
@@ -756,7 +756,7 @@ static void mtk_jpeg_stop_streaming(struct vb2_queue *q)
 	pm_runtime_put_sync(ctx->jpeg->dev);
 }
 
-static struct vb2_ops mtk_jpeg_qops = {
+static const struct vb2_ops mtk_jpeg_qops = {
 	.queue_setup        = mtk_jpeg_queue_setup,
 	.buf_prepare        = mtk_jpeg_buf_prepare,
 	.buf_queue          = mtk_jpeg_buf_queue,
@@ -865,7 +865,7 @@ static void mtk_jpeg_job_abort(void *priv)
 {
 }
 
-static struct v4l2_m2m_ops mtk_jpeg_m2m_ops = {
+static const struct v4l2_m2m_ops mtk_jpeg_m2m_ops = {
 	.device_run = mtk_jpeg_device_run,
 	.job_ready  = mtk_jpeg_job_ready,
 	.job_abort  = mtk_jpeg_job_abort,
diff --git a/drivers/media/platform/mtk-mdp/mtk_mdp_comp.c b/drivers/media/platform/mtk-mdp/mtk_mdp_comp.c
index aa8f9fd1f1a2..03aba03a24c8 100644
--- a/drivers/media/platform/mtk-mdp/mtk_mdp_comp.c
+++ b/drivers/media/platform/mtk-mdp/mtk_mdp_comp.c
@@ -75,7 +75,7 @@ void mtk_mdp_comp_clock_on(struct device *dev, struct mtk_mdp_comp *comp)
 	}
 
 	for (i = 0; i < ARRAY_SIZE(comp->clk); i++) {
-		if (!comp->clk[i])
+		if (IS_ERR(comp->clk[i]))
 			continue;
 		err = clk_prepare_enable(comp->clk[i]);
 		if (err)
@@ -90,7 +90,7 @@ void mtk_mdp_comp_clock_off(struct device *dev, struct mtk_mdp_comp *comp)
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(comp->clk); i++) {
-		if (!comp->clk[i])
+		if (IS_ERR(comp->clk[i]))
 			continue;
 		clk_disable_unprepare(comp->clk[i]);
 	}
@@ -134,15 +134,13 @@ int mtk_mdp_comp_init(struct device *dev, struct device_node *node,
 	larb_node = of_parse_phandle(node, "mediatek,larb", 0);
 	if (!larb_node) {
 		dev_err(dev,
-			"Missing mediadek,larb phandle in %s node\n",
-			node->full_name);
+			"Missing mediadek,larb phandle in %pOF node\n", node);
 		return -EINVAL;
 	}
 
 	larb_pdev = of_find_device_by_node(larb_node);
 	if (!larb_pdev) {
-		dev_warn(dev, "Waiting for larb device %s\n",
-			 larb_node->full_name);
+		dev_warn(dev, "Waiting for larb device %pOF\n", larb_node);
 		of_node_put(larb_node);
 		return -EPROBE_DEFER;
 	}
diff --git a/drivers/media/platform/mtk-mdp/mtk_mdp_core.c b/drivers/media/platform/mtk-mdp/mtk_mdp_core.c
index 81347558b24a..bbb24fb95b95 100644
--- a/drivers/media/platform/mtk-mdp/mtk_mdp_core.c
+++ b/drivers/media/platform/mtk-mdp/mtk_mdp_core.c
@@ -137,16 +137,16 @@ static int mtk_mdp_probe(struct platform_device *pdev)
 			continue;
 
 		if (!of_device_is_available(node)) {
-			dev_err(dev, "Skipping disabled component %s\n",
-				node->full_name);
+			dev_err(dev, "Skipping disabled component %pOF\n",
+				node);
 			continue;
 		}
 
 		comp_type = (enum mtk_mdp_comp_type)of_id->data;
 		comp_id = mtk_mdp_comp_get_id(dev, node, comp_type);
 		if (comp_id < 0) {
-			dev_warn(dev, "Skipping unknown component %s\n",
-				 node->full_name);
+			dev_warn(dev, "Skipping unknown component %pOF\n",
+				 node);
 			continue;
 		}
 
diff --git a/drivers/media/platform/mtk-mdp/mtk_mdp_m2m.c b/drivers/media/platform/mtk-mdp/mtk_mdp_m2m.c
index 13afe48b9dc5..583d47724ee8 100644
--- a/drivers/media/platform/mtk-mdp/mtk_mdp_m2m.c
+++ b/drivers/media/platform/mtk-mdp/mtk_mdp_m2m.c
@@ -621,7 +621,7 @@ static void mtk_mdp_m2m_buf_queue(struct vb2_buffer *vb)
 	v4l2_m2m_buf_queue(ctx->m2m_ctx, to_vb2_v4l2_buffer(vb));
 }
 
-static struct vb2_ops mtk_mdp_m2m_qops = {
+static const struct vb2_ops mtk_mdp_m2m_qops = {
 	.queue_setup	 = mtk_mdp_m2m_queue_setup,
 	.buf_prepare	 = mtk_mdp_m2m_buf_prepare,
 	.buf_queue	 = mtk_mdp_m2m_buf_queue,
@@ -1225,7 +1225,7 @@ static const struct v4l2_file_operations mtk_mdp_m2m_fops = {
 	.mmap		= v4l2_m2m_fop_mmap,
 };
 
-static struct v4l2_m2m_ops mtk_mdp_m2m_ops = {
+static const struct v4l2_m2m_ops mtk_mdp_m2m_ops = {
 	.device_run	= mtk_mdp_m2m_device_run,
 	.job_abort	= mtk_mdp_m2m_job_abort,
 };
diff --git a/drivers/media/platform/mtk-vcodec/vdec/vdec_vp9_if.c b/drivers/media/platform/mtk-vcodec/vdec/vdec_vp9_if.c
index 1daee1207469..bc8349bc2e80 100644
--- a/drivers/media/platform/mtk-vcodec/vdec/vdec_vp9_if.c
+++ b/drivers/media/platform/mtk-vcodec/vdec/vdec_vp9_if.c
@@ -31,6 +31,7 @@
 #define MAX_NUM_REF_FRAMES 8
 #define VP9_MAX_FRM_BUF_NUM 9
 #define VP9_MAX_FRM_BUF_NODE_NUM (VP9_MAX_FRM_BUF_NUM * 2)
+#define VP9_SEG_ID_SZ 0x12000
 
 /**
  * struct vp9_dram_buf - contains buffer info for vpu
@@ -132,6 +133,7 @@ struct vp9_sf_ref_fb {
  * @frm_num : decoded frame number, include sub-frame count (AP-R, VPU-W)
  * @mv_buf : motion vector working buffer (AP-W, VPU-R)
  * @frm_refs : maintain three reference buffer info (AP-R/W, VPU-R/W)
+ * @seg_id_buf : segmentation map working buffer (AP-W, VPU-R)
  */
 struct vdec_vp9_vsi {
 	unsigned char sf_bs_buf[VP9_SUPER_FRAME_BS_SZ];
@@ -167,11 +169,14 @@ struct vdec_vp9_vsi {
 	struct vp9_dram_buf mv_buf;
 
 	struct vp9_ref_buf frm_refs[REFS_PER_FRAME];
+	struct vp9_dram_buf seg_id_buf;
+
 };
 
 /*
  * struct vdec_vp9_inst - vp9 decode instance
  * @mv_buf : working buffer for mv
+ * @seg_id_buf : working buffer for segmentation map
  * @dec_fb : vdec_fb node to link fb to different fb_xxx_list
  * @available_fb_node_list : current available vdec_fb node
  * @fb_use_list : current used or referenced vdec_fb
@@ -187,6 +192,7 @@ struct vdec_vp9_vsi {
  */
 struct vdec_vp9_inst {
 	struct mtk_vcodec_mem mv_buf;
+	struct mtk_vcodec_mem seg_id_buf;
 
 	struct vdec_fb_node dec_fb[VP9_MAX_FRM_BUF_NODE_NUM];
 	struct list_head available_fb_node_list;
@@ -388,13 +394,11 @@ static bool vp9_alloc_work_buf(struct vdec_vp9_inst *inst)
 			vsi->buf_h);
 
 	mem = &inst->mv_buf;
-
 	if (mem->va)
 		mtk_vcodec_mem_free(inst->ctx, mem);
 
 	mem->size = ((vsi->buf_w / 64) *
 		    (vsi->buf_h / 64) + 2) * 36 * 16;
-
 	result = mtk_vcodec_mem_alloc(inst->ctx, mem);
 	if (result) {
 		mem->size = 0;
@@ -406,6 +410,24 @@ static bool vp9_alloc_work_buf(struct vdec_vp9_inst *inst)
 	vsi->mv_buf.pa = (unsigned long)mem->dma_addr;
 	vsi->mv_buf.sz = (unsigned int)mem->size;
 
+
+	mem = &inst->seg_id_buf;
+	if (mem->va)
+		mtk_vcodec_mem_free(inst->ctx, mem);
+
+	mem->size = VP9_SEG_ID_SZ;
+	result = mtk_vcodec_mem_alloc(inst->ctx, mem);
+	if (result) {
+		mem->size = 0;
+		mtk_vcodec_err(inst, "Cannot allocate seg_id_buf");
+		return false;
+	}
+	/* Set the va again */
+	vsi->seg_id_buf.va = (unsigned long)mem->va;
+	vsi->seg_id_buf.pa = (unsigned long)mem->dma_addr;
+	vsi->seg_id_buf.sz = (unsigned int)mem->size;
+
+
 	vp9_free_all_sf_ref_fb(inst);
 	vsi->sf_next_ref_fb_idx = vp9_get_sf_ref_fb(inst);
 
@@ -653,6 +675,12 @@ static void vp9_reset(struct vdec_vp9_inst *inst)
 	inst->vsi->mv_buf.va = (unsigned long)inst->mv_buf.va;
 	inst->vsi->mv_buf.pa = (unsigned long)inst->mv_buf.dma_addr;
 	inst->vsi->mv_buf.sz = (unsigned long)inst->mv_buf.size;
+
+	/* Set the va again, since vpu_dec_reset will clear seg_id_buf in vpu */
+	inst->vsi->seg_id_buf.va = (unsigned long)inst->seg_id_buf.va;
+	inst->vsi->seg_id_buf.pa = (unsigned long)inst->seg_id_buf.dma_addr;
+	inst->vsi->seg_id_buf.sz = (unsigned long)inst->seg_id_buf.size;
+
 }
 
 static void init_all_fb_lists(struct vdec_vp9_inst *inst)
@@ -752,6 +780,10 @@ static void vdec_vp9_deinit(unsigned long h_vdec)
 	if (mem->va)
 		mtk_vcodec_mem_free(inst->ctx, mem);
 
+	mem = &inst->seg_id_buf;
+	if (mem->va)
+		mtk_vcodec_mem_free(inst->ctx, mem);
+
 	vp9_free_all_sf_ref_fb(inst);
 	vp9_free_inst(inst);
 }
@@ -848,6 +880,7 @@ static int vdec_vp9_decode(unsigned long h_vdec, struct mtk_vcodec_mem *bs,
 					vsi->sf_frm_sz[idx]);
 			}
 		}
+		memset(inst->seg_id_buf.va, 0, inst->seg_id_buf.size);
 		ret = vpu_dec_start(&inst->vpu, data, 3);
 		if (ret) {
 			mtk_vcodec_err(inst, "vpu_dec_start failed");
diff --git a/drivers/media/platform/mx2_emmaprp.c b/drivers/media/platform/mx2_emmaprp.c
index 03e47e0f778d..4a2b1afa19c4 100644
--- a/drivers/media/platform/mx2_emmaprp.c
+++ b/drivers/media/platform/mx2_emmaprp.c
@@ -873,7 +873,7 @@ static const struct v4l2_file_operations emmaprp_fops = {
 	.mmap		= emmaprp_mmap,
 };
 
-static struct video_device emmaprp_videodev = {
+static const struct video_device emmaprp_videodev = {
 	.name		= MEM2MEM_NAME,
 	.fops		= &emmaprp_fops,
 	.ioctl_ops	= &emmaprp_ioctl_ops,
@@ -882,7 +882,7 @@ static struct video_device emmaprp_videodev = {
 	.vfl_dir	= VFL_DIR_M2M,
 };
 
-static struct v4l2_m2m_ops m2m_ops = {
+static const struct v4l2_m2m_ops m2m_ops = {
 	.device_run	= emmaprp_device_run,
 	.job_abort	= emmaprp_job_abort,
 	.lock		= emmaprp_lock,
@@ -942,6 +942,8 @@ static int emmaprp_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, pcdev);
 
 	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return irq;
 	ret = devm_request_irq(&pdev->dev, irq, emmaprp_irq, 0,
 			       dev_name(&pdev->dev), pcdev);
 	if (ret)
diff --git a/drivers/media/platform/omap/omap_vout_vrfb.c b/drivers/media/platform/omap/omap_vout_vrfb.c
index 45a553d4f5b2..123c2b26a933 100644
--- a/drivers/media/platform/omap/omap_vout_vrfb.c
+++ b/drivers/media/platform/omap/omap_vout_vrfb.c
@@ -12,6 +12,7 @@
 #include <linux/sched.h>
 #include <linux/platform_device.h>
 #include <linux/videodev2.h>
+#include <linux/slab.h>
 
 #include <media/videobuf-dma-contig.h>
 #include <media/v4l2-device.h>
@@ -233,7 +234,7 @@ int omap_vout_prepare_vrfb(struct omap_vout_device *vout,
 			   struct videobuf_buffer *vb)
 {
 	struct dma_async_tx_descriptor *tx;
-	enum dma_ctrl_flags flags;
+	enum dma_ctrl_flags flags = DMA_PREP_INTERRUPT | DMA_CTRL_ACK;
 	struct dma_chan *chan = vout->vrfb_dma_tx.chan;
 	struct dma_device *dmadev = chan->device;
 	struct dma_interleaved_template *xt = vout->vrfb_dma_tx.xt;
diff --git a/drivers/media/platform/omap3isp/isp.c b/drivers/media/platform/omap3isp/isp.c
index 9df64c189883..1a428fe9f070 100644
--- a/drivers/media/platform/omap3isp/isp.c
+++ b/drivers/media/platform/omap3isp/isp.c
@@ -1859,6 +1859,7 @@ static void isp_cleanup_modules(struct isp_device *isp)
 	omap3isp_ccdc_cleanup(isp);
 	omap3isp_ccp2_cleanup(isp);
 	omap3isp_csi2_cleanup(isp);
+	omap3isp_csiphy_cleanup(isp);
 }
 
 static int isp_initialize_modules(struct isp_device *isp)
@@ -1868,7 +1869,7 @@ static int isp_initialize_modules(struct isp_device *isp)
 	ret = omap3isp_csiphy_init(isp);
 	if (ret < 0) {
 		dev_err(isp->dev, "CSI PHY initialization failed\n");
-		goto error_csiphy;
+		return ret;
 	}
 
 	ret = omap3isp_csi2_init(isp);
@@ -1879,7 +1880,8 @@ static int isp_initialize_modules(struct isp_device *isp)
 
 	ret = omap3isp_ccp2_init(isp);
 	if (ret < 0) {
-		dev_err(isp->dev, "CCP2 initialization failed\n");
+		if (ret != -EPROBE_DEFER)
+			dev_err(isp->dev, "CCP2 initialization failed\n");
 		goto error_ccp2;
 	}
 
@@ -1936,7 +1938,8 @@ error_ccdc:
 error_ccp2:
 	omap3isp_csi2_cleanup(isp);
 error_csi2:
-error_csiphy:
+	omap3isp_csiphy_cleanup(isp);
+
 	return ret;
 }
 
@@ -2015,13 +2018,14 @@ static int isp_fwnode_parse(struct device *dev, struct fwnode_handle *fwnode,
 	struct v4l2_fwnode_endpoint vep;
 	unsigned int i;
 	int ret;
+	bool csi1 = false;
 
 	ret = v4l2_fwnode_endpoint_parse(fwnode, &vep);
 	if (ret)
 		return ret;
 
-	dev_dbg(dev, "parsing endpoint %s, interface %u\n",
-		to_of_node(fwnode)->full_name, vep.base.port);
+	dev_dbg(dev, "parsing endpoint %pOF, interface %u\n",
+		to_of_node(fwnode), vep.base.port);
 
 	switch (vep.base.port) {
 	case ISP_OF_PHY_PARALLEL:
@@ -2039,48 +2043,102 @@ static int isp_fwnode_parse(struct device *dev, struct fwnode_handle *fwnode,
 			!!(vep.bus.parallel.flags & V4L2_MBUS_FIELD_EVEN_LOW);
 		buscfg->bus.parallel.data_pol =
 			!!(vep.bus.parallel.flags & V4L2_MBUS_DATA_ACTIVE_LOW);
+		buscfg->bus.parallel.bt656 = vep.bus_type == V4L2_MBUS_BT656;
 		break;
 
 	case ISP_OF_PHY_CSIPHY1:
 	case ISP_OF_PHY_CSIPHY2:
-		/* FIXME: always assume CSI-2 for now. */
+		switch (vep.bus_type) {
+		case V4L2_MBUS_CCP2:
+		case V4L2_MBUS_CSI1:
+			dev_dbg(dev, "CSI-1/CCP-2 configuration\n");
+			csi1 = true;
+			break;
+		case V4L2_MBUS_CSI2:
+			dev_dbg(dev, "CSI-2 configuration\n");
+			csi1 = false;
+			break;
+		default:
+			dev_err(dev, "unsupported bus type %u\n",
+				vep.bus_type);
+			return -EINVAL;
+		}
+
 		switch (vep.base.port) {
 		case ISP_OF_PHY_CSIPHY1:
-			buscfg->interface = ISP_INTERFACE_CSI2C_PHY1;
+			if (csi1)
+				buscfg->interface = ISP_INTERFACE_CCP2B_PHY1;
+			else
+				buscfg->interface = ISP_INTERFACE_CSI2C_PHY1;
 			break;
 		case ISP_OF_PHY_CSIPHY2:
-			buscfg->interface = ISP_INTERFACE_CSI2A_PHY2;
+			if (csi1)
+				buscfg->interface = ISP_INTERFACE_CCP2B_PHY2;
+			else
+				buscfg->interface = ISP_INTERFACE_CSI2A_PHY2;
 			break;
 		}
-		buscfg->bus.csi2.lanecfg.clk.pos = vep.bus.mipi_csi2.clock_lane;
-		buscfg->bus.csi2.lanecfg.clk.pol =
-			vep.bus.mipi_csi2.lane_polarities[0];
-		dev_dbg(dev, "clock lane polarity %u, pos %u\n",
-			buscfg->bus.csi2.lanecfg.clk.pol,
-			buscfg->bus.csi2.lanecfg.clk.pos);
-
-		for (i = 0; i < ISP_CSIPHY2_NUM_DATA_LANES; i++) {
-			buscfg->bus.csi2.lanecfg.data[i].pos =
-				vep.bus.mipi_csi2.data_lanes[i];
-			buscfg->bus.csi2.lanecfg.data[i].pol =
-				vep.bus.mipi_csi2.lane_polarities[i + 1];
-			dev_dbg(dev, "data lane %u polarity %u, pos %u\n", i,
-				buscfg->bus.csi2.lanecfg.data[i].pol,
-				buscfg->bus.csi2.lanecfg.data[i].pos);
+		if (csi1) {
+			buscfg->bus.ccp2.lanecfg.clk.pos =
+				vep.bus.mipi_csi1.clock_lane;
+			buscfg->bus.ccp2.lanecfg.clk.pol =
+				vep.bus.mipi_csi1.lane_polarity[0];
+			dev_dbg(dev, "clock lane polarity %u, pos %u\n",
+				buscfg->bus.ccp2.lanecfg.clk.pol,
+				buscfg->bus.ccp2.lanecfg.clk.pos);
+
+			buscfg->bus.ccp2.lanecfg.data[0].pos =
+				vep.bus.mipi_csi1.data_lane;
+			buscfg->bus.ccp2.lanecfg.data[0].pol =
+				vep.bus.mipi_csi1.lane_polarity[1];
+
+			dev_dbg(dev, "data lane polarity %u, pos %u\n",
+				buscfg->bus.ccp2.lanecfg.data[0].pol,
+				buscfg->bus.ccp2.lanecfg.data[0].pos);
+
+			buscfg->bus.ccp2.strobe_clk_pol =
+				vep.bus.mipi_csi1.clock_inv;
+			buscfg->bus.ccp2.phy_layer = vep.bus.mipi_csi1.strobe;
+			buscfg->bus.ccp2.ccp2_mode =
+				vep.bus_type == V4L2_MBUS_CCP2;
+			buscfg->bus.ccp2.vp_clk_pol = 1;
+
+			buscfg->bus.ccp2.crc = 1;
+		} else {
+			buscfg->bus.csi2.lanecfg.clk.pos =
+				vep.bus.mipi_csi2.clock_lane;
+			buscfg->bus.csi2.lanecfg.clk.pol =
+				vep.bus.mipi_csi2.lane_polarities[0];
+			dev_dbg(dev, "clock lane polarity %u, pos %u\n",
+				buscfg->bus.csi2.lanecfg.clk.pol,
+				buscfg->bus.csi2.lanecfg.clk.pos);
+
+			buscfg->bus.csi2.num_data_lanes =
+				vep.bus.mipi_csi2.num_data_lanes;
+
+			for (i = 0; i < buscfg->bus.csi2.num_data_lanes; i++) {
+				buscfg->bus.csi2.lanecfg.data[i].pos =
+					vep.bus.mipi_csi2.data_lanes[i];
+				buscfg->bus.csi2.lanecfg.data[i].pol =
+					vep.bus.mipi_csi2.lane_polarities[i + 1];
+				dev_dbg(dev,
+					"data lane %u polarity %u, pos %u\n", i,
+					buscfg->bus.csi2.lanecfg.data[i].pol,
+					buscfg->bus.csi2.lanecfg.data[i].pos);
+			}
+			/*
+			 * FIXME: now we assume the CRC is always there.
+			 * Implement a way to obtain this information from the
+			 * sensor. Frame descriptors, perhaps?
+			 */
+			buscfg->bus.csi2.crc = 1;
 		}
-
-		/*
-		 * FIXME: now we assume the CRC is always there.
-		 * Implement a way to obtain this information from the
-		 * sensor. Frame descriptors, perhaps?
-		 */
-		buscfg->bus.csi2.crc = 1;
 		break;
 
 	default:
-		dev_warn(dev, "%s: invalid interface %u\n",
-			 to_of_node(fwnode)->full_name, vep.base.port);
-		break;
+		dev_warn(dev, "%pOF: invalid interface %u\n",
+			 to_of_node(fwnode), vep.base.port);
+		return -EINVAL;
 	}
 
 	return 0;
@@ -2105,10 +2163,12 @@ static int isp_fwnodes_parse(struct device *dev,
 		if (!isd)
 			goto error;
 
-		notifier->subdevs[notifier->num_subdevs] = &isd->asd;
+		if (isp_fwnode_parse(dev, fwnode, isd)) {
+			devm_kfree(dev, isd);
+			continue;
+		}
 
-		if (isp_fwnode_parse(dev, fwnode, isd))
-			goto error;
+		notifier->subdevs[notifier->num_subdevs] = &isd->asd;
 
 		isd->asd.match.fwnode.fwnode =
 			fwnode_graph_get_remote_port_parent(fwnode);
@@ -2128,26 +2188,12 @@ error:
 	return -EINVAL;
 }
 
-static int isp_subdev_notifier_bound(struct v4l2_async_notifier *async,
-				     struct v4l2_subdev *subdev,
-				     struct v4l2_async_subdev *asd)
-{
-	struct isp_async_subdev *isd =
-		container_of(asd, struct isp_async_subdev, asd);
-
-	isd->sd = subdev;
-	isd->sd->host_priv = &isd->bus;
-
-	return 0;
-}
-
 static int isp_subdev_notifier_complete(struct v4l2_async_notifier *async)
 {
 	struct isp_device *isp = container_of(async, struct isp_device,
 					      notifier);
 	struct v4l2_device *v4l2_dev = &isp->v4l2_dev;
 	struct v4l2_subdev *sd;
-	struct isp_bus_cfg *bus;
 	int ret;
 
 	ret = media_entity_enum_init(&isp->crashed, &isp->media_dev);
@@ -2155,13 +2201,13 @@ static int isp_subdev_notifier_complete(struct v4l2_async_notifier *async)
 		return ret;
 
 	list_for_each_entry(sd, &v4l2_dev->subdevs, list) {
-		/* Only try to link entities whose interface was set on bound */
-		if (sd->host_priv) {
-			bus = (struct isp_bus_cfg *)sd->host_priv;
-			ret = isp_link_entity(isp, &sd->entity, bus->interface);
-			if (ret < 0)
-				return ret;
-		}
+		if (!sd->asd)
+			continue;
+
+		ret = isp_link_entity(isp, &sd->entity,
+				      v4l2_subdev_to_bus_cfg(sd)->interface);
+		if (ret < 0)
+			return ret;
 	}
 
 	ret = v4l2_device_register_subdev_nodes(&isp->v4l2_dev);
@@ -2339,7 +2385,6 @@ static int isp_probe(struct platform_device *pdev)
 	if (ret < 0)
 		goto error_register_entities;
 
-	isp->notifier.bound = isp_subdev_notifier_bound;
 	isp->notifier.complete = isp_subdev_notifier_complete;
 
 	ret = v4l2_async_notifier_register(&isp->v4l2_dev, &isp->notifier);
diff --git a/drivers/media/platform/omap3isp/isp.h b/drivers/media/platform/omap3isp/isp.h
index 2f2ae609c548..e528df6efc09 100644
--- a/drivers/media/platform/omap3isp/isp.h
+++ b/drivers/media/platform/omap3isp/isp.h
@@ -226,11 +226,13 @@ struct isp_device {
 };
 
 struct isp_async_subdev {
-	struct v4l2_subdev *sd;
 	struct isp_bus_cfg bus;
 	struct v4l2_async_subdev asd;
 };
 
+#define v4l2_subdev_to_bus_cfg(sd) \
+	(&container_of((sd)->asd, struct isp_async_subdev, asd)->bus)
+
 #define v4l2_dev_to_isp_device(dev) \
 	container_of(dev, struct isp_device, v4l2_dev)
 
diff --git a/drivers/media/platform/omap3isp/ispccdc.c b/drivers/media/platform/omap3isp/ispccdc.c
index 7207558d722c..b66276ab5765 100644
--- a/drivers/media/platform/omap3isp/ispccdc.c
+++ b/drivers/media/platform/omap3isp/ispccdc.c
@@ -1139,15 +1139,11 @@ static void ccdc_configure(struct isp_ccdc_device *ccdc)
 	pad = media_entity_remote_pad(&ccdc->pads[CCDC_PAD_SINK]);
 	sensor = media_entity_to_v4l2_subdev(pad->entity);
 	if (ccdc->input == CCDC_INPUT_PARALLEL) {
-		struct v4l2_mbus_config cfg;
-		int ret;
+		struct v4l2_subdev *sd =
+			to_isp_pipeline(&ccdc->subdev.entity)->external;
 
-		ret = v4l2_subdev_call(sensor, video, g_mbus_config, &cfg);
-		if (!ret)
-			ccdc->bt656 = cfg.type == V4L2_MBUS_BT656;
-
-		parcfg = &((struct isp_bus_cfg *)sensor->host_priv)
-			->bus.parallel;
+		parcfg = &v4l2_subdev_to_bus_cfg(sd)->bus.parallel;
+		ccdc->bt656 = parcfg->bt656;
 	}
 
 	/* CCDC_PAD_SINK */
@@ -2418,11 +2414,11 @@ static int ccdc_link_validate(struct v4l2_subdev *sd,
 
 	/* We've got a parallel sensor here. */
 	if (ccdc->input == CCDC_INPUT_PARALLEL) {
-		struct isp_parallel_cfg *parcfg =
-			&((struct isp_bus_cfg *)
-			  media_entity_to_v4l2_subdev(link->source->entity)
-			  ->host_priv)->bus.parallel;
-		parallel_shift = parcfg->data_lane_shift;
+		struct v4l2_subdev *sd =
+			media_entity_to_v4l2_subdev(link->source->entity);
+		struct isp_bus_cfg *bus_cfg = v4l2_subdev_to_bus_cfg(sd);
+
+		parallel_shift = bus_cfg->bus.parallel.data_lane_shift;
 	} else {
 		parallel_shift = 0;
 	}
diff --git a/drivers/media/platform/omap3isp/ispccp2.c b/drivers/media/platform/omap3isp/ispccp2.c
index ca095238510d..e062939d0d05 100644
--- a/drivers/media/platform/omap3isp/ispccp2.c
+++ b/drivers/media/platform/omap3isp/ispccp2.c
@@ -213,14 +213,17 @@ static int ccp2_phyif_config(struct isp_ccp2_device *ccp2,
 	struct isp_device *isp = to_isp_device(ccp2);
 	u32 val;
 
-	/* CCP2B mode */
 	val = isp_reg_readl(isp, OMAP3_ISP_IOMEM_CCP2, ISPCCP2_CTRL) |
-			    ISPCCP2_CTRL_IO_OUT_SEL | ISPCCP2_CTRL_MODE;
+			    ISPCCP2_CTRL_MODE;
 	/* Data/strobe physical layer */
 	BIT_SET(val, ISPCCP2_CTRL_PHY_SEL_SHIFT, ISPCCP2_CTRL_PHY_SEL_MASK,
 		buscfg->phy_layer);
+	BIT_SET(val, ISPCCP2_CTRL_IO_OUT_SEL_SHIFT,
+		ISPCCP2_CTRL_IO_OUT_SEL_MASK, buscfg->ccp2_mode);
 	BIT_SET(val, ISPCCP2_CTRL_INV_SHIFT, ISPCCP2_CTRL_INV_MASK,
 		buscfg->strobe_clk_pol);
+	BIT_SET(val, ISPCCP2_CTRL_VP_CLK_POL_SHIFT,
+		ISPCCP2_CTRL_VP_CLK_POL_MASK, buscfg->vp_clk_pol);
 	isp_reg_writel(isp, val, OMAP3_ISP_IOMEM_CCP2, ISPCCP2_CTRL);
 
 	val = isp_reg_readl(isp, OMAP3_ISP_IOMEM_CCP2, ISPCCP2_CTRL);
@@ -347,6 +350,7 @@ static void ccp2_lcx_config(struct isp_ccp2_device *ccp2,
  */
 static int ccp2_if_configure(struct isp_ccp2_device *ccp2)
 {
+	struct isp_pipeline *pipe = to_isp_pipeline(&ccp2->subdev.entity);
 	const struct isp_bus_cfg *buscfg;
 	struct v4l2_mbus_framefmt *format;
 	struct media_pad *pad;
@@ -358,7 +362,7 @@ static int ccp2_if_configure(struct isp_ccp2_device *ccp2)
 
 	pad = media_entity_remote_pad(&ccp2->pads[CCP2_PAD_SINK]);
 	sensor = media_entity_to_v4l2_subdev(pad->entity);
-	buscfg = sensor->host_priv;
+	buscfg = v4l2_subdev_to_bus_cfg(pipe->external);
 
 	ret = ccp2_phyif_config(ccp2, &buscfg->bus.ccp2);
 	if (ret < 0)
@@ -838,7 +842,7 @@ static int ccp2_s_stream(struct v4l2_subdev *sd, int enable)
 	switch (enable) {
 	case ISP_PIPELINE_STREAM_CONTINUOUS:
 		if (ccp2->phy) {
-			ret = omap3isp_csiphy_acquire(ccp2->phy);
+			ret = omap3isp_csiphy_acquire(ccp2->phy, &sd->entity);
 			if (ret < 0)
 				return ret;
 		}
@@ -1137,10 +1141,16 @@ int omap3isp_ccp2_init(struct isp_device *isp)
 	if (isp->revision == ISP_REVISION_2_0) {
 		ccp2->vdds_csib = devm_regulator_get(isp->dev, "vdds_csib");
 		if (IS_ERR(ccp2->vdds_csib)) {
+			if (PTR_ERR(ccp2->vdds_csib) == -EPROBE_DEFER) {
+				dev_dbg(isp->dev,
+					"Can't get regulator vdds_csib, deferring probing\n");
+				return -EPROBE_DEFER;
+			}
 			dev_dbg(isp->dev,
 				"Could not get regulator vdds_csib\n");
 			ccp2->vdds_csib = NULL;
 		}
+		ccp2->phy = &isp->isp_csiphy2;
 	} else if (isp->revision == ISP_REVISION_15_0) {
 		ccp2->phy = &isp->isp_csiphy1;
 	}
diff --git a/drivers/media/platform/omap3isp/ispcsi2.c b/drivers/media/platform/omap3isp/ispcsi2.c
index 7dae2fe0d42d..a4d3d030e81e 100644
--- a/drivers/media/platform/omap3isp/ispcsi2.c
+++ b/drivers/media/platform/omap3isp/ispcsi2.c
@@ -490,7 +490,7 @@ int omap3isp_csi2_reset(struct isp_csi2_device *csi2)
 	if (!csi2->available)
 		return -ENODEV;
 
-	if (csi2->phy->phy_in_use)
+	if (csi2->phy->entity)
 		return -EBUSY;
 
 	isp_reg_set(isp, csi2->regs1, ISPCSI2_SYSCONFIG,
@@ -566,7 +566,7 @@ static int csi2_configure(struct isp_csi2_device *csi2)
 
 	pad = media_entity_remote_pad(&csi2->pads[CSI2_PAD_SINK]);
 	sensor = media_entity_to_v4l2_subdev(pad->entity);
-	buscfg = sensor->host_priv;
+	buscfg = v4l2_subdev_to_bus_cfg(pipe->external);
 
 	csi2->frame_skip = 0;
 	v4l2_subdev_call(sensor, sensor, g_skip_frames, &csi2->frame_skip);
@@ -1053,7 +1053,7 @@ static int csi2_set_stream(struct v4l2_subdev *sd, int enable)
 
 	switch (enable) {
 	case ISP_PIPELINE_STREAM_CONTINUOUS:
-		if (omap3isp_csiphy_acquire(csi2->phy) < 0)
+		if (omap3isp_csiphy_acquire(csi2->phy, &sd->entity) < 0)
 			return -ENODEV;
 		if (csi2->output & CSI2_OUTPUT_MEMORY)
 			omap3isp_sbl_enable(isp, OMAP3_ISP_SBL_CSI2A_WRITE);
diff --git a/drivers/media/platform/omap3isp/ispcsiphy.c b/drivers/media/platform/omap3isp/ispcsiphy.c
index 871d4fe09c7f..a28fb79abaac 100644
--- a/drivers/media/platform/omap3isp/ispcsiphy.c
+++ b/drivers/media/platform/omap3isp/ispcsiphy.c
@@ -164,30 +164,28 @@ static int csiphy_set_power(struct isp_csiphy *phy, u32 power)
 
 static int omap3isp_csiphy_config(struct isp_csiphy *phy)
 {
-	struct isp_csi2_device *csi2 = phy->csi2;
-	struct isp_pipeline *pipe = to_isp_pipeline(&csi2->subdev.entity);
-	struct isp_bus_cfg *buscfg = pipe->external->host_priv;
+	struct isp_pipeline *pipe = to_isp_pipeline(phy->entity);
+	struct isp_bus_cfg *buscfg = v4l2_subdev_to_bus_cfg(pipe->external);
 	struct isp_csiphy_lanes_cfg *lanes;
 	int csi2_ddrclk_khz;
-	unsigned int used_lanes = 0;
+	unsigned int num_data_lanes, used_lanes = 0;
 	unsigned int i;
 	u32 reg;
 
-	if (!buscfg) {
-		struct isp_async_subdev *isd =
-			container_of(pipe->external->asd,
-				     struct isp_async_subdev, asd);
-		buscfg = &isd->bus;
-	}
-
 	if (buscfg->interface == ISP_INTERFACE_CCP2B_PHY1
-	    || buscfg->interface == ISP_INTERFACE_CCP2B_PHY2)
+	    || buscfg->interface == ISP_INTERFACE_CCP2B_PHY2) {
 		lanes = &buscfg->bus.ccp2.lanecfg;
-	else
+		num_data_lanes = 1;
+	} else {
 		lanes = &buscfg->bus.csi2.lanecfg;
+		num_data_lanes = buscfg->bus.csi2.num_data_lanes;
+	}
+
+	if (num_data_lanes > phy->num_data_lanes)
+		return -EINVAL;
 
 	/* Clock and data lanes verification */
-	for (i = 0; i < phy->num_data_lanes; i++) {
+	for (i = 0; i < num_data_lanes; i++) {
 		if (lanes->data[i].pol > 1 || lanes->data[i].pos > 3)
 			return -EINVAL;
 
@@ -216,7 +214,7 @@ static int omap3isp_csiphy_config(struct isp_csiphy *phy)
 	csi2_ddrclk_khz = pipe->external_rate / 1000
 		/ (2 * hweight32(used_lanes)) * pipe->external_width;
 
-	reg = isp_reg_readl(csi2->isp, phy->phy_regs, ISPCSIPHY_REG0);
+	reg = isp_reg_readl(phy->isp, phy->phy_regs, ISPCSIPHY_REG0);
 
 	reg &= ~(ISPCSIPHY_REG0_THS_TERM_MASK |
 		 ISPCSIPHY_REG0_THS_SETTLE_MASK);
@@ -227,9 +225,9 @@ static int omap3isp_csiphy_config(struct isp_csiphy *phy)
 	reg |= (DIV_ROUND_UP(90 * csi2_ddrclk_khz, 1000000) + 3)
 		<< ISPCSIPHY_REG0_THS_SETTLE_SHIFT;
 
-	isp_reg_writel(csi2->isp, reg, phy->phy_regs, ISPCSIPHY_REG0);
+	isp_reg_writel(phy->isp, reg, phy->phy_regs, ISPCSIPHY_REG0);
 
-	reg = isp_reg_readl(csi2->isp, phy->phy_regs, ISPCSIPHY_REG1);
+	reg = isp_reg_readl(phy->isp, phy->phy_regs, ISPCSIPHY_REG1);
 
 	reg &= ~(ISPCSIPHY_REG1_TCLK_TERM_MASK |
 		 ISPCSIPHY_REG1_TCLK_MISS_MASK |
@@ -238,12 +236,12 @@ static int omap3isp_csiphy_config(struct isp_csiphy *phy)
 	reg |= TCLK_MISS << ISPCSIPHY_REG1_TCLK_MISS_SHIFT;
 	reg |= TCLK_SETTLE << ISPCSIPHY_REG1_TCLK_SETTLE_SHIFT;
 
-	isp_reg_writel(csi2->isp, reg, phy->phy_regs, ISPCSIPHY_REG1);
+	isp_reg_writel(phy->isp, reg, phy->phy_regs, ISPCSIPHY_REG1);
 
 	/* DPHY lane configuration */
-	reg = isp_reg_readl(csi2->isp, phy->cfg_regs, ISPCSI2_PHY_CFG);
+	reg = isp_reg_readl(phy->isp, phy->cfg_regs, ISPCSI2_PHY_CFG);
 
-	for (i = 0; i < phy->num_data_lanes; i++) {
+	for (i = 0; i < num_data_lanes; i++) {
 		reg &= ~(ISPCSI2_PHY_CFG_DATA_POL_MASK(i + 1) |
 			 ISPCSI2_PHY_CFG_DATA_POSITION_MASK(i + 1));
 		reg |= (lanes->data[i].pol <<
@@ -257,12 +255,12 @@ static int omap3isp_csiphy_config(struct isp_csiphy *phy)
 	reg |= lanes->clk.pol << ISPCSI2_PHY_CFG_CLOCK_POL_SHIFT;
 	reg |= lanes->clk.pos << ISPCSI2_PHY_CFG_CLOCK_POSITION_SHIFT;
 
-	isp_reg_writel(csi2->isp, reg, phy->cfg_regs, ISPCSI2_PHY_CFG);
+	isp_reg_writel(phy->isp, reg, phy->cfg_regs, ISPCSI2_PHY_CFG);
 
 	return 0;
 }
 
-int omap3isp_csiphy_acquire(struct isp_csiphy *phy)
+int omap3isp_csiphy_acquire(struct isp_csiphy *phy, struct media_entity *entity)
 {
 	int rval;
 
@@ -282,20 +280,25 @@ int omap3isp_csiphy_acquire(struct isp_csiphy *phy)
 	if (rval < 0)
 		goto done;
 
+	phy->entity = entity;
+
 	rval = omap3isp_csiphy_config(phy);
 	if (rval < 0)
 		goto done;
 
-	rval = csiphy_set_power(phy, ISPCSI2_PHY_CFG_PWR_CMD_ON);
-	if (rval) {
-		regulator_disable(phy->vdd);
-		goto done;
-	}
-
-	csiphy_power_autoswitch_enable(phy, true);
-	phy->phy_in_use = 1;
+	if (phy->isp->revision == ISP_REVISION_15_0) {
+		rval = csiphy_set_power(phy, ISPCSI2_PHY_CFG_PWR_CMD_ON);
+		if (rval) {
+			regulator_disable(phy->vdd);
+			goto done;
+		}
 
+		csiphy_power_autoswitch_enable(phy, true);
+	}
 done:
+	if (rval < 0)
+		phy->entity = NULL;
+
 	mutex_unlock(&phy->mutex);
 	return rval;
 }
@@ -303,18 +306,19 @@ done:
 void omap3isp_csiphy_release(struct isp_csiphy *phy)
 {
 	mutex_lock(&phy->mutex);
-	if (phy->phy_in_use) {
-		struct isp_csi2_device *csi2 = phy->csi2;
-		struct isp_pipeline *pipe =
-			to_isp_pipeline(&csi2->subdev.entity);
-		struct isp_bus_cfg *buscfg = pipe->external->host_priv;
+	if (phy->entity) {
+		struct isp_pipeline *pipe = to_isp_pipeline(phy->entity);
+		struct isp_bus_cfg *buscfg =
+			v4l2_subdev_to_bus_cfg(pipe->external);
 
 		csiphy_routing_cfg(phy, buscfg->interface, false,
 				   buscfg->bus.ccp2.phy_layer);
-		csiphy_power_autoswitch_enable(phy, false);
-		csiphy_set_power(phy, ISPCSI2_PHY_CFG_PWR_CMD_OFF);
+		if (phy->isp->revision == ISP_REVISION_15_0) {
+			csiphy_power_autoswitch_enable(phy, false);
+			csiphy_set_power(phy, ISPCSI2_PHY_CFG_PWR_CMD_OFF);
+		}
 		regulator_disable(phy->vdd);
-		phy->phy_in_use = 0;
+		phy->entity = NULL;
 	}
 	mutex_unlock(&phy->mutex);
 }
@@ -334,14 +338,21 @@ int omap3isp_csiphy_init(struct isp_device *isp)
 	phy2->phy_regs = OMAP3_ISP_IOMEM_CSIPHY2;
 	mutex_init(&phy2->mutex);
 
+	phy1->isp = isp;
+	mutex_init(&phy1->mutex);
+
 	if (isp->revision == ISP_REVISION_15_0) {
-		phy1->isp = isp;
 		phy1->csi2 = &isp->isp_csi2c;
 		phy1->num_data_lanes = ISP_CSIPHY1_NUM_DATA_LANES;
 		phy1->cfg_regs = OMAP3_ISP_IOMEM_CSI2C_REGS1;
 		phy1->phy_regs = OMAP3_ISP_IOMEM_CSIPHY1;
-		mutex_init(&phy1->mutex);
 	}
 
 	return 0;
 }
+
+void omap3isp_csiphy_cleanup(struct isp_device *isp)
+{
+	mutex_destroy(&isp->isp_csiphy1.mutex);
+	mutex_destroy(&isp->isp_csiphy2.mutex);
+}
diff --git a/drivers/media/platform/omap3isp/ispcsiphy.h b/drivers/media/platform/omap3isp/ispcsiphy.h
index 28b63b28f9f7..91543a09b28a 100644
--- a/drivers/media/platform/omap3isp/ispcsiphy.h
+++ b/drivers/media/platform/omap3isp/ispcsiphy.h
@@ -25,9 +25,10 @@ struct regulator;
 struct isp_csiphy {
 	struct isp_device *isp;
 	struct mutex mutex;	/* serialize csiphy configuration */
-	u8 phy_in_use;
 	struct isp_csi2_device *csi2;
 	struct regulator *vdd;
+	/* the entity that acquired the phy */
+	struct media_entity *entity;
 
 	/* mem resources - enums as defined in enum isp_mem_resources */
 	unsigned int cfg_regs;
@@ -36,8 +37,10 @@ struct isp_csiphy {
 	u8 num_data_lanes;	/* number of CSI2 Data Lanes supported */
 };
 
-int omap3isp_csiphy_acquire(struct isp_csiphy *phy);
+int omap3isp_csiphy_acquire(struct isp_csiphy *phy,
+			    struct media_entity *entity);
 void omap3isp_csiphy_release(struct isp_csiphy *phy);
 int omap3isp_csiphy_init(struct isp_device *isp);
+void omap3isp_csiphy_cleanup(struct isp_device *isp);
 
 #endif	/* OMAP3_ISP_CSI_PHY_H */
diff --git a/drivers/media/platform/omap3isp/ispreg.h b/drivers/media/platform/omap3isp/ispreg.h
index b5ea8da0b904..d08483919a77 100644
--- a/drivers/media/platform/omap3isp/ispreg.h
+++ b/drivers/media/platform/omap3isp/ispreg.h
@@ -87,6 +87,8 @@
 #define ISPCCP2_CTRL_PHY_SEL_MASK	0x1
 #define ISPCCP2_CTRL_PHY_SEL_SHIFT	1
 #define ISPCCP2_CTRL_IO_OUT_SEL		(1 << 2)
+#define ISPCCP2_CTRL_IO_OUT_SEL_MASK	0x1
+#define ISPCCP2_CTRL_IO_OUT_SEL_SHIFT	2
 #define ISPCCP2_CTRL_MODE		(1 << 4)
 #define ISPCCP2_CTRL_VP_CLK_FORCE_ON	(1 << 9)
 #define ISPCCP2_CTRL_INV		(1 << 10)
@@ -94,6 +96,8 @@
 #define ISPCCP2_CTRL_INV_SHIFT		10
 #define ISPCCP2_CTRL_VP_ONLY_EN		(1 << 11)
 #define ISPCCP2_CTRL_VP_CLK_POL		(1 << 12)
+#define ISPCCP2_CTRL_VP_CLK_POL_MASK	0x1
+#define ISPCCP2_CTRL_VP_CLK_POL_SHIFT	12
 #define ISPCCP2_CTRL_VPCLK_DIV_SHIFT	15
 #define ISPCCP2_CTRL_VPCLK_DIV_MASK	0x1ffff /* [31:15] */
 #define ISPCCP2_CTRL_VP_OUT_CTRL_SHIFT	8 /* 3430 bits */
diff --git a/drivers/media/platform/omap3isp/omap3isp.h b/drivers/media/platform/omap3isp/omap3isp.h
index 443e8f7673e2..9fb4d5bce004 100644
--- a/drivers/media/platform/omap3isp/omap3isp.h
+++ b/drivers/media/platform/omap3isp/omap3isp.h
@@ -46,6 +46,7 @@ enum isp_interface_type {
  *		0 - Positive, 1 - Negative
  * @data_pol: Data polarity
  *		0 - Normal, 1 - One's complement
+ * @bt656: Data contain BT.656 embedded synchronization
  */
 struct isp_parallel_cfg {
 	unsigned int data_lane_shift:3;
@@ -54,6 +55,7 @@ struct isp_parallel_cfg {
 	unsigned int vs_pol:1;
 	unsigned int fld_pol:1;
 	unsigned int data_pol:1;
+	unsigned int bt656:1;
 };
 
 enum {
@@ -108,16 +110,20 @@ struct isp_ccp2_cfg {
 	unsigned int ccp2_mode:1;
 	unsigned int phy_layer:1;
 	unsigned int vpclk_div:2;
+	unsigned int vp_clk_pol:1;
 	struct isp_csiphy_lanes_cfg lanecfg;
 };
 
 /**
  * struct isp_csi2_cfg - CSI2 interface configuration
  * @crc: Enable the cyclic redundancy check
+ * @lanecfg: CSI-2 lane configuration
+ * @num_data_lanes: The number of data lanes in use
  */
 struct isp_csi2_cfg {
 	unsigned crc:1;
 	struct isp_csiphy_lanes_cfg lanecfg;
+	u8 num_data_lanes;
 };
 
 struct isp_bus_cfg {
diff --git a/drivers/media/platform/pxa_camera.c b/drivers/media/platform/pxa_camera.c
index 399095170b6e..edca993c2b1f 100644
--- a/drivers/media/platform/pxa_camera.c
+++ b/drivers/media/platform/pxa_camera.c
@@ -638,6 +638,9 @@ static unsigned int pxa_mbus_config_compatible(const struct v4l2_mbus_config *cf
 		mipi_clock = common_flags & (V4L2_MBUS_CSI2_NONCONTINUOUS_CLOCK |
 					     V4L2_MBUS_CSI2_CONTINUOUS_CLOCK);
 		return (!mipi_lanes || !mipi_clock) ? 0 : common_flags;
+	default:
+		WARN_ON(1);
+		return -EINVAL;
 	}
 	return 0;
 }
@@ -1557,7 +1560,7 @@ static void pxac_vb2_stop_streaming(struct vb2_queue *vq)
 		pxa_camera_wakeup(pcdev, buf, VB2_BUF_STATE_ERROR);
 }
 
-static struct vb2_ops pxac_vb2_ops = {
+static const struct vb2_ops pxac_vb2_ops = {
 	.queue_setup		= pxac_vb2_queue_setup,
 	.buf_init		= pxac_vb2_init,
 	.buf_prepare		= pxac_vb2_prepare,
@@ -2097,7 +2100,7 @@ static const struct v4l2_ioctl_ops pxa_camera_ioctl_ops = {
 	.vidioc_unsubscribe_event	= v4l2_event_unsubscribe,
 };
 
-static struct v4l2_clk_ops pxa_camera_mclk_ops = {
+static const struct v4l2_clk_ops pxa_camera_mclk_ops = {
 };
 
 static const struct video_device pxa_camera_videodev_template = {
@@ -2328,7 +2331,7 @@ static int pxa_camera_pdata_from_dt(struct device *dev,
 		asd->match.fwnode.fwnode = of_fwnode_handle(remote);
 		of_node_put(remote);
 	} else {
-		dev_notice(dev, "no remote for %s\n", of_node_full_name(np));
+		dev_notice(dev, "no remote for %pOF\n", np);
 	}
 
 out:
diff --git a/drivers/media/platform/qcom/camss-8x16/Makefile b/drivers/media/platform/qcom/camss-8x16/Makefile
new file mode 100644
index 000000000000..3c4024fbb768
--- /dev/null
+++ b/drivers/media/platform/qcom/camss-8x16/Makefile
@@ -0,0 +1,11 @@
+# Makefile for Qualcomm CAMSS driver
+
+qcom-camss-objs += \
+		camss.o \
+		camss-csid.o \
+		camss-csiphy.o \
+		camss-ispif.o \
+		camss-vfe.o \
+		camss-video.o \
+
+obj-$(CONFIG_VIDEO_QCOM_CAMSS) += qcom-camss.o
diff --git a/drivers/media/platform/qcom/camss-8x16/camss-csid.c b/drivers/media/platform/qcom/camss-8x16/camss-csid.c
new file mode 100644
index 000000000000..64df82817de3
--- /dev/null
+++ b/drivers/media/platform/qcom/camss-8x16/camss-csid.c
@@ -0,0 +1,1092 @@
+/*
+ * camss-csid.c
+ *
+ * Qualcomm MSM Camera Subsystem - CSID (CSI Decoder) Module
+ *
+ * Copyright (c) 2011-2015, The Linux Foundation. All rights reserved.
+ * Copyright (C) 2015-2017 Linaro Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#include <linux/clk.h>
+#include <linux/completion.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regulator/consumer.h>
+#include <media/media-entity.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-subdev.h>
+
+#include "camss-csid.h"
+#include "camss.h"
+
+#define MSM_CSID_NAME "msm_csid"
+
+#define CAMSS_CSID_HW_VERSION		0x0
+#define CAMSS_CSID_CORE_CTRL_0		0x004
+#define CAMSS_CSID_CORE_CTRL_1		0x008
+#define CAMSS_CSID_RST_CMD		0x00c
+#define CAMSS_CSID_CID_LUT_VC_n(n)	(0x010 + 0x4 * (n))
+#define CAMSS_CSID_CID_n_CFG(n)		(0x020 + 0x4 * (n))
+#define CAMSS_CSID_IRQ_CLEAR_CMD	0x060
+#define CAMSS_CSID_IRQ_MASK		0x064
+#define CAMSS_CSID_IRQ_STATUS		0x068
+#define CAMSS_CSID_TG_CTRL		0x0a0
+#define CAMSS_CSID_TG_CTRL_DISABLE	0xa06436
+#define CAMSS_CSID_TG_CTRL_ENABLE	0xa06437
+#define CAMSS_CSID_TG_VC_CFG		0x0a4
+#define CAMSS_CSID_TG_VC_CFG_H_BLANKING		0x3ff
+#define CAMSS_CSID_TG_VC_CFG_V_BLANKING		0x7f
+#define CAMSS_CSID_TG_DT_n_CGG_0(n)	(0x0ac + 0xc * (n))
+#define CAMSS_CSID_TG_DT_n_CGG_1(n)	(0x0b0 + 0xc * (n))
+#define CAMSS_CSID_TG_DT_n_CGG_2(n)	(0x0b4 + 0xc * (n))
+
+#define DATA_TYPE_EMBEDDED_DATA_8BIT	0x12
+#define DATA_TYPE_YUV422_8BIT		0x1e
+#define DATA_TYPE_RAW_6BIT		0x28
+#define DATA_TYPE_RAW_8BIT		0x2a
+#define DATA_TYPE_RAW_10BIT		0x2b
+#define DATA_TYPE_RAW_12BIT		0x2c
+
+#define DECODE_FORMAT_UNCOMPRESSED_6_BIT	0x0
+#define DECODE_FORMAT_UNCOMPRESSED_8_BIT	0x1
+#define DECODE_FORMAT_UNCOMPRESSED_10_BIT	0x2
+#define DECODE_FORMAT_UNCOMPRESSED_12_BIT	0x3
+
+#define CSID_RESET_TIMEOUT_MS 500
+
+struct csid_fmts {
+	u32 code;
+	u8 data_type;
+	u8 decode_format;
+	u8 bpp;
+	u8 spp; /* bus samples per pixel */
+};
+
+static const struct csid_fmts csid_input_fmts[] = {
+	{
+		MEDIA_BUS_FMT_UYVY8_2X8,
+		DATA_TYPE_YUV422_8BIT,
+		DECODE_FORMAT_UNCOMPRESSED_8_BIT,
+		8,
+		2,
+	},
+	{
+		MEDIA_BUS_FMT_VYUY8_2X8,
+		DATA_TYPE_YUV422_8BIT,
+		DECODE_FORMAT_UNCOMPRESSED_8_BIT,
+		8,
+		2,
+	},
+	{
+		MEDIA_BUS_FMT_YUYV8_2X8,
+		DATA_TYPE_YUV422_8BIT,
+		DECODE_FORMAT_UNCOMPRESSED_8_BIT,
+		8,
+		2,
+	},
+	{
+		MEDIA_BUS_FMT_YVYU8_2X8,
+		DATA_TYPE_YUV422_8BIT,
+		DECODE_FORMAT_UNCOMPRESSED_8_BIT,
+		8,
+		2,
+	},
+	{
+		MEDIA_BUS_FMT_SBGGR8_1X8,
+		DATA_TYPE_RAW_8BIT,
+		DECODE_FORMAT_UNCOMPRESSED_8_BIT,
+		8,
+		1,
+	},
+	{
+		MEDIA_BUS_FMT_SGBRG8_1X8,
+		DATA_TYPE_RAW_8BIT,
+		DECODE_FORMAT_UNCOMPRESSED_8_BIT,
+		8,
+		1,
+	},
+	{
+		MEDIA_BUS_FMT_SGRBG8_1X8,
+		DATA_TYPE_RAW_8BIT,
+		DECODE_FORMAT_UNCOMPRESSED_8_BIT,
+		8,
+		1,
+	},
+	{
+		MEDIA_BUS_FMT_SRGGB8_1X8,
+		DATA_TYPE_RAW_8BIT,
+		DECODE_FORMAT_UNCOMPRESSED_8_BIT,
+		8,
+		1,
+	},
+	{
+		MEDIA_BUS_FMT_SBGGR10_1X10,
+		DATA_TYPE_RAW_10BIT,
+		DECODE_FORMAT_UNCOMPRESSED_10_BIT,
+		10,
+		1,
+	},
+	{
+		MEDIA_BUS_FMT_SGBRG10_1X10,
+		DATA_TYPE_RAW_10BIT,
+		DECODE_FORMAT_UNCOMPRESSED_10_BIT,
+		10,
+		1,
+	},
+	{
+		MEDIA_BUS_FMT_SGRBG10_1X10,
+		DATA_TYPE_RAW_10BIT,
+		DECODE_FORMAT_UNCOMPRESSED_10_BIT,
+		10,
+		1,
+	},
+	{
+		MEDIA_BUS_FMT_SRGGB10_1X10,
+		DATA_TYPE_RAW_10BIT,
+		DECODE_FORMAT_UNCOMPRESSED_10_BIT,
+		10,
+		1,
+	},
+	{
+		MEDIA_BUS_FMT_SBGGR12_1X12,
+		DATA_TYPE_RAW_12BIT,
+		DECODE_FORMAT_UNCOMPRESSED_12_BIT,
+		12,
+		1,
+	},
+	{
+		MEDIA_BUS_FMT_SGBRG12_1X12,
+		DATA_TYPE_RAW_12BIT,
+		DECODE_FORMAT_UNCOMPRESSED_12_BIT,
+		12,
+		1,
+	},
+	{
+		MEDIA_BUS_FMT_SGRBG12_1X12,
+		DATA_TYPE_RAW_12BIT,
+		DECODE_FORMAT_UNCOMPRESSED_12_BIT,
+		12,
+		1,
+	},
+	{
+		MEDIA_BUS_FMT_SRGGB12_1X12,
+		DATA_TYPE_RAW_12BIT,
+		DECODE_FORMAT_UNCOMPRESSED_12_BIT,
+		12,
+		1,
+	}
+};
+
+static const struct csid_fmts *csid_get_fmt_entry(u32 code)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(csid_input_fmts); i++)
+		if (code == csid_input_fmts[i].code)
+			return &csid_input_fmts[i];
+
+	WARN(1, "Unknown format\n");
+
+	return &csid_input_fmts[0];
+}
+
+/*
+ * csid_isr - CSID module interrupt handler
+ * @irq: Interrupt line
+ * @dev: CSID device
+ *
+ * Return IRQ_HANDLED on success
+ */
+static irqreturn_t csid_isr(int irq, void *dev)
+{
+	struct csid_device *csid = dev;
+	u32 value;
+
+	value = readl_relaxed(csid->base + CAMSS_CSID_IRQ_STATUS);
+	writel_relaxed(value, csid->base + CAMSS_CSID_IRQ_CLEAR_CMD);
+
+	if ((value >> 11) & 0x1)
+		complete(&csid->reset_complete);
+
+	return IRQ_HANDLED;
+}
+
+/*
+ * csid_set_clock_rates - Calculate and set clock rates on CSID module
+ * @csiphy: CSID device
+ */
+static int csid_set_clock_rates(struct csid_device *csid)
+{
+	struct device *dev = to_device_index(csid, csid->id);
+	u32 pixel_clock;
+	int i, j;
+	int ret;
+
+	ret = camss_get_pixel_clock(&csid->subdev.entity, &pixel_clock);
+	if (ret)
+		pixel_clock = 0;
+
+	for (i = 0; i < csid->nclocks; i++) {
+		struct camss_clock *clock = &csid->clock[i];
+
+		if (!strcmp(clock->name, "csi0") ||
+			!strcmp(clock->name, "csi1")) {
+			u8 bpp = csid_get_fmt_entry(
+				csid->fmt[MSM_CSIPHY_PAD_SINK].code)->bpp;
+			u8 num_lanes = csid->phy.lane_cnt;
+			u64 min_rate = pixel_clock * bpp / (2 * num_lanes * 4);
+			long rate;
+
+			camss_add_clock_margin(&min_rate);
+
+			for (j = 0; j < clock->nfreqs; j++)
+				if (min_rate < clock->freq[j])
+					break;
+
+			if (j == clock->nfreqs) {
+				dev_err(dev,
+					"Pixel clock is too high for CSID\n");
+				return -EINVAL;
+			}
+
+			/* if sensor pixel clock is not available */
+			/* set highest possible CSID clock rate */
+			if (min_rate == 0)
+				j = clock->nfreqs - 1;
+
+			rate = clk_round_rate(clock->clk, clock->freq[j]);
+			if (rate < 0) {
+				dev_err(dev, "clk round rate failed: %ld\n",
+					rate);
+				return -EINVAL;
+			}
+
+			ret = clk_set_rate(clock->clk, rate);
+			if (ret < 0) {
+				dev_err(dev, "clk set rate failed: %d\n", ret);
+				return ret;
+			}
+		}
+	}
+
+	return 0;
+}
+
+/*
+ * csid_reset - Trigger reset on CSID module and wait to complete
+ * @csid: CSID device
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int csid_reset(struct csid_device *csid)
+{
+	unsigned long time;
+
+	reinit_completion(&csid->reset_complete);
+
+	writel_relaxed(0x7fff, csid->base + CAMSS_CSID_RST_CMD);
+
+	time = wait_for_completion_timeout(&csid->reset_complete,
+		msecs_to_jiffies(CSID_RESET_TIMEOUT_MS));
+	if (!time) {
+		dev_err(to_device_index(csid, csid->id),
+			"CSID reset timeout\n");
+		return -EIO;
+	}
+
+	return 0;
+}
+
+/*
+ * csid_set_power - Power on/off CSID module
+ * @sd: CSID V4L2 subdevice
+ * @on: Requested power state
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int csid_set_power(struct v4l2_subdev *sd, int on)
+{
+	struct csid_device *csid = v4l2_get_subdevdata(sd);
+	struct device *dev = to_device_index(csid, csid->id);
+	int ret;
+
+	if (on) {
+		u32 hw_version;
+
+		ret = regulator_enable(csid->vdda);
+		if (ret < 0)
+			return ret;
+
+		ret = csid_set_clock_rates(csid);
+		if (ret < 0) {
+			regulator_disable(csid->vdda);
+			return ret;
+		}
+
+		ret = camss_enable_clocks(csid->nclocks, csid->clock, dev);
+		if (ret < 0) {
+			regulator_disable(csid->vdda);
+			return ret;
+		}
+
+		enable_irq(csid->irq);
+
+		ret = csid_reset(csid);
+		if (ret < 0) {
+			disable_irq(csid->irq);
+			camss_disable_clocks(csid->nclocks, csid->clock);
+			regulator_disable(csid->vdda);
+			return ret;
+		}
+
+		hw_version = readl_relaxed(csid->base + CAMSS_CSID_HW_VERSION);
+		dev_dbg(dev, "CSID HW Version = 0x%08x\n", hw_version);
+	} else {
+		disable_irq(csid->irq);
+		camss_disable_clocks(csid->nclocks, csid->clock);
+		ret = regulator_disable(csid->vdda);
+	}
+
+	return ret;
+}
+
+/*
+ * csid_set_stream - Enable/disable streaming on CSID module
+ * @sd: CSID V4L2 subdevice
+ * @enable: Requested streaming state
+ *
+ * Main configuration of CSID module is also done here.
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int csid_set_stream(struct v4l2_subdev *sd, int enable)
+{
+	struct csid_device *csid = v4l2_get_subdevdata(sd);
+	struct csid_testgen_config *tg = &csid->testgen;
+	u32 val;
+
+	if (enable) {
+		u8 vc = 0; /* Virtual Channel 0 */
+		u8 cid = vc * 4; /* id of Virtual Channel and Data Type set */
+		u8 dt, dt_shift, df;
+		int ret;
+
+		ret = v4l2_ctrl_handler_setup(&csid->ctrls);
+		if (ret < 0) {
+			dev_err(to_device_index(csid, csid->id),
+				"could not sync v4l2 controls: %d\n", ret);
+			return ret;
+		}
+
+		if (!tg->enabled &&
+		    !media_entity_remote_pad(&csid->pads[MSM_CSID_PAD_SINK]))
+			return -ENOLINK;
+
+		dt = csid_get_fmt_entry(csid->fmt[MSM_CSID_PAD_SRC].code)->
+								data_type;
+
+		if (tg->enabled) {
+			/* Config Test Generator */
+			struct v4l2_mbus_framefmt *f =
+					&csid->fmt[MSM_CSID_PAD_SRC];
+			u8 bpp = csid_get_fmt_entry(f->code)->bpp;
+			u8 spp = csid_get_fmt_entry(f->code)->spp;
+			u32 num_bytes_per_line = f->width * bpp * spp / 8;
+			u32 num_lines = f->height;
+
+			/* 31:24 V blank, 23:13 H blank, 3:2 num of active DT */
+			/* 1:0 VC */
+			val = ((CAMSS_CSID_TG_VC_CFG_V_BLANKING & 0xff) << 24) |
+			      ((CAMSS_CSID_TG_VC_CFG_H_BLANKING & 0x7ff) << 13);
+			writel_relaxed(val, csid->base + CAMSS_CSID_TG_VC_CFG);
+
+			/* 28:16 bytes per lines, 12:0 num of lines */
+			val = ((num_bytes_per_line & 0x1fff) << 16) |
+			      (num_lines & 0x1fff);
+			writel_relaxed(val, csid->base +
+				       CAMSS_CSID_TG_DT_n_CGG_0(0));
+
+			/* 5:0 data type */
+			val = dt;
+			writel_relaxed(val, csid->base +
+				       CAMSS_CSID_TG_DT_n_CGG_1(0));
+
+			/* 2:0 output test pattern */
+			val = tg->payload_mode;
+			writel_relaxed(val, csid->base +
+				       CAMSS_CSID_TG_DT_n_CGG_2(0));
+		} else {
+			struct csid_phy_config *phy = &csid->phy;
+
+			val = phy->lane_cnt - 1;
+			val |= phy->lane_assign << 4;
+
+			writel_relaxed(val,
+				       csid->base + CAMSS_CSID_CORE_CTRL_0);
+
+			val = phy->csiphy_id << 17;
+			val |= 0x9;
+
+			writel_relaxed(val,
+				       csid->base + CAMSS_CSID_CORE_CTRL_1);
+		}
+
+		/* Config LUT */
+
+		dt_shift = (cid % 4) * 8;
+		df = csid_get_fmt_entry(csid->fmt[MSM_CSID_PAD_SINK].code)->
+								decode_format;
+
+		val = readl_relaxed(csid->base + CAMSS_CSID_CID_LUT_VC_n(vc));
+		val &= ~(0xff << dt_shift);
+		val |= dt << dt_shift;
+		writel_relaxed(val, csid->base + CAMSS_CSID_CID_LUT_VC_n(vc));
+
+		val = (df << 4) | 0x3;
+		writel_relaxed(val, csid->base + CAMSS_CSID_CID_n_CFG(cid));
+
+		if (tg->enabled) {
+			val = CAMSS_CSID_TG_CTRL_ENABLE;
+			writel_relaxed(val, csid->base + CAMSS_CSID_TG_CTRL);
+		}
+	} else {
+		if (tg->enabled) {
+			val = CAMSS_CSID_TG_CTRL_DISABLE;
+			writel_relaxed(val, csid->base + CAMSS_CSID_TG_CTRL);
+		}
+	}
+
+	return 0;
+}
+
+/*
+ * __csid_get_format - Get pointer to format structure
+ * @csid: CSID device
+ * @cfg: V4L2 subdev pad configuration
+ * @pad: pad from which format is requested
+ * @which: TRY or ACTIVE format
+ *
+ * Return pointer to TRY or ACTIVE format structure
+ */
+static struct v4l2_mbus_framefmt *
+__csid_get_format(struct csid_device *csid,
+		  struct v4l2_subdev_pad_config *cfg,
+		  unsigned int pad,
+		  enum v4l2_subdev_format_whence which)
+{
+	if (which == V4L2_SUBDEV_FORMAT_TRY)
+		return v4l2_subdev_get_try_format(&csid->subdev, cfg, pad);
+
+	return &csid->fmt[pad];
+}
+
+/*
+ * csid_try_format - Handle try format by pad subdev method
+ * @csid: CSID device
+ * @cfg: V4L2 subdev pad configuration
+ * @pad: pad on which format is requested
+ * @fmt: pointer to v4l2 format structure
+ * @which: wanted subdev format
+ */
+static void csid_try_format(struct csid_device *csid,
+			    struct v4l2_subdev_pad_config *cfg,
+			    unsigned int pad,
+			    struct v4l2_mbus_framefmt *fmt,
+			    enum v4l2_subdev_format_whence which)
+{
+	unsigned int i;
+
+	switch (pad) {
+	case MSM_CSID_PAD_SINK:
+		/* Set format on sink pad */
+
+		for (i = 0; i < ARRAY_SIZE(csid_input_fmts); i++)
+			if (fmt->code == csid_input_fmts[i].code)
+				break;
+
+		/* If not found, use UYVY as default */
+		if (i >= ARRAY_SIZE(csid_input_fmts))
+			fmt->code = MEDIA_BUS_FMT_UYVY8_2X8;
+
+		fmt->width = clamp_t(u32, fmt->width, 1, 8191);
+		fmt->height = clamp_t(u32, fmt->height, 1, 8191);
+
+		fmt->field = V4L2_FIELD_NONE;
+		fmt->colorspace = V4L2_COLORSPACE_SRGB;
+
+		break;
+
+	case MSM_CSID_PAD_SRC:
+		if (csid->testgen_mode->cur.val == 0) {
+			/* Test generator is disabled, keep pad formats */
+			/* in sync - set and return a format same as sink pad */
+			struct v4l2_mbus_framefmt format;
+
+			format = *__csid_get_format(csid, cfg,
+						    MSM_CSID_PAD_SINK, which);
+			*fmt = format;
+		} else {
+			/* Test generator is enabled, set format on source*/
+			/* pad to allow test generator usage */
+
+			for (i = 0; i < ARRAY_SIZE(csid_input_fmts); i++)
+				if (csid_input_fmts[i].code == fmt->code)
+					break;
+
+			/* If not found, use UYVY as default */
+			if (i >= ARRAY_SIZE(csid_input_fmts))
+				fmt->code = MEDIA_BUS_FMT_UYVY8_2X8;
+
+			fmt->width = clamp_t(u32, fmt->width, 1, 8191);
+			fmt->height = clamp_t(u32, fmt->height, 1, 8191);
+
+			fmt->field = V4L2_FIELD_NONE;
+		}
+		break;
+	}
+
+	fmt->colorspace = V4L2_COLORSPACE_SRGB;
+}
+
+/*
+ * csid_enum_mbus_code - Handle pixel format enumeration
+ * @sd: CSID V4L2 subdevice
+ * @cfg: V4L2 subdev pad configuration
+ * @code: pointer to v4l2_subdev_mbus_code_enum structure
+ * return -EINVAL or zero on success
+ */
+static int csid_enum_mbus_code(struct v4l2_subdev *sd,
+			       struct v4l2_subdev_pad_config *cfg,
+			       struct v4l2_subdev_mbus_code_enum *code)
+{
+	struct csid_device *csid = v4l2_get_subdevdata(sd);
+	struct v4l2_mbus_framefmt *format;
+
+	if (code->pad == MSM_CSID_PAD_SINK) {
+		if (code->index >= ARRAY_SIZE(csid_input_fmts))
+			return -EINVAL;
+
+		code->code = csid_input_fmts[code->index].code;
+	} else {
+		if (csid->testgen_mode->cur.val == 0) {
+			if (code->index > 0)
+				return -EINVAL;
+
+			format = __csid_get_format(csid, cfg, MSM_CSID_PAD_SINK,
+						   code->which);
+
+			code->code = format->code;
+		} else {
+			if (code->index >= ARRAY_SIZE(csid_input_fmts))
+				return -EINVAL;
+
+			code->code = csid_input_fmts[code->index].code;
+		}
+	}
+
+	return 0;
+}
+
+/*
+ * csid_enum_frame_size - Handle frame size enumeration
+ * @sd: CSID V4L2 subdevice
+ * @cfg: V4L2 subdev pad configuration
+ * @fse: pointer to v4l2_subdev_frame_size_enum structure
+ * return -EINVAL or zero on success
+ */
+static int csid_enum_frame_size(struct v4l2_subdev *sd,
+				struct v4l2_subdev_pad_config *cfg,
+				struct v4l2_subdev_frame_size_enum *fse)
+{
+	struct csid_device *csid = v4l2_get_subdevdata(sd);
+	struct v4l2_mbus_framefmt format;
+
+	if (fse->index != 0)
+		return -EINVAL;
+
+	format.code = fse->code;
+	format.width = 1;
+	format.height = 1;
+	csid_try_format(csid, cfg, fse->pad, &format, fse->which);
+	fse->min_width = format.width;
+	fse->min_height = format.height;
+
+	if (format.code != fse->code)
+		return -EINVAL;
+
+	format.code = fse->code;
+	format.width = -1;
+	format.height = -1;
+	csid_try_format(csid, cfg, fse->pad, &format, fse->which);
+	fse->max_width = format.width;
+	fse->max_height = format.height;
+
+	return 0;
+}
+
+/*
+ * csid_get_format - Handle get format by pads subdev method
+ * @sd: CSID V4L2 subdevice
+ * @cfg: V4L2 subdev pad configuration
+ * @fmt: pointer to v4l2 subdev format structure
+ *
+ * Return -EINVAL or zero on success
+ */
+static int csid_get_format(struct v4l2_subdev *sd,
+			   struct v4l2_subdev_pad_config *cfg,
+			   struct v4l2_subdev_format *fmt)
+{
+	struct csid_device *csid = v4l2_get_subdevdata(sd);
+	struct v4l2_mbus_framefmt *format;
+
+	format = __csid_get_format(csid, cfg, fmt->pad, fmt->which);
+	if (format == NULL)
+		return -EINVAL;
+
+	fmt->format = *format;
+
+	return 0;
+}
+
+/*
+ * csid_set_format - Handle set format by pads subdev method
+ * @sd: CSID V4L2 subdevice
+ * @cfg: V4L2 subdev pad configuration
+ * @fmt: pointer to v4l2 subdev format structure
+ *
+ * Return -EINVAL or zero on success
+ */
+static int csid_set_format(struct v4l2_subdev *sd,
+			   struct v4l2_subdev_pad_config *cfg,
+			   struct v4l2_subdev_format *fmt)
+{
+	struct csid_device *csid = v4l2_get_subdevdata(sd);
+	struct v4l2_mbus_framefmt *format;
+
+	format = __csid_get_format(csid, cfg, fmt->pad, fmt->which);
+	if (format == NULL)
+		return -EINVAL;
+
+	csid_try_format(csid, cfg, fmt->pad, &fmt->format, fmt->which);
+	*format = fmt->format;
+
+	/* Propagate the format from sink to source */
+	if (fmt->pad == MSM_CSID_PAD_SINK) {
+		format = __csid_get_format(csid, cfg, MSM_CSID_PAD_SRC,
+					   fmt->which);
+
+		*format = fmt->format;
+		csid_try_format(csid, cfg, MSM_CSID_PAD_SRC, format,
+				fmt->which);
+	}
+
+	return 0;
+}
+
+/*
+ * csid_init_formats - Initialize formats on all pads
+ * @sd: CSID V4L2 subdevice
+ * @fh: V4L2 subdev file handle
+ *
+ * Initialize all pad formats with default values.
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int csid_init_formats(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
+{
+	struct v4l2_subdev_format format = {
+		.pad = MSM_CSID_PAD_SINK,
+		.which = fh ? V4L2_SUBDEV_FORMAT_TRY :
+			      V4L2_SUBDEV_FORMAT_ACTIVE,
+		.format = {
+			.code = MEDIA_BUS_FMT_UYVY8_2X8,
+			.width = 1920,
+			.height = 1080
+		}
+	};
+
+	return csid_set_format(sd, fh ? fh->pad : NULL, &format);
+}
+
+static const char * const csid_test_pattern_menu[] = {
+	"Disabled",
+	"Incrementing",
+	"Alternating 0x55/0xAA",
+	"All Zeros 0x00",
+	"All Ones 0xFF",
+	"Pseudo-random Data",
+};
+
+/*
+ * csid_set_test_pattern - Set test generator's pattern mode
+ * @csid: CSID device
+ * @value: desired test pattern mode
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int csid_set_test_pattern(struct csid_device *csid, s32 value)
+{
+	struct csid_testgen_config *tg = &csid->testgen;
+
+	/* If CSID is linked to CSIPHY, do not allow to enable test generator */
+	if (value && media_entity_remote_pad(&csid->pads[MSM_CSID_PAD_SINK]))
+		return -EBUSY;
+
+	tg->enabled = !!value;
+
+	switch (value) {
+	case 1:
+		tg->payload_mode = CSID_PAYLOAD_MODE_INCREMENTING;
+		break;
+	case 2:
+		tg->payload_mode = CSID_PAYLOAD_MODE_ALTERNATING_55_AA;
+		break;
+	case 3:
+		tg->payload_mode = CSID_PAYLOAD_MODE_ALL_ZEROES;
+		break;
+	case 4:
+		tg->payload_mode = CSID_PAYLOAD_MODE_ALL_ONES;
+		break;
+	case 5:
+		tg->payload_mode = CSID_PAYLOAD_MODE_RANDOM;
+		break;
+	}
+
+	return 0;
+}
+
+/*
+ * csid_s_ctrl - Handle set control subdev method
+ * @ctrl: pointer to v4l2 control structure
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int csid_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct csid_device *csid = container_of(ctrl->handler,
+						struct csid_device, ctrls);
+	int ret = -EINVAL;
+
+	switch (ctrl->id) {
+	case V4L2_CID_TEST_PATTERN:
+		ret = csid_set_test_pattern(csid, ctrl->val);
+		break;
+	}
+
+	return ret;
+}
+
+static const struct v4l2_ctrl_ops csid_ctrl_ops = {
+	.s_ctrl = csid_s_ctrl,
+};
+
+/*
+ * msm_csid_subdev_init - Initialize CSID device structure and resources
+ * @csid: CSID device
+ * @res: CSID module resources table
+ * @id: CSID module id
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+int msm_csid_subdev_init(struct csid_device *csid,
+			 const struct resources *res, u8 id)
+{
+	struct device *dev = to_device_index(csid, id);
+	struct platform_device *pdev = to_platform_device(dev);
+	struct resource *r;
+	int i, j;
+	int ret;
+
+	csid->id = id;
+
+	/* Memory */
+
+	r = platform_get_resource_byname(pdev, IORESOURCE_MEM, res->reg[0]);
+	csid->base = devm_ioremap_resource(dev, r);
+	if (IS_ERR(csid->base)) {
+		dev_err(dev, "could not map memory\n");
+		return PTR_ERR(csid->base);
+	}
+
+	/* Interrupt */
+
+	r = platform_get_resource_byname(pdev, IORESOURCE_IRQ,
+					 res->interrupt[0]);
+	if (!r) {
+		dev_err(dev, "missing IRQ\n");
+		return -EINVAL;
+	}
+
+	csid->irq = r->start;
+	snprintf(csid->irq_name, sizeof(csid->irq_name), "%s_%s%d",
+		 dev_name(dev), MSM_CSID_NAME, csid->id);
+	ret = devm_request_irq(dev, csid->irq, csid_isr,
+		IRQF_TRIGGER_RISING, csid->irq_name, csid);
+	if (ret < 0) {
+		dev_err(dev, "request_irq failed: %d\n", ret);
+		return ret;
+	}
+
+	disable_irq(csid->irq);
+
+	/* Clocks */
+
+	csid->nclocks = 0;
+	while (res->clock[csid->nclocks])
+		csid->nclocks++;
+
+	csid->clock = devm_kzalloc(dev, csid->nclocks * sizeof(*csid->clock),
+				    GFP_KERNEL);
+	if (!csid->clock)
+		return -ENOMEM;
+
+	for (i = 0; i < csid->nclocks; i++) {
+		struct camss_clock *clock = &csid->clock[i];
+
+		clock->clk = devm_clk_get(dev, res->clock[i]);
+		if (IS_ERR(clock->clk))
+			return PTR_ERR(clock->clk);
+
+		clock->name = res->clock[i];
+
+		clock->nfreqs = 0;
+		while (res->clock_rate[i][clock->nfreqs])
+			clock->nfreqs++;
+
+		if (!clock->nfreqs) {
+			clock->freq = NULL;
+			continue;
+		}
+
+		clock->freq = devm_kzalloc(dev, clock->nfreqs *
+					   sizeof(*clock->freq), GFP_KERNEL);
+		if (!clock->freq)
+			return -ENOMEM;
+
+		for (j = 0; j < clock->nfreqs; j++)
+			clock->freq[j] = res->clock_rate[i][j];
+	}
+
+	/* Regulator */
+
+	csid->vdda = devm_regulator_get(dev, res->regulator[0]);
+	if (IS_ERR(csid->vdda)) {
+		dev_err(dev, "could not get regulator\n");
+		return PTR_ERR(csid->vdda);
+	}
+
+	init_completion(&csid->reset_complete);
+
+	return 0;
+}
+
+/*
+ * msm_csid_get_csid_id - Get CSID HW module id
+ * @entity: Pointer to CSID media entity structure
+ * @id: Return CSID HW module id here
+ */
+void msm_csid_get_csid_id(struct media_entity *entity, u8 *id)
+{
+	struct v4l2_subdev *sd = media_entity_to_v4l2_subdev(entity);
+	struct csid_device *csid = v4l2_get_subdevdata(sd);
+
+	*id = csid->id;
+}
+
+/*
+ * csid_get_lane_assign - Calculate CSI2 lane assign configuration parameter
+ * @lane_cfg - CSI2 lane configuration
+ *
+ * Return lane assign
+ */
+static u32 csid_get_lane_assign(struct csiphy_lanes_cfg *lane_cfg)
+{
+	u32 lane_assign = 0;
+	int i;
+
+	for (i = 0; i < lane_cfg->num_data; i++)
+		lane_assign |= lane_cfg->data[i].pos << (i * 4);
+
+	return lane_assign;
+}
+
+/*
+ * csid_link_setup - Setup CSID connections
+ * @entity: Pointer to media entity structure
+ * @local: Pointer to local pad
+ * @remote: Pointer to remote pad
+ * @flags: Link flags
+ *
+ * Return 0 on success
+ */
+static int csid_link_setup(struct media_entity *entity,
+			   const struct media_pad *local,
+			   const struct media_pad *remote, u32 flags)
+{
+	if (flags & MEDIA_LNK_FL_ENABLED)
+		if (media_entity_remote_pad(local))
+			return -EBUSY;
+
+	if ((local->flags & MEDIA_PAD_FL_SINK) &&
+	    (flags & MEDIA_LNK_FL_ENABLED)) {
+		struct v4l2_subdev *sd;
+		struct csid_device *csid;
+		struct csiphy_device *csiphy;
+		struct csiphy_lanes_cfg *lane_cfg;
+		struct v4l2_subdev_format format = { 0 };
+
+		sd = media_entity_to_v4l2_subdev(entity);
+		csid = v4l2_get_subdevdata(sd);
+
+		/* If test generator is enabled */
+		/* do not allow a link from CSIPHY to CSID */
+		if (csid->testgen_mode->cur.val != 0)
+			return -EBUSY;
+
+		sd = media_entity_to_v4l2_subdev(remote->entity);
+		csiphy = v4l2_get_subdevdata(sd);
+
+		/* If a sensor is not linked to CSIPHY */
+		/* do no allow a link from CSIPHY to CSID */
+		if (!csiphy->cfg.csi2)
+			return -EPERM;
+
+		csid->phy.csiphy_id = csiphy->id;
+
+		lane_cfg = &csiphy->cfg.csi2->lane_cfg;
+		csid->phy.lane_cnt = lane_cfg->num_data;
+		csid->phy.lane_assign = csid_get_lane_assign(lane_cfg);
+
+		/* Reset format on source pad to sink pad format */
+		format.pad = MSM_CSID_PAD_SRC;
+		format.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+		csid_set_format(&csid->subdev, NULL, &format);
+	}
+
+	return 0;
+}
+
+static const struct v4l2_subdev_core_ops csid_core_ops = {
+	.s_power = csid_set_power,
+};
+
+static const struct v4l2_subdev_video_ops csid_video_ops = {
+	.s_stream = csid_set_stream,
+};
+
+static const struct v4l2_subdev_pad_ops csid_pad_ops = {
+	.enum_mbus_code = csid_enum_mbus_code,
+	.enum_frame_size = csid_enum_frame_size,
+	.get_fmt = csid_get_format,
+	.set_fmt = csid_set_format,
+};
+
+static const struct v4l2_subdev_ops csid_v4l2_ops = {
+	.core = &csid_core_ops,
+	.video = &csid_video_ops,
+	.pad = &csid_pad_ops,
+};
+
+static const struct v4l2_subdev_internal_ops csid_v4l2_internal_ops = {
+	.open = csid_init_formats,
+};
+
+static const struct media_entity_operations csid_media_ops = {
+	.link_setup = csid_link_setup,
+	.link_validate = v4l2_subdev_link_validate,
+};
+
+/*
+ * msm_csid_register_entity - Register subdev node for CSID module
+ * @csid: CSID device
+ * @v4l2_dev: V4L2 device
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+int msm_csid_register_entity(struct csid_device *csid,
+			     struct v4l2_device *v4l2_dev)
+{
+	struct v4l2_subdev *sd = &csid->subdev;
+	struct media_pad *pads = csid->pads;
+	struct device *dev = to_device_index(csid, csid->id);
+	int ret;
+
+	v4l2_subdev_init(sd, &csid_v4l2_ops);
+	sd->internal_ops = &csid_v4l2_internal_ops;
+	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+	snprintf(sd->name, ARRAY_SIZE(sd->name), "%s%d",
+		 MSM_CSID_NAME, csid->id);
+	v4l2_set_subdevdata(sd, csid);
+
+	ret = v4l2_ctrl_handler_init(&csid->ctrls, 1);
+	if (ret < 0) {
+		dev_err(dev, "Failed to init ctrl handler: %d\n", ret);
+		return ret;
+	}
+
+	csid->testgen_mode = v4l2_ctrl_new_std_menu_items(&csid->ctrls,
+				&csid_ctrl_ops, V4L2_CID_TEST_PATTERN,
+				ARRAY_SIZE(csid_test_pattern_menu) - 1, 0, 0,
+				csid_test_pattern_menu);
+
+	if (csid->ctrls.error) {
+		dev_err(dev, "Failed to init ctrl: %d\n", csid->ctrls.error);
+		ret = csid->ctrls.error;
+		goto free_ctrl;
+	}
+
+	csid->subdev.ctrl_handler = &csid->ctrls;
+
+	ret = csid_init_formats(sd, NULL);
+	if (ret < 0) {
+		dev_err(dev, "Failed to init format: %d\n", ret);
+		goto free_ctrl;
+	}
+
+	pads[MSM_CSID_PAD_SINK].flags = MEDIA_PAD_FL_SINK;
+	pads[MSM_CSID_PAD_SRC].flags = MEDIA_PAD_FL_SOURCE;
+
+	sd->entity.function = MEDIA_ENT_F_IO_V4L;
+	sd->entity.ops = &csid_media_ops;
+	ret = media_entity_pads_init(&sd->entity, MSM_CSID_PADS_NUM, pads);
+	if (ret < 0) {
+		dev_err(dev, "Failed to init media entity: %d\n", ret);
+		goto free_ctrl;
+	}
+
+	ret = v4l2_device_register_subdev(v4l2_dev, sd);
+	if (ret < 0) {
+		dev_err(dev, "Failed to register subdev: %d\n", ret);
+		goto media_cleanup;
+	}
+
+	return 0;
+
+media_cleanup:
+	media_entity_cleanup(&sd->entity);
+free_ctrl:
+	v4l2_ctrl_handler_free(&csid->ctrls);
+
+	return ret;
+}
+
+/*
+ * msm_csid_unregister_entity - Unregister CSID module subdev node
+ * @csid: CSID device
+ */
+void msm_csid_unregister_entity(struct csid_device *csid)
+{
+	v4l2_device_unregister_subdev(&csid->subdev);
+	media_entity_cleanup(&csid->subdev.entity);
+	v4l2_ctrl_handler_free(&csid->ctrls);
+}
diff --git a/drivers/media/platform/qcom/camss-8x16/camss-csid.h b/drivers/media/platform/qcom/camss-8x16/camss-csid.h
new file mode 100644
index 000000000000..8682d3081bc3
--- /dev/null
+++ b/drivers/media/platform/qcom/camss-8x16/camss-csid.h
@@ -0,0 +1,82 @@
+/*
+ * camss-csid.h
+ *
+ * Qualcomm MSM Camera Subsystem - CSID (CSI Decoder) Module
+ *
+ * Copyright (c) 2011-2014, The Linux Foundation. All rights reserved.
+ * Copyright (C) 2015-2017 Linaro Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#ifndef QC_MSM_CAMSS_CSID_H
+#define QC_MSM_CAMSS_CSID_H
+
+#include <linux/clk.h>
+#include <media/media-entity.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-mediabus.h>
+#include <media/v4l2-subdev.h>
+
+#define MSM_CSID_PAD_SINK 0
+#define MSM_CSID_PAD_SRC 1
+#define MSM_CSID_PADS_NUM 2
+
+enum csid_payload_mode {
+	CSID_PAYLOAD_MODE_INCREMENTING = 0,
+	CSID_PAYLOAD_MODE_ALTERNATING_55_AA = 1,
+	CSID_PAYLOAD_MODE_ALL_ZEROES = 2,
+	CSID_PAYLOAD_MODE_ALL_ONES = 3,
+	CSID_PAYLOAD_MODE_RANDOM = 4,
+	CSID_PAYLOAD_MODE_USER_SPECIFIED = 5,
+};
+
+struct csid_testgen_config {
+	u8 enabled;
+	enum csid_payload_mode payload_mode;
+};
+
+struct csid_phy_config {
+	u8 csiphy_id;
+	u8 lane_cnt;
+	u32 lane_assign;
+};
+
+struct csid_device {
+	u8 id;
+	struct v4l2_subdev subdev;
+	struct media_pad pads[MSM_CSID_PADS_NUM];
+	void __iomem *base;
+	u32 irq;
+	char irq_name[30];
+	struct camss_clock *clock;
+	int nclocks;
+	struct regulator *vdda;
+	struct completion reset_complete;
+	struct csid_testgen_config testgen;
+	struct csid_phy_config phy;
+	struct v4l2_mbus_framefmt fmt[MSM_CSID_PADS_NUM];
+	struct v4l2_ctrl_handler ctrls;
+	struct v4l2_ctrl *testgen_mode;
+};
+
+struct resources;
+
+int msm_csid_subdev_init(struct csid_device *csid,
+			 const struct resources *res, u8 id);
+
+int msm_csid_register_entity(struct csid_device *csid,
+			     struct v4l2_device *v4l2_dev);
+
+void msm_csid_unregister_entity(struct csid_device *csid);
+
+void msm_csid_get_csid_id(struct media_entity *entity, u8 *id);
+
+#endif /* QC_MSM_CAMSS_CSID_H */
diff --git a/drivers/media/platform/qcom/camss-8x16/camss-csiphy.c b/drivers/media/platform/qcom/camss-8x16/camss-csiphy.c
new file mode 100644
index 000000000000..072c6cf053f6
--- /dev/null
+++ b/drivers/media/platform/qcom/camss-8x16/camss-csiphy.c
@@ -0,0 +1,890 @@
+/*
+ * camss-csiphy.c
+ *
+ * Qualcomm MSM Camera Subsystem - CSIPHY Module
+ *
+ * Copyright (c) 2011-2015, The Linux Foundation. All rights reserved.
+ * Copyright (C) 2016-2017 Linaro Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <media/media-entity.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-subdev.h>
+
+#include "camss-csiphy.h"
+#include "camss.h"
+
+#define MSM_CSIPHY_NAME "msm_csiphy"
+
+#define CAMSS_CSI_PHY_LNn_CFG2(n)		(0x004 + 0x40 * (n))
+#define CAMSS_CSI_PHY_LNn_CFG3(n)		(0x008 + 0x40 * (n))
+#define CAMSS_CSI_PHY_GLBL_RESET		0x140
+#define CAMSS_CSI_PHY_GLBL_PWR_CFG		0x144
+#define CAMSS_CSI_PHY_GLBL_IRQ_CMD		0x164
+#define CAMSS_CSI_PHY_HW_VERSION		0x188
+#define CAMSS_CSI_PHY_INTERRUPT_STATUSn(n)	(0x18c + 0x4 * (n))
+#define CAMSS_CSI_PHY_INTERRUPT_MASKn(n)	(0x1ac + 0x4 * (n))
+#define CAMSS_CSI_PHY_INTERRUPT_CLEARn(n)	(0x1cc + 0x4 * (n))
+#define CAMSS_CSI_PHY_GLBL_T_INIT_CFG0		0x1ec
+#define CAMSS_CSI_PHY_T_WAKEUP_CFG0		0x1f4
+
+static const struct {
+	u32 code;
+	u8 bpp;
+} csiphy_formats[] = {
+	{
+		MEDIA_BUS_FMT_UYVY8_2X8,
+		8,
+	},
+	{
+		MEDIA_BUS_FMT_VYUY8_2X8,
+		8,
+	},
+	{
+		MEDIA_BUS_FMT_YUYV8_2X8,
+		8,
+	},
+	{
+		MEDIA_BUS_FMT_YVYU8_2X8,
+		8,
+	},
+	{
+		MEDIA_BUS_FMT_SBGGR8_1X8,
+		8,
+	},
+	{
+		MEDIA_BUS_FMT_SGBRG8_1X8,
+		8,
+	},
+	{
+		MEDIA_BUS_FMT_SGRBG8_1X8,
+		8,
+	},
+	{
+		MEDIA_BUS_FMT_SRGGB8_1X8,
+		8,
+	},
+	{
+		MEDIA_BUS_FMT_SBGGR10_1X10,
+		10,
+	},
+	{
+		MEDIA_BUS_FMT_SGBRG10_1X10,
+		10,
+	},
+	{
+		MEDIA_BUS_FMT_SGRBG10_1X10,
+		10,
+	},
+	{
+		MEDIA_BUS_FMT_SRGGB10_1X10,
+		10,
+	},
+	{
+		MEDIA_BUS_FMT_SBGGR12_1X12,
+		12,
+	},
+	{
+		MEDIA_BUS_FMT_SGBRG12_1X12,
+		12,
+	},
+	{
+		MEDIA_BUS_FMT_SGRBG12_1X12,
+		12,
+	},
+	{
+		MEDIA_BUS_FMT_SRGGB12_1X12,
+		12,
+	}
+};
+
+/*
+ * csiphy_get_bpp - map media bus format to bits per pixel
+ * @code: media bus format code
+ *
+ * Return number of bits per pixel
+ */
+static u8 csiphy_get_bpp(u32 code)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(csiphy_formats); i++)
+		if (code == csiphy_formats[i].code)
+			return csiphy_formats[i].bpp;
+
+	WARN(1, "Unknown format\n");
+
+	return csiphy_formats[0].bpp;
+}
+
+/*
+ * csiphy_isr - CSIPHY module interrupt handler
+ * @irq: Interrupt line
+ * @dev: CSIPHY device
+ *
+ * Return IRQ_HANDLED on success
+ */
+static irqreturn_t csiphy_isr(int irq, void *dev)
+{
+	struct csiphy_device *csiphy = dev;
+	u8 i;
+
+	for (i = 0; i < 8; i++) {
+		u8 val = readl_relaxed(csiphy->base +
+				       CAMSS_CSI_PHY_INTERRUPT_STATUSn(i));
+		writel_relaxed(val, csiphy->base +
+			       CAMSS_CSI_PHY_INTERRUPT_CLEARn(i));
+		writel_relaxed(0x1, csiphy->base + CAMSS_CSI_PHY_GLBL_IRQ_CMD);
+		writel_relaxed(0x0, csiphy->base + CAMSS_CSI_PHY_GLBL_IRQ_CMD);
+		writel_relaxed(0x0, csiphy->base +
+			       CAMSS_CSI_PHY_INTERRUPT_CLEARn(i));
+	}
+
+	return IRQ_HANDLED;
+}
+
+/*
+ * csiphy_set_clock_rates - Calculate and set clock rates on CSIPHY module
+ * @csiphy: CSIPHY device
+ */
+static int csiphy_set_clock_rates(struct csiphy_device *csiphy)
+{
+	struct device *dev = to_device_index(csiphy, csiphy->id);
+	u32 pixel_clock;
+	int i, j;
+	int ret;
+
+	ret = camss_get_pixel_clock(&csiphy->subdev.entity, &pixel_clock);
+	if (ret)
+		pixel_clock = 0;
+
+	for (i = 0; i < csiphy->nclocks; i++) {
+		struct camss_clock *clock = &csiphy->clock[i];
+
+		if (!strcmp(clock->name, "csiphy0_timer") ||
+			!strcmp(clock->name, "csiphy1_timer")) {
+			u8 bpp = csiphy_get_bpp(
+					csiphy->fmt[MSM_CSIPHY_PAD_SINK].code);
+			u8 num_lanes = csiphy->cfg.csi2->lane_cfg.num_data;
+			u64 min_rate = pixel_clock * bpp / (2 * num_lanes * 4);
+			long round_rate;
+
+			camss_add_clock_margin(&min_rate);
+
+			for (j = 0; j < clock->nfreqs; j++)
+				if (min_rate < clock->freq[j])
+					break;
+
+			if (j == clock->nfreqs) {
+				dev_err(dev,
+					"Pixel clock is too high for CSIPHY\n");
+				return -EINVAL;
+			}
+
+			/* if sensor pixel clock is not available */
+			/* set highest possible CSIPHY clock rate */
+			if (min_rate == 0)
+				j = clock->nfreqs - 1;
+
+			round_rate = clk_round_rate(clock->clk, clock->freq[j]);
+			if (round_rate < 0) {
+				dev_err(dev, "clk round rate failed: %ld\n",
+					round_rate);
+				return -EINVAL;
+			}
+
+			csiphy->timer_clk_rate = round_rate;
+
+			ret = clk_set_rate(clock->clk, csiphy->timer_clk_rate);
+			if (ret < 0) {
+				dev_err(dev, "clk set rate failed: %d\n", ret);
+				return ret;
+			}
+		}
+	}
+
+	return 0;
+}
+
+/*
+ * csiphy_reset - Perform software reset on CSIPHY module
+ * @csiphy: CSIPHY device
+ */
+static void csiphy_reset(struct csiphy_device *csiphy)
+{
+	writel_relaxed(0x1, csiphy->base + CAMSS_CSI_PHY_GLBL_RESET);
+	usleep_range(5000, 8000);
+	writel_relaxed(0x0, csiphy->base + CAMSS_CSI_PHY_GLBL_RESET);
+}
+
+/*
+ * csiphy_set_power - Power on/off CSIPHY module
+ * @sd: CSIPHY V4L2 subdevice
+ * @on: Requested power state
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int csiphy_set_power(struct v4l2_subdev *sd, int on)
+{
+	struct csiphy_device *csiphy = v4l2_get_subdevdata(sd);
+	struct device *dev = to_device_index(csiphy, csiphy->id);
+
+	if (on) {
+		u8 hw_version;
+		int ret;
+
+		ret = csiphy_set_clock_rates(csiphy);
+		if (ret < 0)
+			return ret;
+
+		ret = camss_enable_clocks(csiphy->nclocks, csiphy->clock, dev);
+		if (ret < 0)
+			return ret;
+
+		enable_irq(csiphy->irq);
+
+		csiphy_reset(csiphy);
+
+		hw_version = readl_relaxed(csiphy->base +
+					   CAMSS_CSI_PHY_HW_VERSION);
+		dev_dbg(dev, "CSIPHY HW Version = 0x%02x\n", hw_version);
+	} else {
+		disable_irq(csiphy->irq);
+
+		camss_disable_clocks(csiphy->nclocks, csiphy->clock);
+	}
+
+	return 0;
+}
+
+/*
+ * csiphy_get_lane_mask - Calculate CSI2 lane mask configuration parameter
+ * @lane_cfg - CSI2 lane configuration
+ *
+ * Return lane mask
+ */
+static u8 csiphy_get_lane_mask(struct csiphy_lanes_cfg *lane_cfg)
+{
+	u8 lane_mask;
+	int i;
+
+	lane_mask = 1 << lane_cfg->clk.pos;
+
+	for (i = 0; i < lane_cfg->num_data; i++)
+		lane_mask |= 1 << lane_cfg->data[i].pos;
+
+	return lane_mask;
+}
+
+/*
+ * csiphy_settle_cnt_calc - Calculate settle count value
+ * @csiphy: CSIPHY device
+ *
+ * Helper function to calculate settle count value. This is
+ * based on the CSI2 T_hs_settle parameter which in turn
+ * is calculated based on the CSI2 transmitter pixel clock
+ * frequency.
+ *
+ * Return settle count value or 0 if the CSI2 pixel clock
+ * frequency is not available
+ */
+static u8 csiphy_settle_cnt_calc(struct csiphy_device *csiphy)
+{
+	u8 bpp = csiphy_get_bpp(
+			csiphy->fmt[MSM_CSIPHY_PAD_SINK].code);
+	u8 num_lanes = csiphy->cfg.csi2->lane_cfg.num_data;
+	u32 pixel_clock; /* Hz */
+	u32 mipi_clock; /* Hz */
+	u32 ui; /* ps */
+	u32 timer_period; /* ps */
+	u32 t_hs_prepare_max; /* ps */
+	u32 t_hs_prepare_zero_min; /* ps */
+	u32 t_hs_settle; /* ps */
+	u8 settle_cnt;
+	int ret;
+
+	ret = camss_get_pixel_clock(&csiphy->subdev.entity, &pixel_clock);
+	if (ret) {
+		dev_err(to_device_index(csiphy, csiphy->id),
+			"Cannot get CSI2 transmitter's pixel clock\n");
+		return 0;
+	}
+	if (!pixel_clock) {
+		dev_err(to_device_index(csiphy, csiphy->id),
+			"Got pixel clock == 0, cannot continue\n");
+		return 0;
+	}
+
+	mipi_clock = pixel_clock * bpp / (2 * num_lanes);
+	ui = div_u64(1000000000000LL, mipi_clock);
+	ui /= 2;
+	t_hs_prepare_max = 85000 + 6 * ui;
+	t_hs_prepare_zero_min = 145000 + 10 * ui;
+	t_hs_settle = (t_hs_prepare_max + t_hs_prepare_zero_min) / 2;
+
+	timer_period = div_u64(1000000000000LL, csiphy->timer_clk_rate);
+	settle_cnt = t_hs_settle / timer_period;
+
+	return settle_cnt;
+}
+
+/*
+ * csiphy_stream_on - Enable streaming on CSIPHY module
+ * @csiphy: CSIPHY device
+ *
+ * Helper function to enable streaming on CSIPHY module.
+ * Main configuration of CSIPHY module is also done here.
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int csiphy_stream_on(struct csiphy_device *csiphy)
+{
+	struct csiphy_config *cfg = &csiphy->cfg;
+	u8 lane_mask = csiphy_get_lane_mask(&cfg->csi2->lane_cfg);
+	u8 settle_cnt;
+	u8 val;
+	int i = 0;
+
+	settle_cnt = csiphy_settle_cnt_calc(csiphy);
+	if (!settle_cnt)
+		return -EINVAL;
+
+	val = readl_relaxed(csiphy->base_clk_mux);
+	if (cfg->combo_mode && (lane_mask & 0x18) == 0x18) {
+		val &= ~0xf0;
+		val |= cfg->csid_id << 4;
+	} else {
+		val &= ~0xf;
+		val |= cfg->csid_id;
+	}
+	writel_relaxed(val, csiphy->base_clk_mux);
+
+	writel_relaxed(0x1, csiphy->base +
+		       CAMSS_CSI_PHY_GLBL_T_INIT_CFG0);
+	writel_relaxed(0x1, csiphy->base +
+		       CAMSS_CSI_PHY_T_WAKEUP_CFG0);
+
+	val = 0x1;
+	val |= lane_mask << 1;
+	writel_relaxed(val, csiphy->base + CAMSS_CSI_PHY_GLBL_PWR_CFG);
+
+	val = cfg->combo_mode << 4;
+	writel_relaxed(val, csiphy->base + CAMSS_CSI_PHY_GLBL_RESET);
+
+	while (lane_mask) {
+		if (lane_mask & 0x1) {
+			writel_relaxed(0x10, csiphy->base +
+				       CAMSS_CSI_PHY_LNn_CFG2(i));
+			writel_relaxed(settle_cnt, csiphy->base +
+				       CAMSS_CSI_PHY_LNn_CFG3(i));
+			writel_relaxed(0x3f, csiphy->base +
+				       CAMSS_CSI_PHY_INTERRUPT_MASKn(i));
+			writel_relaxed(0x3f, csiphy->base +
+				       CAMSS_CSI_PHY_INTERRUPT_CLEARn(i));
+		}
+
+		lane_mask >>= 1;
+		i++;
+	}
+
+	return 0;
+}
+
+/*
+ * csiphy_stream_off - Disable streaming on CSIPHY module
+ * @csiphy: CSIPHY device
+ *
+ * Helper function to disable streaming on CSIPHY module
+ */
+static void csiphy_stream_off(struct csiphy_device *csiphy)
+{
+	u8 lane_mask = csiphy_get_lane_mask(&csiphy->cfg.csi2->lane_cfg);
+	int i = 0;
+
+	while (lane_mask) {
+		if (lane_mask & 0x1)
+			writel_relaxed(0x0, csiphy->base +
+				       CAMSS_CSI_PHY_LNn_CFG2(i));
+
+		lane_mask >>= 1;
+		i++;
+	}
+
+	writel_relaxed(0x0, csiphy->base + CAMSS_CSI_PHY_GLBL_PWR_CFG);
+}
+
+
+/*
+ * csiphy_set_stream - Enable/disable streaming on CSIPHY module
+ * @sd: CSIPHY V4L2 subdevice
+ * @enable: Requested streaming state
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int csiphy_set_stream(struct v4l2_subdev *sd, int enable)
+{
+	struct csiphy_device *csiphy = v4l2_get_subdevdata(sd);
+	int ret = 0;
+
+	if (enable)
+		ret = csiphy_stream_on(csiphy);
+	else
+		csiphy_stream_off(csiphy);
+
+	return ret;
+}
+
+/*
+ * __csiphy_get_format - Get pointer to format structure
+ * @csiphy: CSIPHY device
+ * @cfg: V4L2 subdev pad configuration
+ * @pad: pad from which format is requested
+ * @which: TRY or ACTIVE format
+ *
+ * Return pointer to TRY or ACTIVE format structure
+ */
+static struct v4l2_mbus_framefmt *
+__csiphy_get_format(struct csiphy_device *csiphy,
+		    struct v4l2_subdev_pad_config *cfg,
+		    unsigned int pad,
+		    enum v4l2_subdev_format_whence which)
+{
+	if (which == V4L2_SUBDEV_FORMAT_TRY)
+		return v4l2_subdev_get_try_format(&csiphy->subdev, cfg, pad);
+
+	return &csiphy->fmt[pad];
+}
+
+/*
+ * csiphy_try_format - Handle try format by pad subdev method
+ * @csiphy: CSIPHY device
+ * @cfg: V4L2 subdev pad configuration
+ * @pad: pad on which format is requested
+ * @fmt: pointer to v4l2 format structure
+ * @which: wanted subdev format
+ */
+static void csiphy_try_format(struct csiphy_device *csiphy,
+			      struct v4l2_subdev_pad_config *cfg,
+			      unsigned int pad,
+			      struct v4l2_mbus_framefmt *fmt,
+			      enum v4l2_subdev_format_whence which)
+{
+	unsigned int i;
+
+	switch (pad) {
+	case MSM_CSIPHY_PAD_SINK:
+		/* Set format on sink pad */
+
+		for (i = 0; i < ARRAY_SIZE(csiphy_formats); i++)
+			if (fmt->code == csiphy_formats[i].code)
+				break;
+
+		/* If not found, use UYVY as default */
+		if (i >= ARRAY_SIZE(csiphy_formats))
+			fmt->code = MEDIA_BUS_FMT_UYVY8_2X8;
+
+		fmt->width = clamp_t(u32, fmt->width, 1, 8191);
+		fmt->height = clamp_t(u32, fmt->height, 1, 8191);
+
+		fmt->field = V4L2_FIELD_NONE;
+		fmt->colorspace = V4L2_COLORSPACE_SRGB;
+
+		break;
+
+	case MSM_CSIPHY_PAD_SRC:
+		/* Set and return a format same as sink pad */
+
+		*fmt = *__csiphy_get_format(csiphy, cfg, MSM_CSID_PAD_SINK,
+					    which);
+
+		break;
+	}
+}
+
+/*
+ * csiphy_enum_mbus_code - Handle pixel format enumeration
+ * @sd: CSIPHY V4L2 subdevice
+ * @cfg: V4L2 subdev pad configuration
+ * @code: pointer to v4l2_subdev_mbus_code_enum structure
+ * return -EINVAL or zero on success
+ */
+static int csiphy_enum_mbus_code(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_pad_config *cfg,
+				 struct v4l2_subdev_mbus_code_enum *code)
+{
+	struct csiphy_device *csiphy = v4l2_get_subdevdata(sd);
+	struct v4l2_mbus_framefmt *format;
+
+	if (code->pad == MSM_CSIPHY_PAD_SINK) {
+		if (code->index >= ARRAY_SIZE(csiphy_formats))
+			return -EINVAL;
+
+		code->code = csiphy_formats[code->index].code;
+	} else {
+		if (code->index > 0)
+			return -EINVAL;
+
+		format = __csiphy_get_format(csiphy, cfg, MSM_CSIPHY_PAD_SINK,
+					     code->which);
+
+		code->code = format->code;
+	}
+
+	return 0;
+}
+
+/*
+ * csiphy_enum_frame_size - Handle frame size enumeration
+ * @sd: CSIPHY V4L2 subdevice
+ * @cfg: V4L2 subdev pad configuration
+ * @fse: pointer to v4l2_subdev_frame_size_enum structure
+ * return -EINVAL or zero on success
+ */
+static int csiphy_enum_frame_size(struct v4l2_subdev *sd,
+				  struct v4l2_subdev_pad_config *cfg,
+				  struct v4l2_subdev_frame_size_enum *fse)
+{
+	struct csiphy_device *csiphy = v4l2_get_subdevdata(sd);
+	struct v4l2_mbus_framefmt format;
+
+	if (fse->index != 0)
+		return -EINVAL;
+
+	format.code = fse->code;
+	format.width = 1;
+	format.height = 1;
+	csiphy_try_format(csiphy, cfg, fse->pad, &format, fse->which);
+	fse->min_width = format.width;
+	fse->min_height = format.height;
+
+	if (format.code != fse->code)
+		return -EINVAL;
+
+	format.code = fse->code;
+	format.width = -1;
+	format.height = -1;
+	csiphy_try_format(csiphy, cfg, fse->pad, &format, fse->which);
+	fse->max_width = format.width;
+	fse->max_height = format.height;
+
+	return 0;
+}
+
+/*
+ * csiphy_get_format - Handle get format by pads subdev method
+ * @sd: CSIPHY V4L2 subdevice
+ * @cfg: V4L2 subdev pad configuration
+ * @fmt: pointer to v4l2 subdev format structure
+ *
+ * Return -EINVAL or zero on success
+ */
+static int csiphy_get_format(struct v4l2_subdev *sd,
+			     struct v4l2_subdev_pad_config *cfg,
+			     struct v4l2_subdev_format *fmt)
+{
+	struct csiphy_device *csiphy = v4l2_get_subdevdata(sd);
+	struct v4l2_mbus_framefmt *format;
+
+	format = __csiphy_get_format(csiphy, cfg, fmt->pad, fmt->which);
+	if (format == NULL)
+		return -EINVAL;
+
+	fmt->format = *format;
+
+	return 0;
+}
+
+/*
+ * csiphy_set_format - Handle set format by pads subdev method
+ * @sd: CSIPHY V4L2 subdevice
+ * @cfg: V4L2 subdev pad configuration
+ * @fmt: pointer to v4l2 subdev format structure
+ *
+ * Return -EINVAL or zero on success
+ */
+static int csiphy_set_format(struct v4l2_subdev *sd,
+			     struct v4l2_subdev_pad_config *cfg,
+			     struct v4l2_subdev_format *fmt)
+{
+	struct csiphy_device *csiphy = v4l2_get_subdevdata(sd);
+	struct v4l2_mbus_framefmt *format;
+
+	format = __csiphy_get_format(csiphy, cfg, fmt->pad, fmt->which);
+	if (format == NULL)
+		return -EINVAL;
+
+	csiphy_try_format(csiphy, cfg, fmt->pad, &fmt->format, fmt->which);
+	*format = fmt->format;
+
+	/* Propagate the format from sink to source */
+	if (fmt->pad == MSM_CSIPHY_PAD_SINK) {
+		format = __csiphy_get_format(csiphy, cfg, MSM_CSIPHY_PAD_SRC,
+					     fmt->which);
+
+		*format = fmt->format;
+		csiphy_try_format(csiphy, cfg, MSM_CSIPHY_PAD_SRC, format,
+				  fmt->which);
+	}
+
+	return 0;
+}
+
+/*
+ * csiphy_init_formats - Initialize formats on all pads
+ * @sd: CSIPHY V4L2 subdevice
+ * @fh: V4L2 subdev file handle
+ *
+ * Initialize all pad formats with default values.
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int csiphy_init_formats(struct v4l2_subdev *sd,
+			       struct v4l2_subdev_fh *fh)
+{
+	struct v4l2_subdev_format format = {
+		.pad = MSM_CSIPHY_PAD_SINK,
+		.which = fh ? V4L2_SUBDEV_FORMAT_TRY :
+			      V4L2_SUBDEV_FORMAT_ACTIVE,
+		.format = {
+			.code = MEDIA_BUS_FMT_UYVY8_2X8,
+			.width = 1920,
+			.height = 1080
+		}
+	};
+
+	return csiphy_set_format(sd, fh ? fh->pad : NULL, &format);
+}
+
+/*
+ * msm_csiphy_subdev_init - Initialize CSIPHY device structure and resources
+ * @csiphy: CSIPHY device
+ * @res: CSIPHY module resources table
+ * @id: CSIPHY module id
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+int msm_csiphy_subdev_init(struct csiphy_device *csiphy,
+			   const struct resources *res, u8 id)
+{
+	struct device *dev = to_device_index(csiphy, id);
+	struct platform_device *pdev = to_platform_device(dev);
+	struct resource *r;
+	int i, j;
+	int ret;
+
+	csiphy->id = id;
+	csiphy->cfg.combo_mode = 0;
+
+	/* Memory */
+
+	r = platform_get_resource_byname(pdev, IORESOURCE_MEM, res->reg[0]);
+	csiphy->base = devm_ioremap_resource(dev, r);
+	if (IS_ERR(csiphy->base)) {
+		dev_err(dev, "could not map memory\n");
+		return PTR_ERR(csiphy->base);
+	}
+
+	r = platform_get_resource_byname(pdev, IORESOURCE_MEM, res->reg[1]);
+	csiphy->base_clk_mux = devm_ioremap_resource(dev, r);
+	if (IS_ERR(csiphy->base_clk_mux)) {
+		dev_err(dev, "could not map memory\n");
+		return PTR_ERR(csiphy->base_clk_mux);
+	}
+
+	/* Interrupt */
+
+	r = platform_get_resource_byname(pdev, IORESOURCE_IRQ,
+					 res->interrupt[0]);
+	if (!r) {
+		dev_err(dev, "missing IRQ\n");
+		return -EINVAL;
+	}
+
+	csiphy->irq = r->start;
+	snprintf(csiphy->irq_name, sizeof(csiphy->irq_name), "%s_%s%d",
+		 dev_name(dev), MSM_CSIPHY_NAME, csiphy->id);
+	ret = devm_request_irq(dev, csiphy->irq, csiphy_isr,
+			       IRQF_TRIGGER_RISING, csiphy->irq_name, csiphy);
+	if (ret < 0) {
+		dev_err(dev, "request_irq failed: %d\n", ret);
+		return ret;
+	}
+
+	disable_irq(csiphy->irq);
+
+	/* Clocks */
+
+	csiphy->nclocks = 0;
+	while (res->clock[csiphy->nclocks])
+		csiphy->nclocks++;
+
+	csiphy->clock = devm_kzalloc(dev, csiphy->nclocks *
+				     sizeof(*csiphy->clock), GFP_KERNEL);
+	if (!csiphy->clock)
+		return -ENOMEM;
+
+	for (i = 0; i < csiphy->nclocks; i++) {
+		struct camss_clock *clock = &csiphy->clock[i];
+
+		clock->clk = devm_clk_get(dev, res->clock[i]);
+		if (IS_ERR(clock->clk))
+			return PTR_ERR(clock->clk);
+
+		clock->name = res->clock[i];
+
+		clock->nfreqs = 0;
+		while (res->clock_rate[i][clock->nfreqs])
+			clock->nfreqs++;
+
+		if (!clock->nfreqs) {
+			clock->freq = NULL;
+			continue;
+		}
+
+		clock->freq = devm_kzalloc(dev, clock->nfreqs *
+					   sizeof(*clock->freq), GFP_KERNEL);
+		if (!clock->freq)
+			return -ENOMEM;
+
+		for (j = 0; j < clock->nfreqs; j++)
+			clock->freq[j] = res->clock_rate[i][j];
+	}
+
+	return 0;
+}
+
+/*
+ * csiphy_link_setup - Setup CSIPHY connections
+ * @entity: Pointer to media entity structure
+ * @local: Pointer to local pad
+ * @remote: Pointer to remote pad
+ * @flags: Link flags
+ *
+ * Rreturn 0 on success
+ */
+static int csiphy_link_setup(struct media_entity *entity,
+			     const struct media_pad *local,
+			     const struct media_pad *remote, u32 flags)
+{
+	if ((local->flags & MEDIA_PAD_FL_SOURCE) &&
+	    (flags & MEDIA_LNK_FL_ENABLED)) {
+		struct v4l2_subdev *sd;
+		struct csiphy_device *csiphy;
+		struct csid_device *csid;
+
+		if (media_entity_remote_pad(local))
+			return -EBUSY;
+
+		sd = media_entity_to_v4l2_subdev(entity);
+		csiphy = v4l2_get_subdevdata(sd);
+
+		sd = media_entity_to_v4l2_subdev(remote->entity);
+		csid = v4l2_get_subdevdata(sd);
+
+		csiphy->cfg.csid_id = csid->id;
+	}
+
+	return 0;
+}
+
+static const struct v4l2_subdev_core_ops csiphy_core_ops = {
+	.s_power = csiphy_set_power,
+};
+
+static const struct v4l2_subdev_video_ops csiphy_video_ops = {
+	.s_stream = csiphy_set_stream,
+};
+
+static const struct v4l2_subdev_pad_ops csiphy_pad_ops = {
+	.enum_mbus_code = csiphy_enum_mbus_code,
+	.enum_frame_size = csiphy_enum_frame_size,
+	.get_fmt = csiphy_get_format,
+	.set_fmt = csiphy_set_format,
+};
+
+static const struct v4l2_subdev_ops csiphy_v4l2_ops = {
+	.core = &csiphy_core_ops,
+	.video = &csiphy_video_ops,
+	.pad = &csiphy_pad_ops,
+};
+
+static const struct v4l2_subdev_internal_ops csiphy_v4l2_internal_ops = {
+	.open = csiphy_init_formats,
+};
+
+static const struct media_entity_operations csiphy_media_ops = {
+	.link_setup = csiphy_link_setup,
+	.link_validate = v4l2_subdev_link_validate,
+};
+
+/*
+ * msm_csiphy_register_entity - Register subdev node for CSIPHY module
+ * @csiphy: CSIPHY device
+ * @v4l2_dev: V4L2 device
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+int msm_csiphy_register_entity(struct csiphy_device *csiphy,
+			       struct v4l2_device *v4l2_dev)
+{
+	struct v4l2_subdev *sd = &csiphy->subdev;
+	struct media_pad *pads = csiphy->pads;
+	struct device *dev = to_device_index(csiphy, csiphy->id);
+	int ret;
+
+	v4l2_subdev_init(sd, &csiphy_v4l2_ops);
+	sd->internal_ops = &csiphy_v4l2_internal_ops;
+	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+	snprintf(sd->name, ARRAY_SIZE(sd->name), "%s%d",
+		 MSM_CSIPHY_NAME, csiphy->id);
+	v4l2_set_subdevdata(sd, csiphy);
+
+	ret = csiphy_init_formats(sd, NULL);
+	if (ret < 0) {
+		dev_err(dev, "Failed to init format: %d\n", ret);
+		return ret;
+	}
+
+	pads[MSM_CSIPHY_PAD_SINK].flags = MEDIA_PAD_FL_SINK;
+	pads[MSM_CSIPHY_PAD_SRC].flags = MEDIA_PAD_FL_SOURCE;
+
+	sd->entity.function = MEDIA_ENT_F_IO_V4L;
+	sd->entity.ops = &csiphy_media_ops;
+	ret = media_entity_pads_init(&sd->entity, MSM_CSIPHY_PADS_NUM, pads);
+	if (ret < 0) {
+		dev_err(dev, "Failed to init media entity: %d\n", ret);
+		return ret;
+	}
+
+	ret = v4l2_device_register_subdev(v4l2_dev, sd);
+	if (ret < 0) {
+		dev_err(dev, "Failed to register subdev: %d\n", ret);
+		media_entity_cleanup(&sd->entity);
+	}
+
+	return ret;
+}
+
+/*
+ * msm_csiphy_unregister_entity - Unregister CSIPHY module subdev node
+ * @csiphy: CSIPHY device
+ */
+void msm_csiphy_unregister_entity(struct csiphy_device *csiphy)
+{
+	v4l2_device_unregister_subdev(&csiphy->subdev);
+	media_entity_cleanup(&csiphy->subdev.entity);
+}
diff --git a/drivers/media/platform/qcom/camss-8x16/camss-csiphy.h b/drivers/media/platform/qcom/camss-8x16/camss-csiphy.h
new file mode 100644
index 000000000000..ba8781122065
--- /dev/null
+++ b/drivers/media/platform/qcom/camss-8x16/camss-csiphy.h
@@ -0,0 +1,77 @@
+/*
+ * camss-csiphy.h
+ *
+ * Qualcomm MSM Camera Subsystem - CSIPHY Module
+ *
+ * Copyright (c) 2011-2015, The Linux Foundation. All rights reserved.
+ * Copyright (C) 2016-2017 Linaro Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#ifndef QC_MSM_CAMSS_CSIPHY_H
+#define QC_MSM_CAMSS_CSIPHY_H
+
+#include <linux/clk.h>
+#include <media/media-entity.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-mediabus.h>
+#include <media/v4l2-subdev.h>
+
+#define MSM_CSIPHY_PAD_SINK 0
+#define MSM_CSIPHY_PAD_SRC 1
+#define MSM_CSIPHY_PADS_NUM 2
+
+struct csiphy_lane {
+	u8 pos;
+	u8 pol;
+};
+
+struct csiphy_lanes_cfg {
+	int num_data;
+	struct csiphy_lane *data;
+	struct csiphy_lane clk;
+};
+
+struct csiphy_csi2_cfg {
+	struct csiphy_lanes_cfg lane_cfg;
+};
+
+struct csiphy_config {
+	u8 combo_mode;
+	u8 csid_id;
+	struct csiphy_csi2_cfg *csi2;
+};
+
+struct csiphy_device {
+	u8 id;
+	struct v4l2_subdev subdev;
+	struct media_pad pads[MSM_CSIPHY_PADS_NUM];
+	void __iomem *base;
+	void __iomem *base_clk_mux;
+	u32 irq;
+	char irq_name[30];
+	struct camss_clock *clock;
+	int nclocks;
+	u32 timer_clk_rate;
+	struct csiphy_config cfg;
+	struct v4l2_mbus_framefmt fmt[MSM_CSIPHY_PADS_NUM];
+};
+
+struct resources;
+
+int msm_csiphy_subdev_init(struct csiphy_device *csiphy,
+			   const struct resources *res, u8 id);
+
+int msm_csiphy_register_entity(struct csiphy_device *csiphy,
+			       struct v4l2_device *v4l2_dev);
+
+void msm_csiphy_unregister_entity(struct csiphy_device *csiphy);
+
+#endif /* QC_MSM_CAMSS_CSIPHY_H */
diff --git a/drivers/media/platform/qcom/camss-8x16/camss-ispif.c b/drivers/media/platform/qcom/camss-8x16/camss-ispif.c
new file mode 100644
index 000000000000..24da529397b5
--- /dev/null
+++ b/drivers/media/platform/qcom/camss-8x16/camss-ispif.c
@@ -0,0 +1,1175 @@
+/*
+ * camss-ispif.c
+ *
+ * Qualcomm MSM Camera Subsystem - ISPIF (ISP Interface) Module
+ *
+ * Copyright (c) 2013-2015, The Linux Foundation. All rights reserved.
+ * Copyright (C) 2015-2017 Linaro Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#include <linux/clk.h>
+#include <linux/completion.h>
+#include <linux/interrupt.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/mutex.h>
+#include <linux/platform_device.h>
+#include <media/media-entity.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-subdev.h>
+
+#include "camss-ispif.h"
+#include "camss.h"
+
+#define MSM_ISPIF_NAME "msm_ispif"
+
+#define ispif_line_array(ptr_line)	\
+	((const struct ispif_line (*)[]) &(ptr_line[-(ptr_line->id)]))
+
+#define to_ispif(ptr_line)	\
+	container_of(ispif_line_array(ptr_line), struct ispif_device, ptr_line)
+
+#define ISPIF_RST_CMD_0			0x008
+#define ISPIF_RST_CMD_0_STROBED_RST_EN		(1 << 0)
+#define ISPIF_RST_CMD_0_MISC_LOGIC_RST		(1 << 1)
+#define ISPIF_RST_CMD_0_SW_REG_RST		(1 << 2)
+#define ISPIF_RST_CMD_0_PIX_INTF_0_CSID_RST	(1 << 3)
+#define ISPIF_RST_CMD_0_PIX_INTF_0_VFE_RST	(1 << 4)
+#define ISPIF_RST_CMD_0_PIX_INTF_1_CSID_RST	(1 << 5)
+#define ISPIF_RST_CMD_0_PIX_INTF_1_VFE_RST	(1 << 6)
+#define ISPIF_RST_CMD_0_RDI_INTF_0_CSID_RST	(1 << 7)
+#define ISPIF_RST_CMD_0_RDI_INTF_0_VFE_RST	(1 << 8)
+#define ISPIF_RST_CMD_0_RDI_INTF_1_CSID_RST	(1 << 9)
+#define ISPIF_RST_CMD_0_RDI_INTF_1_VFE_RST	(1 << 10)
+#define ISPIF_RST_CMD_0_RDI_INTF_2_CSID_RST	(1 << 11)
+#define ISPIF_RST_CMD_0_RDI_INTF_2_VFE_RST	(1 << 12)
+#define ISPIF_RST_CMD_0_PIX_OUTPUT_0_MISR_RST	(1 << 16)
+#define ISPIF_RST_CMD_0_RDI_OUTPUT_0_MISR_RST	(1 << 17)
+#define ISPIF_RST_CMD_0_RDI_OUTPUT_1_MISR_RST	(1 << 18)
+#define ISPIF_RST_CMD_0_RDI_OUTPUT_2_MISR_RST	(1 << 19)
+#define ISPIF_IRQ_GLOBAL_CLEAR_CMD	0x01c
+#define ISPIF_VFE_m_CTRL_0(m)		(0x200 + 0x200 * (m))
+#define ISPIF_VFE_m_CTRL_0_PIX0_LINE_BUF_EN	(1 << 6)
+#define ISPIF_VFE_m_IRQ_MASK_0(m)	(0x208 + 0x200 * (m))
+#define ISPIF_VFE_m_IRQ_MASK_0_PIX0_ENABLE	0x00001249
+#define ISPIF_VFE_m_IRQ_MASK_0_PIX0_MASK	0x00001fff
+#define ISPIF_VFE_m_IRQ_MASK_0_RDI0_ENABLE	0x02492000
+#define ISPIF_VFE_m_IRQ_MASK_0_RDI0_MASK	0x03ffe000
+#define ISPIF_VFE_m_IRQ_MASK_1(m)	(0x20c + 0x200 * (m))
+#define ISPIF_VFE_m_IRQ_MASK_1_PIX1_ENABLE	0x00001249
+#define ISPIF_VFE_m_IRQ_MASK_1_PIX1_MASK	0x00001fff
+#define ISPIF_VFE_m_IRQ_MASK_1_RDI1_ENABLE	0x02492000
+#define ISPIF_VFE_m_IRQ_MASK_1_RDI1_MASK	0x03ffe000
+#define ISPIF_VFE_m_IRQ_MASK_2(m)	(0x210 + 0x200 * (m))
+#define ISPIF_VFE_m_IRQ_MASK_2_RDI2_ENABLE	0x00001249
+#define ISPIF_VFE_m_IRQ_MASK_2_RDI2_MASK	0x00001fff
+#define ISPIF_VFE_m_IRQ_STATUS_0(m)	(0x21c + 0x200 * (m))
+#define ISPIF_VFE_m_IRQ_STATUS_0_PIX0_OVERFLOW	(1 << 12)
+#define ISPIF_VFE_m_IRQ_STATUS_0_RDI0_OVERFLOW	(1 << 25)
+#define ISPIF_VFE_m_IRQ_STATUS_1(m)	(0x220 + 0x200 * (m))
+#define ISPIF_VFE_m_IRQ_STATUS_1_PIX1_OVERFLOW	(1 << 12)
+#define ISPIF_VFE_m_IRQ_STATUS_1_RDI1_OVERFLOW	(1 << 25)
+#define ISPIF_VFE_m_IRQ_STATUS_2(m)	(0x224 + 0x200 * (m))
+#define ISPIF_VFE_m_IRQ_STATUS_2_RDI2_OVERFLOW	(1 << 12)
+#define ISPIF_VFE_m_IRQ_CLEAR_0(m)	(0x230 + 0x200 * (m))
+#define ISPIF_VFE_m_IRQ_CLEAR_1(m)	(0x234 + 0x200 * (m))
+#define ISPIF_VFE_m_IRQ_CLEAR_2(m)	(0x238 + 0x200 * (m))
+#define ISPIF_VFE_m_INTF_INPUT_SEL(m)	(0x244 + 0x200 * (m))
+#define ISPIF_VFE_m_INTF_CMD_0(m)	(0x248 + 0x200 * (m))
+#define ISPIF_VFE_m_INTF_CMD_1(m)	(0x24c + 0x200 * (m))
+#define ISPIF_VFE_m_PIX_INTF_n_CID_MASK(m, n)	\
+					(0x254 + 0x200 * (m) + 0x4 * (n))
+#define ISPIF_VFE_m_RDI_INTF_n_CID_MASK(m, n)	\
+					(0x264 + 0x200 * (m) + 0x4 * (n))
+#define ISPIF_VFE_m_PIX_INTF_n_STATUS(m, n)	\
+					(0x2c0 + 0x200 * (m) + 0x4 * (n))
+#define ISPIF_VFE_m_RDI_INTF_n_STATUS(m, n)	\
+					(0x2d0 + 0x200 * (m) + 0x4 * (n))
+
+#define CSI_PIX_CLK_MUX_SEL		0x000
+#define CSI_RDI_CLK_MUX_SEL		0x008
+
+#define ISPIF_TIMEOUT_SLEEP_US		1000
+#define ISPIF_TIMEOUT_ALL_US		1000000
+#define ISPIF_RESET_TIMEOUT_MS		500
+
+enum ispif_intf_cmd {
+	CMD_DISABLE_FRAME_BOUNDARY = 0x0,
+	CMD_ENABLE_FRAME_BOUNDARY = 0x1,
+	CMD_DISABLE_IMMEDIATELY = 0x2,
+	CMD_ALL_DISABLE_IMMEDIATELY = 0xaaaaaaaa,
+	CMD_ALL_NO_CHANGE = 0xffffffff,
+};
+
+static const u32 ispif_formats[] = {
+	MEDIA_BUS_FMT_UYVY8_2X8,
+	MEDIA_BUS_FMT_VYUY8_2X8,
+	MEDIA_BUS_FMT_YUYV8_2X8,
+	MEDIA_BUS_FMT_YVYU8_2X8,
+	MEDIA_BUS_FMT_SBGGR8_1X8,
+	MEDIA_BUS_FMT_SGBRG8_1X8,
+	MEDIA_BUS_FMT_SGRBG8_1X8,
+	MEDIA_BUS_FMT_SRGGB8_1X8,
+	MEDIA_BUS_FMT_SBGGR10_1X10,
+	MEDIA_BUS_FMT_SGBRG10_1X10,
+	MEDIA_BUS_FMT_SGRBG10_1X10,
+	MEDIA_BUS_FMT_SRGGB10_1X10,
+	MEDIA_BUS_FMT_SBGGR12_1X12,
+	MEDIA_BUS_FMT_SGBRG12_1X12,
+	MEDIA_BUS_FMT_SGRBG12_1X12,
+	MEDIA_BUS_FMT_SRGGB12_1X12,
+};
+
+/*
+ * ispif_isr - ISPIF module interrupt handler
+ * @irq: Interrupt line
+ * @dev: ISPIF device
+ *
+ * Return IRQ_HANDLED on success
+ */
+static irqreturn_t ispif_isr(int irq, void *dev)
+{
+	struct ispif_device *ispif = dev;
+	u32 value0, value1, value2;
+
+	value0 = readl_relaxed(ispif->base + ISPIF_VFE_m_IRQ_STATUS_0(0));
+	value1 = readl_relaxed(ispif->base + ISPIF_VFE_m_IRQ_STATUS_1(0));
+	value2 = readl_relaxed(ispif->base + ISPIF_VFE_m_IRQ_STATUS_2(0));
+
+	writel_relaxed(value0, ispif->base + ISPIF_VFE_m_IRQ_CLEAR_0(0));
+	writel_relaxed(value1, ispif->base + ISPIF_VFE_m_IRQ_CLEAR_1(0));
+	writel_relaxed(value2, ispif->base + ISPIF_VFE_m_IRQ_CLEAR_2(0));
+
+	writel(0x1, ispif->base + ISPIF_IRQ_GLOBAL_CLEAR_CMD);
+
+	if ((value0 >> 27) & 0x1)
+		complete(&ispif->reset_complete);
+
+	if (unlikely(value0 & ISPIF_VFE_m_IRQ_STATUS_0_PIX0_OVERFLOW))
+		dev_err_ratelimited(to_device(ispif), "VFE0 pix0 overflow\n");
+
+	if (unlikely(value0 & ISPIF_VFE_m_IRQ_STATUS_0_RDI0_OVERFLOW))
+		dev_err_ratelimited(to_device(ispif), "VFE0 rdi0 overflow\n");
+
+	if (unlikely(value1 & ISPIF_VFE_m_IRQ_STATUS_1_PIX1_OVERFLOW))
+		dev_err_ratelimited(to_device(ispif), "VFE0 pix1 overflow\n");
+
+	if (unlikely(value1 & ISPIF_VFE_m_IRQ_STATUS_1_RDI1_OVERFLOW))
+		dev_err_ratelimited(to_device(ispif), "VFE0 rdi1 overflow\n");
+
+	if (unlikely(value2 & ISPIF_VFE_m_IRQ_STATUS_2_RDI2_OVERFLOW))
+		dev_err_ratelimited(to_device(ispif), "VFE0 rdi2 overflow\n");
+
+	return IRQ_HANDLED;
+}
+
+/*
+ * ispif_reset - Trigger reset on ISPIF module and wait to complete
+ * @ispif: ISPIF device
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int ispif_reset(struct ispif_device *ispif)
+{
+	unsigned long time;
+	u32 val;
+	int ret;
+
+	ret = camss_enable_clocks(ispif->nclocks_for_reset,
+				  ispif->clock_for_reset,
+				  to_device(ispif));
+	if (ret < 0)
+		return ret;
+
+	reinit_completion(&ispif->reset_complete);
+
+	val = ISPIF_RST_CMD_0_STROBED_RST_EN |
+		ISPIF_RST_CMD_0_MISC_LOGIC_RST |
+		ISPIF_RST_CMD_0_SW_REG_RST |
+		ISPIF_RST_CMD_0_PIX_INTF_0_CSID_RST |
+		ISPIF_RST_CMD_0_PIX_INTF_0_VFE_RST |
+		ISPIF_RST_CMD_0_PIX_INTF_1_CSID_RST |
+		ISPIF_RST_CMD_0_PIX_INTF_1_VFE_RST |
+		ISPIF_RST_CMD_0_RDI_INTF_0_CSID_RST |
+		ISPIF_RST_CMD_0_RDI_INTF_0_VFE_RST |
+		ISPIF_RST_CMD_0_RDI_INTF_1_CSID_RST |
+		ISPIF_RST_CMD_0_RDI_INTF_1_VFE_RST |
+		ISPIF_RST_CMD_0_RDI_INTF_2_CSID_RST |
+		ISPIF_RST_CMD_0_RDI_INTF_2_VFE_RST |
+		ISPIF_RST_CMD_0_PIX_OUTPUT_0_MISR_RST |
+		ISPIF_RST_CMD_0_RDI_OUTPUT_0_MISR_RST |
+		ISPIF_RST_CMD_0_RDI_OUTPUT_1_MISR_RST |
+		ISPIF_RST_CMD_0_RDI_OUTPUT_2_MISR_RST;
+
+	writel_relaxed(val, ispif->base + ISPIF_RST_CMD_0);
+
+	time = wait_for_completion_timeout(&ispif->reset_complete,
+		msecs_to_jiffies(ISPIF_RESET_TIMEOUT_MS));
+	if (!time) {
+		dev_err(to_device(ispif), "ISPIF reset timeout\n");
+		return -EIO;
+	}
+
+	camss_disable_clocks(ispif->nclocks_for_reset, ispif->clock_for_reset);
+
+	return 0;
+}
+
+/*
+ * ispif_set_power - Power on/off ISPIF module
+ * @sd: ISPIF V4L2 subdevice
+ * @on: Requested power state
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int ispif_set_power(struct v4l2_subdev *sd, int on)
+{
+	struct ispif_line *line = v4l2_get_subdevdata(sd);
+	struct ispif_device *ispif = to_ispif(line);
+	struct device *dev = to_device(ispif);
+	int ret = 0;
+
+	mutex_lock(&ispif->power_lock);
+
+	if (on) {
+		if (ispif->power_count) {
+			/* Power is already on */
+			ispif->power_count++;
+			goto exit;
+		}
+
+		ret = camss_enable_clocks(ispif->nclocks, ispif->clock, dev);
+		if (ret < 0)
+			goto exit;
+
+		ret = ispif_reset(ispif);
+		if (ret < 0) {
+			camss_disable_clocks(ispif->nclocks, ispif->clock);
+			goto exit;
+		}
+
+		ispif->intf_cmd[line->vfe_id].cmd_0 = CMD_ALL_NO_CHANGE;
+		ispif->intf_cmd[line->vfe_id].cmd_1 = CMD_ALL_NO_CHANGE;
+
+		ispif->power_count++;
+	} else {
+		if (ispif->power_count == 0) {
+			dev_err(dev, "ispif power off on power_count == 0\n");
+			goto exit;
+		} else if (ispif->power_count == 1) {
+			camss_disable_clocks(ispif->nclocks, ispif->clock);
+		}
+
+		ispif->power_count--;
+	}
+
+exit:
+	mutex_unlock(&ispif->power_lock);
+
+	return ret;
+}
+
+/*
+ * ispif_select_clk_mux - Select clock for PIX/RDI interface
+ * @ispif: ISPIF device
+ * @intf: VFE interface
+ * @csid: CSID HW module id
+ * @vfe: VFE HW module id
+ * @enable: enable or disable the selected clock
+ */
+static void ispif_select_clk_mux(struct ispif_device *ispif,
+				 enum ispif_intf intf, u8 csid,
+				 u8 vfe, u8 enable)
+{
+	u32 val;
+
+	switch (intf) {
+	case PIX0:
+		val = readl_relaxed(ispif->base_clk_mux + CSI_PIX_CLK_MUX_SEL);
+		val &= ~(0xf << (vfe * 8));
+		if (enable)
+			val |= (csid << (vfe * 8));
+		writel_relaxed(val, ispif->base_clk_mux + CSI_PIX_CLK_MUX_SEL);
+		break;
+
+	case RDI0:
+		val = readl_relaxed(ispif->base_clk_mux + CSI_RDI_CLK_MUX_SEL);
+		val &= ~(0xf << (vfe * 12));
+		if (enable)
+			val |= (csid << (vfe * 12));
+		writel_relaxed(val, ispif->base_clk_mux + CSI_RDI_CLK_MUX_SEL);
+		break;
+
+	case PIX1:
+		val = readl_relaxed(ispif->base_clk_mux + CSI_PIX_CLK_MUX_SEL);
+		val &= ~(0xf << (4 + (vfe * 8)));
+		if (enable)
+			val |= (csid << (4 + (vfe * 8)));
+		writel_relaxed(val, ispif->base_clk_mux + CSI_PIX_CLK_MUX_SEL);
+		break;
+
+	case RDI1:
+		val = readl_relaxed(ispif->base_clk_mux + CSI_RDI_CLK_MUX_SEL);
+		val &= ~(0xf << (4 + (vfe * 12)));
+		if (enable)
+			val |= (csid << (4 + (vfe * 12)));
+		writel_relaxed(val, ispif->base_clk_mux + CSI_RDI_CLK_MUX_SEL);
+		break;
+
+	case RDI2:
+		val = readl_relaxed(ispif->base_clk_mux + CSI_RDI_CLK_MUX_SEL);
+		val &= ~(0xf << (8 + (vfe * 12)));
+		if (enable)
+			val |= (csid << (8 + (vfe * 12)));
+		writel_relaxed(val, ispif->base_clk_mux + CSI_RDI_CLK_MUX_SEL);
+		break;
+	}
+
+	mb();
+}
+
+/*
+ * ispif_validate_intf_status - Validate current status of PIX/RDI interface
+ * @ispif: ISPIF device
+ * @intf: VFE interface
+ * @vfe: VFE HW module id
+ *
+ * Return 0 when interface is idle or -EBUSY otherwise
+ */
+static int ispif_validate_intf_status(struct ispif_device *ispif,
+				      enum ispif_intf intf, u8 vfe)
+{
+	int ret = 0;
+	u32 val = 0;
+
+	switch (intf) {
+	case PIX0:
+		val = readl_relaxed(ispif->base +
+			ISPIF_VFE_m_PIX_INTF_n_STATUS(vfe, 0));
+		break;
+	case RDI0:
+		val = readl_relaxed(ispif->base +
+			ISPIF_VFE_m_RDI_INTF_n_STATUS(vfe, 0));
+		break;
+	case PIX1:
+		val = readl_relaxed(ispif->base +
+			ISPIF_VFE_m_PIX_INTF_n_STATUS(vfe, 1));
+		break;
+	case RDI1:
+		val = readl_relaxed(ispif->base +
+			ISPIF_VFE_m_RDI_INTF_n_STATUS(vfe, 1));
+		break;
+	case RDI2:
+		val = readl_relaxed(ispif->base +
+			ISPIF_VFE_m_RDI_INTF_n_STATUS(vfe, 2));
+		break;
+	}
+
+	if ((val & 0xf) != 0xf) {
+		dev_err(to_device(ispif), "%s: ispif is busy: 0x%x\n",
+			__func__, val);
+		ret = -EBUSY;
+	}
+
+	return ret;
+}
+
+/*
+ * ispif_wait_for_stop - Wait for PIX/RDI interface to stop
+ * @ispif: ISPIF device
+ * @intf: VFE interface
+ * @vfe: VFE HW module id
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int ispif_wait_for_stop(struct ispif_device *ispif,
+			       enum ispif_intf intf, u8 vfe)
+{
+	u32 addr = 0;
+	u32 stop_flag = 0;
+	int ret;
+
+	switch (intf) {
+	case PIX0:
+		addr = ISPIF_VFE_m_PIX_INTF_n_STATUS(vfe, 0);
+		break;
+	case RDI0:
+		addr = ISPIF_VFE_m_RDI_INTF_n_STATUS(vfe, 0);
+		break;
+	case PIX1:
+		addr = ISPIF_VFE_m_PIX_INTF_n_STATUS(vfe, 1);
+		break;
+	case RDI1:
+		addr = ISPIF_VFE_m_RDI_INTF_n_STATUS(vfe, 1);
+		break;
+	case RDI2:
+		addr = ISPIF_VFE_m_RDI_INTF_n_STATUS(vfe, 2);
+		break;
+	}
+
+	ret = readl_poll_timeout(ispif->base + addr,
+				 stop_flag,
+				 (stop_flag & 0xf) == 0xf,
+				 ISPIF_TIMEOUT_SLEEP_US,
+				 ISPIF_TIMEOUT_ALL_US);
+	if (ret < 0)
+		dev_err(to_device(ispif), "%s: ispif stop timeout\n",
+			__func__);
+
+	return ret;
+}
+
+/*
+ * ispif_select_csid - Select CSID HW module for input from
+ * @ispif: ISPIF device
+ * @intf: VFE interface
+ * @csid: CSID HW module id
+ * @vfe: VFE HW module id
+ * @enable: enable or disable the selected input
+ */
+static void ispif_select_csid(struct ispif_device *ispif, enum ispif_intf intf,
+			      u8 csid, u8 vfe, u8 enable)
+{
+	u32 val;
+
+	val = readl_relaxed(ispif->base + ISPIF_VFE_m_INTF_INPUT_SEL(vfe));
+	switch (intf) {
+	case PIX0:
+		val &= ~(BIT(1) | BIT(0));
+		if (enable)
+			val |= csid;
+		break;
+	case RDI0:
+		val &= ~(BIT(5) | BIT(4));
+		if (enable)
+			val |= (csid << 4);
+		break;
+	case PIX1:
+		val &= ~(BIT(9) | BIT(8));
+		if (enable)
+			val |= (csid << 8);
+		break;
+	case RDI1:
+		val &= ~(BIT(13) | BIT(12));
+		if (enable)
+			val |= (csid << 12);
+		break;
+	case RDI2:
+		val &= ~(BIT(21) | BIT(20));
+		if (enable)
+			val |= (csid << 20);
+		break;
+	}
+
+	writel(val, ispif->base + ISPIF_VFE_m_INTF_INPUT_SEL(vfe));
+}
+
+/*
+ * ispif_select_cid - Enable/disable desired CID
+ * @ispif: ISPIF device
+ * @intf: VFE interface
+ * @cid: desired CID to enable/disable
+ * @vfe: VFE HW module id
+ * @enable: enable or disable the desired CID
+ */
+static void ispif_select_cid(struct ispif_device *ispif, enum ispif_intf intf,
+			     u8 cid, u8 vfe, u8 enable)
+{
+	u32 cid_mask = 1 << cid;
+	u32 addr = 0;
+	u32 val;
+
+	switch (intf) {
+	case PIX0:
+		addr = ISPIF_VFE_m_PIX_INTF_n_CID_MASK(vfe, 0);
+		break;
+	case RDI0:
+		addr = ISPIF_VFE_m_RDI_INTF_n_CID_MASK(vfe, 0);
+		break;
+	case PIX1:
+		addr = ISPIF_VFE_m_PIX_INTF_n_CID_MASK(vfe, 1);
+		break;
+	case RDI1:
+		addr = ISPIF_VFE_m_RDI_INTF_n_CID_MASK(vfe, 1);
+		break;
+	case RDI2:
+		addr = ISPIF_VFE_m_RDI_INTF_n_CID_MASK(vfe, 2);
+		break;
+	}
+
+	val = readl_relaxed(ispif->base + addr);
+	if (enable)
+		val |= cid_mask;
+	else
+		val &= ~cid_mask;
+
+	writel(val, ispif->base + addr);
+}
+
+/*
+ * ispif_config_irq - Enable/disable interrupts for PIX/RDI interface
+ * @ispif: ISPIF device
+ * @intf: VFE interface
+ * @vfe: VFE HW module id
+ * @enable: enable or disable
+ */
+static void ispif_config_irq(struct ispif_device *ispif, enum ispif_intf intf,
+			     u8 vfe, u8 enable)
+{
+	u32 val;
+
+	switch (intf) {
+	case PIX0:
+		val = readl_relaxed(ispif->base + ISPIF_VFE_m_IRQ_MASK_0(vfe));
+		val &= ~ISPIF_VFE_m_IRQ_MASK_0_PIX0_MASK;
+		if (enable)
+			val |= ISPIF_VFE_m_IRQ_MASK_0_PIX0_ENABLE;
+		writel_relaxed(val, ispif->base + ISPIF_VFE_m_IRQ_MASK_0(vfe));
+		writel_relaxed(ISPIF_VFE_m_IRQ_MASK_0_PIX0_ENABLE,
+			       ispif->base + ISPIF_VFE_m_IRQ_CLEAR_0(vfe));
+		break;
+	case RDI0:
+		val = readl_relaxed(ispif->base + ISPIF_VFE_m_IRQ_MASK_0(vfe));
+		val &= ~ISPIF_VFE_m_IRQ_MASK_0_RDI0_MASK;
+		if (enable)
+			val |= ISPIF_VFE_m_IRQ_MASK_0_RDI0_ENABLE;
+		writel_relaxed(val, ispif->base + ISPIF_VFE_m_IRQ_MASK_0(vfe));
+		writel_relaxed(ISPIF_VFE_m_IRQ_MASK_0_RDI0_ENABLE,
+			       ispif->base + ISPIF_VFE_m_IRQ_CLEAR_0(vfe));
+		break;
+	case PIX1:
+		val = readl_relaxed(ispif->base + ISPIF_VFE_m_IRQ_MASK_1(vfe));
+		val &= ~ISPIF_VFE_m_IRQ_MASK_1_PIX1_MASK;
+		if (enable)
+			val |= ISPIF_VFE_m_IRQ_MASK_1_PIX1_ENABLE;
+		writel_relaxed(val, ispif->base + ISPIF_VFE_m_IRQ_MASK_1(vfe));
+		writel_relaxed(ISPIF_VFE_m_IRQ_MASK_1_PIX1_ENABLE,
+			       ispif->base + ISPIF_VFE_m_IRQ_CLEAR_1(vfe));
+		break;
+	case RDI1:
+		val = readl_relaxed(ispif->base + ISPIF_VFE_m_IRQ_MASK_1(vfe));
+		val &= ~ISPIF_VFE_m_IRQ_MASK_1_RDI1_MASK;
+		if (enable)
+			val |= ISPIF_VFE_m_IRQ_MASK_1_RDI1_ENABLE;
+		writel_relaxed(val, ispif->base + ISPIF_VFE_m_IRQ_MASK_1(vfe));
+		writel_relaxed(ISPIF_VFE_m_IRQ_MASK_1_RDI1_ENABLE,
+			       ispif->base + ISPIF_VFE_m_IRQ_CLEAR_1(vfe));
+		break;
+	case RDI2:
+		val = readl_relaxed(ispif->base + ISPIF_VFE_m_IRQ_MASK_2(vfe));
+		val &= ~ISPIF_VFE_m_IRQ_MASK_2_RDI2_MASK;
+		if (enable)
+			val |= ISPIF_VFE_m_IRQ_MASK_2_RDI2_ENABLE;
+		writel_relaxed(val, ispif->base + ISPIF_VFE_m_IRQ_MASK_2(vfe));
+		writel_relaxed(ISPIF_VFE_m_IRQ_MASK_2_RDI2_ENABLE,
+			       ispif->base + ISPIF_VFE_m_IRQ_CLEAR_2(vfe));
+		break;
+	}
+
+	writel(0x1, ispif->base + ISPIF_IRQ_GLOBAL_CLEAR_CMD);
+}
+
+/*
+ * ispif_set_intf_cmd - Set command to enable/disable interface
+ * @ispif: ISPIF device
+ * @cmd: interface command
+ * @intf: VFE interface
+ * @vfe: VFE HW module id
+ * @vc: virtual channel
+ */
+static void ispif_set_intf_cmd(struct ispif_device *ispif, u8 cmd,
+			       enum ispif_intf intf, u8 vfe, u8 vc)
+{
+	u32 *val;
+
+	if (intf == RDI2) {
+		val = &ispif->intf_cmd[vfe].cmd_1;
+		*val &= ~(0x3 << (vc * 2 + 8));
+		*val |= (cmd << (vc * 2 + 8));
+		wmb();
+		writel_relaxed(*val, ispif->base + ISPIF_VFE_m_INTF_CMD_1(vfe));
+		wmb();
+	} else {
+		val = &ispif->intf_cmd[vfe].cmd_0;
+		*val &= ~(0x3 << (vc * 2 + intf * 8));
+		*val |= (cmd << (vc * 2 + intf * 8));
+		wmb();
+		writel_relaxed(*val, ispif->base + ISPIF_VFE_m_INTF_CMD_0(vfe));
+		wmb();
+	}
+}
+
+/*
+ * ispif_set_stream - Enable/disable streaming on ISPIF module
+ * @sd: ISPIF V4L2 subdevice
+ * @enable: Requested streaming state
+ *
+ * Main configuration of ISPIF module is also done here.
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int ispif_set_stream(struct v4l2_subdev *sd, int enable)
+{
+	struct ispif_line *line = v4l2_get_subdevdata(sd);
+	struct ispif_device *ispif = to_ispif(line);
+	enum ispif_intf intf = line->interface;
+	u8 csid = line->csid_id;
+	u8 vfe = line->vfe_id;
+	u8 vc = 0; /* Virtual Channel 0 */
+	u8 cid = vc * 4; /* id of Virtual Channel and Data Type set */
+	int ret;
+
+	if (enable) {
+		if (!media_entity_remote_pad(&line->pads[MSM_ISPIF_PAD_SINK]))
+			return -ENOLINK;
+
+		/* Config */
+
+		mutex_lock(&ispif->config_lock);
+		ispif_select_clk_mux(ispif, intf, csid, vfe, 1);
+
+		ret = ispif_validate_intf_status(ispif, intf, vfe);
+		if (ret < 0) {
+			mutex_unlock(&ispif->config_lock);
+			return ret;
+		}
+
+		ispif_select_csid(ispif, intf, csid, vfe, 1);
+		ispif_select_cid(ispif, intf, cid, vfe, 1);
+		ispif_config_irq(ispif, intf, vfe, 1);
+		ispif_set_intf_cmd(ispif, CMD_ENABLE_FRAME_BOUNDARY,
+				   intf, vfe, vc);
+	} else {
+		mutex_lock(&ispif->config_lock);
+		ispif_set_intf_cmd(ispif, CMD_DISABLE_FRAME_BOUNDARY,
+				   intf, vfe, vc);
+		mutex_unlock(&ispif->config_lock);
+
+		ret = ispif_wait_for_stop(ispif, intf, vfe);
+		if (ret < 0)
+			return ret;
+
+		mutex_lock(&ispif->config_lock);
+		ispif_config_irq(ispif, intf, vfe, 0);
+		ispif_select_cid(ispif, intf, cid, vfe, 0);
+		ispif_select_csid(ispif, intf, csid, vfe, 0);
+		ispif_select_clk_mux(ispif, intf, csid, vfe, 0);
+	}
+
+	mutex_unlock(&ispif->config_lock);
+
+	return 0;
+}
+
+/*
+ * __ispif_get_format - Get pointer to format structure
+ * @ispif: ISPIF line
+ * @cfg: V4L2 subdev pad configuration
+ * @pad: pad from which format is requested
+ * @which: TRY or ACTIVE format
+ *
+ * Return pointer to TRY or ACTIVE format structure
+ */
+static struct v4l2_mbus_framefmt *
+__ispif_get_format(struct ispif_line *line,
+		   struct v4l2_subdev_pad_config *cfg,
+		   unsigned int pad,
+		   enum v4l2_subdev_format_whence which)
+{
+	if (which == V4L2_SUBDEV_FORMAT_TRY)
+		return v4l2_subdev_get_try_format(&line->subdev, cfg, pad);
+
+	return &line->fmt[pad];
+}
+
+/*
+ * ispif_try_format - Handle try format by pad subdev method
+ * @ispif: ISPIF line
+ * @cfg: V4L2 subdev pad configuration
+ * @pad: pad on which format is requested
+ * @fmt: pointer to v4l2 format structure
+ * @which: wanted subdev format
+ */
+static void ispif_try_format(struct ispif_line *line,
+			     struct v4l2_subdev_pad_config *cfg,
+			     unsigned int pad,
+			     struct v4l2_mbus_framefmt *fmt,
+			     enum v4l2_subdev_format_whence which)
+{
+	unsigned int i;
+
+	switch (pad) {
+	case MSM_ISPIF_PAD_SINK:
+		/* Set format on sink pad */
+
+		for (i = 0; i < ARRAY_SIZE(ispif_formats); i++)
+			if (fmt->code == ispif_formats[i])
+				break;
+
+		/* If not found, use UYVY as default */
+		if (i >= ARRAY_SIZE(ispif_formats))
+			fmt->code = MEDIA_BUS_FMT_UYVY8_2X8;
+
+		fmt->width = clamp_t(u32, fmt->width, 1, 8191);
+		fmt->height = clamp_t(u32, fmt->height, 1, 8191);
+
+		fmt->field = V4L2_FIELD_NONE;
+		fmt->colorspace = V4L2_COLORSPACE_SRGB;
+
+		break;
+
+	case MSM_ISPIF_PAD_SRC:
+		/* Set and return a format same as sink pad */
+
+		*fmt = *__ispif_get_format(line, cfg, MSM_ISPIF_PAD_SINK,
+					   which);
+
+		break;
+	}
+
+	fmt->colorspace = V4L2_COLORSPACE_SRGB;
+}
+
+/*
+ * ispif_enum_mbus_code - Handle pixel format enumeration
+ * @sd: ISPIF V4L2 subdevice
+ * @cfg: V4L2 subdev pad configuration
+ * @code: pointer to v4l2_subdev_mbus_code_enum structure
+ * return -EINVAL or zero on success
+ */
+static int ispif_enum_mbus_code(struct v4l2_subdev *sd,
+				struct v4l2_subdev_pad_config *cfg,
+				struct v4l2_subdev_mbus_code_enum *code)
+{
+	struct ispif_line *line = v4l2_get_subdevdata(sd);
+	struct v4l2_mbus_framefmt *format;
+
+	if (code->pad == MSM_ISPIF_PAD_SINK) {
+		if (code->index >= ARRAY_SIZE(ispif_formats))
+			return -EINVAL;
+
+		code->code = ispif_formats[code->index];
+	} else {
+		if (code->index > 0)
+			return -EINVAL;
+
+		format = __ispif_get_format(line, cfg, MSM_ISPIF_PAD_SINK,
+					    code->which);
+
+		code->code = format->code;
+	}
+
+	return 0;
+}
+
+/*
+ * ispif_enum_frame_size - Handle frame size enumeration
+ * @sd: ISPIF V4L2 subdevice
+ * @cfg: V4L2 subdev pad configuration
+ * @fse: pointer to v4l2_subdev_frame_size_enum structure
+ * return -EINVAL or zero on success
+ */
+static int ispif_enum_frame_size(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_pad_config *cfg,
+				 struct v4l2_subdev_frame_size_enum *fse)
+{
+	struct ispif_line *line = v4l2_get_subdevdata(sd);
+	struct v4l2_mbus_framefmt format;
+
+	if (fse->index != 0)
+		return -EINVAL;
+
+	format.code = fse->code;
+	format.width = 1;
+	format.height = 1;
+	ispif_try_format(line, cfg, fse->pad, &format, fse->which);
+	fse->min_width = format.width;
+	fse->min_height = format.height;
+
+	if (format.code != fse->code)
+		return -EINVAL;
+
+	format.code = fse->code;
+	format.width = -1;
+	format.height = -1;
+	ispif_try_format(line, cfg, fse->pad, &format, fse->which);
+	fse->max_width = format.width;
+	fse->max_height = format.height;
+
+	return 0;
+}
+
+/*
+ * ispif_get_format - Handle get format by pads subdev method
+ * @sd: ISPIF V4L2 subdevice
+ * @cfg: V4L2 subdev pad configuration
+ * @fmt: pointer to v4l2 subdev format structure
+ *
+ * Return -EINVAL or zero on success
+ */
+static int ispif_get_format(struct v4l2_subdev *sd,
+			    struct v4l2_subdev_pad_config *cfg,
+			    struct v4l2_subdev_format *fmt)
+{
+	struct ispif_line *line = v4l2_get_subdevdata(sd);
+	struct v4l2_mbus_framefmt *format;
+
+	format = __ispif_get_format(line, cfg, fmt->pad, fmt->which);
+	if (format == NULL)
+		return -EINVAL;
+
+	fmt->format = *format;
+
+	return 0;
+}
+
+/*
+ * ispif_set_format - Handle set format by pads subdev method
+ * @sd: ISPIF V4L2 subdevice
+ * @cfg: V4L2 subdev pad configuration
+ * @fmt: pointer to v4l2 subdev format structure
+ *
+ * Return -EINVAL or zero on success
+ */
+static int ispif_set_format(struct v4l2_subdev *sd,
+			    struct v4l2_subdev_pad_config *cfg,
+			    struct v4l2_subdev_format *fmt)
+{
+	struct ispif_line *line = v4l2_get_subdevdata(sd);
+	struct v4l2_mbus_framefmt *format;
+
+	format = __ispif_get_format(line, cfg, fmt->pad, fmt->which);
+	if (format == NULL)
+		return -EINVAL;
+
+	ispif_try_format(line, cfg, fmt->pad, &fmt->format, fmt->which);
+	*format = fmt->format;
+
+	/* Propagate the format from sink to source */
+	if (fmt->pad == MSM_ISPIF_PAD_SINK) {
+		format = __ispif_get_format(line, cfg, MSM_ISPIF_PAD_SRC,
+					    fmt->which);
+
+		*format = fmt->format;
+		ispif_try_format(line, cfg, MSM_ISPIF_PAD_SRC, format,
+				 fmt->which);
+	}
+
+	return 0;
+}
+
+/*
+ * ispif_init_formats - Initialize formats on all pads
+ * @sd: ISPIF V4L2 subdevice
+ * @fh: V4L2 subdev file handle
+ *
+ * Initialize all pad formats with default values.
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int ispif_init_formats(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
+{
+	struct v4l2_subdev_format format = {
+		.pad = MSM_ISPIF_PAD_SINK,
+		.which = fh ? V4L2_SUBDEV_FORMAT_TRY :
+			      V4L2_SUBDEV_FORMAT_ACTIVE,
+		.format = {
+			.code = MEDIA_BUS_FMT_UYVY8_2X8,
+			.width = 1920,
+			.height = 1080
+		}
+	};
+
+	return ispif_set_format(sd, fh ? fh->pad : NULL, &format);
+}
+
+/*
+ * msm_ispif_subdev_init - Initialize ISPIF device structure and resources
+ * @ispif: ISPIF device
+ * @res: ISPIF module resources table
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+int msm_ispif_subdev_init(struct ispif_device *ispif,
+			  const struct resources_ispif *res)
+{
+	struct device *dev = to_device(ispif);
+	struct platform_device *pdev = to_platform_device(dev);
+	struct resource *r;
+	int i;
+	int ret;
+
+	/* Memory */
+
+	r = platform_get_resource_byname(pdev, IORESOURCE_MEM, res->reg[0]);
+	ispif->base = devm_ioremap_resource(dev, r);
+	if (IS_ERR(ispif->base)) {
+		dev_err(dev, "could not map memory\n");
+		return PTR_ERR(ispif->base);
+	}
+
+	r = platform_get_resource_byname(pdev, IORESOURCE_MEM, res->reg[1]);
+	ispif->base_clk_mux = devm_ioremap_resource(dev, r);
+	if (IS_ERR(ispif->base_clk_mux)) {
+		dev_err(dev, "could not map memory\n");
+		return PTR_ERR(ispif->base_clk_mux);
+	}
+
+	/* Interrupt */
+
+	r = platform_get_resource_byname(pdev, IORESOURCE_IRQ, res->interrupt);
+
+	if (!r) {
+		dev_err(dev, "missing IRQ\n");
+		return -EINVAL;
+	}
+
+	ispif->irq = r->start;
+	snprintf(ispif->irq_name, sizeof(ispif->irq_name), "%s_%s",
+		 dev_name(dev), MSM_ISPIF_NAME);
+	ret = devm_request_irq(dev, ispif->irq, ispif_isr,
+			       IRQF_TRIGGER_RISING, ispif->irq_name, ispif);
+	if (ret < 0) {
+		dev_err(dev, "request_irq failed: %d\n", ret);
+		return ret;
+	}
+
+	/* Clocks */
+
+	ispif->nclocks = 0;
+	while (res->clock[ispif->nclocks])
+		ispif->nclocks++;
+
+	ispif->clock = devm_kzalloc(dev, ispif->nclocks * sizeof(*ispif->clock),
+				    GFP_KERNEL);
+	if (!ispif->clock)
+		return -ENOMEM;
+
+	for (i = 0; i < ispif->nclocks; i++) {
+		struct camss_clock *clock = &ispif->clock[i];
+
+		clock->clk = devm_clk_get(dev, res->clock[i]);
+		if (IS_ERR(clock->clk))
+			return PTR_ERR(clock->clk);
+
+		clock->freq = NULL;
+		clock->nfreqs = 0;
+	}
+
+	ispif->nclocks_for_reset = 0;
+	while (res->clock_for_reset[ispif->nclocks_for_reset])
+		ispif->nclocks_for_reset++;
+
+	ispif->clock_for_reset = devm_kzalloc(dev, ispif->nclocks_for_reset *
+			sizeof(*ispif->clock_for_reset), GFP_KERNEL);
+	if (!ispif->clock_for_reset)
+		return -ENOMEM;
+
+	for (i = 0; i < ispif->nclocks_for_reset; i++) {
+		struct camss_clock *clock = &ispif->clock_for_reset[i];
+
+		clock->clk = devm_clk_get(dev, res->clock_for_reset[i]);
+		if (IS_ERR(clock->clk))
+			return PTR_ERR(clock->clk);
+
+		clock->freq = NULL;
+		clock->nfreqs = 0;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(ispif->line); i++)
+		ispif->line[i].id = i;
+
+	mutex_init(&ispif->power_lock);
+	ispif->power_count = 0;
+
+	mutex_init(&ispif->config_lock);
+
+	init_completion(&ispif->reset_complete);
+
+	return 0;
+}
+
+/*
+ * ispif_get_intf - Get ISPIF interface to use by VFE line id
+ * @line_id: VFE line id that the ISPIF line is connected to
+ *
+ * Return ISPIF interface to use
+ */
+static enum ispif_intf ispif_get_intf(enum vfe_line_id line_id)
+{
+	switch (line_id) {
+	case (VFE_LINE_RDI0):
+		return RDI0;
+	case (VFE_LINE_RDI1):
+		return RDI1;
+	case (VFE_LINE_RDI2):
+		return RDI2;
+	case (VFE_LINE_PIX):
+		return PIX0;
+	default:
+		return RDI0;
+	}
+}
+
+/*
+ * ispif_link_setup - Setup ISPIF connections
+ * @entity: Pointer to media entity structure
+ * @local: Pointer to local pad
+ * @remote: Pointer to remote pad
+ * @flags: Link flags
+ *
+ * Return 0 on success
+ */
+static int ispif_link_setup(struct media_entity *entity,
+			    const struct media_pad *local,
+			    const struct media_pad *remote, u32 flags)
+{
+	if (flags & MEDIA_LNK_FL_ENABLED) {
+		if (media_entity_remote_pad(local))
+			return -EBUSY;
+
+		if (local->flags & MEDIA_PAD_FL_SINK) {
+			struct v4l2_subdev *sd;
+			struct ispif_line *line;
+
+			sd = media_entity_to_v4l2_subdev(entity);
+			line = v4l2_get_subdevdata(sd);
+
+			msm_csid_get_csid_id(remote->entity, &line->csid_id);
+		} else { /* MEDIA_PAD_FL_SOURCE */
+			struct v4l2_subdev *sd;
+			struct ispif_line *line;
+			enum vfe_line_id id;
+
+			sd = media_entity_to_v4l2_subdev(entity);
+			line = v4l2_get_subdevdata(sd);
+
+			msm_vfe_get_vfe_id(remote->entity, &line->vfe_id);
+			msm_vfe_get_vfe_line_id(remote->entity, &id);
+			line->interface = ispif_get_intf(id);
+		}
+	}
+
+	return 0;
+}
+
+static const struct v4l2_subdev_core_ops ispif_core_ops = {
+	.s_power = ispif_set_power,
+};
+
+static const struct v4l2_subdev_video_ops ispif_video_ops = {
+	.s_stream = ispif_set_stream,
+};
+
+static const struct v4l2_subdev_pad_ops ispif_pad_ops = {
+	.enum_mbus_code = ispif_enum_mbus_code,
+	.enum_frame_size = ispif_enum_frame_size,
+	.get_fmt = ispif_get_format,
+	.set_fmt = ispif_set_format,
+};
+
+static const struct v4l2_subdev_ops ispif_v4l2_ops = {
+	.core = &ispif_core_ops,
+	.video = &ispif_video_ops,
+	.pad = &ispif_pad_ops,
+};
+
+static const struct v4l2_subdev_internal_ops ispif_v4l2_internal_ops = {
+	.open = ispif_init_formats,
+};
+
+static const struct media_entity_operations ispif_media_ops = {
+	.link_setup = ispif_link_setup,
+	.link_validate = v4l2_subdev_link_validate,
+};
+
+/*
+ * msm_ispif_register_entities - Register subdev node for ISPIF module
+ * @ispif: ISPIF device
+ * @v4l2_dev: V4L2 device
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+int msm_ispif_register_entities(struct ispif_device *ispif,
+				struct v4l2_device *v4l2_dev)
+{
+	struct device *dev = to_device(ispif);
+	int ret;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(ispif->line); i++) {
+		struct v4l2_subdev *sd = &ispif->line[i].subdev;
+		struct media_pad *pads = ispif->line[i].pads;
+
+		v4l2_subdev_init(sd, &ispif_v4l2_ops);
+		sd->internal_ops = &ispif_v4l2_internal_ops;
+		sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+		snprintf(sd->name, ARRAY_SIZE(sd->name), "%s%d",
+			 MSM_ISPIF_NAME, i);
+		v4l2_set_subdevdata(sd, &ispif->line[i]);
+
+		ret = ispif_init_formats(sd, NULL);
+		if (ret < 0) {
+			dev_err(dev, "Failed to init format: %d\n", ret);
+			goto error;
+		}
+
+		pads[MSM_ISPIF_PAD_SINK].flags = MEDIA_PAD_FL_SINK;
+		pads[MSM_ISPIF_PAD_SRC].flags = MEDIA_PAD_FL_SOURCE;
+
+		sd->entity.function = MEDIA_ENT_F_IO_V4L;
+		sd->entity.ops = &ispif_media_ops;
+		ret = media_entity_pads_init(&sd->entity, MSM_ISPIF_PADS_NUM,
+					     pads);
+		if (ret < 0) {
+			dev_err(dev, "Failed to init media entity: %d\n", ret);
+			goto error;
+		}
+
+		ret = v4l2_device_register_subdev(v4l2_dev, sd);
+		if (ret < 0) {
+			dev_err(dev, "Failed to register subdev: %d\n", ret);
+			media_entity_cleanup(&sd->entity);
+			goto error;
+		}
+	}
+
+	return 0;
+
+error:
+	for (i--; i >= 0; i--) {
+		struct v4l2_subdev *sd = &ispif->line[i].subdev;
+
+		v4l2_device_unregister_subdev(sd);
+		media_entity_cleanup(&sd->entity);
+	}
+
+	return ret;
+}
+
+/*
+ * msm_ispif_unregister_entities - Unregister ISPIF module subdev node
+ * @ispif: ISPIF device
+ */
+void msm_ispif_unregister_entities(struct ispif_device *ispif)
+{
+	int i;
+
+	mutex_destroy(&ispif->power_lock);
+	mutex_destroy(&ispif->config_lock);
+
+	for (i = 0; i < ARRAY_SIZE(ispif->line); i++) {
+		struct v4l2_subdev *sd = &ispif->line[i].subdev;
+
+		v4l2_device_unregister_subdev(sd);
+		media_entity_cleanup(&sd->entity);
+	}
+}
diff --git a/drivers/media/platform/qcom/camss-8x16/camss-ispif.h b/drivers/media/platform/qcom/camss-8x16/camss-ispif.h
new file mode 100644
index 000000000000..f668306020c3
--- /dev/null
+++ b/drivers/media/platform/qcom/camss-8x16/camss-ispif.h
@@ -0,0 +1,85 @@
+/*
+ * camss-ispif.h
+ *
+ * Qualcomm MSM Camera Subsystem - ISPIF (ISP Interface) Module
+ *
+ * Copyright (c) 2013-2014, The Linux Foundation. All rights reserved.
+ * Copyright (C) 2015-2017 Linaro Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#ifndef QC_MSM_CAMSS_ISPIF_H
+#define QC_MSM_CAMSS_ISPIF_H
+
+#include <linux/clk.h>
+#include <media/media-entity.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-subdev.h>
+
+/* Number of ISPIF lines - same as number of CSID hardware modules */
+#define MSM_ISPIF_LINE_NUM 2
+
+#define MSM_ISPIF_PAD_SINK 0
+#define MSM_ISPIF_PAD_SRC 1
+#define MSM_ISPIF_PADS_NUM 2
+
+#define MSM_ISPIF_VFE_NUM 1
+
+enum ispif_intf {
+	PIX0,
+	RDI0,
+	PIX1,
+	RDI1,
+	RDI2
+};
+
+struct ispif_intf_cmd_reg {
+	u32 cmd_0;
+	u32 cmd_1;
+};
+
+struct ispif_line {
+	u8 id;
+	u8 csid_id;
+	u8 vfe_id;
+	enum ispif_intf interface;
+	struct v4l2_subdev subdev;
+	struct media_pad pads[MSM_ISPIF_PADS_NUM];
+	struct v4l2_mbus_framefmt fmt[MSM_ISPIF_PADS_NUM];
+};
+
+struct ispif_device {
+	void __iomem *base;
+	void __iomem *base_clk_mux;
+	u32 irq;
+	char irq_name[30];
+	struct camss_clock *clock;
+	int nclocks;
+	struct camss_clock  *clock_for_reset;
+	int nclocks_for_reset;
+	struct completion reset_complete;
+	int power_count;
+	struct mutex power_lock;
+	struct ispif_intf_cmd_reg intf_cmd[MSM_ISPIF_VFE_NUM];
+	struct mutex config_lock;
+	struct ispif_line line[MSM_ISPIF_LINE_NUM];
+};
+
+struct resources_ispif;
+
+int msm_ispif_subdev_init(struct ispif_device *ispif,
+			  const struct resources_ispif *res);
+
+int msm_ispif_register_entities(struct ispif_device *ispif,
+				struct v4l2_device *v4l2_dev);
+
+void msm_ispif_unregister_entities(struct ispif_device *ispif);
+
+#endif /* QC_MSM_CAMSS_ISPIF_H */
diff --git a/drivers/media/platform/qcom/camss-8x16/camss-vfe.c b/drivers/media/platform/qcom/camss-8x16/camss-vfe.c
new file mode 100644
index 000000000000..b21b3c2dc77f
--- /dev/null
+++ b/drivers/media/platform/qcom/camss-8x16/camss-vfe.c
@@ -0,0 +1,3088 @@
+/*
+ * camss-vfe.c
+ *
+ * Qualcomm MSM Camera Subsystem - VFE (Video Front End) Module
+ *
+ * Copyright (c) 2013-2015, The Linux Foundation. All rights reserved.
+ * Copyright (C) 2015-2017 Linaro Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#include <linux/clk.h>
+#include <linux/completion.h>
+#include <linux/interrupt.h>
+#include <linux/iommu.h>
+#include <linux/iopoll.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/spinlock_types.h>
+#include <linux/spinlock.h>
+#include <media/media-entity.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-subdev.h>
+
+#include "camss-vfe.h"
+#include "camss.h"
+
+#define MSM_VFE_NAME "msm_vfe"
+
+#define vfe_line_array(ptr_line)	\
+	((const struct vfe_line (*)[]) &(ptr_line[-(ptr_line->id)]))
+
+#define to_vfe(ptr_line)	\
+	container_of(vfe_line_array(ptr_line), struct vfe_device, ptr_line)
+
+#define VFE_0_HW_VERSION		0x000
+
+#define VFE_0_GLOBAL_RESET_CMD		0x00c
+#define VFE_0_GLOBAL_RESET_CMD_CORE	(1 << 0)
+#define VFE_0_GLOBAL_RESET_CMD_CAMIF	(1 << 1)
+#define VFE_0_GLOBAL_RESET_CMD_BUS	(1 << 2)
+#define VFE_0_GLOBAL_RESET_CMD_BUS_BDG	(1 << 3)
+#define VFE_0_GLOBAL_RESET_CMD_REGISTER	(1 << 4)
+#define VFE_0_GLOBAL_RESET_CMD_TIMER	(1 << 5)
+#define VFE_0_GLOBAL_RESET_CMD_PM	(1 << 6)
+#define VFE_0_GLOBAL_RESET_CMD_BUS_MISR	(1 << 7)
+#define VFE_0_GLOBAL_RESET_CMD_TESTGEN	(1 << 8)
+
+#define VFE_0_MODULE_CFG		0x018
+#define VFE_0_MODULE_CFG_DEMUX			(1 << 2)
+#define VFE_0_MODULE_CFG_CHROMA_UPSAMPLE	(1 << 3)
+#define VFE_0_MODULE_CFG_SCALE_ENC		(1 << 23)
+#define VFE_0_MODULE_CFG_CROP_ENC		(1 << 27)
+
+#define VFE_0_CORE_CFG			0x01c
+#define VFE_0_CORE_CFG_PIXEL_PATTERN_YCBYCR	0x4
+#define VFE_0_CORE_CFG_PIXEL_PATTERN_YCRYCB	0x5
+#define VFE_0_CORE_CFG_PIXEL_PATTERN_CBYCRY	0x6
+#define VFE_0_CORE_CFG_PIXEL_PATTERN_CRYCBY	0x7
+
+#define VFE_0_IRQ_CMD			0x024
+#define VFE_0_IRQ_CMD_GLOBAL_CLEAR	(1 << 0)
+
+#define VFE_0_IRQ_MASK_0		0x028
+#define VFE_0_IRQ_MASK_0_CAMIF_SOF			(1 << 0)
+#define VFE_0_IRQ_MASK_0_CAMIF_EOF			(1 << 1)
+#define VFE_0_IRQ_MASK_0_RDIn_REG_UPDATE(n)		(1 << ((n) + 5))
+#define VFE_0_IRQ_MASK_0_line_n_REG_UPDATE(n)		\
+	((n) == VFE_LINE_PIX ? (1 << 4) : VFE_0_IRQ_MASK_0_RDIn_REG_UPDATE(n))
+#define VFE_0_IRQ_MASK_0_IMAGE_MASTER_n_PING_PONG(n)	(1 << ((n) + 8))
+#define VFE_0_IRQ_MASK_0_IMAGE_COMPOSITE_DONE_n(n)	(1 << ((n) + 25))
+#define VFE_0_IRQ_MASK_0_RESET_ACK			(1 << 31)
+#define VFE_0_IRQ_MASK_1		0x02c
+#define VFE_0_IRQ_MASK_1_CAMIF_ERROR			(1 << 0)
+#define VFE_0_IRQ_MASK_1_VIOLATION			(1 << 7)
+#define VFE_0_IRQ_MASK_1_BUS_BDG_HALT_ACK		(1 << 8)
+#define VFE_0_IRQ_MASK_1_IMAGE_MASTER_n_BUS_OVERFLOW(n)	(1 << ((n) + 9))
+#define VFE_0_IRQ_MASK_1_RDIn_SOF(n)			(1 << ((n) + 29))
+
+#define VFE_0_IRQ_CLEAR_0		0x030
+#define VFE_0_IRQ_CLEAR_1		0x034
+
+#define VFE_0_IRQ_STATUS_0		0x038
+#define VFE_0_IRQ_STATUS_0_CAMIF_SOF			(1 << 0)
+#define VFE_0_IRQ_STATUS_0_RDIn_REG_UPDATE(n)		(1 << ((n) + 5))
+#define VFE_0_IRQ_STATUS_0_line_n_REG_UPDATE(n)		\
+	((n) == VFE_LINE_PIX ? (1 << 4) : VFE_0_IRQ_STATUS_0_RDIn_REG_UPDATE(n))
+#define VFE_0_IRQ_STATUS_0_IMAGE_MASTER_n_PING_PONG(n)	(1 << ((n) + 8))
+#define VFE_0_IRQ_STATUS_0_IMAGE_COMPOSITE_DONE_n(n)	(1 << ((n) + 25))
+#define VFE_0_IRQ_STATUS_0_RESET_ACK			(1 << 31)
+#define VFE_0_IRQ_STATUS_1		0x03c
+#define VFE_0_IRQ_STATUS_1_VIOLATION			(1 << 7)
+#define VFE_0_IRQ_STATUS_1_BUS_BDG_HALT_ACK		(1 << 8)
+#define VFE_0_IRQ_STATUS_1_RDIn_SOF(n)			(1 << ((n) + 29))
+
+#define VFE_0_IRQ_COMPOSITE_MASK_0	0x40
+#define VFE_0_VIOLATION_STATUS		0x48
+
+#define VFE_0_BUS_CMD			0x4c
+#define VFE_0_BUS_CMD_Mx_RLD_CMD(x)	(1 << (x))
+
+#define VFE_0_BUS_CFG			0x050
+
+#define VFE_0_BUS_XBAR_CFG_x(x)		(0x58 + 0x4 * ((x) / 2))
+#define VFE_0_BUS_XBAR_CFG_x_M_PAIR_STREAM_EN			(1 << 1)
+#define VFE_0_BUS_XBAR_CFG_x_M_PAIR_STREAM_SWAP_INTER_INTRA	(0x3 << 4)
+#define VFE_0_BUS_XBAR_CFG_x_M_SINGLE_STREAM_SEL_SHIFT		8
+#define VFE_0_BUS_XBAR_CFG_x_M_SINGLE_STREAM_SEL_LUMA		0
+#define VFE_0_BUS_XBAR_CFG_x_M_SINGLE_STREAM_SEL_VAL_RDI0	5
+#define VFE_0_BUS_XBAR_CFG_x_M_SINGLE_STREAM_SEL_VAL_RDI1	6
+#define VFE_0_BUS_XBAR_CFG_x_M_SINGLE_STREAM_SEL_VAL_RDI2	7
+
+#define VFE_0_BUS_IMAGE_MASTER_n_WR_CFG(n)		(0x06c + 0x24 * (n))
+#define VFE_0_BUS_IMAGE_MASTER_n_WR_CFG_WR_PATH_SHIFT	0
+#define VFE_0_BUS_IMAGE_MASTER_n_WR_CFG_FRM_BASED_SHIFT	1
+#define VFE_0_BUS_IMAGE_MASTER_n_WR_PING_ADDR(n)	(0x070 + 0x24 * (n))
+#define VFE_0_BUS_IMAGE_MASTER_n_WR_PONG_ADDR(n)	(0x074 + 0x24 * (n))
+#define VFE_0_BUS_IMAGE_MASTER_n_WR_ADDR_CFG(n)		(0x078 + 0x24 * (n))
+#define VFE_0_BUS_IMAGE_MASTER_n_WR_ADDR_CFG_FRM_DROP_PER_SHIFT	2
+#define VFE_0_BUS_IMAGE_MASTER_n_WR_ADDR_CFG_FRM_DROP_PER_MASK	(0x1F << 2)
+
+#define VFE_0_BUS_IMAGE_MASTER_n_WR_UB_CFG(n)		(0x07c + 0x24 * (n))
+#define VFE_0_BUS_IMAGE_MASTER_n_WR_UB_CFG_OFFSET_SHIFT	16
+#define VFE_0_BUS_IMAGE_MASTER_n_WR_IMAGE_SIZE(n)	(0x080 + 0x24 * (n))
+#define VFE_0_BUS_IMAGE_MASTER_n_WR_BUFFER_CFG(n)	(0x084 + 0x24 * (n))
+#define VFE_0_BUS_IMAGE_MASTER_n_WR_FRAMEDROP_PATTERN(n)	\
+							(0x088 + 0x24 * (n))
+#define VFE_0_BUS_IMAGE_MASTER_n_WR_IRQ_SUBSAMPLE_PATTERN(n)	\
+							(0x08c + 0x24 * (n))
+#define VFE_0_BUS_IMAGE_MASTER_n_WR_IRQ_SUBSAMPLE_PATTERN_DEF	0xffffffff
+
+#define VFE_0_BUS_PING_PONG_STATUS	0x268
+
+#define VFE_0_BUS_BDG_CMD		0x2c0
+#define VFE_0_BUS_BDG_CMD_HALT_REQ	1
+
+#define VFE_0_BUS_BDG_QOS_CFG_0		0x2c4
+#define VFE_0_BUS_BDG_QOS_CFG_0_CFG	0xaaa5aaa5
+#define VFE_0_BUS_BDG_QOS_CFG_1		0x2c8
+#define VFE_0_BUS_BDG_QOS_CFG_2		0x2cc
+#define VFE_0_BUS_BDG_QOS_CFG_3		0x2d0
+#define VFE_0_BUS_BDG_QOS_CFG_4		0x2d4
+#define VFE_0_BUS_BDG_QOS_CFG_5		0x2d8
+#define VFE_0_BUS_BDG_QOS_CFG_6		0x2dc
+#define VFE_0_BUS_BDG_QOS_CFG_7		0x2e0
+#define VFE_0_BUS_BDG_QOS_CFG_7_CFG	0x0001aaa5
+
+#define VFE_0_RDI_CFG_x(x)		(0x2e8 + (0x4 * (x)))
+#define VFE_0_RDI_CFG_x_RDI_STREAM_SEL_SHIFT	28
+#define VFE_0_RDI_CFG_x_RDI_STREAM_SEL_MASK	(0xf << 28)
+#define VFE_0_RDI_CFG_x_RDI_M0_SEL_SHIFT	4
+#define VFE_0_RDI_CFG_x_RDI_M0_SEL_MASK		(0xf << 4)
+#define VFE_0_RDI_CFG_x_RDI_EN_BIT		(1 << 2)
+#define VFE_0_RDI_CFG_x_MIPI_EN_BITS		0x3
+#define VFE_0_RDI_CFG_x_RDI_Mr_FRAME_BASED_EN(r)	(1 << (16 + (r)))
+
+#define VFE_0_CAMIF_CMD				0x2f4
+#define VFE_0_CAMIF_CMD_DISABLE_FRAME_BOUNDARY	0
+#define VFE_0_CAMIF_CMD_ENABLE_FRAME_BOUNDARY	1
+#define VFE_0_CAMIF_CMD_CLEAR_CAMIF_STATUS	(1 << 2)
+#define VFE_0_CAMIF_CFG				0x2f8
+#define VFE_0_CAMIF_CFG_VFE_OUTPUT_EN		(1 << 6)
+#define VFE_0_CAMIF_FRAME_CFG			0x300
+#define VFE_0_CAMIF_WINDOW_WIDTH_CFG		0x304
+#define VFE_0_CAMIF_WINDOW_HEIGHT_CFG		0x308
+#define VFE_0_CAMIF_SUBSAMPLE_CFG_0		0x30c
+#define VFE_0_CAMIF_IRQ_SUBSAMPLE_PATTERN	0x314
+#define VFE_0_CAMIF_STATUS			0x31c
+#define VFE_0_CAMIF_STATUS_HALT			(1 << 31)
+
+#define VFE_0_REG_UPDATE			0x378
+#define VFE_0_REG_UPDATE_RDIn(n)		(1 << (1 + (n)))
+#define VFE_0_REG_UPDATE_line_n(n)		\
+			((n) == VFE_LINE_PIX ? 1 : VFE_0_REG_UPDATE_RDIn(n))
+
+#define VFE_0_DEMUX_CFG				0x424
+#define VFE_0_DEMUX_CFG_PERIOD			0x3
+#define VFE_0_DEMUX_GAIN_0			0x428
+#define VFE_0_DEMUX_GAIN_0_CH0_EVEN		(0x80 << 0)
+#define VFE_0_DEMUX_GAIN_0_CH0_ODD		(0x80 << 16)
+#define VFE_0_DEMUX_GAIN_1			0x42c
+#define VFE_0_DEMUX_GAIN_1_CH1			(0x80 << 0)
+#define VFE_0_DEMUX_GAIN_1_CH2			(0x80 << 16)
+#define VFE_0_DEMUX_EVEN_CFG			0x438
+#define VFE_0_DEMUX_EVEN_CFG_PATTERN_YUYV	0x9cac
+#define VFE_0_DEMUX_EVEN_CFG_PATTERN_YVYU	0xac9c
+#define VFE_0_DEMUX_EVEN_CFG_PATTERN_UYVY	0xc9ca
+#define VFE_0_DEMUX_EVEN_CFG_PATTERN_VYUY	0xcac9
+#define VFE_0_DEMUX_ODD_CFG			0x43c
+#define VFE_0_DEMUX_ODD_CFG_PATTERN_YUYV	0x9cac
+#define VFE_0_DEMUX_ODD_CFG_PATTERN_YVYU	0xac9c
+#define VFE_0_DEMUX_ODD_CFG_PATTERN_UYVY	0xc9ca
+#define VFE_0_DEMUX_ODD_CFG_PATTERN_VYUY	0xcac9
+
+#define VFE_0_SCALE_ENC_Y_CFG			0x75c
+#define VFE_0_SCALE_ENC_Y_H_IMAGE_SIZE		0x760
+#define VFE_0_SCALE_ENC_Y_H_PHASE		0x764
+#define VFE_0_SCALE_ENC_Y_V_IMAGE_SIZE		0x76c
+#define VFE_0_SCALE_ENC_Y_V_PHASE		0x770
+#define VFE_0_SCALE_ENC_CBCR_CFG		0x778
+#define VFE_0_SCALE_ENC_CBCR_H_IMAGE_SIZE	0x77c
+#define VFE_0_SCALE_ENC_CBCR_H_PHASE		0x780
+#define VFE_0_SCALE_ENC_CBCR_V_IMAGE_SIZE	0x790
+#define VFE_0_SCALE_ENC_CBCR_V_PHASE		0x794
+
+#define VFE_0_CROP_ENC_Y_WIDTH			0x854
+#define VFE_0_CROP_ENC_Y_HEIGHT			0x858
+#define VFE_0_CROP_ENC_CBCR_WIDTH		0x85c
+#define VFE_0_CROP_ENC_CBCR_HEIGHT		0x860
+
+#define VFE_0_CLAMP_ENC_MAX_CFG			0x874
+#define VFE_0_CLAMP_ENC_MAX_CFG_CH0		(0xff << 0)
+#define VFE_0_CLAMP_ENC_MAX_CFG_CH1		(0xff << 8)
+#define VFE_0_CLAMP_ENC_MAX_CFG_CH2		(0xff << 16)
+#define VFE_0_CLAMP_ENC_MIN_CFG			0x878
+#define VFE_0_CLAMP_ENC_MIN_CFG_CH0		(0x0 << 0)
+#define VFE_0_CLAMP_ENC_MIN_CFG_CH1		(0x0 << 8)
+#define VFE_0_CLAMP_ENC_MIN_CFG_CH2		(0x0 << 16)
+
+#define VFE_0_CGC_OVERRIDE_1			0x974
+#define VFE_0_CGC_OVERRIDE_1_IMAGE_Mx_CGC_OVERRIDE(x)	(1 << (x))
+
+/* VFE reset timeout */
+#define VFE_RESET_TIMEOUT_MS 50
+/* VFE halt timeout */
+#define VFE_HALT_TIMEOUT_MS 100
+/* Max number of frame drop updates per frame */
+#define VFE_FRAME_DROP_UPDATES 5
+/* Frame drop value. NOTE: VAL + UPDATES should not exceed 31 */
+#define VFE_FRAME_DROP_VAL 20
+
+#define VFE_NEXT_SOF_MS 500
+
+#define CAMIF_TIMEOUT_SLEEP_US 1000
+#define CAMIF_TIMEOUT_ALL_US 1000000
+
+#define SCALER_RATIO_MAX 16
+
+static const struct {
+	u32 code;
+	u8 bpp;
+} vfe_formats[] = {
+	{
+		MEDIA_BUS_FMT_UYVY8_2X8,
+		8,
+	},
+	{
+		MEDIA_BUS_FMT_VYUY8_2X8,
+		8,
+	},
+	{
+		MEDIA_BUS_FMT_YUYV8_2X8,
+		8,
+	},
+	{
+		MEDIA_BUS_FMT_YVYU8_2X8,
+		8,
+	},
+	{
+		MEDIA_BUS_FMT_SBGGR8_1X8,
+		8,
+	},
+	{
+		MEDIA_BUS_FMT_SGBRG8_1X8,
+		8,
+	},
+	{
+		MEDIA_BUS_FMT_SGRBG8_1X8,
+		8,
+	},
+	{
+		MEDIA_BUS_FMT_SRGGB8_1X8,
+		8,
+	},
+	{
+		MEDIA_BUS_FMT_SBGGR10_1X10,
+		10,
+	},
+	{
+		MEDIA_BUS_FMT_SGBRG10_1X10,
+		10,
+	},
+	{
+		MEDIA_BUS_FMT_SGRBG10_1X10,
+		10,
+	},
+	{
+		MEDIA_BUS_FMT_SRGGB10_1X10,
+		10,
+	},
+	{
+		MEDIA_BUS_FMT_SBGGR12_1X12,
+		12,
+	},
+	{
+		MEDIA_BUS_FMT_SGBRG12_1X12,
+		12,
+	},
+	{
+		MEDIA_BUS_FMT_SGRBG12_1X12,
+		12,
+	},
+	{
+		MEDIA_BUS_FMT_SRGGB12_1X12,
+		12,
+	}
+};
+
+/*
+ * vfe_get_bpp - map media bus format to bits per pixel
+ * @code: media bus format code
+ *
+ * Return number of bits per pixel
+ */
+static u8 vfe_get_bpp(u32 code)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(vfe_formats); i++)
+		if (code == vfe_formats[i].code)
+			return vfe_formats[i].bpp;
+
+	WARN(1, "Unknown format\n");
+
+	return vfe_formats[0].bpp;
+}
+
+static inline void vfe_reg_clr(struct vfe_device *vfe, u32 reg, u32 clr_bits)
+{
+	u32 bits = readl_relaxed(vfe->base + reg);
+
+	writel_relaxed(bits & ~clr_bits, vfe->base + reg);
+}
+
+static inline void vfe_reg_set(struct vfe_device *vfe, u32 reg, u32 set_bits)
+{
+	u32 bits = readl_relaxed(vfe->base + reg);
+
+	writel_relaxed(bits | set_bits, vfe->base + reg);
+}
+
+static void vfe_global_reset(struct vfe_device *vfe)
+{
+	u32 reset_bits = VFE_0_GLOBAL_RESET_CMD_TESTGEN		|
+			 VFE_0_GLOBAL_RESET_CMD_BUS_MISR	|
+			 VFE_0_GLOBAL_RESET_CMD_PM		|
+			 VFE_0_GLOBAL_RESET_CMD_TIMER		|
+			 VFE_0_GLOBAL_RESET_CMD_REGISTER	|
+			 VFE_0_GLOBAL_RESET_CMD_BUS_BDG		|
+			 VFE_0_GLOBAL_RESET_CMD_BUS		|
+			 VFE_0_GLOBAL_RESET_CMD_CAMIF		|
+			 VFE_0_GLOBAL_RESET_CMD_CORE;
+
+	writel_relaxed(reset_bits, vfe->base + VFE_0_GLOBAL_RESET_CMD);
+}
+
+static void vfe_wm_enable(struct vfe_device *vfe, u8 wm, u8 enable)
+{
+	if (enable)
+		vfe_reg_set(vfe, VFE_0_BUS_IMAGE_MASTER_n_WR_CFG(wm),
+			    1 << VFE_0_BUS_IMAGE_MASTER_n_WR_CFG_WR_PATH_SHIFT);
+	else
+		vfe_reg_clr(vfe, VFE_0_BUS_IMAGE_MASTER_n_WR_CFG(wm),
+			    1 << VFE_0_BUS_IMAGE_MASTER_n_WR_CFG_WR_PATH_SHIFT);
+}
+
+static void vfe_wm_frame_based(struct vfe_device *vfe, u8 wm, u8 enable)
+{
+	if (enable)
+		vfe_reg_set(vfe, VFE_0_BUS_IMAGE_MASTER_n_WR_CFG(wm),
+			1 << VFE_0_BUS_IMAGE_MASTER_n_WR_CFG_FRM_BASED_SHIFT);
+	else
+		vfe_reg_clr(vfe, VFE_0_BUS_IMAGE_MASTER_n_WR_CFG(wm),
+			1 << VFE_0_BUS_IMAGE_MASTER_n_WR_CFG_FRM_BASED_SHIFT);
+}
+
+#define CALC_WORD(width, M, N) (((width) * (M) + (N) - 1) / (N))
+
+static int vfe_word_per_line(uint32_t format, uint32_t pixel_per_line)
+{
+	int val = 0;
+
+	switch (format) {
+	case V4L2_PIX_FMT_NV12:
+	case V4L2_PIX_FMT_NV21:
+	case V4L2_PIX_FMT_NV16:
+	case V4L2_PIX_FMT_NV61:
+		val = CALC_WORD(pixel_per_line, 1, 8);
+		break;
+	case V4L2_PIX_FMT_YUYV:
+	case V4L2_PIX_FMT_YVYU:
+	case V4L2_PIX_FMT_UYVY:
+	case V4L2_PIX_FMT_VYUY:
+		val = CALC_WORD(pixel_per_line, 2, 8);
+		break;
+	}
+
+	return val;
+}
+
+static void vfe_get_wm_sizes(struct v4l2_pix_format_mplane *pix, u8 plane,
+			     u16 *width, u16 *height, u16 *bytesperline)
+{
+	switch (pix->pixelformat) {
+	case V4L2_PIX_FMT_NV12:
+	case V4L2_PIX_FMT_NV21:
+		*width = pix->width;
+		*height = pix->height;
+		*bytesperline = pix->plane_fmt[0].bytesperline;
+		if (plane == 1)
+			*height /= 2;
+		break;
+	case V4L2_PIX_FMT_NV16:
+	case V4L2_PIX_FMT_NV61:
+		*width = pix->width;
+		*height = pix->height;
+		*bytesperline = pix->plane_fmt[0].bytesperline;
+		break;
+	}
+}
+
+static void vfe_wm_line_based(struct vfe_device *vfe, u32 wm,
+			      struct v4l2_pix_format_mplane *pix,
+			      u8 plane, u32 enable)
+{
+	u32 reg;
+
+	if (enable) {
+		u16 width = 0, height = 0, bytesperline = 0, wpl;
+
+		vfe_get_wm_sizes(pix, plane, &width, &height, &bytesperline);
+
+		wpl = vfe_word_per_line(pix->pixelformat, width);
+
+		reg = height - 1;
+		reg |= ((wpl + 1) / 2 - 1) << 16;
+
+		writel_relaxed(reg, vfe->base +
+			       VFE_0_BUS_IMAGE_MASTER_n_WR_IMAGE_SIZE(wm));
+
+		wpl = vfe_word_per_line(pix->pixelformat, bytesperline);
+
+		reg = 0x3;
+		reg |= (height - 1) << 4;
+		reg |= wpl << 16;
+
+		writel_relaxed(reg, vfe->base +
+			       VFE_0_BUS_IMAGE_MASTER_n_WR_BUFFER_CFG(wm));
+	} else {
+		writel_relaxed(0, vfe->base +
+			       VFE_0_BUS_IMAGE_MASTER_n_WR_IMAGE_SIZE(wm));
+		writel_relaxed(0, vfe->base +
+			       VFE_0_BUS_IMAGE_MASTER_n_WR_BUFFER_CFG(wm));
+	}
+}
+
+static void vfe_wm_set_framedrop_period(struct vfe_device *vfe, u8 wm, u8 per)
+{
+	u32 reg;
+
+	reg = readl_relaxed(vfe->base +
+			    VFE_0_BUS_IMAGE_MASTER_n_WR_ADDR_CFG(wm));
+
+	reg &= ~(VFE_0_BUS_IMAGE_MASTER_n_WR_ADDR_CFG_FRM_DROP_PER_MASK);
+
+	reg |= (per << VFE_0_BUS_IMAGE_MASTER_n_WR_ADDR_CFG_FRM_DROP_PER_SHIFT)
+		& VFE_0_BUS_IMAGE_MASTER_n_WR_ADDR_CFG_FRM_DROP_PER_MASK;
+
+	writel_relaxed(reg,
+		       vfe->base + VFE_0_BUS_IMAGE_MASTER_n_WR_ADDR_CFG(wm));
+}
+
+static void vfe_wm_set_framedrop_pattern(struct vfe_device *vfe, u8 wm,
+					 u32 pattern)
+{
+	writel_relaxed(pattern,
+	       vfe->base + VFE_0_BUS_IMAGE_MASTER_n_WR_FRAMEDROP_PATTERN(wm));
+}
+
+static void vfe_wm_set_ub_cfg(struct vfe_device *vfe, u8 wm, u16 offset,
+			      u16 depth)
+{
+	u32 reg;
+
+	reg = (offset << VFE_0_BUS_IMAGE_MASTER_n_WR_UB_CFG_OFFSET_SHIFT) |
+		depth;
+	writel_relaxed(reg, vfe->base + VFE_0_BUS_IMAGE_MASTER_n_WR_UB_CFG(wm));
+}
+
+static void vfe_bus_reload_wm(struct vfe_device *vfe, u8 wm)
+{
+	wmb();
+	writel_relaxed(VFE_0_BUS_CMD_Mx_RLD_CMD(wm), vfe->base + VFE_0_BUS_CMD);
+	wmb();
+}
+
+static void vfe_wm_set_ping_addr(struct vfe_device *vfe, u8 wm, u32 addr)
+{
+	writel_relaxed(addr,
+		       vfe->base + VFE_0_BUS_IMAGE_MASTER_n_WR_PING_ADDR(wm));
+}
+
+static void vfe_wm_set_pong_addr(struct vfe_device *vfe, u8 wm, u32 addr)
+{
+	writel_relaxed(addr,
+		       vfe->base + VFE_0_BUS_IMAGE_MASTER_n_WR_PONG_ADDR(wm));
+}
+
+static int vfe_wm_get_ping_pong_status(struct vfe_device *vfe, u8 wm)
+{
+	u32 reg;
+
+	reg = readl_relaxed(vfe->base + VFE_0_BUS_PING_PONG_STATUS);
+
+	return (reg >> wm) & 0x1;
+}
+
+static void vfe_bus_enable_wr_if(struct vfe_device *vfe, u8 enable)
+{
+	if (enable)
+		writel_relaxed(0x10000009, vfe->base + VFE_0_BUS_CFG);
+	else
+		writel_relaxed(0, vfe->base + VFE_0_BUS_CFG);
+}
+
+static void vfe_bus_connect_wm_to_rdi(struct vfe_device *vfe, u8 wm,
+				      enum vfe_line_id id)
+{
+	u32 reg;
+
+	reg = VFE_0_RDI_CFG_x_MIPI_EN_BITS;
+	reg |= VFE_0_RDI_CFG_x_RDI_Mr_FRAME_BASED_EN(id);
+	vfe_reg_set(vfe, VFE_0_RDI_CFG_x(0), reg);
+
+	reg = VFE_0_RDI_CFG_x_RDI_EN_BIT;
+	reg |= ((3 * id) << VFE_0_RDI_CFG_x_RDI_STREAM_SEL_SHIFT) &
+		VFE_0_RDI_CFG_x_RDI_STREAM_SEL_MASK;
+	vfe_reg_set(vfe, VFE_0_RDI_CFG_x(id), reg);
+
+	switch (id) {
+	case VFE_LINE_RDI0:
+	default:
+		reg = VFE_0_BUS_XBAR_CFG_x_M_SINGLE_STREAM_SEL_VAL_RDI0 <<
+		      VFE_0_BUS_XBAR_CFG_x_M_SINGLE_STREAM_SEL_SHIFT;
+		break;
+	case VFE_LINE_RDI1:
+		reg = VFE_0_BUS_XBAR_CFG_x_M_SINGLE_STREAM_SEL_VAL_RDI1 <<
+		      VFE_0_BUS_XBAR_CFG_x_M_SINGLE_STREAM_SEL_SHIFT;
+		break;
+	case VFE_LINE_RDI2:
+		reg = VFE_0_BUS_XBAR_CFG_x_M_SINGLE_STREAM_SEL_VAL_RDI2 <<
+		      VFE_0_BUS_XBAR_CFG_x_M_SINGLE_STREAM_SEL_SHIFT;
+		break;
+	}
+
+	if (wm % 2 == 1)
+		reg <<= 16;
+
+	vfe_reg_set(vfe, VFE_0_BUS_XBAR_CFG_x(wm), reg);
+}
+
+static void vfe_wm_set_subsample(struct vfe_device *vfe, u8 wm)
+{
+	writel_relaxed(VFE_0_BUS_IMAGE_MASTER_n_WR_IRQ_SUBSAMPLE_PATTERN_DEF,
+	       vfe->base +
+	       VFE_0_BUS_IMAGE_MASTER_n_WR_IRQ_SUBSAMPLE_PATTERN(wm));
+}
+
+static void vfe_bus_disconnect_wm_from_rdi(struct vfe_device *vfe, u8 wm,
+					   enum vfe_line_id id)
+{
+	u32 reg;
+
+	reg = VFE_0_RDI_CFG_x_RDI_Mr_FRAME_BASED_EN(id);
+	vfe_reg_clr(vfe, VFE_0_RDI_CFG_x(0), reg);
+
+	reg = VFE_0_RDI_CFG_x_RDI_EN_BIT;
+	vfe_reg_clr(vfe, VFE_0_RDI_CFG_x(id), reg);
+
+	switch (id) {
+	case VFE_LINE_RDI0:
+	default:
+		reg = VFE_0_BUS_XBAR_CFG_x_M_SINGLE_STREAM_SEL_VAL_RDI0 <<
+		      VFE_0_BUS_XBAR_CFG_x_M_SINGLE_STREAM_SEL_SHIFT;
+		break;
+	case VFE_LINE_RDI1:
+		reg = VFE_0_BUS_XBAR_CFG_x_M_SINGLE_STREAM_SEL_VAL_RDI1 <<
+		      VFE_0_BUS_XBAR_CFG_x_M_SINGLE_STREAM_SEL_SHIFT;
+		break;
+	case VFE_LINE_RDI2:
+		reg = VFE_0_BUS_XBAR_CFG_x_M_SINGLE_STREAM_SEL_VAL_RDI2 <<
+		      VFE_0_BUS_XBAR_CFG_x_M_SINGLE_STREAM_SEL_SHIFT;
+		break;
+	}
+
+	if (wm % 2 == 1)
+		reg <<= 16;
+
+	vfe_reg_clr(vfe, VFE_0_BUS_XBAR_CFG_x(wm), reg);
+}
+
+static void vfe_set_xbar_cfg(struct vfe_device *vfe, struct vfe_output *output,
+			     u8 enable)
+{
+	struct vfe_line *line = container_of(output, struct vfe_line, output);
+	u32 p = line->video_out.active_fmt.fmt.pix_mp.pixelformat;
+	u32 reg;
+	unsigned int i;
+
+	for (i = 0; i < output->wm_num; i++) {
+		if (i == 0) {
+			reg = VFE_0_BUS_XBAR_CFG_x_M_SINGLE_STREAM_SEL_LUMA <<
+				VFE_0_BUS_XBAR_CFG_x_M_SINGLE_STREAM_SEL_SHIFT;
+		} else if (i == 1) {
+			reg = VFE_0_BUS_XBAR_CFG_x_M_PAIR_STREAM_EN;
+			if (p == V4L2_PIX_FMT_NV12 || p == V4L2_PIX_FMT_NV16)
+				reg |= VFE_0_BUS_XBAR_CFG_x_M_PAIR_STREAM_SWAP_INTER_INTRA;
+		}
+
+		if (output->wm_idx[i] % 2 == 1)
+			reg <<= 16;
+
+		if (enable)
+			vfe_reg_set(vfe,
+				    VFE_0_BUS_XBAR_CFG_x(output->wm_idx[i]),
+				    reg);
+		else
+			vfe_reg_clr(vfe,
+				    VFE_0_BUS_XBAR_CFG_x(output->wm_idx[i]),
+				    reg);
+	}
+}
+
+static void vfe_set_rdi_cid(struct vfe_device *vfe, enum vfe_line_id id, u8 cid)
+{
+	vfe_reg_clr(vfe, VFE_0_RDI_CFG_x(id),
+		    VFE_0_RDI_CFG_x_RDI_M0_SEL_MASK);
+
+	vfe_reg_set(vfe, VFE_0_RDI_CFG_x(id),
+		    cid << VFE_0_RDI_CFG_x_RDI_M0_SEL_SHIFT);
+}
+
+static void vfe_reg_update(struct vfe_device *vfe, enum vfe_line_id line_id)
+{
+	vfe->reg_update |= VFE_0_REG_UPDATE_line_n(line_id);
+	wmb();
+	writel_relaxed(vfe->reg_update, vfe->base + VFE_0_REG_UPDATE);
+	wmb();
+}
+
+static void vfe_enable_irq_wm_line(struct vfe_device *vfe, u8 wm,
+				   enum vfe_line_id line_id, u8 enable)
+{
+	u32 irq_en0 = VFE_0_IRQ_MASK_0_IMAGE_MASTER_n_PING_PONG(wm) |
+		      VFE_0_IRQ_MASK_0_line_n_REG_UPDATE(line_id);
+	u32 irq_en1 = VFE_0_IRQ_MASK_1_IMAGE_MASTER_n_BUS_OVERFLOW(wm) |
+		      VFE_0_IRQ_MASK_1_RDIn_SOF(line_id);
+
+	if (enable) {
+		vfe_reg_set(vfe, VFE_0_IRQ_MASK_0, irq_en0);
+		vfe_reg_set(vfe, VFE_0_IRQ_MASK_1, irq_en1);
+	} else {
+		vfe_reg_clr(vfe, VFE_0_IRQ_MASK_0, irq_en0);
+		vfe_reg_clr(vfe, VFE_0_IRQ_MASK_1, irq_en1);
+	}
+}
+
+static void vfe_enable_irq_pix_line(struct vfe_device *vfe, u8 comp,
+				    enum vfe_line_id line_id, u8 enable)
+{
+	struct vfe_output *output = &vfe->line[line_id].output;
+	unsigned int i;
+	u32 irq_en0;
+	u32 irq_en1;
+	u32 comp_mask = 0;
+
+	irq_en0 = VFE_0_IRQ_MASK_0_CAMIF_SOF;
+	irq_en0 |= VFE_0_IRQ_MASK_0_CAMIF_EOF;
+	irq_en0 |= VFE_0_IRQ_MASK_0_IMAGE_COMPOSITE_DONE_n(comp);
+	irq_en0 |= VFE_0_IRQ_MASK_0_line_n_REG_UPDATE(line_id);
+	irq_en1 = VFE_0_IRQ_MASK_1_CAMIF_ERROR;
+	for (i = 0; i < output->wm_num; i++) {
+		irq_en1 |= VFE_0_IRQ_MASK_1_IMAGE_MASTER_n_BUS_OVERFLOW(
+							output->wm_idx[i]);
+		comp_mask |= (1 << output->wm_idx[i]) << comp * 8;
+	}
+
+	if (enable) {
+		vfe_reg_set(vfe, VFE_0_IRQ_MASK_0, irq_en0);
+		vfe_reg_set(vfe, VFE_0_IRQ_MASK_1, irq_en1);
+		vfe_reg_set(vfe, VFE_0_IRQ_COMPOSITE_MASK_0, comp_mask);
+	} else {
+		vfe_reg_clr(vfe, VFE_0_IRQ_MASK_0, irq_en0);
+		vfe_reg_clr(vfe, VFE_0_IRQ_MASK_1, irq_en1);
+		vfe_reg_clr(vfe, VFE_0_IRQ_COMPOSITE_MASK_0, comp_mask);
+	}
+}
+
+static void vfe_enable_irq_common(struct vfe_device *vfe)
+{
+	u32 irq_en0 = VFE_0_IRQ_MASK_0_RESET_ACK;
+	u32 irq_en1 = VFE_0_IRQ_MASK_1_VIOLATION |
+		      VFE_0_IRQ_MASK_1_BUS_BDG_HALT_ACK;
+
+	vfe_reg_set(vfe, VFE_0_IRQ_MASK_0, irq_en0);
+	vfe_reg_set(vfe, VFE_0_IRQ_MASK_1, irq_en1);
+}
+
+static void vfe_set_demux_cfg(struct vfe_device *vfe, struct vfe_line *line)
+{
+	u32 val, even_cfg, odd_cfg;
+
+	writel_relaxed(VFE_0_DEMUX_CFG_PERIOD, vfe->base + VFE_0_DEMUX_CFG);
+
+	val = VFE_0_DEMUX_GAIN_0_CH0_EVEN | VFE_0_DEMUX_GAIN_0_CH0_ODD;
+	writel_relaxed(val, vfe->base + VFE_0_DEMUX_GAIN_0);
+
+	val = VFE_0_DEMUX_GAIN_1_CH1 | VFE_0_DEMUX_GAIN_1_CH2;
+	writel_relaxed(val, vfe->base + VFE_0_DEMUX_GAIN_1);
+
+	switch (line->fmt[MSM_VFE_PAD_SINK].code) {
+	case MEDIA_BUS_FMT_YUYV8_2X8:
+		even_cfg = VFE_0_DEMUX_EVEN_CFG_PATTERN_YUYV;
+		odd_cfg = VFE_0_DEMUX_ODD_CFG_PATTERN_YUYV;
+		break;
+	case MEDIA_BUS_FMT_YVYU8_2X8:
+		even_cfg = VFE_0_DEMUX_EVEN_CFG_PATTERN_YVYU;
+		odd_cfg = VFE_0_DEMUX_ODD_CFG_PATTERN_YVYU;
+		break;
+	case MEDIA_BUS_FMT_UYVY8_2X8:
+	default:
+		even_cfg = VFE_0_DEMUX_EVEN_CFG_PATTERN_UYVY;
+		odd_cfg = VFE_0_DEMUX_ODD_CFG_PATTERN_UYVY;
+		break;
+	case MEDIA_BUS_FMT_VYUY8_2X8:
+		even_cfg = VFE_0_DEMUX_EVEN_CFG_PATTERN_VYUY;
+		odd_cfg = VFE_0_DEMUX_ODD_CFG_PATTERN_VYUY;
+		break;
+	}
+
+	writel_relaxed(even_cfg, vfe->base + VFE_0_DEMUX_EVEN_CFG);
+	writel_relaxed(odd_cfg, vfe->base + VFE_0_DEMUX_ODD_CFG);
+}
+
+static inline u8 vfe_calc_interp_reso(u16 input, u16 output)
+{
+	if (input / output >= 16)
+		return 0;
+
+	if (input / output >= 8)
+		return 1;
+
+	if (input / output >= 4)
+		return 2;
+
+	return 3;
+}
+
+static void vfe_set_scale_cfg(struct vfe_device *vfe, struct vfe_line *line)
+{
+	u32 p = line->video_out.active_fmt.fmt.pix_mp.pixelformat;
+	u32 reg;
+	u16 input, output;
+	u8 interp_reso;
+	u32 phase_mult;
+
+	writel_relaxed(0x3, vfe->base + VFE_0_SCALE_ENC_Y_CFG);
+
+	input = line->fmt[MSM_VFE_PAD_SINK].width;
+	output = line->compose.width;
+	reg = (output << 16) | input;
+	writel_relaxed(reg, vfe->base + VFE_0_SCALE_ENC_Y_H_IMAGE_SIZE);
+
+	interp_reso = vfe_calc_interp_reso(input, output);
+	phase_mult = input * (1 << (13 + interp_reso)) / output;
+	reg = (interp_reso << 20) | phase_mult;
+	writel_relaxed(reg, vfe->base + VFE_0_SCALE_ENC_Y_H_PHASE);
+
+	input = line->fmt[MSM_VFE_PAD_SINK].height;
+	output = line->compose.height;
+	reg = (output << 16) | input;
+	writel_relaxed(reg, vfe->base + VFE_0_SCALE_ENC_Y_V_IMAGE_SIZE);
+
+	interp_reso = vfe_calc_interp_reso(input, output);
+	phase_mult = input * (1 << (13 + interp_reso)) / output;
+	reg = (interp_reso << 20) | phase_mult;
+	writel_relaxed(reg, vfe->base + VFE_0_SCALE_ENC_Y_V_PHASE);
+
+	writel_relaxed(0x3, vfe->base + VFE_0_SCALE_ENC_CBCR_CFG);
+
+	input = line->fmt[MSM_VFE_PAD_SINK].width;
+	output = line->compose.width / 2;
+	reg = (output << 16) | input;
+	writel_relaxed(reg, vfe->base + VFE_0_SCALE_ENC_CBCR_H_IMAGE_SIZE);
+
+	interp_reso = vfe_calc_interp_reso(input, output);
+	phase_mult = input * (1 << (13 + interp_reso)) / output;
+	reg = (interp_reso << 20) | phase_mult;
+	writel_relaxed(reg, vfe->base + VFE_0_SCALE_ENC_CBCR_H_PHASE);
+
+	input = line->fmt[MSM_VFE_PAD_SINK].height;
+	output = line->compose.height;
+	if (p == V4L2_PIX_FMT_NV12 || p == V4L2_PIX_FMT_NV21)
+		output = line->compose.height / 2;
+	reg = (output << 16) | input;
+	writel_relaxed(reg, vfe->base + VFE_0_SCALE_ENC_CBCR_V_IMAGE_SIZE);
+
+	interp_reso = vfe_calc_interp_reso(input, output);
+	phase_mult = input * (1 << (13 + interp_reso)) / output;
+	reg = (interp_reso << 20) | phase_mult;
+	writel_relaxed(reg, vfe->base + VFE_0_SCALE_ENC_CBCR_V_PHASE);
+}
+
+static void vfe_set_crop_cfg(struct vfe_device *vfe, struct vfe_line *line)
+{
+	u32 p = line->video_out.active_fmt.fmt.pix_mp.pixelformat;
+	u32 reg;
+	u16 first, last;
+
+	first = line->crop.left;
+	last = line->crop.left + line->crop.width - 1;
+	reg = (first << 16) | last;
+	writel_relaxed(reg, vfe->base + VFE_0_CROP_ENC_Y_WIDTH);
+
+	first = line->crop.top;
+	last = line->crop.top + line->crop.height - 1;
+	reg = (first << 16) | last;
+	writel_relaxed(reg, vfe->base + VFE_0_CROP_ENC_Y_HEIGHT);
+
+	first = line->crop.left / 2;
+	last = line->crop.left / 2 + line->crop.width / 2 - 1;
+	reg = (first << 16) | last;
+	writel_relaxed(reg, vfe->base + VFE_0_CROP_ENC_CBCR_WIDTH);
+
+	first = line->crop.top;
+	last = line->crop.top + line->crop.height - 1;
+	if (p == V4L2_PIX_FMT_NV12 || p == V4L2_PIX_FMT_NV21) {
+		first = line->crop.top / 2;
+		last = line->crop.top / 2 + line->crop.height / 2 - 1;
+	}
+	reg = (first << 16) | last;
+	writel_relaxed(reg, vfe->base + VFE_0_CROP_ENC_CBCR_HEIGHT);
+}
+
+static void vfe_set_clamp_cfg(struct vfe_device *vfe)
+{
+	u32 val = VFE_0_CLAMP_ENC_MAX_CFG_CH0 |
+		VFE_0_CLAMP_ENC_MAX_CFG_CH1 |
+		VFE_0_CLAMP_ENC_MAX_CFG_CH2;
+
+	writel_relaxed(val, vfe->base + VFE_0_CLAMP_ENC_MAX_CFG);
+
+	val = VFE_0_CLAMP_ENC_MIN_CFG_CH0 |
+		VFE_0_CLAMP_ENC_MIN_CFG_CH1 |
+		VFE_0_CLAMP_ENC_MIN_CFG_CH2;
+
+	writel_relaxed(val, vfe->base + VFE_0_CLAMP_ENC_MIN_CFG);
+}
+
+/*
+ * vfe_reset - Trigger reset on VFE module and wait to complete
+ * @vfe: VFE device
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int vfe_reset(struct vfe_device *vfe)
+{
+	unsigned long time;
+
+	reinit_completion(&vfe->reset_complete);
+
+	vfe_global_reset(vfe);
+
+	time = wait_for_completion_timeout(&vfe->reset_complete,
+		msecs_to_jiffies(VFE_RESET_TIMEOUT_MS));
+	if (!time) {
+		dev_err(to_device(vfe), "VFE reset timeout\n");
+		return -EIO;
+	}
+
+	return 0;
+}
+
+/*
+ * vfe_halt - Trigger halt on VFE module and wait to complete
+ * @vfe: VFE device
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int vfe_halt(struct vfe_device *vfe)
+{
+	unsigned long time;
+
+	reinit_completion(&vfe->halt_complete);
+
+	writel_relaxed(VFE_0_BUS_BDG_CMD_HALT_REQ,
+		       vfe->base + VFE_0_BUS_BDG_CMD);
+
+	time = wait_for_completion_timeout(&vfe->halt_complete,
+		msecs_to_jiffies(VFE_HALT_TIMEOUT_MS));
+	if (!time) {
+		dev_err(to_device(vfe), "VFE halt timeout\n");
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static void vfe_init_outputs(struct vfe_device *vfe)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(vfe->line); i++) {
+		struct vfe_output *output = &vfe->line[i].output;
+
+		output->state = VFE_OUTPUT_OFF;
+		output->buf[0] = NULL;
+		output->buf[1] = NULL;
+		INIT_LIST_HEAD(&output->pending_bufs);
+
+		output->wm_num = 1;
+		if (vfe->line[i].id == VFE_LINE_PIX)
+			output->wm_num = 2;
+	}
+}
+
+static void vfe_reset_output_maps(struct vfe_device *vfe)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(vfe->wm_output_map); i++)
+		vfe->wm_output_map[i] = VFE_LINE_NONE;
+}
+
+static void vfe_set_qos(struct vfe_device *vfe)
+{
+	u32 val = VFE_0_BUS_BDG_QOS_CFG_0_CFG;
+	u32 val7 = VFE_0_BUS_BDG_QOS_CFG_7_CFG;
+
+	writel_relaxed(val, vfe->base + VFE_0_BUS_BDG_QOS_CFG_0);
+	writel_relaxed(val, vfe->base + VFE_0_BUS_BDG_QOS_CFG_1);
+	writel_relaxed(val, vfe->base + VFE_0_BUS_BDG_QOS_CFG_2);
+	writel_relaxed(val, vfe->base + VFE_0_BUS_BDG_QOS_CFG_3);
+	writel_relaxed(val, vfe->base + VFE_0_BUS_BDG_QOS_CFG_4);
+	writel_relaxed(val, vfe->base + VFE_0_BUS_BDG_QOS_CFG_5);
+	writel_relaxed(val, vfe->base + VFE_0_BUS_BDG_QOS_CFG_6);
+	writel_relaxed(val7, vfe->base + VFE_0_BUS_BDG_QOS_CFG_7);
+}
+
+static void vfe_set_cgc_override(struct vfe_device *vfe, u8 wm, u8 enable)
+{
+	u32 val = VFE_0_CGC_OVERRIDE_1_IMAGE_Mx_CGC_OVERRIDE(wm);
+
+	if (enable)
+		vfe_reg_set(vfe, VFE_0_CGC_OVERRIDE_1, val);
+	else
+		vfe_reg_clr(vfe, VFE_0_CGC_OVERRIDE_1, val);
+
+	wmb();
+}
+
+static void vfe_set_module_cfg(struct vfe_device *vfe, u8 enable)
+{
+	u32 val = VFE_0_MODULE_CFG_DEMUX |
+		  VFE_0_MODULE_CFG_CHROMA_UPSAMPLE |
+		  VFE_0_MODULE_CFG_SCALE_ENC |
+		  VFE_0_MODULE_CFG_CROP_ENC;
+
+	if (enable)
+		writel_relaxed(val, vfe->base + VFE_0_MODULE_CFG);
+	else
+		writel_relaxed(0x0, vfe->base + VFE_0_MODULE_CFG);
+}
+
+static void vfe_set_camif_cfg(struct vfe_device *vfe, struct vfe_line *line)
+{
+	u32 val;
+
+	switch (line->fmt[MSM_VFE_PAD_SINK].code) {
+	case MEDIA_BUS_FMT_YUYV8_2X8:
+		val = VFE_0_CORE_CFG_PIXEL_PATTERN_YCBYCR;
+		break;
+	case MEDIA_BUS_FMT_YVYU8_2X8:
+		val = VFE_0_CORE_CFG_PIXEL_PATTERN_YCRYCB;
+		break;
+	case MEDIA_BUS_FMT_UYVY8_2X8:
+	default:
+		val = VFE_0_CORE_CFG_PIXEL_PATTERN_CBYCRY;
+		break;
+	case MEDIA_BUS_FMT_VYUY8_2X8:
+		val = VFE_0_CORE_CFG_PIXEL_PATTERN_CRYCBY;
+		break;
+	}
+
+	writel_relaxed(val, vfe->base + VFE_0_CORE_CFG);
+
+	val = line->fmt[MSM_VFE_PAD_SINK].width * 2;
+	val |= line->fmt[MSM_VFE_PAD_SINK].height << 16;
+	writel_relaxed(val, vfe->base + VFE_0_CAMIF_FRAME_CFG);
+
+	val = line->fmt[MSM_VFE_PAD_SINK].width * 2 - 1;
+	writel_relaxed(val, vfe->base + VFE_0_CAMIF_WINDOW_WIDTH_CFG);
+
+	val = line->fmt[MSM_VFE_PAD_SINK].height - 1;
+	writel_relaxed(val, vfe->base + VFE_0_CAMIF_WINDOW_HEIGHT_CFG);
+
+	val = 0xffffffff;
+	writel_relaxed(val, vfe->base + VFE_0_CAMIF_SUBSAMPLE_CFG_0);
+
+	val = 0xffffffff;
+	writel_relaxed(val, vfe->base + VFE_0_CAMIF_IRQ_SUBSAMPLE_PATTERN);
+
+	val = VFE_0_RDI_CFG_x_MIPI_EN_BITS;
+	vfe_reg_set(vfe, VFE_0_RDI_CFG_x(0), val);
+
+	val = VFE_0_CAMIF_CFG_VFE_OUTPUT_EN;
+	writel_relaxed(val, vfe->base + VFE_0_CAMIF_CFG);
+}
+
+static void vfe_set_camif_cmd(struct vfe_device *vfe, u32 cmd)
+{
+	writel_relaxed(VFE_0_CAMIF_CMD_CLEAR_CAMIF_STATUS,
+		       vfe->base + VFE_0_CAMIF_CMD);
+
+	writel_relaxed(cmd, vfe->base + VFE_0_CAMIF_CMD);
+}
+
+static int vfe_camif_wait_for_stop(struct vfe_device *vfe)
+{
+	u32 val;
+	int ret;
+
+	ret = readl_poll_timeout(vfe->base + VFE_0_CAMIF_STATUS,
+				 val,
+				 (val & VFE_0_CAMIF_STATUS_HALT),
+				 CAMIF_TIMEOUT_SLEEP_US,
+				 CAMIF_TIMEOUT_ALL_US);
+	if (ret < 0)
+		dev_err(to_device(vfe), "%s: camif stop timeout\n", __func__);
+
+	return ret;
+}
+
+static void vfe_output_init_addrs(struct vfe_device *vfe,
+				  struct vfe_output *output, u8 sync)
+{
+	u32 ping_addr;
+	u32 pong_addr;
+	unsigned int i;
+
+	output->active_buf = 0;
+
+	for (i = 0; i < output->wm_num; i++) {
+		if (output->buf[0])
+			ping_addr = output->buf[0]->addr[i];
+		else
+			ping_addr = 0;
+
+		if (output->buf[1])
+			pong_addr = output->buf[1]->addr[i];
+		else
+			pong_addr = ping_addr;
+
+		vfe_wm_set_ping_addr(vfe, output->wm_idx[i], ping_addr);
+		vfe_wm_set_pong_addr(vfe, output->wm_idx[i], pong_addr);
+		if (sync)
+			vfe_bus_reload_wm(vfe, output->wm_idx[i]);
+	}
+}
+
+static void vfe_output_update_ping_addr(struct vfe_device *vfe,
+					struct vfe_output *output, u8 sync)
+{
+	u32 addr;
+	unsigned int i;
+
+	for (i = 0; i < output->wm_num; i++) {
+		if (output->buf[0])
+			addr = output->buf[0]->addr[i];
+		else
+			addr = 0;
+
+		vfe_wm_set_ping_addr(vfe, output->wm_idx[i], addr);
+		if (sync)
+			vfe_bus_reload_wm(vfe, output->wm_idx[i]);
+	}
+}
+
+static void vfe_output_update_pong_addr(struct vfe_device *vfe,
+					struct vfe_output *output, u8 sync)
+{
+	u32 addr;
+	unsigned int i;
+
+	for (i = 0; i < output->wm_num; i++) {
+		if (output->buf[1])
+			addr = output->buf[1]->addr[i];
+		else
+			addr = 0;
+
+		vfe_wm_set_pong_addr(vfe, output->wm_idx[i], addr);
+		if (sync)
+			vfe_bus_reload_wm(vfe, output->wm_idx[i]);
+	}
+
+}
+
+static int vfe_reserve_wm(struct vfe_device *vfe, enum vfe_line_id line_id)
+{
+	int ret = -EBUSY;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(vfe->wm_output_map); i++) {
+		if (vfe->wm_output_map[i] == VFE_LINE_NONE) {
+			vfe->wm_output_map[i] = line_id;
+			ret = i;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static int vfe_release_wm(struct vfe_device *vfe, u8 wm)
+{
+	if (wm >= ARRAY_SIZE(vfe->wm_output_map))
+		return -EINVAL;
+
+	vfe->wm_output_map[wm] = VFE_LINE_NONE;
+
+	return 0;
+}
+
+static void vfe_output_frame_drop(struct vfe_device *vfe,
+				  struct vfe_output *output,
+				  u32 drop_pattern)
+{
+	u8 drop_period;
+	unsigned int i;
+
+	/* We need to toggle update period to be valid on next frame */
+	output->drop_update_idx++;
+	output->drop_update_idx %= VFE_FRAME_DROP_UPDATES;
+	drop_period = VFE_FRAME_DROP_VAL + output->drop_update_idx;
+
+	for (i = 0; i < output->wm_num; i++) {
+		vfe_wm_set_framedrop_period(vfe, output->wm_idx[i],
+					    drop_period);
+		vfe_wm_set_framedrop_pattern(vfe, output->wm_idx[i],
+					     drop_pattern);
+	}
+	vfe_reg_update(vfe, container_of(output, struct vfe_line, output)->id);
+}
+
+static struct camss_buffer *vfe_buf_get_pending(struct vfe_output *output)
+{
+	struct camss_buffer *buffer = NULL;
+
+	if (!list_empty(&output->pending_bufs)) {
+		buffer = list_first_entry(&output->pending_bufs,
+					  struct camss_buffer,
+					  queue);
+		list_del(&buffer->queue);
+	}
+
+	return buffer;
+}
+
+/*
+ * vfe_buf_add_pending - Add output buffer to list of pending
+ * @output: VFE output
+ * @buffer: Video buffer
+ */
+static void vfe_buf_add_pending(struct vfe_output *output,
+				struct camss_buffer *buffer)
+{
+	INIT_LIST_HEAD(&buffer->queue);
+	list_add_tail(&buffer->queue, &output->pending_bufs);
+}
+
+/*
+ * vfe_buf_flush_pending - Flush all pending buffers.
+ * @output: VFE output
+ * @state: vb2 buffer state
+ */
+static void vfe_buf_flush_pending(struct vfe_output *output,
+				  enum vb2_buffer_state state)
+{
+	struct camss_buffer *buf;
+	struct camss_buffer *t;
+
+	list_for_each_entry_safe(buf, t, &output->pending_bufs, queue) {
+		vb2_buffer_done(&buf->vb.vb2_buf, state);
+		list_del(&buf->queue);
+	}
+}
+
+static void vfe_buf_update_wm_on_next(struct vfe_device *vfe,
+				      struct vfe_output *output)
+{
+	switch (output->state) {
+	case VFE_OUTPUT_CONTINUOUS:
+		vfe_output_frame_drop(vfe, output, 3);
+		break;
+	case VFE_OUTPUT_SINGLE:
+	default:
+		dev_err_ratelimited(to_device(vfe),
+				    "Next buf in wrong state! %d\n",
+				    output->state);
+		break;
+	}
+}
+
+static void vfe_buf_update_wm_on_last(struct vfe_device *vfe,
+				      struct vfe_output *output)
+{
+	switch (output->state) {
+	case VFE_OUTPUT_CONTINUOUS:
+		output->state = VFE_OUTPUT_SINGLE;
+		vfe_output_frame_drop(vfe, output, 1);
+		break;
+	case VFE_OUTPUT_SINGLE:
+		output->state = VFE_OUTPUT_STOPPING;
+		vfe_output_frame_drop(vfe, output, 0);
+		break;
+	default:
+		dev_err_ratelimited(to_device(vfe),
+				    "Last buff in wrong state! %d\n",
+				    output->state);
+		break;
+	}
+}
+
+static void vfe_buf_update_wm_on_new(struct vfe_device *vfe,
+				     struct vfe_output *output,
+				     struct camss_buffer *new_buf)
+{
+	int inactive_idx;
+
+	switch (output->state) {
+	case VFE_OUTPUT_SINGLE:
+		inactive_idx = !output->active_buf;
+
+		if (!output->buf[inactive_idx]) {
+			output->buf[inactive_idx] = new_buf;
+
+			if (inactive_idx)
+				vfe_output_update_pong_addr(vfe, output, 0);
+			else
+				vfe_output_update_ping_addr(vfe, output, 0);
+
+			vfe_output_frame_drop(vfe, output, 3);
+			output->state = VFE_OUTPUT_CONTINUOUS;
+		} else {
+			vfe_buf_add_pending(output, new_buf);
+			dev_err_ratelimited(to_device(vfe),
+					    "Inactive buffer is busy\n");
+		}
+		break;
+
+	case VFE_OUTPUT_IDLE:
+		if (!output->buf[0]) {
+			output->buf[0] = new_buf;
+
+			vfe_output_init_addrs(vfe, output, 1);
+
+			vfe_output_frame_drop(vfe, output, 1);
+			output->state = VFE_OUTPUT_SINGLE;
+		} else {
+			vfe_buf_add_pending(output, new_buf);
+			dev_err_ratelimited(to_device(vfe),
+					    "Output idle with buffer set!\n");
+		}
+		break;
+
+	case VFE_OUTPUT_CONTINUOUS:
+	default:
+		vfe_buf_add_pending(output, new_buf);
+		break;
+	}
+}
+
+static int vfe_get_output(struct vfe_line *line)
+{
+	struct vfe_device *vfe = to_vfe(line);
+	struct vfe_output *output;
+	unsigned long flags;
+	int i;
+	int wm_idx;
+
+	spin_lock_irqsave(&vfe->output_lock, flags);
+
+	output = &line->output;
+	if (output->state != VFE_OUTPUT_OFF) {
+		dev_err(to_device(vfe), "Output is running\n");
+		goto error;
+	}
+	output->state = VFE_OUTPUT_RESERVED;
+
+	output->active_buf = 0;
+
+	for (i = 0; i < output->wm_num; i++) {
+		wm_idx = vfe_reserve_wm(vfe, line->id);
+		if (wm_idx < 0) {
+			dev_err(to_device(vfe), "Can not reserve wm\n");
+			goto error_get_wm;
+		}
+		output->wm_idx[i] = wm_idx;
+	}
+
+	output->drop_update_idx = 0;
+
+	spin_unlock_irqrestore(&vfe->output_lock, flags);
+
+	return 0;
+
+error_get_wm:
+	for (i--; i >= 0; i--)
+		vfe_release_wm(vfe, output->wm_idx[i]);
+	output->state = VFE_OUTPUT_OFF;
+error:
+	spin_unlock_irqrestore(&vfe->output_lock, flags);
+
+	return -EINVAL;
+}
+
+static int vfe_put_output(struct vfe_line *line)
+{
+	struct vfe_device *vfe = to_vfe(line);
+	struct vfe_output *output = &line->output;
+	unsigned long flags;
+	unsigned int i;
+
+	spin_lock_irqsave(&vfe->output_lock, flags);
+
+	for (i = 0; i < output->wm_num; i++)
+		vfe_release_wm(vfe, output->wm_idx[i]);
+
+	output->state = VFE_OUTPUT_OFF;
+
+	spin_unlock_irqrestore(&vfe->output_lock, flags);
+	return 0;
+}
+
+static int vfe_enable_output(struct vfe_line *line)
+{
+	struct vfe_device *vfe = to_vfe(line);
+	struct vfe_output *output = &line->output;
+	unsigned long flags;
+	unsigned int i;
+	u16 ub_size;
+
+	switch (vfe->id) {
+	case 0:
+		ub_size = MSM_VFE_VFE0_UB_SIZE_RDI;
+		break;
+	case 1:
+		ub_size = MSM_VFE_VFE1_UB_SIZE_RDI;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	spin_lock_irqsave(&vfe->output_lock, flags);
+
+	vfe->reg_update &= ~VFE_0_REG_UPDATE_line_n(line->id);
+
+	if (output->state != VFE_OUTPUT_RESERVED) {
+		dev_err(to_device(vfe), "Output is not in reserved state %d\n",
+			output->state);
+		spin_unlock_irqrestore(&vfe->output_lock, flags);
+		return -EINVAL;
+	}
+	output->state = VFE_OUTPUT_IDLE;
+
+	output->buf[0] = vfe_buf_get_pending(output);
+	output->buf[1] = vfe_buf_get_pending(output);
+
+	if (!output->buf[0] && output->buf[1]) {
+		output->buf[0] = output->buf[1];
+		output->buf[1] = NULL;
+	}
+
+	if (output->buf[0])
+		output->state = VFE_OUTPUT_SINGLE;
+
+	if (output->buf[1])
+		output->state = VFE_OUTPUT_CONTINUOUS;
+
+	switch (output->state) {
+	case VFE_OUTPUT_SINGLE:
+		vfe_output_frame_drop(vfe, output, 1);
+		break;
+	case VFE_OUTPUT_CONTINUOUS:
+		vfe_output_frame_drop(vfe, output, 3);
+		break;
+	default:
+		vfe_output_frame_drop(vfe, output, 0);
+		break;
+	}
+
+	output->sequence = 0;
+	output->wait_sof = 0;
+	output->wait_reg_update = 0;
+	reinit_completion(&output->sof);
+	reinit_completion(&output->reg_update);
+
+	vfe_output_init_addrs(vfe, output, 0);
+
+	if (line->id != VFE_LINE_PIX) {
+		vfe_set_cgc_override(vfe, output->wm_idx[0], 1);
+		vfe_enable_irq_wm_line(vfe, output->wm_idx[0], line->id, 1);
+		vfe_bus_connect_wm_to_rdi(vfe, output->wm_idx[0], line->id);
+		vfe_wm_set_subsample(vfe, output->wm_idx[0]);
+		vfe_set_rdi_cid(vfe, line->id, 0);
+		vfe_wm_set_ub_cfg(vfe, output->wm_idx[0],
+				  (ub_size + 1) * output->wm_idx[0], ub_size);
+		vfe_wm_frame_based(vfe, output->wm_idx[0], 1);
+		vfe_wm_enable(vfe, output->wm_idx[0], 1);
+		vfe_bus_reload_wm(vfe, output->wm_idx[0]);
+	} else {
+		ub_size /= output->wm_num;
+		for (i = 0; i < output->wm_num; i++) {
+			vfe_set_cgc_override(vfe, output->wm_idx[i], 1);
+			vfe_wm_set_subsample(vfe, output->wm_idx[i]);
+			vfe_wm_set_ub_cfg(vfe, output->wm_idx[i],
+					  (ub_size + 1) * output->wm_idx[i],
+					  ub_size);
+			vfe_wm_line_based(vfe, output->wm_idx[i],
+					&line->video_out.active_fmt.fmt.pix_mp,
+					i, 1);
+			vfe_wm_enable(vfe, output->wm_idx[i], 1);
+			vfe_bus_reload_wm(vfe, output->wm_idx[i]);
+		}
+		vfe_enable_irq_pix_line(vfe, 0, line->id, 1);
+		vfe_set_module_cfg(vfe, 1);
+		vfe_set_camif_cfg(vfe, line);
+		vfe_set_xbar_cfg(vfe, output, 1);
+		vfe_set_demux_cfg(vfe, line);
+		vfe_set_scale_cfg(vfe, line);
+		vfe_set_crop_cfg(vfe, line);
+		vfe_set_clamp_cfg(vfe);
+		vfe_set_camif_cmd(vfe, VFE_0_CAMIF_CMD_ENABLE_FRAME_BOUNDARY);
+	}
+
+	vfe_reg_update(vfe, line->id);
+
+	spin_unlock_irqrestore(&vfe->output_lock, flags);
+
+	return 0;
+}
+
+static int vfe_disable_output(struct vfe_line *line)
+{
+	struct vfe_device *vfe = to_vfe(line);
+	struct vfe_output *output = &line->output;
+	unsigned long flags;
+	unsigned long time;
+	unsigned int i;
+
+	spin_lock_irqsave(&vfe->output_lock, flags);
+
+	output->wait_sof = 1;
+	spin_unlock_irqrestore(&vfe->output_lock, flags);
+
+	time = wait_for_completion_timeout(&output->sof,
+					   msecs_to_jiffies(VFE_NEXT_SOF_MS));
+	if (!time)
+		dev_err(to_device(vfe), "VFE sof timeout\n");
+
+	spin_lock_irqsave(&vfe->output_lock, flags);
+	for (i = 0; i < output->wm_num; i++)
+		vfe_wm_enable(vfe, output->wm_idx[i], 0);
+
+	vfe_reg_update(vfe, line->id);
+	output->wait_reg_update = 1;
+	spin_unlock_irqrestore(&vfe->output_lock, flags);
+
+	time = wait_for_completion_timeout(&output->reg_update,
+					   msecs_to_jiffies(VFE_NEXT_SOF_MS));
+	if (!time)
+		dev_err(to_device(vfe), "VFE reg update timeout\n");
+
+	spin_lock_irqsave(&vfe->output_lock, flags);
+
+	if (line->id != VFE_LINE_PIX) {
+		vfe_wm_frame_based(vfe, output->wm_idx[0], 0);
+		vfe_bus_disconnect_wm_from_rdi(vfe, output->wm_idx[0], line->id);
+		vfe_enable_irq_wm_line(vfe, output->wm_idx[0], line->id, 0);
+		vfe_set_cgc_override(vfe, output->wm_idx[0], 0);
+		spin_unlock_irqrestore(&vfe->output_lock, flags);
+	} else {
+		for (i = 0; i < output->wm_num; i++) {
+			vfe_wm_line_based(vfe, output->wm_idx[i], NULL, i, 0);
+			vfe_set_cgc_override(vfe, output->wm_idx[i], 0);
+		}
+
+		vfe_enable_irq_pix_line(vfe, 0, line->id, 0);
+		vfe_set_module_cfg(vfe, 0);
+		vfe_set_xbar_cfg(vfe, output, 0);
+
+		vfe_set_camif_cmd(vfe, VFE_0_CAMIF_CMD_DISABLE_FRAME_BOUNDARY);
+		spin_unlock_irqrestore(&vfe->output_lock, flags);
+
+		vfe_camif_wait_for_stop(vfe);
+	}
+
+	return 0;
+}
+
+/*
+ * vfe_enable - Enable streaming on VFE line
+ * @line: VFE line
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int vfe_enable(struct vfe_line *line)
+{
+	struct vfe_device *vfe = to_vfe(line);
+	int ret;
+
+	mutex_lock(&vfe->stream_lock);
+
+	if (!vfe->stream_count) {
+		vfe_enable_irq_common(vfe);
+
+		vfe_bus_enable_wr_if(vfe, 1);
+
+		vfe_set_qos(vfe);
+	}
+
+	vfe->stream_count++;
+
+	mutex_unlock(&vfe->stream_lock);
+
+	ret = vfe_get_output(line);
+	if (ret < 0)
+		goto error_get_output;
+
+	ret = vfe_enable_output(line);
+	if (ret < 0)
+		goto error_enable_output;
+
+	vfe->was_streaming = 1;
+
+	return 0;
+
+
+error_enable_output:
+	vfe_put_output(line);
+
+error_get_output:
+	mutex_lock(&vfe->stream_lock);
+
+	if (vfe->stream_count == 1)
+		vfe_bus_enable_wr_if(vfe, 0);
+
+	vfe->stream_count--;
+
+	mutex_unlock(&vfe->stream_lock);
+
+	return ret;
+}
+
+/*
+ * vfe_disable - Disable streaming on VFE line
+ * @line: VFE line
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int vfe_disable(struct vfe_line *line)
+{
+	struct vfe_device *vfe = to_vfe(line);
+
+	vfe_disable_output(line);
+
+	vfe_put_output(line);
+
+	mutex_lock(&vfe->stream_lock);
+
+	if (vfe->stream_count == 1)
+		vfe_bus_enable_wr_if(vfe, 0);
+
+	vfe->stream_count--;
+
+	mutex_unlock(&vfe->stream_lock);
+
+	return 0;
+}
+
+/*
+ * vfe_isr_sof - Process start of frame interrupt
+ * @vfe: VFE Device
+ * @line_id: VFE line
+ */
+static void vfe_isr_sof(struct vfe_device *vfe, enum vfe_line_id line_id)
+{
+	struct vfe_output *output;
+	unsigned long flags;
+
+	spin_lock_irqsave(&vfe->output_lock, flags);
+	output = &vfe->line[line_id].output;
+	if (output->wait_sof) {
+		output->wait_sof = 0;
+		complete(&output->sof);
+	}
+	spin_unlock_irqrestore(&vfe->output_lock, flags);
+}
+
+/*
+ * vfe_isr_reg_update - Process reg update interrupt
+ * @vfe: VFE Device
+ * @line_id: VFE line
+ */
+static void vfe_isr_reg_update(struct vfe_device *vfe, enum vfe_line_id line_id)
+{
+	struct vfe_output *output;
+	unsigned long flags;
+
+	spin_lock_irqsave(&vfe->output_lock, flags);
+	vfe->reg_update &= ~VFE_0_REG_UPDATE_line_n(line_id);
+
+	output = &vfe->line[line_id].output;
+
+	if (output->wait_reg_update) {
+		output->wait_reg_update = 0;
+		complete(&output->reg_update);
+		spin_unlock_irqrestore(&vfe->output_lock, flags);
+		return;
+	}
+
+	if (output->state == VFE_OUTPUT_STOPPING) {
+		/* Release last buffer when hw is idle */
+		if (output->last_buffer) {
+			vb2_buffer_done(&output->last_buffer->vb.vb2_buf,
+					VB2_BUF_STATE_DONE);
+			output->last_buffer = NULL;
+		}
+		output->state = VFE_OUTPUT_IDLE;
+
+		/* Buffers received in stopping state are queued in */
+		/* dma pending queue, start next capture here */
+
+		output->buf[0] = vfe_buf_get_pending(output);
+		output->buf[1] = vfe_buf_get_pending(output);
+
+		if (!output->buf[0] && output->buf[1]) {
+			output->buf[0] = output->buf[1];
+			output->buf[1] = NULL;
+		}
+
+		if (output->buf[0])
+			output->state = VFE_OUTPUT_SINGLE;
+
+		if (output->buf[1])
+			output->state = VFE_OUTPUT_CONTINUOUS;
+
+		switch (output->state) {
+		case VFE_OUTPUT_SINGLE:
+			vfe_output_frame_drop(vfe, output, 2);
+			break;
+		case VFE_OUTPUT_CONTINUOUS:
+			vfe_output_frame_drop(vfe, output, 3);
+			break;
+		default:
+			vfe_output_frame_drop(vfe, output, 0);
+			break;
+		}
+
+		vfe_output_init_addrs(vfe, output, 1);
+	}
+
+	spin_unlock_irqrestore(&vfe->output_lock, flags);
+}
+
+/*
+ * vfe_isr_wm_done - Process write master done interrupt
+ * @vfe: VFE Device
+ * @wm: Write master id
+ */
+static void vfe_isr_wm_done(struct vfe_device *vfe, u8 wm)
+{
+	struct camss_buffer *ready_buf;
+	struct vfe_output *output;
+	dma_addr_t *new_addr;
+	unsigned long flags;
+	u32 active_index;
+	u64 ts = ktime_get_ns();
+	unsigned int i;
+
+	active_index = vfe_wm_get_ping_pong_status(vfe, wm);
+
+	spin_lock_irqsave(&vfe->output_lock, flags);
+
+	if (vfe->wm_output_map[wm] == VFE_LINE_NONE) {
+		dev_err_ratelimited(to_device(vfe),
+				    "Received wm done for unmapped index\n");
+		goto out_unlock;
+	}
+	output = &vfe->line[vfe->wm_output_map[wm]].output;
+
+	if (output->active_buf == active_index) {
+		dev_err_ratelimited(to_device(vfe),
+				    "Active buffer mismatch!\n");
+		goto out_unlock;
+	}
+	output->active_buf = active_index;
+
+	ready_buf = output->buf[!active_index];
+	if (!ready_buf) {
+		dev_err_ratelimited(to_device(vfe),
+				    "Missing ready buf %d %d!\n",
+				    !active_index, output->state);
+		goto out_unlock;
+	}
+
+	ready_buf->vb.vb2_buf.timestamp = ts;
+	ready_buf->vb.sequence = output->sequence++;
+
+	/* Get next buffer */
+	output->buf[!active_index] = vfe_buf_get_pending(output);
+	if (!output->buf[!active_index]) {
+		/* No next buffer - set same address */
+		new_addr = ready_buf->addr;
+		vfe_buf_update_wm_on_last(vfe, output);
+	} else {
+		new_addr = output->buf[!active_index]->addr;
+		vfe_buf_update_wm_on_next(vfe, output);
+	}
+
+	if (active_index)
+		for (i = 0; i < output->wm_num; i++)
+			vfe_wm_set_ping_addr(vfe, output->wm_idx[i],
+					     new_addr[i]);
+	else
+		for (i = 0; i < output->wm_num; i++)
+			vfe_wm_set_pong_addr(vfe, output->wm_idx[i],
+					     new_addr[i]);
+
+	spin_unlock_irqrestore(&vfe->output_lock, flags);
+
+	if (output->state == VFE_OUTPUT_STOPPING)
+		output->last_buffer = ready_buf;
+	else
+		vb2_buffer_done(&ready_buf->vb.vb2_buf, VB2_BUF_STATE_DONE);
+
+	return;
+
+out_unlock:
+	spin_unlock_irqrestore(&vfe->output_lock, flags);
+}
+
+/*
+ * vfe_isr_wm_done - Process composite image done interrupt
+ * @vfe: VFE Device
+ * @comp: Composite image id
+ */
+static void vfe_isr_comp_done(struct vfe_device *vfe, u8 comp)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(vfe->wm_output_map); i++)
+		if (vfe->wm_output_map[i] == VFE_LINE_PIX) {
+			vfe_isr_wm_done(vfe, i);
+			break;
+		}
+}
+
+/*
+ * vfe_isr - ISPIF module interrupt handler
+ * @irq: Interrupt line
+ * @dev: VFE device
+ *
+ * Return IRQ_HANDLED on success
+ */
+static irqreturn_t vfe_isr(int irq, void *dev)
+{
+	struct vfe_device *vfe = dev;
+	u32 value0, value1;
+	u32 violation;
+	int i, j;
+
+	value0 = readl_relaxed(vfe->base + VFE_0_IRQ_STATUS_0);
+	value1 = readl_relaxed(vfe->base + VFE_0_IRQ_STATUS_1);
+
+	writel_relaxed(value0, vfe->base + VFE_0_IRQ_CLEAR_0);
+	writel_relaxed(value1, vfe->base + VFE_0_IRQ_CLEAR_1);
+
+	wmb();
+	writel_relaxed(VFE_0_IRQ_CMD_GLOBAL_CLEAR, vfe->base + VFE_0_IRQ_CMD);
+
+	if (value0 & VFE_0_IRQ_STATUS_0_RESET_ACK)
+		complete(&vfe->reset_complete);
+
+	if (value1 & VFE_0_IRQ_STATUS_1_VIOLATION) {
+		violation = readl_relaxed(vfe->base + VFE_0_VIOLATION_STATUS);
+		dev_err_ratelimited(to_device(vfe),
+				    "VFE: violation = 0x%08x\n", violation);
+	}
+
+	if (value1 & VFE_0_IRQ_STATUS_1_BUS_BDG_HALT_ACK) {
+		complete(&vfe->halt_complete);
+		writel_relaxed(0x0, vfe->base + VFE_0_BUS_BDG_CMD);
+	}
+
+	for (i = VFE_LINE_RDI0; i <= VFE_LINE_PIX; i++)
+		if (value0 & VFE_0_IRQ_STATUS_0_line_n_REG_UPDATE(i))
+			vfe_isr_reg_update(vfe, i);
+
+	if (value0 & VFE_0_IRQ_STATUS_0_CAMIF_SOF)
+		vfe_isr_sof(vfe, VFE_LINE_PIX);
+
+	for (i = VFE_LINE_RDI0; i <= VFE_LINE_RDI2; i++)
+		if (value1 & VFE_0_IRQ_STATUS_1_RDIn_SOF(i))
+			vfe_isr_sof(vfe, i);
+
+	for (i = 0; i < MSM_VFE_COMPOSITE_IRQ_NUM; i++)
+		if (value0 & VFE_0_IRQ_STATUS_0_IMAGE_COMPOSITE_DONE_n(i)) {
+			vfe_isr_comp_done(vfe, i);
+			for (j = 0; j < ARRAY_SIZE(vfe->wm_output_map); j++)
+				if (vfe->wm_output_map[j] == VFE_LINE_PIX)
+					value0 &= ~VFE_0_IRQ_MASK_0_IMAGE_MASTER_n_PING_PONG(j);
+		}
+
+	for (i = 0; i < MSM_VFE_IMAGE_MASTERS_NUM; i++)
+		if (value0 & VFE_0_IRQ_STATUS_0_IMAGE_MASTER_n_PING_PONG(i))
+			vfe_isr_wm_done(vfe, i);
+
+	return IRQ_HANDLED;
+}
+
+/*
+ * vfe_set_clock_rates - Calculate and set clock rates on VFE module
+ * @vfe: VFE device
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int vfe_set_clock_rates(struct vfe_device *vfe)
+{
+	struct device *dev = to_device(vfe);
+	u32 pixel_clock[MSM_VFE_LINE_NUM];
+	int i, j;
+	int ret;
+
+	for (i = VFE_LINE_RDI0; i <= VFE_LINE_PIX; i++) {
+		ret = camss_get_pixel_clock(&vfe->line[i].subdev.entity,
+					    &pixel_clock[i]);
+		if (ret)
+			pixel_clock[i] = 0;
+	}
+
+	for (i = 0; i < vfe->nclocks; i++) {
+		struct camss_clock *clock = &vfe->clock[i];
+
+		if (!strcmp(clock->name, "camss_vfe_vfe")) {
+			u64 min_rate = 0;
+			long rate;
+
+			for (j = VFE_LINE_RDI0; j <= VFE_LINE_PIX; j++) {
+				u32 tmp;
+				u8 bpp;
+
+				if (j == VFE_LINE_PIX) {
+					tmp = pixel_clock[j];
+				} else {
+					bpp = vfe_get_bpp(vfe->line[j].
+						fmt[MSM_VFE_PAD_SINK].code);
+					tmp = pixel_clock[j] * bpp / 64;
+				}
+
+				if (min_rate < tmp)
+					min_rate = tmp;
+			}
+
+			camss_add_clock_margin(&min_rate);
+
+			for (j = 0; j < clock->nfreqs; j++)
+				if (min_rate < clock->freq[j])
+					break;
+
+			if (j == clock->nfreqs) {
+				dev_err(dev,
+					"Pixel clock is too high for VFE");
+				return -EINVAL;
+			}
+
+			/* if sensor pixel clock is not available */
+			/* set highest possible VFE clock rate */
+			if (min_rate == 0)
+				j = clock->nfreqs - 1;
+
+			rate = clk_round_rate(clock->clk, clock->freq[j]);
+			if (rate < 0) {
+				dev_err(dev, "clk round rate failed: %ld\n",
+					rate);
+				return -EINVAL;
+			}
+
+			ret = clk_set_rate(clock->clk, rate);
+			if (ret < 0) {
+				dev_err(dev, "clk set rate failed: %d\n", ret);
+				return ret;
+			}
+		}
+	}
+
+	return 0;
+}
+
+/*
+ * vfe_check_clock_rates - Check current clock rates on VFE module
+ * @vfe: VFE device
+ *
+ * Return 0 if current clock rates are suitable for a new pipeline
+ * or a negative error code otherwise
+ */
+static int vfe_check_clock_rates(struct vfe_device *vfe)
+{
+	u32 pixel_clock[MSM_VFE_LINE_NUM];
+	int i, j;
+	int ret;
+
+	for (i = VFE_LINE_RDI0; i <= VFE_LINE_PIX; i++) {
+		ret = camss_get_pixel_clock(&vfe->line[i].subdev.entity,
+					    &pixel_clock[i]);
+		if (ret)
+			pixel_clock[i] = 0;
+	}
+
+	for (i = 0; i < vfe->nclocks; i++) {
+		struct camss_clock *clock = &vfe->clock[i];
+
+		if (!strcmp(clock->name, "camss_vfe_vfe")) {
+			u64 min_rate = 0;
+			unsigned long rate;
+
+			for (j = VFE_LINE_RDI0; j <= VFE_LINE_PIX; j++) {
+				u32 tmp;
+				u8 bpp;
+
+				if (j == VFE_LINE_PIX) {
+					tmp = pixel_clock[j];
+				} else {
+					bpp = vfe_get_bpp(vfe->line[j].
+						fmt[MSM_VFE_PAD_SINK].code);
+					tmp = pixel_clock[j] * bpp / 64;
+				}
+
+				if (min_rate < tmp)
+					min_rate = tmp;
+			}
+
+			camss_add_clock_margin(&min_rate);
+
+			rate = clk_get_rate(clock->clk);
+			if (rate < min_rate)
+				return -EBUSY;
+		}
+	}
+
+	return 0;
+}
+
+/*
+ * vfe_get - Power up and reset VFE module
+ * @vfe: VFE Device
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int vfe_get(struct vfe_device *vfe)
+{
+	int ret;
+
+	mutex_lock(&vfe->power_lock);
+
+	if (vfe->power_count == 0) {
+		ret = vfe_set_clock_rates(vfe);
+		if (ret < 0)
+			goto error_clocks;
+
+		ret = camss_enable_clocks(vfe->nclocks, vfe->clock,
+					  to_device(vfe));
+		if (ret < 0)
+			goto error_clocks;
+
+		ret = vfe_reset(vfe);
+		if (ret < 0)
+			goto error_reset;
+
+		vfe_reset_output_maps(vfe);
+
+		vfe_init_outputs(vfe);
+	} else {
+		ret = vfe_check_clock_rates(vfe);
+		if (ret < 0)
+			goto error_clocks;
+	}
+	vfe->power_count++;
+
+	mutex_unlock(&vfe->power_lock);
+
+	return 0;
+
+error_reset:
+	camss_disable_clocks(vfe->nclocks, vfe->clock);
+
+error_clocks:
+	mutex_unlock(&vfe->power_lock);
+
+	return ret;
+}
+
+/*
+ * vfe_put - Power down VFE module
+ * @vfe: VFE Device
+ */
+static void vfe_put(struct vfe_device *vfe)
+{
+	mutex_lock(&vfe->power_lock);
+
+	if (vfe->power_count == 0) {
+		dev_err(to_device(vfe), "vfe power off on power_count == 0\n");
+		goto exit;
+	} else if (vfe->power_count == 1) {
+		if (vfe->was_streaming) {
+			vfe->was_streaming = 0;
+			vfe_halt(vfe);
+		}
+		camss_disable_clocks(vfe->nclocks, vfe->clock);
+	}
+
+	vfe->power_count--;
+
+exit:
+	mutex_unlock(&vfe->power_lock);
+}
+
+/*
+ * vfe_video_pad_to_line - Get pointer to VFE line by media pad
+ * @pad: Media pad
+ *
+ * Return pointer to vfe line structure
+ */
+static struct vfe_line *vfe_video_pad_to_line(struct media_pad *pad)
+{
+	struct media_pad *vfe_pad;
+	struct v4l2_subdev *subdev;
+
+	vfe_pad = media_entity_remote_pad(pad);
+	if (vfe_pad == NULL)
+		return NULL;
+
+	subdev = media_entity_to_v4l2_subdev(vfe_pad->entity);
+
+	return container_of(subdev, struct vfe_line, subdev);
+}
+
+/*
+ * vfe_queue_buffer - Add empty buffer
+ * @vid: Video device structure
+ * @buf: Buffer to be enqueued
+ *
+ * Add an empty buffer - depending on the current number of buffers it will be
+ * put in pending buffer queue or directly given to the hardware to be filled.
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int vfe_queue_buffer(struct camss_video *vid,
+			    struct camss_buffer *buf)
+{
+	struct vfe_device *vfe = &vid->camss->vfe;
+	struct vfe_line *line;
+	struct vfe_output *output;
+	unsigned long flags;
+
+	line = vfe_video_pad_to_line(&vid->pad);
+	if (!line) {
+		dev_err(to_device(vfe), "Can not queue buffer\n");
+		return -1;
+	}
+	output = &line->output;
+
+	spin_lock_irqsave(&vfe->output_lock, flags);
+
+	vfe_buf_update_wm_on_new(vfe, output, buf);
+
+	spin_unlock_irqrestore(&vfe->output_lock, flags);
+
+	return 0;
+}
+
+/*
+ * vfe_flush_buffers - Return all vb2 buffers
+ * @vid: Video device structure
+ * @state: vb2 buffer state of the returned buffers
+ *
+ * Return all buffers to vb2. This includes queued pending buffers (still
+ * unused) and any buffers given to the hardware but again still not used.
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int vfe_flush_buffers(struct camss_video *vid,
+			     enum vb2_buffer_state state)
+{
+	struct vfe_device *vfe = &vid->camss->vfe;
+	struct vfe_line *line;
+	struct vfe_output *output;
+	unsigned long flags;
+
+	line = vfe_video_pad_to_line(&vid->pad);
+	if (!line) {
+		dev_err(to_device(vfe),	"Can not flush buffers\n");
+		return -1;
+	}
+	output = &line->output;
+
+	spin_lock_irqsave(&vfe->output_lock, flags);
+
+	vfe_buf_flush_pending(output, state);
+
+	if (output->buf[0])
+		vb2_buffer_done(&output->buf[0]->vb.vb2_buf, state);
+
+	if (output->buf[1])
+		vb2_buffer_done(&output->buf[1]->vb.vb2_buf, state);
+
+	if (output->last_buffer) {
+		vb2_buffer_done(&output->last_buffer->vb.vb2_buf, state);
+		output->last_buffer = NULL;
+	}
+
+	spin_unlock_irqrestore(&vfe->output_lock, flags);
+
+	return 0;
+}
+
+/*
+ * vfe_set_power - Power on/off VFE module
+ * @sd: VFE V4L2 subdevice
+ * @on: Requested power state
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int vfe_set_power(struct v4l2_subdev *sd, int on)
+{
+	struct vfe_line *line = v4l2_get_subdevdata(sd);
+	struct vfe_device *vfe = to_vfe(line);
+	int ret;
+
+	if (on) {
+		u32 hw_version;
+
+		ret = vfe_get(vfe);
+		if (ret < 0)
+			return ret;
+
+		hw_version = readl_relaxed(vfe->base + VFE_0_HW_VERSION);
+		dev_dbg(to_device(vfe),
+			"VFE HW Version = 0x%08x\n", hw_version);
+	} else {
+		vfe_put(vfe);
+	}
+
+	return 0;
+}
+
+/*
+ * vfe_set_stream - Enable/disable streaming on VFE module
+ * @sd: VFE V4L2 subdevice
+ * @enable: Requested streaming state
+ *
+ * Main configuration of VFE module is triggered here.
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int vfe_set_stream(struct v4l2_subdev *sd, int enable)
+{
+	struct vfe_line *line = v4l2_get_subdevdata(sd);
+	struct vfe_device *vfe = to_vfe(line);
+	int ret;
+
+	if (enable) {
+		ret = vfe_enable(line);
+		if (ret < 0)
+			dev_err(to_device(vfe),
+				"Failed to enable vfe outputs\n");
+	} else {
+		ret = vfe_disable(line);
+		if (ret < 0)
+			dev_err(to_device(vfe),
+				"Failed to disable vfe outputs\n");
+	}
+
+	return ret;
+}
+
+/*
+ * __vfe_get_format - Get pointer to format structure
+ * @line: VFE line
+ * @cfg: V4L2 subdev pad configuration
+ * @pad: pad from which format is requested
+ * @which: TRY or ACTIVE format
+ *
+ * Return pointer to TRY or ACTIVE format structure
+ */
+static struct v4l2_mbus_framefmt *
+__vfe_get_format(struct vfe_line *line,
+		 struct v4l2_subdev_pad_config *cfg,
+		 unsigned int pad,
+		 enum v4l2_subdev_format_whence which)
+{
+	if (which == V4L2_SUBDEV_FORMAT_TRY)
+		return v4l2_subdev_get_try_format(&line->subdev, cfg, pad);
+
+	return &line->fmt[pad];
+}
+
+/*
+ * __vfe_get_compose - Get pointer to compose selection structure
+ * @line: VFE line
+ * @cfg: V4L2 subdev pad configuration
+ * @which: TRY or ACTIVE format
+ *
+ * Return pointer to TRY or ACTIVE compose rectangle structure
+ */
+static struct v4l2_rect *
+__vfe_get_compose(struct vfe_line *line,
+		  struct v4l2_subdev_pad_config *cfg,
+		  enum v4l2_subdev_format_whence which)
+{
+	if (which == V4L2_SUBDEV_FORMAT_TRY)
+		return v4l2_subdev_get_try_compose(&line->subdev, cfg,
+						   MSM_VFE_PAD_SINK);
+
+	return &line->compose;
+}
+
+/*
+ * __vfe_get_crop - Get pointer to crop selection structure
+ * @line: VFE line
+ * @cfg: V4L2 subdev pad configuration
+ * @which: TRY or ACTIVE format
+ *
+ * Return pointer to TRY or ACTIVE crop rectangle structure
+ */
+static struct v4l2_rect *
+__vfe_get_crop(struct vfe_line *line,
+	       struct v4l2_subdev_pad_config *cfg,
+	       enum v4l2_subdev_format_whence which)
+{
+	if (which == V4L2_SUBDEV_FORMAT_TRY)
+		return v4l2_subdev_get_try_crop(&line->subdev, cfg,
+						MSM_VFE_PAD_SRC);
+
+	return &line->crop;
+}
+
+/*
+ * vfe_try_format - Handle try format by pad subdev method
+ * @line: VFE line
+ * @cfg: V4L2 subdev pad configuration
+ * @pad: pad on which format is requested
+ * @fmt: pointer to v4l2 format structure
+ * @which: wanted subdev format
+ */
+static void vfe_try_format(struct vfe_line *line,
+			   struct v4l2_subdev_pad_config *cfg,
+			   unsigned int pad,
+			   struct v4l2_mbus_framefmt *fmt,
+			   enum v4l2_subdev_format_whence which)
+{
+	unsigned int i;
+	u32 code;
+
+	switch (pad) {
+	case MSM_VFE_PAD_SINK:
+		/* Set format on sink pad */
+
+		for (i = 0; i < ARRAY_SIZE(vfe_formats); i++)
+			if (fmt->code == vfe_formats[i].code)
+				break;
+
+		/* If not found, use UYVY as default */
+		if (i >= ARRAY_SIZE(vfe_formats))
+			fmt->code = MEDIA_BUS_FMT_UYVY8_2X8;
+
+		fmt->width = clamp_t(u32, fmt->width, 1, 8191);
+		fmt->height = clamp_t(u32, fmt->height, 1, 8191);
+
+		fmt->field = V4L2_FIELD_NONE;
+		fmt->colorspace = V4L2_COLORSPACE_SRGB;
+
+		break;
+
+	case MSM_VFE_PAD_SRC:
+		/* Set and return a format same as sink pad */
+
+		code = fmt->code;
+
+		*fmt = *__vfe_get_format(line, cfg, MSM_VFE_PAD_SINK,
+					 which);
+
+		if (line->id == VFE_LINE_PIX) {
+			struct v4l2_rect *rect;
+
+			rect = __vfe_get_crop(line, cfg, which);
+
+			fmt->width = rect->width;
+			fmt->height = rect->height;
+
+			switch (fmt->code) {
+			case MEDIA_BUS_FMT_YUYV8_2X8:
+				if (code == MEDIA_BUS_FMT_YUYV8_1_5X8)
+					fmt->code = MEDIA_BUS_FMT_YUYV8_1_5X8;
+				else
+					fmt->code = MEDIA_BUS_FMT_YUYV8_2X8;
+				break;
+			case MEDIA_BUS_FMT_YVYU8_2X8:
+				if (code == MEDIA_BUS_FMT_YVYU8_1_5X8)
+					fmt->code = MEDIA_BUS_FMT_YVYU8_1_5X8;
+				else
+					fmt->code = MEDIA_BUS_FMT_YVYU8_2X8;
+				break;
+			case MEDIA_BUS_FMT_UYVY8_2X8:
+			default:
+				if (code == MEDIA_BUS_FMT_UYVY8_1_5X8)
+					fmt->code = MEDIA_BUS_FMT_UYVY8_1_5X8;
+				else
+					fmt->code = MEDIA_BUS_FMT_UYVY8_2X8;
+				break;
+			case MEDIA_BUS_FMT_VYUY8_2X8:
+				if (code == MEDIA_BUS_FMT_VYUY8_1_5X8)
+					fmt->code = MEDIA_BUS_FMT_VYUY8_1_5X8;
+				else
+					fmt->code = MEDIA_BUS_FMT_VYUY8_2X8;
+				break;
+			}
+		}
+
+		break;
+	}
+
+	fmt->colorspace = V4L2_COLORSPACE_SRGB;
+}
+
+/*
+ * vfe_try_compose - Handle try compose selection by pad subdev method
+ * @line: VFE line
+ * @cfg: V4L2 subdev pad configuration
+ * @rect: pointer to v4l2 rect structure
+ * @which: wanted subdev format
+ */
+static void vfe_try_compose(struct vfe_line *line,
+			    struct v4l2_subdev_pad_config *cfg,
+			    struct v4l2_rect *rect,
+			    enum v4l2_subdev_format_whence which)
+{
+	struct v4l2_mbus_framefmt *fmt;
+
+	fmt = __vfe_get_format(line, cfg, MSM_VFE_PAD_SINK, which);
+
+	if (rect->width > fmt->width)
+		rect->width = fmt->width;
+
+	if (rect->height > fmt->height)
+		rect->height = fmt->height;
+
+	if (fmt->width > rect->width * SCALER_RATIO_MAX)
+		rect->width = (fmt->width + SCALER_RATIO_MAX - 1) /
+							SCALER_RATIO_MAX;
+
+	rect->width &= ~0x1;
+
+	if (fmt->height > rect->height * SCALER_RATIO_MAX)
+		rect->height = (fmt->height + SCALER_RATIO_MAX - 1) /
+							SCALER_RATIO_MAX;
+
+	if (rect->width < 16)
+		rect->width = 16;
+
+	if (rect->height < 4)
+		rect->height = 4;
+}
+
+/*
+ * vfe_try_crop - Handle try crop selection by pad subdev method
+ * @line: VFE line
+ * @cfg: V4L2 subdev pad configuration
+ * @rect: pointer to v4l2 rect structure
+ * @which: wanted subdev format
+ */
+static void vfe_try_crop(struct vfe_line *line,
+			 struct v4l2_subdev_pad_config *cfg,
+			 struct v4l2_rect *rect,
+			 enum v4l2_subdev_format_whence which)
+{
+	struct v4l2_rect *compose;
+
+	compose = __vfe_get_compose(line, cfg, which);
+
+	if (rect->width > compose->width)
+		rect->width = compose->width;
+
+	if (rect->width + rect->left > compose->width)
+		rect->left = compose->width - rect->width;
+
+	if (rect->height > compose->height)
+		rect->height = compose->height;
+
+	if (rect->height + rect->top > compose->height)
+		rect->top = compose->height - rect->height;
+
+	/* wm in line based mode writes multiple of 16 horizontally */
+	rect->left += (rect->width & 0xf) >> 1;
+	rect->width &= ~0xf;
+
+	if (rect->width < 16) {
+		rect->left = 0;
+		rect->width = 16;
+	}
+
+	if (rect->height < 4) {
+		rect->top = 0;
+		rect->height = 4;
+	}
+}
+
+/*
+ * vfe_enum_mbus_code - Handle pixel format enumeration
+ * @sd: VFE V4L2 subdevice
+ * @cfg: V4L2 subdev pad configuration
+ * @code: pointer to v4l2_subdev_mbus_code_enum structure
+ *
+ * return -EINVAL or zero on success
+ */
+static int vfe_enum_mbus_code(struct v4l2_subdev *sd,
+			      struct v4l2_subdev_pad_config *cfg,
+			      struct v4l2_subdev_mbus_code_enum *code)
+{
+	struct vfe_line *line = v4l2_get_subdevdata(sd);
+	struct v4l2_mbus_framefmt *format;
+
+	if (code->pad == MSM_VFE_PAD_SINK) {
+		if (code->index >= ARRAY_SIZE(vfe_formats))
+			return -EINVAL;
+
+		code->code = vfe_formats[code->index].code;
+	} else {
+		if (code->index > 0)
+			return -EINVAL;
+
+		format = __vfe_get_format(line, cfg, MSM_VFE_PAD_SINK,
+					  code->which);
+
+		code->code = format->code;
+	}
+
+	return 0;
+}
+
+/*
+ * vfe_enum_frame_size - Handle frame size enumeration
+ * @sd: VFE V4L2 subdevice
+ * @cfg: V4L2 subdev pad configuration
+ * @fse: pointer to v4l2_subdev_frame_size_enum structure
+ *
+ * Return -EINVAL or zero on success
+ */
+static int vfe_enum_frame_size(struct v4l2_subdev *sd,
+			       struct v4l2_subdev_pad_config *cfg,
+			       struct v4l2_subdev_frame_size_enum *fse)
+{
+	struct vfe_line *line = v4l2_get_subdevdata(sd);
+	struct v4l2_mbus_framefmt format;
+
+	if (fse->index != 0)
+		return -EINVAL;
+
+	format.code = fse->code;
+	format.width = 1;
+	format.height = 1;
+	vfe_try_format(line, cfg, fse->pad, &format, fse->which);
+	fse->min_width = format.width;
+	fse->min_height = format.height;
+
+	if (format.code != fse->code)
+		return -EINVAL;
+
+	format.code = fse->code;
+	format.width = -1;
+	format.height = -1;
+	vfe_try_format(line, cfg, fse->pad, &format, fse->which);
+	fse->max_width = format.width;
+	fse->max_height = format.height;
+
+	return 0;
+}
+
+/*
+ * vfe_get_format - Handle get format by pads subdev method
+ * @sd: VFE V4L2 subdevice
+ * @cfg: V4L2 subdev pad configuration
+ * @fmt: pointer to v4l2 subdev format structure
+ *
+ * Return -EINVAL or zero on success
+ */
+static int vfe_get_format(struct v4l2_subdev *sd,
+			  struct v4l2_subdev_pad_config *cfg,
+			  struct v4l2_subdev_format *fmt)
+{
+	struct vfe_line *line = v4l2_get_subdevdata(sd);
+	struct v4l2_mbus_framefmt *format;
+
+	format = __vfe_get_format(line, cfg, fmt->pad, fmt->which);
+	if (format == NULL)
+		return -EINVAL;
+
+	fmt->format = *format;
+
+	return 0;
+}
+
+static int vfe_set_selection(struct v4l2_subdev *sd,
+			     struct v4l2_subdev_pad_config *cfg,
+			     struct v4l2_subdev_selection *sel);
+
+/*
+ * vfe_set_format - Handle set format by pads subdev method
+ * @sd: VFE V4L2 subdevice
+ * @cfg: V4L2 subdev pad configuration
+ * @fmt: pointer to v4l2 subdev format structure
+ *
+ * Return -EINVAL or zero on success
+ */
+static int vfe_set_format(struct v4l2_subdev *sd,
+			  struct v4l2_subdev_pad_config *cfg,
+			  struct v4l2_subdev_format *fmt)
+{
+	struct vfe_line *line = v4l2_get_subdevdata(sd);
+	struct v4l2_mbus_framefmt *format;
+
+	format = __vfe_get_format(line, cfg, fmt->pad, fmt->which);
+	if (format == NULL)
+		return -EINVAL;
+
+	vfe_try_format(line, cfg, fmt->pad, &fmt->format, fmt->which);
+	*format = fmt->format;
+
+	if (fmt->pad == MSM_VFE_PAD_SINK) {
+		struct v4l2_subdev_selection sel = { 0 };
+		int ret;
+
+		/* Propagate the format from sink to source */
+		format = __vfe_get_format(line, cfg, MSM_VFE_PAD_SRC,
+					  fmt->which);
+
+		*format = fmt->format;
+		vfe_try_format(line, cfg, MSM_VFE_PAD_SRC, format,
+			       fmt->which);
+
+		if (line->id != VFE_LINE_PIX)
+			return 0;
+
+		/* Reset sink pad compose selection */
+		sel.which = fmt->which;
+		sel.pad = MSM_VFE_PAD_SINK;
+		sel.target = V4L2_SEL_TGT_COMPOSE;
+		sel.r.width = fmt->format.width;
+		sel.r.height = fmt->format.height;
+		ret = vfe_set_selection(sd, cfg, &sel);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+
+/*
+ * vfe_get_selection - Handle get selection by pads subdev method
+ * @sd: VFE V4L2 subdevice
+ * @cfg: V4L2 subdev pad configuration
+ * @sel: pointer to v4l2 subdev selection structure
+ *
+ * Return -EINVAL or zero on success
+ */
+static int vfe_get_selection(struct v4l2_subdev *sd,
+			     struct v4l2_subdev_pad_config *cfg,
+			     struct v4l2_subdev_selection *sel)
+{
+	struct vfe_line *line = v4l2_get_subdevdata(sd);
+	struct v4l2_subdev_format fmt = { 0 };
+	struct v4l2_rect *rect;
+	int ret;
+
+	if (line->id != VFE_LINE_PIX)
+		return -EINVAL;
+
+	if (sel->pad == MSM_VFE_PAD_SINK)
+		switch (sel->target) {
+		case V4L2_SEL_TGT_COMPOSE_BOUNDS:
+			fmt.pad = sel->pad;
+			fmt.which = sel->which;
+			ret = vfe_get_format(sd, cfg, &fmt);
+			if (ret < 0)
+				return ret;
+
+			sel->r.left = 0;
+			sel->r.top = 0;
+			sel->r.width = fmt.format.width;
+			sel->r.height = fmt.format.height;
+			break;
+		case V4L2_SEL_TGT_COMPOSE:
+			rect = __vfe_get_compose(line, cfg, sel->which);
+			if (rect == NULL)
+				return -EINVAL;
+
+			sel->r = *rect;
+			break;
+		default:
+			return -EINVAL;
+		}
+	else if (sel->pad == MSM_VFE_PAD_SRC)
+		switch (sel->target) {
+		case V4L2_SEL_TGT_CROP_BOUNDS:
+			rect = __vfe_get_compose(line, cfg, sel->which);
+			if (rect == NULL)
+				return -EINVAL;
+
+			sel->r.left = rect->left;
+			sel->r.top = rect->top;
+			sel->r.width = rect->width;
+			sel->r.height = rect->height;
+			break;
+		case V4L2_SEL_TGT_CROP:
+			rect = __vfe_get_crop(line, cfg, sel->which);
+			if (rect == NULL)
+				return -EINVAL;
+
+			sel->r = *rect;
+			break;
+		default:
+			return -EINVAL;
+		}
+
+	return 0;
+}
+
+/*
+ * vfe_set_selection - Handle set selection by pads subdev method
+ * @sd: VFE V4L2 subdevice
+ * @cfg: V4L2 subdev pad configuration
+ * @sel: pointer to v4l2 subdev selection structure
+ *
+ * Return -EINVAL or zero on success
+ */
+int vfe_set_selection(struct v4l2_subdev *sd,
+			     struct v4l2_subdev_pad_config *cfg,
+			     struct v4l2_subdev_selection *sel)
+{
+	struct vfe_line *line = v4l2_get_subdevdata(sd);
+	struct v4l2_rect *rect;
+	int ret;
+
+	if (line->id != VFE_LINE_PIX)
+		return -EINVAL;
+
+	if (sel->target == V4L2_SEL_TGT_COMPOSE &&
+		sel->pad == MSM_VFE_PAD_SINK) {
+		struct v4l2_subdev_selection crop = { 0 };
+
+		rect = __vfe_get_compose(line, cfg, sel->which);
+		if (rect == NULL)
+			return -EINVAL;
+
+		vfe_try_compose(line, cfg, &sel->r, sel->which);
+		*rect = sel->r;
+
+		/* Reset source crop selection */
+		crop.which = sel->which;
+		crop.pad = MSM_VFE_PAD_SRC;
+		crop.target = V4L2_SEL_TGT_CROP;
+		crop.r = *rect;
+		ret = vfe_set_selection(sd, cfg, &crop);
+	} else if (sel->target == V4L2_SEL_TGT_CROP &&
+		sel->pad == MSM_VFE_PAD_SRC) {
+		struct v4l2_subdev_format fmt = { 0 };
+
+		rect = __vfe_get_crop(line, cfg, sel->which);
+		if (rect == NULL)
+			return -EINVAL;
+
+		vfe_try_crop(line, cfg, &sel->r, sel->which);
+		*rect = sel->r;
+
+		/* Reset source pad format width and height */
+		fmt.which = sel->which;
+		fmt.pad = MSM_VFE_PAD_SRC;
+		ret = vfe_get_format(sd, cfg, &fmt);
+		if (ret < 0)
+			return ret;
+
+		fmt.format.width = rect->width;
+		fmt.format.height = rect->height;
+		ret = vfe_set_format(sd, cfg, &fmt);
+	} else {
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+/*
+ * vfe_init_formats - Initialize formats on all pads
+ * @sd: VFE V4L2 subdevice
+ * @fh: V4L2 subdev file handle
+ *
+ * Initialize all pad formats with default values.
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int vfe_init_formats(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
+{
+	struct v4l2_subdev_format format = {
+		.pad = MSM_VFE_PAD_SINK,
+		.which = fh ? V4L2_SUBDEV_FORMAT_TRY :
+			      V4L2_SUBDEV_FORMAT_ACTIVE,
+		.format = {
+			.code = MEDIA_BUS_FMT_UYVY8_2X8,
+			.width = 1920,
+			.height = 1080
+		}
+	};
+
+	return vfe_set_format(sd, fh ? fh->pad : NULL, &format);
+}
+
+/*
+ * msm_vfe_subdev_init - Initialize VFE device structure and resources
+ * @vfe: VFE device
+ * @res: VFE module resources table
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+int msm_vfe_subdev_init(struct vfe_device *vfe, const struct resources *res)
+{
+	struct device *dev = to_device(vfe);
+	struct platform_device *pdev = to_platform_device(dev);
+	struct resource *r;
+	struct camss *camss = to_camss(vfe);
+	int i, j;
+	int ret;
+
+	/* Memory */
+
+	r = platform_get_resource_byname(pdev, IORESOURCE_MEM, res->reg[0]);
+	vfe->base = devm_ioremap_resource(dev, r);
+	if (IS_ERR(vfe->base)) {
+		dev_err(dev, "could not map memory\n");
+		return PTR_ERR(vfe->base);
+	}
+
+	/* Interrupt */
+
+	r = platform_get_resource_byname(pdev, IORESOURCE_IRQ,
+					 res->interrupt[0]);
+	if (!r) {
+		dev_err(dev, "missing IRQ\n");
+		return -EINVAL;
+	}
+
+	vfe->irq = r->start;
+	snprintf(vfe->irq_name, sizeof(vfe->irq_name), "%s_%s%d",
+		 dev_name(dev), MSM_VFE_NAME, vfe->id);
+	ret = devm_request_irq(dev, vfe->irq, vfe_isr,
+			       IRQF_TRIGGER_RISING, vfe->irq_name, vfe);
+	if (ret < 0) {
+		dev_err(dev, "request_irq failed: %d\n", ret);
+		return ret;
+	}
+
+	/* Clocks */
+
+	vfe->nclocks = 0;
+	while (res->clock[vfe->nclocks])
+		vfe->nclocks++;
+
+	vfe->clock = devm_kzalloc(dev, vfe->nclocks * sizeof(*vfe->clock),
+				  GFP_KERNEL);
+	if (!vfe->clock)
+		return -ENOMEM;
+
+	for (i = 0; i < vfe->nclocks; i++) {
+		struct camss_clock *clock = &vfe->clock[i];
+
+		clock->clk = devm_clk_get(dev, res->clock[i]);
+		if (IS_ERR(clock->clk))
+			return PTR_ERR(clock->clk);
+
+		clock->name = res->clock[i];
+
+		clock->nfreqs = 0;
+		while (res->clock_rate[i][clock->nfreqs])
+			clock->nfreqs++;
+
+		if (!clock->nfreqs) {
+			clock->freq = NULL;
+			continue;
+		}
+
+		clock->freq = devm_kzalloc(dev, clock->nfreqs *
+					   sizeof(*clock->freq), GFP_KERNEL);
+		if (!clock->freq)
+			return -ENOMEM;
+
+		for (j = 0; j < clock->nfreqs; j++)
+			clock->freq[j] = res->clock_rate[i][j];
+	}
+
+	mutex_init(&vfe->power_lock);
+	vfe->power_count = 0;
+
+	mutex_init(&vfe->stream_lock);
+	vfe->stream_count = 0;
+
+	spin_lock_init(&vfe->output_lock);
+
+	vfe->id = 0;
+	vfe->reg_update = 0;
+
+	for (i = VFE_LINE_RDI0; i <= VFE_LINE_PIX; i++) {
+		vfe->line[i].video_out.type =
+					V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+		vfe->line[i].video_out.camss = camss;
+		vfe->line[i].id = i;
+		init_completion(&vfe->line[i].output.sof);
+		init_completion(&vfe->line[i].output.reg_update);
+	}
+
+	init_completion(&vfe->reset_complete);
+	init_completion(&vfe->halt_complete);
+
+	return 0;
+}
+
+/*
+ * msm_vfe_get_vfe_id - Get VFE HW module id
+ * @entity: Pointer to VFE media entity structure
+ * @id: Return CSID HW module id here
+ */
+void msm_vfe_get_vfe_id(struct media_entity *entity, u8 *id)
+{
+	struct v4l2_subdev *sd;
+	struct vfe_line *line;
+	struct vfe_device *vfe;
+
+	sd = media_entity_to_v4l2_subdev(entity);
+	line = v4l2_get_subdevdata(sd);
+	vfe = to_vfe(line);
+
+	*id = vfe->id;
+}
+
+/*
+ * msm_vfe_get_vfe_line_id - Get VFE line id by media entity
+ * @entity: Pointer to VFE media entity structure
+ * @id: Return VFE line id here
+ */
+void msm_vfe_get_vfe_line_id(struct media_entity *entity, enum vfe_line_id *id)
+{
+	struct v4l2_subdev *sd;
+	struct vfe_line *line;
+
+	sd = media_entity_to_v4l2_subdev(entity);
+	line = v4l2_get_subdevdata(sd);
+
+	*id = line->id;
+}
+
+/*
+ * vfe_link_setup - Setup VFE connections
+ * @entity: Pointer to media entity structure
+ * @local: Pointer to local pad
+ * @remote: Pointer to remote pad
+ * @flags: Link flags
+ *
+ * Return 0 on success
+ */
+static int vfe_link_setup(struct media_entity *entity,
+			  const struct media_pad *local,
+			  const struct media_pad *remote, u32 flags)
+{
+	if (flags & MEDIA_LNK_FL_ENABLED)
+		if (media_entity_remote_pad(local))
+			return -EBUSY;
+
+	return 0;
+}
+
+static const struct v4l2_subdev_core_ops vfe_core_ops = {
+	.s_power = vfe_set_power,
+};
+
+static const struct v4l2_subdev_video_ops vfe_video_ops = {
+	.s_stream = vfe_set_stream,
+};
+
+static const struct v4l2_subdev_pad_ops vfe_pad_ops = {
+	.enum_mbus_code = vfe_enum_mbus_code,
+	.enum_frame_size = vfe_enum_frame_size,
+	.get_fmt = vfe_get_format,
+	.set_fmt = vfe_set_format,
+	.get_selection = vfe_get_selection,
+	.set_selection = vfe_set_selection,
+};
+
+static const struct v4l2_subdev_ops vfe_v4l2_ops = {
+	.core = &vfe_core_ops,
+	.video = &vfe_video_ops,
+	.pad = &vfe_pad_ops,
+};
+
+static const struct v4l2_subdev_internal_ops vfe_v4l2_internal_ops = {
+	.open = vfe_init_formats,
+};
+
+static const struct media_entity_operations vfe_media_ops = {
+	.link_setup = vfe_link_setup,
+	.link_validate = v4l2_subdev_link_validate,
+};
+
+static const struct camss_video_ops camss_vfe_video_ops = {
+	.queue_buffer = vfe_queue_buffer,
+	.flush_buffers = vfe_flush_buffers,
+};
+
+void msm_vfe_stop_streaming(struct vfe_device *vfe)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(vfe->line); i++)
+		msm_video_stop_streaming(&vfe->line[i].video_out);
+}
+
+/*
+ * msm_vfe_register_entities - Register subdev node for VFE module
+ * @vfe: VFE device
+ * @v4l2_dev: V4L2 device
+ *
+ * Initialize and register a subdev node for the VFE module. Then
+ * call msm_video_register() to register the video device node which
+ * will be connected to this subdev node. Then actually create the
+ * media link between them.
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+int msm_vfe_register_entities(struct vfe_device *vfe,
+			      struct v4l2_device *v4l2_dev)
+{
+	struct device *dev = to_device(vfe);
+	struct v4l2_subdev *sd;
+	struct media_pad *pads;
+	struct camss_video *video_out;
+	int ret;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(vfe->line); i++) {
+		char name[32];
+
+		sd = &vfe->line[i].subdev;
+		pads = vfe->line[i].pads;
+		video_out = &vfe->line[i].video_out;
+
+		v4l2_subdev_init(sd, &vfe_v4l2_ops);
+		sd->internal_ops = &vfe_v4l2_internal_ops;
+		sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+		if (i == VFE_LINE_PIX)
+			snprintf(sd->name, ARRAY_SIZE(sd->name), "%s%d_%s",
+				 MSM_VFE_NAME, vfe->id, "pix");
+		else
+			snprintf(sd->name, ARRAY_SIZE(sd->name), "%s%d_%s%d",
+				 MSM_VFE_NAME, vfe->id, "rdi", i);
+
+		v4l2_set_subdevdata(sd, &vfe->line[i]);
+
+		ret = vfe_init_formats(sd, NULL);
+		if (ret < 0) {
+			dev_err(dev, "Failed to init format: %d\n", ret);
+			goto error_init;
+		}
+
+		pads[MSM_VFE_PAD_SINK].flags = MEDIA_PAD_FL_SINK;
+		pads[MSM_VFE_PAD_SRC].flags = MEDIA_PAD_FL_SOURCE;
+
+		sd->entity.function = MEDIA_ENT_F_PROC_VIDEO_PIXEL_FORMATTER;
+		sd->entity.ops = &vfe_media_ops;
+		ret = media_entity_pads_init(&sd->entity, MSM_VFE_PADS_NUM,
+					     pads);
+		if (ret < 0) {
+			dev_err(dev, "Failed to init media entity: %d\n", ret);
+			goto error_init;
+		}
+
+		ret = v4l2_device_register_subdev(v4l2_dev, sd);
+		if (ret < 0) {
+			dev_err(dev, "Failed to register subdev: %d\n", ret);
+			goto error_reg_subdev;
+		}
+
+		video_out->ops = &camss_vfe_video_ops;
+		video_out->bpl_alignment = 8;
+		video_out->line_based = 0;
+		if (i == VFE_LINE_PIX) {
+			video_out->bpl_alignment = 16;
+			video_out->line_based = 1;
+		}
+		snprintf(name, ARRAY_SIZE(name), "%s%d_%s%d",
+			 MSM_VFE_NAME, vfe->id, "video", i);
+		ret = msm_video_register(video_out, v4l2_dev, name,
+					 i == VFE_LINE_PIX ? 1 : 0);
+		if (ret < 0) {
+			dev_err(dev, "Failed to register video node: %d\n",
+				ret);
+			goto error_reg_video;
+		}
+
+		ret = media_create_pad_link(
+				&sd->entity, MSM_VFE_PAD_SRC,
+				&video_out->vdev.entity, 0,
+				MEDIA_LNK_FL_IMMUTABLE | MEDIA_LNK_FL_ENABLED);
+		if (ret < 0) {
+			dev_err(dev, "Failed to link %s->%s entities: %d\n",
+				sd->entity.name, video_out->vdev.entity.name,
+				ret);
+			goto error_link;
+		}
+	}
+
+	return 0;
+
+error_link:
+	msm_video_unregister(video_out);
+
+error_reg_video:
+	v4l2_device_unregister_subdev(sd);
+
+error_reg_subdev:
+	media_entity_cleanup(&sd->entity);
+
+error_init:
+	for (i--; i >= 0; i--) {
+		sd = &vfe->line[i].subdev;
+		video_out = &vfe->line[i].video_out;
+
+		msm_video_unregister(video_out);
+		v4l2_device_unregister_subdev(sd);
+		media_entity_cleanup(&sd->entity);
+	}
+
+	return ret;
+}
+
+/*
+ * msm_vfe_unregister_entities - Unregister VFE module subdev node
+ * @vfe: VFE device
+ */
+void msm_vfe_unregister_entities(struct vfe_device *vfe)
+{
+	int i;
+
+	mutex_destroy(&vfe->power_lock);
+	mutex_destroy(&vfe->stream_lock);
+
+	for (i = 0; i < ARRAY_SIZE(vfe->line); i++) {
+		struct v4l2_subdev *sd = &vfe->line[i].subdev;
+		struct camss_video *video_out = &vfe->line[i].video_out;
+
+		msm_video_unregister(video_out);
+		v4l2_device_unregister_subdev(sd);
+		media_entity_cleanup(&sd->entity);
+	}
+}
diff --git a/drivers/media/platform/qcom/camss-8x16/camss-vfe.h b/drivers/media/platform/qcom/camss-8x16/camss-vfe.h
new file mode 100644
index 000000000000..53d5b66a9dfb
--- /dev/null
+++ b/drivers/media/platform/qcom/camss-8x16/camss-vfe.h
@@ -0,0 +1,123 @@
+/*
+ * camss-vfe.h
+ *
+ * Qualcomm MSM Camera Subsystem - VFE (Video Front End) Module
+ *
+ * Copyright (c) 2013-2015, The Linux Foundation. All rights reserved.
+ * Copyright (C) 2015-2017 Linaro Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#ifndef QC_MSM_CAMSS_VFE_H
+#define QC_MSM_CAMSS_VFE_H
+
+#include <linux/clk.h>
+#include <linux/spinlock_types.h>
+#include <media/media-entity.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-subdev.h>
+
+#include "camss-video.h"
+
+#define MSM_VFE_PAD_SINK 0
+#define MSM_VFE_PAD_SRC 1
+#define MSM_VFE_PADS_NUM 2
+
+#define MSM_VFE_LINE_NUM 4
+#define MSM_VFE_IMAGE_MASTERS_NUM 7
+#define MSM_VFE_COMPOSITE_IRQ_NUM 4
+
+#define MSM_VFE_VFE0_UB_SIZE 1023
+#define MSM_VFE_VFE0_UB_SIZE_RDI (MSM_VFE_VFE0_UB_SIZE / 3)
+#define MSM_VFE_VFE1_UB_SIZE 1535
+#define MSM_VFE_VFE1_UB_SIZE_RDI (MSM_VFE_VFE1_UB_SIZE / 3)
+
+enum vfe_output_state {
+	VFE_OUTPUT_OFF,
+	VFE_OUTPUT_RESERVED,
+	VFE_OUTPUT_SINGLE,
+	VFE_OUTPUT_CONTINUOUS,
+	VFE_OUTPUT_IDLE,
+	VFE_OUTPUT_STOPPING
+};
+
+enum vfe_line_id {
+	VFE_LINE_NONE = -1,
+	VFE_LINE_RDI0 = 0,
+	VFE_LINE_RDI1 = 1,
+	VFE_LINE_RDI2 = 2,
+	VFE_LINE_PIX = 3
+};
+
+struct vfe_output {
+	u8 wm_num;
+	u8 wm_idx[3];
+
+	int active_buf;
+	struct camss_buffer *buf[2];
+	struct camss_buffer *last_buffer;
+	struct list_head pending_bufs;
+
+	unsigned int drop_update_idx;
+
+	enum vfe_output_state state;
+	unsigned int sequence;
+	int wait_sof;
+	int wait_reg_update;
+	struct completion sof;
+	struct completion reg_update;
+};
+
+struct vfe_line {
+	enum vfe_line_id id;
+	struct v4l2_subdev subdev;
+	struct media_pad pads[MSM_VFE_PADS_NUM];
+	struct v4l2_mbus_framefmt fmt[MSM_VFE_PADS_NUM];
+	struct v4l2_rect compose;
+	struct v4l2_rect crop;
+	struct camss_video video_out;
+	struct vfe_output output;
+};
+
+struct vfe_device {
+	u8 id;
+	void __iomem *base;
+	u32 irq;
+	char irq_name[30];
+	struct camss_clock *clock;
+	int nclocks;
+	struct completion reset_complete;
+	struct completion halt_complete;
+	struct mutex power_lock;
+	int power_count;
+	struct mutex stream_lock;
+	int stream_count;
+	spinlock_t output_lock;
+	enum vfe_line_id wm_output_map[MSM_VFE_IMAGE_MASTERS_NUM];
+	struct vfe_line line[MSM_VFE_LINE_NUM];
+	u32 reg_update;
+	u8 was_streaming;
+};
+
+struct resources;
+
+int msm_vfe_subdev_init(struct vfe_device *vfe, const struct resources *res);
+
+int msm_vfe_register_entities(struct vfe_device *vfe,
+			      struct v4l2_device *v4l2_dev);
+
+void msm_vfe_unregister_entities(struct vfe_device *vfe);
+
+void msm_vfe_get_vfe_id(struct media_entity *entity, u8 *id);
+void msm_vfe_get_vfe_line_id(struct media_entity *entity, enum vfe_line_id *id);
+
+void msm_vfe_stop_streaming(struct vfe_device *vfe);
+
+#endif /* QC_MSM_CAMSS_VFE_H */
diff --git a/drivers/media/platform/qcom/camss-8x16/camss-video.c b/drivers/media/platform/qcom/camss-8x16/camss-video.c
new file mode 100644
index 000000000000..cf4219e871bd
--- /dev/null
+++ b/drivers/media/platform/qcom/camss-8x16/camss-video.c
@@ -0,0 +1,860 @@
+/*
+ * camss-video.c
+ *
+ * Qualcomm MSM Camera Subsystem - V4L2 device node
+ *
+ * Copyright (c) 2013-2015, The Linux Foundation. All rights reserved.
+ * Copyright (C) 2015-2017 Linaro Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#include <linux/slab.h>
+#include <media/media-entity.h>
+#include <media/v4l2-dev.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-ioctl.h>
+#include <media/v4l2-mc.h>
+#include <media/videobuf-core.h>
+#include <media/videobuf2-dma-sg.h>
+
+#include "camss-video.h"
+#include "camss.h"
+
+struct fract {
+	u8 numerator;
+	u8 denominator;
+};
+
+/*
+ * struct camss_format_info - ISP media bus format information
+ * @code: V4L2 media bus format code
+ * @pixelformat: V4L2 pixel format FCC identifier
+ * @planes: Number of planes
+ * @hsub: Horizontal subsampling (for each plane)
+ * @vsub: Vertical subsampling (for each plane)
+ * @bpp: Bits per pixel when stored in memory (for each plane)
+ */
+struct camss_format_info {
+	u32 code;
+	u32 pixelformat;
+	u8 planes;
+	struct fract hsub[3];
+	struct fract vsub[3];
+	unsigned int bpp[3];
+};
+
+static const struct camss_format_info formats_rdi[] = {
+	{ MEDIA_BUS_FMT_UYVY8_2X8, V4L2_PIX_FMT_UYVY, 1,
+	  { { 1, 1 } }, { { 1, 1 } }, { 16 } },
+	{ MEDIA_BUS_FMT_VYUY8_2X8, V4L2_PIX_FMT_VYUY, 1,
+	  { { 1, 1 } }, { { 1, 1 } }, { 16 } },
+	{ MEDIA_BUS_FMT_YUYV8_2X8, V4L2_PIX_FMT_YUYV, 1,
+	  { { 1, 1 } }, { { 1, 1 } }, { 16 } },
+	{ MEDIA_BUS_FMT_YVYU8_2X8, V4L2_PIX_FMT_YVYU, 1,
+	  { { 1, 1 } }, { { 1, 1 } }, { 16 } },
+	{ MEDIA_BUS_FMT_SBGGR8_1X8, V4L2_PIX_FMT_SBGGR8, 1,
+	  { { 1, 1 } }, { { 1, 1 } }, { 8 } },
+	{ MEDIA_BUS_FMT_SGBRG8_1X8, V4L2_PIX_FMT_SGBRG8, 1,
+	  { { 1, 1 } }, { { 1, 1 } }, { 8 } },
+	{ MEDIA_BUS_FMT_SGRBG8_1X8, V4L2_PIX_FMT_SGRBG8, 1,
+	  { { 1, 1 } }, { { 1, 1 } }, { 8 } },
+	{ MEDIA_BUS_FMT_SRGGB8_1X8, V4L2_PIX_FMT_SRGGB8, 1,
+	  { { 1, 1 } }, { { 1, 1 } }, { 8 } },
+	{ MEDIA_BUS_FMT_SBGGR10_1X10, V4L2_PIX_FMT_SBGGR10P, 1,
+	  { { 1, 1 } }, { { 1, 1 } }, { 10 } },
+	{ MEDIA_BUS_FMT_SGBRG10_1X10, V4L2_PIX_FMT_SGBRG10P, 1,
+	  { { 1, 1 } }, { { 1, 1 } }, { 10 } },
+	{ MEDIA_BUS_FMT_SGRBG10_1X10, V4L2_PIX_FMT_SGRBG10P, 1,
+	  { { 1, 1 } }, { { 1, 1 } }, { 10 } },
+	{ MEDIA_BUS_FMT_SRGGB10_1X10, V4L2_PIX_FMT_SRGGB10P, 1,
+	  { { 1, 1 } }, { { 1, 1 } }, { 10 } },
+	{ MEDIA_BUS_FMT_SBGGR12_1X12, V4L2_PIX_FMT_SBGGR12P, 1,
+	  { { 1, 1 } }, { { 1, 1 } }, { 12 } },
+	{ MEDIA_BUS_FMT_SGBRG12_1X12, V4L2_PIX_FMT_SGBRG12P, 1,
+	  { { 1, 1 } }, { { 1, 1 } }, { 12 } },
+	{ MEDIA_BUS_FMT_SGRBG12_1X12, V4L2_PIX_FMT_SGRBG12P, 1,
+	  { { 1, 1 } }, { { 1, 1 } }, { 12 } },
+	{ MEDIA_BUS_FMT_SRGGB12_1X12, V4L2_PIX_FMT_SRGGB12P, 1,
+	  { { 1, 1 } }, { { 1, 1 } }, { 12 } },
+};
+
+static const struct camss_format_info formats_pix[] = {
+	{ MEDIA_BUS_FMT_YUYV8_1_5X8, V4L2_PIX_FMT_NV12, 1,
+	  { { 1, 1 } }, { { 2, 3 } }, { 8 } },
+	{ MEDIA_BUS_FMT_YVYU8_1_5X8, V4L2_PIX_FMT_NV12, 1,
+	  { { 1, 1 } }, { { 2, 3 } }, { 8 } },
+	{ MEDIA_BUS_FMT_UYVY8_1_5X8, V4L2_PIX_FMT_NV12, 1,
+	  { { 1, 1 } }, { { 2, 3 } }, { 8 } },
+	{ MEDIA_BUS_FMT_VYUY8_1_5X8, V4L2_PIX_FMT_NV12, 1,
+	  { { 1, 1 } }, { { 2, 3 } }, { 8 } },
+	{ MEDIA_BUS_FMT_YUYV8_1_5X8, V4L2_PIX_FMT_NV21, 1,
+	  { { 1, 1 } }, { { 2, 3 } }, { 8 } },
+	{ MEDIA_BUS_FMT_YVYU8_1_5X8, V4L2_PIX_FMT_NV21, 1,
+	  { { 1, 1 } }, { { 2, 3 } }, { 8 } },
+	{ MEDIA_BUS_FMT_UYVY8_1_5X8, V4L2_PIX_FMT_NV21, 1,
+	  { { 1, 1 } }, { { 2, 3 } }, { 8 } },
+	{ MEDIA_BUS_FMT_VYUY8_1_5X8, V4L2_PIX_FMT_NV21, 1,
+	  { { 1, 1 } }, { { 2, 3 } }, { 8 } },
+	{ MEDIA_BUS_FMT_YUYV8_2X8, V4L2_PIX_FMT_NV16, 1,
+	  { { 1, 1 } }, { { 1, 2 } }, { 8 } },
+	{ MEDIA_BUS_FMT_YVYU8_2X8, V4L2_PIX_FMT_NV16, 1,
+	  { { 1, 1 } }, { { 1, 2 } }, { 8 } },
+	{ MEDIA_BUS_FMT_UYVY8_2X8, V4L2_PIX_FMT_NV16, 1,
+	  { { 1, 1 } }, { { 1, 2 } }, { 8 } },
+	{ MEDIA_BUS_FMT_VYUY8_2X8, V4L2_PIX_FMT_NV16, 1,
+	  { { 1, 1 } }, { { 1, 2 } }, { 8 } },
+	{ MEDIA_BUS_FMT_YUYV8_2X8, V4L2_PIX_FMT_NV61, 1,
+	  { { 1, 1 } }, { { 1, 2 } }, { 8 } },
+	{ MEDIA_BUS_FMT_YVYU8_2X8, V4L2_PIX_FMT_NV61, 1,
+	  { { 1, 1 } }, { { 1, 2 } }, { 8 } },
+	{ MEDIA_BUS_FMT_UYVY8_2X8, V4L2_PIX_FMT_NV61, 1,
+	  { { 1, 1 } }, { { 1, 2 } }, { 8 } },
+	{ MEDIA_BUS_FMT_VYUY8_2X8, V4L2_PIX_FMT_NV61, 1,
+	  { { 1, 1 } }, { { 1, 2 } }, { 8 } },
+};
+
+/* -----------------------------------------------------------------------------
+ * Helper functions
+ */
+
+static int video_find_format(u32 code, u32 pixelformat,
+			     const struct camss_format_info *formats,
+			     unsigned int nformats)
+{
+	int i;
+
+	for (i = 0; i < nformats; i++) {
+		if (formats[i].code == code &&
+		    formats[i].pixelformat == pixelformat)
+			return i;
+	}
+
+	for (i = 0; i < nformats; i++)
+		if (formats[i].code == code)
+			return i;
+
+	WARN_ON(1);
+
+	return -EINVAL;
+}
+
+/*
+ * video_mbus_to_pix_mp - Convert v4l2_mbus_framefmt to v4l2_pix_format_mplane
+ * @mbus: v4l2_mbus_framefmt format (input)
+ * @pix: v4l2_pix_format_mplane format (output)
+ * @f: a pointer to formats array element to be used for the conversion
+ * @alignment: bytesperline alignment value
+ *
+ * Fill the output pix structure with information from the input mbus format.
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int video_mbus_to_pix_mp(const struct v4l2_mbus_framefmt *mbus,
+				struct v4l2_pix_format_mplane *pix,
+				const struct camss_format_info *f,
+				unsigned int alignment)
+{
+	unsigned int i;
+	u32 bytesperline;
+
+	memset(pix, 0, sizeof(*pix));
+	v4l2_fill_pix_format_mplane(pix, mbus);
+	pix->pixelformat = f->pixelformat;
+	pix->num_planes = f->planes;
+	for (i = 0; i < pix->num_planes; i++) {
+		bytesperline = pix->width / f->hsub[i].numerator *
+			f->hsub[i].denominator * f->bpp[i] / 8;
+		bytesperline = ALIGN(bytesperline, alignment);
+		pix->plane_fmt[i].bytesperline = bytesperline;
+		pix->plane_fmt[i].sizeimage = pix->height /
+				f->vsub[i].numerator * f->vsub[i].denominator *
+				bytesperline;
+	}
+
+	return 0;
+}
+
+static struct v4l2_subdev *video_remote_subdev(struct camss_video *video,
+					       u32 *pad)
+{
+	struct media_pad *remote;
+
+	remote = media_entity_remote_pad(&video->pad);
+
+	if (!remote || !is_media_entity_v4l2_subdev(remote->entity))
+		return NULL;
+
+	if (pad)
+		*pad = remote->index;
+
+	return media_entity_to_v4l2_subdev(remote->entity);
+}
+
+static int video_get_subdev_format(struct camss_video *video,
+				   struct v4l2_format *format)
+{
+	struct v4l2_subdev_format fmt;
+	struct v4l2_subdev *subdev;
+	u32 pad;
+	int ret;
+
+	subdev = video_remote_subdev(video, &pad);
+	if (subdev == NULL)
+		return -EPIPE;
+
+	fmt.pad = pad;
+	fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+
+	ret = v4l2_subdev_call(subdev, pad, get_fmt, NULL, &fmt);
+	if (ret)
+		return ret;
+
+	ret = video_find_format(fmt.format.code,
+				format->fmt.pix_mp.pixelformat,
+				video->formats, video->nformats);
+	if (ret < 0)
+		return ret;
+
+	format->type = video->type;
+
+	return video_mbus_to_pix_mp(&fmt.format, &format->fmt.pix_mp,
+				    &video->formats[ret], video->bpl_alignment);
+}
+
+/* -----------------------------------------------------------------------------
+ * Video queue operations
+ */
+
+static int video_queue_setup(struct vb2_queue *q,
+	unsigned int *num_buffers, unsigned int *num_planes,
+	unsigned int sizes[], struct device *alloc_devs[])
+{
+	struct camss_video *video = vb2_get_drv_priv(q);
+	const struct v4l2_pix_format_mplane *format =
+						&video->active_fmt.fmt.pix_mp;
+	unsigned int i;
+
+	if (*num_planes) {
+		if (*num_planes != format->num_planes)
+			return -EINVAL;
+
+		for (i = 0; i < *num_planes; i++)
+			if (sizes[i] < format->plane_fmt[i].sizeimage)
+				return -EINVAL;
+
+		return 0;
+	}
+
+	*num_planes = format->num_planes;
+
+	for (i = 0; i < *num_planes; i++)
+		sizes[i] = format->plane_fmt[i].sizeimage;
+
+	return 0;
+}
+
+static int video_buf_init(struct vb2_buffer *vb)
+{
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct camss_video *video = vb2_get_drv_priv(vb->vb2_queue);
+	struct camss_buffer *buffer = container_of(vbuf, struct camss_buffer,
+						   vb);
+	const struct v4l2_pix_format_mplane *format =
+						&video->active_fmt.fmt.pix_mp;
+	struct sg_table *sgt;
+	unsigned int i;
+
+	for (i = 0; i < format->num_planes; i++) {
+		sgt = vb2_dma_sg_plane_desc(vb, i);
+		if (!sgt)
+			return -EFAULT;
+
+		buffer->addr[i] = sg_dma_address(sgt->sgl);
+	}
+
+	if (format->pixelformat == V4L2_PIX_FMT_NV12 ||
+			format->pixelformat == V4L2_PIX_FMT_NV21 ||
+			format->pixelformat == V4L2_PIX_FMT_NV16 ||
+			format->pixelformat == V4L2_PIX_FMT_NV61)
+		buffer->addr[1] = buffer->addr[0] +
+				format->plane_fmt[0].bytesperline *
+				format->height;
+
+	return 0;
+}
+
+static int video_buf_prepare(struct vb2_buffer *vb)
+{
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct camss_video *video = vb2_get_drv_priv(vb->vb2_queue);
+	const struct v4l2_pix_format_mplane *format =
+						&video->active_fmt.fmt.pix_mp;
+	unsigned int i;
+
+	for (i = 0; i < format->num_planes; i++) {
+		if (format->plane_fmt[i].sizeimage > vb2_plane_size(vb, i))
+			return -EINVAL;
+
+		vb2_set_plane_payload(vb, i, format->plane_fmt[i].sizeimage);
+	}
+
+	vbuf->field = V4L2_FIELD_NONE;
+
+	return 0;
+}
+
+static void video_buf_queue(struct vb2_buffer *vb)
+{
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct camss_video *video = vb2_get_drv_priv(vb->vb2_queue);
+	struct camss_buffer *buffer = container_of(vbuf, struct camss_buffer,
+						   vb);
+
+	video->ops->queue_buffer(video, buffer);
+}
+
+static int video_check_format(struct camss_video *video)
+{
+	struct v4l2_pix_format_mplane *pix = &video->active_fmt.fmt.pix_mp;
+	struct v4l2_format format;
+	struct v4l2_pix_format_mplane *sd_pix = &format.fmt.pix_mp;
+	int ret;
+
+	sd_pix->pixelformat = pix->pixelformat;
+	ret = video_get_subdev_format(video, &format);
+	if (ret < 0)
+		return ret;
+
+	if (pix->pixelformat != sd_pix->pixelformat ||
+	    pix->height != sd_pix->height ||
+	    pix->width != sd_pix->width ||
+	    pix->num_planes != sd_pix->num_planes ||
+	    pix->field != format.fmt.pix_mp.field)
+		return -EPIPE;
+
+	return 0;
+}
+
+static int video_start_streaming(struct vb2_queue *q, unsigned int count)
+{
+	struct camss_video *video = vb2_get_drv_priv(q);
+	struct video_device *vdev = &video->vdev;
+	struct media_entity *entity;
+	struct media_pad *pad;
+	struct v4l2_subdev *subdev;
+	int ret;
+
+	ret = media_pipeline_start(&vdev->entity, &video->pipe);
+	if (ret < 0)
+		return ret;
+
+	ret = video_check_format(video);
+	if (ret < 0)
+		goto error;
+
+	entity = &vdev->entity;
+	while (1) {
+		pad = &entity->pads[0];
+		if (!(pad->flags & MEDIA_PAD_FL_SINK))
+			break;
+
+		pad = media_entity_remote_pad(pad);
+		if (!pad || !is_media_entity_v4l2_subdev(pad->entity))
+			break;
+
+		entity = pad->entity;
+		subdev = media_entity_to_v4l2_subdev(entity);
+
+		ret = v4l2_subdev_call(subdev, video, s_stream, 1);
+		if (ret < 0 && ret != -ENOIOCTLCMD)
+			goto error;
+	}
+
+	return 0;
+
+error:
+	media_pipeline_stop(&vdev->entity);
+
+	video->ops->flush_buffers(video, VB2_BUF_STATE_QUEUED);
+
+	return ret;
+}
+
+static void video_stop_streaming(struct vb2_queue *q)
+{
+	struct camss_video *video = vb2_get_drv_priv(q);
+	struct video_device *vdev = &video->vdev;
+	struct media_entity *entity;
+	struct media_pad *pad;
+	struct v4l2_subdev *subdev;
+
+	entity = &vdev->entity;
+	while (1) {
+		pad = &entity->pads[0];
+		if (!(pad->flags & MEDIA_PAD_FL_SINK))
+			break;
+
+		pad = media_entity_remote_pad(pad);
+		if (!pad || !is_media_entity_v4l2_subdev(pad->entity))
+			break;
+
+		entity = pad->entity;
+		subdev = media_entity_to_v4l2_subdev(entity);
+
+		v4l2_subdev_call(subdev, video, s_stream, 0);
+	}
+
+	media_pipeline_stop(&vdev->entity);
+
+	video->ops->flush_buffers(video, VB2_BUF_STATE_ERROR);
+}
+
+static const struct vb2_ops msm_video_vb2_q_ops = {
+	.queue_setup     = video_queue_setup,
+	.wait_prepare    = vb2_ops_wait_prepare,
+	.wait_finish     = vb2_ops_wait_finish,
+	.buf_init        = video_buf_init,
+	.buf_prepare     = video_buf_prepare,
+	.buf_queue       = video_buf_queue,
+	.start_streaming = video_start_streaming,
+	.stop_streaming  = video_stop_streaming,
+};
+
+/* -----------------------------------------------------------------------------
+ * V4L2 ioctls
+ */
+
+static int video_querycap(struct file *file, void *fh,
+			  struct v4l2_capability *cap)
+{
+	struct camss_video *video = video_drvdata(file);
+
+	strlcpy(cap->driver, "qcom-camss", sizeof(cap->driver));
+	strlcpy(cap->card, "Qualcomm Camera Subsystem", sizeof(cap->card));
+	snprintf(cap->bus_info, sizeof(cap->bus_info), "platform:%s",
+		 dev_name(video->camss->dev));
+
+	return 0;
+}
+
+static int video_enum_fmt(struct file *file, void *fh, struct v4l2_fmtdesc *f)
+{
+	struct camss_video *video = video_drvdata(file);
+	int i, j, k;
+
+	if (f->type != video->type)
+		return -EINVAL;
+
+	if (f->index >= video->nformats)
+		return -EINVAL;
+
+	/* find index "i" of "k"th unique pixelformat in formats array */
+	k = -1;
+	for (i = 0; i < video->nformats; i++) {
+		for (j = 0; j < i; j++) {
+			if (video->formats[i].pixelformat ==
+					video->formats[j].pixelformat)
+				break;
+		}
+
+		if (j == i)
+			k++;
+
+		if (k == f->index)
+			break;
+	}
+
+	if (k < f->index)
+		return -EINVAL;
+
+	f->pixelformat = video->formats[i].pixelformat;
+
+	return 0;
+}
+
+static int video_g_fmt(struct file *file, void *fh, struct v4l2_format *f)
+{
+	struct camss_video *video = video_drvdata(file);
+
+	*f = video->active_fmt;
+
+	return 0;
+}
+
+static int __video_try_fmt(struct camss_video *video, struct v4l2_format *f)
+{
+	struct v4l2_pix_format_mplane *pix_mp;
+	const struct camss_format_info *fi;
+	struct v4l2_plane_pix_format *p;
+	u32 bytesperline[3] = { 0 };
+	u32 sizeimage[3] = { 0 };
+	u32 width, height;
+	u32 bpl, lines;
+	int i, j;
+
+	pix_mp = &f->fmt.pix_mp;
+
+	if (video->line_based)
+		for (i = 0; i < pix_mp->num_planes && i < 3; i++) {
+			p = &pix_mp->plane_fmt[i];
+			bytesperline[i] = clamp_t(u32, p->bytesperline,
+						  1, 65528);
+			sizeimage[i] = clamp_t(u32, p->sizeimage,
+					       bytesperline[i],
+					       bytesperline[i] * 4096);
+		}
+
+	for (j = 0; j < video->nformats; j++)
+		if (pix_mp->pixelformat == video->formats[j].pixelformat)
+			break;
+
+	if (j == video->nformats)
+		j = 0; /* default format */
+
+	fi = &video->formats[j];
+	width = pix_mp->width;
+	height = pix_mp->height;
+
+	memset(pix_mp, 0, sizeof(*pix_mp));
+
+	pix_mp->pixelformat = fi->pixelformat;
+	pix_mp->width = clamp_t(u32, width, 1, 8191);
+	pix_mp->height = clamp_t(u32, height, 1, 8191);
+	pix_mp->num_planes = fi->planes;
+	for (i = 0; i < pix_mp->num_planes; i++) {
+		bpl = pix_mp->width / fi->hsub[i].numerator *
+			fi->hsub[i].denominator * fi->bpp[i] / 8;
+		bpl = ALIGN(bpl, video->bpl_alignment);
+		pix_mp->plane_fmt[i].bytesperline = bpl;
+		pix_mp->plane_fmt[i].sizeimage = pix_mp->height /
+			fi->vsub[i].numerator * fi->vsub[i].denominator * bpl;
+	}
+
+	pix_mp->field = V4L2_FIELD_NONE;
+	pix_mp->colorspace = V4L2_COLORSPACE_SRGB;
+	pix_mp->flags = 0;
+	pix_mp->ycbcr_enc = V4L2_MAP_YCBCR_ENC_DEFAULT(pix_mp->colorspace);
+	pix_mp->quantization = V4L2_MAP_QUANTIZATION_DEFAULT(true,
+					pix_mp->colorspace, pix_mp->ycbcr_enc);
+	pix_mp->xfer_func = V4L2_MAP_XFER_FUNC_DEFAULT(pix_mp->colorspace);
+
+	if (video->line_based)
+		for (i = 0; i < pix_mp->num_planes; i++) {
+			p = &pix_mp->plane_fmt[i];
+			p->bytesperline = clamp_t(u32, p->bytesperline,
+						  1, 65528);
+			p->sizeimage = clamp_t(u32, p->sizeimage,
+					       p->bytesperline,
+					       p->bytesperline * 4096);
+			lines = p->sizeimage / p->bytesperline;
+
+			if (p->bytesperline < bytesperline[i])
+				p->bytesperline = ALIGN(bytesperline[i], 8);
+
+			if (p->sizeimage < p->bytesperline * lines)
+				p->sizeimage = p->bytesperline * lines;
+
+			if (p->sizeimage < sizeimage[i])
+				p->sizeimage = sizeimage[i];
+		}
+
+	return 0;
+}
+
+static int video_try_fmt(struct file *file, void *fh, struct v4l2_format *f)
+{
+	struct camss_video *video = video_drvdata(file);
+
+	return __video_try_fmt(video, f);
+}
+
+static int video_s_fmt(struct file *file, void *fh, struct v4l2_format *f)
+{
+	struct camss_video *video = video_drvdata(file);
+	int ret;
+
+	if (vb2_is_busy(&video->vb2_q))
+		return -EBUSY;
+
+	ret = __video_try_fmt(video, f);
+	if (ret < 0)
+		return ret;
+
+	video->active_fmt = *f;
+
+	return 0;
+}
+
+static int video_enum_input(struct file *file, void *fh,
+			    struct v4l2_input *input)
+{
+	if (input->index > 0)
+		return -EINVAL;
+
+	strlcpy(input->name, "camera", sizeof(input->name));
+	input->type = V4L2_INPUT_TYPE_CAMERA;
+
+	return 0;
+}
+
+static int video_g_input(struct file *file, void *fh, unsigned int *input)
+{
+	*input = 0;
+
+	return 0;
+}
+
+static int video_s_input(struct file *file, void *fh, unsigned int input)
+{
+	return input == 0 ? 0 : -EINVAL;
+}
+
+static const struct v4l2_ioctl_ops msm_vid_ioctl_ops = {
+	.vidioc_querycap		= video_querycap,
+	.vidioc_enum_fmt_vid_cap_mplane	= video_enum_fmt,
+	.vidioc_g_fmt_vid_cap_mplane	= video_g_fmt,
+	.vidioc_s_fmt_vid_cap_mplane	= video_s_fmt,
+	.vidioc_try_fmt_vid_cap_mplane	= video_try_fmt,
+	.vidioc_reqbufs			= vb2_ioctl_reqbufs,
+	.vidioc_querybuf		= vb2_ioctl_querybuf,
+	.vidioc_qbuf			= vb2_ioctl_qbuf,
+	.vidioc_expbuf			= vb2_ioctl_expbuf,
+	.vidioc_dqbuf			= vb2_ioctl_dqbuf,
+	.vidioc_create_bufs		= vb2_ioctl_create_bufs,
+	.vidioc_prepare_buf		= vb2_ioctl_prepare_buf,
+	.vidioc_streamon		= vb2_ioctl_streamon,
+	.vidioc_streamoff		= vb2_ioctl_streamoff,
+	.vidioc_enum_input		= video_enum_input,
+	.vidioc_g_input			= video_g_input,
+	.vidioc_s_input			= video_s_input,
+};
+
+/* -----------------------------------------------------------------------------
+ * V4L2 file operations
+ */
+
+static int video_open(struct file *file)
+{
+	struct video_device *vdev = video_devdata(file);
+	struct camss_video *video = video_drvdata(file);
+	struct v4l2_fh *vfh;
+	int ret;
+
+	mutex_lock(&video->lock);
+
+	vfh = kzalloc(sizeof(*vfh), GFP_KERNEL);
+	if (vfh == NULL) {
+		ret = -ENOMEM;
+		goto error_alloc;
+	}
+
+	v4l2_fh_init(vfh, vdev);
+	v4l2_fh_add(vfh);
+
+	file->private_data = vfh;
+
+	ret = v4l2_pipeline_pm_use(&vdev->entity, 1);
+	if (ret < 0) {
+		dev_err(video->camss->dev, "Failed to power up pipeline: %d\n",
+			ret);
+		goto error_pm_use;
+	}
+
+	mutex_unlock(&video->lock);
+
+	return 0;
+
+error_pm_use:
+	v4l2_fh_release(file);
+
+error_alloc:
+	mutex_unlock(&video->lock);
+
+	return ret;
+}
+
+static int video_release(struct file *file)
+{
+	struct video_device *vdev = video_devdata(file);
+
+	vb2_fop_release(file);
+
+	v4l2_pipeline_pm_use(&vdev->entity, 0);
+
+	file->private_data = NULL;
+
+	return 0;
+}
+
+static const struct v4l2_file_operations msm_vid_fops = {
+	.owner          = THIS_MODULE,
+	.unlocked_ioctl = video_ioctl2,
+	.open           = video_open,
+	.release        = video_release,
+	.poll           = vb2_fop_poll,
+	.mmap		= vb2_fop_mmap,
+	.read		= vb2_fop_read,
+};
+
+/* -----------------------------------------------------------------------------
+ * CAMSS video core
+ */
+
+static void msm_video_release(struct video_device *vdev)
+{
+	struct camss_video *video = video_get_drvdata(vdev);
+
+	media_entity_cleanup(&vdev->entity);
+
+	mutex_destroy(&video->q_lock);
+	mutex_destroy(&video->lock);
+
+	if (atomic_dec_and_test(&video->camss->ref_count))
+		camss_delete(video->camss);
+}
+
+/*
+ * msm_video_init_format - Helper function to initialize format
+ * @video: struct camss_video
+ *
+ * Initialize pad format with default value.
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+static int msm_video_init_format(struct camss_video *video)
+{
+	int ret;
+	struct v4l2_format format = {
+		.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
+		.fmt.pix_mp = {
+			.width = 1920,
+			.height = 1080,
+			.pixelformat = video->formats[0].pixelformat,
+		},
+	};
+
+	ret = __video_try_fmt(video, &format);
+	if (ret < 0)
+		return ret;
+
+	video->active_fmt = format;
+
+	return 0;
+}
+
+/*
+ * msm_video_register - Register a video device node
+ * @video: struct camss_video
+ * @v4l2_dev: V4L2 device
+ * @name: name to be used for the video device node
+ *
+ * Initialize and register a video device node to a V4L2 device. Also
+ * initialize the vb2 queue.
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+
+int msm_video_register(struct camss_video *video, struct v4l2_device *v4l2_dev,
+		       const char *name, int is_pix)
+{
+	struct media_pad *pad = &video->pad;
+	struct video_device *vdev;
+	struct vb2_queue *q;
+	int ret;
+
+	vdev = &video->vdev;
+
+	mutex_init(&video->q_lock);
+
+	q = &video->vb2_q;
+	q->drv_priv = video;
+	q->mem_ops = &vb2_dma_sg_memops;
+	q->ops = &msm_video_vb2_q_ops;
+	q->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+	q->io_modes = VB2_DMABUF | VB2_MMAP | VB2_READ;
+	q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
+	q->buf_struct_size = sizeof(struct camss_buffer);
+	q->dev = video->camss->dev;
+	q->lock = &video->q_lock;
+	ret = vb2_queue_init(q);
+	if (ret < 0) {
+		dev_err(v4l2_dev->dev, "Failed to init vb2 queue: %d\n", ret);
+		goto error_vb2_init;
+	}
+
+	pad->flags = MEDIA_PAD_FL_SINK;
+	ret = media_entity_pads_init(&vdev->entity, 1, pad);
+	if (ret < 0) {
+		dev_err(v4l2_dev->dev, "Failed to init video entity: %d\n",
+			ret);
+		goto error_media_init;
+	}
+
+	mutex_init(&video->lock);
+
+	video->formats = formats_rdi;
+	video->nformats = ARRAY_SIZE(formats_rdi);
+	if (is_pix) {
+		video->formats = formats_pix;
+		video->nformats = ARRAY_SIZE(formats_pix);
+	}
+
+	ret = msm_video_init_format(video);
+	if (ret < 0) {
+		dev_err(v4l2_dev->dev, "Failed to init format: %d\n", ret);
+		goto error_video_register;
+	}
+
+	vdev->fops = &msm_vid_fops;
+	vdev->device_caps = V4L2_CAP_VIDEO_CAPTURE_MPLANE | V4L2_CAP_STREAMING |
+							V4L2_CAP_READWRITE;
+	vdev->ioctl_ops = &msm_vid_ioctl_ops;
+	vdev->release = msm_video_release;
+	vdev->v4l2_dev = v4l2_dev;
+	vdev->vfl_dir = VFL_DIR_RX;
+	vdev->queue = &video->vb2_q;
+	vdev->lock = &video->lock;
+	strlcpy(vdev->name, name, sizeof(vdev->name));
+
+	ret = video_register_device(vdev, VFL_TYPE_GRABBER, -1);
+	if (ret < 0) {
+		dev_err(v4l2_dev->dev, "Failed to register video device: %d\n",
+			ret);
+		goto error_video_register;
+	}
+
+	video_set_drvdata(vdev, video);
+	atomic_inc(&video->camss->ref_count);
+
+	return 0;
+
+error_video_register:
+	media_entity_cleanup(&vdev->entity);
+	mutex_destroy(&video->lock);
+error_media_init:
+	vb2_queue_release(&video->vb2_q);
+error_vb2_init:
+	mutex_destroy(&video->q_lock);
+
+	return ret;
+}
+
+void msm_video_stop_streaming(struct camss_video *video)
+{
+	if (vb2_is_streaming(&video->vb2_q))
+		vb2_queue_release(&video->vb2_q);
+}
+
+void msm_video_unregister(struct camss_video *video)
+{
+	atomic_inc(&video->camss->ref_count);
+	video_unregister_device(&video->vdev);
+	atomic_dec(&video->camss->ref_count);
+}
diff --git a/drivers/media/platform/qcom/camss-8x16/camss-video.h b/drivers/media/platform/qcom/camss-8x16/camss-video.h
new file mode 100644
index 000000000000..38bd1f2eec54
--- /dev/null
+++ b/drivers/media/platform/qcom/camss-8x16/camss-video.h
@@ -0,0 +1,70 @@
+/*
+ * camss-video.h
+ *
+ * Qualcomm MSM Camera Subsystem - V4L2 device node
+ *
+ * Copyright (c) 2013-2015, The Linux Foundation. All rights reserved.
+ * Copyright (C) 2015-2017 Linaro Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#ifndef QC_MSM_CAMSS_VIDEO_H
+#define QC_MSM_CAMSS_VIDEO_H
+
+#include <linux/mutex.h>
+#include <linux/videodev2.h>
+#include <media/media-entity.h>
+#include <media/v4l2-dev.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-fh.h>
+#include <media/v4l2-mediabus.h>
+#include <media/videobuf2-v4l2.h>
+
+struct camss_buffer {
+	struct vb2_v4l2_buffer vb;
+	dma_addr_t addr[3];
+	struct list_head queue;
+};
+
+struct camss_video;
+
+struct camss_video_ops {
+	int (*queue_buffer)(struct camss_video *vid, struct camss_buffer *buf);
+	int (*flush_buffers)(struct camss_video *vid,
+			     enum vb2_buffer_state state);
+};
+
+struct camss_format_info;
+
+struct camss_video {
+	struct camss *camss;
+	struct vb2_queue vb2_q;
+	struct video_device vdev;
+	struct media_pad pad;
+	struct v4l2_format active_fmt;
+	enum v4l2_buf_type type;
+	struct media_pipeline pipe;
+	const struct camss_video_ops *ops;
+	struct mutex lock;
+	struct mutex q_lock;
+	unsigned int bpl_alignment;
+	unsigned int line_based;
+	const struct camss_format_info *formats;
+	unsigned int nformats;
+};
+
+void msm_video_stop_streaming(struct camss_video *video);
+
+int msm_video_register(struct camss_video *video, struct v4l2_device *v4l2_dev,
+		       const char *name, int is_pix);
+
+void msm_video_unregister(struct camss_video *video);
+
+#endif /* QC_MSM_CAMSS_VIDEO_H */
diff --git a/drivers/media/platform/qcom/camss-8x16/camss.c b/drivers/media/platform/qcom/camss-8x16/camss.c
new file mode 100644
index 000000000000..a3760b5dd1d1
--- /dev/null
+++ b/drivers/media/platform/qcom/camss-8x16/camss.c
@@ -0,0 +1,746 @@
+/*
+ * camss.c
+ *
+ * Qualcomm MSM Camera Subsystem - Core
+ *
+ * Copyright (c) 2015, The Linux Foundation. All rights reserved.
+ * Copyright (C) 2015-2017 Linaro Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#include <linux/clk.h>
+#include <linux/media-bus-format.h>
+#include <linux/media.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/of.h>
+#include <linux/of_graph.h>
+#include <linux/slab.h>
+#include <linux/videodev2.h>
+
+#include <media/media-device.h>
+#include <media/v4l2-async.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-mc.h>
+#include <media/v4l2-fwnode.h>
+
+#include "camss.h"
+
+#define CAMSS_CLOCK_MARGIN_NUMERATOR 105
+#define CAMSS_CLOCK_MARGIN_DENOMINATOR 100
+
+static const struct resources csiphy_res[] = {
+	/* CSIPHY0 */
+	{
+		.regulator = { NULL },
+		.clock = { "camss_top_ahb", "ispif_ahb",
+			   "camss_ahb", "csiphy0_timer" },
+		.clock_rate = { { 0 },
+				{ 0 },
+				{ 0 },
+				{ 100000000, 200000000 } },
+		.reg = { "csiphy0", "csiphy0_clk_mux" },
+		.interrupt = { "csiphy0" }
+	},
+
+	/* CSIPHY1 */
+	{
+		.regulator = { NULL },
+		.clock = { "camss_top_ahb", "ispif_ahb",
+			   "camss_ahb", "csiphy1_timer" },
+		.clock_rate = { { 0 },
+				{ 0 },
+				{ 0 },
+				{ 100000000, 200000000 } },
+		.reg = { "csiphy1", "csiphy1_clk_mux" },
+		.interrupt = { "csiphy1" }
+	}
+};
+
+static const struct resources csid_res[] = {
+	/* CSID0 */
+	{
+		.regulator = { "vdda" },
+		.clock = { "camss_top_ahb", "ispif_ahb",
+			   "csi0_ahb", "camss_ahb",
+			   "csi0", "csi0_phy", "csi0_pix", "csi0_rdi" },
+		.clock_rate = { { 0 },
+				{ 0 },
+				{ 0 },
+				{ 0 },
+				{ 100000000, 200000000 },
+				{ 0 },
+				{ 0 },
+				{ 0 } },
+		.reg = { "csid0" },
+		.interrupt = { "csid0" }
+	},
+
+	/* CSID1 */
+	{
+		.regulator = { "vdda" },
+		.clock = { "camss_top_ahb", "ispif_ahb",
+			   "csi1_ahb", "camss_ahb",
+			   "csi1", "csi1_phy", "csi1_pix", "csi1_rdi" },
+		.clock_rate = { { 0 },
+				{ 0 },
+				{ 0 },
+				{ 0 },
+				{ 100000000, 200000000 },
+				{ 0 },
+				{ 0 },
+				{ 0 } },
+		.reg = { "csid1" },
+		.interrupt = { "csid1" }
+	},
+};
+
+static const struct resources_ispif ispif_res = {
+	/* ISPIF */
+	.clock = { "camss_top_ahb", "camss_ahb", "ispif_ahb",
+		   "csi0", "csi0_pix", "csi0_rdi",
+		   "csi1", "csi1_pix", "csi1_rdi" },
+	.clock_for_reset = { "camss_vfe_vfe", "camss_csi_vfe" },
+	.reg = { "ispif", "csi_clk_mux" },
+	.interrupt = "ispif"
+
+};
+
+static const struct resources vfe_res = {
+	/* VFE0 */
+	.regulator = { NULL },
+	.clock = { "camss_top_ahb", "camss_vfe_vfe", "camss_csi_vfe",
+		   "iface", "bus", "camss_ahb" },
+	.clock_rate = { { 0 },
+			{ 50000000, 80000000, 100000000, 160000000,
+			  177780000, 200000000, 266670000, 320000000,
+			  400000000, 465000000 },
+			{ 0 },
+			{ 0 },
+			{ 0 },
+			{ 0 },
+			{ 0 },
+			{ 0 },
+			{ 0 } },
+	.reg = { "vfe0" },
+	.interrupt = { "vfe0" }
+};
+
+/*
+ * camss_add_clock_margin - Add margin to clock frequency rate
+ * @rate: Clock frequency rate
+ *
+ * When making calculations with physical clock frequency values
+ * some safety margin must be added. Add it.
+ */
+inline void camss_add_clock_margin(u64 *rate)
+{
+	*rate *= CAMSS_CLOCK_MARGIN_NUMERATOR;
+	*rate = div_u64(*rate, CAMSS_CLOCK_MARGIN_DENOMINATOR);
+}
+
+/*
+ * camss_enable_clocks - Enable multiple clocks
+ * @nclocks: Number of clocks in clock array
+ * @clock: Clock array
+ * @dev: Device
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+int camss_enable_clocks(int nclocks, struct camss_clock *clock,
+			struct device *dev)
+{
+	int ret;
+	int i;
+
+	for (i = 0; i < nclocks; i++) {
+		ret = clk_prepare_enable(clock[i].clk);
+		if (ret) {
+			dev_err(dev, "clock enable failed: %d\n", ret);
+			goto error;
+		}
+	}
+
+	return 0;
+
+error:
+	for (i--; i >= 0; i--)
+		clk_disable_unprepare(clock[i].clk);
+
+	return ret;
+}
+
+/*
+ * camss_disable_clocks - Disable multiple clocks
+ * @nclocks: Number of clocks in clock array
+ * @clock: Clock array
+ */
+void camss_disable_clocks(int nclocks, struct camss_clock *clock)
+{
+	int i;
+
+	for (i = nclocks - 1; i >= 0; i--)
+		clk_disable_unprepare(clock[i].clk);
+}
+
+/*
+ * camss_find_sensor - Find a linked media entity which represents a sensor
+ * @entity: Media entity to start searching from
+ *
+ * Return a pointer to sensor media entity or NULL if not found
+ */
+static struct media_entity *camss_find_sensor(struct media_entity *entity)
+{
+	struct media_pad *pad;
+
+	while (1) {
+		pad = &entity->pads[0];
+		if (!(pad->flags & MEDIA_PAD_FL_SINK))
+			return NULL;
+
+		pad = media_entity_remote_pad(pad);
+		if (!pad || !is_media_entity_v4l2_subdev(pad->entity))
+			return NULL;
+
+		entity = pad->entity;
+
+		if (entity->function == MEDIA_ENT_F_CAM_SENSOR)
+			return entity;
+	}
+}
+
+/*
+ * camss_get_pixel_clock - Get pixel clock rate from sensor
+ * @entity: Media entity in the current pipeline
+ * @pixel_clock: Received pixel clock value
+ *
+ * Return 0 on success or a negative error code otherwise
+ */
+int camss_get_pixel_clock(struct media_entity *entity, u32 *pixel_clock)
+{
+	struct media_entity *sensor;
+	struct v4l2_subdev *subdev;
+	struct v4l2_ctrl *ctrl;
+
+	sensor = camss_find_sensor(entity);
+	if (!sensor)
+		return -ENODEV;
+
+	subdev = media_entity_to_v4l2_subdev(sensor);
+
+	ctrl = v4l2_ctrl_find(subdev->ctrl_handler, V4L2_CID_PIXEL_RATE);
+
+	if (!ctrl)
+		return -EINVAL;
+
+	*pixel_clock = v4l2_ctrl_g_ctrl_int64(ctrl);
+
+	return 0;
+}
+
+/*
+ * camss_of_parse_endpoint_node - Parse port endpoint node
+ * @dev: Device
+ * @node: Device node to be parsed
+ * @csd: Parsed data from port endpoint node
+ *
+ * Return 0 on success or a negative error code on failure
+ */
+static int camss_of_parse_endpoint_node(struct device *dev,
+					struct device_node *node,
+					struct camss_async_subdev *csd)
+{
+	struct csiphy_lanes_cfg *lncfg = &csd->interface.csi2.lane_cfg;
+	struct v4l2_fwnode_bus_mipi_csi2 *mipi_csi2;
+	struct v4l2_fwnode_endpoint vep = { { 0 } };
+	unsigned int i;
+
+	v4l2_fwnode_endpoint_parse(of_fwnode_handle(node), &vep);
+
+	csd->interface.csiphy_id = vep.base.port;
+
+	mipi_csi2 = &vep.bus.mipi_csi2;
+	lncfg->clk.pos = mipi_csi2->clock_lane;
+	lncfg->clk.pol = mipi_csi2->lane_polarities[0];
+	lncfg->num_data = mipi_csi2->num_data_lanes;
+
+	lncfg->data = devm_kzalloc(dev, lncfg->num_data * sizeof(*lncfg->data),
+				   GFP_KERNEL);
+	if (!lncfg->data)
+		return -ENOMEM;
+
+	for (i = 0; i < lncfg->num_data; i++) {
+		lncfg->data[i].pos = mipi_csi2->data_lanes[i];
+		lncfg->data[i].pol = mipi_csi2->lane_polarities[i + 1];
+	}
+
+	return 0;
+}
+
+/*
+ * camss_of_parse_ports - Parse ports node
+ * @dev: Device
+ * @notifier: v4l2_device notifier data
+ *
+ * Return number of "port" nodes found in "ports" node
+ */
+static int camss_of_parse_ports(struct device *dev,
+				struct v4l2_async_notifier *notifier)
+{
+	struct device_node *node = NULL;
+	struct device_node *remote = NULL;
+	unsigned int size, i;
+	int ret;
+
+	while ((node = of_graph_get_next_endpoint(dev->of_node, node)))
+		if (of_device_is_available(node))
+			notifier->num_subdevs++;
+
+	size = sizeof(*notifier->subdevs) * notifier->num_subdevs;
+	notifier->subdevs = devm_kzalloc(dev, size, GFP_KERNEL);
+	if (!notifier->subdevs) {
+		dev_err(dev, "Failed to allocate memory\n");
+		return -ENOMEM;
+	}
+
+	i = 0;
+	while ((node = of_graph_get_next_endpoint(dev->of_node, node))) {
+		struct camss_async_subdev *csd;
+
+		if (!of_device_is_available(node))
+			continue;
+
+		csd = devm_kzalloc(dev, sizeof(*csd), GFP_KERNEL);
+		if (!csd) {
+			of_node_put(node);
+			dev_err(dev, "Failed to allocate memory\n");
+			return -ENOMEM;
+		}
+
+		notifier->subdevs[i++] = &csd->asd;
+
+		ret = camss_of_parse_endpoint_node(dev, node, csd);
+		if (ret < 0) {
+			of_node_put(node);
+			return ret;
+		}
+
+		remote = of_graph_get_remote_port_parent(node);
+		of_node_put(node);
+
+		if (!remote) {
+			dev_err(dev, "Cannot get remote parent\n");
+			return -EINVAL;
+		}
+
+		csd->asd.match_type = V4L2_ASYNC_MATCH_FWNODE;
+		csd->asd.match.fwnode.fwnode = of_fwnode_handle(remote);
+	}
+
+	return notifier->num_subdevs;
+}
+
+/*
+ * camss_init_subdevices - Initialize subdev structures and resources
+ * @camss: CAMSS device
+ *
+ * Return 0 on success or a negative error code on failure
+ */
+static int camss_init_subdevices(struct camss *camss)
+{
+	unsigned int i;
+	int ret;
+
+	for (i = 0; i < ARRAY_SIZE(camss->csiphy); i++) {
+		ret = msm_csiphy_subdev_init(&camss->csiphy[i],
+					     &csiphy_res[i], i);
+		if (ret < 0) {
+			dev_err(camss->dev,
+				"Failed to init csiphy%d sub-device: %d\n",
+				i, ret);
+			return ret;
+		}
+	}
+
+	for (i = 0; i < ARRAY_SIZE(camss->csid); i++) {
+		ret = msm_csid_subdev_init(&camss->csid[i],
+					   &csid_res[i], i);
+		if (ret < 0) {
+			dev_err(camss->dev,
+				"Failed to init csid%d sub-device: %d\n",
+				i, ret);
+			return ret;
+		}
+	}
+
+	ret = msm_ispif_subdev_init(&camss->ispif, &ispif_res);
+	if (ret < 0) {
+		dev_err(camss->dev, "Failed to init ispif sub-device: %d\n",
+			ret);
+		return ret;
+	}
+
+	ret = msm_vfe_subdev_init(&camss->vfe, &vfe_res);
+	if (ret < 0) {
+		dev_err(camss->dev, "Fail to init vfe sub-device: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+/*
+ * camss_register_entities - Register subdev nodes and create links
+ * @camss: CAMSS device
+ *
+ * Return 0 on success or a negative error code on failure
+ */
+static int camss_register_entities(struct camss *camss)
+{
+	int i, j;
+	int ret;
+
+	for (i = 0; i < ARRAY_SIZE(camss->csiphy); i++) {
+		ret = msm_csiphy_register_entity(&camss->csiphy[i],
+						 &camss->v4l2_dev);
+		if (ret < 0) {
+			dev_err(camss->dev,
+				"Failed to register csiphy%d entity: %d\n",
+				i, ret);
+			goto err_reg_csiphy;
+		}
+	}
+
+	for (i = 0; i < ARRAY_SIZE(camss->csid); i++) {
+		ret = msm_csid_register_entity(&camss->csid[i],
+					       &camss->v4l2_dev);
+		if (ret < 0) {
+			dev_err(camss->dev,
+				"Failed to register csid%d entity: %d\n",
+				i, ret);
+			goto err_reg_csid;
+		}
+	}
+
+	ret = msm_ispif_register_entities(&camss->ispif, &camss->v4l2_dev);
+	if (ret < 0) {
+		dev_err(camss->dev, "Failed to register ispif entities: %d\n",
+			ret);
+		goto err_reg_ispif;
+	}
+
+	ret = msm_vfe_register_entities(&camss->vfe, &camss->v4l2_dev);
+	if (ret < 0) {
+		dev_err(camss->dev, "Failed to register vfe entities: %d\n",
+			ret);
+		goto err_reg_vfe;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(camss->csiphy); i++) {
+		for (j = 0; j < ARRAY_SIZE(camss->csid); j++) {
+			ret = media_create_pad_link(
+				&camss->csiphy[i].subdev.entity,
+				MSM_CSIPHY_PAD_SRC,
+				&camss->csid[j].subdev.entity,
+				MSM_CSID_PAD_SINK,
+				0);
+			if (ret < 0) {
+				dev_err(camss->dev,
+					"Failed to link %s->%s entities: %d\n",
+					camss->csiphy[i].subdev.entity.name,
+					camss->csid[j].subdev.entity.name,
+					ret);
+				goto err_link;
+			}
+		}
+	}
+
+	for (i = 0; i < ARRAY_SIZE(camss->csid); i++) {
+		for (j = 0; j < ARRAY_SIZE(camss->ispif.line); j++) {
+			ret = media_create_pad_link(
+				&camss->csid[i].subdev.entity,
+				MSM_CSID_PAD_SRC,
+				&camss->ispif.line[j].subdev.entity,
+				MSM_ISPIF_PAD_SINK,
+				0);
+			if (ret < 0) {
+				dev_err(camss->dev,
+					"Failed to link %s->%s entities: %d\n",
+					camss->csid[i].subdev.entity.name,
+					camss->ispif.line[j].subdev.entity.name,
+					ret);
+				goto err_link;
+			}
+		}
+	}
+
+	for (i = 0; i < ARRAY_SIZE(camss->ispif.line); i++) {
+		for (j = 0; j < ARRAY_SIZE(camss->vfe.line); j++) {
+			ret = media_create_pad_link(
+				&camss->ispif.line[i].subdev.entity,
+				MSM_ISPIF_PAD_SRC,
+				&camss->vfe.line[j].subdev.entity,
+				MSM_VFE_PAD_SINK,
+				0);
+			if (ret < 0) {
+				dev_err(camss->dev,
+					"Failed to link %s->%s entities: %d\n",
+					camss->ispif.line[i].subdev.entity.name,
+					camss->vfe.line[j].subdev.entity.name,
+					ret);
+				goto err_link;
+			}
+		}
+	}
+
+	return 0;
+
+err_link:
+	msm_vfe_unregister_entities(&camss->vfe);
+err_reg_vfe:
+	msm_ispif_unregister_entities(&camss->ispif);
+err_reg_ispif:
+
+	i = ARRAY_SIZE(camss->csid);
+err_reg_csid:
+	for (i--; i >= 0; i--)
+		msm_csid_unregister_entity(&camss->csid[i]);
+
+	i = ARRAY_SIZE(camss->csiphy);
+err_reg_csiphy:
+	for (i--; i >= 0; i--)
+		msm_csiphy_unregister_entity(&camss->csiphy[i]);
+
+	return ret;
+}
+
+/*
+ * camss_unregister_entities - Unregister subdev nodes
+ * @camss: CAMSS device
+ *
+ * Return 0 on success or a negative error code on failure
+ */
+static void camss_unregister_entities(struct camss *camss)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(camss->csiphy); i++)
+		msm_csiphy_unregister_entity(&camss->csiphy[i]);
+
+	for (i = 0; i < ARRAY_SIZE(camss->csid); i++)
+		msm_csid_unregister_entity(&camss->csid[i]);
+
+	msm_ispif_unregister_entities(&camss->ispif);
+	msm_vfe_unregister_entities(&camss->vfe);
+}
+
+static int camss_subdev_notifier_bound(struct v4l2_async_notifier *async,
+				       struct v4l2_subdev *subdev,
+				       struct v4l2_async_subdev *asd)
+{
+	struct camss *camss = container_of(async, struct camss, notifier);
+	struct camss_async_subdev *csd =
+		container_of(asd, struct camss_async_subdev, asd);
+	u8 id = csd->interface.csiphy_id;
+	struct csiphy_device *csiphy = &camss->csiphy[id];
+
+	csiphy->cfg.csi2 = &csd->interface.csi2;
+	subdev->host_priv = csiphy;
+
+	return 0;
+}
+
+static int camss_subdev_notifier_complete(struct v4l2_async_notifier *async)
+{
+	struct camss *camss = container_of(async, struct camss, notifier);
+	struct v4l2_device *v4l2_dev = &camss->v4l2_dev;
+	struct v4l2_subdev *sd;
+	int ret;
+
+	list_for_each_entry(sd, &v4l2_dev->subdevs, list) {
+		if (sd->host_priv) {
+			struct media_entity *sensor = &sd->entity;
+			struct csiphy_device *csiphy =
+					(struct csiphy_device *) sd->host_priv;
+			struct media_entity *input = &csiphy->subdev.entity;
+			unsigned int i;
+
+			for (i = 0; i < sensor->num_pads; i++) {
+				if (sensor->pads[i].flags & MEDIA_PAD_FL_SOURCE)
+					break;
+			}
+			if (i == sensor->num_pads) {
+				dev_err(camss->dev,
+					"No source pad in external entity\n");
+				return -EINVAL;
+			}
+
+			ret = media_create_pad_link(sensor, i,
+				input, MSM_CSIPHY_PAD_SINK,
+				MEDIA_LNK_FL_IMMUTABLE | MEDIA_LNK_FL_ENABLED);
+			if (ret < 0) {
+				dev_err(camss->dev,
+					"Failed to link %s->%s entities: %d\n",
+					sensor->name, input->name, ret);
+				return ret;
+			}
+		}
+	}
+
+	ret = v4l2_device_register_subdev_nodes(&camss->v4l2_dev);
+	if (ret < 0)
+		return ret;
+
+	return media_device_register(&camss->media_dev);
+}
+
+static const struct media_device_ops camss_media_ops = {
+	.link_notify = v4l2_pipeline_link_notify,
+};
+
+/*
+ * camss_probe - Probe CAMSS platform device
+ * @pdev: Pointer to CAMSS platform device
+ *
+ * Return 0 on success or a negative error code on failure
+ */
+static int camss_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct camss *camss;
+	int ret;
+
+	camss = kzalloc(sizeof(*camss), GFP_KERNEL);
+	if (!camss)
+		return -ENOMEM;
+
+	atomic_set(&camss->ref_count, 0);
+	camss->dev = dev;
+	platform_set_drvdata(pdev, camss);
+
+	ret = camss_of_parse_ports(dev, &camss->notifier);
+	if (ret < 0)
+		return ret;
+
+	ret = camss_init_subdevices(camss);
+	if (ret < 0)
+		return ret;
+
+	ret = dma_set_mask_and_coherent(dev, 0xffffffff);
+	if (ret)
+		return ret;
+
+	camss->media_dev.dev = camss->dev;
+	strlcpy(camss->media_dev.model, "Qualcomm Camera Subsystem",
+		sizeof(camss->media_dev.model));
+	camss->media_dev.ops = &camss_media_ops;
+	media_device_init(&camss->media_dev);
+
+	camss->v4l2_dev.mdev = &camss->media_dev;
+	ret = v4l2_device_register(camss->dev, &camss->v4l2_dev);
+	if (ret < 0) {
+		dev_err(dev, "Failed to register V4L2 device: %d\n", ret);
+		return ret;
+	}
+
+	ret = camss_register_entities(camss);
+	if (ret < 0)
+		goto err_register_entities;
+
+	if (camss->notifier.num_subdevs) {
+		camss->notifier.bound = camss_subdev_notifier_bound;
+		camss->notifier.complete = camss_subdev_notifier_complete;
+
+		ret = v4l2_async_notifier_register(&camss->v4l2_dev,
+						   &camss->notifier);
+		if (ret) {
+			dev_err(dev,
+				"Failed to register async subdev nodes: %d\n",
+				ret);
+			goto err_register_subdevs;
+		}
+	} else {
+		ret = v4l2_device_register_subdev_nodes(&camss->v4l2_dev);
+		if (ret < 0) {
+			dev_err(dev, "Failed to register subdev nodes: %d\n",
+				ret);
+			goto err_register_subdevs;
+		}
+
+		ret = media_device_register(&camss->media_dev);
+		if (ret < 0) {
+			dev_err(dev, "Failed to register media device: %d\n",
+				ret);
+			goto err_register_subdevs;
+		}
+	}
+
+	return 0;
+
+err_register_subdevs:
+	camss_unregister_entities(camss);
+err_register_entities:
+	v4l2_device_unregister(&camss->v4l2_dev);
+
+	return ret;
+}
+
+void camss_delete(struct camss *camss)
+{
+	v4l2_device_unregister(&camss->v4l2_dev);
+	media_device_unregister(&camss->media_dev);
+	media_device_cleanup(&camss->media_dev);
+
+	kfree(camss);
+}
+
+/*
+ * camss_remove - Remove CAMSS platform device
+ * @pdev: Pointer to CAMSS platform device
+ *
+ * Always returns 0.
+ */
+static int camss_remove(struct platform_device *pdev)
+{
+	struct camss *camss = platform_get_drvdata(pdev);
+
+	msm_vfe_stop_streaming(&camss->vfe);
+
+	v4l2_async_notifier_unregister(&camss->notifier);
+	camss_unregister_entities(camss);
+
+	if (atomic_read(&camss->ref_count) == 0)
+		camss_delete(camss);
+
+	return 0;
+}
+
+static const struct of_device_id camss_dt_match[] = {
+	{ .compatible = "qcom,msm8916-camss" },
+	{ }
+};
+
+MODULE_DEVICE_TABLE(of, camss_dt_match);
+
+static struct platform_driver qcom_camss_driver = {
+	.probe = camss_probe,
+	.remove = camss_remove,
+	.driver = {
+		.name = "qcom-camss",
+		.of_match_table = camss_dt_match,
+	},
+};
+
+module_platform_driver(qcom_camss_driver);
+
+MODULE_ALIAS("platform:qcom-camss");
+MODULE_DESCRIPTION("Qualcomm Camera Subsystem driver");
+MODULE_AUTHOR("Todor Tomov <todor.tomov@linaro.org>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/platform/qcom/camss-8x16/camss.h b/drivers/media/platform/qcom/camss-8x16/camss.h
new file mode 100644
index 000000000000..4ad223443e4b
--- /dev/null
+++ b/drivers/media/platform/qcom/camss-8x16/camss.h
@@ -0,0 +1,106 @@
+/*
+ * camss.h
+ *
+ * Qualcomm MSM Camera Subsystem - Core
+ *
+ * Copyright (c) 2015, The Linux Foundation. All rights reserved.
+ * Copyright (C) 2015-2017 Linaro Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#ifndef QC_MSM_CAMSS_H
+#define QC_MSM_CAMSS_H
+
+#include <linux/types.h>
+#include <media/v4l2-async.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-subdev.h>
+#include <media/media-device.h>
+#include <media/media-entity.h>
+#include <linux/device.h>
+
+#include "camss-csid.h"
+#include "camss-csiphy.h"
+#include "camss-ispif.h"
+#include "camss-vfe.h"
+
+#define CAMSS_CSID_NUM 2
+#define CAMSS_CSIPHY_NUM 2
+
+#define to_camss(ptr_module)	\
+	container_of(ptr_module, struct camss, ptr_module)
+
+#define to_device(ptr_module)	\
+	(to_camss(ptr_module)->dev)
+
+#define module_pointer(ptr_module, index)	\
+	((const struct ptr_module##_device (*)[]) &(ptr_module[-(index)]))
+
+#define to_camss_index(ptr_module, index)	\
+	container_of(module_pointer(ptr_module, index),	\
+		     struct camss, ptr_module)
+
+#define to_device_index(ptr_module, index)	\
+	(to_camss_index(ptr_module, index)->dev)
+
+#define CAMSS_RES_MAX 15
+
+struct resources {
+	char *regulator[CAMSS_RES_MAX];
+	char *clock[CAMSS_RES_MAX];
+	u32 clock_rate[CAMSS_RES_MAX][CAMSS_RES_MAX];
+	char *reg[CAMSS_RES_MAX];
+	char *interrupt[CAMSS_RES_MAX];
+};
+
+struct resources_ispif {
+	char *clock[CAMSS_RES_MAX];
+	char *clock_for_reset[CAMSS_RES_MAX];
+	char *reg[CAMSS_RES_MAX];
+	char *interrupt;
+};
+
+struct camss {
+	struct v4l2_device v4l2_dev;
+	struct v4l2_async_notifier notifier;
+	struct media_device media_dev;
+	struct device *dev;
+	struct csiphy_device csiphy[CAMSS_CSIPHY_NUM];
+	struct csid_device csid[CAMSS_CSID_NUM];
+	struct ispif_device ispif;
+	struct vfe_device vfe;
+	atomic_t ref_count;
+};
+
+struct camss_camera_interface {
+	u8 csiphy_id;
+	struct csiphy_csi2_cfg csi2;
+};
+
+struct camss_async_subdev {
+	struct camss_camera_interface interface;
+	struct v4l2_async_subdev asd;
+};
+
+struct camss_clock {
+	struct clk *clk;
+	const char *name;
+	u32 *freq;
+	u32 nfreqs;
+};
+
+void camss_add_clock_margin(u64 *rate);
+int camss_enable_clocks(int nclocks, struct camss_clock *clock,
+			struct device *dev);
+void camss_disable_clocks(int nclocks, struct camss_clock *clock);
+int camss_get_pixel_clock(struct media_entity *entity, u32 *pixel_clock);
+void camss_delete(struct camss *camss);
+
+#endif /* QC_MSM_CAMSS_H */
diff --git a/drivers/media/platform/qcom/venus/helpers.c b/drivers/media/platform/qcom/venus/helpers.c
index 5f4434c0a8f1..68933d208063 100644
--- a/drivers/media/platform/qcom/venus/helpers.c
+++ b/drivers/media/platform/qcom/venus/helpers.c
@@ -34,6 +34,55 @@ struct intbuf {
 	unsigned long attrs;
 };
 
+bool venus_helper_check_codec(struct venus_inst *inst, u32 v4l2_pixfmt)
+{
+	struct venus_core *core = inst->core;
+	u32 session_type = inst->session_type;
+	u32 codec;
+
+	switch (v4l2_pixfmt) {
+	case V4L2_PIX_FMT_H264:
+		codec = HFI_VIDEO_CODEC_H264;
+		break;
+	case V4L2_PIX_FMT_H263:
+		codec = HFI_VIDEO_CODEC_H263;
+		break;
+	case V4L2_PIX_FMT_MPEG1:
+		codec = HFI_VIDEO_CODEC_MPEG1;
+		break;
+	case V4L2_PIX_FMT_MPEG2:
+		codec = HFI_VIDEO_CODEC_MPEG2;
+		break;
+	case V4L2_PIX_FMT_MPEG4:
+		codec = HFI_VIDEO_CODEC_MPEG4;
+		break;
+	case V4L2_PIX_FMT_VC1_ANNEX_G:
+	case V4L2_PIX_FMT_VC1_ANNEX_L:
+		codec = HFI_VIDEO_CODEC_VC1;
+		break;
+	case V4L2_PIX_FMT_VP8:
+		codec = HFI_VIDEO_CODEC_VP8;
+		break;
+	case V4L2_PIX_FMT_VP9:
+		codec = HFI_VIDEO_CODEC_VP9;
+		break;
+	case V4L2_PIX_FMT_XVID:
+		codec = HFI_VIDEO_CODEC_DIVX;
+		break;
+	default:
+		return false;
+	}
+
+	if (session_type == VIDC_SESSION_TYPE_ENC && core->enc_codecs & codec)
+		return true;
+
+	if (session_type == VIDC_SESSION_TYPE_DEC && core->dec_codecs & codec)
+		return true;
+
+	return false;
+}
+EXPORT_SYMBOL_GPL(venus_helper_check_codec);
+
 static int intbufs_set_buffer(struct venus_inst *inst, u32 type)
 {
 	struct venus_core *core = inst->core;
@@ -243,7 +292,7 @@ static void return_buf_error(struct venus_inst *inst,
 	if (vbuf->vb2_buf.type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE)
 		v4l2_m2m_src_buf_remove_by_buf(m2m_ctx, vbuf);
 	else
-		v4l2_m2m_src_buf_remove_by_buf(m2m_ctx, vbuf);
+		v4l2_m2m_dst_buf_remove_by_buf(m2m_ctx, vbuf);
 
 	v4l2_m2m_buf_done(vbuf, VB2_BUF_STATE_ERROR);
 }
diff --git a/drivers/media/platform/qcom/venus/helpers.h b/drivers/media/platform/qcom/venus/helpers.h
index 6a061b417a93..971392be5df5 100644
--- a/drivers/media/platform/qcom/venus/helpers.h
+++ b/drivers/media/platform/qcom/venus/helpers.h
@@ -19,6 +19,7 @@
 
 struct venus_inst;
 
+bool venus_helper_check_codec(struct venus_inst *inst, u32 v4l2_pixfmt);
 struct vb2_v4l2_buffer *venus_helper_find_buf(struct venus_inst *inst,
 					      unsigned int type, u32 idx);
 void venus_helper_buffers_done(struct venus_inst *inst,
diff --git a/drivers/media/platform/qcom/venus/vdec.c b/drivers/media/platform/qcom/venus/vdec.c
index eb0c1c51cfef..da611a5eb670 100644
--- a/drivers/media/platform/qcom/venus/vdec.c
+++ b/drivers/media/platform/qcom/venus/vdec.c
@@ -102,7 +102,8 @@ static const struct venus_format vdec_formats[] = {
 	},
 };
 
-static const struct venus_format *find_format(u32 pixfmt, u32 type)
+static const struct venus_format *
+find_format(struct venus_inst *inst, u32 pixfmt, u32 type)
 {
 	const struct venus_format *fmt = vdec_formats;
 	unsigned int size = ARRAY_SIZE(vdec_formats);
@@ -116,11 +117,15 @@ static const struct venus_format *find_format(u32 pixfmt, u32 type)
 	if (i == size || fmt[i].type != type)
 		return NULL;
 
+	if (type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE &&
+	    !venus_helper_check_codec(inst, fmt[i].pixfmt))
+		return NULL;
+
 	return &fmt[i];
 }
 
 static const struct venus_format *
-find_format_by_index(unsigned int index, u32 type)
+find_format_by_index(struct venus_inst *inst, unsigned int index, u32 type)
 {
 	const struct venus_format *fmt = vdec_formats;
 	unsigned int size = ARRAY_SIZE(vdec_formats);
@@ -140,6 +145,10 @@ find_format_by_index(unsigned int index, u32 type)
 	if (i == size)
 		return NULL;
 
+	if (type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE &&
+	    !venus_helper_check_codec(inst, fmt[i].pixfmt))
+		return NULL;
+
 	return &fmt[i];
 }
 
@@ -154,7 +163,7 @@ vdec_try_fmt_common(struct venus_inst *inst, struct v4l2_format *f)
 	memset(pfmt[0].reserved, 0, sizeof(pfmt[0].reserved));
 	memset(pixmp->reserved, 0, sizeof(pixmp->reserved));
 
-	fmt = find_format(pixmp->pixelformat, f->type);
+	fmt = find_format(inst, pixmp->pixelformat, f->type);
 	if (!fmt) {
 		if (f->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)
 			pixmp->pixelformat = V4L2_PIX_FMT_NV12;
@@ -162,7 +171,7 @@ vdec_try_fmt_common(struct venus_inst *inst, struct v4l2_format *f)
 			pixmp->pixelformat = V4L2_PIX_FMT_H264;
 		else
 			return NULL;
-		fmt = find_format(pixmp->pixelformat, f->type);
+		fmt = find_format(inst, pixmp->pixelformat, f->type);
 		pixmp->width = 1280;
 		pixmp->height = 720;
 	}
@@ -364,11 +373,12 @@ vdec_querycap(struct file *file, void *fh, struct v4l2_capability *cap)
 
 static int vdec_enum_fmt(struct file *file, void *fh, struct v4l2_fmtdesc *f)
 {
+	struct venus_inst *inst = to_inst(file);
 	const struct venus_format *fmt;
 
 	memset(f->reserved, 0, sizeof(f->reserved));
 
-	fmt = find_format_by_index(f->index, f->type);
+	fmt = find_format_by_index(inst, f->index, f->type);
 	if (!fmt)
 		return -EINVAL;
 
@@ -417,10 +427,10 @@ static int vdec_enum_framesizes(struct file *file, void *fh,
 	struct venus_inst *inst = to_inst(file);
 	const struct venus_format *fmt;
 
-	fmt = find_format(fsize->pixel_format,
+	fmt = find_format(inst, fsize->pixel_format,
 			  V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
 	if (!fmt) {
-		fmt = find_format(fsize->pixel_format,
+		fmt = find_format(inst, fsize->pixel_format,
 				  V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE);
 		if (!fmt)
 			return -EINVAL;
@@ -1069,6 +1079,7 @@ static int vdec_probe(struct platform_device *pdev)
 	if (!vdev)
 		return -ENOMEM;
 
+	strlcpy(vdev->name, "qcom-venus-decoder", sizeof(vdev->name));
 	vdev->release = video_device_release;
 	vdev->fops = &vdec_fops;
 	vdev->ioctl_ops = &vdec_ioctl_ops;
@@ -1103,8 +1114,7 @@ static int vdec_remove(struct platform_device *pdev)
 	return 0;
 }
 
-#ifdef CONFIG_PM
-static int vdec_runtime_suspend(struct device *dev)
+static __maybe_unused int vdec_runtime_suspend(struct device *dev)
 {
 	struct venus_core *core = dev_get_drvdata(dev);
 
@@ -1118,7 +1128,7 @@ static int vdec_runtime_suspend(struct device *dev)
 	return 0;
 }
 
-static int vdec_runtime_resume(struct device *dev)
+static __maybe_unused int vdec_runtime_resume(struct device *dev)
 {
 	struct venus_core *core = dev_get_drvdata(dev);
 	int ret;
@@ -1132,7 +1142,6 @@ static int vdec_runtime_resume(struct device *dev)
 
 	return ret;
 }
-#endif
 
 static const struct dev_pm_ops vdec_pm_ops = {
 	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
diff --git a/drivers/media/platform/qcom/venus/venc.c b/drivers/media/platform/qcom/venus/venc.c
index 39748e7a08e4..6f123a387cf9 100644
--- a/drivers/media/platform/qcom/venus/venc.c
+++ b/drivers/media/platform/qcom/venus/venc.c
@@ -84,14 +84,11 @@ static const struct venus_format venc_formats[] = {
 		.pixfmt = V4L2_PIX_FMT_VP8,
 		.num_planes = 1,
 		.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
-	}, {
-		.pixfmt = V4L2_PIX_FMT_VP9,
-		.num_planes = 1,
-		.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
 	},
 };
 
-static const struct venus_format *find_format(u32 pixfmt, u32 type)
+static const struct venus_format *
+find_format(struct venus_inst *inst, u32 pixfmt, u32 type)
 {
 	const struct venus_format *fmt = venc_formats;
 	unsigned int size = ARRAY_SIZE(venc_formats);
@@ -105,11 +102,15 @@ static const struct venus_format *find_format(u32 pixfmt, u32 type)
 	if (i == size || fmt[i].type != type)
 		return NULL;
 
+	if (type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE &&
+	    !venus_helper_check_codec(inst, fmt[i].pixfmt))
+		return NULL;
+
 	return &fmt[i];
 }
 
 static const struct venus_format *
-find_format_by_index(unsigned int index, u32 type)
+find_format_by_index(struct venus_inst *inst, unsigned int index, u32 type)
 {
 	const struct venus_format *fmt = venc_formats;
 	unsigned int size = ARRAY_SIZE(venc_formats);
@@ -129,6 +130,10 @@ find_format_by_index(unsigned int index, u32 type)
 	if (i == size)
 		return NULL;
 
+	if (type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE &&
+	    !venus_helper_check_codec(inst, fmt[i].pixfmt))
+		return NULL;
+
 	return &fmt[i];
 }
 
@@ -246,9 +251,10 @@ venc_querycap(struct file *file, void *fh, struct v4l2_capability *cap)
 
 static int venc_enum_fmt(struct file *file, void *fh, struct v4l2_fmtdesc *f)
 {
+	struct venus_inst *inst = to_inst(file);
 	const struct venus_format *fmt;
 
-	fmt = find_format_by_index(f->index, f->type);
+	fmt = find_format_by_index(inst, f->index, f->type);
 
 	memset(f->reserved, 0, sizeof(f->reserved));
 
@@ -271,7 +277,7 @@ venc_try_fmt_common(struct venus_inst *inst, struct v4l2_format *f)
 	memset(pfmt[0].reserved, 0, sizeof(pfmt[0].reserved));
 	memset(pixmp->reserved, 0, sizeof(pixmp->reserved));
 
-	fmt = find_format(pixmp->pixelformat, f->type);
+	fmt = find_format(inst, pixmp->pixelformat, f->type);
 	if (!fmt) {
 		if (f->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)
 			pixmp->pixelformat = V4L2_PIX_FMT_H264;
@@ -279,7 +285,7 @@ venc_try_fmt_common(struct venus_inst *inst, struct v4l2_format *f)
 			pixmp->pixelformat = V4L2_PIX_FMT_NV12;
 		else
 			return NULL;
-		fmt = find_format(pixmp->pixelformat, f->type);
+		fmt = find_format(inst, pixmp->pixelformat, f->type);
 		pixmp->width = 1280;
 		pixmp->height = 720;
 	}
@@ -289,7 +295,7 @@ venc_try_fmt_common(struct venus_inst *inst, struct v4l2_format *f)
 	pixmp->height = clamp(pixmp->height, inst->cap_height.min,
 			      inst->cap_height.max);
 
-	if (inst->core->res->hfi_version == HFI_VERSION_1XX)
+	if (f->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE)
 		pixmp->height = ALIGN(pixmp->height, 32);
 
 	pixmp->width = ALIGN(pixmp->width, 2);
@@ -524,10 +530,10 @@ static int venc_enum_framesizes(struct file *file, void *fh,
 
 	fsize->type = V4L2_FRMSIZE_TYPE_STEPWISE;
 
-	fmt = find_format(fsize->pixel_format,
+	fmt = find_format(inst, fsize->pixel_format,
 			  V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
 	if (!fmt) {
-		fmt = find_format(fsize->pixel_format,
+		fmt = find_format(inst, fsize->pixel_format,
 				  V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE);
 		if (!fmt)
 			return -EINVAL;
@@ -554,10 +560,10 @@ static int venc_enum_frameintervals(struct file *file, void *fh,
 
 	fival->type = V4L2_FRMIVAL_TYPE_STEPWISE;
 
-	fmt = find_format(fival->pixel_format,
+	fmt = find_format(inst, fival->pixel_format,
 			  V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
 	if (!fmt) {
-		fmt = find_format(fival->pixel_format,
+		fmt = find_format(inst, fival->pixel_format,
 				  V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE);
 		if (!fmt)
 			return -EINVAL;
@@ -747,8 +753,8 @@ static int venc_init_session(struct venus_inst *inst)
 	if (ret)
 		return ret;
 
-	ret = venus_helper_set_input_resolution(inst, inst->out_width,
-						inst->out_height);
+	ret = venus_helper_set_input_resolution(inst, inst->width,
+						inst->height);
 	if (ret)
 		goto deinit;
 
@@ -1010,6 +1016,8 @@ static int m2m_queue_init(void *priv, struct vb2_queue *src_vq,
 	src_vq->allow_zero_bytesused = 1;
 	src_vq->min_buffers_needed = 1;
 	src_vq->dev = inst->core->dev;
+	if (inst->core->res->hfi_version == HFI_VERSION_1XX)
+		src_vq->bidirectional = 1;
 	ret = vb2_queue_init(src_vq);
 	if (ret)
 		return ret;
@@ -1190,6 +1198,7 @@ static int venc_probe(struct platform_device *pdev)
 	if (!vdev)
 		return -ENOMEM;
 
+	strlcpy(vdev->name, "qcom-venus-encoder", sizeof(vdev->name));
 	vdev->release = video_device_release;
 	vdev->fops = &venc_fops;
 	vdev->ioctl_ops = &venc_ioctl_ops;
@@ -1224,8 +1233,7 @@ static int venc_remove(struct platform_device *pdev)
 	return 0;
 }
 
-#ifdef CONFIG_PM
-static int venc_runtime_suspend(struct device *dev)
+static __maybe_unused int venc_runtime_suspend(struct device *dev)
 {
 	struct venus_core *core = dev_get_drvdata(dev);
 
@@ -1239,7 +1247,7 @@ static int venc_runtime_suspend(struct device *dev)
 	return 0;
 }
 
-static int venc_runtime_resume(struct device *dev)
+static __maybe_unused int venc_runtime_resume(struct device *dev)
 {
 	struct venus_core *core = dev_get_drvdata(dev);
 	int ret;
@@ -1253,7 +1261,6 @@ static int venc_runtime_resume(struct device *dev)
 
 	return ret;
 }
-#endif
 
 static const struct dev_pm_ops venc_pm_ops = {
 	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
diff --git a/drivers/media/platform/rcar-vin/rcar-core.c b/drivers/media/platform/rcar-vin/rcar-core.c
index 77dff047c41c..142de447aaaa 100644
--- a/drivers/media/platform/rcar-vin/rcar-core.c
+++ b/drivers/media/platform/rcar-vin/rcar-core.c
@@ -222,8 +222,8 @@ static int rvin_digital_graph_init(struct rvin_dev *vin)
 
 	subdevs[0] = &vin->digital.asd;
 
-	vin_dbg(vin, "Found digital subdevice %s\n",
-		of_node_full_name(to_of_node(subdevs[0]->match.fwnode.fwnode)));
+	vin_dbg(vin, "Found digital subdevice %pOF\n",
+		to_of_node(subdevs[0]->match.fwnode.fwnode));
 
 	vin->notifier.num_subdevs = 1;
 	vin->notifier.subdevs = subdevs;
diff --git a/drivers/media/platform/rcar_fdp1.c b/drivers/media/platform/rcar_fdp1.c
index 3ee51fc3bb50..3245bc45f4a0 100644
--- a/drivers/media/platform/rcar_fdp1.c
+++ b/drivers/media/platform/rcar_fdp1.c
@@ -2032,7 +2032,7 @@ static void fdp1_stop_streaming(struct vb2_queue *q)
 	}
 }
 
-static struct vb2_ops fdp1_qops = {
+static const struct vb2_ops fdp1_qops = {
 	.queue_setup	 = fdp1_queue_setup,
 	.buf_prepare	 = fdp1_buf_prepare,
 	.buf_queue	 = fdp1_buf_queue,
diff --git a/drivers/media/platform/rcar_jpu.c b/drivers/media/platform/rcar_jpu.c
index d1746ecc645d..070bac36d766 100644
--- a/drivers/media/platform/rcar_jpu.c
+++ b/drivers/media/platform/rcar_jpu.c
@@ -1506,7 +1506,7 @@ static void jpu_job_abort(void *priv)
 		jpu_cleanup(ctx, true);
 }
 
-static struct v4l2_m2m_ops jpu_m2m_ops = {
+static const struct v4l2_m2m_ops jpu_m2m_ops = {
 	.device_run	= jpu_device_run,
 	.job_ready	= jpu_job_ready,
 	.job_abort	= jpu_job_abort,
diff --git a/drivers/media/platform/s3c-camif/camif-core.c b/drivers/media/platform/s3c-camif/camif-core.c
index ec4001970313..c4ab63986c8f 100644
--- a/drivers/media/platform/s3c-camif/camif-core.c
+++ b/drivers/media/platform/s3c-camif/camif-core.c
@@ -317,7 +317,6 @@ static int camif_media_dev_init(struct camif_dev *camif)
 		 ip_rev == S3C6410_CAMIF_IP_REV ? "6410" : "244X");
 	strlcpy(md->bus_info, "platform", sizeof(md->bus_info));
 	md->hw_revision = ip_rev;
-	md->driver_version = KERNEL_VERSION(1, 0, 0);
 
 	md->dev = camif->dev;
 
diff --git a/drivers/media/platform/s5p-cec/s5p_cec.c b/drivers/media/platform/s5p-cec/s5p_cec.c
index 8e06071a7977..58d200e7c838 100644
--- a/drivers/media/platform/s5p-cec/s5p_cec.c
+++ b/drivers/media/platform/s5p-cec/s5p_cec.c
@@ -219,11 +219,8 @@ static int s5p_cec_probe(struct platform_device *pdev)
 	if (cec->notifier == NULL)
 		return -ENOMEM;
 
-	cec->adap = cec_allocate_adapter(&s5p_cec_adap_ops, cec,
-		CEC_NAME,
-		CEC_CAP_LOG_ADDRS | CEC_CAP_TRANSMIT |
-		CEC_CAP_PASSTHROUGH | CEC_CAP_RC |
-		(needs_hpd ? CEC_CAP_NEEDS_HPD : 0), 1);
+	cec->adap = cec_allocate_adapter(&s5p_cec_adap_ops, cec, CEC_NAME,
+		CEC_CAP_DEFAULTS | (needs_hpd ? CEC_CAP_NEEDS_HPD : 0), 1);
 	ret = PTR_ERR_OR_ZERO(cec->adap);
 	if (ret)
 		return ret;
diff --git a/drivers/media/platform/s5p-g2d/g2d.c b/drivers/media/platform/s5p-g2d/g2d.c
index 81ed5cd5cd5d..66aa8cf1d048 100644
--- a/drivers/media/platform/s5p-g2d/g2d.c
+++ b/drivers/media/platform/s5p-g2d/g2d.c
@@ -602,7 +602,7 @@ static const struct v4l2_ioctl_ops g2d_ioctl_ops = {
 	.vidioc_cropcap			= vidioc_cropcap,
 };
 
-static struct video_device g2d_videodev = {
+static const struct video_device g2d_videodev = {
 	.name		= G2D_NAME,
 	.fops		= &g2d_fops,
 	.ioctl_ops	= &g2d_ioctl_ops,
@@ -611,7 +611,7 @@ static struct video_device g2d_videodev = {
 	.vfl_dir	= VFL_DIR_M2M,
 };
 
-static struct v4l2_m2m_ops g2d_m2m_ops = {
+static const struct v4l2_m2m_ops g2d_m2m_ops = {
 	.device_run	= device_run,
 	.job_abort	= job_abort,
 };
diff --git a/drivers/media/platform/s5p-jpeg/jpeg-core.c b/drivers/media/platform/s5p-jpeg/jpeg-core.c
index d1e3ebb22577..faac8161b683 100644
--- a/drivers/media/platform/s5p-jpeg/jpeg-core.c
+++ b/drivers/media/platform/s5p-jpeg/jpeg-core.c
@@ -24,6 +24,7 @@
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/string.h>
+#include <media/v4l2-event.h>
 #include <media/v4l2-mem2mem.h>
 #include <media/v4l2-ioctl.h>
 #include <media/videobuf2-v4l2.h>
@@ -614,24 +615,27 @@ static inline struct s5p_jpeg_ctx *fh_to_ctx(struct v4l2_fh *fh)
 
 static int s5p_jpeg_to_user_subsampling(struct s5p_jpeg_ctx *ctx)
 {
-	WARN_ON(ctx->subsampling > 3);
-
 	switch (ctx->jpeg->variant->version) {
 	case SJPEG_S5P:
+		WARN_ON(ctx->subsampling > 3);
 		if (ctx->subsampling > 2)
 			return V4L2_JPEG_CHROMA_SUBSAMPLING_GRAY;
 		return ctx->subsampling;
 	case SJPEG_EXYNOS3250:
 	case SJPEG_EXYNOS5420:
+		WARN_ON(ctx->subsampling > 6);
 		if (ctx->subsampling > 3)
 			return V4L2_JPEG_CHROMA_SUBSAMPLING_411;
 		return exynos3250_decoded_subsampling[ctx->subsampling];
 	case SJPEG_EXYNOS4:
-	case SJPEG_EXYNOS5433:
+		WARN_ON(ctx->subsampling > 3);
 		if (ctx->subsampling > 2)
 			return V4L2_JPEG_CHROMA_SUBSAMPLING_420;
 		return exynos4x12_decoded_subsampling[ctx->subsampling];
+	case SJPEG_EXYNOS5433:
+		return ctx->subsampling; /* parsed from header */
 	default:
+		WARN_ON(ctx->subsampling > 3);
 		return V4L2_JPEG_CHROMA_SUBSAMPLING_GRAY;
 	}
 }
@@ -1094,6 +1098,44 @@ static void skip(struct s5p_jpeg_buffer *buf, long len)
 		get_byte(buf);
 }
 
+static bool s5p_jpeg_subsampling_decode(struct s5p_jpeg_ctx *ctx,
+					unsigned int subsampling)
+{
+	unsigned int version;
+
+	switch (subsampling) {
+	case 0x11:
+		ctx->subsampling = V4L2_JPEG_CHROMA_SUBSAMPLING_444;
+		break;
+	case 0x21:
+		ctx->subsampling = V4L2_JPEG_CHROMA_SUBSAMPLING_422;
+		break;
+	case 0x22:
+		ctx->subsampling = V4L2_JPEG_CHROMA_SUBSAMPLING_420;
+		break;
+	case 0x33:
+		ctx->subsampling = V4L2_JPEG_CHROMA_SUBSAMPLING_GRAY;
+		break;
+	case 0x41:
+		/*
+		 * 4:1:1 subsampling only supported by 3250, 5420, and 5433
+		 * variants
+		 */
+		version = ctx->jpeg->variant->version;
+		if (version != SJPEG_EXYNOS3250 &&
+		    version != SJPEG_EXYNOS5420 &&
+		    version != SJPEG_EXYNOS5433)
+			return false;
+
+		ctx->subsampling = V4L2_JPEG_CHROMA_SUBSAMPLING_411;
+		break;
+	default:
+		return false;
+	}
+
+	return true;
+}
+
 static bool s5p_jpeg_parse_hdr(struct s5p_jpeg_q_data *result,
 			       unsigned long buffer, unsigned long size,
 			       struct s5p_jpeg_ctx *ctx)
@@ -1204,6 +1246,10 @@ static bool s5p_jpeg_parse_hdr(struct s5p_jpeg_q_data *result,
 			break;
 		}
 	}
+
+	if (notfound || !sos || !s5p_jpeg_subsampling_decode(ctx, subsampling))
+		return false;
+
 	result->w = width;
 	result->h = height;
 	result->sos = sos;
@@ -1219,26 +1265,9 @@ static bool s5p_jpeg_parse_hdr(struct s5p_jpeg_q_data *result,
 	}
 	result->sof = sof;
 	result->sof_len = sof_len;
-	result->size = result->components = components;
-
-	switch (subsampling) {
-	case 0x11:
-		ctx->subsampling = V4L2_JPEG_CHROMA_SUBSAMPLING_444;
-		break;
-	case 0x21:
-		ctx->subsampling = V4L2_JPEG_CHROMA_SUBSAMPLING_422;
-		break;
-	case 0x22:
-		ctx->subsampling = V4L2_JPEG_CHROMA_SUBSAMPLING_420;
-		break;
-	case 0x33:
-		ctx->subsampling = V4L2_JPEG_CHROMA_SUBSAMPLING_GRAY;
-		break;
-	default:
-		return false;
-	}
+	result->components = components;
 
-	return !notfound && sos;
+	return true;
 }
 
 static int s5p_jpeg_querycap(struct file *file, void *priv,
@@ -1606,8 +1635,12 @@ static int s5p_jpeg_s_fmt(struct s5p_jpeg_ctx *ct, struct v4l2_format *f)
 			FMT_TYPE_OUTPUT : FMT_TYPE_CAPTURE;
 
 	q_data->fmt = s5p_jpeg_find_format(ct, pix->pixelformat, f_type);
-	q_data->w = pix->width;
-	q_data->h = pix->height;
+	if (ct->mode == S5P_JPEG_ENCODE ||
+		(ct->mode == S5P_JPEG_DECODE &&
+		q_data->fmt->fourcc != V4L2_PIX_FMT_JPEG)) {
+		q_data->w = pix->width;
+		q_data->h = pix->height;
+	}
 	if (q_data->fmt->fourcc != V4L2_PIX_FMT_JPEG) {
 		/*
 		 * During encoding Exynos4x12 SoCs access wider memory area
@@ -1690,6 +1723,15 @@ static int s5p_jpeg_s_fmt_vid_out(struct file *file, void *priv,
 	return s5p_jpeg_s_fmt(fh_to_ctx(priv), f);
 }
 
+static int s5p_jpeg_subscribe_event(struct v4l2_fh *fh,
+				    const struct v4l2_event_subscription *sub)
+{
+	if (sub->type == V4L2_EVENT_SOURCE_CHANGE)
+		return v4l2_src_change_event_subscribe(fh, sub);
+
+	return -EINVAL;
+}
+
 static int exynos3250_jpeg_try_downscale(struct s5p_jpeg_ctx *ctx,
 				   struct v4l2_rect *r)
 {
@@ -2015,6 +2057,9 @@ static const struct v4l2_ioctl_ops s5p_jpeg_ioctl_ops = {
 
 	.vidioc_g_selection		= s5p_jpeg_g_selection,
 	.vidioc_s_selection		= s5p_jpeg_s_selection,
+
+	.vidioc_subscribe_event		= s5p_jpeg_subscribe_event,
+	.vidioc_unsubscribe_event	= v4l2_event_unsubscribe,
 };
 
 /*
@@ -2259,6 +2304,7 @@ static void exynos4_jpeg_device_run(void *priv)
 		exynos4_jpeg_set_dec_bitstream_size(jpeg->regs, bitstream_size);
 	}
 
+	exynos4_jpeg_set_sys_int_enable(jpeg->regs, 1);
 	exynos4_jpeg_set_enc_dec_mode(jpeg->regs, ctx->mode);
 
 	spin_unlock_irqrestore(&jpeg->slock, flags);
@@ -2407,8 +2453,17 @@ static int s5p_jpeg_job_ready(void *priv)
 {
 	struct s5p_jpeg_ctx *ctx = priv;
 
-	if (ctx->mode == S5P_JPEG_DECODE)
+	if (ctx->mode == S5P_JPEG_DECODE) {
+		/*
+		 * We have only one input buffer and one output buffer. If there
+		 * is a resolution change event, no need to continue decoding.
+		 */
+		if (ctx->state == JPEGCTX_RESOLUTION_CHANGE)
+			return 0;
+
 		return ctx->hdr_parsed;
+	}
+
 	return 1;
 }
 
@@ -2487,6 +2542,30 @@ static int s5p_jpeg_buf_prepare(struct vb2_buffer *vb)
 	return 0;
 }
 
+static void s5p_jpeg_set_capture_queue_data(struct s5p_jpeg_ctx *ctx)
+{
+	struct s5p_jpeg_q_data *q_data = &ctx->cap_q;
+
+	q_data->w = ctx->out_q.w;
+	q_data->h = ctx->out_q.h;
+
+	/*
+	 * This call to jpeg_bound_align_image() takes care of width and
+	 * height values alignment when user space calls the QBUF of
+	 * OUTPUT buffer after the S_FMT of CAPTURE buffer.
+	 * Please note that on Exynos4x12 SoCs, resigning from executing
+	 * S_FMT on capture buffer for each JPEG image can result in a
+	 * hardware hangup if subsampling is lower than the one of input
+	 * JPEG.
+	 */
+	jpeg_bound_align_image(ctx, &q_data->w, S5P_JPEG_MIN_WIDTH,
+			       S5P_JPEG_MAX_WIDTH, q_data->fmt->h_align,
+			       &q_data->h, S5P_JPEG_MIN_HEIGHT,
+			       S5P_JPEG_MAX_HEIGHT, q_data->fmt->v_align);
+
+	q_data->size = q_data->w * q_data->h * q_data->fmt->depth >> 3;
+}
+
 static void s5p_jpeg_buf_queue(struct vb2_buffer *vb)
 {
 	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
@@ -2494,9 +2573,20 @@ static void s5p_jpeg_buf_queue(struct vb2_buffer *vb)
 
 	if (ctx->mode == S5P_JPEG_DECODE &&
 	    vb->vb2_queue->type == V4L2_BUF_TYPE_VIDEO_OUTPUT) {
-		struct s5p_jpeg_q_data tmp, *q_data;
-
-		ctx->hdr_parsed = s5p_jpeg_parse_hdr(&tmp,
+		static const struct v4l2_event ev_src_ch = {
+			.type = V4L2_EVENT_SOURCE_CHANGE,
+			.u.src_change.changes = V4L2_EVENT_SRC_CH_RESOLUTION,
+		};
+		struct vb2_queue *dst_vq;
+		u32 ori_w;
+		u32 ori_h;
+
+		dst_vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx,
+					 V4L2_BUF_TYPE_VIDEO_CAPTURE);
+		ori_w = ctx->out_q.w;
+		ori_h = ctx->out_q.h;
+
+		ctx->hdr_parsed = s5p_jpeg_parse_hdr(&ctx->out_q,
 		     (unsigned long)vb2_plane_vaddr(vb, 0),
 		     min((unsigned long)ctx->out_q.size,
 			 vb2_get_plane_payload(vb, 0)), ctx);
@@ -2505,24 +2595,18 @@ static void s5p_jpeg_buf_queue(struct vb2_buffer *vb)
 			return;
 		}
 
-		q_data = &ctx->out_q;
-		q_data->w = tmp.w;
-		q_data->h = tmp.h;
-		q_data->sos = tmp.sos;
-		memcpy(q_data->dht.marker, tmp.dht.marker,
-		       sizeof(tmp.dht.marker));
-		memcpy(q_data->dht.len, tmp.dht.len, sizeof(tmp.dht.len));
-		q_data->dht.n = tmp.dht.n;
-		memcpy(q_data->dqt.marker, tmp.dqt.marker,
-		       sizeof(tmp.dqt.marker));
-		memcpy(q_data->dqt.len, tmp.dqt.len, sizeof(tmp.dqt.len));
-		q_data->dqt.n = tmp.dqt.n;
-		q_data->sof = tmp.sof;
-		q_data->sof_len = tmp.sof_len;
-
-		q_data = &ctx->cap_q;
-		q_data->w = tmp.w;
-		q_data->h = tmp.h;
+		/*
+		 * If there is a resolution change event, only update capture
+		 * queue when it is not streaming. Otherwise, update it in
+		 * STREAMOFF. See s5p_jpeg_stop_streaming for detail.
+		 */
+		if (ctx->out_q.w != ori_w || ctx->out_q.h != ori_h) {
+			v4l2_event_queue_fh(&ctx->fh, &ev_src_ch);
+			if (vb2_is_streaming(dst_vq))
+				ctx->state = JPEGCTX_RESOLUTION_CHANGE;
+			else
+				s5p_jpeg_set_capture_queue_data(ctx);
+		}
 	}
 
 	v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, vbuf);
@@ -2542,6 +2626,17 @@ static void s5p_jpeg_stop_streaming(struct vb2_queue *q)
 {
 	struct s5p_jpeg_ctx *ctx = vb2_get_drv_priv(q);
 
+	/*
+	 * STREAMOFF is an acknowledgment for resolution change event.
+	 * Before STREAMOFF, we still have to return the old resolution and
+	 * subsampling. Update capture queue when the stream is off.
+	 */
+	if (ctx->state == JPEGCTX_RESOLUTION_CHANGE &&
+	    q->type == V4L2_BUF_TYPE_VIDEO_CAPTURE) {
+		s5p_jpeg_set_capture_queue_data(ctx);
+		ctx->state = JPEGCTX_RUNNING;
+	}
+
 	pm_runtime_put(ctx->jpeg->dev);
 }
 
@@ -2662,6 +2757,8 @@ static irqreturn_t exynos4_jpeg_irq(int irq, void *priv)
 
 	spin_lock(&jpeg->slock);
 
+	exynos4_jpeg_set_sys_int_enable(jpeg->regs, 0);
+
 	curr_ctx = v4l2_m2m_get_curr_priv(jpeg->m2m_dev);
 
 	src_vb = v4l2_m2m_src_buf_remove(curr_ctx->fh.m2m_ctx);
@@ -2710,6 +2807,8 @@ static irqreturn_t exynos4_jpeg_irq(int irq, void *priv)
 	if (jpeg->variant->version == SJPEG_EXYNOS4)
 		curr_ctx->subsampling = exynos4_jpeg_get_frame_fmt(jpeg->regs);
 
+	exynos4_jpeg_set_enc_dec_mode(jpeg->regs, S5P_JPEG_DISABLE);
+
 	spin_unlock(&jpeg->slock);
 
 	v4l2_m2m_job_finish(jpeg->m2m_dev, curr_ctx->fh.m2m_ctx);
@@ -2724,6 +2823,7 @@ static irqreturn_t exynos3250_jpeg_irq(int irq, void *dev_id)
 	unsigned long payload_size = 0;
 	enum vb2_buffer_state state = VB2_BUF_STATE_DONE;
 	bool interrupt_timeout = false;
+	bool stream_error = false;
 	u32 irq_status;
 
 	spin_lock(&jpeg->slock);
@@ -2740,6 +2840,12 @@ static irqreturn_t exynos3250_jpeg_irq(int irq, void *dev_id)
 
 	jpeg->irq_status |= irq_status;
 
+	if (jpeg->variant->version == SJPEG_EXYNOS5420 &&
+	    irq_status & EXYNOS3250_STREAM_STAT) {
+		stream_error = true;
+		dev_err(jpeg->dev, "Syntax error or unrecoverable error occurred.\n");
+	}
+
 	curr_ctx = v4l2_m2m_get_curr_priv(jpeg->m2m_dev);
 
 	if (!curr_ctx)
@@ -2756,7 +2862,7 @@ static irqreturn_t exynos3250_jpeg_irq(int irq, void *dev_id)
 				EXYNOS3250_RDMA_DONE |
 				EXYNOS3250_RESULT_STAT))
 		payload_size = exynos3250_jpeg_compressed_size(jpeg->regs);
-	else if (interrupt_timeout)
+	else if (interrupt_timeout || stream_error)
 		state = VB2_BUF_STATE_ERROR;
 	else
 		goto exit_unlock;
diff --git a/drivers/media/platform/s5p-jpeg/jpeg-core.h b/drivers/media/platform/s5p-jpeg/jpeg-core.h
index 4492a3535df5..a46465e10351 100644
--- a/drivers/media/platform/s5p-jpeg/jpeg-core.h
+++ b/drivers/media/platform/s5p-jpeg/jpeg-core.h
@@ -63,6 +63,7 @@
 
 #define S5P_JPEG_ENCODE		0
 #define S5P_JPEG_DECODE		1
+#define S5P_JPEG_DISABLE	-1
 
 #define FMT_TYPE_OUTPUT		0
 #define FMT_TYPE_CAPTURE	1
@@ -98,6 +99,11 @@ enum  exynos4_jpeg_img_quality_level {
 	QUALITY_LEVEL_4,	/* low */
 };
 
+enum s5p_jpeg_ctx_state {
+	JPEGCTX_RUNNING = 0,
+	JPEGCTX_RESOLUTION_CHANGE,
+};
+
 /**
  * struct s5p_jpeg - JPEG IP abstraction
  * @lock:		the mutex protecting this structure
@@ -220,6 +226,7 @@ struct s5p_jpeg_q_data {
  * @hdr_parsed:		set if header has been parsed during decompression
  * @crop_altered:	set if crop rectangle has been altered by the user space
  * @ctrl_handler:	controls handler
+ * @state:		state of the context
  */
 struct s5p_jpeg_ctx {
 	struct s5p_jpeg		*jpeg;
@@ -235,6 +242,7 @@ struct s5p_jpeg_ctx {
 	bool			hdr_parsed;
 	bool			crop_altered;
 	struct v4l2_ctrl_handler ctrl_handler;
+	enum s5p_jpeg_ctx_state	state;
 };
 
 /**
diff --git a/drivers/media/platform/s5p-jpeg/jpeg-hw-exynos4.c b/drivers/media/platform/s5p-jpeg/jpeg-hw-exynos4.c
index a1d823ab0c63..c72789bae6ed 100644
--- a/drivers/media/platform/s5p-jpeg/jpeg-hw-exynos4.c
+++ b/drivers/media/platform/s5p-jpeg/jpeg-hw-exynos4.c
@@ -21,6 +21,10 @@ void exynos4_jpeg_sw_reset(void __iomem *base)
 	unsigned int reg;
 
 	reg = readl(base + EXYNOS4_JPEG_CNTL_REG);
+	writel(reg & ~(EXYNOS4_DEC_MODE | EXYNOS4_ENC_MODE),
+				base + EXYNOS4_JPEG_CNTL_REG);
+
+	reg = readl(base + EXYNOS4_JPEG_CNTL_REG);
 	writel(reg & ~EXYNOS4_SOFT_RESET_HI, base + EXYNOS4_JPEG_CNTL_REG);
 
 	udelay(100);
@@ -38,10 +42,13 @@ void exynos4_jpeg_set_enc_dec_mode(void __iomem *base, unsigned int mode)
 		writel((reg & EXYNOS4_ENC_DEC_MODE_MASK) |
 					EXYNOS4_DEC_MODE,
 			base + EXYNOS4_JPEG_CNTL_REG);
-	} else {/* encode */
+	} else if (mode == S5P_JPEG_ENCODE) {/* encode */
 		writel((reg & EXYNOS4_ENC_DEC_MODE_MASK) |
 					EXYNOS4_ENC_MODE,
 			base + EXYNOS4_JPEG_CNTL_REG);
+	} else { /* disable both */
+		writel(reg & EXYNOS4_ENC_DEC_MODE_MASK,
+			base + EXYNOS4_JPEG_CNTL_REG);
 	}
 }
 
diff --git a/drivers/media/platform/s5p-jpeg/jpeg-regs.h b/drivers/media/platform/s5p-jpeg/jpeg-regs.h
index 1870400468b2..df790b10140c 100644
--- a/drivers/media/platform/s5p-jpeg/jpeg-regs.h
+++ b/drivers/media/platform/s5p-jpeg/jpeg-regs.h
@@ -371,7 +371,7 @@
 #define EXYNOS4_NF_SHIFT			16
 #define EXYNOS4_NF_MASK				0xff
 #define EXYNOS4_NF(x)				\
-	(((x) << EXYNOS4_NF_SHIFT) & EXYNOS4_NF_MASK)
+	(((x) & EXYNOS4_NF_MASK) << EXYNOS4_NF_SHIFT)
 
 /* JPEG quantizer table register */
 #define EXYNOS4_QTBL_CONTENT(n)	(0x100 + (n) * 0x40)
diff --git a/drivers/media/platform/soc_camera/sh_mobile_ceu_camera.c b/drivers/media/platform/soc_camera/sh_mobile_ceu_camera.c
index 96dc01750bc0..36762ec954e7 100644
--- a/drivers/media/platform/soc_camera/sh_mobile_ceu_camera.c
+++ b/drivers/media/platform/soc_camera/sh_mobile_ceu_camera.c
@@ -1708,11 +1708,10 @@ static int sh_mobile_ceu_probe(struct platform_device *pdev)
 		err = dma_declare_coherent_memory(&pdev->dev, res->start,
 						  res->start,
 						  resource_size(res),
-						  DMA_MEMORY_MAP |
 						  DMA_MEMORY_EXCLUSIVE);
-		if (!err) {
+		if (err) {
 			dev_err(&pdev->dev, "Unable to declare CEU memory.\n");
-			return -ENXIO;
+			return err;
 		}
 
 		pcdev->video_limit = resource_size(res);
diff --git a/drivers/media/platform/soc_camera/soc_camera.c b/drivers/media/platform/soc_camera/soc_camera.c
index 45a0429d75bb..1f3c450c7a69 100644
--- a/drivers/media/platform/soc_camera/soc_camera.c
+++ b/drivers/media/platform/soc_camera/soc_camera.c
@@ -820,7 +820,7 @@ static unsigned int soc_camera_poll(struct file *file, poll_table *pt)
 	return res;
 }
 
-static struct v4l2_file_operations soc_camera_fops = {
+static const struct v4l2_file_operations soc_camera_fops = {
 	.owner		= THIS_MODULE,
 	.open		= soc_camera_open,
 	.release	= soc_camera_close,
@@ -1550,8 +1550,7 @@ static int soc_of_bind(struct soc_camera_host *ici,
 		v4l2_clk_name_i2c(clk_name, sizeof(clk_name),
 				  client->adapter->nr, client->addr);
 	else
-		v4l2_clk_name_of(clk_name, sizeof(clk_name),
-				 of_node_full_name(remote));
+		v4l2_clk_name_of(clk_name, sizeof(clk_name), remote);
 
 	icd->clk = v4l2_clk_register(&soc_camera_clk_ops, clk_name, icd);
 	if (IS_ERR(icd->clk)) {
@@ -1590,8 +1589,7 @@ static void scan_of_host(struct soc_camera_host *ici)
 
 		ren = of_graph_get_remote_port(epn);
 		if (!ren) {
-			dev_notice(dev, "no remote for %s\n",
-				   of_node_full_name(epn));
+			dev_notice(dev, "no remote for %pOF\n", epn);
 			continue;
 		}
 
diff --git a/drivers/media/platform/soc_camera/soc_mediabus.c b/drivers/media/platform/soc_camera/soc_mediabus.c
index 57581f626f4c..0ad4b28266e4 100644
--- a/drivers/media/platform/soc_camera/soc_mediabus.c
+++ b/drivers/media/platform/soc_camera/soc_mediabus.c
@@ -508,6 +508,9 @@ unsigned int soc_mbus_config_compatible(const struct v4l2_mbus_config *cfg,
 		mipi_clock = common_flags & (V4L2_MBUS_CSI2_NONCONTINUOUS_CLOCK |
 					     V4L2_MBUS_CSI2_CONTINUOUS_CLOCK);
 		return (!mipi_lanes || !mipi_clock) ? 0 : common_flags;
+	default:
+		WARN_ON(1);
+		return -EINVAL;
 	}
 	return 0;
 }
diff --git a/drivers/media/platform/sti/bdisp/bdisp-v4l2.c b/drivers/media/platform/sti/bdisp/bdisp-v4l2.c
index 7918b928f058..939da6da7644 100644
--- a/drivers/media/platform/sti/bdisp/bdisp-v4l2.c
+++ b/drivers/media/platform/sti/bdisp/bdisp-v4l2.c
@@ -360,7 +360,7 @@ out:
 		bdisp_job_finish(ctx, VB2_BUF_STATE_ERROR);
 }
 
-static struct v4l2_m2m_ops bdisp_m2m_ops = {
+static const struct v4l2_m2m_ops bdisp_m2m_ops = {
 	.device_run     = bdisp_device_run,
 	.job_abort      = bdisp_job_abort,
 };
diff --git a/drivers/media/platform/sti/cec/stih-cec.c b/drivers/media/platform/sti/cec/stih-cec.c
index dccbdaebb7a8..70160df36de9 100644
--- a/drivers/media/platform/sti/cec/stih-cec.c
+++ b/drivers/media/platform/sti/cec/stih-cec.c
@@ -351,9 +351,7 @@ static int stih_cec_probe(struct platform_device *pdev)
 	}
 
 	cec->adap = cec_allocate_adapter(&sti_cec_adap_ops, cec,
-			CEC_NAME,
-			CEC_CAP_LOG_ADDRS | CEC_CAP_PASSTHROUGH |
-			CEC_CAP_TRANSMIT, CEC_MAX_LOG_ADDRS);
+			CEC_NAME, CEC_CAP_DEFAULTS, CEC_MAX_LOG_ADDRS);
 	ret = PTR_ERR_OR_ZERO(cec->adap);
 	if (ret)
 		return ret;
diff --git a/drivers/media/platform/sti/delta/delta-v4l2.c b/drivers/media/platform/sti/delta/delta-v4l2.c
index c6f2e244b7a8..b2dc3d223a9c 100644
--- a/drivers/media/platform/sti/delta/delta-v4l2.c
+++ b/drivers/media/platform/sti/delta/delta-v4l2.c
@@ -1095,7 +1095,7 @@ static int delta_job_ready(void *priv)
 }
 
 /* mem-to-mem ops */
-static struct v4l2_m2m_ops delta_m2m_ops = {
+static const struct v4l2_m2m_ops delta_m2m_ops = {
 	.device_run     = delta_device_run,
 	.job_ready	= delta_job_ready,
 	.job_abort      = delta_job_abort,
@@ -1574,7 +1574,7 @@ static void delta_vb2_frame_stop_streaming(struct vb2_queue *q)
 }
 
 /* VB2 queue ops */
-static struct vb2_ops delta_vb2_au_ops = {
+static const struct vb2_ops delta_vb2_au_ops = {
 	.queue_setup = delta_vb2_au_queue_setup,
 	.buf_prepare = delta_vb2_au_prepare,
 	.buf_queue = delta_vb2_au_queue,
@@ -1584,7 +1584,7 @@ static struct vb2_ops delta_vb2_au_ops = {
 	.stop_streaming = delta_vb2_au_stop_streaming,
 };
 
-static struct vb2_ops delta_vb2_frame_ops = {
+static const struct vb2_ops delta_vb2_frame_ops = {
 	.queue_setup = delta_vb2_frame_queue_setup,
 	.buf_prepare = delta_vb2_frame_prepare,
 	.buf_finish = delta_vb2_frame_finish,
diff --git a/drivers/media/platform/stm32/stm32-cec.c b/drivers/media/platform/stm32/stm32-cec.c
index 9ab896b01ee8..0e5aa17bdd40 100644
--- a/drivers/media/platform/stm32/stm32-cec.c
+++ b/drivers/media/platform/stm32/stm32-cec.c
@@ -246,9 +246,7 @@ static const struct regmap_config stm32_cec_regmap_cfg = {
 
 static int stm32_cec_probe(struct platform_device *pdev)
 {
-	u32 caps = CEC_CAP_LOG_ADDRS | CEC_CAP_PASSTHROUGH |
-		   CEC_CAP_TRANSMIT | CEC_CAP_RC | CEC_CAP_PHYS_ADDR |
-		   CEC_MODE_MONITOR_ALL;
+	u32 caps = CEC_CAP_DEFAULTS | CEC_CAP_PHYS_ADDR | CEC_MODE_MONITOR_ALL;
 	struct resource *res;
 	struct stm32_cec *cec;
 	void __iomem *mmio;
diff --git a/drivers/media/platform/stm32/stm32-dcmi.c b/drivers/media/platform/stm32/stm32-dcmi.c
index 83d32a5d0f40..35ba6f211b79 100644
--- a/drivers/media/platform/stm32/stm32-dcmi.c
+++ b/drivers/media/platform/stm32/stm32-dcmi.c
@@ -33,6 +33,7 @@
 #include <media/v4l2-fwnode.h>
 #include <media/v4l2-image-sizes.h>
 #include <media/v4l2-ioctl.h>
+#include <media/v4l2-rect.h>
 #include <media/videobuf2-dma-contig.h>
 
 #define DRV_NAME "stm32-dcmi"
@@ -107,6 +108,11 @@ struct dcmi_format {
 	u8	bpp;
 };
 
+struct dcmi_framesize {
+	u32	width;
+	u32	height;
+};
+
 struct dcmi_buf {
 	struct vb2_v4l2_buffer	vb;
 	bool			prepared;
@@ -131,10 +137,16 @@ struct stm32_dcmi {
 	struct v4l2_async_notifier	notifier;
 	struct dcmi_graph_entity	entity;
 	struct v4l2_format		fmt;
+	struct v4l2_rect		crop;
+	bool				do_crop;
 
-	const struct dcmi_format	**user_formats;
-	unsigned int			num_user_formats;
-	const struct dcmi_format	*current_fmt;
+	const struct dcmi_format	**sd_formats;
+	unsigned int			num_of_sd_formats;
+	const struct dcmi_format	*sd_format;
+	struct dcmi_framesize		*sd_framesizes;
+	unsigned int			num_of_sd_framesizes;
+	struct dcmi_framesize		sd_framesize;
+	struct v4l2_rect		sd_bounds;
 
 	/* Protect this data structure */
 	struct mutex			lock;
@@ -295,6 +307,10 @@ static int dcmi_start_dma(struct stm32_dcmi *dcmi,
 
 	/* Push current DMA transaction in the pending queue */
 	dcmi->dma_cookie = dmaengine_submit(desc);
+	if (dma_submit_error(dcmi->dma_cookie)) {
+		dev_err(dcmi->dev, "%s: DMA submission failed\n", __func__);
+		return -ENXIO;
+	}
 
 	dma_async_issue_pending(dcmi->dma_chan);
 
@@ -321,6 +337,28 @@ static int dcmi_start_capture(struct stm32_dcmi *dcmi)
 	return 0;
 }
 
+static void dcmi_set_crop(struct stm32_dcmi *dcmi)
+{
+	u32 size, start;
+
+	/* Crop resolution */
+	size = ((dcmi->crop.height - 1) << 16) |
+		((dcmi->crop.width << 1) - 1);
+	reg_write(dcmi->regs, DCMI_CWSIZE, size);
+
+	/* Crop start point */
+	start = ((dcmi->crop.top) << 16) |
+		 ((dcmi->crop.left << 1));
+	reg_write(dcmi->regs, DCMI_CWSTRT, start);
+
+	dev_dbg(dcmi->dev, "Cropping to %ux%u@%u:%u\n",
+		dcmi->crop.width, dcmi->crop.height,
+		dcmi->crop.left, dcmi->crop.top);
+
+	/* Enable crop */
+	reg_set(dcmi->regs, DCMI_CR, CR_CROP);
+}
+
 static irqreturn_t dcmi_irq_thread(int irq, void *arg)
 {
 	struct stm32_dcmi *dcmi = arg;
@@ -486,7 +524,7 @@ static int dcmi_start_streaming(struct vb2_queue *vq, unsigned int count)
 {
 	struct stm32_dcmi *dcmi = vb2_get_drv_priv(vq);
 	struct dcmi_buf *buf, *node;
-	u32 val;
+	u32 val = 0;
 	int ret;
 
 	ret = clk_enable(dcmi->mclk);
@@ -506,22 +544,16 @@ static int dcmi_start_streaming(struct vb2_queue *vq, unsigned int count)
 
 	spin_lock_irq(&dcmi->irqlock);
 
-	val = reg_read(dcmi->regs, DCMI_CR);
-
-	val &= ~(CR_PCKPOL | CR_HSPOL | CR_VSPOL |
-		 CR_EDM_0 | CR_EDM_1 | CR_FCRC_0 |
-		 CR_FCRC_1 | CR_JPEG | CR_ESS);
-
 	/* Set bus width */
 	switch (dcmi->bus.bus_width) {
 	case 14:
-		val &= CR_EDM_0 + CR_EDM_1;
+		val |= CR_EDM_0 | CR_EDM_1;
 		break;
 	case 12:
-		val &= CR_EDM_1;
+		val |= CR_EDM_1;
 		break;
 	case 10:
-		val &= CR_EDM_0;
+		val |= CR_EDM_0;
 		break;
 	default:
 		/* Set bus width to 8 bits by default */
@@ -542,6 +574,10 @@ static int dcmi_start_streaming(struct vb2_queue *vq, unsigned int count)
 
 	reg_write(dcmi->regs, DCMI_CR, val);
 
+	/* Set crop */
+	if (dcmi->do_crop)
+		dcmi_set_crop(dcmi);
+
 	/* Enable dcmi */
 	reg_set(dcmi->regs, DCMI_CR, CR_ENABLE);
 
@@ -662,7 +698,7 @@ static void dcmi_stop_streaming(struct vb2_queue *vq)
 		dcmi->errors_count, dcmi->buffers_count);
 }
 
-static struct vb2_ops dcmi_video_qops = {
+static const struct vb2_ops dcmi_video_qops = {
 	.queue_setup		= dcmi_queue_setup,
 	.buf_init		= dcmi_buf_init,
 	.buf_prepare		= dcmi_buf_prepare,
@@ -686,12 +722,12 @@ static int dcmi_g_fmt_vid_cap(struct file *file, void *priv,
 static const struct dcmi_format *find_format_by_fourcc(struct stm32_dcmi *dcmi,
 						       unsigned int fourcc)
 {
-	unsigned int num_formats = dcmi->num_user_formats;
+	unsigned int num_formats = dcmi->num_of_sd_formats;
 	const struct dcmi_format *fmt;
 	unsigned int i;
 
 	for (i = 0; i < num_formats; i++) {
-		fmt = dcmi->user_formats[i];
+		fmt = dcmi->sd_formats[i];
 		if (fmt->fourcc == fourcc)
 			return fmt;
 	}
@@ -699,41 +735,108 @@ static const struct dcmi_format *find_format_by_fourcc(struct stm32_dcmi *dcmi,
 	return NULL;
 }
 
+static void __find_outer_frame_size(struct stm32_dcmi *dcmi,
+				    struct v4l2_pix_format *pix,
+				    struct dcmi_framesize *framesize)
+{
+	struct dcmi_framesize *match = NULL;
+	unsigned int i;
+	unsigned int min_err = UINT_MAX;
+
+	for (i = 0; i < dcmi->num_of_sd_framesizes; i++) {
+		struct dcmi_framesize *fsize = &dcmi->sd_framesizes[i];
+		int w_err = (fsize->width - pix->width);
+		int h_err = (fsize->height - pix->height);
+		int err = w_err + h_err;
+
+		if ((w_err >= 0) && (h_err >= 0) && (err < min_err)) {
+			min_err = err;
+			match = fsize;
+		}
+	}
+	if (!match)
+		match = &dcmi->sd_framesizes[0];
+
+	*framesize = *match;
+}
+
 static int dcmi_try_fmt(struct stm32_dcmi *dcmi, struct v4l2_format *f,
-			const struct dcmi_format **current_fmt)
+			const struct dcmi_format **sd_format,
+			struct dcmi_framesize *sd_framesize)
 {
-	const struct dcmi_format *dcmi_fmt;
-	struct v4l2_pix_format *pixfmt = &f->fmt.pix;
+	const struct dcmi_format *sd_fmt;
+	struct dcmi_framesize sd_fsize;
+	struct v4l2_pix_format *pix = &f->fmt.pix;
 	struct v4l2_subdev_pad_config pad_cfg;
 	struct v4l2_subdev_format format = {
 		.which = V4L2_SUBDEV_FORMAT_TRY,
 	};
 	int ret;
 
-	dcmi_fmt = find_format_by_fourcc(dcmi, pixfmt->pixelformat);
-	if (!dcmi_fmt) {
-		dcmi_fmt = dcmi->user_formats[dcmi->num_user_formats - 1];
-		pixfmt->pixelformat = dcmi_fmt->fourcc;
+	sd_fmt = find_format_by_fourcc(dcmi, pix->pixelformat);
+	if (!sd_fmt) {
+		sd_fmt = dcmi->sd_formats[dcmi->num_of_sd_formats - 1];
+		pix->pixelformat = sd_fmt->fourcc;
 	}
 
 	/* Limit to hardware capabilities */
-	pixfmt->width = clamp(pixfmt->width, MIN_WIDTH, MAX_WIDTH);
-	pixfmt->height = clamp(pixfmt->height, MIN_HEIGHT, MAX_HEIGHT);
+	pix->width = clamp(pix->width, MIN_WIDTH, MAX_WIDTH);
+	pix->height = clamp(pix->height, MIN_HEIGHT, MAX_HEIGHT);
 
-	v4l2_fill_mbus_format(&format.format, pixfmt, dcmi_fmt->mbus_code);
+	if (dcmi->do_crop && dcmi->num_of_sd_framesizes) {
+		struct dcmi_framesize outer_sd_fsize;
+		/*
+		 * If crop is requested and sensor have discrete frame sizes,
+		 * select the frame size that is just larger than request
+		 */
+		__find_outer_frame_size(dcmi, pix, &outer_sd_fsize);
+		pix->width = outer_sd_fsize.width;
+		pix->height = outer_sd_fsize.height;
+	}
+
+	v4l2_fill_mbus_format(&format.format, pix, sd_fmt->mbus_code);
 	ret = v4l2_subdev_call(dcmi->entity.subdev, pad, set_fmt,
 			       &pad_cfg, &format);
 	if (ret < 0)
 		return ret;
 
-	v4l2_fill_pix_format(pixfmt, &format.format);
+	/* Update pix regarding to what sensor can do */
+	v4l2_fill_pix_format(pix, &format.format);
+
+	/* Save resolution that sensor can actually do */
+	sd_fsize.width = pix->width;
+	sd_fsize.height = pix->height;
 
-	pixfmt->field = V4L2_FIELD_NONE;
-	pixfmt->bytesperline = pixfmt->width * dcmi_fmt->bpp;
-	pixfmt->sizeimage = pixfmt->bytesperline * pixfmt->height;
+	if (dcmi->do_crop) {
+		struct v4l2_rect c = dcmi->crop;
+		struct v4l2_rect max_rect;
 
-	if (current_fmt)
-		*current_fmt = dcmi_fmt;
+		/*
+		 * Adjust crop by making the intersection between
+		 * format resolution request and crop request
+		 */
+		max_rect.top = 0;
+		max_rect.left = 0;
+		max_rect.width = pix->width;
+		max_rect.height = pix->height;
+		v4l2_rect_map_inside(&c, &max_rect);
+		c.top  = clamp_t(s32, c.top, 0, pix->height - c.height);
+		c.left = clamp_t(s32, c.left, 0, pix->width - c.width);
+		dcmi->crop = c;
+
+		/* Adjust format resolution request to crop */
+		pix->width = dcmi->crop.width;
+		pix->height = dcmi->crop.height;
+	}
+
+	pix->field = V4L2_FIELD_NONE;
+	pix->bytesperline = pix->width * sd_fmt->bpp;
+	pix->sizeimage = pix->bytesperline * pix->height;
+
+	if (sd_format)
+		*sd_format = sd_fmt;
+	if (sd_framesize)
+		*sd_framesize = sd_fsize;
 
 	return 0;
 }
@@ -743,22 +846,42 @@ static int dcmi_set_fmt(struct stm32_dcmi *dcmi, struct v4l2_format *f)
 	struct v4l2_subdev_format format = {
 		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
 	};
-	const struct dcmi_format *current_fmt;
+	const struct dcmi_format *sd_format;
+	struct dcmi_framesize sd_framesize;
+	struct v4l2_mbus_framefmt *mf = &format.format;
+	struct v4l2_pix_format *pix = &f->fmt.pix;
 	int ret;
 
-	ret = dcmi_try_fmt(dcmi, f, &current_fmt);
+	/*
+	 * Try format, fmt.width/height could have been changed
+	 * to match sensor capability or crop request
+	 * sd_format & sd_framesize will contain what subdev
+	 * can do for this request.
+	 */
+	ret = dcmi_try_fmt(dcmi, f, &sd_format, &sd_framesize);
 	if (ret)
 		return ret;
 
-	v4l2_fill_mbus_format(&format.format, &f->fmt.pix,
-			      current_fmt->mbus_code);
+	/* pix to mbus format */
+	v4l2_fill_mbus_format(mf, pix,
+			      sd_format->mbus_code);
+	mf->width = sd_framesize.width;
+	mf->height = sd_framesize.height;
+
 	ret = v4l2_subdev_call(dcmi->entity.subdev, pad,
 			       set_fmt, NULL, &format);
 	if (ret < 0)
 		return ret;
 
+	dev_dbg(dcmi->dev, "Sensor format set to 0x%x %ux%u\n",
+		mf->code, mf->width, mf->height);
+	dev_dbg(dcmi->dev, "Buffer format set to %4.4s %ux%u\n",
+		(char *)&pix->pixelformat,
+		pix->width, pix->height);
+
 	dcmi->fmt = *f;
-	dcmi->current_fmt = current_fmt;
+	dcmi->sd_format = sd_format;
+	dcmi->sd_framesize = sd_framesize;
 
 	return 0;
 }
@@ -779,7 +902,7 @@ static int dcmi_try_fmt_vid_cap(struct file *file, void *priv,
 {
 	struct stm32_dcmi *dcmi = video_drvdata(file);
 
-	return dcmi_try_fmt(dcmi, f, NULL);
+	return dcmi_try_fmt(dcmi, f, NULL, NULL);
 }
 
 static int dcmi_enum_fmt_vid_cap(struct file *file, void  *priv,
@@ -787,10 +910,197 @@ static int dcmi_enum_fmt_vid_cap(struct file *file, void  *priv,
 {
 	struct stm32_dcmi *dcmi = video_drvdata(file);
 
-	if (f->index >= dcmi->num_user_formats)
+	if (f->index >= dcmi->num_of_sd_formats)
 		return -EINVAL;
 
-	f->pixelformat = dcmi->user_formats[f->index]->fourcc;
+	f->pixelformat = dcmi->sd_formats[f->index]->fourcc;
+	return 0;
+}
+
+static int dcmi_get_sensor_format(struct stm32_dcmi *dcmi,
+				  struct v4l2_pix_format *pix)
+{
+	struct v4l2_subdev_format fmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+	};
+	int ret;
+
+	ret = v4l2_subdev_call(dcmi->entity.subdev, pad, get_fmt, NULL, &fmt);
+	if (ret)
+		return ret;
+
+	v4l2_fill_pix_format(pix, &fmt.format);
+
+	return 0;
+}
+
+static int dcmi_set_sensor_format(struct stm32_dcmi *dcmi,
+				  struct v4l2_pix_format *pix)
+{
+	const struct dcmi_format *sd_fmt;
+	struct v4l2_subdev_format format = {
+		.which = V4L2_SUBDEV_FORMAT_TRY,
+	};
+	struct v4l2_subdev_pad_config pad_cfg;
+	int ret;
+
+	sd_fmt = find_format_by_fourcc(dcmi, pix->pixelformat);
+	if (!sd_fmt) {
+		sd_fmt = dcmi->sd_formats[dcmi->num_of_sd_formats - 1];
+		pix->pixelformat = sd_fmt->fourcc;
+	}
+
+	v4l2_fill_mbus_format(&format.format, pix, sd_fmt->mbus_code);
+	ret = v4l2_subdev_call(dcmi->entity.subdev, pad, set_fmt,
+			       &pad_cfg, &format);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int dcmi_get_sensor_bounds(struct stm32_dcmi *dcmi,
+				  struct v4l2_rect *r)
+{
+	struct v4l2_subdev_selection bounds = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		.target = V4L2_SEL_TGT_CROP_BOUNDS,
+	};
+	unsigned int max_width, max_height, max_pixsize;
+	struct v4l2_pix_format pix;
+	unsigned int i;
+	int ret;
+
+	/*
+	 * Get sensor bounds first
+	 */
+	ret = v4l2_subdev_call(dcmi->entity.subdev, pad, get_selection,
+			       NULL, &bounds);
+	if (!ret)
+		*r = bounds.r;
+	if (ret != -ENOIOCTLCMD)
+		return ret;
+
+	/*
+	 * If selection is not implemented,
+	 * fallback by enumerating sensor frame sizes
+	 * and take the largest one
+	 */
+	max_width = 0;
+	max_height = 0;
+	max_pixsize = 0;
+	for (i = 0; i < dcmi->num_of_sd_framesizes; i++) {
+		struct dcmi_framesize *fsize = &dcmi->sd_framesizes[i];
+		unsigned int pixsize = fsize->width * fsize->height;
+
+		if (pixsize > max_pixsize) {
+			max_pixsize = pixsize;
+			max_width = fsize->width;
+			max_height = fsize->height;
+		}
+	}
+	if (max_pixsize > 0) {
+		r->top = 0;
+		r->left = 0;
+		r->width = max_width;
+		r->height = max_height;
+		return 0;
+	}
+
+	/*
+	 * If frame sizes enumeration is not implemented,
+	 * fallback by getting current sensor frame size
+	 */
+	ret = dcmi_get_sensor_format(dcmi, &pix);
+	if (ret)
+		return ret;
+
+	r->top = 0;
+	r->left = 0;
+	r->width = pix.width;
+	r->height = pix.height;
+
+	return 0;
+}
+
+static int dcmi_g_selection(struct file *file, void *fh,
+			    struct v4l2_selection *s)
+{
+	struct stm32_dcmi *dcmi = video_drvdata(file);
+
+	if (s->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+		return -EINVAL;
+
+	switch (s->target) {
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+		s->r = dcmi->sd_bounds;
+		return 0;
+	case V4L2_SEL_TGT_CROP:
+		if (dcmi->do_crop) {
+			s->r = dcmi->crop;
+		} else {
+			s->r.top = 0;
+			s->r.left = 0;
+			s->r.width = dcmi->fmt.fmt.pix.width;
+			s->r.height = dcmi->fmt.fmt.pix.height;
+		}
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int dcmi_s_selection(struct file *file, void *priv,
+			    struct v4l2_selection *s)
+{
+	struct stm32_dcmi *dcmi = video_drvdata(file);
+	struct v4l2_rect r = s->r;
+	struct v4l2_rect max_rect;
+	struct v4l2_pix_format pix;
+
+	if (s->type != V4L2_BUF_TYPE_VIDEO_CAPTURE ||
+	    s->target != V4L2_SEL_TGT_CROP)
+		return -EINVAL;
+
+	/* Reset sensor resolution to max resolution */
+	pix.pixelformat = dcmi->fmt.fmt.pix.pixelformat;
+	pix.width = dcmi->sd_bounds.width;
+	pix.height = dcmi->sd_bounds.height;
+	dcmi_set_sensor_format(dcmi, &pix);
+
+	/*
+	 * Make the intersection between
+	 * sensor resolution
+	 * and crop request
+	 */
+	max_rect.top = 0;
+	max_rect.left = 0;
+	max_rect.width = pix.width;
+	max_rect.height = pix.height;
+	v4l2_rect_map_inside(&r, &max_rect);
+	r.top  = clamp_t(s32, r.top, 0, pix.height - r.height);
+	r.left = clamp_t(s32, r.left, 0, pix.width - r.width);
+
+	if (!((r.top == dcmi->sd_bounds.top) &&
+	      (r.left == dcmi->sd_bounds.left) &&
+	      (r.width == dcmi->sd_bounds.width) &&
+	      (r.height == dcmi->sd_bounds.height))) {
+		/* Crop if request is different than sensor resolution */
+		dcmi->do_crop = true;
+		dcmi->crop = r;
+		dev_dbg(dcmi->dev, "s_selection: crop %ux%u@(%u,%u) from %ux%u\n",
+			r.width, r.height, r.left, r.top,
+			pix.width, pix.height);
+	} else {
+		/* Disable crop */
+		dcmi->do_crop = false;
+		dev_dbg(dcmi->dev, "s_selection: crop is disabled\n");
+	}
+
+	s->r = r;
 	return 0;
 }
 
@@ -832,18 +1142,18 @@ static int dcmi_enum_framesizes(struct file *file, void *fh,
 				struct v4l2_frmsizeenum *fsize)
 {
 	struct stm32_dcmi *dcmi = video_drvdata(file);
-	const struct dcmi_format *dcmi_fmt;
+	const struct dcmi_format *sd_fmt;
 	struct v4l2_subdev_frame_size_enum fse = {
 		.index = fsize->index,
 		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
 	};
 	int ret;
 
-	dcmi_fmt = find_format_by_fourcc(dcmi, fsize->pixel_format);
-	if (!dcmi_fmt)
+	sd_fmt = find_format_by_fourcc(dcmi, fsize->pixel_format);
+	if (!sd_fmt)
 		return -EINVAL;
 
-	fse.code = dcmi_fmt->mbus_code;
+	fse.code = sd_fmt->mbus_code;
 
 	ret = v4l2_subdev_call(dcmi->entity.subdev, pad, enum_frame_size,
 			       NULL, &fse);
@@ -861,7 +1171,7 @@ static int dcmi_enum_frameintervals(struct file *file, void *fh,
 				    struct v4l2_frmivalenum *fival)
 {
 	struct stm32_dcmi *dcmi = video_drvdata(file);
-	const struct dcmi_format *dcmi_fmt;
+	const struct dcmi_format *sd_fmt;
 	struct v4l2_subdev_frame_interval_enum fie = {
 		.index = fival->index,
 		.width = fival->width,
@@ -870,11 +1180,11 @@ static int dcmi_enum_frameintervals(struct file *file, void *fh,
 	};
 	int ret;
 
-	dcmi_fmt = find_format_by_fourcc(dcmi, fival->pixel_format);
-	if (!dcmi_fmt)
+	sd_fmt = find_format_by_fourcc(dcmi, fival->pixel_format);
+	if (!sd_fmt)
 		return -EINVAL;
 
-	fie.code = dcmi_fmt->mbus_code;
+	fie.code = sd_fmt->mbus_code;
 
 	ret = v4l2_subdev_call(dcmi->entity.subdev, pad,
 			       enum_frame_interval, NULL, &fie);
@@ -952,6 +1262,8 @@ static const struct v4l2_ioctl_ops dcmi_ioctl_ops = {
 	.vidioc_g_fmt_vid_cap		= dcmi_g_fmt_vid_cap,
 	.vidioc_s_fmt_vid_cap		= dcmi_s_fmt_vid_cap,
 	.vidioc_enum_fmt_vid_cap	= dcmi_enum_fmt_vid_cap,
+	.vidioc_g_selection		= dcmi_g_selection,
+	.vidioc_s_selection		= dcmi_s_selection,
 
 	.vidioc_enum_input		= dcmi_enum_input,
 	.vidioc_g_input			= dcmi_g_input,
@@ -996,15 +1308,15 @@ static int dcmi_set_default_fmt(struct stm32_dcmi *dcmi)
 			.width		= CIF_WIDTH,
 			.height		= CIF_HEIGHT,
 			.field		= V4L2_FIELD_NONE,
-			.pixelformat	= dcmi->user_formats[0]->fourcc,
+			.pixelformat	= dcmi->sd_formats[0]->fourcc,
 		},
 	};
 	int ret;
 
-	ret = dcmi_try_fmt(dcmi, &f, NULL);
+	ret = dcmi_try_fmt(dcmi, &f, NULL, NULL);
 	if (ret)
 		return ret;
-	dcmi->current_fmt = dcmi->user_formats[0];
+	dcmi->sd_format = dcmi->sd_formats[0];
 	dcmi->fmt = f;
 	return 0;
 }
@@ -1027,7 +1339,7 @@ static const struct dcmi_format dcmi_formats[] = {
 
 static int dcmi_formats_init(struct stm32_dcmi *dcmi)
 {
-	const struct dcmi_format *dcmi_fmts[ARRAY_SIZE(dcmi_formats)];
+	const struct dcmi_format *sd_fmts[ARRAY_SIZE(dcmi_formats)];
 	unsigned int num_fmts = 0, i, j;
 	struct v4l2_subdev *subdev = dcmi->entity.subdev;
 	struct v4l2_subdev_mbus_code_enum mbus_code = {
@@ -1042,13 +1354,13 @@ static int dcmi_formats_init(struct stm32_dcmi *dcmi)
 
 			/* Code supported, have we got this fourcc yet? */
 			for (j = 0; j < num_fmts; j++)
-				if (dcmi_fmts[j]->fourcc ==
+				if (sd_fmts[j]->fourcc ==
 						dcmi_formats[i].fourcc)
 					/* Already available */
 					break;
 			if (j == num_fmts)
 				/* New */
-				dcmi_fmts[num_fmts++] = dcmi_formats + i;
+				sd_fmts[num_fmts++] = dcmi_formats + i;
 		}
 		mbus_code.index++;
 	}
@@ -1056,18 +1368,63 @@ static int dcmi_formats_init(struct stm32_dcmi *dcmi)
 	if (!num_fmts)
 		return -ENXIO;
 
-	dcmi->num_user_formats = num_fmts;
-	dcmi->user_formats = devm_kcalloc(dcmi->dev,
-					 num_fmts, sizeof(struct dcmi_format *),
-					 GFP_KERNEL);
-	if (!dcmi->user_formats) {
-		dev_err(dcmi->dev, "could not allocate memory\n");
+	dcmi->num_of_sd_formats = num_fmts;
+	dcmi->sd_formats = devm_kcalloc(dcmi->dev,
+					num_fmts, sizeof(struct dcmi_format *),
+					GFP_KERNEL);
+	if (!dcmi->sd_formats) {
+		dev_err(dcmi->dev, "Could not allocate memory\n");
 		return -ENOMEM;
 	}
 
-	memcpy(dcmi->user_formats, dcmi_fmts,
+	memcpy(dcmi->sd_formats, sd_fmts,
 	       num_fmts * sizeof(struct dcmi_format *));
-	dcmi->current_fmt = dcmi->user_formats[0];
+	dcmi->sd_format = dcmi->sd_formats[0];
+
+	return 0;
+}
+
+static int dcmi_framesizes_init(struct stm32_dcmi *dcmi)
+{
+	unsigned int num_fsize = 0;
+	struct v4l2_subdev *subdev = dcmi->entity.subdev;
+	struct v4l2_subdev_frame_size_enum fse = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		.code = dcmi->sd_format->mbus_code,
+	};
+	unsigned int ret;
+	unsigned int i;
+
+	/* Allocate discrete framesizes array */
+	while (!v4l2_subdev_call(subdev, pad, enum_frame_size,
+				 NULL, &fse))
+		fse.index++;
+
+	num_fsize = fse.index;
+	if (!num_fsize)
+		return 0;
+
+	dcmi->num_of_sd_framesizes = num_fsize;
+	dcmi->sd_framesizes = devm_kcalloc(dcmi->dev, num_fsize,
+					   sizeof(struct dcmi_framesize),
+					   GFP_KERNEL);
+	if (!dcmi->sd_framesizes) {
+		dev_err(dcmi->dev, "Could not allocate memory\n");
+		return -ENOMEM;
+	}
+
+	/* Fill array with sensor supported framesizes */
+	dev_dbg(dcmi->dev, "Sensor supports %u frame sizes:\n", num_fsize);
+	for (i = 0; i < dcmi->num_of_sd_framesizes; i++) {
+		fse.index = i;
+		ret = v4l2_subdev_call(subdev, pad, enum_frame_size,
+				       NULL, &fse);
+		if (ret)
+			return ret;
+		dcmi->sd_framesizes[fse.index].width = fse.max_width;
+		dcmi->sd_framesizes[fse.index].height = fse.max_height;
+		dev_dbg(dcmi->dev, "%ux%u\n", fse.max_width, fse.max_height);
+	}
 
 	return 0;
 }
@@ -1084,6 +1441,18 @@ static int dcmi_graph_notify_complete(struct v4l2_async_notifier *notifier)
 		return ret;
 	}
 
+	ret = dcmi_framesizes_init(dcmi);
+	if (ret) {
+		dev_err(dcmi->dev, "Could not initialize framesizes\n");
+		return ret;
+	}
+
+	ret = dcmi_get_sensor_bounds(dcmi, &dcmi->sd_bounds);
+	if (ret) {
+		dev_err(dcmi->dev, "Could not get sensor bounds\n");
+		return ret;
+	}
+
 	ret = dcmi_set_default_fmt(dcmi);
 	if (ret) {
 		dev_err(dcmi->dev, "Could not set default format\n");
@@ -1209,7 +1578,7 @@ static int dcmi_probe(struct platform_device *pdev)
 	if (!dcmi)
 		return -ENOMEM;
 
-	dcmi->rstc = devm_reset_control_get(&pdev->dev, NULL);
+	dcmi->rstc = devm_reset_control_get_exclusive(&pdev->dev, NULL);
 	if (IS_ERR(dcmi->rstc)) {
 		dev_err(&pdev->dev, "Could not get reset control\n");
 		return -ENODEV;
diff --git a/drivers/media/platform/ti-vpe/cal.c b/drivers/media/platform/ti-vpe/cal.c
index 177faa36bc16..42e383a48ffe 100644
--- a/drivers/media/platform/ti-vpe/cal.c
+++ b/drivers/media/platform/ti-vpe/cal.c
@@ -1420,7 +1420,7 @@ static const struct v4l2_ioctl_ops cal_ioctl_ops = {
 	.vidioc_unsubscribe_event = v4l2_event_unsubscribe,
 };
 
-static struct video_device cal_videodev = {
+static const struct video_device cal_videodev = {
 	.name		= CAL_MODULE_NAME,
 	.fops		= &cal_fops,
 	.ioctl_ops	= &cal_ioctl_ops,
@@ -1702,7 +1702,7 @@ static int of_cal_create_instance(struct cal_ctx *ctx, int inst)
 	asd->match_type = V4L2_ASYNC_MATCH_FWNODE;
 	asd->match.fwnode.fwnode = of_fwnode_handle(sensor_node);
 
-	remote_ep = of_parse_phandle(ep_node, "remote-endpoint", 0);
+	remote_ep = of_graph_get_remote_endpoint(ep_node);
 	if (!remote_ep) {
 		ctx_dbg(3, ctx, "can't get remote-endpoint\n");
 		goto cleanup_exit;
diff --git a/drivers/media/platform/ti-vpe/vpe.c b/drivers/media/platform/ti-vpe/vpe.c
index c47151495b6f..45bd10544189 100644
--- a/drivers/media/platform/ti-vpe/vpe.c
+++ b/drivers/media/platform/ti-vpe/vpe.c
@@ -2421,7 +2421,7 @@ static const struct v4l2_file_operations vpe_fops = {
 	.mmap		= v4l2_m2m_fop_mmap,
 };
 
-static struct video_device vpe_videodev = {
+static const struct video_device vpe_videodev = {
 	.name		= VPE_MODULE_NAME,
 	.fops		= &vpe_fops,
 	.ioctl_ops	= &vpe_ioctl_ops,
@@ -2430,7 +2430,7 @@ static struct video_device vpe_videodev = {
 	.vfl_dir	= VFL_DIR_M2M,
 };
 
-static struct v4l2_m2m_ops m2m_ops = {
+static const struct v4l2_m2m_ops m2m_ops = {
 	.device_run	= device_run,
 	.job_ready	= job_ready,
 	.job_abort	= job_abort,
diff --git a/drivers/media/platform/via-camera.c b/drivers/media/platform/via-camera.c
index e16f70a5df1d..805d4a8fc17e 100644
--- a/drivers/media/platform/via-camera.c
+++ b/drivers/media/platform/via-camera.c
@@ -1259,7 +1259,7 @@ static struct viafb_pm_hooks viacam_pm_hooks = {
  * Setup stuff.
  */
 
-static struct video_device viacam_v4l_template = {
+static const struct video_device viacam_v4l_template = {
 	.name		= "via-camera",
 	.minor		= -1,
 	.tvnorms	= V4L2_STD_NTSC_M,
diff --git a/drivers/media/platform/video-mux.c b/drivers/media/platform/video-mux.c
index 665744716f73..ee89ad76bee2 100644
--- a/drivers/media/platform/video-mux.c
+++ b/drivers/media/platform/video-mux.c
@@ -17,8 +17,7 @@
 #include <linux/err.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
-#include <linux/regmap.h>
-#include <linux/mfd/syscon.h>
+#include <linux/mux/consumer.h>
 #include <linux/of.h>
 #include <linux/of_graph.h>
 #include <linux/platform_device.h>
@@ -30,7 +29,7 @@ struct video_mux {
 	struct v4l2_subdev subdev;
 	struct media_pad *pads;
 	struct v4l2_mbus_framefmt *format_mbus;
-	struct regmap_field *field;
+	struct mux_control *mux;
 	struct mutex lock;
 	int active;
 };
@@ -71,7 +70,7 @@ static int video_mux_link_setup(struct media_entity *entity,
 		}
 
 		dev_dbg(sd->dev, "setting %d active\n", local->index);
-		ret = regmap_field_write(vmux->field, local->index);
+		ret = mux_control_try_select(vmux->mux, local->index);
 		if (ret < 0)
 			goto out;
 		vmux->active = local->index;
@@ -80,6 +79,7 @@ static int video_mux_link_setup(struct media_entity *entity,
 			goto out;
 
 		dev_dbg(sd->dev, "going inactive\n");
+		mux_control_deselect(vmux->mux);
 		vmux->active = -1;
 	}
 
@@ -193,46 +193,6 @@ static const struct v4l2_subdev_ops video_mux_subdev_ops = {
 	.video = &video_mux_subdev_video_ops,
 };
 
-static int video_mux_probe_mmio_mux(struct video_mux *vmux)
-{
-	struct device *dev = vmux->subdev.dev;
-	struct of_phandle_args args;
-	struct reg_field field;
-	struct regmap *regmap;
-	u32 reg, mask;
-	int ret;
-
-	ret = of_parse_phandle_with_args(dev->of_node, "mux-controls",
-					 "#mux-control-cells", 0, &args);
-	if (ret)
-		return ret;
-
-	if (!of_device_is_compatible(args.np, "mmio-mux"))
-		return -EINVAL;
-
-	regmap = syscon_node_to_regmap(args.np->parent);
-	if (IS_ERR(regmap))
-		return PTR_ERR(regmap);
-
-	ret = of_property_read_u32_index(args.np, "mux-reg-masks",
-					 2 * args.args[0], &reg);
-	if (!ret)
-		ret = of_property_read_u32_index(args.np, "mux-reg-masks",
-						 2 * args.args[0] + 1, &mask);
-	if (ret < 0)
-		return ret;
-
-	field.reg = reg;
-	field.msb = fls(mask) - 1;
-	field.lsb = ffs(mask) - 1;
-
-	vmux->field = devm_regmap_field_alloc(dev, regmap, field);
-	if (IS_ERR(vmux->field))
-		return PTR_ERR(vmux->field);
-
-	return 0;
-}
-
 static int video_mux_probe(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node;
@@ -270,8 +230,9 @@ static int video_mux_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
-	ret = video_mux_probe_mmio_mux(vmux);
-	if (ret) {
+	vmux->mux = devm_mux_control_get(dev, NULL);
+	if (IS_ERR(vmux->mux)) {
+		ret = PTR_ERR(vmux->mux);
 		if (ret != -EPROBE_DEFER)
 			dev_err(dev, "Failed to get mux: %d\n", ret);
 		return ret;
diff --git a/drivers/media/platform/vim2m.c b/drivers/media/platform/vim2m.c
index 970b9b6dab25..b01fba020d5f 100644
--- a/drivers/media/platform/vim2m.c
+++ b/drivers/media/platform/vim2m.c
@@ -974,7 +974,7 @@ static const struct v4l2_file_operations vim2m_fops = {
 	.mmap		= v4l2_m2m_fop_mmap,
 };
 
-static struct video_device vim2m_videodev = {
+static const struct video_device vim2m_videodev = {
 	.name		= MEM2MEM_NAME,
 	.vfl_dir	= VFL_DIR_M2M,
 	.fops		= &vim2m_fops,
@@ -983,7 +983,7 @@ static struct video_device vim2m_videodev = {
 	.release	= video_device_release_empty,
 };
 
-static struct v4l2_m2m_ops m2m_ops = {
+static const struct v4l2_m2m_ops m2m_ops = {
 	.device_run	= device_run,
 	.job_ready	= job_ready,
 	.job_abort	= job_abort,
diff --git a/drivers/media/platform/vimc/vimc-debayer.c b/drivers/media/platform/vimc/vimc-debayer.c
index 033a131f67af..4d663e89d33f 100644
--- a/drivers/media/platform/vimc/vimc-debayer.c
+++ b/drivers/media/platform/vimc/vimc-debayer.c
@@ -373,7 +373,7 @@ static int vimc_deb_s_stream(struct v4l2_subdev *sd, int enable)
 	return 0;
 }
 
-static struct v4l2_subdev_video_ops vimc_deb_video_ops = {
+static const struct v4l2_subdev_video_ops vimc_deb_video_ops = {
 	.s_stream = vimc_deb_s_stream,
 };
 
diff --git a/drivers/media/platform/vimc/vimc-scaler.c b/drivers/media/platform/vimc/vimc-scaler.c
index 0a3e086e12f3..e1602e0bc230 100644
--- a/drivers/media/platform/vimc/vimc-scaler.c
+++ b/drivers/media/platform/vimc/vimc-scaler.c
@@ -267,7 +267,7 @@ static int vimc_sca_s_stream(struct v4l2_subdev *sd, int enable)
 	return 0;
 }
 
-static struct v4l2_subdev_video_ops vimc_sca_video_ops = {
+static const struct v4l2_subdev_video_ops vimc_sca_video_ops = {
 	.s_stream = vimc_sca_s_stream,
 };
 
diff --git a/drivers/media/platform/vimc/vimc-sensor.c b/drivers/media/platform/vimc/vimc-sensor.c
index 615c2b18dcfc..02e68c8fc02b 100644
--- a/drivers/media/platform/vimc/vimc-sensor.c
+++ b/drivers/media/platform/vimc/vimc-sensor.c
@@ -282,7 +282,7 @@ static int vimc_sen_s_stream(struct v4l2_subdev *sd, int enable)
 	return 0;
 }
 
-static struct v4l2_subdev_video_ops vimc_sen_video_ops = {
+static const struct v4l2_subdev_video_ops vimc_sen_video_ops = {
 	.s_stream = vimc_sen_s_stream,
 };
 
diff --git a/drivers/media/platform/vivid/vivid-cec.c b/drivers/media/platform/vivid/vivid-cec.c
index e15705758969..b55d278d38a7 100644
--- a/drivers/media/platform/vivid/vivid-cec.c
+++ b/drivers/media/platform/vivid/vivid-cec.c
@@ -22,6 +22,15 @@
 #include "vivid-core.h"
 #include "vivid-cec.h"
 
+#define CEC_TIM_START_BIT_TOTAL		4500
+#define CEC_TIM_START_BIT_LOW		3700
+#define CEC_TIM_START_BIT_HIGH		800
+#define CEC_TIM_DATA_BIT_TOTAL		2400
+#define CEC_TIM_DATA_BIT_0_LOW		1500
+#define CEC_TIM_DATA_BIT_0_HIGH		900
+#define CEC_TIM_DATA_BIT_1_LOW		600
+#define CEC_TIM_DATA_BIT_1_HIGH		1800
+
 void vivid_cec_bus_free_work(struct vivid_dev *dev)
 {
 	spin_lock(&dev->cec_slock);
@@ -64,6 +73,58 @@ static bool vivid_cec_find_dest_adap(struct vivid_dev *dev,
 	return false;
 }
 
+static void vivid_cec_pin_adap_events(struct cec_adapter *adap, ktime_t ts,
+				      const struct cec_msg *msg, bool nacked)
+{
+	unsigned int len = nacked ? 1 : msg->len;
+	unsigned int i;
+	bool bit;
+
+	if (adap == NULL)
+		return;
+	ts = ktime_sub_us(ts, (CEC_TIM_START_BIT_TOTAL +
+			       len * 10 * CEC_TIM_DATA_BIT_TOTAL));
+	cec_queue_pin_cec_event(adap, false, ts);
+	ts = ktime_add_us(ts, CEC_TIM_START_BIT_LOW);
+	cec_queue_pin_cec_event(adap, true, ts);
+	ts = ktime_add_us(ts, CEC_TIM_START_BIT_HIGH);
+
+	for (i = 0; i < 10 * len; i++) {
+		switch (i % 10) {
+		case 0 ... 7:
+			bit = msg->msg[i / 10] & (0x80 >> (i % 10));
+			break;
+		case 8: /* EOM */
+			bit = i / 10 == msg->len - 1;
+			break;
+		case 9: /* ACK */
+			bit = cec_msg_is_broadcast(msg) ^ nacked;
+			break;
+		}
+		cec_queue_pin_cec_event(adap, false, ts);
+		if (bit)
+			ts = ktime_add_us(ts, CEC_TIM_DATA_BIT_1_LOW);
+		else
+			ts = ktime_add_us(ts, CEC_TIM_DATA_BIT_0_LOW);
+		cec_queue_pin_cec_event(adap, true, ts);
+		if (bit)
+			ts = ktime_add_us(ts, CEC_TIM_DATA_BIT_1_HIGH);
+		else
+			ts = ktime_add_us(ts, CEC_TIM_DATA_BIT_0_HIGH);
+	}
+}
+
+static void vivid_cec_pin_events(struct vivid_dev *dev,
+				 const struct cec_msg *msg, bool nacked)
+{
+	ktime_t ts = ktime_get();
+	unsigned int i;
+
+	vivid_cec_pin_adap_events(dev->cec_rx_adap, ts, msg, nacked);
+	for (i = 0; i < MAX_OUTPUTS; i++)
+		vivid_cec_pin_adap_events(dev->cec_tx_adap[i], ts, msg, nacked);
+}
+
 static void vivid_cec_xfer_done_worker(struct work_struct *work)
 {
 	struct vivid_cec_work *cw =
@@ -84,6 +145,7 @@ static void vivid_cec_xfer_done_worker(struct work_struct *work)
 	dev->cec_xfer_start_jiffies = 0;
 	list_del(&cw->list);
 	spin_unlock(&dev->cec_slock);
+	vivid_cec_pin_events(dev, &cw->msg, !valid_dest);
 	cec_transmit_attempt_done(cw->adap, cw->tx_status);
 
 	/* Broadcast message */
@@ -118,6 +180,7 @@ static void vivid_cec_xfer_try_worker(struct work_struct *work)
 
 static int vivid_cec_adap_enable(struct cec_adapter *adap, bool enable)
 {
+	adap->cec_pin_is_high = true;
 	return 0;
 }
 
@@ -219,8 +282,7 @@ struct cec_adapter *vivid_cec_alloc_adap(struct vivid_dev *dev,
 					 bool is_source)
 {
 	char name[sizeof(dev->vid_out_dev.name) + 2];
-	u32 caps = CEC_CAP_TRANSMIT | CEC_CAP_LOG_ADDRS |
-		CEC_CAP_PASSTHROUGH | CEC_CAP_RC | CEC_CAP_MONITOR_ALL;
+	u32 caps = CEC_CAP_DEFAULTS | CEC_CAP_MONITOR_ALL | CEC_CAP_MONITOR_PIN;
 
 	snprintf(name, sizeof(name), "%s%d",
 		 is_source ? dev->vid_out_dev.name : dev->vid_cap_dev.name,
diff --git a/drivers/media/platform/vivid/vivid-core.c b/drivers/media/platform/vivid/vivid-core.c
index ef344b9a48af..5f316a5e38db 100644
--- a/drivers/media/platform/vivid/vivid-core.c
+++ b/drivers/media/platform/vivid/vivid-core.c
@@ -1201,8 +1201,8 @@ static int vivid_create_instance(struct platform_device *pdev, int inst)
 				goto unreg_dev;
 			}
 			cec_s_phys_addr(adap, 0, false);
-			v4l2_info(&dev->v4l2_dev, "CEC adapter %s registered for HDMI input %d\n",
-				  dev_name(&adap->devnode.dev), i);
+			v4l2_info(&dev->v4l2_dev, "CEC adapter %s registered for HDMI input 0\n",
+				  dev_name(&adap->devnode.dev));
 		}
 #endif
 
@@ -1255,13 +1255,13 @@ static int vivid_create_instance(struct platform_device *pdev, int inst)
 				dev->cec_tx_adap[bus_cnt] = NULL;
 				goto unreg_dev;
 			}
+			v4l2_info(&dev->v4l2_dev, "CEC adapter %s registered for HDMI output %d\n",
+				  dev_name(&adap->devnode.dev), bus_cnt);
 			bus_cnt++;
 			if (bus_cnt <= out_type_counter[HDMI])
 				cec_s_phys_addr(adap, bus_cnt << 12, false);
 			else
 				cec_s_phys_addr(adap, 0x1000, false);
-			v4l2_info(&dev->v4l2_dev, "CEC adapter %s registered for HDMI output %d\n",
-				  dev_name(&adap->devnode.dev), i);
 		}
 #endif
 
diff --git a/drivers/media/platform/vsp1/vsp1.h b/drivers/media/platform/vsp1/vsp1.h
index 847963b6e9eb..78ef838416b3 100644
--- a/drivers/media/platform/vsp1/vsp1.h
+++ b/drivers/media/platform/vsp1/vsp1.h
@@ -41,11 +41,11 @@ struct vsp1_rwpf;
 struct vsp1_sru;
 struct vsp1_uds;
 
+#define VSP1_MAX_LIF		2
 #define VSP1_MAX_RPF		5
 #define VSP1_MAX_UDS		3
 #define VSP1_MAX_WPF		4
 
-#define VSP1_HAS_LIF		(1 << 0)
 #define VSP1_HAS_LUT		(1 << 1)
 #define VSP1_HAS_SRU		(1 << 2)
 #define VSP1_HAS_BRU		(1 << 3)
@@ -54,12 +54,14 @@ struct vsp1_uds;
 #define VSP1_HAS_WPF_HFLIP	(1 << 6)
 #define VSP1_HAS_HGO		(1 << 7)
 #define VSP1_HAS_HGT		(1 << 8)
+#define VSP1_HAS_BRS		(1 << 9)
 
 struct vsp1_device_info {
 	u32 version;
 	const char *model;
 	unsigned int gen;
 	unsigned int features;
+	unsigned int lif_count;
 	unsigned int rpf_count;
 	unsigned int uds_count;
 	unsigned int wpf_count;
@@ -76,13 +78,14 @@ struct vsp1_device {
 	struct rcar_fcp_device *fcp;
 	struct device *bus_master;
 
+	struct vsp1_bru *brs;
 	struct vsp1_bru *bru;
 	struct vsp1_clu *clu;
 	struct vsp1_hgo *hgo;
 	struct vsp1_hgt *hgt;
 	struct vsp1_hsit *hsi;
 	struct vsp1_hsit *hst;
-	struct vsp1_lif *lif;
+	struct vsp1_lif *lif[VSP1_MAX_LIF];
 	struct vsp1_lut *lut;
 	struct vsp1_rwpf *rpf[VSP1_MAX_RPF];
 	struct vsp1_sru *sru;
diff --git a/drivers/media/platform/vsp1/vsp1_bru.c b/drivers/media/platform/vsp1/vsp1_bru.c
index 85362c5ef57a..e8fd2ae3b3eb 100644
--- a/drivers/media/platform/vsp1/vsp1_bru.c
+++ b/drivers/media/platform/vsp1/vsp1_bru.c
@@ -33,7 +33,7 @@
 static inline void vsp1_bru_write(struct vsp1_bru *bru, struct vsp1_dl_list *dl,
 				  u32 reg, u32 data)
 {
-	vsp1_dl_list_write(dl, reg, data);
+	vsp1_dl_list_write(dl, bru->base + reg, data);
 }
 
 /* -----------------------------------------------------------------------------
@@ -332,11 +332,14 @@ static void bru_configure(struct vsp1_entity *entity,
 	/*
 	 * Route BRU input 1 as SRC input to the ROP unit and configure the ROP
 	 * unit with a NOP operation to make BRU input 1 available as the
-	 * Blend/ROP unit B SRC input.
+	 * Blend/ROP unit B SRC input. Only needed for BRU, the BRS has no ROP
+	 * unit.
 	 */
-	vsp1_bru_write(bru, dl, VI6_BRU_ROP, VI6_BRU_ROP_DSTSEL_BRUIN(1) |
-		       VI6_BRU_ROP_CROP(VI6_ROP_NOP) |
-		       VI6_BRU_ROP_AROP(VI6_ROP_NOP));
+	if (entity->type == VSP1_ENTITY_BRU)
+		vsp1_bru_write(bru, dl, VI6_BRU_ROP,
+			       VI6_BRU_ROP_DSTSEL_BRUIN(1) |
+			       VI6_BRU_ROP_CROP(VI6_ROP_NOP) |
+			       VI6_BRU_ROP_AROP(VI6_ROP_NOP));
 
 	for (i = 0; i < bru->entity.source_pad; ++i) {
 		bool premultiplied = false;
@@ -366,12 +369,13 @@ static void bru_configure(struct vsp1_entity *entity,
 			ctrl |= VI6_BRU_CTRL_DSTSEL_VRPF;
 
 		/*
-		 * Route BRU inputs 0 to 3 as SRC inputs to Blend/ROP units A to
-		 * D in that order. The Blend/ROP unit B SRC is hardwired to the
-		 * ROP unit output, the corresponding register bits must be set
-		 * to 0.
+		 * Route inputs 0 to 3 as SRC inputs to Blend/ROP units A to D
+		 * in that order. In the BRU the Blend/ROP unit B SRC is
+		 * hardwired to the ROP unit output, the corresponding register
+		 * bits must be set to 0. The BRS has no ROP unit and doesn't
+		 * need any special processing.
 		 */
-		if (i != 1)
+		if (!(entity->type == VSP1_ENTITY_BRU && i == 1))
 			ctrl |= VI6_BRU_CTRL_SRCSEL_BRUIN(i);
 
 		vsp1_bru_write(bru, dl, VI6_BRU_CTRL(i), ctrl);
@@ -407,20 +411,31 @@ static const struct vsp1_entity_operations bru_entity_ops = {
  * Initialization and Cleanup
  */
 
-struct vsp1_bru *vsp1_bru_create(struct vsp1_device *vsp1)
+struct vsp1_bru *vsp1_bru_create(struct vsp1_device *vsp1,
+				 enum vsp1_entity_type type)
 {
 	struct vsp1_bru *bru;
+	unsigned int num_pads;
+	const char *name;
 	int ret;
 
 	bru = devm_kzalloc(vsp1->dev, sizeof(*bru), GFP_KERNEL);
 	if (bru == NULL)
 		return ERR_PTR(-ENOMEM);
 
+	bru->base = type == VSP1_ENTITY_BRU ? VI6_BRU_BASE : VI6_BRS_BASE;
 	bru->entity.ops = &bru_entity_ops;
-	bru->entity.type = VSP1_ENTITY_BRU;
+	bru->entity.type = type;
+
+	if (type == VSP1_ENTITY_BRU) {
+		num_pads = vsp1->info->num_bru_inputs + 1;
+		name = "bru";
+	} else {
+		num_pads = 3;
+		name = "brs";
+	}
 
-	ret = vsp1_entity_init(vsp1, &bru->entity, "bru",
-			       vsp1->info->num_bru_inputs + 1, &bru_ops,
+	ret = vsp1_entity_init(vsp1, &bru->entity, name, num_pads, &bru_ops,
 			       MEDIA_ENT_F_PROC_VIDEO_COMPOSER);
 	if (ret < 0)
 		return ERR_PTR(ret);
@@ -435,7 +450,7 @@ struct vsp1_bru *vsp1_bru_create(struct vsp1_device *vsp1)
 	bru->entity.subdev.ctrl_handler = &bru->ctrls;
 
 	if (bru->ctrls.error) {
-		dev_err(vsp1->dev, "bru: failed to initialize controls\n");
+		dev_err(vsp1->dev, "%s: failed to initialize controls\n", name);
 		ret = bru->ctrls.error;
 		vsp1_entity_destroy(&bru->entity);
 		return ERR_PTR(ret);
diff --git a/drivers/media/platform/vsp1/vsp1_bru.h b/drivers/media/platform/vsp1/vsp1_bru.h
index 828a3fcadea8..c98ed96d8de6 100644
--- a/drivers/media/platform/vsp1/vsp1_bru.h
+++ b/drivers/media/platform/vsp1/vsp1_bru.h
@@ -26,6 +26,7 @@ struct vsp1_rwpf;
 
 struct vsp1_bru {
 	struct vsp1_entity entity;
+	unsigned int base;
 
 	struct v4l2_ctrl_handler ctrls;
 
@@ -41,6 +42,7 @@ static inline struct vsp1_bru *to_bru(struct v4l2_subdev *subdev)
 	return container_of(subdev, struct vsp1_bru, entity.subdev);
 }
 
-struct vsp1_bru *vsp1_bru_create(struct vsp1_device *vsp1);
+struct vsp1_bru *vsp1_bru_create(struct vsp1_device *vsp1,
+				 enum vsp1_entity_type type);
 
 #endif /* __VSP1_BRU_H__ */
diff --git a/drivers/media/platform/vsp1/vsp1_dl.c b/drivers/media/platform/vsp1/vsp1_dl.c
index aaf17b13fd78..8b5cbb6b7a70 100644
--- a/drivers/media/platform/vsp1/vsp1_dl.c
+++ b/drivers/media/platform/vsp1/vsp1_dl.c
@@ -95,6 +95,7 @@ enum vsp1_dl_mode {
  * struct vsp1_dl_manager - Display List manager
  * @index: index of the related WPF
  * @mode: display list operation mode (header or headerless)
+ * @singleshot: execute the display list in single-shot mode
  * @vsp1: the VSP1 device
  * @lock: protects the free, active, queued, pending and gc_fragments lists
  * @free: array of all free display lists
@@ -107,6 +108,7 @@ enum vsp1_dl_mode {
 struct vsp1_dl_manager {
 	unsigned int index;
 	enum vsp1_dl_mode mode;
+	bool singleshot;
 	struct vsp1_device *vsp1;
 
 	spinlock_t lock;
@@ -437,6 +439,7 @@ int vsp1_dl_list_add_chain(struct vsp1_dl_list *head,
 
 static void vsp1_dl_list_fill_header(struct vsp1_dl_list *dl, bool is_last)
 {
+	struct vsp1_dl_manager *dlm = dl->dlm;
 	struct vsp1_dl_header_list *hdr = dl->header->lists;
 	struct vsp1_dl_body *dlb;
 	unsigned int num_lists = 0;
@@ -461,106 +464,152 @@ static void vsp1_dl_list_fill_header(struct vsp1_dl_list *dl, bool is_last)
 
 	dl->header->num_lists = num_lists;
 
-	/*
-	 * If this display list's chain is not empty, we are on a list, where
-	 * the next item in the list is the display list entity which should be
-	 * automatically queued by the hardware.
-	 */
 	if (!list_empty(&dl->chain) && !is_last) {
+		/*
+		 * If this display list's chain is not empty, we are on a list,
+		 * and the next item is the display list that we must queue for
+		 * automatic processing by the hardware.
+		 */
 		struct vsp1_dl_list *next = list_next_entry(dl, chain);
 
 		dl->header->next_header = next->dma;
 		dl->header->flags = VSP1_DLH_AUTO_START;
+	} else if (!dlm->singleshot) {
+		/*
+		 * if the display list manager works in continuous mode, the VSP
+		 * should loop over the display list continuously until
+		 * instructed to do otherwise.
+		 */
+		dl->header->next_header = dl->dma;
+		dl->header->flags = VSP1_DLH_INT_ENABLE | VSP1_DLH_AUTO_START;
 	} else {
+		/*
+		 * Otherwise, in mem-to-mem mode, we work in single-shot mode
+		 * and the next display list must not be started automatically.
+		 */
 		dl->header->flags = VSP1_DLH_INT_ENABLE;
 	}
 }
 
-void vsp1_dl_list_commit(struct vsp1_dl_list *dl)
+static bool vsp1_dl_list_hw_update_pending(struct vsp1_dl_manager *dlm)
 {
-	struct vsp1_dl_manager *dlm = dl->dlm;
 	struct vsp1_device *vsp1 = dlm->vsp1;
-	unsigned long flags;
-	bool update;
 
-	spin_lock_irqsave(&dlm->lock, flags);
+	if (!dlm->queued)
+		return false;
 
-	if (dl->dlm->mode == VSP1_DL_MODE_HEADER) {
-		struct vsp1_dl_list *dl_child;
+	/*
+	 * Check whether the VSP1 has taken the update. In headerless mode the
+	 * hardware indicates this by clearing the UPD bit in the DL_BODY_SIZE
+	 * register, and in header mode by clearing the UPDHDR bit in the CMD
+	 * register.
+	 */
+	if (dlm->mode == VSP1_DL_MODE_HEADERLESS)
+		return !!(vsp1_read(vsp1, VI6_DL_BODY_SIZE)
+			  & VI6_DL_BODY_SIZE_UPD);
+	else
+		return !!(vsp1_read(vsp1, VI6_CMD(dlm->index) & VI6_CMD_UPDHDR));
+}
 
+static void vsp1_dl_list_hw_enqueue(struct vsp1_dl_list *dl)
+{
+	struct vsp1_dl_manager *dlm = dl->dlm;
+	struct vsp1_device *vsp1 = dlm->vsp1;
+
+	if (dlm->mode == VSP1_DL_MODE_HEADERLESS) {
 		/*
-		 * In header mode the caller guarantees that the hardware is
-		 * idle at this point.
+		 * In headerless mode, program the hardware directly with the
+		 * display list body address and size and set the UPD bit. The
+		 * bit will be cleared by the hardware when the display list
+		 * processing starts.
 		 */
-
-		/* Fill the header for the head and chained display lists. */
-		vsp1_dl_list_fill_header(dl, list_empty(&dl->chain));
-
-		list_for_each_entry(dl_child, &dl->chain, chain) {
-			bool last = list_is_last(&dl_child->chain, &dl->chain);
-
-			vsp1_dl_list_fill_header(dl_child, last);
-		}
-
+		vsp1_write(vsp1, VI6_DL_HDR_ADDR(0), dl->body0.dma);
+		vsp1_write(vsp1, VI6_DL_BODY_SIZE, VI6_DL_BODY_SIZE_UPD |
+			   (dl->body0.num_entries * sizeof(*dl->header->lists)));
+	} else {
 		/*
-		 * Commit the head display list to hardware. Chained headers
-		 * will auto-start.
+		 * In header mode, program the display list header address. If
+		 * the hardware is idle (single-shot mode or first frame in
+		 * continuous mode) it will then be started independently. If
+		 * the hardware is operating, the VI6_DL_HDR_REF_ADDR register
+		 * will be updated with the display list address.
 		 */
 		vsp1_write(vsp1, VI6_DL_HDR_ADDR(dlm->index), dl->dma);
-
-		dlm->active = dl;
-		goto done;
 	}
+}
+
+static void vsp1_dl_list_commit_continuous(struct vsp1_dl_list *dl)
+{
+	struct vsp1_dl_manager *dlm = dl->dlm;
 
 	/*
-	 * Once the UPD bit has been set the hardware can start processing the
-	 * display list at any time and we can't touch the address and size
-	 * registers. In that case mark the update as pending, it will be
-	 * queued up to the hardware by the frame end interrupt handler.
+	 * If a previous display list has been queued to the hardware but not
+	 * processed yet, the VSP can start processing it at any time. In that
+	 * case we can't replace the queued list by the new one, as we could
+	 * race with the hardware. We thus mark the update as pending, it will
+	 * be queued up to the hardware by the frame end interrupt handler.
 	 */
-	update = !!(vsp1_read(vsp1, VI6_DL_BODY_SIZE) & VI6_DL_BODY_SIZE_UPD);
-	if (update) {
+	if (vsp1_dl_list_hw_update_pending(dlm)) {
 		__vsp1_dl_list_put(dlm->pending);
 		dlm->pending = dl;
-		goto done;
+		return;
 	}
 
 	/*
-	 * Program the hardware with the display list body address and size.
-	 * The UPD bit will be cleared by the device when the display list is
-	 * processed.
+	 * Pass the new display list to the hardware and mark it as queued. It
+	 * will become active when the hardware starts processing it.
 	 */
-	vsp1_write(vsp1, VI6_DL_HDR_ADDR(0), dl->body0.dma);
-	vsp1_write(vsp1, VI6_DL_BODY_SIZE, VI6_DL_BODY_SIZE_UPD |
-		   (dl->body0.num_entries * sizeof(*dl->header->lists)));
+	vsp1_dl_list_hw_enqueue(dl);
 
 	__vsp1_dl_list_put(dlm->queued);
 	dlm->queued = dl;
-
-done:
-	spin_unlock_irqrestore(&dlm->lock, flags);
 }
 
-/* -----------------------------------------------------------------------------
- * Display List Manager
- */
-
-/* Interrupt Handling */
-void vsp1_dlm_irq_display_start(struct vsp1_dl_manager *dlm)
+static void vsp1_dl_list_commit_singleshot(struct vsp1_dl_list *dl)
 {
-	spin_lock(&dlm->lock);
+	struct vsp1_dl_manager *dlm = dl->dlm;
 
 	/*
-	 * The display start interrupt signals the end of the display list
-	 * processing by the device. The active display list, if any, won't be
-	 * accessed anymore and can be reused.
+	 * When working in single-shot mode, the caller guarantees that the
+	 * hardware is idle at this point. Just commit the head display list
+	 * to hardware. Chained lists will be started automatically.
 	 */
-	__vsp1_dl_list_put(dlm->active);
-	dlm->active = NULL;
+	vsp1_dl_list_hw_enqueue(dl);
 
-	spin_unlock(&dlm->lock);
+	dlm->active = dl;
+}
+
+void vsp1_dl_list_commit(struct vsp1_dl_list *dl)
+{
+	struct vsp1_dl_manager *dlm = dl->dlm;
+	struct vsp1_dl_list *dl_child;
+	unsigned long flags;
+
+	if (dlm->mode == VSP1_DL_MODE_HEADER) {
+		/* Fill the header for the head and chained display lists. */
+		vsp1_dl_list_fill_header(dl, list_empty(&dl->chain));
+
+		list_for_each_entry(dl_child, &dl->chain, chain) {
+			bool last = list_is_last(&dl_child->chain, &dl->chain);
+
+			vsp1_dl_list_fill_header(dl_child, last);
+		}
+	}
+
+	spin_lock_irqsave(&dlm->lock, flags);
+
+	if (dlm->singleshot)
+		vsp1_dl_list_commit_singleshot(dl);
+	else
+		vsp1_dl_list_commit_continuous(dl);
+
+	spin_unlock_irqrestore(&dlm->lock, flags);
 }
 
+/* -----------------------------------------------------------------------------
+ * Display List Manager
+ */
+
 /**
  * vsp1_dlm_irq_frame_end - Display list handler for the frame end interrupt
  * @dlm: the display list manager
@@ -572,31 +621,28 @@ void vsp1_dlm_irq_display_start(struct vsp1_dl_manager *dlm)
  */
 bool vsp1_dlm_irq_frame_end(struct vsp1_dl_manager *dlm)
 {
-	struct vsp1_device *vsp1 = dlm->vsp1;
 	bool completed = false;
 
 	spin_lock(&dlm->lock);
 
-	__vsp1_dl_list_put(dlm->active);
-	dlm->active = NULL;
-
 	/*
-	 * Header mode is used for mem-to-mem pipelines only. We don't need to
-	 * perform any operation as there can't be any new display list queued
-	 * in that case.
+	 * The mem-to-mem pipelines work in single-shot mode. No new display
+	 * list can be queued, we don't have to do anything.
 	 */
-	if (dlm->mode == VSP1_DL_MODE_HEADER) {
+	if (dlm->singleshot) {
+		__vsp1_dl_list_put(dlm->active);
+		dlm->active = NULL;
 		completed = true;
 		goto done;
 	}
 
 	/*
-	 * The UPD bit set indicates that the commit operation raced with the
-	 * interrupt and occurred after the frame end event and UPD clear but
-	 * before interrupt processing. The hardware hasn't taken the update
-	 * into account yet, we'll thus skip one frame and retry.
+	 * If the commit operation raced with the interrupt and occurred after
+	 * the frame end event but before interrupt processing, the hardware
+	 * hasn't taken the update into account yet. We have to skip one frame
+	 * and retry.
 	 */
-	if (vsp1_read(vsp1, VI6_DL_BODY_SIZE) & VI6_DL_BODY_SIZE_UPD)
+	if (vsp1_dl_list_hw_update_pending(dlm))
 		goto done;
 
 	/*
@@ -604,24 +650,20 @@ bool vsp1_dlm_irq_frame_end(struct vsp1_dl_manager *dlm)
 	 * frame end interrupt. The display list thus becomes active.
 	 */
 	if (dlm->queued) {
+		__vsp1_dl_list_put(dlm->active);
 		dlm->active = dlm->queued;
 		dlm->queued = NULL;
 		completed = true;
 	}
 
 	/*
-	 * Now that the UPD bit has been cleared we can queue the next display
-	 * list to the hardware if one has been prepared.
+	 * Now that the VSP has started processing the queued display list, we
+	 * can queue the pending display list to the hardware if one has been
+	 * prepared.
 	 */
 	if (dlm->pending) {
-		struct vsp1_dl_list *dl = dlm->pending;
-
-		vsp1_write(vsp1, VI6_DL_HDR_ADDR(0), dl->body0.dma);
-		vsp1_write(vsp1, VI6_DL_BODY_SIZE, VI6_DL_BODY_SIZE_UPD |
-			   (dl->body0.num_entries *
-			    sizeof(*dl->header->lists)));
-
-		dlm->queued = dl;
+		vsp1_dl_list_hw_enqueue(dlm->pending);
+		dlm->queued = dlm->pending;
 		dlm->pending = NULL;
 	}
 
@@ -714,6 +756,7 @@ struct vsp1_dl_manager *vsp1_dlm_create(struct vsp1_device *vsp1,
 	dlm->index = index;
 	dlm->mode = index == 0 && !vsp1->info->uapi
 		  ? VSP1_DL_MODE_HEADERLESS : VSP1_DL_MODE_HEADER;
+	dlm->singleshot = vsp1->info->uapi;
 	dlm->vsp1 = vsp1;
 
 	spin_lock_init(&dlm->lock);
diff --git a/drivers/media/platform/vsp1/vsp1_dl.h b/drivers/media/platform/vsp1/vsp1_dl.h
index 6ec1380a10af..ee3508172f0a 100644
--- a/drivers/media/platform/vsp1/vsp1_dl.h
+++ b/drivers/media/platform/vsp1/vsp1_dl.h
@@ -27,7 +27,6 @@ struct vsp1_dl_manager *vsp1_dlm_create(struct vsp1_device *vsp1,
 					unsigned int prealloc);
 void vsp1_dlm_destroy(struct vsp1_dl_manager *dlm);
 void vsp1_dlm_reset(struct vsp1_dl_manager *dlm);
-void vsp1_dlm_irq_display_start(struct vsp1_dl_manager *dlm);
 bool vsp1_dlm_irq_frame_end(struct vsp1_dl_manager *dlm);
 
 struct vsp1_dl_list *vsp1_dl_list_get(struct vsp1_dl_manager *dlm);
diff --git a/drivers/media/platform/vsp1/vsp1_drm.c b/drivers/media/platform/vsp1/vsp1_drm.c
index 9377aafa8996..4dfbeac8f42c 100644
--- a/drivers/media/platform/vsp1/vsp1_drm.c
+++ b/drivers/media/platform/vsp1/vsp1_drm.c
@@ -32,17 +32,13 @@
  * Interrupt Handling
  */
 
-void vsp1_drm_display_start(struct vsp1_device *vsp1)
+static void vsp1_du_pipeline_frame_end(struct vsp1_pipeline *pipe,
+				       bool completed)
 {
-	vsp1_dlm_irq_display_start(vsp1->drm->pipe.output->dlm);
-}
-
-static void vsp1_du_pipeline_frame_end(struct vsp1_pipeline *pipe)
-{
-	struct vsp1_drm *drm = to_vsp1_drm(pipe);
+	struct vsp1_drm_pipeline *drm_pipe = to_vsp1_drm_pipeline(pipe);
 
-	if (drm->du_complete)
-		drm->du_complete(drm->du_private);
+	if (drm_pipe->du_complete)
+		drm_pipe->du_complete(drm_pipe->du_private, completed);
 }
 
 /* -----------------------------------------------------------------------------
@@ -63,29 +59,44 @@ EXPORT_SYMBOL_GPL(vsp1_du_init);
 /**
  * vsp1_du_setup_lif - Setup the output part of the VSP pipeline
  * @dev: the VSP device
+ * @pipe_index: the DRM pipeline index
  * @cfg: the LIF configuration
  *
  * Configure the output part of VSP DRM pipeline for the given frame @cfg.width
- * and @cfg.height. This sets up formats on the BRU source pad, the WPF0 sink
- * and source pads, and the LIF sink pad.
+ * and @cfg.height. This sets up formats on the blend unit (BRU or BRS) source
+ * pad, the WPF sink and source pads, and the LIF sink pad.
+ *
+ * The @pipe_index argument selects which DRM pipeline to setup. The number of
+ * available pipelines depend on the VSP instance.
  *
- * As the media bus code on the BRU source pad is conditioned by the
- * configuration of the BRU sink 0 pad, we also set up the formats on all BRU
+ * As the media bus code on the blend unit source pad is conditioned by the
+ * configuration of its sink 0 pad, we also set up the formats on all blend unit
  * sinks, even if the configuration will be overwritten later by
- * vsp1_du_setup_rpf(). This ensures that the BRU configuration is set to a well
- * defined state.
+ * vsp1_du_setup_rpf(). This ensures that the blend unit configuration is set to
+ * a well defined state.
  *
  * Return 0 on success or a negative error code on failure.
  */
-int vsp1_du_setup_lif(struct device *dev, const struct vsp1_du_lif_config *cfg)
+int vsp1_du_setup_lif(struct device *dev, unsigned int pipe_index,
+		      const struct vsp1_du_lif_config *cfg)
 {
 	struct vsp1_device *vsp1 = dev_get_drvdata(dev);
-	struct vsp1_pipeline *pipe = &vsp1->drm->pipe;
-	struct vsp1_bru *bru = vsp1->bru;
+	struct vsp1_drm_pipeline *drm_pipe;
+	struct vsp1_pipeline *pipe;
+	struct vsp1_bru *bru;
 	struct v4l2_subdev_format format;
+	const char *bru_name;
 	unsigned int i;
 	int ret;
 
+	if (pipe_index >= vsp1->info->lif_count)
+		return -EINVAL;
+
+	drm_pipe = &vsp1->drm->pipe[pipe_index];
+	pipe = &drm_pipe->pipe;
+	bru = to_bru(&pipe->bru->subdev);
+	bru_name = pipe->bru->type == VSP1_ENTITY_BRU ? "BRU" : "BRS";
+
 	if (!cfg) {
 		/*
 		 * NULL configuration means the CRTC is being disabled, stop
@@ -97,14 +108,25 @@ int vsp1_du_setup_lif(struct device *dev, const struct vsp1_du_lif_config *cfg)
 
 		media_pipeline_stop(&pipe->output->entity.subdev.entity);
 
-		for (i = 0; i < bru->entity.source_pad; ++i) {
-			vsp1->drm->inputs[i].enabled = false;
-			bru->inputs[i].rpf = NULL;
+		for (i = 0; i < ARRAY_SIZE(pipe->inputs); ++i) {
+			struct vsp1_rwpf *rpf = pipe->inputs[i];
+
+			if (!rpf)
+				continue;
+
+			/*
+			 * Remove the RPF from the pipe and the list of BRU
+			 * inputs.
+			 */
+			WARN_ON(list_empty(&rpf->entity.list_pipe));
+			list_del_init(&rpf->entity.list_pipe);
 			pipe->inputs[i] = NULL;
+
+			bru->inputs[rpf->bru_input].rpf = NULL;
 		}
 
+		drm_pipe->du_complete = NULL;
 		pipe->num_inputs = 0;
-		vsp1->drm->du_complete = NULL;
 
 		vsp1_dlm_reset(pipe->output->dlm);
 		vsp1_device_put(vsp1);
@@ -114,8 +136,8 @@ int vsp1_du_setup_lif(struct device *dev, const struct vsp1_du_lif_config *cfg)
 		return 0;
 	}
 
-	dev_dbg(vsp1->dev, "%s: configuring LIF with format %ux%u\n",
-		__func__, cfg->width, cfg->height);
+	dev_dbg(vsp1->dev, "%s: configuring LIF%u with format %ux%u\n",
+		__func__, pipe_index, cfg->width, cfg->height);
 
 	/*
 	 * Configure the format at the BRU sinks and propagate it through the
@@ -124,7 +146,7 @@ int vsp1_du_setup_lif(struct device *dev, const struct vsp1_du_lif_config *cfg)
 	memset(&format, 0, sizeof(format));
 	format.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 
-	for (i = 0; i < bru->entity.source_pad; ++i) {
+	for (i = 0; i < pipe->bru->source_pad; ++i) {
 		format.pad = i;
 
 		format.format.width = cfg->width;
@@ -132,60 +154,60 @@ int vsp1_du_setup_lif(struct device *dev, const struct vsp1_du_lif_config *cfg)
 		format.format.code = MEDIA_BUS_FMT_ARGB8888_1X32;
 		format.format.field = V4L2_FIELD_NONE;
 
-		ret = v4l2_subdev_call(&bru->entity.subdev, pad,
+		ret = v4l2_subdev_call(&pipe->bru->subdev, pad,
 				       set_fmt, NULL, &format);
 		if (ret < 0)
 			return ret;
 
-		dev_dbg(vsp1->dev, "%s: set format %ux%u (%x) on BRU pad %u\n",
+		dev_dbg(vsp1->dev, "%s: set format %ux%u (%x) on %s pad %u\n",
 			__func__, format.format.width, format.format.height,
-			format.format.code, i);
+			format.format.code, bru_name, i);
 	}
 
-	format.pad = bru->entity.source_pad;
+	format.pad = pipe->bru->source_pad;
 	format.format.width = cfg->width;
 	format.format.height = cfg->height;
 	format.format.code = MEDIA_BUS_FMT_ARGB8888_1X32;
 	format.format.field = V4L2_FIELD_NONE;
 
-	ret = v4l2_subdev_call(&bru->entity.subdev, pad, set_fmt, NULL,
+	ret = v4l2_subdev_call(&pipe->bru->subdev, pad, set_fmt, NULL,
 			       &format);
 	if (ret < 0)
 		return ret;
 
-	dev_dbg(vsp1->dev, "%s: set format %ux%u (%x) on BRU pad %u\n",
+	dev_dbg(vsp1->dev, "%s: set format %ux%u (%x) on %s pad %u\n",
 		__func__, format.format.width, format.format.height,
-		format.format.code, i);
+		format.format.code, bru_name, i);
 
 	format.pad = RWPF_PAD_SINK;
-	ret = v4l2_subdev_call(&vsp1->wpf[0]->entity.subdev, pad, set_fmt, NULL,
+	ret = v4l2_subdev_call(&pipe->output->entity.subdev, pad, set_fmt, NULL,
 			       &format);
 	if (ret < 0)
 		return ret;
 
-	dev_dbg(vsp1->dev, "%s: set format %ux%u (%x) on WPF0 sink\n",
+	dev_dbg(vsp1->dev, "%s: set format %ux%u (%x) on WPF%u sink\n",
 		__func__, format.format.width, format.format.height,
-		format.format.code);
+		format.format.code, pipe->output->entity.index);
 
 	format.pad = RWPF_PAD_SOURCE;
-	ret = v4l2_subdev_call(&vsp1->wpf[0]->entity.subdev, pad, get_fmt, NULL,
+	ret = v4l2_subdev_call(&pipe->output->entity.subdev, pad, get_fmt, NULL,
 			       &format);
 	if (ret < 0)
 		return ret;
 
-	dev_dbg(vsp1->dev, "%s: got format %ux%u (%x) on WPF0 source\n",
+	dev_dbg(vsp1->dev, "%s: got format %ux%u (%x) on WPF%u source\n",
 		__func__, format.format.width, format.format.height,
-		format.format.code);
+		format.format.code, pipe->output->entity.index);
 
 	format.pad = LIF_PAD_SINK;
-	ret = v4l2_subdev_call(&vsp1->lif->entity.subdev, pad, set_fmt, NULL,
+	ret = v4l2_subdev_call(&pipe->lif->subdev, pad, set_fmt, NULL,
 			       &format);
 	if (ret < 0)
 		return ret;
 
-	dev_dbg(vsp1->dev, "%s: set format %ux%u (%x) on LIF sink\n",
+	dev_dbg(vsp1->dev, "%s: set format %ux%u (%x) on LIF%u sink\n",
 		__func__, format.format.width, format.format.height,
-		format.format.code);
+		format.format.code, pipe_index);
 
 	/*
 	 * Verify that the format at the output of the pipeline matches the
@@ -213,8 +235,8 @@ int vsp1_du_setup_lif(struct device *dev, const struct vsp1_du_lif_config *cfg)
 	 * Register a callback to allow us to notify the DRM driver of frame
 	 * completion events.
 	 */
-	vsp1->drm->du_complete = cfg->callback;
-	vsp1->drm->du_private = cfg->callback_data;
+	drm_pipe->du_complete = cfg->callback;
+	drm_pipe->du_private = cfg->callback_data;
 
 	ret = media_pipeline_start(&pipe->output->entity.subdev.entity,
 					  &pipe->pipe);
@@ -224,6 +246,10 @@ int vsp1_du_setup_lif(struct device *dev, const struct vsp1_du_lif_config *cfg)
 		return ret;
 	}
 
+	/* Disable the display interrupts. */
+	vsp1_write(vsp1, VI6_DISP_IRQ_STA, 0);
+	vsp1_write(vsp1, VI6_DISP_IRQ_ENB, 0);
+
 	dev_dbg(vsp1->dev, "%s: pipeline enabled\n", __func__);
 
 	return 0;
@@ -233,19 +259,21 @@ EXPORT_SYMBOL_GPL(vsp1_du_setup_lif);
 /**
  * vsp1_du_atomic_begin - Prepare for an atomic update
  * @dev: the VSP device
+ * @pipe_index: the DRM pipeline index
  */
-void vsp1_du_atomic_begin(struct device *dev)
+void vsp1_du_atomic_begin(struct device *dev, unsigned int pipe_index)
 {
 	struct vsp1_device *vsp1 = dev_get_drvdata(dev);
-	struct vsp1_pipeline *pipe = &vsp1->drm->pipe;
+	struct vsp1_drm_pipeline *drm_pipe = &vsp1->drm->pipe[pipe_index];
 
-	vsp1->drm->num_inputs = pipe->num_inputs;
+	drm_pipe->enabled = drm_pipe->pipe.num_inputs != 0;
 }
 EXPORT_SYMBOL_GPL(vsp1_du_atomic_begin);
 
 /**
  * vsp1_du_atomic_update - Setup one RPF input of the VSP pipeline
  * @dev: the VSP device
+ * @pipe_index: the DRM pipeline index
  * @rpf_index: index of the RPF to setup (0-based)
  * @cfg: the RPF configuration
  *
@@ -272,10 +300,12 @@ EXPORT_SYMBOL_GPL(vsp1_du_atomic_begin);
  *
  * Return 0 on success or a negative error code on failure.
  */
-int vsp1_du_atomic_update(struct device *dev, unsigned int rpf_index,
+int vsp1_du_atomic_update(struct device *dev, unsigned int pipe_index,
+			  unsigned int rpf_index,
 			  const struct vsp1_du_atomic_config *cfg)
 {
 	struct vsp1_device *vsp1 = dev_get_drvdata(dev);
+	struct vsp1_drm_pipeline *drm_pipe = &vsp1->drm->pipe[pipe_index];
 	const struct vsp1_format_info *fmtinfo;
 	struct vsp1_rwpf *rpf;
 
@@ -288,7 +318,12 @@ int vsp1_du_atomic_update(struct device *dev, unsigned int rpf_index,
 		dev_dbg(vsp1->dev, "%s: RPF%u: disable requested\n", __func__,
 			rpf_index);
 
-		vsp1->drm->inputs[rpf_index].enabled = false;
+		/*
+		 * Remove the RPF from the pipe's inputs. The atomic flush
+		 * handler will disable the input and remove the entity from the
+		 * pipe's entities list.
+		 */
+		drm_pipe->pipe.inputs[rpf_index] = NULL;
 		return 0;
 	}
 
@@ -324,13 +359,15 @@ int vsp1_du_atomic_update(struct device *dev, unsigned int rpf_index,
 	vsp1->drm->inputs[rpf_index].crop = cfg->src;
 	vsp1->drm->inputs[rpf_index].compose = cfg->dst;
 	vsp1->drm->inputs[rpf_index].zpos = cfg->zpos;
-	vsp1->drm->inputs[rpf_index].enabled = true;
+
+	drm_pipe->pipe.inputs[rpf_index] = rpf;
 
 	return 0;
 }
 EXPORT_SYMBOL_GPL(vsp1_du_atomic_update);
 
 static int vsp1_du_setup_rpf_pipe(struct vsp1_device *vsp1,
+				  struct vsp1_pipeline *pipe,
 				  struct vsp1_rwpf *rpf, unsigned int bru_input)
 {
 	struct v4l2_subdev_selection sel;
@@ -404,7 +441,7 @@ static int vsp1_du_setup_rpf_pipe(struct vsp1_device *vsp1,
 	/* BRU sink, propagate the format from the RPF source. */
 	format.pad = bru_input;
 
-	ret = v4l2_subdev_call(&vsp1->bru->entity.subdev, pad, set_fmt, NULL,
+	ret = v4l2_subdev_call(&pipe->bru->subdev, pad, set_fmt, NULL,
 			       &format);
 	if (ret < 0)
 		return ret;
@@ -417,8 +454,8 @@ static int vsp1_du_setup_rpf_pipe(struct vsp1_device *vsp1,
 	sel.target = V4L2_SEL_TGT_COMPOSE;
 	sel.r = vsp1->drm->inputs[rpf->entity.index].compose;
 
-	ret = v4l2_subdev_call(&vsp1->bru->entity.subdev, pad, set_selection,
-			       NULL, &sel);
+	ret = v4l2_subdev_call(&pipe->bru->subdev, pad, set_selection, NULL,
+			       &sel);
 	if (ret < 0)
 		return ret;
 
@@ -438,18 +475,25 @@ static unsigned int rpf_zpos(struct vsp1_device *vsp1, struct vsp1_rwpf *rpf)
 /**
  * vsp1_du_atomic_flush - Commit an atomic update
  * @dev: the VSP device
+ * @pipe_index: the DRM pipeline index
  */
-void vsp1_du_atomic_flush(struct device *dev)
+void vsp1_du_atomic_flush(struct device *dev, unsigned int pipe_index)
 {
 	struct vsp1_device *vsp1 = dev_get_drvdata(dev);
-	struct vsp1_pipeline *pipe = &vsp1->drm->pipe;
+	struct vsp1_drm_pipeline *drm_pipe = &vsp1->drm->pipe[pipe_index];
+	struct vsp1_pipeline *pipe = &drm_pipe->pipe;
 	struct vsp1_rwpf *inputs[VSP1_MAX_RPF] = { NULL, };
+	struct vsp1_bru *bru = to_bru(&pipe->bru->subdev);
 	struct vsp1_entity *entity;
+	struct vsp1_entity *next;
 	struct vsp1_dl_list *dl;
+	const char *bru_name;
 	unsigned long flags;
 	unsigned int i;
 	int ret;
 
+	bru_name = pipe->bru->type == VSP1_ENTITY_BRU ? "BRU" : "BRS";
+
 	/* Prepare the display list. */
 	dl = vsp1_dl_list_get(pipe->output->dlm);
 
@@ -460,12 +504,8 @@ void vsp1_du_atomic_flush(struct device *dev)
 		struct vsp1_rwpf *rpf = vsp1->rpf[i];
 		unsigned int j;
 
-		if (!vsp1->drm->inputs[i].enabled) {
-			pipe->inputs[i] = NULL;
+		if (!pipe->inputs[i])
 			continue;
-		}
-
-		pipe->inputs[i] = rpf;
 
 		/* Insert the RPF in the sorted RPFs array. */
 		for (j = pipe->num_inputs++; j > 0; --j) {
@@ -478,22 +518,26 @@ void vsp1_du_atomic_flush(struct device *dev)
 	}
 
 	/* Setup the RPF input pipeline for every enabled input. */
-	for (i = 0; i < vsp1->info->num_bru_inputs; ++i) {
+	for (i = 0; i < pipe->bru->source_pad; ++i) {
 		struct vsp1_rwpf *rpf = inputs[i];
 
 		if (!rpf) {
-			vsp1->bru->inputs[i].rpf = NULL;
+			bru->inputs[i].rpf = NULL;
 			continue;
 		}
 
-		vsp1->bru->inputs[i].rpf = rpf;
+		if (list_empty(&rpf->entity.list_pipe))
+			list_add_tail(&rpf->entity.list_pipe, &pipe->entities);
+
+		bru->inputs[i].rpf = rpf;
 		rpf->bru_input = i;
+		rpf->entity.sink = pipe->bru;
 		rpf->entity.sink_pad = i;
 
-		dev_dbg(vsp1->dev, "%s: connecting RPF.%u to BRU:%u\n",
-			__func__, rpf->entity.index, i);
+		dev_dbg(vsp1->dev, "%s: connecting RPF.%u to %s:%u\n",
+			__func__, rpf->entity.index, bru_name, i);
 
-		ret = vsp1_du_setup_rpf_pipe(vsp1, rpf, i);
+		ret = vsp1_du_setup_rpf_pipe(vsp1, pipe, rpf, i);
 		if (ret < 0)
 			dev_err(vsp1->dev,
 				"%s: failed to setup RPF.%u\n",
@@ -501,16 +545,16 @@ void vsp1_du_atomic_flush(struct device *dev)
 	}
 
 	/* Configure all entities in the pipeline. */
-	list_for_each_entry(entity, &pipe->entities, list_pipe) {
+	list_for_each_entry_safe(entity, next, &pipe->entities, list_pipe) {
 		/* Disconnect unused RPFs from the pipeline. */
-		if (entity->type == VSP1_ENTITY_RPF) {
-			struct vsp1_rwpf *rpf = to_rwpf(&entity->subdev);
+		if (entity->type == VSP1_ENTITY_RPF &&
+		    !pipe->inputs[entity->index]) {
+			vsp1_dl_list_write(dl, entity->route->reg,
+					   VI6_DPR_NODE_UNUSED);
 
-			if (!pipe->inputs[rpf->entity.index]) {
-				vsp1_dl_list_write(dl, entity->route->reg,
-						   VI6_DPR_NODE_UNUSED);
-				continue;
-			}
+			list_del_init(&entity->list_pipe);
+
+			continue;
 		}
 
 		vsp1_entity_route_setup(entity, pipe, dl);
@@ -528,14 +572,11 @@ void vsp1_du_atomic_flush(struct device *dev)
 	vsp1_dl_list_commit(dl);
 
 	/* Start or stop the pipeline if needed. */
-	if (!vsp1->drm->num_inputs && pipe->num_inputs) {
-		vsp1_write(vsp1, VI6_DISP_IRQ_STA, 0);
-		vsp1_write(vsp1, VI6_DISP_IRQ_ENB, VI6_DISP_IRQ_ENB_DSTE);
+	if (!drm_pipe->enabled && pipe->num_inputs) {
 		spin_lock_irqsave(&pipe->irqlock, flags);
 		vsp1_pipeline_run(pipe);
 		spin_unlock_irqrestore(&pipe->irqlock, flags);
-	} else if (vsp1->drm->num_inputs && !pipe->num_inputs) {
-		vsp1_write(vsp1, VI6_DISP_IRQ_ENB, 0);
+	} else if (drm_pipe->enabled && !pipe->num_inputs) {
 		vsp1_pipeline_stop(pipe);
 	}
 }
@@ -568,83 +609,48 @@ EXPORT_SYMBOL_GPL(vsp1_du_unmap_sg);
  * Initialization
  */
 
-int vsp1_drm_create_links(struct vsp1_device *vsp1)
-{
-	const u32 flags = MEDIA_LNK_FL_ENABLED | MEDIA_LNK_FL_IMMUTABLE;
-	unsigned int i;
-	int ret;
-
-	/*
-	 * VSPD instances require a BRU to perform composition and a LIF to
-	 * output to the DU.
-	 */
-	if (!vsp1->bru || !vsp1->lif)
-		return -ENXIO;
-
-	for (i = 0; i < vsp1->info->rpf_count; ++i) {
-		struct vsp1_rwpf *rpf = vsp1->rpf[i];
-
-		ret = media_create_pad_link(&rpf->entity.subdev.entity,
-					    RWPF_PAD_SOURCE,
-					    &vsp1->bru->entity.subdev.entity,
-					    i, flags);
-		if (ret < 0)
-			return ret;
-
-		rpf->entity.sink = &vsp1->bru->entity.subdev.entity;
-		rpf->entity.sink_pad = i;
-	}
-
-	ret = media_create_pad_link(&vsp1->bru->entity.subdev.entity,
-				    vsp1->bru->entity.source_pad,
-				    &vsp1->wpf[0]->entity.subdev.entity,
-				    RWPF_PAD_SINK, flags);
-	if (ret < 0)
-		return ret;
-
-	vsp1->bru->entity.sink = &vsp1->wpf[0]->entity.subdev.entity;
-	vsp1->bru->entity.sink_pad = RWPF_PAD_SINK;
-
-	ret = media_create_pad_link(&vsp1->wpf[0]->entity.subdev.entity,
-				    RWPF_PAD_SOURCE,
-				    &vsp1->lif->entity.subdev.entity,
-				    LIF_PAD_SINK, flags);
-	if (ret < 0)
-		return ret;
-
-	return 0;
-}
-
 int vsp1_drm_init(struct vsp1_device *vsp1)
 {
-	struct vsp1_pipeline *pipe;
 	unsigned int i;
 
 	vsp1->drm = devm_kzalloc(vsp1->dev, sizeof(*vsp1->drm), GFP_KERNEL);
 	if (!vsp1->drm)
 		return -ENOMEM;
 
-	pipe = &vsp1->drm->pipe;
+	/* Create one DRM pipeline per LIF. */
+	for (i = 0; i < vsp1->info->lif_count; ++i) {
+		struct vsp1_drm_pipeline *drm_pipe = &vsp1->drm->pipe[i];
+		struct vsp1_pipeline *pipe = &drm_pipe->pipe;
 
-	vsp1_pipeline_init(pipe);
+		vsp1_pipeline_init(pipe);
 
-	/* The DRM pipeline is static, add entities manually. */
+		/*
+		 * The DRM pipeline is static, add entities manually. The first
+		 * pipeline uses the BRU and the second pipeline the BRS.
+		 */
+		pipe->bru = i == 0 ? &vsp1->bru->entity : &vsp1->brs->entity;
+		pipe->lif = &vsp1->lif[i]->entity;
+		pipe->output = vsp1->wpf[i];
+		pipe->output->pipe = pipe;
+		pipe->frame_end = vsp1_du_pipeline_frame_end;
+
+		pipe->bru->sink = &pipe->output->entity;
+		pipe->bru->sink_pad = 0;
+		pipe->output->entity.sink = pipe->lif;
+		pipe->output->entity.sink_pad = 0;
+
+		list_add_tail(&pipe->bru->list_pipe, &pipe->entities);
+		list_add_tail(&pipe->lif->list_pipe, &pipe->entities);
+		list_add_tail(&pipe->output->entity.list_pipe, &pipe->entities);
+	}
+
+	/* Disable all RPFs initially. */
 	for (i = 0; i < vsp1->info->rpf_count; ++i) {
 		struct vsp1_rwpf *input = vsp1->rpf[i];
 
-		list_add_tail(&input->entity.list_pipe, &pipe->entities);
+		INIT_LIST_HEAD(&input->entity.list_pipe);
 	}
 
-	list_add_tail(&vsp1->bru->entity.list_pipe, &pipe->entities);
-	list_add_tail(&vsp1->wpf[0]->entity.list_pipe, &pipe->entities);
-	list_add_tail(&vsp1->lif->entity.list_pipe, &pipe->entities);
-
-	pipe->bru = &vsp1->bru->entity;
-	pipe->lif = &vsp1->lif->entity;
-	pipe->output = vsp1->wpf[0];
-	pipe->output->pipe = pipe;
-	pipe->frame_end = vsp1_du_pipeline_frame_end;
-
 	return 0;
 }
 
diff --git a/drivers/media/platform/vsp1/vsp1_drm.h b/drivers/media/platform/vsp1/vsp1_drm.h
index e9f80727ff92..1cd9db785bf7 100644
--- a/drivers/media/platform/vsp1/vsp1_drm.h
+++ b/drivers/media/platform/vsp1/vsp1_drm.h
@@ -18,38 +18,44 @@
 #include "vsp1_pipe.h"
 
 /**
- * vsp1_drm - State for the API exposed to the DRM driver
+ * vsp1_drm_pipeline - State for the API exposed to the DRM driver
  * @pipe: the VSP1 pipeline used for display
- * @num_inputs: number of active pipeline inputs at the beginning of an update
- * @inputs: source crop rectangle, destination compose rectangle and z-order
- *	position for every input
+ * @enabled: pipeline state at the beginning of an update
  * @du_complete: frame completion callback for the DU driver (optional)
  * @du_private: data to be passed to the du_complete callback
  */
-struct vsp1_drm {
+struct vsp1_drm_pipeline {
 	struct vsp1_pipeline pipe;
-	unsigned int num_inputs;
+	bool enabled;
+
+	/* Frame synchronisation */
+	void (*du_complete)(void *, bool);
+	void *du_private;
+};
+
+/**
+ * vsp1_drm - State for the API exposed to the DRM driver
+ * @pipe: the VSP1 DRM pipeline used for display
+ * @inputs: source crop rectangle, destination compose rectangle and z-order
+ *	position for every input (indexed by RPF index)
+ */
+struct vsp1_drm {
+	struct vsp1_drm_pipeline pipe[VSP1_MAX_LIF];
+
 	struct {
-		bool enabled;
 		struct v4l2_rect crop;
 		struct v4l2_rect compose;
 		unsigned int zpos;
 	} inputs[VSP1_MAX_RPF];
-
-	/* Frame synchronisation */
-	void (*du_complete)(void *);
-	void *du_private;
 };
 
-static inline struct vsp1_drm *to_vsp1_drm(struct vsp1_pipeline *pipe)
+static inline struct vsp1_drm_pipeline *
+to_vsp1_drm_pipeline(struct vsp1_pipeline *pipe)
 {
-	return container_of(pipe, struct vsp1_drm, pipe);
+	return container_of(pipe, struct vsp1_drm_pipeline, pipe);
 }
 
 int vsp1_drm_init(struct vsp1_device *vsp1);
 void vsp1_drm_cleanup(struct vsp1_device *vsp1);
-int vsp1_drm_create_links(struct vsp1_device *vsp1);
-
-void vsp1_drm_display_start(struct vsp1_device *vsp1);
 
 #endif /* __VSP1_DRM_H__ */
diff --git a/drivers/media/platform/vsp1/vsp1_drv.c b/drivers/media/platform/vsp1/vsp1_drv.c
index 95c26edead85..962e4c304076 100644
--- a/drivers/media/platform/vsp1/vsp1_drv.c
+++ b/drivers/media/platform/vsp1/vsp1_drv.c
@@ -68,14 +68,6 @@ static irqreturn_t vsp1_irq_handler(int irq, void *data)
 		}
 	}
 
-	status = vsp1_read(vsp1, VI6_DISP_IRQ_STA);
-	vsp1_write(vsp1, VI6_DISP_IRQ_STA, ~status & VI6_DISP_IRQ_STA_DST);
-
-	if (status & VI6_DISP_IRQ_STA_DST) {
-		vsp1_drm_display_start(vsp1);
-		ret = IRQ_HANDLED;
-	}
-
 	return ret;
 }
 
@@ -92,6 +84,10 @@ static irqreturn_t vsp1_irq_handler(int irq, void *data)
  *
  * - from a UDS to a UDS (UDS entities can't be chained)
  * - from an entity to itself (no loops are allowed)
+ *
+ * Furthermore, the BRS can't be connected to histogram generators, but no
+ * special check is currently needed as all VSP instances that include a BRS
+ * have no histogram generator.
  */
 static int vsp1_create_sink_links(struct vsp1_device *vsp1,
 				  struct vsp1_entity *sink)
@@ -129,7 +125,7 @@ static int vsp1_create_sink_links(struct vsp1_device *vsp1,
 				return ret;
 
 			if (flags & MEDIA_LNK_FL_ENABLED)
-				source->sink = entity;
+				source->sink = sink;
 		}
 	}
 
@@ -172,10 +168,13 @@ static int vsp1_uapi_create_links(struct vsp1_device *vsp1)
 			return ret;
 	}
 
-	if (vsp1->lif) {
-		ret = media_create_pad_link(&vsp1->wpf[0]->entity.subdev.entity,
+	for (i = 0; i < vsp1->info->lif_count; ++i) {
+		if (!vsp1->lif[i])
+			continue;
+
+		ret = media_create_pad_link(&vsp1->wpf[i]->entity.subdev.entity,
 					    RWPF_PAD_SOURCE,
-					    &vsp1->lif->entity.subdev.entity,
+					    &vsp1->lif[i]->entity.subdev.entity,
 					    LIF_PAD_SINK, 0);
 		if (ret < 0)
 			return ret;
@@ -269,8 +268,18 @@ static int vsp1_create_entities(struct vsp1_device *vsp1)
 	}
 
 	/* Instantiate all the entities. */
+	if (vsp1->info->features & VSP1_HAS_BRS) {
+		vsp1->brs = vsp1_bru_create(vsp1, VSP1_ENTITY_BRS);
+		if (IS_ERR(vsp1->brs)) {
+			ret = PTR_ERR(vsp1->brs);
+			goto done;
+		}
+
+		list_add_tail(&vsp1->brs->entity.list_dev, &vsp1->entities);
+	}
+
 	if (vsp1->info->features & VSP1_HAS_BRU) {
-		vsp1->bru = vsp1_bru_create(vsp1);
+		vsp1->bru = vsp1_bru_create(vsp1, VSP1_ENTITY_BRU);
 		if (IS_ERR(vsp1->bru)) {
 			ret = PTR_ERR(vsp1->bru);
 			goto done;
@@ -328,18 +337,23 @@ static int vsp1_create_entities(struct vsp1_device *vsp1)
 	}
 
 	/*
-	 * The LIF is only supported when used in conjunction with the DU, in
+	 * The LIFs are only supported when used in conjunction with the DU, in
 	 * which case the userspace API is disabled. If the userspace API is
-	 * enabled skip the LIF, even when present.
+	 * enabled skip the LIFs, even when present.
 	 */
-	if (vsp1->info->features & VSP1_HAS_LIF && !vsp1->info->uapi) {
-		vsp1->lif = vsp1_lif_create(vsp1);
-		if (IS_ERR(vsp1->lif)) {
-			ret = PTR_ERR(vsp1->lif);
-			goto done;
-		}
+	if (!vsp1->info->uapi) {
+		for (i = 0; i < vsp1->info->lif_count; ++i) {
+			struct vsp1_lif *lif;
+
+			lif = vsp1_lif_create(vsp1, i);
+			if (IS_ERR(lif)) {
+				ret = PTR_ERR(lif);
+				goto done;
+			}
 
-		list_add_tail(&vsp1->lif->entity.list_dev, &vsp1->entities);
+			vsp1->lif[i] = lif;
+			list_add_tail(&lif->entity.list_dev, &vsp1->entities);
+		}
 	}
 
 	if (vsp1->info->features & VSP1_HAS_LUT) {
@@ -420,7 +434,6 @@ static int vsp1_create_entities(struct vsp1_device *vsp1)
 			}
 
 			list_add_tail(&video->list, &vsp1->videos);
-			wpf->entity.sink = &video->video.entity;
 		}
 	}
 
@@ -432,19 +445,15 @@ static int vsp1_create_entities(struct vsp1_device *vsp1)
 			goto done;
 	}
 
-	/* Create links. */
-	if (vsp1->info->uapi)
-		ret = vsp1_uapi_create_links(vsp1);
-	else
-		ret = vsp1_drm_create_links(vsp1);
-	if (ret < 0)
-		goto done;
-
 	/*
-	 * Register subdev nodes if the userspace API is enabled or initialize
-	 * the DRM pipeline otherwise.
+	 * Create links and register subdev nodes if the userspace API is
+	 * enabled or initialize the DRM pipeline otherwise.
 	 */
 	if (vsp1->info->uapi) {
+		ret = vsp1_uapi_create_links(vsp1);
+		if (ret < 0)
+			goto done;
+
 		ret = v4l2_device_register_subdev_nodes(&vsp1->v4l2_dev);
 		if (ret < 0)
 			goto done;
@@ -515,6 +524,9 @@ static int vsp1_device_init(struct vsp1_device *vsp1)
 	vsp1_write(vsp1, VI6_DPR_HSI_ROUTE, VI6_DPR_NODE_UNUSED);
 	vsp1_write(vsp1, VI6_DPR_BRU_ROUTE, VI6_DPR_NODE_UNUSED);
 
+	if (vsp1->info->features & VSP1_HAS_BRS)
+		vsp1_write(vsp1, VI6_DPR_ILV_BRS_ROUTE, VI6_DPR_NODE_UNUSED);
+
 	vsp1_write(vsp1, VI6_DPR_HGO_SMPPT, (7 << VI6_DPR_SMPPT_TGW_SHIFT) |
 		   (VI6_DPR_NODE_UNUSED << VI6_DPR_SMPPT_PT_SHIFT));
 	vsp1_write(vsp1, VI6_DPR_HGT_SMPPT, (7 << VI6_DPR_SMPPT_TGW_SHIFT) |
@@ -634,8 +646,8 @@ static const struct vsp1_device_info vsp1_device_infos[] = {
 		.version = VI6_IP_VERSION_MODEL_VSPD_GEN2,
 		.model = "VSP1-D",
 		.gen = 2,
-		.features = VSP1_HAS_BRU | VSP1_HAS_HGO | VSP1_HAS_LIF
-			  | VSP1_HAS_LUT,
+		.features = VSP1_HAS_BRU | VSP1_HAS_HGO | VSP1_HAS_LUT,
+		.lif_count = 1,
 		.rpf_count = 4,
 		.uds_count = 1,
 		.wpf_count = 1,
@@ -668,8 +680,8 @@ static const struct vsp1_device_info vsp1_device_infos[] = {
 		.version = VI6_IP_VERSION_MODEL_VSPD_V2H,
 		.model = "VSP1V-D",
 		.gen = 2,
-		.features = VSP1_HAS_BRU | VSP1_HAS_CLU | VSP1_HAS_LUT
-			  | VSP1_HAS_LIF,
+		.features = VSP1_HAS_BRU | VSP1_HAS_CLU | VSP1_HAS_LUT,
+		.lif_count = 1,
 		.rpf_count = 4,
 		.uds_count = 1,
 		.wpf_count = 1,
@@ -706,10 +718,37 @@ static const struct vsp1_device_info vsp1_device_infos[] = {
 		.num_bru_inputs = 5,
 		.uapi = true,
 	}, {
+		.version = VI6_IP_VERSION_MODEL_VSPBS_GEN3,
+		.model = "VSP2-BS",
+		.gen = 3,
+		.features = VSP1_HAS_BRS | VSP1_HAS_WPF_VFLIP,
+		.rpf_count = 2,
+		.wpf_count = 1,
+		.uapi = true,
+	}, {
 		.version = VI6_IP_VERSION_MODEL_VSPD_GEN3,
 		.model = "VSP2-D",
 		.gen = 3,
-		.features = VSP1_HAS_BRU | VSP1_HAS_LIF | VSP1_HAS_WPF_VFLIP,
+		.features = VSP1_HAS_BRU | VSP1_HAS_WPF_VFLIP,
+		.lif_count = 1,
+		.rpf_count = 5,
+		.wpf_count = 2,
+		.num_bru_inputs = 5,
+	}, {
+		.version = VI6_IP_VERSION_MODEL_VSPD_V3,
+		.model = "VSP2-D",
+		.gen = 3,
+		.features = VSP1_HAS_BRS | VSP1_HAS_BRU,
+		.lif_count = 1,
+		.rpf_count = 5,
+		.wpf_count = 1,
+		.num_bru_inputs = 5,
+	}, {
+		.version = VI6_IP_VERSION_MODEL_VSPDL_GEN3,
+		.model = "VSP2-DL",
+		.gen = 3,
+		.features = VSP1_HAS_BRS | VSP1_HAS_BRU,
+		.lif_count = 2,
 		.rpf_count = 5,
 		.wpf_count = 2,
 		.num_bru_inputs = 5,
diff --git a/drivers/media/platform/vsp1/vsp1_entity.c b/drivers/media/platform/vsp1/vsp1_entity.c
index 4bdb3b141611..54de15095709 100644
--- a/drivers/media/platform/vsp1/vsp1_entity.c
+++ b/drivers/media/platform/vsp1/vsp1_entity.c
@@ -24,18 +24,12 @@
 #include "vsp1_pipe.h"
 #include "vsp1_rwpf.h"
 
-static inline struct vsp1_entity *
-media_entity_to_vsp1_entity(struct media_entity *entity)
-{
-	return container_of(entity, struct vsp1_entity, subdev.entity);
-}
-
 void vsp1_entity_route_setup(struct vsp1_entity *entity,
 			     struct vsp1_pipeline *pipe,
 			     struct vsp1_dl_list *dl)
 {
 	struct vsp1_entity *source;
-	struct vsp1_entity *sink;
+	u32 route;
 
 	if (entity->type == VSP1_ENTITY_HGO) {
 		u32 smppt;
@@ -44,7 +38,7 @@ void vsp1_entity_route_setup(struct vsp1_entity *entity,
 		 * The HGO is a special case, its routing is configured on the
 		 * sink pad.
 		 */
-		source = media_entity_to_vsp1_entity(entity->sources[0]);
+		source = entity->sources[0];
 		smppt = (pipe->output->entity.index << VI6_DPR_SMPPT_TGW_SHIFT)
 		      | (source->route->output << VI6_DPR_SMPPT_PT_SHIFT);
 
@@ -57,7 +51,7 @@ void vsp1_entity_route_setup(struct vsp1_entity *entity,
 		 * The HGT is a special case, its routing is configured on the
 		 * sink pad.
 		 */
-		source = media_entity_to_vsp1_entity(entity->sources[0]);
+		source = entity->sources[0];
 		smppt = (pipe->output->entity.index << VI6_DPR_SMPPT_TGW_SHIFT)
 		      | (source->route->output << VI6_DPR_SMPPT_PT_SHIFT);
 
@@ -69,9 +63,14 @@ void vsp1_entity_route_setup(struct vsp1_entity *entity,
 	if (source->route->reg == 0)
 		return;
 
-	sink = media_entity_to_vsp1_entity(source->sink);
-	vsp1_dl_list_write(dl, source->route->reg,
-			   sink->route->inputs[source->sink_pad]);
+	route = source->sink->route->inputs[source->sink_pad];
+	/*
+	 * The ILV and BRS share the same data path route. The extra BRSSEL bit
+	 * selects between the ILV and BRS.
+	 */
+	if (source->type == VSP1_ENTITY_BRS)
+		route |= VI6_DPR_ROUTE_BRSSEL;
+	vsp1_dl_list_write(dl, source->route->reg, route);
 }
 
 /* -----------------------------------------------------------------------------
@@ -316,6 +315,12 @@ done:
  * Media Operations
  */
 
+static inline struct vsp1_entity *
+media_entity_to_vsp1_entity(struct media_entity *entity)
+{
+	return container_of(entity, struct vsp1_entity, subdev.entity);
+}
+
 static int vsp1_entity_link_setup_source(const struct media_pad *source_pad,
 					 const struct media_pad *sink_pad,
 					 u32 flags)
@@ -339,7 +344,7 @@ static int vsp1_entity_link_setup_source(const struct media_pad *source_pad,
 		    sink->type != VSP1_ENTITY_HGT) {
 			if (source->sink)
 				return -EBUSY;
-			source->sink = sink_pad->entity;
+			source->sink = sink;
 			source->sink_pad = sink_pad->index;
 		}
 	} else {
@@ -355,15 +360,17 @@ static int vsp1_entity_link_setup_sink(const struct media_pad *source_pad,
 				       u32 flags)
 {
 	struct vsp1_entity *sink;
+	struct vsp1_entity *source;
 
 	sink = media_entity_to_vsp1_entity(sink_pad->entity);
+	source = media_entity_to_vsp1_entity(source_pad->entity);
 
 	if (flags & MEDIA_LNK_FL_ENABLED) {
 		/* Fan-in is limited to one. */
 		if (sink->sources[sink_pad->index])
 			return -EBUSY;
 
-		sink->sources[sink_pad->index] = source_pad->entity;
+		sink->sources[sink_pad->index] = source;
 	} else {
 		sink->sources[sink_pad->index] = NULL;
 	}
@@ -450,6 +457,8 @@ struct media_pad *vsp1_entity_remote_pad(struct media_pad *pad)
 	  { VI6_DPR_NODE_WPF(idx) }, VI6_DPR_NODE_WPF(idx) }
 
 static const struct vsp1_route vsp1_routes[] = {
+	{ VSP1_ENTITY_BRS, 0, VI6_DPR_ILV_BRS_ROUTE,
+	  { VI6_DPR_NODE_BRS_IN(0), VI6_DPR_NODE_BRS_IN(1) }, 0 },
 	{ VSP1_ENTITY_BRU, 0, VI6_DPR_BRU_ROUTE,
 	  { VI6_DPR_NODE_BRU_IN(0), VI6_DPR_NODE_BRU_IN(1),
 	    VI6_DPR_NODE_BRU_IN(2), VI6_DPR_NODE_BRU_IN(3),
@@ -459,7 +468,8 @@ static const struct vsp1_route vsp1_routes[] = {
 	{ VSP1_ENTITY_HGT, 0, 0, { 0, }, 0 },
 	VSP1_ENTITY_ROUTE(HSI),
 	VSP1_ENTITY_ROUTE(HST),
-	{ VSP1_ENTITY_LIF, 0, 0, { VI6_DPR_NODE_LIF, }, VI6_DPR_NODE_LIF },
+	{ VSP1_ENTITY_LIF, 0, 0, { 0, }, 0 },
+	{ VSP1_ENTITY_LIF, 1, 0, { 0, }, 0 },
 	VSP1_ENTITY_ROUTE(LUT),
 	VSP1_ENTITY_ROUTE_RPF(0),
 	VSP1_ENTITY_ROUTE_RPF(1),
diff --git a/drivers/media/platform/vsp1/vsp1_entity.h b/drivers/media/platform/vsp1/vsp1_entity.h
index c169a060b6d2..408602ebeb97 100644
--- a/drivers/media/platform/vsp1/vsp1_entity.h
+++ b/drivers/media/platform/vsp1/vsp1_entity.h
@@ -21,8 +21,11 @@
 struct vsp1_device;
 struct vsp1_dl_list;
 struct vsp1_pipeline;
+struct vsp1_partition;
+struct vsp1_partition_window;
 
 enum vsp1_entity_type {
+	VSP1_ENTITY_BRS,
 	VSP1_ENTITY_BRU,
 	VSP1_ENTITY_CLU,
 	VSP1_ENTITY_HGO,
@@ -81,12 +84,17 @@ struct vsp1_route {
  *		selection rectangles, ...)
  * @max_width:	Return the max supported width of data that the entity can
  *		process in a single operation.
+ * @partition:	Process the partition construction based on this entity's
+ *		configuration.
  */
 struct vsp1_entity_operations {
 	void (*destroy)(struct vsp1_entity *);
 	void (*configure)(struct vsp1_entity *, struct vsp1_pipeline *,
 			  struct vsp1_dl_list *, enum vsp1_entity_params);
 	unsigned int (*max_width)(struct vsp1_entity *, struct vsp1_pipeline *);
+	void (*partition)(struct vsp1_entity *, struct vsp1_pipeline *,
+			  struct vsp1_partition *, unsigned int,
+			  struct vsp1_partition_window *);
 };
 
 struct vsp1_entity {
@@ -104,8 +112,8 @@ struct vsp1_entity {
 	struct media_pad *pads;
 	unsigned int source_pad;
 
-	struct media_entity **sources;
-	struct media_entity *sink;
+	struct vsp1_entity **sources;
+	struct vsp1_entity *sink;
 	unsigned int sink_pad;
 
 	struct v4l2_subdev subdev;
diff --git a/drivers/media/platform/vsp1/vsp1_lif.c b/drivers/media/platform/vsp1/vsp1_lif.c
index 702487f895b3..e6fa16d7fda8 100644
--- a/drivers/media/platform/vsp1/vsp1_lif.c
+++ b/drivers/media/platform/vsp1/vsp1_lif.c
@@ -30,7 +30,7 @@
 static inline void vsp1_lif_write(struct vsp1_lif *lif, struct vsp1_dl_list *dl,
 				  u32 reg, u32 data)
 {
-	vsp1_dl_list_write(dl, reg, data);
+	vsp1_dl_list_write(dl, reg + lif->entity.index * VI6_LIF_OFFSET, data);
 }
 
 /* -----------------------------------------------------------------------------
@@ -165,7 +165,7 @@ static const struct vsp1_entity_operations lif_entity_ops = {
  * Initialization and Cleanup
  */
 
-struct vsp1_lif *vsp1_lif_create(struct vsp1_device *vsp1)
+struct vsp1_lif *vsp1_lif_create(struct vsp1_device *vsp1, unsigned int index)
 {
 	struct vsp1_lif *lif;
 	int ret;
@@ -176,6 +176,7 @@ struct vsp1_lif *vsp1_lif_create(struct vsp1_device *vsp1)
 
 	lif->entity.ops = &lif_entity_ops;
 	lif->entity.type = VSP1_ENTITY_LIF;
+	lif->entity.index = index;
 
 	/*
 	 * The LIF is never exposed to userspace, but media entity registration
diff --git a/drivers/media/platform/vsp1/vsp1_lif.h b/drivers/media/platform/vsp1/vsp1_lif.h
index 7b35879028de..3417339379b1 100644
--- a/drivers/media/platform/vsp1/vsp1_lif.h
+++ b/drivers/media/platform/vsp1/vsp1_lif.h
@@ -32,6 +32,6 @@ static inline struct vsp1_lif *to_lif(struct v4l2_subdev *subdev)
 	return container_of(subdev, struct vsp1_lif, entity.subdev);
 }
 
-struct vsp1_lif *vsp1_lif_create(struct vsp1_device *vsp1);
+struct vsp1_lif *vsp1_lif_create(struct vsp1_device *vsp1, unsigned int index);
 
 #endif /* __VSP1_LIF_H__ */
diff --git a/drivers/media/platform/vsp1/vsp1_pipe.c b/drivers/media/platform/vsp1/vsp1_pipe.c
index e817623b84e0..44944ac86d9b 100644
--- a/drivers/media/platform/vsp1/vsp1_pipe.c
+++ b/drivers/media/platform/vsp1/vsp1_pipe.c
@@ -335,16 +335,12 @@ void vsp1_pipeline_frame_end(struct vsp1_pipeline *pipe)
 	if (pipe == NULL)
 		return;
 
+	/*
+	 * If the DL commit raced with the frame end interrupt, the commit ends
+	 * up being postponed by one frame. @completed represents whether the
+	 * active frame was finished or postponed.
+	 */
 	completed = vsp1_dlm_irq_frame_end(pipe->output->dlm);
-	if (!completed) {
-		/*
-		 * If the DL commit raced with the frame end interrupt, the
-		 * commit ends up being postponed by one frame. Return
-		 * immediately without calling the pipeline's frame end handler
-		 * or incrementing the sequence number.
-		 */
-		return;
-	}
 
 	if (pipe->hgo)
 		vsp1_hgo_frame_end(pipe->hgo);
@@ -352,8 +348,12 @@ void vsp1_pipeline_frame_end(struct vsp1_pipeline *pipe)
 	if (pipe->hgt)
 		vsp1_hgt_frame_end(pipe->hgt);
 
+	/*
+	 * Regardless of frame completion we still need to notify the pipe
+	 * frame_end to account for vblank events.
+	 */
 	if (pipe->frame_end)
-		pipe->frame_end(pipe);
+		pipe->frame_end(pipe, completed);
 
 	pipe->sequence++;
 }
@@ -373,15 +373,38 @@ void vsp1_pipeline_propagate_alpha(struct vsp1_pipeline *pipe,
 		return;
 
 	/*
-	 * The BRU background color has a fixed alpha value set to 255, the
-	 * output alpha value is thus always equal to 255.
+	 * The BRU and BRS background color has a fixed alpha value set to 255,
+	 * the output alpha value is thus always equal to 255.
 	 */
-	if (pipe->uds_input->type == VSP1_ENTITY_BRU)
+	if (pipe->uds_input->type == VSP1_ENTITY_BRU ||
+	    pipe->uds_input->type == VSP1_ENTITY_BRS)
 		alpha = 255;
 
 	vsp1_uds_set_alpha(pipe->uds, dl, alpha);
 }
 
+/*
+ * Propagate the partition calculations through the pipeline
+ *
+ * Work backwards through the pipe, allowing each entity to update the partition
+ * parameters based on its configuration, and the entity connected to its
+ * source. Each entity must produce the partition required for the previous
+ * entity in the pipeline.
+ */
+void vsp1_pipeline_propagate_partition(struct vsp1_pipeline *pipe,
+				       struct vsp1_partition *partition,
+				       unsigned int index,
+				       struct vsp1_partition_window *window)
+{
+	struct vsp1_entity *entity;
+
+	list_for_each_entry_reverse(entity, &pipe->entities, list_pipe) {
+		if (entity->ops->partition)
+			entity->ops->partition(entity, pipe, partition, index,
+					       window);
+	}
+}
+
 void vsp1_pipelines_suspend(struct vsp1_device *vsp1)
 {
 	unsigned long flags;
diff --git a/drivers/media/platform/vsp1/vsp1_pipe.h b/drivers/media/platform/vsp1/vsp1_pipe.h
index 91a784a13422..dfff9b5685fe 100644
--- a/drivers/media/platform/vsp1/vsp1_pipe.h
+++ b/drivers/media/platform/vsp1/vsp1_pipe.h
@@ -58,6 +58,33 @@ enum vsp1_pipeline_state {
 };
 
 /*
+ * struct vsp1_partition_window - Partition window coordinates
+ * @left: horizontal coordinate of the partition start in pixels relative to the
+ *	  left edge of the image
+ * @width: partition width in pixels
+ */
+struct vsp1_partition_window {
+	unsigned int left;
+	unsigned int width;
+};
+
+/*
+ * struct vsp1_partition - A description of a slice for the partition algorithm
+ * @rpf: The RPF partition window configuration
+ * @uds_sink: The UDS input partition window configuration
+ * @uds_source: The UDS output partition window configuration
+ * @sru: The SRU partition window configuration
+ * @wpf: The WPF partition window configuration
+ */
+struct vsp1_partition {
+	struct vsp1_partition_window rpf;
+	struct vsp1_partition_window uds_sink;
+	struct vsp1_partition_window uds_source;
+	struct vsp1_partition_window sru;
+	struct vsp1_partition_window wpf;
+};
+
+/*
  * struct vsp1_pipeline - A VSP1 hardware pipeline
  * @pipe: the media pipeline
  * @irqlock: protects the pipeline state
@@ -80,9 +107,9 @@ enum vsp1_pipeline_state {
  * @uds_input: entity at the input of the UDS, if the UDS is present
  * @entities: list of entities in the pipeline
  * @dl: display list associated with the pipeline
- * @div_size: The maximum allowed partition size for the pipeline
  * @partitions: The number of partitions used to process one frame
- * @current_partition: The partition number currently being configured
+ * @partition: The current partition for configuration to process
+ * @part_table: The pre-calculated partitions used by the pipeline
  */
 struct vsp1_pipeline {
 	struct media_pipeline pipe;
@@ -91,7 +118,7 @@ struct vsp1_pipeline {
 	enum vsp1_pipeline_state state;
 	wait_queue_head_t wq;
 
-	void (*frame_end)(struct vsp1_pipeline *pipe);
+	void (*frame_end)(struct vsp1_pipeline *pipe, bool completed);
 
 	struct mutex lock;
 	struct kref kref;
@@ -109,14 +136,18 @@ struct vsp1_pipeline {
 	struct vsp1_entity *uds;
 	struct vsp1_entity *uds_input;
 
+	/*
+	 * The order of this list must be identical to the order of the entities
+	 * in the pipeline, as it is assumed by the partition algorithm that we
+	 * can walk this list in sequence.
+	 */
 	struct list_head entities;
 
 	struct vsp1_dl_list *dl;
 
-	unsigned int div_size;
 	unsigned int partitions;
-	struct v4l2_rect partition;
-	unsigned int current_partition;
+	struct vsp1_partition *partition;
+	struct vsp1_partition *part_table;
 };
 
 void vsp1_pipeline_reset(struct vsp1_pipeline *pipe);
@@ -132,6 +163,11 @@ void vsp1_pipeline_frame_end(struct vsp1_pipeline *pipe);
 void vsp1_pipeline_propagate_alpha(struct vsp1_pipeline *pipe,
 				   struct vsp1_dl_list *dl, unsigned int alpha);
 
+void vsp1_pipeline_propagate_partition(struct vsp1_pipeline *pipe,
+				       struct vsp1_partition *partition,
+				       unsigned int index,
+				       struct vsp1_partition_window *window);
+
 void vsp1_pipelines_suspend(struct vsp1_device *vsp1);
 void vsp1_pipelines_resume(struct vsp1_device *vsp1);
 
diff --git a/drivers/media/platform/vsp1/vsp1_regs.h b/drivers/media/platform/vsp1/vsp1_regs.h
index cd3e32af6e3b..26c4ffad2f46 100644
--- a/drivers/media/platform/vsp1/vsp1_regs.h
+++ b/drivers/media/platform/vsp1/vsp1_regs.h
@@ -18,6 +18,7 @@
  */
 
 #define VI6_CMD(n)			(0x0000 + (n) * 4)
+#define VI6_CMD_UPDHDR			(1 << 4)
 #define VI6_CMD_STRCMD			(1 << 0)
 
 #define VI6_CLK_DCSWT			0x0018
@@ -238,6 +239,10 @@
 #define VI6_WPF_SRCRPF_VIRACT_SUB	(1 << 28)
 #define VI6_WPF_SRCRPF_VIRACT_MST	(2 << 28)
 #define VI6_WPF_SRCRPF_VIRACT_MASK	(3 << 28)
+#define VI6_WPF_SRCRPF_VIRACT2_DIS	(0 << 24)
+#define VI6_WPF_SRCRPF_VIRACT2_SUB	(1 << 24)
+#define VI6_WPF_SRCRPF_VIRACT2_MST	(2 << 24)
+#define VI6_WPF_SRCRPF_VIRACT2_MASK	(3 << 24)
 #define VI6_WPF_SRCRPF_RPF_ACT_DIS(n)	(0 << ((n) * 2))
 #define VI6_WPF_SRCRPF_RPF_ACT_SUB(n)	(1 << ((n) * 2))
 #define VI6_WPF_SRCRPF_RPF_ACT_MST(n)	(2 << ((n) * 2))
@@ -321,6 +326,8 @@
 #define VI6_DPR_HST_ROUTE		0x2044
 #define VI6_DPR_HSI_ROUTE		0x2048
 #define VI6_DPR_BRU_ROUTE		0x204c
+#define VI6_DPR_ILV_BRS_ROUTE		0x2050
+#define VI6_DPR_ROUTE_BRSSEL		(1 << 28)
 #define VI6_DPR_ROUTE_FXA_MASK		(0xff << 16)
 #define VI6_DPR_ROUTE_FXA_SHIFT		16
 #define VI6_DPR_ROUTE_FP_MASK		(0x3f << 8)
@@ -344,7 +351,8 @@
 #define VI6_DPR_NODE_CLU		29
 #define VI6_DPR_NODE_HST		30
 #define VI6_DPR_NODE_HSI		31
-#define VI6_DPR_NODE_LIF		55
+#define VI6_DPR_NODE_BRS_IN(n)		(38 + (n))
+#define VI6_DPR_NODE_LIF		55		/* Gen2 only */
 #define VI6_DPR_NODE_WPF(n)		(56 + (n))
 #define VI6_DPR_NODE_UNUSED		63
 
@@ -388,6 +396,7 @@
 #define VI6_UDS_CTRL_NE_RCR		(1 << 18)
 #define VI6_UDS_CTRL_NE_GY		(1 << 17)
 #define VI6_UDS_CTRL_NE_BCB		(1 << 16)
+#define VI6_UDS_CTRL_AMDSLH		(1 << 2)
 #define VI6_UDS_CTRL_TDIPC		(1 << 1)
 
 #define VI6_UDS_SCALE			0x2304
@@ -420,11 +429,24 @@
 #define VI6_UDS_PASS_BWIDTH_V_MASK	(0x7f << 0)
 #define VI6_UDS_PASS_BWIDTH_V_SHIFT	0
 
+#define VI6_UDS_HPHASE			0x2314
+#define VI6_UDS_HPHASE_HSTP_MASK	(0xfff << 16)
+#define VI6_UDS_HPHASE_HSTP_SHIFT	16
+#define VI6_UDS_HPHASE_HEDP_MASK	(0xfff << 0)
+#define VI6_UDS_HPHASE_HEDP_SHIFT	0
+
 #define VI6_UDS_IPC			0x2318
 #define VI6_UDS_IPC_FIELD		(1 << 27)
 #define VI6_UDS_IPC_VEDP_MASK		(0xfff << 0)
 #define VI6_UDS_IPC_VEDP_SHIFT		0
 
+#define VI6_UDS_HSZCLIP			0x231c
+#define VI6_UDS_HSZCLIP_HCEN		(1 << 28)
+#define VI6_UDS_HSZCLIP_HCL_OFST_MASK	(0xff << 16)
+#define VI6_UDS_HSZCLIP_HCL_OFST_SHIFT	16
+#define VI6_UDS_HSZCLIP_HCL_SIZE_MASK	(0x1fff << 0)
+#define VI6_UDS_HSZCLIP_HCL_SIZE_SHIFT	0
+
 #define VI6_UDS_CLIP_SIZE		0x2324
 #define VI6_UDS_CLIP_SIZE_HSIZE_MASK	(0x1fff << 16)
 #define VI6_UDS_CLIP_SIZE_HSIZE_SHIFT	16
@@ -476,7 +498,7 @@
 #define VI6_HSI_CTRL_EN			(1 << 0)
 
 /* -----------------------------------------------------------------------------
- * BRU Control Registers
+ * BRS and BRU Control Registers
  */
 
 #define VI6_ROP_NOP			0
@@ -496,7 +518,10 @@
 #define VI6_ROP_NAND			14
 #define VI6_ROP_SET			15
 
-#define VI6_BRU_INCTRL			0x2c00
+#define VI6_BRU_BASE			0x2c00
+#define VI6_BRS_BASE			0x3900
+
+#define VI6_BRU_INCTRL			0x0000
 #define VI6_BRU_INCTRL_NRM		(1 << 28)
 #define VI6_BRU_INCTRL_DnON		(1 << (16 + (n)))
 #define VI6_BRU_INCTRL_DITHn_OFF	(0 << ((n) * 4))
@@ -508,19 +533,19 @@
 #define VI6_BRU_INCTRL_DITHn_MASK	(7 << ((n) * 4))
 #define VI6_BRU_INCTRL_DITHn_SHIFT	((n) * 4)
 
-#define VI6_BRU_VIRRPF_SIZE		0x2c04
+#define VI6_BRU_VIRRPF_SIZE		0x0004
 #define VI6_BRU_VIRRPF_SIZE_HSIZE_MASK	(0x1fff << 16)
 #define VI6_BRU_VIRRPF_SIZE_HSIZE_SHIFT	16
 #define VI6_BRU_VIRRPF_SIZE_VSIZE_MASK	(0x1fff << 0)
 #define VI6_BRU_VIRRPF_SIZE_VSIZE_SHIFT	0
 
-#define VI6_BRU_VIRRPF_LOC		0x2c08
+#define VI6_BRU_VIRRPF_LOC		0x0008
 #define VI6_BRU_VIRRPF_LOC_HCOORD_MASK	(0x1fff << 16)
 #define VI6_BRU_VIRRPF_LOC_HCOORD_SHIFT	16
 #define VI6_BRU_VIRRPF_LOC_VCOORD_MASK	(0x1fff << 0)
 #define VI6_BRU_VIRRPF_LOC_VCOORD_SHIFT	0
 
-#define VI6_BRU_VIRRPF_COL		0x2c0c
+#define VI6_BRU_VIRRPF_COL		0x000c
 #define VI6_BRU_VIRRPF_COL_A_MASK	(0xff << 24)
 #define VI6_BRU_VIRRPF_COL_A_SHIFT	24
 #define VI6_BRU_VIRRPF_COL_RCR_MASK	(0xff << 16)
@@ -530,7 +555,7 @@
 #define VI6_BRU_VIRRPF_COL_BCB_MASK	(0xff << 0)
 #define VI6_BRU_VIRRPF_COL_BCB_SHIFT	0
 
-#define VI6_BRU_CTRL(n)			(0x2c10 + (n) * 8 + ((n) <= 3 ? 0 : 4))
+#define VI6_BRU_CTRL(n)			(0x0010 + (n) * 8 + ((n) <= 3 ? 0 : 4))
 #define VI6_BRU_CTRL_RBC		(1 << 31)
 #define VI6_BRU_CTRL_DSTSEL_BRUIN(n)	(((n) <= 3 ? (n) : (n)+1) << 20)
 #define VI6_BRU_CTRL_DSTSEL_VRPF	(4 << 20)
@@ -543,7 +568,7 @@
 #define VI6_BRU_CTRL_AROP(rop)		((rop) << 0)
 #define VI6_BRU_CTRL_AROP_MASK		(0xf << 0)
 
-#define VI6_BRU_BLD(n)			(0x2c14 + (n) * 8 + ((n) <= 3 ? 0 : 4))
+#define VI6_BRU_BLD(n)			(0x0014 + (n) * 8 + ((n) <= 3 ? 0 : 4))
 #define VI6_BRU_BLD_CBES		(1 << 31)
 #define VI6_BRU_BLD_CCMDX_DST_A		(0 << 28)
 #define VI6_BRU_BLD_CCMDX_255_DST_A	(1 << 28)
@@ -576,7 +601,7 @@
 #define VI6_BRU_BLD_COEFY_MASK		(0xff << 0)
 #define VI6_BRU_BLD_COEFY_SHIFT		0
 
-#define VI6_BRU_ROP			0x2c30
+#define VI6_BRU_ROP			0x0030	/* Only available on BRU */
 #define VI6_BRU_ROP_DSTSEL_BRUIN(n)	(((n) <= 3 ? (n) : (n)+1) << 20)
 #define VI6_BRU_ROP_DSTSEL_VRPF		(4 << 20)
 #define VI6_BRU_ROP_DSTSEL_MASK		(7 << 20)
@@ -653,6 +678,8 @@
  * LIF Control Registers
  */
 
+#define VI6_LIF_OFFSET			(-0x100)
+
 #define VI6_LIF_CTRL			0x3b00
 #define VI6_LIF_CTRL_OBTH_MASK		(0x7ff << 16)
 #define VI6_LIF_CTRL_OBTH_SHIFT		16
@@ -689,9 +716,20 @@
 #define VI6_IP_VERSION_MODEL_VSPBD_GEN3	(0x15 << 8)
 #define VI6_IP_VERSION_MODEL_VSPBC_GEN3	(0x16 << 8)
 #define VI6_IP_VERSION_MODEL_VSPD_GEN3	(0x17 << 8)
+#define VI6_IP_VERSION_MODEL_VSPD_V3	(0x18 << 8)
+#define VI6_IP_VERSION_MODEL_VSPDL_GEN3	(0x19 << 8)
+#define VI6_IP_VERSION_MODEL_VSPBS_GEN3	(0x1a << 8)
 #define VI6_IP_VERSION_SOC_MASK		(0xff << 0)
-#define VI6_IP_VERSION_SOC_H		(0x01 << 0)
-#define VI6_IP_VERSION_SOC_M		(0x02 << 0)
+#define VI6_IP_VERSION_SOC_H2		(0x01 << 0)
+#define VI6_IP_VERSION_SOC_V2H		(0x01 << 0)
+#define VI6_IP_VERSION_SOC_V3M		(0x01 << 0)
+#define VI6_IP_VERSION_SOC_M2		(0x02 << 0)
+#define VI6_IP_VERSION_SOC_M3W		(0x02 << 0)
+#define VI6_IP_VERSION_SOC_V3H		(0x02 << 0)
+#define VI6_IP_VERSION_SOC_H3		(0x03 << 0)
+#define VI6_IP_VERSION_SOC_D3		(0x04 << 0)
+#define VI6_IP_VERSION_SOC_M3N		(0x04 << 0)
+#define VI6_IP_VERSION_SOC_E3		(0x04 << 0)
 
 /* -----------------------------------------------------------------------------
  * RPF CLUT Registers
diff --git a/drivers/media/platform/vsp1/vsp1_rpf.c b/drivers/media/platform/vsp1/vsp1_rpf.c
index 8feddd59cf8d..fe0633da5a5f 100644
--- a/drivers/media/platform/vsp1/vsp1_rpf.c
+++ b/drivers/media/platform/vsp1/vsp1_rpf.c
@@ -97,21 +97,8 @@ static void rpf_configure(struct vsp1_entity *entity,
 		 * 'width' need to be adjusted.
 		 */
 		if (pipe->partitions > 1) {
-			const struct v4l2_mbus_framefmt *output;
-			struct vsp1_entity *wpf = &pipe->output->entity;
-			unsigned int input_width = crop.width;
-
-			/*
-			 * Scale the partition window based on the configuration
-			 * of the pipeline.
-			 */
-			output = vsp1_entity_get_pad_format(wpf, wpf->config,
-							    RWPF_PAD_SINK);
-
-			crop.width = pipe->partition.width * input_width
-				   / output->width;
-			crop.left += pipe->partition.left * input_width
-				   / output->width;
+			crop.width = pipe->partition->rpf.width;
+			crop.left += pipe->partition->rpf.left;
 		}
 
 		vsp1_rpf_write(rpf, dl, VI6_RPF_SRC_BSIZE,
@@ -260,8 +247,18 @@ static void rpf_configure(struct vsp1_entity *entity,
 
 }
 
+static void rpf_partition(struct vsp1_entity *entity,
+			  struct vsp1_pipeline *pipe,
+			  struct vsp1_partition *partition,
+			  unsigned int partition_idx,
+			  struct vsp1_partition_window *window)
+{
+	partition->rpf = *window;
+}
+
 static const struct vsp1_entity_operations rpf_entity_ops = {
 	.configure = rpf_configure,
+	.partition = rpf_partition,
 };
 
 /* -----------------------------------------------------------------------------
diff --git a/drivers/media/platform/vsp1/vsp1_sru.c b/drivers/media/platform/vsp1/vsp1_sru.c
index 30142793dfcd..51e5691187c3 100644
--- a/drivers/media/platform/vsp1/vsp1_sru.c
+++ b/drivers/media/platform/vsp1/vsp1_sru.c
@@ -18,6 +18,7 @@
 
 #include "vsp1.h"
 #include "vsp1_dl.h"
+#include "vsp1_pipe.h"
 #include "vsp1_sru.h"
 
 #define SRU_MIN_SIZE				4U
@@ -325,9 +326,34 @@ static unsigned int sru_max_width(struct vsp1_entity *entity,
 		return 256;
 }
 
+static void sru_partition(struct vsp1_entity *entity,
+			  struct vsp1_pipeline *pipe,
+			  struct vsp1_partition *partition,
+			  unsigned int partition_idx,
+			  struct vsp1_partition_window *window)
+{
+	struct vsp1_sru *sru = to_sru(&entity->subdev);
+	struct v4l2_mbus_framefmt *input;
+	struct v4l2_mbus_framefmt *output;
+
+	input = vsp1_entity_get_pad_format(&sru->entity, sru->entity.config,
+					   SRU_PAD_SINK);
+	output = vsp1_entity_get_pad_format(&sru->entity, sru->entity.config,
+					    SRU_PAD_SOURCE);
+
+	/* Adapt if SRUx2 is enabled */
+	if (input->width != output->width) {
+		window->width /= 2;
+		window->left /= 2;
+	}
+
+	partition->sru = *window;
+}
+
 static const struct vsp1_entity_operations sru_entity_ops = {
 	.configure = sru_configure,
 	.max_width = sru_max_width,
+	.partition = sru_partition,
 };
 
 /* -----------------------------------------------------------------------------
diff --git a/drivers/media/platform/vsp1/vsp1_uds.c b/drivers/media/platform/vsp1/vsp1_uds.c
index 4226403ad235..72f72a9d2152 100644
--- a/drivers/media/platform/vsp1/vsp1_uds.c
+++ b/drivers/media/platform/vsp1/vsp1_uds.c
@@ -271,23 +271,32 @@ static void uds_configure(struct vsp1_entity *entity,
 	unsigned int vscale;
 	bool multitap;
 
+	input = vsp1_entity_get_pad_format(&uds->entity, uds->entity.config,
+					   UDS_PAD_SINK);
+	output = vsp1_entity_get_pad_format(&uds->entity, uds->entity.config,
+					    UDS_PAD_SOURCE);
+
 	if (params == VSP1_ENTITY_PARAMS_PARTITION) {
-		const struct v4l2_rect *clip = &pipe->partition;
+		struct vsp1_partition *partition = pipe->partition;
+
+		/* Input size clipping */
+		vsp1_uds_write(uds, dl, VI6_UDS_HSZCLIP, VI6_UDS_HSZCLIP_HCEN |
+			       (0 << VI6_UDS_HSZCLIP_HCL_OFST_SHIFT) |
+			       (partition->uds_sink.width
+					<< VI6_UDS_HSZCLIP_HCL_SIZE_SHIFT));
 
+		/* Output size clipping */
 		vsp1_uds_write(uds, dl, VI6_UDS_CLIP_SIZE,
-			       (clip->width << VI6_UDS_CLIP_SIZE_HSIZE_SHIFT) |
-			       (clip->height << VI6_UDS_CLIP_SIZE_VSIZE_SHIFT));
+			       (partition->uds_source.width
+					<< VI6_UDS_CLIP_SIZE_HSIZE_SHIFT) |
+			       (output->height
+					<< VI6_UDS_CLIP_SIZE_VSIZE_SHIFT));
 		return;
 	}
 
 	if (params != VSP1_ENTITY_PARAMS_INIT)
 		return;
 
-	input = vsp1_entity_get_pad_format(&uds->entity, uds->entity.config,
-					   UDS_PAD_SINK);
-	output = vsp1_entity_get_pad_format(&uds->entity, uds->entity.config,
-					    UDS_PAD_SOURCE);
-
 	hscale = uds_compute_ratio(input->width, output->width);
 	vscale = uds_compute_ratio(input->height, output->height);
 
@@ -343,9 +352,41 @@ static unsigned int uds_max_width(struct vsp1_entity *entity,
 		return 2048;
 }
 
+/* -----------------------------------------------------------------------------
+ * Partition Algorithm Support
+ */
+
+static void uds_partition(struct vsp1_entity *entity,
+			  struct vsp1_pipeline *pipe,
+			  struct vsp1_partition *partition,
+			  unsigned int partition_idx,
+			  struct vsp1_partition_window *window)
+{
+	struct vsp1_uds *uds = to_uds(&entity->subdev);
+	const struct v4l2_mbus_framefmt *output;
+	const struct v4l2_mbus_framefmt *input;
+
+	/* Initialise the partition state */
+	partition->uds_sink = *window;
+	partition->uds_source = *window;
+
+	input = vsp1_entity_get_pad_format(&uds->entity, uds->entity.config,
+					   UDS_PAD_SINK);
+	output = vsp1_entity_get_pad_format(&uds->entity, uds->entity.config,
+					    UDS_PAD_SOURCE);
+
+	partition->uds_sink.width = window->width * input->width
+				  / output->width;
+	partition->uds_sink.left = window->left * input->width
+				 / output->width;
+
+	*window = partition->uds_sink;
+}
+
 static const struct vsp1_entity_operations uds_entity_ops = {
 	.configure = uds_configure,
 	.max_width = uds_max_width,
+	.partition = uds_partition,
 };
 
 /* -----------------------------------------------------------------------------
diff --git a/drivers/media/platform/vsp1/vsp1_video.c b/drivers/media/platform/vsp1/vsp1_video.c
index 5af3486afe07..c2d3b8f0f487 100644
--- a/drivers/media/platform/vsp1/vsp1_video.c
+++ b/drivers/media/platform/vsp1/vsp1_video.c
@@ -182,57 +182,21 @@ static int __vsp1_video_try_format(struct vsp1_video *video,
  * VSP1 Partition Algorithm support
  */
 
-static void vsp1_video_pipeline_setup_partitions(struct vsp1_pipeline *pipe)
-{
-	struct vsp1_device *vsp1 = pipe->output->entity.vsp1;
-	const struct v4l2_mbus_framefmt *format;
-	struct vsp1_entity *entity;
-	unsigned int div_size;
-
-	/*
-	 * Partitions are computed on the size before rotation, use the format
-	 * at the WPF sink.
-	 */
-	format = vsp1_entity_get_pad_format(&pipe->output->entity,
-					    pipe->output->entity.config,
-					    RWPF_PAD_SINK);
-	div_size = format->width;
-
-	/* Gen2 hardware doesn't require image partitioning. */
-	if (vsp1->info->gen == 2) {
-		pipe->div_size = div_size;
-		pipe->partitions = 1;
-		return;
-	}
-
-	list_for_each_entry(entity, &pipe->entities, list_pipe) {
-		unsigned int entity_max = VSP1_VIDEO_MAX_WIDTH;
-
-		if (entity->ops->max_width) {
-			entity_max = entity->ops->max_width(entity, pipe);
-			if (entity_max)
-				div_size = min(div_size, entity_max);
-		}
-	}
-
-	pipe->div_size = div_size;
-	pipe->partitions = DIV_ROUND_UP(format->width, div_size);
-}
-
 /**
- * vsp1_video_partition - Calculate the active partition output window
+ * vsp1_video_calculate_partition - Calculate the active partition output window
  *
+ * @pipe: the pipeline
+ * @partition: partition that will hold the calculated values
  * @div_size: pre-determined maximum partition division size
  * @index: partition index
- *
- * Returns a v4l2_rect describing the partition window.
  */
-static struct v4l2_rect vsp1_video_partition(struct vsp1_pipeline *pipe,
-					     unsigned int div_size,
-					     unsigned int index)
+static void vsp1_video_calculate_partition(struct vsp1_pipeline *pipe,
+					   struct vsp1_partition *partition,
+					   unsigned int div_size,
+					   unsigned int index)
 {
 	const struct v4l2_mbus_framefmt *format;
-	struct v4l2_rect partition;
+	struct vsp1_partition_window window;
 	unsigned int modulus;
 
 	/*
@@ -245,18 +209,17 @@ static struct v4l2_rect vsp1_video_partition(struct vsp1_pipeline *pipe,
 
 	/* A single partition simply processes the output size in full. */
 	if (pipe->partitions <= 1) {
-		partition.left = 0;
-		partition.top = 0;
-		partition.width = format->width;
-		partition.height = format->height;
-		return partition;
+		window.left = 0;
+		window.width = format->width;
+
+		vsp1_pipeline_propagate_partition(pipe, partition, index,
+						  &window);
+		return;
 	}
 
 	/* Initialise the partition with sane starting conditions. */
-	partition.left = index * div_size;
-	partition.top = 0;
-	partition.width = div_size;
-	partition.height = format->height;
+	window.left = index * div_size;
+	window.width = div_size;
 
 	modulus = format->width % div_size;
 
@@ -279,18 +242,65 @@ static struct v4l2_rect vsp1_video_partition(struct vsp1_pipeline *pipe,
 		if (modulus < div_size / 2) {
 			if (index == partitions - 1) {
 				/* Halve the penultimate partition. */
-				partition.width = div_size / 2;
+				window.width = div_size / 2;
 			} else if (index == partitions) {
 				/* Increase the final partition. */
-				partition.width = (div_size / 2) + modulus;
-				partition.left -= div_size / 2;
+				window.width = (div_size / 2) + modulus;
+				window.left -= div_size / 2;
 			}
 		} else if (index == partitions) {
-			partition.width = modulus;
+			window.width = modulus;
 		}
 	}
 
-	return partition;
+	vsp1_pipeline_propagate_partition(pipe, partition, index, &window);
+}
+
+static int vsp1_video_pipeline_setup_partitions(struct vsp1_pipeline *pipe)
+{
+	struct vsp1_device *vsp1 = pipe->output->entity.vsp1;
+	const struct v4l2_mbus_framefmt *format;
+	struct vsp1_entity *entity;
+	unsigned int div_size;
+	unsigned int i;
+
+	/*
+	 * Partitions are computed on the size before rotation, use the format
+	 * at the WPF sink.
+	 */
+	format = vsp1_entity_get_pad_format(&pipe->output->entity,
+					    pipe->output->entity.config,
+					    RWPF_PAD_SINK);
+	div_size = format->width;
+
+	/*
+	 * Only Gen3 hardware requires image partitioning, Gen2 will operate
+	 * with a single partition that covers the whole output.
+	 */
+	if (vsp1->info->gen == 3) {
+		list_for_each_entry(entity, &pipe->entities, list_pipe) {
+			unsigned int entity_max;
+
+			if (!entity->ops->max_width)
+				continue;
+
+			entity_max = entity->ops->max_width(entity, pipe);
+			if (entity_max)
+				div_size = min(div_size, entity_max);
+		}
+	}
+
+	pipe->partitions = DIV_ROUND_UP(format->width, div_size);
+	pipe->part_table = kcalloc(pipe->partitions, sizeof(*pipe->part_table),
+				   GFP_KERNEL);
+	if (!pipe->part_table)
+		return -ENOMEM;
+
+	for (i = 0; i < pipe->partitions; ++i)
+		vsp1_video_calculate_partition(pipe, &pipe->part_table[i],
+					       div_size, i);
+
+	return 0;
 }
 
 /* -----------------------------------------------------------------------------
@@ -369,12 +379,12 @@ static void vsp1_video_frame_end(struct vsp1_pipeline *pipe,
 }
 
 static void vsp1_video_pipeline_run_partition(struct vsp1_pipeline *pipe,
-					      struct vsp1_dl_list *dl)
+					      struct vsp1_dl_list *dl,
+					      unsigned int partition)
 {
 	struct vsp1_entity *entity;
 
-	pipe->partition = vsp1_video_partition(pipe, pipe->div_size,
-					       pipe->current_partition);
+	pipe->partition = &pipe->part_table[partition];
 
 	list_for_each_entry(entity, &pipe->entities, list_pipe) {
 		if (entity->ops->configure)
@@ -387,6 +397,7 @@ static void vsp1_video_pipeline_run(struct vsp1_pipeline *pipe)
 {
 	struct vsp1_device *vsp1 = pipe->output->entity.vsp1;
 	struct vsp1_entity *entity;
+	unsigned int partition;
 
 	if (!pipe->dl)
 		pipe->dl = vsp1_dl_list_get(pipe->output->dlm);
@@ -403,20 +414,12 @@ static void vsp1_video_pipeline_run(struct vsp1_pipeline *pipe)
 	}
 
 	/* Run the first partition */
-	pipe->current_partition = 0;
-	vsp1_video_pipeline_run_partition(pipe, pipe->dl);
+	vsp1_video_pipeline_run_partition(pipe, pipe->dl, 0);
 
 	/* Process consecutive partitions as necessary */
-	for (pipe->current_partition = 1;
-	     pipe->current_partition < pipe->partitions;
-	     pipe->current_partition++) {
+	for (partition = 1; partition < pipe->partitions; ++partition) {
 		struct vsp1_dl_list *dl;
 
-		/*
-		 * Partition configuration operations will utilise
-		 * the pipe->current_partition variable to determine
-		 * the work they should complete.
-		 */
 		dl = vsp1_dl_list_get(pipe->output->dlm);
 
 		/*
@@ -429,7 +432,7 @@ static void vsp1_video_pipeline_run(struct vsp1_pipeline *pipe)
 			break;
 		}
 
-		vsp1_video_pipeline_run_partition(pipe, dl);
+		vsp1_video_pipeline_run_partition(pipe, dl, partition);
 		vsp1_dl_list_add_chain(pipe->dl, dl);
 	}
 
@@ -440,13 +443,17 @@ static void vsp1_video_pipeline_run(struct vsp1_pipeline *pipe)
 	vsp1_pipeline_run(pipe);
 }
 
-static void vsp1_video_pipeline_frame_end(struct vsp1_pipeline *pipe)
+static void vsp1_video_pipeline_frame_end(struct vsp1_pipeline *pipe,
+					  bool completed)
 {
 	struct vsp1_device *vsp1 = pipe->output->entity.vsp1;
 	enum vsp1_pipeline_state state;
 	unsigned long flags;
 	unsigned int i;
 
+	/* M2M Pipelines should never call here with an incomplete frame. */
+	WARN_ON_ONCE(!completed);
+
 	spin_lock_irqsave(&pipe->irqlock, flags);
 
 	/* Complete buffers on all video nodes. */
@@ -481,7 +488,7 @@ static int vsp1_video_pipeline_build_branch(struct vsp1_pipeline *pipe,
 	struct media_entity_enum ent_enum;
 	struct vsp1_entity *entity;
 	struct media_pad *pad;
-	bool bru_found = false;
+	struct vsp1_bru *bru = NULL;
 	int ret;
 
 	ret = media_entity_enum_init(&ent_enum, &input->entity.vsp1->media_dev);
@@ -511,16 +518,20 @@ static int vsp1_video_pipeline_build_branch(struct vsp1_pipeline *pipe,
 			media_entity_to_v4l2_subdev(pad->entity));
 
 		/*
-		 * A BRU is present in the pipeline, store the BRU input pad
+		 * A BRU or BRS is present in the pipeline, store its input pad
 		 * number in the input RPF for use when configuring the RPF.
 		 */
-		if (entity->type == VSP1_ENTITY_BRU) {
-			struct vsp1_bru *bru = to_bru(&entity->subdev);
+		if (entity->type == VSP1_ENTITY_BRU ||
+		    entity->type == VSP1_ENTITY_BRS) {
+			/* BRU and BRS can't be chained. */
+			if (bru) {
+				ret = -EPIPE;
+				goto out;
+			}
 
+			bru = to_bru(&entity->subdev);
 			bru->inputs[pad->index].rpf = input;
 			input->bru_input = pad->index;
-
-			bru_found = true;
 		}
 
 		/* We've reached the WPF, we're done. */
@@ -542,8 +553,7 @@ static int vsp1_video_pipeline_build_branch(struct vsp1_pipeline *pipe,
 			}
 
 			pipe->uds = entity;
-			pipe->uds_input = bru_found ? pipe->bru
-					: &input->entity;
+			pipe->uds_input = bru ? &bru->entity : &input->entity;
 		}
 
 		/* Follow the source link, ignoring any HGO or HGT. */
@@ -589,30 +599,42 @@ static int vsp1_video_pipeline_build(struct vsp1_pipeline *pipe,
 		e = to_vsp1_entity(subdev);
 		list_add_tail(&e->list_pipe, &pipe->entities);
 
-		if (e->type == VSP1_ENTITY_RPF) {
+		switch (e->type) {
+		case VSP1_ENTITY_RPF:
 			rwpf = to_rwpf(subdev);
 			pipe->inputs[rwpf->entity.index] = rwpf;
 			rwpf->video->pipe_index = ++pipe->num_inputs;
 			rwpf->pipe = pipe;
-		} else if (e->type == VSP1_ENTITY_WPF) {
+			break;
+
+		case VSP1_ENTITY_WPF:
 			rwpf = to_rwpf(subdev);
 			pipe->output = rwpf;
 			rwpf->video->pipe_index = 0;
 			rwpf->pipe = pipe;
-		} else if (e->type == VSP1_ENTITY_LIF) {
+			break;
+
+		case VSP1_ENTITY_LIF:
 			pipe->lif = e;
-		} else if (e->type == VSP1_ENTITY_BRU) {
+			break;
+
+		case VSP1_ENTITY_BRU:
+		case VSP1_ENTITY_BRS:
 			pipe->bru = e;
-		} else if (e->type == VSP1_ENTITY_HGO) {
-			struct vsp1_hgo *hgo = to_hgo(subdev);
+			break;
 
+		case VSP1_ENTITY_HGO:
 			pipe->hgo = e;
-			hgo->histo.pipe = pipe;
-		} else if (e->type == VSP1_ENTITY_HGT) {
-			struct vsp1_hgt *hgt = to_hgt(subdev);
+			to_hgo(subdev)->histo.pipe = pipe;
+			break;
 
+		case VSP1_ENTITY_HGT:
 			pipe->hgt = e;
-			hgt->histo.pipe = pipe;
+			to_hgt(subdev)->histo.pipe = pipe;
+			break;
+
+		default:
+			break;
 		}
 	}
 
@@ -783,9 +805,12 @@ static void vsp1_video_buffer_queue(struct vb2_buffer *vb)
 static int vsp1_video_setup_pipeline(struct vsp1_pipeline *pipe)
 {
 	struct vsp1_entity *entity;
+	int ret;
 
 	/* Determine this pipelines sizes for image partitioning support. */
-	vsp1_video_pipeline_setup_partitions(pipe);
+	ret = vsp1_video_pipeline_setup_partitions(pipe);
+	if (ret < 0)
+		return ret;
 
 	/* Prepare the display list. */
 	pipe->dl = vsp1_dl_list_get(pipe->output->dlm);
@@ -796,12 +821,14 @@ static int vsp1_video_setup_pipeline(struct vsp1_pipeline *pipe)
 		struct vsp1_uds *uds = to_uds(&pipe->uds->subdev);
 
 		/*
-		 * If a BRU is present in the pipeline before the UDS, the alpha
-		 * component doesn't need to be scaled as the BRU output alpha
-		 * value is fixed to 255. Otherwise we need to scale the alpha
-		 * component only when available at the input RPF.
+		 * If a BRU or BRS is present in the pipeline before the UDS,
+		 * the alpha component doesn't need to be scaled as the BRU and
+		 * BRS output alpha value is fixed to 255. Otherwise we need to
+		 * scale the alpha component only when available at the input
+		 * RPF.
 		 */
-		if (pipe->uds_input->type == VSP1_ENTITY_BRU) {
+		if (pipe->uds_input->type == VSP1_ENTITY_BRU ||
+		    pipe->uds_input->type == VSP1_ENTITY_BRS) {
 			uds->scale_alpha = false;
 		} else {
 			struct vsp1_rwpf *rpf =
@@ -822,6 +849,26 @@ static int vsp1_video_setup_pipeline(struct vsp1_pipeline *pipe)
 	return 0;
 }
 
+static void vsp1_video_cleanup_pipeline(struct vsp1_pipeline *pipe)
+{
+	struct vsp1_video *video = pipe->output->video;
+	struct vsp1_vb2_buffer *buffer;
+	unsigned long flags;
+
+	/* Remove all buffers from the IRQ queue. */
+	spin_lock_irqsave(&video->irqlock, flags);
+	list_for_each_entry(buffer, &video->irqqueue, queue)
+		vb2_buffer_done(&buffer->buf.vb2_buf, VB2_BUF_STATE_ERROR);
+	INIT_LIST_HEAD(&video->irqqueue);
+	spin_unlock_irqrestore(&video->irqlock, flags);
+
+	/* Release our partition table allocation */
+	mutex_lock(&pipe->lock);
+	kfree(pipe->part_table);
+	pipe->part_table = NULL;
+	mutex_unlock(&pipe->lock);
+}
+
 static int vsp1_video_start_streaming(struct vb2_queue *vq, unsigned int count)
 {
 	struct vsp1_video *video = vb2_get_drv_priv(vq);
@@ -835,6 +882,7 @@ static int vsp1_video_start_streaming(struct vb2_queue *vq, unsigned int count)
 		ret = vsp1_video_setup_pipeline(pipe);
 		if (ret < 0) {
 			mutex_unlock(&pipe->lock);
+			vsp1_video_cleanup_pipeline(pipe);
 			return ret;
 		}
 
@@ -866,7 +914,6 @@ static void vsp1_video_stop_streaming(struct vb2_queue *vq)
 {
 	struct vsp1_video *video = vb2_get_drv_priv(vq);
 	struct vsp1_pipeline *pipe = video->rwpf->pipe;
-	struct vsp1_vb2_buffer *buffer;
 	unsigned long flags;
 	int ret;
 
@@ -891,14 +938,8 @@ static void vsp1_video_stop_streaming(struct vb2_queue *vq)
 	mutex_unlock(&pipe->lock);
 
 	media_pipeline_stop(&video->video.entity);
+	vsp1_video_cleanup_pipeline(pipe);
 	vsp1_video_pipeline_put(pipe);
-
-	/* Remove all buffers from the IRQ queue. */
-	spin_lock_irqsave(&video->irqlock, flags);
-	list_for_each_entry(buffer, &video->irqqueue, queue)
-		vb2_buffer_done(&buffer->buf.vb2_buf, VB2_BUF_STATE_ERROR);
-	INIT_LIST_HEAD(&video->irqqueue);
-	spin_unlock_irqrestore(&video->irqlock, flags);
 }
 
 static const struct vb2_ops vsp1_video_queue_qops = {
diff --git a/drivers/media/platform/vsp1/vsp1_wpf.c b/drivers/media/platform/vsp1/vsp1_wpf.c
index 32df109b119f..f7f3b4b2c2de 100644
--- a/drivers/media/platform/vsp1/vsp1_wpf.c
+++ b/drivers/media/platform/vsp1/vsp1_wpf.c
@@ -291,7 +291,7 @@ static void wpf_configure(struct vsp1_entity *entity,
 		 * multiple slices.
 		 */
 		if (pipe->partitions > 1)
-			width = pipe->partition.width;
+			width = pipe->partition->wpf.width;
 
 		vsp1_wpf_write(wpf, dl, VI6_WPF_HSZCLIP, VI6_WPF_SZCLIP_EN |
 			       (0 << VI6_WPF_SZCLIP_OFST_SHIFT) |
@@ -320,13 +320,13 @@ static void wpf_configure(struct vsp1_entity *entity,
 		 * is applied horizontally or vertically accordingly.
 		 */
 		if (flip & BIT(WPF_CTRL_HFLIP) && !wpf->flip.rotate)
-			offset = format->width - pipe->partition.left
-				- pipe->partition.width;
+			offset = format->width - pipe->partition->wpf.left
+				- pipe->partition->wpf.width;
 		else if (flip & BIT(WPF_CTRL_VFLIP) && wpf->flip.rotate)
-			offset = format->height - pipe->partition.left
-				- pipe->partition.width;
+			offset = format->height - pipe->partition->wpf.left
+				- pipe->partition->wpf.width;
 		else
-			offset = pipe->partition.left;
+			offset = pipe->partition->wpf.left;
 
 		for (i = 0; i < format->num_planes; ++i) {
 			unsigned int hsub = i > 0 ? fmtinfo->hsub : 1;
@@ -348,7 +348,7 @@ static void wpf_configure(struct vsp1_entity *entity,
 			 * image height.
 			 */
 			if (wpf->flip.rotate)
-				height = pipe->partition.width;
+				height = pipe->partition->wpf.width;
 			else
 				height = format->height;
 
@@ -453,7 +453,9 @@ static void wpf_configure(struct vsp1_entity *entity,
 	}
 
 	if (pipe->bru || pipe->num_inputs > 1)
-		srcrpf |= VI6_WPF_SRCRPF_VIRACT_MST;
+		srcrpf |= pipe->bru->type == VSP1_ENTITY_BRU
+			? VI6_WPF_SRCRPF_VIRACT_MST
+			: VI6_WPF_SRCRPF_VIRACT2_MST;
 
 	vsp1_wpf_write(wpf, dl, VI6_WPF_SRCRPF, srcrpf);
 
@@ -471,10 +473,20 @@ static unsigned int wpf_max_width(struct vsp1_entity *entity,
 	return wpf->flip.rotate ? 256 : wpf->max_width;
 }
 
+static void wpf_partition(struct vsp1_entity *entity,
+			  struct vsp1_pipeline *pipe,
+			  struct vsp1_partition *partition,
+			  unsigned int partition_idx,
+			  struct vsp1_partition_window *window)
+{
+	partition->wpf = *window;
+}
+
 static const struct vsp1_entity_operations wpf_entity_ops = {
 	.destroy = vsp1_wpf_destroy,
 	.configure = wpf_configure,
 	.max_width = wpf_max_width,
+	.partition = wpf_partition,
 };
 
 /* -----------------------------------------------------------------------------
diff --git a/drivers/media/platform/xilinx/xilinx-vipp.c b/drivers/media/platform/xilinx/xilinx-vipp.c
index ac4704388920..ebfdf334d99c 100644
--- a/drivers/media/platform/xilinx/xilinx-vipp.c
+++ b/drivers/media/platform/xilinx/xilinx-vipp.c
@@ -90,12 +90,12 @@ static int xvip_graph_build_one(struct xvip_composite_device *xdev,
 		of_node_put(ep);
 		ep = next;
 
-		dev_dbg(xdev->dev, "processing endpoint %s\n", ep->full_name);
+		dev_dbg(xdev->dev, "processing endpoint %pOF\n", ep);
 
 		ret = v4l2_fwnode_parse_link(of_fwnode_handle(ep), &link);
 		if (ret < 0) {
-			dev_err(xdev->dev, "failed to parse link for %s\n",
-				ep->full_name);
+			dev_err(xdev->dev, "failed to parse link for %pOF\n",
+				ep);
 			continue;
 		}
 
@@ -103,9 +103,9 @@ static int xvip_graph_build_one(struct xvip_composite_device *xdev,
 		 * the link.
 		 */
 		if (link.local_port >= local->num_pads) {
-			dev_err(xdev->dev, "invalid port number %u for %s\n",
+			dev_err(xdev->dev, "invalid port number %u for %pOF\n",
 				link.local_port,
-				to_of_node(link.local_node)->full_name);
+				to_of_node(link.local_node));
 			v4l2_fwnode_put_link(&link);
 			ret = -EINVAL;
 			break;
@@ -114,8 +114,8 @@ static int xvip_graph_build_one(struct xvip_composite_device *xdev,
 		local_pad = &local->pads[link.local_port];
 
 		if (local_pad->flags & MEDIA_PAD_FL_SINK) {
-			dev_dbg(xdev->dev, "skipping sink port %s:%u\n",
-				to_of_node(link.local_node)->full_name,
+			dev_dbg(xdev->dev, "skipping sink port %pOF:%u\n",
+				to_of_node(link.local_node),
 				link.local_port);
 			v4l2_fwnode_put_link(&link);
 			continue;
@@ -123,8 +123,8 @@ static int xvip_graph_build_one(struct xvip_composite_device *xdev,
 
 		/* Skip DMA engines, they will be processed separately. */
 		if (link.remote_node == of_fwnode_handle(xdev->dev->of_node)) {
-			dev_dbg(xdev->dev, "skipping DMA port %s:%u\n",
-				to_of_node(link.local_node)->full_name,
+			dev_dbg(xdev->dev, "skipping DMA port %pOF:%u\n",
+				to_of_node(link.local_node),
 				link.local_port);
 			v4l2_fwnode_put_link(&link);
 			continue;
@@ -134,8 +134,8 @@ static int xvip_graph_build_one(struct xvip_composite_device *xdev,
 		ent = xvip_graph_find_entity(xdev,
 					     to_of_node(link.remote_node));
 		if (ent == NULL) {
-			dev_err(xdev->dev, "no entity found for %s\n",
-				to_of_node(link.remote_node)->full_name);
+			dev_err(xdev->dev, "no entity found for %pOF\n",
+				to_of_node(link.remote_node));
 			v4l2_fwnode_put_link(&link);
 			ret = -ENODEV;
 			break;
@@ -144,9 +144,8 @@ static int xvip_graph_build_one(struct xvip_composite_device *xdev,
 		remote = ent->entity;
 
 		if (link.remote_port >= remote->num_pads) {
-			dev_err(xdev->dev, "invalid port number %u on %s\n",
-				link.remote_port,
-				to_of_node(link.remote_node)->full_name);
+			dev_err(xdev->dev, "invalid port number %u on %pOF\n",
+				link.remote_port, to_of_node(link.remote_node));
 			v4l2_fwnode_put_link(&link);
 			ret = -EINVAL;
 			break;
@@ -216,12 +215,12 @@ static int xvip_graph_build_dma(struct xvip_composite_device *xdev)
 		of_node_put(ep);
 		ep = next;
 
-		dev_dbg(xdev->dev, "processing endpoint %s\n", ep->full_name);
+		dev_dbg(xdev->dev, "processing endpoint %pOF\n", ep);
 
 		ret = v4l2_fwnode_parse_link(of_fwnode_handle(ep), &link);
 		if (ret < 0) {
-			dev_err(xdev->dev, "failed to parse link for %s\n",
-				ep->full_name);
+			dev_err(xdev->dev, "failed to parse link for %pOF\n",
+				ep);
 			continue;
 		}
 
@@ -242,17 +241,17 @@ static int xvip_graph_build_dma(struct xvip_composite_device *xdev)
 		ent = xvip_graph_find_entity(xdev,
 					     to_of_node(link.remote_node));
 		if (ent == NULL) {
-			dev_err(xdev->dev, "no entity found for %s\n",
-				to_of_node(link.remote_node)->full_name);
+			dev_err(xdev->dev, "no entity found for %pOF\n",
+				to_of_node(link.remote_node));
 			v4l2_fwnode_put_link(&link);
 			ret = -ENODEV;
 			break;
 		}
 
 		if (link.remote_port >= ent->entity->num_pads) {
-			dev_err(xdev->dev, "invalid port number %u on %s\n",
+			dev_err(xdev->dev, "invalid port number %u on %pOF\n",
 				link.remote_port,
-				to_of_node(link.remote_node)->full_name);
+				to_of_node(link.remote_node));
 			v4l2_fwnode_put_link(&link);
 			ret = -EINVAL;
 			break;
@@ -337,8 +336,8 @@ static int xvip_graph_notify_bound(struct v4l2_async_notifier *notifier,
 			continue;
 
 		if (entity->subdev) {
-			dev_err(xdev->dev, "duplicate subdev for node %s\n",
-				entity->node->full_name);
+			dev_err(xdev->dev, "duplicate subdev for node %pOF\n",
+				entity->node);
 			return -EINVAL;
 		}
 
@@ -360,14 +359,14 @@ static int xvip_graph_parse_one(struct xvip_composite_device *xdev,
 	struct device_node *ep = NULL;
 	int ret = 0;
 
-	dev_dbg(xdev->dev, "parsing node %s\n", node->full_name);
+	dev_dbg(xdev->dev, "parsing node %pOF\n", node);
 
 	while (1) {
 		ep = of_graph_get_next_endpoint(node, ep);
 		if (ep == NULL)
 			break;
 
-		dev_dbg(xdev->dev, "handling endpoint %s\n", ep->full_name);
+		dev_dbg(xdev->dev, "handling endpoint %pOF\n", ep);
 
 		remote = of_graph_get_remote_port_parent(ep);
 		if (remote == NULL) {
@@ -452,8 +451,7 @@ static int xvip_graph_dma_init_one(struct xvip_composite_device *xdev,
 
 	ret = xvip_dma_init(xdev, dma, type, index);
 	if (ret < 0) {
-		dev_err(xdev->dev, "%s initialization failed\n",
-			node->full_name);
+		dev_err(xdev->dev, "%pOF initialization failed\n", node);
 		return ret;
 	}
 
diff --git a/drivers/media/radio/dsbr100.c b/drivers/media/radio/dsbr100.c
index 53bc8c010035..8521bb2825e8 100644
--- a/drivers/media/radio/dsbr100.c
+++ b/drivers/media/radio/dsbr100.c
@@ -408,7 +408,7 @@ err_reg_dev:
 	return retval;
 }
 
-static struct usb_device_id usb_dsbr100_device_table[] = {
+static const struct usb_device_id usb_dsbr100_device_table[] = {
 	{ USB_DEVICE(DSB100_VENDOR, DSB100_PRODUCT) },
 	{ }						/* Terminating entry */
 };
diff --git a/drivers/media/radio/radio-cadet.c b/drivers/media/radio/radio-cadet.c
index cbaf850f4791..6888b7db449d 100644
--- a/drivers/media/radio/radio-cadet.c
+++ b/drivers/media/radio/radio-cadet.c
@@ -528,7 +528,7 @@ static const struct v4l2_ctrl_ops cadet_ctrl_ops = {
 
 #ifdef CONFIG_PNP
 
-static struct pnp_device_id cadet_pnp_devices[] = {
+static const struct pnp_device_id cadet_pnp_devices[] = {
 	/* ADS Cadet AM/FM Radio Card */
 	{.id = "MSM0c24", .driver_data = 0},
 	{.id = ""}
diff --git a/drivers/media/radio/radio-gemtek.c b/drivers/media/radio/radio-gemtek.c
index ca051ccbc3e4..ddc12b16f77c 100644
--- a/drivers/media/radio/radio-gemtek.c
+++ b/drivers/media/radio/radio-gemtek.c
@@ -281,7 +281,7 @@ static const struct radio_isa_ops gemtek_ops = {
 static const int gemtek_ioports[] = { 0x20c, 0x30c, 0x24c, 0x34c, 0x248, 0x28c };
 
 #ifdef CONFIG_PNP
-static struct pnp_device_id gemtek_pnp_devices[] = {
+static const struct pnp_device_id gemtek_pnp_devices[] = {
 	/* AOpen FX-3D/Pro Radio */
 	{.id = "ADS7183", .driver_data = 0},
 	{.id = ""}
diff --git a/drivers/media/radio/radio-keene.c b/drivers/media/radio/radio-keene.c
index 53a7c2e87762..f2ea8bc5f5ee 100644
--- a/drivers/media/radio/radio-keene.c
+++ b/drivers/media/radio/radio-keene.c
@@ -45,7 +45,7 @@ MODULE_LICENSE("GPL");
 #define FREQ_MUL 16000U
 
 /* USB Device ID List */
-static struct usb_device_id usb_keene_device_table[] = {
+static const struct usb_device_id usb_keene_device_table[] = {
 	{USB_DEVICE_AND_INTERFACE_INFO(USB_KEENE_VENDOR, USB_KEENE_PRODUCT,
 							USB_CLASS_HID, 0, 0) },
 	{ }						/* Terminating entry */
diff --git a/drivers/media/radio/radio-ma901.c b/drivers/media/radio/radio-ma901.c
index c2010a905a47..fdc481257efd 100644
--- a/drivers/media/radio/radio-ma901.c
+++ b/drivers/media/radio/radio-ma901.c
@@ -444,7 +444,7 @@ err:
 }
 
 /* USB Device ID List */
-static struct usb_device_id usb_ma901radio_device_table[] = {
+static const struct usb_device_id usb_ma901radio_device_table[] = {
 	{ USB_DEVICE_AND_INTERFACE_INFO(USB_MA901_VENDOR, USB_MA901_PRODUCT,
 							USB_CLASS_HID, 0, 0) },
 	{ }						/* Terminating entry */
diff --git a/drivers/media/radio/radio-maxiradio.c b/drivers/media/radio/radio-maxiradio.c
index 8253f79d5d75..3aa5ad391581 100644
--- a/drivers/media/radio/radio-maxiradio.c
+++ b/drivers/media/radio/radio-maxiradio.c
@@ -186,7 +186,7 @@ static void maxiradio_remove(struct pci_dev *pdev)
 	kfree(dev);
 }
 
-static struct pci_device_id maxiradio_pci_tbl[] = {
+static const struct pci_device_id maxiradio_pci_tbl[] = {
 	{ PCI_VENDOR_ID_GUILLEMOT, PCI_DEVICE_ID_GUILLEMOT_MAXIRADIO,
 		PCI_ANY_ID, PCI_ANY_ID, },
 	{ 0 }
diff --git a/drivers/media/radio/radio-mr800.c b/drivers/media/radio/radio-mr800.c
index 95c12532e87a..c9f59129af79 100644
--- a/drivers/media/radio/radio-mr800.c
+++ b/drivers/media/radio/radio-mr800.c
@@ -587,7 +587,7 @@ err:
 }
 
 /* USB Device ID List */
-static struct usb_device_id usb_amradio_device_table[] = {
+static const struct usb_device_id usb_amradio_device_table[] = {
 	{ USB_DEVICE_AND_INTERFACE_INFO(USB_AMRADIO_VENDOR, USB_AMRADIO_PRODUCT,
 							USB_CLASS_HID, 0, 0) },
 	{ }						/* Terminating entry */
diff --git a/drivers/media/radio/radio-raremono.c b/drivers/media/radio/radio-raremono.c
index bfb3a6d051ba..3c0a22a54113 100644
--- a/drivers/media/radio/radio-raremono.c
+++ b/drivers/media/radio/radio-raremono.c
@@ -58,7 +58,7 @@ MODULE_LICENSE("GPL v2");
  */
 
 /* USB Device ID List */
-static struct usb_device_id usb_raremono_device_table[] = {
+static const struct usb_device_id usb_raremono_device_table[] = {
 	{USB_DEVICE_AND_INTERFACE_INFO(0x10c4, 0x818a, USB_CLASS_HID, 0, 0) },
 	{ }						/* Terminating entry */
 };
diff --git a/drivers/media/radio/radio-sf16fmr2.c b/drivers/media/radio/radio-sf16fmr2.c
index dc81d422b394..de79d5569c2a 100644
--- a/drivers/media/radio/radio-sf16fmr2.c
+++ b/drivers/media/radio/radio-sf16fmr2.c
@@ -197,7 +197,7 @@ static int fmr2_tea_ext_init(struct snd_tea575x *tea)
 	return 0;
 }
 
-static struct pnp_device_id fmr2_pnp_ids[] = {
+static const struct pnp_device_id fmr2_pnp_ids[] = {
 	{ .id = "MFRad13" }, /* tuner subdevice of SF16-FMD2 */
 	{ .id = "" }
 };
diff --git a/drivers/media/radio/radio-shark.c b/drivers/media/radio/radio-shark.c
index 23971f5502a8..22f3466af2b1 100644
--- a/drivers/media/radio/radio-shark.c
+++ b/drivers/media/radio/radio-shark.c
@@ -392,7 +392,7 @@ static int usb_shark_resume(struct usb_interface *intf)
 #endif
 
 /* Specify the bcdDevice value, as the radioSHARK and radioSHARK2 share ids */
-static struct usb_device_id usb_shark_device_table[] = {
+static const struct usb_device_id usb_shark_device_table[] = {
 	{ .match_flags = USB_DEVICE_ID_MATCH_DEVICE_AND_VERSION |
 			 USB_DEVICE_ID_MATCH_INT_CLASS,
 	  .idVendor     = 0x077d,
diff --git a/drivers/media/radio/radio-shark2.c b/drivers/media/radio/radio-shark2.c
index b50638ec5f09..4d1a4b3d669c 100644
--- a/drivers/media/radio/radio-shark2.c
+++ b/drivers/media/radio/radio-shark2.c
@@ -358,7 +358,7 @@ static int usb_shark_resume(struct usb_interface *intf)
 #endif
 
 /* Specify the bcdDevice value, as the radioSHARK and radioSHARK2 share ids */
-static struct usb_device_id usb_shark_device_table[] = {
+static const struct usb_device_id usb_shark_device_table[] = {
 	{ .match_flags = USB_DEVICE_ID_MATCH_DEVICE_AND_VERSION |
 			 USB_DEVICE_ID_MATCH_INT_CLASS,
 	  .idVendor     = 0x077d,
diff --git a/drivers/media/radio/radio-tea5764.c b/drivers/media/radio/radio-tea5764.c
index 9db8331a0c75..bc7e69e7e32e 100644
--- a/drivers/media/radio/radio-tea5764.c
+++ b/drivers/media/radio/radio-tea5764.c
@@ -414,7 +414,7 @@ static const struct v4l2_ioctl_ops tea5764_ioctl_ops = {
 };
 
 /* V4L2 interface */
-static struct video_device tea5764_radio_template = {
+static const struct video_device tea5764_radio_template = {
 	.name		= "TEA5764 FM-Radio",
 	.fops           = &tea5764_fops,
 	.ioctl_ops 	= &tea5764_ioctl_ops,
diff --git a/drivers/media/radio/radio-wl1273.c b/drivers/media/radio/radio-wl1273.c
index 17e82a9a0109..903fcd5e99c0 100644
--- a/drivers/media/radio/radio-wl1273.c
+++ b/drivers/media/radio/radio-wl1273.c
@@ -1982,7 +1982,7 @@ static const struct v4l2_ioctl_ops wl1273_ioctl_ops = {
 	.vidioc_log_status      = wl1273_fm_vidioc_log_status,
 };
 
-static struct video_device wl1273_viddev_template = {
+static const struct video_device wl1273_viddev_template = {
 	.fops			= &wl1273_fops,
 	.ioctl_ops		= &wl1273_ioctl_ops,
 	.name			= WL1273_FM_DRIVER_NAME,
diff --git a/drivers/media/radio/si470x/radio-si470x-usb.c b/drivers/media/radio/si470x/radio-si470x-usb.c
index 571f29a34bf8..c311f9951d80 100644
--- a/drivers/media/radio/si470x/radio-si470x-usb.c
+++ b/drivers/media/radio/si470x/radio-si470x-usb.c
@@ -38,7 +38,7 @@
 
 
 /* USB Device ID List */
-static struct usb_device_id si470x_usb_driver_id_table[] = {
+static const struct usb_device_id si470x_usb_driver_id_table[] = {
 	/* Silicon Labs USB FM Radio Reference Design */
 	{ USB_DEVICE_AND_INTERFACE_INFO(0x10c4, 0x818a, USB_CLASS_HID, 0, 0) },
 	/* ADS/Tech FM Radio Receiver (formerly Instant FM Music) */
diff --git a/drivers/media/radio/si4713/radio-platform-si4713.c b/drivers/media/radio/si4713/radio-platform-si4713.c
index 6f93ef1249a6..27339ec495f6 100644
--- a/drivers/media/radio/si4713/radio-platform-si4713.c
+++ b/drivers/media/radio/si4713/radio-platform-si4713.c
@@ -135,7 +135,7 @@ static struct v4l2_ioctl_ops radio_si4713_ioctl_ops = {
 };
 
 /* radio_si4713_vdev_template - video device interface */
-static struct video_device radio_si4713_vdev_template = {
+static const struct video_device radio_si4713_vdev_template = {
 	.fops			= &radio_si4713_fops,
 	.name			= "radio-si4713",
 	.release		= video_device_release_empty,
diff --git a/drivers/media/radio/si4713/radio-usb-si4713.c b/drivers/media/radio/si4713/radio-usb-si4713.c
index e5e5a1672bdb..a115db24667b 100644
--- a/drivers/media/radio/si4713/radio-usb-si4713.c
+++ b/drivers/media/radio/si4713/radio-usb-si4713.c
@@ -49,7 +49,7 @@ MODULE_LICENSE("GPL v2");
 #define USB_RESP_TIMEOUT		50000
 
 /* USB Device ID List */
-static struct usb_device_id usb_si4713_usb_device_table[] = {
+static const struct usb_device_id usb_si4713_usb_device_table[] = {
 	{USB_DEVICE_AND_INTERFACE_INFO(USB_SI4713_VENDOR, USB_SI4713_PRODUCT,
 							USB_CLASS_HID, 0, 0) },
 	{ }						/* Terminating entry */
@@ -409,7 +409,7 @@ static const struct i2c_algorithm si4713_algo = {
 
 /* This name value shows up in the sysfs filename associated
 		with this I2C adapter */
-static struct i2c_adapter si4713_i2c_adapter_template = {
+static const struct i2c_adapter si4713_i2c_adapter_template = {
 	.name   = "si4713-i2c",
 	.owner  = THIS_MODULE,
 	.algo   = &si4713_algo,
diff --git a/drivers/media/radio/wl128x/fmdrv_v4l2.c b/drivers/media/radio/wl128x/fmdrv_v4l2.c
index 71423f45c05c..fc5a7abc83d2 100644
--- a/drivers/media/radio/wl128x/fmdrv_v4l2.c
+++ b/drivers/media/radio/wl128x/fmdrv_v4l2.c
@@ -509,7 +509,7 @@ static const struct v4l2_ioctl_ops fm_drv_ioctl_ops = {
 };
 
 /* V4L2 RADIO device parent structure */
-static struct video_device fm_viddev_template = {
+static const struct video_device fm_viddev_template = {
 	.fops = &fm_drv_fops,
 	.ioctl_ops = &fm_drv_ioctl_ops,
 	.name = FM_DRV_NAME,
diff --git a/drivers/media/rc/Kconfig b/drivers/media/rc/Kconfig
index 5e83b76495f7..d9ce8ff55d0c 100644
--- a/drivers/media/rc/Kconfig
+++ b/drivers/media/rc/Kconfig
@@ -1,9 +1,20 @@
-config RC_CORE
-	tristate
-	depends on MEDIA_RC_SUPPORT
+
+menuconfig RC_CORE
+	tristate "Remote Controller support"
 	depends on INPUT
 	default y
+	---help---
+	  Enable support for Remote Controllers on Linux. This is
+	  needed in order to support several video capture adapters,
+	  standalone IR receivers/transmitters, and RF receivers.
+
+	  Enable this option if you have a video capture board even
+	  if you don't need IR, as otherwise, you may not be able to
+	  compile the driver for your adapter.
 
+	  Say Y when you have a TV or an IR device.
+
+if RC_CORE
 source "drivers/media/rc/keymaps/Kconfig"
 
 menuconfig RC_DECODERS
@@ -388,6 +399,29 @@ config IR_GPIO_CIR
 	   To compile this driver as a module, choose M here: the module will
 	   be called gpio-ir-recv.
 
+config IR_GPIO_TX
+	tristate "GPIO IR Bit Banging Transmitter"
+	depends on RC_CORE
+	depends on LIRC
+	---help---
+	   Say Y if you want to a GPIO based IR transmitter. This is a
+	   bit banging driver.
+
+	   To compile this driver as a module, choose M here: the module will
+	   be called gpio-ir-tx.
+
+config IR_PWM_TX
+	tristate "PWM IR transmitter"
+	depends on RC_CORE
+	depends on LIRC
+	depends on PWM
+	---help---
+	   Say Y if you want to use a PWM based IR transmitter. This is
+	   more power efficient than the bit banging gpio driver.
+
+	   To compile this driver as a module, choose M here: the module will
+	   be called pwm-ir-tx.
+
 config RC_ST
 	tristate "ST remote control receiver"
 	depends on RC_CORE
@@ -435,4 +469,17 @@ config IR_SIR
 	   To compile this driver as a module, choose M here: the module will
 	   be called sir-ir.
 
+config IR_ZX
+	tristate "ZTE ZX IR remote control"
+	depends on RC_CORE
+	depends on ARCH_ZX || COMPILE_TEST
+	---help---
+	   Say Y if you want to use the IR remote control available
+	   on ZTE ZX family SoCs.
+
+	   To compile this driver as a module, choose M here: the
+	   module will be called zx-irdec.
+
 endif #RC_DEVICES
+
+endif #RC_CORE
diff --git a/drivers/media/rc/Makefile b/drivers/media/rc/Makefile
index 245e2c2d0b22..9bc6a3980ed0 100644
--- a/drivers/media/rc/Makefile
+++ b/drivers/media/rc/Makefile
@@ -32,6 +32,8 @@ obj-$(CONFIG_IR_STREAMZAP) += streamzap.o
 obj-$(CONFIG_IR_WINBOND_CIR) += winbond-cir.o
 obj-$(CONFIG_RC_LOOPBACK) += rc-loopback.o
 obj-$(CONFIG_IR_GPIO_CIR) += gpio-ir-recv.o
+obj-$(CONFIG_IR_GPIO_TX) += gpio-ir-tx.o
+obj-$(CONFIG_IR_PWM_TX) += pwm-ir-tx.o
 obj-$(CONFIG_IR_IGORPLUGUSB) += igorplugusb.o
 obj-$(CONFIG_IR_IGUANA) += iguanair.o
 obj-$(CONFIG_IR_TTUSBIR) += ttusbir.o
@@ -41,3 +43,4 @@ obj-$(CONFIG_IR_IMG) += img-ir/
 obj-$(CONFIG_IR_SERIAL) += serial_ir.o
 obj-$(CONFIG_IR_SIR) += sir_ir.o
 obj-$(CONFIG_IR_MTK) += mtk-cir.o
+obj-$(CONFIG_IR_ZX) += zx-irdec.o
diff --git a/drivers/media/rc/ati_remote.c b/drivers/media/rc/ati_remote.c
index a4c6ad4f67c1..d0871d60a723 100644
--- a/drivers/media/rc/ati_remote.c
+++ b/drivers/media/rc/ati_remote.c
@@ -622,7 +622,8 @@ static void ati_remote_input_report(struct urb *urb)
 				* it would cause ghost repeats which would be a
 				* regression for this driver.
 				*/
-				rc_keydown_notimeout(ati_remote->rdev, RC_TYPE_OTHER,
+				rc_keydown_notimeout(ati_remote->rdev,
+						     RC_PROTO_OTHER,
 						     scancode, data[2]);
 				rc_keyup(ati_remote->rdev);
 			}
@@ -760,13 +761,13 @@ static void ati_remote_rc_init(struct ati_remote *ati_remote)
 	struct rc_dev *rdev = ati_remote->rdev;
 
 	rdev->priv = ati_remote;
-	rdev->allowed_protocols = RC_BIT_OTHER;
+	rdev->allowed_protocols = RC_PROTO_BIT_OTHER;
 	rdev->driver_name = "ati_remote";
 
 	rdev->open = ati_remote_rc_open;
 	rdev->close = ati_remote_rc_close;
 
-	rdev->input_name = ati_remote->rc_name;
+	rdev->device_name = ati_remote->rc_name;
 	rdev->input_phys = ati_remote->rc_phys;
 
 	usb_to_input_id(ati_remote->udev, &rdev->input_id);
diff --git a/drivers/media/rc/ene_ir.c b/drivers/media/rc/ene_ir.c
index 60da963f40dc..af7ba23e16e1 100644
--- a/drivers/media/rc/ene_ir.c
+++ b/drivers/media/rc/ene_ir.c
@@ -1053,14 +1053,14 @@ static int ene_probe(struct pnp_dev *pnp_dev, const struct pnp_device_id *id)
 	if (!dev->hw_learning_and_tx_capable)
 		learning_mode_force = false;
 
-	rdev->allowed_protocols = RC_BIT_ALL_IR_DECODER;
+	rdev->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
 	rdev->priv = dev;
 	rdev->open = ene_open;
 	rdev->close = ene_close;
 	rdev->s_idle = ene_set_idle;
 	rdev->driver_name = ENE_DRIVER_NAME;
 	rdev->map_name = RC_MAP_RC6_MCE;
-	rdev->input_name = "ENE eHome Infrared Remote Receiver";
+	rdev->device_name = "ENE eHome Infrared Remote Receiver";
 
 	if (dev->hw_learning_and_tx_capable) {
 		rdev->s_learning_mode = ene_set_learning_mode;
@@ -1070,7 +1070,7 @@ static int ene_probe(struct pnp_dev *pnp_dev, const struct pnp_device_id *id)
 		rdev->s_tx_carrier = ene_set_tx_carrier;
 		rdev->s_tx_duty_cycle = ene_set_tx_duty_cycle;
 		rdev->s_carrier_report = ene_set_carrier_report;
-		rdev->input_name = "ENE eHome Infrared Remote Transceiver";
+		rdev->device_name = "ENE eHome Infrared Remote Transceiver";
 	}
 
 	dev->rdev = rdev;
diff --git a/drivers/media/rc/fintek-cir.c b/drivers/media/rc/fintek-cir.c
index 0d3562712f27..f2639d0c2fca 100644
--- a/drivers/media/rc/fintek-cir.c
+++ b/drivers/media/rc/fintek-cir.c
@@ -529,10 +529,10 @@ static int fintek_probe(struct pnp_dev *pdev, const struct pnp_device_id *dev_id
 
 	/* Set up the rc device */
 	rdev->priv = fintek;
-	rdev->allowed_protocols = RC_BIT_ALL_IR_DECODER;
+	rdev->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
 	rdev->open = fintek_open;
 	rdev->close = fintek_close;
-	rdev->input_name = FINTEK_DESCRIPTION;
+	rdev->device_name = FINTEK_DESCRIPTION;
 	rdev->input_phys = "fintek/cir0";
 	rdev->input_id.bustype = BUS_HOST;
 	rdev->input_id.vendor = VENDOR_ID_FINTEK;
diff --git a/drivers/media/rc/gpio-ir-recv.c b/drivers/media/rc/gpio-ir-recv.c
index b4f773b9dc1d..7248b3662285 100644
--- a/drivers/media/rc/gpio-ir-recv.c
+++ b/drivers/media/rc/gpio-ir-recv.c
@@ -30,7 +30,6 @@ struct gpio_rc_dev {
 	struct rc_dev *rcdev;
 	int gpio_nr;
 	bool active_low;
-	struct timer_list flush_timer;
 };
 
 #ifdef CONFIG_OF
@@ -77,7 +76,6 @@ static irqreturn_t gpio_ir_recv_irq(int irq, void *dev_id)
 	struct gpio_rc_dev *gpio_dev = dev_id;
 	int gval;
 	int rc = 0;
-	enum raw_event_type type = IR_SPACE;
 
 	gval = gpio_get_value(gpio_dev->gpio_nr);
 
@@ -87,33 +85,14 @@ static irqreturn_t gpio_ir_recv_irq(int irq, void *dev_id)
 	if (gpio_dev->active_low)
 		gval = !gval;
 
-	if (gval == 1)
-		type = IR_PULSE;
-
-	rc = ir_raw_event_store_edge(gpio_dev->rcdev, type);
+	rc = ir_raw_event_store_edge(gpio_dev->rcdev, gval == 1);
 	if (rc < 0)
 		goto err_get_value;
 
-	mod_timer(&gpio_dev->flush_timer,
-		  jiffies + nsecs_to_jiffies(gpio_dev->rcdev->timeout));
-
-	ir_raw_event_handle(gpio_dev->rcdev);
-
 err_get_value:
 	return IRQ_HANDLED;
 }
 
-static void flush_timer(unsigned long arg)
-{
-	struct gpio_rc_dev *gpio_dev = (struct gpio_rc_dev *)arg;
-	DEFINE_IR_RAW_EVENT(ev);
-
-	ev.timeout = true;
-	ev.duration = gpio_dev->rcdev->timeout;
-	ir_raw_event_store(gpio_dev->rcdev, &ev);
-	ir_raw_event_handle(gpio_dev->rcdev);
-}
-
 static int gpio_ir_recv_probe(struct platform_device *pdev)
 {
 	struct gpio_rc_dev *gpio_dev;
@@ -150,7 +129,7 @@ static int gpio_ir_recv_probe(struct platform_device *pdev)
 	}
 
 	rcdev->priv = gpio_dev;
-	rcdev->input_name = GPIO_IR_DEVICE_NAME;
+	rcdev->device_name = GPIO_IR_DEVICE_NAME;
 	rcdev->input_phys = GPIO_IR_DEVICE_NAME "/input0";
 	rcdev->input_id.bustype = BUS_HOST;
 	rcdev->input_id.vendor = 0x0001;
@@ -164,16 +143,13 @@ static int gpio_ir_recv_probe(struct platform_device *pdev)
 	if (pdata->allowed_protos)
 		rcdev->allowed_protocols = pdata->allowed_protos;
 	else
-		rcdev->allowed_protocols = RC_BIT_ALL_IR_DECODER;
+		rcdev->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
 	rcdev->map_name = pdata->map_name ?: RC_MAP_EMPTY;
 
 	gpio_dev->rcdev = rcdev;
 	gpio_dev->gpio_nr = pdata->gpio_nr;
 	gpio_dev->active_low = pdata->active_low;
 
-	setup_timer(&gpio_dev->flush_timer, flush_timer,
-		    (unsigned long)gpio_dev);
-
 	rc = gpio_request(pdata->gpio_nr, "gpio-ir-recv");
 	if (rc < 0)
 		goto err_gpio_request;
@@ -216,7 +192,6 @@ static int gpio_ir_recv_remove(struct platform_device *pdev)
 	struct gpio_rc_dev *gpio_dev = platform_get_drvdata(pdev);
 
 	free_irq(gpio_to_irq(gpio_dev->gpio_nr), gpio_dev);
-	del_timer_sync(&gpio_dev->flush_timer);
 	rc_unregister_device(gpio_dev->rcdev);
 	gpio_free(gpio_dev->gpio_nr);
 	kfree(gpio_dev);
diff --git a/drivers/media/rc/gpio-ir-tx.c b/drivers/media/rc/gpio-ir-tx.c
new file mode 100644
index 000000000000..cd476cab9782
--- /dev/null
+++ b/drivers/media/rc/gpio-ir-tx.c
@@ -0,0 +1,176 @@
+/*
+ * Copyright (C) 2017 Sean Young <sean@mess.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/gpio/consumer.h>
+#include <linux/delay.h>
+#include <linux/slab.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <media/rc-core.h>
+
+#define DRIVER_NAME	"gpio-ir-tx"
+#define DEVICE_NAME	"GPIO IR Bit Banging Transmitter"
+
+struct gpio_ir {
+	struct gpio_desc *gpio;
+	unsigned int carrier;
+	unsigned int duty_cycle;
+	/* we need a spinlock to hold the cpu while transmitting */
+	spinlock_t lock;
+};
+
+static const struct of_device_id gpio_ir_tx_of_match[] = {
+	{ .compatible = "gpio-ir-tx", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, gpio_ir_tx_of_match);
+
+static int gpio_ir_tx_set_duty_cycle(struct rc_dev *dev, u32 duty_cycle)
+{
+	struct gpio_ir *gpio_ir = dev->priv;
+
+	gpio_ir->duty_cycle = duty_cycle;
+
+	return 0;
+}
+
+static int gpio_ir_tx_set_carrier(struct rc_dev *dev, u32 carrier)
+{
+	struct gpio_ir *gpio_ir = dev->priv;
+
+	if (!carrier)
+		return -EINVAL;
+
+	gpio_ir->carrier = carrier;
+
+	return 0;
+}
+
+static int gpio_ir_tx(struct rc_dev *dev, unsigned int *txbuf,
+		      unsigned int count)
+{
+	struct gpio_ir *gpio_ir = dev->priv;
+	unsigned long flags;
+	ktime_t edge;
+	/*
+	 * delta should never exceed 0.5 seconds (IR_MAX_DURATION) and on
+	 * m68k ndelay(s64) does not compile; so use s32 rather than s64.
+	 */
+	s32 delta;
+	int i;
+	unsigned int pulse, space;
+
+	/* Ensure the dividend fits into 32 bit */
+	pulse = DIV_ROUND_CLOSEST(gpio_ir->duty_cycle * (NSEC_PER_SEC / 100),
+				  gpio_ir->carrier);
+	space = DIV_ROUND_CLOSEST((100 - gpio_ir->duty_cycle) *
+				  (NSEC_PER_SEC / 100), gpio_ir->carrier);
+
+	spin_lock_irqsave(&gpio_ir->lock, flags);
+
+	edge = ktime_get();
+
+	for (i = 0; i < count; i++) {
+		if (i % 2) {
+			// space
+			edge = ktime_add_us(edge, txbuf[i]);
+			delta = ktime_us_delta(edge, ktime_get());
+			if (delta > 10) {
+				spin_unlock_irqrestore(&gpio_ir->lock, flags);
+				usleep_range(delta, delta + 10);
+				spin_lock_irqsave(&gpio_ir->lock, flags);
+			} else if (delta > 0) {
+				udelay(delta);
+			}
+		} else {
+			// pulse
+			ktime_t last = ktime_add_us(edge, txbuf[i]);
+
+			while (ktime_before(ktime_get(), last)) {
+				gpiod_set_value(gpio_ir->gpio, 1);
+				edge = ktime_add_ns(edge, pulse);
+				delta = ktime_to_ns(ktime_sub(edge,
+							      ktime_get()));
+				if (delta > 0)
+					ndelay(delta);
+				gpiod_set_value(gpio_ir->gpio, 0);
+				edge = ktime_add_ns(edge, space);
+				delta = ktime_to_ns(ktime_sub(edge,
+							      ktime_get()));
+				if (delta > 0)
+					ndelay(delta);
+			}
+
+			edge = last;
+		}
+	}
+
+	spin_unlock_irqrestore(&gpio_ir->lock, flags);
+
+	return count;
+}
+
+static int gpio_ir_tx_probe(struct platform_device *pdev)
+{
+	struct gpio_ir *gpio_ir;
+	struct rc_dev *rcdev;
+	int rc;
+
+	gpio_ir = devm_kmalloc(&pdev->dev, sizeof(*gpio_ir), GFP_KERNEL);
+	if (!gpio_ir)
+		return -ENOMEM;
+
+	rcdev = devm_rc_allocate_device(&pdev->dev, RC_DRIVER_IR_RAW_TX);
+	if (!rcdev)
+		return -ENOMEM;
+
+	gpio_ir->gpio = devm_gpiod_get(&pdev->dev, NULL, GPIOD_OUT_LOW);
+	if (IS_ERR(gpio_ir->gpio)) {
+		if (PTR_ERR(gpio_ir->gpio) != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "Failed to get gpio (%ld)\n",
+				PTR_ERR(gpio_ir->gpio));
+		return PTR_ERR(gpio_ir->gpio);
+	}
+
+	rcdev->priv = gpio_ir;
+	rcdev->driver_name = DRIVER_NAME;
+	rcdev->device_name = DEVICE_NAME;
+	rcdev->tx_ir = gpio_ir_tx;
+	rcdev->s_tx_duty_cycle = gpio_ir_tx_set_duty_cycle;
+	rcdev->s_tx_carrier = gpio_ir_tx_set_carrier;
+
+	gpio_ir->carrier = 38000;
+	gpio_ir->duty_cycle = 50;
+	spin_lock_init(&gpio_ir->lock);
+
+	rc = devm_rc_register_device(&pdev->dev, rcdev);
+	if (rc < 0)
+		dev_err(&pdev->dev, "failed to register rc device\n");
+
+	return rc;
+}
+
+static struct platform_driver gpio_ir_tx_driver = {
+	.probe	= gpio_ir_tx_probe,
+	.driver = {
+		.name	= DRIVER_NAME,
+		.of_match_table = of_match_ptr(gpio_ir_tx_of_match),
+	},
+};
+module_platform_driver(gpio_ir_tx_driver);
+
+MODULE_DESCRIPTION("GPIO IR Bit Banging Transmitter");
+MODULE_AUTHOR("Sean Young <sean@mess.org>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/rc/igorplugusb.c b/drivers/media/rc/igorplugusb.c
index cb6d4f1247da..a5ea86be8f44 100644
--- a/drivers/media/rc/igorplugusb.c
+++ b/drivers/media/rc/igorplugusb.c
@@ -194,7 +194,7 @@ static int igorplugusb_probe(struct usb_interface *intf,
 	if (!rc)
 		goto fail;
 
-	rc->input_name = DRIVER_DESC;
+	rc->device_name = DRIVER_DESC;
 	rc->input_phys = ir->phys;
 	usb_to_input_id(udev, &rc->input_id);
 	rc->dev.parent = &intf->dev;
@@ -202,10 +202,11 @@ static int igorplugusb_probe(struct usb_interface *intf,
 	 * This device can only store 36 pulses + spaces, which is not enough
 	 * for the NEC protocol and many others.
 	 */
-	rc->allowed_protocols = RC_BIT_ALL_IR_DECODER & ~(RC_BIT_NEC |
-			RC_BIT_NECX | RC_BIT_NEC32 | RC_BIT_RC6_6A_20 |
-			RC_BIT_RC6_6A_24 | RC_BIT_RC6_6A_32 | RC_BIT_RC6_MCE |
-			RC_BIT_SONY20 | RC_BIT_SANYO);
+	rc->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER &
+		~(RC_PROTO_BIT_NEC | RC_PROTO_BIT_NECX | RC_PROTO_BIT_NEC32 |
+		  RC_PROTO_BIT_RC6_6A_20 | RC_PROTO_BIT_RC6_6A_24 |
+		  RC_PROTO_BIT_RC6_6A_32 | RC_PROTO_BIT_RC6_MCE |
+		  RC_PROTO_BIT_SONY20 | RC_PROTO_BIT_SANYO);
 
 	rc->priv = ir;
 	rc->driver_name = DRIVER_NAME;
diff --git a/drivers/media/rc/iguanair.c b/drivers/media/rc/iguanair.c
index 8711a7ff55cc..30e24da67226 100644
--- a/drivers/media/rc/iguanair.c
+++ b/drivers/media/rc/iguanair.c
@@ -487,11 +487,11 @@ static int iguanair_probe(struct usb_interface *intf,
 
 	usb_make_path(ir->udev, ir->phys, sizeof(ir->phys));
 
-	rc->input_name = ir->name;
+	rc->device_name = ir->name;
 	rc->input_phys = ir->phys;
 	usb_to_input_id(ir->udev, &rc->input_id);
 	rc->dev.parent = &intf->dev;
-	rc->allowed_protocols = RC_BIT_ALL_IR_DECODER;
+	rc->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
 	rc->priv = ir;
 	rc->open = iguanair_open;
 	rc->close = iguanair_close;
diff --git a/drivers/media/rc/img-ir/img-ir-hw.c b/drivers/media/rc/img-ir/img-ir-hw.c
index 8d1439622533..82fdf4cc0824 100644
--- a/drivers/media/rc/img-ir/img-ir-hw.c
+++ b/drivers/media/rc/img-ir/img-ir-hw.c
@@ -589,7 +589,7 @@ static void img_ir_set_decoder(struct img_ir_priv *priv,
 	/* clear the wakeup scancode filter */
 	rdev->scancode_wakeup_filter.data = 0;
 	rdev->scancode_wakeup_filter.mask = 0;
-	rdev->wakeup_protocol = RC_TYPE_UNKNOWN;
+	rdev->wakeup_protocol = RC_PROTO_UNKNOWN;
 
 	/* clear raw filters */
 	_img_ir_set_filter(priv, NULL);
@@ -823,7 +823,7 @@ static void img_ir_handle_data(struct img_ir_priv *priv, u32 len, u64 raw)
 	int ret = IMG_IR_SCANCODE;
 	struct img_ir_scancode_req request;
 
-	request.protocol = RC_TYPE_UNKNOWN;
+	request.protocol = RC_PROTO_UNKNOWN;
 	request.toggle   = 0;
 
 	if (dec->scancode)
@@ -1083,7 +1083,7 @@ int img_ir_probe_hw(struct img_ir_priv *priv)
 	rdev->priv = priv;
 	rdev->map_name = RC_MAP_EMPTY;
 	rdev->allowed_protocols = img_ir_allowed_protos(priv);
-	rdev->input_name = "IMG Infrared Decoder";
+	rdev->device_name = "IMG Infrared Decoder";
 	rdev->s_filter = img_ir_set_normal_filter;
 	rdev->s_wakeup_filter = img_ir_set_wakeup_filter;
 
diff --git a/drivers/media/rc/img-ir/img-ir-hw.h b/drivers/media/rc/img-ir/img-ir-hw.h
index 91a297731661..58b68dd6c67d 100644
--- a/drivers/media/rc/img-ir/img-ir-hw.h
+++ b/drivers/media/rc/img-ir/img-ir-hw.h
@@ -135,13 +135,13 @@ struct img_ir_timing_regvals {
 /**
  * struct img_ir_scancode_req - Scancode request data.
  * @protocol:	Protocol code of received message (defaults to
- *		RC_TYPE_UNKNOWN).
+ *		RC_PROTO_UNKNOWN).
  * @scancode:	Scan code of received message (must be written by
  *		handler if IMG_IR_SCANCODE is returned).
  * @toggle:	Toggle bit (defaults to 0).
  */
 struct img_ir_scancode_req {
-	enum rc_type protocol;
+	enum rc_proto protocol;
 	u32 scancode;
 	u8 toggle;
 };
diff --git a/drivers/media/rc/img-ir/img-ir-jvc.c b/drivers/media/rc/img-ir/img-ir-jvc.c
index d3e2fc0bcfe1..4b07c76fbe1b 100644
--- a/drivers/media/rc/img-ir/img-ir-jvc.c
+++ b/drivers/media/rc/img-ir/img-ir-jvc.c
@@ -23,7 +23,7 @@ static int img_ir_jvc_scancode(int len, u64 raw, u64 enabled_protocols,
 	cust = (raw >> 0) & 0xff;
 	data = (raw >> 8) & 0xff;
 
-	request->protocol = RC_TYPE_JVC;
+	request->protocol = RC_PROTO_JVC;
 	request->scancode = cust << 8 | data;
 	return IMG_IR_SCANCODE;
 }
@@ -52,7 +52,7 @@ static int img_ir_jvc_filter(const struct rc_scancode_filter *in,
  *          http://support.jvc.com/consumer/support/documents/RemoteCodes.pdf
  */
 struct img_ir_decoder img_ir_jvc = {
-	.type = RC_BIT_JVC,
+	.type = RC_PROTO_BIT_JVC,
 	.control = {
 		.decoden = 1,
 		.code_type = IMG_IR_CODETYPE_PULSEDIST,
diff --git a/drivers/media/rc/img-ir/img-ir-nec.c b/drivers/media/rc/img-ir/img-ir-nec.c
index 044fd42b22a0..2fc0678ad2d7 100644
--- a/drivers/media/rc/img-ir/img-ir-nec.c
+++ b/drivers/media/rc/img-ir/img-ir-nec.c
@@ -35,20 +35,20 @@ static int img_ir_nec_scancode(int len, u64 raw, u64 enabled_protocols,
 				bitrev8(addr_inv) << 16 |
 				bitrev8(data)     <<  8 |
 				bitrev8(data_inv);
-		request->protocol = RC_TYPE_NEC32;
+		request->protocol = RC_PROTO_NEC32;
 	} else if ((addr_inv ^ addr) != 0xff) {
 		/* Extended NEC */
 		/* scan encoding: AAaaDD */
 		request->scancode = addr     << 16 |
 				addr_inv <<  8 |
 				data;
-		request->protocol = RC_TYPE_NECX;
+		request->protocol = RC_PROTO_NECX;
 	} else {
 		/* Normal NEC */
 		/* scan encoding: AADD */
 		request->scancode = addr << 8 |
 				data;
-		request->protocol = RC_TYPE_NEC;
+		request->protocol = RC_PROTO_NEC;
 	}
 	return IMG_IR_SCANCODE;
 }
@@ -63,7 +63,7 @@ static int img_ir_nec_filter(const struct rc_scancode_filter *in,
 	data       = in->data & 0xff;
 	data_m     = in->mask & 0xff;
 
-	protocols &= RC_BIT_NEC | RC_BIT_NECX | RC_BIT_NEC32;
+	protocols &= RC_PROTO_BIT_NEC | RC_PROTO_BIT_NECX | RC_PROTO_BIT_NEC32;
 
 	/*
 	 * If only one bit is set, we were requested to do an exact
@@ -72,14 +72,14 @@ static int img_ir_nec_filter(const struct rc_scancode_filter *in,
 	 */
 	if (!is_power_of_2(protocols)) {
 		if ((in->data | in->mask) & 0xff000000)
-			protocols = RC_BIT_NEC32;
+			protocols = RC_PROTO_BIT_NEC32;
 		else if ((in->data | in->mask) & 0x00ff0000)
-			protocols = RC_BIT_NECX;
+			protocols = RC_PROTO_BIT_NECX;
 		else
-			protocols = RC_BIT_NEC;
+			protocols = RC_PROTO_BIT_NEC;
 	}
 
-	if (protocols == RC_BIT_NEC32) {
+	if (protocols == RC_PROTO_BIT_NEC32) {
 		/* 32-bit NEC (used by Apple and TiVo remotes) */
 		/* scan encoding: as transmitted, MSBit = first received bit */
 		addr       = bitrev8(in->data >> 24);
@@ -90,7 +90,7 @@ static int img_ir_nec_filter(const struct rc_scancode_filter *in,
 		data_m     = bitrev8(in->mask >>  8);
 		data_inv   = bitrev8(in->data >>  0);
 		data_inv_m = bitrev8(in->mask >>  0);
-	} else if (protocols == RC_BIT_NECX) {
+	} else if (protocols == RC_PROTO_BIT_NECX) {
 		/* Extended NEC */
 		/* scan encoding AAaaDD */
 		addr       = (in->data >> 16) & 0xff;
@@ -128,7 +128,7 @@ static int img_ir_nec_filter(const struct rc_scancode_filter *in,
  *        http://wiki.altium.com/display/ADOH/NEC+Infrared+Transmission+Protocol
  */
 struct img_ir_decoder img_ir_nec = {
-	.type = RC_BIT_NEC | RC_BIT_NECX | RC_BIT_NEC32,
+	.type = RC_PROTO_BIT_NEC | RC_PROTO_BIT_NECX | RC_PROTO_BIT_NEC32,
 	.control = {
 		.decoden = 1,
 		.code_type = IMG_IR_CODETYPE_PULSEDIST,
diff --git a/drivers/media/rc/img-ir/img-ir-raw.c b/drivers/media/rc/img-ir/img-ir-raw.c
index 8d2f8e2006e7..64714efc1145 100644
--- a/drivers/media/rc/img-ir/img-ir-raw.c
+++ b/drivers/media/rc/img-ir/img-ir-raw.c
@@ -40,9 +40,9 @@ static void img_ir_refresh_raw(struct img_ir_priv *priv, u32 irq_status)
 
 	/* report the edge to the IR raw decoders */
 	if (ir_status) /* low */
-		ir_raw_event_store_edge(rc_dev, IR_SPACE);
+		ir_raw_event_store_edge(rc_dev, false);
 	else /* high */
-		ir_raw_event_store_edge(rc_dev, IR_PULSE);
+		ir_raw_event_store_edge(rc_dev, true);
 	ir_raw_event_handle(rc_dev);
 }
 
@@ -117,7 +117,7 @@ int img_ir_probe_raw(struct img_ir_priv *priv)
 	}
 	rdev->priv = priv;
 	rdev->map_name = RC_MAP_EMPTY;
-	rdev->input_name = "IMG Infrared Decoder Raw";
+	rdev->device_name = "IMG Infrared Decoder Raw";
 
 	/* Register raw decoder */
 	error = rc_register_device(rdev);
diff --git a/drivers/media/rc/img-ir/img-ir-rc5.c b/drivers/media/rc/img-ir/img-ir-rc5.c
index a8a28a377eee..a1bc8705472b 100644
--- a/drivers/media/rc/img-ir/img-ir-rc5.c
+++ b/drivers/media/rc/img-ir/img-ir-rc5.c
@@ -33,7 +33,7 @@ static int img_ir_rc5_scancode(int len, u64 raw, u64 enabled_protocols,
 	if (!start)
 		return -EINVAL;
 
-	request->protocol = RC_TYPE_RC5;
+	request->protocol = RC_PROTO_RC5;
 	request->scancode = addr << 8 | cmd;
 	request->toggle   = tgl;
 	return IMG_IR_SCANCODE;
@@ -52,7 +52,7 @@ static int img_ir_rc5_filter(const struct rc_scancode_filter *in,
  * see http://www.sbprojects.com/knowledge/ir/rc5.php
  */
 struct img_ir_decoder img_ir_rc5 = {
-	.type      = RC_BIT_RC5,
+	.type      = RC_PROTO_BIT_RC5,
 	.control   = {
 		.bitoriend2	= 1,
 		.code_type	= IMG_IR_CODETYPE_BIPHASE,
diff --git a/drivers/media/rc/img-ir/img-ir-rc6.c b/drivers/media/rc/img-ir/img-ir-rc6.c
index de1e27534968..5f34f59ca257 100644
--- a/drivers/media/rc/img-ir/img-ir-rc6.c
+++ b/drivers/media/rc/img-ir/img-ir-rc6.c
@@ -54,7 +54,7 @@ static int img_ir_rc6_scancode(int len, u64 raw, u64 enabled_protocols,
 	if (mode)
 		return -EINVAL;
 
-	request->protocol = RC_TYPE_RC6_0;
+	request->protocol = RC_PROTO_RC6_0;
 	request->scancode = addr << 8 | cmd;
 	request->toggle	  = trl2;
 	return IMG_IR_SCANCODE;
@@ -73,7 +73,7 @@ static int img_ir_rc6_filter(const struct rc_scancode_filter *in,
  * see http://www.sbprojects.com/knowledge/ir/rc6.php
  */
 struct img_ir_decoder img_ir_rc6 = {
-	.type		= RC_BIT_RC6_0,
+	.type		= RC_PROTO_BIT_RC6_0,
 	.control	= {
 		.bitorien	= 1,
 		.code_type	= IMG_IR_CODETYPE_BIPHASE,
diff --git a/drivers/media/rc/img-ir/img-ir-sanyo.c b/drivers/media/rc/img-ir/img-ir-sanyo.c
index f394994ffc22..55a755bb437c 100644
--- a/drivers/media/rc/img-ir/img-ir-sanyo.c
+++ b/drivers/media/rc/img-ir/img-ir-sanyo.c
@@ -44,7 +44,7 @@ static int img_ir_sanyo_scancode(int len, u64 raw, u64 enabled_protocols,
 		return -EINVAL;
 
 	/* Normal Sanyo */
-	request->protocol = RC_TYPE_SANYO;
+	request->protocol = RC_PROTO_SANYO;
 	request->scancode = addr << 8 | data;
 	return IMG_IR_SCANCODE;
 }
@@ -80,7 +80,7 @@ static int img_ir_sanyo_filter(const struct rc_scancode_filter *in,
 
 /* Sanyo decoder */
 struct img_ir_decoder img_ir_sanyo = {
-	.type = RC_BIT_SANYO,
+	.type = RC_PROTO_BIT_SANYO,
 	.control = {
 		.decoden = 1,
 		.code_type = IMG_IR_CODETYPE_PULSEDIST,
diff --git a/drivers/media/rc/img-ir/img-ir-sharp.c b/drivers/media/rc/img-ir/img-ir-sharp.c
index fe5acc4f030e..2d2530902cfa 100644
--- a/drivers/media/rc/img-ir/img-ir-sharp.c
+++ b/drivers/media/rc/img-ir/img-ir-sharp.c
@@ -32,7 +32,7 @@ static int img_ir_sharp_scancode(int len, u64 raw, u64 enabled_protocols,
 		/* probably the second half of the message */
 		return -EINVAL;
 
-	request->protocol = RC_TYPE_SHARP;
+	request->protocol = RC_PROTO_SHARP;
 	request->scancode = addr << 8 | cmd;
 	return IMG_IR_SCANCODE;
 }
@@ -73,7 +73,7 @@ static int img_ir_sharp_filter(const struct rc_scancode_filter *in,
  * See also http://www.sbprojects.com/knowledge/ir/sharp.php
  */
 struct img_ir_decoder img_ir_sharp = {
-	.type = RC_BIT_SHARP,
+	.type = RC_PROTO_BIT_SHARP,
 	.control = {
 		.decoden = 0,
 		.decodend2 = 1,
diff --git a/drivers/media/rc/img-ir/img-ir-sony.c b/drivers/media/rc/img-ir/img-ir-sony.c
index 3fcba271a419..a942d0be908c 100644
--- a/drivers/media/rc/img-ir/img-ir-sony.c
+++ b/drivers/media/rc/img-ir/img-ir-sony.c
@@ -19,32 +19,32 @@ static int img_ir_sony_scancode(int len, u64 raw, u64 enabled_protocols,
 
 	switch (len) {
 	case 12:
-		if (!(enabled_protocols & RC_BIT_SONY12))
+		if (!(enabled_protocols & RC_PROTO_BIT_SONY12))
 			return -EINVAL;
 		func   = raw & 0x7f;	/* first 7 bits */
 		raw    >>= 7;
 		dev    = raw & 0x1f;	/* next 5 bits */
 		subdev = 0;
-		request->protocol = RC_TYPE_SONY12;
+		request->protocol = RC_PROTO_SONY12;
 		break;
 	case 15:
-		if (!(enabled_protocols & RC_BIT_SONY15))
+		if (!(enabled_protocols & RC_PROTO_BIT_SONY15))
 			return -EINVAL;
 		func   = raw & 0x7f;	/* first 7 bits */
 		raw    >>= 7;
 		dev    = raw & 0xff;	/* next 8 bits */
 		subdev = 0;
-		request->protocol = RC_TYPE_SONY15;
+		request->protocol = RC_PROTO_SONY15;
 		break;
 	case 20:
-		if (!(enabled_protocols & RC_BIT_SONY20))
+		if (!(enabled_protocols & RC_PROTO_BIT_SONY20))
 			return -EINVAL;
 		func   = raw & 0x7f;	/* first 7 bits */
 		raw    >>= 7;
 		dev    = raw & 0x1f;	/* next 5 bits */
 		raw    >>= 5;
 		subdev = raw & 0xff;	/* next 8 bits */
-		request->protocol = RC_TYPE_SONY20;
+		request->protocol = RC_PROTO_SONY20;
 		break;
 	default:
 		return -EINVAL;
@@ -68,7 +68,8 @@ static int img_ir_sony_filter(const struct rc_scancode_filter *in,
 	func     = (in->data >> 0)  & 0x7f;
 	func_m   = (in->mask >> 0)  & 0x7f;
 
-	protocols &= RC_BIT_SONY12 | RC_BIT_SONY15 | RC_BIT_SONY20;
+	protocols &= RC_PROTO_BIT_SONY12 | RC_PROTO_BIT_SONY15 |
+							RC_PROTO_BIT_SONY20;
 
 	/*
 	 * If only one bit is set, we were requested to do an exact
@@ -77,20 +78,20 @@ static int img_ir_sony_filter(const struct rc_scancode_filter *in,
 	 */
 	if (!is_power_of_2(protocols)) {
 		if (subdev & subdev_m)
-			protocols = RC_BIT_SONY20;
+			protocols = RC_PROTO_BIT_SONY20;
 		else if (dev & dev_m & 0xe0)
-			protocols = RC_BIT_SONY15;
+			protocols = RC_PROTO_BIT_SONY15;
 		else
-			protocols = RC_BIT_SONY12;
+			protocols = RC_PROTO_BIT_SONY12;
 	}
 
-	if (protocols == RC_BIT_SONY20) {
+	if (protocols == RC_PROTO_BIT_SONY20) {
 		/* can't encode subdev and higher device bits */
 		if (dev & dev_m & 0xe0)
 			return -EINVAL;
 		len = 20;
 		dev_m &= 0x1f;
-	} else if (protocols == RC_BIT_SONY15) {
+	} else if (protocols == RC_PROTO_BIT_SONY15) {
 		len = 15;
 		subdev_m = 0;
 	} else {
@@ -128,7 +129,7 @@ static int img_ir_sony_filter(const struct rc_scancode_filter *in,
  *          http://picprojects.org.uk/projects/sirc/sonysirc.pdf
  */
 struct img_ir_decoder img_ir_sony = {
-	.type = RC_BIT_SONY12 | RC_BIT_SONY15 | RC_BIT_SONY20,
+	.type = RC_PROTO_BIT_SONY12 | RC_PROTO_BIT_SONY15 | RC_PROTO_BIT_SONY20,
 	.control = {
 		.decoden = 1,
 		.code_type = IMG_IR_CODETYPE_PULSELEN,
diff --git a/drivers/media/rc/imon.c b/drivers/media/rc/imon.c
index bd76534a2749..7b3f31cc63d2 100644
--- a/drivers/media/rc/imon.c
+++ b/drivers/media/rc/imon.c
@@ -148,7 +148,7 @@ struct imon_context {
 	u32 last_keycode;		/* last reported input keycode */
 	u32 rc_scancode;		/* the computed remote scancode */
 	u8 rc_toggle;			/* the computed remote toggle bit */
-	u64 rc_type;			/* iMON or MCE (RC6) IR protocol? */
+	u64 rc_proto;			/* iMON or MCE (RC6) IR protocol? */
 	bool release_code;		/* some keys send a release code */
 
 	u8 display_type;		/* store the display type */
@@ -911,7 +911,7 @@ static struct attribute *imon_display_sysfs_entries[] = {
 	NULL
 };
 
-static struct attribute_group imon_display_attr_group = {
+static const struct attribute_group imon_display_attr_group = {
 	.attrs = imon_display_sysfs_entries
 };
 
@@ -920,7 +920,7 @@ static struct attribute *imon_rf_sysfs_entries[] = {
 	NULL
 };
 
-static struct attribute_group imon_rf_attr_group = {
+static const struct attribute_group imon_rf_attr_group = {
 	.attrs = imon_rf_sysfs_entries
 };
 
@@ -1118,7 +1118,7 @@ static void imon_touch_display_timeout(unsigned long data)
  * it is not, so we must acquire it prior to calling send_packet, which
  * requires that the lock is held.
  */
-static int imon_ir_change_protocol(struct rc_dev *rc, u64 *rc_type)
+static int imon_ir_change_protocol(struct rc_dev *rc, u64 *rc_proto)
 {
 	int retval;
 	struct imon_context *ictx = rc->priv;
@@ -1127,25 +1127,25 @@ static int imon_ir_change_protocol(struct rc_dev *rc, u64 *rc_type)
 	unsigned char ir_proto_packet[] = {
 		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x86 };
 
-	if (*rc_type && !(*rc_type & rc->allowed_protocols))
+	if (*rc_proto && !(*rc_proto & rc->allowed_protocols))
 		dev_warn(dev, "Looks like you're trying to use an IR protocol this device does not support\n");
 
-	if (*rc_type & RC_BIT_RC6_MCE) {
+	if (*rc_proto & RC_PROTO_BIT_RC6_MCE) {
 		dev_dbg(dev, "Configuring IR receiver for MCE protocol\n");
 		ir_proto_packet[0] = 0x01;
-		*rc_type = RC_BIT_RC6_MCE;
-	} else if (*rc_type & RC_BIT_OTHER) {
+		*rc_proto = RC_PROTO_BIT_RC6_MCE;
+	} else if (*rc_proto & RC_PROTO_BIT_OTHER) {
 		dev_dbg(dev, "Configuring IR receiver for iMON protocol\n");
 		if (!pad_stabilize)
 			dev_dbg(dev, "PAD stabilize functionality disabled\n");
 		/* ir_proto_packet[0] = 0x00; // already the default */
-		*rc_type = RC_BIT_OTHER;
+		*rc_proto = RC_PROTO_BIT_OTHER;
 	} else {
 		dev_warn(dev, "Unsupported IR protocol specified, overriding to iMON IR protocol\n");
 		if (!pad_stabilize)
 			dev_dbg(dev, "PAD stabilize functionality disabled\n");
 		/* ir_proto_packet[0] = 0x00; // already the default */
-		*rc_type = RC_BIT_OTHER;
+		*rc_proto = RC_PROTO_BIT_OTHER;
 	}
 
 	memcpy(ictx->usb_tx_buf, &ir_proto_packet, sizeof(ir_proto_packet));
@@ -1159,7 +1159,7 @@ static int imon_ir_change_protocol(struct rc_dev *rc, u64 *rc_type)
 	if (retval)
 		goto out;
 
-	ictx->rc_type = *rc_type;
+	ictx->rc_proto = *rc_proto;
 	ictx->pad_mouse = false;
 
 out:
@@ -1435,7 +1435,7 @@ static void imon_pad_to_keys(struct imon_context *ictx, unsigned char *buf)
 		rel_x = buf[2];
 		rel_y = buf[3];
 
-		if (ictx->rc_type == RC_BIT_OTHER && pad_stabilize) {
+		if (ictx->rc_proto == RC_PROTO_BIT_OTHER && pad_stabilize) {
 			if ((buf[1] == 0) && ((rel_x != 0) || (rel_y != 0))) {
 				dir = stabilize((int)rel_x, (int)rel_y,
 						timeout, threshold);
@@ -1502,7 +1502,7 @@ static void imon_pad_to_keys(struct imon_context *ictx, unsigned char *buf)
 		buf[0] = 0x01;
 		buf[1] = buf[4] = buf[5] = buf[6] = buf[7] = 0;
 
-		if (ictx->rc_type == RC_BIT_OTHER && pad_stabilize) {
+		if (ictx->rc_proto == RC_PROTO_BIT_OTHER && pad_stabilize) {
 			dir = stabilize((int)rel_x, (int)rel_y,
 					timeout, threshold);
 			if (!dir) {
@@ -1706,7 +1706,7 @@ static void imon_incoming_scancode(struct imon_context *ictx,
 		ictx->release_code = false;
 	} else {
 		scancode = be32_to_cpu(*((__be32 *)buf));
-		if (ictx->rc_type == RC_BIT_RC6_MCE) {
+		if (ictx->rc_proto == RC_PROTO_BIT_RC6_MCE) {
 			ktype = IMON_KEY_IMON;
 			if (buf[0] == 0x80)
 				ktype = IMON_KEY_MCE;
@@ -1769,10 +1769,10 @@ static void imon_incoming_scancode(struct imon_context *ictx,
 		if (press_type == 0)
 			rc_keyup(ictx->rdev);
 		else {
-			if (ictx->rc_type == RC_BIT_RC6_MCE ||
-			    ictx->rc_type == RC_BIT_OTHER)
+			if (ictx->rc_proto == RC_PROTO_BIT_RC6_MCE ||
+			    ictx->rc_proto == RC_PROTO_BIT_OTHER)
 				rc_keydown(ictx->rdev,
-					   ictx->rc_type == RC_BIT_RC6_MCE ? RC_TYPE_RC6_MCE : RC_TYPE_OTHER,
+					   ictx->rc_proto == RC_PROTO_BIT_RC6_MCE ? RC_PROTO_RC6_MCE : RC_PROTO_OTHER,
 					   ictx->rc_scancode, ictx->rc_toggle);
 			spin_lock_irqsave(&ictx->kc_lock, flags);
 			ictx->last_keycode = ictx->kc;
@@ -1936,7 +1936,7 @@ static void imon_get_ffdc_type(struct imon_context *ictx)
 {
 	u8 ffdc_cfg_byte = ictx->usb_rx_buf[6];
 	u8 detected_display_type = IMON_DISPLAY_TYPE_NONE;
-	u64 allowed_protos = RC_BIT_OTHER;
+	u64 allowed_protos = RC_PROTO_BIT_OTHER;
 
 	switch (ffdc_cfg_byte) {
 	/* iMON Knob, no display, iMON IR + vol knob */
@@ -1967,27 +1967,27 @@ static void imon_get_ffdc_type(struct imon_context *ictx)
 	case 0x9e:
 		dev_info(ictx->dev, "0xffdc iMON VFD, MCE IR");
 		detected_display_type = IMON_DISPLAY_TYPE_VFD;
-		allowed_protos = RC_BIT_RC6_MCE;
+		allowed_protos = RC_PROTO_BIT_RC6_MCE;
 		break;
 	/* iMON LCD, MCE IR */
 	case 0x9f:
 		dev_info(ictx->dev, "0xffdc iMON LCD, MCE IR");
 		detected_display_type = IMON_DISPLAY_TYPE_LCD;
-		allowed_protos = RC_BIT_RC6_MCE;
+		allowed_protos = RC_PROTO_BIT_RC6_MCE;
 		break;
 	default:
 		dev_info(ictx->dev, "Unknown 0xffdc device, defaulting to VFD and iMON IR");
 		detected_display_type = IMON_DISPLAY_TYPE_VFD;
 		/* We don't know which one it is, allow user to set the
 		 * RC6 one from userspace if OTHER wasn't correct. */
-		allowed_protos |= RC_BIT_RC6_MCE;
+		allowed_protos |= RC_PROTO_BIT_RC6_MCE;
 		break;
 	}
 
 	printk(KERN_CONT " (id 0x%02x)\n", ffdc_cfg_byte);
 
 	ictx->display_type = detected_display_type;
-	ictx->rc_type = allowed_protos;
+	ictx->rc_proto = allowed_protos;
 }
 
 static void imon_set_display_type(struct imon_context *ictx)
@@ -2063,17 +2063,18 @@ static struct rc_dev *imon_init_rdev(struct imon_context *ictx)
 		      sizeof(ictx->phys_rdev));
 	strlcat(ictx->phys_rdev, "/input0", sizeof(ictx->phys_rdev));
 
-	rdev->input_name = ictx->name_rdev;
+	rdev->device_name = ictx->name_rdev;
 	rdev->input_phys = ictx->phys_rdev;
 	usb_to_input_id(ictx->usbdev_intf0, &rdev->input_id);
 	rdev->dev.parent = ictx->dev;
 
 	rdev->priv = ictx;
 	if (ictx->dev_descr->flags & IMON_IR_RAW)
-		rdev->allowed_protocols = RC_BIT_ALL_IR_DECODER;
+		rdev->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
 	else
 		/* iMON PAD or MCE */
-		rdev->allowed_protocols = RC_BIT_OTHER | RC_BIT_RC6_MCE;
+		rdev->allowed_protocols = RC_PROTO_BIT_OTHER |
+					  RC_PROTO_BIT_RC6_MCE;
 	rdev->change_protocol = imon_ir_change_protocol;
 	rdev->driver_name = MOD_NAME;
 
@@ -2086,12 +2087,12 @@ static struct rc_dev *imon_init_rdev(struct imon_context *ictx)
 
 	if (ictx->product == 0xffdc) {
 		imon_get_ffdc_type(ictx);
-		rdev->allowed_protocols = ictx->rc_type;
+		rdev->allowed_protocols = ictx->rc_proto;
 	}
 
 	imon_set_display_type(ictx);
 
-	if (ictx->rc_type == RC_BIT_RC6_MCE ||
+	if (ictx->rc_proto == RC_PROTO_BIT_RC6_MCE ||
 	    ictx->dev_descr->flags & IMON_IR_RAW)
 		rdev->map_name = RC_MAP_IMON_MCE;
 	else
diff --git a/drivers/media/rc/ir-hix5hd2.c b/drivers/media/rc/ir-hix5hd2.c
index 50951f686852..0ce11c41dfae 100644
--- a/drivers/media/rc/ir-hix5hd2.c
+++ b/drivers/media/rc/ir-hix5hd2.c
@@ -242,14 +242,14 @@ static int hix5hd2_ir_probe(struct platform_device *pdev)
 	clk_prepare_enable(priv->clock);
 	priv->rate = clk_get_rate(priv->clock);
 
-	rdev->allowed_protocols = RC_BIT_ALL_IR_DECODER;
+	rdev->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
 	rdev->priv = priv;
 	rdev->open = hix5hd2_ir_open;
 	rdev->close = hix5hd2_ir_close;
 	rdev->driver_name = IR_HIX5HD2_NAME;
 	map_name = of_get_property(node, "linux,rc-map-name", NULL);
 	rdev->map_name = map_name ?: RC_MAP_EMPTY;
-	rdev->input_name = IR_HIX5HD2_NAME;
+	rdev->device_name = IR_HIX5HD2_NAME;
 	rdev->input_phys = IR_HIX5HD2_NAME "/input0";
 	rdev->input_id.bustype = BUS_HOST;
 	rdev->input_id.vendor = 0x0001;
diff --git a/drivers/media/rc/ir-jvc-decoder.c b/drivers/media/rc/ir-jvc-decoder.c
index 674bf156edcb..e2bd68c42edf 100644
--- a/drivers/media/rc/ir-jvc-decoder.c
+++ b/drivers/media/rc/ir-jvc-decoder.c
@@ -137,7 +137,7 @@ again:
 			scancode = (bitrev8((data->bits >> 8) & 0xff) << 8) |
 				   (bitrev8((data->bits >> 0) & 0xff) << 0);
 			IR_dprintk(1, "JVC scancode 0x%04x\n", scancode);
-			rc_keydown(dev, RC_TYPE_JVC, scancode, data->toggle);
+			rc_keydown(dev, RC_PROTO_JVC, scancode, data->toggle);
 			data->first = false;
 			data->old_bits = data->bits;
 		} else if (data->bits == data->old_bits) {
@@ -193,7 +193,7 @@ static const struct ir_raw_timings_pd ir_jvc_timings = {
  *		-ENOBUFS if there isn't enough space in the array to fit the
  *		encoding. In this case all @max events will have been written.
  */
-static int ir_jvc_encode(enum rc_type protocol, u32 scancode,
+static int ir_jvc_encode(enum rc_proto protocol, u32 scancode,
 			 struct ir_raw_event *events, unsigned int max)
 {
 	struct ir_raw_event *e = events;
@@ -209,7 +209,7 @@ static int ir_jvc_encode(enum rc_type protocol, u32 scancode,
 }
 
 static struct ir_raw_handler jvc_handler = {
-	.protocols	= RC_BIT_JVC,
+	.protocols	= RC_PROTO_BIT_JVC,
 	.decode		= ir_jvc_decode,
 	.encode		= ir_jvc_encode,
 };
diff --git a/drivers/media/rc/ir-mce_kbd-decoder.c b/drivers/media/rc/ir-mce_kbd-decoder.c
index 6a4d58b88d91..7c572a643656 100644
--- a/drivers/media/rc/ir-mce_kbd-decoder.c
+++ b/drivers/media/rc/ir-mce_kbd-decoder.c
@@ -358,6 +358,9 @@ static int ir_mce_kbd_register(struct rc_dev *dev)
 	struct input_dev *idev;
 	int i, ret;
 
+	if (dev->driver_type == RC_DRIVER_IR_RAW_TX)
+		return 0;
+
 	idev = input_allocate_device();
 	if (!idev)
 		return -ENOMEM;
@@ -413,6 +416,9 @@ static int ir_mce_kbd_unregister(struct rc_dev *dev)
 	struct mce_kbd_dec *mce_kbd = &dev->raw->mce_kbd;
 	struct input_dev *idev = mce_kbd->idev;
 
+	if (dev->driver_type == RC_DRIVER_IR_RAW_TX)
+		return 0;
+
 	del_timer_sync(&mce_kbd->rx_timeout);
 	input_unregister_device(idev);
 
@@ -438,14 +444,14 @@ static const struct ir_raw_timings_manchester ir_mce_kbd_timings = {
  *              -ENOBUFS if there isn't enough space in the array to fit the
  *              encoding. In this case all @max events will have been written.
  */
-static int ir_mce_kbd_encode(enum rc_type protocol, u32 scancode,
+static int ir_mce_kbd_encode(enum rc_proto protocol, u32 scancode,
 			     struct ir_raw_event *events, unsigned int max)
 {
 	struct ir_raw_event *e = events;
 	int len, ret;
 	u64 raw;
 
-	if (protocol == RC_TYPE_MCIR2_KBD) {
+	if (protocol == RC_PROTO_MCIR2_KBD) {
 		raw = scancode |
 		      ((u64)MCIR2_KEYBOARD_HEADER << MCIR2_KEYBOARD_NBITS);
 		len = MCIR2_KEYBOARD_NBITS + MCIR2_HEADER_NBITS + 1;
@@ -463,7 +469,7 @@ static int ir_mce_kbd_encode(enum rc_type protocol, u32 scancode,
 }
 
 static struct ir_raw_handler mce_kbd_handler = {
-	.protocols	= RC_BIT_MCIR2_KBD | RC_BIT_MCIR2_MSE,
+	.protocols	= RC_PROTO_BIT_MCIR2_KBD | RC_PROTO_BIT_MCIR2_MSE,
 	.decode		= ir_mce_kbd_decode,
 	.encode		= ir_mce_kbd_encode,
 	.raw_register	= ir_mce_kbd_register,
diff --git a/drivers/media/rc/ir-nec-decoder.c b/drivers/media/rc/ir-nec-decoder.c
index 3ce850314dca..817c18f2ddd1 100644
--- a/drivers/media/rc/ir-nec-decoder.c
+++ b/drivers/media/rc/ir-nec-decoder.c
@@ -49,9 +49,8 @@ static int ir_nec_decode(struct rc_dev *dev, struct ir_raw_event ev)
 {
 	struct nec_dec *data = &dev->raw->nec;
 	u32 scancode;
-	enum rc_type rc_type;
+	enum rc_proto rc_proto;
 	u8 address, not_address, command, not_command;
-	bool send_32bits = false;
 
 	if (!is_timing_event(ev)) {
 		if (ev.reset)
@@ -88,13 +87,9 @@ static int ir_nec_decode(struct rc_dev *dev, struct ir_raw_event ev)
 			data->state = STATE_BIT_PULSE;
 			return 0;
 		} else if (eq_margin(ev.duration, NEC_REPEAT_SPACE, NEC_UNIT / 2)) {
-			if (!dev->keypressed) {
-				IR_dprintk(1, "Discarding last key repeat: event after key up\n");
-			} else {
-				rc_repeat(dev);
-				IR_dprintk(1, "Repeat last key\n");
-				data->state = STATE_TRAILER_PULSE;
-			}
+			rc_repeat(dev);
+			IR_dprintk(1, "Repeat last key\n");
+			data->state = STATE_TRAILER_PULSE;
 			return 0;
 		}
 
@@ -161,39 +156,14 @@ static int ir_nec_decode(struct rc_dev *dev, struct ir_raw_event ev)
 		command	    = bitrev8((data->bits >>  8) & 0xff);
 		not_command = bitrev8((data->bits >>  0) & 0xff);
 
-		if ((command ^ not_command) != 0xff) {
-			IR_dprintk(1, "NEC checksum error: received 0x%08x\n",
-				   data->bits);
-			send_32bits = true;
-		}
-
-		if (send_32bits) {
-			/* NEC transport, but modified protocol, used by at
-			 * least Apple and TiVo remotes */
-			scancode = not_address << 24 |
-				address     << 16 |
-				not_command <<  8 |
-				command;
-			IR_dprintk(1, "NEC (modified) scancode 0x%08x\n", scancode);
-			rc_type = RC_TYPE_NEC32;
-		} else if ((address ^ not_address) != 0xff) {
-			/* Extended NEC */
-			scancode = address     << 16 |
-				   not_address <<  8 |
-				   command;
-			IR_dprintk(1, "NEC (Ext) scancode 0x%06x\n", scancode);
-			rc_type = RC_TYPE_NECX;
-		} else {
-			/* Normal NEC */
-			scancode = address << 8 | command;
-			IR_dprintk(1, "NEC scancode 0x%04x\n", scancode);
-			rc_type = RC_TYPE_NEC;
-		}
+		scancode = ir_nec_bytes_to_scancode(address, not_address,
+						    command, not_command,
+						    &rc_proto);
 
 		if (data->is_nec_x)
 			data->necx_repeat = true;
 
-		rc_keydown(dev, rc_type, scancode, 0);
+		rc_keydown(dev, rc_proto, scancode, 0);
 		data->state = STATE_INACTIVE;
 		return 0;
 	}
@@ -210,19 +180,19 @@ static int ir_nec_decode(struct rc_dev *dev, struct ir_raw_event ev)
  * @scancode:	a single NEC scancode.
  * @raw:	raw data to be modulated.
  */
-static u32 ir_nec_scancode_to_raw(enum rc_type protocol, u32 scancode)
+static u32 ir_nec_scancode_to_raw(enum rc_proto protocol, u32 scancode)
 {
 	unsigned int addr, addr_inv, data, data_inv;
 
 	data = scancode & 0xff;
 
-	if (protocol == RC_TYPE_NEC32) {
+	if (protocol == RC_PROTO_NEC32) {
 		/* 32-bit NEC (used by Apple and TiVo remotes) */
 		/* scan encoding: aaAAddDD */
 		addr_inv   = (scancode >> 24) & 0xff;
 		addr       = (scancode >> 16) & 0xff;
 		data_inv   = (scancode >>  8) & 0xff;
-	} else if (protocol == RC_TYPE_NECX) {
+	} else if (protocol == RC_PROTO_NECX) {
 		/* Extended NEC */
 		/* scan encoding AAaaDD */
 		addr       = (scancode >> 16) & 0xff;
@@ -266,7 +236,7 @@ static const struct ir_raw_timings_pd ir_nec_timings = {
  *		-ENOBUFS if there isn't enough space in the array to fit the
  *		encoding. In this case all @max events will have been written.
  */
-static int ir_nec_encode(enum rc_type protocol, u32 scancode,
+static int ir_nec_encode(enum rc_proto protocol, u32 scancode,
 			 struct ir_raw_event *events, unsigned int max)
 {
 	struct ir_raw_event *e = events;
@@ -285,7 +255,8 @@ static int ir_nec_encode(enum rc_type protocol, u32 scancode,
 }
 
 static struct ir_raw_handler nec_handler = {
-	.protocols	= RC_BIT_NEC | RC_BIT_NECX | RC_BIT_NEC32,
+	.protocols	= RC_PROTO_BIT_NEC | RC_PROTO_BIT_NECX |
+							RC_PROTO_BIT_NEC32,
 	.decode		= ir_nec_decode,
 	.encode		= ir_nec_encode,
 };
diff --git a/drivers/media/rc/ir-rc5-decoder.c b/drivers/media/rc/ir-rc5-decoder.c
index fcfedf95def7..1292f534de43 100644
--- a/drivers/media/rc/ir-rc5-decoder.c
+++ b/drivers/media/rc/ir-rc5-decoder.c
@@ -51,7 +51,7 @@ static int ir_rc5_decode(struct rc_dev *dev, struct ir_raw_event ev)
 	struct rc5_dec *data = &dev->raw->rc5;
 	u8 toggle;
 	u32 scancode;
-	enum rc_type protocol;
+	enum rc_proto protocol;
 
 	if (!is_timing_event(ev)) {
 		if (ev.reset)
@@ -124,7 +124,7 @@ again:
 		if (data->is_rc5x && data->count == RC5X_NBITS) {
 			/* RC5X */
 			u8 xdata, command, system;
-			if (!(dev->enabled_protocols & RC_BIT_RC5X_20)) {
+			if (!(dev->enabled_protocols & RC_PROTO_BIT_RC5X_20)) {
 				data->state = STATE_INACTIVE;
 				return 0;
 			}
@@ -134,12 +134,12 @@ again:
 			toggle   = (data->bits & 0x20000) ? 1 : 0;
 			command += (data->bits & 0x40000) ? 0 : 0x40;
 			scancode = system << 16 | command << 8 | xdata;
-			protocol = RC_TYPE_RC5X_20;
+			protocol = RC_PROTO_RC5X_20;
 
 		} else if (!data->is_rc5x && data->count == RC5_NBITS) {
 			/* RC5 */
 			u8 command, system;
-			if (!(dev->enabled_protocols & RC_BIT_RC5)) {
+			if (!(dev->enabled_protocols & RC_PROTO_BIT_RC5)) {
 				data->state = STATE_INACTIVE;
 				return 0;
 			}
@@ -148,12 +148,12 @@ again:
 			toggle   = (data->bits & 0x00800) ? 1 : 0;
 			command += (data->bits & 0x01000) ? 0 : 0x40;
 			scancode = system << 8 | command;
-			protocol = RC_TYPE_RC5;
+			protocol = RC_PROTO_RC5;
 
 		} else if (!data->is_rc5x && data->count == RC5_SZ_NBITS) {
 			/* RC5 StreamZap */
 			u8 command, system;
-			if (!(dev->enabled_protocols & RC_BIT_RC5_SZ)) {
+			if (!(dev->enabled_protocols & RC_PROTO_BIT_RC5_SZ)) {
 				data->state = STATE_INACTIVE;
 				return 0;
 			}
@@ -161,7 +161,7 @@ again:
 			system   = (data->bits & 0x02FC0) >> 6;
 			toggle   = (data->bits & 0x01000) ? 1 : 0;
 			scancode = system << 6 | command;
-			protocol = RC_TYPE_RC5_SZ;
+			protocol = RC_PROTO_RC5_SZ;
 
 		} else
 			break;
@@ -221,7 +221,7 @@ static const struct ir_raw_timings_manchester ir_rc5_sz_timings = {
  *		encoding. In this case all @max events will have been written.
  *		-EINVAL if the scancode is ambiguous or invalid.
  */
-static int ir_rc5_encode(enum rc_type protocol, u32 scancode,
+static int ir_rc5_encode(enum rc_proto protocol, u32 scancode,
 			 struct ir_raw_event *events, unsigned int max)
 {
 	int ret;
@@ -229,7 +229,7 @@ static int ir_rc5_encode(enum rc_type protocol, u32 scancode,
 	unsigned int data, xdata, command, commandx, system, pre_space_data;
 
 	/* Detect protocol and convert scancode to raw data */
-	if (protocol == RC_TYPE_RC5) {
+	if (protocol == RC_PROTO_RC5) {
 		/* decode scancode */
 		command  = (scancode & 0x003f) >> 0;
 		commandx = (scancode & 0x0040) >> 6;
@@ -242,7 +242,7 @@ static int ir_rc5_encode(enum rc_type protocol, u32 scancode,
 					    RC5_NBITS, data);
 		if (ret < 0)
 			return ret;
-	} else if (protocol == RC_TYPE_RC5X_20) {
+	} else if (protocol == RC_PROTO_RC5X_20) {
 		/* decode scancode */
 		xdata    = (scancode & 0x00003f) >> 0;
 		command  = (scancode & 0x003f00) >> 8;
@@ -264,7 +264,7 @@ static int ir_rc5_encode(enum rc_type protocol, u32 scancode,
 					    data);
 		if (ret < 0)
 			return ret;
-	} else if (protocol == RC_TYPE_RC5_SZ) {
+	} else if (protocol == RC_PROTO_RC5_SZ) {
 		/* RC5-SZ scancode is raw enough for Manchester as it is */
 		ret = ir_raw_gen_manchester(&e, max, &ir_rc5_sz_timings,
 					    RC5_SZ_NBITS, scancode & 0x2fff);
@@ -278,7 +278,8 @@ static int ir_rc5_encode(enum rc_type protocol, u32 scancode,
 }
 
 static struct ir_raw_handler rc5_handler = {
-	.protocols	= RC_BIT_RC5 | RC_BIT_RC5X_20 | RC_BIT_RC5_SZ,
+	.protocols	= RC_PROTO_BIT_RC5 | RC_PROTO_BIT_RC5X_20 |
+							RC_PROTO_BIT_RC5_SZ,
 	.decode		= ir_rc5_decode,
 	.encode		= ir_rc5_encode,
 };
diff --git a/drivers/media/rc/ir-rc6-decoder.c b/drivers/media/rc/ir-rc6-decoder.c
index 6fe2268dada0..5d0d2fe3b7a7 100644
--- a/drivers/media/rc/ir-rc6-decoder.c
+++ b/drivers/media/rc/ir-rc6-decoder.c
@@ -88,7 +88,7 @@ static int ir_rc6_decode(struct rc_dev *dev, struct ir_raw_event ev)
 	struct rc6_dec *data = &dev->raw->rc6;
 	u32 scancode;
 	u8 toggle;
-	enum rc_type protocol;
+	enum rc_proto protocol;
 
 	if (!is_timing_event(ev)) {
 		if (ev.reset)
@@ -229,7 +229,7 @@ again:
 		case RC6_MODE_0:
 			scancode = data->body;
 			toggle = data->toggle;
-			protocol = RC_TYPE_RC6_0;
+			protocol = RC_PROTO_RC6_0;
 			IR_dprintk(1, "RC6(0) scancode 0x%04x (toggle: %u)\n",
 				   scancode, toggle);
 			break;
@@ -244,20 +244,20 @@ again:
 			scancode = data->body;
 			switch (data->count) {
 			case 20:
-				protocol = RC_TYPE_RC6_6A_20;
+				protocol = RC_PROTO_RC6_6A_20;
 				toggle = 0;
 				break;
 			case 24:
-				protocol = RC_TYPE_RC6_6A_24;
+				protocol = RC_PROTO_RC6_6A_24;
 				toggle = 0;
 				break;
 			case 32:
 				if ((scancode & RC6_6A_LCC_MASK) == RC6_6A_MCE_CC) {
-					protocol = RC_TYPE_RC6_MCE;
+					protocol = RC_PROTO_RC6_MCE;
 					toggle = !!(scancode & RC6_6A_MCE_TOGGLE_MASK);
 					scancode &= ~RC6_6A_MCE_TOGGLE_MASK;
 				} else {
-					protocol = RC_TYPE_RC6_6A_32;
+					protocol = RC_PROTO_RC6_6A_32;
 					toggle = 0;
 				}
 				break;
@@ -322,13 +322,13 @@ static const struct ir_raw_timings_manchester ir_rc6_timings[4] = {
  *		encoding. In this case all @max events will have been written.
  *		-EINVAL if the scancode is ambiguous or invalid.
  */
-static int ir_rc6_encode(enum rc_type protocol, u32 scancode,
+static int ir_rc6_encode(enum rc_proto protocol, u32 scancode,
 			 struct ir_raw_event *events, unsigned int max)
 {
 	int ret;
 	struct ir_raw_event *e = events;
 
-	if (protocol == RC_TYPE_RC6_0) {
+	if (protocol == RC_PROTO_RC6_0) {
 		/* Modulate the preamble */
 		ret = ir_raw_gen_manchester(&e, max, &ir_rc6_timings[0], 0, 0);
 		if (ret < 0)
@@ -358,14 +358,14 @@ static int ir_rc6_encode(enum rc_type protocol, u32 scancode,
 		int bits;
 
 		switch (protocol) {
-		case RC_TYPE_RC6_MCE:
-		case RC_TYPE_RC6_6A_32:
+		case RC_PROTO_RC6_MCE:
+		case RC_PROTO_RC6_6A_32:
 			bits = 32;
 			break;
-		case RC_TYPE_RC6_6A_24:
+		case RC_PROTO_RC6_6A_24:
 			bits = 24;
 			break;
-		case RC_TYPE_RC6_6A_20:
+		case RC_PROTO_RC6_6A_20:
 			bits = 20;
 			break;
 		default:
@@ -403,9 +403,9 @@ static int ir_rc6_encode(enum rc_type protocol, u32 scancode,
 }
 
 static struct ir_raw_handler rc6_handler = {
-	.protocols	= RC_BIT_RC6_0 | RC_BIT_RC6_6A_20 |
-			  RC_BIT_RC6_6A_24 | RC_BIT_RC6_6A_32 |
-			  RC_BIT_RC6_MCE,
+	.protocols	= RC_PROTO_BIT_RC6_0 | RC_PROTO_BIT_RC6_6A_20 |
+			  RC_PROTO_BIT_RC6_6A_24 | RC_PROTO_BIT_RC6_6A_32 |
+			  RC_PROTO_BIT_RC6_MCE,
 	.decode		= ir_rc6_decode,
 	.encode		= ir_rc6_encode,
 };
diff --git a/drivers/media/rc/ir-sanyo-decoder.c b/drivers/media/rc/ir-sanyo-decoder.c
index 520bb77dcb62..758c60956850 100644
--- a/drivers/media/rc/ir-sanyo-decoder.c
+++ b/drivers/media/rc/ir-sanyo-decoder.c
@@ -110,13 +110,9 @@ static int ir_sanyo_decode(struct rc_dev *dev, struct ir_raw_event ev)
 			break;
 
 		if (!data->count && geq_margin(ev.duration, SANYO_REPEAT_SPACE, SANYO_UNIT / 2)) {
-			if (!dev->keypressed) {
-				IR_dprintk(1, "SANYO discarding last key repeat: event after key up\n");
-			} else {
-				rc_repeat(dev);
-				IR_dprintk(1, "SANYO repeat last key\n");
-				data->state = STATE_INACTIVE;
-			}
+			rc_repeat(dev);
+			IR_dprintk(1, "SANYO repeat last key\n");
+			data->state = STATE_INACTIVE;
 			return 0;
 		}
 
@@ -165,7 +161,7 @@ static int ir_sanyo_decode(struct rc_dev *dev, struct ir_raw_event ev)
 
 		scancode = address << 8 | command;
 		IR_dprintk(1, "SANYO scancode: 0x%06x\n", scancode);
-		rc_keydown(dev, RC_TYPE_SANYO, scancode, 0);
+		rc_keydown(dev, RC_PROTO_SANYO, scancode, 0);
 		data->state = STATE_INACTIVE;
 		return 0;
 	}
@@ -199,7 +195,7 @@ static const struct ir_raw_timings_pd ir_sanyo_timings = {
  *		-ENOBUFS if there isn't enough space in the array to fit the
  *		encoding. In this case all @max events will have been written.
  */
-static int ir_sanyo_encode(enum rc_type protocol, u32 scancode,
+static int ir_sanyo_encode(enum rc_proto protocol, u32 scancode,
 			   struct ir_raw_event *events, unsigned int max)
 {
 	struct ir_raw_event *e = events;
@@ -219,7 +215,7 @@ static int ir_sanyo_encode(enum rc_type protocol, u32 scancode,
 }
 
 static struct ir_raw_handler sanyo_handler = {
-	.protocols	= RC_BIT_SANYO,
+	.protocols	= RC_PROTO_BIT_SANYO,
 	.decode		= ir_sanyo_decode,
 	.encode		= ir_sanyo_encode,
 };
diff --git a/drivers/media/rc/ir-sharp-decoder.c b/drivers/media/rc/ir-sharp-decoder.c
index b47e89e2c1bd..ed43a4212479 100644
--- a/drivers/media/rc/ir-sharp-decoder.c
+++ b/drivers/media/rc/ir-sharp-decoder.c
@@ -161,7 +161,7 @@ static int ir_sharp_decode(struct rc_dev *dev, struct ir_raw_event ev)
 		scancode = address << 8 | command;
 		IR_dprintk(1, "Sharp scancode 0x%04x\n", scancode);
 
-		rc_keydown(dev, RC_TYPE_SHARP, scancode, 0);
+		rc_keydown(dev, RC_PROTO_SHARP, scancode, 0);
 		data->state = STATE_INACTIVE;
 		return 0;
 	}
@@ -196,7 +196,7 @@ static const struct ir_raw_timings_pd ir_sharp_timings = {
  *		-ENOBUFS if there isn't enough space in the array to fit the
  *		encoding. In this case all @max events will have been written.
  */
-static int ir_sharp_encode(enum rc_type protocol, u32 scancode,
+static int ir_sharp_encode(enum rc_proto protocol, u32 scancode,
 			   struct ir_raw_event *events, unsigned int max)
 {
 	struct ir_raw_event *e = events;
@@ -223,7 +223,7 @@ static int ir_sharp_encode(enum rc_type protocol, u32 scancode,
 }
 
 static struct ir_raw_handler sharp_handler = {
-	.protocols	= RC_BIT_SHARP,
+	.protocols	= RC_PROTO_BIT_SHARP,
 	.decode		= ir_sharp_decode,
 	.encode		= ir_sharp_encode,
 };
diff --git a/drivers/media/rc/ir-sony-decoder.c b/drivers/media/rc/ir-sony-decoder.c
index 355fa8198f5a..a47ced763031 100644
--- a/drivers/media/rc/ir-sony-decoder.c
+++ b/drivers/media/rc/ir-sony-decoder.c
@@ -42,7 +42,7 @@ enum sony_state {
 static int ir_sony_decode(struct rc_dev *dev, struct ir_raw_event ev)
 {
 	struct sony_dec *data = &dev->raw->sony;
-	enum rc_type protocol;
+	enum rc_proto protocol;
 	u32 scancode;
 	u8 device, subdevice, function;
 
@@ -121,31 +121,31 @@ static int ir_sony_decode(struct rc_dev *dev, struct ir_raw_event ev)
 
 		switch (data->count) {
 		case 12:
-			if (!(dev->enabled_protocols & RC_BIT_SONY12))
+			if (!(dev->enabled_protocols & RC_PROTO_BIT_SONY12))
 				goto finish_state_machine;
 
 			device    = bitrev8((data->bits <<  3) & 0xF8);
 			subdevice = 0;
 			function  = bitrev8((data->bits >>  4) & 0xFE);
-			protocol = RC_TYPE_SONY12;
+			protocol = RC_PROTO_SONY12;
 			break;
 		case 15:
-			if (!(dev->enabled_protocols & RC_BIT_SONY15))
+			if (!(dev->enabled_protocols & RC_PROTO_BIT_SONY15))
 				goto finish_state_machine;
 
 			device    = bitrev8((data->bits >>  0) & 0xFF);
 			subdevice = 0;
 			function  = bitrev8((data->bits >>  7) & 0xFE);
-			protocol = RC_TYPE_SONY15;
+			protocol = RC_PROTO_SONY15;
 			break;
 		case 20:
-			if (!(dev->enabled_protocols & RC_BIT_SONY20))
+			if (!(dev->enabled_protocols & RC_PROTO_BIT_SONY20))
 				goto finish_state_machine;
 
 			device    = bitrev8((data->bits >>  5) & 0xF8);
 			subdevice = bitrev8((data->bits >>  0) & 0xFF);
 			function  = bitrev8((data->bits >> 12) & 0xFE);
-			protocol = RC_TYPE_SONY20;
+			protocol = RC_PROTO_SONY20;
 			break;
 		default:
 			IR_dprintk(1, "Sony invalid bitcount %u\n", data->count);
@@ -190,17 +190,17 @@ static const struct ir_raw_timings_pl ir_sony_timings = {
  *		-ENOBUFS if there isn't enough space in the array to fit the
  *		encoding. In this case all @max events will have been written.
  */
-static int ir_sony_encode(enum rc_type protocol, u32 scancode,
+static int ir_sony_encode(enum rc_proto protocol, u32 scancode,
 			  struct ir_raw_event *events, unsigned int max)
 {
 	struct ir_raw_event *e = events;
 	u32 raw, len;
 	int ret;
 
-	if (protocol == RC_TYPE_SONY12) {
+	if (protocol == RC_PROTO_SONY12) {
 		raw = (scancode & 0x7f) | ((scancode & 0x1f0000) >> 9);
 		len = 12;
-	} else if (protocol == RC_TYPE_SONY15) {
+	} else if (protocol == RC_PROTO_SONY15) {
 		raw = (scancode & 0x7f) | ((scancode & 0xff0000) >> 9);
 		len = 15;
 	} else {
@@ -217,7 +217,8 @@ static int ir_sony_encode(enum rc_type protocol, u32 scancode,
 }
 
 static struct ir_raw_handler sony_handler = {
-	.protocols	= RC_BIT_SONY12 | RC_BIT_SONY15 | RC_BIT_SONY20,
+	.protocols	= RC_PROTO_BIT_SONY12 | RC_PROTO_BIT_SONY15 |
+							RC_PROTO_BIT_SONY20,
 	.decode		= ir_sony_decode,
 	.encode		= ir_sony_encode,
 };
diff --git a/drivers/media/rc/ir-spi.c b/drivers/media/rc/ir-spi.c
index 7e383b3fedd5..29ed0638cb74 100644
--- a/drivers/media/rc/ir-spi.c
+++ b/drivers/media/rc/ir-spi.c
@@ -155,6 +155,7 @@ static int ir_spi_probe(struct spi_device *spi)
 	idata->rc->tx_ir           = ir_spi_tx;
 	idata->rc->s_tx_carrier    = ir_spi_set_tx_carrier;
 	idata->rc->s_tx_duty_cycle = ir_spi_set_duty_cycle;
+	idata->rc->device_name	   = "IR SPI";
 	idata->rc->driver_name     = IR_SPI_DRIVER_NAME;
 	idata->rc->priv            = idata;
 	idata->spi                 = spi;
diff --git a/drivers/media/rc/ir-xmp-decoder.c b/drivers/media/rc/ir-xmp-decoder.c
index 18596190bbb8..6f464be1c8d7 100644
--- a/drivers/media/rc/ir-xmp-decoder.c
+++ b/drivers/media/rc/ir-xmp-decoder.c
@@ -141,7 +141,7 @@ static int ir_xmp_decode(struct rc_dev *dev, struct ir_raw_event ev)
 			IR_dprintk(1, "XMP scancode 0x%06x\n", scancode);
 
 			if (toggle == 0) {
-				rc_keydown(dev, RC_TYPE_XMP, scancode, 0);
+				rc_keydown(dev, RC_PROTO_XMP, scancode, 0);
 			} else {
 				rc_repeat(dev);
 				IR_dprintk(1, "Repeat last key\n");
@@ -196,7 +196,7 @@ static int ir_xmp_decode(struct rc_dev *dev, struct ir_raw_event ev)
 }
 
 static struct ir_raw_handler xmp_handler = {
-	.protocols	= RC_BIT_XMP,
+	.protocols	= RC_PROTO_BIT_XMP,
 	.decode		= ir_xmp_decode,
 };
 
diff --git a/drivers/media/rc/ite-cir.c b/drivers/media/rc/ite-cir.c
index e9e4befbbebb..65e104c7ddfc 100644
--- a/drivers/media/rc/ite-cir.c
+++ b/drivers/media/rc/ite-cir.c
@@ -1556,7 +1556,7 @@ static int ite_probe(struct pnp_dev *pdev, const struct pnp_device_id
 
 	/* set up ir-core props */
 	rdev->priv = itdev;
-	rdev->allowed_protocols = RC_BIT_ALL_IR_DECODER;
+	rdev->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
 	rdev->open = ite_open;
 	rdev->close = ite_close;
 	rdev->s_idle = ite_s_idle;
@@ -1576,7 +1576,7 @@ static int ite_probe(struct pnp_dev *pdev, const struct pnp_device_id
 		rdev->s_tx_duty_cycle = ite_set_tx_duty_cycle;
 	}
 
-	rdev->input_name = dev_desc->model;
+	rdev->device_name = dev_desc->model;
 	rdev->input_id.bustype = BUS_HOST;
 	rdev->input_id.vendor = PCI_VENDOR_ID_ITE;
 	rdev->input_id.product = 0;
diff --git a/drivers/media/rc/keymaps/Makefile b/drivers/media/rc/keymaps/Makefile
index 2945f99907b5..af6496d709fb 100644
--- a/drivers/media/rc/keymaps/Makefile
+++ b/drivers/media/rc/keymaps/Makefile
@@ -109,4 +109,5 @@ obj-$(CONFIG_RC_MAP) += rc-adstech-dvb-t-pci.o \
 			rc-videomate-tv-pvr.o \
 			rc-winfast.o \
 			rc-winfast-usbii-deluxe.o \
-			rc-su3000.o
+			rc-su3000.o \
+			rc-zx-irdec.o
diff --git a/drivers/media/rc/keymaps/rc-adstech-dvb-t-pci.c b/drivers/media/rc/keymaps/rc-adstech-dvb-t-pci.c
index 01d901fbfc8b..2d303c2cee3b 100644
--- a/drivers/media/rc/keymaps/rc-adstech-dvb-t-pci.c
+++ b/drivers/media/rc/keymaps/rc-adstech-dvb-t-pci.c
@@ -66,10 +66,10 @@ static struct rc_map_table adstech_dvb_t_pci[] = {
 
 static struct rc_map_list adstech_dvb_t_pci_map = {
 	.map = {
-		.scan    = adstech_dvb_t_pci,
-		.size    = ARRAY_SIZE(adstech_dvb_t_pci),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_ADSTECH_DVB_T_PCI,
+		.scan     = adstech_dvb_t_pci,
+		.size     = ARRAY_SIZE(adstech_dvb_t_pci),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_ADSTECH_DVB_T_PCI,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-alink-dtu-m.c b/drivers/media/rc/keymaps/rc-alink-dtu-m.c
index 4e6ade8e616f..3818c33734a1 100644
--- a/drivers/media/rc/keymaps/rc-alink-dtu-m.c
+++ b/drivers/media/rc/keymaps/rc-alink-dtu-m.c
@@ -45,10 +45,10 @@ static struct rc_map_table alink_dtu_m[] = {
 
 static struct rc_map_list alink_dtu_m_map = {
 	.map = {
-		.scan    = alink_dtu_m,
-		.size    = ARRAY_SIZE(alink_dtu_m),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_ALINK_DTU_M,
+		.scan     = alink_dtu_m,
+		.size     = ARRAY_SIZE(alink_dtu_m),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_ALINK_DTU_M,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-anysee.c b/drivers/media/rc/keymaps/rc-anysee.c
index c735fe10a390..e75e51b34d29 100644
--- a/drivers/media/rc/keymaps/rc-anysee.c
+++ b/drivers/media/rc/keymaps/rc-anysee.c
@@ -70,10 +70,10 @@ static struct rc_map_table anysee[] = {
 
 static struct rc_map_list anysee_map = {
 	.map = {
-		.scan    = anysee,
-		.size    = ARRAY_SIZE(anysee),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_ANYSEE,
+		.scan     = anysee,
+		.size     = ARRAY_SIZE(anysee),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_ANYSEE,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-apac-viewcomp.c b/drivers/media/rc/keymaps/rc-apac-viewcomp.c
index bf9efa007e1c..65bc8957d9c3 100644
--- a/drivers/media/rc/keymaps/rc-apac-viewcomp.c
+++ b/drivers/media/rc/keymaps/rc-apac-viewcomp.c
@@ -57,10 +57,10 @@ static struct rc_map_table apac_viewcomp[] = {
 
 static struct rc_map_list apac_viewcomp_map = {
 	.map = {
-		.scan    = apac_viewcomp,
-		.size    = ARRAY_SIZE(apac_viewcomp),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_APAC_VIEWCOMP,
+		.scan     = apac_viewcomp,
+		.size     = ARRAY_SIZE(apac_viewcomp),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_APAC_VIEWCOMP,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-asus-pc39.c b/drivers/media/rc/keymaps/rc-asus-pc39.c
index 9e674ba5dd4f..530e1d1158d1 100644
--- a/drivers/media/rc/keymaps/rc-asus-pc39.c
+++ b/drivers/media/rc/keymaps/rc-asus-pc39.c
@@ -68,10 +68,10 @@ static struct rc_map_table asus_pc39[] = {
 
 static struct rc_map_list asus_pc39_map = {
 	.map = {
-		.scan    = asus_pc39,
-		.size    = ARRAY_SIZE(asus_pc39),
-		.rc_type = RC_TYPE_RC5,
-		.name    = RC_MAP_ASUS_PC39,
+		.scan     = asus_pc39,
+		.size     = ARRAY_SIZE(asus_pc39),
+		.rc_proto = RC_PROTO_RC5,
+		.name     = RC_MAP_ASUS_PC39,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-asus-ps3-100.c b/drivers/media/rc/keymaps/rc-asus-ps3-100.c
index e45de35f528f..c91ba332984c 100644
--- a/drivers/media/rc/keymaps/rc-asus-ps3-100.c
+++ b/drivers/media/rc/keymaps/rc-asus-ps3-100.c
@@ -67,10 +67,10 @@ static struct rc_map_table asus_ps3_100[] = {
 
 static struct rc_map_list asus_ps3_100_map = {
 .map = {
-	.scan    = asus_ps3_100,
-	.size    = ARRAY_SIZE(asus_ps3_100),
-	.rc_type = RC_TYPE_RC5,
-	.name    = RC_MAP_ASUS_PS3_100,
+	.scan     = asus_ps3_100,
+	.size     = ARRAY_SIZE(asus_ps3_100),
+	.rc_proto = RC_PROTO_RC5,
+	.name     = RC_MAP_ASUS_PS3_100,
 }
 };
 
diff --git a/drivers/media/rc/keymaps/rc-ati-tv-wonder-hd-600.c b/drivers/media/rc/keymaps/rc-ati-tv-wonder-hd-600.c
index 91392d4cfd6d..11b4bdd2392b 100644
--- a/drivers/media/rc/keymaps/rc-ati-tv-wonder-hd-600.c
+++ b/drivers/media/rc/keymaps/rc-ati-tv-wonder-hd-600.c
@@ -46,10 +46,10 @@ static struct rc_map_table ati_tv_wonder_hd_600[] = {
 
 static struct rc_map_list ati_tv_wonder_hd_600_map = {
 	.map = {
-		.scan    = ati_tv_wonder_hd_600,
-		.size    = ARRAY_SIZE(ati_tv_wonder_hd_600),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_ATI_TV_WONDER_HD_600,
+		.scan     = ati_tv_wonder_hd_600,
+		.size     = ARRAY_SIZE(ati_tv_wonder_hd_600),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_ATI_TV_WONDER_HD_600,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-ati-x10.c b/drivers/media/rc/keymaps/rc-ati-x10.c
index 4bdc709ec54d..11f1eb6ad712 100644
--- a/drivers/media/rc/keymaps/rc-ati-x10.c
+++ b/drivers/media/rc/keymaps/rc-ati-x10.c
@@ -114,10 +114,10 @@ static struct rc_map_table ati_x10[] = {
 
 static struct rc_map_list ati_x10_map = {
 	.map = {
-		.scan    = ati_x10,
-		.size    = ARRAY_SIZE(ati_x10),
-		.rc_type = RC_TYPE_OTHER,
-		.name    = RC_MAP_ATI_X10,
+		.scan     = ati_x10,
+		.size     = ARRAY_SIZE(ati_x10),
+		.rc_proto = RC_PROTO_OTHER,
+		.name     = RC_MAP_ATI_X10,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-avermedia-a16d.c b/drivers/media/rc/keymaps/rc-avermedia-a16d.c
index ff30a71d623e..510dc90ebf49 100644
--- a/drivers/media/rc/keymaps/rc-avermedia-a16d.c
+++ b/drivers/media/rc/keymaps/rc-avermedia-a16d.c
@@ -52,10 +52,10 @@ static struct rc_map_table avermedia_a16d[] = {
 
 static struct rc_map_list avermedia_a16d_map = {
 	.map = {
-		.scan    = avermedia_a16d,
-		.size    = ARRAY_SIZE(avermedia_a16d),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_AVERMEDIA_A16D,
+		.scan     = avermedia_a16d,
+		.size     = ARRAY_SIZE(avermedia_a16d),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_AVERMEDIA_A16D,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-avermedia-cardbus.c b/drivers/media/rc/keymaps/rc-avermedia-cardbus.c
index d7471a6de9b4..4bbc1e68d1b8 100644
--- a/drivers/media/rc/keymaps/rc-avermedia-cardbus.c
+++ b/drivers/media/rc/keymaps/rc-avermedia-cardbus.c
@@ -74,10 +74,10 @@ static struct rc_map_table avermedia_cardbus[] = {
 
 static struct rc_map_list avermedia_cardbus_map = {
 	.map = {
-		.scan    = avermedia_cardbus,
-		.size    = ARRAY_SIZE(avermedia_cardbus),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_AVERMEDIA_CARDBUS,
+		.scan     = avermedia_cardbus,
+		.size     = ARRAY_SIZE(avermedia_cardbus),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_AVERMEDIA_CARDBUS,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-avermedia-dvbt.c b/drivers/media/rc/keymaps/rc-avermedia-dvbt.c
index e2417d6331fe..f6b8547dbad3 100644
--- a/drivers/media/rc/keymaps/rc-avermedia-dvbt.c
+++ b/drivers/media/rc/keymaps/rc-avermedia-dvbt.c
@@ -55,10 +55,10 @@ static struct rc_map_table avermedia_dvbt[] = {
 
 static struct rc_map_list avermedia_dvbt_map = {
 	.map = {
-		.scan    = avermedia_dvbt,
-		.size    = ARRAY_SIZE(avermedia_dvbt),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_AVERMEDIA_DVBT,
+		.scan     = avermedia_dvbt,
+		.size     = ARRAY_SIZE(avermedia_dvbt),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_AVERMEDIA_DVBT,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-avermedia-m135a.c b/drivers/media/rc/keymaps/rc-avermedia-m135a.c
index 843598a5f1b5..9882e2cde975 100644
--- a/drivers/media/rc/keymaps/rc-avermedia-m135a.c
+++ b/drivers/media/rc/keymaps/rc-avermedia-m135a.c
@@ -124,10 +124,10 @@ static struct rc_map_table avermedia_m135a[] = {
 
 static struct rc_map_list avermedia_m135a_map = {
 	.map = {
-		.scan    = avermedia_m135a,
-		.size    = ARRAY_SIZE(avermedia_m135a),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_AVERMEDIA_M135A,
+		.scan     = avermedia_m135a,
+		.size     = ARRAY_SIZE(avermedia_m135a),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_AVERMEDIA_M135A,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-avermedia-m733a-rm-k6.c b/drivers/media/rc/keymaps/rc-avermedia-m733a-rm-k6.c
index b24e7481ac21..d86126e10375 100644
--- a/drivers/media/rc/keymaps/rc-avermedia-m733a-rm-k6.c
+++ b/drivers/media/rc/keymaps/rc-avermedia-m733a-rm-k6.c
@@ -72,10 +72,10 @@ static struct rc_map_table avermedia_m733a_rm_k6[] = {
 
 static struct rc_map_list avermedia_m733a_rm_k6_map = {
 	.map = {
-		.scan    = avermedia_m733a_rm_k6,
-		.size    = ARRAY_SIZE(avermedia_m733a_rm_k6),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_AVERMEDIA_M733A_RM_K6,
+		.scan     = avermedia_m733a_rm_k6,
+		.size     = ARRAY_SIZE(avermedia_m733a_rm_k6),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_AVERMEDIA_M733A_RM_K6,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-avermedia-rm-ks.c b/drivers/media/rc/keymaps/rc-avermedia-rm-ks.c
index 2583400ca1b4..5d92d36d9174 100644
--- a/drivers/media/rc/keymaps/rc-avermedia-rm-ks.c
+++ b/drivers/media/rc/keymaps/rc-avermedia-rm-ks.c
@@ -56,10 +56,10 @@ static struct rc_map_table avermedia_rm_ks[] = {
 
 static struct rc_map_list avermedia_rm_ks_map = {
 	.map = {
-		.scan    = avermedia_rm_ks,
-		.size    = ARRAY_SIZE(avermedia_rm_ks),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_AVERMEDIA_RM_KS,
+		.scan     = avermedia_rm_ks,
+		.size     = ARRAY_SIZE(avermedia_rm_ks),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_AVERMEDIA_RM_KS,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-avermedia.c b/drivers/media/rc/keymaps/rc-avermedia.c
index 3f68fbecc188..6503f11c7df5 100644
--- a/drivers/media/rc/keymaps/rc-avermedia.c
+++ b/drivers/media/rc/keymaps/rc-avermedia.c
@@ -63,10 +63,10 @@ static struct rc_map_table avermedia[] = {
 
 static struct rc_map_list avermedia_map = {
 	.map = {
-		.scan    = avermedia,
-		.size    = ARRAY_SIZE(avermedia),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_AVERMEDIA,
+		.scan     = avermedia,
+		.size     = ARRAY_SIZE(avermedia),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_AVERMEDIA,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-avertv-303.c b/drivers/media/rc/keymaps/rc-avertv-303.c
index c35bc5b835c4..fbdd7ada57ce 100644
--- a/drivers/media/rc/keymaps/rc-avertv-303.c
+++ b/drivers/media/rc/keymaps/rc-avertv-303.c
@@ -62,10 +62,10 @@ static struct rc_map_table avertv_303[] = {
 
 static struct rc_map_list avertv_303_map = {
 	.map = {
-		.scan    = avertv_303,
-		.size    = ARRAY_SIZE(avertv_303),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_AVERTV_303,
+		.scan     = avertv_303,
+		.size     = ARRAY_SIZE(avertv_303),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_AVERTV_303,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-azurewave-ad-tu700.c b/drivers/media/rc/keymaps/rc-azurewave-ad-tu700.c
index ea7f2d0f31eb..18d7dcb869b0 100644
--- a/drivers/media/rc/keymaps/rc-azurewave-ad-tu700.c
+++ b/drivers/media/rc/keymaps/rc-azurewave-ad-tu700.c
@@ -79,10 +79,10 @@ static struct rc_map_table azurewave_ad_tu700[] = {
 
 static struct rc_map_list azurewave_ad_tu700_map = {
 	.map = {
-		.scan    = azurewave_ad_tu700,
-		.size    = ARRAY_SIZE(azurewave_ad_tu700),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_AZUREWAVE_AD_TU700,
+		.scan     = azurewave_ad_tu700,
+		.size     = ARRAY_SIZE(azurewave_ad_tu700),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_AZUREWAVE_AD_TU700,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-behold-columbus.c b/drivers/media/rc/keymaps/rc-behold-columbus.c
index 1fc344e9daa7..d256743be998 100644
--- a/drivers/media/rc/keymaps/rc-behold-columbus.c
+++ b/drivers/media/rc/keymaps/rc-behold-columbus.c
@@ -85,10 +85,10 @@ static struct rc_map_table behold_columbus[] = {
 
 static struct rc_map_list behold_columbus_map = {
 	.map = {
-		.scan    = behold_columbus,
-		.size    = ARRAY_SIZE(behold_columbus),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_BEHOLD_COLUMBUS,
+		.scan     = behold_columbus,
+		.size     = ARRAY_SIZE(behold_columbus),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_BEHOLD_COLUMBUS,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-behold.c b/drivers/media/rc/keymaps/rc-behold.c
index 520a96f2ff86..93dc795adc67 100644
--- a/drivers/media/rc/keymaps/rc-behold.c
+++ b/drivers/media/rc/keymaps/rc-behold.c
@@ -118,10 +118,10 @@ static struct rc_map_table behold[] = {
 
 static struct rc_map_list behold_map = {
 	.map = {
-		.scan    = behold,
-		.size    = ARRAY_SIZE(behold),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_BEHOLD,
+		.scan     = behold,
+		.size     = ARRAY_SIZE(behold),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_BEHOLD,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-budget-ci-old.c b/drivers/media/rc/keymaps/rc-budget-ci-old.c
index b196a5f436a3..81ea1424d9e5 100644
--- a/drivers/media/rc/keymaps/rc-budget-ci-old.c
+++ b/drivers/media/rc/keymaps/rc-budget-ci-old.c
@@ -70,10 +70,10 @@ static struct rc_map_table budget_ci_old[] = {
 
 static struct rc_map_list budget_ci_old_map = {
 	.map = {
-		.scan    = budget_ci_old,
-		.size    = ARRAY_SIZE(budget_ci_old),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_BUDGET_CI_OLD,
+		.scan     = budget_ci_old,
+		.size     = ARRAY_SIZE(budget_ci_old),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_BUDGET_CI_OLD,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-cec.c b/drivers/media/rc/keymaps/rc-cec.c
index 354c8e724b8e..76d34abb7c85 100644
--- a/drivers/media/rc/keymaps/rc-cec.c
+++ b/drivers/media/rc/keymaps/rc-cec.c
@@ -160,7 +160,7 @@ static struct rc_map_list cec_map = {
 	.map = {
 		.scan		= cec,
 		.size		= ARRAY_SIZE(cec),
-		.rc_type	= RC_TYPE_CEC,
+		.rc_proto	= RC_PROTO_CEC,
 		.name		= RC_MAP_CEC,
 	}
 };
diff --git a/drivers/media/rc/keymaps/rc-cinergy-1400.c b/drivers/media/rc/keymaps/rc-cinergy-1400.c
index a099c080bf8c..bcb96b3dda85 100644
--- a/drivers/media/rc/keymaps/rc-cinergy-1400.c
+++ b/drivers/media/rc/keymaps/rc-cinergy-1400.c
@@ -61,10 +61,10 @@ static struct rc_map_table cinergy_1400[] = {
 
 static struct rc_map_list cinergy_1400_map = {
 	.map = {
-		.scan    = cinergy_1400,
-		.size    = ARRAY_SIZE(cinergy_1400),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_CINERGY_1400,
+		.scan     = cinergy_1400,
+		.size     = ARRAY_SIZE(cinergy_1400),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_CINERGY_1400,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-cinergy.c b/drivers/media/rc/keymaps/rc-cinergy.c
index b0f4328bdd6f..fd56c402aae5 100644
--- a/drivers/media/rc/keymaps/rc-cinergy.c
+++ b/drivers/media/rc/keymaps/rc-cinergy.c
@@ -55,10 +55,10 @@ static struct rc_map_table cinergy[] = {
 
 static struct rc_map_list cinergy_map = {
 	.map = {
-		.scan    = cinergy,
-		.size    = ARRAY_SIZE(cinergy),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_CINERGY,
+		.scan     = cinergy,
+		.size     = ARRAY_SIZE(cinergy),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_CINERGY,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-d680-dmb.c b/drivers/media/rc/keymaps/rc-d680-dmb.c
index bb5745d29d8a..2c94b9d88b67 100644
--- a/drivers/media/rc/keymaps/rc-d680-dmb.c
+++ b/drivers/media/rc/keymaps/rc-d680-dmb.c
@@ -51,10 +51,10 @@ static struct rc_map_table rc_map_d680_dmb_table[] = {
 
 static struct rc_map_list d680_dmb_map = {
 	.map = {
-		.scan    = rc_map_d680_dmb_table,
-		.size    = ARRAY_SIZE(rc_map_d680_dmb_table),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_D680_DMB,
+		.scan     = rc_map_d680_dmb_table,
+		.size     = ARRAY_SIZE(rc_map_d680_dmb_table),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_D680_DMB,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-delock-61959.c b/drivers/media/rc/keymaps/rc-delock-61959.c
index 01bed864f09d..62de69d78d92 100644
--- a/drivers/media/rc/keymaps/rc-delock-61959.c
+++ b/drivers/media/rc/keymaps/rc-delock-61959.c
@@ -58,10 +58,10 @@ static struct rc_map_table delock_61959[] = {
 
 static struct rc_map_list delock_61959_map = {
 	.map = {
-		.scan    = delock_61959,
-		.size    = ARRAY_SIZE(delock_61959),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_DELOCK_61959,
+		.scan     = delock_61959,
+		.size     = ARRAY_SIZE(delock_61959),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_DELOCK_61959,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-dib0700-nec.c b/drivers/media/rc/keymaps/rc-dib0700-nec.c
index a0fa543c9f9e..1b4df106b7b5 100644
--- a/drivers/media/rc/keymaps/rc-dib0700-nec.c
+++ b/drivers/media/rc/keymaps/rc-dib0700-nec.c
@@ -101,10 +101,10 @@ static struct rc_map_table dib0700_nec_table[] = {
 
 static struct rc_map_list dib0700_nec_map = {
 	.map = {
-		.scan    = dib0700_nec_table,
-		.size    = ARRAY_SIZE(dib0700_nec_table),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_DIB0700_NEC_TABLE,
+		.scan     = dib0700_nec_table,
+		.size     = ARRAY_SIZE(dib0700_nec_table),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_DIB0700_NEC_TABLE,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-dib0700-rc5.c b/drivers/media/rc/keymaps/rc-dib0700-rc5.c
index 907941145eb7..b0f8151bb824 100644
--- a/drivers/media/rc/keymaps/rc-dib0700-rc5.c
+++ b/drivers/media/rc/keymaps/rc-dib0700-rc5.c
@@ -212,10 +212,10 @@ static struct rc_map_table dib0700_rc5_table[] = {
 
 static struct rc_map_list dib0700_rc5_map = {
 	.map = {
-		.scan    = dib0700_rc5_table,
-		.size    = ARRAY_SIZE(dib0700_rc5_table),
-		.rc_type = RC_TYPE_RC5,
-		.name    = RC_MAP_DIB0700_RC5_TABLE,
+		.scan     = dib0700_rc5_table,
+		.size     = ARRAY_SIZE(dib0700_rc5_table),
+		.rc_proto = RC_PROTO_RC5,
+		.name     = RC_MAP_DIB0700_RC5_TABLE,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-digitalnow-tinytwin.c b/drivers/media/rc/keymaps/rc-digitalnow-tinytwin.c
index bed78acb9198..01ca8b39359f 100644
--- a/drivers/media/rc/keymaps/rc-digitalnow-tinytwin.c
+++ b/drivers/media/rc/keymaps/rc-digitalnow-tinytwin.c
@@ -75,10 +75,10 @@ static struct rc_map_table digitalnow_tinytwin[] = {
 
 static struct rc_map_list digitalnow_tinytwin_map = {
 	.map = {
-		.scan    = digitalnow_tinytwin,
-		.size    = ARRAY_SIZE(digitalnow_tinytwin),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_DIGITALNOW_TINYTWIN,
+		.scan     = digitalnow_tinytwin,
+		.size     = ARRAY_SIZE(digitalnow_tinytwin),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_DIGITALNOW_TINYTWIN,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-digittrade.c b/drivers/media/rc/keymaps/rc-digittrade.c
index a3b97a1fe223..a54b1d632ca6 100644
--- a/drivers/media/rc/keymaps/rc-digittrade.c
+++ b/drivers/media/rc/keymaps/rc-digittrade.c
@@ -59,10 +59,10 @@ static struct rc_map_table digittrade[] = {
 
 static struct rc_map_list digittrade_map = {
 	.map = {
-		.scan    = digittrade,
-		.size    = ARRAY_SIZE(digittrade),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_DIGITTRADE,
+		.scan     = digittrade,
+		.size     = ARRAY_SIZE(digittrade),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_DIGITTRADE,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-dm1105-nec.c b/drivers/media/rc/keymaps/rc-dm1105-nec.c
index 46e7ae414cc8..c353445d10ed 100644
--- a/drivers/media/rc/keymaps/rc-dm1105-nec.c
+++ b/drivers/media/rc/keymaps/rc-dm1105-nec.c
@@ -53,10 +53,10 @@ static struct rc_map_table dm1105_nec[] = {
 
 static struct rc_map_list dm1105_nec_map = {
 	.map = {
-		.scan    = dm1105_nec,
-		.size    = ARRAY_SIZE(dm1105_nec),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_DM1105_NEC,
+		.scan     = dm1105_nec,
+		.size     = ARRAY_SIZE(dm1105_nec),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_DM1105_NEC,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-dntv-live-dvb-t.c b/drivers/media/rc/keymaps/rc-dntv-live-dvb-t.c
index d2826b46fea2..5bafd5b70f5e 100644
--- a/drivers/media/rc/keymaps/rc-dntv-live-dvb-t.c
+++ b/drivers/media/rc/keymaps/rc-dntv-live-dvb-t.c
@@ -55,10 +55,10 @@ static struct rc_map_table dntv_live_dvb_t[] = {
 
 static struct rc_map_list dntv_live_dvb_t_map = {
 	.map = {
-		.scan    = dntv_live_dvb_t,
-		.size    = ARRAY_SIZE(dntv_live_dvb_t),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_DNTV_LIVE_DVB_T,
+		.scan     = dntv_live_dvb_t,
+		.size     = ARRAY_SIZE(dntv_live_dvb_t),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_DNTV_LIVE_DVB_T,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-dntv-live-dvbt-pro.c b/drivers/media/rc/keymaps/rc-dntv-live-dvbt-pro.c
index 0d74769467b5..360167c8829b 100644
--- a/drivers/media/rc/keymaps/rc-dntv-live-dvbt-pro.c
+++ b/drivers/media/rc/keymaps/rc-dntv-live-dvbt-pro.c
@@ -74,10 +74,10 @@ static struct rc_map_table dntv_live_dvbt_pro[] = {
 
 static struct rc_map_list dntv_live_dvbt_pro_map = {
 	.map = {
-		.scan    = dntv_live_dvbt_pro,
-		.size    = ARRAY_SIZE(dntv_live_dvbt_pro),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_DNTV_LIVE_DVBT_PRO,
+		.scan     = dntv_live_dvbt_pro,
+		.size     = ARRAY_SIZE(dntv_live_dvbt_pro),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_DNTV_LIVE_DVBT_PRO,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-dtt200u.c b/drivers/media/rc/keymaps/rc-dtt200u.c
index 25650e9e4664..c932d8b6c509 100644
--- a/drivers/media/rc/keymaps/rc-dtt200u.c
+++ b/drivers/media/rc/keymaps/rc-dtt200u.c
@@ -35,10 +35,10 @@ static struct rc_map_table dtt200u_table[] = {
 
 static struct rc_map_list dtt200u_map = {
 	.map = {
-		.scan    = dtt200u_table,
-		.size    = ARRAY_SIZE(dtt200u_table),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_DTT200U,
+		.scan     = dtt200u_table,
+		.size     = ARRAY_SIZE(dtt200u_table),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_DTT200U,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-dvbsky.c b/drivers/media/rc/keymaps/rc-dvbsky.c
index c5115a1165d1..d6c0b4c1e20e 100644
--- a/drivers/media/rc/keymaps/rc-dvbsky.c
+++ b/drivers/media/rc/keymaps/rc-dvbsky.c
@@ -54,10 +54,10 @@ static struct rc_map_table rc5_dvbsky[] = {
 
 static struct rc_map_list rc5_dvbsky_map = {
 	.map = {
-		.scan    = rc5_dvbsky,
-		.size    = ARRAY_SIZE(rc5_dvbsky),
-		.rc_type = RC_TYPE_RC5,
-		.name    = RC_MAP_DVBSKY,
+		.scan     = rc5_dvbsky,
+		.size     = ARRAY_SIZE(rc5_dvbsky),
+		.rc_proto = RC_PROTO_RC5,
+		.name     = RC_MAP_DVBSKY,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-dvico-mce.c b/drivers/media/rc/keymaps/rc-dvico-mce.c
index d1e861f4d095..e4cee190b923 100644
--- a/drivers/media/rc/keymaps/rc-dvico-mce.c
+++ b/drivers/media/rc/keymaps/rc-dvico-mce.c
@@ -61,10 +61,10 @@ static struct rc_map_table rc_map_dvico_mce_table[] = {
 
 static struct rc_map_list dvico_mce_map = {
 	.map = {
-		.scan    = rc_map_dvico_mce_table,
-		.size    = ARRAY_SIZE(rc_map_dvico_mce_table),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_DVICO_MCE,
+		.scan     = rc_map_dvico_mce_table,
+		.size     = ARRAY_SIZE(rc_map_dvico_mce_table),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_DVICO_MCE,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-dvico-portable.c b/drivers/media/rc/keymaps/rc-dvico-portable.c
index ac4cb515cbf1..cdd21f54aa61 100644
--- a/drivers/media/rc/keymaps/rc-dvico-portable.c
+++ b/drivers/media/rc/keymaps/rc-dvico-portable.c
@@ -52,10 +52,10 @@ static struct rc_map_table rc_map_dvico_portable_table[] = {
 
 static struct rc_map_list dvico_portable_map = {
 	.map = {
-		.scan    = rc_map_dvico_portable_table,
-		.size    = ARRAY_SIZE(rc_map_dvico_portable_table),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_DVICO_PORTABLE,
+		.scan     = rc_map_dvico_portable_table,
+		.size     = ARRAY_SIZE(rc_map_dvico_portable_table),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_DVICO_PORTABLE,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-em-terratec.c b/drivers/media/rc/keymaps/rc-em-terratec.c
index 7f1e06be175b..18e1a2679c20 100644
--- a/drivers/media/rc/keymaps/rc-em-terratec.c
+++ b/drivers/media/rc/keymaps/rc-em-terratec.c
@@ -46,10 +46,10 @@ static struct rc_map_table em_terratec[] = {
 
 static struct rc_map_list em_terratec_map = {
 	.map = {
-		.scan    = em_terratec,
-		.size    = ARRAY_SIZE(em_terratec),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_EM_TERRATEC,
+		.scan     = em_terratec,
+		.size     = ARRAY_SIZE(em_terratec),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_EM_TERRATEC,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-encore-enltv-fm53.c b/drivers/media/rc/keymaps/rc-encore-enltv-fm53.c
index 4fc3904daf06..72ffd5cb0108 100644
--- a/drivers/media/rc/keymaps/rc-encore-enltv-fm53.c
+++ b/drivers/media/rc/keymaps/rc-encore-enltv-fm53.c
@@ -58,10 +58,10 @@ static struct rc_map_table encore_enltv_fm53[] = {
 
 static struct rc_map_list encore_enltv_fm53_map = {
 	.map = {
-		.scan    = encore_enltv_fm53,
-		.size    = ARRAY_SIZE(encore_enltv_fm53),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_ENCORE_ENLTV_FM53,
+		.scan     = encore_enltv_fm53,
+		.size     = ARRAY_SIZE(encore_enltv_fm53),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_ENCORE_ENLTV_FM53,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-encore-enltv.c b/drivers/media/rc/keymaps/rc-encore-enltv.c
index f1914e23d203..e0381e7aa964 100644
--- a/drivers/media/rc/keymaps/rc-encore-enltv.c
+++ b/drivers/media/rc/keymaps/rc-encore-enltv.c
@@ -89,10 +89,10 @@ static struct rc_map_table encore_enltv[] = {
 
 static struct rc_map_list encore_enltv_map = {
 	.map = {
-		.scan    = encore_enltv,
-		.size    = ARRAY_SIZE(encore_enltv),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_ENCORE_ENLTV,
+		.scan     = encore_enltv,
+		.size     = ARRAY_SIZE(encore_enltv),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_ENCORE_ENLTV,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-encore-enltv2.c b/drivers/media/rc/keymaps/rc-encore-enltv2.c
index 9c6c55240d18..e9b0bfba319c 100644
--- a/drivers/media/rc/keymaps/rc-encore-enltv2.c
+++ b/drivers/media/rc/keymaps/rc-encore-enltv2.c
@@ -67,10 +67,10 @@ static struct rc_map_table encore_enltv2[] = {
 
 static struct rc_map_list encore_enltv2_map = {
 	.map = {
-		.scan    = encore_enltv2,
-		.size    = ARRAY_SIZE(encore_enltv2),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_ENCORE_ENLTV2,
+		.scan     = encore_enltv2,
+		.size     = ARRAY_SIZE(encore_enltv2),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_ENCORE_ENLTV2,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-evga-indtube.c b/drivers/media/rc/keymaps/rc-evga-indtube.c
index 2370d2a3deb6..b77c5e908668 100644
--- a/drivers/media/rc/keymaps/rc-evga-indtube.c
+++ b/drivers/media/rc/keymaps/rc-evga-indtube.c
@@ -38,10 +38,10 @@ static struct rc_map_table evga_indtube[] = {
 
 static struct rc_map_list evga_indtube_map = {
 	.map = {
-		.scan    = evga_indtube,
-		.size    = ARRAY_SIZE(evga_indtube),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_EVGA_INDTUBE,
+		.scan     = evga_indtube,
+		.size     = ARRAY_SIZE(evga_indtube),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_EVGA_INDTUBE,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-eztv.c b/drivers/media/rc/keymaps/rc-eztv.c
index b5c96ed84376..5013b3b2aa93 100644
--- a/drivers/media/rc/keymaps/rc-eztv.c
+++ b/drivers/media/rc/keymaps/rc-eztv.c
@@ -73,10 +73,10 @@ static struct rc_map_table eztv[] = {
 
 static struct rc_map_list eztv_map = {
 	.map = {
-		.scan    = eztv,
-		.size    = ARRAY_SIZE(eztv),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_EZTV,
+		.scan     = eztv,
+		.size     = ARRAY_SIZE(eztv),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_EZTV,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-flydvb.c b/drivers/media/rc/keymaps/rc-flydvb.c
index 25cb89fac03c..418b32521273 100644
--- a/drivers/media/rc/keymaps/rc-flydvb.c
+++ b/drivers/media/rc/keymaps/rc-flydvb.c
@@ -54,10 +54,10 @@ static struct rc_map_table flydvb[] = {
 
 static struct rc_map_list flydvb_map = {
 	.map = {
-		.scan    = flydvb,
-		.size    = ARRAY_SIZE(flydvb),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_FLYDVB,
+		.scan     = flydvb,
+		.size     = ARRAY_SIZE(flydvb),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_FLYDVB,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-flyvideo.c b/drivers/media/rc/keymaps/rc-flyvideo.c
index e71377dd0534..93fb87ecf061 100644
--- a/drivers/media/rc/keymaps/rc-flyvideo.c
+++ b/drivers/media/rc/keymaps/rc-flyvideo.c
@@ -47,10 +47,10 @@ static struct rc_map_table flyvideo[] = {
 
 static struct rc_map_list flyvideo_map = {
 	.map = {
-		.scan    = flyvideo,
-		.size    = ARRAY_SIZE(flyvideo),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_FLYVIDEO,
+		.scan     = flyvideo,
+		.size     = ARRAY_SIZE(flyvideo),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_FLYVIDEO,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-fusionhdtv-mce.c b/drivers/media/rc/keymaps/rc-fusionhdtv-mce.c
index cf0608dc83d5..9ed3f749262b 100644
--- a/drivers/media/rc/keymaps/rc-fusionhdtv-mce.c
+++ b/drivers/media/rc/keymaps/rc-fusionhdtv-mce.c
@@ -75,10 +75,10 @@ static struct rc_map_table fusionhdtv_mce[] = {
 
 static struct rc_map_list fusionhdtv_mce_map = {
 	.map = {
-		.scan    = fusionhdtv_mce,
-		.size    = ARRAY_SIZE(fusionhdtv_mce),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_FUSIONHDTV_MCE,
+		.scan     = fusionhdtv_mce,
+		.size     = ARRAY_SIZE(fusionhdtv_mce),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_FUSIONHDTV_MCE,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-gadmei-rm008z.c b/drivers/media/rc/keymaps/rc-gadmei-rm008z.c
index 03575bdb2eca..3443b721d092 100644
--- a/drivers/media/rc/keymaps/rc-gadmei-rm008z.c
+++ b/drivers/media/rc/keymaps/rc-gadmei-rm008z.c
@@ -58,10 +58,10 @@ static struct rc_map_table gadmei_rm008z[] = {
 
 static struct rc_map_list gadmei_rm008z_map = {
 	.map = {
-		.scan    = gadmei_rm008z,
-		.size    = ARRAY_SIZE(gadmei_rm008z),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_GADMEI_RM008Z,
+		.scan     = gadmei_rm008z,
+		.size     = ARRAY_SIZE(gadmei_rm008z),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_GADMEI_RM008Z,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-geekbox.c b/drivers/media/rc/keymaps/rc-geekbox.c
index affc4c481888..4aa1b54bb52e 100644
--- a/drivers/media/rc/keymaps/rc-geekbox.c
+++ b/drivers/media/rc/keymaps/rc-geekbox.c
@@ -31,10 +31,10 @@ static struct rc_map_table geekbox[] = {
 
 static struct rc_map_list geekbox_map = {
 	.map = {
-		.scan    = geekbox,
-		.size    = ARRAY_SIZE(geekbox),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_GEEKBOX,
+		.scan     = geekbox,
+		.size     = ARRAY_SIZE(geekbox),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_GEEKBOX,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-genius-tvgo-a11mce.c b/drivers/media/rc/keymaps/rc-genius-tvgo-a11mce.c
index b2ab13b0dcb1..d140e8d45bcc 100644
--- a/drivers/media/rc/keymaps/rc-genius-tvgo-a11mce.c
+++ b/drivers/media/rc/keymaps/rc-genius-tvgo-a11mce.c
@@ -61,10 +61,10 @@ static struct rc_map_table genius_tvgo_a11mce[] = {
 
 static struct rc_map_list genius_tvgo_a11mce_map = {
 	.map = {
-		.scan    = genius_tvgo_a11mce,
-		.size    = ARRAY_SIZE(genius_tvgo_a11mce),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_GENIUS_TVGO_A11MCE,
+		.scan     = genius_tvgo_a11mce,
+		.size     = ARRAY_SIZE(genius_tvgo_a11mce),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_GENIUS_TVGO_A11MCE,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-gotview7135.c b/drivers/media/rc/keymaps/rc-gotview7135.c
index 229a36ac7f0a..51230fbb52ba 100644
--- a/drivers/media/rc/keymaps/rc-gotview7135.c
+++ b/drivers/media/rc/keymaps/rc-gotview7135.c
@@ -56,10 +56,10 @@ static struct rc_map_table gotview7135[] = {
 
 static struct rc_map_list gotview7135_map = {
 	.map = {
-		.scan    = gotview7135,
-		.size    = ARRAY_SIZE(gotview7135),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_GOTVIEW7135,
+		.scan     = gotview7135,
+		.size     = ARRAY_SIZE(gotview7135),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_GOTVIEW7135,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-hauppauge.c b/drivers/media/rc/keymaps/rc-hauppauge.c
index 36d57f7c532b..890164b68d64 100644
--- a/drivers/media/rc/keymaps/rc-hauppauge.c
+++ b/drivers/media/rc/keymaps/rc-hauppauge.c
@@ -269,10 +269,10 @@ static struct rc_map_table rc5_hauppauge_new[] = {
 
 static struct rc_map_list rc5_hauppauge_new_map = {
 	.map = {
-		.scan    = rc5_hauppauge_new,
-		.size    = ARRAY_SIZE(rc5_hauppauge_new),
-		.rc_type = RC_TYPE_RC5,
-		.name    = RC_MAP_HAUPPAUGE,
+		.scan     = rc5_hauppauge_new,
+		.size     = ARRAY_SIZE(rc5_hauppauge_new),
+		.rc_proto = RC_PROTO_RC5,
+		.name     = RC_MAP_HAUPPAUGE,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-imon-mce.c b/drivers/media/rc/keymaps/rc-imon-mce.c
index f0da960560b0..6a69ce1451f1 100644
--- a/drivers/media/rc/keymaps/rc-imon-mce.c
+++ b/drivers/media/rc/keymaps/rc-imon-mce.c
@@ -118,11 +118,11 @@ static struct rc_map_table imon_mce[] = {
 
 static struct rc_map_list imon_mce_map = {
 	.map = {
-		.scan    = imon_mce,
-		.size    = ARRAY_SIZE(imon_mce),
+		.scan     = imon_mce,
+		.size     = ARRAY_SIZE(imon_mce),
 		/* its RC6, but w/a hardware decoder */
-		.rc_type = RC_TYPE_RC6_MCE,
-		.name    = RC_MAP_IMON_MCE,
+		.rc_proto = RC_PROTO_RC6_MCE,
+		.name     = RC_MAP_IMON_MCE,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-imon-pad.c b/drivers/media/rc/keymaps/rc-imon-pad.c
index 999c6295c70e..a7296ffbf218 100644
--- a/drivers/media/rc/keymaps/rc-imon-pad.c
+++ b/drivers/media/rc/keymaps/rc-imon-pad.c
@@ -132,11 +132,11 @@ static struct rc_map_table imon_pad[] = {
 
 static struct rc_map_list imon_pad_map = {
 	.map = {
-		.scan    = imon_pad,
-		.size    = ARRAY_SIZE(imon_pad),
+		.scan     = imon_pad,
+		.size     = ARRAY_SIZE(imon_pad),
 		/* actual protocol details unknown, hardware decoder */
-		.rc_type = RC_TYPE_OTHER,
-		.name    = RC_MAP_IMON_PAD,
+		.rc_proto = RC_PROTO_OTHER,
+		.name     = RC_MAP_IMON_PAD,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-iodata-bctv7e.c b/drivers/media/rc/keymaps/rc-iodata-bctv7e.c
index 9ee154cb0c6b..8cf87a15c4f2 100644
--- a/drivers/media/rc/keymaps/rc-iodata-bctv7e.c
+++ b/drivers/media/rc/keymaps/rc-iodata-bctv7e.c
@@ -65,10 +65,10 @@ static struct rc_map_table iodata_bctv7e[] = {
 
 static struct rc_map_list iodata_bctv7e_map = {
 	.map = {
-		.scan    = iodata_bctv7e,
-		.size    = ARRAY_SIZE(iodata_bctv7e),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_IODATA_BCTV7E,
+		.scan     = iodata_bctv7e,
+		.size     = ARRAY_SIZE(iodata_bctv7e),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_IODATA_BCTV7E,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-it913x-v1.c b/drivers/media/rc/keymaps/rc-it913x-v1.c
index 0ac775fd109d..908d14848ae8 100644
--- a/drivers/media/rc/keymaps/rc-it913x-v1.c
+++ b/drivers/media/rc/keymaps/rc-it913x-v1.c
@@ -71,10 +71,10 @@ static struct rc_map_table it913x_v1_rc[] = {
 
 static struct rc_map_list it913x_v1_map = {
 	.map = {
-		.scan    = it913x_v1_rc,
-		.size    = ARRAY_SIZE(it913x_v1_rc),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_IT913X_V1,
+		.scan     = it913x_v1_rc,
+		.size     = ARRAY_SIZE(it913x_v1_rc),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_IT913X_V1,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-it913x-v2.c b/drivers/media/rc/keymaps/rc-it913x-v2.c
index bd42a30ec06f..05ab7fa4f90b 100644
--- a/drivers/media/rc/keymaps/rc-it913x-v2.c
+++ b/drivers/media/rc/keymaps/rc-it913x-v2.c
@@ -70,10 +70,10 @@ static struct rc_map_table it913x_v2_rc[] = {
 
 static struct rc_map_list it913x_v2_map = {
 	.map = {
-		.scan    = it913x_v2_rc,
-		.size    = ARRAY_SIZE(it913x_v2_rc),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_IT913X_V2,
+		.scan     = it913x_v2_rc,
+		.size     = ARRAY_SIZE(it913x_v2_rc),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_IT913X_V2,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-kaiomy.c b/drivers/media/rc/keymaps/rc-kaiomy.c
index 60803a732c08..e791f1e1b43b 100644
--- a/drivers/media/rc/keymaps/rc-kaiomy.c
+++ b/drivers/media/rc/keymaps/rc-kaiomy.c
@@ -64,10 +64,10 @@ static struct rc_map_table kaiomy[] = {
 
 static struct rc_map_list kaiomy_map = {
 	.map = {
-		.scan    = kaiomy,
-		.size    = ARRAY_SIZE(kaiomy),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_KAIOMY,
+		.scan     = kaiomy,
+		.size     = ARRAY_SIZE(kaiomy),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_KAIOMY,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-kworld-315u.c b/drivers/media/rc/keymaps/rc-kworld-315u.c
index ba087eed1ed9..71dce0138f0e 100644
--- a/drivers/media/rc/keymaps/rc-kworld-315u.c
+++ b/drivers/media/rc/keymaps/rc-kworld-315u.c
@@ -60,10 +60,10 @@ static struct rc_map_table kworld_315u[] = {
 
 static struct rc_map_list kworld_315u_map = {
 	.map = {
-		.scan    = kworld_315u,
-		.size    = ARRAY_SIZE(kworld_315u),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_KWORLD_315U,
+		.scan     = kworld_315u,
+		.size     = ARRAY_SIZE(kworld_315u),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_KWORLD_315U,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-kworld-pc150u.c b/drivers/media/rc/keymaps/rc-kworld-pc150u.c
index b92e571f4def..3846059060aa 100644
--- a/drivers/media/rc/keymaps/rc-kworld-pc150u.c
+++ b/drivers/media/rc/keymaps/rc-kworld-pc150u.c
@@ -78,10 +78,10 @@ static struct rc_map_table kworld_pc150u[] = {
 
 static struct rc_map_list kworld_pc150u_map = {
 	.map = {
-		.scan    = kworld_pc150u,
-		.size    = ARRAY_SIZE(kworld_pc150u),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_KWORLD_PC150U,
+		.scan     = kworld_pc150u,
+		.size     = ARRAY_SIZE(kworld_pc150u),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_KWORLD_PC150U,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-kworld-plus-tv-analog.c b/drivers/media/rc/keymaps/rc-kworld-plus-tv-analog.c
index edc868564f99..e0322ed16c94 100644
--- a/drivers/media/rc/keymaps/rc-kworld-plus-tv-analog.c
+++ b/drivers/media/rc/keymaps/rc-kworld-plus-tv-analog.c
@@ -76,10 +76,10 @@ static struct rc_map_table kworld_plus_tv_analog[] = {
 
 static struct rc_map_list kworld_plus_tv_analog_map = {
 	.map = {
-		.scan    = kworld_plus_tv_analog,
-		.size    = ARRAY_SIZE(kworld_plus_tv_analog),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_KWORLD_PLUS_TV_ANALOG,
+		.scan     = kworld_plus_tv_analog,
+		.size     = ARRAY_SIZE(kworld_plus_tv_analog),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_KWORLD_PLUS_TV_ANALOG,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-leadtek-y04g0051.c b/drivers/media/rc/keymaps/rc-leadtek-y04g0051.c
index 03d762d986ee..e534a5601b6d 100644
--- a/drivers/media/rc/keymaps/rc-leadtek-y04g0051.c
+++ b/drivers/media/rc/keymaps/rc-leadtek-y04g0051.c
@@ -76,10 +76,10 @@ static struct rc_map_table leadtek_y04g0051[] = {
 
 static struct rc_map_list leadtek_y04g0051_map = {
 	.map = {
-		.scan    = leadtek_y04g0051,
-		.size    = ARRAY_SIZE(leadtek_y04g0051),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_LEADTEK_Y04G0051,
+		.scan     = leadtek_y04g0051,
+		.size     = ARRAY_SIZE(leadtek_y04g0051),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_LEADTEK_Y04G0051,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-lme2510.c b/drivers/media/rc/keymaps/rc-lme2510.c
index 2b0027c41332..9c93f90f5c2b 100644
--- a/drivers/media/rc/keymaps/rc-lme2510.c
+++ b/drivers/media/rc/keymaps/rc-lme2510.c
@@ -87,10 +87,10 @@ static struct rc_map_table lme2510_rc[] = {
 
 static struct rc_map_list lme2510_map = {
 	.map = {
-		.scan    = lme2510_rc,
-		.size    = ARRAY_SIZE(lme2510_rc),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_LME2510,
+		.scan     = lme2510_rc,
+		.size     = ARRAY_SIZE(lme2510_rc),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_LME2510,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-manli.c b/drivers/media/rc/keymaps/rc-manli.c
index 92424ef2aaa6..da566902a4dd 100644
--- a/drivers/media/rc/keymaps/rc-manli.c
+++ b/drivers/media/rc/keymaps/rc-manli.c
@@ -111,10 +111,10 @@ static struct rc_map_table manli[] = {
 
 static struct rc_map_list manli_map = {
 	.map = {
-		.scan    = manli,
-		.size    = ARRAY_SIZE(manli),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_MANLI,
+		.scan     = manli,
+		.size     = ARRAY_SIZE(manli),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_MANLI,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-medion-x10-digitainer.c b/drivers/media/rc/keymaps/rc-medion-x10-digitainer.c
index 966f9b3c71da..c9973340e546 100644
--- a/drivers/media/rc/keymaps/rc-medion-x10-digitainer.c
+++ b/drivers/media/rc/keymaps/rc-medion-x10-digitainer.c
@@ -98,10 +98,10 @@ static struct rc_map_table medion_x10_digitainer[] = {
 
 static struct rc_map_list medion_x10_digitainer_map = {
 	.map = {
-		.scan    = medion_x10_digitainer,
-		.size    = ARRAY_SIZE(medion_x10_digitainer),
-		.rc_type = RC_TYPE_OTHER,
-		.name    = RC_MAP_MEDION_X10_DIGITAINER,
+		.scan     = medion_x10_digitainer,
+		.size     = ARRAY_SIZE(medion_x10_digitainer),
+		.rc_proto = RC_PROTO_OTHER,
+		.name     = RC_MAP_MEDION_X10_DIGITAINER,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-medion-x10-or2x.c b/drivers/media/rc/keymaps/rc-medion-x10-or2x.c
index b077300ecb5c..103ad88d242c 100644
--- a/drivers/media/rc/keymaps/rc-medion-x10-or2x.c
+++ b/drivers/media/rc/keymaps/rc-medion-x10-or2x.c
@@ -83,10 +83,10 @@ static struct rc_map_table medion_x10_or2x[] = {
 
 static struct rc_map_list medion_x10_or2x_map = {
 	.map = {
-		.scan    = medion_x10_or2x,
-		.size    = ARRAY_SIZE(medion_x10_or2x),
-		.rc_type = RC_TYPE_OTHER,
-		.name    = RC_MAP_MEDION_X10_OR2X,
+		.scan     = medion_x10_or2x,
+		.size     = ARRAY_SIZE(medion_x10_or2x),
+		.rc_proto = RC_PROTO_OTHER,
+		.name     = RC_MAP_MEDION_X10_OR2X,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-medion-x10.c b/drivers/media/rc/keymaps/rc-medion-x10.c
index 479cdb897810..bbffa5dfe420 100644
--- a/drivers/media/rc/keymaps/rc-medion-x10.c
+++ b/drivers/media/rc/keymaps/rc-medion-x10.c
@@ -93,10 +93,10 @@ static struct rc_map_table medion_x10[] = {
 
 static struct rc_map_list medion_x10_map = {
 	.map = {
-		.scan    = medion_x10,
-		.size    = ARRAY_SIZE(medion_x10),
-		.rc_type = RC_TYPE_OTHER,
-		.name    = RC_MAP_MEDION_X10,
+		.scan     = medion_x10,
+		.size     = ARRAY_SIZE(medion_x10),
+		.rc_proto = RC_PROTO_OTHER,
+		.name     = RC_MAP_MEDION_X10,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-msi-digivox-ii.c b/drivers/media/rc/keymaps/rc-msi-digivox-ii.c
index 2fa71d0d72d7..94aa12d4b73c 100644
--- a/drivers/media/rc/keymaps/rc-msi-digivox-ii.c
+++ b/drivers/media/rc/keymaps/rc-msi-digivox-ii.c
@@ -44,10 +44,10 @@ static struct rc_map_table msi_digivox_ii[] = {
 
 static struct rc_map_list msi_digivox_ii_map = {
 	.map = {
-		.scan    = msi_digivox_ii,
-		.size    = ARRAY_SIZE(msi_digivox_ii),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_MSI_DIGIVOX_II,
+		.scan     = msi_digivox_ii,
+		.size     = ARRAY_SIZE(msi_digivox_ii),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_MSI_DIGIVOX_II,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-msi-digivox-iii.c b/drivers/media/rc/keymaps/rc-msi-digivox-iii.c
index 303a0b73175b..8fec0c1dcb12 100644
--- a/drivers/media/rc/keymaps/rc-msi-digivox-iii.c
+++ b/drivers/media/rc/keymaps/rc-msi-digivox-iii.c
@@ -62,10 +62,10 @@ static struct rc_map_table msi_digivox_iii[] = {
 
 static struct rc_map_list msi_digivox_iii_map = {
 	.map = {
-		.scan    = msi_digivox_iii,
-		.size    = ARRAY_SIZE(msi_digivox_iii),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_MSI_DIGIVOX_III,
+		.scan     = msi_digivox_iii,
+		.size     = ARRAY_SIZE(msi_digivox_iii),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_MSI_DIGIVOX_III,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-msi-tvanywhere-plus.c b/drivers/media/rc/keymaps/rc-msi-tvanywhere-plus.c
index fd7a55c56167..dfa0ed1d7667 100644
--- a/drivers/media/rc/keymaps/rc-msi-tvanywhere-plus.c
+++ b/drivers/media/rc/keymaps/rc-msi-tvanywhere-plus.c
@@ -100,10 +100,10 @@ static struct rc_map_table msi_tvanywhere_plus[] = {
 
 static struct rc_map_list msi_tvanywhere_plus_map = {
 	.map = {
-		.scan    = msi_tvanywhere_plus,
-		.size    = ARRAY_SIZE(msi_tvanywhere_plus),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_MSI_TVANYWHERE_PLUS,
+		.scan     = msi_tvanywhere_plus,
+		.size     = ARRAY_SIZE(msi_tvanywhere_plus),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_MSI_TVANYWHERE_PLUS,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-msi-tvanywhere.c b/drivers/media/rc/keymaps/rc-msi-tvanywhere.c
index 4233a8d4d63e..2111816a3f59 100644
--- a/drivers/media/rc/keymaps/rc-msi-tvanywhere.c
+++ b/drivers/media/rc/keymaps/rc-msi-tvanywhere.c
@@ -46,10 +46,10 @@ static struct rc_map_table msi_tvanywhere[] = {
 
 static struct rc_map_list msi_tvanywhere_map = {
 	.map = {
-		.scan    = msi_tvanywhere,
-		.size    = ARRAY_SIZE(msi_tvanywhere),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_MSI_TVANYWHERE,
+		.scan     = msi_tvanywhere,
+		.size     = ARRAY_SIZE(msi_tvanywhere),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_MSI_TVANYWHERE,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-nebula.c b/drivers/media/rc/keymaps/rc-nebula.c
index 4c50f33c7c41..109b6e1a8b1a 100644
--- a/drivers/media/rc/keymaps/rc-nebula.c
+++ b/drivers/media/rc/keymaps/rc-nebula.c
@@ -73,10 +73,10 @@ static struct rc_map_table nebula[] = {
 
 static struct rc_map_list nebula_map = {
 	.map = {
-		.scan    = nebula,
-		.size    = ARRAY_SIZE(nebula),
-		.rc_type = RC_TYPE_RC5,
-		.name    = RC_MAP_NEBULA,
+		.scan     = nebula,
+		.size     = ARRAY_SIZE(nebula),
+		.rc_proto = RC_PROTO_RC5,
+		.name     = RC_MAP_NEBULA,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-nec-terratec-cinergy-xs.c b/drivers/media/rc/keymaps/rc-nec-terratec-cinergy-xs.c
index 292bbad35d21..bb2d3a2962c0 100644
--- a/drivers/media/rc/keymaps/rc-nec-terratec-cinergy-xs.c
+++ b/drivers/media/rc/keymaps/rc-nec-terratec-cinergy-xs.c
@@ -134,10 +134,10 @@ static struct rc_map_table nec_terratec_cinergy_xs[] = {
 
 static struct rc_map_list nec_terratec_cinergy_xs_map = {
 	.map = {
-		.scan    = nec_terratec_cinergy_xs,
-		.size    = ARRAY_SIZE(nec_terratec_cinergy_xs),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_NEC_TERRATEC_CINERGY_XS,
+		.scan     = nec_terratec_cinergy_xs,
+		.size     = ARRAY_SIZE(nec_terratec_cinergy_xs),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_NEC_TERRATEC_CINERGY_XS,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-norwood.c b/drivers/media/rc/keymaps/rc-norwood.c
index ca1b82a2c54f..cd25df336749 100644
--- a/drivers/media/rc/keymaps/rc-norwood.c
+++ b/drivers/media/rc/keymaps/rc-norwood.c
@@ -62,10 +62,10 @@ static struct rc_map_table norwood[] = {
 
 static struct rc_map_list norwood_map = {
 	.map = {
-		.scan    = norwood,
-		.size    = ARRAY_SIZE(norwood),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_NORWOOD,
+		.scan     = norwood,
+		.size     = ARRAY_SIZE(norwood),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_NORWOOD,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-npgtech.c b/drivers/media/rc/keymaps/rc-npgtech.c
index 1fb946024512..140bbc20a764 100644
--- a/drivers/media/rc/keymaps/rc-npgtech.c
+++ b/drivers/media/rc/keymaps/rc-npgtech.c
@@ -57,10 +57,10 @@ static struct rc_map_table npgtech[] = {
 
 static struct rc_map_list npgtech_map = {
 	.map = {
-		.scan    = npgtech,
-		.size    = ARRAY_SIZE(npgtech),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_NPGTECH,
+		.scan     = npgtech,
+		.size     = ARRAY_SIZE(npgtech),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_NPGTECH,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-pctv-sedna.c b/drivers/media/rc/keymaps/rc-pctv-sedna.c
index 5ef01ab3fd50..52b4558b7bd0 100644
--- a/drivers/media/rc/keymaps/rc-pctv-sedna.c
+++ b/drivers/media/rc/keymaps/rc-pctv-sedna.c
@@ -57,10 +57,10 @@ static struct rc_map_table pctv_sedna[] = {
 
 static struct rc_map_list pctv_sedna_map = {
 	.map = {
-		.scan    = pctv_sedna,
-		.size    = ARRAY_SIZE(pctv_sedna),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_PCTV_SEDNA,
+		.scan     = pctv_sedna,
+		.size     = ARRAY_SIZE(pctv_sedna),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_PCTV_SEDNA,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-pinnacle-color.c b/drivers/media/rc/keymaps/rc-pinnacle-color.c
index a218b471a4ca..973c9c34e304 100644
--- a/drivers/media/rc/keymaps/rc-pinnacle-color.c
+++ b/drivers/media/rc/keymaps/rc-pinnacle-color.c
@@ -71,10 +71,10 @@ static struct rc_map_table pinnacle_color[] = {
 
 static struct rc_map_list pinnacle_color_map = {
 	.map = {
-		.scan    = pinnacle_color,
-		.size    = ARRAY_SIZE(pinnacle_color),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_PINNACLE_COLOR,
+		.scan     = pinnacle_color,
+		.size     = ARRAY_SIZE(pinnacle_color),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_PINNACLE_COLOR,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-pinnacle-grey.c b/drivers/media/rc/keymaps/rc-pinnacle-grey.c
index 4a3f467a47a2..22e44b0d2a93 100644
--- a/drivers/media/rc/keymaps/rc-pinnacle-grey.c
+++ b/drivers/media/rc/keymaps/rc-pinnacle-grey.c
@@ -66,10 +66,10 @@ static struct rc_map_table pinnacle_grey[] = {
 
 static struct rc_map_list pinnacle_grey_map = {
 	.map = {
-		.scan    = pinnacle_grey,
-		.size    = ARRAY_SIZE(pinnacle_grey),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_PINNACLE_GREY,
+		.scan     = pinnacle_grey,
+		.size     = ARRAY_SIZE(pinnacle_grey),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_PINNACLE_GREY,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-pinnacle-pctv-hd.c b/drivers/media/rc/keymaps/rc-pinnacle-pctv-hd.c
index e89cc10b68bf..186dcf8e0491 100644
--- a/drivers/media/rc/keymaps/rc-pinnacle-pctv-hd.c
+++ b/drivers/media/rc/keymaps/rc-pinnacle-pctv-hd.c
@@ -47,10 +47,10 @@ static struct rc_map_table pinnacle_pctv_hd[] = {
 
 static struct rc_map_list pinnacle_pctv_hd_map = {
 	.map = {
-		.scan    = pinnacle_pctv_hd,
-		.size    = ARRAY_SIZE(pinnacle_pctv_hd),
-		.rc_type = RC_TYPE_RC5,
-		.name    = RC_MAP_PINNACLE_PCTV_HD,
+		.scan     = pinnacle_pctv_hd,
+		.size     = ARRAY_SIZE(pinnacle_pctv_hd),
+		.rc_proto = RC_PROTO_RC5,
+		.name     = RC_MAP_PINNACLE_PCTV_HD,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-pixelview-002t.c b/drivers/media/rc/keymaps/rc-pixelview-002t.c
index d967c3816fdc..b235ada2e28f 100644
--- a/drivers/media/rc/keymaps/rc-pixelview-002t.c
+++ b/drivers/media/rc/keymaps/rc-pixelview-002t.c
@@ -54,10 +54,10 @@ static struct rc_map_table pixelview_002t[] = {
 
 static struct rc_map_list pixelview_map = {
 	.map = {
-		.scan    = pixelview_002t,
-		.size    = ARRAY_SIZE(pixelview_002t),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_PIXELVIEW_002T,
+		.scan     = pixelview_002t,
+		.size     = ARRAY_SIZE(pixelview_002t),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_PIXELVIEW_002T,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-pixelview-mk12.c b/drivers/media/rc/keymaps/rc-pixelview-mk12.c
index 224d0efaa6e5..453d52d663fe 100644
--- a/drivers/media/rc/keymaps/rc-pixelview-mk12.c
+++ b/drivers/media/rc/keymaps/rc-pixelview-mk12.c
@@ -60,10 +60,10 @@ static struct rc_map_table pixelview_mk12[] = {
 
 static struct rc_map_list pixelview_map = {
 	.map = {
-		.scan    = pixelview_mk12,
-		.size    = ARRAY_SIZE(pixelview_mk12),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_PIXELVIEW_MK12,
+		.scan     = pixelview_mk12,
+		.size     = ARRAY_SIZE(pixelview_mk12),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_PIXELVIEW_MK12,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-pixelview-new.c b/drivers/media/rc/keymaps/rc-pixelview-new.c
index 781d788d6b6d..ef97095ec8f1 100644
--- a/drivers/media/rc/keymaps/rc-pixelview-new.c
+++ b/drivers/media/rc/keymaps/rc-pixelview-new.c
@@ -60,10 +60,10 @@ static struct rc_map_table pixelview_new[] = {
 
 static struct rc_map_list pixelview_new_map = {
 	.map = {
-		.scan    = pixelview_new,
-		.size    = ARRAY_SIZE(pixelview_new),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_PIXELVIEW_NEW,
+		.scan     = pixelview_new,
+		.size     = ARRAY_SIZE(pixelview_new),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_PIXELVIEW_NEW,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-pixelview.c b/drivers/media/rc/keymaps/rc-pixelview.c
index 39e6feaa35a3..cfd8f80d3617 100644
--- a/drivers/media/rc/keymaps/rc-pixelview.c
+++ b/drivers/media/rc/keymaps/rc-pixelview.c
@@ -59,10 +59,10 @@ static struct rc_map_table pixelview[] = {
 
 static struct rc_map_list pixelview_map = {
 	.map = {
-		.scan    = pixelview,
-		.size    = ARRAY_SIZE(pixelview),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_PIXELVIEW,
+		.scan     = pixelview,
+		.size     = ARRAY_SIZE(pixelview),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_PIXELVIEW,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-powercolor-real-angel.c b/drivers/media/rc/keymaps/rc-powercolor-real-angel.c
index e96fa3ab9f4b..b63f82bcf29a 100644
--- a/drivers/media/rc/keymaps/rc-powercolor-real-angel.c
+++ b/drivers/media/rc/keymaps/rc-powercolor-real-angel.c
@@ -58,10 +58,10 @@ static struct rc_map_table powercolor_real_angel[] = {
 
 static struct rc_map_list powercolor_real_angel_map = {
 	.map = {
-		.scan    = powercolor_real_angel,
-		.size    = ARRAY_SIZE(powercolor_real_angel),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_POWERCOLOR_REAL_ANGEL,
+		.scan     = powercolor_real_angel,
+		.size     = ARRAY_SIZE(powercolor_real_angel),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_POWERCOLOR_REAL_ANGEL,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-proteus-2309.c b/drivers/media/rc/keymaps/rc-proteus-2309.c
index eef626ee02df..be34c517e4e1 100644
--- a/drivers/media/rc/keymaps/rc-proteus-2309.c
+++ b/drivers/media/rc/keymaps/rc-proteus-2309.c
@@ -46,10 +46,10 @@ static struct rc_map_table proteus_2309[] = {
 
 static struct rc_map_list proteus_2309_map = {
 	.map = {
-		.scan    = proteus_2309,
-		.size    = ARRAY_SIZE(proteus_2309),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_PROTEUS_2309,
+		.scan     = proteus_2309,
+		.size     = ARRAY_SIZE(proteus_2309),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_PROTEUS_2309,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-purpletv.c b/drivers/media/rc/keymaps/rc-purpletv.c
index cec6fe466829..84c40b97ee00 100644
--- a/drivers/media/rc/keymaps/rc-purpletv.c
+++ b/drivers/media/rc/keymaps/rc-purpletv.c
@@ -58,10 +58,10 @@ static struct rc_map_table purpletv[] = {
 
 static struct rc_map_list purpletv_map = {
 	.map = {
-		.scan    = purpletv,
-		.size    = ARRAY_SIZE(purpletv),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_PURPLETV,
+		.scan     = purpletv,
+		.size     = ARRAY_SIZE(purpletv),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_PURPLETV,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-pv951.c b/drivers/media/rc/keymaps/rc-pv951.c
index 5ac89ce8c053..be190ddebfc4 100644
--- a/drivers/media/rc/keymaps/rc-pv951.c
+++ b/drivers/media/rc/keymaps/rc-pv951.c
@@ -55,10 +55,10 @@ static struct rc_map_table pv951[] = {
 
 static struct rc_map_list pv951_map = {
 	.map = {
-		.scan    = pv951,
-		.size    = ARRAY_SIZE(pv951),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_PV951,
+		.scan     = pv951,
+		.size     = ARRAY_SIZE(pv951),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_PV951,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-rc6-mce.c b/drivers/media/rc/keymaps/rc-rc6-mce.c
index 5be567506bcd..0d87b20a0c43 100644
--- a/drivers/media/rc/keymaps/rc-rc6-mce.c
+++ b/drivers/media/rc/keymaps/rc-rc6-mce.c
@@ -96,10 +96,10 @@ static struct rc_map_table rc6_mce[] = {
 
 static struct rc_map_list rc6_mce_map = {
 	.map = {
-		.scan    = rc6_mce,
-		.size    = ARRAY_SIZE(rc6_mce),
-		.rc_type = RC_TYPE_RC6_MCE,
-		.name    = RC_MAP_RC6_MCE,
+		.scan     = rc6_mce,
+		.size     = ARRAY_SIZE(rc6_mce),
+		.rc_proto = RC_PROTO_RC6_MCE,
+		.name     = RC_MAP_RC6_MCE,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-real-audio-220-32-keys.c b/drivers/media/rc/keymaps/rc-real-audio-220-32-keys.c
index 9f778bd091db..957fa21747ea 100644
--- a/drivers/media/rc/keymaps/rc-real-audio-220-32-keys.c
+++ b/drivers/media/rc/keymaps/rc-real-audio-220-32-keys.c
@@ -55,10 +55,10 @@ static struct rc_map_table real_audio_220_32_keys[] = {
 
 static struct rc_map_list real_audio_220_32_keys_map = {
 	.map = {
-		.scan    = real_audio_220_32_keys,
-		.size    = ARRAY_SIZE(real_audio_220_32_keys),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_REAL_AUDIO_220_32_KEYS,
+		.scan     = real_audio_220_32_keys,
+		.size     = ARRAY_SIZE(real_audio_220_32_keys),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_REAL_AUDIO_220_32_KEYS,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-reddo.c b/drivers/media/rc/keymaps/rc-reddo.c
index b80b336e9284..3b37acc7b144 100644
--- a/drivers/media/rc/keymaps/rc-reddo.c
+++ b/drivers/media/rc/keymaps/rc-reddo.c
@@ -62,10 +62,10 @@ static struct rc_map_table reddo[] = {
 
 static struct rc_map_list reddo_map = {
 	.map = {
-		.scan    = reddo,
-		.size    = ARRAY_SIZE(reddo),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_REDDO,
+		.scan     = reddo,
+		.size     = ARRAY_SIZE(reddo),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_REDDO,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-snapstream-firefly.c b/drivers/media/rc/keymaps/rc-snapstream-firefly.c
index c7f33ec719b4..30630a6f76ac 100644
--- a/drivers/media/rc/keymaps/rc-snapstream-firefly.c
+++ b/drivers/media/rc/keymaps/rc-snapstream-firefly.c
@@ -83,10 +83,10 @@ static struct rc_map_table snapstream_firefly[] = {
 
 static struct rc_map_list snapstream_firefly_map = {
 	.map = {
-		.scan    = snapstream_firefly,
-		.size    = ARRAY_SIZE(snapstream_firefly),
-		.rc_type = RC_TYPE_OTHER,
-		.name    = RC_MAP_SNAPSTREAM_FIREFLY,
+		.scan     = snapstream_firefly,
+		.size     = ARRAY_SIZE(snapstream_firefly),
+		.rc_proto = RC_PROTO_OTHER,
+		.name     = RC_MAP_SNAPSTREAM_FIREFLY,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-streamzap.c b/drivers/media/rc/keymaps/rc-streamzap.c
index 23c061174ed7..b53bca9e4576 100644
--- a/drivers/media/rc/keymaps/rc-streamzap.c
+++ b/drivers/media/rc/keymaps/rc-streamzap.c
@@ -57,10 +57,10 @@ static struct rc_map_table streamzap[] = {
 
 static struct rc_map_list streamzap_map = {
 	.map = {
-		.scan    = streamzap,
-		.size    = ARRAY_SIZE(streamzap),
-		.rc_type = RC_TYPE_RC5_SZ,
-		.name    = RC_MAP_STREAMZAP,
+		.scan     = streamzap,
+		.size     = ARRAY_SIZE(streamzap),
+		.rc_proto = RC_PROTO_RC5_SZ,
+		.name     = RC_MAP_STREAMZAP,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-su3000.c b/drivers/media/rc/keymaps/rc-su3000.c
index 8dbd3e9bc951..d9af7e3c55d9 100644
--- a/drivers/media/rc/keymaps/rc-su3000.c
+++ b/drivers/media/rc/keymaps/rc-su3000.c
@@ -51,10 +51,10 @@ static struct rc_map_table su3000[] = {
 
 static struct rc_map_list su3000_map = {
 	.map = {
-		.scan    = su3000,
-		.size    = ARRAY_SIZE(su3000),
-		.rc_type = RC_TYPE_RC5,
-		.name    = RC_MAP_SU3000,
+		.scan     = su3000,
+		.size     = ARRAY_SIZE(su3000),
+		.rc_proto = RC_PROTO_RC5,
+		.name     = RC_MAP_SU3000,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-tbs-nec.c b/drivers/media/rc/keymaps/rc-tbs-nec.c
index 24ce2a252502..05facc043272 100644
--- a/drivers/media/rc/keymaps/rc-tbs-nec.c
+++ b/drivers/media/rc/keymaps/rc-tbs-nec.c
@@ -52,10 +52,10 @@ static struct rc_map_table tbs_nec[] = {
 
 static struct rc_map_list tbs_nec_map = {
 	.map = {
-		.scan    = tbs_nec,
-		.size    = ARRAY_SIZE(tbs_nec),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_TBS_NEC,
+		.scan     = tbs_nec,
+		.size     = ARRAY_SIZE(tbs_nec),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_TBS_NEC,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-technisat-ts35.c b/drivers/media/rc/keymaps/rc-technisat-ts35.c
index 3328cbefabad..dff7021734ba 100644
--- a/drivers/media/rc/keymaps/rc-technisat-ts35.c
+++ b/drivers/media/rc/keymaps/rc-technisat-ts35.c
@@ -53,10 +53,10 @@ static struct rc_map_table technisat_ts35[] = {
 
 static struct rc_map_list technisat_ts35_map = {
 	.map = {
-		.scan    = technisat_ts35,
-		.size    = ARRAY_SIZE(technisat_ts35),
-		.rc_type = RC_TYPE_UNKNOWN,
-		.name    = RC_MAP_TECHNISAT_TS35,
+		.scan     = technisat_ts35,
+		.size     = ARRAY_SIZE(technisat_ts35),
+		.rc_proto = RC_PROTO_UNKNOWN,
+		.name     = RC_MAP_TECHNISAT_TS35,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-technisat-usb2.c b/drivers/media/rc/keymaps/rc-technisat-usb2.c
index 02c9c243c060..58b3baf5ee96 100644
--- a/drivers/media/rc/keymaps/rc-technisat-usb2.c
+++ b/drivers/media/rc/keymaps/rc-technisat-usb2.c
@@ -66,10 +66,10 @@ static struct rc_map_table technisat_usb2[] = {
 
 static struct rc_map_list technisat_usb2_map = {
 	.map = {
-		.scan    = technisat_usb2,
-		.size    = ARRAY_SIZE(technisat_usb2),
-		.rc_type = RC_TYPE_RC5,
-		.name    = RC_MAP_TECHNISAT_USB2,
+		.scan     = technisat_usb2,
+		.size     = ARRAY_SIZE(technisat_usb2),
+		.rc_proto = RC_PROTO_RC5,
+		.name     = RC_MAP_TECHNISAT_USB2,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-terratec-cinergy-c-pci.c b/drivers/media/rc/keymaps/rc-terratec-cinergy-c-pci.c
index 7958f458527a..7ae88ccf1def 100644
--- a/drivers/media/rc/keymaps/rc-terratec-cinergy-c-pci.c
+++ b/drivers/media/rc/keymaps/rc-terratec-cinergy-c-pci.c
@@ -65,10 +65,10 @@ static struct rc_map_table terratec_cinergy_c_pci[] = {
 
 static struct rc_map_list terratec_cinergy_c_pci_map = {
 	.map = {
-		.scan    = terratec_cinergy_c_pci,
-		.size    = ARRAY_SIZE(terratec_cinergy_c_pci),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_TERRATEC_CINERGY_C_PCI,
+		.scan     = terratec_cinergy_c_pci,
+		.size     = ARRAY_SIZE(terratec_cinergy_c_pci),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_TERRATEC_CINERGY_C_PCI,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-terratec-cinergy-s2-hd.c b/drivers/media/rc/keymaps/rc-terratec-cinergy-s2-hd.c
index 1e096bbda4a0..bf0171b05ac2 100644
--- a/drivers/media/rc/keymaps/rc-terratec-cinergy-s2-hd.c
+++ b/drivers/media/rc/keymaps/rc-terratec-cinergy-s2-hd.c
@@ -63,10 +63,10 @@ static struct rc_map_table terratec_cinergy_s2_hd[] = {
 
 static struct rc_map_list terratec_cinergy_s2_hd_map = {
 	.map = {
-		.scan    = terratec_cinergy_s2_hd,
-		.size    = ARRAY_SIZE(terratec_cinergy_s2_hd),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_TERRATEC_CINERGY_S2_HD,
+		.scan     = terratec_cinergy_s2_hd,
+		.size     = ARRAY_SIZE(terratec_cinergy_s2_hd),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_TERRATEC_CINERGY_S2_HD,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-terratec-cinergy-xs.c b/drivers/media/rc/keymaps/rc-terratec-cinergy-xs.c
index 97eb83ab5a35..3d0f6f7e5bea 100644
--- a/drivers/media/rc/keymaps/rc-terratec-cinergy-xs.c
+++ b/drivers/media/rc/keymaps/rc-terratec-cinergy-xs.c
@@ -69,10 +69,10 @@ static struct rc_map_table terratec_cinergy_xs[] = {
 
 static struct rc_map_list terratec_cinergy_xs_map = {
 	.map = {
-		.scan    = terratec_cinergy_xs,
-		.size    = ARRAY_SIZE(terratec_cinergy_xs),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_TERRATEC_CINERGY_XS,
+		.scan     = terratec_cinergy_xs,
+		.size     = ARRAY_SIZE(terratec_cinergy_xs),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_TERRATEC_CINERGY_XS,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-terratec-slim-2.c b/drivers/media/rc/keymaps/rc-terratec-slim-2.c
index 4c149ef712dc..df57e0a45820 100644
--- a/drivers/media/rc/keymaps/rc-terratec-slim-2.c
+++ b/drivers/media/rc/keymaps/rc-terratec-slim-2.c
@@ -49,10 +49,10 @@ static struct rc_map_table terratec_slim_2[] = {
 
 static struct rc_map_list terratec_slim_2_map = {
 	.map = {
-		.scan    = terratec_slim_2,
-		.size    = ARRAY_SIZE(terratec_slim_2),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_TERRATEC_SLIM_2,
+		.scan     = terratec_slim_2,
+		.size     = ARRAY_SIZE(terratec_slim_2),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_TERRATEC_SLIM_2,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-terratec-slim.c b/drivers/media/rc/keymaps/rc-terratec-slim.c
index 3d8a19cdb5a2..628272c58d65 100644
--- a/drivers/media/rc/keymaps/rc-terratec-slim.c
+++ b/drivers/media/rc/keymaps/rc-terratec-slim.c
@@ -56,10 +56,10 @@ static struct rc_map_table terratec_slim[] = {
 
 static struct rc_map_list terratec_slim_map = {
 	.map = {
-		.scan    = terratec_slim,
-		.size    = ARRAY_SIZE(terratec_slim),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_TERRATEC_SLIM,
+		.scan     = terratec_slim,
+		.size     = ARRAY_SIZE(terratec_slim),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_TERRATEC_SLIM,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-tevii-nec.c b/drivers/media/rc/keymaps/rc-tevii-nec.c
index 38e0c0875596..31f8a0fd1f2c 100644
--- a/drivers/media/rc/keymaps/rc-tevii-nec.c
+++ b/drivers/media/rc/keymaps/rc-tevii-nec.c
@@ -65,10 +65,10 @@ static struct rc_map_table tevii_nec[] = {
 
 static struct rc_map_list tevii_nec_map = {
 	.map = {
-		.scan    = tevii_nec,
-		.size    = ARRAY_SIZE(tevii_nec),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_TEVII_NEC,
+		.scan     = tevii_nec,
+		.size     = ARRAY_SIZE(tevii_nec),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_TEVII_NEC,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-tivo.c b/drivers/media/rc/keymaps/rc-tivo.c
index 5cc1b456e329..1962e33c8f4e 100644
--- a/drivers/media/rc/keymaps/rc-tivo.c
+++ b/drivers/media/rc/keymaps/rc-tivo.c
@@ -75,10 +75,10 @@ static struct rc_map_table tivo[] = {
 
 static struct rc_map_list tivo_map = {
 	.map = {
-		.scan    = tivo,
-		.size    = ARRAY_SIZE(tivo),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_TIVO,
+		.scan     = tivo,
+		.size     = ARRAY_SIZE(tivo),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_TIVO,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-total-media-in-hand-02.c b/drivers/media/rc/keymaps/rc-total-media-in-hand-02.c
index 47270f72ebf0..eeeca142f7b1 100644
--- a/drivers/media/rc/keymaps/rc-total-media-in-hand-02.c
+++ b/drivers/media/rc/keymaps/rc-total-media-in-hand-02.c
@@ -62,10 +62,10 @@ static struct rc_map_table total_media_in_hand_02[] = {
 
 static struct rc_map_list total_media_in_hand_02_map = {
 	.map = {
-		.scan    = total_media_in_hand_02,
-		.size    = ARRAY_SIZE(total_media_in_hand_02),
-		.rc_type = RC_TYPE_RC5,
-		.name    = RC_MAP_TOTAL_MEDIA_IN_HAND_02,
+		.scan     = total_media_in_hand_02,
+		.size     = ARRAY_SIZE(total_media_in_hand_02),
+		.rc_proto = RC_PROTO_RC5,
+		.name     = RC_MAP_TOTAL_MEDIA_IN_HAND_02,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-total-media-in-hand.c b/drivers/media/rc/keymaps/rc-total-media-in-hand.c
index 5b9f9ec13680..bc73bee309d8 100644
--- a/drivers/media/rc/keymaps/rc-total-media-in-hand.c
+++ b/drivers/media/rc/keymaps/rc-total-media-in-hand.c
@@ -62,10 +62,10 @@ static struct rc_map_table total_media_in_hand[] = {
 
 static struct rc_map_list total_media_in_hand_map = {
 	.map = {
-		.scan    = total_media_in_hand,
-		.size    = ARRAY_SIZE(total_media_in_hand),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_TOTAL_MEDIA_IN_HAND,
+		.scan     = total_media_in_hand,
+		.size     = ARRAY_SIZE(total_media_in_hand),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_TOTAL_MEDIA_IN_HAND,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-trekstor.c b/drivers/media/rc/keymaps/rc-trekstor.c
index f9a2e0fabb9f..63f966219342 100644
--- a/drivers/media/rc/keymaps/rc-trekstor.c
+++ b/drivers/media/rc/keymaps/rc-trekstor.c
@@ -57,10 +57,10 @@ static struct rc_map_table trekstor[] = {
 
 static struct rc_map_list trekstor_map = {
 	.map = {
-		.scan    = trekstor,
-		.size    = ARRAY_SIZE(trekstor),
-		.rc_type = RC_TYPE_NEC,
-		.name    = RC_MAP_TREKSTOR,
+		.scan     = trekstor,
+		.size     = ARRAY_SIZE(trekstor),
+		.rc_proto = RC_PROTO_NEC,
+		.name     = RC_MAP_TREKSTOR,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-tt-1500.c b/drivers/media/rc/keymaps/rc-tt-1500.c
index c766d3b2b6b0..374c230705d2 100644
--- a/drivers/media/rc/keymaps/rc-tt-1500.c
+++ b/drivers/media/rc/keymaps/rc-tt-1500.c
@@ -59,10 +59,10 @@ static struct rc_map_table tt_1500[] = {
 
 static struct rc_map_list tt_1500_map = {
 	.map = {
-		.scan    = tt_1500,
-		.size    = ARRAY_SIZE(tt_1500),
-		.rc_type = RC_TYPE_RC5,
-		.name    = RC_MAP_TT_1500,
+		.scan     = tt_1500,
+		.size     = ARRAY_SIZE(tt_1500),
+		.rc_proto = RC_PROTO_RC5,
+		.name     = RC_MAP_TT_1500,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-twinhan-dtv-cab-ci.c b/drivers/media/rc/keymaps/rc-twinhan-dtv-cab-ci.c
index 202500cb3061..240d720d440c 100644
--- a/drivers/media/rc/keymaps/rc-twinhan-dtv-cab-ci.c
+++ b/drivers/media/rc/keymaps/rc-twinhan-dtv-cab-ci.c
@@ -75,10 +75,10 @@ static struct rc_map_table twinhan_dtv_cab_ci[] = {
 
 static struct rc_map_list twinhan_dtv_cab_ci_map = {
 	.map = {
-		.scan    = twinhan_dtv_cab_ci,
-		.size    = ARRAY_SIZE(twinhan_dtv_cab_ci),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_TWINHAN_DTV_CAB_CI,
+		.scan     = twinhan_dtv_cab_ci,
+		.size     = ARRAY_SIZE(twinhan_dtv_cab_ci),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_TWINHAN_DTV_CAB_CI,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-twinhan1027.c b/drivers/media/rc/keymaps/rc-twinhan1027.c
index 509299b90c90..2275b37c61d2 100644
--- a/drivers/media/rc/keymaps/rc-twinhan1027.c
+++ b/drivers/media/rc/keymaps/rc-twinhan1027.c
@@ -64,10 +64,10 @@ static struct rc_map_table twinhan_vp1027[] = {
 
 static struct rc_map_list twinhan_vp1027_map = {
 	.map = {
-		.scan    = twinhan_vp1027,
-		.size    = ARRAY_SIZE(twinhan_vp1027),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_TWINHAN_VP1027_DVBS,
+		.scan     = twinhan_vp1027,
+		.size     = ARRAY_SIZE(twinhan_vp1027),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_TWINHAN_VP1027_DVBS,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-videomate-m1f.c b/drivers/media/rc/keymaps/rc-videomate-m1f.c
index 23ee05e53949..fe02e047bd01 100644
--- a/drivers/media/rc/keymaps/rc-videomate-m1f.c
+++ b/drivers/media/rc/keymaps/rc-videomate-m1f.c
@@ -69,10 +69,10 @@ static struct rc_map_table videomate_k100[] = {
 
 static struct rc_map_list videomate_k100_map = {
 	.map = {
-		.scan    = videomate_k100,
-		.size    = ARRAY_SIZE(videomate_k100),
-		.rc_type = RC_TYPE_UNKNOWN,     /* Legacy IR type */
-		.name    = RC_MAP_VIDEOMATE_K100,
+		.scan     = videomate_k100,
+		.size     = ARRAY_SIZE(videomate_k100),
+		.rc_proto = RC_PROTO_UNKNOWN,     /* Legacy IR type */
+		.name     = RC_MAP_VIDEOMATE_K100,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-videomate-s350.c b/drivers/media/rc/keymaps/rc-videomate-s350.c
index 8a354775a2d8..b4f103269872 100644
--- a/drivers/media/rc/keymaps/rc-videomate-s350.c
+++ b/drivers/media/rc/keymaps/rc-videomate-s350.c
@@ -62,10 +62,10 @@ static struct rc_map_table videomate_s350[] = {
 
 static struct rc_map_list videomate_s350_map = {
 	.map = {
-		.scan    = videomate_s350,
-		.size    = ARRAY_SIZE(videomate_s350),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_VIDEOMATE_S350,
+		.scan     = videomate_s350,
+		.size     = ARRAY_SIZE(videomate_s350),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_VIDEOMATE_S350,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-videomate-tv-pvr.c b/drivers/media/rc/keymaps/rc-videomate-tv-pvr.c
index eb0cda7766c4..c431fdf44057 100644
--- a/drivers/media/rc/keymaps/rc-videomate-tv-pvr.c
+++ b/drivers/media/rc/keymaps/rc-videomate-tv-pvr.c
@@ -64,10 +64,10 @@ static struct rc_map_table videomate_tv_pvr[] = {
 
 static struct rc_map_list videomate_tv_pvr_map = {
 	.map = {
-		.scan    = videomate_tv_pvr,
-		.size    = ARRAY_SIZE(videomate_tv_pvr),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_VIDEOMATE_TV_PVR,
+		.scan     = videomate_tv_pvr,
+		.size     = ARRAY_SIZE(videomate_tv_pvr),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_VIDEOMATE_TV_PVR,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-winfast-usbii-deluxe.c b/drivers/media/rc/keymaps/rc-winfast-usbii-deluxe.c
index c1dd598e828e..5a437e61bd5d 100644
--- a/drivers/media/rc/keymaps/rc-winfast-usbii-deluxe.c
+++ b/drivers/media/rc/keymaps/rc-winfast-usbii-deluxe.c
@@ -59,10 +59,10 @@ static struct rc_map_table winfast_usbii_deluxe[] = {
 
 static struct rc_map_list winfast_usbii_deluxe_map = {
 	.map = {
-		.scan    = winfast_usbii_deluxe,
-		.size    = ARRAY_SIZE(winfast_usbii_deluxe),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_WINFAST_USBII_DELUXE,
+		.scan     = winfast_usbii_deluxe,
+		.size     = ARRAY_SIZE(winfast_usbii_deluxe),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_WINFAST_USBII_DELUXE,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-winfast.c b/drivers/media/rc/keymaps/rc-winfast.c
index 8a779da1e973..53685d1f9a47 100644
--- a/drivers/media/rc/keymaps/rc-winfast.c
+++ b/drivers/media/rc/keymaps/rc-winfast.c
@@ -79,10 +79,10 @@ static struct rc_map_table winfast[] = {
 
 static struct rc_map_list winfast_map = {
 	.map = {
-		.scan    = winfast,
-		.size    = ARRAY_SIZE(winfast),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_WINFAST,
+		.scan     = winfast,
+		.size     = ARRAY_SIZE(winfast),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_WINFAST,
 	}
 };
 
diff --git a/drivers/media/rc/keymaps/rc-zx-irdec.c b/drivers/media/rc/keymaps/rc-zx-irdec.c
new file mode 100644
index 000000000000..5bf3ab002afc
--- /dev/null
+++ b/drivers/media/rc/keymaps/rc-zx-irdec.c
@@ -0,0 +1,79 @@
+/*
+ * Copyright (C) 2017 Sanechips Technology Co., Ltd.
+ * Copyright 2017 Linaro Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/module.h>
+#include <media/rc-map.h>
+
+static struct rc_map_table zx_irdec_table[] = {
+	{ 0x01, KEY_1 },
+	{ 0x02, KEY_2 },
+	{ 0x03, KEY_3 },
+	{ 0x04, KEY_4 },
+	{ 0x05, KEY_5 },
+	{ 0x06, KEY_6 },
+	{ 0x07, KEY_7 },
+	{ 0x08, KEY_8 },
+	{ 0x09, KEY_9 },
+	{ 0x31, KEY_0 },
+	{ 0x16, KEY_DELETE },
+	{ 0x0a, KEY_MODE },		/* Input method */
+	{ 0x0c, KEY_VOLUMEUP },
+	{ 0x18, KEY_VOLUMEDOWN },
+	{ 0x0b, KEY_CHANNELUP },
+	{ 0x15, KEY_CHANNELDOWN },
+	{ 0x0d, KEY_PAGEUP },
+	{ 0x13, KEY_PAGEDOWN },
+	{ 0x46, KEY_FASTFORWARD },
+	{ 0x43, KEY_REWIND },
+	{ 0x44, KEY_PLAYPAUSE },
+	{ 0x45, KEY_STOP },
+	{ 0x49, KEY_OK },
+	{ 0x47, KEY_UP },
+	{ 0x4b, KEY_DOWN },
+	{ 0x48, KEY_LEFT },
+	{ 0x4a, KEY_RIGHT },
+	{ 0x4d, KEY_MENU },
+	{ 0x56, KEY_APPSELECT },	/* Application */
+	{ 0x4c, KEY_BACK },
+	{ 0x1e, KEY_INFO },
+	{ 0x4e, KEY_F1 },
+	{ 0x4f, KEY_F2 },
+	{ 0x50, KEY_F3 },
+	{ 0x51, KEY_F4 },
+	{ 0x1c, KEY_AUDIO },
+	{ 0x12, KEY_MUTE },
+	{ 0x11, KEY_DOT },		/* Location */
+	{ 0x1d, KEY_SETUP },
+	{ 0x40, KEY_POWER },
+};
+
+static struct rc_map_list zx_irdec_map = {
+	.map = {
+		.scan = zx_irdec_table,
+		.size = ARRAY_SIZE(zx_irdec_table),
+		.rc_proto = RC_PROTO_NEC,
+		.name = RC_MAP_ZX_IRDEC,
+	}
+};
+
+static int __init init_rc_map_zx_irdec(void)
+{
+	return rc_map_register(&zx_irdec_map);
+}
+
+static void __exit exit_rc_map_zx_irdec(void)
+{
+	rc_map_unregister(&zx_irdec_map);
+}
+
+module_init(init_rc_map_zx_irdec)
+module_exit(exit_rc_map_zx_irdec)
+
+MODULE_AUTHOR("Shawn Guo <shawn.guo@linaro.org>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/rc/lirc_dev.c b/drivers/media/rc/lirc_dev.c
index db1e7b70c998..9080e39ea391 100644
--- a/drivers/media/rc/lirc_dev.c
+++ b/drivers/media/rc/lirc_dev.c
@@ -59,6 +59,8 @@ static void lirc_release(struct device *ld)
 {
 	struct irctl *ir = container_of(ld, struct irctl, dev);
 
+	put_device(ir->dev.parent);
+
 	if (ir->buf_internal) {
 		lirc_buffer_free(ir->buf);
 		kfree(ir->buf);
@@ -218,6 +220,8 @@ int lirc_register_driver(struct lirc_driver *d)
 
 	mutex_unlock(&lirc_dev_lock);
 
+	get_device(ir->dev.parent);
+
 	dev_info(ir->d.dev, "lirc_dev: driver %s registered at minor = %d\n",
 		 ir->d.name, ir->d.minor);
 
diff --git a/drivers/media/rc/mceusb.c b/drivers/media/rc/mceusb.c
index eb130694bbb8..bf7aaff3aa37 100644
--- a/drivers/media/rc/mceusb.c
+++ b/drivers/media/rc/mceusb.c
@@ -538,12 +538,12 @@ static int mceusb_cmd_datasize(u8 cmd, u8 subcmd)
 	return datasize;
 }
 
-static void mceusb_dev_printdata(struct mceusb_dev *ir, char *buf,
-				 int buf_len, int offset, int len, bool out)
+static void mceusb_dev_printdata(struct mceusb_dev *ir, u8 *buf, int buf_len,
+				 int offset, int len, bool out)
 {
 #if defined(DEBUG) || defined(CONFIG_DYNAMIC_DEBUG)
 	char *inout;
-	u8 cmd, subcmd, data1, data2, data3, data4;
+	u8 cmd, subcmd, *data;
 	struct device *dev = ir->dev;
 	int start, skip = 0;
 	u32 carrier, period;
@@ -564,17 +564,14 @@ static void mceusb_dev_printdata(struct mceusb_dev *ir, char *buf,
 	start  = offset + skip;
 	cmd    = buf[start] & 0xff;
 	subcmd = buf[start + 1] & 0xff;
-	data1  = buf[start + 2] & 0xff;
-	data2  = buf[start + 3] & 0xff;
-	data3  = buf[start + 4] & 0xff;
-	data4  = buf[start + 5] & 0xff;
+	data = buf + start + 2;
 
 	switch (cmd) {
 	case MCE_CMD_NULL:
 		if (subcmd == MCE_CMD_NULL)
 			break;
 		if ((subcmd == MCE_CMD_PORT_SYS) &&
-		    (data1 == MCE_CMD_RESUME))
+		    (data[0] == MCE_CMD_RESUME))
 			dev_dbg(dev, "Device resume requested");
 		else
 			dev_dbg(dev, "Unknown command 0x%02x 0x%02x",
@@ -585,7 +582,7 @@ static void mceusb_dev_printdata(struct mceusb_dev *ir, char *buf,
 		case MCE_RSP_EQEMVER:
 			if (!out)
 				dev_dbg(dev, "Emulator interface version %x",
-					 data1);
+					 data[0]);
 			break;
 		case MCE_CMD_G_REVISION:
 			if (len == 2)
@@ -603,13 +600,13 @@ static void mceusb_dev_printdata(struct mceusb_dev *ir, char *buf,
 		case MCE_RSP_EQWAKEVERSION:
 			if (!out)
 				dev_dbg(dev, "Wake version, proto: 0x%02x, payload: 0x%02x, address: 0x%02x, version: 0x%02x",
-					 data1, data2, data3, data4);
+					data[0], data[1], data[2], data[3]);
 			break;
 		case MCE_RSP_GETPORTSTATUS:
 			if (!out)
 				/* We use data1 + 1 here, to match hw labels */
 				dev_dbg(dev, "TX port %d: blaster is%s connected",
-					 data1 + 1, data4 ? " not" : "");
+					 data[0] + 1, data[3] ? " not" : "");
 			break;
 		case MCE_CMD_FLASHLED:
 			dev_dbg(dev, "Attempting to flash LED");
@@ -630,11 +627,11 @@ static void mceusb_dev_printdata(struct mceusb_dev *ir, char *buf,
 			break;
 		case MCE_CMD_UNKNOWN:
 			dev_dbg(dev, "Resp to 9f 05 of 0x%02x 0x%02x",
-				 data1, data2);
+				data[0], data[1]);
 			break;
 		case MCE_RSP_EQIRCFS:
-			period = DIV_ROUND_CLOSEST(
-					(1U << data1 * 2) * (data2 + 1), 10);
+			period = DIV_ROUND_CLOSEST((1U << data[0] * 2) *
+						   (data[1] + 1), 10);
 			if (!period)
 				break;
 			carrier = (1000 * 1000) / period;
@@ -646,11 +643,12 @@ static void mceusb_dev_printdata(struct mceusb_dev *ir, char *buf,
 			break;
 		case MCE_RSP_EQIRTXPORTS:
 			dev_dbg(dev, "%s transmit blaster mask of 0x%02x",
-				 inout, data1);
+				 inout, data[0]);
 			break;
 		case MCE_RSP_EQIRTIMEOUT:
 			/* value is in units of 50us, so x*50/1000 ms */
-			period = ((data1 << 8) | data2) * MCE_TIME_UNIT / 1000;
+			period = ((data[0] << 8) | data[1]) *
+				  MCE_TIME_UNIT / 1000;
 			dev_dbg(dev, "%s receive timeout of %d ms",
 				 inout, period);
 			break;
@@ -662,7 +660,7 @@ static void mceusb_dev_printdata(struct mceusb_dev *ir, char *buf,
 			break;
 		case MCE_RSP_EQIRRXPORTEN:
 			dev_dbg(dev, "%s %s-range receive sensor in use",
-				 inout, data1 == 0x02 ? "short" : "long");
+				 inout, data[0] == 0x02 ? "short" : "long");
 			break;
 		case MCE_CMD_GETIRRXPORTEN:
 		/* aka MCE_RSP_EQIRRXCFCNT */
@@ -670,13 +668,13 @@ static void mceusb_dev_printdata(struct mceusb_dev *ir, char *buf,
 				dev_dbg(dev, "Get receive sensor");
 			else if (ir->learning_enabled)
 				dev_dbg(dev, "RX pulse count: %d",
-					 ((data1 << 8) | data2));
+					((data[0] << 8) | data[1]));
 			break;
 		case MCE_RSP_EQIRNUMPORTS:
 			if (out)
 				break;
 			dev_dbg(dev, "Num TX ports: %x, num RX ports: %x",
-				 data1, data2);
+				data[0], data[1]);
 			break;
 		case MCE_RSP_CMD_ILLEGAL:
 			dev_dbg(dev, "Illegal PORT_IR command");
@@ -1264,12 +1262,12 @@ static struct rc_dev *mceusb_init_rc_dev(struct mceusb_dev *ir)
 
 	usb_make_path(ir->usbdev, ir->phys, sizeof(ir->phys));
 
-	rc->input_name = ir->name;
+	rc->device_name = ir->name;
 	rc->input_phys = ir->phys;
 	usb_to_input_id(ir->usbdev, &rc->input_id);
 	rc->dev.parent = dev;
 	rc->priv = ir;
-	rc->allowed_protocols = RC_BIT_ALL_IR_DECODER;
+	rc->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
 	rc->timeout = MS_TO_NS(100);
 	if (!ir->flags.no_tx) {
 		rc->s_tx_mask = mceusb_set_tx_mask;
diff --git a/drivers/media/rc/meson-ir.c b/drivers/media/rc/meson-ir.c
index 65566d569cb1..f2204eb77e2a 100644
--- a/drivers/media/rc/meson-ir.c
+++ b/drivers/media/rc/meson-ir.c
@@ -138,12 +138,12 @@ static int meson_ir_probe(struct platform_device *pdev)
 	}
 
 	ir->rc->priv = ir;
-	ir->rc->input_name = DRIVER_NAME;
+	ir->rc->device_name = DRIVER_NAME;
 	ir->rc->input_phys = DRIVER_NAME "/input0";
 	ir->rc->input_id.bustype = BUS_HOST;
 	map_name = of_get_property(node, "linux,rc-map-name", NULL);
 	ir->rc->map_name = map_name ? map_name : RC_MAP_EMPTY;
-	ir->rc->allowed_protocols = RC_BIT_ALL_IR_DECODER;
+	ir->rc->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
 	ir->rc->rx_resolution = US_TO_NS(MESON_TRATE);
 	ir->rc->timeout = MS_TO_NS(200);
 	ir->rc->driver_name = DRIVER_NAME;
diff --git a/drivers/media/rc/mtk-cir.c b/drivers/media/rc/mtk-cir.c
index f1e164e441e8..e88eb64e8e69 100644
--- a/drivers/media/rc/mtk-cir.c
+++ b/drivers/media/rc/mtk-cir.c
@@ -25,35 +25,28 @@
 
 /* Register to enable PWM and IR */
 #define MTK_CONFIG_HIGH_REG       0x0c
-/* Enable IR pulse width detection */
+
+/* Bit to enable IR pulse width detection */
 #define MTK_PWM_EN		  BIT(13)
-/* Enable IR hardware function */
-#define MTK_IR_EN		  BIT(0)
 
-/* Register to setting sample period */
-#define MTK_CONFIG_LOW_REG        0x10
-/* Field to set sample period */
-#define CHK_PERIOD		  DIV_ROUND_CLOSEST(MTK_IR_SAMPLE,  \
-						    MTK_IR_CLK_PERIOD)
-#define MTK_CHK_PERIOD            (((CHK_PERIOD) << 8) & (GENMASK(20, 8)))
-#define MTK_CHK_PERIOD_MASK	  (GENMASK(20, 8))
+/*
+ * Register to setting ok count whose unit based on hardware sampling period
+ * indicating IR receiving completion and then making IRQ fires
+ */
+#define MTK_OK_COUNT(x)		  (((x) & GENMASK(23, 16)) << 16)
+
+/* Bit to enable IR hardware function */
+#define MTK_IR_EN		  BIT(0)
 
-/* Register to clear state of state machine */
-#define MTK_IRCLR_REG             0x20
 /* Bit to restart IR receiving */
 #define MTK_IRCLR		  BIT(0)
 
-/* Register containing pulse width data */
-#define MTK_CHKDATA_REG(i)        (0x88 + 4 * (i))
+/* Fields containing pulse width data */
 #define MTK_WIDTH_MASK		  (GENMASK(7, 0))
 
-/* Register to enable IR interrupt */
-#define MTK_IRINT_EN_REG          0xcc
 /* Bit to enable interrupt */
 #define MTK_IRINT_EN		  BIT(0)
 
-/* Register to ack IR interrupt */
-#define MTK_IRINT_CLR_REG         0xd0
 /* Bit to clear interrupt status */
 #define MTK_IRINT_CLR		  BIT(0)
 
@@ -63,24 +56,85 @@
 #define MTK_IR_END(v, p)	  ((v) == MTK_MAX_SAMPLES && (p) == 0)
 /* Number of registers to record the pulse width */
 #define MTK_CHKDATA_SZ		  17
-/* Source clock frequency */
-#define MTK_IR_BASE_CLK		  273000000
-/* Frequency after IR internal divider */
-#define MTK_IR_CLK_FREQ		  (MTK_IR_BASE_CLK / 4)
-/* Period for MTK_IR_CLK in ns*/
-#define MTK_IR_CLK_PERIOD	  DIV_ROUND_CLOSEST(1000000000ul,  \
-						    MTK_IR_CLK_FREQ)
 /* Sample period in ns */
-#define MTK_IR_SAMPLE		  (MTK_IR_CLK_PERIOD * 0xc00)
+#define MTK_IR_SAMPLE		  46000
+
+enum mtk_fields {
+	/* Register to setting software sampling period */
+	MTK_CHK_PERIOD,
+	/* Register to setting hardware sampling period */
+	MTK_HW_PERIOD,
+};
+
+enum mtk_regs {
+	/* Register to clear state of state machine */
+	MTK_IRCLR_REG,
+	/* Register containing pulse width data */
+	MTK_CHKDATA_REG,
+	/* Register to enable IR interrupt */
+	MTK_IRINT_EN_REG,
+	/* Register to ack IR interrupt */
+	MTK_IRINT_CLR_REG
+};
+
+static const u32 mt7623_regs[] = {
+	[MTK_IRCLR_REG] =	0x20,
+	[MTK_CHKDATA_REG] =	0x88,
+	[MTK_IRINT_EN_REG] =	0xcc,
+	[MTK_IRINT_CLR_REG] =	0xd0,
+};
+
+static const u32 mt7622_regs[] = {
+	[MTK_IRCLR_REG] =	0x18,
+	[MTK_CHKDATA_REG] =	0x30,
+	[MTK_IRINT_EN_REG] =	0x1c,
+	[MTK_IRINT_CLR_REG] =	0x20,
+};
+
+struct mtk_field_type {
+	u32 reg;
+	u8 offset;
+	u32 mask;
+};
+
+/*
+ * struct mtk_ir_data -	This is the structure holding all differences among
+			various hardwares
+ * @regs:		The pointer to the array holding registers offset
+ * @fields:		The pointer to the array holding fields location
+ * @div:		The internal divisor for the based reference clock
+ * @ok_count:		The count indicating the completion of IR data
+ *			receiving when count is reached
+ * @hw_period:		The value indicating the hardware sampling period
+ */
+struct mtk_ir_data {
+	const u32 *regs;
+	const struct mtk_field_type *fields;
+	u8 div;
+	u8 ok_count;
+	u32 hw_period;
+};
+
+static const struct mtk_field_type mt7623_fields[] = {
+	[MTK_CHK_PERIOD] = {0x10, 8, GENMASK(20, 8)},
+	[MTK_HW_PERIOD] = {0x10, 0, GENMASK(7, 0)},
+};
+
+static const struct mtk_field_type mt7622_fields[] = {
+	[MTK_CHK_PERIOD] = {0x24, 0, GENMASK(24, 0)},
+	[MTK_HW_PERIOD] = {0x10, 0, GENMASK(24, 0)},
+};
 
 /*
  * struct mtk_ir -	This is the main datasructure for holding the state
  *			of the driver
  * @dev:		The device pointer
  * @rc:			The rc instrance
- * @irq:		The IRQ that we are using
  * @base:		The mapped register i/o base
- * @clk:		The clock that we are using
+ * @irq:		The IRQ that we are using
+ * @clk:		The clock that IR internal is using
+ * @bus:		The clock that software decoder is using
+ * @data:		Holding specific data for vaious platform
  */
 struct mtk_ir {
 	struct device	*dev;
@@ -88,8 +142,36 @@ struct mtk_ir {
 	void __iomem	*base;
 	int		irq;
 	struct clk	*clk;
+	struct clk	*bus;
+	const struct mtk_ir_data *data;
 };
 
+static inline u32 mtk_chkdata_reg(struct mtk_ir *ir, u32 i)
+{
+	return ir->data->regs[MTK_CHKDATA_REG] + 4 * i;
+}
+
+static inline u32 mtk_chk_period(struct mtk_ir *ir)
+{
+	u32 val;
+
+	/* Period of raw software sampling in ns */
+	val = DIV_ROUND_CLOSEST(1000000000ul,
+				clk_get_rate(ir->bus) / ir->data->div);
+
+	/*
+	 * Period for software decoder used in the
+	 * unit of raw software sampling
+	 */
+	val = DIV_ROUND_CLOSEST(MTK_IR_SAMPLE, val);
+
+	dev_dbg(ir->dev, "@pwm clk  = \t%lu\n",
+		clk_get_rate(ir->bus) / ir->data->div);
+	dev_dbg(ir->dev, "@chkperiod = %08x\n", val);
+
+	return val;
+}
+
 static void mtk_w32_mask(struct mtk_ir *ir, u32 val, u32 mask, unsigned int reg)
 {
 	u32 tmp;
@@ -113,16 +195,16 @@ static inline void mtk_irq_disable(struct mtk_ir *ir, u32 mask)
 {
 	u32 val;
 
-	val = mtk_r32(ir, MTK_IRINT_EN_REG);
-	mtk_w32(ir, val & ~mask, MTK_IRINT_EN_REG);
+	val = mtk_r32(ir, ir->data->regs[MTK_IRINT_EN_REG]);
+	mtk_w32(ir, val & ~mask, ir->data->regs[MTK_IRINT_EN_REG]);
 }
 
 static inline void mtk_irq_enable(struct mtk_ir *ir, u32 mask)
 {
 	u32 val;
 
-	val = mtk_r32(ir, MTK_IRINT_EN_REG);
-	mtk_w32(ir, val | mask, MTK_IRINT_EN_REG);
+	val = mtk_r32(ir, ir->data->regs[MTK_IRINT_EN_REG]);
+	mtk_w32(ir, val | mask, ir->data->regs[MTK_IRINT_EN_REG]);
 }
 
 static irqreturn_t mtk_ir_irq(int irqno, void *dev_id)
@@ -140,7 +222,7 @@ static irqreturn_t mtk_ir_irq(int irqno, void *dev_id)
 	 * every decoder to reset themselves through long enough
 	 * trailing spaces and 2) the IRQ handler guarantees that
 	 * start of IR message is always contained in and starting
-	 * from register MTK_CHKDATA_REG(0).
+	 * from register mtk_chkdata_reg(ir, i).
 	 */
 	ir_raw_event_reset(ir->rc);
 
@@ -149,7 +231,7 @@ static irqreturn_t mtk_ir_irq(int irqno, void *dev_id)
 
 	/* Handle all pulse and space IR controller captures */
 	for (i = 0 ; i < MTK_CHKDATA_SZ ; i++) {
-		val = mtk_r32(ir, MTK_CHKDATA_REG(i));
+		val = mtk_r32(ir, mtk_chkdata_reg(ir, i));
 		dev_dbg(ir->dev, "@reg%d=0x%08x\n", i, val);
 
 		for (j = 0 ; j < 4 ; j++) {
@@ -181,18 +263,44 @@ static irqreturn_t mtk_ir_irq(int irqno, void *dev_id)
 	 * Restart controller for the next receive that would
 	 * clear up all CHKDATA registers
 	 */
-	mtk_w32_mask(ir, 0x1, MTK_IRCLR, MTK_IRCLR_REG);
+	mtk_w32_mask(ir, 0x1, MTK_IRCLR, ir->data->regs[MTK_IRCLR_REG]);
 
 	/* Clear interrupt status */
-	mtk_w32_mask(ir, 0x1, MTK_IRINT_CLR, MTK_IRINT_CLR_REG);
+	mtk_w32_mask(ir, 0x1, MTK_IRINT_CLR,
+		     ir->data->regs[MTK_IRINT_CLR_REG]);
 
 	return IRQ_HANDLED;
 }
 
+static const struct mtk_ir_data mt7623_data = {
+	.regs = mt7623_regs,
+	.fields = mt7623_fields,
+	.ok_count = 0xf,
+	.hw_period = 0xff,
+	.div	= 4,
+};
+
+static const struct mtk_ir_data mt7622_data = {
+	.regs = mt7622_regs,
+	.fields = mt7622_fields,
+	.ok_count = 0xf,
+	.hw_period = 0xffff,
+	.div	= 32,
+};
+
+static const struct of_device_id mtk_ir_match[] = {
+	{ .compatible = "mediatek,mt7623-cir", .data = &mt7623_data},
+	{ .compatible = "mediatek,mt7622-cir", .data = &mt7622_data},
+	{},
+};
+MODULE_DEVICE_TABLE(of, mtk_ir_match);
+
 static int mtk_ir_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct device_node *dn = dev->of_node;
+	const struct of_device_id *of_id =
+		of_match_device(mtk_ir_match, &pdev->dev);
 	struct resource *res;
 	struct mtk_ir *ir;
 	u32 val;
@@ -204,9 +312,7 @@ static int mtk_ir_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	ir->dev = dev;
-
-	if (!of_device_is_compatible(dn, "mediatek,mt7623-cir"))
-		return -ENODEV;
+	ir->data = of_id->data;
 
 	ir->clk = devm_clk_get(dev, "clk");
 	if (IS_ERR(ir->clk)) {
@@ -214,6 +320,15 @@ static int mtk_ir_probe(struct platform_device *pdev)
 		return PTR_ERR(ir->clk);
 	}
 
+	ir->bus = devm_clk_get(dev, "bus");
+	if (IS_ERR(ir->bus)) {
+		/*
+		 * For compatibility with older device trees try unnamed
+		 * ir->bus uses the same clock as ir->clock.
+		 */
+		ir->bus = ir->clk;
+	}
+
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	ir->base = devm_ioremap_resource(dev, res);
 	if (IS_ERR(ir->base)) {
@@ -228,7 +343,7 @@ static int mtk_ir_probe(struct platform_device *pdev)
 	}
 
 	ir->rc->priv = ir;
-	ir->rc->input_name = MTK_IR_DEV;
+	ir->rc->device_name = MTK_IR_DEV;
 	ir->rc->input_phys = MTK_IR_DEV "/input0";
 	ir->rc->input_id.bustype = BUS_HOST;
 	ir->rc->input_id.vendor = 0x0001;
@@ -238,7 +353,7 @@ static int mtk_ir_probe(struct platform_device *pdev)
 	ir->rc->map_name = map_name ?: RC_MAP_EMPTY;
 	ir->rc->dev.parent = dev;
 	ir->rc->driver_name = MTK_IR_DEV;
-	ir->rc->allowed_protocols = RC_BIT_ALL;
+	ir->rc->allowed_protocols = RC_PROTO_BIT_ALL;
 	ir->rc->rx_resolution = MTK_IR_SAMPLE;
 	ir->rc->timeout = MTK_MAX_SAMPLES * (MTK_IR_SAMPLE + 1);
 
@@ -256,40 +371,60 @@ static int mtk_ir_probe(struct platform_device *pdev)
 		return -ENODEV;
 	}
 
-	/*
-	 * Enable interrupt after proper hardware
-	 * setup and IRQ handler registration
-	 */
 	if (clk_prepare_enable(ir->clk)) {
 		dev_err(dev, "try to enable ir_clk failed\n");
+		return -EINVAL;
+	}
+
+	if (clk_prepare_enable(ir->bus)) {
+		dev_err(dev, "try to enable ir_clk failed\n");
 		ret = -EINVAL;
 		goto exit_clkdisable_clk;
 	}
 
+	/*
+	 * Enable interrupt after proper hardware
+	 * setup and IRQ handler registration
+	 */
 	mtk_irq_disable(ir, MTK_IRINT_EN);
 
 	ret = devm_request_irq(dev, ir->irq, mtk_ir_irq, 0, MTK_IR_DEV, ir);
 	if (ret) {
 		dev_err(dev, "failed request irq\n");
-		goto exit_clkdisable_clk;
+		goto exit_clkdisable_bus;
 	}
 
+	/*
+	 * Setup software sample period as the reference of software decoder
+	 */
+	val = (mtk_chk_period(ir) << ir->data->fields[MTK_CHK_PERIOD].offset) &
+	       ir->data->fields[MTK_CHK_PERIOD].mask;
+	mtk_w32_mask(ir, val, ir->data->fields[MTK_CHK_PERIOD].mask,
+		     ir->data->fields[MTK_CHK_PERIOD].reg);
+
+	/*
+	 * Setup hardware sampling period used to setup the proper timeout for
+	 * indicating end of IR receiving completion
+	 */
+	val = (ir->data->hw_period << ir->data->fields[MTK_HW_PERIOD].offset) &
+	       ir->data->fields[MTK_HW_PERIOD].mask;
+	mtk_w32_mask(ir, val, ir->data->fields[MTK_HW_PERIOD].mask,
+		     ir->data->fields[MTK_HW_PERIOD].reg);
+
 	/* Enable IR and PWM */
 	val = mtk_r32(ir, MTK_CONFIG_HIGH_REG);
-	val |= MTK_PWM_EN | MTK_IR_EN;
+	val |= MTK_OK_COUNT(ir->data->ok_count) |  MTK_PWM_EN | MTK_IR_EN;
 	mtk_w32(ir, val, MTK_CONFIG_HIGH_REG);
 
-	/* Setting sample period */
-	mtk_w32_mask(ir, MTK_CHK_PERIOD, MTK_CHK_PERIOD_MASK,
-		     MTK_CONFIG_LOW_REG);
-
 	mtk_irq_enable(ir, MTK_IRINT_EN);
 
-	dev_info(dev, "Initialized MT7623 IR driver, sample period = %luus\n",
+	dev_info(dev, "Initialized MT7623 IR driver, sample period = %dus\n",
 		 DIV_ROUND_CLOSEST(MTK_IR_SAMPLE, 1000));
 
 	return 0;
 
+exit_clkdisable_bus:
+	clk_disable_unprepare(ir->bus);
 exit_clkdisable_clk:
 	clk_disable_unprepare(ir->clk);
 
@@ -308,17 +443,12 @@ static int mtk_ir_remove(struct platform_device *pdev)
 	mtk_irq_disable(ir, MTK_IRINT_EN);
 	synchronize_irq(ir->irq);
 
+	clk_disable_unprepare(ir->bus);
 	clk_disable_unprepare(ir->clk);
 
 	return 0;
 }
 
-static const struct of_device_id mtk_ir_match[] = {
-	{ .compatible = "mediatek,mt7623-cir" },
-	{},
-};
-MODULE_DEVICE_TABLE(of, mtk_ir_match);
-
 static struct platform_driver mtk_ir_driver = {
 	.probe          = mtk_ir_probe,
 	.remove         = mtk_ir_remove,
diff --git a/drivers/media/rc/nuvoton-cir.c b/drivers/media/rc/nuvoton-cir.c
index ec4b25bd2ec2..5e1d866a61a5 100644
--- a/drivers/media/rc/nuvoton-cir.c
+++ b/drivers/media/rc/nuvoton-cir.c
@@ -727,70 +727,6 @@ out_raw:
 	return ret;
 }
 
-/*
- * nvt_tx_ir
- *
- * 1) clean TX fifo first (handled by AP)
- * 2) copy data from user space
- * 3) disable RX interrupts, enable TX interrupts: TTR & TFU
- * 4) send 9 packets to TX FIFO to open TTR
- * in interrupt_handler:
- * 5) send all data out
- * go back to write():
- * 6) disable TX interrupts, re-enable RX interupts
- *
- * The key problem of this function is user space data may larger than
- * driver's data buf length. So nvt_tx_ir() will only copy TX_BUF_LEN data to
- * buf, and keep current copied data buf num in cur_buf_num. But driver's buf
- * number may larger than TXFCONT (0xff). So in interrupt_handler, it has to
- * set TXFCONT as 0xff, until buf_count less than 0xff.
- */
-static int nvt_tx_ir(struct rc_dev *dev, unsigned *txbuf, unsigned n)
-{
-	struct nvt_dev *nvt = dev->priv;
-	unsigned long flags;
-	unsigned int i;
-	u8 iren;
-	int ret;
-
-	spin_lock_irqsave(&nvt->lock, flags);
-
-	ret = min((unsigned)(TX_BUF_LEN / sizeof(unsigned)), n);
-	nvt->tx.buf_count = (ret * sizeof(unsigned));
-
-	memcpy(nvt->tx.buf, txbuf, nvt->tx.buf_count);
-
-	nvt->tx.cur_buf_num = 0;
-
-	/* save currently enabled interrupts */
-	iren = nvt_cir_reg_read(nvt, CIR_IREN);
-
-	/* now disable all interrupts, save TFU & TTR */
-	nvt_cir_reg_write(nvt, CIR_IREN_TFU | CIR_IREN_TTR, CIR_IREN);
-
-	nvt->tx.tx_state = ST_TX_REPLY;
-
-	nvt_cir_reg_write(nvt, CIR_FIFOCON_TX_TRIGGER_LEV_8 |
-			  CIR_FIFOCON_RXFIFOCLR, CIR_FIFOCON);
-
-	/* trigger TTR interrupt by writing out ones, (yes, it's ugly) */
-	for (i = 0; i < 9; i++)
-		nvt_cir_reg_write(nvt, 0x01, CIR_STXFIFO);
-
-	spin_unlock_irqrestore(&nvt->lock, flags);
-
-	wait_event(nvt->tx.queue, nvt->tx.tx_state == ST_TX_REQUEST);
-
-	spin_lock_irqsave(&nvt->lock, flags);
-	nvt->tx.tx_state = ST_TX_NONE;
-	spin_unlock_irqrestore(&nvt->lock, flags);
-
-	/* restore enabled interrupts to prior state */
-	nvt_cir_reg_write(nvt, iren, CIR_IREN);
-
-	return ret;
-}
-
 /* dump contents of the last rx buffer we got from the hw rx fifo */
 static void nvt_dump_rx_buf(struct nvt_dev *nvt)
 {
@@ -895,11 +831,6 @@ static void nvt_cir_log_irqs(u8 status, u8 iren)
 			   CIR_IRSTS_TFU | CIR_IRSTS_GH) ? " ?" : "");
 }
 
-static bool nvt_cir_tx_inactive(struct nvt_dev *nvt)
-{
-	return nvt->tx.tx_state == ST_TX_NONE;
-}
-
 /* interrupt service routine for incoming and outgoing CIR data */
 static irqreturn_t nvt_cir_isr(int irq, void *data)
 {
@@ -952,40 +883,8 @@ static irqreturn_t nvt_cir_isr(int irq, void *data)
 
 	if (status & CIR_IRSTS_RFO)
 		nvt_handle_rx_fifo_overrun(nvt);
-
-	else if (status & (CIR_IRSTS_RTR | CIR_IRSTS_PE)) {
-		/* We only do rx if not tx'ing */
-		if (nvt_cir_tx_inactive(nvt))
-			nvt_get_rx_ir_data(nvt);
-	}
-
-	if (status & CIR_IRSTS_TE)
-		nvt_clear_tx_fifo(nvt);
-
-	if (status & CIR_IRSTS_TTR) {
-		unsigned int pos, count;
-		u8 tmp;
-
-		pos = nvt->tx.cur_buf_num;
-		count = nvt->tx.buf_count;
-
-		/* Write data into the hardware tx fifo while pos < count */
-		if (pos < count) {
-			nvt_cir_reg_write(nvt, nvt->tx.buf[pos], CIR_STXFIFO);
-			nvt->tx.cur_buf_num++;
-		/* Disable TX FIFO Trigger Level Reach (TTR) interrupt */
-		} else {
-			tmp = nvt_cir_reg_read(nvt, CIR_IREN);
-			nvt_cir_reg_write(nvt, tmp & ~CIR_IREN_TTR, CIR_IREN);
-		}
-	}
-
-	if (status & CIR_IRSTS_TFU) {
-		if (nvt->tx.tx_state == ST_TX_REPLY) {
-			nvt->tx.tx_state = ST_TX_REQUEST;
-			wake_up(&nvt->tx.queue);
-		}
-	}
+	else if (status & (CIR_IRSTS_RTR | CIR_IRSTS_PE))
+		nvt_get_rx_ir_data(nvt);
 
 	spin_unlock(&nvt->lock);
 
@@ -1062,7 +961,7 @@ static int nvt_probe(struct pnp_dev *pdev, const struct pnp_device_id *dev_id)
 	if (!nvt)
 		return -ENOMEM;
 
-	/* input device for IR remote (and tx) */
+	/* input device for IR remote */
 	nvt->rdev = devm_rc_allocate_device(&pdev->dev, RC_DRIVER_IR_RAW);
 	if (!nvt->rdev)
 		return -ENOMEM;
@@ -1105,8 +1004,6 @@ static int nvt_probe(struct pnp_dev *pdev, const struct pnp_device_id *dev_id)
 
 	pnp_set_drvdata(pdev, nvt);
 
-	init_waitqueue_head(&nvt->tx.queue);
-
 	ret = nvt_hw_detect(nvt);
 	if (ret)
 		return ret;
@@ -1126,15 +1023,14 @@ static int nvt_probe(struct pnp_dev *pdev, const struct pnp_device_id *dev_id)
 
 	/* Set up the rc device */
 	rdev->priv = nvt;
-	rdev->allowed_protocols = RC_BIT_ALL_IR_DECODER;
-	rdev->allowed_wakeup_protocols = RC_BIT_ALL_IR_ENCODER;
+	rdev->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
+	rdev->allowed_wakeup_protocols = RC_PROTO_BIT_ALL_IR_ENCODER;
 	rdev->encode_wakeup = true;
 	rdev->open = nvt_open;
 	rdev->close = nvt_close;
-	rdev->tx_ir = nvt_tx_ir;
 	rdev->s_tx_carrier = nvt_set_tx_carrier;
 	rdev->s_wakeup_filter = nvt_ir_raw_set_wakeup_filter;
-	rdev->input_name = "Nuvoton w836x7hg Infrared Remote Transceiver";
+	rdev->device_name = "Nuvoton w836x7hg Infrared Remote Transceiver";
 	rdev->input_phys = "nuvoton/cir0";
 	rdev->input_id.bustype = BUS_HOST;
 	rdev->input_id.vendor = PCI_VENDOR_ID_WINBOND2;
@@ -1148,8 +1044,6 @@ static int nvt_probe(struct pnp_dev *pdev, const struct pnp_device_id *dev_id)
 #if 0
 	rdev->min_timeout = XYZ;
 	rdev->max_timeout = XYZ;
-	/* tx bits */
-	rdev->tx_resolution = XYZ;
 #endif
 	ret = devm_rc_register_device(&pdev->dev, rdev);
 	if (ret)
@@ -1205,8 +1099,6 @@ static int nvt_suspend(struct pnp_dev *pdev, pm_message_t state)
 
 	spin_lock_irqsave(&nvt->lock, flags);
 
-	nvt->tx.tx_state = ST_TX_NONE;
-
 	/* disable all CIR interrupts */
 	nvt_cir_reg_write(nvt, 0, CIR_IREN);
 
diff --git a/drivers/media/rc/nuvoton-cir.h b/drivers/media/rc/nuvoton-cir.h
index 88a29df38a57..0737c27f7ddc 100644
--- a/drivers/media/rc/nuvoton-cir.h
+++ b/drivers/media/rc/nuvoton-cir.h
@@ -46,14 +46,6 @@ static int debug;
 			KBUILD_MODNAME ": " text "\n" , ## __VA_ARGS__)
 
 
-/*
- * Original lirc driver said min value of 76, and recommended value of 256
- * for the buffer length, but then used 2048. Never mind that the size of the
- * RX FIFO is 32 bytes... So I'm using 32 for RX and 256 for TX atm, but I'm
- * not sure if maybe that TX value is off by a factor of 8 (bits vs. bytes),
- * and I don't have TX-capable hardware to test/debug on...
- */
-#define TX_BUF_LEN 256
 #define RX_BUF_LEN 32
 
 #define SIO_ID_MASK 0xfff0
@@ -81,14 +73,6 @@ struct nvt_dev {
 	u8 buf[RX_BUF_LEN];
 	unsigned int pkts;
 
-	struct {
-		u8 buf[TX_BUF_LEN];
-		unsigned int buf_count;
-		unsigned int cur_buf_num;
-		wait_queue_head_t queue;
-		u8 tx_state;
-	} tx;
-
 	/* EFER Config register index/data pair */
 	u32 cr_efir;
 	u32 cr_efdr;
@@ -103,18 +87,10 @@ struct nvt_dev {
 	u8 chip_major;
 	u8 chip_minor;
 
-	/* hardware features */
-	bool hw_tx_capable;
-
 	/* carrier period = 1 / frequency */
 	u32 carrier;
 };
 
-/* send states */
-#define ST_TX_NONE	0x0
-#define ST_TX_REQUEST	0x2
-#define ST_TX_REPLY	0x4
-
 /* buffer packet constants */
 #define BUF_PULSE_BIT	0x80
 #define BUF_LEN_MASK	0x7f
diff --git a/drivers/media/rc/pwm-ir-tx.c b/drivers/media/rc/pwm-ir-tx.c
new file mode 100644
index 000000000000..27d0f5837a76
--- /dev/null
+++ b/drivers/media/rc/pwm-ir-tx.c
@@ -0,0 +1,138 @@
+/*
+ * Copyright (C) 2017 Sean Young <sean@mess.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pwm.h>
+#include <linux/delay.h>
+#include <linux/slab.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <media/rc-core.h>
+
+#define DRIVER_NAME	"pwm-ir-tx"
+#define DEVICE_NAME	"PWM IR Transmitter"
+
+struct pwm_ir {
+	struct pwm_device *pwm;
+	unsigned int carrier;
+	unsigned int duty_cycle;
+};
+
+static const struct of_device_id pwm_ir_of_match[] = {
+	{ .compatible = "pwm-ir-tx", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, pwm_ir_of_match);
+
+static int pwm_ir_set_duty_cycle(struct rc_dev *dev, u32 duty_cycle)
+{
+	struct pwm_ir *pwm_ir = dev->priv;
+
+	pwm_ir->duty_cycle = duty_cycle;
+
+	return 0;
+}
+
+static int pwm_ir_set_carrier(struct rc_dev *dev, u32 carrier)
+{
+	struct pwm_ir *pwm_ir = dev->priv;
+
+	if (!carrier)
+		return -EINVAL;
+
+	pwm_ir->carrier = carrier;
+
+	return 0;
+}
+
+static int pwm_ir_tx(struct rc_dev *dev, unsigned int *txbuf,
+		     unsigned int count)
+{
+	struct pwm_ir *pwm_ir = dev->priv;
+	struct pwm_device *pwm = pwm_ir->pwm;
+	int i, duty, period;
+	ktime_t edge;
+	long delta;
+
+	period = DIV_ROUND_CLOSEST(NSEC_PER_SEC, pwm_ir->carrier);
+	duty = DIV_ROUND_CLOSEST(pwm_ir->duty_cycle * period, 100);
+
+	pwm_config(pwm, duty, period);
+
+	edge = ktime_get();
+
+	for (i = 0; i < count; i++) {
+		if (i % 2) // space
+			pwm_disable(pwm);
+		else
+			pwm_enable(pwm);
+
+		edge = ktime_add_us(edge, txbuf[i]);
+		delta = ktime_us_delta(edge, ktime_get());
+		if (delta > 0)
+			usleep_range(delta, delta + 10);
+	}
+
+	pwm_disable(pwm);
+
+	return count;
+}
+
+static int pwm_ir_probe(struct platform_device *pdev)
+{
+	struct pwm_ir *pwm_ir;
+	struct rc_dev *rcdev;
+	int rc;
+
+	pwm_ir = devm_kmalloc(&pdev->dev, sizeof(*pwm_ir), GFP_KERNEL);
+	if (!pwm_ir)
+		return -ENOMEM;
+
+	pwm_ir->pwm = devm_pwm_get(&pdev->dev, NULL);
+	if (IS_ERR(pwm_ir->pwm))
+		return PTR_ERR(pwm_ir->pwm);
+
+	pwm_ir->carrier = 38000;
+	pwm_ir->duty_cycle = 50;
+
+	rcdev = devm_rc_allocate_device(&pdev->dev, RC_DRIVER_IR_RAW_TX);
+	if (!rcdev)
+		return -ENOMEM;
+
+	rcdev->priv = pwm_ir;
+	rcdev->driver_name = DRIVER_NAME;
+	rcdev->device_name = DEVICE_NAME;
+	rcdev->tx_ir = pwm_ir_tx;
+	rcdev->s_tx_duty_cycle = pwm_ir_set_duty_cycle;
+	rcdev->s_tx_carrier = pwm_ir_set_carrier;
+
+	rc = devm_rc_register_device(&pdev->dev, rcdev);
+	if (rc < 0)
+		dev_err(&pdev->dev, "failed to register rc device\n");
+
+	return rc;
+}
+
+static struct platform_driver pwm_ir_driver = {
+	.probe = pwm_ir_probe,
+	.driver = {
+		.name	= DRIVER_NAME,
+		.of_match_table = of_match_ptr(pwm_ir_of_match),
+	},
+};
+module_platform_driver(pwm_ir_driver);
+
+MODULE_DESCRIPTION("PWM IR Transmitter");
+MODULE_AUTHOR("Sean Young <sean@mess.org>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/rc/rc-core-priv.h b/drivers/media/rc/rc-core-priv.h
index b3e7cac2c3ee..7da9c96cb058 100644
--- a/drivers/media/rc/rc-core-priv.h
+++ b/drivers/media/rc/rc-core-priv.h
@@ -27,7 +27,7 @@ struct ir_raw_handler {
 
 	u64 protocols; /* which are handled by this handler */
 	int (*decode)(struct rc_dev *dev, struct ir_raw_event event);
-	int (*encode)(enum rc_type protocol, u32 scancode,
+	int (*encode)(enum rc_proto protocol, u32 scancode,
 		      struct ir_raw_event *events, unsigned int max);
 
 	/* These two should only be used by the lirc decoder */
@@ -41,8 +41,9 @@ struct ir_raw_event_ctrl {
 	/* fifo for the pulse/space durations */
 	DECLARE_KFIFO(kfifo, struct ir_raw_event, MAX_IR_EVENT_SIZE);
 	ktime_t				last_event;	/* when last event occurred */
-	enum raw_event_type		last_type;	/* last event type */
 	struct rc_dev			*dev;		/* pointer to the parent rc_dev */
+	/* edge driver */
+	struct timer_list edge_handle;
 
 	/* raw decoder state follows */
 	struct ir_raw_event prev_ev;
diff --git a/drivers/media/rc/rc-ir-raw.c b/drivers/media/rc/rc-ir-raw.c
index b6d256f03847..503bc425a187 100644
--- a/drivers/media/rc/rc-ir-raw.c
+++ b/drivers/media/rc/rc-ir-raw.c
@@ -88,7 +88,7 @@ EXPORT_SYMBOL_GPL(ir_raw_event_store);
 /**
  * ir_raw_event_store_edge() - notify raw ir decoders of the start of a pulse/space
  * @dev:	the struct rc_dev device descriptor
- * @type:	the type of the event that has occurred
+ * @pulse:	true for pulse, false for space
  *
  * This routine (which may be called from an interrupt context) is used to
  * store the beginning of an ir pulse or space (or the start/end of ir
@@ -96,43 +96,31 @@ EXPORT_SYMBOL_GPL(ir_raw_event_store);
  * hardware which does not provide durations directly but only interrupts
  * (or similar events) on state change.
  */
-int ir_raw_event_store_edge(struct rc_dev *dev, enum raw_event_type type)
+int ir_raw_event_store_edge(struct rc_dev *dev, bool pulse)
 {
 	ktime_t			now;
-	s64			delta; /* ns */
 	DEFINE_IR_RAW_EVENT(ev);
 	int			rc = 0;
-	int			delay;
 
 	if (!dev->raw)
 		return -EINVAL;
 
 	now = ktime_get();
-	delta = ktime_to_ns(ktime_sub(now, dev->raw->last_event));
-	delay = MS_TO_NS(dev->input_dev->rep[REP_DELAY]);
+	ev.duration = ktime_to_ns(ktime_sub(now, dev->raw->last_event));
+	ev.pulse = !pulse;
 
-	/* Check for a long duration since last event or if we're
-	 * being called for the first time, note that delta can't
-	 * possibly be negative.
-	 */
-	if (delta > delay || !dev->raw->last_type)
-		type |= IR_START_EVENT;
-	else
-		ev.duration = delta;
-
-	if (type & IR_START_EVENT)
-		ir_raw_event_reset(dev);
-	else if (dev->raw->last_type & IR_SPACE) {
-		ev.pulse = false;
-		rc = ir_raw_event_store(dev, &ev);
-	} else if (dev->raw->last_type & IR_PULSE) {
-		ev.pulse = true;
-		rc = ir_raw_event_store(dev, &ev);
-	} else
-		return 0;
+	rc = ir_raw_event_store(dev, &ev);
 
 	dev->raw->last_event = now;
-	dev->raw->last_type = type;
+
+	/* timer could be set to timeout (125ms by default) */
+	if (!timer_pending(&dev->raw->edge_handle) ||
+	    time_after(dev->raw->edge_handle.expires,
+		       jiffies + msecs_to_jiffies(15))) {
+		mod_timer(&dev->raw->edge_handle,
+			  jiffies + msecs_to_jiffies(15));
+	}
+
 	return rc;
 }
 EXPORT_SYMBOL_GPL(ir_raw_event_store_edge);
@@ -225,7 +213,7 @@ ir_raw_get_allowed_protocols(void)
 	return atomic64_read(&available_protocols);
 }
 
-static int change_protocol(struct rc_dev *dev, u64 *rc_type)
+static int change_protocol(struct rc_dev *dev, u64 *rc_proto)
 {
 	/* the caller will update dev->enabled_protocols */
 	return 0;
@@ -462,7 +450,7 @@ EXPORT_SYMBOL(ir_raw_gen_pl);
  *		-EINVAL if the scancode is ambiguous or invalid, or if no
  *		compatible encoder was found.
  */
-int ir_raw_encode_scancode(enum rc_type protocol, u32 scancode,
+int ir_raw_encode_scancode(enum rc_proto protocol, u32 scancode,
 			   struct ir_raw_event *events, unsigned int max)
 {
 	struct ir_raw_handler *handler;
@@ -483,6 +471,27 @@ int ir_raw_encode_scancode(enum rc_type protocol, u32 scancode,
 }
 EXPORT_SYMBOL(ir_raw_encode_scancode);
 
+static void edge_handle(unsigned long arg)
+{
+	struct rc_dev *dev = (struct rc_dev *)arg;
+	ktime_t interval = ktime_sub(ktime_get(), dev->raw->last_event);
+
+	if (ktime_to_ns(interval) >= dev->timeout) {
+		DEFINE_IR_RAW_EVENT(ev);
+
+		ev.timeout = true;
+		ev.duration = ktime_to_ns(interval);
+
+		ir_raw_event_store(dev, &ev);
+	} else {
+		mod_timer(&dev->raw->edge_handle,
+			  jiffies + nsecs_to_jiffies(dev->timeout -
+						     ktime_to_ns(interval)));
+	}
+
+	ir_raw_event_handle(dev);
+}
+
 /*
  * Used to (un)register raw event clients
  */
@@ -504,6 +513,8 @@ int ir_raw_event_prepare(struct rc_dev *dev)
 
 	dev->raw->dev = dev;
 	dev->change_protocol = change_protocol;
+	setup_timer(&dev->raw->edge_handle, edge_handle,
+		    (unsigned long)dev);
 	INIT_KFIFO(dev->raw->kfifo);
 
 	return 0;
@@ -555,6 +566,7 @@ void ir_raw_event_unregister(struct rc_dev *dev)
 		return;
 
 	kthread_stop(dev->raw->thread);
+	del_timer_sync(&dev->raw->edge_handle);
 
 	mutex_lock(&ir_raw_handler_lock);
 	list_del(&dev->raw->list);
diff --git a/drivers/media/rc/rc-loopback.c b/drivers/media/rc/rc-loopback.c
index 62195af24fbe..3822d9ebcb46 100644
--- a/drivers/media/rc/rc-loopback.c
+++ b/drivers/media/rc/rc-loopback.c
@@ -219,15 +219,15 @@ static int __init loop_init(void)
 		return -ENOMEM;
 	}
 
-	rc->input_name		= "rc-core loopback device";
+	rc->device_name		= "rc-core loopback device";
 	rc->input_phys		= "rc-core/virtual";
 	rc->input_id.bustype	= BUS_VIRTUAL;
 	rc->input_id.version	= 1;
 	rc->driver_name		= DRIVER_NAME;
 	rc->map_name		= RC_MAP_EMPTY;
 	rc->priv		= &loopdev;
-	rc->allowed_protocols	= RC_BIT_ALL_IR_DECODER;
-	rc->allowed_wakeup_protocols = RC_BIT_ALL_IR_ENCODER;
+	rc->allowed_protocols	= RC_PROTO_BIT_ALL_IR_DECODER;
+	rc->allowed_wakeup_protocols = RC_PROTO_BIT_ALL_IR_ENCODER;
 	rc->encode_wakeup	= true;
 	rc->timeout		= 100 * 1000 * 1000; /* 100 ms */
 	rc->min_timeout		= 1;
diff --git a/drivers/media/rc/rc-main.c b/drivers/media/rc/rc-main.c
index a9eba0013525..981cccd6b988 100644
--- a/drivers/media/rc/rc-main.c
+++ b/drivers/media/rc/rc-main.c
@@ -30,8 +30,54 @@
 #define IR_TAB_MAX_SIZE	8192
 #define RC_DEV_MAX	256
 
-/* FIXME: IR_KEYPRESS_TIMEOUT should be protocol specific */
-#define IR_KEYPRESS_TIMEOUT 250
+static const struct {
+	const char *name;
+	unsigned int repeat_period;
+	unsigned int scancode_bits;
+} protocols[] = {
+	[RC_PROTO_UNKNOWN] = { .name = "unknown", .repeat_period = 250 },
+	[RC_PROTO_OTHER] = { .name = "other", .repeat_period = 250 },
+	[RC_PROTO_RC5] = { .name = "rc-5",
+		.scancode_bits = 0x1f7f, .repeat_period = 164 },
+	[RC_PROTO_RC5X_20] = { .name = "rc-5x-20",
+		.scancode_bits = 0x1f7f3f, .repeat_period = 164 },
+	[RC_PROTO_RC5_SZ] = { .name = "rc-5-sz",
+		.scancode_bits = 0x2fff, .repeat_period = 164 },
+	[RC_PROTO_JVC] = { .name = "jvc",
+		.scancode_bits = 0xffff, .repeat_period = 250 },
+	[RC_PROTO_SONY12] = { .name = "sony-12",
+		.scancode_bits = 0x1f007f, .repeat_period = 100 },
+	[RC_PROTO_SONY15] = { .name = "sony-15",
+		.scancode_bits = 0xff007f, .repeat_period = 100 },
+	[RC_PROTO_SONY20] = { .name = "sony-20",
+		.scancode_bits = 0x1fff7f, .repeat_period = 100 },
+	[RC_PROTO_NEC] = { .name = "nec",
+		.scancode_bits = 0xffff, .repeat_period = 160 },
+	[RC_PROTO_NECX] = { .name = "nec-x",
+		.scancode_bits = 0xffffff, .repeat_period = 160 },
+	[RC_PROTO_NEC32] = { .name = "nec-32",
+		.scancode_bits = 0xffffffff, .repeat_period = 160 },
+	[RC_PROTO_SANYO] = { .name = "sanyo",
+		.scancode_bits = 0x1fffff, .repeat_period = 250 },
+	[RC_PROTO_MCIR2_KBD] = { .name = "mcir2-kbd",
+		.scancode_bits = 0xffff, .repeat_period = 150 },
+	[RC_PROTO_MCIR2_MSE] = { .name = "mcir2-mse",
+		.scancode_bits = 0x1fffff, .repeat_period = 150 },
+	[RC_PROTO_RC6_0] = { .name = "rc-6-0",
+		.scancode_bits = 0xffff, .repeat_period = 164 },
+	[RC_PROTO_RC6_6A_20] = { .name = "rc-6-6a-20",
+		.scancode_bits = 0xfffff, .repeat_period = 164 },
+	[RC_PROTO_RC6_6A_24] = { .name = "rc-6-6a-24",
+		.scancode_bits = 0xffffff, .repeat_period = 164 },
+	[RC_PROTO_RC6_6A_32] = { .name = "rc-6-6a-32",
+		.scancode_bits = 0xffffffff, .repeat_period = 164 },
+	[RC_PROTO_RC6_MCE] = { .name = "rc-6-mce",
+		.scancode_bits = 0xffff7fff, .repeat_period = 164 },
+	[RC_PROTO_SHARP] = { .name = "sharp",
+		.scancode_bits = 0x1fff, .repeat_period = 250 },
+	[RC_PROTO_XMP] = { .name = "xmp", .repeat_period = 250 },
+	[RC_PROTO_CEC] = { .name = "cec", .repeat_period = 550 },
+};
 
 /* Used to keep track of known keymaps */
 static LIST_HEAD(rc_map_list);
@@ -110,10 +156,10 @@ static struct rc_map_table empty[] = {
 
 static struct rc_map_list empty_map = {
 	.map = {
-		.scan    = empty,
-		.size    = ARRAY_SIZE(empty),
-		.rc_type = RC_TYPE_UNKNOWN,	/* Legacy IR type */
-		.name    = RC_MAP_EMPTY,
+		.scan     = empty,
+		.size     = ARRAY_SIZE(empty),
+		.rc_proto = RC_PROTO_UNKNOWN,	/* Legacy IR type */
+		.name     = RC_MAP_EMPTY,
 	}
 };
 
@@ -121,7 +167,7 @@ static struct rc_map_list empty_map = {
  * ir_create_table() - initializes a scancode table
  * @rc_map:	the rc_map to initialize
  * @name:	name to assign to the table
- * @rc_type:	ir type to assign to the new table
+ * @rc_proto:	ir type to assign to the new table
  * @size:	initial size of the table
  * @return:	zero on success or a negative error code
  *
@@ -129,12 +175,12 @@ static struct rc_map_list empty_map = {
  * memory to hold at least the specified number of elements.
  */
 static int ir_create_table(struct rc_map *rc_map,
-			   const char *name, u64 rc_type, size_t size)
+			   const char *name, u64 rc_proto, size_t size)
 {
 	rc_map->name = kstrdup(name, GFP_KERNEL);
 	if (!rc_map->name)
 		return -ENOMEM;
-	rc_map->rc_type = rc_type;
+	rc_map->rc_proto = rc_proto;
 	rc_map->alloc = roundup_pow_of_two(size * sizeof(struct rc_map_table));
 	rc_map->size = rc_map->alloc / sizeof(struct rc_map_table);
 	rc_map->scan = kmalloc(rc_map->alloc, GFP_KERNEL);
@@ -389,7 +435,7 @@ static int ir_setkeytable(struct rc_dev *dev,
 	int rc;
 
 	rc = ir_create_table(rc_map, from->name,
-			     from->rc_type, from->size);
+			     from->rc_proto, from->size);
 	if (rc)
 		return rc;
 
@@ -530,7 +576,7 @@ u32 rc_g_keycode_from_table(struct rc_dev *dev, u32 scancode)
 
 	if (keycode != KEY_RESERVED)
 		IR_dprintk(1, "%s: scancode 0x%04x keycode 0x%02x\n",
-			   dev->input_name, scancode, keycode);
+			   dev->device_name, scancode, keycode);
 
 	return keycode;
 }
@@ -613,16 +659,17 @@ static void ir_timer_keyup(unsigned long cookie)
 void rc_repeat(struct rc_dev *dev)
 {
 	unsigned long flags;
+	unsigned int timeout = protocols[dev->last_protocol].repeat_period;
 
 	spin_lock_irqsave(&dev->keylock, flags);
 
-	input_event(dev->input_dev, EV_MSC, MSC_SCAN, dev->last_scancode);
-	input_sync(dev->input_dev);
-
 	if (!dev->keypressed)
 		goto out;
 
-	dev->keyup_jiffies = jiffies + msecs_to_jiffies(IR_KEYPRESS_TIMEOUT);
+	input_event(dev->input_dev, EV_MSC, MSC_SCAN, dev->last_scancode);
+	input_sync(dev->input_dev);
+
+	dev->keyup_jiffies = jiffies + msecs_to_jiffies(timeout);
 	mod_timer(&dev->timer_keyup, dev->keyup_jiffies);
 
 out:
@@ -641,7 +688,7 @@ EXPORT_SYMBOL_GPL(rc_repeat);
  * This function is used internally to register a keypress, it must be
  * called with keylock held.
  */
-static void ir_do_keydown(struct rc_dev *dev, enum rc_type protocol,
+static void ir_do_keydown(struct rc_dev *dev, enum rc_proto protocol,
 			  u32 scancode, u32 keycode, u8 toggle)
 {
 	bool new_event = (!dev->keypressed		 ||
@@ -663,7 +710,7 @@ static void ir_do_keydown(struct rc_dev *dev, enum rc_type protocol,
 		dev->last_keycode = keycode;
 
 		IR_dprintk(1, "%s: key down event, key 0x%04x, protocol 0x%04x, scancode 0x%08x\n",
-			   dev->input_name, keycode, protocol, scancode);
+			   dev->device_name, keycode, protocol, scancode);
 		input_report_key(dev->input_dev, keycode, 1);
 
 		led_trigger_event(led_feedback, LED_FULL);
@@ -683,7 +730,8 @@ static void ir_do_keydown(struct rc_dev *dev, enum rc_type protocol,
  * This routine is used to signal that a key has been pressed on the
  * remote control.
  */
-void rc_keydown(struct rc_dev *dev, enum rc_type protocol, u32 scancode, u8 toggle)
+void rc_keydown(struct rc_dev *dev, enum rc_proto protocol, u32 scancode,
+		u8 toggle)
 {
 	unsigned long flags;
 	u32 keycode = rc_g_keycode_from_table(dev, scancode);
@@ -692,7 +740,8 @@ void rc_keydown(struct rc_dev *dev, enum rc_type protocol, u32 scancode, u8 togg
 	ir_do_keydown(dev, protocol, scancode, keycode, toggle);
 
 	if (dev->keypressed) {
-		dev->keyup_jiffies = jiffies + msecs_to_jiffies(IR_KEYPRESS_TIMEOUT);
+		dev->keyup_jiffies = jiffies +
+			msecs_to_jiffies(protocols[protocol].repeat_period);
 		mod_timer(&dev->timer_keyup, dev->keyup_jiffies);
 	}
 	spin_unlock_irqrestore(&dev->keylock, flags);
@@ -711,7 +760,7 @@ EXPORT_SYMBOL_GPL(rc_keydown);
  * This routine is used to signal that a key has been pressed on the
  * remote control. The driver must manually call rc_keyup() at a later stage.
  */
-void rc_keydown_notimeout(struct rc_dev *dev, enum rc_type protocol,
+void rc_keydown_notimeout(struct rc_dev *dev, enum rc_proto protocol,
 			  u32 scancode, u8 toggle)
 {
 	unsigned long flags;
@@ -733,44 +782,28 @@ EXPORT_SYMBOL_GPL(rc_keydown_notimeout);
 static int rc_validate_filter(struct rc_dev *dev,
 			      struct rc_scancode_filter *filter)
 {
-	static u32 masks[] = {
-		[RC_TYPE_RC5] = 0x1f7f,
-		[RC_TYPE_RC5X_20] = 0x1f7f3f,
-		[RC_TYPE_RC5_SZ] = 0x2fff,
-		[RC_TYPE_SONY12] = 0x1f007f,
-		[RC_TYPE_SONY15] = 0xff007f,
-		[RC_TYPE_SONY20] = 0x1fff7f,
-		[RC_TYPE_JVC] = 0xffff,
-		[RC_TYPE_NEC] = 0xffff,
-		[RC_TYPE_NECX] = 0xffffff,
-		[RC_TYPE_NEC32] = 0xffffffff,
-		[RC_TYPE_SANYO] = 0x1fffff,
-		[RC_TYPE_MCIR2_KBD] = 0xffff,
-		[RC_TYPE_MCIR2_MSE] = 0x1fffff,
-		[RC_TYPE_RC6_0] = 0xffff,
-		[RC_TYPE_RC6_6A_20] = 0xfffff,
-		[RC_TYPE_RC6_6A_24] = 0xffffff,
-		[RC_TYPE_RC6_6A_32] = 0xffffffff,
-		[RC_TYPE_RC6_MCE] = 0xffff7fff,
-		[RC_TYPE_SHARP] = 0x1fff,
-	};
-	u32 s = filter->data;
-	enum rc_type protocol = dev->wakeup_protocol;
+	u32 mask, s = filter->data;
+	enum rc_proto protocol = dev->wakeup_protocol;
+
+	if (protocol >= ARRAY_SIZE(protocols))
+		return -EINVAL;
+
+	mask = protocols[protocol].scancode_bits;
 
 	switch (protocol) {
-	case RC_TYPE_NECX:
+	case RC_PROTO_NECX:
 		if ((((s >> 16) ^ ~(s >> 8)) & 0xff) == 0)
 			return -EINVAL;
 		break;
-	case RC_TYPE_NEC32:
+	case RC_PROTO_NEC32:
 		if ((((s >> 24) ^ ~(s >> 16)) & 0xff) == 0)
 			return -EINVAL;
 		break;
-	case RC_TYPE_RC6_MCE:
+	case RC_PROTO_RC6_MCE:
 		if ((s & 0xffff0000) != 0x800f0000)
 			return -EINVAL;
 		break;
-	case RC_TYPE_RC6_6A_32:
+	case RC_PROTO_RC6_6A_32:
 		if ((s & 0xffff0000) == 0x800f0000)
 			return -EINVAL;
 		break;
@@ -778,14 +811,13 @@ static int rc_validate_filter(struct rc_dev *dev,
 		break;
 	}
 
-	filter->data &= masks[protocol];
-	filter->mask &= masks[protocol];
+	filter->data &= mask;
+	filter->mask &= mask;
 
 	/*
 	 * If we have to raw encode the IR for wakeup, we cannot have a mask
 	 */
-	if (dev->encode_wakeup &&
-	    filter->mask != 0 && filter->mask != masks[protocol])
+	if (dev->encode_wakeup && filter->mask != 0 && filter->mask != mask)
 		return -EINVAL;
 
 	return 0;
@@ -859,30 +891,30 @@ static const struct {
 	const char	*name;
 	const char	*module_name;
 } proto_names[] = {
-	{ RC_BIT_NONE,		"none",		NULL			},
-	{ RC_BIT_OTHER,		"other",	NULL			},
-	{ RC_BIT_UNKNOWN,	"unknown",	NULL			},
-	{ RC_BIT_RC5 |
-	  RC_BIT_RC5X_20,	"rc-5",		"ir-rc5-decoder"	},
-	{ RC_BIT_NEC |
-	  RC_BIT_NECX |
-	  RC_BIT_NEC32,		"nec",		"ir-nec-decoder"	},
-	{ RC_BIT_RC6_0 |
-	  RC_BIT_RC6_6A_20 |
-	  RC_BIT_RC6_6A_24 |
-	  RC_BIT_RC6_6A_32 |
-	  RC_BIT_RC6_MCE,	"rc-6",		"ir-rc6-decoder"	},
-	{ RC_BIT_JVC,		"jvc",		"ir-jvc-decoder"	},
-	{ RC_BIT_SONY12 |
-	  RC_BIT_SONY15 |
-	  RC_BIT_SONY20,	"sony",		"ir-sony-decoder"	},
-	{ RC_BIT_RC5_SZ,	"rc-5-sz",	"ir-rc5-decoder"	},
-	{ RC_BIT_SANYO,		"sanyo",	"ir-sanyo-decoder"	},
-	{ RC_BIT_SHARP,		"sharp",	"ir-sharp-decoder"	},
-	{ RC_BIT_MCIR2_KBD |
-	  RC_BIT_MCIR2_MSE,	"mce_kbd",	"ir-mce_kbd-decoder"	},
-	{ RC_BIT_XMP,		"xmp",		"ir-xmp-decoder"	},
-	{ RC_BIT_CEC,		"cec",		NULL			},
+	{ RC_PROTO_BIT_NONE,	"none",		NULL			},
+	{ RC_PROTO_BIT_OTHER,	"other",	NULL			},
+	{ RC_PROTO_BIT_UNKNOWN,	"unknown",	NULL			},
+	{ RC_PROTO_BIT_RC5 |
+	  RC_PROTO_BIT_RC5X_20,	"rc-5",		"ir-rc5-decoder"	},
+	{ RC_PROTO_BIT_NEC |
+	  RC_PROTO_BIT_NECX |
+	  RC_PROTO_BIT_NEC32,	"nec",		"ir-nec-decoder"	},
+	{ RC_PROTO_BIT_RC6_0 |
+	  RC_PROTO_BIT_RC6_6A_20 |
+	  RC_PROTO_BIT_RC6_6A_24 |
+	  RC_PROTO_BIT_RC6_6A_32 |
+	  RC_PROTO_BIT_RC6_MCE,	"rc-6",		"ir-rc6-decoder"	},
+	{ RC_PROTO_BIT_JVC,	"jvc",		"ir-jvc-decoder"	},
+	{ RC_PROTO_BIT_SONY12 |
+	  RC_PROTO_BIT_SONY15 |
+	  RC_PROTO_BIT_SONY20,	"sony",		"ir-sony-decoder"	},
+	{ RC_PROTO_BIT_RC5_SZ,	"rc-5-sz",	"ir-rc5-decoder"	},
+	{ RC_PROTO_BIT_SANYO,	"sanyo",	"ir-sanyo-decoder"	},
+	{ RC_PROTO_BIT_SHARP,	"sharp",	"ir-sharp-decoder"	},
+	{ RC_PROTO_BIT_MCIR2_KBD |
+	  RC_PROTO_BIT_MCIR2_MSE, "mce_kbd",	"ir-mce_kbd-decoder"	},
+	{ RC_PROTO_BIT_XMP,	"xmp",		"ir-xmp-decoder"	},
+	{ RC_PROTO_BIT_CEC,	"cec",		NULL			},
 };
 
 /**
@@ -1052,8 +1084,9 @@ static void ir_raw_load_modules(u64 *protocols)
 	int i, ret;
 
 	for (i = 0; i < ARRAY_SIZE(proto_names); i++) {
-		if (proto_names[i].type == RC_BIT_NONE ||
-		    proto_names[i].type & (RC_BIT_OTHER | RC_BIT_UNKNOWN))
+		if (proto_names[i].type == RC_PROTO_BIT_NONE ||
+		    proto_names[i].type & (RC_PROTO_BIT_OTHER |
+					   RC_PROTO_BIT_UNKNOWN))
 			continue;
 
 		available = ir_raw_get_allowed_protocols();
@@ -1271,7 +1304,7 @@ static ssize_t store_filter(struct device *device,
 		 * Refuse to set a filter unless a protocol is enabled
 		 * and the filter is valid for that protocol
 		 */
-		if (dev->wakeup_protocol != RC_TYPE_UNKNOWN)
+		if (dev->wakeup_protocol != RC_PROTO_UNKNOWN)
 			ret = rc_validate_filter(dev, &new_filter);
 		else
 			ret = -EINVAL;
@@ -1298,40 +1331,6 @@ unlock:
 	return (ret < 0) ? ret : len;
 }
 
-/*
- * This is the list of all variants of all protocols, which is used by
- * the wakeup_protocols sysfs entry. In the protocols sysfs entry some
- * some protocols are grouped together (e.g. nec = nec + necx + nec32).
- *
- * For wakeup we need to know the exact protocol variant so the hardware
- * can be programmed exactly what to expect.
- */
-static const char * const proto_variant_names[] = {
-	[RC_TYPE_UNKNOWN] = "unknown",
-	[RC_TYPE_OTHER] = "other",
-	[RC_TYPE_RC5] = "rc-5",
-	[RC_TYPE_RC5X_20] = "rc-5x-20",
-	[RC_TYPE_RC5_SZ] = "rc-5-sz",
-	[RC_TYPE_JVC] = "jvc",
-	[RC_TYPE_SONY12] = "sony-12",
-	[RC_TYPE_SONY15] = "sony-15",
-	[RC_TYPE_SONY20] = "sony-20",
-	[RC_TYPE_NEC] = "nec",
-	[RC_TYPE_NECX] = "nec-x",
-	[RC_TYPE_NEC32] = "nec-32",
-	[RC_TYPE_SANYO] = "sanyo",
-	[RC_TYPE_MCIR2_KBD] = "mcir2-kbd",
-	[RC_TYPE_MCIR2_MSE] = "mcir2-mse",
-	[RC_TYPE_RC6_0] = "rc-6-0",
-	[RC_TYPE_RC6_6A_20] = "rc-6-6a-20",
-	[RC_TYPE_RC6_6A_24] = "rc-6-6a-24",
-	[RC_TYPE_RC6_6A_32] = "rc-6-6a-32",
-	[RC_TYPE_RC6_MCE] = "rc-6-mce",
-	[RC_TYPE_SHARP] = "sharp",
-	[RC_TYPE_XMP] = "xmp",
-	[RC_TYPE_CEC] = "cec",
-};
-
 /**
  * show_wakeup_protocols() - shows the wakeup IR protocol
  * @device:	the device descriptor
@@ -1352,7 +1351,7 @@ static ssize_t show_wakeup_protocols(struct device *device,
 {
 	struct rc_dev *dev = to_rc_dev(device);
 	u64 allowed;
-	enum rc_type enabled;
+	enum rc_proto enabled;
 	char *tmp = buf;
 	int i;
 
@@ -1366,14 +1365,12 @@ static ssize_t show_wakeup_protocols(struct device *device,
 	IR_dprintk(1, "%s: allowed - 0x%llx, enabled - %d\n",
 		   __func__, (long long)allowed, enabled);
 
-	for (i = 0; i < ARRAY_SIZE(proto_variant_names); i++) {
+	for (i = 0; i < ARRAY_SIZE(protocols); i++) {
 		if (allowed & (1ULL << i)) {
 			if (i == enabled)
-				tmp += sprintf(tmp, "[%s] ",
-						proto_variant_names[i]);
+				tmp += sprintf(tmp, "[%s] ", protocols[i].name);
 			else
-				tmp += sprintf(tmp, "%s ",
-						proto_variant_names[i]);
+				tmp += sprintf(tmp, "%s ", protocols[i].name);
 		}
 	}
 
@@ -1403,7 +1400,7 @@ static ssize_t store_wakeup_protocols(struct device *device,
 				      const char *buf, size_t len)
 {
 	struct rc_dev *dev = to_rc_dev(device);
-	enum rc_type protocol;
+	enum rc_proto protocol;
 	ssize_t rc;
 	u64 allowed;
 	int i;
@@ -1413,17 +1410,17 @@ static ssize_t store_wakeup_protocols(struct device *device,
 	allowed = dev->allowed_wakeup_protocols;
 
 	if (sysfs_streq(buf, "none")) {
-		protocol = RC_TYPE_UNKNOWN;
+		protocol = RC_PROTO_UNKNOWN;
 	} else {
-		for (i = 0; i < ARRAY_SIZE(proto_variant_names); i++) {
+		for (i = 0; i < ARRAY_SIZE(protocols); i++) {
 			if ((allowed & (1ULL << i)) &&
-			    sysfs_streq(buf, proto_variant_names[i])) {
+			    sysfs_streq(buf, protocols[i].name)) {
 				protocol = i;
 				break;
 			}
 		}
 
-		if (i == ARRAY_SIZE(proto_variant_names)) {
+		if (i == ARRAY_SIZE(protocols)) {
 			rc = -EINVAL;
 			goto out;
 		}
@@ -1443,7 +1440,7 @@ static ssize_t store_wakeup_protocols(struct device *device,
 		dev->wakeup_protocol = protocol;
 		IR_dprintk(1, "Wakeup protocol changed to %d\n", protocol);
 
-		if (protocol == RC_TYPE_RC6_MCE)
+		if (protocol == RC_PROTO_RC6_MCE)
 			dev->scancode_wakeup_filter.data = 0x800f0000;
 		else
 			dev->scancode_wakeup_filter.data = 0;
@@ -1507,7 +1504,7 @@ static struct attribute *rc_dev_protocol_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group rc_dev_protocol_attr_grp = {
+static const struct attribute_group rc_dev_protocol_attr_grp = {
 	.attrs	= rc_dev_protocol_attrs,
 };
 
@@ -1517,7 +1514,7 @@ static struct attribute *rc_dev_filter_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group rc_dev_filter_attr_grp = {
+static const struct attribute_group rc_dev_filter_attr_grp = {
 	.attrs	= rc_dev_filter_attrs,
 };
 
@@ -1528,7 +1525,7 @@ static struct attribute *rc_dev_wakeup_filter_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group rc_dev_wakeup_filter_attr_grp = {
+static const struct attribute_group rc_dev_wakeup_filter_attr_grp = {
 	.attrs	= rc_dev_wakeup_filter_attrs,
 };
 
@@ -1624,7 +1621,7 @@ static int rc_prepare_rx_device(struct rc_dev *dev)
 {
 	int rc;
 	struct rc_map *rc_map;
-	u64 rc_type;
+	u64 rc_proto;
 
 	if (!dev->map_name)
 		return -EINVAL;
@@ -1639,17 +1636,17 @@ static int rc_prepare_rx_device(struct rc_dev *dev)
 	if (rc)
 		return rc;
 
-	rc_type = BIT_ULL(rc_map->rc_type);
+	rc_proto = BIT_ULL(rc_map->rc_proto);
 
 	if (dev->change_protocol) {
-		rc = dev->change_protocol(dev, &rc_type);
+		rc = dev->change_protocol(dev, &rc_proto);
 		if (rc < 0)
 			goto out_table;
-		dev->enabled_protocols = rc_type;
+		dev->enabled_protocols = rc_proto;
 	}
 
 	if (dev->driver_type == RC_DRIVER_IR_RAW)
-		ir_raw_load_modules(&rc_type);
+		ir_raw_load_modules(&rc_proto);
 
 	set_bit(EV_KEY, dev->input_dev->evbit);
 	set_bit(EV_REP, dev->input_dev->evbit);
@@ -1663,7 +1660,7 @@ static int rc_prepare_rx_device(struct rc_dev *dev)
 	dev->input_dev->dev.parent = &dev->dev;
 	memcpy(&dev->input_dev->id, &dev->input_id, sizeof(dev->input_id));
 	dev->input_dev->phys = dev->input_phys;
-	dev->input_dev->name = dev->input_name;
+	dev->input_dev->name = dev->device_name;
 
 	return 0;
 
@@ -1759,7 +1756,7 @@ int rc_register_device(struct rc_dev *dev)
 
 	path = kobject_get_path(&dev->dev.kobj, GFP_KERNEL);
 	dev_info(&dev->dev, "%s as %s\n",
-		dev->input_name ?: "Unspecified device", path ?: "N/A");
+		 dev->device_name ?: "Unspecified device", path ?: "N/A");
 	kfree(path);
 
 	if (dev->driver_type != RC_DRIVER_IR_RAW_TX) {
diff --git a/drivers/media/rc/redrat3.c b/drivers/media/rc/redrat3.c
index 56d43be2756b..6784cb9fc4e7 100644
--- a/drivers/media/rc/redrat3.c
+++ b/drivers/media/rc/redrat3.c
@@ -951,12 +951,12 @@ static struct rc_dev *redrat3_init_rc_dev(struct redrat3_dev *rr3)
 
 	usb_make_path(rr3->udev, rr3->phys, sizeof(rr3->phys));
 
-	rc->input_name = rr3->name;
+	rc->device_name = rr3->name;
 	rc->input_phys = rr3->phys;
 	usb_to_input_id(rr3->udev, &rc->input_id);
 	rc->dev.parent = dev;
 	rc->priv = rr3;
-	rc->allowed_protocols = RC_BIT_ALL_IR_DECODER;
+	rc->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
 	rc->min_timeout = MS_TO_NS(RR3_RX_MIN_TIMEOUT);
 	rc->max_timeout = MS_TO_NS(RR3_RX_MAX_TIMEOUT);
 	rc->timeout = US_TO_NS(redrat3_get_timeout(rr3));
diff --git a/drivers/media/rc/serial_ir.c b/drivers/media/rc/serial_ir.c
index 77d5d4cbed0a..8b66926bc16a 100644
--- a/drivers/media/rc/serial_ir.c
+++ b/drivers/media/rc/serial_ir.c
@@ -139,10 +139,8 @@ struct serial_ir {
 	struct platform_device *pdev;
 	struct timer_list timeout_timer;
 
-	unsigned int freq;
+	unsigned int carrier;
 	unsigned int duty_cycle;
-
-	unsigned int pulse_width, space_width;
 };
 
 static struct serial_ir serial_ir;
@@ -183,18 +181,6 @@ static void off(void)
 		soutp(UART_MCR, hardware[type].off);
 }
 
-static void init_timing_params(unsigned int new_duty_cycle,
-			       unsigned int new_freq)
-{
-	serial_ir.duty_cycle = new_duty_cycle;
-	serial_ir.freq = new_freq;
-
-	serial_ir.pulse_width = DIV_ROUND_CLOSEST(
-		new_duty_cycle * NSEC_PER_SEC, new_freq * 100l);
-	serial_ir.space_width = DIV_ROUND_CLOSEST(
-		(100l - new_duty_cycle) * NSEC_PER_SEC, new_freq * 100l);
-}
-
 static void send_pulse_irdeo(unsigned int length, ktime_t target)
 {
 	long rawbits;
@@ -241,13 +227,20 @@ static void send_pulse_homebrew_softcarrier(unsigned int length, ktime_t edge)
 	 * ndelay(s64) does not compile; so use s32 rather than s64.
 	 */
 	s32 delta;
+	unsigned int pulse, space;
+
+	/* Ensure the dividend fits into 32 bit */
+	pulse = DIV_ROUND_CLOSEST(serial_ir.duty_cycle * (NSEC_PER_SEC / 100),
+				  serial_ir.carrier);
+	space = DIV_ROUND_CLOSEST((100 - serial_ir.duty_cycle) *
+				  (NSEC_PER_SEC / 100), serial_ir.carrier);
 
 	for (;;) {
 		now = ktime_get();
 		if (ktime_compare(now, target) >= 0)
 			break;
 		on();
-		edge = ktime_add_ns(edge, serial_ir.pulse_width);
+		edge = ktime_add_ns(edge, pulse);
 		delta = ktime_to_ns(ktime_sub(edge, now));
 		if (delta > 0)
 			ndelay(delta);
@@ -255,7 +248,7 @@ static void send_pulse_homebrew_softcarrier(unsigned int length, ktime_t edge)
 		off();
 		if (ktime_compare(now, target) >= 0)
 			break;
-		edge = ktime_add_ns(edge, serial_ir.space_width);
+		edge = ktime_add_ns(edge, space);
 		delta = ktime_to_ns(ktime_sub(edge, now));
 		if (delta > 0)
 			ndelay(delta);
@@ -513,19 +506,19 @@ static int serial_ir_probe(struct platform_device *dev)
 
 	switch (type) {
 	case IR_HOMEBREW:
-		rcdev->input_name = "Serial IR type home-brew";
+		rcdev->device_name = "Serial IR type home-brew";
 		break;
 	case IR_IRDEO:
-		rcdev->input_name = "Serial IR type IRdeo";
+		rcdev->device_name = "Serial IR type IRdeo";
 		break;
 	case IR_IRDEO_REMOTE:
-		rcdev->input_name = "Serial IR type IRdeo remote";
+		rcdev->device_name = "Serial IR type IRdeo remote";
 		break;
 	case IR_ANIMAX:
-		rcdev->input_name = "Serial IR type AnimaX";
+		rcdev->device_name = "Serial IR type AnimaX";
 		break;
 	case IR_IGOR:
-		rcdev->input_name = "Serial IR type IgorPlug";
+		rcdev->device_name = "Serial IR type IgorPlug";
 		break;
 	}
 
@@ -537,7 +530,7 @@ static int serial_ir_probe(struct platform_device *dev)
 	rcdev->open = serial_ir_open;
 	rcdev->close = serial_ir_close;
 	rcdev->dev.parent = &serial_ir.pdev->dev;
-	rcdev->allowed_protocols = RC_BIT_ALL_IR_DECODER;
+	rcdev->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
 	rcdev->driver_name = KBUILD_MODNAME;
 	rcdev->map_name = RC_MAP_RC6_MCE;
 	rcdev->min_timeout = 1;
@@ -580,7 +573,8 @@ static int serial_ir_probe(struct platform_device *dev)
 		return result;
 
 	/* Initialize pulse/space widths */
-	init_timing_params(50, 38000);
+	serial_ir.duty_cycle = 50;
+	serial_ir.carrier = 38000;
 
 	/* If pin is high, then this must be an active low receiver. */
 	if (sense == -1) {
@@ -684,7 +678,7 @@ static int serial_ir_tx(struct rc_dev *dev, unsigned int *txbuf,
 
 static int serial_ir_tx_duty_cycle(struct rc_dev *dev, u32 cycle)
 {
-	init_timing_params(cycle, serial_ir.freq);
+	serial_ir.duty_cycle = cycle;
 	return 0;
 }
 
@@ -693,7 +687,7 @@ static int serial_ir_tx_carrier(struct rc_dev *dev, u32 carrier)
 	if (carrier > 500000 || carrier < 20000)
 		return -EINVAL;
 
-	init_timing_params(serial_ir.duty_cycle, carrier);
+	serial_ir.carrier = carrier;
 	return 0;
 }
 
diff --git a/drivers/media/rc/sir_ir.c b/drivers/media/rc/sir_ir.c
index 20234ba0b318..bc906fb128d5 100644
--- a/drivers/media/rc/sir_ir.c
+++ b/drivers/media/rc/sir_ir.c
@@ -155,7 +155,7 @@ static irqreturn_t sir_interrupt(int irq, void *dev_id)
 {
 	unsigned char data;
 	ktime_t curr_time;
-	static unsigned long delt;
+	unsigned long delt;
 	unsigned long deltintr;
 	unsigned long flags;
 	int counter = 0;
@@ -308,14 +308,14 @@ static int sir_ir_probe(struct platform_device *dev)
 	if (!rcdev)
 		return -ENOMEM;
 
-	rcdev->input_name = "SIR IrDA port";
+	rcdev->device_name = "SIR IrDA port";
 	rcdev->input_phys = KBUILD_MODNAME "/input0";
 	rcdev->input_id.bustype = BUS_HOST;
 	rcdev->input_id.vendor = 0x0001;
 	rcdev->input_id.product = 0x0001;
 	rcdev->input_id.version = 0x0100;
 	rcdev->tx_ir = sir_tx_ir;
-	rcdev->allowed_protocols = RC_BIT_ALL_IR_DECODER;
+	rcdev->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
 	rcdev->driver_name = KBUILD_MODNAME;
 	rcdev->map_name = RC_MAP_RC6_MCE;
 	rcdev->timeout = IR_DEFAULT_TIMEOUT;
diff --git a/drivers/media/rc/st_rc.c b/drivers/media/rc/st_rc.c
index a08e1dd06124..a8e39c635f34 100644
--- a/drivers/media/rc/st_rc.c
+++ b/drivers/media/rc/st_rc.c
@@ -280,7 +280,7 @@ static int st_rc_probe(struct platform_device *pdev)
 	else
 		rc_dev->rx_base = rc_dev->base;
 
-	rc_dev->rstc = reset_control_get_optional(dev, NULL);
+	rc_dev->rstc = reset_control_get_optional_exclusive(dev, NULL);
 	if (IS_ERR(rc_dev->rstc)) {
 		ret = PTR_ERR(rc_dev->rstc);
 		goto err;
@@ -290,7 +290,7 @@ static int st_rc_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, rc_dev);
 	st_rc_hardware_init(rc_dev);
 
-	rdev->allowed_protocols = RC_BIT_ALL_IR_DECODER;
+	rdev->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
 	/* rx sampling rate is 10Mhz */
 	rdev->rx_resolution = 100;
 	rdev->timeout = US_TO_NS(MAX_SYMB_TIME);
@@ -299,7 +299,7 @@ static int st_rc_probe(struct platform_device *pdev)
 	rdev->close = st_rc_close;
 	rdev->driver_name = IR_ST_NAME;
 	rdev->map_name = RC_MAP_EMPTY;
-	rdev->input_name = "ST Remote Control Receiver";
+	rdev->device_name = "ST Remote Control Receiver";
 
 	ret = rc_register_device(rdev);
 	if (ret < 0)
diff --git a/drivers/media/rc/streamzap.c b/drivers/media/rc/streamzap.c
index b09c45abb5f3..f03a174ddf9d 100644
--- a/drivers/media/rc/streamzap.c
+++ b/drivers/media/rc/streamzap.c
@@ -299,12 +299,12 @@ static struct rc_dev *streamzap_init_rc_dev(struct streamzap_ir *sz)
 	usb_make_path(sz->usbdev, sz->phys, sizeof(sz->phys));
 	strlcat(sz->phys, "/input0", sizeof(sz->phys));
 
-	rdev->input_name = sz->name;
+	rdev->device_name = sz->name;
 	rdev->input_phys = sz->phys;
 	usb_to_input_id(sz->usbdev, &rdev->input_id);
 	rdev->dev.parent = dev;
 	rdev->priv = sz;
-	rdev->allowed_protocols = RC_BIT_ALL_IR_DECODER;
+	rdev->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
 	rdev->driver_name = DRIVER_NAME;
 	rdev->map_name = RC_MAP_STREAMZAP;
 
diff --git a/drivers/media/rc/sunxi-cir.c b/drivers/media/rc/sunxi-cir.c
index 4b785dd775c1..97f367b446c4 100644
--- a/drivers/media/rc/sunxi-cir.c
+++ b/drivers/media/rc/sunxi-cir.c
@@ -173,7 +173,7 @@ static int sunxi_ir_probe(struct platform_device *pdev)
 	}
 
 	/* Reset (optional) */
-	ir->rst = devm_reset_control_get_optional(dev, NULL);
+	ir->rst = devm_reset_control_get_optional_exclusive(dev, NULL);
 	if (IS_ERR(ir->rst))
 		return PTR_ERR(ir->rst);
 	ret = reset_control_deassert(ir->rst);
@@ -215,7 +215,7 @@ static int sunxi_ir_probe(struct platform_device *pdev)
 	}
 
 	ir->rc->priv = ir;
-	ir->rc->input_name = SUNXI_IR_DEV;
+	ir->rc->device_name = SUNXI_IR_DEV;
 	ir->rc->input_phys = "sunxi-ir/input0";
 	ir->rc->input_id.bustype = BUS_HOST;
 	ir->rc->input_id.vendor = 0x0001;
@@ -224,7 +224,7 @@ static int sunxi_ir_probe(struct platform_device *pdev)
 	ir->map_name = of_get_property(dn, "linux,rc-map-name", NULL);
 	ir->rc->map_name = ir->map_name ?: RC_MAP_EMPTY;
 	ir->rc->dev.parent = dev;
-	ir->rc->allowed_protocols = RC_BIT_ALL_IR_DECODER;
+	ir->rc->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
 	ir->rc->rx_resolution = SUNXI_IR_SAMPLE;
 	ir->rc->timeout = MS_TO_NS(SUNXI_IR_TIMEOUT);
 	ir->rc->driver_name = SUNXI_IR_DEV;
diff --git a/drivers/media/rc/ttusbir.c b/drivers/media/rc/ttusbir.c
index 23be7702e2df..aafea3c5170b 100644
--- a/drivers/media/rc/ttusbir.c
+++ b/drivers/media/rc/ttusbir.c
@@ -309,11 +309,11 @@ static int ttusbir_probe(struct usb_interface *intf,
 
 	usb_make_path(tt->udev, tt->phys, sizeof(tt->phys));
 
-	rc->input_name = DRIVER_DESC;
+	rc->device_name = DRIVER_DESC;
 	rc->input_phys = tt->phys;
 	usb_to_input_id(tt->udev, &rc->input_id);
 	rc->dev.parent = &intf->dev;
-	rc->allowed_protocols = RC_BIT_ALL_IR_DECODER;
+	rc->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
 	rc->priv = tt;
 	rc->driver_name = DRIVER_NAME;
 	rc->map_name = RC_MAP_TT_1500;
diff --git a/drivers/media/rc/winbond-cir.c b/drivers/media/rc/winbond-cir.c
index 5a4d4a611197..3ca7ab48293d 100644
--- a/drivers/media/rc/winbond-cir.c
+++ b/drivers/media/rc/winbond-cir.c
@@ -429,7 +429,7 @@ wbcir_irq_tx(struct wbcir_data *data)
 		bytes[used] = byte;
 	}
 
-	while (data->txbuf[data->txoff] == 0 && data->txoff != data->txlen)
+	while (data->txoff != data->txlen && data->txbuf[data->txoff] == 0)
 		data->txoff++;
 
 	if (used == 0) {
@@ -697,7 +697,7 @@ wbcir_shutdown(struct pnp_dev *device)
 	}
 
 	switch (rc->wakeup_protocol) {
-	case RC_TYPE_RC5:
+	case RC_PROTO_RC5:
 		/* Mask = 13 bits, ex toggle */
 		mask[0]  = (mask_sc & 0x003f);
 		mask[0] |= (mask_sc & 0x0300) >> 2;
@@ -714,7 +714,7 @@ wbcir_shutdown(struct pnp_dev *device)
 		proto = IR_PROTOCOL_RC5;
 		break;
 
-	case RC_TYPE_NEC:
+	case RC_PROTO_NEC:
 		mask[1] = bitrev8(mask_sc);
 		mask[0] = mask[1];
 		mask[3] = bitrev8(mask_sc >> 8);
@@ -728,7 +728,7 @@ wbcir_shutdown(struct pnp_dev *device)
 		proto = IR_PROTOCOL_NEC;
 		break;
 
-	case RC_TYPE_NECX:
+	case RC_PROTO_NECX:
 		mask[1] = bitrev8(mask_sc);
 		mask[0] = mask[1];
 		mask[2] = bitrev8(mask_sc >> 8);
@@ -742,7 +742,7 @@ wbcir_shutdown(struct pnp_dev *device)
 		proto = IR_PROTOCOL_NEC;
 		break;
 
-	case RC_TYPE_NEC32:
+	case RC_PROTO_NEC32:
 		mask[0] = bitrev8(mask_sc);
 		mask[1] = bitrev8(mask_sc >> 8);
 		mask[2] = bitrev8(mask_sc >> 16);
@@ -756,7 +756,7 @@ wbcir_shutdown(struct pnp_dev *device)
 		proto = IR_PROTOCOL_NEC;
 		break;
 
-	case RC_TYPE_RC6_0:
+	case RC_PROTO_RC6_0:
 		/* Command */
 		match[0] = wbcir_to_rc6cells(wake_sc >> 0);
 		mask[0]  = wbcir_to_rc6cells(mask_sc >> 0);
@@ -779,9 +779,9 @@ wbcir_shutdown(struct pnp_dev *device)
 		proto = IR_PROTOCOL_RC6;
 		break;
 
-	case RC_TYPE_RC6_6A_24:
-	case RC_TYPE_RC6_6A_32:
-	case RC_TYPE_RC6_MCE:
+	case RC_PROTO_RC6_6A_24:
+	case RC_PROTO_RC6_6A_32:
+	case RC_PROTO_RC6_MCE:
 		i = 0;
 
 		/* Command */
@@ -800,13 +800,13 @@ wbcir_shutdown(struct pnp_dev *device)
 		match[i]  = wbcir_to_rc6cells(wake_sc >> 16);
 		mask[i++] = wbcir_to_rc6cells(mask_sc >> 16);
 
-		if (rc->wakeup_protocol == RC_TYPE_RC6_6A_20) {
+		if (rc->wakeup_protocol == RC_PROTO_RC6_6A_20) {
 			rc6_csl = 52;
 		} else {
 			match[i]  = wbcir_to_rc6cells(wake_sc >> 20);
 			mask[i++] = wbcir_to_rc6cells(mask_sc >> 20);
 
-			if (rc->wakeup_protocol == RC_TYPE_RC6_6A_24) {
+			if (rc->wakeup_protocol == RC_PROTO_RC6_6A_24) {
 				rc6_csl = 60;
 			} else {
 				/* Customer range bit and bits 15 - 8 */
@@ -1068,7 +1068,7 @@ wbcir_probe(struct pnp_dev *device, const struct pnp_device_id *dev_id)
 	}
 
 	data->dev->driver_name = DRVNAME;
-	data->dev->input_name = WBCIR_NAME;
+	data->dev->device_name = WBCIR_NAME;
 	data->dev->input_phys = "wbcir/cir0";
 	data->dev->input_id.bustype = BUS_HOST;
 	data->dev->input_id.vendor = PCI_VENDOR_ID_WINBOND;
@@ -1086,12 +1086,13 @@ wbcir_probe(struct pnp_dev *device, const struct pnp_device_id *dev_id)
 	data->dev->timeout = IR_DEFAULT_TIMEOUT;
 	data->dev->max_timeout = 10 * IR_DEFAULT_TIMEOUT;
 	data->dev->rx_resolution = US_TO_NS(2);
-	data->dev->allowed_protocols = RC_BIT_ALL_IR_DECODER;
-	data->dev->allowed_wakeup_protocols = RC_BIT_NEC | RC_BIT_NECX |
-			RC_BIT_NEC32 | RC_BIT_RC5 | RC_BIT_RC6_0 |
-			RC_BIT_RC6_6A_20 | RC_BIT_RC6_6A_24 |
-			RC_BIT_RC6_6A_32 | RC_BIT_RC6_MCE;
-	data->dev->wakeup_protocol = RC_TYPE_RC6_MCE;
+	data->dev->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
+	data->dev->allowed_wakeup_protocols = RC_PROTO_BIT_NEC |
+		RC_PROTO_BIT_NECX | RC_PROTO_BIT_NEC32 | RC_PROTO_BIT_RC5 |
+		RC_PROTO_BIT_RC6_0 | RC_PROTO_BIT_RC6_6A_20 |
+		RC_PROTO_BIT_RC6_6A_24 | RC_PROTO_BIT_RC6_6A_32 |
+		RC_PROTO_BIT_RC6_MCE;
+	data->dev->wakeup_protocol = RC_PROTO_RC6_MCE;
 	data->dev->scancode_wakeup_filter.data = 0x800f040c;
 	data->dev->scancode_wakeup_filter.mask = 0xffff7fff;
 	data->dev->s_wakeup_filter = wbcir_set_wakeup_filter;
diff --git a/drivers/media/rc/zx-irdec.c b/drivers/media/rc/zx-irdec.c
new file mode 100644
index 000000000000..12d322ec8a29
--- /dev/null
+++ b/drivers/media/rc/zx-irdec.c
@@ -0,0 +1,184 @@
+/*
+ * Copyright (C) 2017 Sanechips Technology Co., Ltd.
+ * Copyright 2017 Linaro Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+
+#include <media/rc-core.h>
+
+#define DRIVER_NAME		"zx-irdec"
+
+#define ZX_IR_ENABLE		0x04
+#define ZX_IREN			BIT(0)
+#define ZX_IR_CTRL		0x08
+#define ZX_DEGL_MASK		GENMASK(21, 20)
+#define ZX_DEGL_VALUE(x)	(((x) << 20) & ZX_DEGL_MASK)
+#define ZX_WDBEGIN_MASK		GENMASK(18, 8)
+#define ZX_WDBEGIN_VALUE(x)	(((x) << 8) & ZX_WDBEGIN_MASK)
+#define ZX_IR_INTEN		0x10
+#define ZX_IR_INTSTCLR		0x14
+#define ZX_IR_CODE		0x30
+#define ZX_IR_CNUM		0x34
+#define ZX_NECRPT		BIT(16)
+
+struct zx_irdec {
+	void __iomem *base;
+	struct rc_dev *rcd;
+};
+
+static void zx_irdec_set_mask(struct zx_irdec *irdec, unsigned int reg,
+			      u32 mask, u32 value)
+{
+	u32 data;
+
+	data = readl(irdec->base + reg);
+	data &= ~mask;
+	data |= value & mask;
+	writel(data, irdec->base + reg);
+}
+
+static irqreturn_t zx_irdec_irq(int irq, void *dev_id)
+{
+	struct zx_irdec *irdec = dev_id;
+	u8 address, not_address;
+	u8 command, not_command;
+	u32 rawcode, scancode;
+	enum rc_proto rc_proto;
+
+	/* Clear interrupt */
+	writel(1, irdec->base + ZX_IR_INTSTCLR);
+
+	/* Check repeat frame */
+	if (readl(irdec->base + ZX_IR_CNUM) & ZX_NECRPT) {
+		rc_repeat(irdec->rcd);
+		goto done;
+	}
+
+	rawcode = readl(irdec->base + ZX_IR_CODE);
+	not_command = (rawcode >> 24) & 0xff;
+	command = (rawcode >> 16) & 0xff;
+	not_address = (rawcode >> 8) & 0xff;
+	address = rawcode & 0xff;
+
+	scancode = ir_nec_bytes_to_scancode(address, not_address,
+					    command, not_command,
+					    &rc_proto);
+	rc_keydown(irdec->rcd, rc_proto, scancode, 0);
+
+done:
+	return IRQ_HANDLED;
+}
+
+static int zx_irdec_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct zx_irdec *irdec;
+	struct resource *res;
+	struct rc_dev *rcd;
+	int irq;
+	int ret;
+
+	irdec = devm_kzalloc(dev, sizeof(*irdec), GFP_KERNEL);
+	if (!irdec)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	irdec->base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(irdec->base))
+		return PTR_ERR(irdec->base);
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return irq;
+
+	rcd = devm_rc_allocate_device(dev, RC_DRIVER_SCANCODE);
+	if (!rcd) {
+		dev_err(dev, "failed to allocate rc device\n");
+		return -ENOMEM;
+	}
+
+	irdec->rcd = rcd;
+
+	rcd->priv = irdec;
+	rcd->input_phys = DRIVER_NAME "/input0";
+	rcd->input_id.bustype = BUS_HOST;
+	rcd->map_name = RC_MAP_ZX_IRDEC;
+	rcd->allowed_protocols = RC_PROTO_BIT_NEC | RC_PROTO_BIT_NECX |
+							RC_PROTO_BIT_NEC32;
+	rcd->driver_name = DRIVER_NAME;
+	rcd->device_name = DRIVER_NAME;
+
+	platform_set_drvdata(pdev, irdec);
+
+	ret = devm_rc_register_device(dev, rcd);
+	if (ret) {
+		dev_err(dev, "failed to register rc device\n");
+		return ret;
+	}
+
+	ret = devm_request_irq(dev, irq, zx_irdec_irq, 0, NULL, irdec);
+	if (ret) {
+		dev_err(dev, "failed to request irq\n");
+		return ret;
+	}
+
+	/*
+	 * Initialize deglitch level and watchdog counter beginner as
+	 * recommended by vendor BSP code.
+	 */
+	zx_irdec_set_mask(irdec, ZX_IR_CTRL, ZX_DEGL_MASK, ZX_DEGL_VALUE(0));
+	zx_irdec_set_mask(irdec, ZX_IR_CTRL, ZX_WDBEGIN_MASK,
+			  ZX_WDBEGIN_VALUE(0x21c));
+
+	/* Enable interrupt */
+	writel(1, irdec->base + ZX_IR_INTEN);
+
+	/* Enable the decoder */
+	zx_irdec_set_mask(irdec, ZX_IR_ENABLE, ZX_IREN, ZX_IREN);
+
+	return 0;
+}
+
+static int zx_irdec_remove(struct platform_device *pdev)
+{
+	struct zx_irdec *irdec = platform_get_drvdata(pdev);
+
+	/* Disable the decoder */
+	zx_irdec_set_mask(irdec, ZX_IR_ENABLE, ZX_IREN, 0);
+
+	/* Disable interrupt */
+	writel(0, irdec->base + ZX_IR_INTEN);
+
+	return 0;
+}
+
+static const struct of_device_id zx_irdec_match[] = {
+	{ .compatible = "zte,zx296718-irdec" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, zx_irdec_match);
+
+static struct platform_driver zx_irdec_driver = {
+	.probe = zx_irdec_probe,
+	.remove = zx_irdec_remove,
+	.driver = {
+		.name = DRIVER_NAME,
+		.of_match_table	= zx_irdec_match,
+	},
+};
+module_platform_driver(zx_irdec_driver);
+
+MODULE_DESCRIPTION("ZTE ZX IR remote control driver");
+MODULE_AUTHOR("Shawn Guo <shawn.guo@linaro.org>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/tuners/fc0012.c b/drivers/media/tuners/fc0012.c
index dcc323ffbde7..625ac6f51c39 100644
--- a/drivers/media/tuners/fc0012.c
+++ b/drivers/media/tuners/fc0012.c
@@ -351,7 +351,7 @@ static int fc0012_get_rf_strength(struct dvb_frontend *fe, u16 *strength)
 	int ret;
 	unsigned char tmp;
 	int int_temp, lna_gain, int_lna, tot_agc_gain, power;
-	const int fc0012_lna_gain_table[] = {
+	static const int fc0012_lna_gain_table[] = {
 		/* low gain */
 		-63, -58, -99, -73,
 		-63, -65, -54, -60,
diff --git a/drivers/media/tuners/fc0013.c b/drivers/media/tuners/fc0013.c
index 91dfa770a5cc..e606118d1a9b 100644
--- a/drivers/media/tuners/fc0013.c
+++ b/drivers/media/tuners/fc0013.c
@@ -511,7 +511,7 @@ static int fc0013_get_rf_strength(struct dvb_frontend *fe, u16 *strength)
 	int ret;
 	unsigned char tmp;
 	int int_temp, lna_gain, int_lna, tot_agc_gain, power;
-	const int fc0013_lna_gain_table[] = {
+	static const int fc0013_lna_gain_table[] = {
 		/* low gain */
 		-63, -58, -99, -73,
 		-63, -65, -54, -60,
diff --git a/drivers/media/tuners/tda18271-maps.c b/drivers/media/tuners/tda18271-maps.c
index 7d114677b4ca..9679804fd219 100644
--- a/drivers/media/tuners/tda18271-maps.c
+++ b/drivers/media/tuners/tda18271-maps.c
@@ -1182,7 +1182,7 @@ fail:
 
 /*---------------------------------------------------------------------*/
 
-static struct tda18271_std_map tda18271c1_std_map = {
+static const struct tda18271_std_map tda18271c1_std_map = {
 	.fm_radio = { .if_freq = 1250, .fm_rfn = 1, .agc_mode = 3, .std = 0,
 		      .if_lvl = 0, .rfagc_top = 0x2c, }, /* EP3[4:0] 0x18 */
 	.atv_b    = { .if_freq = 6750, .fm_rfn = 0, .agc_mode = 1, .std = 6,
@@ -1215,7 +1215,7 @@ static struct tda18271_std_map tda18271c1_std_map = {
 		      .if_lvl = 1, .rfagc_top = 0x37, }, /* EP3[4:0] 0x1f */
 };
 
-static struct tda18271_std_map tda18271c2_std_map = {
+static const struct tda18271_std_map tda18271c2_std_map = {
 	.fm_radio = { .if_freq = 1250, .fm_rfn = 1, .agc_mode = 3, .std = 0,
 		      .if_lvl = 0, .rfagc_top = 0x2c, }, /* EP3[4:0] 0x18 */
 	.atv_b    = { .if_freq = 6000, .fm_rfn = 0, .agc_mode = 1, .std = 5,
diff --git a/drivers/media/tuners/tuner-simple.c b/drivers/media/tuners/tuner-simple.c
index 3339b13dd3f5..cf44d3657f55 100644
--- a/drivers/media/tuners/tuner-simple.c
+++ b/drivers/media/tuners/tuner-simple.c
@@ -846,7 +846,7 @@ static u32 simple_dvb_configure(struct dvb_frontend *fe, u8 *buf,
 	/* This function returns the tuned frequency on success, 0 on error */
 	struct tuner_simple_priv *priv = fe->tuner_priv;
 	struct tunertype *tun = priv->tun;
-	static struct tuner_params *t_params;
+	struct tuner_params *t_params;
 	u8 config, cb;
 	u32 div;
 	int ret;
diff --git a/drivers/media/usb/airspy/airspy.c b/drivers/media/usb/airspy/airspy.c
index 8251942bcd12..e70c9e2f3798 100644
--- a/drivers/media/usb/airspy/airspy.c
+++ b/drivers/media/usb/airspy/airspy.c
@@ -859,7 +859,7 @@ static const struct v4l2_file_operations airspy_fops = {
 	.unlocked_ioctl           = video_ioctl2,
 };
 
-static struct video_device airspy_template = {
+static const struct video_device airspy_template = {
 	.name                     = "AirSpy SDR",
 	.release                  = video_device_release_empty,
 	.fops                     = &airspy_fops,
@@ -1087,7 +1087,7 @@ err_free_mem:
 }
 
 /* USB device ID list */
-static struct usb_device_id airspy_id_table[] = {
+static const struct usb_device_id airspy_id_table[] = {
 	{ USB_DEVICE(0x1d50, 0x60a1) }, /* AirSpy */
 	{ }
 };
diff --git a/drivers/media/usb/as102/as102_usb_drv.c b/drivers/media/usb/as102/as102_usb_drv.c
index 68c3a80ce349..ea57859aee77 100644
--- a/drivers/media/usb/as102/as102_usb_drv.c
+++ b/drivers/media/usb/as102/as102_usb_drv.c
@@ -33,7 +33,7 @@ static void as102_usb_stop_stream(struct as102_dev_t *dev);
 static int as102_open(struct inode *inode, struct file *file);
 static int as102_release(struct inode *inode, struct file *file);
 
-static struct usb_device_id as102_usb_id_table[] = {
+static const struct usb_device_id as102_usb_id_table[] = {
 	{ USB_DEVICE(AS102_USB_DEVICE_VENDOR_ID, AS102_USB_DEVICE_PID_0001) },
 	{ USB_DEVICE(PCTV_74E_USB_VID, PCTV_74E_USB_PID) },
 	{ USB_DEVICE(ELGATO_EYETV_DTT_USB_VID, ELGATO_EYETV_DTT_USB_PID) },
diff --git a/drivers/media/usb/au0828/Kconfig b/drivers/media/usb/au0828/Kconfig
index 78b797e0b434..70521e0b4c53 100644
--- a/drivers/media/usb/au0828/Kconfig
+++ b/drivers/media/usb/au0828/Kconfig
@@ -31,6 +31,7 @@ config VIDEO_AU0828_V4L2
 config VIDEO_AU0828_RC
 	bool "AU0828 Remote Controller support"
 	depends on RC_CORE
+	depends on !(RC_CORE=m && VIDEO_AU0828=y)
 	depends on VIDEO_AU0828
 	---help---
 	   Enables Remote Controller support on au0828 driver.
diff --git a/drivers/media/usb/au0828/au0828-core.c b/drivers/media/usb/au0828/au0828-core.c
index 739df61cec4f..cd363a2100d4 100644
--- a/drivers/media/usb/au0828/au0828-core.c
+++ b/drivers/media/usb/au0828/au0828-core.c
@@ -628,6 +628,8 @@ static int au0828_usb_probe(struct usb_interface *interface,
 	if (retval) {
 		pr_err("%s() au0282_dev_register failed to register on V4L2\n",
 			__func__);
+		mutex_unlock(&dev->lock);
+		kfree(dev);
 		goto done;
 	}
 
diff --git a/drivers/media/usb/au0828/au0828-i2c.c b/drivers/media/usb/au0828/au0828-i2c.c
index 42b352bb4f02..ef7d1b830ca3 100644
--- a/drivers/media/usb/au0828/au0828-i2c.c
+++ b/drivers/media/usb/au0828/au0828-i2c.c
@@ -329,14 +329,14 @@ static u32 au0828_functionality(struct i2c_adapter *adap)
 	return I2C_FUNC_SMBUS_EMUL | I2C_FUNC_I2C;
 }
 
-static struct i2c_algorithm au0828_i2c_algo_template = {
+static const struct i2c_algorithm au0828_i2c_algo_template = {
 	.master_xfer	= i2c_xfer,
 	.functionality	= au0828_functionality,
 };
 
 /* ----------------------------------------------------------------------- */
 
-static struct i2c_adapter au0828_i2c_adap_template = {
+static const struct i2c_adapter au0828_i2c_adap_template = {
 	.name              = KBUILD_MODNAME,
 	.owner             = THIS_MODULE,
 	.algo              = &au0828_i2c_algo_template,
diff --git a/drivers/media/usb/au0828/au0828-input.c b/drivers/media/usb/au0828/au0828-input.c
index 9d82ec0a4b64..7996eb83a54e 100644
--- a/drivers/media/usb/au0828/au0828-input.c
+++ b/drivers/media/usb/au0828/au0828-input.c
@@ -335,7 +335,7 @@ int au0828_rc_register(struct au0828_dev *dev)
 	usb_make_path(dev->usbdev, ir->phys, sizeof(ir->phys));
 	strlcat(ir->phys, "/input0", sizeof(ir->phys));
 
-	rc->input_name = ir->name;
+	rc->device_name = ir->name;
 	rc->input_phys = ir->phys;
 	rc->input_id.bustype = BUS_USB;
 	rc->input_id.version = 1;
@@ -343,8 +343,8 @@ int au0828_rc_register(struct au0828_dev *dev)
 	rc->input_id.product = le16_to_cpu(dev->usbdev->descriptor.idProduct);
 	rc->dev.parent = &dev->usbdev->dev;
 	rc->driver_name = "au0828-input";
-	rc->allowed_protocols = RC_BIT_NEC | RC_BIT_NECX | RC_BIT_NEC32 |
-								RC_BIT_RC5;
+	rc->allowed_protocols = RC_PROTO_BIT_NEC | RC_PROTO_BIT_NECX |
+				RC_PROTO_BIT_NEC32 | RC_PROTO_BIT_RC5;
 
 	/* all done */
 	err = rc_register_device(rc);
diff --git a/drivers/media/usb/au0828/au0828-video.c b/drivers/media/usb/au0828/au0828-video.c
index 2a255bd32bb3..9342402b92f7 100644
--- a/drivers/media/usb/au0828/au0828-video.c
+++ b/drivers/media/usb/au0828/au0828-video.c
@@ -1740,7 +1740,7 @@ void au0828_v4l2_resume(struct au0828_dev *dev)
 	}
 }
 
-static struct v4l2_file_operations au0828_v4l_fops = {
+static const struct v4l2_file_operations au0828_v4l_fops = {
 	.owner      = THIS_MODULE,
 	.open       = au0828_v4l2_open,
 	.release    = au0828_v4l2_close,
diff --git a/drivers/media/usb/b2c2/flexcop-usb.c b/drivers/media/usb/b2c2/flexcop-usb.c
index 788c73803138..a8f3169e30b3 100644
--- a/drivers/media/usb/b2c2/flexcop-usb.c
+++ b/drivers/media/usb/b2c2/flexcop-usb.c
@@ -596,7 +596,7 @@ static void flexcop_usb_disconnect(struct usb_interface *intf)
 	info("%s successfully deinitialized and disconnected.", DRIVER_NAME);
 }
 
-static struct usb_device_id flexcop_usb_table [] = {
+static const struct usb_device_id flexcop_usb_table[] = {
 	{ USB_DEVICE(0x0af7, 0x0101) },
 	{ }
 };
diff --git a/drivers/media/usb/cpia2/cpia2_usb.c b/drivers/media/usb/cpia2/cpia2_usb.c
index 1c7e16e5d88b..6089036049d9 100644
--- a/drivers/media/usb/cpia2/cpia2_usb.c
+++ b/drivers/media/usb/cpia2/cpia2_usb.c
@@ -60,7 +60,7 @@ static int submit_urbs(struct camera_data *cam);
 static int set_alternate(struct camera_data *cam, unsigned int alt);
 static int configure_transfer_mode(struct camera_data *cam, unsigned int alt);
 
-static struct usb_device_id cpia2_id_table[] = {
+static const struct usb_device_id cpia2_id_table[] = {
 	{USB_DEVICE(0x0553, 0x0100)},
 	{USB_DEVICE(0x0553, 0x0140)},
 	{USB_DEVICE(0x0553, 0x0151)},  /* STV0676 */
diff --git a/drivers/media/usb/cpia2/cpia2_v4l.c b/drivers/media/usb/cpia2/cpia2_v4l.c
index 7122023e7004..3dedd83f0b19 100644
--- a/drivers/media/usb/cpia2/cpia2_v4l.c
+++ b/drivers/media/usb/cpia2/cpia2_v4l.c
@@ -1075,7 +1075,7 @@ static const struct v4l2_file_operations cpia2_fops = {
 	.mmap		= cpia2_mmap,
 };
 
-static struct video_device cpia2_template = {
+static const struct video_device cpia2_template = {
 	/* I could not find any place for the old .initialize initializer?? */
 	.name =		"CPiA2 Camera",
 	.fops =		&cpia2_fops,
diff --git a/drivers/media/usb/cx231xx/cx231xx-417.c b/drivers/media/usb/cx231xx/cx231xx-417.c
index 509d9711d590..d538fa407742 100644
--- a/drivers/media/usb/cx231xx/cx231xx-417.c
+++ b/drivers/media/usb/cx231xx/cx231xx-417.c
@@ -1490,7 +1490,7 @@ static void bb_buf_release(struct videobuf_queue *q,
 	free_buffer(q, buf);
 }
 
-static struct videobuf_queue_ops cx231xx_qops = {
+static const struct videobuf_queue_ops cx231xx_qops = {
 	.buf_setup    = bb_buf_setup,
 	.buf_prepare  = bb_buf_prepare,
 	.buf_queue    = bb_buf_queue,
@@ -1843,7 +1843,7 @@ static int mpeg_mmap(struct file *file, struct vm_area_struct *vma)
 	return videobuf_mmap_mapper(&fh->vidq, vma);
 }
 
-static struct v4l2_file_operations mpeg_fops = {
+static const struct v4l2_file_operations mpeg_fops = {
 	.owner	       = THIS_MODULE,
 	.open	       = mpeg_open,
 	.release       = mpeg_release,
diff --git a/drivers/media/usb/cx231xx/cx231xx-audio.c b/drivers/media/usb/cx231xx/cx231xx-audio.c
index a050d125934c..06f10d7fc4b0 100644
--- a/drivers/media/usb/cx231xx/cx231xx-audio.c
+++ b/drivers/media/usb/cx231xx/cx231xx-audio.c
@@ -403,7 +403,7 @@ static int snd_pcm_alloc_vmalloc_buffer(struct snd_pcm_substream *subs,
 	return 0;
 }
 
-static struct snd_pcm_hardware snd_cx231xx_hw_capture = {
+static const struct snd_pcm_hardware snd_cx231xx_hw_capture = {
 	.info = SNDRV_PCM_INFO_BLOCK_TRANSFER 	|
 	    SNDRV_PCM_INFO_MMAP 		|
 	    SNDRV_PCM_INFO_INTERLEAVED 		|
diff --git a/drivers/media/usb/cx231xx/cx231xx-core.c b/drivers/media/usb/cx231xx/cx231xx-core.c
index 46646ecd2dbc..f372ad3917a8 100644
--- a/drivers/media/usb/cx231xx/cx231xx-core.c
+++ b/drivers/media/usb/cx231xx/cx231xx-core.c
@@ -1311,6 +1311,7 @@ int cx231xx_dev_init(struct cx231xx *dev)
 	dev->i2c_bus[0].i2c_period = I2C_SPEED_100K;	/* 100 KHz */
 	dev->i2c_bus[0].i2c_nostop = 0;
 	dev->i2c_bus[0].i2c_reserve = 0;
+	dev->i2c_bus[0].i2c_rc = -ENODEV;
 
 	/* External Master 2 Bus */
 	dev->i2c_bus[1].nr = 1;
@@ -1318,6 +1319,7 @@ int cx231xx_dev_init(struct cx231xx *dev)
 	dev->i2c_bus[1].i2c_period = I2C_SPEED_100K;	/* 100 KHz */
 	dev->i2c_bus[1].i2c_nostop = 0;
 	dev->i2c_bus[1].i2c_reserve = 0;
+	dev->i2c_bus[1].i2c_rc = -ENODEV;
 
 	/* Internal Master 3 Bus */
 	dev->i2c_bus[2].nr = 2;
@@ -1325,6 +1327,7 @@ int cx231xx_dev_init(struct cx231xx *dev)
 	dev->i2c_bus[2].i2c_period = I2C_SPEED_100K;	/* 100kHz */
 	dev->i2c_bus[2].i2c_nostop = 0;
 	dev->i2c_bus[2].i2c_reserve = 0;
+	dev->i2c_bus[2].i2c_rc = -ENODEV;
 
 	/* register I2C buses */
 	errCode = cx231xx_i2c_register(&dev->i2c_bus[0]);
diff --git a/drivers/media/usb/cx231xx/cx231xx-dvb.c b/drivers/media/usb/cx231xx/cx231xx-dvb.c
index ee3eeeb600f8..c18bb33e060e 100644
--- a/drivers/media/usb/cx231xx/cx231xx-dvb.c
+++ b/drivers/media/usb/cx231xx/cx231xx-dvb.c
@@ -585,6 +585,9 @@ static void unregister_dvb(struct cx231xx_dvb *dvb)
 	dvb->demux.dmx.remove_frontend(&dvb->demux.dmx, &dvb->fe_hw);
 	dvb_dmxdev_release(&dvb->dmxdev);
 	dvb_dmx_release(&dvb->demux);
+	dvb_unregister_frontend(dvb->frontend);
+	dvb_frontend_detach(dvb->frontend);
+	dvb_unregister_adapter(&dvb->adapter);
 	/* remove I2C tuner */
 	client = dvb->i2c_client_tuner;
 	if (client) {
@@ -597,9 +600,6 @@ static void unregister_dvb(struct cx231xx_dvb *dvb)
 		module_put(client->dev.driver->owner);
 		i2c_unregister_device(client);
 	}
-	dvb_unregister_frontend(dvb->frontend);
-	dvb_frontend_detach(dvb->frontend);
-	dvb_unregister_adapter(&dvb->adapter);
 }
 
 static int dvb_init(struct cx231xx *dev)
diff --git a/drivers/media/usb/cx231xx/cx231xx-i2c.c b/drivers/media/usb/cx231xx/cx231xx-i2c.c
index 8d95b1154e12..23648dab7be8 100644
--- a/drivers/media/usb/cx231xx/cx231xx-i2c.c
+++ b/drivers/media/usb/cx231xx/cx231xx-i2c.c
@@ -459,7 +459,7 @@ static const struct i2c_algorithm cx231xx_algo = {
 	.functionality = functionality,
 };
 
-static struct i2c_adapter cx231xx_adap_template = {
+static const struct i2c_adapter cx231xx_adap_template = {
 	.owner = THIS_MODULE,
 	.name = "cx231xx",
 	.algo = &cx231xx_algo,
@@ -538,7 +538,7 @@ int cx231xx_i2c_register(struct cx231xx_i2c *bus)
 
 	bus->i2c_adap.algo_data = bus;
 	i2c_set_adapdata(&bus->i2c_adap, &dev->v4l2_dev);
-	i2c_add_adapter(&bus->i2c_adap);
+	bus->i2c_rc = i2c_add_adapter(&bus->i2c_adap);
 
 	if (0 != bus->i2c_rc)
 		dev_warn(dev->dev,
@@ -551,10 +551,10 @@ int cx231xx_i2c_register(struct cx231xx_i2c *bus)
  * cx231xx_i2c_unregister()
  * unregister i2c_bus
  */
-int cx231xx_i2c_unregister(struct cx231xx_i2c *bus)
+void cx231xx_i2c_unregister(struct cx231xx_i2c *bus)
 {
-	i2c_del_adapter(&bus->i2c_adap);
-	return 0;
+	if (!bus->i2c_rc)
+		i2c_del_adapter(&bus->i2c_adap);
 }
 
 /*
diff --git a/drivers/media/usb/cx231xx/cx231xx-input.c b/drivers/media/usb/cx231xx/cx231xx-input.c
index eecf074b0a48..02ebeb16055f 100644
--- a/drivers/media/usb/cx231xx/cx231xx-input.c
+++ b/drivers/media/usb/cx231xx/cx231xx-input.c
@@ -24,7 +24,7 @@
 
 #define MODULE_NAME "cx231xx-input"
 
-static int get_key_isdbt(struct IR_i2c *ir, enum rc_type *protocol,
+static int get_key_isdbt(struct IR_i2c *ir, enum rc_proto *protocol,
 			 u32 *pscancode, u8 *toggle)
 {
 	int	rc;
@@ -50,7 +50,7 @@ static int get_key_isdbt(struct IR_i2c *ir, enum rc_type *protocol,
 
 	dev_dbg(&ir->rc->dev, "cmd %02x, scan = %02x\n", cmd, scancode);
 
-	*protocol = RC_TYPE_OTHER;
+	*protocol = RC_PROTO_OTHER;
 	*pscancode = scancode;
 	*toggle = 0;
 	return 1;
@@ -91,7 +91,7 @@ int cx231xx_ir_init(struct cx231xx *dev)
 	/* The i2c micro-controller only outputs the cmd part of NEC protocol */
 	dev->init_data.rc_dev->scancode_mask = 0xff;
 	dev->init_data.rc_dev->driver_name = "cx231xx";
-	dev->init_data.type = RC_BIT_NEC;
+	dev->init_data.type = RC_PROTO_BIT_NEC;
 	info.addr = 0x30;
 
 	/* Load and bind ir-kbd-i2c */
diff --git a/drivers/media/usb/cx231xx/cx231xx-video.c b/drivers/media/usb/cx231xx/cx231xx-video.c
index f67f86876625..179b8481a870 100644
--- a/drivers/media/usb/cx231xx/cx231xx-video.c
+++ b/drivers/media/usb/cx231xx/cx231xx-video.c
@@ -859,7 +859,7 @@ static void buffer_release(struct videobuf_queue *vq,
 	free_buffer(vq, buf);
 }
 
-static struct videobuf_queue_ops cx231xx_video_qops = {
+static const struct videobuf_queue_ops cx231xx_video_qops = {
 	.buf_setup = buffer_setup,
 	.buf_prepare = buffer_prepare,
 	.buf_queue = buffer_queue,
diff --git a/drivers/media/usb/cx231xx/cx231xx.h b/drivers/media/usb/cx231xx/cx231xx.h
index 986c64ba5b56..72d5937a087e 100644
--- a/drivers/media/usb/cx231xx/cx231xx.h
+++ b/drivers/media/usb/cx231xx/cx231xx.h
@@ -476,7 +476,7 @@ struct cx231xx_i2c {
 
 	/* i2c i/o */
 	struct i2c_adapter i2c_adap;
-	u32 i2c_rc;
+	int i2c_rc;
 
 	/* different settings for each bus */
 	u8 i2c_period;
@@ -762,7 +762,7 @@ int cx231xx_reset_analog_tuner(struct cx231xx *dev);
 /* Provided by cx231xx-i2c.c */
 void cx231xx_do_i2c_scan(struct cx231xx *dev, int i2c_port);
 int cx231xx_i2c_register(struct cx231xx_i2c *bus);
-int cx231xx_i2c_unregister(struct cx231xx_i2c *bus);
+void cx231xx_i2c_unregister(struct cx231xx_i2c *bus);
 int cx231xx_i2c_mux_create(struct cx231xx *dev);
 int cx231xx_i2c_mux_register(struct cx231xx *dev, int mux_no);
 void cx231xx_i2c_mux_unregister(struct cx231xx *dev);
diff --git a/drivers/media/usb/dvb-usb-v2/af9015.c b/drivers/media/usb/dvb-usb-v2/af9015.c
index 23bbbf367b51..8013659c41b1 100644
--- a/drivers/media/usb/dvb-usb-v2/af9015.c
+++ b/drivers/media/usb/dvb-usb-v2/af9015.c
@@ -1237,7 +1237,7 @@ static int af9015_rc_query(struct dvb_usb_device *d)
 
 	/* Only process key if canary killed */
 	if (buf[16] != 0xff && buf[0] != 0x01) {
-		enum rc_type proto;
+		enum rc_proto proto;
 		dev_dbg(&d->udev->dev, "%s: key pressed %*ph\n",
 				__func__, 4, buf + 12);
 
@@ -1253,13 +1253,13 @@ static int af9015_rc_query(struct dvb_usb_device *d)
 				/* NEC */
 				state->rc_keycode = RC_SCANCODE_NEC(buf[12],
 								    buf[14]);
-				proto = RC_TYPE_NEC;
+				proto = RC_PROTO_NEC;
 			} else {
 				/* NEC extended*/
 				state->rc_keycode = RC_SCANCODE_NECX(buf[12] << 8 |
 								     buf[13],
 								     buf[14]);
-				proto = RC_TYPE_NECX;
+				proto = RC_PROTO_NECX;
 			}
 		} else {
 			/* 32 bit NEC */
@@ -1267,7 +1267,7 @@ static int af9015_rc_query(struct dvb_usb_device *d)
 							      buf[13] << 16 |
 							      buf[14] << 8  |
 							      buf[15]);
-			proto = RC_TYPE_NEC32;
+			proto = RC_PROTO_NEC32;
 		}
 		rc_keydown(d->rc_dev, proto, state->rc_keycode, 0);
 	} else {
@@ -1336,7 +1336,8 @@ static int af9015_get_rc_config(struct dvb_usb_device *d, struct dvb_usb_rc *rc)
 	if (!rc->map_name)
 		rc->map_name = RC_MAP_EMPTY;
 
-	rc->allowed_protos = RC_BIT_NEC | RC_BIT_NECX | RC_BIT_NEC32;
+	rc->allowed_protos = RC_PROTO_BIT_NEC | RC_PROTO_BIT_NECX |
+						RC_PROTO_BIT_NEC32;
 	rc->query = af9015_rc_query;
 	rc->interval = 500;
 
diff --git a/drivers/media/usb/dvb-usb-v2/af9035.c b/drivers/media/usb/dvb-usb-v2/af9035.c
index 4df9486e19b9..666d319d3d1a 100644
--- a/drivers/media/usb/dvb-usb-v2/af9035.c
+++ b/drivers/media/usb/dvb-usb-v2/af9035.c
@@ -1828,7 +1828,7 @@ static int af9035_rc_query(struct dvb_usb_device *d)
 {
 	struct usb_interface *intf = d->intf;
 	int ret;
-	enum rc_type proto;
+	enum rc_proto proto;
 	u32 key;
 	u8 buf[4];
 	struct usb_req req = { CMD_IR_GET, 0, 0, NULL, 4, buf };
@@ -1843,17 +1843,17 @@ static int af9035_rc_query(struct dvb_usb_device *d)
 		if ((buf[0] + buf[1]) == 0xff) {
 			/* NEC standard 16bit */
 			key = RC_SCANCODE_NEC(buf[0], buf[2]);
-			proto = RC_TYPE_NEC;
+			proto = RC_PROTO_NEC;
 		} else {
 			/* NEC extended 24bit */
 			key = RC_SCANCODE_NECX(buf[0] << 8 | buf[1], buf[2]);
-			proto = RC_TYPE_NECX;
+			proto = RC_PROTO_NECX;
 		}
 	} else {
 		/* NEC full code 32bit */
 		key = RC_SCANCODE_NEC32(buf[0] << 24 | buf[1] << 16 |
 					buf[2] << 8  | buf[3]);
-		proto = RC_TYPE_NEC32;
+		proto = RC_PROTO_NEC32;
 	}
 
 	dev_dbg(&intf->dev, "%*ph\n", 4, buf);
@@ -1881,11 +1881,11 @@ static int af9035_get_rc_config(struct dvb_usb_device *d, struct dvb_usb_rc *rc)
 		switch (state->ir_type) {
 		case 0: /* NEC */
 		default:
-			rc->allowed_protos = RC_BIT_NEC | RC_BIT_NECX |
-								RC_BIT_NEC32;
+			rc->allowed_protos = RC_PROTO_BIT_NEC |
+					RC_PROTO_BIT_NECX | RC_PROTO_BIT_NEC32;
 			break;
 		case 1: /* RC6 */
-			rc->allowed_protos = RC_BIT_RC6_MCE;
+			rc->allowed_protos = RC_PROTO_BIT_RC6_MCE;
 			break;
 		}
 
@@ -2108,6 +2108,8 @@ static const struct usb_device_id af9035_id_table[] = {
 	{ DVB_USB_DEVICE(USB_VID_KWORLD_2, USB_PID_CTVDIGDUAL_V2,
 		&af9035_props, "Digital Dual TV Receiver CTVDIGDUAL_V2",
 							RC_MAP_IT913X_V1) },
+	{ DVB_USB_DEVICE(USB_VID_TERRATEC, USB_PID_TERRATEC_T1,
+		&af9035_props, "TerraTec T1", RC_MAP_IT913X_V1) },
 	/* XXX: that same ID [0ccd:0099] is used by af9015 driver too */
 	{ DVB_USB_DEVICE(USB_VID_TERRATEC, 0x0099,
 		&af9035_props, "TerraTec Cinergy T Stick Dual RC (rev. 2)",
diff --git a/drivers/media/usb/dvb-usb-v2/anysee.c b/drivers/media/usb/dvb-usb-v2/anysee.c
index 6795c0c609b1..20ee7eea2a91 100644
--- a/drivers/media/usb/dvb-usb-v2/anysee.c
+++ b/drivers/media/usb/dvb-usb-v2/anysee.c
@@ -1142,7 +1142,7 @@ static int anysee_rc_query(struct dvb_usb_device *d)
 	if (ircode[0]) {
 		dev_dbg(&d->udev->dev, "%s: key pressed %02x\n", __func__,
 				ircode[1]);
-		rc_keydown(d->rc_dev, RC_TYPE_NEC,
+		rc_keydown(d->rc_dev, RC_PROTO_NEC,
 			   RC_SCANCODE_NEC(0x08, ircode[1]), 0);
 	}
 
@@ -1151,7 +1151,7 @@ static int anysee_rc_query(struct dvb_usb_device *d)
 
 static int anysee_get_rc_config(struct dvb_usb_device *d, struct dvb_usb_rc *rc)
 {
-	rc->allowed_protos = RC_BIT_NEC;
+	rc->allowed_protos = RC_PROTO_BIT_NEC;
 	rc->query          = anysee_rc_query;
 	rc->interval       = 250;  /* windows driver uses 500ms */
 
diff --git a/drivers/media/usb/dvb-usb-v2/az6007.c b/drivers/media/usb/dvb-usb-v2/az6007.c
index 50c07fe7dacb..1414d59e85ba 100644
--- a/drivers/media/usb/dvb-usb-v2/az6007.c
+++ b/drivers/media/usb/dvb-usb-v2/az6007.c
@@ -208,7 +208,7 @@ static int az6007_rc_query(struct dvb_usb_device *d)
 {
 	struct az6007_device_state *st = d_to_priv(d);
 	unsigned code;
-	enum rc_type proto;
+	enum rc_proto proto;
 
 	az6007_read(d, AZ6007_READ_IR, 0, 0, st->data, 10);
 
@@ -218,18 +218,18 @@ static int az6007_rc_query(struct dvb_usb_device *d)
 	if ((st->data[3] ^ st->data[4]) == 0xff) {
 		if ((st->data[1] ^ st->data[2]) == 0xff) {
 			code = RC_SCANCODE_NEC(st->data[1], st->data[3]);
-			proto = RC_TYPE_NEC;
+			proto = RC_PROTO_NEC;
 		} else {
 			code = RC_SCANCODE_NECX(st->data[1] << 8 | st->data[2],
 						st->data[3]);
-			proto = RC_TYPE_NECX;
+			proto = RC_PROTO_NECX;
 		}
 	} else {
 		code = RC_SCANCODE_NEC32(st->data[1] << 24 |
 					 st->data[2] << 16 |
 					 st->data[3] << 8  |
 					 st->data[4]);
-		proto = RC_TYPE_NEC32;
+		proto = RC_PROTO_NEC32;
 	}
 
 	rc_keydown(d->rc_dev, proto, code, st->data[5]);
@@ -241,7 +241,8 @@ static int az6007_get_rc_config(struct dvb_usb_device *d, struct dvb_usb_rc *rc)
 {
 	pr_debug("Getting az6007 Remote Control properties\n");
 
-	rc->allowed_protos = RC_BIT_NEC | RC_BIT_NECX | RC_BIT_NEC32;
+	rc->allowed_protos = RC_PROTO_BIT_NEC | RC_PROTO_BIT_NECX |
+						RC_PROTO_BIT_NEC32;
 	rc->query          = az6007_rc_query;
 	rc->interval       = 400;
 
@@ -933,7 +934,7 @@ static struct dvb_usb_device_properties az6007_cablestar_hdci_props = {
 	}
 };
 
-static struct usb_device_id az6007_usb_table[] = {
+static const struct usb_device_id az6007_usb_table[] = {
 	{DVB_USB_DEVICE(USB_VID_AZUREWAVE, USB_PID_AZUREWAVE_6007,
 		&az6007_props, "Azurewave 6007", RC_MAP_EMPTY)},
 	{DVB_USB_DEVICE(USB_VID_TERRATEC, USB_PID_TERRATEC_H7,
diff --git a/drivers/media/usb/dvb-usb-v2/dvb_usb.h b/drivers/media/usb/dvb-usb-v2/dvb_usb.h
index 35f27e2e4e28..0005bdb2207d 100644
--- a/drivers/media/usb/dvb-usb-v2/dvb_usb.h
+++ b/drivers/media/usb/dvb-usb-v2/dvb_usb.h
@@ -138,7 +138,7 @@ struct dvb_usb_driver_info {
 struct dvb_usb_rc {
 	const char *map_name;
 	u64 allowed_protos;
-	int (*change_protocol)(struct rc_dev *dev, u64 *rc_type);
+	int (*change_protocol)(struct rc_dev *dev, u64 *rc_proto);
 	int (*query) (struct dvb_usb_device *d);
 	unsigned int interval;
 	enum rc_driver_type driver_type;
diff --git a/drivers/media/usb/dvb-usb-v2/dvb_usb_core.c b/drivers/media/usb/dvb-usb-v2/dvb_usb_core.c
index 955fb0d07507..096bb75a24e5 100644
--- a/drivers/media/usb/dvb-usb-v2/dvb_usb_core.c
+++ b/drivers/media/usb/dvb-usb-v2/dvb_usb_core.c
@@ -154,13 +154,12 @@ static int dvb_usbv2_remote_init(struct dvb_usb_device *d)
 	}
 
 	dev->dev.parent = &d->udev->dev;
-	dev->input_name = d->name;
+	dev->device_name = d->name;
 	usb_make_path(d->udev, d->rc_phys, sizeof(d->rc_phys));
 	strlcat(d->rc_phys, "/ir0", sizeof(d->rc_phys));
 	dev->input_phys = d->rc_phys;
 	usb_to_input_id(d->udev, &dev->input_id);
-	/* TODO: likely RC-core should took const char * */
-	dev->driver_name = (char *) d->props->driver_name;
+	dev->driver_name = d->props->driver_name;
 	dev->map_name = d->rc.map_name;
 	dev->allowed_protocols = d->rc.allowed_protos;
 	dev->change_protocol = d->rc.change_protocol;
diff --git a/drivers/media/usb/dvb-usb-v2/dvbsky.c b/drivers/media/usb/dvb-usb-v2/dvbsky.c
index 5730760e4e93..131b6c08e199 100644
--- a/drivers/media/usb/dvb-usb-v2/dvbsky.c
+++ b/drivers/media/usb/dvb-usb-v2/dvbsky.c
@@ -211,7 +211,7 @@ static int dvbsky_rc_query(struct dvb_usb_device *d)
 		rc5_system = (code & 0x7C0) >> 6;
 		toggle = (code & 0x800) ? 1 : 0;
 		scancode = rc5_system << 8 | rc5_command;
-		rc_keydown(d->rc_dev, RC_TYPE_RC5, scancode, toggle);
+		rc_keydown(d->rc_dev, RC_PROTO_RC5, scancode, toggle);
 	}
 	return 0;
 }
@@ -223,7 +223,7 @@ static int dvbsky_get_rc_config(struct dvb_usb_device *d, struct dvb_usb_rc *rc)
 		return 0;
 	}
 
-	rc->allowed_protos = RC_BIT_RC5;
+	rc->allowed_protos = RC_PROTO_BIT_RC5;
 	rc->query          = dvbsky_rc_query;
 	rc->interval       = 300;
 	return 0;
diff --git a/drivers/media/usb/dvb-usb-v2/lmedm04.c b/drivers/media/usb/dvb-usb-v2/lmedm04.c
index a91fdad8f8d4..5e320fa4a795 100644
--- a/drivers/media/usb/dvb-usb-v2/lmedm04.c
+++ b/drivers/media/usb/dvb-usb-v2/lmedm04.c
@@ -347,8 +347,8 @@ static void lme2510_int_response(struct urb *lme_urb)
 						ibuf[5]);
 
 			deb_info(1, "INT Key = 0x%08x", key);
-			rc_keydown(adap_to_d(adap)->rc_dev, RC_TYPE_NEC32, key,
-									0);
+			rc_keydown(adap_to_d(adap)->rc_dev, RC_PROTO_NEC32, key,
+				   0);
 			break;
 		case 0xbb:
 			switch (st->tuner_config) {
@@ -1232,7 +1232,7 @@ static int lme2510_get_stream_config(struct dvb_frontend *fe, u8 *ts_type,
 static int lme2510_get_rc_config(struct dvb_usb_device *d,
 	struct dvb_usb_rc *rc)
 {
-	rc->allowed_protos = RC_BIT_NEC32;
+	rc->allowed_protos = RC_PROTO_BIT_NEC32;
 	return 0;
 }
 
diff --git a/drivers/media/usb/dvb-usb-v2/mxl111sf.c b/drivers/media/usb/dvb-usb-v2/mxl111sf.c
index b0d5904a4ea6..67953360fda5 100644
--- a/drivers/media/usb/dvb-usb-v2/mxl111sf.c
+++ b/drivers/media/usb/dvb-usb-v2/mxl111sf.c
@@ -77,7 +77,9 @@ int mxl111sf_ctrl_msg(struct mxl111sf_state *state,
 		dvb_usbv2_generic_rw(d, state->sndbuf, 1+wlen, state->rcvbuf,
 				     rlen);
 
-	memcpy(rbuf, state->rcvbuf, rlen);
+	if (rbuf)
+		memcpy(rbuf, state->rcvbuf, rlen);
+
 	mutex_unlock(&state->msg_lock);
 
 	mxl_fail(ret);
diff --git a/drivers/media/usb/dvb-usb-v2/rtl28xxu.c b/drivers/media/usb/dvb-usb-v2/rtl28xxu.c
index e16ca07acf1d..95a7b9123f8e 100644
--- a/drivers/media/usb/dvb-usb-v2/rtl28xxu.c
+++ b/drivers/media/usb/dvb-usb-v2/rtl28xxu.c
@@ -1631,24 +1631,24 @@ static int rtl2831u_rc_query(struct dvb_usb_device *d)
 		goto err;
 
 	if (buf[4] & 0x01) {
-		enum rc_type proto;
+		enum rc_proto proto;
 
 		if (buf[2] == (u8) ~buf[3]) {
 			if (buf[0] == (u8) ~buf[1]) {
 				/* NEC standard (16 bit) */
 				rc_code = RC_SCANCODE_NEC(buf[0], buf[2]);
-				proto = RC_TYPE_NEC;
+				proto = RC_PROTO_NEC;
 			} else {
 				/* NEC extended (24 bit) */
 				rc_code = RC_SCANCODE_NECX(buf[0] << 8 | buf[1],
 							   buf[2]);
-				proto = RC_TYPE_NECX;
+				proto = RC_PROTO_NECX;
 			}
 		} else {
 			/* NEC full (32 bit) */
 			rc_code = RC_SCANCODE_NEC32(buf[0] << 24 | buf[1] << 16 |
 						    buf[2] << 8  | buf[3]);
-			proto = RC_TYPE_NEC32;
+			proto = RC_PROTO_NEC32;
 		}
 
 		rc_keydown(d->rc_dev, proto, rc_code, 0);
@@ -1673,7 +1673,8 @@ static int rtl2831u_get_rc_config(struct dvb_usb_device *d,
 		struct dvb_usb_rc *rc)
 {
 	rc->map_name = RC_MAP_EMPTY;
-	rc->allowed_protos = RC_BIT_NEC | RC_BIT_NECX | RC_BIT_NEC32;
+	rc->allowed_protos = RC_PROTO_BIT_NEC | RC_PROTO_BIT_NECX |
+							RC_PROTO_BIT_NEC32;
 	rc->query = rtl2831u_rc_query;
 	rc->interval = 400;
 
@@ -1778,7 +1779,7 @@ static int rtl2832u_get_rc_config(struct dvb_usb_device *d,
 	/* load empty to enable rc */
 	if (!rc->map_name)
 		rc->map_name = RC_MAP_EMPTY;
-	rc->allowed_protos = RC_BIT_ALL_IR_DECODER;
+	rc->allowed_protos = RC_PROTO_BIT_ALL_IR_DECODER;
 	rc->driver_type = RC_DRIVER_IR_RAW;
 	rc->query = rtl2832u_rc_query;
 	rc->interval = 200;
diff --git a/drivers/media/usb/dvb-usb/cxusb.c b/drivers/media/usb/dvb-usb/cxusb.c
index 99a3f3625944..37dea0adc695 100644
--- a/drivers/media/usb/dvb-usb/cxusb.c
+++ b/drivers/media/usb/dvb-usb/cxusb.c
@@ -458,7 +458,7 @@ static int cxusb_rc_query(struct dvb_usb_device *d)
 	cxusb_ctrl_msg(d, CMD_GET_IR_CODE, NULL, 0, ircode, 4);
 
 	if (ircode[2] || ircode[3])
-		rc_keydown(d->rc_dev, RC_TYPE_NEC,
+		rc_keydown(d->rc_dev, RC_PROTO_NEC,
 			   RC_SCANCODE_NEC(~ircode[2] & 0xff, ircode[3]), 0);
 	return 0;
 }
@@ -473,7 +473,7 @@ static int cxusb_bluebird2_rc_query(struct dvb_usb_device *d)
 		return 0;
 
 	if (ircode[1] || ircode[2])
-		rc_keydown(d->rc_dev, RC_TYPE_NEC,
+		rc_keydown(d->rc_dev, RC_PROTO_NEC,
 			   RC_SCANCODE_NEC(~ircode[1] & 0xff, ircode[2]), 0);
 	return 0;
 }
@@ -486,7 +486,7 @@ static int cxusb_d680_dmb_rc_query(struct dvb_usb_device *d)
 		return 0;
 
 	if (ircode[0] || ircode[1])
-		rc_keydown(d->rc_dev, RC_TYPE_UNKNOWN,
+		rc_keydown(d->rc_dev, RC_PROTO_UNKNOWN,
 			   RC_SCANCODE_RC5(ircode[0], ircode[1]), 0);
 	return 0;
 }
@@ -1646,7 +1646,7 @@ static struct dvb_usb_device_properties cxusb_bluebird_lgh064f_properties = {
 		.rc_codes	= RC_MAP_DVICO_PORTABLE,
 		.module_name	= KBUILD_MODNAME,
 		.rc_query	= cxusb_rc_query,
-		.allowed_protos = RC_BIT_NEC,
+		.allowed_protos = RC_PROTO_BIT_NEC,
 	},
 
 	.generic_bulk_ctrl_endpoint = 0x01,
@@ -1703,7 +1703,7 @@ static struct dvb_usb_device_properties cxusb_bluebird_dee1601_properties = {
 		.rc_codes	= RC_MAP_DVICO_MCE,
 		.module_name	= KBUILD_MODNAME,
 		.rc_query	= cxusb_rc_query,
-		.allowed_protos = RC_BIT_NEC,
+		.allowed_protos = RC_PROTO_BIT_NEC,
 	},
 
 	.generic_bulk_ctrl_endpoint = 0x01,
@@ -1768,7 +1768,7 @@ static struct dvb_usb_device_properties cxusb_bluebird_lgz201_properties = {
 		.rc_codes	= RC_MAP_DVICO_PORTABLE,
 		.module_name	= KBUILD_MODNAME,
 		.rc_query	= cxusb_rc_query,
-		.allowed_protos = RC_BIT_NEC,
+		.allowed_protos = RC_PROTO_BIT_NEC,
 	},
 
 	.generic_bulk_ctrl_endpoint = 0x01,
@@ -1824,7 +1824,7 @@ static struct dvb_usb_device_properties cxusb_bluebird_dtt7579_properties = {
 		.rc_codes	= RC_MAP_DVICO_PORTABLE,
 		.module_name	= KBUILD_MODNAME,
 		.rc_query	= cxusb_rc_query,
-		.allowed_protos = RC_BIT_NEC,
+		.allowed_protos = RC_PROTO_BIT_NEC,
 	},
 
 	.generic_bulk_ctrl_endpoint = 0x01,
@@ -1879,7 +1879,7 @@ static struct dvb_usb_device_properties cxusb_bluebird_dualdig4_properties = {
 		.rc_codes	= RC_MAP_DVICO_MCE,
 		.module_name	= KBUILD_MODNAME,
 		.rc_query	= cxusb_bluebird2_rc_query,
-		.allowed_protos = RC_BIT_NEC,
+		.allowed_protos = RC_PROTO_BIT_NEC,
 	},
 
 	.num_device_descs = 1,
@@ -1933,7 +1933,7 @@ static struct dvb_usb_device_properties cxusb_bluebird_nano2_properties = {
 		.rc_codes	= RC_MAP_DVICO_PORTABLE,
 		.module_name	= KBUILD_MODNAME,
 		.rc_query       = cxusb_bluebird2_rc_query,
-		.allowed_protos = RC_BIT_NEC,
+		.allowed_protos = RC_PROTO_BIT_NEC,
 	},
 
 	.num_device_descs = 1,
@@ -1989,7 +1989,7 @@ static struct dvb_usb_device_properties cxusb_bluebird_nano2_needsfirmware_prope
 		.rc_codes	= RC_MAP_DVICO_PORTABLE,
 		.module_name	= KBUILD_MODNAME,
 		.rc_query	= cxusb_rc_query,
-		.allowed_protos = RC_BIT_NEC,
+		.allowed_protos = RC_PROTO_BIT_NEC,
 	},
 
 	.num_device_descs = 1,
@@ -2088,7 +2088,7 @@ struct dvb_usb_device_properties cxusb_bluebird_dualdig4_rev2_properties = {
 		.rc_codes	= RC_MAP_DVICO_MCE,
 		.module_name	= KBUILD_MODNAME,
 		.rc_query	= cxusb_rc_query,
-		.allowed_protos = RC_BIT_NEC,
+		.allowed_protos = RC_PROTO_BIT_NEC,
 	},
 
 	.num_device_descs = 1,
@@ -2142,7 +2142,7 @@ static struct dvb_usb_device_properties cxusb_d680_dmb_properties = {
 		.rc_codes	= RC_MAP_D680_DMB,
 		.module_name	= KBUILD_MODNAME,
 		.rc_query       = cxusb_d680_dmb_rc_query,
-		.allowed_protos = RC_BIT_UNKNOWN,
+		.allowed_protos = RC_PROTO_BIT_UNKNOWN,
 	},
 
 	.num_device_descs = 1,
@@ -2197,7 +2197,7 @@ static struct dvb_usb_device_properties cxusb_mygica_d689_properties = {
 		.rc_codes	= RC_MAP_D680_DMB,
 		.module_name	= KBUILD_MODNAME,
 		.rc_query       = cxusb_d680_dmb_rc_query,
-		.allowed_protos = RC_BIT_UNKNOWN,
+		.allowed_protos = RC_PROTO_BIT_UNKNOWN,
 	},
 
 	.num_device_descs = 1,
@@ -2251,7 +2251,7 @@ static struct dvb_usb_device_properties cxusb_mygica_t230_properties = {
 		.rc_codes	= RC_MAP_TOTAL_MEDIA_IN_HAND_02,
 		.module_name	= KBUILD_MODNAME,
 		.rc_query       = cxusb_d680_dmb_rc_query,
-		.allowed_protos = RC_BIT_UNKNOWN,
+		.allowed_protos = RC_PROTO_BIT_UNKNOWN,
 	},
 
 	.num_device_descs = 1,
@@ -2305,7 +2305,7 @@ static struct dvb_usb_device_properties cxusb_mygica_t230c_properties = {
 		.rc_codes	= RC_MAP_TOTAL_MEDIA_IN_HAND_02,
 		.module_name	= KBUILD_MODNAME,
 		.rc_query       = cxusb_d680_dmb_rc_query,
-		.allowed_protos = RC_BIT_UNKNOWN,
+		.allowed_protos = RC_PROTO_BIT_UNKNOWN,
 	},
 
 	.num_device_descs = 1,
diff --git a/drivers/media/usb/dvb-usb/dib0700.h b/drivers/media/usb/dvb-usb/dib0700.h
index 8fd8f5b489d2..f89ab3b5a6c4 100644
--- a/drivers/media/usb/dvb-usb/dib0700.h
+++ b/drivers/media/usb/dvb-usb/dib0700.h
@@ -64,7 +64,7 @@ extern int dib0700_streaming_ctrl(struct dvb_usb_adapter *adap, int onoff);
 extern struct i2c_algorithm dib0700_i2c_algo;
 extern int dib0700_identify_state(struct usb_device *udev, struct dvb_usb_device_properties *props,
 			struct dvb_usb_device_description **desc, int *cold);
-extern int dib0700_change_protocol(struct rc_dev *dev, u64 *rc_type);
+extern int dib0700_change_protocol(struct rc_dev *dev, u64 *rc_proto);
 extern int dib0700_set_i2c_speed(struct dvb_usb_device *d, u16 scl_kHz);
 
 extern int dib0700_device_count;
diff --git a/drivers/media/usb/dvb-usb/dib0700_core.c b/drivers/media/usb/dvb-usb/dib0700_core.c
index bea1b4764a66..1ee7ec558293 100644
--- a/drivers/media/usb/dvb-usb/dib0700_core.c
+++ b/drivers/media/usb/dvb-usb/dib0700_core.c
@@ -638,7 +638,7 @@ int dib0700_streaming_ctrl(struct dvb_usb_adapter *adap, int onoff)
 	return ret;
 }
 
-int dib0700_change_protocol(struct rc_dev *rc, u64 *rc_type)
+int dib0700_change_protocol(struct rc_dev *rc, u64 *rc_proto)
 {
 	struct dvb_usb_device *d = rc->priv;
 	struct dib0700_state *st = d->priv;
@@ -654,19 +654,19 @@ int dib0700_change_protocol(struct rc_dev *rc, u64 *rc_type)
 	st->buf[2] = 0;
 
 	/* Set the IR mode */
-	if (*rc_type & RC_BIT_RC5) {
+	if (*rc_proto & RC_PROTO_BIT_RC5) {
 		new_proto = 1;
-		*rc_type = RC_BIT_RC5;
-	} else if (*rc_type & RC_BIT_NEC) {
+		*rc_proto = RC_PROTO_BIT_RC5;
+	} else if (*rc_proto & RC_PROTO_BIT_NEC) {
 		new_proto = 0;
-		*rc_type = RC_BIT_NEC;
-	} else if (*rc_type & RC_BIT_RC6_MCE) {
+		*rc_proto = RC_PROTO_BIT_NEC;
+	} else if (*rc_proto & RC_PROTO_BIT_RC6_MCE) {
 		if (st->fw_version < 0x10200) {
 			ret = -EINVAL;
 			goto out;
 		}
 		new_proto = 2;
-		*rc_type = RC_BIT_RC6_MCE;
+		*rc_proto = RC_PROTO_BIT_RC6_MCE;
 	} else {
 		ret = -EINVAL;
 		goto out;
@@ -680,7 +680,7 @@ int dib0700_change_protocol(struct rc_dev *rc, u64 *rc_type)
 		goto out;
 	}
 
-	d->props.rc.core.protocol = *rc_type;
+	d->props.rc.core.protocol = *rc_proto;
 
 out:
 	mutex_unlock(&d->usb_mutex);
@@ -712,7 +712,7 @@ static void dib0700_rc_urb_completion(struct urb *purb)
 {
 	struct dvb_usb_device *d = purb->context;
 	struct dib0700_rc_response *poll_reply;
-	enum rc_type protocol;
+	enum rc_proto protocol;
 	u32 keycode;
 	u8 toggle;
 
@@ -745,7 +745,7 @@ static void dib0700_rc_urb_completion(struct urb *purb)
 		 purb->actual_length);
 
 	switch (d->props.rc.core.protocol) {
-	case RC_BIT_NEC:
+	case RC_PROTO_BIT_NEC:
 		toggle = 0;
 
 		/* NEC protocol sends repeat code as 0 0 0 FF */
@@ -764,25 +764,25 @@ static void dib0700_rc_urb_completion(struct urb *purb)
 						     poll_reply->nec.not_system << 16 |
 						     poll_reply->nec.data       << 8  |
 						     poll_reply->nec.not_data);
-			protocol = RC_TYPE_NEC32;
+			protocol = RC_PROTO_NEC32;
 		} else if ((poll_reply->nec.system ^ poll_reply->nec.not_system) != 0xff) {
 			deb_data("NEC extended protocol\n");
 			keycode = RC_SCANCODE_NECX(poll_reply->nec.system << 8 |
 						    poll_reply->nec.not_system,
 						    poll_reply->nec.data);
 
-			protocol = RC_TYPE_NECX;
+			protocol = RC_PROTO_NECX;
 		} else {
 			deb_data("NEC normal protocol\n");
 			keycode = RC_SCANCODE_NEC(poll_reply->nec.system,
 						   poll_reply->nec.data);
-			protocol = RC_TYPE_NEC;
+			protocol = RC_PROTO_NEC;
 		}
 
 		break;
 	default:
 		deb_data("RC5 protocol\n");
-		protocol = RC_TYPE_RC5;
+		protocol = RC_PROTO_RC5;
 		toggle = poll_reply->report_id;
 		keycode = RC_SCANCODE_RC5(poll_reply->rc5.system, poll_reply->rc5.data);
 
diff --git a/drivers/media/usb/dvb-usb/dib0700_devices.c b/drivers/media/usb/dvb-usb/dib0700_devices.c
index 6a57fc6d3472..6020170fe99a 100644
--- a/drivers/media/usb/dvb-usb/dib0700_devices.c
+++ b/drivers/media/usb/dvb-usb/dib0700_devices.c
@@ -514,7 +514,7 @@ static int stk7700ph_tuner_attach(struct dvb_usb_adapter *adap)
  */
 static int dib0700_rc_query_old_firmware(struct dvb_usb_device *d)
 {
-	enum rc_type protocol;
+	enum rc_proto protocol;
 	u32 scancode;
 	u8 toggle;
 	int i;
@@ -547,7 +547,7 @@ static int dib0700_rc_query_old_firmware(struct dvb_usb_device *d)
 	dib0700_rc_setup(d, NULL); /* reset ir sensor data to prevent false events */
 
 	switch (d->props.rc.core.protocol) {
-	case RC_BIT_NEC:
+	case RC_PROTO_BIT_NEC:
 		/* NEC protocol sends repeat code as 0 0 0 FF */
 		if ((st->buf[3 - 2] == 0x00) && (st->buf[3 - 3] == 0x00) &&
 		    (st->buf[3] == 0xff)) {
@@ -555,14 +555,14 @@ static int dib0700_rc_query_old_firmware(struct dvb_usb_device *d)
 			return 0;
 		}
 
-		protocol = RC_TYPE_NEC;
+		protocol = RC_PROTO_NEC;
 		scancode = RC_SCANCODE_NEC(st->buf[3 - 2], st->buf[3 - 3]);
 		toggle = 0;
 		break;
 
 	default:
 		/* RC-5 protocol changes toggle bit on new keypress */
-		protocol = RC_TYPE_RC5;
+		protocol = RC_PROTO_RC5;
 		scancode = RC_SCANCODE_RC5(st->buf[3 - 2], st->buf[3 - 3]);
 		toggle = st->buf[3 - 1];
 		break;
@@ -3909,9 +3909,9 @@ struct dvb_usb_device_properties dib0700_devices[] = {
 			.rc_interval      = DEFAULT_RC_INTERVAL,
 			.rc_codes         = RC_MAP_DIB0700_RC5_TABLE,
 			.rc_query         = dib0700_rc_query_old_firmware,
-			.allowed_protos   = RC_BIT_RC5 |
-					    RC_BIT_RC6_MCE |
-					    RC_BIT_NEC,
+			.allowed_protos   = RC_PROTO_BIT_RC5 |
+					    RC_PROTO_BIT_RC6_MCE |
+					    RC_PROTO_BIT_NEC,
 			.change_protocol  = dib0700_change_protocol,
 		},
 	}, { DIB0700_DEFAULT_DEVICE_PROPERTIES,
@@ -3949,9 +3949,9 @@ struct dvb_usb_device_properties dib0700_devices[] = {
 			.rc_interval      = DEFAULT_RC_INTERVAL,
 			.rc_codes         = RC_MAP_DIB0700_RC5_TABLE,
 			.rc_query         = dib0700_rc_query_old_firmware,
-			.allowed_protos   = RC_BIT_RC5 |
-					    RC_BIT_RC6_MCE |
-					    RC_BIT_NEC,
+			.allowed_protos   = RC_PROTO_BIT_RC5 |
+					    RC_PROTO_BIT_RC6_MCE |
+					    RC_PROTO_BIT_NEC,
 			.change_protocol = dib0700_change_protocol,
 		},
 	}, { DIB0700_DEFAULT_DEVICE_PROPERTIES,
@@ -4014,9 +4014,9 @@ struct dvb_usb_device_properties dib0700_devices[] = {
 			.rc_interval      = DEFAULT_RC_INTERVAL,
 			.rc_codes         = RC_MAP_DIB0700_RC5_TABLE,
 			.rc_query         = dib0700_rc_query_old_firmware,
-			.allowed_protos   = RC_BIT_RC5 |
-					    RC_BIT_RC6_MCE |
-					    RC_BIT_NEC,
+			.allowed_protos   = RC_PROTO_BIT_RC5 |
+					    RC_PROTO_BIT_RC6_MCE |
+					    RC_PROTO_BIT_NEC,
 			.change_protocol = dib0700_change_protocol,
 		},
 	}, { DIB0700_DEFAULT_DEVICE_PROPERTIES,
@@ -4059,9 +4059,9 @@ struct dvb_usb_device_properties dib0700_devices[] = {
 			.rc_codes         = RC_MAP_DIB0700_RC5_TABLE,
 			.module_name	  = "dib0700",
 			.rc_query         = dib0700_rc_query_old_firmware,
-			.allowed_protos   = RC_BIT_RC5 |
-					    RC_BIT_RC6_MCE |
-					    RC_BIT_NEC,
+			.allowed_protos   = RC_PROTO_BIT_RC5 |
+					    RC_PROTO_BIT_RC6_MCE |
+					    RC_PROTO_BIT_NEC,
 			.change_protocol = dib0700_change_protocol,
 		},
 	}, { DIB0700_DEFAULT_DEVICE_PROPERTIES,
@@ -4140,9 +4140,9 @@ struct dvb_usb_device_properties dib0700_devices[] = {
 			.rc_codes         = RC_MAP_DIB0700_RC5_TABLE,
 			.module_name	  = "dib0700",
 			.rc_query         = dib0700_rc_query_old_firmware,
-			.allowed_protos   = RC_BIT_RC5 |
-					    RC_BIT_RC6_MCE |
-					    RC_BIT_NEC,
+			.allowed_protos   = RC_PROTO_BIT_RC5 |
+					    RC_PROTO_BIT_RC6_MCE |
+					    RC_PROTO_BIT_NEC,
 			.change_protocol  = dib0700_change_protocol,
 		},
 	}, { DIB0700_DEFAULT_DEVICE_PROPERTIES,
@@ -4185,9 +4185,9 @@ struct dvb_usb_device_properties dib0700_devices[] = {
 			.rc_codes         = RC_MAP_DIB0700_RC5_TABLE,
 			.module_name	  = "dib0700",
 			.rc_query         = dib0700_rc_query_old_firmware,
-			.allowed_protos   = RC_BIT_RC5 |
-					    RC_BIT_RC6_MCE |
-					    RC_BIT_NEC,
+			.allowed_protos   = RC_PROTO_BIT_RC5 |
+					    RC_PROTO_BIT_RC6_MCE |
+					    RC_PROTO_BIT_NEC,
 			.change_protocol  = dib0700_change_protocol,
 		},
 	}, { DIB0700_DEFAULT_DEVICE_PROPERTIES,
@@ -4242,9 +4242,9 @@ struct dvb_usb_device_properties dib0700_devices[] = {
 			.rc_codes         = RC_MAP_DIB0700_RC5_TABLE,
 			.module_name	  = "dib0700",
 			.rc_query         = dib0700_rc_query_old_firmware,
-			.allowed_protos   = RC_BIT_RC5 |
-					    RC_BIT_RC6_MCE |
-					    RC_BIT_NEC,
+			.allowed_protos   = RC_PROTO_BIT_RC5 |
+					    RC_PROTO_BIT_RC6_MCE |
+					    RC_PROTO_BIT_NEC,
 			.change_protocol = dib0700_change_protocol,
 		},
 	}, { DIB0700_DEFAULT_DEVICE_PROPERTIES,
@@ -4308,9 +4308,9 @@ struct dvb_usb_device_properties dib0700_devices[] = {
 			.rc_codes         = RC_MAP_DIB0700_RC5_TABLE,
 			.module_name	  = "dib0700",
 			.rc_query         = dib0700_rc_query_old_firmware,
-			.allowed_protos   = RC_BIT_RC5 |
-					    RC_BIT_RC6_MCE |
-					    RC_BIT_NEC,
+			.allowed_protos   = RC_PROTO_BIT_RC5 |
+					    RC_PROTO_BIT_RC6_MCE |
+					    RC_PROTO_BIT_NEC,
 			.change_protocol = dib0700_change_protocol,
 		},
 	}, { DIB0700_DEFAULT_DEVICE_PROPERTIES,
@@ -4357,9 +4357,9 @@ struct dvb_usb_device_properties dib0700_devices[] = {
 			.rc_codes         = RC_MAP_DIB0700_NEC_TABLE,
 			.module_name	  = "dib0700",
 			.rc_query         = dib0700_rc_query_old_firmware,
-			.allowed_protos   = RC_BIT_RC5 |
-					    RC_BIT_RC6_MCE |
-					    RC_BIT_NEC,
+			.allowed_protos   = RC_PROTO_BIT_RC5 |
+					    RC_PROTO_BIT_RC6_MCE |
+					    RC_PROTO_BIT_NEC,
 			.change_protocol  = dib0700_change_protocol,
 		},
 	}, { DIB0700_DEFAULT_DEVICE_PROPERTIES,
@@ -4430,9 +4430,9 @@ struct dvb_usb_device_properties dib0700_devices[] = {
 			.rc_codes         = RC_MAP_DIB0700_RC5_TABLE,
 			.module_name	  = "dib0700",
 			.rc_query         = dib0700_rc_query_old_firmware,
-			.allowed_protos   = RC_BIT_RC5 |
-					    RC_BIT_RC6_MCE |
-					    RC_BIT_NEC,
+			.allowed_protos   = RC_PROTO_BIT_RC5 |
+					    RC_PROTO_BIT_RC6_MCE |
+					    RC_PROTO_BIT_NEC,
 			.change_protocol  = dib0700_change_protocol,
 		},
 	}, { DIB0700_DEFAULT_DEVICE_PROPERTIES,
@@ -4466,9 +4466,9 @@ struct dvb_usb_device_properties dib0700_devices[] = {
 			.rc_codes         = RC_MAP_DIB0700_RC5_TABLE,
 			.module_name	  = "dib0700",
 			.rc_query         = dib0700_rc_query_old_firmware,
-			.allowed_protos   = RC_BIT_RC5 |
-					    RC_BIT_RC6_MCE |
-					    RC_BIT_NEC,
+			.allowed_protos   = RC_PROTO_BIT_RC5 |
+					    RC_PROTO_BIT_RC6_MCE |
+					    RC_PROTO_BIT_NEC,
 			.change_protocol  = dib0700_change_protocol,
 		},
 	}, { DIB0700_DEFAULT_DEVICE_PROPERTIES,
@@ -4542,9 +4542,9 @@ struct dvb_usb_device_properties dib0700_devices[] = {
 			.rc_codes         = RC_MAP_DIB0700_RC5_TABLE,
 			.module_name	  = "dib0700",
 			.rc_query         = dib0700_rc_query_old_firmware,
-			.allowed_protos   = RC_BIT_RC5 |
-					    RC_BIT_RC6_MCE |
-					    RC_BIT_NEC,
+			.allowed_protos   = RC_PROTO_BIT_RC5 |
+					    RC_PROTO_BIT_RC6_MCE |
+					    RC_PROTO_BIT_NEC,
 			.change_protocol  = dib0700_change_protocol,
 		},
 	}, { DIB0700_DEFAULT_DEVICE_PROPERTIES,
@@ -4586,9 +4586,9 @@ struct dvb_usb_device_properties dib0700_devices[] = {
 			.rc_codes         = RC_MAP_DIB0700_NEC_TABLE,
 			.module_name	  = "dib0700",
 			.rc_query         = dib0700_rc_query_old_firmware,
-			.allowed_protos   = RC_BIT_RC5 |
-					    RC_BIT_RC6_MCE |
-					    RC_BIT_NEC,
+			.allowed_protos   = RC_PROTO_BIT_RC5 |
+					    RC_PROTO_BIT_RC6_MCE |
+					    RC_PROTO_BIT_NEC,
 			.change_protocol  = dib0700_change_protocol,
 		},
 	}, { DIB0700_DEFAULT_DEVICE_PROPERTIES,
@@ -4635,9 +4635,9 @@ struct dvb_usb_device_properties dib0700_devices[] = {
 			.rc_codes         = RC_MAP_DIB0700_RC5_TABLE,
 			.module_name	  = "dib0700",
 			.rc_query         = dib0700_rc_query_old_firmware,
-			.allowed_protos   = RC_BIT_RC5 |
-					    RC_BIT_RC6_MCE |
-					    RC_BIT_NEC,
+			.allowed_protos   = RC_PROTO_BIT_RC5 |
+					    RC_PROTO_BIT_RC6_MCE |
+					    RC_PROTO_BIT_NEC,
 			.change_protocol  = dib0700_change_protocol,
 		},
 	}, { DIB0700_DEFAULT_DEVICE_PROPERTIES,
@@ -4672,9 +4672,9 @@ struct dvb_usb_device_properties dib0700_devices[] = {
 			.rc_codes         = RC_MAP_DIB0700_RC5_TABLE,
 			.module_name	  = "dib0700",
 			.rc_query         = dib0700_rc_query_old_firmware,
-			.allowed_protos   = RC_BIT_RC5 |
-					    RC_BIT_RC6_MCE |
-					    RC_BIT_NEC,
+			.allowed_protos   = RC_PROTO_BIT_RC5 |
+					    RC_PROTO_BIT_RC6_MCE |
+					    RC_PROTO_BIT_NEC,
 			.change_protocol  = dib0700_change_protocol,
 		},
 	}, { DIB0700_DEFAULT_DEVICE_PROPERTIES,
@@ -4709,9 +4709,9 @@ struct dvb_usb_device_properties dib0700_devices[] = {
 			.rc_codes         = RC_MAP_DIB0700_RC5_TABLE,
 			.module_name	  = "dib0700",
 			.rc_query         = dib0700_rc_query_old_firmware,
-			.allowed_protos   = RC_BIT_RC5 |
-					    RC_BIT_RC6_MCE |
-					    RC_BIT_NEC,
+			.allowed_protos   = RC_PROTO_BIT_RC5 |
+					    RC_PROTO_BIT_RC6_MCE |
+					    RC_PROTO_BIT_NEC,
 			.change_protocol  = dib0700_change_protocol,
 		},
 	}, { DIB0700_DEFAULT_DEVICE_PROPERTIES,
@@ -4746,9 +4746,9 @@ struct dvb_usb_device_properties dib0700_devices[] = {
 			.rc_codes         = RC_MAP_DIB0700_RC5_TABLE,
 			.module_name	  = "dib0700",
 			.rc_query         = dib0700_rc_query_old_firmware,
-			.allowed_protos   = RC_BIT_RC5 |
-					    RC_BIT_RC6_MCE |
-					    RC_BIT_NEC,
+			.allowed_protos   = RC_PROTO_BIT_RC5 |
+					    RC_PROTO_BIT_RC6_MCE |
+					    RC_PROTO_BIT_NEC,
 			.change_protocol  = dib0700_change_protocol,
 		},
 	}, { DIB0700_DEFAULT_DEVICE_PROPERTIES,
@@ -4783,9 +4783,9 @@ struct dvb_usb_device_properties dib0700_devices[] = {
 			.rc_codes         = RC_MAP_DIB0700_RC5_TABLE,
 			.module_name	  = "dib0700",
 			.rc_query         = dib0700_rc_query_old_firmware,
-			.allowed_protos   = RC_BIT_RC5 |
-					    RC_BIT_RC6_MCE |
-					    RC_BIT_NEC,
+			.allowed_protos   = RC_PROTO_BIT_RC5 |
+					    RC_PROTO_BIT_RC6_MCE |
+					    RC_PROTO_BIT_NEC,
 			.change_protocol  = dib0700_change_protocol,
 		},
 	}, { DIB0700_DEFAULT_DEVICE_PROPERTIES,
@@ -4820,9 +4820,9 @@ struct dvb_usb_device_properties dib0700_devices[] = {
 			.rc_codes         = RC_MAP_DIB0700_RC5_TABLE,
 			.module_name	  = "dib0700",
 			.rc_query         = dib0700_rc_query_old_firmware,
-			.allowed_protos   = RC_BIT_RC5 |
-					    RC_BIT_RC6_MCE |
-					    RC_BIT_NEC,
+			.allowed_protos   = RC_PROTO_BIT_RC5 |
+					    RC_PROTO_BIT_RC6_MCE |
+					    RC_PROTO_BIT_NEC,
 			.change_protocol  = dib0700_change_protocol,
 		},
 	}, { DIB0700_DEFAULT_DEVICE_PROPERTIES,
@@ -4871,9 +4871,9 @@ struct dvb_usb_device_properties dib0700_devices[] = {
 			.rc_codes         = RC_MAP_DIB0700_RC5_TABLE,
 			.module_name	  = "dib0700",
 			.rc_query         = dib0700_rc_query_old_firmware,
-			.allowed_protos   = RC_BIT_RC5 |
-					    RC_BIT_RC6_MCE |
-					    RC_BIT_NEC,
+			.allowed_protos   = RC_PROTO_BIT_RC5 |
+					    RC_PROTO_BIT_RC6_MCE |
+					    RC_PROTO_BIT_NEC,
 			.change_protocol  = dib0700_change_protocol,
 		},
 	}, { DIB0700_DEFAULT_DEVICE_PROPERTIES,
@@ -4906,9 +4906,9 @@ struct dvb_usb_device_properties dib0700_devices[] = {
 			.rc_codes         = RC_MAP_DIB0700_RC5_TABLE,
 			.module_name	  = "dib0700",
 			.rc_query         = dib0700_rc_query_old_firmware,
-			.allowed_protos   = RC_BIT_RC5 |
-					    RC_BIT_RC6_MCE |
-					    RC_BIT_NEC,
+			.allowed_protos   = RC_PROTO_BIT_RC5 |
+					    RC_PROTO_BIT_RC6_MCE |
+					    RC_PROTO_BIT_NEC,
 			.change_protocol  = dib0700_change_protocol,
 		},
 	}, { DIB0700_DEFAULT_DEVICE_PROPERTIES,
@@ -4943,9 +4943,9 @@ struct dvb_usb_device_properties dib0700_devices[] = {
 			.rc_codes         = RC_MAP_DIB0700_RC5_TABLE,
 			.module_name	  = "dib0700",
 			.rc_query         = dib0700_rc_query_old_firmware,
-			.allowed_protos   = RC_BIT_RC5 |
-					    RC_BIT_RC6_MCE |
-					    RC_BIT_NEC,
+			.allowed_protos   = RC_PROTO_BIT_RC5 |
+					    RC_PROTO_BIT_RC6_MCE |
+					    RC_PROTO_BIT_NEC,
 			.change_protocol  = dib0700_change_protocol,
 		},
 	}, { DIB0700_DEFAULT_DEVICE_PROPERTIES,
@@ -4981,9 +4981,9 @@ struct dvb_usb_device_properties dib0700_devices[] = {
 			.rc_codes         = RC_MAP_DIB0700_RC5_TABLE,
 			.module_name	  = "dib0700",
 			.rc_query         = dib0700_rc_query_old_firmware,
-			.allowed_protos   = RC_BIT_RC5 |
-					    RC_BIT_RC6_MCE |
-					    RC_BIT_NEC,
+			.allowed_protos   = RC_PROTO_BIT_RC5 |
+					    RC_PROTO_BIT_RC6_MCE |
+					    RC_PROTO_BIT_NEC,
 			.change_protocol  = dib0700_change_protocol,
 		},
 	}, { DIB0700_DEFAULT_DEVICE_PROPERTIES,
@@ -5035,9 +5035,9 @@ struct dvb_usb_device_properties dib0700_devices[] = {
 			.rc_codes         = RC_MAP_DIB0700_RC5_TABLE,
 			.module_name  = "dib0700",
 			.rc_query         = dib0700_rc_query_old_firmware,
-			.allowed_protos   = RC_BIT_RC5 |
-				RC_BIT_RC6_MCE |
-				RC_BIT_NEC,
+			.allowed_protos   = RC_PROTO_BIT_RC5 |
+				RC_PROTO_BIT_RC6_MCE |
+				RC_PROTO_BIT_NEC,
 			.change_protocol  = dib0700_change_protocol,
 		},
 	},
diff --git a/drivers/media/usb/dvb-usb/dtt200u.c b/drivers/media/usb/dvb-usb/dtt200u.c
index fcbff7fb0c4e..512370786696 100644
--- a/drivers/media/usb/dvb-usb/dtt200u.c
+++ b/drivers/media/usb/dvb-usb/dtt200u.c
@@ -100,14 +100,14 @@ static int dtt200u_rc_query(struct dvb_usb_device *d)
 		goto ret;
 
 	if (st->data[0] == 1) {
-		enum rc_type proto = RC_TYPE_NEC;
+		enum rc_proto proto = RC_PROTO_NEC;
 
 		scancode = st->data[1];
 		if ((u8) ~st->data[1] != st->data[2]) {
 			/* Extended NEC */
 			scancode = scancode << 8;
 			scancode |= st->data[2];
-			proto = RC_TYPE_NECX;
+			proto = RC_PROTO_NECX;
 		}
 		scancode = scancode << 8;
 		scancode |= st->data[3];
@@ -213,7 +213,7 @@ static struct dvb_usb_device_properties dtt200u_properties = {
 		.rc_interval     = 300,
 		.rc_codes        = RC_MAP_DTT200U,
 		.rc_query        = dtt200u_rc_query,
-		.allowed_protos  = RC_BIT_NEC,
+		.allowed_protos  = RC_PROTO_BIT_NEC,
 	},
 
 	.generic_bulk_ctrl_endpoint = 0x01,
@@ -265,7 +265,7 @@ static struct dvb_usb_device_properties wt220u_properties = {
 		.rc_interval     = 300,
 		.rc_codes        = RC_MAP_DTT200U,
 		.rc_query        = dtt200u_rc_query,
-		.allowed_protos  = RC_BIT_NEC,
+		.allowed_protos  = RC_PROTO_BIT_NEC,
 	},
 
 	.generic_bulk_ctrl_endpoint = 0x01,
@@ -317,7 +317,7 @@ static struct dvb_usb_device_properties wt220u_fc_properties = {
 		.rc_interval     = 300,
 		.rc_codes        = RC_MAP_DTT200U,
 		.rc_query        = dtt200u_rc_query,
-		.allowed_protos  = RC_BIT_NEC,
+		.allowed_protos  = RC_PROTO_BIT_NEC,
 	},
 
 	.generic_bulk_ctrl_endpoint = 0x01,
@@ -369,7 +369,7 @@ static struct dvb_usb_device_properties wt220u_zl0353_properties = {
 		.rc_interval     = 300,
 		.rc_codes        = RC_MAP_DTT200U,
 		.rc_query        = dtt200u_rc_query,
-		.allowed_protos  = RC_BIT_NEC,
+		.allowed_protos  = RC_PROTO_BIT_NEC,
 	},
 
 	.generic_bulk_ctrl_endpoint = 0x01,
diff --git a/drivers/media/usb/dvb-usb/dvb-usb-remote.c b/drivers/media/usb/dvb-usb/dvb-usb-remote.c
index f05f1fc80729..0b03f9bd9c26 100644
--- a/drivers/media/usb/dvb-usb/dvb-usb-remote.c
+++ b/drivers/media/usb/dvb-usb/dvb-usb-remote.c
@@ -279,7 +279,7 @@ static int rc_core_dvb_usb_remote_init(struct dvb_usb_device *d)
 	dev->change_protocol = d->props.rc.core.change_protocol;
 	dev->allowed_protocols = d->props.rc.core.allowed_protos;
 	usb_to_input_id(d->udev, &dev->input_id);
-	dev->input_name = "IR-receiver inside an USB DVB receiver";
+	dev->device_name = "IR-receiver inside an USB DVB receiver";
 	dev->input_phys = d->rc_phys;
 	dev->dev.parent = &d->udev->dev;
 	dev->priv = d;
diff --git a/drivers/media/usb/dvb-usb/dvb-usb.h b/drivers/media/usb/dvb-usb/dvb-usb.h
index 67f898b6f6d0..72468fdffa18 100644
--- a/drivers/media/usb/dvb-usb/dvb-usb.h
+++ b/drivers/media/usb/dvb-usb/dvb-usb.h
@@ -202,7 +202,7 @@ struct dvb_rc {
 	u64 protocol;
 	u64 allowed_protos;
 	enum rc_driver_type driver_type;
-	int (*change_protocol)(struct rc_dev *dev, u64 *rc_type);
+	int (*change_protocol)(struct rc_dev *dev, u64 *rc_proto);
 	char *module_name;
 	int (*rc_query) (struct dvb_usb_device *d);
 	int rc_interval;
diff --git a/drivers/media/usb/dvb-usb/dw2102.c b/drivers/media/usb/dvb-usb/dw2102.c
index 57b187240110..b421329b21fa 100644
--- a/drivers/media/usb/dvb-usb/dw2102.c
+++ b/drivers/media/usb/dvb-usb/dw2102.c
@@ -1671,7 +1671,7 @@ static int dw2102_rc_query(struct dvb_usb_device *d)
 		if (msg.buf[0] != 0xff) {
 			deb_rc("%s: rc code: %x, %x\n",
 					__func__, key[0], key[1]);
-			rc_keydown(d->rc_dev, RC_TYPE_UNKNOWN, key[0], 0);
+			rc_keydown(d->rc_dev, RC_PROTO_UNKNOWN, key[0], 0);
 		}
 	}
 
@@ -1692,7 +1692,8 @@ static int prof_rc_query(struct dvb_usb_device *d)
 		if (msg.buf[0] != 0xff) {
 			deb_rc("%s: rc code: %x, %x\n",
 					__func__, key[0], key[1]);
-			rc_keydown(d->rc_dev, RC_TYPE_UNKNOWN, key[0]^0xff, 0);
+			rc_keydown(d->rc_dev, RC_PROTO_UNKNOWN, key[0] ^ 0xff,
+				   0);
 		}
 	}
 
@@ -1713,7 +1714,7 @@ static int su3000_rc_query(struct dvb_usb_device *d)
 		if (msg.buf[0] != 0xff) {
 			deb_rc("%s: rc code: %x, %x\n",
 					__func__, key[0], key[1]);
-			rc_keydown(d->rc_dev, RC_TYPE_RC5,
+			rc_keydown(d->rc_dev, RC_PROTO_RC5,
 				   RC_SCANCODE_RC5(key[1], key[0]), 0);
 		}
 	}
@@ -1912,7 +1913,7 @@ static struct dvb_usb_device_properties dw2102_properties = {
 		.rc_interval = 150,
 		.rc_codes = RC_MAP_DM1105_NEC,
 		.module_name = "dw2102",
-		.allowed_protos   = RC_BIT_NEC,
+		.allowed_protos   = RC_PROTO_BIT_NEC,
 		.rc_query = dw2102_rc_query,
 	},
 
@@ -1967,7 +1968,7 @@ static struct dvb_usb_device_properties dw2104_properties = {
 		.rc_interval = 150,
 		.rc_codes = RC_MAP_DM1105_NEC,
 		.module_name = "dw2102",
-		.allowed_protos   = RC_BIT_NEC,
+		.allowed_protos   = RC_PROTO_BIT_NEC,
 		.rc_query = dw2102_rc_query,
 	},
 
@@ -2018,7 +2019,7 @@ static struct dvb_usb_device_properties dw3101_properties = {
 		.rc_interval = 150,
 		.rc_codes = RC_MAP_DM1105_NEC,
 		.module_name = "dw2102",
-		.allowed_protos   = RC_BIT_NEC,
+		.allowed_protos   = RC_PROTO_BIT_NEC,
 		.rc_query = dw2102_rc_query,
 	},
 
@@ -2067,7 +2068,7 @@ static struct dvb_usb_device_properties s6x0_properties = {
 		.rc_interval = 150,
 		.rc_codes = RC_MAP_TEVII_NEC,
 		.module_name = "dw2102",
-		.allowed_protos   = RC_BIT_NEC,
+		.allowed_protos   = RC_PROTO_BIT_NEC,
 		.rc_query = dw2102_rc_query,
 	},
 
@@ -2103,46 +2104,46 @@ static struct dvb_usb_device_properties s6x0_properties = {
 };
 
 static struct dvb_usb_device_properties *p1100;
-static struct dvb_usb_device_description d1100 = {
+static const struct dvb_usb_device_description d1100 = {
 	"Prof 1100 USB ",
 	{&dw2102_table[PROF_1100], NULL},
 	{NULL},
 };
 
 static struct dvb_usb_device_properties *s660;
-static struct dvb_usb_device_description d660 = {
+static const struct dvb_usb_device_description d660 = {
 	"TeVii S660 USB",
 	{&dw2102_table[TEVII_S660], NULL},
 	{NULL},
 };
 
-static struct dvb_usb_device_description d480_1 = {
+static const struct dvb_usb_device_description d480_1 = {
 	"TeVii S480.1 USB",
 	{&dw2102_table[TEVII_S480_1], NULL},
 	{NULL},
 };
 
-static struct dvb_usb_device_description d480_2 = {
+static const struct dvb_usb_device_description d480_2 = {
 	"TeVii S480.2 USB",
 	{&dw2102_table[TEVII_S480_2], NULL},
 	{NULL},
 };
 
 static struct dvb_usb_device_properties *p7500;
-static struct dvb_usb_device_description d7500 = {
+static const struct dvb_usb_device_description d7500 = {
 	"Prof 7500 USB DVB-S2",
 	{&dw2102_table[PROF_7500], NULL},
 	{NULL},
 };
 
 static struct dvb_usb_device_properties *s421;
-static struct dvb_usb_device_description d421 = {
+static const struct dvb_usb_device_description d421 = {
 	"TeVii S421 PCI",
 	{&dw2102_table[TEVII_S421], NULL},
 	{NULL},
 };
 
-static struct dvb_usb_device_description d632 = {
+static const struct dvb_usb_device_description d632 = {
 	"TeVii S632 USB",
 	{&dw2102_table[TEVII_S632], NULL},
 	{NULL},
@@ -2161,7 +2162,7 @@ static struct dvb_usb_device_properties su3000_properties = {
 		.rc_interval = 150,
 		.rc_codes = RC_MAP_SU3000,
 		.module_name = "dw2102",
-		.allowed_protos   = RC_BIT_RC5,
+		.allowed_protos   = RC_PROTO_BIT_RC5,
 		.rc_query = su3000_rc_query,
 	},
 
@@ -2230,7 +2231,7 @@ static struct dvb_usb_device_properties t220_properties = {
 		.rc_interval = 150,
 		.rc_codes = RC_MAP_SU3000,
 		.module_name = "dw2102",
-		.allowed_protos   = RC_BIT_RC5,
+		.allowed_protos   = RC_PROTO_BIT_RC5,
 		.rc_query = su3000_rc_query,
 	},
 
@@ -2279,7 +2280,7 @@ static struct dvb_usb_device_properties tt_s2_4600_properties = {
 		.rc_interval = 250,
 		.rc_codes = RC_MAP_TT_1500,
 		.module_name = "dw2102",
-		.allowed_protos   = RC_BIT_RC5,
+		.allowed_protos   = RC_PROTO_BIT_RC5,
 		.rc_query = su3000_rc_query,
 	},
 
@@ -2334,10 +2335,12 @@ static struct dvb_usb_device_properties tt_s2_4600_properties = {
 static int dw2102_probe(struct usb_interface *intf,
 		const struct usb_device_id *id)
 {
+	int retval = -ENOMEM;
 	p1100 = kmemdup(&s6x0_properties,
 			sizeof(struct dvb_usb_device_properties), GFP_KERNEL);
 	if (!p1100)
-		return -ENOMEM;
+		goto err0;
+
 	/* copy default structure */
 	/* fill only different fields */
 	p1100->firmware = P1100_FIRMWARE;
@@ -2348,10 +2351,9 @@ static int dw2102_probe(struct usb_interface *intf,
 
 	s660 = kmemdup(&s6x0_properties,
 		       sizeof(struct dvb_usb_device_properties), GFP_KERNEL);
-	if (!s660) {
-		kfree(p1100);
-		return -ENOMEM;
-	}
+	if (!s660)
+		goto err1;
+
 	s660->firmware = S660_FIRMWARE;
 	s660->num_device_descs = 3;
 	s660->devices[0] = d660;
@@ -2361,11 +2363,9 @@ static int dw2102_probe(struct usb_interface *intf,
 
 	p7500 = kmemdup(&s6x0_properties,
 			sizeof(struct dvb_usb_device_properties), GFP_KERNEL);
-	if (!p7500) {
-		kfree(p1100);
-		kfree(s660);
-		return -ENOMEM;
-	}
+	if (!p7500)
+		goto err2;
+
 	p7500->firmware = P7500_FIRMWARE;
 	p7500->devices[0] = d7500;
 	p7500->rc.core.rc_query = prof_rc_query;
@@ -2375,12 +2375,9 @@ static int dw2102_probe(struct usb_interface *intf,
 
 	s421 = kmemdup(&su3000_properties,
 		       sizeof(struct dvb_usb_device_properties), GFP_KERNEL);
-	if (!s421) {
-		kfree(p1100);
-		kfree(s660);
-		kfree(p7500);
-		return -ENOMEM;
-	}
+	if (!s421)
+		goto err3;
+
 	s421->num_device_descs = 2;
 	s421->devices[0] = d421;
 	s421->devices[1] = d632;
@@ -2410,7 +2407,16 @@ static int dw2102_probe(struct usb_interface *intf,
 			 THIS_MODULE, NULL, adapter_nr))
 		return 0;
 
-	return -ENODEV;
+	retval = -ENODEV;
+	kfree(s421);
+err3:
+	kfree(p7500);
+err2:
+	kfree(s660);
+err1:
+	kfree(p1100);
+err0:
+	return retval;
 }
 
 static void dw2102_disconnect(struct usb_interface *intf)
diff --git a/drivers/media/usb/dvb-usb/m920x.c b/drivers/media/usb/dvb-usb/m920x.c
index 70672e1e5ec7..32081c2ce0da 100644
--- a/drivers/media/usb/dvb-usb/m920x.c
+++ b/drivers/media/usb/dvb-usb/m920x.c
@@ -241,7 +241,7 @@ static int m920x_rc_core_query(struct dvb_usb_device *d)
 	else if (state == REMOTE_KEY_REPEAT)
 		rc_repeat(d->rc_dev);
 	else
-		rc_keydown(d->rc_dev, RC_TYPE_UNKNOWN, rc_state[1], 0);
+		rc_keydown(d->rc_dev, RC_PROTO_UNKNOWN, rc_state[1], 0);
 
 out:
 	kfree(rc_state);
@@ -1208,7 +1208,7 @@ static struct dvb_usb_device_properties vp7049_properties = {
 		.rc_interval    = 150,
 		.rc_codes       = RC_MAP_TWINHAN_VP1027_DVBS,
 		.rc_query       = m920x_rc_core_query,
-		.allowed_protos = RC_BIT_UNKNOWN,
+		.allowed_protos = RC_PROTO_BIT_UNKNOWN,
 	},
 
 	.size_of_priv     = sizeof(struct m920x_state),
diff --git a/drivers/media/usb/dvb-usb/pctv452e.c b/drivers/media/usb/dvb-usb/pctv452e.c
index d54ebe7e0215..601ade7ca48d 100644
--- a/drivers/media/usb/dvb-usb/pctv452e.c
+++ b/drivers/media/usb/dvb-usb/pctv452e.c
@@ -600,7 +600,7 @@ static int pctv452e_rc_query(struct dvb_usb_device *d)
 			info("%s: cmd=0x%02x sys=0x%02x\n",
 				__func__, rx[6], rx[7]);
 
-		rc_keydown(d->rc_dev, RC_TYPE_RC5, state->last_rc_key, 0);
+		rc_keydown(d->rc_dev, RC_PROTO_RC5, state->last_rc_key, 0);
 	} else if (state->last_rc_key) {
 		rc_keyup(d->rc_dev);
 		state->last_rc_key = 0;
@@ -958,7 +958,7 @@ static struct dvb_usb_device_properties pctv452e_properties = {
 
 	.rc.core = {
 		.rc_codes	= RC_MAP_DIB0700_RC5_TABLE,
-		.allowed_protos	= RC_BIT_RC5,
+		.allowed_protos	= RC_PROTO_BIT_RC5,
 		.rc_query	= pctv452e_rc_query,
 		.rc_interval	= 100,
 	},
@@ -1011,7 +1011,7 @@ static struct dvb_usb_device_properties tt_connect_s2_3600_properties = {
 
 	.rc.core = {
 		.rc_codes	= RC_MAP_TT_1500,
-		.allowed_protos	= RC_BIT_RC5,
+		.allowed_protos	= RC_PROTO_BIT_RC5,
 		.rc_query	= pctv452e_rc_query,
 		.rc_interval	= 100,
 	},
diff --git a/drivers/media/usb/dvb-usb/technisat-usb2.c b/drivers/media/usb/dvb-usb/technisat-usb2.c
index 9f7dd1afcb15..18d0f8f5283f 100644
--- a/drivers/media/usb/dvb-usb/technisat-usb2.c
+++ b/drivers/media/usb/dvb-usb/technisat-usb2.c
@@ -749,7 +749,7 @@ static struct dvb_usb_device_properties technisat_usb2_devices = {
 		.rc_codes    = RC_MAP_TECHNISAT_USB2,
 		.module_name = "technisat-usb2",
 		.rc_query    = technisat_usb2_rc_query,
-		.allowed_protos = RC_BIT_ALL_IR_DECODER,
+		.allowed_protos = RC_PROTO_BIT_ALL_IR_DECODER,
 		.driver_type    = RC_DRIVER_IR_RAW,
 	}
 };
diff --git a/drivers/media/usb/dvb-usb/ttusb2.c b/drivers/media/usb/dvb-usb/ttusb2.c
index 9e0d6a4166d2..e7020f245f53 100644
--- a/drivers/media/usb/dvb-usb/ttusb2.c
+++ b/drivers/media/usb/dvb-usb/ttusb2.c
@@ -459,7 +459,7 @@ static int tt3650_rc_query(struct dvb_usb_device *d)
 		/* got a "press" event */
 		st->last_rc_key = RC_SCANCODE_RC5(rx[3], rx[2]);
 		deb_info("%s: cmd=0x%02x sys=0x%02x\n", __func__, rx[2], rx[3]);
-		rc_keydown(d->rc_dev, RC_TYPE_RC5, st->last_rc_key, rx[1]);
+		rc_keydown(d->rc_dev, RC_PROTO_RC5, st->last_rc_key, rx[1]);
 	} else if (st->last_rc_key) {
 		rc_keyup(d->rc_dev);
 		st->last_rc_key = 0;
@@ -766,7 +766,7 @@ static struct dvb_usb_device_properties ttusb2_properties_ct3650 = {
 		.rc_interval      = 150, /* Less than IR_KEYPRESS_TIMEOUT */
 		.rc_codes         = RC_MAP_TT_1500,
 		.rc_query         = tt3650_rc_query,
-		.allowed_protos   = RC_BIT_RC5,
+		.allowed_protos   = RC_PROTO_BIT_RC5,
 	},
 
 	.num_adapters = 1,
diff --git a/drivers/media/usb/em28xx/em28xx-audio.c b/drivers/media/usb/em28xx/em28xx-audio.c
index ffad7f1af166..4628d73f46f2 100644
--- a/drivers/media/usb/em28xx/em28xx-audio.c
+++ b/drivers/media/usb/em28xx/em28xx-audio.c
@@ -216,7 +216,7 @@ static int snd_pcm_alloc_vmalloc_buffer(struct snd_pcm_substream *subs,
 	return 0;
 }
 
-static struct snd_pcm_hardware snd_em28xx_hw_capture = {
+static const struct snd_pcm_hardware snd_em28xx_hw_capture = {
 	.info = SNDRV_PCM_INFO_BLOCK_TRANSFER |
 		SNDRV_PCM_INFO_MMAP           |
 		SNDRV_PCM_INFO_INTERLEAVED    |
@@ -564,7 +564,7 @@ static int em28xx_vol_get(struct snd_kcontrol *kcontrol,
 		val, (int)kcontrol->private_value);
 
 	value->value.integer.value[0] = 0x1f - (val & 0x1f);
-	value->value.integer.value[1] = 0x1f - ((val << 8) & 0x1f);
+	value->value.integer.value[1] = 0x1f - ((val >> 8) & 0x1f);
 
 	return 0;
 }
diff --git a/drivers/media/usb/em28xx/em28xx-i2c.c b/drivers/media/usb/em28xx/em28xx-i2c.c
index 60b195c157b8..66c5012a628a 100644
--- a/drivers/media/usb/em28xx/em28xx-i2c.c
+++ b/drivers/media/usb/em28xx/em28xx-i2c.c
@@ -876,7 +876,7 @@ static const struct i2c_algorithm em28xx_algo = {
 	.functionality = functionality,
 };
 
-static struct i2c_adapter em28xx_adap_template = {
+static const struct i2c_adapter em28xx_adap_template = {
 	.owner = THIS_MODULE,
 	.name = "em28xx",
 	.algo = &em28xx_algo,
diff --git a/drivers/media/usb/em28xx/em28xx-input.c b/drivers/media/usb/em28xx/em28xx-input.c
index ca9673917ad5..046223de1e91 100644
--- a/drivers/media/usb/em28xx/em28xx-input.c
+++ b/drivers/media/usb/em28xx/em28xx-input.c
@@ -55,7 +55,7 @@ struct em28xx_ir_poll_result {
 	unsigned int toggle_bit:1;
 	unsigned int read_count:7;
 
-	enum rc_type protocol;
+	enum rc_proto protocol;
 	u32 scancode;
 };
 
@@ -70,11 +70,12 @@ struct em28xx_IR {
 	struct delayed_work work;
 	unsigned int full_code:1;
 	unsigned int last_readcount;
-	u64 rc_type;
+	u64 rc_proto;
 
 	struct i2c_client *i2c_client;
 
-	int  (*get_key_i2c)(struct i2c_client *ir, enum rc_type *protocol, u32 *scancode);
+	int  (*get_key_i2c)(struct i2c_client *ir, enum rc_proto *protocol,
+			    u32 *scancode);
 	int  (*get_key)(struct em28xx_IR *, struct em28xx_ir_poll_result *);
 };
 
@@ -83,7 +84,7 @@ struct em28xx_IR {
  **********************************************************/
 
 static int em28xx_get_key_terratec(struct i2c_client *i2c_dev,
-				   enum rc_type *protocol, u32 *scancode)
+				   enum rc_proto *protocol, u32 *scancode)
 {
 	unsigned char b;
 
@@ -101,13 +102,13 @@ static int em28xx_get_key_terratec(struct i2c_client *i2c_dev,
 		/* keep old data */
 		return 1;
 
-	*protocol = RC_TYPE_UNKNOWN;
+	*protocol = RC_PROTO_UNKNOWN;
 	*scancode = b;
 	return 1;
 }
 
 static int em28xx_get_key_em_haup(struct i2c_client *i2c_dev,
-				  enum rc_type *protocol, u32 *scancode)
+				  enum rc_proto *protocol, u32 *scancode)
 {
 	unsigned char buf[2];
 	int size;
@@ -131,13 +132,14 @@ static int em28xx_get_key_em_haup(struct i2c_client *i2c_dev,
 	 * So, the code translation is not complete. Yet, it is enough to
 	 * work with the provided RC5 IR.
 	 */
-	*protocol = RC_TYPE_RC5;
+	*protocol = RC_PROTO_RC5;
 	*scancode = (bitrev8(buf[1]) & 0x1f) << 8 | bitrev8(buf[0]) >> 2;
 	return 1;
 }
 
 static int em28xx_get_key_pinnacle_usb_grey(struct i2c_client *i2c_dev,
-					    enum rc_type *protocol, u32 *scancode)
+					    enum rc_proto *protocol,
+					    u32 *scancode)
 {
 	unsigned char buf[3];
 
@@ -149,13 +151,14 @@ static int em28xx_get_key_pinnacle_usb_grey(struct i2c_client *i2c_dev,
 	if (buf[0] != 0x00)
 		return 0;
 
-	*protocol = RC_TYPE_UNKNOWN;
+	*protocol = RC_PROTO_UNKNOWN;
 	*scancode = buf[2] & 0x3f;
 	return 1;
 }
 
 static int em28xx_get_key_winfast_usbii_deluxe(struct i2c_client *i2c_dev,
-					       enum rc_type *protocol, u32 *scancode)
+					       enum rc_proto *protocol,
+					       u32 *scancode)
 {
 	unsigned char subaddr, keydetect, key;
 
@@ -175,7 +178,7 @@ static int em28xx_get_key_winfast_usbii_deluxe(struct i2c_client *i2c_dev,
 	if (key == 0x00)
 		return 0;
 
-	*protocol = RC_TYPE_UNKNOWN;
+	*protocol = RC_PROTO_UNKNOWN;
 	*scancode = key;
 	return 1;
 }
@@ -207,19 +210,19 @@ static int default_polling_getkey(struct em28xx_IR *ir,
 	poll_result->read_count = (msg[0] & 0x7f);
 
 	/* Remote Control Address/Data (Regs 0x46/0x47) */
-	switch (ir->rc_type) {
-	case RC_BIT_RC5:
-		poll_result->protocol = RC_TYPE_RC5;
+	switch (ir->rc_proto) {
+	case RC_PROTO_BIT_RC5:
+		poll_result->protocol = RC_PROTO_RC5;
 		poll_result->scancode = RC_SCANCODE_RC5(msg[1], msg[2]);
 		break;
 
-	case RC_BIT_NEC:
-		poll_result->protocol = RC_TYPE_NEC;
+	case RC_PROTO_BIT_NEC:
+		poll_result->protocol = RC_PROTO_NEC;
 		poll_result->scancode = RC_SCANCODE_NEC(msg[1], msg[2]);
 		break;
 
 	default:
-		poll_result->protocol = RC_TYPE_UNKNOWN;
+		poll_result->protocol = RC_PROTO_UNKNOWN;
 		poll_result->scancode = msg[1] << 8 | msg[2];
 		break;
 	}
@@ -252,37 +255,37 @@ static int em2874_polling_getkey(struct em28xx_IR *ir,
 	 * Remote Control Address (Reg 0x52)
 	 * Remote Control Data (Reg 0x53-0x55)
 	 */
-	switch (ir->rc_type) {
-	case RC_BIT_RC5:
-		poll_result->protocol = RC_TYPE_RC5;
+	switch (ir->rc_proto) {
+	case RC_PROTO_BIT_RC5:
+		poll_result->protocol = RC_PROTO_RC5;
 		poll_result->scancode = RC_SCANCODE_RC5(msg[1], msg[2]);
 		break;
 
-	case RC_BIT_NEC:
+	case RC_PROTO_BIT_NEC:
 		poll_result->scancode = msg[1] << 8 | msg[2];
 		if ((msg[3] ^ msg[4]) != 0xff) {	/* 32 bits NEC */
-			poll_result->protocol = RC_TYPE_NEC32;
+			poll_result->protocol = RC_PROTO_NEC32;
 			poll_result->scancode = RC_SCANCODE_NEC32((msg[1] << 24) |
 								  (msg[2] << 16) |
 								  (msg[3] << 8)  |
 								  (msg[4]));
 		} else if ((msg[1] ^ msg[2]) != 0xff) {	/* 24 bits NEC */
-			poll_result->protocol = RC_TYPE_NECX;
+			poll_result->protocol = RC_PROTO_NECX;
 			poll_result->scancode = RC_SCANCODE_NECX(msg[1] << 8 |
 								 msg[2], msg[3]);
 		} else {				/* Normal NEC */
-			poll_result->protocol = RC_TYPE_NEC;
+			poll_result->protocol = RC_PROTO_NEC;
 			poll_result->scancode = RC_SCANCODE_NEC(msg[1], msg[3]);
 		}
 		break;
 
-	case RC_BIT_RC6_0:
-		poll_result->protocol = RC_TYPE_RC6_0;
+	case RC_PROTO_BIT_RC6_0:
+		poll_result->protocol = RC_PROTO_RC6_0;
 		poll_result->scancode = RC_SCANCODE_RC6_0(msg[1], msg[2]);
 		break;
 
 	default:
-		poll_result->protocol = RC_TYPE_UNKNOWN;
+		poll_result->protocol = RC_PROTO_UNKNOWN;
 		poll_result->scancode = (msg[1] << 24) | (msg[2] << 16) |
 					(msg[3] << 8)  | msg[4];
 		break;
@@ -298,7 +301,7 @@ static int em2874_polling_getkey(struct em28xx_IR *ir,
 static int em28xx_i2c_ir_handle_key(struct em28xx_IR *ir)
 {
 	static u32 scancode;
-	enum rc_type protocol;
+	enum rc_proto protocol;
 	int rc;
 
 	rc = ir->get_key_i2c(ir->i2c_client, &protocol, &scancode);
@@ -338,7 +341,7 @@ static void em28xx_ir_handle_key(struct em28xx_IR *ir)
 				   poll_result.toggle_bit);
 		else
 			rc_keydown(ir->rc,
-				   RC_TYPE_UNKNOWN,
+				   RC_PROTO_UNKNOWN,
 				   poll_result.scancode & 0xff,
 				   poll_result.toggle_bit);
 
@@ -383,70 +386,71 @@ static void em28xx_ir_stop(struct rc_dev *rc)
 	cancel_delayed_work_sync(&ir->work);
 }
 
-static int em2860_ir_change_protocol(struct rc_dev *rc_dev, u64 *rc_type)
+static int em2860_ir_change_protocol(struct rc_dev *rc_dev, u64 *rc_proto)
 {
 	struct em28xx_IR *ir = rc_dev->priv;
 	struct em28xx *dev = ir->dev;
 
 	/* Adjust xclk based on IR table for RC5/NEC tables */
-	if (*rc_type & RC_BIT_RC5) {
+	if (*rc_proto & RC_PROTO_BIT_RC5) {
 		dev->board.xclk |= EM28XX_XCLK_IR_RC5_MODE;
 		ir->full_code = 1;
-		*rc_type = RC_BIT_RC5;
-	} else if (*rc_type & RC_BIT_NEC) {
+		*rc_proto = RC_PROTO_BIT_RC5;
+	} else if (*rc_proto & RC_PROTO_BIT_NEC) {
 		dev->board.xclk &= ~EM28XX_XCLK_IR_RC5_MODE;
 		ir->full_code = 1;
-		*rc_type = RC_BIT_NEC;
-	} else if (*rc_type & RC_BIT_UNKNOWN) {
-		*rc_type = RC_BIT_UNKNOWN;
+		*rc_proto = RC_PROTO_BIT_NEC;
+	} else if (*rc_proto & RC_PROTO_BIT_UNKNOWN) {
+		*rc_proto = RC_PROTO_BIT_UNKNOWN;
 	} else {
-		*rc_type = ir->rc_type;
+		*rc_proto = ir->rc_proto;
 		return -EINVAL;
 	}
 	em28xx_write_reg_bits(dev, EM28XX_R0F_XCLK, dev->board.xclk,
 			      EM28XX_XCLK_IR_RC5_MODE);
 
-	ir->rc_type = *rc_type;
+	ir->rc_proto = *rc_proto;
 
 	return 0;
 }
 
-static int em2874_ir_change_protocol(struct rc_dev *rc_dev, u64 *rc_type)
+static int em2874_ir_change_protocol(struct rc_dev *rc_dev, u64 *rc_proto)
 {
 	struct em28xx_IR *ir = rc_dev->priv;
 	struct em28xx *dev = ir->dev;
 	u8 ir_config = EM2874_IR_RC5;
 
 	/* Adjust xclk and set type based on IR table for RC5/NEC/RC6 tables */
-	if (*rc_type & RC_BIT_RC5) {
+	if (*rc_proto & RC_PROTO_BIT_RC5) {
 		dev->board.xclk |= EM28XX_XCLK_IR_RC5_MODE;
 		ir->full_code = 1;
-		*rc_type = RC_BIT_RC5;
-	} else if (*rc_type & RC_BIT_NEC) {
+		*rc_proto = RC_PROTO_BIT_RC5;
+	} else if (*rc_proto & RC_PROTO_BIT_NEC) {
 		dev->board.xclk &= ~EM28XX_XCLK_IR_RC5_MODE;
 		ir_config = EM2874_IR_NEC | EM2874_IR_NEC_NO_PARITY;
 		ir->full_code = 1;
-		*rc_type = RC_BIT_NEC;
-	} else if (*rc_type & RC_BIT_RC6_0) {
+		*rc_proto = RC_PROTO_BIT_NEC;
+	} else if (*rc_proto & RC_PROTO_BIT_RC6_0) {
 		dev->board.xclk |= EM28XX_XCLK_IR_RC5_MODE;
 		ir_config = EM2874_IR_RC6_MODE_0;
 		ir->full_code = 1;
-		*rc_type = RC_BIT_RC6_0;
-	} else if (*rc_type & RC_BIT_UNKNOWN) {
-		*rc_type = RC_BIT_UNKNOWN;
+		*rc_proto = RC_PROTO_BIT_RC6_0;
+	} else if (*rc_proto & RC_PROTO_BIT_UNKNOWN) {
+		*rc_proto = RC_PROTO_BIT_UNKNOWN;
 	} else {
-		*rc_type = ir->rc_type;
+		*rc_proto = ir->rc_proto;
 		return -EINVAL;
 	}
 	em28xx_write_regs(dev, EM2874_R50_IR_CONFIG, &ir_config, 1);
 	em28xx_write_reg_bits(dev, EM28XX_R0F_XCLK, dev->board.xclk,
 			      EM28XX_XCLK_IR_RC5_MODE);
 
-	ir->rc_type = *rc_type;
+	ir->rc_proto = *rc_proto;
 
 	return 0;
 }
-static int em28xx_ir_change_protocol(struct rc_dev *rc_dev, u64 *rc_type)
+
+static int em28xx_ir_change_protocol(struct rc_dev *rc_dev, u64 *rc_proto)
 {
 	struct em28xx_IR *ir = rc_dev->priv;
 	struct em28xx *dev = ir->dev;
@@ -455,12 +459,12 @@ static int em28xx_ir_change_protocol(struct rc_dev *rc_dev, u64 *rc_type)
 	switch (dev->chip_id) {
 	case CHIP_ID_EM2860:
 	case CHIP_ID_EM2883:
-		return em2860_ir_change_protocol(rc_dev, rc_type);
+		return em2860_ir_change_protocol(rc_dev, rc_proto);
 	case CHIP_ID_EM2884:
 	case CHIP_ID_EM2874:
 	case CHIP_ID_EM28174:
 	case CHIP_ID_EM28178:
-		return em2874_ir_change_protocol(rc_dev, rc_type);
+		return em2874_ir_change_protocol(rc_dev, rc_proto);
 	default:
 		dev_err(&ir->dev->intf->dev,
 			"Unrecognized em28xx chip id 0x%02x: IR not supported\n",
@@ -686,7 +690,7 @@ static int em28xx_ir_init(struct em28xx *dev)
 	struct em28xx_IR *ir;
 	struct rc_dev *rc;
 	int err = -ENOMEM;
-	u64 rc_type;
+	u64 rc_proto;
 	u16 i2c_rc_dev_addr = 0;
 
 	if (dev->is_audio_only) {
@@ -749,7 +753,7 @@ static int em28xx_ir_init(struct em28xx *dev)
 		case EM2820_BOARD_HAUPPAUGE_WINTV_USB_2:
 			rc->map_name = RC_MAP_HAUPPAUGE;
 			ir->get_key_i2c = em28xx_get_key_em_haup;
-			rc->allowed_protocols = RC_BIT_RC5;
+			rc->allowed_protocols = RC_PROTO_BIT_RC5;
 			break;
 		case EM2820_BOARD_LEADTEK_WINFAST_USBII_DELUXE:
 			rc->map_name = RC_MAP_WINFAST_USBII_DELUXE;
@@ -771,7 +775,8 @@ static int em28xx_ir_init(struct em28xx *dev)
 		switch (dev->chip_id) {
 		case CHIP_ID_EM2860:
 		case CHIP_ID_EM2883:
-			rc->allowed_protocols = RC_BIT_RC5 | RC_BIT_NEC;
+			rc->allowed_protocols = RC_PROTO_BIT_RC5 |
+						RC_PROTO_BIT_NEC;
 			ir->get_key = default_polling_getkey;
 			break;
 		case CHIP_ID_EM2884:
@@ -779,8 +784,9 @@ static int em28xx_ir_init(struct em28xx *dev)
 		case CHIP_ID_EM28174:
 		case CHIP_ID_EM28178:
 			ir->get_key = em2874_polling_getkey;
-			rc->allowed_protocols = RC_BIT_RC5 | RC_BIT_NEC |
-				RC_BIT_NECX | RC_BIT_NEC32 | RC_BIT_RC6_0;
+			rc->allowed_protocols = RC_PROTO_BIT_RC5 |
+				RC_PROTO_BIT_NEC | RC_PROTO_BIT_NECX |
+				RC_PROTO_BIT_NEC32 | RC_PROTO_BIT_RC6_0;
 			break;
 		default:
 			err = -ENODEV;
@@ -791,8 +797,8 @@ static int em28xx_ir_init(struct em28xx *dev)
 		rc->map_name = dev->board.ir_codes;
 
 		/* By default, keep protocol field untouched */
-		rc_type = RC_BIT_UNKNOWN;
-		err = em28xx_ir_change_protocol(rc, &rc_type);
+		rc_proto = RC_PROTO_BIT_UNKNOWN;
+		err = em28xx_ir_change_protocol(rc, &rc_proto);
 		if (err)
 			goto error;
 	}
@@ -807,7 +813,7 @@ static int em28xx_ir_init(struct em28xx *dev)
 	usb_make_path(udev, ir->phys, sizeof(ir->phys));
 	strlcat(ir->phys, "/input0", sizeof(ir->phys));
 
-	rc->input_name = ir->name;
+	rc->device_name = ir->name;
 	rc->input_phys = ir->phys;
 	rc->input_id.bustype = BUS_USB;
 	rc->input_id.version = 1;
diff --git a/drivers/media/usb/go7007/go7007-v4l2.c b/drivers/media/usb/go7007/go7007-v4l2.c
index ed5ec9773969..98cd57eaf36a 100644
--- a/drivers/media/usb/go7007/go7007-v4l2.c
+++ b/drivers/media/usb/go7007/go7007-v4l2.c
@@ -857,7 +857,7 @@ static int go7007_s_ctrl(struct v4l2_ctrl *ctrl)
 	return 0;
 }
 
-static struct v4l2_file_operations go7007_fops = {
+static const struct v4l2_file_operations go7007_fops = {
 	.owner		= THIS_MODULE,
 	.open		= v4l2_fh_open,
 	.release	= vb2_fop_release,
@@ -901,7 +901,7 @@ static const struct v4l2_ioctl_ops video_ioctl_ops = {
 	.vidioc_unsubscribe_event = v4l2_event_unsubscribe,
 };
 
-static struct video_device go7007_template = {
+static const struct video_device go7007_template = {
 	.name		= "go7007",
 	.fops		= &go7007_fops,
 	.release	= video_device_release_empty,
diff --git a/drivers/media/usb/go7007/snd-go7007.c b/drivers/media/usb/go7007/snd-go7007.c
index 070871fb1fc4..c618764480c6 100644
--- a/drivers/media/usb/go7007/snd-go7007.c
+++ b/drivers/media/usb/go7007/snd-go7007.c
@@ -52,7 +52,7 @@ struct go7007_snd {
 	int capturing;
 };
 
-static struct snd_pcm_hardware go7007_snd_capture_hw = {
+static const struct snd_pcm_hardware go7007_snd_capture_hw = {
 	.info			= (SNDRV_PCM_INFO_MMAP |
 					SNDRV_PCM_INFO_INTERLEAVED |
 					SNDRV_PCM_INFO_BLOCK_TRANSFER |
diff --git a/drivers/media/usb/gspca/gspca.c b/drivers/media/usb/gspca/gspca.c
index 16bc1dde2c8c..0f141762abf1 100644
--- a/drivers/media/usb/gspca/gspca.c
+++ b/drivers/media/usb/gspca/gspca.c
@@ -1964,7 +1964,7 @@ out:
 	return ret;
 }
 
-static struct v4l2_file_operations dev_fops = {
+static const struct v4l2_file_operations dev_fops = {
 	.owner = THIS_MODULE,
 	.open = dev_open,
 	.release = dev_close,
diff --git a/drivers/media/usb/gspca/xirlink_cit.c b/drivers/media/usb/gspca/xirlink_cit.c
index b600ea6460d3..68656e7986c7 100644
--- a/drivers/media/usb/gspca/xirlink_cit.c
+++ b/drivers/media/usb/gspca/xirlink_cit.c
@@ -1315,7 +1315,7 @@ static int cit_set_sharpness(struct gspca_dev *gspca_dev, s32 val)
 		break;
 	case CIT_MODEL1: {
 		int i;
-		const unsigned short sa[] = {
+		static const unsigned short sa[] = {
 			0x11, 0x13, 0x16, 0x18, 0x1a, 0x8, 0x0a };
 
 		for (i = 0; i < cit_model1_ntries; i++)
diff --git a/drivers/media/usb/hackrf/hackrf.c b/drivers/media/usb/hackrf/hackrf.c
index d9a525260511..7eb53517a82f 100644
--- a/drivers/media/usb/hackrf/hackrf.c
+++ b/drivers/media/usb/hackrf/hackrf.c
@@ -1263,7 +1263,7 @@ static const struct v4l2_file_operations hackrf_fops = {
 	.unlocked_ioctl           = video_ioctl2,
 };
 
-static struct video_device hackrf_template = {
+static const struct video_device hackrf_template = {
 	.name                     = "HackRF One",
 	.release                  = video_device_release_empty,
 	.fops                     = &hackrf_fops,
@@ -1545,7 +1545,7 @@ err:
 }
 
 /* USB device ID list */
-static struct usb_device_id hackrf_id_table[] = {
+static const struct usb_device_id hackrf_id_table[] = {
 	{ USB_DEVICE(0x1d50, 0x6089) }, /* HackRF One */
 	{ }
 };
diff --git a/drivers/media/usb/hdpvr/hdpvr-core.c b/drivers/media/usb/hdpvr/hdpvr-core.c
index 15f016ad5b89..dbe29c6c4d8b 100644
--- a/drivers/media/usb/hdpvr/hdpvr-core.c
+++ b/drivers/media/usb/hdpvr/hdpvr-core.c
@@ -53,7 +53,7 @@ MODULE_PARM_DESC(boost_audio, "boost the audio signal");
 
 
 /* table of devices that work with this driver */
-static struct usb_device_id hdpvr_table[] = {
+static const struct usb_device_id hdpvr_table[] = {
 	{ USB_DEVICE(HD_PVR_VENDOR_ID, HD_PVR_PRODUCT_ID) },
 	{ USB_DEVICE(HD_PVR_VENDOR_ID, HD_PVR_PRODUCT_ID1) },
 	{ USB_DEVICE(HD_PVR_VENDOR_ID, HD_PVR_PRODUCT_ID2) },
diff --git a/drivers/media/usb/hdpvr/hdpvr-i2c.c b/drivers/media/usb/hdpvr/hdpvr-i2c.c
index fcab55038d99..1db49ed5eaf1 100644
--- a/drivers/media/usb/hdpvr/hdpvr-i2c.c
+++ b/drivers/media/usb/hdpvr/hdpvr-i2c.c
@@ -55,7 +55,8 @@ struct i2c_client *hdpvr_register_ir_rx_i2c(struct hdpvr_device *dev)
 	/* Our default information for ir-kbd-i2c.c to use */
 	init_data->ir_codes = RC_MAP_HAUPPAUGE;
 	init_data->internal_get_key_func = IR_KBD_GET_KEY_HAUP_XVR;
-	init_data->type = RC_BIT_RC5 | RC_BIT_RC6_MCE | RC_BIT_RC6_6A_32;
+	init_data->type = RC_PROTO_BIT_RC5 | RC_PROTO_BIT_RC6_MCE |
+			  RC_PROTO_BIT_RC6_6A_32;
 	init_data->name = "HD-PVR";
 	init_data->polling_interval = 405; /* ms, duplicated from Windows */
 	hdpvr_ir_rx_i2c_board_info.platform_data = init_data;
@@ -184,7 +185,7 @@ static const struct i2c_algorithm hdpvr_algo = {
 	.functionality = hdpvr_functionality,
 };
 
-static struct i2c_adapter hdpvr_i2c_adapter_template = {
+static const struct i2c_adapter hdpvr_i2c_adapter_template = {
 	.name   = "Hauppage HD PVR I2C",
 	.owner  = THIS_MODULE,
 	.algo   = &hdpvr_algo,
diff --git a/drivers/media/usb/msi2500/msi2500.c b/drivers/media/usb/msi2500/msi2500.c
index bb3d31e2a0b5..a097d3dbc141 100644
--- a/drivers/media/usb/msi2500/msi2500.c
+++ b/drivers/media/usb/msi2500/msi2500.c
@@ -1143,7 +1143,7 @@ static const struct v4l2_file_operations msi2500_fops = {
 	.unlocked_ioctl           = video_ioctl2,
 };
 
-static struct video_device msi2500_template = {
+static const struct video_device msi2500_template = {
 	.name                     = "Mirics MSi3101 SDR Dongle",
 	.release                  = video_device_release_empty,
 	.fops                     = &msi2500_fops,
@@ -1308,7 +1308,7 @@ err:
 }
 
 /* USB device ID list */
-static struct usb_device_id msi2500_id_table[] = {
+static const struct usb_device_id msi2500_id_table[] = {
 	{USB_DEVICE(0x1df7, 0x2500)}, /* Mirics MSi3101 SDR Dongle */
 	{USB_DEVICE(0x2040, 0xd300)}, /* Hauppauge WinTV 133559 LF */
 	{}
diff --git a/drivers/media/usb/pulse8-cec/pulse8-cec.c b/drivers/media/usb/pulse8-cec/pulse8-cec.c
index f9ed9c950247..50146f263d90 100644
--- a/drivers/media/usb/pulse8-cec/pulse8-cec.c
+++ b/drivers/media/usb/pulse8-cec/pulse8-cec.c
@@ -642,8 +642,7 @@ static const struct cec_adap_ops pulse8_cec_adap_ops = {
 
 static int pulse8_connect(struct serio *serio, struct serio_driver *drv)
 {
-	u32 caps = CEC_CAP_TRANSMIT | CEC_CAP_LOG_ADDRS | CEC_CAP_PHYS_ADDR |
-		CEC_CAP_PASSTHROUGH | CEC_CAP_RC | CEC_CAP_MONITOR_ALL;
+	u32 caps = CEC_CAP_DEFAULTS | CEC_CAP_PHYS_ADDR | CEC_CAP_MONITOR_ALL;
 	struct pulse8 *pulse8;
 	int err = -ENOMEM;
 	struct cec_log_addrs log_addrs = {};
@@ -656,7 +655,7 @@ static int pulse8_connect(struct serio *serio, struct serio_driver *drv)
 
 	pulse8->serio = serio;
 	pulse8->adap = cec_allocate_adapter(&pulse8_cec_adap_ops, pulse8,
-		"HDMI CEC", caps, 1);
+					    dev_name(&serio->dev), caps, 1);
 	err = PTR_ERR_OR_ZERO(pulse8->adap);
 	if (err < 0)
 		goto free_device;
@@ -732,7 +731,7 @@ static void pulse8_ping_eeprom_work_handler(struct work_struct *work)
 	mutex_unlock(&pulse8->config_lock);
 }
 
-static struct serio_device_id pulse8_serio_ids[] = {
+static const struct serio_device_id pulse8_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_PULSE8_CEC,
diff --git a/drivers/media/usb/pvrusb2/pvrusb2-encoder.c b/drivers/media/usb/pvrusb2/pvrusb2-encoder.c
index ca637074fa1f..43e43404095f 100644
--- a/drivers/media/usb/pvrusb2/pvrusb2-encoder.c
+++ b/drivers/media/usb/pvrusb2/pvrusb2-encoder.c
@@ -198,7 +198,7 @@ static int pvr2_encoder_cmd(void *ctxt,
 	}
 
 
-	LOCK_TAKE(hdw->ctl_lock); do {
+	LOCK_TAKE(hdw->ctl_lock); while (1) {
 
 		if (!hdw->state_encoder_ok) {
 			ret = -EIO;
@@ -293,9 +293,9 @@ rdData[0]);
 
 		wrData[0] = 0x0;
 		ret = pvr2_encoder_write_words(hdw,MBOX_BASE,wrData,1);
-		if (ret) break;
+		break;
 
-	} while(0); LOCK_GIVE(hdw->ctl_lock);
+	}; LOCK_GIVE(hdw->ctl_lock);
 
 	return ret;
 }
diff --git a/drivers/media/usb/pvrusb2/pvrusb2-i2c-core.c b/drivers/media/usb/pvrusb2/pvrusb2-i2c-core.c
index 20a52b785fff..ff7b4d1d385d 100644
--- a/drivers/media/usb/pvrusb2/pvrusb2-i2c-core.c
+++ b/drivers/media/usb/pvrusb2/pvrusb2-i2c-core.c
@@ -514,12 +514,12 @@ static u32 pvr2_i2c_functionality(struct i2c_adapter *adap)
 	return I2C_FUNC_SMBUS_EMUL | I2C_FUNC_I2C;
 }
 
-static struct i2c_algorithm pvr2_i2c_algo_template = {
+static const struct i2c_algorithm pvr2_i2c_algo_template = {
 	.master_xfer   = pvr2_i2c_xfer,
 	.functionality = pvr2_i2c_functionality,
 };
 
-static struct i2c_adapter pvr2_i2c_adap_template = {
+static const struct i2c_adapter pvr2_i2c_adap_template = {
 	.owner         = THIS_MODULE,
 	.class	       = 0,
 };
@@ -567,7 +567,7 @@ static void pvr2_i2c_register_ir(struct pvr2_hdw *hdw)
 	case PVR2_IR_SCHEME_29XXX: /* Original 29xxx device */
 		init_data->ir_codes              = RC_MAP_HAUPPAUGE;
 		init_data->internal_get_key_func = IR_KBD_GET_KEY_HAUP;
-		init_data->type                  = RC_BIT_RC5;
+		init_data->type                  = RC_PROTO_BIT_RC5;
 		init_data->name                  = hdw->hdw_desc->description;
 		init_data->polling_interval      = 100; /* ms From ir-kbd-i2c */
 		/* IR Receiver */
@@ -580,11 +580,11 @@ static void pvr2_i2c_register_ir(struct pvr2_hdw *hdw)
 		break;
 	case PVR2_IR_SCHEME_ZILOG:     /* HVR-1950 style */
 	case PVR2_IR_SCHEME_24XXX_MCE: /* 24xxx MCE device */
-		init_data->ir_codes              = RC_MAP_HAUPPAUGE;
+		init_data->ir_codes = RC_MAP_HAUPPAUGE;
 		init_data->internal_get_key_func = IR_KBD_GET_KEY_HAUP_XVR;
-		init_data->type                  = RC_BIT_RC5 | RC_BIT_RC6_MCE |
-							RC_BIT_RC6_6A_32;
-		init_data->name                  = hdw->hdw_desc->description;
+		init_data->type = RC_PROTO_BIT_RC5 | RC_PROTO_BIT_RC6_MCE |
+							RC_PROTO_BIT_RC6_6A_32;
+		init_data->name = hdw->hdw_desc->description;
 		/* IR Receiver */
 		info.addr          = 0x71;
 		info.platform_data = init_data;
diff --git a/drivers/media/usb/pvrusb2/pvrusb2-v4l2.c b/drivers/media/usb/pvrusb2/pvrusb2-v4l2.c
index 8f13c60198ed..4320bda9352d 100644
--- a/drivers/media/usb/pvrusb2/pvrusb2-v4l2.c
+++ b/drivers/media/usb/pvrusb2/pvrusb2-v4l2.c
@@ -1226,7 +1226,7 @@ static const struct v4l2_file_operations vdev_fops = {
 };
 
 
-static struct video_device vdev_template = {
+static const struct video_device vdev_template = {
 	.fops       = &vdev_fops,
 };
 
diff --git a/drivers/media/usb/pwc/pwc-if.c b/drivers/media/usb/pwc/pwc-if.c
index 22420c14ac98..eb6921d2743e 100644
--- a/drivers/media/usb/pwc/pwc-if.c
+++ b/drivers/media/usb/pwc/pwc-if.c
@@ -146,7 +146,7 @@ static const struct v4l2_file_operations pwc_fops = {
 	.mmap =		vb2_fop_mmap,
 	.unlocked_ioctl = video_ioctl2,
 };
-static struct video_device pwc_template = {
+static const struct video_device pwc_template = {
 	.name =		"Philips Webcam",	/* Filled in later */
 	.release =	video_device_release_empty,
 	.fops =         &pwc_fops,
diff --git a/drivers/media/usb/rainshadow-cec/rainshadow-cec.c b/drivers/media/usb/rainshadow-cec/rainshadow-cec.c
index 65692576690f..cecdcbcd400c 100644
--- a/drivers/media/usb/rainshadow-cec/rainshadow-cec.c
+++ b/drivers/media/usb/rainshadow-cec/rainshadow-cec.c
@@ -309,8 +309,7 @@ static const struct cec_adap_ops rain_cec_adap_ops = {
 
 static int rain_connect(struct serio *serio, struct serio_driver *drv)
 {
-	u32 caps = CEC_CAP_TRANSMIT | CEC_CAP_LOG_ADDRS | CEC_CAP_PHYS_ADDR |
-		CEC_CAP_PASSTHROUGH | CEC_CAP_RC | CEC_CAP_MONITOR_ALL;
+	u32 caps = CEC_CAP_DEFAULTS | CEC_CAP_PHYS_ADDR | CEC_CAP_MONITOR_ALL;
 	struct rain *rain;
 	int err = -ENOMEM;
 	struct cec_log_addrs log_addrs = {};
@@ -323,7 +322,7 @@ static int rain_connect(struct serio *serio, struct serio_driver *drv)
 
 	rain->serio = serio;
 	rain->adap = cec_allocate_adapter(&rain_cec_adap_ops, rain,
-		"HDMI CEC", caps, 1);
+					  dev_name(&serio->dev), caps, 1);
 	err = PTR_ERR_OR_ZERO(rain->adap);
 	if (err < 0)
 		goto free_device;
@@ -359,7 +358,7 @@ free_device:
 	return err;
 }
 
-static struct serio_device_id rain_serio_ids[] = {
+static const struct serio_device_id rain_serio_ids[] = {
 	{
 		.type	= SERIO_RS232,
 		.proto	= SERIO_RAINSHADOW_CEC,
diff --git a/drivers/media/usb/s2255/s2255drv.c b/drivers/media/usb/s2255/s2255drv.c
index 6a88b1dbb3a0..b2f239c4ba42 100644
--- a/drivers/media/usb/s2255/s2255drv.c
+++ b/drivers/media/usb/s2255/s2255drv.c
@@ -381,7 +381,7 @@ MODULE_PARM_DESC(jpeg_enable, "Jpeg enable(1-on 0-off) default 1");
 
 /* USB device table */
 #define USB_SENSORAY_VID	0x1943
-static struct usb_device_id s2255_table[] = {
+static const struct usb_device_id s2255_table[] = {
 	{USB_DEVICE(USB_SENSORAY_VID, 0x2255)},
 	{USB_DEVICE(USB_SENSORAY_VID, 0x2257)}, /*same family as 2255*/
 	{ }			/* Terminating entry */
@@ -1590,7 +1590,7 @@ static void s2255_video_device_release(struct video_device *vdev)
 	return;
 }
 
-static struct video_device template = {
+static const struct video_device template = {
 	.name = "s2255v",
 	.fops = &s2255_fops_v4l,
 	.ioctl_ops = &s2255_ioctl_ops,
diff --git a/drivers/media/usb/stk1160/stk1160-core.c b/drivers/media/usb/stk1160/stk1160-core.c
index c86eb6164713..bea8bbbb84fb 100644
--- a/drivers/media/usb/stk1160/stk1160-core.c
+++ b/drivers/media/usb/stk1160/stk1160-core.c
@@ -47,7 +47,7 @@ MODULE_AUTHOR("Ezequiel Garcia");
 MODULE_DESCRIPTION("STK1160 driver");
 
 /* Devices supported by this driver */
-static struct usb_device_id stk1160_id_table[] = {
+static const struct usb_device_id stk1160_id_table[] = {
 	{ USB_DEVICE(0x05e1, 0x0408) },
 	{ }
 };
diff --git a/drivers/media/usb/stk1160/stk1160-i2c.c b/drivers/media/usb/stk1160/stk1160-i2c.c
index 3f2517be02bb..2c70173e3c82 100644
--- a/drivers/media/usb/stk1160/stk1160-i2c.c
+++ b/drivers/media/usb/stk1160/stk1160-i2c.c
@@ -240,7 +240,7 @@ static const struct i2c_algorithm algo = {
 	.functionality = functionality,
 };
 
-static struct i2c_adapter adap_template = {
+static const struct i2c_adapter adap_template = {
 	.owner = THIS_MODULE,
 	.name = "stk1160",
 	.algo = &algo,
diff --git a/drivers/media/usb/stk1160/stk1160-v4l.c b/drivers/media/usb/stk1160/stk1160-v4l.c
index a005d262392a..77b759a0bcd9 100644
--- a/drivers/media/usb/stk1160/stk1160-v4l.c
+++ b/drivers/media/usb/stk1160/stk1160-v4l.c
@@ -326,7 +326,7 @@ static int stk1160_stop_streaming(struct stk1160 *dev)
 	return 0;
 }
 
-static struct v4l2_file_operations stk1160_fops = {
+static const struct v4l2_file_operations stk1160_fops = {
 	.owner = THIS_MODULE,
 	.open = v4l2_fh_open,
 	.release = vb2_fop_release,
@@ -751,7 +751,7 @@ static const struct vb2_ops stk1160_video_qops = {
 	.wait_finish		= vb2_ops_wait_finish,
 };
 
-static struct video_device v4l_template = {
+static const struct video_device v4l_template = {
 	.name = "stk1160",
 	.tvnorms = V4L2_STD_525_60 | V4L2_STD_625_50,
 	.fops = &stk1160_fops,
diff --git a/drivers/media/usb/stkwebcam/stk-webcam.c b/drivers/media/usb/stkwebcam/stk-webcam.c
index 90d4a08cda31..c0bba773db25 100644
--- a/drivers/media/usb/stkwebcam/stk-webcam.c
+++ b/drivers/media/usb/stkwebcam/stk-webcam.c
@@ -55,7 +55,7 @@ MODULE_AUTHOR("Jaime Velasco Juan <jsagarribay@gmail.com> and Nicolas VIVIEN");
 MODULE_DESCRIPTION("Syntek DC1125 webcam driver");
 
 /* Some cameras have audio interfaces, we aren't interested in those */
-static struct usb_device_id stkwebcam_table[] = {
+static const struct usb_device_id stkwebcam_table[] = {
 	{ USB_DEVICE_AND_INTERFACE_INFO(0x174f, 0xa311, 0xff, 0xff, 0xff) },
 	{ USB_DEVICE_AND_INTERFACE_INFO(0x05e1, 0x0501, 0xff, 0xff, 0xff) },
 	{ }
@@ -1202,7 +1202,7 @@ static const struct v4l2_ctrl_ops stk_ctrl_ops = {
 	.s_ctrl = stk_s_ctrl,
 };
 
-static struct v4l2_file_operations v4l_stk_fops = {
+static const struct v4l2_file_operations v4l_stk_fops = {
 	.owner = THIS_MODULE,
 	.open = v4l_stk_open,
 	.release = v4l_stk_release,
@@ -1244,7 +1244,7 @@ static void stk_v4l_dev_release(struct video_device *vd)
 	kfree(dev);
 }
 
-static struct video_device stk_v4l_data = {
+static const struct video_device stk_v4l_data = {
 	.name = "stkwebcam",
 	.fops = &v4l_stk_fops,
 	.ioctl_ops = &v4l_stk_ioctl_ops,
diff --git a/drivers/media/usb/tm6000/tm6000-alsa.c b/drivers/media/usb/tm6000/tm6000-alsa.c
index 422322541af6..3717a6844ea8 100644
--- a/drivers/media/usb/tm6000/tm6000-alsa.c
+++ b/drivers/media/usb/tm6000/tm6000-alsa.c
@@ -143,7 +143,7 @@ static int dsp_buffer_alloc(struct snd_pcm_substream *substream, int size)
  */
 #define DEFAULT_FIFO_SIZE	4096
 
-static struct snd_pcm_hardware snd_tm6000_digital_hw = {
+static const struct snd_pcm_hardware snd_tm6000_digital_hw = {
 	.info = SNDRV_PCM_INFO_BATCH |
 		SNDRV_PCM_INFO_MMAP |
 		SNDRV_PCM_INFO_INTERLEAVED |
diff --git a/drivers/media/usb/tm6000/tm6000-cards.c b/drivers/media/usb/tm6000/tm6000-cards.c
index b293dea6554f..2537643a1808 100644
--- a/drivers/media/usb/tm6000/tm6000-cards.c
+++ b/drivers/media/usb/tm6000/tm6000-cards.c
@@ -613,7 +613,7 @@ static struct tm6000_board tm6000_boards[] = {
 };
 
 /* table of devices that work with this driver */
-static struct usb_device_id tm6000_id_table[] = {
+static const struct usb_device_id tm6000_id_table[] = {
 	{ USB_DEVICE(0x6000, 0x0001), .driver_info = TM5600_BOARD_GENERIC },
 	{ USB_DEVICE(0x6000, 0x0002), .driver_info = TM6010_BOARD_GENERIC },
 	{ USB_DEVICE(0x06e1, 0xf332), .driver_info = TM6000_BOARD_ADSTECH_DUAL_TV },
diff --git a/drivers/media/usb/tm6000/tm6000-input.c b/drivers/media/usb/tm6000/tm6000-input.c
index 1a033f57fcc1..91889ad9cdd7 100644
--- a/drivers/media/usb/tm6000/tm6000-input.c
+++ b/drivers/media/usb/tm6000/tm6000-input.c
@@ -66,7 +66,7 @@ struct tm6000_IR {
 	struct urb		*int_urb;
 
 	/* IR device properties */
-	u64			rc_type;
+	u64			rc_proto;
 };
 
 void tm6000_ir_wait(struct tm6000_core *dev, u8 state)
@@ -103,13 +103,13 @@ static int tm6000_ir_config(struct tm6000_IR *ir)
 	 * IR, in order to discard such decoding
 	 */
 
-	switch (ir->rc_type) {
-	case RC_BIT_NEC:
+	switch (ir->rc_proto) {
+	case RC_PROTO_BIT_NEC:
 		leader = 900;	/* ms */
 		pulse  = 700;	/* ms - the actual value would be 562 */
 		break;
 	default:
-	case RC_BIT_RC5:
+	case RC_PROTO_BIT_RC5:
 		leader = 900;	/* ms - from the NEC decoding */
 		pulse  = 1780;	/* ms - The actual value would be 1776 */
 		break;
@@ -117,12 +117,12 @@ static int tm6000_ir_config(struct tm6000_IR *ir)
 
 	pulse = ir_clock_mhz * pulse;
 	leader = ir_clock_mhz * leader;
-	if (ir->rc_type == RC_BIT_NEC)
+	if (ir->rc_proto == RC_PROTO_BIT_NEC)
 		leader = leader | 0x8000;
 
 	dprintk(2, "%s: %s, %d MHz, leader = 0x%04x, pulse = 0x%06x \n",
 		__func__,
-		(ir->rc_type == RC_BIT_NEC) ? "NEC" : "RC-5",
+		(ir->rc_proto == RC_PROTO_BIT_NEC) ? "NEC" : "RC-5",
 		ir_clock_mhz, leader, pulse);
 
 	/* Remote WAKEUP = enable, normal mode, from IR decoder output */
@@ -162,24 +162,24 @@ static void tm6000_ir_keydown(struct tm6000_IR *ir,
 {
 	u8 device, command;
 	u32 scancode;
-	enum rc_type protocol;
+	enum rc_proto protocol;
 
 	if (len < 1)
 		return;
 
 	command = buf[0];
 	device = (len > 1 ? buf[1] : 0x0);
-	switch (ir->rc_type) {
-	case RC_BIT_RC5:
-		protocol = RC_TYPE_RC5;
+	switch (ir->rc_proto) {
+	case RC_PROTO_BIT_RC5:
+		protocol = RC_PROTO_RC5;
 		scancode = RC_SCANCODE_RC5(device, command);
 		break;
-	case RC_BIT_NEC:
-		protocol = RC_TYPE_NEC;
+	case RC_PROTO_BIT_NEC:
+		protocol = RC_PROTO_NEC;
 		scancode = RC_SCANCODE_NEC(device, command);
 		break;
 	default:
-		protocol = RC_TYPE_OTHER;
+		protocol = RC_PROTO_OTHER;
 		scancode = RC_SCANCODE_OTHER(device << 8 | command);
 		break;
 	}
@@ -311,7 +311,7 @@ static void tm6000_ir_stop(struct rc_dev *rc)
 	cancel_delayed_work_sync(&ir->work);
 }
 
-static int tm6000_ir_change_protocol(struct rc_dev *rc, u64 *rc_type)
+static int tm6000_ir_change_protocol(struct rc_dev *rc, u64 *rc_proto)
 {
 	struct tm6000_IR *ir = rc->priv;
 
@@ -320,7 +320,7 @@ static int tm6000_ir_change_protocol(struct rc_dev *rc, u64 *rc_type)
 
 	dprintk(2, "%s\n",__func__);
 
-	ir->rc_type = *rc_type;
+	ir->rc_proto = *rc_proto;
 
 	tm6000_ir_config(ir);
 	/* TODO */
@@ -409,7 +409,7 @@ int tm6000_ir_init(struct tm6000_core *dev)
 	struct tm6000_IR *ir;
 	struct rc_dev *rc;
 	int err = -ENOMEM;
-	u64 rc_type;
+	u64 rc_proto;
 
 	if (!enable_ir)
 		return -ENODEV;
@@ -433,7 +433,7 @@ int tm6000_ir_init(struct tm6000_core *dev)
 	ir->rc = rc;
 
 	/* input setup */
-	rc->allowed_protocols = RC_BIT_RC5 | RC_BIT_NEC;
+	rc->allowed_protocols = RC_PROTO_BIT_RC5 | RC_PROTO_BIT_NEC;
 	/* Needed, in order to support NEC remotes with 24 or 32 bits */
 	rc->scancode_mask = 0xffff;
 	rc->priv = ir;
@@ -455,10 +455,10 @@ int tm6000_ir_init(struct tm6000_core *dev)
 	usb_make_path(dev->udev, ir->phys, sizeof(ir->phys));
 	strlcat(ir->phys, "/input0", sizeof(ir->phys));
 
-	rc_type = RC_BIT_UNKNOWN;
-	tm6000_ir_change_protocol(rc, &rc_type);
+	rc_proto = RC_PROTO_BIT_UNKNOWN;
+	tm6000_ir_change_protocol(rc, &rc_proto);
 
-	rc->input_name = ir->name;
+	rc->device_name = ir->name;
 	rc->input_phys = ir->phys;
 	rc->input_id.bustype = BUS_USB;
 	rc->input_id.version = 1;
diff --git a/drivers/media/usb/tm6000/tm6000-video.c b/drivers/media/usb/tm6000/tm6000-video.c
index 7e960d0a5b92..ec8c4d2534dc 100644
--- a/drivers/media/usb/tm6000/tm6000-video.c
+++ b/drivers/media/usb/tm6000/tm6000-video.c
@@ -801,7 +801,7 @@ static void buffer_release(struct videobuf_queue *vq, struct videobuf_buffer *vb
 	free_buffer(vq, buf);
 }
 
-static struct videobuf_queue_ops tm6000_video_qops = {
+static const struct videobuf_queue_ops tm6000_video_qops = {
 	.buf_setup      = buffer_setup,
 	.buf_prepare    = buffer_prepare,
 	.buf_queue      = buffer_queue,
@@ -1532,7 +1532,7 @@ static int tm6000_mmap(struct file *file, struct vm_area_struct * vma)
 	return res;
 }
 
-static struct v4l2_file_operations tm6000_fops = {
+static const struct v4l2_file_operations tm6000_fops = {
 	.owner = THIS_MODULE,
 	.open = tm6000_open,
 	.release = tm6000_release,
diff --git a/drivers/media/usb/ttusb-budget/dvb-ttusb-budget.c b/drivers/media/usb/ttusb-budget/dvb-ttusb-budget.c
index 361e40b56045..b842f367249f 100644
--- a/drivers/media/usb/ttusb-budget/dvb-ttusb-budget.c
+++ b/drivers/media/usb/ttusb-budget/dvb-ttusb-budget.c
@@ -1640,7 +1640,7 @@ static void frontend_init(struct ttusb* ttusb)
 
 
 
-static struct i2c_algorithm ttusb_dec_algo = {
+static const struct i2c_algorithm ttusb_dec_algo = {
 	.master_xfer	= master_xfer,
 	.functionality	= functionality,
 };
@@ -1795,7 +1795,7 @@ static void ttusb_disconnect(struct usb_interface *intf)
 	dprintk("%s: TTUSB DVB disconnected\n", __func__);
 }
 
-static struct usb_device_id ttusb_table[] = {
+static const struct usb_device_id ttusb_table[] = {
 	{USB_DEVICE(0xb48, 0x1003)},
 	{USB_DEVICE(0xb48, 0x1004)},
 	{USB_DEVICE(0xb48, 0x1005)},
diff --git a/drivers/media/usb/ttusb-dec/ttusb_dec.c b/drivers/media/usb/ttusb-dec/ttusb_dec.c
index 01c7e6d4481c..cdefb5dfbbdc 100644
--- a/drivers/media/usb/ttusb-dec/ttusb_dec.c
+++ b/drivers/media/usb/ttusb-dec/ttusb_dec.c
@@ -1791,7 +1791,7 @@ static void ttusb_dec_set_model(struct ttusb_dec *dec,
 	}
 }
 
-static struct usb_device_id ttusb_dec_table[] = {
+static const struct usb_device_id ttusb_dec_table[] = {
 	{USB_DEVICE(0x0b48, 0x1006)},	/* DEC3000-s */
 	/*{USB_DEVICE(0x0b48, 0x1007)},	   Unconfirmed */
 	{USB_DEVICE(0x0b48, 0x1008)},	/* DEC2000-t */
diff --git a/drivers/media/usb/usbtv/usbtv-audio.c b/drivers/media/usb/usbtv/usbtv-audio.c
index 9db31db7d9ac..2c2ca77fa01f 100644
--- a/drivers/media/usb/usbtv/usbtv-audio.c
+++ b/drivers/media/usb/usbtv/usbtv-audio.c
@@ -43,7 +43,7 @@
 
 #include "usbtv.h"
 
-static struct snd_pcm_hardware snd_usbtv_digital_hw = {
+static const struct snd_pcm_hardware snd_usbtv_digital_hw = {
 	.info = SNDRV_PCM_INFO_BATCH |
 		SNDRV_PCM_INFO_MMAP |
 		SNDRV_PCM_INFO_INTERLEAVED |
diff --git a/drivers/media/usb/usbtv/usbtv-core.c b/drivers/media/usb/usbtv/usbtv-core.c
index ceb953be0770..f06f09a0876e 100644
--- a/drivers/media/usb/usbtv/usbtv-core.c
+++ b/drivers/media/usb/usbtv/usbtv-core.c
@@ -142,7 +142,7 @@ static void usbtv_disconnect(struct usb_interface *intf)
 	v4l2_device_put(&usbtv->v4l2_dev);
 }
 
-static struct usb_device_id usbtv_id_table[] = {
+static const struct usb_device_id usbtv_id_table[] = {
 	{ USB_DEVICE(0x1b71, 0x3002) },
 	{}
 };
diff --git a/drivers/media/usb/usbtv/usbtv-video.c b/drivers/media/usb/usbtv/usbtv-video.c
index 8135614f395a..95b5f4319ec2 100644
--- a/drivers/media/usb/usbtv/usbtv-video.c
+++ b/drivers/media/usb/usbtv/usbtv-video.c
@@ -629,7 +629,7 @@ static struct v4l2_ioctl_ops usbtv_ioctl_ops = {
 	.vidioc_streamoff = vb2_ioctl_streamoff,
 };
 
-static struct v4l2_file_operations usbtv_fops = {
+static const struct v4l2_file_operations usbtv_fops = {
 	.owner = THIS_MODULE,
 	.unlocked_ioctl = video_ioctl2,
 	.mmap = vb2_fop_mmap,
diff --git a/drivers/media/usb/usbvision/usbvision-i2c.c b/drivers/media/usb/usbvision/usbvision-i2c.c
index fdf6b6e285da..837bd4d9db41 100644
--- a/drivers/media/usb/usbvision/usbvision-i2c.c
+++ b/drivers/media/usb/usbvision/usbvision-i2c.c
@@ -163,7 +163,7 @@ static u32 functionality(struct i2c_adapter *adap)
 
 /* -----exported algorithm data: -------------------------------------	*/
 
-static struct i2c_algorithm usbvision_algo = {
+static const struct i2c_algorithm usbvision_algo = {
 	.master_xfer   = usbvision_i2c_xfer,
 	.smbus_xfer    = NULL,
 	.functionality = functionality,
@@ -173,7 +173,7 @@ static struct i2c_algorithm usbvision_algo = {
 /* ----------------------------------------------------------------------- */
 /* usbvision specific I2C functions                                        */
 /* ----------------------------------------------------------------------- */
-static struct i2c_adapter i2c_adap_template;
+static const struct i2c_adapter i2c_adap_template;
 
 int usbvision_i2c_register(struct usb_usbvision *usbvision)
 {
@@ -187,8 +187,9 @@ int usbvision_i2c_register(struct usb_usbvision *usbvision)
 
 	usbvision->i2c_adap = i2c_adap_template;
 
-	sprintf(usbvision->i2c_adap.name, "%s-%d-%s", i2c_adap_template.name,
-		usbvision->dev->bus->busnum, usbvision->dev->devpath);
+	snprintf(usbvision->i2c_adap.name, sizeof(usbvision->i2c_adap.name),
+		 "usbvision-%d-%s",
+		 usbvision->dev->bus->busnum, usbvision->dev->devpath);
 	PDEBUG(DBG_I2C, "Adaptername: %s", usbvision->i2c_adap.name);
 	usbvision->i2c_adap.dev.parent = &usbvision->dev->dev;
 
@@ -440,7 +441,7 @@ static int usbvision_i2c_read(struct usb_usbvision *usbvision, unsigned char add
 	return rdcount;
 }
 
-static struct i2c_adapter i2c_adap_template = {
+static const struct i2c_adapter i2c_adap_template = {
 	.owner = THIS_MODULE,
 	.name              = "usbvision",
 };
diff --git a/drivers/media/usb/usbvision/usbvision-video.c b/drivers/media/usb/usbvision/usbvision-video.c
index 756322c4ac05..960272d3c924 100644
--- a/drivers/media/usb/usbvision/usbvision-video.c
+++ b/drivers/media/usb/usbvision/usbvision-video.c
@@ -904,7 +904,7 @@ static ssize_t usbvision_read(struct file *file, char __user *buf,
 	PDEBUG(DBG_IO, "%s: %ld bytes, noblock=%d", __func__,
 	       (unsigned long)count, noblock);
 
-	if (!USBVISION_IS_OPERATIONAL(usbvision) || (buf == NULL))
+	if (!USBVISION_IS_OPERATIONAL(usbvision) || !buf)
 		return -EFAULT;
 
 	/* This entry point is compatible with the mmap routines
@@ -1234,7 +1234,7 @@ static void usbvision_vdev_init(struct usb_usbvision *usbvision,
 {
 	struct usb_device *usb_dev = usbvision->dev;
 
-	if (usb_dev == NULL) {
+	if (!usb_dev) {
 		dev_err(&usbvision->dev->dev,
 			"%s: usbvision->dev is not set\n", __func__);
 		return;
@@ -1319,8 +1319,8 @@ static struct usb_usbvision *usbvision_alloc(struct usb_device *dev,
 {
 	struct usb_usbvision *usbvision;
 
-	usbvision = kzalloc(sizeof(struct usb_usbvision), GFP_KERNEL);
-	if (usbvision == NULL)
+	usbvision = kzalloc(sizeof(*usbvision), GFP_KERNEL);
+	if (!usbvision)
 		return NULL;
 
 	usbvision->dev = dev;
@@ -1334,7 +1334,7 @@ static struct usb_usbvision *usbvision_alloc(struct usb_device *dev,
 
 	/* prepare control urb for control messages during interrupts */
 	usbvision->ctrl_urb = usb_alloc_urb(USBVISION_URB_FRAMES, GFP_KERNEL);
-	if (usbvision->ctrl_urb == NULL)
+	if (!usbvision->ctrl_urb)
 		goto err_unreg;
 
 	return usbvision;
@@ -1380,7 +1380,7 @@ static void usbvision_configure_video(struct usb_usbvision *usbvision)
 {
 	int model;
 
-	if (usbvision == NULL)
+	if (!usbvision)
 		return;
 
 	model = usbvision->dev_model;
@@ -1474,7 +1474,7 @@ static int usbvision_probe(struct usb_interface *intf,
 	}
 
 	usbvision = usbvision_alloc(dev, intf);
-	if (usbvision == NULL) {
+	if (!usbvision) {
 		dev_err(&intf->dev, "%s: couldn't allocate USBVision struct\n", __func__);
 		ret = -ENOMEM;
 		goto err_usb;
@@ -1494,8 +1494,7 @@ static int usbvision_probe(struct usb_interface *intf,
 	usbvision->num_alt = uif->num_altsetting;
 	PDEBUG(DBG_PROBE, "Alternate settings: %i", usbvision->num_alt);
 	usbvision->alt_max_pkt_size = kmalloc(32 * usbvision->num_alt, GFP_KERNEL);
-	if (usbvision->alt_max_pkt_size == NULL) {
-		dev_err(&intf->dev, "usbvision: out of memory!\n");
+	if (!usbvision->alt_max_pkt_size) {
 		ret = -ENOMEM;
 		goto err_pkt;
 	}
@@ -1566,7 +1565,7 @@ static void usbvision_disconnect(struct usb_interface *intf)
 
 	PDEBUG(DBG_PROBE, "");
 
-	if (usbvision == NULL) {
+	if (!usbvision) {
 		pr_err("%s: usb_get_intfdata() failed\n", __func__);
 		return;
 	}
diff --git a/drivers/media/usb/uvc/uvc_ctrl.c b/drivers/media/usb/uvc/uvc_ctrl.c
index c2ee6e39fd0c..20397aba6849 100644
--- a/drivers/media/usb/uvc/uvc_ctrl.c
+++ b/drivers/media/usb/uvc/uvc_ctrl.c
@@ -2002,6 +2002,13 @@ int uvc_ctrl_add_mapping(struct uvc_video_chain *chain,
 		goto done;
 	}
 
+	/* Validate the user-provided bit-size and offset */
+	if (mapping->size > 32 ||
+	    mapping->offset + mapping->size > ctrl->info.size * 8) {
+		ret = -EINVAL;
+		goto done;
+	}
+
 	list_for_each_entry(map, &ctrl->info.mappings, list) {
 		if (mapping->id == map->id) {
 			uvc_trace(UVC_TRACE_CONTROL, "Can't add mapping '%s', "
diff --git a/drivers/media/usb/uvc/uvc_driver.c b/drivers/media/usb/uvc/uvc_driver.c
index 70842c5af05b..6d22b22cb35b 100644
--- a/drivers/media/usb/uvc/uvc_driver.c
+++ b/drivers/media/usb/uvc/uvc_driver.c
@@ -1802,8 +1802,9 @@ static int uvc_scan_device(struct uvc_device *dev)
  * already been canceled by the USB core. There is no need to kill the
  * interrupt URB manually.
  */
-static void uvc_delete(struct uvc_device *dev)
+static void uvc_delete(struct kref *kref)
 {
+	struct uvc_device *dev = container_of(kref, struct uvc_device, ref);
 	struct list_head *p, *n;
 
 	uvc_status_cleanup(dev);
@@ -1854,11 +1855,7 @@ static void uvc_release(struct video_device *vdev)
 	struct uvc_streaming *stream = video_get_drvdata(vdev);
 	struct uvc_device *dev = stream->dev;
 
-	/* Decrement the registered streams count and delete the device when it
-	 * reaches zero.
-	 */
-	if (atomic_dec_and_test(&dev->nstreams))
-		uvc_delete(dev);
+	kref_put(&dev->ref, uvc_delete);
 }
 
 /*
@@ -1870,10 +1867,10 @@ static void uvc_unregister_video(struct uvc_device *dev)
 
 	/* Unregistering all video devices might result in uvc_delete() being
 	 * called from inside the loop if there's no open file handle. To avoid
-	 * that, increment the stream count before iterating over the streams
-	 * and decrement it when done.
+	 * that, increment the refcount before iterating over the streams and
+	 * decrement it when done.
 	 */
-	atomic_inc(&dev->nstreams);
+	kref_get(&dev->ref);
 
 	list_for_each_entry(stream, &dev->streams, list) {
 		if (!video_is_registered(&stream->vdev))
@@ -1884,11 +1881,7 @@ static void uvc_unregister_video(struct uvc_device *dev)
 		uvc_debugfs_cleanup_stream(stream);
 	}
 
-	/* Decrement the stream count and call uvc_delete explicitly if there
-	 * are no stream left.
-	 */
-	if (atomic_dec_and_test(&dev->nstreams))
-		uvc_delete(dev);
+	kref_put(&dev->ref, uvc_delete);
 }
 
 static int uvc_register_video(struct uvc_device *dev,
@@ -1946,7 +1939,7 @@ static int uvc_register_video(struct uvc_device *dev,
 	else
 		stream->chain->caps |= V4L2_CAP_VIDEO_OUTPUT;
 
-	atomic_inc(&dev->nstreams);
+	kref_get(&dev->ref);
 	return 0;
 }
 
@@ -2031,7 +2024,7 @@ static int uvc_probe(struct usb_interface *intf,
 	INIT_LIST_HEAD(&dev->entities);
 	INIT_LIST_HEAD(&dev->chains);
 	INIT_LIST_HEAD(&dev->streams);
-	atomic_set(&dev->nstreams, 0);
+	kref_init(&dev->ref);
 	atomic_set(&dev->nmappings, 0);
 	mutex_init(&dev->lock);
 
@@ -2096,7 +2089,6 @@ static int uvc_probe(struct usb_interface *intf,
 			sizeof(dev->mdev.serial));
 	strcpy(dev->mdev.bus_info, udev->devpath);
 	dev->mdev.hw_revision = le16_to_cpu(udev->descriptor.bcdDevice);
-	dev->mdev.driver_version = LINUX_VERSION_CODE;
 	media_device_init(&dev->mdev);
 
 	dev->vdev.mdev = &dev->mdev;
@@ -2284,7 +2276,7 @@ MODULE_PARM_DESC(timeout, "Streaming control requests timeout");
  * VENDOR_SPEC because they don't announce themselves as UVC devices, even
  * though they are compliant.
  */
-static struct usb_device_id uvc_ids[] = {
+static const struct usb_device_id uvc_ids[] = {
 	/* LogiLink Wireless Webcam */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
diff --git a/drivers/media/usb/uvc/uvc_entity.c b/drivers/media/usb/uvc/uvc_entity.c
index ac386bb547e6..554063c07d7a 100644
--- a/drivers/media/usb/uvc/uvc_entity.c
+++ b/drivers/media/usb/uvc/uvc_entity.c
@@ -61,7 +61,7 @@ static int uvc_mc_create_links(struct uvc_video_chain *chain,
 	return 0;
 }
 
-static struct v4l2_subdev_ops uvc_subdev_ops = {
+static const struct v4l2_subdev_ops uvc_subdev_ops = {
 };
 
 void uvc_mc_cleanup_entity(struct uvc_entity *entity)
diff --git a/drivers/media/usb/uvc/uvc_queue.c b/drivers/media/usb/uvc/uvc_queue.c
index aa2199775cb8..c8d78b2f3de4 100644
--- a/drivers/media/usb/uvc/uvc_queue.c
+++ b/drivers/media/usb/uvc/uvc_queue.c
@@ -82,9 +82,14 @@ static int uvc_queue_setup(struct vb2_queue *vq,
 	struct uvc_streaming *stream = uvc_queue_to_stream(queue);
 	unsigned size = stream->ctrl.dwMaxVideoFrameSize;
 
-	/* Make sure the image size is large enough. */
+	/*
+	 * When called with plane sizes, validate them. The driver supports
+	 * single planar formats only, and requires buffers to be large enough
+	 * to store a complete frame.
+	 */
 	if (*nplanes)
-		return sizes[0] < size ? -EINVAL : 0;
+		return *nplanes != 1 || sizes[0] < size ? -EINVAL : 0;
+
 	*nplanes = 1;
 	sizes[0] = size;
 	return 0;
diff --git a/drivers/media/usb/uvc/uvcvideo.h b/drivers/media/usb/uvc/uvcvideo.h
index 15e415e32c7f..34c7ee6cc9e5 100644
--- a/drivers/media/usb/uvc/uvcvideo.h
+++ b/drivers/media/usb/uvc/uvcvideo.h
@@ -166,7 +166,7 @@
 /* Maximum status buffer size in bytes of interrupt URB. */
 #define UVC_MAX_STATUS_SIZE	16
 
-#define UVC_CTRL_CONTROL_TIMEOUT	300
+#define UVC_CTRL_CONTROL_TIMEOUT	500
 #define UVC_CTRL_STREAMING_TIMEOUT	5000
 
 /* Maximum allowed number of control mappings per device */
@@ -575,7 +575,7 @@ struct uvc_device {
 
 	/* Video Streaming interfaces */
 	struct list_head streams;
-	atomic_t nstreams;
+	struct kref ref;
 
 	/* Status Interrupt Endpoint */
 	struct usb_host_endpoint *int_ep;
diff --git a/drivers/media/usb/zr364xx/zr364xx.c b/drivers/media/usb/zr364xx/zr364xx.c
index efdcd5bd6a4c..4ff8d0aed015 100644
--- a/drivers/media/usb/zr364xx/zr364xx.c
+++ b/drivers/media/usb/zr364xx/zr364xx.c
@@ -93,7 +93,7 @@ MODULE_PARM_DESC(mode, "0 = 320x240, 1 = 160x120, 2 = 640x480");
 
 /* Devices supported by this driver
  * .driver_info contains the init method used by the camera */
-static struct usb_device_id device_table[] = {
+static const struct usb_device_id device_table[] = {
 	{USB_DEVICE(0x08ca, 0x0109), .driver_info = METHOD0 },
 	{USB_DEVICE(0x041e, 0x4024), .driver_info = METHOD0 },
 	{USB_DEVICE(0x0d64, 0x0108), .driver_info = METHOD0 },
@@ -439,7 +439,7 @@ static void buffer_release(struct videobuf_queue *vq,
 	free_buffer(vq, buf);
 }
 
-static struct videobuf_queue_ops zr364xx_video_qops = {
+static const struct videobuf_queue_ops zr364xx_video_qops = {
 	.buf_setup = buffer_setup,
 	.buf_prepare = buffer_prepare,
 	.buf_queue = buffer_queue,
@@ -1335,7 +1335,7 @@ static const struct v4l2_ioctl_ops zr364xx_ioctl_ops = {
 	.vidioc_unsubscribe_event = v4l2_event_unsubscribe,
 };
 
-static struct video_device zr364xx_template = {
+static const struct video_device zr364xx_template = {
 	.name = DRIVER_DESC,
 	.fops = &zr364xx_fops,
 	.ioctl_ops = &zr364xx_ioctl_ops,
diff --git a/drivers/media/v4l2-core/v4l2-async.c b/drivers/media/v4l2-core/v4l2-async.c
index 851f128eba22..d741a8e0fdac 100644
--- a/drivers/media/v4l2-core/v4l2-async.c
+++ b/drivers/media/v4l2-core/v4l2-async.c
@@ -44,12 +44,7 @@ static bool match_devname(struct v4l2_subdev *sd,
 
 static bool match_fwnode(struct v4l2_subdev *sd, struct v4l2_async_subdev *asd)
 {
-	if (!is_of_node(sd->fwnode) || !is_of_node(asd->match.fwnode.fwnode))
-		return sd->fwnode == asd->match.fwnode.fwnode;
-
-	return !of_node_cmp(of_node_full_name(to_of_node(sd->fwnode)),
-			    of_node_full_name(
-				    to_of_node(asd->match.fwnode.fwnode)));
+	return sd->fwnode == asd->match.fwnode.fwnode;
 }
 
 static bool match_custom(struct v4l2_subdev *sd, struct v4l2_async_subdev *asd)
diff --git a/drivers/media/v4l2-core/v4l2-clk.c b/drivers/media/v4l2-core/v4l2-clk.c
index 297e10e69898..90628d7a04de 100644
--- a/drivers/media/v4l2-core/v4l2-clk.c
+++ b/drivers/media/v4l2-core/v4l2-clk.c
@@ -61,8 +61,7 @@ struct v4l2_clk *v4l2_clk_get(struct device *dev, const char *id)
 
 	/* if dev_name is not found, try use the OF name to find again  */
 	if (PTR_ERR(clk) == -ENODEV && dev->of_node) {
-		v4l2_clk_name_of(clk_name, sizeof(clk_name),
-				 of_node_full_name(dev->of_node));
+		v4l2_clk_name_of(clk_name, sizeof(clk_name), dev->of_node);
 		clk = v4l2_clk_find(clk_name);
 	}
 
diff --git a/drivers/media/v4l2-core/v4l2-compat-ioctl32.c b/drivers/media/v4l2-core/v4l2-compat-ioctl32.c
index 6f52970f8b54..821f2aa299ae 100644
--- a/drivers/media/v4l2-core/v4l2-compat-ioctl32.c
+++ b/drivers/media/v4l2-core/v4l2-compat-ioctl32.c
@@ -43,6 +43,7 @@ struct v4l2_window32 {
 	compat_caddr_t		clips; /* actually struct v4l2_clip32 * */
 	__u32			clipcount;
 	compat_caddr_t		bitmap;
+	__u8                    global_alpha;
 };
 
 static int get_v4l2_window32(struct v4l2_window *kp, struct v4l2_window32 __user *up)
@@ -51,7 +52,8 @@ static int get_v4l2_window32(struct v4l2_window *kp, struct v4l2_window32 __user
 		copy_from_user(&kp->w, &up->w, sizeof(up->w)) ||
 		get_user(kp->field, &up->field) ||
 		get_user(kp->chromakey, &up->chromakey) ||
-		get_user(kp->clipcount, &up->clipcount))
+		get_user(kp->clipcount, &up->clipcount) ||
+		get_user(kp->global_alpha, &up->global_alpha))
 			return -EFAULT;
 	if (kp->clipcount > 2048)
 		return -EINVAL;
@@ -84,7 +86,8 @@ static int put_v4l2_window32(struct v4l2_window *kp, struct v4l2_window32 __user
 	if (copy_to_user(&up->w, &kp->w, sizeof(kp->w)) ||
 		put_user(kp->field, &up->field) ||
 		put_user(kp->chromakey, &up->chromakey) ||
-		put_user(kp->clipcount, &up->clipcount))
+		put_user(kp->clipcount, &up->clipcount) ||
+		put_user(kp->global_alpha, &up->global_alpha))
 			return -EFAULT;
 	return 0;
 }
@@ -627,7 +630,8 @@ struct v4l2_input32 {
 	__u32        tuner;             /*  Associated tuner */
 	compat_u64   std;
 	__u32	     status;
-	__u32	     reserved[4];
+	__u32	     capabilities;
+	__u32	     reserved[3];
 };
 
 /* The 64-bit v4l2_input struct has extra padding at the end of the struct.
@@ -796,7 +800,8 @@ static int put_v4l2_event32(struct v4l2_event *kp, struct v4l2_event32 __user *u
 		copy_to_user(&up->u, &kp->u, sizeof(kp->u)) ||
 		put_user(kp->pending, &up->pending) ||
 		put_user(kp->sequence, &up->sequence) ||
-		compat_put_timespec(&kp->timestamp, &up->timestamp) ||
+		put_user(kp->timestamp.tv_sec, &up->timestamp.tv_sec) ||
+		put_user(kp->timestamp.tv_nsec, &up->timestamp.tv_nsec) ||
 		put_user(kp->id, &up->id) ||
 		copy_to_user(up->reserved, kp->reserved, 8 * sizeof(__u32)))
 			return -EFAULT;
diff --git a/drivers/media/v4l2-core/v4l2-flash-led-class.c b/drivers/media/v4l2-core/v4l2-flash-led-class.c
index 7b8288108e8a..4ceef217de83 100644
--- a/drivers/media/v4l2-core/v4l2-flash-led-class.c
+++ b/drivers/media/v4l2-core/v4l2-flash-led-class.c
@@ -18,7 +18,7 @@
 #include <media/v4l2-flash-led-class.h>
 
 #define has_flash_op(v4l2_flash, op)				\
-	(v4l2_flash && v4l2_flash->ops->op)
+	(v4l2_flash && v4l2_flash->ops && v4l2_flash->ops->op)
 
 #define call_flash_op(v4l2_flash, op, arg)			\
 		(has_flash_op(v4l2_flash, op) ?			\
@@ -110,7 +110,7 @@ static void v4l2_flash_set_led_brightness(struct v4l2_flash *v4l2_flash,
 		led_set_brightness_sync(&v4l2_flash->fled_cdev->led_cdev,
 					brightness);
 	} else {
-		led_set_brightness_sync(&v4l2_flash->iled_cdev->led_cdev,
+		led_set_brightness_sync(v4l2_flash->iled_cdev,
 					brightness);
 	}
 }
@@ -133,7 +133,7 @@ static int v4l2_flash_update_led_brightness(struct v4l2_flash *v4l2_flash,
 			return 0;
 		led_cdev = &v4l2_flash->fled_cdev->led_cdev;
 	} else {
-		led_cdev = &v4l2_flash->iled_cdev->led_cdev;
+		led_cdev = v4l2_flash->iled_cdev;
 	}
 
 	ret = led_update_brightness(led_cdev);
@@ -197,7 +197,7 @@ static int v4l2_flash_s_ctrl(struct v4l2_ctrl *c)
 {
 	struct v4l2_flash *v4l2_flash = v4l2_ctrl_to_v4l2_flash(c);
 	struct led_classdev_flash *fled_cdev = v4l2_flash->fled_cdev;
-	struct led_classdev *led_cdev = &fled_cdev->led_cdev;
+	struct led_classdev *led_cdev = fled_cdev ? &fled_cdev->led_cdev : NULL;
 	struct v4l2_ctrl **ctrls = v4l2_flash->ctrls;
 	bool external_strobe;
 	int ret = 0;
@@ -299,11 +299,26 @@ static void __fill_ctrl_init_data(struct v4l2_flash *v4l2_flash,
 			  struct v4l2_flash_ctrl_data *ctrl_init_data)
 {
 	struct led_classdev_flash *fled_cdev = v4l2_flash->fled_cdev;
-	const struct led_flash_ops *fled_cdev_ops = fled_cdev->ops;
-	struct led_classdev *led_cdev = &fled_cdev->led_cdev;
+	struct led_classdev *led_cdev = fled_cdev ? &fled_cdev->led_cdev : NULL;
 	struct v4l2_ctrl_config *ctrl_cfg;
 	u32 mask;
 
+	/* Init INDICATOR_INTENSITY ctrl data */
+	if (v4l2_flash->iled_cdev) {
+		ctrl_init_data[INDICATOR_INTENSITY].cid =
+					V4L2_CID_FLASH_INDICATOR_INTENSITY;
+		ctrl_cfg = &ctrl_init_data[INDICATOR_INTENSITY].config;
+		__lfs_to_v4l2_ctrl_config(&flash_cfg->intensity,
+					  ctrl_cfg);
+		ctrl_cfg->id = V4L2_CID_FLASH_INDICATOR_INTENSITY;
+		ctrl_cfg->min = 0;
+		ctrl_cfg->flags = V4L2_CTRL_FLAG_VOLATILE |
+				  V4L2_CTRL_FLAG_EXECUTE_ON_WRITE;
+	}
+
+	if (!led_cdev || WARN_ON(!(led_cdev->flags & LED_DEV_CAP_FLASH)))
+		return;
+
 	/* Init FLASH_FAULT ctrl data */
 	if (flash_cfg->flash_faults) {
 		ctrl_init_data[FLASH_FAULT].cid = V4L2_CID_FLASH_FAULT;
@@ -331,27 +346,11 @@ static void __fill_ctrl_init_data(struct v4l2_flash *v4l2_flash,
 	/* Init TORCH_INTENSITY ctrl data */
 	ctrl_init_data[TORCH_INTENSITY].cid = V4L2_CID_FLASH_TORCH_INTENSITY;
 	ctrl_cfg = &ctrl_init_data[TORCH_INTENSITY].config;
-	__lfs_to_v4l2_ctrl_config(&flash_cfg->torch_intensity, ctrl_cfg);
+	__lfs_to_v4l2_ctrl_config(&flash_cfg->intensity, ctrl_cfg);
 	ctrl_cfg->id = V4L2_CID_FLASH_TORCH_INTENSITY;
 	ctrl_cfg->flags = V4L2_CTRL_FLAG_VOLATILE |
 			  V4L2_CTRL_FLAG_EXECUTE_ON_WRITE;
 
-	/* Init INDICATOR_INTENSITY ctrl data */
-	if (v4l2_flash->iled_cdev) {
-		ctrl_init_data[INDICATOR_INTENSITY].cid =
-					V4L2_CID_FLASH_INDICATOR_INTENSITY;
-		ctrl_cfg = &ctrl_init_data[INDICATOR_INTENSITY].config;
-		__lfs_to_v4l2_ctrl_config(&flash_cfg->indicator_intensity,
-					  ctrl_cfg);
-		ctrl_cfg->id = V4L2_CID_FLASH_INDICATOR_INTENSITY;
-		ctrl_cfg->min = 0;
-		ctrl_cfg->flags = V4L2_CTRL_FLAG_VOLATILE |
-				  V4L2_CTRL_FLAG_EXECUTE_ON_WRITE;
-	}
-
-	if (!(led_cdev->flags & LED_DEV_CAP_FLASH))
-		return;
-
 	/* Init FLASH_STROBE ctrl data */
 	ctrl_init_data[FLASH_STROBE].cid = V4L2_CID_FLASH_STROBE;
 	ctrl_cfg = &ctrl_init_data[FLASH_STROBE].config;
@@ -376,7 +375,7 @@ static void __fill_ctrl_init_data(struct v4l2_flash *v4l2_flash,
 	}
 
 	/* Init STROBE_STATUS ctrl data */
-	if (fled_cdev_ops->strobe_get) {
+	if (has_flash_op(fled_cdev, strobe_get)) {
 		ctrl_init_data[STROBE_STATUS].cid =
 					V4L2_CID_FLASH_STROBE_STATUS;
 		ctrl_cfg = &ctrl_init_data[STROBE_STATUS].config;
@@ -386,7 +385,7 @@ static void __fill_ctrl_init_data(struct v4l2_flash *v4l2_flash,
 	}
 
 	/* Init FLASH_TIMEOUT ctrl data */
-	if (fled_cdev_ops->timeout_set) {
+	if (has_flash_op(fled_cdev, timeout_set)) {
 		ctrl_init_data[FLASH_TIMEOUT].cid = V4L2_CID_FLASH_TIMEOUT;
 		ctrl_cfg = &ctrl_init_data[FLASH_TIMEOUT].config;
 		__lfs_to_v4l2_ctrl_config(&fled_cdev->timeout, ctrl_cfg);
@@ -394,7 +393,7 @@ static void __fill_ctrl_init_data(struct v4l2_flash *v4l2_flash,
 	}
 
 	/* Init FLASH_INTENSITY ctrl data */
-	if (fled_cdev_ops->flash_brightness_set) {
+	if (has_flash_op(fled_cdev, flash_brightness_set)) {
 		ctrl_init_data[FLASH_INTENSITY].cid = V4L2_CID_FLASH_INTENSITY;
 		ctrl_cfg = &ctrl_init_data[FLASH_INTENSITY].config;
 		__lfs_to_v4l2_ctrl_config(&fled_cdev->brightness, ctrl_cfg);
@@ -486,7 +485,9 @@ static int __sync_device_with_v4l2_controls(struct v4l2_flash *v4l2_flash)
 	struct v4l2_ctrl **ctrls = v4l2_flash->ctrls;
 	int ret = 0;
 
-	v4l2_flash_set_led_brightness(v4l2_flash, ctrls[TORCH_INTENSITY]);
+	if (ctrls[TORCH_INTENSITY])
+		v4l2_flash_set_led_brightness(v4l2_flash,
+					      ctrls[TORCH_INTENSITY]);
 
 	if (ctrls[INDICATOR_INTENSITY])
 		v4l2_flash_set_led_brightness(v4l2_flash,
@@ -528,24 +529,23 @@ static int v4l2_flash_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
 {
 	struct v4l2_flash *v4l2_flash = v4l2_subdev_to_v4l2_flash(sd);
 	struct led_classdev_flash *fled_cdev = v4l2_flash->fled_cdev;
-	struct led_classdev *led_cdev = &fled_cdev->led_cdev;
-	struct led_classdev_flash *iled_cdev = v4l2_flash->iled_cdev;
-	struct led_classdev *led_cdev_ind = NULL;
+	struct led_classdev *led_cdev = fled_cdev ? &fled_cdev->led_cdev : NULL;
+	struct led_classdev *led_cdev_ind = v4l2_flash->iled_cdev;
 	int ret = 0;
 
 	if (!v4l2_fh_is_singular(&fh->vfh))
 		return 0;
 
-	mutex_lock(&led_cdev->led_access);
-
-	led_sysfs_disable(led_cdev);
-	led_trigger_remove(led_cdev);
+	if (led_cdev) {
+		mutex_lock(&led_cdev->led_access);
 
-	mutex_unlock(&led_cdev->led_access);
+		led_sysfs_disable(led_cdev);
+		led_trigger_remove(led_cdev);
 
-	if (iled_cdev) {
-		led_cdev_ind = &iled_cdev->led_cdev;
+		mutex_unlock(&led_cdev->led_access);
+	}
 
+	if (led_cdev_ind) {
 		mutex_lock(&led_cdev_ind->led_access);
 
 		led_sysfs_disable(led_cdev_ind);
@@ -560,9 +560,11 @@ static int v4l2_flash_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
 
 	return 0;
 out_sync_device:
-	mutex_lock(&led_cdev->led_access);
-	led_sysfs_enable(led_cdev);
-	mutex_unlock(&led_cdev->led_access);
+	if (led_cdev) {
+		mutex_lock(&led_cdev->led_access);
+		led_sysfs_enable(led_cdev);
+		mutex_unlock(&led_cdev->led_access);
+	}
 
 	if (led_cdev_ind) {
 		mutex_lock(&led_cdev_ind->led_access);
@@ -577,25 +579,26 @@ static int v4l2_flash_close(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
 {
 	struct v4l2_flash *v4l2_flash = v4l2_subdev_to_v4l2_flash(sd);
 	struct led_classdev_flash *fled_cdev = v4l2_flash->fled_cdev;
-	struct led_classdev *led_cdev = &fled_cdev->led_cdev;
-	struct led_classdev_flash *iled_cdev = v4l2_flash->iled_cdev;
+	struct led_classdev *led_cdev = fled_cdev ? &fled_cdev->led_cdev : NULL;
+	struct led_classdev *led_cdev_ind = v4l2_flash->iled_cdev;
 	int ret = 0;
 
 	if (!v4l2_fh_is_singular(&fh->vfh))
 		return 0;
 
-	mutex_lock(&led_cdev->led_access);
+	if (led_cdev) {
+		mutex_lock(&led_cdev->led_access);
 
-	if (v4l2_flash->ctrls[STROBE_SOURCE])
-		ret = v4l2_ctrl_s_ctrl(v4l2_flash->ctrls[STROBE_SOURCE],
+		if (v4l2_flash->ctrls[STROBE_SOURCE])
+			ret = v4l2_ctrl_s_ctrl(
+				v4l2_flash->ctrls[STROBE_SOURCE],
 				V4L2_FLASH_STROBE_SOURCE_SOFTWARE);
-	led_sysfs_enable(led_cdev);
-
-	mutex_unlock(&led_cdev->led_access);
+		led_sysfs_enable(led_cdev);
 
-	if (iled_cdev) {
-		struct led_classdev *led_cdev_ind = &iled_cdev->led_cdev;
+		mutex_unlock(&led_cdev->led_access);
+	}
 
+	if (led_cdev_ind) {
 		mutex_lock(&led_cdev_ind->led_access);
 		led_sysfs_enable(led_cdev_ind);
 		mutex_unlock(&led_cdev_ind->led_access);
@@ -611,25 +614,19 @@ static const struct v4l2_subdev_internal_ops v4l2_flash_subdev_internal_ops = {
 
 static const struct v4l2_subdev_ops v4l2_flash_subdev_ops;
 
-struct v4l2_flash *v4l2_flash_init(
+static struct v4l2_flash *__v4l2_flash_init(
 	struct device *dev, struct fwnode_handle *fwn,
-	struct led_classdev_flash *fled_cdev,
-	struct led_classdev_flash *iled_cdev,
-	const struct v4l2_flash_ops *ops,
-	struct v4l2_flash_config *config)
+	struct led_classdev_flash *fled_cdev, struct led_classdev *iled_cdev,
+	const struct v4l2_flash_ops *ops, struct v4l2_flash_config *config)
 {
 	struct v4l2_flash *v4l2_flash;
-	struct led_classdev *led_cdev;
 	struct v4l2_subdev *sd;
 	int ret;
 
-	if (!fled_cdev || !ops || !config)
+	if (!config)
 		return ERR_PTR(-EINVAL);
 
-	led_cdev = &fled_cdev->led_cdev;
-
-	v4l2_flash = devm_kzalloc(led_cdev->dev, sizeof(*v4l2_flash),
-					GFP_KERNEL);
+	v4l2_flash = devm_kzalloc(dev, sizeof(*v4l2_flash), GFP_KERNEL);
 	if (!v4l2_flash)
 		return ERR_PTR(-ENOMEM);
 
@@ -638,7 +635,7 @@ struct v4l2_flash *v4l2_flash_init(
 	v4l2_flash->iled_cdev = iled_cdev;
 	v4l2_flash->ops = ops;
 	sd->dev = dev;
-	sd->fwnode = fwn ? fwn : dev_fwnode(led_cdev->dev);
+	sd->fwnode = fwn ? fwn : dev_fwnode(dev);
 	v4l2_subdev_init(sd, &v4l2_flash_subdev_ops);
 	sd->internal_ops = &v4l2_flash_subdev_internal_ops;
 	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
@@ -670,8 +667,26 @@ err_init_controls:
 
 	return ERR_PTR(ret);
 }
+
+struct v4l2_flash *v4l2_flash_init(
+	struct device *dev, struct fwnode_handle *fwn,
+	struct led_classdev_flash *fled_cdev,
+	const struct v4l2_flash_ops *ops,
+	struct v4l2_flash_config *config)
+{
+	return __v4l2_flash_init(dev, fwn, fled_cdev, NULL, ops, config);
+}
 EXPORT_SYMBOL_GPL(v4l2_flash_init);
 
+struct v4l2_flash *v4l2_flash_indicator_init(
+	struct device *dev, struct fwnode_handle *fwn,
+	struct led_classdev *iled_cdev,
+	struct v4l2_flash_config *config)
+{
+	return __v4l2_flash_init(dev, fwn, NULL, iled_cdev, NULL, config);
+}
+EXPORT_SYMBOL_GPL(v4l2_flash_indicator_init);
+
 void v4l2_flash_release(struct v4l2_flash *v4l2_flash)
 {
 	struct v4l2_subdev *sd;
diff --git a/drivers/media/v4l2-core/v4l2-fwnode.c b/drivers/media/v4l2-core/v4l2-fwnode.c
index 153c53ca3925..40b2fbfe8865 100644
--- a/drivers/media/v4l2-core/v4l2-fwnode.c
+++ b/drivers/media/v4l2-core/v4l2-fwnode.c
@@ -28,8 +28,16 @@
 
 #include <media/v4l2-fwnode.h>
 
-static int v4l2_fwnode_endpoint_parse_csi_bus(struct fwnode_handle *fwnode,
-					      struct v4l2_fwnode_endpoint *vep)
+enum v4l2_fwnode_bus_type {
+	V4L2_FWNODE_BUS_TYPE_GUESS = 0,
+	V4L2_FWNODE_BUS_TYPE_CSI2_CPHY,
+	V4L2_FWNODE_BUS_TYPE_CSI1,
+	V4L2_FWNODE_BUS_TYPE_CCP2,
+	NR_OF_V4L2_FWNODE_BUS_TYPE,
+};
+
+static int v4l2_fwnode_endpoint_parse_csi2_bus(struct fwnode_handle *fwnode,
+					       struct v4l2_fwnode_endpoint *vep)
 {
 	struct v4l2_fwnode_bus_mipi_csi2 *bus = &vep->bus.mipi_csi2;
 	bool have_clk_lane = false;
@@ -40,10 +48,10 @@ static int v4l2_fwnode_endpoint_parse_csi_bus(struct fwnode_handle *fwnode,
 
 	rval = fwnode_property_read_u32_array(fwnode, "data-lanes", NULL, 0);
 	if (rval > 0) {
-		u32 array[ARRAY_SIZE(bus->data_lanes)];
+		u32 array[1 + V4L2_FWNODE_CSI2_MAX_DATA_LANES];
 
 		bus->num_data_lanes =
-			min_t(int, ARRAY_SIZE(bus->data_lanes), rval);
+			min_t(int, V4L2_FWNODE_CSI2_MAX_DATA_LANES, rval);
 
 		fwnode_property_read_u32_array(fwnode, "data-lanes", array,
 					       bus->num_data_lanes);
@@ -56,24 +64,25 @@ static int v4l2_fwnode_endpoint_parse_csi_bus(struct fwnode_handle *fwnode,
 
 			bus->data_lanes[i] = array[i];
 		}
-	}
-
-	rval = fwnode_property_read_u32_array(fwnode, "lane-polarities", NULL,
-					      0);
-	if (rval > 0) {
-		u32 array[ARRAY_SIZE(bus->lane_polarities)];
 
-		if (rval < 1 + bus->num_data_lanes /* clock + data */) {
-			pr_warn("too few lane-polarities entries (need %u, got %u)\n",
-				1 + bus->num_data_lanes, rval);
-			return -EINVAL;
+		rval = fwnode_property_read_u32_array(fwnode,
+						      "lane-polarities", NULL,
+						      0);
+		if (rval > 0) {
+			if (rval != 1 + bus->num_data_lanes /* clock+data */) {
+				pr_warn("invalid number of lane-polarities entries (need %u, got %u)\n",
+					1 + bus->num_data_lanes, rval);
+				return -EINVAL;
+			}
+
+			fwnode_property_read_u32_array(fwnode,
+						       "lane-polarities", array,
+						       1 + bus->num_data_lanes);
+
+			for (i = 0; i < 1 + bus->num_data_lanes; i++)
+				bus->lane_polarities[i] = array[i];
 		}
 
-		fwnode_property_read_u32_array(fwnode, "lane-polarities", array,
-					       1 + bus->num_data_lanes);
-
-		for (i = 0; i < 1 + bus->num_data_lanes; i++)
-			bus->lane_polarities[i] = array[i];
 	}
 
 	if (!fwnode_property_read_u32(fwnode, "clock-lanes", &v)) {
@@ -146,6 +155,32 @@ static void v4l2_fwnode_endpoint_parse_parallel_bus(
 
 }
 
+static void
+v4l2_fwnode_endpoint_parse_csi1_bus(struct fwnode_handle *fwnode,
+				    struct v4l2_fwnode_endpoint *vep,
+				    u32 bus_type)
+{
+	struct v4l2_fwnode_bus_mipi_csi1 *bus = &vep->bus.mipi_csi1;
+	u32 v;
+
+	if (!fwnode_property_read_u32(fwnode, "clock-inv", &v))
+		bus->clock_inv = v;
+
+	if (!fwnode_property_read_u32(fwnode, "strobe", &v))
+		bus->strobe = v;
+
+	if (!fwnode_property_read_u32(fwnode, "data-lanes", &v))
+		bus->data_lane = v;
+
+	if (!fwnode_property_read_u32(fwnode, "clock-lanes", &v))
+		bus->clock_lane = v;
+
+	if (bus_type == V4L2_FWNODE_BUS_TYPE_CCP2)
+		vep->bus_type = V4L2_MBUS_CCP2;
+	else
+		vep->bus_type = V4L2_MBUS_CSI1;
+}
+
 /**
  * v4l2_fwnode_endpoint_parse() - parse all fwnode node properties
  * @fwnode: pointer to the endpoint's fwnode handle
@@ -168,6 +203,7 @@ static void v4l2_fwnode_endpoint_parse_parallel_bus(
 int v4l2_fwnode_endpoint_parse(struct fwnode_handle *fwnode,
 			       struct v4l2_fwnode_endpoint *vep)
 {
+	u32 bus_type = 0;
 	int rval;
 
 	fwnode_graph_parse_endpoint(fwnode, &vep->base);
@@ -176,17 +212,30 @@ int v4l2_fwnode_endpoint_parse(struct fwnode_handle *fwnode,
 	memset(&vep->bus_type, 0, sizeof(*vep) -
 	       offsetof(typeof(*vep), bus_type));
 
-	rval = v4l2_fwnode_endpoint_parse_csi_bus(fwnode, vep);
-	if (rval)
-		return rval;
-	/*
-	 * Parse the parallel video bus properties only if none
-	 * of the MIPI CSI-2 specific properties were found.
-	 */
-	if (vep->bus.mipi_csi2.flags == 0)
-		v4l2_fwnode_endpoint_parse_parallel_bus(fwnode, vep);
-
-	return 0;
+	fwnode_property_read_u32(fwnode, "bus-type", &bus_type);
+
+	switch (bus_type) {
+	case V4L2_FWNODE_BUS_TYPE_GUESS:
+		rval = v4l2_fwnode_endpoint_parse_csi2_bus(fwnode, vep);
+		if (rval)
+			return rval;
+		/*
+		 * Parse the parallel video bus properties only if none
+		 * of the MIPI CSI-2 specific properties were found.
+		 */
+		if (vep->bus.mipi_csi2.flags == 0)
+			v4l2_fwnode_endpoint_parse_parallel_bus(fwnode, vep);
+
+		return 0;
+	case V4L2_FWNODE_BUS_TYPE_CCP2:
+	case V4L2_FWNODE_BUS_TYPE_CSI1:
+		v4l2_fwnode_endpoint_parse_csi1_bus(fwnode, vep, bus_type);
+
+		return 0;
+	default:
+		pr_warn("unsupported bus type %u\n", bus_type);
+		return -EINVAL;
+	}
 }
 EXPORT_SYMBOL_GPL(v4l2_fwnode_endpoint_parse);
 
@@ -247,23 +296,23 @@ struct v4l2_fwnode_endpoint *v4l2_fwnode_endpoint_alloc_parse(
 
 	rval = fwnode_property_read_u64_array(fwnode, "link-frequencies",
 					      NULL, 0);
-	if (rval < 0)
-		goto out_err;
-
-	vep->link_frequencies =
-		kmalloc_array(rval, sizeof(*vep->link_frequencies), GFP_KERNEL);
-	if (!vep->link_frequencies) {
-		rval = -ENOMEM;
-		goto out_err;
-	}
+	if (rval > 0) {
+		vep->link_frequencies =
+			kmalloc_array(rval, sizeof(*vep->link_frequencies),
+				      GFP_KERNEL);
+		if (!vep->link_frequencies) {
+			rval = -ENOMEM;
+			goto out_err;
+		}
 
-	vep->nr_of_link_frequencies = rval;
+		vep->nr_of_link_frequencies = rval;
 
-	rval = fwnode_property_read_u64_array(fwnode, "link-frequencies",
-					      vep->link_frequencies,
-					      vep->nr_of_link_frequencies);
-	if (rval < 0)
-		goto out_err;
+		rval = fwnode_property_read_u64_array(
+			fwnode, "link-frequencies", vep->link_frequencies,
+			vep->nr_of_link_frequencies);
+		if (rval < 0)
+			goto out_err;
+	}
 
 	return vep;
 
diff --git a/drivers/media/v4l2-core/v4l2-ioctl.c b/drivers/media/v4l2-core/v4l2-ioctl.c
index cab63bb49c97..b60a6b0841d1 100644
--- a/drivers/media/v4l2-core/v4l2-ioctl.c
+++ b/drivers/media/v4l2-core/v4l2-ioctl.c
@@ -1195,10 +1195,6 @@ static void v4l_fill_fmtdesc(struct v4l2_fmtdesc *fmt)
 	case V4L2_PIX_FMT_SGBRG10:	descr = "10-bit Bayer GBGB/RGRG"; break;
 	case V4L2_PIX_FMT_SGRBG10:	descr = "10-bit Bayer GRGR/BGBG"; break;
 	case V4L2_PIX_FMT_SRGGB10:	descr = "10-bit Bayer RGRG/GBGB"; break;
-	case V4L2_PIX_FMT_SBGGR12:	descr = "12-bit Bayer BGBG/GRGR"; break;
-	case V4L2_PIX_FMT_SGBRG12:	descr = "12-bit Bayer GBGB/RGRG"; break;
-	case V4L2_PIX_FMT_SGRBG12:	descr = "12-bit Bayer GRGR/BGBG"; break;
-	case V4L2_PIX_FMT_SRGGB12:	descr = "12-bit Bayer RGRG/GBGB"; break;
 	case V4L2_PIX_FMT_SBGGR10P:	descr = "10-bit Bayer BGBG/GRGR Packed"; break;
 	case V4L2_PIX_FMT_SGBRG10P:	descr = "10-bit Bayer GBGB/RGRG Packed"; break;
 	case V4L2_PIX_FMT_SGRBG10P:	descr = "10-bit Bayer GRGR/BGBG Packed"; break;
@@ -1211,6 +1207,14 @@ static void v4l_fill_fmtdesc(struct v4l2_fmtdesc *fmt)
 	case V4L2_PIX_FMT_SGBRG10DPCM8:	descr = "8-bit Bayer GBGB/RGRG (DPCM)"; break;
 	case V4L2_PIX_FMT_SGRBG10DPCM8:	descr = "8-bit Bayer GRGR/BGBG (DPCM)"; break;
 	case V4L2_PIX_FMT_SRGGB10DPCM8:	descr = "8-bit Bayer RGRG/GBGB (DPCM)"; break;
+	case V4L2_PIX_FMT_SBGGR12:	descr = "12-bit Bayer BGBG/GRGR"; break;
+	case V4L2_PIX_FMT_SGBRG12:	descr = "12-bit Bayer GBGB/RGRG"; break;
+	case V4L2_PIX_FMT_SGRBG12:	descr = "12-bit Bayer GRGR/BGBG"; break;
+	case V4L2_PIX_FMT_SRGGB12:	descr = "12-bit Bayer RGRG/GBGB"; break;
+	case V4L2_PIX_FMT_SBGGR12P:	descr = "12-bit Bayer BGBG/GRGR Packed"; break;
+	case V4L2_PIX_FMT_SGBRG12P:	descr = "12-bit Bayer GBGB/RGRG Packed"; break;
+	case V4L2_PIX_FMT_SGRBG12P:	descr = "12-bit Bayer GRGR/BGBG Packed"; break;
+	case V4L2_PIX_FMT_SRGGB12P:	descr = "12-bit Bayer RGRG/GBGB Packed"; break;
 	case V4L2_PIX_FMT_SBGGR16:	descr = "16-bit Bayer BGBG/GRGR"; break;
 	case V4L2_PIX_FMT_SGBRG16:	descr = "16-bit Bayer GBGB/RGRG"; break;
 	case V4L2_PIX_FMT_SGRBG16:	descr = "16-bit Bayer GRGR/BGBG"; break;
diff --git a/drivers/media/v4l2-core/videobuf2-core.c b/drivers/media/v4l2-core/videobuf2-core.c
index 14f83cecfa92..cb115ba6a1d2 100644
--- a/drivers/media/v4l2-core/videobuf2-core.c
+++ b/drivers/media/v4l2-core/videobuf2-core.c
@@ -194,8 +194,6 @@ static void __enqueue_in_driver(struct vb2_buffer *vb);
 static int __vb2_buf_mem_alloc(struct vb2_buffer *vb)
 {
 	struct vb2_queue *q = vb->vb2_queue;
-	enum dma_data_direction dma_dir =
-		q->is_output ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
 	void *mem_priv;
 	int plane;
 	int ret = -ENOMEM;
@@ -209,7 +207,7 @@ static int __vb2_buf_mem_alloc(struct vb2_buffer *vb)
 
 		mem_priv = call_ptr_memop(vb, alloc,
 				q->alloc_devs[plane] ? : q->dev,
-				q->dma_attrs, size, dma_dir, q->gfp_flags);
+				q->dma_attrs, size, q->dma_dir, q->gfp_flags);
 		if (IS_ERR_OR_NULL(mem_priv)) {
 			if (mem_priv)
 				ret = PTR_ERR(mem_priv);
@@ -978,8 +976,6 @@ static int __prepare_userptr(struct vb2_buffer *vb, const void *pb)
 	void *mem_priv;
 	unsigned int plane;
 	int ret = 0;
-	enum dma_data_direction dma_dir =
-		q->is_output ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
 	bool reacquired = vb->planes[0].mem_priv == NULL;
 
 	memset(planes, 0, sizeof(planes[0]) * vb->num_planes);
@@ -1030,7 +1026,7 @@ static int __prepare_userptr(struct vb2_buffer *vb, const void *pb)
 		mem_priv = call_ptr_memop(vb, get_userptr,
 				q->alloc_devs[plane] ? : q->dev,
 				planes[plane].m.userptr,
-				planes[plane].length, dma_dir);
+				planes[plane].length, q->dma_dir);
 		if (IS_ERR(mem_priv)) {
 			dprintk(1, "failed acquiring userspace memory for plane %d\n",
 				plane);
@@ -1096,8 +1092,6 @@ static int __prepare_dmabuf(struct vb2_buffer *vb, const void *pb)
 	void *mem_priv;
 	unsigned int plane;
 	int ret = 0;
-	enum dma_data_direction dma_dir =
-		q->is_output ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
 	bool reacquired = vb->planes[0].mem_priv == NULL;
 
 	memset(planes, 0, sizeof(planes[0]) * vb->num_planes);
@@ -1139,7 +1133,7 @@ static int __prepare_dmabuf(struct vb2_buffer *vb, const void *pb)
 			continue;
 		}
 
-		dprintk(1, "buffer for plane %d changed\n", plane);
+		dprintk(3, "buffer for plane %d changed\n", plane);
 
 		if (!reacquired) {
 			reacquired = true;
@@ -1156,7 +1150,7 @@ static int __prepare_dmabuf(struct vb2_buffer *vb, const void *pb)
 		/* Acquire each plane's memory */
 		mem_priv = call_ptr_memop(vb, attach_dmabuf,
 				q->alloc_devs[plane] ? : q->dev,
-				dbuf, planes[plane].length, dma_dir);
+				dbuf, planes[plane].length, q->dma_dir);
 		if (IS_ERR(mem_priv)) {
 			dprintk(1, "failed to attach dmabuf\n");
 			ret = PTR_ERR(mem_priv);
@@ -1298,7 +1292,7 @@ int vb2_core_prepare_buf(struct vb2_queue *q, unsigned int index, void *pb)
 	/* Fill buffer information for the userspace */
 	call_void_bufop(q, fill_user_buffer, vb, pb);
 
-	dprintk(1, "prepare of buffer %d succeeded\n", vb->index);
+	dprintk(2, "prepare of buffer %d succeeded\n", vb->index);
 
 	return ret;
 }
@@ -1428,7 +1422,7 @@ int vb2_core_qbuf(struct vb2_queue *q, unsigned int index, void *pb)
 			return ret;
 	}
 
-	dprintk(1, "qbuf of buffer %d succeeded\n", vb->index);
+	dprintk(2, "qbuf of buffer %d succeeded\n", vb->index);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(vb2_core_qbuf);
@@ -1476,7 +1470,7 @@ static int __vb2_wait_for_done_vb(struct vb2_queue *q, int nonblocking)
 		}
 
 		if (nonblocking) {
-			dprintk(1, "nonblocking and no buffers to dequeue, will not wait\n");
+			dprintk(3, "nonblocking and no buffers to dequeue, will not wait\n");
 			return -EAGAIN;
 		}
 
@@ -1623,7 +1617,7 @@ int vb2_core_dqbuf(struct vb2_queue *q, unsigned int *pindex, void *pb,
 	/* go back to dequeued state */
 	__vb2_dqbuf(vb);
 
-	dprintk(1, "dqbuf of buffer %d, with state %d\n",
+	dprintk(2, "dqbuf of buffer %d, with state %d\n",
 			vb->index, vb->state);
 
 	return 0;
@@ -2003,6 +1997,11 @@ int vb2_core_queue_init(struct vb2_queue *q)
 	if (q->buf_struct_size == 0)
 		q->buf_struct_size = sizeof(struct vb2_buffer);
 
+	if (q->bidirectional)
+		q->dma_dir = DMA_BIDIRECTIONAL;
+	else
+		q->dma_dir = q->is_output ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
+
 	return 0;
 }
 EXPORT_SYMBOL_GPL(vb2_core_queue_init);
diff --git a/drivers/media/v4l2-core/videobuf2-dma-contig.c b/drivers/media/v4l2-core/videobuf2-dma-contig.c
index 4f246d166111..9f389f36566d 100644
--- a/drivers/media/v4l2-core/videobuf2-dma-contig.c
+++ b/drivers/media/v4l2-core/videobuf2-dma-contig.c
@@ -352,7 +352,7 @@ static int vb2_dc_dmabuf_ops_mmap(struct dma_buf *dbuf,
 	return vb2_dc_mmap(dbuf->priv, vma);
 }
 
-static struct dma_buf_ops vb2_dc_dmabuf_ops = {
+static const struct dma_buf_ops vb2_dc_dmabuf_ops = {
 	.attach = vb2_dc_dmabuf_ops_attach,
 	.detach = vb2_dc_dmabuf_ops_detach,
 	.map_dma_buf = vb2_dc_dmabuf_ops_map,
@@ -508,7 +508,8 @@ static void *vb2_dc_get_userptr(struct device *dev, unsigned long vaddr,
 	buf->dma_dir = dma_dir;
 
 	offset = vaddr & ~PAGE_MASK;
-	vec = vb2_create_framevec(vaddr, size, dma_dir == DMA_FROM_DEVICE);
+	vec = vb2_create_framevec(vaddr, size, dma_dir == DMA_FROM_DEVICE ||
+					       dma_dir == DMA_BIDIRECTIONAL);
 	if (IS_ERR(vec)) {
 		ret = PTR_ERR(vec);
 		goto fail_buf;
diff --git a/drivers/media/v4l2-core/videobuf2-dma-sg.c b/drivers/media/v4l2-core/videobuf2-dma-sg.c
index 5defa1f22ca2..6808231a6bdc 100644
--- a/drivers/media/v4l2-core/videobuf2-dma-sg.c
+++ b/drivers/media/v4l2-core/videobuf2-dma-sg.c
@@ -239,7 +239,8 @@ static void *vb2_dma_sg_get_userptr(struct device *dev, unsigned long vaddr,
 	buf->offset = vaddr & ~PAGE_MASK;
 	buf->size = size;
 	buf->dma_sgt = &buf->sg_table;
-	vec = vb2_create_framevec(vaddr, size, buf->dma_dir == DMA_FROM_DEVICE);
+	vec = vb2_create_framevec(vaddr, size, dma_dir == DMA_FROM_DEVICE ||
+					       dma_dir == DMA_BIDIRECTIONAL);
 	if (IS_ERR(vec))
 		goto userptr_fail_pfnvec;
 	buf->vec = vec;
@@ -292,7 +293,8 @@ static void vb2_dma_sg_put_userptr(void *buf_priv)
 		vm_unmap_ram(buf->vaddr, buf->num_pages);
 	sg_free_table(buf->dma_sgt);
 	while (--i >= 0) {
-		if (buf->dma_dir == DMA_FROM_DEVICE)
+		if (buf->dma_dir == DMA_FROM_DEVICE ||
+		    buf->dma_dir == DMA_BIDIRECTIONAL)
 			set_page_dirty_lock(buf->pages[i]);
 	}
 	vb2_destroy_framevec(buf->vec);
@@ -500,7 +502,7 @@ static int vb2_dma_sg_dmabuf_ops_mmap(struct dma_buf *dbuf,
 	return vb2_dma_sg_mmap(dbuf->priv, vma);
 }
 
-static struct dma_buf_ops vb2_dma_sg_dmabuf_ops = {
+static const struct dma_buf_ops vb2_dma_sg_dmabuf_ops = {
 	.attach = vb2_dma_sg_dmabuf_ops_attach,
 	.detach = vb2_dma_sg_dmabuf_ops_detach,
 	.map_dma_buf = vb2_dma_sg_dmabuf_ops_map,
diff --git a/drivers/media/v4l2-core/videobuf2-vmalloc.c b/drivers/media/v4l2-core/videobuf2-vmalloc.c
index b337d780844c..3a7c80cd1a17 100644
--- a/drivers/media/v4l2-core/videobuf2-vmalloc.c
+++ b/drivers/media/v4l2-core/videobuf2-vmalloc.c
@@ -87,7 +87,8 @@ static void *vb2_vmalloc_get_userptr(struct device *dev, unsigned long vaddr,
 	buf->dma_dir = dma_dir;
 	offset = vaddr & ~PAGE_MASK;
 	buf->size = size;
-	vec = vb2_create_framevec(vaddr, size, dma_dir == DMA_FROM_DEVICE);
+	vec = vb2_create_framevec(vaddr, size, dma_dir == DMA_FROM_DEVICE ||
+					       dma_dir == DMA_BIDIRECTIONAL);
 	if (IS_ERR(vec)) {
 		ret = PTR_ERR(vec);
 		goto fail_pfnvec_create;
@@ -137,7 +138,8 @@ static void vb2_vmalloc_put_userptr(void *buf_priv)
 		pages = frame_vector_pages(buf->vec);
 		if (vaddr)
 			vm_unmap_ram((void *)vaddr, n_pages);
-		if (buf->dma_dir == DMA_FROM_DEVICE)
+		if (buf->dma_dir == DMA_FROM_DEVICE ||
+		    buf->dma_dir == DMA_BIDIRECTIONAL)
 			for (i = 0; i < n_pages; i++)
 				set_page_dirty_lock(pages[i]);
 	} else {
@@ -338,7 +340,7 @@ static int vb2_vmalloc_dmabuf_ops_mmap(struct dma_buf *dbuf,
 	return vb2_vmalloc_mmap(dbuf->priv, vma);
 }
 
-static struct dma_buf_ops vb2_vmalloc_dmabuf_ops = {
+static const struct dma_buf_ops vb2_vmalloc_dmabuf_ops = {
 	.attach = vb2_vmalloc_dmabuf_ops_attach,
 	.detach = vb2_vmalloc_dmabuf_ops_detach,
 	.map_dma_buf = vb2_vmalloc_dmabuf_ops_map,
diff --git a/drivers/memory/atmel-ebi.c b/drivers/memory/atmel-ebi.c
index 99e644cda4d1..b907865d4664 100644
--- a/drivers/memory/atmel-ebi.c
+++ b/drivers/memory/atmel-ebi.c
@@ -51,6 +51,7 @@ struct atmel_ebi {
 	struct  {
 		struct regmap *regmap;
 		struct clk *clk;
+		const struct atmel_hsmc_reg_layout *layout;
 	} smc;
 
 	struct device *dev;
@@ -72,7 +73,7 @@ struct atmel_smc_timing_xlate {
 	{ .name = nm, .converter = atmel_smc_cs_conf_set_pulse, .shift = pos}
 
 #define ATMEL_SMC_CYCLE_XLATE(nm, pos)	\
-	{ .name = nm, .converter = atmel_smc_cs_conf_set_setup, .shift = pos}
+	{ .name = nm, .converter = atmel_smc_cs_conf_set_cycle, .shift = pos}
 
 static void at91sam9_ebi_get_config(struct atmel_ebi_dev *ebid,
 				    struct atmel_ebi_dev_config *conf)
@@ -84,8 +85,8 @@ static void at91sam9_ebi_get_config(struct atmel_ebi_dev *ebid,
 static void sama5_ebi_get_config(struct atmel_ebi_dev *ebid,
 				 struct atmel_ebi_dev_config *conf)
 {
-	atmel_hsmc_cs_conf_get(ebid->ebi->smc.regmap, conf->cs,
-			       &conf->smcconf);
+	atmel_hsmc_cs_conf_get(ebid->ebi->smc.regmap, ebid->ebi->smc.layout,
+			       conf->cs, &conf->smcconf);
 }
 
 static const struct atmel_smc_timing_xlate timings_xlate_table[] = {
@@ -120,12 +121,14 @@ static int atmel_ebi_xslate_smc_timings(struct atmel_ebi_dev *ebid,
 	if (!ret) {
 		required = true;
 		ncycles = DIV_ROUND_UP(val, clk_period_ns);
-		if (ncycles > ATMEL_SMC_MODE_TDF_MAX ||
-		    ncycles < ATMEL_SMC_MODE_TDF_MIN) {
+		if (ncycles > ATMEL_SMC_MODE_TDF_MAX) {
 			ret = -EINVAL;
 			goto out;
 		}
 
+		if (ncycles < ATMEL_SMC_MODE_TDF_MIN)
+			ncycles = ATMEL_SMC_MODE_TDF_MIN;
+
 		smcconf->mode |= ATMEL_SMC_MODE_TDF(ncycles);
 	}
 
@@ -156,8 +159,8 @@ static int atmel_ebi_xslate_smc_timings(struct atmel_ebi_dev *ebid,
 out:
 	if (ret) {
 		dev_err(ebid->ebi->dev,
-			"missing or invalid timings definition in %s",
-			np->full_name);
+			"missing or invalid timings definition in %pOF",
+			np);
 		return ret;
 	}
 
@@ -263,12 +266,12 @@ static int atmel_ebi_xslate_smc_config(struct atmel_ebi_dev *ebid,
 	}
 
 	ret = atmel_ebi_xslate_smc_timings(ebid, np, &conf->smcconf);
-	if (ret)
+	if (ret < 0)
 		return -EINVAL;
 
 	if ((ret > 0 && !required) || (!ret && required)) {
-		dev_err(ebid->ebi->dev, "missing atmel,smc- properties in %s",
-			np->full_name);
+		dev_err(ebid->ebi->dev, "missing atmel,smc- properties in %pOF",
+			np);
 		return -EINVAL;
 	}
 
@@ -285,8 +288,8 @@ static void at91sam9_ebi_apply_config(struct atmel_ebi_dev *ebid,
 static void sama5_ebi_apply_config(struct atmel_ebi_dev *ebid,
 				   struct atmel_ebi_dev_config *conf)
 {
-	atmel_hsmc_cs_conf_apply(ebid->ebi->smc.regmap, conf->cs,
-				 &conf->smcconf);
+	atmel_hsmc_cs_conf_apply(ebid->ebi->smc.regmap, ebid->ebi->smc.layout,
+				 conf->cs, &conf->smcconf);
 }
 
 static int atmel_ebi_dev_setup(struct atmel_ebi *ebi, struct device_node *np,
@@ -311,8 +314,7 @@ static int atmel_ebi_dev_setup(struct atmel_ebi *ebi, struct device_node *np,
 
 		if (cs >= AT91_MATRIX_EBI_NUM_CS ||
 		    !(ebi->caps->available_cs & BIT(cs))) {
-			dev_err(dev, "invalid reg property in %s\n",
-				np->full_name);
+			dev_err(dev, "invalid reg property in %pOF\n", np);
 			return -EINVAL;
 		}
 
@@ -321,7 +323,7 @@ static int atmel_ebi_dev_setup(struct atmel_ebi *ebi, struct device_node *np,
 	}
 
 	if (!numcs) {
-		dev_err(dev, "invalid reg property in %s\n", np->full_name);
+		dev_err(dev, "invalid reg property in %pOF\n", np);
 		return -EINVAL;
 	}
 
@@ -525,6 +527,10 @@ static int atmel_ebi_probe(struct platform_device *pdev)
 	if (IS_ERR(ebi->smc.regmap))
 		return PTR_ERR(ebi->smc.regmap);
 
+	ebi->smc.layout = atmel_hsmc_get_reg_layout(smc_np);
+	if (IS_ERR(ebi->smc.layout))
+		return PTR_ERR(ebi->smc.layout);
+
 	ebi->smc.clk = of_clk_get(smc_np, 0);
 	if (IS_ERR(ebi->smc.clk)) {
 		if (PTR_ERR(ebi->smc.clk) != -ENOENT)
@@ -569,8 +575,8 @@ static int atmel_ebi_probe(struct platform_device *pdev)
 
 		ret = atmel_ebi_dev_setup(ebi, child, reg_cells);
 		if (ret) {
-			dev_err(dev, "failed to configure EBI bus for %s, disabling the device",
-				child->full_name);
+			dev_err(dev, "failed to configure EBI bus for %pOF, disabling the device",
+				child);
 
 			ret = atmel_ebi_dev_disable(ebi, child);
 			if (ret)
diff --git a/drivers/memory/jz4780-nemc.c b/drivers/memory/jz4780-nemc.c
index 919d1925acb9..bcf06adefc96 100644
--- a/drivers/memory/jz4780-nemc.c
+++ b/drivers/memory/jz4780-nemc.c
@@ -322,8 +322,8 @@ static int jz4780_nemc_probe(struct platform_device *pdev)
 			bank = of_read_number(prop, 1);
 			if (bank < 1 || bank >= JZ4780_NEMC_NUM_BANKS) {
 				dev_err(nemc->dev,
-					"%s requests invalid bank %u\n",
-					child->full_name, bank);
+					"%pOF requests invalid bank %u\n",
+					child, bank);
 
 				/* Will continue the outer loop below. */
 				referenced = 0;
@@ -334,12 +334,12 @@ static int jz4780_nemc_probe(struct platform_device *pdev)
 		}
 
 		if (!referenced) {
-			dev_err(nemc->dev, "%s has no addresses\n",
-				child->full_name);
+			dev_err(nemc->dev, "%pOF has no addresses\n",
+				child);
 			continue;
 		} else if (nemc->banks_present & referenced) {
-			dev_err(nemc->dev, "%s conflicts with another node\n",
-				child->full_name);
+			dev_err(nemc->dev, "%pOF conflicts with another node\n",
+				child);
 			continue;
 		}
 
diff --git a/drivers/memory/mtk-smi.c b/drivers/memory/mtk-smi.c
index 4afbc412f959..8f2d152a78b8 100644
--- a/drivers/memory/mtk-smi.c
+++ b/drivers/memory/mtk-smi.c
@@ -16,6 +16,7 @@
 #include <linux/device.h>
 #include <linux/err.h>
 #include <linux/io.h>
+#include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
@@ -23,7 +24,10 @@
 #include <soc/mediatek/smi.h>
 #include <dt-bindings/memory/mt2701-larb-port.h>
 
+/* mt8173 */
 #define SMI_LARB_MMU_EN		0xf00
+
+/* mt2701 */
 #define REG_SMI_SECUR_CON_BASE		0x5c0
 
 /* every register control 8 port, register offset 0x4 */
@@ -41,7 +45,12 @@
 /* mt2701 domain should be set to 3 */
 #define SMI_SECUR_CON_VAL_DOMAIN(id)	(0x3 << ((((id) & 0x7) << 2) + 1))
 
+/* mt2712 */
+#define SMI_LARB_NONSEC_CON(id)	(0x380 + ((id) * 4))
+#define F_MMU_EN		BIT(0)
+
 struct mtk_smi_larb_gen {
+	bool need_larbid;
 	int port_in_larb[MTK_LARB_NR_MAX + 1];
 	void (*config_port)(struct device *);
 };
@@ -148,6 +157,15 @@ mtk_smi_larb_bind(struct device *dev, struct device *master, void *data)
 	struct mtk_smi_iommu *smi_iommu = data;
 	unsigned int         i;
 
+	if (larb->larb_gen->need_larbid) {
+		larb->mmu = &smi_iommu->larb_imu[larb->larbid].mmu;
+		return 0;
+	}
+
+	/*
+	 * If there is no larbid property, Loop to find the corresponding
+	 * iommu information.
+	 */
 	for (i = 0; i < smi_iommu->larb_nr; i++) {
 		if (dev == smi_iommu->larb_imu[i].dev) {
 			/* The 'mmu' may be updated in iommu-attach/detach. */
@@ -158,13 +176,32 @@ mtk_smi_larb_bind(struct device *dev, struct device *master, void *data)
 	return -ENODEV;
 }
 
-static void mtk_smi_larb_config_port(struct device *dev)
+static void mtk_smi_larb_config_port_mt2712(struct device *dev)
 {
 	struct mtk_smi_larb *larb = dev_get_drvdata(dev);
+	u32 reg;
+	int i;
 
-	writel(*larb->mmu, larb->base + SMI_LARB_MMU_EN);
+	/*
+	 * larb 8/9 is the bdpsys larb, the iommu_en is enabled defaultly.
+	 * Don't need to set it again.
+	 */
+	if (larb->larbid == 8 || larb->larbid == 9)
+		return;
+
+	for_each_set_bit(i, (unsigned long *)larb->mmu, 32) {
+		reg = readl_relaxed(larb->base + SMI_LARB_NONSEC_CON(i));
+		reg |= F_MMU_EN;
+		writel(reg, larb->base + SMI_LARB_NONSEC_CON(i));
+	}
 }
 
+static void mtk_smi_larb_config_port_mt8173(struct device *dev)
+{
+	struct mtk_smi_larb *larb = dev_get_drvdata(dev);
+
+	writel(*larb->mmu, larb->base + SMI_LARB_MMU_EN);
+}
 
 static void mtk_smi_larb_config_port_gen1(struct device *dev)
 {
@@ -210,10 +247,11 @@ static const struct component_ops mtk_smi_larb_component_ops = {
 
 static const struct mtk_smi_larb_gen mtk_smi_larb_mt8173 = {
 	/* mt8173 do not need the port in larb */
-	.config_port = mtk_smi_larb_config_port,
+	.config_port = mtk_smi_larb_config_port_mt8173,
 };
 
 static const struct mtk_smi_larb_gen mtk_smi_larb_mt2701 = {
+	.need_larbid = true,
 	.port_in_larb = {
 		LARB0_PORT_OFFSET, LARB1_PORT_OFFSET,
 		LARB2_PORT_OFFSET, LARB3_PORT_OFFSET
@@ -221,6 +259,11 @@ static const struct mtk_smi_larb_gen mtk_smi_larb_mt2701 = {
 	.config_port = mtk_smi_larb_config_port_gen1,
 };
 
+static const struct mtk_smi_larb_gen mtk_smi_larb_mt2712 = {
+	.need_larbid = true,
+	.config_port = mtk_smi_larb_config_port_mt2712,
+};
+
 static const struct of_device_id mtk_smi_larb_of_ids[] = {
 	{
 		.compatible = "mediatek,mt8173-smi-larb",
@@ -230,6 +273,10 @@ static const struct of_device_id mtk_smi_larb_of_ids[] = {
 		.compatible = "mediatek,mt2701-smi-larb",
 		.data = &mtk_smi_larb_mt2701
 	},
+	{
+		.compatible = "mediatek,mt2712-smi-larb",
+		.data = &mtk_smi_larb_mt2712
+	},
 	{}
 };
 
@@ -240,20 +287,13 @@ static int mtk_smi_larb_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct device_node *smi_node;
 	struct platform_device *smi_pdev;
-	const struct of_device_id *of_id;
-
-	if (!dev->pm_domain)
-		return -EPROBE_DEFER;
-
-	of_id = of_match_node(mtk_smi_larb_of_ids, pdev->dev.of_node);
-	if (!of_id)
-		return -EINVAL;
+	int err;
 
 	larb = devm_kzalloc(dev, sizeof(*larb), GFP_KERNEL);
 	if (!larb)
 		return -ENOMEM;
 
-	larb->larb_gen = of_id->data;
+	larb->larb_gen = of_device_get_match_data(dev);
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	larb->base = devm_ioremap_resource(dev, res);
 	if (IS_ERR(larb->base))
@@ -268,6 +308,15 @@ static int mtk_smi_larb_probe(struct platform_device *pdev)
 		return PTR_ERR(larb->smi.clk_smi);
 	larb->smi.dev = dev;
 
+	if (larb->larb_gen->need_larbid) {
+		err = of_property_read_u32(dev->of_node, "mediatek,larb-id",
+					   &larb->larbid);
+		if (err) {
+			dev_err(dev, "missing larbid property\n");
+			return err;
+		}
+	}
+
 	smi_node = of_parse_phandle(dev->of_node, "mediatek,smi", 0);
 	if (!smi_node)
 		return -EINVAL;
@@ -275,6 +324,8 @@ static int mtk_smi_larb_probe(struct platform_device *pdev)
 	smi_pdev = of_find_device_by_node(smi_node);
 	of_node_put(smi_node);
 	if (smi_pdev) {
+		if (!platform_get_drvdata(smi_pdev))
+			return -EPROBE_DEFER;
 		larb->smi_common_dev = &smi_pdev->dev;
 	} else {
 		dev_err(dev, "Failed to get the smi_common device\n");
@@ -311,6 +362,10 @@ static const struct of_device_id mtk_smi_common_of_ids[] = {
 		.compatible = "mediatek,mt2701-smi-common",
 		.data = (void *)MTK_SMI_GEN1
 	},
+	{
+		.compatible = "mediatek,mt2712-smi-common",
+		.data = (void *)MTK_SMI_GEN2
+	},
 	{}
 };
 
@@ -319,11 +374,8 @@ static int mtk_smi_common_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct mtk_smi *common;
 	struct resource *res;
-	const struct of_device_id *of_id;
 	enum mtk_smi_gen smi_gen;
-
-	if (!dev->pm_domain)
-		return -EPROBE_DEFER;
+	int ret;
 
 	common = devm_kzalloc(dev, sizeof(*common), GFP_KERNEL);
 	if (!common)
@@ -338,17 +390,13 @@ static int mtk_smi_common_probe(struct platform_device *pdev)
 	if (IS_ERR(common->clk_smi))
 		return PTR_ERR(common->clk_smi);
 
-	of_id = of_match_node(mtk_smi_common_of_ids, pdev->dev.of_node);
-	if (!of_id)
-		return -EINVAL;
-
 	/*
 	 * for mtk smi gen 1, we need to get the ao(always on) base to config
 	 * m4u port, and we need to enable the aync clock for transform the smi
 	 * clock into emi clock domain, but for mtk smi gen2, there's no smi ao
 	 * base.
 	 */
-	smi_gen = (enum mtk_smi_gen)of_id->data;
+	smi_gen = (enum mtk_smi_gen)of_device_get_match_data(dev);
 	if (smi_gen == MTK_SMI_GEN1) {
 		res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 		common->smi_ao_base = devm_ioremap_resource(dev, res);
@@ -359,7 +407,9 @@ static int mtk_smi_common_probe(struct platform_device *pdev)
 		if (IS_ERR(common->clk_async))
 			return PTR_ERR(common->clk_async);
 
-		clk_prepare_enable(common->clk_async);
+		ret = clk_prepare_enable(common->clk_async);
+		if (ret)
+			return ret;
 	}
 	pm_runtime_enable(dev);
 	platform_set_drvdata(pdev, common);
@@ -403,4 +453,4 @@ err_unreg_smi:
 	return ret;
 }
 
-subsys_initcall(mtk_smi_init);
+module_init(mtk_smi_init);
diff --git a/drivers/memory/mvebu-devbus.c b/drivers/memory/mvebu-devbus.c
index 24852812fd44..981860879d02 100644
--- a/drivers/memory/mvebu-devbus.c
+++ b/drivers/memory/mvebu-devbus.c
@@ -105,8 +105,8 @@ static int get_timing_param_ps(struct devbus *devbus,
 
 	err = of_property_read_u32(node, name, &time_ps);
 	if (err < 0) {
-		dev_err(devbus->dev, "%s has no '%s' property\n",
-			name, node->full_name);
+		dev_err(devbus->dev, "%pOF has no '%s' property\n",
+			node, name);
 		return err;
 	}
 
@@ -127,8 +127,8 @@ static int devbus_get_timing_params(struct devbus *devbus,
 	err = of_property_read_u32(node, "devbus,bus-width", &r->bus_width);
 	if (err < 0) {
 		dev_err(devbus->dev,
-			"%s has no 'devbus,bus-width' property\n",
-			node->full_name);
+			"%pOF has no 'devbus,bus-width' property\n",
+			node);
 		return err;
 	}
 
@@ -180,8 +180,8 @@ static int devbus_get_timing_params(struct devbus *devbus,
 					   &w->sync_enable);
 		if (err < 0) {
 			dev_err(devbus->dev,
-				"%s has no 'devbus,sync-enable' property\n",
-				node->full_name);
+				"%pOF has no 'devbus,sync-enable' property\n",
+				node);
 			return err;
 		}
 	}
diff --git a/drivers/memory/omap-gpmc.c b/drivers/memory/omap-gpmc.c
index a80e17de906d..7059bbda2fac 100644
--- a/drivers/memory/omap-gpmc.c
+++ b/drivers/memory/omap-gpmc.c
@@ -1930,8 +1930,8 @@ static int gpmc_probe_onenand_child(struct platform_device *pdev,
 	struct omap_onenand_platform_data *gpmc_onenand_data;
 
 	if (of_property_read_u32(child, "reg", &val) < 0) {
-		dev_err(&pdev->dev, "%s has no 'reg' property\n",
-			child->full_name);
+		dev_err(&pdev->dev, "%pOF has no 'reg' property\n",
+			child);
 		return -ENODEV;
 	}
 
@@ -1979,14 +1979,14 @@ static int gpmc_probe_generic_child(struct platform_device *pdev,
 	struct gpmc_device *gpmc = platform_get_drvdata(pdev);
 
 	if (of_property_read_u32(child, "reg", &cs) < 0) {
-		dev_err(&pdev->dev, "%s has no 'reg' property\n",
-			child->full_name);
+		dev_err(&pdev->dev, "%pOF has no 'reg' property\n",
+			child);
 		return -ENODEV;
 	}
 
 	if (of_address_to_resource(child, 0, &res) < 0) {
-		dev_err(&pdev->dev, "%s has malformed 'reg' property\n",
-			child->full_name);
+		dev_err(&pdev->dev, "%pOF has malformed 'reg' property\n",
+			child);
 		return -ENODEV;
 	}
 
@@ -2084,8 +2084,8 @@ static int gpmc_probe_generic_child(struct platform_device *pdev,
 		ret = of_property_read_u32(child, "bank-width",
 					   &gpmc_s.device_width);
 		if (ret < 0) {
-			dev_err(&pdev->dev, "%s has no 'bank-width' property\n",
-				child->full_name);
+			dev_err(&pdev->dev, "%pOF has no 'bank-width' property\n",
+				child);
 			goto err;
 		}
 	}
diff --git a/drivers/message/fusion/mptbase.c b/drivers/message/fusion/mptbase.c
index 62cff5afc6bd..84eab28665f3 100644
--- a/drivers/message/fusion/mptbase.c
+++ b/drivers/message/fusion/mptbase.c
@@ -2079,7 +2079,7 @@ void
 mpt_detach(struct pci_dev *pdev)
 {
 	MPT_ADAPTER 	*ioc = pci_get_drvdata(pdev);
-	char pname[32];
+	char pname[64];
 	u8 cb_idx;
 	unsigned long flags;
 	struct workqueue_struct *wq;
@@ -2100,11 +2100,11 @@ mpt_detach(struct pci_dev *pdev)
 	spin_unlock_irqrestore(&ioc->fw_event_lock, flags);
 	destroy_workqueue(wq);
 
-	sprintf(pname, MPT_PROCFS_MPTBASEDIR "/%s/summary", ioc->name);
+	snprintf(pname, sizeof(pname), MPT_PROCFS_MPTBASEDIR "/%s/summary", ioc->name);
 	remove_proc_entry(pname, NULL);
-	sprintf(pname, MPT_PROCFS_MPTBASEDIR "/%s/info", ioc->name);
+	snprintf(pname, sizeof(pname), MPT_PROCFS_MPTBASEDIR "/%s/info", ioc->name);
 	remove_proc_entry(pname, NULL);
-	sprintf(pname, MPT_PROCFS_MPTBASEDIR "/%s", ioc->name);
+	snprintf(pname, sizeof(pname), MPT_PROCFS_MPTBASEDIR "/%s", ioc->name);
 	remove_proc_entry(pname, NULL);
 
 	/* call per device driver remove entry point */
diff --git a/drivers/message/fusion/mptfc.c b/drivers/message/fusion/mptfc.c
index d065062240bc..6d461ca97150 100644
--- a/drivers/message/fusion/mptfc.c
+++ b/drivers/message/fusion/mptfc.c
@@ -104,7 +104,6 @@ static void mptfc_remove(struct pci_dev *pdev);
 static int mptfc_abort(struct scsi_cmnd *SCpnt);
 static int mptfc_dev_reset(struct scsi_cmnd *SCpnt);
 static int mptfc_bus_reset(struct scsi_cmnd *SCpnt);
-static int mptfc_host_reset(struct scsi_cmnd *SCpnt);
 
 static struct scsi_host_template mptfc_driver_template = {
 	.module				= THIS_MODULE,
@@ -123,7 +122,7 @@ static struct scsi_host_template mptfc_driver_template = {
 	.eh_abort_handler		= mptfc_abort,
 	.eh_device_reset_handler	= mptfc_dev_reset,
 	.eh_bus_reset_handler		= mptfc_bus_reset,
-	.eh_host_reset_handler		= mptfc_host_reset,
+	.eh_host_reset_handler		= mptscsih_host_reset,
 	.bios_param			= mptscsih_bios_param,
 	.can_queue			= MPT_FC_CAN_QUEUE,
 	.this_id			= -1,
@@ -254,13 +253,6 @@ mptfc_bus_reset(struct scsi_cmnd *SCpnt)
 	    mptfc_block_error_handler(SCpnt, mptscsih_bus_reset, __func__);
 }
 
-static int
-mptfc_host_reset(struct scsi_cmnd *SCpnt)
-{
-	return
-	    mptfc_block_error_handler(SCpnt, mptscsih_host_reset, __func__);
-}
-
 static void
 mptfc_set_rport_loss_tmo(struct fc_rport *rport, uint32_t timeout)
 {
diff --git a/drivers/message/fusion/mptsas.c b/drivers/message/fusion/mptsas.c
index f6308ad35b19..345f6035599e 100644
--- a/drivers/message/fusion/mptsas.c
+++ b/drivers/message/fusion/mptsas.c
@@ -2210,33 +2210,26 @@ mptsas_get_bay_identifier(struct sas_rphy *rphy)
 	return rc;
 }
 
-static int mptsas_smp_handler(struct Scsi_Host *shost, struct sas_rphy *rphy,
-			      struct request *req)
+static void mptsas_smp_handler(struct bsg_job *job, struct Scsi_Host *shost,
+		struct sas_rphy *rphy)
 {
 	MPT_ADAPTER *ioc = ((MPT_SCSI_HOST *) shost->hostdata)->ioc;
 	MPT_FRAME_HDR *mf;
 	SmpPassthroughRequest_t *smpreq;
-	struct request *rsp = req->next_rq;
-	int ret;
 	int flagsLength;
 	unsigned long timeleft;
 	char *psge;
-	dma_addr_t dma_addr_in = 0;
-	dma_addr_t dma_addr_out = 0;
 	u64 sas_address = 0;
-
-	if (!rsp) {
-		printk(MYIOC_s_ERR_FMT "%s: the smp response space is missing\n",
-		    ioc->name, __func__);
-		return -EINVAL;
-	}
+	unsigned int reslen = 0;
+	int ret = -EINVAL;
 
 	/* do we need to support multiple segments? */
-	if (bio_multiple_segments(req->bio) ||
-	    bio_multiple_segments(rsp->bio)) {
+	if (job->request_payload.sg_cnt > 1 ||
+	    job->reply_payload.sg_cnt > 1) {
 		printk(MYIOC_s_ERR_FMT "%s: multiple segments req %u, rsp %u\n",
-		    ioc->name, __func__, blk_rq_bytes(req), blk_rq_bytes(rsp));
-		return -EINVAL;
+		    ioc->name, __func__, job->request_payload.payload_len,
+		    job->reply_payload.payload_len);
+		goto out;
 	}
 
 	ret = mutex_lock_interruptible(&ioc->sas_mgmt.mutex);
@@ -2252,7 +2245,8 @@ static int mptsas_smp_handler(struct Scsi_Host *shost, struct sas_rphy *rphy,
 	smpreq = (SmpPassthroughRequest_t *)mf;
 	memset(smpreq, 0, sizeof(*smpreq));
 
-	smpreq->RequestDataLength = cpu_to_le16(blk_rq_bytes(req) - 4);
+	smpreq->RequestDataLength =
+		cpu_to_le16(job->request_payload.payload_len - 4);
 	smpreq->Function = MPI_FUNCTION_SMP_PASSTHROUGH;
 
 	if (rphy)
@@ -2278,13 +2272,14 @@ static int mptsas_smp_handler(struct Scsi_Host *shost, struct sas_rphy *rphy,
 		       MPI_SGE_FLAGS_END_OF_BUFFER |
 		       MPI_SGE_FLAGS_DIRECTION)
 		       << MPI_SGE_FLAGS_SHIFT;
-	flagsLength |= (blk_rq_bytes(req) - 4);
 
-	dma_addr_out = pci_map_single(ioc->pcidev, bio_data(req->bio),
-				      blk_rq_bytes(req), PCI_DMA_BIDIRECTIONAL);
-	if (pci_dma_mapping_error(ioc->pcidev, dma_addr_out))
+	if (!dma_map_sg(&ioc->pcidev->dev, job->request_payload.sg_list,
+			1, PCI_DMA_BIDIRECTIONAL))
 		goto put_mf;
-	ioc->add_sge(psge, flagsLength, dma_addr_out);
+
+	flagsLength |= (sg_dma_len(job->request_payload.sg_list) - 4);
+	ioc->add_sge(psge, flagsLength,
+			sg_dma_address(job->request_payload.sg_list));
 	psge += ioc->SGE_size;
 
 	/* response */
@@ -2294,12 +2289,13 @@ static int mptsas_smp_handler(struct Scsi_Host *shost, struct sas_rphy *rphy,
 		MPI_SGE_FLAGS_END_OF_BUFFER;
 
 	flagsLength = flagsLength << MPI_SGE_FLAGS_SHIFT;
-	flagsLength |= blk_rq_bytes(rsp) + 4;
-	dma_addr_in =  pci_map_single(ioc->pcidev, bio_data(rsp->bio),
-				      blk_rq_bytes(rsp), PCI_DMA_BIDIRECTIONAL);
-	if (pci_dma_mapping_error(ioc->pcidev, dma_addr_in))
-		goto unmap;
-	ioc->add_sge(psge, flagsLength, dma_addr_in);
+
+	if (!dma_map_sg(&ioc->pcidev->dev, job->reply_payload.sg_list,
+			1, PCI_DMA_BIDIRECTIONAL))
+		goto unmap_out;
+	flagsLength |= sg_dma_len(job->reply_payload.sg_list) + 4;
+	ioc->add_sge(psge, flagsLength,
+			sg_dma_address(job->reply_payload.sg_list));
 
 	INITIALIZE_MGMT_STATUS(ioc->sas_mgmt.status)
 	mpt_put_msg_frame(mptsasMgmtCtx, ioc, mf);
@@ -2310,10 +2306,10 @@ static int mptsas_smp_handler(struct Scsi_Host *shost, struct sas_rphy *rphy,
 		mpt_free_msg_frame(ioc, mf);
 		mf = NULL;
 		if (ioc->sas_mgmt.status & MPT_MGMT_STATUS_DID_IOCRESET)
-			goto unmap;
+			goto unmap_in;
 		if (!timeleft)
 			mpt_Soft_Hard_ResetHandler(ioc, CAN_SLEEP);
-		goto unmap;
+		goto unmap_in;
 	}
 	mf = NULL;
 
@@ -2321,23 +2317,22 @@ static int mptsas_smp_handler(struct Scsi_Host *shost, struct sas_rphy *rphy,
 		SmpPassthroughReply_t *smprep;
 
 		smprep = (SmpPassthroughReply_t *)ioc->sas_mgmt.reply;
-		memcpy(scsi_req(req)->sense, smprep, sizeof(*smprep));
-		scsi_req(req)->sense_len = sizeof(*smprep);
-		scsi_req(req)->resid_len = 0;
-		scsi_req(rsp)->resid_len -= smprep->ResponseDataLength;
+		memcpy(job->reply, smprep, sizeof(*smprep));
+		job->reply_len = sizeof(*smprep);
+		reslen = smprep->ResponseDataLength;
 	} else {
 		printk(MYIOC_s_ERR_FMT
 		    "%s: smp passthru reply failed to be returned\n",
 		    ioc->name, __func__);
 		ret = -ENXIO;
 	}
-unmap:
-	if (dma_addr_out)
-		pci_unmap_single(ioc->pcidev, dma_addr_out, blk_rq_bytes(req),
-				 PCI_DMA_BIDIRECTIONAL);
-	if (dma_addr_in)
-		pci_unmap_single(ioc->pcidev, dma_addr_in, blk_rq_bytes(rsp),
-				 PCI_DMA_BIDIRECTIONAL);
+
+unmap_in:
+	dma_unmap_sg(&ioc->pcidev->dev, job->reply_payload.sg_list, 1,
+			PCI_DMA_BIDIRECTIONAL);
+unmap_out:
+	dma_unmap_sg(&ioc->pcidev->dev, job->request_payload.sg_list, 1,
+			PCI_DMA_BIDIRECTIONAL);
 put_mf:
 	if (mf)
 		mpt_free_msg_frame(ioc, mf);
@@ -2345,7 +2340,7 @@ out_unlock:
 	CLEAR_MGMT_STATUS(ioc->sas_mgmt.status)
 	mutex_unlock(&ioc->sas_mgmt.mutex);
 out:
-	return ret;
+	bsg_job_done(job, ret, reslen);
 }
 
 static struct sas_function_template mptsas_transport_functions = {
@@ -4352,11 +4347,10 @@ mptsas_hotplug_work(MPT_ADAPTER *ioc, struct fw_event_work *fw_event,
 			return;
 
 		phy_info = mptsas_refreshing_device_handles(ioc, &sas_device);
-		/* Only For SATA Device ADD */
-		if (!phy_info && (sas_device.device_info &
-				MPI_SAS_DEVICE_INFO_SATA_DEVICE)) {
+		/* Device hot plug */
+		if (!phy_info) {
 			devtprintk(ioc, printk(MYIOC_s_DEBUG_FMT
-				"%s %d SATA HOT PLUG: "
+				"%s %d HOT PLUG: "
 				"parent handle of device %x\n", ioc->name,
 				__func__, __LINE__, sas_device.handle_parent));
 			port_info = mptsas_find_portinfo_by_handle(ioc,
diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig
index 94ad2c1c3d90..fc5e4fef89d2 100644
--- a/drivers/mfd/Kconfig
+++ b/drivers/mfd/Kconfig
@@ -133,6 +133,20 @@ config MFD_BCM590XX
 	help
 	  Support for the BCM590xx PMUs from Broadcom
 
+config MFD_BD9571MWV
+	tristate "ROHM BD9571MWV PMIC"
+	select MFD_CORE
+	select REGMAP_I2C
+	select REGMAP_IRQ
+	depends on I2C
+	help
+	  Support for the ROHM BD9571MWV PMIC, which contains single
+	  voltage regulator, voltage sampling units, GPIO block and
+	  watchdog block.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called bd9571mwv.
+
 config MFD_AC100
 	tristate "X-Powers AC100"
 	select MFD_CORE
@@ -453,12 +467,12 @@ config LPC_SCH
 
 config INTEL_SOC_PMIC
 	bool "Support for Crystal Cove PMIC"
-	depends on HAS_IOMEM && I2C=y && GPIOLIB && COMMON_CLK
+	depends on ACPI && HAS_IOMEM && I2C=y && GPIOLIB && COMMON_CLK
 	depends on X86 || COMPILE_TEST
 	select MFD_CORE
 	select REGMAP_I2C
 	select REGMAP_IRQ
-	select I2C_DESIGNWARE_PLATFORM if ACPI
+	select I2C_DESIGNWARE_PLATFORM
 	help
 	  Select this option to enable support for Crystal Cove PMIC
 	  on some Intel SoC systems. The PMIC provides ADC, GPIO,
@@ -481,7 +495,7 @@ config INTEL_SOC_PMIC_BXTWC
 	  on these systems.
 
 config INTEL_SOC_PMIC_CHTWC
-	tristate "Support for Intel Cherry Trail Whiskey Cove PMIC"
+	bool "Support for Intel Cherry Trail Whiskey Cove PMIC"
 	depends on ACPI && HAS_IOMEM && I2C=y && COMMON_CLK
 	depends on X86 || COMPILE_TEST
 	select MFD_CORE
@@ -951,13 +965,13 @@ config MFD_RC5T583
 	  different functionality of the device.
 
 config MFD_RK808
-	tristate "Rockchip RK808/RK818 Power Management Chip"
+	tristate "Rockchip RK805/RK808/RK818 Power Management Chip"
 	depends on I2C && OF
 	select MFD_CORE
 	select REGMAP_I2C
 	select REGMAP_IRQ
 	help
-	  If you say yes here you get support for the RK808 and RK818
+	  If you say yes here you get support for the RK805, RK808 and RK818
 	  Power Management chips.
 	  This driver provides common support for accessing the device
 	  through I2C interface. The device supports multiple sub-devices
@@ -1294,6 +1308,7 @@ config TPS6507X
 
 config MFD_TPS65086
 	tristate "TI TPS65086 Power Management Integrated Chips (PMICs)"
+	select MFD_CORE
 	select REGMAP
 	select REGMAP_IRQ
 	select REGMAP_I2C
@@ -1337,6 +1352,24 @@ config MFD_TPS65217
 	  This driver can also be built as a module.  If so, the module
 	  will be called tps65217.
 
+config MFD_TPS68470
+	bool "TI TPS68470 Power Management / LED chips"
+	depends on ACPI && I2C=y
+	select MFD_CORE
+	select REGMAP_I2C
+	select I2C_DESIGNWARE_PLATFORM
+	help
+	  If you say yes here you get support for the TPS68470 series of
+	  Power Management / LED chips.
+
+	  These include voltage regulators, LEDs and other features
+	  that are often used in portable devices.
+
+	  This option is a bool as it provides an ACPI operation
+	  region, which must be available before any of the devices
+	  using this are probed. This option also configures the
+	  designware-i2c driver to be built-in, for the same reason.
+
 config MFD_TI_LP873X
 	tristate "TI LP873X Power Management IC"
 	depends on I2C
@@ -1723,6 +1756,20 @@ config MFD_STW481X
 	  in various ST Microelectronics and ST-Ericsson embedded
 	  Nomadik series.
 
+config MFD_STM32_LPTIMER
+	tristate "Support for STM32 Low-Power Timer"
+	depends on (ARCH_STM32 && OF) || COMPILE_TEST
+	select MFD_CORE
+	select REGMAP
+	select REGMAP_MMIO
+	help
+	  Select this option to enable STM32 Low-Power Timer driver
+	  used for PWM, IIO Trigger, IIO Encoder and Counter. Shared
+	  resources are also dealt with here.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called stm32-lptimer.
+
 config MFD_STM32_TIMERS
 	tristate "Support for STM32 Timers"
 	depends on (ARCH_STM32 && OF) || COMPILE_TEST
diff --git a/drivers/mfd/Makefile b/drivers/mfd/Makefile
index 080793b3fd0e..c3d0a1b39bb6 100644
--- a/drivers/mfd/Makefile
+++ b/drivers/mfd/Makefile
@@ -10,6 +10,7 @@ obj-$(CONFIG_MFD_ACT8945A)	+= act8945a.o
 obj-$(CONFIG_MFD_SM501)		+= sm501.o
 obj-$(CONFIG_MFD_ASIC3)		+= asic3.o tmio_core.o
 obj-$(CONFIG_MFD_BCM590XX)	+= bcm590xx.o
+obj-$(CONFIG_MFD_BD9571MWV)	+= bd9571mwv.o
 cros_ec_core-objs		:= cros_ec.o
 cros_ec_core-$(CONFIG_ACPI)	+= cros_ec_acpi_gpe.o
 obj-$(CONFIG_MFD_CROS_EC)	+= cros_ec_core.o
@@ -83,6 +84,7 @@ obj-$(CONFIG_MFD_TPS65910)	+= tps65910.o
 obj-$(CONFIG_MFD_TPS65912)	+= tps65912-core.o
 obj-$(CONFIG_MFD_TPS65912_I2C)	+= tps65912-i2c.o
 obj-$(CONFIG_MFD_TPS65912_SPI)  += tps65912-spi.o
+obj-$(CONFIG_MFD_TPS68470)	+= tps68470.o
 obj-$(CONFIG_MFD_TPS80031)	+= tps80031.o
 obj-$(CONFIG_MENELAUS)		+= menelaus.o
 
@@ -221,5 +223,6 @@ obj-$(CONFIG_MFD_MT6397)	+= mt6397-core.o
 obj-$(CONFIG_MFD_ALTERA_A10SR)	+= altera-a10sr.o
 obj-$(CONFIG_MFD_SUN4I_GPADC)	+= sun4i-gpadc.o
 
+obj-$(CONFIG_MFD_STM32_LPTIMER)	+= stm32-lptimer.o
 obj-$(CONFIG_MFD_STM32_TIMERS) 	+= stm32-timers.o
 obj-$(CONFIG_MFD_MXS_LRADC)     += mxs-lradc.o
diff --git a/drivers/mfd/ab8500-core.c b/drivers/mfd/ab8500-core.c
index 8511c068a610..30d09d177171 100644
--- a/drivers/mfd/ab8500-core.c
+++ b/drivers/mfd/ab8500-core.c
@@ -1059,15 +1059,15 @@ static struct attribute *ab9540_sysfs_entries[] = {
 	NULL,
 };
 
-static struct attribute_group ab8500_attr_group = {
+static const struct attribute_group ab8500_attr_group = {
 	.attrs	= ab8500_sysfs_entries,
 };
 
-static struct attribute_group ab8505_attr_group = {
+static const struct attribute_group ab8505_attr_group = {
 	.attrs	= ab8505_sysfs_entries,
 };
 
-static struct attribute_group ab9540_attr_group = {
+static const struct attribute_group ab9540_attr_group = {
 	.attrs	= ab9540_sysfs_entries,
 };
 
diff --git a/drivers/mfd/atmel-smc.c b/drivers/mfd/atmel-smc.c
index 954cf0f66a31..7d77948567d7 100644
--- a/drivers/mfd/atmel-smc.c
+++ b/drivers/mfd/atmel-smc.c
@@ -206,7 +206,7 @@ EXPORT_SYMBOL_GPL(atmel_smc_cs_conf_set_pulse);
  *	     parameter
  *
  * This function encodes the @ncycles value as described in the datasheet
- * (section "SMC Pulse Register"), and then stores the result in the
+ * (section "SMC Cycle Register"), and then stores the result in the
  * @conf->setup field at @shift position.
  *
  * Returns -EINVAL if @shift is invalid, -ERANGE if @ncycles does not fit in
@@ -258,19 +258,21 @@ EXPORT_SYMBOL_GPL(atmel_smc_cs_conf_apply);
  * atmel_hsmc_cs_conf_apply - apply an SMC CS conf
  * @regmap: the HSMC regmap
  * @cs: the CS id
+ * @layout: the layout of registers
  * @conf the SMC CS conf to apply
  *
  * Applies an SMC CS configuration.
  * Only valid on post-sama5 SoCs.
  */
-void atmel_hsmc_cs_conf_apply(struct regmap *regmap, int cs,
-			      const struct atmel_smc_cs_conf *conf)
+void atmel_hsmc_cs_conf_apply(struct regmap *regmap,
+			      const struct atmel_hsmc_reg_layout *layout,
+			      int cs, const struct atmel_smc_cs_conf *conf)
 {
-	regmap_write(regmap, ATMEL_HSMC_SETUP(cs), conf->setup);
-	regmap_write(regmap, ATMEL_HSMC_PULSE(cs), conf->pulse);
-	regmap_write(regmap, ATMEL_HSMC_CYCLE(cs), conf->cycle);
-	regmap_write(regmap, ATMEL_HSMC_TIMINGS(cs), conf->timings);
-	regmap_write(regmap, ATMEL_HSMC_MODE(cs), conf->mode);
+	regmap_write(regmap, ATMEL_HSMC_SETUP(layout, cs), conf->setup);
+	regmap_write(regmap, ATMEL_HSMC_PULSE(layout, cs), conf->pulse);
+	regmap_write(regmap, ATMEL_HSMC_CYCLE(layout, cs), conf->cycle);
+	regmap_write(regmap, ATMEL_HSMC_TIMINGS(layout, cs), conf->timings);
+	regmap_write(regmap, ATMEL_HSMC_MODE(layout, cs), conf->mode);
 }
 EXPORT_SYMBOL_GPL(atmel_hsmc_cs_conf_apply);
 
@@ -297,18 +299,55 @@ EXPORT_SYMBOL_GPL(atmel_smc_cs_conf_get);
  * atmel_hsmc_cs_conf_get - retrieve the current SMC CS conf
  * @regmap: the HSMC regmap
  * @cs: the CS id
+ * @layout: the layout of registers
  * @conf: the SMC CS conf object to store the current conf
  *
  * Retrieve the SMC CS configuration.
  * Only valid on post-sama5 SoCs.
  */
-void atmel_hsmc_cs_conf_get(struct regmap *regmap, int cs,
-			    struct atmel_smc_cs_conf *conf)
+void atmel_hsmc_cs_conf_get(struct regmap *regmap,
+			    const struct atmel_hsmc_reg_layout *layout,
+			    int cs, struct atmel_smc_cs_conf *conf)
 {
-	regmap_read(regmap, ATMEL_HSMC_SETUP(cs), &conf->setup);
-	regmap_read(regmap, ATMEL_HSMC_PULSE(cs), &conf->pulse);
-	regmap_read(regmap, ATMEL_HSMC_CYCLE(cs), &conf->cycle);
-	regmap_read(regmap, ATMEL_HSMC_TIMINGS(cs), &conf->timings);
-	regmap_read(regmap, ATMEL_HSMC_MODE(cs), &conf->mode);
+	regmap_read(regmap, ATMEL_HSMC_SETUP(layout, cs), &conf->setup);
+	regmap_read(regmap, ATMEL_HSMC_PULSE(layout, cs), &conf->pulse);
+	regmap_read(regmap, ATMEL_HSMC_CYCLE(layout, cs), &conf->cycle);
+	regmap_read(regmap, ATMEL_HSMC_TIMINGS(layout, cs), &conf->timings);
+	regmap_read(regmap, ATMEL_HSMC_MODE(layout, cs), &conf->mode);
 }
 EXPORT_SYMBOL_GPL(atmel_hsmc_cs_conf_get);
+
+static const struct atmel_hsmc_reg_layout sama5d3_reg_layout = {
+	.timing_regs_offset = 0x600,
+};
+
+static const struct atmel_hsmc_reg_layout sama5d2_reg_layout = {
+	.timing_regs_offset = 0x700,
+};
+
+static const struct of_device_id atmel_smc_ids[] = {
+	{ .compatible = "atmel,at91sam9260-smc", .data = NULL },
+	{ .compatible = "atmel,sama5d3-smc", .data = &sama5d3_reg_layout },
+	{ .compatible = "atmel,sama5d2-smc", .data = &sama5d2_reg_layout },
+	{ /* sentinel */ },
+};
+
+/**
+ * atmel_hsmc_get_reg_layout - retrieve the layout of HSMC registers
+ * @np: the HSMC regmap
+ *
+ * Retrieve the layout of HSMC registers.
+ *
+ * Returns NULL in case of SMC, a struct atmel_hsmc_reg_layout pointer
+ * in HSMC case, otherwise ERR_PTR(-EINVAL).
+ */
+const struct atmel_hsmc_reg_layout *
+atmel_hsmc_get_reg_layout(struct device_node *np)
+{
+	const struct of_device_id *match;
+
+	match = of_match_node(atmel_smc_ids, np);
+
+	return match ? match->data : ERR_PTR(-EINVAL);
+}
+EXPORT_SYMBOL_GPL(atmel_hsmc_get_reg_layout);
diff --git a/drivers/mfd/axp20x-rsb.c b/drivers/mfd/axp20x-rsb.c
index fd5c7267b136..7ddbd9e8dd03 100644
--- a/drivers/mfd/axp20x-rsb.c
+++ b/drivers/mfd/axp20x-rsb.c
@@ -64,6 +64,7 @@ static const struct of_device_id axp20x_rsb_of_match[] = {
 	{ .compatible = "x-powers,axp803", .data = (void *)AXP803_ID },
 	{ .compatible = "x-powers,axp806", .data = (void *)AXP806_ID },
 	{ .compatible = "x-powers,axp809", .data = (void *)AXP809_ID },
+	{ .compatible = "x-powers,axp813", .data = (void *)AXP813_ID },
 	{ },
 };
 MODULE_DEVICE_TABLE(of, axp20x_rsb_of_match);
diff --git a/drivers/mfd/axp20x.c b/drivers/mfd/axp20x.c
index 917b6ddc4f15..336de66ca408 100644
--- a/drivers/mfd/axp20x.c
+++ b/drivers/mfd/axp20x.c
@@ -44,6 +44,7 @@ static const char * const axp20x_model_names[] = {
 	"AXP803",
 	"AXP806",
 	"AXP809",
+	"AXP813",
 };
 
 static const struct regmap_range axp152_writeable_ranges[] = {
@@ -676,7 +677,7 @@ static struct mfd_cell axp20x_cells[] = {
 
 static struct mfd_cell axp221_cells[] = {
 	{
-		.name		= "axp20x-pek",
+		.name		= "axp221-pek",
 		.num_resources	= ARRAY_SIZE(axp22x_pek_resources),
 		.resources	= axp22x_pek_resources,
 	}, {
@@ -701,7 +702,7 @@ static struct mfd_cell axp221_cells[] = {
 
 static struct mfd_cell axp223_cells[] = {
 	{
-		.name			= "axp20x-pek",
+		.name			= "axp221-pek",
 		.num_resources		= ARRAY_SIZE(axp22x_pek_resources),
 		.resources		= axp22x_pek_resources,
 	}, {
@@ -834,7 +835,7 @@ static struct mfd_cell axp288_cells[] = {
 		.resources = axp288_fuel_gauge_resources,
 	},
 	{
-		.name = "axp20x-pek",
+		.name = "axp221-pek",
 		.num_resources = ARRAY_SIZE(axp288_power_button_resources),
 		.resources = axp288_power_button_resources,
 	},
@@ -845,7 +846,7 @@ static struct mfd_cell axp288_cells[] = {
 
 static struct mfd_cell axp803_cells[] = {
 	{
-		.name			= "axp20x-pek",
+		.name			= "axp221-pek",
 		.num_resources		= ARRAY_SIZE(axp803_pek_resources),
 		.resources		= axp803_pek_resources,
 	},
@@ -861,7 +862,7 @@ static struct mfd_cell axp806_cells[] = {
 
 static struct mfd_cell axp809_cells[] = {
 	{
-		.name			= "axp20x-pek",
+		.name			= "axp221-pek",
 		.num_resources		= ARRAY_SIZE(axp809_pek_resources),
 		.resources		= axp809_pek_resources,
 	}, {
@@ -870,6 +871,14 @@ static struct mfd_cell axp809_cells[] = {
 	},
 };
 
+static struct mfd_cell axp813_cells[] = {
+	{
+		.name			= "axp221-pek",
+		.num_resources		= ARRAY_SIZE(axp803_pek_resources),
+		.resources		= axp803_pek_resources,
+	}
+};
+
 static struct axp20x_dev *axp20x_pm_power_off;
 static void axp20x_power_off(void)
 {
@@ -956,6 +965,19 @@ int axp20x_match_device(struct axp20x_dev *axp20x)
 		axp20x->regmap_cfg = &axp22x_regmap_config;
 		axp20x->regmap_irq_chip = &axp809_regmap_irq_chip;
 		break;
+	case AXP813_ID:
+		axp20x->nr_cells = ARRAY_SIZE(axp813_cells);
+		axp20x->cells = axp813_cells;
+		axp20x->regmap_cfg = &axp288_regmap_config;
+		/*
+		 * The IRQ table given in the datasheet is incorrect.
+		 * In IRQ enable/status registers 1, there are separate
+		 * IRQs for ACIN and VBUS, instead of bits [7:5] being
+		 * the same as bits [4:2]. So it shares the same IRQs
+		 * as the AXP803, rather than the AXP288.
+		 */
+		axp20x->regmap_irq_chip = &axp803_regmap_irq_chip;
+		break;
 	default:
 		dev_err(dev, "unsupported AXP20X ID %lu\n", axp20x->variant);
 		return -EINVAL;
diff --git a/drivers/mfd/bd9571mwv.c b/drivers/mfd/bd9571mwv.c
new file mode 100644
index 000000000000..64e088dfe7b0
--- /dev/null
+++ b/drivers/mfd/bd9571mwv.c
@@ -0,0 +1,230 @@
+/*
+ * ROHM BD9571MWV-M MFD driver
+ *
+ * Copyright (C) 2017 Marek Vasut <marek.vasut+renesas@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether expressed or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License version 2 for more details.
+ *
+ * Based on the TPS65086 driver
+ */
+
+#include <linux/i2c.h>
+#include <linux/interrupt.h>
+#include <linux/mfd/core.h>
+#include <linux/module.h>
+
+#include <linux/mfd/bd9571mwv.h>
+
+static const struct mfd_cell bd9571mwv_cells[] = {
+	{ .name = "bd9571mwv-regulator", },
+	{ .name = "bd9571mwv-gpio", },
+};
+
+static const struct regmap_range bd9571mwv_readable_yes_ranges[] = {
+	regmap_reg_range(BD9571MWV_VENDOR_CODE, BD9571MWV_PRODUCT_REVISION),
+	regmap_reg_range(BD9571MWV_AVS_SET_MONI, BD9571MWV_AVS_DVFS_VID(3)),
+	regmap_reg_range(BD9571MWV_VD18_VID, BD9571MWV_VD33_VID),
+	regmap_reg_range(BD9571MWV_DVFS_VINIT, BD9571MWV_DVFS_VINIT),
+	regmap_reg_range(BD9571MWV_DVFS_SETVMAX, BD9571MWV_DVFS_MONIVDAC),
+	regmap_reg_range(BD9571MWV_GPIO_IN, BD9571MWV_GPIO_IN),
+	regmap_reg_range(BD9571MWV_GPIO_INT, BD9571MWV_GPIO_INTMASK),
+	regmap_reg_range(BD9571MWV_INT_INTREQ, BD9571MWV_INT_INTMASK),
+};
+
+static const struct regmap_access_table bd9571mwv_readable_table = {
+	.yes_ranges	= bd9571mwv_readable_yes_ranges,
+	.n_yes_ranges	= ARRAY_SIZE(bd9571mwv_readable_yes_ranges),
+};
+
+static const struct regmap_range bd9571mwv_writable_yes_ranges[] = {
+	regmap_reg_range(BD9571MWV_AVS_VD09_VID(0), BD9571MWV_AVS_VD09_VID(3)),
+	regmap_reg_range(BD9571MWV_DVFS_SETVID, BD9571MWV_DVFS_SETVID),
+	regmap_reg_range(BD9571MWV_GPIO_DIR, BD9571MWV_GPIO_OUT),
+	regmap_reg_range(BD9571MWV_GPIO_INT_SET, BD9571MWV_GPIO_INTMASK),
+	regmap_reg_range(BD9571MWV_INT_INTREQ, BD9571MWV_INT_INTMASK),
+};
+
+static const struct regmap_access_table bd9571mwv_writable_table = {
+	.yes_ranges	= bd9571mwv_writable_yes_ranges,
+	.n_yes_ranges	= ARRAY_SIZE(bd9571mwv_writable_yes_ranges),
+};
+
+static const struct regmap_range bd9571mwv_volatile_yes_ranges[] = {
+	regmap_reg_range(BD9571MWV_GPIO_IN, BD9571MWV_GPIO_IN),
+	regmap_reg_range(BD9571MWV_GPIO_INT, BD9571MWV_GPIO_INT),
+	regmap_reg_range(BD9571MWV_INT_INTREQ, BD9571MWV_INT_INTREQ),
+};
+
+static const struct regmap_access_table bd9571mwv_volatile_table = {
+	.yes_ranges	= bd9571mwv_volatile_yes_ranges,
+	.n_yes_ranges	= ARRAY_SIZE(bd9571mwv_volatile_yes_ranges),
+};
+
+static const struct regmap_config bd9571mwv_regmap_config = {
+	.reg_bits	= 8,
+	.val_bits	= 8,
+	.cache_type	= REGCACHE_RBTREE,
+	.rd_table	= &bd9571mwv_readable_table,
+	.wr_table	= &bd9571mwv_writable_table,
+	.volatile_table	= &bd9571mwv_volatile_table,
+	.max_register	= 0xff,
+};
+
+static const struct regmap_irq bd9571mwv_irqs[] = {
+	REGMAP_IRQ_REG(BD9571MWV_IRQ_MD1, 0,
+		       BD9571MWV_INT_INTREQ_MD1_INT),
+	REGMAP_IRQ_REG(BD9571MWV_IRQ_MD2_E1, 0,
+		       BD9571MWV_INT_INTREQ_MD2_E1_INT),
+	REGMAP_IRQ_REG(BD9571MWV_IRQ_MD2_E2, 0,
+		       BD9571MWV_INT_INTREQ_MD2_E2_INT),
+	REGMAP_IRQ_REG(BD9571MWV_IRQ_PROT_ERR, 0,
+		       BD9571MWV_INT_INTREQ_PROT_ERR_INT),
+	REGMAP_IRQ_REG(BD9571MWV_IRQ_GP, 0,
+		       BD9571MWV_INT_INTREQ_GP_INT),
+	REGMAP_IRQ_REG(BD9571MWV_IRQ_128H_OF, 0,
+		       BD9571MWV_INT_INTREQ_128H_OF_INT),
+	REGMAP_IRQ_REG(BD9571MWV_IRQ_WDT_OF, 0,
+		       BD9571MWV_INT_INTREQ_WDT_OF_INT),
+	REGMAP_IRQ_REG(BD9571MWV_IRQ_BKUP_TRG, 0,
+		       BD9571MWV_INT_INTREQ_BKUP_TRG_INT),
+};
+
+static struct regmap_irq_chip bd9571mwv_irq_chip = {
+	.name		= "bd9571mwv",
+	.status_base	= BD9571MWV_INT_INTREQ,
+	.mask_base	= BD9571MWV_INT_INTMASK,
+	.ack_base	= BD9571MWV_INT_INTREQ,
+	.init_ack_masked = true,
+	.num_regs	= 1,
+	.irqs		= bd9571mwv_irqs,
+	.num_irqs	= ARRAY_SIZE(bd9571mwv_irqs),
+};
+
+static int bd9571mwv_identify(struct bd9571mwv *bd)
+{
+	struct device *dev = bd->dev;
+	unsigned int value;
+	int ret;
+
+	ret = regmap_read(bd->regmap, BD9571MWV_VENDOR_CODE, &value);
+	if (ret) {
+		dev_err(dev, "Failed to read vendor code register (ret=%i)\n",
+			ret);
+		return ret;
+	}
+
+	if (value != BD9571MWV_VENDOR_CODE_VAL) {
+		dev_err(dev, "Invalid vendor code ID %02x (expected %02x)\n",
+			value, BD9571MWV_VENDOR_CODE_VAL);
+		return -EINVAL;
+	}
+
+	ret = regmap_read(bd->regmap, BD9571MWV_PRODUCT_CODE, &value);
+	if (ret) {
+		dev_err(dev, "Failed to read product code register (ret=%i)\n",
+			ret);
+		return ret;
+	}
+
+	if (value != BD9571MWV_PRODUCT_CODE_VAL) {
+		dev_err(dev, "Invalid product code ID %02x (expected %02x)\n",
+			value, BD9571MWV_PRODUCT_CODE_VAL);
+		return -EINVAL;
+	}
+
+	ret = regmap_read(bd->regmap, BD9571MWV_PRODUCT_REVISION, &value);
+	if (ret) {
+		dev_err(dev, "Failed to read revision register (ret=%i)\n",
+			ret);
+		return ret;
+	}
+
+	dev_info(dev, "Device: BD9571MWV rev. %d\n", value & 0xff);
+
+	return 0;
+}
+
+static int bd9571mwv_probe(struct i2c_client *client,
+			  const struct i2c_device_id *ids)
+{
+	struct bd9571mwv *bd;
+	int ret;
+
+	bd = devm_kzalloc(&client->dev, sizeof(*bd), GFP_KERNEL);
+	if (!bd)
+		return -ENOMEM;
+
+	i2c_set_clientdata(client, bd);
+	bd->dev = &client->dev;
+	bd->irq = client->irq;
+
+	bd->regmap = devm_regmap_init_i2c(client, &bd9571mwv_regmap_config);
+	if (IS_ERR(bd->regmap)) {
+		dev_err(bd->dev, "Failed to initialize register map\n");
+		return PTR_ERR(bd->regmap);
+	}
+
+	ret = bd9571mwv_identify(bd);
+	if (ret)
+		return ret;
+
+	ret = regmap_add_irq_chip(bd->regmap, bd->irq, IRQF_ONESHOT, 0,
+				  &bd9571mwv_irq_chip, &bd->irq_data);
+	if (ret) {
+		dev_err(bd->dev, "Failed to register IRQ chip\n");
+		return ret;
+	}
+
+	ret = mfd_add_devices(bd->dev, PLATFORM_DEVID_AUTO, bd9571mwv_cells,
+			      ARRAY_SIZE(bd9571mwv_cells), NULL, 0,
+			      regmap_irq_get_domain(bd->irq_data));
+	if (ret) {
+		regmap_del_irq_chip(bd->irq, bd->irq_data);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int bd9571mwv_remove(struct i2c_client *client)
+{
+	struct bd9571mwv *bd = i2c_get_clientdata(client);
+
+	regmap_del_irq_chip(bd->irq, bd->irq_data);
+
+	return 0;
+}
+
+static const struct of_device_id bd9571mwv_of_match_table[] = {
+	{ .compatible = "rohm,bd9571mwv", },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, bd9571mwv_of_match_table);
+
+static const struct i2c_device_id bd9571mwv_id_table[] = {
+	{ "bd9571mwv", 0 },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(i2c, bd9571mwv_id_table);
+
+static struct i2c_driver bd9571mwv_driver = {
+	.driver		= {
+		.name	= "bd9571mwv",
+		.of_match_table = bd9571mwv_of_match_table,
+	},
+	.probe		= bd9571mwv_probe,
+	.remove		= bd9571mwv_remove,
+	.id_table       = bd9571mwv_id_table,
+};
+module_i2c_driver(bd9571mwv_driver);
+
+MODULE_AUTHOR("Marek Vasut <marek.vasut+renesas@gmail.com>");
+MODULE_DESCRIPTION("BD9571MWV PMIC Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/mfd/da9052-core.c b/drivers/mfd/da9052-core.c
index a88c2065d8ab..433add43a0a9 100644
--- a/drivers/mfd/da9052-core.c
+++ b/drivers/mfd/da9052-core.c
@@ -18,6 +18,7 @@
 #include <linux/mfd/core.h>
 #include <linux/slab.h>
 #include <linux/module.h>
+#include <linux/property.h>
 
 #include <linux/mfd/da9052/da9052.h>
 #include <linux/mfd/da9052/pdata.h>
@@ -386,6 +387,8 @@ int da9052_adc_manual_read(struct da9052 *da9052, unsigned char channel)
 
 	mutex_lock(&da9052->auxadc_lock);
 
+	reinit_completion(&da9052->done);
+
 	/* Channel gets activated on enabling the Conversion bit */
 	mux_sel = chan_mux[channel] | DA9052_ADC_MAN_MAN_CONV;
 
@@ -519,9 +522,6 @@ static const struct mfd_cell da9052_subdev_info[] = {
 		.name = "da9052-wled3",
 	},
 	{
-		.name = "da9052-tsi",
-	},
-	{
 		.name = "da9052-bat",
 	},
 	{
@@ -529,6 +529,10 @@ static const struct mfd_cell da9052_subdev_info[] = {
 	},
 };
 
+static const struct mfd_cell da9052_tsi_subdev_info[] = {
+	{ .name = "da9052-tsi" },
+};
+
 const struct regmap_config da9052_regmap_config = {
 	.reg_bits = 8,
 	.val_bits = 8,
@@ -619,9 +623,27 @@ int da9052_device_init(struct da9052 *da9052, u8 chip_id)
 		goto err;
 	}
 
+	/*
+	 * Check if touchscreen pins are used are analogue input instead
+	 * of having a touchscreen connected to them. The analogue input
+	 * functionality will be provided by hwmon driver (if enabled).
+	 */
+	if (!device_property_read_bool(da9052->dev, "dlg,tsi-as-adc")) {
+		ret = mfd_add_devices(da9052->dev, PLATFORM_DEVID_AUTO,
+				      da9052_tsi_subdev_info,
+				      ARRAY_SIZE(da9052_tsi_subdev_info),
+				      NULL, 0, NULL);
+		if (ret) {
+			dev_err(da9052->dev, "failed to add TSI subdev: %d\n",
+				ret);
+			goto err;
+		}
+	}
+
 	return 0;
 
 err:
+	mfd_remove_devices(da9052->dev);
 	da9052_irq_exit(da9052);
 
 	return ret;
diff --git a/drivers/mfd/da9052-spi.c b/drivers/mfd/da9052-spi.c
index b9ea1b27db64..abfb11818fdc 100644
--- a/drivers/mfd/da9052-spi.c
+++ b/drivers/mfd/da9052-spi.c
@@ -67,7 +67,7 @@ static int da9052_spi_remove(struct spi_device *spi)
 	return 0;
 }
 
-static struct spi_device_id da9052_spi_id[] = {
+static const struct spi_device_id da9052_spi_id[] = {
 	{"da9052", DA9052},
 	{"da9053-aa", DA9053_AA},
 	{"da9053-ba", DA9053_BA},
diff --git a/drivers/mfd/da9055-i2c.c b/drivers/mfd/da9055-i2c.c
index b53e100f577c..8169a5c2fa20 100644
--- a/drivers/mfd/da9055-i2c.c
+++ b/drivers/mfd/da9055-i2c.c
@@ -62,7 +62,7 @@ static int da9055_i2c_remove(struct i2c_client *i2c)
  * purposes separate). As a result there are specific DA9055 ids for PMIC
  * and CODEC, which must be different to operate together.
  */
-static struct i2c_device_id da9055_i2c_id[] = {
+static const struct i2c_device_id da9055_i2c_id[] = {
 	{"da9055-pmic", 0},
 	{ }
 };
diff --git a/drivers/mfd/da9062-core.c b/drivers/mfd/da9062-core.c
index fbe0f245ce8e..fe1811523e4a 100644
--- a/drivers/mfd/da9062-core.c
+++ b/drivers/mfd/da9062-core.c
@@ -645,6 +645,9 @@ static const struct regmap_range da9062_aa_readable_ranges[] = {
 		.range_min = DA9062AA_VLDO1_B,
 		.range_max = DA9062AA_VLDO4_B,
 	}, {
+		.range_min = DA9062AA_BBAT_CONT,
+		.range_max = DA9062AA_BBAT_CONT,
+	}, {
 		.range_min = DA9062AA_INTERFACE,
 		.range_max = DA9062AA_CONFIG_E,
 	}, {
@@ -721,6 +724,9 @@ static const struct regmap_range da9062_aa_writeable_ranges[] = {
 		.range_min = DA9062AA_VLDO1_B,
 		.range_max = DA9062AA_VLDO4_B,
 	}, {
+		.range_min = DA9062AA_BBAT_CONT,
+		.range_max = DA9062AA_BBAT_CONT,
+	}, {
 		.range_min = DA9062AA_GP_ID_0,
 		.range_max = DA9062AA_GP_ID_19,
 	},
diff --git a/drivers/mfd/db8500-prcmu.c b/drivers/mfd/db8500-prcmu.c
index 5c739ac752e8..5970b8def548 100644
--- a/drivers/mfd/db8500-prcmu.c
+++ b/drivers/mfd/db8500-prcmu.c
@@ -33,7 +33,6 @@
 #include <linux/mfd/abx500/ab8500.h>
 #include <linux/regulator/db8500-prcmu.h>
 #include <linux/regulator/machine.h>
-#include <linux/cpufreq.h>
 #include <linux/platform_data/ux500_wdt.h>
 #include <linux/platform_data/db8500_thermal.h>
 #include "dbx500-prcmu-regs.h"
@@ -1692,32 +1691,27 @@ static long round_clock_rate(u8 clock, unsigned long rate)
 	return rounded_rate;
 }
 
-/* CPU FREQ table, may be changed due to if MAX_OPP is supported. */
-static struct cpufreq_frequency_table db8500_cpufreq_table[] = {
-	{ .frequency = 200000, .driver_data = ARM_EXTCLK,},
-	{ .frequency = 400000, .driver_data = ARM_50_OPP,},
-	{ .frequency = 800000, .driver_data = ARM_100_OPP,},
-	{ .frequency = CPUFREQ_TABLE_END,}, /* To be used for MAX_OPP. */
-	{ .frequency = CPUFREQ_TABLE_END,},
+static const unsigned long armss_freqs[] = {
+	200000000,
+	400000000,
+	800000000,
+	998400000
 };
 
 static long round_armss_rate(unsigned long rate)
 {
-	struct cpufreq_frequency_table *pos;
-	long freq = 0;
-
-	/* cpufreq table frequencies is in KHz. */
-	rate = rate / 1000;
+	unsigned long freq = 0;
+	int i;
 
 	/* Find the corresponding arm opp from the cpufreq table. */
-	cpufreq_for_each_entry(pos, db8500_cpufreq_table) {
-		freq = pos->frequency;
-		if (freq == rate)
+	for (i = 0; i < ARRAY_SIZE(armss_freqs); i++) {
+		freq = armss_freqs[i];
+		if (rate <= freq)
 			break;
 	}
 
 	/* Return the last valid value, even if a match was not found. */
-	return freq * 1000;
+	return freq;
 }
 
 #define MIN_PLL_VCO_RATE 600000000ULL
@@ -1854,21 +1848,23 @@ static void set_clock_rate(u8 clock, unsigned long rate)
 
 static int set_armss_rate(unsigned long rate)
 {
-	struct cpufreq_frequency_table *pos;
-
-	/* cpufreq table frequencies is in KHz. */
-	rate = rate / 1000;
+	unsigned long freq;
+	u8 opps[] = { ARM_EXTCLK, ARM_50_OPP, ARM_100_OPP, ARM_MAX_OPP };
+	int i;
 
 	/* Find the corresponding arm opp from the cpufreq table. */
-	cpufreq_for_each_entry(pos, db8500_cpufreq_table)
-		if (pos->frequency == rate)
+	for (i = 0; i < ARRAY_SIZE(armss_freqs); i++) {
+		freq = armss_freqs[i];
+		if (rate == freq)
 			break;
+	}
 
-	if (pos->frequency != rate)
+	if (rate != freq)
 		return -EINVAL;
 
 	/* Set the new arm opp. */
-	return db8500_prcmu_set_arm_opp(pos->driver_data);
+	pr_debug("SET ARM OPP 0x%02x\n", opps[i]);
+	return db8500_prcmu_set_arm_opp(opps[i]);
 }
 
 static int set_plldsi_rate(unsigned long rate)
@@ -3049,12 +3045,6 @@ static const struct mfd_cell db8500_prcmu_devs[] = {
 		.pdata_size = sizeof(db8500_regulators),
 	},
 	{
-		.name = "cpufreq-ux500",
-		.of_compatible = "stericsson,cpufreq-ux500",
-		.platform_data = &db8500_cpufreq_table,
-		.pdata_size = sizeof(db8500_cpufreq_table),
-	},
-	{
 		.name = "cpuidle-dbx500",
 		.of_compatible = "stericsson,cpuidle-dbx500",
 	},
@@ -3067,14 +3057,6 @@ static const struct mfd_cell db8500_prcmu_devs[] = {
 	},
 };
 
-static void db8500_prcmu_update_cpufreq(void)
-{
-	if (prcmu_has_arm_maxopp()) {
-		db8500_cpufreq_table[3].frequency = 1000000;
-		db8500_cpufreq_table[3].driver_data = ARM_MAX_OPP;
-	}
-}
-
 static int db8500_prcmu_register_ab8500(struct device *parent)
 {
 	struct device_node *np;
@@ -3160,8 +3142,6 @@ static int db8500_prcmu_probe(struct platform_device *pdev)
 
 	prcmu_config_esram0_deep_sleep(ESRAM0_DEEP_SLEEP_STATE_RET);
 
-	db8500_prcmu_update_cpufreq();
-
 	err = mfd_add_devices(&pdev->dev, 0, common_prcmu_devs,
 			      ARRAY_SIZE(common_prcmu_devs), NULL, 0, db8500_irq_domain);
 	if (err) {
diff --git a/drivers/mfd/dm355evm_msp.c b/drivers/mfd/dm355evm_msp.c
index 86eca614507b..2a2756709f22 100644
--- a/drivers/mfd/dm355evm_msp.c
+++ b/drivers/mfd/dm355evm_msp.c
@@ -18,7 +18,7 @@
 #include <linux/gpio.h>
 #include <linux/leds.h>
 #include <linux/i2c.h>
-#include <linux/i2c/dm355evm_msp.h>
+#include <linux/mfd/dm355evm_msp.h>
 
 
 /*
diff --git a/drivers/mfd/hi6421-pmic-core.c b/drivers/mfd/hi6421-pmic-core.c
index 3fd703fe3aba..6fb7ba272e09 100644
--- a/drivers/mfd/hi6421-pmic-core.c
+++ b/drivers/mfd/hi6421-pmic-core.c
@@ -1,40 +1,35 @@
 /*
- * Device driver for Hi6421 IC
+ * Device driver for Hi6421 PMIC
  *
  * Copyright (c) <2011-2014> HiSilicon Technologies Co., Ltd.
  *              http://www.hisilicon.com
- * Copyright (c) <2013-2014> Linaro Ltd.
+ * Copyright (c) <2013-2017> Linaro Ltd.
  *              http://www.linaro.org
  *
  * Author: Guodong Xu <guodong.xu@linaro.org>
  *
  * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
  */
 
 #include <linux/device.h>
 #include <linux/err.h>
 #include <linux/mfd/core.h>
+#include <linux/mfd/hi6421-pmic.h>
 #include <linux/module.h>
-#include <linux/of.h>
+#include <linux/of_device.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
-#include <linux/mfd/hi6421-pmic.h>
 
 static const struct mfd_cell hi6421_devs[] = {
 	{ .name = "hi6421-regulator", },
 };
 
+static const struct mfd_cell hi6421v530_devs[] = {
+	{ .name = "hi6421v530-regulator", },
+};
+
 static const struct regmap_config hi6421_regmap_config = {
 	.reg_bits = 32,
 	.reg_stride = 4,
@@ -42,12 +37,33 @@ static const struct regmap_config hi6421_regmap_config = {
 	.max_register = HI6421_REG_TO_BUS_ADDR(HI6421_REG_MAX),
 };
 
+static const struct of_device_id of_hi6421_pmic_match[] = {
+	{
+		.compatible = "hisilicon,hi6421-pmic",
+		.data = (void *)HI6421
+	},
+	{
+		.compatible = "hisilicon,hi6421v530-pmic",
+		.data = (void *)HI6421_V530
+	},
+	{ },
+};
+MODULE_DEVICE_TABLE(of, of_hi6421_pmic_match);
+
 static int hi6421_pmic_probe(struct platform_device *pdev)
 {
 	struct hi6421_pmic *pmic;
 	struct resource *res;
+	const struct of_device_id *id;
+	const struct mfd_cell *subdevs;
+	enum hi6421_type type;
 	void __iomem *base;
-	int ret;
+	int n_subdevs, ret;
+
+	id = of_match_device(of_hi6421_pmic_match, &pdev->dev);
+	if (!id)
+		return -EINVAL;
+	type = (enum hi6421_type)id->data;
 
 	pmic = devm_kzalloc(&pdev->dev, sizeof(*pmic), GFP_KERNEL);
 	if (!pmic)
@@ -61,41 +77,50 @@ static int hi6421_pmic_probe(struct platform_device *pdev)
 	pmic->regmap = devm_regmap_init_mmio_clk(&pdev->dev, NULL, base,
 						 &hi6421_regmap_config);
 	if (IS_ERR(pmic->regmap)) {
-		dev_err(&pdev->dev,
-			"regmap init failed: %ld\n", PTR_ERR(pmic->regmap));
+		dev_err(&pdev->dev, "Failed to initialise Regmap: %ld\n",
+						PTR_ERR(pmic->regmap));
 		return PTR_ERR(pmic->regmap);
 	}
 
-	/* set over-current protection debounce 8ms */
-	regmap_update_bits(pmic->regmap, HI6421_OCP_DEB_CTRL_REG,
+	platform_set_drvdata(pdev, pmic);
+
+	switch (type) {
+	case HI6421:
+		/* set over-current protection debounce 8ms */
+		regmap_update_bits(pmic->regmap, HI6421_OCP_DEB_CTRL_REG,
 				(HI6421_OCP_DEB_SEL_MASK
 				 | HI6421_OCP_EN_DEBOUNCE_MASK
 				 | HI6421_OCP_AUTO_STOP_MASK),
 				(HI6421_OCP_DEB_SEL_8MS
 				 | HI6421_OCP_EN_DEBOUNCE_ENABLE));
 
-	platform_set_drvdata(pdev, pmic);
+		subdevs = hi6421_devs;
+		n_subdevs = ARRAY_SIZE(hi6421_devs);
+		break;
+	case HI6421_V530:
+		subdevs = hi6421v530_devs;
+		n_subdevs = ARRAY_SIZE(hi6421v530_devs);
+		break;
+	default:
+		dev_err(&pdev->dev, "Unknown device type %d\n",
+						(unsigned int)type);
+		return -EINVAL;
+	}
 
-	ret = devm_mfd_add_devices(&pdev->dev, 0, hi6421_devs,
-				   ARRAY_SIZE(hi6421_devs), NULL, 0, NULL);
+	ret = devm_mfd_add_devices(&pdev->dev, PLATFORM_DEVID_NONE,
+				   subdevs, n_subdevs, NULL, 0, NULL);
 	if (ret) {
-		dev_err(&pdev->dev, "add mfd devices failed: %d\n", ret);
+		dev_err(&pdev->dev, "Failed to add child devices: %d\n", ret);
 		return ret;
 	}
 
 	return 0;
 }
 
-static const struct of_device_id of_hi6421_pmic_match_tbl[] = {
-	{ .compatible = "hisilicon,hi6421-pmic", },
-	{ },
-};
-MODULE_DEVICE_TABLE(of, of_hi6421_pmic_match_tbl);
-
 static struct platform_driver hi6421_pmic_driver = {
 	.driver = {
-		.name	= "hi6421_pmic",
-		.of_match_table = of_hi6421_pmic_match_tbl,
+		.name = "hi6421_pmic",
+		.of_match_table = of_hi6421_pmic_match,
 	},
 	.probe	= hi6421_pmic_probe,
 };
diff --git a/drivers/mfd/intel-lpss-pci.c b/drivers/mfd/intel-lpss-pci.c
index ad388bb056cd..d1c46de89eb4 100644
--- a/drivers/mfd/intel-lpss-pci.c
+++ b/drivers/mfd/intel-lpss-pci.c
@@ -221,6 +221,7 @@ static const struct pci_device_id intel_lpss_pci_ids[] = {
 	{ PCI_VDEVICE(INTEL, 0xa12a), (kernel_ulong_t)&spt_info },
 	{ PCI_VDEVICE(INTEL, 0xa160), (kernel_ulong_t)&spt_i2c_info },
 	{ PCI_VDEVICE(INTEL, 0xa161), (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa162), (kernel_ulong_t)&spt_i2c_info },
 	{ PCI_VDEVICE(INTEL, 0xa166), (kernel_ulong_t)&spt_uart_info },
 	/* KBL-H */
 	{ PCI_VDEVICE(INTEL, 0xa2a7), (kernel_ulong_t)&spt_uart_info },
diff --git a/drivers/mfd/intel-lpss.c b/drivers/mfd/intel-lpss.c
index 70c646b0097d..0e0ab9bb1530 100644
--- a/drivers/mfd/intel-lpss.c
+++ b/drivers/mfd/intel-lpss.c
@@ -502,6 +502,14 @@ int intel_lpss_suspend(struct device *dev)
 	for (i = 0; i < LPSS_PRIV_REG_COUNT; i++)
 		lpss->priv_ctx[i] = readl(lpss->priv + i * 4);
 
+	/*
+	 * If the device type is not UART, then put the controller into
+	 * reset. UART cannot be put into reset since S3/S0ix fail when
+	 * no_console_suspend flag is enabled.
+	 */
+	if (lpss->type != LPSS_DEV_UART)
+		writel(0, lpss->priv + LPSS_PRIV_RESETS);
+
 	return 0;
 }
 EXPORT_SYMBOL_GPL(intel_lpss_suspend);
diff --git a/drivers/mfd/intel_soc_pmic_core.c b/drivers/mfd/intel_soc_pmic_core.c
index 13737be6df35..36adf9e8153e 100644
--- a/drivers/mfd/intel_soc_pmic_core.c
+++ b/drivers/mfd/intel_soc_pmic_core.c
@@ -16,6 +16,7 @@
  * Author: Zhu, Lejun <lejun.zhu@linux.intel.com>
  */
 
+#include <linux/acpi.h>
 #include <linux/module.h>
 #include <linux/mfd/core.h>
 #include <linux/i2c.h>
@@ -28,6 +29,10 @@
 #include <linux/pwm.h>
 #include "intel_soc_pmic_core.h"
 
+/* Crystal Cove PMIC shares same ACPI ID between different platforms */
+#define BYT_CRC_HRV		2
+#define CHT_CRC_HRV		3
+
 /* Lookup table for the Panel Enable/Disable line as GPIO signals */
 static struct gpiod_lookup_table panel_gpio_table = {
 	/* Intel GFX is consumer */
@@ -48,16 +53,33 @@ static int intel_soc_pmic_i2c_probe(struct i2c_client *i2c,
 				    const struct i2c_device_id *i2c_id)
 {
 	struct device *dev = &i2c->dev;
-	const struct acpi_device_id *id;
 	struct intel_soc_pmic_config *config;
 	struct intel_soc_pmic *pmic;
+	unsigned long long hrv;
+	acpi_status status;
 	int ret;
 
-	id = acpi_match_device(dev->driver->acpi_match_table, dev);
-	if (!id || !id->driver_data)
+	/*
+	 * There are 2 different Crystal Cove PMICs a Bay Trail and Cherry
+	 * Trail version, use _HRV to differentiate between the 2.
+	 */
+	status = acpi_evaluate_integer(ACPI_HANDLE(dev), "_HRV", NULL, &hrv);
+	if (ACPI_FAILURE(status)) {
+		dev_err(dev, "Failed to get PMIC hardware revision\n");
 		return -ENODEV;
-
-	config = (struct intel_soc_pmic_config *)id->driver_data;
+	}
+
+	switch (hrv) {
+	case BYT_CRC_HRV:
+		config = &intel_soc_pmic_config_byt_crc;
+		break;
+	case CHT_CRC_HRV:
+		config = &intel_soc_pmic_config_cht_crc;
+		break;
+	default:
+		dev_warn(dev, "Unknown hardware rev %llu, assuming BYT\n", hrv);
+		config = &intel_soc_pmic_config_byt_crc;
+	}
 
 	pmic = devm_kzalloc(dev, sizeof(*pmic), GFP_KERNEL);
 	if (!pmic)
@@ -157,7 +179,7 @@ MODULE_DEVICE_TABLE(i2c, intel_soc_pmic_i2c_id);
 
 #if defined(CONFIG_ACPI)
 static const struct acpi_device_id intel_soc_pmic_acpi_match[] = {
-	{"INT33FD", (kernel_ulong_t)&intel_soc_pmic_config_crc},
+	{ "INT33FD" },
 	{ },
 };
 MODULE_DEVICE_TABLE(acpi, intel_soc_pmic_acpi_match);
diff --git a/drivers/mfd/intel_soc_pmic_core.h b/drivers/mfd/intel_soc_pmic_core.h
index ff2464bc172f..90a1416d4dac 100644
--- a/drivers/mfd/intel_soc_pmic_core.h
+++ b/drivers/mfd/intel_soc_pmic_core.h
@@ -27,6 +27,7 @@ struct intel_soc_pmic_config {
 	const struct regmap_irq_chip *irq_chip;
 };
 
-extern struct intel_soc_pmic_config intel_soc_pmic_config_crc;
+extern struct intel_soc_pmic_config intel_soc_pmic_config_byt_crc;
+extern struct intel_soc_pmic_config intel_soc_pmic_config_cht_crc;
 
 #endif	/* __INTEL_SOC_PMIC_CORE_H__ */
diff --git a/drivers/mfd/intel_soc_pmic_crc.c b/drivers/mfd/intel_soc_pmic_crc.c
index 4a7494872da2..6d19a6d0fb97 100644
--- a/drivers/mfd/intel_soc_pmic_crc.c
+++ b/drivers/mfd/intel_soc_pmic_crc.c
@@ -80,7 +80,7 @@ static struct resource bcu_resources[] = {
 	},
 };
 
-static struct mfd_cell crystal_cove_dev[] = {
+static struct mfd_cell crystal_cove_byt_dev[] = {
 	{
 		.name = "crystal_cove_pwrsrc",
 		.num_resources = ARRAY_SIZE(pwrsrc_resources),
@@ -114,6 +114,17 @@ static struct mfd_cell crystal_cove_dev[] = {
 	},
 };
 
+static struct mfd_cell crystal_cove_cht_dev[] = {
+	{
+		.name = "crystal_cove_gpio",
+		.num_resources = ARRAY_SIZE(gpio_resources),
+		.resources = gpio_resources,
+	},
+	{
+		.name = "crystal_cove_pwm",
+	},
+};
+
 static const struct regmap_config crystal_cove_regmap_config = {
 	.reg_bits = 8,
 	.val_bits = 8,
@@ -155,10 +166,18 @@ static const struct regmap_irq_chip crystal_cove_irq_chip = {
 	.mask_base = CRYSTAL_COVE_REG_MIRQLVL1,
 };
 
-struct intel_soc_pmic_config intel_soc_pmic_config_crc = {
+struct intel_soc_pmic_config intel_soc_pmic_config_byt_crc = {
+	.irq_flags = IRQF_TRIGGER_RISING,
+	.cell_dev = crystal_cove_byt_dev,
+	.n_cell_devs = ARRAY_SIZE(crystal_cove_byt_dev),
+	.regmap_config = &crystal_cove_regmap_config,
+	.irq_chip = &crystal_cove_irq_chip,
+};
+
+struct intel_soc_pmic_config intel_soc_pmic_config_cht_crc = {
 	.irq_flags = IRQF_TRIGGER_RISING,
-	.cell_dev = crystal_cove_dev,
-	.n_cell_devs = ARRAY_SIZE(crystal_cove_dev),
+	.cell_dev = crystal_cove_cht_dev,
+	.n_cell_devs = ARRAY_SIZE(crystal_cove_cht_dev),
 	.regmap_config = &crystal_cove_regmap_config,
 	.irq_chip = &crystal_cove_irq_chip,
 };
diff --git a/drivers/mfd/kempld-core.c b/drivers/mfd/kempld-core.c
index 895f655780a7..55d824b3a808 100644
--- a/drivers/mfd/kempld-core.c
+++ b/drivers/mfd/kempld-core.c
@@ -494,7 +494,7 @@ static struct platform_driver kempld_driver = {
 	.remove		= kempld_remove,
 };
 
-static struct dmi_system_id kempld_dmi_table[] __initdata = {
+static const struct dmi_system_id kempld_dmi_table[] __initconst = {
 	{
 		.ident = "BBD6",
 		.matches = {
diff --git a/drivers/mfd/lp87565.c b/drivers/mfd/lp87565.c
index 340ad0c63744..32d2a07d4354 100644
--- a/drivers/mfd/lp87565.c
+++ b/drivers/mfd/lp87565.c
@@ -73,10 +73,9 @@ static int lp87565_probe(struct i2c_client *client,
 
 	i2c_set_clientdata(client, lp87565);
 
-	ret = mfd_add_devices(lp87565->dev, PLATFORM_DEVID_AUTO, lp87565_cells,
-			      ARRAY_SIZE(lp87565_cells), NULL, 0, NULL);
-
-	return ret;
+	return devm_mfd_add_devices(lp87565->dev, PLATFORM_DEVID_AUTO,
+				    lp87565_cells, ARRAY_SIZE(lp87565_cells),
+				    NULL, 0, NULL);
 }
 
 static const struct i2c_device_id lp87565_id_table[] = {
diff --git a/drivers/mfd/lpc_ich.c b/drivers/mfd/lpc_ich.c
index 773f1554d2f9..450ae36645aa 100644
--- a/drivers/mfd/lpc_ich.c
+++ b/drivers/mfd/lpc_ich.c
@@ -1119,17 +1119,7 @@ static int lpc_ich_init_spi(struct pci_dev *dev)
 			res->start = spi_base + SPIBASE_LPT;
 			res->end = res->start + SPIBASE_LPT_SZ - 1;
 
-			/*
-			 * Try to make the flash chip writeable now by
-			 * setting BCR_WPD. It it fails we tell the driver
-			 * that it can only read the chip.
-			 */
 			pci_read_config_dword(dev, BCR, &bcr);
-			if (!(bcr & BCR_WPD)) {
-				bcr |= BCR_WPD;
-				pci_write_config_dword(dev, BCR, bcr);
-				pci_read_config_dword(dev, BCR, &bcr);
-			}
 			info->writeable = !!(bcr & BCR_WPD);
 		}
 		break;
diff --git a/drivers/mfd/max8925-i2c.c b/drivers/mfd/max8925-i2c.c
index 5c80aea3211f..10063236132c 100644
--- a/drivers/mfd/max8925-i2c.c
+++ b/drivers/mfd/max8925-i2c.c
@@ -151,7 +151,7 @@ static int max8925_probe(struct i2c_client *client,
 				   const struct i2c_device_id *id)
 {
 	struct max8925_platform_data *pdata = dev_get_platdata(&client->dev);
-	static struct max8925_chip *chip;
+	struct max8925_chip *chip;
 	struct device_node *node = client->dev.of_node;
 
 	if (node && !pdata) {
diff --git a/drivers/mfd/max8998.c b/drivers/mfd/max8998.c
index 4c33b8063bc3..b1d3f70782d9 100644
--- a/drivers/mfd/max8998.c
+++ b/drivers/mfd/max8998.c
@@ -192,10 +192,8 @@ static int max8998_i2c_probe(struct i2c_client *i2c,
 
 	if (IS_ENABLED(CONFIG_OF) && i2c->dev.of_node) {
 		pdata = max8998_i2c_parse_dt_pdata(&i2c->dev);
-		if (IS_ERR(pdata)) {
-			ret = PTR_ERR(pdata);
-			goto err;
-		}
+		if (IS_ERR(pdata))
+			return PTR_ERR(pdata);
 	}
 
 	i2c_set_clientdata(i2c, max8998);
diff --git a/drivers/mfd/omap-usb-tll.c b/drivers/mfd/omap-usb-tll.c
index 6f5300b0eb31..44a5d66314c6 100644
--- a/drivers/mfd/omap-usb-tll.c
+++ b/drivers/mfd/omap-usb-tll.c
@@ -131,12 +131,12 @@ static inline u32 usbtll_read(void __iomem *base, u32 reg)
 	return readl_relaxed(base + reg);
 }
 
-static inline void usbtll_writeb(void __iomem *base, u8 reg, u8 val)
+static inline void usbtll_writeb(void __iomem *base, u32 reg, u8 val)
 {
 	writeb_relaxed(val, base + reg);
 }
 
-static inline u8 usbtll_readb(void __iomem *base, u8 reg)
+static inline u8 usbtll_readb(void __iomem *base, u32 reg)
 {
 	return readb_relaxed(base + reg);
 }
diff --git a/drivers/mfd/retu-mfd.c b/drivers/mfd/retu-mfd.c
index d4c114abeb75..e7d27b7861c1 100644
--- a/drivers/mfd/retu-mfd.c
+++ b/drivers/mfd/retu-mfd.c
@@ -302,15 +302,23 @@ static int retu_remove(struct i2c_client *i2c)
 }
 
 static const struct i2c_device_id retu_id[] = {
-	{ "retu-mfd", 0 },
-	{ "tahvo-mfd", 0 },
+	{ "retu", 0 },
+	{ "tahvo", 0 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, retu_id);
 
+static const struct of_device_id retu_of_match[] = {
+	{ .compatible = "nokia,retu" },
+	{ .compatible = "nokia,tahvo" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, retu_of_match);
+
 static struct i2c_driver retu_driver = {
 	.driver		= {
 		.name = "retu-mfd",
+		.of_match_table = retu_of_match,
 	},
 	.probe		= retu_probe,
 	.remove		= retu_remove,
diff --git a/drivers/mfd/rk808.c b/drivers/mfd/rk808.c
index fd087cbb0bde..216fbf6adec9 100644
--- a/drivers/mfd/rk808.c
+++ b/drivers/mfd/rk808.c
@@ -70,6 +70,14 @@ static const struct regmap_config rk818_regmap_config = {
 	.volatile_reg = rk808_is_volatile_reg,
 };
 
+static const struct regmap_config rk805_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = RK805_OFF_SOURCE_REG,
+	.cache_type = REGCACHE_RBTREE,
+	.volatile_reg = rk808_is_volatile_reg,
+};
+
 static const struct regmap_config rk808_regmap_config = {
 	.reg_bits = 8,
 	.val_bits = 8,
@@ -86,6 +94,34 @@ static struct resource rtc_resources[] = {
 	}
 };
 
+static struct resource rk805_key_resources[] = {
+	{
+		.start  = RK805_IRQ_PWRON_FALL,
+		.end    = RK805_IRQ_PWRON_FALL,
+		.flags  = IORESOURCE_IRQ,
+	},
+	{
+		.start  = RK805_IRQ_PWRON_RISE,
+		.end    = RK805_IRQ_PWRON_RISE,
+		.flags  = IORESOURCE_IRQ,
+	}
+};
+
+static const struct mfd_cell rk805s[] = {
+	{ .name = "rk808-clkout", },
+	{ .name = "rk808-regulator", },
+	{ .name = "rk805-pinctrl", },
+	{
+		.name = "rk808-rtc",
+		.num_resources = ARRAY_SIZE(rtc_resources),
+		.resources = &rtc_resources[0],
+	},
+	{	.name = "rk805-pwrkey",
+		.num_resources = ARRAY_SIZE(rk805_key_resources),
+		.resources = &rk805_key_resources[0],
+	},
+};
+
 static const struct mfd_cell rk808s[] = {
 	{ .name = "rk808-clkout", },
 	{ .name = "rk808-regulator", },
@@ -106,6 +142,20 @@ static const struct mfd_cell rk818s[] = {
 	},
 };
 
+static const struct rk808_reg_data rk805_pre_init_reg[] = {
+	{RK805_BUCK1_CONFIG_REG, RK805_BUCK1_2_ILMAX_MASK,
+				 RK805_BUCK1_2_ILMAX_4000MA},
+	{RK805_BUCK2_CONFIG_REG, RK805_BUCK1_2_ILMAX_MASK,
+				 RK805_BUCK1_2_ILMAX_4000MA},
+	{RK805_BUCK3_CONFIG_REG, RK805_BUCK3_4_ILMAX_MASK,
+				 RK805_BUCK3_ILMAX_3000MA},
+	{RK805_BUCK4_CONFIG_REG, RK805_BUCK3_4_ILMAX_MASK,
+				 RK805_BUCK4_ILMAX_3500MA},
+	{RK805_BUCK4_CONFIG_REG, BUCK_ILMIN_MASK, BUCK_ILMIN_400MA},
+	{RK805_GPIO_IO_POL_REG, SLP_SD_MSK, SLEEP_FUN},
+	{RK805_THERMAL_REG, TEMP_HOTDIE_MSK, TEMP115C},
+};
+
 static const struct rk808_reg_data rk808_pre_init_reg[] = {
 	{ RK808_BUCK3_CONFIG_REG, BUCK_ILMIN_MASK,  BUCK_ILMIN_150MA },
 	{ RK808_BUCK4_CONFIG_REG, BUCK_ILMIN_MASK,  BUCK_ILMIN_200MA },
@@ -135,6 +185,41 @@ static const struct rk808_reg_data rk818_pre_init_reg[] = {
 						    VB_LO_SEL_3500MV },
 };
 
+static const struct regmap_irq rk805_irqs[] = {
+	[RK805_IRQ_PWRON_RISE] = {
+		.mask = RK805_IRQ_PWRON_RISE_MSK,
+		.reg_offset = 0,
+	},
+	[RK805_IRQ_VB_LOW] = {
+		.mask = RK805_IRQ_VB_LOW_MSK,
+		.reg_offset = 0,
+	},
+	[RK805_IRQ_PWRON] = {
+		.mask = RK805_IRQ_PWRON_MSK,
+		.reg_offset = 0,
+	},
+	[RK805_IRQ_PWRON_LP] = {
+		.mask = RK805_IRQ_PWRON_LP_MSK,
+		.reg_offset = 0,
+	},
+	[RK805_IRQ_HOTDIE] = {
+		.mask = RK805_IRQ_HOTDIE_MSK,
+		.reg_offset = 0,
+	},
+	[RK805_IRQ_RTC_ALARM] = {
+		.mask = RK805_IRQ_RTC_ALARM_MSK,
+		.reg_offset = 0,
+	},
+	[RK805_IRQ_RTC_PERIOD] = {
+		.mask = RK805_IRQ_RTC_PERIOD_MSK,
+		.reg_offset = 0,
+	},
+	[RK805_IRQ_PWRON_FALL] = {
+		.mask = RK805_IRQ_PWRON_FALL_MSK,
+		.reg_offset = 0,
+	},
+};
+
 static const struct regmap_irq rk808_irqs[] = {
 	/* INT_STS */
 	[RK808_IRQ_VOUT_LO] = {
@@ -247,6 +332,17 @@ static const struct regmap_irq rk818_irqs[] = {
 	},
 };
 
+static struct regmap_irq_chip rk805_irq_chip = {
+	.name = "rk805",
+	.irqs = rk805_irqs,
+	.num_irqs = ARRAY_SIZE(rk805_irqs),
+	.num_regs = 1,
+	.status_base = RK805_INT_STS_REG,
+	.mask_base = RK805_INT_STS_MSK_REG,
+	.ack_base = RK805_INT_STS_REG,
+	.init_ack_masked = true,
+};
+
 static const struct regmap_irq_chip rk808_irq_chip = {
 	.name = "rk808",
 	.irqs = rk808_irqs,
@@ -272,6 +368,25 @@ static const struct regmap_irq_chip rk818_irq_chip = {
 };
 
 static struct i2c_client *rk808_i2c_client;
+
+static void rk805_device_shutdown(void)
+{
+	int ret;
+	struct rk808 *rk808 = i2c_get_clientdata(rk808_i2c_client);
+
+	if (!rk808) {
+		dev_warn(&rk808_i2c_client->dev,
+			 "have no rk805, so do nothing here\n");
+		return;
+	}
+
+	ret = regmap_update_bits(rk808->regmap,
+				 RK805_DEV_CTRL_REG,
+				 DEV_OFF, DEV_OFF);
+	if (ret)
+		dev_err(&rk808_i2c_client->dev, "power off error!\n");
+}
+
 static void rk808_device_shutdown(void)
 {
 	int ret;
@@ -309,6 +424,7 @@ static void rk818_device_shutdown(void)
 }
 
 static const struct of_device_id rk808_of_match[] = {
+	{ .compatible = "rockchip,rk805" },
 	{ .compatible = "rockchip,rk808" },
 	{ .compatible = "rockchip,rk818" },
 	{ },
@@ -325,7 +441,7 @@ static int rk808_probe(struct i2c_client *client,
 	void (*pm_pwroff_fn)(void);
 	int nr_pre_init_regs;
 	int nr_cells;
-	int pm_off = 0;
+	int pm_off = 0, msb, lsb;
 	int ret;
 	int i;
 
@@ -333,16 +449,34 @@ static int rk808_probe(struct i2c_client *client,
 	if (!rk808)
 		return -ENOMEM;
 
-	rk808->variant = i2c_smbus_read_word_data(client, RK808_ID_MSB);
-	if (rk808->variant < 0) {
-		dev_err(&client->dev, "Failed to read the chip id at 0x%02x\n",
+	/* Read chip variant */
+	msb = i2c_smbus_read_byte_data(client, RK808_ID_MSB);
+	if (msb < 0) {
+		dev_err(&client->dev, "failed to read the chip id at 0x%x\n",
 			RK808_ID_MSB);
-		return rk808->variant;
+		return msb;
+	}
+
+	lsb = i2c_smbus_read_byte_data(client, RK808_ID_LSB);
+	if (lsb < 0) {
+		dev_err(&client->dev, "failed to read the chip id at 0x%x\n",
+			RK808_ID_LSB);
+		return lsb;
 	}
 
-	dev_dbg(&client->dev, "Chip id: 0x%x\n", (unsigned int)rk808->variant);
+	rk808->variant = ((msb << 8) | lsb) & RK8XX_ID_MSK;
+	dev_info(&client->dev, "chip id: 0x%x\n", (unsigned int)rk808->variant);
 
 	switch (rk808->variant) {
+	case RK805_ID:
+		rk808->regmap_cfg = &rk805_regmap_config;
+		rk808->regmap_irq_chip = &rk805_irq_chip;
+		pre_init_reg = rk805_pre_init_reg;
+		nr_pre_init_regs = ARRAY_SIZE(rk805_pre_init_reg);
+		cells = rk805s;
+		nr_cells = ARRAY_SIZE(rk805s);
+		pm_pwroff_fn = rk805_device_shutdown;
+		break;
 	case RK808_ID:
 		rk808->regmap_cfg = &rk808_regmap_config;
 		rk808->regmap_irq_chip = &rk808_irq_chip;
@@ -435,6 +569,7 @@ static int rk808_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id rk808_ids[] = {
+	{ "rk805" },
 	{ "rk808" },
 	{ "rk818" },
 	{ },
diff --git a/drivers/mfd/rtsx_pcr.c b/drivers/mfd/rtsx_pcr.c
index a0ac89dfdf0f..3cf69e5c5703 100644
--- a/drivers/mfd/rtsx_pcr.c
+++ b/drivers/mfd/rtsx_pcr.c
@@ -644,7 +644,7 @@ int rtsx_pci_switch_clock(struct rtsx_pcr *pcr, unsigned int card_clock,
 {
 	int err, clk;
 	u8 n, clk_divider, mcu_cnt, div;
-	u8 depth[] = {
+	static const u8 depth[] = {
 		[RTSX_SSC_DEPTH_4M] = SSC_DEPTH_4M,
 		[RTSX_SSC_DEPTH_2M] = SSC_DEPTH_2M,
 		[RTSX_SSC_DEPTH_1M] = SSC_DEPTH_1M,
@@ -768,7 +768,7 @@ EXPORT_SYMBOL_GPL(rtsx_pci_card_power_off);
 
 int rtsx_pci_card_exclusive_check(struct rtsx_pcr *pcr, int card)
 {
-	unsigned int cd_mask[] = {
+	static const unsigned int cd_mask[] = {
 		[RTSX_SD_CARD] = SD_EXIST,
 		[RTSX_MS_CARD] = MS_EXIST
 	};
diff --git a/drivers/mfd/stm32-lptimer.c b/drivers/mfd/stm32-lptimer.c
new file mode 100644
index 000000000000..075330a25f61
--- /dev/null
+++ b/drivers/mfd/stm32-lptimer.c
@@ -0,0 +1,107 @@
+/*
+ * STM32 Low-Power Timer parent driver.
+ *
+ * Copyright (C) STMicroelectronics 2017
+ *
+ * Author: Fabrice Gasnier <fabrice.gasnier@st.com>
+ *
+ * Inspired by Benjamin Gaignard's stm32-timers driver
+ *
+ * License terms:  GNU General Public License (GPL), version 2
+ */
+
+#include <linux/mfd/stm32-lptimer.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+
+#define STM32_LPTIM_MAX_REGISTER	0x3fc
+
+static const struct regmap_config stm32_lptimer_regmap_cfg = {
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = sizeof(u32),
+	.max_register = STM32_LPTIM_MAX_REGISTER,
+};
+
+static int stm32_lptimer_detect_encoder(struct stm32_lptimer *ddata)
+{
+	u32 val;
+	int ret;
+
+	/*
+	 * Quadrature encoder mode bit can only be written and read back when
+	 * Low-Power Timer supports it.
+	 */
+	ret = regmap_update_bits(ddata->regmap, STM32_LPTIM_CFGR,
+				 STM32_LPTIM_ENC, STM32_LPTIM_ENC);
+	if (ret)
+		return ret;
+
+	ret = regmap_read(ddata->regmap, STM32_LPTIM_CFGR, &val);
+	if (ret)
+		return ret;
+
+	ret = regmap_update_bits(ddata->regmap, STM32_LPTIM_CFGR,
+				 STM32_LPTIM_ENC, 0);
+	if (ret)
+		return ret;
+
+	ddata->has_encoder = !!(val & STM32_LPTIM_ENC);
+
+	return 0;
+}
+
+static int stm32_lptimer_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct stm32_lptimer *ddata;
+	struct resource *res;
+	void __iomem *mmio;
+	int ret;
+
+	ddata = devm_kzalloc(dev, sizeof(*ddata), GFP_KERNEL);
+	if (!ddata)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	mmio = devm_ioremap_resource(dev, res);
+	if (IS_ERR(mmio))
+		return PTR_ERR(mmio);
+
+	ddata->regmap = devm_regmap_init_mmio_clk(dev, "mux", mmio,
+						  &stm32_lptimer_regmap_cfg);
+	if (IS_ERR(ddata->regmap))
+		return PTR_ERR(ddata->regmap);
+
+	ddata->clk = devm_clk_get(dev, NULL);
+	if (IS_ERR(ddata->clk))
+		return PTR_ERR(ddata->clk);
+
+	ret = stm32_lptimer_detect_encoder(ddata);
+	if (ret)
+		return ret;
+
+	platform_set_drvdata(pdev, ddata);
+
+	return devm_of_platform_populate(&pdev->dev);
+}
+
+static const struct of_device_id stm32_lptimer_of_match[] = {
+	{ .compatible = "st,stm32-lptimer", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, stm32_lptimer_of_match);
+
+static struct platform_driver stm32_lptimer_driver = {
+	.probe = stm32_lptimer_probe,
+	.driver = {
+		.name = "stm32-lptimer",
+		.of_match_table = stm32_lptimer_of_match,
+	},
+};
+module_platform_driver(stm32_lptimer_driver);
+
+MODULE_AUTHOR("Fabrice Gasnier <fabrice.gasnier@st.com>");
+MODULE_DESCRIPTION("STMicroelectronics STM32 Low-Power Timer");
+MODULE_ALIAS("platform:stm32-lptimer");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/mfd/t7l66xb.c b/drivers/mfd/t7l66xb.c
index 22c811396edc..43d8683266de 100644
--- a/drivers/mfd/t7l66xb.c
+++ b/drivers/mfd/t7l66xb.c
@@ -86,8 +86,11 @@ static int t7l66xb_mmc_enable(struct platform_device *mmc)
 	struct t7l66xb *t7l66xb = platform_get_drvdata(dev);
 	unsigned long flags;
 	u8 dev_ctl;
+	int ret;
 
-	clk_prepare_enable(t7l66xb->clk32k);
+	ret = clk_prepare_enable(t7l66xb->clk32k);
+	if (ret)
+		return ret;
 
 	raw_spin_lock_irqsave(&t7l66xb->lock, flags);
 
@@ -286,8 +289,12 @@ static int t7l66xb_resume(struct platform_device *dev)
 {
 	struct t7l66xb *t7l66xb = platform_get_drvdata(dev);
 	struct t7l66xb_platform_data *pdata = dev_get_platdata(&dev->dev);
+	int ret;
+
+	ret = clk_prepare_enable(t7l66xb->clk48m);
+	if (ret)
+		return ret;
 
-	clk_prepare_enable(t7l66xb->clk48m);
 	if (pdata && pdata->resume)
 		pdata->resume(dev);
 
@@ -361,7 +368,9 @@ static int t7l66xb_probe(struct platform_device *dev)
 		goto err_ioremap;
 	}
 
-	clk_prepare_enable(t7l66xb->clk48m);
+	ret = clk_prepare_enable(t7l66xb->clk48m);
+	if (ret)
+		goto err_clk_enable;
 
 	if (pdata->enable)
 		pdata->enable(dev);
@@ -386,6 +395,8 @@ static int t7l66xb_probe(struct platform_device *dev)
 		return 0;
 
 	t7l66xb_detach_irq(dev);
+	clk_disable_unprepare(t7l66xb->clk48m);
+err_clk_enable:
 	iounmap(t7l66xb->scr);
 err_ioremap:
 	release_resource(&t7l66xb->rscr);
diff --git a/drivers/mfd/tps6105x.c b/drivers/mfd/tps6105x.c
index baa12ea666fb..187848c93779 100644
--- a/drivers/mfd/tps6105x.c
+++ b/drivers/mfd/tps6105x.c
@@ -173,9 +173,17 @@ static const struct i2c_device_id tps6105x_id[] = {
 };
 MODULE_DEVICE_TABLE(i2c, tps6105x_id);
 
+static const struct of_device_id tps6105x_of_match[] = {
+	{ .compatible = "ti,tps61050" },
+	{ .compatible = "ti,tps61052" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, tps6105x_of_match);
+
 static struct i2c_driver tps6105x_driver = {
 	.driver = {
 		.name	= "tps6105x",
+		.of_match_table = tps6105x_of_match,
 	},
 	.probe		= tps6105x_probe,
 	.remove		= tps6105x_remove,
diff --git a/drivers/mfd/tps65010.c b/drivers/mfd/tps65010.c
index d829a6131f09..2ab67386b4ef 100644
--- a/drivers/mfd/tps65010.c
+++ b/drivers/mfd/tps65010.c
@@ -32,7 +32,7 @@
 #include <linux/mutex.h>
 #include <linux/platform_device.h>
 
-#include <linux/i2c/tps65010.h>
+#include <linux/mfd/tps65010.h>
 
 #include <linux/gpio/driver.h>
 
diff --git a/drivers/mfd/tps68470.c b/drivers/mfd/tps68470.c
new file mode 100644
index 000000000000..189efaea054c
--- /dev/null
+++ b/drivers/mfd/tps68470.c
@@ -0,0 +1,106 @@
+/*
+ * TPS68470 chip Parent driver
+ *
+ * Copyright (C) 2017 Intel Corporation
+ *
+ * Authors:
+ *	Rajmohan Mani <rajmohan.mani@intel.com>
+ *	Tianshu Qiu <tian.shu.qiu@intel.com>
+ *	Jian Xu Zheng <jian.xu.zheng@intel.com>
+ *	Yuning Pu <yuning.pu@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/acpi.h>
+#include <linux/delay.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/mfd/core.h>
+#include <linux/mfd/tps68470.h>
+#include <linux/regmap.h>
+
+static const struct mfd_cell tps68470s[] = {
+	{ .name = "tps68470-gpio" },
+	{ .name = "tps68470_pmic_opregion" },
+};
+
+static const struct regmap_config tps68470_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = TPS68470_REG_MAX,
+};
+
+static int tps68470_chip_init(struct device *dev, struct regmap *regmap)
+{
+	unsigned int version;
+	int ret;
+
+	/* Force software reset */
+	ret = regmap_write(regmap, TPS68470_REG_RESET, TPS68470_REG_RESET_MASK);
+	if (ret)
+		return ret;
+
+	ret = regmap_read(regmap, TPS68470_REG_REVID, &version);
+	if (ret) {
+		dev_err(dev, "Failed to read revision register: %d\n", ret);
+		return ret;
+	}
+
+	dev_info(dev, "TPS68470 REVID: 0x%x\n", version);
+
+	return 0;
+}
+
+static int tps68470_probe(struct i2c_client *client)
+{
+	struct device *dev = &client->dev;
+	struct regmap *regmap;
+	int ret;
+
+	regmap = devm_regmap_init_i2c(client, &tps68470_regmap_config);
+	if (IS_ERR(regmap)) {
+		dev_err(dev, "devm_regmap_init_i2c Error %ld\n",
+			PTR_ERR(regmap));
+		return PTR_ERR(regmap);
+	}
+
+	i2c_set_clientdata(client, regmap);
+
+	ret = tps68470_chip_init(dev, regmap);
+	if (ret < 0) {
+		dev_err(dev, "TPS68470 Init Error %d\n", ret);
+		return ret;
+	}
+
+	ret = devm_mfd_add_devices(dev, PLATFORM_DEVID_NONE, tps68470s,
+			      ARRAY_SIZE(tps68470s), NULL, 0, NULL);
+	if (ret < 0) {
+		dev_err(dev, "devm_mfd_add_devices failed: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static const struct acpi_device_id tps68470_acpi_ids[] = {
+	{"INT3472"},
+	{},
+};
+MODULE_DEVICE_TABLE(acpi, tps68470_acpi_ids);
+
+static struct i2c_driver tps68470_driver = {
+	.driver = {
+		   .name = "tps68470",
+		   .acpi_match_table = tps68470_acpi_ids,
+	},
+	.probe_new = tps68470_probe,
+};
+builtin_i2c_driver(tps68470_driver);
diff --git a/drivers/mfd/twl-core.c b/drivers/mfd/twl-core.c
index c64615dca2bd..d3133a371e27 100644
--- a/drivers/mfd/twl-core.c
+++ b/drivers/mfd/twl-core.c
@@ -44,7 +44,7 @@
 #include <linux/regulator/machine.h>
 
 #include <linux/i2c.h>
-#include <linux/i2c/twl.h>
+#include <linux/mfd/twl.h>
 
 /* Register descriptions for audio */
 #include <linux/mfd/twl4030-audio.h>
@@ -173,7 +173,7 @@ static struct twl_private *twl_priv;
 static struct twl_mapping twl4030_map[] = {
 	/*
 	 * NOTE:  don't change this table without updating the
-	 * <linux/i2c/twl.h> defines for TWL4030_MODULE_*
+	 * <linux/mfd/twl.h> defines for TWL4030_MODULE_*
 	 * so they continue to match the order in this table.
 	 */
 
@@ -344,7 +344,7 @@ static const struct regmap_config twl4030_regmap_config[4] = {
 static struct twl_mapping twl6030_map[] = {
 	/*
 	 * NOTE:  don't change this table without updating the
-	 * <linux/i2c/twl.h> defines for TWL4030_MODULE_*
+	 * <linux/mfd/twl.h> defines for TWL4030_MODULE_*
 	 * so they continue to match the order in this table.
 	 */
 
@@ -448,7 +448,7 @@ static struct regmap *twl_get_regmap(u8 mod_no)
  * @reg: register address (just offset will do)
  * @num_bytes: number of bytes to transfer
  *
- * Returns the result of operation - 0 is success
+ * Returns 0 on success or else a negative error code.
  */
 int twl_i2c_write(u8 mod_no, u8 *value, u8 reg, unsigned num_bytes)
 {
@@ -476,7 +476,7 @@ EXPORT_SYMBOL(twl_i2c_write);
  * @reg: register address (just offset will do)
  * @num_bytes: number of bytes to transfer
  *
- * Returns result of operation - num_bytes is success else failure.
+ * Returns 0 on success or else a negative error code.
  */
 int twl_i2c_read(u8 mod_no, u8 *value, u8 reg, unsigned num_bytes)
 {
diff --git a/drivers/mfd/twl4030-audio.c b/drivers/mfd/twl4030-audio.c
index 0a1606480023..da16bf45fab4 100644
--- a/drivers/mfd/twl4030-audio.c
+++ b/drivers/mfd/twl4030-audio.c
@@ -30,7 +30,7 @@
 #include <linux/platform_device.h>
 #include <linux/of.h>
 #include <linux/of_platform.h>
-#include <linux/i2c/twl.h>
+#include <linux/mfd/twl.h>
 #include <linux/mfd/core.h>
 #include <linux/mfd/twl4030-audio.h>
 
diff --git a/drivers/mfd/twl4030-irq.c b/drivers/mfd/twl4030-irq.c
index 378c02d43bf7..b16c16f194fd 100644
--- a/drivers/mfd/twl4030-irq.c
+++ b/drivers/mfd/twl4030-irq.c
@@ -33,7 +33,7 @@
 #include <linux/slab.h>
 #include <linux/of.h>
 #include <linux/irqdomain.h>
-#include <linux/i2c/twl.h>
+#include <linux/mfd/twl.h>
 
 #include "twl-core.h"
 
diff --git a/drivers/mfd/twl4030-power.c b/drivers/mfd/twl4030-power.c
index f4b2c29d77e3..6b36932263ba 100644
--- a/drivers/mfd/twl4030-power.c
+++ b/drivers/mfd/twl4030-power.c
@@ -25,7 +25,7 @@
 
 #include <linux/module.h>
 #include <linux/pm.h>
-#include <linux/i2c/twl.h>
+#include <linux/mfd/twl.h>
 #include <linux/platform_device.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
diff --git a/drivers/mfd/twl6030-irq.c b/drivers/mfd/twl6030-irq.c
index 53574508a613..e3ec8dfa9f1e 100644
--- a/drivers/mfd/twl6030-irq.c
+++ b/drivers/mfd/twl6030-irq.c
@@ -35,7 +35,7 @@
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/kthread.h>
-#include <linux/i2c/twl.h>
+#include <linux/mfd/twl.h>
 #include <linux/platform_device.h>
 #include <linux/suspend.h>
 #include <linux/of.h>
diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile
index b0b766416306..d84819dc2468 100644
--- a/drivers/misc/Makefile
+++ b/drivers/misc/Makefile
@@ -60,6 +60,7 @@ lkdtm-$(CONFIG_LKDTM)		+= lkdtm_core.o
 lkdtm-$(CONFIG_LKDTM)		+= lkdtm_bugs.o
 lkdtm-$(CONFIG_LKDTM)		+= lkdtm_heap.o
 lkdtm-$(CONFIG_LKDTM)		+= lkdtm_perms.o
+lkdtm-$(CONFIG_LKDTM)		+= lkdtm_refcount.o
 lkdtm-$(CONFIG_LKDTM)		+= lkdtm_rodata_objcopy.o
 lkdtm-$(CONFIG_LKDTM)		+= lkdtm_usercopy.o
 
diff --git a/drivers/misc/apds9802als.c b/drivers/misc/apds9802als.c
index c6cc3dc8ae1f..d8ac036f01ab 100644
--- a/drivers/misc/apds9802als.c
+++ b/drivers/misc/apds9802als.c
@@ -197,7 +197,7 @@ static struct attribute *mid_att_als[] = {
 	NULL
 };
 
-static struct attribute_group m_als_gr = {
+static const struct attribute_group m_als_gr = {
 	.name = "apds9802als",
 	.attrs = mid_att_als
 };
@@ -298,7 +298,7 @@ static UNIVERSAL_DEV_PM_OPS(apds9802als_pm_ops, apds9802als_suspend,
 #define APDS9802ALS_PM_OPS NULL
 #endif	/* CONFIG_PM */
 
-static struct i2c_device_id apds9802als_id[] = {
+static const struct i2c_device_id apds9802als_id[] = {
 	{ DRIVER_NAME, 0 },
 	{ }
 };
diff --git a/drivers/misc/apds990x.c b/drivers/misc/apds990x.c
index 84e5b949399e..c9f07032c2fc 100644
--- a/drivers/misc/apds990x.c
+++ b/drivers/misc/apds990x.c
@@ -1051,7 +1051,7 @@ static struct attribute *sysfs_attrs_ctrl[] = {
 	NULL
 };
 
-static struct attribute_group apds990x_attribute_group[] = {
+static const struct attribute_group apds990x_attribute_group[] = {
 	{.attrs = sysfs_attrs_ctrl },
 };
 
diff --git a/drivers/misc/aspeed-lpc-snoop.c b/drivers/misc/aspeed-lpc-snoop.c
index 593905565b74..cb78c98bc78d 100644
--- a/drivers/misc/aspeed-lpc-snoop.c
+++ b/drivers/misc/aspeed-lpc-snoop.c
@@ -20,6 +20,7 @@
 #include <linux/mfd/syscon.h>
 #include <linux/module.h>
 #include <linux/of.h>
+#include <linux/of_device.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
 
@@ -51,6 +52,13 @@
 #define HICRB_ENSNP0D		BIT(14)
 #define HICRB_ENSNP1D		BIT(15)
 
+struct aspeed_lpc_snoop_model_data {
+	/* The ast2400 has bits 14 and 15 as reserved, whereas the ast2500
+	 * can use them.
+	 */
+	unsigned int has_hicrb_ensnp;
+};
+
 struct aspeed_lpc_snoop {
 	struct regmap		*regmap;
 	int			irq;
@@ -123,10 +131,13 @@ static int aspeed_lpc_snoop_config_irq(struct aspeed_lpc_snoop *lpc_snoop,
 }
 
 static int aspeed_lpc_enable_snoop(struct aspeed_lpc_snoop *lpc_snoop,
-				  int channel, u16 lpc_port)
+				   struct device *dev,
+				   int channel, u16 lpc_port)
 {
 	int rc = 0;
 	u32 hicr5_en, snpwadr_mask, snpwadr_shift, hicrb_en;
+	const struct aspeed_lpc_snoop_model_data *model_data =
+		of_device_get_match_data(dev);
 
 	/* Create FIFO datastructure */
 	rc = kfifo_alloc(&lpc_snoop->snoop_fifo[channel],
@@ -155,7 +166,9 @@ static int aspeed_lpc_enable_snoop(struct aspeed_lpc_snoop *lpc_snoop,
 	regmap_update_bits(lpc_snoop->regmap, HICR5, hicr5_en, hicr5_en);
 	regmap_update_bits(lpc_snoop->regmap, SNPWADR, snpwadr_mask,
 			   lpc_port << snpwadr_shift);
-	regmap_update_bits(lpc_snoop->regmap, HICRB, hicrb_en, hicrb_en);
+	if (model_data->has_hicrb_ensnp)
+		regmap_update_bits(lpc_snoop->regmap, HICRB,
+				hicrb_en, hicrb_en);
 
 	return rc;
 }
@@ -213,14 +226,14 @@ static int aspeed_lpc_snoop_probe(struct platform_device *pdev)
 	if (rc)
 		return rc;
 
-	rc = aspeed_lpc_enable_snoop(lpc_snoop, 0, port);
+	rc = aspeed_lpc_enable_snoop(lpc_snoop, dev, 0, port);
 	if (rc)
 		return rc;
 
 	/* Configuration of 2nd snoop channel port is optional */
 	if (of_property_read_u32_index(dev->of_node, "snoop-ports",
 				       1, &port) == 0) {
-		rc = aspeed_lpc_enable_snoop(lpc_snoop, 1, port);
+		rc = aspeed_lpc_enable_snoop(lpc_snoop, dev, 1, port);
 		if (rc)
 			aspeed_lpc_disable_snoop(lpc_snoop, 0);
 	}
@@ -239,8 +252,19 @@ static int aspeed_lpc_snoop_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static const struct aspeed_lpc_snoop_model_data ast2400_model_data = {
+	.has_hicrb_ensnp = 0,
+};
+
+static const struct aspeed_lpc_snoop_model_data ast2500_model_data = {
+	.has_hicrb_ensnp = 1,
+};
+
 static const struct of_device_id aspeed_lpc_snoop_match[] = {
-	{ .compatible = "aspeed,ast2500-lpc-snoop" },
+	{ .compatible = "aspeed,ast2400-lpc-snoop",
+	  .data = &ast2400_model_data },
+	{ .compatible = "aspeed,ast2500-lpc-snoop",
+	  .data = &ast2500_model_data },
 	{ },
 };
 
diff --git a/drivers/misc/bh1770glc.c b/drivers/misc/bh1770glc.c
index 38fcfe219d1c..9c62bf064f77 100644
--- a/drivers/misc/bh1770glc.c
+++ b/drivers/misc/bh1770glc.c
@@ -1175,7 +1175,7 @@ static struct attribute *sysfs_attrs[] = {
 	NULL
 };
 
-static struct attribute_group bh1770_attribute_group = {
+static const struct attribute_group bh1770_attribute_group = {
 	.attrs = sysfs_attrs
 };
 
diff --git a/drivers/misc/cxl/api.c b/drivers/misc/cxl/api.c
index 1a138c83f877..a0c44d16bf30 100644
--- a/drivers/misc/cxl/api.c
+++ b/drivers/misc/cxl/api.c
@@ -336,6 +336,10 @@ int cxl_start_context(struct cxl_context *ctx, u64 wed,
 			mmput(ctx->mm);
 	}
 
+	/*
+	 * Increment driver use count. Enables global TLBIs for hash
+	 * and callbacks to handle the segment table
+	 */
 	cxl_ctx_get();
 
 	if ((rc = cxl_ops->attach_process(ctx, kernel, wed, 0))) {
diff --git a/drivers/misc/cxl/fault.c b/drivers/misc/cxl/fault.c
index 6eed7d03e2b5..f17f72ea0545 100644
--- a/drivers/misc/cxl/fault.c
+++ b/drivers/misc/cxl/fault.c
@@ -138,6 +138,22 @@ int cxl_handle_mm_fault(struct mm_struct *mm, u64 dsisr, u64 dar)
 	int result;
 	unsigned long access, flags, inv_flags = 0;
 
+	/*
+	 * Add the fault handling cpu to task mm cpumask so that we
+	 * can do a safe lockless page table walk when inserting the
+	 * hash page table entry. This function get called with a
+	 * valid mm for user space addresses. Hence using the if (mm)
+	 * check is sufficient here.
+	 */
+	if (mm && !cpumask_test_cpu(smp_processor_id(), mm_cpumask(mm))) {
+		cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm));
+		/*
+		 * We need to make sure we walk the table only after
+		 * we update the cpumask. The other side of the barrier
+		 * is explained in serialize_against_pte_lookup()
+		 */
+		smp_mb();
+	}
 	if ((result = copro_handle_mm_fault(mm, dar, dsisr, &flt))) {
 		pr_devel("copro_handle_mm_fault failed: %#x\n", result);
 		return result;
diff --git a/drivers/misc/cxl/file.c b/drivers/misc/cxl/file.c
index 0761271d68c5..4bfad9f6dc9f 100644
--- a/drivers/misc/cxl/file.c
+++ b/drivers/misc/cxl/file.c
@@ -95,7 +95,6 @@ static int __afu_open(struct inode *inode, struct file *file, bool master)
 
 	pr_devel("afu_open pe: %i\n", ctx->pe);
 	file->private_data = ctx;
-	cxl_ctx_get();
 
 	/* indicate success */
 	rc = 0;
@@ -225,6 +224,12 @@ static long afu_ioctl_start_work(struct cxl_context *ctx,
 	if (ctx->mm)
 		mmput(ctx->mm);
 
+	/*
+	 * Increment driver use count. Enables global TLBIs for hash
+	 * and callbacks to handle the segment table
+	 */
+	cxl_ctx_get();
+
 	trace_cxl_attach(ctx, work.work_element_descriptor, work.num_interrupts, amr);
 
 	if ((rc = cxl_ops->attach_process(ctx, false, work.work_element_descriptor,
@@ -233,6 +238,7 @@ static long afu_ioctl_start_work(struct cxl_context *ctx,
 		cxl_adapter_context_put(ctx->afu->adapter);
 		put_pid(ctx->pid);
 		ctx->pid = NULL;
+		cxl_ctx_put();
 		cxl_context_mm_count_put(ctx);
 		goto out;
 	}
diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c
index d18b3d9292fd..3ba04f371380 100644
--- a/drivers/misc/cxl/pci.c
+++ b/drivers/misc/cxl/pci.c
@@ -1279,7 +1279,7 @@ ssize_t cxl_pci_afu_read_err_buffer(struct cxl_afu *afu, char *buf,
 	}
 
 	/* use bounce buffer for copy */
-	tbuf = (void *)__get_free_page(GFP_TEMPORARY);
+	tbuf = (void *)__get_free_page(GFP_KERNEL);
 	if (!tbuf)
 		return -ENOMEM;
 
diff --git a/drivers/misc/ds1682.c b/drivers/misc/ds1682.c
index 28bb495f0cf1..7231260ac287 100644
--- a/drivers/misc/ds1682.c
+++ b/drivers/misc/ds1682.c
@@ -173,7 +173,7 @@ static ssize_t ds1682_eeprom_write(struct file *filp, struct kobject *kobj,
 	return count;
 }
 
-static struct bin_attribute ds1682_eeprom_attr = {
+static const struct bin_attribute ds1682_eeprom_attr = {
 	.attr = {
 		.name = "eeprom",
 		.mode = S_IRUGO | S_IWUSR,
diff --git a/drivers/misc/eeprom/eeprom.c b/drivers/misc/eeprom/eeprom.c
index 2fad790db3bf..60e3d91b5e03 100644
--- a/drivers/misc/eeprom/eeprom.c
+++ b/drivers/misc/eeprom/eeprom.c
@@ -114,7 +114,7 @@ static ssize_t eeprom_read(struct file *filp, struct kobject *kobj,
 	return count;
 }
 
-static struct bin_attribute eeprom_attr = {
+static const struct bin_attribute eeprom_attr = {
 	.attr = {
 		.name = "eeprom",
 		.mode = S_IRUGO,
diff --git a/drivers/misc/eeprom/eeprom_93xx46.c b/drivers/misc/eeprom/eeprom_93xx46.c
index 94cc035aa841..38766968bfa2 100644
--- a/drivers/misc/eeprom/eeprom_93xx46.c
+++ b/drivers/misc/eeprom/eeprom_93xx46.c
@@ -377,8 +377,6 @@ static int eeprom_93xx46_probe_dt(struct spi_device *spi)
 	struct device_node *np = spi->dev.of_node;
 	struct eeprom_93xx46_platform_data *pd;
 	u32 tmp;
-	int gpio;
-	enum of_gpio_flags of_flags;
 	int ret;
 
 	pd = devm_kzalloc(&spi->dev, sizeof(*pd), GFP_KERNEL);
@@ -403,22 +401,14 @@ static int eeprom_93xx46_probe_dt(struct spi_device *spi)
 	if (of_property_read_bool(np, "read-only"))
 		pd->flags |= EE_READONLY;
 
-	gpio = of_get_named_gpio_flags(np, "select-gpios", 0, &of_flags);
-	if (gpio_is_valid(gpio)) {
-		unsigned long flags =
-			of_flags == OF_GPIO_ACTIVE_LOW ? GPIOF_ACTIVE_LOW : 0;
+	pd->select = devm_gpiod_get_optional(&spi->dev, "select",
+					     GPIOD_OUT_LOW);
+	if (IS_ERR(pd->select))
+		return PTR_ERR(pd->select);
 
-		ret = devm_gpio_request_one(&spi->dev, gpio, flags,
-					    "eeprom_93xx46_select");
-		if (ret)
-			return ret;
-
-		pd->select = gpio_to_desc(gpio);
-		pd->prepare = select_assert;
-		pd->finish = select_deassert;
-
-		gpiod_direction_output(pd->select, 0);
-	}
+	pd->prepare = select_assert;
+	pd->finish = select_deassert;
+	gpiod_direction_output(pd->select, 0);
 
 	if (of_id->data) {
 		const struct eeprom_93xx46_devtype_data *data = of_id->data;
diff --git a/drivers/misc/eeprom/idt_89hpesx.c b/drivers/misc/eeprom/idt_89hpesx.c
index ab0df6a17690..34a5a41578d7 100644
--- a/drivers/misc/eeprom/idt_89hpesx.c
+++ b/drivers/misc/eeprom/idt_89hpesx.c
@@ -77,7 +77,7 @@
 #include <linux/sysfs.h>
 #include <linux/debugfs.h>
 #include <linux/mod_devicetable.h>
-#include <linux/of.h>
+#include <linux/property.h>
 #include <linux/i2c.h>
 #include <linux/pci_ids.h>
 #include <linux/delay.h>
@@ -1089,101 +1089,85 @@ static void idt_set_defval(struct idt_89hpesx_dev *pdev)
 	pdev->eeaddr = 0;
 }
 
-#ifdef CONFIG_OF
 static const struct i2c_device_id ee_ids[];
+
 /*
  * idt_ee_match_id() - check whether the node belongs to compatible EEPROMs
  */
-static const struct i2c_device_id *idt_ee_match_id(struct device_node *node)
+static const struct i2c_device_id *idt_ee_match_id(struct fwnode_handle *fwnode)
 {
 	const struct i2c_device_id *id = ee_ids;
+	const char *compatible, *p;
 	char devname[I2C_NAME_SIZE];
+	int ret;
 
-	/* Retrieve the device name without manufacturer name */
-	if (of_modalias_node(node, devname, sizeof(devname)))
+	ret = fwnode_property_read_string(fwnode, "compatible", &compatible);
+	if (ret)
 		return NULL;
 
+	p = strchr(compatible, ',');
+	strlcpy(devname, p ? p + 1 : compatible, sizeof(devname));
 	/* Search through the device name */
-        while (id->name[0]) {
-                if (strcmp(devname, id->name) == 0)
-                        return id;
-                id++;
-        }
-        return NULL;
+	while (id->name[0]) {
+		if (strcmp(devname, id->name) == 0)
+			return id;
+		id++;
+	}
+	return NULL;
 }
 
 /*
- * idt_get_ofdata() - get IDT i2c-device parameters from device tree
+ * idt_get_fw_data() - get IDT i2c-device parameters from device tree
  * @pdev:	Pointer to the driver data
  */
-static void idt_get_ofdata(struct idt_89hpesx_dev *pdev)
+static void idt_get_fw_data(struct idt_89hpesx_dev *pdev)
 {
-	const struct device_node *node = pdev->client->dev.of_node;
 	struct device *dev = &pdev->client->dev;
+	struct fwnode_handle *fwnode;
+	const struct i2c_device_id *ee_id = NULL;
+	u32 eeprom_addr;
+	int ret;
 
-	/* Read dts node parameters */
-	if (node) {
-		const struct i2c_device_id *ee_id = NULL;
-		struct device_node *child;
-		const __be32 *addr_be;
-		int len;
-
-		/* Walk through all child nodes looking for compatible one */
-		for_each_available_child_of_node(node, child) {
-			ee_id = idt_ee_match_id(child);
-			if (IS_ERR_OR_NULL(ee_id)) {
-				dev_warn(dev, "Skip unsupported child node %s",
-					child->full_name);
-				continue;
-			} else
-				break;
-		}
-
-		/* If there is no child EEPROM device, then set zero size */
-		if (!ee_id) {
-			idt_set_defval(pdev);
-			return;
-		}
+	device_for_each_child_node(dev, fwnode) {
+		ee_id = idt_ee_match_id(fwnode);
+		if (IS_ERR_OR_NULL(ee_id)) {
+			dev_warn(dev, "Skip unsupported EEPROM device");
+			continue;
+		} else
+			break;
+	}
 
-		/* Retrieve EEPROM size */
-		pdev->eesize = (u32)ee_id->driver_data;
-
-		/* Get custom EEPROM address from 'reg' attribute */
-		addr_be = of_get_property(child, "reg", &len);
-		if (!addr_be || (len < sizeof(*addr_be))) {
-			dev_warn(dev, "No reg on %s, use default address %d",
-				child->full_name, EEPROM_DEF_ADDR);
-			pdev->inieecmd = 0;
-			pdev->eeaddr = EEPROM_DEF_ADDR << 1;
-		} else {
-			pdev->inieecmd = EEPROM_USA;
-			pdev->eeaddr = be32_to_cpup(addr_be) << 1;
-		}
+	/* If there is no fwnode EEPROM device, then set zero size */
+	if (!ee_id) {
+		dev_warn(dev, "No fwnode, EEPROM access disabled");
+		idt_set_defval(pdev);
+		return;
+	}
 
-		/* Check EEPROM 'read-only' flag */
-		if (of_get_property(child, "read-only", NULL))
-			pdev->eero = true;
-		else /* if (!of_get_property(node, "read-only", NULL)) */
-			pdev->eero = false;
+	/* Retrieve EEPROM size */
+	pdev->eesize = (u32)ee_id->driver_data;
 
-		dev_dbg(dev, "EEPROM of %u bytes found by %hhu",
-			pdev->eesize, pdev->eeaddr);
+	/* Get custom EEPROM address from 'reg' attribute */
+	ret = fwnode_property_read_u32(fwnode, "reg", &eeprom_addr);
+	if (ret || (eeprom_addr == 0)) {
+		dev_warn(dev, "No EEPROM reg found, use default address 0x%x",
+			 EEPROM_DEF_ADDR);
+		pdev->inieecmd = 0;
+		pdev->eeaddr = EEPROM_DEF_ADDR << 1;
 	} else {
-		dev_warn(dev, "No dts node, EEPROM access disabled");
-		idt_set_defval(pdev);
+		pdev->inieecmd = EEPROM_USA;
+		pdev->eeaddr = eeprom_addr << 1;
 	}
-}
-#else
-static void idt_get_ofdata(struct idt_89hpesx_dev *pdev)
-{
-	struct device *dev = &pdev->client->dev;
 
-	dev_warn(dev, "OF table is unsupported, EEPROM access disabled");
+	/* Check EEPROM 'read-only' flag */
+	if (fwnode_property_read_bool(fwnode, "read-only"))
+		pdev->eero = true;
+	else /* if (!fwnode_property_read_bool(node, "read-only")) */
+		pdev->eero = false;
 
-	/* Nothing we can do, just set the default values */
-	idt_set_defval(pdev);
+	dev_info(dev, "EEPROM of %d bytes found by 0x%x",
+		pdev->eesize, pdev->eeaddr);
 }
-#endif /* CONFIG_OF */
 
 /*
  * idt_create_pdev() - create and init data structure of the driver
@@ -1203,8 +1187,8 @@ static struct idt_89hpesx_dev *idt_create_pdev(struct i2c_client *client)
 	pdev->client = client;
 	i2c_set_clientdata(client, pdev);
 
-	/* Read OF nodes information */
-	idt_get_ofdata(pdev);
+	/* Read firmware nodes information */
+	idt_get_fw_data(pdev);
 
 	/* Initialize basic CSR CMD field - use full DWORD-sized r/w ops */
 	pdev->inicsrcmd = CSR_DWE;
diff --git a/drivers/misc/eeprom/max6875.c b/drivers/misc/eeprom/max6875.c
index e4dd93b2518c..0e32709d1022 100644
--- a/drivers/misc/eeprom/max6875.c
+++ b/drivers/misc/eeprom/max6875.c
@@ -124,7 +124,7 @@ static ssize_t max6875_read(struct file *filp, struct kobject *kobj,
 	return count;
 }
 
-static struct bin_attribute user_eeprom_attr = {
+static const struct bin_attribute user_eeprom_attr = {
 	.attr = {
 		.name = "eeprom",
 		.mode = S_IRUGO,
diff --git a/drivers/misc/hmc6352.c b/drivers/misc/hmc6352.c
index 90520d76633f..eeb7eef62174 100644
--- a/drivers/misc/hmc6352.c
+++ b/drivers/misc/hmc6352.c
@@ -132,7 +132,7 @@ static int hmc6352_remove(struct i2c_client *client)
 	return 0;
 }
 
-static struct i2c_device_id hmc6352_id[] = {
+static const struct i2c_device_id hmc6352_id[] = {
 	{ "hmc6352", 0 },
 	{ }
 };
diff --git a/drivers/misc/hpilo.c b/drivers/misc/hpilo.c
index fea8ff40440f..097e3092c158 100644
--- a/drivers/misc/hpilo.c
+++ b/drivers/misc/hpilo.c
@@ -857,7 +857,7 @@ out:
 	return error;
 }
 
-static struct pci_device_id ilo_devices[] = {
+static const struct pci_device_id ilo_devices[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_COMPAQ, 0xB204) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_HP, 0x3307) },
 	{ }
diff --git a/drivers/misc/ioc4.c b/drivers/misc/ioc4.c
index 8758d033db23..ec0832278170 100644
--- a/drivers/misc/ioc4.c
+++ b/drivers/misc/ioc4.c
@@ -454,7 +454,7 @@ ioc4_remove(struct pci_dev *pdev)
 	kfree(idd);
 }
 
-static struct pci_device_id ioc4_id_table[] = {
+static const struct pci_device_id ioc4_id_table[] = {
 	{PCI_VENDOR_ID_SGI, PCI_DEVICE_ID_SGI_IOC4, PCI_ANY_ID,
 	 PCI_ANY_ID, 0x0b4000, 0xFFFFFF},
 	{0}
diff --git a/drivers/misc/isl29020.c b/drivers/misc/isl29020.c
index 4a9c50a43afb..e3bd3c1d2574 100644
--- a/drivers/misc/isl29020.c
+++ b/drivers/misc/isl29020.c
@@ -145,7 +145,7 @@ static struct attribute *mid_att_als[] = {
 	NULL
 };
 
-static struct attribute_group m_als_gr = {
+static const struct attribute_group m_als_gr = {
 	.name = "isl29020",
 	.attrs = mid_att_als
 };
@@ -188,7 +188,7 @@ static int isl29020_remove(struct i2c_client *client)
 	return 0;
 }
 
-static struct i2c_device_id isl29020_id[] = {
+static const struct i2c_device_id isl29020_id[] = {
 	{ "isl29020", 0 },
 	{ }
 };
diff --git a/drivers/misc/lis3lv02d/lis3lv02d.c b/drivers/misc/lis3lv02d/lis3lv02d.c
index e389b0b5278d..8d53609861d8 100644
--- a/drivers/misc/lis3lv02d/lis3lv02d.c
+++ b/drivers/misc/lis3lv02d/lis3lv02d.c
@@ -856,7 +856,7 @@ static struct attribute *lis3lv02d_attributes[] = {
 	NULL
 };
 
-static struct attribute_group lis3lv02d_attribute_group = {
+static const struct attribute_group lis3lv02d_attribute_group = {
 	.attrs = lis3lv02d_attributes
 };
 
diff --git a/drivers/misc/lkdtm.h b/drivers/misc/lkdtm.h
index 3b4976396ec4..bfb6c45b6130 100644
--- a/drivers/misc/lkdtm.h
+++ b/drivers/misc/lkdtm.h
@@ -14,20 +14,17 @@ void lkdtm_EXCEPTION(void);
 void lkdtm_LOOP(void);
 void lkdtm_OVERFLOW(void);
 void lkdtm_CORRUPT_STACK(void);
+void lkdtm_CORRUPT_STACK_STRONG(void);
 void lkdtm_UNALIGNED_LOAD_STORE_WRITE(void);
 void lkdtm_SOFTLOCKUP(void);
 void lkdtm_HARDLOCKUP(void);
 void lkdtm_SPINLOCKUP(void);
 void lkdtm_HUNG_TASK(void);
-void lkdtm_REFCOUNT_SATURATE_INC(void);
-void lkdtm_REFCOUNT_SATURATE_ADD(void);
-void lkdtm_REFCOUNT_ZERO_DEC(void);
-void lkdtm_REFCOUNT_ZERO_INC(void);
-void lkdtm_REFCOUNT_ZERO_SUB(void);
-void lkdtm_REFCOUNT_ZERO_ADD(void);
 void lkdtm_CORRUPT_LIST_ADD(void);
 void lkdtm_CORRUPT_LIST_DEL(void);
 void lkdtm_CORRUPT_USER_DS(void);
+void lkdtm_STACK_GUARD_PAGE_LEADING(void);
+void lkdtm_STACK_GUARD_PAGE_TRAILING(void);
 
 /* lkdtm_heap.c */
 void lkdtm_OVERWRITE_ALLOCATION(void);
@@ -49,6 +46,27 @@ void lkdtm_EXEC_RODATA(void);
 void lkdtm_EXEC_USERSPACE(void);
 void lkdtm_ACCESS_USERSPACE(void);
 
+/* lkdtm_refcount.c */
+void lkdtm_REFCOUNT_INC_OVERFLOW(void);
+void lkdtm_REFCOUNT_ADD_OVERFLOW(void);
+void lkdtm_REFCOUNT_INC_NOT_ZERO_OVERFLOW(void);
+void lkdtm_REFCOUNT_ADD_NOT_ZERO_OVERFLOW(void);
+void lkdtm_REFCOUNT_DEC_ZERO(void);
+void lkdtm_REFCOUNT_DEC_NEGATIVE(void);
+void lkdtm_REFCOUNT_DEC_AND_TEST_NEGATIVE(void);
+void lkdtm_REFCOUNT_SUB_AND_TEST_NEGATIVE(void);
+void lkdtm_REFCOUNT_INC_ZERO(void);
+void lkdtm_REFCOUNT_ADD_ZERO(void);
+void lkdtm_REFCOUNT_INC_SATURATED(void);
+void lkdtm_REFCOUNT_DEC_SATURATED(void);
+void lkdtm_REFCOUNT_ADD_SATURATED(void);
+void lkdtm_REFCOUNT_INC_NOT_ZERO_SATURATED(void);
+void lkdtm_REFCOUNT_ADD_NOT_ZERO_SATURATED(void);
+void lkdtm_REFCOUNT_DEC_AND_TEST_SATURATED(void);
+void lkdtm_REFCOUNT_SUB_AND_TEST_SATURATED(void);
+void lkdtm_REFCOUNT_TIMING(void);
+void lkdtm_ATOMIC_TIMING(void);
+
 /* lkdtm_rodata.c */
 void lkdtm_rodata_do_nothing(void);
 
diff --git a/drivers/misc/lkdtm_bugs.c b/drivers/misc/lkdtm_bugs.c
index d9028ef50fbe..9e0b4f959987 100644
--- a/drivers/misc/lkdtm_bugs.c
+++ b/drivers/misc/lkdtm_bugs.c
@@ -6,9 +6,9 @@
  */
 #include "lkdtm.h"
 #include <linux/list.h>
-#include <linux/refcount.h>
 #include <linux/sched.h>
 #include <linux/sched/signal.h>
+#include <linux/sched/task_stack.h>
 #include <linux/uaccess.h>
 
 struct lkdtm_list {
@@ -85,16 +85,31 @@ void lkdtm_OVERFLOW(void)
 
 static noinline void __lkdtm_CORRUPT_STACK(void *stack)
 {
-	memset(stack, 'a', 64);
+	memset(stack, '\xff', 64);
 }
 
+/* This should trip the stack canary, not corrupt the return address. */
 noinline void lkdtm_CORRUPT_STACK(void)
 {
 	/* Use default char array length that triggers stack protection. */
-	char data[8];
+	char data[8] __aligned(sizeof(void *));
+
+	__lkdtm_CORRUPT_STACK(&data);
+
+	pr_info("Corrupted stack containing char array ...\n");
+}
+
+/* Same as above but will only get a canary with -fstack-protector-strong */
+noinline void lkdtm_CORRUPT_STACK_STRONG(void)
+{
+	union {
+		unsigned short shorts[4];
+		unsigned long *ptr;
+	} data __aligned(sizeof(void *));
+
 	__lkdtm_CORRUPT_STACK(&data);
 
-	pr_info("Corrupted stack with '%16s'...\n", data);
+	pr_info("Corrupted stack containing union ...\n");
 }
 
 void lkdtm_UNALIGNED_LOAD_STORE_WRITE(void)
@@ -137,88 +152,6 @@ void lkdtm_HUNG_TASK(void)
 	schedule();
 }
 
-void lkdtm_REFCOUNT_SATURATE_INC(void)
-{
-	refcount_t over = REFCOUNT_INIT(UINT_MAX - 1);
-
-	pr_info("attempting good refcount decrement\n");
-	refcount_dec(&over);
-	refcount_inc(&over);
-
-	pr_info("attempting bad refcount inc overflow\n");
-	refcount_inc(&over);
-	refcount_inc(&over);
-	if (refcount_read(&over) == UINT_MAX)
-		pr_err("Correctly stayed saturated, but no BUG?!\n");
-	else
-		pr_err("Fail: refcount wrapped\n");
-}
-
-void lkdtm_REFCOUNT_SATURATE_ADD(void)
-{
-	refcount_t over = REFCOUNT_INIT(UINT_MAX - 1);
-
-	pr_info("attempting good refcount decrement\n");
-	refcount_dec(&over);
-	refcount_inc(&over);
-
-	pr_info("attempting bad refcount add overflow\n");
-	refcount_add(2, &over);
-	if (refcount_read(&over) == UINT_MAX)
-		pr_err("Correctly stayed saturated, but no BUG?!\n");
-	else
-		pr_err("Fail: refcount wrapped\n");
-}
-
-void lkdtm_REFCOUNT_ZERO_DEC(void)
-{
-	refcount_t zero = REFCOUNT_INIT(1);
-
-	pr_info("attempting bad refcount decrement to zero\n");
-	refcount_dec(&zero);
-	if (refcount_read(&zero) == 0)
-		pr_err("Stayed at zero, but no BUG?!\n");
-	else
-		pr_err("Fail: refcount went crazy\n");
-}
-
-void lkdtm_REFCOUNT_ZERO_SUB(void)
-{
-	refcount_t zero = REFCOUNT_INIT(1);
-
-	pr_info("attempting bad refcount subtract past zero\n");
-	if (!refcount_sub_and_test(2, &zero))
-		pr_info("wrap attempt was noticed\n");
-	if (refcount_read(&zero) == 1)
-		pr_err("Correctly stayed above 0, but no BUG?!\n");
-	else
-		pr_err("Fail: refcount wrapped\n");
-}
-
-void lkdtm_REFCOUNT_ZERO_INC(void)
-{
-	refcount_t zero = REFCOUNT_INIT(0);
-
-	pr_info("attempting bad refcount increment from zero\n");
-	refcount_inc(&zero);
-	if (refcount_read(&zero) == 0)
-		pr_err("Stayed at zero, but no BUG?!\n");
-	else
-		pr_err("Fail: refcount went past zero\n");
-}
-
-void lkdtm_REFCOUNT_ZERO_ADD(void)
-{
-	refcount_t zero = REFCOUNT_INIT(0);
-
-	pr_info("attempting bad refcount addition from zero\n");
-	refcount_add(2, &zero);
-	if (refcount_read(&zero) == 0)
-		pr_err("Stayed at zero, but no BUG?!\n");
-	else
-		pr_err("Fail: refcount went past zero\n");
-}
-
 void lkdtm_CORRUPT_LIST_ADD(void)
 {
 	/*
@@ -282,6 +215,7 @@ void lkdtm_CORRUPT_LIST_DEL(void)
 		pr_err("list_del() corruption not detected!\n");
 }
 
+/* Test if unbalanced set_fs(KERNEL_DS)/set_fs(USER_DS) check exists. */
 void lkdtm_CORRUPT_USER_DS(void)
 {
 	pr_info("setting bad task size limit\n");
@@ -290,3 +224,31 @@ void lkdtm_CORRUPT_USER_DS(void)
 	/* Make sure we do not keep running with a KERNEL_DS! */
 	force_sig(SIGKILL, current);
 }
+
+/* Test that VMAP_STACK is actually allocating with a leading guard page */
+void lkdtm_STACK_GUARD_PAGE_LEADING(void)
+{
+	const unsigned char *stack = task_stack_page(current);
+	const unsigned char *ptr = stack - 1;
+	volatile unsigned char byte;
+
+	pr_info("attempting bad read from page below current stack\n");
+
+	byte = *ptr;
+
+	pr_err("FAIL: accessed page before stack!\n");
+}
+
+/* Test that VMAP_STACK is actually allocating with a trailing guard page */
+void lkdtm_STACK_GUARD_PAGE_TRAILING(void)
+{
+	const unsigned char *stack = task_stack_page(current);
+	const unsigned char *ptr = stack + THREAD_SIZE;
+	volatile unsigned char byte;
+
+	pr_info("attempting bad read from page above current stack\n");
+
+	byte = *ptr;
+
+	pr_err("FAIL: accessed page after stack!\n");
+}
diff --git a/drivers/misc/lkdtm_core.c b/drivers/misc/lkdtm_core.c
index 42d2b8e31e6b..981b3ef71e47 100644
--- a/drivers/misc/lkdtm_core.c
+++ b/drivers/misc/lkdtm_core.c
@@ -201,6 +201,9 @@ struct crashtype crashtypes[] = {
 	CRASHTYPE(CORRUPT_LIST_DEL),
 	CRASHTYPE(CORRUPT_USER_DS),
 	CRASHTYPE(CORRUPT_STACK),
+	CRASHTYPE(CORRUPT_STACK_STRONG),
+	CRASHTYPE(STACK_GUARD_PAGE_LEADING),
+	CRASHTYPE(STACK_GUARD_PAGE_TRAILING),
 	CRASHTYPE(UNALIGNED_LOAD_STORE_WRITE),
 	CRASHTYPE(OVERWRITE_ALLOCATION),
 	CRASHTYPE(WRITE_AFTER_FREE),
@@ -221,12 +224,25 @@ struct crashtype crashtypes[] = {
 	CRASHTYPE(WRITE_RO),
 	CRASHTYPE(WRITE_RO_AFTER_INIT),
 	CRASHTYPE(WRITE_KERN),
-	CRASHTYPE(REFCOUNT_SATURATE_INC),
-	CRASHTYPE(REFCOUNT_SATURATE_ADD),
-	CRASHTYPE(REFCOUNT_ZERO_DEC),
-	CRASHTYPE(REFCOUNT_ZERO_INC),
-	CRASHTYPE(REFCOUNT_ZERO_SUB),
-	CRASHTYPE(REFCOUNT_ZERO_ADD),
+	CRASHTYPE(REFCOUNT_INC_OVERFLOW),
+	CRASHTYPE(REFCOUNT_ADD_OVERFLOW),
+	CRASHTYPE(REFCOUNT_INC_NOT_ZERO_OVERFLOW),
+	CRASHTYPE(REFCOUNT_ADD_NOT_ZERO_OVERFLOW),
+	CRASHTYPE(REFCOUNT_DEC_ZERO),
+	CRASHTYPE(REFCOUNT_DEC_NEGATIVE),
+	CRASHTYPE(REFCOUNT_DEC_AND_TEST_NEGATIVE),
+	CRASHTYPE(REFCOUNT_SUB_AND_TEST_NEGATIVE),
+	CRASHTYPE(REFCOUNT_INC_ZERO),
+	CRASHTYPE(REFCOUNT_ADD_ZERO),
+	CRASHTYPE(REFCOUNT_INC_SATURATED),
+	CRASHTYPE(REFCOUNT_DEC_SATURATED),
+	CRASHTYPE(REFCOUNT_ADD_SATURATED),
+	CRASHTYPE(REFCOUNT_INC_NOT_ZERO_SATURATED),
+	CRASHTYPE(REFCOUNT_ADD_NOT_ZERO_SATURATED),
+	CRASHTYPE(REFCOUNT_DEC_AND_TEST_SATURATED),
+	CRASHTYPE(REFCOUNT_SUB_AND_TEST_SATURATED),
+	CRASHTYPE(REFCOUNT_TIMING),
+	CRASHTYPE(ATOMIC_TIMING),
 	CRASHTYPE(USERCOPY_HEAP_SIZE_TO),
 	CRASHTYPE(USERCOPY_HEAP_SIZE_FROM),
 	CRASHTYPE(USERCOPY_HEAP_FLAG_TO),
diff --git a/drivers/misc/lkdtm_refcount.c b/drivers/misc/lkdtm_refcount.c
new file mode 100644
index 000000000000..2b99d448e7fd
--- /dev/null
+++ b/drivers/misc/lkdtm_refcount.c
@@ -0,0 +1,400 @@
+/*
+ * This is for all the tests related to refcount bugs (e.g. overflow,
+ * underflow, reaching zero untested, etc).
+ */
+#include "lkdtm.h"
+#include <linux/refcount.h>
+
+#ifdef CONFIG_REFCOUNT_FULL
+#define REFCOUNT_MAX		(UINT_MAX - 1)
+#define REFCOUNT_SATURATED	UINT_MAX
+#else
+#define REFCOUNT_MAX		INT_MAX
+#define REFCOUNT_SATURATED	(INT_MIN / 2)
+#endif
+
+static void overflow_check(refcount_t *ref)
+{
+	switch (refcount_read(ref)) {
+	case REFCOUNT_SATURATED:
+		pr_info("Overflow detected: saturated\n");
+		break;
+	case REFCOUNT_MAX:
+		pr_warn("Overflow detected: unsafely reset to max\n");
+		break;
+	default:
+		pr_err("Fail: refcount wrapped to %d\n", refcount_read(ref));
+	}
+}
+
+/*
+ * A refcount_inc() above the maximum value of the refcount implementation,
+ * should at least saturate, and at most also WARN.
+ */
+void lkdtm_REFCOUNT_INC_OVERFLOW(void)
+{
+	refcount_t over = REFCOUNT_INIT(REFCOUNT_MAX - 1);
+
+	pr_info("attempting good refcount_inc() without overflow\n");
+	refcount_dec(&over);
+	refcount_inc(&over);
+
+	pr_info("attempting bad refcount_inc() overflow\n");
+	refcount_inc(&over);
+	refcount_inc(&over);
+
+	overflow_check(&over);
+}
+
+/* refcount_add() should behave just like refcount_inc() above. */
+void lkdtm_REFCOUNT_ADD_OVERFLOW(void)
+{
+	refcount_t over = REFCOUNT_INIT(REFCOUNT_MAX - 1);
+
+	pr_info("attempting good refcount_add() without overflow\n");
+	refcount_dec(&over);
+	refcount_dec(&over);
+	refcount_dec(&over);
+	refcount_dec(&over);
+	refcount_add(4, &over);
+
+	pr_info("attempting bad refcount_add() overflow\n");
+	refcount_add(4, &over);
+
+	overflow_check(&over);
+}
+
+/* refcount_inc_not_zero() should behave just like refcount_inc() above. */
+void lkdtm_REFCOUNT_INC_NOT_ZERO_OVERFLOW(void)
+{
+	refcount_t over = REFCOUNT_INIT(REFCOUNT_MAX);
+
+	pr_info("attempting bad refcount_inc_not_zero() overflow\n");
+	if (!refcount_inc_not_zero(&over))
+		pr_warn("Weird: refcount_inc_not_zero() reported zero\n");
+
+	overflow_check(&over);
+}
+
+/* refcount_add_not_zero() should behave just like refcount_inc() above. */
+void lkdtm_REFCOUNT_ADD_NOT_ZERO_OVERFLOW(void)
+{
+	refcount_t over = REFCOUNT_INIT(REFCOUNT_MAX);
+
+	pr_info("attempting bad refcount_add_not_zero() overflow\n");
+	if (!refcount_add_not_zero(6, &over))
+		pr_warn("Weird: refcount_add_not_zero() reported zero\n");
+
+	overflow_check(&over);
+}
+
+static void check_zero(refcount_t *ref)
+{
+	switch (refcount_read(ref)) {
+	case REFCOUNT_SATURATED:
+		pr_info("Zero detected: saturated\n");
+		break;
+	case REFCOUNT_MAX:
+		pr_warn("Zero detected: unsafely reset to max\n");
+		break;
+	case 0:
+		pr_warn("Still at zero: refcount_inc/add() must not inc-from-0\n");
+		break;
+	default:
+		pr_err("Fail: refcount went crazy: %d\n", refcount_read(ref));
+	}
+}
+
+/*
+ * A refcount_dec(), as opposed to a refcount_dec_and_test(), when it hits
+ * zero it should either saturate (when inc-from-zero isn't protected)
+ * or stay at zero (when inc-from-zero is protected) and should WARN for both.
+ */
+void lkdtm_REFCOUNT_DEC_ZERO(void)
+{
+	refcount_t zero = REFCOUNT_INIT(2);
+
+	pr_info("attempting good refcount_dec()\n");
+	refcount_dec(&zero);
+
+	pr_info("attempting bad refcount_dec() to zero\n");
+	refcount_dec(&zero);
+
+	check_zero(&zero);
+}
+
+static void check_negative(refcount_t *ref, int start)
+{
+	/*
+	 * CONFIG_REFCOUNT_FULL refuses to move a refcount at all on an
+	 * over-sub, so we have to track our starting position instead of
+	 * looking only at zero-pinning.
+	 */
+	if (refcount_read(ref) == start) {
+		pr_warn("Still at %d: refcount_inc/add() must not inc-from-0\n",
+			start);
+		return;
+	}
+
+	switch (refcount_read(ref)) {
+	case REFCOUNT_SATURATED:
+		pr_info("Negative detected: saturated\n");
+		break;
+	case REFCOUNT_MAX:
+		pr_warn("Negative detected: unsafely reset to max\n");
+		break;
+	default:
+		pr_err("Fail: refcount went crazy: %d\n", refcount_read(ref));
+	}
+}
+
+/* A refcount_dec() going negative should saturate and may WARN. */
+void lkdtm_REFCOUNT_DEC_NEGATIVE(void)
+{
+	refcount_t neg = REFCOUNT_INIT(0);
+
+	pr_info("attempting bad refcount_dec() below zero\n");
+	refcount_dec(&neg);
+
+	check_negative(&neg, 0);
+}
+
+/*
+ * A refcount_dec_and_test() should act like refcount_dec() above when
+ * going negative.
+ */
+void lkdtm_REFCOUNT_DEC_AND_TEST_NEGATIVE(void)
+{
+	refcount_t neg = REFCOUNT_INIT(0);
+
+	pr_info("attempting bad refcount_dec_and_test() below zero\n");
+	if (refcount_dec_and_test(&neg))
+		pr_warn("Weird: refcount_dec_and_test() reported zero\n");
+
+	check_negative(&neg, 0);
+}
+
+/*
+ * A refcount_sub_and_test() should act like refcount_dec_and_test()
+ * above when going negative.
+ */
+void lkdtm_REFCOUNT_SUB_AND_TEST_NEGATIVE(void)
+{
+	refcount_t neg = REFCOUNT_INIT(3);
+
+	pr_info("attempting bad refcount_sub_and_test() below zero\n");
+	if (refcount_sub_and_test(5, &neg))
+		pr_warn("Weird: refcount_sub_and_test() reported zero\n");
+
+	check_negative(&neg, 3);
+}
+
+static void check_from_zero(refcount_t *ref)
+{
+	switch (refcount_read(ref)) {
+	case 0:
+		pr_info("Zero detected: stayed at zero\n");
+		break;
+	case REFCOUNT_SATURATED:
+		pr_info("Zero detected: saturated\n");
+		break;
+	case REFCOUNT_MAX:
+		pr_warn("Zero detected: unsafely reset to max\n");
+		break;
+	default:
+		pr_info("Fail: zero not detected, incremented to %d\n",
+			refcount_read(ref));
+	}
+}
+
+/*
+ * A refcount_inc() from zero should pin to zero or saturate and may WARN.
+ * Only CONFIG_REFCOUNT_FULL provides this protection currently.
+ */
+void lkdtm_REFCOUNT_INC_ZERO(void)
+{
+	refcount_t zero = REFCOUNT_INIT(0);
+
+	pr_info("attempting safe refcount_inc_not_zero() from zero\n");
+	if (!refcount_inc_not_zero(&zero)) {
+		pr_info("Good: zero detected\n");
+		if (refcount_read(&zero) == 0)
+			pr_info("Correctly stayed at zero\n");
+		else
+			pr_err("Fail: refcount went past zero!\n");
+	} else {
+		pr_err("Fail: Zero not detected!?\n");
+	}
+
+	pr_info("attempting bad refcount_inc() from zero\n");
+	refcount_inc(&zero);
+
+	check_from_zero(&zero);
+}
+
+/*
+ * A refcount_add() should act like refcount_inc() above when starting
+ * at zero.
+ */
+void lkdtm_REFCOUNT_ADD_ZERO(void)
+{
+	refcount_t zero = REFCOUNT_INIT(0);
+
+	pr_info("attempting safe refcount_add_not_zero() from zero\n");
+	if (!refcount_add_not_zero(3, &zero)) {
+		pr_info("Good: zero detected\n");
+		if (refcount_read(&zero) == 0)
+			pr_info("Correctly stayed at zero\n");
+		else
+			pr_err("Fail: refcount went past zero\n");
+	} else {
+		pr_err("Fail: Zero not detected!?\n");
+	}
+
+	pr_info("attempting bad refcount_add() from zero\n");
+	refcount_add(3, &zero);
+
+	check_from_zero(&zero);
+}
+
+static void check_saturated(refcount_t *ref)
+{
+	switch (refcount_read(ref)) {
+	case REFCOUNT_SATURATED:
+		pr_info("Saturation detected: still saturated\n");
+		break;
+	case REFCOUNT_MAX:
+		pr_warn("Saturation detected: unsafely reset to max\n");
+		break;
+	default:
+		pr_err("Fail: refcount went crazy: %d\n", refcount_read(ref));
+	}
+}
+
+/*
+ * A refcount_inc() from a saturated value should at most warn about
+ * being saturated already.
+ */
+void lkdtm_REFCOUNT_INC_SATURATED(void)
+{
+	refcount_t sat = REFCOUNT_INIT(REFCOUNT_SATURATED);
+
+	pr_info("attempting bad refcount_inc() from saturated\n");
+	refcount_inc(&sat);
+
+	check_saturated(&sat);
+}
+
+/* Should act like refcount_inc() above from saturated. */
+void lkdtm_REFCOUNT_DEC_SATURATED(void)
+{
+	refcount_t sat = REFCOUNT_INIT(REFCOUNT_SATURATED);
+
+	pr_info("attempting bad refcount_dec() from saturated\n");
+	refcount_dec(&sat);
+
+	check_saturated(&sat);
+}
+
+/* Should act like refcount_inc() above from saturated. */
+void lkdtm_REFCOUNT_ADD_SATURATED(void)
+{
+	refcount_t sat = REFCOUNT_INIT(REFCOUNT_SATURATED);
+
+	pr_info("attempting bad refcount_dec() from saturated\n");
+	refcount_add(8, &sat);
+
+	check_saturated(&sat);
+}
+
+/* Should act like refcount_inc() above from saturated. */
+void lkdtm_REFCOUNT_INC_NOT_ZERO_SATURATED(void)
+{
+	refcount_t sat = REFCOUNT_INIT(REFCOUNT_SATURATED);
+
+	pr_info("attempting bad refcount_inc_not_zero() from saturated\n");
+	if (!refcount_inc_not_zero(&sat))
+		pr_warn("Weird: refcount_inc_not_zero() reported zero\n");
+
+	check_saturated(&sat);
+}
+
+/* Should act like refcount_inc() above from saturated. */
+void lkdtm_REFCOUNT_ADD_NOT_ZERO_SATURATED(void)
+{
+	refcount_t sat = REFCOUNT_INIT(REFCOUNT_SATURATED);
+
+	pr_info("attempting bad refcount_add_not_zero() from saturated\n");
+	if (!refcount_add_not_zero(7, &sat))
+		pr_warn("Weird: refcount_add_not_zero() reported zero\n");
+
+	check_saturated(&sat);
+}
+
+/* Should act like refcount_inc() above from saturated. */
+void lkdtm_REFCOUNT_DEC_AND_TEST_SATURATED(void)
+{
+	refcount_t sat = REFCOUNT_INIT(REFCOUNT_SATURATED);
+
+	pr_info("attempting bad refcount_dec_and_test() from saturated\n");
+	if (refcount_dec_and_test(&sat))
+		pr_warn("Weird: refcount_dec_and_test() reported zero\n");
+
+	check_saturated(&sat);
+}
+
+/* Should act like refcount_inc() above from saturated. */
+void lkdtm_REFCOUNT_SUB_AND_TEST_SATURATED(void)
+{
+	refcount_t sat = REFCOUNT_INIT(REFCOUNT_SATURATED);
+
+	pr_info("attempting bad refcount_sub_and_test() from saturated\n");
+	if (refcount_sub_and_test(8, &sat))
+		pr_warn("Weird: refcount_sub_and_test() reported zero\n");
+
+	check_saturated(&sat);
+}
+
+/* Used to time the existing atomic_t when used for reference counting */
+void lkdtm_ATOMIC_TIMING(void)
+{
+	unsigned int i;
+	atomic_t count = ATOMIC_INIT(1);
+
+	for (i = 0; i < INT_MAX - 1; i++)
+		atomic_inc(&count);
+
+	for (i = INT_MAX; i > 0; i--)
+		if (atomic_dec_and_test(&count))
+			break;
+
+	if (i != 1)
+		pr_err("atomic timing: out of sync up/down cycle: %u\n", i - 1);
+	else
+		pr_info("atomic timing: done\n");
+}
+
+/*
+ * This can be compared to ATOMIC_TIMING when implementing fast refcount
+ * protections. Looking at the number of CPU cycles tells the real story
+ * about performance. For example:
+ *    cd /sys/kernel/debug/provoke-crash
+ *    perf stat -B -- cat <(echo REFCOUNT_TIMING) > DIRECT
+ */
+void lkdtm_REFCOUNT_TIMING(void)
+{
+	unsigned int i;
+	refcount_t count = REFCOUNT_INIT(1);
+
+	for (i = 0; i < INT_MAX - 1; i++)
+		refcount_inc(&count);
+
+	for (i = INT_MAX; i > 0; i--)
+		if (refcount_dec_and_test(&count))
+			break;
+
+	if (i != 1)
+		pr_err("refcount: out of sync up/down cycle: %u\n", i - 1);
+	else
+		pr_info("refcount timing: done\n");
+}
diff --git a/drivers/misc/mei/bus.c b/drivers/misc/mei/bus.c
index 40c79089e548..1ac10cb64d6e 100644
--- a/drivers/misc/mei/bus.c
+++ b/drivers/misc/mei/bus.c
@@ -845,7 +845,7 @@ static void mei_cl_bus_dev_release(struct device *dev)
 	kfree(cldev);
 }
 
-static struct device_type mei_cl_device_type = {
+static const struct device_type mei_cl_device_type = {
 	.release	= mei_cl_bus_dev_release,
 };
 
diff --git a/drivers/misc/mei/hw-me.c b/drivers/misc/mei/hw-me.c
index 71216affcab1..10dcf4ff99a5 100644
--- a/drivers/misc/mei/hw-me.c
+++ b/drivers/misc/mei/hw-me.c
@@ -1354,10 +1354,10 @@ static bool mei_me_fw_type_sps(struct pci_dev *pdev)
 	.quirk_probe = mei_me_fw_type_sps
 
 
-#define MEI_CFG_LEGACY_HFS                      \
+#define MEI_CFG_ICH_HFS                      \
 	.fw_status.count = 0
 
-#define MEI_CFG_ICH_HFS                        \
+#define MEI_CFG_ICH10_HFS                        \
 	.fw_status.count = 1,                   \
 	.fw_status.status[0] = PCI_CFG_HFS_1
 
@@ -1376,38 +1376,61 @@ static bool mei_me_fw_type_sps(struct pci_dev *pdev)
 	.fw_status.status[5] = PCI_CFG_HFS_6
 
 /* ICH Legacy devices */
-const struct mei_cfg mei_me_legacy_cfg = {
-	MEI_CFG_LEGACY_HFS,
+static const struct mei_cfg mei_me_ich_cfg = {
+	MEI_CFG_ICH_HFS,
 };
 
 /* ICH devices */
-const struct mei_cfg mei_me_ich_cfg = {
-	MEI_CFG_ICH_HFS,
+static const struct mei_cfg mei_me_ich10_cfg = {
+	MEI_CFG_ICH10_HFS,
 };
 
 /* PCH devices */
-const struct mei_cfg mei_me_pch_cfg = {
+static const struct mei_cfg mei_me_pch_cfg = {
 	MEI_CFG_PCH_HFS,
 };
 
-
 /* PCH Cougar Point and Patsburg with quirk for Node Manager exclusion */
-const struct mei_cfg mei_me_pch_cpt_pbg_cfg = {
+static const struct mei_cfg mei_me_pch_cpt_pbg_cfg = {
 	MEI_CFG_PCH_HFS,
 	MEI_CFG_FW_NM,
 };
 
 /* PCH8 Lynx Point and newer devices */
-const struct mei_cfg mei_me_pch8_cfg = {
+static const struct mei_cfg mei_me_pch8_cfg = {
 	MEI_CFG_PCH8_HFS,
 };
 
 /* PCH8 Lynx Point with quirk for SPS Firmware exclusion */
-const struct mei_cfg mei_me_pch8_sps_cfg = {
+static const struct mei_cfg mei_me_pch8_sps_cfg = {
 	MEI_CFG_PCH8_HFS,
 	MEI_CFG_FW_SPS,
 };
 
+/*
+ * mei_cfg_list - A list of platform platform specific configurations.
+ * Note: has to be synchronized with  enum mei_cfg_idx.
+ */
+static const struct mei_cfg *const mei_cfg_list[] = {
+	[MEI_ME_UNDEF_CFG] = NULL,
+	[MEI_ME_ICH_CFG] = &mei_me_ich_cfg,
+	[MEI_ME_ICH10_CFG] = &mei_me_ich10_cfg,
+	[MEI_ME_PCH_CFG] = &mei_me_pch_cfg,
+	[MEI_ME_PCH_CPT_PBG_CFG] = &mei_me_pch_cpt_pbg_cfg,
+	[MEI_ME_PCH8_CFG] = &mei_me_pch8_cfg,
+	[MEI_ME_PCH8_SPS_CFG] = &mei_me_pch8_sps_cfg,
+};
+
+const struct mei_cfg *mei_me_get_cfg(kernel_ulong_t idx)
+{
+	BUILD_BUG_ON(ARRAY_SIZE(mei_cfg_list) != MEI_ME_NUM_CFG);
+
+	if (idx >= MEI_ME_NUM_CFG)
+		return NULL;
+
+	return mei_cfg_list[idx];
+};
+
 /**
  * mei_me_dev_init - allocates and initializes the mei device structure
  *
diff --git a/drivers/misc/mei/hw-me.h b/drivers/misc/mei/hw-me.h
index cf64847a35b9..67892533576e 100644
--- a/drivers/misc/mei/hw-me.h
+++ b/drivers/misc/mei/hw-me.h
@@ -41,8 +41,7 @@ struct mei_cfg {
 #define MEI_PCI_DEVICE(dev, cfg) \
 	.vendor = PCI_VENDOR_ID_INTEL, .device = (dev), \
 	.subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID, \
-	.driver_data = (kernel_ulong_t)&(cfg)
-
+	.driver_data = (kernel_ulong_t)(cfg),
 
 #define MEI_ME_RPM_TIMEOUT    500 /* ms */
 
@@ -63,12 +62,36 @@ struct mei_me_hw {
 
 #define to_me_hw(dev) (struct mei_me_hw *)((dev)->hw)
 
-extern const struct mei_cfg mei_me_legacy_cfg;
-extern const struct mei_cfg mei_me_ich_cfg;
-extern const struct mei_cfg mei_me_pch_cfg;
-extern const struct mei_cfg mei_me_pch_cpt_pbg_cfg;
-extern const struct mei_cfg mei_me_pch8_cfg;
-extern const struct mei_cfg mei_me_pch8_sps_cfg;
+/**
+ * enum mei_cfg_idx - indices to platform specific configurations.
+ *
+ * Note: has to be synchronized with mei_cfg_list[]
+ *
+ * @MEI_ME_UNDEF_CFG:      Lower sentinel.
+ * @MEI_ME_ICH_CFG:        I/O Controller Hub legacy devices.
+ * @MEI_ME_ICH10_CFG:      I/O Controller Hub platforms Gen10
+ * @MEI_ME_PCH_CFG:        Platform Controller Hub platforms (Up to Gen8).
+ * @MEI_ME_PCH_CPT_PBG_CFG:Platform Controller Hub workstations
+ *                         with quirk for Node Manager exclusion.
+ * @MEI_ME_PCH8_CFG:       Platform Controller Hub Gen8 and newer
+ *                         client platforms.
+ * @MEI_ME_PCH8_SPS_CFG:   Platform Controller Hub Gen8 and newer
+ *                         servers platforms with quirk for
+ *                         SPS firmware exclusion.
+ * @MEI_ME_NUM_CFG:        Upper Sentinel.
+ */
+enum mei_cfg_idx {
+	MEI_ME_UNDEF_CFG,
+	MEI_ME_ICH_CFG,
+	MEI_ME_ICH10_CFG,
+	MEI_ME_PCH_CFG,
+	MEI_ME_PCH_CPT_PBG_CFG,
+	MEI_ME_PCH8_CFG,
+	MEI_ME_PCH8_SPS_CFG,
+	MEI_ME_NUM_CFG,
+};
+
+const struct mei_cfg *mei_me_get_cfg(kernel_ulong_t idx);
 
 struct mei_device *mei_me_dev_init(struct pci_dev *pdev,
 				   const struct mei_cfg *cfg);
diff --git a/drivers/misc/mei/pci-me.c b/drivers/misc/mei/pci-me.c
index bac33311f55a..4ff40d319676 100644
--- a/drivers/misc/mei/pci-me.c
+++ b/drivers/misc/mei/pci-me.c
@@ -43,57 +43,58 @@
 
 /* mei_pci_tbl - PCI Device ID Table */
 static const struct pci_device_id mei_me_pci_tbl[] = {
-	{MEI_PCI_DEVICE(MEI_DEV_ID_82946GZ, mei_me_legacy_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_82G35, mei_me_legacy_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_82Q965, mei_me_legacy_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_82G965, mei_me_legacy_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_82GM965, mei_me_legacy_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_82GME965, mei_me_legacy_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_82Q35, mei_me_legacy_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_82G33, mei_me_legacy_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_82Q33, mei_me_legacy_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_82X38, mei_me_legacy_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_3200, mei_me_legacy_cfg)},
-
-	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_6, mei_me_legacy_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_7, mei_me_legacy_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_8, mei_me_legacy_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_9, mei_me_legacy_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_10, mei_me_legacy_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9M_1, mei_me_legacy_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9M_2, mei_me_legacy_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9M_3, mei_me_legacy_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9M_4, mei_me_legacy_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH10_1, mei_me_ich_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH10_2, mei_me_ich_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH10_3, mei_me_ich_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH10_4, mei_me_ich_cfg)},
-
-	{MEI_PCI_DEVICE(MEI_DEV_ID_IBXPK_1, mei_me_pch_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_IBXPK_2, mei_me_pch_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_CPT_1, mei_me_pch_cpt_pbg_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_PBG_1, mei_me_pch_cpt_pbg_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_PPT_1, mei_me_pch_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_PPT_2, mei_me_pch_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_PPT_3, mei_me_pch_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_LPT_H, mei_me_pch8_sps_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_LPT_W, mei_me_pch8_sps_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_LPT_LP, mei_me_pch8_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_LPT_HR, mei_me_pch8_sps_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_WPT_LP, mei_me_pch8_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_WPT_LP_2, mei_me_pch8_cfg)},
-
-	{MEI_PCI_DEVICE(MEI_DEV_ID_SPT, mei_me_pch8_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_SPT_2, mei_me_pch8_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_SPT_H, mei_me_pch8_sps_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_SPT_H_2, mei_me_pch8_sps_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_LBG, mei_me_pch8_cfg)},
-
-	{MEI_PCI_DEVICE(MEI_DEV_ID_BXT_M, mei_me_pch8_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_APL_I, mei_me_pch8_cfg)},
-
-	{MEI_PCI_DEVICE(MEI_DEV_ID_KBP, mei_me_pch8_cfg)},
-	{MEI_PCI_DEVICE(MEI_DEV_ID_KBP_2, mei_me_pch8_cfg)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_82946GZ, MEI_ME_ICH_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_82G35, MEI_ME_ICH_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_82Q965, MEI_ME_ICH_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_82G965, MEI_ME_ICH_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_82GM965, MEI_ME_ICH_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_82GME965, MEI_ME_ICH_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_82Q35, MEI_ME_ICH_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_82G33, MEI_ME_ICH_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_82Q33, MEI_ME_ICH_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_82X38, MEI_ME_ICH_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_3200, MEI_ME_ICH_CFG)},
+
+	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_6, MEI_ME_ICH_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_7, MEI_ME_ICH_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_8, MEI_ME_ICH_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_9, MEI_ME_ICH_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_10, MEI_ME_ICH_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9M_1, MEI_ME_ICH_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9M_2, MEI_ME_ICH_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9M_3, MEI_ME_ICH_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH9M_4, MEI_ME_ICH_CFG)},
+
+	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH10_1, MEI_ME_ICH10_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH10_2, MEI_ME_ICH10_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH10_3, MEI_ME_ICH10_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_ICH10_4, MEI_ME_ICH10_CFG)},
+
+	{MEI_PCI_DEVICE(MEI_DEV_ID_IBXPK_1, MEI_ME_PCH_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_IBXPK_2, MEI_ME_PCH_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_CPT_1, MEI_ME_PCH_CPT_PBG_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_PBG_1, MEI_ME_PCH_CPT_PBG_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_PPT_1, MEI_ME_PCH_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_PPT_2, MEI_ME_PCH_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_PPT_3, MEI_ME_PCH_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_LPT_H, MEI_ME_PCH8_SPS_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_LPT_W, MEI_ME_PCH8_SPS_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_LPT_LP, MEI_ME_PCH8_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_LPT_HR, MEI_ME_PCH8_SPS_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_WPT_LP, MEI_ME_PCH8_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_WPT_LP_2, MEI_ME_PCH8_CFG)},
+
+	{MEI_PCI_DEVICE(MEI_DEV_ID_SPT, MEI_ME_PCH8_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_SPT_2, MEI_ME_PCH8_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_SPT_H, MEI_ME_PCH8_SPS_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_SPT_H_2, MEI_ME_PCH8_SPS_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_LBG, MEI_ME_PCH8_CFG)},
+
+	{MEI_PCI_DEVICE(MEI_DEV_ID_BXT_M, MEI_ME_PCH8_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_APL_I, MEI_ME_PCH8_CFG)},
+
+	{MEI_PCI_DEVICE(MEI_DEV_ID_KBP, MEI_ME_PCH8_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_KBP_2, MEI_ME_PCH8_CFG)},
 
 	/* required last entry */
 	{0, }
@@ -138,12 +139,15 @@ static bool mei_me_quirk_probe(struct pci_dev *pdev,
  */
 static int mei_me_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
-	const struct mei_cfg *cfg = (struct mei_cfg *)(ent->driver_data);
+	const struct mei_cfg *cfg;
 	struct mei_device *dev;
 	struct mei_me_hw *hw;
 	unsigned int irqflags;
 	int err;
 
+	cfg = mei_me_get_cfg(ent->driver_data);
+	if (!cfg)
+		return -ENODEV;
 
 	if (!mei_me_quirk_probe(pdev, cfg))
 		return -ENODEV;
@@ -491,6 +495,7 @@ static struct pci_driver mei_me_driver = {
 	.remove = mei_me_remove,
 	.shutdown = mei_me_shutdown,
 	.driver.pm = MEI_ME_PM_OPS,
+	.driver.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 };
 
 module_pci_driver(mei_me_driver);
diff --git a/drivers/misc/mic/scif/scif_dma.c b/drivers/misc/mic/scif/scif_dma.c
index 64d5760d069a..63d6246d6dff 100644
--- a/drivers/misc/mic/scif/scif_dma.c
+++ b/drivers/misc/mic/scif/scif_dma.c
@@ -200,16 +200,6 @@ static void scif_mmu_notifier_release(struct mmu_notifier *mn,
 	schedule_work(&scif_info.misc_work);
 }
 
-static void scif_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
-					      struct mm_struct *mm,
-					      unsigned long address)
-{
-	struct scif_mmu_notif	*mmn;
-
-	mmn = container_of(mn, struct scif_mmu_notif, ep_mmu_notifier);
-	scif_rma_destroy_tcw(mmn, address, PAGE_SIZE);
-}
-
 static void scif_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
 						     struct mm_struct *mm,
 						     unsigned long start,
@@ -235,7 +225,6 @@ static void scif_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
 static const struct mmu_notifier_ops scif_mmu_notifier_ops = {
 	.release = scif_mmu_notifier_release,
 	.clear_flush_young = NULL,
-	.invalidate_page = scif_mmu_notifier_invalidate_page,
 	.invalidate_range_start = scif_mmu_notifier_invalidate_range_start,
 	.invalidate_range_end = scif_mmu_notifier_invalidate_range_end};
 
diff --git a/drivers/misc/pch_phub.c b/drivers/misc/pch_phub.c
index e42bdc90fa27..540845651b8c 100644
--- a/drivers/misc/pch_phub.c
+++ b/drivers/misc/pch_phub.c
@@ -659,7 +659,7 @@ static ssize_t store_pch_mac(struct device *dev, struct device_attribute *attr,
 
 static DEVICE_ATTR(pch_mac, S_IRUGO | S_IWUSR, show_pch_mac, store_pch_mac);
 
-static struct bin_attribute pch_bin_attr = {
+static const struct bin_attribute pch_bin_attr = {
 	.attr = {
 		.name = "pch_firmware",
 		.mode = S_IRUGO | S_IWUSR,
@@ -891,7 +891,7 @@ static int pch_phub_resume(struct pci_dev *pdev)
 #define pch_phub_resume NULL
 #endif /* CONFIG_PM */
 
-static struct pci_device_id pch_phub_pcidev_id[] = {
+static const struct pci_device_id pch_phub_pcidev_id[] = {
 	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_PCH1_PHUB),       1,  },
 	{ PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ROHM_ML7213_PHUB), 2,  },
 	{ PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ROHM_ML7223_mPHUB), 3,  },
diff --git a/drivers/misc/pci_endpoint_test.c b/drivers/misc/pci_endpoint_test.c
index 09c10f426b64..deb203026496 100644
--- a/drivers/misc/pci_endpoint_test.c
+++ b/drivers/misc/pci_endpoint_test.c
@@ -72,6 +72,11 @@ static DEFINE_IDA(pci_endpoint_test_ida);
 
 #define to_endpoint_test(priv) container_of((priv), struct pci_endpoint_test, \
 					    miscdev)
+
+static bool no_msi;
+module_param(no_msi, bool, 0444);
+MODULE_PARM_DESC(no_msi, "Disable MSI interrupt in pci_endpoint_test");
+
 enum pci_barno {
 	BAR_0,
 	BAR_1,
@@ -90,9 +95,15 @@ struct pci_endpoint_test {
 	/* mutex to protect the ioctls */
 	struct mutex	mutex;
 	struct miscdevice miscdev;
+	enum pci_barno test_reg_bar;
+	size_t alignment;
 };
 
-static int bar_size[] = { 4, 512, 1024, 16384, 131072, 1048576 };
+struct pci_endpoint_test_data {
+	enum pci_barno test_reg_bar;
+	size_t alignment;
+	bool no_msi;
+};
 
 static inline u32 pci_endpoint_test_readl(struct pci_endpoint_test *test,
 					  u32 offset)
@@ -141,11 +152,15 @@ static bool pci_endpoint_test_bar(struct pci_endpoint_test *test,
 	int j;
 	u32 val;
 	int size;
+	struct pci_dev *pdev = test->pdev;
 
 	if (!test->bar[barno])
 		return false;
 
-	size = bar_size[barno];
+	size = pci_resource_len(pdev, barno);
+
+	if (barno == test->test_reg_bar)
+		size = 0x4;
 
 	for (j = 0; j < size; j += 4)
 		pci_endpoint_test_bar_writel(test, barno, j, 0xA0A0A0A0);
@@ -202,16 +217,32 @@ static bool pci_endpoint_test_copy(struct pci_endpoint_test *test, size_t size)
 	dma_addr_t dst_phys_addr;
 	struct pci_dev *pdev = test->pdev;
 	struct device *dev = &pdev->dev;
+	void *orig_src_addr;
+	dma_addr_t orig_src_phys_addr;
+	void *orig_dst_addr;
+	dma_addr_t orig_dst_phys_addr;
+	size_t offset;
+	size_t alignment = test->alignment;
 	u32 src_crc32;
 	u32 dst_crc32;
 
-	src_addr = dma_alloc_coherent(dev, size, &src_phys_addr, GFP_KERNEL);
-	if (!src_addr) {
+	orig_src_addr = dma_alloc_coherent(dev, size + alignment,
+					   &orig_src_phys_addr, GFP_KERNEL);
+	if (!orig_src_addr) {
 		dev_err(dev, "failed to allocate source buffer\n");
 		ret = false;
 		goto err;
 	}
 
+	if (alignment && !IS_ALIGNED(orig_src_phys_addr, alignment)) {
+		src_phys_addr = PTR_ALIGN(orig_src_phys_addr, alignment);
+		offset = src_phys_addr - orig_src_phys_addr;
+		src_addr = orig_src_addr + offset;
+	} else {
+		src_phys_addr = orig_src_phys_addr;
+		src_addr = orig_src_addr;
+	}
+
 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_LOWER_SRC_ADDR,
 				 lower_32_bits(src_phys_addr));
 
@@ -221,11 +252,21 @@ static bool pci_endpoint_test_copy(struct pci_endpoint_test *test, size_t size)
 	get_random_bytes(src_addr, size);
 	src_crc32 = crc32_le(~0, src_addr, size);
 
-	dst_addr = dma_alloc_coherent(dev, size, &dst_phys_addr, GFP_KERNEL);
-	if (!dst_addr) {
+	orig_dst_addr = dma_alloc_coherent(dev, size + alignment,
+					   &orig_dst_phys_addr, GFP_KERNEL);
+	if (!orig_dst_addr) {
 		dev_err(dev, "failed to allocate destination address\n");
 		ret = false;
-		goto err_src_addr;
+		goto err_orig_src_addr;
+	}
+
+	if (alignment && !IS_ALIGNED(orig_dst_phys_addr, alignment)) {
+		dst_phys_addr = PTR_ALIGN(orig_dst_phys_addr, alignment);
+		offset = dst_phys_addr - orig_dst_phys_addr;
+		dst_addr = orig_dst_addr + offset;
+	} else {
+		dst_phys_addr = orig_dst_phys_addr;
+		dst_addr = orig_dst_addr;
 	}
 
 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_LOWER_DST_ADDR,
@@ -245,10 +286,12 @@ static bool pci_endpoint_test_copy(struct pci_endpoint_test *test, size_t size)
 	if (dst_crc32 == src_crc32)
 		ret = true;
 
-	dma_free_coherent(dev, size, dst_addr, dst_phys_addr);
+	dma_free_coherent(dev, size + alignment, orig_dst_addr,
+			  orig_dst_phys_addr);
 
-err_src_addr:
-	dma_free_coherent(dev, size, src_addr, src_phys_addr);
+err_orig_src_addr:
+	dma_free_coherent(dev, size + alignment, orig_src_addr,
+			  orig_src_phys_addr);
 
 err:
 	return ret;
@@ -262,15 +305,29 @@ static bool pci_endpoint_test_write(struct pci_endpoint_test *test, size_t size)
 	dma_addr_t phys_addr;
 	struct pci_dev *pdev = test->pdev;
 	struct device *dev = &pdev->dev;
+	void *orig_addr;
+	dma_addr_t orig_phys_addr;
+	size_t offset;
+	size_t alignment = test->alignment;
 	u32 crc32;
 
-	addr = dma_alloc_coherent(dev, size, &phys_addr, GFP_KERNEL);
-	if (!addr) {
+	orig_addr = dma_alloc_coherent(dev, size + alignment, &orig_phys_addr,
+				       GFP_KERNEL);
+	if (!orig_addr) {
 		dev_err(dev, "failed to allocate address\n");
 		ret = false;
 		goto err;
 	}
 
+	if (alignment && !IS_ALIGNED(orig_phys_addr, alignment)) {
+		phys_addr =  PTR_ALIGN(orig_phys_addr, alignment);
+		offset = phys_addr - orig_phys_addr;
+		addr = orig_addr + offset;
+	} else {
+		phys_addr = orig_phys_addr;
+		addr = orig_addr;
+	}
+
 	get_random_bytes(addr, size);
 
 	crc32 = crc32_le(~0, addr, size);
@@ -293,7 +350,7 @@ static bool pci_endpoint_test_write(struct pci_endpoint_test *test, size_t size)
 	if (reg & STATUS_READ_SUCCESS)
 		ret = true;
 
-	dma_free_coherent(dev, size, addr, phys_addr);
+	dma_free_coherent(dev, size + alignment, orig_addr, orig_phys_addr);
 
 err:
 	return ret;
@@ -306,15 +363,29 @@ static bool pci_endpoint_test_read(struct pci_endpoint_test *test, size_t size)
 	dma_addr_t phys_addr;
 	struct pci_dev *pdev = test->pdev;
 	struct device *dev = &pdev->dev;
+	void *orig_addr;
+	dma_addr_t orig_phys_addr;
+	size_t offset;
+	size_t alignment = test->alignment;
 	u32 crc32;
 
-	addr = dma_alloc_coherent(dev, size, &phys_addr, GFP_KERNEL);
-	if (!addr) {
+	orig_addr = dma_alloc_coherent(dev, size + alignment, &orig_phys_addr,
+				       GFP_KERNEL);
+	if (!orig_addr) {
 		dev_err(dev, "failed to allocate destination address\n");
 		ret = false;
 		goto err;
 	}
 
+	if (alignment && !IS_ALIGNED(orig_phys_addr, alignment)) {
+		phys_addr = PTR_ALIGN(orig_phys_addr, alignment);
+		offset = phys_addr - orig_phys_addr;
+		addr = orig_addr + offset;
+	} else {
+		phys_addr = orig_phys_addr;
+		addr = orig_addr;
+	}
+
 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_LOWER_DST_ADDR,
 				 lower_32_bits(phys_addr));
 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_UPPER_DST_ADDR,
@@ -331,7 +402,7 @@ static bool pci_endpoint_test_read(struct pci_endpoint_test *test, size_t size)
 	if (crc32 == pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_CHECKSUM))
 		ret = true;
 
-	dma_free_coherent(dev, size, addr, phys_addr);
+	dma_free_coherent(dev, size + alignment, orig_addr, orig_phys_addr);
 err:
 	return ret;
 }
@@ -383,13 +454,15 @@ static int pci_endpoint_test_probe(struct pci_dev *pdev,
 {
 	int i;
 	int err;
-	int irq;
+	int irq = 0;
 	int id;
 	char name[20];
 	enum pci_barno bar;
 	void __iomem *base;
 	struct device *dev = &pdev->dev;
 	struct pci_endpoint_test *test;
+	struct pci_endpoint_test_data *data;
+	enum pci_barno test_reg_bar = BAR_0;
 	struct miscdevice *misc_device;
 
 	if (pci_is_bridge(pdev))
@@ -399,7 +472,17 @@ static int pci_endpoint_test_probe(struct pci_dev *pdev,
 	if (!test)
 		return -ENOMEM;
 
+	test->test_reg_bar = 0;
+	test->alignment = 0;
 	test->pdev = pdev;
+
+	data = (struct pci_endpoint_test_data *)ent->driver_data;
+	if (data) {
+		test_reg_bar = data->test_reg_bar;
+		test->alignment = data->alignment;
+		no_msi = data->no_msi;
+	}
+
 	init_completion(&test->irq_raised);
 	mutex_init(&test->mutex);
 
@@ -417,9 +500,11 @@ static int pci_endpoint_test_probe(struct pci_dev *pdev,
 
 	pci_set_master(pdev);
 
-	irq = pci_alloc_irq_vectors(pdev, 1, 32, PCI_IRQ_MSI);
-	if (irq < 0)
-		dev_err(dev, "failed to get MSI interrupts\n");
+	if (!no_msi) {
+		irq = pci_alloc_irq_vectors(pdev, 1, 32, PCI_IRQ_MSI);
+		if (irq < 0)
+			dev_err(dev, "failed to get MSI interrupts\n");
+	}
 
 	err = devm_request_irq(dev, pdev->irq, pci_endpoint_test_irqhandler,
 			       IRQF_SHARED, DRV_MODULE_NAME, test);
@@ -441,14 +526,15 @@ static int pci_endpoint_test_probe(struct pci_dev *pdev,
 		base = pci_ioremap_bar(pdev, bar);
 		if (!base) {
 			dev_err(dev, "failed to read BAR%d\n", bar);
-			WARN_ON(bar == BAR_0);
+			WARN_ON(bar == test_reg_bar);
 		}
 		test->bar[bar] = base;
 	}
 
-	test->base = test->bar[0];
+	test->base = test->bar[test_reg_bar];
 	if (!test->base) {
-		dev_err(dev, "Cannot perform PCI test without BAR0\n");
+		dev_err(dev, "Cannot perform PCI test without BAR%d\n",
+			test_reg_bar);
 		goto err_iounmap;
 	}
 
diff --git a/drivers/misc/sgi-gru/grutlbpurge.c b/drivers/misc/sgi-gru/grutlbpurge.c
index e936d43895d2..9918eda0e05f 100644
--- a/drivers/misc/sgi-gru/grutlbpurge.c
+++ b/drivers/misc/sgi-gru/grutlbpurge.c
@@ -247,17 +247,6 @@ static void gru_invalidate_range_end(struct mmu_notifier *mn,
 	gru_dbg(grudev, "gms %p, start 0x%lx, end 0x%lx\n", gms, start, end);
 }
 
-static void gru_invalidate_page(struct mmu_notifier *mn, struct mm_struct *mm,
-				unsigned long address)
-{
-	struct gru_mm_struct *gms = container_of(mn, struct gru_mm_struct,
-						 ms_notifier);
-
-	STAT(mmu_invalidate_page);
-	gru_flush_tlb_range(gms, address, PAGE_SIZE);
-	gru_dbg(grudev, "gms %p, address 0x%lx\n", gms, address);
-}
-
 static void gru_release(struct mmu_notifier *mn, struct mm_struct *mm)
 {
 	struct gru_mm_struct *gms = container_of(mn, struct gru_mm_struct,
@@ -269,7 +258,6 @@ static void gru_release(struct mmu_notifier *mn, struct mm_struct *mm)
 
 
 static const struct mmu_notifier_ops gru_mmuops = {
-	.invalidate_page	= gru_invalidate_page,
 	.invalidate_range_start	= gru_invalidate_range_start,
 	.invalidate_range_end	= gru_invalidate_range_end,
 	.release		= gru_release,
diff --git a/drivers/misc/sram.c b/drivers/misc/sram.c
index d1185b78cf9a..fc0415771c00 100644
--- a/drivers/misc/sram.c
+++ b/drivers/misc/sram.c
@@ -196,15 +196,15 @@ static int sram_reserve_regions(struct sram_dev *sram, struct resource *res)
 		ret = of_address_to_resource(child, 0, &child_res);
 		if (ret < 0) {
 			dev_err(sram->dev,
-				"could not get address for node %s\n",
-				child->full_name);
+				"could not get address for node %pOF\n",
+				child);
 			goto err_chunks;
 		}
 
 		if (child_res.start < res->start || child_res.end > res->end) {
 			dev_err(sram->dev,
-				"reserved block %s outside the sram area\n",
-				child->full_name);
+				"reserved block %pOF outside the sram area\n",
+				child);
 			ret = -EINVAL;
 			goto err_chunks;
 		}
@@ -230,8 +230,8 @@ static int sram_reserve_regions(struct sram_dev *sram, struct resource *res)
 			ret = of_property_read_string(child, "label", &label);
 			if (ret && ret != -EINVAL) {
 				dev_err(sram->dev,
-					"%s has invalid label name\n",
-					child->full_name);
+					"%pOF has invalid label name\n",
+					child);
 				goto err_chunks;
 			}
 			if (!label)
diff --git a/drivers/misc/ti-st/st_kim.c b/drivers/misc/ti-st/st_kim.c
index e74413f882ab..b77aacafc3fc 100644
--- a/drivers/misc/ti-st/st_kim.c
+++ b/drivers/misc/ti-st/st_kim.c
@@ -660,7 +660,7 @@ static struct attribute *uim_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group uim_attr_grp = {
+static const struct attribute_group uim_attr_grp = {
 	.attrs = uim_attrs,
 };
 
diff --git a/drivers/misc/tifm_7xx1.c b/drivers/misc/tifm_7xx1.c
index a37a42f67088..e5f108713dd8 100644
--- a/drivers/misc/tifm_7xx1.c
+++ b/drivers/misc/tifm_7xx1.c
@@ -415,7 +415,7 @@ static void tifm_7xx1_remove(struct pci_dev *dev)
 	tifm_free_adapter(fm);
 }
 
-static struct pci_device_id tifm_7xx1_pci_tbl [] = {
+static const struct pci_device_id tifm_7xx1_pci_tbl[] = {
 	{ PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_XX21_XX11_FM, PCI_ANY_ID,
 	  PCI_ANY_ID, 0, 0, 0 }, /* xx21 - the one I have */
         { PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_XX12_FM, PCI_ANY_ID,
diff --git a/drivers/misc/vmw_vmci/vmci_queue_pair.c b/drivers/misc/vmw_vmci/vmci_queue_pair.c
index 06c4974ee8dd..8af5c2672f71 100644
--- a/drivers/misc/vmw_vmci/vmci_queue_pair.c
+++ b/drivers/misc/vmw_vmci/vmci_queue_pair.c
@@ -2235,14 +2235,8 @@ int vmci_qp_broker_detach(struct vmci_handle handle, struct vmci_ctx *context)
 					handle.context, handle.resource,
 					result);
 
-			if (entry->vmci_page_files)
-				qp_host_unregister_user_memory(entry->produce_q,
-							       entry->
-							       consume_q);
-			else
-				qp_host_unregister_user_memory(entry->produce_q,
-							       entry->
-							       consume_q);
+			qp_host_unregister_user_memory(entry->produce_q,
+						       entry->consume_q);
 
 		}
 
diff --git a/drivers/mmc/Kconfig b/drivers/mmc/Kconfig
index 7e803fc454d1..ec21388311db 100644
--- a/drivers/mmc/Kconfig
+++ b/drivers/mmc/Kconfig
@@ -12,13 +12,6 @@ menuconfig MMC
 	  If you want MMC/SD/SDIO support, you should say Y here and
 	  also to your specific host controller driver.
 
-config MMC_DEBUG
-	bool "MMC debugging"
-	depends on MMC != n
-	help
-	  This is an option for use by developers; most people should
-	  say N here.  This enables MMC core and driver debugging.
-
 if MMC
 
 source "drivers/mmc/core/Kconfig"
diff --git a/drivers/mmc/Makefile b/drivers/mmc/Makefile
index 416b6d1c9ec6..26ab7af4e0f9 100644
--- a/drivers/mmc/Makefile
+++ b/drivers/mmc/Makefile
@@ -2,7 +2,5 @@
 # Makefile for the kernel mmc device drivers.
 #
 
-subdir-ccflags-$(CONFIG_MMC_DEBUG) := -DDEBUG
-
 obj-$(CONFIG_MMC)		+= core/
 obj-$(subst m,y,$(CONFIG_MMC))	+= host/
diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c
index f1bbfd389367..29fc1e662891 100644
--- a/drivers/mmc/core/block.c
+++ b/drivers/mmc/core/block.c
@@ -36,6 +36,7 @@
 #include <linux/compat.h>
 #include <linux/pm_runtime.h>
 #include <linux/idr.h>
+#include <linux/debugfs.h>
 
 #include <linux/mmc/ioctl.h>
 #include <linux/mmc/card.h>
@@ -126,7 +127,7 @@ module_param(perdev_minors, int, 0444);
 MODULE_PARM_DESC(perdev_minors, "Minors numbers to allocate per device");
 
 static inline int mmc_blk_part_switch(struct mmc_card *card,
-				      struct mmc_blk_data *md);
+				      unsigned int part_type);
 
 static struct mmc_blk_data *mmc_blk_get(struct gendisk *disk)
 {
@@ -188,7 +189,6 @@ static ssize_t power_ro_lock_store(struct device *dev,
 {
 	int ret;
 	struct mmc_blk_data *md, *part_md;
-	struct mmc_card *card;
 	struct mmc_queue *mq;
 	struct request *req;
 	unsigned long set;
@@ -201,7 +201,6 @@ static ssize_t power_ro_lock_store(struct device *dev,
 
 	md = mmc_blk_get(dev_to_disk(dev));
 	mq = &md->queue;
-	card = md->queue.card;
 
 	/* Dispatch locking to the block layer */
 	req = blk_get_request(mq->queue, REQ_OP_DRV_OUT, __GFP_RECLAIM);
@@ -489,7 +488,7 @@ static int __mmc_blk_ioctl_cmd(struct mmc_card *card, struct mmc_blk_data *md,
 
 	mrq.cmd = &cmd;
 
-	err = mmc_blk_part_switch(card, md);
+	err = mmc_blk_part_switch(card, md->part_type);
 	if (err)
 		return err;
 
@@ -554,35 +553,20 @@ static int __mmc_blk_ioctl_cmd(struct mmc_card *card, struct mmc_blk_data *md,
 	return err;
 }
 
-static int mmc_blk_ioctl_cmd(struct block_device *bdev,
+static int mmc_blk_ioctl_cmd(struct mmc_blk_data *md,
 			     struct mmc_ioc_cmd __user *ic_ptr)
 {
 	struct mmc_blk_ioc_data *idata;
 	struct mmc_blk_ioc_data *idatas[1];
-	struct mmc_blk_data *md;
 	struct mmc_queue *mq;
 	struct mmc_card *card;
 	int err = 0, ioc_err = 0;
 	struct request *req;
 
-	/*
-	 * The caller must have CAP_SYS_RAWIO, and must be calling this on the
-	 * whole block device, not on a partition.  This prevents overspray
-	 * between sibling partitions.
-	 */
-	if ((!capable(CAP_SYS_RAWIO)) || (bdev != bdev->bd_contains))
-		return -EPERM;
-
 	idata = mmc_blk_ioctl_copy_from_user(ic_ptr);
 	if (IS_ERR(idata))
 		return PTR_ERR(idata);
 
-	md = mmc_blk_get(bdev->bd_disk);
-	if (!md) {
-		err = -EINVAL;
-		goto cmd_err;
-	}
-
 	card = md->queue.card;
 	if (IS_ERR(card)) {
 		err = PTR_ERR(card);
@@ -598,7 +582,7 @@ static int mmc_blk_ioctl_cmd(struct block_device *bdev,
 		__GFP_RECLAIM);
 	idatas[0] = idata;
 	req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_IOCTL;
-	req_to_mmc_queue_req(req)->idata = idatas;
+	req_to_mmc_queue_req(req)->drv_op_data = idatas;
 	req_to_mmc_queue_req(req)->ioc_count = 1;
 	blk_execute_rq(mq->queue, NULL, req, 0);
 	ioc_err = req_to_mmc_queue_req(req)->drv_op_result;
@@ -606,33 +590,22 @@ static int mmc_blk_ioctl_cmd(struct block_device *bdev,
 	blk_put_request(req);
 
 cmd_done:
-	mmc_blk_put(md);
-cmd_err:
 	kfree(idata->buf);
 	kfree(idata);
 	return ioc_err ? ioc_err : err;
 }
 
-static int mmc_blk_ioctl_multi_cmd(struct block_device *bdev,
+static int mmc_blk_ioctl_multi_cmd(struct mmc_blk_data *md,
 				   struct mmc_ioc_multi_cmd __user *user)
 {
 	struct mmc_blk_ioc_data **idata = NULL;
 	struct mmc_ioc_cmd __user *cmds = user->cmds;
 	struct mmc_card *card;
-	struct mmc_blk_data *md;
 	struct mmc_queue *mq;
 	int i, err = 0, ioc_err = 0;
 	__u64 num_of_cmds;
 	struct request *req;
 
-	/*
-	 * The caller must have CAP_SYS_RAWIO, and must be calling this on the
-	 * whole block device, not on a partition.  This prevents overspray
-	 * between sibling partitions.
-	 */
-	if ((!capable(CAP_SYS_RAWIO)) || (bdev != bdev->bd_contains))
-		return -EPERM;
-
 	if (copy_from_user(&num_of_cmds, &user->num_of_cmds,
 			   sizeof(num_of_cmds)))
 		return -EFAULT;
@@ -656,16 +629,10 @@ static int mmc_blk_ioctl_multi_cmd(struct block_device *bdev,
 		}
 	}
 
-	md = mmc_blk_get(bdev->bd_disk);
-	if (!md) {
-		err = -EINVAL;
-		goto cmd_err;
-	}
-
 	card = md->queue.card;
 	if (IS_ERR(card)) {
 		err = PTR_ERR(card);
-		goto cmd_done;
+		goto cmd_err;
 	}
 
 
@@ -677,7 +644,7 @@ static int mmc_blk_ioctl_multi_cmd(struct block_device *bdev,
 		idata[0]->ic.write_flag ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN,
 		__GFP_RECLAIM);
 	req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_IOCTL;
-	req_to_mmc_queue_req(req)->idata = idata;
+	req_to_mmc_queue_req(req)->drv_op_data = idata;
 	req_to_mmc_queue_req(req)->ioc_count = num_of_cmds;
 	blk_execute_rq(mq->queue, NULL, req, 0);
 	ioc_err = req_to_mmc_queue_req(req)->drv_op_result;
@@ -688,8 +655,6 @@ static int mmc_blk_ioctl_multi_cmd(struct block_device *bdev,
 
 	blk_put_request(req);
 
-cmd_done:
-	mmc_blk_put(md);
 cmd_err:
 	for (i = 0; i < num_of_cmds; i++) {
 		kfree(idata[i]->buf);
@@ -699,16 +664,47 @@ cmd_err:
 	return ioc_err ? ioc_err : err;
 }
 
+static int mmc_blk_check_blkdev(struct block_device *bdev)
+{
+	/*
+	 * The caller must have CAP_SYS_RAWIO, and must be calling this on the
+	 * whole block device, not on a partition.  This prevents overspray
+	 * between sibling partitions.
+	 */
+	if ((!capable(CAP_SYS_RAWIO)) || (bdev != bdev->bd_contains))
+		return -EPERM;
+	return 0;
+}
+
 static int mmc_blk_ioctl(struct block_device *bdev, fmode_t mode,
 	unsigned int cmd, unsigned long arg)
 {
+	struct mmc_blk_data *md;
+	int ret;
+
 	switch (cmd) {
 	case MMC_IOC_CMD:
-		return mmc_blk_ioctl_cmd(bdev,
-				(struct mmc_ioc_cmd __user *)arg);
+		ret = mmc_blk_check_blkdev(bdev);
+		if (ret)
+			return ret;
+		md = mmc_blk_get(bdev->bd_disk);
+		if (!md)
+			return -EINVAL;
+		ret = mmc_blk_ioctl_cmd(md,
+					(struct mmc_ioc_cmd __user *)arg);
+		mmc_blk_put(md);
+		return ret;
 	case MMC_IOC_MULTI_CMD:
-		return mmc_blk_ioctl_multi_cmd(bdev,
-				(struct mmc_ioc_multi_cmd __user *)arg);
+		ret = mmc_blk_check_blkdev(bdev);
+		if (ret)
+			return ret;
+		md = mmc_blk_get(bdev->bd_disk);
+		if (!md)
+			return -EINVAL;
+		ret = mmc_blk_ioctl_multi_cmd(md,
+					(struct mmc_ioc_multi_cmd __user *)arg);
+		mmc_blk_put(md);
+		return ret;
 	default:
 		return -EINVAL;
 	}
@@ -765,29 +761,29 @@ static int mmc_blk_part_switch_post(struct mmc_card *card,
 }
 
 static inline int mmc_blk_part_switch(struct mmc_card *card,
-				      struct mmc_blk_data *md)
+				      unsigned int part_type)
 {
 	int ret = 0;
 	struct mmc_blk_data *main_md = dev_get_drvdata(&card->dev);
 
-	if (main_md->part_curr == md->part_type)
+	if (main_md->part_curr == part_type)
 		return 0;
 
 	if (mmc_card_mmc(card)) {
 		u8 part_config = card->ext_csd.part_config;
 
-		ret = mmc_blk_part_switch_pre(card, md->part_type);
+		ret = mmc_blk_part_switch_pre(card, part_type);
 		if (ret)
 			return ret;
 
 		part_config &= ~EXT_CSD_PART_CONFIG_ACC_MASK;
-		part_config |= md->part_type;
+		part_config |= part_type;
 
 		ret = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
 				 EXT_CSD_PART_CONFIG, part_config,
 				 card->ext_csd.part_time);
 		if (ret) {
-			mmc_blk_part_switch_post(card, md->part_type);
+			mmc_blk_part_switch_post(card, part_type);
 			return ret;
 		}
 
@@ -796,7 +792,7 @@ static inline int mmc_blk_part_switch(struct mmc_card *card,
 		ret = mmc_blk_part_switch_post(card, main_md->part_curr);
 	}
 
-	main_md->part_curr = md->part_type;
+	main_md->part_curr = part_type;
 	return ret;
 }
 
@@ -1139,7 +1135,7 @@ static int mmc_blk_reset(struct mmc_blk_data *md, struct mmc_host *host,
 		int part_err;
 
 		main_md->part_curr = main_md->part_type;
-		part_err = mmc_blk_part_switch(host->card, md);
+		part_err = mmc_blk_part_switch(host->card, md->part_type);
 		if (part_err) {
 			/*
 			 * We have failed to get back into the correct
@@ -1178,6 +1174,10 @@ static void mmc_blk_issue_drv_op(struct mmc_queue *mq, struct request *req)
 	struct mmc_queue_req *mq_rq;
 	struct mmc_card *card = mq->card;
 	struct mmc_blk_data *md = mq->blkdata;
+	struct mmc_blk_data *main_md = dev_get_drvdata(&card->dev);
+	struct mmc_blk_ioc_data **idata;
+	u8 **ext_csd;
+	u32 status;
 	int ret;
 	int i;
 
@@ -1185,14 +1185,15 @@ static void mmc_blk_issue_drv_op(struct mmc_queue *mq, struct request *req)
 
 	switch (mq_rq->drv_op) {
 	case MMC_DRV_OP_IOCTL:
+		idata = mq_rq->drv_op_data;
 		for (i = 0, ret = 0; i < mq_rq->ioc_count; i++) {
-			ret = __mmc_blk_ioctl_cmd(card, md, mq_rq->idata[i]);
+			ret = __mmc_blk_ioctl_cmd(card, md, idata[i]);
 			if (ret)
 				break;
 		}
 		/* Always switch back to main area after RPMB access */
 		if (md->area_type & MMC_BLK_DATA_AREA_RPMB)
-			mmc_blk_part_switch(card, dev_get_drvdata(&card->dev));
+			mmc_blk_part_switch(card, main_md->part_type);
 		break;
 	case MMC_DRV_OP_BOOT_WP:
 		ret = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL, EXT_CSD_BOOT_WP,
@@ -1206,6 +1207,15 @@ static void mmc_blk_issue_drv_op(struct mmc_queue *mq, struct request *req)
 			card->ext_csd.boot_ro_lock |=
 				EXT_CSD_BOOT_WP_B_PWR_WP_EN;
 		break;
+	case MMC_DRV_OP_GET_CARD_STATUS:
+		ret = mmc_send_status(card, &status);
+		if (!ret)
+			ret = status;
+		break;
+	case MMC_DRV_OP_GET_EXT_CSD:
+		ext_csd = mq_rq->drv_op_data;
+		ret = mmc_get_ext_csd(card, ext_csd);
+		break;
 	default:
 		pr_err("%s: unknown driver specific operation\n",
 		       md->disk->disk_name);
@@ -1213,7 +1223,7 @@ static void mmc_blk_issue_drv_op(struct mmc_queue *mq, struct request *req)
 		break;
 	}
 	mq_rq->drv_op_result = ret;
-	blk_end_request_all(req, ret);
+	blk_end_request_all(req, ret ? BLK_STS_IOERR : BLK_STS_OK);
 }
 
 static void mmc_blk_issue_discard_rq(struct mmc_queue *mq, struct request *req)
@@ -1371,12 +1381,46 @@ static inline void mmc_apply_rel_rw(struct mmc_blk_request *brq,
 	 R1_CC_ERROR |		/* Card controller error */		\
 	 R1_ERROR)		/* General/unknown error */
 
-static bool mmc_blk_has_cmd_err(struct mmc_command *cmd)
+static void mmc_blk_eval_resp_error(struct mmc_blk_request *brq)
 {
-	if (!cmd->error && cmd->resp[0] & CMD_ERRORS)
-		cmd->error = -EIO;
+	u32 val;
 
-	return cmd->error;
+	/*
+	 * Per the SD specification(physical layer version 4.10)[1],
+	 * section 4.3.3, it explicitly states that "When the last
+	 * block of user area is read using CMD18, the host should
+	 * ignore OUT_OF_RANGE error that may occur even the sequence
+	 * is correct". And JESD84-B51 for eMMC also has a similar
+	 * statement on section 6.8.3.
+	 *
+	 * Multiple block read/write could be done by either predefined
+	 * method, namely CMD23, or open-ending mode. For open-ending mode,
+	 * we should ignore the OUT_OF_RANGE error as it's normal behaviour.
+	 *
+	 * However the spec[1] doesn't tell us whether we should also
+	 * ignore that for predefined method. But per the spec[1], section
+	 * 4.15 Set Block Count Command, it says"If illegal block count
+	 * is set, out of range error will be indicated during read/write
+	 * operation (For example, data transfer is stopped at user area
+	 * boundary)." In another word, we could expect a out of range error
+	 * in the response for the following CMD18/25. And if argument of
+	 * CMD23 + the argument of CMD18/25 exceed the max number of blocks,
+	 * we could also expect to get a -ETIMEDOUT or any error number from
+	 * the host drivers due to missing data response(for write)/data(for
+	 * read), as the cards will stop the data transfer by itself per the
+	 * spec. So we only need to check R1_OUT_OF_RANGE for open-ending mode.
+	 */
+
+	if (!brq->stop.error) {
+		bool oor_with_open_end;
+		/* If there is no error yet, check R1 response */
+
+		val = brq->stop.resp[0] & CMD_ERRORS;
+		oor_with_open_end = val & R1_OUT_OF_RANGE && !brq->mrq.sbc;
+
+		if (val && !oor_with_open_end)
+			brq->stop.error = -EIO;
+	}
 }
 
 static enum mmc_blk_status mmc_blk_err_check(struct mmc_card *card,
@@ -1400,8 +1444,11 @@ static enum mmc_blk_status mmc_blk_err_check(struct mmc_card *card,
 	 * stop.error indicates a problem with the stop command.  Data
 	 * may have been transferred, or may still be transferring.
 	 */
-	if (brq->sbc.error || brq->cmd.error || mmc_blk_has_cmd_err(&brq->stop) ||
-	    brq->data.error) {
+
+	mmc_blk_eval_resp_error(brq);
+
+	if (brq->sbc.error || brq->cmd.error ||
+	    brq->stop.error || brq->data.error) {
 		switch (mmc_blk_cmd_recovery(card, req, brq, &ecc_err, &gen_err)) {
 		case ERR_RETRY:
 			return MMC_BLK_RETRY;
@@ -1681,9 +1728,9 @@ static bool mmc_blk_rw_cmd_err(struct mmc_blk_data *md, struct mmc_card *card,
 		if (err)
 			req_pending = old_req_pending;
 		else
-			req_pending = blk_end_request(req, 0, blocks << 9);
+			req_pending = blk_end_request(req, BLK_STS_OK, blocks << 9);
 	} else {
-		req_pending = blk_end_request(req, 0, brq->data.bytes_xfered);
+		req_pending = blk_end_request(req, BLK_STS_OK, brq->data.bytes_xfered);
 	}
 	return req_pending;
 }
@@ -1906,7 +1953,7 @@ void mmc_blk_issue_rq(struct mmc_queue *mq, struct request *req)
 		/* claim host only for the first request */
 		mmc_get_card(card);
 
-	ret = mmc_blk_part_switch(card, md);
+	ret = mmc_blk_part_switch(card, md->part_type);
 	if (ret) {
 		if (req) {
 			blk_end_request_all(req, BLK_STS_IOERR);
@@ -1987,8 +2034,20 @@ static struct mmc_blk_data *mmc_blk_alloc_req(struct mmc_card *card,
 	int devidx, ret;
 
 	devidx = ida_simple_get(&mmc_blk_ida, 0, max_devices, GFP_KERNEL);
-	if (devidx < 0)
+	if (devidx < 0) {
+		/*
+		 * We get -ENOSPC because there are no more any available
+		 * devidx. The reason may be that, either userspace haven't yet
+		 * unmounted the partitions, which postpones mmc_blk_release()
+		 * from being called, or the device has more partitions than
+		 * what we support.
+		 */
+		if (devidx == -ENOSPC)
+			dev_err(mmc_dev(card->host),
+				"no more device IDs available\n");
+
 		return ERR_PTR(devidx);
+	}
 
 	md = kzalloc(sizeof(struct mmc_blk_data), GFP_KERNEL);
 	if (!md) {
@@ -2246,6 +2305,134 @@ force_ro_fail:
 	return ret;
 }
 
+#ifdef CONFIG_DEBUG_FS
+
+static int mmc_dbg_card_status_get(void *data, u64 *val)
+{
+	struct mmc_card *card = data;
+	struct mmc_blk_data *md = dev_get_drvdata(&card->dev);
+	struct mmc_queue *mq = &md->queue;
+	struct request *req;
+	int ret;
+
+	/* Ask the block layer about the card status */
+	req = blk_get_request(mq->queue, REQ_OP_DRV_IN, __GFP_RECLAIM);
+	req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_CARD_STATUS;
+	blk_execute_rq(mq->queue, NULL, req, 0);
+	ret = req_to_mmc_queue_req(req)->drv_op_result;
+	if (ret >= 0) {
+		*val = ret;
+		ret = 0;
+	}
+
+	return ret;
+}
+DEFINE_SIMPLE_ATTRIBUTE(mmc_dbg_card_status_fops, mmc_dbg_card_status_get,
+		NULL, "%08llx\n");
+
+/* That is two digits * 512 + 1 for newline */
+#define EXT_CSD_STR_LEN 1025
+
+static int mmc_ext_csd_open(struct inode *inode, struct file *filp)
+{
+	struct mmc_card *card = inode->i_private;
+	struct mmc_blk_data *md = dev_get_drvdata(&card->dev);
+	struct mmc_queue *mq = &md->queue;
+	struct request *req;
+	char *buf;
+	ssize_t n = 0;
+	u8 *ext_csd;
+	int err, i;
+
+	buf = kmalloc(EXT_CSD_STR_LEN + 1, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	/* Ask the block layer for the EXT CSD */
+	req = blk_get_request(mq->queue, REQ_OP_DRV_IN, __GFP_RECLAIM);
+	req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_EXT_CSD;
+	req_to_mmc_queue_req(req)->drv_op_data = &ext_csd;
+	blk_execute_rq(mq->queue, NULL, req, 0);
+	err = req_to_mmc_queue_req(req)->drv_op_result;
+	if (err) {
+		pr_err("FAILED %d\n", err);
+		goto out_free;
+	}
+
+	for (i = 0; i < 512; i++)
+		n += sprintf(buf + n, "%02x", ext_csd[i]);
+	n += sprintf(buf + n, "\n");
+
+	if (n != EXT_CSD_STR_LEN) {
+		err = -EINVAL;
+		goto out_free;
+	}
+
+	filp->private_data = buf;
+	kfree(ext_csd);
+	return 0;
+
+out_free:
+	kfree(buf);
+	return err;
+}
+
+static ssize_t mmc_ext_csd_read(struct file *filp, char __user *ubuf,
+				size_t cnt, loff_t *ppos)
+{
+	char *buf = filp->private_data;
+
+	return simple_read_from_buffer(ubuf, cnt, ppos,
+				       buf, EXT_CSD_STR_LEN);
+}
+
+static int mmc_ext_csd_release(struct inode *inode, struct file *file)
+{
+	kfree(file->private_data);
+	return 0;
+}
+
+static const struct file_operations mmc_dbg_ext_csd_fops = {
+	.open		= mmc_ext_csd_open,
+	.read		= mmc_ext_csd_read,
+	.release	= mmc_ext_csd_release,
+	.llseek		= default_llseek,
+};
+
+static int mmc_blk_add_debugfs(struct mmc_card *card)
+{
+	struct dentry *root;
+
+	if (!card->debugfs_root)
+		return 0;
+
+	root = card->debugfs_root;
+
+	if (mmc_card_mmc(card) || mmc_card_sd(card)) {
+		if (!debugfs_create_file("status", S_IRUSR, root, card,
+					 &mmc_dbg_card_status_fops))
+			return -EIO;
+	}
+
+	if (mmc_card_mmc(card)) {
+		if (!debugfs_create_file("ext_csd", S_IRUSR, root, card,
+					 &mmc_dbg_ext_csd_fops))
+			return -EIO;
+	}
+
+	return 0;
+}
+
+
+#else
+
+static int mmc_blk_add_debugfs(struct mmc_card *card)
+{
+	return 0;
+}
+
+#endif /* CONFIG_DEBUG_FS */
+
 static int mmc_blk_probe(struct mmc_card *card)
 {
 	struct mmc_blk_data *md, *part_md;
@@ -2282,6 +2469,9 @@ static int mmc_blk_probe(struct mmc_card *card)
 			goto out;
 	}
 
+	/* Add two debugfs entries */
+	mmc_blk_add_debugfs(card);
+
 	pm_runtime_set_autosuspend_delay(&card->dev, 3000);
 	pm_runtime_use_autosuspend(&card->dev);
 
@@ -2309,7 +2499,7 @@ static void mmc_blk_remove(struct mmc_card *card)
 	mmc_blk_remove_parts(card, md);
 	pm_runtime_get_sync(&card->dev);
 	mmc_claim_host(card->host);
-	mmc_blk_part_switch(card, md);
+	mmc_blk_part_switch(card, md->part_type);
 	mmc_release_host(card->host);
 	if (card->type != MMC_TYPE_SD_COMBO)
 		pm_runtime_disable(&card->dev);
diff --git a/drivers/mmc/core/core.c b/drivers/mmc/core/core.c
index 26431267a3e2..66c9cf49ad2f 100644
--- a/drivers/mmc/core/core.c
+++ b/drivers/mmc/core/core.c
@@ -260,6 +260,9 @@ static void __mmc_start_request(struct mmc_host *host, struct mmc_request *mrq)
 
 	trace_mmc_request_start(host, mrq);
 
+	if (host->cqe_on)
+		host->cqe_ops->cqe_off(host);
+
 	host->ops->request(host, mrq);
 }
 
@@ -295,10 +298,8 @@ static void mmc_mrq_pr_debug(struct mmc_host *host, struct mmc_request *mrq)
 
 static int mmc_mrq_prep(struct mmc_host *host, struct mmc_request *mrq)
 {
-#ifdef CONFIG_MMC_DEBUG
-	unsigned int i, sz;
+	unsigned int i, sz = 0;
 	struct scatterlist *sg;
-#endif
 
 	if (mrq->cmd) {
 		mrq->cmd->error = 0;
@@ -314,13 +315,12 @@ static int mmc_mrq_prep(struct mmc_host *host, struct mmc_request *mrq)
 		    mrq->data->blocks > host->max_blk_count ||
 		    mrq->data->blocks * mrq->data->blksz > host->max_req_size)
 			return -EINVAL;
-#ifdef CONFIG_MMC_DEBUG
-		sz = 0;
+
 		for_each_sg(mrq->data->sg, sg, mrq->data->sg_len, i)
 			sz += sg->length;
 		if (sz != mrq->data->blocks * mrq->data->blksz)
 			return -EINVAL;
-#endif
+
 		mrq->data->error = 0;
 		mrq->data->mrq = mrq;
 		if (mrq->stop) {
@@ -736,8 +736,8 @@ void mmc_set_data_timeout(struct mmc_data *data, const struct mmc_card *card)
 	if (data->flags & MMC_DATA_WRITE)
 		mult <<= card->csd.r2w_factor;
 
-	data->timeout_ns = card->csd.tacc_ns * mult;
-	data->timeout_clks = card->csd.tacc_clks * mult;
+	data->timeout_ns = card->csd.taac_ns * mult;
+	data->timeout_clks = card->csd.taac_clks * mult;
 
 	/*
 	 * SD cards also have an upper limit on the timeout.
@@ -766,7 +766,7 @@ void mmc_set_data_timeout(struct mmc_data *data, const struct mmc_card *card)
 		/*
 		 * SDHC cards always use these fixed values.
 		 */
-		if (timeout_us > limit_us || mmc_card_blockaddr(card)) {
+		if (timeout_us > limit_us) {
 			data->timeout_ns = limit_us * 1000;
 			data->timeout_clks = 0;
 		}
@@ -982,6 +982,9 @@ int mmc_execute_tuning(struct mmc_card *card)
 	if (!host->ops->execute_tuning)
 		return 0;
 
+	if (host->cqe_on)
+		host->cqe_ops->cqe_off(host);
+
 	if (mmc_card_mmc(card))
 		opcode = MMC_SEND_TUNING_BLOCK_HS200;
 	else
@@ -1021,6 +1024,9 @@ void mmc_set_bus_width(struct mmc_host *host, unsigned int width)
  */
 void mmc_set_initial_state(struct mmc_host *host)
 {
+	if (host->cqe_on)
+		host->cqe_ops->cqe_off(host);
+
 	mmc_retune_disable(host);
 
 	if (mmc_host_is_spi(host))
@@ -1137,11 +1143,11 @@ int mmc_of_parse_voltage(struct device_node *np, u32 *mask)
 	voltage_ranges = of_get_property(np, "voltage-ranges", &num_ranges);
 	num_ranges = num_ranges / sizeof(*voltage_ranges) / 2;
 	if (!voltage_ranges) {
-		pr_debug("%s: voltage-ranges unspecified\n", np->full_name);
+		pr_debug("%pOF: voltage-ranges unspecified\n", np);
 		return 0;
 	}
 	if (!num_ranges) {
-		pr_err("%s: voltage-ranges empty\n", np->full_name);
+		pr_err("%pOF: voltage-ranges empty\n", np);
 		return -EINVAL;
 	}
 
@@ -1153,8 +1159,8 @@ int mmc_of_parse_voltage(struct device_node *np, u32 *mask)
 				be32_to_cpu(voltage_ranges[j]),
 				be32_to_cpu(voltage_ranges[j + 1]));
 		if (!ocr_mask) {
-			pr_err("%s: voltage-range #%d is invalid\n",
-				np->full_name, i);
+			pr_err("%pOF: voltage-range #%d is invalid\n",
+				np, i);
 			return -EINVAL;
 		}
 		*mask |= ocr_mask;
@@ -1769,13 +1775,6 @@ void mmc_detach_bus(struct mmc_host *host)
 static void _mmc_detect_change(struct mmc_host *host, unsigned long delay,
 				bool cd_irq)
 {
-#ifdef CONFIG_MMC_DEBUG
-	unsigned long flags;
-	spin_lock_irqsave(&host->lock, flags);
-	WARN_ON(host->removed);
-	spin_unlock_irqrestore(&host->lock, flags);
-#endif
-
 	/*
 	 * If the device is configured as wakeup, we prevent a new sleep for
 	 * 5 s to give provision for user space to consume the event.
@@ -1869,14 +1868,14 @@ static unsigned int mmc_mmc_erase_timeout(struct mmc_card *card,
 	} else {
 		/* CSD Erase Group Size uses write timeout */
 		unsigned int mult = (10 << card->csd.r2w_factor);
-		unsigned int timeout_clks = card->csd.tacc_clks * mult;
+		unsigned int timeout_clks = card->csd.taac_clks * mult;
 		unsigned int timeout_us;
 
-		/* Avoid overflow: e.g. tacc_ns=80000000 mult=1280 */
-		if (card->csd.tacc_ns < 1000000)
-			timeout_us = (card->csd.tacc_ns * mult) / 1000;
+		/* Avoid overflow: e.g. taac_ns=80000000 mult=1280 */
+		if (card->csd.taac_ns < 1000000)
+			timeout_us = (card->csd.taac_ns * mult) / 1000;
 		else
-			timeout_us = (card->csd.tacc_ns / 1000) * mult;
+			timeout_us = (card->csd.taac_ns / 1000) * mult;
 
 		/*
 		 * ios.clock is only a target.  The real clock rate might be
@@ -2446,10 +2445,9 @@ static int mmc_rescan_try_freq(struct mmc_host *host, unsigned freq)
 {
 	host->f_init = freq;
 
-#ifdef CONFIG_MMC_DEBUG
-	pr_info("%s: %s: trying to init card at %u Hz\n",
+	pr_debug("%s: %s: trying to init card at %u Hz\n",
 		mmc_hostname(host), __func__, host->f_init);
-#endif
+
 	mmc_power_up(host, host->ocr_avail);
 
 	/*
@@ -2646,12 +2644,6 @@ void mmc_start_host(struct mmc_host *host)
 
 void mmc_stop_host(struct mmc_host *host)
 {
-#ifdef CONFIG_MMC_DEBUG
-	unsigned long flags;
-	spin_lock_irqsave(&host->lock, flags);
-	host->removed = 1;
-	spin_unlock_irqrestore(&host->lock, flags);
-#endif
 	if (host->slot.cd_irq >= 0) {
 		if (host->slot.cd_wake_enabled)
 			disable_irq_wake(host->slot.cd_irq);
@@ -2686,9 +2678,7 @@ int mmc_power_save_host(struct mmc_host *host)
 {
 	int ret = 0;
 
-#ifdef CONFIG_MMC_DEBUG
-	pr_info("%s: %s: powering down\n", mmc_hostname(host), __func__);
-#endif
+	pr_debug("%s: %s: powering down\n", mmc_hostname(host), __func__);
 
 	mmc_bus_get(host);
 
@@ -2712,9 +2702,7 @@ int mmc_power_restore_host(struct mmc_host *host)
 {
 	int ret;
 
-#ifdef CONFIG_MMC_DEBUG
-	pr_info("%s: %s: powering up\n", mmc_hostname(host), __func__);
-#endif
+	pr_debug("%s: %s: powering up\n", mmc_hostname(host), __func__);
 
 	mmc_bus_get(host);
 
diff --git a/drivers/mmc/core/core.h b/drivers/mmc/core/core.h
index 55f543fd37c4..ca861091a776 100644
--- a/drivers/mmc/core/core.h
+++ b/drivers/mmc/core/core.h
@@ -107,6 +107,12 @@ static inline void mmc_unregister_pm_notifier(struct mmc_host *host) { }
 void mmc_wait_for_req_done(struct mmc_host *host, struct mmc_request *mrq);
 bool mmc_is_req_done(struct mmc_host *host, struct mmc_request *mrq);
 
+struct mmc_async_req;
+
+struct mmc_async_req *mmc_start_areq(struct mmc_host *host,
+				     struct mmc_async_req *areq,
+				     enum mmc_blk_status *ret_stat);
+
 int mmc_erase(struct mmc_card *card, unsigned int from, unsigned int nr,
 		unsigned int arg);
 int mmc_can_erase(struct mmc_card *card);
diff --git a/drivers/mmc/core/debugfs.c b/drivers/mmc/core/debugfs.c
index a1fba5732d66..01e459a34f33 100644
--- a/drivers/mmc/core/debugfs.c
+++ b/drivers/mmc/core/debugfs.c
@@ -281,85 +281,6 @@ void mmc_remove_host_debugfs(struct mmc_host *host)
 	debugfs_remove_recursive(host->debugfs_root);
 }
 
-static int mmc_dbg_card_status_get(void *data, u64 *val)
-{
-	struct mmc_card	*card = data;
-	u32		status;
-	int		ret;
-
-	mmc_get_card(card);
-
-	ret = mmc_send_status(data, &status);
-	if (!ret)
-		*val = status;
-
-	mmc_put_card(card);
-
-	return ret;
-}
-DEFINE_SIMPLE_ATTRIBUTE(mmc_dbg_card_status_fops, mmc_dbg_card_status_get,
-		NULL, "%08llx\n");
-
-#define EXT_CSD_STR_LEN 1025
-
-static int mmc_ext_csd_open(struct inode *inode, struct file *filp)
-{
-	struct mmc_card *card = inode->i_private;
-	char *buf;
-	ssize_t n = 0;
-	u8 *ext_csd;
-	int err, i;
-
-	buf = kmalloc(EXT_CSD_STR_LEN + 1, GFP_KERNEL);
-	if (!buf)
-		return -ENOMEM;
-
-	mmc_get_card(card);
-	err = mmc_get_ext_csd(card, &ext_csd);
-	mmc_put_card(card);
-	if (err)
-		goto out_free;
-
-	for (i = 0; i < 512; i++)
-		n += sprintf(buf + n, "%02x", ext_csd[i]);
-	n += sprintf(buf + n, "\n");
-
-	if (n != EXT_CSD_STR_LEN) {
-		err = -EINVAL;
-		goto out_free;
-	}
-
-	filp->private_data = buf;
-	kfree(ext_csd);
-	return 0;
-
-out_free:
-	kfree(buf);
-	return err;
-}
-
-static ssize_t mmc_ext_csd_read(struct file *filp, char __user *ubuf,
-				size_t cnt, loff_t *ppos)
-{
-	char *buf = filp->private_data;
-
-	return simple_read_from_buffer(ubuf, cnt, ppos,
-				       buf, EXT_CSD_STR_LEN);
-}
-
-static int mmc_ext_csd_release(struct inode *inode, struct file *file)
-{
-	kfree(file->private_data);
-	return 0;
-}
-
-static const struct file_operations mmc_dbg_ext_csd_fops = {
-	.open		= mmc_ext_csd_open,
-	.read		= mmc_ext_csd_read,
-	.release	= mmc_ext_csd_release,
-	.llseek		= default_llseek,
-};
-
 void mmc_add_card_debugfs(struct mmc_card *card)
 {
 	struct mmc_host	*host = card->host;
@@ -382,16 +303,6 @@ void mmc_add_card_debugfs(struct mmc_card *card)
 	if (!debugfs_create_x32("state", S_IRUSR, root, &card->state))
 		goto err;
 
-	if (mmc_card_mmc(card) || mmc_card_sd(card))
-		if (!debugfs_create_file("status", S_IRUSR, root, card,
-					&mmc_dbg_card_status_fops))
-			goto err;
-
-	if (mmc_card_mmc(card))
-		if (!debugfs_create_file("ext_csd", S_IRUSR, root, card,
-					&mmc_dbg_ext_csd_fops))
-			goto err;
-
 	return;
 
 err:
diff --git a/drivers/mmc/core/host.c b/drivers/mmc/core/host.c
index 1503412f826c..ad88deb2e8f3 100644
--- a/drivers/mmc/core/host.c
+++ b/drivers/mmc/core/host.c
@@ -111,6 +111,12 @@ void mmc_retune_hold(struct mmc_host *host)
 	host->hold_retune += 1;
 }
 
+void mmc_retune_hold_now(struct mmc_host *host)
+{
+	host->retune_now = 0;
+	host->hold_retune += 1;
+}
+
 void mmc_retune_release(struct mmc_host *host)
 {
 	if (host->hold_retune)
diff --git a/drivers/mmc/core/host.h b/drivers/mmc/core/host.h
index fb6a76a03833..77d6f60d1bf9 100644
--- a/drivers/mmc/core/host.h
+++ b/drivers/mmc/core/host.h
@@ -19,6 +19,7 @@ void mmc_unregister_host_class(void);
 void mmc_retune_enable(struct mmc_host *host);
 void mmc_retune_disable(struct mmc_host *host);
 void mmc_retune_hold(struct mmc_host *host);
+void mmc_retune_hold_now(struct mmc_host *host);
 void mmc_retune_release(struct mmc_host *host);
 int mmc_retune(struct mmc_host *host);
 void mmc_retune_pause(struct mmc_host *host);
diff --git a/drivers/mmc/core/mmc.c b/drivers/mmc/core/mmc.c
index 2bae69e39544..a7eb623f8daa 100644
--- a/drivers/mmc/core/mmc.c
+++ b/drivers/mmc/core/mmc.c
@@ -41,11 +41,11 @@ static const unsigned char tran_mant[] = {
 	35,	40,	45,	50,	55,	60,	70,	80,
 };
 
-static const unsigned int tacc_exp[] = {
+static const unsigned int taac_exp[] = {
 	1,	10,	100,	1000,	10000,	100000,	1000000, 10000000,
 };
 
-static const unsigned int tacc_mant[] = {
+static const unsigned int taac_mant[] = {
 	0,	10,	12,	13,	15,	20,	25,	30,
 	35,	40,	45,	50,	55,	60,	70,	80,
 };
@@ -153,8 +153,8 @@ static int mmc_decode_csd(struct mmc_card *card)
 	csd->mmca_vsn	 = UNSTUFF_BITS(resp, 122, 4);
 	m = UNSTUFF_BITS(resp, 115, 4);
 	e = UNSTUFF_BITS(resp, 112, 3);
-	csd->tacc_ns	 = (tacc_exp[e] * tacc_mant[m] + 9) / 10;
-	csd->tacc_clks	 = UNSTUFF_BITS(resp, 104, 8) * 100;
+	csd->taac_ns	 = (taac_exp[e] * taac_mant[m] + 9) / 10;
+	csd->taac_clks	 = UNSTUFF_BITS(resp, 104, 8) * 100;
 
 	m = UNSTUFF_BITS(resp, 99, 4);
 	e = UNSTUFF_BITS(resp, 96, 3);
@@ -1790,29 +1790,6 @@ static int mmc_init_card(struct mmc_host *host, u32 ocr,
 	 */
 	card->reenable_cmdq = card->ext_csd.cmdq_en;
 
-	/*
-	 * The mandatory minimum values are defined for packed command.
-	 * read: 5, write: 3
-	 */
-	if (card->ext_csd.max_packed_writes >= 3 &&
-	    card->ext_csd.max_packed_reads >= 5 &&
-	    host->caps2 & MMC_CAP2_PACKED_CMD) {
-		err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
-				EXT_CSD_EXP_EVENTS_CTRL,
-				EXT_CSD_PACKED_EVENT_EN,
-				card->ext_csd.generic_cmd6_time);
-		if (err && err != -EBADMSG)
-			goto free_card;
-		if (err) {
-			pr_warn("%s: Enabling packed event failed\n",
-				mmc_hostname(card->host));
-			card->ext_csd.packed_event_en = 0;
-			err = 0;
-		} else {
-			card->ext_csd.packed_event_en = 1;
-		}
-	}
-
 	if (!oldcard)
 		host->card = card;
 
diff --git a/drivers/mmc/core/mmc_ops.c b/drivers/mmc/core/mmc_ops.c
index 5f7c5920231a..54686ca4bfb7 100644
--- a/drivers/mmc/core/mmc_ops.c
+++ b/drivers/mmc/core/mmc_ops.c
@@ -83,6 +83,7 @@ int mmc_send_status(struct mmc_card *card, u32 *status)
 {
 	return __mmc_send_status(card, status, MMC_CMD_RETRIES);
 }
+EXPORT_SYMBOL_GPL(mmc_send_status);
 
 static int _mmc_select_card(struct mmc_host *host, struct mmc_card *card)
 {
@@ -946,7 +947,7 @@ static int mmc_read_bkops_status(struct mmc_card *card)
 /**
  *	mmc_start_bkops - start BKOPS for supported cards
  *	@card: MMC card to start BKOPS
- *	@form_exception: A flag to indicate if this function was
+ *	@from_exception: A flag to indicate if this function was
  *			 called due to an exception raised by the card
  *
  *	Start background operations whenever requested.
diff --git a/drivers/mmc/core/mmc_test.c b/drivers/mmc/core/mmc_test.c
index 7a304a6e5bf1..478869805b96 100644
--- a/drivers/mmc/core/mmc_test.c
+++ b/drivers/mmc/core/mmc_test.c
@@ -800,38 +800,44 @@ static int mmc_test_check_broken_result(struct mmc_test_card *test,
 	return ret;
 }
 
+struct mmc_test_req {
+	struct mmc_request mrq;
+	struct mmc_command sbc;
+	struct mmc_command cmd;
+	struct mmc_command stop;
+	struct mmc_command status;
+	struct mmc_data data;
+};
+
 /*
  * Tests nonblock transfer with certain parameters
  */
-static void mmc_test_nonblock_reset(struct mmc_request *mrq,
-				    struct mmc_command *cmd,
-				    struct mmc_command *stop,
-				    struct mmc_data *data)
+static void mmc_test_req_reset(struct mmc_test_req *rq)
+{
+	memset(rq, 0, sizeof(struct mmc_test_req));
+
+	rq->mrq.cmd = &rq->cmd;
+	rq->mrq.data = &rq->data;
+	rq->mrq.stop = &rq->stop;
+}
+
+static struct mmc_test_req *mmc_test_req_alloc(void)
 {
-	memset(mrq, 0, sizeof(struct mmc_request));
-	memset(cmd, 0, sizeof(struct mmc_command));
-	memset(data, 0, sizeof(struct mmc_data));
-	memset(stop, 0, sizeof(struct mmc_command));
+	struct mmc_test_req *rq = kmalloc(sizeof(*rq), GFP_KERNEL);
 
-	mrq->cmd = cmd;
-	mrq->data = data;
-	mrq->stop = stop;
+	if (rq)
+		mmc_test_req_reset(rq);
+
+	return rq;
 }
+
+
 static int mmc_test_nonblock_transfer(struct mmc_test_card *test,
 				      struct scatterlist *sg, unsigned sg_len,
 				      unsigned dev_addr, unsigned blocks,
 				      unsigned blksz, int write, int count)
 {
-	struct mmc_request mrq1;
-	struct mmc_command cmd1;
-	struct mmc_command stop1;
-	struct mmc_data data1;
-
-	struct mmc_request mrq2;
-	struct mmc_command cmd2;
-	struct mmc_command stop2;
-	struct mmc_data data2;
-
+	struct mmc_test_req *rq1, *rq2;
 	struct mmc_test_async_req test_areq[2];
 	struct mmc_async_req *done_areq;
 	struct mmc_async_req *cur_areq = &test_areq[0].areq;
@@ -843,12 +849,16 @@ static int mmc_test_nonblock_transfer(struct mmc_test_card *test,
 	test_areq[0].test = test;
 	test_areq[1].test = test;
 
-	mmc_test_nonblock_reset(&mrq1, &cmd1, &stop1, &data1);
-	mmc_test_nonblock_reset(&mrq2, &cmd2, &stop2, &data2);
+	rq1 = mmc_test_req_alloc();
+	rq2 = mmc_test_req_alloc();
+	if (!rq1 || !rq2) {
+		ret = RESULT_FAIL;
+		goto err;
+	}
 
-	cur_areq->mrq = &mrq1;
+	cur_areq->mrq = &rq1->mrq;
 	cur_areq->err_check = mmc_test_check_result_async;
-	other_areq->mrq = &mrq2;
+	other_areq->mrq = &rq2->mrq;
 	other_areq->err_check = mmc_test_check_result_async;
 
 	for (i = 0; i < count; i++) {
@@ -861,14 +871,10 @@ static int mmc_test_nonblock_transfer(struct mmc_test_card *test,
 			goto err;
 		}
 
-		if (done_areq) {
-			if (done_areq->mrq == &mrq2)
-				mmc_test_nonblock_reset(&mrq2, &cmd2,
-							&stop2, &data2);
-			else
-				mmc_test_nonblock_reset(&mrq1, &cmd1,
-							&stop1, &data1);
-		}
+		if (done_areq)
+			mmc_test_req_reset(container_of(done_areq->mrq,
+						struct mmc_test_req, mrq));
+
 		swap(cur_areq, other_areq);
 		dev_addr += blocks;
 	}
@@ -877,8 +883,9 @@ static int mmc_test_nonblock_transfer(struct mmc_test_card *test,
 	if (status != MMC_BLK_SUCCESS)
 		ret = RESULT_FAIL;
 
-	return ret;
 err:
+	kfree(rq1);
+	kfree(rq2);
 	return ret;
 }
 
@@ -2329,28 +2336,6 @@ static int mmc_test_reset(struct mmc_test_card *test)
 	return RESULT_FAIL;
 }
 
-struct mmc_test_req {
-	struct mmc_request mrq;
-	struct mmc_command sbc;
-	struct mmc_command cmd;
-	struct mmc_command stop;
-	struct mmc_command status;
-	struct mmc_data data;
-};
-
-static struct mmc_test_req *mmc_test_req_alloc(void)
-{
-	struct mmc_test_req *rq = kzalloc(sizeof(*rq), GFP_KERNEL);
-
-	if (rq) {
-		rq->mrq.cmd = &rq->cmd;
-		rq->mrq.data = &rq->data;
-		rq->mrq.stop = &rq->stop;
-	}
-
-	return rq;
-}
-
 static int mmc_test_send_status(struct mmc_test_card *test,
 				struct mmc_command *cmd)
 {
diff --git a/drivers/mmc/core/queue.c b/drivers/mmc/core/queue.c
index affa7370ba82..74c663b1c0a7 100644
--- a/drivers/mmc/core/queue.c
+++ b/drivers/mmc/core/queue.c
@@ -242,6 +242,12 @@ int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card,
 	if (mmc_dev(host)->dma_mask && *mmc_dev(host)->dma_mask)
 		limit = (u64)dma_max_pfn(mmc_dev(host)) << PAGE_SHIFT;
 
+	/*
+	 * mmc_init_request() depends on card->bouncesz so it must be calculated
+	 * before blk_init_allocated_queue() starts allocating requests.
+	 */
+	card->bouncesz = mmc_queue_calc_bouncesz(host);
+
 	mq->card = card;
 	mq->queue = blk_alloc_queue(GFP_KERNEL);
 	if (!mq->queue)
@@ -265,7 +271,6 @@ int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card,
 	if (mmc_can_erase(card))
 		mmc_queue_setup_discard(mq->queue, card);
 
-	card->bouncesz = mmc_queue_calc_bouncesz(host);
 	if (card->bouncesz) {
 		blk_queue_max_hw_sectors(mq->queue, card->bouncesz / 512);
 		blk_queue_max_segments(mq->queue, card->bouncesz / 512);
diff --git a/drivers/mmc/core/queue.h b/drivers/mmc/core/queue.h
index 361b46408e0f..04fc89360a7a 100644
--- a/drivers/mmc/core/queue.h
+++ b/drivers/mmc/core/queue.h
@@ -36,10 +36,14 @@ struct mmc_blk_request {
  * enum mmc_drv_op - enumerates the operations in the mmc_queue_req
  * @MMC_DRV_OP_IOCTL: ioctl operation
  * @MMC_DRV_OP_BOOT_WP: write protect boot partitions
+ * @MMC_DRV_OP_GET_CARD_STATUS: get card status
+ * @MMC_DRV_OP_GET_EXT_CSD: get the EXT CSD from an eMMC card
  */
 enum mmc_drv_op {
 	MMC_DRV_OP_IOCTL,
 	MMC_DRV_OP_BOOT_WP,
+	MMC_DRV_OP_GET_CARD_STATUS,
+	MMC_DRV_OP_GET_EXT_CSD,
 };
 
 struct mmc_queue_req {
@@ -51,7 +55,7 @@ struct mmc_queue_req {
 	struct mmc_async_req	areq;
 	enum mmc_drv_op		drv_op;
 	int			drv_op_result;
-	struct mmc_blk_ioc_data	**idata;
+	void			*drv_op_data;
 	unsigned int		ioc_count;
 };
 
diff --git a/drivers/mmc/core/sd.c b/drivers/mmc/core/sd.c
index a1b0aa14d5e3..4fd1620b732d 100644
--- a/drivers/mmc/core/sd.c
+++ b/drivers/mmc/core/sd.c
@@ -39,11 +39,11 @@ static const unsigned char tran_mant[] = {
 	35,	40,	45,	50,	55,	60,	70,	80,
 };
 
-static const unsigned int tacc_exp[] = {
+static const unsigned int taac_exp[] = {
 	1,	10,	100,	1000,	10000,	100000,	1000000, 10000000,
 };
 
-static const unsigned int tacc_mant[] = {
+static const unsigned int taac_mant[] = {
 	0,	10,	12,	13,	15,	20,	25,	30,
 	35,	40,	45,	50,	55,	60,	70,	80,
 };
@@ -111,8 +111,8 @@ static int mmc_decode_csd(struct mmc_card *card)
 	case 0:
 		m = UNSTUFF_BITS(resp, 115, 4);
 		e = UNSTUFF_BITS(resp, 112, 3);
-		csd->tacc_ns	 = (tacc_exp[e] * tacc_mant[m] + 9) / 10;
-		csd->tacc_clks	 = UNSTUFF_BITS(resp, 104, 8) * 100;
+		csd->taac_ns	 = (taac_exp[e] * taac_mant[m] + 9) / 10;
+		csd->taac_clks	 = UNSTUFF_BITS(resp, 104, 8) * 100;
 
 		m = UNSTUFF_BITS(resp, 99, 4);
 		e = UNSTUFF_BITS(resp, 96, 3);
@@ -148,8 +148,8 @@ static int mmc_decode_csd(struct mmc_card *card)
 		 */
 		mmc_card_set_blockaddr(card);
 
-		csd->tacc_ns	 = 0; /* Unused */
-		csd->tacc_clks	 = 0; /* Unused */
+		csd->taac_ns	 = 0; /* Unused */
+		csd->taac_clks	 = 0; /* Unused */
 
 		m = UNSTUFF_BITS(resp, 99, 4);
 		e = UNSTUFF_BITS(resp, 96, 3);
diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig
index 5755b69f2f72..8c15637178ff 100644
--- a/drivers/mmc/host/Kconfig
+++ b/drivers/mmc/host/Kconfig
@@ -4,6 +4,15 @@
 
 comment "MMC/SD/SDIO Host Controller Drivers"
 
+config MMC_DEBUG
+	bool "MMC host drivers debugging"
+	depends on MMC != n
+	help
+	  This is an option for use by developers; most people should
+	  say N here. This enables MMC host driver debugging. And further
+	  added host drivers please don't invent their private macro for
+	  debugging.
+
 config MMC_ARMMMCI
 	tristate "ARM AMBA Multimedia Card Interface support"
 	depends on ARM_AMBA
@@ -15,7 +24,7 @@ config MMC_ARMMMCI
 	  If unsure, say N.
 
 config MMC_QCOM_DML
-	tristate "Qualcomm Data Mover for SD Card Controller"
+	bool "Qualcomm Data Mover for SD Card Controller"
 	depends on MMC_ARMMMCI && QCOM_BAM_DMA
 	default y
 	help
@@ -354,7 +363,7 @@ config MMC_MOXART
 
 config MMC_SDHCI_ST
 	tristate "SDHCI support on STMicroelectronics SoC"
-	depends on ARCH_STI
+	depends on ARCH_STI || FSP2
 	depends on MMC_SDHCI_PLTFM
 	select MMC_SDHCI_IO_ACCESSORS
 	help
@@ -494,7 +503,7 @@ config MMC_GOLDFISH
 
 config MMC_SPI
 	tristate "MMC/SD/SDIO over SPI"
-	depends on SPI_MASTER && !HIGHMEM && HAS_DMA
+	depends on SPI_MASTER && HAS_DMA
 	select CRC7
 	select CRC_ITU_T
 	help
@@ -575,10 +584,29 @@ config MMC_SDHI
 	depends on SUPERH || ARM || ARM64
 	depends on SUPERH || ARCH_RENESAS || COMPILE_TEST
 	select MMC_TMIO_CORE
+	select MMC_SDHI_SYS_DMAC if (SUPERH || ARM)
+	select MMC_SDHI_INTERNAL_DMAC if ARM64
 	help
 	  This provides support for the SDHI SD/SDIO controller found in
 	  Renesas SuperH, ARM and ARM64 based SoCs
 
+config MMC_SDHI_SYS_DMAC
+	tristate "DMA for SDHI SD/SDIO controllers using SYS-DMAC"
+	depends on MMC_SDHI
+	help
+	  This provides DMA support for SDHI SD/SDIO controllers
+	  using SYS-DMAC via DMA Engine. This supports the controllers
+	  found in SuperH and Renesas ARM based SoCs.
+
+config MMC_SDHI_INTERNAL_DMAC
+	tristate "DMA for SDHI SD/SDIO controllers using on-chip bus mastering"
+	depends on ARM64 || COMPILE_TEST
+	depends on MMC_SDHI
+	help
+	  This provides DMA support for SDHI SD/SDIO controllers
+	  using on-chip bus mastering. This supports the controllers
+	  found in arm64 based SoCs.
+
 config MMC_CB710
 	tristate "ENE CB710 MMC/SD Interface support"
 	depends on PCI
diff --git a/drivers/mmc/host/Makefile b/drivers/mmc/host/Makefile
index 4d4547116311..303f5cd46cd9 100644
--- a/drivers/mmc/host/Makefile
+++ b/drivers/mmc/host/Makefile
@@ -2,8 +2,9 @@
 # Makefile for MMC/SD host controller drivers
 #
 
-obj-$(CONFIG_MMC_ARMMMCI)	+= mmci.o
-obj-$(CONFIG_MMC_QCOM_DML)	+= mmci_qcom_dml.o
+obj-$(CONFIG_MMC_ARMMMCI) += armmmci.o
+armmmci-y := mmci.o
+armmmci-$(CONFIG_MMC_QCOM_DML) += mmci_qcom_dml.o
 obj-$(CONFIG_MMC_PXA)		+= pxamci.o
 obj-$(CONFIG_MMC_MXC)		+= mxcmmc.o
 obj-$(CONFIG_MMC_MXS)		+= mxs-mmc.o
@@ -36,7 +37,13 @@ obj-$(CONFIG_MMC_S3C)   	+= s3cmci.o
 obj-$(CONFIG_MMC_SDRICOH_CS)	+= sdricoh_cs.o
 obj-$(CONFIG_MMC_TMIO)		+= tmio_mmc.o
 obj-$(CONFIG_MMC_TMIO_CORE)	+= tmio_mmc_core.o
-obj-$(CONFIG_MMC_SDHI)		+= renesas_sdhi_core.o renesas_sdhi_sys_dmac.o
+obj-$(CONFIG_MMC_SDHI)		+= renesas_sdhi_core.o
+ifeq ($(subst m,y,$(CONFIG_MMC_SDHI_SYS_DMAC)),y)
+obj-$(CONFIG_MMC_SDHI)		+= renesas_sdhi_sys_dmac.o
+endif
+ifeq ($(subst m,y,$(CONFIG_MMC_SDHI_INTERNAL_DMAC)),y)
+obj-$(CONFIG_MMC_SDHI)		+= renesas_sdhi_internal_dmac.o
+endif
 obj-$(CONFIG_MMC_CB710)		+= cb710-mmc.o
 obj-$(CONFIG_MMC_VIA_SDMMC)	+= via-sdmmc.o
 obj-$(CONFIG_SDH_BFIN)		+= bfin_sdh.o
diff --git a/drivers/mmc/host/android-goldfish.c b/drivers/mmc/host/android-goldfish.c
index 5b3e1c9bb75f..63fe5091ca59 100644
--- a/drivers/mmc/host/android-goldfish.c
+++ b/drivers/mmc/host/android-goldfish.c
@@ -290,7 +290,6 @@ static irqreturn_t goldfish_mmc_irq(int irq, void *dev_id)
 	u16 status;
 	int end_command = 0;
 	int end_transfer = 0;
-	int transfer_error = 0;
 	int state_changed = 0;
 	int cmd_timeout = 0;
 
@@ -322,9 +321,7 @@ static irqreturn_t goldfish_mmc_irq(int irq, void *dev_id)
 	if (end_command)
 		goldfish_mmc_cmd_done(host, host->cmd);
 
-	if (transfer_error)
-		goldfish_mmc_xfer_done(host, host->data);
-	else if (end_transfer) {
+	if (end_transfer) {
 		host->dma_done = 1;
 		goldfish_mmc_end_of_data(host, host->data);
 	} else if (host->data != NULL) {
@@ -347,8 +344,7 @@ static irqreturn_t goldfish_mmc_irq(int irq, void *dev_id)
 		mmc_detect_change(host->mmc, 0);
 	}
 
-	if (!end_command && !end_transfer &&
-	    !transfer_error && !state_changed && !cmd_timeout) {
+	if (!end_command && !end_transfer && !state_changed && !cmd_timeout) {
 		status = GOLDFISH_MMC_READ(host, MMC_INT_STATUS);
 		dev_info(mmc_dev(host->mmc),"spurious irq 0x%04x\n", status);
 		if (status != 0) {
diff --git a/drivers/mmc/host/atmel-mci.c b/drivers/mmc/host/atmel-mci.c
index 97de2d32ba84..0a0ebf3a096d 100644
--- a/drivers/mmc/host/atmel-mci.c
+++ b/drivers/mmc/host/atmel-mci.c
@@ -665,14 +665,15 @@ atmci_of_init(struct platform_device *pdev)
 
 	for_each_child_of_node(np, cnp) {
 		if (of_property_read_u32(cnp, "reg", &slot_id)) {
-			dev_warn(&pdev->dev, "reg property is missing for %s\n",
-				 cnp->full_name);
+			dev_warn(&pdev->dev, "reg property is missing for %pOF\n",
+				 cnp);
 			continue;
 		}
 
 		if (slot_id >= ATMCI_MAX_NR_SLOTS) {
 			dev_warn(&pdev->dev, "can't have more than %d slots\n",
 			         ATMCI_MAX_NR_SLOTS);
+			of_node_put(cnp);
 			break;
 		}
 
@@ -1083,7 +1084,6 @@ static u32
 atmci_prepare_data_pdc(struct atmel_mci *host, struct mmc_data *data)
 {
 	u32 iflags, tmp;
-	unsigned int sg_len;
 	int i;
 
 	data->error = -EINPROGRESS;
@@ -1108,8 +1108,8 @@ atmci_prepare_data_pdc(struct atmel_mci *host, struct mmc_data *data)
 
 	/* Configure PDC */
 	host->data_size = data->blocks * data->blksz;
-	sg_len = dma_map_sg(&host->pdev->dev, data->sg, data->sg_len,
-			    mmc_get_dma_dir(data));
+	dma_map_sg(&host->pdev->dev, data->sg, data->sg_len,
+		   mmc_get_dma_dir(data));
 
 	if ((!host->caps.has_rwproof)
 	    && (host->data->flags & MMC_DATA_WRITE)) {
diff --git a/drivers/mmc/host/bcm2835.c b/drivers/mmc/host/bcm2835.c
index abba9a2a78b8..229dc18f0581 100644
--- a/drivers/mmc/host/bcm2835.c
+++ b/drivers/mmc/host/bcm2835.c
@@ -1252,7 +1252,7 @@ static void bcm2835_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 	mutex_unlock(&host->mutex);
 }
 
-static struct mmc_host_ops bcm2835_ops = {
+static const struct mmc_host_ops bcm2835_ops = {
 	.request = bcm2835_request,
 	.set_ios = bcm2835_set_ios,
 	.hw_reset = bcm2835_reset,
diff --git a/drivers/mmc/host/cavium-octeon.c b/drivers/mmc/host/cavium-octeon.c
index 951d2cdd7888..22aded1065ae 100644
--- a/drivers/mmc/host/cavium-octeon.c
+++ b/drivers/mmc/host/cavium-octeon.c
@@ -342,18 +342,7 @@ static struct platform_driver octeon_mmc_driver = {
 	},
 };
 
-static int __init octeon_mmc_init(void)
-{
-	return platform_driver_register(&octeon_mmc_driver);
-}
-
-static void __exit octeon_mmc_cleanup(void)
-{
-	platform_driver_unregister(&octeon_mmc_driver);
-}
-
-module_init(octeon_mmc_init);
-module_exit(octeon_mmc_cleanup);
+module_platform_driver(octeon_mmc_driver);
 
 MODULE_AUTHOR("Cavium Inc. <support@cavium.com>");
 MODULE_DESCRIPTION("Low-level driver for Cavium OCTEON MMC/SSD card");
diff --git a/drivers/mmc/host/cavium-thunderx.c b/drivers/mmc/host/cavium-thunderx.c
index b9cc95998799..eee08d81b242 100644
--- a/drivers/mmc/host/cavium-thunderx.c
+++ b/drivers/mmc/host/cavium-thunderx.c
@@ -7,6 +7,7 @@
  *
  * Copyright (C) 2016 Cavium Inc.
  */
+#include <linux/device.h>
 #include <linux/dma-mapping.h>
 #include <linux/interrupt.h>
 #include <linux/mmc/mmc.h>
@@ -149,8 +150,11 @@ error:
 	for (i = 0; i < CAVIUM_MAX_MMC; i++) {
 		if (host->slot[i])
 			cvm_mmc_of_slot_remove(host->slot[i]);
-		if (host->slot_pdev[i])
+		if (host->slot_pdev[i]) {
+			get_device(&host->slot_pdev[i]->dev);
 			of_platform_device_destroy(&host->slot_pdev[i]->dev, NULL);
+			put_device(&host->slot_pdev[i]->dev);
+		}
 	}
 	clk_disable_unprepare(host->clk);
 	return ret;
diff --git a/drivers/mmc/host/cavium.c b/drivers/mmc/host/cavium.c
index 3686d77c717b..27fb625cbcf3 100644
--- a/drivers/mmc/host/cavium.c
+++ b/drivers/mmc/host/cavium.c
@@ -957,14 +957,12 @@ static int cvm_mmc_of_parse(struct device *dev, struct cvm_mmc_slot *slot)
 
 	ret = of_property_read_u32(node, "reg", &id);
 	if (ret) {
-		dev_err(dev, "Missing or invalid reg property on %s\n",
-			of_node_full_name(node));
+		dev_err(dev, "Missing or invalid reg property on %pOF\n", node);
 		return ret;
 	}
 
 	if (id >= CAVIUM_MAX_MMC || slot->host->slot[id]) {
-		dev_err(dev, "Invalid reg property on %s\n",
-			of_node_full_name(node));
+		dev_err(dev, "Invalid reg property on %pOF\n", node);
 		return -EINVAL;
 	}
 
diff --git a/drivers/mmc/host/davinci_mmc.c b/drivers/mmc/host/davinci_mmc.c
index 621ce47e0e4a..351330dfb954 100644
--- a/drivers/mmc/host/davinci_mmc.c
+++ b/drivers/mmc/host/davinci_mmc.c
@@ -1062,7 +1062,7 @@ static void mmc_davinci_enable_sdio_irq(struct mmc_host *mmc, int enable)
 	}
 }
 
-static struct mmc_host_ops mmc_davinci_ops = {
+static const struct mmc_host_ops mmc_davinci_ops = {
 	.request	= mmc_davinci_request,
 	.set_ios	= mmc_davinci_set_ios,
 	.get_cd		= mmc_davinci_get_cd,
diff --git a/drivers/mmc/host/dw_mmc-k3.c b/drivers/mmc/host/dw_mmc-k3.c
index e38fb0020bb1..64cda84b2302 100644
--- a/drivers/mmc/host/dw_mmc-k3.c
+++ b/drivers/mmc/host/dw_mmc-k3.c
@@ -8,6 +8,8 @@
  * (at your option) any later version.
  */
 
+#include <linux/bitops.h>
+#include <linux/bitfield.h>
 #include <linux/clk.h>
 #include <linux/mfd/syscon.h>
 #include <linux/mmc/host.h>
@@ -28,7 +30,35 @@
 #define AO_SCTRL_SEL18		BIT(10)
 #define AO_SCTRL_CTRL3		0x40C
 
+#define DWMMC_SDIO_ID 2
+
+#define SOC_SCTRL_SCPERCTRL5    (0x314)
+#define SDCARD_IO_SEL18         BIT(2)
+
+#define SDCARD_RD_THRESHOLD  (512)
+
+#define GENCLK_DIV (7)
+
+#define GPIO_CLK_ENABLE                   BIT(16)
+#define GPIO_CLK_DIV_MASK                 GENMASK(11, 8)
+#define GPIO_USE_SAMPLE_DLY_MASK          GENMASK(13, 13)
+#define UHS_REG_EXT_SAMPLE_PHASE_MASK     GENMASK(20, 16)
+#define UHS_REG_EXT_SAMPLE_DRVPHASE_MASK  GENMASK(25, 21)
+#define UHS_REG_EXT_SAMPLE_DLY_MASK       GENMASK(30, 26)
+
+#define TIMING_MODE     3
+#define TIMING_CFG_NUM 10
+
+#define NUM_PHASES (40)
+
+#define ENABLE_SHIFT_MIN_SMPL (4)
+#define ENABLE_SHIFT_MAX_SMPL (12)
+#define USE_DLY_MIN_SMPL (11)
+#define USE_DLY_MAX_SMPL (14)
+
 struct k3_priv {
+	int ctrl_id;
+	u32 cur_speed;
 	struct regmap	*reg;
 };
 
@@ -38,6 +68,41 @@ static unsigned long dw_mci_hi6220_caps[] = {
 	0
 };
 
+struct hs_timing {
+	u32 drv_phase;
+	u32 smpl_dly;
+	u32 smpl_phase_max;
+	u32 smpl_phase_min;
+};
+
+struct hs_timing hs_timing_cfg[TIMING_MODE][TIMING_CFG_NUM] = {
+	{ /* reserved */ },
+	{ /* SD */
+		{7, 0, 15, 15,},  /* 0: LEGACY 400k */
+		{6, 0,  4,  4,},  /* 1: MMC_HS */
+		{6, 0,  3,  3,},  /* 2: SD_HS */
+		{6, 0, 15, 15,},  /* 3: SDR12 */
+		{6, 0,  2,  2,},  /* 4: SDR25 */
+		{4, 0, 11,  0,},  /* 5: SDR50 */
+		{6, 4, 15,  0,},  /* 6: SDR104 */
+		{0},              /* 7: DDR50 */
+		{0},              /* 8: DDR52 */
+		{0},              /* 9: HS200 */
+	},
+	{ /* SDIO */
+		{7, 0, 15, 15,},  /* 0: LEGACY 400k */
+		{0},              /* 1: MMC_HS */
+		{6, 0, 15, 15,},  /* 2: SD_HS */
+		{6, 0, 15, 15,},  /* 3: SDR12 */
+		{6, 0,  0,  0,},  /* 4: SDR25 */
+		{4, 0, 12,  0,},  /* 5: SDR50 */
+		{5, 4, 15,  0,},  /* 6: SDR104 */
+		{0},              /* 7: DDR50 */
+		{0},              /* 8: DDR52 */
+		{0},              /* 9: HS200 */
+	}
+};
+
 static void dw_mci_k3_set_ios(struct dw_mci *host, struct mmc_ios *ios)
 {
 	int ret;
@@ -66,6 +131,10 @@ static int dw_mci_hi6220_parse_dt(struct dw_mci *host)
 	if (IS_ERR(priv->reg))
 		priv->reg = NULL;
 
+	priv->ctrl_id = of_alias_get_id(host->dev->of_node, "mshc");
+	if (priv->ctrl_id < 0)
+		priv->ctrl_id = 0;
+
 	host->priv = priv;
 	return 0;
 }
@@ -144,7 +213,236 @@ static const struct dw_mci_drv_data hi6220_data = {
 	.execute_tuning		= dw_mci_hi6220_execute_tuning,
 };
 
+static void dw_mci_hs_set_timing(struct dw_mci *host, int timing,
+				     int smpl_phase)
+{
+	u32 drv_phase;
+	u32 smpl_dly;
+	u32 use_smpl_dly = 0;
+	u32 enable_shift = 0;
+	u32 reg_value;
+	int ctrl_id;
+	struct k3_priv *priv;
+
+	priv = host->priv;
+	ctrl_id = priv->ctrl_id;
+
+	drv_phase = hs_timing_cfg[ctrl_id][timing].drv_phase;
+	smpl_dly   = hs_timing_cfg[ctrl_id][timing].smpl_dly;
+	if (smpl_phase == -1)
+		smpl_phase = (hs_timing_cfg[ctrl_id][timing].smpl_phase_max +
+			     hs_timing_cfg[ctrl_id][timing].smpl_phase_min) / 2;
+
+	switch (timing) {
+	case MMC_TIMING_UHS_SDR104:
+		if (smpl_phase >= USE_DLY_MIN_SMPL &&
+				smpl_phase <= USE_DLY_MAX_SMPL)
+			use_smpl_dly = 1;
+			/* fallthrough */
+	case MMC_TIMING_UHS_SDR50:
+		if (smpl_phase >= ENABLE_SHIFT_MIN_SMPL &&
+				smpl_phase <= ENABLE_SHIFT_MAX_SMPL)
+			enable_shift = 1;
+		break;
+	}
+
+	mci_writel(host, GPIO, 0x0);
+	usleep_range(5, 10);
+
+	reg_value = FIELD_PREP(UHS_REG_EXT_SAMPLE_PHASE_MASK, smpl_phase) |
+		    FIELD_PREP(UHS_REG_EXT_SAMPLE_DLY_MASK, smpl_dly) |
+		    FIELD_PREP(UHS_REG_EXT_SAMPLE_DRVPHASE_MASK, drv_phase);
+	mci_writel(host, UHS_REG_EXT, reg_value);
+
+	mci_writel(host, ENABLE_SHIFT, enable_shift);
+
+	reg_value = FIELD_PREP(GPIO_CLK_DIV_MASK, GENCLK_DIV) |
+			     FIELD_PREP(GPIO_USE_SAMPLE_DLY_MASK, use_smpl_dly);
+	mci_writel(host, GPIO, (unsigned int)reg_value | GPIO_CLK_ENABLE);
+
+	/* We should delay 1ms wait for timing setting finished. */
+	usleep_range(1000, 2000);
+}
+
+static int dw_mci_hi3660_init(struct dw_mci *host)
+{
+	mci_writel(host, CDTHRCTL, SDMMC_SET_THLD(SDCARD_RD_THRESHOLD,
+		    SDMMC_CARD_RD_THR_EN));
+
+	dw_mci_hs_set_timing(host, MMC_TIMING_LEGACY, -1);
+	host->bus_hz /= (GENCLK_DIV + 1);
+
+	return 0;
+}
+
+static int dw_mci_set_sel18(struct dw_mci *host, bool set)
+{
+	int ret;
+	unsigned int val;
+	struct k3_priv *priv;
+
+	priv = host->priv;
+
+	val = set ? SDCARD_IO_SEL18 : 0;
+	ret = regmap_update_bits(priv->reg, SOC_SCTRL_SCPERCTRL5,
+				 SDCARD_IO_SEL18, val);
+	if (ret) {
+		dev_err(host->dev, "sel18 %u error\n", val);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void dw_mci_hi3660_set_ios(struct dw_mci *host, struct mmc_ios *ios)
+{
+	int ret;
+	unsigned long wanted;
+	unsigned long actual;
+	struct k3_priv *priv = host->priv;
+
+	if (!ios->clock || ios->clock == priv->cur_speed)
+		return;
+
+	wanted = ios->clock * (GENCLK_DIV + 1);
+	ret = clk_set_rate(host->ciu_clk, wanted);
+	if (ret) {
+		dev_err(host->dev, "failed to set rate %luHz\n", wanted);
+		return;
+	}
+	actual = clk_get_rate(host->ciu_clk);
+
+	dw_mci_hs_set_timing(host, ios->timing, -1);
+	host->bus_hz = actual / (GENCLK_DIV + 1);
+	host->current_speed = 0;
+	priv->cur_speed = host->bus_hz;
+}
+
+static int dw_mci_get_best_clksmpl(unsigned int sample_flag)
+{
+	int i;
+	int interval;
+	unsigned int v;
+	unsigned int len;
+	unsigned int range_start = 0;
+	unsigned int range_length = 0;
+	unsigned int middle_range = 0;
+
+	if (!sample_flag)
+		return -EIO;
+
+	if (~sample_flag == 0)
+		return 0;
+
+	i = ffs(sample_flag) - 1;
+
+	/*
+	* A clock cycle is divided into 32 phases,
+	* each of which is represented by a bit,
+	* finding the optimal phase.
+	*/
+	while (i < 32) {
+		v = ror32(sample_flag, i);
+		len = ffs(~v) - 1;
+
+		if (len > range_length) {
+			range_length = len;
+			range_start = i;
+		}
+
+		interval = ffs(v >> len) - 1;
+		if (interval < 0)
+			break;
+
+		i += len + interval;
+	}
+
+	middle_range = range_start + range_length / 2;
+	if (middle_range >= 32)
+		middle_range %= 32;
+
+	return middle_range;
+}
+
+static int dw_mci_hi3660_execute_tuning(struct dw_mci_slot *slot, u32 opcode)
+{
+	int i = 0;
+	struct dw_mci *host = slot->host;
+	struct mmc_host *mmc = slot->mmc;
+	int smpl_phase = 0;
+	u32 tuning_sample_flag = 0;
+	int best_clksmpl = 0;
+
+	for (i = 0; i < NUM_PHASES; ++i, ++smpl_phase) {
+		smpl_phase %= 32;
+
+		mci_writel(host, TMOUT, ~0);
+		dw_mci_hs_set_timing(host, mmc->ios.timing, smpl_phase);
+
+		if (!mmc_send_tuning(mmc, opcode, NULL))
+			tuning_sample_flag |= (1 << smpl_phase);
+		else
+			tuning_sample_flag &= ~(1 << smpl_phase);
+	}
+
+	best_clksmpl = dw_mci_get_best_clksmpl(tuning_sample_flag);
+	if (best_clksmpl < 0) {
+		dev_err(host->dev, "All phases bad!\n");
+		return -EIO;
+	}
+
+	dw_mci_hs_set_timing(host, mmc->ios.timing, best_clksmpl);
+
+	dev_info(host->dev, "tuning ok best_clksmpl %u tuning_sample_flag %x\n",
+		 best_clksmpl, tuning_sample_flag);
+	return 0;
+}
+
+static int dw_mci_hi3660_switch_voltage(struct mmc_host *mmc,
+					struct mmc_ios *ios)
+{
+	int ret = 0;
+	struct dw_mci_slot *slot = mmc_priv(mmc);
+	struct k3_priv *priv;
+	struct dw_mci *host;
+
+	host = slot->host;
+	priv = host->priv;
+
+	if (!priv || !priv->reg)
+		return 0;
+
+	if (priv->ctrl_id == DWMMC_SDIO_ID)
+		return 0;
+
+	if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_330)
+		ret = dw_mci_set_sel18(host, 0);
+	else if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_180)
+		ret = dw_mci_set_sel18(host, 1);
+	if (ret)
+		return ret;
+
+	if (!IS_ERR(mmc->supply.vqmmc)) {
+		ret = mmc_regulator_set_vqmmc(mmc, ios);
+		if (ret) {
+			dev_err(host->dev, "Regulator set error %d\n", ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static const struct dw_mci_drv_data hi3660_data = {
+	.init = dw_mci_hi3660_init,
+	.set_ios = dw_mci_hi3660_set_ios,
+	.parse_dt = dw_mci_hi6220_parse_dt,
+	.execute_tuning = dw_mci_hi3660_execute_tuning,
+	.switch_voltage  = dw_mci_hi3660_switch_voltage,
+};
+
 static const struct of_device_id dw_mci_k3_match[] = {
+	{ .compatible = "hisilicon,hi3660-dw-mshc", .data = &hi3660_data, },
 	{ .compatible = "hisilicon,hi4511-dw-mshc", .data = &k3_drv_data, },
 	{ .compatible = "hisilicon,hi6220-dw-mshc", .data = &hi6220_data, },
 	{},
diff --git a/drivers/mmc/host/dw_mmc.c b/drivers/mmc/host/dw_mmc.c
index 250dc6ec4c82..860313bd952a 100644
--- a/drivers/mmc/host/dw_mmc.c
+++ b/drivers/mmc/host/dw_mmc.c
@@ -398,6 +398,21 @@ static u32 dw_mci_prep_stop_abort(struct dw_mci *host, struct mmc_command *cmd)
 	return cmdr;
 }
 
+static inline void dw_mci_set_cto(struct dw_mci *host)
+{
+	unsigned int cto_clks;
+	unsigned int cto_ms;
+
+	cto_clks = mci_readl(host, TMOUT) & 0xff;
+	cto_ms = DIV_ROUND_UP(cto_clks, host->bus_hz / 1000);
+
+	/* add a bit spare time */
+	cto_ms += 10;
+
+	mod_timer(&host->cto_timer,
+		  jiffies + msecs_to_jiffies(cto_ms) + 1);
+}
+
 static void dw_mci_start_command(struct dw_mci *host,
 				 struct mmc_command *cmd, u32 cmd_flags)
 {
@@ -410,6 +425,10 @@ static void dw_mci_start_command(struct dw_mci *host,
 	wmb(); /* drain writebuffer */
 	dw_mci_wait_while_busy(host, cmd_flags);
 
+	/* response expected command only */
+	if (cmd_flags & SDMMC_CMD_RESP_EXP)
+		dw_mci_set_cto(host);
+
 	mci_writel(host, CMD, cmd_flags | SDMMC_CMD_START);
 }
 
@@ -2599,6 +2618,7 @@ static irqreturn_t dw_mci_interrupt(int irq, void *dev_id)
 		}
 
 		if (pending & DW_MCI_CMD_ERROR_FLAGS) {
+			del_timer(&host->cto_timer);
 			mci_writel(host, RINTSTS, DW_MCI_CMD_ERROR_FLAGS);
 			host->cmd_status = pending;
 			smp_wmb(); /* drain writebuffer */
@@ -2642,6 +2662,7 @@ static irqreturn_t dw_mci_interrupt(int irq, void *dev_id)
 		}
 
 		if (pending & SDMMC_INT_CMD_DONE) {
+			del_timer(&host->cto_timer);
 			mci_writel(host, RINTSTS, SDMMC_INT_CMD_DONE);
 			dw_mci_cmd_interrupt(host, pending);
 		}
@@ -2914,6 +2935,30 @@ static void dw_mci_cmd11_timer(unsigned long arg)
 	tasklet_schedule(&host->tasklet);
 }
 
+static void dw_mci_cto_timer(unsigned long arg)
+{
+	struct dw_mci *host = (struct dw_mci *)arg;
+
+	switch (host->state) {
+	case STATE_SENDING_CMD11:
+	case STATE_SENDING_CMD:
+	case STATE_SENDING_STOP:
+		/*
+		 * If CMD_DONE interrupt does NOT come in sending command
+		 * state, we should notify the driver to terminate current
+		 * transfer and report a command timeout to the core.
+		 */
+		host->cmd_status = SDMMC_INT_RTO;
+		set_bit(EVENT_CMD_COMPLETE, &host->pending_events);
+		tasklet_schedule(&host->tasklet);
+		break;
+	default:
+		dev_warn(host->dev, "Unexpected command timeout, state %d\n",
+			 host->state);
+		break;
+	}
+}
+
 static void dw_mci_dto_timer(unsigned long arg)
 {
 	struct dw_mci *host = (struct dw_mci *)arg;
@@ -2950,7 +2995,7 @@ static struct dw_mci_board *dw_mci_parse_dt(struct dw_mci *host)
 		return ERR_PTR(-ENOMEM);
 
 	/* find reset controller when exist */
-	pdata->rstc = devm_reset_control_get_optional(dev, "reset");
+	pdata->rstc = devm_reset_control_get_optional_exclusive(dev, "reset");
 	if (IS_ERR(pdata->rstc)) {
 		if (PTR_ERR(pdata->rstc) == -EPROBE_DEFER)
 			return ERR_PTR(-EPROBE_DEFER);
@@ -3067,6 +3112,12 @@ int dw_mci_probe(struct dw_mci *host)
 		goto err_clk_ciu;
 	}
 
+	if (!IS_ERR(host->pdata->rstc)) {
+		reset_control_assert(host->pdata->rstc);
+		usleep_range(10, 50);
+		reset_control_deassert(host->pdata->rstc);
+	}
+
 	if (drv_data && drv_data->init) {
 		ret = drv_data->init(host);
 		if (ret) {
@@ -3076,15 +3127,12 @@ int dw_mci_probe(struct dw_mci *host)
 		}
 	}
 
-	if (!IS_ERR(host->pdata->rstc)) {
-		reset_control_assert(host->pdata->rstc);
-		usleep_range(10, 50);
-		reset_control_deassert(host->pdata->rstc);
-	}
-
 	setup_timer(&host->cmd11_timer,
 		    dw_mci_cmd11_timer, (unsigned long)host);
 
+	setup_timer(&host->cto_timer,
+		    dw_mci_cto_timer, (unsigned long)host);
+
 	setup_timer(&host->dto_timer,
 		    dw_mci_dto_timer, (unsigned long)host);
 
diff --git a/drivers/mmc/host/dw_mmc.h b/drivers/mmc/host/dw_mmc.h
index 75da3756955d..34474ad731aa 100644
--- a/drivers/mmc/host/dw_mmc.h
+++ b/drivers/mmc/host/dw_mmc.h
@@ -126,6 +126,7 @@ struct dw_mci_dma_slave {
  * @irq: The irq value to be passed to request_irq.
  * @sdio_id0: Number of slot0 in the SDIO interrupt registers.
  * @cmd11_timer: Timer for SD3.0 voltage switch over scheme.
+ * @cto_timer: Timer for broken command transfer over scheme.
  * @dto_timer: Timer for broken data transfer over scheme.
  *
  * Locking
@@ -232,6 +233,7 @@ struct dw_mci {
 	int			sdio_id0;
 
 	struct timer_list       cmd11_timer;
+	struct timer_list       cto_timer;
 	struct timer_list       dto_timer;
 };
 
@@ -314,6 +316,8 @@ struct dw_mci_board {
 #define SDMMC_DSCADDR		0x094
 #define SDMMC_BUFADDR		0x098
 #define SDMMC_CDTHRCTL		0x100
+#define SDMMC_UHS_REG_EXT	0x108
+#define SDMMC_ENABLE_SHIFT	0x110
 #define SDMMC_DATA(x)		(x)
 /*
 * Registers to support idmac 64-bit address mode
diff --git a/drivers/mmc/host/meson-gx-mmc.c b/drivers/mmc/host/meson-gx-mmc.c
index de962c2d5e00..c885c2d4b904 100644
--- a/drivers/mmc/host/meson-gx-mmc.c
+++ b/drivers/mmc/host/meson-gx-mmc.c
@@ -42,22 +42,18 @@
 
 #define SD_EMMC_CLOCK 0x0
 #define   CLK_DIV_MASK GENMASK(5, 0)
-#define   CLK_DIV_MAX 63
 #define   CLK_SRC_MASK GENMASK(7, 6)
-#define   CLK_SRC_XTAL 0   /* external crystal */
-#define   CLK_SRC_XTAL_RATE 24000000
-#define   CLK_SRC_PLL 1    /* FCLK_DIV2 */
-#define   CLK_SRC_PLL_RATE 1000000000
 #define   CLK_CORE_PHASE_MASK GENMASK(9, 8)
 #define   CLK_TX_PHASE_MASK GENMASK(11, 10)
 #define   CLK_RX_PHASE_MASK GENMASK(13, 12)
-#define   CLK_PHASE_0 0
-#define   CLK_PHASE_90 1
-#define   CLK_PHASE_180 2
-#define   CLK_PHASE_270 3
+#define   CLK_TX_DELAY_MASK GENMASK(19, 16)
+#define   CLK_RX_DELAY_MASK GENMASK(23, 20)
+#define   CLK_DELAY_STEP_PS 200
+#define   CLK_PHASE_STEP 30
+#define   CLK_PHASE_POINT_NUM (360 / CLK_PHASE_STEP)
 #define   CLK_ALWAYS_ON BIT(24)
 
-#define SD_EMMC_DElAY 0x4
+#define SD_EMMC_DELAY 0x4
 #define SD_EMMC_ADJUST 0x8
 #define SD_EMMC_CALOUT 0x10
 #define SD_EMMC_START 0x40
@@ -81,18 +77,25 @@
 
 #define SD_EMMC_STATUS 0x48
 #define   STATUS_BUSY BIT(31)
+#define   STATUS_DATI GENMASK(23, 16)
 
 #define SD_EMMC_IRQ_EN 0x4c
-#define   IRQ_EN_MASK GENMASK(13, 0)
 #define   IRQ_RXD_ERR_MASK GENMASK(7, 0)
 #define   IRQ_TXD_ERR BIT(8)
 #define   IRQ_DESC_ERR BIT(9)
 #define   IRQ_RESP_ERR BIT(10)
+#define   IRQ_CRC_ERR \
+	(IRQ_RXD_ERR_MASK | IRQ_TXD_ERR | IRQ_DESC_ERR | IRQ_RESP_ERR)
 #define   IRQ_RESP_TIMEOUT BIT(11)
 #define   IRQ_DESC_TIMEOUT BIT(12)
+#define   IRQ_TIMEOUTS \
+	(IRQ_RESP_TIMEOUT | IRQ_DESC_TIMEOUT)
 #define   IRQ_END_OF_CHAIN BIT(13)
 #define   IRQ_RESP_STATUS BIT(14)
 #define   IRQ_SDIO BIT(15)
+#define   IRQ_EN_MASK \
+	(IRQ_CRC_ERR | IRQ_TIMEOUTS | IRQ_END_OF_CHAIN | IRQ_RESP_STATUS |\
+	 IRQ_SDIO)
 
 #define SD_EMMC_CMD_CFG 0x50
 #define SD_EMMC_CMD_ARG 0x54
@@ -118,12 +121,6 @@
 
 #define MUX_CLK_NUM_PARENTS 2
 
-struct meson_tuning_params {
-	u8 core_phase;
-	u8 tx_phase;
-	u8 rx_phase;
-};
-
 struct sd_emmc_desc {
 	u32 cmd_cfg;
 	u32 cmd_arg;
@@ -139,12 +136,14 @@ struct meson_host {
 	spinlock_t lock;
 	void __iomem *regs;
 	struct clk *core_clk;
-	struct clk_mux mux;
-	struct clk *mux_clk;
-	unsigned long current_clock;
+	struct clk *mmc_clk;
+	struct clk *rx_clk;
+	struct clk *tx_clk;
+	unsigned long req_rate;
 
-	struct clk_divider cfg_div;
-	struct clk *cfg_div_clk;
+	struct pinctrl *pinctrl;
+	struct pinctrl_state *pins_default;
+	struct pinctrl_state *pins_clk_gate;
 
 	unsigned int bounce_buf_size;
 	void *bounce_buf;
@@ -152,7 +151,6 @@ struct meson_host {
 	struct sd_emmc_desc *descs;
 	dma_addr_t descs_dma_addr;
 
-	struct meson_tuning_params tp;
 	bool vqmmc_enabled;
 };
 
@@ -179,6 +177,90 @@ struct meson_host {
 #define CMD_RESP_MASK GENMASK(31, 1)
 #define CMD_RESP_SRAM BIT(0)
 
+struct meson_mmc_phase {
+	struct clk_hw hw;
+	void __iomem *reg;
+	unsigned long phase_mask;
+	unsigned long delay_mask;
+	unsigned int delay_step_ps;
+};
+
+#define to_meson_mmc_phase(_hw) container_of(_hw, struct meson_mmc_phase, hw)
+
+static int meson_mmc_clk_get_phase(struct clk_hw *hw)
+{
+	struct meson_mmc_phase *mmc = to_meson_mmc_phase(hw);
+	unsigned int phase_num = 1 <<  hweight_long(mmc->phase_mask);
+	unsigned long period_ps, p, d;
+		int degrees;
+	u32 val;
+
+	val = readl(mmc->reg);
+	p = (val & mmc->phase_mask) >> __ffs(mmc->phase_mask);
+	degrees = p * 360 / phase_num;
+
+	if (mmc->delay_mask) {
+		period_ps = DIV_ROUND_UP((unsigned long)NSEC_PER_SEC * 1000,
+					 clk_get_rate(hw->clk));
+		d = (val & mmc->delay_mask) >> __ffs(mmc->delay_mask);
+		degrees += d * mmc->delay_step_ps * 360 / period_ps;
+		degrees %= 360;
+	}
+
+	return degrees;
+}
+
+static void meson_mmc_apply_phase_delay(struct meson_mmc_phase *mmc,
+					unsigned int phase,
+					unsigned int delay)
+{
+	u32 val;
+
+	val = readl(mmc->reg);
+	val &= ~mmc->phase_mask;
+	val |= phase << __ffs(mmc->phase_mask);
+
+	if (mmc->delay_mask) {
+		val &= ~mmc->delay_mask;
+		val |= delay << __ffs(mmc->delay_mask);
+	}
+
+	writel(val, mmc->reg);
+}
+
+static int meson_mmc_clk_set_phase(struct clk_hw *hw, int degrees)
+{
+	struct meson_mmc_phase *mmc = to_meson_mmc_phase(hw);
+	unsigned int phase_num = 1 <<  hweight_long(mmc->phase_mask);
+	unsigned long period_ps, d = 0, r;
+	uint64_t p;
+
+	p = degrees % 360;
+
+	if (!mmc->delay_mask) {
+		p = DIV_ROUND_CLOSEST_ULL(p, 360 / phase_num);
+	} else {
+		period_ps = DIV_ROUND_UP((unsigned long)NSEC_PER_SEC * 1000,
+					 clk_get_rate(hw->clk));
+
+		/* First compute the phase index (p), the remainder (r) is the
+		 * part we'll try to acheive using the delays (d).
+		 */
+		r = do_div(p, 360 / phase_num);
+		d = DIV_ROUND_CLOSEST(r * period_ps,
+				      360 * mmc->delay_step_ps);
+		d = min(d, mmc->delay_mask >> __ffs(mmc->delay_mask));
+	}
+
+	meson_mmc_apply_phase_delay(mmc, p, d);
+	return 0;
+}
+
+static const struct clk_ops meson_mmc_clk_phase_ops = {
+	.get_phase = meson_mmc_clk_get_phase,
+	.set_phase = meson_mmc_clk_set_phase,
+};
+
 static unsigned int meson_mmc_get_timeout_msecs(struct mmc_data *data)
 {
 	unsigned int timeout = data->timeout_ns / NSEC_PER_MSEC;
@@ -271,58 +353,102 @@ static void meson_mmc_post_req(struct mmc_host *mmc, struct mmc_request *mrq,
 			     mmc_get_dma_dir(data));
 }
 
-static int meson_mmc_clk_set(struct meson_host *host, unsigned long clk_rate)
+static bool meson_mmc_timing_is_ddr(struct mmc_ios *ios)
+{
+	if (ios->timing == MMC_TIMING_MMC_DDR52 ||
+	    ios->timing == MMC_TIMING_UHS_DDR50 ||
+	    ios->timing == MMC_TIMING_MMC_HS400)
+		return true;
+
+	return false;
+}
+
+/*
+ * Gating the clock on this controller is tricky.  It seems the mmc clock
+ * is also used by the controller.  It may crash during some operation if the
+ * clock is stopped.  The safest thing to do, whenever possible, is to keep
+ * clock running at stop it at the pad using the pinmux.
+ */
+static void meson_mmc_clk_gate(struct meson_host *host)
+{
+	u32 cfg;
+
+	if (host->pins_clk_gate) {
+		pinctrl_select_state(host->pinctrl, host->pins_clk_gate);
+	} else {
+		/*
+		 * If the pinmux is not provided - default to the classic and
+		 * unsafe method
+		 */
+		cfg = readl(host->regs + SD_EMMC_CFG);
+		cfg |= CFG_STOP_CLOCK;
+		writel(cfg, host->regs + SD_EMMC_CFG);
+	}
+}
+
+static void meson_mmc_clk_ungate(struct meson_host *host)
+{
+	u32 cfg;
+
+	if (host->pins_clk_gate)
+		pinctrl_select_state(host->pinctrl, host->pins_default);
+
+	/* Make sure the clock is not stopped in the controller */
+	cfg = readl(host->regs + SD_EMMC_CFG);
+	cfg &= ~CFG_STOP_CLOCK;
+	writel(cfg, host->regs + SD_EMMC_CFG);
+}
+
+static int meson_mmc_clk_set(struct meson_host *host, struct mmc_ios *ios)
 {
 	struct mmc_host *mmc = host->mmc;
+	unsigned long rate = ios->clock;
 	int ret;
 	u32 cfg;
 
-	if (clk_rate) {
-		if (WARN_ON(clk_rate > mmc->f_max))
-			clk_rate = mmc->f_max;
-		else if (WARN_ON(clk_rate < mmc->f_min))
-			clk_rate = mmc->f_min;
-	}
+	/* DDR modes require higher module clock */
+	if (meson_mmc_timing_is_ddr(ios))
+		rate <<= 1;
 
-	if (clk_rate == host->current_clock)
+	/* Same request - bail-out */
+	if (host->req_rate == rate)
 		return 0;
 
 	/* stop clock */
-	cfg = readl(host->regs + SD_EMMC_CFG);
-	if (!(cfg & CFG_STOP_CLOCK)) {
-		cfg |= CFG_STOP_CLOCK;
-		writel(cfg, host->regs + SD_EMMC_CFG);
-	}
-
-	dev_dbg(host->dev, "change clock rate %u -> %lu\n",
-		mmc->actual_clock, clk_rate);
+	meson_mmc_clk_gate(host);
+	host->req_rate = 0;
 
-	if (!clk_rate) {
+	if (!rate) {
 		mmc->actual_clock = 0;
-		host->current_clock = 0;
 		/* return with clock being stopped */
 		return 0;
 	}
 
-	ret = clk_set_rate(host->cfg_div_clk, clk_rate);
+	/* Stop the clock during rate change to avoid glitches */
+	cfg = readl(host->regs + SD_EMMC_CFG);
+	cfg |= CFG_STOP_CLOCK;
+	writel(cfg, host->regs + SD_EMMC_CFG);
+
+	ret = clk_set_rate(host->mmc_clk, rate);
 	if (ret) {
 		dev_err(host->dev, "Unable to set cfg_div_clk to %lu. ret=%d\n",
-			clk_rate, ret);
+			rate, ret);
 		return ret;
 	}
 
-	mmc->actual_clock = clk_get_rate(host->cfg_div_clk);
-	host->current_clock = clk_rate;
+	host->req_rate = rate;
+	mmc->actual_clock = clk_get_rate(host->mmc_clk);
 
-	if (clk_rate != mmc->actual_clock)
-		dev_dbg(host->dev,
-			"divider requested rate %lu != actual rate %u\n",
-			clk_rate, mmc->actual_clock);
+	/* We should report the real output frequency of the controller */
+	if (meson_mmc_timing_is_ddr(ios))
+		mmc->actual_clock >>= 1;
+
+	dev_dbg(host->dev, "clk rate: %u Hz\n", mmc->actual_clock);
+	if (ios->clock != mmc->actual_clock)
+		dev_dbg(host->dev, "requested rate was %u\n", ios->clock);
 
 	/* (re)start clock */
-	cfg = readl(host->regs + SD_EMMC_CFG);
-	cfg &= ~CFG_STOP_CLOCK;
-	writel(cfg, host->regs + SD_EMMC_CFG);
+	meson_mmc_clk_ungate(host);
 
 	return 0;
 }
@@ -335,11 +461,21 @@ static int meson_mmc_clk_set(struct meson_host *host, unsigned long clk_rate)
 static int meson_mmc_clk_init(struct meson_host *host)
 {
 	struct clk_init_data init;
+	struct clk_mux *mux;
+	struct clk_divider *div;
+	struct meson_mmc_phase *core, *tx, *rx;
+	struct clk *clk;
 	char clk_name[32];
 	int i, ret = 0;
 	const char *mux_parent_names[MUX_CLK_NUM_PARENTS];
-	const char *clk_div_parents[1];
-	u32 clk_reg, cfg;
+	const char *clk_parent[1];
+	u32 clk_reg;
+
+	/* init SD_EMMC_CLOCK to sane defaults w/min clock rate */
+	clk_reg = 0;
+	clk_reg |= CLK_ALWAYS_ON;
+	clk_reg |= CLK_DIV_MASK;
+	writel(clk_reg, host->regs + SD_EMMC_CLOCK);
 
 	/* get the mux parents */
 	for (i = 0; i < MUX_CLK_NUM_PARENTS; i++) {
@@ -358,103 +494,238 @@ static int meson_mmc_clk_init(struct meson_host *host)
 	}
 
 	/* create the mux */
+	mux = devm_kzalloc(host->dev, sizeof(*mux), GFP_KERNEL);
+	if (!mux)
+		return -ENOMEM;
+
 	snprintf(clk_name, sizeof(clk_name), "%s#mux", dev_name(host->dev));
 	init.name = clk_name;
 	init.ops = &clk_mux_ops;
 	init.flags = 0;
 	init.parent_names = mux_parent_names;
 	init.num_parents = MUX_CLK_NUM_PARENTS;
-	host->mux.reg = host->regs + SD_EMMC_CLOCK;
-	host->mux.shift = __bf_shf(CLK_SRC_MASK);
-	host->mux.mask = CLK_SRC_MASK;
-	host->mux.flags = 0;
-	host->mux.table = NULL;
-	host->mux.hw.init = &init;
 
-	host->mux_clk = devm_clk_register(host->dev, &host->mux.hw);
-	if (WARN_ON(IS_ERR(host->mux_clk)))
-		return PTR_ERR(host->mux_clk);
+	mux->reg = host->regs + SD_EMMC_CLOCK;
+	mux->shift = __ffs(CLK_SRC_MASK);
+	mux->mask = CLK_SRC_MASK >> mux->shift;
+	mux->hw.init = &init;
+
+	clk = devm_clk_register(host->dev, &mux->hw);
+	if (WARN_ON(IS_ERR(clk)))
+		return PTR_ERR(clk);
 
 	/* create the divider */
+	div = devm_kzalloc(host->dev, sizeof(*div), GFP_KERNEL);
+	if (!div)
+		return -ENOMEM;
+
 	snprintf(clk_name, sizeof(clk_name), "%s#div", dev_name(host->dev));
 	init.name = clk_name;
 	init.ops = &clk_divider_ops;
 	init.flags = CLK_SET_RATE_PARENT;
-	clk_div_parents[0] = __clk_get_name(host->mux_clk);
-	init.parent_names = clk_div_parents;
-	init.num_parents = ARRAY_SIZE(clk_div_parents);
+	clk_parent[0] = __clk_get_name(clk);
+	init.parent_names = clk_parent;
+	init.num_parents = 1;
+
+	div->reg = host->regs + SD_EMMC_CLOCK;
+	div->shift = __ffs(CLK_DIV_MASK);
+	div->width = __builtin_popcountl(CLK_DIV_MASK);
+	div->hw.init = &init;
+	div->flags = (CLK_DIVIDER_ONE_BASED |
+		      CLK_DIVIDER_ROUND_CLOSEST);
+
+	clk = devm_clk_register(host->dev, &div->hw);
+	if (WARN_ON(IS_ERR(clk)))
+		return PTR_ERR(clk);
+
+	/* create the mmc core clock */
+	core = devm_kzalloc(host->dev, sizeof(*core), GFP_KERNEL);
+	if (!core)
+		return -ENOMEM;
+
+	snprintf(clk_name, sizeof(clk_name), "%s#core", dev_name(host->dev));
+	init.name = clk_name;
+	init.ops = &meson_mmc_clk_phase_ops;
+	init.flags = CLK_SET_RATE_PARENT;
+	clk_parent[0] = __clk_get_name(clk);
+	init.parent_names = clk_parent;
+	init.num_parents = 1;
 
-	host->cfg_div.reg = host->regs + SD_EMMC_CLOCK;
-	host->cfg_div.shift = __bf_shf(CLK_DIV_MASK);
-	host->cfg_div.width = __builtin_popcountl(CLK_DIV_MASK);
-	host->cfg_div.hw.init = &init;
-	host->cfg_div.flags = CLK_DIVIDER_ONE_BASED |
-		CLK_DIVIDER_ROUND_CLOSEST | CLK_DIVIDER_ALLOW_ZERO;
+	core->reg = host->regs + SD_EMMC_CLOCK;
+	core->phase_mask = CLK_CORE_PHASE_MASK;
+	core->hw.init = &init;
 
-	host->cfg_div_clk = devm_clk_register(host->dev, &host->cfg_div.hw);
-	if (WARN_ON(PTR_ERR_OR_ZERO(host->cfg_div_clk)))
-		return PTR_ERR(host->cfg_div_clk);
+	host->mmc_clk = devm_clk_register(host->dev, &core->hw);
+	if (WARN_ON(PTR_ERR_OR_ZERO(host->mmc_clk)))
+		return PTR_ERR(host->mmc_clk);
 
-	/* init SD_EMMC_CLOCK to sane defaults w/min clock rate */
-	clk_reg = 0;
-	clk_reg |= FIELD_PREP(CLK_CORE_PHASE_MASK, host->tp.core_phase);
-	clk_reg |= FIELD_PREP(CLK_TX_PHASE_MASK, host->tp.tx_phase);
-	clk_reg |= FIELD_PREP(CLK_RX_PHASE_MASK, host->tp.rx_phase);
-	clk_reg |= FIELD_PREP(CLK_SRC_MASK, CLK_SRC_XTAL);
-	clk_reg |= FIELD_PREP(CLK_DIV_MASK, CLK_DIV_MAX);
-	clk_reg &= ~CLK_ALWAYS_ON;
-	writel(clk_reg, host->regs + SD_EMMC_CLOCK);
+	/* create the mmc tx clock */
+	tx = devm_kzalloc(host->dev, sizeof(*tx), GFP_KERNEL);
+	if (!tx)
+		return -ENOMEM;
 
-	/* Ensure clock starts in "auto" mode, not "always on" */
-	cfg = readl(host->regs + SD_EMMC_CFG);
-	cfg &= ~CFG_CLK_ALWAYS_ON;
-	cfg |= CFG_AUTO_CLK;
-	writel(cfg, host->regs + SD_EMMC_CFG);
+	snprintf(clk_name, sizeof(clk_name), "%s#tx", dev_name(host->dev));
+	init.name = clk_name;
+	init.ops = &meson_mmc_clk_phase_ops;
+	init.flags = 0;
+	clk_parent[0] = __clk_get_name(host->mmc_clk);
+	init.parent_names = clk_parent;
+	init.num_parents = 1;
+
+	tx->reg = host->regs + SD_EMMC_CLOCK;
+	tx->phase_mask = CLK_TX_PHASE_MASK;
+	tx->delay_mask = CLK_TX_DELAY_MASK;
+	tx->delay_step_ps = CLK_DELAY_STEP_PS;
+	tx->hw.init = &init;
+
+	host->tx_clk = devm_clk_register(host->dev, &tx->hw);
+	if (WARN_ON(PTR_ERR_OR_ZERO(host->tx_clk)))
+		return PTR_ERR(host->tx_clk);
+
+	/* create the mmc rx clock */
+	rx = devm_kzalloc(host->dev, sizeof(*rx), GFP_KERNEL);
+	if (!rx)
+		return -ENOMEM;
+
+	snprintf(clk_name, sizeof(clk_name), "%s#rx", dev_name(host->dev));
+	init.name = clk_name;
+	init.ops = &meson_mmc_clk_phase_ops;
+	init.flags = 0;
+	clk_parent[0] = __clk_get_name(host->mmc_clk);
+	init.parent_names = clk_parent;
+	init.num_parents = 1;
 
-	ret = clk_prepare_enable(host->cfg_div_clk);
+	rx->reg = host->regs + SD_EMMC_CLOCK;
+	rx->phase_mask = CLK_RX_PHASE_MASK;
+	rx->delay_mask = CLK_RX_DELAY_MASK;
+	rx->delay_step_ps = CLK_DELAY_STEP_PS;
+	rx->hw.init = &init;
+
+	host->rx_clk = devm_clk_register(host->dev, &rx->hw);
+	if (WARN_ON(PTR_ERR_OR_ZERO(host->rx_clk)))
+		return PTR_ERR(host->rx_clk);
+
+	/* init SD_EMMC_CLOCK to sane defaults w/min clock rate */
+	host->mmc->f_min = clk_round_rate(host->mmc_clk, 400000);
+	ret = clk_set_rate(host->mmc_clk, host->mmc->f_min);
 	if (ret)
 		return ret;
 
-	/* Get the nearest minimum clock to 400KHz */
-	host->mmc->f_min = clk_round_rate(host->cfg_div_clk, 400000);
+	/*
+	 * Set phases : These values are mostly the datasheet recommended ones
+	 * except for the Tx phase. Datasheet recommends 180 but some cards
+	 * fail at initialisation with it. 270 works just fine, it fixes these
+	 * initialisation issues and enable eMMC DDR52 mode.
+	 */
+	clk_set_phase(host->mmc_clk, 180);
+	clk_set_phase(host->tx_clk, 270);
+	clk_set_phase(host->rx_clk, 0);
 
-	ret = meson_mmc_clk_set(host, host->mmc->f_min);
-	if (ret)
-		clk_disable_unprepare(host->cfg_div_clk);
+	return clk_prepare_enable(host->mmc_clk);
+}
 
-	return ret;
+static void meson_mmc_shift_map(unsigned long *map, unsigned long shift)
+{
+	DECLARE_BITMAP(left, CLK_PHASE_POINT_NUM);
+	DECLARE_BITMAP(right, CLK_PHASE_POINT_NUM);
+
+	/*
+	 * shift the bitmap right and reintroduce the dropped bits on the left
+	 * of the bitmap
+	 */
+	bitmap_shift_right(right, map, shift, CLK_PHASE_POINT_NUM);
+	bitmap_shift_left(left, map, CLK_PHASE_POINT_NUM - shift,
+			  CLK_PHASE_POINT_NUM);
+	bitmap_or(map, left, right, CLK_PHASE_POINT_NUM);
 }
 
-static void meson_mmc_set_tuning_params(struct mmc_host *mmc)
+static void meson_mmc_find_next_region(unsigned long *map,
+				       unsigned long *start,
+				       unsigned long *stop)
+{
+	*start = find_next_bit(map, CLK_PHASE_POINT_NUM, *start);
+	*stop = find_next_zero_bit(map, CLK_PHASE_POINT_NUM, *start);
+}
+
+static int meson_mmc_find_tuning_point(unsigned long *test)
+{
+	unsigned long shift, stop, offset = 0, start = 0, size = 0;
+
+	/* Get the all good/all bad situation out the way */
+	if (bitmap_full(test, CLK_PHASE_POINT_NUM))
+		return 0; /* All points are good so point 0 will do */
+	else if (bitmap_empty(test, CLK_PHASE_POINT_NUM))
+		return -EIO; /* No successful tuning point */
+
+	/*
+	 * Now we know there is a least one region find. Make sure it does
+	 * not wrap by the shifting the bitmap if necessary
+	 */
+	shift = find_first_zero_bit(test, CLK_PHASE_POINT_NUM);
+	if (shift != 0)
+		meson_mmc_shift_map(test, shift);
+
+	while (start < CLK_PHASE_POINT_NUM) {
+		meson_mmc_find_next_region(test, &start, &stop);
+
+		if ((stop - start) > size) {
+			offset = start;
+			size = stop - start;
+		}
+
+		start = stop;
+	}
+
+	/* Get the center point of the region */
+	offset += (size / 2);
+
+	/* Shift the result back */
+	offset = (offset + shift) % CLK_PHASE_POINT_NUM;
+
+	return offset;
+}
+
+static int meson_mmc_clk_phase_tuning(struct mmc_host *mmc, u32 opcode,
+				      struct clk *clk)
+{
+	int point, ret;
+	DECLARE_BITMAP(test, CLK_PHASE_POINT_NUM);
+
+	dev_dbg(mmc_dev(mmc), "%s phase/delay tunning...\n",
+		__clk_get_name(clk));
+	bitmap_zero(test, CLK_PHASE_POINT_NUM);
+
+	/* Explore tuning points */
+	for (point = 0; point < CLK_PHASE_POINT_NUM; point++) {
+		clk_set_phase(clk, point * CLK_PHASE_STEP);
+		ret = mmc_send_tuning(mmc, opcode, NULL);
+		if (!ret)
+			set_bit(point, test);
+	}
+
+	/* Find the optimal tuning point and apply it */
+	point = meson_mmc_find_tuning_point(test);
+	if (point < 0)
+		return point; /* tuning failed */
+
+	clk_set_phase(clk, point * CLK_PHASE_STEP);
+	dev_dbg(mmc_dev(mmc), "success with phase: %d\n",
+		clk_get_phase(clk));
+	return 0;
+}
+
+static int meson_mmc_execute_tuning(struct mmc_host *mmc, u32 opcode)
 {
 	struct meson_host *host = mmc_priv(mmc);
-	u32 regval;
 
-	/* stop clock */
-	regval = readl(host->regs + SD_EMMC_CFG);
-	regval |= CFG_STOP_CLOCK;
-	writel(regval, host->regs + SD_EMMC_CFG);
-
-	regval = readl(host->regs + SD_EMMC_CLOCK);
-	regval &= ~CLK_CORE_PHASE_MASK;
-	regval |= FIELD_PREP(CLK_CORE_PHASE_MASK, host->tp.core_phase);
-	regval &= ~CLK_TX_PHASE_MASK;
-	regval |= FIELD_PREP(CLK_TX_PHASE_MASK, host->tp.tx_phase);
-	regval &= ~CLK_RX_PHASE_MASK;
-	regval |= FIELD_PREP(CLK_RX_PHASE_MASK, host->tp.rx_phase);
-	writel(regval, host->regs + SD_EMMC_CLOCK);
-
-	/* start clock */
-	regval = readl(host->regs + SD_EMMC_CFG);
-	regval &= ~CFG_STOP_CLOCK;
-	writel(regval, host->regs + SD_EMMC_CFG);
+	return meson_mmc_clk_phase_tuning(mmc, opcode, host->rx_clk);
 }
 
 static void meson_mmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 {
 	struct meson_host *host = mmc_priv(mmc);
-	u32 bus_width;
-	u32 val, orig;
+	u32 bus_width, val;
+	int err;
 
 	/*
 	 * GPIO regulator, only controls switching between 1v8 and
@@ -482,18 +753,17 @@ static void meson_mmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 			int ret = regulator_enable(mmc->supply.vqmmc);
 
 			if (ret < 0)
-				dev_err(mmc_dev(mmc),
+				dev_err(host->dev,
 					"failed to enable vqmmc regulator\n");
 			else
 				host->vqmmc_enabled = true;
 		}
 
+		/* Reset rx phase */
+		clk_set_phase(host->rx_clk, 0);
 		break;
 	}
 
-
-	meson_mmc_clk_set(host, ios->clock);
-
 	/* Bus width */
 	switch (ios->bus_width) {
 	case MMC_BUS_WIDTH_1:
@@ -512,26 +782,23 @@ static void meson_mmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 	}
 
 	val = readl(host->regs + SD_EMMC_CFG);
-	orig = val;
-
 	val &= ~CFG_BUS_WIDTH_MASK;
 	val |= FIELD_PREP(CFG_BUS_WIDTH_MASK, bus_width);
 
 	val &= ~CFG_DDR;
-	if (ios->timing == MMC_TIMING_UHS_DDR50 ||
-	    ios->timing == MMC_TIMING_MMC_DDR52 ||
-	    ios->timing == MMC_TIMING_MMC_HS400)
+	if (meson_mmc_timing_is_ddr(ios))
 		val |= CFG_DDR;
 
 	val &= ~CFG_CHK_DS;
 	if (ios->timing == MMC_TIMING_MMC_HS400)
 		val |= CFG_CHK_DS;
 
-	if (val != orig) {
-		writel(val, host->regs + SD_EMMC_CFG);
-		dev_dbg(host->dev, "%s: SD_EMMC_CFG: 0x%08x -> 0x%08x\n",
-			__func__, orig, val);
-	}
+	err = meson_mmc_clk_set(host, ios);
+	if (err)
+		dev_err(host->dev, "Failed to set clock: %d\n,", err);
+
+	writel(val, host->regs + SD_EMMC_CFG);
+	dev_dbg(host->dev, "SD_EMMC_CFG:  0x%08x\n", val);
 }
 
 static void meson_mmc_request_done(struct mmc_host *mmc,
@@ -729,57 +996,40 @@ static irqreturn_t meson_mmc_irq(int irq, void *dev_id)
 	struct mmc_command *cmd;
 	struct mmc_data *data;
 	u32 irq_en, status, raw_status;
-	irqreturn_t ret = IRQ_HANDLED;
+	irqreturn_t ret = IRQ_NONE;
 
-	if (WARN_ON(!host))
+	if (WARN_ON(!host) || WARN_ON(!host->cmd))
 		return IRQ_NONE;
 
-	cmd = host->cmd;
-
-	if (WARN_ON(!cmd))
-		return IRQ_NONE;
+	spin_lock(&host->lock);
 
+	cmd = host->cmd;
 	data = cmd->data;
-
-	spin_lock(&host->lock);
 	irq_en = readl(host->regs + SD_EMMC_IRQ_EN);
 	raw_status = readl(host->regs + SD_EMMC_STATUS);
 	status = raw_status & irq_en;
 
-	if (!status) {
-		dev_warn(host->dev, "Spurious IRQ! status=0x%08x, irq_en=0x%08x\n",
-			 raw_status, irq_en);
-		ret = IRQ_NONE;
-		goto out;
-	}
-
-	meson_mmc_read_resp(host->mmc, cmd);
-
 	cmd->error = 0;
-	if (status & IRQ_RXD_ERR_MASK) {
-		dev_dbg(host->dev, "Unhandled IRQ: RXD error\n");
-		cmd->error = -EILSEQ;
-	}
-	if (status & IRQ_TXD_ERR) {
-		dev_dbg(host->dev, "Unhandled IRQ: TXD error\n");
-		cmd->error = -EILSEQ;
-	}
-	if (status & IRQ_DESC_ERR)
-		dev_dbg(host->dev, "Unhandled IRQ: Descriptor error\n");
-	if (status & IRQ_RESP_ERR) {
-		dev_dbg(host->dev, "Unhandled IRQ: Response error\n");
+	if (status & IRQ_CRC_ERR) {
+		dev_dbg(host->dev, "CRC Error - status 0x%08x\n", status);
 		cmd->error = -EILSEQ;
+		ret = IRQ_HANDLED;
+		goto out;
 	}
-	if (status & IRQ_RESP_TIMEOUT) {
-		dev_dbg(host->dev, "Unhandled IRQ: Response timeout\n");
+
+	if (status & IRQ_TIMEOUTS) {
+		dev_dbg(host->dev, "Timeout - status 0x%08x\n", status);
 		cmd->error = -ETIMEDOUT;
+		ret = IRQ_HANDLED;
+		goto out;
 	}
-	if (status & IRQ_DESC_TIMEOUT) {
-		dev_dbg(host->dev, "Unhandled IRQ: Descriptor timeout\n");
-		cmd->error = -ETIMEDOUT;
+
+	meson_mmc_read_resp(host->mmc, cmd);
+
+	if (status & IRQ_SDIO) {
+		dev_dbg(host->dev, "IRQ: SDIO TODO.\n");
+		ret = IRQ_HANDLED;
 	}
-	if (status & IRQ_SDIO)
-		dev_dbg(host->dev, "Unhandled IRQ: SDIO.\n");
 
 	if (status & (IRQ_END_OF_CHAIN | IRQ_RESP_STATUS)) {
 		if (data && !cmd->error)
@@ -787,26 +1037,20 @@ static irqreturn_t meson_mmc_irq(int irq, void *dev_id)
 		if (meson_mmc_bounce_buf_read(data) ||
 		    meson_mmc_get_next_command(cmd))
 			ret = IRQ_WAKE_THREAD;
-	} else {
-		dev_warn(host->dev, "Unknown IRQ! status=0x%04x: MMC CMD%u arg=0x%08x flags=0x%08x stop=%d\n",
-			 status, cmd->opcode, cmd->arg,
-			 cmd->flags, cmd->mrq->stop ? 1 : 0);
-		if (cmd->data) {
-			struct mmc_data *data = cmd->data;
-
-			dev_warn(host->dev, "\tblksz %u blocks %u flags 0x%08x (%s%s)",
-				 data->blksz, data->blocks, data->flags,
-				 data->flags & MMC_DATA_WRITE ? "write" : "",
-				 data->flags & MMC_DATA_READ ? "read" : "");
-		}
+		else
+			ret = IRQ_HANDLED;
 	}
 
 out:
-	/* ack all (enabled) interrupts */
-	writel(status, host->regs + SD_EMMC_STATUS);
+	/* ack all enabled interrupts */
+	writel(irq_en, host->regs + SD_EMMC_STATUS);
 
 	if (ret == IRQ_HANDLED)
 		meson_mmc_request_done(host->mmc, cmd->mrq);
+	else if (ret == IRQ_NONE)
+		dev_warn(host->dev,
+			 "Unexpected IRQ! status=0x%08x, irq_en=0x%08x\n",
+			 raw_status, irq_en);
 
 	spin_unlock(&host->lock);
 	return ret;
@@ -839,29 +1083,6 @@ static irqreturn_t meson_mmc_irq_thread(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-static int meson_mmc_execute_tuning(struct mmc_host *mmc, u32 opcode)
-{
-	struct meson_host *host = mmc_priv(mmc);
-	struct meson_tuning_params tp_old = host->tp;
-	int ret = -EINVAL, i, cmd_error;
-
-	dev_info(mmc_dev(mmc), "(re)tuning...\n");
-
-	for (i = CLK_PHASE_0; i <= CLK_PHASE_270; i++) {
-		host->tp.rx_phase = i;
-		/* exclude the active parameter set if retuning */
-		if (!memcmp(&tp_old, &host->tp, sizeof(tp_old)) &&
-		    mmc->doing_retune)
-			continue;
-		meson_mmc_set_tuning_params(mmc);
-		ret = mmc_send_tuning(mmc, opcode, &cmd_error);
-		if (!ret)
-			break;
-	}
-
-	return ret;
-}
-
 /*
  * NOTE: we only need this until the GPIO/pinctrl driver can handle
  * interrupts.  For now, the MMC core will use this for polling.
@@ -888,6 +1109,38 @@ static void meson_mmc_cfg_init(struct meson_host *host)
 	writel(cfg, host->regs + SD_EMMC_CFG);
 }
 
+static int meson_mmc_card_busy(struct mmc_host *mmc)
+{
+	struct meson_host *host = mmc_priv(mmc);
+	u32 regval;
+
+	regval = readl(host->regs + SD_EMMC_STATUS);
+
+	/* We are only interrested in lines 0 to 3, so mask the other ones */
+	return !(FIELD_GET(STATUS_DATI, regval) & 0xf);
+}
+
+static int meson_mmc_voltage_switch(struct mmc_host *mmc, struct mmc_ios *ios)
+{
+	/* vqmmc regulator is available */
+	if (!IS_ERR(mmc->supply.vqmmc)) {
+		/*
+		 * The usual amlogic setup uses a GPIO to switch from one
+		 * regulator to the other. While the voltage ramp up is
+		 * pretty fast, care must be taken when switching from 3.3v
+		 * to 1.8v. Please make sure the regulator framework is aware
+		 * of your own regulator constraints
+		 */
+		return mmc_regulator_set_vqmmc(mmc, ios);
+	}
+
+	/* no vqmmc regulator, assume fixed regulator at 3/3.3V */
+	if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_330)
+		return 0;
+
+	return -EINVAL;
+}
+
 static const struct mmc_host_ops meson_mmc_ops = {
 	.request	= meson_mmc_request,
 	.set_ios	= meson_mmc_set_ios,
@@ -895,6 +1148,8 @@ static const struct mmc_host_ops meson_mmc_ops = {
 	.pre_req	= meson_mmc_pre_req,
 	.post_req	= meson_mmc_post_req,
 	.execute_tuning = meson_mmc_execute_tuning,
+	.card_busy	= meson_mmc_card_busy,
+	.start_signal_voltage_switch = meson_mmc_voltage_switch,
 };
 
 static int meson_mmc_probe(struct platform_device *pdev)
@@ -941,6 +1196,27 @@ static int meson_mmc_probe(struct platform_device *pdev)
 		goto free_host;
 	}
 
+	host->pinctrl = devm_pinctrl_get(&pdev->dev);
+	if (IS_ERR(host->pinctrl)) {
+		ret = PTR_ERR(host->pinctrl);
+		goto free_host;
+	}
+
+	host->pins_default = pinctrl_lookup_state(host->pinctrl,
+						  PINCTRL_STATE_DEFAULT);
+	if (IS_ERR(host->pins_default)) {
+		ret = PTR_ERR(host->pins_default);
+		goto free_host;
+	}
+
+	host->pins_clk_gate = pinctrl_lookup_state(host->pinctrl,
+						   "clk-gate");
+	if (IS_ERR(host->pins_clk_gate)) {
+		dev_warn(&pdev->dev,
+			 "can't get clk-gate pinctrl, using clk_stop bit\n");
+		host->pins_clk_gate = NULL;
+	}
+
 	host->core_clk = devm_clk_get(&pdev->dev, "core");
 	if (IS_ERR(host->core_clk)) {
 		ret = PTR_ERR(host->core_clk);
@@ -951,30 +1227,28 @@ static int meson_mmc_probe(struct platform_device *pdev)
 	if (ret)
 		goto free_host;
 
-	host->tp.core_phase = CLK_PHASE_180;
-	host->tp.tx_phase = CLK_PHASE_0;
-	host->tp.rx_phase = CLK_PHASE_0;
-
 	ret = meson_mmc_clk_init(host);
 	if (ret)
 		goto err_core_clk;
 
+	/* set config to sane default */
+	meson_mmc_cfg_init(host);
+
 	/* Stop execution */
 	writel(0, host->regs + SD_EMMC_START);
 
-	/* clear, ack, enable all interrupts */
+	/* clear, ack and enable interrupts */
 	writel(0, host->regs + SD_EMMC_IRQ_EN);
-	writel(IRQ_EN_MASK, host->regs + SD_EMMC_STATUS);
-	writel(IRQ_EN_MASK, host->regs + SD_EMMC_IRQ_EN);
-
-	/* set config to sane default */
-	meson_mmc_cfg_init(host);
+	writel(IRQ_CRC_ERR | IRQ_TIMEOUTS | IRQ_END_OF_CHAIN,
+	       host->regs + SD_EMMC_STATUS);
+	writel(IRQ_CRC_ERR | IRQ_TIMEOUTS | IRQ_END_OF_CHAIN,
+	       host->regs + SD_EMMC_IRQ_EN);
 
 	ret = devm_request_threaded_irq(&pdev->dev, irq, meson_mmc_irq,
 					meson_mmc_irq_thread, IRQF_SHARED,
 					NULL, host);
 	if (ret)
-		goto err_div_clk;
+		goto err_init_clk;
 
 	mmc->caps |= MMC_CAP_CMD23;
 	mmc->max_blk_count = CMD_CFG_LENGTH_MASK;
@@ -990,7 +1264,7 @@ static int meson_mmc_probe(struct platform_device *pdev)
 	if (host->bounce_buf == NULL) {
 		dev_err(host->dev, "Unable to map allocate DMA bounce buffer.\n");
 		ret = -ENOMEM;
-		goto err_div_clk;
+		goto err_init_clk;
 	}
 
 	host->descs = dma_alloc_coherent(host->dev, SD_EMMC_DESC_BUF_LEN,
@@ -1009,8 +1283,8 @@ static int meson_mmc_probe(struct platform_device *pdev)
 err_bounce_buf:
 	dma_free_coherent(host->dev, host->bounce_buf_size,
 			  host->bounce_buf, host->bounce_dma_addr);
-err_div_clk:
-	clk_disable_unprepare(host->cfg_div_clk);
+err_init_clk:
+	clk_disable_unprepare(host->mmc_clk);
 err_core_clk:
 	clk_disable_unprepare(host->core_clk);
 free_host:
@@ -1032,7 +1306,7 @@ static int meson_mmc_remove(struct platform_device *pdev)
 	dma_free_coherent(host->dev, host->bounce_buf_size,
 			  host->bounce_buf, host->bounce_dma_addr);
 
-	clk_disable_unprepare(host->cfg_div_clk);
+	clk_disable_unprepare(host->mmc_clk);
 	clk_disable_unprepare(host->core_clk);
 
 	mmc_free_host(host->mmc);
diff --git a/drivers/mmc/host/mmci.c b/drivers/mmc/host/mmci.c
index d1ca2f489054..f1f54a818489 100644
--- a/drivers/mmc/host/mmci.c
+++ b/drivers/mmc/host/mmci.c
@@ -1904,7 +1904,7 @@ static const struct dev_pm_ops mmci_dev_pm_ops = {
 	SET_RUNTIME_PM_OPS(mmci_runtime_suspend, mmci_runtime_resume, NULL)
 };
 
-static struct amba_id mmci_ids[] = {
+static const struct amba_id mmci_ids[] = {
 	{
 		.id	= 0x00041180,
 		.mask	= 0xff0fffff,
diff --git a/drivers/mmc/host/moxart-mmc.c b/drivers/mmc/host/moxart-mmc.c
index d4dc55ac7dea..a0670e9cd012 100644
--- a/drivers/mmc/host/moxart-mmc.c
+++ b/drivers/mmc/host/moxart-mmc.c
@@ -546,7 +546,7 @@ static int moxart_get_ro(struct mmc_host *mmc)
 	return !!(readl(host->base + REG_STATUS) & WRITE_PROT);
 }
 
-static struct mmc_host_ops moxart_ops = {
+static const struct mmc_host_ops moxart_ops = {
 	.request = moxart_request,
 	.set_ios = moxart_set_ios,
 	.get_ro = moxart_get_ro,
diff --git a/drivers/mmc/host/mtk-sd.c b/drivers/mmc/host/mtk-sd.c
index 5a672a5218ad..267f7ab08420 100644
--- a/drivers/mmc/host/mtk-sd.c
+++ b/drivers/mmc/host/mtk-sd.c
@@ -1579,12 +1579,13 @@ static void msdc_hw_reset(struct mmc_host *mmc)
 	sdr_clr_bits(host->base + EMMC_IOCON, 1);
 }
 
-static struct mmc_host_ops mt_msdc_ops = {
+static const struct mmc_host_ops mt_msdc_ops = {
 	.post_req = msdc_post_req,
 	.pre_req = msdc_pre_req,
 	.request = msdc_ops_request,
 	.set_ios = msdc_ops_set_ios,
 	.get_ro = mmc_gpio_get_ro,
+	.get_cd = mmc_gpio_get_cd,
 	.start_signal_voltage_switch = msdc_ops_switch_volt,
 	.card_busy = msdc_card_busy,
 	.execute_tuning = msdc_execute_tuning,
diff --git a/drivers/mmc/host/mxcmmc.c b/drivers/mmc/host/mxcmmc.c
index fb3ca8296273..1d5418e4efae 100644
--- a/drivers/mmc/host/mxcmmc.c
+++ b/drivers/mmc/host/mxcmmc.c
@@ -681,6 +681,9 @@ static void mxcmci_data_done(struct mxcmci_host *host, unsigned int stat)
 
 	spin_unlock_irqrestore(&host->lock, flags);
 
+	if (data_error)
+		return;
+
 	mxcmci_read_response(host, stat);
 	host->cmd = NULL;
 
@@ -1014,8 +1017,10 @@ static int mxcmci_probe(struct platform_device *pdev)
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0)
-		return -EINVAL;
+	if (irq < 0) {
+		dev_err(&pdev->dev, "failed to get IRQ: %d\n", irq);
+		return irq;
+	}
 
 	mmc = mmc_alloc_host(sizeof(*host), &pdev->dev);
 	if (!mmc)
@@ -1098,8 +1103,13 @@ static int mxcmci_probe(struct platform_device *pdev)
 		goto out_free;
 	}
 
-	clk_prepare_enable(host->clk_per);
-	clk_prepare_enable(host->clk_ipg);
+	ret = clk_prepare_enable(host->clk_per);
+	if (ret)
+		goto out_free;
+
+	ret = clk_prepare_enable(host->clk_ipg);
+	if (ret)
+		goto out_clk_per_put;
 
 	mxcmci_softreset(host);
 
@@ -1168,8 +1178,9 @@ out_free_dma:
 		dma_release_channel(host->dma);
 
 out_clk_put:
-	clk_disable_unprepare(host->clk_per);
 	clk_disable_unprepare(host->clk_ipg);
+out_clk_per_put:
+	clk_disable_unprepare(host->clk_per);
 
 out_free:
 	mmc_free_host(mmc);
@@ -1212,10 +1223,17 @@ static int __maybe_unused mxcmci_resume(struct device *dev)
 {
 	struct mmc_host *mmc = dev_get_drvdata(dev);
 	struct mxcmci_host *host = mmc_priv(mmc);
+	int ret;
 
-	clk_prepare_enable(host->clk_per);
-	clk_prepare_enable(host->clk_ipg);
-	return 0;
+	ret = clk_prepare_enable(host->clk_per);
+	if (ret)
+		return ret;
+
+	ret = clk_prepare_enable(host->clk_ipg);
+	if (ret)
+		clk_disable_unprepare(host->clk_per);
+
+	return ret;
 }
 
 static SIMPLE_DEV_PM_OPS(mxcmci_pm_ops, mxcmci_suspend, mxcmci_resume);
diff --git a/drivers/mmc/host/of_mmc_spi.c b/drivers/mmc/host/of_mmc_spi.c
index 85bbebfde02e..c9eed8436b6b 100644
--- a/drivers/mmc/host/of_mmc_spi.c
+++ b/drivers/mmc/host/of_mmc_spi.c
@@ -71,7 +71,7 @@ struct mmc_spi_platform_data *mmc_spi_get_pdata(struct spi_device *spi)
 	struct device *dev = &spi->dev;
 	struct device_node *np = dev->of_node;
 	struct of_mmc_spi *oms;
-	const u32 *voltage_ranges;
+	const __be32 *voltage_ranges;
 	int num_ranges;
 	int i;
 
diff --git a/drivers/mmc/host/omap_hsmmc.c b/drivers/mmc/host/omap_hsmmc.c
index 2ab4788d021f..3b5e6d11069b 100644
--- a/drivers/mmc/host/omap_hsmmc.c
+++ b/drivers/mmc/host/omap_hsmmc.c
@@ -2076,9 +2076,9 @@ static int omap_hsmmc_probe(struct platform_device *pdev)
 		host->dbclk = NULL;
 	}
 
-	/* Since we do only SG emulation, we can have as many segs
-	 * as we want. */
-	mmc->max_segs = 1024;
+	/* Set this to a value that allows allocating an entire descriptor
+	 * list within a page (zero order allocation). */
+	mmc->max_segs = 64;
 
 	mmc->max_blk_size = 512;       /* Block Length at max can be 1024 */
 	mmc->max_blk_count = 0xFFFF;    /* No. of Blocks is 16 bits */
@@ -2322,7 +2322,7 @@ static int omap_hsmmc_runtime_resume(struct device *dev)
 	return 0;
 }
 
-static struct dev_pm_ops omap_hsmmc_dev_pm_ops = {
+static const struct dev_pm_ops omap_hsmmc_dev_pm_ops = {
 	SET_SYSTEM_SLEEP_PM_OPS(omap_hsmmc_suspend, omap_hsmmc_resume)
 	.runtime_suspend = omap_hsmmc_runtime_suspend,
 	.runtime_resume = omap_hsmmc_runtime_resume,
diff --git a/drivers/mmc/host/renesas_sdhi.h b/drivers/mmc/host/renesas_sdhi.h
index ca83acc113b8..b9dfea5d8193 100644
--- a/drivers/mmc/host/renesas_sdhi.h
+++ b/drivers/mmc/host/renesas_sdhi.h
@@ -31,6 +31,8 @@ struct renesas_sdhi_of_data {
 	int scc_offset;
 	struct renesas_sdhi_scc *taps;
 	int taps_num;
+	unsigned int max_blk_count;
+	unsigned short max_segs;
 };
 
 int renesas_sdhi_probe(struct platform_device *pdev,
diff --git a/drivers/mmc/host/renesas_sdhi_core.c b/drivers/mmc/host/renesas_sdhi_core.c
index a4fb07d0ea91..fcf7235d5742 100644
--- a/drivers/mmc/host/renesas_sdhi_core.c
+++ b/drivers/mmc/host/renesas_sdhi_core.c
@@ -40,6 +40,7 @@
 #define EXT_ACC           0xe4
 
 #define SDHI_VER_GEN2_SDR50	0x490c
+#define SDHI_VER_RZ_A1		0x820b
 /* very old datasheets said 0x490c for SDR104, too. They are wrong! */
 #define SDHI_VER_GEN2_SDR104	0xcb0d
 #define SDHI_VER_GEN3_SD	0xcc10
@@ -398,12 +399,14 @@ static void renesas_sdhi_hw_reset(struct tmio_mmc_host *host)
 		       sd_scc_read32(host, priv, SH_MOBILE_SDHI_SCC_RVSCNTL));
 }
 
-static int renesas_sdhi_wait_idle(struct tmio_mmc_host *host)
+static int renesas_sdhi_wait_idle(struct tmio_mmc_host *host, u32 bit)
 {
 	int timeout = 1000;
+	/* CBSY is set when busy, SCLKDIVEN is cleared when busy */
+	u32 wait_state = (bit == TMIO_STAT_CMD_BUSY ? TMIO_STAT_CMD_BUSY : 0);
 
-	while (--timeout && !(sd_ctrl_read16_and_16_as_32(host, CTL_STATUS)
-			      & TMIO_STAT_SCLKDIVEN))
+	while (--timeout && (sd_ctrl_read16_and_16_as_32(host, CTL_STATUS)
+			      & bit) == wait_state)
 		udelay(1);
 
 	if (!timeout) {
@@ -416,17 +419,22 @@ static int renesas_sdhi_wait_idle(struct tmio_mmc_host *host)
 
 static int renesas_sdhi_write16_hook(struct tmio_mmc_host *host, int addr)
 {
+	u32 bit = TMIO_STAT_SCLKDIVEN;
+
 	switch (addr) {
 	case CTL_SD_CMD:
 	case CTL_STOP_INTERNAL_ACTION:
 	case CTL_XFER_BLK_COUNT:
-	case CTL_SD_CARD_CLK_CTL:
 	case CTL_SD_XFER_LEN:
 	case CTL_SD_MEM_CARD_OPT:
 	case CTL_TRANSACTION_CTL:
 	case CTL_DMA_ENABLE:
 	case EXT_ACC:
-		return renesas_sdhi_wait_idle(host);
+		if (host->pdata->flags & TMIO_MMC_HAVE_CBSY)
+			bit = TMIO_STAT_CMD_BUSY;
+		/* fallthrough */
+	case CTL_SD_CARD_CLK_CTL:
+		return renesas_sdhi_wait_idle(host, bit);
 	}
 
 	return 0;
@@ -452,10 +460,11 @@ static int renesas_sdhi_multi_io_quirk(struct mmc_card *card,
 
 static void renesas_sdhi_enable_dma(struct tmio_mmc_host *host, bool enable)
 {
-	sd_ctrl_write16(host, CTL_DMA_ENABLE, enable ? 2 : 0);
+	/* Iff regs are 8 byte apart, sdbuf is 64 bit. Otherwise always 32. */
+	int width = (host->bus_shift == 2) ? 64 : 32;
 
-	/* enable 32bit access if DMA mode if possibile */
-	renesas_sdhi_sdbuf_width(host, enable ? 32 : 16);
+	sd_ctrl_write16(host, CTL_DMA_ENABLE, enable ? DMA_ENABLE_DMASDRW : 0);
+	renesas_sdhi_sdbuf_width(host, enable ? width : 16);
 }
 
 int renesas_sdhi_probe(struct platform_device *pdev,
@@ -526,6 +535,8 @@ int renesas_sdhi_probe(struct platform_device *pdev,
 		mmc_data->capabilities |= of_data->capabilities;
 		mmc_data->capabilities2 |= of_data->capabilities2;
 		mmc_data->dma_rx_offset = of_data->dma_rx_offset;
+		mmc_data->max_blk_count = of_data->max_blk_count;
+		mmc_data->max_segs = of_data->max_segs;
 		dma_priv->dma_buswidth = of_data->dma_buswidth;
 		host->bus_shift = of_data->bus_shift;
 	}
@@ -579,6 +590,10 @@ int renesas_sdhi_probe(struct platform_device *pdev,
 	if (ret < 0)
 		goto efree;
 
+	/* One Gen2 SDHI incarnation does NOT have a CBSY bit */
+	if (sd_ctrl_read16(host, CTL_VERSION) == SDHI_VER_GEN2_SDR50)
+		mmc_data->flags &= ~TMIO_MMC_HAVE_CBSY;
+
 	/* Enable tuning iff we have an SCC and a supported mode */
 	if (of_data && of_data->scc_offset &&
 	    (host->mmc->caps & MMC_CAP_UHS_SDR104 ||
diff --git a/drivers/mmc/host/renesas_sdhi_internal_dmac.c b/drivers/mmc/host/renesas_sdhi_internal_dmac.c
new file mode 100644
index 000000000000..f905f2361d12
--- /dev/null
+++ b/drivers/mmc/host/renesas_sdhi_internal_dmac.c
@@ -0,0 +1,287 @@
+/*
+ * DMA support for Internal DMAC with SDHI SD/SDIO controller
+ *
+ * Copyright (C) 2016-17 Renesas Electronics Corporation
+ * Copyright (C) 2016-17 Horms Solutions, Simon Horman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/io-64-nonatomic-hi-lo.h>
+#include <linux/mfd/tmio.h>
+#include <linux/mmc/host.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/pagemap.h>
+#include <linux/scatterlist.h>
+#include <linux/sys_soc.h>
+
+#include "renesas_sdhi.h"
+#include "tmio_mmc.h"
+
+#define DM_CM_DTRAN_MODE	0x820
+#define DM_CM_DTRAN_CTRL	0x828
+#define DM_CM_RST		0x830
+#define DM_CM_INFO1		0x840
+#define DM_CM_INFO1_MASK	0x848
+#define DM_CM_INFO2		0x850
+#define DM_CM_INFO2_MASK	0x858
+#define DM_DTRAN_ADDR		0x880
+
+/* DM_CM_DTRAN_MODE */
+#define DTRAN_MODE_CH_NUM_CH0	0	/* "downstream" = for write commands */
+#define DTRAN_MODE_CH_NUM_CH1	BIT(16)	/* "uptream" = for read commands */
+#define DTRAN_MODE_BUS_WID_TH	(BIT(5) | BIT(4))
+#define DTRAN_MODE_ADDR_MODE	BIT(0)	/* 1 = Increment address */
+
+/* DM_CM_DTRAN_CTRL */
+#define DTRAN_CTRL_DM_START	BIT(0)
+
+/* DM_CM_RST */
+#define RST_DTRANRST1		BIT(9)
+#define RST_DTRANRST0		BIT(8)
+#define RST_RESERVED_BITS	GENMASK_ULL(32, 0)
+
+/* DM_CM_INFO1 and DM_CM_INFO1_MASK */
+#define INFO1_CLEAR		0
+#define INFO1_DTRANEND1		BIT(17)
+#define INFO1_DTRANEND0		BIT(16)
+
+/* DM_CM_INFO2 and DM_CM_INFO2_MASK */
+#define INFO2_DTRANERR1		BIT(17)
+#define INFO2_DTRANERR0		BIT(16)
+
+/*
+ * Specification of this driver:
+ * - host->chan_{rx,tx} will be used as a flag of enabling/disabling the dma
+ * - Since this SDHI DMAC register set has 16 but 32-bit width, we
+ *   need a custom accessor.
+ */
+
+/* Definitions for sampling clocks */
+static struct renesas_sdhi_scc rcar_gen3_scc_taps[] = {
+	{
+		.clk_rate = 0,
+		.tap = 0x00000300,
+	},
+};
+
+static const struct renesas_sdhi_of_data of_rcar_gen3_compatible = {
+	.tmio_flags	= TMIO_MMC_HAS_IDLE_WAIT | TMIO_MMC_WRPROTECT_DISABLE |
+			  TMIO_MMC_CLK_ACTUAL | TMIO_MMC_HAVE_CBSY |
+			  TMIO_MMC_MIN_RCAR2,
+	.capabilities	= MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ |
+			  MMC_CAP_CMD23,
+	.bus_shift	= 2,
+	.scc_offset	= 0x1000,
+	.taps		= rcar_gen3_scc_taps,
+	.taps_num	= ARRAY_SIZE(rcar_gen3_scc_taps),
+	/* Gen3 SDHI DMAC can handle 0xffffffff blk count, but seg = 1 */
+	.max_blk_count	= 0xffffffff,
+	.max_segs	= 1,
+};
+
+static const struct of_device_id renesas_sdhi_internal_dmac_of_match[] = {
+	{ .compatible = "renesas,sdhi-r8a7795", .data = &of_rcar_gen3_compatible, },
+	{ .compatible = "renesas,sdhi-r8a7796", .data = &of_rcar_gen3_compatible, },
+	{},
+};
+MODULE_DEVICE_TABLE(of, renesas_sdhi_internal_dmac_of_match);
+
+static void
+renesas_sdhi_internal_dmac_dm_write(struct tmio_mmc_host *host,
+				    int addr, u64 val)
+{
+	writeq(val, host->ctl + addr);
+}
+
+static void
+renesas_sdhi_internal_dmac_enable_dma(struct tmio_mmc_host *host, bool enable)
+{
+	if (!host->chan_tx || !host->chan_rx)
+		return;
+
+	if (!enable)
+		renesas_sdhi_internal_dmac_dm_write(host, DM_CM_INFO1,
+						    INFO1_CLEAR);
+
+	if (host->dma->enable)
+		host->dma->enable(host, enable);
+}
+
+static void
+renesas_sdhi_internal_dmac_abort_dma(struct tmio_mmc_host *host) {
+	u64 val = RST_DTRANRST1 | RST_DTRANRST0;
+
+	renesas_sdhi_internal_dmac_enable_dma(host, false);
+
+	renesas_sdhi_internal_dmac_dm_write(host, DM_CM_RST,
+					    RST_RESERVED_BITS & ~val);
+	renesas_sdhi_internal_dmac_dm_write(host, DM_CM_RST,
+					    RST_RESERVED_BITS | val);
+
+	renesas_sdhi_internal_dmac_enable_dma(host, true);
+}
+
+static void
+renesas_sdhi_internal_dmac_dataend_dma(struct tmio_mmc_host *host) {
+	tasklet_schedule(&host->dma_complete);
+}
+
+static void
+renesas_sdhi_internal_dmac_start_dma(struct tmio_mmc_host *host,
+				     struct mmc_data *data)
+{
+	struct scatterlist *sg = host->sg_ptr;
+	u32 dtran_mode = DTRAN_MODE_BUS_WID_TH | DTRAN_MODE_ADDR_MODE;
+	enum dma_data_direction dir;
+	int ret;
+	u32 irq_mask;
+
+	/* This DMAC cannot handle if sg_len is not 1 */
+	WARN_ON(host->sg_len > 1);
+
+	/* This DMAC cannot handle if buffer is not 8-bytes alignment */
+	if (!IS_ALIGNED(sg->offset, 8)) {
+		host->force_pio = true;
+		renesas_sdhi_internal_dmac_enable_dma(host, false);
+		return;
+	}
+
+	if (data->flags & MMC_DATA_READ) {
+		dtran_mode |= DTRAN_MODE_CH_NUM_CH1;
+		dir = DMA_FROM_DEVICE;
+		irq_mask = TMIO_STAT_RXRDY;
+	} else {
+		dtran_mode |= DTRAN_MODE_CH_NUM_CH0;
+		dir = DMA_TO_DEVICE;
+		irq_mask = TMIO_STAT_TXRQ;
+	}
+
+	ret = dma_map_sg(&host->pdev->dev, sg, host->sg_len, dir);
+	if (ret < 0)
+		return;
+
+	renesas_sdhi_internal_dmac_enable_dma(host, true);
+
+	/* disable PIO irqs to avoid "PIO IRQ in DMA mode!" */
+	tmio_mmc_disable_mmc_irqs(host, irq_mask);
+
+	/* set dma parameters */
+	renesas_sdhi_internal_dmac_dm_write(host, DM_CM_DTRAN_MODE,
+					    dtran_mode);
+	renesas_sdhi_internal_dmac_dm_write(host, DM_DTRAN_ADDR,
+					    sg->dma_address);
+}
+
+static void renesas_sdhi_internal_dmac_issue_tasklet_fn(unsigned long arg)
+{
+	struct tmio_mmc_host *host = (struct tmio_mmc_host *)arg;
+
+	tmio_mmc_enable_mmc_irqs(host, TMIO_STAT_DATAEND);
+
+	/* start the DMAC */
+	renesas_sdhi_internal_dmac_dm_write(host, DM_CM_DTRAN_CTRL,
+					    DTRAN_CTRL_DM_START);
+}
+
+static void renesas_sdhi_internal_dmac_complete_tasklet_fn(unsigned long arg)
+{
+	struct tmio_mmc_host *host = (struct tmio_mmc_host *)arg;
+	enum dma_data_direction dir;
+
+	spin_lock_irq(&host->lock);
+
+	if (!host->data)
+		goto out;
+
+	if (host->data->flags & MMC_DATA_READ)
+		dir = DMA_FROM_DEVICE;
+	else
+		dir = DMA_TO_DEVICE;
+
+	renesas_sdhi_internal_dmac_enable_dma(host, false);
+	dma_unmap_sg(&host->pdev->dev, host->sg_ptr, host->sg_len, dir);
+
+	tmio_mmc_do_data_irq(host);
+out:
+	spin_unlock_irq(&host->lock);
+}
+
+static void
+renesas_sdhi_internal_dmac_request_dma(struct tmio_mmc_host *host,
+				       struct tmio_mmc_data *pdata)
+{
+	/* Each value is set to non-zero to assume "enabling" each DMA */
+	host->chan_rx = host->chan_tx = (void *)0xdeadbeaf;
+
+	tasklet_init(&host->dma_complete,
+		     renesas_sdhi_internal_dmac_complete_tasklet_fn,
+		     (unsigned long)host);
+	tasklet_init(&host->dma_issue,
+		     renesas_sdhi_internal_dmac_issue_tasklet_fn,
+		     (unsigned long)host);
+}
+
+static void
+renesas_sdhi_internal_dmac_release_dma(struct tmio_mmc_host *host)
+{
+	/* Each value is set to zero to assume "disabling" each DMA */
+	host->chan_rx = host->chan_tx = NULL;
+}
+
+static const struct tmio_mmc_dma_ops renesas_sdhi_internal_dmac_dma_ops = {
+	.start = renesas_sdhi_internal_dmac_start_dma,
+	.enable = renesas_sdhi_internal_dmac_enable_dma,
+	.request = renesas_sdhi_internal_dmac_request_dma,
+	.release = renesas_sdhi_internal_dmac_release_dma,
+	.abort = renesas_sdhi_internal_dmac_abort_dma,
+	.dataend = renesas_sdhi_internal_dmac_dataend_dma,
+};
+
+/*
+ * Whitelist of specific R-Car Gen3 SoC ES versions to use this DMAC
+ * implementation as others may use a different implementation.
+ */
+static const struct soc_device_attribute gen3_soc_whitelist[] = {
+        { .soc_id = "r8a7795", .revision = "ES1.*" },
+        { .soc_id = "r8a7795", .revision = "ES2.0" },
+        { .soc_id = "r8a7796", .revision = "ES1.0" },
+        { /* sentinel */ }
+};
+
+static int renesas_sdhi_internal_dmac_probe(struct platform_device *pdev)
+{
+	if (!soc_device_match(gen3_soc_whitelist))
+		return -ENODEV;
+
+	return renesas_sdhi_probe(pdev, &renesas_sdhi_internal_dmac_dma_ops);
+}
+
+static const struct dev_pm_ops renesas_sdhi_internal_dmac_dev_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
+	SET_RUNTIME_PM_OPS(tmio_mmc_host_runtime_suspend,
+			   tmio_mmc_host_runtime_resume,
+			   NULL)
+};
+
+static struct platform_driver renesas_internal_dmac_sdhi_driver = {
+	.driver		= {
+		.name	= "renesas_sdhi_internal_dmac",
+		.pm	= &renesas_sdhi_internal_dmac_dev_pm_ops,
+		.of_match_table = renesas_sdhi_internal_dmac_of_match,
+	},
+	.probe		= renesas_sdhi_internal_dmac_probe,
+	.remove		= renesas_sdhi_remove,
+};
+
+module_platform_driver(renesas_internal_dmac_sdhi_driver);
+
+MODULE_DESCRIPTION("Renesas SDHI driver for internal DMAC");
+MODULE_AUTHOR("Yoshihiro Shimoda");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/mmc/host/renesas_sdhi_sys_dmac.c b/drivers/mmc/host/renesas_sdhi_sys_dmac.c
index 642a0dcc8c5c..df4465439e13 100644
--- a/drivers/mmc/host/renesas_sdhi_sys_dmac.c
+++ b/drivers/mmc/host/renesas_sdhi_sys_dmac.c
@@ -1,5 +1,5 @@
 /*
- * DMA function for TMIO MMC implementations
+ * DMA support use of SYS DMAC with SDHI SD/SDIO controller
  *
  * Copyright (C) 2016-17 Renesas Electronics Corporation
  * Copyright (C) 2016-17 Sang Engineering, Wolfram Sang
@@ -18,8 +18,10 @@
 #include <linux/mmc/host.h>
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
+#include <linux/of_device.h>
 #include <linux/pagemap.h>
 #include <linux/scatterlist.h>
+#include <linux/sys_soc.h>
 
 #include "renesas_sdhi.h"
 #include "tmio_mmc.h"
@@ -31,7 +33,8 @@ static const struct renesas_sdhi_of_data of_default_cfg = {
 };
 
 static const struct renesas_sdhi_of_data of_rz_compatible = {
-	.tmio_flags	= TMIO_MMC_HAS_IDLE_WAIT | TMIO_MMC_32BIT_DATA_PORT,
+	.tmio_flags	= TMIO_MMC_HAS_IDLE_WAIT | TMIO_MMC_32BIT_DATA_PORT |
+			  TMIO_MMC_HAVE_CBSY,
 	.tmio_ocr_mask	= MMC_VDD_32_33,
 	.capabilities	= MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ,
 };
@@ -56,7 +59,8 @@ static struct renesas_sdhi_scc rcar_gen2_scc_taps[] = {
 
 static const struct renesas_sdhi_of_data of_rcar_gen2_compatible = {
 	.tmio_flags	= TMIO_MMC_HAS_IDLE_WAIT | TMIO_MMC_WRPROTECT_DISABLE |
-			  TMIO_MMC_CLK_ACTUAL | TMIO_MMC_MIN_RCAR2,
+			  TMIO_MMC_CLK_ACTUAL | TMIO_MMC_HAVE_CBSY |
+			  TMIO_MMC_MIN_RCAR2,
 	.capabilities	= MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ |
 			  MMC_CAP_CMD23,
 	.dma_buswidth	= DMA_SLAVE_BUSWIDTH_4_BYTES,
@@ -76,7 +80,8 @@ static struct renesas_sdhi_scc rcar_gen3_scc_taps[] = {
 
 static const struct renesas_sdhi_of_data of_rcar_gen3_compatible = {
 	.tmio_flags	= TMIO_MMC_HAS_IDLE_WAIT | TMIO_MMC_WRPROTECT_DISABLE |
-			  TMIO_MMC_CLK_ACTUAL | TMIO_MMC_MIN_RCAR2,
+			  TMIO_MMC_CLK_ACTUAL | TMIO_MMC_HAVE_CBSY |
+			  TMIO_MMC_MIN_RCAR2,
 	.capabilities	= MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ |
 			  MMC_CAP_CMD23,
 	.bus_shift	= 2,
@@ -93,6 +98,8 @@ static const struct of_device_id renesas_sdhi_sys_dmac_of_match[] = {
 	{ .compatible = "renesas,sdhi-r7s72100", .data = &of_rz_compatible, },
 	{ .compatible = "renesas,sdhi-r8a7778", .data = &of_rcar_gen1_compatible, },
 	{ .compatible = "renesas,sdhi-r8a7779", .data = &of_rcar_gen1_compatible, },
+	{ .compatible = "renesas,sdhi-r8a7743", .data = &of_rcar_gen2_compatible, },
+	{ .compatible = "renesas,sdhi-r8a7745", .data = &of_rcar_gen2_compatible, },
 	{ .compatible = "renesas,sdhi-r8a7790", .data = &of_rcar_gen2_compatible, },
 	{ .compatible = "renesas,sdhi-r8a7791", .data = &of_rcar_gen2_compatible, },
 	{ .compatible = "renesas,sdhi-r8a7792", .data = &of_rcar_gen2_compatible, },
@@ -126,6 +133,11 @@ static void renesas_sdhi_sys_dmac_abort_dma(struct tmio_mmc_host *host)
 	renesas_sdhi_sys_dmac_enable_dma(host, true);
 }
 
+static void renesas_sdhi_sys_dmac_dataend_dma(struct tmio_mmc_host *host)
+{
+	complete(&host->dma_dataend);
+}
+
 static void renesas_sdhi_sys_dmac_dma_callback(void *arg)
 {
 	struct tmio_mmc_host *host = arg;
@@ -451,10 +463,24 @@ static const struct tmio_mmc_dma_ops renesas_sdhi_sys_dmac_dma_ops = {
 	.request = renesas_sdhi_sys_dmac_request_dma,
 	.release = renesas_sdhi_sys_dmac_release_dma,
 	.abort = renesas_sdhi_sys_dmac_abort_dma,
+	.dataend = renesas_sdhi_sys_dmac_dataend_dma,
+};
+
+/*
+ * Whitelist of specific R-Car Gen3 SoC ES versions to use this DMAC
+ * implementation. Currently empty as all supported ES versions use
+ * the internal DMAC.
+ */
+static const struct soc_device_attribute gen3_soc_whitelist[] = {
+        { /* sentinel */ }
 };
 
 static int renesas_sdhi_sys_dmac_probe(struct platform_device *pdev)
 {
+	if (of_device_get_match_data(&pdev->dev) == &of_rcar_gen3_compatible &&
+	    !soc_device_match(gen3_soc_whitelist))
+		return -ENODEV;
+
 	return renesas_sdhi_probe(pdev, &renesas_sdhi_sys_dmac_dma_ops);
 }
 
diff --git a/drivers/mmc/host/rtsx_usb_sdmmc.c b/drivers/mmc/host/rtsx_usb_sdmmc.c
index 12d2fbe9c520..76da1687ab37 100644
--- a/drivers/mmc/host/rtsx_usb_sdmmc.c
+++ b/drivers/mmc/host/rtsx_usb_sdmmc.c
@@ -909,7 +909,7 @@ static int sd_set_bus_width(struct rtsx_usb_sdmmc *host,
 		unsigned char bus_width)
 {
 	int err = 0;
-	u8 width[] = {
+	static const u8 width[] = {
 		[MMC_BUS_WIDTH_1] = SD_BUS_WIDTH_1BIT,
 		[MMC_BUS_WIDTH_4] = SD_BUS_WIDTH_4BIT,
 		[MMC_BUS_WIDTH_8] = SD_BUS_WIDTH_8BIT,
diff --git a/drivers/mmc/host/s3cmci.c b/drivers/mmc/host/s3cmci.c
index 8896bf533dc7..f7f157a62a4a 100644
--- a/drivers/mmc/host/s3cmci.c
+++ b/drivers/mmc/host/s3cmci.c
@@ -1313,7 +1313,7 @@ static void s3cmci_enable_sdio_irq(struct mmc_host *mmc, int enable)
 	s3cmci_check_sdio_irq(host);
 }
 
-static struct mmc_host_ops s3cmci_ops = {
+static const struct mmc_host_ops s3cmci_ops = {
 	.request	= s3cmci_request,
 	.set_ios	= s3cmci_set_ios,
 	.get_ro		= mmc_gpio_get_ro,
diff --git a/drivers/mmc/host/sdhci-acpi.c b/drivers/mmc/host/sdhci-acpi.c
index ac678e9fb19a..08ae0ff13513 100644
--- a/drivers/mmc/host/sdhci-acpi.c
+++ b/drivers/mmc/host/sdhci-acpi.c
@@ -294,13 +294,10 @@ static int sdhci_acpi_sdio_probe_slot(struct platform_device *pdev,
 				      const char *hid, const char *uid)
 {
 	struct sdhci_acpi_host *c = platform_get_drvdata(pdev);
-	struct sdhci_host *host;
 
 	if (!c || !c->host)
 		return 0;
 
-	host = c->host;
-
 	/* Platform specific code during sdio probe slot goes here */
 
 	return 0;
@@ -432,7 +429,6 @@ static const struct sdhci_acpi_slot *sdhci_acpi_get_slot(const char *hid,
 static int sdhci_acpi_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	acpi_handle handle = ACPI_HANDLE(dev);
 	struct acpi_device *device, *child;
 	struct sdhci_acpi_host *c;
 	struct sdhci_host *host;
@@ -442,7 +438,8 @@ static int sdhci_acpi_probe(struct platform_device *pdev)
 	const char *uid;
 	int err;
 
-	if (acpi_bus_get_device(handle, &device))
+	device = ACPI_COMPANION(dev);
+	if (!device)
 		return -ENODEV;
 
 	hid = acpi_device_hid(device);
diff --git a/drivers/mmc/host/sdhci-bcm-kona.c b/drivers/mmc/host/sdhci-bcm-kona.c
index 51dd2fd65000..11ca95c60bcf 100644
--- a/drivers/mmc/host/sdhci-bcm-kona.c
+++ b/drivers/mmc/host/sdhci-bcm-kona.c
@@ -186,7 +186,7 @@ static void sdhci_bcm_kona_init_74_clocks(struct sdhci_host *host,
 		udelay(740);
 }
 
-static struct sdhci_ops sdhci_bcm_kona_ops = {
+static const struct sdhci_ops sdhci_bcm_kona_ops = {
 	.set_clock = sdhci_set_clock,
 	.get_max_clock = sdhci_pltfm_clk_get_max_clock,
 	.get_timeout_clock = sdhci_pltfm_clk_get_max_clock,
@@ -197,7 +197,7 @@ static struct sdhci_ops sdhci_bcm_kona_ops = {
 	.card_event = sdhci_bcm_kona_card_event,
 };
 
-static struct sdhci_pltfm_data sdhci_pltfm_data_kona = {
+static const struct sdhci_pltfm_data sdhci_pltfm_data_kona = {
 	.ops    = &sdhci_bcm_kona_ops,
 	.quirks = SDHCI_QUIRK_NO_CARD_NO_RESET |
 		SDHCI_QUIRK_BROKEN_TIMEOUT_VAL | SDHCI_QUIRK_32BIT_DMA_ADDR |
diff --git a/drivers/mmc/host/sdhci-brcmstb.c b/drivers/mmc/host/sdhci-brcmstb.c
index e2f638338e8f..552bddc5096c 100644
--- a/drivers/mmc/host/sdhci-brcmstb.c
+++ b/drivers/mmc/host/sdhci-brcmstb.c
@@ -21,41 +21,6 @@
 
 #include "sdhci-pltfm.h"
 
-#ifdef CONFIG_PM_SLEEP
-
-static int sdhci_brcmstb_suspend(struct device *dev)
-{
-	struct sdhci_host *host = dev_get_drvdata(dev);
-	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
-	int res;
-
-	if (host->tuning_mode != SDHCI_TUNING_MODE_3)
-		mmc_retune_needed(host->mmc);
-
-	res = sdhci_suspend_host(host);
-	if (res)
-		return res;
-	clk_disable_unprepare(pltfm_host->clk);
-	return res;
-}
-
-static int sdhci_brcmstb_resume(struct device *dev)
-{
-	struct sdhci_host *host = dev_get_drvdata(dev);
-	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
-	int err;
-
-	err = clk_prepare_enable(pltfm_host->clk);
-	if (err)
-		return err;
-	return sdhci_resume_host(host);
-}
-
-#endif /* CONFIG_PM_SLEEP */
-
-static SIMPLE_DEV_PM_OPS(sdhci_brcmstb_pmops, sdhci_brcmstb_suspend,
-			sdhci_brcmstb_resume);
-
 static const struct sdhci_ops sdhci_brcmstb_ops = {
 	.set_clock = sdhci_set_clock,
 	.set_bus_width = sdhci_set_bus_width,
@@ -63,7 +28,7 @@ static const struct sdhci_ops sdhci_brcmstb_ops = {
 	.set_uhs_signaling = sdhci_set_uhs_signaling,
 };
 
-static struct sdhci_pltfm_data sdhci_brcmstb_pdata = {
+static const struct sdhci_pltfm_data sdhci_brcmstb_pdata = {
 	.ops = &sdhci_brcmstb_ops,
 };
 
@@ -131,7 +96,7 @@ MODULE_DEVICE_TABLE(of, sdhci_brcm_of_match);
 static struct platform_driver sdhci_brcmstb_driver = {
 	.driver		= {
 		.name	= "sdhci-brcmstb",
-		.pm	= &sdhci_brcmstb_pmops,
+		.pm	= &sdhci_pltfm_pmops,
 		.of_match_table = of_match_ptr(sdhci_brcm_of_match),
 	},
 	.probe		= sdhci_brcmstb_probe,
diff --git a/drivers/mmc/host/sdhci-cadence.c b/drivers/mmc/host/sdhci-cadence.c
index 19d5698244b5..56529c3d389a 100644
--- a/drivers/mmc/host/sdhci-cadence.c
+++ b/drivers/mmc/host/sdhci-cadence.c
@@ -67,9 +67,16 @@
  */
 #define SDHCI_CDNS_MAX_TUNING_LOOP	40
 
+struct sdhci_cdns_phy_param {
+	u8 addr;
+	u8 data;
+};
+
 struct sdhci_cdns_priv {
 	void __iomem *hrs_addr;
 	bool enhanced_strobe;
+	unsigned int nr_phy_params;
+	struct sdhci_cdns_phy_param phy_params[0];
 };
 
 struct sdhci_cdns_phy_cfg {
@@ -115,9 +122,22 @@ static int sdhci_cdns_write_phy_reg(struct sdhci_cdns_priv *priv,
 	return 0;
 }
 
-static int sdhci_cdns_phy_init(struct device_node *np,
-			       struct sdhci_cdns_priv *priv)
+static unsigned int sdhci_cdns_phy_param_count(struct device_node *np)
+{
+	unsigned int count = 0;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(sdhci_cdns_phy_cfgs); i++)
+		if (of_property_read_bool(np, sdhci_cdns_phy_cfgs[i].property))
+			count++;
+
+	return count;
+}
+
+static void sdhci_cdns_phy_param_parse(struct device_node *np,
+				       struct sdhci_cdns_priv *priv)
 {
+	struct sdhci_cdns_phy_param *p = priv->phy_params;
 	u32 val;
 	int ret, i;
 
@@ -127,9 +147,19 @@ static int sdhci_cdns_phy_init(struct device_node *np,
 		if (ret)
 			continue;
 
-		ret = sdhci_cdns_write_phy_reg(priv,
-					       sdhci_cdns_phy_cfgs[i].addr,
-					       val);
+		p->addr = sdhci_cdns_phy_cfgs[i].addr;
+		p->data = val;
+		p++;
+	}
+}
+
+static int sdhci_cdns_phy_init(struct sdhci_cdns_priv *priv)
+{
+	int ret, i;
+
+	for (i = 0; i < priv->nr_phy_params; i++) {
+		ret = sdhci_cdns_write_phy_reg(priv, priv->phy_params[i].addr,
+					       priv->phy_params[i].data);
 		if (ret)
 			return ret;
 	}
@@ -302,6 +332,8 @@ static int sdhci_cdns_probe(struct platform_device *pdev)
 	struct sdhci_pltfm_host *pltfm_host;
 	struct sdhci_cdns_priv *priv;
 	struct clk *clk;
+	size_t priv_size;
+	unsigned int nr_phy_params;
 	int ret;
 	struct device *dev = &pdev->dev;
 
@@ -313,7 +345,9 @@ static int sdhci_cdns_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	host = sdhci_pltfm_init(pdev, &sdhci_cdns_pltfm_data, sizeof(*priv));
+	nr_phy_params = sdhci_cdns_phy_param_count(dev->of_node);
+	priv_size = sizeof(*priv) + sizeof(priv->phy_params[0]) * nr_phy_params;
+	host = sdhci_pltfm_init(pdev, &sdhci_cdns_pltfm_data, priv_size);
 	if (IS_ERR(host)) {
 		ret = PTR_ERR(host);
 		goto disable_clk;
@@ -322,7 +356,8 @@ static int sdhci_cdns_probe(struct platform_device *pdev)
 	pltfm_host = sdhci_priv(host);
 	pltfm_host->clk = clk;
 
-	priv = sdhci_cdns_priv(host);
+	priv = sdhci_pltfm_priv(pltfm_host);
+	priv->nr_phy_params = nr_phy_params;
 	priv->hrs_addr = host->ioaddr;
 	priv->enhanced_strobe = false;
 	host->ioaddr += SDHCI_CDNS_SRS_BASE;
@@ -336,7 +371,9 @@ static int sdhci_cdns_probe(struct platform_device *pdev)
 	if (ret)
 		goto free;
 
-	ret = sdhci_cdns_phy_init(dev->of_node, priv);
+	sdhci_cdns_phy_param_parse(dev->of_node, priv);
+
+	ret = sdhci_cdns_phy_init(priv);
 	if (ret)
 		goto free;
 
@@ -353,6 +390,39 @@ disable_clk:
 	return ret;
 }
 
+#ifdef CONFIG_PM_SLEEP
+static int sdhci_cdns_resume(struct device *dev)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_cdns_priv *priv = sdhci_pltfm_priv(pltfm_host);
+	int ret;
+
+	ret = clk_prepare_enable(pltfm_host->clk);
+	if (ret)
+		return ret;
+
+	ret = sdhci_cdns_phy_init(priv);
+	if (ret)
+		goto disable_clk;
+
+	ret = sdhci_resume_host(host);
+	if (ret)
+		goto disable_clk;
+
+	return 0;
+
+disable_clk:
+	clk_disable_unprepare(pltfm_host->clk);
+
+	return ret;
+}
+#endif
+
+static const struct dev_pm_ops sdhci_cdns_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(sdhci_pltfm_suspend, sdhci_cdns_resume)
+};
+
 static const struct of_device_id sdhci_cdns_match[] = {
 	{ .compatible = "socionext,uniphier-sd4hc" },
 	{ .compatible = "cdns,sd4hc" },
@@ -363,7 +433,7 @@ MODULE_DEVICE_TABLE(of, sdhci_cdns_match);
 static struct platform_driver sdhci_cdns_driver = {
 	.driver = {
 		.name = "sdhci-cdns",
-		.pm = &sdhci_pltfm_pmops,
+		.pm = &sdhci_cdns_pm_ops,
 		.of_match_table = sdhci_cdns_match,
 	},
 	.probe = sdhci_cdns_probe,
diff --git a/drivers/mmc/host/sdhci-esdhc.h b/drivers/mmc/host/sdhci-esdhc.h
index e7893f21b65e..dfa58f8b8dfa 100644
--- a/drivers/mmc/host/sdhci-esdhc.h
+++ b/drivers/mmc/host/sdhci-esdhc.h
@@ -54,6 +54,9 @@
 #define ESDHC_CLOCK_HCKEN		0x00000002
 #define ESDHC_CLOCK_IPGEN		0x00000001
 
+/* Host Controller Capabilities Register 2 */
+#define ESDHC_CAPABILITIES_1		0x114
+
 /* Tuning Block Control Register */
 #define ESDHC_TBCTL			0x120
 #define ESDHC_TB_EN			0x00000004
diff --git a/drivers/mmc/host/sdhci-msm.c b/drivers/mmc/host/sdhci-msm.c
index 9d601dc0d646..fc73e56eb1e2 100644
--- a/drivers/mmc/host/sdhci-msm.c
+++ b/drivers/mmc/host/sdhci-msm.c
@@ -611,7 +611,7 @@ static void msm_hc_select_hs400(struct sdhci_host *host)
  * HS400 - divided clock (free running MCLK/2)
  * All other modes - default (free running MCLK)
  */
-void sdhci_msm_hc_select_mode(struct sdhci_host *host)
+static void sdhci_msm_hc_select_mode(struct sdhci_host *host)
 {
 	struct mmc_ios ios = host->mmc->ios;
 
@@ -1049,7 +1049,7 @@ static unsigned int sdhci_msm_get_min_clock(struct sdhci_host *host)
  * instead directly control the GCC clock as per
  * HW recommendation.
  **/
-void __sdhci_msm_set_clock(struct sdhci_host *host, unsigned int clock)
+static void __sdhci_msm_set_clock(struct sdhci_host *host, unsigned int clock)
 {
 	u16 clk;
 	/*
@@ -1133,6 +1133,7 @@ static int sdhci_msm_probe(struct platform_device *pdev)
 	if (IS_ERR(host))
 		return PTR_ERR(host);
 
+	host->sdma_boundary = 0;
 	pltfm_host = sdhci_priv(host);
 	msm_host = sdhci_pltfm_priv(pltfm_host);
 	msm_host->mmc = host->mmc;
diff --git a/drivers/mmc/host/sdhci-of-arasan.c b/drivers/mmc/host/sdhci-of-arasan.c
index b13c0a7d50e4..0720ea717011 100644
--- a/drivers/mmc/host/sdhci-of-arasan.c
+++ b/drivers/mmc/host/sdhci-of-arasan.c
@@ -216,13 +216,13 @@ static void sdhci_arasan_hs400_enhanced_strobe(struct mmc_host *mmc,
 	u32 vendor;
 	struct sdhci_host *host = mmc_priv(mmc);
 
-	vendor = readl(host->ioaddr + SDHCI_ARASAN_VENDOR_REGISTER);
+	vendor = sdhci_readl(host, SDHCI_ARASAN_VENDOR_REGISTER);
 	if (ios->enhanced_strobe)
 		vendor |= VENDOR_ENHANCED_STROBE;
 	else
 		vendor &= ~VENDOR_ENHANCED_STROBE;
 
-	writel(vendor, host->ioaddr + SDHCI_ARASAN_VENDOR_REGISTER);
+	sdhci_writel(host, vendor, SDHCI_ARASAN_VENDOR_REGISTER);
 }
 
 static void sdhci_arasan_reset(struct sdhci_host *host, u8 mask)
@@ -262,7 +262,7 @@ static int sdhci_arasan_voltage_switch(struct mmc_host *mmc,
 	return -EINVAL;
 }
 
-static struct sdhci_ops sdhci_arasan_ops = {
+static const struct sdhci_ops sdhci_arasan_ops = {
 	.set_clock = sdhci_arasan_set_clock,
 	.get_max_clock = sdhci_pltfm_clk_get_max_clock,
 	.get_timeout_clock = sdhci_pltfm_clk_get_max_clock,
@@ -271,7 +271,7 @@ static struct sdhci_ops sdhci_arasan_ops = {
 	.set_uhs_signaling = sdhci_set_uhs_signaling,
 };
 
-static struct sdhci_pltfm_data sdhci_arasan_pdata = {
+static const struct sdhci_pltfm_data sdhci_arasan_pdata = {
 	.ops = &sdhci_arasan_ops,
 	.quirks = SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
 	.quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN |
diff --git a/drivers/mmc/host/sdhci-of-at91.c b/drivers/mmc/host/sdhci-of-at91.c
index 1485530c3592..4e47ed6bc716 100644
--- a/drivers/mmc/host/sdhci-of-at91.c
+++ b/drivers/mmc/host/sdhci-of-at91.c
@@ -42,6 +42,7 @@ struct sdhci_at91_priv {
 	struct clk *hclock;
 	struct clk *gck;
 	struct clk *mainck;
+	bool restore_needed;
 };
 
 static void sdhci_at91_set_force_card_detect(struct sdhci_host *host)
@@ -146,6 +147,100 @@ static const struct of_device_id sdhci_at91_dt_match[] = {
 };
 MODULE_DEVICE_TABLE(of, sdhci_at91_dt_match);
 
+static int sdhci_at91_set_clks_presets(struct device *dev)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_at91_priv *priv = sdhci_pltfm_priv(pltfm_host);
+	int ret;
+	unsigned int			caps0, caps1;
+	unsigned int			clk_base, clk_mul;
+	unsigned int			gck_rate, real_gck_rate;
+	unsigned int			preset_div;
+
+	/*
+	 * The mult clock is provided by as a generated clock by the PMC
+	 * controller. In order to set the rate of gck, we have to get the
+	 * base clock rate and the clock mult from capabilities.
+	 */
+	clk_prepare_enable(priv->hclock);
+	caps0 = readl(host->ioaddr + SDHCI_CAPABILITIES);
+	caps1 = readl(host->ioaddr + SDHCI_CAPABILITIES_1);
+	clk_base = (caps0 & SDHCI_CLOCK_V3_BASE_MASK) >> SDHCI_CLOCK_BASE_SHIFT;
+	clk_mul = (caps1 & SDHCI_CLOCK_MUL_MASK) >> SDHCI_CLOCK_MUL_SHIFT;
+	gck_rate = clk_base * 1000000 * (clk_mul + 1);
+	ret = clk_set_rate(priv->gck, gck_rate);
+	if (ret < 0) {
+		dev_err(dev, "failed to set gck");
+		clk_disable_unprepare(priv->hclock);
+		return ret;
+	}
+	/*
+	 * We need to check if we have the requested rate for gck because in
+	 * some cases this rate could be not supported. If it happens, the rate
+	 * is the closest one gck can provide. We have to update the value
+	 * of clk mul.
+	 */
+	real_gck_rate = clk_get_rate(priv->gck);
+	if (real_gck_rate != gck_rate) {
+		clk_mul = real_gck_rate / (clk_base * 1000000) - 1;
+		caps1 &= (~SDHCI_CLOCK_MUL_MASK);
+		caps1 |= ((clk_mul << SDHCI_CLOCK_MUL_SHIFT) &
+			  SDHCI_CLOCK_MUL_MASK);
+		/* Set capabilities in r/w mode. */
+		writel(SDMMC_CACR_KEY | SDMMC_CACR_CAPWREN,
+		       host->ioaddr + SDMMC_CACR);
+		writel(caps1, host->ioaddr + SDHCI_CAPABILITIES_1);
+		/* Set capabilities in ro mode. */
+		writel(0, host->ioaddr + SDMMC_CACR);
+		dev_info(dev, "update clk mul to %u as gck rate is %u Hz\n",
+			 clk_mul, real_gck_rate);
+	}
+
+	/*
+	 * We have to set preset values because it depends on the clk_mul
+	 * value. Moreover, SDR104 is supported in a degraded mode since the
+	 * maximum sd clock value is 120 MHz instead of 208 MHz. For that
+	 * reason, we need to use presets to support SDR104.
+	 */
+	preset_div = DIV_ROUND_UP(real_gck_rate, 24000000) - 1;
+	writew(SDHCI_AT91_PRESET_COMMON_CONF | preset_div,
+	       host->ioaddr + SDHCI_PRESET_FOR_SDR12);
+	preset_div = DIV_ROUND_UP(real_gck_rate, 50000000) - 1;
+	writew(SDHCI_AT91_PRESET_COMMON_CONF | preset_div,
+	       host->ioaddr + SDHCI_PRESET_FOR_SDR25);
+	preset_div = DIV_ROUND_UP(real_gck_rate, 100000000) - 1;
+	writew(SDHCI_AT91_PRESET_COMMON_CONF | preset_div,
+	       host->ioaddr + SDHCI_PRESET_FOR_SDR50);
+	preset_div = DIV_ROUND_UP(real_gck_rate, 120000000) - 1;
+	writew(SDHCI_AT91_PRESET_COMMON_CONF | preset_div,
+	       host->ioaddr + SDHCI_PRESET_FOR_SDR104);
+	preset_div = DIV_ROUND_UP(real_gck_rate, 50000000) - 1;
+	writew(SDHCI_AT91_PRESET_COMMON_CONF | preset_div,
+	       host->ioaddr + SDHCI_PRESET_FOR_DDR50);
+
+	clk_prepare_enable(priv->mainck);
+	clk_prepare_enable(priv->gck);
+
+	return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int sdhci_at91_suspend(struct device *dev)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_at91_priv *priv = sdhci_pltfm_priv(pltfm_host);
+	int ret;
+
+	ret = pm_runtime_force_suspend(dev);
+
+	priv->restore_needed = true;
+
+	return ret;
+}
+#endif /* CONFIG_PM_SLEEP */
+
 #ifdef CONFIG_PM
 static int sdhci_at91_runtime_suspend(struct device *dev)
 {
@@ -173,6 +268,15 @@ static int sdhci_at91_runtime_resume(struct device *dev)
 	struct sdhci_at91_priv *priv = sdhci_pltfm_priv(pltfm_host);
 	int ret;
 
+	if (priv->restore_needed) {
+		ret = sdhci_at91_set_clks_presets(dev);
+		if (ret)
+			return ret;
+
+		priv->restore_needed = false;
+		goto out;
+	}
+
 	ret = clk_prepare_enable(priv->mainck);
 	if (ret) {
 		dev_err(dev, "can't enable mainck\n");
@@ -191,13 +295,13 @@ static int sdhci_at91_runtime_resume(struct device *dev)
 		return ret;
 	}
 
+out:
 	return sdhci_runtime_resume_host(host);
 }
 #endif /* CONFIG_PM */
 
 static const struct dev_pm_ops sdhci_at91_dev_pm_ops = {
-	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
-				pm_runtime_force_resume)
+	SET_SYSTEM_SLEEP_PM_OPS(sdhci_at91_suspend, pm_runtime_force_resume)
 	SET_RUNTIME_PM_OPS(sdhci_at91_runtime_suspend,
 			   sdhci_at91_runtime_resume,
 			   NULL)
@@ -210,11 +314,7 @@ static int sdhci_at91_probe(struct platform_device *pdev)
 	struct sdhci_host		*host;
 	struct sdhci_pltfm_host		*pltfm_host;
 	struct sdhci_at91_priv		*priv;
-	unsigned int			caps0, caps1;
-	unsigned int			clk_base, clk_mul;
-	unsigned int			gck_rate, real_gck_rate;
 	int				ret;
-	unsigned int			preset_div;
 
 	match = of_match_device(sdhci_at91_dt_match, &pdev->dev);
 	if (!match)
@@ -246,66 +346,11 @@ static int sdhci_at91_probe(struct platform_device *pdev)
 		return PTR_ERR(priv->gck);
 	}
 
-	/*
-	 * The mult clock is provided by as a generated clock by the PMC
-	 * controller. In order to set the rate of gck, we have to get the
-	 * base clock rate and the clock mult from capabilities.
-	 */
-	clk_prepare_enable(priv->hclock);
-	caps0 = readl(host->ioaddr + SDHCI_CAPABILITIES);
-	caps1 = readl(host->ioaddr + SDHCI_CAPABILITIES_1);
-	clk_base = (caps0 & SDHCI_CLOCK_V3_BASE_MASK) >> SDHCI_CLOCK_BASE_SHIFT;
-	clk_mul = (caps1 & SDHCI_CLOCK_MUL_MASK) >> SDHCI_CLOCK_MUL_SHIFT;
-	gck_rate = clk_base * 1000000 * (clk_mul + 1);
-	ret = clk_set_rate(priv->gck, gck_rate);
-	if (ret < 0) {
-		dev_err(&pdev->dev, "failed to set gck");
-		goto hclock_disable_unprepare;
-	}
-	/*
-	 * We need to check if we have the requested rate for gck because in
-	 * some cases this rate could be not supported. If it happens, the rate
-	 * is the closest one gck can provide. We have to update the value
-	 * of clk mul.
-	 */
-	real_gck_rate = clk_get_rate(priv->gck);
-	if (real_gck_rate != gck_rate) {
-		clk_mul = real_gck_rate / (clk_base * 1000000) - 1;
-		caps1 &= (~SDHCI_CLOCK_MUL_MASK);
-		caps1 |= ((clk_mul << SDHCI_CLOCK_MUL_SHIFT) & SDHCI_CLOCK_MUL_MASK);
-		/* Set capabilities in r/w mode. */
-		writel(SDMMC_CACR_KEY | SDMMC_CACR_CAPWREN, host->ioaddr + SDMMC_CACR);
-		writel(caps1, host->ioaddr + SDHCI_CAPABILITIES_1);
-		/* Set capabilities in ro mode. */
-		writel(0, host->ioaddr + SDMMC_CACR);
-		dev_info(&pdev->dev, "update clk mul to %u as gck rate is %u Hz\n",
-			 clk_mul, real_gck_rate);
-	}
-
-	/*
-	 * We have to set preset values because it depends on the clk_mul
-	 * value. Moreover, SDR104 is supported in a degraded mode since the
-	 * maximum sd clock value is 120 MHz instead of 208 MHz. For that
-	 * reason, we need to use presets to support SDR104.
-	 */
-	preset_div = DIV_ROUND_UP(real_gck_rate, 24000000) - 1;
-	writew(SDHCI_AT91_PRESET_COMMON_CONF | preset_div,
-	       host->ioaddr + SDHCI_PRESET_FOR_SDR12);
-	preset_div = DIV_ROUND_UP(real_gck_rate, 50000000) - 1;
-	writew(SDHCI_AT91_PRESET_COMMON_CONF | preset_div,
-	       host->ioaddr + SDHCI_PRESET_FOR_SDR25);
-	preset_div = DIV_ROUND_UP(real_gck_rate, 100000000) - 1;
-	writew(SDHCI_AT91_PRESET_COMMON_CONF | preset_div,
-	       host->ioaddr + SDHCI_PRESET_FOR_SDR50);
-	preset_div = DIV_ROUND_UP(real_gck_rate, 120000000) - 1;
-	writew(SDHCI_AT91_PRESET_COMMON_CONF | preset_div,
-	       host->ioaddr + SDHCI_PRESET_FOR_SDR104);
-	preset_div = DIV_ROUND_UP(real_gck_rate, 50000000) - 1;
-	writew(SDHCI_AT91_PRESET_COMMON_CONF | preset_div,
-	       host->ioaddr + SDHCI_PRESET_FOR_DDR50);
+	ret = sdhci_at91_set_clks_presets(&pdev->dev);
+	if (ret)
+		goto sdhci_pltfm_free;
 
-	clk_prepare_enable(priv->mainck);
-	clk_prepare_enable(priv->gck);
+	priv->restore_needed = false;
 
 	ret = mmc_of_parse(host->mmc);
 	if (ret)
@@ -368,8 +413,8 @@ pm_runtime_disable:
 clocks_disable_unprepare:
 	clk_disable_unprepare(priv->gck);
 	clk_disable_unprepare(priv->mainck);
-hclock_disable_unprepare:
 	clk_disable_unprepare(priv->hclock);
+sdhci_pltfm_free:
 	sdhci_pltfm_free(pdev);
 	return ret;
 }
diff --git a/drivers/mmc/host/sdhci-of-esdhc.c b/drivers/mmc/host/sdhci-of-esdhc.c
index 44b016baa585..d96a057a7db8 100644
--- a/drivers/mmc/host/sdhci-of-esdhc.c
+++ b/drivers/mmc/host/sdhci-of-esdhc.c
@@ -86,6 +86,17 @@ static u32 esdhc_readl_fixup(struct sdhci_host *host,
 		return ret;
 	}
 
+	/*
+	 * DTS properties of mmc host are used to enable each speed mode
+	 * according to soc and board capability. So clean up
+	 * SDR50/SDR104/DDR50 support bits here.
+	 */
+	if (spec_reg == SDHCI_CAPABILITIES_1) {
+		ret = value & ~(SDHCI_SUPPORT_SDR50 | SDHCI_SUPPORT_SDR104 |
+				SDHCI_SUPPORT_DDR50);
+		return ret;
+	}
+
 	ret = value;
 	return ret;
 }
@@ -249,7 +260,11 @@ static u32 esdhc_be_readl(struct sdhci_host *host, int reg)
 	u32 ret;
 	u32 value;
 
-	value = ioread32be(host->ioaddr + reg);
+	if (reg == SDHCI_CAPABILITIES_1)
+		value = ioread32be(host->ioaddr + ESDHC_CAPABILITIES_1);
+	else
+		value = ioread32be(host->ioaddr + reg);
+
 	ret = esdhc_readl_fixup(host, reg, value);
 
 	return ret;
@@ -260,7 +275,11 @@ static u32 esdhc_le_readl(struct sdhci_host *host, int reg)
 	u32 ret;
 	u32 value;
 
-	value = ioread32(host->ioaddr + reg);
+	if (reg == SDHCI_CAPABILITIES_1)
+		value = ioread32(host->ioaddr + ESDHC_CAPABILITIES_1);
+	else
+		value = ioread32(host->ioaddr + reg);
+
 	ret = esdhc_readl_fixup(host, reg, value);
 
 	return ret;
diff --git a/drivers/mmc/host/sdhci-pci-core.c b/drivers/mmc/host/sdhci-pci-core.c
index e1721ac37919..bbaddf18a1b3 100644
--- a/drivers/mmc/host/sdhci-pci-core.c
+++ b/drivers/mmc/host/sdhci-pci-core.c
@@ -35,7 +35,6 @@
 #include "sdhci-pci-o2micro.h"
 
 static int sdhci_pci_enable_dma(struct sdhci_host *host);
-static void sdhci_pci_set_bus_width(struct sdhci_host *host, int width);
 static void sdhci_pci_hw_reset(struct sdhci_host *host);
 
 #ifdef CONFIG_PM_SLEEP
@@ -562,7 +561,7 @@ static const struct sdhci_ops sdhci_intel_byt_ops = {
 	.set_clock		= sdhci_set_clock,
 	.set_power		= sdhci_intel_set_power,
 	.enable_dma		= sdhci_pci_enable_dma,
-	.set_bus_width		= sdhci_pci_set_bus_width,
+	.set_bus_width		= sdhci_set_bus_width,
 	.reset			= sdhci_reset,
 	.set_uhs_signaling	= sdhci_set_uhs_signaling,
 	.hw_reset		= sdhci_pci_hw_reset,
@@ -730,6 +729,24 @@ static const struct sdhci_pci_fixes sdhci_intel_byt_sd = {
 #define INTEL_MRFLD_SD		2
 #define INTEL_MRFLD_SDIO	3
 
+#ifdef CONFIG_ACPI
+static void intel_mrfld_mmc_fix_up_power_slot(struct sdhci_pci_slot *slot)
+{
+	struct acpi_device *device, *child;
+
+	device = ACPI_COMPANION(&slot->chip->pdev->dev);
+	if (!device)
+		return;
+
+	acpi_device_fix_up_power(device);
+	list_for_each_entry(child, &device->children, node)
+		if (child->status.present && child->status.enabled)
+			acpi_device_fix_up_power(child);
+}
+#else
+static inline void intel_mrfld_mmc_fix_up_power_slot(struct sdhci_pci_slot *slot) {}
+#endif
+
 static int intel_mrfld_mmc_probe_slot(struct sdhci_pci_slot *slot)
 {
 	unsigned int func = PCI_FUNC(slot->chip->pdev->devfn);
@@ -751,6 +768,8 @@ static int intel_mrfld_mmc_probe_slot(struct sdhci_pci_slot *slot)
 	default:
 		return -ENODEV;
 	}
+
+	intel_mrfld_mmc_fix_up_power_slot(slot);
 	return 0;
 }
 
@@ -1197,7 +1216,7 @@ static int amd_probe(struct sdhci_pci_chip *chip)
 static const struct sdhci_ops amd_sdhci_pci_ops = {
 	.set_clock			= sdhci_set_clock,
 	.enable_dma			= sdhci_pci_enable_dma,
-	.set_bus_width			= sdhci_pci_set_bus_width,
+	.set_bus_width			= sdhci_set_bus_width,
 	.reset				= sdhci_reset,
 	.set_uhs_signaling		= sdhci_set_uhs_signaling,
 	.platform_execute_tuning	= amd_execute_tuning,
@@ -1313,29 +1332,6 @@ static int sdhci_pci_enable_dma(struct sdhci_host *host)
 	return 0;
 }
 
-static void sdhci_pci_set_bus_width(struct sdhci_host *host, int width)
-{
-	u8 ctrl;
-
-	ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
-
-	switch (width) {
-	case MMC_BUS_WIDTH_8:
-		ctrl |= SDHCI_CTRL_8BITBUS;
-		ctrl &= ~SDHCI_CTRL_4BITBUS;
-		break;
-	case MMC_BUS_WIDTH_4:
-		ctrl |= SDHCI_CTRL_4BITBUS;
-		ctrl &= ~SDHCI_CTRL_8BITBUS;
-		break;
-	default:
-		ctrl &= ~(SDHCI_CTRL_8BITBUS | SDHCI_CTRL_4BITBUS);
-		break;
-	}
-
-	sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
-}
-
 static void sdhci_pci_gpio_hw_reset(struct sdhci_host *host)
 {
 	struct sdhci_pci_slot *slot = sdhci_priv(host);
@@ -1362,7 +1358,7 @@ static void sdhci_pci_hw_reset(struct sdhci_host *host)
 static const struct sdhci_ops sdhci_pci_ops = {
 	.set_clock	= sdhci_set_clock,
 	.enable_dma	= sdhci_pci_enable_dma,
-	.set_bus_width	= sdhci_pci_set_bus_width,
+	.set_bus_width	= sdhci_set_bus_width,
 	.reset		= sdhci_reset,
 	.set_uhs_signaling = sdhci_set_uhs_signaling,
 	.hw_reset		= sdhci_pci_hw_reset,
diff --git a/drivers/mmc/host/sdhci-pic32.c b/drivers/mmc/host/sdhci-pic32.c
index 72c13b6f05f9..a6caa49ca25a 100644
--- a/drivers/mmc/host/sdhci-pic32.c
+++ b/drivers/mmc/host/sdhci-pic32.c
@@ -97,7 +97,7 @@ static const struct sdhci_ops pic32_sdhci_ops = {
 	.get_ro = pic32_sdhci_get_ro,
 };
 
-static struct sdhci_pltfm_data sdhci_pic32_pdata = {
+static const struct sdhci_pltfm_data sdhci_pic32_pdata = {
 	.ops = &pic32_sdhci_ops,
 	.quirks = SDHCI_QUIRK_NO_HISPD_BIT,
 	.quirks2 = SDHCI_QUIRK2_NO_1_8_V,
diff --git a/drivers/mmc/host/sdhci-pltfm.c b/drivers/mmc/host/sdhci-pltfm.c
index e090d8c42ddb..02bea6159d79 100644
--- a/drivers/mmc/host/sdhci-pltfm.c
+++ b/drivers/mmc/host/sdhci-pltfm.c
@@ -209,22 +209,42 @@ int sdhci_pltfm_unregister(struct platform_device *pdev)
 EXPORT_SYMBOL_GPL(sdhci_pltfm_unregister);
 
 #ifdef CONFIG_PM_SLEEP
-static int sdhci_pltfm_suspend(struct device *dev)
+int sdhci_pltfm_suspend(struct device *dev)
 {
 	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	int ret;
 
 	if (host->tuning_mode != SDHCI_TUNING_MODE_3)
 		mmc_retune_needed(host->mmc);
 
-	return sdhci_suspend_host(host);
+	ret = sdhci_suspend_host(host);
+	if (ret)
+		return ret;
+
+	clk_disable_unprepare(pltfm_host->clk);
+
+	return 0;
 }
+EXPORT_SYMBOL_GPL(sdhci_pltfm_suspend);
 
-static int sdhci_pltfm_resume(struct device *dev)
+int sdhci_pltfm_resume(struct device *dev)
 {
 	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	int ret;
+
+	ret = clk_prepare_enable(pltfm_host->clk);
+	if (ret)
+		return ret;
 
-	return sdhci_resume_host(host);
+	ret = sdhci_resume_host(host);
+	if (ret)
+		clk_disable_unprepare(pltfm_host->clk);
+
+	return ret;
 }
+EXPORT_SYMBOL_GPL(sdhci_pltfm_resume);
 #endif
 
 const struct dev_pm_ops sdhci_pltfm_pmops = {
diff --git a/drivers/mmc/host/sdhci-pltfm.h b/drivers/mmc/host/sdhci-pltfm.h
index 957839d0fe37..1e91fb1c020e 100644
--- a/drivers/mmc/host/sdhci-pltfm.h
+++ b/drivers/mmc/host/sdhci-pltfm.h
@@ -109,6 +109,8 @@ static inline void *sdhci_pltfm_priv(struct sdhci_pltfm_host *host)
 	return host->private;
 }
 
+int sdhci_pltfm_suspend(struct device *dev);
+int sdhci_pltfm_resume(struct device *dev);
 extern const struct dev_pm_ops sdhci_pltfm_pmops;
 
 #endif /* _DRIVERS_MMC_SDHCI_PLTFM_H */
diff --git a/drivers/mmc/host/sdhci-pxav2.c b/drivers/mmc/host/sdhci-pxav2.c
index 995083ce1c46..8986f9d9cf98 100644
--- a/drivers/mmc/host/sdhci-pxav2.c
+++ b/drivers/mmc/host/sdhci-pxav2.c
@@ -178,17 +178,17 @@ static int sdhci_pxav2_probe(struct platform_device *pdev)
 
 	pltfm_host = sdhci_priv(host);
 
-	clk = clk_get(dev, "PXA-SDHCLK");
+	clk = devm_clk_get(dev, "PXA-SDHCLK");
 	if (IS_ERR(clk)) {
 		dev_err(dev, "failed to get io clock\n");
 		ret = PTR_ERR(clk);
-		goto err_clk_get;
+		goto free;
 	}
 	pltfm_host->clk = clk;
 	ret = clk_prepare_enable(clk);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to enable io clock\n");
-		goto err_clk_enable;
+		goto free;
 	}
 
 	host->quirks = SDHCI_QUIRK_BROKEN_ADMA
@@ -223,34 +223,18 @@ static int sdhci_pxav2_probe(struct platform_device *pdev)
 	ret = sdhci_add_host(host);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to add host\n");
-		goto err_add_host;
+		goto disable_clk;
 	}
 
 	return 0;
 
-err_add_host:
+disable_clk:
 	clk_disable_unprepare(clk);
-err_clk_enable:
-	clk_put(clk);
-err_clk_get:
+free:
 	sdhci_pltfm_free(pdev);
 	return ret;
 }
 
-static int sdhci_pxav2_remove(struct platform_device *pdev)
-{
-	struct sdhci_host *host = platform_get_drvdata(pdev);
-	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
-
-	sdhci_remove_host(host, 1);
-
-	clk_disable_unprepare(pltfm_host->clk);
-	clk_put(pltfm_host->clk);
-	sdhci_pltfm_free(pdev);
-
-	return 0;
-}
-
 static struct platform_driver sdhci_pxav2_driver = {
 	.driver		= {
 		.name	= "sdhci-pxav2",
@@ -258,7 +242,7 @@ static struct platform_driver sdhci_pxav2_driver = {
 		.pm	= &sdhci_pltfm_pmops,
 	},
 	.probe		= sdhci_pxav2_probe,
-	.remove		= sdhci_pxav2_remove,
+	.remove		= sdhci_pltfm_unregister,
 };
 
 module_platform_driver(sdhci_pxav2_driver);
diff --git a/drivers/mmc/host/sdhci-pxav3.c b/drivers/mmc/host/sdhci-pxav3.c
index f953f35c2624..a34434166ca7 100644
--- a/drivers/mmc/host/sdhci-pxav3.c
+++ b/drivers/mmc/host/sdhci-pxav3.c
@@ -337,7 +337,7 @@ static const struct sdhci_ops pxav3_sdhci_ops = {
 	.set_uhs_signaling = pxav3_set_uhs_signaling,
 };
 
-static struct sdhci_pltfm_data sdhci_pxav3_pdata = {
+static const struct sdhci_pltfm_data sdhci_pxav3_pdata = {
 	.quirks = SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK
 		| SDHCI_QUIRK_NO_ENDATTR_IN_NOPDESC
 		| SDHCI_QUIRK_32BIT_ADMA_SIZE
diff --git a/drivers/mmc/host/sdhci-s3c.c b/drivers/mmc/host/sdhci-s3c.c
index 7c065a70f92b..d328fcf284d1 100644
--- a/drivers/mmc/host/sdhci-s3c.c
+++ b/drivers/mmc/host/sdhci-s3c.c
@@ -414,43 +414,11 @@ static void sdhci_cmu_set_clock(struct sdhci_host *host, unsigned int clock)
 	sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL);
 }
 
-/**
- * sdhci_s3c_set_bus_width - support 8bit buswidth
- * @host: The SDHCI host being queried
- * @width: MMC_BUS_WIDTH_ macro for the bus width being requested
- *
- * We have 8-bit width support but is not a v3 controller.
- * So we add platform_bus_width() and support 8bit width.
- */
-static void sdhci_s3c_set_bus_width(struct sdhci_host *host, int width)
-{
-	u8 ctrl;
-
-	ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
-
-	switch (width) {
-	case MMC_BUS_WIDTH_8:
-		ctrl |= SDHCI_CTRL_8BITBUS;
-		ctrl &= ~SDHCI_CTRL_4BITBUS;
-		break;
-	case MMC_BUS_WIDTH_4:
-		ctrl |= SDHCI_CTRL_4BITBUS;
-		ctrl &= ~SDHCI_CTRL_8BITBUS;
-		break;
-	default:
-		ctrl &= ~SDHCI_CTRL_4BITBUS;
-		ctrl &= ~SDHCI_CTRL_8BITBUS;
-		break;
-	}
-
-	sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
-}
-
 static struct sdhci_ops sdhci_s3c_ops = {
 	.get_max_clock		= sdhci_s3c_get_max_clk,
 	.set_clock		= sdhci_s3c_set_clock,
 	.get_min_clock		= sdhci_s3c_get_min_clock,
-	.set_bus_width		= sdhci_s3c_set_bus_width,
+	.set_bus_width		= sdhci_set_bus_width,
 	.reset			= sdhci_reset,
 	.set_uhs_signaling	= sdhci_set_uhs_signaling,
 };
diff --git a/drivers/mmc/host/sdhci-sirf.c b/drivers/mmc/host/sdhci-sirf.c
index c251c6c0a112..391d52b467ca 100644
--- a/drivers/mmc/host/sdhci-sirf.c
+++ b/drivers/mmc/host/sdhci-sirf.c
@@ -146,7 +146,7 @@ retry:
 	return rc;
 }
 
-static struct sdhci_ops sdhci_sirf_ops = {
+static const struct sdhci_ops sdhci_sirf_ops = {
 	.read_l = sdhci_sirf_readl_le,
 	.read_w = sdhci_sirf_readw_le,
 	.platform_execute_tuning = sdhci_sirf_execute_tuning,
@@ -157,7 +157,7 @@ static struct sdhci_ops sdhci_sirf_ops = {
 	.set_uhs_signaling = sdhci_set_uhs_signaling,
 };
 
-static struct sdhci_pltfm_data sdhci_sirf_pdata = {
+static const struct sdhci_pltfm_data sdhci_sirf_pdata = {
 	.ops = &sdhci_sirf_ops,
 	.quirks = SDHCI_QUIRK_BROKEN_TIMEOUT_VAL |
 		SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK |
@@ -230,43 +230,6 @@ err_clk_prepare:
 	return ret;
 }
 
-#ifdef CONFIG_PM_SLEEP
-static int sdhci_sirf_suspend(struct device *dev)
-{
-	struct sdhci_host *host = dev_get_drvdata(dev);
-	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
-	int ret;
-
-	if (host->tuning_mode != SDHCI_TUNING_MODE_3)
-		mmc_retune_needed(host->mmc);
-
-	ret = sdhci_suspend_host(host);
-	if (ret)
-		return ret;
-
-	clk_disable(pltfm_host->clk);
-
-	return 0;
-}
-
-static int sdhci_sirf_resume(struct device *dev)
-{
-	struct sdhci_host *host = dev_get_drvdata(dev);
-	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
-	int ret;
-
-	ret = clk_enable(pltfm_host->clk);
-	if (ret) {
-		dev_dbg(dev, "Resume: Error enabling clock\n");
-		return ret;
-	}
-
-	return sdhci_resume_host(host);
-}
-#endif
-
-static SIMPLE_DEV_PM_OPS(sdhci_sirf_pm_ops, sdhci_sirf_suspend, sdhci_sirf_resume);
-
 static const struct of_device_id sdhci_sirf_of_match[] = {
 	{ .compatible = "sirf,prima2-sdhc" },
 	{ }
@@ -277,7 +240,7 @@ static struct platform_driver sdhci_sirf_driver = {
 	.driver		= {
 		.name	= "sdhci-sirf",
 		.of_match_table = sdhci_sirf_of_match,
-		.pm	= &sdhci_sirf_pm_ops,
+		.pm	= &sdhci_pltfm_pmops,
 	},
 	.probe		= sdhci_sirf_probe,
 	.remove		= sdhci_pltfm_unregister,
diff --git a/drivers/mmc/host/sdhci-st.c b/drivers/mmc/host/sdhci-st.c
index 68c36c9fa231..c32daed0d418 100644
--- a/drivers/mmc/host/sdhci-st.c
+++ b/drivers/mmc/host/sdhci-st.c
@@ -371,7 +371,7 @@ static int sdhci_st_probe(struct platform_device *pdev)
 	if (IS_ERR(icnclk))
 		icnclk = NULL;
 
-	rstc = devm_reset_control_get(&pdev->dev, NULL);
+	rstc = devm_reset_control_get_exclusive(&pdev->dev, NULL);
 	if (IS_ERR(rstc))
 		rstc = NULL;
 	else
@@ -394,8 +394,17 @@ static int sdhci_st_probe(struct platform_device *pdev)
 		goto err_of;
 	}
 
-	clk_prepare_enable(clk);
-	clk_prepare_enable(icnclk);
+	ret = clk_prepare_enable(clk);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to prepare clock\n");
+		goto err_of;
+	}
+
+	ret = clk_prepare_enable(icnclk);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to prepare icn clock\n");
+		goto err_icnclk;
+	}
 
 	/* Configure the FlashSS Top registers for setting eMMC TX/RX delay */
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
@@ -429,6 +438,7 @@ static int sdhci_st_probe(struct platform_device *pdev)
 
 err_out:
 	clk_disable_unprepare(icnclk);
+err_icnclk:
 	clk_disable_unprepare(clk);
 err_of:
 	sdhci_pltfm_free(pdev);
@@ -487,9 +497,17 @@ static int sdhci_st_resume(struct device *dev)
 	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 	struct st_mmc_platform_data *pdata = sdhci_pltfm_priv(pltfm_host);
 	struct device_node *np = dev->of_node;
+	int ret;
+
+	ret = clk_prepare_enable(pltfm_host->clk);
+	if (ret)
+		return ret;
 
-	clk_prepare_enable(pltfm_host->clk);
-	clk_prepare_enable(pdata->icnclk);
+	ret = clk_prepare_enable(pdata->icnclk);
+	if (ret) {
+		clk_disable_unprepare(pltfm_host->clk);
+		return ret;
+	}
 
 	if (pdata->rstc)
 		reset_control_deassert(pdata->rstc);
diff --git a/drivers/mmc/host/sdhci-tegra.c b/drivers/mmc/host/sdhci-tegra.c
index 7f93079c7a3a..0cd6fa80db66 100644
--- a/drivers/mmc/host/sdhci-tegra.c
+++ b/drivers/mmc/host/sdhci-tegra.c
@@ -190,25 +190,6 @@ static void tegra_sdhci_reset(struct sdhci_host *host, u8 mask)
 	tegra_host->ddr_signaling = false;
 }
 
-static void tegra_sdhci_set_bus_width(struct sdhci_host *host, int bus_width)
-{
-	u32 ctrl;
-
-	ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
-	if ((host->mmc->caps & MMC_CAP_8_BIT_DATA) &&
-	    (bus_width == MMC_BUS_WIDTH_8)) {
-		ctrl &= ~SDHCI_CTRL_4BITBUS;
-		ctrl |= SDHCI_CTRL_8BITBUS;
-	} else {
-		ctrl &= ~SDHCI_CTRL_8BITBUS;
-		if (bus_width == MMC_BUS_WIDTH_4)
-			ctrl |= SDHCI_CTRL_4BITBUS;
-		else
-			ctrl &= ~SDHCI_CTRL_4BITBUS;
-	}
-	sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
-}
-
 static void tegra_sdhci_pad_autocalib(struct sdhci_host *host)
 {
 	u32 val;
@@ -323,7 +304,7 @@ static const struct sdhci_ops tegra_sdhci_ops = {
 	.read_w     = tegra_sdhci_readw,
 	.write_l    = tegra_sdhci_writel,
 	.set_clock  = tegra_sdhci_set_clock,
-	.set_bus_width = tegra_sdhci_set_bus_width,
+	.set_bus_width = sdhci_set_bus_width,
 	.reset      = tegra_sdhci_reset,
 	.platform_execute_tuning = tegra_sdhci_execute_tuning,
 	.set_uhs_signaling = tegra_sdhci_set_uhs_signaling,
@@ -371,7 +352,7 @@ static const struct sdhci_ops tegra114_sdhci_ops = {
 	.write_w    = tegra_sdhci_writew,
 	.write_l    = tegra_sdhci_writel,
 	.set_clock  = tegra_sdhci_set_clock,
-	.set_bus_width = tegra_sdhci_set_bus_width,
+	.set_bus_width = sdhci_set_bus_width,
 	.reset      = tegra_sdhci_reset,
 	.platform_execute_tuning = tegra_sdhci_execute_tuning,
 	.set_uhs_signaling = tegra_sdhci_set_uhs_signaling,
@@ -508,7 +489,8 @@ static int sdhci_tegra_probe(struct platform_device *pdev)
 	clk_prepare_enable(clk);
 	pltfm_host->clk = clk;
 
-	tegra_host->rst = devm_reset_control_get(&pdev->dev, "sdhci");
+	tegra_host->rst = devm_reset_control_get_exclusive(&pdev->dev,
+							   "sdhci");
 	if (IS_ERR(tegra_host->rst)) {
 		rc = PTR_ERR(tegra_host->rst);
 		dev_err(&pdev->dev, "failed to get reset control: %d\n", rc);
diff --git a/drivers/mmc/host/sdhci-xenon-phy.c b/drivers/mmc/host/sdhci-xenon-phy.c
index f7e26b031e76..ec8794335241 100644
--- a/drivers/mmc/host/sdhci-xenon-phy.c
+++ b/drivers/mmc/host/sdhci-xenon-phy.c
@@ -409,17 +409,30 @@ static int xenon_emmc_phy_config_tuning(struct sdhci_host *host)
 	return 0;
 }
 
-static void xenon_emmc_phy_disable_data_strobe(struct sdhci_host *host)
+static void xenon_emmc_phy_disable_strobe(struct sdhci_host *host)
 {
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct xenon_priv *priv = sdhci_pltfm_priv(pltfm_host);
 	u32 reg;
 
-	/* Disable SDHC Data Strobe */
+	/* Disable both SDHC Data Strobe and Enhanced Strobe */
 	reg = sdhci_readl(host, XENON_SLOT_EMMC_CTRL);
-	reg &= ~XENON_ENABLE_DATA_STROBE;
+	reg &= ~(XENON_ENABLE_DATA_STROBE | XENON_ENABLE_RESP_STROBE);
 	sdhci_writel(host, reg, XENON_SLOT_EMMC_CTRL);
+
+	/* Clear Strobe line Pull down or Pull up */
+	if (priv->phy_type == EMMC_5_0_PHY) {
+		reg = sdhci_readl(host, XENON_EMMC_5_0_PHY_PAD_CONTROL);
+		reg &= ~(XENON_EMMC5_FC_QSP_PD | XENON_EMMC5_FC_QSP_PU);
+		sdhci_writel(host, reg, XENON_EMMC_5_0_PHY_PAD_CONTROL);
+	} else {
+		reg = sdhci_readl(host, XENON_EMMC_PHY_PAD_CONTROL1);
+		reg &= ~(XENON_EMMC5_1_FC_QSP_PD | XENON_EMMC5_1_FC_QSP_PU);
+		sdhci_writel(host, reg, XENON_EMMC_PHY_PAD_CONTROL1);
+	}
 }
 
-/* Set HS400 Data Strobe */
+/* Set HS400 Data Strobe and Enhanced Strobe */
 static void xenon_emmc_phy_strobe_delay_adj(struct sdhci_host *host)
 {
 	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
@@ -439,6 +452,15 @@ static void xenon_emmc_phy_strobe_delay_adj(struct sdhci_host *host)
 	/* Enable SDHC Data Strobe */
 	reg = sdhci_readl(host, XENON_SLOT_EMMC_CTRL);
 	reg |= XENON_ENABLE_DATA_STROBE;
+	/*
+	 * Enable SDHC Enhanced Strobe if supported
+	 * Xenon Enhanced Strobe should be enabled only when
+	 * 1. card is in HS400 mode and
+	 * 2. SDCLK is higher than 52MHz
+	 * 3. DLL is enabled
+	 */
+	if (host->mmc->ios.enhanced_strobe)
+		reg |= XENON_ENABLE_RESP_STROBE;
 	sdhci_writel(host, reg, XENON_SLOT_EMMC_CTRL);
 
 	/* Set Data Strobe Pull down */
@@ -615,7 +637,7 @@ static void xenon_emmc_phy_set(struct sdhci_host *host,
 		sdhci_writel(host, phy_regs->logic_timing_val,
 			     phy_regs->logic_timing_adj);
 	else
-		xenon_emmc_phy_disable_data_strobe(host);
+		xenon_emmc_phy_disable_strobe(host);
 
 phy_init:
 	xenon_emmc_phy_init(host);
@@ -705,7 +727,7 @@ void xenon_soc_pad_ctrl(struct sdhci_host *host,
 
 /*
  * Setting PHY when card is working in High Speed Mode.
- * HS400 set data strobe line.
+ * HS400 set Data Strobe and Enhanced Strobe if it is supported.
  * HS200/SDR104 set tuning config to prepare for tuning.
  */
 static int xenon_hs_delay_adj(struct sdhci_host *host)
diff --git a/drivers/mmc/host/sdhci-xenon.c b/drivers/mmc/host/sdhci-xenon.c
index bc1781bb070b..2eec2e652c53 100644
--- a/drivers/mmc/host/sdhci-xenon.c
+++ b/drivers/mmc/host/sdhci-xenon.c
@@ -18,6 +18,8 @@
 #include <linux/ktime.h>
 #include <linux/module.h>
 #include <linux/of.h>
+#include <linux/pm.h>
+#include <linux/pm_runtime.h>
 
 #include "sdhci-pltfm.h"
 #include "sdhci-xenon.h"
@@ -210,8 +212,27 @@ static void xenon_set_uhs_signaling(struct sdhci_host *host,
 	sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2);
 }
 
+static void xenon_set_power(struct sdhci_host *host, unsigned char mode,
+		unsigned short vdd)
+{
+	struct mmc_host *mmc = host->mmc;
+	u8 pwr = host->pwr;
+
+	sdhci_set_power_noreg(host, mode, vdd);
+
+	if (host->pwr == pwr)
+		return;
+
+	if (host->pwr == 0)
+		vdd = 0;
+
+	if (!IS_ERR(mmc->supply.vmmc))
+		mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, vdd);
+}
+
 static const struct sdhci_ops sdhci_xenon_ops = {
 	.set_clock		= sdhci_set_clock,
+	.set_power		= xenon_set_power,
 	.set_bus_width		= sdhci_set_bus_width,
 	.reset			= xenon_reset,
 	.set_uhs_signaling	= xenon_set_uhs_signaling,
@@ -311,7 +332,8 @@ static int xenon_execute_tuning(struct mmc_host *mmc, u32 opcode)
 {
 	struct sdhci_host *host = mmc_priv(mmc);
 
-	if (host->timing == MMC_TIMING_UHS_DDR50)
+	if (host->timing == MMC_TIMING_UHS_DDR50 ||
+		host->timing == MMC_TIMING_MMC_DDR52)
 		return 0;
 
 	/*
@@ -444,7 +466,6 @@ static int xenon_probe(struct platform_device *pdev)
 {
 	struct sdhci_pltfm_host *pltfm_host;
 	struct sdhci_host *host;
-	struct xenon_priv *priv;
 	int err;
 
 	host = sdhci_pltfm_init(pdev, &sdhci_xenon_pdata,
@@ -453,7 +474,6 @@ static int xenon_probe(struct platform_device *pdev)
 		return PTR_ERR(host);
 
 	pltfm_host = sdhci_priv(host);
-	priv = sdhci_pltfm_priv(pltfm_host);
 
 	/*
 	 * Link Xenon specific mmc_host_ops function,
@@ -488,13 +508,24 @@ static int xenon_probe(struct platform_device *pdev)
 	if (err)
 		goto err_clk;
 
+	pm_runtime_get_noresume(&pdev->dev);
+	pm_runtime_set_active(&pdev->dev);
+	pm_runtime_set_autosuspend_delay(&pdev->dev, 50);
+	pm_runtime_use_autosuspend(&pdev->dev);
+	pm_runtime_enable(&pdev->dev);
+	pm_suspend_ignore_children(&pdev->dev, 1);
+
 	err = sdhci_add_host(host);
 	if (err)
 		goto remove_sdhc;
 
+	pm_runtime_put_autosuspend(&pdev->dev);
+
 	return 0;
 
 remove_sdhc:
+	pm_runtime_disable(&pdev->dev);
+	pm_runtime_put_noidle(&pdev->dev);
 	xenon_sdhc_unprepare(host);
 err_clk:
 	clk_disable_unprepare(pltfm_host->clk);
@@ -508,6 +539,10 @@ static int xenon_remove(struct platform_device *pdev)
 	struct sdhci_host *host = platform_get_drvdata(pdev);
 	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 
+	pm_runtime_get_sync(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+	pm_runtime_put_noidle(&pdev->dev);
+
 	sdhci_remove_host(host, 0);
 
 	xenon_sdhc_unprepare(host);
@@ -519,6 +554,84 @@ static int xenon_remove(struct platform_device *pdev)
 	return 0;
 }
 
+#ifdef CONFIG_PM_SLEEP
+static int xenon_suspend(struct device *dev)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct xenon_priv *priv = sdhci_pltfm_priv(pltfm_host);
+	int ret;
+
+	ret = pm_runtime_force_suspend(dev);
+
+	priv->restore_needed = true;
+	return ret;
+}
+#endif
+
+#ifdef CONFIG_PM
+static int xenon_runtime_suspend(struct device *dev)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct xenon_priv *priv = sdhci_pltfm_priv(pltfm_host);
+	int ret;
+
+	ret = sdhci_runtime_suspend_host(host);
+	if (ret)
+		return ret;
+
+	if (host->tuning_mode != SDHCI_TUNING_MODE_3)
+		mmc_retune_needed(host->mmc);
+
+	clk_disable_unprepare(pltfm_host->clk);
+	/*
+	 * Need to update the priv->clock here, or when runtime resume
+	 * back, phy don't aware the clock change and won't adjust phy
+	 * which will cause cmd err
+	 */
+	priv->clock = 0;
+	return 0;
+}
+
+static int xenon_runtime_resume(struct device *dev)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct xenon_priv *priv = sdhci_pltfm_priv(pltfm_host);
+	int ret;
+
+	ret = clk_prepare_enable(pltfm_host->clk);
+	if (ret) {
+		dev_err(dev, "can't enable mainck\n");
+		return ret;
+	}
+
+	if (priv->restore_needed) {
+		ret = xenon_sdhc_prepare(host);
+		if (ret)
+			goto out;
+		priv->restore_needed = false;
+	}
+
+	ret = sdhci_runtime_resume_host(host);
+	if (ret)
+		goto out;
+	return 0;
+out:
+	clk_disable_unprepare(pltfm_host->clk);
+	return ret;
+}
+#endif /* CONFIG_PM */
+
+static const struct dev_pm_ops sdhci_xenon_dev_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(xenon_suspend,
+				pm_runtime_force_resume)
+	SET_RUNTIME_PM_OPS(xenon_runtime_suspend,
+			   xenon_runtime_resume,
+			   NULL)
+};
+
 static const struct of_device_id sdhci_xenon_dt_ids[] = {
 	{ .compatible = "marvell,armada-ap806-sdhci",},
 	{ .compatible = "marvell,armada-cp110-sdhci",},
@@ -531,7 +644,7 @@ static struct platform_driver sdhci_xenon_driver = {
 	.driver	= {
 		.name	= "xenon-sdhci",
 		.of_match_table = sdhci_xenon_dt_ids,
-		.pm = &sdhci_pltfm_pmops,
+		.pm = &sdhci_xenon_dev_pm_ops,
 	},
 	.probe	= xenon_probe,
 	.remove	= xenon_remove,
diff --git a/drivers/mmc/host/sdhci-xenon.h b/drivers/mmc/host/sdhci-xenon.h
index 73debb42dc2f..2bc0510c0769 100644
--- a/drivers/mmc/host/sdhci-xenon.h
+++ b/drivers/mmc/host/sdhci-xenon.h
@@ -33,6 +33,7 @@
 #define XENON_TUNING_STEP_DIVIDER		BIT(6)
 
 #define XENON_SLOT_EMMC_CTRL			0x0130
+#define XENON_ENABLE_RESP_STROBE		BIT(25)
 #define XENON_ENABLE_DATA_STROBE		BIT(24)
 
 #define XENON_SLOT_RETUNING_REQ_CTRL		0x0144
@@ -90,6 +91,7 @@ struct xenon_priv {
 	 */
 	void		*phy_params;
 	struct xenon_emmc_phy_regs *emmc_phy_regs;
+	bool restore_needed;
 };
 
 int xenon_phy_adj(struct sdhci_host *host, struct mmc_ios *ios);
diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c
index ecd0d4350e8a..0d5fcca18c9e 100644
--- a/drivers/mmc/host/sdhci.c
+++ b/drivers/mmc/host/sdhci.c
@@ -897,8 +897,8 @@ static void sdhci_prepare_data(struct sdhci_host *host, struct mmc_command *cmd)
 	sdhci_set_transfer_irqs(host);
 
 	/* Set the DMA boundary value and block size */
-	sdhci_writew(host, SDHCI_MAKE_BLKSZ(SDHCI_DEFAULT_BOUNDARY_ARG,
-		data->blksz), SDHCI_BLOCK_SIZE);
+	sdhci_writew(host, SDHCI_MAKE_BLKSZ(host->sdma_boundary, data->blksz),
+		     SDHCI_BLOCK_SIZE);
 	sdhci_writew(host, data->blocks, SDHCI_BLOCK_COUNT);
 }
 
@@ -1173,24 +1173,35 @@ void sdhci_send_command(struct sdhci_host *host, struct mmc_command *cmd)
 }
 EXPORT_SYMBOL_GPL(sdhci_send_command);
 
+static void sdhci_read_rsp_136(struct sdhci_host *host, struct mmc_command *cmd)
+{
+	int i, reg;
+
+	for (i = 0; i < 4; i++) {
+		reg = SDHCI_RESPONSE + (3 - i) * 4;
+		cmd->resp[i] = sdhci_readl(host, reg);
+	}
+
+	if (host->quirks2 & SDHCI_QUIRK2_RSP_136_HAS_CRC)
+		return;
+
+	/* CRC is stripped so we need to do some shifting */
+	for (i = 0; i < 4; i++) {
+		cmd->resp[i] <<= 8;
+		if (i != 3)
+			cmd->resp[i] |= cmd->resp[i + 1] >> 24;
+	}
+}
+
 static void sdhci_finish_command(struct sdhci_host *host)
 {
 	struct mmc_command *cmd = host->cmd;
-	int i;
 
 	host->cmd = NULL;
 
 	if (cmd->flags & MMC_RSP_PRESENT) {
 		if (cmd->flags & MMC_RSP_136) {
-			/* CRC is stripped so we need to do some shifting. */
-			for (i = 0;i < 4;i++) {
-				cmd->resp[i] = sdhci_readl(host,
-					SDHCI_RESPONSE + (3-i)*4) << 8;
-				if (i != 3)
-					cmd->resp[i] |=
-						sdhci_readb(host,
-						SDHCI_RESPONSE + (3-i)*4-1);
-			}
+			sdhci_read_rsp_136(host, cmd);
 		} else {
 			cmd->resp[0] = sdhci_readl(host, SDHCI_RESPONSE);
 		}
@@ -1544,10 +1555,9 @@ void sdhci_set_bus_width(struct sdhci_host *host, int width)
 	ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
 	if (width == MMC_BUS_WIDTH_8) {
 		ctrl &= ~SDHCI_CTRL_4BITBUS;
-		if (host->version >= SDHCI_SPEC_300)
-			ctrl |= SDHCI_CTRL_8BITBUS;
+		ctrl |= SDHCI_CTRL_8BITBUS;
 	} else {
-		if (host->version >= SDHCI_SPEC_300)
+		if (host->mmc->caps & MMC_CAP_8_BIT_DATA)
 			ctrl &= ~SDHCI_CTRL_8BITBUS;
 		if (width == MMC_BUS_WIDTH_4)
 			ctrl |= SDHCI_CTRL_4BITBUS;
@@ -1641,19 +1651,20 @@ void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 
 	ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL);
 
-	if ((ios->timing == MMC_TIMING_SD_HS ||
-	     ios->timing == MMC_TIMING_MMC_HS ||
-	     ios->timing == MMC_TIMING_MMC_HS400 ||
-	     ios->timing == MMC_TIMING_MMC_HS200 ||
-	     ios->timing == MMC_TIMING_MMC_DDR52 ||
-	     ios->timing == MMC_TIMING_UHS_SDR50 ||
-	     ios->timing == MMC_TIMING_UHS_SDR104 ||
-	     ios->timing == MMC_TIMING_UHS_DDR50 ||
-	     ios->timing == MMC_TIMING_UHS_SDR25)
-	    && !(host->quirks & SDHCI_QUIRK_NO_HISPD_BIT))
-		ctrl |= SDHCI_CTRL_HISPD;
-	else
-		ctrl &= ~SDHCI_CTRL_HISPD;
+	if (!(host->quirks & SDHCI_QUIRK_NO_HISPD_BIT)) {
+		if (ios->timing == MMC_TIMING_SD_HS ||
+		     ios->timing == MMC_TIMING_MMC_HS ||
+		     ios->timing == MMC_TIMING_MMC_HS400 ||
+		     ios->timing == MMC_TIMING_MMC_HS200 ||
+		     ios->timing == MMC_TIMING_MMC_DDR52 ||
+		     ios->timing == MMC_TIMING_UHS_SDR50 ||
+		     ios->timing == MMC_TIMING_UHS_SDR104 ||
+		     ios->timing == MMC_TIMING_UHS_DDR50 ||
+		     ios->timing == MMC_TIMING_UHS_SDR25)
+			ctrl |= SDHCI_CTRL_HISPD;
+		else
+			ctrl &= ~SDHCI_CTRL_HISPD;
+	}
 
 	if (host->version >= SDHCI_SPEC_300) {
 		u16 clk, ctrl_2;
@@ -2037,6 +2048,7 @@ static void sdhci_send_tuning(struct sdhci_host *host, u32 opcode)
 	struct mmc_command cmd = {};
 	struct mmc_request mrq = {};
 	unsigned long flags;
+	u32 b = host->sdma_boundary;
 
 	spin_lock_irqsave(&host->lock, flags);
 
@@ -2052,9 +2064,9 @@ static void sdhci_send_tuning(struct sdhci_host *host, u32 opcode)
 	 */
 	if (cmd.opcode == MMC_SEND_TUNING_BLOCK_HS200 &&
 	    mmc->ios.bus_width == MMC_BUS_WIDTH_8)
-		sdhci_writew(host, SDHCI_MAKE_BLKSZ(7, 128), SDHCI_BLOCK_SIZE);
+		sdhci_writew(host, SDHCI_MAKE_BLKSZ(b, 128), SDHCI_BLOCK_SIZE);
 	else
-		sdhci_writew(host, SDHCI_MAKE_BLKSZ(7, 64), SDHCI_BLOCK_SIZE);
+		sdhci_writew(host, SDHCI_MAKE_BLKSZ(b, 64), SDHCI_BLOCK_SIZE);
 
 	/*
 	 * The tuning block is sent by the card to the host controller.
@@ -2502,7 +2514,6 @@ static void sdhci_cmd_irq(struct sdhci_host *host, u32 intmask)
 		sdhci_finish_command(host);
 }
 
-#ifdef CONFIG_MMC_DEBUG
 static void sdhci_adma_show_error(struct sdhci_host *host)
 {
 	void *desc = host->adma_table;
@@ -2530,9 +2541,6 @@ static void sdhci_adma_show_error(struct sdhci_host *host)
 			break;
 	}
 }
-#else
-static void sdhci_adma_show_error(struct sdhci_host *host) { }
-#endif
 
 static void sdhci_data_irq(struct sdhci_host *host, u32 intmask)
 {
@@ -2938,7 +2946,8 @@ int sdhci_runtime_resume_host(struct sdhci_host *host)
 
 	sdhci_init(host, 0);
 
-	if (mmc->ios.power_mode != MMC_POWER_UNDEFINED) {
+	if (mmc->ios.power_mode != MMC_POWER_UNDEFINED &&
+	    mmc->ios.power_mode != MMC_POWER_OFF) {
 		/* Force clock and power re-program */
 		host->pwr = 0;
 		host->clock = 0;
@@ -2998,7 +3007,7 @@ void sdhci_cqe_enable(struct mmc_host *mmc)
 		ctrl |= SDHCI_CTRL_ADMA32;
 	sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
 
-	sdhci_writew(host, SDHCI_MAKE_BLKSZ(SDHCI_DEFAULT_BOUNDARY_ARG, 512),
+	sdhci_writew(host, SDHCI_MAKE_BLKSZ(host->sdma_boundary, 512),
 		     SDHCI_BLOCK_SIZE);
 
 	/* Set maximum timeout */
@@ -3119,6 +3128,8 @@ struct sdhci_host *sdhci_alloc_host(struct device *dev,
 
 	host->tuning_delay = -1;
 
+	host->sdma_boundary = SDHCI_DEFAULT_BOUNDARY_ARG;
+
 	return host;
 }
 
@@ -3230,6 +3241,13 @@ int sdhci_setup_host(struct sdhci_host *host)
 	if (ret == -EPROBE_DEFER)
 		return ret;
 
+	DBG("Version:   0x%08x | Present:  0x%08x\n",
+	    sdhci_readw(host, SDHCI_HOST_VERSION),
+	    sdhci_readl(host, SDHCI_PRESENT_STATE));
+	DBG("Caps:      0x%08x | Caps_1:   0x%08x\n",
+	    sdhci_readl(host, SDHCI_CAPABILITIES),
+	    sdhci_readl(host, SDHCI_CAPABILITIES_1));
+
 	sdhci_read_caps(host);
 
 	override_timeout_clk = host->timeout_clk;
@@ -3747,10 +3765,6 @@ int __sdhci_add_host(struct sdhci_host *host)
 		goto untasklet;
 	}
 
-#ifdef CONFIG_MMC_DEBUG
-	sdhci_dumpregs(host);
-#endif
-
 	ret = sdhci_led_register(host);
 	if (ret) {
 		pr_err("%s: Failed to register LED device: %d\n",
diff --git a/drivers/mmc/host/sdhci.h b/drivers/mmc/host/sdhci.h
index 0469fa191493..54bc444c317f 100644
--- a/drivers/mmc/host/sdhci.h
+++ b/drivers/mmc/host/sdhci.h
@@ -435,6 +435,8 @@ struct sdhci_host {
 #define SDHCI_QUIRK2_ACMD23_BROKEN			(1<<14)
 /* Broken Clock divider zero in controller */
 #define SDHCI_QUIRK2_CLOCK_DIV_ZERO_BROKEN		(1<<15)
+/* Controller has CRC in 136 bit Command Response */
+#define SDHCI_QUIRK2_RSP_136_HAS_CRC			(1<<16)
 
 	int irq;		/* Device IRQ */
 	void __iomem *ioaddr;	/* Mapped address */
@@ -541,6 +543,9 @@ struct sdhci_host {
 	/* Delay (ms) between tuning commands */
 	int			tuning_delay;
 
+	/* Host SDMA buffer boundary. */
+	u32			sdma_boundary;
+
 	unsigned long private[0] ____cacheline_aligned;
 };
 
diff --git a/drivers/mmc/host/sdricoh_cs.c b/drivers/mmc/host/sdricoh_cs.c
index 70cb00aa79a0..9e46039282d2 100644
--- a/drivers/mmc/host/sdricoh_cs.c
+++ b/drivers/mmc/host/sdricoh_cs.c
@@ -385,7 +385,7 @@ static int sdricoh_get_ro(struct mmc_host *mmc)
 	return (status & STATUS_CARD_LOCKED);
 }
 
-static struct mmc_host_ops sdricoh_ops = {
+static const struct mmc_host_ops sdricoh_ops = {
 	.request = sdricoh_request,
 	.set_ios = sdricoh_set_ios,
 	.get_ro = sdricoh_get_ro,
diff --git a/drivers/mmc/host/sh_mmcif.c b/drivers/mmc/host/sh_mmcif.c
index 4062d6bef3c8..53fb18bb7bee 100644
--- a/drivers/mmc/host/sh_mmcif.c
+++ b/drivers/mmc/host/sh_mmcif.c
@@ -1079,7 +1079,7 @@ static void sh_mmcif_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 	host->state = STATE_IDLE;
 }
 
-static struct mmc_host_ops sh_mmcif_ops = {
+static const struct mmc_host_ops sh_mmcif_ops = {
 	.request	= sh_mmcif_request,
 	.set_ios	= sh_mmcif_set_ios,
 	.get_cd		= mmc_gpio_get_cd,
diff --git a/drivers/mmc/host/sunxi-mmc.c b/drivers/mmc/host/sunxi-mmc.c
index 0fb4e4c119e1..53c970fe0873 100644
--- a/drivers/mmc/host/sunxi-mmc.c
+++ b/drivers/mmc/host/sunxi-mmc.c
@@ -22,6 +22,7 @@
 #include <linux/err.h>
 
 #include <linux/clk.h>
+#include <linux/clk/sunxi-ng.h>
 #include <linux/gpio.h>
 #include <linux/platform_device.h>
 #include <linux/spinlock.h>
@@ -259,7 +260,11 @@ struct sunxi_mmc_cfg {
 	/* Does DATA0 needs to be masked while the clock is updated */
 	bool mask_data0;
 
+	/* hardware only supports new timing mode */
 	bool needs_new_timings;
+
+	/* hardware can switch between old and new timing modes */
+	bool has_timings_switch;
 };
 
 struct sunxi_mmc_host {
@@ -293,6 +298,9 @@ struct sunxi_mmc_host {
 
 	/* vqmmc */
 	bool		vqmmc_enabled;
+
+	/* timings */
+	bool		use_new_timings;
 };
 
 static int sunxi_mmc_reset_host(struct sunxi_mmc_host *host)
@@ -714,6 +722,11 @@ static int sunxi_mmc_clk_set_phase(struct sunxi_mmc_host *host,
 {
 	int index;
 
+	/* clk controller delays not used under new timings mode */
+	if (host->use_new_timings)
+		return 0;
+
+	/* some old controllers don't support delays */
 	if (!host->cfg->clk_delays)
 		return 0;
 
@@ -747,7 +760,7 @@ static int sunxi_mmc_clk_set_rate(struct sunxi_mmc_host *host,
 {
 	struct mmc_host *mmc = host->mmc;
 	long rate;
-	u32 rval, clock = ios->clock;
+	u32 rval, clock = ios->clock, div = 1;
 	int ret;
 
 	ret = sunxi_mmc_oclk_onoff(host, 0);
@@ -760,10 +773,30 @@ static int sunxi_mmc_clk_set_rate(struct sunxi_mmc_host *host,
 	if (!ios->clock)
 		return 0;
 
-	/* 8 bit DDR requires a higher module clock */
+	/*
+	 * Under the old timing mode, 8 bit DDR requires the module
+	 * clock to be double the card clock. Under the new timing
+	 * mode, all DDR modes require a doubled module clock.
+	 *
+	 * We currently only support the standard MMC DDR52 mode.
+	 * This block should be updated once support for other DDR
+	 * modes is added.
+	 */
 	if (ios->timing == MMC_TIMING_MMC_DDR52 &&
-	    ios->bus_width == MMC_BUS_WIDTH_8)
+	    (host->use_new_timings ||
+	     ios->bus_width == MMC_BUS_WIDTH_8)) {
+		div = 2;
 		clock <<= 1;
+	}
+
+	if (host->use_new_timings && host->cfg->has_timings_switch) {
+		ret = sunxi_ccu_set_mmc_timing_mode(host->clk_mmc, true);
+		if (ret) {
+			dev_err(mmc_dev(mmc),
+				"error setting new timing mode\n");
+			return ret;
+		}
+	}
 
 	rate = clk_round_rate(host->clk_mmc, clock);
 	if (rate < 0) {
@@ -782,24 +815,23 @@ static int sunxi_mmc_clk_set_rate(struct sunxi_mmc_host *host,
 		return ret;
 	}
 
-	/* clear internal divider */
+	/* set internal divider */
 	rval = mmc_readl(host, REG_CLKCR);
 	rval &= ~0xff;
-	/* set internal divider for 8 bit eMMC DDR, so card clock is right */
-	if (ios->timing == MMC_TIMING_MMC_DDR52 &&
-	    ios->bus_width == MMC_BUS_WIDTH_8) {
-		rval |= 1;
-		rate >>= 1;
-	}
+	rval |= div - 1;
 	mmc_writel(host, REG_CLKCR, rval);
 
-	if (host->cfg->needs_new_timings) {
+	/* update card clock rate to account for internal divider */
+	rate /= div;
+
+	if (host->use_new_timings) {
 		/* Don't touch the delay bits */
 		rval = mmc_readl(host, REG_SD_NTSR);
 		rval |= SDXC_2X_TIMING_MODE;
 		mmc_writel(host, REG_SD_NTSR, rval);
 	}
 
+	/* sunxi_mmc_clk_set_phase expects the actual card clock rate */
 	ret = sunxi_mmc_clk_set_phase(host, ios, rate);
 	if (ret)
 		return ret;
@@ -1048,7 +1080,7 @@ static int sunxi_mmc_card_busy(struct mmc_host *mmc)
 	return !!(mmc_readl(host, REG_STAS) & SDXC_CARD_DATA_BUSY);
 }
 
-static struct mmc_host_ops sunxi_mmc_ops = {
+static const struct mmc_host_ops sunxi_mmc_ops = {
 	.request	 = sunxi_mmc_request,
 	.set_ios	 = sunxi_mmc_set_ios,
 	.get_ro		 = mmc_gpio_get_ro,
@@ -1094,6 +1126,13 @@ static const struct sunxi_mmc_cfg sun7i_a20_cfg = {
 	.can_calibrate = false,
 };
 
+static const struct sunxi_mmc_cfg sun8i_a83t_emmc_cfg = {
+	.idma_des_size_bits = 16,
+	.clk_delays = sunxi_mmc_clk_delays,
+	.can_calibrate = false,
+	.has_timings_switch = true,
+};
+
 static const struct sunxi_mmc_cfg sun9i_a80_cfg = {
 	.idma_des_size_bits = 16,
 	.clk_delays = sun9i_mmc_clk_delays,
@@ -1118,6 +1157,7 @@ static const struct of_device_id sunxi_mmc_of_match[] = {
 	{ .compatible = "allwinner,sun4i-a10-mmc", .data = &sun4i_a10_cfg },
 	{ .compatible = "allwinner,sun5i-a13-mmc", .data = &sun5i_a13_cfg },
 	{ .compatible = "allwinner,sun7i-a20-mmc", .data = &sun7i_a20_cfg },
+	{ .compatible = "allwinner,sun8i-a83t-emmc", .data = &sun8i_a83t_emmc_cfg },
 	{ .compatible = "allwinner,sun9i-a80-mmc", .data = &sun9i_a80_cfg },
 	{ .compatible = "allwinner,sun50i-a64-mmc", .data = &sun50i_a64_cfg },
 	{ .compatible = "allwinner,sun50i-a64-emmc", .data = &sun50i_a64_emmc_cfg },
@@ -1172,7 +1212,8 @@ static int sunxi_mmc_resource_request(struct sunxi_mmc_host *host,
 		}
 	}
 
-	host->reset = devm_reset_control_get_optional(&pdev->dev, "ahb");
+	host->reset = devm_reset_control_get_optional_exclusive(&pdev->dev,
+								"ahb");
 	if (PTR_ERR(host->reset) == -EPROBE_DEFER)
 		return PTR_ERR(host->reset);
 
@@ -1201,7 +1242,7 @@ static int sunxi_mmc_resource_request(struct sunxi_mmc_host *host,
 	}
 
 	if (!IS_ERR(host->reset)) {
-		ret = reset_control_deassert(host->reset);
+		ret = reset_control_reset(host->reset);
 		if (ret) {
 			dev_err(&pdev->dev, "reset err %d\n", ret);
 			goto error_disable_clk_sample;
@@ -1262,6 +1303,30 @@ static int sunxi_mmc_probe(struct platform_device *pdev)
 		goto error_free_host;
 	}
 
+	if (host->cfg->has_timings_switch) {
+		/*
+		 * Supports both old and new timing modes.
+		 * Try setting the clk to new timing mode.
+		 */
+		sunxi_ccu_set_mmc_timing_mode(host->clk_mmc, true);
+
+		/* And check the result */
+		ret = sunxi_ccu_get_mmc_timing_mode(host->clk_mmc);
+		if (ret < 0) {
+			/*
+			 * For whatever reason we were not able to get
+			 * the current active mode. Default to old mode.
+			 */
+			dev_warn(&pdev->dev, "MMC clk timing mode unknown\n");
+			host->use_new_timings = false;
+		} else {
+			host->use_new_timings = !!ret;
+		}
+	} else if (host->cfg->needs_new_timings) {
+		/* Supports new timing mode only */
+		host->use_new_timings = true;
+	}
+
 	mmc->ops		= &sunxi_mmc_ops;
 	mmc->max_blk_count	= 8192;
 	mmc->max_blk_size	= 4096;
@@ -1274,7 +1339,7 @@ static int sunxi_mmc_probe(struct platform_device *pdev)
 	mmc->caps	       |= MMC_CAP_MMC_HIGHSPEED | MMC_CAP_SD_HIGHSPEED |
 				  MMC_CAP_ERASE | MMC_CAP_SDIO_IRQ;
 
-	if (host->cfg->clk_delays)
+	if (host->cfg->clk_delays || host->use_new_timings)
 		mmc->caps      |= MMC_CAP_1_8V_DDR;
 
 	ret = mmc_of_parse(mmc);
diff --git a/drivers/mmc/host/tmio_mmc.h b/drivers/mmc/host/tmio_mmc.h
index 6ad6704175dc..3e6ff8921440 100644
--- a/drivers/mmc/host/tmio_mmc.h
+++ b/drivers/mmc/host/tmio_mmc.h
@@ -81,14 +81,14 @@
 #define TMIO_STAT_CMD_BUSY      BIT(30)
 #define TMIO_STAT_ILL_ACCESS    BIT(31)
 
+/* Definitions for values the CTL_SD_CARD_CLK_CTL register can take */
 #define	CLK_CTL_DIV_MASK	0xff
 #define	CLK_CTL_SCLKEN		BIT(8)
 
+/* Definitions for values the CTL_SD_MEM_CARD_OPT register can take */
 #define CARD_OPT_WIDTH8		BIT(13)
 #define CARD_OPT_WIDTH		BIT(15)
 
-#define TMIO_BBS		512		/* Boot block size */
-
 /* Definitions for values the CTL_SDIO_STATUS register can take */
 #define TMIO_SDIO_STAT_IOIRQ	0x0001
 #define TMIO_SDIO_STAT_EXPUB52	0x4000
@@ -97,6 +97,9 @@
 
 #define TMIO_SDIO_SETBITS_MASK	0x0006
 
+/* Definitions for values the CTL_DMA_ENABLE register can take */
+#define DMA_ENABLE_DMASDRW	BIT(1)
+
 /* Define some IRQ masks */
 /* This is the mask used at reset by the chip */
 #define TMIO_MASK_ALL           0x837f031d
@@ -122,6 +125,7 @@ struct tmio_mmc_dma_ops {
 			struct tmio_mmc_data *pdata);
 	void (*release)(struct tmio_mmc_host *host);
 	void (*abort)(struct tmio_mmc_host *host);
+	void (*dataend)(struct tmio_mmc_host *host);
 };
 
 struct tmio_mmc_host {
@@ -151,6 +155,7 @@ struct tmio_mmc_host {
 	struct dma_chan		*chan_rx;
 	struct dma_chan		*chan_tx;
 	struct completion	dma_dataend;
+	struct tasklet_struct	dma_complete;
 	struct tasklet_struct	dma_issue;
 	struct scatterlist	bounce_sg;
 	u8			*bounce_buf;
diff --git a/drivers/mmc/host/tmio_mmc_core.c b/drivers/mmc/host/tmio_mmc_core.c
index 88a94355ac90..12cf8288d663 100644
--- a/drivers/mmc/host/tmio_mmc_core.c
+++ b/drivers/mmc/host/tmio_mmc_core.c
@@ -87,6 +87,12 @@ static inline void tmio_mmc_abort_dma(struct tmio_mmc_host *host)
 		host->dma_ops->abort(host);
 }
 
+static inline void tmio_mmc_dataend_dma(struct tmio_mmc_host *host)
+{
+	if (host->dma_ops)
+		host->dma_ops->dataend(host);
+}
+
 void tmio_mmc_enable_mmc_irqs(struct tmio_mmc_host *host, u32 i)
 {
 	host->sdcard_irq_mask &= ~(i & TMIO_MASK_IRQ);
@@ -201,7 +207,10 @@ static void tmio_mmc_clk_start(struct tmio_mmc_host *host)
 {
 	sd_ctrl_write16(host, CTL_SD_CARD_CLK_CTL, CLK_CTL_SCLKEN |
 		sd_ctrl_read16(host, CTL_SD_CARD_CLK_CTL));
-	msleep(host->pdata->flags & TMIO_MMC_MIN_RCAR2 ? 1 : 10);
+
+	/* HW engineers overrode docs: no sleep needed on R-Car2+ */
+	if (!(host->pdata->flags & TMIO_MMC_MIN_RCAR2))
+		msleep(10);
 
 	if (host->pdata->flags & TMIO_MMC_HAVE_HIGH_REG) {
 		sd_ctrl_write16(host, CTL_CLK_AND_WAIT_CTL, 0x0100);
@@ -218,7 +227,10 @@ static void tmio_mmc_clk_stop(struct tmio_mmc_host *host)
 
 	sd_ctrl_write16(host, CTL_SD_CARD_CLK_CTL, ~CLK_CTL_SCLKEN &
 		sd_ctrl_read16(host, CTL_SD_CARD_CLK_CTL));
-	msleep(host->pdata->flags & TMIO_MMC_MIN_RCAR2 ? 5 : 10);
+
+	/* HW engineers overrode docs: no sleep needed on R-Car2+ */
+	if (!(host->pdata->flags & TMIO_MMC_MIN_RCAR2))
+		msleep(10);
 }
 
 static void tmio_mmc_set_clock(struct tmio_mmc_host *host,
@@ -343,12 +355,6 @@ static int tmio_mmc_start_command(struct tmio_mmc_host *host,
 	int c = cmd->opcode;
 	u32 irq_mask = TMIO_MASK_CMD;
 
-	/* CMD12 is handled by hardware */
-	if (cmd->opcode == MMC_STOP_TRANSMISSION && !cmd->arg) {
-		sd_ctrl_write16(host, CTL_STOP_INTERNAL_ACTION, TMIO_STOP_STP);
-		return 0;
-	}
-
 	switch (mmc_resp_type(cmd)) {
 	case MMC_RSP_NONE: c |= RESP_NONE; break;
 	case MMC_RSP_R1:
@@ -605,11 +611,11 @@ static void tmio_mmc_data_irq(struct tmio_mmc_host *host, unsigned int stat)
 
 		if (done) {
 			tmio_mmc_disable_mmc_irqs(host, TMIO_STAT_DATAEND);
-			complete(&host->dma_dataend);
+			tmio_mmc_dataend_dma(host);
 		}
 	} else if (host->chan_rx && (data->flags & MMC_DATA_READ) && !host->force_pio) {
 		tmio_mmc_disable_mmc_irqs(host, TMIO_STAT_DATAEND);
-		complete(&host->dma_dataend);
+		tmio_mmc_dataend_dma(host);
 	} else {
 		tmio_mmc_do_data_irq(host);
 		tmio_mmc_disable_mmc_irqs(host, TMIO_MASK_READOP | TMIO_MASK_WRITEOP);
@@ -1251,10 +1257,10 @@ int tmio_mmc_host_probe(struct tmio_mmc_host *_host,
 
 	mmc->caps |= MMC_CAP_4_BIT_DATA | pdata->capabilities;
 	mmc->caps2 |= pdata->capabilities2;
-	mmc->max_segs = 32;
+	mmc->max_segs = pdata->max_segs ? : 32;
 	mmc->max_blk_size = 512;
-	mmc->max_blk_count = (PAGE_SIZE / mmc->max_blk_size) *
-		mmc->max_segs;
+	mmc->max_blk_count = pdata->max_blk_count ? :
+		(PAGE_SIZE / mmc->max_blk_size) * mmc->max_segs;
 	mmc->max_req_size = mmc->max_blk_size * mmc->max_blk_count;
 	mmc->max_seg_size = mmc->max_req_size;
 
diff --git a/drivers/mmc/host/toshsd.c b/drivers/mmc/host/toshsd.c
index 553ef41bb806..dd961c54a6a9 100644
--- a/drivers/mmc/host/toshsd.c
+++ b/drivers/mmc/host/toshsd.c
@@ -550,7 +550,7 @@ static int toshsd_get_cd(struct mmc_host *mmc)
 	return !!(ioread16(host->ioaddr + SD_CARDSTATUS) & SD_CARD_PRESENT_0);
 }
 
-static struct mmc_host_ops toshsd_ops = {
+static const struct mmc_host_ops toshsd_ops = {
 	.request = toshsd_request,
 	.set_ios = toshsd_set_ios,
 	.get_ro = toshsd_get_ro,
diff --git a/drivers/mmc/host/usdhi6rol0.c b/drivers/mmc/host/usdhi6rol0.c
index 1bd5f1a18d4e..64da6a88cfb9 100644
--- a/drivers/mmc/host/usdhi6rol0.c
+++ b/drivers/mmc/host/usdhi6rol0.c
@@ -1185,7 +1185,7 @@ static int usdhi6_sig_volt_switch(struct mmc_host *mmc, struct mmc_ios *ios)
 	return ret;
 }
 
-static struct mmc_host_ops usdhi6_ops = {
+static const struct mmc_host_ops usdhi6_ops = {
 	.request	= usdhi6_request,
 	.set_ios	= usdhi6_set_ios,
 	.get_cd		= usdhi6_get_cd,
diff --git a/drivers/mmc/host/via-sdmmc.c b/drivers/mmc/host/via-sdmmc.c
index 6380044c0628..a838bf5480d8 100644
--- a/drivers/mmc/host/via-sdmmc.c
+++ b/drivers/mmc/host/via-sdmmc.c
@@ -323,7 +323,7 @@ struct via_crdr_mmc_host {
 /* some devices need a very long delay for power to stabilize */
 #define VIA_CRDR_QUIRK_300MS_PWRDELAY	0x0001
 
-static struct pci_device_id via_ids[] = {
+static const struct pci_device_id via_ids[] = {
 	{PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_9530,
 	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0,},
 	{0,}
diff --git a/drivers/mmc/host/vub300.c b/drivers/mmc/host/vub300.c
index fbeea1a491a6..8f569d257405 100644
--- a/drivers/mmc/host/vub300.c
+++ b/drivers/mmc/host/vub300.c
@@ -266,7 +266,7 @@ MODULE_PARM_DESC(firmware_rom_wait_states,
 #define ELAN_VENDOR_ID		0x2201
 #define VUB300_VENDOR_ID	0x0424
 #define VUB300_PRODUCT_ID	0x012C
-static struct usb_device_id vub300_table[] = {
+static const struct usb_device_id vub300_table[] = {
 	{USB_DEVICE(ELAN_VENDOR_ID, VUB300_PRODUCT_ID)},
 	{USB_DEVICE(VUB300_VENDOR_ID, VUB300_PRODUCT_ID)},
 	{} /* Terminating entry */
@@ -2079,7 +2079,7 @@ static void vub300_init_card(struct mmc_host *mmc, struct mmc_card *card)
 	dev_info(&vub300->udev->dev, "NO host QUIRKS for this card\n");
 }
 
-static struct mmc_host_ops vub300_mmc_ops = {
+static const struct mmc_host_ops vub300_mmc_ops = {
 	.request = vub300_mmc_request,
 	.set_ios = vub300_mmc_set_ios,
 	.get_ro = vub300_mmc_get_ro,
diff --git a/drivers/mmc/host/wbsd.c b/drivers/mmc/host/wbsd.c
index 9668616faf16..546aaf8d1507 100644
--- a/drivers/mmc/host/wbsd.c
+++ b/drivers/mmc/host/wbsd.c
@@ -802,10 +802,8 @@ static void wbsd_request(struct mmc_host *mmc, struct mmc_request *mrq)
 			break;
 
 		default:
-#ifdef CONFIG_MMC_DEBUG
 			pr_warn("%s: Data command %d is not supported by this controller\n",
 				mmc_hostname(host->mmc), cmd->opcode);
-#endif
 			cmd->error = -EINVAL;
 
 			goto done;
diff --git a/drivers/mmc/host/wmt-sdmmc.c b/drivers/mmc/host/wmt-sdmmc.c
index 21ebba88679c..fd30ac7da5e5 100644
--- a/drivers/mmc/host/wmt-sdmmc.c
+++ b/drivers/mmc/host/wmt-sdmmc.c
@@ -726,7 +726,7 @@ static int wmt_mci_get_cd(struct mmc_host *mmc)
 	return !(cd ^ priv->cd_inverted);
 }
 
-static struct mmc_host_ops wmt_mci_ops = {
+static const struct mmc_host_ops wmt_mci_ops = {
 	.request = wmt_mci_request,
 	.set_ios = wmt_mci_set_ios,
 	.get_ro = wmt_mci_get_ro,
@@ -856,7 +856,9 @@ static int wmt_mci_probe(struct platform_device *pdev)
 		goto fail5;
 	}
 
-	clk_prepare_enable(priv->clk_sdmmc);
+	ret = clk_prepare_enable(priv->clk_sdmmc);
+	if (ret)
+		goto fail6;
 
 	/* configure the controller to a known 'ready' state */
 	wmt_reset_hardware(mmc);
@@ -866,6 +868,8 @@ static int wmt_mci_probe(struct platform_device *pdev)
 	dev_info(&pdev->dev, "WMT SDHC Controller initialized\n");
 
 	return 0;
+fail6:
+	clk_put(priv->clk_sdmmc);
 fail5:
 	free_irq(dma_irq, priv);
 fail4:
diff --git a/drivers/mtd/devices/docg3.c b/drivers/mtd/devices/docg3.c
index b833e6cc684c..84b16133554b 100644
--- a/drivers/mtd/devices/docg3.c
+++ b/drivers/mtd/devices/docg3.c
@@ -1809,37 +1809,22 @@ static int dbg_protection_show(struct seq_file *s, void *p)
 }
 DEBUGFS_RO_ATTR(protection, dbg_protection_show);
 
-static int __init doc_dbg_register(struct docg3 *docg3)
-{
-	struct dentry *root, *entry;
-
-	root = debugfs_create_dir("docg3", NULL);
-	if (!root)
-		return -ENOMEM;
-
-	entry = debugfs_create_file("flashcontrol", S_IRUSR, root, docg3,
-				  &flashcontrol_fops);
-	if (entry)
-		entry = debugfs_create_file("asic_mode", S_IRUSR, root,
-					    docg3, &asic_mode_fops);
-	if (entry)
-		entry = debugfs_create_file("device_id", S_IRUSR, root,
-					    docg3, &device_id_fops);
-	if (entry)
-		entry = debugfs_create_file("protection", S_IRUSR, root,
-					    docg3, &protection_fops);
-	if (entry) {
-		docg3->debugfs_root = root;
-		return 0;
-	} else {
-		debugfs_remove_recursive(root);
-		return -ENOMEM;
-	}
-}
-
-static void doc_dbg_unregister(struct docg3 *docg3)
+static void __init doc_dbg_register(struct mtd_info *floor)
 {
-	debugfs_remove_recursive(docg3->debugfs_root);
+	struct dentry *root = floor->dbg.dfs_dir;
+	struct docg3 *docg3 = floor->priv;
+
+	if (IS_ERR_OR_NULL(root))
+		return;
+
+	debugfs_create_file("docg3_flashcontrol", S_IRUSR, root, docg3,
+			    &flashcontrol_fops);
+	debugfs_create_file("docg3_asic_mode", S_IRUSR, root, docg3,
+			    &asic_mode_fops);
+	debugfs_create_file("docg3_device_id", S_IRUSR, root, docg3,
+			    &device_id_fops);
+	debugfs_create_file("docg3_protection", S_IRUSR, root, docg3,
+			    &protection_fops);
 }
 
 /**
@@ -2114,6 +2099,8 @@ static int __init docg3_probe(struct platform_device *pdev)
 						0);
 		if (ret)
 			goto err_probe;
+
+		doc_dbg_register(cascade->floors[floor]);
 	}
 
 	ret = doc_register_sysfs(pdev, cascade);
@@ -2121,7 +2108,6 @@ static int __init docg3_probe(struct platform_device *pdev)
 		goto err_probe;
 
 	platform_set_drvdata(pdev, cascade);
-	doc_dbg_register(cascade->floors[0]->priv);
 	return 0;
 
 notfound:
@@ -2148,7 +2134,6 @@ static int docg3_release(struct platform_device *pdev)
 	int floor;
 
 	doc_unregister_sysfs(pdev, cascade);
-	doc_dbg_unregister(docg3);
 	for (floor = 0; floor < DOC_MAX_NBFLOORS; floor++)
 		if (cascade->floors[floor])
 			doc_release_device(cascade->floors[floor]);
diff --git a/drivers/mtd/devices/docg3.h b/drivers/mtd/devices/docg3.h
index 19fb93f96a3a..e99946575398 100644
--- a/drivers/mtd/devices/docg3.h
+++ b/drivers/mtd/devices/docg3.h
@@ -299,7 +299,6 @@ struct docg3_cascade {
  * @oob_autoecc: if 1, use only bytes 0-7, 15, and fill the others with HW ECC
  *               if 0, use all the 16 bytes.
  * @oob_write_buf: prepared OOB for next page_write
- * @debugfs_root: debugfs root node
  */
 struct docg3 {
 	struct device *dev;
@@ -312,7 +311,6 @@ struct docg3 {
 	loff_t oob_write_ofs;
 	int oob_autoecc;
 	u8 oob_write_buf[DOC_LAYOUT_OOB_SIZE];
-	struct dentry *debugfs_root;
 };
 
 #define doc_err(fmt, arg...) dev_err(docg3->dev, (fmt), ## arg)
diff --git a/drivers/mtd/devices/spear_smi.c b/drivers/mtd/devices/spear_smi.c
index dd5069876537..ddf478976013 100644
--- a/drivers/mtd/devices/spear_smi.c
+++ b/drivers/mtd/devices/spear_smi.c
@@ -775,6 +775,8 @@ static int spear_smi_probe_config_dt(struct platform_device *pdev,
 	pdata->board_flash_info = devm_kzalloc(&pdev->dev,
 					       sizeof(*pdata->board_flash_info),
 					       GFP_KERNEL);
+	if (!pdata->board_flash_info)
+		return -ENOMEM;
 
 	/* Fill structs for each subnode (flash device) */
 	while ((pp = of_get_next_child(np, pp))) {
diff --git a/drivers/mtd/devices/st_spi_fsm.c b/drivers/mtd/devices/st_spi_fsm.c
index 21afd94cd904..7bc29d725200 100644
--- a/drivers/mtd/devices/st_spi_fsm.c
+++ b/drivers/mtd/devices/st_spi_fsm.c
@@ -2073,15 +2073,17 @@ static int stfsm_probe(struct platform_device *pdev)
 	ret = stfsm_init(fsm);
 	if (ret) {
 		dev_err(&pdev->dev, "Failed to initialise FSM Controller\n");
-		return ret;
+		goto err_clk_unprepare;
 	}
 
 	stfsm_fetch_platform_configs(pdev);
 
 	/* Detect SPI FLASH device */
 	info = stfsm_jedec_probe(fsm);
-	if (!info)
-		return -ENODEV;
+	if (!info) {
+		ret = -ENODEV;
+		goto err_clk_unprepare;
+	}
 	fsm->info = info;
 
 	/* Use device size to determine address width */
@@ -2095,11 +2097,11 @@ static int stfsm_probe(struct platform_device *pdev)
 	if (info->config) {
 		ret = info->config(fsm);
 		if (ret)
-			return ret;
+			goto err_clk_unprepare;
 	} else {
 		ret = stfsm_prepare_rwe_seqs_default(fsm);
 		if (ret)
-			return ret;
+			goto err_clk_unprepare;
 	}
 
 	fsm->mtd.name		= info->name;
@@ -2124,6 +2126,10 @@ static int stfsm_probe(struct platform_device *pdev)
 		fsm->mtd.erasesize, (fsm->mtd.erasesize >> 10));
 
 	return mtd_device_register(&fsm->mtd, NULL, 0);
+
+err_clk_unprepare:
+	clk_disable_unprepare(fsm->clk);
+	return ret;
 }
 
 static int stfsm_remove(struct platform_device *pdev)
@@ -2147,9 +2153,7 @@ static int stfsmfsm_resume(struct device *dev)
 {
 	struct stfsm *fsm = dev_get_drvdata(dev);
 
-	clk_prepare_enable(fsm->clk);
-
-	return 0;
+	return clk_prepare_enable(fsm->clk);
 }
 #endif
 
diff --git a/drivers/mtd/inftlcore.c b/drivers/mtd/inftlcore.c
index 8db740d6eb08..57ef1fb42a04 100644
--- a/drivers/mtd/inftlcore.c
+++ b/drivers/mtd/inftlcore.c
@@ -33,7 +33,7 @@
 #include <linux/mtd/mtd.h>
 #include <linux/mtd/nftl.h>
 #include <linux/mtd/inftl.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/uaccess.h>
 #include <asm/errno.h>
 #include <asm/io.h>
diff --git a/drivers/mtd/maps/amd76xrom.c b/drivers/mtd/maps/amd76xrom.c
index f2b68667ea59..26de0a1d08cf 100644
--- a/drivers/mtd/maps/amd76xrom.c
+++ b/drivers/mtd/maps/amd76xrom.c
@@ -296,7 +296,7 @@ static void amd76xrom_remove_one(struct pci_dev *pdev)
 	amd76xrom_cleanup(window);
 }
 
-static struct pci_device_id amd76xrom_pci_tbl[] = {
+static const struct pci_device_id amd76xrom_pci_tbl[] = {
 	{ PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_VIPER_7410,
 		PCI_ANY_ID, PCI_ANY_ID, },
 	{ PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_VIPER_7440,
@@ -319,7 +319,7 @@ static struct pci_driver amd76xrom_driver = {
 static int __init init_amd76xrom(void)
 {
 	struct pci_dev *pdev;
-	struct pci_device_id *id;
+	const struct pci_device_id *id;
 	pdev = NULL;
 	for(id = amd76xrom_pci_tbl; id->vendor; id++) {
 		pdev = pci_get_device(id->vendor, id->device, NULL);
diff --git a/drivers/mtd/maps/ck804xrom.c b/drivers/mtd/maps/ck804xrom.c
index 4f206a99164c..584962ec49f8 100644
--- a/drivers/mtd/maps/ck804xrom.c
+++ b/drivers/mtd/maps/ck804xrom.c
@@ -326,7 +326,7 @@ static void ck804xrom_remove_one(struct pci_dev *pdev)
 	ck804xrom_cleanup(window);
 }
 
-static struct pci_device_id ck804xrom_pci_tbl[] = {
+static const struct pci_device_id ck804xrom_pci_tbl[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, 0x0051), .driver_data = DEV_CK804 },
 	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, 0x0360), .driver_data = DEV_MCP55 },
 	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, 0x0361), .driver_data = DEV_MCP55 },
@@ -353,7 +353,7 @@ static struct pci_driver ck804xrom_driver = {
 static int __init init_ck804xrom(void)
 {
 	struct pci_dev *pdev;
-	struct pci_device_id *id;
+	const struct pci_device_id *id;
 	int retVal;
 	pdev = NULL;
 
diff --git a/drivers/mtd/maps/esb2rom.c b/drivers/mtd/maps/esb2rom.c
index 9646b0766ce0..da9f6d76ce1d 100644
--- a/drivers/mtd/maps/esb2rom.c
+++ b/drivers/mtd/maps/esb2rom.c
@@ -384,7 +384,7 @@ static void esb2rom_remove_one(struct pci_dev *pdev)
 	esb2rom_cleanup(window);
 }
 
-static struct pci_device_id esb2rom_pci_tbl[] = {
+static const struct pci_device_id esb2rom_pci_tbl[] = {
 	{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801BA_0,
 	  PCI_ANY_ID, PCI_ANY_ID, },
 	{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801CA_0,
@@ -414,7 +414,7 @@ static struct pci_driver esb2rom_driver = {
 static int __init init_esb2rom(void)
 {
 	struct pci_dev *pdev;
-	struct pci_device_id *id;
+	const struct pci_device_id *id;
 	int retVal;
 
 	pdev = NULL;
diff --git a/drivers/mtd/maps/ichxrom.c b/drivers/mtd/maps/ichxrom.c
index 976d42f63aef..1888c5bf13f8 100644
--- a/drivers/mtd/maps/ichxrom.c
+++ b/drivers/mtd/maps/ichxrom.c
@@ -323,7 +323,7 @@ static void ichxrom_remove_one(struct pci_dev *pdev)
 	ichxrom_cleanup(window);
 }
 
-static struct pci_device_id ichxrom_pci_tbl[] = {
+static const struct pci_device_id ichxrom_pci_tbl[] = {
 	{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801BA_0,
 	  PCI_ANY_ID, PCI_ANY_ID, },
 	{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801CA_0,
@@ -351,7 +351,7 @@ static struct pci_driver ichxrom_driver = {
 static int __init init_ichxrom(void)
 {
 	struct pci_dev *pdev;
-	struct pci_device_id *id;
+	const struct pci_device_id *id;
 
 	pdev = NULL;
 	for (id = ichxrom_pci_tbl; id->vendor; id++) {
diff --git a/drivers/mtd/maps/intel_vr_nor.c b/drivers/mtd/maps/intel_vr_nor.c
index 8bf79775e7c1..dd5d6855f543 100644
--- a/drivers/mtd/maps/intel_vr_nor.c
+++ b/drivers/mtd/maps/intel_vr_nor.c
@@ -170,7 +170,7 @@ static int vr_nor_init_maps(struct vr_nor_mtd *p)
 	return err;
 }
 
-static struct pci_device_id vr_nor_pci_ids[] = {
+static const struct pci_device_id vr_nor_pci_ids[] = {
 	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x500D)},
 	{0,}
 };
diff --git a/drivers/mtd/maps/lantiq-flash.c b/drivers/mtd/maps/lantiq-flash.c
index 3e33ab66eb24..77b1d8013295 100644
--- a/drivers/mtd/maps/lantiq-flash.c
+++ b/drivers/mtd/maps/lantiq-flash.c
@@ -114,12 +114,6 @@ ltq_mtd_probe(struct platform_device *pdev)
 	struct cfi_private *cfi;
 	int err;
 
-	if (of_machine_is_compatible("lantiq,falcon") &&
-			(ltq_boot_select() != BS_FLASH)) {
-		dev_err(&pdev->dev, "invalid bootstrap options\n");
-		return -ENODEV;
-	}
-
 	ltq_mtd = devm_kzalloc(&pdev->dev, sizeof(struct ltq_mtd), GFP_KERNEL);
 	if (!ltq_mtd)
 		return -ENOMEM;
diff --git a/drivers/mtd/maps/pci.c b/drivers/mtd/maps/pci.c
index eb0242e0b2d9..7b3bb40aff72 100644
--- a/drivers/mtd/maps/pci.c
+++ b/drivers/mtd/maps/pci.c
@@ -228,7 +228,7 @@ static struct mtd_pci_info intel_dc21285_info = {
  * PCI device ID table
  */
 
-static struct pci_device_id mtd_pci_ids[] = {
+static const struct pci_device_id mtd_pci_ids[] = {
 	{
 		.vendor =	PCI_VENDOR_ID_INTEL,
 		.device =	0x530d,
diff --git a/drivers/mtd/maps/physmap_of_core.c b/drivers/mtd/maps/physmap_of_core.c
index 62fa6836f218..b1bd4faecfb2 100644
--- a/drivers/mtd/maps/physmap_of_core.c
+++ b/drivers/mtd/maps/physmap_of_core.c
@@ -178,8 +178,8 @@ static int of_flash_probe(struct platform_device *dev)
 	 */
 	p = of_get_property(dp, "reg", &count);
 	if (!p || count % reg_tuple_size != 0) {
-		dev_err(&dev->dev, "Malformed reg property on %s\n",
-				dev->dev.of_node->full_name);
+		dev_err(&dev->dev, "Malformed reg property on %pOF\n",
+				dev->dev.of_node);
 		err = -EINVAL;
 		goto err_flash_remove;
 	}
@@ -235,10 +235,10 @@ static int of_flash_probe(struct platform_device *dev)
 
 		err = of_flash_probe_gemini(dev, dp, &info->list[i].map);
 		if (err)
-			return err;
+			goto err_out;
 		err = of_flash_probe_versatile(dev, dp, &info->list[i].map);
 		if (err)
-			return err;
+			goto err_out;
 
 		err = -ENOMEM;
 		info->list[i].map.virt = ioremap(info->list[i].map.phys,
diff --git a/drivers/mtd/maps/physmap_of_gemini.c b/drivers/mtd/maps/physmap_of_gemini.c
index 05b286b5289f..4ed1a6bb4d3c 100644
--- a/drivers/mtd/maps/physmap_of_gemini.c
+++ b/drivers/mtd/maps/physmap_of_gemini.c
@@ -43,13 +43,6 @@
 
 #define FLASH_PARALLEL_HIGH_PIN_CNT	(1 << 20)	/* else low pin cnt */
 
-/* Miscellaneous Control Register */
-#define GLOBAL_MISC_CTRL		0x30
-#define FLASH_PADS_MASK			0x07
-#define NAND_PADS_DISABLE		BIT(2)
-#define PFLASH_PADS_DISABLE		BIT(1)
-#define SFLASH_PADS_DISABLE		BIT(0)
-
 static const struct of_device_id syscon_match[] = {
 	{ .compatible = "cortina,gemini-syscon" },
 	{ },
@@ -102,15 +95,6 @@ int of_flash_probe_gemini(struct platform_device *pdev,
 				 map->bankwidth * 8);
 	}
 
-	/* Activate parallel (NOR flash) mode */
-	ret = regmap_update_bits(rmap, GLOBAL_MISC_CTRL,
-				 FLASH_PADS_MASK,
-				 SFLASH_PADS_DISABLE | NAND_PADS_DISABLE);
-	if (ret) {
-		dev_err(dev, "unable to set up physmap pads\n");
-		return -ENODEV;
-	}
-
 	dev_info(&pdev->dev, "initialized Gemini-specific physmap control\n");
 
 	return 0;
diff --git a/drivers/mtd/maps/physmap_of_versatile.c b/drivers/mtd/maps/physmap_of_versatile.c
index 8c6ccded9be8..03f2b6e7bc7e 100644
--- a/drivers/mtd/maps/physmap_of_versatile.c
+++ b/drivers/mtd/maps/physmap_of_versatile.c
@@ -97,7 +97,7 @@ static const struct of_device_id ebi_match[] = {
 static int ap_flash_init(struct platform_device *pdev)
 {
 	struct device_node *ebi;
-	static void __iomem *ebi_base;
+	void __iomem *ebi_base;
 	u32 val;
 	int ret;
 
diff --git a/drivers/mtd/maps/sun_uflash.c b/drivers/mtd/maps/sun_uflash.c
index 414956eca0c9..1e73bba6e286 100644
--- a/drivers/mtd/maps/sun_uflash.c
+++ b/drivers/mtd/maps/sun_uflash.c
@@ -55,8 +55,8 @@ int uflash_devinit(struct platform_device *op, struct device_node *dp)
 		/* Non-CFI userflash device-- once I find one we
 		 * can work on supporting it.
 		 */
-		printk(KERN_ERR PFX "Unsupported device at %s, 0x%llx\n",
-		       dp->full_name, (unsigned long long)op->resource[0].start);
+		printk(KERN_ERR PFX "Unsupported device at %pOF, 0x%llx\n",
+		       dp, (unsigned long long)op->resource[0].start);
 
 		return -ENODEV;
 	}
diff --git a/drivers/mtd/mtdcore.c b/drivers/mtd/mtdcore.c
index 956382cea256..e7ea842ba3db 100644
--- a/drivers/mtd/mtdcore.c
+++ b/drivers/mtd/mtdcore.c
@@ -40,6 +40,7 @@
 #include <linux/slab.h>
 #include <linux/reboot.h>
 #include <linux/leds.h>
+#include <linux/debugfs.h>
 
 #include <linux/mtd/mtd.h>
 #include <linux/mtd/partitions.h>
@@ -339,7 +340,7 @@ static struct attribute *mtd_attrs[] = {
 };
 ATTRIBUTE_GROUPS(mtd);
 
-static struct device_type mtd_devtype = {
+static const struct device_type mtd_devtype = {
 	.name		= "mtd",
 	.groups		= mtd_groups,
 	.release	= mtd_release,
@@ -477,6 +478,8 @@ int mtd_pairing_groups(struct mtd_info *mtd)
 }
 EXPORT_SYMBOL_GPL(mtd_pairing_groups);
 
+static struct dentry *dfs_dir_mtd;
+
 /**
  *	add_mtd_device - register an MTD device
  *	@mtd: pointer to new MTD device info structure
@@ -552,6 +555,14 @@ int add_mtd_device(struct mtd_info *mtd)
 	if (error)
 		goto fail_added;
 
+	if (!IS_ERR_OR_NULL(dfs_dir_mtd)) {
+		mtd->dbg.dfs_dir = debugfs_create_dir(dev_name(&mtd->dev), dfs_dir_mtd);
+		if (IS_ERR_OR_NULL(mtd->dbg.dfs_dir)) {
+			pr_debug("mtd device %s won't show data in debugfs\n",
+				 dev_name(&mtd->dev));
+		}
+	}
+
 	device_create(&mtd_class, mtd->dev.parent, MTD_DEVT(i) + 1, NULL,
 		      "mtd%dro", i);
 
@@ -594,6 +605,8 @@ int del_mtd_device(struct mtd_info *mtd)
 
 	mutex_lock(&mtd_table_mutex);
 
+	debugfs_remove_recursive(mtd->dbg.dfs_dir);
+
 	if (idr_find(&mtd_idr, mtd->index) != mtd) {
 		ret = -ENODEV;
 		goto out_error;
@@ -1811,6 +1824,8 @@ static int __init init_mtd(void)
 	if (ret)
 		goto out_procfs;
 
+	dfs_dir_mtd = debugfs_create_dir("mtd", NULL);
+
 	return 0;
 
 out_procfs:
@@ -1826,6 +1841,7 @@ err_reg:
 
 static void __exit cleanup_mtd(void)
 {
+	debugfs_remove_recursive(dfs_dir_mtd);
 	cleanup_mtdchar();
 	if (proc_mtd)
 		remove_proc_entry("mtd", NULL);
diff --git a/drivers/mtd/mtdswap.c b/drivers/mtd/mtdswap.c
index f12879a3d4ff..7d9080e33865 100644
--- a/drivers/mtd/mtdswap.c
+++ b/drivers/mtd/mtdswap.c
@@ -138,8 +138,6 @@ struct mtdswap_dev {
 
 	char *page_buf;
 	char *oob_buf;
-
-	struct dentry *debugfs_root;
 };
 
 struct mtdswap_oobdata {
@@ -1315,29 +1313,19 @@ static const struct file_operations mtdswap_fops = {
 
 static int mtdswap_add_debugfs(struct mtdswap_dev *d)
 {
-	struct gendisk *gd = d->mbd_dev->disk;
-	struct device *dev = disk_to_dev(gd);
-
-	struct dentry *root;
+	struct dentry *root = d->mtd->dbg.dfs_dir;
 	struct dentry *dent;
 
-	root = debugfs_create_dir(gd->disk_name, NULL);
-	if (IS_ERR(root))
+	if (!IS_ENABLED(CONFIG_DEBUG_FS))
 		return 0;
 
-	if (!root) {
-		dev_err(dev, "failed to initialize debugfs\n");
+	if (IS_ERR_OR_NULL(root))
 		return -1;
-	}
-
-	d->debugfs_root = root;
 
-	dent = debugfs_create_file("stats", S_IRUSR, root, d,
+	dent = debugfs_create_file("mtdswap_stats", S_IRUSR, root, d,
 				&mtdswap_fops);
 	if (!dent) {
 		dev_err(d->dev, "debugfs_create_file failed\n");
-		debugfs_remove_recursive(root);
-		d->debugfs_root = NULL;
 		return -1;
 	}
 
@@ -1540,7 +1528,6 @@ static void mtdswap_remove_dev(struct mtd_blktrans_dev *dev)
 {
 	struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
 
-	debugfs_remove_recursive(d->debugfs_root);
 	del_mtd_blktrans_dev(dev);
 	mtdswap_cleanup(d);
 	kfree(d);
diff --git a/drivers/mtd/nand/Kconfig b/drivers/mtd/nand/Kconfig
index dbfa72d61d5a..3f2036f31da4 100644
--- a/drivers/mtd/nand/Kconfig
+++ b/drivers/mtd/nand/Kconfig
@@ -315,7 +315,7 @@ config MTD_NAND_ATMEL
 
 config MTD_NAND_PXA3xx
 	tristate "NAND support on PXA3xx and Armada 370/XP"
-	depends on PXA3xx || ARCH_MMP || PLAT_ORION
+	depends on PXA3xx || ARCH_MMP || PLAT_ORION || ARCH_MVEBU
 	help
 	  This enables the driver for the NAND flash device found on
 	  PXA3xx processors (NFCv1) and also on Armada 370/XP (NFCv2).
diff --git a/drivers/mtd/nand/ams-delta.c b/drivers/mtd/nand/ams-delta.c
index 5d6c26f3cf7f..dcec9cf4983f 100644
--- a/drivers/mtd/nand/ams-delta.c
+++ b/drivers/mtd/nand/ams-delta.c
@@ -20,7 +20,7 @@
 #include <linux/module.h>
 #include <linux/delay.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 #include <linux/gpio.h>
 #include <linux/platform_data/gpio-omap.h>
diff --git a/drivers/mtd/nand/atmel/nand-controller.c b/drivers/mtd/nand/atmel/nand-controller.c
index 2c8baa0c2c4e..f25eca79f4e5 100644
--- a/drivers/mtd/nand/atmel/nand-controller.c
+++ b/drivers/mtd/nand/atmel/nand-controller.c
@@ -59,7 +59,7 @@
 #include <linux/mfd/syscon/atmel-matrix.h>
 #include <linux/mfd/syscon/atmel-smc.h>
 #include <linux/module.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/of_address.h>
 #include <linux/of_irq.h>
 #include <linux/of_platform.h>
@@ -247,6 +247,7 @@ struct atmel_hsmc_nand_controller {
 		void __iomem *virt;
 		dma_addr_t dma;
 	} sram;
+	const struct atmel_hsmc_reg_layout *hsmc_layout;
 	struct regmap *io;
 	struct atmel_nfc_op op;
 	struct completion complete;
@@ -1364,7 +1365,18 @@ static int atmel_smc_nand_prepare_smcconf(struct atmel_nand *nand,
 	ret = atmel_smc_cs_conf_set_timing(smcconf,
 					   ATMEL_HSMC_TIMINGS_TADL_SHIFT,
 					   ncycles);
-	if (ret)
+	/*
+	 * Version 4 of the ONFI spec mandates that tADL be at least 400
+	 * nanoseconds, but, depending on the master clock rate, 400 ns may not
+	 * fit in the tADL field of the SMC reg. We need to relax the check and
+	 * accept the -ERANGE return code.
+	 *
+	 * Note that previous versions of the ONFI spec had a lower tADL_min
+	 * (100 or 200 ns). It's not clear why this timing constraint got
+	 * increased but it seems most NANDs are fine with values lower than
+	 * 400ns, so we should be safe.
+	 */
+	if (ret && ret != -ERANGE)
 		return ret;
 
 	ncycles = DIV_ROUND_UP(conf->timings.sdr.tAR_min, mckperiodps);
@@ -1431,12 +1443,12 @@ static int atmel_hsmc_nand_setup_data_interface(struct atmel_nand *nand,
 					int csline,
 					const struct nand_data_interface *conf)
 {
-	struct atmel_nand_controller *nc;
+	struct atmel_hsmc_nand_controller *nc;
 	struct atmel_smc_cs_conf smcconf;
 	struct atmel_nand_cs *cs;
 	int ret;
 
-	nc = to_nand_controller(nand->base.controller);
+	nc = to_hsmc_nand_controller(nand->base.controller);
 
 	ret = atmel_smc_nand_prepare_smcconf(nand, conf, &smcconf);
 	if (ret)
@@ -1451,7 +1463,8 @@ static int atmel_hsmc_nand_setup_data_interface(struct atmel_nand *nand,
 	if (cs->rb.type == ATMEL_NAND_NATIVE_RB)
 		cs->smcconf.timings |= ATMEL_HSMC_TIMINGS_RBNSEL(cs->rb.id);
 
-	atmel_hsmc_cs_conf_apply(nc->smc, cs->id, &cs->smcconf);
+	atmel_hsmc_cs_conf_apply(nc->base.smc, nc->hsmc_layout, cs->id,
+				 &cs->smcconf);
 
 	return 0;
 }
@@ -2078,8 +2091,8 @@ atmel_hsmc_nand_controller_legacy_init(struct atmel_hsmc_nand_controller *nc)
 	}
 
 	nc->irq = of_irq_get(nand_np, 0);
-	if (nc->irq < 0) {
-		ret = nc->irq;
+	if (nc->irq <= 0) {
+		ret = nc->irq ?: -ENXIO;
 		if (ret != -EPROBE_DEFER)
 			dev_err(dev, "Failed to get IRQ number (err = %d)\n",
 				ret);
@@ -2166,13 +2179,16 @@ atmel_hsmc_nand_controller_init(struct atmel_hsmc_nand_controller *nc)
 		return -EINVAL;
 	}
 
+	nc->hsmc_layout = atmel_hsmc_get_reg_layout(np);
+
 	nc->irq = of_irq_get(np, 0);
 	of_node_put(np);
-	if (nc->irq < 0) {
-		if (nc->irq != -EPROBE_DEFER)
+	if (nc->irq <= 0) {
+		ret = nc->irq ?: -ENXIO;
+		if (ret != -EPROBE_DEFER)
 			dev_err(dev, "Failed to get IRQ number (err = %d)\n",
-				nc->irq);
-		return nc->irq;
+				ret);
+		return ret;
 	}
 
 	np = of_parse_phandle(dev->of_node, "atmel,nfc-io", 0);
diff --git a/drivers/mtd/nand/atmel/pmecc.c b/drivers/mtd/nand/atmel/pmecc.c
index 8c210a5776bc..146af8218314 100644
--- a/drivers/mtd/nand/atmel/pmecc.c
+++ b/drivers/mtd/nand/atmel/pmecc.c
@@ -47,7 +47,7 @@
 #include <linux/genalloc.h>
 #include <linux/iopoll.h>
 #include <linux/module.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/of_irq.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
diff --git a/drivers/mtd/nand/au1550nd.c b/drivers/mtd/nand/au1550nd.c
index 9bf6d9915694..9d4a28fa6b73 100644
--- a/drivers/mtd/nand/au1550nd.c
+++ b/drivers/mtd/nand/au1550nd.c
@@ -14,7 +14,7 @@
 #include <linux/module.h>
 #include <linux/interrupt.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 #include <linux/platform_device.h>
 #include <asm/io.h>
diff --git a/drivers/mtd/nand/bcm47xxnflash/bcm47xxnflash.h b/drivers/mtd/nand/bcm47xxnflash/bcm47xxnflash.h
index 8ea75710a854..c8834767ab6d 100644
--- a/drivers/mtd/nand/bcm47xxnflash/bcm47xxnflash.h
+++ b/drivers/mtd/nand/bcm47xxnflash/bcm47xxnflash.h
@@ -6,7 +6,7 @@
 #endif
 
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 
 struct bcm47xxnflash {
 	struct bcma_drv_cc *cc;
diff --git a/drivers/mtd/nand/bf5xx_nand.c b/drivers/mtd/nand/bf5xx_nand.c
index 3962f55bd034..5655dca6ce43 100644
--- a/drivers/mtd/nand/bf5xx_nand.c
+++ b/drivers/mtd/nand/bf5xx_nand.c
@@ -49,7 +49,7 @@
 #include <linux/bitops.h>
 
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/nand_ecc.h>
 #include <linux/mtd/partitions.h>
 
diff --git a/drivers/mtd/nand/brcmnand/brcmnand.c b/drivers/mtd/nand/brcmnand/brcmnand.c
index 7419c5ce63f8..e0eb51d8c012 100644
--- a/drivers/mtd/nand/brcmnand/brcmnand.c
+++ b/drivers/mtd/nand/brcmnand/brcmnand.c
@@ -29,7 +29,7 @@
 #include <linux/bitops.h>
 #include <linux/mm.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 #include <linux/of.h>
 #include <linux/of_platform.h>
diff --git a/drivers/mtd/nand/cafe_nand.c b/drivers/mtd/nand/cafe_nand.c
index 2fd733eba0a3..bc558c438a57 100644
--- a/drivers/mtd/nand/cafe_nand.c
+++ b/drivers/mtd/nand/cafe_nand.c
@@ -13,7 +13,7 @@
 #include <linux/device.h>
 #undef DEBUG
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 #include <linux/rslib.h>
 #include <linux/pci.h>
diff --git a/drivers/mtd/nand/cmx270_nand.c b/drivers/mtd/nand/cmx270_nand.c
index 949b9400dcb7..1fc435f994e1 100644
--- a/drivers/mtd/nand/cmx270_nand.c
+++ b/drivers/mtd/nand/cmx270_nand.c
@@ -18,7 +18,7 @@
  *   CM-X270 board.
  */
 
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 #include <linux/slab.h>
 #include <linux/gpio.h>
diff --git a/drivers/mtd/nand/cs553x_nand.c b/drivers/mtd/nand/cs553x_nand.c
index 594b28684138..d48877540f14 100644
--- a/drivers/mtd/nand/cs553x_nand.c
+++ b/drivers/mtd/nand/cs553x_nand.c
@@ -24,7 +24,7 @@
 #include <linux/module.h>
 #include <linux/delay.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/nand_ecc.h>
 #include <linux/mtd/partitions.h>
 
diff --git a/drivers/mtd/nand/davinci_nand.c b/drivers/mtd/nand/davinci_nand.c
index 7b26e53b95b1..ccc8c43abcff 100644
--- a/drivers/mtd/nand/davinci_nand.c
+++ b/drivers/mtd/nand/davinci_nand.c
@@ -29,7 +29,7 @@
 #include <linux/err.h>
 #include <linux/clk.h>
 #include <linux/io.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 #include <linux/slab.h>
 #include <linux/of_device.h>
diff --git a/drivers/mtd/nand/denali.c b/drivers/mtd/nand/denali.c
index d723be352148..3087b0ba7b7f 100644
--- a/drivers/mtd/nand/denali.c
+++ b/drivers/mtd/nand/denali.c
@@ -980,9 +980,6 @@ static int denali_erase(struct mtd_info *mtd, int page)
 	return irq_status & INTR__ERASE_COMP ? 0 : NAND_STATUS_FAIL;
 }
 
-#define DIV_ROUND_DOWN_ULL(ll, d) \
-	({ unsigned long long _tmp = (ll); do_div(_tmp, d); _tmp; })
-
 static int denali_setup_data_interface(struct mtd_info *mtd, int chipnr,
 				       const struct nand_data_interface *conf)
 {
diff --git a/drivers/mtd/nand/denali.h b/drivers/mtd/nand/denali.h
index 237cc706b0fb..9239e6793e6e 100644
--- a/drivers/mtd/nand/denali.h
+++ b/drivers/mtd/nand/denali.h
@@ -21,7 +21,7 @@
 #define __DENALI_H__
 
 #include <linux/bitops.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 
 #define DEVICE_RESET				0x0
 #define     DEVICE_RESET__BANK(bank)			BIT(bank)
diff --git a/drivers/mtd/nand/denali_dt.c b/drivers/mtd/nand/denali_dt.c
index 47f398edf18f..56e2e177644d 100644
--- a/drivers/mtd/nand/denali_dt.c
+++ b/drivers/mtd/nand/denali_dt.c
@@ -118,7 +118,9 @@ static int denali_dt_probe(struct platform_device *pdev)
 		dev_err(&pdev->dev, "no clk available\n");
 		return PTR_ERR(dt->clk);
 	}
-	clk_prepare_enable(dt->clk);
+	ret = clk_prepare_enable(dt->clk);
+	if (ret)
+		return ret;
 
 	denali->clk_x_rate = clk_get_rate(dt->clk);
 
diff --git a/drivers/mtd/nand/diskonchip.c b/drivers/mtd/nand/diskonchip.c
index a023ab9e9cbf..c3aa53caab5c 100644
--- a/drivers/mtd/nand/diskonchip.c
+++ b/drivers/mtd/nand/diskonchip.c
@@ -27,7 +27,7 @@
 #include <linux/io.h>
 
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/doc2000.h>
 #include <linux/mtd/partitions.h>
 #include <linux/mtd/inftl.h>
diff --git a/drivers/mtd/nand/docg4.c b/drivers/mtd/nand/docg4.c
index a27a84fbfb84..2436cbc71662 100644
--- a/drivers/mtd/nand/docg4.c
+++ b/drivers/mtd/nand/docg4.c
@@ -41,7 +41,7 @@
 #include <linux/bitops.h>
 #include <linux/mtd/partitions.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/bch.h>
 #include <linux/bitrev.h>
 #include <linux/jiffies.h>
diff --git a/drivers/mtd/nand/fsl_elbc_nand.c b/drivers/mtd/nand/fsl_elbc_nand.c
index b9ac16f05057..17db2f90aa2c 100644
--- a/drivers/mtd/nand/fsl_elbc_nand.c
+++ b/drivers/mtd/nand/fsl_elbc_nand.c
@@ -34,7 +34,7 @@
 #include <linux/interrupt.h>
 
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/nand_ecc.h>
 #include <linux/mtd/partitions.h>
 
diff --git a/drivers/mtd/nand/fsl_ifc_nand.c b/drivers/mtd/nand/fsl_ifc_nand.c
index 59408ec2c69f..9e03bac7f34c 100644
--- a/drivers/mtd/nand/fsl_ifc_nand.c
+++ b/drivers/mtd/nand/fsl_ifc_nand.c
@@ -26,7 +26,7 @@
 #include <linux/of_address.h>
 #include <linux/slab.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 #include <linux/mtd/nand_ecc.h>
 #include <linux/fsl_ifc.h>
diff --git a/drivers/mtd/nand/fsl_upm.c b/drivers/mtd/nand/fsl_upm.c
index d85fa2555b68..a88e2cf66e0f 100644
--- a/drivers/mtd/nand/fsl_upm.c
+++ b/drivers/mtd/nand/fsl_upm.c
@@ -14,7 +14,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/delay.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/nand_ecc.h>
 #include <linux/mtd/partitions.h>
 #include <linux/mtd/mtd.h>
diff --git a/drivers/mtd/nand/fsmc_nand.c b/drivers/mtd/nand/fsmc_nand.c
index 9d8b051d3187..eac15d9bf49e 100644
--- a/drivers/mtd/nand/fsmc_nand.c
+++ b/drivers/mtd/nand/fsmc_nand.c
@@ -28,7 +28,7 @@
 #include <linux/sched.h>
 #include <linux/types.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/nand_ecc.h>
 #include <linux/platform_device.h>
 #include <linux/of.h>
diff --git a/drivers/mtd/nand/gpio.c b/drivers/mtd/nand/gpio.c
index 85294f150f4f..fd3648952b5a 100644
--- a/drivers/mtd/nand/gpio.c
+++ b/drivers/mtd/nand/gpio.c
@@ -26,7 +26,7 @@
 #include <linux/gpio.h>
 #include <linux/io.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 #include <linux/mtd/nand-gpio.h>
 #include <linux/of.h>
diff --git a/drivers/mtd/nand/gpmi-nand/gpmi-nand.h b/drivers/mtd/nand/gpmi-nand/gpmi-nand.h
index 9df0ad64e7e0..a45e4ce13d10 100644
--- a/drivers/mtd/nand/gpmi-nand/gpmi-nand.h
+++ b/drivers/mtd/nand/gpmi-nand/gpmi-nand.h
@@ -17,7 +17,7 @@
 #ifndef __DRIVERS_MTD_NAND_GPMI_NAND_H
 #define __DRIVERS_MTD_NAND_GPMI_NAND_H
 
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/platform_device.h>
 #include <linux/dma-mapping.h>
 #include <linux/dmaengine.h>
diff --git a/drivers/mtd/nand/hisi504_nand.c b/drivers/mtd/nand/hisi504_nand.c
index 530caa80b1b6..d9ee1a7e6956 100644
--- a/drivers/mtd/nand/hisi504_nand.c
+++ b/drivers/mtd/nand/hisi504_nand.c
@@ -26,7 +26,7 @@
 #include <linux/module.h>
 #include <linux/delay.h>
 #include <linux/interrupt.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/dma-mapping.h>
 #include <linux/platform_device.h>
 #include <linux/mtd/partitions.h>
diff --git a/drivers/mtd/nand/jz4740_nand.c b/drivers/mtd/nand/jz4740_nand.c
index 0d06a1f07d82..ad827d4af3e9 100644
--- a/drivers/mtd/nand/jz4740_nand.c
+++ b/drivers/mtd/nand/jz4740_nand.c
@@ -20,7 +20,7 @@
 #include <linux/slab.h>
 
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 
 #include <linux/gpio.h>
diff --git a/drivers/mtd/nand/jz4780_nand.c b/drivers/mtd/nand/jz4780_nand.c
index 8bc835f71b26..e69f6ae4c539 100644
--- a/drivers/mtd/nand/jz4780_nand.c
+++ b/drivers/mtd/nand/jz4780_nand.c
@@ -20,7 +20,7 @@
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 
 #include <linux/jz4780-nemc.h>
diff --git a/drivers/mtd/nand/lpc32xx_mlc.c b/drivers/mtd/nand/lpc32xx_mlc.c
index 846a66c1b133..5796468db653 100644
--- a/drivers/mtd/nand/lpc32xx_mlc.c
+++ b/drivers/mtd/nand/lpc32xx_mlc.c
@@ -27,7 +27,7 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 #include <linux/clk.h>
 #include <linux/err.h>
@@ -705,7 +705,9 @@ static int lpc32xx_nand_probe(struct platform_device *pdev)
 		res = -ENOENT;
 		goto err_exit1;
 	}
-	clk_prepare_enable(host->clk);
+	res = clk_prepare_enable(host->clk);
+	if (res)
+		goto err_put_clk;
 
 	nand_chip->cmd_ctrl = lpc32xx_nand_cmd_ctrl;
 	nand_chip->dev_ready = lpc32xx_nand_device_ready;
@@ -812,6 +814,7 @@ err_exit3:
 		dma_release_channel(host->dma_chan);
 err_exit2:
 	clk_disable_unprepare(host->clk);
+err_put_clk:
 	clk_put(host->clk);
 err_exit1:
 	lpc32xx_wp_enable(host);
@@ -846,9 +849,12 @@ static int lpc32xx_nand_remove(struct platform_device *pdev)
 static int lpc32xx_nand_resume(struct platform_device *pdev)
 {
 	struct lpc32xx_nand_host *host = platform_get_drvdata(pdev);
+	int ret;
 
 	/* Re-enable NAND clock */
-	clk_prepare_enable(host->clk);
+	ret = clk_prepare_enable(host->clk);
+	if (ret)
+		return ret;
 
 	/* Fresh init of NAND controller */
 	lpc32xx_nand_setup(host);
diff --git a/drivers/mtd/nand/lpc32xx_slc.c b/drivers/mtd/nand/lpc32xx_slc.c
index a0669a33f8fe..b61f28a1554d 100644
--- a/drivers/mtd/nand/lpc32xx_slc.c
+++ b/drivers/mtd/nand/lpc32xx_slc.c
@@ -23,7 +23,7 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 #include <linux/clk.h>
 #include <linux/err.h>
@@ -840,7 +840,9 @@ static int lpc32xx_nand_probe(struct platform_device *pdev)
 		res = -ENOENT;
 		goto err_exit1;
 	}
-	clk_prepare_enable(host->clk);
+	res = clk_prepare_enable(host->clk);
+	if (res)
+		goto err_exit1;
 
 	/* Set NAND IO addresses and command/ready functions */
 	chip->IO_ADDR_R = SLC_DATA(host->io_base);
@@ -972,9 +974,12 @@ static int lpc32xx_nand_remove(struct platform_device *pdev)
 static int lpc32xx_nand_resume(struct platform_device *pdev)
 {
 	struct lpc32xx_nand_host *host = platform_get_drvdata(pdev);
+	int ret;
 
 	/* Re-enable NAND clock */
-	clk_prepare_enable(host->clk);
+	ret = clk_prepare_enable(host->clk);
+	if (ret)
+		return ret;
 
 	/* Fresh init of NAND controller */
 	lpc32xx_nand_setup(host);
diff --git a/drivers/mtd/nand/mpc5121_nfc.c b/drivers/mtd/nand/mpc5121_nfc.c
index 0e86fb6277c3..b6b97cc9fba6 100644
--- a/drivers/mtd/nand/mpc5121_nfc.c
+++ b/drivers/mtd/nand/mpc5121_nfc.c
@@ -33,7 +33,7 @@
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 #include <linux/of_address.h>
 #include <linux/of_device.h>
diff --git a/drivers/mtd/nand/mtk_ecc.c b/drivers/mtd/nand/mtk_ecc.c
index 6c3a4aab0b48..7f3b065b6b8f 100644
--- a/drivers/mtd/nand/mtk_ecc.c
+++ b/drivers/mtd/nand/mtk_ecc.c
@@ -464,8 +464,8 @@ static int mtk_ecc_probe(struct platform_device *pdev)
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
-		dev_err(dev, "failed to get irq\n");
-		return -EINVAL;
+		dev_err(dev, "failed to get irq: %d\n", irq);
+		return irq;
 	}
 
 	ret = dma_set_mask(dev, DMA_BIT_MASK(32));
diff --git a/drivers/mtd/nand/mtk_nand.c b/drivers/mtd/nand/mtk_nand.c
index f7ae99464375..d86a7d131cc0 100644
--- a/drivers/mtd/nand/mtk_nand.c
+++ b/drivers/mtd/nand/mtk_nand.c
@@ -19,7 +19,7 @@
 #include <linux/interrupt.h>
 #include <linux/delay.h>
 #include <linux/clk.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/mtd.h>
 #include <linux/module.h>
 #include <linux/iopoll.h>
diff --git a/drivers/mtd/nand/mxc_nand.c b/drivers/mtd/nand/mxc_nand.c
index a764d5ca7536..53e5e0337c3e 100644
--- a/drivers/mtd/nand/mxc_nand.c
+++ b/drivers/mtd/nand/mxc_nand.c
@@ -22,7 +22,7 @@
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 #include <linux/interrupt.h>
 #include <linux/device.h>
@@ -876,6 +876,8 @@ static void mxc_do_addr_cycle(struct mtd_info *mtd, int column, int page_addr)
 	}
 }
 
+#define MXC_V1_ECCBYTES		5
+
 static int mxc_v1_ooblayout_ecc(struct mtd_info *mtd, int section,
 				struct mtd_oob_region *oobregion)
 {
@@ -885,7 +887,7 @@ static int mxc_v1_ooblayout_ecc(struct mtd_info *mtd, int section,
 		return -ERANGE;
 
 	oobregion->offset = (section * 16) + 6;
-	oobregion->length = nand_chip->ecc.bytes;
+	oobregion->length = MXC_V1_ECCBYTES;
 
 	return 0;
 }
@@ -907,8 +909,7 @@ static int mxc_v1_ooblayout_free(struct mtd_info *mtd, int section,
 			oobregion->length = 4;
 		}
 	} else {
-		oobregion->offset = ((section - 1) * 16) +
-				    nand_chip->ecc.bytes + 6;
+		oobregion->offset = ((section - 1) * 16) + MXC_V1_ECCBYTES + 6;
 		if (section < nand_chip->ecc.steps)
 			oobregion->length = (section * 16) + 6 -
 					    oobregion->offset;
diff --git a/drivers/mtd/nand/nand_amd.c b/drivers/mtd/nand/nand_amd.c
index 170403a3bfa8..22f060f38123 100644
--- a/drivers/mtd/nand/nand_amd.c
+++ b/drivers/mtd/nand/nand_amd.c
@@ -15,7 +15,7 @@
  * GNU General Public License for more details.
  */
 
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 
 static void amd_nand_decode_id(struct nand_chip *chip)
 {
diff --git a/drivers/mtd/nand/nand_base.c b/drivers/mtd/nand/nand_base.c
index c6c18b82f8f4..12edaae17d81 100644
--- a/drivers/mtd/nand/nand_base.c
+++ b/drivers/mtd/nand/nand_base.c
@@ -39,7 +39,7 @@
 #include <linux/nmi.h>
 #include <linux/types.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/nand_ecc.h>
 #include <linux/mtd/nand_bch.h>
 #include <linux/interrupt.h>
@@ -1248,179 +1248,6 @@ int nand_reset(struct nand_chip *chip, int chipnr)
 }
 
 /**
- * __nand_unlock - [REPLACEABLE] unlocks specified locked blocks
- * @mtd: mtd info
- * @ofs: offset to start unlock from
- * @len: length to unlock
- * @invert:
- *        - when = 0, unlock the range of blocks within the lower and
- *                    upper boundary address
- *        - when = 1, unlock the range of blocks outside the boundaries
- *                    of the lower and upper boundary address
- *
- * Returs unlock status.
- */
-static int __nand_unlock(struct mtd_info *mtd, loff_t ofs,
-					uint64_t len, int invert)
-{
-	int ret = 0;
-	int status, page;
-	struct nand_chip *chip = mtd_to_nand(mtd);
-
-	/* Submit address of first page to unlock */
-	page = ofs >> chip->page_shift;
-	chip->cmdfunc(mtd, NAND_CMD_UNLOCK1, -1, page & chip->pagemask);
-
-	/* Submit address of last page to unlock */
-	page = (ofs + len) >> chip->page_shift;
-	chip->cmdfunc(mtd, NAND_CMD_UNLOCK2, -1,
-				(page | invert) & chip->pagemask);
-
-	/* Call wait ready function */
-	status = chip->waitfunc(mtd, chip);
-	/* See if device thinks it succeeded */
-	if (status & NAND_STATUS_FAIL) {
-		pr_debug("%s: error status = 0x%08x\n",
-					__func__, status);
-		ret = -EIO;
-	}
-
-	return ret;
-}
-
-/**
- * nand_unlock - [REPLACEABLE] unlocks specified locked blocks
- * @mtd: mtd info
- * @ofs: offset to start unlock from
- * @len: length to unlock
- *
- * Returns unlock status.
- */
-int nand_unlock(struct mtd_info *mtd, loff_t ofs, uint64_t len)
-{
-	int ret = 0;
-	int chipnr;
-	struct nand_chip *chip = mtd_to_nand(mtd);
-
-	pr_debug("%s: start = 0x%012llx, len = %llu\n",
-			__func__, (unsigned long long)ofs, len);
-
-	if (check_offs_len(mtd, ofs, len))
-		return -EINVAL;
-
-	/* Align to last block address if size addresses end of the device */
-	if (ofs + len == mtd->size)
-		len -= mtd->erasesize;
-
-	nand_get_device(mtd, FL_UNLOCKING);
-
-	/* Shift to get chip number */
-	chipnr = ofs >> chip->chip_shift;
-
-	/*
-	 * Reset the chip.
-	 * If we want to check the WP through READ STATUS and check the bit 7
-	 * we must reset the chip
-	 * some operation can also clear the bit 7 of status register
-	 * eg. erase/program a locked block
-	 */
-	nand_reset(chip, chipnr);
-
-	chip->select_chip(mtd, chipnr);
-
-	/* Check, if it is write protected */
-	if (nand_check_wp(mtd)) {
-		pr_debug("%s: device is write protected!\n",
-					__func__);
-		ret = -EIO;
-		goto out;
-	}
-
-	ret = __nand_unlock(mtd, ofs, len, 0);
-
-out:
-	chip->select_chip(mtd, -1);
-	nand_release_device(mtd);
-
-	return ret;
-}
-EXPORT_SYMBOL(nand_unlock);
-
-/**
- * nand_lock - [REPLACEABLE] locks all blocks present in the device
- * @mtd: mtd info
- * @ofs: offset to start unlock from
- * @len: length to unlock
- *
- * This feature is not supported in many NAND parts. 'Micron' NAND parts do
- * have this feature, but it allows only to lock all blocks, not for specified
- * range for block. Implementing 'lock' feature by making use of 'unlock', for
- * now.
- *
- * Returns lock status.
- */
-int nand_lock(struct mtd_info *mtd, loff_t ofs, uint64_t len)
-{
-	int ret = 0;
-	int chipnr, status, page;
-	struct nand_chip *chip = mtd_to_nand(mtd);
-
-	pr_debug("%s: start = 0x%012llx, len = %llu\n",
-			__func__, (unsigned long long)ofs, len);
-
-	if (check_offs_len(mtd, ofs, len))
-		return -EINVAL;
-
-	nand_get_device(mtd, FL_LOCKING);
-
-	/* Shift to get chip number */
-	chipnr = ofs >> chip->chip_shift;
-
-	/*
-	 * Reset the chip.
-	 * If we want to check the WP through READ STATUS and check the bit 7
-	 * we must reset the chip
-	 * some operation can also clear the bit 7 of status register
-	 * eg. erase/program a locked block
-	 */
-	nand_reset(chip, chipnr);
-
-	chip->select_chip(mtd, chipnr);
-
-	/* Check, if it is write protected */
-	if (nand_check_wp(mtd)) {
-		pr_debug("%s: device is write protected!\n",
-					__func__);
-		status = MTD_ERASE_FAILED;
-		ret = -EIO;
-		goto out;
-	}
-
-	/* Submit address of first page to lock */
-	page = ofs >> chip->page_shift;
-	chip->cmdfunc(mtd, NAND_CMD_LOCK, -1, page & chip->pagemask);
-
-	/* Call wait ready function */
-	status = chip->waitfunc(mtd, chip);
-	/* See if device thinks it succeeded */
-	if (status & NAND_STATUS_FAIL) {
-		pr_debug("%s: error status = 0x%08x\n",
-					__func__, status);
-		ret = -EIO;
-		goto out;
-	}
-
-	ret = __nand_unlock(mtd, ofs, len, 0x1);
-
-out:
-	chip->select_chip(mtd, -1);
-	nand_release_device(mtd);
-
-	return ret;
-}
-EXPORT_SYMBOL(nand_lock);
-
-/**
  * nand_check_erased_buf - check if a buffer contains (almost) only 0xff data
  * @buf: buffer to test
  * @len: buffer length
@@ -2841,7 +2668,7 @@ static uint8_t *nand_fill_oob(struct mtd_info *mtd, uint8_t *oob, size_t len,
 static int nand_do_write_ops(struct mtd_info *mtd, loff_t to,
 			     struct mtd_oob_ops *ops)
 {
-	int chipnr, realpage, page, blockmask, column;
+	int chipnr, realpage, page, column;
 	struct nand_chip *chip = mtd_to_nand(mtd);
 	uint32_t writelen = ops->len;
 
@@ -2877,7 +2704,6 @@ static int nand_do_write_ops(struct mtd_info *mtd, loff_t to,
 
 	realpage = (int)(to >> chip->page_shift);
 	page = realpage & chip->pagemask;
-	blockmask = (1 << (chip->phys_erase_shift - chip->page_shift)) - 1;
 
 	/* Invalidate the page cache, when we write to the cached page */
 	if (to <= ((loff_t)chip->pagebuf << chip->page_shift) &&
@@ -3993,10 +3819,13 @@ static void nand_manufacturer_detect(struct nand_chip *chip)
 	 * nand_decode_ext_id() otherwise.
 	 */
 	if (chip->manufacturer.desc && chip->manufacturer.desc->ops &&
-	    chip->manufacturer.desc->ops->detect)
+	    chip->manufacturer.desc->ops->detect) {
+		/* The 3rd id byte holds MLC / multichip data */
+		chip->bits_per_cell = nand_get_bits_per_cell(chip->id.data[2]);
 		chip->manufacturer.desc->ops->detect(chip);
-	else
+	} else {
 		nand_decode_ext_id(chip);
+	}
 }
 
 /*
@@ -4036,7 +3865,7 @@ static int nand_detect(struct nand_chip *chip, struct nand_flash_dev *type)
 	const struct nand_manufacturer *manufacturer;
 	struct mtd_info *mtd = nand_to_mtd(chip);
 	int busw;
-	int i, ret;
+	int i;
 	u8 *id_data = chip->id.data;
 	u8 maf_id, dev_id;
 
@@ -4066,7 +3895,7 @@ static int nand_detect(struct nand_chip *chip, struct nand_flash_dev *type)
 	chip->cmdfunc(mtd, NAND_CMD_READID, 0x00, -1);
 
 	/* Read entire ID string */
-	for (i = 0; i < 8; i++)
+	for (i = 0; i < ARRAY_SIZE(chip->id.data); i++)
 		id_data[i] = chip->read_byte(mtd);
 
 	if (id_data[0] != maf_id || id_data[1] != dev_id) {
@@ -4075,7 +3904,7 @@ static int nand_detect(struct nand_chip *chip, struct nand_flash_dev *type)
 		return -ENODEV;
 	}
 
-	chip->id.len = nand_id_len(id_data, 8);
+	chip->id.len = nand_id_len(id_data, ARRAY_SIZE(chip->id.data));
 
 	/* Try to identify manufacturer */
 	manufacturer = nand_get_manufacturer(maf_id);
@@ -4177,10 +4006,6 @@ ident_done:
 	if (mtd->writesize > 512 && chip->cmdfunc == nand_command)
 		chip->cmdfunc = nand_command_lp;
 
-	ret = nand_manufacturer_init(chip);
-	if (ret)
-		return ret;
-
 	pr_info("device found, Manufacturer ID: 0x%02x, Chip ID: 0x%02x\n",
 		maf_id, dev_id);
 
@@ -4388,23 +4213,6 @@ int nand_scan_ident(struct mtd_info *mtd, int maxchips,
 		return ret;
 	}
 
-	/* Initialize the ->data_interface field. */
-	ret = nand_init_data_interface(chip);
-	if (ret)
-		goto err_nand_init;
-
-	/*
-	 * Setup the data interface correctly on the chip and controller side.
-	 * This explicit call to nand_setup_data_interface() is only required
-	 * for the first die, because nand_reset() has been called before
-	 * ->data_interface and ->default_onfi_timing_mode were set.
-	 * For the other dies, nand_reset() will automatically switch to the
-	 * best mode for us.
-	 */
-	ret = nand_setup_data_interface(chip, 0);
-	if (ret)
-		goto err_nand_init;
-
 	nand_maf_id = chip->id.data[0];
 	nand_dev_id = chip->id.data[1];
 
@@ -4434,12 +4242,6 @@ int nand_scan_ident(struct mtd_info *mtd, int maxchips,
 	mtd->size = i * chip->chipsize;
 
 	return 0;
-
-err_nand_init:
-	/* Free manufacturer priv data. */
-	nand_manufacturer_cleanup(chip);
-
-	return ret;
 }
 EXPORT_SYMBOL(nand_scan_ident);
 
@@ -4826,55 +4628,60 @@ int nand_scan_tail(struct mtd_info *mtd)
 	struct nand_chip *chip = mtd_to_nand(mtd);
 	struct nand_ecc_ctrl *ecc = &chip->ecc;
 	struct nand_buffers *nbuf = NULL;
-	int ret;
+	int ret, i;
 
 	/* New bad blocks should be marked in OOB, flash-based BBT, or both */
 	if (WARN_ON((chip->bbt_options & NAND_BBT_NO_OOB_BBM) &&
 		   !(chip->bbt_options & NAND_BBT_USE_FLASH))) {
-		ret = -EINVAL;
-		goto err_ident;
+		return -EINVAL;
 	}
 
 	if (invalid_ecc_page_accessors(chip)) {
 		pr_err("Invalid ECC page accessors setup\n");
-		ret = -EINVAL;
-		goto err_ident;
+		return -EINVAL;
 	}
 
 	if (!(chip->options & NAND_OWN_BUFFERS)) {
 		nbuf = kzalloc(sizeof(*nbuf), GFP_KERNEL);
-		if (!nbuf) {
-			ret = -ENOMEM;
-			goto err_ident;
-		}
+		if (!nbuf)
+			return -ENOMEM;
 
 		nbuf->ecccalc = kmalloc(mtd->oobsize, GFP_KERNEL);
 		if (!nbuf->ecccalc) {
 			ret = -ENOMEM;
-			goto err_free;
+			goto err_free_nbuf;
 		}
 
 		nbuf->ecccode = kmalloc(mtd->oobsize, GFP_KERNEL);
 		if (!nbuf->ecccode) {
 			ret = -ENOMEM;
-			goto err_free;
+			goto err_free_nbuf;
 		}
 
 		nbuf->databuf = kmalloc(mtd->writesize + mtd->oobsize,
 					GFP_KERNEL);
 		if (!nbuf->databuf) {
 			ret = -ENOMEM;
-			goto err_free;
+			goto err_free_nbuf;
 		}
 
 		chip->buffers = nbuf;
-	} else {
-		if (!chip->buffers) {
-			ret = -ENOMEM;
-			goto err_ident;
-		}
+	} else if (!chip->buffers) {
+		return -ENOMEM;
 	}
 
+	/*
+	 * FIXME: some NAND manufacturer drivers expect the first die to be
+	 * selected when manufacturer->init() is called. They should be fixed
+	 * to explictly select the relevant die when interacting with the NAND
+	 * chip.
+	 */
+	chip->select_chip(mtd, 0);
+	ret = nand_manufacturer_init(chip);
+	chip->select_chip(mtd, -1);
+	if (ret)
+		goto err_free_nbuf;
+
 	/* Set the internal oob buffer location, just after the page data */
 	chip->oob_poi = chip->buffers->databuf + mtd->writesize;
 
@@ -4896,7 +4703,7 @@ int nand_scan_tail(struct mtd_info *mtd)
 			WARN(1, "No oob scheme defined for oobsize %d\n",
 				mtd->oobsize);
 			ret = -EINVAL;
-			goto err_free;
+			goto err_nand_manuf_cleanup;
 		}
 	}
 
@@ -4911,7 +4718,7 @@ int nand_scan_tail(struct mtd_info *mtd)
 		if (!ecc->calculate || !ecc->correct || !ecc->hwctl) {
 			WARN(1, "No ECC functions supplied; hardware ECC not possible\n");
 			ret = -EINVAL;
-			goto err_free;
+			goto err_nand_manuf_cleanup;
 		}
 		if (!ecc->read_page)
 			ecc->read_page = nand_read_page_hwecc_oob_first;
@@ -4943,7 +4750,7 @@ int nand_scan_tail(struct mtd_info *mtd)
 		     ecc->write_page == nand_write_page_hwecc)) {
 			WARN(1, "No ECC functions supplied; hardware ECC not possible\n");
 			ret = -EINVAL;
-			goto err_free;
+			goto err_nand_manuf_cleanup;
 		}
 		/* Use standard syndrome read/write page function? */
 		if (!ecc->read_page)
@@ -4963,7 +4770,7 @@ int nand_scan_tail(struct mtd_info *mtd)
 			if (!ecc->strength) {
 				WARN(1, "Driver must set ecc.strength when using hardware ECC\n");
 				ret = -EINVAL;
-				goto err_free;
+				goto err_nand_manuf_cleanup;
 			}
 			break;
 		}
@@ -4976,7 +4783,7 @@ int nand_scan_tail(struct mtd_info *mtd)
 		ret = nand_set_ecc_soft_ops(mtd);
 		if (ret) {
 			ret = -EINVAL;
-			goto err_free;
+			goto err_nand_manuf_cleanup;
 		}
 		break;
 
@@ -4984,7 +4791,7 @@ int nand_scan_tail(struct mtd_info *mtd)
 		if (!ecc->read_page || !ecc->write_page) {
 			WARN(1, "No ECC functions supplied; on-die ECC not possible\n");
 			ret = -EINVAL;
-			goto err_free;
+			goto err_nand_manuf_cleanup;
 		}
 		if (!ecc->read_oob)
 			ecc->read_oob = nand_read_oob_std;
@@ -5008,7 +4815,7 @@ int nand_scan_tail(struct mtd_info *mtd)
 	default:
 		WARN(1, "Invalid NAND_ECC_MODE %d\n", ecc->mode);
 		ret = -EINVAL;
-		goto err_free;
+		goto err_nand_manuf_cleanup;
 	}
 
 	/* For many systems, the standard OOB write also works for raw */
@@ -5029,13 +4836,13 @@ int nand_scan_tail(struct mtd_info *mtd)
 	if (ecc->steps * ecc->size != mtd->writesize) {
 		WARN(1, "Invalid ECC parameters\n");
 		ret = -EINVAL;
-		goto err_free;
+		goto err_nand_manuf_cleanup;
 	}
 	ecc->total = ecc->steps * ecc->bytes;
 	if (ecc->total > mtd->oobsize) {
 		WARN(1, "Total number of ECC bytes exceeded oobsize\n");
 		ret = -EINVAL;
-		goto err_free;
+		goto err_nand_manuf_cleanup;
 	}
 
 	/*
@@ -5117,6 +4924,21 @@ int nand_scan_tail(struct mtd_info *mtd)
 	if (!mtd->bitflip_threshold)
 		mtd->bitflip_threshold = DIV_ROUND_UP(mtd->ecc_strength * 3, 4);
 
+	/* Initialize the ->data_interface field. */
+	ret = nand_init_data_interface(chip);
+	if (ret)
+		goto err_nand_manuf_cleanup;
+
+	/* Enter fastest possible mode on all dies. */
+	for (i = 0; i < chip->numchips; i++) {
+		chip->select_chip(mtd, i);
+		ret = nand_setup_data_interface(chip, i);
+		chip->select_chip(mtd, -1);
+
+		if (ret)
+			goto err_nand_data_iface_cleanup;
+	}
+
 	/* Check, if we should skip the bad block table scan */
 	if (chip->options & NAND_SKIP_BBTSCAN)
 		return 0;
@@ -5124,10 +4946,17 @@ int nand_scan_tail(struct mtd_info *mtd)
 	/* Build bad block table */
 	ret = chip->scan_bbt(mtd);
 	if (ret)
-		goto err_free;
+		goto err_nand_data_iface_cleanup;
+
 	return 0;
 
-err_free:
+err_nand_data_iface_cleanup:
+	nand_release_data_interface(chip);
+
+err_nand_manuf_cleanup:
+	nand_manufacturer_cleanup(chip);
+
+err_free_nbuf:
 	if (nbuf) {
 		kfree(nbuf->databuf);
 		kfree(nbuf->ecccode);
@@ -5135,12 +4964,6 @@ err_free:
 		kfree(nbuf);
 	}
 
-err_ident:
-	/* Clean up nand_scan_ident(). */
-
-	/* Free manufacturer priv data. */
-	nand_manufacturer_cleanup(chip);
-
 	return ret;
 }
 EXPORT_SYMBOL(nand_scan_tail);
diff --git a/drivers/mtd/nand/nand_bbt.c b/drivers/mtd/nand/nand_bbt.c
index 7695efea65f2..2915b6739bf8 100644
--- a/drivers/mtd/nand/nand_bbt.c
+++ b/drivers/mtd/nand/nand_bbt.c
@@ -61,7 +61,7 @@
 #include <linux/types.h>
 #include <linux/mtd/mtd.h>
 #include <linux/mtd/bbm.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/bitops.h>
 #include <linux/delay.h>
 #include <linux/vmalloc.h>
diff --git a/drivers/mtd/nand/nand_bch.c b/drivers/mtd/nand/nand_bch.c
index 44763f87eae4..505441c9373b 100644
--- a/drivers/mtd/nand/nand_bch.c
+++ b/drivers/mtd/nand/nand_bch.c
@@ -25,7 +25,7 @@
 #include <linux/slab.h>
 #include <linux/bitops.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/nand_bch.h>
 #include <linux/bch.h>
 
diff --git a/drivers/mtd/nand/nand_ecc.c b/drivers/mtd/nand/nand_ecc.c
index d1770b066396..7613a0388044 100644
--- a/drivers/mtd/nand/nand_ecc.c
+++ b/drivers/mtd/nand/nand_ecc.c
@@ -43,7 +43,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/nand_ecc.h>
 #include <asm/byteorder.h>
 #else
diff --git a/drivers/mtd/nand/nand_hynix.c b/drivers/mtd/nand/nand_hynix.c
index b12dc7325378..985751eda317 100644
--- a/drivers/mtd/nand/nand_hynix.c
+++ b/drivers/mtd/nand/nand_hynix.c
@@ -15,7 +15,7 @@
  * GNU General Public License for more details.
  */
 
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/sizes.h>
 #include <linux/slab.h>
 
@@ -477,7 +477,7 @@ static void hynix_nand_extract_ecc_requirements(struct nand_chip *chip,
 		 * The ECC requirements field meaning depends on the
 		 * NAND technology.
 		 */
-		u8 nand_tech = chip->id.data[5] & 0x3;
+		u8 nand_tech = chip->id.data[5] & 0x7;
 
 		if (nand_tech < 3) {
 			/* > 26nm, reference: H27UBG8T2A datasheet */
@@ -533,7 +533,7 @@ static void hynix_nand_extract_scrambling_requirements(struct nand_chip *chip,
 		if (nand_tech > 0)
 			chip->options |= NAND_NEED_SCRAMBLING;
 	} else {
-		nand_tech = chip->id.data[5] & 0x3;
+		nand_tech = chip->id.data[5] & 0x7;
 
 		/* < 32nm */
 		if (nand_tech > 2)
diff --git a/drivers/mtd/nand/nand_ids.c b/drivers/mtd/nand/nand_ids.c
index 92e2cf8e9ff9..5423c3bb388e 100644
--- a/drivers/mtd/nand/nand_ids.c
+++ b/drivers/mtd/nand/nand_ids.c
@@ -6,7 +6,7 @@
  * published by the Free Software Foundation.
  *
  */
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/sizes.h>
 
 #define LP_OPTIONS 0
diff --git a/drivers/mtd/nand/nand_macronix.c b/drivers/mtd/nand/nand_macronix.c
index 84855c3e1a02..d290ff2a6d2f 100644
--- a/drivers/mtd/nand/nand_macronix.c
+++ b/drivers/mtd/nand/nand_macronix.c
@@ -15,7 +15,7 @@
  * GNU General Public License for more details.
  */
 
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 
 static int macronix_nand_init(struct nand_chip *chip)
 {
diff --git a/drivers/mtd/nand/nand_micron.c b/drivers/mtd/nand/nand_micron.c
index c30ab60f8e1b..abf6a3c376e8 100644
--- a/drivers/mtd/nand/nand_micron.c
+++ b/drivers/mtd/nand/nand_micron.c
@@ -15,7 +15,7 @@
  * GNU General Public License for more details.
  */
 
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 
 /*
  * Special Micron status bit that indicates when the block has been
diff --git a/drivers/mtd/nand/nand_samsung.c b/drivers/mtd/nand/nand_samsung.c
index 1e0755997762..d348f0129ae7 100644
--- a/drivers/mtd/nand/nand_samsung.c
+++ b/drivers/mtd/nand/nand_samsung.c
@@ -15,7 +15,7 @@
  * GNU General Public License for more details.
  */
 
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 
 static void samsung_nand_decode_id(struct nand_chip *chip)
 {
diff --git a/drivers/mtd/nand/nand_timings.c b/drivers/mtd/nand/nand_timings.c
index 7e36d7d13c26..5d1533bcc5bd 100644
--- a/drivers/mtd/nand/nand_timings.c
+++ b/drivers/mtd/nand/nand_timings.c
@@ -11,7 +11,7 @@
 #include <linux/kernel.h>
 #include <linux/err.h>
 #include <linux/export.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 
 static const struct nand_data_interface onfi_sdr_timings[] = {
 	/* Mode 0 */
diff --git a/drivers/mtd/nand/nand_toshiba.c b/drivers/mtd/nand/nand_toshiba.c
index fa787ba38dcd..57df857074e6 100644
--- a/drivers/mtd/nand/nand_toshiba.c
+++ b/drivers/mtd/nand/nand_toshiba.c
@@ -15,7 +15,7 @@
  * GNU General Public License for more details.
  */
 
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 
 static void toshiba_nand_decode_id(struct nand_chip *chip)
 {
diff --git a/drivers/mtd/nand/nandsim.c b/drivers/mtd/nand/nandsim.c
index 03a0d057bf2f..246b4393118e 100644
--- a/drivers/mtd/nand/nandsim.c
+++ b/drivers/mtd/nand/nandsim.c
@@ -33,7 +33,7 @@
 #include <linux/errno.h>
 #include <linux/string.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/nand_bch.h>
 #include <linux/mtd/partitions.h>
 #include <linux/delay.h>
@@ -287,11 +287,6 @@ MODULE_PARM_DESC(bch,		 "Enable BCH ecc and set how many bits should "
 /* Maximum page cache pages needed to read or write a NAND page to the cache_file */
 #define NS_MAX_HELD_PAGES 16
 
-struct nandsim_debug_info {
-	struct dentry *dfs_root;
-	struct dentry *dfs_wear_report;
-};
-
 /*
  * A union to represent flash memory contents and flash buffer.
  */
@@ -370,8 +365,6 @@ struct nandsim {
 	void *file_buf;
 	struct page *held_pages[NS_MAX_HELD_PAGES];
 	int held_cnt;
-
-	struct nandsim_debug_info dbg;
 };
 
 /*
@@ -524,39 +517,23 @@ static const struct file_operations dfs_fops = {
  */
 static int nandsim_debugfs_create(struct nandsim *dev)
 {
-	struct nandsim_debug_info *dbg = &dev->dbg;
+	struct dentry *root = nsmtd->dbg.dfs_dir;
 	struct dentry *dent;
 
 	if (!IS_ENABLED(CONFIG_DEBUG_FS))
 		return 0;
 
-	dent = debugfs_create_dir("nandsim", NULL);
-	if (!dent) {
-		NS_ERR("cannot create \"nandsim\" debugfs directory\n");
-		return -ENODEV;
-	}
-	dbg->dfs_root = dent;
+	if (IS_ERR_OR_NULL(root))
+		return -1;
 
-	dent = debugfs_create_file("wear_report", S_IRUSR,
-				   dbg->dfs_root, dev, &dfs_fops);
-	if (!dent)
-		goto out_remove;
-	dbg->dfs_wear_report = dent;
+	dent = debugfs_create_file("nandsim_wear_report", S_IRUSR,
+				   root, dev, &dfs_fops);
+	if (IS_ERR_OR_NULL(dent)) {
+		NS_ERR("cannot create \"nandsim_wear_report\" debugfs entry\n");
+		return -1;
+	}
 
 	return 0;
-
-out_remove:
-	debugfs_remove_recursive(dbg->dfs_root);
-	return -ENODEV;
-}
-
-/**
- * nandsim_debugfs_remove - destroy all debugfs files
- */
-static void nandsim_debugfs_remove(struct nandsim *ns)
-{
-	if (IS_ENABLED(CONFIG_DEBUG_FS))
-		debugfs_remove_recursive(ns->dbg.dfs_root);
 }
 
 /*
@@ -1379,7 +1356,7 @@ static ssize_t read_file(struct nandsim *ns, struct file *file, void *buf, size_
 	if (err)
 		return err;
 	noreclaim_flag = memalloc_noreclaim_save();
-	tx = kernel_read(file, pos, buf, count);
+	tx = kernel_read(file, buf, count, &pos);
 	memalloc_noreclaim_restore(noreclaim_flag);
 	put_pages(ns);
 	return tx;
@@ -1395,7 +1372,7 @@ static ssize_t write_file(struct nandsim *ns, struct file *file, void *buf, size
 	if (err)
 		return err;
 	noreclaim_flag = memalloc_noreclaim_save();
-	tx = kernel_write(file, buf, count, pos);
+	tx = kernel_write(file, buf, count, &pos);
 	memalloc_noreclaim_restore(noreclaim_flag);
 	put_pages(ns);
 	return tx;
@@ -2352,9 +2329,6 @@ static int __init ns_init_module(void)
 	if ((retval = setup_wear_reporting(nsmtd)) != 0)
 		goto err_exit;
 
-	if ((retval = nandsim_debugfs_create(nand)) != 0)
-		goto err_exit;
-
 	if ((retval = init_nandsim(nsmtd)) != 0)
 		goto err_exit;
 
@@ -2370,6 +2344,9 @@ static int __init ns_init_module(void)
 	if (retval != 0)
 		goto err_exit;
 
+	if ((retval = nandsim_debugfs_create(nand)) != 0)
+		goto err_exit;
+
         return 0;
 
 err_exit:
@@ -2395,7 +2372,6 @@ static void __exit ns_cleanup_module(void)
 	struct nandsim *ns = nand_get_controller_data(chip);
 	int i;
 
-	nandsim_debugfs_remove(ns);
 	free_nandsim(ns);    /* Free nandsim private resources */
 	nand_release(nsmtd); /* Unregister driver */
 	for (i = 0;i < ARRAY_SIZE(ns->partitions); ++i)
diff --git a/drivers/mtd/nand/ndfc.c b/drivers/mtd/nand/ndfc.c
index 28e6118362f7..d8a806894937 100644
--- a/drivers/mtd/nand/ndfc.c
+++ b/drivers/mtd/nand/ndfc.c
@@ -22,7 +22,7 @@
  *
  */
 #include <linux/module.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/nand_ecc.h>
 #include <linux/mtd/partitions.h>
 #include <linux/mtd/ndfc.h>
diff --git a/drivers/mtd/nand/nuc900_nand.c b/drivers/mtd/nand/nuc900_nand.c
index 8f64011d32ef..7bb4d2ea9342 100644
--- a/drivers/mtd/nand/nuc900_nand.c
+++ b/drivers/mtd/nand/nuc900_nand.c
@@ -19,7 +19,7 @@
 #include <linux/err.h>
 
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 
 #define REG_FMICSR   	0x00
diff --git a/drivers/mtd/nand/omap2.c b/drivers/mtd/nand/omap2.c
index 084934a9f19c..54540c8fa1a2 100644
--- a/drivers/mtd/nand/omap2.c
+++ b/drivers/mtd/nand/omap2.c
@@ -18,7 +18,7 @@
 #include <linux/jiffies.h>
 #include <linux/sched.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 #include <linux/omap-dma.h>
 #include <linux/io.h>
diff --git a/drivers/mtd/nand/orion_nand.c b/drivers/mtd/nand/orion_nand.c
index 209170ed2b76..5a5aa1f07d07 100644
--- a/drivers/mtd/nand/orion_nand.c
+++ b/drivers/mtd/nand/orion_nand.c
@@ -15,7 +15,7 @@
 #include <linux/platform_device.h>
 #include <linux/of.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 #include <linux/clk.h>
 #include <linux/err.h>
@@ -54,13 +54,16 @@ static void orion_nand_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
 {
 	struct nand_chip *chip = mtd_to_nand(mtd);
 	void __iomem *io_base = chip->IO_ADDR_R;
+#if __LINUX_ARM_ARCH__ >= 5
 	uint64_t *buf64;
+#endif
 	int i = 0;
 
 	while (len && (unsigned long)buf & 7) {
 		*buf++ = readb(io_base);
 		len--;
 	}
+#if __LINUX_ARM_ARCH__ >= 5
 	buf64 = (uint64_t *)buf;
 	while (i < len/8) {
 		/*
@@ -74,6 +77,10 @@ static void orion_nand_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
 		buf64[i++] = x;
 	}
 	i *= 8;
+#else
+	readsl(io_base, buf, len/4);
+	i = len / 4 * 4;
+#endif
 	while (i < len)
 		buf[i++] = readb(io_base);
 }
diff --git a/drivers/mtd/nand/oxnas_nand.c b/drivers/mtd/nand/oxnas_nand.c
index 1b207aac840c..d649d5944826 100644
--- a/drivers/mtd/nand/oxnas_nand.c
+++ b/drivers/mtd/nand/oxnas_nand.c
@@ -21,7 +21,7 @@
 #include <linux/clk.h>
 #include <linux/reset.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 #include <linux/of.h>
 
@@ -112,14 +112,19 @@ static int oxnas_nand_probe(struct platform_device *pdev)
 	if (count > 1)
 		return -EINVAL;
 
-	clk_prepare_enable(oxnas->clk);
+	err = clk_prepare_enable(oxnas->clk);
+	if (err)
+		return err;
+
 	device_reset_optional(&pdev->dev);
 
 	for_each_child_of_node(np, nand_np) {
 		chip = devm_kzalloc(&pdev->dev, sizeof(struct nand_chip),
 				    GFP_KERNEL);
-		if (!chip)
-			return -ENOMEM;
+		if (!chip) {
+			err = -ENOMEM;
+			goto err_clk_unprepare;
+		}
 
 		chip->controller = &oxnas->base;
 
@@ -139,12 +144,12 @@ static int oxnas_nand_probe(struct platform_device *pdev)
 		/* Scan to find existence of the device */
 		err = nand_scan(mtd, 1);
 		if (err)
-			return err;
+			goto err_clk_unprepare;
 
 		err = mtd_device_register(mtd, NULL, 0);
 		if (err) {
 			nand_release(mtd);
-			return err;
+			goto err_clk_unprepare;
 		}
 
 		oxnas->chips[nchips] = chip;
@@ -152,12 +157,18 @@ static int oxnas_nand_probe(struct platform_device *pdev)
 	}
 
 	/* Exit if no chips found */
-	if (!nchips)
-		return -ENODEV;
+	if (!nchips) {
+		err = -ENODEV;
+		goto err_clk_unprepare;
+	}
 
 	platform_set_drvdata(pdev, oxnas);
 
 	return 0;
+
+err_clk_unprepare:
+	clk_disable_unprepare(oxnas->clk);
+	return err;
 }
 
 static int oxnas_nand_remove(struct platform_device *pdev)
diff --git a/drivers/mtd/nand/pasemi_nand.c b/drivers/mtd/nand/pasemi_nand.c
index 074b8b01289e..a47a7e4bd25a 100644
--- a/drivers/mtd/nand/pasemi_nand.c
+++ b/drivers/mtd/nand/pasemi_nand.c
@@ -25,7 +25,7 @@
 #include <linux/slab.h>
 #include <linux/module.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/nand_ecc.h>
 #include <linux/of_address.h>
 #include <linux/of_irq.h>
diff --git a/drivers/mtd/nand/plat_nand.c b/drivers/mtd/nand/plat_nand.c
index 791de3e4bbb6..925a1323604d 100644
--- a/drivers/mtd/nand/plat_nand.c
+++ b/drivers/mtd/nand/plat_nand.c
@@ -15,7 +15,7 @@
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 
 struct plat_nand_data {
diff --git a/drivers/mtd/nand/pxa3xx_nand.c b/drivers/mtd/nand/pxa3xx_nand.c
index 74dae4bbdac8..85cff68643e0 100644
--- a/drivers/mtd/nand/pxa3xx_nand.c
+++ b/drivers/mtd/nand/pxa3xx_nand.c
@@ -21,7 +21,7 @@
 #include <linux/delay.h>
 #include <linux/clk.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 #include <linux/io.h>
 #include <linux/iopoll.h>
diff --git a/drivers/mtd/nand/qcom_nandc.c b/drivers/mtd/nand/qcom_nandc.c
index 88af7145a51a..3baddfc997d1 100644
--- a/drivers/mtd/nand/qcom_nandc.c
+++ b/drivers/mtd/nand/qcom_nandc.c
@@ -17,7 +17,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/dmaengine.h>
 #include <linux/module.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
@@ -53,6 +53,8 @@
 #define	NAND_VERSION			0xf08
 #define	NAND_READ_LOCATION_0		0xf20
 #define	NAND_READ_LOCATION_1		0xf24
+#define	NAND_READ_LOCATION_2		0xf28
+#define	NAND_READ_LOCATION_3		0xf2c
 
 /* dummy register offsets, used by write_reg_dma */
 #define	NAND_DEV_CMD1_RESTORE		0xdead
@@ -109,7 +111,11 @@
 #define	READ_ADDR			0
 
 /* NAND_DEV_CMD_VLD bits */
-#define	READ_START_VLD			0
+#define	READ_START_VLD			BIT(0)
+#define	READ_STOP_VLD			BIT(1)
+#define	WRITE_START_VLD			BIT(2)
+#define	ERASE_START_VLD			BIT(3)
+#define	SEQ_READ_START_VLD		BIT(4)
 
 /* NAND_EBI2_ECC_BUF_CFG bits */
 #define	NUM_STEPS			0
@@ -131,6 +137,11 @@
 #define	ERASED_PAGE			(PAGE_ALL_ERASED | PAGE_ERASED)
 #define	ERASED_CW			(CODEWORD_ALL_ERASED | CODEWORD_ERASED)
 
+/* NAND_READ_LOCATION_n bits */
+#define READ_LOCATION_OFFSET		0
+#define READ_LOCATION_SIZE		16
+#define READ_LOCATION_LAST		31
+
 /* Version Mask */
 #define	NAND_VERSION_MAJOR_MASK		0xf0000000
 #define	NAND_VERSION_MAJOR_SHIFT	28
@@ -148,6 +159,13 @@
 #define	FETCH_ID			0xb
 #define	RESET_DEVICE			0xd
 
+/* Default Value for NAND_DEV_CMD_VLD */
+#define NAND_DEV_CMD_VLD_VAL		(READ_START_VLD | WRITE_START_VLD | \
+					 ERASE_START_VLD | SEQ_READ_START_VLD)
+
+/* NAND_CTRL bits */
+#define	BAM_MODE_EN			BIT(0)
+
 /*
  * the NAND controller performs reads/writes with ECC in 516 byte chunks.
  * the driver calls the chunks 'step' or 'codeword' interchangeably
@@ -169,11 +187,81 @@
 #define	ECC_BCH_4BIT	BIT(2)
 #define	ECC_BCH_8BIT	BIT(3)
 
+#define nandc_set_read_loc(nandc, reg, offset, size, is_last)	\
+nandc_set_reg(nandc, NAND_READ_LOCATION_##reg,			\
+	      ((offset) << READ_LOCATION_OFFSET) |		\
+	      ((size) << READ_LOCATION_SIZE) |			\
+	      ((is_last) << READ_LOCATION_LAST))
+
+/*
+ * Returns the actual register address for all NAND_DEV_ registers
+ * (i.e. NAND_DEV_CMD0, NAND_DEV_CMD1, NAND_DEV_CMD2 and NAND_DEV_CMD_VLD)
+ */
+#define dev_cmd_reg_addr(nandc, reg) ((nandc)->props->dev_cmd_reg_start + (reg))
+
+#define QPIC_PER_CW_CMD_SGL		32
+#define QPIC_PER_CW_DATA_SGL		8
+
+/*
+ * Flags used in DMA descriptor preparation helper functions
+ * (i.e. read_reg_dma/write_reg_dma/read_data_dma/write_data_dma)
+ */
+/* Don't set the EOT in current tx BAM sgl */
+#define NAND_BAM_NO_EOT			BIT(0)
+/* Set the NWD flag in current BAM sgl */
+#define NAND_BAM_NWD			BIT(1)
+/* Finish writing in the current BAM sgl and start writing in another BAM sgl */
+#define NAND_BAM_NEXT_SGL		BIT(2)
+/*
+ * Erased codeword status is being used two times in single transfer so this
+ * flag will determine the current value of erased codeword status register
+ */
+#define NAND_ERASED_CW_SET		BIT(4)
+
+/*
+ * This data type corresponds to the BAM transaction which will be used for all
+ * NAND transfers.
+ * @cmd_sgl - sgl for NAND BAM command pipe
+ * @data_sgl - sgl for NAND BAM consumer/producer pipe
+ * @cmd_sgl_pos - current index in command sgl.
+ * @cmd_sgl_start - start index in command sgl.
+ * @tx_sgl_pos - current index in data sgl for tx.
+ * @tx_sgl_start - start index in data sgl for tx.
+ * @rx_sgl_pos - current index in data sgl for rx.
+ * @rx_sgl_start - start index in data sgl for rx.
+ */
+struct bam_transaction {
+	struct scatterlist *cmd_sgl;
+	struct scatterlist *data_sgl;
+	u32 cmd_sgl_pos;
+	u32 cmd_sgl_start;
+	u32 tx_sgl_pos;
+	u32 tx_sgl_start;
+	u32 rx_sgl_pos;
+	u32 rx_sgl_start;
+};
+
+/*
+ * This data type corresponds to the nand dma descriptor
+ * @list - list for desc_info
+ * @dir - DMA transfer direction
+ * @adm_sgl - sgl which will be used for single sgl dma descriptor. Only used by
+ *	      ADM
+ * @bam_sgl - sgl which will be used for dma descriptor. Only used by BAM
+ * @sgl_cnt - number of SGL in bam_sgl. Only used by BAM
+ * @dma_desc - low level DMA engine descriptor
+ */
 struct desc_info {
 	struct list_head node;
 
 	enum dma_data_direction dir;
-	struct scatterlist sgl;
+	union {
+		struct scatterlist adm_sgl;
+		struct {
+			struct scatterlist *bam_sgl;
+			int sgl_cnt;
+		};
+	};
 	struct dma_async_tx_descriptor *dma_desc;
 };
 
@@ -202,6 +290,13 @@ struct nandc_regs {
 	__le32 orig_vld;
 
 	__le32 ecc_buf_cfg;
+	__le32 read_location0;
+	__le32 read_location1;
+	__le32 read_location2;
+	__le32 read_location3;
+
+	__le32 erased_cw_detect_cfg_clr;
+	__le32 erased_cw_detect_cfg_set;
 };
 
 /*
@@ -226,14 +321,17 @@ struct nandc_regs {
  *				by upper layers directly
  * @buf_size/count/start:	markers for chip->read_buf/write_buf functions
  * @reg_read_buf:		local buffer for reading back registers via DMA
+ * @reg_read_dma:		contains dma address for register read buffer
  * @reg_read_pos:		marker for data read in reg_read_buf
  *
  * @regs:			a contiguous chunk of memory for DMA register
  *				writes. contains the register values to be
  *				written to controller
  * @cmd1/vld:			some fixed controller register values
- * @ecc_modes:			supported ECC modes by the current controller,
+ * @props:			properties of current NAND controller,
  *				initialized via DT match data
+ * @max_cwperpage:		maximum QPIC codewords required. calculated
+ *				from all connected NAND devices pagesize
  */
 struct qcom_nand_controller {
 	struct nand_hw_control controller;
@@ -247,23 +345,39 @@ struct qcom_nand_controller {
 	struct clk *core_clk;
 	struct clk *aon_clk;
 
-	struct dma_chan *chan;
-	unsigned int cmd_crci;
-	unsigned int data_crci;
+	union {
+		/* will be used only by QPIC for BAM DMA */
+		struct {
+			struct dma_chan *tx_chan;
+			struct dma_chan *rx_chan;
+			struct dma_chan *cmd_chan;
+		};
+
+		/* will be used only by EBI2 for ADM DMA */
+		struct {
+			struct dma_chan *chan;
+			unsigned int cmd_crci;
+			unsigned int data_crci;
+		};
+	};
+
 	struct list_head desc_list;
+	struct bam_transaction *bam_txn;
 
 	u8		*data_buffer;
 	int		buf_size;
 	int		buf_count;
 	int		buf_start;
+	unsigned int	max_cwperpage;
 
 	__le32 *reg_read_buf;
+	dma_addr_t reg_read_dma;
 	int reg_read_pos;
 
 	struct nandc_regs *regs;
 
 	u32 cmd1, vld;
-	u32 ecc_modes;
+	const struct qcom_nandc_props *props;
 };
 
 /*
@@ -316,6 +430,78 @@ struct qcom_nand_host {
 	u32 clrreadstatus;
 };
 
+/*
+ * This data type corresponds to the NAND controller properties which varies
+ * among different NAND controllers.
+ * @ecc_modes - ecc mode for NAND
+ * @is_bam - whether NAND controller is using BAM
+ * @dev_cmd_reg_start - NAND_DEV_CMD_* registers starting offset
+ */
+struct qcom_nandc_props {
+	u32 ecc_modes;
+	bool is_bam;
+	u32 dev_cmd_reg_start;
+};
+
+/* Frees the BAM transaction memory */
+static void free_bam_transaction(struct qcom_nand_controller *nandc)
+{
+	struct bam_transaction *bam_txn = nandc->bam_txn;
+
+	devm_kfree(nandc->dev, bam_txn);
+}
+
+/* Allocates and Initializes the BAM transaction */
+static struct bam_transaction *
+alloc_bam_transaction(struct qcom_nand_controller *nandc)
+{
+	struct bam_transaction *bam_txn;
+	size_t bam_txn_size;
+	unsigned int num_cw = nandc->max_cwperpage;
+	void *bam_txn_buf;
+
+	bam_txn_size =
+		sizeof(*bam_txn) + num_cw *
+		((sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL) +
+		(sizeof(*bam_txn->data_sgl) * QPIC_PER_CW_DATA_SGL));
+
+	bam_txn_buf = devm_kzalloc(nandc->dev, bam_txn_size, GFP_KERNEL);
+	if (!bam_txn_buf)
+		return NULL;
+
+	bam_txn = bam_txn_buf;
+	bam_txn_buf += sizeof(*bam_txn);
+
+	bam_txn->cmd_sgl = bam_txn_buf;
+	bam_txn_buf +=
+		sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL * num_cw;
+
+	bam_txn->data_sgl = bam_txn_buf;
+
+	return bam_txn;
+}
+
+/* Clears the BAM transaction indexes */
+static void clear_bam_transaction(struct qcom_nand_controller *nandc)
+{
+	struct bam_transaction *bam_txn = nandc->bam_txn;
+
+	if (!nandc->props->is_bam)
+		return;
+
+	bam_txn->cmd_sgl_pos = 0;
+	bam_txn->cmd_sgl_start = 0;
+	bam_txn->tx_sgl_pos = 0;
+	bam_txn->tx_sgl_start = 0;
+	bam_txn->rx_sgl_pos = 0;
+	bam_txn->rx_sgl_start = 0;
+
+	sg_init_table(bam_txn->cmd_sgl, nandc->max_cwperpage *
+		      QPIC_PER_CW_CMD_SGL);
+	sg_init_table(bam_txn->data_sgl, nandc->max_cwperpage *
+		      QPIC_PER_CW_DATA_SGL);
+}
+
 static inline struct qcom_nand_host *to_qcom_nand_host(struct nand_chip *chip)
 {
 	return container_of(chip, struct qcom_nand_host, chip);
@@ -339,6 +525,24 @@ static inline void nandc_write(struct qcom_nand_controller *nandc, int offset,
 	iowrite32(val, nandc->base + offset);
 }
 
+static inline void nandc_read_buffer_sync(struct qcom_nand_controller *nandc,
+					  bool is_cpu)
+{
+	if (!nandc->props->is_bam)
+		return;
+
+	if (is_cpu)
+		dma_sync_single_for_cpu(nandc->dev, nandc->reg_read_dma,
+					MAX_REG_RD *
+					sizeof(*nandc->reg_read_buf),
+					DMA_FROM_DEVICE);
+	else
+		dma_sync_single_for_device(nandc->dev, nandc->reg_read_dma,
+					   MAX_REG_RD *
+					   sizeof(*nandc->reg_read_buf),
+					   DMA_FROM_DEVICE);
+}
+
 static __le32 *offset_to_nandc_reg(struct nandc_regs *regs, int offset)
 {
 	switch (offset) {
@@ -372,6 +576,14 @@ static __le32 *offset_to_nandc_reg(struct nandc_regs *regs, int offset)
 		return &regs->orig_vld;
 	case NAND_EBI2_ECC_BUF_CFG:
 		return &regs->ecc_buf_cfg;
+	case NAND_READ_LOCATION_0:
+		return &regs->read_location0;
+	case NAND_READ_LOCATION_1:
+		return &regs->read_location1;
+	case NAND_READ_LOCATION_2:
+		return &regs->read_location2;
+	case NAND_READ_LOCATION_3:
+		return &regs->read_location3;
 	default:
 		return NULL;
 	}
@@ -446,11 +658,119 @@ static void update_rw_regs(struct qcom_nand_host *host, int num_cw, bool read)
 	nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
 	nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);
 	nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
+
+	if (read)
+		nandc_set_read_loc(nandc, 0, 0, host->use_ecc ?
+				   host->cw_data : host->cw_size, 1);
+}
+
+/*
+ * Maps the scatter gather list for DMA transfer and forms the DMA descriptor
+ * for BAM. This descriptor will be added in the NAND DMA descriptor queue
+ * which will be submitted to DMA engine.
+ */
+static int prepare_bam_async_desc(struct qcom_nand_controller *nandc,
+				  struct dma_chan *chan,
+				  unsigned long flags)
+{
+	struct desc_info *desc;
+	struct scatterlist *sgl;
+	unsigned int sgl_cnt;
+	int ret;
+	struct bam_transaction *bam_txn = nandc->bam_txn;
+	enum dma_transfer_direction dir_eng;
+	struct dma_async_tx_descriptor *dma_desc;
+
+	desc = kzalloc(sizeof(*desc), GFP_KERNEL);
+	if (!desc)
+		return -ENOMEM;
+
+	if (chan == nandc->cmd_chan) {
+		sgl = &bam_txn->cmd_sgl[bam_txn->cmd_sgl_start];
+		sgl_cnt = bam_txn->cmd_sgl_pos - bam_txn->cmd_sgl_start;
+		bam_txn->cmd_sgl_start = bam_txn->cmd_sgl_pos;
+		dir_eng = DMA_MEM_TO_DEV;
+		desc->dir = DMA_TO_DEVICE;
+	} else if (chan == nandc->tx_chan) {
+		sgl = &bam_txn->data_sgl[bam_txn->tx_sgl_start];
+		sgl_cnt = bam_txn->tx_sgl_pos - bam_txn->tx_sgl_start;
+		bam_txn->tx_sgl_start = bam_txn->tx_sgl_pos;
+		dir_eng = DMA_MEM_TO_DEV;
+		desc->dir = DMA_TO_DEVICE;
+	} else {
+		sgl = &bam_txn->data_sgl[bam_txn->rx_sgl_start];
+		sgl_cnt = bam_txn->rx_sgl_pos - bam_txn->rx_sgl_start;
+		bam_txn->rx_sgl_start = bam_txn->rx_sgl_pos;
+		dir_eng = DMA_DEV_TO_MEM;
+		desc->dir = DMA_FROM_DEVICE;
+	}
+
+	sg_mark_end(sgl + sgl_cnt - 1);
+	ret = dma_map_sg(nandc->dev, sgl, sgl_cnt, desc->dir);
+	if (ret == 0) {
+		dev_err(nandc->dev, "failure in mapping desc\n");
+		kfree(desc);
+		return -ENOMEM;
+	}
+
+	desc->sgl_cnt = sgl_cnt;
+	desc->bam_sgl = sgl;
+
+	dma_desc = dmaengine_prep_slave_sg(chan, sgl, sgl_cnt, dir_eng,
+					   flags);
+
+	if (!dma_desc) {
+		dev_err(nandc->dev, "failure in prep desc\n");
+		dma_unmap_sg(nandc->dev, sgl, sgl_cnt, desc->dir);
+		kfree(desc);
+		return -EINVAL;
+	}
+
+	desc->dma_desc = dma_desc;
+
+	list_add_tail(&desc->node, &nandc->desc_list);
+
+	return 0;
+}
+
+/*
+ * Prepares the data descriptor for BAM DMA which will be used for NAND
+ * data reads and writes.
+ */
+static int prep_bam_dma_desc_data(struct qcom_nand_controller *nandc, bool read,
+				  const void *vaddr,
+				  int size, unsigned int flags)
+{
+	int ret;
+	struct bam_transaction *bam_txn = nandc->bam_txn;
+
+	if (read) {
+		sg_set_buf(&bam_txn->data_sgl[bam_txn->rx_sgl_pos],
+			   vaddr, size);
+		bam_txn->rx_sgl_pos++;
+	} else {
+		sg_set_buf(&bam_txn->data_sgl[bam_txn->tx_sgl_pos],
+			   vaddr, size);
+		bam_txn->tx_sgl_pos++;
+
+		/*
+		 * BAM will only set EOT for DMA_PREP_INTERRUPT so if this flag
+		 * is not set, form the DMA descriptor
+		 */
+		if (!(flags & NAND_BAM_NO_EOT)) {
+			ret = prepare_bam_async_desc(nandc, nandc->tx_chan,
+						     DMA_PREP_INTERRUPT);
+			if (ret)
+				return ret;
+		}
+	}
+
+	return 0;
 }
 
-static int prep_dma_desc(struct qcom_nand_controller *nandc, bool read,
-			 int reg_off, const void *vaddr, int size,
-			 bool flow_control)
+static int prep_adm_dma_desc(struct qcom_nand_controller *nandc, bool read,
+			     int reg_off, const void *vaddr, int size,
+			     bool flow_control)
 {
 	struct desc_info *desc;
 	struct dma_async_tx_descriptor *dma_desc;
@@ -463,7 +783,7 @@ static int prep_dma_desc(struct qcom_nand_controller *nandc, bool read,
 	if (!desc)
 		return -ENOMEM;
 
-	sgl = &desc->sgl;
+	sgl = &desc->adm_sgl;
 
 	sg_init_one(sgl, vaddr, size);
 
@@ -524,9 +844,10 @@ err:
  *
  * @first:		offset of the first register in the contiguous block
  * @num_regs:		number of registers to read
+ * @flags:		flags to control DMA descriptor preparation
  */
 static int read_reg_dma(struct qcom_nand_controller *nandc, int first,
-			int num_regs)
+			int num_regs, unsigned int flags)
 {
 	bool flow_control = false;
 	void *vaddr;
@@ -535,11 +856,14 @@ static int read_reg_dma(struct qcom_nand_controller *nandc, int first,
 	if (first == NAND_READ_ID || first == NAND_FLASH_STATUS)
 		flow_control = true;
 
+	if (first == NAND_DEV_CMD_VLD || first == NAND_DEV_CMD1)
+		first = dev_cmd_reg_addr(nandc, first);
+
 	size = num_regs * sizeof(u32);
 	vaddr = nandc->reg_read_buf + nandc->reg_read_pos;
 	nandc->reg_read_pos += num_regs;
 
-	return prep_dma_desc(nandc, true, first, vaddr, size, flow_control);
+	return prep_adm_dma_desc(nandc, true, first, vaddr, size, flow_control);
 }
 
 /*
@@ -548,9 +872,10 @@ static int read_reg_dma(struct qcom_nand_controller *nandc, int first,
  *
  * @first:		offset of the first register in the contiguous block
  * @num_regs:		number of registers to write
+ * @flags:		flags to control DMA descriptor preparation
  */
 static int write_reg_dma(struct qcom_nand_controller *nandc, int first,
-			 int num_regs)
+			 int num_regs, unsigned int flags)
 {
 	bool flow_control = false;
 	struct nandc_regs *regs = nandc->regs;
@@ -562,15 +887,26 @@ static int write_reg_dma(struct qcom_nand_controller *nandc, int first,
 	if (first == NAND_FLASH_CMD)
 		flow_control = true;
 
-	if (first == NAND_DEV_CMD1_RESTORE)
-		first = NAND_DEV_CMD1;
+	if (first == NAND_ERASED_CW_DETECT_CFG) {
+		if (flags & NAND_ERASED_CW_SET)
+			vaddr = &regs->erased_cw_detect_cfg_set;
+		else
+			vaddr = &regs->erased_cw_detect_cfg_clr;
+	}
+
+	if (first == NAND_EXEC_CMD)
+		flags |= NAND_BAM_NWD;
+
+	if (first == NAND_DEV_CMD1_RESTORE || first == NAND_DEV_CMD1)
+		first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD1);
 
-	if (first == NAND_DEV_CMD_VLD_RESTORE)
-		first = NAND_DEV_CMD_VLD;
+	if (first == NAND_DEV_CMD_VLD_RESTORE || first == NAND_DEV_CMD_VLD)
+		first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD);
 
 	size = num_regs * sizeof(u32);
 
-	return prep_dma_desc(nandc, false, first, vaddr, size, flow_control);
+	return prep_adm_dma_desc(nandc, false, first, vaddr, size,
+				 flow_control);
 }
 
 /*
@@ -580,11 +916,15 @@ static int write_reg_dma(struct qcom_nand_controller *nandc, int first,
  * @reg_off:		offset within the controller's data buffer
  * @vaddr:		virtual address of the buffer we want to write to
  * @size:		DMA transaction size in bytes
+ * @flags:		flags to control DMA descriptor preparation
  */
 static int read_data_dma(struct qcom_nand_controller *nandc, int reg_off,
-			 const u8 *vaddr, int size)
+			 const u8 *vaddr, int size, unsigned int flags)
 {
-	return prep_dma_desc(nandc, true, reg_off, vaddr, size, false);
+	if (nandc->props->is_bam)
+		return prep_bam_dma_desc_data(nandc, true, vaddr, size, flags);
+
+	return prep_adm_dma_desc(nandc, true, reg_off, vaddr, size, false);
 }
 
 /*
@@ -594,48 +934,84 @@ static int read_data_dma(struct qcom_nand_controller *nandc, int reg_off,
  * @reg_off:		offset within the controller's data buffer
  * @vaddr:		virtual address of the buffer we want to read from
  * @size:		DMA transaction size in bytes
+ * @flags:		flags to control DMA descriptor preparation
  */
 static int write_data_dma(struct qcom_nand_controller *nandc, int reg_off,
-			  const u8 *vaddr, int size)
+			  const u8 *vaddr, int size, unsigned int flags)
 {
-	return prep_dma_desc(nandc, false, reg_off, vaddr, size, false);
+	if (nandc->props->is_bam)
+		return prep_bam_dma_desc_data(nandc, false, vaddr, size, flags);
+
+	return prep_adm_dma_desc(nandc, false, reg_off, vaddr, size, false);
 }
 
 /*
- * helper to prepare dma descriptors to configure registers needed for reading a
- * codeword/step in a page
+ * Helper to prepare DMA descriptors for configuring registers
+ * before reading a NAND page.
  */
-static void config_cw_read(struct qcom_nand_controller *nandc)
+static void config_nand_page_read(struct qcom_nand_controller *nandc)
 {
-	write_reg_dma(nandc, NAND_FLASH_CMD, 3);
-	write_reg_dma(nandc, NAND_DEV0_CFG0, 3);
-	write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1);
+	write_reg_dma(nandc, NAND_ADDR0, 2, 0);
+	write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0);
+	write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1, 0);
+	write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1, 0);
+	write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1,
+		      NAND_ERASED_CW_SET | NAND_BAM_NEXT_SGL);
+}
 
-	write_reg_dma(nandc, NAND_EXEC_CMD, 1);
+/*
+ * Helper to prepare DMA descriptors for configuring registers
+ * before reading each codeword in NAND page.
+ */
+static void config_nand_cw_read(struct qcom_nand_controller *nandc)
+{
+	if (nandc->props->is_bam)
+		write_reg_dma(nandc, NAND_READ_LOCATION_0, 4,
+			      NAND_BAM_NEXT_SGL);
 
-	read_reg_dma(nandc, NAND_FLASH_STATUS, 2);
-	read_reg_dma(nandc, NAND_ERASED_CW_DETECT_STATUS, 1);
+	write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
+	write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
+
+	read_reg_dma(nandc, NAND_FLASH_STATUS, 2, 0);
+	read_reg_dma(nandc, NAND_ERASED_CW_DETECT_STATUS, 1,
+		     NAND_BAM_NEXT_SGL);
 }
 
 /*
- * helpers to prepare dma descriptors used to configure registers needed for
- * writing a codeword/step in a page
+ * Helper to prepare dma descriptors to configure registers needed for reading a
+ * single codeword in page
  */
-static void config_cw_write_pre(struct qcom_nand_controller *nandc)
+static void config_nand_single_cw_page_read(struct qcom_nand_controller *nandc)
 {
-	write_reg_dma(nandc, NAND_FLASH_CMD, 3);
-	write_reg_dma(nandc, NAND_DEV0_CFG0, 3);
-	write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1);
+	config_nand_page_read(nandc);
+	config_nand_cw_read(nandc);
 }
 
-static void config_cw_write_post(struct qcom_nand_controller *nandc)
+/*
+ * Helper to prepare DMA descriptors used to configure registers needed for
+ * before writing a NAND page.
+ */
+static void config_nand_page_write(struct qcom_nand_controller *nandc)
 {
-	write_reg_dma(nandc, NAND_EXEC_CMD, 1);
+	write_reg_dma(nandc, NAND_ADDR0, 2, 0);
+	write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0);
+	write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1,
+		      NAND_BAM_NEXT_SGL);
+}
 
-	read_reg_dma(nandc, NAND_FLASH_STATUS, 1);
+/*
+ * Helper to prepare DMA descriptors for configuring registers
+ * before writing each codeword in NAND page.
+ */
+static void config_nand_cw_write(struct qcom_nand_controller *nandc)
+{
+	write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
+	write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
 
-	write_reg_dma(nandc, NAND_FLASH_STATUS, 1);
-	write_reg_dma(nandc, NAND_READ_STATUS, 1);
+	read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
+
+	write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0);
+	write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL);
 }
 
 /*
@@ -672,8 +1048,7 @@ static int nandc_param(struct qcom_nand_host *host)
 
 	/* configure CMD1 and VLD for ONFI param probing */
 	nandc_set_reg(nandc, NAND_DEV_CMD_VLD,
-		      (nandc->vld & ~(1 << READ_START_VLD))
-		      | 0 << READ_START_VLD);
+		      (nandc->vld & ~READ_START_VLD));
 	nandc_set_reg(nandc, NAND_DEV_CMD1,
 		      (nandc->cmd1 & ~(0xFF << READ_ADDR))
 		      | NAND_CMD_PARAM << READ_ADDR);
@@ -682,21 +1057,22 @@ static int nandc_param(struct qcom_nand_host *host)
 
 	nandc_set_reg(nandc, NAND_DEV_CMD1_RESTORE, nandc->cmd1);
 	nandc_set_reg(nandc, NAND_DEV_CMD_VLD_RESTORE, nandc->vld);
+	nandc_set_read_loc(nandc, 0, 0, 512, 1);
 
-	write_reg_dma(nandc, NAND_DEV_CMD_VLD, 1);
-	write_reg_dma(nandc, NAND_DEV_CMD1, 1);
+	write_reg_dma(nandc, NAND_DEV_CMD_VLD, 1, 0);
+	write_reg_dma(nandc, NAND_DEV_CMD1, 1, NAND_BAM_NEXT_SGL);
 
 	nandc->buf_count = 512;
 	memset(nandc->data_buffer, 0xff, nandc->buf_count);
 
-	config_cw_read(nandc);
+	config_nand_single_cw_page_read(nandc);
 
 	read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer,
-		      nandc->buf_count);
+		      nandc->buf_count, 0);
 
 	/* restore CMD1 and VLD regs */
-	write_reg_dma(nandc, NAND_DEV_CMD1_RESTORE, 1);
-	write_reg_dma(nandc, NAND_DEV_CMD_VLD_RESTORE, 1);
+	write_reg_dma(nandc, NAND_DEV_CMD1_RESTORE, 1, 0);
+	write_reg_dma(nandc, NAND_DEV_CMD_VLD_RESTORE, 1, NAND_BAM_NEXT_SGL);
 
 	return 0;
 }
@@ -718,14 +1094,14 @@ static int erase_block(struct qcom_nand_host *host, int page_addr)
 	nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
 	nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);
 
-	write_reg_dma(nandc, NAND_FLASH_CMD, 3);
-	write_reg_dma(nandc, NAND_DEV0_CFG0, 2);
-	write_reg_dma(nandc, NAND_EXEC_CMD, 1);
+	write_reg_dma(nandc, NAND_FLASH_CMD, 3, NAND_BAM_NEXT_SGL);
+	write_reg_dma(nandc, NAND_DEV0_CFG0, 2, NAND_BAM_NEXT_SGL);
+	write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
 
-	read_reg_dma(nandc, NAND_FLASH_STATUS, 1);
+	read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
 
-	write_reg_dma(nandc, NAND_FLASH_STATUS, 1);
-	write_reg_dma(nandc, NAND_READ_STATUS, 1);
+	write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0);
+	write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL);
 
 	return 0;
 }
@@ -742,13 +1118,14 @@ static int read_id(struct qcom_nand_host *host, int column)
 	nandc_set_reg(nandc, NAND_FLASH_CMD, FETCH_ID);
 	nandc_set_reg(nandc, NAND_ADDR0, column);
 	nandc_set_reg(nandc, NAND_ADDR1, 0);
-	nandc_set_reg(nandc, NAND_FLASH_CHIP_SELECT, DM_EN);
+	nandc_set_reg(nandc, NAND_FLASH_CHIP_SELECT,
+		      nandc->props->is_bam ? 0 : DM_EN);
 	nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
 
-	write_reg_dma(nandc, NAND_FLASH_CMD, 4);
-	write_reg_dma(nandc, NAND_EXEC_CMD, 1);
+	write_reg_dma(nandc, NAND_FLASH_CMD, 4, NAND_BAM_NEXT_SGL);
+	write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
 
-	read_reg_dma(nandc, NAND_READ_ID, 1);
+	read_reg_dma(nandc, NAND_READ_ID, 1, NAND_BAM_NEXT_SGL);
 
 	return 0;
 }
@@ -762,10 +1139,10 @@ static int reset(struct qcom_nand_host *host)
 	nandc_set_reg(nandc, NAND_FLASH_CMD, RESET_DEVICE);
 	nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
 
-	write_reg_dma(nandc, NAND_FLASH_CMD, 1);
-	write_reg_dma(nandc, NAND_EXEC_CMD, 1);
+	write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
+	write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
 
-	read_reg_dma(nandc, NAND_FLASH_STATUS, 1);
+	read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
 
 	return 0;
 }
@@ -775,12 +1152,43 @@ static int submit_descs(struct qcom_nand_controller *nandc)
 {
 	struct desc_info *desc;
 	dma_cookie_t cookie = 0;
+	struct bam_transaction *bam_txn = nandc->bam_txn;
+	int r;
+
+	if (nandc->props->is_bam) {
+		if (bam_txn->rx_sgl_pos > bam_txn->rx_sgl_start) {
+			r = prepare_bam_async_desc(nandc, nandc->rx_chan, 0);
+			if (r)
+				return r;
+		}
+
+		if (bam_txn->tx_sgl_pos > bam_txn->tx_sgl_start) {
+			r = prepare_bam_async_desc(nandc, nandc->tx_chan,
+						   DMA_PREP_INTERRUPT);
+			if (r)
+				return r;
+		}
+
+		if (bam_txn->cmd_sgl_pos > bam_txn->cmd_sgl_start) {
+			r = prepare_bam_async_desc(nandc, nandc->cmd_chan, 0);
+			if (r)
+				return r;
+		}
+	}
 
 	list_for_each_entry(desc, &nandc->desc_list, node)
 		cookie = dmaengine_submit(desc->dma_desc);
 
-	if (dma_sync_wait(nandc->chan, cookie) != DMA_COMPLETE)
-		return -ETIMEDOUT;
+	if (nandc->props->is_bam) {
+		dma_async_issue_pending(nandc->tx_chan);
+		dma_async_issue_pending(nandc->rx_chan);
+
+		if (dma_sync_wait(nandc->cmd_chan, cookie) != DMA_COMPLETE)
+			return -ETIMEDOUT;
+	} else {
+		if (dma_sync_wait(nandc->chan, cookie) != DMA_COMPLETE)
+			return -ETIMEDOUT;
+	}
 
 	return 0;
 }
@@ -791,7 +1199,14 @@ static void free_descs(struct qcom_nand_controller *nandc)
 
 	list_for_each_entry_safe(desc, n, &nandc->desc_list, node) {
 		list_del(&desc->node);
-		dma_unmap_sg(nandc->dev, &desc->sgl, 1, desc->dir);
+
+		if (nandc->props->is_bam)
+			dma_unmap_sg(nandc->dev, desc->bam_sgl,
+				     desc->sgl_cnt, desc->dir);
+		else
+			dma_unmap_sg(nandc->dev, &desc->adm_sgl, 1,
+				     desc->dir);
+
 		kfree(desc);
 	}
 }
@@ -800,8 +1215,7 @@ static void free_descs(struct qcom_nand_controller *nandc)
 static void clear_read_regs(struct qcom_nand_controller *nandc)
 {
 	nandc->reg_read_pos = 0;
-	memset(nandc->reg_read_buf, 0,
-	       MAX_REG_RD * sizeof(*nandc->reg_read_buf));
+	nandc_read_buffer_sync(nandc, false);
 }
 
 static void pre_command(struct qcom_nand_host *host, int command)
@@ -815,6 +1229,10 @@ static void pre_command(struct qcom_nand_host *host, int command)
 	host->last_command = command;
 
 	clear_read_regs(nandc);
+
+	if (command == NAND_CMD_RESET || command == NAND_CMD_READID ||
+	    command == NAND_CMD_PARAM || command == NAND_CMD_ERASE1)
+		clear_bam_transaction(nandc);
 }
 
 /*
@@ -831,6 +1249,7 @@ static void parse_erase_write_errors(struct qcom_nand_host *host, int command)
 	int i;
 
 	num_cw = command == NAND_CMD_PAGEPROG ? ecc->steps : 1;
+	nandc_read_buffer_sync(nandc, true);
 
 	for (i = 0; i < num_cw; i++) {
 		u32 flash_status = le32_to_cpu(nandc->reg_read_buf[i]);
@@ -852,6 +1271,7 @@ static void post_command(struct qcom_nand_host *host, int command)
 
 	switch (command) {
 	case NAND_CMD_READID:
+		nandc_read_buffer_sync(nandc, true);
 		memcpy(nandc->data_buffer, nandc->reg_read_buf,
 		       nandc->buf_count);
 		break;
@@ -1015,6 +1435,7 @@ static int parse_read_errors(struct qcom_nand_host *host, u8 *data_buf,
 	int i;
 
 	buf = (struct read_stats *)nandc->reg_read_buf;
+	nandc_read_buffer_sync(nandc, true);
 
 	for (i = 0; i < ecc->steps; i++, buf++) {
 		u32 flash, buffer, erased_cw;
@@ -1102,6 +1523,8 @@ static int read_page_ecc(struct qcom_nand_host *host, u8 *data_buf,
 	struct nand_ecc_ctrl *ecc = &chip->ecc;
 	int i, ret;
 
+	config_nand_page_read(nandc);
+
 	/* queue cmd descs for each codeword */
 	for (i = 0; i < ecc->steps; i++) {
 		int data_size, oob_size;
@@ -1115,11 +1538,24 @@ static int read_page_ecc(struct qcom_nand_host *host, u8 *data_buf,
 			oob_size = host->ecc_bytes_hw + host->spare_bytes;
 		}
 
-		config_cw_read(nandc);
+		if (nandc->props->is_bam) {
+			if (data_buf && oob_buf) {
+				nandc_set_read_loc(nandc, 0, 0, data_size, 0);
+				nandc_set_read_loc(nandc, 1, data_size,
+						   oob_size, 1);
+			} else if (data_buf) {
+				nandc_set_read_loc(nandc, 0, 0, data_size, 1);
+			} else {
+				nandc_set_read_loc(nandc, 0, data_size,
+						   oob_size, 1);
+			}
+		}
+
+		config_nand_cw_read(nandc);
 
 		if (data_buf)
 			read_data_dma(nandc, FLASH_BUF_ACC, data_buf,
-				      data_size);
+				      data_size, 0);
 
 		/*
 		 * when ecc is enabled, the controller doesn't read the real
@@ -1135,7 +1571,7 @@ static int read_page_ecc(struct qcom_nand_host *host, u8 *data_buf,
 				*oob_buf++ = 0xff;
 
 			read_data_dma(nandc, FLASH_BUF_ACC + data_size,
-				      oob_buf, oob_size);
+				      oob_buf, oob_size, 0);
 		}
 
 		if (data_buf)
@@ -1175,9 +1611,9 @@ static int copy_last_cw(struct qcom_nand_host *host, int page)
 	set_address(host, host->cw_size * (ecc->steps - 1), page);
 	update_rw_regs(host, 1, true);
 
-	config_cw_read(nandc);
+	config_nand_single_cw_page_read(nandc);
 
-	read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, size);
+	read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, size, 0);
 
 	ret = submit_descs(nandc);
 	if (ret)
@@ -1200,6 +1636,7 @@ static int qcom_nandc_read_page(struct mtd_info *mtd, struct nand_chip *chip,
 	data_buf = buf;
 	oob_buf = oob_required ? chip->oob_poi : NULL;
 
+	clear_bam_transaction(nandc);
 	ret = read_page_ecc(host, data_buf, oob_buf);
 	if (ret) {
 		dev_err(nandc->dev, "failure to read page\n");
@@ -1219,12 +1656,16 @@ static int qcom_nandc_read_page_raw(struct mtd_info *mtd,
 	u8 *data_buf, *oob_buf;
 	struct nand_ecc_ctrl *ecc = &chip->ecc;
 	int i, ret;
+	int read_loc;
 
 	data_buf = buf;
 	oob_buf = chip->oob_poi;
 
 	host->use_ecc = false;
+
+	clear_bam_transaction(nandc);
 	update_rw_regs(host, ecc->steps, true);
+	config_nand_page_read(nandc);
 
 	for (i = 0; i < ecc->steps; i++) {
 		int data_size1, data_size2, oob_size1, oob_size2;
@@ -1243,21 +1684,35 @@ static int qcom_nandc_read_page_raw(struct mtd_info *mtd,
 			oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
 		}
 
-		config_cw_read(nandc);
+		if (nandc->props->is_bam) {
+			read_loc = 0;
+			nandc_set_read_loc(nandc, 0, read_loc, data_size1, 0);
+			read_loc += data_size1;
+
+			nandc_set_read_loc(nandc, 1, read_loc, oob_size1, 0);
+			read_loc += oob_size1;
+
+			nandc_set_read_loc(nandc, 2, read_loc, data_size2, 0);
+			read_loc += data_size2;
 
-		read_data_dma(nandc, reg_off, data_buf, data_size1);
+			nandc_set_read_loc(nandc, 3, read_loc, oob_size2, 1);
+		}
+
+		config_nand_cw_read(nandc);
+
+		read_data_dma(nandc, reg_off, data_buf, data_size1, 0);
 		reg_off += data_size1;
 		data_buf += data_size1;
 
-		read_data_dma(nandc, reg_off, oob_buf, oob_size1);
+		read_data_dma(nandc, reg_off, oob_buf, oob_size1, 0);
 		reg_off += oob_size1;
 		oob_buf += oob_size1;
 
-		read_data_dma(nandc, reg_off, data_buf, data_size2);
+		read_data_dma(nandc, reg_off, data_buf, data_size2, 0);
 		reg_off += data_size2;
 		data_buf += data_size2;
 
-		read_data_dma(nandc, reg_off, oob_buf, oob_size2);
+		read_data_dma(nandc, reg_off, oob_buf, oob_size2, 0);
 		oob_buf += oob_size2;
 	}
 
@@ -1280,6 +1735,7 @@ static int qcom_nandc_read_oob(struct mtd_info *mtd, struct nand_chip *chip,
 	int ret;
 
 	clear_read_regs(nandc);
+	clear_bam_transaction(nandc);
 
 	host->use_ecc = true;
 	set_address(host, 0, page);
@@ -1303,12 +1759,14 @@ static int qcom_nandc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
 	int i, ret;
 
 	clear_read_regs(nandc);
+	clear_bam_transaction(nandc);
 
 	data_buf = (u8 *)buf;
 	oob_buf = chip->oob_poi;
 
 	host->use_ecc = true;
 	update_rw_regs(host, ecc->steps, false);
+	config_nand_page_write(nandc);
 
 	for (i = 0; i < ecc->steps; i++) {
 		int data_size, oob_size;
@@ -1322,9 +1780,9 @@ static int qcom_nandc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
 			oob_size = ecc->bytes;
 		}
 
-		config_cw_write_pre(nandc);
 
-		write_data_dma(nandc, FLASH_BUF_ACC, data_buf, data_size);
+		write_data_dma(nandc, FLASH_BUF_ACC, data_buf, data_size,
+			       i == (ecc->steps - 1) ? NAND_BAM_NO_EOT : 0);
 
 		/*
 		 * when ECC is enabled, we don't really need to write anything
@@ -1337,10 +1795,10 @@ static int qcom_nandc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
 			oob_buf += host->bbm_size;
 
 			write_data_dma(nandc, FLASH_BUF_ACC + data_size,
-				       oob_buf, oob_size);
+				       oob_buf, oob_size, 0);
 		}
 
-		config_cw_write_post(nandc);
+		config_nand_cw_write(nandc);
 
 		data_buf += data_size;
 		oob_buf += oob_size;
@@ -1367,12 +1825,14 @@ static int qcom_nandc_write_page_raw(struct mtd_info *mtd,
 	int i, ret;
 
 	clear_read_regs(nandc);
+	clear_bam_transaction(nandc);
 
 	data_buf = (u8 *)buf;
 	oob_buf = chip->oob_poi;
 
 	host->use_ecc = false;
 	update_rw_regs(host, ecc->steps, false);
+	config_nand_page_write(nandc);
 
 	for (i = 0; i < ecc->steps; i++) {
 		int data_size1, data_size2, oob_size1, oob_size2;
@@ -1391,24 +1851,25 @@ static int qcom_nandc_write_page_raw(struct mtd_info *mtd,
 			oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
 		}
 
-		config_cw_write_pre(nandc);
-
-		write_data_dma(nandc, reg_off, data_buf, data_size1);
+		write_data_dma(nandc, reg_off, data_buf, data_size1,
+			       NAND_BAM_NO_EOT);
 		reg_off += data_size1;
 		data_buf += data_size1;
 
-		write_data_dma(nandc, reg_off, oob_buf, oob_size1);
+		write_data_dma(nandc, reg_off, oob_buf, oob_size1,
+			       NAND_BAM_NO_EOT);
 		reg_off += oob_size1;
 		oob_buf += oob_size1;
 
-		write_data_dma(nandc, reg_off, data_buf, data_size2);
+		write_data_dma(nandc, reg_off, data_buf, data_size2,
+			       NAND_BAM_NO_EOT);
 		reg_off += data_size2;
 		data_buf += data_size2;
 
-		write_data_dma(nandc, reg_off, oob_buf, oob_size2);
+		write_data_dma(nandc, reg_off, oob_buf, oob_size2, 0);
 		oob_buf += oob_size2;
 
-		config_cw_write_post(nandc);
+		config_nand_cw_write(nandc);
 	}
 
 	ret = submit_descs(nandc);
@@ -1441,11 +1902,13 @@ static int qcom_nandc_write_oob(struct mtd_info *mtd, struct nand_chip *chip,
 
 	host->use_ecc = true;
 
+	clear_bam_transaction(nandc);
 	ret = copy_last_cw(host, page);
 	if (ret)
 		return ret;
 
 	clear_read_regs(nandc);
+	clear_bam_transaction(nandc);
 
 	/* calculate the data and oob size for the last codeword/step */
 	data_size = ecc->size - ((ecc->steps - 1) << 2);
@@ -1458,10 +1921,10 @@ static int qcom_nandc_write_oob(struct mtd_info *mtd, struct nand_chip *chip,
 	set_address(host, host->cw_size * (ecc->steps - 1), page);
 	update_rw_regs(host, 1, false);
 
-	config_cw_write_pre(nandc);
-	write_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer,
-		       data_size + oob_size);
-	config_cw_write_post(nandc);
+	config_nand_page_write(nandc);
+	write_data_dma(nandc, FLASH_BUF_ACC,
+		       nandc->data_buffer, data_size + oob_size, 0);
+	config_nand_cw_write(nandc);
 
 	ret = submit_descs(nandc);
 
@@ -1498,6 +1961,7 @@ static int qcom_nandc_block_bad(struct mtd_info *mtd, loff_t ofs)
 	 */
 	host->use_ecc = false;
 
+	clear_bam_transaction(nandc);
 	ret = copy_last_cw(host, page);
 	if (ret)
 		goto err;
@@ -1528,6 +1992,7 @@ static int qcom_nandc_block_markbad(struct mtd_info *mtd, loff_t ofs)
 	int page, ret, status = 0;
 
 	clear_read_regs(nandc);
+	clear_bam_transaction(nandc);
 
 	/*
 	 * to mark the BBM as bad, we flash the entire last codeword with 0s.
@@ -1543,9 +2008,10 @@ static int qcom_nandc_block_markbad(struct mtd_info *mtd, loff_t ofs)
 	set_address(host, host->cw_size * (ecc->steps - 1), page);
 	update_rw_regs(host, 1, false);
 
-	config_cw_write_pre(nandc);
-	write_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, host->cw_size);
-	config_cw_write_post(nandc);
+	config_nand_page_write(nandc);
+	write_data_dma(nandc, FLASH_BUF_ACC,
+		       nandc->data_buffer, host->cw_size, 0);
+	config_nand_cw_write(nandc);
 
 	ret = submit_descs(nandc);
 
@@ -1794,7 +2260,7 @@ static int qcom_nand_host_setup(struct qcom_nand_host *host)
 		 * uses lesser bytes for ECC. If RS is used, the ECC bytes is
 		 * always 10 bytes
 		 */
-		if (nandc->ecc_modes & ECC_BCH_4BIT) {
+		if (nandc->props->ecc_modes & ECC_BCH_4BIT) {
 			/* BCH */
 			host->bch_enabled = true;
 			ecc_mode = 0;
@@ -1842,6 +2308,8 @@ static int qcom_nand_host_setup(struct qcom_nand_host *host)
 	mtd_set_ooblayout(mtd, &qcom_nand_ooblayout_ops);
 
 	cwperpage = mtd->writesize / ecc->size;
+	nandc->max_cwperpage = max_t(unsigned int, nandc->max_cwperpage,
+				     cwperpage);
 
 	/*
 	 * DATA_UD_BYTES varies based on whether the read/write command protects
@@ -1893,7 +2361,7 @@ static int qcom_nand_host_setup(struct qcom_nand_host *host)
 				| wide_bus << WIDE_FLASH
 				| 1 << DEV0_CFG1_ECC_DISABLE;
 
-	host->ecc_bch_cfg = host->bch_enabled << ECC_CFG_ECC_DISABLE
+	host->ecc_bch_cfg = !host->bch_enabled << ECC_CFG_ECC_DISABLE
 				| 0 << ECC_SW_RESET
 				| host->cw_data << ECC_NUM_DATA_BYTES
 				| 1 << ECC_FORCE_CLK_OPEN
@@ -1904,6 +2372,10 @@ static int qcom_nand_host_setup(struct qcom_nand_host *host)
 
 	host->clrflashstatus = FS_READY_BSY_N;
 	host->clrreadstatus = 0xc0;
+	nandc->regs->erased_cw_detect_cfg_clr =
+		cpu_to_le32(CLR_ERASED_PAGE_DET);
+	nandc->regs->erased_cw_detect_cfg_set =
+		cpu_to_le32(SET_ERASED_PAGE_DET);
 
 	dev_dbg(nandc->dev,
 		"cfg0 %x cfg1 %x ecc_buf_cfg %x ecc_bch cfg %x cw_size %d cw_data %d strength %d parity_bytes %d steps %d\n",
@@ -1948,10 +2420,55 @@ static int qcom_nandc_alloc(struct qcom_nand_controller *nandc)
 	if (!nandc->reg_read_buf)
 		return -ENOMEM;
 
-	nandc->chan = dma_request_slave_channel(nandc->dev, "rxtx");
-	if (!nandc->chan) {
-		dev_err(nandc->dev, "failed to request slave channel\n");
-		return -ENODEV;
+	if (nandc->props->is_bam) {
+		nandc->reg_read_dma =
+			dma_map_single(nandc->dev, nandc->reg_read_buf,
+				       MAX_REG_RD *
+				       sizeof(*nandc->reg_read_buf),
+				       DMA_FROM_DEVICE);
+		if (dma_mapping_error(nandc->dev, nandc->reg_read_dma)) {
+			dev_err(nandc->dev, "failed to DMA MAP reg buffer\n");
+			return -EIO;
+		}
+
+		nandc->tx_chan = dma_request_slave_channel(nandc->dev, "tx");
+		if (!nandc->tx_chan) {
+			dev_err(nandc->dev, "failed to request tx channel\n");
+			return -ENODEV;
+		}
+
+		nandc->rx_chan = dma_request_slave_channel(nandc->dev, "rx");
+		if (!nandc->rx_chan) {
+			dev_err(nandc->dev, "failed to request rx channel\n");
+			return -ENODEV;
+		}
+
+		nandc->cmd_chan = dma_request_slave_channel(nandc->dev, "cmd");
+		if (!nandc->cmd_chan) {
+			dev_err(nandc->dev, "failed to request cmd channel\n");
+			return -ENODEV;
+		}
+
+		/*
+		 * Initially allocate BAM transaction to read ONFI param page.
+		 * After detecting all the devices, this BAM transaction will
+		 * be freed and the next BAM tranasction will be allocated with
+		 * maximum codeword size
+		 */
+		nandc->max_cwperpage = 1;
+		nandc->bam_txn = alloc_bam_transaction(nandc);
+		if (!nandc->bam_txn) {
+			dev_err(nandc->dev,
+				"failed to allocate bam transaction\n");
+			return -ENOMEM;
+		}
+	} else {
+		nandc->chan = dma_request_slave_channel(nandc->dev, "rxtx");
+		if (!nandc->chan) {
+			dev_err(nandc->dev,
+				"failed to request slave channel\n");
+			return -ENODEV;
+		}
 	}
 
 	INIT_LIST_HEAD(&nandc->desc_list);
@@ -1964,21 +2481,48 @@ static int qcom_nandc_alloc(struct qcom_nand_controller *nandc)
 
 static void qcom_nandc_unalloc(struct qcom_nand_controller *nandc)
 {
-	dma_release_channel(nandc->chan);
+	if (nandc->props->is_bam) {
+		if (!dma_mapping_error(nandc->dev, nandc->reg_read_dma))
+			dma_unmap_single(nandc->dev, nandc->reg_read_dma,
+					 MAX_REG_RD *
+					 sizeof(*nandc->reg_read_buf),
+					 DMA_FROM_DEVICE);
+
+		if (nandc->tx_chan)
+			dma_release_channel(nandc->tx_chan);
+
+		if (nandc->rx_chan)
+			dma_release_channel(nandc->rx_chan);
+
+		if (nandc->cmd_chan)
+			dma_release_channel(nandc->cmd_chan);
+	} else {
+		if (nandc->chan)
+			dma_release_channel(nandc->chan);
+	}
 }
 
 /* one time setup of a few nand controller registers */
 static int qcom_nandc_setup(struct qcom_nand_controller *nandc)
 {
+	u32 nand_ctrl;
+
 	/* kill onenand */
 	nandc_write(nandc, SFLASHC_BURST_CFG, 0);
+	nandc_write(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD),
+		    NAND_DEV_CMD_VLD_VAL);
 
-	/* enable ADM DMA */
-	nandc_write(nandc, NAND_FLASH_CHIP_SELECT, DM_EN);
+	/* enable ADM or BAM DMA */
+	if (nandc->props->is_bam) {
+		nand_ctrl = nandc_read(nandc, NAND_CTRL);
+		nandc_write(nandc, NAND_CTRL, nand_ctrl | BAM_MODE_EN);
+	} else {
+		nandc_write(nandc, NAND_FLASH_CHIP_SELECT, DM_EN);
+	}
 
 	/* save the original values of these registers */
-	nandc->cmd1 = nandc_read(nandc, NAND_DEV_CMD1);
-	nandc->vld = nandc_read(nandc, NAND_DEV_CMD_VLD);
+	nandc->cmd1 = nandc_read(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD1));
+	nandc->vld = NAND_DEV_CMD_VLD_VAL;
 
 	return 0;
 }
@@ -2034,14 +2578,77 @@ static int qcom_nand_host_init(struct qcom_nand_controller *nandc,
 		return ret;
 
 	ret = qcom_nand_host_setup(host);
-	if (ret)
-		return ret;
+
+	return ret;
+}
+
+static int qcom_nand_mtd_register(struct qcom_nand_controller *nandc,
+				  struct qcom_nand_host *host,
+				  struct device_node *dn)
+{
+	struct nand_chip *chip = &host->chip;
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	int ret;
 
 	ret = nand_scan_tail(mtd);
 	if (ret)
 		return ret;
 
-	return mtd_device_register(mtd, NULL, 0);
+	ret = mtd_device_register(mtd, NULL, 0);
+	if (ret)
+		nand_cleanup(mtd_to_nand(mtd));
+
+	return ret;
+}
+
+static int qcom_probe_nand_devices(struct qcom_nand_controller *nandc)
+{
+	struct device *dev = nandc->dev;
+	struct device_node *dn = dev->of_node, *child;
+	struct qcom_nand_host *host, *tmp;
+	int ret;
+
+	for_each_available_child_of_node(dn, child) {
+		host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
+		if (!host) {
+			of_node_put(child);
+			return -ENOMEM;
+		}
+
+		ret = qcom_nand_host_init(nandc, host, child);
+		if (ret) {
+			devm_kfree(dev, host);
+			continue;
+		}
+
+		list_add_tail(&host->node, &nandc->host_list);
+	}
+
+	if (list_empty(&nandc->host_list))
+		return -ENODEV;
+
+	if (nandc->props->is_bam) {
+		free_bam_transaction(nandc);
+		nandc->bam_txn = alloc_bam_transaction(nandc);
+		if (!nandc->bam_txn) {
+			dev_err(nandc->dev,
+				"failed to allocate bam transaction\n");
+			return -ENOMEM;
+		}
+	}
+
+	list_for_each_entry_safe(host, tmp, &nandc->host_list, node) {
+		ret = qcom_nand_mtd_register(nandc, host, child);
+		if (ret) {
+			list_del(&host->node);
+			devm_kfree(dev, host);
+		}
+	}
+
+	if (list_empty(&nandc->host_list))
+		return -ENODEV;
+
+	return 0;
 }
 
 /* parse custom DT properties here */
@@ -2051,16 +2658,20 @@ static int qcom_nandc_parse_dt(struct platform_device *pdev)
 	struct device_node *np = nandc->dev->of_node;
 	int ret;
 
-	ret = of_property_read_u32(np, "qcom,cmd-crci", &nandc->cmd_crci);
-	if (ret) {
-		dev_err(nandc->dev, "command CRCI unspecified\n");
-		return ret;
-	}
+	if (!nandc->props->is_bam) {
+		ret = of_property_read_u32(np, "qcom,cmd-crci",
+					   &nandc->cmd_crci);
+		if (ret) {
+			dev_err(nandc->dev, "command CRCI unspecified\n");
+			return ret;
+		}
 
-	ret = of_property_read_u32(np, "qcom,data-crci", &nandc->data_crci);
-	if (ret) {
-		dev_err(nandc->dev, "data CRCI unspecified\n");
-		return ret;
+		ret = of_property_read_u32(np, "qcom,data-crci",
+					   &nandc->data_crci);
+		if (ret) {
+			dev_err(nandc->dev, "data CRCI unspecified\n");
+			return ret;
+		}
 	}
 
 	return 0;
@@ -2069,10 +2680,8 @@ static int qcom_nandc_parse_dt(struct platform_device *pdev)
 static int qcom_nandc_probe(struct platform_device *pdev)
 {
 	struct qcom_nand_controller *nandc;
-	struct qcom_nand_host *host;
 	const void *dev_data;
 	struct device *dev = &pdev->dev;
-	struct device_node *dn = dev->of_node, *child;
 	struct resource *res;
 	int ret;
 
@@ -2089,7 +2698,7 @@ static int qcom_nandc_probe(struct platform_device *pdev)
 		return -ENODEV;
 	}
 
-	nandc->ecc_modes = (unsigned long)dev_data;
+	nandc->props = dev_data;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	nandc->base = devm_ioremap_resource(dev, res);
@@ -2112,7 +2721,7 @@ static int qcom_nandc_probe(struct platform_device *pdev)
 
 	ret = qcom_nandc_alloc(nandc);
 	if (ret)
-		return ret;
+		goto err_core_clk;
 
 	ret = clk_prepare_enable(nandc->core_clk);
 	if (ret)
@@ -2126,35 +2735,12 @@ static int qcom_nandc_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_setup;
 
-	for_each_available_child_of_node(dn, child) {
-		if (of_device_is_compatible(child, "qcom,nandcs")) {
-			host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
-			if (!host) {
-				of_node_put(child);
-				ret = -ENOMEM;
-				goto err_cs_init;
-			}
-
-			ret = qcom_nand_host_init(nandc, host, child);
-			if (ret) {
-				devm_kfree(dev, host);
-				continue;
-			}
-
-			list_add_tail(&host->node, &nandc->host_list);
-		}
-	}
-
-	if (list_empty(&nandc->host_list)) {
-		ret = -ENODEV;
-		goto err_cs_init;
-	}
+	ret = qcom_probe_nand_devices(nandc);
+	if (ret)
+		goto err_setup;
 
 	return 0;
 
-err_cs_init:
-	list_for_each_entry(host, &nandc->host_list, node)
-		nand_release(nand_to_mtd(&host->chip));
 err_setup:
 	clk_disable_unprepare(nandc->aon_clk);
 err_aon_clk:
@@ -2181,15 +2767,40 @@ static int qcom_nandc_remove(struct platform_device *pdev)
 	return 0;
 }
 
-#define EBI2_NANDC_ECC_MODES	(ECC_RS_4BIT | ECC_BCH_8BIT)
+static const struct qcom_nandc_props ipq806x_nandc_props = {
+	.ecc_modes = (ECC_RS_4BIT | ECC_BCH_8BIT),
+	.is_bam = false,
+	.dev_cmd_reg_start = 0x0,
+};
+
+static const struct qcom_nandc_props ipq4019_nandc_props = {
+	.ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT),
+	.is_bam = true,
+	.dev_cmd_reg_start = 0x0,
+};
+
+static const struct qcom_nandc_props ipq8074_nandc_props = {
+	.ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT),
+	.is_bam = true,
+	.dev_cmd_reg_start = 0x7000,
+};
 
 /*
  * data will hold a struct pointer containing more differences once we support
  * more controller variants
  */
 static const struct of_device_id qcom_nandc_of_match[] = {
-	{	.compatible = "qcom,ipq806x-nand",
-		.data = (void *)EBI2_NANDC_ECC_MODES,
+	{
+		.compatible = "qcom,ipq806x-nand",
+		.data = &ipq806x_nandc_props,
+	},
+	{
+		.compatible = "qcom,ipq4019-nand",
+		.data = &ipq4019_nandc_props,
+	},
+	{
+		.compatible = "qcom,ipq8074-nand",
+		.data = &ipq8074_nandc_props,
 	},
 	{}
 };
diff --git a/drivers/mtd/nand/r852.h b/drivers/mtd/nand/r852.h
index d042ddb71a8b..8713c57f6207 100644
--- a/drivers/mtd/nand/r852.h
+++ b/drivers/mtd/nand/r852.h
@@ -10,7 +10,7 @@
 #include <linux/pci.h>
 #include <linux/completion.h>
 #include <linux/workqueue.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/spinlock.h>
 
 
diff --git a/drivers/mtd/nand/s3c2410.c b/drivers/mtd/nand/s3c2410.c
index 9e0c849607b9..4c383eeec6f6 100644
--- a/drivers/mtd/nand/s3c2410.c
+++ b/drivers/mtd/nand/s3c2410.c
@@ -43,7 +43,7 @@
 #include <linux/of_device.h>
 
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/nand_ecc.h>
 #include <linux/mtd/partitions.h>
 
diff --git a/drivers/mtd/nand/sh_flctl.c b/drivers/mtd/nand/sh_flctl.c
index 891ac7b99305..e7f3c98487e6 100644
--- a/drivers/mtd/nand/sh_flctl.c
+++ b/drivers/mtd/nand/sh_flctl.c
@@ -38,7 +38,7 @@
 #include <linux/string.h>
 
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 #include <linux/mtd/sh_flctl.h>
 
@@ -411,7 +411,7 @@ static int flctl_dma_fifo0_transfer(struct sh_flctl *flctl, unsigned long *buf,
 
 	dma_addr = dma_map_single(chan->device->dev, buf, len, dir);
 
-	if (dma_addr)
+	if (!dma_mapping_error(chan->device->dev, dma_addr))
 		desc = dmaengine_prep_slave_single(chan, dma_addr, len,
 			tr_dir, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 
@@ -1141,8 +1141,8 @@ static int flctl_probe(struct platform_device *pdev)
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
-		dev_err(&pdev->dev, "failed to get flste irq data\n");
-		return -ENXIO;
+		dev_err(&pdev->dev, "failed to get flste irq data: %d\n", irq);
+		return irq;
 	}
 
 	ret = devm_request_irq(&pdev->dev, irq, flctl_handle_flste, IRQF_SHARED,
diff --git a/drivers/mtd/nand/sharpsl.c b/drivers/mtd/nand/sharpsl.c
index 064ca1757589..f59c455d9f51 100644
--- a/drivers/mtd/nand/sharpsl.c
+++ b/drivers/mtd/nand/sharpsl.c
@@ -17,7 +17,7 @@
 #include <linux/module.h>
 #include <linux/delay.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/nand_ecc.h>
 #include <linux/mtd/partitions.h>
 #include <linux/mtd/sharpsl.h>
@@ -183,7 +183,7 @@ static int sharpsl_nand_probe(struct platform_device *pdev)
 	/* Register the partitions */
 	mtd->name = "sharpsl-nand";
 
-	err = mtd_device_parse_register(mtd, NULL, NULL,
+	err = mtd_device_parse_register(mtd, data->part_parsers, NULL,
 					data->partitions, data->nr_partitions);
 	if (err)
 		goto err_add;
diff --git a/drivers/mtd/nand/sm_common.c b/drivers/mtd/nand/sm_common.c
index 5939dff253c2..c378705c6e2b 100644
--- a/drivers/mtd/nand/sm_common.c
+++ b/drivers/mtd/nand/sm_common.c
@@ -7,7 +7,7 @@
  * published by the Free Software Foundation.
  */
 #include <linux/kernel.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/module.h>
 #include <linux/sizes.h>
 #include "sm_common.h"
diff --git a/drivers/mtd/nand/socrates_nand.c b/drivers/mtd/nand/socrates_nand.c
index 72369bd079af..575997d0ef8a 100644
--- a/drivers/mtd/nand/socrates_nand.c
+++ b/drivers/mtd/nand/socrates_nand.c
@@ -13,7 +13,7 @@
 #include <linux/slab.h>
 #include <linux/module.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 #include <linux/of_address.h>
 #include <linux/of_platform.h>
diff --git a/drivers/mtd/nand/sunxi_nand.c b/drivers/mtd/nand/sunxi_nand.c
index 6abd142b1324..82244be3e766 100644
--- a/drivers/mtd/nand/sunxi_nand.c
+++ b/drivers/mtd/nand/sunxi_nand.c
@@ -31,7 +31,7 @@
 #include <linux/of_device.h>
 #include <linux/of_gpio.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
@@ -2212,7 +2212,7 @@ static int sunxi_nfc_probe(struct platform_device *pdev)
 	if (ret)
 		goto out_ahb_clk_unprepare;
 
-	nfc->reset = devm_reset_control_get_optional(dev, "ahb");
+	nfc->reset = devm_reset_control_get_optional_exclusive(dev, "ahb");
 	if (IS_ERR(nfc->reset)) {
 		ret = PTR_ERR(nfc->reset);
 		goto out_mod_clk_unprepare;
diff --git a/drivers/mtd/nand/tango_nand.c b/drivers/mtd/nand/tango_nand.c
index 9d40b793b1c4..766906f03943 100644
--- a/drivers/mtd/nand/tango_nand.c
+++ b/drivers/mtd/nand/tango_nand.c
@@ -11,7 +11,7 @@
 #include <linux/clk.h>
 #include <linux/iopoll.h>
 #include <linux/module.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/dmaengine.h>
 #include <linux/dma-mapping.h>
 #include <linux/platform_device.h>
diff --git a/drivers/mtd/nand/tmio_nand.c b/drivers/mtd/nand/tmio_nand.c
index fc5e773f8b60..84dbf32332e1 100644
--- a/drivers/mtd/nand/tmio_nand.c
+++ b/drivers/mtd/nand/tmio_nand.c
@@ -34,7 +34,7 @@
 #include <linux/interrupt.h>
 #include <linux/ioport.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/nand_ecc.h>
 #include <linux/mtd/partitions.h>
 #include <linux/slab.h>
@@ -440,7 +440,9 @@ static int tmio_probe(struct platform_device *dev)
 		goto err_irq;
 
 	/* Register the partitions */
-	retval = mtd_device_parse_register(mtd, NULL, NULL,
+	retval = mtd_device_parse_register(mtd,
+					   data ? data->part_parsers : NULL,
+					   NULL,
 					   data ? data->partition : NULL,
 					   data ? data->num_partitions : 0);
 	if (!retval)
diff --git a/drivers/mtd/nand/txx9ndfmc.c b/drivers/mtd/nand/txx9ndfmc.c
index 0a14fda2e41b..b567d212fe7d 100644
--- a/drivers/mtd/nand/txx9ndfmc.c
+++ b/drivers/mtd/nand/txx9ndfmc.c
@@ -16,7 +16,7 @@
 #include <linux/platform_device.h>
 #include <linux/delay.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/nand_ecc.h>
 #include <linux/mtd/partitions.h>
 #include <linux/io.h>
diff --git a/drivers/mtd/nand/vf610_nfc.c b/drivers/mtd/nand/vf610_nfc.c
index 744ab10e8962..8037d4b48a05 100644
--- a/drivers/mtd/nand/vf610_nfc.c
+++ b/drivers/mtd/nand/vf610_nfc.c
@@ -31,10 +31,9 @@
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 #include <linux/of_device.h>
-#include <linux/pinctrl/consumer.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 
@@ -814,12 +813,14 @@ static int vf610_nfc_suspend(struct device *dev)
 
 static int vf610_nfc_resume(struct device *dev)
 {
+	int err;
+
 	struct mtd_info *mtd = dev_get_drvdata(dev);
 	struct vf610_nfc *nfc = mtd_to_nfc(mtd);
 
-	pinctrl_pm_select_default_state(dev);
-
-	clk_prepare_enable(nfc->clk);
+	err = clk_prepare_enable(nfc->clk);
+	if (err)
+		return err;
 
 	vf610_nfc_preinit_controller(nfc);
 	vf610_nfc_init_controller(nfc);
diff --git a/drivers/mtd/nand/xway_nand.c b/drivers/mtd/nand/xway_nand.c
index ddee4005248c..9926b4e3d69d 100644
--- a/drivers/mtd/nand/xway_nand.c
+++ b/drivers/mtd/nand/xway_nand.c
@@ -7,7 +7,7 @@
  *  Copyright © 2016 Hauke Mehrtens <hauke@hauke-m.de>
  */
 
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/of_gpio.h>
 #include <linux/of_platform.h>
 
diff --git a/drivers/mtd/nftlcore.c b/drivers/mtd/nftlcore.c
index e21161353e76..1f1a61168b3d 100644
--- a/drivers/mtd/nftlcore.c
+++ b/drivers/mtd/nftlcore.c
@@ -34,7 +34,7 @@
 
 #include <linux/kmod.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/nftl.h>
 #include <linux/mtd/blktrans.h>
 
diff --git a/drivers/mtd/nftlmount.c b/drivers/mtd/nftlmount.c
index a5dfbfbebfca..184c8fbfe465 100644
--- a/drivers/mtd/nftlmount.c
+++ b/drivers/mtd/nftlmount.c
@@ -25,7 +25,7 @@
 #include <linux/delay.h>
 #include <linux/slab.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/nftl.h>
 
 #define SECTORSIZE 512
diff --git a/drivers/mtd/ofpart.c b/drivers/mtd/ofpart.c
index 2861c7079d7b..6bdf4e525677 100644
--- a/drivers/mtd/ofpart.c
+++ b/drivers/mtd/ofpart.c
@@ -50,8 +50,8 @@ static int parse_ofpart_partitions(struct mtd_info *master,
 		 * when using another parser), so don't be louder than
 		 * KERN_DEBUG
 		 */
-		pr_debug("%s: 'partitions' subnode not found on %s. Trying to parse direct subnodes as partitions.\n",
-			 master->name, mtd_node->full_name);
+		pr_debug("%s: 'partitions' subnode not found on %pOF. Trying to parse direct subnodes as partitions.\n",
+			 master->name, mtd_node);
 		ofpart_node = mtd_node;
 		dedicated = false;
 	} else if (!of_device_is_compatible(ofpart_node, "fixed-partitions")) {
@@ -87,9 +87,9 @@ static int parse_ofpart_partitions(struct mtd_info *master,
 		reg = of_get_property(pp, "reg", &len);
 		if (!reg) {
 			if (dedicated) {
-				pr_debug("%s: ofpart partition %s (%s) missing reg property.\n",
-					 master->name, pp->full_name,
-					 mtd_node->full_name);
+				pr_debug("%s: ofpart partition %pOF (%pOF) missing reg property.\n",
+					 master->name, pp,
+					 mtd_node);
 				goto ofpart_fail;
 			} else {
 				nr_parts--;
@@ -100,9 +100,9 @@ static int parse_ofpart_partitions(struct mtd_info *master,
 		a_cells = of_n_addr_cells(pp);
 		s_cells = of_n_size_cells(pp);
 		if (len / 4 != a_cells + s_cells) {
-			pr_debug("%s: ofpart partition %s (%s) error parsing reg property.\n",
-				 master->name, pp->full_name,
-				 mtd_node->full_name);
+			pr_debug("%s: ofpart partition %pOF (%pOF) error parsing reg property.\n",
+				 master->name, pp,
+				 mtd_node);
 			goto ofpart_fail;
 		}
 
@@ -131,8 +131,8 @@ static int parse_ofpart_partitions(struct mtd_info *master,
 	return nr_parts;
 
 ofpart_fail:
-	pr_err("%s: error parsing ofpart partition %s (%s)\n",
-	       master->name, pp->full_name, mtd_node->full_name);
+	pr_err("%s: error parsing ofpart partition %pOF (%pOF)\n",
+	       master->name, pp, mtd_node);
 	ret = -EINVAL;
 ofpart_none:
 	of_node_put(pp);
@@ -166,8 +166,7 @@ static int parse_ofoldpart_partitions(struct mtd_info *master,
 	if (!part)
 		return 0; /* No partitions found */
 
-	pr_warn("Device tree uses obsolete partition map binding: %s\n",
-		dp->full_name);
+	pr_warn("Device tree uses obsolete partition map binding: %pOF\n", dp);
 
 	nr_parts = plen / sizeof(part[0]);
 
diff --git a/drivers/mtd/spi-nor/Kconfig b/drivers/mtd/spi-nor/Kconfig
index 293c8a4d1e49..69c638dd0484 100644
--- a/drivers/mtd/spi-nor/Kconfig
+++ b/drivers/mtd/spi-nor/Kconfig
@@ -89,6 +89,22 @@ config SPI_NXP_SPIFI
 config SPI_INTEL_SPI
 	tristate
 
+config SPI_INTEL_SPI_PCI
+	tristate "Intel PCH/PCU SPI flash PCI driver" if EXPERT
+	depends on X86 && PCI
+	select SPI_INTEL_SPI
+	help
+	  This enables PCI support for the Intel PCH/PCU SPI controller in
+	  master mode. This controller is present in modern Intel hardware
+	  and is used to hold BIOS and other persistent settings. Using
+	  this driver it is possible to upgrade BIOS directly from Linux.
+
+	  Say N here unless you know what you are doing. Overwriting the
+	  SPI flash may render the system unbootable.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called intel-spi-pci.
+
 config SPI_INTEL_SPI_PLATFORM
 	tristate "Intel PCH/PCU SPI flash platform driver" if EXPERT
 	depends on X86
diff --git a/drivers/mtd/spi-nor/Makefile b/drivers/mtd/spi-nor/Makefile
index 285aab86c7ca..7d84c5108e17 100644
--- a/drivers/mtd/spi-nor/Makefile
+++ b/drivers/mtd/spi-nor/Makefile
@@ -7,5 +7,6 @@ obj-$(CONFIG_SPI_HISI_SFC)	+= hisi-sfc.o
 obj-$(CONFIG_MTD_MT81xx_NOR)    += mtk-quadspi.o
 obj-$(CONFIG_SPI_NXP_SPIFI)	+= nxp-spifi.o
 obj-$(CONFIG_SPI_INTEL_SPI)	+= intel-spi.o
+obj-$(CONFIG_SPI_INTEL_SPI_PCI)	+= intel-spi-pci.o
 obj-$(CONFIG_SPI_INTEL_SPI_PLATFORM)	+= intel-spi-platform.o
-obj-$(CONFIG_SPI_STM32_QUADSPI)	+= stm32-quadspi.o
\ No newline at end of file
+obj-$(CONFIG_SPI_STM32_QUADSPI)	+= stm32-quadspi.o
diff --git a/drivers/mtd/spi-nor/aspeed-smc.c b/drivers/mtd/spi-nor/aspeed-smc.c
index 0106357421bd..8d3cbe27efb6 100644
--- a/drivers/mtd/spi-nor/aspeed-smc.c
+++ b/drivers/mtd/spi-nor/aspeed-smc.c
@@ -621,19 +621,18 @@ static void aspeed_smc_chip_set_type(struct aspeed_smc_chip *chip, int type)
 }
 
 /*
- * The AST2500 FMC flash controller should be strapped by hardware, or
- * autodetected, but the AST2500 SPI flash needs to be set.
+ * The first chip of the AST2500 FMC flash controller is strapped by
+ * hardware, or autodetected, but other chips need to be set. Enforce
+ * the 4B setting for all chips.
  */
 static void aspeed_smc_chip_set_4b(struct aspeed_smc_chip *chip)
 {
 	struct aspeed_smc_controller *controller = chip->controller;
 	u32 reg;
 
-	if (chip->controller->info == &spi_2500_info) {
-		reg = readl(controller->regs + CE_CONTROL_REG);
-		reg |= 1 << chip->cs;
-		writel(reg, controller->regs + CE_CONTROL_REG);
-	}
+	reg = readl(controller->regs + CE_CONTROL_REG);
+	reg |= 1 << chip->cs;
+	writel(reg, controller->regs + CE_CONTROL_REG);
 }
 
 /*
diff --git a/drivers/mtd/spi-nor/atmel-quadspi.c b/drivers/mtd/spi-nor/atmel-quadspi.c
index ba76fa8f2031..6c5708bacad8 100644
--- a/drivers/mtd/spi-nor/atmel-quadspi.c
+++ b/drivers/mtd/spi-nor/atmel-quadspi.c
@@ -35,7 +35,6 @@
 
 #include <linux/io.h>
 #include <linux/gpio.h>
-#include <linux/pinctrl/consumer.h>
 
 /* QSPI register offsets */
 #define QSPI_CR      0x0000  /* Control Register */
diff --git a/drivers/mtd/spi-nor/hisi-sfc.c b/drivers/mtd/spi-nor/hisi-sfc.c
index d1106832b9d5..04f9fb5cd9b6 100644
--- a/drivers/mtd/spi-nor/hisi-sfc.c
+++ b/drivers/mtd/spi-nor/hisi-sfc.c
@@ -355,16 +355,16 @@ static int hisi_spi_nor_register(struct device_node *np,
 
 	ret = of_property_read_u32(np, "reg", &priv->chipselect);
 	if (ret) {
-		dev_err(dev, "There's no reg property for %s\n",
-			np->full_name);
+		dev_err(dev, "There's no reg property for %pOF\n",
+			np);
 		return ret;
 	}
 
 	ret = of_property_read_u32(np, "spi-max-frequency",
 			&priv->clkrate);
 	if (ret) {
-		dev_err(dev, "There's no spi-max-frequency property for %s\n",
-			np->full_name);
+		dev_err(dev, "There's no spi-max-frequency property for %pOF\n",
+			np);
 		return ret;
 	}
 	priv->host = host;
diff --git a/drivers/mtd/spi-nor/intel-spi-pci.c b/drivers/mtd/spi-nor/intel-spi-pci.c
new file mode 100644
index 000000000000..e82652335ede
--- /dev/null
+++ b/drivers/mtd/spi-nor/intel-spi-pci.c
@@ -0,0 +1,82 @@
+/*
+ * Intel PCH/PCU SPI flash PCI driver.
+ *
+ * Copyright (C) 2016, Intel Corporation
+ * Author: Mika Westerberg <mika.westerberg@linux.intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/ioport.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+
+#include "intel-spi.h"
+
+#define BCR		0xdc
+#define BCR_WPD		BIT(0)
+
+static const struct intel_spi_boardinfo bxt_info = {
+	.type = INTEL_SPI_BXT,
+};
+
+static int intel_spi_pci_probe(struct pci_dev *pdev,
+			       const struct pci_device_id *id)
+{
+	struct intel_spi_boardinfo *info;
+	struct intel_spi *ispi;
+	u32 bcr;
+	int ret;
+
+	ret = pcim_enable_device(pdev);
+	if (ret)
+		return ret;
+
+	info = devm_kmemdup(&pdev->dev, (void *)id->driver_data, sizeof(*info),
+			    GFP_KERNEL);
+	if (!info)
+		return -ENOMEM;
+
+	/* Try to make the chip read/write */
+	pci_read_config_dword(pdev, BCR, &bcr);
+	if (!(bcr & BCR_WPD)) {
+		bcr |= BCR_WPD;
+		pci_write_config_dword(pdev, BCR, bcr);
+		pci_read_config_dword(pdev, BCR, &bcr);
+	}
+	info->writeable = !!(bcr & BCR_WPD);
+
+	ispi = intel_spi_probe(&pdev->dev, &pdev->resource[0], info);
+	if (IS_ERR(ispi))
+		return PTR_ERR(ispi);
+
+	pci_set_drvdata(pdev, ispi);
+	return 0;
+}
+
+static void intel_spi_pci_remove(struct pci_dev *pdev)
+{
+	intel_spi_remove(pci_get_drvdata(pdev));
+}
+
+static const struct pci_device_id intel_spi_pci_ids[] = {
+	{ PCI_VDEVICE(INTEL, 0x19e0), (unsigned long)&bxt_info },
+	{ },
+};
+MODULE_DEVICE_TABLE(pci, intel_spi_pci_ids);
+
+static struct pci_driver intel_spi_pci_driver = {
+	.name = "intel-spi",
+	.id_table = intel_spi_pci_ids,
+	.probe = intel_spi_pci_probe,
+	.remove = intel_spi_pci_remove,
+};
+
+module_pci_driver(intel_spi_pci_driver);
+
+MODULE_DESCRIPTION("Intel PCH/PCU SPI flash PCI driver");
+MODULE_AUTHOR("Mika Westerberg <mika.westerberg@linux.intel.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/mtd/spi-nor/mtk-quadspi.c b/drivers/mtd/spi-nor/mtk-quadspi.c
index 8a20ec4991c8..c258c7adf1c5 100644
--- a/drivers/mtd/spi-nor/mtk-quadspi.c
+++ b/drivers/mtd/spi-nor/mtk-quadspi.c
@@ -24,7 +24,6 @@
 #include <linux/mutex.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
-#include <linux/pinctrl/consumer.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/mtd/mtd.h>
diff --git a/drivers/mtd/spi-nor/spi-nor.c b/drivers/mtd/spi-nor/spi-nor.c
index 1413828ff1fb..19c000722cbc 100644
--- a/drivers/mtd/spi-nor/spi-nor.c
+++ b/drivers/mtd/spi-nor/spi-nor.c
@@ -17,6 +17,7 @@
 #include <linux/mutex.h>
 #include <linux/math64.h>
 #include <linux/sizes.h>
+#include <linux/slab.h>
 
 #include <linux/mtd/mtd.h>
 #include <linux/of_platform.h>
@@ -86,6 +87,8 @@ struct flash_info {
 					 * to support memory size above 128Mib.
 					 */
 #define NO_CHIP_ERASE		BIT(12) /* Chip does not support chip erase */
+#define SPI_NOR_SKIP_SFDP	BIT(13)	/* Skip parsing of SFDP tables */
+#define USE_CLSR		BIT(14)	/* use CLSR command */
 };
 
 #define JEDEC_MFR(info)	((info)->id[0])
@@ -306,8 +309,18 @@ static inline int spi_nor_sr_ready(struct spi_nor *nor)
 	int sr = read_sr(nor);
 	if (sr < 0)
 		return sr;
-	else
-		return !(sr & SR_WIP);
+
+	if (nor->flags & SNOR_F_USE_CLSR && sr & (SR_E_ERR | SR_P_ERR)) {
+		if (sr & SR_E_ERR)
+			dev_err(nor->dev, "Erase Error occurred\n");
+		else
+			dev_err(nor->dev, "Programming Error occurred\n");
+
+		nor->write_reg(nor, SPINOR_OP_CLSR, NULL, 0);
+		return -EIO;
+	}
+
+	return !(sr & SR_WIP);
 }
 
 static inline int spi_nor_fsr_ready(struct spi_nor *nor)
@@ -1041,15 +1054,15 @@ static const struct flash_info spi_nor_ids[] = {
 	 */
 	{ "s25sl032p",  INFO(0x010215, 0x4d00,  64 * 1024,  64, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) },
 	{ "s25sl064p",  INFO(0x010216, 0x4d00,  64 * 1024, 128, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) },
-	{ "s25fl256s0", INFO(0x010219, 0x4d00, 256 * 1024, 128, 0) },
-	{ "s25fl256s1", INFO(0x010219, 0x4d01,  64 * 1024, 512, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) },
-	{ "s25fl512s",  INFO(0x010220, 0x4d00, 256 * 1024, 256, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) },
+	{ "s25fl256s0", INFO(0x010219, 0x4d00, 256 * 1024, 128, USE_CLSR) },
+	{ "s25fl256s1", INFO(0x010219, 0x4d01,  64 * 1024, 512, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | USE_CLSR) },
+	{ "s25fl512s",  INFO(0x010220, 0x4d00, 256 * 1024, 256, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | USE_CLSR) },
 	{ "s70fl01gs",  INFO(0x010221, 0x4d00, 256 * 1024, 256, 0) },
 	{ "s25sl12800", INFO(0x012018, 0x0300, 256 * 1024,  64, 0) },
 	{ "s25sl12801", INFO(0x012018, 0x0301,  64 * 1024, 256, 0) },
-	{ "s25fl128s",	INFO6(0x012018, 0x4d0180, 64 * 1024, 256, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) },
-	{ "s25fl129p0", INFO(0x012018, 0x4d00, 256 * 1024,  64, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) },
-	{ "s25fl129p1", INFO(0x012018, 0x4d01,  64 * 1024, 256, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) },
+	{ "s25fl128s",  INFO6(0x012018, 0x4d0180, 64 * 1024, 256, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | USE_CLSR) },
+	{ "s25fl129p0", INFO(0x012018, 0x4d00, 256 * 1024,  64, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | USE_CLSR) },
+	{ "s25fl129p1", INFO(0x012018, 0x4d01,  64 * 1024, 256, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | USE_CLSR) },
 	{ "s25sl004a",  INFO(0x010212,      0,  64 * 1024,   8, 0) },
 	{ "s25sl008a",  INFO(0x010213,      0,  64 * 1024,  16, 0) },
 	{ "s25sl016a",  INFO(0x010214,      0,  64 * 1024,  32, 0) },
@@ -1079,6 +1092,7 @@ static const struct flash_info spi_nor_ids[] = {
 	{ "sst25wf040b", INFO(0x621613, 0, 64 * 1024,  8, SECT_4K) },
 	{ "sst25wf040",  INFO(0xbf2504, 0, 64 * 1024,  8, SECT_4K | SST_WRITE) },
 	{ "sst25wf080",  INFO(0xbf2505, 0, 64 * 1024, 16, SECT_4K | SST_WRITE) },
+	{ "sst26vf064b", INFO(0xbf2643, 0, 64 * 1024, 128, SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) },
 
 	/* ST Microelectronics -- newer production may have feature updates */
 	{ "m25p05",  INFO(0x202010,  0,  32 * 1024,   2, 0) },
@@ -1380,6 +1394,16 @@ write_err:
 	return ret;
 }
 
+/**
+ * macronix_quad_enable() - set QE bit in Status Register.
+ * @nor:	pointer to a 'struct spi_nor'
+ *
+ * Set the Quad Enable (QE) bit in the Status Register.
+ *
+ * bit 6 of the Status Register is the QE bit for Macronix like QSPI memories.
+ *
+ * Return: 0 on success, -errno otherwise.
+ */
 static int macronix_quad_enable(struct spi_nor *nor)
 {
 	int ret, val;
@@ -1413,22 +1437,13 @@ static int macronix_quad_enable(struct spi_nor *nor)
  * second byte will be written to the configuration register.
  * Return negative if error occurred.
  */
-static int write_sr_cr(struct spi_nor *nor, u16 val)
-{
-	nor->cmd_buf[0] = val & 0xff;
-	nor->cmd_buf[1] = (val >> 8);
-
-	return nor->write_reg(nor, SPINOR_OP_WRSR, nor->cmd_buf, 2);
-}
-
-static int spansion_quad_enable(struct spi_nor *nor)
+static int write_sr_cr(struct spi_nor *nor, u8 *sr_cr)
 {
 	int ret;
-	int quad_en = CR_QUAD_EN_SPAN << 8;
 
 	write_enable(nor);
 
-	ret = write_sr_cr(nor, quad_en);
+	ret = nor->write_reg(nor, SPINOR_OP_WRSR, sr_cr, 2);
 	if (ret < 0) {
 		dev_err(nor->dev,
 			"error while writing configuration register\n");
@@ -1442,6 +1457,41 @@ static int spansion_quad_enable(struct spi_nor *nor)
 		return ret;
 	}
 
+	return 0;
+}
+
+/**
+ * spansion_quad_enable() - set QE bit in Configuraiton Register.
+ * @nor:	pointer to a 'struct spi_nor'
+ *
+ * Set the Quad Enable (QE) bit in the Configuration Register.
+ * This function is kept for legacy purpose because it has been used for a
+ * long time without anybody complaining but it should be considered as
+ * deprecated and maybe buggy.
+ * First, this function doesn't care about the previous values of the Status
+ * and Configuration Registers when it sets the QE bit (bit 1) in the
+ * Configuration Register: all other bits are cleared, which may have unwanted
+ * side effects like removing some block protections.
+ * Secondly, it uses the Read Configuration Register (35h) instruction though
+ * some very old and few memories don't support this instruction. If a pull-up
+ * resistor is present on the MISO/IO1 line, we might still be able to pass the
+ * "read back" test because the QSPI memory doesn't recognize the command,
+ * so leaves the MISO/IO1 line state unchanged, hence read_cr() returns 0xFF.
+ *
+ * bit 1 of the Configuration Register is the QE bit for Spansion like QSPI
+ * memories.
+ *
+ * Return: 0 on success, -errno otherwise.
+ */
+static int spansion_quad_enable(struct spi_nor *nor)
+{
+	u8 sr_cr[2] = {0, CR_QUAD_EN_SPAN};
+	int ret;
+
+	ret = write_sr_cr(nor, sr_cr);
+	if (ret)
+		return ret;
+
 	/* read back and check it */
 	ret = read_cr(nor);
 	if (!(ret > 0 && (ret & CR_QUAD_EN_SPAN))) {
@@ -1452,6 +1502,140 @@ static int spansion_quad_enable(struct spi_nor *nor)
 	return 0;
 }
 
+/**
+ * spansion_no_read_cr_quad_enable() - set QE bit in Configuration Register.
+ * @nor:	pointer to a 'struct spi_nor'
+ *
+ * Set the Quad Enable (QE) bit in the Configuration Register.
+ * This function should be used with QSPI memories not supporting the Read
+ * Configuration Register (35h) instruction.
+ *
+ * bit 1 of the Configuration Register is the QE bit for Spansion like QSPI
+ * memories.
+ *
+ * Return: 0 on success, -errno otherwise.
+ */
+static int spansion_no_read_cr_quad_enable(struct spi_nor *nor)
+{
+	u8 sr_cr[2];
+	int ret;
+
+	/* Keep the current value of the Status Register. */
+	ret = read_sr(nor);
+	if (ret < 0) {
+		dev_err(nor->dev, "error while reading status register\n");
+		return -EINVAL;
+	}
+	sr_cr[0] = ret;
+	sr_cr[1] = CR_QUAD_EN_SPAN;
+
+	return write_sr_cr(nor, sr_cr);
+}
+
+/**
+ * spansion_read_cr_quad_enable() - set QE bit in Configuration Register.
+ * @nor:	pointer to a 'struct spi_nor'
+ *
+ * Set the Quad Enable (QE) bit in the Configuration Register.
+ * This function should be used with QSPI memories supporting the Read
+ * Configuration Register (35h) instruction.
+ *
+ * bit 1 of the Configuration Register is the QE bit for Spansion like QSPI
+ * memories.
+ *
+ * Return: 0 on success, -errno otherwise.
+ */
+static int spansion_read_cr_quad_enable(struct spi_nor *nor)
+{
+	struct device *dev = nor->dev;
+	u8 sr_cr[2];
+	int ret;
+
+	/* Check current Quad Enable bit value. */
+	ret = read_cr(nor);
+	if (ret < 0) {
+		dev_err(dev, "error while reading configuration register\n");
+		return -EINVAL;
+	}
+
+	if (ret & CR_QUAD_EN_SPAN)
+		return 0;
+
+	sr_cr[1] = ret | CR_QUAD_EN_SPAN;
+
+	/* Keep the current value of the Status Register. */
+	ret = read_sr(nor);
+	if (ret < 0) {
+		dev_err(dev, "error while reading status register\n");
+		return -EINVAL;
+	}
+	sr_cr[0] = ret;
+
+	ret = write_sr_cr(nor, sr_cr);
+	if (ret)
+		return ret;
+
+	/* Read back and check it. */
+	ret = read_cr(nor);
+	if (!(ret > 0 && (ret & CR_QUAD_EN_SPAN))) {
+		dev_err(nor->dev, "Spansion Quad bit not set\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/**
+ * sr2_bit7_quad_enable() - set QE bit in Status Register 2.
+ * @nor:	pointer to a 'struct spi_nor'
+ *
+ * Set the Quad Enable (QE) bit in the Status Register 2.
+ *
+ * This is one of the procedures to set the QE bit described in the SFDP
+ * (JESD216 rev B) specification but no manufacturer using this procedure has
+ * been identified yet, hence the name of the function.
+ *
+ * Return: 0 on success, -errno otherwise.
+ */
+static int sr2_bit7_quad_enable(struct spi_nor *nor)
+{
+	u8 sr2;
+	int ret;
+
+	/* Check current Quad Enable bit value. */
+	ret = nor->read_reg(nor, SPINOR_OP_RDSR2, &sr2, 1);
+	if (ret)
+		return ret;
+	if (sr2 & SR2_QUAD_EN_BIT7)
+		return 0;
+
+	/* Update the Quad Enable bit. */
+	sr2 |= SR2_QUAD_EN_BIT7;
+
+	write_enable(nor);
+
+	ret = nor->write_reg(nor, SPINOR_OP_WRSR2, &sr2, 1);
+	if (ret < 0) {
+		dev_err(nor->dev, "error while writing status register 2\n");
+		return -EINVAL;
+	}
+
+	ret = spi_nor_wait_till_ready(nor);
+	if (ret < 0) {
+		dev_err(nor->dev, "timeout while writing status register 2\n");
+		return ret;
+	}
+
+	/* Read back and check it. */
+	ret = nor->read_reg(nor, SPINOR_OP_RDSR2, &sr2, 1);
+	if (!(ret > 0 && (sr2 & SR2_QUAD_EN_BIT7))) {
+		dev_err(nor->dev, "SR2 Quad bit not set\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static int spi_nor_check(struct spi_nor *nor)
 {
 	if (!nor->dev || !nor->read || !nor->write ||
@@ -1591,6 +1775,599 @@ spi_nor_set_pp_settings(struct spi_nor_pp_command *pp,
 	pp->proto = proto;
 }
 
+/*
+ * Serial Flash Discoverable Parameters (SFDP) parsing.
+ */
+
+/**
+ * spi_nor_read_sfdp() - read Serial Flash Discoverable Parameters.
+ * @nor:	pointer to a 'struct spi_nor'
+ * @addr:	offset in the SFDP area to start reading data from
+ * @len:	number of bytes to read
+ * @buf:	buffer where the SFDP data are copied into (dma-safe memory)
+ *
+ * Whatever the actual numbers of bytes for address and dummy cycles are
+ * for (Fast) Read commands, the Read SFDP (5Ah) instruction is always
+ * followed by a 3-byte address and 8 dummy clock cycles.
+ *
+ * Return: 0 on success, -errno otherwise.
+ */
+static int spi_nor_read_sfdp(struct spi_nor *nor, u32 addr,
+			     size_t len, void *buf)
+{
+	u8 addr_width, read_opcode, read_dummy;
+	int ret;
+
+	read_opcode = nor->read_opcode;
+	addr_width = nor->addr_width;
+	read_dummy = nor->read_dummy;
+
+	nor->read_opcode = SPINOR_OP_RDSFDP;
+	nor->addr_width = 3;
+	nor->read_dummy = 8;
+
+	while (len) {
+		ret = nor->read(nor, addr, len, (u8 *)buf);
+		if (!ret || ret > len) {
+			ret = -EIO;
+			goto read_err;
+		}
+		if (ret < 0)
+			goto read_err;
+
+		buf += ret;
+		addr += ret;
+		len -= ret;
+	}
+	ret = 0;
+
+read_err:
+	nor->read_opcode = read_opcode;
+	nor->addr_width = addr_width;
+	nor->read_dummy = read_dummy;
+
+	return ret;
+}
+
+/**
+ * spi_nor_read_sfdp_dma_unsafe() - read Serial Flash Discoverable Parameters.
+ * @nor:	pointer to a 'struct spi_nor'
+ * @addr:	offset in the SFDP area to start reading data from
+ * @len:	number of bytes to read
+ * @buf:	buffer where the SFDP data are copied into
+ *
+ * Wrap spi_nor_read_sfdp() using a kmalloc'ed bounce buffer as @buf is now not
+ * guaranteed to be dma-safe.
+ *
+ * Return: -ENOMEM if kmalloc() fails, the return code of spi_nor_read_sfdp()
+ *          otherwise.
+ */
+static int spi_nor_read_sfdp_dma_unsafe(struct spi_nor *nor, u32 addr,
+					size_t len, void *buf)
+{
+	void *dma_safe_buf;
+	int ret;
+
+	dma_safe_buf = kmalloc(len, GFP_KERNEL);
+	if (!dma_safe_buf)
+		return -ENOMEM;
+
+	ret = spi_nor_read_sfdp(nor, addr, len, dma_safe_buf);
+	memcpy(buf, dma_safe_buf, len);
+	kfree(dma_safe_buf);
+
+	return ret;
+}
+
+struct sfdp_parameter_header {
+	u8		id_lsb;
+	u8		minor;
+	u8		major;
+	u8		length; /* in double words */
+	u8		parameter_table_pointer[3]; /* byte address */
+	u8		id_msb;
+};
+
+#define SFDP_PARAM_HEADER_ID(p)	(((p)->id_msb << 8) | (p)->id_lsb)
+#define SFDP_PARAM_HEADER_PTP(p) \
+	(((p)->parameter_table_pointer[2] << 16) | \
+	 ((p)->parameter_table_pointer[1] <<  8) | \
+	 ((p)->parameter_table_pointer[0] <<  0))
+
+#define SFDP_BFPT_ID		0xff00	/* Basic Flash Parameter Table */
+#define SFDP_SECTOR_MAP_ID	0xff81	/* Sector Map Table */
+
+#define SFDP_SIGNATURE		0x50444653U
+#define SFDP_JESD216_MAJOR	1
+#define SFDP_JESD216_MINOR	0
+#define SFDP_JESD216A_MINOR	5
+#define SFDP_JESD216B_MINOR	6
+
+struct sfdp_header {
+	u32		signature; /* Ox50444653U <=> "SFDP" */
+	u8		minor;
+	u8		major;
+	u8		nph; /* 0-base number of parameter headers */
+	u8		unused;
+
+	/* Basic Flash Parameter Table. */
+	struct sfdp_parameter_header	bfpt_header;
+};
+
+/* Basic Flash Parameter Table */
+
+/*
+ * JESD216 rev B defines a Basic Flash Parameter Table of 16 DWORDs.
+ * They are indexed from 1 but C arrays are indexed from 0.
+ */
+#define BFPT_DWORD(i)		((i) - 1)
+#define BFPT_DWORD_MAX		16
+
+/* The first version of JESB216 defined only 9 DWORDs. */
+#define BFPT_DWORD_MAX_JESD216			9
+
+/* 1st DWORD. */
+#define BFPT_DWORD1_FAST_READ_1_1_2		BIT(16)
+#define BFPT_DWORD1_ADDRESS_BYTES_MASK		GENMASK(18, 17)
+#define BFPT_DWORD1_ADDRESS_BYTES_3_ONLY	(0x0UL << 17)
+#define BFPT_DWORD1_ADDRESS_BYTES_3_OR_4	(0x1UL << 17)
+#define BFPT_DWORD1_ADDRESS_BYTES_4_ONLY	(0x2UL << 17)
+#define BFPT_DWORD1_DTR				BIT(19)
+#define BFPT_DWORD1_FAST_READ_1_2_2		BIT(20)
+#define BFPT_DWORD1_FAST_READ_1_4_4		BIT(21)
+#define BFPT_DWORD1_FAST_READ_1_1_4		BIT(22)
+
+/* 5th DWORD. */
+#define BFPT_DWORD5_FAST_READ_2_2_2		BIT(0)
+#define BFPT_DWORD5_FAST_READ_4_4_4		BIT(4)
+
+/* 11th DWORD. */
+#define BFPT_DWORD11_PAGE_SIZE_SHIFT		4
+#define BFPT_DWORD11_PAGE_SIZE_MASK		GENMASK(7, 4)
+
+/* 15th DWORD. */
+
+/*
+ * (from JESD216 rev B)
+ * Quad Enable Requirements (QER):
+ * - 000b: Device does not have a QE bit. Device detects 1-1-4 and 1-4-4
+ *         reads based on instruction. DQ3/HOLD# functions are hold during
+ *         instruction phase.
+ * - 001b: QE is bit 1 of status register 2. It is set via Write Status with
+ *         two data bytes where bit 1 of the second byte is one.
+ *         [...]
+ *         Writing only one byte to the status register has the side-effect of
+ *         clearing status register 2, including the QE bit. The 100b code is
+ *         used if writing one byte to the status register does not modify
+ *         status register 2.
+ * - 010b: QE is bit 6 of status register 1. It is set via Write Status with
+ *         one data byte where bit 6 is one.
+ *         [...]
+ * - 011b: QE is bit 7 of status register 2. It is set via Write status
+ *         register 2 instruction 3Eh with one data byte where bit 7 is one.
+ *         [...]
+ *         The status register 2 is read using instruction 3Fh.
+ * - 100b: QE is bit 1 of status register 2. It is set via Write Status with
+ *         two data bytes where bit 1 of the second byte is one.
+ *         [...]
+ *         In contrast to the 001b code, writing one byte to the status
+ *         register does not modify status register 2.
+ * - 101b: QE is bit 1 of status register 2. Status register 1 is read using
+ *         Read Status instruction 05h. Status register2 is read using
+ *         instruction 35h. QE is set via Writ Status instruction 01h with
+ *         two data bytes where bit 1 of the second byte is one.
+ *         [...]
+ */
+#define BFPT_DWORD15_QER_MASK			GENMASK(22, 20)
+#define BFPT_DWORD15_QER_NONE			(0x0UL << 20) /* Micron */
+#define BFPT_DWORD15_QER_SR2_BIT1_BUGGY		(0x1UL << 20)
+#define BFPT_DWORD15_QER_SR1_BIT6		(0x2UL << 20) /* Macronix */
+#define BFPT_DWORD15_QER_SR2_BIT7		(0x3UL << 20)
+#define BFPT_DWORD15_QER_SR2_BIT1_NO_RD		(0x4UL << 20)
+#define BFPT_DWORD15_QER_SR2_BIT1		(0x5UL << 20) /* Spansion */
+
+struct sfdp_bfpt {
+	u32	dwords[BFPT_DWORD_MAX];
+};
+
+/* Fast Read settings. */
+
+static inline void
+spi_nor_set_read_settings_from_bfpt(struct spi_nor_read_command *read,
+				    u16 half,
+				    enum spi_nor_protocol proto)
+{
+	read->num_mode_clocks = (half >> 5) & 0x07;
+	read->num_wait_states = (half >> 0) & 0x1f;
+	read->opcode = (half >> 8) & 0xff;
+	read->proto = proto;
+}
+
+struct sfdp_bfpt_read {
+	/* The Fast Read x-y-z hardware capability in params->hwcaps.mask. */
+	u32			hwcaps;
+
+	/*
+	 * The <supported_bit> bit in <supported_dword> BFPT DWORD tells us
+	 * whether the Fast Read x-y-z command is supported.
+	 */
+	u32			supported_dword;
+	u32			supported_bit;
+
+	/*
+	 * The half-word at offset <setting_shift> in <setting_dword> BFPT DWORD
+	 * encodes the op code, the number of mode clocks and the number of wait
+	 * states to be used by Fast Read x-y-z command.
+	 */
+	u32			settings_dword;
+	u32			settings_shift;
+
+	/* The SPI protocol for this Fast Read x-y-z command. */
+	enum spi_nor_protocol	proto;
+};
+
+static const struct sfdp_bfpt_read sfdp_bfpt_reads[] = {
+	/* Fast Read 1-1-2 */
+	{
+		SNOR_HWCAPS_READ_1_1_2,
+		BFPT_DWORD(1), BIT(16),	/* Supported bit */
+		BFPT_DWORD(4), 0,	/* Settings */
+		SNOR_PROTO_1_1_2,
+	},
+
+	/* Fast Read 1-2-2 */
+	{
+		SNOR_HWCAPS_READ_1_2_2,
+		BFPT_DWORD(1), BIT(20),	/* Supported bit */
+		BFPT_DWORD(4), 16,	/* Settings */
+		SNOR_PROTO_1_2_2,
+	},
+
+	/* Fast Read 2-2-2 */
+	{
+		SNOR_HWCAPS_READ_2_2_2,
+		BFPT_DWORD(5),  BIT(0),	/* Supported bit */
+		BFPT_DWORD(6), 16,	/* Settings */
+		SNOR_PROTO_2_2_2,
+	},
+
+	/* Fast Read 1-1-4 */
+	{
+		SNOR_HWCAPS_READ_1_1_4,
+		BFPT_DWORD(1), BIT(22),	/* Supported bit */
+		BFPT_DWORD(3), 16,	/* Settings */
+		SNOR_PROTO_1_1_4,
+	},
+
+	/* Fast Read 1-4-4 */
+	{
+		SNOR_HWCAPS_READ_1_4_4,
+		BFPT_DWORD(1), BIT(21),	/* Supported bit */
+		BFPT_DWORD(3), 0,	/* Settings */
+		SNOR_PROTO_1_4_4,
+	},
+
+	/* Fast Read 4-4-4 */
+	{
+		SNOR_HWCAPS_READ_4_4_4,
+		BFPT_DWORD(5), BIT(4),	/* Supported bit */
+		BFPT_DWORD(7), 16,	/* Settings */
+		SNOR_PROTO_4_4_4,
+	},
+};
+
+struct sfdp_bfpt_erase {
+	/*
+	 * The half-word at offset <shift> in DWORD <dwoard> encodes the
+	 * op code and erase sector size to be used by Sector Erase commands.
+	 */
+	u32			dword;
+	u32			shift;
+};
+
+static const struct sfdp_bfpt_erase sfdp_bfpt_erases[] = {
+	/* Erase Type 1 in DWORD8 bits[15:0] */
+	{BFPT_DWORD(8), 0},
+
+	/* Erase Type 2 in DWORD8 bits[31:16] */
+	{BFPT_DWORD(8), 16},
+
+	/* Erase Type 3 in DWORD9 bits[15:0] */
+	{BFPT_DWORD(9), 0},
+
+	/* Erase Type 4 in DWORD9 bits[31:16] */
+	{BFPT_DWORD(9), 16},
+};
+
+static int spi_nor_hwcaps_read2cmd(u32 hwcaps);
+
+/**
+ * spi_nor_parse_bfpt() - read and parse the Basic Flash Parameter Table.
+ * @nor:		pointer to a 'struct spi_nor'
+ * @bfpt_header:	pointer to the 'struct sfdp_parameter_header' describing
+ *			the Basic Flash Parameter Table length and version
+ * @params:		pointer to the 'struct spi_nor_flash_parameter' to be
+ *			filled
+ *
+ * The Basic Flash Parameter Table is the main and only mandatory table as
+ * defined by the SFDP (JESD216) specification.
+ * It provides us with the total size (memory density) of the data array and
+ * the number of address bytes for Fast Read, Page Program and Sector Erase
+ * commands.
+ * For Fast READ commands, it also gives the number of mode clock cycles and
+ * wait states (regrouped in the number of dummy clock cycles) for each
+ * supported instruction op code.
+ * For Page Program, the page size is now available since JESD216 rev A, however
+ * the supported instruction op codes are still not provided.
+ * For Sector Erase commands, this table stores the supported instruction op
+ * codes and the associated sector sizes.
+ * Finally, the Quad Enable Requirements (QER) are also available since JESD216
+ * rev A. The QER bits encode the manufacturer dependent procedure to be
+ * executed to set the Quad Enable (QE) bit in some internal register of the
+ * Quad SPI memory. Indeed the QE bit, when it exists, must be set before
+ * sending any Quad SPI command to the memory. Actually, setting the QE bit
+ * tells the memory to reassign its WP# and HOLD#/RESET# pins to functions IO2
+ * and IO3 hence enabling 4 (Quad) I/O lines.
+ *
+ * Return: 0 on success, -errno otherwise.
+ */
+static int spi_nor_parse_bfpt(struct spi_nor *nor,
+			      const struct sfdp_parameter_header *bfpt_header,
+			      struct spi_nor_flash_parameter *params)
+{
+	struct mtd_info *mtd = &nor->mtd;
+	struct sfdp_bfpt bfpt;
+	size_t len;
+	int i, cmd, err;
+	u32 addr;
+	u16 half;
+
+	/* JESD216 Basic Flash Parameter Table length is at least 9 DWORDs. */
+	if (bfpt_header->length < BFPT_DWORD_MAX_JESD216)
+		return -EINVAL;
+
+	/* Read the Basic Flash Parameter Table. */
+	len = min_t(size_t, sizeof(bfpt),
+		    bfpt_header->length * sizeof(u32));
+	addr = SFDP_PARAM_HEADER_PTP(bfpt_header);
+	memset(&bfpt, 0, sizeof(bfpt));
+	err = spi_nor_read_sfdp_dma_unsafe(nor,  addr, len, &bfpt);
+	if (err < 0)
+		return err;
+
+	/* Fix endianness of the BFPT DWORDs. */
+	for (i = 0; i < BFPT_DWORD_MAX; i++)
+		bfpt.dwords[i] = le32_to_cpu(bfpt.dwords[i]);
+
+	/* Number of address bytes. */
+	switch (bfpt.dwords[BFPT_DWORD(1)] & BFPT_DWORD1_ADDRESS_BYTES_MASK) {
+	case BFPT_DWORD1_ADDRESS_BYTES_3_ONLY:
+		nor->addr_width = 3;
+		break;
+
+	case BFPT_DWORD1_ADDRESS_BYTES_4_ONLY:
+		nor->addr_width = 4;
+		break;
+
+	default:
+		break;
+	}
+
+	/* Flash Memory Density (in bits). */
+	params->size = bfpt.dwords[BFPT_DWORD(2)];
+	if (params->size & BIT(31)) {
+		params->size &= ~BIT(31);
+
+		/*
+		 * Prevent overflows on params->size. Anyway, a NOR of 2^64
+		 * bits is unlikely to exist so this error probably means
+		 * the BFPT we are reading is corrupted/wrong.
+		 */
+		if (params->size > 63)
+			return -EINVAL;
+
+		params->size = 1ULL << params->size;
+	} else {
+		params->size++;
+	}
+	params->size >>= 3; /* Convert to bytes. */
+
+	/* Fast Read settings. */
+	for (i = 0; i < ARRAY_SIZE(sfdp_bfpt_reads); i++) {
+		const struct sfdp_bfpt_read *rd = &sfdp_bfpt_reads[i];
+		struct spi_nor_read_command *read;
+
+		if (!(bfpt.dwords[rd->supported_dword] & rd->supported_bit)) {
+			params->hwcaps.mask &= ~rd->hwcaps;
+			continue;
+		}
+
+		params->hwcaps.mask |= rd->hwcaps;
+		cmd = spi_nor_hwcaps_read2cmd(rd->hwcaps);
+		read = &params->reads[cmd];
+		half = bfpt.dwords[rd->settings_dword] >> rd->settings_shift;
+		spi_nor_set_read_settings_from_bfpt(read, half, rd->proto);
+	}
+
+	/* Sector Erase settings. */
+	for (i = 0; i < ARRAY_SIZE(sfdp_bfpt_erases); i++) {
+		const struct sfdp_bfpt_erase *er = &sfdp_bfpt_erases[i];
+		u32 erasesize;
+		u8 opcode;
+
+		half = bfpt.dwords[er->dword] >> er->shift;
+		erasesize = half & 0xff;
+
+		/* erasesize == 0 means this Erase Type is not supported. */
+		if (!erasesize)
+			continue;
+
+		erasesize = 1U << erasesize;
+		opcode = (half >> 8) & 0xff;
+#ifdef CONFIG_MTD_SPI_NOR_USE_4K_SECTORS
+		if (erasesize == SZ_4K) {
+			nor->erase_opcode = opcode;
+			mtd->erasesize = erasesize;
+			break;
+		}
+#endif
+		if (!mtd->erasesize || mtd->erasesize < erasesize) {
+			nor->erase_opcode = opcode;
+			mtd->erasesize = erasesize;
+		}
+	}
+
+	/* Stop here if not JESD216 rev A or later. */
+	if (bfpt_header->length < BFPT_DWORD_MAX)
+		return 0;
+
+	/* Page size: this field specifies 'N' so the page size = 2^N bytes. */
+	params->page_size = bfpt.dwords[BFPT_DWORD(11)];
+	params->page_size &= BFPT_DWORD11_PAGE_SIZE_MASK;
+	params->page_size >>= BFPT_DWORD11_PAGE_SIZE_SHIFT;
+	params->page_size = 1U << params->page_size;
+
+	/* Quad Enable Requirements. */
+	switch (bfpt.dwords[BFPT_DWORD(15)] & BFPT_DWORD15_QER_MASK) {
+	case BFPT_DWORD15_QER_NONE:
+		params->quad_enable = NULL;
+		break;
+
+	case BFPT_DWORD15_QER_SR2_BIT1_BUGGY:
+	case BFPT_DWORD15_QER_SR2_BIT1_NO_RD:
+		params->quad_enable = spansion_no_read_cr_quad_enable;
+		break;
+
+	case BFPT_DWORD15_QER_SR1_BIT6:
+		params->quad_enable = macronix_quad_enable;
+		break;
+
+	case BFPT_DWORD15_QER_SR2_BIT7:
+		params->quad_enable = sr2_bit7_quad_enable;
+		break;
+
+	case BFPT_DWORD15_QER_SR2_BIT1:
+		params->quad_enable = spansion_read_cr_quad_enable;
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/**
+ * spi_nor_parse_sfdp() - parse the Serial Flash Discoverable Parameters.
+ * @nor:		pointer to a 'struct spi_nor'
+ * @params:		pointer to the 'struct spi_nor_flash_parameter' to be
+ *			filled
+ *
+ * The Serial Flash Discoverable Parameters are described by the JEDEC JESD216
+ * specification. This is a standard which tends to supported by almost all
+ * (Q)SPI memory manufacturers. Those hard-coded tables allow us to learn at
+ * runtime the main parameters needed to perform basic SPI flash operations such
+ * as Fast Read, Page Program or Sector Erase commands.
+ *
+ * Return: 0 on success, -errno otherwise.
+ */
+static int spi_nor_parse_sfdp(struct spi_nor *nor,
+			      struct spi_nor_flash_parameter *params)
+{
+	const struct sfdp_parameter_header *param_header, *bfpt_header;
+	struct sfdp_parameter_header *param_headers = NULL;
+	struct sfdp_header header;
+	struct device *dev = nor->dev;
+	size_t psize;
+	int i, err;
+
+	/* Get the SFDP header. */
+	err = spi_nor_read_sfdp_dma_unsafe(nor, 0, sizeof(header), &header);
+	if (err < 0)
+		return err;
+
+	/* Check the SFDP header version. */
+	if (le32_to_cpu(header.signature) != SFDP_SIGNATURE ||
+	    header.major != SFDP_JESD216_MAJOR ||
+	    header.minor < SFDP_JESD216_MINOR)
+		return -EINVAL;
+
+	/*
+	 * Verify that the first and only mandatory parameter header is a
+	 * Basic Flash Parameter Table header as specified in JESD216.
+	 */
+	bfpt_header = &header.bfpt_header;
+	if (SFDP_PARAM_HEADER_ID(bfpt_header) != SFDP_BFPT_ID ||
+	    bfpt_header->major != SFDP_JESD216_MAJOR)
+		return -EINVAL;
+
+	/*
+	 * Allocate memory then read all parameter headers with a single
+	 * Read SFDP command. These parameter headers will actually be parsed
+	 * twice: a first time to get the latest revision of the basic flash
+	 * parameter table, then a second time to handle the supported optional
+	 * tables.
+	 * Hence we read the parameter headers once for all to reduce the
+	 * processing time. Also we use kmalloc() instead of devm_kmalloc()
+	 * because we don't need to keep these parameter headers: the allocated
+	 * memory is always released with kfree() before exiting this function.
+	 */
+	if (header.nph) {
+		psize = header.nph * sizeof(*param_headers);
+
+		param_headers = kmalloc(psize, GFP_KERNEL);
+		if (!param_headers)
+			return -ENOMEM;
+
+		err = spi_nor_read_sfdp(nor, sizeof(header),
+					psize, param_headers);
+		if (err < 0) {
+			dev_err(dev, "failed to read SFDP parameter headers\n");
+			goto exit;
+		}
+	}
+
+	/*
+	 * Check other parameter headers to get the latest revision of
+	 * the basic flash parameter table.
+	 */
+	for (i = 0; i < header.nph; i++) {
+		param_header = &param_headers[i];
+
+		if (SFDP_PARAM_HEADER_ID(param_header) == SFDP_BFPT_ID &&
+		    param_header->major == SFDP_JESD216_MAJOR &&
+		    (param_header->minor > bfpt_header->minor ||
+		     (param_header->minor == bfpt_header->minor &&
+		      param_header->length > bfpt_header->length)))
+			bfpt_header = param_header;
+	}
+
+	err = spi_nor_parse_bfpt(nor, bfpt_header, params);
+	if (err)
+		goto exit;
+
+	/* Parse other parameter headers. */
+	for (i = 0; i < header.nph; i++) {
+		param_header = &param_headers[i];
+
+		switch (SFDP_PARAM_HEADER_ID(param_header)) {
+		case SFDP_SECTOR_MAP_ID:
+			dev_info(dev, "non-uniform erase sector maps are not supported yet.\n");
+			break;
+
+		default:
+			break;
+		}
+
+		if (err)
+			goto exit;
+	}
+
+exit:
+	kfree(param_headers);
+	return err;
+}
+
 static int spi_nor_init_params(struct spi_nor *nor,
 			       const struct flash_info *info,
 			       struct spi_nor_flash_parameter *params)
@@ -1646,11 +2423,28 @@ static int spi_nor_init_params(struct spi_nor *nor,
 			break;
 
 		default:
+			/* Kept only for backward compatibility purpose. */
 			params->quad_enable = spansion_quad_enable;
 			break;
 		}
 	}
 
+	/* Override the parameters with data read from SFDP tables. */
+	nor->addr_width = 0;
+	nor->mtd.erasesize = 0;
+	if ((info->flags & (SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ)) &&
+	    !(info->flags & SPI_NOR_SKIP_SFDP)) {
+		struct spi_nor_flash_parameter sfdp_params;
+
+		memcpy(&sfdp_params, params, sizeof(sfdp_params));
+		if (spi_nor_parse_sfdp(nor, &sfdp_params)) {
+			nor->addr_width = 0;
+			nor->mtd.erasesize = 0;
+		} else {
+			memcpy(params, &sfdp_params, sizeof(*params));
+		}
+	}
+
 	return 0;
 }
 
@@ -1762,6 +2556,10 @@ static int spi_nor_select_erase(struct spi_nor *nor,
 {
 	struct mtd_info *mtd = &nor->mtd;
 
+	/* Do nothing if already configured from SFDP. */
+	if (mtd->erasesize)
+		return 0;
+
 #ifdef CONFIG_MTD_SPI_NOR_USE_4K_SECTORS
 	/* prefer "small sector" erase if possible */
 	if (info->flags & SECT_4K) {
@@ -1960,6 +2758,8 @@ int spi_nor_scan(struct spi_nor *nor, const char *name,
 		nor->flags |= SNOR_F_HAS_SR_TB;
 	if (info->flags & NO_CHIP_ERASE)
 		nor->flags |= SNOR_F_NO_OP_CHIP_ERASE;
+	if (info->flags & USE_CLSR)
+		nor->flags |= SNOR_F_USE_CLSR;
 
 	if (info->flags & SPI_NOR_NO_ERASE)
 		mtd->flags |= MTD_NO_ERASE;
@@ -1994,9 +2794,11 @@ int spi_nor_scan(struct spi_nor *nor, const char *name,
 	if (ret)
 		return ret;
 
-	if (info->addr_width)
+	if (nor->addr_width) {
+		/* already configured from SFDP */
+	} else if (info->addr_width) {
 		nor->addr_width = info->addr_width;
-	else if (mtd->size > 0x1000000) {
+	} else if (mtd->size > 0x1000000) {
 		/* enable 4-byte addressing if the device exceeds 16MiB */
 		nor->addr_width = 4;
 		if (JEDEC_MFR(info) == SNOR_MFR_SPANSION ||
diff --git a/drivers/mtd/ssfdc.c b/drivers/mtd/ssfdc.c
index 41b13d1cdcc4..95f0bf95f095 100644
--- a/drivers/mtd/ssfdc.c
+++ b/drivers/mtd/ssfdc.c
@@ -16,7 +16,7 @@
 #include <linux/slab.h>
 #include <linux/hdreg.h>
 #include <linux/mtd/mtd.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/mtd/blktrans.h>
 
 struct ssfdcr_record {
diff --git a/drivers/mtd/tests/nandbiterrs.c b/drivers/mtd/tests/nandbiterrs.c
index f26dec896afa..5f03b8c885a9 100644
--- a/drivers/mtd/tests/nandbiterrs.c
+++ b/drivers/mtd/tests/nandbiterrs.c
@@ -47,7 +47,7 @@
 #include <linux/moduleparam.h>
 #include <linux/mtd/mtd.h>
 #include <linux/err.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/slab.h>
 #include "mtd_test.h"
 
diff --git a/drivers/mtd/ubi/block.c b/drivers/mtd/ubi/block.c
index c3963f880448..b210fdb31c98 100644
--- a/drivers/mtd/ubi/block.c
+++ b/drivers/mtd/ubi/block.c
@@ -383,7 +383,7 @@ int ubiblock_create(struct ubi_volume_info *vi)
 	/* Initialize the gendisk of this ubiblock device */
 	gd = alloc_disk(1);
 	if (!gd) {
-		pr_err("UBI: block: alloc_disk failed");
+		pr_err("UBI: block: alloc_disk failed\n");
 		ret = -ENODEV;
 		goto out_free_dev;
 	}
@@ -607,7 +607,7 @@ static void __init ubiblock_create_from_param(void)
 		desc = open_volume_desc(p->name, p->ubi_num, p->vol_id);
 		if (IS_ERR(desc)) {
 			pr_err(
-			       "UBI: block: can't open volume on ubi%d_%d, err=%ld",
+			       "UBI: block: can't open volume on ubi%d_%d, err=%ld\n",
 			       p->ubi_num, p->vol_id, PTR_ERR(desc));
 			continue;
 		}
@@ -618,7 +618,7 @@ static void __init ubiblock_create_from_param(void)
 		ret = ubiblock_create(&vi);
 		if (ret) {
 			pr_err(
-			       "UBI: block: can't add '%s' volume on ubi%d_%d, err=%d",
+			       "UBI: block: can't add '%s' volume on ubi%d_%d, err=%d\n",
 			       vi.name, p->ubi_num, p->vol_id, ret);
 			continue;
 		}
diff --git a/drivers/mtd/ubi/build.c b/drivers/mtd/ubi/build.c
index d854521962ef..842550b5712a 100644
--- a/drivers/mtd/ubi/build.c
+++ b/drivers/mtd/ubi/build.c
@@ -825,7 +825,7 @@ int ubi_attach_mtd_dev(struct mtd_info *mtd, int ubi_num,
 	for (i = 0; i < UBI_MAX_DEVICES; i++) {
 		ubi = ubi_devices[i];
 		if (ubi && mtd->index == ubi->mtd->index) {
-			pr_err("ubi: mtd%d is already attached to ubi%d",
+			pr_err("ubi: mtd%d is already attached to ubi%d\n",
 				mtd->index, i);
 			return -EEXIST;
 		}
@@ -840,7 +840,7 @@ int ubi_attach_mtd_dev(struct mtd_info *mtd, int ubi_num,
 	 * no sense to attach emulated MTD devices, so we prohibit this.
 	 */
 	if (mtd->type == MTD_UBIVOLUME) {
-		pr_err("ubi: refuse attaching mtd%d - it is already emulated on top of UBI",
+		pr_err("ubi: refuse attaching mtd%d - it is already emulated on top of UBI\n",
 			mtd->index);
 		return -EINVAL;
 	}
@@ -851,7 +851,7 @@ int ubi_attach_mtd_dev(struct mtd_info *mtd, int ubi_num,
 			if (!ubi_devices[ubi_num])
 				break;
 		if (ubi_num == UBI_MAX_DEVICES) {
-			pr_err("ubi: only %d UBI devices may be created",
+			pr_err("ubi: only %d UBI devices may be created\n",
 				UBI_MAX_DEVICES);
 			return -ENFILE;
 		}
@@ -861,7 +861,7 @@ int ubi_attach_mtd_dev(struct mtd_info *mtd, int ubi_num,
 
 		/* Make sure ubi_num is not busy */
 		if (ubi_devices[ubi_num]) {
-			pr_err("ubi: ubi%i already exists", ubi_num);
+			pr_err("ubi: ubi%i already exists\n", ubi_num);
 			return -EEXIST;
 		}
 	}
@@ -1166,7 +1166,7 @@ static int __init ubi_init(void)
 	BUILD_BUG_ON(sizeof(struct ubi_vid_hdr) != 64);
 
 	if (mtd_devs > UBI_MAX_DEVICES) {
-		pr_err("UBI error: too many MTD devices, maximum is %d",
+		pr_err("UBI error: too many MTD devices, maximum is %d\n",
 		       UBI_MAX_DEVICES);
 		return -EINVAL;
 	}
@@ -1178,7 +1178,7 @@ static int __init ubi_init(void)
 
 	err = misc_register(&ubi_ctrl_cdev);
 	if (err) {
-		pr_err("UBI error: cannot register device");
+		pr_err("UBI error: cannot register device\n");
 		goto out;
 	}
 
@@ -1205,7 +1205,7 @@ static int __init ubi_init(void)
 		mtd = open_mtd_device(p->name);
 		if (IS_ERR(mtd)) {
 			err = PTR_ERR(mtd);
-			pr_err("UBI error: cannot open mtd %s, error %d",
+			pr_err("UBI error: cannot open mtd %s, error %d\n",
 			       p->name, err);
 			/* See comment below re-ubi_is_module(). */
 			if (ubi_is_module())
@@ -1218,7 +1218,7 @@ static int __init ubi_init(void)
 					 p->vid_hdr_offs, p->max_beb_per1024);
 		mutex_unlock(&ubi_devices_mutex);
 		if (err < 0) {
-			pr_err("UBI error: cannot attach mtd%d",
+			pr_err("UBI error: cannot attach mtd%d\n",
 			       mtd->index);
 			put_mtd_device(mtd);
 
@@ -1242,7 +1242,7 @@ static int __init ubi_init(void)
 
 	err = ubiblock_init();
 	if (err) {
-		pr_err("UBI error: block: cannot initialize, error %d", err);
+		pr_err("UBI error: block: cannot initialize, error %d\n", err);
 
 		/* See comment above re-ubi_is_module(). */
 		if (ubi_is_module())
@@ -1265,7 +1265,7 @@ out_dev_unreg:
 	misc_deregister(&ubi_ctrl_cdev);
 out:
 	class_unregister(&ubi_class);
-	pr_err("UBI error: cannot initialize UBI, error %d", err);
+	pr_err("UBI error: cannot initialize UBI, error %d\n", err);
 	return err;
 }
 late_initcall(ubi_init);
diff --git a/drivers/mtd/ubi/fastmap.c b/drivers/mtd/ubi/fastmap.c
index b44c8d348e78..5a832bc79b1b 100644
--- a/drivers/mtd/ubi/fastmap.c
+++ b/drivers/mtd/ubi/fastmap.c
@@ -1667,7 +1667,7 @@ err:
 
 	ret = invalidate_fastmap(ubi);
 	if (ret < 0) {
-		ubi_err(ubi, "Unable to invalidiate current fastmap!");
+		ubi_err(ubi, "Unable to invalidate current fastmap!");
 		ubi_ro_mode(ubi);
 	} else {
 		return_fm_pebs(ubi, old_fm);
diff --git a/drivers/mtd/ubi/ubi-media.h b/drivers/mtd/ubi/ubi-media.h
index 22ed3f627506..bfceae5a890e 100644
--- a/drivers/mtd/ubi/ubi-media.h
+++ b/drivers/mtd/ubi/ubi-media.h
@@ -229,7 +229,7 @@ struct ubi_ec_hdr {
  * copy. UBI also calculates data CRC when the data is moved and stores it at
  * the @data_crc field of the copy (P1). So when UBI needs to pick one physical
  * eraseblock of two (P or P1), the @copy_flag of the newer one (P1) is
- * examined. If it is cleared, the situation* is simple and the newer one is
+ * examined. If it is cleared, the situation is simple and the newer one is
  * picked. If it is set, the data CRC of the copy (P1) is examined. If the CRC
  * checksum is correct, this physical eraseblock is selected (P1). Otherwise
  * the older one (P) is selected.
@@ -389,7 +389,7 @@ struct ubi_vtbl_record {
 #define UBI_FM_POOL_MAGIC	0x67AF4D08
 #define UBI_FM_EBA_MAGIC	0xf0c040a8
 
-/* A fastmap supber block can be located between PEB 0 and
+/* A fastmap super block can be located between PEB 0 and
  * UBI_FM_MAX_START */
 #define UBI_FM_MAX_START	64
 
diff --git a/drivers/mux/Makefile b/drivers/mux/Makefile
index 6bac5b0fea13..0e1e59760e3f 100644
--- a/drivers/mux/Makefile
+++ b/drivers/mux/Makefile
@@ -2,6 +2,11 @@
 # Makefile for multiplexer devices.
 #
 
+mux-core-objs			:= core.o
+mux-adg792a-objs		:= adg792a.o
+mux-gpio-objs			:= gpio.o
+mux-mmio-objs			:= mmio.o
+
 obj-$(CONFIG_MULTIPLEXER)	+= mux-core.o
 obj-$(CONFIG_MUX_ADG792A)	+= mux-adg792a.o
 obj-$(CONFIG_MUX_GPIO)		+= mux-gpio.o
diff --git a/drivers/mux/mux-adg792a.c b/drivers/mux/adg792a.c
index 12aa221ab90d..12aa221ab90d 100644
--- a/drivers/mux/mux-adg792a.c
+++ b/drivers/mux/adg792a.c
diff --git a/drivers/mux/mux-core.c b/drivers/mux/core.c
index 2fe96c470112..2260063b0ea8 100644
--- a/drivers/mux/mux-core.c
+++ b/drivers/mux/core.c
@@ -58,7 +58,7 @@ static void mux_chip_release(struct device *dev)
 	kfree(mux_chip);
 }
 
-static struct device_type mux_type = {
+static const struct device_type mux_type = {
 	.name = "mux-chip",
 	.release = mux_chip_release,
 };
@@ -452,8 +452,8 @@ struct mux_control *mux_control_get(struct device *dev, const char *mux_name)
 					 "mux-controls", "#mux-control-cells",
 					 index, &args);
 	if (ret) {
-		dev_err(dev, "%s: failed to get mux-control %s(%i)\n",
-			np->full_name, mux_name ?: "", index);
+		dev_err(dev, "%pOF: failed to get mux-control %s(%i)\n",
+			np, mux_name ?: "", index);
 		return ERR_PTR(ret);
 	}
 
@@ -464,8 +464,8 @@ struct mux_control *mux_control_get(struct device *dev, const char *mux_name)
 
 	if (args.args_count > 1 ||
 	    (!args.args_count && (mux_chip->controllers > 1))) {
-		dev_err(dev, "%s: wrong #mux-control-cells for %s\n",
-			np->full_name, args.np->full_name);
+		dev_err(dev, "%pOF: wrong #mux-control-cells for %pOF\n",
+			np, args.np);
 		return ERR_PTR(-EINVAL);
 	}
 
@@ -474,8 +474,8 @@ struct mux_control *mux_control_get(struct device *dev, const char *mux_name)
 		controller = args.args[0];
 
 	if (controller >= mux_chip->controllers) {
-		dev_err(dev, "%s: bad mux controller %u specified in %s\n",
-			np->full_name, controller, args.np->full_name);
+		dev_err(dev, "%pOF: bad mux controller %u specified in %pOF\n",
+			np, controller, args.np);
 		return ERR_PTR(-EINVAL);
 	}
 
diff --git a/drivers/mux/mux-gpio.c b/drivers/mux/gpio.c
index 468bf1709606..468bf1709606 100644
--- a/drivers/mux/mux-gpio.c
+++ b/drivers/mux/gpio.c
diff --git a/drivers/mux/mux-mmio.c b/drivers/mux/mmio.c
index 37c1de359a70..37c1de359a70 100644
--- a/drivers/mux/mux-mmio.c
+++ b/drivers/mux/mmio.c
diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig
index 83a1616903f8..aba0d652095b 100644
--- a/drivers/net/Kconfig
+++ b/drivers/net/Kconfig
@@ -333,7 +333,7 @@ config VIRTIO_NET
 	depends on VIRTIO
 	---help---
 	  This is the virtual network driver for virtio.  It can be used with
-	  lguest or QEMU based VMMs (like KVM or Xen).  Say Y or M.
+	  QEMU based VMMs (like KVM or Xen).  Say Y or M.
 
 config NLMON
 	tristate "Virtual netlink monitoring device"
diff --git a/drivers/net/Makefile b/drivers/net/Makefile
index b2f6556d8848..8dff900085d6 100644
--- a/drivers/net/Makefile
+++ b/drivers/net/Makefile
@@ -45,7 +45,6 @@ obj-$(CONFIG_ETHERNET) += ethernet/
 obj-$(CONFIG_FDDI) += fddi/
 obj-$(CONFIG_HIPPI) += hippi/
 obj-$(CONFIG_HAMRADIO) += hamradio/
-obj-$(CONFIG_IRDA) += irda/
 obj-$(CONFIG_PLIP) += plip/
 obj-$(CONFIG_PPP) += ppp/
 obj-$(CONFIG_PPP_ASYNC) += ppp/
diff --git a/drivers/net/appletalk/ipddp.c b/drivers/net/appletalk/ipddp.c
index a306de4318d7..9375cef22420 100644
--- a/drivers/net/appletalk/ipddp.c
+++ b/drivers/net/appletalk/ipddp.c
@@ -311,9 +311,7 @@ module_param(ipddp_mode, int, 0);
 static int __init ipddp_init_module(void)
 {
 	dev_ipddp = ipddp_init();
-        if (IS_ERR(dev_ipddp))
-                return PTR_ERR(dev_ipddp);
-	return 0;
+	return PTR_ERR_OR_ZERO(dev_ipddp);
 }
 
 static void __exit ipddp_cleanup_module(void)
diff --git a/drivers/net/arcnet/arcdevice.h b/drivers/net/arcnet/arcdevice.h
index cbb4f8566bbe..d09b2b46ab63 100644
--- a/drivers/net/arcnet/arcdevice.h
+++ b/drivers/net/arcnet/arcdevice.h
@@ -20,7 +20,7 @@
 #include <linux/if_arcnet.h>
 
 #ifdef __KERNEL__
-#include  <linux/irqreturn.h>
+#include <linux/interrupt.h>
 
 /*
  * RECON_THRESHOLD is the maximum number of RECON messages to receive
diff --git a/drivers/net/arcnet/com20020-pci.c b/drivers/net/arcnet/com20020-pci.c
index 01cab9548785..eb7f76753c9c 100644
--- a/drivers/net/arcnet/com20020-pci.c
+++ b/drivers/net/arcnet/com20020-pci.c
@@ -109,7 +109,7 @@ static struct attribute *com20020_state_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group com20020_state_group = {
+static const struct attribute_group com20020_state_group = {
 	.name = NULL,
 	.attrs = com20020_state_attrs,
 };
diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c
index 9bee6c1c70cc..c99dc59d729b 100644
--- a/drivers/net/bonding/bond_main.c
+++ b/drivers/net/bonding/bond_main.c
@@ -1569,7 +1569,8 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev)
 	new_slave->delay = 0;
 	new_slave->link_failure_count = 0;
 
-	if (bond_update_speed_duplex(new_slave))
+	if (bond_update_speed_duplex(new_slave) &&
+	    bond_needs_speed_duplex(bond))
 		new_slave->link = BOND_LINK_DOWN;
 
 	new_slave->last_rx = jiffies -
@@ -2140,11 +2141,13 @@ static void bond_miimon_commit(struct bonding *bond)
 			continue;
 
 		case BOND_LINK_UP:
-			if (bond_update_speed_duplex(slave)) {
+			if (bond_update_speed_duplex(slave) &&
+			    bond_needs_speed_duplex(bond)) {
 				slave->link = BOND_LINK_DOWN;
-				netdev_warn(bond->dev,
-					    "failed to get link speed/duplex for %s\n",
-					    slave->dev->name);
+				if (net_ratelimit())
+					netdev_warn(bond->dev,
+						    "failed to get link speed/duplex for %s\n",
+						    slave->dev->name);
 				continue;
 			}
 			bond_set_slave_link_state(slave, BOND_LINK_UP,
@@ -4286,7 +4289,7 @@ static int bond_check_params(struct bond_params *params)
 	int bond_mode	= BOND_MODE_ROUNDROBIN;
 	int xmit_hashtype = BOND_XMIT_POLICY_LAYER2;
 	int lacp_fast = 0;
-	int tlb_dynamic_lb = 0;
+	int tlb_dynamic_lb;
 
 	/* Convert string parameters. */
 	if (mode) {
@@ -4598,16 +4601,13 @@ static int bond_check_params(struct bond_params *params)
 	}
 	ad_user_port_key = valptr->value;
 
-	if ((bond_mode == BOND_MODE_TLB) || (bond_mode == BOND_MODE_ALB)) {
-		bond_opt_initstr(&newval, "default");
-		valptr = bond_opt_parse(bond_opt_get(BOND_OPT_TLB_DYNAMIC_LB),
-					&newval);
-		if (!valptr) {
-			pr_err("Error: No tlb_dynamic_lb default value");
-			return -EINVAL;
-		}
-		tlb_dynamic_lb = valptr->value;
+	bond_opt_initstr(&newval, "default");
+	valptr = bond_opt_parse(bond_opt_get(BOND_OPT_TLB_DYNAMIC_LB), &newval);
+	if (!valptr) {
+		pr_err("Error: No tlb_dynamic_lb default value");
+		return -EINVAL;
 	}
+	tlb_dynamic_lb = valptr->value;
 
 	if (lp_interval == 0) {
 		pr_warn("Warning: ip_interval must be between 1 and %d, so it was reset to %d\n",
diff --git a/drivers/net/bonding/bond_options.c b/drivers/net/bonding/bond_options.c
index a12d603d41c6..5931aa2fe997 100644
--- a/drivers/net/bonding/bond_options.c
+++ b/drivers/net/bonding/bond_options.c
@@ -754,6 +754,9 @@ static int bond_option_mode_set(struct bonding *bond,
 			   bond->params.miimon);
 	}
 
+	if (newval->value == BOND_MODE_ALB)
+		bond->params.tlb_dynamic_lb = 1;
+
 	/* don't cache arp_validate between modes */
 	bond->params.arp_validate = BOND_ARP_VALIDATE_NONE;
 	bond->params.mode = newval->value;
diff --git a/drivers/net/bonding/bond_sysfs.c b/drivers/net/bonding/bond_sysfs.c
index 770623a0cc01..040b493f60ae 100644
--- a/drivers/net/bonding/bond_sysfs.c
+++ b/drivers/net/bonding/bond_sysfs.c
@@ -759,7 +759,7 @@ static struct attribute *per_bond_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group bonding_group = {
+static const struct attribute_group bonding_group = {
 	.name = "bonding",
 	.attrs = per_bond_attrs,
 };
diff --git a/drivers/net/can/at91_can.c b/drivers/net/can/at91_can.c
index 0e0df0ba288c..f37ce0e1b603 100644
--- a/drivers/net/can/at91_can.c
+++ b/drivers/net/can/at91_can.c
@@ -1232,7 +1232,7 @@ static struct attribute *at91_sysfs_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group at91_sysfs_attr_group = {
+static const struct attribute_group at91_sysfs_attr_group = {
 	.attrs = at91_sysfs_attrs,
 };
 
diff --git a/drivers/net/can/c_can/c_can_platform.c b/drivers/net/can/c_can/c_can_platform.c
index e36d10520e24..46a746ee80bb 100644
--- a/drivers/net/can/c_can/c_can_platform.c
+++ b/drivers/net/can/c_can/c_can_platform.c
@@ -220,7 +220,7 @@ static const struct c_can_driver_data am3352_dcan_drvdata = {
 	.raminit_bits = am3352_raminit_bits,
 };
 
-static struct platform_device_id c_can_id_table[] = {
+static const struct platform_device_id c_can_id_table[] = {
 	{
 		.name = KBUILD_MODNAME,
 		.driver_data = (kernel_ulong_t)&c_can_drvdata,
diff --git a/drivers/net/can/janz-ican3.c b/drivers/net/can/janz-ican3.c
index 2ba1a81500c1..12a53c8e8e1d 100644
--- a/drivers/net/can/janz-ican3.c
+++ b/drivers/net/can/janz-ican3.c
@@ -1875,7 +1875,7 @@ static struct attribute *ican3_sysfs_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group ican3_sysfs_attr_group = {
+static const struct attribute_group ican3_sysfs_attr_group = {
 	.attrs = ican3_sysfs_attrs,
 };
 
diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c
index 7f36d3e3c98b..274f3679f33d 100644
--- a/drivers/net/dsa/b53/b53_common.c
+++ b/drivers/net/dsa/b53/b53_common.c
@@ -1053,49 +1053,6 @@ int b53_vlan_del(struct dsa_switch *ds, int port,
 }
 EXPORT_SYMBOL(b53_vlan_del);
 
-int b53_vlan_dump(struct dsa_switch *ds, int port,
-		  struct switchdev_obj_port_vlan *vlan,
-		  switchdev_obj_dump_cb_t *cb)
-{
-	struct b53_device *dev = ds->priv;
-	u16 vid, vid_start = 0, pvid;
-	struct b53_vlan *vl;
-	int err = 0;
-
-	if (is5325(dev) || is5365(dev))
-		vid_start = 1;
-
-	b53_read16(dev, B53_VLAN_PAGE, B53_VLAN_PORT_DEF_TAG(port), &pvid);
-
-	/* Use our software cache for dumps, since we do not have any HW
-	 * operation returning only the used/valid VLANs
-	 */
-	for (vid = vid_start; vid < dev->num_vlans; vid++) {
-		vl = &dev->vlans[vid];
-
-		if (!vl->valid)
-			continue;
-
-		if (!(vl->members & BIT(port)))
-			continue;
-
-		vlan->vid_begin = vlan->vid_end = vid;
-		vlan->flags = 0;
-
-		if (vl->untag & BIT(port))
-			vlan->flags |= BRIDGE_VLAN_INFO_UNTAGGED;
-		if (pvid == vid)
-			vlan->flags |= BRIDGE_VLAN_INFO_PVID;
-
-		err = cb(&vlan->obj);
-		if (err)
-			break;
-	}
-
-	return err;
-}
-EXPORT_SYMBOL(b53_vlan_dump);
-
 /* Address Resolution Logic routines */
 static int b53_arl_op_wait(struct b53_device *dev)
 {
@@ -1213,9 +1170,8 @@ static int b53_arl_op(struct b53_device *dev, int op, int port,
 	return b53_arl_rw_op(dev, 0);
 }
 
-int b53_fdb_prepare(struct dsa_switch *ds, int port,
-		    const struct switchdev_obj_port_fdb *fdb,
-		    struct switchdev_trans *trans)
+int b53_fdb_add(struct dsa_switch *ds, int port,
+		const unsigned char *addr, u16 vid)
 {
 	struct b53_device *priv = ds->priv;
 
@@ -1225,27 +1181,16 @@ int b53_fdb_prepare(struct dsa_switch *ds, int port,
 	if (is5325(priv) || is5365(priv))
 		return -EOPNOTSUPP;
 
-	return 0;
-}
-EXPORT_SYMBOL(b53_fdb_prepare);
-
-void b53_fdb_add(struct dsa_switch *ds, int port,
-		 const struct switchdev_obj_port_fdb *fdb,
-		 struct switchdev_trans *trans)
-{
-	struct b53_device *priv = ds->priv;
-
-	if (b53_arl_op(priv, 0, port, fdb->addr, fdb->vid, true))
-		pr_err("%s: failed to add MAC address\n", __func__);
+	return b53_arl_op(priv, 0, port, addr, vid, true);
 }
 EXPORT_SYMBOL(b53_fdb_add);
 
 int b53_fdb_del(struct dsa_switch *ds, int port,
-		const struct switchdev_obj_port_fdb *fdb)
+		const unsigned char *addr, u16 vid)
 {
 	struct b53_device *priv = ds->priv;
 
-	return b53_arl_op(priv, 0, port, fdb->addr, fdb->vid, false);
+	return b53_arl_op(priv, 0, port, addr, vid, false);
 }
 EXPORT_SYMBOL(b53_fdb_del);
 
@@ -1282,8 +1227,7 @@ static void b53_arl_search_rd(struct b53_device *dev, u8 idx,
 }
 
 static int b53_fdb_copy(int port, const struct b53_arl_entry *ent,
-			struct switchdev_obj_port_fdb *fdb,
-			switchdev_obj_dump_cb_t *cb)
+			dsa_fdb_dump_cb_t *cb, void *data)
 {
 	if (!ent->is_valid)
 		return 0;
@@ -1291,16 +1235,11 @@ static int b53_fdb_copy(int port, const struct b53_arl_entry *ent,
 	if (port != ent->port)
 		return 0;
 
-	ether_addr_copy(fdb->addr, ent->mac);
-	fdb->vid = ent->vid;
-	fdb->ndm_state = ent->is_static ? NUD_NOARP : NUD_REACHABLE;
-
-	return cb(&fdb->obj);
+	return cb(ent->mac, ent->vid, ent->is_static, data);
 }
 
 int b53_fdb_dump(struct dsa_switch *ds, int port,
-		 struct switchdev_obj_port_fdb *fdb,
-		 switchdev_obj_dump_cb_t *cb)
+		 dsa_fdb_dump_cb_t *cb, void *data)
 {
 	struct b53_device *priv = ds->priv;
 	struct b53_arl_entry results[2];
@@ -1318,13 +1257,13 @@ int b53_fdb_dump(struct dsa_switch *ds, int port,
 			return ret;
 
 		b53_arl_search_rd(priv, 0, &results[0]);
-		ret = b53_fdb_copy(port, &results[0], fdb, cb);
+		ret = b53_fdb_copy(port, &results[0], cb, data);
 		if (ret)
 			return ret;
 
 		if (priv->num_arl_entries > 2) {
 			b53_arl_search_rd(priv, 1, &results[1]);
-			ret = b53_fdb_copy(port, &results[1], fdb, cb);
+			ret = b53_fdb_copy(port, &results[1], cb, data);
 			if (ret)
 				return ret;
 
@@ -1564,8 +1503,6 @@ static const struct dsa_switch_ops b53_switch_ops = {
 	.port_vlan_prepare	= b53_vlan_prepare,
 	.port_vlan_add		= b53_vlan_add,
 	.port_vlan_del		= b53_vlan_del,
-	.port_vlan_dump		= b53_vlan_dump,
-	.port_fdb_prepare	= b53_fdb_prepare,
 	.port_fdb_dump		= b53_fdb_dump,
 	.port_fdb_add		= b53_fdb_add,
 	.port_fdb_del		= b53_fdb_del,
diff --git a/drivers/net/dsa/b53/b53_priv.h b/drivers/net/dsa/b53/b53_priv.h
index 155a9c48c317..01bd8cbe9a3f 100644
--- a/drivers/net/dsa/b53/b53_priv.h
+++ b/drivers/net/dsa/b53/b53_priv.h
@@ -393,20 +393,12 @@ void b53_vlan_add(struct dsa_switch *ds, int port,
 		  struct switchdev_trans *trans);
 int b53_vlan_del(struct dsa_switch *ds, int port,
 		 const struct switchdev_obj_port_vlan *vlan);
-int b53_vlan_dump(struct dsa_switch *ds, int port,
-		  struct switchdev_obj_port_vlan *vlan,
-		  switchdev_obj_dump_cb_t *cb);
-int b53_fdb_prepare(struct dsa_switch *ds, int port,
-		    const struct switchdev_obj_port_fdb *fdb,
-		    struct switchdev_trans *trans);
-void b53_fdb_add(struct dsa_switch *ds, int port,
-		 const struct switchdev_obj_port_fdb *fdb,
-		 struct switchdev_trans *trans);
+int b53_fdb_add(struct dsa_switch *ds, int port,
+		const unsigned char *addr, u16 vid);
 int b53_fdb_del(struct dsa_switch *ds, int port,
-		const struct switchdev_obj_port_fdb *fdb);
+		const unsigned char *addr, u16 vid);
 int b53_fdb_dump(struct dsa_switch *ds, int port,
-		 struct switchdev_obj_port_fdb *fdb,
-		 switchdev_obj_dump_cb_t *cb);
+		 dsa_fdb_dump_cb_t *cb, void *data);
 int b53_mirror_add(struct dsa_switch *ds, int port,
 		   struct dsa_mall_mirror_tc_entry *mirror, bool ingress);
 void b53_mirror_del(struct dsa_switch *ds, int port,
diff --git a/drivers/net/dsa/bcm_sf2.c b/drivers/net/dsa/bcm_sf2.c
index 648f91b58d1e..d7b53d53c116 100644
--- a/drivers/net/dsa/bcm_sf2.c
+++ b/drivers/net/dsa/bcm_sf2.c
@@ -103,6 +103,7 @@ static void bcm_sf2_brcm_hdr_setup(struct bcm_sf2_priv *priv, int port)
 static void bcm_sf2_imp_setup(struct dsa_switch *ds, int port)
 {
 	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
+	unsigned int i;
 	u32 reg, offset;
 
 	if (priv->type == BCM7445_DEVICE_ID)
@@ -129,6 +130,14 @@ static void bcm_sf2_imp_setup(struct dsa_switch *ds, int port)
 	reg |= MII_DUMB_FWDG_EN;
 	core_writel(priv, reg, CORE_SWITCH_CTRL);
 
+	/* Configure Traffic Class to QoS mapping, allow each priority to map
+	 * to a different queue number
+	 */
+	reg = core_readl(priv, CORE_PORT_TC2_QOS_MAP_PORT(port));
+	for (i = 0; i < SF2_NUM_EGRESS_QUEUES; i++)
+		reg |= i << (PRT_TO_QID_SHIFT * i);
+	core_writel(priv, reg, CORE_PORT_TC2_QOS_MAP_PORT(port));
+
 	bcm_sf2_brcm_hdr_setup(priv, port);
 
 	/* Force link status for IMP port */
@@ -244,7 +253,7 @@ static int bcm_sf2_port_setup(struct dsa_switch *ds, int port,
 	 * to a different queue number
 	 */
 	reg = core_readl(priv, CORE_PORT_TC2_QOS_MAP_PORT(port));
-	for (i = 0; i < 8; i++)
+	for (i = 0; i < SF2_NUM_EGRESS_QUEUES; i++)
 		reg |= i << (PRT_TO_QID_SHIFT * i);
 	core_writel(priv, reg, CORE_PORT_TC2_QOS_MAP_PORT(port));
 
@@ -327,12 +336,8 @@ static void bcm_sf2_port_disable(struct dsa_switch *ds, int port,
 static int bcm_sf2_eee_init(struct dsa_switch *ds, int port,
 			    struct phy_device *phy)
 {
-	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
-	struct ethtool_eee *p = &priv->port_sts[port].eee;
 	int ret;
 
-	p->supported = (SUPPORTED_1000baseT_Full | SUPPORTED_100baseT_Full);
-
 	ret = phy_init_eee(phy, 0);
 	if (ret)
 		return 0;
@@ -342,8 +347,8 @@ static int bcm_sf2_eee_init(struct dsa_switch *ds, int port,
 	return 1;
 }
 
-static int bcm_sf2_sw_get_eee(struct dsa_switch *ds, int port,
-			      struct ethtool_eee *e)
+static int bcm_sf2_sw_get_mac_eee(struct dsa_switch *ds, int port,
+				  struct ethtool_eee *e)
 {
 	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	struct ethtool_eee *p = &priv->port_sts[port].eee;
@@ -356,22 +361,14 @@ static int bcm_sf2_sw_get_eee(struct dsa_switch *ds, int port,
 	return 0;
 }
 
-static int bcm_sf2_sw_set_eee(struct dsa_switch *ds, int port,
-			      struct phy_device *phydev,
-			      struct ethtool_eee *e)
+static int bcm_sf2_sw_set_mac_eee(struct dsa_switch *ds, int port,
+				  struct ethtool_eee *e)
 {
 	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	struct ethtool_eee *p = &priv->port_sts[port].eee;
 
 	p->eee_enabled = e->eee_enabled;
-
-	if (!p->eee_enabled) {
-		bcm_sf2_eee_enable_set(ds, port, false);
-	} else {
-		p->eee_enabled = bcm_sf2_eee_init(ds, port, phydev);
-		if (!p->eee_enabled)
-			return -EOPNOTSUPP;
-	}
+	bcm_sf2_eee_enable_set(ds, port, e->eee_enabled);
 
 	return 0;
 }
@@ -800,7 +797,7 @@ static int bcm_sf2_sw_resume(struct dsa_switch *ds)
 static void bcm_sf2_sw_get_wol(struct dsa_switch *ds, int port,
 			       struct ethtool_wolinfo *wol)
 {
-	struct net_device *p = ds->dst[ds->index].cpu_dp->netdev;
+	struct net_device *p = ds->dst->cpu_dp->netdev;
 	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	struct ethtool_wolinfo pwol;
 
@@ -823,7 +820,7 @@ static void bcm_sf2_sw_get_wol(struct dsa_switch *ds, int port,
 static int bcm_sf2_sw_set_wol(struct dsa_switch *ds, int port,
 			      struct ethtool_wolinfo *wol)
 {
-	struct net_device *p = ds->dst[ds->index].cpu_dp->netdev;
+	struct net_device *p = ds->dst->cpu_dp->netdev;
 	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	s8 cpu_port = ds->dst->cpu_dp->index;
 	struct ethtool_wolinfo pwol;
@@ -995,7 +992,7 @@ static int bcm_sf2_core_write64(struct b53_device *dev, u8 page, u8 reg,
 	return 0;
 }
 
-static struct b53_io_ops bcm_sf2_io_ops = {
+static const struct b53_io_ops bcm_sf2_io_ops = {
 	.read8	= bcm_sf2_core_read8,
 	.read16	= bcm_sf2_core_read16,
 	.read32	= bcm_sf2_core_read32,
@@ -1023,8 +1020,8 @@ static const struct dsa_switch_ops bcm_sf2_ops = {
 	.set_wol		= bcm_sf2_sw_set_wol,
 	.port_enable		= bcm_sf2_port_setup,
 	.port_disable		= bcm_sf2_port_disable,
-	.get_eee		= bcm_sf2_sw_get_eee,
-	.set_eee		= bcm_sf2_sw_set_eee,
+	.get_mac_eee		= bcm_sf2_sw_get_mac_eee,
+	.set_mac_eee		= bcm_sf2_sw_set_mac_eee,
 	.port_bridge_join	= b53_br_join,
 	.port_bridge_leave	= b53_br_leave,
 	.port_stp_state_set	= b53_br_set_stp_state,
@@ -1033,8 +1030,6 @@ static const struct dsa_switch_ops bcm_sf2_ops = {
 	.port_vlan_prepare	= b53_vlan_prepare,
 	.port_vlan_add		= b53_vlan_add,
 	.port_vlan_del		= b53_vlan_del,
-	.port_vlan_dump		= b53_vlan_dump,
-	.port_fdb_prepare	= b53_fdb_prepare,
 	.port_fdb_dump		= b53_fdb_dump,
 	.port_fdb_add		= b53_fdb_add,
 	.port_fdb_del		= b53_fdb_del,
@@ -1048,6 +1043,7 @@ struct bcm_sf2_of_data {
 	u32 type;
 	const u16 *reg_offsets;
 	unsigned int core_reg_align;
+	unsigned int num_cfp_rules;
 };
 
 /* Register offsets for the SWITCH_REG_* block */
@@ -1071,6 +1067,7 @@ static const struct bcm_sf2_of_data bcm_sf2_7445_data = {
 	.type		= BCM7445_DEVICE_ID,
 	.core_reg_align	= 0,
 	.reg_offsets	= bcm_sf2_7445_reg_offsets,
+	.num_cfp_rules	= 256,
 };
 
 static const u16 bcm_sf2_7278_reg_offsets[] = {
@@ -1093,6 +1090,7 @@ static const struct bcm_sf2_of_data bcm_sf2_7278_data = {
 	.type		= BCM7278_DEVICE_ID,
 	.core_reg_align	= 1,
 	.reg_offsets	= bcm_sf2_7278_reg_offsets,
+	.num_cfp_rules	= 128,
 };
 
 static const struct of_device_id bcm_sf2_of_match[] = {
@@ -1149,6 +1147,7 @@ static int bcm_sf2_sw_probe(struct platform_device *pdev)
 	priv->type = data->type;
 	priv->reg_offsets = data->reg_offsets;
 	priv->core_reg_align = data->core_reg_align;
+	priv->num_cfp_rules = data->num_cfp_rules;
 
 	/* Auto-detection using standard registers will not work, so
 	 * provide an indication of what kind of device we are for
@@ -1161,6 +1160,9 @@ static int bcm_sf2_sw_probe(struct platform_device *pdev)
 	ds = dev->ds;
 	ds->ops = &bcm_sf2_ops;
 
+	/* Advertise the 8 egress queues */
+	ds->num_tx_queues = SF2_NUM_EGRESS_QUEUES;
+
 	dev_set_drvdata(&pdev->dev, priv);
 
 	spin_lock_init(&priv->indir_lock);
diff --git a/drivers/net/dsa/bcm_sf2.h b/drivers/net/dsa/bcm_sf2.h
index 7d3030e04f11..02c499f9c56b 100644
--- a/drivers/net/dsa/bcm_sf2.h
+++ b/drivers/net/dsa/bcm_sf2.h
@@ -72,6 +72,7 @@ struct bcm_sf2_priv {
 	u32 				type;
 	const u16			*reg_offsets;
 	unsigned int			core_reg_align;
+	unsigned int			num_cfp_rules;
 
 	/* spinlock protecting access to the indirect registers */
 	spinlock_t			indir_lock;
@@ -130,12 +131,12 @@ static inline u32 bcm_sf2_mangle_addr(struct bcm_sf2_priv *priv, u32 off)
 #define SF2_IO_MACRO(name) \
 static inline u32 name##_readl(struct bcm_sf2_priv *priv, u32 off)	\
 {									\
-	return __raw_readl(priv->name + off);				\
+	return readl_relaxed(priv->name + off);				\
 }									\
 static inline void name##_writel(struct bcm_sf2_priv *priv,		\
 				  u32 val, u32 off)			\
 {									\
-	__raw_writel(val, priv->name + off);				\
+	writel_relaxed(val, priv->name + off);				\
 }									\
 
 /* Accesses to 64-bits register requires us to latch the hi/lo pairs
@@ -179,23 +180,23 @@ static inline void intrl2_##which##_mask_set(struct bcm_sf2_priv *priv, \
 static inline u32 core_readl(struct bcm_sf2_priv *priv, u32 off)
 {
 	u32 tmp = bcm_sf2_mangle_addr(priv, off);
-	return __raw_readl(priv->core + tmp);
+	return readl_relaxed(priv->core + tmp);
 }
 
 static inline void core_writel(struct bcm_sf2_priv *priv, u32 val, u32 off)
 {
 	u32 tmp = bcm_sf2_mangle_addr(priv, off);
-	__raw_writel(val, priv->core + tmp);
+	writel_relaxed(val, priv->core + tmp);
 }
 
 static inline u32 reg_readl(struct bcm_sf2_priv *priv, u16 off)
 {
-	return __raw_readl(priv->reg + priv->reg_offsets[off]);
+	return readl_relaxed(priv->reg + priv->reg_offsets[off]);
 }
 
 static inline void reg_writel(struct bcm_sf2_priv *priv, u32 val, u16 off)
 {
-	__raw_writel(val, priv->reg + priv->reg_offsets[off]);
+	writel_relaxed(val, priv->reg + priv->reg_offsets[off]);
 }
 
 SF2_IO64_MACRO(core);
diff --git a/drivers/net/dsa/bcm_sf2_cfp.c b/drivers/net/dsa/bcm_sf2_cfp.c
index 2fb32d67065f..8a1da7e67707 100644
--- a/drivers/net/dsa/bcm_sf2_cfp.c
+++ b/drivers/net/dsa/bcm_sf2_cfp.c
@@ -98,7 +98,7 @@ static inline void bcm_sf2_cfp_rule_addr_set(struct bcm_sf2_priv *priv,
 {
 	u32 reg;
 
-	WARN_ON(addr >= CFP_NUM_RULES);
+	WARN_ON(addr >= priv->num_cfp_rules);
 
 	reg = core_readl(priv, CORE_CFP_ACC);
 	reg &= ~(XCESS_ADDR_MASK << XCESS_ADDR_SHIFT);
@@ -109,7 +109,7 @@ static inline void bcm_sf2_cfp_rule_addr_set(struct bcm_sf2_priv *priv,
 static inline unsigned int bcm_sf2_cfp_rule_size(struct bcm_sf2_priv *priv)
 {
 	/* Entry #0 is reserved */
-	return CFP_NUM_RULES - 1;
+	return priv->num_cfp_rules - 1;
 }
 
 static int bcm_sf2_cfp_rule_set(struct dsa_switch *ds, int port,
@@ -523,7 +523,7 @@ static int bcm_sf2_cfp_rule_get_all(struct bcm_sf2_priv *priv,
 		if (!(reg & OP_STR_DONE))
 			break;
 
-	} while (index < CFP_NUM_RULES);
+	} while (index < priv->num_cfp_rules);
 
 	/* Put the TCAM size here */
 	nfc->data = bcm_sf2_cfp_rule_size(priv);
@@ -544,7 +544,7 @@ int bcm_sf2_get_rxnfc(struct dsa_switch *ds, int port,
 	case ETHTOOL_GRXCLSRLCNT:
 		/* Subtract the default, unusable rule */
 		nfc->rule_cnt = bitmap_weight(priv->cfp.used,
-					      CFP_NUM_RULES) - 1;
+					      priv->num_cfp_rules) - 1;
 		/* We support specifying rule locations */
 		nfc->data |= RX_CLS_LOC_SPECIAL;
 		break;
diff --git a/drivers/net/dsa/bcm_sf2_regs.h b/drivers/net/dsa/bcm_sf2_regs.h
index 26052450091e..49695fcc2ea8 100644
--- a/drivers/net/dsa/bcm_sf2_regs.h
+++ b/drivers/net/dsa/bcm_sf2_regs.h
@@ -401,4 +401,7 @@ enum bcm_sf2_reg_offs {
 
 #define CFP_NUM_RULES			256
 
+/* Number of egress queues per port */
+#define SF2_NUM_EGRESS_QUEUES		8
+
 #endif /* __BCM_SF2_REGS_H */
diff --git a/drivers/net/dsa/dsa_loop.c b/drivers/net/dsa/dsa_loop.c
index fdd8f3872102..d55051abf4ed 100644
--- a/drivers/net/dsa/dsa_loop.c
+++ b/drivers/net/dsa/dsa_loop.c
@@ -257,44 +257,7 @@ static int dsa_loop_port_vlan_del(struct dsa_switch *ds, int port,
 	return 0;
 }
 
-static int dsa_loop_port_vlan_dump(struct dsa_switch *ds, int port,
-				   struct switchdev_obj_port_vlan *vlan,
-				   switchdev_obj_dump_cb_t *cb)
-{
-	struct dsa_loop_priv *ps = ds->priv;
-	struct mii_bus *bus = ps->bus;
-	struct dsa_loop_vlan *vl;
-	u16 vid, vid_start = 0;
-	int err = 0;
-
-	dev_dbg(ds->dev, "%s\n", __func__);
-
-	/* Just do a sleeping operation to make lockdep checks effective */
-	mdiobus_read(bus, ps->port_base + port, MII_BMSR);
-
-	for (vid = vid_start; vid < DSA_LOOP_VLANS; vid++) {
-		vl = &ps->vlans[vid];
-
-		if (!(vl->members & BIT(port)))
-			continue;
-
-		vlan->vid_begin = vlan->vid_end = vid;
-		vlan->flags = 0;
-
-		if (vl->untagged & BIT(port))
-			vlan->flags |= BRIDGE_VLAN_INFO_UNTAGGED;
-		if (ps->pvid == vid)
-			vlan->flags |= BRIDGE_VLAN_INFO_PVID;
-
-		err = cb(&vlan->obj);
-		if (err)
-			break;
-	}
-
-	return err;
-}
-
-static struct dsa_switch_ops dsa_loop_driver = {
+static const struct dsa_switch_ops dsa_loop_driver = {
 	.get_tag_protocol	= dsa_loop_get_protocol,
 	.setup			= dsa_loop_setup,
 	.get_strings		= dsa_loop_get_strings,
@@ -310,7 +273,6 @@ static struct dsa_switch_ops dsa_loop_driver = {
 	.port_vlan_prepare	= dsa_loop_port_vlan_prepare,
 	.port_vlan_add		= dsa_loop_port_vlan_add,
 	.port_vlan_del		= dsa_loop_port_vlan_del,
-	.port_vlan_dump		= dsa_loop_port_vlan_dump,
 };
 
 static int dsa_loop_drv_probe(struct mdio_device *mdiodev)
@@ -390,7 +352,7 @@ static void __exit dsa_loop_exit(void)
 
 	mdio_driver_unregister(&dsa_loop_drv);
 	for (i = 0; i < NUM_FIXED_PHYS; i++)
-		if (phydevs[i])
+		if (!IS_ERR(phydevs[i]))
 			fixed_phy_unregister(phydevs[i]);
 }
 module_exit(dsa_loop_exit);
diff --git a/drivers/net/dsa/lan9303-core.c b/drivers/net/dsa/lan9303-core.c
index cd76e61f1fca..b471413d3df9 100644
--- a/drivers/net/dsa/lan9303-core.c
+++ b/drivers/net/dsa/lan9303-core.c
@@ -20,6 +20,11 @@
 
 #include "lan9303.h"
 
+#define LAN9303_NUM_PORTS 3
+
+/* 13.2 System Control and Status Registers
+ * Multiply register number by 4 to get address offset.
+ */
 #define LAN9303_CHIP_REV 0x14
 # define LAN9303_CHIP_ID 0x9303
 #define LAN9303_IRQ_CFG 0x15
@@ -53,6 +58,9 @@
 #define LAN9303_VIRT_PHY_BASE 0x70
 #define LAN9303_VIRT_SPECIAL_CTRL 0x77
 
+/*13.4 Switch Fabric Control and Status Registers
+ * Accessed indirectly via SWITCH_CSR_CMD, SWITCH_CSR_DATA.
+ */
 #define LAN9303_SW_DEV_ID 0x0000
 #define LAN9303_SW_RESET 0x0001
 #define LAN9303_SW_RESET_RESET BIT(0)
@@ -153,9 +161,7 @@
 # define LAN9303_BM_EGRSS_PORT_TYPE_SPECIAL_TAG_PORT1 (BIT(9) | BIT(8))
 # define LAN9303_BM_EGRSS_PORT_TYPE_SPECIAL_TAG_PORT0 (BIT(1) | BIT(0))
 
-#define LAN9303_PORT_0_OFFSET 0x400
-#define LAN9303_PORT_1_OFFSET 0x800
-#define LAN9303_PORT_2_OFFSET 0xc00
+#define LAN9303_SWITCH_PORT_REG(port, reg0) (0x400 * (port) + (reg0))
 
 /* the built-in PHYs are of type LAN911X */
 #define MII_LAN911X_SPECIAL_MODES 0x12
@@ -242,7 +248,7 @@ static int lan9303_virt_phy_reg_write(struct lan9303 *chip, int regnum, u16 val)
 	return regmap_write(chip->regmap, LAN9303_VIRT_PHY_BASE + regnum, val);
 }
 
-static int lan9303_port_phy_reg_wait_for_completion(struct lan9303 *chip)
+static int lan9303_indirect_phy_wait_for_completion(struct lan9303 *chip)
 {
 	int ret, i;
 	u32 reg;
@@ -262,7 +268,7 @@ static int lan9303_port_phy_reg_wait_for_completion(struct lan9303 *chip)
 	return -EIO;
 }
 
-static int lan9303_port_phy_reg_read(struct lan9303 *chip, int addr, int regnum)
+static int lan9303_indirect_phy_read(struct lan9303 *chip, int addr, int regnum)
 {
 	int ret;
 	u32 val;
@@ -272,7 +278,7 @@ static int lan9303_port_phy_reg_read(struct lan9303 *chip, int addr, int regnum)
 
 	mutex_lock(&chip->indirect_mutex);
 
-	ret = lan9303_port_phy_reg_wait_for_completion(chip);
+	ret = lan9303_indirect_phy_wait_for_completion(chip);
 	if (ret)
 		goto on_error;
 
@@ -281,7 +287,7 @@ static int lan9303_port_phy_reg_read(struct lan9303 *chip, int addr, int regnum)
 	if (ret)
 		goto on_error;
 
-	ret = lan9303_port_phy_reg_wait_for_completion(chip);
+	ret = lan9303_indirect_phy_wait_for_completion(chip);
 	if (ret)
 		goto on_error;
 
@@ -299,8 +305,8 @@ on_error:
 	return ret;
 }
 
-static int lan9303_phy_reg_write(struct lan9303 *chip, int addr, int regnum,
-				 unsigned int val)
+static int lan9303_indirect_phy_write(struct lan9303 *chip, int addr,
+				      int regnum, u16 val)
 {
 	int ret;
 	u32 reg;
@@ -311,7 +317,7 @@ static int lan9303_phy_reg_write(struct lan9303 *chip, int addr, int regnum,
 
 	mutex_lock(&chip->indirect_mutex);
 
-	ret = lan9303_port_phy_reg_wait_for_completion(chip);
+	ret = lan9303_indirect_phy_wait_for_completion(chip);
 	if (ret)
 		goto on_error;
 
@@ -328,6 +334,12 @@ on_error:
 	return ret;
 }
 
+const struct lan9303_phy_ops lan9303_indirect_phy_ops = {
+	.phy_read = lan9303_indirect_phy_read,
+	.phy_write = lan9303_indirect_phy_write,
+};
+EXPORT_SYMBOL_GPL(lan9303_indirect_phy_ops);
+
 static int lan9303_switch_wait_for_completion(struct lan9303 *chip)
 {
 	int ret, i;
@@ -416,6 +428,20 @@ on_error:
 	return ret;
 }
 
+static int lan9303_write_switch_port(struct lan9303 *chip, int port,
+				     u16 regnum, u32 val)
+{
+	return lan9303_write_switch_reg(
+		chip, LAN9303_SWITCH_PORT_REG(port, regnum), val);
+}
+
+static int lan9303_read_switch_port(struct lan9303 *chip, int port,
+				    u16 regnum, u32 *val)
+{
+	return lan9303_read_switch_reg(
+		chip, LAN9303_SWITCH_PORT_REG(port, regnum), val);
+}
+
 static int lan9303_detect_phy_setup(struct lan9303 *chip)
 {
 	int reg;
@@ -427,14 +453,15 @@ static int lan9303_detect_phy_setup(struct lan9303 *chip)
 	 * Special reg 18 of phy 3 reads as 0x0000, if 'phy_addr_sel_strap' is 0
 	 * and the IDs are 0-1-2, else it contains something different from
 	 * 0x0000, which means 'phy_addr_sel_strap' is 1 and the IDs are 1-2-3.
+	 * 0xffff is returned on MDIO read with no response.
 	 */
-	reg = lan9303_port_phy_reg_read(chip, 3, MII_LAN911X_SPECIAL_MODES);
+	reg = chip->ops->phy_read(chip, 3, MII_LAN911X_SPECIAL_MODES);
 	if (reg < 0) {
 		dev_err(chip->dev, "Failed to detect phy config: %d\n", reg);
 		return reg;
 	}
 
-	if (reg != 0)
+	if ((reg != 0) && (reg != 0xffff))
 		chip->phy_addr_sel_strap = 1;
 	else
 		chip->phy_addr_sel_strap = 0;
@@ -445,40 +472,37 @@ static int lan9303_detect_phy_setup(struct lan9303 *chip)
 	return 0;
 }
 
-#define LAN9303_MAC_RX_CFG_OFFS (LAN9303_MAC_RX_CFG_0 - LAN9303_PORT_0_OFFSET)
-#define LAN9303_MAC_TX_CFG_OFFS (LAN9303_MAC_TX_CFG_0 - LAN9303_PORT_0_OFFSET)
-
-static int lan9303_disable_packet_processing(struct lan9303 *chip,
-					     unsigned int port)
+static int lan9303_disable_processing_port(struct lan9303 *chip,
+					   unsigned int port)
 {
 	int ret;
 
 	/* disable RX, but keep register reset default values else */
-	ret = lan9303_write_switch_reg(chip, LAN9303_MAC_RX_CFG_OFFS + port,
-				       LAN9303_MAC_RX_CFG_X_REJECT_MAC_TYPES);
+	ret = lan9303_write_switch_port(chip, port, LAN9303_MAC_RX_CFG_0,
+					LAN9303_MAC_RX_CFG_X_REJECT_MAC_TYPES);
 	if (ret)
 		return ret;
 
 	/* disable TX, but keep register reset default values else */
-	return lan9303_write_switch_reg(chip, LAN9303_MAC_TX_CFG_OFFS + port,
+	return lan9303_write_switch_port(chip, port, LAN9303_MAC_TX_CFG_0,
 				LAN9303_MAC_TX_CFG_X_TX_IFG_CONFIG_DEFAULT |
 				LAN9303_MAC_TX_CFG_X_TX_PAD_ENABLE);
 }
 
-static int lan9303_enable_packet_processing(struct lan9303 *chip,
-					    unsigned int port)
+static int lan9303_enable_processing_port(struct lan9303 *chip,
+					  unsigned int port)
 {
 	int ret;
 
 	/* enable RX and keep register reset default values else */
-	ret = lan9303_write_switch_reg(chip, LAN9303_MAC_RX_CFG_OFFS + port,
-				       LAN9303_MAC_RX_CFG_X_REJECT_MAC_TYPES |
-				       LAN9303_MAC_RX_CFG_X_RX_ENABLE);
+	ret = lan9303_write_switch_port(chip, port, LAN9303_MAC_RX_CFG_0,
+					LAN9303_MAC_RX_CFG_X_REJECT_MAC_TYPES |
+					LAN9303_MAC_RX_CFG_X_RX_ENABLE);
 	if (ret)
 		return ret;
 
 	/* enable TX and keep register reset default values else */
-	return lan9303_write_switch_reg(chip, LAN9303_MAC_TX_CFG_OFFS + port,
+	return lan9303_write_switch_port(chip, port, LAN9303_MAC_TX_CFG_0,
 				LAN9303_MAC_TX_CFG_X_TX_IFG_CONFIG_DEFAULT |
 				LAN9303_MAC_TX_CFG_X_TX_PAD_ENABLE |
 				LAN9303_MAC_TX_CFG_X_TX_ENABLE);
@@ -543,15 +567,16 @@ static int lan9303_handle_reset(struct lan9303 *chip)
 /* stop processing packets for all ports */
 static int lan9303_disable_processing(struct lan9303 *chip)
 {
-	int ret;
+	int p;
 
-	ret = lan9303_disable_packet_processing(chip, LAN9303_PORT_0_OFFSET);
-	if (ret)
-		return ret;
-	ret = lan9303_disable_packet_processing(chip, LAN9303_PORT_1_OFFSET);
-	if (ret)
-		return ret;
-	return lan9303_disable_packet_processing(chip, LAN9303_PORT_2_OFFSET);
+	for (p = 0; p < LAN9303_NUM_PORTS; p++) {
+		int ret = lan9303_disable_processing_port(chip, p);
+
+		if (ret)
+			return ret;
+	}
+
+	return 0;
 }
 
 static int lan9303_check_device(struct lan9303 *chip)
@@ -621,7 +646,7 @@ static int lan9303_setup(struct dsa_switch *ds)
 	if (ret)
 		dev_err(chip->dev, "failed to separate ports %d\n", ret);
 
-	ret = lan9303_enable_packet_processing(chip, LAN9303_PORT_0_OFFSET);
+	ret = lan9303_enable_processing_port(chip, 0);
 	if (ret)
 		dev_err(chip->dev, "failed to re-enable switching %d\n", ret);
 
@@ -687,19 +712,18 @@ static void lan9303_get_ethtool_stats(struct dsa_switch *ds, int port,
 				      uint64_t *data)
 {
 	struct lan9303 *chip = ds->priv;
-	u32 reg;
-	unsigned int u, poff;
-	int ret;
-
-	poff = port * 0x400;
+	unsigned int u;
 
 	for (u = 0; u < ARRAY_SIZE(lan9303_mib); u++) {
-		ret = lan9303_read_switch_reg(chip,
-					      lan9303_mib[u].offset + poff,
-					      &reg);
+		u32 reg;
+		int ret;
+
+		ret = lan9303_read_switch_port(
+			chip, port, lan9303_mib[u].offset, &reg);
+
 		if (ret)
-			dev_warn(chip->dev, "Reading status reg %u failed\n",
-				 lan9303_mib[u].offset + poff);
+			dev_warn(chip->dev, "Reading status port %d reg %u failed\n",
+				 port, lan9303_mib[u].offset);
 		data[u] = reg;
 	}
 }
@@ -719,7 +743,7 @@ static int lan9303_phy_read(struct dsa_switch *ds, int phy, int regnum)
 	if (phy > phy_base + 2)
 		return -ENODEV;
 
-	return lan9303_port_phy_reg_read(chip, phy, regnum);
+	return chip->ops->phy_read(chip, phy, regnum);
 }
 
 static int lan9303_phy_write(struct dsa_switch *ds, int phy, int regnum,
@@ -733,7 +757,7 @@ static int lan9303_phy_write(struct dsa_switch *ds, int phy, int regnum,
 	if (phy > phy_base + 2)
 		return -ENODEV;
 
-	return lan9303_phy_reg_write(chip, phy, regnum, val);
+	return chip->ops->phy_write(chip, phy, regnum, val);
 }
 
 static int lan9303_port_enable(struct dsa_switch *ds, int port,
@@ -744,11 +768,8 @@ static int lan9303_port_enable(struct dsa_switch *ds, int port,
 	/* enable internal packet processing */
 	switch (port) {
 	case 1:
-		return lan9303_enable_packet_processing(chip,
-							LAN9303_PORT_1_OFFSET);
 	case 2:
-		return lan9303_enable_packet_processing(chip,
-							LAN9303_PORT_2_OFFSET);
+		return lan9303_enable_processing_port(chip, port);
 	default:
 		dev_dbg(chip->dev,
 			"Error: request to power up invalid port %d\n", port);
@@ -765,14 +786,10 @@ static void lan9303_port_disable(struct dsa_switch *ds, int port,
 	/* disable internal packet processing */
 	switch (port) {
 	case 1:
-		lan9303_disable_packet_processing(chip, LAN9303_PORT_1_OFFSET);
-		lan9303_phy_reg_write(chip, chip->phy_addr_sel_strap + 1,
-				      MII_BMCR, BMCR_PDOWN);
-		break;
 	case 2:
-		lan9303_disable_packet_processing(chip, LAN9303_PORT_2_OFFSET);
-		lan9303_phy_reg_write(chip, chip->phy_addr_sel_strap + 2,
-				      MII_BMCR, BMCR_PDOWN);
+		lan9303_disable_processing_port(chip, port);
+		lan9303_phy_write(ds, chip->phy_addr_sel_strap + port,
+				  MII_BMCR, BMCR_PDOWN);
 		break;
 	default:
 		dev_dbg(chip->dev,
@@ -780,7 +797,7 @@ static void lan9303_port_disable(struct dsa_switch *ds, int port,
 	}
 }
 
-static struct dsa_switch_ops lan9303_switch_ops = {
+static const struct dsa_switch_ops lan9303_switch_ops = {
 	.get_tag_protocol = lan9303_get_tag_protocol,
 	.setup = lan9303_setup,
 	.get_strings = lan9303_get_strings,
@@ -794,7 +811,7 @@ static struct dsa_switch_ops lan9303_switch_ops = {
 
 static int lan9303_register_switch(struct lan9303 *chip)
 {
-	chip->ds = dsa_switch_alloc(chip->dev, DSA_MAX_PORTS);
+	chip->ds = dsa_switch_alloc(chip->dev, LAN9303_NUM_PORTS);
 	if (!chip->ds)
 		return -ENOMEM;
 
diff --git a/drivers/net/dsa/lan9303.h b/drivers/net/dsa/lan9303.h
index d1512dad2d90..4d8be555ff4d 100644
--- a/drivers/net/dsa/lan9303.h
+++ b/drivers/net/dsa/lan9303.h
@@ -2,6 +2,15 @@
 #include <linux/device.h>
 #include <net/dsa.h>
 
+struct lan9303;
+
+struct lan9303_phy_ops {
+	/* PHY 1 and 2 access*/
+	int	(*phy_read)(struct lan9303 *chip, int port, int regnum);
+	int	(*phy_write)(struct lan9303 *chip, int port,
+			     int regnum, u16 val);
+};
+
 struct lan9303 {
 	struct device *dev;
 	struct regmap *regmap;
@@ -11,9 +20,11 @@ struct lan9303 {
 	bool phy_addr_sel_strap;
 	struct dsa_switch *ds;
 	struct mutex indirect_mutex; /* protect indexed register access */
+	const struct lan9303_phy_ops *ops;
 };
 
 extern const struct regmap_access_table lan9303_register_set;
+extern const struct lan9303_phy_ops lan9303_indirect_phy_ops;
 
 int lan9303_probe(struct lan9303 *chip, struct device_node *np);
 int lan9303_remove(struct lan9303 *chip);
diff --git a/drivers/net/dsa/lan9303_i2c.c b/drivers/net/dsa/lan9303_i2c.c
index ab3ce0da5071..24ec20f7f444 100644
--- a/drivers/net/dsa/lan9303_i2c.c
+++ b/drivers/net/dsa/lan9303_i2c.c
@@ -63,6 +63,8 @@ static int lan9303_i2c_probe(struct i2c_client *client,
 	i2c_set_clientdata(client, sw_dev);
 	sw_dev->chip.dev = &client->dev;
 
+	sw_dev->chip.ops = &lan9303_indirect_phy_ops;
+
 	ret = lan9303_probe(&sw_dev->chip, client->dev.of_node);
 	if (ret != 0)
 		return ret;
diff --git a/drivers/net/dsa/lan9303_mdio.c b/drivers/net/dsa/lan9303_mdio.c
index 93c36c0541cf..fc16668a487f 100644
--- a/drivers/net/dsa/lan9303_mdio.c
+++ b/drivers/net/dsa/lan9303_mdio.c
@@ -40,6 +40,7 @@ static int lan9303_mdio_write(void *ctx, uint32_t reg, uint32_t val)
 {
 	struct lan9303_mdio *sw_dev = (struct lan9303_mdio *)ctx;
 
+	reg <<= 2; /* reg num to offset */
 	mutex_lock(&sw_dev->device->bus->mdio_lock);
 	lan9303_mdio_real_write(sw_dev->device, reg, val & 0xffff);
 	lan9303_mdio_real_write(sw_dev->device, reg + 2, (val >> 16) & 0xffff);
@@ -57,6 +58,7 @@ static int lan9303_mdio_read(void *ctx, uint32_t reg, uint32_t *val)
 {
 	struct lan9303_mdio *sw_dev = (struct lan9303_mdio *)ctx;
 
+	reg <<= 2; /* reg num to offset */
 	mutex_lock(&sw_dev->device->bus->mdio_lock);
 	*val = lan9303_mdio_real_read(sw_dev->device, reg);
 	*val |= (lan9303_mdio_real_read(sw_dev->device, reg + 2) << 16);
@@ -65,6 +67,25 @@ static int lan9303_mdio_read(void *ctx, uint32_t reg, uint32_t *val)
 	return 0;
 }
 
+int lan9303_mdio_phy_write(struct lan9303 *chip, int phy, int reg, u16 val)
+{
+	struct lan9303_mdio *sw_dev = dev_get_drvdata(chip->dev);
+
+	return mdiobus_write_nested(sw_dev->device->bus, phy, reg, val);
+}
+
+int lan9303_mdio_phy_read(struct lan9303 *chip, int phy,  int reg)
+{
+	struct lan9303_mdio *sw_dev = dev_get_drvdata(chip->dev);
+
+	return mdiobus_read_nested(sw_dev->device->bus, phy, reg);
+}
+
+static const struct lan9303_phy_ops lan9303_mdio_phy_ops = {
+	.phy_read = lan9303_mdio_phy_read,
+	.phy_write = lan9303_mdio_phy_write,
+};
+
 static const struct regmap_config lan9303_mdio_regmap_config = {
 	.reg_bits = 8,
 	.val_bits = 32,
@@ -106,6 +127,8 @@ static int lan9303_mdio_probe(struct mdio_device *mdiodev)
 	dev_set_drvdata(&mdiodev->dev, sw_dev);
 	sw_dev->chip.dev = &mdiodev->dev;
 
+	sw_dev->chip.ops = &lan9303_mdio_phy_ops;
+
 	ret = lan9303_probe(&sw_dev->chip, mdiodev->dev.of_node);
 	if (ret != 0)
 		return ret;
diff --git a/drivers/net/dsa/microchip/ksz_common.c b/drivers/net/dsa/microchip/ksz_common.c
index b313ecdf2919..56cd6d365352 100644
--- a/drivers/net/dsa/microchip/ksz_common.c
+++ b/drivers/net/dsa/microchip/ksz_common.c
@@ -638,55 +638,6 @@ static int ksz_port_vlan_del(struct dsa_switch *ds, int port,
 	return 0;
 }
 
-static int ksz_port_vlan_dump(struct dsa_switch *ds, int port,
-			      struct switchdev_obj_port_vlan *vlan,
-			      switchdev_obj_dump_cb_t *cb)
-{
-	struct ksz_device *dev = ds->priv;
-	u16 vid;
-	u16 data;
-	struct vlan_table *vlan_cache;
-	int err = 0;
-
-	mutex_lock(&dev->vlan_mutex);
-
-	/* use dev->vlan_cache due to lack of searching valid vlan entry */
-	for (vid = vlan->vid_begin; vid < dev->num_vlans; vid++) {
-		vlan_cache = &dev->vlan_cache[vid];
-
-		if (!(vlan_cache->table[0] & VLAN_VALID))
-			continue;
-
-		vlan->vid_begin = vid;
-		vlan->vid_end = vid;
-		vlan->flags = 0;
-		if (vlan_cache->table[2] & BIT(port)) {
-			if (vlan_cache->table[1] & BIT(port))
-				vlan->flags |= BRIDGE_VLAN_INFO_UNTAGGED;
-			ksz_pread16(dev, port, REG_PORT_DEFAULT_VID, &data);
-			if (vid == (data & 0xFFFFF))
-				vlan->flags |= BRIDGE_VLAN_INFO_PVID;
-
-			err = cb(&vlan->obj);
-			if (err)
-				break;
-		}
-	}
-
-	mutex_unlock(&dev->vlan_mutex);
-
-	return err;
-}
-
-static int ksz_port_fdb_prepare(struct dsa_switch *ds, int port,
-				const struct switchdev_obj_port_fdb *fdb,
-				struct switchdev_trans *trans)
-{
-	/* nothing needed */
-
-	return 0;
-}
-
 struct alu_struct {
 	/* entry 1 */
 	u8	is_static:1;
@@ -706,30 +657,31 @@ struct alu_struct {
 	u8	mac[ETH_ALEN];
 };
 
-static void ksz_port_fdb_add(struct dsa_switch *ds, int port,
-			     const struct switchdev_obj_port_fdb *fdb,
-			     struct switchdev_trans *trans)
+static int ksz_port_fdb_add(struct dsa_switch *ds, int port,
+			    const unsigned char *addr, u16 vid)
 {
 	struct ksz_device *dev = ds->priv;
 	u32 alu_table[4];
 	u32 data;
+	int ret = 0;
 
 	mutex_lock(&dev->alu_mutex);
 
 	/* find any entry with mac & vid */
-	data = fdb->vid << ALU_FID_INDEX_S;
-	data |= ((fdb->addr[0] << 8) | fdb->addr[1]);
+	data = vid << ALU_FID_INDEX_S;
+	data |= ((addr[0] << 8) | addr[1]);
 	ksz_write32(dev, REG_SW_ALU_INDEX_0, data);
 
-	data = ((fdb->addr[2] << 24) | (fdb->addr[3] << 16));
-	data |= ((fdb->addr[4] << 8) | fdb->addr[5]);
+	data = ((addr[2] << 24) | (addr[3] << 16));
+	data |= ((addr[4] << 8) | addr[5]);
 	ksz_write32(dev, REG_SW_ALU_INDEX_1, data);
 
 	/* start read operation */
 	ksz_write32(dev, REG_SW_ALU_CTRL__4, ALU_READ | ALU_START);
 
 	/* wait to be finished */
-	if (wait_alu_ready(dev, ALU_START, 1000) < 0) {
+	ret = wait_alu_ready(dev, ALU_START, 1000);
+	if (ret < 0) {
 		dev_dbg(dev->dev, "Failed to read ALU\n");
 		goto exit;
 	}
@@ -740,27 +692,30 @@ static void ksz_port_fdb_add(struct dsa_switch *ds, int port,
 	/* update ALU entry */
 	alu_table[0] = ALU_V_STATIC_VALID;
 	alu_table[1] |= BIT(port);
-	if (fdb->vid)
+	if (vid)
 		alu_table[1] |= ALU_V_USE_FID;
-	alu_table[2] = (fdb->vid << ALU_V_FID_S);
-	alu_table[2] |= ((fdb->addr[0] << 8) | fdb->addr[1]);
-	alu_table[3] = ((fdb->addr[2] << 24) | (fdb->addr[3] << 16));
-	alu_table[3] |= ((fdb->addr[4] << 8) | fdb->addr[5]);
+	alu_table[2] = (vid << ALU_V_FID_S);
+	alu_table[2] |= ((addr[0] << 8) | addr[1]);
+	alu_table[3] = ((addr[2] << 24) | (addr[3] << 16));
+	alu_table[3] |= ((addr[4] << 8) | addr[5]);
 
 	write_table(ds, alu_table);
 
 	ksz_write32(dev, REG_SW_ALU_CTRL__4, ALU_WRITE | ALU_START);
 
 	/* wait to be finished */
-	if (wait_alu_ready(dev, ALU_START, 1000) < 0)
-		dev_dbg(dev->dev, "Failed to read ALU\n");
+	ret = wait_alu_ready(dev, ALU_START, 1000);
+	if (ret < 0)
+		dev_dbg(dev->dev, "Failed to write ALU\n");
 
 exit:
 	mutex_unlock(&dev->alu_mutex);
+
+	return ret;
 }
 
 static int ksz_port_fdb_del(struct dsa_switch *ds, int port,
-			    const struct switchdev_obj_port_fdb *fdb)
+			    const unsigned char *addr, u16 vid)
 {
 	struct ksz_device *dev = ds->priv;
 	u32 alu_table[4];
@@ -770,12 +725,12 @@ static int ksz_port_fdb_del(struct dsa_switch *ds, int port,
 	mutex_lock(&dev->alu_mutex);
 
 	/* read any entry with mac & vid */
-	data = fdb->vid << ALU_FID_INDEX_S;
-	data |= ((fdb->addr[0] << 8) | fdb->addr[1]);
+	data = vid << ALU_FID_INDEX_S;
+	data |= ((addr[0] << 8) | addr[1]);
 	ksz_write32(dev, REG_SW_ALU_INDEX_0, data);
 
-	data = ((fdb->addr[2] << 24) | (fdb->addr[3] << 16));
-	data |= ((fdb->addr[4] << 8) | fdb->addr[5]);
+	data = ((addr[2] << 24) | (addr[3] << 16));
+	data |= ((addr[4] << 8) | addr[5]);
 	ksz_write32(dev, REG_SW_ALU_INDEX_1, data);
 
 	/* start read operation */
@@ -850,12 +805,11 @@ static void convert_alu(struct alu_struct *alu, u32 *alu_table)
 }
 
 static int ksz_port_fdb_dump(struct dsa_switch *ds, int port,
-			     struct switchdev_obj_port_fdb *fdb,
-			     switchdev_obj_dump_cb_t *cb)
+			     dsa_fdb_dump_cb_t *cb, void *data)
 {
 	struct ksz_device *dev = ds->priv;
 	int ret = 0;
-	u32 data;
+	u32 ksz_data;
 	u32 alu_table[4];
 	struct alu_struct alu;
 	int timeout;
@@ -868,8 +822,8 @@ static int ksz_port_fdb_dump(struct dsa_switch *ds, int port,
 	do {
 		timeout = 1000;
 		do {
-			ksz_read32(dev, REG_SW_ALU_CTRL__4, &data);
-			if ((data & ALU_VALID) || !(data & ALU_START))
+			ksz_read32(dev, REG_SW_ALU_CTRL__4, &ksz_data);
+			if ((ksz_data & ALU_VALID) || !(ksz_data & ALU_START))
 				break;
 			usleep_range(1, 10);
 		} while (timeout-- > 0);
@@ -886,18 +840,11 @@ static int ksz_port_fdb_dump(struct dsa_switch *ds, int port,
 		convert_alu(&alu, alu_table);
 
 		if (alu.port_forward & BIT(port)) {
-			fdb->vid = alu.fid;
-			if (alu.is_static)
-				fdb->ndm_state = NUD_NOARP;
-			else
-				fdb->ndm_state = NUD_REACHABLE;
-			ether_addr_copy(fdb->addr, alu.mac);
-
-			ret = cb(&fdb->obj);
+			ret = cb(alu.mac, alu.fid, alu.is_static, data);
 			if (ret)
 				goto exit;
 		}
-	} while (data & ALU_START);
+	} while (ksz_data & ALU_START);
 
 exit:
 
@@ -1065,14 +1012,6 @@ exit:
 	return ret;
 }
 
-static int ksz_port_mdb_dump(struct dsa_switch *ds, int port,
-			     struct switchdev_obj_port_mdb *mdb,
-			     switchdev_obj_dump_cb_t *cb)
-{
-	/* this is not called by switch layer */
-	return 0;
-}
-
 static int ksz_port_mirror_add(struct dsa_switch *ds, int port,
 			       struct dsa_mall_mirror_tc_entry *mirror,
 			       bool ingress)
@@ -1129,15 +1068,12 @@ static const struct dsa_switch_ops ksz_switch_ops = {
 	.port_vlan_prepare	= ksz_port_vlan_prepare,
 	.port_vlan_add		= ksz_port_vlan_add,
 	.port_vlan_del		= ksz_port_vlan_del,
-	.port_vlan_dump		= ksz_port_vlan_dump,
-	.port_fdb_prepare	= ksz_port_fdb_prepare,
 	.port_fdb_dump		= ksz_port_fdb_dump,
 	.port_fdb_add		= ksz_port_fdb_add,
 	.port_fdb_del		= ksz_port_fdb_del,
 	.port_mdb_prepare       = ksz_port_mdb_prepare,
 	.port_mdb_add           = ksz_port_mdb_add,
 	.port_mdb_del           = ksz_port_mdb_del,
-	.port_mdb_dump          = ksz_port_mdb_dump,
 	.port_mirror_add	= ksz_port_mirror_add,
 	.port_mirror_del	= ksz_port_mirror_del,
 };
diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c
index 264b281eb86b..c142b97add2c 100644
--- a/drivers/net/dsa/mt7530.c
+++ b/drivers/net/dsa/mt7530.c
@@ -839,49 +839,31 @@ mt7530_port_bridge_leave(struct dsa_switch *ds, int port,
 }
 
 static int
-mt7530_port_fdb_prepare(struct dsa_switch *ds, int port,
-			const struct switchdev_obj_port_fdb *fdb,
-			struct switchdev_trans *trans)
+mt7530_port_fdb_add(struct dsa_switch *ds, int port,
+		    const unsigned char *addr, u16 vid)
 {
 	struct mt7530_priv *priv = ds->priv;
 	int ret;
+	u8 port_mask = BIT(port);
 
-	/* Because auto-learned entrie shares the same FDB table.
-	 * an entry is reserved with no port_mask to make sure fdb_add
-	 * is called while the entry is still available.
-	 */
 	mutex_lock(&priv->reg_mutex);
-	mt7530_fdb_write(priv, fdb->vid, 0, fdb->addr, -1, STATIC_ENT);
+	mt7530_fdb_write(priv, vid, port_mask, addr, -1, STATIC_ENT);
 	ret = mt7530_fdb_cmd(priv, MT7530_FDB_WRITE, 0);
 	mutex_unlock(&priv->reg_mutex);
 
 	return ret;
 }
 
-static void
-mt7530_port_fdb_add(struct dsa_switch *ds, int port,
-		    const struct switchdev_obj_port_fdb *fdb,
-		    struct switchdev_trans *trans)
-{
-	struct mt7530_priv *priv = ds->priv;
-	u8 port_mask = BIT(port);
-
-	mutex_lock(&priv->reg_mutex);
-	mt7530_fdb_write(priv, fdb->vid, port_mask, fdb->addr, -1, STATIC_ENT);
-	mt7530_fdb_cmd(priv, MT7530_FDB_WRITE, 0);
-	mutex_unlock(&priv->reg_mutex);
-}
-
 static int
 mt7530_port_fdb_del(struct dsa_switch *ds, int port,
-		    const struct switchdev_obj_port_fdb *fdb)
+		    const unsigned char *addr, u16 vid)
 {
 	struct mt7530_priv *priv = ds->priv;
 	int ret;
 	u8 port_mask = BIT(port);
 
 	mutex_lock(&priv->reg_mutex);
-	mt7530_fdb_write(priv, fdb->vid, port_mask, fdb->addr, -1, STATIC_EMP);
+	mt7530_fdb_write(priv, vid, port_mask, addr, -1, STATIC_EMP);
 	ret = mt7530_fdb_cmd(priv, MT7530_FDB_WRITE, 0);
 	mutex_unlock(&priv->reg_mutex);
 
@@ -890,8 +872,7 @@ mt7530_port_fdb_del(struct dsa_switch *ds, int port,
 
 static int
 mt7530_port_fdb_dump(struct dsa_switch *ds, int port,
-		     struct switchdev_obj_port_fdb *fdb,
-		     switchdev_obj_dump_cb_t *cb)
+		     dsa_fdb_dump_cb_t *cb, void *data)
 {
 	struct mt7530_priv *priv = ds->priv;
 	struct mt7530_fdb _fdb = { 0 };
@@ -909,11 +890,8 @@ mt7530_port_fdb_dump(struct dsa_switch *ds, int port,
 		if (rsp & ATC_SRCH_HIT) {
 			mt7530_fdb_read(priv, &_fdb);
 			if (_fdb.port_mask & BIT(port)) {
-				ether_addr_copy(fdb->addr, _fdb.mac);
-				fdb->vid = _fdb.vid;
-				fdb->ndm_state = _fdb.noarp ?
-						NUD_NOARP : NUD_REACHABLE;
-				ret = cb(&fdb->obj);
+				ret = cb(_fdb.mac, _fdb.vid, _fdb.noarp,
+					 data);
 				if (ret < 0)
 					break;
 			}
@@ -1039,7 +1017,7 @@ mt7530_setup(struct dsa_switch *ds)
 	return 0;
 }
 
-static struct dsa_switch_ops mt7530_switch_ops = {
+static const struct dsa_switch_ops mt7530_switch_ops = {
 	.get_tag_protocol	= mtk_get_tag_protocol,
 	.setup			= mt7530_setup,
 	.get_strings		= mt7530_get_strings,
@@ -1053,7 +1031,6 @@ static struct dsa_switch_ops mt7530_switch_ops = {
 	.port_stp_state_set	= mt7530_stp_state_set,
 	.port_bridge_join	= mt7530_port_bridge_join,
 	.port_bridge_leave	= mt7530_port_bridge_leave,
-	.port_fdb_prepare	= mt7530_port_fdb_prepare,
 	.port_fdb_add		= mt7530_port_fdb_add,
 	.port_fdb_del		= mt7530_port_fdb_del,
 	.port_fdb_dump		= mt7530_port_fdb_dump,
diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c
index 5bcdd33101b0..c6678aa9b4ef 100644
--- a/drivers/net/dsa/mv88e6xxx/chip.c
+++ b/drivers/net/dsa/mv88e6xxx/chip.c
@@ -307,7 +307,7 @@ out:
 	mutex_unlock(&chip->reg_lock);
 }
 
-static struct irq_chip mv88e6xxx_g1_irq_chip = {
+static const struct irq_chip mv88e6xxx_g1_irq_chip = {
 	.name			= "mv88e6xxx-g1",
 	.irq_mask		= mv88e6xxx_g1_irq_mask,
 	.irq_unmask		= mv88e6xxx_g1_irq_unmask,
@@ -810,63 +810,18 @@ static void mv88e6xxx_get_regs(struct dsa_switch *ds, int port,
 	mutex_unlock(&chip->reg_lock);
 }
 
-static int mv88e6xxx_get_eee(struct dsa_switch *ds, int port,
-			     struct ethtool_eee *e)
+static int mv88e6xxx_get_mac_eee(struct dsa_switch *ds, int port,
+				 struct ethtool_eee *e)
 {
-	struct mv88e6xxx_chip *chip = ds->priv;
-	u16 reg;
-	int err;
-
-	if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_EEE))
-		return -EOPNOTSUPP;
-
-	mutex_lock(&chip->reg_lock);
-
-	err = mv88e6xxx_phy_read(chip, port, 16, &reg);
-	if (err)
-		goto out;
-
-	e->eee_enabled = !!(reg & 0x0200);
-	e->tx_lpi_enabled = !!(reg & 0x0100);
-
-	err = mv88e6xxx_port_read(chip, port, MV88E6XXX_PORT_STS, &reg);
-	if (err)
-		goto out;
-
-	e->eee_active = !!(reg & MV88E6352_PORT_STS_EEE);
-out:
-	mutex_unlock(&chip->reg_lock);
-
-	return err;
+	/* Nothing to do on the port's MAC */
+	return 0;
 }
 
-static int mv88e6xxx_set_eee(struct dsa_switch *ds, int port,
-			     struct phy_device *phydev, struct ethtool_eee *e)
+static int mv88e6xxx_set_mac_eee(struct dsa_switch *ds, int port,
+				 struct ethtool_eee *e)
 {
-	struct mv88e6xxx_chip *chip = ds->priv;
-	u16 reg;
-	int err;
-
-	if (!mv88e6xxx_has(chip, MV88E6XXX_FLAG_EEE))
-		return -EOPNOTSUPP;
-
-	mutex_lock(&chip->reg_lock);
-
-	err = mv88e6xxx_phy_read(chip, port, 16, &reg);
-	if (err)
-		goto out;
-
-	reg &= ~0x0300;
-	if (e->eee_enabled)
-		reg |= 0x0200;
-	if (e->tx_lpi_enabled)
-		reg |= 0x0100;
-
-	err = mv88e6xxx_phy_write(chip, port, 16, reg);
-out:
-	mutex_unlock(&chip->reg_lock);
-
-	return err;
+	/* Nothing to do on the port's MAC */
+	return 0;
 }
 
 static u16 mv88e6xxx_port_vlan(struct mv88e6xxx_chip *chip, int dev, int port)
@@ -926,6 +881,22 @@ static void mv88e6xxx_port_stp_state_set(struct dsa_switch *ds, int port,
 		dev_err(ds->dev, "p%d: failed to update state\n", port);
 }
 
+static int mv88e6xxx_pot_setup(struct mv88e6xxx_chip *chip)
+{
+	if (chip->info->ops->pot_clear)
+		return chip->info->ops->pot_clear(chip);
+
+	return 0;
+}
+
+static int mv88e6xxx_rsvd2cpu_setup(struct mv88e6xxx_chip *chip)
+{
+	if (chip->info->ops->mgmt_rsvd2cpu)
+		return chip->info->ops->mgmt_rsvd2cpu(chip);
+
+	return 0;
+}
+
 static int mv88e6xxx_atu_setup(struct mv88e6xxx_chip *chip)
 {
 	int err;
@@ -1040,61 +1011,6 @@ static int mv88e6xxx_vtu_loadpurge(struct mv88e6xxx_chip *chip,
 	return chip->info->ops->vtu_loadpurge(chip, entry);
 }
 
-static int mv88e6xxx_port_vlan_dump(struct dsa_switch *ds, int port,
-				    struct switchdev_obj_port_vlan *vlan,
-				    switchdev_obj_dump_cb_t *cb)
-{
-	struct mv88e6xxx_chip *chip = ds->priv;
-	struct mv88e6xxx_vtu_entry next = {
-		.vid = chip->info->max_vid,
-	};
-	u16 pvid;
-	int err;
-
-	if (!chip->info->max_vid)
-		return -EOPNOTSUPP;
-
-	mutex_lock(&chip->reg_lock);
-
-	err = mv88e6xxx_port_get_pvid(chip, port, &pvid);
-	if (err)
-		goto unlock;
-
-	do {
-		err = mv88e6xxx_vtu_getnext(chip, &next);
-		if (err)
-			break;
-
-		if (!next.valid)
-			break;
-
-		if (next.member[port] ==
-		    MV88E6XXX_G1_VTU_DATA_MEMBER_TAG_NON_MEMBER)
-			continue;
-
-		/* reinit and dump this VLAN obj */
-		vlan->vid_begin = next.vid;
-		vlan->vid_end = next.vid;
-		vlan->flags = 0;
-
-		if (next.member[port] ==
-		    MV88E6XXX_G1_VTU_DATA_MEMBER_TAG_UNTAGGED)
-			vlan->flags |= BRIDGE_VLAN_INFO_UNTAGGED;
-
-		if (next.vid == pvid)
-			vlan->flags |= BRIDGE_VLAN_INFO_PVID;
-
-		err = cb(&vlan->obj);
-		if (err)
-			break;
-	} while (next.vid < chip->info->max_vid);
-
-unlock:
-	mutex_unlock(&chip->reg_lock);
-
-	return err;
-}
-
 static int mv88e6xxx_atu_new(struct mv88e6xxx_chip *chip, u16 *fid)
 {
 	DECLARE_BITMAP(fid_bitmap, MV88E6XXX_N_FID);
@@ -1435,38 +1351,28 @@ static int mv88e6xxx_port_db_load_purge(struct mv88e6xxx_chip *chip, int port,
 	return mv88e6xxx_g1_atu_loadpurge(chip, vlan.fid, &entry);
 }
 
-static int mv88e6xxx_port_fdb_prepare(struct dsa_switch *ds, int port,
-				      const struct switchdev_obj_port_fdb *fdb,
-				      struct switchdev_trans *trans)
-{
-	/* We don't need any dynamic resource from the kernel (yet),
-	 * so skip the prepare phase.
-	 */
-	return 0;
-}
-
-static void mv88e6xxx_port_fdb_add(struct dsa_switch *ds, int port,
-				   const struct switchdev_obj_port_fdb *fdb,
-				   struct switchdev_trans *trans)
+static int mv88e6xxx_port_fdb_add(struct dsa_switch *ds, int port,
+				  const unsigned char *addr, u16 vid)
 {
 	struct mv88e6xxx_chip *chip = ds->priv;
+	int err;
 
 	mutex_lock(&chip->reg_lock);
-	if (mv88e6xxx_port_db_load_purge(chip, port, fdb->addr, fdb->vid,
-					 MV88E6XXX_G1_ATU_DATA_STATE_UC_STATIC))
-		dev_err(ds->dev, "p%d: failed to load unicast MAC address\n",
-			port);
+	err = mv88e6xxx_port_db_load_purge(chip, port, addr, vid,
+					   MV88E6XXX_G1_ATU_DATA_STATE_UC_STATIC);
 	mutex_unlock(&chip->reg_lock);
+
+	return err;
 }
 
 static int mv88e6xxx_port_fdb_del(struct dsa_switch *ds, int port,
-				  const struct switchdev_obj_port_fdb *fdb)
+				  const unsigned char *addr, u16 vid)
 {
 	struct mv88e6xxx_chip *chip = ds->priv;
 	int err;
 
 	mutex_lock(&chip->reg_lock);
-	err = mv88e6xxx_port_db_load_purge(chip, port, fdb->addr, fdb->vid,
+	err = mv88e6xxx_port_db_load_purge(chip, port, addr, vid,
 					   MV88E6XXX_G1_ATU_DATA_STATE_UNUSED);
 	mutex_unlock(&chip->reg_lock);
 
@@ -1475,10 +1381,10 @@ static int mv88e6xxx_port_fdb_del(struct dsa_switch *ds, int port,
 
 static int mv88e6xxx_port_db_dump_fid(struct mv88e6xxx_chip *chip,
 				      u16 fid, u16 vid, int port,
-				      struct switchdev_obj *obj,
-				      switchdev_obj_dump_cb_t *cb)
+				      dsa_fdb_dump_cb_t *cb, void *data)
 {
 	struct mv88e6xxx_atu_entry addr;
+	bool is_static;
 	int err;
 
 	addr.state = MV88E6XXX_G1_ATU_DATA_STATE_UNUSED;
@@ -1495,33 +1401,12 @@ static int mv88e6xxx_port_db_dump_fid(struct mv88e6xxx_chip *chip,
 		if (addr.trunk || (addr.portvec & BIT(port)) == 0)
 			continue;
 
-		if (obj->id == SWITCHDEV_OBJ_ID_PORT_FDB) {
-			struct switchdev_obj_port_fdb *fdb;
-
-			if (!is_unicast_ether_addr(addr.mac))
-				continue;
-
-			fdb = SWITCHDEV_OBJ_PORT_FDB(obj);
-			fdb->vid = vid;
-			ether_addr_copy(fdb->addr, addr.mac);
-			if (addr.state == MV88E6XXX_G1_ATU_DATA_STATE_UC_STATIC)
-				fdb->ndm_state = NUD_NOARP;
-			else
-				fdb->ndm_state = NUD_REACHABLE;
-		} else if (obj->id == SWITCHDEV_OBJ_ID_PORT_MDB) {
-			struct switchdev_obj_port_mdb *mdb;
-
-			if (!is_multicast_ether_addr(addr.mac))
-				continue;
-
-			mdb = SWITCHDEV_OBJ_PORT_MDB(obj);
-			mdb->vid = vid;
-			ether_addr_copy(mdb->addr, addr.mac);
-		} else {
-			return -EOPNOTSUPP;
-		}
+		if (!is_unicast_ether_addr(addr.mac))
+			continue;
 
-		err = cb(obj);
+		is_static = (addr.state ==
+			     MV88E6XXX_G1_ATU_DATA_STATE_UC_STATIC);
+		err = cb(addr.mac, vid, is_static, data);
 		if (err)
 			return err;
 	} while (!is_broadcast_ether_addr(addr.mac));
@@ -1530,8 +1415,7 @@ static int mv88e6xxx_port_db_dump_fid(struct mv88e6xxx_chip *chip,
 }
 
 static int mv88e6xxx_port_db_dump(struct mv88e6xxx_chip *chip, int port,
-				  struct switchdev_obj *obj,
-				  switchdev_obj_dump_cb_t *cb)
+				  dsa_fdb_dump_cb_t *cb, void *data)
 {
 	struct mv88e6xxx_vtu_entry vlan = {
 		.vid = chip->info->max_vid,
@@ -1544,7 +1428,7 @@ static int mv88e6xxx_port_db_dump(struct mv88e6xxx_chip *chip, int port,
 	if (err)
 		return err;
 
-	err = mv88e6xxx_port_db_dump_fid(chip, fid, 0, port, obj, cb);
+	err = mv88e6xxx_port_db_dump_fid(chip, fid, 0, port, cb, data);
 	if (err)
 		return err;
 
@@ -1558,7 +1442,7 @@ static int mv88e6xxx_port_db_dump(struct mv88e6xxx_chip *chip, int port,
 			break;
 
 		err = mv88e6xxx_port_db_dump_fid(chip, vlan.fid, vlan.vid, port,
-						 obj, cb);
+						 cb, data);
 		if (err)
 			return err;
 	} while (vlan.vid < chip->info->max_vid);
@@ -1567,14 +1451,13 @@ static int mv88e6xxx_port_db_dump(struct mv88e6xxx_chip *chip, int port,
 }
 
 static int mv88e6xxx_port_fdb_dump(struct dsa_switch *ds, int port,
-				   struct switchdev_obj_port_fdb *fdb,
-				   switchdev_obj_dump_cb_t *cb)
+				   dsa_fdb_dump_cb_t *cb, void *data)
 {
 	struct mv88e6xxx_chip *chip = ds->priv;
 	int err;
 
 	mutex_lock(&chip->reg_lock);
-	err = mv88e6xxx_port_db_dump(chip, port, &fdb->obj, cb);
+	err = mv88e6xxx_port_db_dump(chip, port, cb, data);
 	mutex_unlock(&chip->reg_lock);
 
 	return err;
@@ -2116,7 +1999,7 @@ static int mv88e6xxx_setup(struct dsa_switch *ds)
 		goto unlock;
 
 	/* Setup Switch Global 2 Registers */
-	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_GLOBAL2)) {
+	if (chip->info->global2_addr) {
 		err = mv88e6xxx_g2_setup(chip);
 		if (err)
 			goto unlock;
@@ -2142,16 +2025,13 @@ static int mv88e6xxx_setup(struct dsa_switch *ds)
 	if (err)
 		goto unlock;
 
-	/* Some generations have the configuration of sending reserved
-	 * management frames to the CPU in global2, others in
-	 * global1. Hence it does not fit the two setup functions
-	 * above.
-	 */
-	if (chip->info->ops->mgmt_rsvd2cpu) {
-		err = chip->info->ops->mgmt_rsvd2cpu(chip);
-		if (err)
-			goto unlock;
-	}
+	err = mv88e6xxx_pot_setup(chip);
+	if (err)
+		goto unlock;
+
+	err = mv88e6xxx_rsvd2cpu_setup(chip);
+	if (err)
+		goto unlock;
 
 unlock:
 	mutex_unlock(&chip->reg_lock);
@@ -2236,7 +2116,7 @@ static int mv88e6xxx_mdio_register(struct mv88e6xxx_chip *chip,
 
 	if (np) {
 		bus->name = np->full_name;
-		snprintf(bus->id, MII_BUS_ID_SIZE, "%s", np->full_name);
+		snprintf(bus->id, MII_BUS_ID_SIZE, "%pOF", np);
 	} else {
 		bus->name = "mv88e6xxx SMI";
 		snprintf(bus->id, MII_BUS_ID_SIZE, "mv88e6xxx-%d", index++);
@@ -2385,7 +2265,8 @@ static const struct mv88e6xxx_ops mv88e6085_ops = {
 	.set_cpu_port = mv88e6095_g1_set_cpu_port,
 	.set_egress_port = mv88e6095_g1_set_egress_port,
 	.watchdog_ops = &mv88e6097_watchdog_ops,
-	.mgmt_rsvd2cpu = mv88e6095_g2_mgmt_rsvd2cpu,
+	.mgmt_rsvd2cpu = mv88e6352_g2_mgmt_rsvd2cpu,
+	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.ppu_enable = mv88e6185_g1_ppu_enable,
 	.ppu_disable = mv88e6185_g1_ppu_disable,
 	.reset = mv88e6185_g1_reset,
@@ -2408,7 +2289,7 @@ static const struct mv88e6xxx_ops mv88e6095_ops = {
 	.stats_get_sset_count = mv88e6095_stats_get_sset_count,
 	.stats_get_strings = mv88e6095_stats_get_strings,
 	.stats_get_stats = mv88e6095_stats_get_stats,
-	.mgmt_rsvd2cpu = mv88e6095_g2_mgmt_rsvd2cpu,
+	.mgmt_rsvd2cpu = mv88e6185_g2_mgmt_rsvd2cpu,
 	.ppu_enable = mv88e6185_g1_ppu_enable,
 	.ppu_disable = mv88e6185_g1_ppu_disable,
 	.reset = mv88e6185_g1_reset,
@@ -2441,7 +2322,8 @@ static const struct mv88e6xxx_ops mv88e6097_ops = {
 	.set_cpu_port = mv88e6095_g1_set_cpu_port,
 	.set_egress_port = mv88e6095_g1_set_egress_port,
 	.watchdog_ops = &mv88e6097_watchdog_ops,
-	.mgmt_rsvd2cpu = mv88e6095_g2_mgmt_rsvd2cpu,
+	.mgmt_rsvd2cpu = mv88e6352_g2_mgmt_rsvd2cpu,
+	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
 	.vtu_getnext = mv88e6352_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6352_g1_vtu_loadpurge,
@@ -2467,7 +2349,8 @@ static const struct mv88e6xxx_ops mv88e6123_ops = {
 	.set_cpu_port = mv88e6095_g1_set_cpu_port,
 	.set_egress_port = mv88e6095_g1_set_egress_port,
 	.watchdog_ops = &mv88e6097_watchdog_ops,
-	.mgmt_rsvd2cpu = mv88e6095_g2_mgmt_rsvd2cpu,
+	.mgmt_rsvd2cpu = mv88e6352_g2_mgmt_rsvd2cpu,
+	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
 	.vtu_getnext = mv88e6352_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6352_g1_vtu_loadpurge,
@@ -2496,7 +2379,7 @@ static const struct mv88e6xxx_ops mv88e6131_ops = {
 	.set_cpu_port = mv88e6095_g1_set_cpu_port,
 	.set_egress_port = mv88e6095_g1_set_egress_port,
 	.watchdog_ops = &mv88e6097_watchdog_ops,
-	.mgmt_rsvd2cpu = mv88e6095_g2_mgmt_rsvd2cpu,
+	.mgmt_rsvd2cpu = mv88e6185_g2_mgmt_rsvd2cpu,
 	.ppu_enable = mv88e6185_g1_ppu_enable,
 	.ppu_disable = mv88e6185_g1_ppu_disable,
 	.reset = mv88e6185_g1_reset,
@@ -2533,6 +2416,7 @@ static const struct mv88e6xxx_ops mv88e6141_ops = {
 	.set_egress_port = mv88e6390_g1_set_egress_port,
 	.watchdog_ops = &mv88e6390_watchdog_ops,
 	.mgmt_rsvd2cpu =  mv88e6390_g1_mgmt_rsvd2cpu,
+	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
 	.vtu_getnext = mv88e6352_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6352_g1_vtu_loadpurge,
@@ -2563,7 +2447,8 @@ static const struct mv88e6xxx_ops mv88e6161_ops = {
 	.set_cpu_port = mv88e6095_g1_set_cpu_port,
 	.set_egress_port = mv88e6095_g1_set_egress_port,
 	.watchdog_ops = &mv88e6097_watchdog_ops,
-	.mgmt_rsvd2cpu = mv88e6095_g2_mgmt_rsvd2cpu,
+	.mgmt_rsvd2cpu = mv88e6352_g2_mgmt_rsvd2cpu,
+	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
 	.vtu_getnext = mv88e6352_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6352_g1_vtu_loadpurge,
@@ -2587,7 +2472,8 @@ static const struct mv88e6xxx_ops mv88e6165_ops = {
 	.set_cpu_port = mv88e6095_g1_set_cpu_port,
 	.set_egress_port = mv88e6095_g1_set_egress_port,
 	.watchdog_ops = &mv88e6097_watchdog_ops,
-	.mgmt_rsvd2cpu = mv88e6095_g2_mgmt_rsvd2cpu,
+	.mgmt_rsvd2cpu = mv88e6352_g2_mgmt_rsvd2cpu,
+	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
 	.vtu_getnext = mv88e6352_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6352_g1_vtu_loadpurge,
@@ -2619,7 +2505,8 @@ static const struct mv88e6xxx_ops mv88e6171_ops = {
 	.set_cpu_port = mv88e6095_g1_set_cpu_port,
 	.set_egress_port = mv88e6095_g1_set_egress_port,
 	.watchdog_ops = &mv88e6097_watchdog_ops,
-	.mgmt_rsvd2cpu = mv88e6095_g2_mgmt_rsvd2cpu,
+	.mgmt_rsvd2cpu = mv88e6352_g2_mgmt_rsvd2cpu,
+	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
 	.vtu_getnext = mv88e6352_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6352_g1_vtu_loadpurge,
@@ -2653,7 +2540,8 @@ static const struct mv88e6xxx_ops mv88e6172_ops = {
 	.set_cpu_port = mv88e6095_g1_set_cpu_port,
 	.set_egress_port = mv88e6095_g1_set_egress_port,
 	.watchdog_ops = &mv88e6097_watchdog_ops,
-	.mgmt_rsvd2cpu = mv88e6095_g2_mgmt_rsvd2cpu,
+	.mgmt_rsvd2cpu = mv88e6352_g2_mgmt_rsvd2cpu,
+	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
 	.vtu_getnext = mv88e6352_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6352_g1_vtu_loadpurge,
@@ -2686,7 +2574,8 @@ static const struct mv88e6xxx_ops mv88e6175_ops = {
 	.set_cpu_port = mv88e6095_g1_set_cpu_port,
 	.set_egress_port = mv88e6095_g1_set_egress_port,
 	.watchdog_ops = &mv88e6097_watchdog_ops,
-	.mgmt_rsvd2cpu = mv88e6095_g2_mgmt_rsvd2cpu,
+	.mgmt_rsvd2cpu = mv88e6352_g2_mgmt_rsvd2cpu,
+	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
 	.vtu_getnext = mv88e6352_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6352_g1_vtu_loadpurge,
@@ -2720,7 +2609,8 @@ static const struct mv88e6xxx_ops mv88e6176_ops = {
 	.set_cpu_port = mv88e6095_g1_set_cpu_port,
 	.set_egress_port = mv88e6095_g1_set_egress_port,
 	.watchdog_ops = &mv88e6097_watchdog_ops,
-	.mgmt_rsvd2cpu = mv88e6095_g2_mgmt_rsvd2cpu,
+	.mgmt_rsvd2cpu = mv88e6352_g2_mgmt_rsvd2cpu,
+	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
 	.vtu_getnext = mv88e6352_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6352_g1_vtu_loadpurge,
@@ -2746,7 +2636,7 @@ static const struct mv88e6xxx_ops mv88e6185_ops = {
 	.set_cpu_port = mv88e6095_g1_set_cpu_port,
 	.set_egress_port = mv88e6095_g1_set_egress_port,
 	.watchdog_ops = &mv88e6097_watchdog_ops,
-	.mgmt_rsvd2cpu = mv88e6095_g2_mgmt_rsvd2cpu,
+	.mgmt_rsvd2cpu = mv88e6185_g2_mgmt_rsvd2cpu,
 	.ppu_enable = mv88e6185_g1_ppu_enable,
 	.ppu_disable = mv88e6185_g1_ppu_disable,
 	.reset = mv88e6185_g1_reset,
@@ -2782,6 +2672,7 @@ static const struct mv88e6xxx_ops mv88e6190_ops = {
 	.set_egress_port = mv88e6390_g1_set_egress_port,
 	.watchdog_ops = &mv88e6390_watchdog_ops,
 	.mgmt_rsvd2cpu = mv88e6390_g1_mgmt_rsvd2cpu,
+	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
 	.vtu_getnext = mv88e6390_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6390_g1_vtu_loadpurge,
@@ -2816,6 +2707,7 @@ static const struct mv88e6xxx_ops mv88e6190x_ops = {
 	.set_egress_port = mv88e6390_g1_set_egress_port,
 	.watchdog_ops = &mv88e6390_watchdog_ops,
 	.mgmt_rsvd2cpu = mv88e6390_g1_mgmt_rsvd2cpu,
+	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
 	.vtu_getnext = mv88e6390_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6390_g1_vtu_loadpurge,
@@ -2850,6 +2742,7 @@ static const struct mv88e6xxx_ops mv88e6191_ops = {
 	.set_egress_port = mv88e6390_g1_set_egress_port,
 	.watchdog_ops = &mv88e6390_watchdog_ops,
 	.mgmt_rsvd2cpu = mv88e6390_g1_mgmt_rsvd2cpu,
+	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
 	.vtu_getnext = mv88e6390_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6390_g1_vtu_loadpurge,
@@ -2884,7 +2777,8 @@ static const struct mv88e6xxx_ops mv88e6240_ops = {
 	.set_cpu_port = mv88e6095_g1_set_cpu_port,
 	.set_egress_port = mv88e6095_g1_set_egress_port,
 	.watchdog_ops = &mv88e6097_watchdog_ops,
-	.mgmt_rsvd2cpu = mv88e6095_g2_mgmt_rsvd2cpu,
+	.mgmt_rsvd2cpu = mv88e6352_g2_mgmt_rsvd2cpu,
+	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
 	.vtu_getnext = mv88e6352_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6352_g1_vtu_loadpurge,
@@ -2920,6 +2814,7 @@ static const struct mv88e6xxx_ops mv88e6290_ops = {
 	.set_egress_port = mv88e6390_g1_set_egress_port,
 	.watchdog_ops = &mv88e6390_watchdog_ops,
 	.mgmt_rsvd2cpu = mv88e6390_g1_mgmt_rsvd2cpu,
+	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
 	.vtu_getnext = mv88e6390_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6390_g1_vtu_loadpurge,
@@ -2952,14 +2847,15 @@ static const struct mv88e6xxx_ops mv88e6320_ops = {
 	.stats_get_stats = mv88e6320_stats_get_stats,
 	.set_cpu_port = mv88e6095_g1_set_cpu_port,
 	.set_egress_port = mv88e6095_g1_set_egress_port,
-	.mgmt_rsvd2cpu = mv88e6095_g2_mgmt_rsvd2cpu,
+	.mgmt_rsvd2cpu = mv88e6352_g2_mgmt_rsvd2cpu,
+	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
 	.vtu_getnext = mv88e6185_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6185_g1_vtu_loadpurge,
 };
 
 static const struct mv88e6xxx_ops mv88e6321_ops = {
-	/* MV88E6XXX_FAMILY_6321 */
+	/* MV88E6XXX_FAMILY_6320 */
 	.irl_init_all = mv88e6352_g2_irl_init_all,
 	.get_eeprom = mv88e6xxx_g2_get_eeprom16,
 	.set_eeprom = mv88e6xxx_g2_set_eeprom16,
@@ -3018,6 +2914,7 @@ static const struct mv88e6xxx_ops mv88e6341_ops = {
 	.set_egress_port = mv88e6390_g1_set_egress_port,
 	.watchdog_ops = &mv88e6390_watchdog_ops,
 	.mgmt_rsvd2cpu =  mv88e6390_g1_mgmt_rsvd2cpu,
+	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
 	.vtu_getnext = mv88e6352_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6352_g1_vtu_loadpurge,
@@ -3049,7 +2946,8 @@ static const struct mv88e6xxx_ops mv88e6350_ops = {
 	.set_cpu_port = mv88e6095_g1_set_cpu_port,
 	.set_egress_port = mv88e6095_g1_set_egress_port,
 	.watchdog_ops = &mv88e6097_watchdog_ops,
-	.mgmt_rsvd2cpu = mv88e6095_g2_mgmt_rsvd2cpu,
+	.mgmt_rsvd2cpu = mv88e6352_g2_mgmt_rsvd2cpu,
+	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
 	.vtu_getnext = mv88e6352_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6352_g1_vtu_loadpurge,
@@ -3081,7 +2979,8 @@ static const struct mv88e6xxx_ops mv88e6351_ops = {
 	.set_cpu_port = mv88e6095_g1_set_cpu_port,
 	.set_egress_port = mv88e6095_g1_set_egress_port,
 	.watchdog_ops = &mv88e6097_watchdog_ops,
-	.mgmt_rsvd2cpu = mv88e6095_g2_mgmt_rsvd2cpu,
+	.mgmt_rsvd2cpu = mv88e6352_g2_mgmt_rsvd2cpu,
+	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
 	.vtu_getnext = mv88e6352_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6352_g1_vtu_loadpurge,
@@ -3115,7 +3014,8 @@ static const struct mv88e6xxx_ops mv88e6352_ops = {
 	.set_cpu_port = mv88e6095_g1_set_cpu_port,
 	.set_egress_port = mv88e6095_g1_set_egress_port,
 	.watchdog_ops = &mv88e6097_watchdog_ops,
-	.mgmt_rsvd2cpu = mv88e6095_g2_mgmt_rsvd2cpu,
+	.mgmt_rsvd2cpu = mv88e6352_g2_mgmt_rsvd2cpu,
+	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
 	.vtu_getnext = mv88e6352_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6352_g1_vtu_loadpurge,
@@ -3153,6 +3053,7 @@ static const struct mv88e6xxx_ops mv88e6390_ops = {
 	.set_egress_port = mv88e6390_g1_set_egress_port,
 	.watchdog_ops = &mv88e6390_watchdog_ops,
 	.mgmt_rsvd2cpu = mv88e6390_g1_mgmt_rsvd2cpu,
+	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
 	.vtu_getnext = mv88e6390_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6390_g1_vtu_loadpurge,
@@ -3190,6 +3091,7 @@ static const struct mv88e6xxx_ops mv88e6390x_ops = {
 	.set_egress_port = mv88e6390_g1_set_egress_port,
 	.watchdog_ops = &mv88e6390_watchdog_ops,
 	.mgmt_rsvd2cpu = mv88e6390_g1_mgmt_rsvd2cpu,
+	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.reset = mv88e6352_g1_reset,
 	.vtu_getnext = mv88e6390_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6390_g1_vtu_loadpurge,
@@ -3206,12 +3108,14 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 4095,
 		.port_base_addr = 0x10,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.age_time_coeff = 15000,
 		.g1_irqs = 8,
+		.g2_irqs = 10,
 		.atu_move_port_mask = 0xf,
 		.pvt = true,
+		.multi_chip = true,
 		.tag_protocol = DSA_TAG_PROTO_DSA,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6097,
 		.ops = &mv88e6085_ops,
 	},
 
@@ -3224,11 +3128,12 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 4095,
 		.port_base_addr = 0x10,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.age_time_coeff = 15000,
 		.g1_irqs = 8,
 		.atu_move_port_mask = 0xf,
+		.multi_chip = true,
 		.tag_protocol = DSA_TAG_PROTO_DSA,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6095,
 		.ops = &mv88e6095_ops,
 	},
 
@@ -3241,12 +3146,14 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 4095,
 		.port_base_addr = 0x10,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.age_time_coeff = 15000,
 		.g1_irqs = 8,
+		.g2_irqs = 10,
 		.atu_move_port_mask = 0xf,
 		.pvt = true,
+		.multi_chip = true,
 		.tag_protocol = DSA_TAG_PROTO_EDSA,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6097,
 		.ops = &mv88e6097_ops,
 	},
 
@@ -3259,12 +3166,14 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 4095,
 		.port_base_addr = 0x10,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.age_time_coeff = 15000,
 		.g1_irqs = 9,
+		.g2_irqs = 10,
 		.atu_move_port_mask = 0xf,
 		.pvt = true,
+		.multi_chip = true,
 		.tag_protocol = DSA_TAG_PROTO_EDSA,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6165,
 		.ops = &mv88e6123_ops,
 	},
 
@@ -3277,11 +3186,12 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 4095,
 		.port_base_addr = 0x10,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.age_time_coeff = 15000,
 		.g1_irqs = 9,
 		.atu_move_port_mask = 0xf,
+		.multi_chip = true,
 		.tag_protocol = DSA_TAG_PROTO_DSA,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6185,
 		.ops = &mv88e6131_ops,
 	},
 
@@ -3294,11 +3204,13 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 4095,
 		.port_base_addr = 0x10,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.age_time_coeff = 3750,
 		.atu_move_port_mask = 0x1f,
+		.g2_irqs = 10,
 		.pvt = true,
+		.multi_chip = true,
 		.tag_protocol = DSA_TAG_PROTO_EDSA,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6341,
 		.ops = &mv88e6141_ops,
 	},
 
@@ -3311,12 +3223,14 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 4095,
 		.port_base_addr = 0x10,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.age_time_coeff = 15000,
 		.g1_irqs = 9,
+		.g2_irqs = 10,
 		.atu_move_port_mask = 0xf,
 		.pvt = true,
+		.multi_chip = true,
 		.tag_protocol = DSA_TAG_PROTO_EDSA,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6165,
 		.ops = &mv88e6161_ops,
 	},
 
@@ -3329,12 +3243,14 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 4095,
 		.port_base_addr = 0x10,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.age_time_coeff = 15000,
 		.g1_irqs = 9,
+		.g2_irqs = 10,
 		.atu_move_port_mask = 0xf,
 		.pvt = true,
+		.multi_chip = true,
 		.tag_protocol = DSA_TAG_PROTO_DSA,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6165,
 		.ops = &mv88e6165_ops,
 	},
 
@@ -3347,12 +3263,14 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 4095,
 		.port_base_addr = 0x10,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.age_time_coeff = 15000,
 		.g1_irqs = 9,
+		.g2_irqs = 10,
 		.atu_move_port_mask = 0xf,
 		.pvt = true,
+		.multi_chip = true,
 		.tag_protocol = DSA_TAG_PROTO_EDSA,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6351,
 		.ops = &mv88e6171_ops,
 	},
 
@@ -3365,12 +3283,14 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 4095,
 		.port_base_addr = 0x10,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.age_time_coeff = 15000,
 		.g1_irqs = 9,
+		.g2_irqs = 10,
 		.atu_move_port_mask = 0xf,
 		.pvt = true,
+		.multi_chip = true,
 		.tag_protocol = DSA_TAG_PROTO_EDSA,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6352,
 		.ops = &mv88e6172_ops,
 	},
 
@@ -3383,12 +3303,14 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 4095,
 		.port_base_addr = 0x10,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.age_time_coeff = 15000,
 		.g1_irqs = 9,
+		.g2_irqs = 10,
 		.atu_move_port_mask = 0xf,
 		.pvt = true,
+		.multi_chip = true,
 		.tag_protocol = DSA_TAG_PROTO_EDSA,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6351,
 		.ops = &mv88e6175_ops,
 	},
 
@@ -3401,12 +3323,14 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 4095,
 		.port_base_addr = 0x10,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.age_time_coeff = 15000,
 		.g1_irqs = 9,
+		.g2_irqs = 10,
 		.atu_move_port_mask = 0xf,
 		.pvt = true,
+		.multi_chip = true,
 		.tag_protocol = DSA_TAG_PROTO_EDSA,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6352,
 		.ops = &mv88e6176_ops,
 	},
 
@@ -3419,11 +3343,12 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 4095,
 		.port_base_addr = 0x10,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.age_time_coeff = 15000,
 		.g1_irqs = 8,
 		.atu_move_port_mask = 0xf,
+		.multi_chip = true,
 		.tag_protocol = DSA_TAG_PROTO_EDSA,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6185,
 		.ops = &mv88e6185_ops,
 	},
 
@@ -3436,12 +3361,14 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 8191,
 		.port_base_addr = 0x0,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.tag_protocol = DSA_TAG_PROTO_DSA,
 		.age_time_coeff = 3750,
 		.g1_irqs = 9,
+		.g2_irqs = 14,
 		.pvt = true,
+		.multi_chip = true,
 		.atu_move_port_mask = 0x1f,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6390,
 		.ops = &mv88e6190_ops,
 	},
 
@@ -3454,12 +3381,14 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 8191,
 		.port_base_addr = 0x0,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.age_time_coeff = 3750,
 		.g1_irqs = 9,
+		.g2_irqs = 14,
 		.atu_move_port_mask = 0x1f,
 		.pvt = true,
+		.multi_chip = true,
 		.tag_protocol = DSA_TAG_PROTO_DSA,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6390,
 		.ops = &mv88e6190x_ops,
 	},
 
@@ -3472,12 +3401,14 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 8191,
 		.port_base_addr = 0x0,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.age_time_coeff = 3750,
 		.g1_irqs = 9,
+		.g2_irqs = 14,
 		.atu_move_port_mask = 0x1f,
 		.pvt = true,
+		.multi_chip = true,
 		.tag_protocol = DSA_TAG_PROTO_DSA,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6390,
 		.ops = &mv88e6191_ops,
 	},
 
@@ -3490,12 +3421,14 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 4095,
 		.port_base_addr = 0x10,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.age_time_coeff = 15000,
 		.g1_irqs = 9,
+		.g2_irqs = 10,
 		.atu_move_port_mask = 0xf,
 		.pvt = true,
+		.multi_chip = true,
 		.tag_protocol = DSA_TAG_PROTO_EDSA,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6352,
 		.ops = &mv88e6240_ops,
 	},
 
@@ -3508,12 +3441,14 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 8191,
 		.port_base_addr = 0x0,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.age_time_coeff = 3750,
 		.g1_irqs = 9,
+		.g2_irqs = 14,
 		.atu_move_port_mask = 0x1f,
 		.pvt = true,
+		.multi_chip = true,
 		.tag_protocol = DSA_TAG_PROTO_DSA,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6390,
 		.ops = &mv88e6290_ops,
 	},
 
@@ -3526,12 +3461,13 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 4095,
 		.port_base_addr = 0x10,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.age_time_coeff = 15000,
 		.g1_irqs = 8,
 		.atu_move_port_mask = 0xf,
 		.pvt = true,
+		.multi_chip = true,
 		.tag_protocol = DSA_TAG_PROTO_EDSA,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6320,
 		.ops = &mv88e6320_ops,
 	},
 
@@ -3544,11 +3480,12 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 4095,
 		.port_base_addr = 0x10,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.age_time_coeff = 15000,
 		.g1_irqs = 8,
 		.atu_move_port_mask = 0xf,
+		.multi_chip = true,
 		.tag_protocol = DSA_TAG_PROTO_EDSA,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6320,
 		.ops = &mv88e6321_ops,
 	},
 
@@ -3561,11 +3498,13 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 4095,
 		.port_base_addr = 0x10,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.age_time_coeff = 3750,
 		.atu_move_port_mask = 0x1f,
+		.g2_irqs = 10,
 		.pvt = true,
+		.multi_chip = true,
 		.tag_protocol = DSA_TAG_PROTO_EDSA,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6341,
 		.ops = &mv88e6341_ops,
 	},
 
@@ -3578,12 +3517,14 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 4095,
 		.port_base_addr = 0x10,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.age_time_coeff = 15000,
 		.g1_irqs = 9,
+		.g2_irqs = 10,
 		.atu_move_port_mask = 0xf,
 		.pvt = true,
+		.multi_chip = true,
 		.tag_protocol = DSA_TAG_PROTO_EDSA,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6351,
 		.ops = &mv88e6350_ops,
 	},
 
@@ -3596,12 +3537,14 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 4095,
 		.port_base_addr = 0x10,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.age_time_coeff = 15000,
 		.g1_irqs = 9,
+		.g2_irqs = 10,
 		.atu_move_port_mask = 0xf,
 		.pvt = true,
+		.multi_chip = true,
 		.tag_protocol = DSA_TAG_PROTO_EDSA,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6351,
 		.ops = &mv88e6351_ops,
 	},
 
@@ -3614,12 +3557,14 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 4095,
 		.port_base_addr = 0x10,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.age_time_coeff = 15000,
 		.g1_irqs = 9,
+		.g2_irqs = 10,
 		.atu_move_port_mask = 0xf,
 		.pvt = true,
+		.multi_chip = true,
 		.tag_protocol = DSA_TAG_PROTO_EDSA,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6352,
 		.ops = &mv88e6352_ops,
 	},
 	[MV88E6390] = {
@@ -3631,12 +3576,14 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 8191,
 		.port_base_addr = 0x0,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.age_time_coeff = 3750,
 		.g1_irqs = 9,
+		.g2_irqs = 14,
 		.atu_move_port_mask = 0x1f,
 		.pvt = true,
+		.multi_chip = true,
 		.tag_protocol = DSA_TAG_PROTO_DSA,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6390,
 		.ops = &mv88e6390_ops,
 	},
 	[MV88E6390X] = {
@@ -3648,12 +3595,14 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.max_vid = 8191,
 		.port_base_addr = 0x0,
 		.global1_addr = 0x1b,
+		.global2_addr = 0x1c,
 		.age_time_coeff = 3750,
 		.g1_irqs = 9,
+		.g2_irqs = 14,
 		.atu_move_port_mask = 0x1f,
 		.pvt = true,
+		.multi_chip = true,
 		.tag_protocol = DSA_TAG_PROTO_DSA,
-		.flags = MV88E6XXX_FLAGS_FAMILY_6390,
 		.ops = &mv88e6390x_ops,
 	},
 };
@@ -3723,7 +3672,7 @@ static int mv88e6xxx_smi_init(struct mv88e6xxx_chip *chip,
 {
 	if (sw_addr == 0)
 		chip->smi_ops = &mv88e6xxx_smi_single_chip_ops;
-	else if (mv88e6xxx_has(chip, MV88E6XXX_FLAGS_MULTI_CHIP))
+	else if (chip->info->multi_chip)
 		chip->smi_ops = &mv88e6xxx_smi_multi_chip_ops;
 	else
 		return -EINVAL;
@@ -3828,20 +3777,6 @@ static int mv88e6xxx_port_mdb_del(struct dsa_switch *ds, int port,
 	return err;
 }
 
-static int mv88e6xxx_port_mdb_dump(struct dsa_switch *ds, int port,
-				   struct switchdev_obj_port_mdb *mdb,
-				   switchdev_obj_dump_cb_t *cb)
-{
-	struct mv88e6xxx_chip *chip = ds->priv;
-	int err;
-
-	mutex_lock(&chip->reg_lock);
-	err = mv88e6xxx_port_db_dump(chip, port, &mdb->obj, cb);
-	mutex_unlock(&chip->reg_lock);
-
-	return err;
-}
-
 static const struct dsa_switch_ops mv88e6xxx_switch_ops = {
 	.probe			= mv88e6xxx_drv_probe,
 	.get_tag_protocol	= mv88e6xxx_get_tag_protocol,
@@ -3853,8 +3788,8 @@ static const struct dsa_switch_ops mv88e6xxx_switch_ops = {
 	.get_sset_count		= mv88e6xxx_get_sset_count,
 	.port_enable		= mv88e6xxx_port_enable,
 	.port_disable		= mv88e6xxx_port_disable,
-	.set_eee		= mv88e6xxx_set_eee,
-	.get_eee		= mv88e6xxx_get_eee,
+	.get_mac_eee		= mv88e6xxx_get_mac_eee,
+	.set_mac_eee		= mv88e6xxx_set_mac_eee,
 	.get_eeprom_len		= mv88e6xxx_get_eeprom_len,
 	.get_eeprom		= mv88e6xxx_get_eeprom,
 	.set_eeprom		= mv88e6xxx_set_eeprom,
@@ -3869,15 +3804,12 @@ static const struct dsa_switch_ops mv88e6xxx_switch_ops = {
 	.port_vlan_prepare	= mv88e6xxx_port_vlan_prepare,
 	.port_vlan_add		= mv88e6xxx_port_vlan_add,
 	.port_vlan_del		= mv88e6xxx_port_vlan_del,
-	.port_vlan_dump		= mv88e6xxx_port_vlan_dump,
-	.port_fdb_prepare       = mv88e6xxx_port_fdb_prepare,
 	.port_fdb_add           = mv88e6xxx_port_fdb_add,
 	.port_fdb_del           = mv88e6xxx_port_fdb_del,
 	.port_fdb_dump          = mv88e6xxx_port_fdb_dump,
 	.port_mdb_prepare       = mv88e6xxx_port_mdb_prepare,
 	.port_mdb_add           = mv88e6xxx_port_mdb_add,
 	.port_mdb_del           = mv88e6xxx_port_mdb_del,
-	.port_mdb_dump          = mv88e6xxx_port_mdb_dump,
 	.crosschip_bridge_join	= mv88e6xxx_crosschip_bridge_join,
 	.crosschip_bridge_leave	= mv88e6xxx_crosschip_bridge_leave,
 };
@@ -3971,7 +3903,7 @@ static int mv88e6xxx_probe(struct mdio_device *mdiodev)
 		if (err)
 			goto out;
 
-		if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_G2_INT)) {
+		if (chip->info->g2_irqs > 0) {
 			err = mv88e6xxx_g2_irq_setup(chip);
 			if (err)
 				goto out_g1_irq;
@@ -3991,7 +3923,7 @@ static int mv88e6xxx_probe(struct mdio_device *mdiodev)
 out_mdio:
 	mv88e6xxx_mdios_unregister(chip);
 out_g2_irq:
-	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_G2_INT) && chip->irq > 0)
+	if (chip->info->g2_irqs > 0 && chip->irq > 0)
 		mv88e6xxx_g2_irq_free(chip);
 out_g1_irq:
 	if (chip->irq > 0) {
@@ -4013,7 +3945,7 @@ static void mv88e6xxx_remove(struct mdio_device *mdiodev)
 	mv88e6xxx_mdios_unregister(chip);
 
 	if (chip->irq > 0) {
-		if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_G2_INT))
+		if (chip->info->g2_irqs > 0)
 			mv88e6xxx_g2_irq_free(chip);
 		mv88e6xxx_g1_irq_free(chip);
 	}
diff --git a/drivers/net/dsa/mv88e6xxx/chip.h b/drivers/net/dsa/mv88e6xxx/chip.h
index 086444016352..334f6f7544ba 100644
--- a/drivers/net/dsa/mv88e6xxx/chip.h
+++ b/drivers/net/dsa/mv88e6xxx/chip.h
@@ -97,133 +97,6 @@ enum mv88e6xxx_family {
 	MV88E6XXX_FAMILY_6390,  /* 6190 6190X 6191 6290 6390 6390X */
 };
 
-enum mv88e6xxx_cap {
-	/* Energy Efficient Ethernet.
-	 */
-	MV88E6XXX_CAP_EEE,
-
-	/* Multi-chip Addressing Mode.
-	 * Some chips respond to only 2 registers of its own SMI device address
-	 * when it is non-zero, and use indirect access to internal registers.
-	 */
-	MV88E6XXX_CAP_SMI_CMD,		/* (0x00) SMI Command */
-	MV88E6XXX_CAP_SMI_DATA,		/* (0x01) SMI Data */
-
-	/* Switch Global (1) Registers.
-	 */
-	MV88E6XXX_CAP_G1_ATU_FID,	/* (0x01) ATU FID Register */
-	MV88E6XXX_CAP_G1_VTU_FID,	/* (0x02) VTU FID Register */
-
-	/* Switch Global 2 Registers.
-	 * The device contains a second set of global 16-bit registers.
-	 */
-	MV88E6XXX_CAP_GLOBAL2,
-	MV88E6XXX_CAP_G2_INT,		/* (0x00) Interrupt Status */
-	MV88E6XXX_CAP_G2_MGMT_EN_2X,	/* (0x02) MGMT Enable Register 2x */
-	MV88E6XXX_CAP_G2_MGMT_EN_0X,	/* (0x03) MGMT Enable Register 0x */
-	MV88E6XXX_CAP_G2_POT,		/* (0x0f) Priority Override Table */
-
-	/* Per VLAN Spanning Tree Unit (STU).
-	 * The Port State database, if present, is accessed through VTU
-	 * operations and dedicated SID registers. See MV88E6352_G1_VTU_SID.
-	 */
-	MV88E6XXX_CAP_STU,
-
-	/* VLAN Table Unit.
-	 * The VTU is used to program 802.1Q VLANs. See MV88E6XXX_G1_VTU_OP.
-	 */
-	MV88E6XXX_CAP_VTU,
-};
-
-/* Bitmask of capabilities */
-#define MV88E6XXX_FLAG_EEE		BIT_ULL(MV88E6XXX_CAP_EEE)
-
-#define MV88E6XXX_FLAG_SMI_CMD		BIT_ULL(MV88E6XXX_CAP_SMI_CMD)
-#define MV88E6XXX_FLAG_SMI_DATA		BIT_ULL(MV88E6XXX_CAP_SMI_DATA)
-
-#define MV88E6XXX_FLAG_G1_VTU_FID	BIT_ULL(MV88E6XXX_CAP_G1_VTU_FID)
-
-#define MV88E6XXX_FLAG_GLOBAL2		BIT_ULL(MV88E6XXX_CAP_GLOBAL2)
-#define MV88E6XXX_FLAG_G2_INT		BIT_ULL(MV88E6XXX_CAP_G2_INT)
-#define MV88E6XXX_FLAG_G2_MGMT_EN_2X	BIT_ULL(MV88E6XXX_CAP_G2_MGMT_EN_2X)
-#define MV88E6XXX_FLAG_G2_MGMT_EN_0X	BIT_ULL(MV88E6XXX_CAP_G2_MGMT_EN_0X)
-#define MV88E6XXX_FLAG_G2_POT		BIT_ULL(MV88E6XXX_CAP_G2_POT)
-
-/* Multi-chip Addressing Mode */
-#define MV88E6XXX_FLAGS_MULTI_CHIP	\
-	(MV88E6XXX_FLAG_SMI_CMD |	\
-	 MV88E6XXX_FLAG_SMI_DATA)
-
-#define MV88E6XXX_FLAGS_FAMILY_6095	\
-	(MV88E6XXX_FLAG_GLOBAL2 |	\
-	 MV88E6XXX_FLAG_G2_MGMT_EN_0X |	\
-	 MV88E6XXX_FLAGS_MULTI_CHIP)
-
-#define MV88E6XXX_FLAGS_FAMILY_6097	\
-	(MV88E6XXX_FLAG_G1_VTU_FID |	\
-	 MV88E6XXX_FLAG_GLOBAL2 |	\
-	 MV88E6XXX_FLAG_G2_INT |        \
-	 MV88E6XXX_FLAG_G2_MGMT_EN_2X |	\
-	 MV88E6XXX_FLAG_G2_MGMT_EN_0X |	\
-	 MV88E6XXX_FLAG_G2_POT |	\
-	 MV88E6XXX_FLAGS_MULTI_CHIP)
-
-#define MV88E6XXX_FLAGS_FAMILY_6165	\
-	(MV88E6XXX_FLAG_G1_VTU_FID |	\
-	 MV88E6XXX_FLAG_GLOBAL2 |	\
-	 MV88E6XXX_FLAG_G2_INT |	\
-	 MV88E6XXX_FLAG_G2_MGMT_EN_2X |	\
-	 MV88E6XXX_FLAG_G2_MGMT_EN_0X |	\
-	 MV88E6XXX_FLAG_G2_POT |	\
-	 MV88E6XXX_FLAGS_MULTI_CHIP)
-
-#define MV88E6XXX_FLAGS_FAMILY_6185	\
-	(MV88E6XXX_FLAG_GLOBAL2 |	\
-	 MV88E6XXX_FLAG_G2_INT |	\
-	 MV88E6XXX_FLAG_G2_MGMT_EN_0X |	\
-	 MV88E6XXX_FLAGS_MULTI_CHIP)
-
-#define MV88E6XXX_FLAGS_FAMILY_6320	\
-	(MV88E6XXX_FLAG_EEE |		\
-	 MV88E6XXX_FLAG_GLOBAL2 |	\
-	 MV88E6XXX_FLAG_G2_MGMT_EN_2X |	\
-	 MV88E6XXX_FLAG_G2_MGMT_EN_0X |	\
-	 MV88E6XXX_FLAG_G2_POT |	\
-	 MV88E6XXX_FLAGS_MULTI_CHIP)
-
-#define MV88E6XXX_FLAGS_FAMILY_6341	\
-	(MV88E6XXX_FLAG_EEE |		\
-	 MV88E6XXX_FLAG_G1_VTU_FID |	\
-	 MV88E6XXX_FLAG_GLOBAL2 |	\
-	 MV88E6XXX_FLAG_G2_INT |	\
-	 MV88E6XXX_FLAG_G2_POT |	\
-	 MV88E6XXX_FLAGS_MULTI_CHIP)
-
-#define MV88E6XXX_FLAGS_FAMILY_6351	\
-	(MV88E6XXX_FLAG_G1_VTU_FID |	\
-	 MV88E6XXX_FLAG_GLOBAL2 |	\
-	 MV88E6XXX_FLAG_G2_INT |	\
-	 MV88E6XXX_FLAG_G2_MGMT_EN_2X |	\
-	 MV88E6XXX_FLAG_G2_MGMT_EN_0X |	\
-	 MV88E6XXX_FLAG_G2_POT |	\
-	 MV88E6XXX_FLAGS_MULTI_CHIP)
-
-#define MV88E6XXX_FLAGS_FAMILY_6352	\
-	(MV88E6XXX_FLAG_EEE |		\
-	 MV88E6XXX_FLAG_G1_VTU_FID |	\
-	 MV88E6XXX_FLAG_GLOBAL2 |	\
-	 MV88E6XXX_FLAG_G2_INT |	\
-	 MV88E6XXX_FLAG_G2_MGMT_EN_2X |	\
-	 MV88E6XXX_FLAG_G2_MGMT_EN_0X |	\
-	 MV88E6XXX_FLAG_G2_POT |	\
-	 MV88E6XXX_FLAGS_MULTI_CHIP)
-
-#define MV88E6XXX_FLAGS_FAMILY_6390	\
-	(MV88E6XXX_FLAG_EEE |		\
-	 MV88E6XXX_FLAG_GLOBAL2 |	\
-	 MV88E6XXX_FLAG_G2_INT |        \
-	 MV88E6XXX_FLAGS_MULTI_CHIP)
-
 struct mv88e6xxx_ops;
 
 struct mv88e6xxx_info {
@@ -235,11 +108,18 @@ struct mv88e6xxx_info {
 	unsigned int max_vid;
 	unsigned int port_base_addr;
 	unsigned int global1_addr;
+	unsigned int global2_addr;
 	unsigned int age_time_coeff;
 	unsigned int g1_irqs;
+	unsigned int g2_irqs;
 	bool pvt;
+
+	/* Multi-chip Addressing Mode.
+	 * Some chips respond to only 2 registers of its own SMI device address
+	 * when it is non-zero, and use indirect access to internal registers.
+	 */
+	bool multi_chip;
 	enum dsa_tag_protocol tag_protocol;
-	unsigned long long flags;
 
 	/* Mask for FromPort and ToPort value of PortVec used in ATU Move
 	 * operation. 0 means that the ATU Move operation is not supported.
@@ -359,6 +239,9 @@ struct mv88e6xxx_ops {
 			 struct mii_bus *bus,
 			 int addr, int reg, u16 val);
 
+	/* Priority Override Table operations */
+	int (*pot_clear)(struct mv88e6xxx_chip *chip);
+
 	/* PHY Polling Unit (PPU) operations */
 	int (*ppu_enable)(struct mv88e6xxx_chip *chip);
 	int (*ppu_disable)(struct mv88e6xxx_chip *chip);
@@ -449,7 +332,6 @@ struct mv88e6xxx_ops {
 	int (*set_egress_port)(struct mv88e6xxx_chip *chip, int port);
 	const struct mv88e6xxx_irq_ops *watchdog_ops;
 
-	/* Can be either in g1 or g2, so don't use a prefix */
 	int (*mgmt_rsvd2cpu)(struct mv88e6xxx_chip *chip);
 
 	/* Power on/off a SERDES interface */
@@ -482,12 +364,6 @@ struct mv88e6xxx_hw_stat {
 	int type;
 };
 
-static inline bool mv88e6xxx_has(struct mv88e6xxx_chip *chip,
-				 unsigned long flags)
-{
-	return (chip->info->flags & flags) == flags;
-}
-
 static inline bool mv88e6xxx_has_pvt(struct mv88e6xxx_chip *chip)
 {
 	return chip->info->pvt;
diff --git a/drivers/net/dsa/mv88e6xxx/global2.c b/drivers/net/dsa/mv88e6xxx/global2.c
index 158d0f499874..af0727877825 100644
--- a/drivers/net/dsa/mv88e6xxx/global2.c
+++ b/drivers/net/dsa/mv88e6xxx/global2.c
@@ -22,48 +22,99 @@
 
 static int mv88e6xxx_g2_read(struct mv88e6xxx_chip *chip, int reg, u16 *val)
 {
-	return mv88e6xxx_read(chip, MV88E6XXX_G2, reg, val);
+	return mv88e6xxx_read(chip, chip->info->global2_addr, reg, val);
 }
 
 static int mv88e6xxx_g2_write(struct mv88e6xxx_chip *chip, int reg, u16 val)
 {
-	return mv88e6xxx_write(chip, MV88E6XXX_G2, reg, val);
+	return mv88e6xxx_write(chip, chip->info->global2_addr, reg, val);
 }
 
 static int mv88e6xxx_g2_update(struct mv88e6xxx_chip *chip, int reg, u16 update)
 {
-	return mv88e6xxx_update(chip, MV88E6XXX_G2, reg, update);
+	return mv88e6xxx_update(chip, chip->info->global2_addr, reg, update);
 }
 
 static int mv88e6xxx_g2_wait(struct mv88e6xxx_chip *chip, int reg, u16 mask)
 {
-	return mv88e6xxx_wait(chip, MV88E6XXX_G2, reg, mask);
+	return mv88e6xxx_wait(chip, chip->info->global2_addr, reg, mask);
+}
+
+/* Offset 0x00: Interrupt Source Register */
+
+static int mv88e6xxx_g2_int_source(struct mv88e6xxx_chip *chip, u16 *src)
+{
+	/* Read (and clear most of) the Interrupt Source bits */
+	return mv88e6xxx_g2_read(chip, MV88E6XXX_G2_INT_SRC, src);
+}
+
+/* Offset 0x01: Interrupt Mask Register */
+
+static int mv88e6xxx_g2_int_mask(struct mv88e6xxx_chip *chip, u16 mask)
+{
+	return mv88e6xxx_g2_write(chip, MV88E6XXX_G2_INT_MASK, mask);
 }
 
 /* Offset 0x02: Management Enable 2x */
+
+static int mv88e6xxx_g2_mgmt_enable_2x(struct mv88e6xxx_chip *chip, u16 en2x)
+{
+	return mv88e6xxx_g2_write(chip, MV88E6XXX_G2_MGMT_EN_2X, en2x);
+}
+
 /* Offset 0x03: Management Enable 0x */
 
-int mv88e6095_g2_mgmt_rsvd2cpu(struct mv88e6xxx_chip *chip)
+static int mv88e6xxx_g2_mgmt_enable_0x(struct mv88e6xxx_chip *chip, u16 en0x)
+{
+	return mv88e6xxx_g2_write(chip, MV88E6XXX_G2_MGMT_EN_0X, en0x);
+}
+
+/* Offset 0x05: Switch Management Register */
+
+static int mv88e6xxx_g2_switch_mgmt_rsvd2cpu(struct mv88e6xxx_chip *chip,
+					     bool enable)
+{
+	u16 val;
+	int err;
+
+	err = mv88e6xxx_g2_read(chip, MV88E6XXX_G2_SWITCH_MGMT, &val);
+	if (err)
+		return err;
+
+	if (enable)
+		val |= MV88E6XXX_G2_SWITCH_MGMT_RSVD2CPU;
+	else
+		val &= ~MV88E6XXX_G2_SWITCH_MGMT_RSVD2CPU;
+
+	return mv88e6xxx_g2_write(chip, MV88E6XXX_G2_SWITCH_MGMT, val);
+}
+
+int mv88e6185_g2_mgmt_rsvd2cpu(struct mv88e6xxx_chip *chip)
 {
 	int err;
 
 	/* Consider the frames with reserved multicast destination
-	 * addresses matching 01:80:c2:00:00:2x as MGMT.
+	 * addresses matching 01:80:c2:00:00:0x as MGMT.
 	 */
-	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_G2_MGMT_EN_2X)) {
-		err = mv88e6xxx_g2_write(chip, MV88E6XXX_G2_MGMT_EN_2X, 0xffff);
-		if (err)
-			return err;
-	}
+	err = mv88e6xxx_g2_mgmt_enable_0x(chip, 0xffff);
+	if (err)
+		return err;
+
+	return mv88e6xxx_g2_switch_mgmt_rsvd2cpu(chip, true);
+}
+
+int mv88e6352_g2_mgmt_rsvd2cpu(struct mv88e6xxx_chip *chip)
+{
+	int err;
 
 	/* Consider the frames with reserved multicast destination
-	 * addresses matching 01:80:c2:00:00:0x as MGMT.
+	 * addresses matching 01:80:c2:00:00:2x as MGMT.
 	 */
-	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_G2_MGMT_EN_0X))
-		return mv88e6xxx_g2_write(chip, MV88E6XXX_G2_MGMT_EN_0X,
-					  0xffff);
+	err = mv88e6xxx_g2_mgmt_enable_2x(chip, 0xffff);
+	if (err)
+		return err;
 
-	return 0;
+	return mv88e6185_g2_mgmt_rsvd2cpu(chip);
 }
 
 /* Offset 0x06: Device Mapping Table register */
@@ -260,7 +311,7 @@ static int mv88e6xxx_g2_pot_write(struct mv88e6xxx_chip *chip, int pointer,
 	return mv88e6xxx_g2_update(chip, MV88E6XXX_G2_PRIO_OVERRIDE, val);
 }
 
-static int mv88e6xxx_g2_clear_pot(struct mv88e6xxx_chip *chip)
+int mv88e6xxx_g2_pot_clear(struct mv88e6xxx_chip *chip)
 {
 	int i, err;
 
@@ -933,7 +984,7 @@ static irqreturn_t mv88e6xxx_g2_irq_thread_fn(int irq, void *dev_id)
 	u16 reg;
 
 	mutex_lock(&chip->reg_lock);
-	err = mv88e6xxx_g2_read(chip, MV88E6XXX_G2_INT_SOURCE, &reg);
+	err = mv88e6xxx_g2_int_source(chip, &reg);
 	mutex_unlock(&chip->reg_lock);
 	if (err)
 		goto out;
@@ -959,13 +1010,16 @@ static void mv88e6xxx_g2_irq_bus_lock(struct irq_data *d)
 static void mv88e6xxx_g2_irq_bus_sync_unlock(struct irq_data *d)
 {
 	struct mv88e6xxx_chip *chip = irq_data_get_irq_chip_data(d);
+	int err;
 
-	mv88e6xxx_g2_write(chip, MV88E6XXX_G2_INT_MASK, ~chip->g2_irq.masked);
+	err = mv88e6xxx_g2_int_mask(chip, ~chip->g2_irq.masked);
+	if (err)
+		dev_err(chip->dev, "failed to mask interrupts\n");
 
 	mutex_unlock(&chip->reg_lock);
 }
 
-static struct irq_chip mv88e6xxx_g2_irq_chip = {
+static const struct irq_chip mv88e6xxx_g2_irq_chip = {
 	.name			= "mv88e6xxx-g2",
 	.irq_mask		= mv88e6xxx_g2_irq_mask,
 	.irq_unmask		= mv88e6xxx_g2_irq_unmask,
@@ -1063,9 +1117,6 @@ int mv88e6xxx_g2_setup(struct mv88e6xxx_chip *chip)
 	 * port at the highest priority.
 	 */
 	reg = MV88E6XXX_G2_SWITCH_MGMT_FORCE_FLOW_CTL_PRI | (0x7 << 4);
-	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_G2_MGMT_EN_0X) ||
-	    mv88e6xxx_has(chip, MV88E6XXX_FLAG_G2_MGMT_EN_2X))
-		reg |= MV88E6XXX_G2_SWITCH_MGMT_RSVD2CPU | 0x7;
 	err = mv88e6xxx_g2_write(chip, MV88E6XXX_G2_SWITCH_MGMT, reg);
 	if (err)
 		return err;
@@ -1080,12 +1131,5 @@ int mv88e6xxx_g2_setup(struct mv88e6xxx_chip *chip)
 	if (err)
 		return err;
 
-	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_G2_POT)) {
-		/* Clear the priority override table. */
-		err = mv88e6xxx_g2_clear_pot(chip);
-		if (err)
-			return err;
-	}
-
 	return 0;
 }
diff --git a/drivers/net/dsa/mv88e6xxx/global2.h b/drivers/net/dsa/mv88e6xxx/global2.h
index 317ffd8f323d..669f59017b12 100644
--- a/drivers/net/dsa/mv88e6xxx/global2.h
+++ b/drivers/net/dsa/mv88e6xxx/global2.h
@@ -17,14 +17,27 @@
 
 #include "chip.h"
 
-#define MV88E6XXX_G2	0x1c
-
 /* Offset 0x00: Interrupt Source Register */
-#define MV88E6XXX_G2_INT_SOURCE			0x00
+#define MV88E6XXX_G2_INT_SRC			0x00
+#define MV88E6XXX_G2_INT_SRC_WDOG		0x8000
+#define MV88E6XXX_G2_INT_SRC_JAM_LIMIT		0x4000
+#define MV88E6XXX_G2_INT_SRC_DUPLEX_MISMATCH	0x2000
+#define MV88E6XXX_G2_INT_SRC_WAKE_EVENT		0x1000
+#define MV88E6352_G2_INT_SRC_SERDES		0x0800
+#define MV88E6352_G2_INT_SRC_PHY		0x001f
+#define MV88E6390_G2_INT_SRC_PHY		0x07fe
+
 #define MV88E6XXX_G2_INT_SOURCE_WATCHDOG	15
 
 /* Offset 0x01: Interrupt Mask Register */
-#define MV88E6XXX_G2_INT_MASK	0x01
+#define MV88E6XXX_G2_INT_MASK			0x01
+#define MV88E6XXX_G2_INT_MASK_WDOG		0x8000
+#define MV88E6XXX_G2_INT_MASK_JAM_LIMIT		0x4000
+#define MV88E6XXX_G2_INT_MASK_DUPLEX_MISMATCH	0x2000
+#define MV88E6XXX_G2_INT_MASK_WAKE_EVENT	0x1000
+#define MV88E6352_G2_INT_MASK_SERDES		0x0800
+#define MV88E6352_G2_INT_MASK_PHY		0x001f
+#define MV88E6390_G2_INT_MASK_PHY		0x07fe
 
 /* Offset 0x02: MGMT Enable Register 2x */
 #define MV88E6XXX_G2_MGMT_EN_2X		0x02
@@ -245,7 +258,11 @@ int mv88e6xxx_g2_misc_4_bit_port(struct mv88e6xxx_chip *chip);
 int mv88e6xxx_g2_setup(struct mv88e6xxx_chip *chip);
 int mv88e6xxx_g2_irq_setup(struct mv88e6xxx_chip *chip);
 void mv88e6xxx_g2_irq_free(struct mv88e6xxx_chip *chip);
-int mv88e6095_g2_mgmt_rsvd2cpu(struct mv88e6xxx_chip *chip);
+
+int mv88e6185_g2_mgmt_rsvd2cpu(struct mv88e6xxx_chip *chip);
+int mv88e6352_g2_mgmt_rsvd2cpu(struct mv88e6xxx_chip *chip);
+
+int mv88e6xxx_g2_pot_clear(struct mv88e6xxx_chip *chip);
 
 extern const struct mv88e6xxx_irq_ops mv88e6097_watchdog_ops;
 extern const struct mv88e6xxx_irq_ops mv88e6390_watchdog_ops;
@@ -254,7 +271,7 @@ extern const struct mv88e6xxx_irq_ops mv88e6390_watchdog_ops;
 
 static inline int mv88e6xxx_g2_require(struct mv88e6xxx_chip *chip)
 {
-	if (mv88e6xxx_has(chip, MV88E6XXX_FLAG_GLOBAL2)) {
+	if (chip->info->global2_addr) {
 		dev_err(chip->dev, "this chip requires CONFIG_NET_DSA_MV88E6XXX_GLOBAL2 enabled\n");
 		return -EOPNOTSUPP;
 	}
@@ -347,7 +364,17 @@ static inline void mv88e6xxx_g2_irq_free(struct mv88e6xxx_chip *chip)
 {
 }
 
-static inline int mv88e6095_g2_mgmt_rsvd2cpu(struct mv88e6xxx_chip *chip)
+static inline int mv88e6185_g2_mgmt_rsvd2cpu(struct mv88e6xxx_chip *chip)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int mv88e6352_g2_mgmt_rsvd2cpu(struct mv88e6xxx_chip *chip)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int mv88e6xxx_g2_pot_clear(struct mv88e6xxx_chip *chip)
 {
 	return -EOPNOTSUPP;
 }
diff --git a/drivers/net/dsa/mv88e6xxx/phy.c b/drivers/net/dsa/mv88e6xxx/phy.c
index 3500ac0ea848..436668bd50dc 100644
--- a/drivers/net/dsa/mv88e6xxx/phy.c
+++ b/drivers/net/dsa/mv88e6xxx/phy.c
@@ -13,7 +13,6 @@
 
 #include <linux/mdio.h>
 #include <linux/module.h>
-#include <net/dsa.h>
 
 #include "chip.h"
 #include "phy.h"
diff --git a/drivers/net/dsa/mv88e6xxx/port.h b/drivers/net/dsa/mv88e6xxx/port.h
index 8f3991bf1851..b16d5f0e6e9c 100644
--- a/drivers/net/dsa/mv88e6xxx/port.h
+++ b/drivers/net/dsa/mv88e6xxx/port.h
@@ -216,9 +216,6 @@
 /* Offset 0x13: OutFiltered Counter */
 #define MV88E6XXX_PORT_OUT_FILTERED	0x13
 
-/* Offset 0x16: LED Control */
-#define MV88E6XXX_PORT_LED_CONTROL	0x16
-
 /* Offset 0x18: IEEE Priority Mapping Table */
 #define MV88E6390_PORT_IEEE_PRIO_MAP_TABLE			0x18
 #define MV88E6390_PORT_IEEE_PRIO_MAP_TABLE_UPDATE		0x8000
diff --git a/drivers/net/dsa/qca8k.c b/drivers/net/dsa/qca8k.c
index b3bee7eab45f..5ada7a41449c 100644
--- a/drivers/net/dsa/qca8k.c
+++ b/drivers/net/dsa/qca8k.c
@@ -250,7 +250,7 @@ static const struct regmap_range qca8k_readable_ranges[] = {
 
 };
 
-static struct regmap_access_table qca8k_readable_table = {
+static const struct regmap_access_table qca8k_readable_table = {
 	.yes_ranges = qca8k_readable_ranges,
 	.n_yes_ranges = ARRAY_SIZE(qca8k_readable_ranges),
 };
@@ -637,8 +637,8 @@ qca8k_get_sset_count(struct dsa_switch *ds)
 	return ARRAY_SIZE(ar8327_mib);
 }
 
-static void
-qca8k_eee_enable_set(struct dsa_switch *ds, int port, bool enable)
+static int
+qca8k_set_mac_eee(struct dsa_switch *ds, int port, struct ethtool_eee *eee)
 {
 	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
 	u32 lpi_en = QCA8K_REG_EEE_CTRL_LPI_EN(port);
@@ -646,73 +646,21 @@ qca8k_eee_enable_set(struct dsa_switch *ds, int port, bool enable)
 
 	mutex_lock(&priv->reg_mutex);
 	reg = qca8k_read(priv, QCA8K_REG_EEE_CTRL);
-	if (enable)
+	if (eee->eee_enabled)
 		reg |= lpi_en;
 	else
 		reg &= ~lpi_en;
 	qca8k_write(priv, QCA8K_REG_EEE_CTRL, reg);
 	mutex_unlock(&priv->reg_mutex);
-}
-
-static int
-qca8k_eee_init(struct dsa_switch *ds, int port,
-	       struct phy_device *phy)
-{
-	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
-	struct ethtool_eee *p = &priv->port_sts[port].eee;
-	int ret;
-
-	p->supported = (SUPPORTED_1000baseT_Full | SUPPORTED_100baseT_Full);
-
-	ret = phy_init_eee(phy, 0);
-	if (ret)
-		return ret;
-
-	qca8k_eee_enable_set(ds, port, true);
 
 	return 0;
 }
 
 static int
-qca8k_set_eee(struct dsa_switch *ds, int port,
-	      struct phy_device *phydev,
-	      struct ethtool_eee *e)
+qca8k_get_mac_eee(struct dsa_switch *ds, int port, struct ethtool_eee *e)
 {
-	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
-	struct ethtool_eee *p = &priv->port_sts[port].eee;
-	int ret = 0;
-
-	p->eee_enabled = e->eee_enabled;
-
-	if (e->eee_enabled) {
-		p->eee_enabled = qca8k_eee_init(ds, port, phydev);
-		if (!p->eee_enabled)
-			ret = -EOPNOTSUPP;
-	}
-	qca8k_eee_enable_set(ds, port, p->eee_enabled);
-
-	return ret;
-}
-
-static int
-qca8k_get_eee(struct dsa_switch *ds, int port,
-	      struct ethtool_eee *e)
-{
-	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
-	struct ethtool_eee *p = &priv->port_sts[port].eee;
-	struct net_device *netdev = ds->ports[port].netdev;
-	int ret;
-
-	ret = phy_ethtool_get_eee(netdev->phydev, p);
-	if (!ret)
-		e->eee_active =
-			!!(p->supported & p->advertised & p->lp_advertised);
-	else
-		e->eee_active = 0;
-
-	e->eee_enabled = p->eee_enabled;
-
-	return ret;
+	/* Nothing to do on the port's MAC */
+	return 0;
 }
 
 static void
@@ -829,69 +777,44 @@ qca8k_port_fdb_insert(struct qca8k_priv *priv, const u8 *addr,
 }
 
 static int
-qca8k_port_fdb_prepare(struct dsa_switch *ds, int port,
-		       const struct switchdev_obj_port_fdb *fdb,
-		       struct switchdev_trans *trans)
-{
-	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
-
-	/* The FDB table for static and auto learned entries is the same. We
-	 * need to reserve an entry with no port_mask set to make sure that
-	 * when port_fdb_add is called an entry is still available. Otherwise
-	 * the last free entry might have been used up by auto learning
-	 */
-	return qca8k_port_fdb_insert(priv, fdb->addr, 0, fdb->vid);
-}
-
-static void
 qca8k_port_fdb_add(struct dsa_switch *ds, int port,
-		   const struct switchdev_obj_port_fdb *fdb,
-		   struct switchdev_trans *trans)
+		   const unsigned char *addr, u16 vid)
 {
 	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
 	u16 port_mask = BIT(port);
 
-	/* Update the FDB entry adding the port_mask */
-	qca8k_port_fdb_insert(priv, fdb->addr, port_mask, fdb->vid);
+	return qca8k_port_fdb_insert(priv, addr, port_mask, vid);
 }
 
 static int
 qca8k_port_fdb_del(struct dsa_switch *ds, int port,
-		   const struct switchdev_obj_port_fdb *fdb)
+		   const unsigned char *addr, u16 vid)
 {
 	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
 	u16 port_mask = BIT(port);
-	u16 vid = fdb->vid;
 
 	if (!vid)
 		vid = 1;
 
-	return qca8k_fdb_del(priv, fdb->addr, port_mask, vid);
+	return qca8k_fdb_del(priv, addr, port_mask, vid);
 }
 
 static int
 qca8k_port_fdb_dump(struct dsa_switch *ds, int port,
-		    struct switchdev_obj_port_fdb *fdb,
-		    switchdev_obj_dump_cb_t *cb)
+		    dsa_fdb_dump_cb_t *cb, void *data)
 {
 	struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv;
 	struct qca8k_fdb _fdb = { 0 };
 	int cnt = QCA8K_NUM_FDB_RECORDS;
+	bool is_static;
 	int ret = 0;
 
 	mutex_lock(&priv->reg_mutex);
 	while (cnt-- && !qca8k_fdb_next(priv, &_fdb, port)) {
 		if (!_fdb.aging)
 			break;
-
-		ether_addr_copy(fdb->addr, _fdb.mac);
-		fdb->vid = _fdb.vid;
-		if (_fdb.aging == QCA8K_ATU_STATUS_STATIC)
-			fdb->ndm_state = NUD_NOARP;
-		else
-			fdb->ndm_state = NUD_REACHABLE;
-
-		ret = cb(&fdb->obj);
+		is_static = (_fdb.aging == QCA8K_ATU_STATUS_STATIC);
+		ret = cb(_fdb.mac, _fdb.vid, is_static, data);
 		if (ret)
 			break;
 	}
@@ -914,14 +837,13 @@ static const struct dsa_switch_ops qca8k_switch_ops = {
 	.phy_write		= qca8k_phy_write,
 	.get_ethtool_stats	= qca8k_get_ethtool_stats,
 	.get_sset_count		= qca8k_get_sset_count,
-	.get_eee		= qca8k_get_eee,
-	.set_eee		= qca8k_set_eee,
+	.get_mac_eee		= qca8k_get_mac_eee,
+	.set_mac_eee		= qca8k_set_mac_eee,
 	.port_enable		= qca8k_port_enable,
 	.port_disable		= qca8k_port_disable,
 	.port_stp_state_set	= qca8k_port_stp_state_set,
 	.port_bridge_join	= qca8k_port_bridge_join,
 	.port_bridge_leave	= qca8k_port_bridge_leave,
-	.port_fdb_prepare	= qca8k_port_fdb_prepare,
 	.port_fdb_add		= qca8k_port_fdb_add,
 	.port_fdb_del		= qca8k_port_fdb_del,
 	.port_fdb_dump		= qca8k_port_fdb_dump,
diff --git a/drivers/net/dsa/qca8k.h b/drivers/net/dsa/qca8k.h
index 1ed4fac6cd6d..1cf8a920d4ff 100644
--- a/drivers/net/dsa/qca8k.h
+++ b/drivers/net/dsa/qca8k.h
@@ -156,7 +156,6 @@ enum qca8k_fdb_cmd {
 };
 
 struct ar8xxx_port_status {
-	struct ethtool_eee eee;
 	int enabled;
 };
 
diff --git a/drivers/net/dummy.c b/drivers/net/dummy.c
index d0c165d2086e..d0a1f9ce3168 100644
--- a/drivers/net/dummy.c
+++ b/drivers/net/dummy.c
@@ -345,7 +345,7 @@ static void dummy_setup(struct net_device *dev)
 	dev->flags &= ~IFF_MULTICAST;
 	dev->priv_flags |= IFF_LIVE_ADDR_CHANGE | IFF_NO_QUEUE;
 	dev->features	|= NETIF_F_SG | NETIF_F_FRAGLIST;
-	dev->features	|= NETIF_F_ALL_TSO | NETIF_F_UFO;
+	dev->features	|= NETIF_F_ALL_TSO;
 	dev->features	|= NETIF_F_HW_CSUM | NETIF_F_HIGHDMA | NETIF_F_LLTX;
 	dev->features	|= NETIF_F_GSO_ENCAP_ALL;
 	dev->hw_features |= dev->features;
diff --git a/drivers/net/ethernet/3com/3c509.c b/drivers/net/ethernet/3com/3c509.c
index f66c9710cb81..b223769d6a5e 100644
--- a/drivers/net/ethernet/3com/3c509.c
+++ b/drivers/net/ethernet/3com/3c509.c
@@ -392,7 +392,7 @@ static struct isa_driver el3_isa_driver = {
 static int isa_registered;
 
 #ifdef CONFIG_PNP
-static struct pnp_device_id el3_pnp_ids[] = {
+static const struct pnp_device_id el3_pnp_ids[] = {
 	{ .id = "TCM5090" }, /* 3Com Etherlink III (TP) */
 	{ .id = "TCM5091" }, /* 3Com Etherlink III */
 	{ .id = "TCM5094" }, /* 3Com Etherlink III (combo) */
@@ -474,7 +474,7 @@ static int pnp_registered;
 #endif /* CONFIG_PNP */
 
 #ifdef CONFIG_EISA
-static struct eisa_device_id el3_eisa_ids[] = {
+static const struct eisa_device_id el3_eisa_ids[] = {
 		{ "TCM5090" },
 		{ "TCM5091" },
 		{ "TCM5092" },
diff --git a/drivers/net/ethernet/3com/3c59x.c b/drivers/net/ethernet/3com/3c59x.c
index 3b516ebeeddb..402d9090ad29 100644
--- a/drivers/net/ethernet/3com/3c59x.c
+++ b/drivers/net/ethernet/3com/3c59x.c
@@ -900,7 +900,7 @@ static const struct dev_pm_ops vortex_pm_ops = {
 #endif /* !CONFIG_PM */
 
 #ifdef CONFIG_EISA
-static struct eisa_device_id vortex_eisa_ids[] = {
+static const struct eisa_device_id vortex_eisa_ids[] = {
 	{ "TCM5920", CH_3C592 },
 	{ "TCM5970", CH_3C597 },
 	{ "" }
diff --git a/drivers/net/ethernet/8390/ax88796.c b/drivers/net/ethernet/8390/ax88796.c
index 05d9d3e2e92e..245554707163 100644
--- a/drivers/net/ethernet/8390/ax88796.c
+++ b/drivers/net/ethernet/8390/ax88796.c
@@ -585,7 +585,7 @@ static int ax_bb_get_data(struct mdiobb_ctrl *ctrl)
 	return reg_memr & AX_MEMR_MDI ? 1 : 0;
 }
 
-static struct mdiobb_ops bb_ops = {
+static const struct mdiobb_ops bb_ops = {
 	.owner = THIS_MODULE,
 	.set_mdc = ax_bb_mdc,
 	.set_mdio_dir = ax_bb_dir,
diff --git a/drivers/net/ethernet/Kconfig b/drivers/net/ethernet/Kconfig
index edae15ac0e98..c60421339a98 100644
--- a/drivers/net/ethernet/Kconfig
+++ b/drivers/net/ethernet/Kconfig
@@ -78,6 +78,7 @@ source "drivers/net/ethernet/freescale/Kconfig"
 source "drivers/net/ethernet/fujitsu/Kconfig"
 source "drivers/net/ethernet/hisilicon/Kconfig"
 source "drivers/net/ethernet/hp/Kconfig"
+source "drivers/net/ethernet/huawei/Kconfig"
 source "drivers/net/ethernet/ibm/Kconfig"
 source "drivers/net/ethernet/intel/Kconfig"
 source "drivers/net/ethernet/i825xx/Kconfig"
diff --git a/drivers/net/ethernet/Makefile b/drivers/net/ethernet/Makefile
index bf7f4502cabc..a0a03d4d939a 100644
--- a/drivers/net/ethernet/Makefile
+++ b/drivers/net/ethernet/Makefile
@@ -41,6 +41,7 @@ obj-$(CONFIG_NET_VENDOR_FREESCALE) += freescale/
 obj-$(CONFIG_NET_VENDOR_FUJITSU) += fujitsu/
 obj-$(CONFIG_NET_VENDOR_HISILICON) += hisilicon/
 obj-$(CONFIG_NET_VENDOR_HP) += hp/
+obj-$(CONFIG_NET_VENDOR_HUAWEI) += huawei/
 obj-$(CONFIG_NET_VENDOR_IBM) += ibm/
 obj-$(CONFIG_NET_VENDOR_INTEL) += intel/
 obj-$(CONFIG_NET_VENDOR_I825XX) += i825xx/
diff --git a/drivers/net/ethernet/adi/bfin_mac.c b/drivers/net/ethernet/adi/bfin_mac.c
index a9ac58c351a0..a251de8d9a91 100644
--- a/drivers/net/ethernet/adi/bfin_mac.c
+++ b/drivers/net/ethernet/adi/bfin_mac.c
@@ -986,7 +986,7 @@ static int bfin_ptp_enable(struct ptp_clock_info *ptp,
 	return -EOPNOTSUPP;
 }
 
-static struct ptp_clock_info bfin_ptp_caps = {
+static const struct ptp_clock_info bfin_ptp_caps = {
 	.owner		= THIS_MODULE,
 	.name		= "BF518 clock",
 	.max_adj	= 0,
diff --git a/drivers/net/ethernet/amd/a2065.c b/drivers/net/ethernet/amd/a2065.c
index ee4b94e3cda9..e22f976a0d18 100644
--- a/drivers/net/ethernet/amd/a2065.c
+++ b/drivers/net/ethernet/amd/a2065.c
@@ -643,7 +643,7 @@ static int a2065_init_one(struct zorro_dev *z,
 static void a2065_remove_one(struct zorro_dev *z);
 
 
-static struct zorro_device_id a2065_zorro_tbl[] = {
+static const struct zorro_device_id a2065_zorro_tbl[] = {
 	{ ZORRO_PROD_CBM_A2065_1 },
 	{ ZORRO_PROD_CBM_A2065_2 },
 	{ ZORRO_PROD_AMERISTAR_A2065 },
diff --git a/drivers/net/ethernet/amd/ariadne.c b/drivers/net/ethernet/amd/ariadne.c
index 5fd7b15b0574..4b6a5cb85dd2 100644
--- a/drivers/net/ethernet/amd/ariadne.c
+++ b/drivers/net/ethernet/amd/ariadne.c
@@ -692,7 +692,7 @@ static void ariadne_remove_one(struct zorro_dev *z)
 	free_netdev(dev);
 }
 
-static struct zorro_device_id ariadne_zorro_tbl[] = {
+static const struct zorro_device_id ariadne_zorro_tbl[] = {
 	{ ZORRO_PROD_VILLAGE_TRONIC_ARIADNE },
 	{ 0 }
 };
diff --git a/drivers/net/ethernet/amd/au1000_eth.c b/drivers/net/ethernet/amd/au1000_eth.c
index a3c90fe5de00..73ca8879ada7 100644
--- a/drivers/net/ethernet/amd/au1000_eth.c
+++ b/drivers/net/ethernet/amd/au1000_eth.c
@@ -1180,9 +1180,10 @@ static int au1000_probe(struct platform_device *pdev)
 	/* Allocate the data buffers
 	 * Snooping works fine with eth on all au1xxx
 	 */
-	aup->vaddr = (u32)dma_alloc_noncoherent(NULL, MAX_BUF_SIZE *
-						(NUM_TX_BUFFS + NUM_RX_BUFFS),
-						&aup->dma_addr,	0);
+	aup->vaddr = (u32)dma_alloc_attrs(NULL, MAX_BUF_SIZE *
+					  (NUM_TX_BUFFS + NUM_RX_BUFFS),
+					  &aup->dma_addr, 0,
+					  DMA_ATTR_NON_CONSISTENT);
 	if (!aup->vaddr) {
 		dev_err(&pdev->dev, "failed to allocate data buffers\n");
 		err = -ENOMEM;
@@ -1361,8 +1362,9 @@ err_remap3:
 err_remap2:
 	iounmap(aup->mac);
 err_remap1:
-	dma_free_noncoherent(NULL, MAX_BUF_SIZE * (NUM_TX_BUFFS + NUM_RX_BUFFS),
-			     (void *)aup->vaddr, aup->dma_addr);
+	dma_free_attrs(NULL, MAX_BUF_SIZE * (NUM_TX_BUFFS + NUM_RX_BUFFS),
+			(void *)aup->vaddr, aup->dma_addr,
+			DMA_ATTR_NON_CONSISTENT);
 err_vaddr:
 	free_netdev(dev);
 err_alloc:
@@ -1394,9 +1396,9 @@ static int au1000_remove(struct platform_device *pdev)
 		if (aup->tx_db_inuse[i])
 			au1000_ReleaseDB(aup, aup->tx_db_inuse[i]);
 
-	dma_free_noncoherent(NULL, MAX_BUF_SIZE *
-			(NUM_TX_BUFFS + NUM_RX_BUFFS),
-			(void *)aup->vaddr, aup->dma_addr);
+	dma_free_attrs(NULL, MAX_BUF_SIZE * (NUM_TX_BUFFS + NUM_RX_BUFFS),
+			(void *)aup->vaddr, aup->dma_addr,
+			DMA_ATTR_NON_CONSISTENT);
 
 	iounmap(aup->macdma);
 	iounmap(aup->mac);
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-common.h b/drivers/net/ethernet/amd/xgbe/xgbe-common.h
index 9795419aac2d..7ea72ef11a55 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-common.h
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-common.h
@@ -210,11 +210,15 @@
 #define DMA_CH_CR_PBLX8_WIDTH		1
 #define DMA_CH_CR_SPH_INDEX		24
 #define DMA_CH_CR_SPH_WIDTH		1
-#define DMA_CH_IER_AIE_INDEX		15
+#define DMA_CH_IER_AIE20_INDEX		15
+#define DMA_CH_IER_AIE20_WIDTH		1
+#define DMA_CH_IER_AIE_INDEX		14
 #define DMA_CH_IER_AIE_WIDTH		1
 #define DMA_CH_IER_FBEE_INDEX		12
 #define DMA_CH_IER_FBEE_WIDTH		1
-#define DMA_CH_IER_NIE_INDEX		16
+#define DMA_CH_IER_NIE20_INDEX		16
+#define DMA_CH_IER_NIE20_WIDTH		1
+#define DMA_CH_IER_NIE_INDEX		15
 #define DMA_CH_IER_NIE_WIDTH		1
 #define DMA_CH_IER_RBUE_INDEX		7
 #define DMA_CH_IER_RBUE_WIDTH		1
@@ -298,6 +302,7 @@
 #define MAC_RWKPFR			0x00c4
 #define MAC_LPICSR			0x00d0
 #define MAC_LPITCR			0x00d4
+#define MAC_TIR				0x00e0
 #define MAC_VR				0x0110
 #define MAC_DR				0x0114
 #define MAC_HWF0R			0x011c
@@ -364,6 +369,8 @@
 #define MAC_HWF0R_TXCOESEL_WIDTH	1
 #define MAC_HWF0R_VLHASH_INDEX		4
 #define MAC_HWF0R_VLHASH_WIDTH		1
+#define MAC_HWF0R_VXN_INDEX		29
+#define MAC_HWF0R_VXN_WIDTH		1
 #define MAC_HWF1R_ADDR64_INDEX		14
 #define MAC_HWF1R_ADDR64_WIDTH		2
 #define MAC_HWF1R_ADVTHWORD_INDEX	13
@@ -448,6 +455,8 @@
 #define MAC_PFR_PR_WIDTH		1
 #define MAC_PFR_VTFE_INDEX		16
 #define MAC_PFR_VTFE_WIDTH		1
+#define MAC_PFR_VUCC_INDEX		22
+#define MAC_PFR_VUCC_WIDTH		1
 #define MAC_PMTCSR_MGKPKTEN_INDEX	1
 #define MAC_PMTCSR_MGKPKTEN_WIDTH	1
 #define MAC_PMTCSR_PWRDWN_INDEX		0
@@ -510,6 +519,12 @@
 #define MAC_TCR_SS_WIDTH		2
 #define MAC_TCR_TE_INDEX		0
 #define MAC_TCR_TE_WIDTH		1
+#define MAC_TCR_VNE_INDEX		24
+#define MAC_TCR_VNE_WIDTH		1
+#define MAC_TCR_VNM_INDEX		25
+#define MAC_TCR_VNM_WIDTH		1
+#define MAC_TIR_TNID_INDEX		0
+#define MAC_TIR_TNID_WIDTH		16
 #define MAC_TSCR_AV8021ASMEN_INDEX	28
 #define MAC_TSCR_AV8021ASMEN_WIDTH	1
 #define MAC_TSCR_SNAPTYPSEL_INDEX	16
@@ -1153,11 +1168,17 @@
 #define RX_PACKET_ATTRIBUTES_RSS_HASH_WIDTH	1
 #define RX_PACKET_ATTRIBUTES_FIRST_INDEX	7
 #define RX_PACKET_ATTRIBUTES_FIRST_WIDTH	1
+#define RX_PACKET_ATTRIBUTES_TNP_INDEX		8
+#define RX_PACKET_ATTRIBUTES_TNP_WIDTH		1
+#define RX_PACKET_ATTRIBUTES_TNPCSUM_DONE_INDEX	9
+#define RX_PACKET_ATTRIBUTES_TNPCSUM_DONE_WIDTH	1
 
 #define RX_NORMAL_DESC0_OVT_INDEX		0
 #define RX_NORMAL_DESC0_OVT_WIDTH		16
 #define RX_NORMAL_DESC2_HL_INDEX		0
 #define RX_NORMAL_DESC2_HL_WIDTH		10
+#define RX_NORMAL_DESC2_TNP_INDEX		11
+#define RX_NORMAL_DESC2_TNP_WIDTH		1
 #define RX_NORMAL_DESC3_CDA_INDEX		27
 #define RX_NORMAL_DESC3_CDA_WIDTH		1
 #define RX_NORMAL_DESC3_CTXT_INDEX		30
@@ -1184,9 +1205,11 @@
 #define RX_DESC3_L34T_IPV4_TCP			1
 #define RX_DESC3_L34T_IPV4_UDP			2
 #define RX_DESC3_L34T_IPV4_ICMP			3
+#define RX_DESC3_L34T_IPV4_UNKNOWN		7
 #define RX_DESC3_L34T_IPV6_TCP			9
 #define RX_DESC3_L34T_IPV6_UDP			10
 #define RX_DESC3_L34T_IPV6_ICMP			11
+#define RX_DESC3_L34T_IPV6_UNKNOWN		15
 
 #define RX_CONTEXT_DESC3_TSA_INDEX		4
 #define RX_CONTEXT_DESC3_TSA_WIDTH		1
@@ -1201,6 +1224,8 @@
 #define TX_PACKET_ATTRIBUTES_VLAN_CTAG_WIDTH	1
 #define TX_PACKET_ATTRIBUTES_PTP_INDEX		3
 #define TX_PACKET_ATTRIBUTES_PTP_WIDTH		1
+#define TX_PACKET_ATTRIBUTES_VXLAN_INDEX	4
+#define TX_PACKET_ATTRIBUTES_VXLAN_WIDTH	1
 
 #define TX_CONTEXT_DESC2_MSS_INDEX		0
 #define TX_CONTEXT_DESC2_MSS_WIDTH		15
@@ -1241,8 +1266,11 @@
 #define TX_NORMAL_DESC3_TCPPL_WIDTH		18
 #define TX_NORMAL_DESC3_TSE_INDEX		18
 #define TX_NORMAL_DESC3_TSE_WIDTH		1
+#define TX_NORMAL_DESC3_VNP_INDEX		23
+#define TX_NORMAL_DESC3_VNP_WIDTH		3
 
 #define TX_NORMAL_DESC2_VLAN_INSERT		0x2
+#define TX_NORMAL_DESC3_VXLAN_PACKET		0x3
 
 /* MDIO undefined or vendor specific registers */
 #ifndef MDIO_PMA_10GBR_PMD_CTRL
@@ -1339,6 +1367,7 @@
 #define XGBE_AN_CL37_PCS_MODE_BASEX	0x00
 #define XGBE_AN_CL37_PCS_MODE_SGMII	0x04
 #define XGBE_AN_CL37_TX_CONFIG_MASK	0x08
+#define XGBE_AN_CL37_MII_CTRL_8BIT	0x0100
 
 /* Bit setting and getting macros
  *  The get macro will extract the current bit field value from within
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-debugfs.c b/drivers/net/ethernet/amd/xgbe/xgbe-debugfs.c
index 7546b660d6b5..7d128be61310 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-debugfs.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-debugfs.c
@@ -527,3 +527,28 @@ void xgbe_debugfs_exit(struct xgbe_prv_data *pdata)
 	debugfs_remove_recursive(pdata->xgbe_debugfs);
 	pdata->xgbe_debugfs = NULL;
 }
+
+void xgbe_debugfs_rename(struct xgbe_prv_data *pdata)
+{
+	struct dentry *pfile;
+	char *buf;
+
+	if (!pdata->xgbe_debugfs)
+		return;
+
+	buf = kasprintf(GFP_KERNEL, "amd-xgbe-%s", pdata->netdev->name);
+	if (!buf)
+		return;
+
+	if (!strcmp(pdata->xgbe_debugfs->d_name.name, buf))
+		goto out;
+
+	pfile = debugfs_rename(pdata->xgbe_debugfs->d_parent,
+			       pdata->xgbe_debugfs,
+			       pdata->xgbe_debugfs->d_parent, buf);
+	if (!pfile)
+		netdev_err(pdata->netdev, "debugfs_rename failed\n");
+
+out:
+	kfree(buf);
+}
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-dev.c b/drivers/net/ethernet/amd/xgbe/xgbe-dev.c
index 06f953e1e9b2..e107e180e2c8 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-dev.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-dev.c
@@ -479,6 +479,50 @@ static bool xgbe_is_pfc_queue(struct xgbe_prv_data *pdata,
 	return false;
 }
 
+static void xgbe_set_vxlan_id(struct xgbe_prv_data *pdata)
+{
+	/* Program the VXLAN port */
+	XGMAC_IOWRITE_BITS(pdata, MAC_TIR, TNID, pdata->vxlan_port);
+
+	netif_dbg(pdata, drv, pdata->netdev, "VXLAN tunnel id set to %hx\n",
+		  pdata->vxlan_port);
+}
+
+static void xgbe_enable_vxlan(struct xgbe_prv_data *pdata)
+{
+	if (!pdata->hw_feat.vxn)
+		return;
+
+	/* Program the VXLAN port */
+	xgbe_set_vxlan_id(pdata);
+
+	/* Allow for IPv6/UDP zero-checksum VXLAN packets */
+	XGMAC_IOWRITE_BITS(pdata, MAC_PFR, VUCC, 1);
+
+	/* Enable VXLAN tunneling mode */
+	XGMAC_IOWRITE_BITS(pdata, MAC_TCR, VNM, 0);
+	XGMAC_IOWRITE_BITS(pdata, MAC_TCR, VNE, 1);
+
+	netif_dbg(pdata, drv, pdata->netdev, "VXLAN acceleration enabled\n");
+}
+
+static void xgbe_disable_vxlan(struct xgbe_prv_data *pdata)
+{
+	if (!pdata->hw_feat.vxn)
+		return;
+
+	/* Disable tunneling mode */
+	XGMAC_IOWRITE_BITS(pdata, MAC_TCR, VNE, 0);
+
+	/* Clear IPv6/UDP zero-checksum VXLAN packets setting */
+	XGMAC_IOWRITE_BITS(pdata, MAC_PFR, VUCC, 0);
+
+	/* Clear the VXLAN port */
+	XGMAC_IOWRITE_BITS(pdata, MAC_TIR, TNID, 0);
+
+	netif_dbg(pdata, drv, pdata->netdev, "VXLAN acceleration disabled\n");
+}
+
 static int xgbe_disable_tx_flow_control(struct xgbe_prv_data *pdata)
 {
 	unsigned int max_q_count, q_count;
@@ -605,32 +649,38 @@ static void xgbe_config_flow_control(struct xgbe_prv_data *pdata)
 static void xgbe_enable_dma_interrupts(struct xgbe_prv_data *pdata)
 {
 	struct xgbe_channel *channel;
-	unsigned int dma_ch_isr, dma_ch_ier;
-	unsigned int i;
+	unsigned int i, ver;
 
 	/* Set the interrupt mode if supported */
 	if (pdata->channel_irq_mode)
 		XGMAC_IOWRITE_BITS(pdata, DMA_MR, INTM,
 				   pdata->channel_irq_mode);
 
+	ver = XGMAC_GET_BITS(pdata->hw_feat.version, MAC_VR, SNPSVER);
+
 	for (i = 0; i < pdata->channel_count; i++) {
 		channel = pdata->channel[i];
 
 		/* Clear all the interrupts which are set */
-		dma_ch_isr = XGMAC_DMA_IOREAD(channel, DMA_CH_SR);
-		XGMAC_DMA_IOWRITE(channel, DMA_CH_SR, dma_ch_isr);
+		XGMAC_DMA_IOWRITE(channel, DMA_CH_SR,
+				  XGMAC_DMA_IOREAD(channel, DMA_CH_SR));
 
 		/* Clear all interrupt enable bits */
-		dma_ch_ier = 0;
+		channel->curr_ier = 0;
 
 		/* Enable following interrupts
 		 *   NIE  - Normal Interrupt Summary Enable
 		 *   AIE  - Abnormal Interrupt Summary Enable
 		 *   FBEE - Fatal Bus Error Enable
 		 */
-		XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, NIE, 1);
-		XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, AIE, 1);
-		XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, FBEE, 1);
+		if (ver < 0x21) {
+			XGMAC_SET_BITS(channel->curr_ier, DMA_CH_IER, NIE20, 1);
+			XGMAC_SET_BITS(channel->curr_ier, DMA_CH_IER, AIE20, 1);
+		} else {
+			XGMAC_SET_BITS(channel->curr_ier, DMA_CH_IER, NIE, 1);
+			XGMAC_SET_BITS(channel->curr_ier, DMA_CH_IER, AIE, 1);
+		}
+		XGMAC_SET_BITS(channel->curr_ier, DMA_CH_IER, FBEE, 1);
 
 		if (channel->tx_ring) {
 			/* Enable the following Tx interrupts
@@ -639,7 +689,8 @@ static void xgbe_enable_dma_interrupts(struct xgbe_prv_data *pdata)
 			 *          mode)
 			 */
 			if (!pdata->per_channel_irq || pdata->channel_irq_mode)
-				XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, TIE, 1);
+				XGMAC_SET_BITS(channel->curr_ier,
+					       DMA_CH_IER, TIE, 1);
 		}
 		if (channel->rx_ring) {
 			/* Enable following Rx interrupts
@@ -648,12 +699,13 @@ static void xgbe_enable_dma_interrupts(struct xgbe_prv_data *pdata)
 			 *          per channel interrupts in edge triggered
 			 *          mode)
 			 */
-			XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, RBUE, 1);
+			XGMAC_SET_BITS(channel->curr_ier, DMA_CH_IER, RBUE, 1);
 			if (!pdata->per_channel_irq || pdata->channel_irq_mode)
-				XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, RIE, 1);
+				XGMAC_SET_BITS(channel->curr_ier,
+					       DMA_CH_IER, RIE, 1);
 		}
 
-		XGMAC_DMA_IOWRITE(channel, DMA_CH_IER, dma_ch_ier);
+		XGMAC_DMA_IOWRITE(channel, DMA_CH_IER, channel->curr_ier);
 	}
 }
 
@@ -1608,7 +1660,8 @@ static void xgbe_dev_xmit(struct xgbe_channel *channel)
 	struct xgbe_ring_data *rdata;
 	struct xgbe_ring_desc *rdesc;
 	struct xgbe_packet_data *packet = &ring->packet_data;
-	unsigned int csum, tso, vlan;
+	unsigned int tx_packets, tx_bytes;
+	unsigned int csum, tso, vlan, vxlan;
 	unsigned int tso_context, vlan_context;
 	unsigned int tx_set_ic;
 	int start_index = ring->cur;
@@ -1617,12 +1670,17 @@ static void xgbe_dev_xmit(struct xgbe_channel *channel)
 
 	DBGPR("-->xgbe_dev_xmit\n");
 
+	tx_packets = packet->tx_packets;
+	tx_bytes = packet->tx_bytes;
+
 	csum = XGMAC_GET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES,
 			      CSUM_ENABLE);
 	tso = XGMAC_GET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES,
 			     TSO_ENABLE);
 	vlan = XGMAC_GET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES,
 			      VLAN_CTAG);
+	vxlan = XGMAC_GET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES,
+			       VXLAN);
 
 	if (tso && (packet->mss != ring->tx.cur_mss))
 		tso_context = 1;
@@ -1644,13 +1702,12 @@ static void xgbe_dev_xmit(struct xgbe_channel *channel)
 	 *     - Addition of Tx frame count to the frame count since the
 	 *       last interrupt was set does not exceed the frame count setting
 	 */
-	ring->coalesce_count += packet->tx_packets;
+	ring->coalesce_count += tx_packets;
 	if (!pdata->tx_frames)
 		tx_set_ic = 0;
-	else if (packet->tx_packets > pdata->tx_frames)
+	else if (tx_packets > pdata->tx_frames)
 		tx_set_ic = 1;
-	else if ((ring->coalesce_count % pdata->tx_frames) <
-		 packet->tx_packets)
+	else if ((ring->coalesce_count % pdata->tx_frames) < tx_packets)
 		tx_set_ic = 1;
 	else
 		tx_set_ic = 0;
@@ -1740,7 +1797,7 @@ static void xgbe_dev_xmit(struct xgbe_channel *channel)
 		XGMAC_SET_BITS_LE(rdesc->desc3, TX_NORMAL_DESC3, TCPHDRLEN,
 				  packet->tcp_header_len / 4);
 
-		pdata->ext_stats.tx_tso_packets++;
+		pdata->ext_stats.tx_tso_packets += tx_packets;
 	} else {
 		/* Enable CRC and Pad Insertion */
 		XGMAC_SET_BITS_LE(rdesc->desc3, TX_NORMAL_DESC3, CPC, 0);
@@ -1755,6 +1812,13 @@ static void xgbe_dev_xmit(struct xgbe_channel *channel)
 				  packet->length);
 	}
 
+	if (vxlan) {
+		XGMAC_SET_BITS_LE(rdesc->desc3, TX_NORMAL_DESC3, VNP,
+				  TX_NORMAL_DESC3_VXLAN_PACKET);
+
+		pdata->ext_stats.tx_vxlan_packets += packet->tx_packets;
+	}
+
 	for (i = cur_index - start_index + 1; i < packet->rdesc_count; i++) {
 		cur_index++;
 		rdata = XGBE_GET_DESC_DATA(ring, cur_index);
@@ -1788,8 +1852,11 @@ static void xgbe_dev_xmit(struct xgbe_channel *channel)
 		XGMAC_SET_BITS_LE(rdesc->desc2, TX_NORMAL_DESC2, IC, 1);
 
 	/* Save the Tx info to report back during cleanup */
-	rdata->tx.packets = packet->tx_packets;
-	rdata->tx.bytes = packet->tx_bytes;
+	rdata->tx.packets = tx_packets;
+	rdata->tx.bytes = tx_bytes;
+
+	pdata->ext_stats.txq_packets[channel->queue_index] += tx_packets;
+	pdata->ext_stats.txq_bytes[channel->queue_index] += tx_bytes;
 
 	/* In case the Tx DMA engine is running, make sure everything
 	 * is written to the descriptor(s) before setting the OWN bit
@@ -1913,9 +1980,28 @@ static int xgbe_dev_read(struct xgbe_channel *channel)
 	rdata->rx.len = XGMAC_GET_BITS_LE(rdesc->desc3, RX_NORMAL_DESC3, PL);
 
 	/* Set checksum done indicator as appropriate */
-	if (netdev->features & NETIF_F_RXCSUM)
+	if (netdev->features & NETIF_F_RXCSUM) {
 		XGMAC_SET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES,
 			       CSUM_DONE, 1);
+		XGMAC_SET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES,
+			       TNPCSUM_DONE, 1);
+	}
+
+	/* Set the tunneled packet indicator */
+	if (XGMAC_GET_BITS_LE(rdesc->desc2, RX_NORMAL_DESC2, TNP)) {
+		XGMAC_SET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES,
+			       TNP, 1);
+		pdata->ext_stats.rx_vxlan_packets++;
+
+		l34t = XGMAC_GET_BITS_LE(rdesc->desc3, RX_NORMAL_DESC3, L34T);
+		switch (l34t) {
+		case RX_DESC3_L34T_IPV4_UNKNOWN:
+		case RX_DESC3_L34T_IPV6_UNKNOWN:
+			XGMAC_SET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES,
+				       TNPCSUM_DONE, 0);
+			break;
+		}
+	}
 
 	/* Check for errors (only valid in last descriptor) */
 	err = XGMAC_GET_BITS_LE(rdesc->desc3, RX_NORMAL_DESC3, ES);
@@ -1935,14 +2021,30 @@ static int xgbe_dev_read(struct xgbe_channel *channel)
 				  packet->vlan_ctag);
 		}
 	} else {
-		if ((etlt == 0x05) || (etlt == 0x06))
+		unsigned int tnp = XGMAC_GET_BITS(packet->attributes,
+						  RX_PACKET_ATTRIBUTES, TNP);
+
+		if ((etlt == 0x05) || (etlt == 0x06)) {
 			XGMAC_SET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES,
 				       CSUM_DONE, 0);
-		else
+			XGMAC_SET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES,
+				       TNPCSUM_DONE, 0);
+			pdata->ext_stats.rx_csum_errors++;
+		} else if (tnp && ((etlt == 0x09) || (etlt == 0x0a))) {
+			XGMAC_SET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES,
+				       CSUM_DONE, 0);
+			XGMAC_SET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES,
+				       TNPCSUM_DONE, 0);
+			pdata->ext_stats.rx_vxlan_csum_errors++;
+		} else {
 			XGMAC_SET_BITS(packet->errors, RX_PACKET_ERRORS,
 				       FRAME, 1);
+		}
 	}
 
+	pdata->ext_stats.rxq_packets[channel->queue_index]++;
+	pdata->ext_stats.rxq_bytes[channel->queue_index] += rdata->rx.len;
+
 	DBGPR("<--xgbe_dev_read: %s - descriptor=%u (cur=%d)\n", channel->name,
 	      ring->cur & (ring->rdesc_count - 1), ring->cur);
 
@@ -1964,44 +2066,40 @@ static int xgbe_is_last_desc(struct xgbe_ring_desc *rdesc)
 static int xgbe_enable_int(struct xgbe_channel *channel,
 			   enum xgbe_int int_id)
 {
-	unsigned int dma_ch_ier;
-
-	dma_ch_ier = XGMAC_DMA_IOREAD(channel, DMA_CH_IER);
-
 	switch (int_id) {
 	case XGMAC_INT_DMA_CH_SR_TI:
-		XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, TIE, 1);
+		XGMAC_SET_BITS(channel->curr_ier, DMA_CH_IER, TIE, 1);
 		break;
 	case XGMAC_INT_DMA_CH_SR_TPS:
-		XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, TXSE, 1);
+		XGMAC_SET_BITS(channel->curr_ier, DMA_CH_IER, TXSE, 1);
 		break;
 	case XGMAC_INT_DMA_CH_SR_TBU:
-		XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, TBUE, 1);
+		XGMAC_SET_BITS(channel->curr_ier, DMA_CH_IER, TBUE, 1);
 		break;
 	case XGMAC_INT_DMA_CH_SR_RI:
-		XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, RIE, 1);
+		XGMAC_SET_BITS(channel->curr_ier, DMA_CH_IER, RIE, 1);
 		break;
 	case XGMAC_INT_DMA_CH_SR_RBU:
-		XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, RBUE, 1);
+		XGMAC_SET_BITS(channel->curr_ier, DMA_CH_IER, RBUE, 1);
 		break;
 	case XGMAC_INT_DMA_CH_SR_RPS:
-		XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, RSE, 1);
+		XGMAC_SET_BITS(channel->curr_ier, DMA_CH_IER, RSE, 1);
 		break;
 	case XGMAC_INT_DMA_CH_SR_TI_RI:
-		XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, TIE, 1);
-		XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, RIE, 1);
+		XGMAC_SET_BITS(channel->curr_ier, DMA_CH_IER, TIE, 1);
+		XGMAC_SET_BITS(channel->curr_ier, DMA_CH_IER, RIE, 1);
 		break;
 	case XGMAC_INT_DMA_CH_SR_FBE:
-		XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, FBEE, 1);
+		XGMAC_SET_BITS(channel->curr_ier, DMA_CH_IER, FBEE, 1);
 		break;
 	case XGMAC_INT_DMA_ALL:
-		dma_ch_ier |= channel->saved_ier;
+		channel->curr_ier |= channel->saved_ier;
 		break;
 	default:
 		return -1;
 	}
 
-	XGMAC_DMA_IOWRITE(channel, DMA_CH_IER, dma_ch_ier);
+	XGMAC_DMA_IOWRITE(channel, DMA_CH_IER, channel->curr_ier);
 
 	return 0;
 }
@@ -2009,45 +2107,41 @@ static int xgbe_enable_int(struct xgbe_channel *channel,
 static int xgbe_disable_int(struct xgbe_channel *channel,
 			    enum xgbe_int int_id)
 {
-	unsigned int dma_ch_ier;
-
-	dma_ch_ier = XGMAC_DMA_IOREAD(channel, DMA_CH_IER);
-
 	switch (int_id) {
 	case XGMAC_INT_DMA_CH_SR_TI:
-		XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, TIE, 0);
+		XGMAC_SET_BITS(channel->curr_ier, DMA_CH_IER, TIE, 0);
 		break;
 	case XGMAC_INT_DMA_CH_SR_TPS:
-		XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, TXSE, 0);
+		XGMAC_SET_BITS(channel->curr_ier, DMA_CH_IER, TXSE, 0);
 		break;
 	case XGMAC_INT_DMA_CH_SR_TBU:
-		XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, TBUE, 0);
+		XGMAC_SET_BITS(channel->curr_ier, DMA_CH_IER, TBUE, 0);
 		break;
 	case XGMAC_INT_DMA_CH_SR_RI:
-		XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, RIE, 0);
+		XGMAC_SET_BITS(channel->curr_ier, DMA_CH_IER, RIE, 0);
 		break;
 	case XGMAC_INT_DMA_CH_SR_RBU:
-		XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, RBUE, 0);
+		XGMAC_SET_BITS(channel->curr_ier, DMA_CH_IER, RBUE, 0);
 		break;
 	case XGMAC_INT_DMA_CH_SR_RPS:
-		XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, RSE, 0);
+		XGMAC_SET_BITS(channel->curr_ier, DMA_CH_IER, RSE, 0);
 		break;
 	case XGMAC_INT_DMA_CH_SR_TI_RI:
-		XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, TIE, 0);
-		XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, RIE, 0);
+		XGMAC_SET_BITS(channel->curr_ier, DMA_CH_IER, TIE, 0);
+		XGMAC_SET_BITS(channel->curr_ier, DMA_CH_IER, RIE, 0);
 		break;
 	case XGMAC_INT_DMA_CH_SR_FBE:
-		XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, FBEE, 0);
+		XGMAC_SET_BITS(channel->curr_ier, DMA_CH_IER, FBEE, 0);
 		break;
 	case XGMAC_INT_DMA_ALL:
-		channel->saved_ier = dma_ch_ier & XGBE_DMA_INTERRUPT_MASK;
-		dma_ch_ier &= ~XGBE_DMA_INTERRUPT_MASK;
+		channel->saved_ier = channel->curr_ier;
+		channel->curr_ier = 0;
 		break;
 	default:
 		return -1;
 	}
 
-	XGMAC_DMA_IOWRITE(channel, DMA_CH_IER, dma_ch_ier);
+	XGMAC_DMA_IOWRITE(channel, DMA_CH_IER, channel->curr_ier);
 
 	return 0;
 }
@@ -3534,5 +3628,10 @@ void xgbe_init_function_ptrs_dev(struct xgbe_hw_if *hw_if)
 	hw_if->disable_ecc_ded = xgbe_disable_ecc_ded;
 	hw_if->disable_ecc_sec = xgbe_disable_ecc_sec;
 
+	/* For VXLAN */
+	hw_if->enable_vxlan = xgbe_enable_vxlan;
+	hw_if->disable_vxlan = xgbe_disable_vxlan;
+	hw_if->set_vxlan_id = xgbe_set_vxlan_id;
+
 	DBGPR("<--xgbe_init_function_ptrs\n");
 }
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
index ecef3ee87b17..608693d11bd7 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
@@ -124,6 +124,7 @@
 #include <linux/if_ether.h>
 #include <linux/net_tstamp.h>
 #include <linux/phy.h>
+#include <net/vxlan.h>
 
 #include "xgbe.h"
 #include "xgbe-common.h"
@@ -732,8 +733,6 @@ void xgbe_get_all_hw_features(struct xgbe_prv_data *pdata)
 	unsigned int mac_hfr0, mac_hfr1, mac_hfr2;
 	struct xgbe_hw_features *hw_feat = &pdata->hw_feat;
 
-	DBGPR("-->xgbe_get_all_hw_features\n");
-
 	mac_hfr0 = XGMAC_IOREAD(pdata, MAC_HWF0R);
 	mac_hfr1 = XGMAC_IOREAD(pdata, MAC_HWF1R);
 	mac_hfr2 = XGMAC_IOREAD(pdata, MAC_HWF2R);
@@ -758,6 +757,7 @@ void xgbe_get_all_hw_features(struct xgbe_prv_data *pdata)
 					      ADDMACADRSEL);
 	hw_feat->ts_src      = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, TSSTSSEL);
 	hw_feat->sa_vlan_ins = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, SAVLANINS);
+	hw_feat->vxn         = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, VXN);
 
 	/* Hardware feature register 1 */
 	hw_feat->rx_fifo_size  = XGMAC_GET_BITS(mac_hfr1, MAC_HWF1R,
@@ -828,7 +828,193 @@ void xgbe_get_all_hw_features(struct xgbe_prv_data *pdata)
 	hw_feat->rx_fifo_size = 1 << (hw_feat->rx_fifo_size + 7);
 	hw_feat->tx_fifo_size = 1 << (hw_feat->tx_fifo_size + 7);
 
-	DBGPR("<--xgbe_get_all_hw_features\n");
+	if (netif_msg_probe(pdata)) {
+		dev_dbg(pdata->dev, "Hardware features:\n");
+
+		/* Hardware feature register 0 */
+		dev_dbg(pdata->dev, "  1GbE support              : %s\n",
+			hw_feat->gmii ? "yes" : "no");
+		dev_dbg(pdata->dev, "  VLAN hash filter          : %s\n",
+			hw_feat->vlhash ? "yes" : "no");
+		dev_dbg(pdata->dev, "  MDIO interface            : %s\n",
+			hw_feat->sma ? "yes" : "no");
+		dev_dbg(pdata->dev, "  Wake-up packet support    : %s\n",
+			hw_feat->rwk ? "yes" : "no");
+		dev_dbg(pdata->dev, "  Magic packet support      : %s\n",
+			hw_feat->mgk ? "yes" : "no");
+		dev_dbg(pdata->dev, "  Management counters       : %s\n",
+			hw_feat->mmc ? "yes" : "no");
+		dev_dbg(pdata->dev, "  ARP offload               : %s\n",
+			hw_feat->aoe ? "yes" : "no");
+		dev_dbg(pdata->dev, "  IEEE 1588-2008 Timestamp  : %s\n",
+			hw_feat->ts ? "yes" : "no");
+		dev_dbg(pdata->dev, "  Energy Efficient Ethernet : %s\n",
+			hw_feat->eee ? "yes" : "no");
+		dev_dbg(pdata->dev, "  TX checksum offload       : %s\n",
+			hw_feat->tx_coe ? "yes" : "no");
+		dev_dbg(pdata->dev, "  RX checksum offload       : %s\n",
+			hw_feat->rx_coe ? "yes" : "no");
+		dev_dbg(pdata->dev, "  Additional MAC addresses  : %u\n",
+			hw_feat->addn_mac);
+		dev_dbg(pdata->dev, "  Timestamp source          : %s\n",
+			(hw_feat->ts_src == 1) ? "internal" :
+			(hw_feat->ts_src == 2) ? "external" :
+			(hw_feat->ts_src == 3) ? "internal/external" : "n/a");
+		dev_dbg(pdata->dev, "  SA/VLAN insertion         : %s\n",
+			hw_feat->sa_vlan_ins ? "yes" : "no");
+		dev_dbg(pdata->dev, "  VXLAN/NVGRE support       : %s\n",
+			hw_feat->vxn ? "yes" : "no");
+
+		/* Hardware feature register 1 */
+		dev_dbg(pdata->dev, "  RX fifo size              : %u\n",
+			hw_feat->rx_fifo_size);
+		dev_dbg(pdata->dev, "  TX fifo size              : %u\n",
+			hw_feat->tx_fifo_size);
+		dev_dbg(pdata->dev, "  IEEE 1588 high word       : %s\n",
+			hw_feat->adv_ts_hi ? "yes" : "no");
+		dev_dbg(pdata->dev, "  DMA width                 : %u\n",
+			hw_feat->dma_width);
+		dev_dbg(pdata->dev, "  Data Center Bridging      : %s\n",
+			hw_feat->dcb ? "yes" : "no");
+		dev_dbg(pdata->dev, "  Split header              : %s\n",
+			hw_feat->sph ? "yes" : "no");
+		dev_dbg(pdata->dev, "  TCP Segmentation Offload  : %s\n",
+			hw_feat->tso ? "yes" : "no");
+		dev_dbg(pdata->dev, "  Debug memory interface    : %s\n",
+			hw_feat->dma_debug ? "yes" : "no");
+		dev_dbg(pdata->dev, "  Receive Side Scaling      : %s\n",
+			hw_feat->rss ? "yes" : "no");
+		dev_dbg(pdata->dev, "  Traffic Class count       : %u\n",
+			hw_feat->tc_cnt);
+		dev_dbg(pdata->dev, "  Hash table size           : %u\n",
+			hw_feat->hash_table_size);
+		dev_dbg(pdata->dev, "  L3/L4 Filters             : %u\n",
+			hw_feat->l3l4_filter_num);
+
+		/* Hardware feature register 2 */
+		dev_dbg(pdata->dev, "  RX queue count            : %u\n",
+			hw_feat->rx_q_cnt);
+		dev_dbg(pdata->dev, "  TX queue count            : %u\n",
+			hw_feat->tx_q_cnt);
+		dev_dbg(pdata->dev, "  RX DMA channel count      : %u\n",
+			hw_feat->rx_ch_cnt);
+		dev_dbg(pdata->dev, "  TX DMA channel count      : %u\n",
+			hw_feat->rx_ch_cnt);
+		dev_dbg(pdata->dev, "  PPS outputs               : %u\n",
+			hw_feat->pps_out_num);
+		dev_dbg(pdata->dev, "  Auxiliary snapshot inputs : %u\n",
+			hw_feat->aux_snap_num);
+	}
+}
+
+static void xgbe_disable_vxlan_offloads(struct xgbe_prv_data *pdata)
+{
+	struct net_device *netdev = pdata->netdev;
+
+	if (!pdata->vxlan_offloads_set)
+		return;
+
+	netdev_info(netdev, "disabling VXLAN offloads\n");
+
+	netdev->hw_enc_features &= ~(NETIF_F_SG |
+				     NETIF_F_IP_CSUM |
+				     NETIF_F_IPV6_CSUM |
+				     NETIF_F_RXCSUM |
+				     NETIF_F_TSO |
+				     NETIF_F_TSO6 |
+				     NETIF_F_GRO |
+				     NETIF_F_GSO_UDP_TUNNEL |
+				     NETIF_F_GSO_UDP_TUNNEL_CSUM);
+
+	netdev->features &= ~(NETIF_F_GSO_UDP_TUNNEL |
+			      NETIF_F_GSO_UDP_TUNNEL_CSUM);
+
+	pdata->vxlan_offloads_set = 0;
+}
+
+static void xgbe_disable_vxlan_hw(struct xgbe_prv_data *pdata)
+{
+	if (!pdata->vxlan_port_set)
+		return;
+
+	pdata->hw_if.disable_vxlan(pdata);
+
+	pdata->vxlan_port_set = 0;
+	pdata->vxlan_port = 0;
+}
+
+static void xgbe_disable_vxlan_accel(struct xgbe_prv_data *pdata)
+{
+	xgbe_disable_vxlan_offloads(pdata);
+
+	xgbe_disable_vxlan_hw(pdata);
+}
+
+static void xgbe_enable_vxlan_offloads(struct xgbe_prv_data *pdata)
+{
+	struct net_device *netdev = pdata->netdev;
+
+	if (pdata->vxlan_offloads_set)
+		return;
+
+	netdev_info(netdev, "enabling VXLAN offloads\n");
+
+	netdev->hw_enc_features |= NETIF_F_SG |
+				   NETIF_F_IP_CSUM |
+				   NETIF_F_IPV6_CSUM |
+				   NETIF_F_RXCSUM |
+				   NETIF_F_TSO |
+				   NETIF_F_TSO6 |
+				   NETIF_F_GRO |
+				   pdata->vxlan_features;
+
+	netdev->features |= pdata->vxlan_features;
+
+	pdata->vxlan_offloads_set = 1;
+}
+
+static void xgbe_enable_vxlan_hw(struct xgbe_prv_data *pdata)
+{
+	struct xgbe_vxlan_data *vdata;
+
+	if (pdata->vxlan_port_set)
+		return;
+
+	if (list_empty(&pdata->vxlan_ports))
+		return;
+
+	vdata = list_first_entry(&pdata->vxlan_ports,
+				 struct xgbe_vxlan_data, list);
+
+	pdata->vxlan_port_set = 1;
+	pdata->vxlan_port = be16_to_cpu(vdata->port);
+
+	pdata->hw_if.enable_vxlan(pdata);
+}
+
+static void xgbe_enable_vxlan_accel(struct xgbe_prv_data *pdata)
+{
+	/* VXLAN acceleration desired? */
+	if (!pdata->vxlan_features)
+		return;
+
+	/* VXLAN acceleration possible? */
+	if (pdata->vxlan_force_disable)
+		return;
+
+	xgbe_enable_vxlan_hw(pdata);
+
+	xgbe_enable_vxlan_offloads(pdata);
+}
+
+static void xgbe_reset_vxlan_accel(struct xgbe_prv_data *pdata)
+{
+	xgbe_disable_vxlan_hw(pdata);
+
+	if (pdata->vxlan_features)
+		xgbe_enable_vxlan_offloads(pdata);
+
+	pdata->vxlan_force_disable = 0;
 }
 
 static void xgbe_napi_enable(struct xgbe_prv_data *pdata, unsigned int add)
@@ -887,7 +1073,7 @@ static int xgbe_request_irqs(struct xgbe_prv_data *pdata)
 		     (unsigned long)pdata);
 
 	ret = devm_request_irq(pdata->dev, pdata->dev_irq, xgbe_isr, 0,
-			       netdev->name, pdata);
+			       netdev_name(netdev), pdata);
 	if (ret) {
 		netdev_alert(netdev, "error requesting irq %d\n",
 			     pdata->dev_irq);
@@ -1154,6 +1340,8 @@ static int xgbe_start(struct xgbe_prv_data *pdata)
 	hw_if->enable_tx(pdata);
 	hw_if->enable_rx(pdata);
 
+	udp_tunnel_get_rx_info(netdev);
+
 	netif_tx_start_all_queues(netdev);
 
 	xgbe_start_timers(pdata);
@@ -1195,6 +1383,8 @@ static void xgbe_stop(struct xgbe_prv_data *pdata)
 	xgbe_stop_timers(pdata);
 	flush_workqueue(pdata->dev_workqueue);
 
+	xgbe_reset_vxlan_accel(pdata);
+
 	hw_if->disable_tx(pdata);
 	hw_if->disable_rx(pdata);
 
@@ -1483,10 +1673,18 @@ static int xgbe_prep_tso(struct sk_buff *skb, struct xgbe_packet_data *packet)
 	if (ret)
 		return ret;
 
-	packet->header_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
-	packet->tcp_header_len = tcp_hdrlen(skb);
+	if (XGMAC_GET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES, VXLAN)) {
+		packet->header_len = skb_inner_transport_offset(skb) +
+				     inner_tcp_hdrlen(skb);
+		packet->tcp_header_len = inner_tcp_hdrlen(skb);
+	} else {
+		packet->header_len = skb_transport_offset(skb) +
+				     tcp_hdrlen(skb);
+		packet->tcp_header_len = tcp_hdrlen(skb);
+	}
 	packet->tcp_payload_len = skb->len - packet->header_len;
 	packet->mss = skb_shinfo(skb)->gso_size;
+
 	DBGPR("  packet->header_len=%u\n", packet->header_len);
 	DBGPR("  packet->tcp_header_len=%u, packet->tcp_payload_len=%u\n",
 	      packet->tcp_header_len, packet->tcp_payload_len);
@@ -1501,6 +1699,49 @@ static int xgbe_prep_tso(struct sk_buff *skb, struct xgbe_packet_data *packet)
 	return 0;
 }
 
+static bool xgbe_is_vxlan(struct xgbe_prv_data *pdata, struct sk_buff *skb)
+{
+	struct xgbe_vxlan_data *vdata;
+
+	if (pdata->vxlan_force_disable)
+		return false;
+
+	if (!skb->encapsulation)
+		return false;
+
+	if (skb->ip_summed != CHECKSUM_PARTIAL)
+		return false;
+
+	switch (skb->protocol) {
+	case htons(ETH_P_IP):
+		if (ip_hdr(skb)->protocol != IPPROTO_UDP)
+			return false;
+		break;
+
+	case htons(ETH_P_IPV6):
+		if (ipv6_hdr(skb)->nexthdr != IPPROTO_UDP)
+			return false;
+		break;
+
+	default:
+		return false;
+	}
+
+	/* See if we have the UDP port in our list */
+	list_for_each_entry(vdata, &pdata->vxlan_ports, list) {
+		if ((skb->protocol == htons(ETH_P_IP)) &&
+		    (vdata->sa_family == AF_INET) &&
+		    (vdata->port == udp_hdr(skb)->dest))
+			return true;
+		else if ((skb->protocol == htons(ETH_P_IPV6)) &&
+			 (vdata->sa_family == AF_INET6) &&
+			 (vdata->port == udp_hdr(skb)->dest))
+			return true;
+	}
+
+	return false;
+}
+
 static int xgbe_is_tso(struct sk_buff *skb)
 {
 	if (skb->ip_summed != CHECKSUM_PARTIAL)
@@ -1549,6 +1790,10 @@ static void xgbe_packet_info(struct xgbe_prv_data *pdata,
 		XGMAC_SET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES,
 			       CSUM_ENABLE, 1);
 
+	if (xgbe_is_vxlan(pdata, skb))
+		XGMAC_SET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES,
+			       VXLAN, 1);
+
 	if (skb_vlan_tag_present(skb)) {
 		/* VLAN requires an extra descriptor if tag is different */
 		if (skb_vlan_tag_get(skb) != ring->tx.cur_vlan_ctag)
@@ -1589,16 +1834,42 @@ static int xgbe_open(struct net_device *netdev)
 
 	DBGPR("-->xgbe_open\n");
 
+	/* Create the various names based on netdev name */
+	snprintf(pdata->an_name, sizeof(pdata->an_name) - 1, "%s-pcs",
+		 netdev_name(netdev));
+
+	snprintf(pdata->ecc_name, sizeof(pdata->ecc_name) - 1, "%s-ecc",
+		 netdev_name(netdev));
+
+	snprintf(pdata->i2c_name, sizeof(pdata->i2c_name) - 1, "%s-i2c",
+		 netdev_name(netdev));
+
+	/* Create workqueues */
+	pdata->dev_workqueue =
+		create_singlethread_workqueue(netdev_name(netdev));
+	if (!pdata->dev_workqueue) {
+		netdev_err(netdev, "device workqueue creation failed\n");
+		return -ENOMEM;
+	}
+
+	pdata->an_workqueue =
+		create_singlethread_workqueue(pdata->an_name);
+	if (!pdata->an_workqueue) {
+		netdev_err(netdev, "phy workqueue creation failed\n");
+		ret = -ENOMEM;
+		goto err_dev_wq;
+	}
+
 	/* Reset the phy settings */
 	ret = xgbe_phy_reset(pdata);
 	if (ret)
-		return ret;
+		goto err_an_wq;
 
 	/* Enable the clocks */
 	ret = clk_prepare_enable(pdata->sysclk);
 	if (ret) {
 		netdev_alert(netdev, "dma clk_prepare_enable failed\n");
-		return ret;
+		goto err_an_wq;
 	}
 
 	ret = clk_prepare_enable(pdata->ptpclk);
@@ -1651,6 +1922,12 @@ err_ptpclk:
 err_sysclk:
 	clk_disable_unprepare(pdata->sysclk);
 
+err_an_wq:
+	destroy_workqueue(pdata->an_workqueue);
+
+err_dev_wq:
+	destroy_workqueue(pdata->dev_workqueue);
+
 	return ret;
 }
 
@@ -1674,6 +1951,12 @@ static int xgbe_close(struct net_device *netdev)
 	clk_disable_unprepare(pdata->ptpclk);
 	clk_disable_unprepare(pdata->sysclk);
 
+	flush_workqueue(pdata->an_workqueue);
+	destroy_workqueue(pdata->an_workqueue);
+
+	flush_workqueue(pdata->dev_workqueue);
+	destroy_workqueue(pdata->dev_workqueue);
+
 	set_bit(XGBE_DOWN, &pdata->dev_state);
 
 	DBGPR("<--xgbe_close\n");
@@ -1918,18 +2201,18 @@ static void xgbe_poll_controller(struct net_device *netdev)
 }
 #endif /* End CONFIG_NET_POLL_CONTROLLER */
 
-static int xgbe_setup_tc(struct net_device *netdev, u32 handle, u32 chain_index,
-			 __be16 proto,
-			 struct tc_to_netdev *tc_to_netdev)
+static int xgbe_setup_tc(struct net_device *netdev, enum tc_setup_type type,
+			 void *type_data)
 {
 	struct xgbe_prv_data *pdata = netdev_priv(netdev);
+	struct tc_mqprio_qopt *mqprio = type_data;
 	u8 tc;
 
-	if (tc_to_netdev->type != TC_SETUP_MQPRIO)
-		return -EINVAL;
+	if (type != TC_SETUP_MQPRIO)
+		return -EOPNOTSUPP;
 
-	tc_to_netdev->mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
-	tc = tc_to_netdev->mqprio->num_tc;
+	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
+	tc = mqprio->num_tc;
 
 	if (tc > pdata->hw_feat.tc_cnt)
 		return -EINVAL;
@@ -1940,18 +2223,83 @@ static int xgbe_setup_tc(struct net_device *netdev, u32 handle, u32 chain_index,
 	return 0;
 }
 
+static netdev_features_t xgbe_fix_features(struct net_device *netdev,
+					   netdev_features_t features)
+{
+	struct xgbe_prv_data *pdata = netdev_priv(netdev);
+	netdev_features_t vxlan_base, vxlan_mask;
+
+	vxlan_base = NETIF_F_GSO_UDP_TUNNEL | NETIF_F_RX_UDP_TUNNEL_PORT;
+	vxlan_mask = vxlan_base | NETIF_F_GSO_UDP_TUNNEL_CSUM;
+
+	pdata->vxlan_features = features & vxlan_mask;
+
+	/* Only fix VXLAN-related features */
+	if (!pdata->vxlan_features)
+		return features;
+
+	/* If VXLAN isn't supported then clear any features:
+	 *   This is needed because NETIF_F_RX_UDP_TUNNEL_PORT gets
+	 *   automatically set if ndo_udp_tunnel_add is set.
+	 */
+	if (!pdata->hw_feat.vxn)
+		return features & ~vxlan_mask;
+
+	/* VXLAN CSUM requires VXLAN base */
+	if ((features & NETIF_F_GSO_UDP_TUNNEL_CSUM) &&
+	    !(features & NETIF_F_GSO_UDP_TUNNEL)) {
+		netdev_notice(netdev,
+			      "forcing tx udp tunnel support\n");
+		features |= NETIF_F_GSO_UDP_TUNNEL;
+	}
+
+	/* Can't do one without doing the other */
+	if ((features & vxlan_base) != vxlan_base) {
+		netdev_notice(netdev,
+			      "forcing both tx and rx udp tunnel support\n");
+		features |= vxlan_base;
+	}
+
+	if (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
+		if (!(features & NETIF_F_GSO_UDP_TUNNEL_CSUM)) {
+			netdev_notice(netdev,
+				      "forcing tx udp tunnel checksumming on\n");
+			features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
+		}
+	} else {
+		if (features & NETIF_F_GSO_UDP_TUNNEL_CSUM) {
+			netdev_notice(netdev,
+				      "forcing tx udp tunnel checksumming off\n");
+			features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM;
+		}
+	}
+
+	pdata->vxlan_features = features & vxlan_mask;
+
+	/* Adjust UDP Tunnel based on current state */
+	if (pdata->vxlan_force_disable) {
+		netdev_notice(netdev,
+			      "VXLAN acceleration disabled, turning off udp tunnel features\n");
+		features &= ~vxlan_mask;
+	}
+
+	return features;
+}
+
 static int xgbe_set_features(struct net_device *netdev,
 			     netdev_features_t features)
 {
 	struct xgbe_prv_data *pdata = netdev_priv(netdev);
 	struct xgbe_hw_if *hw_if = &pdata->hw_if;
 	netdev_features_t rxhash, rxcsum, rxvlan, rxvlan_filter;
+	netdev_features_t udp_tunnel;
 	int ret = 0;
 
 	rxhash = pdata->netdev_features & NETIF_F_RXHASH;
 	rxcsum = pdata->netdev_features & NETIF_F_RXCSUM;
 	rxvlan = pdata->netdev_features & NETIF_F_HW_VLAN_CTAG_RX;
 	rxvlan_filter = pdata->netdev_features & NETIF_F_HW_VLAN_CTAG_FILTER;
+	udp_tunnel = pdata->netdev_features & NETIF_F_GSO_UDP_TUNNEL;
 
 	if ((features & NETIF_F_RXHASH) && !rxhash)
 		ret = hw_if->enable_rss(pdata);
@@ -1975,6 +2323,11 @@ static int xgbe_set_features(struct net_device *netdev,
 	else if (!(features & NETIF_F_HW_VLAN_CTAG_FILTER) && rxvlan_filter)
 		hw_if->disable_rx_vlan_filtering(pdata);
 
+	if ((features & NETIF_F_GSO_UDP_TUNNEL) && !udp_tunnel)
+		xgbe_enable_vxlan_accel(pdata);
+	else if (!(features & NETIF_F_GSO_UDP_TUNNEL) && udp_tunnel)
+		xgbe_disable_vxlan_accel(pdata);
+
 	pdata->netdev_features = features;
 
 	DBGPR("<--xgbe_set_features\n");
@@ -1982,6 +2335,111 @@ static int xgbe_set_features(struct net_device *netdev,
 	return 0;
 }
 
+static void xgbe_udp_tunnel_add(struct net_device *netdev,
+				struct udp_tunnel_info *ti)
+{
+	struct xgbe_prv_data *pdata = netdev_priv(netdev);
+	struct xgbe_vxlan_data *vdata;
+
+	if (!pdata->hw_feat.vxn)
+		return;
+
+	if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
+		return;
+
+	pdata->vxlan_port_count++;
+
+	netif_dbg(pdata, drv, netdev,
+		  "adding VXLAN tunnel, family=%hx/port=%hx\n",
+		  ti->sa_family, be16_to_cpu(ti->port));
+
+	if (pdata->vxlan_force_disable)
+		return;
+
+	vdata = kzalloc(sizeof(*vdata), GFP_ATOMIC);
+	if (!vdata) {
+		/* Can no longer properly track VXLAN ports */
+		pdata->vxlan_force_disable = 1;
+		netif_dbg(pdata, drv, netdev,
+			  "internal error, disabling VXLAN accelerations\n");
+
+		xgbe_disable_vxlan_accel(pdata);
+
+		return;
+	}
+	vdata->sa_family = ti->sa_family;
+	vdata->port = ti->port;
+
+	list_add_tail(&vdata->list, &pdata->vxlan_ports);
+
+	/* First port added? */
+	if (pdata->vxlan_port_count == 1) {
+		xgbe_enable_vxlan_accel(pdata);
+
+		return;
+	}
+}
+
+static void xgbe_udp_tunnel_del(struct net_device *netdev,
+				struct udp_tunnel_info *ti)
+{
+	struct xgbe_prv_data *pdata = netdev_priv(netdev);
+	struct xgbe_vxlan_data *vdata;
+
+	if (!pdata->hw_feat.vxn)
+		return;
+
+	if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
+		return;
+
+	netif_dbg(pdata, drv, netdev,
+		  "deleting VXLAN tunnel, family=%hx/port=%hx\n",
+		  ti->sa_family, be16_to_cpu(ti->port));
+
+	/* Don't need safe version since loop terminates with deletion */
+	list_for_each_entry(vdata, &pdata->vxlan_ports, list) {
+		if (vdata->sa_family != ti->sa_family)
+			continue;
+
+		if (vdata->port != ti->port)
+			continue;
+
+		list_del(&vdata->list);
+		kfree(vdata);
+
+		break;
+	}
+
+	pdata->vxlan_port_count--;
+	if (!pdata->vxlan_port_count) {
+		xgbe_reset_vxlan_accel(pdata);
+
+		return;
+	}
+
+	if (pdata->vxlan_force_disable)
+		return;
+
+	/* See if VXLAN tunnel id needs to be changed */
+	vdata = list_first_entry(&pdata->vxlan_ports,
+				 struct xgbe_vxlan_data, list);
+	if (pdata->vxlan_port == be16_to_cpu(vdata->port))
+		return;
+
+	pdata->vxlan_port = be16_to_cpu(vdata->port);
+	pdata->hw_if.set_vxlan_id(pdata);
+}
+
+static netdev_features_t xgbe_features_check(struct sk_buff *skb,
+					     struct net_device *netdev,
+					     netdev_features_t features)
+{
+	features = vlan_features_check(skb, features);
+	features = vxlan_features_check(skb, features);
+
+	return features;
+}
+
 static const struct net_device_ops xgbe_netdev_ops = {
 	.ndo_open		= xgbe_open,
 	.ndo_stop		= xgbe_close,
@@ -1999,7 +2457,11 @@ static const struct net_device_ops xgbe_netdev_ops = {
 	.ndo_poll_controller	= xgbe_poll_controller,
 #endif
 	.ndo_setup_tc		= xgbe_setup_tc,
+	.ndo_fix_features	= xgbe_fix_features,
 	.ndo_set_features	= xgbe_set_features,
+	.ndo_udp_tunnel_add	= xgbe_udp_tunnel_add,
+	.ndo_udp_tunnel_del	= xgbe_udp_tunnel_del,
+	.ndo_features_check	= xgbe_features_check,
 };
 
 const struct net_device_ops *xgbe_get_netdev_ops(void)
@@ -2312,6 +2774,15 @@ skip_data:
 			skb->ip_summed = CHECKSUM_UNNECESSARY;
 
 		if (XGMAC_GET_BITS(packet->attributes,
+				   RX_PACKET_ATTRIBUTES, TNP)) {
+			skb->encapsulation = 1;
+
+			if (XGMAC_GET_BITS(packet->attributes,
+					   RX_PACKET_ATTRIBUTES, TNPCSUM_DONE))
+				skb->csum_level = 1;
+		}
+
+		if (XGMAC_GET_BITS(packet->attributes,
 				   RX_PACKET_ATTRIBUTES, VLAN_CTAG))
 			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
 					       packet->vlan_ctag);
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c b/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c
index 67a2e52ad25d..ff397bb25042 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c
@@ -146,6 +146,7 @@ static const struct xgbe_stats xgbe_gstring_stats[] = {
 	XGMAC_MMC_STAT("tx_broadcast_packets", txbroadcastframes_gb),
 	XGMAC_MMC_STAT("tx_multicast_packets", txmulticastframes_gb),
 	XGMAC_MMC_STAT("tx_vlan_packets", txvlanframes_g),
+	XGMAC_EXT_STAT("tx_vxlan_packets", tx_vxlan_packets),
 	XGMAC_EXT_STAT("tx_tso_packets", tx_tso_packets),
 	XGMAC_MMC_STAT("tx_64_byte_packets", tx64octets_gb),
 	XGMAC_MMC_STAT("tx_65_to_127_byte_packets", tx65to127octets_gb),
@@ -162,6 +163,7 @@ static const struct xgbe_stats xgbe_gstring_stats[] = {
 	XGMAC_MMC_STAT("rx_broadcast_packets", rxbroadcastframes_g),
 	XGMAC_MMC_STAT("rx_multicast_packets", rxmulticastframes_g),
 	XGMAC_MMC_STAT("rx_vlan_packets", rxvlanframes_gb),
+	XGMAC_EXT_STAT("rx_vxlan_packets", rx_vxlan_packets),
 	XGMAC_MMC_STAT("rx_64_byte_packets", rx64octets_gb),
 	XGMAC_MMC_STAT("rx_65_to_127_byte_packets", rx65to127octets_gb),
 	XGMAC_MMC_STAT("rx_128_to_255_byte_packets", rx128to255octets_gb),
@@ -177,6 +179,8 @@ static const struct xgbe_stats xgbe_gstring_stats[] = {
 	XGMAC_MMC_STAT("rx_out_of_range_errors", rxoutofrangetype),
 	XGMAC_MMC_STAT("rx_fifo_overflow_errors", rxfifooverflow),
 	XGMAC_MMC_STAT("rx_watchdog_errors", rxwatchdogerror),
+	XGMAC_EXT_STAT("rx_csum_errors", rx_csum_errors),
+	XGMAC_EXT_STAT("rx_vxlan_csum_errors", rx_vxlan_csum_errors),
 	XGMAC_MMC_STAT("rx_pause_frames", rxpauseframes),
 	XGMAC_EXT_STAT("rx_split_header_packets", rx_split_header_packets),
 	XGMAC_EXT_STAT("rx_buffer_unavailable", rx_buffer_unavailable),
@@ -186,6 +190,7 @@ static const struct xgbe_stats xgbe_gstring_stats[] = {
 
 static void xgbe_get_strings(struct net_device *netdev, u32 stringset, u8 *data)
 {
+	struct xgbe_prv_data *pdata = netdev_priv(netdev);
 	int i;
 
 	switch (stringset) {
@@ -195,6 +200,18 @@ static void xgbe_get_strings(struct net_device *netdev, u32 stringset, u8 *data)
 			       ETH_GSTRING_LEN);
 			data += ETH_GSTRING_LEN;
 		}
+		for (i = 0; i < pdata->tx_ring_count; i++) {
+			sprintf(data, "txq_%u_packets", i);
+			data += ETH_GSTRING_LEN;
+			sprintf(data, "txq_%u_bytes", i);
+			data += ETH_GSTRING_LEN;
+		}
+		for (i = 0; i < pdata->rx_ring_count; i++) {
+			sprintf(data, "rxq_%u_packets", i);
+			data += ETH_GSTRING_LEN;
+			sprintf(data, "rxq_%u_bytes", i);
+			data += ETH_GSTRING_LEN;
+		}
 		break;
 	}
 }
@@ -211,15 +228,26 @@ static void xgbe_get_ethtool_stats(struct net_device *netdev,
 		stat = (u8 *)pdata + xgbe_gstring_stats[i].stat_offset;
 		*data++ = *(u64 *)stat;
 	}
+	for (i = 0; i < pdata->tx_ring_count; i++) {
+		*data++ = pdata->ext_stats.txq_packets[i];
+		*data++ = pdata->ext_stats.txq_bytes[i];
+	}
+	for (i = 0; i < pdata->rx_ring_count; i++) {
+		*data++ = pdata->ext_stats.rxq_packets[i];
+		*data++ = pdata->ext_stats.rxq_bytes[i];
+	}
 }
 
 static int xgbe_get_sset_count(struct net_device *netdev, int stringset)
 {
+	struct xgbe_prv_data *pdata = netdev_priv(netdev);
 	int ret;
 
 	switch (stringset) {
 	case ETH_SS_STATS:
-		ret = XGBE_STATS_COUNT;
+		ret = XGBE_STATS_COUNT +
+		      (pdata->tx_ring_count * 2) +
+		      (pdata->rx_ring_count * 2);
 		break;
 
 	default:
@@ -243,6 +271,7 @@ static int xgbe_set_pauseparam(struct net_device *netdev,
 			       struct ethtool_pauseparam *pause)
 {
 	struct xgbe_prv_data *pdata = netdev_priv(netdev);
+	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
 	int ret = 0;
 
 	if (pause->autoneg && (pdata->phy.autoneg != AUTONEG_ENABLE)) {
@@ -255,16 +284,21 @@ static int xgbe_set_pauseparam(struct net_device *netdev,
 	pdata->phy.tx_pause = pause->tx_pause;
 	pdata->phy.rx_pause = pause->rx_pause;
 
-	pdata->phy.advertising &= ~ADVERTISED_Pause;
-	pdata->phy.advertising &= ~ADVERTISED_Asym_Pause;
+	XGBE_CLR_ADV(lks, Pause);
+	XGBE_CLR_ADV(lks, Asym_Pause);
 
 	if (pause->rx_pause) {
-		pdata->phy.advertising |= ADVERTISED_Pause;
-		pdata->phy.advertising |= ADVERTISED_Asym_Pause;
+		XGBE_SET_ADV(lks, Pause);
+		XGBE_SET_ADV(lks, Asym_Pause);
 	}
 
-	if (pause->tx_pause)
-		pdata->phy.advertising ^= ADVERTISED_Asym_Pause;
+	if (pause->tx_pause) {
+		/* Equivalent to XOR of Asym_Pause */
+		if (XGBE_ADV(lks, Asym_Pause))
+			XGBE_CLR_ADV(lks, Asym_Pause);
+		else
+			XGBE_SET_ADV(lks, Asym_Pause);
+	}
 
 	if (netif_running(netdev))
 		ret = pdata->phy_if.phy_config_aneg(pdata);
@@ -276,22 +310,20 @@ static int xgbe_get_link_ksettings(struct net_device *netdev,
 				   struct ethtool_link_ksettings *cmd)
 {
 	struct xgbe_prv_data *pdata = netdev_priv(netdev);
+	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
 
 	cmd->base.phy_address = pdata->phy.address;
 
-	ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported,
-						pdata->phy.supported);
-	ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising,
-						pdata->phy.advertising);
-	ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.lp_advertising,
-						pdata->phy.lp_advertising);
-
 	cmd->base.autoneg = pdata->phy.autoneg;
 	cmd->base.speed = pdata->phy.speed;
 	cmd->base.duplex = pdata->phy.duplex;
 
 	cmd->base.port = PORT_NONE;
 
+	XGBE_LM_COPY(cmd, supported, lks, supported);
+	XGBE_LM_COPY(cmd, advertising, lks, advertising);
+	XGBE_LM_COPY(cmd, lp_advertising, lks, lp_advertising);
+
 	return 0;
 }
 
@@ -299,7 +331,8 @@ static int xgbe_set_link_ksettings(struct net_device *netdev,
 				   const struct ethtool_link_ksettings *cmd)
 {
 	struct xgbe_prv_data *pdata = netdev_priv(netdev);
-	u32 advertising;
+	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
+	__ETHTOOL_DECLARE_LINK_MODE_MASK(advertising);
 	u32 speed;
 	int ret;
 
@@ -331,15 +364,17 @@ static int xgbe_set_link_ksettings(struct net_device *netdev,
 		}
 	}
 
-	ethtool_convert_link_mode_to_legacy_u32(&advertising,
-						cmd->link_modes.advertising);
-
 	netif_dbg(pdata, link, netdev,
-		  "requested advertisement %#x, phy supported %#x\n",
-		  advertising, pdata->phy.supported);
+		  "requested advertisement 0x%*pb, phy supported 0x%*pb\n",
+		  __ETHTOOL_LINK_MODE_MASK_NBITS, cmd->link_modes.advertising,
+		  __ETHTOOL_LINK_MODE_MASK_NBITS, lks->link_modes.supported);
+
+	bitmap_and(advertising,
+		   cmd->link_modes.advertising, lks->link_modes.supported,
+		   __ETHTOOL_LINK_MODE_MASK_NBITS);
 
-	advertising &= pdata->phy.supported;
-	if ((cmd->base.autoneg == AUTONEG_ENABLE) && !advertising) {
+	if ((cmd->base.autoneg == AUTONEG_ENABLE) &&
+	    bitmap_empty(advertising, __ETHTOOL_LINK_MODE_MASK_NBITS)) {
 		netdev_err(netdev,
 			   "unsupported requested advertisement\n");
 		return -EINVAL;
@@ -349,12 +384,13 @@ static int xgbe_set_link_ksettings(struct net_device *netdev,
 	pdata->phy.autoneg = cmd->base.autoneg;
 	pdata->phy.speed = speed;
 	pdata->phy.duplex = cmd->base.duplex;
-	pdata->phy.advertising = advertising;
+	bitmap_copy(lks->link_modes.advertising, advertising,
+		    __ETHTOOL_LINK_MODE_MASK_NBITS);
 
 	if (cmd->base.autoneg == AUTONEG_ENABLE)
-		pdata->phy.advertising |= ADVERTISED_Autoneg;
+		XGBE_SET_ADV(lks, Autoneg);
 	else
-		pdata->phy.advertising &= ~ADVERTISED_Autoneg;
+		XGBE_CLR_ADV(lks, Autoneg);
 
 	if (netif_running(netdev))
 		ret = pdata->phy_if.phy_config_aneg(pdata);
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-main.c b/drivers/net/ethernet/amd/xgbe/xgbe-main.c
index 500147d9e3c8..d91fa595be98 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-main.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-main.c
@@ -120,6 +120,7 @@
 #include <linux/netdevice.h>
 #include <linux/etherdevice.h>
 #include <linux/io.h>
+#include <linux/notifier.h>
 
 #include "xgbe.h"
 #include "xgbe-common.h"
@@ -192,6 +193,7 @@ struct xgbe_prv_data *xgbe_alloc_pdata(struct device *dev)
 	mutex_init(&pdata->i2c_mutex);
 	init_completion(&pdata->i2c_complete);
 	init_completion(&pdata->mdio_complete);
+	INIT_LIST_HEAD(&pdata->vxlan_ports);
 
 	pdata->msg_enable = netif_msg_init(debug, default_msg_level);
 
@@ -373,6 +375,28 @@ int xgbe_config_netdev(struct xgbe_prv_data *pdata)
 	if (pdata->hw_feat.rss)
 		netdev->hw_features |= NETIF_F_RXHASH;
 
+	if (pdata->hw_feat.vxn) {
+		netdev->hw_enc_features = NETIF_F_SG |
+					  NETIF_F_IP_CSUM |
+					  NETIF_F_IPV6_CSUM |
+					  NETIF_F_RXCSUM |
+					  NETIF_F_TSO |
+					  NETIF_F_TSO6 |
+					  NETIF_F_GRO |
+					  NETIF_F_GSO_UDP_TUNNEL |
+					  NETIF_F_GSO_UDP_TUNNEL_CSUM |
+					  NETIF_F_RX_UDP_TUNNEL_PORT;
+
+		netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL |
+				       NETIF_F_GSO_UDP_TUNNEL_CSUM |
+				       NETIF_F_RX_UDP_TUNNEL_PORT;
+
+		pdata->vxlan_offloads_set = 1;
+		pdata->vxlan_features = NETIF_F_GSO_UDP_TUNNEL |
+					NETIF_F_GSO_UDP_TUNNEL_CSUM |
+					NETIF_F_RX_UDP_TUNNEL_PORT;
+	}
+
 	netdev->vlan_features |= NETIF_F_SG |
 				 NETIF_F_IP_CSUM |
 				 NETIF_F_IPV6_CSUM |
@@ -399,35 +423,6 @@ int xgbe_config_netdev(struct xgbe_prv_data *pdata)
 		return ret;
 	}
 
-	/* Create the PHY/ANEG name based on netdev name */
-	snprintf(pdata->an_name, sizeof(pdata->an_name) - 1, "%s-pcs",
-		 netdev_name(netdev));
-
-	/* Create the ECC name based on netdev name */
-	snprintf(pdata->ecc_name, sizeof(pdata->ecc_name) - 1, "%s-ecc",
-		 netdev_name(netdev));
-
-	/* Create the I2C name based on netdev name */
-	snprintf(pdata->i2c_name, sizeof(pdata->i2c_name) - 1, "%s-i2c",
-		 netdev_name(netdev));
-
-	/* Create workqueues */
-	pdata->dev_workqueue =
-		create_singlethread_workqueue(netdev_name(netdev));
-	if (!pdata->dev_workqueue) {
-		netdev_err(netdev, "device workqueue creation failed\n");
-		ret = -ENOMEM;
-		goto err_netdev;
-	}
-
-	pdata->an_workqueue =
-		create_singlethread_workqueue(pdata->an_name);
-	if (!pdata->an_workqueue) {
-		netdev_err(netdev, "phy workqueue creation failed\n");
-		ret = -ENOMEM;
-		goto err_wq;
-	}
-
 	if (IS_REACHABLE(CONFIG_PTP_1588_CLOCK))
 		xgbe_ptp_register(pdata);
 
@@ -439,14 +434,6 @@ int xgbe_config_netdev(struct xgbe_prv_data *pdata)
 		  pdata->rx_ring_count);
 
 	return 0;
-
-err_wq:
-	destroy_workqueue(pdata->dev_workqueue);
-
-err_netdev:
-	unregister_netdev(netdev);
-
-	return ret;
 }
 
 void xgbe_deconfig_netdev(struct xgbe_prv_data *pdata)
@@ -458,21 +445,45 @@ void xgbe_deconfig_netdev(struct xgbe_prv_data *pdata)
 	if (IS_REACHABLE(CONFIG_PTP_1588_CLOCK))
 		xgbe_ptp_unregister(pdata);
 
+	unregister_netdev(netdev);
+
 	pdata->phy_if.phy_exit(pdata);
+}
 
-	flush_workqueue(pdata->an_workqueue);
-	destroy_workqueue(pdata->an_workqueue);
+static int xgbe_netdev_event(struct notifier_block *nb, unsigned long event,
+			     void *data)
+{
+	struct net_device *netdev = netdev_notifier_info_to_dev(data);
+	struct xgbe_prv_data *pdata = netdev_priv(netdev);
 
-	flush_workqueue(pdata->dev_workqueue);
-	destroy_workqueue(pdata->dev_workqueue);
+	if (netdev->netdev_ops != xgbe_get_netdev_ops())
+		goto out;
 
-	unregister_netdev(netdev);
+	switch (event) {
+	case NETDEV_CHANGENAME:
+		xgbe_debugfs_rename(pdata);
+		break;
+
+	default:
+		break;
+	}
+
+out:
+	return NOTIFY_DONE;
 }
 
+static struct notifier_block xgbe_netdev_notifier = {
+	.notifier_call = xgbe_netdev_event,
+};
+
 static int __init xgbe_mod_init(void)
 {
 	int ret;
 
+	ret = register_netdevice_notifier(&xgbe_netdev_notifier);
+	if (ret)
+		return ret;
+
 	ret = xgbe_platform_init();
 	if (ret)
 		return ret;
@@ -489,6 +500,8 @@ static void __exit xgbe_mod_exit(void)
 	xgbe_pci_exit();
 
 	xgbe_platform_exit();
+
+	unregister_netdevice_notifier(&xgbe_netdev_notifier);
 }
 
 module_init(xgbe_mod_init);
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c b/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c
index 80684914dd8a..072b9f664597 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c
@@ -615,12 +615,14 @@ static enum xgbe_an xgbe_an73_page_received(struct xgbe_prv_data *pdata)
 
 static enum xgbe_an xgbe_an73_incompat_link(struct xgbe_prv_data *pdata)
 {
+	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
+
 	/* Be sure we aren't looping trying to negotiate */
 	if (xgbe_in_kr_mode(pdata)) {
 		pdata->kr_state = XGBE_RX_ERROR;
 
-		if (!(pdata->phy.advertising & ADVERTISED_1000baseKX_Full) &&
-		    !(pdata->phy.advertising & ADVERTISED_2500baseX_Full))
+		if (!XGBE_ADV(lks, 1000baseKX_Full) &&
+		    !XGBE_ADV(lks, 2500baseX_Full))
 			return XGBE_AN_NO_LINK;
 
 		if (pdata->kx_state != XGBE_RX_BPA)
@@ -628,7 +630,7 @@ static enum xgbe_an xgbe_an73_incompat_link(struct xgbe_prv_data *pdata)
 	} else {
 		pdata->kx_state = XGBE_RX_ERROR;
 
-		if (!(pdata->phy.advertising & ADVERTISED_10000baseKR_Full))
+		if (!XGBE_ADV(lks, 10000baseKR_Full))
 			return XGBE_AN_NO_LINK;
 
 		if (pdata->kr_state != XGBE_RX_BPA)
@@ -944,18 +946,19 @@ static void xgbe_an_state_machine(struct work_struct *work)
 
 static void xgbe_an37_init(struct xgbe_prv_data *pdata)
 {
-	unsigned int advertising, reg;
+	struct ethtool_link_ksettings lks;
+	unsigned int reg;
 
-	advertising = pdata->phy_if.phy_impl.an_advertising(pdata);
+	pdata->phy_if.phy_impl.an_advertising(pdata, &lks);
 
 	/* Set up Advertisement register */
 	reg = XMDIO_READ(pdata, MDIO_MMD_VEND2, MDIO_VEND2_AN_ADVERTISE);
-	if (advertising & ADVERTISED_Pause)
+	if (XGBE_ADV(&lks, Pause))
 		reg |= 0x100;
 	else
 		reg &= ~0x100;
 
-	if (advertising & ADVERTISED_Asym_Pause)
+	if (XGBE_ADV(&lks, Asym_Pause))
 		reg |= 0x80;
 	else
 		reg &= ~0x80;
@@ -982,6 +985,8 @@ static void xgbe_an37_init(struct xgbe_prv_data *pdata)
 		break;
 	}
 
+	reg |= XGBE_AN_CL37_MII_CTRL_8BIT;
+
 	XMDIO_WRITE(pdata, MDIO_MMD_VEND2, MDIO_VEND2_AN_CTRL, reg);
 
 	netif_dbg(pdata, link, pdata->netdev, "CL37 AN (%s) initialized\n",
@@ -990,13 +995,14 @@ static void xgbe_an37_init(struct xgbe_prv_data *pdata)
 
 static void xgbe_an73_init(struct xgbe_prv_data *pdata)
 {
-	unsigned int advertising, reg;
+	struct ethtool_link_ksettings lks;
+	unsigned int reg;
 
-	advertising = pdata->phy_if.phy_impl.an_advertising(pdata);
+	pdata->phy_if.phy_impl.an_advertising(pdata, &lks);
 
 	/* Set up Advertisement register 3 first */
 	reg = XMDIO_READ(pdata, MDIO_MMD_AN, MDIO_AN_ADVERTISE + 2);
-	if (advertising & ADVERTISED_10000baseR_FEC)
+	if (XGBE_ADV(&lks, 10000baseR_FEC))
 		reg |= 0xc000;
 	else
 		reg &= ~0xc000;
@@ -1005,13 +1011,13 @@ static void xgbe_an73_init(struct xgbe_prv_data *pdata)
 
 	/* Set up Advertisement register 2 next */
 	reg = XMDIO_READ(pdata, MDIO_MMD_AN, MDIO_AN_ADVERTISE + 1);
-	if (advertising & ADVERTISED_10000baseKR_Full)
+	if (XGBE_ADV(&lks, 10000baseKR_Full))
 		reg |= 0x80;
 	else
 		reg &= ~0x80;
 
-	if ((advertising & ADVERTISED_1000baseKX_Full) ||
-	    (advertising & ADVERTISED_2500baseX_Full))
+	if (XGBE_ADV(&lks, 1000baseKX_Full) ||
+	    XGBE_ADV(&lks, 2500baseX_Full))
 		reg |= 0x20;
 	else
 		reg &= ~0x20;
@@ -1020,12 +1026,12 @@ static void xgbe_an73_init(struct xgbe_prv_data *pdata)
 
 	/* Set up Advertisement register 1 last */
 	reg = XMDIO_READ(pdata, MDIO_MMD_AN, MDIO_AN_ADVERTISE);
-	if (advertising & ADVERTISED_Pause)
+	if (XGBE_ADV(&lks, Pause))
 		reg |= 0x400;
 	else
 		reg &= ~0x400;
 
-	if (advertising & ADVERTISED_Asym_Pause)
+	if (XGBE_ADV(&lks, Asym_Pause))
 		reg |= 0x800;
 	else
 		reg &= ~0x800;
@@ -1281,9 +1287,10 @@ static enum xgbe_mode xgbe_phy_status_aneg(struct xgbe_prv_data *pdata)
 
 static void xgbe_phy_status_result(struct xgbe_prv_data *pdata)
 {
+	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
 	enum xgbe_mode mode;
 
-	pdata->phy.lp_advertising = 0;
+	XGBE_ZERO_LP_ADV(lks);
 
 	if ((pdata->phy.autoneg != AUTONEG_ENABLE) || pdata->parallel_detect)
 		mode = xgbe_cur_mode(pdata);
@@ -1513,17 +1520,21 @@ static void xgbe_dump_phy_registers(struct xgbe_prv_data *pdata)
 
 static int xgbe_phy_best_advertised_speed(struct xgbe_prv_data *pdata)
 {
-	if (pdata->phy.advertising & ADVERTISED_10000baseKR_Full)
+	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
+
+	if (XGBE_ADV(lks, 10000baseKR_Full))
 		return SPEED_10000;
-	else if (pdata->phy.advertising & ADVERTISED_10000baseT_Full)
+	else if (XGBE_ADV(lks, 10000baseT_Full))
 		return SPEED_10000;
-	else if (pdata->phy.advertising & ADVERTISED_2500baseX_Full)
+	else if (XGBE_ADV(lks, 2500baseX_Full))
 		return SPEED_2500;
-	else if (pdata->phy.advertising & ADVERTISED_1000baseKX_Full)
+	else if (XGBE_ADV(lks, 2500baseT_Full))
+		return SPEED_2500;
+	else if (XGBE_ADV(lks, 1000baseKX_Full))
 		return SPEED_1000;
-	else if (pdata->phy.advertising & ADVERTISED_1000baseT_Full)
+	else if (XGBE_ADV(lks, 1000baseT_Full))
 		return SPEED_1000;
-	else if (pdata->phy.advertising & ADVERTISED_100baseT_Full)
+	else if (XGBE_ADV(lks, 100baseT_Full))
 		return SPEED_100;
 
 	return SPEED_UNKNOWN;
@@ -1531,13 +1542,12 @@ static int xgbe_phy_best_advertised_speed(struct xgbe_prv_data *pdata)
 
 static void xgbe_phy_exit(struct xgbe_prv_data *pdata)
 {
-	xgbe_phy_stop(pdata);
-
 	pdata->phy_if.phy_impl.exit(pdata);
 }
 
 static int xgbe_phy_init(struct xgbe_prv_data *pdata)
 {
+	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
 	int ret;
 
 	mutex_init(&pdata->an_mutex);
@@ -1555,11 +1565,13 @@ static int xgbe_phy_init(struct xgbe_prv_data *pdata)
 	ret = pdata->phy_if.phy_impl.init(pdata);
 	if (ret)
 		return ret;
-	pdata->phy.advertising = pdata->phy.supported;
+
+	/* Copy supported link modes to advertising link modes */
+	XGBE_LM_COPY(lks, advertising, lks, supported);
 
 	pdata->phy.address = 0;
 
-	if (pdata->phy.advertising & ADVERTISED_Autoneg) {
+	if (XGBE_ADV(lks, Autoneg)) {
 		pdata->phy.autoneg = AUTONEG_ENABLE;
 		pdata->phy.speed = SPEED_UNKNOWN;
 		pdata->phy.duplex = DUPLEX_UNKNOWN;
@@ -1576,16 +1588,21 @@ static int xgbe_phy_init(struct xgbe_prv_data *pdata)
 	pdata->phy.rx_pause = pdata->rx_pause;
 
 	/* Fix up Flow Control advertising */
-	pdata->phy.advertising &= ~ADVERTISED_Pause;
-	pdata->phy.advertising &= ~ADVERTISED_Asym_Pause;
+	XGBE_CLR_ADV(lks, Pause);
+	XGBE_CLR_ADV(lks, Asym_Pause);
 
 	if (pdata->rx_pause) {
-		pdata->phy.advertising |= ADVERTISED_Pause;
-		pdata->phy.advertising |= ADVERTISED_Asym_Pause;
+		XGBE_SET_ADV(lks, Pause);
+		XGBE_SET_ADV(lks, Asym_Pause);
 	}
 
-	if (pdata->tx_pause)
-		pdata->phy.advertising ^= ADVERTISED_Asym_Pause;
+	if (pdata->tx_pause) {
+		/* Equivalent to XOR of Asym_Pause */
+		if (XGBE_ADV(lks, Asym_Pause))
+			XGBE_CLR_ADV(lks, Asym_Pause);
+		else
+			XGBE_SET_ADV(lks, Asym_Pause);
+	}
 
 	if (netif_msg_drv(pdata))
 		xgbe_dump_phy_registers(pdata);
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-pci.c b/drivers/net/ethernet/amd/xgbe/xgbe-pci.c
index 1e56ad7bd9a5..3e5833cf1fab 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-pci.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-pci.c
@@ -292,6 +292,10 @@ static int xgbe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	pdata->xpcs_window_size = 1 << (pdata->xpcs_window_size + 7);
 	pdata->xpcs_window_mask = pdata->xpcs_window_size - 1;
 	if (netif_msg_probe(pdata)) {
+		dev_dbg(dev, "xpcs window def  = %#010x\n",
+			pdata->xpcs_window_def_reg);
+		dev_dbg(dev, "xpcs window sel  = %#010x\n",
+			pdata->xpcs_window_sel_reg);
 		dev_dbg(dev, "xpcs window      = %#010x\n",
 			pdata->xpcs_window);
 		dev_dbg(dev, "xpcs window size = %#010x\n",
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v1.c b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v1.c
index c75edcac5e0a..d16eae415f72 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v1.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v1.c
@@ -231,20 +231,21 @@ static void xgbe_phy_kr_training_post(struct xgbe_prv_data *pdata)
 
 static enum xgbe_mode xgbe_phy_an_outcome(struct xgbe_prv_data *pdata)
 {
+	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
 	struct xgbe_phy_data *phy_data = pdata->phy_data;
 	enum xgbe_mode mode;
 	unsigned int ad_reg, lp_reg;
 
-	pdata->phy.lp_advertising |= ADVERTISED_Autoneg;
-	pdata->phy.lp_advertising |= ADVERTISED_Backplane;
+	XGBE_SET_LP_ADV(lks, Autoneg);
+	XGBE_SET_LP_ADV(lks, Backplane);
 
 	/* Compare Advertisement and Link Partner register 1 */
 	ad_reg = XMDIO_READ(pdata, MDIO_MMD_AN, MDIO_AN_ADVERTISE);
 	lp_reg = XMDIO_READ(pdata, MDIO_MMD_AN, MDIO_AN_LPA);
 	if (lp_reg & 0x400)
-		pdata->phy.lp_advertising |= ADVERTISED_Pause;
+		XGBE_SET_LP_ADV(lks, Pause);
 	if (lp_reg & 0x800)
-		pdata->phy.lp_advertising |= ADVERTISED_Asym_Pause;
+		XGBE_SET_LP_ADV(lks, Asym_Pause);
 
 	if (pdata->phy.pause_autoneg) {
 		/* Set flow control based on auto-negotiation result */
@@ -266,12 +267,12 @@ static enum xgbe_mode xgbe_phy_an_outcome(struct xgbe_prv_data *pdata)
 	ad_reg = XMDIO_READ(pdata, MDIO_MMD_AN, MDIO_AN_ADVERTISE + 1);
 	lp_reg = XMDIO_READ(pdata, MDIO_MMD_AN, MDIO_AN_LPA + 1);
 	if (lp_reg & 0x80)
-		pdata->phy.lp_advertising |= ADVERTISED_10000baseKR_Full;
+		XGBE_SET_LP_ADV(lks, 10000baseKR_Full);
 	if (lp_reg & 0x20) {
 		if (phy_data->speed_set == XGBE_SPEEDSET_2500_10000)
-			pdata->phy.lp_advertising |= ADVERTISED_2500baseX_Full;
+			XGBE_SET_LP_ADV(lks, 2500baseX_Full);
 		else
-			pdata->phy.lp_advertising |= ADVERTISED_1000baseKX_Full;
+			XGBE_SET_LP_ADV(lks, 1000baseKX_Full);
 	}
 
 	ad_reg &= lp_reg;
@@ -290,14 +291,17 @@ static enum xgbe_mode xgbe_phy_an_outcome(struct xgbe_prv_data *pdata)
 	ad_reg = XMDIO_READ(pdata, MDIO_MMD_AN, MDIO_AN_ADVERTISE + 2);
 	lp_reg = XMDIO_READ(pdata, MDIO_MMD_AN, MDIO_AN_LPA + 2);
 	if (lp_reg & 0xc000)
-		pdata->phy.lp_advertising |= ADVERTISED_10000baseR_FEC;
+		XGBE_SET_LP_ADV(lks, 10000baseR_FEC);
 
 	return mode;
 }
 
-static unsigned int xgbe_phy_an_advertising(struct xgbe_prv_data *pdata)
+static void xgbe_phy_an_advertising(struct xgbe_prv_data *pdata,
+				    struct ethtool_link_ksettings *dlks)
 {
-	return pdata->phy.advertising;
+	struct ethtool_link_ksettings *slks = &pdata->phy.lks;
+
+	XGBE_LM_COPY(dlks, advertising, slks, advertising);
 }
 
 static int xgbe_phy_an_config(struct xgbe_prv_data *pdata)
@@ -565,11 +569,10 @@ static void xgbe_phy_set_mode(struct xgbe_prv_data *pdata, enum xgbe_mode mode)
 }
 
 static bool xgbe_phy_check_mode(struct xgbe_prv_data *pdata,
-				enum xgbe_mode mode, u32 advert)
+				enum xgbe_mode mode, bool advert)
 {
 	if (pdata->phy.autoneg == AUTONEG_ENABLE) {
-		if (pdata->phy.advertising & advert)
-			return true;
+		return advert;
 	} else {
 		enum xgbe_mode cur_mode;
 
@@ -583,16 +586,18 @@ static bool xgbe_phy_check_mode(struct xgbe_prv_data *pdata,
 
 static bool xgbe_phy_use_mode(struct xgbe_prv_data *pdata, enum xgbe_mode mode)
 {
+	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
+
 	switch (mode) {
 	case XGBE_MODE_KX_1000:
 		return xgbe_phy_check_mode(pdata, mode,
-					   ADVERTISED_1000baseKX_Full);
+					   XGBE_ADV(lks, 1000baseKX_Full));
 	case XGBE_MODE_KX_2500:
 		return xgbe_phy_check_mode(pdata, mode,
-					   ADVERTISED_2500baseX_Full);
+					   XGBE_ADV(lks, 2500baseX_Full));
 	case XGBE_MODE_KR:
 		return xgbe_phy_check_mode(pdata, mode,
-					   ADVERTISED_10000baseKR_Full);
+					   XGBE_ADV(lks, 10000baseKR_Full));
 	default:
 		return false;
 	}
@@ -672,6 +677,7 @@ static void xgbe_phy_exit(struct xgbe_prv_data *pdata)
 
 static int xgbe_phy_init(struct xgbe_prv_data *pdata)
 {
+	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
 	struct xgbe_phy_data *phy_data;
 	int ret;
 
@@ -790,21 +796,23 @@ static int xgbe_phy_init(struct xgbe_prv_data *pdata)
 	}
 
 	/* Initialize supported features */
-	pdata->phy.supported = SUPPORTED_Autoneg;
-	pdata->phy.supported |= SUPPORTED_Pause | SUPPORTED_Asym_Pause;
-	pdata->phy.supported |= SUPPORTED_Backplane;
-	pdata->phy.supported |= SUPPORTED_10000baseKR_Full;
+	XGBE_ZERO_SUP(lks);
+	XGBE_SET_SUP(lks, Autoneg);
+	XGBE_SET_SUP(lks, Pause);
+	XGBE_SET_SUP(lks, Asym_Pause);
+	XGBE_SET_SUP(lks, Backplane);
+	XGBE_SET_SUP(lks, 10000baseKR_Full);
 	switch (phy_data->speed_set) {
 	case XGBE_SPEEDSET_1000_10000:
-		pdata->phy.supported |= SUPPORTED_1000baseKX_Full;
+		XGBE_SET_SUP(lks, 1000baseKX_Full);
 		break;
 	case XGBE_SPEEDSET_2500_10000:
-		pdata->phy.supported |= SUPPORTED_2500baseX_Full;
+		XGBE_SET_SUP(lks, 2500baseX_Full);
 		break;
 	}
 
 	if (pdata->fec_ability & MDIO_PMA_10GBR_FECABLE_ABLE)
-		pdata->phy.supported |= SUPPORTED_10000baseR_FEC;
+		XGBE_SET_SUP(lks, 10000baseR_FEC);
 
 	pdata->phy_data = phy_data;
 
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c
index 04b5c149caca..3304a291aa96 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c
@@ -709,18 +709,13 @@ static int xgbe_phy_mii_read(struct mii_bus *mii, int addr, int reg)
 
 static void xgbe_phy_sfp_phy_settings(struct xgbe_prv_data *pdata)
 {
+	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
 	struct xgbe_phy_data *phy_data = pdata->phy_data;
 
 	if (!phy_data->sfp_mod_absent && !phy_data->sfp_changed)
 		return;
 
-	pdata->phy.supported &= ~SUPPORTED_Autoneg;
-	pdata->phy.supported &= ~(SUPPORTED_Pause | SUPPORTED_Asym_Pause);
-	pdata->phy.supported &= ~SUPPORTED_TP;
-	pdata->phy.supported &= ~SUPPORTED_FIBRE;
-	pdata->phy.supported &= ~SUPPORTED_100baseT_Full;
-	pdata->phy.supported &= ~SUPPORTED_1000baseT_Full;
-	pdata->phy.supported &= ~SUPPORTED_10000baseT_Full;
+	XGBE_ZERO_SUP(lks);
 
 	if (phy_data->sfp_mod_absent) {
 		pdata->phy.speed = SPEED_UNKNOWN;
@@ -728,18 +723,13 @@ static void xgbe_phy_sfp_phy_settings(struct xgbe_prv_data *pdata)
 		pdata->phy.autoneg = AUTONEG_ENABLE;
 		pdata->phy.pause_autoneg = AUTONEG_ENABLE;
 
-		pdata->phy.supported |= SUPPORTED_Autoneg;
-		pdata->phy.supported |= SUPPORTED_Pause | SUPPORTED_Asym_Pause;
-		pdata->phy.supported |= SUPPORTED_TP;
-		pdata->phy.supported |= SUPPORTED_FIBRE;
-		if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_100)
-			pdata->phy.supported |= SUPPORTED_100baseT_Full;
-		if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_1000)
-			pdata->phy.supported |= SUPPORTED_1000baseT_Full;
-		if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_10000)
-			pdata->phy.supported |= SUPPORTED_10000baseT_Full;
+		XGBE_SET_SUP(lks, Autoneg);
+		XGBE_SET_SUP(lks, Pause);
+		XGBE_SET_SUP(lks, Asym_Pause);
+		XGBE_SET_SUP(lks, TP);
+		XGBE_SET_SUP(lks, FIBRE);
 
-		pdata->phy.advertising = pdata->phy.supported;
+		XGBE_LM_COPY(lks, advertising, lks, supported);
 
 		return;
 	}
@@ -753,8 +743,18 @@ static void xgbe_phy_sfp_phy_settings(struct xgbe_prv_data *pdata)
 		pdata->phy.duplex = DUPLEX_UNKNOWN;
 		pdata->phy.autoneg = AUTONEG_ENABLE;
 		pdata->phy.pause_autoneg = AUTONEG_ENABLE;
-		pdata->phy.supported |= SUPPORTED_Autoneg;
-		pdata->phy.supported |= SUPPORTED_Pause | SUPPORTED_Asym_Pause;
+		XGBE_SET_SUP(lks, Autoneg);
+		XGBE_SET_SUP(lks, Pause);
+		XGBE_SET_SUP(lks, Asym_Pause);
+		if (phy_data->sfp_base == XGBE_SFP_BASE_1000_T) {
+			if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_100)
+				XGBE_SET_SUP(lks, 100baseT_Full);
+			if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_1000)
+				XGBE_SET_SUP(lks, 1000baseT_Full);
+		} else {
+			if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_1000)
+				XGBE_SET_SUP(lks, 1000baseX_Full);
+		}
 		break;
 	case XGBE_SFP_BASE_10000_SR:
 	case XGBE_SFP_BASE_10000_LR:
@@ -765,6 +765,27 @@ static void xgbe_phy_sfp_phy_settings(struct xgbe_prv_data *pdata)
 		pdata->phy.duplex = DUPLEX_FULL;
 		pdata->phy.autoneg = AUTONEG_DISABLE;
 		pdata->phy.pause_autoneg = AUTONEG_DISABLE;
+		if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_10000) {
+			switch (phy_data->sfp_base) {
+			case XGBE_SFP_BASE_10000_SR:
+				XGBE_SET_SUP(lks, 10000baseSR_Full);
+				break;
+			case XGBE_SFP_BASE_10000_LR:
+				XGBE_SET_SUP(lks, 10000baseLR_Full);
+				break;
+			case XGBE_SFP_BASE_10000_LRM:
+				XGBE_SET_SUP(lks, 10000baseLRM_Full);
+				break;
+			case XGBE_SFP_BASE_10000_ER:
+				XGBE_SET_SUP(lks, 10000baseER_Full);
+				break;
+			case XGBE_SFP_BASE_10000_CR:
+				XGBE_SET_SUP(lks, 10000baseCR_Full);
+				break;
+			default:
+				break;
+			}
+		}
 		break;
 	default:
 		pdata->phy.speed = SPEED_UNKNOWN;
@@ -778,38 +799,14 @@ static void xgbe_phy_sfp_phy_settings(struct xgbe_prv_data *pdata)
 	case XGBE_SFP_BASE_1000_T:
 	case XGBE_SFP_BASE_1000_CX:
 	case XGBE_SFP_BASE_10000_CR:
-		pdata->phy.supported |= SUPPORTED_TP;
+		XGBE_SET_SUP(lks, TP);
 		break;
 	default:
-		pdata->phy.supported |= SUPPORTED_FIBRE;
-	}
-
-	switch (phy_data->sfp_speed) {
-	case XGBE_SFP_SPEED_100_1000:
-		if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_100)
-			pdata->phy.supported |= SUPPORTED_100baseT_Full;
-		if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_1000)
-			pdata->phy.supported |= SUPPORTED_1000baseT_Full;
-		break;
-	case XGBE_SFP_SPEED_1000:
-		if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_1000)
-			pdata->phy.supported |= SUPPORTED_1000baseT_Full;
+		XGBE_SET_SUP(lks, FIBRE);
 		break;
-	case XGBE_SFP_SPEED_10000:
-		if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_10000)
-			pdata->phy.supported |= SUPPORTED_10000baseT_Full;
-		break;
-	default:
-		/* Choose the fastest supported speed */
-		if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_10000)
-			pdata->phy.supported |= SUPPORTED_10000baseT_Full;
-		else if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_1000)
-			pdata->phy.supported |= SUPPORTED_1000baseT_Full;
-		else if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_100)
-			pdata->phy.supported |= SUPPORTED_100baseT_Full;
 	}
 
-	pdata->phy.advertising = pdata->phy.supported;
+	XGBE_LM_COPY(lks, advertising, lks, supported);
 }
 
 static bool xgbe_phy_sfp_bit_rate(struct xgbe_sfp_eeprom *sfp_eeprom,
@@ -886,8 +883,10 @@ static void xgbe_phy_external_phy_quirks(struct xgbe_prv_data *pdata)
 
 static int xgbe_phy_find_phy_device(struct xgbe_prv_data *pdata)
 {
+	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
 	struct xgbe_phy_data *phy_data = pdata->phy_data;
 	struct phy_device *phydev;
+	u32 advertising;
 	int ret;
 
 	/* If we already have a PHY, just return */
@@ -943,7 +942,10 @@ static int xgbe_phy_find_phy_device(struct xgbe_prv_data *pdata)
 	phy_data->phydev = phydev;
 
 	xgbe_phy_external_phy_quirks(pdata);
-	phydev->advertising &= pdata->phy.advertising;
+
+	ethtool_convert_link_mode_to_legacy_u32(&advertising,
+						lks->link_modes.advertising);
+	phydev->advertising &= advertising;
 
 	phy_start_aneg(phy_data->phydev);
 
@@ -1277,6 +1279,7 @@ put:
 
 static void xgbe_phy_phydev_flowctrl(struct xgbe_prv_data *pdata)
 {
+	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
 	struct xgbe_phy_data *phy_data = pdata->phy_data;
 	u16 lcl_adv = 0, rmt_adv = 0;
 	u8 fc;
@@ -1293,11 +1296,11 @@ static void xgbe_phy_phydev_flowctrl(struct xgbe_prv_data *pdata)
 		lcl_adv |= ADVERTISE_PAUSE_ASYM;
 
 	if (phy_data->phydev->pause) {
-		pdata->phy.lp_advertising |= ADVERTISED_Pause;
+		XGBE_SET_LP_ADV(lks, Pause);
 		rmt_adv |= LPA_PAUSE_CAP;
 	}
 	if (phy_data->phydev->asym_pause) {
-		pdata->phy.lp_advertising |= ADVERTISED_Asym_Pause;
+		XGBE_SET_LP_ADV(lks, Asym_Pause);
 		rmt_adv |= LPA_PAUSE_ASYM;
 	}
 
@@ -1310,10 +1313,11 @@ static void xgbe_phy_phydev_flowctrl(struct xgbe_prv_data *pdata)
 
 static enum xgbe_mode xgbe_phy_an37_sgmii_outcome(struct xgbe_prv_data *pdata)
 {
+	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
 	enum xgbe_mode mode;
 
-	pdata->phy.lp_advertising |= ADVERTISED_Autoneg;
-	pdata->phy.lp_advertising |= ADVERTISED_TP;
+	XGBE_SET_LP_ADV(lks, Autoneg);
+	XGBE_SET_LP_ADV(lks, TP);
 
 	/* Use external PHY to determine flow control */
 	if (pdata->phy.pause_autoneg)
@@ -1322,21 +1326,21 @@ static enum xgbe_mode xgbe_phy_an37_sgmii_outcome(struct xgbe_prv_data *pdata)
 	switch (pdata->an_status & XGBE_SGMII_AN_LINK_SPEED) {
 	case XGBE_SGMII_AN_LINK_SPEED_100:
 		if (pdata->an_status & XGBE_SGMII_AN_LINK_DUPLEX) {
-			pdata->phy.lp_advertising |= ADVERTISED_100baseT_Full;
+			XGBE_SET_LP_ADV(lks, 100baseT_Full);
 			mode = XGBE_MODE_SGMII_100;
 		} else {
 			/* Half-duplex not supported */
-			pdata->phy.lp_advertising |= ADVERTISED_100baseT_Half;
+			XGBE_SET_LP_ADV(lks, 100baseT_Half);
 			mode = XGBE_MODE_UNKNOWN;
 		}
 		break;
 	case XGBE_SGMII_AN_LINK_SPEED_1000:
 		if (pdata->an_status & XGBE_SGMII_AN_LINK_DUPLEX) {
-			pdata->phy.lp_advertising |= ADVERTISED_1000baseT_Full;
+			XGBE_SET_LP_ADV(lks, 1000baseT_Full);
 			mode = XGBE_MODE_SGMII_1000;
 		} else {
 			/* Half-duplex not supported */
-			pdata->phy.lp_advertising |= ADVERTISED_1000baseT_Half;
+			XGBE_SET_LP_ADV(lks, 1000baseT_Half);
 			mode = XGBE_MODE_UNKNOWN;
 		}
 		break;
@@ -1349,19 +1353,20 @@ static enum xgbe_mode xgbe_phy_an37_sgmii_outcome(struct xgbe_prv_data *pdata)
 
 static enum xgbe_mode xgbe_phy_an37_outcome(struct xgbe_prv_data *pdata)
 {
+	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
 	enum xgbe_mode mode;
 	unsigned int ad_reg, lp_reg;
 
-	pdata->phy.lp_advertising |= ADVERTISED_Autoneg;
-	pdata->phy.lp_advertising |= ADVERTISED_FIBRE;
+	XGBE_SET_LP_ADV(lks, Autoneg);
+	XGBE_SET_LP_ADV(lks, FIBRE);
 
 	/* Compare Advertisement and Link Partner register */
 	ad_reg = XMDIO_READ(pdata, MDIO_MMD_VEND2, MDIO_VEND2_AN_ADVERTISE);
 	lp_reg = XMDIO_READ(pdata, MDIO_MMD_VEND2, MDIO_VEND2_AN_LP_ABILITY);
 	if (lp_reg & 0x100)
-		pdata->phy.lp_advertising |= ADVERTISED_Pause;
+		XGBE_SET_LP_ADV(lks, Pause);
 	if (lp_reg & 0x80)
-		pdata->phy.lp_advertising |= ADVERTISED_Asym_Pause;
+		XGBE_SET_LP_ADV(lks, Asym_Pause);
 
 	if (pdata->phy.pause_autoneg) {
 		/* Set flow control based on auto-negotiation result */
@@ -1379,10 +1384,8 @@ static enum xgbe_mode xgbe_phy_an37_outcome(struct xgbe_prv_data *pdata)
 		}
 	}
 
-	if (lp_reg & 0x40)
-		pdata->phy.lp_advertising |= ADVERTISED_1000baseT_Half;
 	if (lp_reg & 0x20)
-		pdata->phy.lp_advertising |= ADVERTISED_1000baseT_Full;
+		XGBE_SET_LP_ADV(lks, 1000baseX_Full);
 
 	/* Half duplex is not supported */
 	ad_reg &= lp_reg;
@@ -1393,12 +1396,13 @@ static enum xgbe_mode xgbe_phy_an37_outcome(struct xgbe_prv_data *pdata)
 
 static enum xgbe_mode xgbe_phy_an73_redrv_outcome(struct xgbe_prv_data *pdata)
 {
+	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
 	struct xgbe_phy_data *phy_data = pdata->phy_data;
 	enum xgbe_mode mode;
 	unsigned int ad_reg, lp_reg;
 
-	pdata->phy.lp_advertising |= ADVERTISED_Autoneg;
-	pdata->phy.lp_advertising |= ADVERTISED_Backplane;
+	XGBE_SET_LP_ADV(lks, Autoneg);
+	XGBE_SET_LP_ADV(lks, Backplane);
 
 	/* Use external PHY to determine flow control */
 	if (pdata->phy.pause_autoneg)
@@ -1408,9 +1412,9 @@ static enum xgbe_mode xgbe_phy_an73_redrv_outcome(struct xgbe_prv_data *pdata)
 	ad_reg = XMDIO_READ(pdata, MDIO_MMD_AN, MDIO_AN_ADVERTISE + 1);
 	lp_reg = XMDIO_READ(pdata, MDIO_MMD_AN, MDIO_AN_LPA + 1);
 	if (lp_reg & 0x80)
-		pdata->phy.lp_advertising |= ADVERTISED_10000baseKR_Full;
+		XGBE_SET_LP_ADV(lks, 10000baseKR_Full);
 	if (lp_reg & 0x20)
-		pdata->phy.lp_advertising |= ADVERTISED_1000baseKX_Full;
+		XGBE_SET_LP_ADV(lks, 1000baseKX_Full);
 
 	ad_reg &= lp_reg;
 	if (ad_reg & 0x80) {
@@ -1463,26 +1467,27 @@ static enum xgbe_mode xgbe_phy_an73_redrv_outcome(struct xgbe_prv_data *pdata)
 	ad_reg = XMDIO_READ(pdata, MDIO_MMD_AN, MDIO_AN_ADVERTISE + 2);
 	lp_reg = XMDIO_READ(pdata, MDIO_MMD_AN, MDIO_AN_LPA + 2);
 	if (lp_reg & 0xc000)
-		pdata->phy.lp_advertising |= ADVERTISED_10000baseR_FEC;
+		XGBE_SET_LP_ADV(lks, 10000baseR_FEC);
 
 	return mode;
 }
 
 static enum xgbe_mode xgbe_phy_an73_outcome(struct xgbe_prv_data *pdata)
 {
+	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
 	enum xgbe_mode mode;
 	unsigned int ad_reg, lp_reg;
 
-	pdata->phy.lp_advertising |= ADVERTISED_Autoneg;
-	pdata->phy.lp_advertising |= ADVERTISED_Backplane;
+	XGBE_SET_LP_ADV(lks, Autoneg);
+	XGBE_SET_LP_ADV(lks, Backplane);
 
 	/* Compare Advertisement and Link Partner register 1 */
 	ad_reg = XMDIO_READ(pdata, MDIO_MMD_AN, MDIO_AN_ADVERTISE);
 	lp_reg = XMDIO_READ(pdata, MDIO_MMD_AN, MDIO_AN_LPA);
 	if (lp_reg & 0x400)
-		pdata->phy.lp_advertising |= ADVERTISED_Pause;
+		XGBE_SET_LP_ADV(lks, Pause);
 	if (lp_reg & 0x800)
-		pdata->phy.lp_advertising |= ADVERTISED_Asym_Pause;
+		XGBE_SET_LP_ADV(lks, Asym_Pause);
 
 	if (pdata->phy.pause_autoneg) {
 		/* Set flow control based on auto-negotiation result */
@@ -1504,9 +1509,9 @@ static enum xgbe_mode xgbe_phy_an73_outcome(struct xgbe_prv_data *pdata)
 	ad_reg = XMDIO_READ(pdata, MDIO_MMD_AN, MDIO_AN_ADVERTISE + 1);
 	lp_reg = XMDIO_READ(pdata, MDIO_MMD_AN, MDIO_AN_LPA + 1);
 	if (lp_reg & 0x80)
-		pdata->phy.lp_advertising |= ADVERTISED_10000baseKR_Full;
+		XGBE_SET_LP_ADV(lks, 10000baseKR_Full);
 	if (lp_reg & 0x20)
-		pdata->phy.lp_advertising |= ADVERTISED_1000baseKX_Full;
+		XGBE_SET_LP_ADV(lks, 1000baseKX_Full);
 
 	ad_reg &= lp_reg;
 	if (ad_reg & 0x80)
@@ -1520,7 +1525,7 @@ static enum xgbe_mode xgbe_phy_an73_outcome(struct xgbe_prv_data *pdata)
 	ad_reg = XMDIO_READ(pdata, MDIO_MMD_AN, MDIO_AN_ADVERTISE + 2);
 	lp_reg = XMDIO_READ(pdata, MDIO_MMD_AN, MDIO_AN_LPA + 2);
 	if (lp_reg & 0xc000)
-		pdata->phy.lp_advertising |= ADVERTISED_10000baseR_FEC;
+		XGBE_SET_LP_ADV(lks, 10000baseR_FEC);
 
 	return mode;
 }
@@ -1541,41 +1546,43 @@ static enum xgbe_mode xgbe_phy_an_outcome(struct xgbe_prv_data *pdata)
 	}
 }
 
-static unsigned int xgbe_phy_an_advertising(struct xgbe_prv_data *pdata)
+static void xgbe_phy_an_advertising(struct xgbe_prv_data *pdata,
+				    struct ethtool_link_ksettings *dlks)
 {
+	struct ethtool_link_ksettings *slks = &pdata->phy.lks;
 	struct xgbe_phy_data *phy_data = pdata->phy_data;
-	unsigned int advertising;
+
+	XGBE_LM_COPY(dlks, advertising, slks, advertising);
 
 	/* Without a re-driver, just return current advertising */
 	if (!phy_data->redrv)
-		return pdata->phy.advertising;
+		return;
 
 	/* With the KR re-driver we need to advertise a single speed */
-	advertising = pdata->phy.advertising;
-	advertising &= ~ADVERTISED_1000baseKX_Full;
-	advertising &= ~ADVERTISED_10000baseKR_Full;
+	XGBE_CLR_ADV(dlks, 1000baseKX_Full);
+	XGBE_CLR_ADV(dlks, 10000baseKR_Full);
 
 	switch (phy_data->port_mode) {
 	case XGBE_PORT_MODE_BACKPLANE:
-		advertising |= ADVERTISED_10000baseKR_Full;
+		XGBE_SET_ADV(dlks, 10000baseKR_Full);
 		break;
 	case XGBE_PORT_MODE_BACKPLANE_2500:
-		advertising |= ADVERTISED_1000baseKX_Full;
+		XGBE_SET_ADV(dlks, 1000baseKX_Full);
 		break;
 	case XGBE_PORT_MODE_1000BASE_T:
 	case XGBE_PORT_MODE_1000BASE_X:
 	case XGBE_PORT_MODE_NBASE_T:
-		advertising |= ADVERTISED_1000baseKX_Full;
+		XGBE_SET_ADV(dlks, 1000baseKX_Full);
 		break;
 	case XGBE_PORT_MODE_10GBASE_T:
 		if (phy_data->phydev &&
 		    (phy_data->phydev->speed == SPEED_10000))
-			advertising |= ADVERTISED_10000baseKR_Full;
+			XGBE_SET_ADV(dlks, 10000baseKR_Full);
 		else
-			advertising |= ADVERTISED_1000baseKX_Full;
+			XGBE_SET_ADV(dlks, 1000baseKX_Full);
 		break;
 	case XGBE_PORT_MODE_10GBASE_R:
-		advertising |= ADVERTISED_10000baseKR_Full;
+		XGBE_SET_ADV(dlks, 10000baseKR_Full);
 		break;
 	case XGBE_PORT_MODE_SFP:
 		switch (phy_data->sfp_base) {
@@ -1583,24 +1590,24 @@ static unsigned int xgbe_phy_an_advertising(struct xgbe_prv_data *pdata)
 		case XGBE_SFP_BASE_1000_SX:
 		case XGBE_SFP_BASE_1000_LX:
 		case XGBE_SFP_BASE_1000_CX:
-			advertising |= ADVERTISED_1000baseKX_Full;
+			XGBE_SET_ADV(dlks, 1000baseKX_Full);
 			break;
 		default:
-			advertising |= ADVERTISED_10000baseKR_Full;
+			XGBE_SET_ADV(dlks, 10000baseKR_Full);
 			break;
 		}
 		break;
 	default:
-		advertising |= ADVERTISED_10000baseKR_Full;
+		XGBE_SET_ADV(dlks, 10000baseKR_Full);
 		break;
 	}
-
-	return advertising;
 }
 
 static int xgbe_phy_an_config(struct xgbe_prv_data *pdata)
 {
+	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
 	struct xgbe_phy_data *phy_data = pdata->phy_data;
+	u32 advertising;
 	int ret;
 
 	ret = xgbe_phy_find_phy_device(pdata);
@@ -1610,9 +1617,12 @@ static int xgbe_phy_an_config(struct xgbe_prv_data *pdata)
 	if (!phy_data->phydev)
 		return 0;
 
+	ethtool_convert_link_mode_to_legacy_u32(&advertising,
+						lks->link_modes.advertising);
+
 	phy_data->phydev->autoneg = pdata->phy.autoneg;
 	phy_data->phydev->advertising = phy_data->phydev->supported &
-					pdata->phy.advertising;
+					advertising;
 
 	if (pdata->phy.autoneg != AUTONEG_ENABLE) {
 		phy_data->phydev->speed = pdata->phy.speed;
@@ -2073,11 +2083,10 @@ static void xgbe_phy_set_mode(struct xgbe_prv_data *pdata, enum xgbe_mode mode)
 }
 
 static bool xgbe_phy_check_mode(struct xgbe_prv_data *pdata,
-				enum xgbe_mode mode, u32 advert)
+				enum xgbe_mode mode, bool advert)
 {
 	if (pdata->phy.autoneg == AUTONEG_ENABLE) {
-		if (pdata->phy.advertising & advert)
-			return true;
+		return advert;
 	} else {
 		enum xgbe_mode cur_mode;
 
@@ -2092,13 +2101,15 @@ static bool xgbe_phy_check_mode(struct xgbe_prv_data *pdata,
 static bool xgbe_phy_use_basex_mode(struct xgbe_prv_data *pdata,
 				    enum xgbe_mode mode)
 {
+	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
+
 	switch (mode) {
 	case XGBE_MODE_X:
 		return xgbe_phy_check_mode(pdata, mode,
-					   ADVERTISED_1000baseT_Full);
+					   XGBE_ADV(lks, 1000baseX_Full));
 	case XGBE_MODE_KR:
 		return xgbe_phy_check_mode(pdata, mode,
-					   ADVERTISED_10000baseT_Full);
+					   XGBE_ADV(lks, 10000baseKR_Full));
 	default:
 		return false;
 	}
@@ -2107,19 +2118,21 @@ static bool xgbe_phy_use_basex_mode(struct xgbe_prv_data *pdata,
 static bool xgbe_phy_use_baset_mode(struct xgbe_prv_data *pdata,
 				    enum xgbe_mode mode)
 {
+	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
+
 	switch (mode) {
 	case XGBE_MODE_SGMII_100:
 		return xgbe_phy_check_mode(pdata, mode,
-					   ADVERTISED_100baseT_Full);
+					   XGBE_ADV(lks, 100baseT_Full));
 	case XGBE_MODE_SGMII_1000:
 		return xgbe_phy_check_mode(pdata, mode,
-					   ADVERTISED_1000baseT_Full);
+					   XGBE_ADV(lks, 1000baseT_Full));
 	case XGBE_MODE_KX_2500:
 		return xgbe_phy_check_mode(pdata, mode,
-					   ADVERTISED_2500baseX_Full);
+					   XGBE_ADV(lks, 2500baseT_Full));
 	case XGBE_MODE_KR:
 		return xgbe_phy_check_mode(pdata, mode,
-					   ADVERTISED_10000baseT_Full);
+					   XGBE_ADV(lks, 10000baseT_Full));
 	default:
 		return false;
 	}
@@ -2128,6 +2141,7 @@ static bool xgbe_phy_use_baset_mode(struct xgbe_prv_data *pdata,
 static bool xgbe_phy_use_sfp_mode(struct xgbe_prv_data *pdata,
 				  enum xgbe_mode mode)
 {
+	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
 	struct xgbe_phy_data *phy_data = pdata->phy_data;
 
 	switch (mode) {
@@ -2135,22 +2149,26 @@ static bool xgbe_phy_use_sfp_mode(struct xgbe_prv_data *pdata,
 		if (phy_data->sfp_base == XGBE_SFP_BASE_1000_T)
 			return false;
 		return xgbe_phy_check_mode(pdata, mode,
-					   ADVERTISED_1000baseT_Full);
+					   XGBE_ADV(lks, 1000baseX_Full));
 	case XGBE_MODE_SGMII_100:
 		if (phy_data->sfp_base != XGBE_SFP_BASE_1000_T)
 			return false;
 		return xgbe_phy_check_mode(pdata, mode,
-					   ADVERTISED_100baseT_Full);
+					   XGBE_ADV(lks, 100baseT_Full));
 	case XGBE_MODE_SGMII_1000:
 		if (phy_data->sfp_base != XGBE_SFP_BASE_1000_T)
 			return false;
 		return xgbe_phy_check_mode(pdata, mode,
-					   ADVERTISED_1000baseT_Full);
+					   XGBE_ADV(lks, 1000baseT_Full));
 	case XGBE_MODE_SFI:
 		if (phy_data->sfp_mod_absent)
 			return true;
 		return xgbe_phy_check_mode(pdata, mode,
-					   ADVERTISED_10000baseT_Full);
+					   XGBE_ADV(lks, 10000baseSR_Full)  ||
+					   XGBE_ADV(lks, 10000baseLR_Full)  ||
+					   XGBE_ADV(lks, 10000baseLRM_Full) ||
+					   XGBE_ADV(lks, 10000baseER_Full)  ||
+					   XGBE_ADV(lks, 10000baseCR_Full));
 	default:
 		return false;
 	}
@@ -2159,10 +2177,12 @@ static bool xgbe_phy_use_sfp_mode(struct xgbe_prv_data *pdata,
 static bool xgbe_phy_use_bp_2500_mode(struct xgbe_prv_data *pdata,
 				      enum xgbe_mode mode)
 {
+	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
+
 	switch (mode) {
 	case XGBE_MODE_KX_2500:
 		return xgbe_phy_check_mode(pdata, mode,
-					   ADVERTISED_2500baseX_Full);
+					   XGBE_ADV(lks, 2500baseX_Full));
 	default:
 		return false;
 	}
@@ -2171,13 +2191,15 @@ static bool xgbe_phy_use_bp_2500_mode(struct xgbe_prv_data *pdata,
 static bool xgbe_phy_use_bp_mode(struct xgbe_prv_data *pdata,
 				 enum xgbe_mode mode)
 {
+	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
+
 	switch (mode) {
 	case XGBE_MODE_KX_1000:
 		return xgbe_phy_check_mode(pdata, mode,
-					   ADVERTISED_1000baseKX_Full);
+					   XGBE_ADV(lks, 1000baseKX_Full));
 	case XGBE_MODE_KR:
 		return xgbe_phy_check_mode(pdata, mode,
-					   ADVERTISED_10000baseKR_Full);
+					   XGBE_ADV(lks, 10000baseKR_Full));
 	default:
 		return false;
 	}
@@ -2744,6 +2766,7 @@ static void xgbe_phy_exit(struct xgbe_prv_data *pdata)
 
 static int xgbe_phy_init(struct xgbe_prv_data *pdata)
 {
+	struct ethtool_link_ksettings *lks = &pdata->phy.lks;
 	struct xgbe_phy_data *phy_data;
 	struct mii_bus *mii;
 	unsigned int reg;
@@ -2823,32 +2846,33 @@ static int xgbe_phy_init(struct xgbe_prv_data *pdata)
 	phy_data->cur_mode = XGBE_MODE_UNKNOWN;
 
 	/* Initialize supported features */
-	pdata->phy.supported = 0;
+	XGBE_ZERO_SUP(lks);
 
 	switch (phy_data->port_mode) {
 	/* Backplane support */
 	case XGBE_PORT_MODE_BACKPLANE:
-		pdata->phy.supported |= SUPPORTED_Autoneg;
-		pdata->phy.supported |= SUPPORTED_Pause | SUPPORTED_Asym_Pause;
-		pdata->phy.supported |= SUPPORTED_Backplane;
+		XGBE_SET_SUP(lks, Autoneg);
+		XGBE_SET_SUP(lks, Pause);
+		XGBE_SET_SUP(lks, Asym_Pause);
+		XGBE_SET_SUP(lks, Backplane);
 		if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_1000) {
-			pdata->phy.supported |= SUPPORTED_1000baseKX_Full;
+			XGBE_SET_SUP(lks, 1000baseKX_Full);
 			phy_data->start_mode = XGBE_MODE_KX_1000;
 		}
 		if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_10000) {
-			pdata->phy.supported |= SUPPORTED_10000baseKR_Full;
+			XGBE_SET_SUP(lks, 10000baseKR_Full);
 			if (pdata->fec_ability & MDIO_PMA_10GBR_FECABLE_ABLE)
-				pdata->phy.supported |=
-					SUPPORTED_10000baseR_FEC;
+				XGBE_SET_SUP(lks, 10000baseR_FEC);
 			phy_data->start_mode = XGBE_MODE_KR;
 		}
 
 		phy_data->phydev_mode = XGBE_MDIO_MODE_NONE;
 		break;
 	case XGBE_PORT_MODE_BACKPLANE_2500:
-		pdata->phy.supported |= SUPPORTED_Pause | SUPPORTED_Asym_Pause;
-		pdata->phy.supported |= SUPPORTED_Backplane;
-		pdata->phy.supported |= SUPPORTED_2500baseX_Full;
+		XGBE_SET_SUP(lks, Pause);
+		XGBE_SET_SUP(lks, Asym_Pause);
+		XGBE_SET_SUP(lks, Backplane);
+		XGBE_SET_SUP(lks, 2500baseX_Full);
 		phy_data->start_mode = XGBE_MODE_KX_2500;
 
 		phy_data->phydev_mode = XGBE_MDIO_MODE_NONE;
@@ -2856,15 +2880,16 @@ static int xgbe_phy_init(struct xgbe_prv_data *pdata)
 
 	/* MDIO 1GBase-T support */
 	case XGBE_PORT_MODE_1000BASE_T:
-		pdata->phy.supported |= SUPPORTED_Autoneg;
-		pdata->phy.supported |= SUPPORTED_Pause | SUPPORTED_Asym_Pause;
-		pdata->phy.supported |= SUPPORTED_TP;
+		XGBE_SET_SUP(lks, Autoneg);
+		XGBE_SET_SUP(lks, Pause);
+		XGBE_SET_SUP(lks, Asym_Pause);
+		XGBE_SET_SUP(lks, TP);
 		if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_100) {
-			pdata->phy.supported |= SUPPORTED_100baseT_Full;
+			XGBE_SET_SUP(lks, 100baseT_Full);
 			phy_data->start_mode = XGBE_MODE_SGMII_100;
 		}
 		if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_1000) {
-			pdata->phy.supported |= SUPPORTED_1000baseT_Full;
+			XGBE_SET_SUP(lks, 1000baseT_Full);
 			phy_data->start_mode = XGBE_MODE_SGMII_1000;
 		}
 
@@ -2873,10 +2898,11 @@ static int xgbe_phy_init(struct xgbe_prv_data *pdata)
 
 	/* MDIO Base-X support */
 	case XGBE_PORT_MODE_1000BASE_X:
-		pdata->phy.supported |= SUPPORTED_Autoneg;
-		pdata->phy.supported |= SUPPORTED_Pause | SUPPORTED_Asym_Pause;
-		pdata->phy.supported |= SUPPORTED_FIBRE;
-		pdata->phy.supported |= SUPPORTED_1000baseT_Full;
+		XGBE_SET_SUP(lks, Autoneg);
+		XGBE_SET_SUP(lks, Pause);
+		XGBE_SET_SUP(lks, Asym_Pause);
+		XGBE_SET_SUP(lks, FIBRE);
+		XGBE_SET_SUP(lks, 1000baseX_Full);
 		phy_data->start_mode = XGBE_MODE_X;
 
 		phy_data->phydev_mode = XGBE_MDIO_MODE_CL22;
@@ -2884,19 +2910,20 @@ static int xgbe_phy_init(struct xgbe_prv_data *pdata)
 
 	/* MDIO NBase-T support */
 	case XGBE_PORT_MODE_NBASE_T:
-		pdata->phy.supported |= SUPPORTED_Autoneg;
-		pdata->phy.supported |= SUPPORTED_Pause | SUPPORTED_Asym_Pause;
-		pdata->phy.supported |= SUPPORTED_TP;
+		XGBE_SET_SUP(lks, Autoneg);
+		XGBE_SET_SUP(lks, Pause);
+		XGBE_SET_SUP(lks, Asym_Pause);
+		XGBE_SET_SUP(lks, TP);
 		if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_100) {
-			pdata->phy.supported |= SUPPORTED_100baseT_Full;
+			XGBE_SET_SUP(lks, 100baseT_Full);
 			phy_data->start_mode = XGBE_MODE_SGMII_100;
 		}
 		if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_1000) {
-			pdata->phy.supported |= SUPPORTED_1000baseT_Full;
+			XGBE_SET_SUP(lks, 1000baseT_Full);
 			phy_data->start_mode = XGBE_MODE_SGMII_1000;
 		}
 		if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_2500) {
-			pdata->phy.supported |= SUPPORTED_2500baseX_Full;
+			XGBE_SET_SUP(lks, 2500baseT_Full);
 			phy_data->start_mode = XGBE_MODE_KX_2500;
 		}
 
@@ -2905,33 +2932,38 @@ static int xgbe_phy_init(struct xgbe_prv_data *pdata)
 
 	/* 10GBase-T support */
 	case XGBE_PORT_MODE_10GBASE_T:
-		pdata->phy.supported |= SUPPORTED_Autoneg;
-		pdata->phy.supported |= SUPPORTED_Pause | SUPPORTED_Asym_Pause;
-		pdata->phy.supported |= SUPPORTED_TP;
+		XGBE_SET_SUP(lks, Autoneg);
+		XGBE_SET_SUP(lks, Pause);
+		XGBE_SET_SUP(lks, Asym_Pause);
+		XGBE_SET_SUP(lks, TP);
 		if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_100) {
-			pdata->phy.supported |= SUPPORTED_100baseT_Full;
+			XGBE_SET_SUP(lks, 100baseT_Full);
 			phy_data->start_mode = XGBE_MODE_SGMII_100;
 		}
 		if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_1000) {
-			pdata->phy.supported |= SUPPORTED_1000baseT_Full;
+			XGBE_SET_SUP(lks, 1000baseT_Full);
 			phy_data->start_mode = XGBE_MODE_SGMII_1000;
 		}
 		if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_10000) {
-			pdata->phy.supported |= SUPPORTED_10000baseT_Full;
+			XGBE_SET_SUP(lks, 10000baseT_Full);
 			phy_data->start_mode = XGBE_MODE_KR;
 		}
 
-		phy_data->phydev_mode = XGBE_MDIO_MODE_NONE;
+		phy_data->phydev_mode = XGBE_MDIO_MODE_CL45;
 		break;
 
 	/* 10GBase-R support */
 	case XGBE_PORT_MODE_10GBASE_R:
-		pdata->phy.supported |= SUPPORTED_Autoneg;
-		pdata->phy.supported |= SUPPORTED_Pause | SUPPORTED_Asym_Pause;
-		pdata->phy.supported |= SUPPORTED_TP;
-		pdata->phy.supported |= SUPPORTED_10000baseT_Full;
+		XGBE_SET_SUP(lks, Autoneg);
+		XGBE_SET_SUP(lks, Pause);
+		XGBE_SET_SUP(lks, Asym_Pause);
+		XGBE_SET_SUP(lks, FIBRE);
+		XGBE_SET_SUP(lks, 10000baseSR_Full);
+		XGBE_SET_SUP(lks, 10000baseLR_Full);
+		XGBE_SET_SUP(lks, 10000baseLRM_Full);
+		XGBE_SET_SUP(lks, 10000baseER_Full);
 		if (pdata->fec_ability & MDIO_PMA_10GBR_FECABLE_ABLE)
-			pdata->phy.supported |= SUPPORTED_10000baseR_FEC;
+			XGBE_SET_SUP(lks, 10000baseR_FEC);
 		phy_data->start_mode = XGBE_MODE_SFI;
 
 		phy_data->phydev_mode = XGBE_MDIO_MODE_NONE;
@@ -2939,22 +2971,17 @@ static int xgbe_phy_init(struct xgbe_prv_data *pdata)
 
 	/* SFP support */
 	case XGBE_PORT_MODE_SFP:
-		pdata->phy.supported |= SUPPORTED_Autoneg;
-		pdata->phy.supported |= SUPPORTED_Pause | SUPPORTED_Asym_Pause;
-		pdata->phy.supported |= SUPPORTED_TP;
-		pdata->phy.supported |= SUPPORTED_FIBRE;
-		if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_100) {
-			pdata->phy.supported |= SUPPORTED_100baseT_Full;
+		XGBE_SET_SUP(lks, Autoneg);
+		XGBE_SET_SUP(lks, Pause);
+		XGBE_SET_SUP(lks, Asym_Pause);
+		XGBE_SET_SUP(lks, TP);
+		XGBE_SET_SUP(lks, FIBRE);
+		if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_100)
 			phy_data->start_mode = XGBE_MODE_SGMII_100;
-		}
-		if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_1000) {
-			pdata->phy.supported |= SUPPORTED_1000baseT_Full;
+		if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_1000)
 			phy_data->start_mode = XGBE_MODE_SGMII_1000;
-		}
-		if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_10000) {
-			pdata->phy.supported |= SUPPORTED_10000baseT_Full;
+		if (phy_data->port_speeds & XGBE_PHY_PORT_SPEED_10000)
 			phy_data->start_mode = XGBE_MODE_SFI;
-		}
 
 		phy_data->phydev_mode = XGBE_MDIO_MODE_CL22;
 
@@ -2965,8 +2992,9 @@ static int xgbe_phy_init(struct xgbe_prv_data *pdata)
 	}
 
 	if (netif_msg_probe(pdata))
-		dev_dbg(pdata->dev, "phy supported=%#x\n",
-			pdata->phy.supported);
+		dev_dbg(pdata->dev, "phy supported=0x%*pb\n",
+			__ETHTOOL_LINK_MODE_MASK_NBITS,
+			lks->link_modes.supported);
 
 	if ((phy_data->conn_type & XGBE_CONN_TYPE_MDIO) &&
 	    (phy_data->phydev_mode != XGBE_MDIO_MODE_NONE)) {
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe.h b/drivers/net/ethernet/amd/xgbe/xgbe.h
index 0938294f640a..ad102c8bac7b 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe.h
+++ b/drivers/net/ethernet/amd/xgbe/xgbe.h
@@ -129,6 +129,10 @@
 #include <net/dcbnl.h>
 #include <linux/completion.h>
 #include <linux/cpumask.h>
+#include <linux/interrupt.h>
+#include <linux/dcache.h>
+#include <linux/ethtool.h>
+#include <linux/list.h>
 
 #define XGBE_DRV_NAME		"amd-xgbe"
 #define XGBE_DRV_VERSION	"1.0.3"
@@ -180,8 +184,6 @@
 #define XGBE_IRQ_MODE_EDGE	0
 #define XGBE_IRQ_MODE_LEVEL	1
 
-#define XGBE_DMA_INTERRUPT_MASK	0x31c7
-
 #define XGMAC_MIN_PACKET	60
 #define XGMAC_STD_PACKET_MTU	1500
 #define XGMAC_MAX_STD_PACKET	1518
@@ -296,6 +298,48 @@
 /* MDIO port types */
 #define XGMAC_MAX_C22_PORT		3
 
+/* Link mode bit operations */
+#define XGBE_ZERO_SUP(_ls)		\
+	ethtool_link_ksettings_zero_link_mode((_ls), supported)
+
+#define XGBE_SET_SUP(_ls, _mode)	\
+	ethtool_link_ksettings_add_link_mode((_ls), supported, _mode)
+
+#define XGBE_CLR_SUP(_ls, _mode)	\
+	ethtool_link_ksettings_del_link_mode((_ls), supported, _mode)
+
+#define XGBE_IS_SUP(_ls, _mode)	\
+	ethtool_link_ksettings_test_link_mode((_ls), supported, _mode)
+
+#define XGBE_ZERO_ADV(_ls)		\
+	ethtool_link_ksettings_zero_link_mode((_ls), advertising)
+
+#define XGBE_SET_ADV(_ls, _mode)	\
+	ethtool_link_ksettings_add_link_mode((_ls), advertising, _mode)
+
+#define XGBE_CLR_ADV(_ls, _mode)	\
+	ethtool_link_ksettings_del_link_mode((_ls), advertising, _mode)
+
+#define XGBE_ADV(_ls, _mode)		\
+	ethtool_link_ksettings_test_link_mode((_ls), advertising, _mode)
+
+#define XGBE_ZERO_LP_ADV(_ls)		\
+	ethtool_link_ksettings_zero_link_mode((_ls), lp_advertising)
+
+#define XGBE_SET_LP_ADV(_ls, _mode)	\
+	ethtool_link_ksettings_add_link_mode((_ls), lp_advertising, _mode)
+
+#define XGBE_CLR_LP_ADV(_ls, _mode)	\
+	ethtool_link_ksettings_del_link_mode((_ls), lp_advertising, _mode)
+
+#define XGBE_LP_ADV(_ls, _mode)		\
+	ethtool_link_ksettings_test_link_mode((_ls), lp_advertising, _mode)
+
+#define XGBE_LM_COPY(_dst, _dname, _src, _sname)	\
+	bitmap_copy((_dst)->link_modes._dname,		\
+		    (_src)->link_modes._sname,		\
+		    __ETHTOOL_LINK_MODE_MASK_NBITS)
+
 struct xgbe_prv_data;
 
 struct xgbe_packet_data {
@@ -460,6 +504,8 @@ struct xgbe_channel {
 	/* Netdev related settings */
 	struct napi_struct napi;
 
+	/* Per channel interrupt enablement tracker */
+	unsigned int curr_ier;
 	unsigned int saved_ier;
 
 	unsigned int tx_timer_active;
@@ -561,9 +607,7 @@ enum xgbe_mdio_mode {
 };
 
 struct xgbe_phy {
-	u32 supported;
-	u32 advertising;
-	u32 lp_advertising;
+	struct ethtool_link_ksettings lks;
 
 	int address;
 
@@ -666,6 +710,16 @@ struct xgbe_ext_stats {
 	u64 tx_tso_packets;
 	u64 rx_split_header_packets;
 	u64 rx_buffer_unavailable;
+
+	u64 txq_packets[XGBE_MAX_DMA_CHANNELS];
+	u64 txq_bytes[XGBE_MAX_DMA_CHANNELS];
+	u64 rxq_packets[XGBE_MAX_DMA_CHANNELS];
+	u64 rxq_bytes[XGBE_MAX_DMA_CHANNELS];
+
+	u64 tx_vxlan_packets;
+	u64 rx_vxlan_packets;
+	u64 rx_csum_errors;
+	u64 rx_vxlan_csum_errors;
 };
 
 struct xgbe_hw_if {
@@ -769,6 +823,11 @@ struct xgbe_hw_if {
 	/* For ECC */
 	void (*disable_ecc_ded)(struct xgbe_prv_data *);
 	void (*disable_ecc_sec)(struct xgbe_prv_data *, enum xgbe_ecc_sec);
+
+	/* For VXLAN */
+	void (*enable_vxlan)(struct xgbe_prv_data *);
+	void (*disable_vxlan)(struct xgbe_prv_data *);
+	void (*set_vxlan_id)(struct xgbe_prv_data *);
 };
 
 /* This structure represents implementation specific routines for an
@@ -810,7 +869,8 @@ struct xgbe_phy_impl_if {
 	int (*an_config)(struct xgbe_prv_data *);
 
 	/* Set/override auto-negotiation advertisement settings */
-	unsigned int (*an_advertising)(struct xgbe_prv_data *);
+	void (*an_advertising)(struct xgbe_prv_data *,
+			       struct ethtool_link_ksettings *);
 
 	/* Process results of auto-negotiation */
 	enum xgbe_mode (*an_outcome)(struct xgbe_prv_data *);
@@ -892,6 +952,7 @@ struct xgbe_hw_features {
 	unsigned int addn_mac;		/* Additional MAC Addresses */
 	unsigned int ts_src;		/* Timestamp Source */
 	unsigned int sa_vlan_ins;	/* Source Address or VLAN Insertion */
+	unsigned int vxn;		/* VXLAN/NVGRE */
 
 	/* HW Feature Register1 */
 	unsigned int rx_fifo_size;	/* MTL Receive FIFO Size */
@@ -930,6 +991,12 @@ struct xgbe_version_data {
 	unsigned int rx_desc_prefetch;
 };
 
+struct xgbe_vxlan_data {
+	struct list_head list;
+	sa_family_t sa_family;
+	__be16 port;
+};
+
 struct xgbe_prv_data {
 	struct net_device *netdev;
 	struct pci_dev *pcidev;
@@ -1071,6 +1138,15 @@ struct xgbe_prv_data {
 	u32 rss_table[XGBE_RSS_MAX_TABLE_SIZE];
 	u32 rss_options;
 
+	/* VXLAN settings */
+	unsigned int vxlan_port_set;
+	unsigned int vxlan_offloads_set;
+	unsigned int vxlan_force_disable;
+	unsigned int vxlan_port_count;
+	struct list_head vxlan_ports;
+	u16 vxlan_port;
+	netdev_features_t vxlan_features;
+
 	/* Netdev related settings */
 	unsigned char mac_addr[ETH_ALEN];
 	netdev_features_t netdev_features;
@@ -1171,7 +1247,6 @@ struct xgbe_prv_data {
 	struct tasklet_struct tasklet_i2c;
 	struct tasklet_struct tasklet_an;
 
-#ifdef CONFIG_DEBUG_FS
 	struct dentry *xgbe_debugfs;
 
 	unsigned int debugfs_xgmac_reg;
@@ -1182,7 +1257,6 @@ struct xgbe_prv_data {
 	unsigned int debugfs_xprop_reg;
 
 	unsigned int debugfs_xi2c_reg;
-#endif
 };
 
 /* Function prototypes*/
@@ -1231,9 +1305,11 @@ void xgbe_init_tx_coalesce(struct xgbe_prv_data *);
 #ifdef CONFIG_DEBUG_FS
 void xgbe_debugfs_init(struct xgbe_prv_data *);
 void xgbe_debugfs_exit(struct xgbe_prv_data *);
+void xgbe_debugfs_rename(struct xgbe_prv_data *pdata);
 #else
 static inline void xgbe_debugfs_init(struct xgbe_prv_data *pdata) {}
 static inline void xgbe_debugfs_exit(struct xgbe_prv_data *pdata) {}
+static inline void xgbe_debugfs_rename(struct xgbe_prv_data *pdata) {}
 #endif /* CONFIG_DEBUG_FS */
 
 /* NOTE: Uncomment for function trace log messages in KERNEL LOG */
diff --git a/drivers/net/ethernet/apm/xgene/xgene_enet_hw.c b/drivers/net/ethernet/apm/xgene/xgene_enet_hw.c
index e45b587c2994..3188f553da35 100644
--- a/drivers/net/ethernet/apm/xgene/xgene_enet_hw.c
+++ b/drivers/net/ethernet/apm/xgene/xgene_enet_hw.c
@@ -468,7 +468,6 @@ static void xgene_enet_configure_clock(struct xgene_enet_pdata *pdata)
 
 static void xgene_gmac_set_speed(struct xgene_enet_pdata *pdata)
 {
-	struct device *dev = &pdata->pdev->dev;
 	u32 icm0, icm2, mc2;
 	u32 intf_ctl, rgmii, value;
 
@@ -500,10 +499,8 @@ static void xgene_gmac_set_speed(struct xgene_enet_pdata *pdata)
 		intf_ctl |= ENET_GHD_MODE;
 		CFG_MACMODE_SET(&icm0, 2);
 		CFG_WAITASYNCRD_SET(&icm2, 0);
-		if (dev->of_node) {
-			CFG_TXCLK_MUXSEL0_SET(&rgmii, pdata->tx_delay);
-			CFG_RXCLK_MUXSEL0_SET(&rgmii, pdata->rx_delay);
-		}
+		CFG_TXCLK_MUXSEL0_SET(&rgmii, pdata->tx_delay);
+		CFG_RXCLK_MUXSEL0_SET(&rgmii, pdata->rx_delay);
 		rgmii |= CFG_SPEED_1250;
 
 		xgene_enet_rd_csr(pdata, DEBUG_REG_ADDR, &value);
diff --git a/drivers/net/ethernet/apm/xgene/xgene_enet_main.c b/drivers/net/ethernet/apm/xgene/xgene_enet_main.c
index 1d307f2def2d..3b889efddf78 100644
--- a/drivers/net/ethernet/apm/xgene/xgene_enet_main.c
+++ b/drivers/net/ethernet/apm/xgene/xgene_enet_main.c
@@ -1591,7 +1591,7 @@ static int xgene_get_tx_delay(struct xgene_enet_pdata *pdata)
 	struct device *dev = &pdata->pdev->dev;
 	int delay, ret;
 
-	ret = of_property_read_u32(dev->of_node, "tx-delay", &delay);
+	ret = device_property_read_u32(dev, "tx-delay", &delay);
 	if (ret) {
 		pdata->tx_delay = 4;
 		return 0;
@@ -1612,7 +1612,7 @@ static int xgene_get_rx_delay(struct xgene_enet_pdata *pdata)
 	struct device *dev = &pdata->pdev->dev;
 	int delay, ret;
 
-	ret = of_property_read_u32(dev->of_node, "rx-delay", &delay);
+	ret = device_property_read_u32(dev, "rx-delay", &delay);
 	if (ret) {
 		pdata->rx_delay = 2;
 		return 0;
@@ -1661,21 +1661,19 @@ static int xgene_enet_get_irqs(struct xgene_enet_pdata *pdata)
 	return 0;
 }
 
-static int xgene_enet_check_phy_handle(struct xgene_enet_pdata *pdata)
+static void xgene_enet_check_phy_handle(struct xgene_enet_pdata *pdata)
 {
 	int ret;
 
 	if (pdata->phy_mode == PHY_INTERFACE_MODE_XGMII)
-		return 0;
+		return;
 
 	if (!IS_ENABLED(CONFIG_MDIO_XGENE))
-		return 0;
+		return;
 
 	ret = xgene_enet_phy_connect(pdata->ndev);
 	if (!ret)
 		pdata->mdio_driver = true;
-
-	return 0;
 }
 
 static void xgene_enet_gpiod_get(struct xgene_enet_pdata *pdata)
@@ -1779,10 +1777,6 @@ static int xgene_enet_get_resources(struct xgene_enet_pdata *pdata)
 	if (ret)
 		return ret;
 
-	ret = xgene_enet_check_phy_handle(pdata);
-	if (ret)
-		return ret;
-
 	xgene_enet_gpiod_get(pdata);
 
 	pdata->clk = devm_clk_get(&pdev->dev, NULL);
@@ -2097,9 +2091,11 @@ static int xgene_enet_probe(struct platform_device *pdev)
 		goto err;
 	}
 
+	xgene_enet_check_phy_handle(pdata);
+
 	ret = xgene_enet_init_hw(pdata);
 	if (ret)
-		goto err;
+		goto err2;
 
 	link_state = pdata->mac_ops->link_state;
 	if (pdata->phy_mode == PHY_INTERFACE_MODE_XGMII) {
@@ -2117,29 +2113,30 @@ static int xgene_enet_probe(struct platform_device *pdev)
 	spin_lock_init(&pdata->stats_lock);
 	ret = xgene_extd_stats_init(pdata);
 	if (ret)
-		goto err2;
+		goto err1;
 
 	xgene_enet_napi_add(pdata);
 	ret = register_netdev(ndev);
 	if (ret) {
 		netdev_err(ndev, "Failed to register netdev\n");
-		goto err2;
+		goto err1;
 	}
 
 	return 0;
 
-err2:
+err1:
 	/*
 	 * If necessary, free_netdev() will call netif_napi_del() and undo
 	 * the effects of xgene_enet_napi_add()'s calls to netif_napi_add().
 	 */
 
+	xgene_enet_delete_desc_rings(pdata);
+
+err2:
 	if (pdata->mdio_driver)
 		xgene_enet_phy_disconnect(pdata);
 	else if (phy_interface_mode_is_rgmii(pdata->phy_mode))
 		xgene_enet_mdio_remove(pdata);
-err1:
-	xgene_enet_delete_desc_rings(pdata);
 err:
 	free_netdev(ndev);
 	return ret;
diff --git a/drivers/net/ethernet/apple/mace.c b/drivers/net/ethernet/apple/mace.c
index 96dd5300e0e5..e58b157b7d7c 100644
--- a/drivers/net/ethernet/apple/mace.c
+++ b/drivers/net/ethernet/apple/mace.c
@@ -114,8 +114,8 @@ static int mace_probe(struct macio_dev *mdev, const struct of_device_id *match)
 	int j, rev, rc = -EBUSY;
 
 	if (macio_resource_count(mdev) != 3 || macio_irq_count(mdev) != 3) {
-		printk(KERN_ERR "can't use MACE %s: need 3 addrs and 3 irqs\n",
-		       mace->full_name);
+		printk(KERN_ERR "can't use MACE %pOF: need 3 addrs and 3 irqs\n",
+		       mace);
 		return -ENODEV;
 	}
 
@@ -123,8 +123,8 @@ static int mace_probe(struct macio_dev *mdev, const struct of_device_id *match)
 	if (addr == NULL) {
 		addr = of_get_property(mace, "local-mac-address", NULL);
 		if (addr == NULL) {
-			printk(KERN_ERR "Can't get mac-address for MACE %s\n",
-			       mace->full_name);
+			printk(KERN_ERR "Can't get mac-address for MACE %pOF\n",
+			       mace);
 			return -ENODEV;
 		}
 	}
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_hw.h b/drivers/net/ethernet/aquantia/atlantic/aq_hw.h
index fce0fd3f23ff..bf9b3f020e10 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_hw.h
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_hw.h
@@ -105,8 +105,7 @@ struct aq_hw_ops {
 
 	int (*hw_set_mac_address)(struct aq_hw_s *self, u8 *mac_addr);
 
-	int (*hw_get_link_status)(struct aq_hw_s *self,
-				  struct aq_hw_link_status_s *link_status);
+	int (*hw_get_link_status)(struct aq_hw_s *self);
 
 	int (*hw_set_link_speed)(struct aq_hw_s *self, u32 speed);
 
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_nic.c b/drivers/net/ethernet/aquantia/atlantic/aq_nic.c
index 9ee1c5016784..6ac9e2602d6d 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_nic.c
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_nic.c
@@ -103,6 +103,8 @@ int aq_nic_cfg_start(struct aq_nic_s *self)
 	else
 		cfg->vecs = 1U;
 
+	cfg->num_rss_queues = min(cfg->vecs, AQ_CFG_NUM_RSS_QUEUES_DEF);
+
 	cfg->irq_type = aq_pci_func_get_irq_type(self->aq_pci_func);
 
 	if ((cfg->irq_type == AQ_HW_IRQ_LEGACY) ||
@@ -123,33 +125,30 @@ static void aq_nic_service_timer_cb(unsigned long param)
 	struct net_device *ndev = aq_nic_get_ndev(self);
 	int err = 0;
 	unsigned int i = 0U;
-	struct aq_hw_link_status_s link_status;
 	struct aq_ring_stats_rx_s stats_rx;
 	struct aq_ring_stats_tx_s stats_tx;
 
 	if (aq_utils_obj_test(&self->header.flags, AQ_NIC_FLAGS_IS_NOT_READY))
 		goto err_exit;
 
-	err = self->aq_hw_ops.hw_get_link_status(self->aq_hw, &link_status);
+	err = self->aq_hw_ops.hw_get_link_status(self->aq_hw);
 	if (err < 0)
 		goto err_exit;
 
-	self->aq_hw_ops.hw_interrupt_moderation_set(self->aq_hw,
-			    self->aq_nic_cfg.is_interrupt_moderation);
-
-	if (memcmp(&link_status, &self->link_status, sizeof(link_status))) {
-		if (link_status.mbps) {
-			aq_utils_obj_set(&self->header.flags,
-					 AQ_NIC_FLAG_STARTED);
-			aq_utils_obj_clear(&self->header.flags,
-					   AQ_NIC_LINK_DOWN);
-			netif_carrier_on(self->ndev);
-		} else {
-			netif_carrier_off(self->ndev);
-			aq_utils_obj_set(&self->header.flags, AQ_NIC_LINK_DOWN);
-		}
+	self->link_status = self->aq_hw->aq_link_status;
 
-		self->link_status = link_status;
+	self->aq_hw_ops.hw_interrupt_moderation_set(self->aq_hw,
+		    self->aq_nic_cfg.is_interrupt_moderation);
+
+	if (self->link_status.mbps) {
+		aq_utils_obj_set(&self->header.flags,
+				 AQ_NIC_FLAG_STARTED);
+		aq_utils_obj_clear(&self->header.flags,
+				   AQ_NIC_LINK_DOWN);
+		netif_carrier_on(self->ndev);
+	} else {
+		netif_carrier_off(self->ndev);
+		aq_utils_obj_set(&self->header.flags, AQ_NIC_LINK_DOWN);
 	}
 
 	memset(&stats_rx, 0U, sizeof(struct aq_ring_stats_rx_s));
@@ -597,14 +596,11 @@ exit:
 }
 
 int aq_nic_xmit(struct aq_nic_s *self, struct sk_buff *skb)
-__releases(&ring->lock)
-__acquires(&ring->lock)
 {
 	struct aq_ring_s *ring = NULL;
 	unsigned int frags = 0U;
 	unsigned int vec = skb->queue_mapping % self->aq_nic_cfg.vecs;
 	unsigned int tc = 0U;
-	unsigned int trys = AQ_CFG_LOCK_TRYS;
 	int err = NETDEV_TX_OK;
 	bool is_nic_in_bad_state;
 
@@ -628,36 +624,21 @@ __acquires(&ring->lock)
 		goto err_exit;
 	}
 
-	do {
-		if (spin_trylock(&ring->header.lock)) {
-			frags = aq_nic_map_skb(self, skb, ring);
-
-			if (likely(frags)) {
-				err = self->aq_hw_ops.hw_ring_tx_xmit(
-								self->aq_hw,
-								ring, frags);
-				if (err >= 0) {
-					if (aq_ring_avail_dx(ring) <
-					    AQ_CFG_SKB_FRAGS_MAX + 1)
-						aq_nic_ndev_queue_stop(
-								self,
-								ring->idx);
-
-					++ring->stats.tx.packets;
-					ring->stats.tx.bytes += skb->len;
-				}
-			} else {
-				err = NETDEV_TX_BUSY;
-			}
+	frags = aq_nic_map_skb(self, skb, ring);
 
-			spin_unlock(&ring->header.lock);
-			break;
-		}
-	} while (--trys);
+	if (likely(frags)) {
+		err = self->aq_hw_ops.hw_ring_tx_xmit(self->aq_hw,
+						      ring,
+						      frags);
+		if (err >= 0) {
+			if (aq_ring_avail_dx(ring) < AQ_CFG_SKB_FRAGS_MAX + 1)
+				aq_nic_ndev_queue_stop(self, ring->idx);
 
-	if (!trys) {
+			++ring->stats.tx.packets;
+			ring->stats.tx.bytes += skb->len;
+		}
+	} else {
 		err = NETDEV_TX_BUSY;
-		goto err_exit;
 	}
 
 err_exit:
@@ -688,11 +669,26 @@ int aq_nic_set_multicast_list(struct aq_nic_s *self, struct net_device *ndev)
 	netdev_for_each_mc_addr(ha, ndev) {
 		ether_addr_copy(self->mc_list.ar[i++], ha->addr);
 		++self->mc_list.count;
+
+		if (i >= AQ_CFG_MULTICAST_ADDRESS_MAX)
+			break;
 	}
 
-	return self->aq_hw_ops.hw_multicast_list_set(self->aq_hw,
+	if (i >= AQ_CFG_MULTICAST_ADDRESS_MAX) {
+		/* Number of filters is too big: atlantic does not support this.
+		 * Force all multi filter to support this.
+		 * With this we disable all UC filters and setup "all pass"
+		 * multicast mask
+		 */
+		self->packet_filter |= IFF_ALLMULTI;
+		self->aq_hw->aq_nic_cfg->mc_list_count = 0;
+		return self->aq_hw_ops.hw_packet_filter_set(self->aq_hw,
+							self->packet_filter);
+	} else {
+		return self->aq_hw_ops.hw_multicast_list_set(self->aq_hw,
 						    self->mc_list.ar,
 						    self->mc_list.count);
+	}
 }
 
 int aq_nic_set_mtu(struct aq_nic_s *self, int new_mtu)
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_ring.c b/drivers/net/ethernet/aquantia/atlantic/aq_ring.c
index 9a0817938eca..4eee1996a825 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_ring.c
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_ring.c
@@ -101,7 +101,6 @@ int aq_ring_init(struct aq_ring_s *self)
 	self->hw_head = 0;
 	self->sw_head = 0;
 	self->sw_tail = 0;
-	spin_lock_init(&self->header.lock);
 	return 0;
 }
 
@@ -134,7 +133,10 @@ static inline unsigned int aq_ring_dx_in_range(unsigned int h, unsigned int i,
 }
 
 #define AQ_SKB_ALIGN SKB_DATA_ALIGN(sizeof(struct skb_shared_info))
-int aq_ring_rx_clean(struct aq_ring_s *self, int *work_done, int budget)
+int aq_ring_rx_clean(struct aq_ring_s *self,
+		     struct napi_struct *napi,
+		     int *work_done,
+		     int budget)
 {
 	struct net_device *ndev = aq_nic_get_ndev(self->aq_nic);
 	int err = 0;
@@ -240,7 +242,7 @@ int aq_ring_rx_clean(struct aq_ring_s *self, int *work_done, int budget)
 
 		skb_record_rx_queue(skb, self->idx);
 
-		netif_receive_skb(skb);
+		napi_gro_receive(napi, skb);
 
 		++self->stats.rx.packets;
 		self->stats.rx.bytes += skb->len;
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_ring.h b/drivers/net/ethernet/aquantia/atlantic/aq_ring.h
index eecd6d1c4d73..782176c5f4f8 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_ring.h
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_ring.h
@@ -148,7 +148,10 @@ int aq_ring_init(struct aq_ring_s *self);
 void aq_ring_rx_deinit(struct aq_ring_s *self);
 void aq_ring_free(struct aq_ring_s *self);
 void aq_ring_tx_clean(struct aq_ring_s *self);
-int aq_ring_rx_clean(struct aq_ring_s *self, int *work_done, int budget);
+int aq_ring_rx_clean(struct aq_ring_s *self,
+		     struct napi_struct *napi,
+		     int *work_done,
+		     int budget);
 int aq_ring_rx_fill(struct aq_ring_s *self);
 
 #endif /* AQ_RING_H */
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_utils.h b/drivers/net/ethernet/aquantia/atlantic/aq_utils.h
index f6012b34abe6..e12bcdfb874a 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_utils.h
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_utils.h
@@ -17,7 +17,6 @@
 #define AQ_DIMOF(_ARY_)  ARRAY_SIZE(_ARY_)
 
 struct aq_obj_s {
-	spinlock_t lock; /* spinlock for nic/rings processing */
 	atomic_t flags;
 };
 
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_vec.c b/drivers/net/ethernet/aquantia/atlantic/aq_vec.c
index ad5b4d4dac7f..ebf588004c46 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_vec.c
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_vec.c
@@ -34,8 +34,6 @@ struct aq_vec_s {
 #define AQ_VEC_RX_ID 1
 
 static int aq_vec_poll(struct napi_struct *napi, int budget)
-__releases(&self->lock)
-__acquires(&self->lock)
 {
 	struct aq_vec_s *self = container_of(napi, struct aq_vec_s, napi);
 	struct aq_ring_s *ring = NULL;
@@ -47,7 +45,7 @@ __acquires(&self->lock)
 
 	if (!self) {
 		err = -EINVAL;
-	} else if (spin_trylock(&self->header.lock)) {
+	} else {
 		for (i = 0U, ring = self->ring[0];
 			self->tx_rings > i; ++i, ring = self->ring[i]) {
 			if (self->aq_hw_ops->hw_ring_tx_head_update) {
@@ -78,6 +76,7 @@ __acquires(&self->lock)
 			if (ring[AQ_VEC_RX_ID].sw_head !=
 				ring[AQ_VEC_RX_ID].hw_head) {
 				err = aq_ring_rx_clean(&ring[AQ_VEC_RX_ID],
+						       napi,
 						       &work_done,
 						       budget - work_done);
 				if (err < 0)
@@ -105,11 +104,8 @@ __acquires(&self->lock)
 			self->aq_hw_ops->hw_irq_enable(self->aq_hw,
 					1U << self->aq_ring_param.vec_idx);
 		}
-
-err_exit:
-		spin_unlock(&self->header.lock);
 	}
-
+err_exit:
 	return work_done;
 }
 
@@ -185,8 +181,6 @@ int aq_vec_init(struct aq_vec_s *self, struct aq_hw_ops *aq_hw_ops,
 	self->aq_hw_ops = aq_hw_ops;
 	self->aq_hw = aq_hw;
 
-	spin_lock_init(&self->header.lock);
-
 	for (i = 0U, ring = self->ring[0];
 		self->tx_rings > i; ++i, ring = self->ring[i]) {
 		err = aq_ring_init(&ring[AQ_VEC_TX_ID]);
diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_a0.c b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_a0.c
index faeb4935ef3e..c5a02df7a48b 100644
--- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_a0.c
+++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_a0.c
@@ -629,6 +629,12 @@ static int hw_atl_a0_hw_ring_rx_receive(struct aq_hw_s *self,
 				buff->is_udp_cso = (is_err & 0x10U) ? 0 : 1;
 			else if (0x0U == (pkt_type & 0x1CU))
 				buff->is_tcp_cso = (is_err & 0x10U) ? 0 : 1;
+
+			/* Checksum offload workaround for small packets */
+			if (rxd_wb->pkt_len <= 60) {
+				buff->is_ip_cso = 0U;
+				buff->is_cso_err = 0U;
+			}
 		}
 
 		is_err &= ~0x18U;
diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.c b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.c
index 1bceb7358e5c..21784cc39dab 100644
--- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.c
+++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.c
@@ -645,6 +645,12 @@ static int hw_atl_b0_hw_ring_rx_receive(struct aq_hw_s *self,
 				buff->is_udp_cso = buff->is_cso_err ? 0U : 1U;
 			else if (0x0U == (pkt_type & 0x1CU))
 				buff->is_tcp_cso = buff->is_cso_err ? 0U : 1U;
+
+			/* Checksum offload workaround for small packets */
+			if (rxd_wb->pkt_len <= 60) {
+				buff->is_ip_cso = 0U;
+				buff->is_cso_err = 0U;
+			}
 		}
 
 		is_err &= ~0x18U;
diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.c b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.c
index 8d6d8f5804da..4f5ec9a0fbfb 100644
--- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.c
+++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.c
@@ -141,6 +141,12 @@ static int hw_atl_utils_init_ucp(struct aq_hw_s *self,
 
 	err = hw_atl_utils_ver_match(aq_hw_caps->fw_ver_expected,
 				     aq_hw_read_reg(self, 0x18U));
+
+	if (err < 0)
+		pr_err("%s: Bad FW version detected: expected=%x, actual=%x\n",
+		       AQ_CFG_DRV_NAME,
+		       aq_hw_caps->fw_ver_expected,
+		       aq_hw_read_reg(self, 0x18U));
 	return err;
 }
 
@@ -313,11 +319,11 @@ void hw_atl_utils_mpi_set(struct aq_hw_s *self,
 err_exit:;
 }
 
-int hw_atl_utils_mpi_get_link_status(struct aq_hw_s *self,
-				     struct aq_hw_link_status_s *link_status)
+int hw_atl_utils_mpi_get_link_status(struct aq_hw_s *self)
 {
 	u32 cp0x036C = aq_hw_read_reg(self, HW_ATL_MPI_STATE_ADR);
 	u32 link_speed_mask = cp0x036C >> HW_ATL_MPI_SPEED_SHIFT;
+	struct aq_hw_link_status_s *link_status = &self->aq_link_status;
 
 	if (!link_speed_mask) {
 		link_status->mbps = 0U;
diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.h b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.h
index a66aee51ab5b..e0360a6b2202 100644
--- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.h
+++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.h
@@ -180,8 +180,7 @@ void hw_atl_utils_mpi_set(struct aq_hw_s *self,
 int hw_atl_utils_mpi_set_speed(struct aq_hw_s *self, u32 speed,
 			       enum hal_atl_utils_fw_state_e state);
 
-int hw_atl_utils_mpi_get_link_status(struct aq_hw_s *self,
-				     struct aq_hw_link_status_s *link_status);
+int hw_atl_utils_mpi_get_link_status(struct aq_hw_s *self);
 
 int hw_atl_utils_get_mac_permanent(struct aq_hw_s *self,
 				   struct aq_hw_caps_s *aq_hw_caps,
diff --git a/drivers/net/ethernet/arc/emac_main.c b/drivers/net/ethernet/arc/emac_main.c
index 68de2f2652f2..3241af1ce718 100644
--- a/drivers/net/ethernet/arc/emac_main.c
+++ b/drivers/net/ethernet/arc/emac_main.c
@@ -720,6 +720,18 @@ static int arc_emac_set_address(struct net_device *ndev, void *p)
 	return 0;
 }
 
+static int arc_emac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
+{
+	if (!netif_running(dev))
+		return -EINVAL;
+
+	if (!dev->phydev)
+		return -ENODEV;
+
+	return phy_mii_ioctl(dev->phydev, rq, cmd);
+}
+
+
 static const struct net_device_ops arc_emac_netdev_ops = {
 	.ndo_open		= arc_emac_open,
 	.ndo_stop		= arc_emac_stop,
@@ -727,6 +739,7 @@ static const struct net_device_ops arc_emac_netdev_ops = {
 	.ndo_set_mac_address	= arc_emac_set_address,
 	.ndo_get_stats		= arc_emac_stats,
 	.ndo_set_rx_mode	= arc_emac_set_rx_mode,
+	.ndo_do_ioctl		= arc_emac_ioctl,
 #ifdef CONFIG_NET_POLL_CONTROLLER
 	.ndo_poll_controller	= arc_emac_poll_controller,
 #endif
diff --git a/drivers/net/ethernet/broadcom/Kconfig b/drivers/net/ethernet/broadcom/Kconfig
index 96413808c726..67134ece1107 100644
--- a/drivers/net/ethernet/broadcom/Kconfig
+++ b/drivers/net/ethernet/broadcom/Kconfig
@@ -61,10 +61,12 @@ config BCM63XX_ENET
 
 config BCMGENET
 	tristate "Broadcom GENET internal MAC support"
+	depends on OF && HAS_IOMEM
 	select MII
 	select PHYLIB
 	select FIXED_PHY
 	select BCM7XXX_PHY
+	select MDIO_BCM_UNIMAC
 	help
 	  This driver supports the built-in Ethernet MACs found in the
 	  Broadcom BCM7xxx Set Top Box family chipset.
@@ -193,6 +195,7 @@ config SYSTEMPORT
 config BNXT
 	tristate "Broadcom NetXtreme-C/E support"
 	depends on PCI
+	depends on MAY_USE_DEVLINK
 	select FW_LOADER
 	select LIBCRC32C
 	---help---
@@ -209,6 +212,15 @@ config BNXT_SRIOV
 	  Virtualization support in the NetXtreme-C/E products. This
 	  allows for virtual function acceleration in virtual environments.
 
+config BNXT_FLOWER_OFFLOAD
+	bool "TC Flower offload support for NetXtreme-C/E"
+	depends on BNXT
+	default y
+	---help---
+	  This configuration parameter enables TC Flower packet classifier
+	  offload for eswitch.  This option enables SR-IOV switchdev eswitch
+	  offload.
+
 config BNXT_DCB
 	bool "Data Center Bridging (DCB) Support"
 	default n
diff --git a/drivers/net/ethernet/broadcom/bcm63xx_enet.c b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
index 61a88b64bd39..4f3845a58126 100644
--- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c
+++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
@@ -2674,7 +2674,7 @@ static int bcm_enetsw_set_ringparam(struct net_device *dev,
 	return 0;
 }
 
-static struct ethtool_ops bcm_enetsw_ethtool_ops = {
+static const struct ethtool_ops bcm_enetsw_ethtool_ops = {
 	.get_strings		= bcm_enetsw_get_strings,
 	.get_sset_count		= bcm_enetsw_get_sset_count,
 	.get_ethtool_stats      = bcm_enetsw_get_ethtool_stats,
diff --git a/drivers/net/ethernet/broadcom/bcmsysport.c b/drivers/net/ethernet/broadcom/bcmsysport.c
index dc3052751bc1..c3c53f6cd9e6 100644
--- a/drivers/net/ethernet/broadcom/bcmsysport.c
+++ b/drivers/net/ethernet/broadcom/bcmsysport.c
@@ -32,13 +32,13 @@
 #define BCM_SYSPORT_IO_MACRO(name, offset) \
 static inline u32 name##_readl(struct bcm_sysport_priv *priv, u32 off)	\
 {									\
-	u32 reg = __raw_readl(priv->base + offset + off);		\
+	u32 reg = readl_relaxed(priv->base + offset + off);		\
 	return reg;							\
 }									\
 static inline void name##_writel(struct bcm_sysport_priv *priv,		\
 				  u32 val, u32 off)			\
 {									\
-	__raw_writel(val, priv->base + offset + off);			\
+	writel_relaxed(val, priv->base + offset + off);			\
 }									\
 
 BCM_SYSPORT_IO_MACRO(intrl2_0, SYS_PORT_INTRL2_0_OFFSET);
@@ -59,14 +59,14 @@ static inline u32 rdma_readl(struct bcm_sysport_priv *priv, u32 off)
 {
 	if (priv->is_lite && off >= RDMA_STATUS)
 		off += 4;
-	return __raw_readl(priv->base + SYS_PORT_RDMA_OFFSET + off);
+	return readl_relaxed(priv->base + SYS_PORT_RDMA_OFFSET + off);
 }
 
 static inline void rdma_writel(struct bcm_sysport_priv *priv, u32 val, u32 off)
 {
 	if (priv->is_lite && off >= RDMA_STATUS)
 		off += 4;
-	__raw_writel(val, priv->base + SYS_PORT_RDMA_OFFSET + off);
+	writel_relaxed(val, priv->base + SYS_PORT_RDMA_OFFSET + off);
 }
 
 static inline u32 tdma_control_bit(struct bcm_sysport_priv *priv, u32 bit)
@@ -110,10 +110,10 @@ static inline void dma_desc_set_addr(struct bcm_sysport_priv *priv,
 				     dma_addr_t addr)
 {
 #ifdef CONFIG_PHYS_ADDR_T_64BIT
-	__raw_writel(upper_32_bits(addr) & DESC_ADDR_HI_MASK,
+	writel_relaxed(upper_32_bits(addr) & DESC_ADDR_HI_MASK,
 		     d + DESC_ADDR_HI_STATUS_LEN);
 #endif
-	__raw_writel(lower_32_bits(addr), d + DESC_ADDR_LO);
+	writel_relaxed(lower_32_bits(addr), d + DESC_ADDR_LO);
 }
 
 static inline void tdma_port_write_desc_addr(struct bcm_sysport_priv *priv,
@@ -201,10 +201,10 @@ static int bcm_sysport_set_features(struct net_device *dev,
  */
 static const struct bcm_sysport_stats bcm_sysport_gstrings_stats[] = {
 	/* general stats */
-	STAT_NETDEV(rx_packets),
-	STAT_NETDEV(tx_packets),
-	STAT_NETDEV(rx_bytes),
-	STAT_NETDEV(tx_bytes),
+	STAT_NETDEV64(rx_packets),
+	STAT_NETDEV64(tx_packets),
+	STAT_NETDEV64(rx_bytes),
+	STAT_NETDEV64(tx_bytes),
 	STAT_NETDEV(rx_errors),
 	STAT_NETDEV(tx_errors),
 	STAT_NETDEV(rx_dropped),
@@ -316,6 +316,7 @@ static inline bool bcm_sysport_lite_stat_valid(enum bcm_sysport_stat_type type)
 {
 	switch (type) {
 	case BCM_SYSPORT_STAT_NETDEV:
+	case BCM_SYSPORT_STAT_NETDEV64:
 	case BCM_SYSPORT_STAT_RXCHK:
 	case BCM_SYSPORT_STAT_RBUF:
 	case BCM_SYSPORT_STAT_SOFT:
@@ -398,6 +399,7 @@ static void bcm_sysport_update_mib_counters(struct bcm_sysport_priv *priv)
 		s = &bcm_sysport_gstrings_stats[i];
 		switch (s->type) {
 		case BCM_SYSPORT_STAT_NETDEV:
+		case BCM_SYSPORT_STAT_NETDEV64:
 		case BCM_SYSPORT_STAT_SOFT:
 			continue;
 		case BCM_SYSPORT_STAT_MIB_RX:
@@ -434,7 +436,10 @@ static void bcm_sysport_get_stats(struct net_device *dev,
 				  struct ethtool_stats *stats, u64 *data)
 {
 	struct bcm_sysport_priv *priv = netdev_priv(dev);
+	struct bcm_sysport_stats64 *stats64 = &priv->stats64;
+	struct u64_stats_sync *syncp = &priv->syncp;
 	struct bcm_sysport_tx_ring *ring;
+	unsigned int start;
 	int i, j;
 
 	if (netif_running(dev))
@@ -447,14 +452,22 @@ static void bcm_sysport_get_stats(struct net_device *dev,
 		s = &bcm_sysport_gstrings_stats[i];
 		if (s->type == BCM_SYSPORT_STAT_NETDEV)
 			p = (char *)&dev->stats;
+		else if (s->type == BCM_SYSPORT_STAT_NETDEV64)
+			p = (char *)stats64;
 		else
 			p = (char *)priv;
 
 		if (priv->is_lite && !bcm_sysport_lite_stat_valid(s->type))
 			continue;
-
 		p += s->stat_offset;
-		data[j] = *(unsigned long *)p;
+
+		if (s->stat_sizeof == sizeof(u64))
+			do {
+				start = u64_stats_fetch_begin_irq(syncp);
+				data[i] = *(u64 *)p;
+			} while (u64_stats_fetch_retry_irq(syncp, start));
+		else
+			data[i] = *(u32 *)p;
 		j++;
 	}
 
@@ -597,7 +610,7 @@ static int bcm_sysport_set_coalesce(struct net_device *dev,
 
 static void bcm_sysport_free_cb(struct bcm_sysport_cb *cb)
 {
-	dev_kfree_skb_any(cb->skb);
+	dev_consume_skb_any(cb->skb);
 	cb->skb = NULL;
 	dma_unmap_addr_set(cb, dma_addr, 0);
 }
@@ -666,6 +679,7 @@ static int bcm_sysport_alloc_rx_bufs(struct bcm_sysport_priv *priv)
 static unsigned int bcm_sysport_desc_rx(struct bcm_sysport_priv *priv,
 					unsigned int budget)
 {
+	struct bcm_sysport_stats64 *stats64 = &priv->stats64;
 	struct net_device *ndev = priv->netdev;
 	unsigned int processed = 0, to_process;
 	struct bcm_sysport_cb *cb;
@@ -769,6 +783,10 @@ static unsigned int bcm_sysport_desc_rx(struct bcm_sysport_priv *priv,
 		skb->protocol = eth_type_trans(skb, ndev);
 		ndev->stats.rx_packets++;
 		ndev->stats.rx_bytes += len;
+		u64_stats_update_begin(&priv->syncp);
+		stats64->rx_packets++;
+		stats64->rx_bytes += len;
+		u64_stats_update_end(&priv->syncp);
 
 		napi_gro_receive(&priv->napi, skb);
 next:
@@ -791,17 +809,15 @@ static void bcm_sysport_tx_reclaim_one(struct bcm_sysport_tx_ring *ring,
 	struct device *kdev = &priv->pdev->dev;
 
 	if (cb->skb) {
-		ring->bytes += cb->skb->len;
 		*bytes_compl += cb->skb->len;
 		dma_unmap_single(kdev, dma_unmap_addr(cb, dma_addr),
 				 dma_unmap_len(cb, dma_len),
 				 DMA_TO_DEVICE);
-		ring->packets++;
 		(*pkts_compl)++;
 		bcm_sysport_free_cb(cb);
 	/* SKB fragment */
 	} else if (dma_unmap_addr(cb, dma_addr)) {
-		ring->bytes += dma_unmap_len(cb, dma_len);
+		*bytes_compl += dma_unmap_len(cb, dma_len);
 		dma_unmap_page(kdev, dma_unmap_addr(cb, dma_addr),
 			       dma_unmap_len(cb, dma_len), DMA_TO_DEVICE);
 		dma_unmap_addr_set(cb, dma_addr, 0);
@@ -812,9 +828,9 @@ static void bcm_sysport_tx_reclaim_one(struct bcm_sysport_tx_ring *ring,
 static unsigned int __bcm_sysport_tx_reclaim(struct bcm_sysport_priv *priv,
 					     struct bcm_sysport_tx_ring *ring)
 {
-	struct net_device *ndev = priv->netdev;
 	unsigned int c_index, last_c_index, last_tx_cn, num_tx_cbs;
 	unsigned int pkts_compl = 0, bytes_compl = 0;
+	struct net_device *ndev = priv->netdev;
 	struct bcm_sysport_cb *cb;
 	u32 hw_ind;
 
@@ -853,6 +869,11 @@ static unsigned int __bcm_sysport_tx_reclaim(struct bcm_sysport_priv *priv,
 		last_c_index &= (num_tx_cbs - 1);
 	}
 
+	u64_stats_update_begin(&priv->syncp);
+	ring->packets += pkts_compl;
+	ring->bytes += bytes_compl;
+	u64_stats_update_end(&priv->syncp);
+
 	ring->c_index = c_index;
 
 	netif_dbg(priv, tx_done, ndev,
@@ -1346,6 +1367,8 @@ static int bcm_sysport_init_tx_ring(struct bcm_sysport_priv *priv,
 
 	ring->cbs = kcalloc(size, sizeof(struct bcm_sysport_cb), GFP_KERNEL);
 	if (!ring->cbs) {
+		dma_free_coherent(kdev, sizeof(struct dma_desc),
+				  ring->desc_cpu, ring->desc_dma);
 		netif_err(priv, hw, priv->netdev, "CB allocation failed\n");
 		return -ENOMEM;
 	}
@@ -1369,6 +1392,19 @@ static int bcm_sysport_init_tx_ring(struct bcm_sysport_priv *priv,
 	tdma_writel(priv, RING_IGNORE_STATUS, TDMA_DESC_RING_MAPPING(index));
 	tdma_writel(priv, 0, TDMA_DESC_RING_PCP_DEI_VID(index));
 
+	/* Do not use tdma_control_bit() here because TSB_SWAP1 collides
+	 * with the original definition of ACB_ALGO
+	 */
+	reg = tdma_readl(priv, TDMA_CONTROL);
+	if (priv->is_lite)
+		reg &= ~BIT(TSB_SWAP1);
+	/* Set a correct TSB format based on host endian */
+	if (!IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
+		reg |= tdma_control_bit(priv, TSB_SWAP0);
+	else
+		reg &= ~tdma_control_bit(priv, TSB_SWAP0);
+	tdma_writel(priv, reg, TDMA_CONTROL);
+
 	/* Program the number of descriptors as MAX_THRESHOLD and half of
 	 * its size for the hysteresis trigger
 	 */
@@ -1675,22 +1711,38 @@ static int bcm_sysport_change_mac(struct net_device *dev, void *p)
 	return 0;
 }
 
-static struct net_device_stats *bcm_sysport_get_nstats(struct net_device *dev)
+static void bcm_sysport_get_stats64(struct net_device *dev,
+				    struct rtnl_link_stats64 *stats)
 {
 	struct bcm_sysport_priv *priv = netdev_priv(dev);
-	unsigned long tx_bytes = 0, tx_packets = 0;
+	struct bcm_sysport_stats64 *stats64 = &priv->stats64;
 	struct bcm_sysport_tx_ring *ring;
+	u64 tx_packets = 0, tx_bytes = 0;
+	unsigned int start;
 	unsigned int q;
 
+	netdev_stats_to_stats64(stats, &dev->stats);
+
 	for (q = 0; q < dev->num_tx_queues; q++) {
 		ring = &priv->tx_rings[q];
-		tx_bytes += ring->bytes;
-		tx_packets += ring->packets;
+		do {
+			start = u64_stats_fetch_begin_irq(&priv->syncp);
+			tx_bytes = ring->bytes;
+			tx_packets = ring->packets;
+		} while (u64_stats_fetch_retry_irq(&priv->syncp, start));
+
+		stats->tx_bytes += tx_bytes;
+		stats->tx_packets += tx_packets;
 	}
 
-	dev->stats.tx_bytes = tx_bytes;
-	dev->stats.tx_packets = tx_packets;
-	return &dev->stats;
+	stats64->tx_bytes = stats->tx_bytes;
+	stats64->tx_packets = stats->tx_packets;
+
+	do {
+		start = u64_stats_fetch_begin_irq(&priv->syncp);
+		stats->rx_packets = stats64->rx_packets;
+		stats->rx_bytes = stats64->rx_bytes;
+	} while (u64_stats_fetch_retry_irq(&priv->syncp, start));
 }
 
 static void bcm_sysport_netif_start(struct net_device *dev)
@@ -1722,10 +1774,14 @@ static void rbuf_init(struct bcm_sysport_priv *priv)
 	reg = rbuf_readl(priv, RBUF_CONTROL);
 	reg |= RBUF_4B_ALGN | RBUF_RSB_EN;
 	/* Set a correct RSB format on SYSTEMPORT Lite */
-	if (priv->is_lite) {
+	if (priv->is_lite)
 		reg &= ~RBUF_RSB_SWAP1;
+
+	/* Set a correct RSB format based on host endian */
+	if (!IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
 		reg |= RBUF_RSB_SWAP0;
-	}
+	else
+		reg &= ~RBUF_RSB_SWAP0;
 	rbuf_writel(priv, reg, RBUF_CONTROL);
 }
 
@@ -1954,7 +2010,7 @@ static const struct net_device_ops bcm_sysport_netdev_ops = {
 #ifdef CONFIG_NET_POLL_CONTROLLER
 	.ndo_poll_controller	= bcm_sysport_poll_controller,
 #endif
-	.ndo_get_stats		= bcm_sysport_get_nstats,
+	.ndo_get_stats64	= bcm_sysport_get_stats64,
 };
 
 #define REV_FMT	"v%2x.%02x"
@@ -2102,6 +2158,8 @@ static int bcm_sysport_probe(struct platform_device *pdev)
 	/* libphy will adjust the link state accordingly */
 	netif_carrier_off(dev);
 
+	u64_stats_init(&priv->syncp);
+
 	ret = register_netdev(dev);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to register net_device\n");
diff --git a/drivers/net/ethernet/broadcom/bcmsysport.h b/drivers/net/ethernet/broadcom/bcmsysport.h
index 77a51c167a69..82e401df199e 100644
--- a/drivers/net/ethernet/broadcom/bcmsysport.h
+++ b/drivers/net/ethernet/broadcom/bcmsysport.h
@@ -449,7 +449,8 @@ struct bcm_rsb {
 /* Uses 2 bits on SYSTEMPORT Lite and shifts everything by 1 bit, we
  * keep the SYSTEMPORT layout here and adjust with tdma_control_bit()
  */
-#define  TSB_SWAP			2
+#define  TSB_SWAP0			2
+#define  TSB_SWAP1			3
 #define  ACB_ALGO			3
 #define  BUF_DATA_OFFSET_SHIFT		4
 #define  BUF_DATA_OFFSET_MASK		0x3ff
@@ -603,6 +604,7 @@ struct bcm_sysport_mib {
 /* HW maintains a large list of counters */
 enum bcm_sysport_stat_type {
 	BCM_SYSPORT_STAT_NETDEV = -1,
+	BCM_SYSPORT_STAT_NETDEV64,
 	BCM_SYSPORT_STAT_MIB_RX,
 	BCM_SYSPORT_STAT_MIB_TX,
 	BCM_SYSPORT_STAT_RUNT,
@@ -619,6 +621,13 @@ enum bcm_sysport_stat_type {
 	.type = BCM_SYSPORT_STAT_NETDEV, \
 }
 
+#define STAT_NETDEV64(m) { \
+	.stat_string = __stringify(m), \
+	.stat_sizeof = sizeof(((struct bcm_sysport_stats64 *)0)->m), \
+	.stat_offset = offsetof(struct bcm_sysport_stats64, m), \
+	.type = BCM_SYSPORT_STAT_NETDEV64, \
+}
+
 #define STAT_MIB(str, m, _type) { \
 	.stat_string = str, \
 	.stat_sizeof = sizeof(((struct bcm_sysport_priv *)0)->m), \
@@ -659,6 +668,14 @@ struct bcm_sysport_stats {
 	u16 reg_offset;
 };
 
+struct bcm_sysport_stats64 {
+	/* 64bit stats on 32bit/64bit Machine */
+	u64	rx_packets;
+	u64	rx_bytes;
+	u64	tx_packets;
+	u64	tx_bytes;
+};
+
 /* Software house keeping helper structure */
 struct bcm_sysport_cb {
 	struct sk_buff	*skb;		/* SKB for RX packets */
@@ -743,5 +760,10 @@ struct bcm_sysport_priv {
 
 	/* Ethtool */
 	u32			msg_enable;
+
+	struct bcm_sysport_stats64	stats64;
+
+	/* For atomic update generic 64bit value on 32bit Machine */
+	struct u64_stats_sync	syncp;
 };
 #endif /* __BCM_SYSPORT_H */
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
index 67fe3d826566..1216c1f1e052 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
@@ -4284,15 +4284,17 @@ int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
 	return 0;
 }
 
-int __bnx2x_setup_tc(struct net_device *dev, u32 handle, u32 chain_index,
-		     __be16 proto, struct tc_to_netdev *tc)
+int __bnx2x_setup_tc(struct net_device *dev, enum tc_setup_type type,
+		     void *type_data)
 {
-	if (tc->type != TC_SETUP_MQPRIO)
-		return -EINVAL;
+	struct tc_mqprio_qopt *mqprio = type_data;
+
+	if (type != TC_SETUP_MQPRIO)
+		return -EOPNOTSUPP;
 
-	tc->mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
+	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
 
-	return bnx2x_setup_tc(dev, tc->mqprio->num_tc);
+	return bnx2x_setup_tc(dev, mqprio->num_tc);
 }
 
 /* called with rtnl_lock */
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.h b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.h
index c26688d2f326..a5265e1344f1 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.h
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.h
@@ -486,8 +486,8 @@ netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev);
 
 /* setup_tc callback */
 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc);
-int __bnx2x_setup_tc(struct net_device *dev, u32 handle, u32 chain_index,
-		     __be16 proto, struct tc_to_netdev *tc);
+int __bnx2x_setup_tc(struct net_device *dev, enum tc_setup_type type,
+		     void *type_data);
 
 int bnx2x_get_vf_config(struct net_device *dev, int vf,
 			struct ifla_vf_info *ivi);
diff --git a/drivers/net/ethernet/broadcom/bnxt/Makefile b/drivers/net/ethernet/broadcom/bnxt/Makefile
index a7ca45b251cb..4f0cb8e1ffc0 100644
--- a/drivers/net/ethernet/broadcom/bnxt/Makefile
+++ b/drivers/net/ethernet/broadcom/bnxt/Makefile
@@ -1,3 +1,3 @@
 obj-$(CONFIG_BNXT) += bnxt_en.o
 
-bnxt_en-y := bnxt.o bnxt_sriov.o bnxt_ethtool.o bnxt_dcb.o bnxt_ulp.o bnxt_xdp.o
+bnxt_en-y := bnxt.o bnxt_sriov.o bnxt_ethtool.o bnxt_dcb.o bnxt_ulp.o bnxt_xdp.o bnxt_vfr.o bnxt_tc.o
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
index e7c8539cbddf..aacec8bc19d5 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
@@ -33,6 +33,7 @@
 #include <linux/mii.h>
 #include <linux/if.h>
 #include <linux/if_vlan.h>
+#include <linux/if_bridge.h>
 #include <linux/rtc.h>
 #include <linux/bpf.h>
 #include <net/ip.h>
@@ -48,6 +49,8 @@
 #include <linux/aer.h>
 #include <linux/bitmap.h>
 #include <linux/cpu_rmap.h>
+#include <linux/cpumask.h>
+#include <net/pkt_cls.h>
 
 #include "bnxt_hsi.h"
 #include "bnxt.h"
@@ -56,6 +59,8 @@
 #include "bnxt_ethtool.h"
 #include "bnxt_dcb.h"
 #include "bnxt_xdp.h"
+#include "bnxt_vfr.h"
+#include "bnxt_tc.h"
 
 #define BNXT_TX_TIMEOUT		(5 * HZ)
 
@@ -101,6 +106,8 @@ enum board_idx {
 	BCM57416_NPAR,
 	BCM57452,
 	BCM57454,
+	BCM58802,
+	BCM58808,
 	NETXTREME_E_VF,
 	NETXTREME_C_VF,
 };
@@ -109,39 +116,42 @@ enum board_idx {
 static const struct {
 	char *name;
 } board_info[] = {
-	{ "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
-	{ "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
-	{ "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
-	{ "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
-	{ "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
-	{ "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
-	{ "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
-	{ "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
-	{ "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
-	{ "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
-	{ "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
-	{ "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
-	{ "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
-	{ "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
-	{ "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
-	{ "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
-	{ "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
-	{ "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
-	{ "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
-	{ "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
-	{ "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
-	{ "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
-	{ "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
-	{ "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
-	{ "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
-	{ "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
-	{ "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
-	{ "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
-	{ "Broadcom NetXtreme-E Ethernet Virtual Function" },
-	{ "Broadcom NetXtreme-C Ethernet Virtual Function" },
+	[BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
+	[BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
+	[BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
+	[BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
+	[BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
+	[BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
+	[BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
+	[BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
+	[BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
+	[BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
+	[BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
+	[BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
+	[BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
+	[BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
+	[BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
+	[BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
+	[BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
+	[BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
+	[BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
+	[BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
+	[BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
+	[BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
+	[BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
+	[BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
+	[BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
+	[BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
+	[BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
+	[BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
+	[BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
+	[BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
+	[NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
+	[NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
 };
 
 static const struct pci_device_id bnxt_pci_tbl[] = {
+	{ PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
 	{ PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
 	{ PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
 	{ PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
@@ -172,8 +182,9 @@ static const struct pci_device_id bnxt_pci_tbl[] = {
 	{ PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
 	{ PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
 	{ PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
+	{ PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
 	{ PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
-	{ PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
+	{ PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
 #ifdef CONFIG_BNXT_SRIOV
 	{ PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
 	{ PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
@@ -243,6 +254,16 @@ const u16 bnxt_lhint_arr[] = {
 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 };
 
+static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
+{
+	struct metadata_dst *md_dst = skb_metadata_dst(skb);
+
+	if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
+		return 0;
+
+	return md_dst->u.port_info.port_id;
+}
+
 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 {
 	struct bnxt *bp = netdev_priv(dev);
@@ -287,7 +308,7 @@ static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 	tx_buf->nr_frags = last_frag;
 
 	vlan_tag_flags = 0;
-	cfa_action = 0;
+	cfa_action = bnxt_xmit_get_cfa_action(skb);
 	if (skb_vlan_tag_present(skb)) {
 		vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
 				 skb_vlan_tag_get(skb);
@@ -322,7 +343,8 @@ static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 			tx_push1->tx_bd_hsize_lflags = 0;
 
 		tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
-		tx_push1->tx_bd_cfa_action = cpu_to_le32(cfa_action);
+		tx_push1->tx_bd_cfa_action =
+			cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 
 		end = pdata + length;
 		end = PTR_ALIGN(end, 8) - 1;
@@ -427,7 +449,8 @@ normal_tx:
 	txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 
 	txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
-	txbd1->tx_bd_cfa_action = cpu_to_le32(cfa_action);
+	txbd1->tx_bd_cfa_action =
+			cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 	for (i = 0; i < last_frag; i++) {
 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 
@@ -1032,7 +1055,10 @@ static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
 		bnxt_sched_reset(bp, rxr);
 		return;
 	}
-
+	/* Store cfa_code in tpa_info to use in tpa_end
+	 * completion processing.
+	 */
+	tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
 	prod_rx_buf->data = tpa_info->data;
 	prod_rx_buf->data_ptr = tpa_info->data_ptr;
 
@@ -1267,6 +1293,17 @@ static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
 	return skb;
 }
 
+/* Given the cfa_code of a received packet determine which
+ * netdev (vf-rep or PF) the packet is destined to.
+ */
+static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
+{
+	struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
+
+	/* if vf-rep dev is NULL, the must belongs to the PF */
+	return dev ? dev : bp->dev;
+}
+
 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
 					   struct bnxt_napi *bnapi,
 					   u32 *raw_cons,
@@ -1360,7 +1397,9 @@ static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
 			return NULL;
 		}
 	}
-	skb->protocol = eth_type_trans(skb, bp->dev);
+
+	skb->protocol =
+		eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
 
 	if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
 		skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
@@ -1387,6 +1426,18 @@ static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
 	return skb;
 }
 
+static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
+			     struct sk_buff *skb)
+{
+	if (skb->dev != bp->dev) {
+		/* this packet belongs to a vf-rep */
+		bnxt_vf_rep_rx(bp, skb);
+		return;
+	}
+	skb_record_rx_queue(skb, bnapi->index);
+	napi_gro_receive(&bnapi->napi, skb);
+}
+
 /* returns the following:
  * 1       - 1 packet successfully received
  * 0       - successful TPA_START, packet not completed yet
@@ -1403,7 +1454,7 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi *bnapi, u32 *raw_cons,
 	struct rx_cmp *rxcmp;
 	struct rx_cmp_ext *rxcmp1;
 	u32 tmp_raw_cons = *raw_cons;
-	u16 cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
+	u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
 	struct bnxt_sw_rx_bd *rx_buf;
 	unsigned int len;
 	u8 *data_ptr, agg_bufs, cmp_type;
@@ -1445,8 +1496,7 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi *bnapi, u32 *raw_cons,
 
 		rc = -ENOMEM;
 		if (likely(skb)) {
-			skb_record_rx_queue(skb, bnapi->index);
-			napi_gro_receive(&bnapi->napi, skb);
+			bnxt_deliver_skb(bp, bnapi, skb);
 			rc = 1;
 		}
 		*event |= BNXT_RX_EVENT;
@@ -1535,7 +1585,8 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi *bnapi, u32 *raw_cons,
 		skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
 	}
 
-	skb->protocol = eth_type_trans(skb, dev);
+	cfa_code = RX_CMP_CFA_CODE(rxcmp1);
+	skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
 
 	if ((rxcmp1->rx_cmp_flags2 &
 	     cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
@@ -1560,8 +1611,7 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi *bnapi, u32 *raw_cons,
 		}
 	}
 
-	skb_record_rx_queue(skb, bnapi->index);
-	napi_gro_receive(&bnapi->napi, skb);
+	bnxt_deliver_skb(bp, bnapi, skb);
 	rc = 1;
 
 next_rx:
@@ -1802,6 +1852,13 @@ static int bnxt_poll_work(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
 							   &event);
 			if (likely(rc >= 0))
 				rx_pkts += rc;
+			/* Increment rx_pkts when rc is -ENOMEM to count towards
+			 * the NAPI budget.  Otherwise, we may potentially loop
+			 * here forever if we consistently cannot allocate
+			 * buffers.
+			 */
+			else if (rc == -ENOMEM)
+				rx_pkts++;
 			else if (rc == -EBUSY)	/* partial completion */
 				break;
 		} else if (unlikely((TX_CMP_TYPE(txcmp) ==
@@ -4420,9 +4477,33 @@ static int bnxt_hwrm_reserve_tx_rings(struct bnxt *bp, int *tx_rings)
 	mutex_lock(&bp->hwrm_cmd_lock);
 	rc = __bnxt_hwrm_get_tx_rings(bp, 0xffff, tx_rings);
 	mutex_unlock(&bp->hwrm_cmd_lock);
+	if (!rc)
+		bp->tx_reserved_rings = *tx_rings;
 	return rc;
 }
 
+static int bnxt_hwrm_check_tx_rings(struct bnxt *bp, int tx_rings)
+{
+	struct hwrm_func_cfg_input req = {0};
+	int rc;
+
+	if (bp->hwrm_spec_code < 0x10801)
+		return 0;
+
+	if (BNXT_VF(bp))
+		return 0;
+
+	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
+	req.fid = cpu_to_le16(0xffff);
+	req.flags = cpu_to_le32(FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST);
+	req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS);
+	req.num_tx_rings = cpu_to_le16(tx_rings);
+	rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
+	if (rc)
+		return -ENOMEM;
+	return 0;
+}
+
 static void bnxt_hwrm_set_coal_params(struct bnxt *bp, u32 max_bufs,
 	u32 buf_tmrs, u16 flags,
 	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
@@ -4577,6 +4658,7 @@ static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
 {
 	struct hwrm_func_qcfg_input req = {0};
 	struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
+	u16 flags;
 	int rc;
 
 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
@@ -4593,15 +4675,15 @@ static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
 		vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
 	}
 #endif
-	if (BNXT_PF(bp)) {
-		u16 flags = le16_to_cpu(resp->flags);
-
-		if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
-			     FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED))
-			bp->flags |= BNXT_FLAG_FW_LLDP_AGENT;
-		if (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST)
-			bp->flags |= BNXT_FLAG_MULTI_HOST;
+	flags = le16_to_cpu(resp->flags);
+	if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
+		     FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
+		bp->flags |= BNXT_FLAG_FW_LLDP_AGENT;
+		if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
+			bp->flags |= BNXT_FLAG_FW_DCBX_AGENT;
 	}
+	if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
+		bp->flags |= BNXT_FLAG_MULTI_HOST;
 
 	switch (resp->port_partition_type) {
 	case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
@@ -4610,6 +4692,13 @@ static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
 		bp->port_partition_type = resp->port_partition_type;
 		break;
 	}
+	if (bp->hwrm_spec_code < 0x10707 ||
+	    resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
+		bp->br_mode = BRIDGE_MODE_VEB;
+	else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
+		bp->br_mode = BRIDGE_MODE_VEPA;
+	else
+		bp->br_mode = BRIDGE_MODE_UNDEF;
 
 func_qcfg_exit:
 	mutex_unlock(&bp->hwrm_cmd_lock);
@@ -4647,7 +4736,6 @@ static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
 		pf->port_id = le16_to_cpu(resp->port_id);
 		bp->dev->dev_port = pf->port_id;
 		memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
-		memcpy(bp->dev->dev_addr, pf->mac_addr, ETH_ALEN);
 		pf->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
 		pf->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
 		pf->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
@@ -4687,16 +4775,6 @@ static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
 		vf->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
 
 		memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
-		mutex_unlock(&bp->hwrm_cmd_lock);
-
-		if (is_valid_ether_addr(vf->mac_addr)) {
-			/* overwrite netdev dev_adr with admin VF MAC */
-			memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
-		} else {
-			eth_hw_addr_random(bp->dev);
-			rc = bnxt_approve_mac(bp, bp->dev->dev_addr);
-		}
-		return rc;
 #endif
 	}
 
@@ -4911,6 +4989,26 @@ static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
 	}
 }
 
+static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
+{
+	struct hwrm_func_cfg_input req = {0};
+	int rc;
+
+	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
+	req.fid = cpu_to_le16(0xffff);
+	req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
+	if (br_mode == BRIDGE_MODE_VEB)
+		req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
+	else if (br_mode == BRIDGE_MODE_VEPA)
+		req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
+	else
+		return -EINVAL;
+	rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
+	if (rc)
+		rc = -EIO;
+	return rc;
+}
+
 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
 {
 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
@@ -5046,6 +5144,15 @@ static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
 				   rc);
 			goto err_out;
 		}
+		if (bp->tx_reserved_rings != bp->tx_nr_rings) {
+			int tx = bp->tx_nr_rings;
+
+			if (bnxt_hwrm_reserve_tx_rings(bp, &tx) ||
+			    tx < bp->tx_nr_rings) {
+				rc = -ENOMEM;
+				goto err_out;
+			}
+		}
 	}
 
 	rc = bnxt_hwrm_ring_alloc(bp);
@@ -5452,8 +5559,15 @@ static void bnxt_free_irq(struct bnxt *bp)
 
 	for (i = 0; i < bp->cp_nr_rings; i++) {
 		irq = &bp->irq_tbl[i];
-		if (irq->requested)
+		if (irq->requested) {
+			if (irq->have_cpumask) {
+				irq_set_affinity_hint(irq->vector, NULL);
+				free_cpumask_var(irq->cpu_mask);
+				irq->have_cpumask = 0;
+			}
 			free_irq(irq->vector, bp->bnapi[i]);
+		}
+
 		irq->requested = 0;
 	}
 }
@@ -5486,6 +5600,21 @@ static int bnxt_request_irq(struct bnxt *bp)
 			break;
 
 		irq->requested = 1;
+
+		if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
+			int numa_node = dev_to_node(&bp->pdev->dev);
+
+			irq->have_cpumask = 1;
+			cpumask_set_cpu(cpumask_local_spread(i, numa_node),
+					irq->cpu_mask);
+			rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
+			if (rc) {
+				netdev_warn(bp->dev,
+					    "Set affinity failed, IRQ = %d\n",
+					    irq->vector);
+				break;
+			}
+		}
 	}
 	return rc;
 }
@@ -5559,12 +5688,10 @@ void bnxt_tx_disable(struct bnxt *bp)
 {
 	int i;
 	struct bnxt_tx_ring_info *txr;
-	struct netdev_queue *txq;
 
 	if (bp->tx_ring) {
 		for (i = 0; i < bp->tx_nr_rings; i++) {
 			txr = &bp->tx_ring[i];
-			txq = netdev_get_tx_queue(bp->dev, i);
 			txr->dev_state = BNXT_DEV_STATE_CLOSING;
 		}
 	}
@@ -5577,11 +5704,9 @@ void bnxt_tx_enable(struct bnxt *bp)
 {
 	int i;
 	struct bnxt_tx_ring_info *txr;
-	struct netdev_queue *txq;
 
 	for (i = 0; i < bp->tx_nr_rings; i++) {
 		txr = &bp->tx_ring[i];
-		txq = netdev_get_tx_queue(bp->dev, i);
 		txr->dev_state = 0;
 	}
 	netif_tx_wake_all_queues(bp->dev);
@@ -5646,7 +5771,7 @@ static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
 	if (rc)
 		goto hwrm_phy_qcaps_exit;
 
-	if (resp->eee_supported & PORT_PHY_QCAPS_RESP_EEE_SUPPORTED) {
+	if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
 		struct ethtool_eee *eee = &bp->eee;
 		u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
 
@@ -5661,6 +5786,8 @@ static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
 		link_info->support_auto_speeds =
 			le16_to_cpu(resp->supported_speeds_auto_mode);
 
+	bp->port_count = resp->port_cnt;
+
 hwrm_phy_qcaps_exit:
 	mutex_unlock(&bp->hwrm_cmd_lock);
 	return rc;
@@ -5686,13 +5813,15 @@ static int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
 
 	memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
 	link_info->phy_link_status = resp->link;
-	link_info->duplex =  resp->duplex;
+	link_info->duplex = resp->duplex_cfg;
+	if (bp->hwrm_spec_code >= 0x10800)
+		link_info->duplex = resp->duplex_state;
 	link_info->pause = resp->pause;
 	link_info->auto_mode = resp->auto_mode;
 	link_info->auto_pause_setting = resp->auto_pause;
 	link_info->lp_pause = resp->link_partner_adv_pause;
 	link_info->force_pause_setting = resp->force_pause;
-	link_info->duplex_setting = resp->duplex;
+	link_info->duplex_setting = resp->duplex_cfg;
 	if (link_info->phy_link_status == BNXT_LINK_LINK)
 		link_info->link_speed = le16_to_cpu(resp->link_speed);
 	else
@@ -6214,6 +6343,9 @@ static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
 	/* Poll link status and check for SFP+ module status */
 	bnxt_get_port_module_status(bp);
 
+	/* VF-reps may need to be re-opened after the PF is re-opened */
+	if (BNXT_PF(bp))
+		bnxt_vf_reps_open(bp);
 	return 0;
 
 open_err:
@@ -6302,6 +6434,10 @@ int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
 		if (rc)
 			netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
 	}
+
+	/* Close the VF-reps before closing PF */
+	if (BNXT_PF(bp))
+		bnxt_vf_reps_close(bp);
 #endif
 	/* Change device state to avoid TX queue wake up's */
 	bnxt_tx_disable(bp);
@@ -6813,7 +6949,8 @@ static void bnxt_timer(unsigned long data)
 	if (atomic_read(&bp->intr_sem) != 0)
 		goto bnxt_restart_timer;
 
-	if (bp->link_info.link_up && (bp->flags & BNXT_FLAG_PORT_STATS)) {
+	if (bp->link_info.link_up && (bp->flags & BNXT_FLAG_PORT_STATS) &&
+	    bp->stats_coal_ticks) {
 		set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
 		schedule_work(&bp->sp_task);
 	}
@@ -6923,8 +7060,8 @@ static void bnxt_sp_task(struct work_struct *work)
 }
 
 /* Under rtnl_lock */
-int bnxt_reserve_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
-		       int tx_xdp)
+int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
+		     int tx_xdp)
 {
 	int max_rx, max_tx, tx_sets = 1;
 	int tx_rings_needed;
@@ -6944,10 +7081,7 @@ int bnxt_reserve_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
 	if (max_tx < tx_rings_needed)
 		return -ENOMEM;
 
-	if (bnxt_hwrm_reserve_tx_rings(bp, &tx_rings_needed) ||
-	    tx_rings_needed < (tx * tx_sets + tx_xdp))
-		return -ENOMEM;
-	return 0;
+	return bnxt_hwrm_check_tx_rings(bp, tx_rings_needed);
 }
 
 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
@@ -7136,8 +7270,8 @@ int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
 	if (bp->flags & BNXT_FLAG_SHARED_RINGS)
 		sh = true;
 
-	rc = bnxt_reserve_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
-				sh, tc, bp->tx_nr_rings_xdp);
+	rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
+			      sh, tc, bp->tx_nr_rings_xdp);
 	if (rc)
 		return rc;
 
@@ -7152,6 +7286,7 @@ int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
 		bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
 		netdev_reset_tc(dev);
 	}
+	bp->tx_nr_rings += bp->tx_nr_rings_xdp;
 	bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
 			       bp->tx_nr_rings + bp->rx_nr_rings;
 	bp->num_stat_ctxs = bp->cp_nr_rings;
@@ -7162,15 +7297,33 @@ int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
 	return 0;
 }
 
-static int bnxt_setup_tc(struct net_device *dev, u32 handle, u32 chain_index,
-			 __be16 proto, struct tc_to_netdev *ntc)
+static int bnxt_setup_flower(struct net_device *dev,
+			     struct tc_cls_flower_offload *cls_flower)
 {
-	if (ntc->type != TC_SETUP_MQPRIO)
-		return -EINVAL;
+	struct bnxt *bp = netdev_priv(dev);
+
+	if (BNXT_VF(bp))
+		return -EOPNOTSUPP;
+
+	return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, cls_flower);
+}
 
-	ntc->mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
+static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
+			 void *type_data)
+{
+	switch (type) {
+	case TC_SETUP_CLSFLOWER:
+		return bnxt_setup_flower(dev, type_data);
+	case TC_SETUP_MQPRIO: {
+		struct tc_mqprio_qopt *mqprio = type_data;
+
+		mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
 
-	return bnxt_setup_mq_tc(dev, ntc->mqprio->num_tc);
+		return bnxt_setup_mq_tc(dev, mqprio->num_tc);
+	}
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
 #ifdef CONFIG_RFS_ACCEL
@@ -7422,6 +7575,102 @@ static void bnxt_udp_tunnel_del(struct net_device *dev,
 	schedule_work(&bp->sp_task);
 }
 
+static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
+			       struct net_device *dev, u32 filter_mask,
+			       int nlflags)
+{
+	struct bnxt *bp = netdev_priv(dev);
+
+	return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
+				       nlflags, filter_mask, NULL);
+}
+
+static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
+			       u16 flags)
+{
+	struct bnxt *bp = netdev_priv(dev);
+	struct nlattr *attr, *br_spec;
+	int rem, rc = 0;
+
+	if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
+		return -EOPNOTSUPP;
+
+	br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
+	if (!br_spec)
+		return -EINVAL;
+
+	nla_for_each_nested(attr, br_spec, rem) {
+		u16 mode;
+
+		if (nla_type(attr) != IFLA_BRIDGE_MODE)
+			continue;
+
+		if (nla_len(attr) < sizeof(mode))
+			return -EINVAL;
+
+		mode = nla_get_u16(attr);
+		if (mode == bp->br_mode)
+			break;
+
+		rc = bnxt_hwrm_set_br_mode(bp, mode);
+		if (!rc)
+			bp->br_mode = mode;
+		break;
+	}
+	return rc;
+}
+
+static int bnxt_get_phys_port_name(struct net_device *dev, char *buf,
+				   size_t len)
+{
+	struct bnxt *bp = netdev_priv(dev);
+	int rc;
+
+	/* The PF and it's VF-reps only support the switchdev framework */
+	if (!BNXT_PF(bp))
+		return -EOPNOTSUPP;
+
+	rc = snprintf(buf, len, "p%d", bp->pf.port_id);
+
+	if (rc >= len)
+		return -EOPNOTSUPP;
+	return 0;
+}
+
+int bnxt_port_attr_get(struct bnxt *bp, struct switchdev_attr *attr)
+{
+	if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
+		return -EOPNOTSUPP;
+
+	/* The PF and it's VF-reps only support the switchdev framework */
+	if (!BNXT_PF(bp))
+		return -EOPNOTSUPP;
+
+	switch (attr->id) {
+	case SWITCHDEV_ATTR_ID_PORT_PARENT_ID:
+		/* In SRIOV each PF-pool (PF + child VFs) serves as a
+		 * switching domain, the PF's perm mac-addr can be used
+		 * as the unique parent-id
+		 */
+		attr->u.ppid.id_len = ETH_ALEN;
+		ether_addr_copy(attr->u.ppid.id, bp->pf.mac_addr);
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+	return 0;
+}
+
+static int bnxt_swdev_port_attr_get(struct net_device *dev,
+				    struct switchdev_attr *attr)
+{
+	return bnxt_port_attr_get(netdev_priv(dev), attr);
+}
+
+static const struct switchdev_ops bnxt_switchdev_ops = {
+	.switchdev_port_attr_get	= bnxt_swdev_port_attr_get
+};
+
 static const struct net_device_ops bnxt_netdev_ops = {
 	.ndo_open		= bnxt_open,
 	.ndo_start_xmit		= bnxt_start_xmit,
@@ -7453,6 +7702,9 @@ static const struct net_device_ops bnxt_netdev_ops = {
 	.ndo_udp_tunnel_add	= bnxt_udp_tunnel_add,
 	.ndo_udp_tunnel_del	= bnxt_udp_tunnel_del,
 	.ndo_xdp		= bnxt_xdp,
+	.ndo_bridge_getlink	= bnxt_bridge_getlink,
+	.ndo_bridge_setlink	= bnxt_bridge_setlink,
+	.ndo_get_phys_port_name = bnxt_get_phys_port_name
 };
 
 static void bnxt_remove_one(struct pci_dev *pdev)
@@ -7460,11 +7712,14 @@ static void bnxt_remove_one(struct pci_dev *pdev)
 	struct net_device *dev = pci_get_drvdata(pdev);
 	struct bnxt *bp = netdev_priv(dev);
 
-	if (BNXT_PF(bp))
+	if (BNXT_PF(bp)) {
 		bnxt_sriov_disable(bp);
+		bnxt_dl_unregister(bp);
+	}
 
 	pci_disable_pcie_error_reporting(pdev);
 	unregister_netdev(dev);
+	bnxt_shutdown_tc(bp);
 	cancel_work_sync(&bp->sp_task);
 	bp->sp_event = 0;
 
@@ -7633,6 +7888,9 @@ static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
 	if (sh)
 		bp->flags |= BNXT_FLAG_SHARED_RINGS;
 	dflt_rings = netif_get_num_default_rss_queues();
+	/* Reduce default rings to reduce memory usage on multi-port cards */
+	if (bp->port_count > 1)
+		dflt_rings = min_t(int, dflt_rings, 4);
 	rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
 	if (rc)
 		return rc;
@@ -7661,6 +7919,28 @@ void bnxt_restore_pf_fw_resources(struct bnxt *bp)
 	bnxt_subtract_ulp_resources(bp, BNXT_ROCE_ULP);
 }
 
+static int bnxt_init_mac_addr(struct bnxt *bp)
+{
+	int rc = 0;
+
+	if (BNXT_PF(bp)) {
+		memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN);
+	} else {
+#ifdef CONFIG_BNXT_SRIOV
+		struct bnxt_vf_info *vf = &bp->vf;
+
+		if (is_valid_ether_addr(vf->mac_addr)) {
+			/* overwrite netdev dev_adr with admin VF MAC */
+			memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
+		} else {
+			eth_hw_addr_random(bp->dev);
+			rc = bnxt_approve_mac(bp, bp->dev->dev_addr);
+		}
+#endif
+	}
+	return rc;
+}
+
 static void bnxt_parse_log_pcie_link(struct bnxt *bp)
 {
 	enum pcie_link_width width = PCIE_LNK_WIDTH_UNKNOWN;
@@ -7710,6 +7990,7 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	dev->netdev_ops = &bnxt_netdev_ops;
 	dev->watchdog_timeo = BNXT_TX_TIMEOUT;
 	dev->ethtool_ops = &bnxt_ethtool_ops;
+	SWITCHDEV_SET_OPS(dev, &bnxt_switchdev_ops);
 	pci_set_drvdata(pdev, dev);
 
 	rc = bnxt_alloc_hwrm_resources(bp);
@@ -7764,6 +8045,7 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 #ifdef CONFIG_BNXT_SRIOV
 	init_waitqueue_head(&bp->sriov_cfg_wait);
+	mutex_init(&bp->sriov_lock);
 #endif
 	bp->gro_func = bnxt_gro_func_5730x;
 	if (BNXT_CHIP_P4_PLUS(bp))
@@ -7789,7 +8071,12 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 		rc = -1;
 		goto init_err_pci_clean;
 	}
-
+	rc = bnxt_init_mac_addr(bp);
+	if (rc) {
+		dev_err(&pdev->dev, "Unable to initialize mac address.\n");
+		rc = -EADDRNOTAVAIL;
+		goto init_err_pci_clean;
+	}
 	rc = bnxt_hwrm_queue_qportcfg(bp);
 	if (rc) {
 		netdev_err(bp->dev, "hwrm query qportcfg failure rc: %x\n",
@@ -7803,6 +8090,10 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	bnxt_ethtool_init(bp);
 	bnxt_dcb_init(bp);
 
+	rc = bnxt_probe_phy(bp);
+	if (rc)
+		goto init_err_pci_clean;
+
 	bnxt_set_rx_skb_mode(bp, false);
 	bnxt_set_tpa_flags(bp);
 	bnxt_set_ring_params(bp);
@@ -7837,10 +8128,6 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if (dev->hw_features & NETIF_F_HW_VLAN_CTAG_RX)
 		bp->flags |= BNXT_FLAG_STRIP_VLAN;
 
-	rc = bnxt_probe_phy(bp);
-	if (rc)
-		goto init_err_pci_clean;
-
 	rc = bnxt_init_int_mode(bp);
 	if (rc)
 		goto init_err_pci_clean;
@@ -7851,9 +8138,15 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	else
 		device_set_wakeup_capable(&pdev->dev, false);
 
+	if (BNXT_PF(bp))
+		bnxt_init_tc(bp);
+
 	rc = register_netdev(dev);
 	if (rc)
-		goto init_err_clr_int;
+		goto init_err_cleanup_tc;
+
+	if (BNXT_PF(bp))
+		bnxt_dl_register(bp);
 
 	netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
 		    board_info[ent->driver_data].name,
@@ -7863,7 +8156,8 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	return 0;
 
-init_err_clr_int:
+init_err_cleanup_tc:
+	bnxt_shutdown_tc(bp);
 	bnxt_clear_int_mode(bp);
 
 init_err_pci_clean:
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
index f34691f85602..7b888d4b2b55 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
@@ -12,13 +12,17 @@
 #define BNXT_H
 
 #define DRV_MODULE_NAME		"bnxt_en"
-#define DRV_MODULE_VERSION	"1.7.0"
+#define DRV_MODULE_VERSION	"1.8.0"
 
 #define DRV_VER_MAJ	1
-#define DRV_VER_MIN	7
+#define DRV_VER_MIN	8
 #define DRV_VER_UPD	0
 
 #include <linux/interrupt.h>
+#include <linux/rhashtable.h>
+#include <net/devlink.h>
+#include <net/dst_metadata.h>
+#include <net/switchdev.h>
 
 struct tx_bd {
 	__le32 tx_bd_len_flags_type;
@@ -242,6 +246,10 @@ struct rx_cmp_ext {
 	    ((le32_to_cpu((rxcmp1)->rx_cmp_flags2) &			\
 	     RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3)
 
+#define RX_CMP_CFA_CODE(rxcmpl1)					\
+	((le32_to_cpu((rxcmpl1)->rx_cmp_cfa_code_errors_v2) &		\
+	  RX_CMPL_CFA_CODE_MASK) >> RX_CMPL_CFA_CODE_SFT)
+
 struct rx_agg_cmp {
 	__le32 rx_agg_cmp_len_flags_type;
 	#define RX_AGG_CMP_TYPE					(0x3f << 0)
@@ -311,6 +319,10 @@ struct rx_tpa_start_cmp_ext {
 	__le32 rx_tpa_start_cmp_hdr_info;
 };
 
+#define TPA_START_CFA_CODE(rx_tpa_start)				\
+	((le32_to_cpu((rx_tpa_start)->rx_tpa_start_cmp_cfa_code_v2) &	\
+	 RX_TPA_START_CMP_CFA_CODE) >> RX_TPA_START_CMPL_CFA_CODE_SHIFT)
+
 struct rx_tpa_end_cmp {
 	__le32 rx_tpa_end_cmp_len_flags_type;
 	#define RX_TPA_END_CMP_TYPE				(0x3f << 0)
@@ -618,6 +630,8 @@ struct bnxt_tpa_info {
 
 #define BNXT_TPA_OUTER_L3_OFF(hdr_info)	\
 	((hdr_info) & 0x1ff)
+
+	u16			cfa_code; /* cfa_code in TPA start compl */
 };
 
 struct bnxt_rx_ring_info {
@@ -688,8 +702,10 @@ struct bnxt_napi {
 struct bnxt_irq {
 	irq_handler_t	handler;
 	unsigned int	vector;
-	u8		requested;
+	u8		requested:1;
+	u8		have_cpumask:1;
 	char		name[IFNAMSIZ + 2];
+	cpumask_var_t	cpu_mask;
 };
 
 #define HWRM_RING_ALLOC_TX	0x1
@@ -825,8 +841,8 @@ struct bnxt_link_info {
 	u8			loop_back;
 	u8			link_up;
 	u8			duplex;
-#define BNXT_LINK_DUPLEX_HALF	PORT_PHY_QCFG_RESP_DUPLEX_HALF
-#define BNXT_LINK_DUPLEX_FULL	PORT_PHY_QCFG_RESP_DUPLEX_FULL
+#define BNXT_LINK_DUPLEX_HALF	PORT_PHY_QCFG_RESP_DUPLEX_STATE_HALF
+#define BNXT_LINK_DUPLEX_FULL	PORT_PHY_QCFG_RESP_DUPLEX_STATE_FULL
 	u8			pause;
 #define BNXT_LINK_PAUSE_TX	PORT_PHY_QCFG_RESP_PAUSE_TX
 #define BNXT_LINK_PAUSE_RX	PORT_PHY_QCFG_RESP_PAUSE_RX
@@ -928,6 +944,45 @@ struct bnxt_test_info {
 #define BNXT_CAG_REG_LEGACY_INT_STATUS	0x4014
 #define BNXT_CAG_REG_BASE		0x300000
 
+struct bnxt_tc_info {
+	bool				enabled;
+
+	/* hash table to store TC offloaded flows */
+	struct rhashtable		flow_table;
+	struct rhashtable_params	flow_ht_params;
+
+	/* hash table to store L2 keys of TC flows */
+	struct rhashtable		l2_table;
+	struct rhashtable_params	l2_ht_params;
+
+	/* lock to atomically add/del an l2 node when a flow is
+	 * added or deleted.
+	 */
+	struct mutex			lock;
+
+	/* Stat counter mask (width) */
+	u64				bytes_mask;
+	u64				packets_mask;
+};
+
+struct bnxt_vf_rep_stats {
+	u64			packets;
+	u64			bytes;
+	u64			dropped;
+};
+
+struct bnxt_vf_rep {
+	struct bnxt			*bp;
+	struct net_device		*dev;
+	struct metadata_dst		*dst;
+	u16				vf_idx;
+	u16				tx_cfa_action;
+	u16				rx_cfa_code;
+
+	struct bnxt_vf_rep_stats	rx_stats;
+	struct bnxt_vf_rep_stats	tx_stats;
+};
+
 struct bnxt {
 	void __iomem		*bar0;
 	void __iomem		*bar1;
@@ -957,6 +1012,9 @@ struct bnxt {
 
 #define CHIP_NUM_5745X		0xd730
 
+#define CHIP_NUM_58802		0xd802
+#define CHIP_NUM_58808		0xd808
+
 #define BNXT_CHIP_NUM_5730X(chip_num)		\
 	((chip_num) >= CHIP_NUM_57301 &&	\
 	 (chip_num) <= CHIP_NUM_57304)
@@ -988,6 +1046,10 @@ struct bnxt {
 #define BNXT_CHIP_NUM_57X1X(chip_num)		\
 	(BNXT_CHIP_NUM_5731X(chip_num) || BNXT_CHIP_NUM_5741X(chip_num))
 
+#define BNXT_CHIP_NUM_588XX(chip_num)		\
+	((chip_num) == CHIP_NUM_58802 ||	\
+	 (chip_num) == CHIP_NUM_58808)
+
 	struct net_device	*dev;
 	struct pci_dev		*pdev;
 
@@ -1027,6 +1089,7 @@ struct bnxt {
 	#define BNXT_FLAG_MULTI_HOST	0x100000
 	#define BNXT_FLAG_SHORT_CMD	0x200000
 	#define BNXT_FLAG_DOUBLE_DB	0x400000
+	#define BNXT_FLAG_FW_DCBX_AGENT	0x800000
 	#define BNXT_FLAG_CHIP_NITRO_A0	0x1000000
 
 	#define BNXT_FLAG_ALL_CONFIG_FEATS (BNXT_FLAG_TPA |		\
@@ -1045,6 +1108,7 @@ struct bnxt {
 #define BNXT_CHIP_P4_PLUS(bp)			\
 	(BNXT_CHIP_NUM_57X1X((bp)->chip_num) ||	\
 	 BNXT_CHIP_NUM_5745X((bp)->chip_num) ||	\
+	 BNXT_CHIP_NUM_588XX((bp)->chip_num) ||	\
 	 (BNXT_CHIP_NUM_58700((bp)->chip_num) &&	\
 	  !BNXT_CHIP_TYPE_NITRO_A0(bp)))
 
@@ -1086,6 +1150,7 @@ struct bnxt {
 	int			tx_nr_rings;
 	int			tx_nr_rings_per_tc;
 	int			tx_nr_rings_xdp;
+	int			tx_reserved_rings;
 
 	int			tx_wake_thresh;
 	int			tx_push_thresh;
@@ -1164,6 +1229,8 @@ struct bnxt {
 	u8			nge_port_cnt;
 	__le16			nge_fw_dst_port_id;
 	u8			port_partition_type;
+	u8			port_count;
+	u16			br_mode;
 
 	u16			rx_coal_ticks;
 	u16			rx_coal_ticks_irq;
@@ -1206,6 +1273,12 @@ struct bnxt {
 	wait_queue_head_t	sriov_cfg_wait;
 	bool			sriov_cfg;
 #define BNXT_SRIOV_CFG_WAIT_TMO	msecs_to_jiffies(10000)
+
+	/* lock to protect VF-rep creation/cleanup via
+	 * multiple paths such as ->sriov_configure() and
+	 * devlink ->eswitch_mode_set()
+	 */
+	struct mutex		sriov_lock;
 #endif
 
 #define BNXT_NTP_FLTR_MAX_FLTR	4096
@@ -1232,6 +1305,13 @@ struct bnxt {
 	struct bnxt_led_info	leds[BNXT_MAX_LED];
 
 	struct bpf_prog		*xdp_prog;
+
+	/* devlink interface and vf-rep structs */
+	struct devlink		*dl;
+	enum devlink_eswitch_mode eswitch_mode;
+	struct bnxt_vf_rep	**vf_reps; /* array of vf-rep ptrs */
+	u16			*cfa_code_map; /* cfa_code -> vf_idx map */
+	struct bnxt_tc_info	tc_info;
 };
 
 #define BNXT_RX_STATS_OFFSET(counter)			\
@@ -1301,9 +1381,10 @@ int bnxt_open_nic(struct bnxt *, bool, bool);
 int bnxt_half_open_nic(struct bnxt *bp);
 void bnxt_half_close_nic(struct bnxt *bp);
 int bnxt_close_nic(struct bnxt *, bool, bool);
-int bnxt_reserve_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
-		       int tx_xdp);
+int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
+		     int tx_xdp);
 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc);
 int bnxt_get_max_rings(struct bnxt *, int *, int *, bool);
 void bnxt_restore_pf_fw_resources(struct bnxt *bp);
+int bnxt_port_attr_get(struct bnxt *bp, struct switchdev_attr *attr);
 #endif
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c
index 5c6dd0ce209f..aa1f3a2c7a78 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c
@@ -93,6 +93,12 @@ static int bnxt_hwrm_queue_cos2bw_cfg(struct bnxt *bp, struct ieee_ets *ets,
 			cos2bw.tsa =
 				QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_TSA_ASSIGN_ETS;
 			cos2bw.bw_weight = ets->tc_tx_bw[i];
+			/* older firmware requires min_bw to be set to the
+			 * same weight value in percent.
+			 */
+			cos2bw.min_bw =
+				cpu_to_le32((ets->tc_tx_bw[i] * 100) |
+					    BW_VALUE_UNIT_PERCENT1_100);
 		}
 		memcpy(data, &cos2bw.queue_id, sizeof(cos2bw) - 4);
 		if (i == 0) {
@@ -549,13 +555,18 @@ static u8 bnxt_dcbnl_setdcbx(struct net_device *dev, u8 mode)
 {
 	struct bnxt *bp = netdev_priv(dev);
 
-	/* only support IEEE */
-	if ((mode & DCB_CAP_DCBX_VER_CEE) || !(mode & DCB_CAP_DCBX_VER_IEEE))
+	/* All firmware DCBX settings are set in NVRAM */
+	if (bp->dcbx_cap & DCB_CAP_DCBX_LLD_MANAGED)
 		return 1;
 
 	if (mode & DCB_CAP_DCBX_HOST) {
 		if (BNXT_VF(bp) || (bp->flags & BNXT_FLAG_FW_LLDP_AGENT))
 			return 1;
+
+		/* only support IEEE */
+		if ((mode & DCB_CAP_DCBX_VER_CEE) ||
+		    !(mode & DCB_CAP_DCBX_VER_IEEE))
+			return 1;
 	}
 
 	if (mode == bp->dcbx_cap)
@@ -584,7 +595,7 @@ void bnxt_dcb_init(struct bnxt *bp)
 	bp->dcbx_cap = DCB_CAP_DCBX_VER_IEEE;
 	if (BNXT_PF(bp) && !(bp->flags & BNXT_FLAG_FW_LLDP_AGENT))
 		bp->dcbx_cap |= DCB_CAP_DCBX_HOST;
-	else
+	else if (bp->flags & BNXT_FLAG_FW_DCBX_AGENT)
 		bp->dcbx_cap |= DCB_CAP_DCBX_LLD_MANAGED;
 	bp->dev->dcbnl_ops = &dcbnl_ops;
 }
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.h
index ecd0a5e46a49..d2e0af960bf5 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.h
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.h
@@ -26,6 +26,7 @@ struct bnxt_cos2bw_cfg {
 	u8			queue_id;
 	__le32			min_bw;
 	__le32			max_bw;
+#define BW_VALUE_UNIT_PERCENT1_100		(0x1UL << 29)
 	u8			tsa;
 	u8			pri_lvl;
 	u8			bw_weight;
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
index be6acadcb202..8eff05a3e0e4 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
@@ -86,9 +86,11 @@ static int bnxt_set_coalesce(struct net_device *dev,
 	if (bp->stats_coal_ticks != coal->stats_block_coalesce_usecs) {
 		u32 stats_ticks = coal->stats_block_coalesce_usecs;
 
-		stats_ticks = clamp_t(u32, stats_ticks,
-				      BNXT_MIN_STATS_COAL_TICKS,
-				      BNXT_MAX_STATS_COAL_TICKS);
+		/* Allow 0, which means disable. */
+		if (stats_ticks)
+			stats_ticks = clamp_t(u32, stats_ticks,
+					      BNXT_MIN_STATS_COAL_TICKS,
+					      BNXT_MAX_STATS_COAL_TICKS);
 		stats_ticks = rounddown(stats_ticks, BNXT_MIN_STATS_COAL_TICKS);
 		bp->stats_coal_ticks = stats_ticks;
 		update_stats = true;
@@ -198,19 +200,23 @@ static const struct {
 
 #define BNXT_NUM_PORT_STATS ARRAY_SIZE(bnxt_port_stats_arr)
 
+static int bnxt_get_num_stats(struct bnxt *bp)
+{
+	int num_stats = BNXT_NUM_STATS * bp->cp_nr_rings;
+
+	if (bp->flags & BNXT_FLAG_PORT_STATS)
+		num_stats += BNXT_NUM_PORT_STATS;
+
+	return num_stats;
+}
+
 static int bnxt_get_sset_count(struct net_device *dev, int sset)
 {
 	struct bnxt *bp = netdev_priv(dev);
 
 	switch (sset) {
-	case ETH_SS_STATS: {
-		int num_stats = BNXT_NUM_STATS * bp->cp_nr_rings;
-
-		if (bp->flags & BNXT_FLAG_PORT_STATS)
-			num_stats += BNXT_NUM_PORT_STATS;
-
-		return num_stats;
-	}
+	case ETH_SS_STATS:
+		return bnxt_get_num_stats(bp);
 	case ETH_SS_TEST:
 		if (!bp->num_tests)
 			return -EOPNOTSUPP;
@@ -225,11 +231,8 @@ static void bnxt_get_ethtool_stats(struct net_device *dev,
 {
 	u32 i, j = 0;
 	struct bnxt *bp = netdev_priv(dev);
-	u32 buf_size = sizeof(struct ctx_hw_stats) * bp->cp_nr_rings;
 	u32 stat_fields = sizeof(struct ctx_hw_stats) / 8;
 
-	memset(buf, 0, buf_size);
-
 	if (!bp->bnapi)
 		return;
 
@@ -432,8 +435,7 @@ static int bnxt_set_channels(struct net_device *dev,
 		}
 		tx_xdp = req_rx_rings;
 	}
-	rc = bnxt_reserve_rings(bp, req_tx_rings, req_rx_rings, sh, tcs,
-				tx_xdp);
+	rc = bnxt_check_rings(bp, req_tx_rings, req_rx_rings, sh, tcs, tx_xdp);
 	if (rc) {
 		netdev_warn(dev, "Unable to allocate the requested rings\n");
 		return rc;
@@ -520,7 +522,7 @@ static int bnxt_grxclsrule(struct bnxt *bp, struct ethtool_rxnfc *cmd)
 	struct flow_keys *fkeys;
 	int i, rc = -EINVAL;
 
-	if (fs->location < 0 || fs->location >= BNXT_NTP_FLTR_MAX_FLTR)
+	if (fs->location >= BNXT_NTP_FLTR_MAX_FLTR)
 		return rc;
 
 	for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
@@ -835,7 +837,7 @@ static void bnxt_get_drvinfo(struct net_device *dev,
 		strlcpy(info->fw_version, bp->fw_ver_str,
 			sizeof(info->fw_version));
 	strlcpy(info->bus_info, pci_name(bp->pdev), sizeof(info->bus_info));
-	info->n_stats = BNXT_NUM_STATS * bp->cp_nr_rings;
+	info->n_stats = bnxt_get_num_stats(bp);
 	info->testinfo_len = bp->num_tests;
 	/* TODO CHIMP_FW: eeprom dump details */
 	info->eedump_len = 0;
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_hsi.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_hsi.h
index 7dc71bb95837..cb04cc76e8ad 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_hsi.h
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_hsi.h
@@ -11,14 +11,14 @@
 #ifndef BNXT_HSI_H
 #define BNXT_HSI_H
 
-/* HSI and HWRM Specification 1.7.6 */
+/* HSI and HWRM Specification 1.8.1 */
 #define HWRM_VERSION_MAJOR	1
-#define HWRM_VERSION_MINOR	7
-#define HWRM_VERSION_UPDATE	6
+#define HWRM_VERSION_MINOR	8
+#define HWRM_VERSION_UPDATE	1
 
-#define HWRM_VERSION_RSVD	2 /* non-zero means beta version */
+#define HWRM_VERSION_RSVD	4 /* non-zero means beta version */
 
-#define HWRM_VERSION_STR	"1.7.6.2"
+#define HWRM_VERSION_STR	"1.8.1.4"
 /*
  * Following is the signature for HWRM message field that indicates not
  * applicable (All F's). Need to cast it the size of the field if needed.
@@ -813,7 +813,7 @@ struct hwrm_func_qcfg_output {
 	#define FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED	    0x4UL
 	#define FUNC_QCFG_RESP_FLAGS_STD_TX_RING_MODE_ENABLED      0x8UL
 	#define FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED	    0x10UL
-	#define FUNC_QCFG_RESP_FLAGS_MULTI_HOST			    0x20UL
+	#define FUNC_QCFG_RESP_FLAGS_MULTI_HOST		    0x20UL
 	u8 mac_address[6];
 	__le16 pci_id;
 	__le16 alloc_rsscos_ctx;
@@ -835,9 +835,8 @@ struct hwrm_func_qcfg_output {
 	u8 port_pf_cnt;
 	#define FUNC_QCFG_RESP_PORT_PF_CNT_UNAVAIL		   0x0UL
 	__le16 dflt_vnic_id;
-	u8 host_cnt;
-	#define FUNC_QCFG_RESP_HOST_CNT_UNAVAIL		   0x0UL
 	u8 unused_0;
+	u8 unused_1;
 	__le32 min_bw;
 	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_MASK		    0xfffffffUL
 	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_SFT		    0
@@ -874,12 +873,56 @@ struct hwrm_func_qcfg_output {
 	#define FUNC_QCFG_RESP_EVB_MODE_NO_EVB			   0x0UL
 	#define FUNC_QCFG_RESP_EVB_MODE_VEB			   0x1UL
 	#define FUNC_QCFG_RESP_EVB_MODE_VEPA			   0x2UL
-	u8 unused_1;
+	u8 unused_2;
 	__le16 alloc_vfs;
 	__le32 alloc_mcast_filters;
 	__le32 alloc_hw_ring_grps;
 	__le16 alloc_sp_tx_rings;
+	u8 unused_3;
+	u8 valid;
+};
+
+/* hwrm_func_vlan_cfg */
+/* Input (48 bytes) */
+struct hwrm_func_vlan_cfg_input {
+	__le16 req_type;
+	__le16 cmpl_ring;
+	__le16 seq_id;
+	__le16 target_id;
+	__le64 resp_addr;
+	__le16 fid;
+	u8 unused_0;
+	u8 unused_1;
+	__le32 enables;
+	#define FUNC_VLAN_CFG_REQ_ENABLES_STAG_VID		    0x1UL
+	#define FUNC_VLAN_CFG_REQ_ENABLES_CTAG_VID		    0x2UL
+	#define FUNC_VLAN_CFG_REQ_ENABLES_STAG_PCP		    0x4UL
+	#define FUNC_VLAN_CFG_REQ_ENABLES_CTAG_PCP		    0x8UL
+	#define FUNC_VLAN_CFG_REQ_ENABLES_STAG_TPID		    0x10UL
+	#define FUNC_VLAN_CFG_REQ_ENABLES_CTAG_TPID		    0x20UL
+	__le16 stag_vid;
+	u8 stag_pcp;
+	u8 unused_2;
+	__be16 stag_tpid;
+	__le16 ctag_vid;
+	u8 ctag_pcp;
+	u8 unused_3;
+	__be16 ctag_tpid;
+	__le32 rsvd1;
+	__le32 rsvd2;
+	__le32 unused_4;
+};
+
+/* Output (16 bytes) */
+struct hwrm_func_vlan_cfg_output {
+	__le16 error_code;
+	__le16 req_type;
+	__le16 seq_id;
+	__le16 resp_len;
+	__le32 unused_0;
+	u8 unused_1;
 	u8 unused_2;
+	u8 unused_3;
 	u8 valid;
 };
 
@@ -902,6 +945,8 @@ struct hwrm_func_cfg_input {
 	#define FUNC_CFG_REQ_FLAGS_STD_TX_RING_MODE_ENABLE	    0x200UL
 	#define FUNC_CFG_REQ_FLAGS_STD_TX_RING_MODE_DISABLE	    0x400UL
 	#define FUNC_CFG_REQ_FLAGS_VIRT_MAC_PERSIST		    0x800UL
+	#define FUNC_CFG_REQ_FLAGS_NO_AUTOCLEAR_STATISTIC	    0x1000UL
+	#define FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST		    0x2000UL
 	__le32 enables;
 	#define FUNC_CFG_REQ_ENABLES_MTU			    0x1UL
 	#define FUNC_CFG_REQ_ENABLES_MRU			    0x2UL
@@ -1456,9 +1501,9 @@ struct hwrm_port_phy_qcfg_output {
 	#define PORT_PHY_QCFG_RESP_LINK_SPEED_50GB		   0x1f4UL
 	#define PORT_PHY_QCFG_RESP_LINK_SPEED_100GB		   0x3e8UL
 	#define PORT_PHY_QCFG_RESP_LINK_SPEED_10MB		   0xffffUL
-	u8 duplex;
-	#define PORT_PHY_QCFG_RESP_DUPLEX_HALF			   0x0UL
-	#define PORT_PHY_QCFG_RESP_DUPLEX_FULL			   0x1UL
+	u8 duplex_cfg;
+	#define PORT_PHY_QCFG_RESP_DUPLEX_CFG_HALF		   0x0UL
+	#define PORT_PHY_QCFG_RESP_DUPLEX_CFG_FULL		   0x1UL
 	u8 pause;
 	#define PORT_PHY_QCFG_RESP_PAUSE_TX			    0x1UL
 	#define PORT_PHY_QCFG_RESP_PAUSE_RX			    0x2UL
@@ -1573,6 +1618,9 @@ struct hwrm_port_phy_qcfg_output {
 	#define PORT_PHY_QCFG_RESP_PHY_TYPE_40G_BASELR4	   0x16UL
 	#define PORT_PHY_QCFG_RESP_PHY_TYPE_40G_BASEER4	   0x17UL
 	#define PORT_PHY_QCFG_RESP_PHY_TYPE_40G_ACTIVE_CABLE      0x18UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_1G_BASET		   0x19UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_1G_BASESX		   0x1aUL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_1G_BASECX		   0x1bUL
 	u8 media_type;
 	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_UNKNOWN		   0x0UL
 	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_TP		   0x1UL
@@ -1651,14 +1699,16 @@ struct hwrm_port_phy_qcfg_output {
 	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_CLAUSE74_ENABLED    0x10UL
 	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_CLAUSE91_SUPPORTED  0x20UL
 	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_CLAUSE91_ENABLED    0x40UL
+	u8 duplex_state;
+	#define PORT_PHY_QCFG_RESP_DUPLEX_STATE_HALF		   0x0UL
+	#define PORT_PHY_QCFG_RESP_DUPLEX_STATE_FULL		   0x1UL
 	u8 unused_1;
-	u8 unused_2;
 	char phy_vendor_name[16];
 	char phy_vendor_partnumber[16];
-	__le32 unused_3;
+	__le32 unused_2;
+	u8 unused_3;
 	u8 unused_4;
 	u8 unused_5;
-	u8 unused_6;
 	u8 valid;
 };
 
@@ -1744,6 +1794,51 @@ struct hwrm_port_mac_cfg_output {
 	u8 valid;
 };
 
+/* hwrm_port_mac_ptp_qcfg */
+/* Input (24 bytes) */
+struct hwrm_port_mac_ptp_qcfg_input {
+	__le16 req_type;
+	__le16 cmpl_ring;
+	__le16 seq_id;
+	__le16 target_id;
+	__le64 resp_addr;
+	__le16 port_id;
+	__le16 unused_0[3];
+};
+
+/* Output (80 bytes) */
+struct hwrm_port_mac_ptp_qcfg_output {
+	__le16 error_code;
+	__le16 req_type;
+	__le16 seq_id;
+	__le16 resp_len;
+	u8 flags;
+	#define PORT_MAC_PTP_QCFG_RESP_FLAGS_DIRECT_ACCESS	    0x1UL
+	#define PORT_MAC_PTP_QCFG_RESP_FLAGS_HWRM_ACCESS	    0x2UL
+	u8 unused_0;
+	__le16 unused_1;
+	__le32 rx_ts_reg_off_lower;
+	__le32 rx_ts_reg_off_upper;
+	__le32 rx_ts_reg_off_seq_id;
+	__le32 rx_ts_reg_off_src_id_0;
+	__le32 rx_ts_reg_off_src_id_1;
+	__le32 rx_ts_reg_off_src_id_2;
+	__le32 rx_ts_reg_off_domain_id;
+	__le32 rx_ts_reg_off_fifo;
+	__le32 rx_ts_reg_off_fifo_adv;
+	__le32 rx_ts_reg_off_granularity;
+	__le32 tx_ts_reg_off_lower;
+	__le32 tx_ts_reg_off_upper;
+	__le32 tx_ts_reg_off_seq_id;
+	__le32 tx_ts_reg_off_fifo;
+	__le32 tx_ts_reg_off_granularity;
+	__le32 unused_2;
+	u8 unused_3;
+	u8 unused_4;
+	u8 unused_5;
+	u8 valid;
+};
+
 /* hwrm_port_qstats */
 /* Input (40 bytes) */
 struct hwrm_port_qstats_input {
@@ -1874,11 +1969,16 @@ struct hwrm_port_phy_qcaps_output {
 	__le16 req_type;
 	__le16 seq_id;
 	__le16 resp_len;
-	u8 eee_supported;
-	#define PORT_PHY_QCAPS_RESP_EEE_SUPPORTED		    0x1UL
-	#define PORT_PHY_QCAPS_RESP_RSVD1_MASK			    0xfeUL
-	#define PORT_PHY_QCAPS_RESP_RSVD1_SFT			    1
-	u8 unused_0;
+	u8 flags;
+	#define PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED	    0x1UL
+	#define PORT_PHY_QCAPS_RESP_FLAGS_RSVD1_MASK		    0xfeUL
+	#define PORT_PHY_QCAPS_RESP_FLAGS_RSVD1_SFT		    1
+	u8 port_cnt;
+	#define PORT_PHY_QCAPS_RESP_PORT_CNT_UNKNOWN		   0x0UL
+	#define PORT_PHY_QCAPS_RESP_PORT_CNT_1			   0x1UL
+	#define PORT_PHY_QCAPS_RESP_PORT_CNT_2			   0x2UL
+	#define PORT_PHY_QCAPS_RESP_PORT_CNT_3			   0x3UL
+	#define PORT_PHY_QCAPS_RESP_PORT_CNT_4			   0x4UL
 	__le16 supported_speeds_force_mode;
 	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_100MBHD 0x1UL
 	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_100MB 0x2UL
@@ -3152,6 +3252,95 @@ struct hwrm_queue_cos2bw_cfg_output {
 	u8 valid;
 };
 
+/* hwrm_queue_dscp_qcaps */
+/* Input (24 bytes) */
+struct hwrm_queue_dscp_qcaps_input {
+	__le16 req_type;
+	__le16 cmpl_ring;
+	__le16 seq_id;
+	__le16 target_id;
+	__le64 resp_addr;
+	u8 port_id;
+	u8 unused_0[7];
+};
+
+/* Output (16 bytes) */
+struct hwrm_queue_dscp_qcaps_output {
+	__le16 error_code;
+	__le16 req_type;
+	__le16 seq_id;
+	__le16 resp_len;
+	u8 num_dscp_bits;
+	u8 unused_0;
+	__le16 max_entries;
+	u8 unused_1;
+	u8 unused_2;
+	u8 unused_3;
+	u8 valid;
+};
+
+/* hwrm_queue_dscp2pri_qcfg */
+/* Input (32 bytes) */
+struct hwrm_queue_dscp2pri_qcfg_input {
+	__le16 req_type;
+	__le16 cmpl_ring;
+	__le16 seq_id;
+	__le16 target_id;
+	__le64 resp_addr;
+	__le64 dest_data_addr;
+	u8 port_id;
+	u8 unused_0;
+	__le16 dest_data_buffer_size;
+	__le32 unused_1;
+};
+
+/* Output (16 bytes) */
+struct hwrm_queue_dscp2pri_qcfg_output {
+	__le16 error_code;
+	__le16 req_type;
+	__le16 seq_id;
+	__le16 resp_len;
+	__le16 entry_cnt;
+	u8 default_pri;
+	u8 unused_0;
+	u8 unused_1;
+	u8 unused_2;
+	u8 unused_3;
+	u8 valid;
+};
+
+/* hwrm_queue_dscp2pri_cfg */
+/* Input (40 bytes) */
+struct hwrm_queue_dscp2pri_cfg_input {
+	__le16 req_type;
+	__le16 cmpl_ring;
+	__le16 seq_id;
+	__le16 target_id;
+	__le64 resp_addr;
+	__le64 src_data_addr;
+	__le32 flags;
+	#define QUEUE_DSCP2PRI_CFG_REQ_FLAGS_USE_HW_DEFAULT_PRI    0x1UL
+	__le32 enables;
+	#define QUEUE_DSCP2PRI_CFG_REQ_ENABLES_DEFAULT_PRI	    0x1UL
+	u8 port_id;
+	u8 default_pri;
+	__le16 entry_cnt;
+	__le32 unused_0;
+};
+
+/* Output (16 bytes) */
+struct hwrm_queue_dscp2pri_cfg_output {
+	__le16 error_code;
+	__le16 req_type;
+	__le16 seq_id;
+	__le16 resp_len;
+	__le32 unused_0;
+	u8 unused_1;
+	u8 unused_2;
+	u8 unused_3;
+	u8 valid;
+};
+
 /* hwrm_vnic_alloc */
 /* Input (24 bytes) */
 struct hwrm_vnic_alloc_input {
@@ -4038,7 +4227,7 @@ struct hwrm_cfa_encap_record_alloc_input {
 	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_IPGRE       0x8UL
 	u8 unused_0;
 	__le16 unused_1;
-	__le32 encap_data[16];
+	__le32 encap_data[20];
 };
 
 /* Output (16 bytes) */
@@ -4120,8 +4309,8 @@ struct hwrm_cfa_ntuple_filter_alloc_input {
 	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6     0x6UL
 	u8 ip_protocol;
 	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_UNKNOWN   0x0UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_UDP       0x6UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_TCP       0x11UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_TCP       0x6UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_UDP       0x11UL
 	__le16 dst_id;
 	__le16 mirror_vnic_id;
 	u8 tunnel_type;
@@ -4224,6 +4413,216 @@ struct hwrm_cfa_ntuple_filter_cfg_output {
 	u8 valid;
 };
 
+/* hwrm_cfa_flow_alloc */
+/* Input (128 bytes) */
+struct hwrm_cfa_flow_alloc_input {
+	__le16 req_type;
+	__le16 cmpl_ring;
+	__le16 seq_id;
+	__le16 target_id;
+	__le64 resp_addr;
+	__le16 flags;
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_TUNNEL		    0x1UL
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_MASK		    0x6UL
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_SFT		    1
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_NONE		   (0x0UL << 1)
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_ONE		   (0x1UL << 1)
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_TWO		   (0x2UL << 1)
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_LAST    CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_TWO
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_MASK		    0x38UL
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_SFT		    3
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_L2		   (0x0UL << 3)
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV4		   (0x1UL << 3)
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV6		   (0x2UL << 3)
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_LAST    CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV6
+	__le16 src_fid;
+	__le32 tunnel_handle;
+	__le16 action_flags;
+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_FWD		    0x1UL
+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_RECYCLE	    0x2UL
+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_DROP		    0x4UL
+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_METER		    0x8UL
+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_TUNNEL		    0x10UL
+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_NAT_SRC	    0x20UL
+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_NAT_DEST	    0x40UL
+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_NAT_IPV4_ADDRESS   0x80UL
+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_L2_HEADER_REWRITE  0x100UL
+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_TTL_DECREMENT      0x200UL
+	__le16 dst_fid;
+	__be16 l2_rewrite_vlan_tpid;
+	__be16 l2_rewrite_vlan_tci;
+	__le16 act_meter_id;
+	__le16 ref_flow_handle;
+	__be16 ethertype;
+	__be16 outer_vlan_tci;
+	__be16 dmac[3];
+	__be16 inner_vlan_tci;
+	__be16 smac[3];
+	u8 ip_dst_mask_len;
+	u8 ip_src_mask_len;
+	__be32 ip_dst[4];
+	__be32 ip_src[4];
+	__be16 l4_src_port;
+	__be16 l4_src_port_mask;
+	__be16 l4_dst_port;
+	__be16 l4_dst_port_mask;
+	__be32 nat_ip_address[4];
+	__be16 l2_rewrite_dmac[3];
+	__be16 nat_port;
+	__be16 l2_rewrite_smac[3];
+	u8 ip_proto;
+	u8 unused_0;
+};
+
+/* Output (16 bytes) */
+struct hwrm_cfa_flow_alloc_output {
+	__le16 error_code;
+	__le16 req_type;
+	__le16 seq_id;
+	__le16 resp_len;
+	__le16 flow_handle;
+	u8 unused_0;
+	u8 unused_1;
+	u8 unused_2;
+	u8 unused_3;
+	u8 unused_4;
+	u8 valid;
+};
+
+/* hwrm_cfa_flow_free */
+/* Input (24 bytes) */
+struct hwrm_cfa_flow_free_input {
+	__le16 req_type;
+	__le16 cmpl_ring;
+	__le16 seq_id;
+	__le16 target_id;
+	__le64 resp_addr;
+	__le16 flow_handle;
+	__le16 unused_0[3];
+};
+
+/* Output (32 bytes) */
+struct hwrm_cfa_flow_free_output {
+	__le16 error_code;
+	__le16 req_type;
+	__le16 seq_id;
+	__le16 resp_len;
+	__le64 packet;
+	__le64 byte;
+	__le32 unused_0;
+	u8 unused_1;
+	u8 unused_2;
+	u8 unused_3;
+	u8 valid;
+};
+
+/* hwrm_cfa_flow_stats */
+/* Input (40 bytes) */
+struct hwrm_cfa_flow_stats_input {
+	__le16 req_type;
+	__le16 cmpl_ring;
+	__le16 seq_id;
+	__le16 target_id;
+	__le64 resp_addr;
+	__le16 num_flows;
+	__le16 flow_handle_0;
+	__le16 flow_handle_1;
+	__le16 flow_handle_2;
+	__le16 flow_handle_3;
+	__le16 flow_handle_4;
+	__le16 flow_handle_5;
+	__le16 flow_handle_6;
+	__le16 flow_handle_7;
+	__le16 flow_handle_8;
+	__le16 flow_handle_9;
+	__le16 unused_0;
+};
+
+/* Output (176 bytes) */
+struct hwrm_cfa_flow_stats_output {
+	__le16 error_code;
+	__le16 req_type;
+	__le16 seq_id;
+	__le16 resp_len;
+	__le64 packet_0;
+	__le64 packet_1;
+	__le64 packet_2;
+	__le64 packet_3;
+	__le64 packet_4;
+	__le64 packet_5;
+	__le64 packet_6;
+	__le64 packet_7;
+	__le64 packet_8;
+	__le64 packet_9;
+	__le64 byte_0;
+	__le64 byte_1;
+	__le64 byte_2;
+	__le64 byte_3;
+	__le64 byte_4;
+	__le64 byte_5;
+	__le64 byte_6;
+	__le64 byte_7;
+	__le64 byte_8;
+	__le64 byte_9;
+	__le32 unused_0;
+	u8 unused_1;
+	u8 unused_2;
+	u8 unused_3;
+	u8 valid;
+};
+
+/* hwrm_cfa_vfr_alloc */
+/* Input (32 bytes) */
+struct hwrm_cfa_vfr_alloc_input {
+	__le16 req_type;
+	__le16 cmpl_ring;
+	__le16 seq_id;
+	__le16 target_id;
+	__le64 resp_addr;
+	__le16 vf_id;
+	__le16 reserved;
+	__le32 unused_0;
+	char vfr_name[32];
+};
+
+/* Output (16 bytes) */
+struct hwrm_cfa_vfr_alloc_output {
+	__le16 error_code;
+	__le16 req_type;
+	__le16 seq_id;
+	__le16 resp_len;
+	__le16 rx_cfa_code;
+	__le16 tx_cfa_action;
+	u8 unused_0;
+	u8 unused_1;
+	u8 unused_2;
+	u8 valid;
+};
+
+/* hwrm_cfa_vfr_free */
+/* Input (24 bytes) */
+struct hwrm_cfa_vfr_free_input {
+	__le16 req_type;
+	__le16 cmpl_ring;
+	__le16 seq_id;
+	__le16 target_id;
+	__le64 resp_addr;
+	char vfr_name[32];
+};
+
+/* Output (16 bytes) */
+struct hwrm_cfa_vfr_free_output {
+	__le16 error_code;
+	__le16 req_type;
+	__le16 seq_id;
+	__le16 resp_len;
+	__le32 unused_0;
+	u8 unused_1;
+	u8 unused_2;
+	u8 unused_3;
+	u8 valid;
+};
+
 /* hwrm_tunnel_dst_port_query */
 /* Input (24 bytes) */
 struct hwrm_tunnel_dst_port_query_input {
@@ -4448,12 +4847,13 @@ struct hwrm_fw_reset_input {
 	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_MGMT		   0x1UL
 	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_NETCTRL	   0x2UL
 	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_ROCE		   0x3UL
-	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_RSVD		   0x4UL
+	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_HOST		   0x4UL
 	u8 selfrst_status;
 	#define FW_RESET_REQ_SELFRST_STATUS_SELFRSTNONE	   0x0UL
 	#define FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP	   0x1UL
 	#define FW_RESET_REQ_SELFRST_STATUS_SELFRSTPCIERST	   0x2UL
-	__le16 unused_0[3];
+	u8 host_idx;
+	u8 unused_0[5];
 };
 
 /* Output (16 bytes) */
@@ -4487,7 +4887,7 @@ struct hwrm_fw_qstatus_input {
 	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_MGMT		   0x1UL
 	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_NETCTRL	   0x2UL
 	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_ROCE		   0x3UL
-	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_RSVD		   0x4UL
+	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_HOST		   0x4UL
 	u8 unused_0[7];
 };
 
@@ -4572,6 +4972,16 @@ struct hwrm_fw_set_structured_data_output {
 	u8 valid;
 };
 
+/* Command specific Error Codes (8 bytes) */
+struct hwrm_fw_set_structured_data_cmd_err {
+	u8 code;
+	#define FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_UNKNOWN       0x0UL
+	#define FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_BAD_HDR_CNT   0x1UL
+	#define FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_BAD_FMT       0x2UL
+	#define FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_BAD_ID	   0x3UL
+	u8 unused_0[7];
+};
+
 /* hwrm_fw_get_structured_data */
 /* Input (32 bytes) */
 struct hwrm_fw_get_structured_data_input {
@@ -4611,6 +5021,14 @@ struct hwrm_fw_get_structured_data_output {
 	u8 valid;
 };
 
+/* Command specific Error Codes (8 bytes) */
+struct hwrm_fw_get_structured_data_cmd_err {
+	u8 code;
+	#define FW_GET_STRUCTURED_DATA_CMD_ERR_CODE_UNKNOWN       0x0UL
+	#define FW_GET_STRUCTURED_DATA_CMD_ERR_CODE_BAD_ID	   0x3UL
+	u8 unused_0[7];
+};
+
 /* hwrm_exec_fwd_resp */
 /* Input (128 bytes) */
 struct hwrm_exec_fwd_resp_input {
@@ -5280,11 +5698,15 @@ struct hwrm_selftest_qlist_output {
 	#define SELFTEST_QLIST_RESP_AVAILABLE_TESTS_LINK_TEST      0x2UL
 	#define SELFTEST_QLIST_RESP_AVAILABLE_TESTS_REGISTER_TEST  0x4UL
 	#define SELFTEST_QLIST_RESP_AVAILABLE_TESTS_MEMORY_TEST    0x8UL
+	#define SELFTEST_QLIST_RESP_AVAILABLE_TESTS_PCIE_EYE_TEST  0x10UL
+	#define SELFTEST_QLIST_RESP_AVAILABLE_TESTS_ETHERNET_EYE_TEST 0x20UL
 	u8 offline_tests;
 	#define SELFTEST_QLIST_RESP_OFFLINE_TESTS_NVM_TEST	    0x1UL
 	#define SELFTEST_QLIST_RESP_OFFLINE_TESTS_LINK_TEST	    0x2UL
 	#define SELFTEST_QLIST_RESP_OFFLINE_TESTS_REGISTER_TEST    0x4UL
 	#define SELFTEST_QLIST_RESP_OFFLINE_TESTS_MEMORY_TEST      0x8UL
+	#define SELFTEST_QLIST_RESP_OFFLINE_TESTS_PCIE_EYE_TEST    0x10UL
+	#define SELFTEST_QLIST_RESP_OFFLINE_TESTS_ETHERNET_EYE_TEST 0x20UL
 	u8 unused_0;
 	__le16 test_timeout;
 	u8 unused_1;
@@ -5312,6 +5734,8 @@ struct hwrm_selftest_exec_input {
 	#define SELFTEST_EXEC_REQ_FLAGS_LINK_TEST		    0x2UL
 	#define SELFTEST_EXEC_REQ_FLAGS_REGISTER_TEST		    0x4UL
 	#define SELFTEST_EXEC_REQ_FLAGS_MEMORY_TEST		    0x8UL
+	#define SELFTEST_EXEC_REQ_FLAGS_PCIE_EYE_TEST		    0x10UL
+	#define SELFTEST_EXEC_REQ_FLAGS_ETHERNET_EYE_TEST	    0x20UL
 	u8 unused_0[7];
 };
 
@@ -5326,11 +5750,15 @@ struct hwrm_selftest_exec_output {
 	#define SELFTEST_EXEC_RESP_REQUESTED_TESTS_LINK_TEST       0x2UL
 	#define SELFTEST_EXEC_RESP_REQUESTED_TESTS_REGISTER_TEST   0x4UL
 	#define SELFTEST_EXEC_RESP_REQUESTED_TESTS_MEMORY_TEST     0x8UL
+	#define SELFTEST_EXEC_RESP_REQUESTED_TESTS_PCIE_EYE_TEST   0x10UL
+	#define SELFTEST_EXEC_RESP_REQUESTED_TESTS_ETHERNET_EYE_TEST 0x20UL
 	u8 test_success;
 	#define SELFTEST_EXEC_RESP_TEST_SUCCESS_NVM_TEST	    0x1UL
 	#define SELFTEST_EXEC_RESP_TEST_SUCCESS_LINK_TEST	    0x2UL
 	#define SELFTEST_EXEC_RESP_TEST_SUCCESS_REGISTER_TEST      0x4UL
 	#define SELFTEST_EXEC_RESP_TEST_SUCCESS_MEMORY_TEST	    0x8UL
+	#define SELFTEST_EXEC_RESP_TEST_SUCCESS_PCIE_EYE_TEST      0x10UL
+	#define SELFTEST_EXEC_RESP_TEST_SUCCESS_ETHERNET_EYE_TEST  0x20UL
 	__le16 unused_0[3];
 };
 
@@ -5411,7 +5839,7 @@ struct cmd_nums {
 	#define HWRM_PORT_LPBK_CLR_STATS			   (0x26UL)
 	#define HWRM_PORT_PHY_QCFG				   (0x27UL)
 	#define HWRM_PORT_MAC_QCFG				   (0x28UL)
-	#define RESERVED7					   (0x29UL)
+	#define HWRM_PORT_MAC_PTP_QCFG				   (0x29UL)
 	#define HWRM_PORT_PHY_QCAPS				   (0x2aUL)
 	#define HWRM_PORT_PHY_I2C_WRITE			   (0x2bUL)
 	#define HWRM_PORT_PHY_I2C_READ				   (0x2cUL)
@@ -5421,14 +5849,17 @@ struct cmd_nums {
 	#define HWRM_QUEUE_QPORTCFG				   (0x30UL)
 	#define HWRM_QUEUE_QCFG				   (0x31UL)
 	#define HWRM_QUEUE_CFG					   (0x32UL)
-	#define RESERVED2					   (0x33UL)
-	#define RESERVED3					   (0x34UL)
+	#define HWRM_FUNC_VLAN_CFG				   (0x33UL)
+	#define HWRM_FUNC_VLAN_QCFG				   (0x34UL)
 	#define HWRM_QUEUE_PFCENABLE_QCFG			   (0x35UL)
 	#define HWRM_QUEUE_PFCENABLE_CFG			   (0x36UL)
 	#define HWRM_QUEUE_PRI2COS_QCFG			   (0x37UL)
 	#define HWRM_QUEUE_PRI2COS_CFG				   (0x38UL)
 	#define HWRM_QUEUE_COS2BW_QCFG				   (0x39UL)
 	#define HWRM_QUEUE_COS2BW_CFG				   (0x3aUL)
+	#define HWRM_QUEUE_DSCP_QCAPS				   (0x3bUL)
+	#define HWRM_QUEUE_DSCP2PRI_QCFG			   (0x3cUL)
+	#define HWRM_QUEUE_DSCP2PRI_CFG			   (0x3dUL)
 	#define HWRM_VNIC_ALLOC				   (0x40UL)
 	#define HWRM_VNIC_FREE					   (0x41UL)
 	#define HWRM_VNIC_CFG					   (0x42UL)
@@ -5455,7 +5886,7 @@ struct cmd_nums {
 	#define HWRM_CFA_L2_FILTER_FREE			   (0x91UL)
 	#define HWRM_CFA_L2_FILTER_CFG				   (0x92UL)
 	#define HWRM_CFA_L2_SET_RX_MASK			   (0x93UL)
-	#define RESERVED4					   (0x94UL)
+	#define HWRM_CFA_VLAN_ANTISPOOF_CFG			   (0x94UL)
 	#define HWRM_CFA_TUNNEL_FILTER_ALLOC			   (0x95UL)
 	#define HWRM_CFA_TUNNEL_FILTER_FREE			   (0x96UL)
 	#define HWRM_CFA_ENCAP_RECORD_ALLOC			   (0x97UL)
@@ -5494,6 +5925,8 @@ struct cmd_nums {
 	#define HWRM_CFA_METER_PROFILE_CFG			   (0xf7UL)
 	#define HWRM_CFA_METER_INSTANCE_ALLOC			   (0xf8UL)
 	#define HWRM_CFA_METER_INSTANCE_FREE			   (0xf9UL)
+	#define HWRM_CFA_VFR_ALLOC				   (0xfdUL)
+	#define HWRM_CFA_VFR_FREE				   (0xfeUL)
 	#define HWRM_CFA_VF_PAIR_ALLOC				   (0x100UL)
 	#define HWRM_CFA_VF_PAIR_FREE				   (0x101UL)
 	#define HWRM_CFA_VF_PAIR_INFO				   (0x102UL)
@@ -5502,14 +5935,20 @@ struct cmd_nums {
 	#define HWRM_CFA_FLOW_FLUSH				   (0x105UL)
 	#define HWRM_CFA_FLOW_STATS				   (0x106UL)
 	#define HWRM_CFA_FLOW_INFO				   (0x107UL)
+	#define HWRM_CFA_DECAP_FILTER_ALLOC			   (0x108UL)
+	#define HWRM_CFA_DECAP_FILTER_FREE			   (0x109UL)
+	#define HWRM_CFA_VLAN_ANTISPOOF_QCFG			   (0x10aUL)
 	#define HWRM_SELFTEST_QLIST				   (0x200UL)
 	#define HWRM_SELFTEST_EXEC				   (0x201UL)
 	#define HWRM_SELFTEST_IRQ				   (0x202UL)
+	#define HWRM_SELFTEST_RETREIVE_EYE_DATA		   (0x203UL)
 	#define HWRM_DBG_READ_DIRECT				   (0xff10UL)
 	#define HWRM_DBG_READ_INDIRECT				   (0xff11UL)
 	#define HWRM_DBG_WRITE_DIRECT				   (0xff12UL)
 	#define HWRM_DBG_WRITE_INDIRECT			   (0xff13UL)
 	#define HWRM_DBG_DUMP					   (0xff14UL)
+	#define HWRM_DBG_ERASE_NVM				   (0xff15UL)
+	#define HWRM_DBG_CFG					   (0xff16UL)
 	#define HWRM_NVM_FACTORY_DEFAULTS			   (0xffeeUL)
 	#define HWRM_NVM_VALIDATE_OPTION			   (0xffefUL)
 	#define HWRM_NVM_FLUSH					   (0xfff0UL)
@@ -5720,6 +6159,7 @@ struct hwrm_struct_hdr {
 	#define STRUCT_HDR_STRUCT_ID_LLDP_DEVICE		   0x426UL
 	#define STRUCT_HDR_STRUCT_ID_AFM_OPAQUE		   0x1UL
 	#define STRUCT_HDR_STRUCT_ID_PORT_DESCRIPTION		   0xaUL
+	#define STRUCT_HDR_STRUCT_ID_RSS_V2			   0x64UL
 	__le16 len;
 	u8 version;
 	u8 count;
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c
index b8e7248294d9..d37925a8a65b 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c
@@ -18,6 +18,7 @@
 #include "bnxt.h"
 #include "bnxt_ulp.h"
 #include "bnxt_sriov.h"
+#include "bnxt_vfr.h"
 #include "bnxt_ethtool.h"
 
 #ifdef CONFIG_BNXT_SRIOV
@@ -587,6 +588,10 @@ void bnxt_sriov_disable(struct bnxt *bp)
 	if (!num_vfs)
 		return;
 
+	/* synchronize VF and VF-rep create and destroy */
+	mutex_lock(&bp->sriov_lock);
+	bnxt_vf_reps_destroy(bp);
+
 	if (pci_vfs_assigned(bp->pdev)) {
 		bnxt_hwrm_fwd_async_event_cmpl(
 			bp, NULL, ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD);
@@ -597,6 +602,7 @@ void bnxt_sriov_disable(struct bnxt *bp)
 		/* Free the HW resources reserved for various VF's */
 		bnxt_hwrm_func_vf_resource_free(bp, num_vfs);
 	}
+	mutex_unlock(&bp->sriov_lock);
 
 	bnxt_free_vf_resources(bp);
 
@@ -794,8 +800,10 @@ static int bnxt_vf_set_link(struct bnxt *bp, struct bnxt_vf_info *vf)
 					PORT_PHY_QCFG_RESP_LINK_LINK;
 				phy_qcfg_resp.link_speed = cpu_to_le16(
 					PORT_PHY_QCFG_RESP_LINK_SPEED_10GB);
-				phy_qcfg_resp.duplex =
-					PORT_PHY_QCFG_RESP_DUPLEX_FULL;
+				phy_qcfg_resp.duplex_cfg =
+					PORT_PHY_QCFG_RESP_DUPLEX_CFG_FULL;
+				phy_qcfg_resp.duplex_state =
+					PORT_PHY_QCFG_RESP_DUPLEX_STATE_FULL;
 				phy_qcfg_resp.pause =
 					(PORT_PHY_QCFG_RESP_PAUSE_TX |
 					 PORT_PHY_QCFG_RESP_PAUSE_RX);
@@ -804,7 +812,8 @@ static int bnxt_vf_set_link(struct bnxt *bp, struct bnxt_vf_info *vf)
 			/* force link down */
 			phy_qcfg_resp.link = PORT_PHY_QCFG_RESP_LINK_NO_LINK;
 			phy_qcfg_resp.link_speed = 0;
-			phy_qcfg_resp.duplex = PORT_PHY_QCFG_RESP_DUPLEX_HALF;
+			phy_qcfg_resp.duplex_state =
+				PORT_PHY_QCFG_RESP_DUPLEX_STATE_HALF;
 			phy_qcfg_resp.pause = 0;
 		}
 		rc = bnxt_hwrm_fwd_resp(bp, vf, &phy_qcfg_resp,
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c
new file mode 100644
index 000000000000..ccd699fb2d70
--- /dev/null
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c
@@ -0,0 +1,834 @@
+/* Broadcom NetXtreme-C/E network driver.
+ *
+ * Copyright (c) 2017 Broadcom Limited
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.
+ */
+
+#include <linux/netdevice.h>
+#include <linux/inetdevice.h>
+#include <linux/if_vlan.h>
+#include <net/flow_dissector.h>
+#include <net/pkt_cls.h>
+#include <net/tc_act/tc_gact.h>
+#include <net/tc_act/tc_skbedit.h>
+#include <net/tc_act/tc_mirred.h>
+#include <net/tc_act/tc_vlan.h>
+
+#include "bnxt_hsi.h"
+#include "bnxt.h"
+#include "bnxt_sriov.h"
+#include "bnxt_tc.h"
+#include "bnxt_vfr.h"
+
+#ifdef CONFIG_BNXT_FLOWER_OFFLOAD
+
+#define BNXT_FID_INVALID			0xffff
+#define VLAN_TCI(vid, prio)	((vid) | ((prio) << VLAN_PRIO_SHIFT))
+
+/* Return the dst fid of the func for flow forwarding
+ * For PFs: src_fid is the fid of the PF
+ * For VF-reps: src_fid the fid of the VF
+ */
+static u16 bnxt_flow_get_dst_fid(struct bnxt *pf_bp, struct net_device *dev)
+{
+	struct bnxt *bp;
+
+	/* check if dev belongs to the same switch */
+	if (!switchdev_port_same_parent_id(pf_bp->dev, dev)) {
+		netdev_info(pf_bp->dev, "dev(ifindex=%d) not on same switch",
+			    dev->ifindex);
+		return BNXT_FID_INVALID;
+	}
+
+	/* Is dev a VF-rep? */
+	if (dev != pf_bp->dev)
+		return bnxt_vf_rep_get_fid(dev);
+
+	bp = netdev_priv(dev);
+	return bp->pf.fw_fid;
+}
+
+static int bnxt_tc_parse_redir(struct bnxt *bp,
+			       struct bnxt_tc_actions *actions,
+			       const struct tc_action *tc_act)
+{
+	int ifindex = tcf_mirred_ifindex(tc_act);
+	struct net_device *dev;
+	u16 dst_fid;
+
+	dev = __dev_get_by_index(dev_net(bp->dev), ifindex);
+	if (!dev) {
+		netdev_info(bp->dev, "no dev for ifindex=%d", ifindex);
+		return -EINVAL;
+	}
+
+	/* find the FID from dev */
+	dst_fid = bnxt_flow_get_dst_fid(bp, dev);
+	if (dst_fid == BNXT_FID_INVALID) {
+		netdev_info(bp->dev, "can't get fid for ifindex=%d", ifindex);
+		return -EINVAL;
+	}
+
+	actions->flags |= BNXT_TC_ACTION_FLAG_FWD;
+	actions->dst_fid = dst_fid;
+	actions->dst_dev = dev;
+	return 0;
+}
+
+static void bnxt_tc_parse_vlan(struct bnxt *bp,
+			       struct bnxt_tc_actions *actions,
+			       const struct tc_action *tc_act)
+{
+	if (tcf_vlan_action(tc_act) == TCA_VLAN_ACT_POP) {
+		actions->flags |= BNXT_TC_ACTION_FLAG_POP_VLAN;
+	} else if (tcf_vlan_action(tc_act) == TCA_VLAN_ACT_PUSH) {
+		actions->flags |= BNXT_TC_ACTION_FLAG_PUSH_VLAN;
+		actions->push_vlan_tci = htons(tcf_vlan_push_vid(tc_act));
+		actions->push_vlan_tpid = tcf_vlan_push_proto(tc_act);
+	}
+}
+
+static int bnxt_tc_parse_actions(struct bnxt *bp,
+				 struct bnxt_tc_actions *actions,
+				 struct tcf_exts *tc_exts)
+{
+	const struct tc_action *tc_act;
+	LIST_HEAD(tc_actions);
+	int rc;
+
+	if (!tcf_exts_has_actions(tc_exts)) {
+		netdev_info(bp->dev, "no actions");
+		return -EINVAL;
+	}
+
+	tcf_exts_to_list(tc_exts, &tc_actions);
+	list_for_each_entry(tc_act, &tc_actions, list) {
+		/* Drop action */
+		if (is_tcf_gact_shot(tc_act)) {
+			actions->flags |= BNXT_TC_ACTION_FLAG_DROP;
+			return 0; /* don't bother with other actions */
+		}
+
+		/* Redirect action */
+		if (is_tcf_mirred_egress_redirect(tc_act)) {
+			rc = bnxt_tc_parse_redir(bp, actions, tc_act);
+			if (rc)
+				return rc;
+			continue;
+		}
+
+		/* Push/pop VLAN */
+		if (is_tcf_vlan(tc_act)) {
+			bnxt_tc_parse_vlan(bp, actions, tc_act);
+			continue;
+		}
+	}
+
+	return 0;
+}
+
+#define GET_KEY(flow_cmd, key_type)					\
+		skb_flow_dissector_target((flow_cmd)->dissector, key_type,\
+					  (flow_cmd)->key)
+#define GET_MASK(flow_cmd, key_type)					\
+		skb_flow_dissector_target((flow_cmd)->dissector, key_type,\
+					  (flow_cmd)->mask)
+
+static int bnxt_tc_parse_flow(struct bnxt *bp,
+			      struct tc_cls_flower_offload *tc_flow_cmd,
+			      struct bnxt_tc_flow *flow)
+{
+	struct flow_dissector *dissector = tc_flow_cmd->dissector;
+	u16 addr_type = 0;
+
+	/* KEY_CONTROL and KEY_BASIC are needed for forming a meaningful key */
+	if ((dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CONTROL)) == 0 ||
+	    (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_BASIC)) == 0) {
+		netdev_info(bp->dev, "cannot form TC key: used_keys = 0x%x",
+			    dissector->used_keys);
+		return -EOPNOTSUPP;
+	}
+
+	if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_CONTROL)) {
+		struct flow_dissector_key_control *key =
+			GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_CONTROL);
+
+		addr_type = key->addr_type;
+	}
+
+	if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_BASIC)) {
+		struct flow_dissector_key_basic *key =
+			GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_BASIC);
+		struct flow_dissector_key_basic *mask =
+			GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_BASIC);
+
+		flow->l2_key.ether_type = key->n_proto;
+		flow->l2_mask.ether_type = mask->n_proto;
+
+		if (key->n_proto == htons(ETH_P_IP) ||
+		    key->n_proto == htons(ETH_P_IPV6)) {
+			flow->l4_key.ip_proto = key->ip_proto;
+			flow->l4_mask.ip_proto = mask->ip_proto;
+		}
+	}
+
+	if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
+		struct flow_dissector_key_eth_addrs *key =
+			GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ETH_ADDRS);
+		struct flow_dissector_key_eth_addrs *mask =
+			GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_ETH_ADDRS);
+
+		flow->flags |= BNXT_TC_FLOW_FLAGS_ETH_ADDRS;
+		ether_addr_copy(flow->l2_key.dmac, key->dst);
+		ether_addr_copy(flow->l2_mask.dmac, mask->dst);
+		ether_addr_copy(flow->l2_key.smac, key->src);
+		ether_addr_copy(flow->l2_mask.smac, mask->src);
+	}
+
+	if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_VLAN)) {
+		struct flow_dissector_key_vlan *key =
+			GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_VLAN);
+		struct flow_dissector_key_vlan *mask =
+			GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_VLAN);
+
+		flow->l2_key.inner_vlan_tci =
+		   cpu_to_be16(VLAN_TCI(key->vlan_id, key->vlan_priority));
+		flow->l2_mask.inner_vlan_tci =
+		   cpu_to_be16((VLAN_TCI(mask->vlan_id, mask->vlan_priority)));
+		flow->l2_key.inner_vlan_tpid = htons(ETH_P_8021Q);
+		flow->l2_mask.inner_vlan_tpid = htons(0xffff);
+		flow->l2_key.num_vlans = 1;
+	}
+
+	if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
+		struct flow_dissector_key_ipv4_addrs *key =
+			GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV4_ADDRS);
+		struct flow_dissector_key_ipv4_addrs *mask =
+			GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV4_ADDRS);
+
+		flow->flags |= BNXT_TC_FLOW_FLAGS_IPV4_ADDRS;
+		flow->l3_key.ipv4.daddr.s_addr = key->dst;
+		flow->l3_mask.ipv4.daddr.s_addr = mask->dst;
+		flow->l3_key.ipv4.saddr.s_addr = key->src;
+		flow->l3_mask.ipv4.saddr.s_addr = mask->src;
+	} else if (dissector_uses_key(dissector,
+				      FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
+		struct flow_dissector_key_ipv6_addrs *key =
+			GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV6_ADDRS);
+		struct flow_dissector_key_ipv6_addrs *mask =
+			GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV6_ADDRS);
+
+		flow->flags |= BNXT_TC_FLOW_FLAGS_IPV6_ADDRS;
+		flow->l3_key.ipv6.daddr = key->dst;
+		flow->l3_mask.ipv6.daddr = mask->dst;
+		flow->l3_key.ipv6.saddr = key->src;
+		flow->l3_mask.ipv6.saddr = mask->src;
+	}
+
+	if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_PORTS)) {
+		struct flow_dissector_key_ports *key =
+			GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_PORTS);
+		struct flow_dissector_key_ports *mask =
+			GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_PORTS);
+
+		flow->flags |= BNXT_TC_FLOW_FLAGS_PORTS;
+		flow->l4_key.ports.dport = key->dst;
+		flow->l4_mask.ports.dport = mask->dst;
+		flow->l4_key.ports.sport = key->src;
+		flow->l4_mask.ports.sport = mask->src;
+	}
+
+	if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ICMP)) {
+		struct flow_dissector_key_icmp *key =
+			GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ICMP);
+		struct flow_dissector_key_icmp *mask =
+			GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_ICMP);
+
+		flow->flags |= BNXT_TC_FLOW_FLAGS_ICMP;
+		flow->l4_key.icmp.type = key->type;
+		flow->l4_key.icmp.code = key->code;
+		flow->l4_mask.icmp.type = mask->type;
+		flow->l4_mask.icmp.code = mask->code;
+	}
+
+	return bnxt_tc_parse_actions(bp, &flow->actions, tc_flow_cmd->exts);
+}
+
+static int bnxt_hwrm_cfa_flow_free(struct bnxt *bp, __le16 flow_handle)
+{
+	struct hwrm_cfa_flow_free_input req = { 0 };
+	int rc;
+
+	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_FLOW_FREE, -1, -1);
+	req.flow_handle = flow_handle;
+
+	rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
+	if (rc)
+		netdev_info(bp->dev, "Error: %s: flow_handle=0x%x rc=%d",
+			    __func__, flow_handle, rc);
+	return rc;
+}
+
+static int ipv6_mask_len(struct in6_addr *mask)
+{
+	int mask_len = 0, i;
+
+	for (i = 0; i < 4; i++)
+		mask_len += inet_mask_len(mask->s6_addr32[i]);
+
+	return mask_len;
+}
+
+static bool is_wildcard(void *mask, int len)
+{
+	const u8 *p = mask;
+	int i;
+
+	for (i = 0; i < len; i++) {
+		if (p[i] != 0)
+			return false;
+	}
+	return true;
+}
+
+static int bnxt_hwrm_cfa_flow_alloc(struct bnxt *bp, struct bnxt_tc_flow *flow,
+				    __le16 ref_flow_handle, __le16 *flow_handle)
+{
+	struct hwrm_cfa_flow_alloc_output *resp = bp->hwrm_cmd_resp_addr;
+	struct bnxt_tc_actions *actions = &flow->actions;
+	struct bnxt_tc_l3_key *l3_mask = &flow->l3_mask;
+	struct bnxt_tc_l3_key *l3_key = &flow->l3_key;
+	struct hwrm_cfa_flow_alloc_input req = { 0 };
+	u16 flow_flags = 0, action_flags = 0;
+	int rc;
+
+	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_FLOW_ALLOC, -1, -1);
+
+	req.src_fid = cpu_to_le16(flow->src_fid);
+	req.ref_flow_handle = ref_flow_handle;
+	req.ethertype = flow->l2_key.ether_type;
+	req.ip_proto = flow->l4_key.ip_proto;
+
+	if (flow->flags & BNXT_TC_FLOW_FLAGS_ETH_ADDRS) {
+		memcpy(req.dmac, flow->l2_key.dmac, ETH_ALEN);
+		memcpy(req.smac, flow->l2_key.smac, ETH_ALEN);
+	}
+
+	if (flow->l2_key.num_vlans > 0) {
+		flow_flags |= CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_ONE;
+		/* FW expects the inner_vlan_tci value to be set
+		 * in outer_vlan_tci when num_vlans is 1 (which is
+		 * always the case in TC.)
+		 */
+		req.outer_vlan_tci = flow->l2_key.inner_vlan_tci;
+	}
+
+	/* If all IP and L4 fields are wildcarded then this is an L2 flow */
+	if (is_wildcard(&l3_mask, sizeof(l3_mask)) &&
+	    is_wildcard(&flow->l4_mask, sizeof(flow->l4_mask))) {
+		flow_flags |= CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_L2;
+	} else {
+		flow_flags |= flow->l2_key.ether_type == htons(ETH_P_IP) ?
+				CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV4 :
+				CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV6;
+
+		if (flow->flags & BNXT_TC_FLOW_FLAGS_IPV4_ADDRS) {
+			req.ip_dst[0] = l3_key->ipv4.daddr.s_addr;
+			req.ip_dst_mask_len =
+				inet_mask_len(l3_mask->ipv4.daddr.s_addr);
+			req.ip_src[0] = l3_key->ipv4.saddr.s_addr;
+			req.ip_src_mask_len =
+				inet_mask_len(l3_mask->ipv4.saddr.s_addr);
+		} else if (flow->flags & BNXT_TC_FLOW_FLAGS_IPV6_ADDRS) {
+			memcpy(req.ip_dst, l3_key->ipv6.daddr.s6_addr32,
+			       sizeof(req.ip_dst));
+			req.ip_dst_mask_len =
+					ipv6_mask_len(&l3_mask->ipv6.daddr);
+			memcpy(req.ip_src, l3_key->ipv6.saddr.s6_addr32,
+			       sizeof(req.ip_src));
+			req.ip_src_mask_len =
+					ipv6_mask_len(&l3_mask->ipv6.saddr);
+		}
+	}
+
+	if (flow->flags & BNXT_TC_FLOW_FLAGS_PORTS) {
+		req.l4_src_port = flow->l4_key.ports.sport;
+		req.l4_src_port_mask = flow->l4_mask.ports.sport;
+		req.l4_dst_port = flow->l4_key.ports.dport;
+		req.l4_dst_port_mask = flow->l4_mask.ports.dport;
+	} else if (flow->flags & BNXT_TC_FLOW_FLAGS_ICMP) {
+		/* l4 ports serve as type/code when ip_proto is ICMP */
+		req.l4_src_port = htons(flow->l4_key.icmp.type);
+		req.l4_src_port_mask = htons(flow->l4_mask.icmp.type);
+		req.l4_dst_port = htons(flow->l4_key.icmp.code);
+		req.l4_dst_port_mask = htons(flow->l4_mask.icmp.code);
+	}
+	req.flags = cpu_to_le16(flow_flags);
+
+	if (actions->flags & BNXT_TC_ACTION_FLAG_DROP) {
+		action_flags |= CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_DROP;
+	} else {
+		if (actions->flags & BNXT_TC_ACTION_FLAG_FWD) {
+			action_flags |= CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_FWD;
+			req.dst_fid = cpu_to_le16(actions->dst_fid);
+		}
+		if (actions->flags & BNXT_TC_ACTION_FLAG_PUSH_VLAN) {
+			action_flags |=
+			    CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_L2_HEADER_REWRITE;
+			req.l2_rewrite_vlan_tpid = actions->push_vlan_tpid;
+			req.l2_rewrite_vlan_tci = actions->push_vlan_tci;
+			memcpy(&req.l2_rewrite_dmac, &req.dmac, ETH_ALEN);
+			memcpy(&req.l2_rewrite_smac, &req.smac, ETH_ALEN);
+		}
+		if (actions->flags & BNXT_TC_ACTION_FLAG_POP_VLAN) {
+			action_flags |=
+			    CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_L2_HEADER_REWRITE;
+			/* Rewrite config with tpid = 0 implies vlan pop */
+			req.l2_rewrite_vlan_tpid = 0;
+			memcpy(&req.l2_rewrite_dmac, &req.dmac, ETH_ALEN);
+			memcpy(&req.l2_rewrite_smac, &req.smac, ETH_ALEN);
+		}
+	}
+	req.action_flags = cpu_to_le16(action_flags);
+
+	mutex_lock(&bp->hwrm_cmd_lock);
+
+	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
+	if (!rc)
+		*flow_handle = resp->flow_handle;
+
+	mutex_unlock(&bp->hwrm_cmd_lock);
+
+	return rc;
+}
+
+/* Add val to accum while handling a possible wraparound
+ * of val. Eventhough val is of type u64, its actual width
+ * is denoted by mask and will wrap-around beyond that width.
+ */
+static void accumulate_val(u64 *accum, u64 val, u64 mask)
+{
+#define low_bits(x, mask)		((x) & (mask))
+#define high_bits(x, mask)		((x) & ~(mask))
+	bool wrapped = val < low_bits(*accum, mask);
+
+	*accum = high_bits(*accum, mask) + val;
+	if (wrapped)
+		*accum += (mask + 1);
+}
+
+/* The HW counters' width is much less than 64bits.
+ * Handle possible wrap-around while updating the stat counters
+ */
+static void bnxt_flow_stats_fix_wraparound(struct bnxt_tc_info *tc_info,
+					   struct bnxt_tc_flow_stats *stats,
+					   struct bnxt_tc_flow_stats *hw_stats)
+{
+	accumulate_val(&stats->bytes, hw_stats->bytes, tc_info->bytes_mask);
+	accumulate_val(&stats->packets, hw_stats->packets,
+		       tc_info->packets_mask);
+}
+
+/* Fix possible wraparound of the stats queried from HW, calculate
+ * the delta from prev_stats, and also update the prev_stats.
+ * The HW flow stats are fetched under the hwrm_cmd_lock mutex.
+ * This routine is best called while under the mutex so that the
+ * stats processing happens atomically.
+ */
+static void bnxt_flow_stats_calc(struct bnxt_tc_info *tc_info,
+				 struct bnxt_tc_flow *flow,
+				 struct bnxt_tc_flow_stats *stats)
+{
+	struct bnxt_tc_flow_stats *acc_stats, *prev_stats;
+
+	acc_stats = &flow->stats;
+	bnxt_flow_stats_fix_wraparound(tc_info, acc_stats, stats);
+
+	prev_stats = &flow->prev_stats;
+	stats->bytes = acc_stats->bytes - prev_stats->bytes;
+	stats->packets = acc_stats->packets - prev_stats->packets;
+	*prev_stats = *acc_stats;
+}
+
+static int bnxt_hwrm_cfa_flow_stats_get(struct bnxt *bp,
+					__le16 flow_handle,
+					struct bnxt_tc_flow *flow,
+					struct bnxt_tc_flow_stats *stats)
+{
+	struct hwrm_cfa_flow_stats_output *resp = bp->hwrm_cmd_resp_addr;
+	struct hwrm_cfa_flow_stats_input req = { 0 };
+	int rc;
+
+	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_FLOW_STATS, -1, -1);
+	req.num_flows = cpu_to_le16(1);
+	req.flow_handle_0 = flow_handle;
+
+	mutex_lock(&bp->hwrm_cmd_lock);
+	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
+	if (!rc) {
+		stats->packets = le64_to_cpu(resp->packet_0);
+		stats->bytes = le64_to_cpu(resp->byte_0);
+		bnxt_flow_stats_calc(&bp->tc_info, flow, stats);
+	} else {
+		netdev_info(bp->dev, "error rc=%d", rc);
+	}
+
+	mutex_unlock(&bp->hwrm_cmd_lock);
+	return rc;
+}
+
+static int bnxt_tc_put_l2_node(struct bnxt *bp,
+			       struct bnxt_tc_flow_node *flow_node)
+{
+	struct bnxt_tc_l2_node *l2_node = flow_node->l2_node;
+	struct bnxt_tc_info *tc_info = &bp->tc_info;
+	int rc;
+
+	/* remove flow_node from the L2 shared flow list */
+	list_del(&flow_node->l2_list_node);
+	if (--l2_node->refcount == 0) {
+		rc =  rhashtable_remove_fast(&tc_info->l2_table, &l2_node->node,
+					     tc_info->l2_ht_params);
+		if (rc)
+			netdev_err(bp->dev,
+				   "Error: %s: rhashtable_remove_fast: %d",
+				   __func__, rc);
+		kfree_rcu(l2_node, rcu);
+	}
+	return 0;
+}
+
+static struct bnxt_tc_l2_node *
+bnxt_tc_get_l2_node(struct bnxt *bp, struct rhashtable *l2_table,
+		    struct rhashtable_params ht_params,
+		    struct bnxt_tc_l2_key *l2_key)
+{
+	struct bnxt_tc_l2_node *l2_node;
+	int rc;
+
+	l2_node = rhashtable_lookup_fast(l2_table, l2_key, ht_params);
+	if (!l2_node) {
+		l2_node = kzalloc(sizeof(*l2_node), GFP_KERNEL);
+		if (!l2_node) {
+			rc = -ENOMEM;
+			return NULL;
+		}
+
+		l2_node->key = *l2_key;
+		rc = rhashtable_insert_fast(l2_table, &l2_node->node,
+					    ht_params);
+		if (rc) {
+			kfree(l2_node);
+			netdev_err(bp->dev,
+				   "Error: %s: rhashtable_insert_fast: %d",
+				   __func__, rc);
+			return NULL;
+		}
+		INIT_LIST_HEAD(&l2_node->common_l2_flows);
+	}
+	return l2_node;
+}
+
+/* Get the ref_flow_handle for a flow by checking if there are any other
+ * flows that share the same L2 key as this flow.
+ */
+static int
+bnxt_tc_get_ref_flow_handle(struct bnxt *bp, struct bnxt_tc_flow *flow,
+			    struct bnxt_tc_flow_node *flow_node,
+			    __le16 *ref_flow_handle)
+{
+	struct bnxt_tc_info *tc_info = &bp->tc_info;
+	struct bnxt_tc_flow_node *ref_flow_node;
+	struct bnxt_tc_l2_node *l2_node;
+
+	l2_node = bnxt_tc_get_l2_node(bp, &tc_info->l2_table,
+				      tc_info->l2_ht_params,
+				      &flow->l2_key);
+	if (!l2_node)
+		return -1;
+
+	/* If any other flow is using this l2_node, use it's flow_handle
+	 * as the ref_flow_handle
+	 */
+	if (l2_node->refcount > 0) {
+		ref_flow_node = list_first_entry(&l2_node->common_l2_flows,
+						 struct bnxt_tc_flow_node,
+						 l2_list_node);
+		*ref_flow_handle = ref_flow_node->flow_handle;
+	} else {
+		*ref_flow_handle = cpu_to_le16(0xffff);
+	}
+
+	/* Insert the l2_node into the flow_node so that subsequent flows
+	 * with a matching l2 key can use the flow_handle of this flow
+	 * as their ref_flow_handle
+	 */
+	flow_node->l2_node = l2_node;
+	list_add(&flow_node->l2_list_node, &l2_node->common_l2_flows);
+	l2_node->refcount++;
+	return 0;
+}
+
+/* After the flow parsing is done, this routine is used for checking
+ * if there are any aspects of the flow that prevent it from being
+ * offloaded.
+ */
+static bool bnxt_tc_can_offload(struct bnxt *bp, struct bnxt_tc_flow *flow)
+{
+	/* If L4 ports are specified then ip_proto must be TCP or UDP */
+	if ((flow->flags & BNXT_TC_FLOW_FLAGS_PORTS) &&
+	    (flow->l4_key.ip_proto != IPPROTO_TCP &&
+	     flow->l4_key.ip_proto != IPPROTO_UDP)) {
+		netdev_info(bp->dev, "Cannot offload non-TCP/UDP (%d) ports",
+			    flow->l4_key.ip_proto);
+		return false;
+	}
+
+	return true;
+}
+
+static int __bnxt_tc_del_flow(struct bnxt *bp,
+			      struct bnxt_tc_flow_node *flow_node)
+{
+	struct bnxt_tc_info *tc_info = &bp->tc_info;
+	int rc;
+
+	/* send HWRM cmd to free the flow-id */
+	bnxt_hwrm_cfa_flow_free(bp, flow_node->flow_handle);
+
+	mutex_lock(&tc_info->lock);
+
+	/* release reference to l2 node */
+	bnxt_tc_put_l2_node(bp, flow_node);
+
+	mutex_unlock(&tc_info->lock);
+
+	rc = rhashtable_remove_fast(&tc_info->flow_table, &flow_node->node,
+				    tc_info->flow_ht_params);
+	if (rc)
+		netdev_err(bp->dev, "Error: %s: rhashtable_remove_fast rc=%d",
+			   __func__, rc);
+
+	kfree_rcu(flow_node, rcu);
+	return 0;
+}
+
+/* Add a new flow or replace an existing flow.
+ * Notes on locking:
+ * There are essentially two critical sections here.
+ * 1. while adding a new flow
+ *    a) lookup l2-key
+ *    b) issue HWRM cmd and get flow_handle
+ *    c) link l2-key with flow
+ * 2. while deleting a flow
+ *    a) unlinking l2-key from flow
+ * A lock is needed to protect these two critical sections.
+ *
+ * The hash-tables are already protected by the rhashtable API.
+ */
+static int bnxt_tc_add_flow(struct bnxt *bp, u16 src_fid,
+			    struct tc_cls_flower_offload *tc_flow_cmd)
+{
+	struct bnxt_tc_flow_node *new_node, *old_node;
+	struct bnxt_tc_info *tc_info = &bp->tc_info;
+	struct bnxt_tc_flow *flow;
+	__le16 ref_flow_handle;
+	int rc;
+
+	/* allocate memory for the new flow and it's node */
+	new_node = kzalloc(sizeof(*new_node), GFP_KERNEL);
+	if (!new_node) {
+		rc = -ENOMEM;
+		goto done;
+	}
+	new_node->cookie = tc_flow_cmd->cookie;
+	flow = &new_node->flow;
+
+	rc = bnxt_tc_parse_flow(bp, tc_flow_cmd, flow);
+	if (rc)
+		goto free_node;
+	flow->src_fid = src_fid;
+
+	if (!bnxt_tc_can_offload(bp, flow)) {
+		rc = -ENOSPC;
+		goto free_node;
+	}
+
+	/* If a flow exists with the same cookie, delete it */
+	old_node = rhashtable_lookup_fast(&tc_info->flow_table,
+					  &tc_flow_cmd->cookie,
+					  tc_info->flow_ht_params);
+	if (old_node)
+		__bnxt_tc_del_flow(bp, old_node);
+
+	/* Check if the L2 part of the flow has been offloaded already.
+	 * If so, bump up it's refcnt and get it's reference handle.
+	 */
+	mutex_lock(&tc_info->lock);
+	rc = bnxt_tc_get_ref_flow_handle(bp, flow, new_node, &ref_flow_handle);
+	if (rc)
+		goto unlock;
+
+	/* send HWRM cmd to alloc the flow */
+	rc = bnxt_hwrm_cfa_flow_alloc(bp, flow, ref_flow_handle,
+				      &new_node->flow_handle);
+	if (rc)
+		goto put_l2;
+
+	/* add new flow to flow-table */
+	rc = rhashtable_insert_fast(&tc_info->flow_table, &new_node->node,
+				    tc_info->flow_ht_params);
+	if (rc)
+		goto hwrm_flow_free;
+
+	mutex_unlock(&tc_info->lock);
+	return 0;
+
+hwrm_flow_free:
+	bnxt_hwrm_cfa_flow_free(bp, new_node->flow_handle);
+put_l2:
+	bnxt_tc_put_l2_node(bp, new_node);
+unlock:
+	mutex_unlock(&tc_info->lock);
+free_node:
+	kfree(new_node);
+done:
+	netdev_err(bp->dev, "Error: %s: cookie=0x%lx error=%d",
+		   __func__, tc_flow_cmd->cookie, rc);
+	return rc;
+}
+
+static int bnxt_tc_del_flow(struct bnxt *bp,
+			    struct tc_cls_flower_offload *tc_flow_cmd)
+{
+	struct bnxt_tc_info *tc_info = &bp->tc_info;
+	struct bnxt_tc_flow_node *flow_node;
+
+	flow_node = rhashtable_lookup_fast(&tc_info->flow_table,
+					   &tc_flow_cmd->cookie,
+					   tc_info->flow_ht_params);
+	if (!flow_node) {
+		netdev_info(bp->dev, "ERROR: no flow_node for cookie %lx",
+			    tc_flow_cmd->cookie);
+		return -EINVAL;
+	}
+
+	return __bnxt_tc_del_flow(bp, flow_node);
+}
+
+static int bnxt_tc_get_flow_stats(struct bnxt *bp,
+				  struct tc_cls_flower_offload *tc_flow_cmd)
+{
+	struct bnxt_tc_info *tc_info = &bp->tc_info;
+	struct bnxt_tc_flow_node *flow_node;
+	struct bnxt_tc_flow_stats stats;
+	int rc;
+
+	flow_node = rhashtable_lookup_fast(&tc_info->flow_table,
+					   &tc_flow_cmd->cookie,
+					   tc_info->flow_ht_params);
+	if (!flow_node) {
+		netdev_info(bp->dev, "Error: no flow_node for cookie %lx",
+			    tc_flow_cmd->cookie);
+		return -1;
+	}
+
+	rc = bnxt_hwrm_cfa_flow_stats_get(bp, flow_node->flow_handle,
+					  &flow_node->flow, &stats);
+	if (rc)
+		return rc;
+
+	tcf_exts_stats_update(tc_flow_cmd->exts, stats.bytes, stats.packets, 0);
+	return 0;
+}
+
+int bnxt_tc_setup_flower(struct bnxt *bp, u16 src_fid,
+			 struct tc_cls_flower_offload *cls_flower)
+{
+	int rc = 0;
+
+	switch (cls_flower->command) {
+	case TC_CLSFLOWER_REPLACE:
+		rc = bnxt_tc_add_flow(bp, src_fid, cls_flower);
+		break;
+
+	case TC_CLSFLOWER_DESTROY:
+		rc = bnxt_tc_del_flow(bp, cls_flower);
+		break;
+
+	case TC_CLSFLOWER_STATS:
+		rc = bnxt_tc_get_flow_stats(bp, cls_flower);
+		break;
+	}
+	return rc;
+}
+
+static const struct rhashtable_params bnxt_tc_flow_ht_params = {
+	.head_offset = offsetof(struct bnxt_tc_flow_node, node),
+	.key_offset = offsetof(struct bnxt_tc_flow_node, cookie),
+	.key_len = sizeof(((struct bnxt_tc_flow_node *)0)->cookie),
+	.automatic_shrinking = true
+};
+
+static const struct rhashtable_params bnxt_tc_l2_ht_params = {
+	.head_offset = offsetof(struct bnxt_tc_l2_node, node),
+	.key_offset = offsetof(struct bnxt_tc_l2_node, key),
+	.key_len = BNXT_TC_L2_KEY_LEN,
+	.automatic_shrinking = true
+};
+
+/* convert counter width in bits to a mask */
+#define mask(width)		((u64)~0 >> (64 - (width)))
+
+int bnxt_init_tc(struct bnxt *bp)
+{
+	struct bnxt_tc_info *tc_info = &bp->tc_info;
+	int rc;
+
+	if (bp->hwrm_spec_code < 0x10800) {
+		netdev_warn(bp->dev,
+			    "Firmware does not support TC flower offload.\n");
+		return -ENOTSUPP;
+	}
+	mutex_init(&tc_info->lock);
+
+	/* Counter widths are programmed by FW */
+	tc_info->bytes_mask = mask(36);
+	tc_info->packets_mask = mask(28);
+
+	tc_info->flow_ht_params = bnxt_tc_flow_ht_params;
+	rc = rhashtable_init(&tc_info->flow_table, &tc_info->flow_ht_params);
+	if (rc)
+		return rc;
+
+	tc_info->l2_ht_params = bnxt_tc_l2_ht_params;
+	rc = rhashtable_init(&tc_info->l2_table, &tc_info->l2_ht_params);
+	if (rc)
+		goto destroy_flow_table;
+
+	tc_info->enabled = true;
+	bp->dev->hw_features |= NETIF_F_HW_TC;
+	bp->dev->features |= NETIF_F_HW_TC;
+	return 0;
+
+destroy_flow_table:
+	rhashtable_destroy(&tc_info->flow_table);
+	return rc;
+}
+
+void bnxt_shutdown_tc(struct bnxt *bp)
+{
+	struct bnxt_tc_info *tc_info = &bp->tc_info;
+
+	if (!tc_info->enabled)
+		return;
+
+	rhashtable_destroy(&tc_info->flow_table);
+	rhashtable_destroy(&tc_info->l2_table);
+}
+
+#else
+#endif
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.h
new file mode 100644
index 000000000000..6c4c1ed279ef
--- /dev/null
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.h
@@ -0,0 +1,158 @@
+/* Broadcom NetXtreme-C/E network driver.
+ *
+ * Copyright (c) 2017 Broadcom Limited
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.
+ */
+
+#ifndef BNXT_TC_H
+#define BNXT_TC_H
+
+#ifdef CONFIG_BNXT_FLOWER_OFFLOAD
+
+/* Structs used for storing the filter/actions of the TC cmd.
+ */
+struct bnxt_tc_l2_key {
+	u8		dmac[ETH_ALEN];
+	u8		smac[ETH_ALEN];
+	__be16		inner_vlan_tpid;
+	__be16		inner_vlan_tci;
+	__be16		ether_type;
+	u8		num_vlans;
+};
+
+struct bnxt_tc_l3_key {
+	union {
+		struct {
+			struct in_addr daddr;
+			struct in_addr saddr;
+		} ipv4;
+		struct {
+			struct in6_addr daddr;
+			struct in6_addr saddr;
+		} ipv6;
+	};
+};
+
+struct bnxt_tc_l4_key {
+	u8  ip_proto;
+	union {
+		struct {
+			__be16 sport;
+			__be16 dport;
+		} ports;
+		struct {
+			u8 type;
+			u8 code;
+		} icmp;
+	};
+};
+
+struct bnxt_tc_actions {
+	u32				flags;
+#define BNXT_TC_ACTION_FLAG_FWD			BIT(0)
+#define BNXT_TC_ACTION_FLAG_FWD_VXLAN		BIT(1)
+#define BNXT_TC_ACTION_FLAG_PUSH_VLAN		BIT(3)
+#define BNXT_TC_ACTION_FLAG_POP_VLAN		BIT(4)
+#define BNXT_TC_ACTION_FLAG_DROP		BIT(5)
+
+	u16				dst_fid;
+	struct net_device		*dst_dev;
+	__be16				push_vlan_tpid;
+	__be16				push_vlan_tci;
+};
+
+struct bnxt_tc_flow_stats {
+	u64		packets;
+	u64		bytes;
+};
+
+struct bnxt_tc_flow {
+	u32				flags;
+#define BNXT_TC_FLOW_FLAGS_ETH_ADDRS		BIT(1)
+#define BNXT_TC_FLOW_FLAGS_IPV4_ADDRS		BIT(2)
+#define BNXT_TC_FLOW_FLAGS_IPV6_ADDRS		BIT(3)
+#define BNXT_TC_FLOW_FLAGS_PORTS		BIT(4)
+#define BNXT_TC_FLOW_FLAGS_ICMP			BIT(5)
+
+	/* flow applicable to pkts ingressing on this fid */
+	u16				src_fid;
+	struct bnxt_tc_l2_key		l2_key;
+	struct bnxt_tc_l2_key		l2_mask;
+	struct bnxt_tc_l3_key		l3_key;
+	struct bnxt_tc_l3_key		l3_mask;
+	struct bnxt_tc_l4_key		l4_key;
+	struct bnxt_tc_l4_key		l4_mask;
+
+	struct bnxt_tc_actions		actions;
+
+	/* updated stats accounting for hw-counter wrap-around */
+	struct bnxt_tc_flow_stats	stats;
+	/* previous snap-shot of stats */
+	struct bnxt_tc_flow_stats	prev_stats;
+	unsigned long			lastused; /* jiffies */
+};
+
+/* L2 hash table
+ * This data-struct is used for L2-flow table.
+ * The L2 part of a flow is stored in a hash table.
+ * A flow that shares the same L2 key/mask with an
+ * already existing flow must refer to it's flow handle.
+ */
+struct bnxt_tc_l2_node {
+	/* hash key: first 16b of key */
+#define BNXT_TC_L2_KEY_LEN			16
+	struct bnxt_tc_l2_key	key;
+	struct rhash_head	node;
+
+	/* a linked list of flows that share the same l2 key */
+	struct list_head	common_l2_flows;
+
+	/* number of flows sharing the l2 key */
+	u16			refcount;
+
+	struct rcu_head		rcu;
+};
+
+struct bnxt_tc_flow_node {
+	/* hash key: provided by TC */
+	unsigned long			cookie;
+	struct rhash_head		node;
+
+	struct bnxt_tc_flow		flow;
+
+	__le16				flow_handle;
+
+	/* L2 node in l2 hashtable that shares flow's l2 key */
+	struct bnxt_tc_l2_node		*l2_node;
+	/* for the shared_flows list maintained in l2_node */
+	struct list_head		l2_list_node;
+
+	struct rcu_head			rcu;
+};
+
+int bnxt_tc_setup_flower(struct bnxt *bp, u16 src_fid,
+			 struct tc_cls_flower_offload *cls_flower);
+int bnxt_init_tc(struct bnxt *bp);
+void bnxt_shutdown_tc(struct bnxt *bp);
+
+#else /* CONFIG_BNXT_FLOWER_OFFLOAD */
+
+static inline int bnxt_tc_setup_flower(struct bnxt *bp, u16 src_fid,
+				       struct tc_cls_flower_offload *cls_flower)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int bnxt_init_tc(struct bnxt *bp)
+{
+	return 0;
+}
+
+static inline void bnxt_shutdown_tc(struct bnxt *bp)
+{
+}
+#endif /* CONFIG_BNXT_FLOWER_OFFLOAD */
+#endif /* BNXT_TC_H */
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c
index 77da75a55c02..997e10e8b863 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c
@@ -84,6 +84,8 @@ static int bnxt_unregister_dev(struct bnxt_en_dev *edev, int ulp_id)
 
 		max_stat_ctxs = bnxt_get_max_func_stat_ctxs(bp);
 		bnxt_set_max_func_stat_ctxs(bp, max_stat_ctxs + 1);
+		if (ulp->msix_requested)
+			edev->en_ops->bnxt_free_msix(edev, ulp_id);
 	}
 	if (ulp->max_async_event_id)
 		bnxt_hwrm_func_rgtr_async_events(bp, NULL, 0);
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c
new file mode 100644
index 000000000000..e75db04c6cdc
--- /dev/null
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c
@@ -0,0 +1,513 @@
+/* Broadcom NetXtreme-C/E network driver.
+ *
+ * Copyright (c) 2016-2017 Broadcom Limited
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.
+ */
+#include <linux/pci.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/rtnetlink.h>
+#include <linux/jhash.h>
+#include <net/pkt_cls.h>
+
+#include "bnxt_hsi.h"
+#include "bnxt.h"
+#include "bnxt_vfr.h"
+#include "bnxt_tc.h"
+
+#ifdef CONFIG_BNXT_SRIOV
+
+#define CFA_HANDLE_INVALID		0xffff
+#define VF_IDX_INVALID			0xffff
+
+static int hwrm_cfa_vfr_alloc(struct bnxt *bp, u16 vf_idx,
+			      u16 *tx_cfa_action, u16 *rx_cfa_code)
+{
+	struct hwrm_cfa_vfr_alloc_output *resp = bp->hwrm_cmd_resp_addr;
+	struct hwrm_cfa_vfr_alloc_input req = { 0 };
+	int rc;
+
+	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_VFR_ALLOC, -1, -1);
+	req.vf_id = cpu_to_le16(vf_idx);
+	sprintf(req.vfr_name, "vfr%d", vf_idx);
+
+	mutex_lock(&bp->hwrm_cmd_lock);
+	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
+	if (!rc) {
+		*tx_cfa_action = le16_to_cpu(resp->tx_cfa_action);
+		*rx_cfa_code = le16_to_cpu(resp->rx_cfa_code);
+		netdev_dbg(bp->dev, "tx_cfa_action=0x%x, rx_cfa_code=0x%x",
+			   *tx_cfa_action, *rx_cfa_code);
+	} else {
+		netdev_info(bp->dev, "%s error rc=%d", __func__, rc);
+	}
+
+	mutex_unlock(&bp->hwrm_cmd_lock);
+	return rc;
+}
+
+static int hwrm_cfa_vfr_free(struct bnxt *bp, u16 vf_idx)
+{
+	struct hwrm_cfa_vfr_free_input req = { 0 };
+	int rc;
+
+	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_VFR_FREE, -1, -1);
+	sprintf(req.vfr_name, "vfr%d", vf_idx);
+
+	rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
+	if (rc)
+		netdev_info(bp->dev, "%s error rc=%d", __func__, rc);
+	return rc;
+}
+
+static int bnxt_vf_rep_open(struct net_device *dev)
+{
+	struct bnxt_vf_rep *vf_rep = netdev_priv(dev);
+	struct bnxt *bp = vf_rep->bp;
+
+	/* Enable link and TX only if the parent PF is open. */
+	if (netif_running(bp->dev)) {
+		netif_carrier_on(dev);
+		netif_tx_start_all_queues(dev);
+	}
+	return 0;
+}
+
+static int bnxt_vf_rep_close(struct net_device *dev)
+{
+	netif_carrier_off(dev);
+	netif_tx_disable(dev);
+
+	return 0;
+}
+
+static netdev_tx_t bnxt_vf_rep_xmit(struct sk_buff *skb,
+				    struct net_device *dev)
+{
+	struct bnxt_vf_rep *vf_rep = netdev_priv(dev);
+	int rc, len = skb->len;
+
+	skb_dst_drop(skb);
+	dst_hold((struct dst_entry *)vf_rep->dst);
+	skb_dst_set(skb, (struct dst_entry *)vf_rep->dst);
+	skb->dev = vf_rep->dst->u.port_info.lower_dev;
+
+	rc = dev_queue_xmit(skb);
+	if (!rc) {
+		vf_rep->tx_stats.packets++;
+		vf_rep->tx_stats.bytes += len;
+	}
+	return rc;
+}
+
+static void
+bnxt_vf_rep_get_stats64(struct net_device *dev,
+			struct rtnl_link_stats64 *stats)
+{
+	struct bnxt_vf_rep *vf_rep = netdev_priv(dev);
+
+	stats->rx_packets = vf_rep->rx_stats.packets;
+	stats->rx_bytes = vf_rep->rx_stats.bytes;
+	stats->tx_packets = vf_rep->tx_stats.packets;
+	stats->tx_bytes = vf_rep->tx_stats.bytes;
+}
+
+static int bnxt_vf_rep_setup_tc(struct net_device *dev, enum tc_setup_type type,
+				void *type_data)
+{
+	struct bnxt_vf_rep *vf_rep = netdev_priv(dev);
+	struct bnxt *bp = vf_rep->bp;
+	int vf_fid = bp->pf.vf[vf_rep->vf_idx].fw_fid;
+
+	switch (type) {
+	case TC_SETUP_CLSFLOWER:
+		return bnxt_tc_setup_flower(bp, vf_fid, type_data);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+struct net_device *bnxt_get_vf_rep(struct bnxt *bp, u16 cfa_code)
+{
+	u16 vf_idx;
+
+	if (cfa_code && bp->cfa_code_map && BNXT_PF(bp)) {
+		vf_idx = bp->cfa_code_map[cfa_code];
+		if (vf_idx != VF_IDX_INVALID)
+			return bp->vf_reps[vf_idx]->dev;
+	}
+	return NULL;
+}
+
+void bnxt_vf_rep_rx(struct bnxt *bp, struct sk_buff *skb)
+{
+	struct bnxt_vf_rep *vf_rep = netdev_priv(skb->dev);
+	struct bnxt_vf_rep_stats *rx_stats;
+
+	rx_stats = &vf_rep->rx_stats;
+	vf_rep->rx_stats.bytes += skb->len;
+	vf_rep->rx_stats.packets++;
+
+	netif_receive_skb(skb);
+}
+
+static int bnxt_vf_rep_get_phys_port_name(struct net_device *dev, char *buf,
+					  size_t len)
+{
+	struct bnxt_vf_rep *vf_rep = netdev_priv(dev);
+	struct pci_dev *pf_pdev = vf_rep->bp->pdev;
+	int rc;
+
+	rc = snprintf(buf, len, "pf%dvf%d", PCI_FUNC(pf_pdev->devfn),
+		      vf_rep->vf_idx);
+	if (rc >= len)
+		return -EOPNOTSUPP;
+	return 0;
+}
+
+static void bnxt_vf_rep_get_drvinfo(struct net_device *dev,
+				    struct ethtool_drvinfo *info)
+{
+	strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
+	strlcpy(info->version, DRV_MODULE_VERSION, sizeof(info->version));
+}
+
+static int bnxt_vf_rep_port_attr_get(struct net_device *dev,
+				     struct switchdev_attr *attr)
+{
+	struct bnxt_vf_rep *vf_rep = netdev_priv(dev);
+
+	/* as only PORT_PARENT_ID is supported currently use common code
+	 * between PF and VF-rep for now.
+	 */
+	return bnxt_port_attr_get(vf_rep->bp, attr);
+}
+
+static const struct switchdev_ops bnxt_vf_rep_switchdev_ops = {
+	.switchdev_port_attr_get	= bnxt_vf_rep_port_attr_get
+};
+
+static const struct ethtool_ops bnxt_vf_rep_ethtool_ops = {
+	.get_drvinfo		= bnxt_vf_rep_get_drvinfo
+};
+
+static const struct net_device_ops bnxt_vf_rep_netdev_ops = {
+	.ndo_open		= bnxt_vf_rep_open,
+	.ndo_stop		= bnxt_vf_rep_close,
+	.ndo_start_xmit		= bnxt_vf_rep_xmit,
+	.ndo_get_stats64	= bnxt_vf_rep_get_stats64,
+	.ndo_setup_tc		= bnxt_vf_rep_setup_tc,
+	.ndo_get_phys_port_name = bnxt_vf_rep_get_phys_port_name
+};
+
+/* Called when the parent PF interface is closed:
+ * As the mode transition from SWITCHDEV to LEGACY
+ * happens under the rtnl_lock() this routine is safe
+ * under the rtnl_lock()
+ */
+void bnxt_vf_reps_close(struct bnxt *bp)
+{
+	struct bnxt_vf_rep *vf_rep;
+	u16 num_vfs, i;
+
+	if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
+		return;
+
+	num_vfs = pci_num_vf(bp->pdev);
+	for (i = 0; i < num_vfs; i++) {
+		vf_rep = bp->vf_reps[i];
+		if (netif_running(vf_rep->dev))
+			bnxt_vf_rep_close(vf_rep->dev);
+	}
+}
+
+/* Called when the parent PF interface is opened (re-opened):
+ * As the mode transition from SWITCHDEV to LEGACY
+ * happen under the rtnl_lock() this routine is safe
+ * under the rtnl_lock()
+ */
+void bnxt_vf_reps_open(struct bnxt *bp)
+{
+	int i;
+
+	if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
+		return;
+
+	for (i = 0; i < pci_num_vf(bp->pdev); i++)
+		bnxt_vf_rep_open(bp->vf_reps[i]->dev);
+}
+
+static void __bnxt_vf_reps_destroy(struct bnxt *bp)
+{
+	u16 num_vfs = pci_num_vf(bp->pdev);
+	struct bnxt_vf_rep *vf_rep;
+	int i;
+
+	for (i = 0; i < num_vfs; i++) {
+		vf_rep = bp->vf_reps[i];
+		if (vf_rep) {
+			dst_release((struct dst_entry *)vf_rep->dst);
+
+			if (vf_rep->tx_cfa_action != CFA_HANDLE_INVALID)
+				hwrm_cfa_vfr_free(bp, vf_rep->vf_idx);
+
+			if (vf_rep->dev) {
+				/* if register_netdev failed, then netdev_ops
+				 * would have been set to NULL
+				 */
+				if (vf_rep->dev->netdev_ops)
+					unregister_netdev(vf_rep->dev);
+				free_netdev(vf_rep->dev);
+			}
+		}
+	}
+
+	kfree(bp->vf_reps);
+	bp->vf_reps = NULL;
+}
+
+void bnxt_vf_reps_destroy(struct bnxt *bp)
+{
+	bool closed = false;
+
+	if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
+		return;
+
+	if (!bp->vf_reps)
+		return;
+
+	/* Ensure that parent PF's and VF-reps' RX/TX has been quiesced
+	 * before proceeding with VF-rep cleanup.
+	 */
+	rtnl_lock();
+	if (netif_running(bp->dev)) {
+		bnxt_close_nic(bp, false, false);
+		closed = true;
+	}
+	/* un-publish cfa_code_map so that RX path can't see it anymore */
+	kfree(bp->cfa_code_map);
+	bp->cfa_code_map = NULL;
+	bp->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY;
+
+	if (closed)
+		bnxt_open_nic(bp, false, false);
+	rtnl_unlock();
+
+	/* Need to call vf_reps_destroy() outside of rntl_lock
+	 * as unregister_netdev takes rtnl_lock
+	 */
+	__bnxt_vf_reps_destroy(bp);
+}
+
+/* Use the OUI of the PF's perm addr and report the same mac addr
+ * for the same VF-rep each time
+ */
+static void bnxt_vf_rep_eth_addr_gen(u8 *src_mac, u16 vf_idx, u8 *mac)
+{
+	u32 addr;
+
+	ether_addr_copy(mac, src_mac);
+
+	addr = jhash(src_mac, ETH_ALEN, 0) + vf_idx;
+	mac[3] = (u8)(addr & 0xFF);
+	mac[4] = (u8)((addr >> 8) & 0xFF);
+	mac[5] = (u8)((addr >> 16) & 0xFF);
+}
+
+static void bnxt_vf_rep_netdev_init(struct bnxt *bp, struct bnxt_vf_rep *vf_rep,
+				    struct net_device *dev)
+{
+	struct net_device *pf_dev = bp->dev;
+
+	dev->netdev_ops = &bnxt_vf_rep_netdev_ops;
+	dev->ethtool_ops = &bnxt_vf_rep_ethtool_ops;
+	SWITCHDEV_SET_OPS(dev, &bnxt_vf_rep_switchdev_ops);
+	/* Just inherit all the featues of the parent PF as the VF-R
+	 * uses the RX/TX rings of the parent PF
+	 */
+	dev->hw_features = pf_dev->hw_features;
+	dev->gso_partial_features = pf_dev->gso_partial_features;
+	dev->vlan_features = pf_dev->vlan_features;
+	dev->hw_enc_features = pf_dev->hw_enc_features;
+	dev->features |= pf_dev->features;
+	bnxt_vf_rep_eth_addr_gen(bp->pf.mac_addr, vf_rep->vf_idx,
+				 dev->perm_addr);
+	ether_addr_copy(dev->dev_addr, dev->perm_addr);
+}
+
+static int bnxt_vf_reps_create(struct bnxt *bp)
+{
+	u16 *cfa_code_map = NULL, num_vfs = pci_num_vf(bp->pdev);
+	struct bnxt_vf_rep *vf_rep;
+	struct net_device *dev;
+	int rc, i;
+
+	bp->vf_reps = kcalloc(num_vfs, sizeof(vf_rep), GFP_KERNEL);
+	if (!bp->vf_reps)
+		return -ENOMEM;
+
+	/* storage for cfa_code to vf-idx mapping */
+	cfa_code_map = kmalloc(sizeof(*bp->cfa_code_map) * MAX_CFA_CODE,
+			       GFP_KERNEL);
+	if (!cfa_code_map) {
+		rc = -ENOMEM;
+		goto err;
+	}
+	for (i = 0; i < MAX_CFA_CODE; i++)
+		cfa_code_map[i] = VF_IDX_INVALID;
+
+	for (i = 0; i < num_vfs; i++) {
+		dev = alloc_etherdev(sizeof(*vf_rep));
+		if (!dev) {
+			rc = -ENOMEM;
+			goto err;
+		}
+
+		vf_rep = netdev_priv(dev);
+		bp->vf_reps[i] = vf_rep;
+		vf_rep->dev = dev;
+		vf_rep->bp = bp;
+		vf_rep->vf_idx = i;
+		vf_rep->tx_cfa_action = CFA_HANDLE_INVALID;
+
+		/* get cfa handles from FW */
+		rc = hwrm_cfa_vfr_alloc(bp, vf_rep->vf_idx,
+					&vf_rep->tx_cfa_action,
+					&vf_rep->rx_cfa_code);
+		if (rc) {
+			rc = -ENOLINK;
+			goto err;
+		}
+		cfa_code_map[vf_rep->rx_cfa_code] = vf_rep->vf_idx;
+
+		vf_rep->dst = metadata_dst_alloc(0, METADATA_HW_PORT_MUX,
+						 GFP_KERNEL);
+		if (!vf_rep->dst) {
+			rc = -ENOMEM;
+			goto err;
+		}
+		/* only cfa_action is needed to mux a packet while TXing */
+		vf_rep->dst->u.port_info.port_id = vf_rep->tx_cfa_action;
+		vf_rep->dst->u.port_info.lower_dev = bp->dev;
+
+		bnxt_vf_rep_netdev_init(bp, vf_rep, dev);
+		rc = register_netdev(dev);
+		if (rc) {
+			/* no need for unregister_netdev in cleanup */
+			dev->netdev_ops = NULL;
+			goto err;
+		}
+	}
+
+	/* publish cfa_code_map only after all VF-reps have been initialized */
+	bp->cfa_code_map = cfa_code_map;
+	bp->eswitch_mode = DEVLINK_ESWITCH_MODE_SWITCHDEV;
+	netif_keep_dst(bp->dev);
+	return 0;
+
+err:
+	netdev_info(bp->dev, "%s error=%d", __func__, rc);
+	kfree(cfa_code_map);
+	__bnxt_vf_reps_destroy(bp);
+	return rc;
+}
+
+/* Devlink related routines */
+static int bnxt_dl_eswitch_mode_get(struct devlink *devlink, u16 *mode)
+{
+	struct bnxt *bp = bnxt_get_bp_from_dl(devlink);
+
+	*mode = bp->eswitch_mode;
+	return 0;
+}
+
+static int bnxt_dl_eswitch_mode_set(struct devlink *devlink, u16 mode)
+{
+	struct bnxt *bp = bnxt_get_bp_from_dl(devlink);
+	int rc = 0;
+
+	mutex_lock(&bp->sriov_lock);
+	if (bp->eswitch_mode == mode) {
+		netdev_info(bp->dev, "already in %s eswitch mode",
+			    mode == DEVLINK_ESWITCH_MODE_LEGACY ?
+			    "legacy" : "switchdev");
+		rc = -EINVAL;
+		goto done;
+	}
+
+	switch (mode) {
+	case DEVLINK_ESWITCH_MODE_LEGACY:
+		bnxt_vf_reps_destroy(bp);
+		break;
+
+	case DEVLINK_ESWITCH_MODE_SWITCHDEV:
+		if (pci_num_vf(bp->pdev) == 0) {
+			netdev_info(bp->dev,
+				    "Enable VFs before setting switchdev mode");
+			rc = -EPERM;
+			goto done;
+		}
+		rc = bnxt_vf_reps_create(bp);
+		break;
+
+	default:
+		rc = -EINVAL;
+		goto done;
+	}
+done:
+	mutex_unlock(&bp->sriov_lock);
+	return rc;
+}
+
+static const struct devlink_ops bnxt_dl_ops = {
+	.eswitch_mode_set = bnxt_dl_eswitch_mode_set,
+	.eswitch_mode_get = bnxt_dl_eswitch_mode_get
+};
+
+int bnxt_dl_register(struct bnxt *bp)
+{
+	struct devlink *dl;
+	int rc;
+
+	if (!pci_find_ext_capability(bp->pdev, PCI_EXT_CAP_ID_SRIOV))
+		return 0;
+
+	if (bp->hwrm_spec_code < 0x10800) {
+		netdev_warn(bp->dev, "Firmware does not support SR-IOV E-Switch SWITCHDEV mode.\n");
+		return -ENOTSUPP;
+	}
+
+	dl = devlink_alloc(&bnxt_dl_ops, sizeof(struct bnxt_dl));
+	if (!dl) {
+		netdev_warn(bp->dev, "devlink_alloc failed");
+		return -ENOMEM;
+	}
+
+	bnxt_link_bp_to_dl(bp, dl);
+	bp->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY;
+	rc = devlink_register(dl, &bp->pdev->dev);
+	if (rc) {
+		bnxt_link_bp_to_dl(bp, NULL);
+		devlink_free(dl);
+		netdev_warn(bp->dev, "devlink_register failed. rc=%d", rc);
+		return rc;
+	}
+
+	return 0;
+}
+
+void bnxt_dl_unregister(struct bnxt *bp)
+{
+	struct devlink *dl = bp->dl;
+
+	if (!dl)
+		return;
+
+	devlink_unregister(dl);
+	devlink_free(dl);
+}
+
+#endif
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.h
new file mode 100644
index 000000000000..7787cd24606a
--- /dev/null
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.h
@@ -0,0 +1,89 @@
+/* Broadcom NetXtreme-C/E network driver.
+ *
+ * Copyright (c) 2016-2017 Broadcom Limited
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.
+ */
+
+#ifndef BNXT_VFR_H
+#define BNXT_VFR_H
+
+#ifdef CONFIG_BNXT_SRIOV
+
+#define	MAX_CFA_CODE			65536
+
+/* Struct to hold housekeeping info needed by devlink interface */
+struct bnxt_dl {
+	struct bnxt *bp;	/* back ptr to the controlling dev */
+};
+
+static inline struct bnxt *bnxt_get_bp_from_dl(struct devlink *dl)
+{
+	return ((struct bnxt_dl *)devlink_priv(dl))->bp;
+}
+
+/* To clear devlink pointer from bp, pass NULL dl */
+static inline void bnxt_link_bp_to_dl(struct bnxt *bp, struct devlink *dl)
+{
+	bp->dl = dl;
+
+	/* add a back pointer in dl to bp */
+	if (dl) {
+		struct bnxt_dl *bp_dl = devlink_priv(dl);
+
+		bp_dl->bp = bp;
+	}
+}
+
+int bnxt_dl_register(struct bnxt *bp);
+void bnxt_dl_unregister(struct bnxt *bp);
+void bnxt_vf_reps_destroy(struct bnxt *bp);
+void bnxt_vf_reps_close(struct bnxt *bp);
+void bnxt_vf_reps_open(struct bnxt *bp);
+void bnxt_vf_rep_rx(struct bnxt *bp, struct sk_buff *skb);
+struct net_device *bnxt_get_vf_rep(struct bnxt *bp, u16 cfa_code);
+
+static inline u16 bnxt_vf_rep_get_fid(struct net_device *dev)
+{
+	struct bnxt_vf_rep *vf_rep = netdev_priv(dev);
+	struct bnxt *bp = vf_rep->bp;
+
+	return bp->pf.vf[vf_rep->vf_idx].fw_fid;
+}
+
+#else
+
+static inline int bnxt_dl_register(struct bnxt *bp)
+{
+	return 0;
+}
+
+static inline void bnxt_dl_unregister(struct bnxt *bp)
+{
+}
+
+static inline void bnxt_vf_reps_close(struct bnxt *bp)
+{
+}
+
+static inline void bnxt_vf_reps_open(struct bnxt *bp)
+{
+}
+
+static inline void bnxt_vf_rep_rx(struct bnxt *bp, struct sk_buff *skb)
+{
+}
+
+static inline struct net_device *bnxt_get_vf_rep(struct bnxt *bp, u16 cfa_code)
+{
+	return NULL;
+}
+
+static inline u16 bnxt_vf_rep_get_fid(struct net_device *dev)
+{
+	return 0;
+}
+#endif /* CONFIG_BNXT_SRIOV */
+#endif /* BNXT_VFR_H */
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c
index 3961a6807454..d8f0c837b72c 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c
@@ -169,8 +169,8 @@ static int bnxt_xdp_set(struct bnxt *bp, struct bpf_prog *prog)
 	tc = netdev_get_num_tc(dev);
 	if (!tc)
 		tc = 1;
-	rc = bnxt_reserve_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
-				true, tc, tx_xdp);
+	rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
+			      true, tc, tx_xdp);
 	if (rc) {
 		netdev_warn(dev, "Unable to reserve enough TX rings to support XDP.\n");
 		return rc;
diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c
index a981c4ee9d72..9cebca896913 100644
--- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c
+++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c
@@ -72,23 +72,42 @@
 #define GENET_RDMA_REG_OFF	(priv->hw_params->rdma_offset + \
 				TOTAL_DESC * DMA_DESC_SIZE)
 
+static inline void bcmgenet_writel(u32 value, void __iomem *offset)
+{
+	/* MIPS chips strapped for BE will automagically configure the
+	 * peripheral registers for CPU-native byte order.
+	 */
+	if (IS_ENABLED(CONFIG_MIPS) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
+		__raw_writel(value, offset);
+	else
+		writel_relaxed(value, offset);
+}
+
+static inline u32 bcmgenet_readl(void __iomem *offset)
+{
+	if (IS_ENABLED(CONFIG_MIPS) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
+		return __raw_readl(offset);
+	else
+		return readl_relaxed(offset);
+}
+
 static inline void dmadesc_set_length_status(struct bcmgenet_priv *priv,
 					     void __iomem *d, u32 value)
 {
-	__raw_writel(value, d + DMA_DESC_LENGTH_STATUS);
+	bcmgenet_writel(value, d + DMA_DESC_LENGTH_STATUS);
 }
 
 static inline u32 dmadesc_get_length_status(struct bcmgenet_priv *priv,
 					    void __iomem *d)
 {
-	return __raw_readl(d + DMA_DESC_LENGTH_STATUS);
+	return bcmgenet_readl(d + DMA_DESC_LENGTH_STATUS);
 }
 
 static inline void dmadesc_set_addr(struct bcmgenet_priv *priv,
 				    void __iomem *d,
 				    dma_addr_t addr)
 {
-	__raw_writel(lower_32_bits(addr), d + DMA_DESC_ADDRESS_LO);
+	bcmgenet_writel(lower_32_bits(addr), d + DMA_DESC_ADDRESS_LO);
 
 	/* Register writes to GISB bus can take couple hundred nanoseconds
 	 * and are done for each packet, save these expensive writes unless
@@ -96,7 +115,7 @@ static inline void dmadesc_set_addr(struct bcmgenet_priv *priv,
 	 */
 #ifdef CONFIG_PHYS_ADDR_T_64BIT
 	if (priv->hw_params->flags & GENET_HAS_40BITS)
-		__raw_writel(upper_32_bits(addr), d + DMA_DESC_ADDRESS_HI);
+		bcmgenet_writel(upper_32_bits(addr), d + DMA_DESC_ADDRESS_HI);
 #endif
 }
 
@@ -113,7 +132,7 @@ static inline dma_addr_t dmadesc_get_addr(struct bcmgenet_priv *priv,
 {
 	dma_addr_t addr;
 
-	addr = __raw_readl(d + DMA_DESC_ADDRESS_LO);
+	addr = bcmgenet_readl(d + DMA_DESC_ADDRESS_LO);
 
 	/* Register writes to GISB bus can take couple hundred nanoseconds
 	 * and are done for each packet, save these expensive writes unless
@@ -121,7 +140,7 @@ static inline dma_addr_t dmadesc_get_addr(struct bcmgenet_priv *priv,
 	 */
 #ifdef CONFIG_PHYS_ADDR_T_64BIT
 	if (priv->hw_params->flags & GENET_HAS_40BITS)
-		addr |= (u64)__raw_readl(d + DMA_DESC_ADDRESS_HI) << 32;
+		addr |= (u64)bcmgenet_readl(d + DMA_DESC_ADDRESS_HI) << 32;
 #endif
 	return addr;
 }
@@ -156,8 +175,8 @@ static inline u32 bcmgenet_tbuf_ctrl_get(struct bcmgenet_priv *priv)
 	if (GENET_IS_V1(priv))
 		return bcmgenet_rbuf_readl(priv, TBUF_CTRL_V1);
 	else
-		return __raw_readl(priv->base +
-				priv->hw_params->tbuf_offset + TBUF_CTRL);
+		return bcmgenet_readl(priv->base +
+				      priv->hw_params->tbuf_offset + TBUF_CTRL);
 }
 
 static inline void bcmgenet_tbuf_ctrl_set(struct bcmgenet_priv *priv, u32 val)
@@ -165,7 +184,7 @@ static inline void bcmgenet_tbuf_ctrl_set(struct bcmgenet_priv *priv, u32 val)
 	if (GENET_IS_V1(priv))
 		bcmgenet_rbuf_writel(priv, val, TBUF_CTRL_V1);
 	else
-		__raw_writel(val, priv->base +
+		bcmgenet_writel(val, priv->base +
 				priv->hw_params->tbuf_offset + TBUF_CTRL);
 }
 
@@ -174,8 +193,8 @@ static inline u32 bcmgenet_bp_mc_get(struct bcmgenet_priv *priv)
 	if (GENET_IS_V1(priv))
 		return bcmgenet_rbuf_readl(priv, TBUF_BP_MC_V1);
 	else
-		return __raw_readl(priv->base +
-				priv->hw_params->tbuf_offset + TBUF_BP_MC);
+		return bcmgenet_readl(priv->base +
+				      priv->hw_params->tbuf_offset + TBUF_BP_MC);
 }
 
 static inline void bcmgenet_bp_mc_set(struct bcmgenet_priv *priv, u32 val)
@@ -183,7 +202,7 @@ static inline void bcmgenet_bp_mc_set(struct bcmgenet_priv *priv, u32 val)
 	if (GENET_IS_V1(priv))
 		bcmgenet_rbuf_writel(priv, val, TBUF_BP_MC_V1);
 	else
-		__raw_writel(val, priv->base +
+		bcmgenet_writel(val, priv->base +
 				priv->hw_params->tbuf_offset + TBUF_BP_MC);
 }
 
@@ -326,28 +345,28 @@ static inline struct bcmgenet_priv *dev_to_priv(struct device *dev)
 static inline u32 bcmgenet_tdma_readl(struct bcmgenet_priv *priv,
 				      enum dma_reg r)
 {
-	return __raw_readl(priv->base + GENET_TDMA_REG_OFF +
-			DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
+	return bcmgenet_readl(priv->base + GENET_TDMA_REG_OFF +
+			      DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
 }
 
 static inline void bcmgenet_tdma_writel(struct bcmgenet_priv *priv,
 					u32 val, enum dma_reg r)
 {
-	__raw_writel(val, priv->base + GENET_TDMA_REG_OFF +
+	bcmgenet_writel(val, priv->base + GENET_TDMA_REG_OFF +
 			DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
 }
 
 static inline u32 bcmgenet_rdma_readl(struct bcmgenet_priv *priv,
 				      enum dma_reg r)
 {
-	return __raw_readl(priv->base + GENET_RDMA_REG_OFF +
-			DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
+	return bcmgenet_readl(priv->base + GENET_RDMA_REG_OFF +
+			      DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
 }
 
 static inline void bcmgenet_rdma_writel(struct bcmgenet_priv *priv,
 					u32 val, enum dma_reg r)
 {
-	__raw_writel(val, priv->base + GENET_RDMA_REG_OFF +
+	bcmgenet_writel(val, priv->base + GENET_RDMA_REG_OFF +
 			DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
 }
 
@@ -418,16 +437,16 @@ static inline u32 bcmgenet_tdma_ring_readl(struct bcmgenet_priv *priv,
 					   unsigned int ring,
 					   enum dma_ring_reg r)
 {
-	return __raw_readl(priv->base + GENET_TDMA_REG_OFF +
-			(DMA_RING_SIZE * ring) +
-			genet_dma_ring_regs[r]);
+	return bcmgenet_readl(priv->base + GENET_TDMA_REG_OFF +
+			      (DMA_RING_SIZE * ring) +
+			      genet_dma_ring_regs[r]);
 }
 
 static inline void bcmgenet_tdma_ring_writel(struct bcmgenet_priv *priv,
 					     unsigned int ring, u32 val,
 					     enum dma_ring_reg r)
 {
-	__raw_writel(val, priv->base + GENET_TDMA_REG_OFF +
+	bcmgenet_writel(val, priv->base + GENET_TDMA_REG_OFF +
 			(DMA_RING_SIZE * ring) +
 			genet_dma_ring_regs[r]);
 }
@@ -436,16 +455,16 @@ static inline u32 bcmgenet_rdma_ring_readl(struct bcmgenet_priv *priv,
 					   unsigned int ring,
 					   enum dma_ring_reg r)
 {
-	return __raw_readl(priv->base + GENET_RDMA_REG_OFF +
-			(DMA_RING_SIZE * ring) +
-			genet_dma_ring_regs[r]);
+	return bcmgenet_readl(priv->base + GENET_RDMA_REG_OFF +
+			      (DMA_RING_SIZE * ring) +
+			      genet_dma_ring_regs[r]);
 }
 
 static inline void bcmgenet_rdma_ring_writel(struct bcmgenet_priv *priv,
 					     unsigned int ring, u32 val,
 					     enum dma_ring_reg r)
 {
-	__raw_writel(val, priv->base + GENET_RDMA_REG_OFF +
+	bcmgenet_writel(val, priv->base + GENET_RDMA_REG_OFF +
 			(DMA_RING_SIZE * ring) +
 			genet_dma_ring_regs[r]);
 }
@@ -991,12 +1010,12 @@ static void bcmgenet_eee_enable_set(struct net_device *dev, bool enable)
 	bcmgenet_umac_writel(priv, reg, UMAC_EEE_CTRL);
 
 	/* Enable EEE and switch to a 27Mhz clock automatically */
-	reg = __raw_readl(priv->base + off);
+	reg = bcmgenet_readl(priv->base + off);
 	if (enable)
 		reg |= TBUF_EEE_EN | TBUF_PM_EN;
 	else
 		reg &= ~(TBUF_EEE_EN | TBUF_PM_EN);
-	__raw_writel(reg, priv->base + off);
+	bcmgenet_writel(reg, priv->base + off);
 
 	/* Do the same for thing for RBUF */
 	reg = bcmgenet_rbuf_readl(priv, RBUF_ENERGY_CTRL);
@@ -1360,7 +1379,7 @@ static unsigned int __bcmgenet_tx_reclaim(struct net_device *dev,
 		if (skb) {
 			pkts_compl++;
 			bytes_compl += GENET_CB(skb)->bytes_sent;
-			dev_kfree_skb_any(skb);
+			dev_consume_skb_any(skb);
 		}
 
 		txbds_processed++;
@@ -1875,7 +1894,7 @@ static int bcmgenet_alloc_rx_buffers(struct bcmgenet_priv *priv,
 		cb = ring->cbs + i;
 		skb = bcmgenet_rx_refill(priv, cb);
 		if (skb)
-			dev_kfree_skb_any(skb);
+			dev_consume_skb_any(skb);
 		if (!cb->skb)
 			return -ENOMEM;
 	}
@@ -1894,7 +1913,7 @@ static void bcmgenet_free_rx_buffers(struct bcmgenet_priv *priv)
 
 		skb = bcmgenet_free_rx_cb(&priv->pdev->dev, cb);
 		if (skb)
-			dev_kfree_skb_any(skb);
+			dev_consume_skb_any(skb);
 	}
 }
 
diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.h b/drivers/net/ethernet/broadcom/genet/bcmgenet.h
index 3a34fdba5301..4c49d0b97748 100644
--- a/drivers/net/ethernet/broadcom/genet/bcmgenet.h
+++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.h
@@ -657,6 +657,7 @@ struct bcmgenet_priv {
 
 	struct clk *clk;
 	struct platform_device *pdev;
+	struct platform_device *mii_pdev;
 
 	/* WOL */
 	struct clk *clk_wol;
@@ -671,12 +672,21 @@ struct bcmgenet_priv {
 static inline u32 bcmgenet_##name##_readl(struct bcmgenet_priv *priv,	\
 					u32 off)			\
 {									\
-	return __raw_readl(priv->base + offset + off);			\
+	/* MIPS chips strapped for BE will automagically configure the	\
+	 * peripheral registers for CPU-native byte order.		\
+	 */								\
+	if (IS_ENABLED(CONFIG_MIPS) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) \
+		return __raw_readl(priv->base + offset + off);		\
+	else								\
+		return readl_relaxed(priv->base + offset + off);	\
 }									\
 static inline void bcmgenet_##name##_writel(struct bcmgenet_priv *priv,	\
 					u32 val, u32 off)		\
 {									\
-	__raw_writel(val, priv->base + offset + off);			\
+	if (IS_ENABLED(CONFIG_MIPS) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) \
+		__raw_writel(val, priv->base + offset + off);		\
+	else								\
+		writel_relaxed(val, priv->base + offset + off);		\
 }
 
 GENET_IO_MACRO(ext, GENET_EXT_OFF);
diff --git a/drivers/net/ethernet/broadcom/genet/bcmmii.c b/drivers/net/ethernet/broadcom/genet/bcmmii.c
index 30cb97b4a1d7..18f5723be2c9 100644
--- a/drivers/net/ethernet/broadcom/genet/bcmmii.c
+++ b/drivers/net/ethernet/broadcom/genet/bcmmii.c
@@ -24,62 +24,10 @@
 #include <linux/of_net.h>
 #include <linux/of_mdio.h>
 #include <linux/platform_data/bcmgenet.h>
+#include <linux/platform_data/mdio-bcm-unimac.h>
 
 #include "bcmgenet.h"
 
-/* read a value from the MII */
-static int bcmgenet_mii_read(struct mii_bus *bus, int phy_id, int location)
-{
-	int ret;
-	struct net_device *dev = bus->priv;
-	struct bcmgenet_priv *priv = netdev_priv(dev);
-	u32 reg;
-
-	bcmgenet_umac_writel(priv, (MDIO_RD | (phy_id << MDIO_PMD_SHIFT) |
-			     (location << MDIO_REG_SHIFT)), UMAC_MDIO_CMD);
-	/* Start MDIO transaction*/
-	reg = bcmgenet_umac_readl(priv, UMAC_MDIO_CMD);
-	reg |= MDIO_START_BUSY;
-	bcmgenet_umac_writel(priv, reg, UMAC_MDIO_CMD);
-	wait_event_timeout(priv->wq,
-			   !(bcmgenet_umac_readl(priv, UMAC_MDIO_CMD)
-			   & MDIO_START_BUSY),
-			   HZ / 100);
-	ret = bcmgenet_umac_readl(priv, UMAC_MDIO_CMD);
-
-	/* Some broken devices are known not to release the line during
-	 * turn-around, e.g: Broadcom BCM53125 external switches, so check for
-	 * that condition here and ignore the MDIO controller read failure
-	 * indication.
-	 */
-	if (!(bus->phy_ignore_ta_mask & 1 << phy_id) && (ret & MDIO_READ_FAIL))
-		return -EIO;
-
-	return ret & 0xffff;
-}
-
-/* write a value to the MII */
-static int bcmgenet_mii_write(struct mii_bus *bus, int phy_id,
-			      int location, u16 val)
-{
-	struct net_device *dev = bus->priv;
-	struct bcmgenet_priv *priv = netdev_priv(dev);
-	u32 reg;
-
-	bcmgenet_umac_writel(priv, (MDIO_WR | (phy_id << MDIO_PMD_SHIFT) |
-			     (location << MDIO_REG_SHIFT) | (0xffff & val)),
-			     UMAC_MDIO_CMD);
-	reg = bcmgenet_umac_readl(priv, UMAC_MDIO_CMD);
-	reg |= MDIO_START_BUSY;
-	bcmgenet_umac_writel(priv, reg, UMAC_MDIO_CMD);
-	wait_event_timeout(priv->wq,
-			   !(bcmgenet_umac_readl(priv, UMAC_MDIO_CMD) &
-			   MDIO_START_BUSY),
-			   HZ / 100);
-
-	return 0;
-}
-
 /* setup netdev link state when PHY link status change and
  * update UMAC and RGMII block when link up
  */
@@ -393,104 +341,121 @@ int bcmgenet_mii_probe(struct net_device *dev)
 	return 0;
 }
 
-/* Workaround for integrated BCM7xxx Gigabit PHYs which have a problem with
- * their internal MDIO management controller making them fail to successfully
- * be read from or written to for the first transaction.  We insert a dummy
- * BMSR read here to make sure that phy_get_device() and get_phy_id() can
- * correctly read the PHY MII_PHYSID1/2 registers and successfully register a
- * PHY device for this peripheral.
- *
- * Once the PHY driver is registered, we can workaround subsequent reads from
- * there (e.g: during system-wide power management).
- *
- * bus->reset is invoked before mdiobus_scan during mdiobus_register and is
- * therefore the right location to stick that workaround. Since we do not want
- * to read from non-existing PHYs, we either use bus->phy_mask or do a manual
- * Device Tree scan to limit the search area.
- */
-static int bcmgenet_mii_bus_reset(struct mii_bus *bus)
+static struct device_node *bcmgenet_mii_of_find_mdio(struct bcmgenet_priv *priv)
 {
-	struct net_device *dev = bus->priv;
-	struct bcmgenet_priv *priv = netdev_priv(dev);
-	struct device_node *np = priv->mdio_dn;
-	struct device_node *child = NULL;
-	u32 read_mask = 0;
-	int addr = 0;
+	struct device_node *dn = priv->pdev->dev.of_node;
+	struct device *kdev = &priv->pdev->dev;
+	char *compat;
 
-	if (!np) {
-		read_mask = 1 << priv->phy_addr;
-	} else {
-		for_each_available_child_of_node(np, child) {
-			addr = of_mdio_parse_addr(&dev->dev, child);
-			if (addr < 0)
-				continue;
+	compat = kasprintf(GFP_KERNEL, "brcm,genet-mdio-v%d", priv->version);
+	if (!compat)
+		return NULL;
 
-			read_mask |= 1 << addr;
-		}
+	priv->mdio_dn = of_find_compatible_node(dn, NULL, compat);
+	kfree(compat);
+	if (!priv->mdio_dn) {
+		dev_err(kdev, "unable to find MDIO bus node\n");
+		return NULL;
 	}
 
-	for (addr = 0; addr < PHY_MAX_ADDR; addr++) {
-		if (read_mask & 1 << addr) {
-			dev_dbg(&dev->dev, "Workaround for PHY @ %d\n", addr);
-			mdiobus_read(bus, addr, MII_BMSR);
-		}
+	return priv->mdio_dn;
+}
+
+static void bcmgenet_mii_pdata_init(struct bcmgenet_priv *priv,
+				    struct unimac_mdio_pdata *ppd)
+{
+	struct device *kdev = &priv->pdev->dev;
+	struct bcmgenet_platform_data *pd = kdev->platform_data;
+
+	if (pd->phy_interface != PHY_INTERFACE_MODE_MOCA && pd->mdio_enabled) {
+		/*
+		 * Internal or external PHY with MDIO access
+		 */
+		if (pd->phy_address >= 0 && pd->phy_address < PHY_MAX_ADDR)
+			ppd->phy_mask = 1 << pd->phy_address;
+		else
+			ppd->phy_mask = 0;
 	}
+}
 
+static int bcmgenet_mii_wait(void *wait_func_data)
+{
+	struct bcmgenet_priv *priv = wait_func_data;
+
+	wait_event_timeout(priv->wq,
+			   !(bcmgenet_umac_readl(priv, UMAC_MDIO_CMD)
+			   & MDIO_START_BUSY),
+			   HZ / 100);
 	return 0;
 }
 
-static int bcmgenet_mii_alloc(struct bcmgenet_priv *priv)
+static int bcmgenet_mii_register(struct bcmgenet_priv *priv)
 {
-	struct mii_bus *bus;
+	struct platform_device *pdev = priv->pdev;
+	struct bcmgenet_platform_data *pdata = pdev->dev.platform_data;
+	struct device_node *dn = pdev->dev.of_node;
+	struct unimac_mdio_pdata ppd;
+	struct platform_device *ppdev;
+	struct resource *pres, res;
+	int id, ret;
+
+	pres = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	memset(&res, 0, sizeof(res));
+	memset(&ppd, 0, sizeof(ppd));
+
+	ppd.wait_func = bcmgenet_mii_wait;
+	ppd.wait_func_data = priv;
+	ppd.bus_name = "bcmgenet MII bus";
+
+	/* Unimac MDIO bus controller starts at UniMAC offset + MDIO_CMD
+	 * and is 2 * 32-bits word long, 8 bytes total.
+	 */
+	res.start = pres->start + GENET_UMAC_OFF + UMAC_MDIO_CMD;
+	res.end = res.start + 8;
+	res.flags = IORESOURCE_MEM;
 
-	if (priv->mii_bus)
-		return 0;
+	if (dn)
+		id = of_alias_get_id(dn, "eth");
+	else
+		id = pdev->id;
 
-	priv->mii_bus = mdiobus_alloc();
-	if (!priv->mii_bus) {
-		pr_err("failed to allocate\n");
+	ppdev = platform_device_alloc(UNIMAC_MDIO_DRV_NAME, id);
+	if (!ppdev)
 		return -ENOMEM;
-	}
 
-	bus = priv->mii_bus;
-	bus->priv = priv->dev;
-	bus->name = "bcmgenet MII bus";
-	bus->parent = &priv->pdev->dev;
-	bus->read = bcmgenet_mii_read;
-	bus->write = bcmgenet_mii_write;
-	bus->reset = bcmgenet_mii_bus_reset;
-	snprintf(bus->id, MII_BUS_ID_SIZE, "%s-%d",
-		 priv->pdev->name, priv->pdev->id);
+	/* Retain this platform_device pointer for later cleanup */
+	priv->mii_pdev = ppdev;
+	ppdev->dev.parent = &pdev->dev;
+	ppdev->dev.of_node = bcmgenet_mii_of_find_mdio(priv);
+	if (pdata)
+		bcmgenet_mii_pdata_init(priv, &ppd);
+
+	ret = platform_device_add_resources(ppdev, &res, 1);
+	if (ret)
+		goto out;
+
+	ret = platform_device_add_data(ppdev, &ppd, sizeof(ppd));
+	if (ret)
+		goto out;
+
+	ret = platform_device_add(ppdev);
+	if (ret)
+		goto out;
 
 	return 0;
+out:
+	platform_device_put(ppdev);
+	return ret;
 }
 
 static int bcmgenet_mii_of_init(struct bcmgenet_priv *priv)
 {
 	struct device_node *dn = priv->pdev->dev.of_node;
 	struct device *kdev = &priv->pdev->dev;
-	struct phy_device *phydev = NULL;
-	char *compat;
+	struct phy_device *phydev;
 	int phy_mode;
 	int ret;
 
-	compat = kasprintf(GFP_KERNEL, "brcm,genet-mdio-v%d", priv->version);
-	if (!compat)
-		return -ENOMEM;
-
-	priv->mdio_dn = of_find_compatible_node(dn, NULL, compat);
-	kfree(compat);
-	if (!priv->mdio_dn) {
-		dev_err(kdev, "unable to find MDIO bus node\n");
-		return -ENODEV;
-	}
-
-	ret = of_mdiobus_register(priv->mii_bus, priv->mdio_dn);
-	if (ret) {
-		dev_err(kdev, "failed to register MDIO bus\n");
-		return ret;
-	}
-
 	/* Fetch the PHY phandle */
 	priv->phy_dn = of_parse_phandle(dn, "phy-handle", 0);
 
@@ -537,33 +502,23 @@ static int bcmgenet_mii_pd_init(struct bcmgenet_priv *priv)
 {
 	struct device *kdev = &priv->pdev->dev;
 	struct bcmgenet_platform_data *pd = kdev->platform_data;
-	struct mii_bus *mdio = priv->mii_bus;
+	char phy_name[MII_BUS_ID_SIZE + 3];
+	char mdio_bus_id[MII_BUS_ID_SIZE];
 	struct phy_device *phydev;
-	int ret;
+
+	snprintf(mdio_bus_id, MII_BUS_ID_SIZE, "%s-%d",
+		 UNIMAC_MDIO_DRV_NAME, priv->pdev->id);
 
 	if (pd->phy_interface != PHY_INTERFACE_MODE_MOCA && pd->mdio_enabled) {
+		snprintf(phy_name, MII_BUS_ID_SIZE, PHY_ID_FMT,
+			 mdio_bus_id, pd->phy_address);
+
 		/*
 		 * Internal or external PHY with MDIO access
 		 */
-		if (pd->phy_address >= 0 && pd->phy_address < PHY_MAX_ADDR)
-			mdio->phy_mask = ~(1 << pd->phy_address);
-		else
-			mdio->phy_mask = 0;
-
-		ret = mdiobus_register(mdio);
-		if (ret) {
-			dev_err(kdev, "failed to register MDIO bus\n");
-			return ret;
-		}
-
-		if (pd->phy_address >= 0 && pd->phy_address < PHY_MAX_ADDR)
-			phydev = mdiobus_get_phy(mdio, pd->phy_address);
-		else
-			phydev = phy_find_first(mdio);
-
+		phydev = phy_attach(priv->dev, phy_name, pd->phy_interface);
 		if (!phydev) {
 			dev_err(kdev, "failed to register PHY device\n");
-			mdiobus_unregister(mdio);
 			return -ENODEV;
 		}
 	} else {
@@ -609,10 +564,9 @@ static int bcmgenet_mii_bus_init(struct bcmgenet_priv *priv)
 int bcmgenet_mii_init(struct net_device *dev)
 {
 	struct bcmgenet_priv *priv = netdev_priv(dev);
-	struct device_node *dn = priv->pdev->dev.of_node;
 	int ret;
 
-	ret = bcmgenet_mii_alloc(priv);
+	ret = bcmgenet_mii_register(priv);
 	if (ret)
 		return ret;
 
@@ -623,11 +577,7 @@ int bcmgenet_mii_init(struct net_device *dev)
 	return 0;
 
 out:
-	if (of_phy_is_fixed_link(dn))
-		of_phy_deregister_fixed_link(dn);
-	of_node_put(priv->phy_dn);
-	mdiobus_unregister(priv->mii_bus);
-	mdiobus_free(priv->mii_bus);
+	bcmgenet_mii_exit(dev);
 	return ret;
 }
 
@@ -639,6 +589,6 @@ void bcmgenet_mii_exit(struct net_device *dev)
 	if (of_phy_is_fixed_link(dn))
 		of_phy_deregister_fixed_link(dn);
 	of_node_put(priv->phy_dn);
-	mdiobus_unregister(priv->mii_bus);
-	mdiobus_free(priv->mii_bus);
+	platform_device_unregister(priv->mii_pdev);
+	platform_device_put(priv->mii_pdev);
 }
diff --git a/drivers/net/ethernet/broadcom/sb1250-mac.c b/drivers/net/ethernet/broadcom/sb1250-mac.c
index 16a0f192daec..ecdef42f0ae6 100644
--- a/drivers/net/ethernet/broadcom/sb1250-mac.c
+++ b/drivers/net/ethernet/broadcom/sb1250-mac.c
@@ -1367,15 +1367,11 @@ static int sbmac_initctx(struct sbmac_softc *s)
 
 static void sbdma_uninitctx(struct sbmacdma *d)
 {
-	if (d->sbdma_dscrtable_unaligned) {
-		kfree(d->sbdma_dscrtable_unaligned);
-		d->sbdma_dscrtable_unaligned = d->sbdma_dscrtable = NULL;
-	}
+	kfree(d->sbdma_dscrtable_unaligned);
+	d->sbdma_dscrtable_unaligned = d->sbdma_dscrtable = NULL;
 
-	if (d->sbdma_ctxtable) {
-		kfree(d->sbdma_ctxtable);
-		d->sbdma_ctxtable = NULL;
-	}
+	kfree(d->sbdma_ctxtable);
+	d->sbdma_ctxtable = NULL;
 }
 
 
diff --git a/drivers/net/ethernet/broadcom/tg3.c b/drivers/net/ethernet/broadcom/tg3.c
index d600c41fb1dc..656e6af70f0a 100644
--- a/drivers/net/ethernet/broadcom/tg3.c
+++ b/drivers/net/ethernet/broadcom/tg3.c
@@ -6587,7 +6587,7 @@ static void tg3_tx(struct tg3_napi *tnapi)
 		pkts_compl++;
 		bytes_compl += skb->len;
 
-		dev_kfree_skb_any(skb);
+		dev_consume_skb_any(skb);
 
 		if (unlikely(tx_bug)) {
 			tg3_tx_recover(tp);
@@ -7829,7 +7829,7 @@ static int tigon3_dma_hwbug_workaround(struct tg3_napi *tnapi,
 		}
 	}
 
-	dev_kfree_skb_any(skb);
+	dev_consume_skb_any(skb);
 	*pskb = new_skb;
 	return ret;
 }
@@ -7882,7 +7882,7 @@ static int tg3_tso_bug(struct tg3 *tp, struct tg3_napi *tnapi,
 	} while (segs);
 
 tg3_tso_bug_end:
-	dev_kfree_skb_any(skb);
+	dev_consume_skb_any(skb);
 
 	return NETDEV_TX_OK;
 }
@@ -8543,7 +8543,7 @@ static void tg3_free_rings(struct tg3 *tp)
 			tg3_tx_skb_unmap(tnapi, i,
 					 skb_shinfo(skb)->nr_frags - 1);
 
-			dev_kfree_skb_any(skb);
+			dev_consume_skb_any(skb);
 		}
 		netdev_tx_reset_queue(netdev_get_tx_queue(tp->dev, j));
 	}
@@ -11536,11 +11536,11 @@ static int tg3_start(struct tg3 *tp, bool reset_phy, bool test_irq,
 	tg3_napi_enable(tp);
 
 	for (i = 0; i < tp->irq_cnt; i++) {
-		struct tg3_napi *tnapi = &tp->napi[i];
 		err = tg3_request_irq(tp, i);
 		if (err) {
 			for (i--; i >= 0; i--) {
-				tnapi = &tp->napi[i];
+				struct tg3_napi *tnapi = &tp->napi[i];
+
 				free_irq(tnapi->irq_vec, tnapi);
 			}
 			goto out_napi_fini;
diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c
index 26d25749c3e4..6df2cad61647 100644
--- a/drivers/net/ethernet/cadence/macb_main.c
+++ b/drivers/net/ethernet/cadence/macb_main.c
@@ -68,7 +68,7 @@
 #define GEM_MAX_TX_LEN		((unsigned int)((1 << GEM_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1)))
 
 #define GEM_MTU_MIN_SIZE	ETH_MIN_MTU
-#define MACB_NETIF_LSO		(NETIF_F_TSO | NETIF_F_UFO)
+#define MACB_NETIF_LSO		NETIF_F_TSO
 
 #define MACB_WOL_HAS_MAGIC_PACKET	(0x1 << 0)
 #define MACB_WOL_ENABLED		(0x1 << 1)
diff --git a/drivers/net/ethernet/cadence/macb_pci.c b/drivers/net/ethernet/cadence/macb_pci.c
index 9906fda76087..248a8fc45069 100644
--- a/drivers/net/ethernet/cadence/macb_pci.c
+++ b/drivers/net/ethernet/cadence/macb_pci.c
@@ -128,7 +128,7 @@ static void macb_remove(struct pci_dev *pdev)
 	clk_unregister(plat_data->hclk);
 }
 
-static struct pci_device_id dev_id_table[] = {
+static const struct pci_device_id dev_id_table[] = {
 	{ PCI_DEVICE(CDNS_VENDOR_ID, CDNS_DEVICE_ID), },
 	{ 0, }
 };
diff --git a/drivers/net/ethernet/cadence/macb_ptp.c b/drivers/net/ethernet/cadence/macb_ptp.c
index 67cca08472b7..2220c771092b 100755
--- a/drivers/net/ethernet/cadence/macb_ptp.c
+++ b/drivers/net/ethernet/cadence/macb_ptp.c
@@ -192,7 +192,7 @@ static int gem_ptp_enable(struct ptp_clock_info *ptp,
 	return -EOPNOTSUPP;
 }
 
-static struct ptp_clock_info gem_ptp_caps_template = {
+static const struct ptp_clock_info gem_ptp_caps_template = {
 	.owner		= THIS_MODULE,
 	.name		= GEM_PTP_TIMER_NAME,
 	.max_adj	= 0,
diff --git a/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.c b/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.c
index 4b0ca9fb2cb4..e8b290473ee2 100644
--- a/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.c
+++ b/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.c
@@ -1150,14 +1150,50 @@ static void cn23xx_get_pcie_qlmport(struct octeon_device *oct)
 		oct->pcie_port);
 }
 
-static void cn23xx_get_pf_num(struct octeon_device *oct)
+static int cn23xx_get_pf_num(struct octeon_device *oct)
 {
 	u32 fdl_bit = 0;
+	u64 pkt0_in_ctl, d64;
+	int pfnum, mac, trs, ret;
+
+	ret = 0;
 
 	/** Read Function Dependency Link reg to get the function number */
-	pci_read_config_dword(oct->pci_dev, CN23XX_PCIE_SRIOV_FDL, &fdl_bit);
-	oct->pf_num = ((fdl_bit >> CN23XX_PCIE_SRIOV_FDL_BIT_POS) &
-		       CN23XX_PCIE_SRIOV_FDL_MASK);
+	if (pci_read_config_dword(oct->pci_dev, CN23XX_PCIE_SRIOV_FDL,
+				  &fdl_bit) == 0) {
+		oct->pf_num = ((fdl_bit >> CN23XX_PCIE_SRIOV_FDL_BIT_POS) &
+			       CN23XX_PCIE_SRIOV_FDL_MASK);
+	} else {
+		ret = EINVAL;
+
+		/* Under some virtual environments, extended PCI regs are
+		 * inaccessible, in which case the above read will have failed.
+		 * In this case, read the PF number from the
+		 * SLI_PKT0_INPUT_CONTROL reg (written by f/w)
+		 */
+		pkt0_in_ctl = octeon_read_csr64(oct,
+						CN23XX_SLI_IQ_PKT_CONTROL64(0));
+		pfnum = (pkt0_in_ctl >> CN23XX_PKT_INPUT_CTL_PF_NUM_POS) &
+			CN23XX_PKT_INPUT_CTL_PF_NUM_MASK;
+		mac = (octeon_read_csr(oct, CN23XX_SLI_MAC_NUMBER)) & 0xff;
+
+		/* validate PF num by reading RINFO; f/w writes RINFO.trs == 1*/
+		d64 = octeon_read_csr64(oct,
+					CN23XX_SLI_PKT_MAC_RINFO64(mac, pfnum));
+		trs = (int)(d64 >> CN23XX_PKT_MAC_CTL_RINFO_TRS_BIT_POS) & 0xff;
+		if (trs == 1) {
+			dev_err(&oct->pci_dev->dev,
+				"OCTEON: error reading PCI cfg space pfnum, re-read %u\n",
+				pfnum);
+			oct->pf_num = pfnum;
+			ret = 0;
+		} else {
+			dev_err(&oct->pci_dev->dev,
+				"OCTEON: error reading PCI cfg space pfnum; could not ascertain PF number\n");
+		}
+	}
+
+	return ret;
 }
 
 static void cn23xx_setup_reg_address(struct octeon_device *oct)
@@ -1269,6 +1305,26 @@ static int cn23xx_sriov_config(struct octeon_device *oct)
 
 int setup_cn23xx_octeon_pf_device(struct octeon_device *oct)
 {
+	u32 data32;
+	u64 BAR0, BAR1;
+
+	pci_read_config_dword(oct->pci_dev, PCI_BASE_ADDRESS_0, &data32);
+	BAR0 = (u64)(data32 & ~0xf);
+	pci_read_config_dword(oct->pci_dev, PCI_BASE_ADDRESS_1, &data32);
+	BAR0 |= ((u64)data32 << 32);
+	pci_read_config_dword(oct->pci_dev, PCI_BASE_ADDRESS_2, &data32);
+	BAR1 = (u64)(data32 & ~0xf);
+	pci_read_config_dword(oct->pci_dev, PCI_BASE_ADDRESS_3, &data32);
+	BAR1 |= ((u64)data32 << 32);
+
+	if (!BAR0 || !BAR1) {
+		if (!BAR0)
+			dev_err(&oct->pci_dev->dev, "device BAR0 unassigned\n");
+		if (!BAR1)
+			dev_err(&oct->pci_dev->dev, "device BAR1 unassigned\n");
+		return 1;
+	}
+
 	if (octeon_map_pci_barx(oct, 0, 0))
 		return 1;
 
@@ -1279,7 +1335,8 @@ int setup_cn23xx_octeon_pf_device(struct octeon_device *oct)
 		return 1;
 	}
 
-	cn23xx_get_pf_num(oct);
+	if (cn23xx_get_pf_num(oct) != 0)
+		return 1;
 
 	if (cn23xx_sriov_config(oct)) {
 		octeon_unmap_pci_barx(oct, 0);
@@ -1405,8 +1462,19 @@ int cn23xx_fw_loaded(struct octeon_device *oct)
 {
 	u64 val;
 
-	val = octeon_read_csr64(oct, CN23XX_SLI_SCRATCH1);
-	return (val >> 1) & 1ULL;
+	/* If there's more than one active PF on this NIC, then that
+	 * implies that the NIC firmware is loaded and running.  This check
+	 * prevents a rare false negative that might occur if we only relied
+	 * on checking the SCR2_BIT_FW_LOADED flag.  The false negative would
+	 * happen if the PF driver sees SCR2_BIT_FW_LOADED as cleared even
+	 * though the firmware was already loaded but still booting and has yet
+	 * to set SCR2_BIT_FW_LOADED.
+	 */
+	if (atomic_read(oct->adapter_refcount) > 1)
+		return 1;
+
+	val = octeon_read_csr64(oct, CN23XX_SLI_SCRATCH2);
+	return (val >> SCR2_BIT_FW_LOADED) & 1ULL;
 }
 
 void cn23xx_tell_vf_its_macaddr_changed(struct octeon_device *oct, int vfidx,
diff --git a/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.h b/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.h
index dee604651ba7..2aba5247b6d8 100644
--- a/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.h
+++ b/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_device.h
@@ -24,8 +24,6 @@
 
 #include "cn23xx_pf_regs.h"
 
-#define LIO_CMD_WAIT_TM 100
-
 /* Register address and configuration for a CN23XX devices.
  * If device specific changes need to be made then add a struct to include
  * device specific fields as shown in the commented section
diff --git a/drivers/net/ethernet/cavium/liquidio/cn23xx_vf_device.h b/drivers/net/ethernet/cavium/liquidio/cn23xx_vf_device.h
index 3f98c7334957..2d06097d3f61 100644
--- a/drivers/net/ethernet/cavium/liquidio/cn23xx_vf_device.h
+++ b/drivers/net/ethernet/cavium/liquidio/cn23xx_vf_device.h
@@ -36,8 +36,6 @@ struct octeon_cn23xx_vf {
 
 #define CN23XX_MAILBOX_MSGPARAM_SIZE		6
 
-#define MAX_VF_IP_OP_PENDING_PKT_COUNT		100
-
 void cn23xx_vf_ask_pf_to_do_flr(struct octeon_device *oct);
 
 int cn23xx_octeon_pfvf_handshake(struct octeon_device *oct);
diff --git a/drivers/net/ethernet/cavium/liquidio/lio_core.c b/drivers/net/ethernet/cavium/liquidio/lio_core.c
index adde7745d069..23f6b60030c5 100644
--- a/drivers/net/ethernet/cavium/liquidio/lio_core.c
+++ b/drivers/net/ethernet/cavium/liquidio/lio_core.c
@@ -165,9 +165,6 @@ void liquidio_link_ctrl_cmd_completion(void *nctrl_ptr)
 		/* If command is successful, change the MTU. */
 		netif_info(lio, probe, lio->netdev, "MTU Changed from %d to %d\n",
 			   netdev->mtu, nctrl->ncmd.s.param1);
-		dev_info(&oct->pci_dev->dev, "%s MTU Changed from %d to %d\n",
-			 netdev->name, netdev->mtu,
-			 nctrl->ncmd.s.param1);
 		netdev->mtu = nctrl->ncmd.s.param1;
 		queue_delayed_work(lio->link_status_wq.wq,
 				   &lio->link_status_wq.wk.work, 0);
@@ -275,6 +272,11 @@ void liquidio_link_ctrl_cmd_completion(void *nctrl_ptr)
 		netif_info(lio, probe, lio->netdev, "Set RX/TX flow control parameters\n");
 		break;
 
+	case OCTNET_CMD_QUEUE_COUNT_CTL:
+		netif_info(lio, probe, lio->netdev, "Queue count updated to %d\n",
+			   nctrl->ncmd.s.param1);
+		break;
+
 	default:
 		dev_err(&oct->pci_dev->dev, "%s Unknown cmd %d\n", __func__,
 			nctrl->ncmd.s.cmd);
@@ -364,3 +366,723 @@ void cleanup_rx_oom_poll_fn(struct net_device *netdev)
 		destroy_workqueue(lio->rxq_status_wq.wq);
 	}
 }
+
+/* Runs in interrupt context. */
+static void lio_update_txq_status(struct octeon_device *oct, int iq_num)
+{
+	struct octeon_instr_queue *iq = oct->instr_queue[iq_num];
+	struct net_device *netdev;
+	struct lio *lio;
+
+	netdev = oct->props[iq->ifidx].netdev;
+
+	/* This is needed because the first IQ does not have
+	 * a netdev associated with it.
+	 */
+	if (!netdev)
+		return;
+
+	lio = GET_LIO(netdev);
+	if (netif_is_multiqueue(netdev)) {
+		if (__netif_subqueue_stopped(netdev, iq->q_index) &&
+		    lio->linfo.link.s.link_up &&
+		    (!octnet_iq_is_full(oct, iq_num))) {
+			netif_wake_subqueue(netdev, iq->q_index);
+			INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq_num,
+						  tx_restart, 1);
+		}
+	} else if (netif_queue_stopped(netdev) &&
+		   lio->linfo.link.s.link_up &&
+		   (!octnet_iq_is_full(oct, lio->txq))) {
+		INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, lio->txq,
+					  tx_restart, 1);
+		netif_wake_queue(netdev);
+	}
+}
+
+/**
+ * \brief Setup output queue
+ * @param oct octeon device
+ * @param q_no which queue
+ * @param num_descs how many descriptors
+ * @param desc_size size of each descriptor
+ * @param app_ctx application context
+ */
+static int octeon_setup_droq(struct octeon_device *oct, int q_no, int num_descs,
+			     int desc_size, void *app_ctx)
+{
+	int ret_val;
+
+	dev_dbg(&oct->pci_dev->dev, "Creating Droq: %d\n", q_no);
+	/* droq creation and local register settings. */
+	ret_val = octeon_create_droq(oct, q_no, num_descs, desc_size, app_ctx);
+	if (ret_val < 0)
+		return ret_val;
+
+	if (ret_val == 1) {
+		dev_dbg(&oct->pci_dev->dev, "Using default droq %d\n", q_no);
+		return 0;
+	}
+
+	/* Enable the droq queues */
+	octeon_set_droq_pkt_op(oct, q_no, 1);
+
+	/* Send Credit for Octeon Output queues. Credits are always
+	 * sent after the output queue is enabled.
+	 */
+	writel(oct->droq[q_no]->max_count, oct->droq[q_no]->pkts_credit_reg);
+
+	return ret_val;
+}
+
+/** Routine to push packets arriving on Octeon interface upto network layer.
+ * @param oct_id   - octeon device id.
+ * @param skbuff   - skbuff struct to be passed to network layer.
+ * @param len      - size of total data received.
+ * @param rh       - Control header associated with the packet
+ * @param param    - additional control data with the packet
+ * @param arg      - farg registered in droq_ops
+ */
+static void
+liquidio_push_packet(u32 octeon_id __attribute__((unused)),
+		     void *skbuff,
+		     u32 len,
+		     union octeon_rh *rh,
+		     void *param,
+		     void *arg)
+{
+	struct net_device *netdev = (struct net_device *)arg;
+	struct octeon_droq *droq =
+	    container_of(param, struct octeon_droq, napi);
+	struct sk_buff *skb = (struct sk_buff *)skbuff;
+	struct skb_shared_hwtstamps *shhwtstamps;
+	struct napi_struct *napi = param;
+	u16 vtag = 0;
+	u32 r_dh_off;
+	u64 ns;
+
+	if (netdev) {
+		struct lio *lio = GET_LIO(netdev);
+		struct octeon_device *oct = lio->oct_dev;
+		int packet_was_received;
+
+		/* Do not proceed if the interface is not in RUNNING state. */
+		if (!ifstate_check(lio, LIO_IFSTATE_RUNNING)) {
+			recv_buffer_free(skb);
+			droq->stats.rx_dropped++;
+			return;
+		}
+
+		skb->dev = netdev;
+
+		skb_record_rx_queue(skb, droq->q_no);
+		if (likely(len > MIN_SKB_SIZE)) {
+			struct octeon_skb_page_info *pg_info;
+			unsigned char *va;
+
+			pg_info = ((struct octeon_skb_page_info *)(skb->cb));
+			if (pg_info->page) {
+				/* For Paged allocation use the frags */
+				va = page_address(pg_info->page) +
+					pg_info->page_offset;
+				memcpy(skb->data, va, MIN_SKB_SIZE);
+				skb_put(skb, MIN_SKB_SIZE);
+				skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
+						pg_info->page,
+						pg_info->page_offset +
+						MIN_SKB_SIZE,
+						len - MIN_SKB_SIZE,
+						LIO_RXBUFFER_SZ);
+			}
+		} else {
+			struct octeon_skb_page_info *pg_info =
+				((struct octeon_skb_page_info *)(skb->cb));
+			skb_copy_to_linear_data(skb, page_address(pg_info->page)
+						+ pg_info->page_offset, len);
+			skb_put(skb, len);
+			put_page(pg_info->page);
+		}
+
+		r_dh_off = (rh->r_dh.len - 1) * BYTES_PER_DHLEN_UNIT;
+
+		if (oct->ptp_enable) {
+			if (rh->r_dh.has_hwtstamp) {
+				/* timestamp is included from the hardware at
+				 * the beginning of the packet.
+				 */
+				if (ifstate_check
+					(lio,
+					 LIO_IFSTATE_RX_TIMESTAMP_ENABLED)) {
+					/* Nanoseconds are in the first 64-bits
+					 * of the packet.
+					 */
+					memcpy(&ns, (skb->data + r_dh_off),
+					       sizeof(ns));
+					r_dh_off -= BYTES_PER_DHLEN_UNIT;
+					shhwtstamps = skb_hwtstamps(skb);
+					shhwtstamps->hwtstamp =
+						ns_to_ktime(ns +
+							    lio->ptp_adjust);
+				}
+			}
+		}
+
+		if (rh->r_dh.has_hash) {
+			__be32 *hash_be = (__be32 *)(skb->data + r_dh_off);
+			u32 hash = be32_to_cpu(*hash_be);
+
+			skb_set_hash(skb, hash, PKT_HASH_TYPE_L4);
+			r_dh_off -= BYTES_PER_DHLEN_UNIT;
+		}
+
+		skb_pull(skb, rh->r_dh.len * BYTES_PER_DHLEN_UNIT);
+		skb->protocol = eth_type_trans(skb, skb->dev);
+
+		if ((netdev->features & NETIF_F_RXCSUM) &&
+		    (((rh->r_dh.encap_on) &&
+		      (rh->r_dh.csum_verified & CNNIC_TUN_CSUM_VERIFIED)) ||
+		     (!(rh->r_dh.encap_on) &&
+		      (rh->r_dh.csum_verified & CNNIC_CSUM_VERIFIED))))
+			/* checksum has already been verified */
+			skb->ip_summed = CHECKSUM_UNNECESSARY;
+		else
+			skb->ip_summed = CHECKSUM_NONE;
+
+		/* Setting Encapsulation field on basis of status received
+		 * from the firmware
+		 */
+		if (rh->r_dh.encap_on) {
+			skb->encapsulation = 1;
+			skb->csum_level = 1;
+			droq->stats.rx_vxlan++;
+		}
+
+		/* inbound VLAN tag */
+		if ((netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
+		    rh->r_dh.vlan) {
+			u16 priority = rh->r_dh.priority;
+			u16 vid = rh->r_dh.vlan;
+
+			vtag = (priority << VLAN_PRIO_SHIFT) | vid;
+			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vtag);
+		}
+
+		packet_was_received = (napi_gro_receive(napi, skb) != GRO_DROP);
+
+		if (packet_was_received) {
+			droq->stats.rx_bytes_received += len;
+			droq->stats.rx_pkts_received++;
+		} else {
+			droq->stats.rx_dropped++;
+			netif_info(lio, rx_err, lio->netdev,
+				   "droq:%d  error rx_dropped:%llu\n",
+				   droq->q_no, droq->stats.rx_dropped);
+		}
+
+	} else {
+		recv_buffer_free(skb);
+	}
+}
+
+/**
+ * \brief wrapper for calling napi_schedule
+ * @param param parameters to pass to napi_schedule
+ *
+ * Used when scheduling on different CPUs
+ */
+static void napi_schedule_wrapper(void *param)
+{
+	struct napi_struct *napi = param;
+
+	napi_schedule(napi);
+}
+
+/**
+ * \brief callback when receive interrupt occurs and we are in NAPI mode
+ * @param arg pointer to octeon output queue
+ */
+static void liquidio_napi_drv_callback(void *arg)
+{
+	struct octeon_device *oct;
+	struct octeon_droq *droq = arg;
+	int this_cpu = smp_processor_id();
+
+	oct = droq->oct_dev;
+
+	if (OCTEON_CN23XX_PF(oct) || OCTEON_CN23XX_VF(oct) ||
+	    droq->cpu_id == this_cpu) {
+		napi_schedule_irqoff(&droq->napi);
+	} else {
+		call_single_data_t *csd = &droq->csd;
+
+		csd->func = napi_schedule_wrapper;
+		csd->info = &droq->napi;
+		csd->flags = 0;
+
+		smp_call_function_single_async(droq->cpu_id, csd);
+	}
+}
+
+/**
+ * \brief Entry point for NAPI polling
+ * @param napi NAPI structure
+ * @param budget maximum number of items to process
+ */
+static int liquidio_napi_poll(struct napi_struct *napi, int budget)
+{
+	struct octeon_instr_queue *iq;
+	struct octeon_device *oct;
+	struct octeon_droq *droq;
+	int tx_done = 0, iq_no;
+	int work_done;
+
+	droq = container_of(napi, struct octeon_droq, napi);
+	oct = droq->oct_dev;
+	iq_no = droq->q_no;
+
+	/* Handle Droq descriptors */
+	work_done = octeon_process_droq_poll_cmd(oct, droq->q_no,
+						 POLL_EVENT_PROCESS_PKTS,
+						 budget);
+
+	/* Flush the instruction queue */
+	iq = oct->instr_queue[iq_no];
+	if (iq) {
+		/* TODO: move this check to inside octeon_flush_iq,
+		 * once check_db_timeout is removed
+		 */
+		if (atomic_read(&iq->instr_pending))
+			/* Process iq buffers with in the budget limits */
+			tx_done = octeon_flush_iq(oct, iq, budget);
+		else
+			tx_done = 1;
+		/* Update iq read-index rather than waiting for next interrupt.
+		 * Return back if tx_done is false.
+		 */
+		/* sub-queue status update */
+		lio_update_txq_status(oct, iq_no);
+	} else {
+		dev_err(&oct->pci_dev->dev, "%s:  iq (%d) num invalid\n",
+			__func__, iq_no);
+	}
+
+#define MAX_REG_CNT  2000000U
+	/* force enable interrupt if reg cnts are high to avoid wraparound */
+	if ((work_done < budget && tx_done) ||
+	    (iq && iq->pkt_in_done >= MAX_REG_CNT) ||
+	    (droq->pkt_count >= MAX_REG_CNT)) {
+		tx_done = 1;
+		napi_complete_done(napi, work_done);
+
+		octeon_process_droq_poll_cmd(droq->oct_dev, droq->q_no,
+					     POLL_EVENT_ENABLE_INTR, 0);
+		return 0;
+	}
+
+	return (!tx_done) ? (budget) : (work_done);
+}
+
+/**
+ * \brief Setup input and output queues
+ * @param octeon_dev octeon device
+ * @param ifidx Interface index
+ *
+ * Note: Queues are with respect to the octeon device. Thus
+ * an input queue is for egress packets, and output queues
+ * are for ingress packets.
+ */
+int liquidio_setup_io_queues(struct octeon_device *octeon_dev, int ifidx,
+			     u32 num_iqs, u32 num_oqs)
+{
+	struct octeon_droq_ops droq_ops;
+	struct net_device *netdev;
+	struct octeon_droq *droq;
+	struct napi_struct *napi;
+	int cpu_id_modulus;
+	int num_tx_descs;
+	struct lio *lio;
+	int retval = 0;
+	int q, q_no;
+	int cpu_id;
+
+	netdev = octeon_dev->props[ifidx].netdev;
+
+	lio = GET_LIO(netdev);
+
+	memset(&droq_ops, 0, sizeof(struct octeon_droq_ops));
+
+	droq_ops.fptr = liquidio_push_packet;
+	droq_ops.farg = netdev;
+
+	droq_ops.poll_mode = 1;
+	droq_ops.napi_fn = liquidio_napi_drv_callback;
+	cpu_id = 0;
+	cpu_id_modulus = num_present_cpus();
+
+	/* set up DROQs. */
+	for (q = 0; q < num_oqs; q++) {
+		q_no = lio->linfo.rxpciq[q].s.q_no;
+		dev_dbg(&octeon_dev->pci_dev->dev,
+			"%s index:%d linfo.rxpciq.s.q_no:%d\n",
+			__func__, q, q_no);
+		retval = octeon_setup_droq(
+		    octeon_dev, q_no,
+		    CFG_GET_NUM_RX_DESCS_NIC_IF(octeon_get_conf(octeon_dev),
+						lio->ifidx),
+		    CFG_GET_NUM_RX_BUF_SIZE_NIC_IF(octeon_get_conf(octeon_dev),
+						   lio->ifidx),
+		    NULL);
+		if (retval) {
+			dev_err(&octeon_dev->pci_dev->dev,
+				"%s : Runtime DROQ(RxQ) creation failed.\n",
+				__func__);
+			return 1;
+		}
+
+		droq = octeon_dev->droq[q_no];
+		napi = &droq->napi;
+		dev_dbg(&octeon_dev->pci_dev->dev, "netif_napi_add netdev:%llx oct:%llx\n",
+			(u64)netdev, (u64)octeon_dev);
+		netif_napi_add(netdev, napi, liquidio_napi_poll, 64);
+
+		/* designate a CPU for this droq */
+		droq->cpu_id = cpu_id;
+		cpu_id++;
+		if (cpu_id >= cpu_id_modulus)
+			cpu_id = 0;
+
+		octeon_register_droq_ops(octeon_dev, q_no, &droq_ops);
+	}
+
+	if (OCTEON_CN23XX_PF(octeon_dev) || OCTEON_CN23XX_VF(octeon_dev)) {
+		/* 23XX PF/VF can send/recv control messages (via the first
+		 * PF/VF-owned droq) from the firmware even if the ethX
+		 * interface is down, so that's why poll_mode must be off
+		 * for the first droq.
+		 */
+		octeon_dev->droq[0]->ops.poll_mode = 0;
+	}
+
+	/* set up IQs. */
+	for (q = 0; q < num_iqs; q++) {
+		num_tx_descs = CFG_GET_NUM_TX_DESCS_NIC_IF(
+		    octeon_get_conf(octeon_dev), lio->ifidx);
+		retval = octeon_setup_iq(octeon_dev, ifidx, q,
+					 lio->linfo.txpciq[q], num_tx_descs,
+					 netdev_get_tx_queue(netdev, q));
+		if (retval) {
+			dev_err(&octeon_dev->pci_dev->dev,
+				" %s : Runtime IQ(TxQ) creation failed.\n",
+				__func__);
+			return 1;
+		}
+
+		/* XPS */
+		if (!OCTEON_CN23XX_VF(octeon_dev) && octeon_dev->msix_on &&
+		    octeon_dev->ioq_vector) {
+			struct octeon_ioq_vector    *ioq_vector;
+
+			ioq_vector = &octeon_dev->ioq_vector[q];
+			netif_set_xps_queue(netdev,
+					    &ioq_vector->affinity_mask,
+					    ioq_vector->iq_index);
+		}
+	}
+
+	return 0;
+}
+
+static
+int liquidio_schedule_msix_droq_pkt_handler(struct octeon_droq *droq, u64 ret)
+{
+	struct octeon_device *oct = droq->oct_dev;
+	struct octeon_device_priv *oct_priv =
+	    (struct octeon_device_priv *)oct->priv;
+
+	if (droq->ops.poll_mode) {
+		droq->ops.napi_fn(droq);
+	} else {
+		if (ret & MSIX_PO_INT) {
+			if (OCTEON_CN23XX_VF(oct))
+				dev_err(&oct->pci_dev->dev,
+					"should not come here should not get rx when poll mode = 0 for vf\n");
+			tasklet_schedule(&oct_priv->droq_tasklet);
+			return 1;
+		}
+		/* this will be flushed periodically by check iq db */
+		if (ret & MSIX_PI_INT)
+			return 0;
+	}
+
+	return 0;
+}
+
+irqreturn_t
+liquidio_msix_intr_handler(int irq __attribute__((unused)), void *dev)
+{
+	struct octeon_ioq_vector *ioq_vector = (struct octeon_ioq_vector *)dev;
+	struct octeon_device *oct = ioq_vector->oct_dev;
+	struct octeon_droq *droq = oct->droq[ioq_vector->droq_index];
+	u64 ret;
+
+	ret = oct->fn_list.msix_interrupt_handler(ioq_vector);
+
+	if (ret & MSIX_PO_INT || ret & MSIX_PI_INT)
+		liquidio_schedule_msix_droq_pkt_handler(droq, ret);
+
+	return IRQ_HANDLED;
+}
+
+/**
+ * \brief Droq packet processor sceduler
+ * @param oct octeon device
+ */
+static void liquidio_schedule_droq_pkt_handlers(struct octeon_device *oct)
+{
+	struct octeon_device_priv *oct_priv =
+		(struct octeon_device_priv *)oct->priv;
+	struct octeon_droq *droq;
+	u64 oq_no;
+
+	if (oct->int_status & OCT_DEV_INTR_PKT_DATA) {
+		for (oq_no = 0; oq_no < MAX_OCTEON_OUTPUT_QUEUES(oct);
+		     oq_no++) {
+			if (!(oct->droq_intr & BIT_ULL(oq_no)))
+				continue;
+
+			droq = oct->droq[oq_no];
+
+			if (droq->ops.poll_mode) {
+				droq->ops.napi_fn(droq);
+				oct_priv->napi_mask |= (1 << oq_no);
+			} else {
+				tasklet_schedule(&oct_priv->droq_tasklet);
+			}
+		}
+	}
+}
+
+/**
+ * \brief Interrupt handler for octeon
+ * @param irq unused
+ * @param dev octeon device
+ */
+static
+irqreturn_t liquidio_legacy_intr_handler(int irq __attribute__((unused)),
+					 void *dev)
+{
+	struct octeon_device *oct = (struct octeon_device *)dev;
+	irqreturn_t ret;
+
+	/* Disable our interrupts for the duration of ISR */
+	oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
+
+	ret = oct->fn_list.process_interrupt_regs(oct);
+
+	if (ret == IRQ_HANDLED)
+		liquidio_schedule_droq_pkt_handlers(oct);
+
+	/* Re-enable our interrupts  */
+	if (!(atomic_read(&oct->status) == OCT_DEV_IN_RESET))
+		oct->fn_list.enable_interrupt(oct, OCTEON_ALL_INTR);
+
+	return ret;
+}
+
+/**
+ * \brief Setup interrupt for octeon device
+ * @param oct octeon device
+ *
+ *  Enable interrupt in Octeon device as given in the PCI interrupt mask.
+ */
+int octeon_setup_interrupt(struct octeon_device *oct, u32 num_ioqs)
+{
+	struct msix_entry *msix_entries;
+	char *queue_irq_names = NULL;
+	int i, num_interrupts = 0;
+	int num_alloc_ioq_vectors;
+	char *aux_irq_name = NULL;
+	int num_ioq_vectors;
+	int irqret, err;
+
+	oct->num_msix_irqs = num_ioqs;
+	if (oct->msix_on) {
+		if (OCTEON_CN23XX_PF(oct)) {
+			num_interrupts = MAX_IOQ_INTERRUPTS_PER_PF + 1;
+
+			/* one non ioq interrupt for handling
+			 * sli_mac_pf_int_sum
+			 */
+			oct->num_msix_irqs += 1;
+		} else if (OCTEON_CN23XX_VF(oct)) {
+			num_interrupts = MAX_IOQ_INTERRUPTS_PER_VF;
+		}
+
+		/* allocate storage for the names assigned to each irq */
+		oct->irq_name_storage =
+			kcalloc(num_interrupts, INTRNAMSIZ, GFP_KERNEL);
+		if (!oct->irq_name_storage) {
+			dev_err(&oct->pci_dev->dev, "Irq name storage alloc failed...\n");
+			return -ENOMEM;
+		}
+
+		queue_irq_names = oct->irq_name_storage;
+
+		if (OCTEON_CN23XX_PF(oct))
+			aux_irq_name = &queue_irq_names
+				[IRQ_NAME_OFF(MAX_IOQ_INTERRUPTS_PER_PF)];
+
+		oct->msix_entries = kcalloc(oct->num_msix_irqs,
+					    sizeof(struct msix_entry),
+					    GFP_KERNEL);
+		if (!oct->msix_entries) {
+			dev_err(&oct->pci_dev->dev, "Memory Alloc failed...\n");
+			kfree(oct->irq_name_storage);
+			oct->irq_name_storage = NULL;
+			return -ENOMEM;
+		}
+
+		msix_entries = (struct msix_entry *)oct->msix_entries;
+
+		/*Assumption is that pf msix vectors start from pf srn to pf to
+		 * trs and not from 0. if not change this code
+		 */
+		if (OCTEON_CN23XX_PF(oct)) {
+			for (i = 0; i < oct->num_msix_irqs - 1; i++)
+				msix_entries[i].entry =
+					oct->sriov_info.pf_srn + i;
+
+			msix_entries[oct->num_msix_irqs - 1].entry =
+				oct->sriov_info.trs;
+		} else if (OCTEON_CN23XX_VF(oct)) {
+			for (i = 0; i < oct->num_msix_irqs; i++)
+				msix_entries[i].entry = i;
+		}
+		num_alloc_ioq_vectors = pci_enable_msix_range(
+						oct->pci_dev, msix_entries,
+						oct->num_msix_irqs,
+						oct->num_msix_irqs);
+		if (num_alloc_ioq_vectors < 0) {
+			dev_err(&oct->pci_dev->dev, "unable to Allocate MSI-X interrupts\n");
+			kfree(oct->msix_entries);
+			oct->msix_entries = NULL;
+			kfree(oct->irq_name_storage);
+			oct->irq_name_storage = NULL;
+			return num_alloc_ioq_vectors;
+		}
+
+		dev_dbg(&oct->pci_dev->dev, "OCTEON: Enough MSI-X interrupts are allocated...\n");
+
+		num_ioq_vectors = oct->num_msix_irqs;
+		/** For PF, there is one non-ioq interrupt handler */
+		if (OCTEON_CN23XX_PF(oct)) {
+			num_ioq_vectors -= 1;
+
+			snprintf(aux_irq_name, INTRNAMSIZ,
+				 "LiquidIO%u-pf%u-aux", oct->octeon_id,
+				 oct->pf_num);
+			irqret = request_irq(
+					msix_entries[num_ioq_vectors].vector,
+					liquidio_legacy_intr_handler, 0,
+					aux_irq_name, oct);
+			if (irqret) {
+				dev_err(&oct->pci_dev->dev,
+					"Request_irq failed for MSIX interrupt Error: %d\n",
+					irqret);
+				pci_disable_msix(oct->pci_dev);
+				kfree(oct->msix_entries);
+				kfree(oct->irq_name_storage);
+				oct->irq_name_storage = NULL;
+				oct->msix_entries = NULL;
+				return irqret;
+			}
+		}
+		for (i = 0 ; i < num_ioq_vectors ; i++) {
+			if (OCTEON_CN23XX_PF(oct))
+				snprintf(&queue_irq_names[IRQ_NAME_OFF(i)],
+					 INTRNAMSIZ, "LiquidIO%u-pf%u-rxtx-%u",
+					 oct->octeon_id, oct->pf_num, i);
+
+			if (OCTEON_CN23XX_VF(oct))
+				snprintf(&queue_irq_names[IRQ_NAME_OFF(i)],
+					 INTRNAMSIZ, "LiquidIO%u-vf%u-rxtx-%u",
+					 oct->octeon_id, oct->vf_num, i);
+
+			irqret = request_irq(msix_entries[i].vector,
+					     liquidio_msix_intr_handler, 0,
+					     &queue_irq_names[IRQ_NAME_OFF(i)],
+					     &oct->ioq_vector[i]);
+
+			if (irqret) {
+				dev_err(&oct->pci_dev->dev,
+					"Request_irq failed for MSIX interrupt Error: %d\n",
+					irqret);
+				/** Freeing the non-ioq irq vector here . */
+				free_irq(msix_entries[num_ioq_vectors].vector,
+					 oct);
+
+				while (i) {
+					i--;
+					/** clearing affinity mask. */
+					irq_set_affinity_hint(
+						      msix_entries[i].vector,
+						      NULL);
+					free_irq(msix_entries[i].vector,
+						 &oct->ioq_vector[i]);
+				}
+				pci_disable_msix(oct->pci_dev);
+				kfree(oct->msix_entries);
+				kfree(oct->irq_name_storage);
+				oct->irq_name_storage = NULL;
+				oct->msix_entries = NULL;
+				return irqret;
+			}
+			oct->ioq_vector[i].vector = msix_entries[i].vector;
+			/* assign the cpu mask for this msix interrupt vector */
+			irq_set_affinity_hint(msix_entries[i].vector,
+					      &oct->ioq_vector[i].affinity_mask
+					      );
+		}
+		dev_dbg(&oct->pci_dev->dev, "OCTEON[%d]: MSI-X enabled\n",
+			oct->octeon_id);
+	} else {
+		err = pci_enable_msi(oct->pci_dev);
+		if (err)
+			dev_warn(&oct->pci_dev->dev, "Reverting to legacy interrupts. Error: %d\n",
+				 err);
+		else
+			oct->flags |= LIO_FLAG_MSI_ENABLED;
+
+		/* allocate storage for the names assigned to the irq */
+		oct->irq_name_storage = kcalloc(1, INTRNAMSIZ, GFP_KERNEL);
+		if (!oct->irq_name_storage)
+			return -ENOMEM;
+
+		queue_irq_names = oct->irq_name_storage;
+
+		if (OCTEON_CN23XX_PF(oct))
+			snprintf(&queue_irq_names[IRQ_NAME_OFF(0)], INTRNAMSIZ,
+				 "LiquidIO%u-pf%u-rxtx-%u",
+				 oct->octeon_id, oct->pf_num, 0);
+
+		if (OCTEON_CN23XX_VF(oct))
+			snprintf(&queue_irq_names[IRQ_NAME_OFF(0)], INTRNAMSIZ,
+				 "LiquidIO%u-vf%u-rxtx-%u",
+				 oct->octeon_id, oct->vf_num, 0);
+
+		irqret = request_irq(oct->pci_dev->irq,
+				     liquidio_legacy_intr_handler,
+				     IRQF_SHARED,
+				     &queue_irq_names[IRQ_NAME_OFF(0)], oct);
+		if (irqret) {
+			if (oct->flags & LIO_FLAG_MSI_ENABLED)
+				pci_disable_msi(oct->pci_dev);
+			dev_err(&oct->pci_dev->dev, "Request IRQ failed with code: %d\n",
+				irqret);
+			kfree(oct->irq_name_storage);
+			oct->irq_name_storage = NULL;
+			return irqret;
+		}
+	}
+	return 0;
+}
diff --git a/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c b/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c
index ebd353bc78ff..a63ddf07f168 100644
--- a/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c
+++ b/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c
@@ -31,6 +31,7 @@
 #include "cn23xx_pf_device.h"
 #include "cn23xx_vf_device.h"
 
+static int lio_reset_queues(struct net_device *netdev, uint32_t num_qs);
 static int octnet_get_link_stats(struct net_device *netdev);
 
 struct oct_intrmod_context {
@@ -105,6 +106,7 @@ static const char oct_stats_strings[][ETH_GSTRING_LEN] = {
 	"tx_total_sent",
 	"tx_total_fwd",
 	"tx_err_pko",
+	"tx_err_pki",
 	"tx_err_link",
 	"tx_err_drop",
 
@@ -299,6 +301,35 @@ lio_get_vf_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo)
 	strncpy(drvinfo->bus_info, pci_name(oct->pci_dev), 32);
 }
 
+static int
+lio_send_queue_count_update(struct net_device *netdev, uint32_t num_queues)
+{
+	struct lio *lio = GET_LIO(netdev);
+	struct octeon_device *oct = lio->oct_dev;
+	struct octnic_ctrl_pkt nctrl;
+	int ret = 0;
+
+	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
+
+	nctrl.ncmd.u64 = 0;
+	nctrl.ncmd.s.cmd = OCTNET_CMD_QUEUE_COUNT_CTL;
+	nctrl.ncmd.s.param1 = num_queues;
+	nctrl.ncmd.s.param2 = num_queues;
+	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
+	nctrl.wait_time = 100;
+	nctrl.netpndev = (u64)netdev;
+	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
+
+	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
+	if (ret < 0) {
+		dev_err(&oct->pci_dev->dev, "Failed to send Queue reset command (ret: 0x%x)\n",
+			ret);
+		return -1;
+	}
+
+	return 0;
+}
+
 static void
 lio_ethtool_get_channels(struct net_device *dev,
 			 struct ethtool_channels *channel)
@@ -306,6 +337,7 @@ lio_ethtool_get_channels(struct net_device *dev,
 	struct lio *lio = GET_LIO(dev);
 	struct octeon_device *oct = lio->oct_dev;
 	u32 max_rx = 0, max_tx = 0, tx_count = 0, rx_count = 0;
+	u32 combined_count = 0, max_combined = 0;
 
 	if (OCTEON_CN6XXX(oct)) {
 		struct octeon_config *conf6x = CHIP_CONF(oct, cn6xxx);
@@ -315,22 +347,137 @@ lio_ethtool_get_channels(struct net_device *dev,
 		rx_count = CFG_GET_NUM_RXQS_NIC_IF(conf6x, lio->ifidx);
 		tx_count = CFG_GET_NUM_TXQS_NIC_IF(conf6x, lio->ifidx);
 	} else if (OCTEON_CN23XX_PF(oct)) {
-
-		max_rx = oct->sriov_info.num_pf_rings;
-		max_tx = oct->sriov_info.num_pf_rings;
-		rx_count = lio->linfo.num_rxpciq;
-		tx_count = lio->linfo.num_txpciq;
+		max_combined = lio->linfo.num_txpciq;
+		combined_count = oct->num_iqs;
 	} else if (OCTEON_CN23XX_VF(oct)) {
-		max_tx = oct->sriov_info.rings_per_vf;
-		max_rx = oct->sriov_info.rings_per_vf;
-		rx_count = lio->linfo.num_rxpciq;
-		tx_count = lio->linfo.num_txpciq;
+		u64 reg_val = 0ULL;
+		u64 ctrl = CN23XX_VF_SLI_IQ_PKT_CONTROL64(0);
+
+		reg_val = octeon_read_csr64(oct, ctrl);
+		reg_val = reg_val >> CN23XX_PKT_INPUT_CTL_RPVF_POS;
+		max_combined = reg_val & CN23XX_PKT_INPUT_CTL_RPVF_MASK;
+		combined_count = oct->num_iqs;
 	}
 
 	channel->max_rx = max_rx;
 	channel->max_tx = max_tx;
+	channel->max_combined = max_combined;
 	channel->rx_count = rx_count;
 	channel->tx_count = tx_count;
+	channel->combined_count = combined_count;
+}
+
+static int
+lio_irq_reallocate_irqs(struct octeon_device *oct, uint32_t num_ioqs)
+{
+	struct msix_entry *msix_entries;
+	int num_msix_irqs = 0;
+	int i;
+
+	if (!oct->msix_on)
+		return 0;
+
+	/* Disable the input and output queues now. No more packets will
+	 * arrive from Octeon.
+	 */
+	oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
+
+	if (oct->msix_on) {
+		if (OCTEON_CN23XX_PF(oct))
+			num_msix_irqs = oct->num_msix_irqs - 1;
+		else if (OCTEON_CN23XX_VF(oct))
+			num_msix_irqs = oct->num_msix_irqs;
+
+		msix_entries = (struct msix_entry *)oct->msix_entries;
+		for (i = 0; i < num_msix_irqs; i++) {
+			if (oct->ioq_vector[i].vector) {
+				/* clear the affinity_cpumask */
+				irq_set_affinity_hint(msix_entries[i].vector,
+						      NULL);
+				free_irq(msix_entries[i].vector,
+					 &oct->ioq_vector[i]);
+				oct->ioq_vector[i].vector = 0;
+			}
+		}
+
+		/* non-iov vector's argument is oct struct */
+		if (OCTEON_CN23XX_PF(oct))
+			free_irq(msix_entries[i].vector, oct);
+
+		pci_disable_msix(oct->pci_dev);
+		kfree(oct->msix_entries);
+		oct->msix_entries = NULL;
+	}
+
+	kfree(oct->irq_name_storage);
+	oct->irq_name_storage = NULL;
+	if (octeon_setup_interrupt(oct, num_ioqs)) {
+		dev_info(&oct->pci_dev->dev, "Setup interrupt failed\n");
+		return 1;
+	}
+
+	/* Enable Octeon device interrupts */
+	oct->fn_list.enable_interrupt(oct, OCTEON_ALL_INTR);
+
+	return 0;
+}
+
+static int
+lio_ethtool_set_channels(struct net_device *dev,
+			 struct ethtool_channels *channel)
+{
+	u32 combined_count, max_combined;
+	struct lio *lio = GET_LIO(dev);
+	struct octeon_device *oct = lio->oct_dev;
+	int stopped = 0;
+
+	if (strcmp(oct->fw_info.liquidio_firmware_version, "1.6.1") < 0) {
+		dev_err(&oct->pci_dev->dev, "Minimum firmware version required is 1.6.1\n");
+		return -EINVAL;
+	}
+
+	if (!channel->combined_count || channel->other_count ||
+	    channel->rx_count || channel->tx_count)
+		return -EINVAL;
+
+	combined_count = channel->combined_count;
+
+	if (OCTEON_CN23XX_PF(oct)) {
+		max_combined = channel->max_combined;
+	} else if (OCTEON_CN23XX_VF(oct)) {
+		u64 reg_val = 0ULL;
+		u64 ctrl = CN23XX_VF_SLI_IQ_PKT_CONTROL64(0);
+
+		reg_val = octeon_read_csr64(oct, ctrl);
+		reg_val = reg_val >> CN23XX_PKT_INPUT_CTL_RPVF_POS;
+		max_combined = reg_val & CN23XX_PKT_INPUT_CTL_RPVF_MASK;
+	} else {
+		return -EINVAL;
+	}
+
+	if (combined_count > max_combined || combined_count < 1)
+		return -EINVAL;
+
+	if (combined_count == oct->num_iqs)
+		return 0;
+
+	ifstate_set(lio, LIO_IFSTATE_RESETTING);
+
+	if (netif_running(dev)) {
+		dev->netdev_ops->ndo_stop(dev);
+		stopped = 1;
+	}
+
+	if (lio_reset_queues(dev, combined_count))
+		return -EINVAL;
+
+	lio_irq_reallocate_irqs(oct, combined_count);
+	if (stopped)
+		dev->netdev_ops->ndo_open(dev);
+
+	ifstate_reset(lio, LIO_IFSTATE_RESETTING);
+
+	return 0;
 }
 
 static int lio_get_eeprom_len(struct net_device *netdev)
@@ -577,23 +724,18 @@ static int lio_set_phys_id(struct net_device *netdev,
 		break;
 
 	case ETHTOOL_ID_ON:
-		if (oct->chip_id == OCTEON_CN66XX) {
+		if (oct->chip_id == OCTEON_CN66XX)
 			octnet_gpio_access(netdev, VITESSE_PHY_GPIO_CFG,
 					   VITESSE_PHY_GPIO_HIGH);
-
-		} else if (oct->chip_id == OCTEON_CN68XX) {
-			return -EINVAL;
-		} else {
+		else
 			return -EINVAL;
-		}
+
 		break;
 
 	case ETHTOOL_ID_OFF:
 		if (oct->chip_id == OCTEON_CN66XX)
 			octnet_gpio_access(netdev, VITESSE_PHY_GPIO_CFG,
 					   VITESSE_PHY_GPIO_LOW);
-		else if (oct->chip_id == OCTEON_CN68XX)
-			return -EINVAL;
 		else
 			return -EINVAL;
 
@@ -641,6 +783,9 @@ lio_ethtool_get_ringparam(struct net_device *netdev,
 	u32 tx_max_pending = 0, rx_max_pending = 0, tx_pending = 0,
 	    rx_pending = 0;
 
+	if (ifstate_check(lio, LIO_IFSTATE_RESETTING))
+		return;
+
 	if (OCTEON_CN6XXX(oct)) {
 		struct octeon_config *conf6x = CHIP_CONF(oct, cn6xxx);
 
@@ -648,33 +793,147 @@ lio_ethtool_get_ringparam(struct net_device *netdev,
 		rx_max_pending = CN6XXX_MAX_OQ_DESCRIPTORS;
 		rx_pending = CFG_GET_NUM_RX_DESCS_NIC_IF(conf6x, lio->ifidx);
 		tx_pending = CFG_GET_NUM_TX_DESCS_NIC_IF(conf6x, lio->ifidx);
-	} else if (OCTEON_CN23XX_PF(oct)) {
-		struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_pf);
-
+	} else if (OCTEON_CN23XX_PF(oct) || OCTEON_CN23XX_VF(oct)) {
 		tx_max_pending = CN23XX_MAX_IQ_DESCRIPTORS;
 		rx_max_pending = CN23XX_MAX_OQ_DESCRIPTORS;
-		rx_pending = CFG_GET_NUM_RX_DESCS_NIC_IF(conf23, lio->ifidx);
-		tx_pending = CFG_GET_NUM_TX_DESCS_NIC_IF(conf23, lio->ifidx);
-	}
-
-	if (lio->mtu > OCTNET_DEFAULT_FRM_SIZE - OCTNET_FRM_HEADER_SIZE) {
-		ering->rx_pending = 0;
-		ering->rx_max_pending = 0;
-		ering->rx_mini_pending = 0;
-		ering->rx_jumbo_pending = rx_pending;
-		ering->rx_mini_max_pending = 0;
-		ering->rx_jumbo_max_pending = rx_max_pending;
-	} else {
-		ering->rx_pending = rx_pending;
-		ering->rx_max_pending = rx_max_pending;
-		ering->rx_mini_pending = 0;
-		ering->rx_jumbo_pending = 0;
-		ering->rx_mini_max_pending = 0;
-		ering->rx_jumbo_max_pending = 0;
+		rx_pending = oct->droq[0]->max_count;
+		tx_pending = oct->instr_queue[0]->max_count;
 	}
 
 	ering->tx_pending = tx_pending;
 	ering->tx_max_pending = tx_max_pending;
+	ering->rx_pending = rx_pending;
+	ering->rx_max_pending = rx_max_pending;
+	ering->rx_mini_pending = 0;
+	ering->rx_jumbo_pending = 0;
+	ering->rx_mini_max_pending = 0;
+	ering->rx_jumbo_max_pending = 0;
+}
+
+static int lio_reset_queues(struct net_device *netdev, uint32_t num_qs)
+{
+	struct lio *lio = GET_LIO(netdev);
+	struct octeon_device *oct = lio->oct_dev;
+	struct napi_struct *napi, *n;
+	int i, update = 0;
+
+	if (wait_for_pending_requests(oct))
+		dev_err(&oct->pci_dev->dev, "There were pending requests\n");
+
+	if (lio_wait_for_instr_fetch(oct))
+		dev_err(&oct->pci_dev->dev, "IQ had pending instructions\n");
+
+	if (octeon_set_io_queues_off(oct)) {
+		dev_err(&oct->pci_dev->dev, "setting io queues off failed\n");
+		return -1;
+	}
+
+	/* Disable the input and output queues now. No more packets will
+	 * arrive from Octeon.
+	 */
+	oct->fn_list.disable_io_queues(oct);
+	/* Delete NAPI */
+	list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
+		netif_napi_del(napi);
+
+	if (num_qs != oct->num_iqs) {
+		netif_set_real_num_rx_queues(netdev, num_qs);
+		netif_set_real_num_tx_queues(netdev, num_qs);
+		update = 1;
+	}
+
+	for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
+		if (!(oct->io_qmask.oq & BIT_ULL(i)))
+			continue;
+		octeon_delete_droq(oct, i);
+	}
+
+	for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
+		if (!(oct->io_qmask.iq & BIT_ULL(i)))
+			continue;
+		octeon_delete_instr_queue(oct, i);
+	}
+
+	if (oct->fn_list.setup_device_regs(oct)) {
+		dev_err(&oct->pci_dev->dev, "Failed to configure device registers\n");
+		return -1;
+	}
+
+	if (liquidio_setup_io_queues(oct, 0, num_qs, num_qs)) {
+		dev_err(&oct->pci_dev->dev, "IO queues initialization failed\n");
+		return -1;
+	}
+
+	/* Enable the input and output queues for this Octeon device */
+	if (oct->fn_list.enable_io_queues(oct)) {
+		dev_err(&oct->pci_dev->dev, "Failed to enable input/output queues");
+		return -1;
+	}
+
+	if (update && lio_send_queue_count_update(netdev, num_qs))
+		return -1;
+
+	return 0;
+}
+
+static int lio_ethtool_set_ringparam(struct net_device *netdev,
+				     struct ethtool_ringparam *ering)
+{
+	u32 rx_count, tx_count, rx_count_old, tx_count_old;
+	struct lio *lio = GET_LIO(netdev);
+	struct octeon_device *oct = lio->oct_dev;
+	int stopped = 0;
+
+	if (!OCTEON_CN23XX_PF(oct) && !OCTEON_CN23XX_VF(oct))
+		return -EINVAL;
+
+	if (ering->rx_mini_pending || ering->rx_jumbo_pending)
+		return -EINVAL;
+
+	rx_count = clamp_t(u32, ering->rx_pending, CN23XX_MIN_OQ_DESCRIPTORS,
+			   CN23XX_MAX_OQ_DESCRIPTORS);
+	tx_count = clamp_t(u32, ering->tx_pending, CN23XX_MIN_IQ_DESCRIPTORS,
+			   CN23XX_MAX_IQ_DESCRIPTORS);
+
+	rx_count_old = oct->droq[0]->max_count;
+	tx_count_old = oct->instr_queue[0]->max_count;
+
+	if (rx_count == rx_count_old && tx_count == tx_count_old)
+		return 0;
+
+	ifstate_set(lio, LIO_IFSTATE_RESETTING);
+
+	if (netif_running(netdev)) {
+		netdev->netdev_ops->ndo_stop(netdev);
+		stopped = 1;
+	}
+
+	/* Change RX/TX DESCS  count */
+	if (tx_count != tx_count_old)
+		CFG_SET_NUM_TX_DESCS_NIC_IF(octeon_get_conf(oct), lio->ifidx,
+					    tx_count);
+	if (rx_count != rx_count_old)
+		CFG_SET_NUM_RX_DESCS_NIC_IF(octeon_get_conf(oct), lio->ifidx,
+					    rx_count);
+
+	if (lio_reset_queues(netdev, lio->linfo.num_txpciq))
+		goto err_lio_reset_queues;
+
+	if (stopped)
+		netdev->netdev_ops->ndo_open(netdev);
+
+	ifstate_reset(lio, LIO_IFSTATE_RESETTING);
+
+	return 0;
+
+err_lio_reset_queues:
+	if (tx_count != tx_count_old)
+		CFG_SET_NUM_TX_DESCS_NIC_IF(octeon_get_conf(oct), lio->ifidx,
+					    tx_count_old);
+	if (rx_count != rx_count_old)
+		CFG_SET_NUM_RX_DESCS_NIC_IF(octeon_get_conf(oct), lio->ifidx,
+					    rx_count_old);
+	return -EINVAL;
 }
 
 static u32 lio_get_msglevel(struct net_device *netdev)
@@ -795,6 +1054,9 @@ lio_get_ethtool_stats(struct net_device *netdev,
 	struct net_device_stats *netstats = &netdev->stats;
 	int i = 0, j;
 
+	if (ifstate_check(lio, LIO_IFSTATE_RESETTING))
+		return;
+
 	netdev->netdev_ops->ndo_get_stats(netdev);
 	octnet_get_link_stats(netdev);
 
@@ -826,6 +1088,8 @@ lio_get_ethtool_stats(struct net_device *netdev,
 	data[i++] = CVM_CAST64(oct_dev->link_stats.fromhost.fw_total_fwd);
 	/*per_core_stats[j].link_stats[i].fromhost.fw_err_pko */
 	data[i++] = CVM_CAST64(oct_dev->link_stats.fromhost.fw_err_pko);
+	/*per_core_stats[j].link_stats[i].fromhost.fw_err_pki */
+	data[i++] = CVM_CAST64(oct_dev->link_stats.fromhost.fw_err_pki);
 	/*per_core_stats[j].link_stats[i].fromhost.fw_err_link */
 	data[i++] = CVM_CAST64(oct_dev->link_stats.fromhost.fw_err_link);
 	/*per_core_stats[cvmx_get_core_num()].link_stats[idx].fromhost.
@@ -1057,6 +1321,9 @@ static void lio_vf_get_ethtool_stats(struct net_device *netdev,
 	struct octeon_device *oct_dev = lio->oct_dev;
 	int i = 0, j, vj;
 
+	if (ifstate_check(lio, LIO_IFSTATE_RESETTING))
+		return;
+
 	netdev->netdev_ops->ndo_get_stats(netdev);
 	/* sum of oct->droq[oq_no]->stats->rx_pkts_received */
 	data[i++] = CVM_CAST64(netstats->rx_packets);
@@ -1079,7 +1346,7 @@ static void lio_vf_get_ethtool_stats(struct net_device *netdev,
 	/* lio->link_changes */
 	data[i++] = CVM_CAST64(lio->link_changes);
 
-	for (vj = 0; vj < lio->linfo.num_txpciq; vj++) {
+	for (vj = 0; vj < oct_dev->num_iqs; vj++) {
 		j = lio->linfo.txpciq[vj].s.q_no;
 
 		/* packets to network port */
@@ -1121,7 +1388,7 @@ static void lio_vf_get_ethtool_stats(struct net_device *netdev,
 	}
 
 	/* RX */
-	for (vj = 0; vj < lio->linfo.num_rxpciq; vj++) {
+	for (vj = 0; vj < oct_dev->num_oqs; vj++) {
 		j = lio->linfo.rxpciq[vj].s.q_no;
 
 		/* packets send to TCP/IP network stack */
@@ -1568,6 +1835,7 @@ octnet_nic_stats_callback(struct octeon_device *oct_dev,
 		tstats->fw_total_sent = rsp_tstats->fw_total_sent;
 		tstats->fw_total_fwd = rsp_tstats->fw_total_fwd;
 		tstats->fw_err_pko = rsp_tstats->fw_err_pko;
+		tstats->fw_err_pki = rsp_tstats->fw_err_pki;
 		tstats->fw_err_link = rsp_tstats->fw_err_link;
 		tstats->fw_err_drop = rsp_tstats->fw_err_drop;
 		tstats->fw_tso = rsp_tstats->fw_tso;
@@ -2587,7 +2855,9 @@ static const struct ethtool_ops lio_ethtool_ops = {
 	.get_link		= ethtool_op_get_link,
 	.get_drvinfo		= lio_get_drvinfo,
 	.get_ringparam		= lio_ethtool_get_ringparam,
+	.set_ringparam		= lio_ethtool_set_ringparam,
 	.get_channels		= lio_ethtool_get_channels,
+	.set_channels		= lio_ethtool_set_channels,
 	.set_phys_id		= lio_set_phys_id,
 	.get_eeprom_len		= lio_get_eeprom_len,
 	.get_eeprom		= lio_get_eeprom,
@@ -2612,7 +2882,9 @@ static const struct ethtool_ops lio_vf_ethtool_ops = {
 	.get_link		= ethtool_op_get_link,
 	.get_drvinfo		= lio_get_vf_drvinfo,
 	.get_ringparam		= lio_ethtool_get_ringparam,
+	.set_ringparam          = lio_ethtool_set_ringparam,
 	.get_channels		= lio_ethtool_get_channels,
+	.set_channels		= lio_ethtool_set_channels,
 	.get_strings		= lio_vf_get_strings,
 	.get_ethtool_stats	= lio_vf_get_ethtool_stats,
 	.get_regs_len		= lio_get_regs_len,
diff --git a/drivers/net/ethernet/cavium/liquidio/lio_main.c b/drivers/net/ethernet/cavium/liquidio/lio_main.c
index 51583ae4b1eb..e7f54948173f 100644
--- a/drivers/net/ethernet/cavium/liquidio/lio_main.c
+++ b/drivers/net/ethernet/cavium/liquidio/lio_main.c
@@ -39,10 +39,14 @@ MODULE_AUTHOR("Cavium Networks, <support@cavium.com>");
 MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Driver");
 MODULE_LICENSE("GPL");
 MODULE_VERSION(LIQUIDIO_VERSION);
-MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210SV_NAME LIO_FW_NAME_SUFFIX);
-MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210NV_NAME LIO_FW_NAME_SUFFIX);
-MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_410NV_NAME LIO_FW_NAME_SUFFIX);
-MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_23XX_NAME LIO_FW_NAME_SUFFIX);
+MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210SV_NAME
+		"_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
+MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210NV_NAME
+		"_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
+MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_410NV_NAME
+		"_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
+MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_23XX_NAME
+		"_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
 
 static int ddr_timeout = 10000;
 module_param(ddr_timeout, int, 0644);
@@ -55,11 +59,24 @@ static int debug = -1;
 module_param(debug, int, 0644);
 MODULE_PARM_DESC(debug, "NETIF_MSG debug bits");
 
-static char fw_type[LIO_MAX_FW_TYPE_LEN];
-module_param_string(fw_type, fw_type, sizeof(fw_type), 0000);
-MODULE_PARM_DESC(fw_type, "Type of firmware to be loaded. Default \"nic\"");
+static char fw_type[LIO_MAX_FW_TYPE_LEN] = LIO_FW_NAME_TYPE_NIC;
+module_param_string(fw_type, fw_type, sizeof(fw_type), 0444);
+MODULE_PARM_DESC(fw_type, "Type of firmware to be loaded. Default \"nic\".  Use \"none\" to load firmware from flash.");
 
-static int ptp_enable = 1;
+static u32 console_bitmask;
+module_param(console_bitmask, int, 0644);
+MODULE_PARM_DESC(console_bitmask,
+		 "Bitmask indicating which consoles have debug output redirected to syslog.");
+
+/**
+ * \brief determines if a given console has debug enabled.
+ * @param console console to check
+ * @returns  1 = enabled. 0 otherwise
+ */
+static int octeon_console_debug_enabled(u32 console)
+{
+	return (console_bitmask >> (console)) & 0x1;
+}
 
 /* Polling interval for determining when NIC application is alive */
 #define LIQUIDIO_STARTER_POLL_INTERVAL_MS 100
@@ -158,16 +175,13 @@ struct handshake {
 	int started_ok;
 };
 
-struct octeon_device_priv {
-	/** Tasklet structures for this device. */
-	struct tasklet_struct droq_tasklet;
-	unsigned long napi_mask;
-};
-
 #ifdef CONFIG_PCI_IOV
 static int liquidio_enable_sriov(struct pci_dev *dev, int num_vfs);
 #endif
 
+static int octeon_dbg_console_print(struct octeon_device *oct, u32 console_num,
+				    char *prefix, char *suffix);
+
 static int octeon_device_init(struct octeon_device *);
 static int liquidio_stop(struct net_device *netdev);
 static void liquidio_remove(struct pci_dev *pdev);
@@ -256,32 +270,6 @@ static void force_io_queues_off(struct octeon_device *oct)
 }
 
 /**
- * \brief wait for all pending requests to complete
- * @param oct Pointer to Octeon device
- *
- * Called during shutdown sequence
- */
-static int wait_for_pending_requests(struct octeon_device *oct)
-{
-	int i, pcount = 0;
-
-	for (i = 0; i < 100; i++) {
-		pcount =
-			atomic_read(&oct->response_list
-				[OCTEON_ORDERED_SC_LIST].pending_req_count);
-		if (pcount)
-			schedule_timeout_uninterruptible(HZ / 10);
-		else
-			break;
-	}
-
-	if (pcount)
-		return 1;
-
-	return 0;
-}
-
-/**
  * \brief Cause device to go quiet so it can be safely removed/reset/etc
  * @param oct Pointer to Octeon device
  */
@@ -572,7 +560,7 @@ static inline void txqs_wake(struct net_device *netdev)
 
 		for (i = 0; i < netdev->num_tx_queues; i++) {
 			int qno = lio->linfo.txpciq[i %
-				(lio->linfo.num_txpciq)].s.q_no;
+				lio->oct_dev->num_iqs].s.q_no;
 
 			if (__netif_subqueue_stopped(netdev, i)) {
 				INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, qno,
@@ -652,7 +640,7 @@ static inline int check_txq_status(struct lio *lio)
 		/* check each sub-queue state */
 		for (q = 0; q < numqs; q++) {
 			iq = lio->linfo.txpciq[q %
-				(lio->linfo.num_txpciq)].s.q_no;
+				lio->oct_dev->num_iqs].s.q_no;
 			if (octnet_iq_is_full(lio->oct_dev, iq))
 				continue;
 			if (__netif_subqueue_stopped(lio->netdev, q)) {
@@ -823,7 +811,8 @@ static void print_link_info(struct net_device *netdev)
 {
 	struct lio *lio = GET_LIO(netdev);
 
-	if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED) {
+	if (!ifstate_check(lio, LIO_IFSTATE_RESETTING) &&
+	    ifstate_check(lio, LIO_IFSTATE_REGISTERED)) {
 		struct oct_link_info *linfo = &lio->linfo;
 
 		if (linfo->link.s.link_up) {
@@ -912,295 +901,6 @@ static inline void update_link_status(struct net_device *netdev,
 	}
 }
 
-/* Runs in interrupt context. */
-static void update_txq_status(struct octeon_device *oct, int iq_num)
-{
-	struct net_device *netdev;
-	struct lio *lio;
-	struct octeon_instr_queue *iq = oct->instr_queue[iq_num];
-
-	netdev = oct->props[iq->ifidx].netdev;
-
-	/* This is needed because the first IQ does not have
-	 * a netdev associated with it.
-	 */
-	if (!netdev)
-		return;
-
-	lio = GET_LIO(netdev);
-	if (netif_is_multiqueue(netdev)) {
-		if (__netif_subqueue_stopped(netdev, iq->q_index) &&
-		    lio->linfo.link.s.link_up &&
-		    (!octnet_iq_is_full(oct, iq_num))) {
-			INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq_num,
-						  tx_restart, 1);
-			netif_wake_subqueue(netdev, iq->q_index);
-		}
-	} else if (netif_queue_stopped(netdev) &&
-		   lio->linfo.link.s.link_up &&
-		   (!octnet_iq_is_full(oct, lio->txq))) {
-		INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev,
-					  lio->txq, tx_restart, 1);
-		netif_wake_queue(netdev);
-	}
-}
-
-static
-int liquidio_schedule_msix_droq_pkt_handler(struct octeon_droq *droq, u64 ret)
-{
-	struct octeon_device *oct = droq->oct_dev;
-	struct octeon_device_priv *oct_priv =
-	    (struct octeon_device_priv *)oct->priv;
-
-	if (droq->ops.poll_mode) {
-		droq->ops.napi_fn(droq);
-	} else {
-		if (ret & MSIX_PO_INT) {
-			tasklet_schedule(&oct_priv->droq_tasklet);
-			return 1;
-		}
-		/* this will be flushed periodically by check iq db */
-		if (ret & MSIX_PI_INT)
-			return 0;
-	}
-	return 0;
-}
-
-/**
- * \brief Droq packet processor sceduler
- * @param oct octeon device
- */
-static void liquidio_schedule_droq_pkt_handlers(struct octeon_device *oct)
-{
-	struct octeon_device_priv *oct_priv =
-		(struct octeon_device_priv *)oct->priv;
-	u64 oq_no;
-	struct octeon_droq *droq;
-
-	if (oct->int_status & OCT_DEV_INTR_PKT_DATA) {
-		for (oq_no = 0; oq_no < MAX_OCTEON_OUTPUT_QUEUES(oct);
-		     oq_no++) {
-			if (!(oct->droq_intr & BIT_ULL(oq_no)))
-				continue;
-
-			droq = oct->droq[oq_no];
-
-			if (droq->ops.poll_mode) {
-				droq->ops.napi_fn(droq);
-				oct_priv->napi_mask |= (1 << oq_no);
-			} else {
-				tasklet_schedule(&oct_priv->droq_tasklet);
-			}
-		}
-	}
-}
-
-static irqreturn_t
-liquidio_msix_intr_handler(int irq __attribute__((unused)), void *dev)
-{
-	u64 ret;
-	struct octeon_ioq_vector *ioq_vector = (struct octeon_ioq_vector *)dev;
-	struct octeon_device *oct = ioq_vector->oct_dev;
-	struct octeon_droq *droq = oct->droq[ioq_vector->droq_index];
-
-	ret = oct->fn_list.msix_interrupt_handler(ioq_vector);
-
-	if ((ret & MSIX_PO_INT) || (ret & MSIX_PI_INT))
-		liquidio_schedule_msix_droq_pkt_handler(droq, ret);
-
-	return IRQ_HANDLED;
-}
-
-/**
- * \brief Interrupt handler for octeon
- * @param irq unused
- * @param dev octeon device
- */
-static
-irqreturn_t liquidio_legacy_intr_handler(int irq __attribute__((unused)),
-					 void *dev)
-{
-	struct octeon_device *oct = (struct octeon_device *)dev;
-	irqreturn_t ret;
-
-	/* Disable our interrupts for the duration of ISR */
-	oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
-
-	ret = oct->fn_list.process_interrupt_regs(oct);
-
-	if (ret == IRQ_HANDLED)
-		liquidio_schedule_droq_pkt_handlers(oct);
-
-	/* Re-enable our interrupts  */
-	if (!(atomic_read(&oct->status) == OCT_DEV_IN_RESET))
-		oct->fn_list.enable_interrupt(oct, OCTEON_ALL_INTR);
-
-	return ret;
-}
-
-/**
- * \brief Setup interrupt for octeon device
- * @param oct octeon device
- *
- *  Enable interrupt in Octeon device as given in the PCI interrupt mask.
- */
-static int octeon_setup_interrupt(struct octeon_device *oct)
-{
-	int irqret, err;
-	struct msix_entry *msix_entries;
-	int i;
-	int num_ioq_vectors;
-	int num_alloc_ioq_vectors;
-	char *queue_irq_names = NULL;
-	char *aux_irq_name = NULL;
-
-	if (OCTEON_CN23XX_PF(oct) && oct->msix_on) {
-		oct->num_msix_irqs = oct->sriov_info.num_pf_rings;
-		/* one non ioq interrupt for handling sli_mac_pf_int_sum */
-		oct->num_msix_irqs += 1;
-
-		/* allocate storage for the names assigned to each irq */
-		oct->irq_name_storage =
-			kcalloc((MAX_IOQ_INTERRUPTS_PER_PF + 1), INTRNAMSIZ,
-				GFP_KERNEL);
-		if (!oct->irq_name_storage) {
-			dev_err(&oct->pci_dev->dev, "Irq name storage alloc failed...\n");
-			return -ENOMEM;
-		}
-
-		queue_irq_names = oct->irq_name_storage;
-		aux_irq_name = &queue_irq_names
-				[IRQ_NAME_OFF(MAX_IOQ_INTERRUPTS_PER_PF)];
-
-		oct->msix_entries = kcalloc(
-		    oct->num_msix_irqs, sizeof(struct msix_entry), GFP_KERNEL);
-		if (!oct->msix_entries) {
-			dev_err(&oct->pci_dev->dev, "Memory Alloc failed...\n");
-			kfree(oct->irq_name_storage);
-			oct->irq_name_storage = NULL;
-			return -ENOMEM;
-		}
-
-		msix_entries = (struct msix_entry *)oct->msix_entries;
-		/*Assumption is that pf msix vectors start from pf srn to pf to
-		 * trs and not from 0. if not change this code
-		 */
-		for (i = 0; i < oct->num_msix_irqs - 1; i++)
-			msix_entries[i].entry = oct->sriov_info.pf_srn + i;
-		msix_entries[oct->num_msix_irqs - 1].entry =
-		    oct->sriov_info.trs;
-		num_alloc_ioq_vectors = pci_enable_msix_range(
-						oct->pci_dev, msix_entries,
-						oct->num_msix_irqs,
-						oct->num_msix_irqs);
-		if (num_alloc_ioq_vectors < 0) {
-			dev_err(&oct->pci_dev->dev, "unable to Allocate MSI-X interrupts\n");
-			kfree(oct->msix_entries);
-			oct->msix_entries = NULL;
-			kfree(oct->irq_name_storage);
-			oct->irq_name_storage = NULL;
-			return num_alloc_ioq_vectors;
-		}
-		dev_dbg(&oct->pci_dev->dev, "OCTEON: Enough MSI-X interrupts are allocated...\n");
-
-		num_ioq_vectors = oct->num_msix_irqs;
-
-		/** For PF, there is one non-ioq interrupt handler */
-		num_ioq_vectors -= 1;
-
-		snprintf(aux_irq_name, INTRNAMSIZ,
-			 "LiquidIO%u-pf%u-aux", oct->octeon_id, oct->pf_num);
-		irqret = request_irq(msix_entries[num_ioq_vectors].vector,
-				     liquidio_legacy_intr_handler, 0,
-				     aux_irq_name, oct);
-		if (irqret) {
-			dev_err(&oct->pci_dev->dev,
-				"OCTEON: Request_irq failed for MSIX interrupt Error: %d\n",
-				irqret);
-			pci_disable_msix(oct->pci_dev);
-			kfree(oct->msix_entries);
-			oct->msix_entries = NULL;
-			kfree(oct->irq_name_storage);
-			oct->irq_name_storage = NULL;
-			return irqret;
-		}
-
-		for (i = 0; i < num_ioq_vectors; i++) {
-			snprintf(&queue_irq_names[IRQ_NAME_OFF(i)], INTRNAMSIZ,
-				 "LiquidIO%u-pf%u-rxtx-%u",
-				 oct->octeon_id, oct->pf_num, i);
-
-			irqret = request_irq(msix_entries[i].vector,
-					     liquidio_msix_intr_handler, 0,
-					     &queue_irq_names[IRQ_NAME_OFF(i)],
-					     &oct->ioq_vector[i]);
-			if (irqret) {
-				dev_err(&oct->pci_dev->dev,
-					"OCTEON: Request_irq failed for MSIX interrupt Error: %d\n",
-					irqret);
-				/** Freeing the non-ioq irq vector here . */
-				free_irq(msix_entries[num_ioq_vectors].vector,
-					 oct);
-
-				while (i) {
-					i--;
-					/** clearing affinity mask. */
-					irq_set_affinity_hint(
-						msix_entries[i].vector, NULL);
-					free_irq(msix_entries[i].vector,
-						 &oct->ioq_vector[i]);
-				}
-				pci_disable_msix(oct->pci_dev);
-				kfree(oct->msix_entries);
-				oct->msix_entries = NULL;
-				kfree(oct->irq_name_storage);
-				oct->irq_name_storage = NULL;
-				return irqret;
-			}
-			oct->ioq_vector[i].vector = msix_entries[i].vector;
-			/* assign the cpu mask for this msix interrupt vector */
-			irq_set_affinity_hint(
-					msix_entries[i].vector,
-					(&oct->ioq_vector[i].affinity_mask));
-		}
-		dev_dbg(&oct->pci_dev->dev, "OCTEON[%d]: MSI-X enabled\n",
-			oct->octeon_id);
-	} else {
-		err = pci_enable_msi(oct->pci_dev);
-		if (err)
-			dev_warn(&oct->pci_dev->dev, "Reverting to legacy interrupts. Error: %d\n",
-				 err);
-		else
-			oct->flags |= LIO_FLAG_MSI_ENABLED;
-
-		/* allocate storage for the names assigned to the irq */
-		oct->irq_name_storage = kcalloc(1, INTRNAMSIZ, GFP_KERNEL);
-		if (!oct->irq_name_storage)
-			return -ENOMEM;
-
-		queue_irq_names = oct->irq_name_storage;
-
-		snprintf(&queue_irq_names[IRQ_NAME_OFF(0)], INTRNAMSIZ,
-			 "LiquidIO%u-pf%u-rxtx-%u",
-			 oct->octeon_id, oct->pf_num, 0);
-
-		irqret = request_irq(oct->pci_dev->irq,
-				     liquidio_legacy_intr_handler,
-				     IRQF_SHARED,
-				     &queue_irq_names[IRQ_NAME_OFF(0)], oct);
-		if (irqret) {
-			if (oct->flags & LIO_FLAG_MSI_ENABLED)
-				pci_disable_msi(oct->pci_dev);
-			dev_err(&oct->pci_dev->dev, "Request IRQ failed with code: %d\n",
-				irqret);
-			kfree(oct->irq_name_storage);
-			oct->irq_name_storage = NULL;
-			return irqret;
-		}
-	}
-	return 0;
-}
-
 static struct octeon_device *get_other_octeon_device(struct octeon_device *oct)
 {
 	struct octeon_device *other_oct;
@@ -1344,6 +1044,13 @@ liquidio_probe(struct pci_dev *pdev,
 	if (pdev->device == OCTEON_CN23XX_PF_VID)
 		oct_dev->msix_on = LIO_FLAG_MSIX_ENABLED;
 
+	/* Enable PTP for 6XXX Device */
+	if (((pdev->device == OCTEON_CN66XX) ||
+	     (pdev->device == OCTEON_CN68XX)))
+		oct_dev->ptp_enable = true;
+	else
+		oct_dev->ptp_enable = false;
+
 	dev_info(&pdev->dev, "Initializing device %x:%x.\n",
 		 (u32)pdev->vendor, (u32)pdev->device);
 
@@ -1415,6 +1122,33 @@ static bool fw_type_is_none(void)
 }
 
 /**
+ * \brief PCI FLR for each Octeon device.
+ * @param oct octeon device
+ */
+static void octeon_pci_flr(struct octeon_device *oct)
+{
+	int rc;
+
+	pci_save_state(oct->pci_dev);
+
+	pci_cfg_access_lock(oct->pci_dev);
+
+	/* Quiesce the device completely */
+	pci_write_config_word(oct->pci_dev, PCI_COMMAND,
+			      PCI_COMMAND_INTX_DISABLE);
+
+	rc = __pci_reset_function_locked(oct->pci_dev);
+
+	if (rc != 0)
+		dev_err(&oct->pci_dev->dev, "Error %d resetting PCI function %d\n",
+			rc, oct->pf_num);
+
+	pci_cfg_access_unlock(oct->pci_dev);
+
+	pci_restore_state(oct->pci_dev);
+}
+
+/**
  *\brief Destroy resources associated with octeon device
  * @param pdev PCI device structure
  * @param ent unused
@@ -1474,11 +1208,15 @@ static void octeon_destroy_resources(struct octeon_device *oct)
 		if (oct->msix_on) {
 			msix_entries = (struct msix_entry *)oct->msix_entries;
 			for (i = 0; i < oct->num_msix_irqs - 1; i++) {
-				/* clear the affinity_cpumask */
-				irq_set_affinity_hint(msix_entries[i].vector,
-						      NULL);
-				free_irq(msix_entries[i].vector,
-					 &oct->ioq_vector[i]);
+				if (oct->ioq_vector[i].vector) {
+					/* clear the affinity_cpumask */
+					irq_set_affinity_hint(
+							msix_entries[i].vector,
+							NULL);
+					free_irq(msix_entries[i].vector,
+						 &oct->ioq_vector[i]);
+					oct->ioq_vector[i].vector = 0;
+				}
 			}
 			/* non-iov vector's argument is oct struct */
 			free_irq(msix_entries[i].vector, oct);
@@ -1558,14 +1296,16 @@ static void octeon_destroy_resources(struct octeon_device *oct)
 	case OCT_DEV_PCI_MAP_DONE:
 		refcount = octeon_deregister_device(oct);
 
-		if (!fw_type_is_none()) {
-			/* Soft reset the octeon device before exiting.
-			 * Implementation note: here, we reset the device
-			 * if it is a CN6XXX OR the last CN23XX device.
-			 */
-			if (OCTEON_CN6XXX(oct) || !refcount)
-				oct->fn_list.soft_reset(oct);
-		}
+		/* Soft reset the octeon device before exiting.
+		 * However, if fw was loaded from card (i.e. autoboot),
+		 * perform an FLR instead.
+		 * Implementation note: only soft-reset the device
+		 * if it is a CN6XXX OR the LAST CN23XX device.
+		 */
+		if (fw_type_is_none())
+			octeon_pci_flr(oct);
+		else if (OCTEON_CN6XXX(oct) || !refcount)
+			oct->fn_list.soft_reset(oct);
 
 		octeon_unmap_pci_barx(oct, 0);
 		octeon_unmap_pci_barx(oct, 1);
@@ -1698,15 +1438,6 @@ static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)
 	if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING)
 		liquidio_stop(netdev);
 
-	if (fw_type_is_none()) {
-		struct octnic_ctrl_pkt nctrl;
-
-		memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
-		nctrl.ncmd.s.cmd = OCTNET_CMD_RESET_PF;
-		nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
-		octnet_send_nic_ctrl_pkt(oct, &nctrl);
-	}
-
 	if (oct->props[lio->ifidx].napi_enabled == 1) {
 		list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
 			napi_disable(napi);
@@ -1717,6 +1448,10 @@ static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)
 			oct->droq[0]->ops.poll_mode = 0;
 	}
 
+	/* Delete NAPI */
+	list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
+		netif_napi_del(napi);
+
 	if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED)
 		unregister_netdev(netdev);
 
@@ -1754,7 +1489,7 @@ static int liquidio_stop_nic_module(struct octeon_device *oct)
 
 	for (i = 0; i < oct->ifcount; i++) {
 		lio = GET_LIO(oct->props[i].netdev);
-		for (j = 0; j < lio->linfo.num_rxpciq; j++)
+		for (j = 0; j < oct->num_oqs; j++)
 			octeon_unregister_droq_ops(oct,
 						   lio->linfo.rxpciq[j].s.q_no);
 	}
@@ -1825,6 +1560,13 @@ static int octeon_chip_specific_setup(struct octeon_device *oct)
 	case OCTEON_CN23XX_PCIID_PF:
 		oct->chip_id = OCTEON_CN23XX_PF_VID;
 		ret = setup_cn23xx_octeon_pf_device(oct);
+		if (ret)
+			break;
+#ifdef CONFIG_PCI_IOV
+		if (!ret)
+			pci_sriov_set_totalvfs(oct->pci_dev,
+					       oct->sriov_info.max_vfs);
+#endif
 		s = "CN23XX";
 		break;
 
@@ -1889,7 +1631,7 @@ static inline int check_txq_state(struct lio *lio, struct sk_buff *skb)
 
 	if (netif_is_multiqueue(lio->netdev)) {
 		q = skb->queue_mapping;
-		iq = lio->linfo.txpciq[(q % (lio->linfo.num_txpciq))].s.q_no;
+		iq = lio->linfo.txpciq[(q % lio->oct_dev->num_iqs)].s.q_no;
 	} else {
 		iq = lio->txq;
 		q = iq;
@@ -2192,11 +1934,6 @@ static int load_firmware(struct octeon_device *oct)
 	char fw_name[LIO_MAX_FW_FILENAME_LEN];
 	char *tmp_fw_type;
 
-	if (fw_type_is_none()) {
-		dev_info(&oct->pci_dev->dev, "Skipping firmware load\n");
-		return ret;
-	}
-
 	if (fw_type[0] == '\0')
 		tmp_fw_type = LIO_FW_NAME_TYPE_NIC;
 	else
@@ -2222,43 +1959,6 @@ static int load_firmware(struct octeon_device *oct)
 }
 
 /**
- * \brief Setup output queue
- * @param oct octeon device
- * @param q_no which queue
- * @param num_descs how many descriptors
- * @param desc_size size of each descriptor
- * @param app_ctx application context
- */
-static int octeon_setup_droq(struct octeon_device *oct, int q_no, int num_descs,
-			     int desc_size, void *app_ctx)
-{
-	int ret_val = 0;
-
-	dev_dbg(&oct->pci_dev->dev, "Creating Droq: %d\n", q_no);
-	/* droq creation and local register settings. */
-	ret_val = octeon_create_droq(oct, q_no, num_descs, desc_size, app_ctx);
-	if (ret_val < 0)
-		return ret_val;
-
-	if (ret_val == 1) {
-		dev_dbg(&oct->pci_dev->dev, "Using default droq %d\n", q_no);
-		return 0;
-	}
-	/* tasklet creation for the droq */
-
-	/* Enable the droq queues */
-	octeon_set_droq_pkt_op(oct, q_no, 1);
-
-	/* Send Credit for Octeon Output queues. Credits are always
-	 * sent after the output queue is enabled.
-	 */
-	writel(oct->droq[q_no]->max_count,
-	       oct->droq[q_no]->pkts_credit_reg);
-
-	return ret_val;
-}
-
-/**
  * \brief Callback for getting interface configuration
  * @param status status of request
  * @param buf pointer to resp structure
@@ -2291,352 +1991,6 @@ static void if_cfg_callback(struct octeon_device *oct,
 	wake_up_interruptible(&ctx->wc);
 }
 
-/** Routine to push packets arriving on Octeon interface upto network layer.
- * @param oct_id   - octeon device id.
- * @param skbuff   - skbuff struct to be passed to network layer.
- * @param len      - size of total data received.
- * @param rh       - Control header associated with the packet
- * @param param    - additional control data with the packet
- * @param arg	   - farg registered in droq_ops
- */
-static void
-liquidio_push_packet(u32 octeon_id __attribute__((unused)),
-		     void *skbuff,
-		     u32 len,
-		     union octeon_rh *rh,
-		     void *param,
-		     void *arg)
-{
-	struct napi_struct *napi = param;
-	struct sk_buff *skb = (struct sk_buff *)skbuff;
-	struct skb_shared_hwtstamps *shhwtstamps;
-	u64 ns;
-	u16 vtag = 0;
-	u32 r_dh_off;
-	struct net_device *netdev = (struct net_device *)arg;
-	struct octeon_droq *droq = container_of(param, struct octeon_droq,
-						napi);
-	if (netdev) {
-		int packet_was_received;
-		struct lio *lio = GET_LIO(netdev);
-		struct octeon_device *oct = lio->oct_dev;
-
-		/* Do not proceed if the interface is not in RUNNING state. */
-		if (!ifstate_check(lio, LIO_IFSTATE_RUNNING)) {
-			recv_buffer_free(skb);
-			droq->stats.rx_dropped++;
-			return;
-		}
-
-		skb->dev = netdev;
-
-		skb_record_rx_queue(skb, droq->q_no);
-		if (likely(len > MIN_SKB_SIZE)) {
-			struct octeon_skb_page_info *pg_info;
-			unsigned char *va;
-
-			pg_info = ((struct octeon_skb_page_info *)(skb->cb));
-			if (pg_info->page) {
-				/* For Paged allocation use the frags */
-				va = page_address(pg_info->page) +
-					pg_info->page_offset;
-				memcpy(skb->data, va, MIN_SKB_SIZE);
-				skb_put(skb, MIN_SKB_SIZE);
-				skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
-						pg_info->page,
-						pg_info->page_offset +
-						MIN_SKB_SIZE,
-						len - MIN_SKB_SIZE,
-						LIO_RXBUFFER_SZ);
-			}
-		} else {
-			struct octeon_skb_page_info *pg_info =
-				((struct octeon_skb_page_info *)(skb->cb));
-			skb_copy_to_linear_data(skb, page_address(pg_info->page)
-						+ pg_info->page_offset, len);
-			skb_put(skb, len);
-			put_page(pg_info->page);
-		}
-
-		r_dh_off = (rh->r_dh.len - 1) * BYTES_PER_DHLEN_UNIT;
-
-		if (((oct->chip_id == OCTEON_CN66XX) ||
-		     (oct->chip_id == OCTEON_CN68XX)) &&
-		    ptp_enable) {
-			if (rh->r_dh.has_hwtstamp) {
-				/* timestamp is included from the hardware at
-				 * the beginning of the packet.
-				 */
-				if (ifstate_check
-				    (lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED)) {
-					/* Nanoseconds are in the first 64-bits
-					 * of the packet.
-					 */
-					memcpy(&ns, (skb->data + r_dh_off),
-					       sizeof(ns));
-					r_dh_off -= BYTES_PER_DHLEN_UNIT;
-					shhwtstamps = skb_hwtstamps(skb);
-					shhwtstamps->hwtstamp =
-						ns_to_ktime(ns +
-							    lio->ptp_adjust);
-				}
-			}
-		}
-
-		if (rh->r_dh.has_hash) {
-			__be32 *hash_be = (__be32 *)(skb->data + r_dh_off);
-			u32 hash = be32_to_cpu(*hash_be);
-
-			skb_set_hash(skb, hash, PKT_HASH_TYPE_L4);
-			r_dh_off -= BYTES_PER_DHLEN_UNIT;
-		}
-
-		skb_pull(skb, rh->r_dh.len * BYTES_PER_DHLEN_UNIT);
-
-		skb->protocol = eth_type_trans(skb, skb->dev);
-		if ((netdev->features & NETIF_F_RXCSUM) &&
-		    (((rh->r_dh.encap_on) &&
-		      (rh->r_dh.csum_verified & CNNIC_TUN_CSUM_VERIFIED)) ||
-		     (!(rh->r_dh.encap_on) &&
-		      (rh->r_dh.csum_verified & CNNIC_CSUM_VERIFIED))))
-			/* checksum has already been verified */
-			skb->ip_summed = CHECKSUM_UNNECESSARY;
-		else
-			skb->ip_summed = CHECKSUM_NONE;
-
-		/* Setting Encapsulation field on basis of status received
-		 * from the firmware
-		 */
-		if (rh->r_dh.encap_on) {
-			skb->encapsulation = 1;
-			skb->csum_level = 1;
-			droq->stats.rx_vxlan++;
-		}
-
-		/* inbound VLAN tag */
-		if ((netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
-		    (rh->r_dh.vlan != 0)) {
-			u16 vid = rh->r_dh.vlan;
-			u16 priority = rh->r_dh.priority;
-
-			vtag = priority << 13 | vid;
-			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vtag);
-		}
-
-		packet_was_received = napi_gro_receive(napi, skb) != GRO_DROP;
-
-		if (packet_was_received) {
-			droq->stats.rx_bytes_received += len;
-			droq->stats.rx_pkts_received++;
-		} else {
-			droq->stats.rx_dropped++;
-			netif_info(lio, rx_err, lio->netdev,
-				   "droq:%d  error rx_dropped:%llu\n",
-				   droq->q_no, droq->stats.rx_dropped);
-		}
-
-	} else {
-		recv_buffer_free(skb);
-	}
-}
-
-/**
- * \brief wrapper for calling napi_schedule
- * @param param parameters to pass to napi_schedule
- *
- * Used when scheduling on different CPUs
- */
-static void napi_schedule_wrapper(void *param)
-{
-	struct napi_struct *napi = param;
-
-	napi_schedule(napi);
-}
-
-/**
- * \brief callback when receive interrupt occurs and we are in NAPI mode
- * @param arg pointer to octeon output queue
- */
-static void liquidio_napi_drv_callback(void *arg)
-{
-	struct octeon_device *oct;
-	struct octeon_droq *droq = arg;
-	int this_cpu = smp_processor_id();
-
-	oct = droq->oct_dev;
-
-	if (OCTEON_CN23XX_PF(oct) || droq->cpu_id == this_cpu) {
-		napi_schedule_irqoff(&droq->napi);
-	} else {
-		struct call_single_data *csd = &droq->csd;
-
-		csd->func = napi_schedule_wrapper;
-		csd->info = &droq->napi;
-		csd->flags = 0;
-
-		smp_call_function_single_async(droq->cpu_id, csd);
-	}
-}
-
-/**
- * \brief Entry point for NAPI polling
- * @param napi NAPI structure
- * @param budget maximum number of items to process
- */
-static int liquidio_napi_poll(struct napi_struct *napi, int budget)
-{
-	struct octeon_droq *droq;
-	int work_done;
-	int tx_done = 0, iq_no;
-	struct octeon_instr_queue *iq;
-	struct octeon_device *oct;
-
-	droq = container_of(napi, struct octeon_droq, napi);
-	oct = droq->oct_dev;
-	iq_no = droq->q_no;
-	/* Handle Droq descriptors */
-	work_done = octeon_process_droq_poll_cmd(oct, droq->q_no,
-						 POLL_EVENT_PROCESS_PKTS,
-						 budget);
-
-	/* Flush the instruction queue */
-	iq = oct->instr_queue[iq_no];
-	if (iq) {
-		if (atomic_read(&iq->instr_pending))
-			/* Process iq buffers with in the budget limits */
-			tx_done = octeon_flush_iq(oct, iq, budget);
-		else
-			tx_done = 1;
-		/* Update iq read-index rather than waiting for next interrupt.
-		 * Return back if tx_done is false.
-		 */
-		update_txq_status(oct, iq_no);
-	} else {
-		dev_err(&oct->pci_dev->dev, "%s:  iq (%d) num invalid\n",
-			__func__, iq_no);
-	}
-
-	/* force enable interrupt if reg cnts are high to avoid wraparound */
-	if ((work_done < budget && tx_done) ||
-	    (iq && iq->pkt_in_done >= MAX_REG_CNT) ||
-	    (droq->pkt_count >= MAX_REG_CNT)) {
-		tx_done = 1;
-		napi_complete_done(napi, work_done);
-		octeon_process_droq_poll_cmd(droq->oct_dev, droq->q_no,
-					     POLL_EVENT_ENABLE_INTR, 0);
-		return 0;
-	}
-
-	return (!tx_done) ? (budget) : (work_done);
-}
-
-/**
- * \brief Setup input and output queues
- * @param octeon_dev octeon device
- * @param ifidx  Interface Index
- *
- * Note: Queues are with respect to the octeon device. Thus
- * an input queue is for egress packets, and output queues
- * are for ingress packets.
- */
-static inline int setup_io_queues(struct octeon_device *octeon_dev,
-				  int ifidx)
-{
-	struct octeon_droq_ops droq_ops;
-	struct net_device *netdev;
-	static int cpu_id;
-	static int cpu_id_modulus;
-	struct octeon_droq *droq;
-	struct napi_struct *napi;
-	int q, q_no, retval = 0;
-	struct lio *lio;
-	int num_tx_descs;
-
-	netdev = octeon_dev->props[ifidx].netdev;
-
-	lio = GET_LIO(netdev);
-
-	memset(&droq_ops, 0, sizeof(struct octeon_droq_ops));
-
-	droq_ops.fptr = liquidio_push_packet;
-	droq_ops.farg = (void *)netdev;
-
-	droq_ops.poll_mode = 1;
-	droq_ops.napi_fn = liquidio_napi_drv_callback;
-	cpu_id = 0;
-	cpu_id_modulus = num_present_cpus();
-
-	/* set up DROQs. */
-	for (q = 0; q < lio->linfo.num_rxpciq; q++) {
-		q_no = lio->linfo.rxpciq[q].s.q_no;
-		dev_dbg(&octeon_dev->pci_dev->dev,
-			"setup_io_queues index:%d linfo.rxpciq.s.q_no:%d\n",
-			q, q_no);
-		retval = octeon_setup_droq(octeon_dev, q_no,
-					   CFG_GET_NUM_RX_DESCS_NIC_IF
-						   (octeon_get_conf(octeon_dev),
-						   lio->ifidx),
-					   CFG_GET_NUM_RX_BUF_SIZE_NIC_IF
-						   (octeon_get_conf(octeon_dev),
-						   lio->ifidx), NULL);
-		if (retval) {
-			dev_err(&octeon_dev->pci_dev->dev,
-				"%s : Runtime DROQ(RxQ) creation failed.\n",
-				__func__);
-			return 1;
-		}
-
-		droq = octeon_dev->droq[q_no];
-		napi = &droq->napi;
-		dev_dbg(&octeon_dev->pci_dev->dev, "netif_napi_add netdev:%llx oct:%llx pf_num:%d\n",
-			(u64)netdev, (u64)octeon_dev, octeon_dev->pf_num);
-		netif_napi_add(netdev, napi, liquidio_napi_poll, 64);
-
-		/* designate a CPU for this droq */
-		droq->cpu_id = cpu_id;
-		cpu_id++;
-		if (cpu_id >= cpu_id_modulus)
-			cpu_id = 0;
-
-		octeon_register_droq_ops(octeon_dev, q_no, &droq_ops);
-	}
-
-	if (OCTEON_CN23XX_PF(octeon_dev)) {
-		/* 23XX PF can receive control messages (via the first PF-owned
-		 * droq) from the firmware even if the ethX interface is down,
-		 * so that's why poll_mode must be off for the first droq.
-		 */
-		octeon_dev->droq[0]->ops.poll_mode = 0;
-	}
-
-	/* set up IQs. */
-	for (q = 0; q < lio->linfo.num_txpciq; q++) {
-		num_tx_descs = CFG_GET_NUM_TX_DESCS_NIC_IF(octeon_get_conf
-							   (octeon_dev),
-							   lio->ifidx);
-		retval = octeon_setup_iq(octeon_dev, ifidx, q,
-					 lio->linfo.txpciq[q], num_tx_descs,
-					 netdev_get_tx_queue(netdev, q));
-		if (retval) {
-			dev_err(&octeon_dev->pci_dev->dev,
-				" %s : Runtime IQ(TxQ) creation failed.\n",
-				__func__);
-			return 1;
-		}
-
-		if (octeon_dev->ioq_vector) {
-			struct octeon_ioq_vector *ioq_vector;
-
-			ioq_vector = &octeon_dev->ioq_vector[q];
-			netif_set_xps_queue(netdev,
-					    &ioq_vector->affinity_mask,
-					    ioq_vector->iq_index);
-		}
-	}
-
-	return 0;
-}
-
 /**
  * \brief Poll routine for checking transmit queue status
  * @param work work_struct data structure
@@ -2707,8 +2061,7 @@ static int liquidio_open(struct net_device *netdev)
 			oct->droq[0]->ops.poll_mode = 1;
 	}
 
-	if ((oct->chip_id == OCTEON_CN66XX || oct->chip_id == OCTEON_CN68XX) &&
-	    ptp_enable)
+	if (oct->ptp_enable)
 		oct_ptp_open(netdev);
 
 	ifstate_set(lio, LIO_IFSTATE_RUNNING);
@@ -2746,6 +2099,17 @@ static int liquidio_stop(struct net_device *netdev)
 {
 	struct lio *lio = GET_LIO(netdev);
 	struct octeon_device *oct = lio->oct_dev;
+	struct napi_struct *napi, *n;
+
+	if (oct->props[lio->ifidx].napi_enabled) {
+		list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
+			napi_disable(napi);
+
+		oct->props[lio->ifidx].napi_enabled = 0;
+
+		if (OCTEON_CN23XX_PF(oct))
+			oct->droq[0]->ops.poll_mode = 0;
+	}
 
 	ifstate_reset(lio, LIO_IFSTATE_RUNNING);
 
@@ -2916,7 +2280,10 @@ static struct net_device_stats *liquidio_get_stats(struct net_device *netdev)
 
 	oct = lio->oct_dev;
 
-	for (i = 0; i < lio->linfo.num_txpciq; i++) {
+	if (ifstate_check(lio, LIO_IFSTATE_RESETTING))
+		return stats;
+
+	for (i = 0; i < oct->num_iqs; i++) {
 		iq_no = lio->linfo.txpciq[i].s.q_no;
 		iq_stats = &oct->instr_queue[iq_no]->stats;
 		pkts += iq_stats->tx_done;
@@ -2932,7 +2299,7 @@ static struct net_device_stats *liquidio_get_stats(struct net_device *netdev)
 	drop = 0;
 	bytes = 0;
 
-	for (i = 0; i < lio->linfo.num_rxpciq; i++) {
+	for (i = 0; i < oct->num_oqs; i++) {
 		oq_no = lio->linfo.rxpciq[i].s.q_no;
 		oq_stats = &oct->droq[oq_no]->stats;
 		pkts += oq_stats->rx_pkts_received;
@@ -3052,8 +2419,7 @@ static int liquidio_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
 
 	switch (cmd) {
 	case SIOCSHWTSTAMP:
-		if ((lio->oct_dev->chip_id == OCTEON_CN66XX ||
-		     lio->oct_dev->chip_id == OCTEON_CN68XX) && ptp_enable)
+		if (lio->oct_dev->ptp_enable)
 			return hwtstamp_ioctl(netdev, ifr);
 	default:
 		return -EOPNOTSUPP;
@@ -4188,7 +3554,9 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
 		 */
 		lio->txq = lio->linfo.txpciq[0].s.q_no;
 		lio->rxq = lio->linfo.rxpciq[0].s.q_no;
-		if (setup_io_queues(octeon_dev, i)) {
+		if (liquidio_setup_io_queues(octeon_dev, i,
+					     lio->linfo.num_txpciq,
+					     lio->linfo.num_rxpciq)) {
 			dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n");
 			goto setup_nic_dev_fail;
 		}
@@ -4516,6 +3884,7 @@ static int octeon_device_init(struct octeon_device *octeon_dev)
 	int j, ret;
 	int fw_loaded = 0;
 	char bootcmd[] = "\n";
+	char *dbg_enb = NULL;
 	struct octeon_device_priv *oct_priv =
 		(struct octeon_device_priv *)octeon_dev->priv;
 	atomic_set(&octeon_dev->status, OCT_DEV_BEGIN_STATE);
@@ -4548,18 +3917,16 @@ static int octeon_device_init(struct octeon_device *octeon_dev)
 	octeon_dev->app_mode = CVM_DRV_INVALID_APP;
 
 	if (OCTEON_CN23XX_PF(octeon_dev)) {
-		if (!cn23xx_fw_loaded(octeon_dev)) {
+		if (!cn23xx_fw_loaded(octeon_dev) && !fw_type_is_none()) {
 			fw_loaded = 0;
-			if (!fw_type_is_none()) {
-				/* Do a soft reset of the Octeon device. */
-				if (octeon_dev->fn_list.soft_reset(octeon_dev))
-					return 1;
-				/* things might have changed */
-				if (!cn23xx_fw_loaded(octeon_dev))
-					fw_loaded = 0;
-				else
-					fw_loaded = 1;
-			}
+			/* Do a soft reset of the Octeon device. */
+			if (octeon_dev->fn_list.soft_reset(octeon_dev))
+				return 1;
+			/* things might have changed */
+			if (!cn23xx_fw_loaded(octeon_dev))
+				fw_loaded = 0;
+			else
+				fw_loaded = 1;
 		} else {
 			fw_loaded = 1;
 		}
@@ -4666,7 +4033,8 @@ static int octeon_device_init(struct octeon_device *octeon_dev)
 
 	/* Setup the interrupt handler and record the INT SUM register address
 	 */
-	if (octeon_setup_interrupt(octeon_dev))
+	if (octeon_setup_interrupt(octeon_dev,
+				   octeon_dev->sriov_info.num_pf_rings))
 		return 1;
 
 	/* Enable Octeon device interrupts */
@@ -4674,6 +4042,18 @@ static int octeon_device_init(struct octeon_device *octeon_dev)
 
 	atomic_set(&octeon_dev->status, OCT_DEV_INTR_SET_DONE);
 
+	/* Send Credit for Octeon Output queues. Credits are always sent BEFORE
+	 * the output queue is enabled.
+	 * This ensures that we'll receive the f/w CORE DRV_ACTIVE message in
+	 * case we've configured CN23XX_SLI_GBL_CONTROL[NOPTR_D] = 0.
+	 * Otherwise, it is possible that the DRV_ACTIVE message will be sent
+	 * before any credits have been issued, causing the ring to be reset
+	 * (and the f/w appear to never have started).
+	 */
+	for (j = 0; j < octeon_dev->num_oqs; j++)
+		writel(octeon_dev->droq[j]->max_count,
+		       octeon_dev->droq[j]->pkts_credit_reg);
+
 	/* Enable the input and output queues for this Octeon device */
 	ret = octeon_dev->fn_list.enable_io_queues(octeon_dev);
 	if (ret) {
@@ -4722,10 +4102,19 @@ static int octeon_device_init(struct octeon_device *octeon_dev)
 			dev_err(&octeon_dev->pci_dev->dev, "Could not access board consoles\n");
 			return 1;
 		}
-		ret = octeon_add_console(octeon_dev, 0);
+		/* If console debug enabled, specify empty string to use default
+		 * enablement ELSE specify NULL string for 'disabled'.
+		 */
+		dbg_enb = octeon_console_debug_enabled(0) ? "" : NULL;
+		ret = octeon_add_console(octeon_dev, 0, dbg_enb);
 		if (ret) {
 			dev_err(&octeon_dev->pci_dev->dev, "Could not access board console\n");
 			return 1;
+		} else if (octeon_console_debug_enabled(0)) {
+			/* If console was added AND we're logging console output
+			 * then set our console print function.
+			 */
+			octeon_dev->console[0].print = octeon_dbg_console_print;
 		}
 
 		atomic_set(&octeon_dev->status, OCT_DEV_CONSOLE_INIT_DONE);
@@ -4736,12 +4125,6 @@ static int octeon_device_init(struct octeon_device *octeon_dev)
 			dev_err(&octeon_dev->pci_dev->dev, "Could not load firmware to board\n");
 			return 1;
 		}
-		/* set bit 1 of SLI_SCRATCH_1 to indicate that firmware is
-		 * loaded
-		 */
-		if (OCTEON_CN23XX_PF(octeon_dev))
-			octeon_write_csr64(octeon_dev, CN23XX_SLI_SCRATCH1,
-					   2ULL);
 	}
 
 	handshake[octeon_dev->octeon_id].init_ok = 1;
@@ -4749,14 +4132,33 @@ static int octeon_device_init(struct octeon_device *octeon_dev)
 
 	atomic_set(&octeon_dev->status, OCT_DEV_HOST_OK);
 
-	/* Send Credit for Octeon Output queues. Credits are always sent after
-	 * the output queue is enabled.
-	 */
-	for (j = 0; j < octeon_dev->num_oqs; j++)
-		writel(octeon_dev->droq[j]->max_count,
-		       octeon_dev->droq[j]->pkts_credit_reg);
+	return 0;
+}
+
+/**
+ * \brief Debug console print function
+ * @param octeon_dev  octeon device
+ * @param console_num console number
+ * @param prefix      first portion of line to display
+ * @param suffix      second portion of line to display
+ *
+ * The OCTEON debug console outputs entire lines (excluding '\n').
+ * Normally, the line will be passed in the 'prefix' parameter.
+ * However, due to buffering, it is possible for a line to be split into two
+ * parts, in which case they will be passed as the 'prefix' parameter and
+ * 'suffix' parameter.
+ */
+static int octeon_dbg_console_print(struct octeon_device *oct, u32 console_num,
+				    char *prefix, char *suffix)
+{
+	if (prefix && suffix)
+		dev_info(&oct->pci_dev->dev, "%u: %s%s\n", console_num, prefix,
+			 suffix);
+	else if (prefix)
+		dev_info(&oct->pci_dev->dev, "%u: %s\n", console_num, prefix);
+	else if (suffix)
+		dev_info(&oct->pci_dev->dev, "%u: %s\n", console_num, suffix);
 
-	/* Packets can start arriving on the output queues from this point. */
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c b/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c
index 9b247102eb92..2e993ce43b66 100644
--- a/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c
+++ b/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c
@@ -107,12 +107,6 @@ struct octnic_gather {
 	dma_addr_t sg_dma_ptr;
 };
 
-struct octeon_device_priv {
-	/* Tasklet structures for this device. */
-	struct tasklet_struct droq_tasklet;
-	unsigned long napi_mask;
-};
-
 static int
 liquidio_vf_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
 static void liquidio_vf_remove(struct pci_dev *pdev);
@@ -123,7 +117,7 @@ static int lio_wait_for_oq_pkts(struct octeon_device *oct)
 {
 	struct octeon_device_priv *oct_priv =
 	    (struct octeon_device_priv *)oct->priv;
-	int retry = MAX_VF_IP_OP_PENDING_PKT_COUNT;
+	int retry = MAX_IO_PENDING_PKT_COUNT;
 	int pkt_cnt = 0, pending_pkts;
 	int i;
 
@@ -148,32 +142,6 @@ static int lio_wait_for_oq_pkts(struct octeon_device *oct)
 }
 
 /**
- * \brief wait for all pending requests to complete
- * @param oct Pointer to Octeon device
- *
- * Called during shutdown sequence
- */
-static int wait_for_pending_requests(struct octeon_device *oct)
-{
-	int i, pcount = 0;
-
-	for (i = 0; i < MAX_VF_IP_OP_PENDING_PKT_COUNT; i++) {
-		pcount = atomic_read(
-		    &oct->response_list[OCTEON_ORDERED_SC_LIST]
-			 .pending_req_count);
-		if (pcount)
-			schedule_timeout_uninterruptible(HZ / 10);
-		else
-			break;
-	}
-
-	if (pcount)
-		return 1;
-
-	return 0;
-}
-
-/**
  * \brief Cause device to go quiet so it can be safely removed/reset/etc
  * @param oct Pointer to Octeon device
  */
@@ -374,7 +342,7 @@ static void txqs_wake(struct net_device *netdev)
 		int i;
 
 		for (i = 0; i < netdev->num_tx_queues; i++) {
-			int qno = lio->linfo.txpciq[i % (lio->linfo.num_txpciq)]
+			int qno = lio->linfo.txpciq[i % lio->oct_dev->num_iqs]
 				      .s.q_no;
 			if (__netif_subqueue_stopped(netdev, i)) {
 				INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, qno,
@@ -574,7 +542,8 @@ static void print_link_info(struct net_device *netdev)
 {
 	struct lio *lio = GET_LIO(netdev);
 
-	if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED) {
+	if (!ifstate_check(lio, LIO_IFSTATE_RESETTING) &&
+	    ifstate_check(lio, LIO_IFSTATE_REGISTERED)) {
 		struct oct_link_info *linfo = &lio->linfo;
 
 		if (linfo->link.s.link_up) {
@@ -661,6 +630,12 @@ static void update_link_status(struct net_device *netdev,
 			txqs_stop(netdev);
 		}
 
+		if (lio->linfo.link.s.mtu != netdev->max_mtu) {
+			dev_info(&oct->pci_dev->dev, "Max MTU Changed from %d to %d\n",
+				 netdev->max_mtu, lio->linfo.link.s.mtu);
+			netdev->max_mtu = lio->linfo.link.s.mtu;
+		}
+
 		if (lio->linfo.link.s.mtu < netdev->mtu) {
 			dev_warn(&oct->pci_dev->dev,
 				 "PF has changed the MTU for gmx port. Reducing the mtu from %d to %d\n",
@@ -673,167 +648,6 @@ static void update_link_status(struct net_device *netdev,
 	}
 }
 
-static void update_txq_status(struct octeon_device *oct, int iq_num)
-{
-	struct octeon_instr_queue *iq = oct->instr_queue[iq_num];
-	struct net_device *netdev;
-	struct lio *lio;
-
-	netdev = oct->props[iq->ifidx].netdev;
-	lio = GET_LIO(netdev);
-	if (netif_is_multiqueue(netdev)) {
-		if (__netif_subqueue_stopped(netdev, iq->q_index) &&
-		    lio->linfo.link.s.link_up &&
-		    (!octnet_iq_is_full(oct, iq_num))) {
-			netif_wake_subqueue(netdev, iq->q_index);
-			INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq_num,
-						  tx_restart, 1);
-		}
-	} else if (netif_queue_stopped(netdev) && lio->linfo.link.s.link_up &&
-		   (!octnet_iq_is_full(oct, lio->txq))) {
-		INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev,
-					  lio->txq, tx_restart, 1);
-		netif_wake_queue(netdev);
-	}
-}
-
-static
-int liquidio_schedule_msix_droq_pkt_handler(struct octeon_droq *droq, u64 ret)
-{
-	struct octeon_device *oct = droq->oct_dev;
-	struct octeon_device_priv *oct_priv =
-	    (struct octeon_device_priv *)oct->priv;
-
-	if (droq->ops.poll_mode) {
-		droq->ops.napi_fn(droq);
-	} else {
-		if (ret & MSIX_PO_INT) {
-			dev_err(&oct->pci_dev->dev,
-				"should not come here should not get rx when poll mode = 0 for vf\n");
-			tasklet_schedule(&oct_priv->droq_tasklet);
-			return 1;
-		}
-		/* this will be flushed periodically by check iq db */
-		if (ret & MSIX_PI_INT)
-			return 0;
-	}
-	return 0;
-}
-
-static irqreturn_t
-liquidio_msix_intr_handler(int irq __attribute__((unused)), void *dev)
-{
-	struct octeon_ioq_vector *ioq_vector = (struct octeon_ioq_vector *)dev;
-	struct octeon_device *oct = ioq_vector->oct_dev;
-	struct octeon_droq *droq = oct->droq[ioq_vector->droq_index];
-	u64 ret;
-
-	ret = oct->fn_list.msix_interrupt_handler(ioq_vector);
-
-	if ((ret & MSIX_PO_INT) || (ret & MSIX_PI_INT))
-		liquidio_schedule_msix_droq_pkt_handler(droq, ret);
-
-	return IRQ_HANDLED;
-}
-
-/**
- * \brief Setup interrupt for octeon device
- * @param oct octeon device
- *
- *  Enable interrupt in Octeon device as given in the PCI interrupt mask.
- */
-static int octeon_setup_interrupt(struct octeon_device *oct)
-{
-	struct msix_entry *msix_entries;
-	char *queue_irq_names = NULL;
-	int num_alloc_ioq_vectors;
-	int num_ioq_vectors;
-	int irqret;
-	int i;
-
-	if (oct->msix_on) {
-		oct->num_msix_irqs = oct->sriov_info.rings_per_vf;
-
-		/* allocate storage for the names assigned to each irq */
-		oct->irq_name_storage =
-			kcalloc(MAX_IOQ_INTERRUPTS_PER_VF, INTRNAMSIZ,
-				GFP_KERNEL);
-		if (!oct->irq_name_storage) {
-			dev_err(&oct->pci_dev->dev, "Irq name storage alloc failed...\n");
-			return -ENOMEM;
-		}
-
-		queue_irq_names = oct->irq_name_storage;
-
-		oct->msix_entries = kcalloc(
-		    oct->num_msix_irqs, sizeof(struct msix_entry), GFP_KERNEL);
-		if (!oct->msix_entries) {
-			dev_err(&oct->pci_dev->dev, "Memory Alloc failed...\n");
-			kfree(oct->irq_name_storage);
-			oct->irq_name_storage = NULL;
-			return -ENOMEM;
-		}
-
-		msix_entries = (struct msix_entry *)oct->msix_entries;
-
-		for (i = 0; i < oct->num_msix_irqs; i++)
-			msix_entries[i].entry = i;
-		num_alloc_ioq_vectors = pci_enable_msix_range(
-						oct->pci_dev, msix_entries,
-						oct->num_msix_irqs,
-						oct->num_msix_irqs);
-		if (num_alloc_ioq_vectors < 0) {
-			dev_err(&oct->pci_dev->dev, "unable to Allocate MSI-X interrupts\n");
-			kfree(oct->msix_entries);
-			oct->msix_entries = NULL;
-			kfree(oct->irq_name_storage);
-			oct->irq_name_storage = NULL;
-			return num_alloc_ioq_vectors;
-		}
-		dev_dbg(&oct->pci_dev->dev, "OCTEON: Enough MSI-X interrupts are allocated...\n");
-
-		num_ioq_vectors = oct->num_msix_irqs;
-
-		for (i = 0; i < num_ioq_vectors; i++) {
-			snprintf(&queue_irq_names[IRQ_NAME_OFF(i)], INTRNAMSIZ,
-				 "LiquidIO%u-vf%u-rxtx-%u",
-				 oct->octeon_id, oct->vf_num, i);
-
-			irqret = request_irq(msix_entries[i].vector,
-					     liquidio_msix_intr_handler, 0,
-					     &queue_irq_names[IRQ_NAME_OFF(i)],
-					     &oct->ioq_vector[i]);
-			if (irqret) {
-				dev_err(&oct->pci_dev->dev,
-					"OCTEON: Request_irq failed for MSIX interrupt Error: %d\n",
-					irqret);
-
-				while (i) {
-					i--;
-					irq_set_affinity_hint(
-					    msix_entries[i].vector, NULL);
-					free_irq(msix_entries[i].vector,
-						 &oct->ioq_vector[i]);
-				}
-				pci_disable_msix(oct->pci_dev);
-				kfree(oct->msix_entries);
-				oct->msix_entries = NULL;
-				kfree(oct->irq_name_storage);
-				oct->irq_name_storage = NULL;
-				return irqret;
-			}
-			oct->ioq_vector[i].vector = msix_entries[i].vector;
-			/* assign the cpu mask for this msix interrupt vector */
-			irq_set_affinity_hint(
-			    msix_entries[i].vector,
-			    (&oct->ioq_vector[i].affinity_mask));
-		}
-		dev_dbg(&oct->pci_dev->dev,
-			"OCTEON[%d]: MSI-X enabled\n", oct->octeon_id);
-	}
-	return 0;
-}
-
 /**
  * \brief PCI probe handler
  * @param pdev PCI device structure
@@ -942,10 +756,14 @@ static void octeon_destroy_resources(struct octeon_device *oct)
 		if (oct->msix_on) {
 			msix_entries = (struct msix_entry *)oct->msix_entries;
 			for (i = 0; i < oct->num_msix_irqs; i++) {
-				irq_set_affinity_hint(msix_entries[i].vector,
-						      NULL);
-				free_irq(msix_entries[i].vector,
-					 &oct->ioq_vector[i]);
+				if (oct->ioq_vector[i].vector) {
+					irq_set_affinity_hint(
+							msix_entries[i].vector,
+							NULL);
+					free_irq(msix_entries[i].vector,
+						 &oct->ioq_vector[i]);
+					oct->ioq_vector[i].vector = 0;
+				}
 			}
 			pci_disable_msix(oct->pci_dev);
 			kfree(oct->msix_entries);
@@ -1137,6 +955,10 @@ static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)
 		oct->droq[0]->ops.poll_mode = 0;
 	}
 
+	/* Delete NAPI */
+	list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
+		netif_napi_del(napi);
+
 	if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED)
 		unregister_netdev(netdev);
 
@@ -1174,7 +996,7 @@ static int liquidio_stop_nic_module(struct octeon_device *oct)
 
 	for (i = 0; i < oct->ifcount; i++) {
 		lio = GET_LIO(oct->props[i].netdev);
-		for (j = 0; j < lio->linfo.num_rxpciq; j++)
+		for (j = 0; j < oct->num_oqs; j++)
 			octeon_unregister_droq_ops(oct,
 						   lio->linfo.rxpciq[j].s.q_no);
 	}
@@ -1262,7 +1084,7 @@ static int check_txq_state(struct lio *lio, struct sk_buff *skb)
 
 	if (netif_is_multiqueue(lio->netdev)) {
 		q = skb->queue_mapping;
-		iq = lio->linfo.txpciq[(q % (lio->linfo.num_txpciq))].s.q_no;
+		iq = lio->linfo.txpciq[q % lio->oct_dev->num_iqs].s.q_no;
 	} else {
 		iq = lio->txq;
 		q = iq;
@@ -1391,41 +1213,6 @@ static void free_netsgbuf_with_resp(void *buf)
 }
 
 /**
- * \brief Setup output queue
- * @param oct octeon device
- * @param q_no which queue
- * @param num_descs how many descriptors
- * @param desc_size size of each descriptor
- * @param app_ctx application context
- */
-static int octeon_setup_droq(struct octeon_device *oct, int q_no, int num_descs,
-			     int desc_size, void *app_ctx)
-{
-	int ret_val;
-
-	dev_dbg(&oct->pci_dev->dev, "Creating Droq: %d\n", q_no);
-	/* droq creation and local register settings. */
-	ret_val = octeon_create_droq(oct, q_no, num_descs, desc_size, app_ctx);
-	if (ret_val < 0)
-		return ret_val;
-
-	if (ret_val == 1) {
-		dev_dbg(&oct->pci_dev->dev, "Using default droq %d\n", q_no);
-		return 0;
-	}
-
-	/* Enable the droq queues */
-	octeon_set_droq_pkt_op(oct, q_no, 1);
-
-	/* Send Credit for Octeon Output queues. Credits are always
-	 * sent after the output queue is enabled.
-	 */
-	writel(oct->droq[q_no]->max_count, oct->droq[q_no]->pkts_credit_reg);
-
-	return ret_val;
-}
-
-/**
  * \brief Callback for getting interface configuration
  * @param status status of request
  * @param buf pointer to resp structure
@@ -1457,290 +1244,6 @@ static void if_cfg_callback(struct octeon_device *oct,
 	wake_up_interruptible(&ctx->wc);
 }
 
-/** Routine to push packets arriving on Octeon interface upto network layer.
- * @param oct_id   - octeon device id.
- * @param skbuff   - skbuff struct to be passed to network layer.
- * @param len      - size of total data received.
- * @param rh       - Control header associated with the packet
- * @param param    - additional control data with the packet
- * @param arg      - farg registered in droq_ops
- */
-static void
-liquidio_push_packet(u32 octeon_id __attribute__((unused)),
-		     void *skbuff,
-		     u32 len,
-		     union octeon_rh *rh,
-		     void *param,
-		     void *arg)
-{
-	struct napi_struct *napi = param;
-	struct octeon_droq *droq =
-		container_of(param, struct octeon_droq, napi);
-	struct net_device *netdev = (struct net_device *)arg;
-	struct sk_buff *skb = (struct sk_buff *)skbuff;
-	u16 vtag = 0;
-	u32 r_dh_off;
-
-	if (netdev) {
-		struct lio *lio = GET_LIO(netdev);
-		int packet_was_received;
-
-		/* Do not proceed if the interface is not in RUNNING state. */
-		if (!ifstate_check(lio, LIO_IFSTATE_RUNNING)) {
-			recv_buffer_free(skb);
-			droq->stats.rx_dropped++;
-			return;
-		}
-
-		skb->dev = netdev;
-
-		skb_record_rx_queue(skb, droq->q_no);
-		if (likely(len > MIN_SKB_SIZE)) {
-			struct octeon_skb_page_info *pg_info;
-			unsigned char *va;
-
-			pg_info = ((struct octeon_skb_page_info *)(skb->cb));
-			if (pg_info->page) {
-				/* For Paged allocation use the frags */
-				va = page_address(pg_info->page) +
-					pg_info->page_offset;
-				memcpy(skb->data, va, MIN_SKB_SIZE);
-				skb_put(skb, MIN_SKB_SIZE);
-				skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
-						pg_info->page,
-						pg_info->page_offset +
-						MIN_SKB_SIZE,
-						len - MIN_SKB_SIZE,
-						LIO_RXBUFFER_SZ);
-			}
-		} else {
-			struct octeon_skb_page_info *pg_info =
-				((struct octeon_skb_page_info *)(skb->cb));
-			skb_copy_to_linear_data(skb,
-						page_address(pg_info->page) +
-						pg_info->page_offset, len);
-			skb_put(skb, len);
-			put_page(pg_info->page);
-		}
-
-		r_dh_off = (rh->r_dh.len - 1) * BYTES_PER_DHLEN_UNIT;
-
-		if (rh->r_dh.has_hwtstamp)
-			r_dh_off -= BYTES_PER_DHLEN_UNIT;
-
-		if (rh->r_dh.has_hash) {
-			__be32 *hash_be = (__be32 *)(skb->data + r_dh_off);
-			u32 hash = be32_to_cpu(*hash_be);
-
-			skb_set_hash(skb, hash, PKT_HASH_TYPE_L4);
-			r_dh_off -= BYTES_PER_DHLEN_UNIT;
-		}
-
-		skb_pull(skb, rh->r_dh.len * BYTES_PER_DHLEN_UNIT);
-		skb->protocol = eth_type_trans(skb, skb->dev);
-
-		if ((netdev->features & NETIF_F_RXCSUM) &&
-		    (((rh->r_dh.encap_on) &&
-		      (rh->r_dh.csum_verified & CNNIC_TUN_CSUM_VERIFIED)) ||
-		     (!(rh->r_dh.encap_on) &&
-		      (rh->r_dh.csum_verified & CNNIC_CSUM_VERIFIED))))
-			/* checksum has already been verified */
-			skb->ip_summed = CHECKSUM_UNNECESSARY;
-		else
-			skb->ip_summed = CHECKSUM_NONE;
-
-		/* Setting Encapsulation field on basis of status received
-		 * from the firmware
-		 */
-		if (rh->r_dh.encap_on) {
-			skb->encapsulation = 1;
-			skb->csum_level = 1;
-			droq->stats.rx_vxlan++;
-		}
-
-		/* inbound VLAN tag */
-		if ((netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
-		    rh->r_dh.vlan) {
-			u16 priority = rh->r_dh.priority;
-			u16 vid = rh->r_dh.vlan;
-
-			vtag = (priority << VLAN_PRIO_SHIFT) | vid;
-			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vtag);
-		}
-
-		packet_was_received = (napi_gro_receive(napi, skb) != GRO_DROP);
-
-		if (packet_was_received) {
-			droq->stats.rx_bytes_received += len;
-			droq->stats.rx_pkts_received++;
-		} else {
-			droq->stats.rx_dropped++;
-			netif_info(lio, rx_err, lio->netdev,
-				   "droq:%d  error rx_dropped:%llu\n",
-				   droq->q_no, droq->stats.rx_dropped);
-		}
-
-	} else {
-		recv_buffer_free(skb);
-	}
-}
-
-/**
- * \brief callback when receive interrupt occurs and we are in NAPI mode
- * @param arg pointer to octeon output queue
- */
-static void liquidio_vf_napi_drv_callback(void *arg)
-{
-	struct octeon_droq *droq = arg;
-
-	napi_schedule_irqoff(&droq->napi);
-}
-
-/**
- * \brief Entry point for NAPI polling
- * @param napi NAPI structure
- * @param budget maximum number of items to process
- */
-static int liquidio_napi_poll(struct napi_struct *napi, int budget)
-{
-	struct octeon_instr_queue *iq;
-	struct octeon_device *oct;
-	struct octeon_droq *droq;
-	int tx_done = 0, iq_no;
-	int work_done;
-
-	droq = container_of(napi, struct octeon_droq, napi);
-	oct = droq->oct_dev;
-	iq_no = droq->q_no;
-
-	/* Handle Droq descriptors */
-	work_done = octeon_process_droq_poll_cmd(oct, droq->q_no,
-						 POLL_EVENT_PROCESS_PKTS,
-						 budget);
-
-	/* Flush the instruction queue */
-	iq = oct->instr_queue[iq_no];
-	if (iq) {
-		if (atomic_read(&iq->instr_pending))
-			/* Process iq buffers with in the budget limits */
-			tx_done = octeon_flush_iq(oct, iq, budget);
-		else
-			tx_done = 1;
-
-		/* Update iq read-index rather than waiting for next interrupt.
-		 * Return back if tx_done is false.
-		 */
-		update_txq_status(oct, iq_no);
-	} else {
-		dev_err(&oct->pci_dev->dev, "%s: iq (%d) num invalid\n",
-			__func__, iq_no);
-	}
-
-	/* force enable interrupt if reg cnts are high to avoid wraparound */
-	if ((work_done < budget && tx_done) ||
-	    (iq && iq->pkt_in_done >= MAX_REG_CNT) ||
-	    (droq->pkt_count >= MAX_REG_CNT)) {
-		tx_done = 1;
-		napi_complete_done(napi, work_done);
-		octeon_process_droq_poll_cmd(droq->oct_dev, droq->q_no,
-					     POLL_EVENT_ENABLE_INTR, 0);
-		return 0;
-	}
-
-	return (!tx_done) ? (budget) : (work_done);
-}
-
-/**
- * \brief Setup input and output queues
- * @param octeon_dev octeon device
- * @param ifidx Interface index
- *
- * Note: Queues are with respect to the octeon device. Thus
- * an input queue is for egress packets, and output queues
- * are for ingress packets.
- */
-static int setup_io_queues(struct octeon_device *octeon_dev, int ifidx)
-{
-	struct octeon_droq_ops droq_ops;
-	struct net_device *netdev;
-	static int cpu_id_modulus;
-	struct octeon_droq *droq;
-	struct napi_struct *napi;
-	static int cpu_id;
-	int num_tx_descs;
-	struct lio *lio;
-	int retval = 0;
-	int q, q_no;
-
-	netdev = octeon_dev->props[ifidx].netdev;
-
-	lio = GET_LIO(netdev);
-
-	memset(&droq_ops, 0, sizeof(struct octeon_droq_ops));
-
-	droq_ops.fptr = liquidio_push_packet;
-	droq_ops.farg = netdev;
-
-	droq_ops.poll_mode = 1;
-	droq_ops.napi_fn = liquidio_vf_napi_drv_callback;
-	cpu_id = 0;
-	cpu_id_modulus = num_present_cpus();
-
-	/* set up DROQs. */
-	for (q = 0; q < lio->linfo.num_rxpciq; q++) {
-		q_no = lio->linfo.rxpciq[q].s.q_no;
-
-		retval = octeon_setup_droq(
-		    octeon_dev, q_no,
-		    CFG_GET_NUM_RX_DESCS_NIC_IF(octeon_get_conf(octeon_dev),
-						lio->ifidx),
-		    CFG_GET_NUM_RX_BUF_SIZE_NIC_IF(octeon_get_conf(octeon_dev),
-						   lio->ifidx),
-		    NULL);
-		if (retval) {
-			dev_err(&octeon_dev->pci_dev->dev,
-				"%s : Runtime DROQ(RxQ) creation failed.\n",
-				__func__);
-			return 1;
-		}
-
-		droq = octeon_dev->droq[q_no];
-		napi = &droq->napi;
-		netif_napi_add(netdev, napi, liquidio_napi_poll, 64);
-
-		/* designate a CPU for this droq */
-		droq->cpu_id = cpu_id;
-		cpu_id++;
-		if (cpu_id >= cpu_id_modulus)
-			cpu_id = 0;
-
-		octeon_register_droq_ops(octeon_dev, q_no, &droq_ops);
-	}
-
-	/* 23XX VF can send/recv control messages (via the first VF-owned
-	 * droq) from the firmware even if the ethX interface is down,
-	 * so that's why poll_mode must be off for the first droq.
-	 */
-	octeon_dev->droq[0]->ops.poll_mode = 0;
-
-	/* set up IQs. */
-	for (q = 0; q < lio->linfo.num_txpciq; q++) {
-		num_tx_descs = CFG_GET_NUM_TX_DESCS_NIC_IF(
-		    octeon_get_conf(octeon_dev), lio->ifidx);
-		retval = octeon_setup_iq(octeon_dev, ifidx, q,
-					 lio->linfo.txpciq[q], num_tx_descs,
-					 netdev_get_tx_queue(netdev, q));
-		if (retval) {
-			dev_err(&octeon_dev->pci_dev->dev,
-				" %s : Runtime IQ(TxQ) creation failed.\n",
-				__func__);
-			return 1;
-		}
-	}
-
-	return 0;
-}
-
 /**
  * \brief Net device open for LiquidIO
  * @param netdev network device
@@ -1784,6 +1287,16 @@ static int liquidio_stop(struct net_device *netdev)
 {
 	struct lio *lio = GET_LIO(netdev);
 	struct octeon_device *oct = lio->oct_dev;
+	struct napi_struct *napi, *n;
+
+	if (oct->props[lio->ifidx].napi_enabled) {
+		list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
+			napi_disable(napi);
+
+		oct->props[lio->ifidx].napi_enabled = 0;
+
+		oct->droq[0]->ops.poll_mode = 0;
+	}
 
 	netif_info(lio, ifdown, lio->netdev, "Stopping interface!\n");
 	/* Inform that netif carrier is down */
@@ -1988,7 +1501,10 @@ static struct net_device_stats *liquidio_get_stats(struct net_device *netdev)
 
 	oct = lio->oct_dev;
 
-	for (i = 0; i < lio->linfo.num_txpciq; i++) {
+	if (ifstate_check(lio, LIO_IFSTATE_RESETTING))
+		return stats;
+
+	for (i = 0; i < oct->num_iqs; i++) {
 		iq_no = lio->linfo.txpciq[i].s.q_no;
 		iq_stats = &oct->instr_queue[iq_no]->stats;
 		pkts += iq_stats->tx_done;
@@ -2004,7 +1520,7 @@ static struct net_device_stats *liquidio_get_stats(struct net_device *netdev)
 	drop = 0;
 	bytes = 0;
 
-	for (i = 0; i < lio->linfo.num_rxpciq; i++) {
+	for (i = 0; i < oct->num_oqs; i++) {
 		oq_no = lio->linfo.rxpciq[i].s.q_no;
 		oq_stats = &oct->droq[oq_no]->stats;
 		pkts += oq_stats->rx_pkts_received;
@@ -2028,17 +1544,31 @@ static struct net_device_stats *liquidio_get_stats(struct net_device *netdev)
  */
 static int liquidio_change_mtu(struct net_device *netdev, int new_mtu)
 {
-	struct lio *lio = GET_LIO(netdev);
-	struct octeon_device *oct = lio->oct_dev;
+	struct octnic_ctrl_pkt nctrl;
+	struct octeon_device *oct;
+	struct lio *lio;
+	int ret = 0;
 
-	lio->mtu = new_mtu;
+	lio = GET_LIO(netdev);
+	oct = lio->oct_dev;
+
+	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
 
-	netif_info(lio, probe, lio->netdev, "MTU Changed from %d to %d\n",
-		   netdev->mtu, new_mtu);
-	dev_info(&oct->pci_dev->dev, "%s MTU Changed from %d to %d\n",
-		 netdev->name, netdev->mtu, new_mtu);
+	nctrl.ncmd.u64 = 0;
+	nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MTU;
+	nctrl.ncmd.s.param1 = new_mtu;
+	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
+	nctrl.wait_time = LIO_CMD_WAIT_TM;
+	nctrl.netpndev = (u64)netdev;
+	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
 
-	netdev->mtu = new_mtu;
+	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
+	if (ret < 0) {
+		dev_err(&oct->pci_dev->dev, "Failed to set MTU\n");
+		return -EIO;
+	}
+
+	lio->mtu = new_mtu;
 
 	return 0;
 }
@@ -2959,7 +2489,9 @@ static int setup_nic_devices(struct octeon_device *octeon_dev)
 		/* Copy MAC Address to OS network device structure */
 		ether_addr_copy(netdev->dev_addr, mac);
 
-		if (setup_io_queues(octeon_dev, i)) {
+		if (liquidio_setup_io_queues(octeon_dev, i,
+					     lio->linfo.num_txpciq,
+					     lio->linfo.num_rxpciq)) {
 			dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n");
 			goto setup_nic_dev_fail;
 		}
@@ -3182,7 +2714,7 @@ static int octeon_device_init(struct octeon_device *oct)
 		 LIQUIDIO_VERSION, oct->sriov_info.rings_per_vf);
 
 	/* Setup the interrupt handler and record the INT SUM register address*/
-	if (octeon_setup_interrupt(oct))
+	if (octeon_setup_interrupt(oct, oct->sriov_info.rings_per_vf))
 		return 1;
 
 	atomic_set(&oct->status, OCT_DEV_INTR_SET_DONE);
diff --git a/drivers/net/ethernet/cavium/liquidio/liquidio_common.h b/drivers/net/ethernet/cavium/liquidio/liquidio_common.h
index 231dd7fbfb80..3788c8cd082a 100644
--- a/drivers/net/ethernet/cavium/liquidio/liquidio_common.h
+++ b/drivers/net/ethernet/cavium/liquidio/liquidio_common.h
@@ -27,7 +27,7 @@
 
 #define LIQUIDIO_PACKAGE ""
 #define LIQUIDIO_BASE_MAJOR_VERSION 1
-#define LIQUIDIO_BASE_MINOR_VERSION 5
+#define LIQUIDIO_BASE_MINOR_VERSION 6
 #define LIQUIDIO_BASE_MICRO_VERSION 1
 #define LIQUIDIO_BASE_VERSION   __stringify(LIQUIDIO_BASE_MAJOR_VERSION) "." \
 				__stringify(LIQUIDIO_BASE_MINOR_VERSION)
@@ -106,6 +106,7 @@ enum octeon_tag_type {
 #define MAX_IOQ_INTERRUPTS_PER_PF   (64 * 2)
 #define MAX_IOQ_INTERRUPTS_PER_VF   (8 * 2)
 
+#define SCR2_BIT_FW_LOADED	    63
 
 static inline u32 incr_index(u32 index, u32 count, u32 max)
 {
@@ -189,7 +190,6 @@ static inline void add_sg_size(struct octeon_sg_entry *sg_entry,
 #define   OCTNET_CMD_Q                0
 
 /* NIC Command types */
-#define   OCTNET_CMD_RESET_PF         0x0
 #define   OCTNET_CMD_CHANGE_MTU       0x1
 #define   OCTNET_CMD_CHANGE_MACADDR   0x2
 #define   OCTNET_CMD_CHANGE_DEVFLAGS  0x3
@@ -226,6 +226,9 @@ static inline void add_sg_size(struct octeon_sg_entry *sg_entry,
 
 #define   OCTNET_CMD_SET_UC_LIST       0x1b
 #define   OCTNET_CMD_SET_VF_LINKSTATE  0x1c
+
+#define   OCTNET_CMD_QUEUE_COUNT_CTL	0x1f
+
 #define   OCTNET_CMD_VXLAN_PORT_ADD    0x0
 #define   OCTNET_CMD_VXLAN_PORT_DEL    0x1
 #define   OCTNET_CMD_RXCSUM_ENABLE     0x0
@@ -235,6 +238,8 @@ static inline void add_sg_size(struct octeon_sg_entry *sg_entry,
 #define   OCTNET_CMD_VLAN_FILTER_ENABLE 0x1
 #define   OCTNET_CMD_VLAN_FILTER_DISABLE 0x0
 
+#define   LIO_CMD_WAIT_TM 100
+
 /* RX(packets coming from wire) Checksum verification flags */
 /* TCP/UDP csum */
 #define   CNNIC_L4SUM_VERIFIED             0x1
@@ -768,6 +773,7 @@ struct nic_rx_stats {
 	/* firmware stats */
 	u64 fw_total_rcvd;
 	u64 fw_total_fwd;
+	u64 fw_total_fwd_bytes;
 	u64 fw_err_pko;
 	u64 fw_err_link;
 	u64 fw_err_drop;
@@ -814,6 +820,7 @@ struct nic_tx_stats {
 	u64 fw_tso;		/* number of tso requests */
 	u64 fw_tso_fwd;		/* number of packets segmented in tso */
 	u64 fw_tx_vxlan;
+	u64 fw_err_pki;
 };
 
 struct oct_link_stats {
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_config.h b/drivers/net/ethernet/cavium/liquidio/octeon_config.h
index f229d792c2b3..63bd9c94e547 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_config.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_config.h
@@ -71,13 +71,17 @@
 #define   CN23XX_MAX_RINGS_PER_VF          8
 
 #define   CN23XX_MAX_INPUT_QUEUES	CN23XX_MAX_RINGS_PER_PF
-#define   CN23XX_MAX_IQ_DESCRIPTORS	512
+#define   CN23XX_MAX_IQ_DESCRIPTORS	2048
+#define   CN23XX_DEFAULT_IQ_DESCRIPTORS	512
+#define   CN23XX_MIN_IQ_DESCRIPTORS	128
 #define   CN23XX_DB_MIN                 1
 #define   CN23XX_DB_MAX                 8
 #define   CN23XX_DB_TIMEOUT             1
 
 #define   CN23XX_MAX_OUTPUT_QUEUES	CN23XX_MAX_RINGS_PER_PF
-#define   CN23XX_MAX_OQ_DESCRIPTORS	512
+#define   CN23XX_MAX_OQ_DESCRIPTORS	2048
+#define   CN23XX_DEFAULT_OQ_DESCRIPTORS	512
+#define   CN23XX_MIN_OQ_DESCRIPTORS	128
 #define   CN23XX_OQ_BUF_SIZE		1664
 #define   CN23XX_OQ_PKTSPER_INTR	128
 /*#define CAVIUM_ONLY_CN23XX_RX_PERF*/
@@ -163,6 +167,11 @@
 				((cfg)->misc.oct_link_query_interval)
 #define CFG_GET_IS_SLI_BP_ON(cfg)                ((cfg)->misc.enable_sli_oq_bp)
 
+#define CFG_SET_NUM_RX_DESCS_NIC_IF(cfg, idx, value) \
+				((cfg)->nic_if_cfg[idx].num_rx_descs = value)
+#define CFG_SET_NUM_TX_DESCS_NIC_IF(cfg, idx, value) \
+				((cfg)->nic_if_cfg[idx].num_tx_descs = value)
+
 /* Max IOQs per OCTEON Link */
 #define MAX_IOQS_PER_NICIF              64
 
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_console.c b/drivers/net/ethernet/cavium/liquidio/octeon_console.c
index e08f7600f986..ec3dd69cd6b2 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_console.c
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_console.c
@@ -37,13 +37,6 @@ static u64 cvmx_bootmem_phy_named_block_find(struct octeon_device *oct,
 					     u32 flags);
 static int octeon_console_read(struct octeon_device *oct, u32 console_num,
 			       char *buffer, u32 buf_size);
-static u32 console_bitmask;
-module_param(console_bitmask, int, 0644);
-MODULE_PARM_DESC(console_bitmask,
-		 "Bitmask indicating which consoles have debug output redirected to syslog.");
-
-#define MIN(a, b) min((a), (b))
-#define CAST_ULL(v) ((u64)(v))
 
 #define BOOTLOADER_PCI_READ_BUFFER_DATA_ADDR    0x0006c008
 #define BOOTLOADER_PCI_READ_BUFFER_LEN_ADDR     0x0006c004
@@ -139,16 +132,6 @@ struct octeon_pci_console_desc {
 };
 
 /**
- * \brief determines if a given console has debug enabled.
- * @param console console to check
- * @returns  1 = enabled. 0 otherwise
- */
-static int octeon_console_debug_enabled(u32 console)
-{
-	return (console_bitmask >> (console)) & 0x1;
-}
-
-/**
  * This function is the implementation of the get macros defined
  * for individual structure members. The argument are generated
  * by the macros inorder to read only the needed memory.
@@ -234,7 +217,7 @@ static int __cvmx_bootmem_check_version(struct octeon_device *oct,
 	    (exact_match && major_version != exact_match)) {
 		dev_err(&oct->pci_dev->dev, "bootmem ver mismatch %d.%d addr:0x%llx\n",
 			major_version, minor_version,
-			CAST_ULL(oct->bootmem_desc_addr));
+			(long long)oct->bootmem_desc_addr);
 		return -1;
 	} else {
 		return 0;
@@ -454,20 +437,31 @@ static void output_console_line(struct octeon_device *oct,
 {
 	char *line;
 	s32 i;
+	size_t len;
 
 	line = console_buffer;
 	for (i = 0; i < bytes_read; i++) {
 		/* Output a line at a time, prefixed */
 		if (console_buffer[i] == '\n') {
 			console_buffer[i] = '\0';
-			if (console->leftover[0]) {
-				dev_info(&oct->pci_dev->dev, "%lu: %s%s\n",
-					 console_num, console->leftover,
-					 line);
+			/* We need to output 'line', prefaced by 'leftover'.
+			 * However, it is possible we're being called to
+			 * output 'leftover' by itself (in the case of nothing
+			 * having been read from the console).
+			 *
+			 * To avoid duplication, check for this condition.
+			 */
+			if (console->leftover[0] &&
+			    (line != console->leftover)) {
+				if (console->print)
+					(*console->print)(oct, (u32)console_num,
+							  console->leftover,
+							  line);
 				console->leftover[0] = '\0';
 			} else {
-				dev_info(&oct->pci_dev->dev, "%lu: %s\n",
-					 console_num, line);
+				if (console->print)
+					(*console->print)(oct, (u32)console_num,
+							  line, NULL);
 			}
 			line = &console_buffer[i + 1];
 		}
@@ -476,13 +470,16 @@ static void output_console_line(struct octeon_device *oct,
 	/* Save off any leftovers */
 	if (line != &console_buffer[bytes_read]) {
 		console_buffer[bytes_read] = '\0';
-		strcpy(console->leftover, line);
+		len = strlen(console->leftover);
+		strncpy(&console->leftover[len], line,
+			sizeof(console->leftover) - len);
 	}
 }
 
 static void check_console(struct work_struct *work)
 {
 	s32 bytes_read, tries, total_read;
+	size_t len;
 	struct octeon_console *console;
 	struct cavium_wk *wk = (struct cavium_wk *)work;
 	struct octeon_device *oct = (struct octeon_device *)wk->ctxptr;
@@ -504,7 +501,7 @@ static void check_console(struct work_struct *work)
 			total_read += bytes_read;
 			if (console->waiting)
 				octeon_console_handle_result(oct, console_num);
-			if (octeon_console_debug_enabled(console_num)) {
+			if (console->print) {
 				output_console_line(oct, console, console_num,
 						    console_buffer, bytes_read);
 			}
@@ -519,10 +516,13 @@ static void check_console(struct work_struct *work)
 	/* If nothing is read after polling the console,
 	 * output any leftovers if any
 	 */
-	if (octeon_console_debug_enabled(console_num) &&
-	    (total_read == 0) && (console->leftover[0])) {
-		dev_info(&oct->pci_dev->dev, "%u: %s\n",
-			 console_num, console->leftover);
+	if (console->print && (total_read == 0) &&
+	    (console->leftover[0])) {
+		/* append '\n' as terminator for 'output_console_line' */
+		len = strlen(console->leftover);
+		console->leftover[len] = '\n';
+		output_console_line(oct, console, console_num,
+				    console->leftover, (s32)(len + 1));
 		console->leftover[0] = '\0';
 	}
 
@@ -574,7 +574,84 @@ int octeon_init_consoles(struct octeon_device *oct)
 	return ret;
 }
 
-int octeon_add_console(struct octeon_device *oct, u32 console_num)
+static void octeon_get_uboot_version(struct octeon_device *oct)
+{
+	s32 bytes_read, tries, total_read;
+	struct octeon_console *console;
+	u32 console_num = 0;
+	char *uboot_ver;
+	char *buf;
+	char *p;
+
+#define OCTEON_UBOOT_VER_BUF_SIZE 512
+	buf = kmalloc(OCTEON_UBOOT_VER_BUF_SIZE, GFP_KERNEL);
+	if (!buf)
+		return;
+
+	if (octeon_console_send_cmd(oct, "setenv stdout pci\n", 50)) {
+		kfree(buf);
+		return;
+	}
+
+	if (octeon_console_send_cmd(oct, "version\n", 1)) {
+		kfree(buf);
+		return;
+	}
+
+	console = &oct->console[console_num];
+	tries = 0;
+	total_read = 0;
+
+	do {
+		/* Take console output regardless of whether it will
+		 * be logged
+		 */
+		bytes_read =
+			octeon_console_read(oct,
+					    console_num, buf + total_read,
+					    OCTEON_UBOOT_VER_BUF_SIZE - 1 -
+					    total_read);
+		if (bytes_read > 0) {
+			buf[bytes_read] = '\0';
+
+			total_read += bytes_read;
+			if (console->waiting)
+				octeon_console_handle_result(oct, console_num);
+		} else if (bytes_read < 0) {
+			dev_err(&oct->pci_dev->dev, "Error reading console %u, ret=%d\n",
+				console_num, bytes_read);
+		}
+
+		tries++;
+	} while ((bytes_read > 0) && (tries < 16));
+
+	/* If nothing is read after polling the console,
+	 * output any leftovers if any
+	 */
+	if ((total_read == 0) && (console->leftover[0])) {
+		dev_dbg(&oct->pci_dev->dev, "%u: %s\n",
+			console_num, console->leftover);
+		console->leftover[0] = '\0';
+	}
+
+	buf[OCTEON_UBOOT_VER_BUF_SIZE - 1] = '\0';
+
+	uboot_ver = strstr(buf, "U-Boot");
+	if (uboot_ver) {
+		p = strstr(uboot_ver, "mips");
+		if (p) {
+			p--;
+			*p = '\0';
+			dev_info(&oct->pci_dev->dev, "%s\n", uboot_ver);
+		}
+	}
+
+	kfree(buf);
+	octeon_console_send_cmd(oct, "setenv stdout serial\n", 50);
+}
+
+int octeon_add_console(struct octeon_device *oct, u32 console_num,
+		       char *dbg_enb)
 {
 	int ret = 0;
 	u32 delay;
@@ -610,17 +687,19 @@ int octeon_add_console(struct octeon_device *oct, u32 console_num)
 
 		work = &oct->console_poll_work[console_num].work;
 
+		octeon_get_uboot_version(oct);
+
 		INIT_DELAYED_WORK(work, check_console);
 		oct->console_poll_work[console_num].ctxptr = (void *)oct;
 		oct->console_poll_work[console_num].ctxul = console_num;
 		delay = OCTEON_CONSOLE_POLL_INTERVAL_MS;
 		schedule_delayed_work(work, msecs_to_jiffies(delay));
 
-		if (octeon_console_debug_enabled(console_num)) {
-			ret = octeon_console_send_cmd(oct,
-						      "setenv pci_console_active 1",
-						      2000);
-		}
+		/* an empty string means use default debug console enablement */
+		if (dbg_enb && !dbg_enb[0])
+			dbg_enb = "setenv pci_console_active 1";
+		if (dbg_enb)
+			ret = octeon_console_send_cmd(oct, dbg_enb, 2000);
 
 		console->active = 1;
 	}
@@ -704,7 +783,7 @@ static int octeon_console_read(struct octeon_device *oct, u32 console_num,
 	if (bytes_to_read <= 0)
 		return bytes_to_read;
 
-	bytes_to_read = MIN(bytes_to_read, (s32)buf_size);
+	bytes_to_read = min_t(s32, bytes_to_read, buf_size);
 
 	/* Check to see if what we want to read is not contiguous, and limit
 	 * ourselves to the contiguous block
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_device.c b/drivers/net/ethernet/cavium/liquidio/octeon_device.c
index 623e28ca736e..29d53b1763a7 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_device.c
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_device.c
@@ -418,7 +418,7 @@ static struct octeon_config default_cn23xx_conf = {
 	/** IQ attributes */
 	.iq = {
 		.max_iqs		= CN23XX_CFG_IO_QUEUES,
-		.pending_list_size	= (CN23XX_MAX_IQ_DESCRIPTORS *
+		.pending_list_size	= (CN23XX_DEFAULT_IQ_DESCRIPTORS *
 					   CN23XX_CFG_IO_QUEUES),
 		.instr_type		= OCTEON_64BYTE_INSTR,
 		.db_min			= CN23XX_DB_MIN,
@@ -436,8 +436,8 @@ static struct octeon_config default_cn23xx_conf = {
 	},
 
 	.num_nic_ports				= DEFAULT_NUM_NIC_PORTS_23XX,
-	.num_def_rx_descs			= CN23XX_MAX_OQ_DESCRIPTORS,
-	.num_def_tx_descs			= CN23XX_MAX_IQ_DESCRIPTORS,
+	.num_def_rx_descs			= CN23XX_DEFAULT_OQ_DESCRIPTORS,
+	.num_def_tx_descs			= CN23XX_DEFAULT_IQ_DESCRIPTORS,
 	.def_rx_buf_size			= CN23XX_OQ_BUF_SIZE,
 
 	/* For ethernet interface 0:  Port cfg Attributes */
@@ -455,10 +455,10 @@ static struct octeon_config default_cn23xx_conf = {
 		.num_rxqs			= DEF_RXQS_PER_INTF,
 
 		/* Num of desc for rx rings */
-		.num_rx_descs			= CN23XX_MAX_OQ_DESCRIPTORS,
+		.num_rx_descs			= CN23XX_DEFAULT_OQ_DESCRIPTORS,
 
 		/* Num of desc for tx rings */
-		.num_tx_descs			= CN23XX_MAX_IQ_DESCRIPTORS,
+		.num_tx_descs			= CN23XX_DEFAULT_IQ_DESCRIPTORS,
 
 		/* SKB size, We need not change buf size even for Jumbo frames.
 		 * Octeon can send jumbo frames in 4 consecutive descriptors,
@@ -484,10 +484,10 @@ static struct octeon_config default_cn23xx_conf = {
 		.num_rxqs			= DEF_RXQS_PER_INTF,
 
 		/* Num of desc for rx rings */
-		.num_rx_descs			= CN23XX_MAX_OQ_DESCRIPTORS,
+		.num_rx_descs			= CN23XX_DEFAULT_OQ_DESCRIPTORS,
 
 		/* Num of desc for tx rings */
-		.num_tx_descs			= CN23XX_MAX_IQ_DESCRIPTORS,
+		.num_tx_descs			= CN23XX_DEFAULT_IQ_DESCRIPTORS,
 
 		/* SKB size, We need not change buf size even for Jumbo frames.
 		 * Octeon can send jumbo frames in 4 consecutive descriptors,
@@ -528,9 +528,10 @@ static struct octeon_config_ptr {
 };
 
 static char oct_dev_state_str[OCT_DEV_STATES + 1][32] = {
-	"BEGIN", "PCI-MAP-DONE", "DISPATCH-INIT-DONE",
+	"BEGIN", "PCI-ENABLE-DONE", "PCI-MAP-DONE", "DISPATCH-INIT-DONE",
 	"IQ-INIT-DONE", "SCBUFF-POOL-INIT-DONE", "RESPLIST-INIT-DONE",
-	"DROQ-INIT-DONE", "IO-QUEUES-INIT-DONE", "CONSOLE-INIT-DONE",
+	"DROQ-INIT-DONE", "MBOX-SETUP-DONE", "MSIX-ALLOC-VECTOR-DONE",
+	"INTR-SET-DONE", "IO-QUEUES-INIT-DONE", "CONSOLE-INIT-DONE",
 	"HOST-READY", "CORE-READY", "RUNNING", "IN-RESET",
 	"INVALID"
 };
@@ -876,11 +877,11 @@ int octeon_setup_instr_queues(struct octeon_device *oct)
 
 	oct->num_iqs = 0;
 
-	oct->instr_queue[0] = vmalloc_node(sizeof(*oct->instr_queue[0]),
+	oct->instr_queue[0] = vzalloc_node(sizeof(*oct->instr_queue[0]),
 				numa_node);
 	if (!oct->instr_queue[0])
 		oct->instr_queue[0] =
-			vmalloc(sizeof(struct octeon_instr_queue));
+			vzalloc(sizeof(struct octeon_instr_queue));
 	if (!oct->instr_queue[0])
 		return 1;
 	memset(oct->instr_queue[0], 0, sizeof(struct octeon_instr_queue));
@@ -923,9 +924,9 @@ int octeon_setup_output_queues(struct octeon_device *oct)
 		desc_size = CFG_GET_DEF_RX_BUF_SIZE(CHIP_CONF(oct, cn23xx_vf));
 	}
 	oct->num_oqs = 0;
-	oct->droq[0] = vmalloc_node(sizeof(*oct->droq[0]), numa_node);
+	oct->droq[0] = vzalloc_node(sizeof(*oct->droq[0]), numa_node);
 	if (!oct->droq[0])
-		oct->droq[0] = vmalloc(sizeof(*oct->droq[0]));
+		oct->droq[0] = vzalloc(sizeof(*oct->droq[0]));
 	if (!oct->droq[0])
 		return 1;
 
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_device.h b/drivers/net/ethernet/cavium/liquidio/octeon_device.h
index c90ed48ae8ab..894af199ddef 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_device.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_device.h
@@ -22,6 +22,8 @@
 #ifndef _OCTEON_DEVICE_H_
 #define  _OCTEON_DEVICE_H_
 
+#include <linux/interrupt.h>
+
 /** PCI VendorId Device Id */
 #define  OCTEON_CN68XX_PCIID          0x91177d
 #define  OCTEON_CN66XX_PCIID          0x92177d
@@ -192,6 +194,8 @@ struct octeon_reg_list {
 };
 
 #define OCTEON_CONSOLE_MAX_READ_BYTES 512
+typedef int (*octeon_console_print_fn)(struct octeon_device *oct,
+				       u32 num, char *pre, char *suf);
 struct octeon_console {
 	u32 active;
 	u32 waiting;
@@ -199,6 +203,7 @@ struct octeon_console {
 	u32 buffer_size;
 	u64 input_base_addr;
 	u64 output_base_addr;
+	octeon_console_print_fn print;
 	char leftover[OCTEON_CONSOLE_MAX_READ_BYTES];
 };
 
@@ -552,6 +557,7 @@ struct octeon_device {
 	} loc;
 
 	atomic_t *adapter_refcount; /* reference count of adapter */
+	bool ptp_enable;
 };
 
 #define  OCT_DRV_ONLINE 1
@@ -565,6 +571,8 @@ struct octeon_device {
 #define CHIP_CONF(oct, TYPE)             \
 	(((struct octeon_ ## TYPE  *)((oct)->chip))->conf)
 
+#define MAX_IO_PENDING_PKT_COUNT 100
+
 /*------------------ Function Prototypes ----------------------*/
 
 /** Initialize device list memory */
@@ -740,11 +748,17 @@ int octeon_init_consoles(struct octeon_device *oct);
 /**
  * Adds access to a console to the device.
  *
- * @param oct which octeon to add to
- * @param console_num which console
+ * @param oct:          which octeon to add to
+ * @param console_num:  which console
+ * @param dbg_enb:      ptr to debug enablement string, one of:
+ *                    * NULL for no debug output (i.e. disabled)
+ *                    * empty string enables debug output (via default method)
+ *                    * specific string to enable debug console output
+ *
  * @return Zero on success, negative on failure.
  */
-int octeon_add_console(struct octeon_device *oct, u32 console_num);
+int octeon_add_console(struct octeon_device *oct, u32 console_num,
+		       char *dbg_enb);
 
 /** write or read from a console */
 int octeon_console_write(struct octeon_device *oct, u32 console_num,
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_droq.c b/drivers/net/ethernet/cavium/liquidio/octeon_droq.c
index 2e190deb2233..9372d4ce9954 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_droq.c
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_droq.c
@@ -145,6 +145,8 @@ octeon_droq_destroy_ring_buffers(struct octeon_device *oct,
 
 	for (i = 0; i < droq->max_count; i++) {
 		pg_info = &droq->recv_buf_list[i].pg_info;
+		if (!pg_info)
+			continue;
 
 		if (pg_info->dma)
 			lio_unmap_ring(oct->pci_dev,
@@ -207,6 +209,10 @@ int octeon_delete_droq(struct octeon_device *oct, u32 q_no)
 			     droq->desc_ring, droq->desc_ring_dma);
 
 	memset(droq, 0, OCT_DROQ_SIZE);
+	oct->io_qmask.oq &= ~(1ULL << q_no);
+	vfree(oct->droq[q_no]);
+	oct->droq[q_no] = NULL;
+	oct->num_oqs--;
 
 	return 0;
 }
@@ -275,12 +281,12 @@ int octeon_init_droq(struct octeon_device *oct,
 		droq->max_count);
 
 	droq->recv_buf_list = (struct octeon_recv_buffer *)
-			      vmalloc_node(droq->max_count *
+			      vzalloc_node(droq->max_count *
 						OCT_DROQ_RECVBUF_SIZE,
 						numa_node);
 	if (!droq->recv_buf_list)
 		droq->recv_buf_list = (struct octeon_recv_buffer *)
-				      vmalloc(droq->max_count *
+				      vzalloc(droq->max_count *
 						OCT_DROQ_RECVBUF_SIZE);
 	if (!droq->recv_buf_list) {
 		dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n");
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_droq.h b/drivers/net/ethernet/cavium/liquidio/octeon_droq.h
index 6efd139b894d..f91bc84d1719 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_droq.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_droq.h
@@ -328,7 +328,7 @@ struct octeon_droq {
 
 	u32 cpu_id;
 
-	struct call_single_data csd;
+	call_single_data_t csd;
 };
 
 #define OCT_DROQ_SIZE   (sizeof(struct octeon_droq))
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_main.h b/drivers/net/ethernet/cavium/liquidio/octeon_main.h
index 7ccffbb0019e..32ef3a7d88d8 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_main.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_main.h
@@ -35,6 +35,12 @@
 
 #define DRV_NAME "LiquidIO"
 
+struct octeon_device_priv {
+	/** Tasklet structures for this device. */
+	struct tasklet_struct droq_tasklet;
+	unsigned long napi_mask;
+};
+
 /** This structure is used by NIC driver to store information required
  * to free the sk_buff when the packet has been fetched by Octeon.
  * Bytes offset below assume worst-case of a 64-bit system.
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_network.h b/drivers/net/ethernet/cavium/liquidio/octeon_network.h
index ec8504b2942d..9e36319cead6 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_network.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_network.h
@@ -33,6 +33,7 @@
 #define   LIO_IFSTATE_REGISTERED           0x02
 #define   LIO_IFSTATE_RUNNING              0x04
 #define   LIO_IFSTATE_RX_TIMESTAMP_ENABLED 0x08
+#define   LIO_IFSTATE_RESETTING		   0x10
 
 struct oct_nic_stats_resp {
 	u64     rh;
@@ -166,6 +167,14 @@ void cleanup_rx_oom_poll_fn(struct net_device *netdev);
  */
 void liquidio_link_ctrl_cmd_completion(void *nctrl_ptr);
 
+int liquidio_setup_io_queues(struct octeon_device *octeon_dev, int ifidx,
+			     u32 num_iqs, u32 num_oqs);
+
+irqreturn_t liquidio_msix_intr_handler(int irq __attribute__((unused)),
+				       void *dev);
+
+int octeon_setup_interrupt(struct octeon_device *oct, u32 num_ioqs);
+
 /**
  * \brief Register ethtool operations
  * @param netdev    pointer to network device
@@ -448,4 +457,30 @@ static inline void ifstate_reset(struct lio *lio, int state_flag)
 	atomic_set(&lio->ifstate, (atomic_read(&lio->ifstate) & ~(state_flag)));
 }
 
+/**
+ * \brief wait for all pending requests to complete
+ * @param oct Pointer to Octeon device
+ *
+ * Called during shutdown sequence
+ */
+static inline int wait_for_pending_requests(struct octeon_device *oct)
+{
+	int i, pcount = 0;
+
+	for (i = 0; i < MAX_IO_PENDING_PKT_COUNT; i++) {
+		pcount = atomic_read(
+		    &oct->response_list[OCTEON_ORDERED_SC_LIST]
+			 .pending_req_count);
+		if (pcount)
+			schedule_timeout_uninterruptible(HZ / 10);
+		else
+			break;
+	}
+
+	if (pcount)
+		return 1;
+
+	return 0;
+}
+
 #endif
diff --git a/drivers/net/ethernet/cavium/liquidio/request_manager.c b/drivers/net/ethernet/cavium/liquidio/request_manager.c
index 7b297f1f6dbe..1e0fbce86d60 100644
--- a/drivers/net/ethernet/cavium/liquidio/request_manager.c
+++ b/drivers/net/ethernet/cavium/liquidio/request_manager.c
@@ -77,13 +77,6 @@ int octeon_init_instr_queue(struct octeon_device *oct,
 		return 1;
 	}
 
-	if (num_descs & (num_descs - 1)) {
-		dev_err(&oct->pci_dev->dev,
-			"Number of descriptors for instr queue %d not in power of 2.\n",
-			iq_no);
-		return 1;
-	}
-
 	q_size = (u32)conf->instr_type * num_descs;
 
 	iq = oct->instr_queue[iq_no];
@@ -190,6 +183,10 @@ int octeon_delete_instr_queue(struct octeon_device *oct, u32 iq_no)
 		q_size = iq->max_count * desc_size;
 		lio_dma_free(oct, (u32)q_size, iq->base_addr,
 			     iq->base_addr_dma);
+		oct->io_qmask.iq &= ~(1ULL << iq_no);
+		vfree(oct->instr_queue[iq_no]);
+		oct->instr_queue[iq_no] = NULL;
+		oct->num_iqs--;
 		return 0;
 	}
 	return 1;
diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_queues.h b/drivers/net/ethernet/cavium/thunder/nicvf_queues.h
index 57858522c33c..67d1a3230773 100644
--- a/drivers/net/ethernet/cavium/thunder/nicvf_queues.h
+++ b/drivers/net/ethernet/cavium/thunder/nicvf_queues.h
@@ -277,7 +277,6 @@ struct snd_queue {
 	u16		xdp_free_cnt;
 	bool		is_xdp;
 
-#define	TSO_HEADER_SIZE	128
 	/* For TSO segment's header */
 	char		*tso_hdrs;
 	dma_addr_t	tso_hdrs_phys;
diff --git a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c
index 0bc6a4ffce30..6a015362c340 100644
--- a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c
@@ -793,7 +793,9 @@ static struct attribute *cxgb3_attrs[] = {
 	NULL
 };
 
-static struct attribute_group cxgb3_attr_group = {.attrs = cxgb3_attrs };
+static const struct attribute_group cxgb3_attr_group = {
+	.attrs = cxgb3_attrs,
+};
 
 static ssize_t tm_attr_show(struct device *d,
 			    char *buf, int sched)
@@ -880,7 +882,9 @@ static struct attribute *offload_attrs[] = {
 	NULL
 };
 
-static struct attribute_group offload_attr_group = {.attrs = offload_attrs };
+static const struct attribute_group offload_attr_group = {
+	.attrs = offload_attrs,
+};
 
 /*
  * Sends an sk_buff to an offload queue driver
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
index ef4be781fd05..ea72d2d2e1b4 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
@@ -104,13 +104,13 @@ enum dev_state {
 	DEV_STATE_ERR
 };
 
-enum {
+enum cc_pause {
 	PAUSE_RX      = 1 << 0,
 	PAUSE_TX      = 1 << 1,
 	PAUSE_AUTONEG = 1 << 2
 };
 
-enum {
+enum cc_fec {
 	FEC_AUTO      = 1 << 0,	 /* IEEE 802.3 "automatic" */
 	FEC_RS        = 1 << 1,  /* Reed-Solomon */
 	FEC_BASER_RS  = 1 << 2   /* BaseR/Reed-Solomon */
@@ -338,10 +338,12 @@ struct adapter_params {
 	unsigned int sf_nsec;             /* # of flash sectors */
 	unsigned int sf_fw_start;         /* start of FW image in flash */
 
-	unsigned int fw_vers;
-	unsigned int bs_vers;		/* bootstrap version */
-	unsigned int tp_vers;
-	unsigned int er_vers;		/* expansion ROM version */
+	unsigned int fw_vers;		  /* firmware version */
+	unsigned int bs_vers;		  /* bootstrap version */
+	unsigned int tp_vers;		  /* TP microcode version */
+	unsigned int er_vers;		  /* expansion ROM version */
+	unsigned int scfg_vers;		  /* Serial Configuration version */
+	unsigned int vpd_vers;		  /* VPD Version */
 	u8 api_vers[7];
 
 	unsigned short mtus[NMTUS];
@@ -364,6 +366,7 @@ struct adapter_params {
 	unsigned int max_ordird_qp;       /* Max read depth per RDMA QP */
 	unsigned int max_ird_adapter;     /* Max read depth per adapter */
 	bool fr_nsmr_tpte_wr_support;	  /* FW support for FR_NSMR_TPTE_WR */
+	u8 fw_caps_support;		/* 32-bit Port Capabilities */
 
 	/* MPS Buffer Group Map[per Port].  Bit i is set if buffer group i is
 	 * used by the Port
@@ -437,18 +440,34 @@ struct trace_params {
 	unsigned char port;
 };
 
+/* Firmware Port Capabilities types. */
+
+typedef u16 fw_port_cap16_t;	/* 16-bit Port Capabilities integral value */
+typedef u32 fw_port_cap32_t;	/* 32-bit Port Capabilities integral value */
+
+enum fw_caps {
+	FW_CAPS_UNKNOWN	= 0,	/* 0'ed out initial state */
+	FW_CAPS16	= 1,	/* old Firmware: 16-bit Port Capabilities */
+	FW_CAPS32	= 2,	/* new Firmware: 32-bit Port Capabilities */
+};
+
 struct link_config {
-	unsigned short supported;        /* link capabilities */
-	unsigned short advertising;      /* advertised capabilities */
-	unsigned short lp_advertising;   /* peer advertised capabilities */
-	unsigned int   requested_speed;  /* speed user has requested */
-	unsigned int   speed;            /* actual link speed */
-	unsigned char  requested_fc;     /* flow control user has requested */
-	unsigned char  fc;               /* actual link flow control */
-	unsigned char  auto_fec;	 /* Forward Error Correction: */
-	unsigned char  requested_fec;	 /* "automatic" (IEEE 802.3), */
-	unsigned char  fec;		 /* requested, and actual in use */
+	fw_port_cap32_t pcaps;           /* link capabilities */
+	fw_port_cap32_t def_acaps;       /* default advertised capabilities */
+	fw_port_cap32_t acaps;           /* advertised capabilities */
+	fw_port_cap32_t lpacaps;         /* peer advertised capabilities */
+
+	fw_port_cap32_t speed_caps;      /* speed(s) user has requested */
+	unsigned int   speed;            /* actual link speed (Mb/s) */
+
+	enum cc_pause  requested_fc;     /* flow control user has requested */
+	enum cc_pause  fc;               /* actual link flow control */
+
+	enum cc_fec    requested_fec;	 /* Forward Error Correction: */
+	enum cc_fec    fec;		 /* requested and actual in use */
+
 	unsigned char  autoneg;          /* autonegotiating? */
+
 	unsigned char  link_ok;          /* link up? */
 	unsigned char  link_down_rc;     /* link down reason */
 };
@@ -529,6 +548,7 @@ enum {                                 /* adapter flags */
 	USING_SOFT_PARAMS  = (1 << 6),
 	MASTER_PF          = (1 << 7),
 	FW_OFLD_CONN       = (1 << 9),
+	ROOT_NO_RELAXED_ORDERING = (1 << 10),
 };
 
 enum {
@@ -1403,10 +1423,15 @@ int t4_fw_upgrade(struct adapter *adap, unsigned int mbox,
 int t4_fl_pkt_align(struct adapter *adap);
 unsigned int t4_flash_cfg_addr(struct adapter *adapter);
 int t4_check_fw_version(struct adapter *adap);
+int t4_load_cfg(struct adapter *adapter, const u8 *cfg_data, unsigned int size);
 int t4_get_fw_version(struct adapter *adapter, u32 *vers);
 int t4_get_bs_version(struct adapter *adapter, u32 *vers);
 int t4_get_tp_version(struct adapter *adapter, u32 *vers);
 int t4_get_exprom_version(struct adapter *adapter, u32 *vers);
+int t4_get_scfg_version(struct adapter *adapter, u32 *vers);
+int t4_get_vpd_version(struct adapter *adapter, u32 *vers);
+int t4_get_version_info(struct adapter *adapter);
+void t4_dump_version_info(struct adapter *adapter);
 int t4_prep_fw(struct adapter *adap, struct fw_info *fw_info,
 	       const u8 *fw_data, unsigned int fw_size,
 	       struct fw_hdr *card_fw, enum dev_state state, int *reset);
@@ -1572,6 +1597,8 @@ int t4_ofld_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf,
 int t4_sge_ctxt_flush(struct adapter *adap, unsigned int mbox);
 void t4_handle_get_port_info(struct port_info *pi, const __be64 *rpl);
 int t4_update_port_info(struct port_info *pi);
+int t4_get_link_params(struct port_info *pi, unsigned int *link_okp,
+		       unsigned int *speedp, unsigned int *mtup);
 int t4_handle_fw_rpl(struct adapter *adap, const __be64 *rpl);
 void t4_db_full(struct adapter *adapter);
 void t4_db_dropped(struct adapter *adapter);
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c
index 26eb00a45db1..a71af1e587e2 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c
@@ -533,17 +533,23 @@ static int from_fw_port_mod_type(enum fw_port_type port_type,
 static unsigned int speed_to_fw_caps(int speed)
 {
 	if (speed == 100)
-		return FW_PORT_CAP_SPEED_100M;
+		return FW_PORT_CAP32_SPEED_100M;
 	if (speed == 1000)
-		return FW_PORT_CAP_SPEED_1G;
+		return FW_PORT_CAP32_SPEED_1G;
 	if (speed == 10000)
-		return FW_PORT_CAP_SPEED_10G;
+		return FW_PORT_CAP32_SPEED_10G;
 	if (speed == 25000)
-		return FW_PORT_CAP_SPEED_25G;
+		return FW_PORT_CAP32_SPEED_25G;
 	if (speed == 40000)
-		return FW_PORT_CAP_SPEED_40G;
+		return FW_PORT_CAP32_SPEED_40G;
+	if (speed == 50000)
+		return FW_PORT_CAP32_SPEED_50G;
 	if (speed == 100000)
-		return FW_PORT_CAP_SPEED_100G;
+		return FW_PORT_CAP32_SPEED_100G;
+	if (speed == 200000)
+		return FW_PORT_CAP32_SPEED_200G;
+	if (speed == 400000)
+		return FW_PORT_CAP32_SPEED_400G;
 	return 0;
 }
 
@@ -560,12 +566,13 @@ static void fw_caps_to_lmm(enum fw_port_type port_type,
 			   unsigned int fw_caps,
 			   unsigned long *link_mode_mask)
 {
-	#define SET_LMM(__lmm_name) __set_bit(ETHTOOL_LINK_MODE_ ## __lmm_name \
-					## _BIT, link_mode_mask)
+	#define SET_LMM(__lmm_name) \
+		__set_bit(ETHTOOL_LINK_MODE_ ## __lmm_name ## _BIT, \
+			  link_mode_mask)
 
 	#define FW_CAPS_TO_LMM(__fw_name, __lmm_name) \
 		do { \
-			if (fw_caps & FW_PORT_CAP_ ## __fw_name) \
+			if (fw_caps & FW_PORT_CAP32_ ## __fw_name) \
 				SET_LMM(__lmm_name); \
 		} while (0)
 
@@ -645,7 +652,10 @@ static void fw_caps_to_lmm(enum fw_port_type port_type,
 	case FW_PORT_TYPE_KR4_100G:
 	case FW_PORT_TYPE_CR4_QSFP:
 		SET_LMM(FIBRE);
-		SET_LMM(100000baseCR4_Full);
+		FW_CAPS_TO_LMM(SPEED_40G, 40000baseSR4_Full);
+		FW_CAPS_TO_LMM(SPEED_25G, 25000baseCR_Full);
+		FW_CAPS_TO_LMM(SPEED_50G, 50000baseCR2_Full);
+		FW_CAPS_TO_LMM(SPEED_100G, 100000baseCR4_Full);
 		break;
 
 	default:
@@ -663,8 +673,7 @@ static void fw_caps_to_lmm(enum fw_port_type port_type,
 /**
  *	lmm_to_fw_caps - translate ethtool Link Mode Mask to Firmware
  *	capabilities
- *
- *	@link_mode_mask: ethtool Link Mode Mask
+ *	@et_lmm: ethtool Link Mode Mask
  *
  *	Translate ethtool Link Mode Mask into a Firmware Port capabilities
  *	value.
@@ -677,7 +686,7 @@ static unsigned int lmm_to_fw_caps(const unsigned long *link_mode_mask)
 		do { \
 			if (test_bit(ETHTOOL_LINK_MODE_ ## __lmm_name ## _BIT, \
 				     link_mode_mask)) \
-				fw_caps |= FW_PORT_CAP_ ## __fw_name; \
+				fw_caps |= FW_PORT_CAP32_ ## __fw_name; \
 		} while (0)
 
 	LMM_TO_FW_CAPS(100baseT_Full, SPEED_100M);
@@ -685,6 +694,7 @@ static unsigned int lmm_to_fw_caps(const unsigned long *link_mode_mask)
 	LMM_TO_FW_CAPS(10000baseT_Full, SPEED_10G);
 	LMM_TO_FW_CAPS(40000baseSR4_Full, SPEED_40G);
 	LMM_TO_FW_CAPS(25000baseCR_Full, SPEED_25G);
+	LMM_TO_FW_CAPS(50000baseCR2_Full, SPEED_50G);
 	LMM_TO_FW_CAPS(100000baseCR4_Full, SPEED_100G);
 
 	#undef LMM_TO_FW_CAPS
@@ -698,10 +708,6 @@ static int get_link_ksettings(struct net_device *dev,
 	struct port_info *pi = netdev_priv(dev);
 	struct ethtool_link_settings *base = &link_ksettings->base;
 
-	ethtool_link_ksettings_zero_link_mode(link_ksettings, supported);
-	ethtool_link_ksettings_zero_link_mode(link_ksettings, advertising);
-	ethtool_link_ksettings_zero_link_mode(link_ksettings, lp_advertising);
-
 	/* For the nonce, the Firmware doesn't send up Port State changes
 	 * when the Virtual Interface attached to the Port is down.  So
 	 * if it's down, let's grab any changes.
@@ -709,6 +715,10 @@ static int get_link_ksettings(struct net_device *dev,
 	if (!netif_running(dev))
 		(void)t4_update_port_info(pi);
 
+	ethtool_link_ksettings_zero_link_mode(link_ksettings, supported);
+	ethtool_link_ksettings_zero_link_mode(link_ksettings, advertising);
+	ethtool_link_ksettings_zero_link_mode(link_ksettings, lp_advertising);
+
 	base->port = from_fw_port_mod_type(pi->port_type, pi->mod_type);
 
 	if (pi->mdio_addr >= 0) {
@@ -721,11 +731,11 @@ static int get_link_ksettings(struct net_device *dev,
 		base->mdio_support = 0;
 	}
 
-	fw_caps_to_lmm(pi->port_type, pi->link_cfg.supported,
+	fw_caps_to_lmm(pi->port_type, pi->link_cfg.pcaps,
 		       link_ksettings->link_modes.supported);
-	fw_caps_to_lmm(pi->port_type, pi->link_cfg.advertising,
+	fw_caps_to_lmm(pi->port_type, pi->link_cfg.acaps,
 		       link_ksettings->link_modes.advertising);
-	fw_caps_to_lmm(pi->port_type, pi->link_cfg.lp_advertising,
+	fw_caps_to_lmm(pi->port_type, pi->link_cfg.lpacaps,
 		       link_ksettings->link_modes.lp_advertising);
 
 	if (netif_carrier_ok(dev)) {
@@ -736,8 +746,24 @@ static int get_link_ksettings(struct net_device *dev,
 		base->duplex = DUPLEX_UNKNOWN;
 	}
 
+	if (pi->link_cfg.fc & PAUSE_RX) {
+		if (pi->link_cfg.fc & PAUSE_TX) {
+			ethtool_link_ksettings_add_link_mode(link_ksettings,
+							     advertising,
+							     Pause);
+		} else {
+			ethtool_link_ksettings_add_link_mode(link_ksettings,
+							     advertising,
+							     Asym_Pause);
+		}
+	} else if (pi->link_cfg.fc & PAUSE_TX) {
+		ethtool_link_ksettings_add_link_mode(link_ksettings,
+						     advertising,
+						     Asym_Pause);
+	}
+
 	base->autoneg = pi->link_cfg.autoneg;
-	if (pi->link_cfg.supported & FW_PORT_CAP_ANEG)
+	if (pi->link_cfg.pcaps & FW_PORT_CAP32_ANEG)
 		ethtool_link_ksettings_add_link_mode(link_ksettings,
 						     supported, Autoneg);
 	if (pi->link_cfg.autoneg)
@@ -748,8 +774,7 @@ static int get_link_ksettings(struct net_device *dev,
 }
 
 static int set_link_ksettings(struct net_device *dev,
-			      const struct ethtool_link_ksettings
-						*link_ksettings)
+			    const struct ethtool_link_ksettings *link_ksettings)
 {
 	struct port_info *pi = netdev_priv(dev);
 	struct link_config *lc = &pi->link_cfg;
@@ -762,12 +787,12 @@ static int set_link_ksettings(struct net_device *dev,
 	if (base->duplex != DUPLEX_FULL)
 		return -EINVAL;
 
-	if (!(lc->supported & FW_PORT_CAP_ANEG)) {
+	if (!(lc->pcaps & FW_PORT_CAP32_ANEG)) {
 		/* PHY offers a single speed.  See if that's what's
 		 * being requested.
 		 */
 		if (base->autoneg == AUTONEG_DISABLE &&
-		    (lc->supported & speed_to_fw_caps(base->speed)))
+		    (lc->pcaps & speed_to_fw_caps(base->speed)))
 			return 0;
 		return -EINVAL;
 	}
@@ -776,18 +801,17 @@ static int set_link_ksettings(struct net_device *dev,
 	if (base->autoneg == AUTONEG_DISABLE) {
 		fw_caps = speed_to_fw_caps(base->speed);
 
-		if (!(lc->supported & fw_caps))
+		if (!(lc->pcaps & fw_caps))
 			return -EINVAL;
-		lc->requested_speed = fw_caps;
-		lc->advertising = 0;
+		lc->speed_caps = fw_caps;
+		lc->acaps = 0;
 	} else {
 		fw_caps =
-			lmm_to_fw_caps(link_ksettings->link_modes.advertising);
-
-		if (!(lc->supported & fw_caps))
+			 lmm_to_fw_caps(link_ksettings->link_modes.advertising);
+		if (!(lc->pcaps & fw_caps))
 			return -EINVAL;
-		lc->requested_speed = 0;
-		lc->advertising = fw_caps | FW_PORT_CAP_ANEG;
+		lc->speed_caps = 0;
+		lc->acaps = fw_caps | FW_PORT_CAP32_ANEG;
 	}
 	lc->autoneg = base->autoneg;
 
@@ -801,6 +825,104 @@ static int set_link_ksettings(struct net_device *dev,
 	return ret;
 }
 
+/* Translate the Firmware FEC value into the ethtool value. */
+static inline unsigned int fwcap_to_eth_fec(unsigned int fw_fec)
+{
+	unsigned int eth_fec = 0;
+
+	if (fw_fec & FW_PORT_CAP32_FEC_RS)
+		eth_fec |= ETHTOOL_FEC_RS;
+	if (fw_fec & FW_PORT_CAP32_FEC_BASER_RS)
+		eth_fec |= ETHTOOL_FEC_BASER;
+
+	/* if nothing is set, then FEC is off */
+	if (!eth_fec)
+		eth_fec = ETHTOOL_FEC_OFF;
+
+	return eth_fec;
+}
+
+/* Translate Common Code FEC value into ethtool value. */
+static inline unsigned int cc_to_eth_fec(unsigned int cc_fec)
+{
+	unsigned int eth_fec = 0;
+
+	if (cc_fec & FEC_AUTO)
+		eth_fec |= ETHTOOL_FEC_AUTO;
+	if (cc_fec & FEC_RS)
+		eth_fec |= ETHTOOL_FEC_RS;
+	if (cc_fec & FEC_BASER_RS)
+		eth_fec |= ETHTOOL_FEC_BASER;
+
+	/* if nothing is set, then FEC is off */
+	if (!eth_fec)
+		eth_fec = ETHTOOL_FEC_OFF;
+
+	return eth_fec;
+}
+
+/* Translate ethtool FEC value into Common Code value. */
+static inline unsigned int eth_to_cc_fec(unsigned int eth_fec)
+{
+	unsigned int cc_fec = 0;
+
+	if (eth_fec & ETHTOOL_FEC_OFF)
+		return cc_fec;
+
+	if (eth_fec & ETHTOOL_FEC_AUTO)
+		cc_fec |= FEC_AUTO;
+	if (eth_fec & ETHTOOL_FEC_RS)
+		cc_fec |= FEC_RS;
+	if (eth_fec & ETHTOOL_FEC_BASER)
+		cc_fec |= FEC_BASER_RS;
+
+	return cc_fec;
+}
+
+static int get_fecparam(struct net_device *dev, struct ethtool_fecparam *fec)
+{
+	const struct port_info *pi = netdev_priv(dev);
+	const struct link_config *lc = &pi->link_cfg;
+
+	/* Translate the Firmware FEC Support into the ethtool value.  We
+	 * always support IEEE 802.3 "automatic" selection of Link FEC type if
+	 * any FEC is supported.
+	 */
+	fec->fec = fwcap_to_eth_fec(lc->pcaps);
+	if (fec->fec != ETHTOOL_FEC_OFF)
+		fec->fec |= ETHTOOL_FEC_AUTO;
+
+	/* Translate the current internal FEC parameters into the
+	 * ethtool values.
+	 */
+	fec->active_fec = cc_to_eth_fec(lc->fec);
+
+	return 0;
+}
+
+static int set_fecparam(struct net_device *dev, struct ethtool_fecparam *fec)
+{
+	struct port_info *pi = netdev_priv(dev);
+	struct link_config *lc = &pi->link_cfg;
+	struct link_config old_lc;
+	int ret;
+
+	/* Save old Link Configuration in case the L1 Configure below
+	 * fails.
+	 */
+	old_lc = *lc;
+
+	/* Try to perform the L1 Configure and return the result of that
+	 * effort.  If it fails, revert the attempted change.
+	 */
+	lc->requested_fec = eth_to_cc_fec(fec->fec);
+	ret = t4_link_l1cfg(pi->adapter, pi->adapter->mbox,
+			    pi->tx_chan, lc);
+	if (ret)
+		*lc = old_lc;
+	return ret;
+}
+
 static void get_pauseparam(struct net_device *dev,
 			   struct ethtool_pauseparam *epause)
 {
@@ -819,7 +941,7 @@ static int set_pauseparam(struct net_device *dev,
 
 	if (epause->autoneg == AUTONEG_DISABLE)
 		lc->requested_fc = 0;
-	else if (lc->supported & FW_PORT_CAP_ANEG)
+	else if (lc->pcaps & FW_PORT_CAP32_ANEG)
 		lc->requested_fc = PAUSE_AUTONEG;
 	else
 		return -EINVAL;
@@ -1255,6 +1377,8 @@ static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
 static const struct ethtool_ops cxgb_ethtool_ops = {
 	.get_link_ksettings = get_link_ksettings,
 	.set_link_ksettings = set_link_ksettings,
+	.get_fecparam      = get_fecparam,
+	.set_fecparam      = set_fecparam,
 	.get_drvinfo       = get_drvinfo,
 	.get_msglevel      = get_msglevel,
 	.set_msglevel      = set_msglevel,
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index e403fa18f1b1..92d9d795d874 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -530,15 +530,22 @@ static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
 			FW_PORT_CMD_ACTION_G(ntohl(pcmd->action_to_len16));
 
 		if (cmd == FW_PORT_CMD &&
-		    action == FW_PORT_ACTION_GET_PORT_INFO) {
+		    (action == FW_PORT_ACTION_GET_PORT_INFO ||
+		     action == FW_PORT_ACTION_GET_PORT_INFO32)) {
 			int port = FW_PORT_CMD_PORTID_G(
 					be32_to_cpu(pcmd->op_to_portid));
-			struct net_device *dev =
-				q->adap->port[q->adap->chan_map[port]];
-			int state_input = ((pcmd->u.info.dcbxdis_pkd &
-					    FW_PORT_CMD_DCBXDIS_F)
-					   ? CXGB4_DCB_INPUT_FW_DISABLED
-					   : CXGB4_DCB_INPUT_FW_ENABLED);
+			struct net_device *dev;
+			int dcbxdis, state_input;
+
+			dev = q->adap->port[q->adap->chan_map[port]];
+			dcbxdis = (action == FW_PORT_ACTION_GET_PORT_INFO
+				   ? !!(pcmd->u.info.dcbxdis_pkd &
+					FW_PORT_CMD_DCBXDIS_F)
+				   : !!(pcmd->u.info32.lstatus32_to_cbllen32 &
+					FW_PORT_CMD_DCBXDIS32_F));
+			state_input = (dcbxdis
+				       ? CXGB4_DCB_INPUT_FW_DISABLED
+				       : CXGB4_DCB_INPUT_FW_ENABLED);
 
 			cxgb4_dcb_state_fsm(dev, state_input);
 		}
@@ -2672,11 +2679,10 @@ static int cxgb_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate,
 {
 	struct port_info *pi = netdev_priv(dev);
 	struct adapter *adap = pi->adapter;
-	struct fw_port_cmd port_cmd, port_rpl;
-	u32 link_status, speed = 0;
+	unsigned int link_ok, speed, mtu;
 	u32 fw_pfvf, fw_class;
 	int class_id = vf;
-	int link_ok, ret;
+	int ret;
 	u16 pktsize;
 
 	if (vf >= adap->num_vfs)
@@ -2688,41 +2694,18 @@ static int cxgb_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate,
 			min_tx_rate, vf);
 		return -EINVAL;
 	}
-	/* Retrieve link details for VF port */
-	memset(&port_cmd, 0, sizeof(port_cmd));
-	port_cmd.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PORT_CMD) |
-					    FW_CMD_REQUEST_F |
-					    FW_CMD_READ_F |
-					    FW_PORT_CMD_PORTID_V(pi->port_id));
-	port_cmd.action_to_len16 =
-		cpu_to_be32(FW_PORT_CMD_ACTION_V(FW_PORT_ACTION_GET_PORT_INFO) |
-			    FW_LEN16(port_cmd));
-	ret = t4_wr_mbox(adap, adap->mbox, &port_cmd, sizeof(port_cmd),
-			 &port_rpl);
+
+	ret = t4_get_link_params(pi, &link_ok, &speed, &mtu);
 	if (ret != FW_SUCCESS) {
 		dev_err(adap->pdev_dev,
-			"Failed to get link status for VF %d\n", vf);
+			"Failed to get link information for VF %d\n", vf);
 		return -EINVAL;
 	}
-	link_status = be32_to_cpu(port_rpl.u.info.lstatus_to_modtype);
-	link_ok = (link_status & FW_PORT_CMD_LSTATUS_F) != 0;
+
 	if (!link_ok) {
 		dev_err(adap->pdev_dev, "Link down for VF %d\n", vf);
 		return -EINVAL;
 	}
-	/* Determine link speed */
-	if (link_status & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_100M))
-		speed = 100;
-	else if (link_status & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_1G))
-		speed = 1000;
-	else if (link_status & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_10G))
-		speed = 10000;
-	else if (link_status & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_25G))
-		speed = 25000;
-	else if (link_status & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_40G))
-		speed = 40000;
-	else if (link_status & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_100G))
-		speed = 100000;
 
 	if (max_tx_rate > speed) {
 		dev_err(adap->pdev_dev,
@@ -2730,7 +2713,8 @@ static int cxgb_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate,
 			max_tx_rate, vf, speed);
 		return -EINVAL;
 	}
-	pktsize = be16_to_cpu(port_rpl.u.info.mtu);
+
+	pktsize = mtu;
 	/* subtract ethhdr size and 4 bytes crc since, f/w appends it */
 	pktsize = pktsize - sizeof(struct ethhdr) - 4;
 	/* subtract ipv4 hdr size, tcp hdr size to get typical IPv4 MSS size */
@@ -2741,7 +2725,7 @@ static int cxgb_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate,
 			      SCHED_CLASS_MODE_CLASS,
 			      SCHED_CLASS_RATEUNIT_BITS,
 			      SCHED_CLASS_RATEMODE_ABS,
-			      pi->port_id, class_id, 0,
+			      pi->tx_chan, class_id, 0,
 			      max_tx_rate * 1000, 0, pktsize);
 	if (ret) {
 		dev_err(adap->pdev_dev, "Err %d for Traffic Class config\n",
@@ -2889,14 +2873,29 @@ static int cxgb_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
 	return err;
 }
 
-static int cxgb_setup_tc(struct net_device *dev, u32 handle, u32 chain_index,
-			 __be16 proto, struct tc_to_netdev *tc)
+static int cxgb_setup_tc_cls_u32(struct net_device *dev,
+				 struct tc_cls_u32_offload *cls_u32)
 {
-	struct port_info *pi = netdev2pinfo(dev);
-	struct adapter *adap = netdev2adap(dev);
+	if (!is_classid_clsact_ingress(cls_u32->common.classid) ||
+	    cls_u32->common.chain_index)
+		return -EOPNOTSUPP;
 
-	if (chain_index)
+	switch (cls_u32->command) {
+	case TC_CLSU32_NEW_KNODE:
+	case TC_CLSU32_REPLACE_KNODE:
+		return cxgb4_config_knode(dev, cls_u32);
+	case TC_CLSU32_DELETE_KNODE:
+		return cxgb4_delete_knode(dev, cls_u32);
+	default:
 		return -EOPNOTSUPP;
+	}
+}
+
+static int cxgb_setup_tc(struct net_device *dev, enum tc_setup_type type,
+			 void *type_data)
+{
+	struct port_info *pi = netdev2pinfo(dev);
+	struct adapter *adap = netdev2adap(dev);
 
 	if (!(adap->flags & FULL_INIT_DONE)) {
 		dev_err(adap->pdev_dev,
@@ -2905,20 +2904,12 @@ static int cxgb_setup_tc(struct net_device *dev, u32 handle, u32 chain_index,
 		return -EINVAL;
 	}
 
-	if (TC_H_MAJ(handle) == TC_H_MAJ(TC_H_INGRESS) &&
-	    tc->type == TC_SETUP_CLSU32) {
-		switch (tc->cls_u32->command) {
-		case TC_CLSU32_NEW_KNODE:
-		case TC_CLSU32_REPLACE_KNODE:
-			return cxgb4_config_knode(dev, proto, tc->cls_u32);
-		case TC_CLSU32_DELETE_KNODE:
-			return cxgb4_delete_knode(dev, proto, tc->cls_u32);
-		default:
-			return -EOPNOTSUPP;
-		}
+	switch (type) {
+	case TC_SETUP_CLSU32:
+		return cxgb_setup_tc_cls_u32(dev, type_data);
+	default:
+		return -EOPNOTSUPP;
 	}
-
-	return -EOPNOTSUPP;
 }
 
 static netdev_features_t cxgb_fix_features(struct net_device *dev,
@@ -3610,11 +3601,8 @@ static int adap_init0(struct adapter *adap)
 	 * later reporting and B. to warn if the currently loaded firmware
 	 * is excessively mismatched relative to the driver.)
 	 */
-	t4_get_fw_version(adap, &adap->params.fw_vers);
-	t4_get_bs_version(adap, &adap->params.bs_vers);
-	t4_get_tp_version(adap, &adap->params.tp_vers);
-	t4_get_exprom_version(adap, &adap->params.er_vers);
 
+	t4_get_version_info(adap);
 	ret = t4_check_fw_version(adap);
 	/* If firmware is too old (not supported by driver) force an update. */
 	if (ret)
@@ -4204,8 +4192,9 @@ static inline bool is_x_10g_port(const struct link_config *lc)
 {
 	unsigned int speeds, high_speeds;
 
-	speeds = FW_PORT_CAP_SPEED_V(FW_PORT_CAP_SPEED_G(lc->supported));
-	high_speeds = speeds & ~(FW_PORT_CAP_SPEED_100M | FW_PORT_CAP_SPEED_1G);
+	speeds = FW_PORT_CAP32_SPEED_V(FW_PORT_CAP32_SPEED_G(lc->pcaps));
+	high_speeds = speeds &
+			~(FW_PORT_CAP32_SPEED_100M | FW_PORT_CAP32_SPEED_1G);
 
 	return high_speeds != 0;
 }
@@ -4560,56 +4549,8 @@ static void cxgb4_check_pcie_caps(struct adapter *adap)
 /* Dump basic information about the adapter */
 static void print_adapter_info(struct adapter *adapter)
 {
-	/* Device information */
-	dev_info(adapter->pdev_dev, "Chelsio %s rev %d\n",
-		 adapter->params.vpd.id,
-		 CHELSIO_CHIP_RELEASE(adapter->params.chip));
-	dev_info(adapter->pdev_dev, "S/N: %s, P/N: %s\n",
-		 adapter->params.vpd.sn, adapter->params.vpd.pn);
-
-	/* Firmware Version */
-	if (!adapter->params.fw_vers)
-		dev_warn(adapter->pdev_dev, "No firmware loaded\n");
-	else
-		dev_info(adapter->pdev_dev, "Firmware version: %u.%u.%u.%u\n",
-			 FW_HDR_FW_VER_MAJOR_G(adapter->params.fw_vers),
-			 FW_HDR_FW_VER_MINOR_G(adapter->params.fw_vers),
-			 FW_HDR_FW_VER_MICRO_G(adapter->params.fw_vers),
-			 FW_HDR_FW_VER_BUILD_G(adapter->params.fw_vers));
-
-	/* Bootstrap Firmware Version. (Some adapters don't have Bootstrap
-	 * Firmware, so dev_info() is more appropriate here.)
-	 */
-	if (!adapter->params.bs_vers)
-		dev_info(adapter->pdev_dev, "No bootstrap loaded\n");
-	else
-		dev_info(adapter->pdev_dev, "Bootstrap version: %u.%u.%u.%u\n",
-			 FW_HDR_FW_VER_MAJOR_G(adapter->params.bs_vers),
-			 FW_HDR_FW_VER_MINOR_G(adapter->params.bs_vers),
-			 FW_HDR_FW_VER_MICRO_G(adapter->params.bs_vers),
-			 FW_HDR_FW_VER_BUILD_G(adapter->params.bs_vers));
-
-	/* TP Microcode Version */
-	if (!adapter->params.tp_vers)
-		dev_warn(adapter->pdev_dev, "No TP Microcode loaded\n");
-	else
-		dev_info(adapter->pdev_dev,
-			 "TP Microcode version: %u.%u.%u.%u\n",
-			 FW_HDR_FW_VER_MAJOR_G(adapter->params.tp_vers),
-			 FW_HDR_FW_VER_MINOR_G(adapter->params.tp_vers),
-			 FW_HDR_FW_VER_MICRO_G(adapter->params.tp_vers),
-			 FW_HDR_FW_VER_BUILD_G(adapter->params.tp_vers));
-
-	/* Expansion ROM version */
-	if (!adapter->params.er_vers)
-		dev_info(adapter->pdev_dev, "No Expansion ROM loaded\n");
-	else
-		dev_info(adapter->pdev_dev,
-			 "Expansion ROM version: %u.%u.%u.%u\n",
-			 FW_HDR_FW_VER_MAJOR_G(adapter->params.er_vers),
-			 FW_HDR_FW_VER_MINOR_G(adapter->params.er_vers),
-			 FW_HDR_FW_VER_MICRO_G(adapter->params.er_vers),
-			 FW_HDR_FW_VER_BUILD_G(adapter->params.er_vers));
+	/* Hardware/Firmware/etc. Version/Revision IDs */
+	t4_dump_version_info(adapter);
 
 	/* Software/Hardware configuration */
 	dev_info(adapter->pdev_dev, "Configuration: %sNIC %s, %s capable\n",
@@ -4634,18 +4575,24 @@ static void print_port_info(const struct net_device *dev)
 	else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_8_0GB)
 		spd = " 8 GT/s";
 
-	if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
+	if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_100M)
 		bufp += sprintf(bufp, "100M/");
-	if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
+	if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_1G)
 		bufp += sprintf(bufp, "1G/");
-	if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
+	if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_10G)
 		bufp += sprintf(bufp, "10G/");
-	if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_25G)
+	if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_25G)
 		bufp += sprintf(bufp, "25G/");
-	if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G)
+	if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_40G)
 		bufp += sprintf(bufp, "40G/");
-	if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100G)
+	if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_50G)
+		bufp += sprintf(bufp, "50G/");
+	if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_100G)
 		bufp += sprintf(bufp, "100G/");
+	if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_200G)
+		bufp += sprintf(bufp, "200G/");
+	if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_400G)
+		bufp += sprintf(bufp, "400G/");
 	if (bufp != buf)
 		--bufp;
 	sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type));
@@ -4654,11 +4601,6 @@ static void print_port_info(const struct net_device *dev)
 		    dev->name, adap->params.vpd.id, adap->name, buf);
 }
 
-static void enable_pcie_relaxed_ordering(struct pci_dev *dev)
-{
-	pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
-}
-
 /*
  * Free the following resources:
  * - memory used for tables
@@ -4756,10 +4698,11 @@ static int config_mgmt_dev(struct pci_dev *pdev)
 
 	pi = netdev_priv(netdev);
 	pi->adapter = adap;
-	pi->port_id = adap->pf % adap->params.nports;
+	pi->tx_chan = adap->pf % adap->params.nports;
 	SET_NETDEV_DEV(netdev, &pdev->dev);
 
 	adap->port[0] = netdev;
+	pi->port_id = 0;
 
 	err = register_netdev(adap->port[0]);
 	if (err) {
@@ -4908,7 +4851,6 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	}
 
 	pci_enable_pcie_error_reporting(pdev);
-	enable_pcie_relaxed_ordering(pdev);
 	pci_set_master(pdev);
 	pci_save_state(pdev);
 
@@ -4947,6 +4889,23 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	adapter->msg_enable = DFLT_MSG_ENABLE;
 	memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
 
+	/* If possible, we use PCIe Relaxed Ordering Attribute to deliver
+	 * Ingress Packet Data to Free List Buffers in order to allow for
+	 * chipset performance optimizations between the Root Complex and
+	 * Memory Controllers.  (Messages to the associated Ingress Queue
+	 * notifying new Packet Placement in the Free Lists Buffers will be
+	 * send without the Relaxed Ordering Attribute thus guaranteeing that
+	 * all preceding PCIe Transaction Layer Packets will be processed
+	 * first.)  But some Root Complexes have various issues with Upstream
+	 * Transaction Layer Packets with the Relaxed Ordering Attribute set.
+	 * The PCIe devices which under the Root Complexes will be cleared the
+	 * Relaxed Ordering bit in the configuration space, So we check our
+	 * PCIe configuration space to see if it's flagged with advice against
+	 * using Relaxed Ordering.
+	 */
+	if (!pcie_relaxed_ordering_enabled(pdev))
+		adapter->flags |= ROOT_NO_RELAXED_ORDERING;
+
 	spin_lock_init(&adapter->stats_lock);
 	spin_lock_init(&adapter->tid_release_lock);
 	spin_lock_init(&adapter->win0_lock);
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32.c
index ef06ce8247ab..48970ba08bdc 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32.c
@@ -96,7 +96,7 @@ static int fill_action_fields(struct adapter *adap,
 	LIST_HEAD(actions);
 
 	exts = cls->knode.exts;
-	if (tc_no_actions(exts))
+	if (!tcf_exts_has_actions(exts))
 		return -EINVAL;
 
 	tcf_exts_to_list(exts, &actions);
@@ -146,11 +146,11 @@ static int fill_action_fields(struct adapter *adap,
 	return 0;
 }
 
-int cxgb4_config_knode(struct net_device *dev, __be16 protocol,
-		       struct tc_cls_u32_offload *cls)
+int cxgb4_config_knode(struct net_device *dev, struct tc_cls_u32_offload *cls)
 {
 	const struct cxgb4_match_field *start, *link_start = NULL;
 	struct adapter *adapter = netdev2adap(dev);
+	__be16 protocol = cls->common.protocol;
 	struct ch_filter_specification fs;
 	struct cxgb4_tc_u32_table *t;
 	struct cxgb4_link *link;
@@ -338,8 +338,7 @@ out:
 	return ret;
 }
 
-int cxgb4_delete_knode(struct net_device *dev, __be16 protocol,
-		       struct tc_cls_u32_offload *cls)
+int cxgb4_delete_knode(struct net_device *dev, struct tc_cls_u32_offload *cls)
 {
 	struct adapter *adapter = netdev2adap(dev);
 	unsigned int filter_id, max_tids, i, j;
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32.h
index 021261a41c13..70a07b7cca56 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32.h
@@ -44,10 +44,8 @@ static inline bool can_tc_u32_offload(struct net_device *dev)
 	return (dev->features & NETIF_F_HW_TC) && adap->tc_u32 ? true : false;
 }
 
-int cxgb4_config_knode(struct net_device *dev, __be16 protocol,
-		       struct tc_cls_u32_offload *cls);
-int cxgb4_delete_knode(struct net_device *dev, __be16 protocol,
-		       struct tc_cls_u32_offload *cls);
+int cxgb4_config_knode(struct net_device *dev, struct tc_cls_u32_offload *cls);
+int cxgb4_delete_knode(struct net_device *dev, struct tc_cls_u32_offload *cls);
 
 void cxgb4_cleanup_tc_u32(struct adapter *adapter);
 struct cxgb4_tc_u32_table *cxgb4_init_tc_u32(struct adapter *adap);
diff --git a/drivers/net/ethernet/chelsio/cxgb4/sched.c b/drivers/net/ethernet/chelsio/cxgb4/sched.c
index 02acff741f11..9148abb7994c 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/sched.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/sched.c
@@ -533,10 +533,10 @@ struct sched_table *t4_init_sched(unsigned int sched_size)
 void t4_cleanup_sched(struct adapter *adap)
 {
 	struct sched_table *s;
-	unsigned int i;
+	unsigned int j, i;
 
-	for_each_port(adap, i) {
-		struct port_info *pi = netdev2pinfo(adap->port[i]);
+	for_each_port(adap, j) {
+		struct port_info *pi = netdev2pinfo(adap->port[j]);
 
 		s = pi->sched_tbl;
 		for (i = 0; i < s->sched_size; i++) {
diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c b/drivers/net/ethernet/chelsio/cxgb4/sge.c
index ede12209f20b..4ef68f69b58c 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/sge.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c
@@ -2719,6 +2719,7 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq,
 	struct fw_iq_cmd c;
 	struct sge *s = &adap->sge;
 	struct port_info *pi = netdev_priv(dev);
+	int relaxed = !(adap->flags & ROOT_NO_RELAXED_ORDERING);
 
 	/* Size needs to be multiple of 16, including status entry. */
 	iq->size = roundup(iq->size, 16);
@@ -2772,8 +2773,8 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq,
 
 		flsz = fl->size / 8 + s->stat_len / sizeof(struct tx_desc);
 		c.iqns_to_fl0congen |= htonl(FW_IQ_CMD_FL0PACKEN_F |
-					     FW_IQ_CMD_FL0FETCHRO_F |
-					     FW_IQ_CMD_FL0DATARO_F |
+					     FW_IQ_CMD_FL0FETCHRO_V(relaxed) |
+					     FW_IQ_CMD_FL0DATARO_V(relaxed) |
 					     FW_IQ_CMD_FL0PADEN_F);
 		if (cong >= 0)
 			c.iqns_to_fl0congen |=
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c
index 82bf7aac6cdb..b65ce26ff72f 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c
@@ -369,12 +369,12 @@ int t4_wr_mbox_meat_timeout(struct adapter *adap, int mbox, const void *cmd,
 		list_del(&entry.list);
 		spin_unlock(&adap->mbox_lock);
 		ret = (v == MBOX_OWNER_FW) ? -EBUSY : -ETIMEDOUT;
-		t4_record_mbox(adap, cmd, MBOX_LEN, access, ret);
+		t4_record_mbox(adap, cmd, size, access, ret);
 		return ret;
 	}
 
 	/* Copy in the new mailbox command and send it on its way ... */
-	t4_record_mbox(adap, cmd, MBOX_LEN, access, 0);
+	t4_record_mbox(adap, cmd, size, access, 0);
 	for (i = 0; i < size; i += 8)
 		t4_write_reg64(adap, data_reg + i, be64_to_cpu(*p++));
 
@@ -426,7 +426,7 @@ int t4_wr_mbox_meat_timeout(struct adapter *adap, int mbox, const void *cmd,
 	}
 
 	ret = (pcie_fw & PCIE_FW_ERR_F) ? -ENXIO : -ETIMEDOUT;
-	t4_record_mbox(adap, cmd, MBOX_LEN, access, ret);
+	t4_record_mbox(adap, cmd, size, access, ret);
 	dev_err(adap->pdev_dev, "command %#x in mailbox %d timed out\n",
 		*(const u8 *)cmd, mbox);
 	t4_report_fw_error(adap);
@@ -913,7 +913,8 @@ void t4_get_regs(struct adapter *adap, void *buf, size_t buf_size)
 		0xd010, 0xd03c,
 		0xdfc0, 0xdfe0,
 		0xe000, 0xea7c,
-		0xf000, 0x11190,
+		0xf000, 0x11110,
+		0x11118, 0x11190,
 		0x19040, 0x1906c,
 		0x19078, 0x19080,
 		0x1908c, 0x190e4,
@@ -1439,8 +1440,6 @@ void t4_get_regs(struct adapter *adap, void *buf, size_t buf_size)
 		0x1ff00, 0x1ff84,
 		0x1ffc0, 0x1ffc8,
 		0x30000, 0x30030,
-		0x30038, 0x30038,
-		0x30040, 0x30040,
 		0x30100, 0x30144,
 		0x30190, 0x301a0,
 		0x301a8, 0x301b8,
@@ -1551,8 +1550,6 @@ void t4_get_regs(struct adapter *adap, void *buf, size_t buf_size)
 		0x33c3c, 0x33c50,
 		0x33cf0, 0x33cfc,
 		0x34000, 0x34030,
-		0x34038, 0x34038,
-		0x34040, 0x34040,
 		0x34100, 0x34144,
 		0x34190, 0x341a0,
 		0x341a8, 0x341b8,
@@ -1663,8 +1660,6 @@ void t4_get_regs(struct adapter *adap, void *buf, size_t buf_size)
 		0x37c3c, 0x37c50,
 		0x37cf0, 0x37cfc,
 		0x38000, 0x38030,
-		0x38038, 0x38038,
-		0x38040, 0x38040,
 		0x38100, 0x38144,
 		0x38190, 0x381a0,
 		0x381a8, 0x381b8,
@@ -1775,8 +1770,6 @@ void t4_get_regs(struct adapter *adap, void *buf, size_t buf_size)
 		0x3bc3c, 0x3bc50,
 		0x3bcf0, 0x3bcfc,
 		0x3c000, 0x3c030,
-		0x3c038, 0x3c038,
-		0x3c040, 0x3c040,
 		0x3c100, 0x3c144,
 		0x3c190, 0x3c1a0,
 		0x3c1a8, 0x3c1b8,
@@ -2040,12 +2033,8 @@ void t4_get_regs(struct adapter *adap, void *buf, size_t buf_size)
 		0x1190, 0x1194,
 		0x11a0, 0x11a4,
 		0x11b0, 0x11b4,
-		0x11fc, 0x1258,
-		0x1280, 0x12d4,
-		0x12d9, 0x12d9,
-		0x12de, 0x12de,
-		0x12e3, 0x12e3,
-		0x12e8, 0x133c,
+		0x11fc, 0x1274,
+		0x1280, 0x133c,
 		0x1800, 0x18fc,
 		0x3000, 0x302c,
 		0x3060, 0x30b0,
@@ -2076,6 +2065,9 @@ void t4_get_regs(struct adapter *adap, void *buf, size_t buf_size)
 		0x5ea0, 0x5eb0,
 		0x5ec0, 0x5ec0,
 		0x5ec8, 0x5ed0,
+		0x5ee0, 0x5ee0,
+		0x5ef0, 0x5ef0,
+		0x5f00, 0x5f00,
 		0x6000, 0x6020,
 		0x6028, 0x6040,
 		0x6058, 0x609c,
@@ -2133,6 +2125,8 @@ void t4_get_regs(struct adapter *adap, void *buf, size_t buf_size)
 		0xd300, 0xd31c,
 		0xdfc0, 0xdfe0,
 		0xe000, 0xf008,
+		0xf010, 0xf018,
+		0xf020, 0xf028,
 		0x11000, 0x11014,
 		0x11048, 0x1106c,
 		0x11074, 0x11088,
@@ -2256,13 +2250,6 @@ void t4_get_regs(struct adapter *adap, void *buf, size_t buf_size)
 		0x1ff00, 0x1ff84,
 		0x1ffc0, 0x1ffc8,
 		0x30000, 0x30030,
-		0x30038, 0x30038,
-		0x30040, 0x30040,
-		0x30048, 0x30048,
-		0x30050, 0x30050,
-		0x3005c, 0x30060,
-		0x30068, 0x30068,
-		0x30070, 0x30070,
 		0x30100, 0x30168,
 		0x30190, 0x301a0,
 		0x301a8, 0x301b8,
@@ -2325,13 +2312,12 @@ void t4_get_regs(struct adapter *adap, void *buf, size_t buf_size)
 		0x326a8, 0x326a8,
 		0x326ec, 0x326ec,
 		0x32a00, 0x32abc,
-		0x32b00, 0x32b38,
+		0x32b00, 0x32b18,
+		0x32b20, 0x32b38,
 		0x32b40, 0x32b58,
 		0x32b60, 0x32b78,
 		0x32c00, 0x32c00,
 		0x32c08, 0x32c3c,
-		0x32e00, 0x32e2c,
-		0x32f00, 0x32f2c,
 		0x33000, 0x3302c,
 		0x33034, 0x33050,
 		0x33058, 0x33058,
@@ -2396,13 +2382,6 @@ void t4_get_regs(struct adapter *adap, void *buf, size_t buf_size)
 		0x33c38, 0x33c50,
 		0x33cf0, 0x33cfc,
 		0x34000, 0x34030,
-		0x34038, 0x34038,
-		0x34040, 0x34040,
-		0x34048, 0x34048,
-		0x34050, 0x34050,
-		0x3405c, 0x34060,
-		0x34068, 0x34068,
-		0x34070, 0x34070,
 		0x34100, 0x34168,
 		0x34190, 0x341a0,
 		0x341a8, 0x341b8,
@@ -2465,13 +2444,12 @@ void t4_get_regs(struct adapter *adap, void *buf, size_t buf_size)
 		0x366a8, 0x366a8,
 		0x366ec, 0x366ec,
 		0x36a00, 0x36abc,
-		0x36b00, 0x36b38,
+		0x36b00, 0x36b18,
+		0x36b20, 0x36b38,
 		0x36b40, 0x36b58,
 		0x36b60, 0x36b78,
 		0x36c00, 0x36c00,
 		0x36c08, 0x36c3c,
-		0x36e00, 0x36e2c,
-		0x36f00, 0x36f2c,
 		0x37000, 0x3702c,
 		0x37034, 0x37050,
 		0x37058, 0x37058,
@@ -2545,8 +2523,7 @@ void t4_get_regs(struct adapter *adap, void *buf, size_t buf_size)
 		0x40280, 0x40280,
 		0x40304, 0x40304,
 		0x40330, 0x4033c,
-		0x41304, 0x413b8,
-		0x413c0, 0x413c8,
+		0x41304, 0x413c8,
 		0x413d0, 0x413dc,
 		0x413f0, 0x413f0,
 		0x41400, 0x4140c,
@@ -3100,6 +3077,179 @@ int t4_get_exprom_version(struct adapter *adap, u32 *vers)
 }
 
 /**
+ *      t4_get_vpd_version - return the VPD version
+ *      @adapter: the adapter
+ *      @vers: where to place the version
+ *
+ *      Reads the VPD via the Firmware interface (thus this can only be called
+ *      once we're ready to issue Firmware commands).  The format of the
+ *      VPD version is adapter specific.  Returns 0 on success, an error on
+ *      failure.
+ *
+ *      Note that early versions of the Firmware didn't include the ability
+ *      to retrieve the VPD version, so we zero-out the return-value parameter
+ *      in that case to avoid leaving it with garbage in it.
+ *
+ *      Also note that the Firmware will return its cached copy of the VPD
+ *      Revision ID, not the actual Revision ID as written in the Serial
+ *      EEPROM.  This is only an issue if a new VPD has been written and the
+ *      Firmware/Chip haven't yet gone through a RESET sequence.  So it's best
+ *      to defer calling this routine till after a FW_RESET_CMD has been issued
+ *      if the Host Driver will be performing a full adapter initialization.
+ */
+int t4_get_vpd_version(struct adapter *adapter, u32 *vers)
+{
+	u32 vpdrev_param;
+	int ret;
+
+	vpdrev_param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
+			FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_VPDREV));
+	ret = t4_query_params(adapter, adapter->mbox, adapter->pf, 0,
+			      1, &vpdrev_param, vers);
+	if (ret)
+		*vers = 0;
+	return ret;
+}
+
+/**
+ *      t4_get_scfg_version - return the Serial Configuration version
+ *      @adapter: the adapter
+ *      @vers: where to place the version
+ *
+ *      Reads the Serial Configuration Version via the Firmware interface
+ *      (thus this can only be called once we're ready to issue Firmware
+ *      commands).  The format of the Serial Configuration version is
+ *      adapter specific.  Returns 0 on success, an error on failure.
+ *
+ *      Note that early versions of the Firmware didn't include the ability
+ *      to retrieve the Serial Configuration version, so we zero-out the
+ *      return-value parameter in that case to avoid leaving it with
+ *      garbage in it.
+ *
+ *      Also note that the Firmware will return its cached copy of the Serial
+ *      Initialization Revision ID, not the actual Revision ID as written in
+ *      the Serial EEPROM.  This is only an issue if a new VPD has been written
+ *      and the Firmware/Chip haven't yet gone through a RESET sequence.  So
+ *      it's best to defer calling this routine till after a FW_RESET_CMD has
+ *      been issued if the Host Driver will be performing a full adapter
+ *      initialization.
+ */
+int t4_get_scfg_version(struct adapter *adapter, u32 *vers)
+{
+	u32 scfgrev_param;
+	int ret;
+
+	scfgrev_param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
+			 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_SCFGREV));
+	ret = t4_query_params(adapter, adapter->mbox, adapter->pf, 0,
+			      1, &scfgrev_param, vers);
+	if (ret)
+		*vers = 0;
+	return ret;
+}
+
+/**
+ *      t4_get_version_info - extract various chip/firmware version information
+ *      @adapter: the adapter
+ *
+ *      Reads various chip/firmware version numbers and stores them into the
+ *      adapter Adapter Parameters structure.  If any of the efforts fails
+ *      the first failure will be returned, but all of the version numbers
+ *      will be read.
+ */
+int t4_get_version_info(struct adapter *adapter)
+{
+	int ret = 0;
+
+	#define FIRST_RET(__getvinfo) \
+	do { \
+		int __ret = __getvinfo; \
+		if (__ret && !ret) \
+			ret = __ret; \
+	} while (0)
+
+	FIRST_RET(t4_get_fw_version(adapter, &adapter->params.fw_vers));
+	FIRST_RET(t4_get_bs_version(adapter, &adapter->params.bs_vers));
+	FIRST_RET(t4_get_tp_version(adapter, &adapter->params.tp_vers));
+	FIRST_RET(t4_get_exprom_version(adapter, &adapter->params.er_vers));
+	FIRST_RET(t4_get_scfg_version(adapter, &adapter->params.scfg_vers));
+	FIRST_RET(t4_get_vpd_version(adapter, &adapter->params.vpd_vers));
+
+	#undef FIRST_RET
+	return ret;
+}
+
+/**
+ *      t4_dump_version_info - dump all of the adapter configuration IDs
+ *      @adapter: the adapter
+ *
+ *      Dumps all of the various bits of adapter configuration version/revision
+ *      IDs information.  This is typically called at some point after
+ *      t4_get_version_info() has been called.
+ */
+void t4_dump_version_info(struct adapter *adapter)
+{
+	/* Device information */
+	dev_info(adapter->pdev_dev, "Chelsio %s rev %d\n",
+		 adapter->params.vpd.id,
+		 CHELSIO_CHIP_RELEASE(adapter->params.chip));
+	dev_info(adapter->pdev_dev, "S/N: %s, P/N: %s\n",
+		 adapter->params.vpd.sn, adapter->params.vpd.pn);
+
+	/* Firmware Version */
+	if (!adapter->params.fw_vers)
+		dev_warn(adapter->pdev_dev, "No firmware loaded\n");
+	else
+		dev_info(adapter->pdev_dev, "Firmware version: %u.%u.%u.%u\n",
+			 FW_HDR_FW_VER_MAJOR_G(adapter->params.fw_vers),
+			 FW_HDR_FW_VER_MINOR_G(adapter->params.fw_vers),
+			 FW_HDR_FW_VER_MICRO_G(adapter->params.fw_vers),
+			 FW_HDR_FW_VER_BUILD_G(adapter->params.fw_vers));
+
+	/* Bootstrap Firmware Version. (Some adapters don't have Bootstrap
+	 * Firmware, so dev_info() is more appropriate here.)
+	 */
+	if (!adapter->params.bs_vers)
+		dev_info(adapter->pdev_dev, "No bootstrap loaded\n");
+	else
+		dev_info(adapter->pdev_dev, "Bootstrap version: %u.%u.%u.%u\n",
+			 FW_HDR_FW_VER_MAJOR_G(adapter->params.bs_vers),
+			 FW_HDR_FW_VER_MINOR_G(adapter->params.bs_vers),
+			 FW_HDR_FW_VER_MICRO_G(adapter->params.bs_vers),
+			 FW_HDR_FW_VER_BUILD_G(adapter->params.bs_vers));
+
+	/* TP Microcode Version */
+	if (!adapter->params.tp_vers)
+		dev_warn(adapter->pdev_dev, "No TP Microcode loaded\n");
+	else
+		dev_info(adapter->pdev_dev,
+			 "TP Microcode version: %u.%u.%u.%u\n",
+			 FW_HDR_FW_VER_MAJOR_G(adapter->params.tp_vers),
+			 FW_HDR_FW_VER_MINOR_G(adapter->params.tp_vers),
+			 FW_HDR_FW_VER_MICRO_G(adapter->params.tp_vers),
+			 FW_HDR_FW_VER_BUILD_G(adapter->params.tp_vers));
+
+	/* Expansion ROM version */
+	if (!adapter->params.er_vers)
+		dev_info(adapter->pdev_dev, "No Expansion ROM loaded\n");
+	else
+		dev_info(adapter->pdev_dev,
+			 "Expansion ROM version: %u.%u.%u.%u\n",
+			 FW_HDR_FW_VER_MAJOR_G(adapter->params.er_vers),
+			 FW_HDR_FW_VER_MINOR_G(adapter->params.er_vers),
+			 FW_HDR_FW_VER_MICRO_G(adapter->params.er_vers),
+			 FW_HDR_FW_VER_BUILD_G(adapter->params.er_vers));
+
+	/* Serial Configuration version */
+	dev_info(adapter->pdev_dev, "Serial Configuration version: %#x\n",
+		 adapter->params.scfg_vers);
+
+	/* VPD Version */
+	dev_info(adapter->pdev_dev, "VPD version: %#x\n",
+		 adapter->params.vpd_vers);
+}
+
+/**
  *	t4_check_fw_version - check if the FW is supported with this driver
  *	@adap: the adapter
  *
@@ -3685,16 +3835,143 @@ void t4_ulprx_read_la(struct adapter *adap, u32 *la_buf)
 	}
 }
 
-#define ADVERT_MASK (FW_PORT_CAP_SPEED_100M | FW_PORT_CAP_SPEED_1G |\
-		     FW_PORT_CAP_SPEED_10G | FW_PORT_CAP_SPEED_25G | \
-		     FW_PORT_CAP_SPEED_40G | FW_PORT_CAP_SPEED_100G | \
-		     FW_PORT_CAP_ANEG)
+#define ADVERT_MASK (FW_PORT_CAP32_SPEED_V(FW_PORT_CAP32_SPEED_M) | \
+		     FW_PORT_CAP32_ANEG)
+
+/**
+ *	fwcaps16_to_caps32 - convert 16-bit Port Capabilities to 32-bits
+ *	@caps16: a 16-bit Port Capabilities value
+ *
+ *	Returns the equivalent 32-bit Port Capabilities value.
+ */
+static fw_port_cap32_t fwcaps16_to_caps32(fw_port_cap16_t caps16)
+{
+	fw_port_cap32_t caps32 = 0;
+
+	#define CAP16_TO_CAP32(__cap) \
+		do { \
+			if (caps16 & FW_PORT_CAP_##__cap) \
+				caps32 |= FW_PORT_CAP32_##__cap; \
+		} while (0)
+
+	CAP16_TO_CAP32(SPEED_100M);
+	CAP16_TO_CAP32(SPEED_1G);
+	CAP16_TO_CAP32(SPEED_25G);
+	CAP16_TO_CAP32(SPEED_10G);
+	CAP16_TO_CAP32(SPEED_40G);
+	CAP16_TO_CAP32(SPEED_100G);
+	CAP16_TO_CAP32(FC_RX);
+	CAP16_TO_CAP32(FC_TX);
+	CAP16_TO_CAP32(ANEG);
+	CAP16_TO_CAP32(MDIX);
+	CAP16_TO_CAP32(MDIAUTO);
+	CAP16_TO_CAP32(FEC_RS);
+	CAP16_TO_CAP32(FEC_BASER_RS);
+	CAP16_TO_CAP32(802_3_PAUSE);
+	CAP16_TO_CAP32(802_3_ASM_DIR);
+
+	#undef CAP16_TO_CAP32
+
+	return caps32;
+}
+
+/**
+ *	fwcaps32_to_caps16 - convert 32-bit Port Capabilities to 16-bits
+ *	@caps32: a 32-bit Port Capabilities value
+ *
+ *	Returns the equivalent 16-bit Port Capabilities value.  Note that
+ *	not all 32-bit Port Capabilities can be represented in the 16-bit
+ *	Port Capabilities and some fields/values may not make it.
+ */
+static fw_port_cap16_t fwcaps32_to_caps16(fw_port_cap32_t caps32)
+{
+	fw_port_cap16_t caps16 = 0;
+
+	#define CAP32_TO_CAP16(__cap) \
+		do { \
+			if (caps32 & FW_PORT_CAP32_##__cap) \
+				caps16 |= FW_PORT_CAP_##__cap; \
+		} while (0)
+
+	CAP32_TO_CAP16(SPEED_100M);
+	CAP32_TO_CAP16(SPEED_1G);
+	CAP32_TO_CAP16(SPEED_10G);
+	CAP32_TO_CAP16(SPEED_25G);
+	CAP32_TO_CAP16(SPEED_40G);
+	CAP32_TO_CAP16(SPEED_100G);
+	CAP32_TO_CAP16(FC_RX);
+	CAP32_TO_CAP16(FC_TX);
+	CAP32_TO_CAP16(802_3_PAUSE);
+	CAP32_TO_CAP16(802_3_ASM_DIR);
+	CAP32_TO_CAP16(ANEG);
+	CAP32_TO_CAP16(MDIX);
+	CAP32_TO_CAP16(MDIAUTO);
+	CAP32_TO_CAP16(FEC_RS);
+	CAP32_TO_CAP16(FEC_BASER_RS);
+
+	#undef CAP32_TO_CAP16
+
+	return caps16;
+}
+
+/* Translate Firmware Port Capabilities Pause specification to Common Code */
+static inline enum cc_pause fwcap_to_cc_pause(fw_port_cap32_t fw_pause)
+{
+	enum cc_pause cc_pause = 0;
+
+	if (fw_pause & FW_PORT_CAP32_FC_RX)
+		cc_pause |= PAUSE_RX;
+	if (fw_pause & FW_PORT_CAP32_FC_TX)
+		cc_pause |= PAUSE_TX;
+
+	return cc_pause;
+}
+
+/* Translate Common Code Pause specification into Firmware Port Capabilities */
+static inline fw_port_cap32_t cc_to_fwcap_pause(enum cc_pause cc_pause)
+{
+	fw_port_cap32_t fw_pause = 0;
+
+	if (cc_pause & PAUSE_RX)
+		fw_pause |= FW_PORT_CAP32_FC_RX;
+	if (cc_pause & PAUSE_TX)
+		fw_pause |= FW_PORT_CAP32_FC_TX;
+
+	return fw_pause;
+}
+
+/* Translate Firmware Forward Error Correction specification to Common Code */
+static inline enum cc_fec fwcap_to_cc_fec(fw_port_cap32_t fw_fec)
+{
+	enum cc_fec cc_fec = 0;
+
+	if (fw_fec & FW_PORT_CAP32_FEC_RS)
+		cc_fec |= FEC_RS;
+	if (fw_fec & FW_PORT_CAP32_FEC_BASER_RS)
+		cc_fec |= FEC_BASER_RS;
+
+	return cc_fec;
+}
+
+/* Translate Common Code Forward Error Correction specification to Firmware */
+static inline fw_port_cap32_t cc_to_fwcap_fec(enum cc_fec cc_fec)
+{
+	fw_port_cap32_t fw_fec = 0;
+
+	if (cc_fec & FEC_RS)
+		fw_fec |= FW_PORT_CAP32_FEC_RS;
+	if (cc_fec & FEC_BASER_RS)
+		fw_fec |= FW_PORT_CAP32_FEC_BASER_RS;
+
+	return fw_fec;
+}
 
 /**
  *	t4_link_l1cfg - apply link configuration to MAC/PHY
- *	@phy: the PHY to setup
- *	@mac: the MAC to setup
- *	@lc: the requested link configuration
+ *	@adapter: the adapter
+ *	@mbox: the Firmware Mailbox to use
+ *	@port: the Port ID
+ *	@lc: the Port's Link Configuration
  *
  *	Set up a port's MAC and PHY according to a desired link configuration.
  *	- If the PHY can auto-negotiate first decide what to advertise, then
@@ -3703,47 +3980,64 @@ void t4_ulprx_read_la(struct adapter *adap, u32 *la_buf)
  *	- If auto-negotiation is off set the MAC to the proper speed/duplex/FC,
  *	  otherwise do it later based on the outcome of auto-negotiation.
  */
-int t4_link_l1cfg(struct adapter *adap, unsigned int mbox, unsigned int port,
-		  struct link_config *lc)
+int t4_link_l1cfg(struct adapter *adapter, unsigned int mbox,
+		  unsigned int port, struct link_config *lc)
 {
-	struct fw_port_cmd c;
-	unsigned int mdi = FW_PORT_CAP_MDI_V(FW_PORT_CAP_MDI_AUTO);
-	unsigned int fc = 0, fec = 0, fw_fec = 0;
+	unsigned int fw_caps = adapter->params.fw_caps_support;
+	struct fw_port_cmd cmd;
+	unsigned int fw_mdi = FW_PORT_CAP32_MDI_V(FW_PORT_CAP32_MDI_AUTO);
+	fw_port_cap32_t fw_fc, cc_fec, fw_fec, rcap;
 
 	lc->link_ok = 0;
-	if (lc->requested_fc & PAUSE_RX)
-		fc |= FW_PORT_CAP_FC_RX;
-	if (lc->requested_fc & PAUSE_TX)
-		fc |= FW_PORT_CAP_FC_TX;
-
-	fec = lc->requested_fec & FEC_AUTO ? lc->auto_fec : lc->requested_fec;
-
-	if (fec & FEC_RS)
-		fw_fec |= FW_PORT_CAP_FEC_RS;
-	if (fec & FEC_BASER_RS)
-		fw_fec |= FW_PORT_CAP_FEC_BASER_RS;
 
-	memset(&c, 0, sizeof(c));
-	c.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PORT_CMD) |
-				     FW_CMD_REQUEST_F | FW_CMD_EXEC_F |
-				     FW_PORT_CMD_PORTID_V(port));
-	c.action_to_len16 =
-		cpu_to_be32(FW_PORT_CMD_ACTION_V(FW_PORT_ACTION_L1_CFG) |
-			    FW_LEN16(c));
+	/* Convert driver coding of Pause Frame Flow Control settings into the
+	 * Firmware's API.
+	 */
+	fw_fc = cc_to_fwcap_pause(lc->requested_fc);
+
+	/* Convert Common Code Forward Error Control settings into the
+	 * Firmware's API.  If the current Requested FEC has "Automatic"
+	 * (IEEE 802.3) specified, then we use whatever the Firmware
+	 * sent us as part of it's IEEE 802.3-based interpratation of
+	 * the Transceiver Module EPROM FEC parameters.  Otherwise we
+	 * use whatever is in the current Requested FEC settings.
+	 */
+	if (lc->requested_fec & FEC_AUTO)
+		cc_fec = fwcap_to_cc_fec(lc->def_acaps);
+	else
+		cc_fec = lc->requested_fec;
+	fw_fec = cc_to_fwcap_fec(cc_fec);
 
-	if (!(lc->supported & FW_PORT_CAP_ANEG)) {
-		c.u.l1cfg.rcap = cpu_to_be32((lc->supported & ADVERT_MASK) |
-					     fc | fw_fec);
-		lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX);
+	/* Figure out what our Requested Port Capabilities are going to be.
+	 */
+	if (!(lc->pcaps & FW_PORT_CAP32_ANEG)) {
+		rcap = (lc->pcaps & ADVERT_MASK) | fw_fc | fw_fec;
+		lc->fc = lc->requested_fc & ~PAUSE_AUTONEG;
+		lc->fec = cc_fec;
 	} else if (lc->autoneg == AUTONEG_DISABLE) {
-		c.u.l1cfg.rcap = cpu_to_be32(lc->requested_speed | fc |
-					     fw_fec | mdi);
-		lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX);
-	} else
-		c.u.l1cfg.rcap = cpu_to_be32(lc->advertising | fc |
-					     fw_fec | mdi);
+		rcap = lc->speed_caps | fw_fc | fw_fec | fw_mdi;
+		lc->fc = lc->requested_fc & ~PAUSE_AUTONEG;
+		lc->fec = cc_fec;
+	} else {
+		rcap = lc->acaps | fw_fc | fw_fec | fw_mdi;
+	}
 
-	return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
+	/* And send that on to the Firmware ...
+	 */
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PORT_CMD) |
+				       FW_CMD_REQUEST_F | FW_CMD_EXEC_F |
+				       FW_PORT_CMD_PORTID_V(port));
+	cmd.action_to_len16 =
+		cpu_to_be32(FW_PORT_CMD_ACTION_V(fw_caps == FW_CAPS16
+						 ? FW_PORT_ACTION_L1_CFG
+						 : FW_PORT_ACTION_L1_CFG32) |
+			    FW_LEN16(cmd));
+	if (fw_caps == FW_CAPS16)
+		cmd.u.l1cfg.rcap = cpu_to_be32(fwcaps32_to_caps16(rcap));
+	else
+		cmd.u.l1cfg32.rcap32 = cpu_to_be32(rcap);
+	return t4_wr_mbox(adapter, mbox, &cmd, sizeof(cmd), NULL);
 }
 
 /**
@@ -3765,7 +4059,7 @@ int t4_restart_aneg(struct adapter *adap, unsigned int mbox, unsigned int port)
 	c.action_to_len16 =
 		cpu_to_be32(FW_PORT_CMD_ACTION_V(FW_PORT_ACTION_L1_CFG) |
 			    FW_LEN16(c));
-	c.u.l1cfg.rcap = cpu_to_be32(FW_PORT_CAP_ANEG);
+	c.u.l1cfg.rcap = cpu_to_be32(FW_PORT_CAP32_ANEG);
 	return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
 }
 
@@ -4254,6 +4548,18 @@ static void mps_intr_handler(struct adapter *adapter)
 		{ FRMERR_F, "MPS Tx framing error", -1, 1 },
 		{ 0 }
 	};
+	static const struct intr_info t6_mps_tx_intr_info[] = {
+		{ TPFIFO_V(TPFIFO_M), "MPS Tx TP FIFO parity error", -1, 1 },
+		{ NCSIFIFO_F, "MPS Tx NC-SI FIFO parity error", -1, 1 },
+		{ TXDATAFIFO_V(TXDATAFIFO_M), "MPS Tx data FIFO parity error",
+		  -1, 1 },
+		{ TXDESCFIFO_V(TXDESCFIFO_M), "MPS Tx desc FIFO parity error",
+		  -1, 1 },
+		/* MPS Tx Bubble is normal for T6 */
+		{ SECNTERR_F, "MPS Tx SOP/EOP error", -1, 1 },
+		{ FRMERR_F, "MPS Tx framing error", -1, 1 },
+		{ 0 }
+	};
 	static const struct intr_info mps_trc_intr_info[] = {
 		{ FILTMEM_V(FILTMEM_M), "MPS TRC filter parity error", -1, 1 },
 		{ PKTFIFO_V(PKTFIFO_M), "MPS TRC packet FIFO parity error",
@@ -4285,7 +4591,9 @@ static void mps_intr_handler(struct adapter *adapter)
 	fat = t4_handle_intr_status(adapter, MPS_RX_PERR_INT_CAUSE_A,
 				    mps_rx_intr_info) +
 	      t4_handle_intr_status(adapter, MPS_TX_INT_CAUSE_A,
-				    mps_tx_intr_info) +
+				    is_t6(adapter->params.chip)
+				    ? t6_mps_tx_intr_info
+				    : mps_tx_intr_info) +
 	      t4_handle_intr_status(adapter, MPS_TRC_INT_CAUSE_A,
 				    mps_trc_intr_info) +
 	      t4_handle_intr_status(adapter, MPS_STAT_PERR_INT_CAUSE_SRAM_A,
@@ -5693,10 +6001,8 @@ void t4_get_port_stats(struct adapter *adap, int idx, struct port_stats *p)
 	p->tx_ppp7             = GET_STAT(TX_PORT_PPP7);
 
 	if (CHELSIO_CHIP_VERSION(adap->params.chip) >= CHELSIO_T5) {
-		if (stat_ctl & COUNTPAUSESTATTX_F) {
-			p->tx_frames -= p->tx_pause;
-			p->tx_octets -= p->tx_pause * 64;
-		}
+		if (stat_ctl & COUNTPAUSESTATTX_F)
+			p->tx_frames_64 -= p->tx_pause;
 		if (stat_ctl & COUNTPAUSEMCTX_F)
 			p->tx_mcast_frames -= p->tx_pause;
 	}
@@ -5729,10 +6035,8 @@ void t4_get_port_stats(struct adapter *adap, int idx, struct port_stats *p)
 	p->rx_ppp7             = GET_STAT(RX_PORT_PPP7);
 
 	if (CHELSIO_CHIP_VERSION(adap->params.chip) >= CHELSIO_T5) {
-		if (stat_ctl & COUNTPAUSESTATRX_F) {
-			p->rx_frames -= p->rx_pause;
-			p->rx_octets -= p->rx_pause * 64;
-		}
+		if (stat_ctl & COUNTPAUSESTATRX_F)
+			p->rx_frames_64 -= p->rx_pause;
 		if (stat_ctl & COUNTPAUSEMCRX_F)
 			p->rx_mcast_frames -= p->rx_pause;
 	}
@@ -6449,6 +6753,17 @@ int t4_fw_upgrade(struct adapter *adap, unsigned int mbox,
 		goto out;
 
 	/*
+	 * If there was a Firmware Configuration File stored in FLASH,
+	 * there's a good chance that it won't be compatible with the new
+	 * Firmware.  In order to prevent difficult to diagnose adapter
+	 * initialization issues, we clear out the Firmware Configuration File
+	 * portion of the FLASH .  The user will need to re-FLASH a new
+	 * Firmware Configuration File which is compatible with the new
+	 * Firmware if that's desired.
+	 */
+	(void)t4_load_cfg(adap, NULL, 0);
+
+	/*
 	 * Older versions of the firmware don't understand the new
 	 * PCIE_FW.HALT flag and so won't know to perform a RESET when they
 	 * restart.  So for newly loaded older firmware we'll have to do the
@@ -7471,6 +7786,98 @@ static const char *t4_link_down_rc_str(unsigned char link_down_rc)
 }
 
 /**
+ * Return the highest speed set in the port capabilities, in Mb/s.
+ */
+static unsigned int fwcap_to_speed(fw_port_cap32_t caps)
+{
+	#define TEST_SPEED_RETURN(__caps_speed, __speed) \
+		do { \
+			if (caps & FW_PORT_CAP32_SPEED_##__caps_speed) \
+				return __speed; \
+		} while (0)
+
+	TEST_SPEED_RETURN(400G, 400000);
+	TEST_SPEED_RETURN(200G, 200000);
+	TEST_SPEED_RETURN(100G, 100000);
+	TEST_SPEED_RETURN(50G,   50000);
+	TEST_SPEED_RETURN(40G,   40000);
+	TEST_SPEED_RETURN(25G,   25000);
+	TEST_SPEED_RETURN(10G,   10000);
+	TEST_SPEED_RETURN(1G,     1000);
+	TEST_SPEED_RETURN(100M,    100);
+
+	#undef TEST_SPEED_RETURN
+
+	return 0;
+}
+
+/**
+ *	fwcap_to_fwspeed - return highest speed in Port Capabilities
+ *	@acaps: advertised Port Capabilities
+ *
+ *	Get the highest speed for the port from the advertised Port
+ *	Capabilities.  It will be either the highest speed from the list of
+ *	speeds or whatever user has set using ethtool.
+ */
+static fw_port_cap32_t fwcap_to_fwspeed(fw_port_cap32_t acaps)
+{
+	#define TEST_SPEED_RETURN(__caps_speed) \
+		do { \
+			if (acaps & FW_PORT_CAP32_SPEED_##__caps_speed) \
+				return FW_PORT_CAP32_SPEED_##__caps_speed; \
+		} while (0)
+
+	TEST_SPEED_RETURN(400G);
+	TEST_SPEED_RETURN(200G);
+	TEST_SPEED_RETURN(100G);
+	TEST_SPEED_RETURN(50G);
+	TEST_SPEED_RETURN(40G);
+	TEST_SPEED_RETURN(25G);
+	TEST_SPEED_RETURN(10G);
+	TEST_SPEED_RETURN(1G);
+	TEST_SPEED_RETURN(100M);
+
+	#undef TEST_SPEED_RETURN
+
+	return 0;
+}
+
+/**
+ *	lstatus_to_fwcap - translate old lstatus to 32-bit Port Capabilities
+ *	@lstatus: old FW_PORT_ACTION_GET_PORT_INFO lstatus value
+ *
+ *	Translates old FW_PORT_ACTION_GET_PORT_INFO lstatus field into new
+ *	32-bit Port Capabilities value.
+ */
+static fw_port_cap32_t lstatus_to_fwcap(u32 lstatus)
+{
+	fw_port_cap32_t linkattr = 0;
+
+	/* Unfortunately the format of the Link Status in the old
+	 * 16-bit Port Information message isn't the same as the
+	 * 16-bit Port Capabilities bitfield used everywhere else ...
+	 */
+	if (lstatus & FW_PORT_CMD_RXPAUSE_F)
+		linkattr |= FW_PORT_CAP32_FC_RX;
+	if (lstatus & FW_PORT_CMD_TXPAUSE_F)
+		linkattr |= FW_PORT_CAP32_FC_TX;
+	if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_100M))
+		linkattr |= FW_PORT_CAP32_SPEED_100M;
+	if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_1G))
+		linkattr |= FW_PORT_CAP32_SPEED_1G;
+	if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_10G))
+		linkattr |= FW_PORT_CAP32_SPEED_10G;
+	if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_25G))
+		linkattr |= FW_PORT_CAP32_SPEED_25G;
+	if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_40G))
+		linkattr |= FW_PORT_CAP32_SPEED_40G;
+	if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_100G))
+		linkattr |= FW_PORT_CAP32_SPEED_100G;
+
+	return linkattr;
+}
+
+/**
  *	t4_handle_get_port_info - process a FW reply message
  *	@pi: the port info
  *	@rpl: start of the FW message
@@ -7479,56 +7886,123 @@ static const char *t4_link_down_rc_str(unsigned char link_down_rc)
  */
 void t4_handle_get_port_info(struct port_info *pi, const __be64 *rpl)
 {
-	const struct fw_port_cmd *p = (const void *)rpl;
-	struct adapter *adap = pi->adapter;
-
-	/* link/module state change message */
-	int speed = 0, fc = 0;
-	struct link_config *lc;
-	u32 stat = be32_to_cpu(p->u.info.lstatus_to_modtype);
-	int link_ok = (stat & FW_PORT_CMD_LSTATUS_F) != 0;
-	u32 mod = FW_PORT_CMD_MODTYPE_G(stat);
-
-	if (stat & FW_PORT_CMD_RXPAUSE_F)
-		fc |= PAUSE_RX;
-	if (stat & FW_PORT_CMD_TXPAUSE_F)
-		fc |= PAUSE_TX;
-	if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_100M))
-		speed = 100;
-	else if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_1G))
-		speed = 1000;
-	else if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_10G))
-		speed = 10000;
-	else if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_25G))
-		speed = 25000;
-	else if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_40G))
-		speed = 40000;
-	else if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_100G))
-		speed = 100000;
-
-	lc = &pi->link_cfg;
-
-	if (mod != pi->mod_type) {
-		pi->mod_type = mod;
-		t4_os_portmod_changed(adap, pi->port_id);
+	const struct fw_port_cmd *cmd = (const void *)rpl;
+	int action = FW_PORT_CMD_ACTION_G(be32_to_cpu(cmd->action_to_len16));
+	struct adapter *adapter = pi->adapter;
+	struct link_config *lc = &pi->link_cfg;
+	int link_ok, linkdnrc;
+	enum fw_port_type port_type;
+	enum fw_port_module_type mod_type;
+	unsigned int speed, fc, fec;
+	fw_port_cap32_t pcaps, acaps, lpacaps, linkattr;
+
+	/* Extract the various fields from the Port Information message.
+	 */
+	switch (action) {
+	case FW_PORT_ACTION_GET_PORT_INFO: {
+		u32 lstatus = be32_to_cpu(cmd->u.info.lstatus_to_modtype);
+
+		link_ok = (lstatus & FW_PORT_CMD_LSTATUS_F) != 0;
+		linkdnrc = FW_PORT_CMD_LINKDNRC_G(lstatus);
+		port_type = FW_PORT_CMD_PTYPE_G(lstatus);
+		mod_type = FW_PORT_CMD_MODTYPE_G(lstatus);
+		pcaps = fwcaps16_to_caps32(be16_to_cpu(cmd->u.info.pcap));
+		acaps = fwcaps16_to_caps32(be16_to_cpu(cmd->u.info.acap));
+		lpacaps = fwcaps16_to_caps32(be16_to_cpu(cmd->u.info.lpacap));
+		linkattr = lstatus_to_fwcap(lstatus);
+		break;
 	}
+
+	case FW_PORT_ACTION_GET_PORT_INFO32: {
+		u32 lstatus32;
+
+		lstatus32 = be32_to_cpu(cmd->u.info32.lstatus32_to_cbllen32);
+		link_ok = (lstatus32 & FW_PORT_CMD_LSTATUS32_F) != 0;
+		linkdnrc = FW_PORT_CMD_LINKDNRC32_G(lstatus32);
+		port_type = FW_PORT_CMD_PORTTYPE32_G(lstatus32);
+		mod_type = FW_PORT_CMD_MODTYPE32_G(lstatus32);
+		pcaps = be32_to_cpu(cmd->u.info32.pcaps32);
+		acaps = be32_to_cpu(cmd->u.info32.acaps32);
+		lpacaps = be32_to_cpu(cmd->u.info32.lpacaps32);
+		linkattr = be32_to_cpu(cmd->u.info32.linkattr32);
+		break;
+	}
+
+	default:
+		dev_err(adapter->pdev_dev, "Handle Port Information: Bad Command/Action %#x\n",
+			be32_to_cpu(cmd->action_to_len16));
+		return;
+	}
+
+	fec = fwcap_to_cc_fec(acaps);
+	fc = fwcap_to_cc_pause(linkattr);
+	speed = fwcap_to_speed(linkattr);
+
+	if (mod_type != pi->mod_type) {
+		/* With the newer SFP28 and QSFP28 Transceiver Module Types,
+		 * various fundamental Port Capabilities which used to be
+		 * immutable can now change radically.  We can now have
+		 * Speeds, Auto-Negotiation, Forward Error Correction, etc.
+		 * all change based on what Transceiver Module is inserted.
+		 * So we need to record the Physical "Port" Capabilities on
+		 * every Transceiver Module change.
+		 */
+		lc->pcaps = pcaps;
+
+		/* When a new Transceiver Module is inserted, the Firmware
+		 * will examine its i2c EPROM to determine its type and
+		 * general operating parameters including things like Forward
+		 * Error Control, etc.  Various IEEE 802.3 standards dictate
+		 * how to interpret these i2c values to determine default
+		 * "sutomatic" settings.  We record these for future use when
+		 * the user explicitly requests these standards-based values.
+		 */
+		lc->def_acaps = acaps;
+
+		/* Some versions of the early T6 Firmware "cheated" when
+		 * handling different Transceiver Modules by changing the
+		 * underlaying Port Type reported to the Host Drivers.  As
+		 * such we need to capture whatever Port Type the Firmware
+		 * sends us and record it in case it's different from what we
+		 * were told earlier.  Unfortunately, since Firmware is
+		 * forever, we'll need to keep this code here forever, but in
+		 * later T6 Firmware it should just be an assignment of the
+		 * same value already recorded.
+		 */
+		pi->port_type = port_type;
+
+		pi->mod_type = mod_type;
+		t4_os_portmod_changed(adapter, pi->port_id);
+	}
+
 	if (link_ok != lc->link_ok || speed != lc->speed ||
-	    fc != lc->fc) {	/* something changed */
+	    fc != lc->fc || fec != lc->fec) {	/* something changed */
 		if (!link_ok && lc->link_ok) {
-			unsigned char rc = FW_PORT_CMD_LINKDNRC_G(stat);
-
-			lc->link_down_rc = rc;
-			dev_warn(adap->pdev_dev,
-				 "Port %d link down, reason: %s\n",
-				 pi->port_id, t4_link_down_rc_str(rc));
+			lc->link_down_rc = linkdnrc;
+			dev_warn(adapter->pdev_dev, "Port %d link down, reason: %s\n",
+				 pi->tx_chan, t4_link_down_rc_str(linkdnrc));
 		}
 		lc->link_ok = link_ok;
 		lc->speed = speed;
 		lc->fc = fc;
-		lc->supported = be16_to_cpu(p->u.info.pcap);
-		lc->lp_advertising = be16_to_cpu(p->u.info.lpacap);
+		lc->fec = fec;
 
-		t4_os_link_changed(adap, pi->port_id, link_ok);
+		lc->lpacaps = lpacaps;
+		lc->acaps = acaps & ADVERT_MASK;
+
+		if (lc->acaps & FW_PORT_CAP32_ANEG) {
+			lc->autoneg = AUTONEG_ENABLE;
+		} else {
+			/* When Autoneg is disabled, user needs to set
+			 * single speed.
+			 * Similar to cxgb4_ethtool.c: set_link_ksettings
+			 */
+			lc->acaps = 0;
+			lc->speed_caps = fwcap_to_fwspeed(acaps);
+			lc->autoneg = AUTONEG_DISABLE;
+		}
+
+		t4_os_link_changed(adapter, pi->port_id, link_ok);
 	}
 }
 
@@ -7542,15 +8016,18 @@ void t4_handle_get_port_info(struct port_info *pi, const __be64 *rpl)
  */
 int t4_update_port_info(struct port_info *pi)
 {
+	unsigned int fw_caps = pi->adapter->params.fw_caps_support;
 	struct fw_port_cmd port_cmd;
 	int ret;
 
 	memset(&port_cmd, 0, sizeof(port_cmd));
 	port_cmd.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PORT_CMD) |
 					    FW_CMD_REQUEST_F | FW_CMD_READ_F |
-					    FW_PORT_CMD_PORTID_V(pi->port_id));
+					    FW_PORT_CMD_PORTID_V(pi->tx_chan));
 	port_cmd.action_to_len16 = cpu_to_be32(
-		FW_PORT_CMD_ACTION_V(FW_PORT_ACTION_GET_PORT_INFO) |
+		FW_PORT_CMD_ACTION_V(fw_caps == FW_CAPS16
+				     ? FW_PORT_ACTION_GET_PORT_INFO
+				     : FW_PORT_ACTION_GET_PORT_INFO32) |
 		FW_LEN16(port_cmd));
 	ret = t4_wr_mbox(pi->adapter, pi->adapter->mbox,
 			 &port_cmd, sizeof(port_cmd), &port_cmd);
@@ -7562,6 +8039,65 @@ int t4_update_port_info(struct port_info *pi)
 }
 
 /**
+ *	t4_get_link_params - retrieve basic link parameters for given port
+ *	@pi: the port
+ *	@link_okp: value return pointer for link up/down
+ *	@speedp: value return pointer for speed (Mb/s)
+ *	@mtup: value return pointer for mtu
+ *
+ *	Retrieves basic link parameters for a port: link up/down, speed (Mb/s),
+ *	and MTU for a specified port.  A negative error is returned on
+ *	failure; 0 on success.
+ */
+int t4_get_link_params(struct port_info *pi, unsigned int *link_okp,
+		       unsigned int *speedp, unsigned int *mtup)
+{
+	unsigned int fw_caps = pi->adapter->params.fw_caps_support;
+	struct fw_port_cmd port_cmd;
+	unsigned int action, link_ok, speed, mtu;
+	fw_port_cap32_t linkattr;
+	int ret;
+
+	memset(&port_cmd, 0, sizeof(port_cmd));
+	port_cmd.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PORT_CMD) |
+					    FW_CMD_REQUEST_F | FW_CMD_READ_F |
+					    FW_PORT_CMD_PORTID_V(pi->tx_chan));
+	action = (fw_caps == FW_CAPS16
+		  ? FW_PORT_ACTION_GET_PORT_INFO
+		  : FW_PORT_ACTION_GET_PORT_INFO32);
+	port_cmd.action_to_len16 = cpu_to_be32(
+		FW_PORT_CMD_ACTION_V(action) |
+		FW_LEN16(port_cmd));
+	ret = t4_wr_mbox(pi->adapter, pi->adapter->mbox,
+			 &port_cmd, sizeof(port_cmd), &port_cmd);
+	if (ret)
+		return ret;
+
+	if (action == FW_PORT_ACTION_GET_PORT_INFO) {
+		u32 lstatus = be32_to_cpu(port_cmd.u.info.lstatus_to_modtype);
+
+		link_ok = !!(lstatus & FW_PORT_CMD_LSTATUS_F);
+		linkattr = lstatus_to_fwcap(lstatus);
+		mtu = be16_to_cpu(port_cmd.u.info.mtu);
+	} else {
+		u32 lstatus32 =
+			   be32_to_cpu(port_cmd.u.info32.lstatus32_to_cbllen32);
+
+		link_ok = !!(lstatus32 & FW_PORT_CMD_LSTATUS32_F);
+		linkattr = be32_to_cpu(port_cmd.u.info32.linkattr32);
+		mtu = FW_PORT_CMD_MTU32_G(
+			be32_to_cpu(port_cmd.u.info32.auxlinfo32_mtu32));
+	}
+	speed = fwcap_to_speed(linkattr);
+
+	*link_okp = link_ok;
+	*speedp = fwcap_to_speed(linkattr);
+	*mtup = mtu;
+
+	return 0;
+}
+
+/**
  *      t4_handle_fw_rpl - process a FW reply message
  *      @adap: the adapter
  *      @rpl: start of the FW message
@@ -7581,7 +8117,9 @@ int t4_handle_fw_rpl(struct adapter *adap, const __be64 *rpl)
 	unsigned int action =
 		FW_PORT_CMD_ACTION_G(be32_to_cpu(p->action_to_len16));
 
-	if (opcode == FW_PORT_CMD && action == FW_PORT_ACTION_GET_PORT_INFO) {
+	if (opcode == FW_PORT_CMD &&
+	    (action == FW_PORT_ACTION_GET_PORT_INFO ||
+	     action == FW_PORT_ACTION_GET_PORT_INFO32)) {
 		int i;
 		int chan = FW_PORT_CMD_PORTID_G(be32_to_cpu(p->op_to_portid));
 		struct port_info *pi = NULL;
@@ -7594,7 +8132,8 @@ int t4_handle_fw_rpl(struct adapter *adap, const __be64 *rpl)
 
 		t4_handle_get_port_info(pi, rpl);
 	} else {
-		dev_warn(adap->pdev_dev, "Unknown firmware reply %d\n", opcode);
+		dev_warn(adap->pdev_dev, "Unknown firmware reply %d\n",
+			 opcode);
 		return -EINVAL;
 	}
 	return 0;
@@ -7613,38 +8152,35 @@ static void get_pci_mode(struct adapter *adapter, struct pci_params *p)
 
 /**
  *	init_link_config - initialize a link's SW state
- *	@lc: structure holding the link state
- *	@caps: link capabilities
+ *	@lc: pointer to structure holding the link state
+ *	@pcaps: link Port Capabilities
+ *	@acaps: link current Advertised Port Capabilities
  *
  *	Initializes the SW state maintained for each link, including the link's
  *	capabilities and default speed/flow-control/autonegotiation settings.
  */
-static void init_link_config(struct link_config *lc, unsigned int pcaps,
-			     unsigned int acaps)
+static void init_link_config(struct link_config *lc, fw_port_cap32_t pcaps,
+			     fw_port_cap32_t acaps)
 {
-	lc->supported = pcaps;
-	lc->lp_advertising = 0;
-	lc->requested_speed = 0;
+	lc->pcaps = pcaps;
+	lc->def_acaps = acaps;
+	lc->lpacaps = 0;
+	lc->speed_caps = 0;
 	lc->speed = 0;
 	lc->requested_fc = lc->fc = PAUSE_RX | PAUSE_TX;
-	lc->auto_fec = 0;
 
 	/* For Forward Error Control, we default to whatever the Firmware
 	 * tells us the Link is currently advertising.
 	 */
-	if (acaps & FW_PORT_CAP_FEC_RS)
-		lc->auto_fec |= FEC_RS;
-	if (acaps & FW_PORT_CAP_FEC_BASER_RS)
-		lc->auto_fec |= FEC_BASER_RS;
 	lc->requested_fec = FEC_AUTO;
-	lc->fec = lc->auto_fec;
+	lc->fec = fwcap_to_cc_fec(lc->def_acaps);
 
-	if (lc->supported & FW_PORT_CAP_ANEG) {
-		lc->advertising = lc->supported & ADVERT_MASK;
+	if (lc->pcaps & FW_PORT_CAP32_ANEG) {
+		lc->acaps = lc->pcaps & ADVERT_MASK;
 		lc->autoneg = AUTONEG_ENABLE;
 		lc->requested_fc |= PAUSE_AUTONEG;
 	} else {
-		lc->advertising = 0;
+		lc->acaps = 0;
 		lc->autoneg = AUTONEG_DISABLE;
 	}
 }
@@ -8169,7 +8705,7 @@ int t4_init_rss_mode(struct adapter *adap, int mbox)
 }
 
 /**
- *	t4_init_portinfo - allocate a virtual interface amd initialize port_info
+ *	t4_init_portinfo - allocate a virtual interface and initialize port_info
  *	@pi: the port_info
  *	@mbox: mailbox to use for the FW command
  *	@port: physical port associated with the VI
@@ -8185,21 +8721,67 @@ int t4_init_rss_mode(struct adapter *adap, int mbox)
 int t4_init_portinfo(struct port_info *pi, int mbox,
 		     int port, int pf, int vf, u8 mac[])
 {
-	int ret;
-	struct fw_port_cmd c;
+	struct adapter *adapter = pi->adapter;
+	unsigned int fw_caps = adapter->params.fw_caps_support;
+	struct fw_port_cmd cmd;
 	unsigned int rss_size;
+	enum fw_port_type port_type;
+	int mdio_addr;
+	fw_port_cap32_t pcaps, acaps;
+	int ret;
 
-	memset(&c, 0, sizeof(c));
-	c.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PORT_CMD) |
-				     FW_CMD_REQUEST_F | FW_CMD_READ_F |
-				     FW_PORT_CMD_PORTID_V(port));
-	c.action_to_len16 = cpu_to_be32(
-		FW_PORT_CMD_ACTION_V(FW_PORT_ACTION_GET_PORT_INFO) |
-		FW_LEN16(c));
-	ret = t4_wr_mbox(pi->adapter, mbox, &c, sizeof(c), &c);
+	/* If we haven't yet determined whether we're talking to Firmware
+	 * which knows the new 32-bit Port Capabilities, it's time to find
+	 * out now.  This will also tell new Firmware to send us Port Status
+	 * Updates using the new 32-bit Port Capabilities version of the
+	 * Port Information message.
+	 */
+	if (fw_caps == FW_CAPS_UNKNOWN) {
+		u32 param, val;
+
+		param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) |
+			 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_PORT_CAPS32));
+		val = 1;
+		ret = t4_set_params(adapter, mbox, pf, vf, 1, &param, &val);
+		fw_caps = (ret == 0 ? FW_CAPS32 : FW_CAPS16);
+		adapter->params.fw_caps_support = fw_caps;
+	}
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PORT_CMD) |
+				       FW_CMD_REQUEST_F | FW_CMD_READ_F |
+				       FW_PORT_CMD_PORTID_V(port));
+	cmd.action_to_len16 = cpu_to_be32(
+		FW_PORT_CMD_ACTION_V(fw_caps == FW_CAPS16
+				     ? FW_PORT_ACTION_GET_PORT_INFO
+				     : FW_PORT_ACTION_GET_PORT_INFO32) |
+		FW_LEN16(cmd));
+	ret = t4_wr_mbox(pi->adapter, mbox, &cmd, sizeof(cmd), &cmd);
 	if (ret)
 		return ret;
 
+	/* Extract the various fields from the Port Information message.
+	 */
+	if (fw_caps == FW_CAPS16) {
+		u32 lstatus = be32_to_cpu(cmd.u.info.lstatus_to_modtype);
+
+		port_type = FW_PORT_CMD_PTYPE_G(lstatus);
+		mdio_addr = ((lstatus & FW_PORT_CMD_MDIOCAP_F)
+			     ? FW_PORT_CMD_MDIOADDR_G(lstatus)
+			     : -1);
+		pcaps = fwcaps16_to_caps32(be16_to_cpu(cmd.u.info.pcap));
+		acaps = fwcaps16_to_caps32(be16_to_cpu(cmd.u.info.acap));
+	} else {
+		u32 lstatus32 = be32_to_cpu(cmd.u.info32.lstatus32_to_cbllen32);
+
+		port_type = FW_PORT_CMD_PORTTYPE32_G(lstatus32);
+		mdio_addr = ((lstatus32 & FW_PORT_CMD_MDIOCAP32_F)
+			     ? FW_PORT_CMD_MDIOADDR32_G(lstatus32)
+			     : -1);
+		pcaps = be32_to_cpu(cmd.u.info32.pcaps32);
+		acaps = be32_to_cpu(cmd.u.info32.acaps32);
+	}
+
 	ret = t4_alloc_vi(pi->adapter, mbox, port, pf, vf, 1, mac, &rss_size);
 	if (ret < 0)
 		return ret;
@@ -8209,14 +8791,11 @@ int t4_init_portinfo(struct port_info *pi, int mbox,
 	pi->lport = port;
 	pi->rss_size = rss_size;
 
-	ret = be32_to_cpu(c.u.info.lstatus_to_modtype);
-	pi->mdio_addr = (ret & FW_PORT_CMD_MDIOCAP_F) ?
-		FW_PORT_CMD_MDIOADDR_G(ret) : -1;
-	pi->port_type = FW_PORT_CMD_PTYPE_G(ret);
+	pi->port_type = port_type;
+	pi->mdio_addr = mdio_addr;
 	pi->mod_type = FW_PORT_MOD_TYPE_NA;
 
-	init_link_config(&pi->link_cfg, be16_to_cpu(c.u.info.pcap),
-			 be16_to_cpu(c.u.info.acap));
+	init_link_config(&pi->link_cfg, pcaps, acaps);
 	return 0;
 }
 
@@ -8664,6 +9243,65 @@ void t4_idma_monitor(struct adapter *adapter,
 }
 
 /**
+ *	t4_load_cfg - download config file
+ *	@adap: the adapter
+ *	@cfg_data: the cfg text file to write
+ *	@size: text file size
+ *
+ *	Write the supplied config text file to the card's serial flash.
+ */
+int t4_load_cfg(struct adapter *adap, const u8 *cfg_data, unsigned int size)
+{
+	int ret, i, n, cfg_addr;
+	unsigned int addr;
+	unsigned int flash_cfg_start_sec;
+	unsigned int sf_sec_size = adap->params.sf_size / adap->params.sf_nsec;
+
+	cfg_addr = t4_flash_cfg_addr(adap);
+	if (cfg_addr < 0)
+		return cfg_addr;
+
+	addr = cfg_addr;
+	flash_cfg_start_sec = addr / SF_SEC_SIZE;
+
+	if (size > FLASH_CFG_MAX_SIZE) {
+		dev_err(adap->pdev_dev, "cfg file too large, max is %u bytes\n",
+			FLASH_CFG_MAX_SIZE);
+		return -EFBIG;
+	}
+
+	i = DIV_ROUND_UP(FLASH_CFG_MAX_SIZE,	/* # of sectors spanned */
+			 sf_sec_size);
+	ret = t4_flash_erase_sectors(adap, flash_cfg_start_sec,
+				     flash_cfg_start_sec + i - 1);
+	/* If size == 0 then we're simply erasing the FLASH sectors associated
+	 * with the on-adapter Firmware Configuration File.
+	 */
+	if (ret || size == 0)
+		goto out;
+
+	/* this will write to the flash up to SF_PAGE_SIZE at a time */
+	for (i = 0; i < size; i += SF_PAGE_SIZE) {
+		if ((size - i) <  SF_PAGE_SIZE)
+			n = size - i;
+		else
+			n = SF_PAGE_SIZE;
+		ret = t4_write_flash(adap, addr, n, cfg_data);
+		if (ret)
+			goto out;
+
+		addr += SF_PAGE_SIZE;
+		cfg_data += SF_PAGE_SIZE;
+	}
+
+out:
+	if (ret)
+		dev_err(adap->pdev_dev, "config file %s failed %d\n",
+			(size == 0 ? "clear" : "download"), ret);
+	return ret;
+}
+
+/**
  *	t4_set_vf_mac - Set MAC address for the specified VF
  *	@adapter: The adapter
  *	@vf: one of the VFs instantiated by the specified PF
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h b/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h
index 0ebed64d62d3..ca2756dcefc5 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h
@@ -1124,6 +1124,8 @@ enum fw_params_param_dev {
 	FW_PARAMS_PARAM_DEV_MAXIRD_ADAPTER = 0x14, /* max supported adap IRD */
 	FW_PARAMS_PARAM_DEV_ULPTX_MEMWRITE_DSGL = 0x17,
 	FW_PARAMS_PARAM_DEV_FWCACHE = 0x18,
+	FW_PARAMS_PARAM_DEV_SCFGREV = 0x1A,
+	FW_PARAMS_PARAM_DEV_VPDREV = 0x1B,
 	FW_PARAMS_PARAM_DEV_RI_FR_NSMR_TPTE_WR	= 0x1C,
 	FW_PARAMS_PARAM_DEV_MPSBGMAP	= 0x1E,
 };
@@ -1171,7 +1173,8 @@ enum fw_params_param_pfvf {
 	FW_PARAMS_PARAM_PFVF_ACTIVE_FILTER_END = 0x2E,
 	FW_PARAMS_PARAM_PFVF_ETHOFLD_END = 0x30,
 	FW_PARAMS_PARAM_PFVF_CPLFW4MSG_ENCAP = 0x31,
-	FW_PARAMS_PARAM_PFVF_NCRYPTO_LOOKASIDE = 0x32
+	FW_PARAMS_PARAM_PFVF_NCRYPTO_LOOKASIDE = 0x32,
+	FW_PARAMS_PARAM_PFVF_PORT_CAPS32 = 0x3A,
 };
 
 /*
@@ -2254,6 +2257,7 @@ struct fw_acl_vlan_cmd {
 #define FW_ACL_VLAN_CMD_FM_S	6
 #define FW_ACL_VLAN_CMD_FM_V(x)	((x) << FW_ACL_VLAN_CMD_FM_S)
 
+/* old 16-bit port capabilities bitmap (fw_port_cap16_t) */
 enum fw_port_cap {
 	FW_PORT_CAP_SPEED_100M		= 0x0001,
 	FW_PORT_CAP_SPEED_1G		= 0x0002,
@@ -2289,6 +2293,84 @@ enum fw_port_mdi {
 #define FW_PORT_CAP_MDI_S 9
 #define FW_PORT_CAP_MDI_V(x) ((x) << FW_PORT_CAP_MDI_S)
 
+/* new 32-bit port capabilities bitmap (fw_port_cap32_t) */
+#define	FW_PORT_CAP32_SPEED_100M	0x00000001UL
+#define	FW_PORT_CAP32_SPEED_1G		0x00000002UL
+#define	FW_PORT_CAP32_SPEED_10G		0x00000004UL
+#define	FW_PORT_CAP32_SPEED_25G		0x00000008UL
+#define	FW_PORT_CAP32_SPEED_40G		0x00000010UL
+#define	FW_PORT_CAP32_SPEED_50G		0x00000020UL
+#define	FW_PORT_CAP32_SPEED_100G	0x00000040UL
+#define	FW_PORT_CAP32_SPEED_200G	0x00000080UL
+#define	FW_PORT_CAP32_SPEED_400G	0x00000100UL
+#define	FW_PORT_CAP32_SPEED_RESERVED1	0x00000200UL
+#define	FW_PORT_CAP32_SPEED_RESERVED2	0x00000400UL
+#define	FW_PORT_CAP32_SPEED_RESERVED3	0x00000800UL
+#define	FW_PORT_CAP32_RESERVED1		0x0000f000UL
+#define	FW_PORT_CAP32_FC_RX		0x00010000UL
+#define	FW_PORT_CAP32_FC_TX		0x00020000UL
+#define	FW_PORT_CAP32_802_3_PAUSE	0x00040000UL
+#define	FW_PORT_CAP32_802_3_ASM_DIR	0x00080000UL
+#define	FW_PORT_CAP32_ANEG		0x00100000UL
+#define	FW_PORT_CAP32_MDIX		0x00200000UL
+#define	FW_PORT_CAP32_MDIAUTO		0x00400000UL
+#define	FW_PORT_CAP32_FEC_RS		0x00800000UL
+#define	FW_PORT_CAP32_FEC_BASER_RS	0x01000000UL
+#define	FW_PORT_CAP32_FEC_RESERVED1	0x02000000UL
+#define	FW_PORT_CAP32_FEC_RESERVED2	0x04000000UL
+#define	FW_PORT_CAP32_FEC_RESERVED3	0x08000000UL
+#define	FW_PORT_CAP32_RESERVED2		0xf0000000UL
+
+#define FW_PORT_CAP32_SPEED_S	0
+#define FW_PORT_CAP32_SPEED_M	0xfff
+#define FW_PORT_CAP32_SPEED_V(x)	((x) << FW_PORT_CAP32_SPEED_S)
+#define FW_PORT_CAP32_SPEED_G(x) \
+	(((x) >> FW_PORT_CAP32_SPEED_S) & FW_PORT_CAP32_SPEED_M)
+
+#define FW_PORT_CAP32_FC_S	16
+#define FW_PORT_CAP32_FC_M	0x3
+#define FW_PORT_CAP32_FC_V(x)	((x) << FW_PORT_CAP32_FC_S)
+#define FW_PORT_CAP32_FC_G(x) \
+	(((x) >> FW_PORT_CAP32_FC_S) & FW_PORT_CAP32_FC_M)
+
+#define FW_PORT_CAP32_802_3_S	18
+#define FW_PORT_CAP32_802_3_M	0x3
+#define FW_PORT_CAP32_802_3_V(x)	((x) << FW_PORT_CAP32_802_3_S)
+#define FW_PORT_CAP32_802_3_G(x) \
+	(((x) >> FW_PORT_CAP32_802_3_S) & FW_PORT_CAP32_802_3_M)
+
+#define FW_PORT_CAP32_ANEG_S	20
+#define FW_PORT_CAP32_ANEG_M	0x1
+#define FW_PORT_CAP32_ANEG_V(x)	((x) << FW_PORT_CAP32_ANEG_S)
+#define FW_PORT_CAP32_ANEG_G(x) \
+	(((x) >> FW_PORT_CAP32_ANEG_S) & FW_PORT_CAP32_ANEG_M)
+
+enum fw_port_mdi32 {
+	FW_PORT_CAP32_MDI_UNCHANGED,
+	FW_PORT_CAP32_MDI_AUTO,
+	FW_PORT_CAP32_MDI_F_STRAIGHT,
+	FW_PORT_CAP32_MDI_F_CROSSOVER
+};
+
+#define FW_PORT_CAP32_MDI_S 21
+#define FW_PORT_CAP32_MDI_M 3
+#define FW_PORT_CAP32_MDI_V(x) ((x) << FW_PORT_CAP32_MDI_S)
+#define FW_PORT_CAP32_MDI_G(x) \
+	(((x) >> FW_PORT_CAP32_MDI_S) & FW_PORT_CAP32_MDI_M)
+
+#define FW_PORT_CAP32_FEC_S	23
+#define FW_PORT_CAP32_FEC_M	0x1f
+#define FW_PORT_CAP32_FEC_V(x)	((x) << FW_PORT_CAP32_FEC_S)
+#define FW_PORT_CAP32_FEC_G(x) \
+	(((x) >> FW_PORT_CAP32_FEC_S) & FW_PORT_CAP32_FEC_M)
+
+/* macros to isolate various 32-bit Port Capabilities sub-fields */
+#define CAP32_SPEED(__cap32) \
+	(FW_PORT_CAP32_SPEED_V(FW_PORT_CAP32_SPEED_M) & __cap32)
+
+#define CAP32_FEC(__cap32) \
+	(FW_PORT_CAP32_FEC_V(FW_PORT_CAP32_FEC_M) & __cap32)
+
 enum fw_port_action {
 	FW_PORT_ACTION_L1_CFG		= 0x0001,
 	FW_PORT_ACTION_L2_CFG		= 0x0002,
@@ -2298,6 +2380,8 @@ enum fw_port_action {
 	FW_PORT_ACTION_DCB_READ_TRANS	= 0x0006,
 	FW_PORT_ACTION_DCB_READ_RECV	= 0x0007,
 	FW_PORT_ACTION_DCB_READ_DET	= 0x0008,
+	FW_PORT_ACTION_L1_CFG32		= 0x0009,
+	FW_PORT_ACTION_GET_PORT_INFO32	= 0x000a,
 	FW_PORT_ACTION_LOW_PWR_TO_NORMAL = 0x0010,
 	FW_PORT_ACTION_L1_LOW_PWR_EN	= 0x0011,
 	FW_PORT_ACTION_L2_WOL_MODE_EN	= 0x0012,
@@ -2445,6 +2529,18 @@ struct fw_port_cmd {
 				__be64 r12;
 			} control;
 		} dcb;
+		struct fw_port_l1cfg32 {
+			__be32 rcap32;
+			__be32 r;
+		} l1cfg32;
+		struct fw_port_info32 {
+			__be32 lstatus32_to_cbllen32;
+			__be32 auxlinfo32_mtu32;
+			__be32 linkattr32;
+			__be32 pcaps32;
+			__be32 acaps32;
+			__be32 lpacaps32;
+		} info32;
 	} u;
 };
 
@@ -2553,6 +2649,85 @@ struct fw_port_cmd {
 #define FW_PORT_CMD_DCB_VERSION_G(x)	\
 	(((x) >> FW_PORT_CMD_DCB_VERSION_S) & FW_PORT_CMD_DCB_VERSION_M)
 
+#define FW_PORT_CMD_LSTATUS32_S		31
+#define FW_PORT_CMD_LSTATUS32_M		0x1
+#define FW_PORT_CMD_LSTATUS32_V(x)	((x) << FW_PORT_CMD_LSTATUS32_S)
+#define FW_PORT_CMD_LSTATUS32_G(x)	\
+	(((x) >> FW_PORT_CMD_LSTATUS32_S) & FW_PORT_CMD_LSTATUS32_M)
+#define FW_PORT_CMD_LSTATUS32_F	FW_PORT_CMD_LSTATUS32_V(1U)
+
+#define FW_PORT_CMD_LINKDNRC32_S	28
+#define FW_PORT_CMD_LINKDNRC32_M	0x7
+#define FW_PORT_CMD_LINKDNRC32_V(x)	((x) << FW_PORT_CMD_LINKDNRC32_S)
+#define FW_PORT_CMD_LINKDNRC32_G(x)	\
+	(((x) >> FW_PORT_CMD_LINKDNRC32_S) & FW_PORT_CMD_LINKDNRC32_M)
+
+#define FW_PORT_CMD_DCBXDIS32_S		27
+#define FW_PORT_CMD_DCBXDIS32_M		0x1
+#define FW_PORT_CMD_DCBXDIS32_V(x)	((x) << FW_PORT_CMD_DCBXDIS32_S)
+#define FW_PORT_CMD_DCBXDIS32_G(x)	\
+	(((x) >> FW_PORT_CMD_DCBXDIS32_S) & FW_PORT_CMD_DCBXDIS32_M)
+#define FW_PORT_CMD_DCBXDIS32_F	FW_PORT_CMD_DCBXDIS32_V(1U)
+
+#define FW_PORT_CMD_MDIOCAP32_S		26
+#define FW_PORT_CMD_MDIOCAP32_M		0x1
+#define FW_PORT_CMD_MDIOCAP32_V(x)	((x) << FW_PORT_CMD_MDIOCAP32_S)
+#define FW_PORT_CMD_MDIOCAP32_G(x)	\
+	(((x) >> FW_PORT_CMD_MDIOCAP32_S) & FW_PORT_CMD_MDIOCAP32_M)
+#define FW_PORT_CMD_MDIOCAP32_F	FW_PORT_CMD_MDIOCAP32_V(1U)
+
+#define FW_PORT_CMD_MDIOADDR32_S	21
+#define FW_PORT_CMD_MDIOADDR32_M	0x1f
+#define FW_PORT_CMD_MDIOADDR32_V(x)	((x) << FW_PORT_CMD_MDIOADDR32_S)
+#define FW_PORT_CMD_MDIOADDR32_G(x)	\
+	(((x) >> FW_PORT_CMD_MDIOADDR32_S) & FW_PORT_CMD_MDIOADDR32_M)
+
+#define FW_PORT_CMD_PORTTYPE32_S	13
+#define FW_PORT_CMD_PORTTYPE32_M	0xff
+#define FW_PORT_CMD_PORTTYPE32_V(x)	((x) << FW_PORT_CMD_PORTTYPE32_S)
+#define FW_PORT_CMD_PORTTYPE32_G(x)	\
+	(((x) >> FW_PORT_CMD_PORTTYPE32_S) & FW_PORT_CMD_PORTTYPE32_M)
+
+#define FW_PORT_CMD_MODTYPE32_S		8
+#define FW_PORT_CMD_MODTYPE32_M		0x1f
+#define FW_PORT_CMD_MODTYPE32_V(x)	((x) << FW_PORT_CMD_MODTYPE32_S)
+#define FW_PORT_CMD_MODTYPE32_G(x)	\
+	(((x) >> FW_PORT_CMD_MODTYPE32_S) & FW_PORT_CMD_MODTYPE32_M)
+
+#define FW_PORT_CMD_CBLLEN32_S		0
+#define FW_PORT_CMD_CBLLEN32_M		0xff
+#define FW_PORT_CMD_CBLLEN32_V(x)	((x) << FW_PORT_CMD_CBLLEN32_S)
+#define FW_PORT_CMD_CBLLEN32_G(x)	\
+	(((x) >> FW_PORT_CMD_CBLLEN32_S) & FW_PORT_CMD_CBLLEN32_M)
+
+#define FW_PORT_CMD_AUXLINFO32_S	24
+#define FW_PORT_CMD_AUXLINFO32_M	0xff
+#define FW_PORT_CMD_AUXLINFO32_V(x)	((x) << FW_PORT_CMD_AUXLINFO32_S)
+#define FW_PORT_CMD_AUXLINFO32_G(x)	\
+	(((x) >> FW_PORT_CMD_AUXLINFO32_S) & FW_PORT_CMD_AUXLINFO32_M)
+
+#define FW_PORT_AUXLINFO32_KX4_S	2
+#define FW_PORT_AUXLINFO32_KX4_M	0x1
+#define FW_PORT_AUXLINFO32_KX4_V(x) \
+	((x) << FW_PORT_AUXLINFO32_KX4_S)
+#define FW_PORT_AUXLINFO32_KX4_G(x) \
+	(((x) >> FW_PORT_AUXLINFO32_KX4_S) & FW_PORT_AUXLINFO32_KX4_M)
+#define FW_PORT_AUXLINFO32_KX4_F	FW_PORT_AUXLINFO32_KX4_V(1U)
+
+#define FW_PORT_AUXLINFO32_KR_S	1
+#define FW_PORT_AUXLINFO32_KR_M	0x1
+#define FW_PORT_AUXLINFO32_KR_V(x) \
+	((x) << FW_PORT_AUXLINFO32_KR_S)
+#define FW_PORT_AUXLINFO32_KR_G(x) \
+	(((x) >> FW_PORT_AUXLINFO32_KR_S) & FW_PORT_AUXLINFO32_KR_M)
+#define FW_PORT_AUXLINFO32_KR_F	FW_PORT_AUXLINFO32_KR_V(1U)
+
+#define FW_PORT_CMD_MTU32_S	0
+#define FW_PORT_CMD_MTU32_M	0xffff
+#define FW_PORT_CMD_MTU32_V(x)	((x) << FW_PORT_CMD_MTU32_S)
+#define FW_PORT_CMD_MTU32_G(x)	\
+	(((x) >> FW_PORT_CMD_MTU32_S) & FW_PORT_CMD_MTU32_M)
+
 enum fw_port_type {
 	FW_PORT_TYPE_FIBER_XFI,
 	FW_PORT_TYPE_FIBER_XAUI,
diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/adapter.h b/drivers/net/ethernet/chelsio/cxgb4vf/adapter.h
index 109bc630408b..08c6ddb84a04 100644
--- a/drivers/net/ethernet/chelsio/cxgb4vf/adapter.h
+++ b/drivers/net/ethernet/chelsio/cxgb4vf/adapter.h
@@ -408,6 +408,7 @@ enum { /* adapter flags */
 	USING_MSI          = (1UL << 1),
 	USING_MSIX         = (1UL << 2),
 	QUEUES_BOUND       = (1UL << 3),
+	ROOT_NO_RELAXED_ORDERING = (1UL << 4),
 };
 
 /*
diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c
index ac7a150c54e9..8996ebbd222e 100644
--- a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c
@@ -182,7 +182,7 @@ void t4vf_os_link_changed(struct adapter *adapter, int pidx, int link_ok)
 			break;
 		}
 
-		switch (pi->link_cfg.fc) {
+		switch ((int)pi->link_cfg.fc) {
 		case PAUSE_RX:
 			fc = "RX";
 			break;
@@ -191,7 +191,7 @@ void t4vf_os_link_changed(struct adapter *adapter, int pidx, int link_ok)
 			fc = "TX";
 			break;
 
-		case PAUSE_RX|PAUSE_TX:
+		case PAUSE_RX | PAUSE_TX:
 			fc = "RX/TX";
 			break;
 
@@ -1213,7 +1213,11 @@ static int from_fw_port_mod_type(enum fw_port_type port_type,
 	} else if (port_type == FW_PORT_TYPE_SFP ||
 		   port_type == FW_PORT_TYPE_QSFP_10G ||
 		   port_type == FW_PORT_TYPE_QSA ||
-		   port_type == FW_PORT_TYPE_QSFP) {
+		   port_type == FW_PORT_TYPE_QSFP ||
+		   port_type == FW_PORT_TYPE_CR4_QSFP ||
+		   port_type == FW_PORT_TYPE_CR_QSFP ||
+		   port_type == FW_PORT_TYPE_CR2_QSFP ||
+		   port_type == FW_PORT_TYPE_SFP28) {
 		if (mod_type == FW_PORT_MOD_TYPE_LR ||
 		    mod_type == FW_PORT_MOD_TYPE_SR ||
 		    mod_type == FW_PORT_MOD_TYPE_ER ||
@@ -1224,6 +1228,9 @@ static int from_fw_port_mod_type(enum fw_port_type port_type,
 			return PORT_DA;
 		else
 			return PORT_OTHER;
+	} else if (port_type == FW_PORT_TYPE_KR4_100G ||
+		   port_type == FW_PORT_TYPE_KR_SFP28) {
+		return PORT_NONE;
 	}
 
 	return PORT_OTHER;
@@ -1242,12 +1249,13 @@ static void fw_caps_to_lmm(enum fw_port_type port_type,
 			   unsigned int fw_caps,
 			   unsigned long *link_mode_mask)
 {
-	#define SET_LMM(__lmm_name) __set_bit(ETHTOOL_LINK_MODE_ ## __lmm_name\
-			 ## _BIT, link_mode_mask)
+	#define SET_LMM(__lmm_name) \
+		__set_bit(ETHTOOL_LINK_MODE_ ## __lmm_name ## _BIT, \
+			  link_mode_mask)
 
 	#define FW_CAPS_TO_LMM(__fw_name, __lmm_name) \
 		do { \
-			if (fw_caps & FW_PORT_CAP_ ## __fw_name) \
+			if (fw_caps & FW_PORT_CAP32_ ## __fw_name) \
 				SET_LMM(__lmm_name); \
 		} while (0)
 
@@ -1310,6 +1318,16 @@ static void fw_caps_to_lmm(enum fw_port_type port_type,
 		SET_LMM(25000baseCR_Full);
 		break;
 
+	case FW_PORT_TYPE_KR_SFP28:
+		SET_LMM(Backplane);
+		SET_LMM(25000baseKR_Full);
+		break;
+
+	case FW_PORT_TYPE_CR2_QSFP:
+		SET_LMM(FIBRE);
+		SET_LMM(50000baseSR2_Full);
+		break;
+
 	case FW_PORT_TYPE_KR4_100G:
 	case FW_PORT_TYPE_CR4_QSFP:
 		SET_LMM(FIBRE);
@@ -1329,12 +1347,18 @@ static void fw_caps_to_lmm(enum fw_port_type port_type,
 }
 
 static int cxgb4vf_get_link_ksettings(struct net_device *dev,
-				      struct ethtool_link_ksettings
-							*link_ksettings)
+				  struct ethtool_link_ksettings *link_ksettings)
 {
-	const struct port_info *pi = netdev_priv(dev);
+	struct port_info *pi = netdev_priv(dev);
 	struct ethtool_link_settings *base = &link_ksettings->base;
 
+	/* For the nonce, the Firmware doesn't send up Port State changes
+	 * when the Virtual Interface attached to the Port is down.  So
+	 * if it's down, let's grab any changes.
+	 */
+	if (!netif_running(dev))
+		(void)t4vf_update_port_info(pi);
+
 	ethtool_link_ksettings_zero_link_mode(link_ksettings, supported);
 	ethtool_link_ksettings_zero_link_mode(link_ksettings, advertising);
 	ethtool_link_ksettings_zero_link_mode(link_ksettings, lp_advertising);
@@ -1351,11 +1375,11 @@ static int cxgb4vf_get_link_ksettings(struct net_device *dev,
 		base->mdio_support = 0;
 	}
 
-	fw_caps_to_lmm(pi->port_type, pi->link_cfg.supported,
+	fw_caps_to_lmm(pi->port_type, pi->link_cfg.pcaps,
 		       link_ksettings->link_modes.supported);
-	fw_caps_to_lmm(pi->port_type, pi->link_cfg.advertising,
+	fw_caps_to_lmm(pi->port_type, pi->link_cfg.acaps,
 		       link_ksettings->link_modes.advertising);
-	fw_caps_to_lmm(pi->port_type, pi->link_cfg.lp_advertising,
+	fw_caps_to_lmm(pi->port_type, pi->link_cfg.lpacaps,
 		       link_ksettings->link_modes.lp_advertising);
 
 	if (netif_carrier_ok(dev)) {
@@ -1367,7 +1391,7 @@ static int cxgb4vf_get_link_ksettings(struct net_device *dev,
 	}
 
 	base->autoneg = pi->link_cfg.autoneg;
-	if (pi->link_cfg.supported & FW_PORT_CAP_ANEG)
+	if (pi->link_cfg.pcaps & FW_PORT_CAP32_ANEG)
 		ethtool_link_ksettings_add_link_mode(link_ksettings,
 						     supported, Autoneg);
 	if (pi->link_cfg.autoneg)
@@ -2888,6 +2912,24 @@ static int cxgb4vf_pci_probe(struct pci_dev *pdev,
 	 */
 	adapter->name = pci_name(pdev);
 	adapter->msg_enable = DFLT_MSG_ENABLE;
+
+	/* If possible, we use PCIe Relaxed Ordering Attribute to deliver
+	 * Ingress Packet Data to Free List Buffers in order to allow for
+	 * chipset performance optimizations between the Root Complex and
+	 * Memory Controllers.  (Messages to the associated Ingress Queue
+	 * notifying new Packet Placement in the Free Lists Buffers will be
+	 * send without the Relaxed Ordering Attribute thus guaranteeing that
+	 * all preceding PCIe Transaction Layer Packets will be processed
+	 * first.)  But some Root Complexes have various issues with Upstream
+	 * Transaction Layer Packets with the Relaxed Ordering Attribute set.
+	 * The PCIe devices which under the Root Complexes will be cleared the
+	 * Relaxed Ordering bit in the configuration space, So we check our
+	 * PCIe configuration space to see if it's flagged with advice against
+	 * using Relaxed Ordering.
+	 */
+	if (!pcie_relaxed_ordering_enabled(pdev))
+		adapter->flags |= ROOT_NO_RELAXED_ORDERING;
+
 	err = adap_init0(adapter);
 	if (err)
 		goto err_unmap_bar;
diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/sge.c b/drivers/net/ethernet/chelsio/cxgb4vf/sge.c
index e37dde2ba97f..05498e7f2840 100644
--- a/drivers/net/ethernet/chelsio/cxgb4vf/sge.c
+++ b/drivers/net/ethernet/chelsio/cxgb4vf/sge.c
@@ -2205,6 +2205,7 @@ int t4vf_sge_alloc_rxq(struct adapter *adapter, struct sge_rspq *rspq,
 	struct port_info *pi = netdev_priv(dev);
 	struct fw_iq_cmd cmd, rpl;
 	int ret, iqandst, flsz = 0;
+	int relaxed = !(adapter->flags & ROOT_NO_RELAXED_ORDERING);
 
 	/*
 	 * If we're using MSI interrupts and we're not initializing the
@@ -2300,6 +2301,8 @@ int t4vf_sge_alloc_rxq(struct adapter *adapter, struct sge_rspq *rspq,
 			cpu_to_be32(
 				FW_IQ_CMD_FL0HOSTFCMODE_V(SGE_HOSTFCMODE_NONE) |
 				FW_IQ_CMD_FL0PACKEN_F |
+				FW_IQ_CMD_FL0FETCHRO_V(relaxed) |
+				FW_IQ_CMD_FL0DATARO_V(relaxed) |
 				FW_IQ_CMD_FL0PADEN_F);
 
 		/* In T6, for egress queue type FL there is internal overhead
diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h
index b3903fe411aa..9cf9c56b0f73 100644
--- a/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h
+++ b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h
@@ -104,24 +104,62 @@ struct t4vf_port_stats {
 /*
  * Per-"port" (Virtual Interface) link configuration ...
  */
-struct link_config {
-	unsigned int   supported;        /* link capabilities */
-	unsigned int   advertising;      /* advertised capabilities */
-	unsigned short lp_advertising;   /* peer advertised capabilities */
-	unsigned int   requested_speed;  /* speed user has requested */
-	unsigned int   speed;            /* actual link speed */
-	unsigned char  requested_fc;     /* flow control user has requested */
-	unsigned char  fc;               /* actual link flow control */
-	unsigned char  autoneg;          /* autonegotiating? */
-	unsigned char  link_ok;          /* link up? */
+typedef u16 fw_port_cap16_t;    /* 16-bit Port Capabilities integral value */
+typedef u32 fw_port_cap32_t;    /* 32-bit Port Capabilities integral value */
+
+enum fw_caps {
+	FW_CAPS_UNKNOWN	= 0,	/* 0'ed out initial state */
+	FW_CAPS16	= 1,	/* old Firmware: 16-bit Port Capabilities */
+	FW_CAPS32	= 2,	/* new Firmware: 32-bit Port Capabilities */
 };
 
-enum {
-	PAUSE_RX      = 1 << 0,
-	PAUSE_TX      = 1 << 1,
-	PAUSE_AUTONEG = 1 << 2
+enum cc_pause {
+	PAUSE_RX	= 1 << 0,
+	PAUSE_TX	= 1 << 1,
+	PAUSE_AUTONEG	= 1 << 2
+};
+
+enum cc_fec {
+	FEC_AUTO	= 1 << 0,	/* IEEE 802.3 "automatic" */
+	FEC_RS		= 1 << 1,	/* Reed-Solomon */
+	FEC_BASER_RS	= 1 << 2,	/* BaseR/Reed-Solomon */
+};
+
+struct link_config {
+	fw_port_cap32_t pcaps;		/* link capabilities */
+	fw_port_cap32_t	acaps;		/* advertised capabilities */
+	fw_port_cap32_t	lpacaps;	/* peer advertised capabilities */
+
+	fw_port_cap32_t	speed_caps;	/* speed(s) user has requested */
+	u32		speed;		/* actual link speed */
+
+	enum cc_pause	requested_fc;	/* flow control user has requested */
+	enum cc_pause	fc;		/* actual link flow control */
+
+	enum cc_fec	auto_fec;	/* Forward Error Correction: */
+	enum cc_fec	requested_fec;	/*   "automatic" (IEEE 802.3), */
+	enum cc_fec	fec;		/*   requested, and actual in use */
+
+	unsigned char	autoneg;	/* autonegotiating? */
+
+	unsigned char	link_ok;	/* link up? */
+	unsigned char	link_down_rc;	/* link down reason */
 };
 
+/* Return true if the Link Configuration supports "High Speeds" (those greater
+ * than 1Gb/s).
+ */
+static inline bool is_x_10g_port(const struct link_config *lc)
+{
+	fw_port_cap32_t speeds, high_speeds;
+
+	speeds = FW_PORT_CAP32_SPEED_V(FW_PORT_CAP32_SPEED_G(lc->pcaps));
+	high_speeds =
+		speeds & ~(FW_PORT_CAP32_SPEED_100M | FW_PORT_CAP32_SPEED_1G);
+
+	return high_speeds != 0;
+}
+
 /*
  * General device parameters ...
  */
@@ -227,6 +265,7 @@ struct adapter_params {
 	struct arch_specific_params arch; /* chip specific params */
 	enum chip_type chip;		/* chip code */
 	u8 nports;			/* # of Ethernet "ports" */
+	u8 fw_caps_support;		/* 32-bit Port Capabilities */
 };
 
 /* Firmware Mailbox Command/Reply log.  All values are in Host-Endian format.
@@ -266,24 +305,6 @@ static inline struct mbox_cmd *mbox_cmd_log_entry(struct mbox_cmd_log *log,
 #define for_each_port(adapter, iter) \
 	for (iter = 0; iter < (adapter)->params.nports; iter++)
 
-static inline bool is_10g_port(const struct link_config *lc)
-{
-	return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0;
-}
-
-/* Return true if the Link Configuration supports "High Speeds" (those greater
- * than 1Gb/s).
- */
-static inline bool is_x_10g_port(const struct link_config *lc)
-{
-	unsigned int speeds, high_speeds;
-
-	speeds = FW_PORT_CAP_SPEED_V(FW_PORT_CAP_SPEED_G(lc->supported));
-	high_speeds = speeds & ~(FW_PORT_CAP_SPEED_100M | FW_PORT_CAP_SPEED_1G);
-
-	return high_speeds != 0;
-}
-
 static inline unsigned int core_ticks_per_usec(const struct adapter *adapter)
 {
 	return adapter->params.vpd.cclk / 1000;
@@ -387,6 +408,7 @@ int t4vf_iq_free(struct adapter *, unsigned int, unsigned int, unsigned int,
 		 unsigned int);
 int t4vf_eth_eq_free(struct adapter *, unsigned int);
 
+int t4vf_update_port_info(struct port_info *pi);
 int t4vf_handle_fw_rpl(struct adapter *, const __be64 *);
 int t4vf_prep_adapter(struct adapter *);
 int t4vf_get_vf_mac_acl(struct adapter *adapter, unsigned int pf,
diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c
index e98248f00fef..a8d94963b4d0 100644
--- a/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c
+++ b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c
@@ -313,32 +313,130 @@ int t4vf_wr_mbox_core(struct adapter *adapter, const void *cmd, int size,
 	return ret;
 }
 
-#define ADVERT_MASK (FW_PORT_CAP_SPEED_100M | FW_PORT_CAP_SPEED_1G |\
-		     FW_PORT_CAP_SPEED_10G | FW_PORT_CAP_SPEED_25G | \
-		     FW_PORT_CAP_SPEED_40G | FW_PORT_CAP_SPEED_100G | \
-		     FW_PORT_CAP_ANEG)
+#define ADVERT_MASK (FW_PORT_CAP32_SPEED_V(FW_PORT_CAP32_SPEED_M) | \
+		     FW_PORT_CAP32_ANEG)
 
 /**
+ *	fwcaps16_to_caps32 - convert 16-bit Port Capabilities to 32-bits
+ *	@caps16: a 16-bit Port Capabilities value
+ *
+ *	Returns the equivalent 32-bit Port Capabilities value.
+ */
+static fw_port_cap32_t fwcaps16_to_caps32(fw_port_cap16_t caps16)
+{
+	fw_port_cap32_t caps32 = 0;
+
+	#define CAP16_TO_CAP32(__cap) \
+		do { \
+			if (caps16 & FW_PORT_CAP_##__cap) \
+				caps32 |= FW_PORT_CAP32_##__cap; \
+		} while (0)
+
+	CAP16_TO_CAP32(SPEED_100M);
+	CAP16_TO_CAP32(SPEED_1G);
+	CAP16_TO_CAP32(SPEED_25G);
+	CAP16_TO_CAP32(SPEED_10G);
+	CAP16_TO_CAP32(SPEED_40G);
+	CAP16_TO_CAP32(SPEED_100G);
+	CAP16_TO_CAP32(FC_RX);
+	CAP16_TO_CAP32(FC_TX);
+	CAP16_TO_CAP32(ANEG);
+	CAP16_TO_CAP32(MDIX);
+	CAP16_TO_CAP32(MDIAUTO);
+	CAP16_TO_CAP32(FEC_RS);
+	CAP16_TO_CAP32(FEC_BASER_RS);
+	CAP16_TO_CAP32(802_3_PAUSE);
+	CAP16_TO_CAP32(802_3_ASM_DIR);
+
+	#undef CAP16_TO_CAP32
+
+	return caps32;
+}
+
+/* Translate Firmware Pause specification to Common Code */
+static inline enum cc_pause fwcap_to_cc_pause(fw_port_cap32_t fw_pause)
+{
+	enum cc_pause cc_pause = 0;
+
+	if (fw_pause & FW_PORT_CAP32_FC_RX)
+		cc_pause |= PAUSE_RX;
+	if (fw_pause & FW_PORT_CAP32_FC_TX)
+		cc_pause |= PAUSE_TX;
+
+	return cc_pause;
+}
+
+/* Translate Firmware Forward Error Correction specification to Common Code */
+static inline enum cc_fec fwcap_to_cc_fec(fw_port_cap32_t fw_fec)
+{
+	enum cc_fec cc_fec = 0;
+
+	if (fw_fec & FW_PORT_CAP32_FEC_RS)
+		cc_fec |= FEC_RS;
+	if (fw_fec & FW_PORT_CAP32_FEC_BASER_RS)
+		cc_fec |= FEC_BASER_RS;
+
+	return cc_fec;
+}
+
+/**
+ * Return the highest speed set in the port capabilities, in Mb/s.
+ */
+static unsigned int fwcap_to_speed(fw_port_cap32_t caps)
+{
+	#define TEST_SPEED_RETURN(__caps_speed, __speed) \
+		do { \
+			if (caps & FW_PORT_CAP32_SPEED_##__caps_speed) \
+				return __speed; \
+		} while (0)
+
+	TEST_SPEED_RETURN(400G, 400000);
+	TEST_SPEED_RETURN(200G, 200000);
+	TEST_SPEED_RETURN(100G, 100000);
+	TEST_SPEED_RETURN(50G,   50000);
+	TEST_SPEED_RETURN(40G,   40000);
+	TEST_SPEED_RETURN(25G,   25000);
+	TEST_SPEED_RETURN(10G,   10000);
+	TEST_SPEED_RETURN(1G,     1000);
+	TEST_SPEED_RETURN(100M,    100);
+
+	#undef TEST_SPEED_RETURN
+
+	return 0;
+}
+
+/*
  *	init_link_config - initialize a link's SW state
  *	@lc: structure holding the link state
- *	@caps: link capabilities
+ *	@pcaps: link Port Capabilities
+ *	@acaps: link current Advertised Port Capabilities
  *
  *	Initializes the SW state maintained for each link, including the link's
  *	capabilities and default speed/flow-control/autonegotiation settings.
  */
-static void init_link_config(struct link_config *lc, unsigned int caps)
+static void init_link_config(struct link_config *lc,
+			     fw_port_cap32_t pcaps,
+			     fw_port_cap32_t acaps)
 {
-	lc->supported = caps;
-	lc->lp_advertising = 0;
-	lc->requested_speed = 0;
+	lc->pcaps = pcaps;
+	lc->lpacaps = 0;
+	lc->speed_caps = 0;
 	lc->speed = 0;
 	lc->requested_fc = lc->fc = PAUSE_RX | PAUSE_TX;
-	if (lc->supported & FW_PORT_CAP_ANEG) {
-		lc->advertising = lc->supported & ADVERT_MASK;
+
+	/* For Forward Error Control, we default to whatever the Firmware
+	 * tells us the Link is currently advertising.
+	 */
+	lc->auto_fec = fwcap_to_cc_fec(acaps);
+	lc->requested_fec = FEC_AUTO;
+	lc->fec = lc->auto_fec;
+
+	if (lc->pcaps & FW_PORT_CAP32_ANEG) {
+		lc->acaps = acaps & ADVERT_MASK;
 		lc->autoneg = AUTONEG_ENABLE;
 		lc->requested_fc |= PAUSE_AUTONEG;
 	} else {
-		lc->advertising = 0;
+		lc->acaps = 0;
 		lc->autoneg = AUTONEG_DISABLE;
 	}
 }
@@ -351,9 +449,30 @@ static void init_link_config(struct link_config *lc, unsigned int caps)
 int t4vf_port_init(struct adapter *adapter, int pidx)
 {
 	struct port_info *pi = adap2pinfo(adapter, pidx);
+	unsigned int fw_caps = adapter->params.fw_caps_support;
 	struct fw_vi_cmd vi_cmd, vi_rpl;
 	struct fw_port_cmd port_cmd, port_rpl;
-	int v;
+	enum fw_port_type port_type;
+	int mdio_addr;
+	fw_port_cap32_t pcaps, acaps;
+	int ret;
+
+	/* If we haven't yet determined whether we're talking to Firmware
+	 * which knows the new 32-bit Port Capabilities, it's time to find
+	 * out now.  This will also tell new Firmware to send us Port Status
+	 * Updates using the new 32-bit Port Capabilities version of the
+	 * Port Information message.
+	 */
+	if (fw_caps == FW_CAPS_UNKNOWN) {
+		u32 param, val;
+
+		param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) |
+			 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_PORT_CAPS32));
+		val = 1;
+		ret = t4vf_set_params(adapter, 1, &param, &val);
+		fw_caps = (ret == 0 ? FW_CAPS32 : FW_CAPS16);
+		adapter->params.fw_caps_support = fw_caps;
+	}
 
 	/*
 	 * Execute a VI Read command to get our Virtual Interface information
@@ -365,9 +484,9 @@ int t4vf_port_init(struct adapter *adapter, int pidx)
 				       FW_CMD_READ_F);
 	vi_cmd.alloc_to_len16 = cpu_to_be32(FW_LEN16(vi_cmd));
 	vi_cmd.type_viid = cpu_to_be16(FW_VI_CMD_VIID_V(pi->viid));
-	v = t4vf_wr_mbox(adapter, &vi_cmd, sizeof(vi_cmd), &vi_rpl);
-	if (v)
-		return v;
+	ret = t4vf_wr_mbox(adapter, &vi_cmd, sizeof(vi_cmd), &vi_rpl);
+	if (ret != FW_SUCCESS)
+		return ret;
 
 	BUG_ON(pi->port_id != FW_VI_CMD_PORTID_G(vi_rpl.portid_pkd));
 	pi->rss_size = FW_VI_CMD_RSSSIZE_G(be16_to_cpu(vi_rpl.rsssize_pkd));
@@ -385,21 +504,42 @@ int t4vf_port_init(struct adapter *adapter, int pidx)
 					    FW_CMD_REQUEST_F |
 					    FW_CMD_READ_F |
 					    FW_PORT_CMD_PORTID_V(pi->port_id));
-	port_cmd.action_to_len16 =
-		cpu_to_be32(FW_PORT_CMD_ACTION_V(FW_PORT_ACTION_GET_PORT_INFO) |
-			    FW_LEN16(port_cmd));
-	v = t4vf_wr_mbox(adapter, &port_cmd, sizeof(port_cmd), &port_rpl);
-	if (v)
-		return v;
+	port_cmd.action_to_len16 = cpu_to_be32(
+		FW_PORT_CMD_ACTION_V(fw_caps == FW_CAPS16
+				     ? FW_PORT_ACTION_GET_PORT_INFO
+				     : FW_PORT_ACTION_GET_PORT_INFO32) |
+		FW_LEN16(port_cmd));
+	ret = t4vf_wr_mbox(adapter, &port_cmd, sizeof(port_cmd), &port_rpl);
+	if (ret != FW_SUCCESS)
+		return ret;
 
-	v = be32_to_cpu(port_rpl.u.info.lstatus_to_modtype);
-	pi->mdio_addr = (v & FW_PORT_CMD_MDIOCAP_F) ?
-			FW_PORT_CMD_MDIOADDR_G(v) : -1;
-	pi->port_type = FW_PORT_CMD_PTYPE_G(v);
-	pi->mod_type = FW_PORT_MOD_TYPE_NA;
+	/* Extract the various fields from the Port Information message. */
+	if (fw_caps == FW_CAPS16) {
+		u32 lstatus = be32_to_cpu(port_rpl.u.info.lstatus_to_modtype);
 
-	init_link_config(&pi->link_cfg, be16_to_cpu(port_rpl.u.info.pcap));
+		port_type = FW_PORT_CMD_PTYPE_G(lstatus);
+		mdio_addr = ((lstatus & FW_PORT_CMD_MDIOCAP_F)
+			     ? FW_PORT_CMD_MDIOADDR_G(lstatus)
+			     : -1);
+		pcaps = fwcaps16_to_caps32(be16_to_cpu(port_rpl.u.info.pcap));
+		acaps = fwcaps16_to_caps32(be16_to_cpu(port_rpl.u.info.acap));
+	} else {
+		u32 lstatus32 =
+			   be32_to_cpu(port_rpl.u.info32.lstatus32_to_cbllen32);
+
+		port_type = FW_PORT_CMD_PORTTYPE32_G(lstatus32);
+		mdio_addr = ((lstatus32 & FW_PORT_CMD_MDIOCAP32_F)
+			     ? FW_PORT_CMD_MDIOADDR32_G(lstatus32)
+			     : -1);
+		pcaps = be32_to_cpu(port_rpl.u.info32.pcaps32);
+		acaps = be32_to_cpu(port_rpl.u.info32.acaps32);
+	}
 
+	pi->port_type = port_type;
+	pi->mdio_addr = mdio_addr;
+	pi->mod_type = FW_PORT_MOD_TYPE_NA;
+
+	init_link_config(&pi->link_cfg, pcaps, acaps);
 	return 0;
 }
 
@@ -1667,6 +1807,202 @@ int t4vf_eth_eq_free(struct adapter *adapter, unsigned int eqid)
 }
 
 /**
+ *	t4vf_link_down_rc_str - return a string for a Link Down Reason Code
+ *	@link_down_rc: Link Down Reason Code
+ *
+ *	Returns a string representation of the Link Down Reason Code.
+ */
+const char *t4vf_link_down_rc_str(unsigned char link_down_rc)
+{
+	static const char * const reason[] = {
+		"Link Down",
+		"Remote Fault",
+		"Auto-negotiation Failure",
+		"Reserved",
+		"Insufficient Airflow",
+		"Unable To Determine Reason",
+		"No RX Signal Detected",
+		"Reserved",
+	};
+
+	if (link_down_rc >= ARRAY_SIZE(reason))
+		return "Bad Reason Code";
+
+	return reason[link_down_rc];
+}
+
+/**
+ *	t4vf_handle_get_port_info - process a FW reply message
+ *	@pi: the port info
+ *	@rpl: start of the FW message
+ *
+ *	Processes a GET_PORT_INFO FW reply message.
+ */
+void t4vf_handle_get_port_info(struct port_info *pi,
+			       const struct fw_port_cmd *cmd)
+{
+	int action = FW_PORT_CMD_ACTION_G(be32_to_cpu(cmd->action_to_len16));
+	struct adapter *adapter = pi->adapter;
+	struct link_config *lc = &pi->link_cfg;
+	int link_ok, linkdnrc;
+	enum fw_port_type port_type;
+	enum fw_port_module_type mod_type;
+	unsigned int speed, fc, fec;
+	fw_port_cap32_t pcaps, acaps, lpacaps, linkattr;
+
+	/* Extract the various fields from the Port Information message. */
+	switch (action) {
+	case FW_PORT_ACTION_GET_PORT_INFO: {
+		u32 lstatus = be32_to_cpu(cmd->u.info.lstatus_to_modtype);
+
+		link_ok = (lstatus & FW_PORT_CMD_LSTATUS_F) != 0;
+		linkdnrc = FW_PORT_CMD_LINKDNRC_G(lstatus);
+		port_type = FW_PORT_CMD_PTYPE_G(lstatus);
+		mod_type = FW_PORT_CMD_MODTYPE_G(lstatus);
+		pcaps = fwcaps16_to_caps32(be16_to_cpu(cmd->u.info.pcap));
+		acaps = fwcaps16_to_caps32(be16_to_cpu(cmd->u.info.acap));
+		lpacaps = fwcaps16_to_caps32(be16_to_cpu(cmd->u.info.lpacap));
+
+		/* Unfortunately the format of the Link Status in the old
+		 * 16-bit Port Information message isn't the same as the
+		 * 16-bit Port Capabilities bitfield used everywhere else ...
+		 */
+		linkattr = 0;
+		if (lstatus & FW_PORT_CMD_RXPAUSE_F)
+			linkattr |= FW_PORT_CAP32_FC_RX;
+		if (lstatus & FW_PORT_CMD_TXPAUSE_F)
+			linkattr |= FW_PORT_CAP32_FC_TX;
+		if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_100M))
+			linkattr |= FW_PORT_CAP32_SPEED_100M;
+		if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_1G))
+			linkattr |= FW_PORT_CAP32_SPEED_1G;
+		if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_10G))
+			linkattr |= FW_PORT_CAP32_SPEED_10G;
+		if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_25G))
+			linkattr |= FW_PORT_CAP32_SPEED_25G;
+		if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_40G))
+			linkattr |= FW_PORT_CAP32_SPEED_40G;
+		if (lstatus & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_100G))
+			linkattr |= FW_PORT_CAP32_SPEED_100G;
+
+		break;
+	}
+
+	case FW_PORT_ACTION_GET_PORT_INFO32: {
+		u32 lstatus32;
+
+		lstatus32 = be32_to_cpu(cmd->u.info32.lstatus32_to_cbllen32);
+		link_ok = (lstatus32 & FW_PORT_CMD_LSTATUS32_F) != 0;
+		linkdnrc = FW_PORT_CMD_LINKDNRC32_G(lstatus32);
+		port_type = FW_PORT_CMD_PORTTYPE32_G(lstatus32);
+		mod_type = FW_PORT_CMD_MODTYPE32_G(lstatus32);
+		pcaps = be32_to_cpu(cmd->u.info32.pcaps32);
+		acaps = be32_to_cpu(cmd->u.info32.acaps32);
+		lpacaps = be32_to_cpu(cmd->u.info32.lpacaps32);
+		linkattr = be32_to_cpu(cmd->u.info32.linkattr32);
+		break;
+	}
+
+	default:
+		dev_err(adapter->pdev_dev, "Handle Port Information: Bad Command/Action %#x\n",
+			be32_to_cpu(cmd->action_to_len16));
+		return;
+	}
+
+	fec = fwcap_to_cc_fec(acaps);
+	fc = fwcap_to_cc_pause(linkattr);
+	speed = fwcap_to_speed(linkattr);
+
+	if (mod_type != pi->mod_type) {
+		/* When a new Transceiver Module is inserted, the Firmware
+		 * will examine any Forward Error Correction parameters
+		 * present in the Transceiver Module i2c EPROM and determine
+		 * the supported and recommended FEC settings from those
+		 * based on IEEE 802.3 standards.  We always record the
+		 * IEEE 802.3 recommended "automatic" settings.
+		 */
+		lc->auto_fec = fec;
+
+		/* Some versions of the early T6 Firmware "cheated" when
+		 * handling different Transceiver Modules by changing the
+		 * underlaying Port Type reported to the Host Drivers.  As
+		 * such we need to capture whatever Port Type the Firmware
+		 * sends us and record it in case it's different from what we
+		 * were told earlier.  Unfortunately, since Firmware is
+		 * forever, we'll need to keep this code here forever, but in
+		 * later T6 Firmware it should just be an assignment of the
+		 * same value already recorded.
+		 */
+		pi->port_type = port_type;
+
+		pi->mod_type = mod_type;
+		t4vf_os_portmod_changed(adapter, pi->pidx);
+	}
+
+	if (link_ok != lc->link_ok || speed != lc->speed ||
+	    fc != lc->fc || fec != lc->fec) {	/* something changed */
+		if (!link_ok && lc->link_ok) {
+			lc->link_down_rc = linkdnrc;
+			dev_warn(adapter->pdev_dev, "Port %d link down, reason: %s\n",
+				 pi->port_id, t4vf_link_down_rc_str(linkdnrc));
+		}
+		lc->link_ok = link_ok;
+		lc->speed = speed;
+		lc->fc = fc;
+		lc->fec = fec;
+
+		lc->pcaps = pcaps;
+		lc->lpacaps = lpacaps;
+		lc->acaps = acaps & ADVERT_MASK;
+
+		if (lc->acaps & FW_PORT_CAP32_ANEG) {
+			lc->autoneg = AUTONEG_ENABLE;
+		} else {
+			/* When Autoneg is disabled, user needs to set
+			 * single speed.
+			 * Similar to cxgb4_ethtool.c: set_link_ksettings
+			 */
+			lc->acaps = 0;
+			lc->speed_caps = fwcap_to_speed(acaps);
+			lc->autoneg = AUTONEG_DISABLE;
+		}
+
+		t4vf_os_link_changed(adapter, pi->pidx, link_ok);
+	}
+}
+
+/**
+ *	t4vf_update_port_info - retrieve and update port information if changed
+ *	@pi: the port_info
+ *
+ *	We issue a Get Port Information Command to the Firmware and, if
+ *	successful, we check to see if anything is different from what we
+ *	last recorded and update things accordingly.
+ */
+int t4vf_update_port_info(struct port_info *pi)
+{
+	unsigned int fw_caps = pi->adapter->params.fw_caps_support;
+	struct fw_port_cmd port_cmd;
+	int ret;
+
+	memset(&port_cmd, 0, sizeof(port_cmd));
+	port_cmd.op_to_portid = cpu_to_be32(FW_CMD_OP_V(FW_PORT_CMD) |
+					    FW_CMD_REQUEST_F | FW_CMD_READ_F |
+					    FW_PORT_CMD_PORTID_V(pi->port_id));
+	port_cmd.action_to_len16 = cpu_to_be32(
+		FW_PORT_CMD_ACTION_V(fw_caps == FW_CAPS16
+				     ? FW_PORT_ACTION_GET_PORT_INFO
+				     : FW_PORT_ACTION_GET_PORT_INFO32) |
+		FW_LEN16(port_cmd));
+	ret = t4vf_wr_mbox(pi->adapter, &port_cmd, sizeof(port_cmd),
+			   &port_cmd);
+	if (ret)
+		return ret;
+	t4vf_handle_get_port_info(pi, &port_cmd);
+	return 0;
+}
+
+/**
  *	t4vf_handle_fw_rpl - process a firmware reply message
  *	@adapter: the adapter
  *	@rpl: start of the firmware message
@@ -1685,15 +2021,12 @@ int t4vf_handle_fw_rpl(struct adapter *adapter, const __be64 *rpl)
 		 */
 		const struct fw_port_cmd *port_cmd =
 			(const struct fw_port_cmd *)rpl;
-		u32 stat, mod;
-		int action, port_id, link_ok, speed, fc, pidx;
-
-		/*
-		 * Extract various fields from port status change message.
-		 */
-		action = FW_PORT_CMD_ACTION_G(
+		int action = FW_PORT_CMD_ACTION_G(
 			be32_to_cpu(port_cmd->action_to_len16));
-		if (action != FW_PORT_ACTION_GET_PORT_INFO) {
+		int port_id, pidx;
+
+		if (action != FW_PORT_ACTION_GET_PORT_INFO &&
+		    action != FW_PORT_ACTION_GET_PORT_INFO32) {
 			dev_err(adapter->pdev_dev,
 				"Unknown firmware PORT reply action %x\n",
 				action);
@@ -1702,61 +2035,12 @@ int t4vf_handle_fw_rpl(struct adapter *adapter, const __be64 *rpl)
 
 		port_id = FW_PORT_CMD_PORTID_G(
 			be32_to_cpu(port_cmd->op_to_portid));
-
-		stat = be32_to_cpu(port_cmd->u.info.lstatus_to_modtype);
-		link_ok = (stat & FW_PORT_CMD_LSTATUS_F) != 0;
-		speed = 0;
-		fc = 0;
-		if (stat & FW_PORT_CMD_RXPAUSE_F)
-			fc |= PAUSE_RX;
-		if (stat & FW_PORT_CMD_TXPAUSE_F)
-			fc |= PAUSE_TX;
-		if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_100M))
-			speed = 100;
-		else if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_1G))
-			speed = 1000;
-		else if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_10G))
-			speed = 10000;
-		else if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_25G))
-			speed = 25000;
-		else if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_40G))
-			speed = 40000;
-		else if (stat & FW_PORT_CMD_LSPEED_V(FW_PORT_CAP_SPEED_100G))
-			speed = 100000;
-
-		/*
-		 * Scan all of our "ports" (Virtual Interfaces) looking for
-		 * those bound to the physical port which has changed.  If
-		 * our recorded state doesn't match the current state,
-		 * signal that change to the OS code.
-		 */
 		for_each_port(adapter, pidx) {
 			struct port_info *pi = adap2pinfo(adapter, pidx);
-			struct link_config *lc;
 
 			if (pi->port_id != port_id)
 				continue;
-
-			lc = &pi->link_cfg;
-
-			mod = FW_PORT_CMD_MODTYPE_G(stat);
-			if (mod != pi->mod_type) {
-				pi->mod_type = mod;
-				t4vf_os_portmod_changed(adapter, pidx);
-			}
-
-			if (link_ok != lc->link_ok || speed != lc->speed ||
-			    fc != lc->fc) {
-				/* something changed */
-				lc->link_ok = link_ok;
-				lc->speed = speed;
-				lc->fc = fc;
-				lc->supported =
-					be16_to_cpu(port_cmd->u.info.pcap);
-				lc->lp_advertising =
-					be16_to_cpu(port_cmd->u.info.lpacap);
-				t4vf_os_link_changed(adapter, pidx, link_ok);
-			}
+			t4vf_handle_get_port_info(pi, port_cmd);
 		}
 		break;
 	}
diff --git a/drivers/net/ethernet/davicom/dm9000.c b/drivers/net/ethernet/davicom/dm9000.c
index 16fe776ddbe5..50222b7b81f3 100644
--- a/drivers/net/ethernet/davicom/dm9000.c
+++ b/drivers/net/ethernet/davicom/dm9000.c
@@ -65,7 +65,7 @@ MODULE_PARM_DESC(watchdog, "transmit timeout in milliseconds");
  */
 static int debug;
 module_param(debug, int, 0644);
-MODULE_PARM_DESC(debug, "dm9000 debug level (0-4)");
+MODULE_PARM_DESC(debug, "dm9000 debug level (0-6)");
 
 /* DM9000 register address locking.
  *
diff --git a/drivers/net/ethernet/dec/tulip/de4x5.c b/drivers/net/ethernet/dec/tulip/de4x5.c
index 47be5018d35d..0affee9c8aa2 100644
--- a/drivers/net/ethernet/dec/tulip/de4x5.c
+++ b/drivers/net/ethernet/dec/tulip/de4x5.c
@@ -2094,7 +2094,7 @@ static int de4x5_eisa_remove(struct device *device)
 	return 0;
 }
 
-static struct eisa_device_id de4x5_eisa_ids[] = {
+static const struct eisa_device_id de4x5_eisa_ids[] = {
         { "DEC4250", 0 },	/* 0 is the board name index... */
         { "" }
 };
diff --git a/drivers/net/ethernet/dec/tulip/tulip.h b/drivers/net/ethernet/dec/tulip/tulip.h
index 38431a155f09..06660dbc44b7 100644
--- a/drivers/net/ethernet/dec/tulip/tulip.h
+++ b/drivers/net/ethernet/dec/tulip/tulip.h
@@ -515,7 +515,7 @@ void comet_timer(unsigned long data);
 extern int tulip_debug;
 extern const char * const medianame[];
 extern const char tulip_media_cap[];
-extern struct tulip_chip_table tulip_tbl[];
+extern const struct tulip_chip_table tulip_tbl[];
 void oom_timer(unsigned long data);
 extern u8 t21040_csr13[];
 
diff --git a/drivers/net/ethernet/dec/tulip/tulip_core.c b/drivers/net/ethernet/dec/tulip/tulip_core.c
index 17e566a8b345..851b6d1f5a42 100644
--- a/drivers/net/ethernet/dec/tulip/tulip_core.c
+++ b/drivers/net/ethernet/dec/tulip/tulip_core.c
@@ -138,7 +138,7 @@ static void tulip_timer(unsigned long data)
  * It is indexed via the values in 'enum chips'
  */
 
-struct tulip_chip_table tulip_tbl[] = {
+const struct tulip_chip_table tulip_tbl[] = {
   { }, /* placeholder for array, slot unused currently */
   { }, /* placeholder for array, slot unused currently */
 
@@ -1303,7 +1303,6 @@ static int tulip_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 		0x00, 'L', 'i', 'n', 'u', 'x'
 	};
 	static int last_irq;
-	static int multiport_cnt;	/* For four-port boards w/one EEPROM */
 	int i, irq;
 	unsigned short sum;
 	unsigned char *ee_data;
@@ -1557,7 +1556,6 @@ static int tulip_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 		} else if (ee_data[0] == 0xff  &&  ee_data[1] == 0xff &&
 				   ee_data[2] == 0) {
 			sa_offset = 2;		/* Grrr, damn Matrox boards. */
-			multiport_cnt = 4;
 		}
 #ifdef CONFIG_MIPS_COBALT
                if ((pdev->bus->number == 0) &&
diff --git a/drivers/net/ethernet/ec_bhf.c b/drivers/net/ethernet/ec_bhf.c
index 4ee042c034a1..1b79a6defd56 100644
--- a/drivers/net/ethernet/ec_bhf.c
+++ b/drivers/net/ethernet/ec_bhf.c
@@ -73,7 +73,7 @@
 
 #define ETHERCAT_MASTER_ID	0x14
 
-static struct pci_device_id ids[] = {
+static const struct pci_device_id ids[] = {
 	{ PCI_DEVICE(0x15ec, 0x5000), },
 	{ 0, }
 };
diff --git a/drivers/net/ethernet/emulex/benet/be.h b/drivers/net/ethernet/emulex/benet/be.h
index 674cf9d13b98..8984c4938881 100644
--- a/drivers/net/ethernet/emulex/benet/be.h
+++ b/drivers/net/ethernet/emulex/benet/be.h
@@ -930,6 +930,14 @@ static inline bool is_ipv4_pkt(struct sk_buff *skb)
 	return skb->protocol == htons(ETH_P_IP) && ip_hdr(skb)->version == 4;
 }
 
+static inline bool is_ipv6_ext_hdr(struct sk_buff *skb)
+{
+	if (ip_hdr(skb)->version == 6)
+		return ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr);
+	else
+		return false;
+}
+
 #define be_error_recovering(adapter)	\
 		(adapter->flags & BE_FLAGS_TRY_RECOVERY)
 
diff --git a/drivers/net/ethernet/emulex/benet/be_main.c b/drivers/net/ethernet/emulex/benet/be_main.c
index 319eee36649b..0e3d9f39a807 100644
--- a/drivers/net/ethernet/emulex/benet/be_main.c
+++ b/drivers/net/ethernet/emulex/benet/be_main.c
@@ -5089,6 +5089,20 @@ static netdev_features_t be_features_check(struct sk_buff *skb,
 	struct be_adapter *adapter = netdev_priv(dev);
 	u8 l4_hdr = 0;
 
+	if (skb_is_gso(skb)) {
+		/* IPv6 TSO requests with extension hdrs are a problem
+		 * to Lancer and BE3 HW. Disable TSO6 feature.
+		 */
+		if (!skyhawk_chip(adapter) && is_ipv6_ext_hdr(skb))
+			features &= ~NETIF_F_TSO6;
+
+		/* Lancer cannot handle the packet with MSS less than 256.
+		 * Disable the GSO support in such cases
+		 */
+		if (lancer_chip(adapter) && skb_shinfo(skb)->gso_size < 256)
+			features &= ~NETIF_F_GSO_MASK;
+	}
+
 	/* The code below restricts offload features for some tunneled and
 	 * Q-in-Q packets.
 	 * Offload features for normal (non tunnel) packets are unchanged.
diff --git a/drivers/net/ethernet/emulex/benet/be_roce.c b/drivers/net/ethernet/emulex/benet/be_roce.c
index 2b62841c4c63..05989aafaf32 100644
--- a/drivers/net/ethernet/emulex/benet/be_roce.c
+++ b/drivers/net/ethernet/emulex/benet/be_roce.c
@@ -139,10 +139,7 @@ int be_roce_register_driver(struct ocrdma_driver *drv)
 	}
 	ocrdma_drv = drv;
 	list_for_each_entry(dev, &be_adapter_list, entry) {
-		struct net_device *netdev;
-
 		_be_roce_dev_add(dev);
-		netdev = dev->netdev;
 	}
 	mutex_unlock(&be_adapter_list_lock);
 	return 0;
diff --git a/drivers/net/ethernet/faraday/ftgmac100.c b/drivers/net/ethernet/faraday/ftgmac100.c
index 34dae51effd4..9ed8e4b81530 100644
--- a/drivers/net/ethernet/faraday/ftgmac100.c
+++ b/drivers/net/ethernet/faraday/ftgmac100.c
@@ -1623,6 +1623,8 @@ static const struct net_device_ops ftgmac100_netdev_ops = {
 #ifdef CONFIG_NET_POLL_CONTROLLER
 	.ndo_poll_controller	= ftgmac100_poll_controller,
 #endif
+	.ndo_vlan_rx_add_vid	= ncsi_vlan_rx_add_vid,
+	.ndo_vlan_rx_kill_vid	= ncsi_vlan_rx_kill_vid,
 };
 
 static int ftgmac100_setup_mdio(struct net_device *netdev)
@@ -1837,6 +1839,9 @@ static int ftgmac100_probe(struct platform_device *pdev)
 		NETIF_F_GRO | NETIF_F_SG | NETIF_F_HW_VLAN_CTAG_RX |
 		NETIF_F_HW_VLAN_CTAG_TX;
 
+	if (priv->use_ncsi)
+		netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
+
 	/* AST2400  doesn't have working HW checksum generation */
 	if (np && (of_device_is_compatible(np, "aspeed,ast2400-mac")))
 		netdev->hw_features &= ~NETIF_F_HW_CSUM;
@@ -1863,7 +1868,6 @@ err_setup_mdio:
 err_ioremap:
 	release_resource(priv->res);
 err_req_mem:
-	netif_napi_del(&priv->napi);
 	free_netdev(netdev);
 err_alloc_etherdev:
 	return err;
diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c
index 757b873735a5..42258060f142 100644
--- a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c
+++ b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c
@@ -158,7 +158,7 @@ MODULE_PARM_DESC(tx_timeout, "The Tx timeout in ms");
 #define DPAA_RX_PRIV_DATA_SIZE	(u16)(DPAA_TX_PRIV_DATA_SIZE + \
 					dpaa_rx_extra_headroom)
 
-#define DPAA_ETH_RX_QUEUES	128
+#define DPAA_ETH_PCD_RXQ_NUM	128
 
 #define DPAA_ENQUEUE_RETRIES	100000
 
@@ -169,6 +169,7 @@ struct fm_port_fqs {
 	struct dpaa_fq *tx_errq;
 	struct dpaa_fq *rx_defq;
 	struct dpaa_fq *rx_errq;
+	struct dpaa_fq *rx_pcdq;
 };
 
 /* All the dpa bps in use at any moment */
@@ -235,7 +236,7 @@ static int dpaa_netdev_init(struct net_device *net_dev,
 	net_dev->max_mtu = dpaa_get_max_mtu();
 
 	net_dev->hw_features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
-				 NETIF_F_LLTX);
+				 NETIF_F_LLTX | NETIF_F_RXHASH);
 
 	net_dev->hw_features |= NETIF_F_SG | NETIF_F_HIGHDMA;
 	/* The kernels enables GSO automatically, if we declare NETIF_F_SG.
@@ -342,18 +343,19 @@ static void dpaa_get_stats64(struct net_device *net_dev,
 	}
 }
 
-static int dpaa_setup_tc(struct net_device *net_dev, u32 handle,
-			 u32 chain_index, __be16 proto, struct tc_to_netdev *tc)
+static int dpaa_setup_tc(struct net_device *net_dev, enum tc_setup_type type,
+			 void *type_data)
 {
 	struct dpaa_priv *priv = netdev_priv(net_dev);
+	struct tc_mqprio_qopt *mqprio = type_data;
 	u8 num_tc;
 	int i;
 
-	if (tc->type != TC_SETUP_MQPRIO)
-		return -EINVAL;
+	if (type != TC_SETUP_MQPRIO)
+		return -EOPNOTSUPP;
 
-	tc->mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
-	num_tc = tc->mqprio->num_tc;
+	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
+	num_tc = mqprio->num_tc;
 
 	if (num_tc == priv->num_tc)
 		return 0;
@@ -398,8 +400,8 @@ static struct mac_device *dpaa_mac_dev_get(struct platform_device *pdev)
 
 	of_dev = of_find_device_by_node(mac_node);
 	if (!of_dev) {
-		dev_err(dpaa_dev, "of_find_device_by_node(%s) failed\n",
-			mac_node->full_name);
+		dev_err(dpaa_dev, "of_find_device_by_node(%pOF) failed\n",
+			mac_node);
 		of_node_put(mac_node);
 		return ERR_PTR(-EINVAL);
 	}
@@ -627,6 +629,7 @@ static inline void dpaa_assign_wq(struct dpaa_fq *fq, int idx)
 		fq->wq = 5;
 		break;
 	case FQ_TYPE_RX_DEFAULT:
+	case FQ_TYPE_RX_PCD:
 		fq->wq = 6;
 		break;
 	case FQ_TYPE_TX:
@@ -687,6 +690,7 @@ static int dpaa_alloc_all_fqs(struct device *dev, struct list_head *list,
 			      struct fm_port_fqs *port_fqs)
 {
 	struct dpaa_fq *dpaa_fq;
+	u32 fq_base, fq_base_aligned, i;
 
 	dpaa_fq = dpaa_fq_alloc(dev, 0, 1, list, FQ_TYPE_RX_ERROR);
 	if (!dpaa_fq)
@@ -700,6 +704,26 @@ static int dpaa_alloc_all_fqs(struct device *dev, struct list_head *list,
 
 	port_fqs->rx_defq = &dpaa_fq[0];
 
+	/* the PCD FQIDs range needs to be aligned for correct operation */
+	if (qman_alloc_fqid_range(&fq_base, 2 * DPAA_ETH_PCD_RXQ_NUM))
+		goto fq_alloc_failed;
+
+	fq_base_aligned = ALIGN(fq_base, DPAA_ETH_PCD_RXQ_NUM);
+
+	for (i = fq_base; i < fq_base_aligned; i++)
+		qman_release_fqid(i);
+
+	for (i = fq_base_aligned + DPAA_ETH_PCD_RXQ_NUM;
+	     i < (fq_base + 2 * DPAA_ETH_PCD_RXQ_NUM); i++)
+		qman_release_fqid(i);
+
+	dpaa_fq = dpaa_fq_alloc(dev, fq_base_aligned, DPAA_ETH_PCD_RXQ_NUM,
+				list, FQ_TYPE_RX_PCD);
+	if (!dpaa_fq)
+		goto fq_alloc_failed;
+
+	port_fqs->rx_pcdq = &dpaa_fq[0];
+
 	if (!dpaa_fq_alloc(dev, 0, DPAA_ETH_TXQ_NUM, list, FQ_TYPE_TX_CONF_MQ))
 		goto fq_alloc_failed;
 
@@ -869,13 +893,14 @@ static void dpaa_fq_setup(struct dpaa_priv *priv,
 			  const struct dpaa_fq_cbs *fq_cbs,
 			  struct fman_port *tx_port)
 {
-	int egress_cnt = 0, conf_cnt = 0, num_portals = 0, cpu;
+	int egress_cnt = 0, conf_cnt = 0, num_portals = 0, portal_cnt = 0, cpu;
 	const cpumask_t *affine_cpus = qman_affine_cpus();
-	u16 portals[NR_CPUS];
+	u16 channels[NR_CPUS];
 	struct dpaa_fq *fq;
 
 	for_each_cpu(cpu, affine_cpus)
-		portals[num_portals++] = qman_affine_channel(cpu);
+		channels[num_portals++] = qman_affine_channel(cpu);
+
 	if (num_portals == 0)
 		dev_err(priv->net_dev->dev.parent,
 			"No Qman software (affine) channels found");
@@ -889,6 +914,12 @@ static void dpaa_fq_setup(struct dpaa_priv *priv,
 		case FQ_TYPE_RX_ERROR:
 			dpaa_setup_ingress(priv, fq, &fq_cbs->rx_errq);
 			break;
+		case FQ_TYPE_RX_PCD:
+			if (!num_portals)
+				continue;
+			dpaa_setup_ingress(priv, fq, &fq_cbs->rx_defq);
+			fq->channel = channels[portal_cnt++ % num_portals];
+			break;
 		case FQ_TYPE_TX:
 			dpaa_setup_egress(priv, fq, tx_port,
 					  &fq_cbs->egress_ern);
@@ -1038,7 +1069,8 @@ static int dpaa_fq_init(struct dpaa_fq *dpaa_fq, bool td_enable)
 		/* Put all the ingress queues in our "ingress CGR". */
 		if (priv->use_ingress_cgr &&
 		    (dpaa_fq->fq_type == FQ_TYPE_RX_DEFAULT ||
-		     dpaa_fq->fq_type == FQ_TYPE_RX_ERROR)) {
+		     dpaa_fq->fq_type == FQ_TYPE_RX_ERROR ||
+		     dpaa_fq->fq_type == FQ_TYPE_RX_PCD)) {
 			initfq.we_mask |= cpu_to_be16(QM_INITFQ_WE_CGID);
 			initfq.fqd.fq_ctrl |= cpu_to_be16(QM_FQCTRL_CGE);
 			initfq.fqd.cgid = (u8)priv->ingress_cgr.cgrid;
@@ -1169,7 +1201,7 @@ static int dpaa_eth_init_tx_port(struct fman_port *port, struct dpaa_fq *errq,
 
 static int dpaa_eth_init_rx_port(struct fman_port *port, struct dpaa_bp **bps,
 				 size_t count, struct dpaa_fq *errq,
-				 struct dpaa_fq *defq,
+				 struct dpaa_fq *defq, struct dpaa_fq *pcdq,
 				 struct dpaa_buffer_layout *buf_layout)
 {
 	struct fman_buffer_prefix_content buf_prefix_content;
@@ -1189,6 +1221,10 @@ static int dpaa_eth_init_rx_port(struct fman_port *port, struct dpaa_bp **bps,
 	rx_p = &params.specific_params.rx_params;
 	rx_p->err_fqid = errq->fqid;
 	rx_p->dflt_fqid = defq->fqid;
+	if (pcdq) {
+		rx_p->pcd_base_fqid = pcdq->fqid;
+		rx_p->pcd_fqs_count = DPAA_ETH_PCD_RXQ_NUM;
+	}
 
 	count = min(ARRAY_SIZE(rx_p->ext_buf_pools.ext_buf_pool), count);
 	rx_p->ext_buf_pools.num_of_pools_used = (u8)count;
@@ -1233,7 +1269,8 @@ static int dpaa_eth_init_ports(struct mac_device *mac_dev,
 		return err;
 
 	err = dpaa_eth_init_rx_port(rxport, bps, count, port_fqs->rx_errq,
-				    port_fqs->rx_defq, &buf_layout[RX]);
+				    port_fqs->rx_defq, port_fqs->rx_pcdq,
+				    &buf_layout[RX]);
 
 	return err;
 }
@@ -2200,12 +2237,13 @@ static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
 	dma_addr_t addr = qm_fd_addr(fd);
 	enum qm_fd_format fd_format;
 	struct net_device *net_dev;
-	u32 fd_status;
+	u32 fd_status, hash_offset;
 	struct dpaa_bp *dpaa_bp;
 	struct dpaa_priv *priv;
 	unsigned int skb_len;
 	struct sk_buff *skb;
 	int *count_ptr;
+	void *vaddr;
 
 	fd_status = be32_to_cpu(fd->status);
 	fd_format = qm_fd_get_format(fd);
@@ -2251,7 +2289,8 @@ static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
 	dma_unmap_single(dpaa_bp->dev, addr, dpaa_bp->size, DMA_FROM_DEVICE);
 
 	/* prefetch the first 64 bytes of the frame or the SGT start */
-	prefetch(phys_to_virt(addr) + qm_fd_get_offset(fd));
+	vaddr = phys_to_virt(addr);
+	prefetch(vaddr + qm_fd_get_offset(fd));
 
 	fd_format = qm_fd_get_format(fd);
 	/* The only FD types that we may receive are contig and S/G */
@@ -2272,6 +2311,18 @@ static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
 
 	skb->protocol = eth_type_trans(skb, net_dev);
 
+	if (net_dev->features & NETIF_F_RXHASH && priv->keygen_in_use &&
+	    !fman_port_get_hash_result_offset(priv->mac_dev->port[RX],
+					      &hash_offset)) {
+		enum pkt_hash_types type;
+
+		/* if L4 exists, it was used in the hash generation */
+		type = be32_to_cpu(fd->status) & FM_FD_STAT_L4CV ?
+			PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3;
+		skb_set_hash(skb, be32_to_cpu(*(u32 *)(vaddr + hash_offset)),
+			     type);
+	}
+
 	skb_len = skb->len;
 
 	if (unlikely(netif_receive_skb(skb) == NET_RX_DROP))
@@ -2510,6 +2561,9 @@ static struct dpaa_bp *dpaa_bp_alloc(struct device *dev)
 
 	dpaa_bp->bpid = FSL_DPAA_BPID_INV;
 	dpaa_bp->percpu_count = devm_alloc_percpu(dev, *dpaa_bp->percpu_count);
+	if (!dpaa_bp->percpu_count)
+		return ERR_PTR(-ENOMEM);
+
 	dpaa_bp->config_count = FSL_DPAA_ETH_MAX_BUF_COUNT;
 
 	dpaa_bp->seed_cb = dpaa_bp_seed;
@@ -2737,6 +2791,9 @@ static int dpaa_eth_probe(struct platform_device *pdev)
 	if (err)
 		goto init_ports_failed;
 
+	/* Rx traffic distribution based on keygen hashing defaults to on */
+	priv->keygen_in_use = true;
+
 	priv->percpu_priv = devm_alloc_percpu(dev, *priv->percpu_priv);
 	if (!priv->percpu_priv) {
 		dev_err(dev, "devm_alloc_percpu() failed\n");
@@ -2829,7 +2886,7 @@ static int dpaa_remove(struct platform_device *pdev)
 	return err;
 }
 
-static struct platform_device_id dpaa_devtype[] = {
+static const struct platform_device_id dpaa_devtype[] = {
 	{
 		.name = "dpaa-ethernet",
 		.driver_data = 0,
diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.h b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.h
index 9941a7866ebe..bd9422082f83 100644
--- a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.h
+++ b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.h
@@ -52,6 +52,7 @@
 enum dpaa_fq_type {
 	FQ_TYPE_RX_DEFAULT = 1, /* Rx Default FQs */
 	FQ_TYPE_RX_ERROR,	/* Rx Error FQs */
+	FQ_TYPE_RX_PCD,		/* Rx Parse Classify Distribute FQs */
 	FQ_TYPE_TX,		/* "Real" Tx FQs */
 	FQ_TYPE_TX_CONFIRM,	/* Tx default Conf FQ (actually an Rx FQ) */
 	FQ_TYPE_TX_CONF_MQ,	/* Tx conf FQs (one for each Tx FQ) */
@@ -158,6 +159,7 @@ struct dpaa_priv {
 	struct list_head dpaa_fq_list;
 
 	u8 num_tc;
+	bool keygen_in_use;
 	u32 msg_enable;	/* net_device message level */
 
 	struct {
diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_eth_sysfs.c b/drivers/net/ethernet/freescale/dpaa/dpaa_eth_sysfs.c
index ec75d1c6fa89..0d9b185e317f 100644
--- a/drivers/net/ethernet/freescale/dpaa/dpaa_eth_sysfs.c
+++ b/drivers/net/ethernet/freescale/dpaa/dpaa_eth_sysfs.c
@@ -71,6 +71,9 @@ static ssize_t dpaa_eth_show_fqids(struct device *dev,
 		case FQ_TYPE_RX_ERROR:
 			str = "Rx error";
 			break;
+		case FQ_TYPE_RX_PCD:
+			str = "Rx PCD";
+			break;
 		case FQ_TYPE_TX_CONFIRM:
 			str = "Tx default confirmation";
 			break;
diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c b/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c
index aad825088357..faea674094b9 100644
--- a/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c
+++ b/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c
@@ -399,6 +399,122 @@ static void dpaa_get_strings(struct net_device *net_dev, u32 stringset,
 	memcpy(strings, dpaa_stats_global, size);
 }
 
+static int dpaa_get_hash_opts(struct net_device *dev,
+			      struct ethtool_rxnfc *cmd)
+{
+	struct dpaa_priv *priv = netdev_priv(dev);
+
+	cmd->data = 0;
+
+	switch (cmd->flow_type) {
+	case TCP_V4_FLOW:
+	case TCP_V6_FLOW:
+	case UDP_V4_FLOW:
+	case UDP_V6_FLOW:
+		if (priv->keygen_in_use)
+			cmd->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3;
+		/* Fall through */
+	case IPV4_FLOW:
+	case IPV6_FLOW:
+	case SCTP_V4_FLOW:
+	case SCTP_V6_FLOW:
+	case AH_ESP_V4_FLOW:
+	case AH_ESP_V6_FLOW:
+	case AH_V4_FLOW:
+	case AH_V6_FLOW:
+	case ESP_V4_FLOW:
+	case ESP_V6_FLOW:
+		if (priv->keygen_in_use)
+			cmd->data |= RXH_IP_SRC | RXH_IP_DST;
+		break;
+	default:
+		cmd->data = 0;
+		break;
+	}
+
+	return 0;
+}
+
+static int dpaa_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd,
+			  u32 *unused)
+{
+	int ret = -EOPNOTSUPP;
+
+	switch (cmd->cmd) {
+	case ETHTOOL_GRXFH:
+		ret = dpaa_get_hash_opts(dev, cmd);
+		break;
+	default:
+		break;
+	}
+
+	return ret;
+}
+
+static void dpaa_set_hash(struct net_device *net_dev, bool enable)
+{
+	struct mac_device *mac_dev;
+	struct fman_port *rxport;
+	struct dpaa_priv *priv;
+
+	priv = netdev_priv(net_dev);
+	mac_dev = priv->mac_dev;
+	rxport = mac_dev->port[0];
+
+	fman_port_use_kg_hash(rxport, enable);
+	priv->keygen_in_use = enable;
+}
+
+static int dpaa_set_hash_opts(struct net_device *dev,
+			      struct ethtool_rxnfc *nfc)
+{
+	int ret = -EINVAL;
+
+	/* we support hashing on IPv4/v6 src/dest IP and L4 src/dest port */
+	if (nfc->data &
+	    ~(RXH_IP_SRC | RXH_IP_DST | RXH_L4_B_0_1 | RXH_L4_B_2_3))
+		return -EINVAL;
+
+	switch (nfc->flow_type) {
+	case TCP_V4_FLOW:
+	case TCP_V6_FLOW:
+	case UDP_V4_FLOW:
+	case UDP_V6_FLOW:
+	case IPV4_FLOW:
+	case IPV6_FLOW:
+	case SCTP_V4_FLOW:
+	case SCTP_V6_FLOW:
+	case AH_ESP_V4_FLOW:
+	case AH_ESP_V6_FLOW:
+	case AH_V4_FLOW:
+	case AH_V6_FLOW:
+	case ESP_V4_FLOW:
+	case ESP_V6_FLOW:
+		dpaa_set_hash(dev, !!nfc->data);
+		ret = 0;
+		break;
+	default:
+		break;
+	}
+
+	return ret;
+}
+
+static int dpaa_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd)
+{
+	int ret = -EOPNOTSUPP;
+
+	switch (cmd->cmd) {
+	case ETHTOOL_SRXFH:
+		ret = dpaa_set_hash_opts(dev, cmd);
+		break;
+	default:
+		break;
+	}
+
+	return ret;
+}
+
 const struct ethtool_ops dpaa_ethtool_ops = {
 	.get_drvinfo = dpaa_get_drvinfo,
 	.get_msglevel = dpaa_get_msglevel,
@@ -412,4 +528,6 @@ const struct ethtool_ops dpaa_ethtool_ops = {
 	.get_strings = dpaa_get_strings,
 	.get_link_ksettings = dpaa_get_link_ksettings,
 	.set_link_ksettings = dpaa_set_link_ksettings,
+	.get_rxnfc = dpaa_get_rxnfc,
+	.set_rxnfc = dpaa_set_rxnfc,
 };
diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c
index a6e323f15637..56f56d6ada9c 100644
--- a/drivers/net/ethernet/freescale/fec_main.c
+++ b/drivers/net/ethernet/freescale/fec_main.c
@@ -173,10 +173,12 @@ MODULE_PARM_DESC(macaddr, "FEC Ethernet MAC address");
 #endif /* CONFIG_M5272 */
 
 /* The FEC stores dest/src/type/vlan, data, and checksum for receive packets.
+ *
+ * 2048 byte skbufs are allocated. However, alignment requirements
+ * varies between FEC variants. Worst case is 64, so round down by 64.
  */
-#define PKT_MAXBUF_SIZE		1522
+#define PKT_MAXBUF_SIZE		(round_down(2048 - 64, 64))
 #define PKT_MINBUF_SIZE		64
-#define PKT_MAXBLR_SIZE		1536
 
 /* FEC receive acceleration */
 #define FEC_RACC_IPDIS		(1 << 1)
@@ -224,7 +226,6 @@ MODULE_PARM_DESC(macaddr, "FEC Ethernet MAC address");
 
 #define COPYBREAK_DEFAULT	256
 
-#define TSO_HEADER_SIZE		128
 /* Max number of allowed TCP segments for software TSO */
 #define FEC_MAX_TSO_SEGS	100
 #define FEC_MAX_SKB_DESCS	(FEC_MAX_TSO_SEGS * 2 + MAX_SKB_FRAGS)
@@ -851,7 +852,7 @@ static void fec_enet_enable_ring(struct net_device *ndev)
 	for (i = 0; i < fep->num_rx_queues; i++) {
 		rxq = fep->rx_queue[i];
 		writel(rxq->bd.dma, fep->hwp + FEC_R_DES_START(i));
-		writel(PKT_MAXBLR_SIZE, fep->hwp + FEC_R_BUFF_SIZE(i));
+		writel(PKT_MAXBUF_SIZE, fep->hwp + FEC_R_BUFF_SIZE(i));
 
 		/* enable DMA1/2 */
 		if (i)
@@ -1904,8 +1905,10 @@ static int fec_enet_mii_probe(struct net_device *ndev)
 		phy_dev = of_phy_connect(ndev, fep->phy_node,
 					 &fec_enet_adjust_link, 0,
 					 fep->phy_interface);
-		if (!phy_dev)
+		if (!phy_dev) {
+			netdev_err(ndev, "Unable to connect to phy\n");
 			return -ENODEV;
+		}
 	} else {
 		/* check for attached phy */
 		for (phy_id = 0; (phy_id < PHY_MAX_ADDR); phy_id++) {
diff --git a/drivers/net/ethernet/freescale/fec_mpc52xx.c b/drivers/net/ethernet/freescale/fec_mpc52xx.c
index aa8cf5d2a53c..6d7269d87a85 100644
--- a/drivers/net/ethernet/freescale/fec_mpc52xx.c
+++ b/drivers/net/ethernet/freescale/fec_mpc52xx.c
@@ -960,8 +960,8 @@ static int mpc52xx_fec_probe(struct platform_device *op)
 
 	/* We're done ! */
 	platform_set_drvdata(op, ndev);
-	netdev_info(ndev, "%s MAC %pM\n",
-		    op->dev.of_node->full_name, ndev->dev_addr);
+	netdev_info(ndev, "%pOF MAC %pM\n",
+		    op->dev.of_node, ndev->dev_addr);
 
 	return 0;
 
diff --git a/drivers/net/ethernet/freescale/fman/Makefile b/drivers/net/ethernet/freescale/fman/Makefile
index 60491779e49f..2c38119b172c 100644
--- a/drivers/net/ethernet/freescale/fman/Makefile
+++ b/drivers/net/ethernet/freescale/fman/Makefile
@@ -4,6 +4,6 @@ obj-$(CONFIG_FSL_FMAN) += fsl_fman.o
 obj-$(CONFIG_FSL_FMAN) += fsl_fman_port.o
 obj-$(CONFIG_FSL_FMAN) += fsl_mac.o
 
-fsl_fman-objs	:= fman_muram.o fman.o fman_sp.o
+fsl_fman-objs	:= fman_muram.o fman.o fman_sp.o fman_keygen.o
 fsl_fman_port-objs := fman_port.o
 fsl_mac-objs:= mac.o fman_dtsec.o fman_memac.o fman_tgec.o
diff --git a/drivers/net/ethernet/freescale/fman/fman.c b/drivers/net/ethernet/freescale/fman/fman.c
index 4aefe2438969..9530405030a7 100644
--- a/drivers/net/ethernet/freescale/fman/fman.c
+++ b/drivers/net/ethernet/freescale/fman/fman.c
@@ -32,9 +32,6 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
-#include "fman.h"
-#include "fman_muram.h"
-
 #include <linux/fsl/guts.h>
 #include <linux/slab.h>
 #include <linux/delay.h>
@@ -46,6 +43,10 @@
 #include <linux/interrupt.h>
 #include <linux/libfdt_env.h>
 
+#include "fman.h"
+#include "fman_muram.h"
+#include "fman_keygen.h"
+
 /* General defines */
 #define FMAN_LIODN_TBL			64	/* size of LIODN table */
 #define MAX_NUM_OF_MACS			10
@@ -56,6 +57,7 @@
 /* Modules registers offsets */
 #define BMI_OFFSET		0x00080000
 #define QMI_OFFSET		0x00080400
+#define KG_OFFSET		0x000C1000
 #define DMA_OFFSET		0x000C2000
 #define FPM_OFFSET		0x000C3000
 #define IMEM_OFFSET		0x000C4000
@@ -564,80 +566,6 @@ struct fman_cfg {
 	u32 qmi_def_tnums_thresh;
 };
 
-/* Structure that holds information received from device tree */
-struct fman_dts_params {
-	void __iomem *base_addr;		/* FMan virtual address */
-	struct resource *res;			/* FMan memory resource */
-	u8 id;					/* FMan ID */
-
-	int err_irq;				/* FMan Error IRQ */
-
-	u16 clk_freq;				/* FMan clock freq (In Mhz) */
-
-	u32 qman_channel_base;			/* QMan channels base */
-	u32 num_of_qman_channels;		/* Number of QMan channels */
-
-	struct resource muram_res;		/* MURAM resource */
-};
-
-/** fman_exceptions_cb
- * fman		- Pointer to FMan
- * exception	- The exception.
- *
- * Exceptions user callback routine, will be called upon an exception
- * passing the exception identification.
- *
- * Return: irq status
- */
-typedef irqreturn_t (fman_exceptions_cb)(struct fman *fman,
-					 enum fman_exceptions exception);
-
-/** fman_bus_error_cb
- * fman		- Pointer to FMan
- * port_id	- Port id
- * addr		- Address that caused the error
- * tnum		- Owner of error
- * liodn	- Logical IO device number
- *
- * Bus error user callback routine, will be called upon bus error,
- * passing parameters describing the errors and the owner.
- *
- * Return: IRQ status
- */
-typedef irqreturn_t (fman_bus_error_cb)(struct fman *fman, u8 port_id,
-					u64 addr, u8 tnum, u16 liodn);
-
-struct fman {
-	struct device *dev;
-	void __iomem *base_addr;
-	struct fman_intr_src intr_mng[FMAN_EV_CNT];
-
-	struct fman_fpm_regs __iomem *fpm_regs;
-	struct fman_bmi_regs __iomem *bmi_regs;
-	struct fman_qmi_regs __iomem *qmi_regs;
-	struct fman_dma_regs __iomem *dma_regs;
-	struct fman_hwp_regs __iomem *hwp_regs;
-	fman_exceptions_cb *exception_cb;
-	fman_bus_error_cb *bus_error_cb;
-	/* Spinlock for FMan use */
-	spinlock_t spinlock;
-	struct fman_state_struct *state;
-
-	struct fman_cfg *cfg;
-	struct muram_info *muram;
-	/* cam section in muram */
-	unsigned long cam_offset;
-	size_t cam_size;
-	/* Fifo in MURAM */
-	unsigned long fifo_offset;
-	size_t fifo_size;
-
-	u32 liodn_base[64];
-	u32 liodn_offset[64];
-
-	struct fman_dts_params dts_params;
-};
-
 static irqreturn_t fman_exceptions(struct fman *fman,
 				   enum fman_exceptions exception)
 {
@@ -1811,6 +1739,7 @@ static int fman_config(struct fman *fman)
 	fman->qmi_regs = base_addr + QMI_OFFSET;
 	fman->dma_regs = base_addr + DMA_OFFSET;
 	fman->hwp_regs = base_addr + HWP_OFFSET;
+	fman->kg_regs = base_addr + KG_OFFSET;
 	fman->base_addr = base_addr;
 
 	spin_lock_init(&fman->spinlock);
@@ -1925,8 +1854,8 @@ static int fman_reset(struct fman *fman)
 
 		guts_regs = of_iomap(guts_node, 0);
 		if (!guts_regs) {
-			dev_err(fman->dev, "%s: Couldn't map %s regs\n",
-				__func__, guts_node->full_name);
+			dev_err(fman->dev, "%s: Couldn't map %pOF regs\n",
+				__func__, guts_node);
 			goto guts_regs;
 		}
 #define FMAN1_ALL_MACS_MASK	0xFCC00000
@@ -2083,6 +2012,11 @@ static int fman_init(struct fman *fman)
 	/* Init HW Parser */
 	hwp_init(fman->hwp_regs);
 
+	/* Init KeyGen */
+	fman->keygen = keygen_init(fman->kg_regs);
+	if (!fman->keygen)
+		return -EINVAL;
+
 	err = enable(fman, cfg);
 	if (err != 0)
 		return err;
@@ -2434,15 +2368,21 @@ u32 fman_get_qman_channel_id(struct fman *fman, u32 port_id)
 	int i;
 
 	if (fman->state->rev_info.major >= 6) {
-		u32 port_ids[] = {0x30, 0x31, 0x28, 0x29, 0x2a, 0x2b,
-				  0x2c, 0x2d, 0x2, 0x3, 0x4, 0x5, 0x7, 0x7};
+		static const u32 port_ids[] = {
+			0x30, 0x31, 0x28, 0x29, 0x2a, 0x2b,
+			0x2c, 0x2d, 0x2, 0x3, 0x4, 0x5, 0x7, 0x7
+		};
+
 		for (i = 0; i < fman->state->num_of_qman_channels; i++) {
 			if (port_ids[i] == port_id)
 				break;
 		}
 	} else {
-		u32 port_ids[] = {0x30, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x1,
-				  0x2, 0x3, 0x4, 0x5, 0x7, 0x7};
+		static const u32 port_ids[] = {
+			0x30, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x1,
+			0x2, 0x3, 0x4, 0x5, 0x7, 0x7
+		};
+
 		for (i = 0; i < fman->state->num_of_qman_channels; i++) {
 			if (port_ids[i] == port_id)
 				break;
@@ -2780,8 +2720,8 @@ static struct fman *read_dts_node(struct platform_device *of_dev)
 
 	err = of_property_read_u32(fm_node, "cell-index", &val);
 	if (err) {
-		dev_err(&of_dev->dev, "%s: failed to read cell-index for %s\n",
-			__func__, fm_node->full_name);
+		dev_err(&of_dev->dev, "%s: failed to read cell-index for %pOF\n",
+			__func__, fm_node);
 		goto fman_node_put;
 	}
 	fman->dts_params.id = (u8)val;
@@ -2834,8 +2774,8 @@ static struct fman *read_dts_node(struct platform_device *of_dev)
 	err = of_property_read_u32_array(fm_node, "fsl,qman-channel-range",
 					 &range[0], 2);
 	if (err) {
-		dev_err(&of_dev->dev, "%s: failed to read fsl,qman-channel-range for %s\n",
-			__func__, fm_node->full_name);
+		dev_err(&of_dev->dev, "%s: failed to read fsl,qman-channel-range for %pOF\n",
+			__func__, fm_node);
 		goto fman_node_put;
 	}
 	fman->dts_params.qman_channel_base = range[0];
diff --git a/drivers/net/ethernet/freescale/fman/fman.h b/drivers/net/ethernet/freescale/fman/fman.h
index f53e1473dbcc..bfa02e0014ae 100644
--- a/drivers/net/ethernet/freescale/fman/fman.h
+++ b/drivers/net/ethernet/freescale/fman/fman.h
@@ -34,6 +34,8 @@
 #define __FM_H
 
 #include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/of_irq.h>
 
 /* FM Frame descriptor macros  */
 /* Frame queue Context Override */
@@ -274,6 +276,81 @@ struct fman_intr_src {
 	void *src_handle;
 };
 
+/** fman_exceptions_cb
+ * fman         - Pointer to FMan
+ * exception    - The exception.
+ *
+ * Exceptions user callback routine, will be called upon an exception
+ * passing the exception identification.
+ *
+ * Return: irq status
+ */
+typedef irqreturn_t (fman_exceptions_cb)(struct fman *fman,
+					 enum fman_exceptions exception);
+/** fman_bus_error_cb
+ * fman         - Pointer to FMan
+ * port_id      - Port id
+ * addr         - Address that caused the error
+ * tnum         - Owner of error
+ * liodn        - Logical IO device number
+ *
+ * Bus error user callback routine, will be called upon bus error,
+ * passing parameters describing the errors and the owner.
+ *
+ * Return: IRQ status
+ */
+typedef irqreturn_t (fman_bus_error_cb)(struct fman *fman, u8 port_id,
+					u64 addr, u8 tnum, u16 liodn);
+
+/* Structure that holds information received from device tree */
+struct fman_dts_params {
+	void __iomem *base_addr;                /* FMan virtual address */
+	struct resource *res;                   /* FMan memory resource */
+	u8 id;                                  /* FMan ID */
+
+	int err_irq;                            /* FMan Error IRQ */
+
+	u16 clk_freq;                           /* FMan clock freq (In Mhz) */
+
+	u32 qman_channel_base;                  /* QMan channels base */
+	u32 num_of_qman_channels;               /* Number of QMan channels */
+
+	struct resource muram_res;              /* MURAM resource */
+};
+
+struct fman {
+	struct device *dev;
+	void __iomem *base_addr;
+	struct fman_intr_src intr_mng[FMAN_EV_CNT];
+
+	struct fman_fpm_regs __iomem *fpm_regs;
+	struct fman_bmi_regs __iomem *bmi_regs;
+	struct fman_qmi_regs __iomem *qmi_regs;
+	struct fman_dma_regs __iomem *dma_regs;
+	struct fman_hwp_regs __iomem *hwp_regs;
+	struct fman_kg_regs __iomem *kg_regs;
+	fman_exceptions_cb *exception_cb;
+	fman_bus_error_cb *bus_error_cb;
+	/* Spinlock for FMan use */
+	spinlock_t spinlock;
+	struct fman_state_struct *state;
+
+	struct fman_cfg *cfg;
+	struct muram_info *muram;
+	struct fman_keygen *keygen;
+	/* cam section in muram */
+	unsigned long cam_offset;
+	size_t cam_size;
+	/* Fifo in MURAM */
+	unsigned long fifo_offset;
+	size_t fifo_size;
+
+	u32 liodn_base[64];
+	u32 liodn_offset[64];
+
+	struct fman_dts_params dts_params;
+};
+
 /* Structure for port-FM communication during fman_port_init. */
 struct fman_port_init_params {
 	u8 port_id;			/* port Id */
diff --git a/drivers/net/ethernet/freescale/fman/fman_dtsec.c b/drivers/net/ethernet/freescale/fman/fman_dtsec.c
index 98bba10fc38c..ea43b4974149 100644
--- a/drivers/net/ethernet/freescale/fman/fman_dtsec.c
+++ b/drivers/net/ethernet/freescale/fman/fman_dtsec.c
@@ -123,7 +123,7 @@
 #define DTSEC_ECNTRL_R100M		0x00000008
 #define DTSEC_ECNTRL_QSGMIIM		0x00000001
 
-#define DTSEC_TCTRL_GTS			0x00000020
+#define TCTRL_GTS			0x00000020
 
 #define RCTRL_PAL_MASK			0x001f0000
 #define RCTRL_PAL_SHIFT			16
@@ -863,6 +863,52 @@ int dtsec_cfg_pad_and_crc(struct fman_mac *dtsec, bool new_val)
 	return 0;
 }
 
+static void graceful_start(struct fman_mac *dtsec, enum comm_mode mode)
+{
+	struct dtsec_regs __iomem *regs = dtsec->regs;
+
+	if (mode & COMM_MODE_TX)
+		iowrite32be(ioread32be(&regs->tctrl) &
+				~TCTRL_GTS, &regs->tctrl);
+	if (mode & COMM_MODE_RX)
+		iowrite32be(ioread32be(&regs->rctrl) &
+				~RCTRL_GRS, &regs->rctrl);
+}
+
+static void graceful_stop(struct fman_mac *dtsec, enum comm_mode mode)
+{
+	struct dtsec_regs __iomem *regs = dtsec->regs;
+	u32 tmp;
+
+	/* Graceful stop - Assert the graceful Rx stop bit */
+	if (mode & COMM_MODE_RX) {
+		tmp = ioread32be(&regs->rctrl) | RCTRL_GRS;
+		iowrite32be(tmp, &regs->rctrl);
+
+		if (dtsec->fm_rev_info.major == 2) {
+			/* Workaround for dTSEC Errata A002 */
+			usleep_range(100, 200);
+		} else {
+			/* Workaround for dTSEC Errata A004839 */
+			usleep_range(10, 50);
+		}
+	}
+
+	/* Graceful stop - Assert the graceful Tx stop bit */
+	if (mode & COMM_MODE_TX) {
+		if (dtsec->fm_rev_info.major == 2) {
+			/* dTSEC Errata A004: Do not use TCTRL[GTS]=1 */
+			pr_debug("GTS not supported due to DTSEC_A004 Errata.\n");
+		} else {
+			tmp = ioread32be(&regs->tctrl) | TCTRL_GTS;
+			iowrite32be(tmp, &regs->tctrl);
+
+			/* Workaround for dTSEC Errata A0012, A0014 */
+			usleep_range(10, 50);
+		}
+	}
+}
+
 int dtsec_enable(struct fman_mac *dtsec, enum comm_mode mode)
 {
 	struct dtsec_regs __iomem *regs = dtsec->regs;
@@ -880,13 +926,8 @@ int dtsec_enable(struct fman_mac *dtsec, enum comm_mode mode)
 
 	iowrite32be(tmp, &regs->maccfg1);
 
-	/* Graceful start - clear the graceful receive stop bit */
-	if (mode & COMM_MODE_TX)
-		iowrite32be(ioread32be(&regs->tctrl) & ~DTSEC_TCTRL_GTS,
-			    &regs->tctrl);
-	if (mode & COMM_MODE_RX)
-		iowrite32be(ioread32be(&regs->rctrl) & ~RCTRL_GRS,
-			    &regs->rctrl);
+	/* Graceful start - clear the graceful Rx/Tx stop bit */
+	graceful_start(dtsec, mode);
 
 	return 0;
 }
@@ -899,23 +940,8 @@ int dtsec_disable(struct fman_mac *dtsec, enum comm_mode mode)
 	if (!is_init_done(dtsec->dtsec_drv_param))
 		return -EINVAL;
 
-	/* Gracefull stop - Assert the graceful transmit stop bit */
-	if (mode & COMM_MODE_RX) {
-		tmp = ioread32be(&regs->rctrl) | RCTRL_GRS;
-		iowrite32be(tmp, &regs->rctrl);
-
-		if (dtsec->fm_rev_info.major == 2)
-			usleep_range(100, 200);
-		else
-			udelay(10);
-	}
-
-	if (mode & COMM_MODE_TX) {
-		if (dtsec->fm_rev_info.major == 2)
-			pr_debug("GTS not supported due to DTSEC_A004 errata.\n");
-		else
-			pr_debug("GTS not supported due to DTSEC_A0014 errata.\n");
-	}
+	/* Graceful stop - Assert the graceful Rx/Tx stop bit */
+	graceful_stop(dtsec, mode);
 
 	tmp = ioread32be(&regs->maccfg1);
 	if (mode & COMM_MODE_RX)
@@ -933,11 +959,19 @@ int dtsec_set_tx_pause_frames(struct fman_mac *dtsec,
 			      u16 pause_time, u16 __maybe_unused thresh_time)
 {
 	struct dtsec_regs __iomem *regs = dtsec->regs;
+	enum comm_mode mode = COMM_MODE_NONE;
 	u32 ptv = 0;
 
 	if (!is_init_done(dtsec->dtsec_drv_param))
 		return -EINVAL;
 
+	if ((ioread32be(&regs->rctrl) & RCTRL_GRS) == 0)
+		mode |= COMM_MODE_RX;
+	if ((ioread32be(&regs->tctrl) & TCTRL_GTS) == 0)
+		mode |= COMM_MODE_TX;
+
+	graceful_stop(dtsec, mode);
+
 	if (pause_time) {
 		/* FM_BAD_TX_TS_IN_B_2_B_ERRATA_DTSEC_A003 Errata workaround */
 		if (dtsec->fm_rev_info.major == 2 && pause_time <= 320) {
@@ -958,17 +992,27 @@ int dtsec_set_tx_pause_frames(struct fman_mac *dtsec,
 		iowrite32be(ioread32be(&regs->maccfg1) & ~MACCFG1_TX_FLOW,
 			    &regs->maccfg1);
 
+	graceful_start(dtsec, mode);
+
 	return 0;
 }
 
 int dtsec_accept_rx_pause_frames(struct fman_mac *dtsec, bool en)
 {
 	struct dtsec_regs __iomem *regs = dtsec->regs;
+	enum comm_mode mode = COMM_MODE_NONE;
 	u32 tmp;
 
 	if (!is_init_done(dtsec->dtsec_drv_param))
 		return -EINVAL;
 
+	if ((ioread32be(&regs->rctrl) & RCTRL_GRS) == 0)
+		mode |= COMM_MODE_RX;
+	if ((ioread32be(&regs->tctrl) & TCTRL_GTS) == 0)
+		mode |= COMM_MODE_TX;
+
+	graceful_stop(dtsec, mode);
+
 	tmp = ioread32be(&regs->maccfg1);
 	if (en)
 		tmp |= MACCFG1_RX_FLOW;
@@ -976,20 +1020,34 @@ int dtsec_accept_rx_pause_frames(struct fman_mac *dtsec, bool en)
 		tmp &= ~MACCFG1_RX_FLOW;
 	iowrite32be(tmp, &regs->maccfg1);
 
+	graceful_start(dtsec, mode);
+
 	return 0;
 }
 
 int dtsec_modify_mac_address(struct fman_mac *dtsec, enet_addr_t *enet_addr)
 {
+	struct dtsec_regs __iomem *regs = dtsec->regs;
+	enum comm_mode mode = COMM_MODE_NONE;
+
 	if (!is_init_done(dtsec->dtsec_drv_param))
 		return -EINVAL;
 
+	if ((ioread32be(&regs->rctrl) & RCTRL_GRS) == 0)
+		mode |= COMM_MODE_RX;
+	if ((ioread32be(&regs->tctrl) & TCTRL_GTS) == 0)
+		mode |= COMM_MODE_TX;
+
+	graceful_stop(dtsec, mode);
+
 	/* Initialize MAC Station Address registers (1 & 2)
 	 * Station address have to be swapped (big endian to little endian
 	 */
 	dtsec->addr = ENET_ADDR_TO_UINT64(*enet_addr);
 	set_mac_address(dtsec->regs, (u8 *)(*enet_addr));
 
+	graceful_start(dtsec, mode);
+
 	return 0;
 }
 
@@ -1162,11 +1220,19 @@ int dtsec_set_promiscuous(struct fman_mac *dtsec, bool new_val)
 int dtsec_adjust_link(struct fman_mac *dtsec, u16 speed)
 {
 	struct dtsec_regs __iomem *regs = dtsec->regs;
+	enum comm_mode mode = COMM_MODE_NONE;
 	u32 tmp;
 
 	if (!is_init_done(dtsec->dtsec_drv_param))
 		return -EINVAL;
 
+	if ((ioread32be(&regs->rctrl) & RCTRL_GRS) == 0)
+		mode |= COMM_MODE_RX;
+	if ((ioread32be(&regs->tctrl) & TCTRL_GTS) == 0)
+		mode |= COMM_MODE_TX;
+
+	graceful_stop(dtsec, mode);
+
 	tmp = ioread32be(&regs->maccfg2);
 
 	/* Full Duplex */
@@ -1186,6 +1252,8 @@ int dtsec_adjust_link(struct fman_mac *dtsec, u16 speed)
 		tmp &= ~DTSEC_ECNTRL_R100M;
 	iowrite32be(tmp, &regs->ecntrl);
 
+	graceful_start(dtsec, mode);
+
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/freescale/fman/fman_keygen.c b/drivers/net/ethernet/freescale/fman/fman_keygen.c
new file mode 100644
index 000000000000..f54da3c684d0
--- /dev/null
+++ b/drivers/net/ethernet/freescale/fman/fman_keygen.c
@@ -0,0 +1,783 @@
+/*
+ * Copyright 2017 NXP
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of NXP nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NXP ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NXP BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/slab.h>
+
+#include "fman_keygen.h"
+
+/* Maximum number of HW Ports */
+#define FMAN_MAX_NUM_OF_HW_PORTS		64
+
+/* Maximum number of KeyGen Schemes */
+#define FM_KG_MAX_NUM_OF_SCHEMES		32
+
+/* Number of generic KeyGen Generic Extract Command Registers */
+#define FM_KG_NUM_OF_GENERIC_REGS		8
+
+/* Dummy port ID */
+#define DUMMY_PORT_ID				0
+
+/* Select Scheme Value Register */
+#define KG_SCH_DEF_USE_KGSE_DV_0		2
+#define KG_SCH_DEF_USE_KGSE_DV_1		3
+
+/* Registers Shifting values */
+#define FM_KG_KGAR_NUM_SHIFT			16
+#define KG_SCH_DEF_L4_PORT_SHIFT		8
+#define KG_SCH_DEF_IP_ADDR_SHIFT		18
+#define KG_SCH_HASH_CONFIG_SHIFT_SHIFT		24
+
+/* KeyGen Registers bit field masks: */
+
+/* Enable bit field mask for KeyGen General Configuration Register */
+#define FM_KG_KGGCR_EN				0x80000000
+
+/* KeyGen Global Registers bit field masks */
+#define FM_KG_KGAR_GO				0x80000000
+#define FM_KG_KGAR_READ				0x40000000
+#define FM_KG_KGAR_WRITE			0x00000000
+#define FM_KG_KGAR_SEL_SCHEME_ENTRY		0x00000000
+#define FM_KG_KGAR_SCM_WSEL_UPDATE_CNT		0x00008000
+
+#define FM_KG_KGAR_ERR				0x20000000
+#define FM_KG_KGAR_SEL_CLS_PLAN_ENTRY		0x01000000
+#define FM_KG_KGAR_SEL_PORT_ENTRY		0x02000000
+#define FM_KG_KGAR_SEL_PORT_WSEL_SP		0x00008000
+#define FM_KG_KGAR_SEL_PORT_WSEL_CPP		0x00004000
+
+/* Error events exceptions */
+#define FM_EX_KG_DOUBLE_ECC			0x80000000
+#define FM_EX_KG_KEYSIZE_OVERFLOW		0x40000000
+
+/* Scheme Registers bit field masks */
+#define KG_SCH_MODE_EN				0x80000000
+#define KG_SCH_VSP_NO_KSP_EN			0x80000000
+#define KG_SCH_HASH_CONFIG_SYM			0x40000000
+
+/* Known Protocol field codes */
+#define KG_SCH_KN_PORT_ID		0x80000000
+#define KG_SCH_KN_MACDST		0x40000000
+#define KG_SCH_KN_MACSRC		0x20000000
+#define KG_SCH_KN_TCI1			0x10000000
+#define KG_SCH_KN_TCI2			0x08000000
+#define KG_SCH_KN_ETYPE			0x04000000
+#define KG_SCH_KN_PPPSID		0x02000000
+#define KG_SCH_KN_PPPID			0x01000000
+#define KG_SCH_KN_MPLS1			0x00800000
+#define KG_SCH_KN_MPLS2			0x00400000
+#define KG_SCH_KN_MPLS_LAST		0x00200000
+#define KG_SCH_KN_IPSRC1		0x00100000
+#define KG_SCH_KN_IPDST1		0x00080000
+#define KG_SCH_KN_PTYPE1		0x00040000
+#define KG_SCH_KN_IPTOS_TC1		0x00020000
+#define KG_SCH_KN_IPV6FL1		0x00010000
+#define KG_SCH_KN_IPSRC2		0x00008000
+#define KG_SCH_KN_IPDST2		0x00004000
+#define KG_SCH_KN_PTYPE2		0x00002000
+#define KG_SCH_KN_IPTOS_TC2		0x00001000
+#define KG_SCH_KN_IPV6FL2		0x00000800
+#define KG_SCH_KN_GREPTYPE		0x00000400
+#define KG_SCH_KN_IPSEC_SPI		0x00000200
+#define KG_SCH_KN_IPSEC_NH		0x00000100
+#define KG_SCH_KN_IPPID			0x00000080
+#define KG_SCH_KN_L4PSRC		0x00000004
+#define KG_SCH_KN_L4PDST		0x00000002
+#define KG_SCH_KN_TFLG			0x00000001
+
+/* NIA values */
+#define NIA_ENG_BMI			0x00500000
+#define NIA_BMI_AC_ENQ_FRAME		0x00000002
+#define ENQUEUE_KG_DFLT_NIA		(NIA_ENG_BMI | NIA_BMI_AC_ENQ_FRAME)
+
+/* Hard-coded configuration:
+ * These values are used as hard-coded values for KeyGen configuration
+ * and they replace user selections for this hard-coded version
+ */
+
+/* Hash distribution shift */
+#define DEFAULT_HASH_DIST_FQID_SHIFT		0
+
+/* Hash shift */
+#define DEFAULT_HASH_SHIFT			0
+
+/* Symmetric hash usage:
+ * Warning:
+ * - the value for symmetric hash usage must be in accordance with hash
+ *	key defined below
+ * - according to tests performed, spreading is not working if symmetric
+ *	hash is set on true
+ * So ultimately symmetric hash functionality should be always disabled:
+ */
+#define DEFAULT_SYMMETRIC_HASH			false
+
+/* Hash Key extraction fields: */
+#define DEFAULT_HASH_KEY_EXTRACT_FIELDS		\
+	(KG_SCH_KN_IPSRC1 | KG_SCH_KN_IPDST1 | \
+	    KG_SCH_KN_L4PSRC | KG_SCH_KN_L4PDST)
+
+/* Default values to be used as hash key in case IPv4 or L4 (TCP, UDP)
+ * don't exist in the frame
+ */
+/* Default IPv4 address */
+#define DEFAULT_HASH_KEY_IPv4_ADDR		0x0A0A0A0A
+/* Default L4 port */
+#define DEFAULT_HASH_KEY_L4_PORT		0x0B0B0B0B
+
+/* KeyGen Memory Mapped Registers: */
+
+/* Scheme Configuration RAM Registers */
+struct fman_kg_scheme_regs {
+	u32 kgse_mode;		/* 0x100: MODE */
+	u32 kgse_ekfc;		/* 0x104: Extract Known Fields Command */
+	u32 kgse_ekdv;		/* 0x108: Extract Known Default Value */
+	u32 kgse_bmch;		/* 0x10C: Bit Mask Command High */
+	u32 kgse_bmcl;		/* 0x110: Bit Mask Command Low */
+	u32 kgse_fqb;		/* 0x114: Frame Queue Base */
+	u32 kgse_hc;		/* 0x118: Hash Command */
+	u32 kgse_ppc;		/* 0x11C: Policer Profile Command */
+	u32 kgse_gec[FM_KG_NUM_OF_GENERIC_REGS];
+			/* 0x120: Generic Extract Command */
+	u32 kgse_spc;
+		/* 0x140: KeyGen Scheme Entry Statistic Packet Counter */
+	u32 kgse_dv0;	/* 0x144: KeyGen Scheme Entry Default Value 0 */
+	u32 kgse_dv1;	/* 0x148: KeyGen Scheme Entry Default Value 1 */
+	u32 kgse_ccbs;
+		/* 0x14C: KeyGen Scheme Entry Coarse Classification Bit*/
+	u32 kgse_mv;	/* 0x150: KeyGen Scheme Entry Match vector */
+	u32 kgse_om;	/* 0x154: KeyGen Scheme Entry Operation Mode bits */
+	u32 kgse_vsp;
+		/* 0x158: KeyGen Scheme Entry Virtual Storage Profile */
+};
+
+/* Port Partition Configuration Registers */
+struct fman_kg_pe_regs {
+	u32 fmkg_pe_sp;		/* 0x100: KeyGen Port entry Scheme Partition */
+	u32 fmkg_pe_cpp;
+		/* 0x104: KeyGen Port Entry Classification Plan Partition */
+};
+
+/* General Configuration and Status Registers
+ * Global Statistic Counters
+ * KeyGen Global Registers
+ */
+struct fman_kg_regs {
+	u32 fmkg_gcr;	/* 0x000: KeyGen General Configuration Register */
+	u32 res004;	/* 0x004: Reserved */
+	u32 res008;	/* 0x008: Reserved */
+	u32 fmkg_eer;	/* 0x00C: KeyGen Error Event Register */
+	u32 fmkg_eeer;	/* 0x010: KeyGen Error Event Enable Register */
+	u32 res014;	/* 0x014: Reserved */
+	u32 res018;	/* 0x018: Reserved */
+	u32 fmkg_seer;	/* 0x01C: KeyGen Scheme Error Event Register */
+	u32 fmkg_seeer;	/* 0x020: KeyGen Scheme Error Event Enable Register */
+	u32 fmkg_gsr;	/* 0x024: KeyGen Global Status Register */
+	u32 fmkg_tpc;	/* 0x028: Total Packet Counter Register */
+	u32 fmkg_serc;	/* 0x02C: Soft Error Capture Register */
+	u32 res030[4];	/* 0x030: Reserved */
+	u32 fmkg_fdor;	/* 0x034: Frame Data Offset Register */
+	u32 fmkg_gdv0r;	/* 0x038: Global Default Value Register 0 */
+	u32 fmkg_gdv1r;	/* 0x03C: Global Default Value Register 1 */
+	u32 res04c[6];	/* 0x040: Reserved */
+	u32 fmkg_feer;	/* 0x044: Force Error Event Register */
+	u32 res068[38];	/* 0x048: Reserved */
+	union {
+		u32 fmkg_indirect[63];	/* 0x100: Indirect Access Registers */
+		struct fman_kg_scheme_regs fmkg_sch; /* Scheme Registers */
+		struct fman_kg_pe_regs fmkg_pe; /* Port Partition Registers */
+	};
+	u32 fmkg_ar;	/* 0x1FC: KeyGen Action Register */
+};
+
+/* KeyGen Scheme data */
+struct keygen_scheme {
+	bool used;	/* Specifies if this scheme is used */
+	u8 hw_port_id;
+		/* Hardware port ID
+		 * schemes sharing between multiple ports is not
+		 * currently supported
+		 * so we have only one port id bound to a scheme
+		 */
+	u32 base_fqid;
+		/* Base FQID:
+		 * Must be between 1 and 2^24-1
+		 * If hash is used and an even distribution is
+		 * expected according to hash_fqid_count,
+		 * base_fqid must be aligned to hash_fqid_count
+		 */
+	u32 hash_fqid_count;
+		/* FQ range for hash distribution:
+		 * Must be a power of 2
+		 * Represents the range of queues for spreading
+		 */
+	bool use_hashing;	/* Usage of Hashing and spreading over FQ */
+	bool symmetric_hash;	/* Symmetric Hash option usage */
+	u8 hashShift;
+		/* Hash result right shift.
+		 * Select the 24 bits out of the 64 hash result.
+		 * 0 means using the 24 LSB's, otherwise
+		 * use the 24 LSB's after shifting right
+		 */
+	u32 match_vector;	/* Match Vector */
+};
+
+/* KeyGen driver data */
+struct fman_keygen {
+	struct keygen_scheme schemes[FM_KG_MAX_NUM_OF_SCHEMES];
+				/* Array of schemes */
+	struct fman_kg_regs __iomem *keygen_regs;	/* KeyGen registers */
+};
+
+/* keygen_write_ar_wait
+ *
+ * Write Action Register with specified value, wait for GO bit field to be
+ * idle and then read the error
+ *
+ * regs: KeyGen registers
+ * fmkg_ar: Action Register value
+ *
+ * Return: Zero for success or error code in case of failure
+ */
+static int keygen_write_ar_wait(struct fman_kg_regs __iomem *regs, u32 fmkg_ar)
+{
+	iowrite32be(fmkg_ar, &regs->fmkg_ar);
+
+	/* Wait for GO bit field to be idle */
+	while (fmkg_ar & FM_KG_KGAR_GO)
+		fmkg_ar = ioread32be(&regs->fmkg_ar);
+
+	if (fmkg_ar & FM_KG_KGAR_ERR)
+		return -EINVAL;
+
+	return 0;
+}
+
+/* build_ar_scheme
+ *
+ * Build Action Register value for scheme settings
+ *
+ * scheme_id: Scheme ID
+ * update_counter: update scheme counter
+ * write: true for action to write the scheme or false for read action
+ *
+ * Return: AR value
+ */
+static u32 build_ar_scheme(u8 scheme_id, bool update_counter, bool write)
+{
+	u32 rw = (u32)(write ? FM_KG_KGAR_WRITE : FM_KG_KGAR_READ);
+
+	return (u32)(FM_KG_KGAR_GO |
+			rw |
+			FM_KG_KGAR_SEL_SCHEME_ENTRY |
+			DUMMY_PORT_ID |
+			((u32)scheme_id << FM_KG_KGAR_NUM_SHIFT) |
+			(update_counter ? FM_KG_KGAR_SCM_WSEL_UPDATE_CNT : 0));
+}
+
+/* build_ar_bind_scheme
+ *
+ * Build Action Register value for port binding to schemes
+ *
+ * hwport_id: HW Port ID
+ * write: true for action to write the bind or false for read action
+ *
+ * Return: AR value
+ */
+static u32 build_ar_bind_scheme(u8 hwport_id, bool write)
+{
+	u32 rw = write ? (u32)FM_KG_KGAR_WRITE : (u32)FM_KG_KGAR_READ;
+
+	return (u32)(FM_KG_KGAR_GO |
+			rw |
+			FM_KG_KGAR_SEL_PORT_ENTRY |
+			hwport_id |
+			FM_KG_KGAR_SEL_PORT_WSEL_SP);
+}
+
+/* keygen_write_sp
+ *
+ * Write Scheme Partition Register with specified value
+ *
+ * regs: KeyGen Registers
+ * sp: Scheme Partition register value
+ * add: true to add a scheme partition or false to clear
+ *
+ * Return: none
+ */
+static void keygen_write_sp(struct fman_kg_regs __iomem *regs, u32 sp, bool add)
+{
+	u32 tmp;
+
+	tmp = ioread32be(&regs->fmkg_pe.fmkg_pe_sp);
+
+	if (add)
+		tmp |= sp;
+	else
+		tmp &= ~sp;
+
+	iowrite32be(tmp, &regs->fmkg_pe.fmkg_pe_sp);
+}
+
+/* build_ar_bind_cls_plan
+ *
+ * Build Action Register value for Classification Plan
+ *
+ * hwport_id: HW Port ID
+ * write: true for action to write the CP or false for read action
+ *
+ * Return: AR value
+ */
+static u32 build_ar_bind_cls_plan(u8 hwport_id, bool write)
+{
+	u32 rw = write ? (u32)FM_KG_KGAR_WRITE : (u32)FM_KG_KGAR_READ;
+
+	return (u32)(FM_KG_KGAR_GO |
+			rw |
+			FM_KG_KGAR_SEL_PORT_ENTRY |
+			hwport_id |
+			FM_KG_KGAR_SEL_PORT_WSEL_CPP);
+}
+
+/* keygen_write_cpp
+ *
+ * Write Classification Plan Partition Register with specified value
+ *
+ * regs: KeyGen Registers
+ * cpp: CPP register value
+ *
+ * Return: none
+ */
+static void keygen_write_cpp(struct fman_kg_regs __iomem *regs, u32 cpp)
+{
+	iowrite32be(cpp, &regs->fmkg_pe.fmkg_pe_cpp);
+}
+
+/* keygen_write_scheme
+ *
+ * Write all Schemes Registers with specified values
+ *
+ * regs: KeyGen Registers
+ * scheme_id: Scheme ID
+ * scheme_regs: Scheme registers values desired to be written
+ * update_counter: update scheme counter
+ *
+ * Return: Zero for success or error code in case of failure
+ */
+static int keygen_write_scheme(struct fman_kg_regs __iomem *regs, u8 scheme_id,
+			       struct fman_kg_scheme_regs *scheme_regs,
+				bool update_counter)
+{
+	u32 ar_reg;
+	int err, i;
+
+	/* Write indirect scheme registers */
+	iowrite32be(scheme_regs->kgse_mode, &regs->fmkg_sch.kgse_mode);
+	iowrite32be(scheme_regs->kgse_ekfc, &regs->fmkg_sch.kgse_ekfc);
+	iowrite32be(scheme_regs->kgse_ekdv, &regs->fmkg_sch.kgse_ekdv);
+	iowrite32be(scheme_regs->kgse_bmch, &regs->fmkg_sch.kgse_bmch);
+	iowrite32be(scheme_regs->kgse_bmcl, &regs->fmkg_sch.kgse_bmcl);
+	iowrite32be(scheme_regs->kgse_fqb, &regs->fmkg_sch.kgse_fqb);
+	iowrite32be(scheme_regs->kgse_hc, &regs->fmkg_sch.kgse_hc);
+	iowrite32be(scheme_regs->kgse_ppc, &regs->fmkg_sch.kgse_ppc);
+	iowrite32be(scheme_regs->kgse_spc, &regs->fmkg_sch.kgse_spc);
+	iowrite32be(scheme_regs->kgse_dv0, &regs->fmkg_sch.kgse_dv0);
+	iowrite32be(scheme_regs->kgse_dv1, &regs->fmkg_sch.kgse_dv1);
+	iowrite32be(scheme_regs->kgse_ccbs, &regs->fmkg_sch.kgse_ccbs);
+	iowrite32be(scheme_regs->kgse_mv, &regs->fmkg_sch.kgse_mv);
+	iowrite32be(scheme_regs->kgse_om, &regs->fmkg_sch.kgse_om);
+	iowrite32be(scheme_regs->kgse_vsp, &regs->fmkg_sch.kgse_vsp);
+
+	for (i = 0 ; i < FM_KG_NUM_OF_GENERIC_REGS ; i++)
+		iowrite32be(scheme_regs->kgse_gec[i],
+			    &regs->fmkg_sch.kgse_gec[i]);
+
+	/* Write AR (Action register) */
+	ar_reg = build_ar_scheme(scheme_id, update_counter, true);
+	err = keygen_write_ar_wait(regs, ar_reg);
+	if (err != 0) {
+		pr_err("Writing Action Register failed\n");
+		return err;
+	}
+
+	return err;
+}
+
+/* get_free_scheme_id
+ *
+ * Find the first free scheme available to be used
+ *
+ * keygen: KeyGen handle
+ * scheme_id: pointer to scheme id
+ *
+ * Return: 0 on success, -EINVAL when the are no available free schemes
+ */
+static int get_free_scheme_id(struct fman_keygen *keygen, u8 *scheme_id)
+{
+	u8 i;
+
+	for (i = 0; i < FM_KG_MAX_NUM_OF_SCHEMES; i++)
+		if (!keygen->schemes[i].used) {
+			*scheme_id = i;
+			return 0;
+		}
+
+	return -EINVAL;
+}
+
+/* get_scheme
+ *
+ * Provides the scheme for specified ID
+ *
+ * keygen: KeyGen handle
+ * scheme_id: Scheme ID
+ *
+ * Return: handle to required scheme
+ */
+static struct keygen_scheme *get_scheme(struct fman_keygen *keygen,
+					u8 scheme_id)
+{
+	if (scheme_id >= FM_KG_MAX_NUM_OF_SCHEMES)
+		return NULL;
+	return &keygen->schemes[scheme_id];
+}
+
+/* keygen_bind_port_to_schemes
+ *
+ * Bind the port to schemes
+ *
+ * keygen: KeyGen handle
+ * scheme_id: id of the scheme to bind to
+ * bind: true to bind the port or false to unbind it
+ *
+ * Return: Zero for success or error code in case of failure
+ */
+static int keygen_bind_port_to_schemes(struct fman_keygen *keygen,
+				       u8 scheme_id,
+					bool bind)
+{
+	struct fman_kg_regs __iomem *keygen_regs = keygen->keygen_regs;
+	struct keygen_scheme *scheme;
+	u32 ar_reg;
+	u32 schemes_vector = 0;
+	int err;
+
+	scheme = get_scheme(keygen, scheme_id);
+	if (!scheme) {
+		pr_err("Requested Scheme does not exist\n");
+		return -EINVAL;
+	}
+	if (!scheme->used) {
+		pr_err("Cannot bind port to an invalid scheme\n");
+		return -EINVAL;
+	}
+
+	schemes_vector |= 1 << (31 - scheme_id);
+
+	ar_reg = build_ar_bind_scheme(scheme->hw_port_id, false);
+	err = keygen_write_ar_wait(keygen_regs, ar_reg);
+	if (err != 0) {
+		pr_err("Reading Action Register failed\n");
+		return err;
+	}
+
+	keygen_write_sp(keygen_regs, schemes_vector, bind);
+
+	ar_reg = build_ar_bind_scheme(scheme->hw_port_id, true);
+	err = keygen_write_ar_wait(keygen_regs, ar_reg);
+	if (err != 0) {
+		pr_err("Writing Action Register failed\n");
+		return err;
+	}
+
+	return 0;
+}
+
+/* keygen_scheme_setup
+ *
+ * Setup the scheme according to required configuration
+ *
+ * keygen: KeyGen handle
+ * scheme_id: scheme ID
+ * enable: true to enable scheme or false to disable it
+ *
+ * Return: Zero for success or error code in case of failure
+ */
+static int keygen_scheme_setup(struct fman_keygen *keygen, u8 scheme_id,
+			       bool enable)
+{
+	struct fman_kg_regs __iomem *keygen_regs = keygen->keygen_regs;
+	struct fman_kg_scheme_regs scheme_regs;
+	struct keygen_scheme *scheme;
+	u32 tmp_reg;
+	int err;
+
+	scheme = get_scheme(keygen, scheme_id);
+	if (!scheme) {
+		pr_err("Requested Scheme does not exist\n");
+		return -EINVAL;
+	}
+	if (enable && scheme->used) {
+		pr_err("The requested Scheme is already used\n");
+		return -EINVAL;
+	}
+
+	/* Clear scheme registers */
+	memset(&scheme_regs, 0, sizeof(struct fman_kg_scheme_regs));
+
+	/* Setup all scheme registers: */
+	tmp_reg = 0;
+
+	if (enable) {
+		/* Enable Scheme */
+		tmp_reg |= KG_SCH_MODE_EN;
+		/* Enqueue frame NIA */
+		tmp_reg |= ENQUEUE_KG_DFLT_NIA;
+	}
+
+	scheme_regs.kgse_mode = tmp_reg;
+
+	scheme_regs.kgse_mv = scheme->match_vector;
+
+	/* Scheme don't override StorageProfile:
+	 * valid only for DPAA_VERSION >= 11
+	 */
+	scheme_regs.kgse_vsp = KG_SCH_VSP_NO_KSP_EN;
+
+	/* Configure Hard-Coded Rx Hashing: */
+
+	if (scheme->use_hashing) {
+		/* configure kgse_ekfc */
+		scheme_regs.kgse_ekfc = DEFAULT_HASH_KEY_EXTRACT_FIELDS;
+
+		/* configure kgse_ekdv */
+		tmp_reg = 0;
+		tmp_reg |= (KG_SCH_DEF_USE_KGSE_DV_0 <<
+				KG_SCH_DEF_IP_ADDR_SHIFT);
+		tmp_reg |= (KG_SCH_DEF_USE_KGSE_DV_1 <<
+				KG_SCH_DEF_L4_PORT_SHIFT);
+		scheme_regs.kgse_ekdv = tmp_reg;
+
+		/* configure kgse_dv0 */
+		scheme_regs.kgse_dv0 = DEFAULT_HASH_KEY_IPv4_ADDR;
+		/* configure kgse_dv1 */
+		scheme_regs.kgse_dv1 = DEFAULT_HASH_KEY_L4_PORT;
+
+		/* configure kgse_hc  */
+		tmp_reg = 0;
+		tmp_reg |= ((scheme->hash_fqid_count - 1) <<
+				DEFAULT_HASH_DIST_FQID_SHIFT);
+		tmp_reg |= scheme->hashShift << KG_SCH_HASH_CONFIG_SHIFT_SHIFT;
+
+		if (scheme->symmetric_hash) {
+			/* Normally extraction key should be verified if
+			 * complies with symmetric hash
+			 * But because extraction is hard-coded, we are sure
+			 * the key is symmetric
+			 */
+			tmp_reg |= KG_SCH_HASH_CONFIG_SYM;
+		}
+		scheme_regs.kgse_hc = tmp_reg;
+	} else {
+		scheme_regs.kgse_ekfc = 0;
+		scheme_regs.kgse_hc = 0;
+		scheme_regs.kgse_ekdv = 0;
+		scheme_regs.kgse_dv0 = 0;
+		scheme_regs.kgse_dv1 = 0;
+	}
+
+	/* configure kgse_fqb: Scheme FQID base */
+	tmp_reg = 0;
+	tmp_reg |= scheme->base_fqid;
+	scheme_regs.kgse_fqb = tmp_reg;
+
+	/* features not used by hard-coded configuration */
+	scheme_regs.kgse_bmch = 0;
+	scheme_regs.kgse_bmcl = 0;
+	scheme_regs.kgse_spc = 0;
+
+	/* Write scheme registers */
+	err = keygen_write_scheme(keygen_regs, scheme_id, &scheme_regs, true);
+	if (err != 0) {
+		pr_err("Writing scheme registers failed\n");
+		return err;
+	}
+
+	/* Update used field for Scheme */
+	scheme->used = enable;
+
+	return 0;
+}
+
+/* keygen_init
+ *
+ * KeyGen initialization:
+ * Initializes and enables KeyGen, allocate driver memory, setup registers,
+ * clear port bindings, invalidate all schemes
+ *
+ * keygen_regs: KeyGen registers base address
+ *
+ * Return: Handle to KeyGen driver
+ */
+struct fman_keygen *keygen_init(struct fman_kg_regs __iomem *keygen_regs)
+{
+	struct fman_keygen *keygen;
+	u32 ar;
+	int i;
+
+	/* Allocate memory for KeyGen driver */
+	keygen = kzalloc(sizeof(*keygen), GFP_KERNEL);
+	if (!keygen)
+		return NULL;
+
+	keygen->keygen_regs = keygen_regs;
+
+	/* KeyGen initialization (for Master partition):
+	 * Setup KeyGen registers
+	 */
+	iowrite32be(ENQUEUE_KG_DFLT_NIA, &keygen_regs->fmkg_gcr);
+
+	iowrite32be(FM_EX_KG_DOUBLE_ECC | FM_EX_KG_KEYSIZE_OVERFLOW,
+		    &keygen_regs->fmkg_eer);
+
+	iowrite32be(0, &keygen_regs->fmkg_fdor);
+	iowrite32be(0, &keygen_regs->fmkg_gdv0r);
+	iowrite32be(0, &keygen_regs->fmkg_gdv1r);
+
+	/* Clear binding between ports to schemes and classification plans
+	 * so that all ports are not bound to any scheme/classification plan
+	 */
+	for (i = 0; i < FMAN_MAX_NUM_OF_HW_PORTS; i++) {
+		/* Clear all pe sp schemes registers */
+		keygen_write_sp(keygen_regs, 0xffffffff, false);
+		ar = build_ar_bind_scheme(i, true);
+		keygen_write_ar_wait(keygen_regs, ar);
+
+		/* Clear all pe cpp classification plans registers */
+		keygen_write_cpp(keygen_regs, 0);
+		ar = build_ar_bind_cls_plan(i, true);
+		keygen_write_ar_wait(keygen_regs, ar);
+	}
+
+	/* Enable all scheme interrupts */
+	iowrite32be(0xFFFFFFFF, &keygen_regs->fmkg_seer);
+	iowrite32be(0xFFFFFFFF, &keygen_regs->fmkg_seeer);
+
+	/* Enable KyeGen */
+	iowrite32be(ioread32be(&keygen_regs->fmkg_gcr) | FM_KG_KGGCR_EN,
+		    &keygen_regs->fmkg_gcr);
+
+	return keygen;
+}
+EXPORT_SYMBOL(keygen_init);
+
+/* keygen_port_hashing_init
+ *
+ * Initializes a port for Rx Hashing with specified configuration parameters
+ *
+ * keygen: KeyGen handle
+ * hw_port_id: HW Port ID
+ * hash_base_fqid: Hashing Base FQID used for spreading
+ * hash_size: Hashing size
+ *
+ * Return: Zero for success or error code in case of failure
+ */
+int keygen_port_hashing_init(struct fman_keygen *keygen, u8 hw_port_id,
+			     u32 hash_base_fqid, u32 hash_size)
+{
+	struct keygen_scheme *scheme;
+	u8 scheme_id;
+	int err;
+
+	/* Validate Scheme configuration parameters */
+	if (hash_base_fqid == 0 || (hash_base_fqid & ~0x00FFFFFF)) {
+		pr_err("Base FQID must be between 1 and 2^24-1\n");
+		return -EINVAL;
+	}
+	if (hash_size == 0 || (hash_size & (hash_size - 1)) != 0) {
+		pr_err("Hash size must be power of two\n");
+		return -EINVAL;
+	}
+
+	/* Find a free scheme */
+	err = get_free_scheme_id(keygen, &scheme_id);
+	if (err) {
+		pr_err("The maximum number of available Schemes has been exceeded\n");
+		return -EINVAL;
+	}
+
+	/* Create and configure Hard-Coded Scheme: */
+
+	scheme = get_scheme(keygen, scheme_id);
+	if (!scheme) {
+		pr_err("Requested Scheme does not exist\n");
+		return -EINVAL;
+	}
+	if (scheme->used) {
+		pr_err("The requested Scheme is already used\n");
+		return -EINVAL;
+	}
+
+	/* Clear all scheme fields because the scheme may have been
+	 * previously used
+	 */
+	memset(scheme, 0, sizeof(struct keygen_scheme));
+
+	/* Setup scheme: */
+	scheme->hw_port_id = hw_port_id;
+	scheme->use_hashing = true;
+	scheme->base_fqid = hash_base_fqid;
+	scheme->hash_fqid_count = hash_size;
+	scheme->symmetric_hash = DEFAULT_SYMMETRIC_HASH;
+	scheme->hashShift = DEFAULT_HASH_SHIFT;
+
+	/* All Schemes in hard-coded configuration
+	 * are Indirect Schemes
+	 */
+	scheme->match_vector = 0;
+
+	err = keygen_scheme_setup(keygen, scheme_id, true);
+	if (err != 0) {
+		pr_err("Scheme setup failed\n");
+		return err;
+	}
+
+	/* Bind Rx port to Scheme */
+	err = keygen_bind_port_to_schemes(keygen, scheme_id, true);
+	if (err != 0) {
+		pr_err("Binding port to schemes failed\n");
+		return err;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(keygen_port_hashing_init);
diff --git a/drivers/net/ethernet/freescale/fman/fman_keygen.h b/drivers/net/ethernet/freescale/fman/fman_keygen.h
new file mode 100644
index 000000000000..c4640de3f4cb
--- /dev/null
+++ b/drivers/net/ethernet/freescale/fman/fman_keygen.h
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2017 NXP
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of NXP nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NXP ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL NXP BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __KEYGEN_H
+#define __KEYGEN_H
+
+#include <linux/io.h>
+
+struct fman_keygen;
+struct fman_kg_regs;
+
+struct fman_keygen *keygen_init(struct fman_kg_regs __iomem *keygen_regs);
+
+int keygen_port_hashing_init(struct fman_keygen *keygen, u8 hw_port_id,
+			     u32 hash_base_fqid, u32 hash_size);
+
+#endif /* __KEYGEN_H */
diff --git a/drivers/net/ethernet/freescale/fman/fman_port.c b/drivers/net/ethernet/freescale/fman/fman_port.c
index 57bf44fa16a1..1789b206be58 100644
--- a/drivers/net/ethernet/freescale/fman/fman_port.c
+++ b/drivers/net/ethernet/freescale/fman/fman_port.c
@@ -32,10 +32,6 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
-#include "fman_port.h"
-#include "fman.h"
-#include "fman_sp.h"
-
 #include <linux/io.h>
 #include <linux/slab.h>
 #include <linux/module.h>
@@ -45,6 +41,11 @@
 #include <linux/delay.h>
 #include <linux/libfdt_env.h>
 
+#include "fman.h"
+#include "fman_port.h"
+#include "fman_sp.h"
+#include "fman_keygen.h"
+
 /* Queue ID */
 #define DFLT_FQ_ID		0x00FFFFFF
 
@@ -184,6 +185,7 @@
 #define NIA_ENG_QMI_ENQ					0x00540000
 #define NIA_ENG_QMI_DEQ					0x00580000
 #define NIA_ENG_HWP					0x00440000
+#define NIA_ENG_HWK					0x00480000
 #define NIA_BMI_AC_ENQ_FRAME				0x00000002
 #define NIA_BMI_AC_TX_RELEASE				0x000002C0
 #define NIA_BMI_AC_RELEASE				0x000000C0
@@ -394,6 +396,8 @@ struct fman_port_bpools {
 struct fman_port_cfg {
 	u32 dflt_fqid;
 	u32 err_fqid;
+	u32 pcd_base_fqid;
+	u32 pcd_fqs_count;
 	u8 deq_sp;
 	bool deq_high_priority;
 	enum fman_port_deq_type deq_type;
@@ -1271,6 +1275,10 @@ static void set_rx_dflt_cfg(struct fman_port *port,
 		port_params->specific_params.rx_params.err_fqid;
 	port->cfg->dflt_fqid =
 		port_params->specific_params.rx_params.dflt_fqid;
+	port->cfg->pcd_base_fqid =
+		port_params->specific_params.rx_params.pcd_base_fqid;
+	port->cfg->pcd_fqs_count =
+		port_params->specific_params.rx_params.pcd_fqs_count;
 }
 
 static void set_tx_dflt_cfg(struct fman_port *port,
@@ -1398,6 +1406,24 @@ err_port_cfg:
 EXPORT_SYMBOL(fman_port_config);
 
 /**
+ * fman_port_use_kg_hash
+ * port:        A pointer to a FM Port module.
+ * Sets the HW KeyGen or the BMI as HW Parser next engine, enabling
+ * or bypassing the KeyGen hashing of Rx traffic
+ */
+void fman_port_use_kg_hash(struct fman_port *port, bool enable)
+{
+	if (enable)
+		/* After the Parser frames go to KeyGen */
+		iowrite32be(NIA_ENG_HWK, &port->bmi_regs->rx.fmbm_rfpne);
+	else
+		/* After the Parser frames go to BMI */
+		iowrite32be(NIA_ENG_BMI | NIA_BMI_AC_ENQ_FRAME,
+			    &port->bmi_regs->rx.fmbm_rfpne);
+}
+EXPORT_SYMBOL(fman_port_use_kg_hash);
+
+/**
  * fman_port_init
  * port:	A pointer to a FM Port module.
  * Initializes the FM PORT module by defining the software structure and
@@ -1407,9 +1433,10 @@ EXPORT_SYMBOL(fman_port_config);
  */
 int fman_port_init(struct fman_port *port)
 {
+	struct fman_port_init_params params;
+	struct fman_keygen *keygen;
 	struct fman_port_cfg *cfg;
 	int err;
-	struct fman_port_init_params params;
 
 	if (is_init_done(port->cfg))
 		return -EINVAL;
@@ -1472,6 +1499,17 @@ int fman_port_init(struct fman_port *port)
 	if (err)
 		return err;
 
+	if (port->cfg->pcd_fqs_count) {
+		keygen = port->dts_params.fman->keygen;
+		err = keygen_port_hashing_init(keygen, port->port_id,
+					       port->cfg->pcd_base_fqid,
+					       port->cfg->pcd_fqs_count);
+		if (err)
+			return err;
+
+		fman_port_use_kg_hash(port, true);
+	}
+
 	kfree(port->cfg);
 	port->cfg = NULL;
 
@@ -1682,6 +1720,17 @@ u32 fman_port_get_qman_channel_id(struct fman_port *port)
 }
 EXPORT_SYMBOL(fman_port_get_qman_channel_id);
 
+int fman_port_get_hash_result_offset(struct fman_port *port, u32 *offset)
+{
+	if (port->buffer_offsets.hash_result_offset == ILLEGAL_BASE)
+		return -EINVAL;
+
+	*offset = port->buffer_offsets.hash_result_offset;
+
+	return 0;
+}
+EXPORT_SYMBOL(fman_port_get_hash_result_offset);
+
 static int fman_port_probe(struct platform_device *of_dev)
 {
 	struct fman_port *port;
@@ -1720,8 +1769,8 @@ static int fman_port_probe(struct platform_device *of_dev)
 
 	err = of_property_read_u32(port_node, "cell-index", &val);
 	if (err) {
-		dev_err(port->dev, "%s: reading cell-index for %s failed\n",
-			__func__, port_node->full_name);
+		dev_err(port->dev, "%s: reading cell-index for %pOF failed\n",
+			__func__, port_node);
 		err = -EINVAL;
 		goto return_err;
 	}
diff --git a/drivers/net/ethernet/freescale/fman/fman_port.h b/drivers/net/ethernet/freescale/fman/fman_port.h
index 8ba901737048..e86ca6a34e4e 100644
--- a/drivers/net/ethernet/freescale/fman/fman_port.h
+++ b/drivers/net/ethernet/freescale/fman/fman_port.h
@@ -100,6 +100,9 @@ struct fman_port;
 struct fman_port_rx_params {
 	u32 err_fqid;			/* Error Queue Id. */
 	u32 dflt_fqid;			/* Default Queue Id. */
+	u32 pcd_base_fqid;		/* PCD base Queue Id. */
+	u32 pcd_fqs_count;		/* Number of PCD FQs. */
+
 	/* Which external buffer pools are used
 	 * (up to FMAN_PORT_MAX_EXT_POOLS_NUM), and their sizes.
 	 */
@@ -134,6 +137,8 @@ struct fman_port_params {
 
 int fman_port_config(struct fman_port *port, struct fman_port_params *params);
 
+void fman_port_use_kg_hash(struct fman_port *port, bool enable);
+
 int fman_port_init(struct fman_port *port);
 
 int fman_port_cfg_buf_prefix_content(struct fman_port *port,
@@ -146,6 +151,8 @@ int fman_port_enable(struct fman_port *port);
 
 u32 fman_port_get_qman_channel_id(struct fman_port *port);
 
+int fman_port_get_hash_result_offset(struct fman_port *port, u32 *offset);
+
 struct fman_port *fman_port_bind(struct device *dev);
 
 #endif /* __FMAN_PORT_H */
diff --git a/drivers/net/ethernet/freescale/fman/mac.c b/drivers/net/ethernet/freescale/fman/mac.c
index 6e67d22fd0d5..387eb4a88b72 100644
--- a/drivers/net/ethernet/freescale/fman/mac.c
+++ b/drivers/net/ethernet/freescale/fman/mac.c
@@ -623,6 +623,8 @@ static struct platform_device *dpaa_eth_add_device(int fman_id,
 		goto no_mem;
 	}
 
+	pdev->dev.of_node = node;
+	pdev->dev.parent = priv->dev;
 	set_dma_ops(&pdev->dev, get_dma_ops(priv->dev));
 
 	ret = platform_device_add_data(pdev, &data, sizeof(data));
@@ -698,8 +700,8 @@ static int mac_probe(struct platform_device *_of_dev)
 		priv->internal_phy_node = of_parse_phandle(mac_node,
 							  "pcsphy-handle", 0);
 	} else {
-		dev_err(dev, "MAC node (%s) contains unsupported MAC\n",
-			mac_node->full_name);
+		dev_err(dev, "MAC node (%pOF) contains unsupported MAC\n",
+			mac_node);
 		err = -EINVAL;
 		goto _return;
 	}
@@ -712,16 +714,15 @@ static int mac_probe(struct platform_device *_of_dev)
 	/* Get the FM node */
 	dev_node = of_get_parent(mac_node);
 	if (!dev_node) {
-		dev_err(dev, "of_get_parent(%s) failed\n",
-			mac_node->full_name);
+		dev_err(dev, "of_get_parent(%pOF) failed\n",
+			mac_node);
 		err = -EINVAL;
 		goto _return_dev_set_drvdata;
 	}
 
 	of_dev = of_find_device_by_node(dev_node);
 	if (!of_dev) {
-		dev_err(dev, "of_find_device_by_node(%s) failed\n",
-			dev_node->full_name);
+		dev_err(dev, "of_find_device_by_node(%pOF) failed\n", dev_node);
 		err = -EINVAL;
 		goto _return_of_node_put;
 	}
@@ -729,8 +730,7 @@ static int mac_probe(struct platform_device *_of_dev)
 	/* Get the FMan cell-index */
 	err = of_property_read_u32(dev_node, "cell-index", &val);
 	if (err) {
-		dev_err(dev, "failed to read cell-index for %s\n",
-			dev_node->full_name);
+		dev_err(dev, "failed to read cell-index for %pOF\n", dev_node);
 		err = -EINVAL;
 		goto _return_of_node_put;
 	}
@@ -739,7 +739,7 @@ static int mac_probe(struct platform_device *_of_dev)
 
 	priv->fman = fman_bind(&of_dev->dev);
 	if (!priv->fman) {
-		dev_err(dev, "fman_bind(%s) failed\n", dev_node->full_name);
+		dev_err(dev, "fman_bind(%pOF) failed\n", dev_node);
 		err = -ENODEV;
 		goto _return_of_node_put;
 	}
@@ -749,8 +749,8 @@ static int mac_probe(struct platform_device *_of_dev)
 	/* Get the address of the memory mapped registers */
 	err = of_address_to_resource(mac_node, 0, &res);
 	if (err < 0) {
-		dev_err(dev, "of_address_to_resource(%s) = %d\n",
-			mac_node->full_name, err);
+		dev_err(dev, "of_address_to_resource(%pOF) = %d\n",
+			mac_node, err);
 		goto _return_dev_set_drvdata;
 	}
 
@@ -784,8 +784,7 @@ static int mac_probe(struct platform_device *_of_dev)
 	/* Get the cell-index */
 	err = of_property_read_u32(mac_node, "cell-index", &val);
 	if (err) {
-		dev_err(dev, "failed to read cell-index for %s\n",
-			mac_node->full_name);
+		dev_err(dev, "failed to read cell-index for %pOF\n", mac_node);
 		err = -EINVAL;
 		goto _return_dev_set_drvdata;
 	}
@@ -794,8 +793,7 @@ static int mac_probe(struct platform_device *_of_dev)
 	/* Get the MAC address */
 	mac_addr = of_get_mac_address(mac_node);
 	if (!mac_addr) {
-		dev_err(dev, "of_get_mac_address(%s) failed\n",
-			mac_node->full_name);
+		dev_err(dev, "of_get_mac_address(%pOF) failed\n", mac_node);
 		err = -EINVAL;
 		goto _return_dev_set_drvdata;
 	}
@@ -804,15 +802,15 @@ static int mac_probe(struct platform_device *_of_dev)
 	/* Get the port handles */
 	nph = of_count_phandle_with_args(mac_node, "fsl,fman-ports", NULL);
 	if (unlikely(nph < 0)) {
-		dev_err(dev, "of_count_phandle_with_args(%s, fsl,fman-ports) failed\n",
-			mac_node->full_name);
+		dev_err(dev, "of_count_phandle_with_args(%pOF, fsl,fman-ports) failed\n",
+			mac_node);
 		err = nph;
 		goto _return_dev_set_drvdata;
 	}
 
 	if (nph != ARRAY_SIZE(mac_dev->port)) {
-		dev_err(dev, "Not supported number of fman-ports handles of mac node %s from device tree\n",
-			mac_node->full_name);
+		dev_err(dev, "Not supported number of fman-ports handles of mac node %pOF from device tree\n",
+			mac_node);
 		err = -EINVAL;
 		goto _return_dev_set_drvdata;
 	}
@@ -821,24 +819,24 @@ static int mac_probe(struct platform_device *_of_dev)
 		/* Find the port node */
 		dev_node = of_parse_phandle(mac_node, "fsl,fman-ports", i);
 		if (!dev_node) {
-			dev_err(dev, "of_parse_phandle(%s, fsl,fman-ports) failed\n",
-				mac_node->full_name);
+			dev_err(dev, "of_parse_phandle(%pOF, fsl,fman-ports) failed\n",
+				mac_node);
 			err = -EINVAL;
 			goto _return_of_node_put;
 		}
 
 		of_dev = of_find_device_by_node(dev_node);
 		if (!of_dev) {
-			dev_err(dev, "of_find_device_by_node(%s) failed\n",
-				dev_node->full_name);
+			dev_err(dev, "of_find_device_by_node(%pOF) failed\n",
+				dev_node);
 			err = -EINVAL;
 			goto _return_of_node_put;
 		}
 
 		mac_dev->port[i] = fman_port_bind(&of_dev->dev);
 		if (!mac_dev->port[i]) {
-			dev_err(dev, "dev_get_drvdata(%s) failed\n",
-				dev_node->full_name);
+			dev_err(dev, "dev_get_drvdata(%pOF) failed\n",
+				dev_node);
 			err = -EINVAL;
 			goto _return_of_node_put;
 		}
@@ -849,8 +847,8 @@ static int mac_probe(struct platform_device *_of_dev)
 	phy_if = of_get_phy_mode(mac_node);
 	if (phy_if < 0) {
 		dev_warn(dev,
-			 "of_get_phy_mode() for %s failed. Defaulting to SGMII\n",
-			 mac_node->full_name);
+			 "of_get_phy_mode() for %pOF failed. Defaulting to SGMII\n",
+			 mac_node);
 		phy_if = PHY_INTERFACE_MODE_SGMII;
 	}
 	priv->phy_if = phy_if;
diff --git a/drivers/net/ethernet/freescale/fs_enet/mii-bitbang.c b/drivers/net/ethernet/freescale/fs_enet/mii-bitbang.c
index 1f015edcca22..c8e5d889bd81 100644
--- a/drivers/net/ethernet/freescale/fs_enet/mii-bitbang.c
+++ b/drivers/net/ethernet/freescale/fs_enet/mii-bitbang.c
@@ -100,7 +100,7 @@ static inline void mdc(struct mdiobb_ctrl *ctrl, int what)
 	in_be32(bitbang->dat);
 }
 
-static struct mdiobb_ops bb_ops = {
+static const struct mdiobb_ops bb_ops = {
 	.owner = THIS_MODULE,
 	.set_mdc = mdc,
 	.set_mdio_dir = mdio_dir,
diff --git a/drivers/net/ethernet/freescale/fsl_pq_mdio.c b/drivers/net/ethernet/freescale/fsl_pq_mdio.c
index a10de1e9c157..80ad16acf0f1 100644
--- a/drivers/net/ethernet/freescale/fsl_pq_mdio.c
+++ b/drivers/net/ethernet/freescale/fsl_pq_mdio.c
@@ -267,8 +267,8 @@ static void ucc_configure(phys_addr_t start, phys_addr_t end)
 
 		ret = of_address_to_resource(np, 0, &res);
 		if (ret < 0) {
-			pr_debug("fsl-pq-mdio: no address range in node %s\n",
-				 np->full_name);
+			pr_debug("fsl-pq-mdio: no address range in node %pOF\n",
+				 np);
 			continue;
 		}
 
@@ -280,8 +280,8 @@ static void ucc_configure(phys_addr_t start, phys_addr_t end)
 		if (!iprop) {
 			iprop = of_get_property(np, "device-id", NULL);
 			if (!iprop) {
-				pr_debug("fsl-pq-mdio: no UCC ID in node %s\n",
-					 np->full_name);
+				pr_debug("fsl-pq-mdio: no UCC ID in node %pOF\n",
+					 np);
 				continue;
 			}
 		}
@@ -293,8 +293,8 @@ static void ucc_configure(phys_addr_t start, phys_addr_t end)
 		 * numbered from 1, not 0.
 		 */
 		if (ucc_set_qe_mux_mii_mng(id - 1) < 0) {
-			pr_debug("fsl-pq-mdio: invalid UCC ID in node %s\n",
-				 np->full_name);
+			pr_debug("fsl-pq-mdio: invalid UCC ID in node %pOF\n",
+				 np);
 			continue;
 		}
 
@@ -442,8 +442,8 @@ static int fsl_pq_mdio_probe(struct platform_device *pdev)
 	if (data->get_tbipa) {
 		for_each_child_of_node(np, tbi) {
 			if (strcmp(tbi->type, "tbi-phy") == 0) {
-				dev_dbg(&pdev->dev, "found TBI PHY node %s\n",
-					strrchr(tbi->full_name, '/') + 1);
+				dev_dbg(&pdev->dev, "found TBI PHY node %pOFP\n",
+					tbi);
 				break;
 			}
 		}
@@ -454,8 +454,8 @@ static int fsl_pq_mdio_probe(struct platform_device *pdev)
 
 			if (!prop) {
 				dev_err(&pdev->dev,
-					"missing 'reg' property in node %s\n",
-					tbi->full_name);
+					"missing 'reg' property in node %pOF\n",
+					tbi);
 				err = -EBUSY;
 				goto error;
 			}
diff --git a/drivers/net/ethernet/freescale/gianfar.c b/drivers/net/ethernet/freescale/gianfar.c
index c4b4b0a1bbf0..5be52d89b182 100644
--- a/drivers/net/ethernet/freescale/gianfar.c
+++ b/drivers/net/ethernet/freescale/gianfar.c
@@ -3687,7 +3687,7 @@ static noinline void gfar_update_link_state(struct gfar_private *priv)
 		u32 tempval1 = gfar_read(&regs->maccfg1);
 		u32 tempval = gfar_read(&regs->maccfg2);
 		u32 ecntrl = gfar_read(&regs->ecntrl);
-		u32 tx_flow_oldval = (tempval & MACCFG1_TX_FLOW);
+		u32 tx_flow_oldval = (tempval1 & MACCFG1_TX_FLOW);
 
 		if (phydev->duplex != priv->oldduplex) {
 			if (!(phydev->duplex))
diff --git a/drivers/net/ethernet/freescale/gianfar_ptp.c b/drivers/net/ethernet/freescale/gianfar_ptp.c
index 721be13081f9..544114281ea7 100644
--- a/drivers/net/ethernet/freescale/gianfar_ptp.c
+++ b/drivers/net/ethernet/freescale/gianfar_ptp.c
@@ -411,7 +411,7 @@ static int ptp_gianfar_enable(struct ptp_clock_info *ptp,
 	return -EOPNOTSUPP;
 }
 
-static struct ptp_clock_info ptp_gianfar_caps = {
+static const struct ptp_clock_info ptp_gianfar_caps = {
 	.owner		= THIS_MODULE,
 	.name		= "gianfar clock",
 	.max_adj	= 512000,
diff --git a/drivers/net/ethernet/hisilicon/Kconfig b/drivers/net/ethernet/hisilicon/Kconfig
index d11287e11371..91c7bdb9b43c 100644
--- a/drivers/net/ethernet/hisilicon/Kconfig
+++ b/drivers/net/ethernet/hisilicon/Kconfig
@@ -76,4 +76,31 @@ config HNS_ENET
 	  This selects the general ethernet driver for HNS.  This module make
 	  use of any HNS AE driver, such as HNS_DSAF
 
+config HNS3
+	tristate "Hisilicon Network Subsystem Support HNS3 (Framework)"
+    depends on PCI
+	---help---
+	  This selects the framework support for Hisilicon Network Subsystem 3.
+	  This layer facilitates clients like ENET, RoCE and user-space ethernet
+	  drivers(like ODP)to register with HNAE devices and their associated
+	  operations.
+
+config HNS3_HCLGE
+	tristate "Hisilicon HNS3 HCLGE Acceleration Engine & Compatibility Layer Support"
+    depends on PCI_MSI
+	depends on HNS3
+	---help---
+	  This selects the HNS3_HCLGE network acceleration engine & its hardware
+	  compatibility layer. The engine would be used in Hisilicon hip08 family of
+	  SoCs and further upcoming SoCs.
+
+config HNS3_ENET
+	tristate "Hisilicon HNS3 Ethernet Device Support"
+    depends on 64BIT && PCI
+	depends on HNS3 && HNS3_HCLGE
+	---help---
+	  This selects the Ethernet Driver for Hisilicon Network Subsystem 3 for hip08
+	  family of SoCs. This module depends upon HNAE3 driver to access the HNAE3
+	  devices and their associated operations.
+
 endif # NET_VENDOR_HISILICON
diff --git a/drivers/net/ethernet/hisilicon/Makefile b/drivers/net/ethernet/hisilicon/Makefile
index 8661695024dc..3828c435c18f 100644
--- a/drivers/net/ethernet/hisilicon/Makefile
+++ b/drivers/net/ethernet/hisilicon/Makefile
@@ -6,4 +6,5 @@ obj-$(CONFIG_HIX5HD2_GMAC) += hix5hd2_gmac.o
 obj-$(CONFIG_HIP04_ETH) += hip04_eth.o
 obj-$(CONFIG_HNS_MDIO) += hns_mdio.o
 obj-$(CONFIG_HNS) += hns/
+obj-$(CONFIG_HNS3) += hns3/
 obj-$(CONFIG_HISI_FEMAC) += hisi_femac.o
diff --git a/drivers/net/ethernet/hisilicon/hns/hnae.c b/drivers/net/ethernet/hisilicon/hns/hnae.c
index 9d9b6e6dd988..a051e582d541 100644
--- a/drivers/net/ethernet/hisilicon/hns/hnae.c
+++ b/drivers/net/ethernet/hisilicon/hns/hnae.c
@@ -202,6 +202,7 @@ hnae_init_ring(struct hnae_queue *q, struct hnae_ring *ring, int flags)
 	ring->q = q;
 	ring->flags = flags;
 	spin_lock_init(&ring->lock);
+	ring->coal_param = q->handle->coal_param;
 	assert(!ring->desc && !ring->desc_cb && !ring->desc_dma_addr);
 
 	/* not matter for tx or rx ring, the ntc and ntc start from 0 */
diff --git a/drivers/net/ethernet/hisilicon/hns/hnae.h b/drivers/net/ethernet/hisilicon/hns/hnae.h
index 7ba653af19cb..3e62692af011 100644
--- a/drivers/net/ethernet/hisilicon/hns/hnae.h
+++ b/drivers/net/ethernet/hisilicon/hns/hnae.h
@@ -89,6 +89,10 @@ do { \
 
 #define RCB_RING_NAME_LEN 16
 
+#define HNAE_LOWEST_LATENCY_COAL_PARAM	30
+#define HNAE_LOW_LATENCY_COAL_PARAM	80
+#define HNAE_BULK_LATENCY_COAL_PARAM	150
+
 enum hnae_led_state {
 	HNAE_LED_INACTIVE,
 	HNAE_LED_ACTIVE,
@@ -292,6 +296,12 @@ struct hnae_ring {
 
 	int flags;          /* ring attribute */
 	int irq_init_flag;
+
+	/* total rx bytes after last rx rate calucated */
+	u64 coal_last_rx_bytes;
+	unsigned long coal_last_jiffies;
+	u32 coal_param;
+	u32 coal_rx_rate;	/* rx rate in MB */
 };
 
 #define ring_ptr_move_fw(ring, p) \
@@ -548,8 +558,13 @@ struct hnae_handle {
 	u32 if_support;
 	int q_num;
 	int vf_id;
+	unsigned long coal_last_jiffies;
+	u32 coal_param;		/* self adapt coalesce param */
+	/* the ring index of last ring that set coal param */
+	u32 coal_ring_idx;
 	u32 eport_id;
 	u32 dport_id;	/* v2 tx bd should fill the dport_id */
+	bool coal_adapt_en;
 	enum hnae_port_type port_type;
 	enum hnae_media_type media_type;
 	struct list_head node;    /* list to hnae_ae_dev->handle_list */
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_ae_adapt.c b/drivers/net/ethernet/hisilicon/hns/hns_ae_adapt.c
index a37166ee577b..bd68379d2bea 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_ae_adapt.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_ae_adapt.c
@@ -99,6 +99,7 @@ struct hnae_handle *hns_ae_get_handle(struct hnae_ae_dev *dev,
 	ae_handle->owner_dev = dsaf_dev->dev;
 	ae_handle->dev = dev;
 	ae_handle->q_num = qnum_per_vf;
+	ae_handle->coal_param = HNAE_LOWEST_LATENCY_COAL_PARAM;
 
 	/* find ring pair, and set vf id*/
 	for (ae_handle->vf_id = 0;
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_enet.c b/drivers/net/ethernet/hisilicon/hns/hns_enet.c
index 3987699f8fe6..36520634c96a 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_enet.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_enet.c
@@ -812,6 +812,113 @@ static int hns_desc_unused(struct hnae_ring *ring)
 	return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
 }
 
+#define HNS_LOWEST_LATENCY_RATE		27	/* 27 MB/s */
+#define HNS_LOW_LATENCY_RATE			80	/* 80 MB/s */
+
+#define HNS_COAL_BDNUM			3
+
+static u32 hns_coal_rx_bdnum(struct hnae_ring *ring)
+{
+	bool coal_enable = ring->q->handle->coal_adapt_en;
+
+	if (coal_enable &&
+	    ring->coal_last_rx_bytes > HNS_LOWEST_LATENCY_RATE)
+		return HNS_COAL_BDNUM;
+	else
+		return 0;
+}
+
+static void hns_update_rx_rate(struct hnae_ring *ring)
+{
+	bool coal_enable = ring->q->handle->coal_adapt_en;
+	u32 time_passed_ms;
+	u64 total_bytes;
+
+	if (!coal_enable ||
+	    time_before(jiffies, ring->coal_last_jiffies + (HZ >> 4)))
+		return;
+
+	/* ring->stats.rx_bytes overflowed */
+	if (ring->coal_last_rx_bytes > ring->stats.rx_bytes) {
+		ring->coal_last_rx_bytes = ring->stats.rx_bytes;
+		ring->coal_last_jiffies = jiffies;
+		return;
+	}
+
+	total_bytes = ring->stats.rx_bytes - ring->coal_last_rx_bytes;
+	time_passed_ms = jiffies_to_msecs(jiffies - ring->coal_last_jiffies);
+	do_div(total_bytes, time_passed_ms);
+	ring->coal_rx_rate = total_bytes >> 10;
+
+	ring->coal_last_rx_bytes = ring->stats.rx_bytes;
+	ring->coal_last_jiffies = jiffies;
+}
+
+/**
+ * smooth_alg - smoothing algrithm for adjusting coalesce parameter
+ **/
+static u32 smooth_alg(u32 new_param, u32 old_param)
+{
+	u32 gap = (new_param > old_param) ? new_param - old_param
+					  : old_param - new_param;
+
+	if (gap > 8)
+		gap >>= 3;
+
+	if (new_param > old_param)
+		return old_param + gap;
+	else
+		return old_param - gap;
+}
+
+/**
+ * hns_nic_adp_coalesce - self adapte coalesce according to rx rate
+ * @ring_data: pointer to hns_nic_ring_data
+ **/
+static void hns_nic_adpt_coalesce(struct hns_nic_ring_data *ring_data)
+{
+	struct hnae_ring *ring = ring_data->ring;
+	struct hnae_handle *handle = ring->q->handle;
+	u32 new_coal_param, old_coal_param = ring->coal_param;
+
+	if (ring->coal_rx_rate < HNS_LOWEST_LATENCY_RATE)
+		new_coal_param = HNAE_LOWEST_LATENCY_COAL_PARAM;
+	else if (ring->coal_rx_rate < HNS_LOW_LATENCY_RATE)
+		new_coal_param = HNAE_LOW_LATENCY_COAL_PARAM;
+	else
+		new_coal_param = HNAE_BULK_LATENCY_COAL_PARAM;
+
+	if (new_coal_param == old_coal_param &&
+	    new_coal_param == handle->coal_param)
+		return;
+
+	new_coal_param = smooth_alg(new_coal_param, old_coal_param);
+	ring->coal_param = new_coal_param;
+
+	/**
+	 * Because all ring in one port has one coalesce param, when one ring
+	 * calculate its own coalesce param, it cannot write to hardware at
+	 * once. There are three conditions as follows:
+	 *       1. current ring's coalesce param is larger than the hardware.
+	 *       2. or ring which adapt last time can change again.
+	 *       3. timeout.
+	 */
+	if (new_coal_param == handle->coal_param) {
+		handle->coal_last_jiffies = jiffies;
+		handle->coal_ring_idx = ring_data->queue_index;
+	} else if (new_coal_param > handle->coal_param ||
+		   handle->coal_ring_idx == ring_data->queue_index ||
+		   time_after(jiffies, handle->coal_last_jiffies + (HZ >> 4))) {
+		handle->dev->ops->set_coalesce_usecs(handle,
+					new_coal_param);
+		handle->dev->ops->set_coalesce_frames(handle,
+					1, new_coal_param);
+		handle->coal_param = new_coal_param;
+		handle->coal_ring_idx = ring_data->queue_index;
+		handle->coal_last_jiffies = jiffies;
+	}
+}
+
 static int hns_nic_rx_poll_one(struct hns_nic_ring_data *ring_data,
 			       int budget, void *v)
 {
@@ -868,20 +975,27 @@ static bool hns_nic_rx_fini_pro(struct hns_nic_ring_data *ring_data)
 {
 	struct hnae_ring *ring = ring_data->ring;
 	int num = 0;
+	bool rx_stopped;
 
-	ring_data->ring->q->handle->dev->ops->toggle_ring_irq(ring, 0);
+	hns_update_rx_rate(ring);
 
 	/* for hardware bug fixed */
+	ring_data->ring->q->handle->dev->ops->toggle_ring_irq(ring, 0);
 	num = readl_relaxed(ring->io_base + RCB_REG_FBDNUM);
 
-	if (num > 0) {
+	if (num <= hns_coal_rx_bdnum(ring)) {
+		if (ring->q->handle->coal_adapt_en)
+			hns_nic_adpt_coalesce(ring_data);
+
+		rx_stopped = true;
+	} else {
 		ring_data->ring->q->handle->dev->ops->toggle_ring_irq(
 			ring_data->ring, 1);
 
-		return false;
-	} else {
-		return true;
+		rx_stopped = false;
 	}
+
+	return rx_stopped;
 }
 
 static bool hns_nic_rx_fini_pro_v2(struct hns_nic_ring_data *ring_data)
@@ -889,12 +1003,17 @@ static bool hns_nic_rx_fini_pro_v2(struct hns_nic_ring_data *ring_data)
 	struct hnae_ring *ring = ring_data->ring;
 	int num;
 
+	hns_update_rx_rate(ring);
 	num = readl_relaxed(ring->io_base + RCB_REG_FBDNUM);
 
-	if (!num)
+	if (num <= hns_coal_rx_bdnum(ring)) {
+		if (ring->q->handle->coal_adapt_en)
+			hns_nic_adpt_coalesce(ring_data);
+
 		return true;
-	else
-		return false;
+	}
+
+	return false;
 }
 
 static inline void hns_nic_reclaim_one_desc(struct hnae_ring *ring,
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_enet.h b/drivers/net/ethernet/hisilicon/hns/hns_enet.h
index 9cb4c7884201..26e9afcbdd50 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_enet.h
+++ b/drivers/net/ethernet/hisilicon/hns/hns_enet.h
@@ -38,7 +38,7 @@ struct hns_nic_ring_data {
 	struct hnae_ring *ring;
 	struct napi_struct napi;
 	cpumask_t mask; /* affinity mask */
-	int queue_index;
+	u32 queue_index;
 	int (*poll_one)(struct hns_nic_ring_data *, int, void *);
 	void (*ex_process)(struct hns_nic_ring_data *, struct sk_buff *);
 	bool (*fini_process)(struct hns_nic_ring_data *);
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c b/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c
index a8db27e86a11..7ea7f8a4aa2a 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c
@@ -595,7 +595,7 @@ static void hns_nic_self_test(struct net_device *ndev,
 		set_bit(NIC_STATE_TESTING, &priv->state);
 
 		if (if_running)
-			(void)dev_close(ndev);
+			dev_close(ndev);
 
 		for (i = 0; i < SELF_TEST_TPYE_NUM; i++) {
 			if (!st_param[i][1])
@@ -735,8 +735,8 @@ static int hns_get_coalesce(struct net_device *net_dev,
 
 	ops = priv->ae_handle->dev->ops;
 
-	ec->use_adaptive_rx_coalesce = 1;
-	ec->use_adaptive_tx_coalesce = 1;
+	ec->use_adaptive_rx_coalesce = priv->ae_handle->coal_adapt_en;
+	ec->use_adaptive_tx_coalesce = priv->ae_handle->coal_adapt_en;
 
 	if ((!ops->get_coalesce_usecs) ||
 	    (!ops->get_max_coalesced_frames))
@@ -787,6 +787,9 @@ static int hns_set_coalesce(struct net_device *net_dev,
 	    (!ops->set_coalesce_frames))
 		return -ESRCH;
 
+	if (ec->use_adaptive_rx_coalesce != priv->ae_handle->coal_adapt_en)
+		priv->ae_handle->coal_adapt_en = ec->use_adaptive_rx_coalesce;
+
 	rc1 = ops->set_coalesce_usecs(priv->ae_handle,
 				      ec->rx_coalesce_usecs);
 
diff --git a/drivers/net/ethernet/hisilicon/hns3/Makefile b/drivers/net/ethernet/hisilicon/hns3/Makefile
new file mode 100644
index 000000000000..a9349e1f3e51
--- /dev/null
+++ b/drivers/net/ethernet/hisilicon/hns3/Makefile
@@ -0,0 +1,7 @@
+#
+# Makefile for the HISILICON network device drivers.
+#
+
+obj-$(CONFIG_HNS3) += hns3pf/
+
+obj-$(CONFIG_HNS3) += hnae3.o
diff --git a/drivers/net/ethernet/hisilicon/hns3/hnae3.c b/drivers/net/ethernet/hisilicon/hns3/hnae3.c
new file mode 100644
index 000000000000..59efbd605416
--- /dev/null
+++ b/drivers/net/ethernet/hisilicon/hns3/hnae3.c
@@ -0,0 +1,300 @@
+/*
+ * Copyright (c) 2016-2017 Hisilicon Limited.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+
+#include "hnae3.h"
+
+static LIST_HEAD(hnae3_ae_algo_list);
+static LIST_HEAD(hnae3_client_list);
+static LIST_HEAD(hnae3_ae_dev_list);
+
+/* we are keeping things simple and using single lock for all the
+ * list. This is a non-critical code so other updations, if happen
+ * in parallel, can wait.
+ */
+static DEFINE_MUTEX(hnae3_common_lock);
+
+static bool hnae3_client_match(enum hnae3_client_type client_type,
+			       enum hnae3_dev_type dev_type)
+{
+	if ((dev_type == HNAE3_DEV_KNIC) && (client_type == HNAE3_CLIENT_KNIC ||
+					     client_type == HNAE3_CLIENT_ROCE))
+		return true;
+
+	if (dev_type == HNAE3_DEV_UNIC && client_type == HNAE3_CLIENT_UNIC)
+		return true;
+
+	return false;
+}
+
+static int hnae3_match_n_instantiate(struct hnae3_client *client,
+				     struct hnae3_ae_dev *ae_dev,
+				     bool is_reg, bool *matched)
+{
+	int ret;
+
+	*matched = false;
+
+	/* check if this client matches the type of ae_dev */
+	if (!(hnae3_client_match(client->type, ae_dev->dev_type) &&
+	      hnae_get_bit(ae_dev->flag, HNAE3_DEV_INITED_B))) {
+		return 0;
+	}
+	/* there is a match of client and dev */
+	*matched = true;
+
+	/* now, (un-)instantiate client by calling lower layer */
+	if (is_reg) {
+		ret = ae_dev->ops->init_client_instance(client, ae_dev);
+		if (ret)
+			dev_err(&ae_dev->pdev->dev,
+				"fail to instantiate client\n");
+		return ret;
+	}
+
+	ae_dev->ops->uninit_client_instance(client, ae_dev);
+	return 0;
+}
+
+int hnae3_register_client(struct hnae3_client *client)
+{
+	struct hnae3_client *client_tmp;
+	struct hnae3_ae_dev *ae_dev;
+	bool matched;
+	int ret = 0;
+
+	mutex_lock(&hnae3_common_lock);
+	/* one system should only have one client for every type */
+	list_for_each_entry(client_tmp, &hnae3_client_list, node) {
+		if (client_tmp->type == client->type)
+			goto exit;
+	}
+
+	list_add_tail(&client->node, &hnae3_client_list);
+
+	/* initialize the client on every matched port */
+	list_for_each_entry(ae_dev, &hnae3_ae_dev_list, node) {
+		/* if the client could not be initialized on current port, for
+		 * any error reasons, move on to next available port
+		 */
+		ret = hnae3_match_n_instantiate(client, ae_dev, true, &matched);
+		if (ret)
+			dev_err(&ae_dev->pdev->dev,
+				"match and instantiation failed for port\n");
+	}
+
+exit:
+	mutex_unlock(&hnae3_common_lock);
+
+	return ret;
+}
+EXPORT_SYMBOL(hnae3_register_client);
+
+void hnae3_unregister_client(struct hnae3_client *client)
+{
+	struct hnae3_ae_dev *ae_dev;
+	bool matched;
+
+	mutex_lock(&hnae3_common_lock);
+	/* un-initialize the client on every matched port */
+	list_for_each_entry(ae_dev, &hnae3_ae_dev_list, node) {
+		hnae3_match_n_instantiate(client, ae_dev, false, &matched);
+	}
+
+	list_del(&client->node);
+	mutex_unlock(&hnae3_common_lock);
+}
+EXPORT_SYMBOL(hnae3_unregister_client);
+
+/* hnae3_register_ae_algo - register a AE algorithm to hnae3 framework
+ * @ae_algo: AE algorithm
+ * NOTE: the duplicated name will not be checked
+ */
+int hnae3_register_ae_algo(struct hnae3_ae_algo *ae_algo)
+{
+	const struct pci_device_id *id;
+	struct hnae3_ae_dev *ae_dev;
+	struct hnae3_client *client;
+	bool matched;
+	int ret = 0;
+
+	mutex_lock(&hnae3_common_lock);
+
+	list_add_tail(&ae_algo->node, &hnae3_ae_algo_list);
+
+	/* Check if this algo/ops matches the list of ae_devs */
+	list_for_each_entry(ae_dev, &hnae3_ae_dev_list, node) {
+		id = pci_match_id(ae_algo->pdev_id_table, ae_dev->pdev);
+		if (!id)
+			continue;
+
+		/* ae_dev init should set flag */
+		ae_dev->ops = ae_algo->ops;
+		ret = ae_algo->ops->init_ae_dev(ae_dev);
+		if (ret) {
+			dev_err(&ae_dev->pdev->dev, "init ae_dev error.\n");
+			continue;
+		}
+
+		hnae_set_bit(ae_dev->flag, HNAE3_DEV_INITED_B, 1);
+
+		/* check the client list for the match with this ae_dev type and
+		 * initialize the figure out client instance
+		 */
+		list_for_each_entry(client, &hnae3_client_list, node) {
+			ret = hnae3_match_n_instantiate(client, ae_dev, true,
+							&matched);
+			if (ret)
+				dev_err(&ae_dev->pdev->dev,
+					"match and instantiation failed\n");
+			if (matched)
+				break;
+		}
+	}
+
+	mutex_unlock(&hnae3_common_lock);
+
+	return ret;
+}
+EXPORT_SYMBOL(hnae3_register_ae_algo);
+
+/* hnae3_unregister_ae_algo - unregisters a AE algorithm
+ * @ae_algo: the AE algorithm to unregister
+ */
+void hnae3_unregister_ae_algo(struct hnae3_ae_algo *ae_algo)
+{
+	const struct pci_device_id *id;
+	struct hnae3_ae_dev *ae_dev;
+	struct hnae3_client *client;
+	bool matched;
+
+	mutex_lock(&hnae3_common_lock);
+	/* Check if there are matched ae_dev */
+	list_for_each_entry(ae_dev, &hnae3_ae_dev_list, node) {
+		id = pci_match_id(ae_algo->pdev_id_table, ae_dev->pdev);
+		if (!id)
+			continue;
+
+		/* check the client list for the match with this ae_dev type and
+		 * un-initialize the figure out client instance
+		 */
+		list_for_each_entry(client, &hnae3_client_list, node) {
+			hnae3_match_n_instantiate(client, ae_dev, false,
+						  &matched);
+			if (matched)
+				break;
+		}
+
+		ae_algo->ops->uninit_ae_dev(ae_dev);
+		hnae_set_bit(ae_dev->flag, HNAE3_DEV_INITED_B, 0);
+	}
+
+	list_del(&ae_algo->node);
+	mutex_unlock(&hnae3_common_lock);
+}
+EXPORT_SYMBOL(hnae3_unregister_ae_algo);
+
+/* hnae3_register_ae_dev - registers a AE device to hnae3 framework
+ * @ae_dev: the AE device
+ * NOTE: the duplicated name will not be checked
+ */
+int hnae3_register_ae_dev(struct hnae3_ae_dev *ae_dev)
+{
+	const struct pci_device_id *id;
+	struct hnae3_ae_algo *ae_algo;
+	struct hnae3_client *client;
+	bool matched;
+	int ret = 0;
+
+	mutex_lock(&hnae3_common_lock);
+	list_add_tail(&ae_dev->node, &hnae3_ae_dev_list);
+
+	/* Check if there are matched ae_algo */
+	list_for_each_entry(ae_algo, &hnae3_ae_algo_list, node) {
+		id = pci_match_id(ae_algo->pdev_id_table, ae_dev->pdev);
+		if (!id)
+			continue;
+
+		ae_dev->ops = ae_algo->ops;
+
+		if (!ae_dev->ops) {
+			dev_err(&ae_dev->pdev->dev, "ae_dev ops are null\n");
+			goto out_err;
+		}
+
+		/* ae_dev init should set flag */
+		ret = ae_dev->ops->init_ae_dev(ae_dev);
+		if (ret) {
+			dev_err(&ae_dev->pdev->dev, "init ae_dev error\n");
+			goto out_err;
+		}
+
+		hnae_set_bit(ae_dev->flag, HNAE3_DEV_INITED_B, 1);
+		break;
+	}
+
+	/* check the client list for the match with this ae_dev type and
+	 * initialize the figure out client instance
+	 */
+	list_for_each_entry(client, &hnae3_client_list, node) {
+		ret = hnae3_match_n_instantiate(client, ae_dev, true,
+						&matched);
+		if (ret)
+			dev_err(&ae_dev->pdev->dev,
+				"match and instantiation failed\n");
+		if (matched)
+			break;
+	}
+
+out_err:
+	mutex_unlock(&hnae3_common_lock);
+
+	return ret;
+}
+EXPORT_SYMBOL(hnae3_register_ae_dev);
+
+/* hnae3_unregister_ae_dev - unregisters a AE device
+ * @ae_dev: the AE device to unregister
+ */
+void hnae3_unregister_ae_dev(struct hnae3_ae_dev *ae_dev)
+{
+	const struct pci_device_id *id;
+	struct hnae3_ae_algo *ae_algo;
+	struct hnae3_client *client;
+	bool matched;
+
+	mutex_lock(&hnae3_common_lock);
+	/* Check if there are matched ae_algo */
+	list_for_each_entry(ae_algo, &hnae3_ae_algo_list, node) {
+		id = pci_match_id(ae_algo->pdev_id_table, ae_dev->pdev);
+		if (!id)
+			continue;
+
+		list_for_each_entry(client, &hnae3_client_list, node) {
+			hnae3_match_n_instantiate(client, ae_dev, false,
+						  &matched);
+			if (matched)
+				break;
+		}
+
+		ae_algo->ops->uninit_ae_dev(ae_dev);
+		hnae_set_bit(ae_dev->flag, HNAE3_DEV_INITED_B, 0);
+	}
+
+	list_del(&ae_dev->node);
+	mutex_unlock(&hnae3_common_lock);
+}
+EXPORT_SYMBOL(hnae3_unregister_ae_dev);
+
+MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("HNAE3(Hisilicon Network Acceleration Engine) Framework");
diff --git a/drivers/net/ethernet/hisilicon/hns3/hnae3.h b/drivers/net/ethernet/hisilicon/hns3/hnae3.h
new file mode 100644
index 000000000000..b2f28ae81273
--- /dev/null
+++ b/drivers/net/ethernet/hisilicon/hns3/hnae3.h
@@ -0,0 +1,444 @@
+/*
+ * Copyright (c) 2016-2017 Hisilicon Limited.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef __HNAE3_H
+#define __HNAE3_H
+
+/* Names used in this framework:
+ *      ae handle (handle):
+ *        a set of queues provided by AE
+ *      ring buffer queue (rbq):
+ *        the channel between upper layer and the AE, can do tx and rx
+ *      ring:
+ *        a tx or rx channel within a rbq
+ *      ring description (desc):
+ *        an element in the ring with packet information
+ *      buffer:
+ *        a memory region referred by desc with the full packet payload
+ *
+ * "num" means a static number set as a parameter, "count" mean a dynamic
+ *   number set while running
+ * "cb" means control block
+ */
+
+#include <linux/acpi.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/pci.h>
+#include <linux/types.h>
+
+/* Device IDs */
+#define HNAE3_DEV_ID_GE				0xA220
+#define HNAE3_DEV_ID_25GE			0xA221
+#define HNAE3_DEV_ID_25GE_RDMA			0xA222
+#define HNAE3_DEV_ID_25GE_RDMA_MACSEC		0xA223
+#define HNAE3_DEV_ID_50GE_RDMA			0xA224
+#define HNAE3_DEV_ID_50GE_RDMA_MACSEC		0xA225
+#define HNAE3_DEV_ID_100G_RDMA_MACSEC		0xA226
+#define HNAE3_DEV_ID_100G_VF			0xA22E
+#define HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF	0xA22F
+
+#define HNAE3_CLASS_NAME_SIZE 16
+
+#define HNAE3_DEV_INITED_B			0x0
+#define HNAE_DEV_SUPPORT_ROCE_B			0x1
+
+#define ring_ptr_move_fw(ring, p) \
+	((ring)->p = ((ring)->p + 1) % (ring)->desc_num)
+#define ring_ptr_move_bw(ring, p) \
+	((ring)->p = ((ring)->p - 1 + (ring)->desc_num) % (ring)->desc_num)
+
+enum hns_desc_type {
+	DESC_TYPE_SKB,
+	DESC_TYPE_PAGE,
+};
+
+struct hnae3_handle;
+
+struct hnae3_queue {
+	void __iomem *io_base;
+	struct hnae3_ae_algo *ae_algo;
+	struct hnae3_handle *handle;
+	int tqp_index;	/* index in a handle */
+	u32 buf_size;	/* size for hnae_desc->addr, preset by AE */
+	u16 desc_num;	/* total number of desc */
+};
+
+/*hnae3 loop mode*/
+enum hnae3_loop {
+	HNAE3_MAC_INTER_LOOP_MAC,
+	HNAE3_MAC_INTER_LOOP_SERDES,
+	HNAE3_MAC_INTER_LOOP_PHY,
+	HNAE3_MAC_LOOP_NONE,
+};
+
+enum hnae3_client_type {
+	HNAE3_CLIENT_KNIC,
+	HNAE3_CLIENT_UNIC,
+	HNAE3_CLIENT_ROCE,
+};
+
+enum hnae3_dev_type {
+	HNAE3_DEV_KNIC,
+	HNAE3_DEV_UNIC,
+};
+
+/* mac media type */
+enum hnae3_media_type {
+	HNAE3_MEDIA_TYPE_UNKNOWN,
+	HNAE3_MEDIA_TYPE_FIBER,
+	HNAE3_MEDIA_TYPE_COPPER,
+	HNAE3_MEDIA_TYPE_BACKPLANE,
+};
+
+struct hnae3_vector_info {
+	u8 __iomem *io_addr;
+	int vector;
+};
+
+#define HNAE3_RING_TYPE_B 0
+#define HNAE3_RING_TYPE_TX 0
+#define HNAE3_RING_TYPE_RX 1
+
+struct hnae3_ring_chain_node {
+	struct hnae3_ring_chain_node *next;
+	u32 tqp_index;
+	u32 flag;
+};
+
+#define HNAE3_IS_TX_RING(node) \
+	(((node)->flag & (1 << HNAE3_RING_TYPE_B)) == HNAE3_RING_TYPE_TX)
+
+struct hnae3_client_ops {
+	int (*init_instance)(struct hnae3_handle *handle);
+	void (*uninit_instance)(struct hnae3_handle *handle, bool reset);
+	void (*link_status_change)(struct hnae3_handle *handle, bool state);
+};
+
+#define HNAE3_CLIENT_NAME_LENGTH 16
+struct hnae3_client {
+	char name[HNAE3_CLIENT_NAME_LENGTH];
+	u16 version;
+	unsigned long state;
+	enum hnae3_client_type type;
+	const struct hnae3_client_ops *ops;
+	struct list_head node;
+};
+
+struct hnae3_ae_dev {
+	struct pci_dev *pdev;
+	const struct hnae3_ae_ops *ops;
+	struct list_head node;
+	u32 flag;
+	enum hnae3_dev_type dev_type;
+	void *priv;
+};
+
+/* This struct defines the operation on the handle.
+ *
+ * init_ae_dev(): (mandatory)
+ *   Get PF configure from pci_dev and initialize PF hardware
+ * uninit_ae_dev()
+ *   Disable PF device and release PF resource
+ * register_client
+ *   Register client to ae_dev
+ * unregister_client()
+ *   Unregister client from ae_dev
+ * start()
+ *   Enable the hardware
+ * stop()
+ *   Disable the hardware
+ * get_status()
+ *   Get the carrier state of the back channel of the handle, 1 for ok, 0 for
+ *   non-ok
+ * get_ksettings_an_result()
+ *   Get negotiation status,speed and duplex
+ * update_speed_duplex_h()
+ *   Update hardware speed and duplex
+ * get_media_type()
+ *   Get media type of MAC
+ * adjust_link()
+ *   Adjust link status
+ * set_loopback()
+ *   Set loopback
+ * set_promisc_mode
+ *   Set promisc mode
+ * set_mtu()
+ *   set mtu
+ * get_pauseparam()
+ *   get tx and rx of pause frame use
+ * set_pauseparam()
+ *   set tx and rx of pause frame use
+ * set_autoneg()
+ *   set auto autonegotiation of pause frame use
+ * get_autoneg()
+ *   get auto autonegotiation of pause frame use
+ * get_coalesce_usecs()
+ *   get usecs to delay a TX interrupt after a packet is sent
+ * get_rx_max_coalesced_frames()
+ *   get Maximum number of packets to be sent before a TX interrupt.
+ * set_coalesce_usecs()
+ *   set usecs to delay a TX interrupt after a packet is sent
+ * set_coalesce_frames()
+ *   set Maximum number of packets to be sent before a TX interrupt.
+ * get_mac_addr()
+ *   get mac address
+ * set_mac_addr()
+ *   set mac address
+ * add_uc_addr
+ *   Add unicast addr to mac table
+ * rm_uc_addr
+ *   Remove unicast addr from mac table
+ * set_mc_addr()
+ *   Set multicast address
+ * add_mc_addr
+ *   Add multicast address to mac table
+ * rm_mc_addr
+ *   Remove multicast address from mac table
+ * update_stats()
+ *   Update Old network device statistics
+ * get_ethtool_stats()
+ *   Get ethtool network device statistics
+ * get_strings()
+ *   Get a set of strings that describe the requested objects
+ * get_sset_count()
+ *   Get number of strings that @get_strings will write
+ * update_led_status()
+ *   Update the led status
+ * set_led_id()
+ *   Set led id
+ * get_regs()
+ *   Get regs dump
+ * get_regs_len()
+ *   Get the len of the regs dump
+ * get_rss_key_size()
+ *   Get rss key size
+ * get_rss_indir_size()
+ *   Get rss indirection table size
+ * get_rss()
+ *   Get rss table
+ * set_rss()
+ *   Set rss table
+ * get_tc_size()
+ *   Get tc size of handle
+ * get_vector()
+ *   Get vector number and vector information
+ * map_ring_to_vector()
+ *   Map rings to vector
+ * unmap_ring_from_vector()
+ *   Unmap rings from vector
+ * add_tunnel_udp()
+ *   Add tunnel information to hardware
+ * del_tunnel_udp()
+ *   Delete tunnel information from hardware
+ * reset_queue()
+ *   Reset queue
+ * get_fw_version()
+ *   Get firmware version
+ * get_mdix_mode()
+ *   Get media typr of phy
+ * set_vlan_filter()
+ *   Set vlan filter config of Ports
+ * set_vf_vlan_filter()
+ *   Set vlan filter config of vf
+ */
+struct hnae3_ae_ops {
+	int (*init_ae_dev)(struct hnae3_ae_dev *ae_dev);
+	void (*uninit_ae_dev)(struct hnae3_ae_dev *ae_dev);
+
+	int (*init_client_instance)(struct hnae3_client *client,
+				    struct hnae3_ae_dev *ae_dev);
+	void (*uninit_client_instance)(struct hnae3_client *client,
+				       struct hnae3_ae_dev *ae_dev);
+	int (*start)(struct hnae3_handle *handle);
+	void (*stop)(struct hnae3_handle *handle);
+	int (*get_status)(struct hnae3_handle *handle);
+	void (*get_ksettings_an_result)(struct hnae3_handle *handle,
+					u8 *auto_neg, u32 *speed, u8 *duplex);
+
+	int (*update_speed_duplex_h)(struct hnae3_handle *handle);
+	int (*cfg_mac_speed_dup_h)(struct hnae3_handle *handle, int speed,
+				   u8 duplex);
+
+	void (*get_media_type)(struct hnae3_handle *handle, u8 *media_type);
+	void (*adjust_link)(struct hnae3_handle *handle, int speed, int duplex);
+	int (*set_loopback)(struct hnae3_handle *handle,
+			    enum hnae3_loop loop_mode, bool en);
+
+	void (*set_promisc_mode)(struct hnae3_handle *handle, u32 en);
+	int (*set_mtu)(struct hnae3_handle *handle, int new_mtu);
+
+	void (*get_pauseparam)(struct hnae3_handle *handle,
+			       u32 *auto_neg, u32 *rx_en, u32 *tx_en);
+	int (*set_pauseparam)(struct hnae3_handle *handle,
+			      u32 auto_neg, u32 rx_en, u32 tx_en);
+
+	int (*set_autoneg)(struct hnae3_handle *handle, bool enable);
+	int (*get_autoneg)(struct hnae3_handle *handle);
+
+	void (*get_coalesce_usecs)(struct hnae3_handle *handle,
+				   u32 *tx_usecs, u32 *rx_usecs);
+	void (*get_rx_max_coalesced_frames)(struct hnae3_handle *handle,
+					    u32 *tx_frames, u32 *rx_frames);
+	int (*set_coalesce_usecs)(struct hnae3_handle *handle, u32 timeout);
+	int (*set_coalesce_frames)(struct hnae3_handle *handle,
+				   u32 coalesce_frames);
+	void (*get_coalesce_range)(struct hnae3_handle *handle,
+				   u32 *tx_frames_low, u32 *rx_frames_low,
+				   u32 *tx_frames_high, u32 *rx_frames_high,
+				   u32 *tx_usecs_low, u32 *rx_usecs_low,
+				   u32 *tx_usecs_high, u32 *rx_usecs_high);
+
+	void (*get_mac_addr)(struct hnae3_handle *handle, u8 *p);
+	int (*set_mac_addr)(struct hnae3_handle *handle, void *p);
+	int (*add_uc_addr)(struct hnae3_handle *handle,
+			   const unsigned char *addr);
+	int (*rm_uc_addr)(struct hnae3_handle *handle,
+			  const unsigned char *addr);
+	int (*set_mc_addr)(struct hnae3_handle *handle, void *addr);
+	int (*add_mc_addr)(struct hnae3_handle *handle,
+			   const unsigned char *addr);
+	int (*rm_mc_addr)(struct hnae3_handle *handle,
+			  const unsigned char *addr);
+
+	void (*set_tso_stats)(struct hnae3_handle *handle, int enable);
+	void (*update_stats)(struct hnae3_handle *handle,
+			     struct net_device_stats *net_stats);
+	void (*get_stats)(struct hnae3_handle *handle, u64 *data);
+
+	void (*get_strings)(struct hnae3_handle *handle,
+			    u32 stringset, u8 *data);
+	int (*get_sset_count)(struct hnae3_handle *handle, int stringset);
+
+	void (*get_regs)(struct hnae3_handle *handle, void *data);
+	int (*get_regs_len)(struct hnae3_handle *handle);
+
+	u32 (*get_rss_key_size)(struct hnae3_handle *handle);
+	u32 (*get_rss_indir_size)(struct hnae3_handle *handle);
+	int (*get_rss)(struct hnae3_handle *handle, u32 *indir, u8 *key,
+		       u8 *hfunc);
+	int (*set_rss)(struct hnae3_handle *handle, const u32 *indir,
+		       const u8 *key, const u8 hfunc);
+
+	int (*get_tc_size)(struct hnae3_handle *handle);
+
+	int (*get_vector)(struct hnae3_handle *handle, u16 vector_num,
+			  struct hnae3_vector_info *vector_info);
+	int (*map_ring_to_vector)(struct hnae3_handle *handle,
+				  int vector_num,
+				  struct hnae3_ring_chain_node *vr_chain);
+	int (*unmap_ring_from_vector)(struct hnae3_handle *handle,
+				      int vector_num,
+				      struct hnae3_ring_chain_node *vr_chain);
+
+	int (*add_tunnel_udp)(struct hnae3_handle *handle, u16 port_num);
+	int (*del_tunnel_udp)(struct hnae3_handle *handle, u16 port_num);
+
+	void (*reset_queue)(struct hnae3_handle *handle, u16 queue_id);
+	u32 (*get_fw_version)(struct hnae3_handle *handle);
+	void (*get_mdix_mode)(struct hnae3_handle *handle,
+			      u8 *tp_mdix_ctrl, u8 *tp_mdix);
+
+	int (*set_vlan_filter)(struct hnae3_handle *handle, __be16 proto,
+			       u16 vlan_id, bool is_kill);
+	int (*set_vf_vlan_filter)(struct hnae3_handle *handle, int vfid,
+				  u16 vlan, u8 qos, __be16 proto);
+};
+
+struct hnae3_ae_algo {
+	const struct hnae3_ae_ops *ops;
+	struct list_head node;
+	char name[HNAE3_CLASS_NAME_SIZE];
+	const struct pci_device_id *pdev_id_table;
+};
+
+#define HNAE3_INT_NAME_LEN        (IFNAMSIZ + 16)
+#define HNAE3_ITR_COUNTDOWN_START 100
+
+struct hnae3_tc_info {
+	u16	tqp_offset;	/* TQP offset from base TQP */
+	u16	tqp_count;	/* Total TQPs */
+	u8	up;		/* user priority */
+	u8	tc;		/* TC index */
+	bool	enable;		/* If this TC is enable or not */
+};
+
+#define HNAE3_MAX_TC		8
+struct hnae3_knic_private_info {
+	struct net_device *netdev; /* Set by KNIC client when init instance */
+	u16 rss_size;		   /* Allocated RSS queues */
+	u16 rx_buf_len;
+	u16 num_desc;
+
+	u8 num_tc;		   /* Total number of enabled TCs */
+	struct hnae3_tc_info tc_info[HNAE3_MAX_TC]; /* Idx of array is HW TC */
+
+	u16 num_tqps;		  /* total number of TQPs in this handle */
+	struct hnae3_queue **tqp;  /* array base of all TQPs in this instance */
+};
+
+struct hnae3_roce_private_info {
+	struct net_device *netdev;
+	void __iomem *roce_io_base;
+	int base_vector;
+	int num_vectors;
+};
+
+struct hnae3_unic_private_info {
+	struct net_device *netdev;
+	u16 rx_buf_len;
+	u16 num_desc;
+	u16 num_tqps;	/* total number of tqps in this handle */
+	struct hnae3_queue **tqp;  /* array base of all TQPs of this instance */
+};
+
+#define HNAE3_SUPPORT_MAC_LOOPBACK    1
+#define HNAE3_SUPPORT_PHY_LOOPBACK    2
+#define HNAE3_SUPPORT_SERDES_LOOPBACK 4
+
+struct hnae3_handle {
+	struct hnae3_client *client;
+	struct pci_dev *pdev;
+	void *priv;
+	struct hnae3_ae_algo *ae_algo;  /* the class who provides this handle */
+	u64 flags; /* Indicate the capabilities for this handle*/
+
+	union {
+		struct net_device *netdev; /* first member */
+		struct hnae3_knic_private_info kinfo;
+		struct hnae3_unic_private_info uinfo;
+		struct hnae3_roce_private_info rinfo;
+	};
+
+	u32 numa_node_mask;	/* for multi-chip support */
+};
+
+#define hnae_set_field(origin, mask, shift, val) \
+	do { \
+		(origin) &= (~(mask)); \
+		(origin) |= ((val) << (shift)) & (mask); \
+	} while (0)
+#define hnae_get_field(origin, mask, shift) (((origin) & (mask)) >> (shift))
+
+#define hnae_set_bit(origin, shift, val) \
+	hnae_set_field((origin), (0x1 << (shift)), (shift), (val))
+#define hnae_get_bit(origin, shift) \
+	hnae_get_field((origin), (0x1 << (shift)), (shift))
+
+int hnae3_register_ae_dev(struct hnae3_ae_dev *ae_dev);
+void hnae3_unregister_ae_dev(struct hnae3_ae_dev *ae_dev);
+
+void hnae3_unregister_ae_algo(struct hnae3_ae_algo *ae_algo);
+int hnae3_register_ae_algo(struct hnae3_ae_algo *ae_algo);
+
+void hnae3_unregister_client(struct hnae3_client *client);
+int hnae3_register_client(struct hnae3_client *client);
+#endif
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/Makefile b/drivers/net/ethernet/hisilicon/hns3/hns3pf/Makefile
new file mode 100644
index 000000000000..162e8a42acd0
--- /dev/null
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/Makefile
@@ -0,0 +1,11 @@
+#
+# Makefile for the HISILICON network device drivers.
+#
+
+ccflags-y := -Idrivers/net/ethernet/hisilicon/hns3
+
+obj-$(CONFIG_HNS3_HCLGE) += hclge.o
+hclge-objs = hclge_main.o hclge_cmd.o hclge_mdio.o hclge_tm.o
+
+obj-$(CONFIG_HNS3_ENET) += hns3.o
+hns3-objs = hns3_enet.o hns3_ethtool.o
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.c
new file mode 100644
index 000000000000..8b511e6e0ce9
--- /dev/null
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.c
@@ -0,0 +1,356 @@
+/*
+ * Copyright (c) 2016~2017 Hisilicon Limited.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/dma-mapping.h>
+#include <linux/slab.h>
+#include <linux/pci.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/dma-direction.h>
+#include "hclge_cmd.h"
+#include "hnae3.h"
+#include "hclge_main.h"
+
+#define hclge_is_csq(ring) ((ring)->flag & HCLGE_TYPE_CSQ)
+#define hclge_ring_to_dma_dir(ring) (hclge_is_csq(ring) ? \
+	DMA_TO_DEVICE : DMA_FROM_DEVICE)
+#define cmq_ring_to_dev(ring)   (&(ring)->dev->pdev->dev)
+
+static int hclge_ring_space(struct hclge_cmq_ring *ring)
+{
+	int ntu = ring->next_to_use;
+	int ntc = ring->next_to_clean;
+	int used = (ntu - ntc + ring->desc_num) % ring->desc_num;
+
+	return ring->desc_num - used - 1;
+}
+
+static int hclge_alloc_cmd_desc(struct hclge_cmq_ring *ring)
+{
+	int size  = ring->desc_num * sizeof(struct hclge_desc);
+
+	ring->desc = kzalloc(size, GFP_KERNEL);
+	if (!ring->desc)
+		return -ENOMEM;
+
+	ring->desc_dma_addr = dma_map_single(cmq_ring_to_dev(ring), ring->desc,
+					     size, DMA_BIDIRECTIONAL);
+	if (dma_mapping_error(cmq_ring_to_dev(ring), ring->desc_dma_addr)) {
+		ring->desc_dma_addr = 0;
+		kfree(ring->desc);
+		ring->desc = NULL;
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static void hclge_free_cmd_desc(struct hclge_cmq_ring *ring)
+{
+	dma_unmap_single(cmq_ring_to_dev(ring), ring->desc_dma_addr,
+			 ring->desc_num * sizeof(ring->desc[0]),
+			 DMA_BIDIRECTIONAL);
+
+	ring->desc_dma_addr = 0;
+	kfree(ring->desc);
+	ring->desc = NULL;
+}
+
+static int hclge_init_cmd_queue(struct hclge_dev *hdev, int ring_type)
+{
+	struct hclge_hw *hw = &hdev->hw;
+	struct hclge_cmq_ring *ring =
+		(ring_type == HCLGE_TYPE_CSQ) ? &hw->cmq.csq : &hw->cmq.crq;
+	int ret;
+
+	ring->flag = ring_type;
+	ring->dev = hdev;
+
+	ret = hclge_alloc_cmd_desc(ring);
+	if (ret) {
+		dev_err(&hdev->pdev->dev, "descriptor %s alloc error %d\n",
+			(ring_type == HCLGE_TYPE_CSQ) ? "CSQ" : "CRQ", ret);
+		return ret;
+	}
+
+	ring->next_to_clean = 0;
+	ring->next_to_use = 0;
+
+	return 0;
+}
+
+void hclge_cmd_setup_basic_desc(struct hclge_desc *desc,
+				enum hclge_opcode_type opcode, bool is_read)
+{
+	memset((void *)desc, 0, sizeof(struct hclge_desc));
+	desc->opcode = cpu_to_le16(opcode);
+	desc->flag = cpu_to_le16(HCLGE_CMD_FLAG_NO_INTR | HCLGE_CMD_FLAG_IN);
+
+	if (is_read)
+		desc->flag |= cpu_to_le16(HCLGE_CMD_FLAG_WR);
+	else
+		desc->flag &= cpu_to_le16(~HCLGE_CMD_FLAG_WR);
+}
+
+static void hclge_cmd_config_regs(struct hclge_cmq_ring *ring)
+{
+	dma_addr_t dma = ring->desc_dma_addr;
+	struct hclge_dev *hdev = ring->dev;
+	struct hclge_hw *hw = &hdev->hw;
+
+	if (ring->flag == HCLGE_TYPE_CSQ) {
+		hclge_write_dev(hw, HCLGE_NIC_CSQ_BASEADDR_L_REG,
+				(u32)dma);
+		hclge_write_dev(hw, HCLGE_NIC_CSQ_BASEADDR_H_REG,
+				(u32)((dma >> 31) >> 1));
+		hclge_write_dev(hw, HCLGE_NIC_CSQ_DEPTH_REG,
+				(ring->desc_num >> HCLGE_NIC_CMQ_DESC_NUM_S) |
+				HCLGE_NIC_CMQ_ENABLE);
+		hclge_write_dev(hw, HCLGE_NIC_CSQ_TAIL_REG, 0);
+		hclge_write_dev(hw, HCLGE_NIC_CSQ_HEAD_REG, 0);
+	} else {
+		hclge_write_dev(hw, HCLGE_NIC_CRQ_BASEADDR_L_REG,
+				(u32)dma);
+		hclge_write_dev(hw, HCLGE_NIC_CRQ_BASEADDR_H_REG,
+				(u32)((dma >> 31) >> 1));
+		hclge_write_dev(hw, HCLGE_NIC_CRQ_DEPTH_REG,
+				(ring->desc_num >> HCLGE_NIC_CMQ_DESC_NUM_S) |
+				HCLGE_NIC_CMQ_ENABLE);
+		hclge_write_dev(hw, HCLGE_NIC_CRQ_TAIL_REG, 0);
+		hclge_write_dev(hw, HCLGE_NIC_CRQ_HEAD_REG, 0);
+	}
+}
+
+static void hclge_cmd_init_regs(struct hclge_hw *hw)
+{
+	hclge_cmd_config_regs(&hw->cmq.csq);
+	hclge_cmd_config_regs(&hw->cmq.crq);
+}
+
+static int hclge_cmd_csq_clean(struct hclge_hw *hw)
+{
+	struct hclge_cmq_ring *csq = &hw->cmq.csq;
+	u16 ntc = csq->next_to_clean;
+	struct hclge_desc *desc;
+	int clean = 0;
+	u32 head;
+
+	desc = &csq->desc[ntc];
+	head = hclge_read_dev(hw, HCLGE_NIC_CSQ_HEAD_REG);
+
+	while (head != ntc) {
+		memset(desc, 0, sizeof(*desc));
+		ntc++;
+		if (ntc == csq->desc_num)
+			ntc = 0;
+		desc = &csq->desc[ntc];
+		clean++;
+	}
+	csq->next_to_clean = ntc;
+
+	return clean;
+}
+
+static int hclge_cmd_csq_done(struct hclge_hw *hw)
+{
+	u32 head = hclge_read_dev(hw, HCLGE_NIC_CSQ_HEAD_REG);
+	return head == hw->cmq.csq.next_to_use;
+}
+
+static bool hclge_is_special_opcode(u16 opcode)
+{
+	u16 spec_opcode[3] = {0x0030, 0x0031, 0x0032};
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(spec_opcode); i++) {
+		if (spec_opcode[i] == opcode)
+			return true;
+	}
+
+	return false;
+}
+
+/**
+ * hclge_cmd_send - send command to command queue
+ * @hw: pointer to the hw struct
+ * @desc: prefilled descriptor for describing the command
+ * @num : the number of descriptors to be sent
+ *
+ * This is the main send command for command queue, it
+ * sends the queue, cleans the queue, etc
+ **/
+int hclge_cmd_send(struct hclge_hw *hw, struct hclge_desc *desc, int num)
+{
+	struct hclge_dev *hdev = (struct hclge_dev *)hw->back;
+	struct hclge_desc *desc_to_use;
+	bool complete = false;
+	u32 timeout = 0;
+	int handle = 0;
+	int retval = 0;
+	u16 opcode, desc_ret;
+	int ntc;
+
+	spin_lock_bh(&hw->cmq.csq.lock);
+
+	if (num > hclge_ring_space(&hw->cmq.csq)) {
+		spin_unlock_bh(&hw->cmq.csq.lock);
+		return -EBUSY;
+	}
+
+	/**
+	 * Record the location of desc in the ring for this time
+	 * which will be use for hardware to write back
+	 */
+	ntc = hw->cmq.csq.next_to_use;
+	opcode = desc[0].opcode;
+	while (handle < num) {
+		desc_to_use = &hw->cmq.csq.desc[hw->cmq.csq.next_to_use];
+		*desc_to_use = desc[handle];
+		(hw->cmq.csq.next_to_use)++;
+		if (hw->cmq.csq.next_to_use == hw->cmq.csq.desc_num)
+			hw->cmq.csq.next_to_use = 0;
+		handle++;
+	}
+
+	/* Write to hardware */
+	hclge_write_dev(hw, HCLGE_NIC_CSQ_TAIL_REG, hw->cmq.csq.next_to_use);
+
+	/**
+	 * If the command is sync, wait for the firmware to write back,
+	 * if multi descriptors to be sent, use the first one to check
+	 */
+	if (HCLGE_SEND_SYNC(desc->flag)) {
+		do {
+			if (hclge_cmd_csq_done(hw))
+				break;
+			udelay(1);
+			timeout++;
+		} while (timeout < hw->cmq.tx_timeout);
+	}
+
+	if (hclge_cmd_csq_done(hw)) {
+		complete = true;
+		handle = 0;
+		while (handle < num) {
+			/* Get the result of hardware write back */
+			desc_to_use = &hw->cmq.csq.desc[ntc];
+			desc[handle] = *desc_to_use;
+			pr_debug("Get cmd desc:\n");
+
+			if (likely(!hclge_is_special_opcode(opcode)))
+				desc_ret = desc[handle].retval;
+			else
+				desc_ret = desc[0].retval;
+
+			if ((enum hclge_cmd_return_status)desc_ret ==
+			    HCLGE_CMD_EXEC_SUCCESS)
+				retval = 0;
+			else
+				retval = -EIO;
+			hw->cmq.last_status = (enum hclge_cmd_status)desc_ret;
+			ntc++;
+			handle++;
+			if (ntc == hw->cmq.csq.desc_num)
+				ntc = 0;
+		}
+	}
+
+	if (!complete)
+		retval = -EAGAIN;
+
+	/* Clean the command send queue */
+	handle = hclge_cmd_csq_clean(hw);
+	if (handle != num) {
+		dev_warn(&hdev->pdev->dev,
+			 "cleaned %d, need to clean %d\n", handle, num);
+	}
+
+	spin_unlock_bh(&hw->cmq.csq.lock);
+
+	return retval;
+}
+
+enum hclge_cmd_status hclge_cmd_query_firmware_version(struct hclge_hw *hw,
+						       u32 *version)
+{
+	struct hclge_query_version *resp;
+	struct hclge_desc desc;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_QUERY_FW_VER, 1);
+	resp = (struct hclge_query_version *)desc.data;
+
+	ret = hclge_cmd_send(hw, &desc, 1);
+	if (!ret)
+		*version = le32_to_cpu(resp->firmware);
+
+	return ret;
+}
+
+int hclge_cmd_init(struct hclge_dev *hdev)
+{
+	u32 version;
+	int ret;
+
+	/* Setup the queue entries for use cmd queue */
+	hdev->hw.cmq.csq.desc_num = HCLGE_NIC_CMQ_DESC_NUM;
+	hdev->hw.cmq.crq.desc_num = HCLGE_NIC_CMQ_DESC_NUM;
+
+	/* Setup the lock for command queue */
+	spin_lock_init(&hdev->hw.cmq.csq.lock);
+	spin_lock_init(&hdev->hw.cmq.crq.lock);
+
+	/* Setup Tx write back timeout */
+	hdev->hw.cmq.tx_timeout = HCLGE_CMDQ_TX_TIMEOUT;
+
+	/* Setup queue rings */
+	ret = hclge_init_cmd_queue(hdev, HCLGE_TYPE_CSQ);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"CSQ ring setup error %d\n", ret);
+		return ret;
+	}
+
+	ret = hclge_init_cmd_queue(hdev, HCLGE_TYPE_CRQ);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"CRQ ring setup error %d\n", ret);
+		goto err_csq;
+	}
+
+	hclge_cmd_init_regs(&hdev->hw);
+
+	ret = hclge_cmd_query_firmware_version(&hdev->hw, &version);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"firmware version query failed %d\n", ret);
+		return ret;
+	}
+	hdev->fw_version = version;
+
+	dev_info(&hdev->pdev->dev, "The firmware version is %08x\n", version);
+
+	return 0;
+err_csq:
+	hclge_free_cmd_desc(&hdev->hw.cmq.csq);
+	return ret;
+}
+
+static void hclge_destroy_queue(struct hclge_cmq_ring *ring)
+{
+	spin_lock_bh(&ring->lock);
+	hclge_free_cmd_desc(ring);
+	spin_unlock_bh(&ring->lock);
+}
+
+void hclge_destroy_cmd_queue(struct hclge_hw *hw)
+{
+	hclge_destroy_queue(&hw->cmq.csq);
+	hclge_destroy_queue(&hw->cmq.crq);
+}
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h
new file mode 100644
index 000000000000..91ae0135ee50
--- /dev/null
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h
@@ -0,0 +1,740 @@
+/*
+ * Copyright (c) 2016~2017 Hisilicon Limited.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef __HCLGE_CMD_H
+#define __HCLGE_CMD_H
+#include <linux/types.h>
+#include <linux/io.h>
+
+#define HCLGE_CMDQ_TX_TIMEOUT		1000
+
+struct hclge_dev;
+struct hclge_desc {
+	__le16 opcode;
+
+#define HCLGE_CMDQ_RX_INVLD_B		0
+#define HCLGE_CMDQ_RX_OUTVLD_B		1
+
+	__le16 flag;
+	__le16 retval;
+	__le16 rsv;
+	__le32 data[6];
+};
+
+struct hclge_desc_cb {
+	dma_addr_t dma;
+	void *va;
+	u32 length;
+};
+
+struct hclge_cmq_ring {
+	dma_addr_t desc_dma_addr;
+	struct hclge_desc *desc;
+	struct hclge_desc_cb *desc_cb;
+	struct hclge_dev  *dev;
+	u32 head;
+	u32 tail;
+
+	u16 buf_size;
+	u16 desc_num;
+	int next_to_use;
+	int next_to_clean;
+	u8 flag;
+	spinlock_t lock; /* Command queue lock */
+};
+
+enum hclge_cmd_return_status {
+	HCLGE_CMD_EXEC_SUCCESS	= 0,
+	HCLGE_CMD_NO_AUTH	= 1,
+	HCLGE_CMD_NOT_EXEC	= 2,
+	HCLGE_CMD_QUEUE_FULL	= 3,
+};
+
+enum hclge_cmd_status {
+	HCLGE_STATUS_SUCCESS	= 0,
+	HCLGE_ERR_CSQ_FULL	= -1,
+	HCLGE_ERR_CSQ_TIMEOUT	= -2,
+	HCLGE_ERR_CSQ_ERROR	= -3,
+};
+
+struct hclge_cmq {
+	struct hclge_cmq_ring csq;
+	struct hclge_cmq_ring crq;
+	u16 tx_timeout; /* Tx timeout */
+	enum hclge_cmd_status last_status;
+};
+
+#define HCLGE_CMD_FLAG_IN_VALID_SHIFT	0
+#define HCLGE_CMD_FLAG_OUT_VALID_SHIFT	1
+#define HCLGE_CMD_FLAG_NEXT_SHIFT	2
+#define HCLGE_CMD_FLAG_WR_OR_RD_SHIFT	3
+#define HCLGE_CMD_FLAG_NO_INTR_SHIFT	4
+#define HCLGE_CMD_FLAG_ERR_INTR_SHIFT	5
+
+#define HCLGE_CMD_FLAG_IN	BIT(HCLGE_CMD_FLAG_IN_VALID_SHIFT)
+#define HCLGE_CMD_FLAG_OUT	BIT(HCLGE_CMD_FLAG_OUT_VALID_SHIFT)
+#define HCLGE_CMD_FLAG_NEXT	BIT(HCLGE_CMD_FLAG_NEXT_SHIFT)
+#define HCLGE_CMD_FLAG_WR	BIT(HCLGE_CMD_FLAG_WR_OR_RD_SHIFT)
+#define HCLGE_CMD_FLAG_NO_INTR	BIT(HCLGE_CMD_FLAG_NO_INTR_SHIFT)
+#define HCLGE_CMD_FLAG_ERR_INTR	BIT(HCLGE_CMD_FLAG_ERR_INTR_SHIFT)
+
+enum hclge_opcode_type {
+	/* Generic command */
+	HCLGE_OPC_QUERY_FW_VER		= 0x0001,
+	HCLGE_OPC_CFG_RST_TRIGGER	= 0x0020,
+	HCLGE_OPC_GBL_RST_STATUS	= 0x0021,
+	HCLGE_OPC_QUERY_FUNC_STATUS	= 0x0022,
+	HCLGE_OPC_QUERY_PF_RSRC		= 0x0023,
+	HCLGE_OPC_QUERY_VF_RSRC		= 0x0024,
+	HCLGE_OPC_GET_CFG_PARAM		= 0x0025,
+
+	HCLGE_OPC_STATS_64_BIT		= 0x0030,
+	HCLGE_OPC_STATS_32_BIT		= 0x0031,
+	HCLGE_OPC_STATS_MAC		= 0x0032,
+	/* Device management command */
+
+	/* MAC commond */
+	HCLGE_OPC_CONFIG_MAC_MODE	= 0x0301,
+	HCLGE_OPC_CONFIG_AN_MODE	= 0x0304,
+	HCLGE_OPC_QUERY_AN_RESULT	= 0x0306,
+	HCLGE_OPC_QUERY_LINK_STATUS	= 0x0307,
+	HCLGE_OPC_CONFIG_MAX_FRM_SIZE	= 0x0308,
+	HCLGE_OPC_CONFIG_SPEED_DUP	= 0x0309,
+	/* MACSEC command */
+
+	/* PFC/Pause CMD*/
+	HCLGE_OPC_CFG_MAC_PAUSE_EN      = 0x0701,
+	HCLGE_OPC_CFG_PFC_PAUSE_EN      = 0x0702,
+	HCLGE_OPC_CFG_MAC_PARA          = 0x0703,
+	HCLGE_OPC_CFG_PFC_PARA          = 0x0704,
+	HCLGE_OPC_QUERY_MAC_TX_PKT_CNT  = 0x0705,
+	HCLGE_OPC_QUERY_MAC_RX_PKT_CNT  = 0x0706,
+	HCLGE_OPC_QUERY_PFC_TX_PKT_CNT  = 0x0707,
+	HCLGE_OPC_QUERY_PFC_RX_PKT_CNT  = 0x0708,
+	HCLGE_OPC_PRI_TO_TC_MAPPING     = 0x0709,
+	HCLGE_OPC_QOS_MAP               = 0x070A,
+
+	/* ETS/scheduler commands */
+	HCLGE_OPC_TM_PG_TO_PRI_LINK	= 0x0804,
+	HCLGE_OPC_TM_QS_TO_PRI_LINK     = 0x0805,
+	HCLGE_OPC_TM_NQ_TO_QS_LINK      = 0x0806,
+	HCLGE_OPC_TM_RQ_TO_QS_LINK      = 0x0807,
+	HCLGE_OPC_TM_PORT_WEIGHT        = 0x0808,
+	HCLGE_OPC_TM_PG_WEIGHT          = 0x0809,
+	HCLGE_OPC_TM_QS_WEIGHT          = 0x080A,
+	HCLGE_OPC_TM_PRI_WEIGHT         = 0x080B,
+	HCLGE_OPC_TM_PRI_C_SHAPPING     = 0x080C,
+	HCLGE_OPC_TM_PRI_P_SHAPPING     = 0x080D,
+	HCLGE_OPC_TM_PG_C_SHAPPING      = 0x080E,
+	HCLGE_OPC_TM_PG_P_SHAPPING      = 0x080F,
+	HCLGE_OPC_TM_PORT_SHAPPING      = 0x0810,
+	HCLGE_OPC_TM_PG_SCH_MODE_CFG    = 0x0812,
+	HCLGE_OPC_TM_PRI_SCH_MODE_CFG   = 0x0813,
+	HCLGE_OPC_TM_QS_SCH_MODE_CFG    = 0x0814,
+	HCLGE_OPC_TM_BP_TO_QSET_MAPPING = 0x0815,
+
+	/* Packet buffer allocate command */
+	HCLGE_OPC_TX_BUFF_ALLOC		= 0x0901,
+	HCLGE_OPC_RX_PRIV_BUFF_ALLOC	= 0x0902,
+	HCLGE_OPC_RX_PRIV_WL_ALLOC	= 0x0903,
+	HCLGE_OPC_RX_COM_THRD_ALLOC	= 0x0904,
+	HCLGE_OPC_RX_COM_WL_ALLOC	= 0x0905,
+	HCLGE_OPC_RX_GBL_PKT_CNT	= 0x0906,
+
+	/* PTP command */
+	/* TQP management command */
+	HCLGE_OPC_SET_TQP_MAP		= 0x0A01,
+
+	/* TQP command */
+	HCLGE_OPC_CFG_TX_QUEUE		= 0x0B01,
+	HCLGE_OPC_QUERY_TX_POINTER	= 0x0B02,
+	HCLGE_OPC_QUERY_TX_STATUS	= 0x0B03,
+	HCLGE_OPC_CFG_RX_QUEUE		= 0x0B11,
+	HCLGE_OPC_QUERY_RX_POINTER	= 0x0B12,
+	HCLGE_OPC_QUERY_RX_STATUS	= 0x0B13,
+	HCLGE_OPC_STASH_RX_QUEUE_LRO	= 0x0B16,
+	HCLGE_OPC_CFG_RX_QUEUE_LRO	= 0x0B17,
+	HCLGE_OPC_CFG_COM_TQP_QUEUE	= 0x0B20,
+	HCLGE_OPC_RESET_TQP_QUEUE	= 0x0B22,
+
+	/* TSO cmd */
+	HCLGE_OPC_TSO_GENERIC_CONFIG	= 0x0C01,
+
+	/* RSS cmd */
+	HCLGE_OPC_RSS_GENERIC_CONFIG	= 0x0D01,
+	HCLGE_OPC_RSS_INDIR_TABLE	= 0x0D07,
+	HCLGE_OPC_RSS_TC_MODE		= 0x0D08,
+	HCLGE_OPC_RSS_INPUT_TUPLE	= 0x0D02,
+
+	/* Promisuous mode command */
+	HCLGE_OPC_CFG_PROMISC_MODE	= 0x0E01,
+
+	/* Interrupts cmd */
+	HCLGE_OPC_ADD_RING_TO_VECTOR	= 0x1503,
+	HCLGE_OPC_DEL_RING_TO_VECTOR	= 0x1504,
+
+	/* MAC command */
+	HCLGE_OPC_MAC_VLAN_ADD		    = 0x1000,
+	HCLGE_OPC_MAC_VLAN_REMOVE	    = 0x1001,
+	HCLGE_OPC_MAC_VLAN_TYPE_ID	    = 0x1002,
+	HCLGE_OPC_MAC_VLAN_INSERT	    = 0x1003,
+	HCLGE_OPC_MAC_ETHTYPE_ADD	    = 0x1010,
+	HCLGE_OPC_MAC_ETHTYPE_REMOVE	= 0x1011,
+
+	/* Multicast linear table cmd */
+	HCLGE_OPC_MTA_MAC_MODE_CFG	    = 0x1020,
+	HCLGE_OPC_MTA_MAC_FUNC_CFG	    = 0x1021,
+	HCLGE_OPC_MTA_TBL_ITEM_CFG	    = 0x1022,
+	HCLGE_OPC_MTA_TBL_ITEM_QUERY	= 0x1023,
+
+	/* VLAN command */
+	HCLGE_OPC_VLAN_FILTER_CTRL	    = 0x1100,
+	HCLGE_OPC_VLAN_FILTER_PF_CFG	= 0x1101,
+	HCLGE_OPC_VLAN_FILTER_VF_CFG	= 0x1102,
+
+	/* MDIO command */
+	HCLGE_OPC_MDIO_CONFIG		= 0x1900,
+
+	/* QCN command */
+	HCLGE_OPC_QCN_MOD_CFG		= 0x1A01,
+	HCLGE_OPC_QCN_GRP_TMPLT_CFG	= 0x1A02,
+	HCLGE_OPC_QCN_SHAPPING_IR_CFG	= 0x1A03,
+	HCLGE_OPC_QCN_SHAPPING_BS_CFG	= 0x1A04,
+	HCLGE_OPC_QCN_QSET_LINK_CFG	= 0x1A05,
+	HCLGE_OPC_QCN_RP_STATUS_GET	= 0x1A06,
+	HCLGE_OPC_QCN_AJUST_INIT	= 0x1A07,
+	HCLGE_OPC_QCN_DFX_CNT_STATUS    = 0x1A08,
+
+	/* Mailbox cmd */
+	HCLGEVF_OPC_MBX_PF_TO_VF	= 0x2000,
+};
+
+#define HCLGE_TQP_REG_OFFSET		0x80000
+#define HCLGE_TQP_REG_SIZE		0x200
+
+#define HCLGE_RCB_INIT_QUERY_TIMEOUT	10
+#define HCLGE_RCB_INIT_FLAG_EN_B	0
+#define HCLGE_RCB_INIT_FLAG_FINI_B	8
+struct hclge_config_rcb_init {
+	__le16 rcb_init_flag;
+	u8 rsv[22];
+};
+
+struct hclge_tqp_map {
+	__le16 tqp_id;	/* Absolute tqp id for in this pf */
+	u8 tqp_vf;	/* VF id */
+#define HCLGE_TQP_MAP_TYPE_PF		0
+#define HCLGE_TQP_MAP_TYPE_VF		1
+#define HCLGE_TQP_MAP_TYPE_B		0
+#define HCLGE_TQP_MAP_EN_B		1
+	u8 tqp_flag;	/* Indicate it's pf or vf tqp */
+	__le16 tqp_vid; /* Virtual id in this pf/vf */
+	u8 rsv[18];
+};
+
+#define HCLGE_VECTOR_ELEMENTS_PER_CMD	11
+
+enum hclge_int_type {
+	HCLGE_INT_TX,
+	HCLGE_INT_RX,
+	HCLGE_INT_EVENT,
+};
+
+struct hclge_ctrl_vector_chain {
+	u8 int_vector_id;
+	u8 int_cause_num;
+#define HCLGE_INT_TYPE_S	0
+#define HCLGE_INT_TYPE_M	0x3
+#define HCLGE_TQP_ID_S		2
+#define HCLGE_TQP_ID_M		(0x3fff << HCLGE_TQP_ID_S)
+	__le16 tqp_type_and_id[HCLGE_VECTOR_ELEMENTS_PER_CMD];
+};
+
+#define HCLGE_TC_NUM		8
+#define HCLGE_TC0_PRI_BUF_EN_B	15 /* Bit 15 indicate enable or not */
+#define HCLGE_BUF_UNIT_S	7  /* Buf size is united by 128 bytes */
+struct hclge_tx_buff_alloc {
+	__le16 tx_pkt_buff[HCLGE_TC_NUM];
+	u8 tx_buff_rsv[8];
+};
+
+struct hclge_rx_priv_buff {
+	__le16 buf_num[HCLGE_TC_NUM];
+	u8 rsv[8];
+};
+
+struct hclge_query_version {
+	__le32 firmware;
+	__le32 firmware_rsv[5];
+};
+
+#define HCLGE_RX_PRIV_EN_B	15
+#define HCLGE_TC_NUM_ONE_DESC	4
+struct hclge_priv_wl {
+	__le16 high;
+	__le16 low;
+};
+
+struct hclge_rx_priv_wl_buf {
+	struct hclge_priv_wl tc_wl[HCLGE_TC_NUM_ONE_DESC];
+};
+
+struct hclge_rx_com_thrd {
+	struct hclge_priv_wl com_thrd[HCLGE_TC_NUM_ONE_DESC];
+};
+
+struct hclge_rx_com_wl {
+	struct hclge_priv_wl com_wl;
+};
+
+struct hclge_waterline {
+	u32 low;
+	u32 high;
+};
+
+struct hclge_tc_thrd {
+	u32 low;
+	u32 high;
+};
+
+struct hclge_priv_buf {
+	struct hclge_waterline wl;	/* Waterline for low and high*/
+	u32 buf_size;	/* TC private buffer size */
+	u32 enable;	/* Enable TC private buffer or not */
+};
+
+#define HCLGE_MAX_TC_NUM	8
+struct hclge_shared_buf {
+	struct hclge_waterline self;
+	struct hclge_tc_thrd tc_thrd[HCLGE_MAX_TC_NUM];
+	u32 buf_size;
+};
+
+#define HCLGE_RX_COM_WL_EN_B	15
+struct hclge_rx_com_wl_buf {
+	__le16 high_wl;
+	__le16 low_wl;
+	u8 rsv[20];
+};
+
+#define HCLGE_RX_PKT_EN_B	15
+struct hclge_rx_pkt_buf {
+	__le16 high_pkt;
+	__le16 low_pkt;
+	u8 rsv[20];
+};
+
+#define HCLGE_PF_STATE_DONE_B	0
+#define HCLGE_PF_STATE_MAIN_B	1
+#define HCLGE_PF_STATE_BOND_B	2
+#define HCLGE_PF_STATE_MAC_N_B	6
+#define HCLGE_PF_MAC_NUM_MASK	0x3
+#define HCLGE_PF_STATE_MAIN	BIT(HCLGE_PF_STATE_MAIN_B)
+#define HCLGE_PF_STATE_DONE	BIT(HCLGE_PF_STATE_DONE_B)
+struct hclge_func_status {
+	__le32  vf_rst_state[4];
+	u8 pf_state;
+	u8 mac_id;
+	u8 rsv1;
+	u8 pf_cnt_in_mac;
+	u8 pf_num;
+	u8 vf_num;
+	u8 rsv[2];
+};
+
+struct hclge_pf_res {
+	__le16 tqp_num;
+	__le16 buf_size;
+	__le16 msixcap_localid_ba_nic;
+	__le16 msixcap_localid_ba_rocee;
+#define HCLGE_PF_VEC_NUM_S		0
+#define HCLGE_PF_VEC_NUM_M		(0xff << HCLGE_PF_VEC_NUM_S)
+	__le16 pf_intr_vector_number;
+	__le16 pf_own_fun_number;
+	__le32 rsv[3];
+};
+
+#define HCLGE_CFG_OFFSET_S	0
+#define HCLGE_CFG_OFFSET_M	0xfffff /* Byte (8-10.3) */
+#define HCLGE_CFG_RD_LEN_S	24
+#define HCLGE_CFG_RD_LEN_M	(0xf << HCLGE_CFG_RD_LEN_S)
+#define HCLGE_CFG_RD_LEN_BYTES	16
+#define HCLGE_CFG_RD_LEN_UNIT	4
+
+#define HCLGE_CFG_VMDQ_S	0
+#define HCLGE_CFG_VMDQ_M	(0xff << HCLGE_CFG_VMDQ_S)
+#define HCLGE_CFG_TC_NUM_S	8
+#define HCLGE_CFG_TC_NUM_M	(0xff << HCLGE_CFG_TC_NUM_S)
+#define HCLGE_CFG_TQP_DESC_N_S	16
+#define HCLGE_CFG_TQP_DESC_N_M	(0xffff << HCLGE_CFG_TQP_DESC_N_S)
+#define HCLGE_CFG_PHY_ADDR_S	0
+#define HCLGE_CFG_PHY_ADDR_M	(0x1f << HCLGE_CFG_PHY_ADDR_S)
+#define HCLGE_CFG_MEDIA_TP_S	8
+#define HCLGE_CFG_MEDIA_TP_M	(0xff << HCLGE_CFG_MEDIA_TP_S)
+#define HCLGE_CFG_RX_BUF_LEN_S	16
+#define HCLGE_CFG_RX_BUF_LEN_M	(0xffff << HCLGE_CFG_RX_BUF_LEN_S)
+#define HCLGE_CFG_MAC_ADDR_H_S	0
+#define HCLGE_CFG_MAC_ADDR_H_M	(0xffff << HCLGE_CFG_MAC_ADDR_H_S)
+#define HCLGE_CFG_DEFAULT_SPEED_S	16
+#define HCLGE_CFG_DEFAULT_SPEED_M	(0xff << HCLGE_CFG_DEFAULT_SPEED_S)
+
+struct hclge_cfg_param {
+	__le32 offset;
+	__le32 rsv;
+	__le32 param[4];
+};
+
+#define HCLGE_MAC_MODE		0x0
+#define HCLGE_DESC_NUM		0x40
+
+#define HCLGE_ALLOC_VALID_B	0
+struct hclge_vf_num {
+	u8 alloc_valid;
+	u8 rsv[23];
+};
+
+#define HCLGE_RSS_DEFAULT_OUTPORT_B	4
+#define HCLGE_RSS_HASH_KEY_OFFSET_B	4
+#define HCLGE_RSS_HASH_KEY_NUM		16
+struct hclge_rss_config {
+	u8 hash_config;
+	u8 rsv[7];
+	u8 hash_key[HCLGE_RSS_HASH_KEY_NUM];
+};
+
+struct hclge_rss_input_tuple {
+	u8 ipv4_tcp_en;
+	u8 ipv4_udp_en;
+	u8 ipv4_sctp_en;
+	u8 ipv4_fragment_en;
+	u8 ipv6_tcp_en;
+	u8 ipv6_udp_en;
+	u8 ipv6_sctp_en;
+	u8 ipv6_fragment_en;
+	u8 rsv[16];
+};
+
+#define HCLGE_RSS_CFG_TBL_SIZE	16
+
+struct hclge_rss_indirection_table {
+	u16 start_table_index;
+	u16 rss_set_bitmap;
+	u8 rsv[4];
+	u8 rss_result[HCLGE_RSS_CFG_TBL_SIZE];
+};
+
+#define HCLGE_RSS_TC_OFFSET_S		0
+#define HCLGE_RSS_TC_OFFSET_M		(0x3ff << HCLGE_RSS_TC_OFFSET_S)
+#define HCLGE_RSS_TC_SIZE_S		12
+#define HCLGE_RSS_TC_SIZE_M		(0x7 << HCLGE_RSS_TC_SIZE_S)
+#define HCLGE_RSS_TC_VALID_B		15
+struct hclge_rss_tc_mode {
+	u16 rss_tc_mode[HCLGE_MAX_TC_NUM];
+	u8 rsv[8];
+};
+
+#define HCLGE_LINK_STS_B	0
+#define HCLGE_LINK_STATUS	BIT(HCLGE_LINK_STS_B)
+struct hclge_link_status {
+	u8 status;
+	u8 rsv[23];
+};
+
+struct hclge_promisc_param {
+	u8 vf_id;
+	u8 enable;
+};
+
+#define HCLGE_PROMISC_EN_B	1
+#define HCLGE_PROMISC_EN_ALL	0x7
+#define HCLGE_PROMISC_EN_UC	0x1
+#define HCLGE_PROMISC_EN_MC	0x2
+#define HCLGE_PROMISC_EN_BC	0x4
+struct hclge_promisc_cfg {
+	u8 flag;
+	u8 vf_id;
+	__le16 rsv0;
+	u8 rsv1[20];
+};
+
+enum hclge_promisc_type {
+	HCLGE_UNICAST	= 1,
+	HCLGE_MULTICAST	= 2,
+	HCLGE_BROADCAST	= 3,
+};
+
+#define HCLGE_MAC_TX_EN_B	6
+#define HCLGE_MAC_RX_EN_B	7
+#define HCLGE_MAC_PAD_TX_B	11
+#define HCLGE_MAC_PAD_RX_B	12
+#define HCLGE_MAC_1588_TX_B	13
+#define HCLGE_MAC_1588_RX_B	14
+#define HCLGE_MAC_APP_LP_B	15
+#define HCLGE_MAC_LINE_LP_B	16
+#define HCLGE_MAC_FCS_TX_B	17
+#define HCLGE_MAC_RX_OVERSIZE_TRUNCATE_B	18
+#define HCLGE_MAC_RX_FCS_STRIP_B	19
+#define HCLGE_MAC_RX_FCS_B	20
+#define HCLGE_MAC_TX_UNDER_MIN_ERR_B		21
+#define HCLGE_MAC_TX_OVERSIZE_TRUNCATE_B	22
+
+struct hclge_config_mac_mode {
+	__le32 txrx_pad_fcs_loop_en;
+	u8 rsv[20];
+};
+
+#define HCLGE_CFG_SPEED_S		0
+#define HCLGE_CFG_SPEED_M		(0x3f << HCLGE_CFG_SPEED_S)
+
+#define HCLGE_CFG_DUPLEX_B		7
+#define HCLGE_CFG_DUPLEX_M		BIT(HCLGE_CFG_DUPLEX_B)
+
+struct hclge_config_mac_speed_dup {
+	u8 speed_dup;
+
+#define HCLGE_CFG_MAC_SPEED_CHANGE_EN_B	0
+	u8 mac_change_fec_en;
+	u8 rsv[22];
+};
+
+#define HCLGE_QUERY_SPEED_S		3
+#define HCLGE_QUERY_AN_B		0
+#define HCLGE_QUERY_DUPLEX_B		2
+
+#define HCLGE_QUERY_SPEED_M		(0x1f << HCLGE_QUERY_SPEED_S)
+#define HCLGE_QUERY_AN_M		BIT(HCLGE_QUERY_AN_B)
+#define HCLGE_QUERY_DUPLEX_M		BIT(HCLGE_QUERY_DUPLEX_B)
+
+struct hclge_query_an_speed_dup {
+	u8 an_syn_dup_speed;
+	u8 pause;
+	u8 rsv[23];
+};
+
+#define HCLGE_RING_ID_MASK		0x3ff
+#define HCLGE_TQP_ENABLE_B		0
+
+#define HCLGE_MAC_CFG_AN_EN_B		0
+#define HCLGE_MAC_CFG_AN_INT_EN_B	1
+#define HCLGE_MAC_CFG_AN_INT_MSK_B	2
+#define HCLGE_MAC_CFG_AN_INT_CLR_B	3
+#define HCLGE_MAC_CFG_AN_RST_B		4
+
+#define HCLGE_MAC_CFG_AN_EN	BIT(HCLGE_MAC_CFG_AN_EN_B)
+
+struct hclge_config_auto_neg {
+	__le32  cfg_an_cmd_flag;
+	u8      rsv[20];
+};
+
+#define HCLGE_MAC_MIN_MTU		64
+#define HCLGE_MAC_MAX_MTU		9728
+#define HCLGE_MAC_UPLINK_PORT		0x100
+
+struct hclge_config_max_frm_size {
+	__le16  max_frm_size;
+	u8      rsv[22];
+};
+
+enum hclge_mac_vlan_tbl_opcode {
+	HCLGE_MAC_VLAN_ADD,	/* Add new or modify mac_vlan */
+	HCLGE_MAC_VLAN_UPDATE,  /* Modify other fields of this table */
+	HCLGE_MAC_VLAN_REMOVE,  /* Remove a entry through mac_vlan key */
+	HCLGE_MAC_VLAN_LKUP,    /* Lookup a entry through mac_vlan key */
+};
+
+#define HCLGE_MAC_VLAN_BIT0_EN_B	0x0
+#define HCLGE_MAC_VLAN_BIT1_EN_B	0x1
+#define HCLGE_MAC_EPORT_SW_EN_B		0xc
+#define HCLGE_MAC_EPORT_TYPE_B		0xb
+#define HCLGE_MAC_EPORT_VFID_S		0x3
+#define HCLGE_MAC_EPORT_VFID_M		(0xff << HCLGE_MAC_EPORT_VFID_S)
+#define HCLGE_MAC_EPORT_PFID_S		0x0
+#define HCLGE_MAC_EPORT_PFID_M		(0x7 << HCLGE_MAC_EPORT_PFID_S)
+struct hclge_mac_vlan_tbl_entry {
+	u8	flags;
+	u8      resp_code;
+	__le16  vlan_tag;
+	__le32  mac_addr_hi32;
+	__le16  mac_addr_lo16;
+	__le16  rsv1;
+	u8      entry_type;
+	u8      mc_mac_en;
+	__le16  egress_port;
+	__le16  egress_queue;
+	u8      rsv2[6];
+};
+
+#define HCLGE_CFG_MTA_MAC_SEL_S		0x0
+#define HCLGE_CFG_MTA_MAC_SEL_M		(0x3 << HCLGE_CFG_MTA_MAC_SEL_S)
+#define HCLGE_CFG_MTA_MAC_EN_B		0x7
+struct hclge_mta_filter_mode {
+	u8	dmac_sel_en; /* Use lowest 2 bit as sel_mode, bit 7 as enable */
+	u8      rsv[23];
+};
+
+#define HCLGE_CFG_FUNC_MTA_ACCEPT_B	0x0
+struct hclge_cfg_func_mta_filter {
+	u8	accept; /* Only used lowest 1 bit */
+	u8      function_id;
+	u8      rsv[22];
+};
+
+#define HCLGE_CFG_MTA_ITEM_ACCEPT_B	0x0
+#define HCLGE_CFG_MTA_ITEM_IDX_S	0x0
+#define HCLGE_CFG_MTA_ITEM_IDX_M	(0xfff << HCLGE_CFG_MTA_ITEM_IDX_S)
+struct hclge_cfg_func_mta_item {
+	u16	item_idx; /* Only used lowest 12 bit */
+	u8      accept;   /* Only used lowest 1 bit */
+	u8      rsv[21];
+};
+
+struct hclge_mac_vlan_add {
+	__le16  flags;
+	__le16  mac_addr_hi16;
+	__le32  mac_addr_lo32;
+	__le32  mac_addr_msk_hi32;
+	__le16  mac_addr_msk_lo16;
+	__le16  vlan_tag;
+	__le16  ingress_port;
+	__le16  egress_port;
+	u8      rsv[4];
+};
+
+#define HNS3_MAC_VLAN_CFG_FLAG_BIT 0
+struct hclge_mac_vlan_remove {
+	__le16  flags;
+	__le16  mac_addr_hi16;
+	__le32  mac_addr_lo32;
+	__le32  mac_addr_msk_hi32;
+	__le16  mac_addr_msk_lo16;
+	__le16  vlan_tag;
+	__le16  ingress_port;
+	__le16  egress_port;
+	u8      rsv[4];
+};
+
+struct hclge_vlan_filter_ctrl {
+	u8 vlan_type;
+	u8 vlan_fe;
+	u8 rsv[22];
+};
+
+struct hclge_vlan_filter_pf_cfg {
+	u8 vlan_offset;
+	u8 vlan_cfg;
+	u8 rsv[2];
+	u8 vlan_offset_bitmap[20];
+};
+
+struct hclge_vlan_filter_vf_cfg {
+	u16 vlan_id;
+	u8  resp_code;
+	u8  rsv;
+	u8  vlan_cfg;
+	u8  rsv1[3];
+	u8  vf_bitmap[16];
+};
+
+struct hclge_cfg_com_tqp_queue {
+	__le16 tqp_id;
+	__le16 stream_id;
+	u8 enable;
+	u8 rsv[19];
+};
+
+struct hclge_cfg_tx_queue_pointer {
+	__le16 tqp_id;
+	__le16 tx_tail;
+	__le16 tx_head;
+	__le16 fbd_num;
+	__le16 ring_offset;
+	u8 rsv[14];
+};
+
+#define HCLGE_TSO_MSS_MIN_S	0
+#define HCLGE_TSO_MSS_MIN_M	(0x3FFF << HCLGE_TSO_MSS_MIN_S)
+
+#define HCLGE_TSO_MSS_MAX_S	16
+#define HCLGE_TSO_MSS_MAX_M	(0x3FFF << HCLGE_TSO_MSS_MAX_S)
+
+struct hclge_cfg_tso_status {
+	__le16 tso_mss_min;
+	__le16 tso_mss_max;
+	u8 rsv[20];
+};
+
+#define HCLGE_TSO_MSS_MIN	256
+#define HCLGE_TSO_MSS_MAX	9668
+
+#define HCLGE_TQP_RESET_B	0
+struct hclge_reset_tqp_queue {
+	__le16 tqp_id;
+	u8 reset_req;
+	u8 ready_to_reset;
+	u8 rsv[20];
+};
+
+#define HCLGE_DEFAULT_TX_BUF		0x4000	 /* 16k  bytes */
+#define HCLGE_TOTAL_PKT_BUF		0x108000 /* 1.03125M bytes */
+#define HCLGE_DEFAULT_DV		0xA000	 /* 40k byte */
+
+#define HCLGE_TYPE_CRQ			0
+#define HCLGE_TYPE_CSQ			1
+#define HCLGE_NIC_CSQ_BASEADDR_L_REG	0x27000
+#define HCLGE_NIC_CSQ_BASEADDR_H_REG	0x27004
+#define HCLGE_NIC_CSQ_DEPTH_REG		0x27008
+#define HCLGE_NIC_CSQ_TAIL_REG		0x27010
+#define HCLGE_NIC_CSQ_HEAD_REG		0x27014
+#define HCLGE_NIC_CRQ_BASEADDR_L_REG	0x27018
+#define HCLGE_NIC_CRQ_BASEADDR_H_REG	0x2701c
+#define HCLGE_NIC_CRQ_DEPTH_REG		0x27020
+#define HCLGE_NIC_CRQ_TAIL_REG		0x27024
+#define HCLGE_NIC_CRQ_HEAD_REG		0x27028
+#define HCLGE_NIC_CMQ_EN_B		16
+#define HCLGE_NIC_CMQ_ENABLE		BIT(HCLGE_NIC_CMQ_EN_B)
+#define HCLGE_NIC_CMQ_DESC_NUM		1024
+#define HCLGE_NIC_CMQ_DESC_NUM_S	3
+
+int hclge_cmd_init(struct hclge_dev *hdev);
+static inline void hclge_write_reg(void __iomem *base, u32 reg, u32 value)
+{
+	writel(value, base + reg);
+}
+
+#define hclge_write_dev(a, reg, value) \
+	hclge_write_reg((a)->io_base, (reg), (value))
+#define hclge_read_dev(a, reg) \
+	hclge_read_reg((a)->io_base, (reg))
+
+static inline u32 hclge_read_reg(u8 __iomem *base, u32 reg)
+{
+	u8 __iomem *reg_addr = READ_ONCE(base);
+
+	return readl(reg_addr + reg);
+}
+
+#define HCLGE_SEND_SYNC(flag) \
+	((flag) & HCLGE_CMD_FLAG_NO_INTR)
+
+struct hclge_hw;
+int hclge_cmd_send(struct hclge_hw *hw, struct hclge_desc *desc, int num);
+void hclge_cmd_setup_basic_desc(struct hclge_desc *desc,
+				enum hclge_opcode_type opcode, bool is_read);
+
+int hclge_cmd_set_promisc_mode(struct hclge_dev *hdev,
+			       struct hclge_promisc_param *param);
+
+enum hclge_cmd_status hclge_cmd_mdio_write(struct hclge_hw *hw,
+					   struct hclge_desc *desc);
+enum hclge_cmd_status hclge_cmd_mdio_read(struct hclge_hw *hw,
+					  struct hclge_desc *desc);
+
+void hclge_destroy_cmd_queue(struct hclge_hw *hw);
+#endif
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
new file mode 100644
index 000000000000..bb45365fb817
--- /dev/null
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
@@ -0,0 +1,4265 @@
+/*
+ * Copyright (c) 2016-2017 Hisilicon Limited.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/acpi.h>
+#include <linux/device.h>
+#include <linux/etherdevice.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+
+#include "hclge_cmd.h"
+#include "hclge_main.h"
+#include "hclge_mdio.h"
+#include "hclge_tm.h"
+#include "hnae3.h"
+
+#define HCLGE_NAME			"hclge"
+#define HCLGE_STATS_READ(p, offset) (*((u64 *)((u8 *)(p) + (offset))))
+#define HCLGE_MAC_STATS_FIELD_OFF(f) (offsetof(struct hclge_mac_stats, f))
+#define HCLGE_64BIT_STATS_FIELD_OFF(f) (offsetof(struct hclge_64_bit_stats, f))
+#define HCLGE_32BIT_STATS_FIELD_OFF(f) (offsetof(struct hclge_32_bit_stats, f))
+
+static int hclge_rss_init_hw(struct hclge_dev *hdev);
+static int hclge_set_mta_filter_mode(struct hclge_dev *hdev,
+				     enum hclge_mta_dmac_sel_type mta_mac_sel,
+				     bool enable);
+static int hclge_init_vlan_config(struct hclge_dev *hdev);
+
+static struct hnae3_ae_algo ae_algo;
+
+static const struct pci_device_id ae_algo_pci_tbl[] = {
+	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
+	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
+	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0},
+	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0},
+	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0},
+	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0},
+	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0},
+	/* Required last entry */
+	{0, }
+};
+
+static const struct pci_device_id roce_pci_tbl[] = {
+	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0},
+	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0},
+	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0},
+	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0},
+	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0},
+	/* Required last entry */
+	{0, }
+};
+
+static const char hns3_nic_test_strs[][ETH_GSTRING_LEN] = {
+	"Mac    Loopback test",
+	"Serdes Loopback test",
+	"Phy    Loopback test"
+};
+
+static const struct hclge_comm_stats_str g_all_64bit_stats_string[] = {
+	{"igu_rx_oversize_pkt",
+		HCLGE_64BIT_STATS_FIELD_OFF(igu_rx_oversize_pkt)},
+	{"igu_rx_undersize_pkt",
+		HCLGE_64BIT_STATS_FIELD_OFF(igu_rx_undersize_pkt)},
+	{"igu_rx_out_all_pkt",
+		HCLGE_64BIT_STATS_FIELD_OFF(igu_rx_out_all_pkt)},
+	{"igu_rx_uni_pkt",
+		HCLGE_64BIT_STATS_FIELD_OFF(igu_rx_uni_pkt)},
+	{"igu_rx_multi_pkt",
+		HCLGE_64BIT_STATS_FIELD_OFF(igu_rx_multi_pkt)},
+	{"igu_rx_broad_pkt",
+		HCLGE_64BIT_STATS_FIELD_OFF(igu_rx_broad_pkt)},
+	{"egu_tx_out_all_pkt",
+		HCLGE_64BIT_STATS_FIELD_OFF(egu_tx_out_all_pkt)},
+	{"egu_tx_uni_pkt",
+		HCLGE_64BIT_STATS_FIELD_OFF(egu_tx_uni_pkt)},
+	{"egu_tx_multi_pkt",
+		HCLGE_64BIT_STATS_FIELD_OFF(egu_tx_multi_pkt)},
+	{"egu_tx_broad_pkt",
+		HCLGE_64BIT_STATS_FIELD_OFF(egu_tx_broad_pkt)},
+	{"ssu_ppp_mac_key_num",
+		HCLGE_64BIT_STATS_FIELD_OFF(ssu_ppp_mac_key_num)},
+	{"ssu_ppp_host_key_num",
+		HCLGE_64BIT_STATS_FIELD_OFF(ssu_ppp_host_key_num)},
+	{"ppp_ssu_mac_rlt_num",
+		HCLGE_64BIT_STATS_FIELD_OFF(ppp_ssu_mac_rlt_num)},
+	{"ppp_ssu_host_rlt_num",
+		HCLGE_64BIT_STATS_FIELD_OFF(ppp_ssu_host_rlt_num)},
+	{"ssu_tx_in_num",
+		HCLGE_64BIT_STATS_FIELD_OFF(ssu_tx_in_num)},
+	{"ssu_tx_out_num",
+		HCLGE_64BIT_STATS_FIELD_OFF(ssu_tx_out_num)},
+	{"ssu_rx_in_num",
+		HCLGE_64BIT_STATS_FIELD_OFF(ssu_rx_in_num)},
+	{"ssu_rx_out_num",
+		HCLGE_64BIT_STATS_FIELD_OFF(ssu_rx_out_num)}
+};
+
+static const struct hclge_comm_stats_str g_all_32bit_stats_string[] = {
+	{"igu_rx_err_pkt",
+		HCLGE_32BIT_STATS_FIELD_OFF(igu_rx_err_pkt)},
+	{"igu_rx_no_eof_pkt",
+		HCLGE_32BIT_STATS_FIELD_OFF(igu_rx_no_eof_pkt)},
+	{"igu_rx_no_sof_pkt",
+		HCLGE_32BIT_STATS_FIELD_OFF(igu_rx_no_sof_pkt)},
+	{"egu_tx_1588_pkt",
+		HCLGE_32BIT_STATS_FIELD_OFF(egu_tx_1588_pkt)},
+	{"ssu_full_drop_num",
+		HCLGE_32BIT_STATS_FIELD_OFF(ssu_full_drop_num)},
+	{"ssu_part_drop_num",
+		HCLGE_32BIT_STATS_FIELD_OFF(ssu_part_drop_num)},
+	{"ppp_key_drop_num",
+		HCLGE_32BIT_STATS_FIELD_OFF(ppp_key_drop_num)},
+	{"ppp_rlt_drop_num",
+		HCLGE_32BIT_STATS_FIELD_OFF(ppp_rlt_drop_num)},
+	{"ssu_key_drop_num",
+		HCLGE_32BIT_STATS_FIELD_OFF(ssu_key_drop_num)},
+	{"pkt_curr_buf_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(pkt_curr_buf_cnt)},
+	{"qcn_fb_rcv_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(qcn_fb_rcv_cnt)},
+	{"qcn_fb_drop_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(qcn_fb_drop_cnt)},
+	{"qcn_fb_invaild_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(qcn_fb_invaild_cnt)},
+	{"rx_packet_tc0_in_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(rx_packet_tc0_in_cnt)},
+	{"rx_packet_tc1_in_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(rx_packet_tc1_in_cnt)},
+	{"rx_packet_tc2_in_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(rx_packet_tc2_in_cnt)},
+	{"rx_packet_tc3_in_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(rx_packet_tc3_in_cnt)},
+	{"rx_packet_tc4_in_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(rx_packet_tc4_in_cnt)},
+	{"rx_packet_tc5_in_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(rx_packet_tc5_in_cnt)},
+	{"rx_packet_tc6_in_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(rx_packet_tc6_in_cnt)},
+	{"rx_packet_tc7_in_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(rx_packet_tc7_in_cnt)},
+	{"rx_packet_tc0_out_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(rx_packet_tc0_out_cnt)},
+	{"rx_packet_tc1_out_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(rx_packet_tc1_out_cnt)},
+	{"rx_packet_tc2_out_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(rx_packet_tc2_out_cnt)},
+	{"rx_packet_tc3_out_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(rx_packet_tc3_out_cnt)},
+	{"rx_packet_tc4_out_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(rx_packet_tc4_out_cnt)},
+	{"rx_packet_tc5_out_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(rx_packet_tc5_out_cnt)},
+	{"rx_packet_tc6_out_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(rx_packet_tc6_out_cnt)},
+	{"rx_packet_tc7_out_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(rx_packet_tc7_out_cnt)},
+	{"tx_packet_tc0_in_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(tx_packet_tc0_in_cnt)},
+	{"tx_packet_tc1_in_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(tx_packet_tc1_in_cnt)},
+	{"tx_packet_tc2_in_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(tx_packet_tc2_in_cnt)},
+	{"tx_packet_tc3_in_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(tx_packet_tc3_in_cnt)},
+	{"tx_packet_tc4_in_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(tx_packet_tc4_in_cnt)},
+	{"tx_packet_tc5_in_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(tx_packet_tc5_in_cnt)},
+	{"tx_packet_tc6_in_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(tx_packet_tc6_in_cnt)},
+	{"tx_packet_tc7_in_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(tx_packet_tc7_in_cnt)},
+	{"tx_packet_tc0_out_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(tx_packet_tc0_out_cnt)},
+	{"tx_packet_tc1_out_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(tx_packet_tc1_out_cnt)},
+	{"tx_packet_tc2_out_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(tx_packet_tc2_out_cnt)},
+	{"tx_packet_tc3_out_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(tx_packet_tc3_out_cnt)},
+	{"tx_packet_tc4_out_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(tx_packet_tc4_out_cnt)},
+	{"tx_packet_tc5_out_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(tx_packet_tc5_out_cnt)},
+	{"tx_packet_tc6_out_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(tx_packet_tc6_out_cnt)},
+	{"tx_packet_tc7_out_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(tx_packet_tc7_out_cnt)},
+	{"pkt_curr_buf_tc0_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(pkt_curr_buf_tc0_cnt)},
+	{"pkt_curr_buf_tc1_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(pkt_curr_buf_tc1_cnt)},
+	{"pkt_curr_buf_tc2_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(pkt_curr_buf_tc2_cnt)},
+	{"pkt_curr_buf_tc3_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(pkt_curr_buf_tc3_cnt)},
+	{"pkt_curr_buf_tc4_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(pkt_curr_buf_tc4_cnt)},
+	{"pkt_curr_buf_tc5_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(pkt_curr_buf_tc5_cnt)},
+	{"pkt_curr_buf_tc6_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(pkt_curr_buf_tc6_cnt)},
+	{"pkt_curr_buf_tc7_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(pkt_curr_buf_tc7_cnt)},
+	{"mb_uncopy_num",
+		HCLGE_32BIT_STATS_FIELD_OFF(mb_uncopy_num)},
+	{"lo_pri_unicast_rlt_drop_num",
+		HCLGE_32BIT_STATS_FIELD_OFF(lo_pri_unicast_rlt_drop_num)},
+	{"hi_pri_multicast_rlt_drop_num",
+		HCLGE_32BIT_STATS_FIELD_OFF(hi_pri_multicast_rlt_drop_num)},
+	{"lo_pri_multicast_rlt_drop_num",
+		HCLGE_32BIT_STATS_FIELD_OFF(lo_pri_multicast_rlt_drop_num)},
+	{"rx_oq_drop_pkt_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(rx_oq_drop_pkt_cnt)},
+	{"tx_oq_drop_pkt_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(tx_oq_drop_pkt_cnt)},
+	{"nic_l2_err_drop_pkt_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(nic_l2_err_drop_pkt_cnt)},
+	{"roc_l2_err_drop_pkt_cnt",
+		HCLGE_32BIT_STATS_FIELD_OFF(roc_l2_err_drop_pkt_cnt)}
+};
+
+static const struct hclge_comm_stats_str g_mac_stats_string[] = {
+	{"mac_tx_mac_pause_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_mac_pause_num)},
+	{"mac_rx_mac_pause_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_mac_pause_num)},
+	{"mac_tx_pfc_pri0_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_pfc_pri0_pkt_num)},
+	{"mac_tx_pfc_pri1_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_pfc_pri1_pkt_num)},
+	{"mac_tx_pfc_pri2_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_pfc_pri2_pkt_num)},
+	{"mac_tx_pfc_pri3_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_pfc_pri3_pkt_num)},
+	{"mac_tx_pfc_pri4_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_pfc_pri4_pkt_num)},
+	{"mac_tx_pfc_pri5_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_pfc_pri5_pkt_num)},
+	{"mac_tx_pfc_pri6_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_pfc_pri6_pkt_num)},
+	{"mac_tx_pfc_pri7_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_pfc_pri7_pkt_num)},
+	{"mac_rx_pfc_pri0_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_pfc_pri0_pkt_num)},
+	{"mac_rx_pfc_pri1_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_pfc_pri1_pkt_num)},
+	{"mac_rx_pfc_pri2_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_pfc_pri2_pkt_num)},
+	{"mac_rx_pfc_pri3_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_pfc_pri3_pkt_num)},
+	{"mac_rx_pfc_pri4_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_pfc_pri4_pkt_num)},
+	{"mac_rx_pfc_pri5_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_pfc_pri5_pkt_num)},
+	{"mac_rx_pfc_pri6_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_pfc_pri6_pkt_num)},
+	{"mac_rx_pfc_pri7_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_pfc_pri7_pkt_num)},
+	{"mac_tx_total_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_total_pkt_num)},
+	{"mac_tx_total_oct_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_total_oct_num)},
+	{"mac_tx_good_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_good_pkt_num)},
+	{"mac_tx_bad_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_bad_pkt_num)},
+	{"mac_tx_good_oct_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_good_oct_num)},
+	{"mac_tx_bad_oct_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_bad_oct_num)},
+	{"mac_tx_uni_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_uni_pkt_num)},
+	{"mac_tx_multi_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_multi_pkt_num)},
+	{"mac_tx_broad_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_broad_pkt_num)},
+	{"mac_tx_undersize_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_undersize_pkt_num)},
+	{"mac_tx_overrsize_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_overrsize_pkt_num)},
+	{"mac_tx_64_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_64_oct_pkt_num)},
+	{"mac_tx_65_127_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_65_127_oct_pkt_num)},
+	{"mac_tx_128_255_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_128_255_oct_pkt_num)},
+	{"mac_tx_256_511_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_256_511_oct_pkt_num)},
+	{"mac_tx_512_1023_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_512_1023_oct_pkt_num)},
+	{"mac_tx_1024_1518_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_1024_1518_oct_pkt_num)},
+	{"mac_tx_1519_max_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_1519_max_oct_pkt_num)},
+	{"mac_rx_total_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_total_pkt_num)},
+	{"mac_rx_total_oct_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_total_oct_num)},
+	{"mac_rx_good_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_good_pkt_num)},
+	{"mac_rx_bad_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_bad_pkt_num)},
+	{"mac_rx_good_oct_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_good_oct_num)},
+	{"mac_rx_bad_oct_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_bad_oct_num)},
+	{"mac_rx_uni_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_uni_pkt_num)},
+	{"mac_rx_multi_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_multi_pkt_num)},
+	{"mac_rx_broad_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_broad_pkt_num)},
+	{"mac_rx_undersize_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_undersize_pkt_num)},
+	{"mac_rx_overrsize_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_overrsize_pkt_num)},
+	{"mac_rx_64_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_64_oct_pkt_num)},
+	{"mac_rx_65_127_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_65_127_oct_pkt_num)},
+	{"mac_rx_128_255_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_128_255_oct_pkt_num)},
+	{"mac_rx_256_511_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_256_511_oct_pkt_num)},
+	{"mac_rx_512_1023_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_512_1023_oct_pkt_num)},
+	{"mac_rx_1024_1518_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_1024_1518_oct_pkt_num)},
+	{"mac_rx_1519_max_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_1519_max_oct_pkt_num)},
+
+	{"mac_trans_fragment_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_trans_fragment_pkt_num)},
+	{"mac_trans_undermin_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_trans_undermin_pkt_num)},
+	{"mac_trans_jabber_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_trans_jabber_pkt_num)},
+	{"mac_trans_err_all_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_trans_err_all_pkt_num)},
+	{"mac_trans_from_app_good_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_trans_from_app_good_pkt_num)},
+	{"mac_trans_from_app_bad_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_trans_from_app_bad_pkt_num)},
+	{"mac_rcv_fragment_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rcv_fragment_pkt_num)},
+	{"mac_rcv_undermin_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rcv_undermin_pkt_num)},
+	{"mac_rcv_jabber_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rcv_jabber_pkt_num)},
+	{"mac_rcv_fcs_err_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rcv_fcs_err_pkt_num)},
+	{"mac_rcv_send_app_good_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rcv_send_app_good_pkt_num)},
+	{"mac_rcv_send_app_bad_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rcv_send_app_bad_pkt_num)}
+};
+
+static int hclge_64_bit_update_stats(struct hclge_dev *hdev)
+{
+#define HCLGE_64_BIT_CMD_NUM 5
+#define HCLGE_64_BIT_RTN_DATANUM 4
+	u64 *data = (u64 *)(&hdev->hw_stats.all_64_bit_stats);
+	struct hclge_desc desc[HCLGE_64_BIT_CMD_NUM];
+	u64 *desc_data;
+	int i, k, n;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc[0], HCLGE_OPC_STATS_64_BIT, true);
+	ret = hclge_cmd_send(&hdev->hw, desc, HCLGE_64_BIT_CMD_NUM);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"Get 64 bit pkt stats fail, status = %d.\n", ret);
+		return ret;
+	}
+
+	for (i = 0; i < HCLGE_64_BIT_CMD_NUM; i++) {
+		if (unlikely(i == 0)) {
+			desc_data = (u64 *)(&desc[i].data[0]);
+			n = HCLGE_64_BIT_RTN_DATANUM - 1;
+		} else {
+			desc_data = (u64 *)(&desc[i]);
+			n = HCLGE_64_BIT_RTN_DATANUM;
+		}
+		for (k = 0; k < n; k++) {
+			*data++ += cpu_to_le64(*desc_data);
+			desc_data++;
+		}
+	}
+
+	return 0;
+}
+
+static void hclge_reset_partial_32bit_counter(struct hclge_32_bit_stats *stats)
+{
+	stats->pkt_curr_buf_cnt     = 0;
+	stats->pkt_curr_buf_tc0_cnt = 0;
+	stats->pkt_curr_buf_tc1_cnt = 0;
+	stats->pkt_curr_buf_tc2_cnt = 0;
+	stats->pkt_curr_buf_tc3_cnt = 0;
+	stats->pkt_curr_buf_tc4_cnt = 0;
+	stats->pkt_curr_buf_tc5_cnt = 0;
+	stats->pkt_curr_buf_tc6_cnt = 0;
+	stats->pkt_curr_buf_tc7_cnt = 0;
+}
+
+static int hclge_32_bit_update_stats(struct hclge_dev *hdev)
+{
+#define HCLGE_32_BIT_CMD_NUM 8
+#define HCLGE_32_BIT_RTN_DATANUM 8
+
+	struct hclge_desc desc[HCLGE_32_BIT_CMD_NUM];
+	struct hclge_32_bit_stats *all_32_bit_stats;
+	u32 *desc_data;
+	int i, k, n;
+	u64 *data;
+	int ret;
+
+	all_32_bit_stats = &hdev->hw_stats.all_32_bit_stats;
+	data = (u64 *)(&all_32_bit_stats->egu_tx_1588_pkt);
+
+	hclge_cmd_setup_basic_desc(&desc[0], HCLGE_OPC_STATS_32_BIT, true);
+	ret = hclge_cmd_send(&hdev->hw, desc, HCLGE_32_BIT_CMD_NUM);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"Get 32 bit pkt stats fail, status = %d.\n", ret);
+
+		return ret;
+	}
+
+	hclge_reset_partial_32bit_counter(all_32_bit_stats);
+	for (i = 0; i < HCLGE_32_BIT_CMD_NUM; i++) {
+		if (unlikely(i == 0)) {
+			all_32_bit_stats->igu_rx_err_pkt +=
+				cpu_to_le32(desc[i].data[0]);
+			all_32_bit_stats->igu_rx_no_eof_pkt +=
+				cpu_to_le32(desc[i].data[1] & 0xffff);
+			all_32_bit_stats->igu_rx_no_sof_pkt +=
+				cpu_to_le32((desc[i].data[1] >> 16) & 0xffff);
+
+			desc_data = (u32 *)(&desc[i].data[2]);
+			n = HCLGE_32_BIT_RTN_DATANUM - 4;
+		} else {
+			desc_data = (u32 *)(&desc[i]);
+			n = HCLGE_32_BIT_RTN_DATANUM;
+		}
+		for (k = 0; k < n; k++) {
+			*data++ += cpu_to_le32(*desc_data);
+			desc_data++;
+		}
+	}
+
+	return 0;
+}
+
+static int hclge_mac_update_stats(struct hclge_dev *hdev)
+{
+#define HCLGE_MAC_CMD_NUM 17
+#define HCLGE_RTN_DATA_NUM 4
+
+	u64 *data = (u64 *)(&hdev->hw_stats.mac_stats);
+	struct hclge_desc desc[HCLGE_MAC_CMD_NUM];
+	u64 *desc_data;
+	int i, k, n;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc[0], HCLGE_OPC_STATS_MAC, true);
+	ret = hclge_cmd_send(&hdev->hw, desc, HCLGE_MAC_CMD_NUM);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"Get MAC pkt stats fail, status = %d.\n", ret);
+
+		return ret;
+	}
+
+	for (i = 0; i < HCLGE_MAC_CMD_NUM; i++) {
+		if (unlikely(i == 0)) {
+			desc_data = (u64 *)(&desc[i].data[0]);
+			n = HCLGE_RTN_DATA_NUM - 2;
+		} else {
+			desc_data = (u64 *)(&desc[i]);
+			n = HCLGE_RTN_DATA_NUM;
+		}
+		for (k = 0; k < n; k++) {
+			*data++ += cpu_to_le64(*desc_data);
+			desc_data++;
+		}
+	}
+
+	return 0;
+}
+
+static int hclge_tqps_update_stats(struct hnae3_handle *handle)
+{
+	struct hnae3_knic_private_info *kinfo = &handle->kinfo;
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+	struct hnae3_queue *queue;
+	struct hclge_desc desc[1];
+	struct hclge_tqp *tqp;
+	int ret, i;
+
+	for (i = 0; i < kinfo->num_tqps; i++) {
+		queue = handle->kinfo.tqp[i];
+		tqp = container_of(queue, struct hclge_tqp, q);
+		/* command : HCLGE_OPC_QUERY_IGU_STAT */
+		hclge_cmd_setup_basic_desc(&desc[0],
+					   HCLGE_OPC_QUERY_RX_STATUS,
+					   true);
+
+		desc[0].data[0] = (tqp->index & 0x1ff);
+		ret = hclge_cmd_send(&hdev->hw, desc, 1);
+		if (ret) {
+			dev_err(&hdev->pdev->dev,
+				"Query tqp stat fail, status = %d,queue = %d\n",
+				ret,	i);
+			return ret;
+		}
+		tqp->tqp_stats.rcb_rx_ring_pktnum_rcd +=
+			cpu_to_le32(desc[0].data[4]);
+	}
+
+	for (i = 0; i < kinfo->num_tqps; i++) {
+		queue = handle->kinfo.tqp[i];
+		tqp = container_of(queue, struct hclge_tqp, q);
+		/* command : HCLGE_OPC_QUERY_IGU_STAT */
+		hclge_cmd_setup_basic_desc(&desc[0],
+					   HCLGE_OPC_QUERY_TX_STATUS,
+					   true);
+
+		desc[0].data[0] = (tqp->index & 0x1ff);
+		ret = hclge_cmd_send(&hdev->hw, desc, 1);
+		if (ret) {
+			dev_err(&hdev->pdev->dev,
+				"Query tqp stat fail, status = %d,queue = %d\n",
+				ret, i);
+			return ret;
+		}
+		tqp->tqp_stats.rcb_tx_ring_pktnum_rcd +=
+			cpu_to_le32(desc[0].data[4]);
+	}
+
+	return 0;
+}
+
+static u64 *hclge_tqps_get_stats(struct hnae3_handle *handle, u64 *data)
+{
+	struct hnae3_knic_private_info *kinfo = &handle->kinfo;
+	struct hclge_tqp *tqp;
+	u64 *buff = data;
+	int i;
+
+	for (i = 0; i < kinfo->num_tqps; i++) {
+		tqp = container_of(kinfo->tqp[i], struct hclge_tqp, q);
+		*buff++ = cpu_to_le64(tqp->tqp_stats.rcb_tx_ring_pktnum_rcd);
+	}
+
+	for (i = 0; i < kinfo->num_tqps; i++) {
+		tqp = container_of(kinfo->tqp[i], struct hclge_tqp, q);
+		*buff++ = cpu_to_le64(tqp->tqp_stats.rcb_rx_ring_pktnum_rcd);
+	}
+
+	return buff;
+}
+
+static int hclge_tqps_get_sset_count(struct hnae3_handle *handle, int stringset)
+{
+	struct hnae3_knic_private_info *kinfo = &handle->kinfo;
+
+	return kinfo->num_tqps * (2);
+}
+
+static u8 *hclge_tqps_get_strings(struct hnae3_handle *handle, u8 *data)
+{
+	struct hnae3_knic_private_info *kinfo = &handle->kinfo;
+	u8 *buff = data;
+	int i = 0;
+
+	for (i = 0; i < kinfo->num_tqps; i++) {
+		struct hclge_tqp *tqp = container_of(handle->kinfo.tqp[i],
+			struct hclge_tqp, q);
+		snprintf(buff, ETH_GSTRING_LEN, "rcb_q%d_tx_pktnum_rcd",
+			 tqp->index);
+		buff = buff + ETH_GSTRING_LEN;
+	}
+
+	for (i = 0; i < kinfo->num_tqps; i++) {
+		struct hclge_tqp *tqp = container_of(kinfo->tqp[i],
+			struct hclge_tqp, q);
+		snprintf(buff, ETH_GSTRING_LEN, "rcb_q%d_rx_pktnum_rcd",
+			 tqp->index);
+		buff = buff + ETH_GSTRING_LEN;
+	}
+
+	return buff;
+}
+
+static u64 *hclge_comm_get_stats(void *comm_stats,
+				 const struct hclge_comm_stats_str strs[],
+				 int size, u64 *data)
+{
+	u64 *buf = data;
+	u32 i;
+
+	for (i = 0; i < size; i++)
+		buf[i] = HCLGE_STATS_READ(comm_stats, strs[i].offset);
+
+	return buf + size;
+}
+
+static u8 *hclge_comm_get_strings(u32 stringset,
+				  const struct hclge_comm_stats_str strs[],
+				  int size, u8 *data)
+{
+	char *buff = (char *)data;
+	u32 i;
+
+	if (stringset != ETH_SS_STATS)
+		return buff;
+
+	for (i = 0; i < size; i++) {
+		snprintf(buff, ETH_GSTRING_LEN,
+			 strs[i].desc);
+		buff = buff + ETH_GSTRING_LEN;
+	}
+
+	return (u8 *)buff;
+}
+
+static void hclge_update_netstat(struct hclge_hw_stats *hw_stats,
+				 struct net_device_stats *net_stats)
+{
+	net_stats->tx_dropped = 0;
+	net_stats->rx_dropped = hw_stats->all_32_bit_stats.ssu_full_drop_num;
+	net_stats->rx_dropped += hw_stats->all_32_bit_stats.ppp_key_drop_num;
+	net_stats->rx_dropped += hw_stats->all_32_bit_stats.ssu_key_drop_num;
+
+	net_stats->rx_errors = hw_stats->mac_stats.mac_rx_overrsize_pkt_num;
+	net_stats->rx_errors += hw_stats->mac_stats.mac_rx_undersize_pkt_num;
+	net_stats->rx_errors += hw_stats->all_32_bit_stats.igu_rx_err_pkt;
+	net_stats->rx_errors += hw_stats->all_32_bit_stats.igu_rx_no_eof_pkt;
+	net_stats->rx_errors += hw_stats->all_32_bit_stats.igu_rx_no_sof_pkt;
+	net_stats->rx_errors += hw_stats->mac_stats.mac_rcv_fcs_err_pkt_num;
+
+	net_stats->multicast = hw_stats->mac_stats.mac_tx_multi_pkt_num;
+	net_stats->multicast += hw_stats->mac_stats.mac_rx_multi_pkt_num;
+
+	net_stats->rx_crc_errors = hw_stats->mac_stats.mac_rcv_fcs_err_pkt_num;
+	net_stats->rx_length_errors =
+		hw_stats->mac_stats.mac_rx_undersize_pkt_num;
+	net_stats->rx_length_errors +=
+		hw_stats->mac_stats.mac_rx_overrsize_pkt_num;
+	net_stats->rx_over_errors =
+		hw_stats->mac_stats.mac_rx_overrsize_pkt_num;
+}
+
+static void hclge_update_stats_for_all(struct hclge_dev *hdev)
+{
+	struct hnae3_handle *handle;
+	int status;
+
+	handle = &hdev->vport[0].nic;
+	if (handle->client) {
+		status = hclge_tqps_update_stats(handle);
+		if (status) {
+			dev_err(&hdev->pdev->dev,
+				"Update TQPS stats fail, status = %d.\n",
+				status);
+		}
+	}
+
+	status = hclge_mac_update_stats(hdev);
+	if (status)
+		dev_err(&hdev->pdev->dev,
+			"Update MAC stats fail, status = %d.\n", status);
+
+	status = hclge_32_bit_update_stats(hdev);
+	if (status)
+		dev_err(&hdev->pdev->dev,
+			"Update 32 bit stats fail, status = %d.\n",
+			status);
+
+	hclge_update_netstat(&hdev->hw_stats, &handle->kinfo.netdev->stats);
+}
+
+static void hclge_update_stats(struct hnae3_handle *handle,
+			       struct net_device_stats *net_stats)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+	struct hclge_hw_stats *hw_stats = &hdev->hw_stats;
+	int status;
+
+	status = hclge_mac_update_stats(hdev);
+	if (status)
+		dev_err(&hdev->pdev->dev,
+			"Update MAC stats fail, status = %d.\n",
+			status);
+
+	status = hclge_32_bit_update_stats(hdev);
+	if (status)
+		dev_err(&hdev->pdev->dev,
+			"Update 32 bit stats fail, status = %d.\n",
+			status);
+
+	status = hclge_64_bit_update_stats(hdev);
+	if (status)
+		dev_err(&hdev->pdev->dev,
+			"Update 64 bit stats fail, status = %d.\n",
+			status);
+
+	status = hclge_tqps_update_stats(handle);
+	if (status)
+		dev_err(&hdev->pdev->dev,
+			"Update TQPS stats fail, status = %d.\n",
+			status);
+
+	hclge_update_netstat(hw_stats, net_stats);
+}
+
+static int hclge_get_sset_count(struct hnae3_handle *handle, int stringset)
+{
+#define HCLGE_LOOPBACK_TEST_FLAGS 0x7
+
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+	int count = 0;
+
+	/* Loopback test support rules:
+	 * mac: only GE mode support
+	 * serdes: all mac mode will support include GE/XGE/LGE/CGE
+	 * phy: only support when phy device exist on board
+	 */
+	if (stringset == ETH_SS_TEST) {
+		/* clear loopback bit flags at first */
+		handle->flags = (handle->flags & (~HCLGE_LOOPBACK_TEST_FLAGS));
+		if (hdev->hw.mac.speed == HCLGE_MAC_SPEED_10M ||
+		    hdev->hw.mac.speed == HCLGE_MAC_SPEED_100M ||
+		    hdev->hw.mac.speed == HCLGE_MAC_SPEED_1G) {
+			count += 1;
+			handle->flags |= HNAE3_SUPPORT_MAC_LOOPBACK;
+		} else {
+			count = -EOPNOTSUPP;
+		}
+	} else if (stringset == ETH_SS_STATS) {
+		count = ARRAY_SIZE(g_mac_stats_string) +
+			ARRAY_SIZE(g_all_32bit_stats_string) +
+			ARRAY_SIZE(g_all_64bit_stats_string) +
+			hclge_tqps_get_sset_count(handle, stringset);
+	}
+
+	return count;
+}
+
+static void hclge_get_strings(struct hnae3_handle *handle,
+			      u32 stringset,
+			      u8 *data)
+{
+	u8 *p = (char *)data;
+	int size;
+
+	if (stringset == ETH_SS_STATS) {
+		size = ARRAY_SIZE(g_mac_stats_string);
+		p = hclge_comm_get_strings(stringset,
+					   g_mac_stats_string,
+					   size,
+					   p);
+		size = ARRAY_SIZE(g_all_32bit_stats_string);
+		p = hclge_comm_get_strings(stringset,
+					   g_all_32bit_stats_string,
+					   size,
+					   p);
+		size = ARRAY_SIZE(g_all_64bit_stats_string);
+		p = hclge_comm_get_strings(stringset,
+					   g_all_64bit_stats_string,
+					   size,
+					   p);
+		p = hclge_tqps_get_strings(handle, p);
+	} else if (stringset == ETH_SS_TEST) {
+		if (handle->flags & HNAE3_SUPPORT_MAC_LOOPBACK) {
+			memcpy(p,
+			       hns3_nic_test_strs[HNAE3_MAC_INTER_LOOP_MAC],
+			       ETH_GSTRING_LEN);
+			p += ETH_GSTRING_LEN;
+		}
+		if (handle->flags & HNAE3_SUPPORT_SERDES_LOOPBACK) {
+			memcpy(p,
+			       hns3_nic_test_strs[HNAE3_MAC_INTER_LOOP_SERDES],
+			       ETH_GSTRING_LEN);
+			p += ETH_GSTRING_LEN;
+		}
+		if (handle->flags & HNAE3_SUPPORT_PHY_LOOPBACK) {
+			memcpy(p,
+			       hns3_nic_test_strs[HNAE3_MAC_INTER_LOOP_PHY],
+			       ETH_GSTRING_LEN);
+			p += ETH_GSTRING_LEN;
+		}
+	}
+}
+
+static void hclge_get_stats(struct hnae3_handle *handle, u64 *data)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+	u64 *p;
+
+	p = hclge_comm_get_stats(&hdev->hw_stats.mac_stats,
+				 g_mac_stats_string,
+				 ARRAY_SIZE(g_mac_stats_string),
+				 data);
+	p = hclge_comm_get_stats(&hdev->hw_stats.all_32_bit_stats,
+				 g_all_32bit_stats_string,
+				 ARRAY_SIZE(g_all_32bit_stats_string),
+				 p);
+	p = hclge_comm_get_stats(&hdev->hw_stats.all_64_bit_stats,
+				 g_all_64bit_stats_string,
+				 ARRAY_SIZE(g_all_64bit_stats_string),
+				 p);
+	p = hclge_tqps_get_stats(handle, p);
+}
+
+static int hclge_parse_func_status(struct hclge_dev *hdev,
+				   struct hclge_func_status *status)
+{
+	if (!(status->pf_state & HCLGE_PF_STATE_DONE))
+		return -EINVAL;
+
+	/* Set the pf to main pf */
+	if (status->pf_state & HCLGE_PF_STATE_MAIN)
+		hdev->flag |= HCLGE_FLAG_MAIN;
+	else
+		hdev->flag &= ~HCLGE_FLAG_MAIN;
+
+	hdev->num_req_vfs = status->vf_num / status->pf_num;
+	return 0;
+}
+
+static int hclge_query_function_status(struct hclge_dev *hdev)
+{
+	struct hclge_func_status *req;
+	struct hclge_desc desc;
+	int timeout = 0;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_QUERY_FUNC_STATUS, true);
+	req = (struct hclge_func_status *)desc.data;
+
+	do {
+		ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+		if (ret) {
+			dev_err(&hdev->pdev->dev,
+				"query function status failed %d.\n",
+				ret);
+
+			return ret;
+		}
+
+		/* Check pf reset is done */
+		if (req->pf_state)
+			break;
+		usleep_range(1000, 2000);
+	} while (timeout++ < 5);
+
+	ret = hclge_parse_func_status(hdev, req);
+
+	return ret;
+}
+
+static int hclge_query_pf_resource(struct hclge_dev *hdev)
+{
+	struct hclge_pf_res *req;
+	struct hclge_desc desc;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_QUERY_PF_RSRC, true);
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"query pf resource failed %d.\n", ret);
+		return ret;
+	}
+
+	req = (struct hclge_pf_res *)desc.data;
+	hdev->num_tqps = __le16_to_cpu(req->tqp_num);
+	hdev->pkt_buf_size = __le16_to_cpu(req->buf_size) << HCLGE_BUF_UNIT_S;
+
+	if (hnae_get_bit(hdev->ae_dev->flag, HNAE_DEV_SUPPORT_ROCE_B)) {
+		hdev->num_roce_msix =
+		hnae_get_field(__le16_to_cpu(req->pf_intr_vector_number),
+			       HCLGE_PF_VEC_NUM_M, HCLGE_PF_VEC_NUM_S);
+
+		/* PF should have NIC vectors and Roce vectors,
+		 * NIC vectors are queued before Roce vectors.
+		 */
+		hdev->num_msi = hdev->num_roce_msix  + HCLGE_ROCE_VECTOR_OFFSET;
+	} else {
+		hdev->num_msi =
+		hnae_get_field(__le16_to_cpu(req->pf_intr_vector_number),
+			       HCLGE_PF_VEC_NUM_M, HCLGE_PF_VEC_NUM_S);
+	}
+
+	return 0;
+}
+
+static int hclge_parse_speed(int speed_cmd, int *speed)
+{
+	switch (speed_cmd) {
+	case 6:
+		*speed = HCLGE_MAC_SPEED_10M;
+		break;
+	case 7:
+		*speed = HCLGE_MAC_SPEED_100M;
+		break;
+	case 0:
+		*speed = HCLGE_MAC_SPEED_1G;
+		break;
+	case 1:
+		*speed = HCLGE_MAC_SPEED_10G;
+		break;
+	case 2:
+		*speed = HCLGE_MAC_SPEED_25G;
+		break;
+	case 3:
+		*speed = HCLGE_MAC_SPEED_40G;
+		break;
+	case 4:
+		*speed = HCLGE_MAC_SPEED_50G;
+		break;
+	case 5:
+		*speed = HCLGE_MAC_SPEED_100G;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void hclge_parse_cfg(struct hclge_cfg *cfg, struct hclge_desc *desc)
+{
+	struct hclge_cfg_param *req;
+	u64 mac_addr_tmp_high;
+	u64 mac_addr_tmp;
+	int i;
+
+	req = (struct hclge_cfg_param *)desc[0].data;
+
+	/* get the configuration */
+	cfg->vmdq_vport_num = hnae_get_field(__le32_to_cpu(req->param[0]),
+					     HCLGE_CFG_VMDQ_M,
+					     HCLGE_CFG_VMDQ_S);
+	cfg->tc_num = hnae_get_field(__le32_to_cpu(req->param[0]),
+				     HCLGE_CFG_TC_NUM_M, HCLGE_CFG_TC_NUM_S);
+	cfg->tqp_desc_num = hnae_get_field(__le32_to_cpu(req->param[0]),
+					   HCLGE_CFG_TQP_DESC_N_M,
+					   HCLGE_CFG_TQP_DESC_N_S);
+
+	cfg->phy_addr = hnae_get_field(__le32_to_cpu(req->param[1]),
+				       HCLGE_CFG_PHY_ADDR_M,
+				       HCLGE_CFG_PHY_ADDR_S);
+	cfg->media_type = hnae_get_field(__le32_to_cpu(req->param[1]),
+					 HCLGE_CFG_MEDIA_TP_M,
+					 HCLGE_CFG_MEDIA_TP_S);
+	cfg->rx_buf_len = hnae_get_field(__le32_to_cpu(req->param[1]),
+					 HCLGE_CFG_RX_BUF_LEN_M,
+					 HCLGE_CFG_RX_BUF_LEN_S);
+	/* get mac_address */
+	mac_addr_tmp = __le32_to_cpu(req->param[2]);
+	mac_addr_tmp_high = hnae_get_field(__le32_to_cpu(req->param[3]),
+					   HCLGE_CFG_MAC_ADDR_H_M,
+					   HCLGE_CFG_MAC_ADDR_H_S);
+
+	mac_addr_tmp |= (mac_addr_tmp_high << 31) << 1;
+
+	cfg->default_speed = hnae_get_field(__le32_to_cpu(req->param[3]),
+					    HCLGE_CFG_DEFAULT_SPEED_M,
+					    HCLGE_CFG_DEFAULT_SPEED_S);
+	for (i = 0; i < ETH_ALEN; i++)
+		cfg->mac_addr[i] = (mac_addr_tmp >> (8 * i)) & 0xff;
+
+	req = (struct hclge_cfg_param *)desc[1].data;
+	cfg->numa_node_map = __le32_to_cpu(req->param[0]);
+}
+
+/* hclge_get_cfg: query the static parameter from flash
+ * @hdev: pointer to struct hclge_dev
+ * @hcfg: the config structure to be getted
+ */
+static int hclge_get_cfg(struct hclge_dev *hdev, struct hclge_cfg *hcfg)
+{
+	struct hclge_desc desc[HCLGE_PF_CFG_DESC_NUM];
+	struct hclge_cfg_param *req;
+	int i, ret;
+
+	for (i = 0; i < HCLGE_PF_CFG_DESC_NUM; i++) {
+		req = (struct hclge_cfg_param *)desc[i].data;
+		hclge_cmd_setup_basic_desc(&desc[i], HCLGE_OPC_GET_CFG_PARAM,
+					   true);
+		hnae_set_field(req->offset, HCLGE_CFG_OFFSET_M,
+			       HCLGE_CFG_OFFSET_S, i * HCLGE_CFG_RD_LEN_BYTES);
+		/* Len should be united by 4 bytes when send to hardware */
+		hnae_set_field(req->offset, HCLGE_CFG_RD_LEN_M,
+			       HCLGE_CFG_RD_LEN_S,
+			       HCLGE_CFG_RD_LEN_BYTES / HCLGE_CFG_RD_LEN_UNIT);
+		req->offset = cpu_to_le32(req->offset);
+	}
+
+	ret = hclge_cmd_send(&hdev->hw, desc, HCLGE_PF_CFG_DESC_NUM);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"get config failed %d.\n", ret);
+		return ret;
+	}
+
+	hclge_parse_cfg(hcfg, desc);
+	return 0;
+}
+
+static int hclge_get_cap(struct hclge_dev *hdev)
+{
+	int ret;
+
+	ret = hclge_query_function_status(hdev);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"query function status error %d.\n", ret);
+		return ret;
+	}
+
+	/* get pf resource */
+	ret = hclge_query_pf_resource(hdev);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"query pf resource error %d.\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int hclge_configure(struct hclge_dev *hdev)
+{
+	struct hclge_cfg cfg;
+	int ret, i;
+
+	ret = hclge_get_cfg(hdev, &cfg);
+	if (ret) {
+		dev_err(&hdev->pdev->dev, "get mac mode error %d.\n", ret);
+		return ret;
+	}
+
+	hdev->num_vmdq_vport = cfg.vmdq_vport_num;
+	hdev->base_tqp_pid = 0;
+	hdev->rss_size_max = 1;
+	hdev->rx_buf_len = cfg.rx_buf_len;
+	for (i = 0; i < ETH_ALEN; i++)
+		hdev->hw.mac.mac_addr[i] = cfg.mac_addr[i];
+	hdev->hw.mac.media_type = cfg.media_type;
+	hdev->num_desc = cfg.tqp_desc_num;
+	hdev->tm_info.num_pg = 1;
+	hdev->tm_info.num_tc = cfg.tc_num;
+	hdev->tm_info.hw_pfc_map = 0;
+
+	ret = hclge_parse_speed(cfg.default_speed, &hdev->hw.mac.speed);
+	if (ret) {
+		dev_err(&hdev->pdev->dev, "Get wrong speed ret=%d.\n", ret);
+		return ret;
+	}
+
+	if ((hdev->tm_info.num_tc > HNAE3_MAX_TC) ||
+	    (hdev->tm_info.num_tc < 1)) {
+		dev_warn(&hdev->pdev->dev, "TC num = %d.\n",
+			 hdev->tm_info.num_tc);
+		hdev->tm_info.num_tc = 1;
+	}
+
+	/* Currently not support uncontiuous tc */
+	for (i = 0; i < cfg.tc_num; i++)
+		hnae_set_bit(hdev->hw_tc_map, i, 1);
+
+	if (!hdev->num_vmdq_vport && !hdev->num_req_vfs)
+		hdev->tx_sch_mode = HCLGE_FLAG_TC_BASE_SCH_MODE;
+	else
+		hdev->tx_sch_mode = HCLGE_FLAG_VNET_BASE_SCH_MODE;
+
+	return ret;
+}
+
+static int hclge_config_tso(struct hclge_dev *hdev, int tso_mss_min,
+			    int tso_mss_max)
+{
+	struct hclge_cfg_tso_status *req;
+	struct hclge_desc desc;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_TSO_GENERIC_CONFIG, false);
+
+	req = (struct hclge_cfg_tso_status *)desc.data;
+	hnae_set_field(req->tso_mss_min, HCLGE_TSO_MSS_MIN_M,
+		       HCLGE_TSO_MSS_MIN_S, tso_mss_min);
+	hnae_set_field(req->tso_mss_max, HCLGE_TSO_MSS_MIN_M,
+		       HCLGE_TSO_MSS_MIN_S, tso_mss_max);
+
+	return hclge_cmd_send(&hdev->hw, &desc, 1);
+}
+
+static int hclge_alloc_tqps(struct hclge_dev *hdev)
+{
+	struct hclge_tqp *tqp;
+	int i;
+
+	hdev->htqp = devm_kcalloc(&hdev->pdev->dev, hdev->num_tqps,
+				  sizeof(struct hclge_tqp), GFP_KERNEL);
+	if (!hdev->htqp)
+		return -ENOMEM;
+
+	tqp = hdev->htqp;
+
+	for (i = 0; i < hdev->num_tqps; i++) {
+		tqp->dev = &hdev->pdev->dev;
+		tqp->index = i;
+
+		tqp->q.ae_algo = &ae_algo;
+		tqp->q.buf_size = hdev->rx_buf_len;
+		tqp->q.desc_num = hdev->num_desc;
+		tqp->q.io_base = hdev->hw.io_base + HCLGE_TQP_REG_OFFSET +
+			i * HCLGE_TQP_REG_SIZE;
+
+		tqp++;
+	}
+
+	return 0;
+}
+
+static int hclge_map_tqps_to_func(struct hclge_dev *hdev, u16 func_id,
+				  u16 tqp_pid, u16 tqp_vid, bool is_pf)
+{
+	struct hclge_tqp_map *req;
+	struct hclge_desc desc;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_SET_TQP_MAP, false);
+
+	req = (struct hclge_tqp_map *)desc.data;
+	req->tqp_id = cpu_to_le16(tqp_pid);
+	req->tqp_vf = cpu_to_le16(func_id);
+	req->tqp_flag = !is_pf << HCLGE_TQP_MAP_TYPE_B |
+			1 << HCLGE_TQP_MAP_EN_B;
+	req->tqp_vid = cpu_to_le16(tqp_vid);
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev, "TQP map failed %d.\n",
+			ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int  hclge_assign_tqp(struct hclge_vport *vport,
+			     struct hnae3_queue **tqp, u16 num_tqps)
+{
+	struct hclge_dev *hdev = vport->back;
+	int i, alloced, func_id, ret;
+	bool is_pf;
+
+	func_id = vport->vport_id;
+	is_pf = (vport->vport_id == 0) ? true : false;
+
+	for (i = 0, alloced = 0; i < hdev->num_tqps &&
+	     alloced < num_tqps; i++) {
+		if (!hdev->htqp[i].alloced) {
+			hdev->htqp[i].q.handle = &vport->nic;
+			hdev->htqp[i].q.tqp_index = alloced;
+			tqp[alloced] = &hdev->htqp[i].q;
+			hdev->htqp[i].alloced = true;
+			ret = hclge_map_tqps_to_func(hdev, func_id,
+						     hdev->htqp[i].index,
+						     alloced, is_pf);
+			if (ret)
+				return ret;
+
+			alloced++;
+		}
+	}
+	vport->alloc_tqps = num_tqps;
+
+	return 0;
+}
+
+static int hclge_knic_setup(struct hclge_vport *vport, u16 num_tqps)
+{
+	struct hnae3_handle *nic = &vport->nic;
+	struct hnae3_knic_private_info *kinfo = &nic->kinfo;
+	struct hclge_dev *hdev = vport->back;
+	int i, ret;
+
+	kinfo->num_desc = hdev->num_desc;
+	kinfo->rx_buf_len = hdev->rx_buf_len;
+	kinfo->num_tc = min_t(u16, num_tqps, hdev->tm_info.num_tc);
+	kinfo->rss_size
+		= min_t(u16, hdev->rss_size_max, num_tqps / kinfo->num_tc);
+	kinfo->num_tqps = kinfo->rss_size * kinfo->num_tc;
+
+	for (i = 0; i < HNAE3_MAX_TC; i++) {
+		if (hdev->hw_tc_map & BIT(i)) {
+			kinfo->tc_info[i].enable = true;
+			kinfo->tc_info[i].tqp_offset = i * kinfo->rss_size;
+			kinfo->tc_info[i].tqp_count = kinfo->rss_size;
+			kinfo->tc_info[i].tc = i;
+		} else {
+			/* Set to default queue if TC is disable */
+			kinfo->tc_info[i].enable = false;
+			kinfo->tc_info[i].tqp_offset = 0;
+			kinfo->tc_info[i].tqp_count = 1;
+			kinfo->tc_info[i].tc = 0;
+		}
+	}
+
+	kinfo->tqp = devm_kcalloc(&hdev->pdev->dev, kinfo->num_tqps,
+				  sizeof(struct hnae3_queue *), GFP_KERNEL);
+	if (!kinfo->tqp)
+		return -ENOMEM;
+
+	ret = hclge_assign_tqp(vport, kinfo->tqp, kinfo->num_tqps);
+	if (ret) {
+		dev_err(&hdev->pdev->dev, "fail to assign TQPs %d.\n", ret);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void hclge_unic_setup(struct hclge_vport *vport, u16 num_tqps)
+{
+	/* this would be initialized later */
+}
+
+static int hclge_vport_setup(struct hclge_vport *vport, u16 num_tqps)
+{
+	struct hnae3_handle *nic = &vport->nic;
+	struct hclge_dev *hdev = vport->back;
+	int ret;
+
+	nic->pdev = hdev->pdev;
+	nic->ae_algo = &ae_algo;
+	nic->numa_node_mask = hdev->numa_node_mask;
+
+	if (hdev->ae_dev->dev_type == HNAE3_DEV_KNIC) {
+		ret = hclge_knic_setup(vport, num_tqps);
+		if (ret) {
+			dev_err(&hdev->pdev->dev, "knic setup failed %d\n",
+				ret);
+			return ret;
+		}
+	} else {
+		hclge_unic_setup(vport, num_tqps);
+	}
+
+	return 0;
+}
+
+static int hclge_alloc_vport(struct hclge_dev *hdev)
+{
+	struct pci_dev *pdev = hdev->pdev;
+	struct hclge_vport *vport;
+	u32 tqp_main_vport;
+	u32 tqp_per_vport;
+	int num_vport, i;
+	int ret;
+
+	/* We need to alloc a vport for main NIC of PF */
+	num_vport = hdev->num_vmdq_vport + hdev->num_req_vfs + 1;
+
+	if (hdev->num_tqps < num_vport)
+		num_vport = hdev->num_tqps;
+
+	/* Alloc the same number of TQPs for every vport */
+	tqp_per_vport = hdev->num_tqps / num_vport;
+	tqp_main_vport = tqp_per_vport + hdev->num_tqps % num_vport;
+
+	vport = devm_kcalloc(&pdev->dev, num_vport, sizeof(struct hclge_vport),
+			     GFP_KERNEL);
+	if (!vport)
+		return -ENOMEM;
+
+	hdev->vport = vport;
+	hdev->num_alloc_vport = num_vport;
+
+#ifdef CONFIG_PCI_IOV
+	/* Enable SRIOV */
+	if (hdev->num_req_vfs) {
+		dev_info(&pdev->dev, "active VFs(%d) found, enabling SRIOV\n",
+			 hdev->num_req_vfs);
+		ret = pci_enable_sriov(hdev->pdev, hdev->num_req_vfs);
+		if (ret) {
+			hdev->num_alloc_vfs = 0;
+			dev_err(&pdev->dev, "SRIOV enable failed %d\n",
+				ret);
+			return ret;
+		}
+	}
+	hdev->num_alloc_vfs = hdev->num_req_vfs;
+#endif
+
+	for (i = 0; i < num_vport; i++) {
+		vport->back = hdev;
+		vport->vport_id = i;
+
+		if (i == 0)
+			ret = hclge_vport_setup(vport, tqp_main_vport);
+		else
+			ret = hclge_vport_setup(vport, tqp_per_vport);
+		if (ret) {
+			dev_err(&pdev->dev,
+				"vport setup failed for vport %d, %d\n",
+				i, ret);
+			return ret;
+		}
+
+		vport++;
+	}
+
+	return 0;
+}
+
+static int  hclge_cmd_alloc_tx_buff(struct hclge_dev *hdev, u16 buf_size)
+{
+/* TX buffer size is unit by 128 byte */
+#define HCLGE_BUF_SIZE_UNIT_SHIFT	7
+#define HCLGE_BUF_SIZE_UPDATE_EN_MSK	BIT(15)
+	struct hclge_tx_buff_alloc *req;
+	struct hclge_desc desc;
+	int ret;
+	u8 i;
+
+	req = (struct hclge_tx_buff_alloc *)desc.data;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_TX_BUFF_ALLOC, 0);
+	for (i = 0; i < HCLGE_TC_NUM; i++)
+		req->tx_pkt_buff[i] =
+			cpu_to_le16((buf_size >> HCLGE_BUF_SIZE_UNIT_SHIFT) |
+				     HCLGE_BUF_SIZE_UPDATE_EN_MSK);
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev, "tx buffer alloc cmd failed %d.\n",
+			ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int hclge_tx_buffer_alloc(struct hclge_dev *hdev, u32 buf_size)
+{
+	int ret = hclge_cmd_alloc_tx_buff(hdev, buf_size);
+
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"tx buffer alloc failed %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int hclge_get_tc_num(struct hclge_dev *hdev)
+{
+	int i, cnt = 0;
+
+	for (i = 0; i < HCLGE_MAX_TC_NUM; i++)
+		if (hdev->hw_tc_map & BIT(i))
+			cnt++;
+	return cnt;
+}
+
+static int hclge_get_pfc_enalbe_num(struct hclge_dev *hdev)
+{
+	int i, cnt = 0;
+
+	for (i = 0; i < HCLGE_MAX_TC_NUM; i++)
+		if (hdev->hw_tc_map & BIT(i) &&
+		    hdev->tm_info.hw_pfc_map & BIT(i))
+			cnt++;
+	return cnt;
+}
+
+/* Get the number of pfc enabled TCs, which have private buffer */
+static int hclge_get_pfc_priv_num(struct hclge_dev *hdev)
+{
+	struct hclge_priv_buf *priv;
+	int i, cnt = 0;
+
+	for (i = 0; i < HCLGE_MAX_TC_NUM; i++) {
+		priv = &hdev->priv_buf[i];
+		if ((hdev->tm_info.hw_pfc_map & BIT(i)) &&
+		    priv->enable)
+			cnt++;
+	}
+
+	return cnt;
+}
+
+/* Get the number of pfc disabled TCs, which have private buffer */
+static int hclge_get_no_pfc_priv_num(struct hclge_dev *hdev)
+{
+	struct hclge_priv_buf *priv;
+	int i, cnt = 0;
+
+	for (i = 0; i < HCLGE_MAX_TC_NUM; i++) {
+		priv = &hdev->priv_buf[i];
+		if (hdev->hw_tc_map & BIT(i) &&
+		    !(hdev->tm_info.hw_pfc_map & BIT(i)) &&
+		    priv->enable)
+			cnt++;
+	}
+
+	return cnt;
+}
+
+static u32 hclge_get_rx_priv_buff_alloced(struct hclge_dev *hdev)
+{
+	struct hclge_priv_buf *priv;
+	u32 rx_priv = 0;
+	int i;
+
+	for (i = 0; i < HCLGE_MAX_TC_NUM; i++) {
+		priv = &hdev->priv_buf[i];
+		if (priv->enable)
+			rx_priv += priv->buf_size;
+	}
+	return rx_priv;
+}
+
+static bool  hclge_is_rx_buf_ok(struct hclge_dev *hdev, u32 rx_all)
+{
+	u32 shared_buf_min, shared_buf_tc, shared_std;
+	int tc_num, pfc_enable_num;
+	u32 shared_buf;
+	u32 rx_priv;
+	int i;
+
+	tc_num = hclge_get_tc_num(hdev);
+	pfc_enable_num = hclge_get_pfc_enalbe_num(hdev);
+
+	shared_buf_min = 2 * hdev->mps + HCLGE_DEFAULT_DV;
+	shared_buf_tc = pfc_enable_num * hdev->mps +
+			(tc_num - pfc_enable_num) * hdev->mps / 2 +
+			hdev->mps;
+	shared_std = max_t(u32, shared_buf_min, shared_buf_tc);
+
+	rx_priv = hclge_get_rx_priv_buff_alloced(hdev);
+	if (rx_all <= rx_priv + shared_std)
+		return false;
+
+	shared_buf = rx_all - rx_priv;
+	hdev->s_buf.buf_size = shared_buf;
+	hdev->s_buf.self.high = shared_buf;
+	hdev->s_buf.self.low =  2 * hdev->mps;
+
+	for (i = 0; i < HCLGE_MAX_TC_NUM; i++) {
+		if ((hdev->hw_tc_map & BIT(i)) &&
+		    (hdev->tm_info.hw_pfc_map & BIT(i))) {
+			hdev->s_buf.tc_thrd[i].low = hdev->mps;
+			hdev->s_buf.tc_thrd[i].high = 2 * hdev->mps;
+		} else {
+			hdev->s_buf.tc_thrd[i].low = 0;
+			hdev->s_buf.tc_thrd[i].high = hdev->mps;
+		}
+	}
+
+	return true;
+}
+
+/* hclge_rx_buffer_calc: calculate the rx private buffer size for all TCs
+ * @hdev: pointer to struct hclge_dev
+ * @tx_size: the allocated tx buffer for all TCs
+ * @return: 0: calculate sucessful, negative: fail
+ */
+int hclge_rx_buffer_calc(struct hclge_dev *hdev, u32 tx_size)
+{
+	u32 rx_all = hdev->pkt_buf_size - tx_size;
+	int no_pfc_priv_num, pfc_priv_num;
+	struct hclge_priv_buf *priv;
+	int i;
+
+	/* step 1, try to alloc private buffer for all enabled tc */
+	for (i = 0; i < HCLGE_MAX_TC_NUM; i++) {
+		priv = &hdev->priv_buf[i];
+		if (hdev->hw_tc_map & BIT(i)) {
+			priv->enable = 1;
+			if (hdev->tm_info.hw_pfc_map & BIT(i)) {
+				priv->wl.low = hdev->mps;
+				priv->wl.high = priv->wl.low + hdev->mps;
+				priv->buf_size = priv->wl.high +
+						HCLGE_DEFAULT_DV;
+			} else {
+				priv->wl.low = 0;
+				priv->wl.high = 2 * hdev->mps;
+				priv->buf_size = priv->wl.high;
+			}
+		}
+	}
+
+	if (hclge_is_rx_buf_ok(hdev, rx_all))
+		return 0;
+
+	/* step 2, try to decrease the buffer size of
+	 * no pfc TC's private buffer
+	 */
+	for (i = 0; i < HCLGE_MAX_TC_NUM; i++) {
+		priv = &hdev->priv_buf[i];
+
+		if (hdev->hw_tc_map & BIT(i))
+			priv->enable = 1;
+
+		if (hdev->tm_info.hw_pfc_map & BIT(i)) {
+			priv->wl.low = 128;
+			priv->wl.high = priv->wl.low + hdev->mps;
+			priv->buf_size = priv->wl.high + HCLGE_DEFAULT_DV;
+		} else {
+			priv->wl.low = 0;
+			priv->wl.high = hdev->mps;
+			priv->buf_size = priv->wl.high;
+		}
+	}
+
+	if (hclge_is_rx_buf_ok(hdev, rx_all))
+		return 0;
+
+	/* step 3, try to reduce the number of pfc disabled TCs,
+	 * which have private buffer
+	 */
+	/* get the total no pfc enable TC number, which have private buffer */
+	no_pfc_priv_num = hclge_get_no_pfc_priv_num(hdev);
+
+	/* let the last to be cleared first */
+	for (i = HCLGE_MAX_TC_NUM - 1; i >= 0; i--) {
+		priv = &hdev->priv_buf[i];
+
+		if (hdev->hw_tc_map & BIT(i) &&
+		    !(hdev->tm_info.hw_pfc_map & BIT(i))) {
+			/* Clear the no pfc TC private buffer */
+			priv->wl.low = 0;
+			priv->wl.high = 0;
+			priv->buf_size = 0;
+			priv->enable = 0;
+			no_pfc_priv_num--;
+		}
+
+		if (hclge_is_rx_buf_ok(hdev, rx_all) ||
+		    no_pfc_priv_num == 0)
+			break;
+	}
+
+	if (hclge_is_rx_buf_ok(hdev, rx_all))
+		return 0;
+
+	/* step 4, try to reduce the number of pfc enabled TCs
+	 * which have private buffer.
+	 */
+	pfc_priv_num = hclge_get_pfc_priv_num(hdev);
+
+	/* let the last to be cleared first */
+	for (i = HCLGE_MAX_TC_NUM - 1; i >= 0; i--) {
+		priv = &hdev->priv_buf[i];
+
+		if (hdev->hw_tc_map & BIT(i) &&
+		    hdev->tm_info.hw_pfc_map & BIT(i)) {
+			/* Reduce the number of pfc TC with private buffer */
+			priv->wl.low = 0;
+			priv->enable = 0;
+			priv->wl.high = 0;
+			priv->buf_size = 0;
+			pfc_priv_num--;
+		}
+
+		if (hclge_is_rx_buf_ok(hdev, rx_all) ||
+		    pfc_priv_num == 0)
+			break;
+	}
+	if (hclge_is_rx_buf_ok(hdev, rx_all))
+		return 0;
+
+	return -ENOMEM;
+}
+
+static int hclge_rx_priv_buf_alloc(struct hclge_dev *hdev)
+{
+	struct hclge_rx_priv_buff *req;
+	struct hclge_desc desc;
+	int ret;
+	int i;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_RX_PRIV_BUFF_ALLOC, false);
+	req = (struct hclge_rx_priv_buff *)desc.data;
+
+	/* Alloc private buffer TCs */
+	for (i = 0; i < HCLGE_MAX_TC_NUM; i++) {
+		struct hclge_priv_buf *priv = &hdev->priv_buf[i];
+
+		req->buf_num[i] =
+			cpu_to_le16(priv->buf_size >> HCLGE_BUF_UNIT_S);
+		req->buf_num[i] |=
+			cpu_to_le16(true << HCLGE_TC0_PRI_BUF_EN_B);
+	}
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"rx private buffer alloc cmd failed %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+#define HCLGE_PRIV_ENABLE(a) ((a) > 0 ? 1 : 0)
+
+static int hclge_rx_priv_wl_config(struct hclge_dev *hdev)
+{
+	struct hclge_rx_priv_wl_buf *req;
+	struct hclge_priv_buf *priv;
+	struct hclge_desc desc[2];
+	int i, j;
+	int ret;
+
+	for (i = 0; i < 2; i++) {
+		hclge_cmd_setup_basic_desc(&desc[i], HCLGE_OPC_RX_PRIV_WL_ALLOC,
+					   false);
+		req = (struct hclge_rx_priv_wl_buf *)desc[i].data;
+
+		/* The first descriptor set the NEXT bit to 1 */
+		if (i == 0)
+			desc[i].flag |= cpu_to_le16(HCLGE_CMD_FLAG_NEXT);
+		else
+			desc[i].flag &= ~cpu_to_le16(HCLGE_CMD_FLAG_NEXT);
+
+		for (j = 0; j < HCLGE_TC_NUM_ONE_DESC; j++) {
+			priv = &hdev->priv_buf[i * HCLGE_TC_NUM_ONE_DESC + j];
+			req->tc_wl[j].high =
+				cpu_to_le16(priv->wl.high >> HCLGE_BUF_UNIT_S);
+			req->tc_wl[j].high |=
+				cpu_to_le16(HCLGE_PRIV_ENABLE(priv->wl.high) <<
+					    HCLGE_RX_PRIV_EN_B);
+			req->tc_wl[j].low =
+				cpu_to_le16(priv->wl.low >> HCLGE_BUF_UNIT_S);
+			req->tc_wl[j].low |=
+				cpu_to_le16(HCLGE_PRIV_ENABLE(priv->wl.low) <<
+					    HCLGE_RX_PRIV_EN_B);
+		}
+	}
+
+	/* Send 2 descriptor at one time */
+	ret = hclge_cmd_send(&hdev->hw, desc, 2);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"rx private waterline config cmd failed %d\n",
+			ret);
+		return ret;
+	}
+	return 0;
+}
+
+static int hclge_common_thrd_config(struct hclge_dev *hdev)
+{
+	struct hclge_shared_buf *s_buf = &hdev->s_buf;
+	struct hclge_rx_com_thrd *req;
+	struct hclge_desc desc[2];
+	struct hclge_tc_thrd *tc;
+	int i, j;
+	int ret;
+
+	for (i = 0; i < 2; i++) {
+		hclge_cmd_setup_basic_desc(&desc[i],
+					   HCLGE_OPC_RX_COM_THRD_ALLOC, false);
+		req = (struct hclge_rx_com_thrd *)&desc[i].data;
+
+		/* The first descriptor set the NEXT bit to 1 */
+		if (i == 0)
+			desc[i].flag |= cpu_to_le16(HCLGE_CMD_FLAG_NEXT);
+		else
+			desc[i].flag &= ~cpu_to_le16(HCLGE_CMD_FLAG_NEXT);
+
+		for (j = 0; j < HCLGE_TC_NUM_ONE_DESC; j++) {
+			tc = &s_buf->tc_thrd[i * HCLGE_TC_NUM_ONE_DESC + j];
+
+			req->com_thrd[j].high =
+				cpu_to_le16(tc->high >> HCLGE_BUF_UNIT_S);
+			req->com_thrd[j].high |=
+				cpu_to_le16(HCLGE_PRIV_ENABLE(tc->high) <<
+					    HCLGE_RX_PRIV_EN_B);
+			req->com_thrd[j].low =
+				cpu_to_le16(tc->low >> HCLGE_BUF_UNIT_S);
+			req->com_thrd[j].low |=
+				cpu_to_le16(HCLGE_PRIV_ENABLE(tc->low) <<
+					    HCLGE_RX_PRIV_EN_B);
+		}
+	}
+
+	/* Send 2 descriptors at one time */
+	ret = hclge_cmd_send(&hdev->hw, desc, 2);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"common threshold config cmd failed %d\n", ret);
+		return ret;
+	}
+	return 0;
+}
+
+static int hclge_common_wl_config(struct hclge_dev *hdev)
+{
+	struct hclge_shared_buf *buf = &hdev->s_buf;
+	struct hclge_rx_com_wl *req;
+	struct hclge_desc desc;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_RX_COM_WL_ALLOC, false);
+
+	req = (struct hclge_rx_com_wl *)desc.data;
+	req->com_wl.high = cpu_to_le16(buf->self.high >> HCLGE_BUF_UNIT_S);
+	req->com_wl.high |=
+		cpu_to_le16(HCLGE_PRIV_ENABLE(buf->self.high) <<
+			    HCLGE_RX_PRIV_EN_B);
+
+	req->com_wl.low = cpu_to_le16(buf->self.low >> HCLGE_BUF_UNIT_S);
+	req->com_wl.low |=
+		cpu_to_le16(HCLGE_PRIV_ENABLE(buf->self.low) <<
+			    HCLGE_RX_PRIV_EN_B);
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"common waterline config cmd failed %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+int hclge_buffer_alloc(struct hclge_dev *hdev)
+{
+	u32 tx_buf_size = HCLGE_DEFAULT_TX_BUF;
+	int ret;
+
+	hdev->priv_buf = devm_kmalloc_array(&hdev->pdev->dev, HCLGE_MAX_TC_NUM,
+					    sizeof(struct hclge_priv_buf),
+					    GFP_KERNEL | __GFP_ZERO);
+	if (!hdev->priv_buf)
+		return -ENOMEM;
+
+	ret = hclge_tx_buffer_alloc(hdev, tx_buf_size);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"could not alloc tx buffers %d\n", ret);
+		return ret;
+	}
+
+	ret = hclge_rx_buffer_calc(hdev, tx_buf_size);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"could not calc rx priv buffer size for all TCs %d\n",
+			ret);
+		return ret;
+	}
+
+	ret = hclge_rx_priv_buf_alloc(hdev);
+	if (ret) {
+		dev_err(&hdev->pdev->dev, "could not alloc rx priv buffer %d\n",
+			ret);
+		return ret;
+	}
+
+	ret = hclge_rx_priv_wl_config(hdev);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"could not configure rx private waterline %d\n", ret);
+		return ret;
+	}
+
+	ret = hclge_common_thrd_config(hdev);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"could not configure common threshold %d\n", ret);
+		return ret;
+	}
+
+	ret = hclge_common_wl_config(hdev);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"could not configure common waterline %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int hclge_init_roce_base_info(struct hclge_vport *vport)
+{
+	struct hnae3_handle *roce = &vport->roce;
+	struct hnae3_handle *nic = &vport->nic;
+
+	roce->rinfo.num_vectors = vport->back->num_roce_msix;
+
+	if (vport->back->num_msi_left < vport->roce.rinfo.num_vectors ||
+	    vport->back->num_msi_left == 0)
+		return -EINVAL;
+
+	roce->rinfo.base_vector = vport->back->roce_base_vector;
+
+	roce->rinfo.netdev = nic->kinfo.netdev;
+	roce->rinfo.roce_io_base = vport->back->hw.io_base;
+
+	roce->pdev = nic->pdev;
+	roce->ae_algo = nic->ae_algo;
+	roce->numa_node_mask = nic->numa_node_mask;
+
+	return 0;
+}
+
+static int hclge_init_msix(struct hclge_dev *hdev)
+{
+	struct pci_dev *pdev = hdev->pdev;
+	int ret, i;
+
+	hdev->msix_entries = devm_kcalloc(&pdev->dev, hdev->num_msi,
+					  sizeof(struct msix_entry),
+					  GFP_KERNEL);
+	if (!hdev->msix_entries)
+		return -ENOMEM;
+
+	hdev->vector_status = devm_kcalloc(&pdev->dev, hdev->num_msi,
+					   sizeof(u16), GFP_KERNEL);
+	if (!hdev->vector_status)
+		return -ENOMEM;
+
+	for (i = 0; i < hdev->num_msi; i++) {
+		hdev->msix_entries[i].entry = i;
+		hdev->vector_status[i] = HCLGE_INVALID_VPORT;
+	}
+
+	hdev->num_msi_left = hdev->num_msi;
+	hdev->base_msi_vector = hdev->pdev->irq;
+	hdev->roce_base_vector = hdev->base_msi_vector +
+				HCLGE_ROCE_VECTOR_OFFSET;
+
+	ret = pci_enable_msix_range(hdev->pdev, hdev->msix_entries,
+				    hdev->num_msi, hdev->num_msi);
+	if (ret < 0) {
+		dev_info(&hdev->pdev->dev,
+			 "MSI-X vector alloc failed: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int hclge_init_msi(struct hclge_dev *hdev)
+{
+	struct pci_dev *pdev = hdev->pdev;
+	int vectors;
+	int i;
+
+	hdev->vector_status = devm_kcalloc(&pdev->dev, hdev->num_msi,
+					   sizeof(u16), GFP_KERNEL);
+	if (!hdev->vector_status)
+		return -ENOMEM;
+
+	for (i = 0; i < hdev->num_msi; i++)
+		hdev->vector_status[i] = HCLGE_INVALID_VPORT;
+
+	vectors = pci_alloc_irq_vectors(pdev, 1, hdev->num_msi, PCI_IRQ_MSI);
+	if (vectors < 0) {
+		dev_err(&pdev->dev, "MSI vectors enable failed %d\n", vectors);
+		return -EINVAL;
+	}
+	hdev->num_msi = vectors;
+	hdev->num_msi_left = vectors;
+	hdev->base_msi_vector = pdev->irq;
+	hdev->roce_base_vector = hdev->base_msi_vector +
+				HCLGE_ROCE_VECTOR_OFFSET;
+
+	return 0;
+}
+
+static void hclge_check_speed_dup(struct hclge_dev *hdev, int duplex, int speed)
+{
+	struct hclge_mac *mac = &hdev->hw.mac;
+
+	if ((speed == HCLGE_MAC_SPEED_10M) || (speed == HCLGE_MAC_SPEED_100M))
+		mac->duplex = (u8)duplex;
+	else
+		mac->duplex = HCLGE_MAC_FULL;
+
+	mac->speed = speed;
+}
+
+int hclge_cfg_mac_speed_dup(struct hclge_dev *hdev, int speed, u8 duplex)
+{
+	struct hclge_config_mac_speed_dup *req;
+	struct hclge_desc desc;
+	int ret;
+
+	req = (struct hclge_config_mac_speed_dup *)desc.data;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_CONFIG_SPEED_DUP, false);
+
+	hnae_set_bit(req->speed_dup, HCLGE_CFG_DUPLEX_B, !!duplex);
+
+	switch (speed) {
+	case HCLGE_MAC_SPEED_10M:
+		hnae_set_field(req->speed_dup, HCLGE_CFG_SPEED_M,
+			       HCLGE_CFG_SPEED_S, 6);
+		break;
+	case HCLGE_MAC_SPEED_100M:
+		hnae_set_field(req->speed_dup, HCLGE_CFG_SPEED_M,
+			       HCLGE_CFG_SPEED_S, 7);
+		break;
+	case HCLGE_MAC_SPEED_1G:
+		hnae_set_field(req->speed_dup, HCLGE_CFG_SPEED_M,
+			       HCLGE_CFG_SPEED_S, 0);
+		break;
+	case HCLGE_MAC_SPEED_10G:
+		hnae_set_field(req->speed_dup, HCLGE_CFG_SPEED_M,
+			       HCLGE_CFG_SPEED_S, 1);
+		break;
+	case HCLGE_MAC_SPEED_25G:
+		hnae_set_field(req->speed_dup, HCLGE_CFG_SPEED_M,
+			       HCLGE_CFG_SPEED_S, 2);
+		break;
+	case HCLGE_MAC_SPEED_40G:
+		hnae_set_field(req->speed_dup, HCLGE_CFG_SPEED_M,
+			       HCLGE_CFG_SPEED_S, 3);
+		break;
+	case HCLGE_MAC_SPEED_50G:
+		hnae_set_field(req->speed_dup, HCLGE_CFG_SPEED_M,
+			       HCLGE_CFG_SPEED_S, 4);
+		break;
+	case HCLGE_MAC_SPEED_100G:
+		hnae_set_field(req->speed_dup, HCLGE_CFG_SPEED_M,
+			       HCLGE_CFG_SPEED_S, 5);
+		break;
+	default:
+		dev_err(&hdev->pdev->dev, "invalid speed (%d)\n", speed);
+		return -EINVAL;
+	}
+
+	hnae_set_bit(req->mac_change_fec_en, HCLGE_CFG_MAC_SPEED_CHANGE_EN_B,
+		     1);
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"mac speed/duplex config cmd failed %d.\n", ret);
+		return ret;
+	}
+
+	hclge_check_speed_dup(hdev, duplex, speed);
+
+	return 0;
+}
+
+static int hclge_cfg_mac_speed_dup_h(struct hnae3_handle *handle, int speed,
+				     u8 duplex)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+
+	return hclge_cfg_mac_speed_dup(hdev, speed, duplex);
+}
+
+static int hclge_query_mac_an_speed_dup(struct hclge_dev *hdev, int *speed,
+					u8 *duplex)
+{
+	struct hclge_query_an_speed_dup *req;
+	struct hclge_desc desc;
+	int speed_tmp;
+	int ret;
+
+	req = (struct hclge_query_an_speed_dup *)desc.data;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_QUERY_AN_RESULT, true);
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"mac speed/autoneg/duplex query cmd failed %d\n",
+			ret);
+		return ret;
+	}
+
+	*duplex = hnae_get_bit(req->an_syn_dup_speed, HCLGE_QUERY_DUPLEX_B);
+	speed_tmp = hnae_get_field(req->an_syn_dup_speed, HCLGE_QUERY_SPEED_M,
+				   HCLGE_QUERY_SPEED_S);
+
+	ret = hclge_parse_speed(speed_tmp, speed);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"could not parse speed(=%d), %d\n", speed_tmp, ret);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static int hclge_query_autoneg_result(struct hclge_dev *hdev)
+{
+	struct hclge_mac *mac = &hdev->hw.mac;
+	struct hclge_query_an_speed_dup *req;
+	struct hclge_desc desc;
+	int ret;
+
+	req = (struct hclge_query_an_speed_dup *)desc.data;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_QUERY_AN_RESULT, true);
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"autoneg result query cmd failed %d.\n", ret);
+		return ret;
+	}
+
+	mac->autoneg = hnae_get_bit(req->an_syn_dup_speed, HCLGE_QUERY_AN_B);
+
+	return 0;
+}
+
+static int hclge_set_autoneg_en(struct hclge_dev *hdev, bool enable)
+{
+	struct hclge_config_auto_neg *req;
+	struct hclge_desc desc;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_CONFIG_AN_MODE, false);
+
+	req = (struct hclge_config_auto_neg *)desc.data;
+	hnae_set_bit(req->cfg_an_cmd_flag, HCLGE_MAC_CFG_AN_EN_B, !!enable);
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev, "auto neg set cmd failed %d.\n",
+			ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int hclge_set_autoneg(struct hnae3_handle *handle, bool enable)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+
+	return hclge_set_autoneg_en(hdev, enable);
+}
+
+static int hclge_get_autoneg(struct hnae3_handle *handle)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+
+	hclge_query_autoneg_result(hdev);
+
+	return hdev->hw.mac.autoneg;
+}
+
+static int hclge_mac_init(struct hclge_dev *hdev)
+{
+	struct hclge_mac *mac = &hdev->hw.mac;
+	int ret;
+
+	ret = hclge_cfg_mac_speed_dup(hdev, hdev->hw.mac.speed, HCLGE_MAC_FULL);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"Config mac speed dup fail ret=%d\n", ret);
+		return ret;
+	}
+
+	mac->link = 0;
+
+	ret = hclge_mac_mdio_config(hdev);
+	if (ret) {
+		dev_warn(&hdev->pdev->dev,
+			 "mdio config fail ret=%d\n", ret);
+		return ret;
+	}
+
+	/* Initialize the MTA table work mode */
+	hdev->accept_mta_mc	= true;
+	hdev->enable_mta	= true;
+	hdev->mta_mac_sel_type	= HCLGE_MAC_ADDR_47_36;
+
+	ret = hclge_set_mta_filter_mode(hdev,
+					hdev->mta_mac_sel_type,
+					hdev->enable_mta);
+	if (ret) {
+		dev_err(&hdev->pdev->dev, "set mta filter mode failed %d\n",
+			ret);
+		return ret;
+	}
+
+	return hclge_cfg_func_mta_filter(hdev, 0, hdev->accept_mta_mc);
+}
+
+static void hclge_task_schedule(struct hclge_dev *hdev)
+{
+	if (!test_bit(HCLGE_STATE_DOWN, &hdev->state) &&
+	    !test_bit(HCLGE_STATE_REMOVING, &hdev->state) &&
+	    !test_and_set_bit(HCLGE_STATE_SERVICE_SCHED, &hdev->state))
+		(void)schedule_work(&hdev->service_task);
+}
+
+static int hclge_get_mac_link_status(struct hclge_dev *hdev)
+{
+	struct hclge_link_status *req;
+	struct hclge_desc desc;
+	int link_status;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_QUERY_LINK_STATUS, true);
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev, "get link status cmd failed %d\n",
+			ret);
+		return ret;
+	}
+
+	req = (struct hclge_link_status *)desc.data;
+	link_status = req->status & HCLGE_LINK_STATUS;
+
+	return !!link_status;
+}
+
+static int hclge_get_mac_phy_link(struct hclge_dev *hdev)
+{
+	int mac_state;
+	int link_stat;
+
+	mac_state = hclge_get_mac_link_status(hdev);
+
+	if (hdev->hw.mac.phydev) {
+		if (!genphy_read_status(hdev->hw.mac.phydev))
+			link_stat = mac_state &
+				hdev->hw.mac.phydev->link;
+		else
+			link_stat = 0;
+
+	} else {
+		link_stat = mac_state;
+	}
+
+	return !!link_stat;
+}
+
+static void hclge_update_link_status(struct hclge_dev *hdev)
+{
+	struct hnae3_client *client = hdev->nic_client;
+	struct hnae3_handle *handle;
+	int state;
+	int i;
+
+	if (!client)
+		return;
+	state = hclge_get_mac_phy_link(hdev);
+	if (state != hdev->hw.mac.link) {
+		for (i = 0; i < hdev->num_vmdq_vport + 1; i++) {
+			handle = &hdev->vport[i].nic;
+			client->ops->link_status_change(handle, state);
+		}
+		hdev->hw.mac.link = state;
+	}
+}
+
+static int hclge_update_speed_duplex(struct hclge_dev *hdev)
+{
+	struct hclge_mac mac = hdev->hw.mac;
+	u8 duplex;
+	int speed;
+	int ret;
+
+	/* get the speed and duplex as autoneg'result from mac cmd when phy
+	 * doesn't exit.
+	 */
+	if (mac.phydev)
+		return 0;
+
+	/* update mac->antoneg. */
+	ret = hclge_query_autoneg_result(hdev);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"autoneg result query failed %d\n", ret);
+		return ret;
+	}
+
+	if (!mac.autoneg)
+		return 0;
+
+	ret = hclge_query_mac_an_speed_dup(hdev, &speed, &duplex);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"mac autoneg/speed/duplex query failed %d\n", ret);
+		return ret;
+	}
+
+	if ((mac.speed != speed) || (mac.duplex != duplex)) {
+		ret = hclge_cfg_mac_speed_dup(hdev, speed, duplex);
+		if (ret) {
+			dev_err(&hdev->pdev->dev,
+				"mac speed/duplex config failed %d\n", ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int hclge_update_speed_duplex_h(struct hnae3_handle *handle)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+
+	return hclge_update_speed_duplex(hdev);
+}
+
+static int hclge_get_status(struct hnae3_handle *handle)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+
+	hclge_update_link_status(hdev);
+
+	return hdev->hw.mac.link;
+}
+
+static void hclge_service_timer(unsigned long data)
+{
+	struct hclge_dev *hdev = (struct hclge_dev *)data;
+	(void)mod_timer(&hdev->service_timer, jiffies + HZ);
+
+	hclge_task_schedule(hdev);
+}
+
+static void hclge_service_complete(struct hclge_dev *hdev)
+{
+	WARN_ON(!test_bit(HCLGE_STATE_SERVICE_SCHED, &hdev->state));
+
+	/* Flush memory before next watchdog */
+	smp_mb__before_atomic();
+	clear_bit(HCLGE_STATE_SERVICE_SCHED, &hdev->state);
+}
+
+static void hclge_service_task(struct work_struct *work)
+{
+	struct hclge_dev *hdev =
+		container_of(work, struct hclge_dev, service_task);
+
+	hclge_update_speed_duplex(hdev);
+	hclge_update_link_status(hdev);
+	hclge_update_stats_for_all(hdev);
+	hclge_service_complete(hdev);
+}
+
+static void hclge_disable_sriov(struct hclge_dev *hdev)
+{
+	/* If our VFs are assigned we cannot shut down SR-IOV
+	 * without causing issues, so just leave the hardware
+	 * available but disabled
+	 */
+	if (pci_vfs_assigned(hdev->pdev)) {
+		dev_warn(&hdev->pdev->dev,
+			 "disabling driver while VFs are assigned\n");
+		return;
+	}
+
+	pci_disable_sriov(hdev->pdev);
+}
+
+struct hclge_vport *hclge_get_vport(struct hnae3_handle *handle)
+{
+	/* VF handle has no client */
+	if (!handle->client)
+		return container_of(handle, struct hclge_vport, nic);
+	else if (handle->client->type == HNAE3_CLIENT_ROCE)
+		return container_of(handle, struct hclge_vport, roce);
+	else
+		return container_of(handle, struct hclge_vport, nic);
+}
+
+static int hclge_get_vector(struct hnae3_handle *handle, u16 vector_num,
+			    struct hnae3_vector_info *vector_info)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hnae3_vector_info *vector = vector_info;
+	struct hclge_dev *hdev = vport->back;
+	int alloc = 0;
+	int i, j;
+
+	vector_num = min(hdev->num_msi_left, vector_num);
+
+	for (j = 0; j < vector_num; j++) {
+		for (i = 1; i < hdev->num_msi; i++) {
+			if (hdev->vector_status[i] == HCLGE_INVALID_VPORT) {
+				vector->vector = pci_irq_vector(hdev->pdev, i);
+				vector->io_addr = hdev->hw.io_base +
+					HCLGE_VECTOR_REG_BASE +
+					(i - 1) * HCLGE_VECTOR_REG_OFFSET +
+					vport->vport_id *
+					HCLGE_VECTOR_VF_OFFSET;
+				hdev->vector_status[i] = vport->vport_id;
+
+				vector++;
+				alloc++;
+
+				break;
+			}
+		}
+	}
+	hdev->num_msi_left -= alloc;
+	hdev->num_msi_used += alloc;
+
+	return alloc;
+}
+
+static int hclge_get_vector_index(struct hclge_dev *hdev, int vector)
+{
+	int i;
+
+	for (i = 0; i < hdev->num_msi; i++) {
+		if (hdev->msix_entries) {
+			if (vector == hdev->msix_entries[i].vector)
+				return i;
+		} else {
+			if (vector == (hdev->base_msi_vector + i))
+				return i;
+		}
+	}
+	return -EINVAL;
+}
+
+static u32 hclge_get_rss_key_size(struct hnae3_handle *handle)
+{
+	return HCLGE_RSS_KEY_SIZE;
+}
+
+static u32 hclge_get_rss_indir_size(struct hnae3_handle *handle)
+{
+	return HCLGE_RSS_IND_TBL_SIZE;
+}
+
+static int hclge_get_rss_algo(struct hclge_dev *hdev)
+{
+	struct hclge_rss_config *req;
+	struct hclge_desc desc;
+	int rss_hash_algo;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_RSS_GENERIC_CONFIG, true);
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"Get link status error, status =%d\n", ret);
+		return ret;
+	}
+
+	req = (struct hclge_rss_config *)desc.data;
+	rss_hash_algo = (req->hash_config & HCLGE_RSS_HASH_ALGO_MASK);
+
+	if (rss_hash_algo == HCLGE_RSS_HASH_ALGO_TOEPLITZ)
+		return ETH_RSS_HASH_TOP;
+
+	return -EINVAL;
+}
+
+static int hclge_set_rss_algo_key(struct hclge_dev *hdev,
+				  const u8 hfunc, const u8 *key)
+{
+	struct hclge_rss_config *req;
+	struct hclge_desc desc;
+	int key_offset;
+	int key_size;
+	int ret;
+
+	req = (struct hclge_rss_config *)desc.data;
+
+	for (key_offset = 0; key_offset < 3; key_offset++) {
+		hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_RSS_GENERIC_CONFIG,
+					   false);
+
+		req->hash_config |= (hfunc & HCLGE_RSS_HASH_ALGO_MASK);
+		req->hash_config |= (key_offset << HCLGE_RSS_HASH_KEY_OFFSET_B);
+
+		if (key_offset == 2)
+			key_size =
+			HCLGE_RSS_KEY_SIZE - HCLGE_RSS_HASH_KEY_NUM * 2;
+		else
+			key_size = HCLGE_RSS_HASH_KEY_NUM;
+
+		memcpy(req->hash_key,
+		       key + key_offset * HCLGE_RSS_HASH_KEY_NUM, key_size);
+
+		ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+		if (ret) {
+			dev_err(&hdev->pdev->dev,
+				"Configure RSS config fail, status = %d\n",
+				ret);
+			return ret;
+		}
+	}
+	return 0;
+}
+
+static int hclge_set_rss_indir_table(struct hclge_dev *hdev, const u32 *indir)
+{
+	struct hclge_rss_indirection_table *req;
+	struct hclge_desc desc;
+	int i, j;
+	int ret;
+
+	req = (struct hclge_rss_indirection_table *)desc.data;
+
+	for (i = 0; i < HCLGE_RSS_CFG_TBL_NUM; i++) {
+		hclge_cmd_setup_basic_desc
+			(&desc, HCLGE_OPC_RSS_INDIR_TABLE, false);
+
+		req->start_table_index = i * HCLGE_RSS_CFG_TBL_SIZE;
+		req->rss_set_bitmap = HCLGE_RSS_SET_BITMAP_MSK;
+
+		for (j = 0; j < HCLGE_RSS_CFG_TBL_SIZE; j++)
+			req->rss_result[j] =
+				indir[i * HCLGE_RSS_CFG_TBL_SIZE + j];
+
+		ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+		if (ret) {
+			dev_err(&hdev->pdev->dev,
+				"Configure rss indir table fail,status = %d\n",
+				ret);
+			return ret;
+		}
+	}
+	return 0;
+}
+
+static int hclge_set_rss_tc_mode(struct hclge_dev *hdev, u16 *tc_valid,
+				 u16 *tc_size, u16 *tc_offset)
+{
+	struct hclge_rss_tc_mode *req;
+	struct hclge_desc desc;
+	int ret;
+	int i;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_RSS_TC_MODE, false);
+	req = (struct hclge_rss_tc_mode *)desc.data;
+
+	for (i = 0; i < HCLGE_MAX_TC_NUM; i++) {
+		hnae_set_bit(req->rss_tc_mode[i], HCLGE_RSS_TC_VALID_B,
+			     (tc_valid[i] & 0x1));
+		hnae_set_field(req->rss_tc_mode[i], HCLGE_RSS_TC_SIZE_M,
+			       HCLGE_RSS_TC_SIZE_S, tc_size[i]);
+		hnae_set_field(req->rss_tc_mode[i], HCLGE_RSS_TC_OFFSET_M,
+			       HCLGE_RSS_TC_OFFSET_S, tc_offset[i]);
+	}
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"Configure rss tc mode fail, status = %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int hclge_set_rss_input_tuple(struct hclge_dev *hdev)
+{
+#define HCLGE_RSS_INPUT_TUPLE_OTHER		0xf
+#define HCLGE_RSS_INPUT_TUPLE_SCTP		0x1f
+	struct hclge_rss_input_tuple *req;
+	struct hclge_desc desc;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_RSS_INPUT_TUPLE, false);
+
+	req = (struct hclge_rss_input_tuple *)desc.data;
+	req->ipv4_tcp_en = HCLGE_RSS_INPUT_TUPLE_OTHER;
+	req->ipv4_udp_en = HCLGE_RSS_INPUT_TUPLE_OTHER;
+	req->ipv4_sctp_en = HCLGE_RSS_INPUT_TUPLE_SCTP;
+	req->ipv4_fragment_en = HCLGE_RSS_INPUT_TUPLE_OTHER;
+	req->ipv6_tcp_en = HCLGE_RSS_INPUT_TUPLE_OTHER;
+	req->ipv6_udp_en = HCLGE_RSS_INPUT_TUPLE_OTHER;
+	req->ipv6_sctp_en = HCLGE_RSS_INPUT_TUPLE_SCTP;
+	req->ipv6_fragment_en = HCLGE_RSS_INPUT_TUPLE_OTHER;
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"Configure rss input fail, status = %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int hclge_get_rss(struct hnae3_handle *handle, u32 *indir,
+			 u8 *key, u8 *hfunc)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+	int i;
+
+	/* Get hash algorithm */
+	if (hfunc)
+		*hfunc = hclge_get_rss_algo(hdev);
+
+	/* Get the RSS Key required by the user */
+	if (key)
+		memcpy(key, vport->rss_hash_key, HCLGE_RSS_KEY_SIZE);
+
+	/* Get indirect table */
+	if (indir)
+		for (i = 0; i < HCLGE_RSS_IND_TBL_SIZE; i++)
+			indir[i] =  vport->rss_indirection_tbl[i];
+
+	return 0;
+}
+
+static int hclge_set_rss(struct hnae3_handle *handle, const u32 *indir,
+			 const  u8 *key, const  u8 hfunc)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+	u8 hash_algo;
+	int ret, i;
+
+	/* Set the RSS Hash Key if specififed by the user */
+	if (key) {
+		/* Update the shadow RSS key with user specified qids */
+		memcpy(vport->rss_hash_key, key, HCLGE_RSS_KEY_SIZE);
+
+		if (hfunc == ETH_RSS_HASH_TOP ||
+		    hfunc == ETH_RSS_HASH_NO_CHANGE)
+			hash_algo = HCLGE_RSS_HASH_ALGO_TOEPLITZ;
+		else
+			return -EINVAL;
+		ret = hclge_set_rss_algo_key(hdev, hash_algo, key);
+		if (ret)
+			return ret;
+	}
+
+	/* Update the shadow RSS table with user specified qids */
+	for (i = 0; i < HCLGE_RSS_IND_TBL_SIZE; i++)
+		vport->rss_indirection_tbl[i] = indir[i];
+
+	/* Update the hardware */
+	ret = hclge_set_rss_indir_table(hdev, indir);
+	return ret;
+}
+
+static int hclge_get_tc_size(struct hnae3_handle *handle)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+
+	return hdev->rss_size_max;
+}
+
+static int hclge_rss_init_hw(struct hclge_dev *hdev)
+{
+	const  u8 hfunc = HCLGE_RSS_HASH_ALGO_TOEPLITZ;
+	struct hclge_vport *vport = hdev->vport;
+	u16 tc_offset[HCLGE_MAX_TC_NUM];
+	u8 rss_key[HCLGE_RSS_KEY_SIZE];
+	u16 tc_valid[HCLGE_MAX_TC_NUM];
+	u16 tc_size[HCLGE_MAX_TC_NUM];
+	u32 *rss_indir = NULL;
+	const u8 *key;
+	int i, ret, j;
+
+	rss_indir = kcalloc(HCLGE_RSS_IND_TBL_SIZE, sizeof(u32), GFP_KERNEL);
+	if (!rss_indir)
+		return -ENOMEM;
+
+	/* Get default RSS key */
+	netdev_rss_key_fill(rss_key, HCLGE_RSS_KEY_SIZE);
+
+	/* Initialize RSS indirect table for each vport */
+	for (j = 0; j < hdev->num_vmdq_vport + 1; j++) {
+		for (i = 0; i < HCLGE_RSS_IND_TBL_SIZE; i++) {
+			vport[j].rss_indirection_tbl[i] =
+				i % hdev->rss_size_max;
+			rss_indir[i] = vport[j].rss_indirection_tbl[i];
+		}
+	}
+	ret = hclge_set_rss_indir_table(hdev, rss_indir);
+	if (ret)
+		goto err;
+
+	key = rss_key;
+	ret = hclge_set_rss_algo_key(hdev, hfunc, key);
+	if (ret)
+		goto err;
+
+	ret = hclge_set_rss_input_tuple(hdev);
+	if (ret)
+		goto err;
+
+	for (i = 0; i < HCLGE_MAX_TC_NUM; i++) {
+		if (hdev->hw_tc_map & BIT(i))
+			tc_valid[i] = 1;
+		else
+			tc_valid[i] = 0;
+
+		switch (hdev->rss_size_max) {
+		case HCLGE_RSS_TC_SIZE_0:
+			tc_size[i] = 0;
+			break;
+		case HCLGE_RSS_TC_SIZE_1:
+			tc_size[i] = 1;
+			break;
+		case HCLGE_RSS_TC_SIZE_2:
+			tc_size[i] = 2;
+			break;
+		case HCLGE_RSS_TC_SIZE_3:
+			tc_size[i] = 3;
+			break;
+		case HCLGE_RSS_TC_SIZE_4:
+			tc_size[i] = 4;
+			break;
+		case HCLGE_RSS_TC_SIZE_5:
+			tc_size[i] = 5;
+			break;
+		case HCLGE_RSS_TC_SIZE_6:
+			tc_size[i] = 6;
+			break;
+		case HCLGE_RSS_TC_SIZE_7:
+			tc_size[i] = 7;
+			break;
+		default:
+			break;
+		}
+		tc_offset[i] = hdev->rss_size_max * i;
+	}
+	ret = hclge_set_rss_tc_mode(hdev, tc_valid, tc_size, tc_offset);
+
+err:
+	kfree(rss_indir);
+
+	return ret;
+}
+
+int hclge_map_vport_ring_to_vector(struct hclge_vport *vport, int vector_id,
+				   struct hnae3_ring_chain_node *ring_chain)
+{
+	struct hclge_dev *hdev = vport->back;
+	struct hclge_ctrl_vector_chain *req;
+	struct hnae3_ring_chain_node *node;
+	struct hclge_desc desc;
+	int ret;
+	int i;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_ADD_RING_TO_VECTOR, false);
+
+	req = (struct hclge_ctrl_vector_chain *)desc.data;
+	req->int_vector_id = vector_id;
+
+	i = 0;
+	for (node = ring_chain; node; node = node->next) {
+		hnae_set_field(req->tqp_type_and_id[i], HCLGE_INT_TYPE_M,
+			       HCLGE_INT_TYPE_S,
+			       hnae_get_bit(node->flag, HNAE3_RING_TYPE_B));
+		hnae_set_field(req->tqp_type_and_id[i], HCLGE_TQP_ID_M,
+			       HCLGE_TQP_ID_S,	node->tqp_index);
+		req->tqp_type_and_id[i] = cpu_to_le16(req->tqp_type_and_id[i]);
+
+		if (++i >= HCLGE_VECTOR_ELEMENTS_PER_CMD) {
+			req->int_cause_num = HCLGE_VECTOR_ELEMENTS_PER_CMD;
+
+			ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+			if (ret) {
+				dev_err(&hdev->pdev->dev,
+					"Map TQP fail, status is %d.\n",
+					ret);
+				return ret;
+			}
+			i = 0;
+
+			hclge_cmd_setup_basic_desc(&desc,
+						   HCLGE_OPC_ADD_RING_TO_VECTOR,
+						   false);
+			req->int_vector_id = vector_id;
+		}
+	}
+
+	if (i > 0) {
+		req->int_cause_num = i;
+
+		ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+		if (ret) {
+			dev_err(&hdev->pdev->dev,
+				"Map TQP fail, status is %d.\n", ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+int hclge_map_handle_ring_to_vector(struct hnae3_handle *handle,
+				    int vector,
+				    struct hnae3_ring_chain_node *ring_chain)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+	int vector_id;
+
+	vector_id = hclge_get_vector_index(hdev, vector);
+	if (vector_id < 0) {
+		dev_err(&hdev->pdev->dev,
+			"Get vector index fail. ret =%d\n", vector_id);
+		return vector_id;
+	}
+
+	return hclge_map_vport_ring_to_vector(vport, vector_id, ring_chain);
+}
+
+static int hclge_unmap_ring_from_vector(
+	struct hnae3_handle *handle, int vector,
+	struct hnae3_ring_chain_node *ring_chain)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+	struct hclge_ctrl_vector_chain *req;
+	struct hnae3_ring_chain_node *node;
+	struct hclge_desc desc;
+	int i, vector_id;
+	int ret;
+
+	vector_id = hclge_get_vector_index(hdev, vector);
+	if (vector_id < 0) {
+		dev_err(&handle->pdev->dev,
+			"Get vector index fail. ret =%d\n", vector_id);
+		return vector_id;
+	}
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_DEL_RING_TO_VECTOR, false);
+
+	req = (struct hclge_ctrl_vector_chain *)desc.data;
+	req->int_vector_id = vector_id;
+
+	i = 0;
+	for (node = ring_chain; node; node = node->next) {
+		hnae_set_field(req->tqp_type_and_id[i], HCLGE_INT_TYPE_M,
+			       HCLGE_INT_TYPE_S,
+			       hnae_get_bit(node->flag, HNAE3_RING_TYPE_B));
+		hnae_set_field(req->tqp_type_and_id[i], HCLGE_TQP_ID_M,
+			       HCLGE_TQP_ID_S,	node->tqp_index);
+
+		req->tqp_type_and_id[i] = cpu_to_le16(req->tqp_type_and_id[i]);
+
+		if (++i >= HCLGE_VECTOR_ELEMENTS_PER_CMD) {
+			req->int_cause_num = HCLGE_VECTOR_ELEMENTS_PER_CMD;
+
+			ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+			if (ret) {
+				dev_err(&hdev->pdev->dev,
+					"Unmap TQP fail, status is %d.\n",
+					ret);
+				return ret;
+			}
+			i = 0;
+			hclge_cmd_setup_basic_desc(&desc,
+						   HCLGE_OPC_ADD_RING_TO_VECTOR,
+						   false);
+			req->int_vector_id = vector_id;
+		}
+	}
+
+	if (i > 0) {
+		req->int_cause_num = i;
+
+		ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+		if (ret) {
+			dev_err(&hdev->pdev->dev,
+				"Unmap TQP fail, status is %d.\n", ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+int hclge_cmd_set_promisc_mode(struct hclge_dev *hdev,
+			       struct hclge_promisc_param *param)
+{
+	struct hclge_promisc_cfg *req;
+	struct hclge_desc desc;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_CFG_PROMISC_MODE, false);
+
+	req = (struct hclge_promisc_cfg *)desc.data;
+	req->vf_id = param->vf_id;
+	req->flag = (param->enable << HCLGE_PROMISC_EN_B);
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"Set promisc mode fail, status is %d.\n", ret);
+		return ret;
+	}
+	return 0;
+}
+
+void hclge_promisc_param_init(struct hclge_promisc_param *param, bool en_uc,
+			      bool en_mc, bool en_bc, int vport_id)
+{
+	if (!param)
+		return;
+
+	memset(param, 0, sizeof(struct hclge_promisc_param));
+	if (en_uc)
+		param->enable = HCLGE_PROMISC_EN_UC;
+	if (en_mc)
+		param->enable |= HCLGE_PROMISC_EN_MC;
+	if (en_bc)
+		param->enable |= HCLGE_PROMISC_EN_BC;
+	param->vf_id = vport_id;
+}
+
+static void hclge_set_promisc_mode(struct hnae3_handle *handle, u32 en)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+	struct hclge_promisc_param param;
+
+	hclge_promisc_param_init(&param, en, en, true, vport->vport_id);
+	hclge_cmd_set_promisc_mode(hdev, &param);
+}
+
+static void hclge_cfg_mac_mode(struct hclge_dev *hdev, bool enable)
+{
+	struct hclge_desc desc;
+	struct hclge_config_mac_mode *req =
+		(struct hclge_config_mac_mode *)desc.data;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_CONFIG_MAC_MODE, false);
+	hnae_set_bit(req->txrx_pad_fcs_loop_en, HCLGE_MAC_TX_EN_B, enable);
+	hnae_set_bit(req->txrx_pad_fcs_loop_en, HCLGE_MAC_RX_EN_B, enable);
+	hnae_set_bit(req->txrx_pad_fcs_loop_en, HCLGE_MAC_PAD_TX_B, enable);
+	hnae_set_bit(req->txrx_pad_fcs_loop_en, HCLGE_MAC_PAD_RX_B, enable);
+	hnae_set_bit(req->txrx_pad_fcs_loop_en, HCLGE_MAC_1588_TX_B, 0);
+	hnae_set_bit(req->txrx_pad_fcs_loop_en, HCLGE_MAC_1588_RX_B, 0);
+	hnae_set_bit(req->txrx_pad_fcs_loop_en, HCLGE_MAC_APP_LP_B, 0);
+	hnae_set_bit(req->txrx_pad_fcs_loop_en, HCLGE_MAC_LINE_LP_B, 0);
+	hnae_set_bit(req->txrx_pad_fcs_loop_en, HCLGE_MAC_FCS_TX_B, enable);
+	hnae_set_bit(req->txrx_pad_fcs_loop_en, HCLGE_MAC_RX_FCS_B, enable);
+	hnae_set_bit(req->txrx_pad_fcs_loop_en,
+		     HCLGE_MAC_RX_FCS_STRIP_B, enable);
+	hnae_set_bit(req->txrx_pad_fcs_loop_en,
+		     HCLGE_MAC_TX_OVERSIZE_TRUNCATE_B, enable);
+	hnae_set_bit(req->txrx_pad_fcs_loop_en,
+		     HCLGE_MAC_RX_OVERSIZE_TRUNCATE_B, enable);
+	hnae_set_bit(req->txrx_pad_fcs_loop_en,
+		     HCLGE_MAC_TX_UNDER_MIN_ERR_B, enable);
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret)
+		dev_err(&hdev->pdev->dev,
+			"mac enable fail, ret =%d.\n", ret);
+}
+
+static int hclge_tqp_enable(struct hclge_dev *hdev, int tqp_id,
+			    int stream_id, bool enable)
+{
+	struct hclge_desc desc;
+	struct hclge_cfg_com_tqp_queue *req =
+		(struct hclge_cfg_com_tqp_queue *)desc.data;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_CFG_COM_TQP_QUEUE, false);
+	req->tqp_id = cpu_to_le16(tqp_id & HCLGE_RING_ID_MASK);
+	req->stream_id = cpu_to_le16(stream_id);
+	req->enable |= enable << HCLGE_TQP_ENABLE_B;
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret)
+		dev_err(&hdev->pdev->dev,
+			"Tqp enable fail, status =%d.\n", ret);
+	return ret;
+}
+
+static void hclge_reset_tqp_stats(struct hnae3_handle *handle)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hnae3_queue *queue;
+	struct hclge_tqp *tqp;
+	int i;
+
+	for (i = 0; i < vport->alloc_tqps; i++) {
+		queue = handle->kinfo.tqp[i];
+		tqp = container_of(queue, struct hclge_tqp, q);
+		memset(&tqp->tqp_stats, 0, sizeof(tqp->tqp_stats));
+	}
+}
+
+static int hclge_ae_start(struct hnae3_handle *handle)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+	int i, queue_id, ret;
+
+	for (i = 0; i < vport->alloc_tqps; i++) {
+		/* todo clear interrupt */
+		/* ring enable */
+		queue_id = hclge_get_queue_id(handle->kinfo.tqp[i]);
+		if (queue_id < 0) {
+			dev_warn(&hdev->pdev->dev,
+				 "Get invalid queue id, ignore it\n");
+			continue;
+		}
+
+		hclge_tqp_enable(hdev, queue_id, 0, true);
+	}
+	/* mac enable */
+	hclge_cfg_mac_mode(hdev, true);
+	clear_bit(HCLGE_STATE_DOWN, &hdev->state);
+	(void)mod_timer(&hdev->service_timer, jiffies + HZ);
+
+	ret = hclge_mac_start_phy(hdev);
+	if (ret)
+		return ret;
+
+	/* reset tqp stats */
+	hclge_reset_tqp_stats(handle);
+
+	return 0;
+}
+
+static void hclge_ae_stop(struct hnae3_handle *handle)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+	int i, queue_id;
+
+	for (i = 0; i < vport->alloc_tqps; i++) {
+		/* Ring disable */
+		queue_id = hclge_get_queue_id(handle->kinfo.tqp[i]);
+		if (queue_id < 0) {
+			dev_warn(&hdev->pdev->dev,
+				 "Get invalid queue id, ignore it\n");
+			continue;
+		}
+
+		hclge_tqp_enable(hdev, queue_id, 0, false);
+	}
+	/* Mac disable */
+	hclge_cfg_mac_mode(hdev, false);
+
+	hclge_mac_stop_phy(hdev);
+
+	/* reset tqp stats */
+	hclge_reset_tqp_stats(handle);
+}
+
+static int hclge_get_mac_vlan_cmd_status(struct hclge_vport *vport,
+					 u16 cmdq_resp, u8  resp_code,
+					 enum hclge_mac_vlan_tbl_opcode op)
+{
+	struct hclge_dev *hdev = vport->back;
+	int return_status = -EIO;
+
+	if (cmdq_resp) {
+		dev_err(&hdev->pdev->dev,
+			"cmdq execute failed for get_mac_vlan_cmd_status,status=%d.\n",
+			cmdq_resp);
+		return -EIO;
+	}
+
+	if (op == HCLGE_MAC_VLAN_ADD) {
+		if ((!resp_code) || (resp_code == 1)) {
+			return_status = 0;
+		} else if (resp_code == 2) {
+			return_status = -EIO;
+			dev_err(&hdev->pdev->dev,
+				"add mac addr failed for uc_overflow.\n");
+		} else if (resp_code == 3) {
+			return_status = -EIO;
+			dev_err(&hdev->pdev->dev,
+				"add mac addr failed for mc_overflow.\n");
+		} else {
+			dev_err(&hdev->pdev->dev,
+				"add mac addr failed for undefined, code=%d.\n",
+				resp_code);
+		}
+	} else if (op == HCLGE_MAC_VLAN_REMOVE) {
+		if (!resp_code) {
+			return_status = 0;
+		} else if (resp_code == 1) {
+			return_status = -EIO;
+			dev_dbg(&hdev->pdev->dev,
+				"remove mac addr failed for miss.\n");
+		} else {
+			dev_err(&hdev->pdev->dev,
+				"remove mac addr failed for undefined, code=%d.\n",
+				resp_code);
+		}
+	} else if (op == HCLGE_MAC_VLAN_LKUP) {
+		if (!resp_code) {
+			return_status = 0;
+		} else if (resp_code == 1) {
+			return_status = -EIO;
+			dev_dbg(&hdev->pdev->dev,
+				"lookup mac addr failed for miss.\n");
+		} else {
+			dev_err(&hdev->pdev->dev,
+				"lookup mac addr failed for undefined, code=%d.\n",
+				resp_code);
+		}
+	} else {
+		return_status = -EIO;
+		dev_err(&hdev->pdev->dev,
+			"unknown opcode for get_mac_vlan_cmd_status,opcode=%d.\n",
+			op);
+	}
+
+	return return_status;
+}
+
+static int hclge_update_desc_vfid(struct hclge_desc *desc, int vfid, bool clr)
+{
+	int word_num;
+	int bit_num;
+
+	if (vfid > 255 || vfid < 0)
+		return -EIO;
+
+	if (vfid >= 0 && vfid <= 191) {
+		word_num = vfid / 32;
+		bit_num  = vfid % 32;
+		if (clr)
+			desc[1].data[word_num] &= ~(1 << bit_num);
+		else
+			desc[1].data[word_num] |= (1 << bit_num);
+	} else {
+		word_num = (vfid - 192) / 32;
+		bit_num  = vfid % 32;
+		if (clr)
+			desc[2].data[word_num] &= ~(1 << bit_num);
+		else
+			desc[2].data[word_num] |= (1 << bit_num);
+	}
+
+	return 0;
+}
+
+static bool hclge_is_all_function_id_zero(struct hclge_desc *desc)
+{
+#define HCLGE_DESC_NUMBER 3
+#define HCLGE_FUNC_NUMBER_PER_DESC 6
+	int i, j;
+
+	for (i = 0; i < HCLGE_DESC_NUMBER; i++)
+		for (j = 0; j < HCLGE_FUNC_NUMBER_PER_DESC; j++)
+			if (desc[i].data[j])
+				return false;
+
+	return true;
+}
+
+static void hclge_prepare_mac_addr(struct hclge_mac_vlan_tbl_entry *new_req,
+				   const u8 *addr)
+{
+	const unsigned char *mac_addr = addr;
+	u32 high_val = mac_addr[2] << 16 | (mac_addr[3] << 24) |
+		       (mac_addr[0]) | (mac_addr[1] << 8);
+	u32 low_val  = mac_addr[4] | (mac_addr[5] << 8);
+
+	new_req->mac_addr_hi32 = cpu_to_le32(high_val);
+	new_req->mac_addr_lo16 = cpu_to_le16(low_val & 0xffff);
+}
+
+u16 hclge_get_mac_addr_to_mta_index(struct hclge_vport *vport,
+				    const u8 *addr)
+{
+	u16 high_val = addr[1] | (addr[0] << 8);
+	struct hclge_dev *hdev = vport->back;
+	u32 rsh = 4 - hdev->mta_mac_sel_type;
+	u16 ret_val = (high_val >> rsh) & 0xfff;
+
+	return ret_val;
+}
+
+static int hclge_set_mta_filter_mode(struct hclge_dev *hdev,
+				     enum hclge_mta_dmac_sel_type mta_mac_sel,
+				     bool enable)
+{
+	struct hclge_mta_filter_mode *req;
+	struct hclge_desc desc;
+	int ret;
+
+	req = (struct hclge_mta_filter_mode *)desc.data;
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_MTA_MAC_MODE_CFG, false);
+
+	hnae_set_bit(req->dmac_sel_en, HCLGE_CFG_MTA_MAC_EN_B,
+		     enable);
+	hnae_set_field(req->dmac_sel_en, HCLGE_CFG_MTA_MAC_SEL_M,
+		       HCLGE_CFG_MTA_MAC_SEL_S, mta_mac_sel);
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"Config mat filter mode failed for cmd_send, ret =%d.\n",
+			ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+int hclge_cfg_func_mta_filter(struct hclge_dev *hdev,
+			      u8 func_id,
+			      bool enable)
+{
+	struct hclge_cfg_func_mta_filter *req;
+	struct hclge_desc desc;
+	int ret;
+
+	req = (struct hclge_cfg_func_mta_filter *)desc.data;
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_MTA_MAC_FUNC_CFG, false);
+
+	hnae_set_bit(req->accept, HCLGE_CFG_FUNC_MTA_ACCEPT_B,
+		     enable);
+	req->function_id = func_id;
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"Config func_id enable failed for cmd_send, ret =%d.\n",
+			ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int hclge_set_mta_table_item(struct hclge_vport *vport,
+				    u16 idx,
+				    bool enable)
+{
+	struct hclge_dev *hdev = vport->back;
+	struct hclge_cfg_func_mta_item *req;
+	struct hclge_desc desc;
+	int ret;
+
+	req = (struct hclge_cfg_func_mta_item *)desc.data;
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_MTA_TBL_ITEM_CFG, false);
+	hnae_set_bit(req->accept, HCLGE_CFG_MTA_ITEM_ACCEPT_B, enable);
+
+	hnae_set_field(req->item_idx, HCLGE_CFG_MTA_ITEM_IDX_M,
+		       HCLGE_CFG_MTA_ITEM_IDX_S, idx);
+	req->item_idx = cpu_to_le16(req->item_idx);
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"Config mta table item failed for cmd_send, ret =%d.\n",
+			ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int hclge_remove_mac_vlan_tbl(struct hclge_vport *vport,
+				     struct hclge_mac_vlan_tbl_entry *req)
+{
+	struct hclge_dev *hdev = vport->back;
+	struct hclge_desc desc;
+	u8 resp_code;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_MAC_VLAN_REMOVE, false);
+
+	memcpy(desc.data, req, sizeof(struct hclge_mac_vlan_tbl_entry));
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"del mac addr failed for cmd_send, ret =%d.\n",
+			ret);
+		return ret;
+	}
+	resp_code = (desc.data[0] >> 8) & 0xff;
+
+	return hclge_get_mac_vlan_cmd_status(vport, desc.retval, resp_code,
+					     HCLGE_MAC_VLAN_REMOVE);
+}
+
+static int hclge_lookup_mac_vlan_tbl(struct hclge_vport *vport,
+				     struct hclge_mac_vlan_tbl_entry *req,
+				     struct hclge_desc *desc,
+				     bool is_mc)
+{
+	struct hclge_dev *hdev = vport->back;
+	u8 resp_code;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc[0], HCLGE_OPC_MAC_VLAN_ADD, true);
+	if (is_mc) {
+		desc[0].flag |= cpu_to_le16(HCLGE_CMD_FLAG_NEXT);
+		memcpy(desc[0].data,
+		       req,
+		       sizeof(struct hclge_mac_vlan_tbl_entry));
+		hclge_cmd_setup_basic_desc(&desc[1],
+					   HCLGE_OPC_MAC_VLAN_ADD,
+					   true);
+		desc[1].flag |= cpu_to_le16(HCLGE_CMD_FLAG_NEXT);
+		hclge_cmd_setup_basic_desc(&desc[2],
+					   HCLGE_OPC_MAC_VLAN_ADD,
+					   true);
+		ret = hclge_cmd_send(&hdev->hw, desc, 3);
+	} else {
+		memcpy(desc[0].data,
+		       req,
+		       sizeof(struct hclge_mac_vlan_tbl_entry));
+		ret = hclge_cmd_send(&hdev->hw, desc, 1);
+	}
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"lookup mac addr failed for cmd_send, ret =%d.\n",
+			ret);
+		return ret;
+	}
+	resp_code = (desc[0].data[0] >> 8) & 0xff;
+
+	return hclge_get_mac_vlan_cmd_status(vport, desc[0].retval, resp_code,
+					     HCLGE_MAC_VLAN_LKUP);
+}
+
+static int hclge_add_mac_vlan_tbl(struct hclge_vport *vport,
+				  struct hclge_mac_vlan_tbl_entry *req,
+				  struct hclge_desc *mc_desc)
+{
+	struct hclge_dev *hdev = vport->back;
+	int cfg_status;
+	u8 resp_code;
+	int ret;
+
+	if (!mc_desc) {
+		struct hclge_desc desc;
+
+		hclge_cmd_setup_basic_desc(&desc,
+					   HCLGE_OPC_MAC_VLAN_ADD,
+					   false);
+		memcpy(desc.data, req, sizeof(struct hclge_mac_vlan_tbl_entry));
+		ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+		resp_code = (desc.data[0] >> 8) & 0xff;
+		cfg_status = hclge_get_mac_vlan_cmd_status(vport, desc.retval,
+							   resp_code,
+							   HCLGE_MAC_VLAN_ADD);
+	} else {
+		mc_desc[0].flag &= cpu_to_le16(~HCLGE_CMD_FLAG_WR);
+		mc_desc[0].flag |= cpu_to_le16(HCLGE_CMD_FLAG_NEXT);
+		mc_desc[1].flag &= cpu_to_le16(~HCLGE_CMD_FLAG_WR);
+		mc_desc[1].flag |= cpu_to_le16(HCLGE_CMD_FLAG_NEXT);
+		mc_desc[2].flag &= cpu_to_le16(~HCLGE_CMD_FLAG_WR);
+		mc_desc[2].flag &= cpu_to_le16(~HCLGE_CMD_FLAG_NEXT);
+		memcpy(mc_desc[0].data, req,
+		       sizeof(struct hclge_mac_vlan_tbl_entry));
+		ret = hclge_cmd_send(&hdev->hw, mc_desc, 3);
+		resp_code = (mc_desc[0].data[0] >> 8) & 0xff;
+		cfg_status = hclge_get_mac_vlan_cmd_status(vport,
+							   mc_desc[0].retval,
+							   resp_code,
+							   HCLGE_MAC_VLAN_ADD);
+	}
+
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"add mac addr failed for cmd_send, ret =%d.\n",
+			ret);
+		return ret;
+	}
+
+	return cfg_status;
+}
+
+static int hclge_add_uc_addr(struct hnae3_handle *handle,
+			     const unsigned char *addr)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+
+	return hclge_add_uc_addr_common(vport, addr);
+}
+
+int hclge_add_uc_addr_common(struct hclge_vport *vport,
+			     const unsigned char *addr)
+{
+	struct hclge_dev *hdev = vport->back;
+	struct hclge_mac_vlan_tbl_entry req;
+	enum hclge_cmd_status status;
+
+	/* mac addr check */
+	if (is_zero_ether_addr(addr) ||
+	    is_broadcast_ether_addr(addr) ||
+	    is_multicast_ether_addr(addr)) {
+		dev_err(&hdev->pdev->dev,
+			"Set_uc mac err! invalid mac:%pM. is_zero:%d,is_br=%d,is_mul=%d\n",
+			 addr,
+			 is_zero_ether_addr(addr),
+			 is_broadcast_ether_addr(addr),
+			 is_multicast_ether_addr(addr));
+		return -EINVAL;
+	}
+
+	memset(&req, 0, sizeof(req));
+	hnae_set_bit(req.flags, HCLGE_MAC_VLAN_BIT0_EN_B, 1);
+	hnae_set_bit(req.entry_type, HCLGE_MAC_VLAN_BIT0_EN_B, 0);
+	hnae_set_bit(req.entry_type, HCLGE_MAC_VLAN_BIT1_EN_B, 0);
+	hnae_set_bit(req.mc_mac_en, HCLGE_MAC_VLAN_BIT0_EN_B, 0);
+	hnae_set_bit(req.egress_port,
+		     HCLGE_MAC_EPORT_SW_EN_B, 0);
+	hnae_set_bit(req.egress_port,
+		     HCLGE_MAC_EPORT_TYPE_B, 0);
+	hnae_set_field(req.egress_port, HCLGE_MAC_EPORT_VFID_M,
+		       HCLGE_MAC_EPORT_VFID_S, vport->vport_id);
+	hnae_set_field(req.egress_port, HCLGE_MAC_EPORT_PFID_M,
+		       HCLGE_MAC_EPORT_PFID_S, 0);
+	req.egress_port = cpu_to_le16(req.egress_port);
+
+	hclge_prepare_mac_addr(&req, addr);
+
+	status = hclge_add_mac_vlan_tbl(vport, &req, NULL);
+
+	return status;
+}
+
+static int hclge_rm_uc_addr(struct hnae3_handle *handle,
+			    const unsigned char *addr)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+
+	return hclge_rm_uc_addr_common(vport, addr);
+}
+
+int hclge_rm_uc_addr_common(struct hclge_vport *vport,
+			    const unsigned char *addr)
+{
+	struct hclge_dev *hdev = vport->back;
+	struct hclge_mac_vlan_tbl_entry req;
+	enum hclge_cmd_status status;
+
+	/* mac addr check */
+	if (is_zero_ether_addr(addr) ||
+	    is_broadcast_ether_addr(addr) ||
+	    is_multicast_ether_addr(addr)) {
+		dev_dbg(&hdev->pdev->dev,
+			"Remove mac err! invalid mac:%pM.\n",
+			 addr);
+		return -EINVAL;
+	}
+
+	memset(&req, 0, sizeof(req));
+	hnae_set_bit(req.flags, HCLGE_MAC_VLAN_BIT0_EN_B, 1);
+	hnae_set_bit(req.entry_type, HCLGE_MAC_VLAN_BIT0_EN_B, 0);
+	hclge_prepare_mac_addr(&req, addr);
+	status = hclge_remove_mac_vlan_tbl(vport, &req);
+
+	return status;
+}
+
+static int hclge_add_mc_addr(struct hnae3_handle *handle,
+			     const unsigned char *addr)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+
+	return	hclge_add_mc_addr_common(vport, addr);
+}
+
+int hclge_add_mc_addr_common(struct hclge_vport *vport,
+			     const unsigned char *addr)
+{
+	struct hclge_dev *hdev = vport->back;
+	struct hclge_mac_vlan_tbl_entry req;
+	struct hclge_desc desc[3];
+	u16 tbl_idx;
+	int status;
+
+	/* mac addr check */
+	if (!is_multicast_ether_addr(addr)) {
+		dev_err(&hdev->pdev->dev,
+			"Add mc mac err! invalid mac:%pM.\n",
+			 addr);
+		return -EINVAL;
+	}
+	memset(&req, 0, sizeof(req));
+	hnae_set_bit(req.flags, HCLGE_MAC_VLAN_BIT0_EN_B, 1);
+	hnae_set_bit(req.entry_type, HCLGE_MAC_VLAN_BIT0_EN_B, 0);
+	hnae_set_bit(req.entry_type, HCLGE_MAC_VLAN_BIT1_EN_B, 1);
+	hnae_set_bit(req.mc_mac_en, HCLGE_MAC_VLAN_BIT0_EN_B, 0);
+	hclge_prepare_mac_addr(&req, addr);
+	status = hclge_lookup_mac_vlan_tbl(vport, &req, desc, true);
+	if (!status) {
+		/* This mac addr exist, update VFID for it */
+		hclge_update_desc_vfid(desc, vport->vport_id, false);
+		status = hclge_add_mac_vlan_tbl(vport, &req, desc);
+	} else {
+		/* This mac addr do not exist, add new entry for it */
+		memset(desc[0].data, 0, sizeof(desc[0].data));
+		memset(desc[1].data, 0, sizeof(desc[0].data));
+		memset(desc[2].data, 0, sizeof(desc[0].data));
+		hclge_update_desc_vfid(desc, vport->vport_id, false);
+		status = hclge_add_mac_vlan_tbl(vport, &req, desc);
+	}
+
+	/* Set MTA table for this MAC address */
+	tbl_idx = hclge_get_mac_addr_to_mta_index(vport, addr);
+	status = hclge_set_mta_table_item(vport, tbl_idx, true);
+
+	return status;
+}
+
+static int hclge_rm_mc_addr(struct hnae3_handle *handle,
+			    const unsigned char *addr)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+
+	return hclge_rm_mc_addr_common(vport, addr);
+}
+
+int hclge_rm_mc_addr_common(struct hclge_vport *vport,
+			    const unsigned char *addr)
+{
+	struct hclge_dev *hdev = vport->back;
+	struct hclge_mac_vlan_tbl_entry req;
+	enum hclge_cmd_status status;
+	struct hclge_desc desc[3];
+	u16 tbl_idx;
+
+	/* mac addr check */
+	if (!is_multicast_ether_addr(addr)) {
+		dev_dbg(&hdev->pdev->dev,
+			"Remove mc mac err! invalid mac:%pM.\n",
+			 addr);
+		return -EINVAL;
+	}
+
+	memset(&req, 0, sizeof(req));
+	hnae_set_bit(req.flags, HCLGE_MAC_VLAN_BIT0_EN_B, 1);
+	hnae_set_bit(req.entry_type, HCLGE_MAC_VLAN_BIT0_EN_B, 0);
+	hnae_set_bit(req.entry_type, HCLGE_MAC_VLAN_BIT1_EN_B, 1);
+	hnae_set_bit(req.mc_mac_en, HCLGE_MAC_VLAN_BIT0_EN_B, 0);
+	hclge_prepare_mac_addr(&req, addr);
+	status = hclge_lookup_mac_vlan_tbl(vport, &req, desc, true);
+	if (!status) {
+		/* This mac addr exist, remove this handle's VFID for it */
+		hclge_update_desc_vfid(desc, vport->vport_id, true);
+
+		if (hclge_is_all_function_id_zero(desc))
+			/* All the vfid is zero, so need to delete this entry */
+			status = hclge_remove_mac_vlan_tbl(vport, &req);
+		else
+			/* Not all the vfid is zero, update the vfid */
+			status = hclge_add_mac_vlan_tbl(vport, &req, desc);
+
+	} else {
+		/* This mac addr do not exist, can't delete it */
+		dev_err(&hdev->pdev->dev,
+			"Rm multicast mac addr failed, ret = %d.\n",
+			status);
+		return -EIO;
+	}
+
+	/* Set MTB table for this MAC address */
+	tbl_idx = hclge_get_mac_addr_to_mta_index(vport, addr);
+	status = hclge_set_mta_table_item(vport, tbl_idx, false);
+
+	return status;
+}
+
+static void hclge_get_mac_addr(struct hnae3_handle *handle, u8 *p)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+
+	ether_addr_copy(p, hdev->hw.mac.mac_addr);
+}
+
+static int hclge_set_mac_addr(struct hnae3_handle *handle, void *p)
+{
+	const unsigned char *new_addr = (const unsigned char *)p;
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+
+	/* mac addr check */
+	if (is_zero_ether_addr(new_addr) ||
+	    is_broadcast_ether_addr(new_addr) ||
+	    is_multicast_ether_addr(new_addr)) {
+		dev_err(&hdev->pdev->dev,
+			"Change uc mac err! invalid mac:%p.\n",
+			 new_addr);
+		return -EINVAL;
+	}
+
+	hclge_rm_uc_addr(handle, hdev->hw.mac.mac_addr);
+
+	if (!hclge_add_uc_addr(handle, new_addr)) {
+		ether_addr_copy(hdev->hw.mac.mac_addr, new_addr);
+		return 0;
+	}
+
+	return -EIO;
+}
+
+static int hclge_set_vlan_filter_ctrl(struct hclge_dev *hdev, u8 vlan_type,
+				      bool filter_en)
+{
+	struct hclge_vlan_filter_ctrl *req;
+	struct hclge_desc desc;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_VLAN_FILTER_CTRL, false);
+
+	req = (struct hclge_vlan_filter_ctrl *)desc.data;
+	req->vlan_type = vlan_type;
+	req->vlan_fe = filter_en;
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev, "set vlan filter fail, ret =%d.\n",
+			ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+int hclge_set_vf_vlan_common(struct hclge_dev *hdev, int vfid,
+			     bool is_kill, u16 vlan, u8 qos, __be16 proto)
+{
+#define HCLGE_MAX_VF_BYTES  16
+	struct hclge_vlan_filter_vf_cfg *req0;
+	struct hclge_vlan_filter_vf_cfg *req1;
+	struct hclge_desc desc[2];
+	u8 vf_byte_val;
+	u8 vf_byte_off;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc[0],
+				   HCLGE_OPC_VLAN_FILTER_VF_CFG, false);
+	hclge_cmd_setup_basic_desc(&desc[1],
+				   HCLGE_OPC_VLAN_FILTER_VF_CFG, false);
+
+	desc[0].flag |= cpu_to_le16(HCLGE_CMD_FLAG_NEXT);
+
+	vf_byte_off = vfid / 8;
+	vf_byte_val = 1 << (vfid % 8);
+
+	req0 = (struct hclge_vlan_filter_vf_cfg *)desc[0].data;
+	req1 = (struct hclge_vlan_filter_vf_cfg *)desc[1].data;
+
+	req0->vlan_id  = vlan;
+	req0->vlan_cfg = is_kill;
+
+	if (vf_byte_off < HCLGE_MAX_VF_BYTES)
+		req0->vf_bitmap[vf_byte_off] = vf_byte_val;
+	else
+		req1->vf_bitmap[vf_byte_off - HCLGE_MAX_VF_BYTES] = vf_byte_val;
+
+	ret = hclge_cmd_send(&hdev->hw, desc, 2);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"Send vf vlan command fail, ret =%d.\n",
+			ret);
+		return ret;
+	}
+
+	if (!is_kill) {
+		if (!req0->resp_code || req0->resp_code == 1)
+			return 0;
+
+		dev_err(&hdev->pdev->dev,
+			"Add vf vlan filter fail, ret =%d.\n",
+			req0->resp_code);
+	} else {
+		if (!req0->resp_code)
+			return 0;
+
+		dev_err(&hdev->pdev->dev,
+			"Kill vf vlan filter fail, ret =%d.\n",
+			req0->resp_code);
+	}
+
+	return -EIO;
+}
+
+static int hclge_set_port_vlan_filter(struct hnae3_handle *handle,
+				      __be16 proto, u16 vlan_id,
+				      bool is_kill)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+	struct hclge_vlan_filter_pf_cfg *req;
+	struct hclge_desc desc;
+	u8 vlan_offset_byte_val;
+	u8 vlan_offset_byte;
+	u8 vlan_offset_160;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_VLAN_FILTER_PF_CFG, false);
+
+	vlan_offset_160 = vlan_id / 160;
+	vlan_offset_byte = (vlan_id % 160) / 8;
+	vlan_offset_byte_val = 1 << (vlan_id % 8);
+
+	req = (struct hclge_vlan_filter_pf_cfg *)desc.data;
+	req->vlan_offset = vlan_offset_160;
+	req->vlan_cfg = is_kill;
+	req->vlan_offset_bitmap[vlan_offset_byte] = vlan_offset_byte_val;
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"port vlan command, send fail, ret =%d.\n",
+			ret);
+		return ret;
+	}
+
+	ret = hclge_set_vf_vlan_common(hdev, 0, is_kill, vlan_id, 0, proto);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"Set pf vlan filter config fail, ret =%d.\n",
+			ret);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static int hclge_set_vf_vlan_filter(struct hnae3_handle *handle, int vfid,
+				    u16 vlan, u8 qos, __be16 proto)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+
+	if ((vfid >= hdev->num_alloc_vfs) || (vlan > 4095) || (qos > 7))
+		return -EINVAL;
+	if (proto != htons(ETH_P_8021Q))
+		return -EPROTONOSUPPORT;
+
+	return hclge_set_vf_vlan_common(hdev, vfid, false, vlan, qos, proto);
+}
+
+static int hclge_init_vlan_config(struct hclge_dev *hdev)
+{
+#define HCLGE_VLAN_TYPE_VF_TABLE   0
+#define HCLGE_VLAN_TYPE_PORT_TABLE 1
+	int ret;
+
+	ret = hclge_set_vlan_filter_ctrl(hdev, HCLGE_VLAN_TYPE_VF_TABLE,
+					 true);
+	if (ret)
+		return ret;
+
+	ret = hclge_set_vlan_filter_ctrl(hdev, HCLGE_VLAN_TYPE_PORT_TABLE,
+					 true);
+
+	return ret;
+}
+
+static int hclge_set_mtu(struct hnae3_handle *handle, int new_mtu)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_config_max_frm_size *req;
+	struct hclge_dev *hdev = vport->back;
+	struct hclge_desc desc;
+	int ret;
+
+	if ((new_mtu < HCLGE_MAC_MIN_MTU) || (new_mtu > HCLGE_MAC_MAX_MTU))
+		return -EINVAL;
+
+	hdev->mps = new_mtu;
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_CONFIG_MAX_FRM_SIZE, false);
+
+	req = (struct hclge_config_max_frm_size *)desc.data;
+	req->max_frm_size = cpu_to_le16(new_mtu);
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev, "set mtu fail, ret =%d.\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int hclge_send_reset_tqp_cmd(struct hclge_dev *hdev, u16 queue_id,
+				    bool enable)
+{
+	struct hclge_reset_tqp_queue *req;
+	struct hclge_desc desc;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_RESET_TQP_QUEUE, false);
+
+	req = (struct hclge_reset_tqp_queue *)desc.data;
+	req->tqp_id = cpu_to_le16(queue_id & HCLGE_RING_ID_MASK);
+	hnae_set_bit(req->reset_req, HCLGE_TQP_RESET_B, enable);
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"Send tqp reset cmd error, status =%d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int hclge_get_reset_status(struct hclge_dev *hdev, u16 queue_id)
+{
+	struct hclge_reset_tqp_queue *req;
+	struct hclge_desc desc;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_RESET_TQP_QUEUE, true);
+
+	req = (struct hclge_reset_tqp_queue *)desc.data;
+	req->tqp_id = cpu_to_le16(queue_id & HCLGE_RING_ID_MASK);
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"Get reset status error, status =%d\n", ret);
+		return ret;
+	}
+
+	return hnae_get_bit(req->ready_to_reset, HCLGE_TQP_RESET_B);
+}
+
+static void hclge_reset_tqp(struct hnae3_handle *handle, u16 queue_id)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+	int reset_try_times = 0;
+	int reset_status;
+	int ret;
+
+	ret = hclge_tqp_enable(hdev, queue_id, 0, false);
+	if (ret) {
+		dev_warn(&hdev->pdev->dev, "Disable tqp fail, ret = %d\n", ret);
+		return;
+	}
+
+	ret = hclge_send_reset_tqp_cmd(hdev, queue_id, true);
+	if (ret) {
+		dev_warn(&hdev->pdev->dev,
+			 "Send reset tqp cmd fail, ret = %d\n", ret);
+		return;
+	}
+
+	reset_try_times = 0;
+	while (reset_try_times++ < HCLGE_TQP_RESET_TRY_TIMES) {
+		/* Wait for tqp hw reset */
+		msleep(20);
+		reset_status = hclge_get_reset_status(hdev, queue_id);
+		if (reset_status)
+			break;
+	}
+
+	if (reset_try_times >= HCLGE_TQP_RESET_TRY_TIMES) {
+		dev_warn(&hdev->pdev->dev, "Reset TQP fail\n");
+		return;
+	}
+
+	ret = hclge_send_reset_tqp_cmd(hdev, queue_id, false);
+	if (ret) {
+		dev_warn(&hdev->pdev->dev,
+			 "Deassert the soft reset fail, ret = %d\n", ret);
+		return;
+	}
+}
+
+static u32 hclge_get_fw_version(struct hnae3_handle *handle)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+
+	return hdev->fw_version;
+}
+
+static void hclge_get_pauseparam(struct hnae3_handle *handle, u32 *auto_neg,
+				 u32 *rx_en, u32 *tx_en)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+
+	*auto_neg = hclge_get_autoneg(handle);
+
+	if (hdev->tm_info.fc_mode == HCLGE_FC_PFC) {
+		*rx_en = 0;
+		*tx_en = 0;
+		return;
+	}
+
+	if (hdev->tm_info.fc_mode == HCLGE_FC_RX_PAUSE) {
+		*rx_en = 1;
+		*tx_en = 0;
+	} else if (hdev->tm_info.fc_mode == HCLGE_FC_TX_PAUSE) {
+		*tx_en = 1;
+		*rx_en = 0;
+	} else if (hdev->tm_info.fc_mode == HCLGE_FC_FULL) {
+		*rx_en = 1;
+		*tx_en = 1;
+	} else {
+		*rx_en = 0;
+		*tx_en = 0;
+	}
+}
+
+static void hclge_get_ksettings_an_result(struct hnae3_handle *handle,
+					  u8 *auto_neg, u32 *speed, u8 *duplex)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+
+	if (speed)
+		*speed = hdev->hw.mac.speed;
+	if (duplex)
+		*duplex = hdev->hw.mac.duplex;
+	if (auto_neg)
+		*auto_neg = hdev->hw.mac.autoneg;
+}
+
+static void hclge_get_media_type(struct hnae3_handle *handle, u8 *media_type)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+
+	if (media_type)
+		*media_type = hdev->hw.mac.media_type;
+}
+
+static void hclge_get_mdix_mode(struct hnae3_handle *handle,
+				u8 *tp_mdix_ctrl, u8 *tp_mdix)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+	struct phy_device *phydev = hdev->hw.mac.phydev;
+	int mdix_ctrl, mdix, retval, is_resolved;
+
+	if (!phydev) {
+		*tp_mdix_ctrl = ETH_TP_MDI_INVALID;
+		*tp_mdix = ETH_TP_MDI_INVALID;
+		return;
+	}
+
+	phy_write(phydev, HCLGE_PHY_PAGE_REG, HCLGE_PHY_PAGE_MDIX);
+
+	retval = phy_read(phydev, HCLGE_PHY_CSC_REG);
+	mdix_ctrl = hnae_get_field(retval, HCLGE_PHY_MDIX_CTRL_M,
+				   HCLGE_PHY_MDIX_CTRL_S);
+
+	retval = phy_read(phydev, HCLGE_PHY_CSS_REG);
+	mdix = hnae_get_bit(retval, HCLGE_PHY_MDIX_STATUS_B);
+	is_resolved = hnae_get_bit(retval, HCLGE_PHY_SPEED_DUP_RESOLVE_B);
+
+	phy_write(phydev, HCLGE_PHY_PAGE_REG, HCLGE_PHY_PAGE_COPPER);
+
+	switch (mdix_ctrl) {
+	case 0x0:
+		*tp_mdix_ctrl = ETH_TP_MDI;
+		break;
+	case 0x1:
+		*tp_mdix_ctrl = ETH_TP_MDI_X;
+		break;
+	case 0x3:
+		*tp_mdix_ctrl = ETH_TP_MDI_AUTO;
+		break;
+	default:
+		*tp_mdix_ctrl = ETH_TP_MDI_INVALID;
+		break;
+	}
+
+	if (!is_resolved)
+		*tp_mdix = ETH_TP_MDI_INVALID;
+	else if (mdix)
+		*tp_mdix = ETH_TP_MDI_X;
+	else
+		*tp_mdix = ETH_TP_MDI;
+}
+
+static int hclge_init_client_instance(struct hnae3_client *client,
+				      struct hnae3_ae_dev *ae_dev)
+{
+	struct hclge_dev *hdev = ae_dev->priv;
+	struct hclge_vport *vport;
+	int i, ret;
+
+	for (i = 0; i <  hdev->num_vmdq_vport + 1; i++) {
+		vport = &hdev->vport[i];
+
+		switch (client->type) {
+		case HNAE3_CLIENT_KNIC:
+
+			hdev->nic_client = client;
+			vport->nic.client = client;
+			ret = client->ops->init_instance(&vport->nic);
+			if (ret)
+				goto err;
+
+			if (hdev->roce_client &&
+			    hnae_get_bit(hdev->ae_dev->flag,
+					 HNAE_DEV_SUPPORT_ROCE_B)) {
+				struct hnae3_client *rc = hdev->roce_client;
+
+				ret = hclge_init_roce_base_info(vport);
+				if (ret)
+					goto err;
+
+				ret = rc->ops->init_instance(&vport->roce);
+				if (ret)
+					goto err;
+			}
+
+			break;
+		case HNAE3_CLIENT_UNIC:
+			hdev->nic_client = client;
+			vport->nic.client = client;
+
+			ret = client->ops->init_instance(&vport->nic);
+			if (ret)
+				goto err;
+
+			break;
+		case HNAE3_CLIENT_ROCE:
+			if (hnae_get_bit(hdev->ae_dev->flag,
+					 HNAE_DEV_SUPPORT_ROCE_B)) {
+				hdev->roce_client = client;
+				vport->roce.client = client;
+			}
+
+			if (hdev->roce_client) {
+				ret = hclge_init_roce_base_info(vport);
+				if (ret)
+					goto err;
+
+				ret = client->ops->init_instance(&vport->roce);
+				if (ret)
+					goto err;
+			}
+		}
+	}
+
+	return 0;
+err:
+	return ret;
+}
+
+static void hclge_uninit_client_instance(struct hnae3_client *client,
+					 struct hnae3_ae_dev *ae_dev)
+{
+	struct hclge_dev *hdev = ae_dev->priv;
+	struct hclge_vport *vport;
+	int i;
+
+	for (i = 0; i < hdev->num_vmdq_vport + 1; i++) {
+		vport = &hdev->vport[i];
+		if (hdev->roce_client)
+			hdev->roce_client->ops->uninit_instance(&vport->roce,
+								0);
+		if (client->type == HNAE3_CLIENT_ROCE)
+			return;
+		if (client->ops->uninit_instance)
+			client->ops->uninit_instance(&vport->nic, 0);
+	}
+}
+
+static int hclge_pci_init(struct hclge_dev *hdev)
+{
+	struct pci_dev *pdev = hdev->pdev;
+	struct hclge_hw *hw;
+	int ret;
+
+	ret = pci_enable_device(pdev);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to enable PCI device\n");
+		goto err_no_drvdata;
+	}
+
+	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+	if (ret) {
+		ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
+		if (ret) {
+			dev_err(&pdev->dev,
+				"can't set consistent PCI DMA");
+			goto err_disable_device;
+		}
+		dev_warn(&pdev->dev, "set DMA mask to 32 bits\n");
+	}
+
+	ret = pci_request_regions(pdev, HCLGE_DRIVER_NAME);
+	if (ret) {
+		dev_err(&pdev->dev, "PCI request regions failed %d\n", ret);
+		goto err_disable_device;
+	}
+
+	pci_set_master(pdev);
+	hw = &hdev->hw;
+	hw->back = hdev;
+	hw->io_base = pcim_iomap(pdev, 2, 0);
+	if (!hw->io_base) {
+		dev_err(&pdev->dev, "Can't map configuration register space\n");
+		ret = -ENOMEM;
+		goto err_clr_master;
+	}
+
+	return 0;
+err_clr_master:
+	pci_clear_master(pdev);
+	pci_release_regions(pdev);
+err_disable_device:
+	pci_disable_device(pdev);
+err_no_drvdata:
+	pci_set_drvdata(pdev, NULL);
+
+	return ret;
+}
+
+static void hclge_pci_uninit(struct hclge_dev *hdev)
+{
+	struct pci_dev *pdev = hdev->pdev;
+
+	if (hdev->flag & HCLGE_FLAG_USE_MSIX) {
+		pci_disable_msix(pdev);
+		devm_kfree(&pdev->dev, hdev->msix_entries);
+		hdev->msix_entries = NULL;
+	} else {
+		pci_disable_msi(pdev);
+	}
+
+	pci_clear_master(pdev);
+	pci_release_mem_regions(pdev);
+	pci_disable_device(pdev);
+}
+
+static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev)
+{
+	struct pci_dev *pdev = ae_dev->pdev;
+	const struct pci_device_id *id;
+	struct hclge_dev *hdev;
+	int ret;
+
+	hdev = devm_kzalloc(&pdev->dev, sizeof(*hdev), GFP_KERNEL);
+	if (!hdev) {
+		ret = -ENOMEM;
+		goto err_hclge_dev;
+	}
+
+	hdev->flag |= HCLGE_FLAG_USE_MSIX;
+	hdev->pdev = pdev;
+	hdev->ae_dev = ae_dev;
+	ae_dev->priv = hdev;
+
+	id = pci_match_id(roce_pci_tbl, ae_dev->pdev);
+	if (id)
+		hnae_set_bit(ae_dev->flag, HNAE_DEV_SUPPORT_ROCE_B, 1);
+
+	ret = hclge_pci_init(hdev);
+	if (ret) {
+		dev_err(&pdev->dev, "PCI init failed\n");
+		goto err_pci_init;
+	}
+
+	/* Command queue initialize */
+	ret = hclge_cmd_init(hdev);
+	if (ret)
+		goto err_cmd_init;
+
+	ret = hclge_get_cap(hdev);
+	if (ret) {
+		dev_err(&pdev->dev, "get hw capability error, ret = %d.\n",
+			ret);
+		return ret;
+	}
+
+	ret = hclge_configure(hdev);
+	if (ret) {
+		dev_err(&pdev->dev, "Configure dev error, ret = %d.\n", ret);
+		return ret;
+	}
+
+	if (hdev->flag & HCLGE_FLAG_USE_MSIX)
+		ret = hclge_init_msix(hdev);
+	else
+		ret = hclge_init_msi(hdev);
+	if (ret) {
+		dev_err(&pdev->dev, "Init msix/msi error, ret = %d.\n", ret);
+		return ret;
+	}
+
+	ret = hclge_alloc_tqps(hdev);
+	if (ret) {
+		dev_err(&pdev->dev, "Allocate TQPs error, ret = %d.\n", ret);
+		return ret;
+	}
+
+	ret = hclge_alloc_vport(hdev);
+	if (ret) {
+		dev_err(&pdev->dev, "Allocate vport error, ret = %d.\n", ret);
+		return ret;
+	}
+
+	ret = hclge_mac_init(hdev);
+	if (ret) {
+		dev_err(&pdev->dev, "Mac init error, ret = %d\n", ret);
+		return ret;
+	}
+	ret = hclge_buffer_alloc(hdev);
+	if (ret) {
+		dev_err(&pdev->dev, "Buffer allocate fail, ret =%d\n", ret);
+		return  ret;
+	}
+
+	ret = hclge_config_tso(hdev, HCLGE_TSO_MSS_MIN, HCLGE_TSO_MSS_MAX);
+	if (ret) {
+		dev_err(&pdev->dev, "Enable tso fail, ret =%d\n", ret);
+		return ret;
+	}
+
+	ret = hclge_rss_init_hw(hdev);
+	if (ret) {
+		dev_err(&pdev->dev, "Rss init fail, ret =%d\n", ret);
+		return  ret;
+	}
+
+	ret = hclge_init_vlan_config(hdev);
+	if (ret) {
+		dev_err(&pdev->dev, "VLAN init fail, ret =%d\n", ret);
+		return  ret;
+	}
+
+	ret = hclge_tm_schd_init(hdev);
+	if (ret) {
+		dev_err(&pdev->dev, "tm schd init fail, ret =%d\n", ret);
+		return ret;
+	}
+
+	setup_timer(&hdev->service_timer, hclge_service_timer,
+		    (unsigned long)hdev);
+	INIT_WORK(&hdev->service_task, hclge_service_task);
+
+	set_bit(HCLGE_STATE_SERVICE_INITED, &hdev->state);
+	set_bit(HCLGE_STATE_DOWN, &hdev->state);
+
+	pr_info("%s driver initialization finished.\n", HCLGE_DRIVER_NAME);
+	return 0;
+
+err_cmd_init:
+	pci_release_regions(pdev);
+err_pci_init:
+	pci_set_drvdata(pdev, NULL);
+err_hclge_dev:
+	return ret;
+}
+
+static void hclge_uninit_ae_dev(struct hnae3_ae_dev *ae_dev)
+{
+	struct hclge_dev *hdev = ae_dev->priv;
+	struct hclge_mac *mac = &hdev->hw.mac;
+
+	set_bit(HCLGE_STATE_DOWN, &hdev->state);
+
+	if (IS_ENABLED(CONFIG_PCI_IOV))
+		hclge_disable_sriov(hdev);
+
+	if (hdev->service_timer.data)
+		del_timer_sync(&hdev->service_timer);
+	if (hdev->service_task.func)
+		cancel_work_sync(&hdev->service_task);
+
+	if (mac->phydev)
+		mdiobus_unregister(mac->mdio_bus);
+
+	hclge_destroy_cmd_queue(&hdev->hw);
+	hclge_pci_uninit(hdev);
+	ae_dev->priv = NULL;
+}
+
+static const struct hnae3_ae_ops hclge_ops = {
+	.init_ae_dev = hclge_init_ae_dev,
+	.uninit_ae_dev = hclge_uninit_ae_dev,
+	.init_client_instance = hclge_init_client_instance,
+	.uninit_client_instance = hclge_uninit_client_instance,
+	.map_ring_to_vector = hclge_map_handle_ring_to_vector,
+	.unmap_ring_from_vector = hclge_unmap_ring_from_vector,
+	.get_vector = hclge_get_vector,
+	.set_promisc_mode = hclge_set_promisc_mode,
+	.start = hclge_ae_start,
+	.stop = hclge_ae_stop,
+	.get_status = hclge_get_status,
+	.get_ksettings_an_result = hclge_get_ksettings_an_result,
+	.update_speed_duplex_h = hclge_update_speed_duplex_h,
+	.cfg_mac_speed_dup_h = hclge_cfg_mac_speed_dup_h,
+	.get_media_type = hclge_get_media_type,
+	.get_rss_key_size = hclge_get_rss_key_size,
+	.get_rss_indir_size = hclge_get_rss_indir_size,
+	.get_rss = hclge_get_rss,
+	.set_rss = hclge_set_rss,
+	.get_tc_size = hclge_get_tc_size,
+	.get_mac_addr = hclge_get_mac_addr,
+	.set_mac_addr = hclge_set_mac_addr,
+	.add_uc_addr = hclge_add_uc_addr,
+	.rm_uc_addr = hclge_rm_uc_addr,
+	.add_mc_addr = hclge_add_mc_addr,
+	.rm_mc_addr = hclge_rm_mc_addr,
+	.set_autoneg = hclge_set_autoneg,
+	.get_autoneg = hclge_get_autoneg,
+	.get_pauseparam = hclge_get_pauseparam,
+	.set_mtu = hclge_set_mtu,
+	.reset_queue = hclge_reset_tqp,
+	.get_stats = hclge_get_stats,
+	.update_stats = hclge_update_stats,
+	.get_strings = hclge_get_strings,
+	.get_sset_count = hclge_get_sset_count,
+	.get_fw_version = hclge_get_fw_version,
+	.get_mdix_mode = hclge_get_mdix_mode,
+	.set_vlan_filter = hclge_set_port_vlan_filter,
+	.set_vf_vlan_filter = hclge_set_vf_vlan_filter,
+};
+
+static struct hnae3_ae_algo ae_algo = {
+	.ops = &hclge_ops,
+	.name = HCLGE_NAME,
+	.pdev_id_table = ae_algo_pci_tbl,
+};
+
+static int hclge_init(void)
+{
+	pr_info("%s is initializing\n", HCLGE_NAME);
+
+	return hnae3_register_ae_algo(&ae_algo);
+}
+
+static void hclge_exit(void)
+{
+	hnae3_unregister_ae_algo(&ae_algo);
+}
+module_init(hclge_init);
+module_exit(hclge_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
+MODULE_DESCRIPTION("HCLGE Driver");
+MODULE_VERSION(HCLGE_MOD_VERSION);
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h
new file mode 100644
index 000000000000..edb10ad075eb
--- /dev/null
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h
@@ -0,0 +1,519 @@
+/*
+ * Copyright (c) 2016~2017 Hisilicon Limited.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef __HCLGE_MAIN_H
+#define __HCLGE_MAIN_H
+#include <linux/fs.h>
+#include <linux/types.h>
+#include <linux/phy.h>
+#include "hclge_cmd.h"
+#include "hnae3.h"
+
+#define HCLGE_MOD_VERSION "v1.0"
+#define HCLGE_DRIVER_NAME "hclge"
+
+#define HCLGE_INVALID_VPORT 0xffff
+
+#define HCLGE_ROCE_VECTOR_OFFSET	96
+
+#define HCLGE_PF_CFG_BLOCK_SIZE		32
+#define HCLGE_PF_CFG_DESC_NUM \
+	(HCLGE_PF_CFG_BLOCK_SIZE / HCLGE_CFG_RD_LEN_BYTES)
+
+#define HCLGE_VECTOR_REG_BASE		0x20000
+
+#define HCLGE_VECTOR_REG_OFFSET		0x4
+#define HCLGE_VECTOR_VF_OFFSET		0x100000
+
+#define HCLGE_RSS_IND_TBL_SIZE		512
+#define HCLGE_RSS_SET_BITMAP_MSK	0xffff
+#define HCLGE_RSS_KEY_SIZE		40
+#define HCLGE_RSS_HASH_ALGO_TOEPLITZ	0
+#define HCLGE_RSS_HASH_ALGO_SIMPLE	1
+#define HCLGE_RSS_HASH_ALGO_SYMMETRIC	2
+#define HCLGE_RSS_HASH_ALGO_MASK	0xf
+#define HCLGE_RSS_CFG_TBL_NUM \
+	(HCLGE_RSS_IND_TBL_SIZE / HCLGE_RSS_CFG_TBL_SIZE)
+
+#define HCLGE_RSS_TC_SIZE_0		1
+#define HCLGE_RSS_TC_SIZE_1		2
+#define HCLGE_RSS_TC_SIZE_2		4
+#define HCLGE_RSS_TC_SIZE_3		8
+#define HCLGE_RSS_TC_SIZE_4		16
+#define HCLGE_RSS_TC_SIZE_5		32
+#define HCLGE_RSS_TC_SIZE_6		64
+#define HCLGE_RSS_TC_SIZE_7		128
+
+#define HCLGE_TQP_RESET_TRY_TIMES	10
+
+#define HCLGE_PHY_PAGE_MDIX		0
+#define HCLGE_PHY_PAGE_COPPER		0
+
+/* Page Selection Reg. */
+#define HCLGE_PHY_PAGE_REG		22
+
+/* Copper Specific Control Register */
+#define HCLGE_PHY_CSC_REG		16
+
+/* Copper Specific Status Register */
+#define HCLGE_PHY_CSS_REG		17
+
+#define HCLGE_PHY_MDIX_CTRL_S		(5)
+#define HCLGE_PHY_MDIX_CTRL_M		(3 << HCLGE_PHY_MDIX_CTRL_S)
+
+#define HCLGE_PHY_MDIX_STATUS_B	(6)
+#define HCLGE_PHY_SPEED_DUP_RESOLVE_B	(11)
+
+enum HCLGE_DEV_STATE {
+	HCLGE_STATE_REINITING,
+	HCLGE_STATE_DOWN,
+	HCLGE_STATE_DISABLED,
+	HCLGE_STATE_REMOVING,
+	HCLGE_STATE_SERVICE_INITED,
+	HCLGE_STATE_SERVICE_SCHED,
+	HCLGE_STATE_MBX_HANDLING,
+	HCLGE_STATE_MBX_IRQ,
+	HCLGE_STATE_MAX
+};
+
+#define HCLGE_MPF_ENBALE 1
+struct hclge_caps {
+	u16 num_tqp;
+	u16 num_buffer_cell;
+	u32 flag;
+	u16 vmdq;
+};
+
+enum HCLGE_MAC_SPEED {
+	HCLGE_MAC_SPEED_10M	= 10,		/* 10 Mbps */
+	HCLGE_MAC_SPEED_100M	= 100,		/* 100 Mbps */
+	HCLGE_MAC_SPEED_1G	= 1000,		/* 1000 Mbps   = 1 Gbps */
+	HCLGE_MAC_SPEED_10G	= 10000,	/* 10000 Mbps  = 10 Gbps */
+	HCLGE_MAC_SPEED_25G	= 25000,	/* 25000 Mbps  = 25 Gbps */
+	HCLGE_MAC_SPEED_40G	= 40000,	/* 40000 Mbps  = 40 Gbps */
+	HCLGE_MAC_SPEED_50G	= 50000,	/* 50000 Mbps  = 50 Gbps */
+	HCLGE_MAC_SPEED_100G	= 100000	/* 100000 Mbps = 100 Gbps */
+};
+
+enum HCLGE_MAC_DUPLEX {
+	HCLGE_MAC_HALF,
+	HCLGE_MAC_FULL
+};
+
+enum hclge_mta_dmac_sel_type {
+	HCLGE_MAC_ADDR_47_36,
+	HCLGE_MAC_ADDR_46_35,
+	HCLGE_MAC_ADDR_45_34,
+	HCLGE_MAC_ADDR_44_33,
+};
+
+struct hclge_mac {
+	u8 phy_addr;
+	u8 flag;
+	u8 media_type;
+	u8 mac_addr[ETH_ALEN];
+	u8 autoneg;
+	u8 duplex;
+	u32 speed;
+	int link;	/* store the link status of mac & phy (if phy exit)*/
+	struct phy_device *phydev;
+	struct mii_bus *mdio_bus;
+	phy_interface_t phy_if;
+};
+
+struct hclge_hw {
+	void __iomem *io_base;
+	struct hclge_mac mac;
+	int num_vec;
+	struct hclge_cmq cmq;
+	struct hclge_caps caps;
+	void *back;
+};
+
+/* TQP stats */
+struct hlcge_tqp_stats {
+	/* query_tqp_tx_queue_statistics ,opcode id:  0x0B03 */
+	u64 rcb_tx_ring_pktnum_rcd; /* 32bit */
+	/* query_tqp_rx_queue_statistics ,opcode id:  0x0B13 */
+	u64 rcb_rx_ring_pktnum_rcd; /* 32bit */
+};
+
+struct hclge_tqp {
+	struct device *dev;	/* Device for DMA mapping */
+	struct hnae3_queue q;
+	struct hlcge_tqp_stats tqp_stats;
+	u16 index;	/* Global index in a NIC controller */
+
+	bool alloced;
+};
+
+enum hclge_fc_mode {
+	HCLGE_FC_NONE,
+	HCLGE_FC_RX_PAUSE,
+	HCLGE_FC_TX_PAUSE,
+	HCLGE_FC_FULL,
+	HCLGE_FC_PFC,
+	HCLGE_FC_DEFAULT
+};
+
+#define HCLGE_PG_NUM		4
+#define HCLGE_SCH_MODE_SP	0
+#define HCLGE_SCH_MODE_DWRR	1
+struct hclge_pg_info {
+	u8 pg_id;
+	u8 pg_sch_mode;		/* 0: sp; 1: dwrr */
+	u8 tc_bit_map;
+	u32 bw_limit;
+	u8 tc_dwrr[HNAE3_MAX_TC];
+};
+
+struct hclge_tc_info {
+	u8 tc_id;
+	u8 tc_sch_mode;		/* 0: sp; 1: dwrr */
+	u8 up;
+	u8 pgid;
+	u32 bw_limit;
+};
+
+struct hclge_cfg {
+	u8 vmdq_vport_num;
+	u8 tc_num;
+	u16 tqp_desc_num;
+	u16 rx_buf_len;
+	u8 phy_addr;
+	u8 media_type;
+	u8 mac_addr[ETH_ALEN];
+	u8 default_speed;
+	u32 numa_node_map;
+};
+
+struct hclge_tm_info {
+	u8 num_tc;
+	u8 num_pg;      /* It must be 1 if vNET-Base schd */
+	u8 pg_dwrr[HCLGE_PG_NUM];
+	struct hclge_pg_info pg_info[HCLGE_PG_NUM];
+	struct hclge_tc_info tc_info[HNAE3_MAX_TC];
+	enum hclge_fc_mode fc_mode;
+	u8 hw_pfc_map; /* Allow for packet drop or not on this TC */
+};
+
+struct hclge_comm_stats_str {
+	char desc[ETH_GSTRING_LEN];
+	unsigned long offset;
+};
+
+/* all 64bit stats, opcode id: 0x0030 */
+struct hclge_64_bit_stats {
+	/* query_igu_stat */
+	u64 igu_rx_oversize_pkt;
+	u64 igu_rx_undersize_pkt;
+	u64 igu_rx_out_all_pkt;
+	u64 igu_rx_uni_pkt;
+	u64 igu_rx_multi_pkt;
+	u64 igu_rx_broad_pkt;
+	u64 rsv0;
+
+	/* query_egu_stat */
+	u64 egu_tx_out_all_pkt;
+	u64 egu_tx_uni_pkt;
+	u64 egu_tx_multi_pkt;
+	u64 egu_tx_broad_pkt;
+
+	/* ssu_ppp packet stats */
+	u64 ssu_ppp_mac_key_num;
+	u64 ssu_ppp_host_key_num;
+	u64 ppp_ssu_mac_rlt_num;
+	u64 ppp_ssu_host_rlt_num;
+
+	/* ssu_tx_in_out_dfx_stats */
+	u64 ssu_tx_in_num;
+	u64 ssu_tx_out_num;
+	/* ssu_rx_in_out_dfx_stats */
+	u64 ssu_rx_in_num;
+	u64 ssu_rx_out_num;
+};
+
+/* all 32bit stats, opcode id: 0x0031 */
+struct hclge_32_bit_stats {
+	u64 igu_rx_err_pkt;
+	u64 igu_rx_no_eof_pkt;
+	u64 igu_rx_no_sof_pkt;
+	u64 egu_tx_1588_pkt;
+	u64 egu_tx_err_pkt;
+	u64 ssu_full_drop_num;
+	u64 ssu_part_drop_num;
+	u64 ppp_key_drop_num;
+	u64 ppp_rlt_drop_num;
+	u64 ssu_key_drop_num;
+	u64 pkt_curr_buf_cnt;
+	u64 qcn_fb_rcv_cnt;
+	u64 qcn_fb_drop_cnt;
+	u64 qcn_fb_invaild_cnt;
+	u64 rsv0;
+	u64 rx_packet_tc0_in_cnt;
+	u64 rx_packet_tc1_in_cnt;
+	u64 rx_packet_tc2_in_cnt;
+	u64 rx_packet_tc3_in_cnt;
+	u64 rx_packet_tc4_in_cnt;
+	u64 rx_packet_tc5_in_cnt;
+	u64 rx_packet_tc6_in_cnt;
+	u64 rx_packet_tc7_in_cnt;
+	u64 rx_packet_tc0_out_cnt;
+	u64 rx_packet_tc1_out_cnt;
+	u64 rx_packet_tc2_out_cnt;
+	u64 rx_packet_tc3_out_cnt;
+	u64 rx_packet_tc4_out_cnt;
+	u64 rx_packet_tc5_out_cnt;
+	u64 rx_packet_tc6_out_cnt;
+	u64 rx_packet_tc7_out_cnt;
+
+	/* Tx packet level statistics */
+	u64 tx_packet_tc0_in_cnt;
+	u64 tx_packet_tc1_in_cnt;
+	u64 tx_packet_tc2_in_cnt;
+	u64 tx_packet_tc3_in_cnt;
+	u64 tx_packet_tc4_in_cnt;
+	u64 tx_packet_tc5_in_cnt;
+	u64 tx_packet_tc6_in_cnt;
+	u64 tx_packet_tc7_in_cnt;
+	u64 tx_packet_tc0_out_cnt;
+	u64 tx_packet_tc1_out_cnt;
+	u64 tx_packet_tc2_out_cnt;
+	u64 tx_packet_tc3_out_cnt;
+	u64 tx_packet_tc4_out_cnt;
+	u64 tx_packet_tc5_out_cnt;
+	u64 tx_packet_tc6_out_cnt;
+	u64 tx_packet_tc7_out_cnt;
+
+	/* packet buffer statistics */
+	u64 pkt_curr_buf_tc0_cnt;
+	u64 pkt_curr_buf_tc1_cnt;
+	u64 pkt_curr_buf_tc2_cnt;
+	u64 pkt_curr_buf_tc3_cnt;
+	u64 pkt_curr_buf_tc4_cnt;
+	u64 pkt_curr_buf_tc5_cnt;
+	u64 pkt_curr_buf_tc6_cnt;
+	u64 pkt_curr_buf_tc7_cnt;
+
+	u64 mb_uncopy_num;
+	u64 lo_pri_unicast_rlt_drop_num;
+	u64 hi_pri_multicast_rlt_drop_num;
+	u64 lo_pri_multicast_rlt_drop_num;
+	u64 rx_oq_drop_pkt_cnt;
+	u64 tx_oq_drop_pkt_cnt;
+	u64 nic_l2_err_drop_pkt_cnt;
+	u64 roc_l2_err_drop_pkt_cnt;
+};
+
+/* mac stats ,opcode id: 0x0032 */
+struct hclge_mac_stats {
+	u64 mac_tx_mac_pause_num;
+	u64 mac_rx_mac_pause_num;
+	u64 mac_tx_pfc_pri0_pkt_num;
+	u64 mac_tx_pfc_pri1_pkt_num;
+	u64 mac_tx_pfc_pri2_pkt_num;
+	u64 mac_tx_pfc_pri3_pkt_num;
+	u64 mac_tx_pfc_pri4_pkt_num;
+	u64 mac_tx_pfc_pri5_pkt_num;
+	u64 mac_tx_pfc_pri6_pkt_num;
+	u64 mac_tx_pfc_pri7_pkt_num;
+	u64 mac_rx_pfc_pri0_pkt_num;
+	u64 mac_rx_pfc_pri1_pkt_num;
+	u64 mac_rx_pfc_pri2_pkt_num;
+	u64 mac_rx_pfc_pri3_pkt_num;
+	u64 mac_rx_pfc_pri4_pkt_num;
+	u64 mac_rx_pfc_pri5_pkt_num;
+	u64 mac_rx_pfc_pri6_pkt_num;
+	u64 mac_rx_pfc_pri7_pkt_num;
+	u64 mac_tx_total_pkt_num;
+	u64 mac_tx_total_oct_num;
+	u64 mac_tx_good_pkt_num;
+	u64 mac_tx_bad_pkt_num;
+	u64 mac_tx_good_oct_num;
+	u64 mac_tx_bad_oct_num;
+	u64 mac_tx_uni_pkt_num;
+	u64 mac_tx_multi_pkt_num;
+	u64 mac_tx_broad_pkt_num;
+	u64 mac_tx_undersize_pkt_num;
+	u64 mac_tx_overrsize_pkt_num;
+	u64 mac_tx_64_oct_pkt_num;
+	u64 mac_tx_65_127_oct_pkt_num;
+	u64 mac_tx_128_255_oct_pkt_num;
+	u64 mac_tx_256_511_oct_pkt_num;
+	u64 mac_tx_512_1023_oct_pkt_num;
+	u64 mac_tx_1024_1518_oct_pkt_num;
+	u64 mac_tx_1519_max_oct_pkt_num;
+	u64 mac_rx_total_pkt_num;
+	u64 mac_rx_total_oct_num;
+	u64 mac_rx_good_pkt_num;
+	u64 mac_rx_bad_pkt_num;
+	u64 mac_rx_good_oct_num;
+	u64 mac_rx_bad_oct_num;
+	u64 mac_rx_uni_pkt_num;
+	u64 mac_rx_multi_pkt_num;
+	u64 mac_rx_broad_pkt_num;
+	u64 mac_rx_undersize_pkt_num;
+	u64 mac_rx_overrsize_pkt_num;
+	u64 mac_rx_64_oct_pkt_num;
+	u64 mac_rx_65_127_oct_pkt_num;
+	u64 mac_rx_128_255_oct_pkt_num;
+	u64 mac_rx_256_511_oct_pkt_num;
+	u64 mac_rx_512_1023_oct_pkt_num;
+	u64 mac_rx_1024_1518_oct_pkt_num;
+	u64 mac_rx_1519_max_oct_pkt_num;
+
+	u64 mac_trans_fragment_pkt_num;
+	u64 mac_trans_undermin_pkt_num;
+	u64 mac_trans_jabber_pkt_num;
+	u64 mac_trans_err_all_pkt_num;
+	u64 mac_trans_from_app_good_pkt_num;
+	u64 mac_trans_from_app_bad_pkt_num;
+	u64 mac_rcv_fragment_pkt_num;
+	u64 mac_rcv_undermin_pkt_num;
+	u64 mac_rcv_jabber_pkt_num;
+	u64 mac_rcv_fcs_err_pkt_num;
+	u64 mac_rcv_send_app_good_pkt_num;
+	u64 mac_rcv_send_app_bad_pkt_num;
+};
+
+struct hclge_hw_stats {
+	struct hclge_mac_stats      mac_stats;
+	struct hclge_64_bit_stats   all_64_bit_stats;
+	struct hclge_32_bit_stats   all_32_bit_stats;
+};
+
+struct hclge_dev {
+	struct pci_dev *pdev;
+	struct hnae3_ae_dev *ae_dev;
+	struct hclge_hw hw;
+	struct hclge_hw_stats hw_stats;
+	unsigned long state;
+
+	u32 fw_version;
+	u16 num_vmdq_vport;		/* Num vmdq vport this PF has set up */
+	u16 num_tqps;			/* Num task queue pairs of this PF */
+	u16 num_req_vfs;		/* Num VFs requested for this PF */
+
+	u16 num_roce_msix;		/* Num of roce vectors for this PF */
+	int roce_base_vector;
+
+	/* Base task tqp physical id of this PF */
+	u16 base_tqp_pid;
+	u16 alloc_rss_size;		/* Allocated RSS task queue */
+	u16 rss_size_max;		/* HW defined max RSS task queue */
+
+	/* Num of guaranteed filters for this PF */
+	u16 fdir_pf_filter_count;
+	u16 num_alloc_vport;		/* Num vports this driver supports */
+	u32 numa_node_mask;
+	u16 rx_buf_len;
+	u16 num_desc;
+	u8 hw_tc_map;
+	u8 tc_num_last_time;
+	enum hclge_fc_mode fc_mode_last_time;
+
+#define HCLGE_FLAG_TC_BASE_SCH_MODE		1
+#define HCLGE_FLAG_VNET_BASE_SCH_MODE		2
+	u8 tx_sch_mode;
+
+	u8 default_up;
+	struct hclge_tm_info tm_info;
+
+	u16 num_msi;
+	u16 num_msi_left;
+	u16 num_msi_used;
+	u32 base_msi_vector;
+	struct msix_entry *msix_entries;
+	u16 *vector_status;
+
+	u16 pending_udp_bitmap;
+
+	u16 rx_itr_default;
+	u16 tx_itr_default;
+
+	u16 adminq_work_limit; /* Num of admin receive queue desc to process */
+	unsigned long service_timer_period;
+	unsigned long service_timer_previous;
+	struct timer_list service_timer;
+	struct work_struct service_task;
+
+	bool cur_promisc;
+	int num_alloc_vfs;	/* Actual number of VFs allocated */
+
+	struct hclge_tqp *htqp;
+	struct hclge_vport *vport;
+
+	struct dentry *hclge_dbgfs;
+
+	struct hnae3_client *nic_client;
+	struct hnae3_client *roce_client;
+
+#define HCLGE_FLAG_USE_MSI	0x00000001
+#define HCLGE_FLAG_USE_MSIX	0x00000002
+#define HCLGE_FLAG_MAIN		0x00000004
+#define HCLGE_FLAG_DCB_CAPABLE	0x00000008
+#define HCLGE_FLAG_DCB_ENABLE	0x00000010
+	u32 flag;
+
+	u32 pkt_buf_size; /* Total pf buf size for tx/rx */
+	u32 mps; /* Max packet size */
+	struct hclge_priv_buf *priv_buf;
+	struct hclge_shared_buf s_buf;
+
+	enum hclge_mta_dmac_sel_type mta_mac_sel_type;
+	bool enable_mta; /* Mutilcast filter enable */
+	bool accept_mta_mc; /* Whether accept mta filter multicast */
+};
+
+struct hclge_vport {
+	u16 alloc_tqps;	/* Allocated Tx/Rx queues */
+
+	u8  rss_hash_key[HCLGE_RSS_KEY_SIZE]; /* User configured hash keys */
+	/* User configured lookup table entries */
+	u8  rss_indirection_tbl[HCLGE_RSS_IND_TBL_SIZE];
+
+	u16 qs_offset;
+	u16 bw_limit;		/* VSI BW Limit (0 = disabled) */
+	u8  dwrr;
+
+	int vport_id;
+	struct hclge_dev *back;  /* Back reference to associated dev */
+	struct hnae3_handle nic;
+	struct hnae3_handle roce;
+};
+
+void hclge_promisc_param_init(struct hclge_promisc_param *param, bool en_uc,
+			      bool en_mc, bool en_bc, int vport_id);
+
+int hclge_add_uc_addr_common(struct hclge_vport *vport,
+			     const unsigned char *addr);
+int hclge_rm_uc_addr_common(struct hclge_vport *vport,
+			    const unsigned char *addr);
+int hclge_add_mc_addr_common(struct hclge_vport *vport,
+			     const unsigned char *addr);
+int hclge_rm_mc_addr_common(struct hclge_vport *vport,
+			    const unsigned char *addr);
+
+int hclge_cfg_func_mta_filter(struct hclge_dev *hdev,
+			      u8 func_id,
+			      bool enable);
+struct hclge_vport *hclge_get_vport(struct hnae3_handle *handle);
+int hclge_map_vport_ring_to_vector(struct hclge_vport *vport, int vector,
+				   struct hnae3_ring_chain_node *ring_chain);
+static inline int hclge_get_queue_id(struct hnae3_queue *queue)
+{
+	struct hclge_tqp *tqp = container_of(queue, struct hclge_tqp, q);
+
+	return tqp->index;
+}
+
+int hclge_cfg_mac_speed_dup(struct hclge_dev *hdev, int speed, u8 duplex);
+int hclge_set_vf_vlan_common(struct hclge_dev *vport, int vfid,
+			     bool is_kill, u16 vlan, u8 qos, __be16 proto);
+#endif
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c
new file mode 100644
index 000000000000..f32d719c4f77
--- /dev/null
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c
@@ -0,0 +1,213 @@
+/*
+ * Copyright (c) 2016~2017 Hisilicon Limited.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/etherdevice.h>
+#include <linux/kernel.h>
+
+#include "hclge_cmd.h"
+#include "hclge_main.h"
+#include "hclge_mdio.h"
+
+enum hclge_mdio_c22_op_seq {
+	HCLGE_MDIO_C22_WRITE = 1,
+	HCLGE_MDIO_C22_READ = 2
+};
+
+#define HCLGE_MDIO_CTRL_START_B		0
+#define HCLGE_MDIO_CTRL_ST_S		1
+#define HCLGE_MDIO_CTRL_ST_M		(0x3 << HCLGE_MDIO_CTRL_ST_S)
+#define HCLGE_MDIO_CTRL_OP_S		3
+#define HCLGE_MDIO_CTRL_OP_M		(0x3 << HCLGE_MDIO_CTRL_OP_S)
+
+#define HCLGE_MDIO_PHYID_S		0
+#define HCLGE_MDIO_PHYID_M		(0x1f << HCLGE_MDIO_PHYID_S)
+
+#define HCLGE_MDIO_PHYREG_S		0
+#define HCLGE_MDIO_PHYREG_M		(0x1f << HCLGE_MDIO_PHYREG_S)
+
+#define HCLGE_MDIO_STA_B		0
+
+struct hclge_mdio_cfg_cmd {
+	u8 ctrl_bit;
+	u8 phyid;
+	u8 phyad;
+	u8 rsvd;
+	__le16 reserve;
+	__le16 data_wr;
+	__le16 data_rd;
+	__le16 sta;
+};
+
+static int hclge_mdio_write(struct mii_bus *bus, int phyid, int regnum,
+			    u16 data)
+{
+	struct hclge_mdio_cfg_cmd *mdio_cmd;
+	struct hclge_dev *hdev = bus->priv;
+	struct hclge_desc desc;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_MDIO_CONFIG, false);
+
+	mdio_cmd = (struct hclge_mdio_cfg_cmd *)desc.data;
+
+	hnae_set_field(mdio_cmd->phyid, HCLGE_MDIO_PHYID_M,
+		       HCLGE_MDIO_PHYID_S, phyid);
+	hnae_set_field(mdio_cmd->phyad, HCLGE_MDIO_PHYREG_M,
+		       HCLGE_MDIO_PHYREG_S, regnum);
+
+	hnae_set_bit(mdio_cmd->ctrl_bit, HCLGE_MDIO_CTRL_START_B, 1);
+	hnae_set_field(mdio_cmd->ctrl_bit, HCLGE_MDIO_CTRL_ST_M,
+		       HCLGE_MDIO_CTRL_ST_S, 1);
+	hnae_set_field(mdio_cmd->ctrl_bit, HCLGE_MDIO_CTRL_OP_M,
+		       HCLGE_MDIO_CTRL_OP_S, HCLGE_MDIO_C22_WRITE);
+
+	mdio_cmd->data_wr = cpu_to_le16(data);
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"mdio write fail when sending cmd, status is %d.\n",
+			ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int hclge_mdio_read(struct mii_bus *bus, int phyid, int regnum)
+{
+	struct hclge_mdio_cfg_cmd *mdio_cmd;
+	struct hclge_dev *hdev = bus->priv;
+	struct hclge_desc desc;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_MDIO_CONFIG, true);
+
+	mdio_cmd = (struct hclge_mdio_cfg_cmd *)desc.data;
+
+	hnae_set_field(mdio_cmd->phyid, HCLGE_MDIO_PHYID_M,
+		       HCLGE_MDIO_PHYID_S, phyid);
+	hnae_set_field(mdio_cmd->phyad, HCLGE_MDIO_PHYREG_M,
+		       HCLGE_MDIO_PHYREG_S, regnum);
+
+	hnae_set_bit(mdio_cmd->ctrl_bit, HCLGE_MDIO_CTRL_START_B, 1);
+	hnae_set_field(mdio_cmd->ctrl_bit, HCLGE_MDIO_CTRL_ST_M,
+		       HCLGE_MDIO_CTRL_ST_S, 1);
+	hnae_set_field(mdio_cmd->ctrl_bit, HCLGE_MDIO_CTRL_OP_M,
+		       HCLGE_MDIO_CTRL_OP_S, HCLGE_MDIO_C22_READ);
+
+	/* Read out phy data */
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"mdio read fail when get data, status is %d.\n",
+			ret);
+		return ret;
+	}
+
+	if (hnae_get_bit(le16_to_cpu(mdio_cmd->sta), HCLGE_MDIO_STA_B)) {
+		dev_err(&hdev->pdev->dev, "mdio read data error\n");
+		return -EIO;
+	}
+
+	return le16_to_cpu(mdio_cmd->data_rd);
+}
+
+int hclge_mac_mdio_config(struct hclge_dev *hdev)
+{
+	struct hclge_mac *mac = &hdev->hw.mac;
+	struct phy_device *phydev;
+	struct mii_bus *mdio_bus;
+	int ret;
+
+	if (hdev->hw.mac.phy_addr >= PHY_MAX_ADDR)
+		return 0;
+
+	mdio_bus = devm_mdiobus_alloc(&hdev->pdev->dev);
+	if (!mdio_bus)
+		return -ENOMEM;
+
+	mdio_bus->name = "hisilicon MII bus";
+	mdio_bus->read = hclge_mdio_read;
+	mdio_bus->write = hclge_mdio_write;
+	snprintf(mdio_bus->id, MII_BUS_ID_SIZE, "%s-%s", "mii",
+		 dev_name(&hdev->pdev->dev));
+
+	mdio_bus->parent = &hdev->pdev->dev;
+	mdio_bus->priv = hdev;
+	mdio_bus->phy_mask = ~(1 << mac->phy_addr);
+	ret = mdiobus_register(mdio_bus);
+	if (ret) {
+		dev_err(mdio_bus->parent,
+			"Failed to register MDIO bus ret = %#x\n", ret);
+		return ret;
+	}
+
+	phydev = mdiobus_get_phy(mdio_bus, mac->phy_addr);
+	if (!phydev) {
+		dev_err(mdio_bus->parent, "Failed to get phy device\n");
+		mdiobus_unregister(mdio_bus);
+		return -EIO;
+	}
+
+	mac->phydev = phydev;
+	mac->mdio_bus = mdio_bus;
+
+	return 0;
+}
+
+static void hclge_mac_adjust_link(struct net_device *netdev)
+{
+	struct hnae3_handle *h = *((void **)netdev_priv(netdev));
+	struct hclge_vport *vport = hclge_get_vport(h);
+	struct hclge_dev *hdev = vport->back;
+	int duplex, speed;
+	int ret;
+
+	speed = netdev->phydev->speed;
+	duplex = netdev->phydev->duplex;
+
+	ret = hclge_cfg_mac_speed_dup(hdev, speed, duplex);
+	if (ret)
+		netdev_err(netdev, "failed to adjust link.\n");
+}
+
+int hclge_mac_start_phy(struct hclge_dev *hdev)
+{
+	struct net_device *netdev = hdev->vport[0].nic.netdev;
+	struct phy_device *phydev = hdev->hw.mac.phydev;
+	int ret;
+
+	if (!phydev)
+		return 0;
+
+	ret = phy_connect_direct(netdev, phydev,
+				 hclge_mac_adjust_link,
+				 PHY_INTERFACE_MODE_SGMII);
+	if (ret) {
+		netdev_err(netdev, "phy_connect_direct err.\n");
+		return ret;
+	}
+
+	phy_start(phydev);
+
+	return 0;
+}
+
+void hclge_mac_stop_phy(struct hclge_dev *hdev)
+{
+	struct net_device *netdev = hdev->vport[0].nic.netdev;
+	struct phy_device *phydev = netdev->phydev;
+
+	if (!phydev)
+		return;
+
+	phy_stop(phydev);
+	phy_disconnect(phydev);
+}
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.h
new file mode 100644
index 000000000000..c5e91cfb8f2c
--- /dev/null
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.h
@@ -0,0 +1,17 @@
+/*
+ * Copyright (c) 2016-2017 Hisilicon Limited.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef __HCLGE_MDIO_H
+#define __HCLGE_MDIO_H
+
+int hclge_mac_mdio_config(struct hclge_dev *hdev);
+int hclge_mac_start_phy(struct hclge_dev *hdev);
+void hclge_mac_stop_phy(struct hclge_dev *hdev);
+
+#endif
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c
new file mode 100644
index 000000000000..1c577d268f00
--- /dev/null
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c
@@ -0,0 +1,1015 @@
+/*
+ * Copyright (c) 2016~2017 Hisilicon Limited.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/etherdevice.h>
+
+#include "hclge_cmd.h"
+#include "hclge_main.h"
+#include "hclge_tm.h"
+
+enum hclge_shaper_level {
+	HCLGE_SHAPER_LVL_PRI	= 0,
+	HCLGE_SHAPER_LVL_PG	= 1,
+	HCLGE_SHAPER_LVL_PORT	= 2,
+	HCLGE_SHAPER_LVL_QSET	= 3,
+	HCLGE_SHAPER_LVL_CNT	= 4,
+	HCLGE_SHAPER_LVL_VF	= 0,
+	HCLGE_SHAPER_LVL_PF	= 1,
+};
+
+#define HCLGE_SHAPER_BS_U_DEF	1
+#define HCLGE_SHAPER_BS_S_DEF	4
+
+#define HCLGE_ETHER_MAX_RATE	100000
+
+/* hclge_shaper_para_calc: calculate ir parameter for the shaper
+ * @ir: Rate to be config, its unit is Mbps
+ * @shaper_level: the shaper level. eg: port, pg, priority, queueset
+ * @ir_b: IR_B parameter of IR shaper
+ * @ir_u: IR_U parameter of IR shaper
+ * @ir_s: IR_S parameter of IR shaper
+ *
+ * the formula:
+ *
+ *		IR_b * (2 ^ IR_u) * 8
+ * IR(Mbps) = -------------------------  *  CLOCK(1000Mbps)
+ *		Tick * (2 ^ IR_s)
+ *
+ * @return: 0: calculate sucessful, negative: fail
+ */
+static int hclge_shaper_para_calc(u32 ir, u8 shaper_level,
+				  u8 *ir_b, u8 *ir_u, u8 *ir_s)
+{
+	const u16 tick_array[HCLGE_SHAPER_LVL_CNT] = {
+		6 * 256,        /* Prioriy level */
+		6 * 32,         /* Prioriy group level */
+		6 * 8,          /* Port level */
+		6 * 256         /* Qset level */
+	};
+	u8 ir_u_calc = 0, ir_s_calc = 0;
+	u32 ir_calc;
+	u32 tick;
+
+	/* Calc tick */
+	if (shaper_level >= HCLGE_SHAPER_LVL_CNT)
+		return -EINVAL;
+
+	tick = tick_array[shaper_level];
+
+	/**
+	 * Calc the speed if ir_b = 126, ir_u = 0 and ir_s = 0
+	 * the formula is changed to:
+	 *		126 * 1 * 8
+	 * ir_calc = ---------------- * 1000
+	 *		tick * 1
+	 */
+	ir_calc = (1008000 + (tick >> 1) - 1) / tick;
+
+	if (ir_calc == ir) {
+		*ir_b = 126;
+		*ir_u = 0;
+		*ir_s = 0;
+
+		return 0;
+	} else if (ir_calc > ir) {
+		/* Increasing the denominator to select ir_s value */
+		while (ir_calc > ir) {
+			ir_s_calc++;
+			ir_calc = 1008000 / (tick * (1 << ir_s_calc));
+		}
+
+		if (ir_calc == ir)
+			*ir_b = 126;
+		else
+			*ir_b = (ir * tick * (1 << ir_s_calc) + 4000) / 8000;
+	} else {
+		/* Increasing the numerator to select ir_u value */
+		u32 numerator;
+
+		while (ir_calc < ir) {
+			ir_u_calc++;
+			numerator = 1008000 * (1 << ir_u_calc);
+			ir_calc = (numerator + (tick >> 1)) / tick;
+		}
+
+		if (ir_calc == ir) {
+			*ir_b = 126;
+		} else {
+			u32 denominator = (8000 * (1 << --ir_u_calc));
+			*ir_b = (ir * tick + (denominator >> 1)) / denominator;
+		}
+	}
+
+	*ir_u = ir_u_calc;
+	*ir_s = ir_s_calc;
+
+	return 0;
+}
+
+static int hclge_mac_pause_en_cfg(struct hclge_dev *hdev, bool tx, bool rx)
+{
+	struct hclge_desc desc;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_CFG_MAC_PAUSE_EN, false);
+
+	desc.data[0] = cpu_to_le32((tx ? HCLGE_TX_MAC_PAUSE_EN_MSK : 0) |
+		(rx ? HCLGE_RX_MAC_PAUSE_EN_MSK : 0));
+
+	return hclge_cmd_send(&hdev->hw, &desc, 1);
+}
+
+static int hclge_fill_pri_array(struct hclge_dev *hdev, u8 *pri, u8 pri_id)
+{
+	u8 tc;
+
+	for (tc = 0; tc < hdev->tm_info.num_tc; tc++)
+		if (hdev->tm_info.tc_info[tc].up == pri_id)
+			break;
+
+	if (tc >= hdev->tm_info.num_tc)
+		return -EINVAL;
+
+	/**
+	 * the register for priority has four bytes, the first bytes includes
+	 *  priority0 and priority1, the higher 4bit stands for priority1
+	 *  while the lower 4bit stands for priority0, as below:
+	 * first byte:	| pri_1 | pri_0 |
+	 * second byte:	| pri_3 | pri_2 |
+	 * third byte:	| pri_5 | pri_4 |
+	 * fourth byte:	| pri_7 | pri_6 |
+	 */
+	pri[pri_id >> 1] |= tc << ((pri_id & 1) * 4);
+
+	return 0;
+}
+
+static int hclge_up_to_tc_map(struct hclge_dev *hdev)
+{
+	struct hclge_desc desc;
+	u8 *pri = (u8 *)desc.data;
+	u8 pri_id;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_PRI_TO_TC_MAPPING, false);
+
+	for (pri_id = 0; pri_id < hdev->tm_info.num_tc; pri_id++) {
+		ret = hclge_fill_pri_array(hdev, pri, pri_id);
+		if (ret)
+			return ret;
+	}
+
+	return hclge_cmd_send(&hdev->hw, &desc, 1);
+}
+
+static int hclge_tm_pg_to_pri_map_cfg(struct hclge_dev *hdev,
+				      u8 pg_id, u8 pri_bit_map)
+{
+	struct hclge_pg_to_pri_link_cmd *map;
+	struct hclge_desc desc;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_TM_PG_TO_PRI_LINK, false);
+
+	map = (struct hclge_pg_to_pri_link_cmd *)desc.data;
+
+	map->pg_id = pg_id;
+	map->pri_bit_map = pri_bit_map;
+
+	return hclge_cmd_send(&hdev->hw, &desc, 1);
+}
+
+static int hclge_tm_qs_to_pri_map_cfg(struct hclge_dev *hdev,
+				      u16 qs_id, u8 pri)
+{
+	struct hclge_qs_to_pri_link_cmd *map;
+	struct hclge_desc desc;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_TM_QS_TO_PRI_LINK, false);
+
+	map = (struct hclge_qs_to_pri_link_cmd *)desc.data;
+
+	map->qs_id = cpu_to_le16(qs_id);
+	map->priority = pri;
+	map->link_vld = HCLGE_TM_QS_PRI_LINK_VLD_MSK;
+
+	return hclge_cmd_send(&hdev->hw, &desc, 1);
+}
+
+static int hclge_tm_q_to_qs_map_cfg(struct hclge_dev *hdev,
+				    u8 q_id, u16 qs_id)
+{
+	struct hclge_nq_to_qs_link_cmd *map;
+	struct hclge_desc desc;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_TM_NQ_TO_QS_LINK, false);
+
+	map = (struct hclge_nq_to_qs_link_cmd *)desc.data;
+
+	map->nq_id = cpu_to_le16(q_id);
+	map->qset_id = cpu_to_le16(qs_id | HCLGE_TM_Q_QS_LINK_VLD_MSK);
+
+	return hclge_cmd_send(&hdev->hw, &desc, 1);
+}
+
+static int hclge_tm_pg_weight_cfg(struct hclge_dev *hdev, u8 pg_id,
+				  u8 dwrr)
+{
+	struct hclge_pg_weight_cmd *weight;
+	struct hclge_desc desc;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_TM_PG_WEIGHT, false);
+
+	weight = (struct hclge_pg_weight_cmd *)desc.data;
+
+	weight->pg_id = pg_id;
+	weight->dwrr = dwrr;
+
+	return hclge_cmd_send(&hdev->hw, &desc, 1);
+}
+
+static int hclge_tm_pri_weight_cfg(struct hclge_dev *hdev, u8 pri_id,
+				   u8 dwrr)
+{
+	struct hclge_priority_weight_cmd *weight;
+	struct hclge_desc desc;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_TM_PRI_WEIGHT, false);
+
+	weight = (struct hclge_priority_weight_cmd *)desc.data;
+
+	weight->pri_id = pri_id;
+	weight->dwrr = dwrr;
+
+	return hclge_cmd_send(&hdev->hw, &desc, 1);
+}
+
+static int hclge_tm_qs_weight_cfg(struct hclge_dev *hdev, u16 qs_id,
+				  u8 dwrr)
+{
+	struct hclge_qs_weight_cmd *weight;
+	struct hclge_desc desc;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_TM_QS_WEIGHT, false);
+
+	weight = (struct hclge_qs_weight_cmd *)desc.data;
+
+	weight->qs_id = cpu_to_le16(qs_id);
+	weight->dwrr = dwrr;
+
+	return hclge_cmd_send(&hdev->hw, &desc, 1);
+}
+
+static int hclge_tm_pg_shapping_cfg(struct hclge_dev *hdev,
+				    enum hclge_shap_bucket bucket, u8 pg_id,
+				    u8 ir_b, u8 ir_u, u8 ir_s, u8 bs_b, u8 bs_s)
+{
+	struct hclge_pg_shapping_cmd *shap_cfg_cmd;
+	enum hclge_opcode_type opcode;
+	struct hclge_desc desc;
+
+	opcode = bucket ? HCLGE_OPC_TM_PG_P_SHAPPING :
+		HCLGE_OPC_TM_PG_C_SHAPPING;
+	hclge_cmd_setup_basic_desc(&desc, opcode, false);
+
+	shap_cfg_cmd = (struct hclge_pg_shapping_cmd *)desc.data;
+
+	shap_cfg_cmd->pg_id = pg_id;
+
+	hclge_tm_set_feild(shap_cfg_cmd->pg_shapping_para, IR_B, ir_b);
+	hclge_tm_set_feild(shap_cfg_cmd->pg_shapping_para, IR_U, ir_u);
+	hclge_tm_set_feild(shap_cfg_cmd->pg_shapping_para, IR_S, ir_s);
+	hclge_tm_set_feild(shap_cfg_cmd->pg_shapping_para, BS_B, bs_b);
+	hclge_tm_set_feild(shap_cfg_cmd->pg_shapping_para, BS_S, bs_s);
+
+	return hclge_cmd_send(&hdev->hw, &desc, 1);
+}
+
+static int hclge_tm_pri_shapping_cfg(struct hclge_dev *hdev,
+				     enum hclge_shap_bucket bucket, u8 pri_id,
+				     u8 ir_b, u8 ir_u, u8 ir_s,
+				     u8 bs_b, u8 bs_s)
+{
+	struct hclge_pri_shapping_cmd *shap_cfg_cmd;
+	enum hclge_opcode_type opcode;
+	struct hclge_desc desc;
+
+	opcode = bucket ? HCLGE_OPC_TM_PRI_P_SHAPPING :
+		HCLGE_OPC_TM_PRI_C_SHAPPING;
+
+	hclge_cmd_setup_basic_desc(&desc, opcode, false);
+
+	shap_cfg_cmd = (struct hclge_pri_shapping_cmd *)desc.data;
+
+	shap_cfg_cmd->pri_id = pri_id;
+
+	hclge_tm_set_feild(shap_cfg_cmd->pri_shapping_para, IR_B, ir_b);
+	hclge_tm_set_feild(shap_cfg_cmd->pri_shapping_para, IR_U, ir_u);
+	hclge_tm_set_feild(shap_cfg_cmd->pri_shapping_para, IR_S, ir_s);
+	hclge_tm_set_feild(shap_cfg_cmd->pri_shapping_para, BS_B, bs_b);
+	hclge_tm_set_feild(shap_cfg_cmd->pri_shapping_para, BS_S, bs_s);
+
+	return hclge_cmd_send(&hdev->hw, &desc, 1);
+}
+
+static int hclge_tm_pg_schd_mode_cfg(struct hclge_dev *hdev, u8 pg_id)
+{
+	struct hclge_desc desc;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_TM_PG_SCH_MODE_CFG, false);
+
+	if (hdev->tm_info.pg_info[pg_id].pg_sch_mode == HCLGE_SCH_MODE_DWRR)
+		desc.data[1] = cpu_to_le32(HCLGE_TM_TX_SCHD_DWRR_MSK);
+	else
+		desc.data[1] = 0;
+
+	desc.data[0] = cpu_to_le32(pg_id);
+
+	return hclge_cmd_send(&hdev->hw, &desc, 1);
+}
+
+static int hclge_tm_pri_schd_mode_cfg(struct hclge_dev *hdev, u8 pri_id)
+{
+	struct hclge_desc desc;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_TM_PRI_SCH_MODE_CFG, false);
+
+	if (hdev->tm_info.tc_info[pri_id].tc_sch_mode == HCLGE_SCH_MODE_DWRR)
+		desc.data[1] = cpu_to_le32(HCLGE_TM_TX_SCHD_DWRR_MSK);
+	else
+		desc.data[1] = 0;
+
+	desc.data[0] = cpu_to_le32(pri_id);
+
+	return hclge_cmd_send(&hdev->hw, &desc, 1);
+}
+
+static int hclge_tm_qs_schd_mode_cfg(struct hclge_dev *hdev, u16 qs_id)
+{
+	struct hclge_desc desc;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_TM_QS_SCH_MODE_CFG, false);
+
+	if (hdev->tm_info.tc_info[qs_id].tc_sch_mode == HCLGE_SCH_MODE_DWRR)
+		desc.data[1] = cpu_to_le32(HCLGE_TM_TX_SCHD_DWRR_MSK);
+	else
+		desc.data[1] = 0;
+
+	desc.data[0] = cpu_to_le32(qs_id);
+
+	return hclge_cmd_send(&hdev->hw, &desc, 1);
+}
+
+static int hclge_tm_qs_bp_cfg(struct hclge_dev *hdev, u8 tc)
+{
+	struct hclge_bp_to_qs_map_cmd *bp_to_qs_map_cmd;
+	struct hclge_desc desc;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_TM_BP_TO_QSET_MAPPING,
+				   false);
+
+	bp_to_qs_map_cmd = (struct hclge_bp_to_qs_map_cmd *)desc.data;
+
+	bp_to_qs_map_cmd->tc_id = tc;
+
+	/* Qset and tc is one by one mapping */
+	bp_to_qs_map_cmd->qs_bit_map = cpu_to_le32(1 << tc);
+
+	return hclge_cmd_send(&hdev->hw, &desc, 1);
+}
+
+static void hclge_tm_vport_tc_info_update(struct hclge_vport *vport)
+{
+	struct hnae3_knic_private_info *kinfo = &vport->nic.kinfo;
+	struct hclge_dev *hdev = vport->back;
+	u8 i;
+
+	kinfo = &vport->nic.kinfo;
+	vport->bw_limit = hdev->tm_info.pg_info[0].bw_limit;
+	kinfo->num_tc =
+		min_t(u16, kinfo->num_tqps, hdev->tm_info.num_tc);
+	kinfo->rss_size
+		= min_t(u16, hdev->rss_size_max,
+			kinfo->num_tqps / kinfo->num_tc);
+	vport->qs_offset = hdev->tm_info.num_tc * vport->vport_id;
+	vport->dwrr = 100;  /* 100 percent as init */
+
+	for (i = 0; i < kinfo->num_tc; i++) {
+		if (hdev->hw_tc_map & BIT(i)) {
+			kinfo->tc_info[i].enable = true;
+			kinfo->tc_info[i].tqp_offset = i * kinfo->rss_size;
+			kinfo->tc_info[i].tqp_count = kinfo->rss_size;
+			kinfo->tc_info[i].tc = i;
+			kinfo->tc_info[i].up = hdev->tm_info.tc_info[i].up;
+		} else {
+			/* Set to default queue if TC is disable */
+			kinfo->tc_info[i].enable = false;
+			kinfo->tc_info[i].tqp_offset = 0;
+			kinfo->tc_info[i].tqp_count = 1;
+			kinfo->tc_info[i].tc = 0;
+			kinfo->tc_info[i].up = 0;
+		}
+	}
+}
+
+static void hclge_tm_vport_info_update(struct hclge_dev *hdev)
+{
+	struct hclge_vport *vport = hdev->vport;
+	u32 i;
+
+	for (i = 0; i < hdev->num_alloc_vport; i++) {
+		hclge_tm_vport_tc_info_update(vport);
+
+		vport++;
+	}
+}
+
+static void hclge_tm_tc_info_init(struct hclge_dev *hdev)
+{
+	u8 i;
+
+	for (i = 0; i < hdev->tm_info.num_tc; i++) {
+		hdev->tm_info.tc_info[i].tc_id = i;
+		hdev->tm_info.tc_info[i].tc_sch_mode = HCLGE_SCH_MODE_DWRR;
+		hdev->tm_info.tc_info[i].up = i;
+		hdev->tm_info.tc_info[i].pgid = 0;
+		hdev->tm_info.tc_info[i].bw_limit =
+			hdev->tm_info.pg_info[0].bw_limit;
+	}
+
+	hdev->flag &= ~HCLGE_FLAG_DCB_ENABLE;
+}
+
+static void hclge_tm_pg_info_init(struct hclge_dev *hdev)
+{
+	u8 i;
+
+	for (i = 0; i < hdev->tm_info.num_pg; i++) {
+		int k;
+
+		hdev->tm_info.pg_dwrr[i] = i ? 0 : 100;
+
+		hdev->tm_info.pg_info[i].pg_id = i;
+		hdev->tm_info.pg_info[i].pg_sch_mode = HCLGE_SCH_MODE_DWRR;
+
+		hdev->tm_info.pg_info[i].bw_limit = HCLGE_ETHER_MAX_RATE;
+
+		if (i != 0)
+			continue;
+
+		hdev->tm_info.pg_info[i].tc_bit_map = hdev->hw_tc_map;
+		for (k = 0; k < hdev->tm_info.num_tc; k++)
+			hdev->tm_info.pg_info[i].tc_dwrr[k] = 100;
+	}
+}
+
+static int hclge_tm_schd_info_init(struct hclge_dev *hdev)
+{
+	if ((hdev->tx_sch_mode != HCLGE_FLAG_TC_BASE_SCH_MODE) &&
+	    (hdev->tm_info.num_pg != 1))
+		return -EINVAL;
+
+	hclge_tm_pg_info_init(hdev);
+
+	hclge_tm_tc_info_init(hdev);
+
+	hclge_tm_vport_info_update(hdev);
+
+	hdev->tm_info.fc_mode = HCLGE_FC_NONE;
+	hdev->fc_mode_last_time = hdev->tm_info.fc_mode;
+
+	return 0;
+}
+
+static int hclge_tm_pg_to_pri_map(struct hclge_dev *hdev)
+{
+	int ret;
+	u32 i;
+
+	if (hdev->tx_sch_mode != HCLGE_FLAG_TC_BASE_SCH_MODE)
+		return 0;
+
+	for (i = 0; i < hdev->tm_info.num_pg; i++) {
+		/* Cfg mapping */
+		ret = hclge_tm_pg_to_pri_map_cfg(
+			hdev, i, hdev->tm_info.pg_info[i].tc_bit_map);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int hclge_tm_pg_shaper_cfg(struct hclge_dev *hdev)
+{
+	u8 ir_u, ir_b, ir_s;
+	int ret;
+	u32 i;
+
+	/* Cfg pg schd */
+	if (hdev->tx_sch_mode != HCLGE_FLAG_TC_BASE_SCH_MODE)
+		return 0;
+
+	/* Pg to pri */
+	for (i = 0; i < hdev->tm_info.num_pg; i++) {
+		/* Calc shaper para */
+		ret = hclge_shaper_para_calc(
+					hdev->tm_info.pg_info[i].bw_limit,
+					HCLGE_SHAPER_LVL_PG,
+					&ir_b, &ir_u, &ir_s);
+		if (ret)
+			return ret;
+
+		ret = hclge_tm_pg_shapping_cfg(hdev,
+					       HCLGE_TM_SHAP_C_BUCKET, i,
+					       0, 0, 0, HCLGE_SHAPER_BS_U_DEF,
+					       HCLGE_SHAPER_BS_S_DEF);
+		if (ret)
+			return ret;
+
+		ret = hclge_tm_pg_shapping_cfg(hdev,
+					       HCLGE_TM_SHAP_P_BUCKET, i,
+					       ir_b, ir_u, ir_s,
+					       HCLGE_SHAPER_BS_U_DEF,
+					       HCLGE_SHAPER_BS_S_DEF);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int hclge_tm_pg_dwrr_cfg(struct hclge_dev *hdev)
+{
+	int ret;
+	u32 i;
+
+	/* cfg pg schd */
+	if (hdev->tx_sch_mode != HCLGE_FLAG_TC_BASE_SCH_MODE)
+		return 0;
+
+	/* pg to prio */
+	for (i = 0; i < hdev->tm_info.num_pg; i++) {
+		/* Cfg dwrr */
+		ret = hclge_tm_pg_weight_cfg(hdev, i,
+					     hdev->tm_info.pg_dwrr[i]);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int hclge_vport_q_to_qs_map(struct hclge_dev *hdev,
+				   struct hclge_vport *vport)
+{
+	struct hnae3_knic_private_info *kinfo = &vport->nic.kinfo;
+	struct hnae3_queue **tqp = kinfo->tqp;
+	struct hnae3_tc_info *v_tc_info;
+	u32 i, j;
+	int ret;
+
+	for (i = 0; i < kinfo->num_tc; i++) {
+		v_tc_info = &kinfo->tc_info[i];
+		for (j = 0; j < v_tc_info->tqp_count; j++) {
+			struct hnae3_queue *q = tqp[v_tc_info->tqp_offset + j];
+
+			ret = hclge_tm_q_to_qs_map_cfg(hdev,
+						       hclge_get_queue_id(q),
+						       vport->qs_offset + i);
+			if (ret)
+				return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int hclge_tm_pri_q_qs_cfg(struct hclge_dev *hdev)
+{
+	struct hclge_vport *vport = hdev->vport;
+	int ret;
+	u32 i;
+
+	if (hdev->tx_sch_mode == HCLGE_FLAG_TC_BASE_SCH_MODE) {
+		/* Cfg qs -> pri mapping, one by one mapping */
+		for (i = 0; i < hdev->tm_info.num_tc; i++) {
+			ret = hclge_tm_qs_to_pri_map_cfg(hdev, i, i);
+			if (ret)
+				return ret;
+		}
+	} else if (hdev->tx_sch_mode == HCLGE_FLAG_VNET_BASE_SCH_MODE) {
+		int k;
+		/* Cfg qs -> pri mapping,  qs = tc, pri = vf, 8 qs -> 1 pri */
+		for (k = 0; k < hdev->num_alloc_vport; k++)
+			for (i = 0; i < HNAE3_MAX_TC; i++) {
+				ret = hclge_tm_qs_to_pri_map_cfg(
+					hdev, vport[k].qs_offset + i, k);
+				if (ret)
+					return ret;
+			}
+	} else {
+		return -EINVAL;
+	}
+
+	/* Cfg q -> qs mapping */
+	for (i = 0; i < hdev->num_alloc_vport; i++) {
+		ret = hclge_vport_q_to_qs_map(hdev, vport);
+		if (ret)
+			return ret;
+
+		vport++;
+	}
+
+	return 0;
+}
+
+static int hclge_tm_pri_tc_base_shaper_cfg(struct hclge_dev *hdev)
+{
+	u8 ir_u, ir_b, ir_s;
+	int ret;
+	u32 i;
+
+	for (i = 0; i < hdev->tm_info.num_tc; i++) {
+		ret = hclge_shaper_para_calc(
+					hdev->tm_info.tc_info[i].bw_limit,
+					HCLGE_SHAPER_LVL_PRI,
+					&ir_b, &ir_u, &ir_s);
+		if (ret)
+			return ret;
+
+		ret = hclge_tm_pri_shapping_cfg(
+			hdev, HCLGE_TM_SHAP_C_BUCKET, i,
+			0, 0, 0, HCLGE_SHAPER_BS_U_DEF,
+			HCLGE_SHAPER_BS_S_DEF);
+		if (ret)
+			return ret;
+
+		ret = hclge_tm_pri_shapping_cfg(
+			hdev, HCLGE_TM_SHAP_P_BUCKET, i,
+			ir_b, ir_u, ir_s, HCLGE_SHAPER_BS_U_DEF,
+			HCLGE_SHAPER_BS_S_DEF);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int hclge_tm_pri_vnet_base_shaper_pri_cfg(struct hclge_vport *vport)
+{
+	struct hclge_dev *hdev = vport->back;
+	u8 ir_u, ir_b, ir_s;
+	int ret;
+
+	ret = hclge_shaper_para_calc(vport->bw_limit, HCLGE_SHAPER_LVL_VF,
+				     &ir_b, &ir_u, &ir_s);
+	if (ret)
+		return ret;
+
+	ret = hclge_tm_pri_shapping_cfg(hdev, HCLGE_TM_SHAP_C_BUCKET,
+					vport->vport_id,
+					0, 0, 0, HCLGE_SHAPER_BS_U_DEF,
+					HCLGE_SHAPER_BS_S_DEF);
+	if (ret)
+		return ret;
+
+	ret = hclge_tm_pri_shapping_cfg(hdev, HCLGE_TM_SHAP_P_BUCKET,
+					vport->vport_id,
+					ir_b, ir_u, ir_s,
+					HCLGE_SHAPER_BS_U_DEF,
+					HCLGE_SHAPER_BS_S_DEF);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int hclge_tm_pri_vnet_base_shaper_qs_cfg(struct hclge_vport *vport)
+{
+	struct hnae3_knic_private_info *kinfo = &vport->nic.kinfo;
+	struct hclge_dev *hdev = vport->back;
+	struct hnae3_tc_info *v_tc_info;
+	u8 ir_u, ir_b, ir_s;
+	u32 i;
+	int ret;
+
+	for (i = 0; i < kinfo->num_tc; i++) {
+		v_tc_info = &kinfo->tc_info[i];
+		ret = hclge_shaper_para_calc(
+					hdev->tm_info.tc_info[i].bw_limit,
+					HCLGE_SHAPER_LVL_QSET,
+					&ir_b, &ir_u, &ir_s);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int hclge_tm_pri_vnet_base_shaper_cfg(struct hclge_dev *hdev)
+{
+	struct hclge_vport *vport = hdev->vport;
+	int ret;
+	u32 i;
+
+	/* Need config vport shaper */
+	for (i = 0; i < hdev->num_alloc_vport; i++) {
+		ret = hclge_tm_pri_vnet_base_shaper_pri_cfg(vport);
+		if (ret)
+			return ret;
+
+		ret = hclge_tm_pri_vnet_base_shaper_qs_cfg(vport);
+		if (ret)
+			return ret;
+
+		vport++;
+	}
+
+	return 0;
+}
+
+static int hclge_tm_pri_shaper_cfg(struct hclge_dev *hdev)
+{
+	int ret;
+
+	if (hdev->tx_sch_mode == HCLGE_FLAG_TC_BASE_SCH_MODE) {
+		ret = hclge_tm_pri_tc_base_shaper_cfg(hdev);
+		if (ret)
+			return ret;
+	} else {
+		ret = hclge_tm_pri_vnet_base_shaper_cfg(hdev);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int hclge_tm_pri_tc_base_dwrr_cfg(struct hclge_dev *hdev)
+{
+	struct hclge_pg_info *pg_info;
+	u8 dwrr;
+	int ret;
+	u32 i;
+
+	for (i = 0; i < hdev->tm_info.num_tc; i++) {
+		pg_info =
+			&hdev->tm_info.pg_info[hdev->tm_info.tc_info[i].pgid];
+		dwrr = pg_info->tc_dwrr[i];
+
+		ret = hclge_tm_pri_weight_cfg(hdev, i, dwrr);
+		if (ret)
+			return ret;
+
+		ret = hclge_tm_qs_weight_cfg(hdev, i, dwrr);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int hclge_tm_pri_vnet_base_dwrr_pri_cfg(struct hclge_vport *vport)
+{
+	struct hnae3_knic_private_info *kinfo = &vport->nic.kinfo;
+	struct hclge_dev *hdev = vport->back;
+	int ret;
+	u8 i;
+
+	/* Vf dwrr */
+	ret = hclge_tm_pri_weight_cfg(hdev, vport->vport_id, vport->dwrr);
+	if (ret)
+		return ret;
+
+	/* Qset dwrr */
+	for (i = 0; i < kinfo->num_tc; i++) {
+		ret = hclge_tm_qs_weight_cfg(
+			hdev, vport->qs_offset + i,
+			hdev->tm_info.pg_info[0].tc_dwrr[i]);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int hclge_tm_pri_vnet_base_dwrr_cfg(struct hclge_dev *hdev)
+{
+	struct hclge_vport *vport = hdev->vport;
+	int ret;
+	u32 i;
+
+	for (i = 0; i < hdev->num_alloc_vport; i++) {
+		ret = hclge_tm_pri_vnet_base_dwrr_pri_cfg(vport);
+		if (ret)
+			return ret;
+
+		vport++;
+	}
+
+	return 0;
+}
+
+static int hclge_tm_pri_dwrr_cfg(struct hclge_dev *hdev)
+{
+	int ret;
+
+	if (hdev->tx_sch_mode == HCLGE_FLAG_TC_BASE_SCH_MODE) {
+		ret = hclge_tm_pri_tc_base_dwrr_cfg(hdev);
+		if (ret)
+			return ret;
+	} else {
+		ret = hclge_tm_pri_vnet_base_dwrr_cfg(hdev);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int hclge_tm_map_cfg(struct hclge_dev *hdev)
+{
+	int ret;
+
+	ret = hclge_tm_pg_to_pri_map(hdev);
+	if (ret)
+		return ret;
+
+	return hclge_tm_pri_q_qs_cfg(hdev);
+}
+
+static int hclge_tm_shaper_cfg(struct hclge_dev *hdev)
+{
+	int ret;
+
+	ret = hclge_tm_pg_shaper_cfg(hdev);
+	if (ret)
+		return ret;
+
+	return hclge_tm_pri_shaper_cfg(hdev);
+}
+
+int hclge_tm_dwrr_cfg(struct hclge_dev *hdev)
+{
+	int ret;
+
+	ret = hclge_tm_pg_dwrr_cfg(hdev);
+	if (ret)
+		return ret;
+
+	return hclge_tm_pri_dwrr_cfg(hdev);
+}
+
+static int hclge_tm_lvl2_schd_mode_cfg(struct hclge_dev *hdev)
+{
+	int ret;
+	u8 i;
+
+	/* Only being config on TC-Based scheduler mode */
+	if (hdev->tx_sch_mode == HCLGE_FLAG_VNET_BASE_SCH_MODE)
+		return 0;
+
+	for (i = 0; i < hdev->tm_info.num_pg; i++) {
+		ret = hclge_tm_pg_schd_mode_cfg(hdev, i);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int hclge_tm_schd_mode_vnet_base_cfg(struct hclge_vport *vport)
+{
+	struct hnae3_knic_private_info *kinfo = &vport->nic.kinfo;
+	struct hclge_dev *hdev = vport->back;
+	int ret;
+	u8 i;
+
+	ret = hclge_tm_pri_schd_mode_cfg(hdev, vport->vport_id);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < kinfo->num_tc; i++) {
+		ret = hclge_tm_qs_schd_mode_cfg(hdev, vport->qs_offset + i);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int hclge_tm_lvl34_schd_mode_cfg(struct hclge_dev *hdev)
+{
+	struct hclge_vport *vport = hdev->vport;
+	int ret;
+	u8 i;
+
+	if (hdev->tx_sch_mode == HCLGE_FLAG_TC_BASE_SCH_MODE) {
+		for (i = 0; i < hdev->tm_info.num_tc; i++) {
+			ret = hclge_tm_pri_schd_mode_cfg(hdev, i);
+			if (ret)
+				return ret;
+
+			ret = hclge_tm_qs_schd_mode_cfg(hdev, i);
+			if (ret)
+				return ret;
+		}
+	} else {
+		for (i = 0; i < hdev->num_alloc_vport; i++) {
+			ret = hclge_tm_schd_mode_vnet_base_cfg(vport);
+			if (ret)
+				return ret;
+
+			vport++;
+		}
+	}
+
+	return 0;
+}
+
+static int hclge_tm_schd_mode_hw(struct hclge_dev *hdev)
+{
+	int ret;
+
+	ret = hclge_tm_lvl2_schd_mode_cfg(hdev);
+	if (ret)
+		return ret;
+
+	return hclge_tm_lvl34_schd_mode_cfg(hdev);
+}
+
+static int hclge_tm_schd_setup_hw(struct hclge_dev *hdev)
+{
+	int ret;
+
+	/* Cfg tm mapping  */
+	ret = hclge_tm_map_cfg(hdev);
+	if (ret)
+		return ret;
+
+	/* Cfg tm shaper */
+	ret = hclge_tm_shaper_cfg(hdev);
+	if (ret)
+		return ret;
+
+	/* Cfg dwrr */
+	ret = hclge_tm_dwrr_cfg(hdev);
+	if (ret)
+		return ret;
+
+	/* Cfg schd mode for each level schd */
+	return hclge_tm_schd_mode_hw(hdev);
+}
+
+int hclge_pause_setup_hw(struct hclge_dev *hdev)
+{
+	bool en = hdev->tm_info.fc_mode != HCLGE_FC_PFC;
+	int ret;
+	u8 i;
+
+	ret = hclge_mac_pause_en_cfg(hdev, en, en);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < hdev->tm_info.num_tc; i++) {
+		ret = hclge_tm_qs_bp_cfg(hdev, i);
+		if (ret)
+			return ret;
+	}
+
+	return hclge_up_to_tc_map(hdev);
+}
+
+int hclge_tm_init_hw(struct hclge_dev *hdev)
+{
+	int ret;
+
+	if ((hdev->tx_sch_mode != HCLGE_FLAG_TC_BASE_SCH_MODE) &&
+	    (hdev->tx_sch_mode != HCLGE_FLAG_VNET_BASE_SCH_MODE))
+		return -ENOTSUPP;
+
+	ret = hclge_tm_schd_setup_hw(hdev);
+	if (ret)
+		return ret;
+
+	ret = hclge_pause_setup_hw(hdev);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+int hclge_tm_schd_init(struct hclge_dev *hdev)
+{
+	int ret = hclge_tm_schd_info_init(hdev);
+
+	if (ret)
+		return ret;
+
+	return hclge_tm_init_hw(hdev);
+}
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h
new file mode 100644
index 000000000000..7e67337dfaf2
--- /dev/null
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h
@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) 2016~2017 Hisilicon Limited.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef __HCLGE_TM_H
+#define __HCLGE_TM_H
+
+#include <linux/types.h>
+
+/* MAC Pause */
+#define HCLGE_TX_MAC_PAUSE_EN_MSK	BIT(0)
+#define HCLGE_RX_MAC_PAUSE_EN_MSK	BIT(1)
+
+#define HCLGE_TM_PORT_BASE_MODE_MSK	BIT(0)
+
+/* SP or DWRR */
+#define HCLGE_TM_TX_SCHD_DWRR_MSK	BIT(0)
+#define HCLGE_TM_TX_SCHD_SP_MSK		(0xFE)
+
+struct hclge_pg_to_pri_link_cmd {
+	u8 pg_id;
+	u8 rsvd1[3];
+	u8 pri_bit_map;
+};
+
+struct hclge_qs_to_pri_link_cmd {
+	__le16 qs_id;
+	__le16 rsvd;
+	u8 priority;
+#define HCLGE_TM_QS_PRI_LINK_VLD_MSK	BIT(0)
+	u8 link_vld;
+};
+
+struct hclge_nq_to_qs_link_cmd {
+	__le16 nq_id;
+	__le16 rsvd;
+#define HCLGE_TM_Q_QS_LINK_VLD_MSK	BIT(10)
+	__le16 qset_id;
+};
+
+struct hclge_pg_weight_cmd {
+	u8 pg_id;
+	u8 dwrr;
+};
+
+struct hclge_priority_weight_cmd {
+	u8 pri_id;
+	u8 dwrr;
+};
+
+struct hclge_qs_weight_cmd {
+	__le16 qs_id;
+	u8 dwrr;
+};
+
+#define HCLGE_TM_SHAP_IR_B_MSK  GENMASK(7, 0)
+#define HCLGE_TM_SHAP_IR_B_LSH	0
+#define HCLGE_TM_SHAP_IR_U_MSK  GENMASK(11, 8)
+#define HCLGE_TM_SHAP_IR_U_LSH	8
+#define HCLGE_TM_SHAP_IR_S_MSK  GENMASK(15, 12)
+#define HCLGE_TM_SHAP_IR_S_LSH	12
+#define HCLGE_TM_SHAP_BS_B_MSK  GENMASK(20, 16)
+#define HCLGE_TM_SHAP_BS_B_LSH	16
+#define HCLGE_TM_SHAP_BS_S_MSK  GENMASK(25, 21)
+#define HCLGE_TM_SHAP_BS_S_LSH	21
+
+enum hclge_shap_bucket {
+	HCLGE_TM_SHAP_C_BUCKET = 0,
+	HCLGE_TM_SHAP_P_BUCKET,
+};
+
+struct hclge_pri_shapping_cmd {
+	u8 pri_id;
+	u8 rsvd[3];
+	__le32 pri_shapping_para;
+};
+
+struct hclge_pg_shapping_cmd {
+	u8 pg_id;
+	u8 rsvd[3];
+	__le32 pg_shapping_para;
+};
+
+struct hclge_bp_to_qs_map_cmd {
+	u8 tc_id;
+	u8 rsvd[2];
+	u8 qs_group_id;
+	__le32 qs_bit_map;
+	u32 rsvd1;
+};
+
+#define hclge_tm_set_feild(dest, string, val) \
+			hnae_set_field((dest), (HCLGE_TM_SHAP_##string##_MSK), \
+				       (HCLGE_TM_SHAP_##string##_LSH), val)
+#define hclge_tm_get_feild(src, string) \
+			hnae_get_field((src), (HCLGE_TM_SHAP_##string##_MSK), \
+				       (HCLGE_TM_SHAP_##string##_LSH))
+
+int hclge_tm_schd_init(struct hclge_dev *hdev);
+int hclge_pause_setup_hw(struct hclge_dev *hdev);
+#endif
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hns3_enet.c
new file mode 100644
index 000000000000..1c3e29447891
--- /dev/null
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hns3_enet.c
@@ -0,0 +1,2891 @@
+/*
+ * Copyright (c) 2016~2017 Hisilicon Limited.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/dma-mapping.h>
+#include <linux/etherdevice.h>
+#include <linux/interrupt.h>
+#include <linux/if_vlan.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/skbuff.h>
+#include <linux/sctp.h>
+#include <linux/vermagic.h>
+#include <net/gre.h>
+#include <net/vxlan.h>
+
+#include "hnae3.h"
+#include "hns3_enet.h"
+
+const char hns3_driver_name[] = "hns3";
+const char hns3_driver_version[] = VERMAGIC_STRING;
+static const char hns3_driver_string[] =
+			"Hisilicon Ethernet Network Driver for Hip08 Family";
+static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation.";
+static struct hnae3_client client;
+
+/* hns3_pci_tbl - PCI Device ID Table
+ *
+ * Last entry must be all 0s
+ *
+ * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
+ *   Class, Class Mask, private data (not used) }
+ */
+static const struct pci_device_id hns3_pci_tbl[] = {
+	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
+	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
+	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0},
+	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0},
+	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0},
+	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0},
+	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0},
+	/* required last entry */
+	{0, }
+};
+MODULE_DEVICE_TABLE(pci, hns3_pci_tbl);
+
+static irqreturn_t hns3_irq_handle(int irq, void *dev)
+{
+	struct hns3_enet_tqp_vector *tqp_vector = dev;
+
+	napi_schedule(&tqp_vector->napi);
+
+	return IRQ_HANDLED;
+}
+
+static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv)
+{
+	struct hns3_enet_tqp_vector *tqp_vectors;
+	unsigned int i;
+
+	for (i = 0; i < priv->vector_num; i++) {
+		tqp_vectors = &priv->tqp_vector[i];
+
+		if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED)
+			continue;
+
+		/* release the irq resource */
+		free_irq(tqp_vectors->vector_irq, tqp_vectors);
+		tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED;
+	}
+}
+
+static int hns3_nic_init_irq(struct hns3_nic_priv *priv)
+{
+	struct hns3_enet_tqp_vector *tqp_vectors;
+	int txrx_int_idx = 0;
+	int rx_int_idx = 0;
+	int tx_int_idx = 0;
+	unsigned int i;
+	int ret;
+
+	for (i = 0; i < priv->vector_num; i++) {
+		tqp_vectors = &priv->tqp_vector[i];
+
+		if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED)
+			continue;
+
+		if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) {
+			snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
+				 "%s-%s-%d", priv->netdev->name, "TxRx",
+				 txrx_int_idx++);
+			txrx_int_idx++;
+		} else if (tqp_vectors->rx_group.ring) {
+			snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
+				 "%s-%s-%d", priv->netdev->name, "Rx",
+				 rx_int_idx++);
+		} else if (tqp_vectors->tx_group.ring) {
+			snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
+				 "%s-%s-%d", priv->netdev->name, "Tx",
+				 tx_int_idx++);
+		} else {
+			/* Skip this unused q_vector */
+			continue;
+		}
+
+		tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0';
+
+		ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0,
+				  tqp_vectors->name,
+				       tqp_vectors);
+		if (ret) {
+			netdev_err(priv->netdev, "request irq(%d) fail\n",
+				   tqp_vectors->vector_irq);
+			return ret;
+		}
+
+		tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED;
+	}
+
+	return 0;
+}
+
+static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,
+				 u32 mask_en)
+{
+	writel(mask_en, tqp_vector->mask_addr);
+}
+
+static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)
+{
+	napi_enable(&tqp_vector->napi);
+
+	/* enable vector */
+	hns3_mask_vector_irq(tqp_vector, 1);
+}
+
+static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)
+{
+	/* disable vector */
+	hns3_mask_vector_irq(tqp_vector, 0);
+
+	disable_irq(tqp_vector->vector_irq);
+	napi_disable(&tqp_vector->napi);
+}
+
+static void hns3_set_vector_coalesc_gl(struct hns3_enet_tqp_vector *tqp_vector,
+				       u32 gl_value)
+{
+	/* this defines the configuration for GL (Interrupt Gap Limiter)
+	 * GL defines inter interrupt gap.
+	 * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
+	 */
+	writel(gl_value, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);
+	writel(gl_value, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);
+	writel(gl_value, tqp_vector->mask_addr + HNS3_VECTOR_GL2_OFFSET);
+}
+
+static void hns3_set_vector_coalesc_rl(struct hns3_enet_tqp_vector *tqp_vector,
+				       u32 rl_value)
+{
+	/* this defines the configuration for RL (Interrupt Rate Limiter).
+	 * Rl defines rate of interrupts i.e. number of interrupts-per-second
+	 * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
+	 */
+	writel(rl_value, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);
+}
+
+static void hns3_vector_gl_rl_init(struct hns3_enet_tqp_vector *tqp_vector)
+{
+	/* initialize the configuration for interrupt coalescing.
+	 * 1. GL (Interrupt Gap Limiter)
+	 * 2. RL (Interrupt Rate Limiter)
+	 */
+
+	/* Default :enable interrupt coalesce */
+	tqp_vector->rx_group.int_gl = HNS3_INT_GL_50K;
+	tqp_vector->tx_group.int_gl = HNS3_INT_GL_50K;
+	hns3_set_vector_coalesc_gl(tqp_vector, HNS3_INT_GL_50K);
+	/* for now we are disabling Interrupt RL - we
+	 * will re-enable later
+	 */
+	hns3_set_vector_coalesc_rl(tqp_vector, 0);
+	tqp_vector->rx_group.flow_level = HNS3_FLOW_LOW;
+	tqp_vector->tx_group.flow_level = HNS3_FLOW_LOW;
+}
+
+static int hns3_nic_net_up(struct net_device *netdev)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+	int i, j;
+	int ret;
+
+	/* get irq resource for all vectors */
+	ret = hns3_nic_init_irq(priv);
+	if (ret) {
+		netdev_err(netdev, "hns init irq failed! ret=%d\n", ret);
+		return ret;
+	}
+
+	/* enable the vectors */
+	for (i = 0; i < priv->vector_num; i++)
+		hns3_vector_enable(&priv->tqp_vector[i]);
+
+	/* start the ae_dev */
+	ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0;
+	if (ret)
+		goto out_start_err;
+
+	return 0;
+
+out_start_err:
+	for (j = i - 1; j >= 0; j--)
+		hns3_vector_disable(&priv->tqp_vector[j]);
+
+	hns3_nic_uninit_irq(priv);
+
+	return ret;
+}
+
+static int hns3_nic_net_open(struct net_device *netdev)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+	int ret;
+
+	netif_carrier_off(netdev);
+
+	ret = netif_set_real_num_tx_queues(netdev, h->kinfo.num_tqps);
+	if (ret) {
+		netdev_err(netdev,
+			   "netif_set_real_num_tx_queues fail, ret=%d!\n",
+			   ret);
+		return ret;
+	}
+
+	ret = netif_set_real_num_rx_queues(netdev, h->kinfo.num_tqps);
+	if (ret) {
+		netdev_err(netdev,
+			   "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
+		return ret;
+	}
+
+	ret = hns3_nic_net_up(netdev);
+	if (ret) {
+		netdev_err(netdev,
+			   "hns net up fail, ret=%d!\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void hns3_nic_net_down(struct net_device *netdev)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	const struct hnae3_ae_ops *ops;
+	int i;
+
+	/* stop ae_dev */
+	ops = priv->ae_handle->ae_algo->ops;
+	if (ops->stop)
+		ops->stop(priv->ae_handle);
+
+	/* disable vectors */
+	for (i = 0; i < priv->vector_num; i++)
+		hns3_vector_disable(&priv->tqp_vector[i]);
+
+	/* free irq resources */
+	hns3_nic_uninit_irq(priv);
+}
+
+static int hns3_nic_net_stop(struct net_device *netdev)
+{
+	netif_tx_stop_all_queues(netdev);
+	netif_carrier_off(netdev);
+
+	hns3_nic_net_down(netdev);
+
+	return 0;
+}
+
+void hns3_set_multicast_list(struct net_device *netdev)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+	struct netdev_hw_addr *ha = NULL;
+
+	if (h->ae_algo->ops->set_mc_addr) {
+		netdev_for_each_mc_addr(ha, netdev)
+			if (h->ae_algo->ops->set_mc_addr(h, ha->addr))
+				netdev_err(netdev, "set multicast fail\n");
+	}
+}
+
+static int hns3_nic_uc_sync(struct net_device *netdev,
+			    const unsigned char *addr)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+
+	if (h->ae_algo->ops->add_uc_addr)
+		return h->ae_algo->ops->add_uc_addr(h, addr);
+
+	return 0;
+}
+
+static int hns3_nic_uc_unsync(struct net_device *netdev,
+			      const unsigned char *addr)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+
+	if (h->ae_algo->ops->rm_uc_addr)
+		return h->ae_algo->ops->rm_uc_addr(h, addr);
+
+	return 0;
+}
+
+static int hns3_nic_mc_sync(struct net_device *netdev,
+			    const unsigned char *addr)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+
+	if (h->ae_algo->ops->add_mc_addr)
+		return h->ae_algo->ops->add_mc_addr(h, addr);
+
+	return 0;
+}
+
+static int hns3_nic_mc_unsync(struct net_device *netdev,
+			      const unsigned char *addr)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+
+	if (h->ae_algo->ops->rm_mc_addr)
+		return h->ae_algo->ops->rm_mc_addr(h, addr);
+
+	return 0;
+}
+
+void hns3_nic_set_rx_mode(struct net_device *netdev)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+
+	if (h->ae_algo->ops->set_promisc_mode) {
+		if (netdev->flags & IFF_PROMISC)
+			h->ae_algo->ops->set_promisc_mode(h, 1);
+		else
+			h->ae_algo->ops->set_promisc_mode(h, 0);
+	}
+	if (__dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync))
+		netdev_err(netdev, "sync uc address fail\n");
+	if (netdev->flags & IFF_MULTICAST)
+		if (__dev_mc_sync(netdev, hns3_nic_mc_sync, hns3_nic_mc_unsync))
+			netdev_err(netdev, "sync mc address fail\n");
+}
+
+static int hns3_set_tso(struct sk_buff *skb, u32 *paylen,
+			u16 *mss, u32 *type_cs_vlan_tso)
+{
+	u32 l4_offset, hdr_len;
+	union l3_hdr_info l3;
+	union l4_hdr_info l4;
+	u32 l4_paylen;
+	int ret;
+
+	if (!skb_is_gso(skb))
+		return 0;
+
+	ret = skb_cow_head(skb, 0);
+	if (ret)
+		return ret;
+
+	l3.hdr = skb_network_header(skb);
+	l4.hdr = skb_transport_header(skb);
+
+	/* Software should clear the IPv4's checksum field when tso is
+	 * needed.
+	 */
+	if (l3.v4->version == 4)
+		l3.v4->check = 0;
+
+	/* tunnel packet.*/
+	if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
+					 SKB_GSO_GRE_CSUM |
+					 SKB_GSO_UDP_TUNNEL |
+					 SKB_GSO_UDP_TUNNEL_CSUM)) {
+		if ((!(skb_shinfo(skb)->gso_type &
+		    SKB_GSO_PARTIAL)) &&
+		    (skb_shinfo(skb)->gso_type &
+		    SKB_GSO_UDP_TUNNEL_CSUM)) {
+			/* Software should clear the udp's checksum
+			 * field when tso is needed.
+			 */
+			l4.udp->check = 0;
+		}
+		/* reset l3&l4 pointers from outer to inner headers */
+		l3.hdr = skb_inner_network_header(skb);
+		l4.hdr = skb_inner_transport_header(skb);
+
+		/* Software should clear the IPv4's checksum field when
+		 * tso is needed.
+		 */
+		if (l3.v4->version == 4)
+			l3.v4->check = 0;
+	}
+
+	/* normal or tunnel packet*/
+	l4_offset = l4.hdr - skb->data;
+	hdr_len = (l4.tcp->doff * 4) + l4_offset;
+
+	/* remove payload length from inner pseudo checksum when tso*/
+	l4_paylen = skb->len - l4_offset;
+	csum_replace_by_diff(&l4.tcp->check,
+			     (__force __wsum)htonl(l4_paylen));
+
+	/* find the txbd field values */
+	*paylen = skb->len - hdr_len;
+	hnae_set_bit(*type_cs_vlan_tso,
+		     HNS3_TXD_TSO_B, 1);
+
+	/* get MSS for TSO */
+	*mss = skb_shinfo(skb)->gso_size;
+
+	return 0;
+}
+
+static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,
+				u8 *il4_proto)
+{
+	union {
+		struct iphdr *v4;
+		struct ipv6hdr *v6;
+		unsigned char *hdr;
+	} l3;
+	unsigned char *l4_hdr;
+	unsigned char *exthdr;
+	u8 l4_proto_tmp;
+	__be16 frag_off;
+
+	/* find outer header point */
+	l3.hdr = skb_network_header(skb);
+	l4_hdr = skb_inner_transport_header(skb);
+
+	if (skb->protocol == htons(ETH_P_IPV6)) {
+		exthdr = l3.hdr + sizeof(*l3.v6);
+		l4_proto_tmp = l3.v6->nexthdr;
+		if (l4_hdr != exthdr)
+			ipv6_skip_exthdr(skb, exthdr - skb->data,
+					 &l4_proto_tmp, &frag_off);
+	} else if (skb->protocol == htons(ETH_P_IP)) {
+		l4_proto_tmp = l3.v4->protocol;
+	} else {
+		return -EINVAL;
+	}
+
+	*ol4_proto = l4_proto_tmp;
+
+	/* tunnel packet */
+	if (!skb->encapsulation) {
+		*il4_proto = 0;
+		return 0;
+	}
+
+	/* find inner header point */
+	l3.hdr = skb_inner_network_header(skb);
+	l4_hdr = skb_inner_transport_header(skb);
+
+	if (l3.v6->version == 6) {
+		exthdr = l3.hdr + sizeof(*l3.v6);
+		l4_proto_tmp = l3.v6->nexthdr;
+		if (l4_hdr != exthdr)
+			ipv6_skip_exthdr(skb, exthdr - skb->data,
+					 &l4_proto_tmp, &frag_off);
+	} else if (l3.v4->version == 4) {
+		l4_proto_tmp = l3.v4->protocol;
+	}
+
+	*il4_proto = l4_proto_tmp;
+
+	return 0;
+}
+
+static void hns3_set_l2l3l4_len(struct sk_buff *skb, u8 ol4_proto,
+				u8 il4_proto, u32 *type_cs_vlan_tso,
+				u32 *ol_type_vlan_len_msec)
+{
+	union {
+		struct iphdr *v4;
+		struct ipv6hdr *v6;
+		unsigned char *hdr;
+	} l3;
+	union {
+		struct tcphdr *tcp;
+		struct udphdr *udp;
+		struct gre_base_hdr *gre;
+		unsigned char *hdr;
+	} l4;
+	unsigned char *l2_hdr;
+	u8 l4_proto = ol4_proto;
+	u32 ol2_len;
+	u32 ol3_len;
+	u32 ol4_len;
+	u32 l2_len;
+	u32 l3_len;
+
+	l3.hdr = skb_network_header(skb);
+	l4.hdr = skb_transport_header(skb);
+
+	/* compute L2 header size for normal packet, defined in 2 Bytes */
+	l2_len = l3.hdr - skb->data;
+	hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
+		       HNS3_TXD_L2LEN_S, l2_len >> 1);
+
+	/* tunnel packet*/
+	if (skb->encapsulation) {
+		/* compute OL2 header size, defined in 2 Bytes */
+		ol2_len = l2_len;
+		hnae_set_field(*ol_type_vlan_len_msec,
+			       HNS3_TXD_L2LEN_M,
+			       HNS3_TXD_L2LEN_S, ol2_len >> 1);
+
+		/* compute OL3 header size, defined in 4 Bytes */
+		ol3_len = l4.hdr - l3.hdr;
+		hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_M,
+			       HNS3_TXD_L3LEN_S, ol3_len >> 2);
+
+		/* MAC in UDP, MAC in GRE (0x6558)*/
+		if ((ol4_proto == IPPROTO_UDP) || (ol4_proto == IPPROTO_GRE)) {
+			/* switch MAC header ptr from outer to inner header.*/
+			l2_hdr = skb_inner_mac_header(skb);
+
+			/* compute OL4 header size, defined in 4 Bytes. */
+			ol4_len = l2_hdr - l4.hdr;
+			hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L4LEN_M,
+				       HNS3_TXD_L4LEN_S, ol4_len >> 2);
+
+			/* switch IP header ptr from outer to inner header */
+			l3.hdr = skb_inner_network_header(skb);
+
+			/* compute inner l2 header size, defined in 2 Bytes. */
+			l2_len = l3.hdr - l2_hdr;
+			hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
+				       HNS3_TXD_L2LEN_S, l2_len >> 1);
+		} else {
+			/* skb packet types not supported by hardware,
+			 * txbd len fild doesn't be filled.
+			 */
+			return;
+		}
+
+		/* switch L4 header pointer from outer to inner */
+		l4.hdr = skb_inner_transport_header(skb);
+
+		l4_proto = il4_proto;
+	}
+
+	/* compute inner(/normal) L3 header size, defined in 4 Bytes */
+	l3_len = l4.hdr - l3.hdr;
+	hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_M,
+		       HNS3_TXD_L3LEN_S, l3_len >> 2);
+
+	/* compute inner(/normal) L4 header size, defined in 4 Bytes */
+	switch (l4_proto) {
+	case IPPROTO_TCP:
+		hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
+			       HNS3_TXD_L4LEN_S, l4.tcp->doff);
+		break;
+	case IPPROTO_SCTP:
+		hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
+			       HNS3_TXD_L4LEN_S, (sizeof(struct sctphdr) >> 2));
+		break;
+	case IPPROTO_UDP:
+		hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
+			       HNS3_TXD_L4LEN_S, (sizeof(struct udphdr) >> 2));
+		break;
+	default:
+		/* skb packet types not supported by hardware,
+		 * txbd len fild doesn't be filled.
+		 */
+		return;
+	}
+}
+
+static int hns3_set_l3l4_type_csum(struct sk_buff *skb, u8 ol4_proto,
+				   u8 il4_proto, u32 *type_cs_vlan_tso,
+				   u32 *ol_type_vlan_len_msec)
+{
+	union {
+		struct iphdr *v4;
+		struct ipv6hdr *v6;
+		unsigned char *hdr;
+	} l3;
+	u32 l4_proto = ol4_proto;
+
+	l3.hdr = skb_network_header(skb);
+
+	/* define OL3 type and tunnel type(OL4).*/
+	if (skb->encapsulation) {
+		/* define outer network header type.*/
+		if (skb->protocol == htons(ETH_P_IP)) {
+			if (skb_is_gso(skb))
+				hnae_set_field(*ol_type_vlan_len_msec,
+					       HNS3_TXD_OL3T_M, HNS3_TXD_OL3T_S,
+					       HNS3_OL3T_IPV4_CSUM);
+			else
+				hnae_set_field(*ol_type_vlan_len_msec,
+					       HNS3_TXD_OL3T_M, HNS3_TXD_OL3T_S,
+					       HNS3_OL3T_IPV4_NO_CSUM);
+
+		} else if (skb->protocol == htons(ETH_P_IPV6)) {
+			hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_M,
+				       HNS3_TXD_OL3T_S, HNS3_OL3T_IPV6);
+		}
+
+		/* define tunnel type(OL4).*/
+		switch (l4_proto) {
+		case IPPROTO_UDP:
+			hnae_set_field(*ol_type_vlan_len_msec,
+				       HNS3_TXD_TUNTYPE_M,
+				       HNS3_TXD_TUNTYPE_S,
+				       HNS3_TUN_MAC_IN_UDP);
+			break;
+		case IPPROTO_GRE:
+			hnae_set_field(*ol_type_vlan_len_msec,
+				       HNS3_TXD_TUNTYPE_M,
+				       HNS3_TXD_TUNTYPE_S,
+				       HNS3_TUN_NVGRE);
+			break;
+		default:
+			/* drop the skb tunnel packet if hardware don't support,
+			 * because hardware can't calculate csum when TSO.
+			 */
+			if (skb_is_gso(skb))
+				return -EDOM;
+
+			/* the stack computes the IP header already,
+			 * driver calculate l4 checksum when not TSO.
+			 */
+			skb_checksum_help(skb);
+			return 0;
+		}
+
+		l3.hdr = skb_inner_network_header(skb);
+		l4_proto = il4_proto;
+	}
+
+	if (l3.v4->version == 4) {
+		hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
+			       HNS3_TXD_L3T_S, HNS3_L3T_IPV4);
+
+		/* the stack computes the IP header already, the only time we
+		 * need the hardware to recompute it is in the case of TSO.
+		 */
+		if (skb_is_gso(skb))
+			hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1);
+
+		hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
+	} else if (l3.v6->version == 6) {
+		hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
+			       HNS3_TXD_L3T_S, HNS3_L3T_IPV6);
+		hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
+	}
+
+	switch (l4_proto) {
+	case IPPROTO_TCP:
+		hnae_set_field(*type_cs_vlan_tso,
+			       HNS3_TXD_L4T_M,
+			       HNS3_TXD_L4T_S,
+			       HNS3_L4T_TCP);
+		break;
+	case IPPROTO_UDP:
+		hnae_set_field(*type_cs_vlan_tso,
+			       HNS3_TXD_L4T_M,
+			       HNS3_TXD_L4T_S,
+			       HNS3_L4T_UDP);
+		break;
+	case IPPROTO_SCTP:
+		hnae_set_field(*type_cs_vlan_tso,
+			       HNS3_TXD_L4T_M,
+			       HNS3_TXD_L4T_S,
+			       HNS3_L4T_SCTP);
+		break;
+	default:
+		/* drop the skb tunnel packet if hardware don't support,
+		 * because hardware can't calculate csum when TSO.
+		 */
+		if (skb_is_gso(skb))
+			return -EDOM;
+
+		/* the stack computes the IP header already,
+		 * driver calculate l4 checksum when not TSO.
+		 */
+		skb_checksum_help(skb);
+		return 0;
+	}
+
+	return 0;
+}
+
+static void hns3_set_txbd_baseinfo(u16 *bdtp_fe_sc_vld_ra_ri, int frag_end)
+{
+	/* Config bd buffer end */
+	hnae_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_BDTYPE_M,
+		       HNS3_TXD_BDTYPE_M, 0);
+	hnae_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_FE_B, !!frag_end);
+	hnae_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_VLD_B, 1);
+	hnae_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_SC_M, HNS3_TXD_SC_S, 1);
+}
+
+static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv,
+			  int size, dma_addr_t dma, int frag_end,
+			  enum hns_desc_type type)
+{
+	struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
+	struct hns3_desc *desc = &ring->desc[ring->next_to_use];
+	u32 ol_type_vlan_len_msec = 0;
+	u16 bdtp_fe_sc_vld_ra_ri = 0;
+	u32 type_cs_vlan_tso = 0;
+	struct sk_buff *skb;
+	u32 paylen = 0;
+	u16 mss = 0;
+	__be16 protocol;
+	u8 ol4_proto;
+	u8 il4_proto;
+	int ret;
+
+	/* The txbd's baseinfo of DESC_TYPE_PAGE & DESC_TYPE_SKB */
+	desc_cb->priv = priv;
+	desc_cb->length = size;
+	desc_cb->dma = dma;
+	desc_cb->type = type;
+
+	/* now, fill the descriptor */
+	desc->addr = cpu_to_le64(dma);
+	desc->tx.send_size = cpu_to_le16((u16)size);
+	hns3_set_txbd_baseinfo(&bdtp_fe_sc_vld_ra_ri, frag_end);
+	desc->tx.bdtp_fe_sc_vld_ra_ri = cpu_to_le16(bdtp_fe_sc_vld_ra_ri);
+
+	if (type == DESC_TYPE_SKB) {
+		skb = (struct sk_buff *)priv;
+		paylen = cpu_to_le16(skb->len);
+
+		if (skb->ip_summed == CHECKSUM_PARTIAL) {
+			skb_reset_mac_len(skb);
+			protocol = skb->protocol;
+
+			/* vlan packet*/
+			if (protocol == htons(ETH_P_8021Q)) {
+				protocol = vlan_get_protocol(skb);
+				skb->protocol = protocol;
+			}
+			ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto);
+			if (ret)
+				return ret;
+			hns3_set_l2l3l4_len(skb, ol4_proto, il4_proto,
+					    &type_cs_vlan_tso,
+					    &ol_type_vlan_len_msec);
+			ret = hns3_set_l3l4_type_csum(skb, ol4_proto, il4_proto,
+						      &type_cs_vlan_tso,
+						      &ol_type_vlan_len_msec);
+			if (ret)
+				return ret;
+
+			ret = hns3_set_tso(skb, &paylen, &mss,
+					   &type_cs_vlan_tso);
+			if (ret)
+				return ret;
+		}
+
+		/* Set txbd */
+		desc->tx.ol_type_vlan_len_msec =
+			cpu_to_le32(ol_type_vlan_len_msec);
+		desc->tx.type_cs_vlan_tso_len =
+			cpu_to_le32(type_cs_vlan_tso);
+		desc->tx.paylen = cpu_to_le16(paylen);
+		desc->tx.mss = cpu_to_le16(mss);
+	}
+
+	/* move ring pointer to next.*/
+	ring_ptr_move_fw(ring, next_to_use);
+
+	return 0;
+}
+
+static int hns3_fill_desc_tso(struct hns3_enet_ring *ring, void *priv,
+			      int size, dma_addr_t dma, int frag_end,
+			      enum hns_desc_type type)
+{
+	unsigned int frag_buf_num;
+	unsigned int k;
+	int sizeoflast;
+	int ret;
+
+	frag_buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
+	sizeoflast = size % HNS3_MAX_BD_SIZE;
+	sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE;
+
+	/* When the frag size is bigger than hardware, split this frag */
+	for (k = 0; k < frag_buf_num; k++) {
+		ret = hns3_fill_desc(ring, priv,
+				     (k == frag_buf_num - 1) ?
+				sizeoflast : HNS3_MAX_BD_SIZE,
+				dma + HNS3_MAX_BD_SIZE * k,
+				frag_end && (k == frag_buf_num - 1) ? 1 : 0,
+				(type == DESC_TYPE_SKB && !k) ?
+					DESC_TYPE_SKB : DESC_TYPE_PAGE);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int hns3_nic_maybe_stop_tso(struct sk_buff **out_skb, int *bnum,
+				   struct hns3_enet_ring *ring)
+{
+	struct sk_buff *skb = *out_skb;
+	struct skb_frag_struct *frag;
+	int bdnum_for_frag;
+	int frag_num;
+	int buf_num;
+	int size;
+	int i;
+
+	size = skb_headlen(skb);
+	buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
+
+	frag_num = skb_shinfo(skb)->nr_frags;
+	for (i = 0; i < frag_num; i++) {
+		frag = &skb_shinfo(skb)->frags[i];
+		size = skb_frag_size(frag);
+		bdnum_for_frag =
+			(size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
+		if (bdnum_for_frag > HNS3_MAX_BD_PER_FRAG)
+			return -ENOMEM;
+
+		buf_num += bdnum_for_frag;
+	}
+
+	if (buf_num > ring_space(ring))
+		return -EBUSY;
+
+	*bnum = buf_num;
+	return 0;
+}
+
+static int hns3_nic_maybe_stop_tx(struct sk_buff **out_skb, int *bnum,
+				  struct hns3_enet_ring *ring)
+{
+	struct sk_buff *skb = *out_skb;
+	int buf_num;
+
+	/* No. of segments (plus a header) */
+	buf_num = skb_shinfo(skb)->nr_frags + 1;
+
+	if (buf_num > ring_space(ring))
+		return -EBUSY;
+
+	*bnum = buf_num;
+
+	return 0;
+}
+
+static void hns_nic_dma_unmap(struct hns3_enet_ring *ring, int next_to_use_orig)
+{
+	struct device *dev = ring_to_dev(ring);
+	unsigned int i;
+
+	for (i = 0; i < ring->desc_num; i++) {
+		/* check if this is where we started */
+		if (ring->next_to_use == next_to_use_orig)
+			break;
+
+		/* unmap the descriptor dma address */
+		if (ring->desc_cb[ring->next_to_use].type == DESC_TYPE_SKB)
+			dma_unmap_single(dev,
+					 ring->desc_cb[ring->next_to_use].dma,
+					ring->desc_cb[ring->next_to_use].length,
+					DMA_TO_DEVICE);
+		else
+			dma_unmap_page(dev,
+				       ring->desc_cb[ring->next_to_use].dma,
+				       ring->desc_cb[ring->next_to_use].length,
+				       DMA_TO_DEVICE);
+
+		/* rollback one */
+		ring_ptr_move_bw(ring, next_to_use);
+	}
+}
+
+static netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb,
+				     struct net_device *netdev)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hns3_nic_ring_data *ring_data =
+		&tx_ring_data(priv, skb->queue_mapping);
+	struct hns3_enet_ring *ring = ring_data->ring;
+	struct device *dev = priv->dev;
+	struct netdev_queue *dev_queue;
+	struct skb_frag_struct *frag;
+	int next_to_use_head;
+	int next_to_use_frag;
+	dma_addr_t dma;
+	int buf_num;
+	int seg_num;
+	int size;
+	int ret;
+	int i;
+
+	/* Prefetch the data used later */
+	prefetch(skb->data);
+
+	switch (priv->ops.maybe_stop_tx(&skb, &buf_num, ring)) {
+	case -EBUSY:
+		u64_stats_update_begin(&ring->syncp);
+		ring->stats.tx_busy++;
+		u64_stats_update_end(&ring->syncp);
+
+		goto out_net_tx_busy;
+	case -ENOMEM:
+		u64_stats_update_begin(&ring->syncp);
+		ring->stats.sw_err_cnt++;
+		u64_stats_update_end(&ring->syncp);
+		netdev_err(netdev, "no memory to xmit!\n");
+
+		goto out_err_tx_ok;
+	default:
+		break;
+	}
+
+	/* No. of segments (plus a header) */
+	seg_num = skb_shinfo(skb)->nr_frags + 1;
+	/* Fill the first part */
+	size = skb_headlen(skb);
+
+	next_to_use_head = ring->next_to_use;
+
+	dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
+	if (dma_mapping_error(dev, dma)) {
+		netdev_err(netdev, "TX head DMA map failed\n");
+		ring->stats.sw_err_cnt++;
+		goto out_err_tx_ok;
+	}
+
+	ret = priv->ops.fill_desc(ring, skb, size, dma, seg_num == 1 ? 1 : 0,
+			   DESC_TYPE_SKB);
+	if (ret)
+		goto head_dma_map_err;
+
+	next_to_use_frag = ring->next_to_use;
+	/* Fill the fragments */
+	for (i = 1; i < seg_num; i++) {
+		frag = &skb_shinfo(skb)->frags[i - 1];
+		size = skb_frag_size(frag);
+		dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
+		if (dma_mapping_error(dev, dma)) {
+			netdev_err(netdev, "TX frag(%d) DMA map failed\n", i);
+			ring->stats.sw_err_cnt++;
+			goto frag_dma_map_err;
+		}
+		ret = priv->ops.fill_desc(ring, skb_frag_page(frag), size, dma,
+				    seg_num - 1 == i ? 1 : 0,
+				    DESC_TYPE_PAGE);
+
+		if (ret)
+			goto frag_dma_map_err;
+	}
+
+	/* Complete translate all packets */
+	dev_queue = netdev_get_tx_queue(netdev, ring_data->queue_index);
+	netdev_tx_sent_queue(dev_queue, skb->len);
+
+	wmb(); /* Commit all data before submit */
+
+	hnae_queue_xmit(ring->tqp, buf_num);
+
+	return NETDEV_TX_OK;
+
+frag_dma_map_err:
+	hns_nic_dma_unmap(ring, next_to_use_frag);
+
+head_dma_map_err:
+	hns_nic_dma_unmap(ring, next_to_use_head);
+
+out_err_tx_ok:
+	dev_kfree_skb_any(skb);
+	return NETDEV_TX_OK;
+
+out_net_tx_busy:
+	netif_stop_subqueue(netdev, ring_data->queue_index);
+	smp_mb(); /* Commit all data before submit */
+
+	return NETDEV_TX_BUSY;
+}
+
+static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+	struct sockaddr *mac_addr = p;
+	int ret;
+
+	if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
+		return -EADDRNOTAVAIL;
+
+	ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data);
+	if (ret) {
+		netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret);
+		return ret;
+	}
+
+	ether_addr_copy(netdev->dev_addr, mac_addr->sa_data);
+
+	return 0;
+}
+
+static int hns3_nic_set_features(struct net_device *netdev,
+				 netdev_features_t features)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+
+	if (features & (NETIF_F_TSO | NETIF_F_TSO6)) {
+		priv->ops.fill_desc = hns3_fill_desc_tso;
+		priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
+	} else {
+		priv->ops.fill_desc = hns3_fill_desc;
+		priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
+	}
+
+	netdev->features = features;
+	return 0;
+}
+
+static void
+hns3_nic_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	int queue_num = priv->ae_handle->kinfo.num_tqps;
+	struct hns3_enet_ring *ring;
+	unsigned int start;
+	unsigned int idx;
+	u64 tx_bytes = 0;
+	u64 rx_bytes = 0;
+	u64 tx_pkts = 0;
+	u64 rx_pkts = 0;
+
+	for (idx = 0; idx < queue_num; idx++) {
+		/* fetch the tx stats */
+		ring = priv->ring_data[idx].ring;
+		do {
+			start = u64_stats_fetch_begin_irq(&ring->syncp);
+			tx_bytes += ring->stats.tx_bytes;
+			tx_pkts += ring->stats.tx_pkts;
+		} while (u64_stats_fetch_retry_irq(&ring->syncp, start));
+
+		/* fetch the rx stats */
+		ring = priv->ring_data[idx + queue_num].ring;
+		do {
+			start = u64_stats_fetch_begin_irq(&ring->syncp);
+			rx_bytes += ring->stats.rx_bytes;
+			rx_pkts += ring->stats.rx_pkts;
+		} while (u64_stats_fetch_retry_irq(&ring->syncp, start));
+	}
+
+	stats->tx_bytes = tx_bytes;
+	stats->tx_packets = tx_pkts;
+	stats->rx_bytes = rx_bytes;
+	stats->rx_packets = rx_pkts;
+
+	stats->rx_errors = netdev->stats.rx_errors;
+	stats->multicast = netdev->stats.multicast;
+	stats->rx_length_errors = netdev->stats.rx_length_errors;
+	stats->rx_crc_errors = netdev->stats.rx_crc_errors;
+	stats->rx_missed_errors = netdev->stats.rx_missed_errors;
+
+	stats->tx_errors = netdev->stats.tx_errors;
+	stats->rx_dropped = netdev->stats.rx_dropped;
+	stats->tx_dropped = netdev->stats.tx_dropped;
+	stats->collisions = netdev->stats.collisions;
+	stats->rx_over_errors = netdev->stats.rx_over_errors;
+	stats->rx_frame_errors = netdev->stats.rx_frame_errors;
+	stats->rx_fifo_errors = netdev->stats.rx_fifo_errors;
+	stats->tx_aborted_errors = netdev->stats.tx_aborted_errors;
+	stats->tx_carrier_errors = netdev->stats.tx_carrier_errors;
+	stats->tx_fifo_errors = netdev->stats.tx_fifo_errors;
+	stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors;
+	stats->tx_window_errors = netdev->stats.tx_window_errors;
+	stats->rx_compressed = netdev->stats.rx_compressed;
+	stats->tx_compressed = netdev->stats.tx_compressed;
+}
+
+static void hns3_add_tunnel_port(struct net_device *netdev, u16 port,
+				 enum hns3_udp_tnl_type type)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hns3_udp_tunnel *udp_tnl = &priv->udp_tnl[type];
+	struct hnae3_handle *h = priv->ae_handle;
+
+	if (udp_tnl->used && udp_tnl->dst_port == port) {
+		udp_tnl->used++;
+		return;
+	}
+
+	if (udp_tnl->used) {
+		netdev_warn(netdev,
+			    "UDP tunnel [%d], port [%d] offload\n", type, port);
+		return;
+	}
+
+	udp_tnl->dst_port = port;
+	udp_tnl->used = 1;
+	/* TBD send command to hardware to add port */
+	if (h->ae_algo->ops->add_tunnel_udp)
+		h->ae_algo->ops->add_tunnel_udp(h, port);
+}
+
+static void hns3_del_tunnel_port(struct net_device *netdev, u16 port,
+				 enum hns3_udp_tnl_type type)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hns3_udp_tunnel *udp_tnl = &priv->udp_tnl[type];
+	struct hnae3_handle *h = priv->ae_handle;
+
+	if (!udp_tnl->used || udp_tnl->dst_port != port) {
+		netdev_warn(netdev,
+			    "Invalid UDP tunnel port %d\n", port);
+		return;
+	}
+
+	udp_tnl->used--;
+	if (udp_tnl->used)
+		return;
+
+	udp_tnl->dst_port = 0;
+	/* TBD send command to hardware to del port  */
+	if (h->ae_algo->ops->del_tunnel_udp)
+		h->ae_algo->ops->del_tunnel_udp(h, port);
+}
+
+/* hns3_nic_udp_tunnel_add - Get notifiacetion about UDP tunnel ports
+ * @netdev: This physical ports's netdev
+ * @ti: Tunnel information
+ */
+static void hns3_nic_udp_tunnel_add(struct net_device *netdev,
+				    struct udp_tunnel_info *ti)
+{
+	u16 port_n = ntohs(ti->port);
+
+	switch (ti->type) {
+	case UDP_TUNNEL_TYPE_VXLAN:
+		hns3_add_tunnel_port(netdev, port_n, HNS3_UDP_TNL_VXLAN);
+		break;
+	case UDP_TUNNEL_TYPE_GENEVE:
+		hns3_add_tunnel_port(netdev, port_n, HNS3_UDP_TNL_GENEVE);
+		break;
+	default:
+		netdev_err(netdev, "unsupported tunnel type %d\n", ti->type);
+		break;
+	}
+}
+
+static void hns3_nic_udp_tunnel_del(struct net_device *netdev,
+				    struct udp_tunnel_info *ti)
+{
+	u16 port_n = ntohs(ti->port);
+
+	switch (ti->type) {
+	case UDP_TUNNEL_TYPE_VXLAN:
+		hns3_del_tunnel_port(netdev, port_n, HNS3_UDP_TNL_VXLAN);
+		break;
+	case UDP_TUNNEL_TYPE_GENEVE:
+		hns3_del_tunnel_port(netdev, port_n, HNS3_UDP_TNL_GENEVE);
+		break;
+	default:
+		break;
+	}
+}
+
+static int hns3_setup_tc(struct net_device *netdev, u8 tc)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+	struct hnae3_knic_private_info *kinfo = &h->kinfo;
+	unsigned int i;
+	int ret;
+
+	if (tc > HNAE3_MAX_TC)
+		return -EINVAL;
+
+	if (kinfo->num_tc == tc)
+		return 0;
+
+	if (!netdev)
+		return -EINVAL;
+
+	if (!tc) {
+		netdev_reset_tc(netdev);
+		return 0;
+	}
+
+	/* Set num_tc for netdev */
+	ret = netdev_set_num_tc(netdev, tc);
+	if (ret)
+		return ret;
+
+	/* Set per TC queues for the VSI */
+	for (i = 0; i < HNAE3_MAX_TC; i++) {
+		if (kinfo->tc_info[i].enable)
+			netdev_set_tc_queue(netdev,
+					    kinfo->tc_info[i].tc,
+					    kinfo->tc_info[i].tqp_count,
+					    kinfo->tc_info[i].tqp_offset);
+	}
+
+	return 0;
+}
+
+static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type,
+			     void *type_data)
+{
+	struct tc_mqprio_qopt *mqprio = type_data;
+
+	if (type != TC_SETUP_MQPRIO)
+		return -EOPNOTSUPP;
+
+	return hns3_setup_tc(dev, mqprio->num_tc);
+}
+
+static int hns3_vlan_rx_add_vid(struct net_device *netdev,
+				__be16 proto, u16 vid)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+	int ret = -EIO;
+
+	if (h->ae_algo->ops->set_vlan_filter)
+		ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false);
+
+	return ret;
+}
+
+static int hns3_vlan_rx_kill_vid(struct net_device *netdev,
+				 __be16 proto, u16 vid)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+	int ret = -EIO;
+
+	if (h->ae_algo->ops->set_vlan_filter)
+		ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true);
+
+	return ret;
+}
+
+static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
+				u8 qos, __be16 vlan_proto)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+	int ret = -EIO;
+
+	if (h->ae_algo->ops->set_vf_vlan_filter)
+		ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan,
+						   qos, vlan_proto);
+
+	return ret;
+}
+
+static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+	bool if_running = netif_running(netdev);
+	int ret;
+
+	if (!h->ae_algo->ops->set_mtu)
+		return -EOPNOTSUPP;
+
+	/* if this was called with netdev up then bring netdevice down */
+	if (if_running) {
+		(void)hns3_nic_net_stop(netdev);
+		msleep(100);
+	}
+
+	ret = h->ae_algo->ops->set_mtu(h, new_mtu);
+	if (ret) {
+		netdev_err(netdev, "failed to change MTU in hardware %d\n",
+			   ret);
+		return ret;
+	}
+
+	/* if the netdev was running earlier, bring it up again */
+	if (if_running && hns3_nic_net_open(netdev))
+		ret = -EINVAL;
+
+	return ret;
+}
+
+static const struct net_device_ops hns3_nic_netdev_ops = {
+	.ndo_open		= hns3_nic_net_open,
+	.ndo_stop		= hns3_nic_net_stop,
+	.ndo_start_xmit		= hns3_nic_net_xmit,
+	.ndo_set_mac_address	= hns3_nic_net_set_mac_address,
+	.ndo_change_mtu		= hns3_nic_change_mtu,
+	.ndo_set_features	= hns3_nic_set_features,
+	.ndo_get_stats64	= hns3_nic_get_stats64,
+	.ndo_setup_tc		= hns3_nic_setup_tc,
+	.ndo_set_rx_mode	= hns3_nic_set_rx_mode,
+	.ndo_udp_tunnel_add	= hns3_nic_udp_tunnel_add,
+	.ndo_udp_tunnel_del	= hns3_nic_udp_tunnel_del,
+	.ndo_vlan_rx_add_vid	= hns3_vlan_rx_add_vid,
+	.ndo_vlan_rx_kill_vid	= hns3_vlan_rx_kill_vid,
+	.ndo_set_vf_vlan	= hns3_ndo_set_vf_vlan,
+};
+
+/* hns3_probe - Device initialization routine
+ * @pdev: PCI device information struct
+ * @ent: entry in hns3_pci_tbl
+ *
+ * hns3_probe initializes a PF identified by a pci_dev structure.
+ * The OS initialization, configuring of the PF private structure,
+ * and a hardware reset occur.
+ *
+ * Returns 0 on success, negative on failure
+ */
+static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+	struct hnae3_ae_dev *ae_dev;
+	int ret;
+
+	ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev),
+			      GFP_KERNEL);
+	if (!ae_dev) {
+		ret = -ENOMEM;
+		return ret;
+	}
+
+	ae_dev->pdev = pdev;
+	ae_dev->dev_type = HNAE3_DEV_KNIC;
+	pci_set_drvdata(pdev, ae_dev);
+
+	return hnae3_register_ae_dev(ae_dev);
+}
+
+/* hns3_remove - Device removal routine
+ * @pdev: PCI device information struct
+ */
+static void hns3_remove(struct pci_dev *pdev)
+{
+	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
+
+	hnae3_unregister_ae_dev(ae_dev);
+
+	devm_kfree(&pdev->dev, ae_dev);
+
+	pci_set_drvdata(pdev, NULL);
+}
+
+static struct pci_driver hns3_driver = {
+	.name     = hns3_driver_name,
+	.id_table = hns3_pci_tbl,
+	.probe    = hns3_probe,
+	.remove   = hns3_remove,
+};
+
+/* set default feature to hns3 */
+static void hns3_set_default_feature(struct net_device *netdev)
+{
+	netdev->priv_flags |= IFF_UNICAST_FLT;
+
+	netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
+		NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
+		NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
+		NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
+		NETIF_F_GSO_UDP_TUNNEL_CSUM;
+
+	netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
+
+	netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
+
+	netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
+		NETIF_F_HW_VLAN_CTAG_FILTER |
+		NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
+		NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
+		NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
+		NETIF_F_GSO_UDP_TUNNEL_CSUM;
+
+	netdev->vlan_features |=
+		NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
+		NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO |
+		NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
+		NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
+		NETIF_F_GSO_UDP_TUNNEL_CSUM;
+
+	netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
+		NETIF_F_HW_VLAN_CTAG_FILTER |
+		NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
+		NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
+		NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
+		NETIF_F_GSO_UDP_TUNNEL_CSUM;
+}
+
+static int hns3_alloc_buffer(struct hns3_enet_ring *ring,
+			     struct hns3_desc_cb *cb)
+{
+	unsigned int order = hnae_page_order(ring);
+	struct page *p;
+
+	p = dev_alloc_pages(order);
+	if (!p)
+		return -ENOMEM;
+
+	cb->priv = p;
+	cb->page_offset = 0;
+	cb->reuse_flag = 0;
+	cb->buf  = page_address(p);
+	cb->length = hnae_page_size(ring);
+	cb->type = DESC_TYPE_PAGE;
+
+	memset(cb->buf, 0, cb->length);
+
+	return 0;
+}
+
+static void hns3_free_buffer(struct hns3_enet_ring *ring,
+			     struct hns3_desc_cb *cb)
+{
+	if (cb->type == DESC_TYPE_SKB)
+		dev_kfree_skb_any((struct sk_buff *)cb->priv);
+	else if (!HNAE3_IS_TX_RING(ring))
+		put_page((struct page *)cb->priv);
+	memset(cb, 0, sizeof(*cb));
+}
+
+static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb)
+{
+	cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,
+			       cb->length, ring_to_dma_dir(ring));
+
+	if (dma_mapping_error(ring_to_dev(ring), cb->dma))
+		return -EIO;
+
+	return 0;
+}
+
+static void hns3_unmap_buffer(struct hns3_enet_ring *ring,
+			      struct hns3_desc_cb *cb)
+{
+	if (cb->type == DESC_TYPE_SKB)
+		dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,
+				 ring_to_dma_dir(ring));
+	else
+		dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,
+			       ring_to_dma_dir(ring));
+}
+
+static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i)
+{
+	hns3_unmap_buffer(ring, &ring->desc_cb[i]);
+	ring->desc[i].addr = 0;
+}
+
+static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i)
+{
+	struct hns3_desc_cb *cb = &ring->desc_cb[i];
+
+	if (!ring->desc_cb[i].dma)
+		return;
+
+	hns3_buffer_detach(ring, i);
+	hns3_free_buffer(ring, cb);
+}
+
+static void hns3_free_buffers(struct hns3_enet_ring *ring)
+{
+	int i;
+
+	for (i = 0; i < ring->desc_num; i++)
+		hns3_free_buffer_detach(ring, i);
+}
+
+/* free desc along with its attached buffer */
+static void hns3_free_desc(struct hns3_enet_ring *ring)
+{
+	hns3_free_buffers(ring);
+
+	dma_unmap_single(ring_to_dev(ring), ring->desc_dma_addr,
+			 ring->desc_num * sizeof(ring->desc[0]),
+			 DMA_BIDIRECTIONAL);
+	ring->desc_dma_addr = 0;
+	kfree(ring->desc);
+	ring->desc = NULL;
+}
+
+static int hns3_alloc_desc(struct hns3_enet_ring *ring)
+{
+	int size = ring->desc_num * sizeof(ring->desc[0]);
+
+	ring->desc = kzalloc(size, GFP_KERNEL);
+	if (!ring->desc)
+		return -ENOMEM;
+
+	ring->desc_dma_addr = dma_map_single(ring_to_dev(ring), ring->desc,
+					     size, DMA_BIDIRECTIONAL);
+	if (dma_mapping_error(ring_to_dev(ring), ring->desc_dma_addr)) {
+		ring->desc_dma_addr = 0;
+		kfree(ring->desc);
+		ring->desc = NULL;
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static int hns3_reserve_buffer_map(struct hns3_enet_ring *ring,
+				   struct hns3_desc_cb *cb)
+{
+	int ret;
+
+	ret = hns3_alloc_buffer(ring, cb);
+	if (ret)
+		goto out;
+
+	ret = hns3_map_buffer(ring, cb);
+	if (ret)
+		goto out_with_buf;
+
+	return 0;
+
+out_with_buf:
+	hns3_free_buffers(ring);
+out:
+	return ret;
+}
+
+static int hns3_alloc_buffer_attach(struct hns3_enet_ring *ring, int i)
+{
+	int ret = hns3_reserve_buffer_map(ring, &ring->desc_cb[i]);
+
+	if (ret)
+		return ret;
+
+	ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
+
+	return 0;
+}
+
+/* Allocate memory for raw pkg, and map with dma */
+static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring)
+{
+	int i, j, ret;
+
+	for (i = 0; i < ring->desc_num; i++) {
+		ret = hns3_alloc_buffer_attach(ring, i);
+		if (ret)
+			goto out_buffer_fail;
+	}
+
+	return 0;
+
+out_buffer_fail:
+	for (j = i - 1; j >= 0; j--)
+		hns3_free_buffer_detach(ring, j);
+	return ret;
+}
+
+/* detach a in-used buffer and replace with a reserved one  */
+static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i,
+				struct hns3_desc_cb *res_cb)
+{
+	hns3_map_buffer(ring, &ring->desc_cb[i]);
+	ring->desc_cb[i] = *res_cb;
+	ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
+}
+
+static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)
+{
+	ring->desc_cb[i].reuse_flag = 0;
+	ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma
+		+ ring->desc_cb[i].page_offset);
+}
+
+static void hns3_nic_reclaim_one_desc(struct hns3_enet_ring *ring, int *bytes,
+				      int *pkts)
+{
+	struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
+
+	(*pkts) += (desc_cb->type == DESC_TYPE_SKB);
+	(*bytes) += desc_cb->length;
+	/* desc_cb will be cleaned, after hnae_free_buffer_detach*/
+	hns3_free_buffer_detach(ring, ring->next_to_clean);
+
+	ring_ptr_move_fw(ring, next_to_clean);
+}
+
+static int is_valid_clean_head(struct hns3_enet_ring *ring, int h)
+{
+	int u = ring->next_to_use;
+	int c = ring->next_to_clean;
+
+	if (unlikely(h > ring->desc_num))
+		return 0;
+
+	return u > c ? (h > c && h <= u) : (h > c || h <= u);
+}
+
+int hns3_clean_tx_ring(struct hns3_enet_ring *ring, int budget)
+{
+	struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
+	struct netdev_queue *dev_queue;
+	int bytes, pkts;
+	int head;
+
+	head = readl_relaxed(ring->tqp->io_base + HNS3_RING_TX_RING_HEAD_REG);
+	rmb(); /* Make sure head is ready before touch any data */
+
+	if (is_ring_empty(ring) || head == ring->next_to_clean)
+		return 0; /* no data to poll */
+
+	if (!is_valid_clean_head(ring, head)) {
+		netdev_err(netdev, "wrong head (%d, %d-%d)\n", head,
+			   ring->next_to_use, ring->next_to_clean);
+
+		u64_stats_update_begin(&ring->syncp);
+		ring->stats.io_err_cnt++;
+		u64_stats_update_end(&ring->syncp);
+		return -EIO;
+	}
+
+	bytes = 0;
+	pkts = 0;
+	while (head != ring->next_to_clean && budget) {
+		hns3_nic_reclaim_one_desc(ring, &bytes, &pkts);
+		/* Issue prefetch for next Tx descriptor */
+		prefetch(&ring->desc_cb[ring->next_to_clean]);
+		budget--;
+	}
+
+	ring->tqp_vector->tx_group.total_bytes += bytes;
+	ring->tqp_vector->tx_group.total_packets += pkts;
+
+	u64_stats_update_begin(&ring->syncp);
+	ring->stats.tx_bytes += bytes;
+	ring->stats.tx_pkts += pkts;
+	u64_stats_update_end(&ring->syncp);
+
+	dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index);
+	netdev_tx_completed_queue(dev_queue, pkts, bytes);
+
+	if (unlikely(pkts && netif_carrier_ok(netdev) &&
+		     (ring_space(ring) > HNS3_MAX_BD_PER_PKT))) {
+		/* Make sure that anybody stopping the queue after this
+		 * sees the new next_to_clean.
+		 */
+		smp_mb();
+		if (netif_tx_queue_stopped(dev_queue)) {
+			netif_tx_wake_queue(dev_queue);
+			ring->stats.restart_queue++;
+		}
+	}
+
+	return !!budget;
+}
+
+static int hns3_desc_unused(struct hns3_enet_ring *ring)
+{
+	int ntc = ring->next_to_clean;
+	int ntu = ring->next_to_use;
+
+	return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
+}
+
+static void
+hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, int cleand_count)
+{
+	struct hns3_desc_cb *desc_cb;
+	struct hns3_desc_cb res_cbs;
+	int i, ret;
+
+	for (i = 0; i < cleand_count; i++) {
+		desc_cb = &ring->desc_cb[ring->next_to_use];
+		if (desc_cb->reuse_flag) {
+			u64_stats_update_begin(&ring->syncp);
+			ring->stats.reuse_pg_cnt++;
+			u64_stats_update_end(&ring->syncp);
+
+			hns3_reuse_buffer(ring, ring->next_to_use);
+		} else {
+			ret = hns3_reserve_buffer_map(ring, &res_cbs);
+			if (ret) {
+				u64_stats_update_begin(&ring->syncp);
+				ring->stats.sw_err_cnt++;
+				u64_stats_update_end(&ring->syncp);
+
+				netdev_err(ring->tqp->handle->kinfo.netdev,
+					   "hnae reserve buffer map failed.\n");
+				break;
+			}
+			hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
+		}
+
+		ring_ptr_move_fw(ring, next_to_use);
+	}
+
+	wmb(); /* Make all data has been write before submit */
+	writel_relaxed(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG);
+}
+
+/* hns3_nic_get_headlen - determine size of header for LRO/GRO
+ * @data: pointer to the start of the headers
+ * @max: total length of section to find headers in
+ *
+ * This function is meant to determine the length of headers that will
+ * be recognized by hardware for LRO, GRO, and RSC offloads.  The main
+ * motivation of doing this is to only perform one pull for IPv4 TCP
+ * packets so that we can do basic things like calculating the gso_size
+ * based on the average data per packet.
+ */
+static unsigned int hns3_nic_get_headlen(unsigned char *data, u32 flag,
+					 unsigned int max_size)
+{
+	unsigned char *network;
+	u8 hlen;
+
+	/* This should never happen, but better safe than sorry */
+	if (max_size < ETH_HLEN)
+		return max_size;
+
+	/* Initialize network frame pointer */
+	network = data;
+
+	/* Set first protocol and move network header forward */
+	network += ETH_HLEN;
+
+	/* Handle any vlan tag if present */
+	if (hnae_get_field(flag, HNS3_RXD_VLAN_M, HNS3_RXD_VLAN_S)
+		== HNS3_RX_FLAG_VLAN_PRESENT) {
+		if ((typeof(max_size))(network - data) > (max_size - VLAN_HLEN))
+			return max_size;
+
+		network += VLAN_HLEN;
+	}
+
+	/* Handle L3 protocols */
+	if (hnae_get_field(flag, HNS3_RXD_L3ID_M, HNS3_RXD_L3ID_S)
+		== HNS3_RX_FLAG_L3ID_IPV4) {
+		if ((typeof(max_size))(network - data) >
+		    (max_size - sizeof(struct iphdr)))
+			return max_size;
+
+		/* Access ihl as a u8 to avoid unaligned access on ia64 */
+		hlen = (network[0] & 0x0F) << 2;
+
+		/* Verify hlen meets minimum size requirements */
+		if (hlen < sizeof(struct iphdr))
+			return network - data;
+
+		/* Record next protocol if header is present */
+	} else if (hnae_get_field(flag, HNS3_RXD_L3ID_M, HNS3_RXD_L3ID_S)
+		== HNS3_RX_FLAG_L3ID_IPV6) {
+		if ((typeof(max_size))(network - data) >
+		    (max_size - sizeof(struct ipv6hdr)))
+			return max_size;
+
+		/* Record next protocol */
+		hlen = sizeof(struct ipv6hdr);
+	} else {
+		return network - data;
+	}
+
+	/* Relocate pointer to start of L4 header */
+	network += hlen;
+
+	/* Finally sort out TCP/UDP */
+	if (hnae_get_field(flag, HNS3_RXD_L4ID_M, HNS3_RXD_L4ID_S)
+		== HNS3_RX_FLAG_L4ID_TCP) {
+		if ((typeof(max_size))(network - data) >
+		    (max_size - sizeof(struct tcphdr)))
+			return max_size;
+
+		/* Access doff as a u8 to avoid unaligned access on ia64 */
+		hlen = (network[12] & 0xF0) >> 2;
+
+		/* Verify hlen meets minimum size requirements */
+		if (hlen < sizeof(struct tcphdr))
+			return network - data;
+
+		network += hlen;
+	} else if (hnae_get_field(flag, HNS3_RXD_L4ID_M, HNS3_RXD_L4ID_S)
+		== HNS3_RX_FLAG_L4ID_UDP) {
+		if ((typeof(max_size))(network - data) >
+		    (max_size - sizeof(struct udphdr)))
+			return max_size;
+
+		network += sizeof(struct udphdr);
+	}
+
+	/* If everything has gone correctly network should be the
+	 * data section of the packet and will be the end of the header.
+	 * If not then it probably represents the end of the last recognized
+	 * header.
+	 */
+	if ((typeof(max_size))(network - data) < max_size)
+		return network - data;
+	else
+		return max_size;
+}
+
+static void hns3_nic_reuse_page(struct sk_buff *skb, int i,
+				struct hns3_enet_ring *ring, int pull_len,
+				struct hns3_desc_cb *desc_cb)
+{
+	struct hns3_desc *desc;
+	int truesize, size;
+	int last_offset;
+	bool twobufs;
+
+	twobufs = ((PAGE_SIZE < 8192) &&
+		hnae_buf_size(ring) == HNS3_BUFFER_SIZE_2048);
+
+	desc = &ring->desc[ring->next_to_clean];
+	size = le16_to_cpu(desc->rx.size);
+
+	if (twobufs) {
+		truesize = hnae_buf_size(ring);
+	} else {
+		truesize = ALIGN(size, L1_CACHE_BYTES);
+		last_offset = hnae_page_size(ring) - hnae_buf_size(ring);
+	}
+
+	skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len,
+			size - pull_len, truesize - pull_len);
+
+	 /* Avoid re-using remote pages,flag default unreuse */
+	if (unlikely(page_to_nid(desc_cb->priv) != numa_node_id()))
+		return;
+
+	if (twobufs) {
+		/* If we are only owner of page we can reuse it */
+		if (likely(page_count(desc_cb->priv) == 1)) {
+			/* Flip page offset to other buffer */
+			desc_cb->page_offset ^= truesize;
+
+			desc_cb->reuse_flag = 1;
+			/* bump ref count on page before it is given*/
+			get_page(desc_cb->priv);
+		}
+		return;
+	}
+
+	/* Move offset up to the next cache line */
+	desc_cb->page_offset += truesize;
+
+	if (desc_cb->page_offset <= last_offset) {
+		desc_cb->reuse_flag = 1;
+		/* Bump ref count on page before it is given*/
+		get_page(desc_cb->priv);
+	}
+}
+
+static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb,
+			     struct hns3_desc *desc)
+{
+	struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
+	int l3_type, l4_type;
+	u32 bd_base_info;
+	int ol4_type;
+	u32 l234info;
+
+	bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
+	l234info = le32_to_cpu(desc->rx.l234_info);
+
+	skb->ip_summed = CHECKSUM_NONE;
+
+	skb_checksum_none_assert(skb);
+
+	if (!(netdev->features & NETIF_F_RXCSUM))
+		return;
+
+	/* check if hardware has done checksum */
+	if (!hnae_get_bit(bd_base_info, HNS3_RXD_L3L4P_B))
+		return;
+
+	if (unlikely(hnae_get_bit(l234info, HNS3_RXD_L3E_B) ||
+		     hnae_get_bit(l234info, HNS3_RXD_L4E_B) ||
+		     hnae_get_bit(l234info, HNS3_RXD_OL3E_B) ||
+		     hnae_get_bit(l234info, HNS3_RXD_OL4E_B))) {
+		netdev_err(netdev, "L3/L4 error pkt\n");
+		u64_stats_update_begin(&ring->syncp);
+		ring->stats.l3l4_csum_err++;
+		u64_stats_update_end(&ring->syncp);
+
+		return;
+	}
+
+	l3_type = hnae_get_field(l234info, HNS3_RXD_L3ID_M,
+				 HNS3_RXD_L3ID_S);
+	l4_type = hnae_get_field(l234info, HNS3_RXD_L4ID_M,
+				 HNS3_RXD_L4ID_S);
+
+	ol4_type = hnae_get_field(l234info, HNS3_RXD_OL4ID_M, HNS3_RXD_OL4ID_S);
+	switch (ol4_type) {
+	case HNS3_OL4_TYPE_MAC_IN_UDP:
+	case HNS3_OL4_TYPE_NVGRE:
+		skb->csum_level = 1;
+	case HNS3_OL4_TYPE_NO_TUN:
+		/* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */
+		if (l3_type == HNS3_L3_TYPE_IPV4 ||
+		    (l3_type == HNS3_L3_TYPE_IPV6 &&
+		     (l4_type == HNS3_L4_TYPE_UDP ||
+		      l4_type == HNS3_L4_TYPE_TCP ||
+		      l4_type == HNS3_L4_TYPE_SCTP)))
+			skb->ip_summed = CHECKSUM_UNNECESSARY;
+		break;
+	}
+}
+
+static int hns3_handle_rx_bd(struct hns3_enet_ring *ring,
+			     struct sk_buff **out_skb, int *out_bnum)
+{
+	struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
+	struct hns3_desc_cb *desc_cb;
+	struct hns3_desc *desc;
+	struct sk_buff *skb;
+	unsigned char *va;
+	u32 bd_base_info;
+	int pull_len;
+	u32 l234info;
+	int length;
+	int bnum;
+
+	desc = &ring->desc[ring->next_to_clean];
+	desc_cb = &ring->desc_cb[ring->next_to_clean];
+
+	prefetch(desc);
+
+	length = le16_to_cpu(desc->rx.pkt_len);
+	bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
+	l234info = le32_to_cpu(desc->rx.l234_info);
+
+	/* Check valid BD */
+	if (!hnae_get_bit(bd_base_info, HNS3_RXD_VLD_B))
+		return -EFAULT;
+
+	va = (unsigned char *)desc_cb->buf + desc_cb->page_offset;
+
+	/* Prefetch first cache line of first page
+	 * Idea is to cache few bytes of the header of the packet. Our L1 Cache
+	 * line size is 64B so need to prefetch twice to make it 128B. But in
+	 * actual we can have greater size of caches with 128B Level 1 cache
+	 * lines. In such a case, single fetch would suffice to cache in the
+	 * relevant part of the header.
+	 */
+	prefetch(va);
+#if L1_CACHE_BYTES < 128
+	prefetch(va + L1_CACHE_BYTES);
+#endif
+
+	skb = *out_skb = napi_alloc_skb(&ring->tqp_vector->napi,
+					HNS3_RX_HEAD_SIZE);
+	if (unlikely(!skb)) {
+		netdev_err(netdev, "alloc rx skb fail\n");
+
+		u64_stats_update_begin(&ring->syncp);
+		ring->stats.sw_err_cnt++;
+		u64_stats_update_end(&ring->syncp);
+
+		return -ENOMEM;
+	}
+
+	prefetchw(skb->data);
+
+	bnum = 1;
+	if (length <= HNS3_RX_HEAD_SIZE) {
+		memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
+
+		/* We can reuse buffer as-is, just make sure it is local */
+		if (likely(page_to_nid(desc_cb->priv) == numa_node_id()))
+			desc_cb->reuse_flag = 1;
+		else /* This page cannot be reused so discard it */
+			put_page(desc_cb->priv);
+
+		ring_ptr_move_fw(ring, next_to_clean);
+	} else {
+		u64_stats_update_begin(&ring->syncp);
+		ring->stats.seg_pkt_cnt++;
+		u64_stats_update_end(&ring->syncp);
+
+		pull_len = hns3_nic_get_headlen(va, l234info,
+						HNS3_RX_HEAD_SIZE);
+		memcpy(__skb_put(skb, pull_len), va,
+		       ALIGN(pull_len, sizeof(long)));
+
+		hns3_nic_reuse_page(skb, 0, ring, pull_len, desc_cb);
+		ring_ptr_move_fw(ring, next_to_clean);
+
+		while (!hnae_get_bit(bd_base_info, HNS3_RXD_FE_B)) {
+			desc = &ring->desc[ring->next_to_clean];
+			desc_cb = &ring->desc_cb[ring->next_to_clean];
+			bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
+			hns3_nic_reuse_page(skb, bnum, ring, 0, desc_cb);
+			ring_ptr_move_fw(ring, next_to_clean);
+			bnum++;
+		}
+	}
+
+	*out_bnum = bnum;
+
+	if (unlikely(!hnae_get_bit(bd_base_info, HNS3_RXD_VLD_B))) {
+		netdev_err(netdev, "no valid bd,%016llx,%016llx\n",
+			   ((u64 *)desc)[0], ((u64 *)desc)[1]);
+		u64_stats_update_begin(&ring->syncp);
+		ring->stats.non_vld_descs++;
+		u64_stats_update_end(&ring->syncp);
+
+		dev_kfree_skb_any(skb);
+		return -EINVAL;
+	}
+
+	if (unlikely((!desc->rx.pkt_len) ||
+		     hnae_get_bit(l234info, HNS3_RXD_TRUNCAT_B))) {
+		netdev_err(netdev, "truncated pkt\n");
+		u64_stats_update_begin(&ring->syncp);
+		ring->stats.err_pkt_len++;
+		u64_stats_update_end(&ring->syncp);
+
+		dev_kfree_skb_any(skb);
+		return -EFAULT;
+	}
+
+	if (unlikely(hnae_get_bit(l234info, HNS3_RXD_L2E_B))) {
+		netdev_err(netdev, "L2 error pkt\n");
+		u64_stats_update_begin(&ring->syncp);
+		ring->stats.l2_err++;
+		u64_stats_update_end(&ring->syncp);
+
+		dev_kfree_skb_any(skb);
+		return -EFAULT;
+	}
+
+	u64_stats_update_begin(&ring->syncp);
+	ring->stats.rx_pkts++;
+	ring->stats.rx_bytes += skb->len;
+	u64_stats_update_end(&ring->syncp);
+
+	ring->tqp_vector->rx_group.total_bytes += skb->len;
+
+	hns3_rx_checksum(ring, skb, desc);
+	return 0;
+}
+
+static int hns3_clean_rx_ring(struct hns3_enet_ring *ring, int budget)
+{
+#define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
+	struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
+	int recv_pkts, recv_bds, clean_count, err;
+	int unused_count = hns3_desc_unused(ring);
+	struct sk_buff *skb = NULL;
+	int num, bnum = 0;
+
+	num = readl_relaxed(ring->tqp->io_base + HNS3_RING_RX_RING_FBDNUM_REG);
+	rmb(); /* Make sure num taken effect before the other data is touched */
+
+	recv_pkts = 0, recv_bds = 0, clean_count = 0;
+	num -= unused_count;
+
+	while (recv_pkts < budget && recv_bds < num) {
+		/* Reuse or realloc buffers */
+		if (clean_count + unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
+			hns3_nic_alloc_rx_buffers(ring,
+						  clean_count + unused_count);
+			clean_count = 0;
+			unused_count = hns3_desc_unused(ring);
+		}
+
+		/* Poll one pkt */
+		err = hns3_handle_rx_bd(ring, &skb, &bnum);
+		if (unlikely(!skb)) /* This fault cannot be repaired */
+			goto out;
+
+		recv_bds += bnum;
+		clean_count += bnum;
+		if (unlikely(err)) {  /* Do jump the err */
+			recv_pkts++;
+			continue;
+		}
+
+		/* Do update ip stack process */
+		skb->protocol = eth_type_trans(skb, netdev);
+		(void)napi_gro_receive(&ring->tqp_vector->napi, skb);
+
+		recv_pkts++;
+	}
+
+out:
+	/* Make all data has been write before submit */
+	if (clean_count + unused_count > 0)
+		hns3_nic_alloc_rx_buffers(ring,
+					  clean_count + unused_count);
+
+	return recv_pkts;
+}
+
+static bool hns3_get_new_int_gl(struct hns3_enet_ring_group *ring_group)
+{
+#define HNS3_RX_ULTRA_PACKET_RATE 40000
+	enum hns3_flow_level_range new_flow_level;
+	struct hns3_enet_tqp_vector *tqp_vector;
+	int packets_per_secs;
+	int bytes_per_usecs;
+	u16 new_int_gl;
+	int usecs;
+
+	if (!ring_group->int_gl)
+		return false;
+
+	if (ring_group->total_packets == 0) {
+		ring_group->int_gl = HNS3_INT_GL_50K;
+		ring_group->flow_level = HNS3_FLOW_LOW;
+		return true;
+	}
+
+	/* Simple throttlerate management
+	 * 0-10MB/s   lower     (50000 ints/s)
+	 * 10-20MB/s   middle    (20000 ints/s)
+	 * 20-1249MB/s high      (18000 ints/s)
+	 * > 40000pps  ultra     (8000 ints/s)
+	 */
+	new_flow_level = ring_group->flow_level;
+	new_int_gl = ring_group->int_gl;
+	tqp_vector = ring_group->ring->tqp_vector;
+	usecs = (ring_group->int_gl << 1);
+	bytes_per_usecs = ring_group->total_bytes / usecs;
+	/* 1000000 microseconds */
+	packets_per_secs = ring_group->total_packets * 1000000 / usecs;
+
+	switch (new_flow_level) {
+	case HNS3_FLOW_LOW:
+		if (bytes_per_usecs > 10)
+			new_flow_level = HNS3_FLOW_MID;
+		break;
+	case HNS3_FLOW_MID:
+		if (bytes_per_usecs > 20)
+			new_flow_level = HNS3_FLOW_HIGH;
+		else if (bytes_per_usecs <= 10)
+			new_flow_level = HNS3_FLOW_LOW;
+		break;
+	case HNS3_FLOW_HIGH:
+	case HNS3_FLOW_ULTRA:
+	default:
+		if (bytes_per_usecs <= 20)
+			new_flow_level = HNS3_FLOW_MID;
+		break;
+	}
+
+	if ((packets_per_secs > HNS3_RX_ULTRA_PACKET_RATE) &&
+	    (&tqp_vector->rx_group == ring_group))
+		new_flow_level = HNS3_FLOW_ULTRA;
+
+	switch (new_flow_level) {
+	case HNS3_FLOW_LOW:
+		new_int_gl = HNS3_INT_GL_50K;
+		break;
+	case HNS3_FLOW_MID:
+		new_int_gl = HNS3_INT_GL_20K;
+		break;
+	case HNS3_FLOW_HIGH:
+		new_int_gl = HNS3_INT_GL_18K;
+		break;
+	case HNS3_FLOW_ULTRA:
+		new_int_gl = HNS3_INT_GL_8K;
+		break;
+	default:
+		break;
+	}
+
+	ring_group->total_bytes = 0;
+	ring_group->total_packets = 0;
+	ring_group->flow_level = new_flow_level;
+	if (new_int_gl != ring_group->int_gl) {
+		ring_group->int_gl = new_int_gl;
+		return true;
+	}
+	return false;
+}
+
+static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector *tqp_vector)
+{
+	u16 rx_int_gl, tx_int_gl;
+	bool rx, tx;
+
+	rx = hns3_get_new_int_gl(&tqp_vector->rx_group);
+	tx = hns3_get_new_int_gl(&tqp_vector->tx_group);
+	rx_int_gl = tqp_vector->rx_group.int_gl;
+	tx_int_gl = tqp_vector->tx_group.int_gl;
+	if (rx && tx) {
+		if (rx_int_gl > tx_int_gl) {
+			tqp_vector->tx_group.int_gl = rx_int_gl;
+			tqp_vector->tx_group.flow_level =
+				tqp_vector->rx_group.flow_level;
+			hns3_set_vector_coalesc_gl(tqp_vector, rx_int_gl);
+		} else {
+			tqp_vector->rx_group.int_gl = tx_int_gl;
+			tqp_vector->rx_group.flow_level =
+				tqp_vector->tx_group.flow_level;
+			hns3_set_vector_coalesc_gl(tqp_vector, tx_int_gl);
+		}
+	}
+}
+
+static int hns3_nic_common_poll(struct napi_struct *napi, int budget)
+{
+	struct hns3_enet_ring *ring;
+	int rx_pkt_total = 0;
+
+	struct hns3_enet_tqp_vector *tqp_vector =
+		container_of(napi, struct hns3_enet_tqp_vector, napi);
+	bool clean_complete = true;
+	int rx_budget;
+
+	/* Since the actual Tx work is minimal, we can give the Tx a larger
+	 * budget and be more aggressive about cleaning up the Tx descriptors.
+	 */
+	hns3_for_each_ring(ring, tqp_vector->tx_group) {
+		if (!hns3_clean_tx_ring(ring, budget))
+			clean_complete = false;
+	}
+
+	/* make sure rx ring budget not smaller than 1 */
+	rx_budget = max(budget / tqp_vector->num_tqps, 1);
+
+	hns3_for_each_ring(ring, tqp_vector->rx_group) {
+		int rx_cleaned = hns3_clean_rx_ring(ring, rx_budget);
+
+		if (rx_cleaned >= rx_budget)
+			clean_complete = false;
+
+		rx_pkt_total += rx_cleaned;
+	}
+
+	tqp_vector->rx_group.total_packets += rx_pkt_total;
+
+	if (!clean_complete)
+		return budget;
+
+	napi_complete(napi);
+	hns3_update_new_int_gl(tqp_vector);
+	hns3_mask_vector_irq(tqp_vector, 1);
+
+	return rx_pkt_total;
+}
+
+static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
+				      struct hnae3_ring_chain_node *head)
+{
+	struct pci_dev *pdev = tqp_vector->handle->pdev;
+	struct hnae3_ring_chain_node *cur_chain = head;
+	struct hnae3_ring_chain_node *chain;
+	struct hns3_enet_ring *tx_ring;
+	struct hns3_enet_ring *rx_ring;
+
+	tx_ring = tqp_vector->tx_group.ring;
+	if (tx_ring) {
+		cur_chain->tqp_index = tx_ring->tqp->tqp_index;
+		hnae_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
+			     HNAE3_RING_TYPE_TX);
+
+		cur_chain->next = NULL;
+
+		while (tx_ring->next) {
+			tx_ring = tx_ring->next;
+
+			chain = devm_kzalloc(&pdev->dev, sizeof(*chain),
+					     GFP_KERNEL);
+			if (!chain)
+				return -ENOMEM;
+
+			cur_chain->next = chain;
+			chain->tqp_index = tx_ring->tqp->tqp_index;
+			hnae_set_bit(chain->flag, HNAE3_RING_TYPE_B,
+				     HNAE3_RING_TYPE_TX);
+
+			cur_chain = chain;
+		}
+	}
+
+	rx_ring = tqp_vector->rx_group.ring;
+	if (!tx_ring && rx_ring) {
+		cur_chain->next = NULL;
+		cur_chain->tqp_index = rx_ring->tqp->tqp_index;
+		hnae_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
+			     HNAE3_RING_TYPE_RX);
+
+		rx_ring = rx_ring->next;
+	}
+
+	while (rx_ring) {
+		chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
+		if (!chain)
+			return -ENOMEM;
+
+		cur_chain->next = chain;
+		chain->tqp_index = rx_ring->tqp->tqp_index;
+		hnae_set_bit(chain->flag, HNAE3_RING_TYPE_B,
+			     HNAE3_RING_TYPE_RX);
+		cur_chain = chain;
+
+		rx_ring = rx_ring->next;
+	}
+
+	return 0;
+}
+
+static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
+					struct hnae3_ring_chain_node *head)
+{
+	struct pci_dev *pdev = tqp_vector->handle->pdev;
+	struct hnae3_ring_chain_node *chain_tmp, *chain;
+
+	chain = head->next;
+
+	while (chain) {
+		chain_tmp = chain->next;
+		devm_kfree(&pdev->dev, chain);
+		chain = chain_tmp;
+	}
+}
+
+static void hns3_add_ring_to_group(struct hns3_enet_ring_group *group,
+				   struct hns3_enet_ring *ring)
+{
+	ring->next = group->ring;
+	group->ring = ring;
+
+	group->count++;
+}
+
+static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv)
+{
+	struct hnae3_ring_chain_node vector_ring_chain;
+	struct hnae3_handle *h = priv->ae_handle;
+	struct hns3_enet_tqp_vector *tqp_vector;
+	struct hnae3_vector_info *vector;
+	struct pci_dev *pdev = h->pdev;
+	u16 tqp_num = h->kinfo.num_tqps;
+	u16 vector_num;
+	int ret = 0;
+	u16 i;
+
+	/* RSS size, cpu online and vector_num should be the same */
+	/* Should consider 2p/4p later */
+	vector_num = min_t(u16, num_online_cpus(), tqp_num);
+	vector = devm_kcalloc(&pdev->dev, vector_num, sizeof(*vector),
+			      GFP_KERNEL);
+	if (!vector)
+		return -ENOMEM;
+
+	vector_num = h->ae_algo->ops->get_vector(h, vector_num, vector);
+
+	priv->vector_num = vector_num;
+	priv->tqp_vector = (struct hns3_enet_tqp_vector *)
+		devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),
+			     GFP_KERNEL);
+	if (!priv->tqp_vector)
+		return -ENOMEM;
+
+	for (i = 0; i < tqp_num; i++) {
+		u16 vector_i = i % vector_num;
+
+		tqp_vector = &priv->tqp_vector[vector_i];
+
+		hns3_add_ring_to_group(&tqp_vector->tx_group,
+				       priv->ring_data[i].ring);
+
+		hns3_add_ring_to_group(&tqp_vector->rx_group,
+				       priv->ring_data[i + tqp_num].ring);
+
+		tqp_vector->idx = vector_i;
+		tqp_vector->mask_addr = vector[vector_i].io_addr;
+		tqp_vector->vector_irq = vector[vector_i].vector;
+		tqp_vector->num_tqps++;
+
+		priv->ring_data[i].ring->tqp_vector = tqp_vector;
+		priv->ring_data[i + tqp_num].ring->tqp_vector = tqp_vector;
+	}
+
+	for (i = 0; i < vector_num; i++) {
+		tqp_vector = &priv->tqp_vector[i];
+
+		tqp_vector->rx_group.total_bytes = 0;
+		tqp_vector->rx_group.total_packets = 0;
+		tqp_vector->tx_group.total_bytes = 0;
+		tqp_vector->tx_group.total_packets = 0;
+		hns3_vector_gl_rl_init(tqp_vector);
+		tqp_vector->handle = h;
+
+		ret = hns3_get_vector_ring_chain(tqp_vector,
+						 &vector_ring_chain);
+		if (ret)
+			goto out;
+
+		ret = h->ae_algo->ops->map_ring_to_vector(h,
+			tqp_vector->vector_irq, &vector_ring_chain);
+		if (ret)
+			goto out;
+
+		hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
+
+		netif_napi_add(priv->netdev, &tqp_vector->napi,
+			       hns3_nic_common_poll, NAPI_POLL_WEIGHT);
+	}
+
+out:
+	devm_kfree(&pdev->dev, vector);
+	return ret;
+}
+
+static int hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv)
+{
+	struct hnae3_ring_chain_node vector_ring_chain;
+	struct hnae3_handle *h = priv->ae_handle;
+	struct hns3_enet_tqp_vector *tqp_vector;
+	struct pci_dev *pdev = h->pdev;
+	int i, ret;
+
+	for (i = 0; i < priv->vector_num; i++) {
+		tqp_vector = &priv->tqp_vector[i];
+
+		ret = hns3_get_vector_ring_chain(tqp_vector,
+						 &vector_ring_chain);
+		if (ret)
+			return ret;
+
+		ret = h->ae_algo->ops->unmap_ring_from_vector(h,
+			tqp_vector->vector_irq, &vector_ring_chain);
+		if (ret)
+			return ret;
+
+		hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
+
+		if (priv->tqp_vector[i].irq_init_flag == HNS3_VECTOR_INITED) {
+			(void)irq_set_affinity_hint(
+				priv->tqp_vector[i].vector_irq,
+						    NULL);
+			devm_free_irq(&pdev->dev,
+				      priv->tqp_vector[i].vector_irq,
+				      &priv->tqp_vector[i]);
+		}
+
+		priv->ring_data[i].ring->irq_init_flag = HNS3_VECTOR_NOT_INITED;
+
+		netif_napi_del(&priv->tqp_vector[i].napi);
+	}
+
+	devm_kfree(&pdev->dev, priv->tqp_vector);
+
+	return 0;
+}
+
+static int hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv,
+			     int ring_type)
+{
+	struct hns3_nic_ring_data *ring_data = priv->ring_data;
+	int queue_num = priv->ae_handle->kinfo.num_tqps;
+	struct pci_dev *pdev = priv->ae_handle->pdev;
+	struct hns3_enet_ring *ring;
+
+	ring = devm_kzalloc(&pdev->dev, sizeof(*ring), GFP_KERNEL);
+	if (!ring)
+		return -ENOMEM;
+
+	if (ring_type == HNAE3_RING_TYPE_TX) {
+		ring_data[q->tqp_index].ring = ring;
+		ring->io_base = (u8 __iomem *)q->io_base + HNS3_TX_REG_OFFSET;
+	} else {
+		ring_data[q->tqp_index + queue_num].ring = ring;
+		ring->io_base = q->io_base;
+	}
+
+	hnae_set_bit(ring->flag, HNAE3_RING_TYPE_B, ring_type);
+
+	ring_data[q->tqp_index].queue_index = q->tqp_index;
+
+	ring->tqp = q;
+	ring->desc = NULL;
+	ring->desc_cb = NULL;
+	ring->dev = priv->dev;
+	ring->desc_dma_addr = 0;
+	ring->buf_size = q->buf_size;
+	ring->desc_num = q->desc_num;
+	ring->next_to_use = 0;
+	ring->next_to_clean = 0;
+
+	return 0;
+}
+
+static int hns3_queue_to_ring(struct hnae3_queue *tqp,
+			      struct hns3_nic_priv *priv)
+{
+	int ret;
+
+	ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX);
+	if (ret)
+		return ret;
+
+	ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int hns3_get_ring_config(struct hns3_nic_priv *priv)
+{
+	struct hnae3_handle *h = priv->ae_handle;
+	struct pci_dev *pdev = h->pdev;
+	int i, ret;
+
+	priv->ring_data =  devm_kzalloc(&pdev->dev, h->kinfo.num_tqps *
+					sizeof(*priv->ring_data) * 2,
+					GFP_KERNEL);
+	if (!priv->ring_data)
+		return -ENOMEM;
+
+	for (i = 0; i < h->kinfo.num_tqps; i++) {
+		ret = hns3_queue_to_ring(h->kinfo.tqp[i], priv);
+		if (ret)
+			goto err;
+	}
+
+	return 0;
+err:
+	devm_kfree(&pdev->dev, priv->ring_data);
+	return ret;
+}
+
+static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring)
+{
+	int ret;
+
+	if (ring->desc_num <= 0 || ring->buf_size <= 0)
+		return -EINVAL;
+
+	ring->desc_cb = kcalloc(ring->desc_num, sizeof(ring->desc_cb[0]),
+				GFP_KERNEL);
+	if (!ring->desc_cb) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	ret = hns3_alloc_desc(ring);
+	if (ret)
+		goto out_with_desc_cb;
+
+	if (!HNAE3_IS_TX_RING(ring)) {
+		ret = hns3_alloc_ring_buffers(ring);
+		if (ret)
+			goto out_with_desc;
+	}
+
+	return 0;
+
+out_with_desc:
+	hns3_free_desc(ring);
+out_with_desc_cb:
+	kfree(ring->desc_cb);
+	ring->desc_cb = NULL;
+out:
+	return ret;
+}
+
+static void hns3_fini_ring(struct hns3_enet_ring *ring)
+{
+	hns3_free_desc(ring);
+	kfree(ring->desc_cb);
+	ring->desc_cb = NULL;
+	ring->next_to_clean = 0;
+	ring->next_to_use = 0;
+}
+
+int hns3_buf_size2type(u32 buf_size)
+{
+	int bd_size_type;
+
+	switch (buf_size) {
+	case 512:
+		bd_size_type = HNS3_BD_SIZE_512_TYPE;
+		break;
+	case 1024:
+		bd_size_type = HNS3_BD_SIZE_1024_TYPE;
+		break;
+	case 2048:
+		bd_size_type = HNS3_BD_SIZE_2048_TYPE;
+		break;
+	case 4096:
+		bd_size_type = HNS3_BD_SIZE_4096_TYPE;
+		break;
+	default:
+		bd_size_type = HNS3_BD_SIZE_2048_TYPE;
+	}
+
+	return bd_size_type;
+}
+
+static void hns3_init_ring_hw(struct hns3_enet_ring *ring)
+{
+	dma_addr_t dma = ring->desc_dma_addr;
+	struct hnae3_queue *q = ring->tqp;
+
+	if (!HNAE3_IS_TX_RING(ring)) {
+		hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_L_REG,
+			       (u32)dma);
+		hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_H_REG,
+			       (u32)((dma >> 31) >> 1));
+
+		hns3_write_dev(q, HNS3_RING_RX_RING_BD_LEN_REG,
+			       hns3_buf_size2type(ring->buf_size));
+		hns3_write_dev(q, HNS3_RING_RX_RING_BD_NUM_REG,
+			       ring->desc_num / 8 - 1);
+
+	} else {
+		hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_L_REG,
+			       (u32)dma);
+		hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_H_REG,
+			       (u32)((dma >> 31) >> 1));
+
+		hns3_write_dev(q, HNS3_RING_TX_RING_BD_LEN_REG,
+			       hns3_buf_size2type(ring->buf_size));
+		hns3_write_dev(q, HNS3_RING_TX_RING_BD_NUM_REG,
+			       ring->desc_num / 8 - 1);
+	}
+}
+
+static int hns3_init_all_ring(struct hns3_nic_priv *priv)
+{
+	struct hnae3_handle *h = priv->ae_handle;
+	int ring_num = h->kinfo.num_tqps * 2;
+	int i, j;
+	int ret;
+
+	for (i = 0; i < ring_num; i++) {
+		ret = hns3_alloc_ring_memory(priv->ring_data[i].ring);
+		if (ret) {
+			dev_err(priv->dev,
+				"Alloc ring memory fail! ret=%d\n", ret);
+			goto out_when_alloc_ring_memory;
+		}
+
+		hns3_init_ring_hw(priv->ring_data[i].ring);
+
+		u64_stats_init(&priv->ring_data[i].ring->syncp);
+	}
+
+	return 0;
+
+out_when_alloc_ring_memory:
+	for (j = i - 1; j >= 0; j--)
+		hns3_fini_ring(priv->ring_data[i].ring);
+
+	return -ENOMEM;
+}
+
+static int hns3_uninit_all_ring(struct hns3_nic_priv *priv)
+{
+	struct hnae3_handle *h = priv->ae_handle;
+	int i;
+
+	for (i = 0; i < h->kinfo.num_tqps; i++) {
+		if (h->ae_algo->ops->reset_queue)
+			h->ae_algo->ops->reset_queue(h, i);
+
+		hns3_fini_ring(priv->ring_data[i].ring);
+		hns3_fini_ring(priv->ring_data[i + h->kinfo.num_tqps].ring);
+	}
+
+	return 0;
+}
+
+/* Set mac addr if it is configured. or leave it to the AE driver */
+static void hns3_init_mac_addr(struct net_device *netdev)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+	u8 mac_addr_temp[ETH_ALEN];
+
+	if (h->ae_algo->ops->get_mac_addr) {
+		h->ae_algo->ops->get_mac_addr(h, mac_addr_temp);
+		ether_addr_copy(netdev->dev_addr, mac_addr_temp);
+	}
+
+	/* Check if the MAC address is valid, if not get a random one */
+	if (!is_valid_ether_addr(netdev->dev_addr)) {
+		eth_hw_addr_random(netdev);
+		dev_warn(priv->dev, "using random MAC address %pM\n",
+			 netdev->dev_addr);
+		/* Also copy this new MAC address into hdev */
+		if (h->ae_algo->ops->set_mac_addr)
+			h->ae_algo->ops->set_mac_addr(h, netdev->dev_addr);
+	}
+}
+
+static void hns3_nic_set_priv_ops(struct net_device *netdev)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+
+	if ((netdev->features & NETIF_F_TSO) ||
+	    (netdev->features & NETIF_F_TSO6)) {
+		priv->ops.fill_desc = hns3_fill_desc_tso;
+		priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
+	} else {
+		priv->ops.fill_desc = hns3_fill_desc;
+		priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
+	}
+}
+
+static int hns3_client_init(struct hnae3_handle *handle)
+{
+	struct pci_dev *pdev = handle->pdev;
+	struct hns3_nic_priv *priv;
+	struct net_device *netdev;
+	int ret;
+
+	netdev = alloc_etherdev_mq(sizeof(struct hns3_nic_priv),
+				   handle->kinfo.num_tqps);
+	if (!netdev)
+		return -ENOMEM;
+
+	priv = netdev_priv(netdev);
+	priv->dev = &pdev->dev;
+	priv->netdev = netdev;
+	priv->ae_handle = handle;
+
+	handle->kinfo.netdev = netdev;
+	handle->priv = (void *)priv;
+
+	hns3_init_mac_addr(netdev);
+
+	hns3_set_default_feature(netdev);
+
+	netdev->watchdog_timeo = HNS3_TX_TIMEOUT;
+	netdev->priv_flags |= IFF_UNICAST_FLT;
+	netdev->netdev_ops = &hns3_nic_netdev_ops;
+	SET_NETDEV_DEV(netdev, &pdev->dev);
+	hns3_ethtool_set_ops(netdev);
+	hns3_nic_set_priv_ops(netdev);
+
+	/* Carrier off reporting is important to ethtool even BEFORE open */
+	netif_carrier_off(netdev);
+
+	ret = hns3_get_ring_config(priv);
+	if (ret) {
+		ret = -ENOMEM;
+		goto out_get_ring_cfg;
+	}
+
+	ret = hns3_nic_init_vector_data(priv);
+	if (ret) {
+		ret = -ENOMEM;
+		goto out_init_vector_data;
+	}
+
+	ret = hns3_init_all_ring(priv);
+	if (ret) {
+		ret = -ENOMEM;
+		goto out_init_ring_data;
+	}
+
+	ret = register_netdev(netdev);
+	if (ret) {
+		dev_err(priv->dev, "probe register netdev fail!\n");
+		goto out_reg_netdev_fail;
+	}
+
+	/* MTU range: (ETH_MIN_MTU(kernel default) - 9706) */
+	netdev->max_mtu = HNS3_MAX_MTU - (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
+
+	return ret;
+
+out_reg_netdev_fail:
+out_init_ring_data:
+	(void)hns3_nic_uninit_vector_data(priv);
+	priv->ring_data = NULL;
+out_init_vector_data:
+out_get_ring_cfg:
+	priv->ae_handle = NULL;
+	free_netdev(netdev);
+	return ret;
+}
+
+static void hns3_client_uninit(struct hnae3_handle *handle, bool reset)
+{
+	struct net_device *netdev = handle->kinfo.netdev;
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	int ret;
+
+	if (netdev->reg_state != NETREG_UNINITIALIZED)
+		unregister_netdev(netdev);
+
+	ret = hns3_nic_uninit_vector_data(priv);
+	if (ret)
+		netdev_err(netdev, "uninit vector error\n");
+
+	ret = hns3_uninit_all_ring(priv);
+	if (ret)
+		netdev_err(netdev, "uninit ring error\n");
+
+	priv->ring_data = NULL;
+
+	free_netdev(netdev);
+}
+
+static void hns3_link_status_change(struct hnae3_handle *handle, bool linkup)
+{
+	struct net_device *netdev = handle->kinfo.netdev;
+
+	if (!netdev)
+		return;
+
+	if (linkup) {
+		netif_carrier_on(netdev);
+		netif_tx_wake_all_queues(netdev);
+		netdev_info(netdev, "link up\n");
+	} else {
+		netif_carrier_off(netdev);
+		netif_tx_stop_all_queues(netdev);
+		netdev_info(netdev, "link down\n");
+	}
+}
+
+const struct hnae3_client_ops client_ops = {
+	.init_instance = hns3_client_init,
+	.uninit_instance = hns3_client_uninit,
+	.link_status_change = hns3_link_status_change,
+};
+
+/* hns3_init_module - Driver registration routine
+ * hns3_init_module is the first routine called when the driver is
+ * loaded. All it does is register with the PCI subsystem.
+ */
+static int __init hns3_init_module(void)
+{
+	int ret;
+
+	pr_info("%s: %s - version\n", hns3_driver_name, hns3_driver_string);
+	pr_info("%s: %s\n", hns3_driver_name, hns3_copyright);
+
+	client.type = HNAE3_CLIENT_KNIC;
+	snprintf(client.name, HNAE3_CLIENT_NAME_LENGTH - 1, "%s",
+		 hns3_driver_name);
+
+	client.ops = &client_ops;
+
+	ret = hnae3_register_client(&client);
+	if (ret)
+		return ret;
+
+	ret = pci_register_driver(&hns3_driver);
+	if (ret)
+		hnae3_unregister_client(&client);
+
+	return ret;
+}
+module_init(hns3_init_module);
+
+/* hns3_exit_module - Driver exit cleanup routine
+ * hns3_exit_module is called just before the driver is removed
+ * from memory.
+ */
+static void __exit hns3_exit_module(void)
+{
+	pci_unregister_driver(&hns3_driver);
+	hnae3_unregister_client(&client);
+}
+module_exit(hns3_exit_module);
+
+MODULE_DESCRIPTION("HNS3: Hisilicon Ethernet Driver");
+MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("pci:hns-nic");
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hns3_enet.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hns3_enet.h
new file mode 100644
index 000000000000..7e8746189747
--- /dev/null
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hns3_enet.h
@@ -0,0 +1,593 @@
+/*
+ * Copyright (c) 2016 Hisilicon Limited.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef __HNS3_ENET_H
+#define __HNS3_ENET_H
+
+#include "hnae3.h"
+
+extern const char hns3_driver_version[];
+
+enum hns3_nic_state {
+	HNS3_NIC_STATE_TESTING,
+	HNS3_NIC_STATE_RESETTING,
+	HNS3_NIC_STATE_REINITING,
+	HNS3_NIC_STATE_DOWN,
+	HNS3_NIC_STATE_DISABLED,
+	HNS3_NIC_STATE_REMOVING,
+	HNS3_NIC_STATE_SERVICE_INITED,
+	HNS3_NIC_STATE_SERVICE_SCHED,
+	HNS3_NIC_STATE2_RESET_REQUESTED,
+	HNS3_NIC_STATE_MAX
+};
+
+#define HNS3_RING_RX_RING_BASEADDR_L_REG	0x00000
+#define HNS3_RING_RX_RING_BASEADDR_H_REG	0x00004
+#define HNS3_RING_RX_RING_BD_NUM_REG		0x00008
+#define HNS3_RING_RX_RING_BD_LEN_REG		0x0000C
+#define HNS3_RING_RX_RING_TAIL_REG		0x00018
+#define HNS3_RING_RX_RING_HEAD_REG		0x0001C
+#define HNS3_RING_RX_RING_FBDNUM_REG		0x00020
+#define HNS3_RING_RX_RING_PKTNUM_RECORD_REG	0x0002C
+
+#define HNS3_RING_TX_RING_BASEADDR_L_REG	0x00040
+#define HNS3_RING_TX_RING_BASEADDR_H_REG	0x00044
+#define HNS3_RING_TX_RING_BD_NUM_REG		0x00048
+#define HNS3_RING_TX_RING_BD_LEN_REG		0x0004C
+#define HNS3_RING_TX_RING_TAIL_REG		0x00058
+#define HNS3_RING_TX_RING_HEAD_REG		0x0005C
+#define HNS3_RING_TX_RING_FBDNUM_REG		0x00060
+#define HNS3_RING_TX_RING_OFFSET_REG		0x00064
+#define HNS3_RING_TX_RING_PKTNUM_RECORD_REG	0x0006C
+
+#define HNS3_RING_PREFETCH_EN_REG		0x0007C
+#define HNS3_RING_CFG_VF_NUM_REG		0x00080
+#define HNS3_RING_ASID_REG			0x0008C
+#define HNS3_RING_RX_VM_REG			0x00090
+#define HNS3_RING_T0_BE_RST			0x00094
+#define HNS3_RING_COULD_BE_RST			0x00098
+#define HNS3_RING_WRR_WEIGHT_REG		0x0009c
+
+#define HNS3_RING_INTMSK_RXWL_REG		0x000A0
+#define HNS3_RING_INTSTS_RX_RING_REG		0x000A4
+#define HNS3_RX_RING_INT_STS_REG		0x000A8
+#define HNS3_RING_INTMSK_TXWL_REG		0x000AC
+#define HNS3_RING_INTSTS_TX_RING_REG		0x000B0
+#define HNS3_TX_RING_INT_STS_REG		0x000B4
+#define HNS3_RING_INTMSK_RX_OVERTIME_REG	0x000B8
+#define HNS3_RING_INTSTS_RX_OVERTIME_REG	0x000BC
+#define HNS3_RING_INTMSK_TX_OVERTIME_REG	0x000C4
+#define HNS3_RING_INTSTS_TX_OVERTIME_REG	0x000C8
+
+#define HNS3_RING_MB_CTRL_REG			0x00100
+#define HNS3_RING_MB_DATA_BASE_REG		0x00200
+
+#define HNS3_TX_REG_OFFSET			0x40
+
+#define HNS3_RX_HEAD_SIZE			256
+
+#define HNS3_TX_TIMEOUT (5 * HZ)
+#define HNS3_RING_NAME_LEN			16
+#define HNS3_BUFFER_SIZE_2048			2048
+#define HNS3_RING_MAX_PENDING			32768
+#define HNS3_MAX_MTU				9728
+
+#define HNS3_BD_SIZE_512_TYPE			0
+#define HNS3_BD_SIZE_1024_TYPE			1
+#define HNS3_BD_SIZE_2048_TYPE			2
+#define HNS3_BD_SIZE_4096_TYPE			3
+
+#define HNS3_RX_FLAG_VLAN_PRESENT		0x1
+#define HNS3_RX_FLAG_L3ID_IPV4			0x0
+#define HNS3_RX_FLAG_L3ID_IPV6			0x1
+#define HNS3_RX_FLAG_L4ID_UDP			0x0
+#define HNS3_RX_FLAG_L4ID_TCP			0x1
+
+#define HNS3_RXD_DMAC_S				0
+#define HNS3_RXD_DMAC_M				(0x3 << HNS3_RXD_DMAC_S)
+#define HNS3_RXD_VLAN_S				2
+#define HNS3_RXD_VLAN_M				(0x3 << HNS3_RXD_VLAN_S)
+#define HNS3_RXD_L3ID_S				4
+#define HNS3_RXD_L3ID_M				(0xf << HNS3_RXD_L3ID_S)
+#define HNS3_RXD_L4ID_S				8
+#define HNS3_RXD_L4ID_M				(0xf << HNS3_RXD_L4ID_S)
+#define HNS3_RXD_FRAG_B				12
+#define HNS3_RXD_L2E_B				16
+#define HNS3_RXD_L3E_B				17
+#define HNS3_RXD_L4E_B				18
+#define HNS3_RXD_TRUNCAT_B			19
+#define HNS3_RXD_HOI_B				20
+#define HNS3_RXD_DOI_B				21
+#define HNS3_RXD_OL3E_B				22
+#define HNS3_RXD_OL4E_B				23
+
+#define HNS3_RXD_ODMAC_S			0
+#define HNS3_RXD_ODMAC_M			(0x3 << HNS3_RXD_ODMAC_S)
+#define HNS3_RXD_OVLAN_S			2
+#define HNS3_RXD_OVLAN_M			(0x3 << HNS3_RXD_OVLAN_S)
+#define HNS3_RXD_OL3ID_S			4
+#define HNS3_RXD_OL3ID_M			(0xf << HNS3_RXD_OL3ID_S)
+#define HNS3_RXD_OL4ID_S			8
+#define HNS3_RXD_OL4ID_M			(0xf << HNS3_RXD_OL4ID_S)
+#define HNS3_RXD_FBHI_S				12
+#define HNS3_RXD_FBHI_M				(0x3 << HNS3_RXD_FBHI_S)
+#define HNS3_RXD_FBLI_S				14
+#define HNS3_RXD_FBLI_M				(0x3 << HNS3_RXD_FBLI_S)
+
+#define HNS3_RXD_BDTYPE_S			0
+#define HNS3_RXD_BDTYPE_M			(0xf << HNS3_RXD_BDTYPE_S)
+#define HNS3_RXD_VLD_B				4
+#define HNS3_RXD_UDP0_B				5
+#define HNS3_RXD_EXTEND_B			7
+#define HNS3_RXD_FE_B				8
+#define HNS3_RXD_LUM_B				9
+#define HNS3_RXD_CRCP_B				10
+#define HNS3_RXD_L3L4P_B			11
+#define HNS3_RXD_TSIND_S			12
+#define HNS3_RXD_TSIND_M			(0x7 << HNS3_RXD_TSIND_S)
+#define HNS3_RXD_LKBK_B				15
+#define HNS3_RXD_HDL_S				16
+#define HNS3_RXD_HDL_M				(0x7ff << HNS3_RXD_HDL_S)
+#define HNS3_RXD_HSIND_B			31
+
+#define HNS3_TXD_L3T_S				0
+#define HNS3_TXD_L3T_M				(0x3 << HNS3_TXD_L3T_S)
+#define HNS3_TXD_L4T_S				2
+#define HNS3_TXD_L4T_M				(0x3 << HNS3_TXD_L4T_S)
+#define HNS3_TXD_L3CS_B				4
+#define HNS3_TXD_L4CS_B				5
+#define HNS3_TXD_VLAN_B				6
+#define HNS3_TXD_TSO_B				7
+
+#define HNS3_TXD_L2LEN_S			8
+#define HNS3_TXD_L2LEN_M			(0xff << HNS3_TXD_L2LEN_S)
+#define HNS3_TXD_L3LEN_S			16
+#define HNS3_TXD_L3LEN_M			(0xff << HNS3_TXD_L3LEN_S)
+#define HNS3_TXD_L4LEN_S			24
+#define HNS3_TXD_L4LEN_M			(0xff << HNS3_TXD_L4LEN_S)
+
+#define HNS3_TXD_OL3T_S				0
+#define HNS3_TXD_OL3T_M				(0x3 << HNS3_TXD_OL3T_S)
+#define HNS3_TXD_OVLAN_B			2
+#define HNS3_TXD_MACSEC_B			3
+#define HNS3_TXD_TUNTYPE_S			4
+#define HNS3_TXD_TUNTYPE_M			(0xf << HNS3_TXD_TUNTYPE_S)
+
+#define HNS3_TXD_BDTYPE_S			0
+#define HNS3_TXD_BDTYPE_M			(0xf << HNS3_TXD_BDTYPE_S)
+#define HNS3_TXD_FE_B				4
+#define HNS3_TXD_SC_S				5
+#define HNS3_TXD_SC_M				(0x3 << HNS3_TXD_SC_S)
+#define HNS3_TXD_EXTEND_B			7
+#define HNS3_TXD_VLD_B				8
+#define HNS3_TXD_RI_B				9
+#define HNS3_TXD_RA_B				10
+#define HNS3_TXD_TSYN_B				11
+#define HNS3_TXD_DECTTL_S			12
+#define HNS3_TXD_DECTTL_M			(0xf << HNS3_TXD_DECTTL_S)
+
+#define HNS3_TXD_MSS_S				0
+#define HNS3_TXD_MSS_M				(0x3fff << HNS3_TXD_MSS_S)
+
+#define HNS3_VECTOR_TX_IRQ			BIT_ULL(0)
+#define HNS3_VECTOR_RX_IRQ			BIT_ULL(1)
+
+#define HNS3_VECTOR_NOT_INITED			0
+#define HNS3_VECTOR_INITED			1
+
+#define HNS3_MAX_BD_SIZE			65535
+#define HNS3_MAX_BD_PER_FRAG			8
+#define HNS3_MAX_BD_PER_PKT			MAX_SKB_FRAGS
+
+#define HNS3_VECTOR_GL0_OFFSET			0x100
+#define HNS3_VECTOR_GL1_OFFSET			0x200
+#define HNS3_VECTOR_GL2_OFFSET			0x300
+#define HNS3_VECTOR_RL_OFFSET			0x900
+#define HNS3_VECTOR_RL_EN_B			6
+
+enum hns3_pkt_l3t_type {
+	HNS3_L3T_NONE,
+	HNS3_L3T_IPV6,
+	HNS3_L3T_IPV4,
+	HNS3_L3T_RESERVED
+};
+
+enum hns3_pkt_l4t_type {
+	HNS3_L4T_UNKNOWN,
+	HNS3_L4T_TCP,
+	HNS3_L4T_UDP,
+	HNS3_L4T_SCTP
+};
+
+enum hns3_pkt_ol3t_type {
+	HNS3_OL3T_NONE,
+	HNS3_OL3T_IPV6,
+	HNS3_OL3T_IPV4_NO_CSUM,
+	HNS3_OL3T_IPV4_CSUM
+};
+
+enum hns3_pkt_tun_type {
+	HNS3_TUN_NONE,
+	HNS3_TUN_MAC_IN_UDP,
+	HNS3_TUN_NVGRE,
+	HNS3_TUN_OTHER
+};
+
+/* hardware spec ring buffer format */
+struct __packed hns3_desc {
+	__le64 addr;
+	union {
+		struct {
+			__le16 vlan_tag;
+			__le16 send_size;
+			union {
+				__le32 type_cs_vlan_tso_len;
+				struct {
+					__u8 type_cs_vlan_tso;
+					__u8 l2_len;
+					__u8 l3_len;
+					__u8 l4_len;
+				};
+			};
+			__le16 outer_vlan_tag;
+			__le16 tv;
+
+		union {
+			__le32 ol_type_vlan_len_msec;
+			struct {
+				__u8 ol_type_vlan_msec;
+				__u8 ol2_len;
+				__u8 ol3_len;
+				__u8 ol4_len;
+			};
+		};
+
+			__le32 paylen;
+			__le16 bdtp_fe_sc_vld_ra_ri;
+			__le16 mss;
+		} tx;
+
+		struct {
+			__le32 l234_info;
+			__le16 pkt_len;
+			__le16 size;
+
+			__le32 rss_hash;
+			__le16 fd_id;
+			__le16 vlan_tag;
+
+			union {
+				__le32 ol_info;
+				struct {
+					__le16 o_dm_vlan_id_fb;
+					__le16 ot_vlan_tag;
+				};
+			};
+
+			__le32 bd_base_info;
+		} rx;
+	};
+};
+
+struct hns3_desc_cb {
+	dma_addr_t dma; /* dma address of this desc */
+	void *buf;      /* cpu addr for a desc */
+
+	/* priv data for the desc, e.g. skb when use with ip stack*/
+	void *priv;
+	u16 page_offset;
+	u16 reuse_flag;
+
+	u16 length;     /* length of the buffer */
+
+       /* desc type, used by the ring user to mark the type of the priv data */
+	u16 type;
+};
+
+enum hns3_pkt_l3type {
+	HNS3_L3_TYPE_IPV4,
+	HNS3_L3_TYPE_IPV6,
+	HNS3_L3_TYPE_ARP,
+	HNS3_L3_TYPE_RARP,
+	HNS3_L3_TYPE_IPV4_OPT,
+	HNS3_L3_TYPE_IPV6_EXT,
+	HNS3_L3_TYPE_LLDP,
+	HNS3_L3_TYPE_BPDU,
+	HNS3_L3_TYPE_MAC_PAUSE,
+	HNS3_L3_TYPE_PFC_PAUSE,/* 0x9*/
+
+	/* reserved for 0xA~0xB*/
+
+	HNS3_L3_TYPE_CNM = 0xc,
+
+	/* reserved for 0xD~0xE*/
+
+	HNS3_L3_TYPE_PARSE_FAIL	= 0xf /* must be last */
+};
+
+enum hns3_pkt_l4type {
+	HNS3_L4_TYPE_UDP,
+	HNS3_L4_TYPE_TCP,
+	HNS3_L4_TYPE_GRE,
+	HNS3_L4_TYPE_SCTP,
+	HNS3_L4_TYPE_IGMP,
+	HNS3_L4_TYPE_ICMP,
+
+	/* reserved for 0x6~0xE */
+
+	HNS3_L4_TYPE_PARSE_FAIL	= 0xf /* must be last */
+};
+
+enum hns3_pkt_ol3type {
+	HNS3_OL3_TYPE_IPV4 = 0,
+	HNS3_OL3_TYPE_IPV6,
+	/* reserved for 0x2~0x3 */
+	HNS3_OL3_TYPE_IPV4_OPT = 4,
+	HNS3_OL3_TYPE_IPV6_EXT,
+
+	/* reserved for 0x6~0xE*/
+
+	HNS3_OL3_TYPE_PARSE_FAIL = 0xf	/* must be last */
+};
+
+enum hns3_pkt_ol4type {
+	HNS3_OL4_TYPE_NO_TUN,
+	HNS3_OL4_TYPE_MAC_IN_UDP,
+	HNS3_OL4_TYPE_NVGRE,
+	HNS3_OL4_TYPE_UNKNOWN
+};
+
+struct ring_stats {
+	u64 io_err_cnt;
+	u64 sw_err_cnt;
+	u64 seg_pkt_cnt;
+	union {
+		struct {
+			u64 tx_pkts;
+			u64 tx_bytes;
+			u64 tx_err_cnt;
+			u64 restart_queue;
+			u64 tx_busy;
+		};
+		struct {
+			u64 rx_pkts;
+			u64 rx_bytes;
+			u64 rx_err_cnt;
+			u64 reuse_pg_cnt;
+			u64 err_pkt_len;
+			u64 non_vld_descs;
+			u64 err_bd_num;
+			u64 l2_err;
+			u64 l3l4_csum_err;
+		};
+	};
+};
+
+struct hns3_enet_ring {
+	u8 __iomem *io_base; /* base io address for the ring */
+	struct hns3_desc *desc; /* dma map address space */
+	struct hns3_desc_cb *desc_cb;
+	struct hns3_enet_ring *next;
+	struct hns3_enet_tqp_vector *tqp_vector;
+	struct hnae3_queue *tqp;
+	char ring_name[HNS3_RING_NAME_LEN];
+	struct device *dev; /* will be used for DMA mapping of descriptors */
+
+	/* statistic */
+	struct ring_stats stats;
+	struct u64_stats_sync syncp;
+
+	dma_addr_t desc_dma_addr;
+	u32 buf_size;       /* size for hnae_desc->addr, preset by AE */
+	u16 desc_num;       /* total number of desc */
+	u16 max_desc_num_per_pkt;
+	u16 max_raw_data_sz_per_desc;
+	u16 max_pkt_size;
+	int next_to_use;    /* idx of next spare desc */
+
+	/* idx of lastest sent desc, the ring is empty when equal to
+	 * next_to_use
+	 */
+	int next_to_clean;
+
+	u32 flag;          /* ring attribute */
+	int irq_init_flag;
+
+	int numa_node;
+	cpumask_t affinity_mask;
+};
+
+struct hns_queue;
+
+struct hns3_nic_ring_data {
+	struct hns3_enet_ring *ring;
+	struct napi_struct napi;
+	int queue_index;
+	int (*poll_one)(struct hns3_nic_ring_data *, int, void *);
+	void (*ex_process)(struct hns3_nic_ring_data *, struct sk_buff *);
+	void (*fini_process)(struct hns3_nic_ring_data *);
+};
+
+struct hns3_nic_ops {
+	int (*fill_desc)(struct hns3_enet_ring *ring, void *priv,
+			 int size, dma_addr_t dma, int frag_end,
+			 enum hns_desc_type type);
+	int (*maybe_stop_tx)(struct sk_buff **out_skb,
+			     int *bnum, struct hns3_enet_ring *ring);
+	void (*get_rxd_bnum)(u32 bnum_flag, int *out_bnum);
+};
+
+enum hns3_flow_level_range {
+	HNS3_FLOW_LOW = 0,
+	HNS3_FLOW_MID = 1,
+	HNS3_FLOW_HIGH = 2,
+	HNS3_FLOW_ULTRA = 3,
+};
+
+enum hns3_link_mode_bits {
+	HNS3_LM_FIBRE_BIT = BIT(0),
+	HNS3_LM_AUTONEG_BIT = BIT(1),
+	HNS3_LM_TP_BIT = BIT(2),
+	HNS3_LM_PAUSE_BIT = BIT(3),
+	HNS3_LM_BACKPLANE_BIT = BIT(4),
+	HNS3_LM_10BASET_HALF_BIT = BIT(5),
+	HNS3_LM_10BASET_FULL_BIT = BIT(6),
+	HNS3_LM_100BASET_HALF_BIT = BIT(7),
+	HNS3_LM_100BASET_FULL_BIT = BIT(8),
+	HNS3_LM_1000BASET_FULL_BIT = BIT(9),
+	HNS3_LM_10000BASEKR_FULL_BIT = BIT(10),
+	HNS3_LM_25000BASEKR_FULL_BIT = BIT(11),
+	HNS3_LM_40000BASELR4_FULL_BIT = BIT(12),
+	HNS3_LM_50000BASEKR2_FULL_BIT = BIT(13),
+	HNS3_LM_100000BASEKR4_FULL_BIT = BIT(14),
+	HNS3_LM_COUNT = 15
+};
+
+#define HNS3_INT_GL_50K		0x000A
+#define HNS3_INT_GL_20K		0x0019
+#define HNS3_INT_GL_18K		0x001B
+#define HNS3_INT_GL_8K		0x003E
+
+struct hns3_enet_ring_group {
+	/* array of pointers to rings */
+	struct hns3_enet_ring *ring;
+	u64 total_bytes;	/* total bytes processed this group */
+	u64 total_packets;	/* total packets processed this group */
+	u16 count;
+	enum hns3_flow_level_range flow_level;
+	u16 int_gl;
+};
+
+struct hns3_enet_tqp_vector {
+	struct hnae3_handle *handle;
+	u8 __iomem *mask_addr;
+	int vector_irq;
+	int irq_init_flag;
+
+	u16 idx;		/* index in the TQP vector array per handle. */
+
+	struct napi_struct napi;
+
+	struct hns3_enet_ring_group rx_group;
+	struct hns3_enet_ring_group tx_group;
+
+	u16 num_tqps;	/* total number of tqps in TQP vector */
+
+	cpumask_t affinity_mask;
+	char name[HNAE3_INT_NAME_LEN];
+
+	/* when 0 should adjust interrupt coalesce parameter */
+	u8 int_adapt_down;
+} ____cacheline_internodealigned_in_smp;
+
+enum hns3_udp_tnl_type {
+	HNS3_UDP_TNL_VXLAN,
+	HNS3_UDP_TNL_GENEVE,
+	HNS3_UDP_TNL_MAX,
+};
+
+struct hns3_udp_tunnel {
+	u16 dst_port;
+	int used;
+};
+
+struct hns3_nic_priv {
+	struct hnae3_handle *ae_handle;
+	u32 enet_ver;
+	u32 port_id;
+	struct net_device *netdev;
+	struct device *dev;
+	struct hns3_nic_ops ops;
+
+	/**
+	 * the cb for nic to manage the ring buffer, the first half of the
+	 * array is for tx_ring and vice versa for the second half
+	 */
+	struct hns3_nic_ring_data *ring_data;
+	struct hns3_enet_tqp_vector *tqp_vector;
+	u16 vector_num;
+
+	/* The most recently read link state */
+	int link;
+	u64 tx_timeout_count;
+
+	unsigned long state;
+
+	struct timer_list service_timer;
+
+	struct work_struct service_task;
+
+	struct notifier_block notifier_block;
+	/* Vxlan/Geneve information */
+	struct hns3_udp_tunnel udp_tnl[HNS3_UDP_TNL_MAX];
+};
+
+union l3_hdr_info {
+	struct iphdr *v4;
+	struct ipv6hdr *v6;
+	unsigned char *hdr;
+};
+
+union l4_hdr_info {
+	struct tcphdr *tcp;
+	struct udphdr *udp;
+	unsigned char *hdr;
+};
+
+/* the distance between [begin, end) in a ring buffer
+ * note: there is a unuse slot between the begin and the end
+ */
+static inline int ring_dist(struct hns3_enet_ring *ring, int begin, int end)
+{
+	return (end - begin + ring->desc_num) % ring->desc_num;
+}
+
+static inline int ring_space(struct hns3_enet_ring *ring)
+{
+	return ring->desc_num -
+		ring_dist(ring, ring->next_to_clean, ring->next_to_use) - 1;
+}
+
+static inline int is_ring_empty(struct hns3_enet_ring *ring)
+{
+	return ring->next_to_use == ring->next_to_clean;
+}
+
+static inline void hns3_write_reg(void __iomem *base, u32 reg, u32 value)
+{
+	u8 __iomem *reg_addr = READ_ONCE(base);
+
+	writel(value, reg_addr + reg);
+}
+
+#define hns3_write_dev(a, reg, value) \
+	hns3_write_reg((a)->io_base, (reg), (value))
+
+#define hnae_queue_xmit(tqp, buf_num) writel_relaxed(buf_num, \
+		(tqp)->io_base + HNS3_RING_TX_RING_TAIL_REG)
+
+#define ring_to_dev(ring) (&(ring)->tqp->handle->pdev->dev)
+
+#define ring_to_dma_dir(ring) (HNAE3_IS_TX_RING(ring) ? \
+	DMA_TO_DEVICE : DMA_FROM_DEVICE)
+
+#define tx_ring_data(priv, idx) ((priv)->ring_data[idx])
+
+#define hnae_buf_size(_ring) ((_ring)->buf_size)
+#define hnae_page_order(_ring) (get_order(hnae_buf_size(_ring)))
+#define hnae_page_size(_ring) (PAGE_SIZE << hnae_page_order(_ring))
+
+/* iterator for handling rings in ring group */
+#define hns3_for_each_ring(pos, head) \
+	for (pos = (head).ring; pos; pos = pos->next)
+
+void hns3_ethtool_set_ops(struct net_device *netdev);
+
+int hns3_clean_tx_ring(struct hns3_enet_ring *ring, int budget);
+#endif
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hns3_ethtool.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hns3_ethtool.c
new file mode 100644
index 000000000000..d636399232fb
--- /dev/null
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hns3_ethtool.c
@@ -0,0 +1,493 @@
+/*
+ * Copyright (c) 2016~2017 Hisilicon Limited.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/etherdevice.h>
+#include <linux/string.h>
+
+#include "hns3_enet.h"
+
+struct hns3_stats {
+	char stats_string[ETH_GSTRING_LEN];
+	int stats_size;
+	int stats_offset;
+};
+
+/* tqp related stats */
+#define HNS3_TQP_STAT(_string, _member)	{			\
+	.stats_string = _string,				\
+	.stats_size = FIELD_SIZEOF(struct ring_stats, _member),	\
+	.stats_offset = offsetof(struct hns3_enet_ring, stats),	\
+}								\
+
+static const struct hns3_stats hns3_txq_stats[] = {
+	/* Tx per-queue statistics */
+	HNS3_TQP_STAT("tx_io_err_cnt", io_err_cnt),
+	HNS3_TQP_STAT("tx_sw_err_cnt", sw_err_cnt),
+	HNS3_TQP_STAT("tx_seg_pkt_cnt", seg_pkt_cnt),
+	HNS3_TQP_STAT("tx_pkts", tx_pkts),
+	HNS3_TQP_STAT("tx_bytes", tx_bytes),
+	HNS3_TQP_STAT("tx_err_cnt", tx_err_cnt),
+	HNS3_TQP_STAT("tx_restart_queue", restart_queue),
+	HNS3_TQP_STAT("tx_busy", tx_busy),
+};
+
+#define HNS3_TXQ_STATS_COUNT ARRAY_SIZE(hns3_txq_stats)
+
+static const struct hns3_stats hns3_rxq_stats[] = {
+	/* Rx per-queue statistics */
+	HNS3_TQP_STAT("rx_io_err_cnt", io_err_cnt),
+	HNS3_TQP_STAT("rx_sw_err_cnt", sw_err_cnt),
+	HNS3_TQP_STAT("rx_seg_pkt_cnt", seg_pkt_cnt),
+	HNS3_TQP_STAT("rx_pkts", rx_pkts),
+	HNS3_TQP_STAT("rx_bytes", rx_bytes),
+	HNS3_TQP_STAT("rx_err_cnt", rx_err_cnt),
+	HNS3_TQP_STAT("rx_reuse_pg_cnt", reuse_pg_cnt),
+	HNS3_TQP_STAT("rx_err_pkt_len", err_pkt_len),
+	HNS3_TQP_STAT("rx_non_vld_descs", non_vld_descs),
+	HNS3_TQP_STAT("rx_err_bd_num", err_bd_num),
+	HNS3_TQP_STAT("rx_l2_err", l2_err),
+	HNS3_TQP_STAT("rx_l3l4_csum_err", l3l4_csum_err),
+};
+
+#define HNS3_RXQ_STATS_COUNT ARRAY_SIZE(hns3_rxq_stats)
+
+#define HNS3_TQP_STATS_COUNT (HNS3_TXQ_STATS_COUNT + HNS3_RXQ_STATS_COUNT)
+
+struct hns3_link_mode_mapping {
+	u32 hns3_link_mode;
+	u32 ethtool_link_mode;
+};
+
+static const struct hns3_link_mode_mapping hns3_lm_map[] = {
+	{HNS3_LM_FIBRE_BIT, ETHTOOL_LINK_MODE_FIBRE_BIT},
+	{HNS3_LM_AUTONEG_BIT, ETHTOOL_LINK_MODE_Autoneg_BIT},
+	{HNS3_LM_TP_BIT, ETHTOOL_LINK_MODE_TP_BIT},
+	{HNS3_LM_PAUSE_BIT, ETHTOOL_LINK_MODE_Pause_BIT},
+	{HNS3_LM_BACKPLANE_BIT, ETHTOOL_LINK_MODE_Backplane_BIT},
+	{HNS3_LM_10BASET_HALF_BIT, ETHTOOL_LINK_MODE_10baseT_Half_BIT},
+	{HNS3_LM_10BASET_FULL_BIT, ETHTOOL_LINK_MODE_10baseT_Full_BIT},
+	{HNS3_LM_100BASET_HALF_BIT, ETHTOOL_LINK_MODE_100baseT_Half_BIT},
+	{HNS3_LM_100BASET_FULL_BIT, ETHTOOL_LINK_MODE_100baseT_Full_BIT},
+	{HNS3_LM_1000BASET_FULL_BIT, ETHTOOL_LINK_MODE_1000baseT_Full_BIT},
+};
+
+static void hns3_driv_to_eth_caps(u32 caps, struct ethtool_link_ksettings *cmd,
+				  bool is_advertised)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(hns3_lm_map); i++) {
+		if (!(caps & hns3_lm_map[i].hns3_link_mode))
+			continue;
+
+		if (is_advertised) {
+			ethtool_link_ksettings_zero_link_mode(cmd,
+							      advertising);
+			__set_bit(hns3_lm_map[i].ethtool_link_mode,
+				  cmd->link_modes.advertising);
+		} else {
+			ethtool_link_ksettings_zero_link_mode(cmd,
+							      supported);
+			__set_bit(hns3_lm_map[i].ethtool_link_mode,
+				  cmd->link_modes.supported);
+		}
+	}
+}
+
+static int hns3_get_sset_count(struct net_device *netdev, int stringset)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+	const struct hnae3_ae_ops *ops = h->ae_algo->ops;
+
+	if (!ops->get_sset_count)
+		return -EOPNOTSUPP;
+
+	switch (stringset) {
+	case ETH_SS_STATS:
+		return ((HNS3_TQP_STATS_COUNT * h->kinfo.num_tqps) +
+			ops->get_sset_count(h, stringset));
+
+	case ETH_SS_TEST:
+		return ops->get_sset_count(h, stringset);
+	}
+
+	return 0;
+}
+
+static void *hns3_update_strings(u8 *data, const struct hns3_stats *stats,
+				 u32 stat_count, u32 num_tqps)
+{
+#define MAX_PREFIX_SIZE (8 + 4)
+	u32 size_left;
+	u32 i, j;
+	u32 n1;
+
+	for (i = 0; i < num_tqps; i++) {
+		for (j = 0; j < stat_count; j++) {
+			data[ETH_GSTRING_LEN - 1] = '\0';
+
+			/* first, prepend the prefix string */
+			n1 = snprintf(data, MAX_PREFIX_SIZE, "rcb_q%d_", i);
+			n1 = min_t(uint, n1, MAX_PREFIX_SIZE - 1);
+			size_left = (ETH_GSTRING_LEN - 1) - n1;
+
+			/* now, concatenate the stats string to it */
+			strncat(data, stats[j].stats_string, size_left);
+			data += ETH_GSTRING_LEN;
+		}
+	}
+
+	return data;
+}
+
+static u8 *hns3_get_strings_tqps(struct hnae3_handle *handle, u8 *data)
+{
+	struct hnae3_knic_private_info *kinfo = &handle->kinfo;
+
+	/* get strings for Tx */
+	data = hns3_update_strings(data, hns3_txq_stats, HNS3_TXQ_STATS_COUNT,
+				   kinfo->num_tqps);
+
+	/* get strings for Rx */
+	data = hns3_update_strings(data, hns3_rxq_stats, HNS3_RXQ_STATS_COUNT,
+				   kinfo->num_tqps);
+
+	return data;
+}
+
+static void hns3_get_strings(struct net_device *netdev, u32 stringset, u8 *data)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+	const struct hnae3_ae_ops *ops = h->ae_algo->ops;
+	char *buff = (char *)data;
+
+	if (!ops->get_strings)
+		return;
+
+	switch (stringset) {
+	case ETH_SS_STATS:
+		buff = hns3_get_strings_tqps(h, buff);
+		h->ae_algo->ops->get_strings(h, stringset, (u8 *)buff);
+		break;
+	case ETH_SS_TEST:
+		ops->get_strings(h, stringset, data);
+		break;
+	}
+}
+
+static u64 *hns3_get_stats_tqps(struct hnae3_handle *handle, u64 *data)
+{
+	struct hns3_nic_priv *nic_priv = (struct hns3_nic_priv *)handle->priv;
+	struct hnae3_knic_private_info *kinfo = &handle->kinfo;
+	struct hns3_enet_ring *ring;
+	u8 *stat;
+	u32 i;
+
+	/* get stats for Tx */
+	for (i = 0; i < kinfo->num_tqps; i++) {
+		ring = nic_priv->ring_data[i].ring;
+		for (i = 0; i < HNS3_TXQ_STATS_COUNT; i++) {
+			stat = (u8 *)ring + hns3_txq_stats[i].stats_offset;
+			*data++ = *(u64 *)stat;
+		}
+	}
+
+	/* get stats for Rx */
+	for (i = 0; i < kinfo->num_tqps; i++) {
+		ring = nic_priv->ring_data[i + kinfo->num_tqps].ring;
+		for (i = 0; i < HNS3_RXQ_STATS_COUNT; i++) {
+			stat = (u8 *)ring + hns3_rxq_stats[i].stats_offset;
+			*data++ = *(u64 *)stat;
+		}
+	}
+
+	return data;
+}
+
+/* hns3_get_stats - get detail statistics.
+ * @netdev: net device
+ * @stats: statistics info.
+ * @data: statistics data.
+ */
+void hns3_get_stats(struct net_device *netdev, struct ethtool_stats *stats,
+		    u64 *data)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+	u64 *p = data;
+
+	if (!h->ae_algo->ops->get_stats || !h->ae_algo->ops->update_stats) {
+		netdev_err(netdev, "could not get any statistics\n");
+		return;
+	}
+
+	h->ae_algo->ops->update_stats(h, &netdev->stats);
+
+	/* get per-queue stats */
+	p = hns3_get_stats_tqps(h, p);
+
+	/* get MAC & other misc hardware stats */
+	h->ae_algo->ops->get_stats(h, p);
+}
+
+static void hns3_get_drvinfo(struct net_device *netdev,
+			     struct ethtool_drvinfo *drvinfo)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+
+	strncpy(drvinfo->version, hns3_driver_version,
+		sizeof(drvinfo->version));
+	drvinfo->version[sizeof(drvinfo->version) - 1] = '\0';
+
+	strncpy(drvinfo->driver, h->pdev->driver->name,
+		sizeof(drvinfo->driver));
+	drvinfo->driver[sizeof(drvinfo->driver) - 1] = '\0';
+
+	strncpy(drvinfo->bus_info, pci_name(h->pdev),
+		sizeof(drvinfo->bus_info));
+	drvinfo->bus_info[ETHTOOL_BUSINFO_LEN - 1] = '\0';
+
+	snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), "0x%08x",
+		 priv->ae_handle->ae_algo->ops->get_fw_version(h));
+}
+
+static u32 hns3_get_link(struct net_device *netdev)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h;
+
+	h = priv->ae_handle;
+
+	if (h->ae_algo && h->ae_algo->ops && h->ae_algo->ops->get_status)
+		return h->ae_algo->ops->get_status(h);
+	else
+		return 0;
+}
+
+static void hns3_get_ringparam(struct net_device *netdev,
+			       struct ethtool_ringparam *param)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	int queue_num = priv->ae_handle->kinfo.num_tqps;
+
+	param->tx_max_pending = HNS3_RING_MAX_PENDING;
+	param->rx_max_pending = HNS3_RING_MAX_PENDING;
+
+	param->tx_pending = priv->ring_data[0].ring->desc_num;
+	param->rx_pending = priv->ring_data[queue_num].ring->desc_num;
+}
+
+static void hns3_get_pauseparam(struct net_device *netdev,
+				struct ethtool_pauseparam *param)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+
+	if (h->ae_algo && h->ae_algo->ops && h->ae_algo->ops->get_pauseparam)
+		h->ae_algo->ops->get_pauseparam(h, &param->autoneg,
+			&param->rx_pause, &param->tx_pause);
+}
+
+static int hns3_get_link_ksettings(struct net_device *netdev,
+				   struct ethtool_link_ksettings *cmd)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+	u32 supported_caps;
+	u32 advertised_caps;
+	u8 media_type = HNAE3_MEDIA_TYPE_UNKNOWN;
+	u8 link_stat;
+	u8 auto_neg;
+	u8 duplex;
+	u32 speed;
+
+	if (!h->ae_algo || !h->ae_algo->ops)
+		return -EOPNOTSUPP;
+
+	/* 1.auto_neg & speed & duplex from cmd */
+	if (h->ae_algo->ops->get_ksettings_an_result) {
+		h->ae_algo->ops->get_ksettings_an_result(h, &auto_neg,
+							 &speed, &duplex);
+		cmd->base.autoneg = auto_neg;
+		cmd->base.speed = speed;
+		cmd->base.duplex = duplex;
+
+		link_stat = hns3_get_link(netdev);
+		if (!link_stat) {
+			cmd->base.speed = (u32)SPEED_UNKNOWN;
+			cmd->base.duplex = DUPLEX_UNKNOWN;
+		}
+	}
+
+	/* 2.media_type get from bios parameter block */
+	if (h->ae_algo->ops->get_media_type) {
+		h->ae_algo->ops->get_media_type(h, &media_type);
+
+		switch (media_type) {
+		case HNAE3_MEDIA_TYPE_FIBER:
+			cmd->base.port = PORT_FIBRE;
+			supported_caps = HNS3_LM_FIBRE_BIT |
+					 HNS3_LM_AUTONEG_BIT |
+					 HNS3_LM_PAUSE_BIT |
+					 HNS3_LM_1000BASET_FULL_BIT;
+
+			advertised_caps = supported_caps;
+			break;
+		case HNAE3_MEDIA_TYPE_COPPER:
+			cmd->base.port = PORT_TP;
+			supported_caps = HNS3_LM_TP_BIT |
+					 HNS3_LM_AUTONEG_BIT |
+					 HNS3_LM_PAUSE_BIT |
+					 HNS3_LM_1000BASET_FULL_BIT |
+					 HNS3_LM_100BASET_FULL_BIT |
+					 HNS3_LM_100BASET_HALF_BIT |
+					 HNS3_LM_10BASET_FULL_BIT |
+					 HNS3_LM_10BASET_HALF_BIT;
+			advertised_caps = supported_caps;
+			break;
+		case HNAE3_MEDIA_TYPE_BACKPLANE:
+			cmd->base.port = PORT_NONE;
+			supported_caps = HNS3_LM_BACKPLANE_BIT |
+					 HNS3_LM_PAUSE_BIT |
+					 HNS3_LM_AUTONEG_BIT |
+					 HNS3_LM_1000BASET_FULL_BIT |
+					 HNS3_LM_100BASET_FULL_BIT |
+					 HNS3_LM_100BASET_HALF_BIT |
+					 HNS3_LM_10BASET_FULL_BIT |
+					 HNS3_LM_10BASET_HALF_BIT;
+
+			advertised_caps = supported_caps;
+			break;
+		case HNAE3_MEDIA_TYPE_UNKNOWN:
+		default:
+			cmd->base.port = PORT_OTHER;
+			supported_caps = 0;
+			advertised_caps = 0;
+			break;
+		}
+
+		/* now, map driver link modes to ethtool link modes */
+		hns3_driv_to_eth_caps(supported_caps, cmd, false);
+		hns3_driv_to_eth_caps(advertised_caps, cmd, true);
+	}
+
+	/* 3.mdix_ctrl&mdix get from phy reg */
+	if (h->ae_algo->ops->get_mdix_mode)
+		h->ae_algo->ops->get_mdix_mode(h, &cmd->base.eth_tp_mdix_ctrl,
+					       &cmd->base.eth_tp_mdix);
+	/* 4.mdio_support */
+	cmd->base.mdio_support = ETH_MDIO_SUPPORTS_C22;
+
+	return 0;
+}
+
+static u32 hns3_get_rss_key_size(struct net_device *netdev)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+
+	if (!h->ae_algo || !h->ae_algo->ops ||
+	    !h->ae_algo->ops->get_rss_key_size)
+		return -EOPNOTSUPP;
+
+	return h->ae_algo->ops->get_rss_key_size(h);
+}
+
+static u32 hns3_get_rss_indir_size(struct net_device *netdev)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+
+	if (!h->ae_algo || !h->ae_algo->ops ||
+	    !h->ae_algo->ops->get_rss_indir_size)
+		return -EOPNOTSUPP;
+
+	return h->ae_algo->ops->get_rss_indir_size(h);
+}
+
+static int hns3_get_rss(struct net_device *netdev, u32 *indir, u8 *key,
+			u8 *hfunc)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+
+	if (!h->ae_algo || !h->ae_algo->ops || !h->ae_algo->ops->get_rss)
+		return -EOPNOTSUPP;
+
+	return h->ae_algo->ops->get_rss(h, indir, key, hfunc);
+}
+
+static int hns3_set_rss(struct net_device *netdev, const u32 *indir,
+			const u8 *key, const u8 hfunc)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+
+	if (!h->ae_algo || !h->ae_algo->ops || !h->ae_algo->ops->set_rss)
+		return -EOPNOTSUPP;
+
+	/* currently we only support Toeplitz hash */
+	if ((hfunc != ETH_RSS_HASH_NO_CHANGE) && (hfunc != ETH_RSS_HASH_TOP)) {
+		netdev_err(netdev,
+			   "hash func not supported (only Toeplitz hash)\n");
+		return -EOPNOTSUPP;
+	}
+	if (!indir) {
+		netdev_err(netdev,
+			   "set rss failed for indir is empty\n");
+		return -EOPNOTSUPP;
+	}
+
+	return h->ae_algo->ops->set_rss(h, indir, key, hfunc);
+}
+
+static int hns3_get_rxnfc(struct net_device *netdev,
+			  struct ethtool_rxnfc *cmd,
+			  u32 *rule_locs)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+
+	if (!h->ae_algo || !h->ae_algo->ops || !h->ae_algo->ops->get_tc_size)
+		return -EOPNOTSUPP;
+
+	switch (cmd->cmd) {
+	case ETHTOOL_GRXRINGS:
+		cmd->data = h->ae_algo->ops->get_tc_size(h);
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static const struct ethtool_ops hns3_ethtool_ops = {
+	.get_drvinfo = hns3_get_drvinfo,
+	.get_link = hns3_get_link,
+	.get_ringparam = hns3_get_ringparam,
+	.get_pauseparam = hns3_get_pauseparam,
+	.get_strings = hns3_get_strings,
+	.get_ethtool_stats = hns3_get_stats,
+	.get_sset_count = hns3_get_sset_count,
+	.get_rxnfc = hns3_get_rxnfc,
+	.get_rxfh_key_size = hns3_get_rss_key_size,
+	.get_rxfh_indir_size = hns3_get_rss_indir_size,
+	.get_rxfh = hns3_get_rss,
+	.set_rxfh = hns3_set_rss,
+	.get_link_ksettings = hns3_get_link_ksettings,
+};
+
+void hns3_ethtool_set_ops(struct net_device *netdev)
+{
+	netdev->ethtool_ops = &hns3_ethtool_ops;
+}
diff --git a/drivers/net/ethernet/hp/hp100.c b/drivers/net/ethernet/hp/hp100.c
index c6164a98f257..c8c7ad2eff77 100644
--- a/drivers/net/ethernet/hp/hp100.c
+++ b/drivers/net/ethernet/hp/hp100.c
@@ -194,7 +194,7 @@ static const char *hp100_isa_tbl[] = {
 };
 #endif
 
-static struct eisa_device_id hp100_eisa_tbl[] = {
+static const struct eisa_device_id hp100_eisa_tbl[] = {
 	{ "HWPF180" }, /* HP J2577 rev A */
 	{ "HWP1920" }, /* HP 27248B */
 	{ "HWP1940" }, /* HP J2577 */
diff --git a/drivers/net/ethernet/huawei/Kconfig b/drivers/net/ethernet/huawei/Kconfig
new file mode 100644
index 000000000000..c1a95ae4058b
--- /dev/null
+++ b/drivers/net/ethernet/huawei/Kconfig
@@ -0,0 +1,19 @@
+#
+# Huawei driver configuration
+#
+
+config NET_VENDOR_HUAWEI
+	bool "Huawei devices"
+	default y
+	---help---
+	  If you have a network (Ethernet) card belonging to this class, say Y.
+	  Note that the answer to this question doesn't directly affect the
+	  kernel: saying N will just cause the configurator to skip all
+	  the questions about Huawei cards. If you say Y, you will be asked
+	  for your specific card in the following questions.
+
+if NET_VENDOR_HUAWEI
+
+source "drivers/net/ethernet/huawei/hinic/Kconfig"
+
+endif # NET_VENDOR_HUAWEI
diff --git a/drivers/net/ethernet/huawei/Makefile b/drivers/net/ethernet/huawei/Makefile
new file mode 100644
index 000000000000..5c37cc8fc1bc
--- /dev/null
+++ b/drivers/net/ethernet/huawei/Makefile
@@ -0,0 +1,5 @@
+#
+# Makefile for the Huawei device drivers.
+#
+
+obj-$(CONFIG_HINIC) += hinic/
diff --git a/drivers/net/ethernet/huawei/hinic/Kconfig b/drivers/net/ethernet/huawei/hinic/Kconfig
new file mode 100644
index 000000000000..08db24954f7e
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/Kconfig
@@ -0,0 +1,12 @@
+#
+# Huawei driver configuration
+#
+
+config HINIC
+	tristate "Huawei Intelligent PCIE Network Interface Card"
+	depends on (PCI_MSI && X86)
+	---help---
+	  This driver supports HiNIC PCIE Ethernet cards.
+	  To compile this driver as part of the kernel, choose Y here.
+	  If unsure, choose N.
+	  The default is compiled as module.
diff --git a/drivers/net/ethernet/huawei/hinic/Makefile b/drivers/net/ethernet/huawei/hinic/Makefile
new file mode 100644
index 000000000000..289ce88bb2d0
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/Makefile
@@ -0,0 +1,6 @@
+obj-$(CONFIG_HINIC) += hinic.o
+
+hinic-y := hinic_main.o hinic_tx.o hinic_rx.o hinic_port.o hinic_hw_dev.o \
+	   hinic_hw_io.o hinic_hw_qp.o hinic_hw_cmdq.o hinic_hw_wq.o \
+	   hinic_hw_mgmt.o hinic_hw_api_cmd.o hinic_hw_eqs.o hinic_hw_if.o \
+	   hinic_common.o
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_common.c b/drivers/net/ethernet/huawei/hinic/hinic_common.c
new file mode 100644
index 000000000000..02c74fd8380e
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_common.c
@@ -0,0 +1,80 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <asm/byteorder.h>
+
+#include "hinic_common.h"
+
+/**
+ * hinic_cpu_to_be32 - convert data to big endian 32 bit format
+ * @data: the data to convert
+ * @len: length of data to convert
+ **/
+void hinic_cpu_to_be32(void *data, int len)
+{
+	u32 *mem = data;
+	int i;
+
+	len = len / sizeof(u32);
+
+	for (i = 0; i < len; i++) {
+		*mem = cpu_to_be32(*mem);
+		mem++;
+	}
+}
+
+/**
+ * hinic_be32_to_cpu - convert data from big endian 32 bit format
+ * @data: the data to convert
+ * @len: length of data to convert
+ **/
+void hinic_be32_to_cpu(void *data, int len)
+{
+	u32 *mem = data;
+	int i;
+
+	len = len / sizeof(u32);
+
+	for (i = 0; i < len; i++) {
+		*mem = be32_to_cpu(*mem);
+		mem++;
+	}
+}
+
+/**
+ * hinic_set_sge - set dma area in scatter gather entry
+ * @sge: scatter gather entry
+ * @addr: dma address
+ * @len: length of relevant data in the dma address
+ **/
+void hinic_set_sge(struct hinic_sge *sge, dma_addr_t addr, int len)
+{
+	sge->hi_addr = upper_32_bits(addr);
+	sge->lo_addr = lower_32_bits(addr);
+	sge->len  = len;
+}
+
+/**
+ * hinic_sge_to_dma - get dma address from scatter gather entry
+ * @sge: scatter gather entry
+ *
+ * Return dma address of sg entry
+ **/
+dma_addr_t hinic_sge_to_dma(struct hinic_sge *sge)
+{
+	return (dma_addr_t)((((u64)sge->hi_addr) << 32) | sge->lo_addr);
+}
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_common.h b/drivers/net/ethernet/huawei/hinic/hinic_common.h
new file mode 100644
index 000000000000..2c06b76e94a1
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_common.h
@@ -0,0 +1,38 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#ifndef HINIC_COMMON_H
+#define HINIC_COMMON_H
+
+#include <linux/types.h>
+
+#define UPPER_8_BITS(data)      (((data) >> 8) & 0xFF)
+#define LOWER_8_BITS(data)      ((data) & 0xFF)
+
+struct hinic_sge {
+	u32             hi_addr;
+	u32             lo_addr;
+	u32             len;
+};
+
+void hinic_cpu_to_be32(void *data, int len);
+
+void hinic_be32_to_cpu(void *data, int len);
+
+void hinic_set_sge(struct hinic_sge *sge, dma_addr_t addr, int len);
+
+dma_addr_t hinic_sge_to_dma(struct hinic_sge *sge);
+
+#endif
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_dev.h b/drivers/net/ethernet/huawei/hinic/hinic_dev.h
new file mode 100644
index 000000000000..5186cc9023aa
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_dev.h
@@ -0,0 +1,64 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#ifndef HINIC_DEV_H
+#define HINIC_DEV_H
+
+#include <linux/netdevice.h>
+#include <linux/types.h>
+#include <linux/semaphore.h>
+#include <linux/workqueue.h>
+#include <linux/bitops.h>
+
+#include "hinic_hw_dev.h"
+#include "hinic_tx.h"
+#include "hinic_rx.h"
+
+#define HINIC_DRV_NAME          "hinic"
+
+enum hinic_flags {
+	HINIC_LINK_UP = BIT(0),
+	HINIC_INTF_UP = BIT(1),
+};
+
+struct hinic_rx_mode_work {
+	struct work_struct      work;
+	u32                     rx_mode;
+};
+
+struct hinic_dev {
+	struct net_device               *netdev;
+	struct hinic_hwdev              *hwdev;
+
+	u32                             msg_enable;
+	unsigned int                    tx_weight;
+	unsigned int                    rx_weight;
+
+	unsigned int                    flags;
+
+	struct semaphore                mgmt_lock;
+	unsigned long                   *vlan_bitmap;
+
+	struct hinic_rx_mode_work       rx_mode_work;
+	struct workqueue_struct         *workq;
+
+	struct hinic_txq                *txqs;
+	struct hinic_rxq                *rxqs;
+
+	struct hinic_txq_stats          tx_stats;
+	struct hinic_rxq_stats          rx_stats;
+};
+
+#endif
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_api_cmd.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_api_cmd.c
new file mode 100644
index 000000000000..c40603a183df
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_api_cmd.c
@@ -0,0 +1,978 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/errno.h>
+#include <linux/pci.h>
+#include <linux/device.h>
+#include <linux/slab.h>
+#include <linux/dma-mapping.h>
+#include <linux/bitops.h>
+#include <linux/err.h>
+#include <linux/jiffies.h>
+#include <linux/delay.h>
+#include <linux/log2.h>
+#include <linux/semaphore.h>
+#include <asm/byteorder.h>
+#include <asm/barrier.h>
+
+#include "hinic_hw_csr.h"
+#include "hinic_hw_if.h"
+#include "hinic_hw_api_cmd.h"
+
+#define API_CHAIN_NUM_CELLS                     32
+
+#define API_CMD_CELL_SIZE_SHIFT                 6
+#define API_CMD_CELL_SIZE_MIN                   (BIT(API_CMD_CELL_SIZE_SHIFT))
+
+#define API_CMD_CELL_SIZE(cell_size)            \
+		(((cell_size) >= API_CMD_CELL_SIZE_MIN) ? \
+		 (1 << (fls(cell_size - 1))) : API_CMD_CELL_SIZE_MIN)
+
+#define API_CMD_CELL_SIZE_VAL(size)             \
+		ilog2((size) >> API_CMD_CELL_SIZE_SHIFT)
+
+#define API_CMD_BUF_SIZE                        2048
+
+/* Sizes of the members in hinic_api_cmd_cell */
+#define API_CMD_CELL_DESC_SIZE          8
+#define API_CMD_CELL_DATA_ADDR_SIZE     8
+
+#define API_CMD_CELL_ALIGNMENT          8
+
+#define API_CMD_TIMEOUT                 1000
+
+#define MASKED_IDX(chain, idx)          ((idx) & ((chain)->num_cells - 1))
+
+#define SIZE_8BYTES(size)               (ALIGN((size), 8) >> 3)
+#define SIZE_4BYTES(size)               (ALIGN((size), 4) >> 2)
+
+#define RD_DMA_ATTR_DEFAULT             0
+#define WR_DMA_ATTR_DEFAULT             0
+
+enum api_cmd_data_format {
+	SGE_DATA = 1,           /* cell data is passed by hw address */
+};
+
+enum api_cmd_type {
+	API_CMD_WRITE = 0,
+};
+
+enum api_cmd_bypass {
+	NO_BYPASS       = 0,
+	BYPASS          = 1,
+};
+
+enum api_cmd_xor_chk_level {
+	XOR_CHK_DIS = 0,
+
+	XOR_CHK_ALL = 3,
+};
+
+static u8 xor_chksum_set(void *data)
+{
+	int idx;
+	u8 *val, checksum = 0;
+
+	val = data;
+
+	for (idx = 0; idx < 7; idx++)
+		checksum ^= val[idx];
+
+	return checksum;
+}
+
+static void set_prod_idx(struct hinic_api_cmd_chain *chain)
+{
+	enum hinic_api_cmd_chain_type chain_type = chain->chain_type;
+	struct hinic_hwif *hwif = chain->hwif;
+	u32 addr, prod_idx;
+
+	addr = HINIC_CSR_API_CMD_CHAIN_PI_ADDR(chain_type);
+	prod_idx = hinic_hwif_read_reg(hwif, addr);
+
+	prod_idx = HINIC_API_CMD_PI_CLEAR(prod_idx, IDX);
+
+	prod_idx |= HINIC_API_CMD_PI_SET(chain->prod_idx, IDX);
+
+	hinic_hwif_write_reg(hwif, addr, prod_idx);
+}
+
+static u32 get_hw_cons_idx(struct hinic_api_cmd_chain *chain)
+{
+	u32 addr, val;
+
+	addr = HINIC_CSR_API_CMD_STATUS_ADDR(chain->chain_type);
+	val  = hinic_hwif_read_reg(chain->hwif, addr);
+
+	return HINIC_API_CMD_STATUS_GET(val, CONS_IDX);
+}
+
+/**
+ * chain_busy - check if the chain is still processing last requests
+ * @chain: chain to check
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int chain_busy(struct hinic_api_cmd_chain *chain)
+{
+	struct hinic_hwif *hwif = chain->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	u32 prod_idx;
+
+	switch (chain->chain_type) {
+	case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
+		chain->cons_idx = get_hw_cons_idx(chain);
+		prod_idx = chain->prod_idx;
+
+		/* check for a space for a new command */
+		if (chain->cons_idx == MASKED_IDX(chain, prod_idx + 1)) {
+			dev_err(&pdev->dev, "API CMD chain %d is busy\n",
+				chain->chain_type);
+			return -EBUSY;
+		}
+		break;
+
+	default:
+		dev_err(&pdev->dev, "Unknown API CMD Chain type\n");
+		break;
+	}
+
+	return 0;
+}
+
+/**
+ * get_cell_data_size - get the data size of a specific cell type
+ * @type: chain type
+ *
+ * Return the data(Desc + Address) size in the cell
+ **/
+static u8 get_cell_data_size(enum hinic_api_cmd_chain_type type)
+{
+	u8 cell_data_size = 0;
+
+	switch (type) {
+	case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
+		cell_data_size = ALIGN(API_CMD_CELL_DESC_SIZE +
+				       API_CMD_CELL_DATA_ADDR_SIZE,
+				       API_CMD_CELL_ALIGNMENT);
+		break;
+	default:
+		break;
+	}
+
+	return cell_data_size;
+}
+
+/**
+ * prepare_cell_ctrl - prepare the ctrl of the cell for the command
+ * @cell_ctrl: the control of the cell to set the control value into it
+ * @data_size: the size of the data in the cell
+ **/
+static void prepare_cell_ctrl(u64 *cell_ctrl, u16 data_size)
+{
+	u8 chksum;
+	u64 ctrl;
+
+	ctrl =  HINIC_API_CMD_CELL_CTRL_SET(SIZE_8BYTES(data_size), DATA_SZ)  |
+		HINIC_API_CMD_CELL_CTRL_SET(RD_DMA_ATTR_DEFAULT, RD_DMA_ATTR) |
+		HINIC_API_CMD_CELL_CTRL_SET(WR_DMA_ATTR_DEFAULT, WR_DMA_ATTR);
+
+	chksum = xor_chksum_set(&ctrl);
+
+	ctrl |= HINIC_API_CMD_CELL_CTRL_SET(chksum, XOR_CHKSUM);
+
+	/* The data in the HW should be in Big Endian Format */
+	*cell_ctrl = cpu_to_be64(ctrl);
+}
+
+/**
+ * prepare_api_cmd - prepare API CMD command
+ * @chain: chain for the command
+ * @dest: destination node on the card that will receive the command
+ * @cmd: command data
+ * @cmd_size: the command size
+ **/
+static void prepare_api_cmd(struct hinic_api_cmd_chain *chain,
+			    enum hinic_node_id dest,
+			    void *cmd, u16 cmd_size)
+{
+	struct hinic_api_cmd_cell *cell = chain->curr_node;
+	struct hinic_api_cmd_cell_ctxt *cell_ctxt;
+	struct hinic_hwif *hwif = chain->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+
+	cell_ctxt = &chain->cell_ctxt[chain->prod_idx];
+
+	switch (chain->chain_type) {
+	case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
+		cell->desc = HINIC_API_CMD_DESC_SET(SGE_DATA, API_TYPE)   |
+			     HINIC_API_CMD_DESC_SET(API_CMD_WRITE, RD_WR) |
+			     HINIC_API_CMD_DESC_SET(NO_BYPASS, MGMT_BYPASS);
+		break;
+
+	default:
+		dev_err(&pdev->dev, "unknown Chain type\n");
+		return;
+	}
+
+	cell->desc |= HINIC_API_CMD_DESC_SET(dest, DEST)        |
+		      HINIC_API_CMD_DESC_SET(SIZE_4BYTES(cmd_size), SIZE);
+
+	cell->desc |= HINIC_API_CMD_DESC_SET(xor_chksum_set(&cell->desc),
+					     XOR_CHKSUM);
+
+	/* The data in the HW should be in Big Endian Format */
+	cell->desc = cpu_to_be64(cell->desc);
+
+	memcpy(cell_ctxt->api_cmd_vaddr, cmd, cmd_size);
+}
+
+/**
+ * prepare_cell - prepare cell ctrl and cmd in the current cell
+ * @chain: chain for the command
+ * @dest: destination node on the card that will receive the command
+ * @cmd: command data
+ * @cmd_size: the command size
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static void prepare_cell(struct hinic_api_cmd_chain *chain,
+			 enum  hinic_node_id dest,
+			 void *cmd, u16 cmd_size)
+{
+	struct hinic_api_cmd_cell *curr_node = chain->curr_node;
+	u16 data_size = get_cell_data_size(chain->chain_type);
+
+	prepare_cell_ctrl(&curr_node->ctrl, data_size);
+	prepare_api_cmd(chain, dest, cmd, cmd_size);
+}
+
+static inline void cmd_chain_prod_idx_inc(struct hinic_api_cmd_chain *chain)
+{
+	chain->prod_idx = MASKED_IDX(chain, chain->prod_idx + 1);
+}
+
+/**
+ * api_cmd_status_update - update the status in the chain struct
+ * @chain: chain to update
+ **/
+static void api_cmd_status_update(struct hinic_api_cmd_chain *chain)
+{
+	enum hinic_api_cmd_chain_type chain_type;
+	struct hinic_api_cmd_status *wb_status;
+	struct hinic_hwif *hwif = chain->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	u64 status_header;
+	u32 status;
+
+	wb_status = chain->wb_status;
+	status_header = be64_to_cpu(wb_status->header);
+
+	status = be32_to_cpu(wb_status->status);
+	if (HINIC_API_CMD_STATUS_GET(status, CHKSUM_ERR)) {
+		dev_err(&pdev->dev, "API CMD status: Xor check error\n");
+		return;
+	}
+
+	chain_type = HINIC_API_CMD_STATUS_HEADER_GET(status_header, CHAIN_ID);
+	if (chain_type >= HINIC_API_CMD_MAX) {
+		dev_err(&pdev->dev, "unknown API CMD Chain %d\n", chain_type);
+		return;
+	}
+
+	chain->cons_idx = HINIC_API_CMD_STATUS_GET(status, CONS_IDX);
+}
+
+/**
+ * wait_for_status_poll - wait for write to api cmd command to complete
+ * @chain: the chain of the command
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int wait_for_status_poll(struct hinic_api_cmd_chain *chain)
+{
+	int err = -ETIMEDOUT;
+	unsigned long end;
+
+	end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT);
+	do {
+		api_cmd_status_update(chain);
+
+		/* wait for CI to be updated - sign for completion */
+		if (chain->cons_idx == chain->prod_idx) {
+			err = 0;
+			break;
+		}
+
+		msleep(20);
+	} while (time_before(jiffies, end));
+
+	return err;
+}
+
+/**
+ * wait_for_api_cmd_completion - wait for command to complete
+ * @chain: chain for the command
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int wait_for_api_cmd_completion(struct hinic_api_cmd_chain *chain)
+{
+	struct hinic_hwif *hwif = chain->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	int err;
+
+	switch (chain->chain_type) {
+	case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
+		err = wait_for_status_poll(chain);
+		if (err) {
+			dev_err(&pdev->dev, "API CMD Poll status timeout\n");
+			break;
+		}
+		break;
+
+	default:
+		dev_err(&pdev->dev, "unknown API CMD Chain type\n");
+		err = -EINVAL;
+		break;
+	}
+
+	return err;
+}
+
+/**
+ * api_cmd - API CMD command
+ * @chain: chain for the command
+ * @dest: destination node on the card that will receive the command
+ * @cmd: command data
+ * @size: the command size
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int api_cmd(struct hinic_api_cmd_chain *chain,
+		   enum hinic_node_id dest, u8 *cmd, u16 cmd_size)
+{
+	struct hinic_api_cmd_cell_ctxt *ctxt;
+	int err;
+
+	down(&chain->sem);
+	if (chain_busy(chain)) {
+		up(&chain->sem);
+		return -EBUSY;
+	}
+
+	prepare_cell(chain, dest, cmd, cmd_size);
+	cmd_chain_prod_idx_inc(chain);
+
+	wmb();  /* inc pi before issue the command */
+
+	set_prod_idx(chain);    /* issue the command */
+
+	ctxt = &chain->cell_ctxt[chain->prod_idx];
+
+	chain->curr_node = ctxt->cell_vaddr;
+
+	err = wait_for_api_cmd_completion(chain);
+
+	up(&chain->sem);
+	return err;
+}
+
+/**
+ * hinic_api_cmd_write - Write API CMD command
+ * @chain: chain for write command
+ * @dest: destination node on the card that will receive the command
+ * @cmd: command data
+ * @size: the command size
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_api_cmd_write(struct hinic_api_cmd_chain *chain,
+			enum hinic_node_id dest, u8 *cmd, u16 size)
+{
+	/* Verify the chain type */
+	if (chain->chain_type == HINIC_API_CMD_WRITE_TO_MGMT_CPU)
+		return api_cmd(chain, dest, cmd, size);
+
+	return -EINVAL;
+}
+
+/**
+ * api_cmd_hw_restart - restart the chain in the HW
+ * @chain: the API CMD specific chain to restart
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int api_cmd_hw_restart(struct hinic_api_cmd_chain *chain)
+{
+	struct hinic_hwif *hwif = chain->hwif;
+	int err = -ETIMEDOUT;
+	unsigned long end;
+	u32 reg_addr, val;
+
+	/* Read Modify Write */
+	reg_addr = HINIC_CSR_API_CMD_CHAIN_REQ_ADDR(chain->chain_type);
+	val = hinic_hwif_read_reg(hwif, reg_addr);
+
+	val = HINIC_API_CMD_CHAIN_REQ_CLEAR(val, RESTART);
+	val |= HINIC_API_CMD_CHAIN_REQ_SET(1, RESTART);
+
+	hinic_hwif_write_reg(hwif, reg_addr, val);
+
+	end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT);
+	do {
+		val = hinic_hwif_read_reg(hwif, reg_addr);
+
+		if (!HINIC_API_CMD_CHAIN_REQ_GET(val, RESTART)) {
+			err = 0;
+			break;
+		}
+
+		msleep(20);
+	} while (time_before(jiffies, end));
+
+	return err;
+}
+
+/**
+ * api_cmd_ctrl_init - set the control register of a chain
+ * @chain: the API CMD specific chain to set control register for
+ **/
+static void api_cmd_ctrl_init(struct hinic_api_cmd_chain *chain)
+{
+	struct hinic_hwif *hwif = chain->hwif;
+	u32 addr, ctrl;
+	u16 cell_size;
+
+	/* Read Modify Write */
+	addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type);
+
+	cell_size = API_CMD_CELL_SIZE_VAL(chain->cell_size);
+
+	ctrl = hinic_hwif_read_reg(hwif, addr);
+
+	ctrl =  HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, RESTART_WB_STAT) &
+		HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_ERR)         &
+		HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN)         &
+		HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_CHK_EN)      &
+		HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE);
+
+	ctrl |= HINIC_API_CMD_CHAIN_CTRL_SET(1, XOR_ERR)              |
+		HINIC_API_CMD_CHAIN_CTRL_SET(XOR_CHK_ALL, XOR_CHK_EN) |
+		HINIC_API_CMD_CHAIN_CTRL_SET(cell_size, CELL_SIZE);
+
+	hinic_hwif_write_reg(hwif, addr, ctrl);
+}
+
+/**
+ * api_cmd_set_status_addr - set the status address of a chain in the HW
+ * @chain: the API CMD specific chain to set in HW status address for
+ **/
+static void api_cmd_set_status_addr(struct hinic_api_cmd_chain *chain)
+{
+	struct hinic_hwif *hwif = chain->hwif;
+	u32 addr, val;
+
+	addr = HINIC_CSR_API_CMD_STATUS_HI_ADDR(chain->chain_type);
+	val = upper_32_bits(chain->wb_status_paddr);
+	hinic_hwif_write_reg(hwif, addr, val);
+
+	addr = HINIC_CSR_API_CMD_STATUS_LO_ADDR(chain->chain_type);
+	val = lower_32_bits(chain->wb_status_paddr);
+	hinic_hwif_write_reg(hwif, addr, val);
+}
+
+/**
+ * api_cmd_set_num_cells - set the number cells of a chain in the HW
+ * @chain: the API CMD specific chain to set in HW the number of cells for
+ **/
+static void api_cmd_set_num_cells(struct hinic_api_cmd_chain *chain)
+{
+	struct hinic_hwif *hwif = chain->hwif;
+	u32 addr, val;
+
+	addr = HINIC_CSR_API_CMD_CHAIN_NUM_CELLS_ADDR(chain->chain_type);
+	val  = chain->num_cells;
+	hinic_hwif_write_reg(hwif, addr, val);
+}
+
+/**
+ * api_cmd_head_init - set the head of a chain in the HW
+ * @chain: the API CMD specific chain to set in HW the head for
+ **/
+static void api_cmd_head_init(struct hinic_api_cmd_chain *chain)
+{
+	struct hinic_hwif *hwif = chain->hwif;
+	u32 addr, val;
+
+	addr = HINIC_CSR_API_CMD_CHAIN_HEAD_HI_ADDR(chain->chain_type);
+	val = upper_32_bits(chain->head_cell_paddr);
+	hinic_hwif_write_reg(hwif, addr, val);
+
+	addr = HINIC_CSR_API_CMD_CHAIN_HEAD_LO_ADDR(chain->chain_type);
+	val = lower_32_bits(chain->head_cell_paddr);
+	hinic_hwif_write_reg(hwif, addr, val);
+}
+
+/**
+ * api_cmd_chain_hw_clean - clean the HW
+ * @chain: the API CMD specific chain
+ **/
+static void api_cmd_chain_hw_clean(struct hinic_api_cmd_chain *chain)
+{
+	struct hinic_hwif *hwif = chain->hwif;
+	u32 addr, ctrl;
+
+	addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type);
+
+	ctrl = hinic_hwif_read_reg(hwif, addr);
+	ctrl = HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, RESTART_WB_STAT) &
+	       HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_ERR)         &
+	       HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN)         &
+	       HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_CHK_EN)      &
+	       HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE);
+
+	hinic_hwif_write_reg(hwif, addr, ctrl);
+}
+
+/**
+ * api_cmd_chain_hw_init - initialize the chain in the HW
+ * @chain: the API CMD specific chain to initialize in HW
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int api_cmd_chain_hw_init(struct hinic_api_cmd_chain *chain)
+{
+	struct hinic_hwif *hwif = chain->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	int err;
+
+	api_cmd_chain_hw_clean(chain);
+
+	api_cmd_set_status_addr(chain);
+
+	err = api_cmd_hw_restart(chain);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to restart API CMD HW\n");
+		return err;
+	}
+
+	api_cmd_ctrl_init(chain);
+	api_cmd_set_num_cells(chain);
+	api_cmd_head_init(chain);
+	return 0;
+}
+
+/**
+ * free_cmd_buf - free the dma buffer of API CMD command
+ * @chain: the API CMD specific chain of the cmd
+ * @cell_idx: the cell index of the cmd
+ **/
+static void free_cmd_buf(struct hinic_api_cmd_chain *chain, int cell_idx)
+{
+	struct hinic_api_cmd_cell_ctxt *cell_ctxt;
+	struct hinic_hwif *hwif = chain->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+
+	cell_ctxt = &chain->cell_ctxt[cell_idx];
+
+	dma_free_coherent(&pdev->dev, API_CMD_BUF_SIZE,
+			  cell_ctxt->api_cmd_vaddr,
+			  cell_ctxt->api_cmd_paddr);
+}
+
+/**
+ * alloc_cmd_buf - allocate a dma buffer for API CMD command
+ * @chain: the API CMD specific chain for the cmd
+ * @cell: the cell in the HW for the cmd
+ * @cell_idx: the index of the cell
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int alloc_cmd_buf(struct hinic_api_cmd_chain *chain,
+			 struct hinic_api_cmd_cell *cell, int cell_idx)
+{
+	struct hinic_api_cmd_cell_ctxt *cell_ctxt;
+	struct hinic_hwif *hwif = chain->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	dma_addr_t cmd_paddr;
+	u8 *cmd_vaddr;
+	int err = 0;
+
+	cmd_vaddr = dma_zalloc_coherent(&pdev->dev, API_CMD_BUF_SIZE,
+					&cmd_paddr, GFP_KERNEL);
+	if (!cmd_vaddr) {
+		dev_err(&pdev->dev, "Failed to allocate API CMD DMA memory\n");
+		return -ENOMEM;
+	}
+
+	cell_ctxt = &chain->cell_ctxt[cell_idx];
+
+	cell_ctxt->api_cmd_vaddr = cmd_vaddr;
+	cell_ctxt->api_cmd_paddr = cmd_paddr;
+
+	/* set the cmd DMA address in the cell */
+	switch (chain->chain_type) {
+	case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
+		/* The data in the HW should be in Big Endian Format */
+		cell->write.hw_cmd_paddr = cpu_to_be64(cmd_paddr);
+		break;
+
+	default:
+		dev_err(&pdev->dev, "Unsupported API CMD chain type\n");
+		free_cmd_buf(chain, cell_idx);
+		err = -EINVAL;
+		break;
+	}
+
+	return err;
+}
+
+/**
+ * api_cmd_create_cell - create API CMD cell for specific chain
+ * @chain: the API CMD specific chain to create its cell
+ * @cell_idx: the index of the cell to create
+ * @pre_node: previous cell
+ * @node_vaddr: the returned virt addr of the cell
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int api_cmd_create_cell(struct hinic_api_cmd_chain *chain,
+			       int cell_idx,
+			       struct hinic_api_cmd_cell *pre_node,
+			       struct hinic_api_cmd_cell **node_vaddr)
+{
+	struct hinic_api_cmd_cell_ctxt *cell_ctxt;
+	struct hinic_hwif *hwif = chain->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	struct hinic_api_cmd_cell *node;
+	dma_addr_t node_paddr;
+	int err;
+
+	node = dma_zalloc_coherent(&pdev->dev, chain->cell_size,
+				   &node_paddr, GFP_KERNEL);
+	if (!node) {
+		dev_err(&pdev->dev, "Failed to allocate dma API CMD cell\n");
+		return -ENOMEM;
+	}
+
+	node->read.hw_wb_resp_paddr = 0;
+
+	cell_ctxt = &chain->cell_ctxt[cell_idx];
+	cell_ctxt->cell_vaddr = node;
+	cell_ctxt->cell_paddr = node_paddr;
+
+	if (!pre_node) {
+		chain->head_cell_paddr = node_paddr;
+		chain->head_node = node;
+	} else {
+		/* The data in the HW should be in Big Endian Format */
+		pre_node->next_cell_paddr = cpu_to_be64(node_paddr);
+	}
+
+	switch (chain->chain_type) {
+	case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
+		err = alloc_cmd_buf(chain, node, cell_idx);
+		if (err) {
+			dev_err(&pdev->dev, "Failed to allocate cmd buffer\n");
+			goto err_alloc_cmd_buf;
+		}
+		break;
+
+	default:
+		dev_err(&pdev->dev, "Unsupported API CMD chain type\n");
+		err = -EINVAL;
+		goto err_alloc_cmd_buf;
+	}
+
+	*node_vaddr = node;
+	return 0;
+
+err_alloc_cmd_buf:
+	dma_free_coherent(&pdev->dev, chain->cell_size, node, node_paddr);
+	return err;
+}
+
+/**
+ * api_cmd_destroy_cell - destroy API CMD cell of specific chain
+ * @chain: the API CMD specific chain to destroy its cell
+ * @cell_idx: the cell to destroy
+ **/
+static void api_cmd_destroy_cell(struct hinic_api_cmd_chain *chain,
+				 int cell_idx)
+{
+	struct hinic_api_cmd_cell_ctxt *cell_ctxt;
+	struct hinic_hwif *hwif = chain->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	struct hinic_api_cmd_cell *node;
+	dma_addr_t node_paddr;
+	size_t node_size;
+
+	cell_ctxt = &chain->cell_ctxt[cell_idx];
+
+	node = cell_ctxt->cell_vaddr;
+	node_paddr = cell_ctxt->cell_paddr;
+	node_size = chain->cell_size;
+
+	if (cell_ctxt->api_cmd_vaddr) {
+		switch (chain->chain_type) {
+		case HINIC_API_CMD_WRITE_TO_MGMT_CPU:
+			free_cmd_buf(chain, cell_idx);
+			break;
+		default:
+			dev_err(&pdev->dev, "Unsupported API CMD chain type\n");
+			break;
+		}
+
+		dma_free_coherent(&pdev->dev, node_size, node,
+				  node_paddr);
+	}
+}
+
+/**
+ * api_cmd_destroy_cells - destroy API CMD cells of specific chain
+ * @chain: the API CMD specific chain to destroy its cells
+ * @num_cells: number of cells to destroy
+ **/
+static void api_cmd_destroy_cells(struct hinic_api_cmd_chain *chain,
+				  int num_cells)
+{
+	int cell_idx;
+
+	for (cell_idx = 0; cell_idx < num_cells; cell_idx++)
+		api_cmd_destroy_cell(chain, cell_idx);
+}
+
+/**
+ * api_cmd_create_cells - create API CMD cells for specific chain
+ * @chain: the API CMD specific chain
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int api_cmd_create_cells(struct hinic_api_cmd_chain *chain)
+{
+	struct hinic_api_cmd_cell *node = NULL, *pre_node = NULL;
+	struct hinic_hwif *hwif = chain->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	int err, cell_idx;
+
+	for (cell_idx = 0; cell_idx < chain->num_cells; cell_idx++) {
+		err = api_cmd_create_cell(chain, cell_idx, pre_node, &node);
+		if (err) {
+			dev_err(&pdev->dev, "Failed to create API CMD cell\n");
+			goto err_create_cell;
+		}
+
+		pre_node = node;
+	}
+
+	/* set the Final node to point on the start */
+	node->next_cell_paddr = cpu_to_be64(chain->head_cell_paddr);
+
+	/* set the current node to be the head */
+	chain->curr_node = chain->head_node;
+	return 0;
+
+err_create_cell:
+	api_cmd_destroy_cells(chain, cell_idx);
+	return err;
+}
+
+/**
+ * api_chain_init - initialize API CMD specific chain
+ * @chain: the API CMD specific chain to initialize
+ * @attr: attributes to set in the chain
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int api_chain_init(struct hinic_api_cmd_chain *chain,
+			  struct hinic_api_cmd_chain_attr *attr)
+{
+	struct hinic_hwif *hwif = attr->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	size_t cell_ctxt_size;
+
+	chain->hwif = hwif;
+	chain->chain_type  = attr->chain_type;
+	chain->num_cells = attr->num_cells;
+	chain->cell_size = attr->cell_size;
+
+	chain->prod_idx  = 0;
+	chain->cons_idx  = 0;
+
+	sema_init(&chain->sem, 1);
+
+	cell_ctxt_size = chain->num_cells * sizeof(*chain->cell_ctxt);
+	chain->cell_ctxt = devm_kzalloc(&pdev->dev, cell_ctxt_size, GFP_KERNEL);
+	if (!chain->cell_ctxt)
+		return -ENOMEM;
+
+	chain->wb_status = dma_zalloc_coherent(&pdev->dev,
+					       sizeof(*chain->wb_status),
+					       &chain->wb_status_paddr,
+					       GFP_KERNEL);
+	if (!chain->wb_status) {
+		dev_err(&pdev->dev, "Failed to allocate DMA wb status\n");
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+/**
+ * api_chain_free - free API CMD specific chain
+ * @chain: the API CMD specific chain to free
+ **/
+static void api_chain_free(struct hinic_api_cmd_chain *chain)
+{
+	struct hinic_hwif *hwif = chain->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+
+	dma_free_coherent(&pdev->dev, sizeof(*chain->wb_status),
+			  chain->wb_status, chain->wb_status_paddr);
+}
+
+/**
+ * api_cmd_create_chain - create API CMD specific chain
+ * @attr: attributes to set the chain
+ *
+ * Return the created chain
+ **/
+static struct hinic_api_cmd_chain *
+	api_cmd_create_chain(struct hinic_api_cmd_chain_attr *attr)
+{
+	struct hinic_hwif *hwif = attr->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	struct hinic_api_cmd_chain *chain;
+	int err;
+
+	if (attr->num_cells & (attr->num_cells - 1)) {
+		dev_err(&pdev->dev, "Invalid number of cells, must be power of 2\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
+	if (!chain)
+		return ERR_PTR(-ENOMEM);
+
+	err = api_chain_init(chain, attr);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to initialize chain\n");
+		return ERR_PTR(err);
+	}
+
+	err = api_cmd_create_cells(chain);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to create cells for API CMD chain\n");
+		goto err_create_cells;
+	}
+
+	err = api_cmd_chain_hw_init(chain);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to initialize chain HW\n");
+		goto err_chain_hw_init;
+	}
+
+	return chain;
+
+err_chain_hw_init:
+	api_cmd_destroy_cells(chain, chain->num_cells);
+
+err_create_cells:
+	api_chain_free(chain);
+	return ERR_PTR(err);
+}
+
+/**
+ * api_cmd_destroy_chain - destroy API CMD specific chain
+ * @chain: the API CMD specific chain to destroy
+ **/
+static void api_cmd_destroy_chain(struct hinic_api_cmd_chain *chain)
+{
+	api_cmd_chain_hw_clean(chain);
+	api_cmd_destroy_cells(chain, chain->num_cells);
+	api_chain_free(chain);
+}
+
+/**
+ * hinic_api_cmd_init - Initialize all the API CMD chains
+ * @chain: the API CMD chains that are initialized
+ * @hwif: the hardware interface of a pci function device
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_api_cmd_init(struct hinic_api_cmd_chain **chain,
+		       struct hinic_hwif *hwif)
+{
+	enum hinic_api_cmd_chain_type type, chain_type;
+	struct hinic_api_cmd_chain_attr attr;
+	struct pci_dev *pdev = hwif->pdev;
+	size_t hw_cell_sz;
+	int err;
+
+	hw_cell_sz = sizeof(struct hinic_api_cmd_cell);
+
+	attr.hwif = hwif;
+	attr.num_cells  = API_CHAIN_NUM_CELLS;
+	attr.cell_size  = API_CMD_CELL_SIZE(hw_cell_sz);
+
+	chain_type = HINIC_API_CMD_WRITE_TO_MGMT_CPU;
+	for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++) {
+		attr.chain_type = chain_type;
+
+		if (chain_type != HINIC_API_CMD_WRITE_TO_MGMT_CPU)
+			continue;
+
+		chain[chain_type] = api_cmd_create_chain(&attr);
+		if (IS_ERR(chain[chain_type])) {
+			dev_err(&pdev->dev, "Failed to create chain %d\n",
+				chain_type);
+			err = PTR_ERR(chain[chain_type]);
+			goto err_create_chain;
+		}
+	}
+
+	return 0;
+
+err_create_chain:
+	type = HINIC_API_CMD_WRITE_TO_MGMT_CPU;
+	for ( ; type < chain_type; type++) {
+		if (type != HINIC_API_CMD_WRITE_TO_MGMT_CPU)
+			continue;
+
+		api_cmd_destroy_chain(chain[type]);
+	}
+
+	return err;
+}
+
+/**
+ * hinic_api_cmd_free - free the API CMD chains
+ * @chain: the API CMD chains that are freed
+ **/
+void hinic_api_cmd_free(struct hinic_api_cmd_chain **chain)
+{
+	enum hinic_api_cmd_chain_type chain_type;
+
+	chain_type = HINIC_API_CMD_WRITE_TO_MGMT_CPU;
+	for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++) {
+		if (chain_type != HINIC_API_CMD_WRITE_TO_MGMT_CPU)
+			continue;
+
+		api_cmd_destroy_chain(chain[chain_type]);
+	}
+}
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_api_cmd.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_api_cmd.h
new file mode 100644
index 000000000000..31b94d5d47f7
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_api_cmd.h
@@ -0,0 +1,208 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#ifndef HINIC_HW_API_CMD_H
+#define HINIC_HW_API_CMD_H
+
+#include <linux/types.h>
+#include <linux/semaphore.h>
+
+#include "hinic_hw_if.h"
+
+#define HINIC_API_CMD_PI_IDX_SHIFT                              0
+
+#define HINIC_API_CMD_PI_IDX_MASK                               0xFFFFFF
+
+#define HINIC_API_CMD_PI_SET(val, member)                       \
+	(((u32)(val) & HINIC_API_CMD_PI_##member##_MASK) <<     \
+	 HINIC_API_CMD_PI_##member##_SHIFT)
+
+#define HINIC_API_CMD_PI_CLEAR(val, member)                     \
+	((val) & (~(HINIC_API_CMD_PI_##member##_MASK            \
+	 << HINIC_API_CMD_PI_##member##_SHIFT)))
+
+#define HINIC_API_CMD_CHAIN_REQ_RESTART_SHIFT                   1
+
+#define HINIC_API_CMD_CHAIN_REQ_RESTART_MASK                    0x1
+
+#define HINIC_API_CMD_CHAIN_REQ_SET(val, member)                \
+	(((u32)(val) & HINIC_API_CMD_CHAIN_REQ_##member##_MASK) << \
+	 HINIC_API_CMD_CHAIN_REQ_##member##_SHIFT)
+
+#define HINIC_API_CMD_CHAIN_REQ_GET(val, member)                \
+	(((val) >> HINIC_API_CMD_CHAIN_REQ_##member##_SHIFT) &  \
+	 HINIC_API_CMD_CHAIN_REQ_##member##_MASK)
+
+#define HINIC_API_CMD_CHAIN_REQ_CLEAR(val, member)              \
+	((val) & (~(HINIC_API_CMD_CHAIN_REQ_##member##_MASK     \
+	 << HINIC_API_CMD_CHAIN_REQ_##member##_SHIFT)))
+
+#define HINIC_API_CMD_CHAIN_CTRL_RESTART_WB_STAT_SHIFT          1
+#define HINIC_API_CMD_CHAIN_CTRL_XOR_ERR_SHIFT                  2
+#define HINIC_API_CMD_CHAIN_CTRL_AEQE_EN_SHIFT                  4
+#define HINIC_API_CMD_CHAIN_CTRL_AEQ_ID_SHIFT                   8
+#define HINIC_API_CMD_CHAIN_CTRL_XOR_CHK_EN_SHIFT               28
+#define HINIC_API_CMD_CHAIN_CTRL_CELL_SIZE_SHIFT                30
+
+#define HINIC_API_CMD_CHAIN_CTRL_RESTART_WB_STAT_MASK           0x1
+#define HINIC_API_CMD_CHAIN_CTRL_XOR_ERR_MASK                   0x1
+#define HINIC_API_CMD_CHAIN_CTRL_AEQE_EN_MASK                   0x1
+#define HINIC_API_CMD_CHAIN_CTRL_AEQ_ID_MASK                    0x3
+#define HINIC_API_CMD_CHAIN_CTRL_XOR_CHK_EN_MASK                0x3
+#define HINIC_API_CMD_CHAIN_CTRL_CELL_SIZE_MASK                 0x3
+
+#define HINIC_API_CMD_CHAIN_CTRL_SET(val, member)               \
+	(((u32)(val) & HINIC_API_CMD_CHAIN_CTRL_##member##_MASK) << \
+	 HINIC_API_CMD_CHAIN_CTRL_##member##_SHIFT)
+
+#define HINIC_API_CMD_CHAIN_CTRL_CLEAR(val, member)             \
+	((val) & (~(HINIC_API_CMD_CHAIN_CTRL_##member##_MASK    \
+	 << HINIC_API_CMD_CHAIN_CTRL_##member##_SHIFT)))
+
+#define HINIC_API_CMD_CELL_CTRL_DATA_SZ_SHIFT                   0
+#define HINIC_API_CMD_CELL_CTRL_RD_DMA_ATTR_SHIFT               16
+#define HINIC_API_CMD_CELL_CTRL_WR_DMA_ATTR_SHIFT               24
+#define HINIC_API_CMD_CELL_CTRL_XOR_CHKSUM_SHIFT                56
+
+#define HINIC_API_CMD_CELL_CTRL_DATA_SZ_MASK                    0x3F
+#define HINIC_API_CMD_CELL_CTRL_RD_DMA_ATTR_MASK                0x3F
+#define HINIC_API_CMD_CELL_CTRL_WR_DMA_ATTR_MASK                0x3F
+#define HINIC_API_CMD_CELL_CTRL_XOR_CHKSUM_MASK                 0xFF
+
+#define HINIC_API_CMD_CELL_CTRL_SET(val, member)                \
+	((((u64)val) & HINIC_API_CMD_CELL_CTRL_##member##_MASK) << \
+	 HINIC_API_CMD_CELL_CTRL_##member##_SHIFT)
+
+#define HINIC_API_CMD_DESC_API_TYPE_SHIFT                       0
+#define HINIC_API_CMD_DESC_RD_WR_SHIFT                          1
+#define HINIC_API_CMD_DESC_MGMT_BYPASS_SHIFT                    2
+#define HINIC_API_CMD_DESC_DEST_SHIFT                           32
+#define HINIC_API_CMD_DESC_SIZE_SHIFT                           40
+#define HINIC_API_CMD_DESC_XOR_CHKSUM_SHIFT                     56
+
+#define HINIC_API_CMD_DESC_API_TYPE_MASK                        0x1
+#define HINIC_API_CMD_DESC_RD_WR_MASK                           0x1
+#define HINIC_API_CMD_DESC_MGMT_BYPASS_MASK                     0x1
+#define HINIC_API_CMD_DESC_DEST_MASK                            0x1F
+#define HINIC_API_CMD_DESC_SIZE_MASK                            0x7FF
+#define HINIC_API_CMD_DESC_XOR_CHKSUM_MASK                      0xFF
+
+#define HINIC_API_CMD_DESC_SET(val, member)                     \
+	((((u64)val) & HINIC_API_CMD_DESC_##member##_MASK) <<   \
+	 HINIC_API_CMD_DESC_##member##_SHIFT)
+
+#define HINIC_API_CMD_STATUS_HEADER_CHAIN_ID_SHIFT              16
+
+#define HINIC_API_CMD_STATUS_HEADER_CHAIN_ID_MASK               0xFF
+
+#define HINIC_API_CMD_STATUS_HEADER_GET(val, member)            \
+	(((val) >> HINIC_API_CMD_STATUS_HEADER_##member##_SHIFT) & \
+	 HINIC_API_CMD_STATUS_HEADER_##member##_MASK)
+
+#define HINIC_API_CMD_STATUS_CONS_IDX_SHIFT                     0
+#define HINIC_API_CMD_STATUS_CHKSUM_ERR_SHIFT                   28
+
+#define HINIC_API_CMD_STATUS_CONS_IDX_MASK                      0xFFFFFF
+#define HINIC_API_CMD_STATUS_CHKSUM_ERR_MASK                    0x3
+
+#define HINIC_API_CMD_STATUS_GET(val, member)                   \
+	(((val) >> HINIC_API_CMD_STATUS_##member##_SHIFT) &     \
+	 HINIC_API_CMD_STATUS_##member##_MASK)
+
+enum hinic_api_cmd_chain_type {
+	HINIC_API_CMD_WRITE_TO_MGMT_CPU = 2,
+
+	HINIC_API_CMD_MAX,
+};
+
+struct hinic_api_cmd_chain_attr {
+	struct hinic_hwif               *hwif;
+	enum hinic_api_cmd_chain_type   chain_type;
+
+	u32                             num_cells;
+	u16                             cell_size;
+};
+
+struct hinic_api_cmd_status {
+	u64     header;
+	u32     status;
+	u32     rsvd0;
+	u32     rsvd1;
+	u32     rsvd2;
+	u64     rsvd3;
+};
+
+/* HW struct */
+struct hinic_api_cmd_cell {
+	u64 ctrl;
+
+	/* address is 64 bit in HW struct */
+	u64 next_cell_paddr;
+
+	u64 desc;
+
+	/* HW struct */
+	union {
+		struct {
+			u64 hw_cmd_paddr;
+		} write;
+
+		struct {
+			u64 hw_wb_resp_paddr;
+			u64 hw_cmd_paddr;
+		} read;
+	};
+};
+
+struct hinic_api_cmd_cell_ctxt {
+	dma_addr_t                      cell_paddr;
+	struct hinic_api_cmd_cell       *cell_vaddr;
+
+	dma_addr_t                      api_cmd_paddr;
+	u8                              *api_cmd_vaddr;
+};
+
+struct hinic_api_cmd_chain {
+	struct hinic_hwif               *hwif;
+	enum hinic_api_cmd_chain_type   chain_type;
+
+	u32                             num_cells;
+	u16                             cell_size;
+
+	/* HW members in 24 bit format */
+	u32                             prod_idx;
+	u32                             cons_idx;
+
+	struct semaphore                sem;
+
+	struct hinic_api_cmd_cell_ctxt  *cell_ctxt;
+
+	dma_addr_t                      wb_status_paddr;
+	struct hinic_api_cmd_status     *wb_status;
+
+	dma_addr_t                      head_cell_paddr;
+	struct hinic_api_cmd_cell       *head_node;
+	struct hinic_api_cmd_cell       *curr_node;
+};
+
+int hinic_api_cmd_write(struct hinic_api_cmd_chain *chain,
+			enum hinic_node_id dest, u8 *cmd, u16 size);
+
+int hinic_api_cmd_init(struct hinic_api_cmd_chain **chain,
+		       struct hinic_hwif *hwif);
+
+void hinic_api_cmd_free(struct hinic_api_cmd_chain **chain);
+
+#endif
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c
new file mode 100644
index 000000000000..7d95f0866fb0
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c
@@ -0,0 +1,946 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/errno.h>
+#include <linux/pci.h>
+#include <linux/device.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/spinlock.h>
+#include <linux/sizes.h>
+#include <linux/atomic.h>
+#include <linux/log2.h>
+#include <linux/io.h>
+#include <linux/completion.h>
+#include <linux/err.h>
+#include <asm/byteorder.h>
+#include <asm/barrier.h>
+
+#include "hinic_common.h"
+#include "hinic_hw_if.h"
+#include "hinic_hw_eqs.h"
+#include "hinic_hw_mgmt.h"
+#include "hinic_hw_wqe.h"
+#include "hinic_hw_wq.h"
+#include "hinic_hw_cmdq.h"
+#include "hinic_hw_io.h"
+#include "hinic_hw_dev.h"
+
+#define CMDQ_CEQE_TYPE_SHIFT                    0
+
+#define CMDQ_CEQE_TYPE_MASK                     0x7
+
+#define CMDQ_CEQE_GET(val, member)              \
+			(((val) >> CMDQ_CEQE_##member##_SHIFT) \
+			 & CMDQ_CEQE_##member##_MASK)
+
+#define CMDQ_WQE_ERRCODE_VAL_SHIFT              20
+
+#define CMDQ_WQE_ERRCODE_VAL_MASK               0xF
+
+#define CMDQ_WQE_ERRCODE_GET(val, member)       \
+			(((val) >> CMDQ_WQE_ERRCODE_##member##_SHIFT) \
+			 & CMDQ_WQE_ERRCODE_##member##_MASK)
+
+#define CMDQ_DB_PI_OFF(pi)              (((u16)LOWER_8_BITS(pi)) << 3)
+
+#define CMDQ_DB_ADDR(db_base, pi)       ((db_base) + CMDQ_DB_PI_OFF(pi))
+
+#define CMDQ_WQE_HEADER(wqe)            ((struct hinic_cmdq_header *)(wqe))
+
+#define CMDQ_WQE_COMPLETED(ctrl_info)   \
+			HINIC_CMDQ_CTRL_GET(ctrl_info, HW_BUSY_BIT)
+
+#define FIRST_DATA_TO_WRITE_LAST        sizeof(u64)
+
+#define CMDQ_DB_OFF                     SZ_2K
+
+#define CMDQ_WQEBB_SIZE                 64
+#define CMDQ_WQE_SIZE                   64
+#define CMDQ_DEPTH                      SZ_4K
+
+#define CMDQ_WQ_PAGE_SIZE               SZ_4K
+
+#define WQE_LCMD_SIZE                   64
+#define WQE_SCMD_SIZE                   64
+
+#define COMPLETE_LEN                    3
+
+#define CMDQ_TIMEOUT                    1000
+
+#define CMDQ_PFN(addr, page_size)       ((addr) >> (ilog2(page_size)))
+
+#define cmdq_to_cmdqs(cmdq)     container_of((cmdq) - (cmdq)->cmdq_type, \
+					     struct hinic_cmdqs, cmdq[0])
+
+#define cmdqs_to_func_to_io(cmdqs)      container_of(cmdqs, \
+						     struct hinic_func_to_io, \
+						     cmdqs)
+
+enum cmdq_wqe_type {
+	WQE_LCMD_TYPE = 0,
+	WQE_SCMD_TYPE = 1,
+};
+
+enum completion_format {
+	COMPLETE_DIRECT = 0,
+	COMPLETE_SGE    = 1,
+};
+
+enum data_format {
+	DATA_SGE        = 0,
+	DATA_DIRECT     = 1,
+};
+
+enum bufdesc_len {
+	BUFDESC_LCMD_LEN = 2,   /* 16 bytes - 2(8 byte unit) */
+	BUFDESC_SCMD_LEN = 3,   /* 24 bytes - 3(8 byte unit) */
+};
+
+enum ctrl_sect_len {
+	CTRL_SECT_LEN        = 1, /* 4 bytes (ctrl) - 1(8 byte unit) */
+	CTRL_DIRECT_SECT_LEN = 2, /* 12 bytes (ctrl + rsvd) - 2(8 byte unit) */
+};
+
+enum cmdq_scmd_type {
+	CMDQ_SET_ARM_CMD = 2,
+};
+
+enum cmdq_cmd_type {
+	CMDQ_CMD_SYNC_DIRECT_RESP = 0,
+	CMDQ_CMD_SYNC_SGE_RESP    = 1,
+};
+
+enum completion_request {
+	NO_CEQ  = 0,
+	CEQ_SET = 1,
+};
+
+/**
+ * hinic_alloc_cmdq_buf - alloc buffer for sending command
+ * @cmdqs: the cmdqs
+ * @cmdq_buf: the buffer returned in this struct
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_alloc_cmdq_buf(struct hinic_cmdqs *cmdqs,
+			 struct hinic_cmdq_buf *cmdq_buf)
+{
+	struct hinic_hwif *hwif = cmdqs->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+
+	cmdq_buf->buf = pci_pool_alloc(cmdqs->cmdq_buf_pool, GFP_KERNEL,
+				       &cmdq_buf->dma_addr);
+	if (!cmdq_buf->buf) {
+		dev_err(&pdev->dev, "Failed to allocate cmd from the pool\n");
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+/**
+ * hinic_free_cmdq_buf - free buffer
+ * @cmdqs: the cmdqs
+ * @cmdq_buf: the buffer to free that is in this struct
+ **/
+void hinic_free_cmdq_buf(struct hinic_cmdqs *cmdqs,
+			 struct hinic_cmdq_buf *cmdq_buf)
+{
+	pci_pool_free(cmdqs->cmdq_buf_pool, cmdq_buf->buf, cmdq_buf->dma_addr);
+}
+
+static unsigned int cmdq_wqe_size_from_bdlen(enum bufdesc_len len)
+{
+	unsigned int wqe_size = 0;
+
+	switch (len) {
+	case BUFDESC_LCMD_LEN:
+		wqe_size = WQE_LCMD_SIZE;
+		break;
+	case BUFDESC_SCMD_LEN:
+		wqe_size = WQE_SCMD_SIZE;
+		break;
+	}
+
+	return wqe_size;
+}
+
+static void cmdq_set_sge_completion(struct hinic_cmdq_completion *completion,
+				    struct hinic_cmdq_buf *buf_out)
+{
+	struct hinic_sge_resp *sge_resp = &completion->sge_resp;
+
+	hinic_set_sge(&sge_resp->sge, buf_out->dma_addr, buf_out->size);
+}
+
+static void cmdq_prepare_wqe_ctrl(struct hinic_cmdq_wqe *wqe, int wrapped,
+				  enum hinic_cmd_ack_type ack_type,
+				  enum hinic_mod_type mod, u8 cmd, u16 prod_idx,
+				  enum completion_format complete_format,
+				  enum data_format data_format,
+				  enum bufdesc_len buf_len)
+{
+	struct hinic_cmdq_wqe_lcmd *wqe_lcmd;
+	struct hinic_cmdq_wqe_scmd *wqe_scmd;
+	enum ctrl_sect_len ctrl_len;
+	struct hinic_ctrl *ctrl;
+	u32 saved_data;
+
+	if (data_format == DATA_SGE) {
+		wqe_lcmd = &wqe->wqe_lcmd;
+
+		wqe_lcmd->status.status_info = 0;
+		ctrl = &wqe_lcmd->ctrl;
+		ctrl_len = CTRL_SECT_LEN;
+	} else {
+		wqe_scmd = &wqe->direct_wqe.wqe_scmd;
+
+		wqe_scmd->status.status_info = 0;
+		ctrl = &wqe_scmd->ctrl;
+		ctrl_len = CTRL_DIRECT_SECT_LEN;
+	}
+
+	ctrl->ctrl_info = HINIC_CMDQ_CTRL_SET(prod_idx, PI)             |
+			  HINIC_CMDQ_CTRL_SET(cmd, CMD)                 |
+			  HINIC_CMDQ_CTRL_SET(mod, MOD)                 |
+			  HINIC_CMDQ_CTRL_SET(ack_type, ACK_TYPE);
+
+	CMDQ_WQE_HEADER(wqe)->header_info =
+		HINIC_CMDQ_WQE_HEADER_SET(buf_len, BUFDESC_LEN)            |
+		HINIC_CMDQ_WQE_HEADER_SET(complete_format, COMPLETE_FMT)   |
+		HINIC_CMDQ_WQE_HEADER_SET(data_format, DATA_FMT)           |
+		HINIC_CMDQ_WQE_HEADER_SET(CEQ_SET, COMPLETE_REQ)           |
+		HINIC_CMDQ_WQE_HEADER_SET(COMPLETE_LEN, COMPLETE_SECT_LEN) |
+		HINIC_CMDQ_WQE_HEADER_SET(ctrl_len, CTRL_LEN)              |
+		HINIC_CMDQ_WQE_HEADER_SET(wrapped, TOGGLED_WRAPPED);
+
+	saved_data = CMDQ_WQE_HEADER(wqe)->saved_data;
+	saved_data = HINIC_SAVED_DATA_CLEAR(saved_data, ARM);
+
+	if ((cmd == CMDQ_SET_ARM_CMD) && (mod == HINIC_MOD_COMM))
+		CMDQ_WQE_HEADER(wqe)->saved_data |=
+						HINIC_SAVED_DATA_SET(1, ARM);
+	else
+		CMDQ_WQE_HEADER(wqe)->saved_data = saved_data;
+}
+
+static void cmdq_set_lcmd_bufdesc(struct hinic_cmdq_wqe_lcmd *wqe_lcmd,
+				  struct hinic_cmdq_buf *buf_in)
+{
+	hinic_set_sge(&wqe_lcmd->buf_desc.sge, buf_in->dma_addr, buf_in->size);
+}
+
+static void cmdq_set_direct_wqe_data(struct hinic_cmdq_direct_wqe *wqe,
+				     void *buf_in, u32 in_size)
+{
+	struct hinic_cmdq_wqe_scmd *wqe_scmd = &wqe->wqe_scmd;
+
+	wqe_scmd->buf_desc.buf_len = in_size;
+	memcpy(wqe_scmd->buf_desc.data, buf_in, in_size);
+}
+
+static void cmdq_set_lcmd_wqe(struct hinic_cmdq_wqe *wqe,
+			      enum cmdq_cmd_type cmd_type,
+			      struct hinic_cmdq_buf *buf_in,
+			      struct hinic_cmdq_buf *buf_out, int wrapped,
+			      enum hinic_cmd_ack_type ack_type,
+			      enum hinic_mod_type mod, u8 cmd, u16 prod_idx)
+{
+	struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &wqe->wqe_lcmd;
+	enum completion_format complete_format;
+
+	switch (cmd_type) {
+	case CMDQ_CMD_SYNC_SGE_RESP:
+		complete_format = COMPLETE_SGE;
+		cmdq_set_sge_completion(&wqe_lcmd->completion, buf_out);
+		break;
+	case CMDQ_CMD_SYNC_DIRECT_RESP:
+		complete_format = COMPLETE_DIRECT;
+		wqe_lcmd->completion.direct_resp = 0;
+		break;
+	}
+
+	cmdq_prepare_wqe_ctrl(wqe, wrapped, ack_type, mod, cmd,
+			      prod_idx, complete_format, DATA_SGE,
+			      BUFDESC_LCMD_LEN);
+
+	cmdq_set_lcmd_bufdesc(wqe_lcmd, buf_in);
+}
+
+static void cmdq_set_direct_wqe(struct hinic_cmdq_wqe *wqe,
+				enum cmdq_cmd_type cmd_type,
+				void *buf_in, u16 in_size,
+				struct hinic_cmdq_buf *buf_out, int wrapped,
+				enum hinic_cmd_ack_type ack_type,
+				enum hinic_mod_type mod, u8 cmd, u16 prod_idx)
+{
+	struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe;
+	enum completion_format complete_format;
+	struct hinic_cmdq_wqe_scmd *wqe_scmd;
+
+	wqe_scmd = &direct_wqe->wqe_scmd;
+
+	switch (cmd_type) {
+	case CMDQ_CMD_SYNC_SGE_RESP:
+		complete_format = COMPLETE_SGE;
+		cmdq_set_sge_completion(&wqe_scmd->completion, buf_out);
+		break;
+	case CMDQ_CMD_SYNC_DIRECT_RESP:
+		complete_format = COMPLETE_DIRECT;
+		wqe_scmd->completion.direct_resp = 0;
+		break;
+	}
+
+	cmdq_prepare_wqe_ctrl(wqe, wrapped, ack_type, mod, cmd, prod_idx,
+			      complete_format, DATA_DIRECT, BUFDESC_SCMD_LEN);
+
+	cmdq_set_direct_wqe_data(direct_wqe, buf_in, in_size);
+}
+
+static void cmdq_wqe_fill(void *dst, void *src)
+{
+	memcpy(dst + FIRST_DATA_TO_WRITE_LAST, src + FIRST_DATA_TO_WRITE_LAST,
+	       CMDQ_WQE_SIZE - FIRST_DATA_TO_WRITE_LAST);
+
+	wmb();          /* The first 8 bytes should be written last */
+
+	*(u64 *)dst = *(u64 *)src;
+}
+
+static void cmdq_fill_db(u32 *db_info,
+			 enum hinic_cmdq_type cmdq_type, u16 prod_idx)
+{
+	*db_info = HINIC_CMDQ_DB_INFO_SET(UPPER_8_BITS(prod_idx), HI_PROD_IDX) |
+		   HINIC_CMDQ_DB_INFO_SET(HINIC_CTRL_PATH, PATH)               |
+		   HINIC_CMDQ_DB_INFO_SET(cmdq_type, CMDQ_TYPE)                |
+		   HINIC_CMDQ_DB_INFO_SET(HINIC_DB_CMDQ_TYPE, DB_TYPE);
+}
+
+static void cmdq_set_db(struct hinic_cmdq *cmdq,
+			enum hinic_cmdq_type cmdq_type, u16 prod_idx)
+{
+	u32 db_info;
+
+	cmdq_fill_db(&db_info, cmdq_type, prod_idx);
+
+	/* The data that is written to HW should be in Big Endian Format */
+	db_info = cpu_to_be32(db_info);
+
+	wmb();  /* write all before the doorbell */
+
+	writel(db_info, CMDQ_DB_ADDR(cmdq->db_base, prod_idx));
+}
+
+static int cmdq_sync_cmd_direct_resp(struct hinic_cmdq *cmdq,
+				     enum hinic_mod_type mod, u8 cmd,
+				     struct hinic_cmdq_buf *buf_in,
+				     u64 *resp)
+{
+	struct hinic_cmdq_wqe *curr_cmdq_wqe, cmdq_wqe;
+	u16 curr_prod_idx, next_prod_idx;
+	int errcode, wrapped, num_wqebbs;
+	struct hinic_wq *wq = cmdq->wq;
+	struct hinic_hw_wqe *hw_wqe;
+	struct completion done;
+
+	/* Keep doorbell index correct. bh - for tasklet(ceq). */
+	spin_lock_bh(&cmdq->cmdq_lock);
+
+	/* WQE_SIZE = WQEBB_SIZE, we will get the wq element and not shadow*/
+	hw_wqe = hinic_get_wqe(wq, WQE_LCMD_SIZE, &curr_prod_idx);
+	if (IS_ERR(hw_wqe)) {
+		spin_unlock_bh(&cmdq->cmdq_lock);
+		return -EBUSY;
+	}
+
+	curr_cmdq_wqe = &hw_wqe->cmdq_wqe;
+
+	wrapped = cmdq->wrapped;
+
+	num_wqebbs = ALIGN(WQE_LCMD_SIZE, wq->wqebb_size) / wq->wqebb_size;
+	next_prod_idx = curr_prod_idx + num_wqebbs;
+	if (next_prod_idx >= wq->q_depth) {
+		cmdq->wrapped = !cmdq->wrapped;
+		next_prod_idx -= wq->q_depth;
+	}
+
+	cmdq->errcode[curr_prod_idx] = &errcode;
+
+	init_completion(&done);
+	cmdq->done[curr_prod_idx] = &done;
+
+	cmdq_set_lcmd_wqe(&cmdq_wqe, CMDQ_CMD_SYNC_DIRECT_RESP, buf_in, NULL,
+			  wrapped, HINIC_CMD_ACK_TYPE_CMDQ, mod, cmd,
+			  curr_prod_idx);
+
+	/* The data that is written to HW should be in Big Endian Format */
+	hinic_cpu_to_be32(&cmdq_wqe, WQE_LCMD_SIZE);
+
+	/* CMDQ WQE is not shadow, therefore wqe will be written to wq */
+	cmdq_wqe_fill(curr_cmdq_wqe, &cmdq_wqe);
+
+	cmdq_set_db(cmdq, HINIC_CMDQ_SYNC, next_prod_idx);
+
+	spin_unlock_bh(&cmdq->cmdq_lock);
+
+	if (!wait_for_completion_timeout(&done, CMDQ_TIMEOUT)) {
+		spin_lock_bh(&cmdq->cmdq_lock);
+
+		if (cmdq->errcode[curr_prod_idx] == &errcode)
+			cmdq->errcode[curr_prod_idx] = NULL;
+
+		if (cmdq->done[curr_prod_idx] == &done)
+			cmdq->done[curr_prod_idx] = NULL;
+
+		spin_unlock_bh(&cmdq->cmdq_lock);
+
+		return -ETIMEDOUT;
+	}
+
+	smp_rmb();      /* read error code after completion */
+
+	if (resp) {
+		struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &curr_cmdq_wqe->wqe_lcmd;
+
+		*resp = cpu_to_be64(wqe_lcmd->completion.direct_resp);
+	}
+
+	if (errcode != 0)
+		return -EFAULT;
+
+	return 0;
+}
+
+static int cmdq_set_arm_bit(struct hinic_cmdq *cmdq, void *buf_in,
+			    u16 in_size)
+{
+	struct hinic_cmdq_wqe *curr_cmdq_wqe, cmdq_wqe;
+	u16 curr_prod_idx, next_prod_idx;
+	struct hinic_wq *wq = cmdq->wq;
+	struct hinic_hw_wqe *hw_wqe;
+	int wrapped, num_wqebbs;
+
+	/* Keep doorbell index correct */
+	spin_lock(&cmdq->cmdq_lock);
+
+	/* WQE_SIZE = WQEBB_SIZE, we will get the wq element and not shadow*/
+	hw_wqe = hinic_get_wqe(wq, WQE_SCMD_SIZE, &curr_prod_idx);
+	if (IS_ERR(hw_wqe)) {
+		spin_unlock(&cmdq->cmdq_lock);
+		return -EBUSY;
+	}
+
+	curr_cmdq_wqe = &hw_wqe->cmdq_wqe;
+
+	wrapped = cmdq->wrapped;
+
+	num_wqebbs = ALIGN(WQE_SCMD_SIZE, wq->wqebb_size) / wq->wqebb_size;
+	next_prod_idx = curr_prod_idx + num_wqebbs;
+	if (next_prod_idx >= wq->q_depth) {
+		cmdq->wrapped = !cmdq->wrapped;
+		next_prod_idx -= wq->q_depth;
+	}
+
+	cmdq_set_direct_wqe(&cmdq_wqe, CMDQ_CMD_SYNC_DIRECT_RESP, buf_in,
+			    in_size, NULL, wrapped, HINIC_CMD_ACK_TYPE_CMDQ,
+			    HINIC_MOD_COMM, CMDQ_SET_ARM_CMD, curr_prod_idx);
+
+	/* The data that is written to HW should be in Big Endian Format */
+	hinic_cpu_to_be32(&cmdq_wqe, WQE_SCMD_SIZE);
+
+	/* cmdq wqe is not shadow, therefore wqe will be written to wq */
+	cmdq_wqe_fill(curr_cmdq_wqe, &cmdq_wqe);
+
+	cmdq_set_db(cmdq, HINIC_CMDQ_SYNC, next_prod_idx);
+
+	spin_unlock(&cmdq->cmdq_lock);
+	return 0;
+}
+
+static int cmdq_params_valid(struct hinic_cmdq_buf *buf_in)
+{
+	if (buf_in->size > HINIC_CMDQ_MAX_DATA_SIZE)
+		return -EINVAL;
+
+	return 0;
+}
+
+/**
+ * hinic_cmdq_direct_resp - send command with direct data as resp
+ * @cmdqs: the cmdqs
+ * @mod: module on the card that will handle the command
+ * @cmd: the command
+ * @buf_in: the buffer for the command
+ * @resp: the response to return
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_cmdq_direct_resp(struct hinic_cmdqs *cmdqs,
+			   enum hinic_mod_type mod, u8 cmd,
+			   struct hinic_cmdq_buf *buf_in, u64 *resp)
+{
+	struct hinic_hwif *hwif = cmdqs->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	int err;
+
+	err = cmdq_params_valid(buf_in);
+	if (err) {
+		dev_err(&pdev->dev, "Invalid CMDQ parameters\n");
+		return err;
+	}
+
+	return cmdq_sync_cmd_direct_resp(&cmdqs->cmdq[HINIC_CMDQ_SYNC],
+					 mod, cmd, buf_in, resp);
+}
+
+/**
+ * hinic_set_arm_bit - set arm bit for enable interrupt again
+ * @cmdqs: the cmdqs
+ * @q_type: type of queue to set the arm bit for
+ * @q_id: the queue number
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_set_arm_bit(struct hinic_cmdqs *cmdqs,
+		      enum hinic_set_arm_qtype q_type, u32 q_id)
+{
+	struct hinic_cmdq *cmdq = &cmdqs->cmdq[HINIC_CMDQ_SYNC];
+	struct hinic_hwif *hwif = cmdqs->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	struct hinic_cmdq_arm_bit arm_bit;
+	int err;
+
+	arm_bit.q_type = q_type;
+	arm_bit.q_id   = q_id;
+
+	err = cmdq_set_arm_bit(cmdq, &arm_bit, sizeof(arm_bit));
+	if (err) {
+		dev_err(&pdev->dev, "Failed to set arm for qid %d\n", q_id);
+		return err;
+	}
+
+	return 0;
+}
+
+static void clear_wqe_complete_bit(struct hinic_cmdq *cmdq,
+				   struct hinic_cmdq_wqe *wqe)
+{
+	u32 header_info = be32_to_cpu(CMDQ_WQE_HEADER(wqe)->header_info);
+	unsigned int bufdesc_len, wqe_size;
+	struct hinic_ctrl *ctrl;
+
+	bufdesc_len = HINIC_CMDQ_WQE_HEADER_GET(header_info, BUFDESC_LEN);
+	wqe_size = cmdq_wqe_size_from_bdlen(bufdesc_len);
+	if (wqe_size == WQE_LCMD_SIZE) {
+		struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &wqe->wqe_lcmd;
+
+		ctrl = &wqe_lcmd->ctrl;
+	} else {
+		struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe;
+		struct hinic_cmdq_wqe_scmd *wqe_scmd;
+
+		wqe_scmd = &direct_wqe->wqe_scmd;
+		ctrl = &wqe_scmd->ctrl;
+	}
+
+	/* clear HW busy bit */
+	ctrl->ctrl_info = 0;
+
+	wmb();  /* verify wqe is clear */
+}
+
+/**
+ * cmdq_arm_ceq_handler - cmdq completion event handler for arm command
+ * @cmdq: the cmdq of the arm command
+ * @wqe: the wqe of the arm command
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int cmdq_arm_ceq_handler(struct hinic_cmdq *cmdq,
+				struct hinic_cmdq_wqe *wqe)
+{
+	struct hinic_cmdq_direct_wqe *direct_wqe = &wqe->direct_wqe;
+	struct hinic_cmdq_wqe_scmd *wqe_scmd;
+	struct hinic_ctrl *ctrl;
+	u32 ctrl_info;
+
+	wqe_scmd = &direct_wqe->wqe_scmd;
+	ctrl = &wqe_scmd->ctrl;
+	ctrl_info = be32_to_cpu(ctrl->ctrl_info);
+
+	/* HW should toggle the HW BUSY BIT */
+	if (!CMDQ_WQE_COMPLETED(ctrl_info))
+		return -EBUSY;
+
+	clear_wqe_complete_bit(cmdq, wqe);
+
+	hinic_put_wqe(cmdq->wq, WQE_SCMD_SIZE);
+	return 0;
+}
+
+static void cmdq_update_errcode(struct hinic_cmdq *cmdq, u16 prod_idx,
+				int errcode)
+{
+	if (cmdq->errcode[prod_idx])
+		*cmdq->errcode[prod_idx] = errcode;
+}
+
+/**
+ * cmdq_arm_ceq_handler - cmdq completion event handler for sync command
+ * @cmdq: the cmdq of the command
+ * @cons_idx: the consumer index to update the error code for
+ * @errcode: the error code
+ **/
+static void cmdq_sync_cmd_handler(struct hinic_cmdq *cmdq, u16 cons_idx,
+				  int errcode)
+{
+	u16 prod_idx = cons_idx;
+
+	spin_lock(&cmdq->cmdq_lock);
+	cmdq_update_errcode(cmdq, prod_idx, errcode);
+
+	wmb();  /* write all before update for the command request */
+
+	if (cmdq->done[prod_idx])
+		complete(cmdq->done[prod_idx]);
+	spin_unlock(&cmdq->cmdq_lock);
+}
+
+static int cmdq_cmd_ceq_handler(struct hinic_cmdq *cmdq, u16 ci,
+				struct hinic_cmdq_wqe *cmdq_wqe)
+{
+	struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &cmdq_wqe->wqe_lcmd;
+	struct hinic_status *status = &wqe_lcmd->status;
+	struct hinic_ctrl *ctrl = &wqe_lcmd->ctrl;
+	int errcode;
+
+	if (!CMDQ_WQE_COMPLETED(be32_to_cpu(ctrl->ctrl_info)))
+		return -EBUSY;
+
+	errcode = CMDQ_WQE_ERRCODE_GET(be32_to_cpu(status->status_info), VAL);
+
+	cmdq_sync_cmd_handler(cmdq, ci, errcode);
+
+	clear_wqe_complete_bit(cmdq, cmdq_wqe);
+	hinic_put_wqe(cmdq->wq, WQE_LCMD_SIZE);
+	return 0;
+}
+
+/**
+ * cmdq_ceq_handler - cmdq completion event handler
+ * @handle: private data for the handler(cmdqs)
+ * @ceqe_data: ceq element data
+ **/
+static void cmdq_ceq_handler(void *handle, u32 ceqe_data)
+{
+	enum hinic_cmdq_type cmdq_type = CMDQ_CEQE_GET(ceqe_data, TYPE);
+	struct hinic_cmdqs *cmdqs = (struct hinic_cmdqs *)handle;
+	struct hinic_cmdq *cmdq = &cmdqs->cmdq[cmdq_type];
+	struct hinic_cmdq_header *header;
+	struct hinic_hw_wqe *hw_wqe;
+	int err, set_arm = 0;
+	u32 saved_data;
+	u16 ci;
+
+	/* Read the smallest wqe size for getting wqe size */
+	while ((hw_wqe = hinic_read_wqe(cmdq->wq, WQE_SCMD_SIZE, &ci))) {
+		if (IS_ERR(hw_wqe))
+			break;
+
+		header = CMDQ_WQE_HEADER(&hw_wqe->cmdq_wqe);
+		saved_data = be32_to_cpu(header->saved_data);
+
+		if (HINIC_SAVED_DATA_GET(saved_data, ARM)) {
+			/* arm_bit was set until here */
+			set_arm = 0;
+
+			if (cmdq_arm_ceq_handler(cmdq, &hw_wqe->cmdq_wqe))
+				break;
+		} else {
+			set_arm = 1;
+
+			hw_wqe = hinic_read_wqe(cmdq->wq, WQE_LCMD_SIZE, &ci);
+			if (IS_ERR(hw_wqe))
+				break;
+
+			if (cmdq_cmd_ceq_handler(cmdq, ci, &hw_wqe->cmdq_wqe))
+				break;
+		}
+	}
+
+	if (set_arm) {
+		struct hinic_hwif *hwif = cmdqs->hwif;
+		struct pci_dev *pdev = hwif->pdev;
+
+		err = hinic_set_arm_bit(cmdqs, HINIC_SET_ARM_CMDQ, cmdq_type);
+		if (err)
+			dev_err(&pdev->dev, "Failed to set arm for CMDQ\n");
+	}
+}
+
+/**
+ * cmdq_init_queue_ctxt - init the queue ctxt of a cmdq
+ * @cmdq_ctxt: cmdq ctxt to initialize
+ * @cmdq: the cmdq
+ * @cmdq_pages: the memory of the queue
+ **/
+static void cmdq_init_queue_ctxt(struct hinic_cmdq_ctxt *cmdq_ctxt,
+				 struct hinic_cmdq *cmdq,
+				 struct hinic_cmdq_pages *cmdq_pages)
+{
+	struct hinic_cmdq_ctxt_info *ctxt_info = &cmdq_ctxt->ctxt_info;
+	u64 wq_first_page_paddr, cmdq_first_block_paddr, pfn;
+	struct hinic_cmdqs *cmdqs = cmdq_to_cmdqs(cmdq);
+	struct hinic_wq *wq = cmdq->wq;
+
+	/* The data in the HW is in Big Endian Format */
+	wq_first_page_paddr = be64_to_cpu(*wq->block_vaddr);
+
+	pfn = CMDQ_PFN(wq_first_page_paddr, wq->wq_page_size);
+
+	ctxt_info->curr_wqe_page_pfn =
+		HINIC_CMDQ_CTXT_PAGE_INFO_SET(pfn, CURR_WQE_PAGE_PFN)   |
+		HINIC_CMDQ_CTXT_PAGE_INFO_SET(HINIC_CEQ_ID_CMDQ, EQ_ID) |
+		HINIC_CMDQ_CTXT_PAGE_INFO_SET(1, CEQ_ARM)               |
+		HINIC_CMDQ_CTXT_PAGE_INFO_SET(1, CEQ_EN)                |
+		HINIC_CMDQ_CTXT_PAGE_INFO_SET(cmdq->wrapped, WRAPPED);
+
+	/* block PFN - Read Modify Write */
+	cmdq_first_block_paddr = cmdq_pages->page_paddr;
+
+	pfn = CMDQ_PFN(cmdq_first_block_paddr, wq->wq_page_size);
+
+	ctxt_info->wq_block_pfn =
+		HINIC_CMDQ_CTXT_BLOCK_INFO_SET(pfn, WQ_BLOCK_PFN) |
+		HINIC_CMDQ_CTXT_BLOCK_INFO_SET(atomic_read(&wq->cons_idx), CI);
+
+	cmdq_ctxt->func_idx = HINIC_HWIF_FUNC_IDX(cmdqs->hwif);
+	cmdq_ctxt->cmdq_type  = cmdq->cmdq_type;
+}
+
+/**
+ * init_cmdq - initialize cmdq
+ * @cmdq: the cmdq
+ * @wq: the wq attaced to the cmdq
+ * @q_type: the cmdq type of the cmdq
+ * @db_area: doorbell area for the cmdq
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int init_cmdq(struct hinic_cmdq *cmdq, struct hinic_wq *wq,
+		     enum hinic_cmdq_type q_type, void __iomem *db_area)
+{
+	int err;
+
+	cmdq->wq = wq;
+	cmdq->cmdq_type = q_type;
+	cmdq->wrapped = 1;
+
+	spin_lock_init(&cmdq->cmdq_lock);
+
+	cmdq->done = vzalloc(wq->q_depth * sizeof(*cmdq->done));
+	if (!cmdq->done)
+		return -ENOMEM;
+
+	cmdq->errcode = vzalloc(wq->q_depth * sizeof(*cmdq->errcode));
+	if (!cmdq->errcode) {
+		err = -ENOMEM;
+		goto err_errcode;
+	}
+
+	cmdq->db_base = db_area + CMDQ_DB_OFF;
+	return 0;
+
+err_errcode:
+	vfree(cmdq->done);
+	return err;
+}
+
+/**
+ * free_cmdq - Free cmdq
+ * @cmdq: the cmdq to free
+ **/
+static void free_cmdq(struct hinic_cmdq *cmdq)
+{
+	vfree(cmdq->errcode);
+	vfree(cmdq->done);
+}
+
+/**
+ * init_cmdqs_ctxt - write the cmdq ctxt to HW after init all cmdq
+ * @hwdev: the NIC HW device
+ * @cmdqs: cmdqs to write the ctxts for
+ * &db_area: db_area for all the cmdqs
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int init_cmdqs_ctxt(struct hinic_hwdev *hwdev,
+			   struct hinic_cmdqs *cmdqs, void __iomem **db_area)
+{
+	struct hinic_hwif *hwif = hwdev->hwif;
+	enum hinic_cmdq_type type, cmdq_type;
+	struct hinic_cmdq_ctxt *cmdq_ctxts;
+	struct pci_dev *pdev = hwif->pdev;
+	struct hinic_pfhwdev *pfhwdev;
+	size_t cmdq_ctxts_size;
+	int err;
+
+	if (!HINIC_IS_PF(hwif) && !HINIC_IS_PPF(hwif)) {
+		dev_err(&pdev->dev, "Unsupported PCI function type\n");
+		return -EINVAL;
+	}
+
+	cmdq_ctxts_size = HINIC_MAX_CMDQ_TYPES * sizeof(*cmdq_ctxts);
+	cmdq_ctxts = devm_kzalloc(&pdev->dev, cmdq_ctxts_size, GFP_KERNEL);
+	if (!cmdq_ctxts)
+		return -ENOMEM;
+
+	pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev);
+
+	cmdq_type = HINIC_CMDQ_SYNC;
+	for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) {
+		err = init_cmdq(&cmdqs->cmdq[cmdq_type],
+				&cmdqs->saved_wqs[cmdq_type], cmdq_type,
+				db_area[cmdq_type]);
+		if (err) {
+			dev_err(&pdev->dev, "Failed to initialize cmdq\n");
+			goto err_init_cmdq;
+		}
+
+		cmdq_init_queue_ctxt(&cmdq_ctxts[cmdq_type],
+				     &cmdqs->cmdq[cmdq_type],
+				     &cmdqs->cmdq_pages);
+	}
+
+	/* Write the CMDQ ctxts */
+	cmdq_type = HINIC_CMDQ_SYNC;
+	for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) {
+		err = hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_COMM,
+					HINIC_COMM_CMD_CMDQ_CTXT_SET,
+					&cmdq_ctxts[cmdq_type],
+					sizeof(cmdq_ctxts[cmdq_type]),
+					NULL, NULL, HINIC_MGMT_MSG_SYNC);
+		if (err) {
+			dev_err(&pdev->dev, "Failed to set CMDQ CTXT type = %d\n",
+				cmdq_type);
+			goto err_write_cmdq_ctxt;
+		}
+	}
+
+	devm_kfree(&pdev->dev, cmdq_ctxts);
+	return 0;
+
+err_write_cmdq_ctxt:
+	cmdq_type = HINIC_MAX_CMDQ_TYPES;
+
+err_init_cmdq:
+	for (type = HINIC_CMDQ_SYNC; type < cmdq_type; type++)
+		free_cmdq(&cmdqs->cmdq[type]);
+
+	devm_kfree(&pdev->dev, cmdq_ctxts);
+	return err;
+}
+
+/**
+ * hinic_init_cmdqs - init all cmdqs
+ * @cmdqs: cmdqs to init
+ * @hwif: HW interface for accessing cmdqs
+ * @db_area: doorbell areas for all the cmdqs
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_init_cmdqs(struct hinic_cmdqs *cmdqs, struct hinic_hwif *hwif,
+		     void __iomem **db_area)
+{
+	struct hinic_func_to_io *func_to_io = cmdqs_to_func_to_io(cmdqs);
+	struct pci_dev *pdev = hwif->pdev;
+	struct hinic_hwdev *hwdev;
+	size_t saved_wqs_size;
+	u16 max_wqe_size;
+	int err;
+
+	cmdqs->hwif = hwif;
+	cmdqs->cmdq_buf_pool = pci_pool_create("hinic_cmdq", pdev,
+					       HINIC_CMDQ_BUF_SIZE,
+					       HINIC_CMDQ_BUF_SIZE, 0);
+	if (!cmdqs->cmdq_buf_pool)
+		return -ENOMEM;
+
+	saved_wqs_size = HINIC_MAX_CMDQ_TYPES * sizeof(struct hinic_wq);
+	cmdqs->saved_wqs = devm_kzalloc(&pdev->dev, saved_wqs_size, GFP_KERNEL);
+	if (!cmdqs->saved_wqs) {
+		err = -ENOMEM;
+		goto err_saved_wqs;
+	}
+
+	max_wqe_size = WQE_LCMD_SIZE;
+	err = hinic_wqs_cmdq_alloc(&cmdqs->cmdq_pages, cmdqs->saved_wqs, hwif,
+				   HINIC_MAX_CMDQ_TYPES, CMDQ_WQEBB_SIZE,
+				   CMDQ_WQ_PAGE_SIZE, CMDQ_DEPTH, max_wqe_size);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to allocate CMDQ wqs\n");
+		goto err_cmdq_wqs;
+	}
+
+	hwdev = container_of(func_to_io, struct hinic_hwdev, func_to_io);
+	err = init_cmdqs_ctxt(hwdev, cmdqs, db_area);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to write cmdq ctxt\n");
+		goto err_cmdq_ctxt;
+	}
+
+	hinic_ceq_register_cb(&func_to_io->ceqs, HINIC_CEQ_CMDQ, cmdqs,
+			      cmdq_ceq_handler);
+	return 0;
+
+err_cmdq_ctxt:
+	hinic_wqs_cmdq_free(&cmdqs->cmdq_pages, cmdqs->saved_wqs,
+			    HINIC_MAX_CMDQ_TYPES);
+
+err_cmdq_wqs:
+	devm_kfree(&pdev->dev, cmdqs->saved_wqs);
+
+err_saved_wqs:
+	pci_pool_destroy(cmdqs->cmdq_buf_pool);
+	return err;
+}
+
+/**
+ * hinic_free_cmdqs - free all cmdqs
+ * @cmdqs: cmdqs to free
+ **/
+void hinic_free_cmdqs(struct hinic_cmdqs *cmdqs)
+{
+	struct hinic_func_to_io *func_to_io = cmdqs_to_func_to_io(cmdqs);
+	struct hinic_hwif *hwif = cmdqs->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	enum hinic_cmdq_type cmdq_type;
+
+	hinic_ceq_unregister_cb(&func_to_io->ceqs, HINIC_CEQ_CMDQ);
+
+	cmdq_type = HINIC_CMDQ_SYNC;
+	for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++)
+		free_cmdq(&cmdqs->cmdq[cmdq_type]);
+
+	hinic_wqs_cmdq_free(&cmdqs->cmdq_pages, cmdqs->saved_wqs,
+			    HINIC_MAX_CMDQ_TYPES);
+
+	devm_kfree(&pdev->dev, cmdqs->saved_wqs);
+
+	pci_pool_destroy(cmdqs->cmdq_buf_pool);
+}
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.h
new file mode 100644
index 000000000000..b35583400cb6
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.h
@@ -0,0 +1,187 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#ifndef HINIC_CMDQ_H
+#define HINIC_CMDQ_H
+
+#include <linux/types.h>
+#include <linux/spinlock.h>
+#include <linux/completion.h>
+#include <linux/pci.h>
+
+#include "hinic_hw_if.h"
+#include "hinic_hw_wq.h"
+
+#define HINIC_CMDQ_CTXT_CURR_WQE_PAGE_PFN_SHIFT         0
+#define HINIC_CMDQ_CTXT_EQ_ID_SHIFT                     56
+#define HINIC_CMDQ_CTXT_CEQ_ARM_SHIFT                   61
+#define HINIC_CMDQ_CTXT_CEQ_EN_SHIFT                    62
+#define HINIC_CMDQ_CTXT_WRAPPED_SHIFT                   63
+
+#define HINIC_CMDQ_CTXT_CURR_WQE_PAGE_PFN_MASK          0xFFFFFFFFFFFFF
+#define HINIC_CMDQ_CTXT_EQ_ID_MASK                      0x1F
+#define HINIC_CMDQ_CTXT_CEQ_ARM_MASK                    0x1
+#define HINIC_CMDQ_CTXT_CEQ_EN_MASK                     0x1
+#define HINIC_CMDQ_CTXT_WRAPPED_MASK                    0x1
+
+#define HINIC_CMDQ_CTXT_PAGE_INFO_SET(val, member)      \
+			(((u64)(val) & HINIC_CMDQ_CTXT_##member##_MASK) \
+			 << HINIC_CMDQ_CTXT_##member##_SHIFT)
+
+#define HINIC_CMDQ_CTXT_PAGE_INFO_CLEAR(val, member)    \
+			((val) & (~((u64)HINIC_CMDQ_CTXT_##member##_MASK \
+			 << HINIC_CMDQ_CTXT_##member##_SHIFT)))
+
+#define HINIC_CMDQ_CTXT_WQ_BLOCK_PFN_SHIFT              0
+#define HINIC_CMDQ_CTXT_CI_SHIFT                        52
+
+#define HINIC_CMDQ_CTXT_WQ_BLOCK_PFN_MASK               0xFFFFFFFFFFFFF
+#define HINIC_CMDQ_CTXT_CI_MASK                         0xFFF
+
+#define HINIC_CMDQ_CTXT_BLOCK_INFO_SET(val, member)     \
+			(((u64)(val) & HINIC_CMDQ_CTXT_##member##_MASK) \
+			 << HINIC_CMDQ_CTXT_##member##_SHIFT)
+
+#define HINIC_CMDQ_CTXT_BLOCK_INFO_CLEAR(val, member)   \
+			((val) & (~((u64)HINIC_CMDQ_CTXT_##member##_MASK \
+			 << HINIC_CMDQ_CTXT_##member##_SHIFT)))
+
+#define HINIC_SAVED_DATA_ARM_SHIFT                      31
+
+#define HINIC_SAVED_DATA_ARM_MASK                       0x1
+
+#define HINIC_SAVED_DATA_SET(val, member)               \
+			(((u32)(val) & HINIC_SAVED_DATA_##member##_MASK) \
+			 << HINIC_SAVED_DATA_##member##_SHIFT)
+
+#define HINIC_SAVED_DATA_GET(val, member)               \
+			(((val) >> HINIC_SAVED_DATA_##member##_SHIFT) \
+			 & HINIC_SAVED_DATA_##member##_MASK)
+
+#define HINIC_SAVED_DATA_CLEAR(val, member)             \
+			((val) & (~(HINIC_SAVED_DATA_##member##_MASK \
+			 << HINIC_SAVED_DATA_##member##_SHIFT)))
+
+#define HINIC_CMDQ_DB_INFO_HI_PROD_IDX_SHIFT            0
+#define HINIC_CMDQ_DB_INFO_PATH_SHIFT                   23
+#define HINIC_CMDQ_DB_INFO_CMDQ_TYPE_SHIFT              24
+#define HINIC_CMDQ_DB_INFO_DB_TYPE_SHIFT                27
+
+#define HINIC_CMDQ_DB_INFO_HI_PROD_IDX_MASK             0xFF
+#define HINIC_CMDQ_DB_INFO_PATH_MASK                    0x1
+#define HINIC_CMDQ_DB_INFO_CMDQ_TYPE_MASK               0x7
+#define HINIC_CMDQ_DB_INFO_DB_TYPE_MASK                 0x1F
+
+#define HINIC_CMDQ_DB_INFO_SET(val, member)             \
+			(((u32)(val) & HINIC_CMDQ_DB_INFO_##member##_MASK) \
+			 << HINIC_CMDQ_DB_INFO_##member##_SHIFT)
+
+#define HINIC_CMDQ_BUF_SIZE             2048
+
+#define HINIC_CMDQ_BUF_HW_RSVD          8
+#define HINIC_CMDQ_MAX_DATA_SIZE        (HINIC_CMDQ_BUF_SIZE - \
+					 HINIC_CMDQ_BUF_HW_RSVD)
+
+enum hinic_cmdq_type {
+	HINIC_CMDQ_SYNC,
+
+	HINIC_MAX_CMDQ_TYPES,
+};
+
+enum hinic_set_arm_qtype {
+	HINIC_SET_ARM_CMDQ,
+};
+
+enum hinic_cmd_ack_type {
+	HINIC_CMD_ACK_TYPE_CMDQ,
+};
+
+struct hinic_cmdq_buf {
+	void            *buf;
+	dma_addr_t      dma_addr;
+	size_t          size;
+};
+
+struct hinic_cmdq_arm_bit {
+	u32     q_type;
+	u32     q_id;
+};
+
+struct hinic_cmdq_ctxt_info {
+	u64     curr_wqe_page_pfn;
+	u64     wq_block_pfn;
+};
+
+struct hinic_cmdq_ctxt {
+	u8      status;
+	u8      version;
+	u8      rsvd0[6];
+
+	u16     func_idx;
+	u8      cmdq_type;
+	u8      rsvd1[1];
+
+	u8      rsvd2[4];
+
+	struct hinic_cmdq_ctxt_info ctxt_info;
+};
+
+struct hinic_cmdq {
+	struct hinic_wq         *wq;
+
+	enum hinic_cmdq_type    cmdq_type;
+	int                     wrapped;
+
+	/* Lock for keeping the doorbell order */
+	spinlock_t              cmdq_lock;
+
+	struct completion       **done;
+	int                     **errcode;
+
+	/* doorbell area */
+	void __iomem            *db_base;
+};
+
+struct hinic_cmdqs {
+	struct hinic_hwif       *hwif;
+
+	struct pci_pool         *cmdq_buf_pool;
+
+	struct hinic_wq         *saved_wqs;
+
+	struct hinic_cmdq_pages cmdq_pages;
+
+	struct hinic_cmdq       cmdq[HINIC_MAX_CMDQ_TYPES];
+};
+
+int hinic_alloc_cmdq_buf(struct hinic_cmdqs *cmdqs,
+			 struct hinic_cmdq_buf *cmdq_buf);
+
+void hinic_free_cmdq_buf(struct hinic_cmdqs *cmdqs,
+			 struct hinic_cmdq_buf *cmdq_buf);
+
+int hinic_cmdq_direct_resp(struct hinic_cmdqs *cmdqs,
+			   enum hinic_mod_type mod, u8 cmd,
+			   struct hinic_cmdq_buf *buf_in, u64 *out_param);
+
+int hinic_set_arm_bit(struct hinic_cmdqs *cmdqs,
+		      enum hinic_set_arm_qtype q_type, u32 q_id);
+
+int hinic_init_cmdqs(struct hinic_cmdqs *cmdqs, struct hinic_hwif *hwif,
+		     void __iomem **db_area);
+
+void hinic_free_cmdqs(struct hinic_cmdqs *cmdqs);
+
+#endif
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_csr.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_csr.h
new file mode 100644
index 000000000000..f39b184f674d
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_csr.h
@@ -0,0 +1,149 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#ifndef HINIC_HW_CSR_H
+#define HINIC_HW_CSR_H
+
+/* HW interface registers */
+#define HINIC_CSR_FUNC_ATTR0_ADDR                       0x0
+#define HINIC_CSR_FUNC_ATTR1_ADDR                       0x4
+
+#define HINIC_CSR_FUNC_ATTR4_ADDR                       0x10
+#define HINIC_CSR_FUNC_ATTR5_ADDR                       0x14
+
+#define HINIC_DMA_ATTR_BASE                             0xC80
+#define HINIC_ELECTION_BASE                             0x4200
+
+#define HINIC_DMA_ATTR_STRIDE                           0x4
+#define HINIC_CSR_DMA_ATTR_ADDR(idx)                    \
+	(HINIC_DMA_ATTR_BASE + (idx) * HINIC_DMA_ATTR_STRIDE)
+
+#define HINIC_PPF_ELECTION_STRIDE                       0x4
+#define HINIC_CSR_MAX_PORTS                             4
+
+#define HINIC_CSR_PPF_ELECTION_ADDR(idx)                \
+	(HINIC_ELECTION_BASE +  (idx) * HINIC_PPF_ELECTION_STRIDE)
+
+/* API CMD registers */
+#define HINIC_CSR_API_CMD_BASE                          0xF000
+
+#define HINIC_CSR_API_CMD_STRIDE                        0x100
+
+#define HINIC_CSR_API_CMD_CHAIN_HEAD_HI_ADDR(idx)       \
+	(HINIC_CSR_API_CMD_BASE + 0x0 + (idx) * HINIC_CSR_API_CMD_STRIDE)
+
+#define HINIC_CSR_API_CMD_CHAIN_HEAD_LO_ADDR(idx)       \
+	(HINIC_CSR_API_CMD_BASE + 0x4 + (idx) * HINIC_CSR_API_CMD_STRIDE)
+
+#define HINIC_CSR_API_CMD_STATUS_HI_ADDR(idx)           \
+	(HINIC_CSR_API_CMD_BASE + 0x8 + (idx) * HINIC_CSR_API_CMD_STRIDE)
+
+#define HINIC_CSR_API_CMD_STATUS_LO_ADDR(idx)           \
+	(HINIC_CSR_API_CMD_BASE + 0xC + (idx) * HINIC_CSR_API_CMD_STRIDE)
+
+#define HINIC_CSR_API_CMD_CHAIN_NUM_CELLS_ADDR(idx)     \
+	(HINIC_CSR_API_CMD_BASE + 0x10 + (idx) * HINIC_CSR_API_CMD_STRIDE)
+
+#define HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(idx)          \
+	(HINIC_CSR_API_CMD_BASE + 0x14 + (idx) * HINIC_CSR_API_CMD_STRIDE)
+
+#define HINIC_CSR_API_CMD_CHAIN_PI_ADDR(idx)            \
+	(HINIC_CSR_API_CMD_BASE + 0x1C + (idx) * HINIC_CSR_API_CMD_STRIDE)
+
+#define HINIC_CSR_API_CMD_CHAIN_REQ_ADDR(idx)           \
+	(HINIC_CSR_API_CMD_BASE + 0x20 + (idx) * HINIC_CSR_API_CMD_STRIDE)
+
+#define HINIC_CSR_API_CMD_STATUS_ADDR(idx)              \
+	(HINIC_CSR_API_CMD_BASE + 0x30 + (idx) * HINIC_CSR_API_CMD_STRIDE)
+
+/* MSI-X registers */
+#define HINIC_CSR_MSIX_CTRL_BASE                        0x2000
+#define HINIC_CSR_MSIX_CNT_BASE                         0x2004
+
+#define HINIC_CSR_MSIX_STRIDE                           0x8
+
+#define HINIC_CSR_MSIX_CTRL_ADDR(idx)                   \
+	(HINIC_CSR_MSIX_CTRL_BASE + (idx) * HINIC_CSR_MSIX_STRIDE)
+
+#define HINIC_CSR_MSIX_CNT_ADDR(idx)                    \
+	(HINIC_CSR_MSIX_CNT_BASE + (idx) * HINIC_CSR_MSIX_STRIDE)
+
+/* EQ registers */
+#define HINIC_AEQ_MTT_OFF_BASE_ADDR                     0x200
+#define HINIC_CEQ_MTT_OFF_BASE_ADDR                     0x400
+
+#define HINIC_EQ_MTT_OFF_STRIDE                         0x40
+
+#define HINIC_CSR_AEQ_MTT_OFF(id)                       \
+	(HINIC_AEQ_MTT_OFF_BASE_ADDR + (id) * HINIC_EQ_MTT_OFF_STRIDE)
+
+#define HINIC_CSR_CEQ_MTT_OFF(id)                       \
+	(HINIC_CEQ_MTT_OFF_BASE_ADDR + (id) * HINIC_EQ_MTT_OFF_STRIDE)
+
+#define HINIC_CSR_EQ_PAGE_OFF_STRIDE                    8
+
+#define HINIC_CSR_AEQ_HI_PHYS_ADDR_REG(q_id, pg_num)    \
+	(HINIC_CSR_AEQ_MTT_OFF(q_id) + \
+	 (pg_num) * HINIC_CSR_EQ_PAGE_OFF_STRIDE)
+
+#define HINIC_CSR_CEQ_HI_PHYS_ADDR_REG(q_id, pg_num)    \
+	(HINIC_CSR_CEQ_MTT_OFF(q_id) +          \
+	 (pg_num) * HINIC_CSR_EQ_PAGE_OFF_STRIDE)
+
+#define HINIC_CSR_AEQ_LO_PHYS_ADDR_REG(q_id, pg_num)    \
+	(HINIC_CSR_AEQ_MTT_OFF(q_id) + \
+	 (pg_num) * HINIC_CSR_EQ_PAGE_OFF_STRIDE + 4)
+
+#define HINIC_CSR_CEQ_LO_PHYS_ADDR_REG(q_id, pg_num)    \
+	(HINIC_CSR_CEQ_MTT_OFF(q_id) +  \
+	 (pg_num) * HINIC_CSR_EQ_PAGE_OFF_STRIDE + 4)
+
+#define HINIC_AEQ_CTRL_0_ADDR_BASE                      0xE00
+#define HINIC_AEQ_CTRL_1_ADDR_BASE                      0xE04
+#define HINIC_AEQ_CONS_IDX_ADDR_BASE                    0xE08
+#define HINIC_AEQ_PROD_IDX_ADDR_BASE                    0xE0C
+
+#define HINIC_CEQ_CTRL_0_ADDR_BASE                      0x1000
+#define HINIC_CEQ_CTRL_1_ADDR_BASE                      0x1004
+#define HINIC_CEQ_CONS_IDX_ADDR_BASE                    0x1008
+#define HINIC_CEQ_PROD_IDX_ADDR_BASE                    0x100C
+
+#define HINIC_EQ_OFF_STRIDE                             0x80
+
+#define HINIC_CSR_AEQ_CTRL_0_ADDR(idx)                  \
+	(HINIC_AEQ_CTRL_0_ADDR_BASE + (idx) * HINIC_EQ_OFF_STRIDE)
+
+#define HINIC_CSR_AEQ_CTRL_1_ADDR(idx)                  \
+	(HINIC_AEQ_CTRL_1_ADDR_BASE + (idx) * HINIC_EQ_OFF_STRIDE)
+
+#define HINIC_CSR_AEQ_CONS_IDX_ADDR(idx)                \
+	(HINIC_AEQ_CONS_IDX_ADDR_BASE + (idx) * HINIC_EQ_OFF_STRIDE)
+
+#define HINIC_CSR_AEQ_PROD_IDX_ADDR(idx)                \
+	(HINIC_AEQ_PROD_IDX_ADDR_BASE + (idx) * HINIC_EQ_OFF_STRIDE)
+
+#define HINIC_CSR_CEQ_CTRL_0_ADDR(idx)                  \
+	(HINIC_CEQ_CTRL_0_ADDR_BASE + (idx) * HINIC_EQ_OFF_STRIDE)
+
+#define HINIC_CSR_CEQ_CTRL_1_ADDR(idx)                  \
+	(HINIC_CEQ_CTRL_1_ADDR_BASE + (idx) * HINIC_EQ_OFF_STRIDE)
+
+#define HINIC_CSR_CEQ_CONS_IDX_ADDR(idx)                \
+	(HINIC_CEQ_CONS_IDX_ADDR_BASE + (idx) * HINIC_EQ_OFF_STRIDE)
+
+#define HINIC_CSR_CEQ_PROD_IDX_ADDR(idx)                \
+	(HINIC_CEQ_PROD_IDX_ADDR_BASE + (idx) * HINIC_EQ_OFF_STRIDE)
+
+#endif
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c
new file mode 100644
index 000000000000..79b567447084
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c
@@ -0,0 +1,1013 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/pci.h>
+#include <linux/device.h>
+#include <linux/errno.h>
+#include <linux/slab.h>
+#include <linux/bitops.h>
+#include <linux/delay.h>
+#include <linux/jiffies.h>
+#include <linux/log2.h>
+#include <linux/err.h>
+
+#include "hinic_hw_if.h"
+#include "hinic_hw_eqs.h"
+#include "hinic_hw_mgmt.h"
+#include "hinic_hw_qp_ctxt.h"
+#include "hinic_hw_qp.h"
+#include "hinic_hw_io.h"
+#include "hinic_hw_dev.h"
+
+#define IO_STATUS_TIMEOUT               100
+#define OUTBOUND_STATE_TIMEOUT          100
+#define DB_STATE_TIMEOUT                100
+
+#define MAX_IRQS(max_qps, num_aeqs, num_ceqs)   \
+		 (2 * (max_qps) + (num_aeqs) + (num_ceqs))
+
+#define ADDR_IN_4BYTES(addr)            ((addr) >> 2)
+
+enum intr_type {
+	INTR_MSIX_TYPE,
+};
+
+enum io_status {
+	IO_STOPPED = 0,
+	IO_RUNNING = 1,
+};
+
+enum hw_ioctxt_set_cmdq_depth {
+	HW_IOCTXT_SET_CMDQ_DEPTH_DEFAULT,
+};
+
+/* HW struct */
+struct hinic_dev_cap {
+	u8      status;
+	u8      version;
+	u8      rsvd0[6];
+
+	u8      rsvd1[5];
+	u8      intr_type;
+	u8      rsvd2[66];
+	u16     max_sqs;
+	u16     max_rqs;
+	u8      rsvd3[208];
+};
+
+/**
+ * get_capability - convert device capabilities to NIC capabilities
+ * @hwdev: the HW device to set and convert device capabilities for
+ * @dev_cap: device capabilities from FW
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int get_capability(struct hinic_hwdev *hwdev,
+			  struct hinic_dev_cap *dev_cap)
+{
+	struct hinic_cap *nic_cap = &hwdev->nic_cap;
+	int num_aeqs, num_ceqs, num_irqs;
+
+	if (!HINIC_IS_PF(hwdev->hwif) && !HINIC_IS_PPF(hwdev->hwif))
+		return -EINVAL;
+
+	if (dev_cap->intr_type != INTR_MSIX_TYPE)
+		return -EFAULT;
+
+	num_aeqs = HINIC_HWIF_NUM_AEQS(hwdev->hwif);
+	num_ceqs = HINIC_HWIF_NUM_CEQS(hwdev->hwif);
+	num_irqs = HINIC_HWIF_NUM_IRQS(hwdev->hwif);
+
+	/* Each QP has its own (SQ + RQ) interrupts */
+	nic_cap->num_qps = (num_irqs - (num_aeqs + num_ceqs)) / 2;
+
+	if (nic_cap->num_qps > HINIC_Q_CTXT_MAX)
+		nic_cap->num_qps = HINIC_Q_CTXT_MAX;
+
+	/* num_qps must be power of 2 */
+	nic_cap->num_qps = BIT(fls(nic_cap->num_qps) - 1);
+
+	nic_cap->max_qps = dev_cap->max_sqs + 1;
+	if (nic_cap->max_qps != (dev_cap->max_rqs + 1))
+		return -EFAULT;
+
+	if (nic_cap->num_qps > nic_cap->max_qps)
+		nic_cap->num_qps = nic_cap->max_qps;
+
+	return 0;
+}
+
+/**
+ * get_cap_from_fw - get device capabilities from FW
+ * @pfhwdev: the PF HW device to get capabilities for
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int get_cap_from_fw(struct hinic_pfhwdev *pfhwdev)
+{
+	struct hinic_hwdev *hwdev = &pfhwdev->hwdev;
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	struct hinic_dev_cap dev_cap;
+	u16 in_len, out_len;
+	int err;
+
+	in_len = 0;
+	out_len = sizeof(dev_cap);
+
+	err = hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_CFGM,
+				HINIC_CFG_NIC_CAP, &dev_cap, in_len, &dev_cap,
+				&out_len, HINIC_MGMT_MSG_SYNC);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to get capability from FW\n");
+		return err;
+	}
+
+	return get_capability(hwdev, &dev_cap);
+}
+
+/**
+ * get_dev_cap - get device capabilities
+ * @hwdev: the NIC HW device to get capabilities for
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int get_dev_cap(struct hinic_hwdev *hwdev)
+{
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	struct hinic_pfhwdev *pfhwdev;
+	int err;
+
+	switch (HINIC_FUNC_TYPE(hwif)) {
+	case HINIC_PPF:
+	case HINIC_PF:
+		pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev);
+
+		err = get_cap_from_fw(pfhwdev);
+		if (err) {
+			dev_err(&pdev->dev, "Failed to get capability from FW\n");
+			return err;
+		}
+		break;
+
+	default:
+		dev_err(&pdev->dev, "Unsupported PCI Function type\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/**
+ * init_msix - enable the msix and save the entries
+ * @hwdev: the NIC HW device
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int init_msix(struct hinic_hwdev *hwdev)
+{
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	int nr_irqs, num_aeqs, num_ceqs;
+	size_t msix_entries_size;
+	int i, err;
+
+	num_aeqs = HINIC_HWIF_NUM_AEQS(hwif);
+	num_ceqs = HINIC_HWIF_NUM_CEQS(hwif);
+	nr_irqs = MAX_IRQS(HINIC_MAX_QPS, num_aeqs, num_ceqs);
+	if (nr_irqs > HINIC_HWIF_NUM_IRQS(hwif))
+		nr_irqs = HINIC_HWIF_NUM_IRQS(hwif);
+
+	msix_entries_size = nr_irqs * sizeof(*hwdev->msix_entries);
+	hwdev->msix_entries = devm_kzalloc(&pdev->dev, msix_entries_size,
+					   GFP_KERNEL);
+	if (!hwdev->msix_entries)
+		return -ENOMEM;
+
+	for (i = 0; i < nr_irqs; i++)
+		hwdev->msix_entries[i].entry = i;
+
+	err = pci_enable_msix_exact(pdev, hwdev->msix_entries, nr_irqs);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to enable pci msix\n");
+		return err;
+	}
+
+	return 0;
+}
+
+/**
+ * disable_msix - disable the msix
+ * @hwdev: the NIC HW device
+ **/
+static void disable_msix(struct hinic_hwdev *hwdev)
+{
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+
+	pci_disable_msix(pdev);
+}
+
+/**
+ * hinic_port_msg_cmd - send port msg to mgmt
+ * @hwdev: the NIC HW device
+ * @cmd: the port command
+ * @buf_in: input buffer
+ * @in_size: input size
+ * @buf_out: output buffer
+ * @out_size: returned output size
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_port_msg_cmd(struct hinic_hwdev *hwdev, enum hinic_port_cmd cmd,
+		       void *buf_in, u16 in_size, void *buf_out, u16 *out_size)
+{
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	struct hinic_pfhwdev *pfhwdev;
+
+	if (!HINIC_IS_PF(hwif) && !HINIC_IS_PPF(hwif)) {
+		dev_err(&pdev->dev, "unsupported PCI Function type\n");
+		return -EINVAL;
+	}
+
+	pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev);
+
+	return hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_L2NIC, cmd,
+				 buf_in, in_size, buf_out, out_size,
+				 HINIC_MGMT_MSG_SYNC);
+}
+
+/**
+ * init_fw_ctxt- Init Firmware tables before network mgmt and io operations
+ * @hwdev: the NIC HW device
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int init_fw_ctxt(struct hinic_hwdev *hwdev)
+{
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	struct hinic_cmd_fw_ctxt fw_ctxt;
+	struct hinic_pfhwdev *pfhwdev;
+	u16 out_size;
+	int err;
+
+	if (!HINIC_IS_PF(hwif) && !HINIC_IS_PPF(hwif)) {
+		dev_err(&pdev->dev, "Unsupported PCI Function type\n");
+		return -EINVAL;
+	}
+
+	fw_ctxt.func_idx = HINIC_HWIF_FUNC_IDX(hwif);
+	fw_ctxt.rx_buf_sz = HINIC_RX_BUF_SZ;
+
+	pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev);
+
+	err = hinic_port_msg_cmd(hwdev, HINIC_PORT_CMD_FWCTXT_INIT,
+				 &fw_ctxt, sizeof(fw_ctxt),
+				 &fw_ctxt, &out_size);
+	if (err || (out_size != sizeof(fw_ctxt)) || fw_ctxt.status) {
+		dev_err(&pdev->dev, "Failed to init FW ctxt, ret = %d\n",
+			fw_ctxt.status);
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+/**
+ * set_hw_ioctxt - set the shape of the IO queues in FW
+ * @hwdev: the NIC HW device
+ * @rq_depth: rq depth
+ * @sq_depth: sq depth
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int set_hw_ioctxt(struct hinic_hwdev *hwdev, unsigned int rq_depth,
+			 unsigned int sq_depth)
+{
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct hinic_cmd_hw_ioctxt hw_ioctxt;
+	struct pci_dev *pdev = hwif->pdev;
+	struct hinic_pfhwdev *pfhwdev;
+
+	if (!HINIC_IS_PF(hwif) && !HINIC_IS_PPF(hwif)) {
+		dev_err(&pdev->dev, "Unsupported PCI Function type\n");
+		return -EINVAL;
+	}
+
+	hw_ioctxt.func_idx = HINIC_HWIF_FUNC_IDX(hwif);
+
+	hw_ioctxt.set_cmdq_depth = HW_IOCTXT_SET_CMDQ_DEPTH_DEFAULT;
+	hw_ioctxt.cmdq_depth = 0;
+
+	hw_ioctxt.rq_depth  = ilog2(rq_depth);
+
+	hw_ioctxt.rx_buf_sz_idx = HINIC_RX_BUF_SZ_IDX;
+
+	hw_ioctxt.sq_depth  = ilog2(sq_depth);
+
+	pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev);
+
+	return hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_COMM,
+				 HINIC_COMM_CMD_HWCTXT_SET,
+				 &hw_ioctxt, sizeof(hw_ioctxt), NULL,
+				 NULL, HINIC_MGMT_MSG_SYNC);
+}
+
+static int wait_for_outbound_state(struct hinic_hwdev *hwdev)
+{
+	enum hinic_outbound_state outbound_state;
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	unsigned long end;
+
+	end = jiffies + msecs_to_jiffies(OUTBOUND_STATE_TIMEOUT);
+	do {
+		outbound_state = hinic_outbound_state_get(hwif);
+
+		if (outbound_state == HINIC_OUTBOUND_ENABLE)
+			return 0;
+
+		msleep(20);
+	} while (time_before(jiffies, end));
+
+	dev_err(&pdev->dev, "Wait for OUTBOUND - Timeout\n");
+	return -EFAULT;
+}
+
+static int wait_for_db_state(struct hinic_hwdev *hwdev)
+{
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	enum hinic_db_state db_state;
+	unsigned long end;
+
+	end = jiffies + msecs_to_jiffies(DB_STATE_TIMEOUT);
+	do {
+		db_state = hinic_db_state_get(hwif);
+
+		if (db_state == HINIC_DB_ENABLE)
+			return 0;
+
+		msleep(20);
+	} while (time_before(jiffies, end));
+
+	dev_err(&pdev->dev, "Wait for DB - Timeout\n");
+	return -EFAULT;
+}
+
+static int wait_for_io_stopped(struct hinic_hwdev *hwdev)
+{
+	struct hinic_cmd_io_status cmd_io_status;
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	struct hinic_pfhwdev *pfhwdev;
+	unsigned long end;
+	u16 out_size;
+	int err;
+
+	if (!HINIC_IS_PF(hwif) && !HINIC_IS_PPF(hwif)) {
+		dev_err(&pdev->dev, "Unsupported PCI Function type\n");
+		return -EINVAL;
+	}
+
+	pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev);
+
+	cmd_io_status.func_idx = HINIC_HWIF_FUNC_IDX(hwif);
+
+	end = jiffies + msecs_to_jiffies(IO_STATUS_TIMEOUT);
+	do {
+		err = hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_COMM,
+					HINIC_COMM_CMD_IO_STATUS_GET,
+					&cmd_io_status, sizeof(cmd_io_status),
+					&cmd_io_status, &out_size,
+					HINIC_MGMT_MSG_SYNC);
+		if ((err) || (out_size != sizeof(cmd_io_status))) {
+			dev_err(&pdev->dev, "Failed to get IO status, ret = %d\n",
+				err);
+			return err;
+		}
+
+		if (cmd_io_status.status == IO_STOPPED) {
+			dev_info(&pdev->dev, "IO stopped\n");
+			return 0;
+		}
+
+		msleep(20);
+	} while (time_before(jiffies, end));
+
+	dev_err(&pdev->dev, "Wait for IO stopped - Timeout\n");
+	return -ETIMEDOUT;
+}
+
+/**
+ * clear_io_resource - set the IO resources as not active in the NIC
+ * @hwdev: the NIC HW device
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int clear_io_resources(struct hinic_hwdev *hwdev)
+{
+	struct hinic_cmd_clear_io_res cmd_clear_io_res;
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	struct hinic_pfhwdev *pfhwdev;
+	int err;
+
+	if (!HINIC_IS_PF(hwif) && !HINIC_IS_PPF(hwif)) {
+		dev_err(&pdev->dev, "Unsupported PCI Function type\n");
+		return -EINVAL;
+	}
+
+	err = wait_for_io_stopped(hwdev);
+	if (err) {
+		dev_err(&pdev->dev, "IO has not stopped yet\n");
+		return err;
+	}
+
+	cmd_clear_io_res.func_idx = HINIC_HWIF_FUNC_IDX(hwif);
+
+	pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev);
+
+	err = hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_COMM,
+				HINIC_COMM_CMD_IO_RES_CLEAR, &cmd_clear_io_res,
+				sizeof(cmd_clear_io_res), NULL, NULL,
+				HINIC_MGMT_MSG_SYNC);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to clear IO resources\n");
+		return err;
+	}
+
+	return 0;
+}
+
+/**
+ * set_resources_state - set the state of the resources in the NIC
+ * @hwdev: the NIC HW device
+ * @state: the state to set
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int set_resources_state(struct hinic_hwdev *hwdev,
+			       enum hinic_res_state state)
+{
+	struct hinic_cmd_set_res_state res_state;
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	struct hinic_pfhwdev *pfhwdev;
+
+	if (!HINIC_IS_PF(hwif) && !HINIC_IS_PPF(hwif)) {
+		dev_err(&pdev->dev, "Unsupported PCI Function type\n");
+		return -EINVAL;
+	}
+
+	res_state.func_idx = HINIC_HWIF_FUNC_IDX(hwif);
+	res_state.state = state;
+
+	pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev);
+
+	return hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt,
+				 HINIC_MOD_COMM,
+				 HINIC_COMM_CMD_RES_STATE_SET,
+				 &res_state, sizeof(res_state), NULL,
+				 NULL, HINIC_MGMT_MSG_SYNC);
+}
+
+/**
+ * get_base_qpn - get the first qp number
+ * @hwdev: the NIC HW device
+ * @base_qpn: returned qp number
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int get_base_qpn(struct hinic_hwdev *hwdev, u16 *base_qpn)
+{
+	struct hinic_cmd_base_qpn cmd_base_qpn;
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	u16 out_size;
+	int err;
+
+	cmd_base_qpn.func_idx = HINIC_HWIF_FUNC_IDX(hwif);
+
+	err = hinic_port_msg_cmd(hwdev, HINIC_PORT_CMD_GET_GLOBAL_QPN,
+				 &cmd_base_qpn, sizeof(cmd_base_qpn),
+				 &cmd_base_qpn, &out_size);
+	if (err || (out_size != sizeof(cmd_base_qpn)) || cmd_base_qpn.status) {
+		dev_err(&pdev->dev, "Failed to get base qpn, status = %d\n",
+			cmd_base_qpn.status);
+		return -EFAULT;
+	}
+
+	*base_qpn = cmd_base_qpn.qpn;
+	return 0;
+}
+
+/**
+ * hinic_hwdev_ifup - Preparing the HW for passing IO
+ * @hwdev: the NIC HW device
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_hwdev_ifup(struct hinic_hwdev *hwdev)
+{
+	struct hinic_func_to_io *func_to_io = &hwdev->func_to_io;
+	struct hinic_cap *nic_cap = &hwdev->nic_cap;
+	struct hinic_hwif *hwif = hwdev->hwif;
+	int err, num_aeqs, num_ceqs, num_qps;
+	struct msix_entry *ceq_msix_entries;
+	struct msix_entry *sq_msix_entries;
+	struct msix_entry *rq_msix_entries;
+	struct pci_dev *pdev = hwif->pdev;
+	u16 base_qpn;
+
+	err = get_base_qpn(hwdev, &base_qpn);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to get global base qp number\n");
+		return err;
+	}
+
+	num_aeqs = HINIC_HWIF_NUM_AEQS(hwif);
+	num_ceqs = HINIC_HWIF_NUM_CEQS(hwif);
+
+	ceq_msix_entries = &hwdev->msix_entries[num_aeqs];
+
+	err = hinic_io_init(func_to_io, hwif, nic_cap->max_qps, num_ceqs,
+			    ceq_msix_entries);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to init IO channel\n");
+		return err;
+	}
+
+	num_qps = nic_cap->num_qps;
+	sq_msix_entries = &hwdev->msix_entries[num_aeqs + num_ceqs];
+	rq_msix_entries = &hwdev->msix_entries[num_aeqs + num_ceqs + num_qps];
+
+	err = hinic_io_create_qps(func_to_io, base_qpn, num_qps,
+				  sq_msix_entries, rq_msix_entries);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to create QPs\n");
+		goto err_create_qps;
+	}
+
+	err = wait_for_db_state(hwdev);
+	if (err) {
+		dev_warn(&pdev->dev, "db - disabled, try again\n");
+		hinic_db_state_set(hwif, HINIC_DB_ENABLE);
+	}
+
+	err = set_hw_ioctxt(hwdev, HINIC_SQ_DEPTH, HINIC_RQ_DEPTH);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to set HW IO ctxt\n");
+		goto err_hw_ioctxt;
+	}
+
+	return 0;
+
+err_hw_ioctxt:
+	hinic_io_destroy_qps(func_to_io, num_qps);
+
+err_create_qps:
+	hinic_io_free(func_to_io);
+	return err;
+}
+
+/**
+ * hinic_hwdev_ifdown - Closing the HW for passing IO
+ * @hwdev: the NIC HW device
+ *
+ **/
+void hinic_hwdev_ifdown(struct hinic_hwdev *hwdev)
+{
+	struct hinic_func_to_io *func_to_io = &hwdev->func_to_io;
+	struct hinic_cap *nic_cap = &hwdev->nic_cap;
+
+	clear_io_resources(hwdev);
+
+	hinic_io_destroy_qps(func_to_io, nic_cap->num_qps);
+	hinic_io_free(func_to_io);
+}
+
+/**
+ * hinic_hwdev_cb_register - register callback handler for MGMT events
+ * @hwdev: the NIC HW device
+ * @cmd: the mgmt event
+ * @handle: private data for the handler
+ * @handler: event handler
+ **/
+void hinic_hwdev_cb_register(struct hinic_hwdev *hwdev,
+			     enum hinic_mgmt_msg_cmd cmd, void *handle,
+			     void (*handler)(void *handle, void *buf_in,
+					     u16 in_size, void *buf_out,
+					     u16 *out_size))
+{
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	struct hinic_pfhwdev *pfhwdev;
+	struct hinic_nic_cb *nic_cb;
+	u8 cmd_cb;
+
+	if (!HINIC_IS_PF(hwif) && !HINIC_IS_PPF(hwif)) {
+		dev_err(&pdev->dev, "unsupported PCI Function type\n");
+		return;
+	}
+
+	pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev);
+
+	cmd_cb = cmd - HINIC_MGMT_MSG_CMD_BASE;
+	nic_cb = &pfhwdev->nic_cb[cmd_cb];
+
+	nic_cb->handler = handler;
+	nic_cb->handle = handle;
+	nic_cb->cb_state = HINIC_CB_ENABLED;
+}
+
+/**
+ * hinic_hwdev_cb_unregister - unregister callback handler for MGMT events
+ * @hwdev: the NIC HW device
+ * @cmd: the mgmt event
+ **/
+void hinic_hwdev_cb_unregister(struct hinic_hwdev *hwdev,
+			       enum hinic_mgmt_msg_cmd cmd)
+{
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	struct hinic_pfhwdev *pfhwdev;
+	struct hinic_nic_cb *nic_cb;
+	u8 cmd_cb;
+
+	if (!HINIC_IS_PF(hwif) && !HINIC_IS_PPF(hwif)) {
+		dev_err(&pdev->dev, "unsupported PCI Function type\n");
+		return;
+	}
+
+	pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev);
+
+	cmd_cb = cmd - HINIC_MGMT_MSG_CMD_BASE;
+	nic_cb = &pfhwdev->nic_cb[cmd_cb];
+
+	nic_cb->cb_state &= ~HINIC_CB_ENABLED;
+
+	while (nic_cb->cb_state & HINIC_CB_RUNNING)
+		schedule();
+
+	nic_cb->handler = NULL;
+}
+
+/**
+ * nic_mgmt_msg_handler - nic mgmt event handler
+ * @handle: private data for the handler
+ * @buf_in: input buffer
+ * @in_size: input size
+ * @buf_out: output buffer
+ * @out_size: returned output size
+ **/
+static void nic_mgmt_msg_handler(void *handle, u8 cmd, void *buf_in,
+				 u16 in_size, void *buf_out, u16 *out_size)
+{
+	struct hinic_pfhwdev *pfhwdev = handle;
+	enum hinic_cb_state cb_state;
+	struct hinic_nic_cb *nic_cb;
+	struct hinic_hwdev *hwdev;
+	struct hinic_hwif *hwif;
+	struct pci_dev *pdev;
+	u8 cmd_cb;
+
+	hwdev = &pfhwdev->hwdev;
+	hwif = hwdev->hwif;
+	pdev = hwif->pdev;
+
+	if ((cmd < HINIC_MGMT_MSG_CMD_BASE) ||
+	    (cmd >= HINIC_MGMT_MSG_CMD_MAX)) {
+		dev_err(&pdev->dev, "unknown L2NIC event, cmd = %d\n", cmd);
+		return;
+	}
+
+	cmd_cb = cmd - HINIC_MGMT_MSG_CMD_BASE;
+
+	nic_cb = &pfhwdev->nic_cb[cmd_cb];
+
+	cb_state = cmpxchg(&nic_cb->cb_state,
+			   HINIC_CB_ENABLED,
+			   HINIC_CB_ENABLED | HINIC_CB_RUNNING);
+
+	if ((cb_state == HINIC_CB_ENABLED) && (nic_cb->handler))
+		nic_cb->handler(nic_cb->handle, buf_in,
+				in_size, buf_out, out_size);
+	else
+		dev_err(&pdev->dev, "Unhandled NIC Event %d\n", cmd);
+
+	nic_cb->cb_state &= ~HINIC_CB_RUNNING;
+}
+
+/**
+ * init_pfhwdev - Initialize the extended components of PF
+ * @pfhwdev: the HW device for PF
+ *
+ * Return 0 - success, negative - failure
+ **/
+static int init_pfhwdev(struct hinic_pfhwdev *pfhwdev)
+{
+	struct hinic_hwdev *hwdev = &pfhwdev->hwdev;
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	int err;
+
+	err = hinic_pf_to_mgmt_init(&pfhwdev->pf_to_mgmt, hwif);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to initialize PF to MGMT channel\n");
+		return err;
+	}
+
+	hinic_register_mgmt_msg_cb(&pfhwdev->pf_to_mgmt, HINIC_MOD_L2NIC,
+				   pfhwdev, nic_mgmt_msg_handler);
+
+	hinic_set_pf_action(hwif, HINIC_PF_MGMT_ACTIVE);
+	return 0;
+}
+
+/**
+ * free_pfhwdev - Free the extended components of PF
+ * @pfhwdev: the HW device for PF
+ **/
+static void free_pfhwdev(struct hinic_pfhwdev *pfhwdev)
+{
+	struct hinic_hwdev *hwdev = &pfhwdev->hwdev;
+
+	hinic_set_pf_action(hwdev->hwif, HINIC_PF_MGMT_INIT);
+
+	hinic_unregister_mgmt_msg_cb(&pfhwdev->pf_to_mgmt, HINIC_MOD_L2NIC);
+
+	hinic_pf_to_mgmt_free(&pfhwdev->pf_to_mgmt);
+}
+
+/**
+ * hinic_init_hwdev - Initialize the NIC HW
+ * @pdev: the NIC pci device
+ *
+ * Return initialized NIC HW device
+ *
+ * Initialize the NIC HW device and return a pointer to it
+ **/
+struct hinic_hwdev *hinic_init_hwdev(struct pci_dev *pdev)
+{
+	struct hinic_pfhwdev *pfhwdev;
+	struct hinic_hwdev *hwdev;
+	struct hinic_hwif *hwif;
+	int err, num_aeqs;
+
+	hwif = devm_kzalloc(&pdev->dev, sizeof(*hwif), GFP_KERNEL);
+	if (!hwif)
+		return ERR_PTR(-ENOMEM);
+
+	err = hinic_init_hwif(hwif, pdev);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to init HW interface\n");
+		return ERR_PTR(err);
+	}
+
+	if (!HINIC_IS_PF(hwif) && !HINIC_IS_PPF(hwif)) {
+		dev_err(&pdev->dev, "Unsupported PCI Function type\n");
+		err = -EFAULT;
+		goto err_func_type;
+	}
+
+	pfhwdev = devm_kzalloc(&pdev->dev, sizeof(*pfhwdev), GFP_KERNEL);
+	if (!pfhwdev) {
+		err = -ENOMEM;
+		goto err_pfhwdev_alloc;
+	}
+
+	hwdev = &pfhwdev->hwdev;
+	hwdev->hwif = hwif;
+
+	err = init_msix(hwdev);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to init msix\n");
+		goto err_init_msix;
+	}
+
+	err = wait_for_outbound_state(hwdev);
+	if (err) {
+		dev_warn(&pdev->dev, "outbound - disabled, try again\n");
+		hinic_outbound_state_set(hwif, HINIC_OUTBOUND_ENABLE);
+	}
+
+	num_aeqs = HINIC_HWIF_NUM_AEQS(hwif);
+
+	err = hinic_aeqs_init(&hwdev->aeqs, hwif, num_aeqs,
+			      HINIC_DEFAULT_AEQ_LEN, HINIC_EQ_PAGE_SIZE,
+			      hwdev->msix_entries);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to init async event queues\n");
+		goto err_aeqs_init;
+	}
+
+	err = init_pfhwdev(pfhwdev);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to init PF HW device\n");
+		goto err_init_pfhwdev;
+	}
+
+	err = get_dev_cap(hwdev);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to get device capabilities\n");
+		goto err_dev_cap;
+	}
+
+	err = init_fw_ctxt(hwdev);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to init function table\n");
+		goto err_init_fw_ctxt;
+	}
+
+	err = set_resources_state(hwdev, HINIC_RES_ACTIVE);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to set resources state\n");
+		goto err_resources_state;
+	}
+
+	return hwdev;
+
+err_resources_state:
+err_init_fw_ctxt:
+err_dev_cap:
+	free_pfhwdev(pfhwdev);
+
+err_init_pfhwdev:
+	hinic_aeqs_free(&hwdev->aeqs);
+
+err_aeqs_init:
+	disable_msix(hwdev);
+
+err_init_msix:
+err_pfhwdev_alloc:
+err_func_type:
+	hinic_free_hwif(hwif);
+	return ERR_PTR(err);
+}
+
+/**
+ * hinic_free_hwdev - Free the NIC HW device
+ * @hwdev: the NIC HW device
+ **/
+void hinic_free_hwdev(struct hinic_hwdev *hwdev)
+{
+	struct hinic_pfhwdev *pfhwdev = container_of(hwdev,
+						     struct hinic_pfhwdev,
+						     hwdev);
+
+	set_resources_state(hwdev, HINIC_RES_CLEAN);
+
+	free_pfhwdev(pfhwdev);
+
+	hinic_aeqs_free(&hwdev->aeqs);
+
+	disable_msix(hwdev);
+
+	hinic_free_hwif(hwdev->hwif);
+}
+
+/**
+ * hinic_hwdev_num_qps - return the number QPs available for use
+ * @hwdev: the NIC HW device
+ *
+ * Return number QPs available for use
+ **/
+int hinic_hwdev_num_qps(struct hinic_hwdev *hwdev)
+{
+	struct hinic_cap *nic_cap = &hwdev->nic_cap;
+
+	return nic_cap->num_qps;
+}
+
+/**
+ * hinic_hwdev_get_sq - get SQ
+ * @hwdev: the NIC HW device
+ * @i: the position of the SQ
+ *
+ * Return: the SQ in the i position
+ **/
+struct hinic_sq *hinic_hwdev_get_sq(struct hinic_hwdev *hwdev, int i)
+{
+	struct hinic_func_to_io *func_to_io = &hwdev->func_to_io;
+	struct hinic_qp *qp = &func_to_io->qps[i];
+
+	if (i >= hinic_hwdev_num_qps(hwdev))
+		return NULL;
+
+	return &qp->sq;
+}
+
+/**
+ * hinic_hwdev_get_sq - get RQ
+ * @hwdev: the NIC HW device
+ * @i: the position of the RQ
+ *
+ * Return: the RQ in the i position
+ **/
+struct hinic_rq *hinic_hwdev_get_rq(struct hinic_hwdev *hwdev, int i)
+{
+	struct hinic_func_to_io *func_to_io = &hwdev->func_to_io;
+	struct hinic_qp *qp = &func_to_io->qps[i];
+
+	if (i >= hinic_hwdev_num_qps(hwdev))
+		return NULL;
+
+	return &qp->rq;
+}
+
+/**
+ * hinic_hwdev_msix_cnt_set - clear message attribute counters for msix entry
+ * @hwdev: the NIC HW device
+ * @msix_index: msix_index
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_hwdev_msix_cnt_set(struct hinic_hwdev *hwdev, u16 msix_index)
+{
+	return hinic_msix_attr_cnt_clear(hwdev->hwif, msix_index);
+}
+
+/**
+ * hinic_hwdev_msix_set - set message attribute for msix entry
+ * @hwdev: the NIC HW device
+ * @msix_index: msix_index
+ * @pending_limit: the maximum pending interrupt events (unit 8)
+ * @coalesc_timer: coalesc period for interrupt (unit 8 us)
+ * @lli_timer: replenishing period for low latency credit (unit 8 us)
+ * @lli_credit_limit: maximum credits for low latency msix messages (unit 8)
+ * @resend_timer: maximum wait for resending msix (unit coalesc period)
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_hwdev_msix_set(struct hinic_hwdev *hwdev, u16 msix_index,
+			 u8 pending_limit, u8 coalesc_timer,
+			 u8 lli_timer_cfg, u8 lli_credit_limit,
+			 u8 resend_timer)
+{
+	return hinic_msix_attr_set(hwdev->hwif, msix_index,
+				   pending_limit, coalesc_timer,
+				   lli_timer_cfg, lli_credit_limit,
+				   resend_timer);
+}
+
+/**
+ * hinic_hwdev_hw_ci_addr_set - set cons idx addr and attributes in HW for sq
+ * @hwdev: the NIC HW device
+ * @sq: send queue
+ * @pending_limit: the maximum pending update ci events (unit 8)
+ * @coalesc_timer: coalesc period for update ci (unit 8 us)
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_hwdev_hw_ci_addr_set(struct hinic_hwdev *hwdev, struct hinic_sq *sq,
+			       u8 pending_limit, u8 coalesc_timer)
+{
+	struct hinic_qp *qp = container_of(sq, struct hinic_qp, sq);
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	struct hinic_pfhwdev *pfhwdev;
+	struct hinic_cmd_hw_ci hw_ci;
+
+	if (!HINIC_IS_PF(hwif) && !HINIC_IS_PPF(hwif)) {
+		dev_err(&pdev->dev, "Unsupported PCI Function type\n");
+		return -EINVAL;
+	}
+
+	hw_ci.dma_attr_off  = 0;
+	hw_ci.pending_limit = pending_limit;
+	hw_ci.coalesc_timer = coalesc_timer;
+
+	hw_ci.msix_en = 1;
+	hw_ci.msix_entry_idx = sq->msix_entry;
+
+	hw_ci.func_idx = HINIC_HWIF_FUNC_IDX(hwif);
+
+	hw_ci.sq_id = qp->q_id;
+
+	hw_ci.ci_addr = ADDR_IN_4BYTES(sq->hw_ci_dma_addr);
+
+	pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev);
+	return hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt,
+				 HINIC_MOD_COMM,
+				 HINIC_COMM_CMD_SQ_HI_CI_SET,
+				 &hw_ci, sizeof(hw_ci), NULL,
+				 NULL, HINIC_MGMT_MSG_SYNC);
+}
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.h
new file mode 100644
index 000000000000..0f5563f3b779
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.h
@@ -0,0 +1,239 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#ifndef HINIC_HW_DEV_H
+#define HINIC_HW_DEV_H
+
+#include <linux/pci.h>
+#include <linux/types.h>
+#include <linux/bitops.h>
+
+#include "hinic_hw_if.h"
+#include "hinic_hw_eqs.h"
+#include "hinic_hw_mgmt.h"
+#include "hinic_hw_qp.h"
+#include "hinic_hw_io.h"
+
+#define HINIC_MAX_QPS   32
+
+#define HINIC_MGMT_NUM_MSG_CMD  (HINIC_MGMT_MSG_CMD_MAX - \
+				 HINIC_MGMT_MSG_CMD_BASE)
+
+struct hinic_cap {
+	u16     max_qps;
+	u16     num_qps;
+};
+
+enum hinic_port_cmd {
+	HINIC_PORT_CMD_CHANGE_MTU       = 2,
+
+	HINIC_PORT_CMD_ADD_VLAN         = 3,
+	HINIC_PORT_CMD_DEL_VLAN         = 4,
+
+	HINIC_PORT_CMD_SET_MAC          = 9,
+	HINIC_PORT_CMD_GET_MAC          = 10,
+	HINIC_PORT_CMD_DEL_MAC          = 11,
+
+	HINIC_PORT_CMD_SET_RX_MODE      = 12,
+
+	HINIC_PORT_CMD_GET_LINK_STATE   = 24,
+
+	HINIC_PORT_CMD_SET_PORT_STATE   = 41,
+
+	HINIC_PORT_CMD_FWCTXT_INIT      = 69,
+
+	HINIC_PORT_CMD_SET_FUNC_STATE   = 93,
+
+	HINIC_PORT_CMD_GET_GLOBAL_QPN   = 102,
+
+	HINIC_PORT_CMD_GET_CAP          = 170,
+};
+
+enum hinic_mgmt_msg_cmd {
+	HINIC_MGMT_MSG_CMD_BASE         = 160,
+
+	HINIC_MGMT_MSG_CMD_LINK_STATUS  = 160,
+
+	HINIC_MGMT_MSG_CMD_MAX,
+};
+
+enum hinic_cb_state {
+	HINIC_CB_ENABLED = BIT(0),
+	HINIC_CB_RUNNING = BIT(1),
+};
+
+enum hinic_res_state {
+	HINIC_RES_CLEAN         = 0,
+	HINIC_RES_ACTIVE        = 1,
+};
+
+struct hinic_cmd_fw_ctxt {
+	u8      status;
+	u8      version;
+	u8      rsvd0[6];
+
+	u16     func_idx;
+	u16     rx_buf_sz;
+
+	u32     rsvd1;
+};
+
+struct hinic_cmd_hw_ioctxt {
+	u8      status;
+	u8      version;
+	u8      rsvd0[6];
+
+	u16     func_idx;
+
+	u16     rsvd1;
+
+	u8      set_cmdq_depth;
+	u8      cmdq_depth;
+
+	u8      rsvd2;
+	u8      rsvd3;
+	u8      rsvd4;
+	u8      rsvd5;
+
+	u16     rq_depth;
+	u16     rx_buf_sz_idx;
+	u16     sq_depth;
+};
+
+struct hinic_cmd_io_status {
+	u8      status;
+	u8      version;
+	u8      rsvd0[6];
+
+	u16     func_idx;
+	u8      rsvd1;
+	u8      rsvd2;
+	u32     io_status;
+};
+
+struct hinic_cmd_clear_io_res {
+	u8      status;
+	u8      version;
+	u8      rsvd0[6];
+
+	u16     func_idx;
+	u8      rsvd1;
+	u8      rsvd2;
+};
+
+struct hinic_cmd_set_res_state {
+	u8      status;
+	u8      version;
+	u8      rsvd0[6];
+
+	u16     func_idx;
+	u8      state;
+	u8      rsvd1;
+	u32     rsvd2;
+};
+
+struct hinic_cmd_base_qpn {
+	u8      status;
+	u8      version;
+	u8      rsvd0[6];
+
+	u16     func_idx;
+	u16     qpn;
+};
+
+struct hinic_cmd_hw_ci {
+	u8      status;
+	u8      version;
+	u8      rsvd0[6];
+
+	u16     func_idx;
+
+	u8      dma_attr_off;
+	u8      pending_limit;
+	u8      coalesc_timer;
+
+	u8      msix_en;
+	u16     msix_entry_idx;
+
+	u32     sq_id;
+	u32     rsvd1;
+	u64     ci_addr;
+};
+
+struct hinic_hwdev {
+	struct hinic_hwif               *hwif;
+	struct msix_entry               *msix_entries;
+
+	struct hinic_aeqs               aeqs;
+	struct hinic_func_to_io         func_to_io;
+
+	struct hinic_cap                nic_cap;
+};
+
+struct hinic_nic_cb {
+	void    (*handler)(void *handle, void *buf_in,
+			   u16 in_size, void *buf_out,
+			   u16 *out_size);
+
+	void            *handle;
+	unsigned long   cb_state;
+};
+
+struct hinic_pfhwdev {
+	struct hinic_hwdev              hwdev;
+
+	struct hinic_pf_to_mgmt         pf_to_mgmt;
+
+	struct hinic_nic_cb             nic_cb[HINIC_MGMT_NUM_MSG_CMD];
+};
+
+void hinic_hwdev_cb_register(struct hinic_hwdev *hwdev,
+			     enum hinic_mgmt_msg_cmd cmd, void *handle,
+			     void (*handler)(void *handle, void *buf_in,
+					     u16 in_size, void *buf_out,
+					     u16 *out_size));
+
+void hinic_hwdev_cb_unregister(struct hinic_hwdev *hwdev,
+			       enum hinic_mgmt_msg_cmd cmd);
+
+int hinic_port_msg_cmd(struct hinic_hwdev *hwdev, enum hinic_port_cmd cmd,
+		       void *buf_in, u16 in_size, void *buf_out,
+		       u16 *out_size);
+
+int hinic_hwdev_ifup(struct hinic_hwdev *hwdev);
+
+void hinic_hwdev_ifdown(struct hinic_hwdev *hwdev);
+
+struct hinic_hwdev *hinic_init_hwdev(struct pci_dev *pdev);
+
+void hinic_free_hwdev(struct hinic_hwdev *hwdev);
+
+int hinic_hwdev_num_qps(struct hinic_hwdev *hwdev);
+
+struct hinic_sq *hinic_hwdev_get_sq(struct hinic_hwdev *hwdev, int i);
+
+struct hinic_rq *hinic_hwdev_get_rq(struct hinic_hwdev *hwdev, int i);
+
+int hinic_hwdev_msix_cnt_set(struct hinic_hwdev *hwdev, u16 msix_index);
+
+int hinic_hwdev_msix_set(struct hinic_hwdev *hwdev, u16 msix_index,
+			 u8 pending_limit, u8 coalesc_timer,
+			 u8 lli_timer_cfg, u8 lli_credit_limit,
+			 u8 resend_timer);
+
+int hinic_hwdev_hw_ci_addr_set(struct hinic_hwdev *hwdev, struct hinic_sq *sq,
+			       u8 pending_limit, u8 coalesc_timer);
+
+#endif
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_eqs.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_eqs.c
new file mode 100644
index 000000000000..7cb8b9b94726
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_eqs.c
@@ -0,0 +1,886 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/errno.h>
+#include <linux/pci.h>
+#include <linux/device.h>
+#include <linux/workqueue.h>
+#include <linux/interrupt.h>
+#include <linux/slab.h>
+#include <linux/dma-mapping.h>
+#include <linux/log2.h>
+#include <asm/byteorder.h>
+#include <asm/barrier.h>
+
+#include "hinic_hw_csr.h"
+#include "hinic_hw_if.h"
+#include "hinic_hw_eqs.h"
+
+#define HINIC_EQS_WQ_NAME                       "hinic_eqs"
+
+#define GET_EQ_NUM_PAGES(eq, pg_size)           \
+		(ALIGN((eq)->q_len * (eq)->elem_size, pg_size) / (pg_size))
+
+#define GET_EQ_NUM_ELEMS_IN_PG(eq, pg_size)     ((pg_size) / (eq)->elem_size)
+
+#define EQ_CONS_IDX_REG_ADDR(eq)        (((eq)->type == HINIC_AEQ) ? \
+			HINIC_CSR_AEQ_CONS_IDX_ADDR((eq)->q_id) : \
+			HINIC_CSR_CEQ_CONS_IDX_ADDR((eq)->q_id))
+
+#define EQ_PROD_IDX_REG_ADDR(eq)        (((eq)->type == HINIC_AEQ) ? \
+			HINIC_CSR_AEQ_PROD_IDX_ADDR((eq)->q_id) : \
+			HINIC_CSR_CEQ_PROD_IDX_ADDR((eq)->q_id))
+
+#define EQ_HI_PHYS_ADDR_REG(eq, pg_num) (((eq)->type == HINIC_AEQ) ? \
+			HINIC_CSR_AEQ_HI_PHYS_ADDR_REG((eq)->q_id, pg_num) : \
+			HINIC_CSR_CEQ_HI_PHYS_ADDR_REG((eq)->q_id, pg_num))
+
+#define EQ_LO_PHYS_ADDR_REG(eq, pg_num) (((eq)->type == HINIC_AEQ) ? \
+			HINIC_CSR_AEQ_LO_PHYS_ADDR_REG((eq)->q_id, pg_num) : \
+			HINIC_CSR_CEQ_LO_PHYS_ADDR_REG((eq)->q_id, pg_num))
+
+#define GET_EQ_ELEMENT(eq, idx)         \
+		((eq)->virt_addr[(idx) / (eq)->num_elem_in_pg] + \
+		 (((idx) & ((eq)->num_elem_in_pg - 1)) * (eq)->elem_size))
+
+#define GET_AEQ_ELEM(eq, idx)           ((struct hinic_aeq_elem *) \
+					GET_EQ_ELEMENT(eq, idx))
+
+#define GET_CEQ_ELEM(eq, idx)           ((u32 *) \
+					 GET_EQ_ELEMENT(eq, idx))
+
+#define GET_CURR_AEQ_ELEM(eq)           GET_AEQ_ELEM(eq, (eq)->cons_idx)
+
+#define GET_CURR_CEQ_ELEM(eq)           GET_CEQ_ELEM(eq, (eq)->cons_idx)
+
+#define PAGE_IN_4K(page_size)           ((page_size) >> 12)
+#define EQ_SET_HW_PAGE_SIZE_VAL(eq)     (ilog2(PAGE_IN_4K((eq)->page_size)))
+
+#define ELEMENT_SIZE_IN_32B(eq)         (((eq)->elem_size) >> 5)
+#define EQ_SET_HW_ELEM_SIZE_VAL(eq)     (ilog2(ELEMENT_SIZE_IN_32B(eq)))
+
+#define EQ_MAX_PAGES                    8
+
+#define CEQE_TYPE_SHIFT                 23
+#define CEQE_TYPE_MASK                  0x7
+
+#define CEQE_TYPE(ceqe)                 (((ceqe) >> CEQE_TYPE_SHIFT) &  \
+					 CEQE_TYPE_MASK)
+
+#define CEQE_DATA_MASK                  0x3FFFFFF
+#define CEQE_DATA(ceqe)                 ((ceqe) & CEQE_DATA_MASK)
+
+#define aeq_to_aeqs(eq)                 \
+		container_of((eq) - (eq)->q_id, struct hinic_aeqs, aeq[0])
+
+#define ceq_to_ceqs(eq)                 \
+		container_of((eq) - (eq)->q_id, struct hinic_ceqs, ceq[0])
+
+#define work_to_aeq_work(work)          \
+		container_of(work, struct hinic_eq_work, work)
+
+#define DMA_ATTR_AEQ_DEFAULT            0
+#define DMA_ATTR_CEQ_DEFAULT            0
+
+/* No coalescence */
+#define THRESH_CEQ_DEFAULT              0
+
+enum eq_int_mode {
+	EQ_INT_MODE_ARMED,
+	EQ_INT_MODE_ALWAYS
+};
+
+enum eq_arm_state {
+	EQ_NOT_ARMED,
+	EQ_ARMED
+};
+
+/**
+ * hinic_aeq_register_hw_cb - register AEQ callback for specific event
+ * @aeqs: pointer to Async eqs of the chip
+ * @event: aeq event to register callback for it
+ * @handle: private data will be used by the callback
+ * @hw_handler: callback function
+ **/
+void hinic_aeq_register_hw_cb(struct hinic_aeqs *aeqs,
+			      enum hinic_aeq_type event, void *handle,
+			      void (*hwe_handler)(void *handle, void *data,
+						  u8 size))
+{
+	struct hinic_hw_event_cb *hwe_cb = &aeqs->hwe_cb[event];
+
+	hwe_cb->hwe_handler = hwe_handler;
+	hwe_cb->handle = handle;
+	hwe_cb->hwe_state = HINIC_EQE_ENABLED;
+}
+
+/**
+ * hinic_aeq_unregister_hw_cb - unregister the AEQ callback for specific event
+ * @aeqs: pointer to Async eqs of the chip
+ * @event: aeq event to unregister callback for it
+ **/
+void hinic_aeq_unregister_hw_cb(struct hinic_aeqs *aeqs,
+				enum hinic_aeq_type event)
+{
+	struct hinic_hw_event_cb *hwe_cb = &aeqs->hwe_cb[event];
+
+	hwe_cb->hwe_state &= ~HINIC_EQE_ENABLED;
+
+	while (hwe_cb->hwe_state & HINIC_EQE_RUNNING)
+		schedule();
+
+	hwe_cb->hwe_handler = NULL;
+}
+
+/**
+ * hinic_ceq_register_cb - register CEQ callback for specific event
+ * @ceqs: pointer to Completion eqs part of the chip
+ * @event: ceq event to register callback for it
+ * @handle: private data will be used by the callback
+ * @handler: callback function
+ **/
+void hinic_ceq_register_cb(struct hinic_ceqs *ceqs,
+			   enum hinic_ceq_type event, void *handle,
+			   void (*handler)(void *handle, u32 ceqe_data))
+{
+	struct hinic_ceq_cb *ceq_cb = &ceqs->ceq_cb[event];
+
+	ceq_cb->handler = handler;
+	ceq_cb->handle = handle;
+	ceq_cb->ceqe_state = HINIC_EQE_ENABLED;
+}
+
+/**
+ * hinic_ceq_unregister_cb - unregister the CEQ callback for specific event
+ * @ceqs: pointer to Completion eqs part of the chip
+ * @event: ceq event to unregister callback for it
+ **/
+void hinic_ceq_unregister_cb(struct hinic_ceqs *ceqs,
+			     enum hinic_ceq_type event)
+{
+	struct hinic_ceq_cb *ceq_cb = &ceqs->ceq_cb[event];
+
+	ceq_cb->ceqe_state &= ~HINIC_EQE_ENABLED;
+
+	while (ceq_cb->ceqe_state & HINIC_EQE_RUNNING)
+		schedule();
+
+	ceq_cb->handler = NULL;
+}
+
+static u8 eq_cons_idx_checksum_set(u32 val)
+{
+	u8 checksum = 0;
+	int idx;
+
+	for (idx = 0; idx < 32; idx += 4)
+		checksum ^= ((val >> idx) & 0xF);
+
+	return (checksum & 0xF);
+}
+
+/**
+ * eq_update_ci - update the HW cons idx of event queue
+ * @eq: the event queue to update the cons idx for
+ **/
+static void eq_update_ci(struct hinic_eq *eq)
+{
+	u32 val, addr = EQ_CONS_IDX_REG_ADDR(eq);
+
+	/* Read Modify Write */
+	val = hinic_hwif_read_reg(eq->hwif, addr);
+
+	val = HINIC_EQ_CI_CLEAR(val, IDX)       &
+	      HINIC_EQ_CI_CLEAR(val, WRAPPED)   &
+	      HINIC_EQ_CI_CLEAR(val, INT_ARMED) &
+	      HINIC_EQ_CI_CLEAR(val, XOR_CHKSUM);
+
+	val |= HINIC_EQ_CI_SET(eq->cons_idx, IDX)    |
+	       HINIC_EQ_CI_SET(eq->wrapped, WRAPPED) |
+	       HINIC_EQ_CI_SET(EQ_ARMED, INT_ARMED);
+
+	val |= HINIC_EQ_CI_SET(eq_cons_idx_checksum_set(val), XOR_CHKSUM);
+
+	hinic_hwif_write_reg(eq->hwif, addr, val);
+}
+
+/**
+ * aeq_irq_handler - handler for the AEQ event
+ * @eq: the Async Event Queue that received the event
+ **/
+static void aeq_irq_handler(struct hinic_eq *eq)
+{
+	struct hinic_aeqs *aeqs = aeq_to_aeqs(eq);
+	struct hinic_hwif *hwif = aeqs->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	struct hinic_aeq_elem *aeqe_curr;
+	struct hinic_hw_event_cb *hwe_cb;
+	enum hinic_aeq_type event;
+	unsigned long eqe_state;
+	u32 aeqe_desc;
+	int i, size;
+
+	for (i = 0; i < eq->q_len; i++) {
+		aeqe_curr = GET_CURR_AEQ_ELEM(eq);
+
+		/* Data in HW is in Big endian Format */
+		aeqe_desc = be32_to_cpu(aeqe_curr->desc);
+
+		/* HW toggles the wrapped bit, when it adds eq element */
+		if (HINIC_EQ_ELEM_DESC_GET(aeqe_desc, WRAPPED) == eq->wrapped)
+			break;
+
+		event = HINIC_EQ_ELEM_DESC_GET(aeqe_desc, TYPE);
+		if (event >= HINIC_MAX_AEQ_EVENTS) {
+			dev_err(&pdev->dev, "Unknown AEQ Event %d\n", event);
+			return;
+		}
+
+		if (!HINIC_EQ_ELEM_DESC_GET(aeqe_desc, SRC)) {
+			hwe_cb = &aeqs->hwe_cb[event];
+
+			size = HINIC_EQ_ELEM_DESC_GET(aeqe_desc, SIZE);
+
+			eqe_state = cmpxchg(&hwe_cb->hwe_state,
+					    HINIC_EQE_ENABLED,
+					    HINIC_EQE_ENABLED |
+					    HINIC_EQE_RUNNING);
+			if ((eqe_state == HINIC_EQE_ENABLED) &&
+			    (hwe_cb->hwe_handler))
+				hwe_cb->hwe_handler(hwe_cb->handle,
+						    aeqe_curr->data, size);
+			else
+				dev_err(&pdev->dev, "Unhandled AEQ Event %d\n",
+					event);
+
+			hwe_cb->hwe_state &= ~HINIC_EQE_RUNNING;
+		}
+
+		eq->cons_idx++;
+
+		if (eq->cons_idx == eq->q_len) {
+			eq->cons_idx = 0;
+			eq->wrapped = !eq->wrapped;
+		}
+	}
+}
+
+/**
+ * ceq_event_handler - handler for the ceq events
+ * @ceqs: ceqs part of the chip
+ * @ceqe: ceq element that describes the event
+ **/
+static void ceq_event_handler(struct hinic_ceqs *ceqs, u32 ceqe)
+{
+	struct hinic_hwif *hwif = ceqs->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	struct hinic_ceq_cb *ceq_cb;
+	enum hinic_ceq_type event;
+	unsigned long eqe_state;
+
+	event = CEQE_TYPE(ceqe);
+	if (event >= HINIC_MAX_CEQ_EVENTS) {
+		dev_err(&pdev->dev, "Unknown CEQ event, event = %d\n", event);
+		return;
+	}
+
+	ceq_cb = &ceqs->ceq_cb[event];
+
+	eqe_state = cmpxchg(&ceq_cb->ceqe_state,
+			    HINIC_EQE_ENABLED,
+			    HINIC_EQE_ENABLED | HINIC_EQE_RUNNING);
+
+	if ((eqe_state == HINIC_EQE_ENABLED) && (ceq_cb->handler))
+		ceq_cb->handler(ceq_cb->handle, CEQE_DATA(ceqe));
+	else
+		dev_err(&pdev->dev, "Unhandled CEQ Event %d\n", event);
+
+	ceq_cb->ceqe_state &= ~HINIC_EQE_RUNNING;
+}
+
+/**
+ * ceq_irq_handler - handler for the CEQ event
+ * @eq: the Completion Event Queue that received the event
+ **/
+static void ceq_irq_handler(struct hinic_eq *eq)
+{
+	struct hinic_ceqs *ceqs = ceq_to_ceqs(eq);
+	u32 ceqe;
+	int i;
+
+	for (i = 0; i < eq->q_len; i++) {
+		ceqe = *(GET_CURR_CEQ_ELEM(eq));
+
+		/* Data in HW is in Big endian Format */
+		ceqe = be32_to_cpu(ceqe);
+
+		/* HW toggles the wrapped bit, when it adds eq element event */
+		if (HINIC_EQ_ELEM_DESC_GET(ceqe, WRAPPED) == eq->wrapped)
+			break;
+
+		ceq_event_handler(ceqs, ceqe);
+
+		eq->cons_idx++;
+
+		if (eq->cons_idx == eq->q_len) {
+			eq->cons_idx = 0;
+			eq->wrapped = !eq->wrapped;
+		}
+	}
+}
+
+/**
+ * eq_irq_handler - handler for the EQ event
+ * @data: the Event Queue that received the event
+ **/
+static void eq_irq_handler(void *data)
+{
+	struct hinic_eq *eq = data;
+
+	if (eq->type == HINIC_AEQ)
+		aeq_irq_handler(eq);
+	else if (eq->type == HINIC_CEQ)
+		ceq_irq_handler(eq);
+
+	eq_update_ci(eq);
+}
+
+/**
+ * eq_irq_work - the work of the EQ that received the event
+ * @work: the work struct that is associated with the EQ
+ **/
+static void eq_irq_work(struct work_struct *work)
+{
+	struct hinic_eq_work *aeq_work = work_to_aeq_work(work);
+	struct hinic_eq *aeq;
+
+	aeq = aeq_work->data;
+	eq_irq_handler(aeq);
+}
+
+/**
+ * ceq_tasklet - the tasklet of the EQ that received the event
+ * @ceq_data: the eq
+ **/
+static void ceq_tasklet(unsigned long ceq_data)
+{
+	struct hinic_eq *ceq = (struct hinic_eq *)ceq_data;
+
+	eq_irq_handler(ceq);
+}
+
+/**
+ * aeq_interrupt - aeq interrupt handler
+ * @irq: irq number
+ * @data: the Async Event Queue that collected the event
+ **/
+static irqreturn_t aeq_interrupt(int irq, void *data)
+{
+	struct hinic_eq_work *aeq_work;
+	struct hinic_eq *aeq = data;
+	struct hinic_aeqs *aeqs;
+
+	/* clear resend timer cnt register */
+	hinic_msix_attr_cnt_clear(aeq->hwif, aeq->msix_entry.entry);
+
+	aeq_work = &aeq->aeq_work;
+	aeq_work->data = aeq;
+
+	aeqs = aeq_to_aeqs(aeq);
+	queue_work(aeqs->workq, &aeq_work->work);
+
+	return IRQ_HANDLED;
+}
+
+/**
+ * ceq_interrupt - ceq interrupt handler
+ * @irq: irq number
+ * @data: the Completion Event Queue that collected the event
+ **/
+static irqreturn_t ceq_interrupt(int irq, void *data)
+{
+	struct hinic_eq *ceq = data;
+
+	/* clear resend timer cnt register */
+	hinic_msix_attr_cnt_clear(ceq->hwif, ceq->msix_entry.entry);
+
+	tasklet_schedule(&ceq->ceq_tasklet);
+
+	return IRQ_HANDLED;
+}
+
+static void set_ctrl0(struct hinic_eq *eq)
+{
+	struct msix_entry *msix_entry = &eq->msix_entry;
+	enum hinic_eq_type type = eq->type;
+	u32 addr, val, ctrl0;
+
+	if (type == HINIC_AEQ) {
+		/* RMW Ctrl0 */
+		addr = HINIC_CSR_AEQ_CTRL_0_ADDR(eq->q_id);
+
+		val = hinic_hwif_read_reg(eq->hwif, addr);
+
+		val = HINIC_AEQ_CTRL_0_CLEAR(val, INT_IDX)      &
+		      HINIC_AEQ_CTRL_0_CLEAR(val, DMA_ATTR)     &
+		      HINIC_AEQ_CTRL_0_CLEAR(val, PCI_INTF_IDX) &
+		      HINIC_AEQ_CTRL_0_CLEAR(val, INT_MODE);
+
+		ctrl0 = HINIC_AEQ_CTRL_0_SET(msix_entry->entry, INT_IDX)     |
+			HINIC_AEQ_CTRL_0_SET(DMA_ATTR_AEQ_DEFAULT, DMA_ATTR) |
+			HINIC_AEQ_CTRL_0_SET(HINIC_HWIF_PCI_INTF(eq->hwif),
+					     PCI_INTF_IDX)                   |
+			HINIC_AEQ_CTRL_0_SET(EQ_INT_MODE_ARMED, INT_MODE);
+
+		val |= ctrl0;
+
+		hinic_hwif_write_reg(eq->hwif, addr, val);
+	} else if (type == HINIC_CEQ) {
+		/* RMW Ctrl0 */
+		addr = HINIC_CSR_CEQ_CTRL_0_ADDR(eq->q_id);
+
+		val = hinic_hwif_read_reg(eq->hwif, addr);
+
+		val = HINIC_CEQ_CTRL_0_CLEAR(val, INTR_IDX)     &
+		      HINIC_CEQ_CTRL_0_CLEAR(val, DMA_ATTR)     &
+		      HINIC_CEQ_CTRL_0_CLEAR(val, KICK_THRESH)  &
+		      HINIC_CEQ_CTRL_0_CLEAR(val, PCI_INTF_IDX) &
+		      HINIC_CEQ_CTRL_0_CLEAR(val, INTR_MODE);
+
+		ctrl0 = HINIC_CEQ_CTRL_0_SET(msix_entry->entry, INTR_IDX)     |
+			HINIC_CEQ_CTRL_0_SET(DMA_ATTR_CEQ_DEFAULT, DMA_ATTR)  |
+			HINIC_CEQ_CTRL_0_SET(THRESH_CEQ_DEFAULT, KICK_THRESH) |
+			HINIC_CEQ_CTRL_0_SET(HINIC_HWIF_PCI_INTF(eq->hwif),
+					     PCI_INTF_IDX)                    |
+			HINIC_CEQ_CTRL_0_SET(EQ_INT_MODE_ARMED, INTR_MODE);
+
+		val |= ctrl0;
+
+		hinic_hwif_write_reg(eq->hwif, addr, val);
+	}
+}
+
+static void set_ctrl1(struct hinic_eq *eq)
+{
+	enum hinic_eq_type type = eq->type;
+	u32 page_size_val, elem_size;
+	u32 addr, val, ctrl1;
+
+	if (type == HINIC_AEQ) {
+		/* RMW Ctrl1 */
+		addr = HINIC_CSR_AEQ_CTRL_1_ADDR(eq->q_id);
+
+		page_size_val = EQ_SET_HW_PAGE_SIZE_VAL(eq);
+		elem_size = EQ_SET_HW_ELEM_SIZE_VAL(eq);
+
+		val = hinic_hwif_read_reg(eq->hwif, addr);
+
+		val = HINIC_AEQ_CTRL_1_CLEAR(val, LEN)          &
+		      HINIC_AEQ_CTRL_1_CLEAR(val, ELEM_SIZE)    &
+		      HINIC_AEQ_CTRL_1_CLEAR(val, PAGE_SIZE);
+
+		ctrl1 = HINIC_AEQ_CTRL_1_SET(eq->q_len, LEN)            |
+			HINIC_AEQ_CTRL_1_SET(elem_size, ELEM_SIZE)      |
+			HINIC_AEQ_CTRL_1_SET(page_size_val, PAGE_SIZE);
+
+		val |= ctrl1;
+
+		hinic_hwif_write_reg(eq->hwif, addr, val);
+	} else if (type == HINIC_CEQ) {
+		/* RMW Ctrl1 */
+		addr = HINIC_CSR_CEQ_CTRL_1_ADDR(eq->q_id);
+
+		page_size_val = EQ_SET_HW_PAGE_SIZE_VAL(eq);
+
+		val = hinic_hwif_read_reg(eq->hwif, addr);
+
+		val = HINIC_CEQ_CTRL_1_CLEAR(val, LEN) &
+		      HINIC_CEQ_CTRL_1_CLEAR(val, PAGE_SIZE);
+
+		ctrl1 = HINIC_CEQ_CTRL_1_SET(eq->q_len, LEN) |
+			HINIC_CEQ_CTRL_1_SET(page_size_val, PAGE_SIZE);
+
+		val |= ctrl1;
+
+		hinic_hwif_write_reg(eq->hwif, addr, val);
+	}
+}
+
+/**
+ * set_eq_ctrls - setting eq's ctrl registers
+ * @eq: the Event Queue for setting
+ **/
+static void set_eq_ctrls(struct hinic_eq *eq)
+{
+	set_ctrl0(eq);
+	set_ctrl1(eq);
+}
+
+/**
+ * aeq_elements_init - initialize all the elements in the aeq
+ * @eq: the Async Event Queue
+ * @init_val: value to initialize the elements with it
+ **/
+static void aeq_elements_init(struct hinic_eq *eq, u32 init_val)
+{
+	struct hinic_aeq_elem *aeqe;
+	int i;
+
+	for (i = 0; i < eq->q_len; i++) {
+		aeqe = GET_AEQ_ELEM(eq, i);
+		aeqe->desc = cpu_to_be32(init_val);
+	}
+
+	wmb();  /* Write the initilzation values */
+}
+
+/**
+ * ceq_elements_init - Initialize all the elements in the ceq
+ * @eq: the event queue
+ * @init_val: value to init with it the elements
+ **/
+static void ceq_elements_init(struct hinic_eq *eq, u32 init_val)
+{
+	u32 *ceqe;
+	int i;
+
+	for (i = 0; i < eq->q_len; i++) {
+		ceqe = GET_CEQ_ELEM(eq, i);
+		*(ceqe) = cpu_to_be32(init_val);
+	}
+
+	wmb();  /* Write the initilzation values */
+}
+
+/**
+ * alloc_eq_pages - allocate the pages for the queue
+ * @eq: the event queue
+ *
+ * Return 0 - Success, Negative - Failure
+ **/
+static int alloc_eq_pages(struct hinic_eq *eq)
+{
+	struct hinic_hwif *hwif = eq->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	u32 init_val, addr, val;
+	size_t addr_size;
+	int err, pg;
+
+	addr_size = eq->num_pages * sizeof(*eq->dma_addr);
+	eq->dma_addr = devm_kzalloc(&pdev->dev, addr_size, GFP_KERNEL);
+	if (!eq->dma_addr)
+		return -ENOMEM;
+
+	addr_size = eq->num_pages * sizeof(*eq->virt_addr);
+	eq->virt_addr = devm_kzalloc(&pdev->dev, addr_size, GFP_KERNEL);
+	if (!eq->virt_addr) {
+		err = -ENOMEM;
+		goto err_virt_addr_alloc;
+	}
+
+	for (pg = 0; pg < eq->num_pages; pg++) {
+		eq->virt_addr[pg] = dma_zalloc_coherent(&pdev->dev,
+							eq->page_size,
+							&eq->dma_addr[pg],
+							GFP_KERNEL);
+		if (!eq->virt_addr[pg]) {
+			err = -ENOMEM;
+			goto err_dma_alloc;
+		}
+
+		addr = EQ_HI_PHYS_ADDR_REG(eq, pg);
+		val = upper_32_bits(eq->dma_addr[pg]);
+
+		hinic_hwif_write_reg(hwif, addr, val);
+
+		addr = EQ_LO_PHYS_ADDR_REG(eq, pg);
+		val = lower_32_bits(eq->dma_addr[pg]);
+
+		hinic_hwif_write_reg(hwif, addr, val);
+	}
+
+	init_val = HINIC_EQ_ELEM_DESC_SET(eq->wrapped, WRAPPED);
+
+	if (eq->type == HINIC_AEQ)
+		aeq_elements_init(eq, init_val);
+	else if (eq->type == HINIC_CEQ)
+		ceq_elements_init(eq, init_val);
+
+	return 0;
+
+err_dma_alloc:
+	while (--pg >= 0)
+		dma_free_coherent(&pdev->dev, eq->page_size,
+				  eq->virt_addr[pg],
+				  eq->dma_addr[pg]);
+
+	devm_kfree(&pdev->dev, eq->virt_addr);
+
+err_virt_addr_alloc:
+	devm_kfree(&pdev->dev, eq->dma_addr);
+	return err;
+}
+
+/**
+ * free_eq_pages - free the pages of the queue
+ * @eq: the Event Queue
+ **/
+static void free_eq_pages(struct hinic_eq *eq)
+{
+	struct hinic_hwif *hwif = eq->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	int pg;
+
+	for (pg = 0; pg < eq->num_pages; pg++)
+		dma_free_coherent(&pdev->dev, eq->page_size,
+				  eq->virt_addr[pg],
+				  eq->dma_addr[pg]);
+
+	devm_kfree(&pdev->dev, eq->virt_addr);
+	devm_kfree(&pdev->dev, eq->dma_addr);
+}
+
+/**
+ * init_eq - initialize Event Queue
+ * @eq: the event queue
+ * @hwif: the HW interface of a PCI function device
+ * @type: the type of the event queue, aeq or ceq
+ * @q_id: Queue id number
+ * @q_len: the number of EQ elements
+ * @page_size: the page size of the pages in the event queue
+ * @entry: msix entry associated with the event queue
+ *
+ * Return 0 - Success, Negative - Failure
+ **/
+static int init_eq(struct hinic_eq *eq, struct hinic_hwif *hwif,
+		   enum hinic_eq_type type, int q_id, u32 q_len, u32 page_size,
+		   struct msix_entry entry)
+{
+	struct pci_dev *pdev = hwif->pdev;
+	int err;
+
+	eq->hwif = hwif;
+	eq->type = type;
+	eq->q_id = q_id;
+	eq->q_len = q_len;
+	eq->page_size = page_size;
+
+	/* Clear PI and CI, also clear the ARM bit */
+	hinic_hwif_write_reg(eq->hwif, EQ_CONS_IDX_REG_ADDR(eq), 0);
+	hinic_hwif_write_reg(eq->hwif, EQ_PROD_IDX_REG_ADDR(eq), 0);
+
+	eq->cons_idx = 0;
+	eq->wrapped = 0;
+
+	if (type == HINIC_AEQ) {
+		eq->elem_size = HINIC_AEQE_SIZE;
+	} else if (type == HINIC_CEQ) {
+		eq->elem_size = HINIC_CEQE_SIZE;
+	} else {
+		dev_err(&pdev->dev, "Invalid EQ type\n");
+		return -EINVAL;
+	}
+
+	eq->num_pages = GET_EQ_NUM_PAGES(eq, page_size);
+	eq->num_elem_in_pg = GET_EQ_NUM_ELEMS_IN_PG(eq, page_size);
+
+	eq->msix_entry = entry;
+
+	if (eq->num_elem_in_pg & (eq->num_elem_in_pg - 1)) {
+		dev_err(&pdev->dev, "num elements in eq page != power of 2\n");
+		return -EINVAL;
+	}
+
+	if (eq->num_pages > EQ_MAX_PAGES) {
+		dev_err(&pdev->dev, "too many pages for eq\n");
+		return -EINVAL;
+	}
+
+	set_eq_ctrls(eq);
+	eq_update_ci(eq);
+
+	err = alloc_eq_pages(eq);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to allocate pages for eq\n");
+		return err;
+	}
+
+	if (type == HINIC_AEQ) {
+		struct hinic_eq_work *aeq_work = &eq->aeq_work;
+
+		INIT_WORK(&aeq_work->work, eq_irq_work);
+	} else if (type == HINIC_CEQ) {
+		tasklet_init(&eq->ceq_tasklet, ceq_tasklet,
+			     (unsigned long)eq);
+	}
+
+	/* set the attributes of the msix entry */
+	hinic_msix_attr_set(eq->hwif, eq->msix_entry.entry,
+			    HINIC_EQ_MSIX_PENDING_LIMIT_DEFAULT,
+			    HINIC_EQ_MSIX_COALESC_TIMER_DEFAULT,
+			    HINIC_EQ_MSIX_LLI_TIMER_DEFAULT,
+			    HINIC_EQ_MSIX_LLI_CREDIT_LIMIT_DEFAULT,
+			    HINIC_EQ_MSIX_RESEND_TIMER_DEFAULT);
+
+	if (type == HINIC_AEQ)
+		err = request_irq(entry.vector, aeq_interrupt, 0,
+				  "hinic_aeq", eq);
+	else if (type == HINIC_CEQ)
+		err = request_irq(entry.vector, ceq_interrupt, 0,
+				  "hinic_ceq", eq);
+
+	if (err) {
+		dev_err(&pdev->dev, "Failed to request irq for the EQ\n");
+		goto err_req_irq;
+	}
+
+	return 0;
+
+err_req_irq:
+	free_eq_pages(eq);
+	return err;
+}
+
+/**
+ * remove_eq - remove Event Queue
+ * @eq: the event queue
+ **/
+static void remove_eq(struct hinic_eq *eq)
+{
+	struct msix_entry *entry = &eq->msix_entry;
+
+	free_irq(entry->vector, eq);
+
+	if (eq->type == HINIC_AEQ) {
+		struct hinic_eq_work *aeq_work = &eq->aeq_work;
+
+		cancel_work_sync(&aeq_work->work);
+	} else if (eq->type == HINIC_CEQ) {
+		tasklet_kill(&eq->ceq_tasklet);
+	}
+
+	free_eq_pages(eq);
+}
+
+/**
+ * hinic_aeqs_init - initialize all the aeqs
+ * @aeqs: pointer to Async eqs of the chip
+ * @hwif: the HW interface of a PCI function device
+ * @num_aeqs: number of AEQs
+ * @q_len: number of EQ elements
+ * @page_size: the page size of the pages in the event queue
+ * @msix_entries: msix entries associated with the event queues
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_aeqs_init(struct hinic_aeqs *aeqs, struct hinic_hwif *hwif,
+		    int num_aeqs, u32 q_len, u32 page_size,
+		    struct msix_entry *msix_entries)
+{
+	struct pci_dev *pdev = hwif->pdev;
+	int err, i, q_id;
+
+	aeqs->workq = create_singlethread_workqueue(HINIC_EQS_WQ_NAME);
+	if (!aeqs->workq)
+		return -ENOMEM;
+
+	aeqs->hwif = hwif;
+	aeqs->num_aeqs = num_aeqs;
+
+	for (q_id = 0; q_id < num_aeqs; q_id++) {
+		err = init_eq(&aeqs->aeq[q_id], hwif, HINIC_AEQ, q_id, q_len,
+			      page_size, msix_entries[q_id]);
+		if (err) {
+			dev_err(&pdev->dev, "Failed to init aeq %d\n", q_id);
+			goto err_init_aeq;
+		}
+	}
+
+	return 0;
+
+err_init_aeq:
+	for (i = 0; i < q_id; i++)
+		remove_eq(&aeqs->aeq[i]);
+
+	destroy_workqueue(aeqs->workq);
+	return err;
+}
+
+/**
+ * hinic_aeqs_free - free all the aeqs
+ * @aeqs: pointer to Async eqs of the chip
+ **/
+void hinic_aeqs_free(struct hinic_aeqs *aeqs)
+{
+	int q_id;
+
+	for (q_id = 0; q_id < aeqs->num_aeqs ; q_id++)
+		remove_eq(&aeqs->aeq[q_id]);
+
+	destroy_workqueue(aeqs->workq);
+}
+
+/**
+ * hinic_ceqs_init - init all the ceqs
+ * @ceqs: ceqs part of the chip
+ * @hwif: the hardware interface of a pci function device
+ * @num_ceqs: number of CEQs
+ * @q_len: number of EQ elements
+ * @page_size: the page size of the event queue
+ * @msix_entries: msix entries associated with the event queues
+ *
+ * Return 0 - Success, Negative - Failure
+ **/
+int hinic_ceqs_init(struct hinic_ceqs *ceqs, struct hinic_hwif *hwif,
+		    int num_ceqs, u32 q_len, u32 page_size,
+		    struct msix_entry *msix_entries)
+{
+	struct pci_dev *pdev = hwif->pdev;
+	int i, q_id, err;
+
+	ceqs->hwif = hwif;
+	ceqs->num_ceqs = num_ceqs;
+
+	for (q_id = 0; q_id < num_ceqs; q_id++) {
+		err = init_eq(&ceqs->ceq[q_id], hwif, HINIC_CEQ, q_id, q_len,
+			      page_size, msix_entries[q_id]);
+		if (err) {
+			dev_err(&pdev->dev, "Failed to init ceq %d\n", q_id);
+			goto err_init_ceq;
+		}
+	}
+
+	return 0;
+
+err_init_ceq:
+	for (i = 0; i < q_id; i++)
+		remove_eq(&ceqs->ceq[i]);
+
+	return err;
+}
+
+/**
+ * hinic_ceqs_free - free all the ceqs
+ * @ceqs: ceqs part of the chip
+ **/
+void hinic_ceqs_free(struct hinic_ceqs *ceqs)
+{
+	int q_id;
+
+	for (q_id = 0; q_id < ceqs->num_ceqs; q_id++)
+		remove_eq(&ceqs->ceq[q_id]);
+}
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_eqs.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_eqs.h
new file mode 100644
index 000000000000..ecb9c2bc6dc8
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_eqs.h
@@ -0,0 +1,265 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#ifndef HINIC_HW_EQS_H
+#define HINIC_HW_EQS_H
+
+#include <linux/types.h>
+#include <linux/workqueue.h>
+#include <linux/pci.h>
+#include <linux/sizes.h>
+#include <linux/bitops.h>
+#include <linux/interrupt.h>
+
+#include "hinic_hw_if.h"
+
+#define HINIC_AEQ_CTRL_0_INT_IDX_SHIFT          0
+#define HINIC_AEQ_CTRL_0_DMA_ATTR_SHIFT         12
+#define HINIC_AEQ_CTRL_0_PCI_INTF_IDX_SHIFT     20
+#define HINIC_AEQ_CTRL_0_INT_MODE_SHIFT         31
+
+#define HINIC_AEQ_CTRL_0_INT_IDX_MASK           0x3FF
+#define HINIC_AEQ_CTRL_0_DMA_ATTR_MASK          0x3F
+#define HINIC_AEQ_CTRL_0_PCI_INTF_IDX_MASK      0x3
+#define HINIC_AEQ_CTRL_0_INT_MODE_MASK          0x1
+
+#define HINIC_AEQ_CTRL_0_SET(val, member)       \
+			(((u32)(val) & HINIC_AEQ_CTRL_0_##member##_MASK) << \
+			 HINIC_AEQ_CTRL_0_##member##_SHIFT)
+
+#define HINIC_AEQ_CTRL_0_CLEAR(val, member)     \
+			((val) & (~(HINIC_AEQ_CTRL_0_##member##_MASK \
+			 << HINIC_AEQ_CTRL_0_##member##_SHIFT)))
+
+#define HINIC_AEQ_CTRL_1_LEN_SHIFT              0
+#define HINIC_AEQ_CTRL_1_ELEM_SIZE_SHIFT        24
+#define HINIC_AEQ_CTRL_1_PAGE_SIZE_SHIFT        28
+
+#define HINIC_AEQ_CTRL_1_LEN_MASK               0x1FFFFF
+#define HINIC_AEQ_CTRL_1_ELEM_SIZE_MASK         0x3
+#define HINIC_AEQ_CTRL_1_PAGE_SIZE_MASK         0xF
+
+#define HINIC_AEQ_CTRL_1_SET(val, member)       \
+			(((u32)(val) & HINIC_AEQ_CTRL_1_##member##_MASK) << \
+			 HINIC_AEQ_CTRL_1_##member##_SHIFT)
+
+#define HINIC_AEQ_CTRL_1_CLEAR(val, member)     \
+			((val) & (~(HINIC_AEQ_CTRL_1_##member##_MASK \
+			 << HINIC_AEQ_CTRL_1_##member##_SHIFT)))
+
+#define HINIC_CEQ_CTRL_0_INTR_IDX_SHIFT         0
+#define HINIC_CEQ_CTRL_0_DMA_ATTR_SHIFT         12
+#define HINIC_CEQ_CTRL_0_KICK_THRESH_SHIFT      20
+#define HINIC_CEQ_CTRL_0_PCI_INTF_IDX_SHIFT     24
+#define HINIC_CEQ_CTRL_0_INTR_MODE_SHIFT        31
+
+#define HINIC_CEQ_CTRL_0_INTR_IDX_MASK          0x3FF
+#define HINIC_CEQ_CTRL_0_DMA_ATTR_MASK          0x3F
+#define HINIC_CEQ_CTRL_0_KICK_THRESH_MASK       0xF
+#define HINIC_CEQ_CTRL_0_PCI_INTF_IDX_MASK      0x3
+#define HINIC_CEQ_CTRL_0_INTR_MODE_MASK         0x1
+
+#define HINIC_CEQ_CTRL_0_SET(val, member)       \
+			(((u32)(val) & HINIC_CEQ_CTRL_0_##member##_MASK) << \
+			 HINIC_CEQ_CTRL_0_##member##_SHIFT)
+
+#define HINIC_CEQ_CTRL_0_CLEAR(val, member)     \
+			((val) & (~(HINIC_CEQ_CTRL_0_##member##_MASK \
+			 << HINIC_CEQ_CTRL_0_##member##_SHIFT)))
+
+#define HINIC_CEQ_CTRL_1_LEN_SHIFT              0
+#define HINIC_CEQ_CTRL_1_PAGE_SIZE_SHIFT        28
+
+#define HINIC_CEQ_CTRL_1_LEN_MASK               0x1FFFFF
+#define HINIC_CEQ_CTRL_1_PAGE_SIZE_MASK         0xF
+
+#define HINIC_CEQ_CTRL_1_SET(val, member)       \
+			(((u32)(val) & HINIC_CEQ_CTRL_1_##member##_MASK) << \
+			 HINIC_CEQ_CTRL_1_##member##_SHIFT)
+
+#define HINIC_CEQ_CTRL_1_CLEAR(val, member)     \
+			((val) & (~(HINIC_CEQ_CTRL_1_##member##_MASK \
+			 << HINIC_CEQ_CTRL_1_##member##_SHIFT)))
+
+#define HINIC_EQ_ELEM_DESC_TYPE_SHIFT           0
+#define HINIC_EQ_ELEM_DESC_SRC_SHIFT            7
+#define HINIC_EQ_ELEM_DESC_SIZE_SHIFT           8
+#define HINIC_EQ_ELEM_DESC_WRAPPED_SHIFT        31
+
+#define HINIC_EQ_ELEM_DESC_TYPE_MASK            0x7F
+#define HINIC_EQ_ELEM_DESC_SRC_MASK             0x1
+#define HINIC_EQ_ELEM_DESC_SIZE_MASK            0xFF
+#define HINIC_EQ_ELEM_DESC_WRAPPED_MASK         0x1
+
+#define HINIC_EQ_ELEM_DESC_SET(val, member)     \
+			(((u32)(val) & HINIC_EQ_ELEM_DESC_##member##_MASK) << \
+			 HINIC_EQ_ELEM_DESC_##member##_SHIFT)
+
+#define HINIC_EQ_ELEM_DESC_GET(val, member)     \
+			(((val) >> HINIC_EQ_ELEM_DESC_##member##_SHIFT) & \
+			 HINIC_EQ_ELEM_DESC_##member##_MASK)
+
+#define HINIC_EQ_CI_IDX_SHIFT                   0
+#define HINIC_EQ_CI_WRAPPED_SHIFT               20
+#define HINIC_EQ_CI_XOR_CHKSUM_SHIFT            24
+#define HINIC_EQ_CI_INT_ARMED_SHIFT             31
+
+#define HINIC_EQ_CI_IDX_MASK                    0xFFFFF
+#define HINIC_EQ_CI_WRAPPED_MASK                0x1
+#define HINIC_EQ_CI_XOR_CHKSUM_MASK             0xF
+#define HINIC_EQ_CI_INT_ARMED_MASK              0x1
+
+#define HINIC_EQ_CI_SET(val, member)            \
+			(((u32)(val) & HINIC_EQ_CI_##member##_MASK) << \
+			 HINIC_EQ_CI_##member##_SHIFT)
+
+#define HINIC_EQ_CI_CLEAR(val, member)          \
+			((val) & (~(HINIC_EQ_CI_##member##_MASK \
+			 << HINIC_EQ_CI_##member##_SHIFT)))
+
+#define HINIC_MAX_AEQS                  4
+#define HINIC_MAX_CEQS                  32
+
+#define HINIC_AEQE_SIZE                 64
+#define HINIC_CEQE_SIZE                 4
+
+#define HINIC_AEQE_DESC_SIZE            4
+#define HINIC_AEQE_DATA_SIZE            \
+			(HINIC_AEQE_SIZE - HINIC_AEQE_DESC_SIZE)
+
+#define HINIC_DEFAULT_AEQ_LEN           64
+#define HINIC_DEFAULT_CEQ_LEN           1024
+
+#define HINIC_EQ_PAGE_SIZE              SZ_4K
+
+#define HINIC_CEQ_ID_CMDQ               0
+
+enum hinic_eq_type {
+	HINIC_AEQ,
+	HINIC_CEQ,
+};
+
+enum hinic_aeq_type {
+	HINIC_MSG_FROM_MGMT_CPU = 2,
+
+	HINIC_MAX_AEQ_EVENTS,
+};
+
+enum hinic_ceq_type {
+	HINIC_CEQ_CMDQ = 3,
+
+	HINIC_MAX_CEQ_EVENTS,
+};
+
+enum hinic_eqe_state {
+	HINIC_EQE_ENABLED = BIT(0),
+	HINIC_EQE_RUNNING = BIT(1),
+};
+
+struct hinic_aeq_elem {
+	u8      data[HINIC_AEQE_DATA_SIZE];
+	u32     desc;
+};
+
+struct hinic_eq_work {
+	struct work_struct      work;
+	void                    *data;
+};
+
+struct hinic_eq {
+	struct hinic_hwif       *hwif;
+
+	enum hinic_eq_type      type;
+	int                     q_id;
+	u32                     q_len;
+	u32                     page_size;
+
+	u32                     cons_idx;
+	int                     wrapped;
+
+	size_t                  elem_size;
+	int                     num_pages;
+	int                     num_elem_in_pg;
+
+	struct msix_entry       msix_entry;
+
+	dma_addr_t              *dma_addr;
+	void                    **virt_addr;
+
+	struct hinic_eq_work    aeq_work;
+
+	struct tasklet_struct   ceq_tasklet;
+};
+
+struct hinic_hw_event_cb {
+	void    (*hwe_handler)(void *handle, void *data, u8 size);
+	void                    *handle;
+	unsigned long           hwe_state;
+};
+
+struct hinic_aeqs {
+	struct hinic_hwif       *hwif;
+
+	struct hinic_eq         aeq[HINIC_MAX_AEQS];
+	int                     num_aeqs;
+
+	struct hinic_hw_event_cb hwe_cb[HINIC_MAX_AEQ_EVENTS];
+
+	struct workqueue_struct *workq;
+};
+
+struct hinic_ceq_cb {
+	void    (*handler)(void *handle, u32 ceqe_data);
+	void                    *handle;
+	enum hinic_eqe_state    ceqe_state;
+};
+
+struct hinic_ceqs {
+	struct hinic_hwif       *hwif;
+
+	struct hinic_eq         ceq[HINIC_MAX_CEQS];
+	int                     num_ceqs;
+
+	struct hinic_ceq_cb     ceq_cb[HINIC_MAX_CEQ_EVENTS];
+};
+
+void hinic_aeq_register_hw_cb(struct hinic_aeqs *aeqs,
+			      enum hinic_aeq_type event, void *handle,
+			      void (*hwe_handler)(void *handle, void *data,
+						  u8 size));
+
+void hinic_aeq_unregister_hw_cb(struct hinic_aeqs *aeqs,
+				enum hinic_aeq_type event);
+
+void hinic_ceq_register_cb(struct hinic_ceqs *ceqs,
+			   enum hinic_ceq_type event, void *handle,
+			   void (*ceq_cb)(void *handle, u32 ceqe_data));
+
+void hinic_ceq_unregister_cb(struct hinic_ceqs *ceqs,
+			     enum hinic_ceq_type event);
+
+int hinic_aeqs_init(struct hinic_aeqs *aeqs, struct hinic_hwif *hwif,
+		    int num_aeqs, u32 q_len, u32 page_size,
+		    struct msix_entry *msix_entries);
+
+void hinic_aeqs_free(struct hinic_aeqs *aeqs);
+
+int hinic_ceqs_init(struct hinic_ceqs *ceqs, struct hinic_hwif *hwif,
+		    int num_ceqs, u32 q_len, u32 page_size,
+		    struct msix_entry *msix_entries);
+
+void hinic_ceqs_free(struct hinic_ceqs *ceqs);
+
+#endif
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_if.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_if.c
new file mode 100644
index 000000000000..823a17061a97
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_if.c
@@ -0,0 +1,351 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#include <linux/pci.h>
+#include <linux/device.h>
+#include <linux/errno.h>
+#include <linux/io.h>
+#include <linux/types.h>
+#include <linux/bitops.h>
+
+#include "hinic_hw_csr.h"
+#include "hinic_hw_if.h"
+
+#define PCIE_ATTR_ENTRY         0
+
+#define VALID_MSIX_IDX(attr, msix_index) ((msix_index) < (attr)->num_irqs)
+
+/**
+ * hinic_msix_attr_set - set message attribute for msix entry
+ * @hwif: the HW interface of a pci function device
+ * @msix_index: msix_index
+ * @pending_limit: the maximum pending interrupt events (unit 8)
+ * @coalesc_timer: coalesc period for interrupt (unit 8 us)
+ * @lli_timer: replenishing period for low latency credit (unit 8 us)
+ * @lli_credit_limit: maximum credits for low latency msix messages (unit 8)
+ * @resend_timer: maximum wait for resending msix (unit coalesc period)
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_msix_attr_set(struct hinic_hwif *hwif, u16 msix_index,
+			u8 pending_limit, u8 coalesc_timer,
+			u8 lli_timer, u8 lli_credit_limit,
+			u8 resend_timer)
+{
+	u32 msix_ctrl, addr;
+
+	if (!VALID_MSIX_IDX(&hwif->attr, msix_index))
+		return -EINVAL;
+
+	msix_ctrl = HINIC_MSIX_ATTR_SET(pending_limit, PENDING_LIMIT)   |
+		    HINIC_MSIX_ATTR_SET(coalesc_timer, COALESC_TIMER)   |
+		    HINIC_MSIX_ATTR_SET(lli_timer, LLI_TIMER)           |
+		    HINIC_MSIX_ATTR_SET(lli_credit_limit, LLI_CREDIT)   |
+		    HINIC_MSIX_ATTR_SET(resend_timer, RESEND_TIMER);
+
+	addr = HINIC_CSR_MSIX_CTRL_ADDR(msix_index);
+
+	hinic_hwif_write_reg(hwif, addr, msix_ctrl);
+	return 0;
+}
+
+/**
+ * hinic_msix_attr_get - get message attribute of msix entry
+ * @hwif: the HW interface of a pci function device
+ * @msix_index: msix_index
+ * @pending_limit: the maximum pending interrupt events (unit 8)
+ * @coalesc_timer: coalesc period for interrupt (unit 8 us)
+ * @lli_timer: replenishing period for low latency credit (unit 8 us)
+ * @lli_credit_limit: maximum credits for low latency msix messages (unit 8)
+ * @resend_timer: maximum wait for resending msix (unit coalesc period)
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_msix_attr_get(struct hinic_hwif *hwif, u16 msix_index,
+			u8 *pending_limit, u8 *coalesc_timer,
+			u8 *lli_timer, u8 *lli_credit_limit,
+			u8 *resend_timer)
+{
+	u32 addr, val;
+
+	if (!VALID_MSIX_IDX(&hwif->attr, msix_index))
+		return -EINVAL;
+
+	addr = HINIC_CSR_MSIX_CTRL_ADDR(msix_index);
+	val  = hinic_hwif_read_reg(hwif, addr);
+
+	*pending_limit    = HINIC_MSIX_ATTR_GET(val, PENDING_LIMIT);
+	*coalesc_timer    = HINIC_MSIX_ATTR_GET(val, COALESC_TIMER);
+	*lli_timer        = HINIC_MSIX_ATTR_GET(val, LLI_TIMER);
+	*lli_credit_limit = HINIC_MSIX_ATTR_GET(val, LLI_CREDIT);
+	*resend_timer     = HINIC_MSIX_ATTR_GET(val, RESEND_TIMER);
+	return 0;
+}
+
+/**
+ * hinic_msix_attr_cnt_clear - clear message attribute counters for msix entry
+ * @hwif: the HW interface of a pci function device
+ * @msix_index: msix_index
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_msix_attr_cnt_clear(struct hinic_hwif *hwif, u16 msix_index)
+{
+	u32 msix_ctrl, addr;
+
+	if (!VALID_MSIX_IDX(&hwif->attr, msix_index))
+		return -EINVAL;
+
+	msix_ctrl = HINIC_MSIX_CNT_SET(1, RESEND_TIMER);
+	addr = HINIC_CSR_MSIX_CNT_ADDR(msix_index);
+
+	hinic_hwif_write_reg(hwif, addr, msix_ctrl);
+	return 0;
+}
+
+/**
+ * hinic_set_pf_action - set action on pf channel
+ * @hwif: the HW interface of a pci function device
+ * @action: action on pf channel
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+void hinic_set_pf_action(struct hinic_hwif *hwif, enum hinic_pf_action action)
+{
+	u32 attr5 = hinic_hwif_read_reg(hwif, HINIC_CSR_FUNC_ATTR5_ADDR);
+
+	attr5 = HINIC_FA5_CLEAR(attr5, PF_ACTION);
+	attr5 |= HINIC_FA5_SET(action, PF_ACTION);
+
+	hinic_hwif_write_reg(hwif, HINIC_CSR_FUNC_ATTR5_ADDR, attr5);
+}
+
+enum hinic_outbound_state hinic_outbound_state_get(struct hinic_hwif *hwif)
+{
+	u32 attr4 = hinic_hwif_read_reg(hwif, HINIC_CSR_FUNC_ATTR4_ADDR);
+
+	return HINIC_FA4_GET(attr4, OUTBOUND_STATE);
+}
+
+void hinic_outbound_state_set(struct hinic_hwif *hwif,
+			      enum hinic_outbound_state outbound_state)
+{
+	u32 attr4 = hinic_hwif_read_reg(hwif, HINIC_CSR_FUNC_ATTR4_ADDR);
+
+	attr4 = HINIC_FA4_CLEAR(attr4, OUTBOUND_STATE);
+	attr4 |= HINIC_FA4_SET(outbound_state, OUTBOUND_STATE);
+
+	hinic_hwif_write_reg(hwif, HINIC_CSR_FUNC_ATTR4_ADDR, attr4);
+}
+
+enum hinic_db_state hinic_db_state_get(struct hinic_hwif *hwif)
+{
+	u32 attr4 = hinic_hwif_read_reg(hwif, HINIC_CSR_FUNC_ATTR4_ADDR);
+
+	return HINIC_FA4_GET(attr4, DB_STATE);
+}
+
+void hinic_db_state_set(struct hinic_hwif *hwif,
+			enum hinic_db_state db_state)
+{
+	u32 attr4 = hinic_hwif_read_reg(hwif, HINIC_CSR_FUNC_ATTR4_ADDR);
+
+	attr4 = HINIC_FA4_CLEAR(attr4, DB_STATE);
+	attr4 |= HINIC_FA4_SET(db_state, DB_STATE);
+
+	hinic_hwif_write_reg(hwif, HINIC_CSR_FUNC_ATTR4_ADDR, attr4);
+}
+
+/**
+ * hwif_ready - test if the HW is ready for use
+ * @hwif: the HW interface of a pci function device
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int hwif_ready(struct hinic_hwif *hwif)
+{
+	struct pci_dev *pdev = hwif->pdev;
+	u32 addr, attr1;
+
+	addr   = HINIC_CSR_FUNC_ATTR1_ADDR;
+	attr1  = hinic_hwif_read_reg(hwif, addr);
+
+	if (!HINIC_FA1_GET(attr1, INIT_STATUS)) {
+		dev_err(&pdev->dev, "hwif status is not ready\n");
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+/**
+ * set_hwif_attr - set the attributes in the relevant members in hwif
+ * @hwif: the HW interface of a pci function device
+ * @attr0: the first attribute that was read from the hw
+ * @attr1: the second attribute that was read from the hw
+ **/
+static void set_hwif_attr(struct hinic_hwif *hwif, u32 attr0, u32 attr1)
+{
+	hwif->attr.func_idx     = HINIC_FA0_GET(attr0, FUNC_IDX);
+	hwif->attr.pf_idx       = HINIC_FA0_GET(attr0, PF_IDX);
+	hwif->attr.pci_intf_idx = HINIC_FA0_GET(attr0, PCI_INTF_IDX);
+	hwif->attr.func_type    = HINIC_FA0_GET(attr0, FUNC_TYPE);
+
+	hwif->attr.num_aeqs = BIT(HINIC_FA1_GET(attr1, AEQS_PER_FUNC));
+	hwif->attr.num_ceqs = BIT(HINIC_FA1_GET(attr1, CEQS_PER_FUNC));
+	hwif->attr.num_irqs = BIT(HINIC_FA1_GET(attr1, IRQS_PER_FUNC));
+	hwif->attr.num_dma_attr = BIT(HINIC_FA1_GET(attr1, DMA_ATTR_PER_FUNC));
+}
+
+/**
+ * read_hwif_attr - read the attributes and set members in hwif
+ * @hwif: the HW interface of a pci function device
+ **/
+static void read_hwif_attr(struct hinic_hwif *hwif)
+{
+	u32 addr, attr0, attr1;
+
+	addr   = HINIC_CSR_FUNC_ATTR0_ADDR;
+	attr0  = hinic_hwif_read_reg(hwif, addr);
+
+	addr   = HINIC_CSR_FUNC_ATTR1_ADDR;
+	attr1  = hinic_hwif_read_reg(hwif, addr);
+
+	set_hwif_attr(hwif, attr0, attr1);
+}
+
+/**
+ * set_ppf - try to set hwif as ppf and set the type of hwif in this case
+ * @hwif: the HW interface of a pci function device
+ **/
+static void set_ppf(struct hinic_hwif *hwif)
+{
+	struct hinic_func_attr *attr = &hwif->attr;
+	u32 addr, val, ppf_election;
+
+	/* Read Modify Write */
+	addr = HINIC_CSR_PPF_ELECTION_ADDR(HINIC_HWIF_PCI_INTF(hwif));
+
+	val = hinic_hwif_read_reg(hwif, addr);
+	val = HINIC_PPF_ELECTION_CLEAR(val, IDX);
+
+	ppf_election = HINIC_PPF_ELECTION_SET(HINIC_HWIF_FUNC_IDX(hwif), IDX);
+
+	val |= ppf_election;
+	hinic_hwif_write_reg(hwif, addr, val);
+
+	/* check PPF */
+	val = hinic_hwif_read_reg(hwif, addr);
+
+	attr->ppf_idx = HINIC_PPF_ELECTION_GET(val, IDX);
+	if (attr->ppf_idx == HINIC_HWIF_FUNC_IDX(hwif))
+		attr->func_type = HINIC_PPF;
+}
+
+/**
+ * set_dma_attr - set the dma attributes in the HW
+ * @hwif: the HW interface of a pci function device
+ * @entry_idx: the entry index in the dma table
+ * @st: PCIE TLP steering tag
+ * @at: PCIE TLP AT field
+ * @ph: PCIE TLP Processing Hint field
+ * @no_snooping: PCIE TLP No snooping
+ * @tph_en: PCIE TLP Processing Hint Enable
+ **/
+static void set_dma_attr(struct hinic_hwif *hwif, u32 entry_idx,
+			 u8 st, u8 at, u8 ph,
+			 enum hinic_pcie_nosnoop no_snooping,
+			 enum hinic_pcie_tph tph_en)
+{
+	u32 addr, val, dma_attr_entry;
+
+	/* Read Modify Write */
+	addr = HINIC_CSR_DMA_ATTR_ADDR(entry_idx);
+
+	val = hinic_hwif_read_reg(hwif, addr);
+	val = HINIC_DMA_ATTR_CLEAR(val, ST)             &
+	      HINIC_DMA_ATTR_CLEAR(val, AT)             &
+	      HINIC_DMA_ATTR_CLEAR(val, PH)             &
+	      HINIC_DMA_ATTR_CLEAR(val, NO_SNOOPING)    &
+	      HINIC_DMA_ATTR_CLEAR(val, TPH_EN);
+
+	dma_attr_entry = HINIC_DMA_ATTR_SET(st, ST)                     |
+			 HINIC_DMA_ATTR_SET(at, AT)                     |
+			 HINIC_DMA_ATTR_SET(ph, PH)                     |
+			 HINIC_DMA_ATTR_SET(no_snooping, NO_SNOOPING)   |
+			 HINIC_DMA_ATTR_SET(tph_en, TPH_EN);
+
+	val |= dma_attr_entry;
+	hinic_hwif_write_reg(hwif, addr, val);
+}
+
+/**
+ * dma_attr_table_init - initialize the the default dma attributes
+ * @hwif: the HW interface of a pci function device
+ **/
+static void dma_attr_init(struct hinic_hwif *hwif)
+{
+	set_dma_attr(hwif, PCIE_ATTR_ENTRY, HINIC_PCIE_ST_DISABLE,
+		     HINIC_PCIE_AT_DISABLE, HINIC_PCIE_PH_DISABLE,
+		     HINIC_PCIE_SNOOP, HINIC_PCIE_TPH_DISABLE);
+}
+
+/**
+ * hinic_init_hwif - initialize the hw interface
+ * @hwif: the HW interface of a pci function device
+ * @pdev: the pci device for acessing PCI resources
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_init_hwif(struct hinic_hwif *hwif, struct pci_dev *pdev)
+{
+	int err;
+
+	hwif->pdev = pdev;
+
+	hwif->cfg_regs_bar = pci_ioremap_bar(pdev, HINIC_PCI_CFG_REGS_BAR);
+	if (!hwif->cfg_regs_bar) {
+		dev_err(&pdev->dev, "Failed to map configuration regs\n");
+		return -ENOMEM;
+	}
+
+	err = hwif_ready(hwif);
+	if (err) {
+		dev_err(&pdev->dev, "HW interface is not ready\n");
+		goto err_hwif_ready;
+	}
+
+	read_hwif_attr(hwif);
+
+	if (HINIC_IS_PF(hwif))
+		set_ppf(hwif);
+
+	/* No transactionss before DMA is initialized */
+	dma_attr_init(hwif);
+	return 0;
+
+err_hwif_ready:
+	iounmap(hwif->cfg_regs_bar);
+	return err;
+}
+
+/**
+ * hinic_free_hwif - free the HW interface
+ * @hwif: the HW interface of a pci function device
+ **/
+void hinic_free_hwif(struct hinic_hwif *hwif)
+{
+	iounmap(hwif->cfg_regs_bar);
+}
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_if.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_if.h
new file mode 100644
index 000000000000..5b4760c0e9f5
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_if.h
@@ -0,0 +1,272 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#ifndef HINIC_HW_IF_H
+#define HINIC_HW_IF_H
+
+#include <linux/pci.h>
+#include <linux/io.h>
+#include <linux/types.h>
+#include <asm/byteorder.h>
+
+#define HINIC_DMA_ATTR_ST_SHIFT                                 0
+#define HINIC_DMA_ATTR_AT_SHIFT                                 8
+#define HINIC_DMA_ATTR_PH_SHIFT                                 10
+#define HINIC_DMA_ATTR_NO_SNOOPING_SHIFT                        12
+#define HINIC_DMA_ATTR_TPH_EN_SHIFT                             13
+
+#define HINIC_DMA_ATTR_ST_MASK                                  0xFF
+#define HINIC_DMA_ATTR_AT_MASK                                  0x3
+#define HINIC_DMA_ATTR_PH_MASK                                  0x3
+#define HINIC_DMA_ATTR_NO_SNOOPING_MASK                         0x1
+#define HINIC_DMA_ATTR_TPH_EN_MASK                              0x1
+
+#define HINIC_DMA_ATTR_SET(val, member)                         \
+	(((u32)(val) & HINIC_DMA_ATTR_##member##_MASK) <<       \
+	 HINIC_DMA_ATTR_##member##_SHIFT)
+
+#define HINIC_DMA_ATTR_CLEAR(val, member)                       \
+	((val) & (~(HINIC_DMA_ATTR_##member##_MASK              \
+	 << HINIC_DMA_ATTR_##member##_SHIFT)))
+
+#define HINIC_FA0_FUNC_IDX_SHIFT                                0
+#define HINIC_FA0_PF_IDX_SHIFT                                  10
+#define HINIC_FA0_PCI_INTF_IDX_SHIFT                            14
+/* reserved members - off 16 */
+#define HINIC_FA0_FUNC_TYPE_SHIFT                               24
+
+#define HINIC_FA0_FUNC_IDX_MASK                                 0x3FF
+#define HINIC_FA0_PF_IDX_MASK                                   0xF
+#define HINIC_FA0_PCI_INTF_IDX_MASK                             0x3
+#define HINIC_FA0_FUNC_TYPE_MASK                                0x1
+
+#define HINIC_FA0_GET(val, member)                              \
+	(((val) >> HINIC_FA0_##member##_SHIFT) & HINIC_FA0_##member##_MASK)
+
+#define HINIC_FA1_AEQS_PER_FUNC_SHIFT                           8
+/* reserved members - off 10 */
+#define HINIC_FA1_CEQS_PER_FUNC_SHIFT                           12
+/* reserved members - off 15 */
+#define HINIC_FA1_IRQS_PER_FUNC_SHIFT                           20
+#define HINIC_FA1_DMA_ATTR_PER_FUNC_SHIFT                       24
+/* reserved members - off 27 */
+#define HINIC_FA1_INIT_STATUS_SHIFT                             30
+
+#define HINIC_FA1_AEQS_PER_FUNC_MASK                            0x3
+#define HINIC_FA1_CEQS_PER_FUNC_MASK                            0x7
+#define HINIC_FA1_IRQS_PER_FUNC_MASK                            0xF
+#define HINIC_FA1_DMA_ATTR_PER_FUNC_MASK                        0x7
+#define HINIC_FA1_INIT_STATUS_MASK                              0x1
+
+#define HINIC_FA1_GET(val, member)                              \
+	(((val) >> HINIC_FA1_##member##_SHIFT) & HINIC_FA1_##member##_MASK)
+
+#define HINIC_FA4_OUTBOUND_STATE_SHIFT                          0
+#define HINIC_FA4_DB_STATE_SHIFT                                1
+
+#define HINIC_FA4_OUTBOUND_STATE_MASK                           0x1
+#define HINIC_FA4_DB_STATE_MASK                                 0x1
+
+#define HINIC_FA4_GET(val, member)                              \
+	(((val) >> HINIC_FA4_##member##_SHIFT) & HINIC_FA4_##member##_MASK)
+
+#define HINIC_FA4_SET(val, member)                              \
+	((((u32)val) & HINIC_FA4_##member##_MASK) << HINIC_FA4_##member##_SHIFT)
+
+#define HINIC_FA4_CLEAR(val, member)                            \
+	((val) & (~(HINIC_FA4_##member##_MASK << HINIC_FA4_##member##_SHIFT)))
+
+#define HINIC_FA5_PF_ACTION_SHIFT                               0
+#define HINIC_FA5_PF_ACTION_MASK                                0xFFFF
+
+#define HINIC_FA5_SET(val, member)                              \
+	(((u32)(val) & HINIC_FA5_##member##_MASK) << HINIC_FA5_##member##_SHIFT)
+
+#define HINIC_FA5_CLEAR(val, member)                            \
+	((val) & (~(HINIC_FA5_##member##_MASK << HINIC_FA5_##member##_SHIFT)))
+
+#define HINIC_PPF_ELECTION_IDX_SHIFT                            0
+#define HINIC_PPF_ELECTION_IDX_MASK                             0x1F
+
+#define HINIC_PPF_ELECTION_SET(val, member)                     \
+	(((u32)(val) & HINIC_PPF_ELECTION_##member##_MASK) <<   \
+	 HINIC_PPF_ELECTION_##member##_SHIFT)
+
+#define HINIC_PPF_ELECTION_GET(val, member)                     \
+	(((val) >> HINIC_PPF_ELECTION_##member##_SHIFT) &       \
+	 HINIC_PPF_ELECTION_##member##_MASK)
+
+#define HINIC_PPF_ELECTION_CLEAR(val, member)                   \
+	((val) & (~(HINIC_PPF_ELECTION_##member##_MASK          \
+	 << HINIC_PPF_ELECTION_##member##_SHIFT)))
+
+#define HINIC_MSIX_PENDING_LIMIT_SHIFT                          0
+#define HINIC_MSIX_COALESC_TIMER_SHIFT                          8
+#define HINIC_MSIX_LLI_TIMER_SHIFT                              16
+#define HINIC_MSIX_LLI_CREDIT_SHIFT                             24
+#define HINIC_MSIX_RESEND_TIMER_SHIFT                           29
+
+#define HINIC_MSIX_PENDING_LIMIT_MASK                           0xFF
+#define HINIC_MSIX_COALESC_TIMER_MASK                           0xFF
+#define HINIC_MSIX_LLI_TIMER_MASK                               0xFF
+#define HINIC_MSIX_LLI_CREDIT_MASK                              0x1F
+#define HINIC_MSIX_RESEND_TIMER_MASK                            0x7
+
+#define HINIC_MSIX_ATTR_SET(val, member)                        \
+	(((u32)(val) & HINIC_MSIX_##member##_MASK) <<           \
+	 HINIC_MSIX_##member##_SHIFT)
+
+#define HINIC_MSIX_ATTR_GET(val, member)                        \
+	(((val) >> HINIC_MSIX_##member##_SHIFT) &               \
+	 HINIC_MSIX_##member##_MASK)
+
+#define HINIC_MSIX_CNT_RESEND_TIMER_SHIFT                       29
+
+#define HINIC_MSIX_CNT_RESEND_TIMER_MASK                        0x1
+
+#define HINIC_MSIX_CNT_SET(val, member)                         \
+	(((u32)(val) & HINIC_MSIX_CNT_##member##_MASK) <<       \
+	 HINIC_MSIX_CNT_##member##_SHIFT)
+
+#define HINIC_HWIF_NUM_AEQS(hwif)       ((hwif)->attr.num_aeqs)
+#define HINIC_HWIF_NUM_CEQS(hwif)       ((hwif)->attr.num_ceqs)
+#define HINIC_HWIF_NUM_IRQS(hwif)       ((hwif)->attr.num_irqs)
+#define HINIC_HWIF_FUNC_IDX(hwif)       ((hwif)->attr.func_idx)
+#define HINIC_HWIF_PCI_INTF(hwif)       ((hwif)->attr.pci_intf_idx)
+#define HINIC_HWIF_PF_IDX(hwif)         ((hwif)->attr.pf_idx)
+
+#define HINIC_FUNC_TYPE(hwif)           ((hwif)->attr.func_type)
+#define HINIC_IS_PF(hwif)               (HINIC_FUNC_TYPE(hwif) == HINIC_PF)
+#define HINIC_IS_PPF(hwif)              (HINIC_FUNC_TYPE(hwif) == HINIC_PPF)
+
+#define HINIC_PCI_CFG_REGS_BAR          0
+#define HINIC_PCI_DB_BAR                4
+
+#define HINIC_PCIE_ST_DISABLE           0
+#define HINIC_PCIE_AT_DISABLE           0
+#define HINIC_PCIE_PH_DISABLE           0
+
+#define HINIC_EQ_MSIX_PENDING_LIMIT_DEFAULT     0       /* Disabled */
+#define HINIC_EQ_MSIX_COALESC_TIMER_DEFAULT     0xFF    /* max */
+#define HINIC_EQ_MSIX_LLI_TIMER_DEFAULT         0       /* Disabled */
+#define HINIC_EQ_MSIX_LLI_CREDIT_LIMIT_DEFAULT  0       /* Disabled */
+#define HINIC_EQ_MSIX_RESEND_TIMER_DEFAULT      7       /* max */
+
+enum hinic_pcie_nosnoop {
+	HINIC_PCIE_SNOOP        = 0,
+	HINIC_PCIE_NO_SNOOP     = 1,
+};
+
+enum hinic_pcie_tph {
+	HINIC_PCIE_TPH_DISABLE  = 0,
+	HINIC_PCIE_TPH_ENABLE   = 1,
+};
+
+enum hinic_func_type {
+	HINIC_PF        = 0,
+	HINIC_PPF       = 2,
+};
+
+enum hinic_mod_type {
+	HINIC_MOD_COMM  = 0,    /* HW communication module */
+	HINIC_MOD_L2NIC = 1,    /* L2NIC module */
+	HINIC_MOD_CFGM  = 7,    /* Configuration module */
+
+	HINIC_MOD_MAX   = 15
+};
+
+enum hinic_node_id {
+	HINIC_NODE_ID_MGMT = 21,
+};
+
+enum hinic_pf_action {
+	HINIC_PF_MGMT_INIT = 0x0,
+
+	HINIC_PF_MGMT_ACTIVE = 0x11,
+};
+
+enum hinic_outbound_state {
+	HINIC_OUTBOUND_ENABLE  = 0,
+	HINIC_OUTBOUND_DISABLE = 1,
+};
+
+enum hinic_db_state {
+	HINIC_DB_ENABLE  = 0,
+	HINIC_DB_DISABLE = 1,
+};
+
+struct hinic_func_attr {
+	u16                     func_idx;
+	u8                      pf_idx;
+	u8                      pci_intf_idx;
+
+	enum hinic_func_type    func_type;
+
+	u8                      ppf_idx;
+
+	u16                     num_irqs;
+	u8                      num_aeqs;
+	u8                      num_ceqs;
+
+	u8                      num_dma_attr;
+};
+
+struct hinic_hwif {
+	struct pci_dev          *pdev;
+	void __iomem            *cfg_regs_bar;
+
+	struct hinic_func_attr  attr;
+};
+
+static inline u32 hinic_hwif_read_reg(struct hinic_hwif *hwif, u32 reg)
+{
+	return be32_to_cpu(readl(hwif->cfg_regs_bar + reg));
+}
+
+static inline void hinic_hwif_write_reg(struct hinic_hwif *hwif, u32 reg,
+					u32 val)
+{
+	writel(cpu_to_be32(val), hwif->cfg_regs_bar + reg);
+}
+
+int hinic_msix_attr_set(struct hinic_hwif *hwif, u16 msix_index,
+			u8 pending_limit, u8 coalesc_timer,
+			u8 lli_timer_cfg, u8 lli_credit_limit,
+			u8 resend_timer);
+
+int hinic_msix_attr_get(struct hinic_hwif *hwif, u16 msix_index,
+			u8 *pending_limit, u8 *coalesc_timer_cfg,
+			u8 *lli_timer, u8 *lli_credit_limit,
+			u8 *resend_timer);
+
+int hinic_msix_attr_cnt_clear(struct hinic_hwif *hwif, u16 msix_index);
+
+void hinic_set_pf_action(struct hinic_hwif *hwif, enum hinic_pf_action action);
+
+enum hinic_outbound_state hinic_outbound_state_get(struct hinic_hwif *hwif);
+
+void hinic_outbound_state_set(struct hinic_hwif *hwif,
+			      enum hinic_outbound_state outbound_state);
+
+enum hinic_db_state hinic_db_state_get(struct hinic_hwif *hwif);
+
+void hinic_db_state_set(struct hinic_hwif *hwif,
+			enum hinic_db_state db_state);
+
+int hinic_init_hwif(struct hinic_hwif *hwif, struct pci_dev *pdev);
+
+void hinic_free_hwif(struct hinic_hwif *hwif);
+
+#endif
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_io.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_io.c
new file mode 100644
index 000000000000..8e5897669a3a
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_io.c
@@ -0,0 +1,533 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/pci.h>
+#include <linux/device.h>
+#include <linux/errno.h>
+#include <linux/slab.h>
+#include <linux/semaphore.h>
+#include <linux/dma-mapping.h>
+#include <linux/io.h>
+#include <linux/err.h>
+
+#include "hinic_hw_if.h"
+#include "hinic_hw_eqs.h"
+#include "hinic_hw_wqe.h"
+#include "hinic_hw_wq.h"
+#include "hinic_hw_cmdq.h"
+#include "hinic_hw_qp_ctxt.h"
+#include "hinic_hw_qp.h"
+#include "hinic_hw_io.h"
+
+#define CI_Q_ADDR_SIZE                  sizeof(u32)
+
+#define CI_ADDR(base_addr, q_id)        ((base_addr) + \
+					 (q_id) * CI_Q_ADDR_SIZE)
+
+#define CI_TABLE_SIZE(num_qps)          ((num_qps) * CI_Q_ADDR_SIZE)
+
+#define DB_IDX(db, db_base)             \
+	(((unsigned long)(db) - (unsigned long)(db_base)) / HINIC_DB_PAGE_SIZE)
+
+enum io_cmd {
+	IO_CMD_MODIFY_QUEUE_CTXT = 0,
+};
+
+static void init_db_area_idx(struct hinic_free_db_area *free_db_area)
+{
+	int i;
+
+	for (i = 0; i < HINIC_DB_MAX_AREAS; i++)
+		free_db_area->db_idx[i] = i;
+
+	free_db_area->alloc_pos = 0;
+	free_db_area->return_pos = HINIC_DB_MAX_AREAS;
+
+	free_db_area->num_free = HINIC_DB_MAX_AREAS;
+
+	sema_init(&free_db_area->idx_lock, 1);
+}
+
+static void __iomem *get_db_area(struct hinic_func_to_io *func_to_io)
+{
+	struct hinic_free_db_area *free_db_area = &func_to_io->free_db_area;
+	int pos, idx;
+
+	down(&free_db_area->idx_lock);
+
+	free_db_area->num_free--;
+
+	if (free_db_area->num_free < 0) {
+		free_db_area->num_free++;
+		up(&free_db_area->idx_lock);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	pos = free_db_area->alloc_pos++;
+	pos &= HINIC_DB_MAX_AREAS - 1;
+
+	idx = free_db_area->db_idx[pos];
+
+	free_db_area->db_idx[pos] = -1;
+
+	up(&free_db_area->idx_lock);
+
+	return func_to_io->db_base + idx * HINIC_DB_PAGE_SIZE;
+}
+
+static void return_db_area(struct hinic_func_to_io *func_to_io,
+			   void __iomem *db_base)
+{
+	struct hinic_free_db_area *free_db_area = &func_to_io->free_db_area;
+	int pos, idx = DB_IDX(db_base, func_to_io->db_base);
+
+	down(&free_db_area->idx_lock);
+
+	pos = free_db_area->return_pos++;
+	pos &= HINIC_DB_MAX_AREAS - 1;
+
+	free_db_area->db_idx[pos] = idx;
+
+	free_db_area->num_free++;
+
+	up(&free_db_area->idx_lock);
+}
+
+static int write_sq_ctxts(struct hinic_func_to_io *func_to_io, u16 base_qpn,
+			  u16 num_sqs)
+{
+	struct hinic_hwif *hwif = func_to_io->hwif;
+	struct hinic_sq_ctxt_block *sq_ctxt_block;
+	struct pci_dev *pdev = hwif->pdev;
+	struct hinic_cmdq_buf cmdq_buf;
+	struct hinic_sq_ctxt *sq_ctxt;
+	struct hinic_qp *qp;
+	u64 out_param;
+	int err, i;
+
+	err = hinic_alloc_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to allocate cmdq buf\n");
+		return err;
+	}
+
+	sq_ctxt_block = cmdq_buf.buf;
+	sq_ctxt = sq_ctxt_block->sq_ctxt;
+
+	hinic_qp_prepare_header(&sq_ctxt_block->hdr, HINIC_QP_CTXT_TYPE_SQ,
+				num_sqs, func_to_io->max_qps);
+	for (i = 0; i < num_sqs; i++) {
+		qp = &func_to_io->qps[i];
+
+		hinic_sq_prepare_ctxt(&sq_ctxt[i], &qp->sq,
+				      base_qpn + qp->q_id);
+	}
+
+	cmdq_buf.size = HINIC_SQ_CTXT_SIZE(num_sqs);
+
+	err = hinic_cmdq_direct_resp(&func_to_io->cmdqs, HINIC_MOD_L2NIC,
+				     IO_CMD_MODIFY_QUEUE_CTXT, &cmdq_buf,
+				     &out_param);
+	if ((err) || (out_param != 0)) {
+		dev_err(&pdev->dev, "Failed to set SQ ctxts\n");
+		err = -EFAULT;
+	}
+
+	hinic_free_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf);
+	return err;
+}
+
+static int write_rq_ctxts(struct hinic_func_to_io *func_to_io, u16 base_qpn,
+			  u16 num_rqs)
+{
+	struct hinic_hwif *hwif = func_to_io->hwif;
+	struct hinic_rq_ctxt_block *rq_ctxt_block;
+	struct pci_dev *pdev = hwif->pdev;
+	struct hinic_cmdq_buf cmdq_buf;
+	struct hinic_rq_ctxt *rq_ctxt;
+	struct hinic_qp *qp;
+	u64 out_param;
+	int err, i;
+
+	err = hinic_alloc_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to allocate cmdq buf\n");
+		return err;
+	}
+
+	rq_ctxt_block = cmdq_buf.buf;
+	rq_ctxt = rq_ctxt_block->rq_ctxt;
+
+	hinic_qp_prepare_header(&rq_ctxt_block->hdr, HINIC_QP_CTXT_TYPE_RQ,
+				num_rqs, func_to_io->max_qps);
+	for (i = 0; i < num_rqs; i++) {
+		qp = &func_to_io->qps[i];
+
+		hinic_rq_prepare_ctxt(&rq_ctxt[i], &qp->rq,
+				      base_qpn + qp->q_id);
+	}
+
+	cmdq_buf.size = HINIC_RQ_CTXT_SIZE(num_rqs);
+
+	err = hinic_cmdq_direct_resp(&func_to_io->cmdqs, HINIC_MOD_L2NIC,
+				     IO_CMD_MODIFY_QUEUE_CTXT, &cmdq_buf,
+				     &out_param);
+	if ((err) || (out_param != 0)) {
+		dev_err(&pdev->dev, "Failed to set RQ ctxts\n");
+		err = -EFAULT;
+	}
+
+	hinic_free_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf);
+	return err;
+}
+
+/**
+ * write_qp_ctxts - write the qp ctxt to HW
+ * @func_to_io: func to io channel that holds the IO components
+ * @base_qpn: first qp number
+ * @num_qps: number of qps to write
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int write_qp_ctxts(struct hinic_func_to_io *func_to_io, u16 base_qpn,
+			  u16 num_qps)
+{
+	return (write_sq_ctxts(func_to_io, base_qpn, num_qps) ||
+		write_rq_ctxts(func_to_io, base_qpn, num_qps));
+}
+
+/**
+ * init_qp - Initialize a Queue Pair
+ * @func_to_io: func to io channel that holds the IO components
+ * @qp: pointer to the qp to initialize
+ * @q_id: the id of the qp
+ * @sq_msix_entry: msix entry for sq
+ * @rq_msix_entry: msix entry for rq
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int init_qp(struct hinic_func_to_io *func_to_io,
+		   struct hinic_qp *qp, int q_id,
+		   struct msix_entry *sq_msix_entry,
+		   struct msix_entry *rq_msix_entry)
+{
+	struct hinic_hwif *hwif = func_to_io->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	void __iomem *db_base;
+	int err;
+
+	qp->q_id = q_id;
+
+	err = hinic_wq_allocate(&func_to_io->wqs, &func_to_io->sq_wq[q_id],
+				HINIC_SQ_WQEBB_SIZE, HINIC_SQ_PAGE_SIZE,
+				HINIC_SQ_DEPTH, HINIC_SQ_WQE_MAX_SIZE);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to allocate WQ for SQ\n");
+		return err;
+	}
+
+	err = hinic_wq_allocate(&func_to_io->wqs, &func_to_io->rq_wq[q_id],
+				HINIC_RQ_WQEBB_SIZE, HINIC_RQ_PAGE_SIZE,
+				HINIC_RQ_DEPTH, HINIC_RQ_WQE_SIZE);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to allocate WQ for RQ\n");
+		goto err_rq_alloc;
+	}
+
+	db_base = get_db_area(func_to_io);
+	if (IS_ERR(db_base)) {
+		dev_err(&pdev->dev, "Failed to get DB area for SQ\n");
+		err = PTR_ERR(db_base);
+		goto err_get_db;
+	}
+
+	func_to_io->sq_db[q_id] = db_base;
+
+	err = hinic_init_sq(&qp->sq, hwif, &func_to_io->sq_wq[q_id],
+			    sq_msix_entry,
+			    CI_ADDR(func_to_io->ci_addr_base, q_id),
+			    CI_ADDR(func_to_io->ci_dma_base, q_id), db_base);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to init SQ\n");
+		goto err_sq_init;
+	}
+
+	err = hinic_init_rq(&qp->rq, hwif, &func_to_io->rq_wq[q_id],
+			    rq_msix_entry);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to init RQ\n");
+		goto err_rq_init;
+	}
+
+	return 0;
+
+err_rq_init:
+	hinic_clean_sq(&qp->sq);
+
+err_sq_init:
+	return_db_area(func_to_io, db_base);
+
+err_get_db:
+	hinic_wq_free(&func_to_io->wqs, &func_to_io->rq_wq[q_id]);
+
+err_rq_alloc:
+	hinic_wq_free(&func_to_io->wqs, &func_to_io->sq_wq[q_id]);
+	return err;
+}
+
+/**
+ * destroy_qp - Clean the resources of a Queue Pair
+ * @func_to_io: func to io channel that holds the IO components
+ * @qp: pointer to the qp to clean
+ **/
+static void destroy_qp(struct hinic_func_to_io *func_to_io,
+		       struct hinic_qp *qp)
+{
+	int q_id = qp->q_id;
+
+	hinic_clean_rq(&qp->rq);
+	hinic_clean_sq(&qp->sq);
+
+	return_db_area(func_to_io, func_to_io->sq_db[q_id]);
+
+	hinic_wq_free(&func_to_io->wqs, &func_to_io->rq_wq[q_id]);
+	hinic_wq_free(&func_to_io->wqs, &func_to_io->sq_wq[q_id]);
+}
+
+/**
+ * hinic_io_create_qps - Create Queue Pairs
+ * @func_to_io: func to io channel that holds the IO components
+ * @base_qpn: base qp number
+ * @num_qps: number queue pairs to create
+ * @sq_msix_entry: msix entries for sq
+ * @rq_msix_entry: msix entries for rq
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_io_create_qps(struct hinic_func_to_io *func_to_io,
+			u16 base_qpn, int num_qps,
+			struct msix_entry *sq_msix_entries,
+			struct msix_entry *rq_msix_entries)
+{
+	struct hinic_hwif *hwif = func_to_io->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	size_t qps_size, wq_size, db_size;
+	void *ci_addr_base;
+	int i, j, err;
+
+	qps_size = num_qps * sizeof(*func_to_io->qps);
+	func_to_io->qps = devm_kzalloc(&pdev->dev, qps_size, GFP_KERNEL);
+	if (!func_to_io->qps)
+		return -ENOMEM;
+
+	wq_size = num_qps * sizeof(*func_to_io->sq_wq);
+	func_to_io->sq_wq = devm_kzalloc(&pdev->dev, wq_size, GFP_KERNEL);
+	if (!func_to_io->sq_wq) {
+		err = -ENOMEM;
+		goto err_sq_wq;
+	}
+
+	wq_size = num_qps * sizeof(*func_to_io->rq_wq);
+	func_to_io->rq_wq = devm_kzalloc(&pdev->dev, wq_size, GFP_KERNEL);
+	if (!func_to_io->rq_wq) {
+		err = -ENOMEM;
+		goto err_rq_wq;
+	}
+
+	db_size = num_qps * sizeof(*func_to_io->sq_db);
+	func_to_io->sq_db = devm_kzalloc(&pdev->dev, db_size, GFP_KERNEL);
+	if (!func_to_io->sq_db) {
+		err = -ENOMEM;
+		goto err_sq_db;
+	}
+
+	ci_addr_base = dma_zalloc_coherent(&pdev->dev, CI_TABLE_SIZE(num_qps),
+					   &func_to_io->ci_dma_base,
+					   GFP_KERNEL);
+	if (!ci_addr_base) {
+		dev_err(&pdev->dev, "Failed to allocate CI area\n");
+		err = -ENOMEM;
+		goto err_ci_base;
+	}
+
+	func_to_io->ci_addr_base = ci_addr_base;
+
+	for (i = 0; i < num_qps; i++) {
+		err = init_qp(func_to_io, &func_to_io->qps[i], i,
+			      &sq_msix_entries[i], &rq_msix_entries[i]);
+		if (err) {
+			dev_err(&pdev->dev, "Failed to create QP %d\n", i);
+			goto err_init_qp;
+		}
+	}
+
+	err = write_qp_ctxts(func_to_io, base_qpn, num_qps);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to init QP ctxts\n");
+		goto err_write_qp_ctxts;
+	}
+
+	return 0;
+
+err_write_qp_ctxts:
+err_init_qp:
+	for (j = 0; j < i; j++)
+		destroy_qp(func_to_io, &func_to_io->qps[j]);
+
+	dma_free_coherent(&pdev->dev, CI_TABLE_SIZE(num_qps),
+			  func_to_io->ci_addr_base, func_to_io->ci_dma_base);
+
+err_ci_base:
+	devm_kfree(&pdev->dev, func_to_io->sq_db);
+
+err_sq_db:
+	devm_kfree(&pdev->dev, func_to_io->rq_wq);
+
+err_rq_wq:
+	devm_kfree(&pdev->dev, func_to_io->sq_wq);
+
+err_sq_wq:
+	devm_kfree(&pdev->dev, func_to_io->qps);
+	return err;
+}
+
+/**
+ * hinic_io_destroy_qps - Destroy the IO Queue Pairs
+ * @func_to_io: func to io channel that holds the IO components
+ * @num_qps: number queue pairs to destroy
+ **/
+void hinic_io_destroy_qps(struct hinic_func_to_io *func_to_io, int num_qps)
+{
+	struct hinic_hwif *hwif = func_to_io->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	size_t ci_table_size;
+	int i;
+
+	ci_table_size = CI_TABLE_SIZE(num_qps);
+
+	for (i = 0; i < num_qps; i++)
+		destroy_qp(func_to_io, &func_to_io->qps[i]);
+
+	dma_free_coherent(&pdev->dev, ci_table_size, func_to_io->ci_addr_base,
+			  func_to_io->ci_dma_base);
+
+	devm_kfree(&pdev->dev, func_to_io->sq_db);
+
+	devm_kfree(&pdev->dev, func_to_io->rq_wq);
+	devm_kfree(&pdev->dev, func_to_io->sq_wq);
+
+	devm_kfree(&pdev->dev, func_to_io->qps);
+}
+
+/**
+ * hinic_io_init - Initialize the IO components
+ * @func_to_io: func to io channel that holds the IO components
+ * @hwif: HW interface for accessing IO
+ * @max_qps: maximum QPs in HW
+ * @num_ceqs: number completion event queues
+ * @ceq_msix_entries: msix entries for ceqs
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_io_init(struct hinic_func_to_io *func_to_io,
+		  struct hinic_hwif *hwif, u16 max_qps, int num_ceqs,
+		  struct msix_entry *ceq_msix_entries)
+{
+	struct pci_dev *pdev = hwif->pdev;
+	enum hinic_cmdq_type cmdq, type;
+	void __iomem *db_area;
+	int err;
+
+	func_to_io->hwif = hwif;
+	func_to_io->qps = NULL;
+	func_to_io->max_qps = max_qps;
+
+	err = hinic_ceqs_init(&func_to_io->ceqs, hwif, num_ceqs,
+			      HINIC_DEFAULT_CEQ_LEN, HINIC_EQ_PAGE_SIZE,
+			      ceq_msix_entries);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to init CEQs\n");
+		return err;
+	}
+
+	err = hinic_wqs_alloc(&func_to_io->wqs, 2 * max_qps, hwif);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to allocate WQS for IO\n");
+		goto err_wqs_alloc;
+	}
+
+	func_to_io->db_base = pci_ioremap_bar(pdev, HINIC_PCI_DB_BAR);
+	if (!func_to_io->db_base) {
+		dev_err(&pdev->dev, "Failed to remap IO DB area\n");
+		err = -ENOMEM;
+		goto err_db_ioremap;
+	}
+
+	init_db_area_idx(&func_to_io->free_db_area);
+
+	for (cmdq = HINIC_CMDQ_SYNC; cmdq < HINIC_MAX_CMDQ_TYPES; cmdq++) {
+		db_area = get_db_area(func_to_io);
+		if (IS_ERR(db_area)) {
+			dev_err(&pdev->dev, "Failed to get cmdq db area\n");
+			err = PTR_ERR(db_area);
+			goto err_db_area;
+		}
+
+		func_to_io->cmdq_db_area[cmdq] = db_area;
+	}
+
+	err = hinic_init_cmdqs(&func_to_io->cmdqs, hwif,
+			       func_to_io->cmdq_db_area);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to initialize cmdqs\n");
+		goto err_init_cmdqs;
+	}
+
+	return 0;
+
+err_init_cmdqs:
+err_db_area:
+	for (type = HINIC_CMDQ_SYNC; type < cmdq; type++)
+		return_db_area(func_to_io, func_to_io->cmdq_db_area[type]);
+
+	iounmap(func_to_io->db_base);
+
+err_db_ioremap:
+	hinic_wqs_free(&func_to_io->wqs);
+
+err_wqs_alloc:
+	hinic_ceqs_free(&func_to_io->ceqs);
+	return err;
+}
+
+/**
+ * hinic_io_free - Free the IO components
+ * @func_to_io: func to io channel that holds the IO components
+ **/
+void hinic_io_free(struct hinic_func_to_io *func_to_io)
+{
+	enum hinic_cmdq_type cmdq;
+
+	hinic_free_cmdqs(&func_to_io->cmdqs);
+
+	for (cmdq = HINIC_CMDQ_SYNC; cmdq < HINIC_MAX_CMDQ_TYPES; cmdq++)
+		return_db_area(func_to_io, func_to_io->cmdq_db_area[cmdq]);
+
+	iounmap(func_to_io->db_base);
+	hinic_wqs_free(&func_to_io->wqs);
+	hinic_ceqs_free(&func_to_io->ceqs);
+}
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_io.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_io.h
new file mode 100644
index 000000000000..adb64179d47d
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_io.h
@@ -0,0 +1,97 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#ifndef HINIC_HW_IO_H
+#define HINIC_HW_IO_H
+
+#include <linux/types.h>
+#include <linux/pci.h>
+#include <linux/semaphore.h>
+#include <linux/sizes.h>
+
+#include "hinic_hw_if.h"
+#include "hinic_hw_eqs.h"
+#include "hinic_hw_wq.h"
+#include "hinic_hw_cmdq.h"
+#include "hinic_hw_qp.h"
+
+#define HINIC_DB_PAGE_SIZE      SZ_4K
+#define HINIC_DB_SIZE           SZ_4M
+
+#define HINIC_DB_MAX_AREAS      (HINIC_DB_SIZE / HINIC_DB_PAGE_SIZE)
+
+enum hinic_db_type {
+	HINIC_DB_CMDQ_TYPE,
+	HINIC_DB_SQ_TYPE,
+};
+
+enum hinic_io_path {
+	HINIC_CTRL_PATH,
+	HINIC_DATA_PATH,
+};
+
+struct hinic_free_db_area {
+	int             db_idx[HINIC_DB_MAX_AREAS];
+
+	int             alloc_pos;
+	int             return_pos;
+
+	int             num_free;
+
+	/* Lock for getting db area */
+	struct semaphore        idx_lock;
+};
+
+struct hinic_func_to_io {
+	struct hinic_hwif       *hwif;
+
+	struct hinic_ceqs       ceqs;
+
+	struct hinic_wqs        wqs;
+
+	struct hinic_wq         *sq_wq;
+	struct hinic_wq         *rq_wq;
+
+	struct hinic_qp         *qps;
+	u16                     max_qps;
+
+	void __iomem            **sq_db;
+	void __iomem            *db_base;
+
+	void                    *ci_addr_base;
+	dma_addr_t              ci_dma_base;
+
+	struct hinic_free_db_area       free_db_area;
+
+	void __iomem                    *cmdq_db_area[HINIC_MAX_CMDQ_TYPES];
+
+	struct hinic_cmdqs              cmdqs;
+};
+
+int hinic_io_create_qps(struct hinic_func_to_io *func_to_io,
+			u16 base_qpn, int num_qps,
+			struct msix_entry *sq_msix_entries,
+			struct msix_entry *rq_msix_entries);
+
+void hinic_io_destroy_qps(struct hinic_func_to_io *func_to_io,
+			  int num_qps);
+
+int hinic_io_init(struct hinic_func_to_io *func_to_io,
+		  struct hinic_hwif *hwif, u16 max_qps, int num_ceqs,
+		  struct msix_entry *ceq_msix_entries);
+
+void hinic_io_free(struct hinic_func_to_io *func_to_io);
+
+#endif
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_mgmt.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_mgmt.c
new file mode 100644
index 000000000000..278dc13f3dae
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_mgmt.c
@@ -0,0 +1,597 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/errno.h>
+#include <linux/pci.h>
+#include <linux/device.h>
+#include <linux/semaphore.h>
+#include <linux/completion.h>
+#include <linux/slab.h>
+#include <asm/barrier.h>
+
+#include "hinic_hw_if.h"
+#include "hinic_hw_eqs.h"
+#include "hinic_hw_api_cmd.h"
+#include "hinic_hw_mgmt.h"
+#include "hinic_hw_dev.h"
+
+#define SYNC_MSG_ID_MASK                0x1FF
+
+#define SYNC_MSG_ID(pf_to_mgmt)         ((pf_to_mgmt)->sync_msg_id)
+
+#define SYNC_MSG_ID_INC(pf_to_mgmt)     (SYNC_MSG_ID(pf_to_mgmt) = \
+					((SYNC_MSG_ID(pf_to_mgmt) + 1) & \
+					 SYNC_MSG_ID_MASK))
+
+#define MSG_SZ_IS_VALID(in_size)        ((in_size) <= MAX_MSG_LEN)
+
+#define MGMT_MSG_LEN_MIN                20
+#define MGMT_MSG_LEN_STEP               16
+#define MGMT_MSG_RSVD_FOR_DEV           8
+
+#define SEGMENT_LEN                     48
+
+#define MAX_PF_MGMT_BUF_SIZE            2048
+
+/* Data should be SEG LEN size aligned */
+#define MAX_MSG_LEN                     2016
+
+#define MSG_NOT_RESP                    0xFFFF
+
+#define MGMT_MSG_TIMEOUT                1000
+
+#define mgmt_to_pfhwdev(pf_mgmt)        \
+		container_of(pf_mgmt, struct hinic_pfhwdev, pf_to_mgmt)
+
+enum msg_segment_type {
+	NOT_LAST_SEGMENT = 0,
+	LAST_SEGMENT     = 1,
+};
+
+enum mgmt_direction_type {
+	MGMT_DIRECT_SEND = 0,
+	MGMT_RESP        = 1,
+};
+
+enum msg_ack_type {
+	MSG_ACK         = 0,
+	MSG_NO_ACK      = 1,
+};
+
+/**
+ * hinic_register_mgmt_msg_cb - register msg handler for a msg from a module
+ * @pf_to_mgmt: PF to MGMT channel
+ * @mod: module in the chip that this handler will handle its messages
+ * @handle: private data for the callback
+ * @callback: the handler that will handle messages
+ **/
+void hinic_register_mgmt_msg_cb(struct hinic_pf_to_mgmt *pf_to_mgmt,
+				enum hinic_mod_type mod,
+				void *handle,
+				void (*callback)(void *handle,
+						 u8 cmd, void *buf_in,
+						 u16 in_size, void *buf_out,
+						 u16 *out_size))
+{
+	struct hinic_mgmt_cb *mgmt_cb = &pf_to_mgmt->mgmt_cb[mod];
+
+	mgmt_cb->cb = callback;
+	mgmt_cb->handle = handle;
+	mgmt_cb->state = HINIC_MGMT_CB_ENABLED;
+}
+
+/**
+ * hinic_unregister_mgmt_msg_cb - unregister msg handler for a msg from a module
+ * @pf_to_mgmt: PF to MGMT channel
+ * @mod: module in the chip that this handler handles its messages
+ **/
+void hinic_unregister_mgmt_msg_cb(struct hinic_pf_to_mgmt *pf_to_mgmt,
+				  enum hinic_mod_type mod)
+{
+	struct hinic_mgmt_cb *mgmt_cb = &pf_to_mgmt->mgmt_cb[mod];
+
+	mgmt_cb->state &= ~HINIC_MGMT_CB_ENABLED;
+
+	while (mgmt_cb->state & HINIC_MGMT_CB_RUNNING)
+		schedule();
+
+	mgmt_cb->cb = NULL;
+}
+
+/**
+ * prepare_header - prepare the header of the message
+ * @pf_to_mgmt: PF to MGMT channel
+ * @msg_len: the length of the message
+ * @mod: module in the chip that will get the message
+ * @ack_type: ask for response
+ * @direction: the direction of the message
+ * @cmd: command of the message
+ * @msg_id: message id
+ *
+ * Return the prepared header value
+ **/
+static u64 prepare_header(struct hinic_pf_to_mgmt *pf_to_mgmt,
+			  u16 msg_len, enum hinic_mod_type mod,
+			  enum msg_ack_type ack_type,
+			  enum mgmt_direction_type direction,
+			  u16 cmd, u16 msg_id)
+{
+	struct hinic_hwif *hwif = pf_to_mgmt->hwif;
+
+	return HINIC_MSG_HEADER_SET(msg_len, MSG_LEN)           |
+	       HINIC_MSG_HEADER_SET(mod, MODULE)                |
+	       HINIC_MSG_HEADER_SET(SEGMENT_LEN, SEG_LEN)       |
+	       HINIC_MSG_HEADER_SET(ack_type, NO_ACK)           |
+	       HINIC_MSG_HEADER_SET(0, ASYNC_MGMT_TO_PF)        |
+	       HINIC_MSG_HEADER_SET(0, SEQID)                   |
+	       HINIC_MSG_HEADER_SET(LAST_SEGMENT, LAST)         |
+	       HINIC_MSG_HEADER_SET(direction, DIRECTION)       |
+	       HINIC_MSG_HEADER_SET(cmd, CMD)                   |
+	       HINIC_MSG_HEADER_SET(HINIC_HWIF_PCI_INTF(hwif), PCI_INTF) |
+	       HINIC_MSG_HEADER_SET(HINIC_HWIF_PF_IDX(hwif), PF_IDX)     |
+	       HINIC_MSG_HEADER_SET(msg_id, MSG_ID);
+}
+
+/**
+ * prepare_mgmt_cmd - prepare the mgmt command
+ * @mgmt_cmd: pointer to the command to prepare
+ * @header: pointer of the header for the message
+ * @msg: the data of the message
+ * @msg_len: the length of the message
+ **/
+static void prepare_mgmt_cmd(u8 *mgmt_cmd, u64 *header, u8 *msg, u16 msg_len)
+{
+	memset(mgmt_cmd, 0, MGMT_MSG_RSVD_FOR_DEV);
+
+	mgmt_cmd += MGMT_MSG_RSVD_FOR_DEV;
+	memcpy(mgmt_cmd, header, sizeof(*header));
+
+	mgmt_cmd += sizeof(*header);
+	memcpy(mgmt_cmd, msg, msg_len);
+}
+
+/**
+ * mgmt_msg_len - calculate the total message length
+ * @msg_data_len: the length of the message data
+ *
+ * Return the total message length
+ **/
+static u16 mgmt_msg_len(u16 msg_data_len)
+{
+	/* RSVD + HEADER_SIZE + DATA_LEN */
+	u16 msg_len = MGMT_MSG_RSVD_FOR_DEV + sizeof(u64) + msg_data_len;
+
+	if (msg_len > MGMT_MSG_LEN_MIN)
+		msg_len = MGMT_MSG_LEN_MIN +
+			   ALIGN((msg_len - MGMT_MSG_LEN_MIN),
+				 MGMT_MSG_LEN_STEP);
+	else
+		msg_len = MGMT_MSG_LEN_MIN;
+
+	return msg_len;
+}
+
+/**
+ * send_msg_to_mgmt - send message to mgmt by API CMD
+ * @pf_to_mgmt: PF to MGMT channel
+ * @mod: module in the chip that will get the message
+ * @cmd: command of the message
+ * @data: the msg data
+ * @data_len: the msg data length
+ * @ack_type: ask for response
+ * @direction: the direction of the original message
+ * @resp_msg_id: msg id to response for
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int send_msg_to_mgmt(struct hinic_pf_to_mgmt *pf_to_mgmt,
+			    enum hinic_mod_type mod, u8 cmd,
+			    u8 *data, u16 data_len,
+			    enum msg_ack_type ack_type,
+			    enum mgmt_direction_type direction,
+			    u16 resp_msg_id)
+{
+	struct hinic_api_cmd_chain *chain;
+	u64 header;
+	u16 msg_id;
+
+	msg_id = SYNC_MSG_ID(pf_to_mgmt);
+
+	if (direction == MGMT_RESP) {
+		header = prepare_header(pf_to_mgmt, data_len, mod, ack_type,
+					direction, cmd, resp_msg_id);
+	} else {
+		SYNC_MSG_ID_INC(pf_to_mgmt);
+		header = prepare_header(pf_to_mgmt, data_len, mod, ack_type,
+					direction, cmd, msg_id);
+	}
+
+	prepare_mgmt_cmd(pf_to_mgmt->sync_msg_buf, &header, data, data_len);
+
+	chain = pf_to_mgmt->cmd_chain[HINIC_API_CMD_WRITE_TO_MGMT_CPU];
+	return hinic_api_cmd_write(chain, HINIC_NODE_ID_MGMT,
+				   pf_to_mgmt->sync_msg_buf,
+				   mgmt_msg_len(data_len));
+}
+
+/**
+ * msg_to_mgmt_sync - send sync message to mgmt
+ * @pf_to_mgmt: PF to MGMT channel
+ * @mod: module in the chip that will get the message
+ * @cmd: command of the message
+ * @buf_in: the msg data
+ * @in_size: the msg data length
+ * @buf_out: response
+ * @out_size: response length
+ * @direction: the direction of the original message
+ * @resp_msg_id: msg id to response for
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int msg_to_mgmt_sync(struct hinic_pf_to_mgmt *pf_to_mgmt,
+			    enum hinic_mod_type mod, u8 cmd,
+			    u8 *buf_in, u16 in_size,
+			    u8 *buf_out, u16 *out_size,
+			    enum mgmt_direction_type direction,
+			    u16 resp_msg_id)
+{
+	struct hinic_hwif *hwif = pf_to_mgmt->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	struct hinic_recv_msg *recv_msg;
+	struct completion *recv_done;
+	u16 msg_id;
+	int err;
+
+	/* Lock the sync_msg_buf */
+	down(&pf_to_mgmt->sync_msg_lock);
+
+	recv_msg = &pf_to_mgmt->recv_resp_msg_from_mgmt;
+	recv_done = &recv_msg->recv_done;
+
+	if (resp_msg_id == MSG_NOT_RESP)
+		msg_id = SYNC_MSG_ID(pf_to_mgmt);
+	else
+		msg_id = resp_msg_id;
+
+	init_completion(recv_done);
+
+	err = send_msg_to_mgmt(pf_to_mgmt, mod, cmd, buf_in, in_size,
+			       MSG_ACK, direction, resp_msg_id);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to send sync msg to mgmt\n");
+		goto unlock_sync_msg;
+	}
+
+	if (!wait_for_completion_timeout(recv_done, MGMT_MSG_TIMEOUT)) {
+		dev_err(&pdev->dev, "MGMT timeout, MSG id = %d\n", msg_id);
+		err = -ETIMEDOUT;
+		goto unlock_sync_msg;
+	}
+
+	smp_rmb();      /* verify reading after completion */
+
+	if (recv_msg->msg_id != msg_id) {
+		dev_err(&pdev->dev, "incorrect MSG for id = %d\n", msg_id);
+		err = -EFAULT;
+		goto unlock_sync_msg;
+	}
+
+	if ((buf_out) && (recv_msg->msg_len <= MAX_PF_MGMT_BUF_SIZE)) {
+		memcpy(buf_out, recv_msg->msg, recv_msg->msg_len);
+		*out_size = recv_msg->msg_len;
+	}
+
+unlock_sync_msg:
+	up(&pf_to_mgmt->sync_msg_lock);
+	return err;
+}
+
+/**
+ * msg_to_mgmt_async - send message to mgmt without response
+ * @pf_to_mgmt: PF to MGMT channel
+ * @mod: module in the chip that will get the message
+ * @cmd: command of the message
+ * @buf_in: the msg data
+ * @in_size: the msg data length
+ * @direction: the direction of the original message
+ * @resp_msg_id: msg id to response for
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int msg_to_mgmt_async(struct hinic_pf_to_mgmt *pf_to_mgmt,
+			     enum hinic_mod_type mod, u8 cmd,
+			     u8 *buf_in, u16 in_size,
+			     enum mgmt_direction_type direction,
+			     u16 resp_msg_id)
+{
+	int err;
+
+	/* Lock the sync_msg_buf */
+	down(&pf_to_mgmt->sync_msg_lock);
+
+	err = send_msg_to_mgmt(pf_to_mgmt, mod, cmd, buf_in, in_size,
+			       MSG_NO_ACK, direction, resp_msg_id);
+
+	up(&pf_to_mgmt->sync_msg_lock);
+	return err;
+}
+
+/**
+ * hinic_msg_to_mgmt - send message to mgmt
+ * @pf_to_mgmt: PF to MGMT channel
+ * @mod: module in the chip that will get the message
+ * @cmd: command of the message
+ * @buf_in: the msg data
+ * @in_size: the msg data length
+ * @buf_out: response
+ * @out_size: returned response length
+ * @sync: sync msg or async msg
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_msg_to_mgmt(struct hinic_pf_to_mgmt *pf_to_mgmt,
+		      enum hinic_mod_type mod, u8 cmd,
+		      void *buf_in, u16 in_size, void *buf_out, u16 *out_size,
+		      enum hinic_mgmt_msg_type sync)
+{
+	struct hinic_hwif *hwif = pf_to_mgmt->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+
+	if (sync != HINIC_MGMT_MSG_SYNC) {
+		dev_err(&pdev->dev, "Invalid MGMT msg type\n");
+		return -EINVAL;
+	}
+
+	if (!MSG_SZ_IS_VALID(in_size)) {
+		dev_err(&pdev->dev, "Invalid MGMT msg buffer size\n");
+		return -EINVAL;
+	}
+
+	return msg_to_mgmt_sync(pf_to_mgmt, mod, cmd, buf_in, in_size,
+				buf_out, out_size, MGMT_DIRECT_SEND,
+				MSG_NOT_RESP);
+}
+
+/**
+ * mgmt_recv_msg_handler - handler for message from mgmt cpu
+ * @pf_to_mgmt: PF to MGMT channel
+ * @recv_msg: received message details
+ **/
+static void mgmt_recv_msg_handler(struct hinic_pf_to_mgmt *pf_to_mgmt,
+				  struct hinic_recv_msg *recv_msg)
+{
+	struct hinic_hwif *hwif = pf_to_mgmt->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	u8 *buf_out = recv_msg->buf_out;
+	struct hinic_mgmt_cb *mgmt_cb;
+	unsigned long cb_state;
+	u16 out_size = 0;
+
+	if (recv_msg->mod >= HINIC_MOD_MAX) {
+		dev_err(&pdev->dev, "Unknown MGMT MSG module = %d\n",
+			recv_msg->mod);
+		return;
+	}
+
+	mgmt_cb = &pf_to_mgmt->mgmt_cb[recv_msg->mod];
+
+	cb_state = cmpxchg(&mgmt_cb->state,
+			   HINIC_MGMT_CB_ENABLED,
+			   HINIC_MGMT_CB_ENABLED | HINIC_MGMT_CB_RUNNING);
+
+	if ((cb_state == HINIC_MGMT_CB_ENABLED) && (mgmt_cb->cb))
+		mgmt_cb->cb(mgmt_cb->handle, recv_msg->cmd,
+			    recv_msg->msg, recv_msg->msg_len,
+			    buf_out, &out_size);
+	else
+		dev_err(&pdev->dev, "No MGMT msg handler, mod = %d\n",
+			recv_msg->mod);
+
+	mgmt_cb->state &= ~HINIC_MGMT_CB_RUNNING;
+
+	if (!recv_msg->async_mgmt_to_pf)
+		/* MGMT sent sync msg, send the response */
+		msg_to_mgmt_async(pf_to_mgmt, recv_msg->mod, recv_msg->cmd,
+				  buf_out, out_size, MGMT_RESP,
+				  recv_msg->msg_id);
+}
+
+/**
+ * mgmt_resp_msg_handler - handler for a response message from mgmt cpu
+ * @pf_to_mgmt: PF to MGMT channel
+ * @recv_msg: received message details
+ **/
+static void mgmt_resp_msg_handler(struct hinic_pf_to_mgmt *pf_to_mgmt,
+				  struct hinic_recv_msg *recv_msg)
+{
+	wmb();  /* verify writing all, before reading */
+
+	complete(&recv_msg->recv_done);
+}
+
+/**
+ * recv_mgmt_msg_handler - handler for a message from mgmt cpu
+ * @pf_to_mgmt: PF to MGMT channel
+ * @header: the header of the message
+ * @recv_msg: received message details
+ **/
+static void recv_mgmt_msg_handler(struct hinic_pf_to_mgmt *pf_to_mgmt,
+				  u64 *header, struct hinic_recv_msg *recv_msg)
+{
+	struct hinic_hwif *hwif = pf_to_mgmt->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	int seq_id, seg_len;
+	u8 *msg_body;
+
+	seq_id = HINIC_MSG_HEADER_GET(*header, SEQID);
+	seg_len = HINIC_MSG_HEADER_GET(*header, SEG_LEN);
+
+	if (seq_id >= (MAX_MSG_LEN / SEGMENT_LEN)) {
+		dev_err(&pdev->dev, "recv big mgmt msg\n");
+		return;
+	}
+
+	msg_body = (u8 *)header + sizeof(*header);
+	memcpy(recv_msg->msg + seq_id * SEGMENT_LEN, msg_body, seg_len);
+
+	if (!HINIC_MSG_HEADER_GET(*header, LAST))
+		return;
+
+	recv_msg->cmd = HINIC_MSG_HEADER_GET(*header, CMD);
+	recv_msg->mod = HINIC_MSG_HEADER_GET(*header, MODULE);
+	recv_msg->async_mgmt_to_pf = HINIC_MSG_HEADER_GET(*header,
+							  ASYNC_MGMT_TO_PF);
+	recv_msg->msg_len = HINIC_MSG_HEADER_GET(*header, MSG_LEN);
+	recv_msg->msg_id = HINIC_MSG_HEADER_GET(*header, MSG_ID);
+
+	if (HINIC_MSG_HEADER_GET(*header, DIRECTION) == MGMT_RESP)
+		mgmt_resp_msg_handler(pf_to_mgmt, recv_msg);
+	else
+		mgmt_recv_msg_handler(pf_to_mgmt, recv_msg);
+}
+
+/**
+ * mgmt_msg_aeqe_handler - handler for a mgmt message event
+ * @handle: PF to MGMT channel
+ * @data: the header of the message
+ * @size: unused
+ **/
+static void mgmt_msg_aeqe_handler(void *handle, void *data, u8 size)
+{
+	struct hinic_pf_to_mgmt *pf_to_mgmt = handle;
+	struct hinic_recv_msg *recv_msg;
+	u64 *header = (u64 *)data;
+
+	recv_msg = HINIC_MSG_HEADER_GET(*header, DIRECTION) ==
+		   MGMT_DIRECT_SEND ?
+		   &pf_to_mgmt->recv_msg_from_mgmt :
+		   &pf_to_mgmt->recv_resp_msg_from_mgmt;
+
+	recv_mgmt_msg_handler(pf_to_mgmt, header, recv_msg);
+}
+
+/**
+ * alloc_recv_msg - allocate receive message memory
+ * @pf_to_mgmt: PF to MGMT channel
+ * @recv_msg: pointer that will hold the allocated data
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int alloc_recv_msg(struct hinic_pf_to_mgmt *pf_to_mgmt,
+			  struct hinic_recv_msg *recv_msg)
+{
+	struct hinic_hwif *hwif = pf_to_mgmt->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+
+	recv_msg->msg = devm_kzalloc(&pdev->dev, MAX_PF_MGMT_BUF_SIZE,
+				     GFP_KERNEL);
+	if (!recv_msg->msg)
+		return -ENOMEM;
+
+	recv_msg->buf_out = devm_kzalloc(&pdev->dev, MAX_PF_MGMT_BUF_SIZE,
+					 GFP_KERNEL);
+	if (!recv_msg->buf_out)
+		return -ENOMEM;
+
+	return 0;
+}
+
+/**
+ * alloc_msg_buf - allocate all the message buffers of PF to MGMT channel
+ * @pf_to_mgmt: PF to MGMT channel
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int alloc_msg_buf(struct hinic_pf_to_mgmt *pf_to_mgmt)
+{
+	struct hinic_hwif *hwif = pf_to_mgmt->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	int err;
+
+	err = alloc_recv_msg(pf_to_mgmt,
+			     &pf_to_mgmt->recv_msg_from_mgmt);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to allocate recv msg\n");
+		return err;
+	}
+
+	err = alloc_recv_msg(pf_to_mgmt,
+			     &pf_to_mgmt->recv_resp_msg_from_mgmt);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to allocate resp recv msg\n");
+		return err;
+	}
+
+	pf_to_mgmt->sync_msg_buf = devm_kzalloc(&pdev->dev,
+						MAX_PF_MGMT_BUF_SIZE,
+						GFP_KERNEL);
+	if (!pf_to_mgmt->sync_msg_buf)
+		return -ENOMEM;
+
+	return 0;
+}
+
+/**
+ * hinic_pf_to_mgmt_init - initialize PF to MGMT channel
+ * @pf_to_mgmt: PF to MGMT channel
+ * @hwif: HW interface the PF to MGMT will use for accessing HW
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_pf_to_mgmt_init(struct hinic_pf_to_mgmt *pf_to_mgmt,
+			  struct hinic_hwif *hwif)
+{
+	struct hinic_pfhwdev *pfhwdev = mgmt_to_pfhwdev(pf_to_mgmt);
+	struct hinic_hwdev *hwdev = &pfhwdev->hwdev;
+	struct pci_dev *pdev = hwif->pdev;
+	int err;
+
+	pf_to_mgmt->hwif = hwif;
+
+	sema_init(&pf_to_mgmt->sync_msg_lock, 1);
+	pf_to_mgmt->sync_msg_id = 0;
+
+	err = alloc_msg_buf(pf_to_mgmt);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to allocate msg buffers\n");
+		return err;
+	}
+
+	err = hinic_api_cmd_init(pf_to_mgmt->cmd_chain, hwif);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to initialize cmd chains\n");
+		return err;
+	}
+
+	hinic_aeq_register_hw_cb(&hwdev->aeqs, HINIC_MSG_FROM_MGMT_CPU,
+				 pf_to_mgmt,
+				 mgmt_msg_aeqe_handler);
+	return 0;
+}
+
+/**
+ * hinic_pf_to_mgmt_free - free PF to MGMT channel
+ * @pf_to_mgmt: PF to MGMT channel
+ **/
+void hinic_pf_to_mgmt_free(struct hinic_pf_to_mgmt *pf_to_mgmt)
+{
+	struct hinic_pfhwdev *pfhwdev = mgmt_to_pfhwdev(pf_to_mgmt);
+	struct hinic_hwdev *hwdev = &pfhwdev->hwdev;
+
+	hinic_aeq_unregister_hw_cb(&hwdev->aeqs, HINIC_MSG_FROM_MGMT_CPU);
+	hinic_api_cmd_free(pf_to_mgmt->cmd_chain);
+}
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_mgmt.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_mgmt.h
new file mode 100644
index 000000000000..320711e8dee6
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_mgmt.h
@@ -0,0 +1,153 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#ifndef HINIC_HW_MGMT_H
+#define HINIC_HW_MGMT_H
+
+#include <linux/types.h>
+#include <linux/semaphore.h>
+#include <linux/completion.h>
+#include <linux/bitops.h>
+
+#include "hinic_hw_if.h"
+#include "hinic_hw_api_cmd.h"
+
+#define HINIC_MSG_HEADER_MSG_LEN_SHIFT                          0
+#define HINIC_MSG_HEADER_MODULE_SHIFT                           11
+#define HINIC_MSG_HEADER_SEG_LEN_SHIFT                          16
+#define HINIC_MSG_HEADER_NO_ACK_SHIFT                           22
+#define HINIC_MSG_HEADER_ASYNC_MGMT_TO_PF_SHIFT                 23
+#define HINIC_MSG_HEADER_SEQID_SHIFT                            24
+#define HINIC_MSG_HEADER_LAST_SHIFT                             30
+#define HINIC_MSG_HEADER_DIRECTION_SHIFT                        31
+#define HINIC_MSG_HEADER_CMD_SHIFT                              32
+#define HINIC_MSG_HEADER_ZEROS_SHIFT                            40
+#define HINIC_MSG_HEADER_PCI_INTF_SHIFT                         48
+#define HINIC_MSG_HEADER_PF_IDX_SHIFT                           50
+#define HINIC_MSG_HEADER_MSG_ID_SHIFT                           54
+
+#define HINIC_MSG_HEADER_MSG_LEN_MASK                           0x7FF
+#define HINIC_MSG_HEADER_MODULE_MASK                            0x1F
+#define HINIC_MSG_HEADER_SEG_LEN_MASK                           0x3F
+#define HINIC_MSG_HEADER_NO_ACK_MASK                            0x1
+#define HINIC_MSG_HEADER_ASYNC_MGMT_TO_PF_MASK                  0x1
+#define HINIC_MSG_HEADER_SEQID_MASK                             0x3F
+#define HINIC_MSG_HEADER_LAST_MASK                              0x1
+#define HINIC_MSG_HEADER_DIRECTION_MASK                         0x1
+#define HINIC_MSG_HEADER_CMD_MASK                               0xFF
+#define HINIC_MSG_HEADER_ZEROS_MASK                             0xFF
+#define HINIC_MSG_HEADER_PCI_INTF_MASK                          0x3
+#define HINIC_MSG_HEADER_PF_IDX_MASK                            0xF
+#define HINIC_MSG_HEADER_MSG_ID_MASK                            0x3FF
+
+#define HINIC_MSG_HEADER_SET(val, member)                       \
+		((u64)((val) & HINIC_MSG_HEADER_##member##_MASK) << \
+		 HINIC_MSG_HEADER_##member##_SHIFT)
+
+#define HINIC_MSG_HEADER_GET(val, member)                       \
+		(((val) >> HINIC_MSG_HEADER_##member##_SHIFT) & \
+		 HINIC_MSG_HEADER_##member##_MASK)
+
+enum hinic_mgmt_msg_type {
+	HINIC_MGMT_MSG_SYNC = 1,
+};
+
+enum hinic_cfg_cmd {
+	HINIC_CFG_NIC_CAP = 0,
+};
+
+enum hinic_comm_cmd {
+	HINIC_COMM_CMD_IO_STATUS_GET    = 0x3,
+
+	HINIC_COMM_CMD_CMDQ_CTXT_SET    = 0x10,
+	HINIC_COMM_CMD_CMDQ_CTXT_GET    = 0x11,
+
+	HINIC_COMM_CMD_HWCTXT_SET       = 0x12,
+	HINIC_COMM_CMD_HWCTXT_GET       = 0x13,
+
+	HINIC_COMM_CMD_SQ_HI_CI_SET     = 0x14,
+
+	HINIC_COMM_CMD_RES_STATE_SET    = 0x24,
+
+	HINIC_COMM_CMD_IO_RES_CLEAR     = 0x29,
+
+	HINIC_COMM_CMD_MAX              = 0x32,
+};
+
+enum hinic_mgmt_cb_state {
+	HINIC_MGMT_CB_ENABLED = BIT(0),
+	HINIC_MGMT_CB_RUNNING = BIT(1),
+};
+
+struct hinic_recv_msg {
+	u8                      *msg;
+	u8                      *buf_out;
+
+	struct completion       recv_done;
+
+	u16                     cmd;
+	enum hinic_mod_type     mod;
+	int                     async_mgmt_to_pf;
+
+	u16                     msg_len;
+	u16                     msg_id;
+};
+
+struct hinic_mgmt_cb {
+	void    (*cb)(void *handle, u8 cmd,
+		      void *buf_in, u16 in_size,
+		      void *buf_out, u16 *out_size);
+
+	void            *handle;
+	unsigned long   state;
+};
+
+struct hinic_pf_to_mgmt {
+	struct hinic_hwif               *hwif;
+
+	struct semaphore                sync_msg_lock;
+	u16                             sync_msg_id;
+	u8                              *sync_msg_buf;
+
+	struct hinic_recv_msg           recv_resp_msg_from_mgmt;
+	struct hinic_recv_msg           recv_msg_from_mgmt;
+
+	struct hinic_api_cmd_chain      *cmd_chain[HINIC_API_CMD_MAX];
+
+	struct hinic_mgmt_cb            mgmt_cb[HINIC_MOD_MAX];
+};
+
+void hinic_register_mgmt_msg_cb(struct hinic_pf_to_mgmt *pf_to_mgmt,
+				enum hinic_mod_type mod,
+				void *handle,
+				void (*callback)(void *handle,
+						 u8 cmd, void *buf_in,
+						 u16 in_size, void *buf_out,
+						 u16 *out_size));
+
+void hinic_unregister_mgmt_msg_cb(struct hinic_pf_to_mgmt *pf_to_mgmt,
+				  enum hinic_mod_type mod);
+
+int hinic_msg_to_mgmt(struct hinic_pf_to_mgmt *pf_to_mgmt,
+		      enum hinic_mod_type mod, u8 cmd,
+		      void *buf_in, u16 in_size, void *buf_out, u16 *out_size,
+		      enum hinic_mgmt_msg_type sync);
+
+int hinic_pf_to_mgmt_init(struct hinic_pf_to_mgmt *pf_to_mgmt,
+			  struct hinic_hwif *hwif);
+
+void hinic_pf_to_mgmt_free(struct hinic_pf_to_mgmt *pf_to_mgmt);
+
+#endif
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.c
new file mode 100644
index 000000000000..b9db6d649743
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.c
@@ -0,0 +1,887 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/pci.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/vmalloc.h>
+#include <linux/errno.h>
+#include <linux/sizes.h>
+#include <linux/atomic.h>
+#include <linux/skbuff.h>
+#include <linux/io.h>
+#include <asm/barrier.h>
+#include <asm/byteorder.h>
+
+#include "hinic_common.h"
+#include "hinic_hw_if.h"
+#include "hinic_hw_wqe.h"
+#include "hinic_hw_wq.h"
+#include "hinic_hw_qp_ctxt.h"
+#include "hinic_hw_qp.h"
+#include "hinic_hw_io.h"
+
+#define SQ_DB_OFF               SZ_2K
+
+/* The number of cache line to prefetch Until threshold state */
+#define WQ_PREFETCH_MAX         2
+/* The number of cache line to prefetch After threshold state */
+#define WQ_PREFETCH_MIN         1
+/* Threshold state */
+#define WQ_PREFETCH_THRESHOLD   256
+
+/* sizes of the SQ/RQ ctxt */
+#define Q_CTXT_SIZE             48
+#define CTXT_RSVD               240
+
+#define SQ_CTXT_OFFSET(max_sqs, max_rqs, q_id)  \
+		(((max_rqs) + (max_sqs)) * CTXT_RSVD + (q_id) * Q_CTXT_SIZE)
+
+#define RQ_CTXT_OFFSET(max_sqs, max_rqs, q_id)  \
+		(((max_rqs) + (max_sqs)) * CTXT_RSVD + \
+		 (max_sqs + (q_id)) * Q_CTXT_SIZE)
+
+#define SIZE_16BYTES(size)              (ALIGN(size, 16) >> 4)
+#define SIZE_8BYTES(size)               (ALIGN(size, 8) >> 3)
+#define SECT_SIZE_FROM_8BYTES(size)     ((size) << 3)
+
+#define SQ_DB_PI_HI_SHIFT       8
+#define SQ_DB_PI_HI(prod_idx)   ((prod_idx) >> SQ_DB_PI_HI_SHIFT)
+
+#define SQ_DB_PI_LOW_MASK       0xFF
+#define SQ_DB_PI_LOW(prod_idx)  ((prod_idx) & SQ_DB_PI_LOW_MASK)
+
+#define SQ_DB_ADDR(sq, pi)      ((u64 *)((sq)->db_base) + SQ_DB_PI_LOW(pi))
+
+#define SQ_MASKED_IDX(sq, idx)  ((idx) & (sq)->wq->mask)
+#define RQ_MASKED_IDX(rq, idx)  ((idx) & (rq)->wq->mask)
+
+#define TX_MAX_MSS_DEFAULT      0x3E00
+
+enum sq_wqe_type {
+	SQ_NORMAL_WQE = 0,
+};
+
+enum rq_completion_fmt {
+	RQ_COMPLETE_SGE = 1
+};
+
+void hinic_qp_prepare_header(struct hinic_qp_ctxt_header *qp_ctxt_hdr,
+			     enum hinic_qp_ctxt_type ctxt_type,
+			     u16 num_queues, u16 max_queues)
+{
+	u16 max_sqs = max_queues;
+	u16 max_rqs = max_queues;
+
+	qp_ctxt_hdr->num_queues = num_queues;
+	qp_ctxt_hdr->queue_type = ctxt_type;
+
+	if (ctxt_type == HINIC_QP_CTXT_TYPE_SQ)
+		qp_ctxt_hdr->addr_offset = SQ_CTXT_OFFSET(max_sqs, max_rqs, 0);
+	else
+		qp_ctxt_hdr->addr_offset = RQ_CTXT_OFFSET(max_sqs, max_rqs, 0);
+
+	qp_ctxt_hdr->addr_offset = SIZE_16BYTES(qp_ctxt_hdr->addr_offset);
+
+	hinic_cpu_to_be32(qp_ctxt_hdr, sizeof(*qp_ctxt_hdr));
+}
+
+void hinic_sq_prepare_ctxt(struct hinic_sq_ctxt *sq_ctxt,
+			   struct hinic_sq *sq, u16 global_qid)
+{
+	u32 wq_page_pfn_hi, wq_page_pfn_lo, wq_block_pfn_hi, wq_block_pfn_lo;
+	u64 wq_page_addr, wq_page_pfn, wq_block_pfn;
+	u16 pi_start, ci_start;
+	struct hinic_wq *wq;
+
+	wq = sq->wq;
+	ci_start = atomic_read(&wq->cons_idx);
+	pi_start = atomic_read(&wq->prod_idx);
+
+	/* Read the first page paddr from the WQ page paddr ptrs */
+	wq_page_addr = be64_to_cpu(*wq->block_vaddr);
+
+	wq_page_pfn = HINIC_WQ_PAGE_PFN(wq_page_addr);
+	wq_page_pfn_hi = upper_32_bits(wq_page_pfn);
+	wq_page_pfn_lo = lower_32_bits(wq_page_pfn);
+
+	wq_block_pfn = HINIC_WQ_BLOCK_PFN(wq->block_paddr);
+	wq_block_pfn_hi = upper_32_bits(wq_block_pfn);
+	wq_block_pfn_lo = lower_32_bits(wq_block_pfn);
+
+	sq_ctxt->ceq_attr = HINIC_SQ_CTXT_CEQ_ATTR_SET(global_qid,
+						       GLOBAL_SQ_ID) |
+			    HINIC_SQ_CTXT_CEQ_ATTR_SET(0, EN);
+
+	sq_ctxt->ci_wrapped = HINIC_SQ_CTXT_CI_SET(ci_start, IDX) |
+			      HINIC_SQ_CTXT_CI_SET(1, WRAPPED);
+
+	sq_ctxt->wq_hi_pfn_pi =
+			HINIC_SQ_CTXT_WQ_PAGE_SET(wq_page_pfn_hi, HI_PFN) |
+			HINIC_SQ_CTXT_WQ_PAGE_SET(pi_start, PI);
+
+	sq_ctxt->wq_lo_pfn = wq_page_pfn_lo;
+
+	sq_ctxt->pref_cache =
+		HINIC_SQ_CTXT_PREF_SET(WQ_PREFETCH_MIN, CACHE_MIN) |
+		HINIC_SQ_CTXT_PREF_SET(WQ_PREFETCH_MAX, CACHE_MAX) |
+		HINIC_SQ_CTXT_PREF_SET(WQ_PREFETCH_THRESHOLD, CACHE_THRESHOLD);
+
+	sq_ctxt->pref_wrapped = 1;
+
+	sq_ctxt->pref_wq_hi_pfn_ci =
+		HINIC_SQ_CTXT_PREF_SET(ci_start, CI) |
+		HINIC_SQ_CTXT_PREF_SET(wq_page_pfn_hi, WQ_HI_PFN);
+
+	sq_ctxt->pref_wq_lo_pfn = wq_page_pfn_lo;
+
+	sq_ctxt->wq_block_hi_pfn =
+		HINIC_SQ_CTXT_WQ_BLOCK_SET(wq_block_pfn_hi, HI_PFN);
+
+	sq_ctxt->wq_block_lo_pfn = wq_block_pfn_lo;
+
+	hinic_cpu_to_be32(sq_ctxt, sizeof(*sq_ctxt));
+}
+
+void hinic_rq_prepare_ctxt(struct hinic_rq_ctxt *rq_ctxt,
+			   struct hinic_rq *rq, u16 global_qid)
+{
+	u32 wq_page_pfn_hi, wq_page_pfn_lo, wq_block_pfn_hi, wq_block_pfn_lo;
+	u64 wq_page_addr, wq_page_pfn, wq_block_pfn;
+	u16 pi_start, ci_start;
+	struct hinic_wq *wq;
+
+	wq = rq->wq;
+	ci_start = atomic_read(&wq->cons_idx);
+	pi_start = atomic_read(&wq->prod_idx);
+
+	/* Read the first page paddr from the WQ page paddr ptrs */
+	wq_page_addr = be64_to_cpu(*wq->block_vaddr);
+
+	wq_page_pfn = HINIC_WQ_PAGE_PFN(wq_page_addr);
+	wq_page_pfn_hi = upper_32_bits(wq_page_pfn);
+	wq_page_pfn_lo = lower_32_bits(wq_page_pfn);
+
+	wq_block_pfn = HINIC_WQ_BLOCK_PFN(wq->block_paddr);
+	wq_block_pfn_hi = upper_32_bits(wq_block_pfn);
+	wq_block_pfn_lo = lower_32_bits(wq_block_pfn);
+
+	rq_ctxt->ceq_attr = HINIC_RQ_CTXT_CEQ_ATTR_SET(0, EN) |
+			    HINIC_RQ_CTXT_CEQ_ATTR_SET(1, WRAPPED);
+
+	rq_ctxt->pi_intr_attr = HINIC_RQ_CTXT_PI_SET(pi_start, IDX) |
+				HINIC_RQ_CTXT_PI_SET(rq->msix_entry, INTR);
+
+	rq_ctxt->wq_hi_pfn_ci = HINIC_RQ_CTXT_WQ_PAGE_SET(wq_page_pfn_hi,
+							  HI_PFN) |
+				HINIC_RQ_CTXT_WQ_PAGE_SET(ci_start, CI);
+
+	rq_ctxt->wq_lo_pfn = wq_page_pfn_lo;
+
+	rq_ctxt->pref_cache =
+		HINIC_RQ_CTXT_PREF_SET(WQ_PREFETCH_MIN, CACHE_MIN) |
+		HINIC_RQ_CTXT_PREF_SET(WQ_PREFETCH_MAX, CACHE_MAX) |
+		HINIC_RQ_CTXT_PREF_SET(WQ_PREFETCH_THRESHOLD, CACHE_THRESHOLD);
+
+	rq_ctxt->pref_wrapped = 1;
+
+	rq_ctxt->pref_wq_hi_pfn_ci =
+		HINIC_RQ_CTXT_PREF_SET(wq_page_pfn_hi, WQ_HI_PFN) |
+		HINIC_RQ_CTXT_PREF_SET(ci_start, CI);
+
+	rq_ctxt->pref_wq_lo_pfn = wq_page_pfn_lo;
+
+	rq_ctxt->pi_paddr_hi = upper_32_bits(rq->pi_dma_addr);
+	rq_ctxt->pi_paddr_lo = lower_32_bits(rq->pi_dma_addr);
+
+	rq_ctxt->wq_block_hi_pfn =
+		HINIC_RQ_CTXT_WQ_BLOCK_SET(wq_block_pfn_hi, HI_PFN);
+
+	rq_ctxt->wq_block_lo_pfn = wq_block_pfn_lo;
+
+	hinic_cpu_to_be32(rq_ctxt, sizeof(*rq_ctxt));
+}
+
+/**
+ * alloc_sq_skb_arr - allocate sq array for saved skb
+ * @sq: HW Send Queue
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int alloc_sq_skb_arr(struct hinic_sq *sq)
+{
+	struct hinic_wq *wq = sq->wq;
+	size_t skb_arr_size;
+
+	skb_arr_size = wq->q_depth * sizeof(*sq->saved_skb);
+	sq->saved_skb = vzalloc(skb_arr_size);
+	if (!sq->saved_skb)
+		return -ENOMEM;
+
+	return 0;
+}
+
+/**
+ * free_sq_skb_arr - free sq array for saved skb
+ * @sq: HW Send Queue
+ **/
+static void free_sq_skb_arr(struct hinic_sq *sq)
+{
+	vfree(sq->saved_skb);
+}
+
+/**
+ * alloc_rq_skb_arr - allocate rq array for saved skb
+ * @rq: HW Receive Queue
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int alloc_rq_skb_arr(struct hinic_rq *rq)
+{
+	struct hinic_wq *wq = rq->wq;
+	size_t skb_arr_size;
+
+	skb_arr_size = wq->q_depth * sizeof(*rq->saved_skb);
+	rq->saved_skb = vzalloc(skb_arr_size);
+	if (!rq->saved_skb)
+		return -ENOMEM;
+
+	return 0;
+}
+
+/**
+ * free_rq_skb_arr - free rq array for saved skb
+ * @rq: HW Receive Queue
+ **/
+static void free_rq_skb_arr(struct hinic_rq *rq)
+{
+	vfree(rq->saved_skb);
+}
+
+/**
+ * hinic_init_sq - Initialize HW Send Queue
+ * @sq: HW Send Queue
+ * @hwif: HW Interface for accessing HW
+ * @wq: Work Queue for the data of the SQ
+ * @entry: msix entry for sq
+ * @ci_addr: address for reading the current HW consumer index
+ * @ci_dma_addr: dma address for reading the current HW consumer index
+ * @db_base: doorbell base address
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_init_sq(struct hinic_sq *sq, struct hinic_hwif *hwif,
+		  struct hinic_wq *wq, struct msix_entry *entry,
+		  void *ci_addr, dma_addr_t ci_dma_addr,
+		  void __iomem *db_base)
+{
+	sq->hwif = hwif;
+
+	sq->wq = wq;
+
+	sq->irq = entry->vector;
+	sq->msix_entry = entry->entry;
+
+	sq->hw_ci_addr = ci_addr;
+	sq->hw_ci_dma_addr = ci_dma_addr;
+
+	sq->db_base = db_base + SQ_DB_OFF;
+
+	return alloc_sq_skb_arr(sq);
+}
+
+/**
+ * hinic_clean_sq - Clean HW Send Queue's Resources
+ * @sq: Send Queue
+ **/
+void hinic_clean_sq(struct hinic_sq *sq)
+{
+	free_sq_skb_arr(sq);
+}
+
+/**
+ * alloc_rq_cqe - allocate rq completion queue elements
+ * @rq: HW Receive Queue
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int alloc_rq_cqe(struct hinic_rq *rq)
+{
+	struct hinic_hwif *hwif = rq->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	size_t cqe_dma_size, cqe_size;
+	struct hinic_wq *wq = rq->wq;
+	int j, i;
+
+	cqe_size = wq->q_depth * sizeof(*rq->cqe);
+	rq->cqe = vzalloc(cqe_size);
+	if (!rq->cqe)
+		return -ENOMEM;
+
+	cqe_dma_size = wq->q_depth * sizeof(*rq->cqe_dma);
+	rq->cqe_dma = vzalloc(cqe_dma_size);
+	if (!rq->cqe_dma)
+		goto err_cqe_dma_arr_alloc;
+
+	for (i = 0; i < wq->q_depth; i++) {
+		rq->cqe[i] = dma_zalloc_coherent(&pdev->dev,
+						 sizeof(*rq->cqe[i]),
+						 &rq->cqe_dma[i], GFP_KERNEL);
+		if (!rq->cqe[i])
+			goto err_cqe_alloc;
+	}
+
+	return 0;
+
+err_cqe_alloc:
+	for (j = 0; j < i; j++)
+		dma_free_coherent(&pdev->dev, sizeof(*rq->cqe[j]), rq->cqe[j],
+				  rq->cqe_dma[j]);
+
+	vfree(rq->cqe_dma);
+
+err_cqe_dma_arr_alloc:
+	vfree(rq->cqe);
+	return -ENOMEM;
+}
+
+/**
+ * free_rq_cqe - free rq completion queue elements
+ * @rq: HW Receive Queue
+ **/
+static void free_rq_cqe(struct hinic_rq *rq)
+{
+	struct hinic_hwif *hwif = rq->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	struct hinic_wq *wq = rq->wq;
+	int i;
+
+	for (i = 0; i < wq->q_depth; i++)
+		dma_free_coherent(&pdev->dev, sizeof(*rq->cqe[i]), rq->cqe[i],
+				  rq->cqe_dma[i]);
+
+	vfree(rq->cqe_dma);
+	vfree(rq->cqe);
+}
+
+/**
+ * hinic_init_rq - Initialize HW Receive Queue
+ * @rq: HW Receive Queue
+ * @hwif: HW Interface for accessing HW
+ * @wq: Work Queue for the data of the RQ
+ * @entry: msix entry for rq
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_init_rq(struct hinic_rq *rq, struct hinic_hwif *hwif,
+		  struct hinic_wq *wq, struct msix_entry *entry)
+{
+	struct pci_dev *pdev = hwif->pdev;
+	size_t pi_size;
+	int err;
+
+	rq->hwif = hwif;
+
+	rq->wq = wq;
+
+	rq->irq = entry->vector;
+	rq->msix_entry = entry->entry;
+
+	rq->buf_sz = HINIC_RX_BUF_SZ;
+
+	err = alloc_rq_skb_arr(rq);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to allocate rq priv data\n");
+		return err;
+	}
+
+	err = alloc_rq_cqe(rq);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to allocate rq cqe\n");
+		goto err_alloc_rq_cqe;
+	}
+
+	/* HW requirements: Must be at least 32 bit */
+	pi_size = ALIGN(sizeof(*rq->pi_virt_addr), sizeof(u32));
+	rq->pi_virt_addr = dma_zalloc_coherent(&pdev->dev, pi_size,
+					       &rq->pi_dma_addr, GFP_KERNEL);
+	if (!rq->pi_virt_addr) {
+		dev_err(&pdev->dev, "Failed to allocate PI address\n");
+		err = -ENOMEM;
+		goto err_pi_virt;
+	}
+
+	return 0;
+
+err_pi_virt:
+	free_rq_cqe(rq);
+
+err_alloc_rq_cqe:
+	free_rq_skb_arr(rq);
+	return err;
+}
+
+/**
+ * hinic_clean_rq - Clean HW Receive Queue's Resources
+ * @rq: HW Receive Queue
+ **/
+void hinic_clean_rq(struct hinic_rq *rq)
+{
+	struct hinic_hwif *hwif = rq->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	size_t pi_size;
+
+	pi_size = ALIGN(sizeof(*rq->pi_virt_addr), sizeof(u32));
+	dma_free_coherent(&pdev->dev, pi_size, rq->pi_virt_addr,
+			  rq->pi_dma_addr);
+
+	free_rq_cqe(rq);
+	free_rq_skb_arr(rq);
+}
+
+/**
+ * hinic_get_sq_free_wqebbs - return number of free wqebbs for use
+ * @sq: send queue
+ *
+ * Return number of free wqebbs
+ **/
+int hinic_get_sq_free_wqebbs(struct hinic_sq *sq)
+{
+	struct hinic_wq *wq = sq->wq;
+
+	return atomic_read(&wq->delta) - 1;
+}
+
+/**
+ * hinic_get_rq_free_wqebbs - return number of free wqebbs for use
+ * @rq: recv queue
+ *
+ * Return number of free wqebbs
+ **/
+int hinic_get_rq_free_wqebbs(struct hinic_rq *rq)
+{
+	struct hinic_wq *wq = rq->wq;
+
+	return atomic_read(&wq->delta) - 1;
+}
+
+static void sq_prepare_ctrl(struct hinic_sq_ctrl *ctrl, u16 prod_idx,
+			    int nr_descs)
+{
+	u32 ctrl_size, task_size, bufdesc_size;
+
+	ctrl_size = SIZE_8BYTES(sizeof(struct hinic_sq_ctrl));
+	task_size = SIZE_8BYTES(sizeof(struct hinic_sq_task));
+	bufdesc_size = nr_descs * sizeof(struct hinic_sq_bufdesc);
+	bufdesc_size = SIZE_8BYTES(bufdesc_size);
+
+	ctrl->ctrl_info = HINIC_SQ_CTRL_SET(bufdesc_size, BUFDESC_SECT_LEN) |
+			  HINIC_SQ_CTRL_SET(task_size, TASKSECT_LEN)        |
+			  HINIC_SQ_CTRL_SET(SQ_NORMAL_WQE, DATA_FORMAT)     |
+			  HINIC_SQ_CTRL_SET(ctrl_size, LEN);
+
+	ctrl->queue_info = HINIC_SQ_CTRL_SET(TX_MAX_MSS_DEFAULT,
+					     QUEUE_INFO_MSS);
+}
+
+static void sq_prepare_task(struct hinic_sq_task *task)
+{
+	task->pkt_info0 =
+		HINIC_SQ_TASK_INFO0_SET(0, L2HDR_LEN) |
+		HINIC_SQ_TASK_INFO0_SET(HINIC_L4_OFF_DISABLE, L4_OFFLOAD) |
+		HINIC_SQ_TASK_INFO0_SET(HINIC_OUTER_L3TYPE_UNKNOWN,
+					INNER_L3TYPE) |
+		HINIC_SQ_TASK_INFO0_SET(HINIC_VLAN_OFF_DISABLE,
+					VLAN_OFFLOAD) |
+		HINIC_SQ_TASK_INFO0_SET(HINIC_PKT_NOT_PARSED, PARSE_FLAG);
+
+	task->pkt_info1 =
+		HINIC_SQ_TASK_INFO1_SET(HINIC_MEDIA_UNKNOWN, MEDIA_TYPE) |
+		HINIC_SQ_TASK_INFO1_SET(0, INNER_L4_LEN) |
+		HINIC_SQ_TASK_INFO1_SET(0, INNER_L3_LEN);
+
+	task->pkt_info2 =
+		HINIC_SQ_TASK_INFO2_SET(0, TUNNEL_L4_LEN) |
+		HINIC_SQ_TASK_INFO2_SET(0, OUTER_L3_LEN)  |
+		HINIC_SQ_TASK_INFO2_SET(HINIC_TUNNEL_L4TYPE_UNKNOWN,
+					TUNNEL_L4TYPE)    |
+		HINIC_SQ_TASK_INFO2_SET(HINIC_OUTER_L3TYPE_UNKNOWN,
+					OUTER_L3TYPE);
+
+	task->ufo_v6_identify = 0;
+
+	task->pkt_info4 = HINIC_SQ_TASK_INFO4_SET(HINIC_L2TYPE_ETH, L2TYPE);
+
+	task->zero_pad = 0;
+}
+
+/**
+ * hinic_sq_prepare_wqe - prepare wqe before insert to the queue
+ * @sq: send queue
+ * @prod_idx: pi value
+ * @sq_wqe: wqe to prepare
+ * @sges: sges for use by the wqe for send for buf addresses
+ * @nr_sges: number of sges
+ **/
+void hinic_sq_prepare_wqe(struct hinic_sq *sq, u16 prod_idx,
+			  struct hinic_sq_wqe *sq_wqe, struct hinic_sge *sges,
+			  int nr_sges)
+{
+	int i;
+
+	sq_prepare_ctrl(&sq_wqe->ctrl, prod_idx, nr_sges);
+
+	sq_prepare_task(&sq_wqe->task);
+
+	for (i = 0; i < nr_sges; i++)
+		sq_wqe->buf_descs[i].sge = sges[i];
+}
+
+/**
+ * sq_prepare_db - prepare doorbell to write
+ * @sq: send queue
+ * @prod_idx: pi value for the doorbell
+ * @cos: cos of the doorbell
+ *
+ * Return db value
+ **/
+static u32 sq_prepare_db(struct hinic_sq *sq, u16 prod_idx, unsigned int cos)
+{
+	struct hinic_qp *qp = container_of(sq, struct hinic_qp, sq);
+	u8 hi_prod_idx = SQ_DB_PI_HI(SQ_MASKED_IDX(sq, prod_idx));
+
+	/* Data should be written to HW in Big Endian Format */
+	return cpu_to_be32(HINIC_SQ_DB_INFO_SET(hi_prod_idx, PI_HI)     |
+			   HINIC_SQ_DB_INFO_SET(HINIC_DB_SQ_TYPE, TYPE) |
+			   HINIC_SQ_DB_INFO_SET(HINIC_DATA_PATH, PATH)  |
+			   HINIC_SQ_DB_INFO_SET(cos, COS)               |
+			   HINIC_SQ_DB_INFO_SET(qp->q_id, QID));
+}
+
+/**
+ * hinic_sq_write_db- write doorbell
+ * @sq: send queue
+ * @prod_idx: pi value for the doorbell
+ * @wqe_size: wqe size
+ * @cos: cos of the wqe
+ **/
+void hinic_sq_write_db(struct hinic_sq *sq, u16 prod_idx, unsigned int wqe_size,
+		       unsigned int cos)
+{
+	struct hinic_wq *wq = sq->wq;
+
+	/* increment prod_idx to the next */
+	prod_idx += ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size;
+
+	wmb();  /* Write all before the doorbell */
+
+	writel(sq_prepare_db(sq, prod_idx, cos), SQ_DB_ADDR(sq, prod_idx));
+}
+
+/**
+ * hinic_sq_get_wqe - get wqe ptr in the current pi and update the pi
+ * @sq: sq to get wqe from
+ * @wqe_size: wqe size
+ * @prod_idx: returned pi
+ *
+ * Return wqe pointer
+ **/
+struct hinic_sq_wqe *hinic_sq_get_wqe(struct hinic_sq *sq,
+				      unsigned int wqe_size, u16 *prod_idx)
+{
+	struct hinic_hw_wqe *hw_wqe = hinic_get_wqe(sq->wq, wqe_size,
+						    prod_idx);
+
+	if (IS_ERR(hw_wqe))
+		return NULL;
+
+	return &hw_wqe->sq_wqe;
+}
+
+/**
+ * hinic_sq_write_wqe - write the wqe to the sq
+ * @sq: send queue
+ * @prod_idx: pi of the wqe
+ * @sq_wqe: the wqe to write
+ * @skb: skb to save
+ * @wqe_size: the size of the wqe
+ **/
+void hinic_sq_write_wqe(struct hinic_sq *sq, u16 prod_idx,
+			struct hinic_sq_wqe *sq_wqe,
+			struct sk_buff *skb, unsigned int wqe_size)
+{
+	struct hinic_hw_wqe *hw_wqe = (struct hinic_hw_wqe *)sq_wqe;
+
+	sq->saved_skb[prod_idx] = skb;
+
+	/* The data in the HW should be in Big Endian Format */
+	hinic_cpu_to_be32(sq_wqe, wqe_size);
+
+	hinic_write_wqe(sq->wq, hw_wqe, wqe_size);
+}
+
+/**
+ * hinic_sq_read_wqe - read wqe ptr in the current ci and update the ci
+ * @sq: send queue
+ * @skb: return skb that was saved
+ * @wqe_size: the size of the wqe
+ * @cons_idx: consumer index of the wqe
+ *
+ * Return wqe in ci position
+ **/
+struct hinic_sq_wqe *hinic_sq_read_wqe(struct hinic_sq *sq,
+				       struct sk_buff **skb,
+				       unsigned int *wqe_size, u16 *cons_idx)
+{
+	struct hinic_hw_wqe *hw_wqe;
+	struct hinic_sq_wqe *sq_wqe;
+	struct hinic_sq_ctrl *ctrl;
+	unsigned int buf_sect_len;
+	u32 ctrl_info;
+
+	/* read the ctrl section for getting wqe size */
+	hw_wqe = hinic_read_wqe(sq->wq, sizeof(*ctrl), cons_idx);
+	if (IS_ERR(hw_wqe))
+		return NULL;
+
+	sq_wqe = &hw_wqe->sq_wqe;
+	ctrl = &sq_wqe->ctrl;
+	ctrl_info = be32_to_cpu(ctrl->ctrl_info);
+	buf_sect_len = HINIC_SQ_CTRL_GET(ctrl_info, BUFDESC_SECT_LEN);
+
+	*wqe_size = sizeof(*ctrl) + sizeof(sq_wqe->task);
+	*wqe_size += SECT_SIZE_FROM_8BYTES(buf_sect_len);
+
+	*skb = sq->saved_skb[*cons_idx];
+
+	/* using the real wqe size to read wqe again */
+	hw_wqe = hinic_read_wqe(sq->wq, *wqe_size, cons_idx);
+
+	return &hw_wqe->sq_wqe;
+}
+
+/**
+ * hinic_sq_put_wqe - release the ci for new wqes
+ * @sq: send queue
+ * @wqe_size: the size of the wqe
+ **/
+void hinic_sq_put_wqe(struct hinic_sq *sq, unsigned int wqe_size)
+{
+	hinic_put_wqe(sq->wq, wqe_size);
+}
+
+/**
+ * hinic_sq_get_sges - get sges from the wqe
+ * @sq_wqe: wqe to get the sges from its buffer addresses
+ * @sges: returned sges
+ * @nr_sges: number sges to return
+ **/
+void hinic_sq_get_sges(struct hinic_sq_wqe *sq_wqe, struct hinic_sge *sges,
+		       int nr_sges)
+{
+	int i;
+
+	for (i = 0; i < nr_sges && i < HINIC_MAX_SQ_BUFDESCS; i++) {
+		sges[i] = sq_wqe->buf_descs[i].sge;
+		hinic_be32_to_cpu(&sges[i], sizeof(sges[i]));
+	}
+}
+
+/**
+ * hinic_rq_get_wqe - get wqe ptr in the current pi and update the pi
+ * @rq: rq to get wqe from
+ * @wqe_size: wqe size
+ * @prod_idx: returned pi
+ *
+ * Return wqe pointer
+ **/
+struct hinic_rq_wqe *hinic_rq_get_wqe(struct hinic_rq *rq,
+				      unsigned int wqe_size, u16 *prod_idx)
+{
+	struct hinic_hw_wqe *hw_wqe = hinic_get_wqe(rq->wq, wqe_size,
+						    prod_idx);
+
+	if (IS_ERR(hw_wqe))
+		return NULL;
+
+	return &hw_wqe->rq_wqe;
+}
+
+/**
+ * hinic_rq_write_wqe - write the wqe to the rq
+ * @rq: recv queue
+ * @prod_idx: pi of the wqe
+ * @rq_wqe: the wqe to write
+ * @skb: skb to save
+ **/
+void hinic_rq_write_wqe(struct hinic_rq *rq, u16 prod_idx,
+			struct hinic_rq_wqe *rq_wqe, struct sk_buff *skb)
+{
+	struct hinic_hw_wqe *hw_wqe = (struct hinic_hw_wqe *)rq_wqe;
+
+	rq->saved_skb[prod_idx] = skb;
+
+	/* The data in the HW should be in Big Endian Format */
+	hinic_cpu_to_be32(rq_wqe, sizeof(*rq_wqe));
+
+	hinic_write_wqe(rq->wq, hw_wqe, sizeof(*rq_wqe));
+}
+
+/**
+ * hinic_rq_read_wqe - read wqe ptr in the current ci and update the ci
+ * @rq: recv queue
+ * @wqe_size: the size of the wqe
+ * @skb: return saved skb
+ * @cons_idx: consumer index of the wqe
+ *
+ * Return wqe in ci position
+ **/
+struct hinic_rq_wqe *hinic_rq_read_wqe(struct hinic_rq *rq,
+				       unsigned int wqe_size,
+				       struct sk_buff **skb, u16 *cons_idx)
+{
+	struct hinic_hw_wqe *hw_wqe;
+	struct hinic_rq_cqe *cqe;
+	int rx_done;
+	u32 status;
+
+	hw_wqe = hinic_read_wqe(rq->wq, wqe_size, cons_idx);
+	if (IS_ERR(hw_wqe))
+		return NULL;
+
+	cqe = rq->cqe[*cons_idx];
+
+	status = be32_to_cpu(cqe->status);
+
+	rx_done = HINIC_RQ_CQE_STATUS_GET(status, RXDONE);
+	if (!rx_done)
+		return NULL;
+
+	*skb = rq->saved_skb[*cons_idx];
+
+	return &hw_wqe->rq_wqe;
+}
+
+/**
+ * hinic_rq_read_next_wqe - increment ci and read the wqe in ci position
+ * @rq: recv queue
+ * @wqe_size: the size of the wqe
+ * @skb: return saved skb
+ * @cons_idx: consumer index in the wq
+ *
+ * Return wqe in incremented ci position
+ **/
+struct hinic_rq_wqe *hinic_rq_read_next_wqe(struct hinic_rq *rq,
+					    unsigned int wqe_size,
+					    struct sk_buff **skb,
+					    u16 *cons_idx)
+{
+	struct hinic_wq *wq = rq->wq;
+	struct hinic_hw_wqe *hw_wqe;
+	unsigned int num_wqebbs;
+
+	wqe_size = ALIGN(wqe_size, wq->wqebb_size);
+	num_wqebbs = wqe_size / wq->wqebb_size;
+
+	*cons_idx = RQ_MASKED_IDX(rq, *cons_idx + num_wqebbs);
+
+	*skb = rq->saved_skb[*cons_idx];
+
+	hw_wqe = hinic_read_wqe_direct(wq, *cons_idx);
+
+	return &hw_wqe->rq_wqe;
+}
+
+/**
+ * hinic_put_wqe - release the ci for new wqes
+ * @rq: recv queue
+ * @cons_idx: consumer index of the wqe
+ * @wqe_size: the size of the wqe
+ **/
+void hinic_rq_put_wqe(struct hinic_rq *rq, u16 cons_idx,
+		      unsigned int wqe_size)
+{
+	struct hinic_rq_cqe *cqe = rq->cqe[cons_idx];
+	u32 status = be32_to_cpu(cqe->status);
+
+	status = HINIC_RQ_CQE_STATUS_CLEAR(status, RXDONE);
+
+	/* Rx WQE size is 1 WQEBB, no wq shadow*/
+	cqe->status = cpu_to_be32(status);
+
+	wmb();          /* clear done flag */
+
+	hinic_put_wqe(rq->wq, wqe_size);
+}
+
+/**
+ * hinic_rq_get_sge - get sge from the wqe
+ * @rq: recv queue
+ * @rq_wqe: wqe to get the sge from its buf address
+ * @cons_idx: consumer index
+ * @sge: returned sge
+ **/
+void hinic_rq_get_sge(struct hinic_rq *rq, struct hinic_rq_wqe *rq_wqe,
+		      u16 cons_idx, struct hinic_sge *sge)
+{
+	struct hinic_rq_cqe *cqe = rq->cqe[cons_idx];
+	u32 len = be32_to_cpu(cqe->len);
+
+	sge->hi_addr = be32_to_cpu(rq_wqe->buf_desc.hi_addr);
+	sge->lo_addr = be32_to_cpu(rq_wqe->buf_desc.lo_addr);
+	sge->len = HINIC_RQ_CQE_SGE_GET(len, LEN);
+}
+
+/**
+ * hinic_rq_prepare_wqe - prepare wqe before insert to the queue
+ * @rq: recv queue
+ * @prod_idx: pi value
+ * @rq_wqe: the wqe
+ * @sge: sge for use by the wqe for recv buf address
+ **/
+void hinic_rq_prepare_wqe(struct hinic_rq *rq, u16 prod_idx,
+			  struct hinic_rq_wqe *rq_wqe, struct hinic_sge *sge)
+{
+	struct hinic_rq_cqe_sect *cqe_sect = &rq_wqe->cqe_sect;
+	struct hinic_rq_bufdesc *buf_desc = &rq_wqe->buf_desc;
+	struct hinic_rq_cqe *cqe = rq->cqe[prod_idx];
+	struct hinic_rq_ctrl *ctrl = &rq_wqe->ctrl;
+	dma_addr_t cqe_dma = rq->cqe_dma[prod_idx];
+
+	ctrl->ctrl_info =
+		HINIC_RQ_CTRL_SET(SIZE_8BYTES(sizeof(*ctrl)), LEN) |
+		HINIC_RQ_CTRL_SET(SIZE_8BYTES(sizeof(*cqe_sect)),
+				  COMPLETE_LEN)                    |
+		HINIC_RQ_CTRL_SET(SIZE_8BYTES(sizeof(*buf_desc)),
+				  BUFDESC_SECT_LEN)                |
+		HINIC_RQ_CTRL_SET(RQ_COMPLETE_SGE, COMPLETE_FORMAT);
+
+	hinic_set_sge(&cqe_sect->sge, cqe_dma, sizeof(*cqe));
+
+	buf_desc->hi_addr = sge->hi_addr;
+	buf_desc->lo_addr = sge->lo_addr;
+}
+
+/**
+ * hinic_rq_update - update pi of the rq
+ * @rq: recv queue
+ * @prod_idx: pi value
+ **/
+void hinic_rq_update(struct hinic_rq *rq, u16 prod_idx)
+{
+	*rq->pi_virt_addr = cpu_to_be16(RQ_MASKED_IDX(rq, prod_idx + 1));
+}
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.h
new file mode 100644
index 000000000000..df729a1587e9
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.h
@@ -0,0 +1,201 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#ifndef HINIC_HW_QP_H
+#define HINIC_HW_QP_H
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/sizes.h>
+#include <linux/pci.h>
+#include <linux/skbuff.h>
+
+#include "hinic_common.h"
+#include "hinic_hw_if.h"
+#include "hinic_hw_wqe.h"
+#include "hinic_hw_wq.h"
+#include "hinic_hw_qp_ctxt.h"
+
+#define HINIC_SQ_DB_INFO_PI_HI_SHIFT            0
+#define HINIC_SQ_DB_INFO_QID_SHIFT              8
+#define HINIC_SQ_DB_INFO_PATH_SHIFT             23
+#define HINIC_SQ_DB_INFO_COS_SHIFT              24
+#define HINIC_SQ_DB_INFO_TYPE_SHIFT             27
+
+#define HINIC_SQ_DB_INFO_PI_HI_MASK             0xFF
+#define HINIC_SQ_DB_INFO_QID_MASK               0x3FF
+#define HINIC_SQ_DB_INFO_PATH_MASK              0x1
+#define HINIC_SQ_DB_INFO_COS_MASK               0x7
+#define HINIC_SQ_DB_INFO_TYPE_MASK              0x1F
+
+#define HINIC_SQ_DB_INFO_SET(val, member)       \
+		(((u32)(val) & HINIC_SQ_DB_INFO_##member##_MASK) \
+		 << HINIC_SQ_DB_INFO_##member##_SHIFT)
+
+#define HINIC_SQ_WQEBB_SIZE                     64
+#define HINIC_RQ_WQEBB_SIZE                     32
+
+#define HINIC_SQ_PAGE_SIZE                      SZ_4K
+#define HINIC_RQ_PAGE_SIZE                      SZ_4K
+
+#define HINIC_SQ_DEPTH                          SZ_4K
+#define HINIC_RQ_DEPTH                          SZ_4K
+
+/* In any change to HINIC_RX_BUF_SZ, HINIC_RX_BUF_SZ_IDX must be changed */
+#define HINIC_RX_BUF_SZ                         2048
+#define HINIC_RX_BUF_SZ_IDX			HINIC_RX_BUF_SZ_2048_IDX
+
+#define HINIC_MIN_TX_WQE_SIZE(wq)               \
+		ALIGN(HINIC_SQ_WQE_SIZE(1), (wq)->wqebb_size)
+
+#define HINIC_MIN_TX_NUM_WQEBBS(sq)             \
+		(HINIC_MIN_TX_WQE_SIZE((sq)->wq) / (sq)->wq->wqebb_size)
+
+enum hinic_rx_buf_sz_idx {
+	HINIC_RX_BUF_SZ_32_IDX,
+	HINIC_RX_BUF_SZ_64_IDX,
+	HINIC_RX_BUF_SZ_96_IDX,
+	HINIC_RX_BUF_SZ_128_IDX,
+	HINIC_RX_BUF_SZ_192_IDX,
+	HINIC_RX_BUF_SZ_256_IDX,
+	HINIC_RX_BUF_SZ_384_IDX,
+	HINIC_RX_BUF_SZ_512_IDX,
+	HINIC_RX_BUF_SZ_768_IDX,
+	HINIC_RX_BUF_SZ_1024_IDX,
+	HINIC_RX_BUF_SZ_1536_IDX,
+	HINIC_RX_BUF_SZ_2048_IDX,
+	HINIC_RX_BUF_SZ_3072_IDX,
+	HINIC_RX_BUF_SZ_4096_IDX,
+	HINIC_RX_BUF_SZ_8192_IDX,
+	HINIC_RX_BUF_SZ_16384_IDX,
+};
+
+struct hinic_sq {
+	struct hinic_hwif       *hwif;
+
+	struct hinic_wq         *wq;
+
+	u32                     irq;
+	u16                     msix_entry;
+
+	void                    *hw_ci_addr;
+	dma_addr_t              hw_ci_dma_addr;
+
+	void __iomem            *db_base;
+
+	struct sk_buff          **saved_skb;
+};
+
+struct hinic_rq {
+	struct hinic_hwif       *hwif;
+
+	struct hinic_wq         *wq;
+
+	u32                     irq;
+	u16                     msix_entry;
+
+	size_t                  buf_sz;
+
+	struct sk_buff          **saved_skb;
+
+	struct hinic_rq_cqe     **cqe;
+	dma_addr_t              *cqe_dma;
+
+	u16                     *pi_virt_addr;
+	dma_addr_t              pi_dma_addr;
+};
+
+struct hinic_qp {
+	struct hinic_sq         sq;
+	struct hinic_rq         rq;
+
+	u16     q_id;
+};
+
+void hinic_qp_prepare_header(struct hinic_qp_ctxt_header *qp_ctxt_hdr,
+			     enum hinic_qp_ctxt_type ctxt_type,
+			     u16 num_queues, u16 max_queues);
+
+void hinic_sq_prepare_ctxt(struct hinic_sq_ctxt *sq_ctxt,
+			   struct hinic_sq *sq, u16 global_qid);
+
+void hinic_rq_prepare_ctxt(struct hinic_rq_ctxt *rq_ctxt,
+			   struct hinic_rq *rq, u16 global_qid);
+
+int hinic_init_sq(struct hinic_sq *sq, struct hinic_hwif *hwif,
+		  struct hinic_wq *wq, struct msix_entry *entry, void *ci_addr,
+		  dma_addr_t ci_dma_addr, void __iomem *db_base);
+
+void hinic_clean_sq(struct hinic_sq *sq);
+
+int hinic_init_rq(struct hinic_rq *rq, struct hinic_hwif *hwif,
+		  struct hinic_wq *wq, struct msix_entry *entry);
+
+void hinic_clean_rq(struct hinic_rq *rq);
+
+int hinic_get_sq_free_wqebbs(struct hinic_sq *sq);
+
+int hinic_get_rq_free_wqebbs(struct hinic_rq *rq);
+
+void hinic_sq_prepare_wqe(struct hinic_sq *sq, u16 prod_idx,
+			  struct hinic_sq_wqe *wqe, struct hinic_sge *sges,
+			  int nr_sges);
+
+void hinic_sq_write_db(struct hinic_sq *sq, u16 prod_idx, unsigned int wqe_size,
+		       unsigned int cos);
+
+struct hinic_sq_wqe *hinic_sq_get_wqe(struct hinic_sq *sq,
+				      unsigned int wqe_size, u16 *prod_idx);
+
+void hinic_sq_write_wqe(struct hinic_sq *sq, u16 prod_idx,
+			struct hinic_sq_wqe *wqe, struct sk_buff *skb,
+			unsigned int wqe_size);
+
+struct hinic_sq_wqe *hinic_sq_read_wqe(struct hinic_sq *sq,
+				       struct sk_buff **skb,
+				       unsigned int *wqe_size, u16 *cons_idx);
+
+void hinic_sq_put_wqe(struct hinic_sq *sq, unsigned int wqe_size);
+
+void hinic_sq_get_sges(struct hinic_sq_wqe *wqe, struct hinic_sge *sges,
+		       int nr_sges);
+
+struct hinic_rq_wqe *hinic_rq_get_wqe(struct hinic_rq *rq,
+				      unsigned int wqe_size, u16 *prod_idx);
+
+void hinic_rq_write_wqe(struct hinic_rq *rq, u16 prod_idx,
+			struct hinic_rq_wqe *wqe, struct sk_buff *skb);
+
+struct hinic_rq_wqe *hinic_rq_read_wqe(struct hinic_rq *rq,
+				       unsigned int wqe_size,
+				       struct sk_buff **skb, u16 *cons_idx);
+
+struct hinic_rq_wqe *hinic_rq_read_next_wqe(struct hinic_rq *rq,
+					    unsigned int wqe_size,
+					    struct sk_buff **skb,
+					    u16 *cons_idx);
+
+void hinic_rq_put_wqe(struct hinic_rq *rq, u16 cons_idx,
+		      unsigned int wqe_size);
+
+void hinic_rq_get_sge(struct hinic_rq *rq, struct hinic_rq_wqe *wqe,
+		      u16 cons_idx, struct hinic_sge *sge);
+
+void hinic_rq_prepare_wqe(struct hinic_rq *rq, u16 prod_idx,
+			  struct hinic_rq_wqe *wqe, struct hinic_sge *sge);
+
+void hinic_rq_update(struct hinic_rq *rq, u16 prod_idx);
+
+#endif
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_qp_ctxt.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_qp_ctxt.h
new file mode 100644
index 000000000000..376abf00762b
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_qp_ctxt.h
@@ -0,0 +1,214 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#ifndef HINIC_HW_QP_CTXT_H
+#define HINIC_HW_QP_CTXT_H
+
+#include <linux/types.h>
+
+#include "hinic_hw_cmdq.h"
+
+#define HINIC_SQ_CTXT_CEQ_ATTR_GLOBAL_SQ_ID_SHIFT       13
+#define HINIC_SQ_CTXT_CEQ_ATTR_EN_SHIFT                 23
+
+#define HINIC_SQ_CTXT_CEQ_ATTR_GLOBAL_SQ_ID_MASK        0x3FF
+#define HINIC_SQ_CTXT_CEQ_ATTR_EN_MASK                  0x1
+
+#define HINIC_SQ_CTXT_CEQ_ATTR_SET(val, member)         \
+	(((u32)(val) & HINIC_SQ_CTXT_CEQ_ATTR_##member##_MASK) \
+	 << HINIC_SQ_CTXT_CEQ_ATTR_##member##_SHIFT)
+
+#define HINIC_SQ_CTXT_CI_IDX_SHIFT                      11
+#define HINIC_SQ_CTXT_CI_WRAPPED_SHIFT                  23
+
+#define HINIC_SQ_CTXT_CI_IDX_MASK                       0xFFF
+#define HINIC_SQ_CTXT_CI_WRAPPED_MASK                   0x1
+
+#define HINIC_SQ_CTXT_CI_SET(val, member)               \
+	(((u32)(val) & HINIC_SQ_CTXT_CI_##member##_MASK) \
+	 << HINIC_SQ_CTXT_CI_##member##_SHIFT)
+
+#define HINIC_SQ_CTXT_WQ_PAGE_HI_PFN_SHIFT              0
+#define HINIC_SQ_CTXT_WQ_PAGE_PI_SHIFT                  20
+
+#define HINIC_SQ_CTXT_WQ_PAGE_HI_PFN_MASK               0xFFFFF
+#define HINIC_SQ_CTXT_WQ_PAGE_PI_MASK                   0xFFF
+
+#define HINIC_SQ_CTXT_WQ_PAGE_SET(val, member)          \
+	(((u32)(val) & HINIC_SQ_CTXT_WQ_PAGE_##member##_MASK) \
+	 << HINIC_SQ_CTXT_WQ_PAGE_##member##_SHIFT)
+
+#define HINIC_SQ_CTXT_PREF_CACHE_THRESHOLD_SHIFT        0
+#define HINIC_SQ_CTXT_PREF_CACHE_MAX_SHIFT              14
+#define HINIC_SQ_CTXT_PREF_CACHE_MIN_SHIFT              25
+
+#define HINIC_SQ_CTXT_PREF_CACHE_THRESHOLD_MASK         0x3FFF
+#define HINIC_SQ_CTXT_PREF_CACHE_MAX_MASK               0x7FF
+#define HINIC_SQ_CTXT_PREF_CACHE_MIN_MASK               0x7F
+
+#define HINIC_SQ_CTXT_PREF_WQ_HI_PFN_SHIFT              0
+#define HINIC_SQ_CTXT_PREF_CI_SHIFT                     20
+
+#define HINIC_SQ_CTXT_PREF_WQ_HI_PFN_MASK               0xFFFFF
+#define HINIC_SQ_CTXT_PREF_CI_MASK                      0xFFF
+
+#define HINIC_SQ_CTXT_PREF_SET(val, member)             \
+	(((u32)(val) & HINIC_SQ_CTXT_PREF_##member##_MASK) \
+	 << HINIC_SQ_CTXT_PREF_##member##_SHIFT)
+
+#define HINIC_SQ_CTXT_WQ_BLOCK_HI_PFN_SHIFT             0
+
+#define HINIC_SQ_CTXT_WQ_BLOCK_HI_PFN_MASK              0x7FFFFF
+
+#define HINIC_SQ_CTXT_WQ_BLOCK_SET(val, member)         \
+	(((u32)(val) & HINIC_SQ_CTXT_WQ_BLOCK_##member##_MASK) \
+	 << HINIC_SQ_CTXT_WQ_BLOCK_##member##_SHIFT)
+
+#define HINIC_RQ_CTXT_CEQ_ATTR_EN_SHIFT                 0
+#define HINIC_RQ_CTXT_CEQ_ATTR_WRAPPED_SHIFT            1
+
+#define HINIC_RQ_CTXT_CEQ_ATTR_EN_MASK                  0x1
+#define HINIC_RQ_CTXT_CEQ_ATTR_WRAPPED_MASK             0x1
+
+#define HINIC_RQ_CTXT_CEQ_ATTR_SET(val, member)         \
+	(((u32)(val) & HINIC_RQ_CTXT_CEQ_ATTR_##member##_MASK) \
+	 << HINIC_RQ_CTXT_CEQ_ATTR_##member##_SHIFT)
+
+#define HINIC_RQ_CTXT_PI_IDX_SHIFT                      0
+#define HINIC_RQ_CTXT_PI_INTR_SHIFT                     22
+
+#define HINIC_RQ_CTXT_PI_IDX_MASK                       0xFFF
+#define HINIC_RQ_CTXT_PI_INTR_MASK                      0x3FF
+
+#define HINIC_RQ_CTXT_PI_SET(val, member)               \
+	(((u32)(val) & HINIC_RQ_CTXT_PI_##member##_MASK) << \
+	 HINIC_RQ_CTXT_PI_##member##_SHIFT)
+
+#define HINIC_RQ_CTXT_WQ_PAGE_HI_PFN_SHIFT              0
+#define HINIC_RQ_CTXT_WQ_PAGE_CI_SHIFT                  20
+
+#define HINIC_RQ_CTXT_WQ_PAGE_HI_PFN_MASK               0xFFFFF
+#define HINIC_RQ_CTXT_WQ_PAGE_CI_MASK                   0xFFF
+
+#define HINIC_RQ_CTXT_WQ_PAGE_SET(val, member)          \
+	(((u32)(val) & HINIC_RQ_CTXT_WQ_PAGE_##member##_MASK) << \
+	 HINIC_RQ_CTXT_WQ_PAGE_##member##_SHIFT)
+
+#define HINIC_RQ_CTXT_PREF_CACHE_THRESHOLD_SHIFT        0
+#define HINIC_RQ_CTXT_PREF_CACHE_MAX_SHIFT              14
+#define HINIC_RQ_CTXT_PREF_CACHE_MIN_SHIFT              25
+
+#define HINIC_RQ_CTXT_PREF_CACHE_THRESHOLD_MASK         0x3FFF
+#define HINIC_RQ_CTXT_PREF_CACHE_MAX_MASK               0x7FF
+#define HINIC_RQ_CTXT_PREF_CACHE_MIN_MASK               0x7F
+
+#define HINIC_RQ_CTXT_PREF_WQ_HI_PFN_SHIFT              0
+#define HINIC_RQ_CTXT_PREF_CI_SHIFT                     20
+
+#define HINIC_RQ_CTXT_PREF_WQ_HI_PFN_MASK               0xFFFFF
+#define HINIC_RQ_CTXT_PREF_CI_MASK                      0xFFF
+
+#define HINIC_RQ_CTXT_PREF_SET(val, member)             \
+	(((u32)(val) & HINIC_RQ_CTXT_PREF_##member##_MASK) << \
+	 HINIC_RQ_CTXT_PREF_##member##_SHIFT)
+
+#define HINIC_RQ_CTXT_WQ_BLOCK_HI_PFN_SHIFT             0
+
+#define HINIC_RQ_CTXT_WQ_BLOCK_HI_PFN_MASK              0x7FFFFF
+
+#define HINIC_RQ_CTXT_WQ_BLOCK_SET(val, member)         \
+	(((u32)(val) & HINIC_RQ_CTXT_WQ_BLOCK_##member##_MASK) << \
+	 HINIC_RQ_CTXT_WQ_BLOCK_##member##_SHIFT)
+
+#define HINIC_SQ_CTXT_SIZE(num_sqs) (sizeof(struct hinic_qp_ctxt_header) \
+				     + (num_sqs) * sizeof(struct hinic_sq_ctxt))
+
+#define HINIC_RQ_CTXT_SIZE(num_rqs) (sizeof(struct hinic_qp_ctxt_header) \
+				     + (num_rqs) * sizeof(struct hinic_rq_ctxt))
+
+#define HINIC_WQ_PAGE_PFN_SHIFT         12
+#define HINIC_WQ_BLOCK_PFN_SHIFT        9
+
+#define HINIC_WQ_PAGE_PFN(page_addr)    ((page_addr) >> HINIC_WQ_PAGE_PFN_SHIFT)
+#define HINIC_WQ_BLOCK_PFN(page_addr)   ((page_addr) >> \
+					 HINIC_WQ_BLOCK_PFN_SHIFT)
+
+#define HINIC_Q_CTXT_MAX                \
+		((HINIC_CMDQ_BUF_SIZE - sizeof(struct hinic_qp_ctxt_header)) \
+		 / sizeof(struct hinic_sq_ctxt))
+
+enum hinic_qp_ctxt_type {
+	HINIC_QP_CTXT_TYPE_SQ,
+	HINIC_QP_CTXT_TYPE_RQ
+};
+
+struct hinic_qp_ctxt_header {
+	u16     num_queues;
+	u16     queue_type;
+	u32     addr_offset;
+};
+
+struct hinic_sq_ctxt {
+	u32     ceq_attr;
+
+	u32     ci_wrapped;
+
+	u32     wq_hi_pfn_pi;
+	u32     wq_lo_pfn;
+
+	u32     pref_cache;
+	u32     pref_wrapped;
+	u32     pref_wq_hi_pfn_ci;
+	u32     pref_wq_lo_pfn;
+
+	u32     rsvd0;
+	u32     rsvd1;
+
+	u32     wq_block_hi_pfn;
+	u32     wq_block_lo_pfn;
+};
+
+struct hinic_rq_ctxt {
+	u32     ceq_attr;
+
+	u32     pi_intr_attr;
+
+	u32     wq_hi_pfn_ci;
+	u32     wq_lo_pfn;
+
+	u32     pref_cache;
+	u32     pref_wrapped;
+
+	u32     pref_wq_hi_pfn_ci;
+	u32     pref_wq_lo_pfn;
+
+	u32     pi_paddr_hi;
+	u32     pi_paddr_lo;
+
+	u32     wq_block_hi_pfn;
+	u32     wq_block_lo_pfn;
+};
+
+struct hinic_sq_ctxt_block {
+	struct hinic_qp_ctxt_header hdr;
+	struct hinic_sq_ctxt sq_ctxt[HINIC_Q_CTXT_MAX];
+};
+
+struct hinic_rq_ctxt_block {
+	struct hinic_qp_ctxt_header hdr;
+	struct hinic_rq_ctxt rq_ctxt[HINIC_Q_CTXT_MAX];
+};
+
+#endif
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c
new file mode 100644
index 000000000000..3e3181c089bd
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c
@@ -0,0 +1,878 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/pci.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/slab.h>
+#include <linux/atomic.h>
+#include <linux/semaphore.h>
+#include <linux/errno.h>
+#include <linux/vmalloc.h>
+#include <linux/err.h>
+#include <asm/byteorder.h>
+
+#include "hinic_hw_if.h"
+#include "hinic_hw_wqe.h"
+#include "hinic_hw_wq.h"
+#include "hinic_hw_cmdq.h"
+
+#define WQS_BLOCKS_PER_PAGE             4
+
+#define WQ_BLOCK_SIZE                   4096
+#define WQS_PAGE_SIZE                   (WQS_BLOCKS_PER_PAGE * WQ_BLOCK_SIZE)
+
+#define WQS_MAX_NUM_BLOCKS              128
+#define WQS_FREE_BLOCKS_SIZE(wqs)       (WQS_MAX_NUM_BLOCKS * \
+					 sizeof((wqs)->free_blocks[0]))
+
+#define WQ_SIZE(wq)                     ((wq)->q_depth * (wq)->wqebb_size)
+
+#define WQ_PAGE_ADDR_SIZE               sizeof(u64)
+#define WQ_MAX_PAGES                    (WQ_BLOCK_SIZE / WQ_PAGE_ADDR_SIZE)
+
+#define CMDQ_BLOCK_SIZE                 512
+#define CMDQ_PAGE_SIZE                  4096
+
+#define CMDQ_WQ_MAX_PAGES               (CMDQ_BLOCK_SIZE / WQ_PAGE_ADDR_SIZE)
+
+#define WQ_BASE_VADDR(wqs, wq)          \
+			((void *)((wqs)->page_vaddr[(wq)->page_idx]) \
+				+ (wq)->block_idx * WQ_BLOCK_SIZE)
+
+#define WQ_BASE_PADDR(wqs, wq)          \
+			((wqs)->page_paddr[(wq)->page_idx] \
+				+ (wq)->block_idx * WQ_BLOCK_SIZE)
+
+#define WQ_BASE_ADDR(wqs, wq)           \
+			((void *)((wqs)->shadow_page_vaddr[(wq)->page_idx]) \
+				+ (wq)->block_idx * WQ_BLOCK_SIZE)
+
+#define CMDQ_BASE_VADDR(cmdq_pages, wq) \
+			((void *)((cmdq_pages)->page_vaddr) \
+				+ (wq)->block_idx * CMDQ_BLOCK_SIZE)
+
+#define CMDQ_BASE_PADDR(cmdq_pages, wq) \
+			((cmdq_pages)->page_paddr \
+				+ (wq)->block_idx * CMDQ_BLOCK_SIZE)
+
+#define CMDQ_BASE_ADDR(cmdq_pages, wq)  \
+			((void *)((cmdq_pages)->shadow_page_vaddr) \
+				+ (wq)->block_idx * CMDQ_BLOCK_SIZE)
+
+#define WQE_PAGE_OFF(wq, idx)   (((idx) & ((wq)->num_wqebbs_per_page - 1)) * \
+					(wq)->wqebb_size)
+
+#define WQE_PAGE_NUM(wq, idx)   (((idx) / ((wq)->num_wqebbs_per_page)) \
+					& ((wq)->num_q_pages - 1))
+
+#define WQ_PAGE_ADDR(wq, idx)           \
+			((wq)->shadow_block_vaddr[WQE_PAGE_NUM(wq, idx)])
+
+#define MASKED_WQE_IDX(wq, idx)         ((idx) & (wq)->mask)
+
+#define WQE_IN_RANGE(wqe, start, end)   \
+		(((unsigned long)(wqe) >= (unsigned long)(start)) && \
+		 ((unsigned long)(wqe) < (unsigned long)(end)))
+
+#define WQE_SHADOW_PAGE(wq, wqe)        \
+		(((unsigned long)(wqe) - (unsigned long)(wq)->shadow_wqe) \
+			/ (wq)->max_wqe_size)
+
+/**
+ * queue_alloc_page - allocate page for Queue
+ * @hwif: HW interface for allocating DMA
+ * @vaddr: virtual address will be returned in this address
+ * @paddr: physical address will be returned in this address
+ * @shadow_vaddr: VM area will be return here for holding WQ page addresses
+ * @page_sz: page size of each WQ page
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int queue_alloc_page(struct hinic_hwif *hwif, u64 **vaddr, u64 *paddr,
+			    void ***shadow_vaddr, size_t page_sz)
+{
+	struct pci_dev *pdev = hwif->pdev;
+	dma_addr_t dma_addr;
+
+	*vaddr = dma_zalloc_coherent(&pdev->dev, page_sz, &dma_addr,
+				     GFP_KERNEL);
+	if (!*vaddr) {
+		dev_err(&pdev->dev, "Failed to allocate dma for wqs page\n");
+		return -ENOMEM;
+	}
+
+	*paddr = (u64)dma_addr;
+
+	/* use vzalloc for big mem */
+	*shadow_vaddr = vzalloc(page_sz);
+	if (!*shadow_vaddr)
+		goto err_shadow_vaddr;
+
+	return 0;
+
+err_shadow_vaddr:
+	dma_free_coherent(&pdev->dev, page_sz, *vaddr, dma_addr);
+	return -ENOMEM;
+}
+
+/**
+ * wqs_allocate_page - allocate page for WQ set
+ * @wqs: Work Queue Set
+ * @page_idx: the page index of the page will be allocated
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int wqs_allocate_page(struct hinic_wqs *wqs, int page_idx)
+{
+	return queue_alloc_page(wqs->hwif, &wqs->page_vaddr[page_idx],
+				&wqs->page_paddr[page_idx],
+				&wqs->shadow_page_vaddr[page_idx],
+				WQS_PAGE_SIZE);
+}
+
+/**
+ * wqs_free_page - free page of WQ set
+ * @wqs: Work Queue Set
+ * @page_idx: the page index of the page will be freed
+ **/
+static void wqs_free_page(struct hinic_wqs *wqs, int page_idx)
+{
+	struct hinic_hwif *hwif = wqs->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+
+	dma_free_coherent(&pdev->dev, WQS_PAGE_SIZE,
+			  wqs->page_vaddr[page_idx],
+			  (dma_addr_t)wqs->page_paddr[page_idx]);
+	vfree(wqs->shadow_page_vaddr[page_idx]);
+}
+
+/**
+ * cmdq_allocate_page - allocate page for cmdq
+ * @cmdq_pages: the pages of the cmdq queue struct to hold the page
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int cmdq_allocate_page(struct hinic_cmdq_pages *cmdq_pages)
+{
+	return queue_alloc_page(cmdq_pages->hwif, &cmdq_pages->page_vaddr,
+				&cmdq_pages->page_paddr,
+				&cmdq_pages->shadow_page_vaddr,
+				CMDQ_PAGE_SIZE);
+}
+
+/**
+ * cmdq_free_page - free page from cmdq
+ * @cmdq_pages: the pages of the cmdq queue struct that hold the page
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static void cmdq_free_page(struct hinic_cmdq_pages *cmdq_pages)
+{
+	struct hinic_hwif *hwif = cmdq_pages->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+
+	dma_free_coherent(&pdev->dev, CMDQ_PAGE_SIZE,
+			  cmdq_pages->page_vaddr,
+			  (dma_addr_t)cmdq_pages->page_paddr);
+	vfree(cmdq_pages->shadow_page_vaddr);
+}
+
+static int alloc_page_arrays(struct hinic_wqs *wqs)
+{
+	struct hinic_hwif *hwif = wqs->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	size_t size;
+
+	size = wqs->num_pages * sizeof(*wqs->page_paddr);
+	wqs->page_paddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
+	if (!wqs->page_paddr)
+		return -ENOMEM;
+
+	size = wqs->num_pages * sizeof(*wqs->page_vaddr);
+	wqs->page_vaddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
+	if (!wqs->page_vaddr)
+		goto err_page_vaddr;
+
+	size = wqs->num_pages * sizeof(*wqs->shadow_page_vaddr);
+	wqs->shadow_page_vaddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
+	if (!wqs->shadow_page_vaddr)
+		goto err_page_shadow_vaddr;
+
+	return 0;
+
+err_page_shadow_vaddr:
+	devm_kfree(&pdev->dev, wqs->page_vaddr);
+
+err_page_vaddr:
+	devm_kfree(&pdev->dev, wqs->page_paddr);
+	return -ENOMEM;
+}
+
+static void free_page_arrays(struct hinic_wqs *wqs)
+{
+	struct hinic_hwif *hwif = wqs->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+
+	devm_kfree(&pdev->dev, wqs->shadow_page_vaddr);
+	devm_kfree(&pdev->dev, wqs->page_vaddr);
+	devm_kfree(&pdev->dev, wqs->page_paddr);
+}
+
+static int wqs_next_block(struct hinic_wqs *wqs, int *page_idx,
+			  int *block_idx)
+{
+	int pos;
+
+	down(&wqs->alloc_blocks_lock);
+
+	wqs->num_free_blks--;
+
+	if (wqs->num_free_blks < 0) {
+		wqs->num_free_blks++;
+		up(&wqs->alloc_blocks_lock);
+		return -ENOMEM;
+	}
+
+	pos = wqs->alloc_blk_pos++;
+	pos &= WQS_MAX_NUM_BLOCKS - 1;
+
+	*page_idx = wqs->free_blocks[pos].page_idx;
+	*block_idx = wqs->free_blocks[pos].block_idx;
+
+	wqs->free_blocks[pos].page_idx = -1;
+	wqs->free_blocks[pos].block_idx = -1;
+
+	up(&wqs->alloc_blocks_lock);
+	return 0;
+}
+
+static void wqs_return_block(struct hinic_wqs *wqs, int page_idx,
+			     int block_idx)
+{
+	int pos;
+
+	down(&wqs->alloc_blocks_lock);
+
+	pos = wqs->return_blk_pos++;
+	pos &= WQS_MAX_NUM_BLOCKS - 1;
+
+	wqs->free_blocks[pos].page_idx = page_idx;
+	wqs->free_blocks[pos].block_idx = block_idx;
+
+	wqs->num_free_blks++;
+
+	up(&wqs->alloc_blocks_lock);
+}
+
+static void init_wqs_blocks_arr(struct hinic_wqs *wqs)
+{
+	int page_idx, blk_idx, pos = 0;
+
+	for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) {
+		for (blk_idx = 0; blk_idx < WQS_BLOCKS_PER_PAGE; blk_idx++) {
+			wqs->free_blocks[pos].page_idx = page_idx;
+			wqs->free_blocks[pos].block_idx = blk_idx;
+			pos++;
+		}
+	}
+
+	wqs->alloc_blk_pos = 0;
+	wqs->return_blk_pos = pos;
+	wqs->num_free_blks = pos;
+
+	sema_init(&wqs->alloc_blocks_lock, 1);
+}
+
+/**
+ * hinic_wqs_alloc - allocate Work Queues set
+ * @wqs: Work Queue Set
+ * @max_wqs: maximum wqs to allocate
+ * @hwif: HW interface for use for the allocation
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_wqs_alloc(struct hinic_wqs *wqs, int max_wqs,
+		    struct hinic_hwif *hwif)
+{
+	struct pci_dev *pdev = hwif->pdev;
+	int err, i, page_idx;
+
+	max_wqs = ALIGN(max_wqs, WQS_BLOCKS_PER_PAGE);
+	if (max_wqs > WQS_MAX_NUM_BLOCKS)  {
+		dev_err(&pdev->dev, "Invalid max_wqs = %d\n", max_wqs);
+		return -EINVAL;
+	}
+
+	wqs->hwif = hwif;
+	wqs->num_pages = max_wqs / WQS_BLOCKS_PER_PAGE;
+
+	if (alloc_page_arrays(wqs)) {
+		dev_err(&pdev->dev,
+			"Failed to allocate mem for page addresses\n");
+		return -ENOMEM;
+	}
+
+	for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) {
+		err = wqs_allocate_page(wqs, page_idx);
+		if (err) {
+			dev_err(&pdev->dev, "Failed wq page allocation\n");
+			goto err_wq_allocate_page;
+		}
+	}
+
+	wqs->free_blocks = devm_kzalloc(&pdev->dev, WQS_FREE_BLOCKS_SIZE(wqs),
+					GFP_KERNEL);
+	if (!wqs->free_blocks) {
+		err = -ENOMEM;
+		goto err_alloc_blocks;
+	}
+
+	init_wqs_blocks_arr(wqs);
+	return 0;
+
+err_alloc_blocks:
+err_wq_allocate_page:
+	for (i = 0; i < page_idx; i++)
+		wqs_free_page(wqs, i);
+
+	free_page_arrays(wqs);
+	return err;
+}
+
+/**
+ * hinic_wqs_free - free Work Queues set
+ * @wqs: Work Queue Set
+ **/
+void hinic_wqs_free(struct hinic_wqs *wqs)
+{
+	struct hinic_hwif *hwif = wqs->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	int page_idx;
+
+	devm_kfree(&pdev->dev, wqs->free_blocks);
+
+	for (page_idx = 0; page_idx < wqs->num_pages; page_idx++)
+		wqs_free_page(wqs, page_idx);
+
+	free_page_arrays(wqs);
+}
+
+/**
+ * alloc_wqes_shadow - allocate WQE shadows for WQ
+ * @wq: WQ to allocate shadows for
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int alloc_wqes_shadow(struct hinic_wq *wq)
+{
+	struct hinic_hwif *hwif = wq->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	size_t size;
+
+	size = wq->num_q_pages * wq->max_wqe_size;
+	wq->shadow_wqe = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
+	if (!wq->shadow_wqe)
+		return -ENOMEM;
+
+	size = wq->num_q_pages * sizeof(wq->prod_idx);
+	wq->shadow_idx = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
+	if (!wq->shadow_idx)
+		goto err_shadow_idx;
+
+	return 0;
+
+err_shadow_idx:
+	devm_kfree(&pdev->dev, wq->shadow_wqe);
+	return -ENOMEM;
+}
+
+/**
+ * free_wqes_shadow - free WQE shadows of WQ
+ * @wq: WQ to free shadows from
+ **/
+static void free_wqes_shadow(struct hinic_wq *wq)
+{
+	struct hinic_hwif *hwif = wq->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+
+	devm_kfree(&pdev->dev, wq->shadow_idx);
+	devm_kfree(&pdev->dev, wq->shadow_wqe);
+}
+
+/**
+ * free_wq_pages - free pages of WQ
+ * @hwif: HW interface for releasing dma addresses
+ * @wq: WQ to free pages from
+ * @num_q_pages: number pages to free
+ **/
+static void free_wq_pages(struct hinic_wq *wq, struct hinic_hwif *hwif,
+			  int num_q_pages)
+{
+	struct pci_dev *pdev = hwif->pdev;
+	int i;
+
+	for (i = 0; i < num_q_pages; i++) {
+		void **vaddr = &wq->shadow_block_vaddr[i];
+		u64 *paddr = &wq->block_vaddr[i];
+		dma_addr_t dma_addr;
+
+		dma_addr = (dma_addr_t)be64_to_cpu(*paddr);
+		dma_free_coherent(&pdev->dev, wq->wq_page_size, *vaddr,
+				  dma_addr);
+	}
+
+	free_wqes_shadow(wq);
+}
+
+/**
+ * alloc_wq_pages - alloc pages for WQ
+ * @hwif: HW interface for allocating dma addresses
+ * @wq: WQ to allocate pages for
+ * @max_pages: maximum pages allowed
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int alloc_wq_pages(struct hinic_wq *wq, struct hinic_hwif *hwif,
+			  int max_pages)
+{
+	struct pci_dev *pdev = hwif->pdev;
+	int i, err, num_q_pages;
+
+	num_q_pages = ALIGN(WQ_SIZE(wq), wq->wq_page_size) / wq->wq_page_size;
+	if (num_q_pages > max_pages) {
+		dev_err(&pdev->dev, "Number wq pages exceeds the limit\n");
+		return -EINVAL;
+	}
+
+	if (num_q_pages & (num_q_pages - 1)) {
+		dev_err(&pdev->dev, "Number wq pages must be power of 2\n");
+		return -EINVAL;
+	}
+
+	wq->num_q_pages = num_q_pages;
+
+	err = alloc_wqes_shadow(wq);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to allocate wqe shadow\n");
+		return err;
+	}
+
+	for (i = 0; i < num_q_pages; i++) {
+		void **vaddr = &wq->shadow_block_vaddr[i];
+		u64 *paddr = &wq->block_vaddr[i];
+		dma_addr_t dma_addr;
+
+		*vaddr = dma_zalloc_coherent(&pdev->dev, wq->wq_page_size,
+					     &dma_addr, GFP_KERNEL);
+		if (!*vaddr) {
+			dev_err(&pdev->dev, "Failed to allocate wq page\n");
+			goto err_alloc_wq_pages;
+		}
+
+		/* HW uses Big Endian Format */
+		*paddr = cpu_to_be64(dma_addr);
+	}
+
+	return 0;
+
+err_alloc_wq_pages:
+	free_wq_pages(wq, hwif, i);
+	return -ENOMEM;
+}
+
+/**
+ * hinic_wq_allocate - Allocate the WQ resources from the WQS
+ * @wqs: WQ set from which to allocate the WQ resources
+ * @wq: WQ to allocate resources for it from the WQ set
+ * @wqebb_size: Work Queue Block Byte Size
+ * @wq_page_size: the page size in the Work Queue
+ * @q_depth: number of wqebbs in WQ
+ * @max_wqe_size: maximum WQE size that will be used in the WQ
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_wq_allocate(struct hinic_wqs *wqs, struct hinic_wq *wq,
+		      u16 wqebb_size, u16 wq_page_size, u16 q_depth,
+		      u16 max_wqe_size)
+{
+	struct hinic_hwif *hwif = wqs->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	u16 num_wqebbs_per_page;
+	int err;
+
+	if (wqebb_size == 0) {
+		dev_err(&pdev->dev, "wqebb_size must be > 0\n");
+		return -EINVAL;
+	}
+
+	if (wq_page_size == 0) {
+		dev_err(&pdev->dev, "wq_page_size must be > 0\n");
+		return -EINVAL;
+	}
+
+	if (q_depth & (q_depth - 1)) {
+		dev_err(&pdev->dev, "WQ q_depth must be power of 2\n");
+		return -EINVAL;
+	}
+
+	num_wqebbs_per_page = ALIGN(wq_page_size, wqebb_size) / wqebb_size;
+
+	if (num_wqebbs_per_page & (num_wqebbs_per_page - 1)) {
+		dev_err(&pdev->dev, "num wqebbs per page must be power of 2\n");
+		return -EINVAL;
+	}
+
+	wq->hwif = hwif;
+
+	err = wqs_next_block(wqs, &wq->page_idx, &wq->block_idx);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to get free wqs next block\n");
+		return err;
+	}
+
+	wq->wqebb_size = wqebb_size;
+	wq->wq_page_size = wq_page_size;
+	wq->q_depth = q_depth;
+	wq->max_wqe_size = max_wqe_size;
+	wq->num_wqebbs_per_page = num_wqebbs_per_page;
+
+	wq->block_vaddr = WQ_BASE_VADDR(wqs, wq);
+	wq->shadow_block_vaddr = WQ_BASE_ADDR(wqs, wq);
+	wq->block_paddr = WQ_BASE_PADDR(wqs, wq);
+
+	err = alloc_wq_pages(wq, wqs->hwif, WQ_MAX_PAGES);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to allocate wq pages\n");
+		goto err_alloc_wq_pages;
+	}
+
+	atomic_set(&wq->cons_idx, 0);
+	atomic_set(&wq->prod_idx, 0);
+	atomic_set(&wq->delta, q_depth);
+	wq->mask = q_depth - 1;
+
+	return 0;
+
+err_alloc_wq_pages:
+	wqs_return_block(wqs, wq->page_idx, wq->block_idx);
+	return err;
+}
+
+/**
+ * hinic_wq_free - Free the WQ resources to the WQS
+ * @wqs: WQ set to free the WQ resources to it
+ * @wq: WQ to free its resources to the WQ set resources
+ **/
+void hinic_wq_free(struct hinic_wqs *wqs, struct hinic_wq *wq)
+{
+	free_wq_pages(wq, wqs->hwif, wq->num_q_pages);
+
+	wqs_return_block(wqs, wq->page_idx, wq->block_idx);
+}
+
+/**
+ * hinic_wqs_cmdq_alloc - Allocate wqs for cmdqs
+ * @cmdq_pages: will hold the pages of the cmdq
+ * @wq: returned wqs
+ * @hwif: HW interface
+ * @cmdq_blocks: number of cmdq blocks/wq to allocate
+ * @wqebb_size: Work Queue Block Byte Size
+ * @wq_page_size: the page size in the Work Queue
+ * @q_depth: number of wqebbs in WQ
+ * @max_wqe_size: maximum WQE size that will be used in the WQ
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_wqs_cmdq_alloc(struct hinic_cmdq_pages *cmdq_pages,
+			 struct hinic_wq *wq, struct hinic_hwif *hwif,
+			 int cmdq_blocks, u16 wqebb_size, u16 wq_page_size,
+			 u16 q_depth, u16 max_wqe_size)
+{
+	struct pci_dev *pdev = hwif->pdev;
+	u16 num_wqebbs_per_page;
+	int i, j, err = -ENOMEM;
+
+	if (wqebb_size == 0) {
+		dev_err(&pdev->dev, "wqebb_size must be > 0\n");
+		return -EINVAL;
+	}
+
+	if (wq_page_size == 0) {
+		dev_err(&pdev->dev, "wq_page_size must be > 0\n");
+		return -EINVAL;
+	}
+
+	if (q_depth & (q_depth - 1)) {
+		dev_err(&pdev->dev, "WQ q_depth must be power of 2\n");
+		return -EINVAL;
+	}
+
+	num_wqebbs_per_page = ALIGN(wq_page_size, wqebb_size) / wqebb_size;
+
+	if (num_wqebbs_per_page & (num_wqebbs_per_page - 1)) {
+		dev_err(&pdev->dev, "num wqebbs per page must be power of 2\n");
+		return -EINVAL;
+	}
+
+	cmdq_pages->hwif = hwif;
+
+	err = cmdq_allocate_page(cmdq_pages);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to allocate CMDQ page\n");
+		return err;
+	}
+
+	for (i = 0; i < cmdq_blocks; i++) {
+		wq[i].hwif = hwif;
+		wq[i].page_idx = 0;
+		wq[i].block_idx = i;
+
+		wq[i].wqebb_size = wqebb_size;
+		wq[i].wq_page_size = wq_page_size;
+		wq[i].q_depth = q_depth;
+		wq[i].max_wqe_size = max_wqe_size;
+		wq[i].num_wqebbs_per_page = num_wqebbs_per_page;
+
+		wq[i].block_vaddr = CMDQ_BASE_VADDR(cmdq_pages, &wq[i]);
+		wq[i].shadow_block_vaddr = CMDQ_BASE_ADDR(cmdq_pages, &wq[i]);
+		wq[i].block_paddr = CMDQ_BASE_PADDR(cmdq_pages, &wq[i]);
+
+		err = alloc_wq_pages(&wq[i], cmdq_pages->hwif,
+				     CMDQ_WQ_MAX_PAGES);
+		if (err) {
+			dev_err(&pdev->dev, "Failed to alloc CMDQ blocks\n");
+			goto err_cmdq_block;
+		}
+
+		atomic_set(&wq[i].cons_idx, 0);
+		atomic_set(&wq[i].prod_idx, 0);
+		atomic_set(&wq[i].delta, q_depth);
+		wq[i].mask = q_depth - 1;
+	}
+
+	return 0;
+
+err_cmdq_block:
+	for (j = 0; j < i; j++)
+		free_wq_pages(&wq[j], cmdq_pages->hwif, wq[j].num_q_pages);
+
+	cmdq_free_page(cmdq_pages);
+	return err;
+}
+
+/**
+ * hinic_wqs_cmdq_free - Free wqs from cmdqs
+ * @cmdq_pages: hold the pages of the cmdq
+ * @wq: wqs to free
+ * @cmdq_blocks: number of wqs to free
+ **/
+void hinic_wqs_cmdq_free(struct hinic_cmdq_pages *cmdq_pages,
+			 struct hinic_wq *wq, int cmdq_blocks)
+{
+	int i;
+
+	for (i = 0; i < cmdq_blocks; i++)
+		free_wq_pages(&wq[i], cmdq_pages->hwif, wq[i].num_q_pages);
+
+	cmdq_free_page(cmdq_pages);
+}
+
+static void copy_wqe_to_shadow(struct hinic_wq *wq, void *shadow_addr,
+			       int num_wqebbs, u16 idx)
+{
+	void *wqebb_addr;
+	int i;
+
+	for (i = 0; i < num_wqebbs; i++, idx++) {
+		idx = MASKED_WQE_IDX(wq, idx);
+		wqebb_addr = WQ_PAGE_ADDR(wq, idx) +
+			     WQE_PAGE_OFF(wq, idx);
+
+		memcpy(shadow_addr, wqebb_addr, wq->wqebb_size);
+
+		shadow_addr += wq->wqebb_size;
+	}
+}
+
+static void copy_wqe_from_shadow(struct hinic_wq *wq, void *shadow_addr,
+				 int num_wqebbs, u16 idx)
+{
+	void *wqebb_addr;
+	int i;
+
+	for (i = 0; i < num_wqebbs; i++, idx++) {
+		idx = MASKED_WQE_IDX(wq, idx);
+		wqebb_addr = WQ_PAGE_ADDR(wq, idx) +
+			     WQE_PAGE_OFF(wq, idx);
+
+		memcpy(wqebb_addr, shadow_addr, wq->wqebb_size);
+		shadow_addr += wq->wqebb_size;
+	}
+}
+
+/**
+ * hinic_get_wqe - get wqe ptr in the current pi and update the pi
+ * @wq: wq to get wqe from
+ * @wqe_size: wqe size
+ * @prod_idx: returned pi
+ *
+ * Return wqe pointer
+ **/
+struct hinic_hw_wqe *hinic_get_wqe(struct hinic_wq *wq, unsigned int wqe_size,
+				   u16 *prod_idx)
+{
+	int curr_pg, end_pg, num_wqebbs;
+	u16 curr_prod_idx, end_prod_idx;
+
+	*prod_idx = MASKED_WQE_IDX(wq, atomic_read(&wq->prod_idx));
+
+	num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size;
+
+	if (atomic_sub_return(num_wqebbs, &wq->delta) <= 0) {
+		atomic_add(num_wqebbs, &wq->delta);
+		return ERR_PTR(-EBUSY);
+	}
+
+	end_prod_idx = atomic_add_return(num_wqebbs, &wq->prod_idx);
+
+	end_prod_idx = MASKED_WQE_IDX(wq, end_prod_idx);
+	curr_prod_idx = end_prod_idx - num_wqebbs;
+	curr_prod_idx = MASKED_WQE_IDX(wq, curr_prod_idx);
+
+	/* end prod index points to the next wqebb, therefore minus 1 */
+	end_prod_idx = MASKED_WQE_IDX(wq, end_prod_idx - 1);
+
+	curr_pg = WQE_PAGE_NUM(wq, curr_prod_idx);
+	end_pg = WQE_PAGE_NUM(wq, end_prod_idx);
+
+	*prod_idx = curr_prod_idx;
+
+	if (curr_pg != end_pg) {
+		void *shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size];
+
+		copy_wqe_to_shadow(wq, shadow_addr, num_wqebbs, *prod_idx);
+
+		wq->shadow_idx[curr_pg] = *prod_idx;
+		return shadow_addr;
+	}
+
+	return WQ_PAGE_ADDR(wq, *prod_idx) + WQE_PAGE_OFF(wq, *prod_idx);
+}
+
+/**
+ * hinic_put_wqe - return the wqe place to use for a new wqe
+ * @wq: wq to return wqe
+ * @wqe_size: wqe size
+ **/
+void hinic_put_wqe(struct hinic_wq *wq, unsigned int wqe_size)
+{
+	int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size;
+
+	atomic_add(num_wqebbs, &wq->cons_idx);
+
+	atomic_add(num_wqebbs, &wq->delta);
+}
+
+/**
+ * hinic_read_wqe - read wqe ptr in the current ci
+ * @wq: wq to get read from
+ * @wqe_size: wqe size
+ * @cons_idx: returned ci
+ *
+ * Return wqe pointer
+ **/
+struct hinic_hw_wqe *hinic_read_wqe(struct hinic_wq *wq, unsigned int wqe_size,
+				    u16 *cons_idx)
+{
+	int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size;
+	u16 curr_cons_idx, end_cons_idx;
+	int curr_pg, end_pg;
+
+	if ((atomic_read(&wq->delta) + num_wqebbs) > wq->q_depth)
+		return ERR_PTR(-EBUSY);
+
+	curr_cons_idx = atomic_read(&wq->cons_idx);
+
+	curr_cons_idx = MASKED_WQE_IDX(wq, curr_cons_idx);
+	end_cons_idx = MASKED_WQE_IDX(wq, curr_cons_idx + num_wqebbs - 1);
+
+	curr_pg = WQE_PAGE_NUM(wq, curr_cons_idx);
+	end_pg = WQE_PAGE_NUM(wq, end_cons_idx);
+
+	*cons_idx = curr_cons_idx;
+
+	if (curr_pg != end_pg) {
+		void *shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size];
+
+		copy_wqe_to_shadow(wq, shadow_addr, num_wqebbs, *cons_idx);
+		return shadow_addr;
+	}
+
+	return WQ_PAGE_ADDR(wq, *cons_idx) + WQE_PAGE_OFF(wq, *cons_idx);
+}
+
+/**
+ * hinic_read_wqe_direct - read wqe directly from ci position
+ * @wq: wq
+ * @cons_idx: ci position
+ *
+ * Return wqe
+ **/
+struct hinic_hw_wqe *hinic_read_wqe_direct(struct hinic_wq *wq, u16 cons_idx)
+{
+	return WQ_PAGE_ADDR(wq, cons_idx) + WQE_PAGE_OFF(wq, cons_idx);
+}
+
+/**
+ * wqe_shadow - check if a wqe is shadow
+ * @wq: wq of the wqe
+ * @wqe: the wqe for shadow checking
+ *
+ * Return true - shadow, false - Not shadow
+ **/
+static inline bool wqe_shadow(struct hinic_wq *wq, struct hinic_hw_wqe *wqe)
+{
+	size_t wqe_shadow_size = wq->num_q_pages * wq->max_wqe_size;
+
+	return WQE_IN_RANGE(wqe, wq->shadow_wqe,
+			    &wq->shadow_wqe[wqe_shadow_size]);
+}
+
+/**
+ * hinic_write_wqe - write the wqe to the wq
+ * @wq: wq to write wqe to
+ * @wqe: wqe to write
+ * @wqe_size: wqe size
+ **/
+void hinic_write_wqe(struct hinic_wq *wq, struct hinic_hw_wqe *wqe,
+		     unsigned int wqe_size)
+{
+	int curr_pg, num_wqebbs;
+	void *shadow_addr;
+	u16 prod_idx;
+
+	if (wqe_shadow(wq, wqe)) {
+		curr_pg = WQE_SHADOW_PAGE(wq, wqe);
+
+		prod_idx = wq->shadow_idx[curr_pg];
+		num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size;
+		shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size];
+
+		copy_wqe_from_shadow(wq, shadow_addr, num_wqebbs, prod_idx);
+	}
+}
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.h
new file mode 100644
index 000000000000..9c030a0f035e
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.h
@@ -0,0 +1,117 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#ifndef HINIC_HW_WQ_H
+#define HINIC_HW_WQ_H
+
+#include <linux/types.h>
+#include <linux/semaphore.h>
+#include <linux/atomic.h>
+
+#include "hinic_hw_if.h"
+#include "hinic_hw_wqe.h"
+
+struct hinic_free_block {
+	int     page_idx;
+	int     block_idx;
+};
+
+struct hinic_wq {
+	struct hinic_hwif       *hwif;
+
+	int             page_idx;
+	int             block_idx;
+
+	u16             wqebb_size;
+	u16             wq_page_size;
+	u16             q_depth;
+	u16             max_wqe_size;
+	u16             num_wqebbs_per_page;
+
+	/* The addresses are 64 bit in the HW */
+	u64             block_paddr;
+	void            **shadow_block_vaddr;
+	u64             *block_vaddr;
+
+	int             num_q_pages;
+	u8              *shadow_wqe;
+	u16             *shadow_idx;
+
+	atomic_t        cons_idx;
+	atomic_t        prod_idx;
+	atomic_t        delta;
+	u16             mask;
+};
+
+struct hinic_wqs {
+	struct hinic_hwif       *hwif;
+	int                     num_pages;
+
+	/* The addresses are 64 bit in the HW */
+	u64                     *page_paddr;
+	u64                     **page_vaddr;
+	void                    ***shadow_page_vaddr;
+
+	struct hinic_free_block *free_blocks;
+	int                     alloc_blk_pos;
+	int                     return_blk_pos;
+	int                     num_free_blks;
+
+	/* Lock for getting a free block from the WQ set */
+	struct semaphore        alloc_blocks_lock;
+};
+
+struct hinic_cmdq_pages {
+	/* The addresses are 64 bit in the HW */
+	u64                     page_paddr;
+	u64                     *page_vaddr;
+	void                    **shadow_page_vaddr;
+
+	struct hinic_hwif       *hwif;
+};
+
+int hinic_wqs_cmdq_alloc(struct hinic_cmdq_pages *cmdq_pages,
+			 struct hinic_wq *wq, struct hinic_hwif *hwif,
+			 int cmdq_blocks, u16 wqebb_size, u16 wq_page_size,
+			 u16 q_depth, u16 max_wqe_size);
+
+void hinic_wqs_cmdq_free(struct hinic_cmdq_pages *cmdq_pages,
+			 struct hinic_wq *wq, int cmdq_blocks);
+
+int hinic_wqs_alloc(struct hinic_wqs *wqs, int num_wqs,
+		    struct hinic_hwif *hwif);
+
+void hinic_wqs_free(struct hinic_wqs *wqs);
+
+int hinic_wq_allocate(struct hinic_wqs *wqs, struct hinic_wq *wq,
+		      u16 wqebb_size, u16 wq_page_size, u16 q_depth,
+		      u16 max_wqe_size);
+
+void hinic_wq_free(struct hinic_wqs *wqs, struct hinic_wq *wq);
+
+struct hinic_hw_wqe *hinic_get_wqe(struct hinic_wq *wq, unsigned int wqe_size,
+				   u16 *prod_idx);
+
+void hinic_put_wqe(struct hinic_wq *wq, unsigned int wqe_size);
+
+struct hinic_hw_wqe *hinic_read_wqe(struct hinic_wq *wq, unsigned int wqe_size,
+				    u16 *cons_idx);
+
+struct hinic_hw_wqe *hinic_read_wqe_direct(struct hinic_wq *wq, u16 cons_idx);
+
+void hinic_write_wqe(struct hinic_wq *wq, struct hinic_hw_wqe *wqe,
+		     unsigned int wqe_size);
+
+#endif
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_wqe.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_wqe.h
new file mode 100644
index 000000000000..bc73485483c5
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_wqe.h
@@ -0,0 +1,368 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#ifndef HINIC_HW_WQE_H
+#define HINIC_HW_WQE_H
+
+#include "hinic_common.h"
+
+#define HINIC_CMDQ_CTRL_PI_SHIFT                        0
+#define HINIC_CMDQ_CTRL_CMD_SHIFT                       16
+#define HINIC_CMDQ_CTRL_MOD_SHIFT                       24
+#define HINIC_CMDQ_CTRL_ACK_TYPE_SHIFT                  29
+#define HINIC_CMDQ_CTRL_HW_BUSY_BIT_SHIFT               31
+
+#define HINIC_CMDQ_CTRL_PI_MASK                         0xFFFF
+#define HINIC_CMDQ_CTRL_CMD_MASK                        0xFF
+#define HINIC_CMDQ_CTRL_MOD_MASK                        0x1F
+#define HINIC_CMDQ_CTRL_ACK_TYPE_MASK                   0x3
+#define HINIC_CMDQ_CTRL_HW_BUSY_BIT_MASK                0x1
+
+#define HINIC_CMDQ_CTRL_SET(val, member)                        \
+			(((u32)(val) & HINIC_CMDQ_CTRL_##member##_MASK) \
+			 << HINIC_CMDQ_CTRL_##member##_SHIFT)
+
+#define HINIC_CMDQ_CTRL_GET(val, member)                        \
+			(((val) >> HINIC_CMDQ_CTRL_##member##_SHIFT) \
+			 & HINIC_CMDQ_CTRL_##member##_MASK)
+
+#define HINIC_CMDQ_WQE_HEADER_BUFDESC_LEN_SHIFT         0
+#define HINIC_CMDQ_WQE_HEADER_COMPLETE_FMT_SHIFT        15
+#define HINIC_CMDQ_WQE_HEADER_DATA_FMT_SHIFT            22
+#define HINIC_CMDQ_WQE_HEADER_COMPLETE_REQ_SHIFT        23
+#define HINIC_CMDQ_WQE_HEADER_COMPLETE_SECT_LEN_SHIFT   27
+#define HINIC_CMDQ_WQE_HEADER_CTRL_LEN_SHIFT            29
+#define HINIC_CMDQ_WQE_HEADER_TOGGLED_WRAPPED_SHIFT     31
+
+#define HINIC_CMDQ_WQE_HEADER_BUFDESC_LEN_MASK          0xFF
+#define HINIC_CMDQ_WQE_HEADER_COMPLETE_FMT_MASK         0x1
+#define HINIC_CMDQ_WQE_HEADER_DATA_FMT_MASK             0x1
+#define HINIC_CMDQ_WQE_HEADER_COMPLETE_REQ_MASK         0x1
+#define HINIC_CMDQ_WQE_HEADER_COMPLETE_SECT_LEN_MASK    0x3
+#define HINIC_CMDQ_WQE_HEADER_CTRL_LEN_MASK             0x3
+#define HINIC_CMDQ_WQE_HEADER_TOGGLED_WRAPPED_MASK      0x1
+
+#define HINIC_CMDQ_WQE_HEADER_SET(val, member)                  \
+			(((u32)(val) & HINIC_CMDQ_WQE_HEADER_##member##_MASK) \
+			 << HINIC_CMDQ_WQE_HEADER_##member##_SHIFT)
+
+#define HINIC_CMDQ_WQE_HEADER_GET(val, member)                  \
+			(((val) >> HINIC_CMDQ_WQE_HEADER_##member##_SHIFT) \
+			 & HINIC_CMDQ_WQE_HEADER_##member##_MASK)
+
+#define HINIC_SQ_CTRL_BUFDESC_SECT_LEN_SHIFT    0
+#define HINIC_SQ_CTRL_TASKSECT_LEN_SHIFT        16
+#define HINIC_SQ_CTRL_DATA_FORMAT_SHIFT         22
+#define HINIC_SQ_CTRL_LEN_SHIFT                 29
+
+#define HINIC_SQ_CTRL_BUFDESC_SECT_LEN_MASK     0xFF
+#define HINIC_SQ_CTRL_TASKSECT_LEN_MASK         0x1F
+#define HINIC_SQ_CTRL_DATA_FORMAT_MASK          0x1
+#define HINIC_SQ_CTRL_LEN_MASK                  0x3
+
+#define HINIC_SQ_CTRL_QUEUE_INFO_MSS_SHIFT      13
+
+#define HINIC_SQ_CTRL_QUEUE_INFO_MSS_MASK       0x3FFF
+
+#define HINIC_SQ_CTRL_SET(val, member)          \
+		(((u32)(val) & HINIC_SQ_CTRL_##member##_MASK) \
+		 << HINIC_SQ_CTRL_##member##_SHIFT)
+
+#define HINIC_SQ_CTRL_GET(val, member)          \
+		(((val) >> HINIC_SQ_CTRL_##member##_SHIFT) \
+		 & HINIC_SQ_CTRL_##member##_MASK)
+
+#define HINIC_SQ_TASK_INFO0_L2HDR_LEN_SHIFT     0
+#define HINIC_SQ_TASK_INFO0_L4_OFFLOAD_SHIFT    8
+#define HINIC_SQ_TASK_INFO0_INNER_L3TYPE_SHIFT  10
+#define HINIC_SQ_TASK_INFO0_VLAN_OFFLOAD_SHIFT  12
+#define HINIC_SQ_TASK_INFO0_PARSE_FLAG_SHIFT    13
+/* 1 bit reserved */
+#define HINIC_SQ_TASK_INFO0_TSO_FLAG_SHIFT      15
+#define HINIC_SQ_TASK_INFO0_VLAN_TAG_SHIFT      16
+
+#define HINIC_SQ_TASK_INFO0_L2HDR_LEN_MASK      0xFF
+#define HINIC_SQ_TASK_INFO0_L4_OFFLOAD_MASK     0x3
+#define HINIC_SQ_TASK_INFO0_INNER_L3TYPE_MASK   0x3
+#define HINIC_SQ_TASK_INFO0_VLAN_OFFLOAD_MASK   0x1
+#define HINIC_SQ_TASK_INFO0_PARSE_FLAG_MASK     0x1
+/* 1 bit reserved */
+#define HINIC_SQ_TASK_INFO0_TSO_FLAG_MASK       0x1
+#define HINIC_SQ_TASK_INFO0_VLAN_TAG_MASK       0xFFFF
+
+#define HINIC_SQ_TASK_INFO0_SET(val, member)    \
+		(((u32)(val) & HINIC_SQ_TASK_INFO0_##member##_MASK) <<  \
+		 HINIC_SQ_TASK_INFO0_##member##_SHIFT)
+
+/* 8 bits reserved */
+#define HINIC_SQ_TASK_INFO1_MEDIA_TYPE_SHIFT    8
+#define HINIC_SQ_TASK_INFO1_INNER_L4_LEN_SHIFT  16
+#define HINIC_SQ_TASK_INFO1_INNER_L3_LEN_SHIFT  24
+
+/* 8 bits reserved */
+#define HINIC_SQ_TASK_INFO1_MEDIA_TYPE_MASK     0xFF
+#define HINIC_SQ_TASK_INFO1_INNER_L4_LEN_MASK   0xFF
+#define HINIC_SQ_TASK_INFO1_INNER_L3_LEN_MASK   0xFF
+
+#define HINIC_SQ_TASK_INFO1_SET(val, member)    \
+		(((u32)(val) & HINIC_SQ_TASK_INFO1_##member##_MASK) <<  \
+		 HINIC_SQ_TASK_INFO1_##member##_SHIFT)
+
+#define HINIC_SQ_TASK_INFO2_TUNNEL_L4_LEN_SHIFT 0
+#define HINIC_SQ_TASK_INFO2_OUTER_L3_LEN_SHIFT  12
+#define HINIC_SQ_TASK_INFO2_TUNNEL_L4TYPE_SHIFT 19
+/* 1 bit reserved */
+#define HINIC_SQ_TASK_INFO2_OUTER_L3TYPE_SHIFT  22
+/* 8 bits reserved */
+
+#define HINIC_SQ_TASK_INFO2_TUNNEL_L4_LEN_MASK  0xFFF
+#define HINIC_SQ_TASK_INFO2_OUTER_L3_LEN_MASK   0x7F
+#define HINIC_SQ_TASK_INFO2_TUNNEL_L4TYPE_MASK  0x3
+/* 1 bit reserved */
+#define HINIC_SQ_TASK_INFO2_OUTER_L3TYPE_MASK   0x3
+/* 8 bits reserved */
+
+#define HINIC_SQ_TASK_INFO2_SET(val, member)    \
+		(((u32)(val) & HINIC_SQ_TASK_INFO2_##member##_MASK) <<  \
+		 HINIC_SQ_TASK_INFO2_##member##_SHIFT)
+
+/* 31 bits reserved */
+#define HINIC_SQ_TASK_INFO4_L2TYPE_SHIFT        31
+
+/* 31 bits reserved */
+#define HINIC_SQ_TASK_INFO4_L2TYPE_MASK         0x1
+
+#define HINIC_SQ_TASK_INFO4_SET(val, member)    \
+		(((u32)(val) & HINIC_SQ_TASK_INFO4_##member##_MASK) << \
+		 HINIC_SQ_TASK_INFO4_##member##_SHIFT)
+
+#define HINIC_RQ_CQE_STATUS_RXDONE_SHIFT        31
+
+#define HINIC_RQ_CQE_STATUS_RXDONE_MASK         0x1
+
+#define HINIC_RQ_CQE_STATUS_GET(val, member)    \
+		(((val) >> HINIC_RQ_CQE_STATUS_##member##_SHIFT) & \
+		 HINIC_RQ_CQE_STATUS_##member##_MASK)
+
+#define HINIC_RQ_CQE_STATUS_CLEAR(val, member)  \
+		((val) & (~(HINIC_RQ_CQE_STATUS_##member##_MASK << \
+		 HINIC_RQ_CQE_STATUS_##member##_SHIFT)))
+
+#define HINIC_RQ_CQE_SGE_LEN_SHIFT              16
+
+#define HINIC_RQ_CQE_SGE_LEN_MASK               0xFFFF
+
+#define HINIC_RQ_CQE_SGE_GET(val, member)       \
+		(((val) >> HINIC_RQ_CQE_SGE_##member##_SHIFT) & \
+		 HINIC_RQ_CQE_SGE_##member##_MASK)
+
+#define HINIC_RQ_CTRL_BUFDESC_SECT_LEN_SHIFT    0
+#define HINIC_RQ_CTRL_COMPLETE_FORMAT_SHIFT     15
+#define HINIC_RQ_CTRL_COMPLETE_LEN_SHIFT        27
+#define HINIC_RQ_CTRL_LEN_SHIFT                 29
+
+#define HINIC_RQ_CTRL_BUFDESC_SECT_LEN_MASK     0xFF
+#define HINIC_RQ_CTRL_COMPLETE_FORMAT_MASK      0x1
+#define HINIC_RQ_CTRL_COMPLETE_LEN_MASK         0x3
+#define HINIC_RQ_CTRL_LEN_MASK                  0x3
+
+#define HINIC_RQ_CTRL_SET(val, member)          \
+		(((u32)(val) & HINIC_RQ_CTRL_##member##_MASK) << \
+		 HINIC_RQ_CTRL_##member##_SHIFT)
+
+#define HINIC_SQ_WQE_SIZE(nr_sges)              \
+		(sizeof(struct hinic_sq_ctrl) + \
+		 sizeof(struct hinic_sq_task) + \
+		 (nr_sges) * sizeof(struct hinic_sq_bufdesc))
+
+#define HINIC_SCMD_DATA_LEN             16
+
+#define HINIC_MAX_SQ_BUFDESCS           17
+
+#define HINIC_SQ_WQE_MAX_SIZE           320
+#define HINIC_RQ_WQE_SIZE               32
+
+enum hinic_l4offload_type {
+	HINIC_L4_OFF_DISABLE            = 0,
+	HINIC_TCP_OFFLOAD_ENABLE        = 1,
+	HINIC_SCTP_OFFLOAD_ENABLE       = 2,
+	HINIC_UDP_OFFLOAD_ENABLE        = 3,
+};
+
+enum hinic_vlan_offload {
+	HINIC_VLAN_OFF_DISABLE = 0,
+	HINIC_VLAN_OFF_ENABLE  = 1,
+};
+
+enum hinic_pkt_parsed {
+	HINIC_PKT_NOT_PARSED = 0,
+	HINIC_PKT_PARSED     = 1,
+};
+
+enum hinic_outer_l3type {
+	HINIC_OUTER_L3TYPE_UNKNOWN              = 0,
+	HINIC_OUTER_L3TYPE_IPV6                 = 1,
+	HINIC_OUTER_L3TYPE_IPV4_NO_CHKSUM       = 2,
+	HINIC_OUTER_L3TYPE_IPV4_CHKSUM          = 3,
+};
+
+enum hinic_media_type {
+	HINIC_MEDIA_UNKNOWN = 0,
+};
+
+enum hinic_l2type {
+	HINIC_L2TYPE_ETH = 0,
+};
+
+enum hinc_tunnel_l4type {
+	HINIC_TUNNEL_L4TYPE_UNKNOWN = 0,
+};
+
+struct hinic_cmdq_header {
+	u32     header_info;
+	u32     saved_data;
+};
+
+struct hinic_status {
+	u32 status_info;
+};
+
+struct hinic_ctrl {
+	u32 ctrl_info;
+};
+
+struct hinic_sge_resp {
+	struct hinic_sge        sge;
+	u32                     rsvd;
+};
+
+struct hinic_cmdq_completion {
+	/* HW Format */
+	union {
+		struct hinic_sge_resp   sge_resp;
+		u64                     direct_resp;
+	};
+};
+
+struct hinic_scmd_bufdesc {
+	u32     buf_len;
+	u32     rsvd;
+	u8      data[HINIC_SCMD_DATA_LEN];
+};
+
+struct hinic_lcmd_bufdesc {
+	struct hinic_sge        sge;
+	u32                     rsvd1;
+	u64                     rsvd2;
+	u64                     rsvd3;
+};
+
+struct hinic_cmdq_wqe_scmd {
+	struct hinic_cmdq_header        header;
+	u64                             rsvd;
+	struct hinic_status             status;
+	struct hinic_ctrl               ctrl;
+	struct hinic_cmdq_completion    completion;
+	struct hinic_scmd_bufdesc       buf_desc;
+};
+
+struct hinic_cmdq_wqe_lcmd {
+	struct hinic_cmdq_header        header;
+	struct hinic_status             status;
+	struct hinic_ctrl               ctrl;
+	struct hinic_cmdq_completion    completion;
+	struct hinic_lcmd_bufdesc       buf_desc;
+};
+
+struct hinic_cmdq_direct_wqe {
+	struct hinic_cmdq_wqe_scmd      wqe_scmd;
+};
+
+struct hinic_cmdq_wqe {
+	/* HW Format */
+	union {
+		struct hinic_cmdq_direct_wqe    direct_wqe;
+		struct hinic_cmdq_wqe_lcmd      wqe_lcmd;
+	};
+};
+
+struct hinic_sq_ctrl {
+	u32     ctrl_info;
+	u32     queue_info;
+};
+
+struct hinic_sq_task {
+	u32     pkt_info0;
+	u32     pkt_info1;
+	u32     pkt_info2;
+	u32     ufo_v6_identify;
+	u32     pkt_info4;
+	u32     zero_pad;
+};
+
+struct hinic_sq_bufdesc {
+	struct hinic_sge sge;
+	u32     rsvd;
+};
+
+struct hinic_sq_wqe {
+	struct hinic_sq_ctrl            ctrl;
+	struct hinic_sq_task            task;
+	struct hinic_sq_bufdesc         buf_descs[HINIC_MAX_SQ_BUFDESCS];
+};
+
+struct hinic_rq_cqe {
+	u32     status;
+	u32     len;
+
+	u32     rsvd2;
+	u32     rsvd3;
+	u32     rsvd4;
+	u32     rsvd5;
+	u32     rsvd6;
+	u32     rsvd7;
+};
+
+struct hinic_rq_ctrl {
+	u32     ctrl_info;
+};
+
+struct hinic_rq_cqe_sect {
+	struct hinic_sge        sge;
+	u32                     rsvd;
+};
+
+struct hinic_rq_bufdesc {
+	u32     hi_addr;
+	u32     lo_addr;
+};
+
+struct hinic_rq_wqe {
+	struct hinic_rq_ctrl            ctrl;
+	u32                             rsvd;
+	struct hinic_rq_cqe_sect        cqe_sect;
+	struct hinic_rq_bufdesc         buf_desc;
+};
+
+struct hinic_hw_wqe {
+	/* HW Format */
+	union {
+		struct hinic_cmdq_wqe   cmdq_wqe;
+		struct hinic_sq_wqe     sq_wqe;
+		struct hinic_rq_wqe     rq_wqe;
+	};
+};
+
+#endif
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_main.c b/drivers/net/ethernet/huawei/hinic/hinic_main.c
new file mode 100644
index 000000000000..eb53bd93065e
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_main.c
@@ -0,0 +1,1112 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/pci.h>
+#include <linux/device.h>
+#include <linux/errno.h>
+#include <linux/types.h>
+#include <linux/etherdevice.h>
+#include <linux/netdevice.h>
+#include <linux/slab.h>
+#include <linux/if_vlan.h>
+#include <linux/semaphore.h>
+#include <linux/workqueue.h>
+#include <net/ip.h>
+#include <linux/bitops.h>
+#include <linux/bitmap.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+
+#include "hinic_hw_qp.h"
+#include "hinic_hw_dev.h"
+#include "hinic_port.h"
+#include "hinic_tx.h"
+#include "hinic_rx.h"
+#include "hinic_dev.h"
+
+MODULE_AUTHOR("Huawei Technologies CO., Ltd");
+MODULE_DESCRIPTION("Huawei Intelligent NIC driver");
+MODULE_LICENSE("GPL");
+
+static unsigned int tx_weight = 64;
+module_param(tx_weight, uint, 0644);
+MODULE_PARM_DESC(tx_weight, "Number Tx packets for NAPI budget (default=64)");
+
+static unsigned int rx_weight = 64;
+module_param(rx_weight, uint, 0644);
+MODULE_PARM_DESC(rx_weight, "Number Rx packets for NAPI budget (default=64)");
+
+#define PCI_DEVICE_ID_HI1822_PF         0x1822
+
+#define HINIC_WQ_NAME                   "hinic_dev"
+
+#define MSG_ENABLE_DEFAULT              (NETIF_MSG_DRV | NETIF_MSG_PROBE | \
+					 NETIF_MSG_IFUP |                  \
+					 NETIF_MSG_TX_ERR | NETIF_MSG_RX_ERR)
+
+#define VLAN_BITMAP_SIZE(nic_dev)       (ALIGN(VLAN_N_VID, 8) / 8)
+
+#define work_to_rx_mode_work(work)      \
+		container_of(work, struct hinic_rx_mode_work, work)
+
+#define rx_mode_work_to_nic_dev(rx_mode_work) \
+		container_of(rx_mode_work, struct hinic_dev, rx_mode_work)
+
+static int change_mac_addr(struct net_device *netdev, const u8 *addr);
+
+static void set_link_speed(struct ethtool_link_ksettings *link_ksettings,
+			   enum hinic_speed speed)
+{
+	switch (speed) {
+	case HINIC_SPEED_10MB_LINK:
+		link_ksettings->base.speed = SPEED_10;
+		break;
+
+	case HINIC_SPEED_100MB_LINK:
+		link_ksettings->base.speed = SPEED_100;
+		break;
+
+	case HINIC_SPEED_1000MB_LINK:
+		link_ksettings->base.speed = SPEED_1000;
+		break;
+
+	case HINIC_SPEED_10GB_LINK:
+		link_ksettings->base.speed = SPEED_10000;
+		break;
+
+	case HINIC_SPEED_25GB_LINK:
+		link_ksettings->base.speed = SPEED_25000;
+		break;
+
+	case HINIC_SPEED_40GB_LINK:
+		link_ksettings->base.speed = SPEED_40000;
+		break;
+
+	case HINIC_SPEED_100GB_LINK:
+		link_ksettings->base.speed = SPEED_100000;
+		break;
+
+	default:
+		link_ksettings->base.speed = SPEED_UNKNOWN;
+		break;
+	}
+}
+
+static int hinic_get_link_ksettings(struct net_device *netdev,
+				    struct ethtool_link_ksettings
+				    *link_ksettings)
+{
+	struct hinic_dev *nic_dev = netdev_priv(netdev);
+	enum hinic_port_link_state link_state;
+	struct hinic_port_cap port_cap;
+	int err;
+
+	ethtool_link_ksettings_zero_link_mode(link_ksettings, advertising);
+	ethtool_link_ksettings_add_link_mode(link_ksettings, supported,
+					     Autoneg);
+
+	link_ksettings->base.speed   = SPEED_UNKNOWN;
+	link_ksettings->base.autoneg = AUTONEG_DISABLE;
+	link_ksettings->base.duplex  = DUPLEX_UNKNOWN;
+
+	err = hinic_port_get_cap(nic_dev, &port_cap);
+	if (err) {
+		netif_err(nic_dev, drv, netdev,
+			  "Failed to get port capabilities\n");
+		return err;
+	}
+
+	err = hinic_port_link_state(nic_dev, &link_state);
+	if (err) {
+		netif_err(nic_dev, drv, netdev,
+			  "Failed to get port link state\n");
+		return err;
+	}
+
+	if (link_state != HINIC_LINK_STATE_UP) {
+		netif_info(nic_dev, drv, netdev, "No link\n");
+		return err;
+	}
+
+	set_link_speed(link_ksettings, port_cap.speed);
+
+	if (!!(port_cap.autoneg_cap & HINIC_AUTONEG_SUPPORTED))
+		ethtool_link_ksettings_add_link_mode(link_ksettings,
+						     advertising, Autoneg);
+
+	if (port_cap.autoneg_state == HINIC_AUTONEG_ACTIVE)
+		link_ksettings->base.autoneg = AUTONEG_ENABLE;
+
+	link_ksettings->base.duplex = (port_cap.duplex == HINIC_DUPLEX_FULL) ?
+				       DUPLEX_FULL : DUPLEX_HALF;
+	return 0;
+}
+
+static void hinic_get_drvinfo(struct net_device *netdev,
+			      struct ethtool_drvinfo *info)
+{
+	struct hinic_dev *nic_dev = netdev_priv(netdev);
+	struct hinic_hwdev *hwdev = nic_dev->hwdev;
+	struct hinic_hwif *hwif = hwdev->hwif;
+
+	strlcpy(info->driver, HINIC_DRV_NAME, sizeof(info->driver));
+	strlcpy(info->bus_info, pci_name(hwif->pdev), sizeof(info->bus_info));
+}
+
+static void hinic_get_ringparam(struct net_device *netdev,
+				struct ethtool_ringparam *ring)
+{
+	ring->rx_max_pending = HINIC_RQ_DEPTH;
+	ring->tx_max_pending = HINIC_SQ_DEPTH;
+	ring->rx_pending = HINIC_RQ_DEPTH;
+	ring->tx_pending = HINIC_SQ_DEPTH;
+}
+
+static void hinic_get_channels(struct net_device *netdev,
+			       struct ethtool_channels *channels)
+{
+	struct hinic_dev *nic_dev = netdev_priv(netdev);
+	struct hinic_hwdev *hwdev = nic_dev->hwdev;
+
+	channels->max_rx = hwdev->nic_cap.max_qps;
+	channels->max_tx = hwdev->nic_cap.max_qps;
+	channels->max_other    = 0;
+	channels->max_combined = 0;
+	channels->rx_count = hinic_hwdev_num_qps(hwdev);
+	channels->tx_count = hinic_hwdev_num_qps(hwdev);
+	channels->other_count    = 0;
+	channels->combined_count = 0;
+}
+
+static const struct ethtool_ops hinic_ethtool_ops = {
+	.get_link_ksettings = hinic_get_link_ksettings,
+	.get_drvinfo = hinic_get_drvinfo,
+	.get_link = ethtool_op_get_link,
+	.get_ringparam = hinic_get_ringparam,
+	.get_channels = hinic_get_channels,
+};
+
+static void update_rx_stats(struct hinic_dev *nic_dev, struct hinic_rxq *rxq)
+{
+	struct hinic_rxq_stats *nic_rx_stats = &nic_dev->rx_stats;
+	struct hinic_rxq_stats rx_stats;
+
+	u64_stats_init(&rx_stats.syncp);
+
+	hinic_rxq_get_stats(rxq, &rx_stats);
+
+	u64_stats_update_begin(&nic_rx_stats->syncp);
+	nic_rx_stats->bytes += rx_stats.bytes;
+	nic_rx_stats->pkts  += rx_stats.pkts;
+	u64_stats_update_end(&nic_rx_stats->syncp);
+
+	hinic_rxq_clean_stats(rxq);
+}
+
+static void update_tx_stats(struct hinic_dev *nic_dev, struct hinic_txq *txq)
+{
+	struct hinic_txq_stats *nic_tx_stats = &nic_dev->tx_stats;
+	struct hinic_txq_stats tx_stats;
+
+	u64_stats_init(&tx_stats.syncp);
+
+	hinic_txq_get_stats(txq, &tx_stats);
+
+	u64_stats_update_begin(&nic_tx_stats->syncp);
+	nic_tx_stats->bytes += tx_stats.bytes;
+	nic_tx_stats->pkts += tx_stats.pkts;
+	nic_tx_stats->tx_busy += tx_stats.tx_busy;
+	nic_tx_stats->tx_wake += tx_stats.tx_wake;
+	nic_tx_stats->tx_dropped += tx_stats.tx_dropped;
+	u64_stats_update_end(&nic_tx_stats->syncp);
+
+	hinic_txq_clean_stats(txq);
+}
+
+static void update_nic_stats(struct hinic_dev *nic_dev)
+{
+	int i, num_qps = hinic_hwdev_num_qps(nic_dev->hwdev);
+
+	for (i = 0; i < num_qps; i++)
+		update_rx_stats(nic_dev, &nic_dev->rxqs[i]);
+
+	for (i = 0; i < num_qps; i++)
+		update_tx_stats(nic_dev, &nic_dev->txqs[i]);
+}
+
+/**
+ * create_txqs - Create the Logical Tx Queues of specific NIC device
+ * @nic_dev: the specific NIC device
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int create_txqs(struct hinic_dev *nic_dev)
+{
+	int err, i, j, num_txqs = hinic_hwdev_num_qps(nic_dev->hwdev);
+	struct net_device *netdev = nic_dev->netdev;
+	size_t txq_size;
+
+	if (nic_dev->txqs)
+		return -EINVAL;
+
+	txq_size = num_txqs * sizeof(*nic_dev->txqs);
+	nic_dev->txqs = devm_kzalloc(&netdev->dev, txq_size, GFP_KERNEL);
+	if (!nic_dev->txqs)
+		return -ENOMEM;
+
+	for (i = 0; i < num_txqs; i++) {
+		struct hinic_sq *sq = hinic_hwdev_get_sq(nic_dev->hwdev, i);
+
+		err = hinic_init_txq(&nic_dev->txqs[i], sq, netdev);
+		if (err) {
+			netif_err(nic_dev, drv, netdev,
+				  "Failed to init Txq\n");
+			goto err_init_txq;
+		}
+	}
+
+	return 0;
+
+err_init_txq:
+	for (j = 0; j < i; j++)
+		hinic_clean_txq(&nic_dev->txqs[j]);
+
+	devm_kfree(&netdev->dev, nic_dev->txqs);
+	return err;
+}
+
+/**
+ * free_txqs - Free the Logical Tx Queues of specific NIC device
+ * @nic_dev: the specific NIC device
+ **/
+static void free_txqs(struct hinic_dev *nic_dev)
+{
+	int i, num_txqs = hinic_hwdev_num_qps(nic_dev->hwdev);
+	struct net_device *netdev = nic_dev->netdev;
+
+	if (!nic_dev->txqs)
+		return;
+
+	for (i = 0; i < num_txqs; i++)
+		hinic_clean_txq(&nic_dev->txqs[i]);
+
+	devm_kfree(&netdev->dev, nic_dev->txqs);
+	nic_dev->txqs = NULL;
+}
+
+/**
+ * create_txqs - Create the Logical Rx Queues of specific NIC device
+ * @nic_dev: the specific NIC device
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int create_rxqs(struct hinic_dev *nic_dev)
+{
+	int err, i, j, num_rxqs = hinic_hwdev_num_qps(nic_dev->hwdev);
+	struct net_device *netdev = nic_dev->netdev;
+	size_t rxq_size;
+
+	if (nic_dev->rxqs)
+		return -EINVAL;
+
+	rxq_size = num_rxqs * sizeof(*nic_dev->rxqs);
+	nic_dev->rxqs = devm_kzalloc(&netdev->dev, rxq_size, GFP_KERNEL);
+	if (!nic_dev->rxqs)
+		return -ENOMEM;
+
+	for (i = 0; i < num_rxqs; i++) {
+		struct hinic_rq *rq = hinic_hwdev_get_rq(nic_dev->hwdev, i);
+
+		err = hinic_init_rxq(&nic_dev->rxqs[i], rq, netdev);
+		if (err) {
+			netif_err(nic_dev, drv, netdev,
+				  "Failed to init rxq\n");
+			goto err_init_rxq;
+		}
+	}
+
+	return 0;
+
+err_init_rxq:
+	for (j = 0; j < i; j++)
+		hinic_clean_rxq(&nic_dev->rxqs[j]);
+
+	devm_kfree(&netdev->dev, nic_dev->rxqs);
+	return err;
+}
+
+/**
+ * free_txqs - Free the Logical Rx Queues of specific NIC device
+ * @nic_dev: the specific NIC device
+ **/
+static void free_rxqs(struct hinic_dev *nic_dev)
+{
+	int i, num_rxqs = hinic_hwdev_num_qps(nic_dev->hwdev);
+	struct net_device *netdev = nic_dev->netdev;
+
+	if (!nic_dev->rxqs)
+		return;
+
+	for (i = 0; i < num_rxqs; i++)
+		hinic_clean_rxq(&nic_dev->rxqs[i]);
+
+	devm_kfree(&netdev->dev, nic_dev->rxqs);
+	nic_dev->rxqs = NULL;
+}
+
+static int hinic_open(struct net_device *netdev)
+{
+	struct hinic_dev *nic_dev = netdev_priv(netdev);
+	enum hinic_port_link_state link_state;
+	int err, ret, num_qps;
+
+	if (!(nic_dev->flags & HINIC_INTF_UP)) {
+		err = hinic_hwdev_ifup(nic_dev->hwdev);
+		if (err) {
+			netif_err(nic_dev, drv, netdev,
+				  "Failed - HW interface up\n");
+			return err;
+		}
+	}
+
+	err = create_txqs(nic_dev);
+	if (err) {
+		netif_err(nic_dev, drv, netdev,
+			  "Failed to create Tx queues\n");
+		goto err_create_txqs;
+	}
+
+	err = create_rxqs(nic_dev);
+	if (err) {
+		netif_err(nic_dev, drv, netdev,
+			  "Failed to create Rx queues\n");
+		goto err_create_rxqs;
+	}
+
+	num_qps = hinic_hwdev_num_qps(nic_dev->hwdev);
+	netif_set_real_num_tx_queues(netdev, num_qps);
+	netif_set_real_num_rx_queues(netdev, num_qps);
+
+	err = hinic_port_set_state(nic_dev, HINIC_PORT_ENABLE);
+	if (err) {
+		netif_err(nic_dev, drv, netdev,
+			  "Failed to set port state\n");
+		goto err_port_state;
+	}
+
+	err = hinic_port_set_func_state(nic_dev, HINIC_FUNC_PORT_ENABLE);
+	if (err) {
+		netif_err(nic_dev, drv, netdev,
+			  "Failed to set func port state\n");
+		goto err_func_port_state;
+	}
+
+	/* Wait up to 3 sec between port enable to link state */
+	msleep(3000);
+
+	down(&nic_dev->mgmt_lock);
+
+	err = hinic_port_link_state(nic_dev, &link_state);
+	if (err) {
+		netif_err(nic_dev, drv, netdev, "Failed to get link state\n");
+		goto err_port_link;
+	}
+
+	if (link_state == HINIC_LINK_STATE_UP)
+		nic_dev->flags |= HINIC_LINK_UP;
+
+	nic_dev->flags |= HINIC_INTF_UP;
+
+	if ((nic_dev->flags & (HINIC_LINK_UP | HINIC_INTF_UP)) ==
+	    (HINIC_LINK_UP | HINIC_INTF_UP)) {
+		netif_info(nic_dev, drv, netdev, "link + intf UP\n");
+		netif_carrier_on(netdev);
+		netif_tx_wake_all_queues(netdev);
+	}
+
+	up(&nic_dev->mgmt_lock);
+
+	netif_info(nic_dev, drv, netdev, "HINIC_INTF is UP\n");
+	return 0;
+
+err_port_link:
+	up(&nic_dev->mgmt_lock);
+	ret = hinic_port_set_func_state(nic_dev, HINIC_FUNC_PORT_DISABLE);
+	if (ret)
+		netif_warn(nic_dev, drv, netdev,
+			   "Failed to revert func port state\n");
+
+err_func_port_state:
+	ret = hinic_port_set_state(nic_dev, HINIC_PORT_DISABLE);
+	if (ret)
+		netif_warn(nic_dev, drv, netdev,
+			   "Failed to revert port state\n");
+
+err_port_state:
+	free_rxqs(nic_dev);
+
+err_create_rxqs:
+	free_txqs(nic_dev);
+
+err_create_txqs:
+	if (!(nic_dev->flags & HINIC_INTF_UP))
+		hinic_hwdev_ifdown(nic_dev->hwdev);
+	return err;
+}
+
+static int hinic_close(struct net_device *netdev)
+{
+	struct hinic_dev *nic_dev = netdev_priv(netdev);
+	unsigned int flags;
+	int err;
+
+	down(&nic_dev->mgmt_lock);
+
+	flags = nic_dev->flags;
+	nic_dev->flags &= ~HINIC_INTF_UP;
+
+	netif_carrier_off(netdev);
+	netif_tx_disable(netdev);
+
+	update_nic_stats(nic_dev);
+
+	up(&nic_dev->mgmt_lock);
+
+	err = hinic_port_set_func_state(nic_dev, HINIC_FUNC_PORT_DISABLE);
+	if (err) {
+		netif_err(nic_dev, drv, netdev,
+			  "Failed to set func port state\n");
+		nic_dev->flags |= (flags & HINIC_INTF_UP);
+		return err;
+	}
+
+	err = hinic_port_set_state(nic_dev, HINIC_PORT_DISABLE);
+	if (err) {
+		netif_err(nic_dev, drv, netdev, "Failed to set port state\n");
+		nic_dev->flags |= (flags & HINIC_INTF_UP);
+		return err;
+	}
+
+	free_rxqs(nic_dev);
+	free_txqs(nic_dev);
+
+	if (flags & HINIC_INTF_UP)
+		hinic_hwdev_ifdown(nic_dev->hwdev);
+
+	netif_info(nic_dev, drv, netdev, "HINIC_INTF is DOWN\n");
+	return 0;
+}
+
+static int hinic_change_mtu(struct net_device *netdev, int new_mtu)
+{
+	struct hinic_dev *nic_dev = netdev_priv(netdev);
+	int err;
+
+	netif_info(nic_dev, drv, netdev, "set_mtu = %d\n", new_mtu);
+
+	err = hinic_port_set_mtu(nic_dev, new_mtu);
+	if (err)
+		netif_err(nic_dev, drv, netdev, "Failed to set port mtu\n");
+	else
+		netdev->mtu = new_mtu;
+
+	return err;
+}
+
+/**
+ * change_mac_addr - change the main mac address of network device
+ * @netdev: network device
+ * @addr: mac address to set
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int change_mac_addr(struct net_device *netdev, const u8 *addr)
+{
+	struct hinic_dev *nic_dev = netdev_priv(netdev);
+	u16 vid = 0;
+	int err;
+
+	if (!is_valid_ether_addr(addr))
+		return -EADDRNOTAVAIL;
+
+	netif_info(nic_dev, drv, netdev, "change mac addr = %02x %02x %02x %02x %02x %02x\n",
+		   addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]);
+
+	down(&nic_dev->mgmt_lock);
+
+	do {
+		err = hinic_port_del_mac(nic_dev, netdev->dev_addr, vid);
+		if (err) {
+			netif_err(nic_dev, drv, netdev,
+				  "Failed to delete mac\n");
+			break;
+		}
+
+		err = hinic_port_add_mac(nic_dev, addr, vid);
+		if (err) {
+			netif_err(nic_dev, drv, netdev, "Failed to add mac\n");
+			break;
+		}
+
+		vid = find_next_bit(nic_dev->vlan_bitmap, VLAN_N_VID, vid + 1);
+	} while (vid != VLAN_N_VID);
+
+	up(&nic_dev->mgmt_lock);
+	return err;
+}
+
+static int hinic_set_mac_addr(struct net_device *netdev, void *addr)
+{
+	unsigned char new_mac[ETH_ALEN];
+	struct sockaddr *saddr = addr;
+	int err;
+
+	memcpy(new_mac, saddr->sa_data, ETH_ALEN);
+
+	err = change_mac_addr(netdev, new_mac);
+	if (!err)
+		memcpy(netdev->dev_addr, new_mac, ETH_ALEN);
+
+	return err;
+}
+
+/**
+ * add_mac_addr - add mac address to network device
+ * @netdev: network device
+ * @addr: mac address to add
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int add_mac_addr(struct net_device *netdev, const u8 *addr)
+{
+	struct hinic_dev *nic_dev = netdev_priv(netdev);
+	u16 vid = 0;
+	int err;
+
+	if (!is_valid_ether_addr(addr))
+		return -EADDRNOTAVAIL;
+
+	netif_info(nic_dev, drv, netdev, "set mac addr = %02x %02x %02x %02x %02x %02x\n",
+		   addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]);
+
+	down(&nic_dev->mgmt_lock);
+
+	do {
+		err = hinic_port_add_mac(nic_dev, addr, vid);
+		if (err) {
+			netif_err(nic_dev, drv, netdev, "Failed to add mac\n");
+			break;
+		}
+
+		vid = find_next_bit(nic_dev->vlan_bitmap, VLAN_N_VID, vid + 1);
+	} while (vid != VLAN_N_VID);
+
+	up(&nic_dev->mgmt_lock);
+	return err;
+}
+
+/**
+ * remove_mac_addr - remove mac address from network device
+ * @netdev: network device
+ * @addr: mac address to remove
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int remove_mac_addr(struct net_device *netdev, const u8 *addr)
+{
+	struct hinic_dev *nic_dev = netdev_priv(netdev);
+	u16 vid = 0;
+	int err;
+
+	if (!is_valid_ether_addr(addr))
+		return -EADDRNOTAVAIL;
+
+	netif_info(nic_dev, drv, netdev, "remove mac addr = %02x %02x %02x %02x %02x %02x\n",
+		   addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]);
+
+	down(&nic_dev->mgmt_lock);
+
+	do {
+		err = hinic_port_del_mac(nic_dev, addr, vid);
+		if (err) {
+			netif_err(nic_dev, drv, netdev,
+				  "Failed to delete mac\n");
+			break;
+		}
+
+		vid = find_next_bit(nic_dev->vlan_bitmap, VLAN_N_VID, vid + 1);
+	} while (vid != VLAN_N_VID);
+
+	up(&nic_dev->mgmt_lock);
+	return err;
+}
+
+static int hinic_vlan_rx_add_vid(struct net_device *netdev,
+				 __always_unused __be16 proto, u16 vid)
+{
+	struct hinic_dev *nic_dev = netdev_priv(netdev);
+	int ret, err;
+
+	netif_info(nic_dev, drv, netdev, "add vid = %d\n", vid);
+
+	down(&nic_dev->mgmt_lock);
+
+	err = hinic_port_add_vlan(nic_dev, vid);
+	if (err) {
+		netif_err(nic_dev, drv, netdev, "Failed to add vlan\n");
+		goto err_vlan_add;
+	}
+
+	err = hinic_port_add_mac(nic_dev, netdev->dev_addr, vid);
+	if (err) {
+		netif_err(nic_dev, drv, netdev, "Failed to set mac\n");
+		goto err_add_mac;
+	}
+
+	bitmap_set(nic_dev->vlan_bitmap, vid, 1);
+
+	up(&nic_dev->mgmt_lock);
+	return 0;
+
+err_add_mac:
+	ret = hinic_port_del_vlan(nic_dev, vid);
+	if (ret)
+		netif_err(nic_dev, drv, netdev,
+			  "Failed to revert by removing vlan\n");
+
+err_vlan_add:
+	up(&nic_dev->mgmt_lock);
+	return err;
+}
+
+static int hinic_vlan_rx_kill_vid(struct net_device *netdev,
+				  __always_unused __be16 proto, u16 vid)
+{
+	struct hinic_dev *nic_dev = netdev_priv(netdev);
+	int err;
+
+	netif_info(nic_dev, drv, netdev, "remove vid = %d\n", vid);
+
+	down(&nic_dev->mgmt_lock);
+
+	err = hinic_port_del_vlan(nic_dev, vid);
+	if (err) {
+		netif_err(nic_dev, drv, netdev, "Failed to delete vlan\n");
+		goto err_del_vlan;
+	}
+
+	bitmap_clear(nic_dev->vlan_bitmap, vid, 1);
+
+	up(&nic_dev->mgmt_lock);
+	return 0;
+
+err_del_vlan:
+	up(&nic_dev->mgmt_lock);
+	return err;
+}
+
+static void set_rx_mode(struct work_struct *work)
+{
+	struct hinic_rx_mode_work *rx_mode_work = work_to_rx_mode_work(work);
+	struct hinic_dev *nic_dev = rx_mode_work_to_nic_dev(rx_mode_work);
+
+	netif_info(nic_dev, drv, nic_dev->netdev, "set rx mode work\n");
+
+	hinic_port_set_rx_mode(nic_dev, rx_mode_work->rx_mode);
+
+	__dev_uc_sync(nic_dev->netdev, add_mac_addr, remove_mac_addr);
+	__dev_mc_sync(nic_dev->netdev, add_mac_addr, remove_mac_addr);
+}
+
+static void hinic_set_rx_mode(struct net_device *netdev)
+{
+	struct hinic_dev *nic_dev = netdev_priv(netdev);
+	struct hinic_rx_mode_work *rx_mode_work;
+	u32 rx_mode;
+
+	rx_mode_work = &nic_dev->rx_mode_work;
+
+	rx_mode = HINIC_RX_MODE_UC |
+		  HINIC_RX_MODE_MC |
+		  HINIC_RX_MODE_BC;
+
+	if (netdev->flags & IFF_PROMISC)
+		rx_mode |= HINIC_RX_MODE_PROMISC;
+	else if (netdev->flags & IFF_ALLMULTI)
+		rx_mode |= HINIC_RX_MODE_MC_ALL;
+
+	rx_mode_work->rx_mode = rx_mode;
+
+	queue_work(nic_dev->workq, &rx_mode_work->work);
+}
+
+static void hinic_tx_timeout(struct net_device *netdev)
+{
+	struct hinic_dev *nic_dev = netdev_priv(netdev);
+
+	netif_err(nic_dev, drv, netdev, "Tx timeout\n");
+}
+
+static void hinic_get_stats64(struct net_device *netdev,
+			      struct rtnl_link_stats64 *stats)
+{
+	struct hinic_dev *nic_dev = netdev_priv(netdev);
+	struct hinic_rxq_stats *nic_rx_stats;
+	struct hinic_txq_stats *nic_tx_stats;
+
+	nic_rx_stats = &nic_dev->rx_stats;
+	nic_tx_stats = &nic_dev->tx_stats;
+
+	down(&nic_dev->mgmt_lock);
+
+	if (nic_dev->flags & HINIC_INTF_UP)
+		update_nic_stats(nic_dev);
+
+	up(&nic_dev->mgmt_lock);
+
+	stats->rx_bytes   = nic_rx_stats->bytes;
+	stats->rx_packets = nic_rx_stats->pkts;
+
+	stats->tx_bytes   = nic_tx_stats->bytes;
+	stats->tx_packets = nic_tx_stats->pkts;
+	stats->tx_errors  = nic_tx_stats->tx_dropped;
+}
+
+#ifdef CONFIG_NET_POLL_CONTROLLER
+static void hinic_netpoll(struct net_device *netdev)
+{
+	struct hinic_dev *nic_dev = netdev_priv(netdev);
+	int i, num_qps;
+
+	num_qps = hinic_hwdev_num_qps(nic_dev->hwdev);
+	for (i = 0; i < num_qps; i++) {
+		struct hinic_txq *txq = &nic_dev->txqs[i];
+		struct hinic_rxq *rxq = &nic_dev->rxqs[i];
+
+		napi_schedule(&txq->napi);
+		napi_schedule(&rxq->napi);
+	}
+}
+#endif
+
+static const struct net_device_ops hinic_netdev_ops = {
+	.ndo_open = hinic_open,
+	.ndo_stop = hinic_close,
+	.ndo_change_mtu = hinic_change_mtu,
+	.ndo_set_mac_address = hinic_set_mac_addr,
+	.ndo_validate_addr = eth_validate_addr,
+	.ndo_vlan_rx_add_vid = hinic_vlan_rx_add_vid,
+	.ndo_vlan_rx_kill_vid = hinic_vlan_rx_kill_vid,
+	.ndo_set_rx_mode = hinic_set_rx_mode,
+	.ndo_start_xmit = hinic_xmit_frame,
+	.ndo_tx_timeout = hinic_tx_timeout,
+	.ndo_get_stats64 = hinic_get_stats64,
+#ifdef CONFIG_NET_POLL_CONTROLLER
+	.ndo_poll_controller = hinic_netpoll,
+#endif
+};
+
+static void netdev_features_init(struct net_device *netdev)
+{
+	netdev->hw_features = NETIF_F_SG | NETIF_F_HIGHDMA;
+
+	netdev->vlan_features = netdev->hw_features;
+
+	netdev->features = netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
+}
+
+/**
+ * link_status_event_handler - link event handler
+ * @handle: nic device for the handler
+ * @buf_in: input buffer
+ * @in_size: input size
+ * @buf_in: output buffer
+ * @out_size: returned output size
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static void link_status_event_handler(void *handle, void *buf_in, u16 in_size,
+				      void *buf_out, u16 *out_size)
+{
+	struct hinic_port_link_status *link_status, *ret_link_status;
+	struct hinic_dev *nic_dev = handle;
+
+	link_status = buf_in;
+
+	if (link_status->link == HINIC_LINK_STATE_UP) {
+		down(&nic_dev->mgmt_lock);
+
+		nic_dev->flags |= HINIC_LINK_UP;
+
+		if ((nic_dev->flags & (HINIC_LINK_UP | HINIC_INTF_UP)) ==
+		    (HINIC_LINK_UP | HINIC_INTF_UP)) {
+			netif_carrier_on(nic_dev->netdev);
+			netif_tx_wake_all_queues(nic_dev->netdev);
+		}
+
+		up(&nic_dev->mgmt_lock);
+
+		netif_info(nic_dev, drv, nic_dev->netdev, "HINIC_Link is UP\n");
+	} else {
+		down(&nic_dev->mgmt_lock);
+
+		nic_dev->flags &= ~HINIC_LINK_UP;
+
+		netif_carrier_off(nic_dev->netdev);
+		netif_tx_disable(nic_dev->netdev);
+
+		up(&nic_dev->mgmt_lock);
+
+		netif_info(nic_dev, drv, nic_dev->netdev, "HINIC_Link is DOWN\n");
+	}
+
+	ret_link_status = buf_out;
+	ret_link_status->status = 0;
+
+	*out_size = sizeof(*ret_link_status);
+}
+
+/**
+ * nic_dev_init - Initialize the NIC device
+ * @pdev: the NIC pci device
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int nic_dev_init(struct pci_dev *pdev)
+{
+	struct hinic_rx_mode_work *rx_mode_work;
+	struct hinic_txq_stats *tx_stats;
+	struct hinic_rxq_stats *rx_stats;
+	struct hinic_dev *nic_dev;
+	struct net_device *netdev;
+	struct hinic_hwdev *hwdev;
+	int err, num_qps;
+
+	hwdev = hinic_init_hwdev(pdev);
+	if (IS_ERR(hwdev)) {
+		dev_err(&pdev->dev, "Failed to initialize HW device\n");
+		return PTR_ERR(hwdev);
+	}
+
+	num_qps = hinic_hwdev_num_qps(hwdev);
+	if (num_qps <= 0) {
+		dev_err(&pdev->dev, "Invalid number of QPS\n");
+		err = -EINVAL;
+		goto err_num_qps;
+	}
+
+	netdev = alloc_etherdev_mq(sizeof(*nic_dev), num_qps);
+	if (!netdev) {
+		dev_err(&pdev->dev, "Failed to allocate Ethernet device\n");
+		err = -ENOMEM;
+		goto err_alloc_etherdev;
+	}
+
+	netdev->netdev_ops = &hinic_netdev_ops;
+	netdev->ethtool_ops = &hinic_ethtool_ops;
+	netdev->max_mtu = ETH_MAX_MTU;
+
+	nic_dev = netdev_priv(netdev);
+	nic_dev->netdev = netdev;
+	nic_dev->hwdev  = hwdev;
+	nic_dev->msg_enable = MSG_ENABLE_DEFAULT;
+	nic_dev->flags = 0;
+	nic_dev->txqs = NULL;
+	nic_dev->rxqs = NULL;
+	nic_dev->tx_weight = tx_weight;
+	nic_dev->rx_weight = rx_weight;
+
+	sema_init(&nic_dev->mgmt_lock, 1);
+
+	tx_stats = &nic_dev->tx_stats;
+	rx_stats = &nic_dev->rx_stats;
+
+	u64_stats_init(&tx_stats->syncp);
+	u64_stats_init(&rx_stats->syncp);
+
+	nic_dev->vlan_bitmap = devm_kzalloc(&pdev->dev,
+					    VLAN_BITMAP_SIZE(nic_dev),
+					    GFP_KERNEL);
+	if (!nic_dev->vlan_bitmap) {
+		err = -ENOMEM;
+		goto err_vlan_bitmap;
+	}
+
+	nic_dev->workq = create_singlethread_workqueue(HINIC_WQ_NAME);
+	if (!nic_dev->workq) {
+		err = -ENOMEM;
+		goto err_workq;
+	}
+
+	pci_set_drvdata(pdev, netdev);
+
+	err = hinic_port_get_mac(nic_dev, netdev->dev_addr);
+	if (err)
+		dev_warn(&pdev->dev, "Failed to get mac address\n");
+
+	err = hinic_port_add_mac(nic_dev, netdev->dev_addr, 0);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to add mac\n");
+		goto err_add_mac;
+	}
+
+	err = hinic_port_set_mtu(nic_dev, netdev->mtu);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to set mtu\n");
+		goto err_set_mtu;
+	}
+
+	rx_mode_work = &nic_dev->rx_mode_work;
+	INIT_WORK(&rx_mode_work->work, set_rx_mode);
+
+	netdev_features_init(netdev);
+
+	netif_carrier_off(netdev);
+
+	hinic_hwdev_cb_register(nic_dev->hwdev, HINIC_MGMT_MSG_CMD_LINK_STATUS,
+				nic_dev, link_status_event_handler);
+
+	err = register_netdev(netdev);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to register netdev\n");
+		goto err_reg_netdev;
+	}
+
+	return 0;
+
+err_reg_netdev:
+	hinic_hwdev_cb_unregister(nic_dev->hwdev,
+				  HINIC_MGMT_MSG_CMD_LINK_STATUS);
+	cancel_work_sync(&rx_mode_work->work);
+
+err_set_mtu:
+err_add_mac:
+	pci_set_drvdata(pdev, NULL);
+	destroy_workqueue(nic_dev->workq);
+
+err_workq:
+err_vlan_bitmap:
+	free_netdev(netdev);
+
+err_alloc_etherdev:
+err_num_qps:
+	hinic_free_hwdev(hwdev);
+	return err;
+}
+
+static int hinic_probe(struct pci_dev *pdev,
+		       const struct pci_device_id *id)
+{
+	int err = pci_enable_device(pdev);
+
+	if (err) {
+		dev_err(&pdev->dev, "Failed to enable PCI device\n");
+		return err;
+	}
+
+	err = pci_request_regions(pdev, HINIC_DRV_NAME);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to request PCI regions\n");
+		goto err_pci_regions;
+	}
+
+	pci_set_master(pdev);
+
+	err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
+	if (err) {
+		dev_warn(&pdev->dev, "Couldn't set 64-bit DMA mask\n");
+		err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
+		if (err) {
+			dev_err(&pdev->dev, "Failed to set DMA mask\n");
+			goto err_dma_mask;
+		}
+	}
+
+	err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
+	if (err) {
+		dev_warn(&pdev->dev,
+			 "Couldn't set 64-bit consistent DMA mask\n");
+		err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
+		if (err) {
+			dev_err(&pdev->dev,
+				"Failed to set consistent DMA mask\n");
+			goto err_dma_consistent_mask;
+		}
+	}
+
+	err = nic_dev_init(pdev);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to initialize NIC device\n");
+		goto err_nic_dev_init;
+	}
+
+	dev_info(&pdev->dev, "HiNIC driver - probed\n");
+	return 0;
+
+err_nic_dev_init:
+err_dma_consistent_mask:
+err_dma_mask:
+	pci_release_regions(pdev);
+
+err_pci_regions:
+	pci_disable_device(pdev);
+	return err;
+}
+
+static void hinic_remove(struct pci_dev *pdev)
+{
+	struct net_device *netdev = pci_get_drvdata(pdev);
+	struct hinic_dev *nic_dev = netdev_priv(netdev);
+	struct hinic_rx_mode_work *rx_mode_work;
+
+	unregister_netdev(netdev);
+
+	hinic_hwdev_cb_unregister(nic_dev->hwdev,
+				  HINIC_MGMT_MSG_CMD_LINK_STATUS);
+
+	rx_mode_work = &nic_dev->rx_mode_work;
+	cancel_work_sync(&rx_mode_work->work);
+
+	pci_set_drvdata(pdev, NULL);
+
+	destroy_workqueue(nic_dev->workq);
+
+	hinic_free_hwdev(nic_dev->hwdev);
+
+	free_netdev(netdev);
+
+	pci_release_regions(pdev);
+	pci_disable_device(pdev);
+
+	dev_info(&pdev->dev, "HiNIC driver - removed\n");
+}
+
+static const struct pci_device_id hinic_pci_table[] = {
+	{ PCI_VDEVICE(HUAWEI, PCI_DEVICE_ID_HI1822_PF), 0},
+	{ 0, 0}
+};
+MODULE_DEVICE_TABLE(pci, hinic_pci_table);
+
+static struct pci_driver hinic_driver = {
+	.name           = HINIC_DRV_NAME,
+	.id_table       = hinic_pci_table,
+	.probe          = hinic_probe,
+	.remove         = hinic_remove,
+};
+
+module_pci_driver(hinic_driver);
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_port.c b/drivers/net/ethernet/huawei/hinic/hinic_port.c
new file mode 100644
index 000000000000..4d4e3f05fb5f
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_port.c
@@ -0,0 +1,379 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#include <linux/types.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/if_vlan.h>
+#include <linux/pci.h>
+#include <linux/device.h>
+#include <linux/errno.h>
+
+#include "hinic_hw_if.h"
+#include "hinic_hw_dev.h"
+#include "hinic_port.h"
+#include "hinic_dev.h"
+
+#define HINIC_MIN_MTU_SIZE              256
+#define HINIC_MAX_JUMBO_FRAME_SIZE      15872
+
+enum mac_op {
+	MAC_DEL,
+	MAC_SET,
+};
+
+/**
+ * change_mac - change(add or delete) mac address
+ * @nic_dev: nic device
+ * @addr: mac address
+ * @vlan_id: vlan number to set with the mac
+ * @op: add or delete the mac
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int change_mac(struct hinic_dev *nic_dev, const u8 *addr,
+		      u16 vlan_id, enum mac_op op)
+{
+	struct net_device *netdev = nic_dev->netdev;
+	struct hinic_hwdev *hwdev = nic_dev->hwdev;
+	struct hinic_port_mac_cmd port_mac_cmd;
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	enum hinic_port_cmd cmd;
+	u16 out_size;
+	int err;
+
+	if (vlan_id >= VLAN_N_VID) {
+		netif_err(nic_dev, drv, netdev, "Invalid VLAN number\n");
+		return -EINVAL;
+	}
+
+	if (op == MAC_SET)
+		cmd = HINIC_PORT_CMD_SET_MAC;
+	else
+		cmd = HINIC_PORT_CMD_DEL_MAC;
+
+	port_mac_cmd.func_idx = HINIC_HWIF_FUNC_IDX(hwif);
+	port_mac_cmd.vlan_id = vlan_id;
+	memcpy(port_mac_cmd.mac, addr, ETH_ALEN);
+
+	err = hinic_port_msg_cmd(hwdev, cmd, &port_mac_cmd,
+				 sizeof(port_mac_cmd),
+				 &port_mac_cmd, &out_size);
+	if (err || (out_size != sizeof(port_mac_cmd)) || port_mac_cmd.status) {
+		dev_err(&pdev->dev, "Failed to change MAC, ret = %d\n",
+			port_mac_cmd.status);
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+/**
+ * hinic_port_add_mac - add mac address
+ * @nic_dev: nic device
+ * @addr: mac address
+ * @vlan_id: vlan number to set with the mac
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_port_add_mac(struct hinic_dev *nic_dev,
+		       const u8 *addr, u16 vlan_id)
+{
+	return change_mac(nic_dev, addr, vlan_id, MAC_SET);
+}
+
+/**
+ * hinic_port_del_mac - remove mac address
+ * @nic_dev: nic device
+ * @addr: mac address
+ * @vlan_id: vlan number that is connected to the mac
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_port_del_mac(struct hinic_dev *nic_dev, const u8 *addr,
+		       u16 vlan_id)
+{
+	return change_mac(nic_dev, addr, vlan_id, MAC_DEL);
+}
+
+/**
+ * hinic_port_get_mac - get the mac address of the nic device
+ * @nic_dev: nic device
+ * @addr: returned mac address
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_port_get_mac(struct hinic_dev *nic_dev, u8 *addr)
+{
+	struct hinic_hwdev *hwdev = nic_dev->hwdev;
+	struct hinic_port_mac_cmd port_mac_cmd;
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	u16 out_size;
+	int err;
+
+	port_mac_cmd.func_idx = HINIC_HWIF_FUNC_IDX(hwif);
+
+	err = hinic_port_msg_cmd(hwdev, HINIC_PORT_CMD_GET_MAC,
+				 &port_mac_cmd, sizeof(port_mac_cmd),
+				 &port_mac_cmd, &out_size);
+	if (err || (out_size != sizeof(port_mac_cmd)) || port_mac_cmd.status) {
+		dev_err(&pdev->dev, "Failed to get mac, ret = %d\n",
+			port_mac_cmd.status);
+		return -EFAULT;
+	}
+
+	memcpy(addr, port_mac_cmd.mac, ETH_ALEN);
+	return 0;
+}
+
+/**
+ * hinic_port_set_mtu - set mtu
+ * @nic_dev: nic device
+ * @new_mtu: new mtu
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_port_set_mtu(struct hinic_dev *nic_dev, int new_mtu)
+{
+	struct net_device *netdev = nic_dev->netdev;
+	struct hinic_hwdev *hwdev = nic_dev->hwdev;
+	struct hinic_port_mtu_cmd port_mtu_cmd;
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	int err, max_frame;
+	u16 out_size;
+
+	if (new_mtu < HINIC_MIN_MTU_SIZE) {
+		netif_err(nic_dev, drv, netdev, "mtu < MIN MTU size");
+		return -EINVAL;
+	}
+
+	max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
+	if (max_frame > HINIC_MAX_JUMBO_FRAME_SIZE) {
+		netif_err(nic_dev, drv, netdev, "mtu > MAX MTU size");
+		return -EINVAL;
+	}
+
+	port_mtu_cmd.func_idx = HINIC_HWIF_FUNC_IDX(hwif);
+	port_mtu_cmd.mtu = new_mtu;
+
+	err = hinic_port_msg_cmd(hwdev, HINIC_PORT_CMD_CHANGE_MTU,
+				 &port_mtu_cmd, sizeof(port_mtu_cmd),
+				 &port_mtu_cmd, &out_size);
+	if (err || (out_size != sizeof(port_mtu_cmd)) || port_mtu_cmd.status) {
+		dev_err(&pdev->dev, "Failed to set mtu, ret = %d\n",
+			port_mtu_cmd.status);
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+/**
+ * hinic_port_add_vlan - add vlan to the nic device
+ * @nic_dev: nic device
+ * @vlan_id: the vlan number to add
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_port_add_vlan(struct hinic_dev *nic_dev, u16 vlan_id)
+{
+	struct hinic_hwdev *hwdev = nic_dev->hwdev;
+	struct hinic_port_vlan_cmd port_vlan_cmd;
+
+	port_vlan_cmd.func_idx = HINIC_HWIF_FUNC_IDX(hwdev->hwif);
+	port_vlan_cmd.vlan_id = vlan_id;
+
+	return hinic_port_msg_cmd(hwdev, HINIC_PORT_CMD_ADD_VLAN,
+				  &port_vlan_cmd, sizeof(port_vlan_cmd),
+				  NULL, NULL);
+}
+
+/**
+ * hinic_port_del_vlan - delete vlan from the nic device
+ * @nic_dev: nic device
+ * @vlan_id: the vlan number to delete
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_port_del_vlan(struct hinic_dev *nic_dev, u16 vlan_id)
+{
+	struct hinic_hwdev *hwdev = nic_dev->hwdev;
+	struct hinic_port_vlan_cmd port_vlan_cmd;
+
+	port_vlan_cmd.func_idx = HINIC_HWIF_FUNC_IDX(hwdev->hwif);
+	port_vlan_cmd.vlan_id = vlan_id;
+
+	return hinic_port_msg_cmd(hwdev, HINIC_PORT_CMD_DEL_VLAN,
+				 &port_vlan_cmd, sizeof(port_vlan_cmd),
+				 NULL, NULL);
+}
+
+/**
+ * hinic_port_set_rx_mode - set rx mode in the nic device
+ * @nic_dev: nic device
+ * @rx_mode: the rx mode to set
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_port_set_rx_mode(struct hinic_dev *nic_dev, u32 rx_mode)
+{
+	struct hinic_hwdev *hwdev = nic_dev->hwdev;
+	struct hinic_port_rx_mode_cmd rx_mode_cmd;
+
+	rx_mode_cmd.func_idx = HINIC_HWIF_FUNC_IDX(hwdev->hwif);
+	rx_mode_cmd.rx_mode = rx_mode;
+
+	return hinic_port_msg_cmd(hwdev, HINIC_PORT_CMD_SET_RX_MODE,
+				  &rx_mode_cmd, sizeof(rx_mode_cmd),
+				  NULL, NULL);
+}
+
+/**
+ * hinic_port_link_state - get the link state
+ * @nic_dev: nic device
+ * @link_state: the returned link state
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_port_link_state(struct hinic_dev *nic_dev,
+			  enum hinic_port_link_state *link_state)
+{
+	struct hinic_hwdev *hwdev = nic_dev->hwdev;
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct hinic_port_link_cmd link_cmd;
+	struct pci_dev *pdev = hwif->pdev;
+	u16 out_size;
+	int err;
+
+	if (!HINIC_IS_PF(hwif) && !HINIC_IS_PPF(hwif)) {
+		dev_err(&pdev->dev, "unsupported PCI Function type\n");
+		return -EINVAL;
+	}
+
+	link_cmd.func_idx = HINIC_HWIF_FUNC_IDX(hwif);
+
+	err = hinic_port_msg_cmd(hwdev, HINIC_PORT_CMD_GET_LINK_STATE,
+				 &link_cmd, sizeof(link_cmd),
+				 &link_cmd, &out_size);
+	if (err || (out_size != sizeof(link_cmd)) || link_cmd.status) {
+		dev_err(&pdev->dev, "Failed to get link state, ret = %d\n",
+			link_cmd.status);
+		return -EINVAL;
+	}
+
+	*link_state = link_cmd.state;
+	return 0;
+}
+
+/**
+ * hinic_port_set_state - set port state
+ * @nic_dev: nic device
+ * @state: the state to set
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_port_set_state(struct hinic_dev *nic_dev, enum hinic_port_state state)
+{
+	struct hinic_hwdev *hwdev = nic_dev->hwdev;
+	struct hinic_port_state_cmd port_state;
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	u16 out_size;
+	int err;
+
+	if (!HINIC_IS_PF(hwif) && !HINIC_IS_PPF(hwif)) {
+		dev_err(&pdev->dev, "unsupported PCI Function type\n");
+		return -EINVAL;
+	}
+
+	port_state.state = state;
+
+	err = hinic_port_msg_cmd(hwdev, HINIC_PORT_CMD_SET_PORT_STATE,
+				 &port_state, sizeof(port_state),
+				 &port_state, &out_size);
+	if (err || (out_size != sizeof(port_state)) || port_state.status) {
+		dev_err(&pdev->dev, "Failed to set port state, ret = %d\n",
+			port_state.status);
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+/**
+ * hinic_port_set_func_state- set func device state
+ * @nic_dev: nic device
+ * @state: the state to set
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_port_set_func_state(struct hinic_dev *nic_dev,
+			      enum hinic_func_port_state state)
+{
+	struct hinic_port_func_state_cmd func_state;
+	struct hinic_hwdev *hwdev = nic_dev->hwdev;
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	u16 out_size;
+	int err;
+
+	func_state.func_idx = HINIC_HWIF_FUNC_IDX(hwif);
+	func_state.state = state;
+
+	err = hinic_port_msg_cmd(hwdev, HINIC_PORT_CMD_SET_FUNC_STATE,
+				 &func_state, sizeof(func_state),
+				 &func_state, &out_size);
+	if (err || (out_size != sizeof(func_state)) || func_state.status) {
+		dev_err(&pdev->dev, "Failed to set port func state, ret = %d\n",
+			func_state.status);
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+/**
+ * hinic_port_get_cap - get port capabilities
+ * @nic_dev: nic device
+ * @port_cap: returned port capabilities
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_port_get_cap(struct hinic_dev *nic_dev,
+		       struct hinic_port_cap *port_cap)
+{
+	struct hinic_hwdev *hwdev = nic_dev->hwdev;
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	u16 out_size;
+	int err;
+
+	port_cap->func_idx = HINIC_HWIF_FUNC_IDX(hwif);
+
+	err = hinic_port_msg_cmd(hwdev, HINIC_PORT_CMD_GET_CAP,
+				 port_cap, sizeof(*port_cap),
+				 port_cap, &out_size);
+	if (err || (out_size != sizeof(*port_cap)) || port_cap->status) {
+		dev_err(&pdev->dev,
+			"Failed to get port capabilities, ret = %d\n",
+			port_cap->status);
+		return -EINVAL;
+	}
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_port.h b/drivers/net/ethernet/huawei/hinic/hinic_port.h
new file mode 100644
index 000000000000..9404365195dd
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_port.h
@@ -0,0 +1,198 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#ifndef HINIC_PORT_H
+#define HINIC_PORT_H
+
+#include <linux/types.h>
+#include <linux/etherdevice.h>
+#include <linux/bitops.h>
+
+#include "hinic_dev.h"
+
+enum hinic_rx_mode {
+	HINIC_RX_MODE_UC        = BIT(0),
+	HINIC_RX_MODE_MC        = BIT(1),
+	HINIC_RX_MODE_BC        = BIT(2),
+	HINIC_RX_MODE_MC_ALL    = BIT(3),
+	HINIC_RX_MODE_PROMISC   = BIT(4),
+};
+
+enum hinic_port_link_state {
+	HINIC_LINK_STATE_DOWN,
+	HINIC_LINK_STATE_UP,
+};
+
+enum hinic_port_state {
+	HINIC_PORT_DISABLE      = 0,
+	HINIC_PORT_ENABLE       = 3,
+};
+
+enum hinic_func_port_state {
+	HINIC_FUNC_PORT_DISABLE = 0,
+	HINIC_FUNC_PORT_ENABLE  = 2,
+};
+
+enum hinic_autoneg_cap {
+	HINIC_AUTONEG_UNSUPPORTED,
+	HINIC_AUTONEG_SUPPORTED,
+};
+
+enum hinic_autoneg_state {
+	HINIC_AUTONEG_DISABLED,
+	HINIC_AUTONEG_ACTIVE,
+};
+
+enum hinic_duplex {
+	HINIC_DUPLEX_HALF,
+	HINIC_DUPLEX_FULL,
+};
+
+enum hinic_speed {
+	HINIC_SPEED_10MB_LINK = 0,
+	HINIC_SPEED_100MB_LINK,
+	HINIC_SPEED_1000MB_LINK,
+	HINIC_SPEED_10GB_LINK,
+	HINIC_SPEED_25GB_LINK,
+	HINIC_SPEED_40GB_LINK,
+	HINIC_SPEED_100GB_LINK,
+
+	HINIC_SPEED_UNKNOWN = 0xFF,
+};
+
+struct hinic_port_mac_cmd {
+	u8              status;
+	u8              version;
+	u8              rsvd0[6];
+
+	u16             func_idx;
+	u16             vlan_id;
+	u16             rsvd1;
+	unsigned char   mac[ETH_ALEN];
+};
+
+struct hinic_port_mtu_cmd {
+	u8      status;
+	u8      version;
+	u8      rsvd0[6];
+
+	u16     func_idx;
+	u16     rsvd1;
+	u32     mtu;
+};
+
+struct hinic_port_vlan_cmd {
+	u8      status;
+	u8      version;
+	u8      rsvd0[6];
+
+	u16     func_idx;
+	u16     vlan_id;
+};
+
+struct hinic_port_rx_mode_cmd {
+	u8      status;
+	u8      version;
+	u8      rsvd0[6];
+
+	u16     func_idx;
+	u16     rsvd;
+	u32     rx_mode;
+};
+
+struct hinic_port_link_cmd {
+	u8      status;
+	u8      version;
+	u8      rsvd0[6];
+
+	u16     func_idx;
+	u8      state;
+	u8      rsvd1;
+};
+
+struct hinic_port_state_cmd {
+	u8      status;
+	u8      version;
+	u8      rsvd0[6];
+
+	u8      state;
+	u8      rsvd1[3];
+};
+
+struct hinic_port_link_status {
+	u8      status;
+	u8      version;
+	u8      rsvd0[6];
+
+	u16     rsvd1;
+	u8      link;
+	u8      rsvd2;
+};
+
+struct hinic_port_func_state_cmd {
+	u8      status;
+	u8      version;
+	u8      rsvd0[6];
+
+	u16     func_idx;
+	u16     rsvd1;
+	u8      state;
+	u8      rsvd2[3];
+};
+
+struct hinic_port_cap {
+	u8      status;
+	u8      version;
+	u8      rsvd0[6];
+
+	u16     func_idx;
+	u16     rsvd1;
+	u8      port_type;
+	u8      autoneg_cap;
+	u8      autoneg_state;
+	u8      duplex;
+	u8      speed;
+	u8      rsvd2[3];
+};
+
+int hinic_port_add_mac(struct hinic_dev *nic_dev, const u8 *addr,
+		       u16 vlan_id);
+
+int hinic_port_del_mac(struct hinic_dev *nic_dev, const u8 *addr,
+		       u16 vlan_id);
+
+int hinic_port_get_mac(struct hinic_dev *nic_dev, u8 *addr);
+
+int hinic_port_set_mtu(struct hinic_dev *nic_dev, int new_mtu);
+
+int hinic_port_add_vlan(struct hinic_dev *nic_dev, u16 vlan_id);
+
+int hinic_port_del_vlan(struct hinic_dev *nic_dev, u16 vlan_id);
+
+int hinic_port_set_rx_mode(struct hinic_dev *nic_dev, u32 rx_mode);
+
+int hinic_port_link_state(struct hinic_dev *nic_dev,
+			  enum hinic_port_link_state *link_state);
+
+int hinic_port_set_state(struct hinic_dev *nic_dev,
+			 enum hinic_port_state state);
+
+int hinic_port_set_func_state(struct hinic_dev *nic_dev,
+			      enum hinic_func_port_state state);
+
+int hinic_port_get_cap(struct hinic_dev *nic_dev,
+		       struct hinic_port_cap *port_cap);
+
+#endif
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_rx.c b/drivers/net/ethernet/huawei/hinic/hinic_rx.c
new file mode 100644
index 000000000000..1d4f712b15a8
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_rx.c
@@ -0,0 +1,509 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/errno.h>
+#include <linux/pci.h>
+#include <linux/device.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/u64_stats_sync.h>
+#include <linux/slab.h>
+#include <linux/interrupt.h>
+#include <linux/skbuff.h>
+#include <linux/dma-mapping.h>
+#include <linux/prefetch.h>
+#include <asm/barrier.h>
+
+#include "hinic_common.h"
+#include "hinic_hw_if.h"
+#include "hinic_hw_wqe.h"
+#include "hinic_hw_wq.h"
+#include "hinic_hw_qp.h"
+#include "hinic_hw_dev.h"
+#include "hinic_rx.h"
+#include "hinic_dev.h"
+
+#define RX_IRQ_NO_PENDING               0
+#define RX_IRQ_NO_COALESC               0
+#define RX_IRQ_NO_LLI_TIMER             0
+#define RX_IRQ_NO_CREDIT                0
+#define RX_IRQ_NO_RESEND_TIMER          0
+
+/**
+ * hinic_rxq_clean_stats - Clean the statistics of specific queue
+ * @rxq: Logical Rx Queue
+ **/
+void hinic_rxq_clean_stats(struct hinic_rxq *rxq)
+{
+	struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;
+
+	u64_stats_update_begin(&rxq_stats->syncp);
+	rxq_stats->pkts  = 0;
+	rxq_stats->bytes = 0;
+	u64_stats_update_end(&rxq_stats->syncp);
+}
+
+/**
+ * hinic_rxq_get_stats - get statistics of Rx Queue
+ * @rxq: Logical Rx Queue
+ * @stats: return updated stats here
+ **/
+void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats)
+{
+	struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;
+	unsigned int start;
+
+	u64_stats_update_begin(&stats->syncp);
+	do {
+		start = u64_stats_fetch_begin(&rxq_stats->syncp);
+		stats->pkts = rxq_stats->pkts;
+		stats->bytes = rxq_stats->bytes;
+	} while (u64_stats_fetch_retry(&rxq_stats->syncp, start));
+	u64_stats_update_end(&stats->syncp);
+}
+
+/**
+ * rxq_stats_init - Initialize the statistics of specific queue
+ * @rxq: Logical Rx Queue
+ **/
+static void rxq_stats_init(struct hinic_rxq *rxq)
+{
+	struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;
+
+	u64_stats_init(&rxq_stats->syncp);
+	hinic_rxq_clean_stats(rxq);
+}
+
+/**
+ * rx_alloc_skb - allocate skb and map it to dma address
+ * @rxq: rx queue
+ * @dma_addr: returned dma address for the skb
+ *
+ * Return skb
+ **/
+static struct sk_buff *rx_alloc_skb(struct hinic_rxq *rxq,
+				    dma_addr_t *dma_addr)
+{
+	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
+	struct hinic_hwdev *hwdev = nic_dev->hwdev;
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	struct sk_buff *skb;
+	dma_addr_t addr;
+	int err;
+
+	skb = netdev_alloc_skb_ip_align(rxq->netdev, rxq->rq->buf_sz);
+	if (!skb) {
+		netdev_err(rxq->netdev, "Failed to allocate Rx SKB\n");
+		return NULL;
+	}
+
+	addr = dma_map_single(&pdev->dev, skb->data, rxq->rq->buf_sz,
+			      DMA_FROM_DEVICE);
+	err = dma_mapping_error(&pdev->dev, addr);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to map Rx DMA, err = %d\n", err);
+		goto err_rx_map;
+	}
+
+	*dma_addr = addr;
+	return skb;
+
+err_rx_map:
+	dev_kfree_skb_any(skb);
+	return NULL;
+}
+
+/**
+ * rx_unmap_skb - unmap the dma address of the skb
+ * @rxq: rx queue
+ * @dma_addr: dma address of the skb
+ **/
+static void rx_unmap_skb(struct hinic_rxq *rxq, dma_addr_t dma_addr)
+{
+	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
+	struct hinic_hwdev *hwdev = nic_dev->hwdev;
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+
+	dma_unmap_single(&pdev->dev, dma_addr, rxq->rq->buf_sz,
+			 DMA_FROM_DEVICE);
+}
+
+/**
+ * rx_free_skb - unmap and free skb
+ * @rxq: rx queue
+ * @skb: skb to free
+ * @dma_addr: dma address of the skb
+ **/
+static void rx_free_skb(struct hinic_rxq *rxq, struct sk_buff *skb,
+			dma_addr_t dma_addr)
+{
+	rx_unmap_skb(rxq, dma_addr);
+	dev_kfree_skb_any(skb);
+}
+
+/**
+ * rx_alloc_pkts - allocate pkts in rx queue
+ * @rxq: rx queue
+ *
+ * Return number of skbs allocated
+ **/
+static int rx_alloc_pkts(struct hinic_rxq *rxq)
+{
+	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
+	struct hinic_rq_wqe *rq_wqe;
+	unsigned int free_wqebbs;
+	struct hinic_sge sge;
+	dma_addr_t dma_addr;
+	struct sk_buff *skb;
+	int i, alloc_more;
+	u16 prod_idx;
+
+	free_wqebbs = hinic_get_rq_free_wqebbs(rxq->rq);
+	alloc_more = 0;
+
+	/* Limit the allocation chunks */
+	if (free_wqebbs > nic_dev->rx_weight)
+		free_wqebbs = nic_dev->rx_weight;
+
+	for (i = 0; i < free_wqebbs; i++) {
+		skb = rx_alloc_skb(rxq, &dma_addr);
+		if (!skb) {
+			netdev_err(rxq->netdev, "Failed to alloc Rx skb\n");
+			alloc_more = 1;
+			goto skb_out;
+		}
+
+		hinic_set_sge(&sge, dma_addr, skb->len);
+
+		rq_wqe = hinic_rq_get_wqe(rxq->rq, HINIC_RQ_WQE_SIZE,
+					  &prod_idx);
+		if (!rq_wqe) {
+			rx_free_skb(rxq, skb, dma_addr);
+			alloc_more = 1;
+			goto skb_out;
+		}
+
+		hinic_rq_prepare_wqe(rxq->rq, prod_idx, rq_wqe, &sge);
+
+		hinic_rq_write_wqe(rxq->rq, prod_idx, rq_wqe, skb);
+	}
+
+skb_out:
+	if (i) {
+		wmb();  /* write all the wqes before update PI */
+
+		hinic_rq_update(rxq->rq, prod_idx);
+	}
+
+	if (alloc_more)
+		tasklet_schedule(&rxq->rx_task);
+
+	return i;
+}
+
+/**
+ * free_all_rx_skbs - free all skbs in rx queue
+ * @rxq: rx queue
+ **/
+static void free_all_rx_skbs(struct hinic_rxq *rxq)
+{
+	struct hinic_rq *rq = rxq->rq;
+	struct hinic_hw_wqe *hw_wqe;
+	struct hinic_sge sge;
+	u16 ci;
+
+	while ((hw_wqe = hinic_read_wqe(rq->wq, HINIC_RQ_WQE_SIZE, &ci))) {
+		if (IS_ERR(hw_wqe))
+			break;
+
+		hinic_rq_get_sge(rq, &hw_wqe->rq_wqe, ci, &sge);
+
+		hinic_put_wqe(rq->wq, HINIC_RQ_WQE_SIZE);
+
+		rx_free_skb(rxq, rq->saved_skb[ci], hinic_sge_to_dma(&sge));
+	}
+}
+
+/**
+ * rx_alloc_task - tasklet for queue allocation
+ * @data: rx queue
+ **/
+static void rx_alloc_task(unsigned long data)
+{
+	struct hinic_rxq *rxq = (struct hinic_rxq *)data;
+
+	(void)rx_alloc_pkts(rxq);
+}
+
+/**
+ * rx_recv_jumbo_pkt - Rx handler for jumbo pkt
+ * @rxq: rx queue
+ * @head_skb: the first skb in the list
+ * @left_pkt_len: left size of the pkt exclude head skb
+ * @ci: consumer index
+ *
+ * Return number of wqes that used for the left of the pkt
+ **/
+static int rx_recv_jumbo_pkt(struct hinic_rxq *rxq, struct sk_buff *head_skb,
+			     unsigned int left_pkt_len, u16 ci)
+{
+	struct sk_buff *skb, *curr_skb = head_skb;
+	struct hinic_rq_wqe *rq_wqe;
+	unsigned int curr_len;
+	struct hinic_sge sge;
+	int num_wqes = 0;
+
+	while (left_pkt_len > 0) {
+		rq_wqe = hinic_rq_read_next_wqe(rxq->rq, HINIC_RQ_WQE_SIZE,
+						&skb, &ci);
+
+		num_wqes++;
+
+		hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge);
+
+		rx_unmap_skb(rxq, hinic_sge_to_dma(&sge));
+
+		prefetch(skb->data);
+
+		curr_len = (left_pkt_len > HINIC_RX_BUF_SZ) ? HINIC_RX_BUF_SZ :
+			    left_pkt_len;
+
+		left_pkt_len -= curr_len;
+
+		__skb_put(skb, curr_len);
+
+		if (curr_skb == head_skb)
+			skb_shinfo(head_skb)->frag_list = skb;
+		else
+			curr_skb->next = skb;
+
+		head_skb->len += skb->len;
+		head_skb->data_len += skb->len;
+		head_skb->truesize += skb->truesize;
+
+		curr_skb = skb;
+	}
+
+	return num_wqes;
+}
+
+/**
+ * rxq_recv - Rx handler
+ * @rxq: rx queue
+ * @budget: maximum pkts to process
+ *
+ * Return number of pkts received
+ **/
+static int rxq_recv(struct hinic_rxq *rxq, int budget)
+{
+	struct hinic_qp *qp = container_of(rxq->rq, struct hinic_qp, rq);
+	u64 pkt_len = 0, rx_bytes = 0;
+	struct hinic_rq_wqe *rq_wqe;
+	int num_wqes, pkts = 0;
+	struct hinic_sge sge;
+	struct sk_buff *skb;
+	u16 ci;
+
+	while (pkts < budget) {
+		num_wqes = 0;
+
+		rq_wqe = hinic_rq_read_wqe(rxq->rq, HINIC_RQ_WQE_SIZE, &skb,
+					   &ci);
+		if (!rq_wqe)
+			break;
+
+		hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge);
+
+		rx_unmap_skb(rxq, hinic_sge_to_dma(&sge));
+
+		prefetch(skb->data);
+
+		pkt_len = sge.len;
+
+		if (pkt_len <= HINIC_RX_BUF_SZ) {
+			__skb_put(skb, pkt_len);
+		} else {
+			__skb_put(skb, HINIC_RX_BUF_SZ);
+			num_wqes = rx_recv_jumbo_pkt(rxq, skb, pkt_len -
+						     HINIC_RX_BUF_SZ, ci);
+		}
+
+		hinic_rq_put_wqe(rxq->rq, ci,
+				 (num_wqes + 1) * HINIC_RQ_WQE_SIZE);
+
+		skb_record_rx_queue(skb, qp->q_id);
+		skb->protocol = eth_type_trans(skb, rxq->netdev);
+
+		napi_gro_receive(&rxq->napi, skb);
+
+		pkts++;
+		rx_bytes += pkt_len;
+	}
+
+	if (pkts)
+		tasklet_schedule(&rxq->rx_task); /* hinic_rx_alloc_pkts */
+
+	u64_stats_update_begin(&rxq->rxq_stats.syncp);
+	rxq->rxq_stats.pkts += pkts;
+	rxq->rxq_stats.bytes += rx_bytes;
+	u64_stats_update_end(&rxq->rxq_stats.syncp);
+
+	return pkts;
+}
+
+static int rx_poll(struct napi_struct *napi, int budget)
+{
+	struct hinic_rxq *rxq = container_of(napi, struct hinic_rxq, napi);
+	struct hinic_rq *rq = rxq->rq;
+	int pkts;
+
+	pkts = rxq_recv(rxq, budget);
+	if (pkts >= budget)
+		return budget;
+
+	napi_complete(napi);
+	enable_irq(rq->irq);
+	return pkts;
+}
+
+static void rx_add_napi(struct hinic_rxq *rxq)
+{
+	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
+
+	netif_napi_add(rxq->netdev, &rxq->napi, rx_poll, nic_dev->rx_weight);
+	napi_enable(&rxq->napi);
+}
+
+static void rx_del_napi(struct hinic_rxq *rxq)
+{
+	napi_disable(&rxq->napi);
+	netif_napi_del(&rxq->napi);
+}
+
+static irqreturn_t rx_irq(int irq, void *data)
+{
+	struct hinic_rxq *rxq = (struct hinic_rxq *)data;
+	struct hinic_rq *rq = rxq->rq;
+	struct hinic_dev *nic_dev;
+
+	/* Disable the interrupt until napi will be completed */
+	disable_irq_nosync(rq->irq);
+
+	nic_dev = netdev_priv(rxq->netdev);
+	hinic_hwdev_msix_cnt_set(nic_dev->hwdev, rq->msix_entry);
+
+	napi_schedule(&rxq->napi);
+	return IRQ_HANDLED;
+}
+
+static int rx_request_irq(struct hinic_rxq *rxq)
+{
+	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
+	struct hinic_hwdev *hwdev = nic_dev->hwdev;
+	struct hinic_rq *rq = rxq->rq;
+	int err;
+
+	rx_add_napi(rxq);
+
+	hinic_hwdev_msix_set(hwdev, rq->msix_entry,
+			     RX_IRQ_NO_PENDING, RX_IRQ_NO_COALESC,
+			     RX_IRQ_NO_LLI_TIMER, RX_IRQ_NO_CREDIT,
+			     RX_IRQ_NO_RESEND_TIMER);
+
+	err = request_irq(rq->irq, rx_irq, 0, rxq->irq_name, rxq);
+	if (err) {
+		rx_del_napi(rxq);
+		return err;
+	}
+
+	return 0;
+}
+
+static void rx_free_irq(struct hinic_rxq *rxq)
+{
+	struct hinic_rq *rq = rxq->rq;
+
+	free_irq(rq->irq, rxq);
+	rx_del_napi(rxq);
+}
+
+/**
+ * hinic_init_rxq - Initialize the Rx Queue
+ * @rxq: Logical Rx Queue
+ * @rq: Hardware Rx Queue to connect the Logical queue with
+ * @netdev: network device to connect the Logical queue with
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_init_rxq(struct hinic_rxq *rxq, struct hinic_rq *rq,
+		   struct net_device *netdev)
+{
+	struct hinic_qp *qp = container_of(rq, struct hinic_qp, rq);
+	int err, pkts, irqname_len;
+
+	rxq->netdev = netdev;
+	rxq->rq = rq;
+
+	rxq_stats_init(rxq);
+
+	irqname_len = snprintf(NULL, 0, "hinic_rxq%d", qp->q_id) + 1;
+	rxq->irq_name = devm_kzalloc(&netdev->dev, irqname_len, GFP_KERNEL);
+	if (!rxq->irq_name)
+		return -ENOMEM;
+
+	sprintf(rxq->irq_name, "hinic_rxq%d", qp->q_id);
+
+	tasklet_init(&rxq->rx_task, rx_alloc_task, (unsigned long)rxq);
+
+	pkts = rx_alloc_pkts(rxq);
+	if (!pkts) {
+		err = -ENOMEM;
+		goto err_rx_pkts;
+	}
+
+	err = rx_request_irq(rxq);
+	if (err) {
+		netdev_err(netdev, "Failed to request Rx irq\n");
+		goto err_req_rx_irq;
+	}
+
+	return 0;
+
+err_req_rx_irq:
+err_rx_pkts:
+	tasklet_kill(&rxq->rx_task);
+	free_all_rx_skbs(rxq);
+	devm_kfree(&netdev->dev, rxq->irq_name);
+	return err;
+}
+
+/**
+ * hinic_clean_rxq - Clean the Rx Queue
+ * @rxq: Logical Rx Queue
+ **/
+void hinic_clean_rxq(struct hinic_rxq *rxq)
+{
+	struct net_device *netdev = rxq->netdev;
+
+	rx_free_irq(rxq);
+
+	tasklet_kill(&rxq->rx_task);
+	free_all_rx_skbs(rxq);
+	devm_kfree(&netdev->dev, rxq->irq_name);
+}
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_rx.h b/drivers/net/ethernet/huawei/hinic/hinic_rx.h
new file mode 100644
index 000000000000..27c9af4b1c12
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_rx.h
@@ -0,0 +1,55 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#ifndef HINIC_RX_H
+#define HINIC_RX_H
+
+#include <linux/types.h>
+#include <linux/netdevice.h>
+#include <linux/u64_stats_sync.h>
+#include <linux/interrupt.h>
+
+#include "hinic_hw_qp.h"
+
+struct hinic_rxq_stats {
+	u64                     pkts;
+	u64                     bytes;
+
+	struct u64_stats_sync   syncp;
+};
+
+struct hinic_rxq {
+	struct net_device       *netdev;
+	struct hinic_rq         *rq;
+
+	struct hinic_rxq_stats  rxq_stats;
+
+	char                    *irq_name;
+
+	struct tasklet_struct   rx_task;
+
+	struct napi_struct      napi;
+};
+
+void hinic_rxq_clean_stats(struct hinic_rxq *rxq);
+
+void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats);
+
+int hinic_init_rxq(struct hinic_rxq *rxq, struct hinic_rq *rq,
+		   struct net_device *netdev);
+
+void hinic_clean_rxq(struct hinic_rxq *rxq);
+
+#endif
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_tx.c b/drivers/net/ethernet/huawei/hinic/hinic_tx.c
new file mode 100644
index 000000000000..abe3e38cd342
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_tx.c
@@ -0,0 +1,504 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/netdevice.h>
+#include <linux/u64_stats_sync.h>
+#include <linux/errno.h>
+#include <linux/types.h>
+#include <linux/pci.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/slab.h>
+#include <linux/interrupt.h>
+#include <linux/skbuff.h>
+#include <linux/smp.h>
+#include <asm/byteorder.h>
+
+#include "hinic_common.h"
+#include "hinic_hw_if.h"
+#include "hinic_hw_wqe.h"
+#include "hinic_hw_wq.h"
+#include "hinic_hw_qp.h"
+#include "hinic_hw_dev.h"
+#include "hinic_dev.h"
+#include "hinic_tx.h"
+
+#define TX_IRQ_NO_PENDING               0
+#define TX_IRQ_NO_COALESC               0
+#define TX_IRQ_NO_LLI_TIMER             0
+#define TX_IRQ_NO_CREDIT                0
+#define TX_IRQ_NO_RESEND_TIMER          0
+
+#define CI_UPDATE_NO_PENDING            0
+#define CI_UPDATE_NO_COALESC            0
+
+#define HW_CONS_IDX(sq)         be16_to_cpu(*(u16 *)((sq)->hw_ci_addr))
+
+#define MIN_SKB_LEN             64
+
+/**
+ * hinic_txq_clean_stats - Clean the statistics of specific queue
+ * @txq: Logical Tx Queue
+ **/
+void hinic_txq_clean_stats(struct hinic_txq *txq)
+{
+	struct hinic_txq_stats *txq_stats = &txq->txq_stats;
+
+	u64_stats_update_begin(&txq_stats->syncp);
+	txq_stats->pkts    = 0;
+	txq_stats->bytes   = 0;
+	txq_stats->tx_busy = 0;
+	txq_stats->tx_wake = 0;
+	txq_stats->tx_dropped = 0;
+	u64_stats_update_end(&txq_stats->syncp);
+}
+
+/**
+ * hinic_txq_get_stats - get statistics of Tx Queue
+ * @txq: Logical Tx Queue
+ * @stats: return updated stats here
+ **/
+void hinic_txq_get_stats(struct hinic_txq *txq, struct hinic_txq_stats *stats)
+{
+	struct hinic_txq_stats *txq_stats = &txq->txq_stats;
+	unsigned int start;
+
+	u64_stats_update_begin(&stats->syncp);
+	do {
+		start = u64_stats_fetch_begin(&txq_stats->syncp);
+		stats->pkts    = txq_stats->pkts;
+		stats->bytes   = txq_stats->bytes;
+		stats->tx_busy = txq_stats->tx_busy;
+		stats->tx_wake = txq_stats->tx_wake;
+		stats->tx_dropped = txq_stats->tx_dropped;
+	} while (u64_stats_fetch_retry(&txq_stats->syncp, start));
+	u64_stats_update_end(&stats->syncp);
+}
+
+/**
+ * txq_stats_init - Initialize the statistics of specific queue
+ * @txq: Logical Tx Queue
+ **/
+static void txq_stats_init(struct hinic_txq *txq)
+{
+	struct hinic_txq_stats *txq_stats = &txq->txq_stats;
+
+	u64_stats_init(&txq_stats->syncp);
+	hinic_txq_clean_stats(txq);
+}
+
+/**
+ * tx_map_skb - dma mapping for skb and return sges
+ * @nic_dev: nic device
+ * @skb: the skb
+ * @sges: returned sges
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int tx_map_skb(struct hinic_dev *nic_dev, struct sk_buff *skb,
+		      struct hinic_sge *sges)
+{
+	struct hinic_hwdev *hwdev = nic_dev->hwdev;
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	struct skb_frag_struct *frag;
+	dma_addr_t dma_addr;
+	int i, j;
+
+	dma_addr = dma_map_single(&pdev->dev, skb->data, skb_headlen(skb),
+				  DMA_TO_DEVICE);
+	if (dma_mapping_error(&pdev->dev, dma_addr)) {
+		dev_err(&pdev->dev, "Failed to map Tx skb data\n");
+		return -EFAULT;
+	}
+
+	hinic_set_sge(&sges[0], dma_addr, skb_headlen(skb));
+
+	for (i = 0 ; i < skb_shinfo(skb)->nr_frags; i++) {
+		frag = &skb_shinfo(skb)->frags[i];
+
+		dma_addr = skb_frag_dma_map(&pdev->dev, frag, 0,
+					    skb_frag_size(frag),
+					    DMA_TO_DEVICE);
+		if (dma_mapping_error(&pdev->dev, dma_addr)) {
+			dev_err(&pdev->dev, "Failed to map Tx skb frag\n");
+			goto err_tx_map;
+		}
+
+		hinic_set_sge(&sges[i + 1], dma_addr, skb_frag_size(frag));
+	}
+
+	return 0;
+
+err_tx_map:
+	for (j = 0; j < i; j++)
+		dma_unmap_page(&pdev->dev, hinic_sge_to_dma(&sges[j + 1]),
+			       sges[j + 1].len, DMA_TO_DEVICE);
+
+	dma_unmap_single(&pdev->dev, hinic_sge_to_dma(&sges[0]), sges[0].len,
+			 DMA_TO_DEVICE);
+	return -EFAULT;
+}
+
+/**
+ * tx_unmap_skb - unmap the dma address of the skb
+ * @nic_dev: nic device
+ * @skb: the skb
+ * @sges: the sges that are connected to the skb
+ **/
+static void tx_unmap_skb(struct hinic_dev *nic_dev, struct sk_buff *skb,
+			 struct hinic_sge *sges)
+{
+	struct hinic_hwdev *hwdev = nic_dev->hwdev;
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	int i;
+
+	for (i = 0; i < skb_shinfo(skb)->nr_frags ; i++)
+		dma_unmap_page(&pdev->dev, hinic_sge_to_dma(&sges[i + 1]),
+			       sges[i + 1].len, DMA_TO_DEVICE);
+
+	dma_unmap_single(&pdev->dev, hinic_sge_to_dma(&sges[0]), sges[0].len,
+			 DMA_TO_DEVICE);
+}
+
+netdev_tx_t hinic_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
+{
+	struct hinic_dev *nic_dev = netdev_priv(netdev);
+	struct netdev_queue *netdev_txq;
+	int nr_sges, err = NETDEV_TX_OK;
+	struct hinic_sq_wqe *sq_wqe;
+	unsigned int wqe_size;
+	struct hinic_txq *txq;
+	struct hinic_qp *qp;
+	u16 prod_idx;
+
+	txq = &nic_dev->txqs[skb->queue_mapping];
+	qp = container_of(txq->sq, struct hinic_qp, sq);
+
+	if (skb->len < MIN_SKB_LEN) {
+		if (skb_pad(skb, MIN_SKB_LEN - skb->len)) {
+			netdev_err(netdev, "Failed to pad skb\n");
+			goto update_error_stats;
+		}
+
+		skb->len = MIN_SKB_LEN;
+	}
+
+	nr_sges = skb_shinfo(skb)->nr_frags + 1;
+	if (nr_sges > txq->max_sges) {
+		netdev_err(netdev, "Too many Tx sges\n");
+		goto skb_error;
+	}
+
+	err = tx_map_skb(nic_dev, skb, txq->sges);
+	if (err)
+		goto skb_error;
+
+	wqe_size = HINIC_SQ_WQE_SIZE(nr_sges);
+
+	sq_wqe = hinic_sq_get_wqe(txq->sq, wqe_size, &prod_idx);
+	if (!sq_wqe) {
+		tx_unmap_skb(nic_dev, skb, txq->sges);
+
+		netif_stop_subqueue(netdev, qp->q_id);
+
+		u64_stats_update_begin(&txq->txq_stats.syncp);
+		txq->txq_stats.tx_busy++;
+		u64_stats_update_end(&txq->txq_stats.syncp);
+		err = NETDEV_TX_BUSY;
+		goto flush_skbs;
+	}
+
+	hinic_sq_prepare_wqe(txq->sq, prod_idx, sq_wqe, txq->sges, nr_sges);
+
+	hinic_sq_write_wqe(txq->sq, prod_idx, sq_wqe, skb, wqe_size);
+
+flush_skbs:
+	netdev_txq = netdev_get_tx_queue(netdev, skb->queue_mapping);
+	if ((!skb->xmit_more) || (netif_xmit_stopped(netdev_txq)))
+		hinic_sq_write_db(txq->sq, prod_idx, wqe_size, 0);
+
+	return err;
+
+skb_error:
+	dev_kfree_skb_any(skb);
+
+update_error_stats:
+	u64_stats_update_begin(&txq->txq_stats.syncp);
+	txq->txq_stats.tx_dropped++;
+	u64_stats_update_end(&txq->txq_stats.syncp);
+	return err;
+}
+
+/**
+ * tx_free_skb - unmap and free skb
+ * @nic_dev: nic device
+ * @skb: the skb
+ * @sges: the sges that are connected to the skb
+ **/
+static void tx_free_skb(struct hinic_dev *nic_dev, struct sk_buff *skb,
+			struct hinic_sge *sges)
+{
+	tx_unmap_skb(nic_dev, skb, sges);
+
+	dev_kfree_skb_any(skb);
+}
+
+/**
+ * free_all_rx_skbs - free all skbs in tx queue
+ * @txq: tx queue
+ **/
+static void free_all_tx_skbs(struct hinic_txq *txq)
+{
+	struct hinic_dev *nic_dev = netdev_priv(txq->netdev);
+	struct hinic_sq *sq = txq->sq;
+	struct hinic_sq_wqe *sq_wqe;
+	unsigned int wqe_size;
+	struct sk_buff *skb;
+	int nr_sges;
+	u16 ci;
+
+	while ((sq_wqe = hinic_sq_read_wqe(sq, &skb, &wqe_size, &ci))) {
+		nr_sges = skb_shinfo(skb)->nr_frags + 1;
+
+		hinic_sq_get_sges(sq_wqe, txq->free_sges, nr_sges);
+
+		hinic_sq_put_wqe(sq, wqe_size);
+
+		tx_free_skb(nic_dev, skb, txq->free_sges);
+	}
+}
+
+/**
+ * free_tx_poll - free finished tx skbs in tx queue that connected to napi
+ * @napi: napi
+ * @budget: number of tx
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+static int free_tx_poll(struct napi_struct *napi, int budget)
+{
+	struct hinic_txq *txq = container_of(napi, struct hinic_txq, napi);
+	struct hinic_qp *qp = container_of(txq->sq, struct hinic_qp, sq);
+	struct hinic_dev *nic_dev = netdev_priv(txq->netdev);
+	struct netdev_queue *netdev_txq;
+	struct hinic_sq *sq = txq->sq;
+	struct hinic_wq *wq = sq->wq;
+	struct hinic_sq_wqe *sq_wqe;
+	unsigned int wqe_size;
+	int nr_sges, pkts = 0;
+	struct sk_buff *skb;
+	u64 tx_bytes = 0;
+	u16 hw_ci, sw_ci;
+
+	do {
+		hw_ci = HW_CONS_IDX(sq) & wq->mask;
+
+		sq_wqe = hinic_sq_read_wqe(sq, &skb, &wqe_size, &sw_ci);
+		if ((!sq_wqe) ||
+		    (((hw_ci - sw_ci) & wq->mask) * wq->wqebb_size < wqe_size))
+			break;
+
+		tx_bytes += skb->len;
+		pkts++;
+
+		nr_sges = skb_shinfo(skb)->nr_frags + 1;
+
+		hinic_sq_get_sges(sq_wqe, txq->free_sges, nr_sges);
+
+		hinic_sq_put_wqe(sq, wqe_size);
+
+		tx_free_skb(nic_dev, skb, txq->free_sges);
+	} while (pkts < budget);
+
+	if (__netif_subqueue_stopped(nic_dev->netdev, qp->q_id) &&
+	    hinic_get_sq_free_wqebbs(sq) >= HINIC_MIN_TX_NUM_WQEBBS(sq)) {
+		netdev_txq = netdev_get_tx_queue(txq->netdev, qp->q_id);
+
+		__netif_tx_lock(netdev_txq, smp_processor_id());
+
+		netif_wake_subqueue(nic_dev->netdev, qp->q_id);
+
+		__netif_tx_unlock(netdev_txq);
+
+		u64_stats_update_begin(&txq->txq_stats.syncp);
+		txq->txq_stats.tx_wake++;
+		u64_stats_update_end(&txq->txq_stats.syncp);
+	}
+
+	u64_stats_update_begin(&txq->txq_stats.syncp);
+	txq->txq_stats.bytes += tx_bytes;
+	txq->txq_stats.pkts += pkts;
+	u64_stats_update_end(&txq->txq_stats.syncp);
+
+	if (pkts < budget) {
+		napi_complete(napi);
+		enable_irq(sq->irq);
+		return pkts;
+	}
+
+	return budget;
+}
+
+static void tx_napi_add(struct hinic_txq *txq, int weight)
+{
+	netif_napi_add(txq->netdev, &txq->napi, free_tx_poll, weight);
+	napi_enable(&txq->napi);
+}
+
+static void tx_napi_del(struct hinic_txq *txq)
+{
+	napi_disable(&txq->napi);
+	netif_napi_del(&txq->napi);
+}
+
+static irqreturn_t tx_irq(int irq, void *data)
+{
+	struct hinic_txq *txq = data;
+	struct hinic_dev *nic_dev;
+
+	nic_dev = netdev_priv(txq->netdev);
+
+	/* Disable the interrupt until napi will be completed */
+	disable_irq_nosync(txq->sq->irq);
+
+	hinic_hwdev_msix_cnt_set(nic_dev->hwdev, txq->sq->msix_entry);
+
+	napi_schedule(&txq->napi);
+	return IRQ_HANDLED;
+}
+
+static int tx_request_irq(struct hinic_txq *txq)
+{
+	struct hinic_dev *nic_dev = netdev_priv(txq->netdev);
+	struct hinic_hwdev *hwdev = nic_dev->hwdev;
+	struct hinic_hwif *hwif = hwdev->hwif;
+	struct pci_dev *pdev = hwif->pdev;
+	struct hinic_sq *sq = txq->sq;
+	int err;
+
+	tx_napi_add(txq, nic_dev->tx_weight);
+
+	hinic_hwdev_msix_set(nic_dev->hwdev, sq->msix_entry,
+			     TX_IRQ_NO_PENDING, TX_IRQ_NO_COALESC,
+			     TX_IRQ_NO_LLI_TIMER, TX_IRQ_NO_CREDIT,
+			     TX_IRQ_NO_RESEND_TIMER);
+
+	err = request_irq(sq->irq, tx_irq, 0, txq->irq_name, txq);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to request Tx irq\n");
+		tx_napi_del(txq);
+		return err;
+	}
+
+	return 0;
+}
+
+static void tx_free_irq(struct hinic_txq *txq)
+{
+	struct hinic_sq *sq = txq->sq;
+
+	free_irq(sq->irq, txq);
+	tx_napi_del(txq);
+}
+
+/**
+ * hinic_init_txq - Initialize the Tx Queue
+ * @txq: Logical Tx Queue
+ * @sq: Hardware Tx Queue to connect the Logical queue with
+ * @netdev: network device to connect the Logical queue with
+ *
+ * Return 0 - Success, negative - Failure
+ **/
+int hinic_init_txq(struct hinic_txq *txq, struct hinic_sq *sq,
+		   struct net_device *netdev)
+{
+	struct hinic_qp *qp = container_of(sq, struct hinic_qp, sq);
+	struct hinic_dev *nic_dev = netdev_priv(netdev);
+	struct hinic_hwdev *hwdev = nic_dev->hwdev;
+	int err, irqname_len;
+	size_t sges_size;
+
+	txq->netdev = netdev;
+	txq->sq = sq;
+
+	txq_stats_init(txq);
+
+	txq->max_sges = HINIC_MAX_SQ_BUFDESCS;
+
+	sges_size = txq->max_sges * sizeof(*txq->sges);
+	txq->sges = devm_kzalloc(&netdev->dev, sges_size, GFP_KERNEL);
+	if (!txq->sges)
+		return -ENOMEM;
+
+	sges_size = txq->max_sges * sizeof(*txq->free_sges);
+	txq->free_sges = devm_kzalloc(&netdev->dev, sges_size, GFP_KERNEL);
+	if (!txq->free_sges) {
+		err = -ENOMEM;
+		goto err_alloc_free_sges;
+	}
+
+	irqname_len = snprintf(NULL, 0, "hinic_txq%d", qp->q_id) + 1;
+	txq->irq_name = devm_kzalloc(&netdev->dev, irqname_len, GFP_KERNEL);
+	if (!txq->irq_name) {
+		err = -ENOMEM;
+		goto err_alloc_irqname;
+	}
+
+	sprintf(txq->irq_name, "hinic_txq%d", qp->q_id);
+
+	err = hinic_hwdev_hw_ci_addr_set(hwdev, sq, CI_UPDATE_NO_PENDING,
+					 CI_UPDATE_NO_COALESC);
+	if (err)
+		goto err_hw_ci;
+
+	err = tx_request_irq(txq);
+	if (err) {
+		netdev_err(netdev, "Failed to request Tx irq\n");
+		goto err_req_tx_irq;
+	}
+
+	return 0;
+
+err_req_tx_irq:
+err_hw_ci:
+	devm_kfree(&netdev->dev, txq->irq_name);
+
+err_alloc_irqname:
+	devm_kfree(&netdev->dev, txq->free_sges);
+
+err_alloc_free_sges:
+	devm_kfree(&netdev->dev, txq->sges);
+	return err;
+}
+
+/**
+ * hinic_clean_txq - Clean the Tx Queue
+ * @txq: Logical Tx Queue
+ **/
+void hinic_clean_txq(struct hinic_txq *txq)
+{
+	struct net_device *netdev = txq->netdev;
+
+	tx_free_irq(txq);
+
+	free_all_tx_skbs(txq);
+
+	devm_kfree(&netdev->dev, txq->irq_name);
+	devm_kfree(&netdev->dev, txq->free_sges);
+	devm_kfree(&netdev->dev, txq->sges);
+}
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_tx.h b/drivers/net/ethernet/huawei/hinic/hinic_tx.h
new file mode 100644
index 000000000000..1fa55dce5aa7
--- /dev/null
+++ b/drivers/net/ethernet/huawei/hinic/hinic_tx.h
@@ -0,0 +1,62 @@
+/*
+ * Huawei HiNIC PCI Express Linux driver
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ */
+
+#ifndef HINIC_TX_H
+#define HINIC_TX_H
+
+#include <linux/types.h>
+#include <linux/netdevice.h>
+#include <linux/skbuff.h>
+#include <linux/u64_stats_sync.h>
+
+#include "hinic_common.h"
+#include "hinic_hw_qp.h"
+
+struct hinic_txq_stats {
+	u64     pkts;
+	u64     bytes;
+	u64     tx_busy;
+	u64     tx_wake;
+	u64     tx_dropped;
+
+	struct u64_stats_sync   syncp;
+};
+
+struct hinic_txq {
+	struct net_device       *netdev;
+	struct hinic_sq         *sq;
+
+	struct hinic_txq_stats  txq_stats;
+
+	int                     max_sges;
+	struct hinic_sge        *sges;
+	struct hinic_sge        *free_sges;
+
+	char                    *irq_name;
+	struct napi_struct      napi;
+};
+
+void hinic_txq_clean_stats(struct hinic_txq *txq);
+
+void hinic_txq_get_stats(struct hinic_txq *txq, struct hinic_txq_stats *stats);
+
+netdev_tx_t hinic_xmit_frame(struct sk_buff *skb, struct net_device *netdev);
+
+int hinic_init_txq(struct hinic_txq *txq, struct hinic_sq *sq,
+		   struct net_device *netdev);
+
+void hinic_clean_txq(struct hinic_txq *txq);
+
+#endif
diff --git a/drivers/net/ethernet/i825xx/lasi_82596.c b/drivers/net/ethernet/i825xx/lasi_82596.c
index d787fdd5db7b..b69c622ba8b2 100644
--- a/drivers/net/ethernet/i825xx/lasi_82596.c
+++ b/drivers/net/ethernet/i825xx/lasi_82596.c
@@ -96,8 +96,6 @@
 
 #define OPT_SWAP_PORT	0x0001	/* Need to wordswp on the MPU port */
 
-#define DMA_ALLOC                        dma_alloc_noncoherent
-#define DMA_FREE                         dma_free_noncoherent
 #define DMA_WBACK(ndev, addr, len) \
 	do { dma_cache_sync((ndev)->dev.parent, (void *)addr, len, DMA_TO_DEVICE); } while (0)
 
@@ -149,7 +147,7 @@ static void mpu_port(struct net_device *dev, int c, dma_addr_t x)
 
 #define LAN_PROM_ADDR	0xF0810000
 
-static int
+static int __init
 lan_init_chip(struct parisc_device *dev)
 {
 	struct	net_device *netdevice;
@@ -194,19 +192,19 @@ lan_init_chip(struct parisc_device *dev)
 	return retval;
 }
 
-static int lan_remove_chip(struct parisc_device *pdev)
+static int __exit lan_remove_chip(struct parisc_device *pdev)
 {
 	struct net_device *dev = parisc_get_drvdata(pdev);
 	struct i596_private *lp = netdev_priv(dev);
 
 	unregister_netdev (dev);
-	DMA_FREE(&pdev->dev, sizeof(struct i596_private),
-		 (void *)lp->dma, lp->dma_addr);
+	dma_free_attrs(&pdev->dev, sizeof(struct i596_private), lp->dma,
+		       lp->dma_addr, DMA_ATTR_NON_CONSISTENT);
 	free_netdev (dev);
 	return 0;
 }
 
-static struct parisc_device_id lan_tbl[] = {
+static const struct parisc_device_id lan_tbl[] __initconst = {
 	{ HPHW_FIO, HVERSION_REV_ANY_ID, HVERSION_ANY_ID, 0x0008a },
 	{ HPHW_FIO, HVERSION_REV_ANY_ID, HVERSION_ANY_ID, 0x00072 },
 	{ 0, }
@@ -214,11 +212,11 @@ static struct parisc_device_id lan_tbl[] = {
 
 MODULE_DEVICE_TABLE(parisc, lan_tbl);
 
-static struct parisc_driver lan_driver = {
+static struct parisc_driver lan_driver __refdata = {
 	.name		= "lasi_82596",
 	.id_table	= lan_tbl,
 	.probe		= lan_init_chip,
-	.remove         = lan_remove_chip,
+	.remove         = __exit_p(lan_remove_chip),
 };
 
 static int lasi_82596_init(void)
diff --git a/drivers/net/ethernet/i825xx/lib82596.c b/drivers/net/ethernet/i825xx/lib82596.c
index 8449c58f01fd..f00a1dc2128c 100644
--- a/drivers/net/ethernet/i825xx/lib82596.c
+++ b/drivers/net/ethernet/i825xx/lib82596.c
@@ -1063,8 +1063,9 @@ static int i82596_probe(struct net_device *dev)
 	if (!dev->base_addr || !dev->irq)
 		return -ENODEV;
 
-	dma = (struct i596_dma *) DMA_ALLOC(dev->dev.parent,
-		sizeof(struct i596_dma), &lp->dma_addr, GFP_KERNEL);
+	dma = dma_alloc_attrs(dev->dev.parent, sizeof(struct i596_dma),
+			      &lp->dma_addr, GFP_KERNEL,
+			      DMA_ATTR_NON_CONSISTENT);
 	if (!dma) {
 		printk(KERN_ERR "%s: Couldn't get shared memory\n", __FILE__);
 		return -ENOMEM;
@@ -1085,8 +1086,8 @@ static int i82596_probe(struct net_device *dev)
 
 	i = register_netdev(dev);
 	if (i) {
-		DMA_FREE(dev->dev.parent, sizeof(struct i596_dma),
-				    (void *)dma, lp->dma_addr);
+		dma_free_attrs(dev->dev.parent, sizeof(struct i596_dma),
+			       dma, lp->dma_addr, DMA_ATTR_NON_CONSISTENT);
 		return i;
 	}
 
diff --git a/drivers/net/ethernet/i825xx/sni_82596.c b/drivers/net/ethernet/i825xx/sni_82596.c
index 2af7f77345fb..b2c04a789744 100644
--- a/drivers/net/ethernet/i825xx/sni_82596.c
+++ b/drivers/net/ethernet/i825xx/sni_82596.c
@@ -23,8 +23,6 @@
 
 static const char sni_82596_string[] = "snirm_82596";
 
-#define DMA_ALLOC                      dma_alloc_coherent
-#define DMA_FREE                       dma_free_coherent
 #define DMA_WBACK(priv, addr, len)     do { } while (0)
 #define DMA_INV(priv, addr, len)       do { } while (0)
 #define DMA_WBACK_INV(priv, addr, len) do { } while (0)
@@ -152,8 +150,8 @@ static int sni_82596_driver_remove(struct platform_device *pdev)
 	struct i596_private *lp = netdev_priv(dev);
 
 	unregister_netdev(dev);
-	DMA_FREE(dev->dev.parent, sizeof(struct i596_private),
-		 lp->dma, lp->dma_addr);
+	dma_free_attrs(dev->dev.parent, sizeof(struct i596_private), lp->dma,
+		       lp->dma_addr, DMA_ATTR_NON_CONSISTENT);
 	iounmap(lp->ca);
 	iounmap(lp->mpu_port);
 	free_netdev (dev);
diff --git a/drivers/net/ethernet/ibm/ehea/ehea_main.c b/drivers/net/ethernet/ibm/ehea/ehea_main.c
index b9d310f20bcc..4878b7169e0f 100644
--- a/drivers/net/ethernet/ibm/ehea/ehea_main.c
+++ b/drivers/net/ethernet/ibm/ehea/ehea_main.c
@@ -3102,8 +3102,7 @@ static int ehea_setup_ports(struct ehea_adapter *adapter)
 		dn_log_port_id = of_get_property(eth_dn, "ibm,hea-port-no",
 						 NULL);
 		if (!dn_log_port_id) {
-			pr_err("bad device node: eth_dn name=%s\n",
-			       eth_dn->full_name);
+			pr_err("bad device node: eth_dn name=%pOF\n", eth_dn);
 			continue;
 		}
 
@@ -3425,7 +3424,7 @@ static int ehea_probe_adapter(struct platform_device *dev)
 
 	if (!adapter->handle) {
 		dev_err(&dev->dev, "failed getting handle for adapter"
-			" '%s'\n", dev->dev.of_node->full_name);
+			" '%pOF'\n", dev->dev.of_node);
 		ret = -ENODEV;
 		goto out_free_ad;
 	}
diff --git a/drivers/net/ethernet/ibm/emac/core.c b/drivers/net/ethernet/ibm/emac/core.c
index 259e69a52ec5..7feff2450ed6 100644
--- a/drivers/net/ethernet/ibm/emac/core.c
+++ b/drivers/net/ethernet/ibm/emac/core.c
@@ -133,8 +133,7 @@ static inline void emac_report_timeout_error(struct emac_instance *dev,
 				  EMAC_FTR_440EP_PHY_CLK_FIX))
 		DBG(dev, "%s" NL, error);
 	else if (net_ratelimit())
-		printk(KERN_ERR "%s: %s\n", dev->ofdev->dev.of_node->full_name,
-			error);
+		printk(KERN_ERR "%pOF: %s\n", dev->ofdev->dev.of_node, error);
 }
 
 /* EMAC PHY clock workaround:
@@ -2258,8 +2257,8 @@ static void emac_ethtool_get_drvinfo(struct net_device *ndev,
 
 	strlcpy(info->driver, "ibm_emac", sizeof(info->driver));
 	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
-	snprintf(info->bus_info, sizeof(info->bus_info), "PPC 4xx EMAC-%d %s",
-		 dev->cell_index, dev->ofdev->dev.of_node->full_name);
+	snprintf(info->bus_info, sizeof(info->bus_info), "PPC 4xx EMAC-%d %pOF",
+		 dev->cell_index, dev->ofdev->dev.of_node);
 }
 
 static const struct ethtool_ops emac_ethtool_ops = {
@@ -2431,8 +2430,8 @@ static int emac_read_uint_prop(struct device_node *np, const char *name,
 	const u32 *prop = of_get_property(np, name, &len);
 	if (prop == NULL || len < sizeof(u32)) {
 		if (fatal)
-			printk(KERN_ERR "%s: missing %s property\n",
-			       np->full_name, name);
+			printk(KERN_ERR "%pOF: missing %s property\n",
+			       np, name);
 		return -ENODEV;
 	}
 	*val = *prop;
@@ -2768,7 +2767,7 @@ static int emac_init_phy(struct emac_instance *dev)
 #endif
 	mutex_unlock(&emac_phy_map_lock);
 	if (i == 0x20) {
-		printk(KERN_WARNING "%s: can't find PHY!\n", np->full_name);
+		printk(KERN_WARNING "%pOF: can't find PHY!\n", np);
 		return -ENXIO;
 	}
 
@@ -2894,8 +2893,8 @@ static int emac_init_config(struct emac_instance *dev)
 #ifdef CONFIG_IBM_EMAC_NO_FLOW_CTRL
 			dev->features |= EMAC_FTR_NO_FLOW_CONTROL_40x;
 #else
-			printk(KERN_ERR "%s: Flow control not disabled!\n",
-					np->full_name);
+			printk(KERN_ERR "%pOF: Flow control not disabled!\n",
+					np);
 			return -ENXIO;
 #endif
 		}
@@ -2918,8 +2917,7 @@ static int emac_init_config(struct emac_instance *dev)
 #ifdef CONFIG_IBM_EMAC_TAH
 		dev->features |= EMAC_FTR_HAS_TAH;
 #else
-		printk(KERN_ERR "%s: TAH support not enabled !\n",
-		       np->full_name);
+		printk(KERN_ERR "%pOF: TAH support not enabled !\n", np);
 		return -ENXIO;
 #endif
 	}
@@ -2928,8 +2926,7 @@ static int emac_init_config(struct emac_instance *dev)
 #ifdef CONFIG_IBM_EMAC_ZMII
 		dev->features |= EMAC_FTR_HAS_ZMII;
 #else
-		printk(KERN_ERR "%s: ZMII support not enabled !\n",
-		       np->full_name);
+		printk(KERN_ERR "%pOF: ZMII support not enabled !\n", np);
 		return -ENXIO;
 #endif
 	}
@@ -2938,8 +2935,7 @@ static int emac_init_config(struct emac_instance *dev)
 #ifdef CONFIG_IBM_EMAC_RGMII
 		dev->features |= EMAC_FTR_HAS_RGMII;
 #else
-		printk(KERN_ERR "%s: RGMII support not enabled !\n",
-		       np->full_name);
+		printk(KERN_ERR "%pOF: RGMII support not enabled !\n", np);
 		return -ENXIO;
 #endif
 	}
@@ -2947,8 +2943,8 @@ static int emac_init_config(struct emac_instance *dev)
 	/* Read MAC-address */
 	p = of_get_property(np, "local-mac-address", NULL);
 	if (p == NULL) {
-		printk(KERN_ERR "%s: Can't find local-mac-address property\n",
-		       np->full_name);
+		printk(KERN_ERR "%pOF: Can't find local-mac-address property\n",
+		       np);
 		return -ENXIO;
 	}
 	memcpy(dev->ndev->dev_addr, p, ETH_ALEN);
@@ -3036,30 +3032,24 @@ static int emac_probe(struct platform_device *ofdev)
 
 	/* Init various config data based on device-tree */
 	err = emac_init_config(dev);
-	if (err != 0)
+	if (err)
 		goto err_free;
 
 	/* Get interrupts. EMAC irq is mandatory, WOL irq is optional */
 	dev->emac_irq = irq_of_parse_and_map(np, 0);
 	dev->wol_irq = irq_of_parse_and_map(np, 1);
 	if (!dev->emac_irq) {
-		printk(KERN_ERR "%s: Can't map main interrupt\n", np->full_name);
+		printk(KERN_ERR "%pOF: Can't map main interrupt\n", np);
+		err = -ENODEV;
 		goto err_free;
 	}
 	ndev->irq = dev->emac_irq;
 
 	/* Map EMAC regs */
-	if (of_address_to_resource(np, 0, &dev->rsrc_regs)) {
-		printk(KERN_ERR "%s: Can't get registers address\n",
-		       np->full_name);
-		goto err_irq_unmap;
-	}
-	// TODO : request_mem_region
-	dev->emacp = ioremap(dev->rsrc_regs.start,
-			     resource_size(&dev->rsrc_regs));
+	// TODO : platform_get_resource() and devm_ioremap_resource()
+	dev->emacp = of_iomap(np, 0);
 	if (dev->emacp == NULL) {
-		printk(KERN_ERR "%s: Can't map device registers!\n",
-		       np->full_name);
+		printk(KERN_ERR "%pOF: Can't map device registers!\n", np);
 		err = -ENOMEM;
 		goto err_irq_unmap;
 	}
@@ -3068,8 +3058,7 @@ static int emac_probe(struct platform_device *ofdev)
 	err = emac_wait_deps(dev);
 	if (err) {
 		printk(KERN_ERR
-		       "%s: Timeout waiting for dependent devices\n",
-		       np->full_name);
+		       "%pOF: Timeout waiting for dependent devices\n", np);
 		/*  display more info about what's missing ? */
 		goto err_reg_unmap;
 	}
@@ -3084,8 +3073,8 @@ static int emac_probe(struct platform_device *ofdev)
 	dev->commac.rx_chan_mask = MAL_CHAN_MASK(dev->mal_rx_chan);
 	err = mal_register_commac(dev->mal, &dev->commac);
 	if (err) {
-		printk(KERN_ERR "%s: failed to register with mal %s!\n",
-		       np->full_name, dev->mal_dev->dev.of_node->full_name);
+		printk(KERN_ERR "%pOF: failed to register with mal %pOF!\n",
+		       np, dev->mal_dev->dev.of_node);
 		goto err_rel_deps;
 	}
 	dev->rx_skb_size = emac_rx_skb_size(ndev->mtu);
@@ -3161,8 +3150,8 @@ static int emac_probe(struct platform_device *ofdev)
 
 	err = register_netdev(ndev);
 	if (err) {
-		printk(KERN_ERR "%s: failed to register net device (%d)!\n",
-		       np->full_name, err);
+		printk(KERN_ERR "%pOF: failed to register net device (%d)!\n",
+		       np, err);
 		goto err_detach_tah;
 	}
 
@@ -3176,8 +3165,8 @@ static int emac_probe(struct platform_device *ofdev)
 	wake_up_all(&emac_probe_wait);
 
 
-	printk(KERN_INFO "%s: EMAC-%d %s, MAC %pM\n",
-	       ndev->name, dev->cell_index, np->full_name, ndev->dev_addr);
+	printk(KERN_INFO "%s: EMAC-%d %pOF, MAC %pM\n",
+	       ndev->name, dev->cell_index, np, ndev->dev_addr);
 
 	if (dev->phy_mode == PHY_MODE_SGMII)
 		printk(KERN_NOTICE "%s: in SGMII mode\n", ndev->name);
diff --git a/drivers/net/ethernet/ibm/emac/core.h b/drivers/net/ethernet/ibm/emac/core.h
index f10e156641d5..369de2cfb15b 100644
--- a/drivers/net/ethernet/ibm/emac/core.h
+++ b/drivers/net/ethernet/ibm/emac/core.h
@@ -167,7 +167,6 @@ struct emac_error_stats {
 
 struct emac_instance {
 	struct net_device		*ndev;
-	struct resource			rsrc_regs;
 	struct emac_regs		__iomem *emacp;
 	struct platform_device		*ofdev;
 	struct device_node		**blist; /* bootlist entry */
diff --git a/drivers/net/ethernet/ibm/emac/debug.h b/drivers/net/ethernet/ibm/emac/debug.h
index 5bdfc174a07e..9d06d3be3161 100644
--- a/drivers/net/ethernet/ibm/emac/debug.h
+++ b/drivers/net/ethernet/ibm/emac/debug.h
@@ -31,7 +31,7 @@
 #endif
 
 #define EMAC_DBG(d, name, fmt, arg...) \
-	printk(KERN_DEBUG #name "%s: " fmt, d->ofdev->dev.of_node->full_name, ## arg)
+	printk(KERN_DEBUG #name "%pOF: " fmt, d->ofdev->dev.of_node, ## arg)
 
 #if DBG_LEVEL > 0
 #  define DBG(d,f,x...)		EMAC_DBG(d, emac, f, ##x)
diff --git a/drivers/net/ethernet/ibm/emac/mal.c b/drivers/net/ethernet/ibm/emac/mal.c
index 91b1a558f37d..2c74baa2398a 100644
--- a/drivers/net/ethernet/ibm/emac/mal.c
+++ b/drivers/net/ethernet/ibm/emac/mal.c
@@ -579,8 +579,8 @@ static int mal_probe(struct platform_device *ofdev)
 		mal->features |= (MAL_FTR_CLEAR_ICINTSTAT |
 				MAL_FTR_COMMON_ERR_INT);
 #else
-		printk(KERN_ERR "%s: Support for 405EZ not enabled!\n",
-				ofdev->dev.of_node->full_name);
+		printk(KERN_ERR "%pOF: Support for 405EZ not enabled!\n",
+				ofdev->dev.of_node);
 		err = -ENODEV;
 		goto fail;
 #endif
@@ -687,8 +687,8 @@ static int mal_probe(struct platform_device *ofdev)
 	mal_enable_eob_irq(mal);
 
 	printk(KERN_INFO
-	       "MAL v%d %s, %d TX channels, %d RX channels\n",
-	       mal->version, ofdev->dev.of_node->full_name,
+	       "MAL v%d %pOF, %d TX channels, %d RX channels\n",
+	       mal->version, ofdev->dev.of_node,
 	       mal->num_tx_chans, mal->num_rx_chans);
 
 	/* Advertise this instance to the rest of the world */
diff --git a/drivers/net/ethernet/ibm/emac/rgmii.c b/drivers/net/ethernet/ibm/emac/rgmii.c
index 206ccbbae7bb..c4a1ac38bba8 100644
--- a/drivers/net/ethernet/ibm/emac/rgmii.c
+++ b/drivers/net/ethernet/ibm/emac/rgmii.c
@@ -104,8 +104,8 @@ int rgmii_attach(struct platform_device *ofdev, int input, int mode)
 
 	/* Check if we need to attach to a RGMII */
 	if (input < 0 || !rgmii_valid_mode(mode)) {
-		printk(KERN_ERR "%s: unsupported settings !\n",
-		       ofdev->dev.of_node->full_name);
+		printk(KERN_ERR "%pOF: unsupported settings !\n",
+		       ofdev->dev.of_node);
 		return -ENODEV;
 	}
 
@@ -114,8 +114,8 @@ int rgmii_attach(struct platform_device *ofdev, int input, int mode)
 	/* Enable this input */
 	out_be32(&p->fer, in_be32(&p->fer) | rgmii_mode_mask(mode, input));
 
-	printk(KERN_NOTICE "%s: input %d in %s mode\n",
-	       ofdev->dev.of_node->full_name, input, rgmii_mode_name(mode));
+	printk(KERN_NOTICE "%pOF: input %d in %s mode\n",
+	       ofdev->dev.of_node, input, rgmii_mode_name(mode));
 
 	++dev->users;
 
@@ -249,8 +249,7 @@ static int rgmii_probe(struct platform_device *ofdev)
 
 	rc = -ENXIO;
 	if (of_address_to_resource(np, 0, &regs)) {
-		printk(KERN_ERR "%s: Can't get registers address\n",
-		       np->full_name);
+		printk(KERN_ERR "%pOF: Can't get registers address\n", np);
 		goto err_free;
 	}
 
@@ -258,8 +257,7 @@ static int rgmii_probe(struct platform_device *ofdev)
 	dev->base = (struct rgmii_regs __iomem *)ioremap(regs.start,
 						 sizeof(struct rgmii_regs));
 	if (dev->base == NULL) {
-		printk(KERN_ERR "%s: Can't map device registers!\n",
-		       np->full_name);
+		printk(KERN_ERR "%pOF: Can't map device registers!\n", np);
 		goto err_free;
 	}
 
@@ -278,8 +276,8 @@ static int rgmii_probe(struct platform_device *ofdev)
 	out_be32(&dev->base->fer, 0);
 
 	printk(KERN_INFO
-	       "RGMII %s initialized with%s MDIO support\n",
-	       ofdev->dev.of_node->full_name,
+	       "RGMII %pOF initialized with%s MDIO support\n",
+	       ofdev->dev.of_node,
 	       (dev->flags & EMAC_RGMII_FLAG_HAS_MDIO) ? "" : "out");
 
 	wmb();
diff --git a/drivers/net/ethernet/ibm/emac/tah.c b/drivers/net/ethernet/ibm/emac/tah.c
index 32cb6c9007c5..9912456dca48 100644
--- a/drivers/net/ethernet/ibm/emac/tah.c
+++ b/drivers/net/ethernet/ibm/emac/tah.c
@@ -58,8 +58,7 @@ void tah_reset(struct platform_device *ofdev)
 		--n;
 
 	if (unlikely(!n))
-		printk(KERN_ERR "%s: reset timeout\n",
-			ofdev->dev.of_node->full_name);
+		printk(KERN_ERR "%pOF: reset timeout\n", ofdev->dev.of_node);
 
 	/* 10KB TAH TX FIFO accommodates the max MTU of 9000 */
 	out_be32(&p->mr,
@@ -105,8 +104,7 @@ static int tah_probe(struct platform_device *ofdev)
 
 	rc = -ENXIO;
 	if (of_address_to_resource(np, 0, &regs)) {
-		printk(KERN_ERR "%s: Can't get registers address\n",
-		       np->full_name);
+		printk(KERN_ERR "%pOF: Can't get registers address\n", np);
 		goto err_free;
 	}
 
@@ -114,8 +112,7 @@ static int tah_probe(struct platform_device *ofdev)
 	dev->base = (struct tah_regs __iomem *)ioremap(regs.start,
 					       sizeof(struct tah_regs));
 	if (dev->base == NULL) {
-		printk(KERN_ERR "%s: Can't map device registers!\n",
-		       np->full_name);
+		printk(KERN_ERR "%pOF: Can't map device registers!\n", np);
 		goto err_free;
 	}
 
@@ -124,8 +121,7 @@ static int tah_probe(struct platform_device *ofdev)
 	/* Initialize TAH and enable IPv4 checksum verification, no TSO yet */
 	tah_reset(ofdev);
 
-	printk(KERN_INFO
-	       "TAH %s initialized\n", ofdev->dev.of_node->full_name);
+	printk(KERN_INFO "TAH %pOF initialized\n", ofdev->dev.of_node);
 	wmb();
 
 	return 0;
diff --git a/drivers/net/ethernet/ibm/emac/zmii.c b/drivers/net/ethernet/ibm/emac/zmii.c
index 8727b865ea02..89c42d362292 100644
--- a/drivers/net/ethernet/ibm/emac/zmii.c
+++ b/drivers/net/ethernet/ibm/emac/zmii.c
@@ -121,15 +121,15 @@ int zmii_attach(struct platform_device *ofdev, int input, int *mode)
 		} else
 			dev->mode = *mode;
 
-		printk(KERN_NOTICE "%s: bridge in %s mode\n",
-		       ofdev->dev.of_node->full_name,
+		printk(KERN_NOTICE "%pOF: bridge in %s mode\n",
+		       ofdev->dev.of_node,
 		       zmii_mode_name(dev->mode));
 	} else {
 		/* All inputs must use the same mode */
 		if (*mode != PHY_MODE_NA && *mode != dev->mode) {
 			printk(KERN_ERR
-			       "%s: invalid mode %d specified for input %d\n",
-			       ofdev->dev.of_node->full_name, *mode, input);
+			       "%pOF: invalid mode %d specified for input %d\n",
+			       ofdev->dev.of_node, *mode, input);
 			mutex_unlock(&dev->lock);
 			return -EINVAL;
 		}
@@ -250,8 +250,7 @@ static int zmii_probe(struct platform_device *ofdev)
 
 	rc = -ENXIO;
 	if (of_address_to_resource(np, 0, &regs)) {
-		printk(KERN_ERR "%s: Can't get registers address\n",
-		       np->full_name);
+		printk(KERN_ERR "%pOF: Can't get registers address\n", np);
 		goto err_free;
 	}
 
@@ -259,8 +258,7 @@ static int zmii_probe(struct platform_device *ofdev)
 	dev->base = (struct zmii_regs __iomem *)ioremap(regs.start,
 						sizeof(struct zmii_regs));
 	if (dev->base == NULL) {
-		printk(KERN_ERR "%s: Can't map device registers!\n",
-		       np->full_name);
+		printk(KERN_ERR "%pOF: Can't map device registers!\n", np);
 		goto err_free;
 	}
 
@@ -270,8 +268,7 @@ static int zmii_probe(struct platform_device *ofdev)
 	/* Disable all inputs by default */
 	out_be32(&dev->base->fer, 0);
 
-	printk(KERN_INFO
-	       "ZMII %s initialized\n", ofdev->dev.of_node->full_name);
+	printk(KERN_INFO "ZMII %pOF initialized\n", ofdev->dev.of_node);
 	wmb();
 	platform_set_drvdata(ofdev, dev);
 
diff --git a/drivers/net/ethernet/ibm/ibmveth.c b/drivers/net/ethernet/ibm/ibmveth.c
index d17c2b03f580..f210398200ec 100644
--- a/drivers/net/ethernet/ibm/ibmveth.c
+++ b/drivers/net/ethernet/ibm/ibmveth.c
@@ -1897,7 +1897,7 @@ static int ibmveth_resume(struct device *dev)
 	return 0;
 }
 
-static struct vio_device_id ibmveth_device_table[] = {
+static const struct vio_device_id ibmveth_device_table[] = {
 	{ "network", "IBM,l-lan"},
 	{ "", "" }
 };
diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c
index c45e8e3b82d3..cb8182f4fdfa 100644
--- a/drivers/net/ethernet/ibm/ibmvnic.c
+++ b/drivers/net/ethernet/ibm/ibmvnic.c
@@ -347,6 +347,31 @@ static void replenish_pools(struct ibmvnic_adapter *adapter)
 	}
 }
 
+static void release_stats_buffers(struct ibmvnic_adapter *adapter)
+{
+	kfree(adapter->tx_stats_buffers);
+	kfree(adapter->rx_stats_buffers);
+}
+
+static int init_stats_buffers(struct ibmvnic_adapter *adapter)
+{
+	adapter->tx_stats_buffers =
+				kcalloc(adapter->req_tx_queues,
+					sizeof(struct ibmvnic_tx_queue_stats),
+					GFP_KERNEL);
+	if (!adapter->tx_stats_buffers)
+		return -ENOMEM;
+
+	adapter->rx_stats_buffers =
+				kcalloc(adapter->req_rx_queues,
+					sizeof(struct ibmvnic_rx_queue_stats),
+					GFP_KERNEL);
+	if (!adapter->rx_stats_buffers)
+		return -ENOMEM;
+
+	return 0;
+}
+
 static void release_stats_token(struct ibmvnic_adapter *adapter)
 {
 	struct device *dev = &adapter->vdev->dev;
@@ -374,6 +399,7 @@ static int init_stats_token(struct ibmvnic_adapter *adapter)
 	}
 
 	adapter->stats_token = stok;
+	netdev_dbg(adapter->netdev, "Stats token initialized (%llx)\n", stok);
 	return 0;
 }
 
@@ -387,6 +413,8 @@ static int reset_rx_pools(struct ibmvnic_adapter *adapter)
 	for (i = 0; i < rx_scrqs; i++) {
 		rx_pool = &adapter->rx_pool[i];
 
+		netdev_dbg(adapter->netdev, "Re-setting rx_pool[%d]\n", i);
+
 		rc = reset_long_term_buff(adapter, &rx_pool->long_term_buff);
 		if (rc)
 			return rc;
@@ -419,6 +447,8 @@ static void release_rx_pools(struct ibmvnic_adapter *adapter)
 	for (i = 0; i < rx_scrqs; i++) {
 		rx_pool = &adapter->rx_pool[i];
 
+		netdev_dbg(adapter->netdev, "Releasing rx_pool[%d]\n", i);
+
 		kfree(rx_pool->free_map);
 		free_long_term_buff(adapter, &rx_pool->long_term_buff);
 
@@ -465,7 +495,7 @@ static int init_rx_pools(struct net_device *netdev)
 		rx_pool = &adapter->rx_pool[i];
 
 		netdev_dbg(adapter->netdev,
-			   "Initializing rx_pool %d, %lld buffs, %lld bytes each\n",
+			   "Initializing rx_pool[%d], %lld buffs, %lld bytes each\n",
 			   i, adapter->req_rx_add_entries_per_subcrq,
 			   be64_to_cpu(size_array[i]));
 
@@ -515,6 +545,8 @@ static int reset_tx_pools(struct ibmvnic_adapter *adapter)
 
 	tx_scrqs = be32_to_cpu(adapter->login_rsp_buf->num_txsubm_subcrqs);
 	for (i = 0; i < tx_scrqs; i++) {
+		netdev_dbg(adapter->netdev, "Re-setting tx_pool[%d]\n", i);
+
 		tx_pool = &adapter->tx_pool[i];
 
 		rc = reset_long_term_buff(adapter, &tx_pool->long_term_buff);
@@ -545,6 +577,7 @@ static void release_tx_pools(struct ibmvnic_adapter *adapter)
 
 	tx_scrqs = be32_to_cpu(adapter->login_rsp_buf->num_txsubm_subcrqs);
 	for (i = 0; i < tx_scrqs; i++) {
+		netdev_dbg(adapter->netdev, "Releasing tx_pool[%d]\n", i);
 		tx_pool = &adapter->tx_pool[i];
 		kfree(tx_pool->tx_buff);
 		free_long_term_buff(adapter, &tx_pool->long_term_buff);
@@ -571,6 +604,11 @@ static int init_tx_pools(struct net_device *netdev)
 
 	for (i = 0; i < tx_subcrqs; i++) {
 		tx_pool = &adapter->tx_pool[i];
+
+		netdev_dbg(adapter->netdev,
+			   "Initializing tx_pool[%d], %lld buffs\n",
+			   i, adapter->req_tx_entries_per_subcrq);
+
 		tx_pool->tx_buff = kcalloc(adapter->req_tx_entries_per_subcrq,
 					   sizeof(struct ibmvnic_tx_buff),
 					   GFP_KERNEL);
@@ -641,8 +679,10 @@ static void ibmvnic_napi_disable(struct ibmvnic_adapter *adapter)
 	if (!adapter->napi_enabled)
 		return;
 
-	for (i = 0; i < adapter->req_rx_queues; i++)
+	for (i = 0; i < adapter->req_rx_queues; i++) {
+		netdev_dbg(adapter->netdev, "Disabling napi[%d]\n", i);
 		napi_disable(&adapter->napi[i]);
+	}
 
 	adapter->napi_enabled = false;
 }
@@ -700,12 +740,16 @@ static void release_resources(struct ibmvnic_adapter *adapter)
 	release_rx_pools(adapter);
 
 	release_stats_token(adapter);
+	release_stats_buffers(adapter);
 	release_error_buffers(adapter);
 
 	if (adapter->napi) {
 		for (i = 0; i < adapter->req_rx_queues; i++) {
-			if (&adapter->napi[i])
+			if (&adapter->napi[i]) {
+				netdev_dbg(adapter->netdev,
+					   "Releasing napi[%d]\n", i);
 				netif_napi_del(&adapter->napi[i]);
+			}
 		}
 	}
 }
@@ -718,7 +762,8 @@ static int set_link_state(struct ibmvnic_adapter *adapter, u8 link_state)
 	bool resend;
 	int rc;
 
-	netdev_err(netdev, "setting link state %d\n", link_state);
+	netdev_dbg(netdev, "setting link state %d\n", link_state);
+
 	memset(&crq, 0, sizeof(crq));
 	crq.logical_link_state.first = IBMVNIC_CRQ_CMD;
 	crq.logical_link_state.cmd = LOGICAL_LINK_STATE;
@@ -755,6 +800,9 @@ static int set_real_num_queues(struct net_device *netdev)
 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
 	int rc;
 
+	netdev_dbg(netdev, "Setting real tx/rx queues (%llx/%llx)\n",
+		   adapter->req_tx_queues, adapter->req_rx_queues);
+
 	rc = netif_set_real_num_tx_queues(netdev, adapter->req_tx_queues);
 	if (rc) {
 		netdev_err(netdev, "failed to set the number of tx queues\n");
@@ -777,6 +825,10 @@ static int init_resources(struct ibmvnic_adapter *adapter)
 	if (rc)
 		return rc;
 
+	rc = init_stats_buffers(adapter);
+	if (rc)
+		return rc;
+
 	rc = init_stats_token(adapter);
 	if (rc)
 		return rc;
@@ -788,6 +840,7 @@ static int init_resources(struct ibmvnic_adapter *adapter)
 		return -ENOMEM;
 
 	for (i = 0; i < adapter->req_rx_queues; i++) {
+		netdev_dbg(netdev, "Adding napi[%d]\n", i);
 		netif_napi_add(netdev, &adapter->napi[i], ibmvnic_poll,
 			       NAPI_POLL_WEIGHT);
 	}
@@ -816,6 +869,7 @@ static int __ibmvnic_open(struct net_device *netdev)
 	 * set the logical link state to up
 	 */
 	for (i = 0; i < adapter->req_rx_queues; i++) {
+		netdev_dbg(netdev, "Enabling rx_scrq[%d] irq\n", i);
 		if (prev_state == VNIC_CLOSED)
 			enable_irq(adapter->rx_scrq[i]->irq);
 		else
@@ -823,6 +877,7 @@ static int __ibmvnic_open(struct net_device *netdev)
 	}
 
 	for (i = 0; i < adapter->req_tx_queues; i++) {
+		netdev_dbg(netdev, "Enabling tx_scrq[%d] irq\n", i);
 		if (prev_state == VNIC_CLOSED)
 			enable_irq(adapter->tx_scrq[i]->irq);
 		else
@@ -896,6 +951,7 @@ static void clean_tx_pools(struct ibmvnic_adapter *adapter)
 		if (!tx_pool)
 			continue;
 
+		netdev_dbg(adapter->netdev, "Cleaning tx_pool[%d]\n", i);
 		for (j = 0; j < tx_entries; j++) {
 			if (tx_pool->tx_buff[j].skb) {
 				dev_kfree_skb_any(tx_pool->tx_buff[j].skb);
@@ -923,8 +979,11 @@ static int __ibmvnic_close(struct net_device *netdev)
 
 	if (adapter->tx_scrq) {
 		for (i = 0; i < adapter->req_tx_queues; i++)
-			if (adapter->tx_scrq[i]->irq)
+			if (adapter->tx_scrq[i]->irq) {
+				netdev_dbg(adapter->netdev,
+					   "Disabling tx_scrq[%d] irq\n", i);
 				disable_irq(adapter->tx_scrq[i]->irq);
+			}
 	}
 
 	rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_DN);
@@ -943,8 +1002,11 @@ static int __ibmvnic_close(struct net_device *netdev)
 					break;
 			}
 
-			if (adapter->rx_scrq[i]->irq)
+			if (adapter->rx_scrq[i]->irq) {
+				netdev_dbg(adapter->netdev,
+					   "Disabling rx_scrq[%d] irq\n", i);
 				disable_irq(adapter->rx_scrq[i]->irq);
+			}
 		}
 	}
 
@@ -1259,6 +1321,9 @@ out:
 	netdev->stats.tx_packets += tx_packets;
 	adapter->tx_send_failed += tx_send_failed;
 	adapter->tx_map_failed += tx_map_failed;
+	adapter->tx_stats_buffers[queue_num].packets += tx_packets;
+	adapter->tx_stats_buffers[queue_num].bytes += tx_bytes;
+	adapter->tx_stats_buffers[queue_num].dropped_packets += tx_dropped;
 
 	return ret;
 }
@@ -1334,6 +1399,9 @@ static int do_reset(struct ibmvnic_adapter *adapter,
 	struct net_device *netdev = adapter->netdev;
 	int i, rc;
 
+	netdev_dbg(adapter->netdev, "Re-setting driver (%d)\n",
+		   rwi->reset_reason);
+
 	netif_carrier_off(netdev);
 	adapter->reset_reason = rwi->reset_reason;
 
@@ -1458,6 +1526,7 @@ static void __ibmvnic_reset(struct work_struct *work)
 	}
 
 	if (rc) {
+		netdev_dbg(adapter->netdev, "Reset failed\n");
 		free_all_rwi(adapter);
 		mutex_unlock(&adapter->reset_lock);
 		return;
@@ -1491,7 +1560,7 @@ static void ibmvnic_reset(struct ibmvnic_adapter *adapter,
 	list_for_each(entry, &adapter->rwi_list) {
 		tmp = list_entry(entry, struct ibmvnic_rwi, list);
 		if (tmp->reset_reason == reason) {
-			netdev_err(netdev, "Matching reset found, skipping\n");
+			netdev_dbg(netdev, "Skipping matching reset\n");
 			mutex_unlock(&adapter->rwi_lock);
 			return;
 		}
@@ -1507,6 +1576,8 @@ static void ibmvnic_reset(struct ibmvnic_adapter *adapter,
 	rwi->reset_reason = reason;
 	list_add_tail(&rwi->list, &adapter->rwi_list);
 	mutex_unlock(&adapter->rwi_lock);
+
+	netdev_dbg(adapter->netdev, "Scheduling reset (reason %d)\n", reason);
 	schedule_work(&adapter->ibmvnic_reset);
 }
 
@@ -1560,7 +1631,8 @@ restart_poll:
 							  rx_comp.correlator);
 		/* do error checking */
 		if (next->rx_comp.rc) {
-			netdev_err(netdev, "rx error %x\n", next->rx_comp.rc);
+			netdev_dbg(netdev, "rx buffer returned with rc %x\n",
+				   be16_to_cpu(next->rx_comp.rc));
 			/* free the entry */
 			next->rx_comp.first = 0;
 			remove_buff_from_pool(adapter, rx_buff);
@@ -1599,6 +1671,8 @@ restart_poll:
 		napi_gro_receive(napi, skb); /* send it up */
 		netdev->stats.rx_packets++;
 		netdev->stats.rx_bytes += length;
+		adapter->rx_stats_buffers[scrq_num].packets++;
+		adapter->rx_stats_buffers[scrq_num].bytes += length;
 		frames_processed++;
 	}
 
@@ -1708,18 +1782,36 @@ static u32 ibmvnic_get_link(struct net_device *netdev)
 static void ibmvnic_get_ringparam(struct net_device *netdev,
 				  struct ethtool_ringparam *ring)
 {
-	ring->rx_max_pending = 0;
-	ring->tx_max_pending = 0;
+	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
+
+	ring->rx_max_pending = adapter->max_rx_add_entries_per_subcrq;
+	ring->tx_max_pending = adapter->max_tx_entries_per_subcrq;
 	ring->rx_mini_max_pending = 0;
 	ring->rx_jumbo_max_pending = 0;
-	ring->rx_pending = 0;
-	ring->tx_pending = 0;
+	ring->rx_pending = adapter->req_rx_add_entries_per_subcrq;
+	ring->tx_pending = adapter->req_tx_entries_per_subcrq;
 	ring->rx_mini_pending = 0;
 	ring->rx_jumbo_pending = 0;
 }
 
+static void ibmvnic_get_channels(struct net_device *netdev,
+				 struct ethtool_channels *channels)
+{
+	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
+
+	channels->max_rx = adapter->max_rx_queues;
+	channels->max_tx = adapter->max_tx_queues;
+	channels->max_other = 0;
+	channels->max_combined = 0;
+	channels->rx_count = adapter->req_rx_queues;
+	channels->tx_count = adapter->req_tx_queues;
+	channels->other_count = 0;
+	channels->combined_count = 0;
+}
+
 static void ibmvnic_get_strings(struct net_device *dev, u32 stringset, u8 *data)
 {
+	struct ibmvnic_adapter *adapter = netdev_priv(dev);
 	int i;
 
 	if (stringset != ETH_SS_STATS)
@@ -1727,13 +1819,39 @@ static void ibmvnic_get_strings(struct net_device *dev, u32 stringset, u8 *data)
 
 	for (i = 0; i < ARRAY_SIZE(ibmvnic_stats); i++, data += ETH_GSTRING_LEN)
 		memcpy(data, ibmvnic_stats[i].name, ETH_GSTRING_LEN);
+
+	for (i = 0; i < adapter->req_tx_queues; i++) {
+		snprintf(data, ETH_GSTRING_LEN, "tx%d_packets", i);
+		data += ETH_GSTRING_LEN;
+
+		snprintf(data, ETH_GSTRING_LEN, "tx%d_bytes", i);
+		data += ETH_GSTRING_LEN;
+
+		snprintf(data, ETH_GSTRING_LEN, "tx%d_dropped_packets", i);
+		data += ETH_GSTRING_LEN;
+	}
+
+	for (i = 0; i < adapter->req_rx_queues; i++) {
+		snprintf(data, ETH_GSTRING_LEN, "rx%d_packets", i);
+		data += ETH_GSTRING_LEN;
+
+		snprintf(data, ETH_GSTRING_LEN, "rx%d_bytes", i);
+		data += ETH_GSTRING_LEN;
+
+		snprintf(data, ETH_GSTRING_LEN, "rx%d_interrupts", i);
+		data += ETH_GSTRING_LEN;
+	}
 }
 
 static int ibmvnic_get_sset_count(struct net_device *dev, int sset)
 {
+	struct ibmvnic_adapter *adapter = netdev_priv(dev);
+
 	switch (sset) {
 	case ETH_SS_STATS:
-		return ARRAY_SIZE(ibmvnic_stats);
+		return ARRAY_SIZE(ibmvnic_stats) +
+		       adapter->req_tx_queues * NUM_TX_STATS +
+		       adapter->req_rx_queues * NUM_RX_STATS;
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -1744,7 +1862,7 @@ static void ibmvnic_get_ethtool_stats(struct net_device *dev,
 {
 	struct ibmvnic_adapter *adapter = netdev_priv(dev);
 	union ibmvnic_crq crq;
-	int i;
+	int i, j;
 
 	memset(&crq, 0, sizeof(crq));
 	crq.request_statistics.first = IBMVNIC_CRQ_CMD;
@@ -1759,7 +1877,26 @@ static void ibmvnic_get_ethtool_stats(struct net_device *dev,
 	wait_for_completion(&adapter->stats_done);
 
 	for (i = 0; i < ARRAY_SIZE(ibmvnic_stats); i++)
-		data[i] = IBMVNIC_GET_STAT(adapter, ibmvnic_stats[i].offset);
+		data[i] = be64_to_cpu(IBMVNIC_GET_STAT(adapter,
+						ibmvnic_stats[i].offset));
+
+	for (j = 0; j < adapter->req_tx_queues; j++) {
+		data[i] = adapter->tx_stats_buffers[j].packets;
+		i++;
+		data[i] = adapter->tx_stats_buffers[j].bytes;
+		i++;
+		data[i] = adapter->tx_stats_buffers[j].dropped_packets;
+		i++;
+	}
+
+	for (j = 0; j < adapter->req_rx_queues; j++) {
+		data[i] = adapter->rx_stats_buffers[j].packets;
+		i++;
+		data[i] = adapter->rx_stats_buffers[j].bytes;
+		i++;
+		data[i] = adapter->rx_stats_buffers[j].interrupts;
+		i++;
+	}
 }
 
 static const struct ethtool_ops ibmvnic_ethtool_ops = {
@@ -1768,6 +1905,7 @@ static const struct ethtool_ops ibmvnic_ethtool_ops = {
 	.set_msglevel		= ibmvnic_set_msglevel,
 	.get_link		= ibmvnic_get_link,
 	.get_ringparam		= ibmvnic_get_ringparam,
+	.get_channels		= ibmvnic_get_channels,
 	.get_strings            = ibmvnic_get_strings,
 	.get_sset_count         = ibmvnic_get_sset_count,
 	.get_ethtool_stats	= ibmvnic_get_ethtool_stats,
@@ -1800,12 +1938,14 @@ static int reset_sub_crq_queues(struct ibmvnic_adapter *adapter)
 	int i, rc;
 
 	for (i = 0; i < adapter->req_tx_queues; i++) {
+		netdev_dbg(adapter->netdev, "Re-setting tx_scrq[%d]\n", i);
 		rc = reset_one_sub_crq_queue(adapter, adapter->tx_scrq[i]);
 		if (rc)
 			return rc;
 	}
 
 	for (i = 0; i < adapter->req_rx_queues; i++) {
+		netdev_dbg(adapter->netdev, "Re-setting rx_scrq[%d]\n", i);
 		rc = reset_one_sub_crq_queue(adapter, adapter->rx_scrq[i]);
 		if (rc)
 			return rc;
@@ -1909,6 +2049,8 @@ static void release_sub_crqs(struct ibmvnic_adapter *adapter)
 			if (!adapter->tx_scrq[i])
 				continue;
 
+			netdev_dbg(adapter->netdev, "Releasing tx_scrq[%d]\n",
+				   i);
 			if (adapter->tx_scrq[i]->irq) {
 				free_irq(adapter->tx_scrq[i]->irq,
 					 adapter->tx_scrq[i]);
@@ -1928,6 +2070,8 @@ static void release_sub_crqs(struct ibmvnic_adapter *adapter)
 			if (!adapter->rx_scrq[i])
 				continue;
 
+			netdev_dbg(adapter->netdev, "Releasing rx_scrq[%d]\n",
+				   i);
 			if (adapter->rx_scrq[i]->irq) {
 				free_irq(adapter->rx_scrq[i]->irq,
 					 adapter->rx_scrq[i]);
@@ -2064,6 +2208,8 @@ static irqreturn_t ibmvnic_interrupt_rx(int irq, void *instance)
 	struct ibmvnic_sub_crq_queue *scrq = instance;
 	struct ibmvnic_adapter *adapter = scrq->adapter;
 
+	adapter->rx_stats_buffers[scrq->scrq_num].interrupts++;
+
 	if (napi_schedule_prep(&adapter->napi[scrq->scrq_num])) {
 		disable_scrq_irq(adapter, scrq);
 		__napi_schedule(&adapter->napi[scrq->scrq_num]);
@@ -2080,6 +2226,8 @@ static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter)
 	int rc = 0;
 
 	for (i = 0; i < adapter->req_tx_queues; i++) {
+		netdev_dbg(adapter->netdev, "Initializing tx_scrq[%d] irq\n",
+			   i);
 		scrq = adapter->tx_scrq[i];
 		scrq->irq = irq_create_mapping(NULL, scrq->hw_irq);
 
@@ -2101,6 +2249,8 @@ static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter)
 	}
 
 	for (i = 0; i < adapter->req_rx_queues; i++) {
+		netdev_dbg(adapter->netdev, "Initializing rx_scrq[%d] irq\n",
+			   i);
 		scrq = adapter->rx_scrq[i];
 		scrq->irq = irq_create_mapping(NULL, scrq->hw_irq);
 		if (!scrq->irq) {
@@ -3739,31 +3889,35 @@ static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id)
 
 	do {
 		rc = ibmvnic_init(adapter);
-		if (rc && rc != EAGAIN) {
-			free_netdev(netdev);
-			return rc;
-		}
+		if (rc && rc != EAGAIN)
+			goto ibmvnic_init_fail;
 	} while (rc == EAGAIN);
 
 	netdev->mtu = adapter->req_mtu - ETH_HLEN;
 
 	rc = device_create_file(&dev->dev, &dev_attr_failover);
-	if (rc) {
-		free_netdev(netdev);
-		return rc;
-	}
+	if (rc)
+		goto ibmvnic_init_fail;
 
 	rc = register_netdev(netdev);
 	if (rc) {
 		dev_err(&dev->dev, "failed to register netdev rc=%d\n", rc);
-		device_remove_file(&dev->dev, &dev_attr_failover);
-		free_netdev(netdev);
-		return rc;
+		goto ibmvnic_register_fail;
 	}
 	dev_info(&dev->dev, "ibmvnic registered\n");
 
 	adapter->state = VNIC_PROBED;
 	return 0;
+
+ibmvnic_register_fail:
+	device_remove_file(&dev->dev, &dev_attr_failover);
+
+ibmvnic_init_fail:
+	release_sub_crqs(adapter);
+	release_crq_queue(adapter);
+	free_netdev(netdev);
+
+	return rc;
 }
 
 static int ibmvnic_remove(struct vio_dev *dev)
@@ -3859,20 +4013,16 @@ static int ibmvnic_resume(struct device *dev)
 {
 	struct net_device *netdev = dev_get_drvdata(dev);
 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
-	int i;
 
 	if (adapter->state != VNIC_OPEN)
 		return 0;
 
-	/* kick the interrupt handlers just in case we lost an interrupt */
-	for (i = 0; i < adapter->req_rx_queues; i++)
-		ibmvnic_interrupt_rx(adapter->rx_scrq[i]->irq,
-				     adapter->rx_scrq[i]);
+	tasklet_schedule(&adapter->tasklet);
 
 	return 0;
 }
 
-static struct vio_device_id ibmvnic_device_table[] = {
+static const struct vio_device_id ibmvnic_device_table[] = {
 	{"network", "IBM,vnic"},
 	{"", "" }
 };
diff --git a/drivers/net/ethernet/ibm/ibmvnic.h b/drivers/net/ethernet/ibm/ibmvnic.h
index 8eff6e15f4bb..d02257ccc377 100644
--- a/drivers/net/ethernet/ibm/ibmvnic.h
+++ b/drivers/net/ethernet/ibm/ibmvnic.h
@@ -166,6 +166,20 @@ struct ibmvnic_statistics {
 	u8 reserved[72];
 } __packed __aligned(8);
 
+#define NUM_TX_STATS 3
+struct ibmvnic_tx_queue_stats {
+	u64 packets;
+	u64 bytes;
+	u64 dropped_packets;
+};
+
+#define NUM_RX_STATS 3
+struct ibmvnic_rx_queue_stats {
+	u64 packets;
+	u64 bytes;
+	u64 interrupts;
+};
+
 struct ibmvnic_acl_buffer {
 	__be32 len;
 	__be32 version;
@@ -956,6 +970,9 @@ struct ibmvnic_adapter {
 	int tx_send_failed;
 	int tx_map_failed;
 
+	struct ibmvnic_tx_queue_stats *tx_stats_buffers;
+	struct ibmvnic_rx_queue_stats *rx_stats_buffers;
+
 	int phys_link_state;
 	int logical_link_state;
 
diff --git a/drivers/net/ethernet/intel/e1000e/hw.h b/drivers/net/ethernet/intel/e1000e/hw.h
index 66bd5060a65b..d803b1a12349 100644
--- a/drivers/net/ethernet/intel/e1000e/hw.h
+++ b/drivers/net/ethernet/intel/e1000e/hw.h
@@ -100,6 +100,10 @@ struct e1000_hw;
 #define E1000_DEV_ID_PCH_CNP_I219_V6		0x15BE
 #define E1000_DEV_ID_PCH_CNP_I219_LM7		0x15BB
 #define E1000_DEV_ID_PCH_CNP_I219_V7		0x15BC
+#define E1000_DEV_ID_PCH_ICP_I219_LM8		0x15DF
+#define E1000_DEV_ID_PCH_ICP_I219_V8		0x15E0
+#define E1000_DEV_ID_PCH_ICP_I219_LM9		0x15E1
+#define E1000_DEV_ID_PCH_ICP_I219_V9		0x15E2
 
 #define E1000_REVISION_4	4
 
diff --git a/drivers/net/ethernet/intel/e1000e/ich8lan.c b/drivers/net/ethernet/intel/e1000e/ich8lan.c
index 68ea8b4555ab..d6d4ed7acf03 100644
--- a/drivers/net/ethernet/intel/e1000e/ich8lan.c
+++ b/drivers/net/ethernet/intel/e1000e/ich8lan.c
@@ -2437,6 +2437,8 @@ static s32 e1000_hv_phy_workarounds_ich8lan(struct e1000_hw *hw)
 		if (hw->phy.revision < 2) {
 			e1000e_phy_sw_reset(hw);
 			ret_val = e1e_wphy(hw, MII_BMCR, 0x3140);
+			if (ret_val)
+				return ret_val;
 		}
 	}
 
diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c
index 2dcb5463d9b8..327dfe5bedc0 100644
--- a/drivers/net/ethernet/intel/e1000e/netdev.c
+++ b/drivers/net/ethernet/intel/e1000e/netdev.c
@@ -7544,6 +7544,10 @@ static const struct pci_device_id e1000_pci_tbl[] = {
 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_V6), board_pch_cnp },
 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_LM7), board_pch_cnp },
 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_V7), board_pch_cnp },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_LM8), board_pch_cnp },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_V8), board_pch_cnp },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_LM9), board_pch_cnp },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_V9), board_pch_cnp },
 
 	{ 0, 0, 0, 0, 0, 0, 0 }	/* terminate list */
 };
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c b/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c
index 5e37387c7082..e69d49d91d67 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c
@@ -1265,15 +1265,17 @@ err_queueing_scheme:
 	return err;
 }
 
-static int __fm10k_setup_tc(struct net_device *dev, u32 handle, u32 chain_index,
-			    __be16 proto, struct tc_to_netdev *tc)
+static int __fm10k_setup_tc(struct net_device *dev, enum tc_setup_type type,
+			    void *type_data)
 {
-	if (tc->type != TC_SETUP_MQPRIO)
-		return -EINVAL;
+	struct tc_mqprio_qopt *mqprio = type_data;
+
+	if (type != TC_SETUP_MQPRIO)
+		return -EOPNOTSUPP;
 
-	tc->mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
+	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
 
-	return fm10k_setup_tc(dev, tc->mqprio->num_tc);
+	return fm10k_setup_tc(dev, mqprio->num_tc);
 }
 
 static void fm10k_assign_l2_accel(struct fm10k_intfc *interface,
diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h
index d616f698e155..d0c1bf5441d8 100644
--- a/drivers/net/ethernet/intel/i40e/i40e.h
+++ b/drivers/net/ethernet/intel/i40e/i40e.h
@@ -75,11 +75,11 @@
 #define I40E_MIN_VSI_ALLOC		83 /* LAN, ATR, FCOE, 64 VF */
 /* max 16 qps */
 #define i40e_default_queues_per_vmdq(pf) \
-		(((pf)->flags & I40E_FLAG_RSS_AQ_CAPABLE) ? 4 : 1)
+		(((pf)->hw_features & I40E_HW_RSS_AQ_CAPABLE) ? 4 : 1)
 #define I40E_DEFAULT_QUEUES_PER_VF	4
 #define I40E_DEFAULT_QUEUES_PER_TC	1 /* should be a power of 2 */
 #define i40e_pf_get_max_q_per_tc(pf) \
-		(((pf)->flags & I40E_FLAG_128_QP_RSS_CAPABLE) ? 128 : 64)
+		(((pf)->hw_features & I40E_HW_128_QP_RSS_CAPABLE) ? 128 : 64)
 #define I40E_FDIR_RING			0
 #define I40E_FDIR_RING_COUNT		32
 #define I40E_MAX_AQ_BUF_SIZE		4096
@@ -401,6 +401,27 @@ struct i40e_pf {
 	struct timer_list service_timer;
 	struct work_struct service_task;
 
+	u64 hw_features;
+#define I40E_HW_RSS_AQ_CAPABLE			BIT_ULL(0)
+#define I40E_HW_128_QP_RSS_CAPABLE		BIT_ULL(1)
+#define I40E_HW_ATR_EVICT_CAPABLE		BIT_ULL(2)
+#define I40E_HW_WB_ON_ITR_CAPABLE		BIT_ULL(3)
+#define I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE	BIT_ULL(4)
+#define I40E_HW_NO_PCI_LINK_CHECK		BIT_ULL(5)
+#define I40E_HW_100M_SGMII_CAPABLE		BIT_ULL(6)
+#define I40E_HW_NO_DCB_SUPPORT			BIT_ULL(7)
+#define I40E_HW_USE_SET_LLDP_MIB		BIT_ULL(8)
+#define I40E_HW_GENEVE_OFFLOAD_CAPABLE		BIT_ULL(9)
+#define I40E_HW_PTP_L4_CAPABLE			BIT_ULL(10)
+#define I40E_HW_WOL_MC_MAGIC_PKT_WAKE		BIT_ULL(11)
+#define I40E_HW_MPLS_HDR_OFFLOAD_CAPABLE	BIT_ULL(12)
+#define I40E_HW_HAVE_CRT_RETIMER		BIT_ULL(13)
+#define I40E_HW_OUTER_UDP_CSUM_CAPABLE		BIT_ULL(14)
+#define I40E_HW_PHY_CONTROLS_LEDS		BIT_ULL(15)
+#define I40E_HW_STOP_FW_LLDP			BIT_ULL(16)
+#define I40E_HW_PORT_ID_VALID			BIT_ULL(17)
+#define I40E_HW_RESTART_AUTONEG			BIT_ULL(18)
+
 	u64 flags;
 #define I40E_FLAG_RX_CSUM_ENABLED		BIT_ULL(1)
 #define I40E_FLAG_MSI_ENABLED			BIT_ULL(2)
@@ -420,33 +441,14 @@ struct i40e_pf {
 #define I40E_FLAG_PTP				BIT_ULL(25)
 #define I40E_FLAG_MFP_ENABLED			BIT_ULL(26)
 #define I40E_FLAG_UDP_FILTER_SYNC		BIT_ULL(27)
-#define I40E_FLAG_PORT_ID_VALID			BIT_ULL(28)
 #define I40E_FLAG_DCB_CAPABLE			BIT_ULL(29)
-#define I40E_FLAG_RSS_AQ_CAPABLE		BIT_ULL(31)
-#define I40E_FLAG_HW_ATR_EVICT_CAPABLE		BIT_ULL(32)
-#define I40E_FLAG_OUTER_UDP_CSUM_CAPABLE	BIT_ULL(33)
-#define I40E_FLAG_128_QP_RSS_CAPABLE		BIT_ULL(34)
-#define I40E_FLAG_WB_ON_ITR_CAPABLE		BIT_ULL(35)
 #define I40E_FLAG_VEB_STATS_ENABLED		BIT_ULL(37)
-#define I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE	BIT_ULL(38)
 #define I40E_FLAG_LINK_POLLING_ENABLED		BIT_ULL(39)
 #define I40E_FLAG_VEB_MODE_ENABLED		BIT_ULL(40)
-#define I40E_FLAG_GENEVE_OFFLOAD_CAPABLE	BIT_ULL(41)
-#define I40E_FLAG_NO_PCI_LINK_CHECK		BIT_ULL(42)
-#define I40E_FLAG_100M_SGMII_CAPABLE		BIT_ULL(43)
-#define I40E_FLAG_RESTART_AUTONEG		BIT_ULL(44)
-#define I40E_FLAG_NO_DCB_SUPPORT		BIT_ULL(45)
-#define I40E_FLAG_USE_SET_LLDP_MIB		BIT_ULL(46)
-#define I40E_FLAG_STOP_FW_LLDP			BIT_ULL(47)
-#define I40E_FLAG_PHY_CONTROLS_LEDS		BIT_ULL(48)
-#define I40E_FLAG_PF_MAC			BIT_ULL(50)
 #define I40E_FLAG_TRUE_PROMISC_SUPPORT		BIT_ULL(51)
-#define I40E_FLAG_HAVE_CRT_RETIMER		BIT_ULL(52)
-#define I40E_FLAG_PTP_L4_CAPABLE		BIT_ULL(53)
 #define I40E_FLAG_CLIENT_RESET			BIT_ULL(54)
 #define I40E_FLAG_TEMP_LINK_POLLING		BIT_ULL(55)
 #define I40E_FLAG_CLIENT_L2_CHANGE		BIT_ULL(56)
-#define I40E_FLAG_WOL_MC_MAGIC_PKT_WAKE		BIT_ULL(57)
 #define I40E_FLAG_LEGACY_RX			BIT_ULL(58)
 
 	struct i40e_client_instance *cinst;
diff --git a/drivers/net/ethernet/intel/i40e/i40e_common.c b/drivers/net/ethernet/intel/i40e/i40e_common.c
index 8e082a946411..111426ba5fbc 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_common.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_common.c
@@ -328,9 +328,9 @@ void i40e_debug_aq(struct i40e_hw *hw, enum i40e_debug_mask mask, void *desc,
 			len = buf_len;
 		/* write the full 16-byte chunks */
 		if (hw->debug_mask & mask) {
-			char prefix[20];
+			char prefix[27];
 
-			snprintf(prefix, 20,
+			snprintf(prefix, sizeof(prefix),
 				 "i40e %02x:%02x.%x: \t0x",
 				 hw->bus.bus_id,
 				 hw->bus.device,
@@ -2529,6 +2529,10 @@ i40e_status i40e_update_link_info(struct i40e_hw *hw)
 		if (status)
 			return status;
 
+		hw->phy.link_info.req_fec_info =
+			abilities.fec_cfg_curr_mod_ext_info &
+			(I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS);
+
 		memcpy(hw->phy.link_info.module_type, &abilities.module_type,
 		       sizeof(hw->phy.link_info.module_type));
 	}
diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
index 9692a5294fa3..05e89864f781 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
@@ -271,7 +271,7 @@ static void i40e_phy_type_to_ethtool(struct i40e_pf *pf, u32 *supported,
 		*advertising |= ADVERTISED_Autoneg;
 		if (hw_link_info->requested_speeds & I40E_LINK_SPEED_1GB)
 			*advertising |= ADVERTISED_1000baseT_Full;
-		if (pf->flags & I40E_FLAG_100M_SGMII_CAPABLE) {
+		if (pf->hw_features & I40E_HW_100M_SGMII_CAPABLE) {
 			*supported |= SUPPORTED_100baseT_Full;
 			*advertising |= ADVERTISED_100baseT_Full;
 		}
@@ -340,12 +340,12 @@ static void i40e_phy_type_to_ethtool(struct i40e_pf *pf, u32 *supported,
 			*advertising |= ADVERTISED_20000baseKR2_Full;
 	}
 	if (phy_types & I40E_CAP_PHY_TYPE_10GBASE_KR) {
-		if (!(pf->flags & I40E_FLAG_HAVE_CRT_RETIMER))
+		if (!(pf->hw_features & I40E_HW_HAVE_CRT_RETIMER))
 			*supported |= SUPPORTED_10000baseKR_Full |
 				      SUPPORTED_Autoneg;
 		*advertising |= ADVERTISED_Autoneg;
 		if (hw_link_info->requested_speeds & I40E_LINK_SPEED_10GB)
-			if (!(pf->flags & I40E_FLAG_HAVE_CRT_RETIMER))
+			if (!(pf->hw_features & I40E_HW_HAVE_CRT_RETIMER))
 				*advertising |= ADVERTISED_10000baseKR_Full;
 	}
 	if (phy_types & I40E_CAP_PHY_TYPE_10GBASE_KX4) {
@@ -356,12 +356,12 @@ static void i40e_phy_type_to_ethtool(struct i40e_pf *pf, u32 *supported,
 			*advertising |= ADVERTISED_10000baseKX4_Full;
 	}
 	if (phy_types & I40E_CAP_PHY_TYPE_1000BASE_KX) {
-		if (!(pf->flags & I40E_FLAG_HAVE_CRT_RETIMER))
+		if (!(pf->hw_features & I40E_HW_HAVE_CRT_RETIMER))
 			*supported |= SUPPORTED_1000baseKX_Full |
 				      SUPPORTED_Autoneg;
 		*advertising |= ADVERTISED_Autoneg;
 		if (hw_link_info->requested_speeds & I40E_LINK_SPEED_1GB)
-			if (!(pf->flags & I40E_FLAG_HAVE_CRT_RETIMER))
+			if (!(pf->hw_features & I40E_HW_HAVE_CRT_RETIMER))
 				*advertising |= ADVERTISED_1000baseKX_Full;
 	}
 	if (phy_types & I40E_CAP_PHY_TYPE_25GBASE_KR ||
@@ -474,7 +474,7 @@ static void i40e_get_settings_link_up(struct i40e_hw *hw,
 			    SUPPORTED_1000baseT_Full;
 		if (hw_link_info->requested_speeds & I40E_LINK_SPEED_1GB)
 			advertising |= ADVERTISED_1000baseT_Full;
-		if (pf->flags & I40E_FLAG_100M_SGMII_CAPABLE) {
+		if (pf->hw_features & I40E_HW_100M_SGMII_CAPABLE) {
 			supported |= SUPPORTED_100baseT_Full;
 			if (hw_link_info->requested_speeds &
 			    I40E_LINK_SPEED_100MB)
@@ -1091,7 +1091,7 @@ static void i40e_get_regs(struct net_device *netdev, struct ethtool_regs *regs,
 	struct i40e_pf *pf = np->vsi->back;
 	struct i40e_hw *hw = &pf->hw;
 	u32 *reg_buf = p;
-	int i, j, ri;
+	unsigned int i, j, ri;
 	u32 reg;
 
 	/* Tell ethtool which driver-version-specific regs output we have.
@@ -1550,9 +1550,9 @@ static void i40e_get_ethtool_stats(struct net_device *netdev,
 	struct i40e_ring *tx_ring, *rx_ring;
 	struct i40e_vsi *vsi = np->vsi;
 	struct i40e_pf *pf = vsi->back;
+	unsigned int j;
 	int i = 0;
 	char *p;
-	int j;
 	struct rtnl_link_stats64 *net_stats = i40e_get_vsi_stats_struct(vsi);
 	unsigned int start;
 
@@ -1637,7 +1637,7 @@ static void i40e_get_strings(struct net_device *netdev, u32 stringset,
 	struct i40e_vsi *vsi = np->vsi;
 	struct i40e_pf *pf = vsi->back;
 	char *p = (char *)data;
-	int i;
+	unsigned int i;
 
 	switch (stringset) {
 	case ETH_SS_TEST:
@@ -1765,7 +1765,7 @@ static int i40e_get_ts_info(struct net_device *dev,
 			   BIT(HWTSTAMP_FILTER_PTP_V2_L2_SYNC) |
 			   BIT(HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ);
 
-	if (pf->flags & I40E_FLAG_PTP_L4_CAPABLE)
+	if (pf->hw_features & I40E_HW_PTP_L4_CAPABLE)
 		info->rx_filters |= BIT(HWTSTAMP_FILTER_PTP_V1_L4_SYNC) |
 				    BIT(HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ) |
 				    BIT(HWTSTAMP_FILTER_PTP_V2_EVENT) |
@@ -2005,7 +2005,7 @@ static int i40e_set_phys_id(struct net_device *netdev,
 
 	switch (state) {
 	case ETHTOOL_ID_ACTIVE:
-		if (!(pf->flags & I40E_FLAG_PHY_CONTROLS_LEDS)) {
+		if (!(pf->hw_features & I40E_HW_PHY_CONTROLS_LEDS)) {
 			pf->led_status = i40e_led_get(hw);
 		} else {
 			i40e_aq_set_phy_debug(hw, I40E_PHY_DEBUG_ALL, NULL);
@@ -2015,19 +2015,19 @@ static int i40e_set_phys_id(struct net_device *netdev,
 		}
 		return blink_freq;
 	case ETHTOOL_ID_ON:
-		if (!(pf->flags & I40E_FLAG_PHY_CONTROLS_LEDS))
+		if (!(pf->hw_features & I40E_HW_PHY_CONTROLS_LEDS))
 			i40e_led_set(hw, 0xf, false);
 		else
 			ret = i40e_led_set_phy(hw, true, pf->led_status, 0);
 		break;
 	case ETHTOOL_ID_OFF:
-		if (!(pf->flags & I40E_FLAG_PHY_CONTROLS_LEDS))
+		if (!(pf->hw_features & I40E_HW_PHY_CONTROLS_LEDS))
 			i40e_led_set(hw, 0x0, false);
 		else
 			ret = i40e_led_set_phy(hw, false, pf->led_status, 0);
 		break;
 	case ETHTOOL_ID_INACTIVE:
-		if (!(pf->flags & I40E_FLAG_PHY_CONTROLS_LEDS)) {
+		if (!(pf->hw_features & I40E_HW_PHY_CONTROLS_LEDS)) {
 			i40e_led_set(hw, pf->led_status, false);
 		} else {
 			ret = i40e_led_set_phy(hw, false, pf->led_status,
@@ -2194,14 +2194,29 @@ static int __i40e_set_coalesce(struct net_device *netdev,
 			       int queue)
 {
 	struct i40e_netdev_priv *np = netdev_priv(netdev);
+	u16 intrl_reg, cur_rx_itr, cur_tx_itr;
 	struct i40e_vsi *vsi = np->vsi;
 	struct i40e_pf *pf = vsi->back;
-	u16 intrl_reg;
 	int i;
 
 	if (ec->tx_max_coalesced_frames_irq || ec->rx_max_coalesced_frames_irq)
 		vsi->work_limit = ec->tx_max_coalesced_frames_irq;
 
+	if (queue < 0) {
+		cur_rx_itr = vsi->rx_rings[0]->rx_itr_setting;
+		cur_tx_itr = vsi->tx_rings[0]->tx_itr_setting;
+	} else if (queue < vsi->num_queue_pairs) {
+		cur_rx_itr = vsi->rx_rings[queue]->rx_itr_setting;
+		cur_tx_itr = vsi->tx_rings[queue]->tx_itr_setting;
+	} else {
+		netif_info(pf, drv, netdev, "Invalid queue value, queue range is 0 - %d\n",
+			   vsi->num_queue_pairs - 1);
+		return -EINVAL;
+	}
+
+	cur_tx_itr &= ~I40E_ITR_DYNAMIC;
+	cur_rx_itr &= ~I40E_ITR_DYNAMIC;
+
 	/* tx_coalesce_usecs_high is ignored, use rx-usecs-high instead */
 	if (ec->tx_coalesce_usecs_high != vsi->int_rate_limit) {
 		netif_info(pf, drv, netdev, "tx-usecs-high is not used, please program rx-usecs-high\n");
@@ -2214,15 +2229,34 @@ static int __i40e_set_coalesce(struct net_device *netdev,
 		return -EINVAL;
 	}
 
-	if (ec->rx_coalesce_usecs == 0) {
-		if (ec->use_adaptive_rx_coalesce)
-			netif_info(pf, drv, netdev, "rx-usecs=0, need to disable adaptive-rx for a complete disable\n");
-	} else if ((ec->rx_coalesce_usecs < (I40E_MIN_ITR << 1)) ||
-		   (ec->rx_coalesce_usecs > (I40E_MAX_ITR << 1))) {
-			netif_info(pf, drv, netdev, "Invalid value, rx-usecs range is 0-8160\n");
-			return -EINVAL;
+	if (ec->rx_coalesce_usecs != cur_rx_itr &&
+	    ec->use_adaptive_rx_coalesce) {
+		netif_info(pf, drv, netdev, "RX interrupt moderation cannot be changed if adaptive-rx is enabled.\n");
+		return -EINVAL;
 	}
 
+	if (ec->rx_coalesce_usecs > (I40E_MAX_ITR << 1)) {
+		netif_info(pf, drv, netdev, "Invalid value, rx-usecs range is 0-8160\n");
+		return -EINVAL;
+	}
+
+	if (ec->tx_coalesce_usecs != cur_tx_itr &&
+	    ec->use_adaptive_tx_coalesce) {
+		netif_info(pf, drv, netdev, "TX interrupt moderation cannot be changed if adaptive-tx is enabled.\n");
+		return -EINVAL;
+	}
+
+	if (ec->tx_coalesce_usecs > (I40E_MAX_ITR << 1)) {
+		netif_info(pf, drv, netdev, "Invalid value, tx-usecs range is 0-8160\n");
+		return -EINVAL;
+	}
+
+	if (ec->use_adaptive_rx_coalesce && !cur_rx_itr)
+		ec->rx_coalesce_usecs = I40E_MIN_ITR << 1;
+
+	if (ec->use_adaptive_tx_coalesce && !cur_tx_itr)
+		ec->tx_coalesce_usecs = I40E_MIN_ITR << 1;
+
 	intrl_reg = i40e_intrl_usec_to_reg(ec->rx_coalesce_usecs_high);
 	vsi->int_rate_limit = INTRL_REG_TO_USEC(intrl_reg);
 	if (vsi->int_rate_limit != ec->rx_coalesce_usecs_high) {
@@ -2230,27 +2264,14 @@ static int __i40e_set_coalesce(struct net_device *netdev,
 			   vsi->int_rate_limit);
 	}
 
-	if (ec->tx_coalesce_usecs == 0) {
-		if (ec->use_adaptive_tx_coalesce)
-			netif_info(pf, drv, netdev, "tx-usecs=0, need to disable adaptive-tx for a complete disable\n");
-	} else if ((ec->tx_coalesce_usecs < (I40E_MIN_ITR << 1)) ||
-		   (ec->tx_coalesce_usecs > (I40E_MAX_ITR << 1))) {
-			netif_info(pf, drv, netdev, "Invalid value, tx-usecs range is 0-8160\n");
-			return -EINVAL;
-	}
-
 	/* rx and tx usecs has per queue value. If user doesn't specify the queue,
 	 * apply to all queues.
 	 */
 	if (queue < 0) {
 		for (i = 0; i < vsi->num_queue_pairs; i++)
 			i40e_set_itr_per_queue(vsi, ec, i);
-	} else if (queue < vsi->num_queue_pairs) {
-		i40e_set_itr_per_queue(vsi, ec, queue);
 	} else {
-		netif_info(pf, drv, netdev, "Invalid queue value, queue range is 0 - %d\n",
-			   vsi->num_queue_pairs - 1);
-		return -EINVAL;
+		i40e_set_itr_per_queue(vsi, ec, queue);
 	}
 
 	return 0;
@@ -2727,22 +2748,22 @@ static int i40e_set_rss_hash_opt(struct i40e_pf *pf, struct ethtool_rxnfc *nfc)
 	switch (nfc->flow_type) {
 	case TCP_V4_FLOW:
 		flow_pctype = I40E_FILTER_PCTYPE_NONF_IPV4_TCP;
-		if (pf->flags & I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE)
+		if (pf->hw_features & I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE)
 			hena |=
 			  BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK);
 		break;
 	case TCP_V6_FLOW:
 		flow_pctype = I40E_FILTER_PCTYPE_NONF_IPV6_TCP;
-		if (pf->flags & I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE)
+		if (pf->hw_features & I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE)
 			hena |=
 			  BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK);
-		if (pf->flags & I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE)
+		if (pf->hw_features & I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE)
 			hena |=
 			  BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK);
 		break;
 	case UDP_V4_FLOW:
 		flow_pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
-		if (pf->flags & I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE)
+		if (pf->hw_features & I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE)
 			hena |=
 			  BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) |
 			  BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP);
@@ -2751,7 +2772,7 @@ static int i40e_set_rss_hash_opt(struct i40e_pf *pf, struct ethtool_rxnfc *nfc)
 		break;
 	case UDP_V6_FLOW:
 		flow_pctype = I40E_FILTER_PCTYPE_NONF_IPV6_UDP;
-		if (pf->flags & I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE)
+		if (pf->hw_features & I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE)
 			hena |=
 			  BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) |
 			  BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP);
@@ -4069,23 +4090,26 @@ static int i40e_set_priv_flags(struct net_device *dev, u32 flags)
 	struct i40e_netdev_priv *np = netdev_priv(dev);
 	struct i40e_vsi *vsi = np->vsi;
 	struct i40e_pf *pf = vsi->back;
-	u64 changed_flags;
+	u64 orig_flags, new_flags, changed_flags;
 	u32 i, j;
 
-	changed_flags = pf->flags;
+	orig_flags = READ_ONCE(pf->flags);
+	new_flags = orig_flags;
 
 	for (i = 0; i < I40E_PRIV_FLAGS_STR_LEN; i++) {
 		const struct i40e_priv_flags *priv_flags;
 
 		priv_flags = &i40e_gstrings_priv_flags[i];
 
-		if (priv_flags->read_only)
-			continue;
-
 		if (flags & BIT(i))
-			pf->flags |= priv_flags->flag;
+			new_flags |= priv_flags->flag;
 		else
-			pf->flags &= ~(priv_flags->flag);
+			new_flags &= ~(priv_flags->flag);
+
+		/* If this is a read-only flag, it can't be changed */
+		if (priv_flags->read_only &&
+		    ((orig_flags ^ new_flags) & ~BIT(i)))
+			return -EOPNOTSUPP;
 	}
 
 	if (pf->hw.pf_id != 0)
@@ -4096,18 +4120,40 @@ static int i40e_set_priv_flags(struct net_device *dev, u32 flags)
 
 		priv_flags = &i40e_gl_gstrings_priv_flags[j];
 
-		if (priv_flags->read_only)
-			continue;
-
 		if (flags & BIT(i + j))
-			pf->flags |= priv_flags->flag;
+			new_flags |= priv_flags->flag;
 		else
-			pf->flags &= ~(priv_flags->flag);
+			new_flags &= ~(priv_flags->flag);
+
+		/* If this is a read-only flag, it can't be changed */
+		if (priv_flags->read_only &&
+		    ((orig_flags ^ new_flags) & ~BIT(i)))
+			return -EOPNOTSUPP;
 	}
 
 flags_complete:
-	/* check for flags that changed */
-	changed_flags ^= pf->flags;
+	/* Before we finalize any flag changes, we need to perform some
+	 * checks to ensure that the changes are supported and safe.
+	 */
+
+	/* ATR eviction is not supported on all devices */
+	if ((new_flags & I40E_FLAG_HW_ATR_EVICT_ENABLED) &&
+	    !(pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE))
+		return -EOPNOTSUPP;
+
+	/* Compare and exchange the new flags into place. If we failed, that
+	 * is if cmpxchg64 returns anything but the old value, this means that
+	 * something else has modified the flags variable since we copied it
+	 * originally. We'll just punt with an error and log something in the
+	 * message buffer.
+	 */
+	if (cmpxchg64(&pf->flags, orig_flags, new_flags) != orig_flags) {
+		dev_warn(&pf->pdev->dev,
+			 "Unable to update pf->flags as it was modified by another thread...\n");
+		return -EAGAIN;
+	}
+
+	changed_flags = orig_flags ^ new_flags;
 
 	/* Process any additional changes needed as a result of flag changes.
 	 * The changed_flags value reflects the list of bits that were
@@ -4121,10 +4167,6 @@ flags_complete:
 		set_bit(__I40E_FD_FLUSH_REQUESTED, pf->state);
 	}
 
-	/* Only allow ATR evict on hardware that is capable of handling it */
-	if (pf->flags & I40E_FLAG_HW_ATR_EVICT_CAPABLE)
-		pf->flags &= ~I40E_FLAG_HW_ATR_EVICT_ENABLED;
-
 	if (changed_flags & I40E_FLAG_TRUE_PROMISC_SUPPORT) {
 		u16 sw_flags = 0, valid_flags = 0;
 		int ret;
diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c
index 2db93d3f6d23..6498da8806cb 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_main.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_main.c
@@ -2595,9 +2595,20 @@ int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid)
 {
 	int err;
 
-	if (!vid || vsi->info.pvid)
+	if (vsi->info.pvid)
 		return -EINVAL;
 
+	/* The network stack will attempt to add VID=0, with the intention to
+	 * receive priority tagged packets with a VLAN of 0. Our HW receives
+	 * these packets by default when configured to receive untagged
+	 * packets, so we don't need to add a filter for this case.
+	 * Additionally, HW interprets adding a VID=0 filter as meaning to
+	 * receive *only* tagged traffic and stops receiving untagged traffic.
+	 * Thus, we do not want to actually add a filter for VID=0
+	 */
+	if (!vid)
+		return 0;
+
 	/* Locked once because all functions invoked below iterates list*/
 	spin_lock_bh(&vsi->mac_filter_hash_lock);
 	err = i40e_add_vlan_all_mac(vsi, vid);
@@ -2674,15 +2685,7 @@ static int i40e_vlan_rx_add_vid(struct net_device *netdev,
 	if (vid >= VLAN_N_VID)
 		return -EINVAL;
 
-	/* If the network stack called us with vid = 0 then
-	 * it is asking to receive priority tagged packets with
-	 * vlan id 0.  Our HW receives them by default when configured
-	 * to receive untagged packets so there is no need to add an
-	 * extra filter for vlan 0 tagged packets.
-	 */
-	if (vid)
-		ret = i40e_vsi_add_vlan(vsi, vid);
-
+	ret = i40e_vsi_add_vlan(vsi, vid);
 	if (!ret)
 		set_bit(vid, vsi->active_vlans);
 
@@ -2714,44 +2717,6 @@ static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
 }
 
 /**
- * i40e_macaddr_init - explicitly write the mac address filters
- *
- * @vsi: pointer to the vsi
- * @macaddr: the MAC address
- *
- * This is needed when the macaddr has been obtained by other
- * means than the default, e.g., from Open Firmware or IDPROM.
- * Returns 0 on success, negative on failure
- **/
-static int i40e_macaddr_init(struct i40e_vsi *vsi, u8 *macaddr)
-{
-	int ret;
-	struct i40e_aqc_add_macvlan_element_data element;
-
-	ret = i40e_aq_mac_address_write(&vsi->back->hw,
-					I40E_AQC_WRITE_TYPE_LAA_WOL,
-					macaddr, NULL);
-	if (ret) {
-		dev_info(&vsi->back->pdev->dev,
-			 "Addr change for VSI failed: %d\n", ret);
-		return -EADDRNOTAVAIL;
-	}
-
-	memset(&element, 0, sizeof(element));
-	ether_addr_copy(element.mac_addr, macaddr);
-	element.flags = cpu_to_le16(I40E_AQC_MACVLAN_ADD_PERFECT_MATCH);
-	ret = i40e_aq_add_macvlan(&vsi->back->hw, vsi->seid, &element, 1, NULL);
-	if (ret) {
-		dev_info(&vsi->back->pdev->dev,
-			 "add filter failed err %s aq_err %s\n",
-			 i40e_stat_str(&vsi->back->hw, ret),
-			 i40e_aq_str(&vsi->back->hw,
-				     vsi->back->hw.aq.asq_last_status));
-	}
-	return ret;
-}
-
-/**
  * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
  * @vsi: the vsi being brought back up
  **/
@@ -2909,22 +2874,15 @@ static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
 static void i40e_config_xps_tx_ring(struct i40e_ring *ring)
 {
 	struct i40e_vsi *vsi = ring->vsi;
-	cpumask_var_t mask;
 
 	if (!ring->q_vector || !ring->netdev)
 		return;
 
-	/* Single TC mode enable XPS */
-	if (vsi->tc_config.numtc <= 1) {
-		if (!test_and_set_bit(__I40E_TX_XPS_INIT_DONE, &ring->state))
-			netif_set_xps_queue(ring->netdev,
-					    &ring->q_vector->affinity_mask,
-					    ring->queue_index);
-	} else if (alloc_cpumask_var(&mask, GFP_KERNEL)) {
-		/* Disable XPS to allow selection based on TC */
-		bitmap_zero(cpumask_bits(mask), nr_cpumask_bits);
-		netif_set_xps_queue(ring->netdev, mask, ring->queue_index);
-		free_cpumask_var(mask);
+	if ((vsi->tc_config.numtc <= 1) &&
+	    !test_and_set_bit(__I40E_TX_XPS_INIT_DONE, &ring->state)) {
+		netif_set_xps_queue(ring->netdev,
+				    get_cpu_mask(ring->q_vector->v_idx),
+				    ring->queue_index);
 	}
 
 	/* schedule our worker thread which will take care of
@@ -3203,19 +3161,8 @@ static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
  **/
 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
 {
-	struct i40e_pf *pf = vsi->back;
-	int err;
-
 	if (vsi->netdev)
 		i40e_set_rx_mode(vsi->netdev);
-
-	if (!!(pf->flags & I40E_FLAG_PF_MAC)) {
-		err = i40e_macaddr_init(vsi, pf->hw.mac.addr);
-		if (err) {
-			dev_warn(&pf->pdev->dev,
-				 "could not set up macaddr; err %d\n", err);
-		}
-	}
 }
 
 /**
@@ -3495,7 +3442,7 @@ static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify,
 	struct i40e_q_vector *q_vector =
 		container_of(notify, struct i40e_q_vector, affinity_notify);
 
-	q_vector->affinity_mask = *mask;
+	cpumask_copy(&q_vector->affinity_mask, mask);
 }
 
 /**
@@ -3559,8 +3506,10 @@ static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
 		q_vector->affinity_notify.notify = i40e_irq_affinity_notify;
 		q_vector->affinity_notify.release = i40e_irq_affinity_release;
 		irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
-		/* assign the mask for this irq */
-		irq_set_affinity_hint(irq_num, &q_vector->affinity_mask);
+		/* get_cpu_mask returns a static constant mask with
+		 * a permanent lifetime so it's ok to use here.
+		 */
+		irq_set_affinity_hint(irq_num, get_cpu_mask(q_vector->v_idx));
 	}
 
 	vsi->irqs_ready = true;
@@ -4342,7 +4291,7 @@ static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
 
 			/* clear the affinity notifier in the IRQ descriptor */
 			irq_set_affinity_notifier(irq_num, NULL);
-			/* clear the affinity_mask in the IRQ descriptor */
+			/* remove our suggested affinity mask for this IRQ */
 			irq_set_affinity_hint(irq_num, NULL);
 			synchronize_irq(irq_num);
 			free_irq(irq_num, vsi->q_vectors[i]);
@@ -4773,7 +4722,7 @@ static void i40e_detect_recover_hung(struct i40e_pf *pf)
 {
 	struct net_device *netdev;
 	struct i40e_vsi *vsi;
-	int i;
+	unsigned int i;
 
 	/* Only for LAN VSI */
 	vsi = pf->vsi[pf->lan_vsi];
@@ -5350,7 +5299,7 @@ static int i40e_init_pf_dcb(struct i40e_pf *pf)
 	int err = 0;
 
 	/* Do not enable DCB for SW1 and SW2 images even if the FW is capable */
-	if (pf->flags & I40E_FLAG_NO_DCB_SUPPORT)
+	if (pf->hw_features & I40E_HW_NO_DCB_SUPPORT)
 		goto out;
 
 	/* Get the initial DCB configuration */
@@ -5400,6 +5349,7 @@ void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
 	char *speed = "Unknown";
 	char *fc = "Unknown";
 	char *fec = "";
+	char *req_fec = "";
 	char *an = "";
 
 	new_speed = vsi->back->hw.phy.link_info.link_speed;
@@ -5461,6 +5411,7 @@ void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
 	}
 
 	if (vsi->back->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) {
+		req_fec = ", Requested FEC: None";
 		fec = ", FEC: None";
 		an = ", Autoneg: False";
 
@@ -5473,10 +5424,22 @@ void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
 		else if (vsi->back->hw.phy.link_info.fec_info &
 			 I40E_AQ_CONFIG_FEC_RS_ENA)
 			fec = ", FEC: CL108 RS-FEC";
+
+		/* 'CL108 RS-FEC' should be displayed when RS is requested, or
+		 * both RS and FC are requested
+		 */
+		if (vsi->back->hw.phy.link_info.req_fec_info &
+		    (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) {
+			if (vsi->back->hw.phy.link_info.req_fec_info &
+			    I40E_AQ_REQUEST_FEC_RS)
+				req_fec = ", Requested FEC: CL108 RS-FEC";
+			else
+				req_fec = ", Requested FEC: CL74 FC-FEC/BASE-R";
+		}
 	}
 
-	netdev_info(vsi->netdev, "NIC Link is Up, %sbps Full Duplex%s%s, Flow Control: %s\n",
-		    speed, fec, an, fc);
+	netdev_info(vsi->netdev, "NIC Link is Up, %sbps Full Duplex%s%s%s, Flow Control: %s\n",
+		    speed, req_fec, fec, an, fc);
 }
 
 /**
@@ -5656,16 +5619,17 @@ exit:
 	return ret;
 }
 
-static int __i40e_setup_tc(struct net_device *netdev, u32 handle,
-			   u32 chain_index, __be16 proto,
-			   struct tc_to_netdev *tc)
+static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type,
+			   void *type_data)
 {
-	if (tc->type != TC_SETUP_MQPRIO)
-		return -EINVAL;
+	struct tc_mqprio_qopt *mqprio = type_data;
 
-	tc->mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
+	if (type != TC_SETUP_MQPRIO)
+		return -EOPNOTSUPP;
 
-	return i40e_setup_tc(netdev, tc->mqprio->num_tc);
+	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
+
+	return i40e_setup_tc(netdev, mqprio->num_tc);
 }
 
 /**
@@ -7331,7 +7295,7 @@ static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
 		wr32(hw, I40E_REG_MSS, val);
 	}
 
-	if (pf->flags & I40E_FLAG_RESTART_AUTONEG) {
+	if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
 		msleep(75);
 		ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
 		if (ret)
@@ -7520,6 +7484,18 @@ static void i40e_handle_mdd_event(struct i40e_pf *pf)
 	i40e_flush(hw);
 }
 
+static const char *i40e_tunnel_name(struct i40e_udp_port_config *port)
+{
+	switch (port->type) {
+	case UDP_TUNNEL_TYPE_VXLAN:
+		return "vxlan";
+	case UDP_TUNNEL_TYPE_GENEVE:
+		return "geneve";
+	default:
+		return "unknown";
+	}
+}
+
 /**
  * i40e_sync_udp_filters - Trigger a sync event for existing UDP filters
  * @pf: board private structure
@@ -7565,14 +7541,14 @@ static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf)
 				ret = i40e_aq_del_udp_tunnel(hw, i, NULL);
 
 			if (ret) {
-				dev_dbg(&pf->pdev->dev,
-					"%s %s port %d, index %d failed, err %s aq_err %s\n",
-					pf->udp_ports[i].type ? "vxlan" : "geneve",
-					port ? "add" : "delete",
-					port, i,
-					i40e_stat_str(&pf->hw, ret),
-					i40e_aq_str(&pf->hw,
-						    pf->hw.aq.asq_last_status));
+				dev_info(&pf->pdev->dev,
+					 "%s %s port %d, index %d failed, err %s aq_err %s\n",
+					 i40e_tunnel_name(&pf->udp_ports[i]),
+					 port ? "add" : "delete",
+					 port, i,
+					 i40e_stat_str(&pf->hw, ret),
+					 i40e_aq_str(&pf->hw,
+						     pf->hw.aq.asq_last_status));
 				pf->udp_ports[i].port = 0;
 			}
 		}
@@ -7957,7 +7933,7 @@ static int i40e_alloc_rings(struct i40e_vsi *vsi)
 		ring->count = vsi->num_desc;
 		ring->size = 0;
 		ring->dcb_tc = 0;
-		if (vsi->back->flags & I40E_FLAG_WB_ON_ITR_CAPABLE)
+		if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
 			ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
 		ring->tx_itr_setting = pf->tx_itr_default;
 		vsi->tx_rings[i] = ring++;
@@ -7974,7 +7950,7 @@ static int i40e_alloc_rings(struct i40e_vsi *vsi)
 		ring->count = vsi->num_desc;
 		ring->size = 0;
 		ring->dcb_tc = 0;
-		if (vsi->back->flags & I40E_FLAG_WB_ON_ITR_CAPABLE)
+		if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
 			ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
 		set_ring_xdp(ring);
 		ring->tx_itr_setting = pf->tx_itr_default;
@@ -8261,7 +8237,7 @@ static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx, int cpu)
 
 	q_vector->vsi = vsi;
 	q_vector->v_idx = v_idx;
-	cpumask_set_cpu(cpu, &q_vector->affinity_mask);
+	cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
 
 	if (vsi->netdev)
 		netif_napi_add(vsi->netdev, &q_vector->napi,
@@ -8510,7 +8486,7 @@ static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
 	u8 *lut;
 	int ret;
 
-	if (!(pf->flags & I40E_FLAG_RSS_AQ_CAPABLE))
+	if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE))
 		return 0;
 
 	if (!vsi->rss_size)
@@ -8640,7 +8616,7 @@ int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
 {
 	struct i40e_pf *pf = vsi->back;
 
-	if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE)
+	if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
 		return i40e_config_rss_aq(vsi, seed, lut, lut_size);
 	else
 		return i40e_config_rss_reg(vsi, seed, lut, lut_size);
@@ -8659,7 +8635,7 @@ int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
 {
 	struct i40e_pf *pf = vsi->back;
 
-	if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE)
+	if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
 		return i40e_get_rss_aq(vsi, seed, lut, lut_size);
 	else
 		return i40e_get_rss_reg(vsi, seed, lut, lut_size);
@@ -8987,25 +8963,56 @@ static int i40e_sw_init(struct i40e_pf *pf)
 				 pf->hw.func_caps.fd_filters_best_effort;
 	}
 
+	if (pf->hw.mac.type == I40E_MAC_X722) {
+		pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE |
+				    I40E_HW_128_QP_RSS_CAPABLE |
+				    I40E_HW_ATR_EVICT_CAPABLE |
+				    I40E_HW_WB_ON_ITR_CAPABLE |
+				    I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE |
+				    I40E_HW_NO_PCI_LINK_CHECK |
+				    I40E_HW_USE_SET_LLDP_MIB |
+				    I40E_HW_GENEVE_OFFLOAD_CAPABLE |
+				    I40E_HW_PTP_L4_CAPABLE |
+				    I40E_HW_WOL_MC_MAGIC_PKT_WAKE |
+				    I40E_HW_OUTER_UDP_CSUM_CAPABLE);
+
+#define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03
+		if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) !=
+		    I40E_FDEVICT_PCTYPE_DEFAULT) {
+			dev_warn(&pf->pdev->dev,
+				 "FD EVICT PCTYPES are not right, disable FD HW EVICT\n");
+			pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE;
+		}
+	} else if ((pf->hw.aq.api_maj_ver > 1) ||
+		   ((pf->hw.aq.api_maj_ver == 1) &&
+		    (pf->hw.aq.api_min_ver > 4))) {
+		/* Supported in FW API version higher than 1.4 */
+		pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE;
+	}
+
+	/* Enable HW ATR eviction if possible */
+	if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE)
+		pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED;
+
 	if ((pf->hw.mac.type == I40E_MAC_XL710) &&
 	    (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) ||
 	    (pf->hw.aq.fw_maj_ver < 4))) {
-		pf->flags |= I40E_FLAG_RESTART_AUTONEG;
+		pf->hw_features |= I40E_HW_RESTART_AUTONEG;
 		/* No DCB support  for FW < v4.33 */
-		pf->flags |= I40E_FLAG_NO_DCB_SUPPORT;
+		pf->hw_features |= I40E_HW_NO_DCB_SUPPORT;
 	}
 
 	/* Disable FW LLDP if FW < v4.3 */
 	if ((pf->hw.mac.type == I40E_MAC_XL710) &&
 	    (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) ||
 	    (pf->hw.aq.fw_maj_ver < 4)))
-		pf->flags |= I40E_FLAG_STOP_FW_LLDP;
+		pf->hw_features |= I40E_HW_STOP_FW_LLDP;
 
 	/* Use the FW Set LLDP MIB API if FW > v4.40 */
 	if ((pf->hw.mac.type == I40E_MAC_XL710) &&
 	    (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) ||
 	    (pf->hw.aq.fw_maj_ver >= 5)))
-		pf->flags |= I40E_FLAG_USE_SET_LLDP_MIB;
+		pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB;
 
 	if (pf->hw.func_caps.vmdq) {
 		pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
@@ -9028,29 +9035,6 @@ static int i40e_sw_init(struct i40e_pf *pf)
 					I40E_MAX_VF_COUNT);
 	}
 #endif /* CONFIG_PCI_IOV */
-	if (pf->hw.mac.type == I40E_MAC_X722) {
-		pf->flags |= I40E_FLAG_RSS_AQ_CAPABLE
-			     | I40E_FLAG_128_QP_RSS_CAPABLE
-			     | I40E_FLAG_HW_ATR_EVICT_CAPABLE
-			     | I40E_FLAG_OUTER_UDP_CSUM_CAPABLE
-			     | I40E_FLAG_WB_ON_ITR_CAPABLE
-			     | I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE
-			     | I40E_FLAG_NO_PCI_LINK_CHECK
-			     | I40E_FLAG_USE_SET_LLDP_MIB
-			     | I40E_FLAG_GENEVE_OFFLOAD_CAPABLE
-			     | I40E_FLAG_PTP_L4_CAPABLE
-			     | I40E_FLAG_WOL_MC_MAGIC_PKT_WAKE;
-	} else if ((pf->hw.aq.api_maj_ver > 1) ||
-		   ((pf->hw.aq.api_maj_ver == 1) &&
-		    (pf->hw.aq.api_min_ver > 4))) {
-		/* Supported in FW API version higher than 1.4 */
-		pf->flags |= I40E_FLAG_GENEVE_OFFLOAD_CAPABLE;
-	}
-
-	/* Enable HW ATR eviction if possible */
-	if (pf->flags & I40E_FLAG_HW_ATR_EVICT_CAPABLE)
-		pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED;
-
 	pf->eeprom_version = 0xDEAD;
 	pf->lan_veb = I40E_NO_VEB;
 	pf->lan_vsi = I40E_NO_VSI;
@@ -9231,7 +9215,7 @@ static void i40e_udp_tunnel_add(struct net_device *netdev,
 		pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN;
 		break;
 	case UDP_TUNNEL_TYPE_GENEVE:
-		if (!(pf->flags & I40E_FLAG_GENEVE_OFFLOAD_CAPABLE))
+		if (!(pf->hw_features & I40E_HW_GENEVE_OFFLOAD_CAPABLE))
 			return;
 		pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE;
 		break;
@@ -9298,7 +9282,7 @@ static int i40e_get_phys_port_id(struct net_device *netdev,
 	struct i40e_pf *pf = np->vsi->back;
 	struct i40e_hw *hw = &pf->hw;
 
-	if (!(pf->flags & I40E_FLAG_PORT_ID_VALID))
+	if (!(pf->hw_features & I40E_HW_PORT_ID_VALID))
 		return -EOPNOTSUPP;
 
 	ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
@@ -9589,6 +9573,7 @@ static int i40e_xdp(struct net_device *dev,
 		return i40e_xdp_setup(vsi, xdp->prog);
 	case XDP_QUERY_PROG:
 		xdp->prog_attached = i40e_enabled_xdp_vsi(vsi);
+		xdp->prog_id = vsi->xdp_prog ? vsi->xdp_prog->aux->id : 0;
 		return 0;
 	default:
 		return -EINVAL;
@@ -9675,7 +9660,7 @@ static int i40e_config_netdev(struct i40e_vsi *vsi)
 			  NETIF_F_RXCSUM		|
 			  0;
 
-	if (!(pf->flags & I40E_FLAG_OUTER_UDP_CSUM_CAPABLE))
+	if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE))
 		netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
 
 	netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
@@ -9714,8 +9699,13 @@ static int i40e_config_netdev(struct i40e_vsi *vsi)
 		i40e_add_mac_filter(vsi, mac_addr);
 		spin_unlock_bh(&vsi->mac_filter_hash_lock);
 	} else {
-		/* relate the VSI_VMDQ name to the VSI_MAIN name */
-		snprintf(netdev->name, IFNAMSIZ, "%sv%%d",
+		/* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we
+		 * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to
+		 * the end, which is 4 bytes long, so force truncation of the
+		 * original name by IFNAMSIZ - 4
+		 */
+		snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d",
+			 IFNAMSIZ - 4,
 			 pf->vsi[pf->lan_vsi]->netdev->name);
 		random_ether_addr(mac_addr);
 
@@ -9756,8 +9746,7 @@ static int i40e_config_netdev(struct i40e_vsi *vsi)
 
 	/* MTU range: 68 - 9706 */
 	netdev->min_mtu = ETH_MIN_MTU;
-	netdev->max_mtu = I40E_MAX_RXBUFFER -
-			  (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
+	netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD;
 
 	return 0;
 }
@@ -9890,13 +9879,15 @@ static int i40e_add_vsi(struct i40e_vsi *vsi)
 			 */
 			ret = i40e_vsi_config_tc(vsi, enabled_tc);
 			if (ret) {
+				/* Single TC condition is not fatal,
+				 * message and continue
+				 */
 				dev_info(&pf->pdev->dev,
 					 "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n",
 					 enabled_tc,
 					 i40e_stat_str(&pf->hw, ret),
 					 i40e_aq_str(&pf->hw,
 						    pf->hw.aq.asq_last_status));
-				ret = -ENOENT;
 			}
 		}
 		break;
@@ -10387,17 +10378,6 @@ struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
 	switch (vsi->type) {
 	/* setup the netdev if needed */
 	case I40E_VSI_MAIN:
-		/* Apply relevant filters if a platform-specific mac
-		 * address was selected.
-		 */
-		if (!!(pf->flags & I40E_FLAG_PF_MAC)) {
-			ret = i40e_macaddr_init(vsi, pf->hw.mac.addr);
-			if (ret) {
-				dev_warn(&pf->pdev->dev,
-					 "could not set up macaddr; err %d\n",
-					 ret);
-			}
-		}
 	case I40E_VSI_VMDQ2:
 		ret = i40e_config_netdev(vsi);
 		if (ret)
@@ -10434,7 +10414,7 @@ struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
 		break;
 	}
 
-	if ((pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) &&
+	if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) &&
 	    (vsi->type == I40E_VSI_VMDQ2)) {
 		ret = i40e_vsi_config_rss(vsi);
 	}
@@ -11443,7 +11423,7 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	 * Ignore error return codes because if it was already disabled via
 	 * hardware settings this will fail
 	 */
-	if (pf->flags & I40E_FLAG_STOP_FW_LLDP) {
+	if (pf->hw_features & I40E_HW_STOP_FW_LLDP) {
 		dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");
 		i40e_aq_stop_lldp(hw, true, NULL);
 	}
@@ -11460,7 +11440,7 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);
 	i40e_get_port_mac_addr(hw, hw->mac.port_addr);
 	if (is_valid_ether_addr(hw->mac.port_addr))
-		pf->flags |= I40E_FLAG_PORT_ID_VALID;
+		pf->hw_features |= I40E_HW_PORT_ID_VALID;
 
 	pci_set_drvdata(pdev, pf);
 	pci_save_state(pdev);
@@ -11576,7 +11556,7 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		wr32(hw, I40E_REG_MSS, val);
 	}
 
-	if (pf->flags & I40E_FLAG_RESTART_AUTONEG) {
+	if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
 		msleep(75);
 		err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
 		if (err)
@@ -11663,7 +11643,7 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	 * and will report PCI Gen 1 x 1 by default so don't bother
 	 * checking them.
 	 */
-	if (!(pf->flags & I40E_FLAG_NO_PCI_LINK_CHECK)) {
+	if (!(pf->hw_features & I40E_HW_NO_PCI_LINK_CHECK)) {
 		char speed[PCI_SPEED_SIZE] = "Unknown";
 		char width[PCI_WIDTH_SIZE] = "Unknown";
 
@@ -11734,9 +11714,9 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||
 		(pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))
-		pf->flags |= I40E_FLAG_PHY_CONTROLS_LEDS;
+		pf->hw_features |= I40E_HW_PHY_CONTROLS_LEDS;
 	if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722)
-		pf->flags |= I40E_FLAG_HAVE_CRT_RETIMER;
+		pf->hw_features |= I40E_HW_HAVE_CRT_RETIMER;
 	/* print a string summarizing features */
 	i40e_print_features(pf);
 
@@ -12048,7 +12028,7 @@ static void i40e_shutdown(struct pci_dev *pdev)
 	 */
 	i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
 
-	if (pf->wol_en && (pf->flags & I40E_FLAG_WOL_MC_MAGIC_PKT_WAKE))
+	if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
 		i40e_enable_mc_magic_wake(pf);
 
 	i40e_prep_for_reset(pf, false);
@@ -12080,7 +12060,7 @@ static int i40e_suspend(struct pci_dev *pdev, pm_message_t state)
 	set_bit(__I40E_SUSPENDED, pf->state);
 	set_bit(__I40E_DOWN, pf->state);
 
-	if (pf->wol_en && (pf->flags & I40E_FLAG_WOL_MC_MAGIC_PKT_WAKE))
+	if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
 		i40e_enable_mc_magic_wake(pf);
 
 	i40e_prep_for_reset(pf, false);
@@ -12089,7 +12069,10 @@ static int i40e_suspend(struct pci_dev *pdev, pm_message_t state)
 	wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
 
 	i40e_stop_misc_vector(pf);
-
+	if (pf->msix_entries) {
+		synchronize_irq(pf->msix_entries[0].vector);
+		free_irq(pf->msix_entries[0].vector, pf);
+	}
 	retval = pci_save_state(pdev);
 	if (retval)
 		return retval;
@@ -12129,6 +12112,15 @@ static int i40e_resume(struct pci_dev *pdev)
 	/* handling the reset will rebuild the device state */
 	if (test_and_clear_bit(__I40E_SUSPENDED, pf->state)) {
 		clear_bit(__I40E_DOWN, pf->state);
+		if (pf->msix_entries) {
+			err = request_irq(pf->msix_entries[0].vector,
+					  i40e_intr, 0, pf->int_name, pf);
+			if (err) {
+				dev_err(&pf->pdev->dev,
+					"request_irq for %s failed: %d\n",
+					pf->int_name, err);
+			}
+		}
 		i40e_reset_and_rebuild(pf, false, false);
 	}
 
@@ -12168,12 +12160,14 @@ static int __init i40e_init_module(void)
 		i40e_driver_string, i40e_driver_version_str);
 	pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
 
-	/* we will see if single thread per module is enough for now,
-	 * it can't be any worse than using the system workqueue which
-	 * was already single threaded
+	/* There is no need to throttle the number of active tasks because
+	 * each device limits its own task using a state bit for scheduling
+	 * the service task, and the device tasks do not interfere with each
+	 * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM
+	 * since we need to be able to guarantee forward progress even under
+	 * memory pressure.
 	 */
-	i40e_wq = alloc_workqueue("%s", WQ_UNBOUND | WQ_MEM_RECLAIM, 1,
-				  i40e_driver_name);
+	i40e_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, i40e_driver_name);
 	if (!i40e_wq) {
 		pr_err("%s: Failed to create workqueue\n", i40e_driver_name);
 		return -ENOMEM;
diff --git a/drivers/net/ethernet/intel/i40e/i40e_nvm.c b/drivers/net/ethernet/intel/i40e/i40e_nvm.c
index 800bd55d0159..57505b1df98d 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_nvm.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_nvm.c
@@ -134,8 +134,25 @@ i40e_i40e_acquire_nvm_exit:
  **/
 void i40e_release_nvm(struct i40e_hw *hw)
 {
-	if (!hw->nvm.blank_nvm_mode)
-		i40e_aq_release_resource(hw, I40E_NVM_RESOURCE_ID, 0, NULL);
+	i40e_status ret_code = I40E_SUCCESS;
+	u32 total_delay = 0;
+
+	if (hw->nvm.blank_nvm_mode)
+		return;
+
+	ret_code = i40e_aq_release_resource(hw, I40E_NVM_RESOURCE_ID, 0, NULL);
+
+	/* there are some rare cases when trying to release the resource
+	 * results in an admin Q timeout, so handle them correctly
+	 */
+	while ((ret_code == I40E_ERR_ADMIN_QUEUE_TIMEOUT) &&
+	       (total_delay < hw->aq.asq_cmd_timeout)) {
+		usleep_range(1000, 2000);
+		ret_code = i40e_aq_release_resource(hw,
+						    I40E_NVM_RESOURCE_ID,
+						    0, NULL);
+		total_delay++;
+	}
 }
 
 /**
@@ -230,6 +247,7 @@ static i40e_status i40e_read_nvm_aq(struct i40e_hw *hw, u8 module_pointer,
 	struct i40e_asq_cmd_details cmd_details;
 
 	memset(&cmd_details, 0, sizeof(cmd_details));
+	cmd_details.wb_desc = &hw->nvm_wb_desc;
 
 	/* Here we are checking the SR limit only for the flat memory model.
 	 * We cannot do it for the module-based model, as we did not acquire
@@ -266,7 +284,7 @@ static i40e_status i40e_read_nvm_aq(struct i40e_hw *hw, u8 module_pointer,
  * @offset: offset of the Shadow RAM word to read (0x000000 - 0x001FFF)
  * @data: word read from the Shadow RAM
  *
- * Reads one 16 bit word from the Shadow RAM using the GLNVM_SRCTL register.
+ * Reads one 16 bit word from the Shadow RAM using the AdminQ
  **/
 static i40e_status i40e_read_nvm_word_aq(struct i40e_hw *hw, u16 offset,
 					 u16 *data)
@@ -280,27 +298,49 @@ static i40e_status i40e_read_nvm_word_aq(struct i40e_hw *hw, u16 offset,
 }
 
 /**
- * i40e_read_nvm_word - Reads Shadow RAM
+ * __i40e_read_nvm_word - Reads nvm word, assumes called does the locking
  * @hw: pointer to the HW structure
  * @offset: offset of the Shadow RAM word to read (0x000000 - 0x001FFF)
  * @data: word read from the Shadow RAM
  *
- * Reads one 16 bit word from the Shadow RAM using the GLNVM_SRCTL register.
+ * Reads one 16 bit word from the Shadow RAM.
+ *
+ * Do not use this function except in cases where the nvm lock is already
+ * taken via i40e_acquire_nvm().
+ **/
+static i40e_status __i40e_read_nvm_word(struct i40e_hw *hw,
+					u16 offset, u16 *data)
+{
+	i40e_status ret_code = 0;
+
+	if (hw->flags & I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE)
+		ret_code = i40e_read_nvm_word_aq(hw, offset, data);
+	else
+		ret_code = i40e_read_nvm_word_srctl(hw, offset, data);
+	return ret_code;
+}
+
+/**
+ * i40e_read_nvm_word - Reads nvm word and acquire lock if necessary
+ * @hw: pointer to the HW structure
+ * @offset: offset of the Shadow RAM word to read (0x000000 - 0x001FFF)
+ * @data: word read from the Shadow RAM
+ *
+ * Reads one 16 bit word from the Shadow RAM.
  **/
 i40e_status i40e_read_nvm_word(struct i40e_hw *hw, u16 offset,
 			       u16 *data)
 {
-	enum i40e_status_code ret_code = 0;
+	i40e_status ret_code = 0;
 
 	ret_code = i40e_acquire_nvm(hw, I40E_RESOURCE_READ);
-	if (!ret_code) {
-		if (hw->flags & I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE) {
-			ret_code = i40e_read_nvm_word_aq(hw, offset, data);
-		} else {
-			ret_code = i40e_read_nvm_word_srctl(hw, offset, data);
-		}
-		i40e_release_nvm(hw);
-	}
+	if (ret_code)
+		return ret_code;
+
+	ret_code = __i40e_read_nvm_word(hw, offset, data);
+
+	i40e_release_nvm(hw);
+
 	return ret_code;
 }
 
@@ -393,31 +433,25 @@ read_nvm_buffer_aq_exit:
 }
 
 /**
- * i40e_read_nvm_buffer - Reads Shadow RAM buffer
+ * __i40e_read_nvm_buffer - Reads nvm buffer, caller must acquire lock
  * @hw: pointer to the HW structure
  * @offset: offset of the Shadow RAM word to read (0x000000 - 0x001FFF).
  * @words: (in) number of words to read; (out) number of words actually read
  * @data: words read from the Shadow RAM
  *
  * Reads 16 bit words (data buffer) from the SR using the i40e_read_nvm_srrd()
- * method. The buffer read is preceded by the NVM ownership take
- * and followed by the release.
+ * method.
  **/
-i40e_status i40e_read_nvm_buffer(struct i40e_hw *hw, u16 offset,
-				 u16 *words, u16 *data)
+static i40e_status __i40e_read_nvm_buffer(struct i40e_hw *hw,
+					  u16 offset, u16 *words,
+					  u16 *data)
 {
-	enum i40e_status_code ret_code = 0;
+	i40e_status ret_code = 0;
 
-	if (hw->flags & I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE) {
-		ret_code = i40e_acquire_nvm(hw, I40E_RESOURCE_READ);
-		if (!ret_code) {
-			ret_code = i40e_read_nvm_buffer_aq(hw, offset, words,
-							   data);
-			i40e_release_nvm(hw);
-		}
-	} else {
+	if (hw->flags & I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE)
+		ret_code = i40e_read_nvm_buffer_aq(hw, offset, words, data);
+	else
 		ret_code = i40e_read_nvm_buffer_srctl(hw, offset, words, data);
-	}
 	return ret_code;
 }
 
@@ -499,15 +533,15 @@ static i40e_status i40e_calc_nvm_checksum(struct i40e_hw *hw,
 	data = (u16 *)vmem.va;
 
 	/* read pointer to VPD area */
-	ret_code = i40e_read_nvm_word(hw, I40E_SR_VPD_PTR, &vpd_module);
+	ret_code = __i40e_read_nvm_word(hw, I40E_SR_VPD_PTR, &vpd_module);
 	if (ret_code) {
 		ret_code = I40E_ERR_NVM_CHECKSUM;
 		goto i40e_calc_nvm_checksum_exit;
 	}
 
 	/* read pointer to PCIe Alt Auto-load module */
-	ret_code = i40e_read_nvm_word(hw, I40E_SR_PCIE_ALT_AUTO_LOAD_PTR,
-				      &pcie_alt_module);
+	ret_code = __i40e_read_nvm_word(hw, I40E_SR_PCIE_ALT_AUTO_LOAD_PTR,
+					&pcie_alt_module);
 	if (ret_code) {
 		ret_code = I40E_ERR_NVM_CHECKSUM;
 		goto i40e_calc_nvm_checksum_exit;
@@ -521,7 +555,7 @@ static i40e_status i40e_calc_nvm_checksum(struct i40e_hw *hw,
 		if ((i % I40E_SR_SECTOR_SIZE_IN_WORDS) == 0) {
 			u16 words = I40E_SR_SECTOR_SIZE_IN_WORDS;
 
-			ret_code = i40e_read_nvm_buffer(hw, i, &words, data);
+			ret_code = __i40e_read_nvm_buffer(hw, i, &words, data);
 			if (ret_code) {
 				ret_code = I40E_ERR_NVM_CHECKSUM;
 				goto i40e_calc_nvm_checksum_exit;
@@ -593,14 +627,19 @@ i40e_status i40e_validate_nvm_checksum(struct i40e_hw *hw,
 	u16 checksum_sr = 0;
 	u16 checksum_local = 0;
 
+	/* We must acquire the NVM lock in order to correctly synchronize the
+	 * NVM accesses across multiple PFs. Without doing so it is possible
+	 * for one of the PFs to read invalid data potentially indicating that
+	 * the checksum is invalid.
+	 */
+	ret_code = i40e_acquire_nvm(hw, I40E_RESOURCE_READ);
+	if (ret_code)
+		return ret_code;
 	ret_code = i40e_calc_nvm_checksum(hw, &checksum_local);
+	__i40e_read_nvm_word(hw, I40E_SR_SW_CHECKSUM_WORD, &checksum_sr);
+	i40e_release_nvm(hw);
 	if (ret_code)
-		goto i40e_validate_nvm_checksum_exit;
-
-	/* Do not use i40e_read_nvm_word() because we do not want to take
-	 * the synchronization semaphores twice here.
-	 */
-	i40e_read_nvm_word(hw, I40E_SR_SW_CHECKSUM_WORD, &checksum_sr);
+		return ret_code;
 
 	/* Verify read checksum from EEPROM is the same as
 	 * calculated checksum
@@ -612,7 +651,6 @@ i40e_status i40e_validate_nvm_checksum(struct i40e_hw *hw,
 	if (checksum)
 		*checksum = checksum_local;
 
-i40e_validate_nvm_checksum_exit:
 	return ret_code;
 }
 
@@ -736,6 +774,15 @@ i40e_status i40e_nvmupd_command(struct i40e_hw *hw,
 		hw->nvmupd_state = I40E_NVMUPD_STATE_INIT;
 	}
 
+	/* Acquire lock to prevent race condition where adminq_task
+	 * can execute after i40e_nvmupd_nvm_read/write but before state
+	 * variables (nvm_wait_opcode, nvm_release_on_done) are updated.
+	 *
+	 * During NVMUpdate, it is observed that lock could be held for
+	 * ~5ms for most commands. However lock is held for ~60ms for
+	 * NVMUPD_CSUM_LCB command.
+	 */
+	mutex_lock(&hw->aq.arq_mutex);
 	switch (hw->nvmupd_state) {
 	case I40E_NVMUPD_STATE_INIT:
 		status = i40e_nvmupd_state_init(hw, cmd, bytes, perrno);
@@ -756,7 +803,8 @@ i40e_status i40e_nvmupd_command(struct i40e_hw *hw,
 		 */
 		if (cmd->offset == 0xffff) {
 			i40e_nvmupd_check_wait_event(hw, hw->nvm_wait_opcode);
-			return 0;
+			status = 0;
+			goto exit;
 		}
 
 		status = I40E_ERR_NOT_READY;
@@ -771,6 +819,8 @@ i40e_status i40e_nvmupd_command(struct i40e_hw *hw,
 		*perrno = -ESRCH;
 		break;
 	}
+exit:
+	mutex_unlock(&hw->aq.arq_mutex);
 	return status;
 }
 
@@ -997,6 +1047,7 @@ retry:
 		break;
 
 	case I40E_NVMUPD_CSUM_CON:
+		/* Assumes the caller has acquired the nvm */
 		status = i40e_update_nvm_checksum(hw);
 		if (status) {
 			*perrno = hw->aq.asq_last_status ?
@@ -1011,6 +1062,7 @@ retry:
 		break;
 
 	case I40E_NVMUPD_CSUM_LCB:
+		/* Assumes the caller has acquired the nvm */
 		status = i40e_update_nvm_checksum(hw);
 		if (status) {
 			*perrno = hw->aq.asq_last_status ?
diff --git a/drivers/net/ethernet/intel/i40e/i40e_prototype.h b/drivers/net/ethernet/intel/i40e/i40e_prototype.h
index df613ea40313..a39b13197891 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_prototype.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_prototype.h
@@ -311,8 +311,6 @@ i40e_status i40e_acquire_nvm(struct i40e_hw *hw,
 void i40e_release_nvm(struct i40e_hw *hw);
 i40e_status i40e_read_nvm_word(struct i40e_hw *hw, u16 offset,
 					 u16 *data);
-i40e_status i40e_read_nvm_buffer(struct i40e_hw *hw, u16 offset,
-					   u16 *words, u16 *data);
 i40e_status i40e_update_nvm_checksum(struct i40e_hw *hw);
 i40e_status i40e_validate_nvm_checksum(struct i40e_hw *hw,
 						 u16 *checksum);
diff --git a/drivers/net/ethernet/intel/i40e/i40e_ptp.c b/drivers/net/ethernet/intel/i40e/i40e_ptp.c
index 1a0be835fa06..d8456c381c99 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_ptp.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_ptp.c
@@ -158,13 +158,12 @@ static int i40e_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
 static int i40e_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
 {
 	struct i40e_pf *pf = container_of(ptp, struct i40e_pf, ptp_caps);
-	struct timespec64 now, then;
+	struct timespec64 now;
 
-	then = ns_to_timespec64(delta);
 	mutex_lock(&pf->tmreg_lock);
 
 	i40e_ptp_read(pf, &now);
-	now = timespec64_add(now, then);
+	timespec64_add_ns(&now, delta);
 	i40e_ptp_write(pf, (const struct timespec64 *)&now);
 
 	mutex_unlock(&pf->tmreg_lock);
@@ -570,7 +569,7 @@ static int i40e_ptp_set_timestamp_mode(struct i40e_pf *pf,
 	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
 	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
 	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
-		if (!(pf->flags & I40E_FLAG_PTP_L4_CAPABLE))
+		if (!(pf->hw_features & I40E_HW_PTP_L4_CAPABLE))
 			return -ERANGE;
 		pf->ptp_rx = true;
 		tsyntype = I40E_PRTTSYN_CTL1_V1MESSTYPE0_MASK |
@@ -584,7 +583,7 @@ static int i40e_ptp_set_timestamp_mode(struct i40e_pf *pf,
 	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
 	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
 	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
-		if (!(pf->flags & I40E_FLAG_PTP_L4_CAPABLE))
+		if (!(pf->hw_features & I40E_HW_PTP_L4_CAPABLE))
 			return -ERANGE;
 		/* fall through */
 	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
@@ -593,7 +592,7 @@ static int i40e_ptp_set_timestamp_mode(struct i40e_pf *pf,
 		pf->ptp_rx = true;
 		tsyntype = I40E_PRTTSYN_CTL1_V2MESSTYPE0_MASK |
 			   I40E_PRTTSYN_CTL1_TSYNTYPE_V2;
-		if (pf->flags & I40E_FLAG_PTP_L4_CAPABLE) {
+		if (pf->hw_features & I40E_HW_PTP_L4_CAPABLE) {
 			tsyntype |= I40E_PRTTSYN_CTL1_UDP_ENA_MASK;
 			config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
 		} else {
diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
index 2194960d5855..1519dfb851d0 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
@@ -860,7 +860,7 @@ static bool i40e_clean_tx_irq(struct i40e_vsi *vsi,
 	netdev_tx_completed_queue(txring_txq(tx_ring),
 				  total_packets, total_bytes);
 
-#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
+#define TX_WAKE_THRESHOLD ((s16)(DESC_NEEDED * 2))
 	if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
 		     (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
 		/* Make sure that anybody stopping the queue after this
@@ -959,19 +959,31 @@ void i40e_force_wb(struct i40e_vsi *vsi, struct i40e_q_vector *q_vector)
 static bool i40e_set_new_dynamic_itr(struct i40e_ring_container *rc)
 {
 	enum i40e_latency_range new_latency_range = rc->latency_range;
-	struct i40e_q_vector *qv = rc->ring->q_vector;
 	u32 new_itr = rc->itr;
 	int bytes_per_int;
-	int usecs;
+	unsigned int usecs, estimated_usecs;
 
 	if (rc->total_packets == 0 || !rc->itr)
 		return false;
 
+	usecs = (rc->itr << 1) * ITR_COUNTDOWN_START;
+	bytes_per_int = rc->total_bytes / usecs;
+
+	/* The calculations in this algorithm depend on interrupts actually
+	 * firing at the ITR rate. This may not happen if the packet rate is
+	 * really low, or if we've been napi polling. Check to make sure
+	 * that's not the case before we continue.
+	 */
+	estimated_usecs = jiffies_to_usecs(jiffies - rc->last_itr_update);
+	if (estimated_usecs > usecs) {
+		new_latency_range = I40E_LOW_LATENCY;
+		goto reset_latency;
+	}
+
 	/* simple throttlerate management
 	 *   0-10MB/s   lowest (50000 ints/s)
 	 *  10-20MB/s   low    (20000 ints/s)
 	 *  20-1249MB/s bulk   (18000 ints/s)
-	 *  > 40000 Rx packets per second (8000 ints/s)
 	 *
 	 * The math works out because the divisor is in 10^(-6) which
 	 * turns the bytes/us input value into MB/s values, but
@@ -979,9 +991,6 @@ static bool i40e_set_new_dynamic_itr(struct i40e_ring_container *rc)
 	 * are in 2 usec increments in the ITR registers, and make sure
 	 * to use the smoothed values that the countdown timer gives us.
 	 */
-	usecs = (rc->itr << 1) * ITR_COUNTDOWN_START;
-	bytes_per_int = rc->total_bytes / usecs;
-
 	switch (new_latency_range) {
 	case I40E_LOWEST_LATENCY:
 		if (bytes_per_int > 10)
@@ -994,24 +1003,13 @@ static bool i40e_set_new_dynamic_itr(struct i40e_ring_container *rc)
 			new_latency_range = I40E_LOWEST_LATENCY;
 		break;
 	case I40E_BULK_LATENCY:
-	case I40E_ULTRA_LATENCY:
 	default:
 		if (bytes_per_int <= 20)
 			new_latency_range = I40E_LOW_LATENCY;
 		break;
 	}
 
-	/* this is to adjust RX more aggressively when streaming small
-	 * packets.  The value of 40000 was picked as it is just beyond
-	 * what the hardware can receive per second if in low latency
-	 * mode.
-	 */
-#define RX_ULTRA_PACKET_RATE 40000
-
-	if ((((rc->total_packets * 1000000) / usecs) > RX_ULTRA_PACKET_RATE) &&
-	    (&qv->rx == rc))
-		new_latency_range = I40E_ULTRA_LATENCY;
-
+reset_latency:
 	rc->latency_range = new_latency_range;
 
 	switch (new_latency_range) {
@@ -1024,21 +1022,18 @@ static bool i40e_set_new_dynamic_itr(struct i40e_ring_container *rc)
 	case I40E_BULK_LATENCY:
 		new_itr = I40E_ITR_18K;
 		break;
-	case I40E_ULTRA_LATENCY:
-		new_itr = I40E_ITR_8K;
-		break;
 	default:
 		break;
 	}
 
 	rc->total_bytes = 0;
 	rc->total_packets = 0;
+	rc->last_itr_update = jiffies;
 
 	if (new_itr != rc->itr) {
 		rc->itr = new_itr;
 		return true;
 	}
-
 	return false;
 }
 
@@ -2065,7 +2060,7 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
 	u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
 	bool failure = false, xdp_xmit = false;
 
-	while (likely(total_rx_packets < budget)) {
+	while (likely(total_rx_packets < (unsigned int)budget)) {
 		struct i40e_rx_buffer *rx_buffer;
 		union i40e_rx_desc *rx_desc;
 		struct xdp_buff xdp;
@@ -2198,7 +2193,7 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
 	rx_ring->q_vector->rx.total_bytes += total_rx_bytes;
 
 	/* guarantee a trip back through this routine if there was a failure */
-	return failure ? budget : total_rx_packets;
+	return failure ? budget : (int)total_rx_packets;
 }
 
 static u32 i40e_buildreg_itr(const int type, const u16 itr)
@@ -2243,6 +2238,12 @@ static inline void i40e_update_enable_itr(struct i40e_vsi *vsi,
 	int idx = q_vector->v_idx;
 	int rx_itr_setting, tx_itr_setting;
 
+	/* If we don't have MSIX, then we only need to re-enable icr0 */
+	if (!(vsi->back->flags & I40E_FLAG_MSIX_ENABLED)) {
+		i40e_irq_dynamic_enable_icr0(vsi->back, false);
+		return;
+	}
+
 	vector = (q_vector->v_idx + vsi->base_vector);
 
 	/* avoid dynamic calculation if in countdown mode OR if
@@ -2363,7 +2364,6 @@ int i40e_napi_poll(struct napi_struct *napi, int budget)
 
 	/* If work not completed, return budget and polling will return */
 	if (!clean_complete) {
-		const cpumask_t *aff_mask = &q_vector->affinity_mask;
 		int cpu_id = smp_processor_id();
 
 		/* It is possible that the interrupt affinity has changed but,
@@ -2373,15 +2373,22 @@ int i40e_napi_poll(struct napi_struct *napi, int budget)
 		 * continue to poll, otherwise we must stop polling so the
 		 * interrupt can move to the correct cpu.
 		 */
-		if (likely(cpumask_test_cpu(cpu_id, aff_mask) ||
-			   !(vsi->back->flags & I40E_FLAG_MSIX_ENABLED))) {
+		if (!cpumask_test_cpu(cpu_id, &q_vector->affinity_mask)) {
+			/* Tell napi that we are done polling */
+			napi_complete_done(napi, work_done);
+
+			/* Force an interrupt */
+			i40e_force_wb(vsi, q_vector);
+
+			/* Return budget-1 so that polling stops */
+			return budget - 1;
+		}
 tx_only:
-			if (arm_wb) {
-				q_vector->tx.ring[0].tx_stats.tx_force_wb++;
-				i40e_enable_wb_on_itr(vsi, q_vector);
-			}
-			return budget;
+		if (arm_wb) {
+			q_vector->tx.ring[0].tx_stats.tx_force_wb++;
+			i40e_enable_wb_on_itr(vsi, q_vector);
 		}
+		return budget;
 	}
 
 	if (vsi->back->flags & I40E_TXR_FLAGS_WB_ON_ITR)
@@ -2390,16 +2397,7 @@ tx_only:
 	/* Work is done so exit the polling mode and re-enable the interrupt */
 	napi_complete_done(napi, work_done);
 
-	/* If we're prematurely stopping polling to fix the interrupt
-	 * affinity we want to make sure polling starts back up so we
-	 * issue a call to i40e_force_wb which triggers a SW interrupt.
-	 */
-	if (!clean_complete)
-		i40e_force_wb(vsi, q_vector);
-	else if (!(vsi->back->flags & I40E_FLAG_MSIX_ENABLED))
-		i40e_irq_dynamic_enable_icr0(vsi->back, false);
-	else
-		i40e_update_enable_itr(vsi, q_vector);
+	i40e_update_enable_itr(vsi, q_vector);
 
 	return min(work_done, budget - 1);
 }
@@ -2453,9 +2451,15 @@ static void i40e_atr(struct i40e_ring *tx_ring, struct sk_buff *skb,
 		hlen = (hdr.network[0] & 0x0F) << 2;
 		l4_proto = hdr.ipv4->protocol;
 	} else {
-		hlen = hdr.network - skb->data;
-		l4_proto = ipv6_find_hdr(skb, &hlen, IPPROTO_TCP, NULL, NULL);
-		hlen -= hdr.network - skb->data;
+		/* find the start of the innermost ipv6 header */
+		unsigned int inner_hlen = hdr.network - skb->data;
+		unsigned int h_offset = inner_hlen;
+
+		/* this function updates h_offset to the end of the header */
+		l4_proto =
+		  ipv6_find_hdr(skb, &h_offset, IPPROTO_TCP, NULL, NULL);
+		/* hlen will contain our best estimate of the tcp header */
+		hlen = h_offset - inner_hlen;
 	}
 
 	if (l4_proto != IPPROTO_TCP)
diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.h b/drivers/net/ethernet/intel/i40e/i40e_txrx.h
index b288d58313a6..2f848bc5e391 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_txrx.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.h
@@ -112,7 +112,7 @@ enum i40e_dyn_idx_t {
 	BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP))
 
 #define i40e_pf_get_default_rss_hena(pf) \
-	(((pf)->flags & I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE) ? \
+	(((pf)->hw_features & I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE) ? \
 	  I40E_DEFAULT_RSS_HENA_EXPANDED : I40E_DEFAULT_RSS_HENA)
 
 /* Supported Rx Buffer Sizes (a multiple of 128) */
@@ -130,6 +130,7 @@ enum i40e_dyn_idx_t {
  * i.e. RXBUFFER_512 --> 1216 byte skb (size-2048 slab)
  */
 #define I40E_RX_HDR_SIZE I40E_RXBUFFER_256
+#define I40E_PACKET_HDR_PAD (ETH_HLEN + ETH_FCS_LEN + (VLAN_HLEN * 2))
 #define i40e_rx_desc i40e_32byte_rx_desc
 
 #define I40E_RX_DMA_ATTR \
@@ -453,7 +454,6 @@ enum i40e_latency_range {
 	I40E_LOWEST_LATENCY = 0,
 	I40E_LOW_LATENCY = 1,
 	I40E_BULK_LATENCY = 2,
-	I40E_ULTRA_LATENCY = 3,
 };
 
 struct i40e_ring_container {
@@ -461,6 +461,7 @@ struct i40e_ring_container {
 	struct i40e_ring *ring;
 	unsigned int total_bytes;	/* total bytes processed this int */
 	unsigned int total_packets;	/* total packets processed this int */
+	unsigned long last_itr_update;	/* jiffies of last ITR update */
 	u16 count;
 	enum i40e_latency_range latency_range;
 	u16 itr;
diff --git a/drivers/net/ethernet/intel/i40e/i40e_type.h b/drivers/net/ethernet/intel/i40e/i40e_type.h
index 3a18ed13edc4..fd4bbdd88b57 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_type.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_type.h
@@ -185,6 +185,7 @@ struct i40e_link_status {
 	enum i40e_aq_link_speed link_speed;
 	u8 link_info;
 	u8 an_info;
+	u8 req_fec_info;
 	u8 fec_info;
 	u8 ext_info;
 	u8 loopback;
diff --git a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
index ecbe40ea8ffe..4d1e670f490e 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
@@ -1528,54 +1528,54 @@ static int i40e_vc_get_vf_resources_msg(struct i40e_vf *vf, u8 *msg)
 				  VIRTCHNL_VF_OFFLOAD_RSS_REG |
 				  VIRTCHNL_VF_OFFLOAD_VLAN;
 
-	vfres->vf_offload_flags = VIRTCHNL_VF_OFFLOAD_L2;
+	vfres->vf_cap_flags = VIRTCHNL_VF_OFFLOAD_L2;
 	vsi = pf->vsi[vf->lan_vsi_idx];
 	if (!vsi->info.pvid)
-		vfres->vf_offload_flags |= VIRTCHNL_VF_OFFLOAD_VLAN;
+		vfres->vf_cap_flags |= VIRTCHNL_VF_OFFLOAD_VLAN;
 
 	if (i40e_vf_client_capable(pf, vf->vf_id) &&
 	    (vf->driver_caps & VIRTCHNL_VF_OFFLOAD_IWARP)) {
-		vfres->vf_offload_flags |= VIRTCHNL_VF_OFFLOAD_IWARP;
+		vfres->vf_cap_flags |= VIRTCHNL_VF_OFFLOAD_IWARP;
 		set_bit(I40E_VF_STATE_IWARPENA, &vf->vf_states);
 	}
 
 	if (vf->driver_caps & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
-		vfres->vf_offload_flags |= VIRTCHNL_VF_OFFLOAD_RSS_PF;
+		vfres->vf_cap_flags |= VIRTCHNL_VF_OFFLOAD_RSS_PF;
 	} else {
-		if ((pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) &&
+		if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) &&
 		    (vf->driver_caps & VIRTCHNL_VF_OFFLOAD_RSS_AQ))
-			vfres->vf_offload_flags |= VIRTCHNL_VF_OFFLOAD_RSS_AQ;
+			vfres->vf_cap_flags |= VIRTCHNL_VF_OFFLOAD_RSS_AQ;
 		else
-			vfres->vf_offload_flags |= VIRTCHNL_VF_OFFLOAD_RSS_REG;
+			vfres->vf_cap_flags |= VIRTCHNL_VF_OFFLOAD_RSS_REG;
 	}
 
-	if (pf->flags & I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE) {
+	if (pf->hw_features & I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE) {
 		if (vf->driver_caps & VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2)
-			vfres->vf_offload_flags |=
+			vfres->vf_cap_flags |=
 				VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2;
 	}
 
 	if (vf->driver_caps & VIRTCHNL_VF_OFFLOAD_ENCAP)
-		vfres->vf_offload_flags |= VIRTCHNL_VF_OFFLOAD_ENCAP;
+		vfres->vf_cap_flags |= VIRTCHNL_VF_OFFLOAD_ENCAP;
 
-	if ((pf->flags & I40E_FLAG_OUTER_UDP_CSUM_CAPABLE) &&
+	if ((pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE) &&
 	    (vf->driver_caps & VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM))
-		vfres->vf_offload_flags |= VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM;
+		vfres->vf_cap_flags |= VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM;
 
 	if (vf->driver_caps & VIRTCHNL_VF_OFFLOAD_RX_POLLING) {
 		if (pf->flags & I40E_FLAG_MFP_ENABLED) {
 			dev_err(&pf->pdev->dev,
 				"VF %d requested polling mode: this feature is supported only when the device is running in single function per port (SFP) mode\n",
 				 vf->vf_id);
-			ret = I40E_ERR_PARAM;
+			aq_ret = I40E_ERR_PARAM;
 			goto err;
 		}
-		vfres->vf_offload_flags |= VIRTCHNL_VF_OFFLOAD_RX_POLLING;
+		vfres->vf_cap_flags |= VIRTCHNL_VF_OFFLOAD_RX_POLLING;
 	}
 
-	if (pf->flags & I40E_FLAG_WB_ON_ITR_CAPABLE) {
+	if (pf->hw_features & I40E_HW_WB_ON_ITR_CAPABLE) {
 		if (vf->driver_caps & VIRTCHNL_VF_OFFLOAD_WB_ON_ITR)
-			vfres->vf_offload_flags |=
+			vfres->vf_cap_flags |=
 					VIRTCHNL_VF_OFFLOAD_WB_ON_ITR;
 	}
 
@@ -1741,16 +1741,14 @@ static int i40e_vc_config_promiscuous_mode_msg(struct i40e_vf *vf,
 							    NULL);
 	} else if (i40e_getnum_vf_vsi_vlan_filters(vsi)) {
 		hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {
-			aq_ret = 0;
-			if (f->vlan >= 0 && f->vlan <= I40E_MAX_VLANID) {
-				aq_ret =
-				i40e_aq_set_vsi_uc_promisc_on_vlan(hw,
-								   vsi->seid,
-								   alluni,
-								   f->vlan,
-								   NULL);
-				aq_err = pf->hw.aq.asq_last_status;
-			}
+			if (f->vlan < 0 || f->vlan > I40E_MAX_VLANID)
+				continue;
+			aq_ret = i40e_aq_set_vsi_uc_promisc_on_vlan(hw,
+								    vsi->seid,
+								    alluni,
+								    f->vlan,
+								    NULL);
+			aq_err = pf->hw.aq.asq_last_status;
 			if (aq_ret)
 				dev_err(&pf->pdev->dev,
 					"Could not add VLAN %d to Unicast promiscuous domain err %s aq_err %s\n",
@@ -1760,7 +1758,7 @@ static int i40e_vc_config_promiscuous_mode_msg(struct i40e_vf *vf,
 		}
 	} else {
 		aq_ret = i40e_aq_set_vsi_unicast_promiscuous(hw, vsi->seid,
-							     allmulti, NULL,
+							     alluni, NULL,
 							     true);
 		aq_err = pf->hw.aq.asq_last_status;
 		if (aq_ret) {
@@ -2532,6 +2530,60 @@ err:
 }
 
 /**
+ * i40e_vc_enable_vlan_stripping
+ * @vf: pointer to the VF info
+ * @msg: pointer to the msg buffer
+ * @msglen: msg length
+ *
+ * Enable vlan header stripping for the VF
+ **/
+static int i40e_vc_enable_vlan_stripping(struct i40e_vf *vf, u8 *msg,
+					 u16 msglen)
+{
+	struct i40e_vsi *vsi = vf->pf->vsi[vf->lan_vsi_idx];
+	i40e_status aq_ret = 0;
+
+	if (!test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states)) {
+		aq_ret = I40E_ERR_PARAM;
+		goto err;
+	}
+
+	i40e_vlan_stripping_enable(vsi);
+
+	/* send the response to the VF */
+err:
+	return i40e_vc_send_resp_to_vf(vf, VIRTCHNL_OP_ENABLE_VLAN_STRIPPING,
+				       aq_ret);
+}
+
+/**
+ * i40e_vc_disable_vlan_stripping
+ * @vf: pointer to the VF info
+ * @msg: pointer to the msg buffer
+ * @msglen: msg length
+ *
+ * Disable vlan header stripping for the VF
+ **/
+static int i40e_vc_disable_vlan_stripping(struct i40e_vf *vf, u8 *msg,
+					  u16 msglen)
+{
+	struct i40e_vsi *vsi = vf->pf->vsi[vf->lan_vsi_idx];
+	i40e_status aq_ret = 0;
+
+	if (!test_bit(I40E_VF_STATE_ACTIVE, &vf->vf_states)) {
+		aq_ret = I40E_ERR_PARAM;
+		goto err;
+	}
+
+	i40e_vlan_stripping_disable(vsi);
+
+	/* send the response to the VF */
+err:
+	return i40e_vc_send_resp_to_vf(vf, VIRTCHNL_OP_DISABLE_VLAN_STRIPPING,
+				       aq_ret);
+}
+
+/**
  * i40e_vc_process_vf_msg
  * @pf: pointer to the PF structure
  * @vf_id: source VF id
@@ -2650,6 +2702,12 @@ int i40e_vc_process_vf_msg(struct i40e_pf *pf, s16 vf_id, u32 v_opcode,
 	case VIRTCHNL_OP_SET_RSS_HENA:
 		ret = i40e_vc_set_rss_hena(vf, msg, msglen);
 		break;
+	case VIRTCHNL_OP_ENABLE_VLAN_STRIPPING:
+		ret = i40e_vc_enable_vlan_stripping(vf, msg, msglen);
+		break;
+	case VIRTCHNL_OP_DISABLE_VLAN_STRIPPING:
+		ret = i40e_vc_disable_vlan_stripping(vf, msg, msglen);
+		break;
 
 	case VIRTCHNL_OP_UNKNOWN:
 	default:
@@ -2764,7 +2822,6 @@ int i40e_ndo_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac)
 
 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
 
-	dev_info(&pf->pdev->dev, "Setting MAC %pM on VF %d\n", mac, vf_id);
 	/* program mac filter */
 	if (i40e_sync_vsi_filters(vsi)) {
 		dev_err(&pf->pdev->dev, "Unable to program ucast filters\n");
@@ -2772,7 +2829,16 @@ int i40e_ndo_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac)
 		goto error_param;
 	}
 	ether_addr_copy(vf->default_lan_addr.addr, mac);
-	vf->pf_set_mac = true;
+
+	if (is_zero_ether_addr(mac)) {
+		vf->pf_set_mac = false;
+		dev_info(&pf->pdev->dev, "Removing MAC on VF %d\n", vf_id);
+	} else {
+		vf->pf_set_mac = true;
+		dev_info(&pf->pdev->dev, "Setting MAC %pM on VF %d\n",
+			 mac, vf_id);
+	}
+
 	/* Force the VF driver stop so it has to reload with new MAC address */
 	i40e_vc_disable_vf(pf, vf);
 	dev_info(&pf->pdev->dev, "Reload the VF driver to make this change effective.\n");
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_common.c b/drivers/net/ethernet/intel/i40evf/i40e_common.c
index 1dd1938f594f..8d3a2bfe186a 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_common.c
+++ b/drivers/net/ethernet/intel/i40evf/i40e_common.c
@@ -333,9 +333,9 @@ void i40evf_debug_aq(struct i40e_hw *hw, enum i40e_debug_mask mask, void *desc,
 			len = buf_len;
 		/* write the full 16-byte chunks */
 		if (hw->debug_mask & mask) {
-			char prefix[20];
+			char prefix[27];
 
-			snprintf(prefix, 20,
+			snprintf(prefix, sizeof(prefix),
 				 "i40evf %02x:%02x.%x: \t0x",
 				 hw->bus.bus_id,
 				 hw->bus.device,
@@ -1104,7 +1104,7 @@ void i40e_vf_parse_hw_config(struct i40e_hw *hw,
 	hw->dev_caps.num_rx_qp = msg->num_queue_pairs;
 	hw->dev_caps.num_tx_qp = msg->num_queue_pairs;
 	hw->dev_caps.num_msix_vectors_vf = msg->max_vectors;
-	hw->dev_caps.dcb = msg->vf_offload_flags &
+	hw->dev_caps.dcb = msg->vf_cap_flags &
 			   VIRTCHNL_VF_OFFLOAD_L2;
 	hw->dev_caps.fcoe = 0;
 	for (i = 0; i < msg->num_vsis; i++) {
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_osdep.h b/drivers/net/ethernet/intel/i40evf/i40e_osdep.h
index 5e314fd3c016..a90737786c34 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_osdep.h
+++ b/drivers/net/ethernet/intel/i40evf/i40e_osdep.h
@@ -54,7 +54,7 @@ struct i40e_dma_mem {
 	void *va;
 	dma_addr_t pa;
 	u32 size;
-} __packed;
+};
 
 #define i40e_allocate_dma_mem(h, m, unused, s, a) \
 	i40evf_allocate_dma_mem_d(h, m, s, a)
@@ -63,7 +63,7 @@ struct i40e_dma_mem {
 struct i40e_virt_mem {
 	void *va;
 	u32 size;
-} __packed;
+};
 #define i40e_allocate_virt_mem(h, m, s) i40evf_allocate_virt_mem_d(h, m, s)
 #define i40e_free_virt_mem(h, m) i40evf_free_virt_mem_d(h, m)
 
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_txrx.c b/drivers/net/ethernet/intel/i40evf/i40e_txrx.c
index 12b02e530503..c32c62462c84 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_txrx.c
+++ b/drivers/net/ethernet/intel/i40evf/i40e_txrx.c
@@ -275,7 +275,7 @@ static bool i40e_clean_tx_irq(struct i40e_vsi *vsi,
 	netdev_tx_completed_queue(txring_txq(tx_ring),
 				  total_packets, total_bytes);
 
-#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
+#define TX_WAKE_THRESHOLD ((s16)(DESC_NEEDED * 2))
 	if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
 		     (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
 		/* Make sure that anybody stopping the queue after this
@@ -357,19 +357,31 @@ void i40evf_force_wb(struct i40e_vsi *vsi, struct i40e_q_vector *q_vector)
 static bool i40e_set_new_dynamic_itr(struct i40e_ring_container *rc)
 {
 	enum i40e_latency_range new_latency_range = rc->latency_range;
-	struct i40e_q_vector *qv = rc->ring->q_vector;
 	u32 new_itr = rc->itr;
 	int bytes_per_int;
-	int usecs;
+	unsigned int usecs, estimated_usecs;
 
 	if (rc->total_packets == 0 || !rc->itr)
 		return false;
 
+	usecs = (rc->itr << 1) * ITR_COUNTDOWN_START;
+	bytes_per_int = rc->total_bytes / usecs;
+
+	/* The calculations in this algorithm depend on interrupts actually
+	 * firing at the ITR rate. This may not happen if the packet rate is
+	 * really low, or if we've been napi polling. Check to make sure
+	 * that's not the case before we continue.
+	 */
+	estimated_usecs = jiffies_to_usecs(jiffies - rc->last_itr_update);
+	if (estimated_usecs > usecs) {
+		new_latency_range = I40E_LOW_LATENCY;
+		goto reset_latency;
+	}
+
 	/* simple throttlerate management
 	 *   0-10MB/s   lowest (50000 ints/s)
 	 *  10-20MB/s   low    (20000 ints/s)
 	 *  20-1249MB/s bulk   (18000 ints/s)
-	 *  > 40000 Rx packets per second (8000 ints/s)
 	 *
 	 * The math works out because the divisor is in 10^(-6) which
 	 * turns the bytes/us input value into MB/s values, but
@@ -377,9 +389,6 @@ static bool i40e_set_new_dynamic_itr(struct i40e_ring_container *rc)
 	 * are in 2 usec increments in the ITR registers, and make sure
 	 * to use the smoothed values that the countdown timer gives us.
 	 */
-	usecs = (rc->itr << 1) * ITR_COUNTDOWN_START;
-	bytes_per_int = rc->total_bytes / usecs;
-
 	switch (new_latency_range) {
 	case I40E_LOWEST_LATENCY:
 		if (bytes_per_int > 10)
@@ -392,24 +401,13 @@ static bool i40e_set_new_dynamic_itr(struct i40e_ring_container *rc)
 			new_latency_range = I40E_LOWEST_LATENCY;
 		break;
 	case I40E_BULK_LATENCY:
-	case I40E_ULTRA_LATENCY:
 	default:
 		if (bytes_per_int <= 20)
 			new_latency_range = I40E_LOW_LATENCY;
 		break;
 	}
 
-	/* this is to adjust RX more aggressively when streaming small
-	 * packets.  The value of 40000 was picked as it is just beyond
-	 * what the hardware can receive per second if in low latency
-	 * mode.
-	 */
-#define RX_ULTRA_PACKET_RATE 40000
-
-	if ((((rc->total_packets * 1000000) / usecs) > RX_ULTRA_PACKET_RATE) &&
-	    (&qv->rx == rc))
-		new_latency_range = I40E_ULTRA_LATENCY;
-
+reset_latency:
 	rc->latency_range = new_latency_range;
 
 	switch (new_latency_range) {
@@ -422,21 +420,18 @@ static bool i40e_set_new_dynamic_itr(struct i40e_ring_container *rc)
 	case I40E_BULK_LATENCY:
 		new_itr = I40E_ITR_18K;
 		break;
-	case I40E_ULTRA_LATENCY:
-		new_itr = I40E_ITR_8K;
-		break;
 	default:
 		break;
 	}
 
 	rc->total_bytes = 0;
 	rc->total_packets = 0;
+	rc->last_itr_update = jiffies;
 
 	if (new_itr != rc->itr) {
 		rc->itr = new_itr;
 		return true;
 	}
-
 	return false;
 }
 
@@ -1299,7 +1294,7 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
 	u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
 	bool failure = false;
 
-	while (likely(total_rx_packets < budget)) {
+	while (likely(total_rx_packets < (unsigned int)budget)) {
 		struct i40e_rx_buffer *rx_buffer;
 		union i40e_rx_desc *rx_desc;
 		unsigned int size;
@@ -1406,7 +1401,7 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
 	rx_ring->q_vector->rx.total_bytes += total_rx_bytes;
 
 	/* guarantee a trip back through this routine if there was a failure */
-	return failure ? budget : total_rx_packets;
+	return failure ? budget : (int)total_rx_packets;
 }
 
 static u32 i40e_buildreg_itr(const int type, const u16 itr)
@@ -1575,7 +1570,6 @@ int i40evf_napi_poll(struct napi_struct *napi, int budget)
 
 	/* If work not completed, return budget and polling will return */
 	if (!clean_complete) {
-		const cpumask_t *aff_mask = &q_vector->affinity_mask;
 		int cpu_id = smp_processor_id();
 
 		/* It is possible that the interrupt affinity has changed but,
@@ -1585,14 +1579,22 @@ int i40evf_napi_poll(struct napi_struct *napi, int budget)
 		 * continue to poll, otherwise we must stop polling so the
 		 * interrupt can move to the correct cpu.
 		 */
-		if (likely(cpumask_test_cpu(cpu_id, aff_mask))) {
+		if (!cpumask_test_cpu(cpu_id, &q_vector->affinity_mask)) {
+			/* Tell napi that we are done polling */
+			napi_complete_done(napi, work_done);
+
+			/* Force an interrupt */
+			i40evf_force_wb(vsi, q_vector);
+
+			/* Return budget-1 so that polling stops */
+			return budget - 1;
+		}
 tx_only:
-			if (arm_wb) {
-				q_vector->tx.ring[0].tx_stats.tx_force_wb++;
-				i40e_enable_wb_on_itr(vsi, q_vector);
-			}
-			return budget;
+		if (arm_wb) {
+			q_vector->tx.ring[0].tx_stats.tx_force_wb++;
+			i40e_enable_wb_on_itr(vsi, q_vector);
 		}
+		return budget;
 	}
 
 	if (vsi->back->flags & I40E_TXR_FLAGS_WB_ON_ITR)
@@ -1601,14 +1603,7 @@ tx_only:
 	/* Work is done so exit the polling mode and re-enable the interrupt */
 	napi_complete_done(napi, work_done);
 
-	/* If we're prematurely stopping polling to fix the interrupt
-	 * affinity we want to make sure polling starts back up so we
-	 * issue a call to i40evf_force_wb which triggers a SW interrupt.
-	 */
-	if (!clean_complete)
-		i40evf_force_wb(vsi, q_vector);
-	else
-		i40e_update_enable_itr(vsi, q_vector);
+	i40e_update_enable_itr(vsi, q_vector);
 
 	return min(work_done, budget - 1);
 }
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_txrx.h b/drivers/net/ethernet/intel/i40evf/i40e_txrx.h
index 901282c87cf6..0d9f98bc07bd 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_txrx.h
+++ b/drivers/net/ethernet/intel/i40evf/i40e_txrx.h
@@ -98,10 +98,6 @@ enum i40e_dyn_idx_t {
 	BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | \
 	BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP))
 
-#define i40e_pf_get_default_rss_hena(pf) \
-	(((pf)->flags & I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE) ? \
-	  I40E_DEFAULT_RSS_HENA_EXPANDED : I40E_DEFAULT_RSS_HENA)
-
 /* Supported Rx Buffer Sizes (a multiple of 128) */
 #define I40E_RXBUFFER_256   256
 #define I40E_RXBUFFER_1536  1536  /* 128B aligned standard Ethernet frame */
@@ -117,6 +113,7 @@ enum i40e_dyn_idx_t {
  * i.e. RXBUFFER_512 --> 1216 byte skb (size-2048 slab)
  */
 #define I40E_RX_HDR_SIZE I40E_RXBUFFER_256
+#define I40E_PACKET_HDR_PAD (ETH_HLEN + ETH_FCS_LEN + (VLAN_HLEN * 2))
 #define i40e_rx_desc i40e_32byte_rx_desc
 
 #define I40E_RX_DMA_ATTR \
@@ -428,7 +425,6 @@ enum i40e_latency_range {
 	I40E_LOWEST_LATENCY = 0,
 	I40E_LOW_LATENCY = 1,
 	I40E_BULK_LATENCY = 2,
-	I40E_ULTRA_LATENCY = 3,
 };
 
 struct i40e_ring_container {
@@ -436,6 +432,7 @@ struct i40e_ring_container {
 	struct i40e_ring *ring;
 	unsigned int total_bytes;	/* total bytes processed this int */
 	unsigned int total_packets;	/* total packets processed this int */
+	unsigned long last_itr_update;	/* jiffies of last ITR update */
 	u16 count;
 	enum i40e_latency_range latency_range;
 	u16 itr;
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_type.h b/drivers/net/ethernet/intel/i40evf/i40e_type.h
index bde7f24af1c6..2ea919d9cdcf 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_type.h
+++ b/drivers/net/ethernet/intel/i40evf/i40e_type.h
@@ -159,6 +159,7 @@ struct i40e_link_status {
 	enum i40e_aq_link_speed link_speed;
 	u8 link_info;
 	u8 an_info;
+	u8 req_fec_info;
 	u8 fec_info;
 	u8 ext_info;
 	u8 loopback;
diff --git a/drivers/net/ethernet/intel/i40evf/i40evf.h b/drivers/net/ethernet/intel/i40evf/i40evf.h
index 6cc92089fecb..82f69031e5cd 100644
--- a/drivers/net/ethernet/intel/i40evf/i40evf.h
+++ b/drivers/net/ethernet/intel/i40evf/i40evf.h
@@ -39,6 +39,18 @@
 #include <linux/tcp.h>
 #include <linux/sctp.h>
 #include <linux/ipv6.h>
+#include <linux/kernel.h>
+#include <linux/bitops.h>
+#include <linux/timer.h>
+#include <linux/workqueue.h>
+#include <linux/wait.h>
+#include <linux/delay.h>
+#include <linux/gfp.h>
+#include <linux/skbuff.h>
+#include <linux/dma-mapping.h>
+#include <linux/etherdevice.h>
+#include <linux/socket.h>
+#include <linux/jiffies.h>
 #include <net/ip6_checksum.h>
 #include <net/udp.h>
 
@@ -109,7 +121,7 @@ struct i40e_q_vector {
 #define ITR_COUNTDOWN_START 100
 	u8 itr_countdown;	/* when 0 or 1 update ITR */
 	int v_idx;	/* vector index in list */
-	char name[IFNAMSIZ + 9];
+	char name[IFNAMSIZ + 15];
 	bool arm_wb_state;
 	cpumask_t affinity_mask;
 	struct irq_affinity_notify affinity_notify;
@@ -183,6 +195,7 @@ struct i40evf_adapter {
 	struct work_struct adminq_task;
 	struct delayed_work client_task;
 	struct delayed_work init_task;
+	wait_queue_head_t down_waitqueue;
 	struct i40e_q_vector *q_vectors;
 	struct list_head vlan_filter_list;
 	char misc_vector_name[IFNAMSIZ + 9];
@@ -225,8 +238,6 @@ struct i40evf_adapter {
 /* duplicates for common code */
 #define I40E_FLAG_DCB_ENABLED			0
 #define I40E_FLAG_RX_CSUM_ENABLED		I40EVF_FLAG_RX_CSUM_ENABLED
-#define I40E_FLAG_WB_ON_ITR_CAPABLE		I40EVF_FLAG_WB_ON_ITR_CAPABLE
-#define I40E_FLAG_OUTER_UDP_CSUM_CAPABLE	I40EVF_FLAG_OUTER_UDP_CSUM_CAPABLE
 #define I40E_FLAG_LEGACY_RX			I40EVF_FLAG_LEGACY_RX
 	/* flags for admin queue service task */
 	u32 aq_required;
@@ -250,6 +261,8 @@ struct i40evf_adapter {
 #define I40EVF_FLAG_AQ_RELEASE_PROMISC		BIT(16)
 #define I40EVF_FLAG_AQ_REQUEST_ALLMULTI		BIT(17)
 #define I40EVF_FLAG_AQ_RELEASE_ALLMULTI		BIT(18)
+#define I40EVF_FLAG_AQ_ENABLE_VLAN_STRIPPING	BIT(19)
+#define I40EVF_FLAG_AQ_DISABLE_VLAN_STRIPPING	BIT(20)
 
 	/* OS defined structs */
 	struct net_device *netdev;
@@ -266,19 +279,19 @@ struct i40evf_adapter {
 	enum virtchnl_link_speed link_speed;
 	enum virtchnl_ops current_op;
 #define CLIENT_ALLOWED(_a) ((_a)->vf_res ? \
-			    (_a)->vf_res->vf_offload_flags & \
+			    (_a)->vf_res->vf_cap_flags & \
 				VIRTCHNL_VF_OFFLOAD_IWARP : \
 			    0)
 #define CLIENT_ENABLED(_a) ((_a)->cinst)
 /* RSS by the PF should be preferred over RSS via other methods. */
-#define RSS_PF(_a) ((_a)->vf_res->vf_offload_flags & \
+#define RSS_PF(_a) ((_a)->vf_res->vf_cap_flags & \
 		    VIRTCHNL_VF_OFFLOAD_RSS_PF)
-#define RSS_AQ(_a) ((_a)->vf_res->vf_offload_flags & \
+#define RSS_AQ(_a) ((_a)->vf_res->vf_cap_flags & \
 		    VIRTCHNL_VF_OFFLOAD_RSS_AQ)
-#define RSS_REG(_a) (!((_a)->vf_res->vf_offload_flags & \
+#define RSS_REG(_a) (!((_a)->vf_res->vf_cap_flags & \
 		       (VIRTCHNL_VF_OFFLOAD_RSS_AQ | \
 			VIRTCHNL_VF_OFFLOAD_RSS_PF)))
-#define VLAN_ALLOWED(_a) ((_a)->vf_res->vf_offload_flags & \
+#define VLAN_ALLOWED(_a) ((_a)->vf_res->vf_cap_flags & \
 			  VIRTCHNL_VF_OFFLOAD_VLAN)
 	struct virtchnl_vf_resource *vf_res; /* incl. all VSIs */
 	struct virtchnl_vsi_resource *vsi_res; /* our LAN VSI */
@@ -347,6 +360,8 @@ void i40evf_get_hena(struct i40evf_adapter *adapter);
 void i40evf_set_hena(struct i40evf_adapter *adapter);
 void i40evf_set_rss_key(struct i40evf_adapter *adapter);
 void i40evf_set_rss_lut(struct i40evf_adapter *adapter);
+void i40evf_enable_vlan_stripping(struct i40evf_adapter *adapter);
+void i40evf_disable_vlan_stripping(struct i40evf_adapter *adapter);
 void i40evf_virtchnl_completion(struct i40evf_adapter *adapter,
 				enum virtchnl_ops v_opcode,
 				i40e_status v_retval, u8 *msg, u16 msglen);
diff --git a/drivers/net/ethernet/intel/i40evf/i40evf_ethtool.c b/drivers/net/ethernet/intel/i40evf/i40evf_ethtool.c
index 9bb2cc7dd4e4..65874d6b3ab9 100644
--- a/drivers/net/ethernet/intel/i40evf/i40evf_ethtool.c
+++ b/drivers/net/ethernet/intel/i40evf/i40evf_ethtool.c
@@ -165,7 +165,7 @@ static void i40evf_get_ethtool_stats(struct net_device *netdev,
 				     struct ethtool_stats *stats, u64 *data)
 {
 	struct i40evf_adapter *adapter = netdev_priv(netdev);
-	int i, j;
+	unsigned int i, j;
 	char *p;
 
 	for (i = 0; i < I40EVF_GLOBAL_STATS_LEN; i++) {
@@ -197,7 +197,7 @@ static void i40evf_get_strings(struct net_device *netdev, u32 sset, u8 *data)
 	int i;
 
 	if (sset == ETH_SS_STATS) {
-		for (i = 0; i < I40EVF_GLOBAL_STATS_LEN; i++) {
+		for (i = 0; i < (int)I40EVF_GLOBAL_STATS_LEN; i++) {
 			memcpy(p, i40evf_gstrings_stats[i].stat_string,
 			       ETH_GSTRING_LEN);
 			p += ETH_GSTRING_LEN;
@@ -258,29 +258,50 @@ static u32 i40evf_get_priv_flags(struct net_device *netdev)
 static int i40evf_set_priv_flags(struct net_device *netdev, u32 flags)
 {
 	struct i40evf_adapter *adapter = netdev_priv(netdev);
-	u64 changed_flags;
+	u32 orig_flags, new_flags, changed_flags;
 	u32 i;
 
-	changed_flags = adapter->flags;
+	orig_flags = READ_ONCE(adapter->flags);
+	new_flags = orig_flags;
 
 	for (i = 0; i < I40EVF_PRIV_FLAGS_STR_LEN; i++) {
 		const struct i40evf_priv_flags *priv_flags;
 
 		priv_flags = &i40evf_gstrings_priv_flags[i];
 
-		if (priv_flags->read_only)
-			continue;
-
 		if (flags & BIT(i))
-			adapter->flags |= priv_flags->flag;
+			new_flags |= priv_flags->flag;
 		else
-			adapter->flags &= ~(priv_flags->flag);
+			new_flags &= ~(priv_flags->flag);
+
+		if (priv_flags->read_only &&
+		    ((orig_flags ^ new_flags) & ~BIT(i)))
+			return -EOPNOTSUPP;
+	}
+
+	/* Before we finalize any flag changes, any checks which we need to
+	 * perform to determine if the new flags will be supported should go
+	 * here...
+	 */
+
+	/* Compare and exchange the new flags into place. If we failed, that
+	 * is if cmpxchg returns anything but the old value, this means
+	 * something else must have modified the flags variable since we
+	 * copied it. We'll just punt with an error and log something in the
+	 * message buffer.
+	 */
+	if (cmpxchg(&adapter->flags, orig_flags, new_flags) != orig_flags) {
+		dev_warn(&adapter->pdev->dev,
+			 "Unable to update adapter->flags as it was modified by another thread...\n");
+		return -EAGAIN;
 	}
 
-	/* check for flags that changed */
-	changed_flags ^= adapter->flags;
+	changed_flags = orig_flags ^ new_flags;
 
-	/* Process any additional changes needed as a result of flag changes. */
+	/* Process any additional changes needed as a result of flag changes.
+	 * The changed_flags value reflects the list of bits that were changed
+	 * in the code above.
+	 */
 
 	/* issue a reset to force legacy-rx change to take effect */
 	if (changed_flags & I40EVF_FLAG_LEGACY_RX) {
diff --git a/drivers/net/ethernet/intel/i40evf/i40evf_main.c b/drivers/net/ethernet/intel/i40evf/i40evf_main.c
index 7c213a347909..1825d956bb00 100644
--- a/drivers/net/ethernet/intel/i40evf/i40evf_main.c
+++ b/drivers/net/ethernet/intel/i40evf/i40evf_main.c
@@ -520,7 +520,7 @@ static void i40evf_irq_affinity_notify(struct irq_affinity_notify *notify,
 	struct i40e_q_vector *q_vector =
 		container_of(notify, struct i40e_q_vector, affinity_notify);
 
-	q_vector->affinity_mask = *mask;
+	cpumask_copy(&q_vector->affinity_mask, mask);
 }
 
 /**
@@ -543,9 +543,9 @@ static void i40evf_irq_affinity_release(struct kref *ref) {}
 static int
 i40evf_request_traffic_irqs(struct i40evf_adapter *adapter, char *basename)
 {
-	int vector, err, q_vectors;
-	int rx_int_idx = 0, tx_int_idx = 0;
-	int irq_num;
+	unsigned int vector, q_vectors;
+	unsigned int rx_int_idx = 0, tx_int_idx = 0;
+	int irq_num, err;
 
 	i40evf_irq_disable(adapter);
 	/* Decrement for Other and TCP Timer vectors */
@@ -556,18 +556,15 @@ i40evf_request_traffic_irqs(struct i40evf_adapter *adapter, char *basename)
 		irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
 
 		if (q_vector->tx.ring && q_vector->rx.ring) {
-			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
-				 "i40evf-%s-%s-%d", basename,
-				 "TxRx", rx_int_idx++);
+			snprintf(q_vector->name, sizeof(q_vector->name),
+				 "i40evf-%s-TxRx-%d", basename, rx_int_idx++);
 			tx_int_idx++;
 		} else if (q_vector->rx.ring) {
-			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
-				 "i40evf-%s-%s-%d", basename,
-				 "rx", rx_int_idx++);
+			snprintf(q_vector->name, sizeof(q_vector->name),
+				 "i40evf-%s-rx-%d", basename, rx_int_idx++);
 		} else if (q_vector->tx.ring) {
-			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
-				 "i40evf-%s-%s-%d", basename,
-				 "tx", tx_int_idx++);
+			snprintf(q_vector->name, sizeof(q_vector->name),
+				 "i40evf-%s-tx-%d", basename, tx_int_idx++);
 		} else {
 			/* skip this unused q_vector */
 			continue;
@@ -587,8 +584,10 @@ i40evf_request_traffic_irqs(struct i40evf_adapter *adapter, char *basename)
 		q_vector->affinity_notify.release =
 						   i40evf_irq_affinity_release;
 		irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
-		/* assign the mask for this irq */
-		irq_set_affinity_hint(irq_num, &q_vector->affinity_mask);
+		/* get_cpu_mask returns a static constant mask with
+		 * a permanent lifetime so it's ok to use here.
+		 */
+		irq_set_affinity_hint(irq_num, get_cpu_mask(q_vector->v_idx));
 	}
 
 	return 0;
@@ -1143,6 +1142,7 @@ void i40evf_down(struct i40evf_adapter *adapter)
 	}
 
 	clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
+	mod_timer_pending(&adapter->watchdog_timer, jiffies + 1);
 }
 
 /**
@@ -1241,7 +1241,7 @@ static int i40evf_alloc_queues(struct i40evf_adapter *adapter)
 		tx_ring->dev = &adapter->pdev->dev;
 		tx_ring->count = adapter->tx_desc_count;
 		tx_ring->tx_itr_setting = (I40E_ITR_DYNAMIC | I40E_ITR_TX_DEF);
-		if (adapter->flags & I40E_FLAG_WB_ON_ITR_CAPABLE)
+		if (adapter->flags & I40EVF_FLAG_WB_ON_ITR_CAPABLE)
 			tx_ring->flags |= I40E_TXR_FLAGS_WB_ON_ITR;
 
 		rx_ring = &adapter->rx_rings[i];
@@ -1417,7 +1417,7 @@ static int i40evf_init_rss(struct i40evf_adapter *adapter)
 
 	if (!RSS_PF(adapter)) {
 		/* Enable PCTYPES for RSS, TCP/UDP with IPv4/IPv6 */
-		if (adapter->vf_res->vf_offload_flags &
+		if (adapter->vf_res->vf_cap_flags &
 		    VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2)
 			adapter->hena = I40E_DEFAULT_RSS_HENA_EXPANDED;
 		else
@@ -1458,6 +1458,7 @@ static int i40evf_alloc_q_vectors(struct i40evf_adapter *adapter)
 		q_vector->adapter = adapter;
 		q_vector->vsi = &adapter->vsi;
 		q_vector->v_idx = q_idx;
+		cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
 		netif_napi_add(adapter->netdev, &q_vector->napi,
 			       i40evf_napi_poll, NAPI_POLL_WEIGHT);
 	}
@@ -1678,6 +1679,16 @@ static void i40evf_watchdog_task(struct work_struct *work)
 		goto watchdog_done;
 	}
 
+	if (adapter->aq_required & I40EVF_FLAG_AQ_ENABLE_VLAN_STRIPPING) {
+		i40evf_enable_vlan_stripping(adapter);
+		goto watchdog_done;
+	}
+
+	if (adapter->aq_required & I40EVF_FLAG_AQ_DISABLE_VLAN_STRIPPING) {
+		i40evf_disable_vlan_stripping(adapter);
+		goto watchdog_done;
+	}
+
 	if (adapter->aq_required & I40EVF_FLAG_AQ_CONFIGURE_QUEUES) {
 		i40evf_configure_queues(adapter);
 		goto watchdog_done;
@@ -1794,6 +1805,7 @@ static void i40evf_disable_vf(struct i40evf_adapter *adapter)
 	clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
 	adapter->flags &= ~I40EVF_FLAG_RESET_PENDING;
 	adapter->state = __I40EVF_DOWN;
+	wake_up(&adapter->down_waitqueue);
 	dev_info(&adapter->pdev->dev, "Reset task did not complete, VF disabled\n");
 }
 
@@ -1877,7 +1889,7 @@ static void i40evf_reset_task(struct work_struct *work)
 	}
 
 continue_reset:
-	if (netif_running(adapter->netdev)) {
+	if (netif_running(netdev)) {
 		netif_carrier_off(netdev);
 		netif_tx_stop_all_queues(netdev);
 		adapter->link_up = false;
@@ -1939,12 +1951,13 @@ continue_reset:
 		i40evf_irq_enable(adapter, true);
 	} else {
 		adapter->state = __I40EVF_DOWN;
+		wake_up(&adapter->down_waitqueue);
 	}
 
 	return;
 reset_err:
 	dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n");
-	i40evf_close(adapter->netdev);
+	i40evf_close(netdev);
 }
 
 /**
@@ -1957,8 +1970,8 @@ static void i40evf_adminq_task(struct work_struct *work)
 		container_of(work, struct i40evf_adapter, adminq_task);
 	struct i40e_hw *hw = &adapter->hw;
 	struct i40e_arq_event_info event;
-	struct virtchnl_msg *v_msg;
-	i40e_status ret;
+	enum virtchnl_ops v_op;
+	i40e_status ret, v_ret;
 	u32 val, oldval;
 	u16 pending;
 
@@ -1970,15 +1983,15 @@ static void i40evf_adminq_task(struct work_struct *work)
 	if (!event.msg_buf)
 		goto out;
 
-	v_msg = (struct virtchnl_msg *)&event.desc;
 	do {
 		ret = i40evf_clean_arq_element(hw, &event, &pending);
-		if (ret || !v_msg->v_opcode)
+		v_op = (enum virtchnl_ops)le32_to_cpu(event.desc.cookie_high);
+		v_ret = (i40e_status)le32_to_cpu(event.desc.cookie_low);
+
+		if (ret || !v_op)
 			break; /* No event to process or error cleaning ARQ */
 
-		i40evf_virtchnl_completion(adapter, v_msg->v_opcode,
-					   (i40e_status)v_msg->v_retval,
-					   event.msg_buf,
+		i40evf_virtchnl_completion(adapter, v_op, v_ret, event.msg_buf,
 					   event.msg_len);
 		if (pending != 0)
 			memset(event.msg_buf, 0, I40EVF_MAX_AQ_BUF_SIZE);
@@ -2238,6 +2251,7 @@ err_setup_tx:
 static int i40evf_close(struct net_device *netdev)
 {
 	struct i40evf_adapter *adapter = netdev_priv(netdev);
+	int status;
 
 	if (adapter->state <= __I40EVF_DOWN_PENDING)
 		return 0;
@@ -2255,7 +2269,18 @@ static int i40evf_close(struct net_device *netdev)
 	 * still active and can DMA into memory. Resources are cleared in
 	 * i40evf_virtchnl_completion() after we get confirmation from the PF
 	 * driver that the rings have been stopped.
+	 *
+	 * Also, we wait for state to transition to __I40EVF_DOWN before
+	 * returning. State change occurs in i40evf_virtchnl_completion() after
+	 * VF resources are released (which occurs after PF driver processes and
+	 * responds to admin queue commands).
 	 */
+
+	status = wait_event_timeout(adapter->down_waitqueue,
+				    adapter->state == __I40EVF_DOWN,
+				    msecs_to_jiffies(200));
+	if (!status)
+		netdev_warn(netdev, "Device resources not yet released\n");
 	return 0;
 }
 
@@ -2282,6 +2307,28 @@ static int i40evf_change_mtu(struct net_device *netdev, int new_mtu)
 }
 
 /**
+ * i40e_set_features - set the netdev feature flags
+ * @netdev: ptr to the netdev being adjusted
+ * @features: the feature set that the stack is suggesting
+ * Note: expects to be called while under rtnl_lock()
+ **/
+static int i40evf_set_features(struct net_device *netdev,
+			       netdev_features_t features)
+{
+	struct i40evf_adapter *adapter = netdev_priv(netdev);
+
+	if (!VLAN_ALLOWED(adapter))
+		return -EINVAL;
+
+	if (features & NETIF_F_HW_VLAN_CTAG_RX)
+		adapter->aq_required |= I40EVF_FLAG_AQ_ENABLE_VLAN_STRIPPING;
+	else
+		adapter->aq_required |= I40EVF_FLAG_AQ_DISABLE_VLAN_STRIPPING;
+
+	return 0;
+}
+
+/**
  * i40evf_features_check - Validate encapsulated packet conforms to limits
  * @skb: skb buff
  * @netdev: This physical port's netdev
@@ -2356,7 +2403,7 @@ static netdev_features_t i40evf_fix_features(struct net_device *netdev,
 	struct i40evf_adapter *adapter = netdev_priv(netdev);
 
 	features &= ~I40EVF_VLAN_FEATURES;
-	if (adapter->vf_res->vf_offload_flags & VIRTCHNL_VF_OFFLOAD_VLAN)
+	if (adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)
 		features |= I40EVF_VLAN_FEATURES;
 	return features;
 }
@@ -2374,6 +2421,7 @@ static const struct net_device_ops i40evf_netdev_ops = {
 	.ndo_vlan_rx_kill_vid	= i40evf_vlan_rx_kill_vid,
 	.ndo_features_check	= i40evf_features_check,
 	.ndo_fix_features	= i40evf_fix_features,
+	.ndo_set_features	= i40evf_set_features,
 #ifdef CONFIG_NET_POLL_CONTROLLER
 	.ndo_poll_controller	= i40evf_netpoll,
 #endif
@@ -2443,7 +2491,7 @@ int i40evf_process_config(struct i40evf_adapter *adapter)
 	/* advertise to stack only if offloads for encapsulated packets is
 	 * supported
 	 */
-	if (vfres->vf_offload_flags & VIRTCHNL_VF_OFFLOAD_ENCAP) {
+	if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ENCAP) {
 		hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL	|
 				   NETIF_F_GSO_GRE		|
 				   NETIF_F_GSO_GRE_CSUM		|
@@ -2453,7 +2501,7 @@ int i40evf_process_config(struct i40evf_adapter *adapter)
 				   NETIF_F_GSO_PARTIAL		|
 				   0;
 
-		if (!(vfres->vf_offload_flags &
+		if (!(vfres->vf_cap_flags &
 		      VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM))
 			netdev->gso_partial_features |=
 				NETIF_F_GSO_UDP_TUNNEL_CSUM;
@@ -2481,7 +2529,7 @@ int i40evf_process_config(struct i40evf_adapter *adapter)
 	adapter->vsi.work_limit = I40E_DEFAULT_IRQ_WORK;
 	vsi->netdev = adapter->netdev;
 	vsi->qs_handle = adapter->vsi_res->qset_handle;
-	if (vfres->vf_offload_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
+	if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
 		adapter->rss_key_size = vfres->rss_key_size;
 		adapter->rss_lut_size = vfres->rss_lut_size;
 	} else {
@@ -2625,7 +2673,7 @@ static void i40evf_init_task(struct work_struct *work)
 
 	/* MTU range: 68 - 9710 */
 	netdev->min_mtu = ETH_MIN_MTU;
-	netdev->max_mtu = I40E_MAX_RXBUFFER - (ETH_HLEN + ETH_FCS_LEN);
+	netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD;
 
 	if (!is_valid_ether_addr(adapter->hw.mac.addr)) {
 		dev_info(&pdev->dev, "Invalid MAC address %pM, using random\n",
@@ -2649,7 +2697,7 @@ static void i40evf_init_task(struct work_struct *work)
 	if (err)
 		goto err_sw_init;
 	i40evf_map_rings_to_vectors(adapter);
-	if (adapter->vf_res->vf_offload_flags &
+	if (adapter->vf_res->vf_cap_flags &
 	    VIRTCHNL_VF_OFFLOAD_WB_ON_ITR)
 		adapter->flags |= I40EVF_FLAG_WB_ON_ITR_CAPABLE;
 
@@ -2683,6 +2731,7 @@ static void i40evf_init_task(struct work_struct *work)
 	adapter->state = __I40EVF_DOWN;
 	set_bit(__I40E_VSI_DOWN, adapter->vsi.state);
 	i40evf_misc_irq_enable(adapter);
+	wake_up(&adapter->down_waitqueue);
 
 	adapter->rss_key = kzalloc(adapter->rss_key_size, GFP_KERNEL);
 	adapter->rss_lut = kzalloc(adapter->rss_lut_size, GFP_KERNEL);
@@ -2844,6 +2893,9 @@ static int i40evf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	schedule_delayed_work(&adapter->init_task,
 			      msecs_to_jiffies(5 * (pdev->devfn & 0x07)));
 
+	/* Setup the wait queue for indicating transition to down status */
+	init_waitqueue_head(&adapter->down_waitqueue);
+
 	return 0;
 
 err_ioremap:
diff --git a/drivers/net/ethernet/intel/i40evf/i40evf_virtchnl.c b/drivers/net/ethernet/intel/i40evf/i40evf_virtchnl.c
index d2bb250a71af..85876f4fb1fb 100644
--- a/drivers/net/ethernet/intel/i40evf/i40evf_virtchnl.c
+++ b/drivers/net/ethernet/intel/i40evf/i40evf_virtchnl.c
@@ -821,6 +821,46 @@ void i40evf_set_rss_lut(struct i40evf_adapter *adapter)
 }
 
 /**
+ * i40evf_enable_vlan_stripping
+ * @adapter: adapter structure
+ *
+ * Request VLAN header stripping to be enabled
+ **/
+void i40evf_enable_vlan_stripping(struct i40evf_adapter *adapter)
+{
+	if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+		/* bail because we already have a command pending */
+		dev_err(&adapter->pdev->dev, "Cannot enable stripping, command %d pending\n",
+			adapter->current_op);
+		return;
+	}
+	adapter->current_op = VIRTCHNL_OP_ENABLE_VLAN_STRIPPING;
+	adapter->aq_required &= ~I40EVF_FLAG_AQ_ENABLE_VLAN_STRIPPING;
+	i40evf_send_pf_msg(adapter, VIRTCHNL_OP_ENABLE_VLAN_STRIPPING,
+			   NULL, 0);
+}
+
+/**
+ * i40evf_disable_vlan_stripping
+ * @adapter: adapter structure
+ *
+ * Request VLAN header stripping to be disabled
+ **/
+void i40evf_disable_vlan_stripping(struct i40evf_adapter *adapter)
+{
+	if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
+		/* bail because we already have a command pending */
+		dev_err(&adapter->pdev->dev, "Cannot disable stripping, command %d pending\n",
+			adapter->current_op);
+		return;
+	}
+	adapter->current_op = VIRTCHNL_OP_DISABLE_VLAN_STRIPPING;
+	adapter->aq_required &= ~I40EVF_FLAG_AQ_DISABLE_VLAN_STRIPPING;
+	i40evf_send_pf_msg(adapter, VIRTCHNL_OP_DISABLE_VLAN_STRIPPING,
+			   NULL, 0);
+}
+
+/**
  * i40evf_print_link_message - print link up or down
  * @adapter: adapter structure
  *
@@ -991,8 +1031,10 @@ void i40evf_virtchnl_completion(struct i40evf_adapter *adapter,
 	case VIRTCHNL_OP_DISABLE_QUEUES:
 		i40evf_free_all_tx_resources(adapter);
 		i40evf_free_all_rx_resources(adapter);
-		if (adapter->state == __I40EVF_DOWN_PENDING)
+		if (adapter->state == __I40EVF_DOWN_PENDING) {
 			adapter->state = __I40EVF_DOWN;
+			wake_up(&adapter->down_waitqueue);
+		}
 		break;
 	case VIRTCHNL_OP_VERSION:
 	case VIRTCHNL_OP_CONFIG_IRQ_MAP:
diff --git a/drivers/net/ethernet/intel/igb/e1000_82575.c b/drivers/net/ethernet/intel/igb/e1000_82575.c
index 4a50870e0fa7..c37cc8bccf47 100644
--- a/drivers/net/ethernet/intel/igb/e1000_82575.c
+++ b/drivers/net/ethernet/intel/igb/e1000_82575.c
@@ -340,6 +340,9 @@ static s32 igb_init_phy_params_82575(struct e1000_hw *hw)
 		phy->ops.set_d3_lplu_state = igb_set_d3_lplu_state_82580;
 		phy->ops.force_speed_duplex = igb_phy_force_speed_duplex_m88;
 		break;
+	case BCM54616_E_PHY_ID:
+		phy->type = e1000_phy_bcm54616;
+		break;
 	default:
 		ret_val = -E1000_ERR_PHY;
 		goto out;
@@ -1659,6 +1662,9 @@ static s32 igb_setup_copper_link_82575(struct e1000_hw *hw)
 	case e1000_phy_82580:
 		ret_val = igb_copper_link_setup_82580(hw);
 		break;
+	case e1000_phy_bcm54616:
+		ret_val = 0;
+		break;
 	default:
 		ret_val = -E1000_ERR_PHY;
 		break;
diff --git a/drivers/net/ethernet/intel/igb/e1000_defines.h b/drivers/net/ethernet/intel/igb/e1000_defines.h
index d8517779439b..1de82f247312 100644
--- a/drivers/net/ethernet/intel/igb/e1000_defines.h
+++ b/drivers/net/ethernet/intel/igb/e1000_defines.h
@@ -889,6 +889,7 @@
 #define I210_I_PHY_ID        0x01410C00
 #define M88E1543_E_PHY_ID    0x01410EA0
 #define M88E1512_E_PHY_ID    0x01410DD0
+#define BCM54616_E_PHY_ID    0x03625D10
 
 /* M88E1000 Specific Registers */
 #define M88E1000_PHY_SPEC_CTRL     0x10  /* PHY Specific Control Register */
diff --git a/drivers/net/ethernet/intel/igb/e1000_hw.h b/drivers/net/ethernet/intel/igb/e1000_hw.h
index 2fb2213cd562..6c9485ab4b57 100644
--- a/drivers/net/ethernet/intel/igb/e1000_hw.h
+++ b/drivers/net/ethernet/intel/igb/e1000_hw.h
@@ -128,6 +128,7 @@ enum e1000_phy_type {
 	e1000_phy_ife,
 	e1000_phy_82580,
 	e1000_phy_i210,
+	e1000_phy_bcm54616,
 };
 
 enum e1000_bus_type {
@@ -491,13 +492,16 @@ struct e1000_fc_info {
 
 struct e1000_mbx_operations {
 	s32 (*init_params)(struct e1000_hw *hw);
-	s32 (*read)(struct e1000_hw *, u32 *, u16,  u16);
-	s32 (*write)(struct e1000_hw *, u32 *, u16, u16);
-	s32 (*read_posted)(struct e1000_hw *, u32 *, u16,  u16);
-	s32 (*write_posted)(struct e1000_hw *, u32 *, u16, u16);
-	s32 (*check_for_msg)(struct e1000_hw *, u16);
-	s32 (*check_for_ack)(struct e1000_hw *, u16);
-	s32 (*check_for_rst)(struct e1000_hw *, u16);
+	s32 (*read)(struct e1000_hw *hw, u32 *msg, u16 size, u16 mbx_id,
+		    bool unlock);
+	s32 (*write)(struct e1000_hw *hw, u32 *msg, u16 size, u16 mbx_id);
+	s32 (*read_posted)(struct e1000_hw *hw, u32 *msg, u16 size, u16 mbx_id);
+	s32 (*write_posted)(struct e1000_hw *hw, u32 *msg, u16 size,
+			    u16 mbx_id);
+	s32 (*check_for_msg)(struct e1000_hw *hw, u16 mbx_id);
+	s32 (*check_for_ack)(struct e1000_hw *hw, u16 mbx_id);
+	s32 (*check_for_rst)(struct e1000_hw *hw, u16 mbx_id);
+	s32 (*unlock)(struct e1000_hw *hw, u16 mbx_id);
 };
 
 struct e1000_mbx_stats {
diff --git a/drivers/net/ethernet/intel/igb/e1000_mbx.c b/drivers/net/ethernet/intel/igb/e1000_mbx.c
index 00e263f0c030..bffd58f7b2a1 100644
--- a/drivers/net/ethernet/intel/igb/e1000_mbx.c
+++ b/drivers/net/ethernet/intel/igb/e1000_mbx.c
@@ -32,7 +32,8 @@
  *
  *  returns SUCCESS if it successfully read message from buffer
  **/
-s32 igb_read_mbx(struct e1000_hw *hw, u32 *msg, u16 size, u16 mbx_id)
+s32 igb_read_mbx(struct e1000_hw *hw, u32 *msg, u16 size, u16 mbx_id,
+		 bool unlock)
 {
 	struct e1000_mbx_info *mbx = &hw->mbx;
 	s32 ret_val = -E1000_ERR_MBX;
@@ -42,7 +43,7 @@ s32 igb_read_mbx(struct e1000_hw *hw, u32 *msg, u16 size, u16 mbx_id)
 		size = mbx->size;
 
 	if (mbx->ops.read)
-		ret_val = mbx->ops.read(hw, msg, size, mbx_id);
+		ret_val = mbx->ops.read(hw, msg, size, mbx_id, unlock);
 
 	return ret_val;
 }
@@ -125,6 +126,24 @@ s32 igb_check_for_rst(struct e1000_hw *hw, u16 mbx_id)
 }
 
 /**
+ *  igb_unlock_mbx - unlock the mailbox
+ *  @hw: pointer to the HW structure
+ *  @mbx_id: id of mailbox to check
+ *
+ *  returns SUCCESS if the mailbox was unlocked or else ERR_MBX
+ **/
+s32 igb_unlock_mbx(struct e1000_hw *hw, u16 mbx_id)
+{
+	struct e1000_mbx_info *mbx = &hw->mbx;
+	s32 ret_val = -E1000_ERR_MBX;
+
+	if (mbx->ops.unlock)
+		ret_val = mbx->ops.unlock(hw, mbx_id);
+
+	return ret_val;
+}
+
+/**
  *  igb_poll_for_msg - Wait for message notification
  *  @hw: pointer to the HW structure
  *  @mbx_id: id of mailbox to write
@@ -204,7 +223,7 @@ static s32 igb_read_posted_mbx(struct e1000_hw *hw, u32 *msg, u16 size,
 	ret_val = igb_poll_for_msg(hw, mbx_id);
 
 	if (!ret_val)
-		ret_val = mbx->ops.read(hw, msg, size, mbx_id);
+		ret_val = mbx->ops.read(hw, msg, size, mbx_id, true);
 out:
 	return ret_val;
 }
@@ -341,6 +360,26 @@ static s32 igb_obtain_mbx_lock_pf(struct e1000_hw *hw, u16 vf_number)
 }
 
 /**
+ *  igb_release_mbx_lock_pf - release mailbox lock
+ *  @hw: pointer to the HW structure
+ *  @vf_number: the VF index
+ *
+ *  return SUCCESS if we released the mailbox lock
+ **/
+static s32 igb_release_mbx_lock_pf(struct e1000_hw *hw, u16 vf_number)
+{
+	u32 p2v_mailbox;
+
+	/* drop PF lock of mailbox, if set */
+	p2v_mailbox = rd32(E1000_P2VMAILBOX(vf_number));
+	if (p2v_mailbox & E1000_P2VMAILBOX_PFU)
+		wr32(E1000_P2VMAILBOX(vf_number),
+		     p2v_mailbox & ~E1000_P2VMAILBOX_PFU);
+
+	return 0;
+}
+
+/**
  *  igb_write_mbx_pf - Places a message in the mailbox
  *  @hw: pointer to the HW structure
  *  @msg: The message buffer
@@ -385,13 +424,14 @@ out_no_write:
  *  @msg: The message buffer
  *  @size: Length of buffer
  *  @vf_number: the VF index
+ *  @unlock: unlock the mailbox when done?
  *
  *  This function copies a message from the mailbox buffer to the caller's
  *  memory buffer.  The presumption is that the caller knows that there was
  *  a message due to a VF request so no polling for message is needed.
  **/
 static s32 igb_read_mbx_pf(struct e1000_hw *hw, u32 *msg, u16 size,
-			   u16 vf_number)
+			   u16 vf_number, bool unlock)
 {
 	s32 ret_val;
 	u16 i;
@@ -405,8 +445,12 @@ static s32 igb_read_mbx_pf(struct e1000_hw *hw, u32 *msg, u16 size,
 	for (i = 0; i < size; i++)
 		msg[i] = array_rd32(E1000_VMBMEM(vf_number), i);
 
-	/* Acknowledge the message and release buffer */
-	wr32(E1000_P2VMAILBOX(vf_number), E1000_P2VMAILBOX_ACK);
+	/* Acknowledge the message and release mailbox lock (or not) */
+	if (unlock)
+		wr32(E1000_P2VMAILBOX(vf_number), E1000_P2VMAILBOX_ACK);
+	else
+		wr32(E1000_P2VMAILBOX(vf_number),
+		     E1000_P2VMAILBOX_ACK | E1000_P2VMAILBOX_PFU);
 
 	/* update stats */
 	hw->mbx.stats.msgs_rx++;
@@ -437,6 +481,7 @@ s32 igb_init_mbx_params_pf(struct e1000_hw *hw)
 	mbx->ops.check_for_msg = igb_check_for_msg_pf;
 	mbx->ops.check_for_ack = igb_check_for_ack_pf;
 	mbx->ops.check_for_rst = igb_check_for_rst_pf;
+	mbx->ops.unlock = igb_release_mbx_lock_pf;
 
 	mbx->stats.msgs_tx = 0;
 	mbx->stats.msgs_rx = 0;
diff --git a/drivers/net/ethernet/intel/igb/e1000_mbx.h b/drivers/net/ethernet/intel/igb/e1000_mbx.h
index 3e7fed73df15..a62b08e1572e 100644
--- a/drivers/net/ethernet/intel/igb/e1000_mbx.h
+++ b/drivers/net/ethernet/intel/igb/e1000_mbx.h
@@ -67,11 +67,13 @@
 
 #define E1000_PF_CONTROL_MSG	0x0100 /* PF control message */
 
-s32 igb_read_mbx(struct e1000_hw *, u32 *, u16, u16);
-s32 igb_write_mbx(struct e1000_hw *, u32 *, u16, u16);
-s32 igb_check_for_msg(struct e1000_hw *, u16);
-s32 igb_check_for_ack(struct e1000_hw *, u16);
-s32 igb_check_for_rst(struct e1000_hw *, u16);
-s32 igb_init_mbx_params_pf(struct e1000_hw *);
+s32 igb_read_mbx(struct e1000_hw *hw, u32 *msg, u16 size, u16 mbx_id,
+		 bool unlock);
+s32 igb_write_mbx(struct e1000_hw *hw, u32 *msg, u16 size, u16 mbx_id);
+s32 igb_check_for_msg(struct e1000_hw *hw, u16 mbx_id);
+s32 igb_check_for_ack(struct e1000_hw *hw, u16 mbx_id);
+s32 igb_check_for_rst(struct e1000_hw *hw, u16 mbx_id);
+s32 igb_unlock_mbx(struct e1000_hw *hw, u16 mbx_id);
+s32 igb_init_mbx_params_pf(struct e1000_hw *hw);
 
 #endif /* _E1000_MBX_H_ */
diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index ec62410b035a..fd4a46b03cc8 100644
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -1791,6 +1791,8 @@ void igb_down(struct igb_adapter *adapter)
 	wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
 	/* flush and sleep below */
 
+	igb_nfc_filter_exit(adapter);
+
 	netif_carrier_off(netdev);
 	netif_tx_stop_all_queues(netdev);
 
@@ -3317,8 +3319,6 @@ static int __igb_close(struct net_device *netdev, bool suspending)
 	igb_down(adapter);
 	igb_free_irq(adapter);
 
-	igb_nfc_filter_exit(adapter);
-
 	igb_free_all_tx_resources(adapter);
 	igb_free_all_rx_resources(adapter);
 
@@ -5380,7 +5380,8 @@ netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
 	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
 		struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
 
-		if (!test_and_set_bit_lock(__IGB_PTP_TX_IN_PROGRESS,
+		if (adapter->tstamp_config.tx_type & HWTSTAMP_TX_ON &&
+		    !test_and_set_bit_lock(__IGB_PTP_TX_IN_PROGRESS,
 					   &adapter->state)) {
 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
 			tx_flags |= IGB_TX_FLAGS_TSTAMP;
@@ -5745,8 +5746,6 @@ static void igb_tsync_interrupt(struct igb_adapter *adapter)
 		event.type = PTP_CLOCK_PPS;
 		if (adapter->ptp_caps.pps)
 			ptp_clock_event(adapter->ptp_clock, &event);
-		else
-			dev_err(&adapter->pdev->dev, "unexpected SYS WRAP");
 		ack |= TSINTR_SYS_WRAP;
 	}
 
@@ -6676,32 +6675,33 @@ static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
 	struct vf_data_storage *vf_data = &adapter->vf_data[vf];
 	s32 retval;
 
-	retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
+	retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf, false);
 
 	if (retval) {
 		/* if receive failed revoke VF CTS stats and restart init */
 		dev_err(&pdev->dev, "Error receiving message from VF\n");
 		vf_data->flags &= ~IGB_VF_FLAG_CTS;
 		if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
-			return;
+			goto unlock;
 		goto out;
 	}
 
 	/* this is a message we already processed, do nothing */
 	if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
-		return;
+		goto unlock;
 
 	/* until the vf completes a reset it should not be
 	 * allowed to start any configuration.
 	 */
 	if (msgbuf[0] == E1000_VF_RESET) {
+		/* unlocks mailbox */
 		igb_vf_reset_msg(adapter, vf);
 		return;
 	}
 
 	if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
 		if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
-			return;
+			goto unlock;
 		retval = -1;
 		goto out;
 	}
@@ -6742,7 +6742,12 @@ out:
 	else
 		msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
 
+	/* unlocks mailbox */
 	igb_write_mbx(hw, msgbuf, 1, vf);
+	return;
+
+unlock:
+	igb_unlock_mbx(hw, vf);
 }
 
 static void igb_msg_task(struct igb_adapter *adapter)
diff --git a/drivers/net/ethernet/intel/igbvf/ethtool.c b/drivers/net/ethernet/intel/igbvf/ethtool.c
index 34faa113a8a0..a127688e83e6 100644
--- a/drivers/net/ethernet/intel/igbvf/ethtool.c
+++ b/drivers/net/ethernet/intel/igbvf/ethtool.c
@@ -296,8 +296,12 @@ static int igbvf_link_test(struct igbvf_adapter *adapter, u64 *data)
 	struct e1000_hw *hw = &adapter->hw;
 	*data = 0;
 
+	spin_lock_bh(&hw->mbx_lock);
+
 	hw->mac.ops.check_for_link(hw);
 
+	spin_unlock_bh(&hw->mbx_lock);
+
 	if (!(er32(STATUS) & E1000_STATUS_LU))
 		*data = 1;
 
diff --git a/drivers/net/ethernet/intel/igbvf/mbx.c b/drivers/net/ethernet/intel/igbvf/mbx.c
index 01752f44ace2..c9a441632e9f 100644
--- a/drivers/net/ethernet/intel/igbvf/mbx.c
+++ b/drivers/net/ethernet/intel/igbvf/mbx.c
@@ -264,6 +264,8 @@ static s32 e1000_write_mbx_vf(struct e1000_hw *hw, u32 *msg, u16 size)
 	s32 err;
 	u16 i;
 
+	WARN_ON_ONCE(!spin_is_locked(&hw->mbx_lock));
+
 	/* lock the mailbox to prevent pf/vf race condition */
 	err = e1000_obtain_mbx_lock_vf(hw);
 	if (err)
@@ -300,6 +302,8 @@ static s32 e1000_read_mbx_vf(struct e1000_hw *hw, u32 *msg, u16 size)
 	s32 err;
 	u16 i;
 
+	WARN_ON_ONCE(!spin_is_locked(&hw->mbx_lock));
+
 	/* lock the mailbox to prevent pf/vf race condition */
 	err = e1000_obtain_mbx_lock_vf(hw);
 	if (err)
diff --git a/drivers/net/ethernet/intel/igbvf/netdev.c b/drivers/net/ethernet/intel/igbvf/netdev.c
index 1b9cbbe88f6f..1ed556911b14 100644
--- a/drivers/net/ethernet/intel/igbvf/netdev.c
+++ b/drivers/net/ethernet/intel/igbvf/netdev.c
@@ -1235,7 +1235,12 @@ static void igbvf_set_rlpml(struct igbvf_adapter *adapter)
 	struct e1000_hw *hw = &adapter->hw;
 
 	max_frame_size = adapter->max_frame_size + VLAN_TAG_SIZE;
+
+	spin_lock_bh(&hw->mbx_lock);
+
 	e1000_rlpml_set_vf(hw, max_frame_size);
+
+	spin_unlock_bh(&hw->mbx_lock);
 }
 
 static int igbvf_vlan_rx_add_vid(struct net_device *netdev,
@@ -1244,10 +1249,16 @@ static int igbvf_vlan_rx_add_vid(struct net_device *netdev,
 	struct igbvf_adapter *adapter = netdev_priv(netdev);
 	struct e1000_hw *hw = &adapter->hw;
 
+	spin_lock_bh(&hw->mbx_lock);
+
 	if (hw->mac.ops.set_vfta(hw, vid, true)) {
 		dev_err(&adapter->pdev->dev, "Failed to add vlan id %d\n", vid);
+		spin_unlock_bh(&hw->mbx_lock);
 		return -EINVAL;
 	}
+
+	spin_unlock_bh(&hw->mbx_lock);
+
 	set_bit(vid, adapter->active_vlans);
 	return 0;
 }
@@ -1258,11 +1269,17 @@ static int igbvf_vlan_rx_kill_vid(struct net_device *netdev,
 	struct igbvf_adapter *adapter = netdev_priv(netdev);
 	struct e1000_hw *hw = &adapter->hw;
 
+	spin_lock_bh(&hw->mbx_lock);
+
 	if (hw->mac.ops.set_vfta(hw, vid, false)) {
 		dev_err(&adapter->pdev->dev,
 			"Failed to remove vlan id %d\n", vid);
+		spin_unlock_bh(&hw->mbx_lock);
 		return -EINVAL;
 	}
+
+	spin_unlock_bh(&hw->mbx_lock);
+
 	clear_bit(vid, adapter->active_vlans);
 	return 0;
 }
@@ -1428,7 +1445,11 @@ static void igbvf_set_multi(struct net_device *netdev)
 	netdev_for_each_mc_addr(ha, netdev)
 		memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
 
+	spin_lock_bh(&hw->mbx_lock);
+
 	hw->mac.ops.update_mc_addr_list(hw, mta_list, i, 0, 0);
+
+	spin_unlock_bh(&hw->mbx_lock);
 	kfree(mta_list);
 }
 
@@ -1449,16 +1470,24 @@ static int igbvf_set_uni(struct net_device *netdev)
 		return -ENOSPC;
 	}
 
+	spin_lock_bh(&hw->mbx_lock);
+
 	/* Clear all unicast MAC filters */
 	hw->mac.ops.set_uc_addr(hw, E1000_VF_MAC_FILTER_CLR, NULL);
 
+	spin_unlock_bh(&hw->mbx_lock);
+
 	if (!netdev_uc_empty(netdev)) {
 		struct netdev_hw_addr *ha;
 
 		/* Add MAC filters one by one */
 		netdev_for_each_uc_addr(ha, netdev) {
+			spin_lock_bh(&hw->mbx_lock);
+
 			hw->mac.ops.set_uc_addr(hw, E1000_VF_MAC_FILTER_ADD,
 						ha->addr);
+
+			spin_unlock_bh(&hw->mbx_lock);
 			udelay(200);
 		}
 	}
@@ -1503,12 +1532,16 @@ static void igbvf_reset(struct igbvf_adapter *adapter)
 	struct net_device *netdev = adapter->netdev;
 	struct e1000_hw *hw = &adapter->hw;
 
+	spin_lock_bh(&hw->mbx_lock);
+
 	/* Allow time for pending master requests to run */
 	if (mac->ops.reset_hw(hw))
 		dev_err(&adapter->pdev->dev, "PF still resetting\n");
 
 	mac->ops.init_hw(hw);
 
+	spin_unlock_bh(&hw->mbx_lock);
+
 	if (is_valid_ether_addr(adapter->hw.mac.addr)) {
 		memcpy(netdev->dev_addr, adapter->hw.mac.addr,
 		       netdev->addr_len);
@@ -1643,6 +1676,7 @@ static int igbvf_sw_init(struct igbvf_adapter *adapter)
 	igbvf_irq_disable(adapter);
 
 	spin_lock_init(&adapter->stats_lock);
+	spin_lock_init(&adapter->hw.mbx_lock);
 
 	set_bit(__IGBVF_DOWN, &adapter->state);
 	return 0;
@@ -1786,8 +1820,12 @@ static int igbvf_set_mac(struct net_device *netdev, void *p)
 
 	memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
 
+	spin_lock_bh(&hw->mbx_lock);
+
 	hw->mac.ops.rar_set(hw, hw->mac.addr, 0);
 
+	spin_unlock_bh(&hw->mbx_lock);
+
 	if (!ether_addr_equal(addr->sa_data, hw->mac.addr))
 		return -EADDRNOTAVAIL;
 
@@ -1858,7 +1896,12 @@ static bool igbvf_has_link(struct igbvf_adapter *adapter)
 	if (test_bit(__IGBVF_DOWN, &adapter->state))
 		return false;
 
+	spin_lock_bh(&hw->mbx_lock);
+
 	ret_val = hw->mac.ops.check_for_link(hw);
+
+	spin_unlock_bh(&hw->mbx_lock);
+
 	link_active = !hw->mac.get_link_status;
 
 	/* if check for link returns error we will need to reset */
@@ -2808,6 +2851,8 @@ static int igbvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	netdev->min_mtu = ETH_MIN_MTU;
 	netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;
 
+	spin_lock_bh(&hw->mbx_lock);
+
 	/*reset the controller to put the device in a known good state */
 	err = hw->mac.ops.reset_hw(hw);
 	if (err) {
@@ -2824,6 +2869,8 @@ static int igbvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		       netdev->addr_len);
 	}
 
+	spin_unlock_bh(&hw->mbx_lock);
+
 	if (!is_valid_ether_addr(netdev->dev_addr)) {
 		dev_info(&pdev->dev, "Assigning random MAC address.\n");
 		eth_hw_addr_random(netdev);
diff --git a/drivers/net/ethernet/intel/igbvf/vf.c b/drivers/net/ethernet/intel/igbvf/vf.c
index 528be116184e..9577ccf4b26a 100644
--- a/drivers/net/ethernet/intel/igbvf/vf.c
+++ b/drivers/net/ethernet/intel/igbvf/vf.c
@@ -149,7 +149,7 @@ static s32 e1000_reset_hw_vf(struct e1000_hw *hw)
 		msgbuf[0] = E1000_VF_RESET;
 		mbx->ops.write_posted(hw, msgbuf, 1);
 
-		msleep(10);
+		mdelay(10);
 
 		/* set our "perm_addr" based on info provided by PF */
 		ret_val = mbx->ops.read_posted(hw, msgbuf, 3);
@@ -230,6 +230,7 @@ static void e1000_update_mc_addr_list_vf(struct e1000_hw *hw,
 	u16 *hash_list = (u16 *)&msgbuf[1];
 	u32 hash_value;
 	u32 cnt, i;
+	s32 ret_val;
 
 	/* Each entry in the list uses 1 16 bit word.  We have 30
 	 * 16 bit words available in our HW msg buffer (minus 1 for the
@@ -250,7 +251,9 @@ static void e1000_update_mc_addr_list_vf(struct e1000_hw *hw,
 		mc_addr_list += ETH_ALEN;
 	}
 
-	mbx->ops.write_posted(hw, msgbuf, E1000_VFMAILBOX_SIZE);
+	ret_val = mbx->ops.write_posted(hw, msgbuf, E1000_VFMAILBOX_SIZE);
+	if (!ret_val)
+		mbx->ops.read_posted(hw, msgbuf, 1);
 }
 
 /**
@@ -293,11 +296,14 @@ void e1000_rlpml_set_vf(struct e1000_hw *hw, u16 max_size)
 {
 	struct e1000_mbx_info *mbx = &hw->mbx;
 	u32 msgbuf[2];
+	s32 ret_val;
 
 	msgbuf[0] = E1000_VF_SET_LPE;
 	msgbuf[1] = max_size;
 
-	mbx->ops.write_posted(hw, msgbuf, 2);
+	ret_val = mbx->ops.write_posted(hw, msgbuf, 2);
+	if (!ret_val)
+		mbx->ops.read_posted(hw, msgbuf, 1);
 }
 
 /**
diff --git a/drivers/net/ethernet/intel/igbvf/vf.h b/drivers/net/ethernet/intel/igbvf/vf.h
index 4cf78b0dec50..d213eefb6169 100644
--- a/drivers/net/ethernet/intel/igbvf/vf.h
+++ b/drivers/net/ethernet/intel/igbvf/vf.h
@@ -245,6 +245,7 @@ struct e1000_hw {
 
 	struct e1000_mac_info  mac;
 	struct e1000_mbx_info mbx;
+	spinlock_t mbx_lock;		/* serializes mailbox ops */
 
 	union {
 		struct e1000_dev_spec_vf vf;
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c
index 4e35e7017f3d..2c19070d2a0b 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c
@@ -79,16 +79,28 @@ bool ixgbe_device_supports_autoneg_fc(struct ixgbe_hw *hw)
 
 	switch (hw->phy.media_type) {
 	case ixgbe_media_type_fiber:
-		hw->mac.ops.check_link(hw, &speed, &link_up, false);
-		/* if link is down, assume supported */
-		if (link_up)
-			supported = speed == IXGBE_LINK_SPEED_1GB_FULL ?
+		/* flow control autoneg black list */
+		switch (hw->device_id) {
+		case IXGBE_DEV_ID_X550EM_A_SFP:
+		case IXGBE_DEV_ID_X550EM_A_SFP_N:
+			supported = false;
+			break;
+		default:
+			hw->mac.ops.check_link(hw, &speed, &link_up, false);
+			/* if link is down, assume supported */
+			if (link_up)
+				supported = speed == IXGBE_LINK_SPEED_1GB_FULL ?
 				true : false;
-		else
-			supported = true;
+			else
+				supported = true;
+		}
+
 		break;
 	case ixgbe_media_type_backplane:
-		supported = true;
+		if (hw->device_id == IXGBE_DEV_ID_X550EM_X_XFI)
+			supported = false;
+		else
+			supported = true;
 		break;
 	case ixgbe_media_type_copper:
 		/* only some copper devices support flow control autoneg */
@@ -111,6 +123,10 @@ bool ixgbe_device_supports_autoneg_fc(struct ixgbe_hw *hw)
 		break;
 	}
 
+	if (!supported)
+		hw_dbg(hw, "Device %x does not support flow control autoneg\n",
+		       hw->device_id);
+
 	return supported;
 }
 
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c
index b45fdc98033d..f1bfae0c41d0 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c
@@ -1018,8 +1018,12 @@ static void ixgbe_free_q_vector(struct ixgbe_adapter *adapter, int v_idx)
 	struct ixgbe_q_vector *q_vector = adapter->q_vector[v_idx];
 	struct ixgbe_ring *ring;
 
-	ixgbe_for_each_ring(ring, q_vector->tx)
-		adapter->tx_ring[ring->queue_index] = NULL;
+	ixgbe_for_each_ring(ring, q_vector->tx) {
+		if (ring_is_xdp(ring))
+			adapter->xdp_ring[ring->queue_index] = NULL;
+		else
+			adapter->tx_ring[ring->queue_index] = NULL;
+	}
 
 	ixgbe_for_each_ring(ring, q_vector->rx)
 		adapter->rx_ring[ring->queue_index] = NULL;
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
index f1dbdf26d8e1..d962368d08d0 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
@@ -386,7 +386,7 @@ u32 ixgbe_read_reg(struct ixgbe_hw *hw, u32 reg)
 	if (ixgbe_removed(reg_addr))
 		return IXGBE_FAILED_READ_REG;
 	if (unlikely(hw->phy.nw_mng_if_sel &
-		     IXGBE_NW_MNG_IF_SEL_ENABLE_10_100M)) {
+		     IXGBE_NW_MNG_IF_SEL_SGMII_ENABLE)) {
 		struct ixgbe_adapter *adapter;
 		int i;
 
@@ -2214,7 +2214,7 @@ static struct sk_buff *ixgbe_run_xdp(struct ixgbe_adapter *adapter,
 				     struct ixgbe_ring *rx_ring,
 				     struct xdp_buff *xdp)
 {
-	int result = IXGBE_XDP_PASS;
+	int err, result = IXGBE_XDP_PASS;
 	struct bpf_prog *xdp_prog;
 	u32 act;
 
@@ -2231,6 +2231,13 @@ static struct sk_buff *ixgbe_run_xdp(struct ixgbe_adapter *adapter,
 	case XDP_TX:
 		result = ixgbe_xmit_xdp_ring(adapter, xdp);
 		break;
+	case XDP_REDIRECT:
+		err = xdp_do_redirect(adapter->netdev, xdp, xdp_prog);
+		if (!err)
+			result = IXGBE_XDP_TX;
+		else
+			result = IXGBE_XDP_CONSUMED;
+		break;
 	default:
 		bpf_warn_invalid_xdp_action(act);
 		/* fallthrough */
@@ -2408,6 +2415,8 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
 		 */
 		wmb();
 		writel(ring->next_to_use, ring->tail);
+
+		xdp_do_flush_map();
 	}
 
 	u64_stats_update_begin(&rx_ring->syncp);
@@ -5810,6 +5819,9 @@ void ixgbe_down(struct ixgbe_adapter *adapter)
 
 	usleep_range(10000, 20000);
 
+	/* synchronize_sched() needed for pending XDP buffers to drain */
+	if (adapter->xdp_ring[0])
+		synchronize_sched();
 	netif_tx_stop_all_queues(netdev);
 
 	/* call carrier off first to avoid false dev_watchdog timeouts */
@@ -8839,7 +8851,6 @@ static int ixgbe_delete_clsu32(struct ixgbe_adapter *adapter,
 }
 
 static int ixgbe_configure_clsu32_add_hnode(struct ixgbe_adapter *adapter,
-					    __be16 protocol,
 					    struct tc_cls_u32_offload *cls)
 {
 	u32 uhtid = TC_U32_USERHTID(cls->hnode.handle);
@@ -8941,7 +8952,7 @@ static int parse_tc_actions(struct ixgbe_adapter *adapter,
 	LIST_HEAD(actions);
 	int err;
 
-	if (tc_no_actions(exts))
+	if (!tcf_exts_has_actions(exts))
 		return -EINVAL;
 
 	tcf_exts_to_list(exts, &actions);
@@ -9025,9 +9036,9 @@ static int ixgbe_clsu32_build_input(struct ixgbe_fdir_filter *input,
 }
 
 static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter,
-				  __be16 protocol,
 				  struct tc_cls_u32_offload *cls)
 {
+	__be16 protocol = cls->common.protocol;
 	u32 loc = cls->knode.handle & 0xfffff;
 	struct ixgbe_hw *hw = &adapter->hw;
 	struct ixgbe_mat_field *field_ptr;
@@ -9214,41 +9225,49 @@ free_jump:
 	return err;
 }
 
-static int __ixgbe_setup_tc(struct net_device *dev, u32 handle, u32 chain_index,
-			    __be16 proto, struct tc_to_netdev *tc)
+static int ixgbe_setup_tc_cls_u32(struct net_device *dev,
+				  struct tc_cls_u32_offload *cls_u32)
 {
 	struct ixgbe_adapter *adapter = netdev_priv(dev);
 
-	if (chain_index)
+	if (!is_classid_clsact_ingress(cls_u32->common.classid) ||
+	    cls_u32->common.chain_index)
 		return -EOPNOTSUPP;
 
-	if (TC_H_MAJ(handle) == TC_H_MAJ(TC_H_INGRESS) &&
-	    tc->type == TC_SETUP_CLSU32) {
-		switch (tc->cls_u32->command) {
-		case TC_CLSU32_NEW_KNODE:
-		case TC_CLSU32_REPLACE_KNODE:
-			return ixgbe_configure_clsu32(adapter,
-						      proto, tc->cls_u32);
-		case TC_CLSU32_DELETE_KNODE:
-			return ixgbe_delete_clsu32(adapter, tc->cls_u32);
-		case TC_CLSU32_NEW_HNODE:
-		case TC_CLSU32_REPLACE_HNODE:
-			return ixgbe_configure_clsu32_add_hnode(adapter, proto,
-								tc->cls_u32);
-		case TC_CLSU32_DELETE_HNODE:
-			return ixgbe_configure_clsu32_del_hnode(adapter,
-								tc->cls_u32);
-		default:
-			return -EINVAL;
-		}
+	switch (cls_u32->command) {
+	case TC_CLSU32_NEW_KNODE:
+	case TC_CLSU32_REPLACE_KNODE:
+		return ixgbe_configure_clsu32(adapter, cls_u32);
+	case TC_CLSU32_DELETE_KNODE:
+		return ixgbe_delete_clsu32(adapter, cls_u32);
+	case TC_CLSU32_NEW_HNODE:
+	case TC_CLSU32_REPLACE_HNODE:
+		return ixgbe_configure_clsu32_add_hnode(adapter, cls_u32);
+	case TC_CLSU32_DELETE_HNODE:
+		return ixgbe_configure_clsu32_del_hnode(adapter, cls_u32);
+	default:
+		return -EOPNOTSUPP;
 	}
+}
 
-	if (tc->type != TC_SETUP_MQPRIO)
-		return -EINVAL;
-
-	tc->mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
+static int ixgbe_setup_tc_mqprio(struct net_device *dev,
+				 struct tc_mqprio_qopt *mqprio)
+{
+	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
+	return ixgbe_setup_tc(dev, mqprio->num_tc);
+}
 
-	return ixgbe_setup_tc(dev, tc->mqprio->num_tc);
+static int __ixgbe_setup_tc(struct net_device *dev, enum tc_setup_type type,
+			    void *type_data)
+{
+	switch (type) {
+	case TC_SETUP_CLSU32:
+		return ixgbe_setup_tc_cls_u32(dev, type_data);
+	case TC_SETUP_MQPRIO:
+		return ixgbe_setup_tc_mqprio(dev, type_data);
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
 #ifdef CONFIG_PCI_IOV
@@ -9823,6 +9842,53 @@ static int ixgbe_xdp(struct net_device *dev, struct netdev_xdp *xdp)
 	}
 }
 
+static int ixgbe_xdp_xmit(struct net_device *dev, struct xdp_buff *xdp)
+{
+	struct ixgbe_adapter *adapter = netdev_priv(dev);
+	struct ixgbe_ring *ring;
+	int err;
+
+	if (unlikely(test_bit(__IXGBE_DOWN, &adapter->state)))
+		return -ENETDOWN;
+
+	/* During program transitions its possible adapter->xdp_prog is assigned
+	 * but ring has not been configured yet. In this case simply abort xmit.
+	 */
+	ring = adapter->xdp_prog ? adapter->xdp_ring[smp_processor_id()] : NULL;
+	if (unlikely(!ring))
+		return -ENXIO;
+
+	err = ixgbe_xmit_xdp_ring(adapter, xdp);
+	if (err != IXGBE_XDP_TX)
+		return -ENOSPC;
+
+	return 0;
+}
+
+static void ixgbe_xdp_flush(struct net_device *dev)
+{
+	struct ixgbe_adapter *adapter = netdev_priv(dev);
+	struct ixgbe_ring *ring;
+
+	/* Its possible the device went down between xdp xmit and flush so
+	 * we need to ensure device is still up.
+	 */
+	if (unlikely(test_bit(__IXGBE_DOWN, &adapter->state)))
+		return;
+
+	ring = adapter->xdp_prog ? adapter->xdp_ring[smp_processor_id()] : NULL;
+	if (unlikely(!ring))
+		return;
+
+	/* Force memory writes to complete before letting h/w know there
+	 * are new descriptors to fetch.
+	 */
+	wmb();
+	writel(ring->next_to_use, ring->tail);
+
+	return;
+}
+
 static const struct net_device_ops ixgbe_netdev_ops = {
 	.ndo_open		= ixgbe_open,
 	.ndo_stop		= ixgbe_close,
@@ -9869,6 +9935,8 @@ static const struct net_device_ops ixgbe_netdev_ops = {
 	.ndo_udp_tunnel_del	= ixgbe_del_udp_tunnel_port,
 	.ndo_features_check	= ixgbe_features_check,
 	.ndo_xdp		= ixgbe_xdp,
+	.ndo_xdp_xmit		= ixgbe_xdp_xmit,
+	.ndo_xdp_flush		= ixgbe_xdp_flush,
 };
 
 /**
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c
index 0760bd7eeb01..112d24c6c9ce 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c
@@ -679,8 +679,9 @@ update_vlvfb:
 static int ixgbe_set_vf_macvlan(struct ixgbe_adapter *adapter,
 				int vf, int index, unsigned char *mac_addr)
 {
-	struct list_head *pos;
 	struct vf_macvlans *entry;
+	struct list_head *pos;
+	int retval = 0;
 
 	if (index <= 1) {
 		list_for_each(pos, &adapter->vf_mvs.l) {
@@ -721,13 +722,15 @@ static int ixgbe_set_vf_macvlan(struct ixgbe_adapter *adapter,
 	if (!entry || !entry->free)
 		return -ENOSPC;
 
+	retval = ixgbe_add_mac_filter(adapter, mac_addr, vf);
+	if (retval < 0)
+		return retval;
+
 	entry->free = false;
 	entry->is_macvlan = true;
 	entry->vf = vf;
 	memcpy(entry->vf_macvlan, mac_addr, ETH_ALEN);
 
-	ixgbe_add_mac_filter(adapter, mac_addr, vf);
-
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h
index 9c2460c5ef1b..ffa0ee5cd0f5 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h
@@ -3778,8 +3778,8 @@ struct ixgbe_info {
 #define IXGBE_NW_MNG_IF_SEL_PHY_SPEED_1G	BIT(19)
 #define IXGBE_NW_MNG_IF_SEL_PHY_SPEED_2_5G	BIT(20)
 #define IXGBE_NW_MNG_IF_SEL_PHY_SPEED_10G	BIT(21)
-#define IXGBE_NW_MNG_IF_SEL_ENABLE_10_100M	BIT(23)
-#define IXGBE_NW_MNG_IF_SEL_INT_PHY_MODE	BIT(24)
+#define IXGBE_NW_MNG_IF_SEL_SGMII_ENABLE	BIT(25)
+#define IXGBE_NW_MNG_IF_SEL_INT_PHY_MODE	BIT(24) /* X552 only */
 #define IXGBE_NW_MNG_IF_SEL_MDIO_PHY_ADD_SHIFT	3
 #define IXGBE_NW_MNG_IF_SEL_MDIO_PHY_ADD	\
 				(0x1F << IXGBE_NW_MNG_IF_SEL_MDIO_PHY_ADD_SHIFT)
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c
index 72d84a065e34..19fbb2f28ea4 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c
@@ -1555,9 +1555,14 @@ static s32 ixgbe_restart_an_internal_phy_x550em(struct ixgbe_hw *hw)
  **/
 static s32 ixgbe_setup_ixfi_x550em(struct ixgbe_hw *hw, ixgbe_link_speed *speed)
 {
+	struct ixgbe_mac_info *mac = &hw->mac;
 	s32 status;
 	u32 reg_val;
 
+	/* iXFI is only supported with X552 */
+	if (mac->type != ixgbe_mac_X550EM_x)
+		return IXGBE_ERR_LINK_SETUP;
+
 	/* Disable AN and force speed to 10G Serial. */
 	status = ixgbe_read_iosf_sb_reg_x550(hw,
 					IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id),
@@ -1874,8 +1879,10 @@ static s32 ixgbe_setup_mac_link_t_X550em(struct ixgbe_hw *hw,
 	else
 		force_speed = IXGBE_LINK_SPEED_1GB_FULL;
 
-	/* If internal link mode is XFI, then setup XFI internal link. */
-	if (!(hw->phy.nw_mng_if_sel & IXGBE_NW_MNG_IF_SEL_INT_PHY_MODE)) {
+	/* If X552 and internal link mode is XFI, then setup XFI internal link.
+	 */
+	if (hw->mac.type == ixgbe_mac_X550EM_x &&
+	    !(hw->phy.nw_mng_if_sel & IXGBE_NW_MNG_IF_SEL_INT_PHY_MODE)) {
 		status = ixgbe_setup_ixfi_x550em(hw, &force_speed);
 
 		if (status)
@@ -2404,17 +2411,30 @@ static s32 ixgbe_enable_lasi_ext_t_x550em(struct ixgbe_hw *hw)
 	status = ixgbe_get_lasi_ext_t_x550em(hw, &lsc);
 
 	/* Enable link status change alarm */
-	status = hw->phy.ops.read_reg(hw, IXGBE_MDIO_PMA_TX_VEN_LASI_INT_MASK,
-				      MDIO_MMD_AN, &reg);
-	if (status)
-		return status;
 
-	reg |= IXGBE_MDIO_PMA_TX_VEN_LASI_INT_EN;
+	/* Enable the LASI interrupts on X552 devices to receive notifications
+	 * of the link configurations of the external PHY and correspondingly
+	 * support the configuration of the internal iXFI link, since iXFI does
+	 * not support auto-negotiation. This is not required for X553 devices
+	 * having KR support, which performs auto-negotiations and which is used
+	 * as the internal link to the external PHY. Hence adding a check here
+	 * to avoid enabling LASI interrupts for X553 devices.
+	 */
+	if (hw->mac.type != ixgbe_mac_x550em_a) {
+		status = hw->phy.ops.read_reg(hw,
+					    IXGBE_MDIO_PMA_TX_VEN_LASI_INT_MASK,
+					    MDIO_MMD_AN, &reg);
+		if (status)
+			return status;
 
-	status = hw->phy.ops.write_reg(hw, IXGBE_MDIO_PMA_TX_VEN_LASI_INT_MASK,
-				       MDIO_MMD_AN, reg);
-	if (status)
-		return status;
+		reg |= IXGBE_MDIO_PMA_TX_VEN_LASI_INT_EN;
+
+		status = hw->phy.ops.write_reg(hw,
+					    IXGBE_MDIO_PMA_TX_VEN_LASI_INT_MASK,
+					    MDIO_MMD_AN, reg);
+		if (status)
+			return status;
+	}
 
 	/* Enable high temperature failure and global fault alarms */
 	status = hw->phy.ops.read_reg(hw, IXGBE_MDIO_GLOBAL_INT_MASK,
@@ -2615,7 +2635,8 @@ static s32 ixgbe_setup_internal_phy_t_x550em(struct ixgbe_hw *hw)
 	if (hw->mac.ops.get_media_type(hw) != ixgbe_media_type_copper)
 		return IXGBE_ERR_CONFIG;
 
-	if (hw->phy.nw_mng_if_sel & IXGBE_NW_MNG_IF_SEL_INT_PHY_MODE) {
+	if (!(hw->mac.type == ixgbe_mac_X550EM_x &&
+	      !(hw->phy.nw_mng_if_sel & IXGBE_NW_MNG_IF_SEL_INT_PHY_MODE))) {
 		speed = IXGBE_LINK_SPEED_10GB_FULL |
 			IXGBE_LINK_SPEED_1GB_FULL;
 		return ixgbe_setup_kr_speed_x550em(hw, speed);
@@ -2822,7 +2843,7 @@ static s32 ixgbe_setup_fc_x550em(struct ixgbe_hw *hw)
 {
 	bool pause, asm_dir;
 	u32 reg_val;
-	s32 rc;
+	s32 rc = 0;
 
 	/* Validate the requested mode */
 	if (hw->fc.strict_ieee && hw->fc.requested_mode == ixgbe_fc_rx_pause) {
@@ -2865,32 +2886,37 @@ static s32 ixgbe_setup_fc_x550em(struct ixgbe_hw *hw)
 		return IXGBE_ERR_CONFIG;
 	}
 
-	if (hw->device_id != IXGBE_DEV_ID_X550EM_X_KR &&
-	    hw->device_id != IXGBE_DEV_ID_X550EM_A_KR &&
-	    hw->device_id != IXGBE_DEV_ID_X550EM_A_KR_L)
-		return 0;
-
-	rc = hw->mac.ops.read_iosf_sb_reg(hw,
-					  IXGBE_KRM_AN_CNTL_1(hw->bus.lan_id),
-					  IXGBE_SB_IOSF_TARGET_KR_PHY,
-					  &reg_val);
-	if (rc)
-		return rc;
-
-	reg_val &= ~(IXGBE_KRM_AN_CNTL_1_SYM_PAUSE |
-		     IXGBE_KRM_AN_CNTL_1_ASM_PAUSE);
-	if (pause)
-		reg_val |= IXGBE_KRM_AN_CNTL_1_SYM_PAUSE;
-	if (asm_dir)
-		reg_val |= IXGBE_KRM_AN_CNTL_1_ASM_PAUSE;
-	rc = hw->mac.ops.write_iosf_sb_reg(hw,
-					   IXGBE_KRM_AN_CNTL_1(hw->bus.lan_id),
-					   IXGBE_SB_IOSF_TARGET_KR_PHY,
-					   reg_val);
-
-	/* This device does not fully support AN. */
-	hw->fc.disable_fc_autoneg = true;
+	switch (hw->device_id) {
+	case IXGBE_DEV_ID_X550EM_X_KR:
+	case IXGBE_DEV_ID_X550EM_A_KR:
+	case IXGBE_DEV_ID_X550EM_A_KR_L:
+		rc = hw->mac.ops.read_iosf_sb_reg(hw,
+					    IXGBE_KRM_AN_CNTL_1(hw->bus.lan_id),
+					    IXGBE_SB_IOSF_TARGET_KR_PHY,
+					    &reg_val);
+		if (rc)
+			return rc;
 
+		reg_val &= ~(IXGBE_KRM_AN_CNTL_1_SYM_PAUSE |
+			     IXGBE_KRM_AN_CNTL_1_ASM_PAUSE);
+		if (pause)
+			reg_val |= IXGBE_KRM_AN_CNTL_1_SYM_PAUSE;
+		if (asm_dir)
+			reg_val |= IXGBE_KRM_AN_CNTL_1_ASM_PAUSE;
+		rc = hw->mac.ops.write_iosf_sb_reg(hw,
+					    IXGBE_KRM_AN_CNTL_1(hw->bus.lan_id),
+					    IXGBE_SB_IOSF_TARGET_KR_PHY,
+					    reg_val);
+
+		/* This device does not fully support AN. */
+		hw->fc.disable_fc_autoneg = true;
+		break;
+	case IXGBE_DEV_ID_X550EM_X_XFI:
+		hw->fc.disable_fc_autoneg = true;
+		break;
+	default:
+		break;
+	}
 	return rc;
 }
 
diff --git a/drivers/net/ethernet/marvell/mv643xx_eth.c b/drivers/net/ethernet/marvell/mv643xx_eth.c
index 9c94ea9b2b80..81c1fac00d33 100644
--- a/drivers/net/ethernet/marvell/mv643xx_eth.c
+++ b/drivers/net/ethernet/marvell/mv643xx_eth.c
@@ -183,8 +183,6 @@ static char mv643xx_eth_driver_version[] = "1.4";
 #define DEFAULT_TX_QUEUE_SIZE	512
 #define SKB_DMA_REALIGN		((PAGE_SIZE - NET_SKB_PAD) % SMP_CACHE_BYTES)
 
-#define TSO_HEADER_SIZE		128
-
 /* Max number of allowed TCP segments for software TSO */
 #define MV643XX_MAX_TSO_SEGS 100
 #define MV643XX_MAX_SKB_DESCS (MV643XX_MAX_TSO_SEGS * 2 + MAX_SKB_FRAGS)
@@ -1123,7 +1121,7 @@ static int txq_reclaim(struct tx_queue *txq, int budget, int force)
 			struct sk_buff *skb = __skb_dequeue(&txq->tx_skb);
 
 			if (!WARN_ON(!skb))
-				dev_kfree_skb(skb);
+				dev_consume_skb_any(skb);
 		}
 
 		if (cmd_sts & ERROR_SUMMARY) {
@@ -2026,7 +2024,7 @@ static void rxq_deinit(struct rx_queue *rxq)
 
 	for (i = 0; i < rxq->rx_ring_size; i++) {
 		if (rxq->rx_skb[i]) {
-			dev_kfree_skb(rxq->rx_skb[i]);
+			dev_consume_skb_any(rxq->rx_skb[i]);
 			rxq->rx_desc_count--;
 		}
 	}
diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c
index 0aab74c2a209..64a04975bcf8 100644
--- a/drivers/net/ethernet/marvell/mvneta.c
+++ b/drivers/net/ethernet/marvell/mvneta.c
@@ -281,9 +281,6 @@
  */
 #define MVNETA_RSS_LU_TABLE_SIZE	1
 
-/* TSO header size */
-#define TSO_HEADER_SIZE 128
-
 /* Max number of Rx descriptors */
 #define MVNETA_MAX_RXD 128
 
@@ -4332,7 +4329,7 @@ static int mvneta_probe(struct platform_device *pdev)
 		}
 	}
 
-	dev->features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO;
+	dev->features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_TSO;
 	dev->hw_features |= dev->features;
 	dev->vlan_features |= dev->features;
 	dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
diff --git a/drivers/net/ethernet/marvell/mvpp2.c b/drivers/net/ethernet/marvell/mvpp2.c
index 48d21c1e09f2..dd0ee2691c86 100644
--- a/drivers/net/ethernet/marvell/mvpp2.c
+++ b/drivers/net/ethernet/marvell/mvpp2.c
@@ -18,6 +18,7 @@
 #include <linux/inetdevice.h>
 #include <linux/mbus.h>
 #include <linux/module.h>
+#include <linux/mfd/syscon.h>
 #include <linux/interrupt.h>
 #include <linux/cpumask.h>
 #include <linux/of.h>
@@ -27,12 +28,15 @@
 #include <linux/of_address.h>
 #include <linux/of_device.h>
 #include <linux/phy.h>
+#include <linux/phy/phy.h>
 #include <linux/clk.h>
 #include <linux/hrtimer.h>
 #include <linux/ktime.h>
+#include <linux/regmap.h>
 #include <uapi/linux/ppp_defs.h>
 #include <net/ip.h>
 #include <net/ipv6.h>
+#include <net/tso.h>
 
 /* RX Fifo Registers */
 #define MVPP2_RX_DATA_FIFO_SIZE_REG(port)	(0x00 + 4 * (port))
@@ -120,6 +124,9 @@
 #define MVPP2_TXQ_DESC_ADDR_REG			0x2084
 #define MVPP2_TXQ_DESC_SIZE_REG			0x2088
 #define     MVPP2_TXQ_DESC_SIZE_MASK		0x3ff0
+#define MVPP2_TXQ_THRESH_REG			0x2094
+#define	    MVPP2_TXQ_THRESH_OFFSET		16
+#define	    MVPP2_TXQ_THRESH_MASK		0x3fff
 #define MVPP2_AGGR_TXQ_UPDATE_REG		0x2090
 #define MVPP2_TXQ_INDEX_REG			0x2098
 #define MVPP2_TXQ_PREF_BUF_REG			0x209c
@@ -183,22 +190,25 @@
 #define MVPP22_AXI_CODE_DOMAIN_SYSTEM		3
 
 /* Interrupt Cause and Mask registers */
+#define MVPP2_ISR_TX_THRESHOLD_REG(port)	(0x5140 + 4 * (port))
+#define     MVPP2_MAX_ISR_TX_THRESHOLD		0xfffff0
+
 #define MVPP2_ISR_RX_THRESHOLD_REG(rxq)		(0x5200 + 4 * (rxq))
 #define     MVPP2_MAX_ISR_RX_THRESHOLD		0xfffff0
-#define MVPP21_ISR_RXQ_GROUP_REG(rxq)		(0x5400 + 4 * (rxq))
+#define MVPP21_ISR_RXQ_GROUP_REG(port)		(0x5400 + 4 * (port))
 
-#define MVPP22_ISR_RXQ_GROUP_INDEX_REG          0x5400
+#define MVPP22_ISR_RXQ_GROUP_INDEX_REG		0x5400
 #define MVPP22_ISR_RXQ_GROUP_INDEX_SUBGROUP_MASK 0xf
-#define MVPP22_ISR_RXQ_GROUP_INDEX_GROUP_MASK   0x380
-#define MVPP22_ISR_RXQ_GROUP_INDEX_GROUP_OFFSET 7
+#define MVPP22_ISR_RXQ_GROUP_INDEX_GROUP_MASK	0x380
+#define MVPP22_ISR_RXQ_GROUP_INDEX_GROUP_OFFSET	7
 
 #define MVPP22_ISR_RXQ_GROUP_INDEX_SUBGROUP_MASK 0xf
-#define MVPP22_ISR_RXQ_GROUP_INDEX_GROUP_MASK   0x380
+#define MVPP22_ISR_RXQ_GROUP_INDEX_GROUP_MASK	0x380
 
-#define MVPP22_ISR_RXQ_SUB_GROUP_CONFIG_REG     0x5404
-#define MVPP22_ISR_RXQ_SUB_GROUP_STARTQ_MASK    0x1f
-#define MVPP22_ISR_RXQ_SUB_GROUP_SIZE_MASK      0xf00
-#define MVPP22_ISR_RXQ_SUB_GROUP_SIZE_OFFSET    8
+#define MVPP22_ISR_RXQ_SUB_GROUP_CONFIG_REG	0x5404
+#define MVPP22_ISR_RXQ_SUB_GROUP_STARTQ_MASK	0x1f
+#define MVPP22_ISR_RXQ_SUB_GROUP_SIZE_MASK	0xf00
+#define MVPP22_ISR_RXQ_SUB_GROUP_SIZE_OFFSET	8
 
 #define MVPP2_ISR_ENABLE_REG(port)		(0x5420 + 4 * (port))
 #define     MVPP2_ISR_ENABLE_INTERRUPT(mask)	((mask) & 0xffff)
@@ -206,6 +216,7 @@
 #define MVPP2_ISR_RX_TX_CAUSE_REG(port)		(0x5480 + 4 * (port))
 #define     MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK	0xffff
 #define     MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_MASK	0xff0000
+#define     MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_OFFSET	16
 #define     MVPP2_CAUSE_RX_FIFO_OVERRUN_MASK	BIT(24)
 #define     MVPP2_CAUSE_FCS_ERR_MASK		BIT(25)
 #define     MVPP2_CAUSE_TX_FIFO_UNDERRUN_MASK	BIT(26)
@@ -265,7 +276,7 @@
 #define MVPP2_BM_VIRT_RLS_REG			0x64c0
 #define MVPP22_BM_ADDR_HIGH_RLS_REG		0x64c4
 #define     MVPP22_BM_ADDR_HIGH_PHYS_RLS_MASK	0xff
-#define	    MVPP22_BM_ADDR_HIGH_VIRT_RLS_MASK	0xff00
+#define     MVPP22_BM_ADDR_HIGH_VIRT_RLS_MASK	0xff00
 #define     MVPP22_BM_ADDR_HIGH_VIRT_RLS_SHIFT	8
 
 /* TX Scheduler registers */
@@ -307,57 +318,87 @@
 
 /* Per-port registers */
 #define MVPP2_GMAC_CTRL_0_REG			0x0
-#define      MVPP2_GMAC_PORT_EN_MASK		BIT(0)
-#define      MVPP2_GMAC_MAX_RX_SIZE_OFFS	2
-#define      MVPP2_GMAC_MAX_RX_SIZE_MASK	0x7ffc
-#define      MVPP2_GMAC_MIB_CNTR_EN_MASK	BIT(15)
+#define     MVPP2_GMAC_PORT_EN_MASK		BIT(0)
+#define     MVPP2_GMAC_PORT_TYPE_MASK		BIT(1)
+#define     MVPP2_GMAC_MAX_RX_SIZE_OFFS		2
+#define     MVPP2_GMAC_MAX_RX_SIZE_MASK		0x7ffc
+#define     MVPP2_GMAC_MIB_CNTR_EN_MASK		BIT(15)
 #define MVPP2_GMAC_CTRL_1_REG			0x4
-#define      MVPP2_GMAC_PERIODIC_XON_EN_MASK	BIT(1)
-#define      MVPP2_GMAC_GMII_LB_EN_MASK		BIT(5)
-#define      MVPP2_GMAC_PCS_LB_EN_BIT		6
-#define      MVPP2_GMAC_PCS_LB_EN_MASK		BIT(6)
-#define      MVPP2_GMAC_SA_LOW_OFFS		7
+#define     MVPP2_GMAC_PERIODIC_XON_EN_MASK	BIT(1)
+#define     MVPP2_GMAC_GMII_LB_EN_MASK		BIT(5)
+#define     MVPP2_GMAC_PCS_LB_EN_BIT		6
+#define     MVPP2_GMAC_PCS_LB_EN_MASK		BIT(6)
+#define     MVPP2_GMAC_SA_LOW_OFFS		7
 #define MVPP2_GMAC_CTRL_2_REG			0x8
-#define      MVPP2_GMAC_INBAND_AN_MASK		BIT(0)
-#define      MVPP2_GMAC_PCS_ENABLE_MASK		BIT(3)
-#define      MVPP2_GMAC_PORT_RGMII_MASK		BIT(4)
-#define      MVPP2_GMAC_PORT_RESET_MASK		BIT(6)
+#define     MVPP2_GMAC_INBAND_AN_MASK		BIT(0)
+#define     MVPP2_GMAC_FLOW_CTRL_MASK		GENMASK(2, 1)
+#define     MVPP2_GMAC_PCS_ENABLE_MASK		BIT(3)
+#define     MVPP2_GMAC_PORT_RGMII_MASK		BIT(4)
+#define     MVPP2_GMAC_DISABLE_PADDING		BIT(5)
+#define     MVPP2_GMAC_PORT_RESET_MASK		BIT(6)
 #define MVPP2_GMAC_AUTONEG_CONFIG		0xc
-#define      MVPP2_GMAC_FORCE_LINK_DOWN		BIT(0)
-#define      MVPP2_GMAC_FORCE_LINK_PASS		BIT(1)
-#define      MVPP2_GMAC_CONFIG_MII_SPEED	BIT(5)
-#define      MVPP2_GMAC_CONFIG_GMII_SPEED	BIT(6)
-#define      MVPP2_GMAC_AN_SPEED_EN		BIT(7)
-#define      MVPP2_GMAC_FC_ADV_EN		BIT(9)
-#define      MVPP2_GMAC_CONFIG_FULL_DUPLEX	BIT(12)
-#define      MVPP2_GMAC_AN_DUPLEX_EN		BIT(13)
+#define     MVPP2_GMAC_FORCE_LINK_DOWN		BIT(0)
+#define     MVPP2_GMAC_FORCE_LINK_PASS		BIT(1)
+#define     MVPP2_GMAC_IN_BAND_AUTONEG		BIT(2)
+#define     MVPP2_GMAC_IN_BAND_AUTONEG_BYPASS	BIT(3)
+#define     MVPP2_GMAC_CONFIG_MII_SPEED	BIT(5)
+#define     MVPP2_GMAC_CONFIG_GMII_SPEED	BIT(6)
+#define     MVPP2_GMAC_AN_SPEED_EN		BIT(7)
+#define     MVPP2_GMAC_FC_ADV_EN		BIT(9)
+#define     MVPP2_GMAC_FLOW_CTRL_AUTONEG	BIT(11)
+#define     MVPP2_GMAC_CONFIG_FULL_DUPLEX	BIT(12)
+#define     MVPP2_GMAC_AN_DUPLEX_EN		BIT(13)
+#define MVPP2_GMAC_STATUS0			0x10
+#define     MVPP2_GMAC_STATUS0_LINK_UP		BIT(0)
 #define MVPP2_GMAC_PORT_FIFO_CFG_1_REG		0x1c
-#define      MVPP2_GMAC_TX_FIFO_MIN_TH_OFFS	6
-#define      MVPP2_GMAC_TX_FIFO_MIN_TH_ALL_MASK	0x1fc0
-#define      MVPP2_GMAC_TX_FIFO_MIN_TH_MASK(v)	(((v) << 6) & \
+#define     MVPP2_GMAC_TX_FIFO_MIN_TH_OFFS	6
+#define     MVPP2_GMAC_TX_FIFO_MIN_TH_ALL_MASK	0x1fc0
+#define     MVPP2_GMAC_TX_FIFO_MIN_TH_MASK(v)	(((v) << 6) & \
 					MVPP2_GMAC_TX_FIFO_MIN_TH_ALL_MASK)
+#define MVPP22_GMAC_INT_STAT			0x20
+#define     MVPP22_GMAC_INT_STAT_LINK		BIT(1)
+#define MVPP22_GMAC_INT_MASK			0x24
+#define     MVPP22_GMAC_INT_MASK_LINK_STAT	BIT(1)
 #define MVPP22_GMAC_CTRL_4_REG			0x90
-#define      MVPP22_CTRL4_EXT_PIN_GMII_SEL	BIT(0)
-#define      MVPP22_CTRL4_DP_CLK_SEL		BIT(5)
-#define      MVPP22_CTRL4_SYNC_BYPASS		BIT(6)
-#define      MVPP22_CTRL4_QSGMII_BYPASS_ACTIVE	BIT(7)
+#define     MVPP22_CTRL4_EXT_PIN_GMII_SEL	BIT(0)
+#define     MVPP22_CTRL4_DP_CLK_SEL		BIT(5)
+#define     MVPP22_CTRL4_SYNC_BYPASS_DIS	BIT(6)
+#define     MVPP22_CTRL4_QSGMII_BYPASS_ACTIVE	BIT(7)
+#define MVPP22_GMAC_INT_SUM_MASK		0xa4
+#define     MVPP22_GMAC_INT_SUM_MASK_LINK_STAT	BIT(1)
 
 /* Per-port XGMAC registers. PPv2.2 only, only for GOP port 0,
  * relative to port->base.
  */
 #define MVPP22_XLG_CTRL0_REG			0x100
-#define      MVPP22_XLG_CTRL0_PORT_EN		BIT(0)
-#define      MVPP22_XLG_CTRL0_MAC_RESET_DIS	BIT(1)
-#define      MVPP22_XLG_CTRL0_MIB_CNT_DIS	BIT(14)
-
+#define     MVPP22_XLG_CTRL0_PORT_EN		BIT(0)
+#define     MVPP22_XLG_CTRL0_MAC_RESET_DIS	BIT(1)
+#define     MVPP22_XLG_CTRL0_RX_FLOW_CTRL_EN	BIT(7)
+#define     MVPP22_XLG_CTRL0_MIB_CNT_DIS	BIT(14)
+#define MVPP22_XLG_CTRL1_REG			0x104
+#define     MVPP22_XLG_CTRL1_FRAMESIZELIMIT_OFFS	0
+#define     MVPP22_XLG_CTRL1_FRAMESIZELIMIT_MASK	0x1fff
+#define MVPP22_XLG_STATUS			0x10c
+#define     MVPP22_XLG_STATUS_LINK_UP		BIT(0)
+#define MVPP22_XLG_INT_STAT			0x114
+#define     MVPP22_XLG_INT_STAT_LINK		BIT(1)
+#define MVPP22_XLG_INT_MASK			0x118
+#define     MVPP22_XLG_INT_MASK_LINK		BIT(1)
 #define MVPP22_XLG_CTRL3_REG			0x11c
-#define      MVPP22_XLG_CTRL3_MACMODESELECT_MASK	(7 << 13)
-#define      MVPP22_XLG_CTRL3_MACMODESELECT_GMAC	(0 << 13)
-#define      MVPP22_XLG_CTRL3_MACMODESELECT_10G		(1 << 13)
+#define     MVPP22_XLG_CTRL3_MACMODESELECT_MASK	(7 << 13)
+#define     MVPP22_XLG_CTRL3_MACMODESELECT_GMAC	(0 << 13)
+#define     MVPP22_XLG_CTRL3_MACMODESELECT_10G	(1 << 13)
+#define MVPP22_XLG_EXT_INT_MASK			0x15c
+#define     MVPP22_XLG_EXT_INT_MASK_XLG		BIT(1)
+#define     MVPP22_XLG_EXT_INT_MASK_GIG		BIT(2)
+#define MVPP22_XLG_CTRL4_REG			0x184
+#define     MVPP22_XLG_CTRL4_FWD_FC		BIT(5)
+#define     MVPP22_XLG_CTRL4_FWD_PFC		BIT(6)
+#define     MVPP22_XLG_CTRL4_MACMODSELECT_GMAC	BIT(12)
 
 /* SMI registers. PPv2.2 only, relative to priv->iface_base. */
 #define MVPP22_SMI_MISC_CFG_REG			0x1204
-#define      MVPP22_SMI_POLLING_EN		BIT(10)
+#define     MVPP22_SMI_POLLING_EN		BIT(10)
 
 #define MVPP22_GMAC_BASE(port)		(0x7000 + (port) * 0x1000 + 0xe00)
 
@@ -367,11 +408,44 @@
 #define MVPP2_QUEUE_NEXT_DESC(q, index) \
 	(((index) < (q)->last_desc) ? ((index) + 1) : 0)
 
+/* XPCS registers. PPv2.2 only */
+#define MVPP22_MPCS_BASE(port)			(0x7000 + (port) * 0x1000)
+#define MVPP22_MPCS_CTRL			0x14
+#define     MVPP22_MPCS_CTRL_FWD_ERR_CONN	BIT(10)
+#define MVPP22_MPCS_CLK_RESET			0x14c
+#define     MAC_CLK_RESET_SD_TX			BIT(0)
+#define     MAC_CLK_RESET_SD_RX			BIT(1)
+#define     MAC_CLK_RESET_MAC			BIT(2)
+#define     MVPP22_MPCS_CLK_RESET_DIV_RATIO(n)	((n) << 4)
+#define     MVPP22_MPCS_CLK_RESET_DIV_SET	BIT(11)
+
+/* XPCS registers. PPv2.2 only */
+#define MVPP22_XPCS_BASE(port)			(0x7400 + (port) * 0x1000)
+#define MVPP22_XPCS_CFG0			0x0
+#define     MVPP22_XPCS_CFG0_PCS_MODE(n)	((n) << 3)
+#define     MVPP22_XPCS_CFG0_ACTIVE_LANE(n)	((n) << 5)
+
+/* System controller registers. Accessed through a regmap. */
+#define GENCONF_SOFT_RESET1				0x1108
+#define     GENCONF_SOFT_RESET1_GOP			BIT(6)
+#define GENCONF_PORT_CTRL0				0x1110
+#define     GENCONF_PORT_CTRL0_BUS_WIDTH_SELECT		BIT(1)
+#define     GENCONF_PORT_CTRL0_RX_DATA_SAMPLE		BIT(29)
+#define     GENCONF_PORT_CTRL0_CLK_DIV_PHASE_CLR	BIT(31)
+#define GENCONF_PORT_CTRL1				0x1114
+#define     GENCONF_PORT_CTRL1_EN(p)			BIT(p)
+#define     GENCONF_PORT_CTRL1_RESET(p)			(BIT(p) << 28)
+#define GENCONF_CTRL0					0x1120
+#define     GENCONF_CTRL0_PORT0_RGMII			BIT(0)
+#define     GENCONF_CTRL0_PORT1_RGMII_MII		BIT(1)
+#define     GENCONF_CTRL0_PORT1_RGMII			BIT(2)
+
 /* Various constants */
 
 /* Coalescing */
 #define MVPP2_TXDONE_COAL_PKTS_THRESH	15
 #define MVPP2_TXDONE_HRTIMER_PERIOD_NS	1000000UL
+#define MVPP2_TXDONE_COAL_USEC		1000
 #define MVPP2_RX_COAL_PKTS		32
 #define MVPP2_RX_COAL_USEC		100
 
@@ -685,7 +759,8 @@ enum mvpp2_prs_l3_cast {
 #define MVPP21_ADDR_SPACE_SZ		0
 #define MVPP22_ADDR_SPACE_SZ		SZ_64K
 
-#define MVPP2_MAX_CPUS			4
+#define MVPP2_MAX_THREADS		8
+#define MVPP2_MAX_QVECS			MVPP2_MAX_THREADS
 
 enum mvpp2_bm_type {
 	MVPP2_BM_FREE,
@@ -701,11 +776,17 @@ struct mvpp2 {
 	void __iomem *lms_base;
 	void __iomem *iface_base;
 
-	/* On PPv2.2, each CPU can access the base register through a
-	 * separate address space, each 64 KB apart from each
-	 * other.
+	/* On PPv2.2, each "software thread" can access the base
+	 * register through a separate address space, each 64 KB apart
+	 * from each other. Typically, such address spaces will be
+	 * used per CPU.
+	 */
+	void __iomem *swth_base[MVPP2_MAX_THREADS];
+
+	/* On PPv2.2, some port control registers are located into the system
+	 * controller space. These registers are accessible through a regmap.
 	 */
-	void __iomem *cpu_base[MVPP2_MAX_CPUS];
+	struct regmap *sysctrl_base;
 
 	/* Common clocks */
 	struct clk *pp_clk;
@@ -752,6 +833,18 @@ struct mvpp2_port_pcpu {
 	struct tasklet_struct tx_done_tasklet;
 };
 
+struct mvpp2_queue_vector {
+	int irq;
+	struct napi_struct napi;
+	enum { MVPP2_QUEUE_VECTOR_SHARED, MVPP2_QUEUE_VECTOR_PRIVATE } type;
+	int sw_thread_id;
+	u16 sw_thread_mask;
+	int first_rxq;
+	int nrxqs;
+	u32 pending_cause_rx;
+	struct mvpp2_port *port;
+};
+
 struct mvpp2_port {
 	u8 id;
 
@@ -760,7 +853,7 @@ struct mvpp2_port {
 	 */
 	int gop_id;
 
-	int irq;
+	int link_irq;
 
 	struct mvpp2 *priv;
 
@@ -768,14 +861,13 @@ struct mvpp2_port {
 	void __iomem *base;
 
 	struct mvpp2_rx_queue **rxqs;
+	unsigned int nrxqs;
 	struct mvpp2_tx_queue **txqs;
+	unsigned int ntxqs;
 	struct net_device *dev;
 
 	int pkt_size;
 
-	u32 pending_cause_rx;
-	struct napi_struct napi;
-
 	/* Per-CPU port control */
 	struct mvpp2_port_pcpu __percpu *pcpu;
 
@@ -788,6 +880,7 @@ struct mvpp2_port {
 
 	phy_interface_t phy_interface;
 	struct device_node *phy_node;
+	struct phy *comphy;
 	unsigned int link;
 	unsigned int duplex;
 	unsigned int speed;
@@ -797,6 +890,12 @@ struct mvpp2_port {
 
 	/* Index of first port's physical RXQ */
 	u8 first_rxq;
+
+	struct mvpp2_queue_vector qvecs[MVPP2_MAX_QVECS];
+	unsigned int nqvecs;
+	bool has_tx_irqs;
+
+	u32 tx_time_coal;
 };
 
 /* The mvpp2_tx_desc and mvpp2_rx_desc structures describe the
@@ -932,6 +1031,10 @@ struct mvpp2_txq_pcpu {
 
 	/* Index of the TX DMA descriptor to be cleaned up */
 	int txq_get_index;
+
+	/* DMA buffer for TSO headers */
+	char *tso_headers;
+	dma_addr_t tso_headers_dma;
 };
 
 struct mvpp2_tx_queue {
@@ -1062,12 +1165,14 @@ struct mvpp2_bm_pool {
 	u32 port_map;
 };
 
-/* Static declaractions */
+/* Queue modes */
+#define MVPP2_QDIST_SINGLE_MODE	0
+#define MVPP2_QDIST_MULTI_MODE	1
 
-/* Number of RXQs used by single port */
-static int rxq_number = MVPP2_DEFAULT_RXQ;
-/* Number of TXQs used by single port */
-static int txq_number = MVPP2_MAX_TXQ;
+static int queue_mode = MVPP2_QDIST_SINGLE_MODE;
+
+module_param(queue_mode, int, 0444);
+MODULE_PARM_DESC(queue_mode, "Set queue_mode (single=0, multi=1)");
 
 #define MVPP2_DRIVER_NAME "mvpp2"
 #define MVPP2_DRIVER_VERSION "1.0"
@@ -1076,12 +1181,12 @@ static int txq_number = MVPP2_MAX_TXQ;
 
 static void mvpp2_write(struct mvpp2 *priv, u32 offset, u32 data)
 {
-	writel(data, priv->cpu_base[0] + offset);
+	writel(data, priv->swth_base[0] + offset);
 }
 
 static u32 mvpp2_read(struct mvpp2 *priv, u32 offset)
 {
-	return readl(priv->cpu_base[0] + offset);
+	return readl(priv->swth_base[0] + offset);
 }
 
 /* These accessors should be used to access:
@@ -1123,13 +1228,13 @@ static u32 mvpp2_read(struct mvpp2 *priv, u32 offset)
 static void mvpp2_percpu_write(struct mvpp2 *priv, int cpu,
 			       u32 offset, u32 data)
 {
-	writel(data, priv->cpu_base[cpu] + offset);
+	writel(data, priv->swth_base[cpu] + offset);
 }
 
 static u32 mvpp2_percpu_read(struct mvpp2 *priv, int cpu,
 			     u32 offset)
 {
-	return readl(priv->cpu_base[cpu] + offset);
+	return readl(priv->swth_base[cpu] + offset);
 }
 
 static dma_addr_t mvpp2_txdesc_dma_addr_get(struct mvpp2_port *port,
@@ -4070,7 +4175,7 @@ static int mvpp2_swf_bm_pool_init(struct mvpp2_port *port)
 
 		port->pool_long->port_map |= (1 << port->id);
 
-		for (rxq = 0; rxq < rxq_number; rxq++)
+		for (rxq = 0; rxq < port->nrxqs; rxq++)
 			mvpp2_rxq_long_pool_set(port, rxq, port->pool_long->id);
 	}
 
@@ -4084,7 +4189,7 @@ static int mvpp2_swf_bm_pool_init(struct mvpp2_port *port)
 
 		port->pool_short->port_map |= (1 << port->id);
 
-		for (rxq = 0; rxq < rxq_number; rxq++)
+		for (rxq = 0; rxq < port->nrxqs; rxq++)
 			mvpp2_rxq_short_pool_set(port, rxq,
 						 port->pool_short->id);
 	}
@@ -4125,22 +4230,40 @@ static int mvpp2_bm_update_mtu(struct net_device *dev, int mtu)
 
 static inline void mvpp2_interrupts_enable(struct mvpp2_port *port)
 {
-	int cpu, cpu_mask = 0;
+	int i, sw_thread_mask = 0;
+
+	for (i = 0; i < port->nqvecs; i++)
+		sw_thread_mask |= port->qvecs[i].sw_thread_mask;
 
-	for_each_present_cpu(cpu)
-		cpu_mask |= 1 << cpu;
 	mvpp2_write(port->priv, MVPP2_ISR_ENABLE_REG(port->id),
-		    MVPP2_ISR_ENABLE_INTERRUPT(cpu_mask));
+		    MVPP2_ISR_ENABLE_INTERRUPT(sw_thread_mask));
 }
 
 static inline void mvpp2_interrupts_disable(struct mvpp2_port *port)
 {
-	int cpu, cpu_mask = 0;
+	int i, sw_thread_mask = 0;
+
+	for (i = 0; i < port->nqvecs; i++)
+		sw_thread_mask |= port->qvecs[i].sw_thread_mask;
 
-	for_each_present_cpu(cpu)
-		cpu_mask |= 1 << cpu;
 	mvpp2_write(port->priv, MVPP2_ISR_ENABLE_REG(port->id),
-		    MVPP2_ISR_DISABLE_INTERRUPT(cpu_mask));
+		    MVPP2_ISR_DISABLE_INTERRUPT(sw_thread_mask));
+}
+
+static inline void mvpp2_qvec_interrupt_enable(struct mvpp2_queue_vector *qvec)
+{
+	struct mvpp2_port *port = qvec->port;
+
+	mvpp2_write(port->priv, MVPP2_ISR_ENABLE_REG(port->id),
+		    MVPP2_ISR_ENABLE_INTERRUPT(qvec->sw_thread_mask));
+}
+
+static inline void mvpp2_qvec_interrupt_disable(struct mvpp2_queue_vector *qvec)
+{
+	struct mvpp2_port *port = qvec->port;
+
+	mvpp2_write(port->priv, MVPP2_ISR_ENABLE_REG(port->id),
+		    MVPP2_ISR_DISABLE_INTERRUPT(qvec->sw_thread_mask));
 }
 
 /* Mask the current CPU's Rx/Tx interrupts
@@ -4162,15 +4285,346 @@ static void mvpp2_interrupts_mask(void *arg)
 static void mvpp2_interrupts_unmask(void *arg)
 {
 	struct mvpp2_port *port = arg;
+	u32 val;
+
+	val = MVPP2_CAUSE_MISC_SUM_MASK |
+		MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK;
+	if (port->has_tx_irqs)
+		val |= MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_MASK;
 
 	mvpp2_percpu_write(port->priv, smp_processor_id(),
-			   MVPP2_ISR_RX_TX_MASK_REG(port->id),
-			   (MVPP2_CAUSE_MISC_SUM_MASK |
-			    MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK));
+			   MVPP2_ISR_RX_TX_MASK_REG(port->id), val);
+}
+
+static void
+mvpp2_shared_interrupt_mask_unmask(struct mvpp2_port *port, bool mask)
+{
+	u32 val;
+	int i;
+
+	if (port->priv->hw_version != MVPP22)
+		return;
+
+	if (mask)
+		val = 0;
+	else
+		val = MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK;
+
+	for (i = 0; i < port->nqvecs; i++) {
+		struct mvpp2_queue_vector *v = port->qvecs + i;
+
+		if (v->type != MVPP2_QUEUE_VECTOR_SHARED)
+			continue;
+
+		mvpp2_percpu_write(port->priv, v->sw_thread_id,
+				   MVPP2_ISR_RX_TX_MASK_REG(port->id), val);
+	}
 }
 
 /* Port configuration routines */
 
+static void mvpp22_gop_init_rgmii(struct mvpp2_port *port)
+{
+	struct mvpp2 *priv = port->priv;
+	u32 val;
+
+	regmap_read(priv->sysctrl_base, GENCONF_PORT_CTRL0, &val);
+	val |= GENCONF_PORT_CTRL0_BUS_WIDTH_SELECT;
+	regmap_write(priv->sysctrl_base, GENCONF_PORT_CTRL0, val);
+
+	regmap_read(priv->sysctrl_base, GENCONF_CTRL0, &val);
+	if (port->gop_id == 2)
+		val |= GENCONF_CTRL0_PORT0_RGMII | GENCONF_CTRL0_PORT1_RGMII;
+	else if (port->gop_id == 3)
+		val |= GENCONF_CTRL0_PORT1_RGMII_MII;
+	regmap_write(priv->sysctrl_base, GENCONF_CTRL0, val);
+}
+
+static void mvpp22_gop_init_sgmii(struct mvpp2_port *port)
+{
+	struct mvpp2 *priv = port->priv;
+	u32 val;
+
+	regmap_read(priv->sysctrl_base, GENCONF_PORT_CTRL0, &val);
+	val |= GENCONF_PORT_CTRL0_BUS_WIDTH_SELECT |
+	       GENCONF_PORT_CTRL0_RX_DATA_SAMPLE;
+	regmap_write(priv->sysctrl_base, GENCONF_PORT_CTRL0, val);
+
+	if (port->gop_id > 1) {
+		regmap_read(priv->sysctrl_base, GENCONF_CTRL0, &val);
+		if (port->gop_id == 2)
+			val &= ~GENCONF_CTRL0_PORT0_RGMII;
+		else if (port->gop_id == 3)
+			val &= ~GENCONF_CTRL0_PORT1_RGMII_MII;
+		regmap_write(priv->sysctrl_base, GENCONF_CTRL0, val);
+	}
+}
+
+static void mvpp22_gop_init_10gkr(struct mvpp2_port *port)
+{
+	struct mvpp2 *priv = port->priv;
+	void __iomem *mpcs = priv->iface_base + MVPP22_MPCS_BASE(port->gop_id);
+	void __iomem *xpcs = priv->iface_base + MVPP22_XPCS_BASE(port->gop_id);
+	u32 val;
+
+	/* XPCS */
+	val = readl(xpcs + MVPP22_XPCS_CFG0);
+	val &= ~(MVPP22_XPCS_CFG0_PCS_MODE(0x3) |
+		 MVPP22_XPCS_CFG0_ACTIVE_LANE(0x3));
+	val |= MVPP22_XPCS_CFG0_ACTIVE_LANE(2);
+	writel(val, xpcs + MVPP22_XPCS_CFG0);
+
+	/* MPCS */
+	val = readl(mpcs + MVPP22_MPCS_CTRL);
+	val &= ~MVPP22_MPCS_CTRL_FWD_ERR_CONN;
+	writel(val, mpcs + MVPP22_MPCS_CTRL);
+
+	val = readl(mpcs + MVPP22_MPCS_CLK_RESET);
+	val &= ~(MVPP22_MPCS_CLK_RESET_DIV_RATIO(0x7) | MAC_CLK_RESET_MAC |
+		 MAC_CLK_RESET_SD_RX | MAC_CLK_RESET_SD_TX);
+	val |= MVPP22_MPCS_CLK_RESET_DIV_RATIO(1);
+	writel(val, mpcs + MVPP22_MPCS_CLK_RESET);
+
+	val &= ~MVPP22_MPCS_CLK_RESET_DIV_SET;
+	val |= MAC_CLK_RESET_MAC | MAC_CLK_RESET_SD_RX | MAC_CLK_RESET_SD_TX;
+	writel(val, mpcs + MVPP22_MPCS_CLK_RESET);
+}
+
+static int mvpp22_gop_init(struct mvpp2_port *port)
+{
+	struct mvpp2 *priv = port->priv;
+	u32 val;
+
+	if (!priv->sysctrl_base)
+		return 0;
+
+	switch (port->phy_interface) {
+	case PHY_INTERFACE_MODE_RGMII:
+	case PHY_INTERFACE_MODE_RGMII_ID:
+	case PHY_INTERFACE_MODE_RGMII_RXID:
+	case PHY_INTERFACE_MODE_RGMII_TXID:
+		if (port->gop_id == 0)
+			goto invalid_conf;
+		mvpp22_gop_init_rgmii(port);
+		break;
+	case PHY_INTERFACE_MODE_SGMII:
+		mvpp22_gop_init_sgmii(port);
+		break;
+	case PHY_INTERFACE_MODE_10GKR:
+		if (port->gop_id != 0)
+			goto invalid_conf;
+		mvpp22_gop_init_10gkr(port);
+		break;
+	default:
+		goto unsupported_conf;
+	}
+
+	regmap_read(priv->sysctrl_base, GENCONF_PORT_CTRL1, &val);
+	val |= GENCONF_PORT_CTRL1_RESET(port->gop_id) |
+	       GENCONF_PORT_CTRL1_EN(port->gop_id);
+	regmap_write(priv->sysctrl_base, GENCONF_PORT_CTRL1, val);
+
+	regmap_read(priv->sysctrl_base, GENCONF_PORT_CTRL0, &val);
+	val |= GENCONF_PORT_CTRL0_CLK_DIV_PHASE_CLR;
+	regmap_write(priv->sysctrl_base, GENCONF_PORT_CTRL0, val);
+
+	regmap_read(priv->sysctrl_base, GENCONF_SOFT_RESET1, &val);
+	val |= GENCONF_SOFT_RESET1_GOP;
+	regmap_write(priv->sysctrl_base, GENCONF_SOFT_RESET1, val);
+
+unsupported_conf:
+	return 0;
+
+invalid_conf:
+	netdev_err(port->dev, "Invalid port configuration\n");
+	return -EINVAL;
+}
+
+static void mvpp22_gop_unmask_irq(struct mvpp2_port *port)
+{
+	u32 val;
+
+	if (phy_interface_mode_is_rgmii(port->phy_interface) ||
+	    port->phy_interface == PHY_INTERFACE_MODE_SGMII) {
+		/* Enable the GMAC link status irq for this port */
+		val = readl(port->base + MVPP22_GMAC_INT_SUM_MASK);
+		val |= MVPP22_GMAC_INT_SUM_MASK_LINK_STAT;
+		writel(val, port->base + MVPP22_GMAC_INT_SUM_MASK);
+	}
+
+	if (port->gop_id == 0) {
+		/* Enable the XLG/GIG irqs for this port */
+		val = readl(port->base + MVPP22_XLG_EXT_INT_MASK);
+		if (port->phy_interface == PHY_INTERFACE_MODE_10GKR)
+			val |= MVPP22_XLG_EXT_INT_MASK_XLG;
+		else
+			val |= MVPP22_XLG_EXT_INT_MASK_GIG;
+		writel(val, port->base + MVPP22_XLG_EXT_INT_MASK);
+	}
+}
+
+static void mvpp22_gop_mask_irq(struct mvpp2_port *port)
+{
+	u32 val;
+
+	if (port->gop_id == 0) {
+		val = readl(port->base + MVPP22_XLG_EXT_INT_MASK);
+		val &= ~(MVPP22_XLG_EXT_INT_MASK_XLG |
+		         MVPP22_XLG_EXT_INT_MASK_GIG);
+		writel(val, port->base + MVPP22_XLG_EXT_INT_MASK);
+	}
+
+	if (phy_interface_mode_is_rgmii(port->phy_interface) ||
+	    port->phy_interface == PHY_INTERFACE_MODE_SGMII) {
+		val = readl(port->base + MVPP22_GMAC_INT_SUM_MASK);
+		val &= ~MVPP22_GMAC_INT_SUM_MASK_LINK_STAT;
+		writel(val, port->base + MVPP22_GMAC_INT_SUM_MASK);
+	}
+}
+
+static void mvpp22_gop_setup_irq(struct mvpp2_port *port)
+{
+	u32 val;
+
+	if (phy_interface_mode_is_rgmii(port->phy_interface) ||
+	    port->phy_interface == PHY_INTERFACE_MODE_SGMII) {
+		val = readl(port->base + MVPP22_GMAC_INT_MASK);
+		val |= MVPP22_GMAC_INT_MASK_LINK_STAT;
+		writel(val, port->base + MVPP22_GMAC_INT_MASK);
+	}
+
+	if (port->gop_id == 0) {
+		val = readl(port->base + MVPP22_XLG_INT_MASK);
+		val |= MVPP22_XLG_INT_MASK_LINK;
+		writel(val, port->base + MVPP22_XLG_INT_MASK);
+	}
+
+	mvpp22_gop_unmask_irq(port);
+}
+
+static int mvpp22_comphy_init(struct mvpp2_port *port)
+{
+	enum phy_mode mode;
+	int ret;
+
+	if (!port->comphy)
+		return 0;
+
+	switch (port->phy_interface) {
+	case PHY_INTERFACE_MODE_SGMII:
+		mode = PHY_MODE_SGMII;
+		break;
+	case PHY_INTERFACE_MODE_10GKR:
+		mode = PHY_MODE_10GKR;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	ret = phy_set_mode(port->comphy, mode);
+	if (ret)
+		return ret;
+
+	return phy_power_on(port->comphy);
+}
+
+static void mvpp2_port_mii_gmac_configure_mode(struct mvpp2_port *port)
+{
+	u32 val;
+
+	if (port->phy_interface == PHY_INTERFACE_MODE_SGMII) {
+		val = readl(port->base + MVPP22_GMAC_CTRL_4_REG);
+		val |= MVPP22_CTRL4_SYNC_BYPASS_DIS | MVPP22_CTRL4_DP_CLK_SEL |
+		       MVPP22_CTRL4_QSGMII_BYPASS_ACTIVE;
+		val &= ~MVPP22_CTRL4_EXT_PIN_GMII_SEL;
+		writel(val, port->base + MVPP22_GMAC_CTRL_4_REG);
+
+		val = readl(port->base + MVPP2_GMAC_CTRL_2_REG);
+		val |= MVPP2_GMAC_DISABLE_PADDING;
+		val &= ~MVPP2_GMAC_FLOW_CTRL_MASK;
+		writel(val, port->base + MVPP2_GMAC_CTRL_2_REG);
+	} else if (phy_interface_mode_is_rgmii(port->phy_interface)) {
+		val = readl(port->base + MVPP22_GMAC_CTRL_4_REG);
+		val |= MVPP22_CTRL4_EXT_PIN_GMII_SEL |
+		       MVPP22_CTRL4_SYNC_BYPASS_DIS |
+		       MVPP22_CTRL4_QSGMII_BYPASS_ACTIVE;
+		val &= ~MVPP22_CTRL4_DP_CLK_SEL;
+		writel(val, port->base + MVPP22_GMAC_CTRL_4_REG);
+
+		val = readl(port->base + MVPP2_GMAC_CTRL_2_REG);
+		val &= ~MVPP2_GMAC_DISABLE_PADDING;
+		writel(val, port->base + MVPP2_GMAC_CTRL_2_REG);
+	}
+
+	/* The port is connected to a copper PHY */
+	val = readl(port->base + MVPP2_GMAC_CTRL_0_REG);
+	val &= ~MVPP2_GMAC_PORT_TYPE_MASK;
+	writel(val, port->base + MVPP2_GMAC_CTRL_0_REG);
+
+	val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+	val |= MVPP2_GMAC_IN_BAND_AUTONEG_BYPASS |
+	       MVPP2_GMAC_AN_SPEED_EN | MVPP2_GMAC_FLOW_CTRL_AUTONEG |
+	       MVPP2_GMAC_AN_DUPLEX_EN;
+	if (port->phy_interface == PHY_INTERFACE_MODE_SGMII)
+		val |= MVPP2_GMAC_IN_BAND_AUTONEG;
+	writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+}
+
+static void mvpp2_port_mii_gmac_configure(struct mvpp2_port *port)
+{
+	u32 val;
+
+	/* Force link down */
+	val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+	val &= ~MVPP2_GMAC_FORCE_LINK_PASS;
+	val |= MVPP2_GMAC_FORCE_LINK_DOWN;
+	writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+
+	/* Set the GMAC in a reset state */
+	val = readl(port->base + MVPP2_GMAC_CTRL_2_REG);
+	val |= MVPP2_GMAC_PORT_RESET_MASK;
+	writel(val, port->base + MVPP2_GMAC_CTRL_2_REG);
+
+	/* Configure the PCS and in-band AN */
+	val = readl(port->base + MVPP2_GMAC_CTRL_2_REG);
+	if (port->phy_interface == PHY_INTERFACE_MODE_SGMII) {
+	        val |= MVPP2_GMAC_INBAND_AN_MASK | MVPP2_GMAC_PCS_ENABLE_MASK;
+	} else if (phy_interface_mode_is_rgmii(port->phy_interface)) {
+		val &= ~MVPP2_GMAC_PCS_ENABLE_MASK;
+		val |= MVPP2_GMAC_PORT_RGMII_MASK;
+	}
+	writel(val, port->base + MVPP2_GMAC_CTRL_2_REG);
+
+	mvpp2_port_mii_gmac_configure_mode(port);
+
+	/* Unset the GMAC reset state */
+	val = readl(port->base + MVPP2_GMAC_CTRL_2_REG);
+	val &= ~MVPP2_GMAC_PORT_RESET_MASK;
+	writel(val, port->base + MVPP2_GMAC_CTRL_2_REG);
+
+	/* Stop forcing link down */
+	val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+	val &= ~MVPP2_GMAC_FORCE_LINK_DOWN;
+	writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+}
+
+static void mvpp2_port_mii_xlg_configure(struct mvpp2_port *port)
+{
+	u32 val;
+
+	if (port->gop_id != 0)
+		return;
+
+	val = readl(port->base + MVPP22_XLG_CTRL0_REG);
+	val |= MVPP22_XLG_CTRL0_RX_FLOW_CTRL_EN;
+	writel(val, port->base + MVPP22_XLG_CTRL0_REG);
+
+	val = readl(port->base + MVPP22_XLG_CTRL4_REG);
+	val &= ~MVPP22_XLG_CTRL4_MACMODSELECT_GMAC;
+	val |= MVPP22_XLG_CTRL4_FWD_FC | MVPP22_XLG_CTRL4_FWD_PFC;
+	writel(val, port->base + MVPP22_XLG_CTRL4_REG);
+}
+
 static void mvpp22_port_mii_set(struct mvpp2_port *port)
 {
 	u32 val;
@@ -4188,38 +4642,18 @@ static void mvpp22_port_mii_set(struct mvpp2_port *port)
 
 		writel(val, port->base + MVPP22_XLG_CTRL3_REG);
 	}
-
-	val = readl(port->base + MVPP22_GMAC_CTRL_4_REG);
-	if (port->phy_interface == PHY_INTERFACE_MODE_RGMII)
-		val |= MVPP22_CTRL4_EXT_PIN_GMII_SEL;
-	else
-		val &= ~MVPP22_CTRL4_EXT_PIN_GMII_SEL;
-	val &= ~MVPP22_CTRL4_DP_CLK_SEL;
-	val |= MVPP22_CTRL4_SYNC_BYPASS;
-	val |= MVPP22_CTRL4_QSGMII_BYPASS_ACTIVE;
-	writel(val, port->base + MVPP22_GMAC_CTRL_4_REG);
 }
 
 static void mvpp2_port_mii_set(struct mvpp2_port *port)
 {
-	u32 val;
-
 	if (port->priv->hw_version == MVPP22)
 		mvpp22_port_mii_set(port);
 
-	val = readl(port->base + MVPP2_GMAC_CTRL_2_REG);
-
-	switch (port->phy_interface) {
-	case PHY_INTERFACE_MODE_SGMII:
-		val |= MVPP2_GMAC_INBAND_AN_MASK;
-		break;
-	case PHY_INTERFACE_MODE_RGMII:
-		val |= MVPP2_GMAC_PORT_RGMII_MASK;
-	default:
-		val &= ~MVPP2_GMAC_PCS_ENABLE_MASK;
-	}
-
-	writel(val, port->base + MVPP2_GMAC_CTRL_2_REG);
+	if (phy_interface_mode_is_rgmii(port->phy_interface) ||
+	    port->phy_interface == PHY_INTERFACE_MODE_SGMII)
+		mvpp2_port_mii_gmac_configure(port);
+	else if (port->phy_interface == PHY_INTERFACE_MODE_10GKR)
+		mvpp2_port_mii_xlg_configure(port);
 }
 
 static void mvpp2_port_fc_adv_enable(struct mvpp2_port *port)
@@ -4326,6 +4760,18 @@ static inline void mvpp2_gmac_max_rx_size_set(struct mvpp2_port *port)
 	writel(val, port->base + MVPP2_GMAC_CTRL_0_REG);
 }
 
+/* Change maximum receive size of the port */
+static inline void mvpp2_xlg_max_rx_size_set(struct mvpp2_port *port)
+{
+	u32 val;
+
+	val =  readl(port->base + MVPP22_XLG_CTRL1_REG);
+	val &= ~MVPP22_XLG_CTRL1_FRAMESIZELIMIT_MASK;
+	val |= ((port->pkt_size - MVPP2_MH_SIZE) / 2) <<
+	       MVPP22_XLG_CTRL1_FRAMESIZELIMIT_OFFS;
+	writel(val, port->base + MVPP22_XLG_CTRL1_REG);
+}
+
 /* Set defaults to the MVPP2 port */
 static void mvpp2_defaults_set(struct mvpp2_port *port)
 {
@@ -4376,7 +4822,7 @@ static void mvpp2_defaults_set(struct mvpp2_port *port)
 		    MVPP2_RX_LOW_LATENCY_PKT_SIZE(256));
 
 	/* Enable Rx cache snoop */
-	for (lrxq = 0; lrxq < rxq_number; lrxq++) {
+	for (lrxq = 0; lrxq < port->nrxqs; lrxq++) {
 		queue = port->rxqs[lrxq]->id;
 		val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(queue));
 		val |= MVPP2_SNOOP_PKT_SIZE_MASK |
@@ -4394,7 +4840,7 @@ static void mvpp2_ingress_enable(struct mvpp2_port *port)
 	u32 val;
 	int lrxq, queue;
 
-	for (lrxq = 0; lrxq < rxq_number; lrxq++) {
+	for (lrxq = 0; lrxq < port->nrxqs; lrxq++) {
 		queue = port->rxqs[lrxq]->id;
 		val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(queue));
 		val &= ~MVPP2_RXQ_DISABLE_MASK;
@@ -4407,7 +4853,7 @@ static void mvpp2_ingress_disable(struct mvpp2_port *port)
 	u32 val;
 	int lrxq, queue;
 
-	for (lrxq = 0; lrxq < rxq_number; lrxq++) {
+	for (lrxq = 0; lrxq < port->nrxqs; lrxq++) {
 		queue = port->rxqs[lrxq]->id;
 		val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(queue));
 		val |= MVPP2_RXQ_DISABLE_MASK;
@@ -4426,7 +4872,7 @@ static void mvpp2_egress_enable(struct mvpp2_port *port)
 
 	/* Enable all initialized TXs. */
 	qmap = 0;
-	for (queue = 0; queue < txq_number; queue++) {
+	for (queue = 0; queue < port->ntxqs; queue++) {
 		struct mvpp2_tx_queue *txq = port->txqs[queue];
 
 		if (txq->descs)
@@ -4712,7 +5158,7 @@ static void mvpp2_txq_sent_counter_clear(void *arg)
 	struct mvpp2_port *port = arg;
 	int queue;
 
-	for (queue = 0; queue < txq_number; queue++) {
+	for (queue = 0; queue < port->ntxqs; queue++) {
 		int id = port->txqs[queue]->id;
 
 		mvpp2_percpu_read(port->priv, smp_processor_id(),
@@ -4753,7 +5199,7 @@ static void mvpp2_txp_max_tx_size_set(struct mvpp2_port *port)
 		mvpp2_write(port->priv, MVPP2_TXP_SCHED_TOKEN_SIZE_REG, val);
 	}
 
-	for (txq = 0; txq < txq_number; txq++) {
+	for (txq = 0; txq < port->ntxqs; txq++) {
 		val = mvpp2_read(port->priv,
 				 MVPP2_TXQ_SCHED_TOKEN_SIZE_REG(txq));
 		size = val & MVPP2_TXQ_TOKEN_SIZE_MAX;
@@ -4787,6 +5233,23 @@ static void mvpp2_rx_pkts_coal_set(struct mvpp2_port *port,
 	put_cpu();
 }
 
+/* For some reason in the LSP this is done on each CPU. Why ? */
+static void mvpp2_tx_pkts_coal_set(struct mvpp2_port *port,
+				   struct mvpp2_tx_queue *txq)
+{
+	int cpu = get_cpu();
+	u32 val;
+
+	if (txq->done_pkts_coal > MVPP2_TXQ_THRESH_MASK)
+		txq->done_pkts_coal = MVPP2_TXQ_THRESH_MASK;
+
+	val = (txq->done_pkts_coal << MVPP2_TXQ_THRESH_OFFSET);
+	mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_NUM_REG, txq->id);
+	mvpp2_percpu_write(port->priv, cpu, MVPP2_TXQ_THRESH_REG, val);
+
+	put_cpu();
+}
+
 static u32 mvpp2_usec_to_cycles(u32 usec, unsigned long clk_hz)
 {
 	u64 tmp = (u64)clk_hz * usec;
@@ -4823,6 +5286,22 @@ static void mvpp2_rx_time_coal_set(struct mvpp2_port *port,
 	mvpp2_write(port->priv, MVPP2_ISR_RX_THRESHOLD_REG(rxq->id), val);
 }
 
+static void mvpp2_tx_time_coal_set(struct mvpp2_port *port)
+{
+	unsigned long freq = port->priv->tclk;
+	u32 val = mvpp2_usec_to_cycles(port->tx_time_coal, freq);
+
+	if (val > MVPP2_MAX_ISR_TX_THRESHOLD) {
+		port->tx_time_coal =
+			mvpp2_cycles_to_usec(MVPP2_MAX_ISR_TX_THRESHOLD, freq);
+
+		/* re-evaluate to get actual register value */
+		val = mvpp2_usec_to_cycles(port->tx_time_coal, freq);
+	}
+
+	mvpp2_write(port->priv, MVPP2_ISR_TX_THRESHOLD_REG(port->id), val);
+}
+
 /* Free Tx queue skbuffs */
 static void mvpp2_txq_bufs_free(struct mvpp2_port *port,
 				struct mvpp2_tx_queue *txq,
@@ -4881,7 +5360,8 @@ static void mvpp2_txq_done(struct mvpp2_port *port, struct mvpp2_tx_queue *txq,
 			netif_tx_wake_queue(nq);
 }
 
-static unsigned int mvpp2_tx_done(struct mvpp2_port *port, u32 cause)
+static unsigned int mvpp2_tx_done(struct mvpp2_port *port, u32 cause,
+				  int cpu)
 {
 	struct mvpp2_tx_queue *txq;
 	struct mvpp2_txq_pcpu *txq_pcpu;
@@ -4892,7 +5372,7 @@ static unsigned int mvpp2_tx_done(struct mvpp2_port *port, u32 cause)
 		if (!txq)
 			break;
 
-		txq_pcpu = this_cpu_ptr(txq->pcpu);
+		txq_pcpu = per_cpu_ptr(txq->pcpu, cpu);
 
 		if (txq_pcpu->count) {
 			mvpp2_txq_done(port, txq, txq_pcpu);
@@ -4908,15 +5388,14 @@ static unsigned int mvpp2_tx_done(struct mvpp2_port *port, u32 cause)
 
 /* Allocate and initialize descriptors for aggr TXQ */
 static int mvpp2_aggr_txq_init(struct platform_device *pdev,
-			       struct mvpp2_tx_queue *aggr_txq,
-			       int desc_num, int cpu,
+			       struct mvpp2_tx_queue *aggr_txq, int cpu,
 			       struct mvpp2 *priv)
 {
 	u32 txq_dma;
 
 	/* Allocate memory for TX descriptors */
 	aggr_txq->descs = dma_alloc_coherent(&pdev->dev,
-				desc_num * MVPP2_DESC_ALIGNED_SIZE,
+				MVPP2_AGGR_TXQ_SIZE * MVPP2_DESC_ALIGNED_SIZE,
 				&aggr_txq->descs_dma, GFP_KERNEL);
 	if (!aggr_txq->descs)
 		return -ENOMEM;
@@ -4937,7 +5416,8 @@ static int mvpp2_aggr_txq_init(struct platform_device *pdev,
 			MVPP22_AGGR_TXQ_DESC_ADDR_OFFS;
 
 	mvpp2_write(priv, MVPP2_AGGR_TXQ_DESC_ADDR_REG(cpu), txq_dma);
-	mvpp2_write(priv, MVPP2_AGGR_TXQ_DESC_SIZE_REG(cpu), desc_num);
+	mvpp2_write(priv, MVPP2_AGGR_TXQ_DESC_SIZE_REG(cpu),
+		    MVPP2_AGGR_TXQ_SIZE);
 
 	return 0;
 }
@@ -5118,6 +5598,14 @@ static int mvpp2_txq_init(struct mvpp2_port *port,
 		txq_pcpu->reserved_num = 0;
 		txq_pcpu->txq_put_index = 0;
 		txq_pcpu->txq_get_index = 0;
+
+		txq_pcpu->tso_headers =
+			dma_alloc_coherent(port->dev->dev.parent,
+					   MVPP2_AGGR_TXQ_SIZE * TSO_HEADER_SIZE,
+					   &txq_pcpu->tso_headers_dma,
+					   GFP_KERNEL);
+		if (!txq_pcpu->tso_headers)
+			goto cleanup;
 	}
 
 	return 0;
@@ -5125,6 +5613,11 @@ cleanup:
 	for_each_present_cpu(cpu) {
 		txq_pcpu = per_cpu_ptr(txq->pcpu, cpu);
 		kfree(txq_pcpu->buffs);
+
+		dma_free_coherent(port->dev->dev.parent,
+				  MVPP2_AGGR_TXQ_SIZE * MVPP2_DESC_ALIGNED_SIZE,
+				  txq_pcpu->tso_headers,
+				  txq_pcpu->tso_headers_dma);
 	}
 
 	dma_free_coherent(port->dev->dev.parent,
@@ -5144,6 +5637,11 @@ static void mvpp2_txq_deinit(struct mvpp2_port *port,
 	for_each_present_cpu(cpu) {
 		txq_pcpu = per_cpu_ptr(txq->pcpu, cpu);
 		kfree(txq_pcpu->buffs);
+
+		dma_free_coherent(port->dev->dev.parent,
+				  MVPP2_AGGR_TXQ_SIZE * MVPP2_DESC_ALIGNED_SIZE,
+				  txq_pcpu->tso_headers,
+				  txq_pcpu->tso_headers_dma);
 	}
 
 	if (txq->descs)
@@ -5229,7 +5727,7 @@ static void mvpp2_cleanup_txqs(struct mvpp2_port *port)
 	val |= MVPP2_TX_PORT_FLUSH_MASK(port->id);
 	mvpp2_write(port->priv, MVPP2_TX_PORT_FLUSH_REG, val);
 
-	for (queue = 0; queue < txq_number; queue++) {
+	for (queue = 0; queue < port->ntxqs; queue++) {
 		txq = port->txqs[queue];
 		mvpp2_txq_clean(port, txq);
 		mvpp2_txq_deinit(port, txq);
@@ -5246,7 +5744,7 @@ static void mvpp2_cleanup_rxqs(struct mvpp2_port *port)
 {
 	int queue;
 
-	for (queue = 0; queue < rxq_number; queue++)
+	for (queue = 0; queue < port->nrxqs; queue++)
 		mvpp2_rxq_deinit(port, port->rxqs[queue]);
 }
 
@@ -5255,7 +5753,7 @@ static int mvpp2_setup_rxqs(struct mvpp2_port *port)
 {
 	int queue, err;
 
-	for (queue = 0; queue < rxq_number; queue++) {
+	for (queue = 0; queue < port->nrxqs; queue++) {
 		err = mvpp2_rxq_init(port, port->rxqs[queue]);
 		if (err)
 			goto err_cleanup;
@@ -5273,13 +5771,21 @@ static int mvpp2_setup_txqs(struct mvpp2_port *port)
 	struct mvpp2_tx_queue *txq;
 	int queue, err;
 
-	for (queue = 0; queue < txq_number; queue++) {
+	for (queue = 0; queue < port->ntxqs; queue++) {
 		txq = port->txqs[queue];
 		err = mvpp2_txq_init(port, txq);
 		if (err)
 			goto err_cleanup;
 	}
 
+	if (port->has_tx_irqs) {
+		mvpp2_tx_time_coal_set(port);
+		for (queue = 0; queue < port->ntxqs; queue++) {
+			txq = port->txqs[queue];
+			mvpp2_tx_pkts_coal_set(port, txq);
+		}
+	}
+
 	on_each_cpu(mvpp2_txq_sent_counter_clear, port, 1);
 	return 0;
 
@@ -5291,72 +5797,170 @@ err_cleanup:
 /* The callback for per-port interrupt */
 static irqreturn_t mvpp2_isr(int irq, void *dev_id)
 {
+	struct mvpp2_queue_vector *qv = dev_id;
+
+	mvpp2_qvec_interrupt_disable(qv);
+
+	napi_schedule(&qv->napi);
+
+	return IRQ_HANDLED;
+}
+
+/* Per-port interrupt for link status changes */
+static irqreturn_t mvpp2_link_status_isr(int irq, void *dev_id)
+{
 	struct mvpp2_port *port = (struct mvpp2_port *)dev_id;
+	struct net_device *dev = port->dev;
+	bool event = false, link = false;
+	u32 val;
 
-	mvpp2_interrupts_disable(port);
+	mvpp22_gop_mask_irq(port);
 
-	napi_schedule(&port->napi);
+	if (port->gop_id == 0 &&
+	    port->phy_interface == PHY_INTERFACE_MODE_10GKR) {
+		val = readl(port->base + MVPP22_XLG_INT_STAT);
+		if (val & MVPP22_XLG_INT_STAT_LINK) {
+			event = true;
+			val = readl(port->base + MVPP22_XLG_STATUS);
+			if (val & MVPP22_XLG_STATUS_LINK_UP)
+				link = true;
+		}
+	} else if (phy_interface_mode_is_rgmii(port->phy_interface) ||
+		   port->phy_interface == PHY_INTERFACE_MODE_SGMII) {
+		val = readl(port->base + MVPP22_GMAC_INT_STAT);
+		if (val & MVPP22_GMAC_INT_STAT_LINK) {
+			event = true;
+			val = readl(port->base + MVPP2_GMAC_STATUS0);
+			if (val & MVPP2_GMAC_STATUS0_LINK_UP)
+				link = true;
+		}
+	}
 
+	if (!netif_running(dev) || !event)
+		goto handled;
+
+	if (link) {
+		mvpp2_interrupts_enable(port);
+
+		mvpp2_egress_enable(port);
+		mvpp2_ingress_enable(port);
+		netif_carrier_on(dev);
+		netif_tx_wake_all_queues(dev);
+	} else {
+		netif_tx_stop_all_queues(dev);
+		netif_carrier_off(dev);
+		mvpp2_ingress_disable(port);
+		mvpp2_egress_disable(port);
+
+		mvpp2_interrupts_disable(port);
+	}
+
+handled:
+	mvpp22_gop_unmask_irq(port);
 	return IRQ_HANDLED;
 }
 
+static void mvpp2_gmac_set_autoneg(struct mvpp2_port *port,
+				   struct phy_device *phydev)
+{
+	u32 val;
+
+	if (port->phy_interface != PHY_INTERFACE_MODE_RGMII &&
+	    port->phy_interface != PHY_INTERFACE_MODE_RGMII_ID &&
+	    port->phy_interface != PHY_INTERFACE_MODE_RGMII_RXID &&
+	    port->phy_interface != PHY_INTERFACE_MODE_RGMII_TXID &&
+	    port->phy_interface != PHY_INTERFACE_MODE_SGMII)
+		return;
+
+	val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+	val &= ~(MVPP2_GMAC_CONFIG_MII_SPEED |
+		 MVPP2_GMAC_CONFIG_GMII_SPEED |
+		 MVPP2_GMAC_CONFIG_FULL_DUPLEX |
+		 MVPP2_GMAC_AN_SPEED_EN |
+		 MVPP2_GMAC_AN_DUPLEX_EN);
+
+	if (phydev->duplex)
+		val |= MVPP2_GMAC_CONFIG_FULL_DUPLEX;
+
+	if (phydev->speed == SPEED_1000)
+		val |= MVPP2_GMAC_CONFIG_GMII_SPEED;
+	else if (phydev->speed == SPEED_100)
+		val |= MVPP2_GMAC_CONFIG_MII_SPEED;
+
+	writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+}
+
 /* Adjust link */
 static void mvpp2_link_event(struct net_device *dev)
 {
 	struct mvpp2_port *port = netdev_priv(dev);
 	struct phy_device *phydev = dev->phydev;
-	int status_change = 0;
+	bool link_reconfigured = false;
 	u32 val;
 
 	if (phydev->link) {
+		if (port->phy_interface != phydev->interface && port->comphy) {
+	                /* disable current port for reconfiguration */
+	                mvpp2_interrupts_disable(port);
+	                netif_carrier_off(port->dev);
+	                mvpp2_port_disable(port);
+			phy_power_off(port->comphy);
+
+	                /* comphy reconfiguration */
+	                port->phy_interface = phydev->interface;
+	                mvpp22_comphy_init(port);
+
+	                /* gop/mac reconfiguration */
+	                mvpp22_gop_init(port);
+	                mvpp2_port_mii_set(port);
+
+	                link_reconfigured = true;
+		}
+
 		if ((port->speed != phydev->speed) ||
 		    (port->duplex != phydev->duplex)) {
-			u32 val;
-
-			val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
-			val &= ~(MVPP2_GMAC_CONFIG_MII_SPEED |
-				 MVPP2_GMAC_CONFIG_GMII_SPEED |
-				 MVPP2_GMAC_CONFIG_FULL_DUPLEX |
-				 MVPP2_GMAC_AN_SPEED_EN |
-				 MVPP2_GMAC_AN_DUPLEX_EN);
-
-			if (phydev->duplex)
-				val |= MVPP2_GMAC_CONFIG_FULL_DUPLEX;
-
-			if (phydev->speed == SPEED_1000)
-				val |= MVPP2_GMAC_CONFIG_GMII_SPEED;
-			else if (phydev->speed == SPEED_100)
-				val |= MVPP2_GMAC_CONFIG_MII_SPEED;
-
-			writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+			mvpp2_gmac_set_autoneg(port, phydev);
 
 			port->duplex = phydev->duplex;
 			port->speed  = phydev->speed;
 		}
 	}
 
-	if (phydev->link != port->link) {
-		if (!phydev->link) {
-			port->duplex = -1;
-			port->speed = 0;
-		}
-
+	if (phydev->link != port->link || link_reconfigured) {
 		port->link = phydev->link;
-		status_change = 1;
-	}
 
-	if (status_change) {
 		if (phydev->link) {
-			val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
-			val |= (MVPP2_GMAC_FORCE_LINK_PASS |
-				MVPP2_GMAC_FORCE_LINK_DOWN);
-			writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+			if (port->phy_interface == PHY_INTERFACE_MODE_RGMII ||
+			    port->phy_interface == PHY_INTERFACE_MODE_RGMII_ID ||
+			    port->phy_interface == PHY_INTERFACE_MODE_RGMII_RXID ||
+			    port->phy_interface == PHY_INTERFACE_MODE_RGMII_TXID ||
+			    port->phy_interface == PHY_INTERFACE_MODE_SGMII) {
+				val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+				val |= (MVPP2_GMAC_FORCE_LINK_PASS |
+					MVPP2_GMAC_FORCE_LINK_DOWN);
+				writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+			}
+
+			mvpp2_interrupts_enable(port);
+			mvpp2_port_enable(port);
+
 			mvpp2_egress_enable(port);
 			mvpp2_ingress_enable(port);
+			netif_carrier_on(dev);
+			netif_tx_wake_all_queues(dev);
 		} else {
+			port->duplex = -1;
+			port->speed = 0;
+
+			netif_tx_stop_all_queues(dev);
+			netif_carrier_off(dev);
 			mvpp2_ingress_disable(port);
 			mvpp2_egress_disable(port);
+
+			mvpp2_port_disable(port);
+			mvpp2_interrupts_disable(port);
 		}
+
 		phy_print_status(phydev);
 	}
 }
@@ -5385,8 +5989,8 @@ static void mvpp2_tx_proc_cb(unsigned long data)
 	port_pcpu->timer_scheduled = false;
 
 	/* Process all the Tx queues */
-	cause = (1 << txq_number) - 1;
-	tx_todo = mvpp2_tx_done(port, cause);
+	cause = (1 << port->ntxqs) - 1;
+	tx_todo = mvpp2_tx_done(port, cause, smp_processor_id());
 
 	/* Set the timer in case not all the packets were processed */
 	if (tx_todo)
@@ -5498,8 +6102,8 @@ static u32 mvpp2_skb_tx_csum(struct mvpp2_port *port, struct sk_buff *skb)
 }
 
 /* Main rx processing */
-static int mvpp2_rx(struct mvpp2_port *port, int rx_todo,
-		    struct mvpp2_rx_queue *rxq)
+static int mvpp2_rx(struct mvpp2_port *port, struct napi_struct *napi,
+		    int rx_todo, struct mvpp2_rx_queue *rxq)
 {
 	struct net_device *dev = port->dev;
 	int rx_received;
@@ -5577,7 +6181,7 @@ err_drop_frame:
 		skb->protocol = eth_type_trans(skb, dev);
 		mvpp2_rx_csum(port, rx_status, skb);
 
-		napi_gro_receive(&port->napi, skb);
+		napi_gro_receive(napi, skb);
 	}
 
 	if (rcvd_pkts) {
@@ -5665,6 +6269,123 @@ cleanup:
 	return -ENOMEM;
 }
 
+static inline void mvpp2_tso_put_hdr(struct sk_buff *skb,
+				     struct net_device *dev,
+				     struct mvpp2_tx_queue *txq,
+				     struct mvpp2_tx_queue *aggr_txq,
+				     struct mvpp2_txq_pcpu *txq_pcpu,
+				     int hdr_sz)
+{
+	struct mvpp2_port *port = netdev_priv(dev);
+	struct mvpp2_tx_desc *tx_desc = mvpp2_txq_next_desc_get(aggr_txq);
+	dma_addr_t addr;
+
+	mvpp2_txdesc_txq_set(port, tx_desc, txq->id);
+	mvpp2_txdesc_size_set(port, tx_desc, hdr_sz);
+
+	addr = txq_pcpu->tso_headers_dma +
+	       txq_pcpu->txq_put_index * TSO_HEADER_SIZE;
+	mvpp2_txdesc_offset_set(port, tx_desc, addr & MVPP2_TX_DESC_ALIGN);
+	mvpp2_txdesc_dma_addr_set(port, tx_desc, addr & ~MVPP2_TX_DESC_ALIGN);
+
+	mvpp2_txdesc_cmd_set(port, tx_desc, mvpp2_skb_tx_csum(port, skb) |
+					    MVPP2_TXD_F_DESC |
+					    MVPP2_TXD_PADDING_DISABLE);
+	mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc);
+}
+
+static inline int mvpp2_tso_put_data(struct sk_buff *skb,
+				     struct net_device *dev, struct tso_t *tso,
+				     struct mvpp2_tx_queue *txq,
+				     struct mvpp2_tx_queue *aggr_txq,
+				     struct mvpp2_txq_pcpu *txq_pcpu,
+				     int sz, bool left, bool last)
+{
+	struct mvpp2_port *port = netdev_priv(dev);
+	struct mvpp2_tx_desc *tx_desc = mvpp2_txq_next_desc_get(aggr_txq);
+	dma_addr_t buf_dma_addr;
+
+	mvpp2_txdesc_txq_set(port, tx_desc, txq->id);
+	mvpp2_txdesc_size_set(port, tx_desc, sz);
+
+	buf_dma_addr = dma_map_single(dev->dev.parent, tso->data, sz,
+				      DMA_TO_DEVICE);
+	if (unlikely(dma_mapping_error(dev->dev.parent, buf_dma_addr))) {
+		mvpp2_txq_desc_put(txq);
+		return -ENOMEM;
+	}
+
+	mvpp2_txdesc_offset_set(port, tx_desc,
+				buf_dma_addr & MVPP2_TX_DESC_ALIGN);
+	mvpp2_txdesc_dma_addr_set(port, tx_desc,
+				  buf_dma_addr & ~MVPP2_TX_DESC_ALIGN);
+
+	if (!left) {
+		mvpp2_txdesc_cmd_set(port, tx_desc, MVPP2_TXD_L_DESC);
+		if (last) {
+			mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc);
+			return 0;
+		}
+	} else {
+		mvpp2_txdesc_cmd_set(port, tx_desc, 0);
+	}
+
+	mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc);
+	return 0;
+}
+
+static int mvpp2_tx_tso(struct sk_buff *skb, struct net_device *dev,
+			struct mvpp2_tx_queue *txq,
+			struct mvpp2_tx_queue *aggr_txq,
+			struct mvpp2_txq_pcpu *txq_pcpu)
+{
+	struct mvpp2_port *port = netdev_priv(dev);
+	struct tso_t tso;
+	int hdr_sz = skb_transport_offset(skb) + tcp_hdrlen(skb);
+	int i, len, descs = 0;
+
+	/* Check number of available descriptors */
+	if (mvpp2_aggr_desc_num_check(port->priv, aggr_txq,
+				      tso_count_descs(skb)) ||
+	    mvpp2_txq_reserved_desc_num_proc(port->priv, txq, txq_pcpu,
+					     tso_count_descs(skb)))
+		return 0;
+
+	tso_start(skb, &tso);
+	len = skb->len - hdr_sz;
+	while (len > 0) {
+		int left = min_t(int, skb_shinfo(skb)->gso_size, len);
+		char *hdr = txq_pcpu->tso_headers +
+			    txq_pcpu->txq_put_index * TSO_HEADER_SIZE;
+
+		len -= left;
+		descs++;
+
+		tso_build_hdr(skb, hdr, &tso, left, len == 0);
+		mvpp2_tso_put_hdr(skb, dev, txq, aggr_txq, txq_pcpu, hdr_sz);
+
+		while (left > 0) {
+			int sz = min_t(int, tso.size, left);
+			left -= sz;
+			descs++;
+
+			if (mvpp2_tso_put_data(skb, dev, &tso, txq, aggr_txq,
+					       txq_pcpu, sz, left, len == 0))
+				goto release;
+			tso_build_data(skb, &tso, sz);
+		}
+	}
+
+	return descs;
+
+release:
+	for (i = descs - 1; i >= 0; i--) {
+		struct mvpp2_tx_desc *tx_desc = txq->descs + i;
+		tx_desc_unmap_put(port, txq, tx_desc);
+	}
+	return 0;
+}
+
 /* Main tx processing */
 static int mvpp2_tx(struct sk_buff *skb, struct net_device *dev)
 {
@@ -5682,6 +6403,10 @@ static int mvpp2_tx(struct sk_buff *skb, struct net_device *dev)
 	txq_pcpu = this_cpu_ptr(txq->pcpu);
 	aggr_txq = &port->priv->aggr_txqs[smp_processor_id()];
 
+	if (skb_is_gso(skb)) {
+		frags = mvpp2_tx_tso(skb, dev, txq, aggr_txq, txq_pcpu);
+		goto out;
+	}
 	frags = skb_shinfo(skb)->nr_frags + 1;
 
 	/* Check number of available descriptors */
@@ -5731,22 +6456,21 @@ static int mvpp2_tx(struct sk_buff *skb, struct net_device *dev)
 		}
 	}
 
-	txq_pcpu->reserved_num -= frags;
-	txq_pcpu->count += frags;
-	aggr_txq->count += frags;
-
-	/* Enable transmit */
-	wmb();
-	mvpp2_aggr_txq_pend_desc_add(port, frags);
-
-	if (txq_pcpu->size - txq_pcpu->count < MAX_SKB_FRAGS + 1) {
-		struct netdev_queue *nq = netdev_get_tx_queue(dev, txq_id);
-
-		netif_tx_stop_queue(nq);
-	}
 out:
 	if (frags > 0) {
 		struct mvpp2_pcpu_stats *stats = this_cpu_ptr(port->stats);
+		struct netdev_queue *nq = netdev_get_tx_queue(dev, txq_id);
+
+		txq_pcpu->reserved_num -= frags;
+		txq_pcpu->count += frags;
+		aggr_txq->count += frags;
+
+		/* Enable transmit */
+		wmb();
+		mvpp2_aggr_txq_pend_desc_add(port, frags);
+
+		if (txq_pcpu->size - txq_pcpu->count < MAX_SKB_FRAGS + 1)
+			netif_tx_stop_queue(nq);
 
 		u64_stats_update_begin(&stats->syncp);
 		stats->tx_packets++;
@@ -5762,7 +6486,8 @@ out:
 		mvpp2_txq_done(port, txq, txq_pcpu);
 
 	/* Set the timer in case not all frags were processed */
-	if (txq_pcpu->count <= frags && txq_pcpu->count > 0) {
+	if (!port->has_tx_irqs && txq_pcpu->count <= frags &&
+	    txq_pcpu->count > 0) {
 		struct mvpp2_port_pcpu *port_pcpu = this_cpu_ptr(port->pcpu);
 
 		mvpp2_timer_set(port_pcpu);
@@ -5783,11 +6508,14 @@ static inline void mvpp2_cause_error(struct net_device *dev, int cause)
 
 static int mvpp2_poll(struct napi_struct *napi, int budget)
 {
-	u32 cause_rx_tx, cause_rx, cause_misc;
+	u32 cause_rx_tx, cause_rx, cause_tx, cause_misc;
 	int rx_done = 0;
 	struct mvpp2_port *port = netdev_priv(napi->dev);
+	struct mvpp2_queue_vector *qv;
 	int cpu = smp_processor_id();
 
+	qv = container_of(napi, struct mvpp2_queue_vector, napi);
+
 	/* Rx/Tx cause register
 	 *
 	 * Bits 0-15: each bit indicates received packets on the Rx queue
@@ -5798,11 +6526,10 @@ static int mvpp2_poll(struct napi_struct *napi, int budget)
 	 *
 	 * Each CPU has its own Rx/Tx cause register
 	 */
-	cause_rx_tx = mvpp2_percpu_read(port->priv, cpu,
+	cause_rx_tx = mvpp2_percpu_read(port->priv, qv->sw_thread_id,
 					MVPP2_ISR_RX_TX_CAUSE_REG(port->id));
-	cause_rx_tx &= ~MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_MASK;
-	cause_misc = cause_rx_tx & MVPP2_CAUSE_MISC_SUM_MASK;
 
+	cause_misc = cause_rx_tx & MVPP2_CAUSE_MISC_SUM_MASK;
 	if (cause_misc) {
 		mvpp2_cause_error(port->dev, cause_misc);
 
@@ -5813,10 +6540,16 @@ static int mvpp2_poll(struct napi_struct *napi, int budget)
 				   cause_rx_tx & ~MVPP2_CAUSE_MISC_SUM_MASK);
 	}
 
-	cause_rx = cause_rx_tx & MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK;
+	cause_tx = cause_rx_tx & MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_MASK;
+	if (cause_tx) {
+		cause_tx >>= MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_OFFSET;
+		mvpp2_tx_done(port, cause_tx, qv->sw_thread_id);
+	}
 
 	/* Process RX packets */
-	cause_rx |= port->pending_cause_rx;
+	cause_rx = cause_rx_tx & MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK;
+	cause_rx <<= qv->first_rxq;
+	cause_rx |= qv->pending_cause_rx;
 	while (cause_rx && budget > 0) {
 		int count;
 		struct mvpp2_rx_queue *rxq;
@@ -5825,7 +6558,7 @@ static int mvpp2_poll(struct napi_struct *napi, int budget)
 		if (!rxq)
 			break;
 
-		count = mvpp2_rx(port, budget, rxq);
+		count = mvpp2_rx(port, napi, budget, rxq);
 		rx_done += count;
 		budget -= count;
 		if (budget > 0) {
@@ -5841,9 +6574,9 @@ static int mvpp2_poll(struct napi_struct *napi, int budget)
 		cause_rx = 0;
 		napi_complete_done(napi, rx_done);
 
-		mvpp2_interrupts_enable(port);
+		mvpp2_qvec_interrupt_enable(qv);
 	}
-	port->pending_cause_rx = cause_rx;
+	qv->pending_cause_rx = cause_rx;
 	return rx_done;
 }
 
@@ -5851,17 +6584,32 @@ static int mvpp2_poll(struct napi_struct *napi, int budget)
 static void mvpp2_start_dev(struct mvpp2_port *port)
 {
 	struct net_device *ndev = port->dev;
+	int i;
+
+	if (port->gop_id == 0 &&
+	    (port->phy_interface == PHY_INTERFACE_MODE_XAUI ||
+	     port->phy_interface == PHY_INTERFACE_MODE_10GKR))
+		mvpp2_xlg_max_rx_size_set(port);
+	else
+		mvpp2_gmac_max_rx_size_set(port);
 
-	mvpp2_gmac_max_rx_size_set(port);
 	mvpp2_txp_max_tx_size_set(port);
 
-	napi_enable(&port->napi);
+	for (i = 0; i < port->nqvecs; i++)
+		napi_enable(&port->qvecs[i].napi);
 
 	/* Enable interrupts on all CPUs */
 	mvpp2_interrupts_enable(port);
 
+	if (port->priv->hw_version == MVPP22) {
+		mvpp22_comphy_init(port);
+		mvpp22_gop_init(port);
+	}
+
+	mvpp2_port_mii_set(port);
 	mvpp2_port_enable(port);
-	phy_start(ndev->phydev);
+	if (ndev->phydev)
+		phy_start(ndev->phydev);
 	netif_tx_start_all_queues(port->dev);
 }
 
@@ -5869,6 +6617,7 @@ static void mvpp2_start_dev(struct mvpp2_port *port)
 static void mvpp2_stop_dev(struct mvpp2_port *port)
 {
 	struct net_device *ndev = port->dev;
+	int i;
 
 	/* Stop new packets from arriving to RXQs */
 	mvpp2_ingress_disable(port);
@@ -5878,14 +6627,17 @@ static void mvpp2_stop_dev(struct mvpp2_port *port)
 	/* Disable interrupts on all CPUs */
 	mvpp2_interrupts_disable(port);
 
-	napi_disable(&port->napi);
+	for (i = 0; i < port->nqvecs; i++)
+		napi_disable(&port->qvecs[i].napi);
 
 	netif_carrier_off(port->dev);
 	netif_tx_stop_all_queues(port->dev);
 
 	mvpp2_egress_disable(port);
 	mvpp2_port_disable(port);
-	phy_stop(ndev->phydev);
+	if (ndev->phydev)
+		phy_stop(ndev->phydev);
+	phy_power_off(port->comphy);
 }
 
 static int mvpp2_check_ringparam_valid(struct net_device *dev,
@@ -5941,6 +6693,10 @@ static int mvpp2_phy_connect(struct mvpp2_port *port)
 {
 	struct phy_device *phy_dev;
 
+	/* No PHY is attached */
+	if (!port->phy_node)
+		return 0;
+
 	phy_dev = of_phy_connect(port->dev, port->phy_node, mvpp2_link_event, 0,
 				 port->phy_interface);
 	if (!phy_dev) {
@@ -5961,12 +6717,56 @@ static void mvpp2_phy_disconnect(struct mvpp2_port *port)
 {
 	struct net_device *ndev = port->dev;
 
+	if (!ndev->phydev)
+		return;
+
 	phy_disconnect(ndev->phydev);
 }
 
+static int mvpp2_irqs_init(struct mvpp2_port *port)
+{
+	int err, i;
+
+	for (i = 0; i < port->nqvecs; i++) {
+		struct mvpp2_queue_vector *qv = port->qvecs + i;
+
+		err = request_irq(qv->irq, mvpp2_isr, 0, port->dev->name, qv);
+		if (err)
+			goto err;
+
+		if (qv->type == MVPP2_QUEUE_VECTOR_PRIVATE)
+			irq_set_affinity_hint(qv->irq,
+					      cpumask_of(qv->sw_thread_id));
+	}
+
+	return 0;
+err:
+	for (i = 0; i < port->nqvecs; i++) {
+		struct mvpp2_queue_vector *qv = port->qvecs + i;
+
+		irq_set_affinity_hint(qv->irq, NULL);
+		free_irq(qv->irq, qv);
+	}
+
+	return err;
+}
+
+static void mvpp2_irqs_deinit(struct mvpp2_port *port)
+{
+	int i;
+
+	for (i = 0; i < port->nqvecs; i++) {
+		struct mvpp2_queue_vector *qv = port->qvecs + i;
+
+		irq_set_affinity_hint(qv->irq, NULL);
+		free_irq(qv->irq, qv);
+	}
+}
+
 static int mvpp2_open(struct net_device *dev)
 {
 	struct mvpp2_port *port = netdev_priv(dev);
+	struct mvpp2 *priv = port->priv;
 	unsigned char mac_bcast[ETH_ALEN] = {
 			0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
 	int err;
@@ -6006,28 +6806,44 @@ static int mvpp2_open(struct net_device *dev)
 		goto err_cleanup_rxqs;
 	}
 
-	err = request_irq(port->irq, mvpp2_isr, 0, dev->name, port);
+	err = mvpp2_irqs_init(port);
 	if (err) {
-		netdev_err(port->dev, "cannot request IRQ %d\n", port->irq);
+		netdev_err(port->dev, "cannot init IRQs\n");
 		goto err_cleanup_txqs;
 	}
 
+	if (priv->hw_version == MVPP22 && !port->phy_node && port->link_irq) {
+		err = request_irq(port->link_irq, mvpp2_link_status_isr, 0,
+				  dev->name, port);
+		if (err) {
+			netdev_err(port->dev, "cannot request link IRQ %d\n",
+				   port->link_irq);
+			goto err_free_irq;
+		}
+
+		mvpp22_gop_setup_irq(port);
+	}
+
 	/* In default link is down */
 	netif_carrier_off(port->dev);
 
 	err = mvpp2_phy_connect(port);
 	if (err < 0)
-		goto err_free_irq;
+		goto err_free_link_irq;
 
 	/* Unmask interrupts on all CPUs */
 	on_each_cpu(mvpp2_interrupts_unmask, port, 1);
+	mvpp2_shared_interrupt_mask_unmask(port, false);
 
 	mvpp2_start_dev(port);
 
 	return 0;
 
+err_free_link_irq:
+	if (priv->hw_version == MVPP22 && !port->phy_node && port->link_irq)
+		free_irq(port->link_irq, port);
 err_free_irq:
-	free_irq(port->irq, port);
+	mvpp2_irqs_deinit(port);
 err_cleanup_txqs:
 	mvpp2_cleanup_txqs(port);
 err_cleanup_rxqs:
@@ -6039,6 +6855,7 @@ static int mvpp2_stop(struct net_device *dev)
 {
 	struct mvpp2_port *port = netdev_priv(dev);
 	struct mvpp2_port_pcpu *port_pcpu;
+	struct mvpp2 *priv = port->priv;
 	int cpu;
 
 	mvpp2_stop_dev(port);
@@ -6046,14 +6863,20 @@ static int mvpp2_stop(struct net_device *dev)
 
 	/* Mask interrupts on all CPUs */
 	on_each_cpu(mvpp2_interrupts_mask, port, 1);
+	mvpp2_shared_interrupt_mask_unmask(port, true);
 
-	free_irq(port->irq, port);
-	for_each_present_cpu(cpu) {
-		port_pcpu = per_cpu_ptr(port->pcpu, cpu);
+	if (priv->hw_version == MVPP22 && !port->phy_node && port->link_irq)
+		free_irq(port->link_irq, port);
 
-		hrtimer_cancel(&port_pcpu->tx_done_timer);
-		port_pcpu->timer_scheduled = false;
-		tasklet_kill(&port_pcpu->tx_done_tasklet);
+	mvpp2_irqs_deinit(port);
+	if (!port->has_tx_irqs) {
+		for_each_present_cpu(cpu) {
+			port_pcpu = per_cpu_ptr(port->pcpu, cpu);
+
+			hrtimer_cancel(&port_pcpu->tx_done_timer);
+			port_pcpu->timer_scheduled = false;
+			tasklet_kill(&port_pcpu->tx_done_tasklet);
+		}
 	}
 	mvpp2_cleanup_rxqs(port);
 	mvpp2_cleanup_txqs(port);
@@ -6228,7 +7051,7 @@ static int mvpp2_ethtool_set_coalesce(struct net_device *dev,
 	struct mvpp2_port *port = netdev_priv(dev);
 	int queue;
 
-	for (queue = 0; queue < rxq_number; queue++) {
+	for (queue = 0; queue < port->nrxqs; queue++) {
 		struct mvpp2_rx_queue *rxq = port->rxqs[queue];
 
 		rxq->time_coal = c->rx_coalesce_usecs;
@@ -6237,10 +7060,18 @@ static int mvpp2_ethtool_set_coalesce(struct net_device *dev,
 		mvpp2_rx_time_coal_set(port, rxq);
 	}
 
-	for (queue = 0; queue < txq_number; queue++) {
+	if (port->has_tx_irqs) {
+		port->tx_time_coal = c->tx_coalesce_usecs;
+		mvpp2_tx_time_coal_set(port);
+	}
+
+	for (queue = 0; queue < port->ntxqs; queue++) {
 		struct mvpp2_tx_queue *txq = port->txqs[queue];
 
 		txq->done_pkts_coal = c->tx_max_coalesced_frames;
+
+		if (port->has_tx_irqs)
+			mvpp2_tx_pkts_coal_set(port, txq);
 	}
 
 	return 0;
@@ -6365,6 +7196,129 @@ static const struct ethtool_ops mvpp2_eth_tool_ops = {
 	.set_link_ksettings = phy_ethtool_set_link_ksettings,
 };
 
+/* Used for PPv2.1, or PPv2.2 with the old Device Tree binding that
+ * had a single IRQ defined per-port.
+ */
+static int mvpp2_simple_queue_vectors_init(struct mvpp2_port *port,
+					   struct device_node *port_node)
+{
+	struct mvpp2_queue_vector *v = &port->qvecs[0];
+
+	v->first_rxq = 0;
+	v->nrxqs = port->nrxqs;
+	v->type = MVPP2_QUEUE_VECTOR_SHARED;
+	v->sw_thread_id = 0;
+	v->sw_thread_mask = *cpumask_bits(cpu_online_mask);
+	v->port = port;
+	v->irq = irq_of_parse_and_map(port_node, 0);
+	if (v->irq <= 0)
+		return -EINVAL;
+	netif_napi_add(port->dev, &v->napi, mvpp2_poll,
+		       NAPI_POLL_WEIGHT);
+
+	port->nqvecs = 1;
+
+	return 0;
+}
+
+static int mvpp2_multi_queue_vectors_init(struct mvpp2_port *port,
+					  struct device_node *port_node)
+{
+	struct mvpp2_queue_vector *v;
+	int i, ret;
+
+	port->nqvecs = num_possible_cpus();
+	if (queue_mode == MVPP2_QDIST_SINGLE_MODE)
+		port->nqvecs += 1;
+
+	for (i = 0; i < port->nqvecs; i++) {
+		char irqname[16];
+
+		v = port->qvecs + i;
+
+		v->port = port;
+		v->type = MVPP2_QUEUE_VECTOR_PRIVATE;
+		v->sw_thread_id = i;
+		v->sw_thread_mask = BIT(i);
+
+		snprintf(irqname, sizeof(irqname), "tx-cpu%d", i);
+
+		if (queue_mode == MVPP2_QDIST_MULTI_MODE) {
+			v->first_rxq = i * MVPP2_DEFAULT_RXQ;
+			v->nrxqs = MVPP2_DEFAULT_RXQ;
+		} else if (queue_mode == MVPP2_QDIST_SINGLE_MODE &&
+			   i == (port->nqvecs - 1)) {
+			v->first_rxq = 0;
+			v->nrxqs = port->nrxqs;
+			v->type = MVPP2_QUEUE_VECTOR_SHARED;
+			strncpy(irqname, "rx-shared", sizeof(irqname));
+		}
+
+		v->irq = of_irq_get_byname(port_node, irqname);
+		if (v->irq <= 0) {
+			ret = -EINVAL;
+			goto err;
+		}
+
+		netif_napi_add(port->dev, &v->napi, mvpp2_poll,
+			       NAPI_POLL_WEIGHT);
+	}
+
+	return 0;
+
+err:
+	for (i = 0; i < port->nqvecs; i++)
+		irq_dispose_mapping(port->qvecs[i].irq);
+	return ret;
+}
+
+static int mvpp2_queue_vectors_init(struct mvpp2_port *port,
+				    struct device_node *port_node)
+{
+	if (port->has_tx_irqs)
+		return mvpp2_multi_queue_vectors_init(port, port_node);
+	else
+		return mvpp2_simple_queue_vectors_init(port, port_node);
+}
+
+static void mvpp2_queue_vectors_deinit(struct mvpp2_port *port)
+{
+	int i;
+
+	for (i = 0; i < port->nqvecs; i++)
+		irq_dispose_mapping(port->qvecs[i].irq);
+}
+
+/* Configure Rx queue group interrupt for this port */
+static void mvpp2_rx_irqs_setup(struct mvpp2_port *port)
+{
+	struct mvpp2 *priv = port->priv;
+	u32 val;
+	int i;
+
+	if (priv->hw_version == MVPP21) {
+		mvpp2_write(priv, MVPP21_ISR_RXQ_GROUP_REG(port->id),
+			    port->nrxqs);
+		return;
+	}
+
+	/* Handle the more complicated PPv2.2 case */
+	for (i = 0; i < port->nqvecs; i++) {
+		struct mvpp2_queue_vector *qv = port->qvecs + i;
+
+		if (!qv->nrxqs)
+			continue;
+
+		val = qv->sw_thread_id;
+		val |= port->id << MVPP22_ISR_RXQ_GROUP_INDEX_GROUP_OFFSET;
+		mvpp2_write(priv, MVPP22_ISR_RXQ_GROUP_INDEX_REG, val);
+
+		val = qv->first_rxq;
+		val |= qv->nrxqs << MVPP22_ISR_RXQ_SUB_GROUP_SIZE_OFFSET;
+		mvpp2_write(priv, MVPP22_ISR_RXQ_SUB_GROUP_CONFIG_REG, val);
+	}
+}
+
 /* Initialize port HW */
 static int mvpp2_port_init(struct mvpp2_port *port)
 {
@@ -6373,15 +7327,22 @@ static int mvpp2_port_init(struct mvpp2_port *port)
 	struct mvpp2_txq_pcpu *txq_pcpu;
 	int queue, cpu, err;
 
-	if (port->first_rxq + rxq_number >
+	/* Checks for hardware constraints */
+	if (port->first_rxq + port->nrxqs >
 	    MVPP2_MAX_PORTS * priv->max_port_rxqs)
 		return -EINVAL;
 
+	if (port->nrxqs % 4 || (port->nrxqs > priv->max_port_rxqs) ||
+	    (port->ntxqs > MVPP2_MAX_TXQ))
+		return -EINVAL;
+
 	/* Disable port */
 	mvpp2_egress_disable(port);
 	mvpp2_port_disable(port);
 
-	port->txqs = devm_kcalloc(dev, txq_number, sizeof(*port->txqs),
+	port->tx_time_coal = MVPP2_TXDONE_COAL_USEC;
+
+	port->txqs = devm_kcalloc(dev, port->ntxqs, sizeof(*port->txqs),
 				  GFP_KERNEL);
 	if (!port->txqs)
 		return -ENOMEM;
@@ -6389,7 +7350,7 @@ static int mvpp2_port_init(struct mvpp2_port *port)
 	/* Associate physical Tx queues to this port and initialize.
 	 * The mapping is predefined.
 	 */
-	for (queue = 0; queue < txq_number; queue++) {
+	for (queue = 0; queue < port->ntxqs; queue++) {
 		int queue_phy_id = mvpp2_txq_phys(port->id, queue);
 		struct mvpp2_tx_queue *txq;
 
@@ -6416,7 +7377,7 @@ static int mvpp2_port_init(struct mvpp2_port *port)
 		port->txqs[queue] = txq;
 	}
 
-	port->rxqs = devm_kcalloc(dev, rxq_number, sizeof(*port->rxqs),
+	port->rxqs = devm_kcalloc(dev, port->nrxqs, sizeof(*port->rxqs),
 				  GFP_KERNEL);
 	if (!port->rxqs) {
 		err = -ENOMEM;
@@ -6424,7 +7385,7 @@ static int mvpp2_port_init(struct mvpp2_port *port)
 	}
 
 	/* Allocate and initialize Rx queue for this port */
-	for (queue = 0; queue < rxq_number; queue++) {
+	for (queue = 0; queue < port->nrxqs; queue++) {
 		struct mvpp2_rx_queue *rxq;
 
 		/* Map physical Rx queue to port's logical Rx queue */
@@ -6441,22 +7402,10 @@ static int mvpp2_port_init(struct mvpp2_port *port)
 		port->rxqs[queue] = rxq;
 	}
 
-	/* Configure Rx queue group interrupt for this port */
-	if (priv->hw_version == MVPP21) {
-		mvpp2_write(priv, MVPP21_ISR_RXQ_GROUP_REG(port->id),
-			    rxq_number);
-	} else {
-		u32 val;
-
-		val = (port->id << MVPP22_ISR_RXQ_GROUP_INDEX_GROUP_OFFSET);
-		mvpp2_write(priv, MVPP22_ISR_RXQ_GROUP_INDEX_REG, val);
-
-		val = (rxq_number << MVPP22_ISR_RXQ_SUB_GROUP_SIZE_OFFSET);
-		mvpp2_write(priv, MVPP22_ISR_RXQ_SUB_GROUP_CONFIG_REG, val);
-	}
+	mvpp2_rx_irqs_setup(port);
 
 	/* Create Rx descriptor rings */
-	for (queue = 0; queue < rxq_number; queue++) {
+	for (queue = 0; queue < port->nrxqs; queue++) {
 		struct mvpp2_rx_queue *rxq = port->rxqs[queue];
 
 		rxq->size = port->rx_ring_size;
@@ -6484,7 +7433,7 @@ static int mvpp2_port_init(struct mvpp2_port *port)
 	return 0;
 
 err_free_percpu:
-	for (queue = 0; queue < txq_number; queue++) {
+	for (queue = 0; queue < port->ntxqs; queue++) {
 		if (!port->txqs[queue])
 			continue;
 		free_percpu(port->txqs[queue]->pcpu);
@@ -6492,35 +7441,93 @@ err_free_percpu:
 	return err;
 }
 
+/* Checks if the port DT description has the TX interrupts
+ * described. On PPv2.1, there are no such interrupts. On PPv2.2,
+ * there are available, but we need to keep support for old DTs.
+ */
+static bool mvpp2_port_has_tx_irqs(struct mvpp2 *priv,
+				   struct device_node *port_node)
+{
+	char *irqs[5] = { "rx-shared", "tx-cpu0", "tx-cpu1",
+			  "tx-cpu2", "tx-cpu3" };
+	int ret, i;
+
+	if (priv->hw_version == MVPP21)
+		return false;
+
+	for (i = 0; i < 5; i++) {
+		ret = of_property_match_string(port_node, "interrupt-names",
+					       irqs[i]);
+		if (ret < 0)
+			return false;
+	}
+
+	return true;
+}
+
+static void mvpp2_port_copy_mac_addr(struct net_device *dev, struct mvpp2 *priv,
+				     struct device_node *port_node,
+				     char **mac_from)
+{
+	struct mvpp2_port *port = netdev_priv(dev);
+	char hw_mac_addr[ETH_ALEN] = {0};
+	const char *dt_mac_addr;
+
+	dt_mac_addr = of_get_mac_address(port_node);
+	if (dt_mac_addr && is_valid_ether_addr(dt_mac_addr)) {
+		*mac_from = "device tree";
+		ether_addr_copy(dev->dev_addr, dt_mac_addr);
+		return;
+	}
+
+	if (priv->hw_version == MVPP21) {
+		mvpp21_get_mac_address(port, hw_mac_addr);
+		if (is_valid_ether_addr(hw_mac_addr)) {
+			*mac_from = "hardware";
+			ether_addr_copy(dev->dev_addr, hw_mac_addr);
+			return;
+		}
+	}
+
+	*mac_from = "random";
+	eth_hw_addr_random(dev);
+}
+
 /* Ports initialization */
 static int mvpp2_port_probe(struct platform_device *pdev,
 			    struct device_node *port_node,
 			    struct mvpp2 *priv)
 {
 	struct device_node *phy_node;
+	struct phy *comphy;
 	struct mvpp2_port *port;
 	struct mvpp2_port_pcpu *port_pcpu;
 	struct net_device *dev;
 	struct resource *res;
-	const char *dt_mac_addr;
-	const char *mac_from;
-	char hw_mac_addr[ETH_ALEN];
+	char *mac_from = "";
+	unsigned int ntxqs, nrxqs;
+	bool has_tx_irqs;
 	u32 id;
 	int features;
 	int phy_mode;
 	int err, i, cpu;
 
-	dev = alloc_etherdev_mqs(sizeof(*port), txq_number, rxq_number);
+	has_tx_irqs = mvpp2_port_has_tx_irqs(priv, port_node);
+
+	if (!has_tx_irqs)
+		queue_mode = MVPP2_QDIST_SINGLE_MODE;
+
+	ntxqs = MVPP2_MAX_TXQ;
+	if (priv->hw_version == MVPP22 && queue_mode == MVPP2_QDIST_MULTI_MODE)
+		nrxqs = MVPP2_DEFAULT_RXQ * num_possible_cpus();
+	else
+		nrxqs = MVPP2_DEFAULT_RXQ;
+
+	dev = alloc_etherdev_mqs(sizeof(*port), ntxqs, nrxqs);
 	if (!dev)
 		return -ENOMEM;
 
 	phy_node = of_parse_phandle(port_node, "phy", 0);
-	if (!phy_node) {
-		dev_err(&pdev->dev, "missing phy\n");
-		err = -ENODEV;
-		goto err_free_netdev;
-	}
-
 	phy_mode = of_get_phy_mode(port_node);
 	if (phy_mode < 0) {
 		dev_err(&pdev->dev, "incorrect phy mode\n");
@@ -6528,6 +7535,15 @@ static int mvpp2_port_probe(struct platform_device *pdev,
 		goto err_free_netdev;
 	}
 
+	comphy = devm_of_phy_get(&pdev->dev, port_node, NULL);
+	if (IS_ERR(comphy)) {
+		if (PTR_ERR(comphy) == -EPROBE_DEFER) {
+			err = -EPROBE_DEFER;
+			goto err_free_netdev;
+		}
+		comphy = NULL;
+	}
+
 	if (of_property_read_u32(port_node, "port-id", &id)) {
 		err = -EINVAL;
 		dev_err(&pdev->dev, "missing port-id value\n");
@@ -6540,25 +7556,37 @@ static int mvpp2_port_probe(struct platform_device *pdev,
 	dev->ethtool_ops = &mvpp2_eth_tool_ops;
 
 	port = netdev_priv(dev);
+	port->dev = dev;
+	port->ntxqs = ntxqs;
+	port->nrxqs = nrxqs;
+	port->priv = priv;
+	port->has_tx_irqs = has_tx_irqs;
 
-	port->irq = irq_of_parse_and_map(port_node, 0);
-	if (port->irq <= 0) {
-		err = -EINVAL;
+	err = mvpp2_queue_vectors_init(port, port_node);
+	if (err)
 		goto err_free_netdev;
+
+	port->link_irq = of_irq_get_byname(port_node, "link");
+	if (port->link_irq == -EPROBE_DEFER) {
+		err = -EPROBE_DEFER;
+		goto err_deinit_qvecs;
 	}
+	if (port->link_irq <= 0)
+		/* the link irq is optional */
+		port->link_irq = 0;
 
 	if (of_property_read_bool(port_node, "marvell,loopback"))
 		port->flags |= MVPP2_F_LOOPBACK;
 
-	port->priv = priv;
 	port->id = id;
 	if (priv->hw_version == MVPP21)
-		port->first_rxq = port->id * rxq_number;
+		port->first_rxq = port->id * port->nrxqs;
 	else
 		port->first_rxq = port->id * priv->max_port_rxqs;
 
 	port->phy_node = phy_node;
 	port->phy_interface = phy_mode;
+	port->comphy = comphy;
 
 	if (priv->hw_version == MVPP21) {
 		res = platform_get_resource(pdev, IORESOURCE_MEM, 2 + id);
@@ -6572,7 +7600,7 @@ static int mvpp2_port_probe(struct platform_device *pdev,
 					 &port->gop_id)) {
 			err = -EINVAL;
 			dev_err(&pdev->dev, "missing gop-port-id value\n");
-			goto err_free_irq;
+			goto err_deinit_qvecs;
 		}
 
 		port->base = priv->iface_base + MVPP22_GMAC_BASE(port->gop_id);
@@ -6585,25 +7613,10 @@ static int mvpp2_port_probe(struct platform_device *pdev,
 		goto err_free_irq;
 	}
 
-	dt_mac_addr = of_get_mac_address(port_node);
-	if (dt_mac_addr && is_valid_ether_addr(dt_mac_addr)) {
-		mac_from = "device tree";
-		ether_addr_copy(dev->dev_addr, dt_mac_addr);
-	} else {
-		if (priv->hw_version == MVPP21)
-			mvpp21_get_mac_address(port, hw_mac_addr);
-		if (is_valid_ether_addr(hw_mac_addr)) {
-			mac_from = "hardware";
-			ether_addr_copy(dev->dev_addr, hw_mac_addr);
-		} else {
-			mac_from = "random";
-			eth_hw_addr_random(dev);
-		}
-	}
+	mvpp2_port_copy_mac_addr(dev, priv, port_node, &mac_from);
 
 	port->tx_ring_size = MVPP2_MAX_TXD;
 	port->rx_ring_size = MVPP2_MAX_RXD;
-	port->dev = dev;
 	SET_NETDEV_DEV(dev, &pdev->dev);
 
 	err = mvpp2_port_init(port);
@@ -6612,7 +7625,6 @@ static int mvpp2_port_probe(struct platform_device *pdev,
 		goto err_free_stats;
 	}
 
-	mvpp2_port_mii_set(port);
 	mvpp2_port_periodic_xon_disable(port);
 
 	if (priv->hw_version == MVPP21)
@@ -6626,20 +7638,22 @@ static int mvpp2_port_probe(struct platform_device *pdev,
 		goto err_free_txq_pcpu;
 	}
 
-	for_each_present_cpu(cpu) {
-		port_pcpu = per_cpu_ptr(port->pcpu, cpu);
+	if (!port->has_tx_irqs) {
+		for_each_present_cpu(cpu) {
+			port_pcpu = per_cpu_ptr(port->pcpu, cpu);
 
-		hrtimer_init(&port_pcpu->tx_done_timer, CLOCK_MONOTONIC,
-			     HRTIMER_MODE_REL_PINNED);
-		port_pcpu->tx_done_timer.function = mvpp2_hr_timer_cb;
-		port_pcpu->timer_scheduled = false;
+			hrtimer_init(&port_pcpu->tx_done_timer, CLOCK_MONOTONIC,
+				     HRTIMER_MODE_REL_PINNED);
+			port_pcpu->tx_done_timer.function = mvpp2_hr_timer_cb;
+			port_pcpu->timer_scheduled = false;
 
-		tasklet_init(&port_pcpu->tx_done_tasklet, mvpp2_tx_proc_cb,
-			     (unsigned long)dev);
+			tasklet_init(&port_pcpu->tx_done_tasklet,
+				     mvpp2_tx_proc_cb,
+				     (unsigned long)dev);
+		}
 	}
 
-	netif_napi_add(dev, &port->napi, mvpp2_poll, NAPI_POLL_WEIGHT);
-	features = NETIF_F_SG | NETIF_F_IP_CSUM;
+	features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO;
 	dev->features = features | NETIF_F_RXCSUM;
 	dev->hw_features |= features | NETIF_F_RXCSUM | NETIF_F_GRO;
 	dev->vlan_features |= features;
@@ -6662,12 +7676,15 @@ static int mvpp2_port_probe(struct platform_device *pdev,
 err_free_port_pcpu:
 	free_percpu(port->pcpu);
 err_free_txq_pcpu:
-	for (i = 0; i < txq_number; i++)
+	for (i = 0; i < port->ntxqs; i++)
 		free_percpu(port->txqs[i]->pcpu);
 err_free_stats:
 	free_percpu(port->stats);
 err_free_irq:
-	irq_dispose_mapping(port->irq);
+	if (port->link_irq)
+		irq_dispose_mapping(port->link_irq);
+err_deinit_qvecs:
+	mvpp2_queue_vectors_deinit(port);
 err_free_netdev:
 	of_node_put(phy_node);
 	free_netdev(dev);
@@ -6683,9 +7700,11 @@ static void mvpp2_port_remove(struct mvpp2_port *port)
 	of_node_put(port->phy_node);
 	free_percpu(port->pcpu);
 	free_percpu(port->stats);
-	for (i = 0; i < txq_number; i++)
+	for (i = 0; i < port->ntxqs; i++)
 		free_percpu(port->txqs[i]->pcpu);
-	irq_dispose_mapping(port->irq);
+	mvpp2_queue_vectors_deinit(port);
+	if (port->link_irq)
+		irq_dispose_mapping(port->link_irq);
 	free_netdev(port->dev);
 }
 
@@ -6800,13 +7819,6 @@ static int mvpp2_init(struct platform_device *pdev, struct mvpp2 *priv)
 	int err, i;
 	u32 val;
 
-	/* Checks for hardware constraints */
-	if (rxq_number % 4 || (rxq_number > priv->max_port_rxqs) ||
-	    (txq_number > MVPP2_MAX_TXQ)) {
-		dev_err(&pdev->dev, "invalid queue size parameter\n");
-		return -EINVAL;
-	}
-
 	/* MBUS windows configuration */
 	dram_target_info = mv_mbus_dram_info();
 	if (dram_target_info)
@@ -6836,8 +7848,7 @@ static int mvpp2_init(struct platform_device *pdev, struct mvpp2 *priv)
 	for_each_present_cpu(i) {
 		priv->aggr_txqs[i].id = i;
 		priv->aggr_txqs[i].size = MVPP2_AGGR_TXQ_SIZE;
-		err = mvpp2_aggr_txq_init(pdev, &priv->aggr_txqs[i],
-					  MVPP2_AGGR_TXQ_SIZE, i, priv);
+		err = mvpp2_aggr_txq_init(pdev, &priv->aggr_txqs[i], i, priv);
 		if (err < 0)
 			return err;
 	}
@@ -6845,23 +7856,6 @@ static int mvpp2_init(struct platform_device *pdev, struct mvpp2 *priv)
 	/* Rx Fifo Init */
 	mvpp2_rx_fifo_init(priv);
 
-	/* Reset Rx queue group interrupt configuration */
-	for (i = 0; i < MVPP2_MAX_PORTS; i++) {
-		if (priv->hw_version == MVPP21) {
-			mvpp2_write(priv, MVPP21_ISR_RXQ_GROUP_REG(i),
-				    rxq_number);
-			continue;
-		} else {
-			u32 val;
-
-			val = (i << MVPP22_ISR_RXQ_GROUP_INDEX_GROUP_OFFSET);
-			mvpp2_write(priv, MVPP22_ISR_RXQ_GROUP_INDEX_REG, val);
-
-			val = (rxq_number << MVPP22_ISR_RXQ_SUB_GROUP_SIZE_OFFSET);
-			mvpp2_write(priv, MVPP22_ISR_RXQ_SUB_GROUP_CONFIG_REG, val);
-		}
-	}
-
 	if (priv->hw_version == MVPP21)
 		writel(MVPP2_EXT_GLOBAL_CTRL_DEFAULT,
 		       priv->lms_base + MVPP2_MNG_EXTENDED_GLOBAL_CTRL_REG);
@@ -6892,7 +7886,7 @@ static int mvpp2_probe(struct platform_device *pdev)
 	struct mvpp2 *priv;
 	struct resource *res;
 	void __iomem *base;
-	int port_count, cpu;
+	int port_count, i;
 	int err;
 
 	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
@@ -6917,14 +7911,25 @@ static int mvpp2_probe(struct platform_device *pdev)
 		priv->iface_base = devm_ioremap_resource(&pdev->dev, res);
 		if (IS_ERR(priv->iface_base))
 			return PTR_ERR(priv->iface_base);
+
+		priv->sysctrl_base =
+			syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
+							"marvell,system-controller");
+		if (IS_ERR(priv->sysctrl_base))
+			/* The system controller regmap is optional for dt
+			 * compatibility reasons. When not provided, the
+			 * configuration of the GoP relies on the
+			 * firmware/bootloader.
+			 */
+			priv->sysctrl_base = NULL;
 	}
 
-	for_each_present_cpu(cpu) {
+	for (i = 0; i < MVPP2_MAX_THREADS; i++) {
 		u32 addr_space_sz;
 
 		addr_space_sz = (priv->hw_version == MVPP21 ?
 				 MVPP21_ADDR_SPACE_SZ : MVPP22_ADDR_SPACE_SZ);
-		priv->cpu_base[cpu] = base + cpu * addr_space_sz;
+		priv->swth_base[i] = base + i * addr_space_sz;
 	}
 
 	if (priv->hw_version == MVPP21)
diff --git a/drivers/net/ethernet/marvell/skge.c b/drivers/net/ethernet/marvell/skge.c
index 5d7d94de4e00..eef35bf3e849 100644
--- a/drivers/net/ethernet/marvell/skge.c
+++ b/drivers/net/ethernet/marvell/skge.c
@@ -3516,7 +3516,7 @@ static const char *skge_board_name(const struct skge_hw *hw)
 		if (skge_chips[i].id == hw->chip_id)
 			return skge_chips[i].name;
 
-	snprintf(buf, sizeof buf, "chipid 0x%x", hw->chip_id);
+	snprintf(buf, sizeof(buf), "chipid 0x%x", hw->chip_id);
 	return buf;
 }
 
@@ -4193,7 +4193,7 @@ static struct pci_driver skge_driver = {
 	.driver.pm =	SKGE_PM_OPS,
 };
 
-static struct dmi_system_id skge_32bit_dma_boards[] = {
+static const struct dmi_system_id skge_32bit_dma_boards[] = {
 	{
 		.ident = "Gigabyte nForce boards",
 		.matches = {
diff --git a/drivers/net/ethernet/mediatek/Kconfig b/drivers/net/ethernet/mediatek/Kconfig
index 698bb89aa901..f9149d2a4694 100644
--- a/drivers/net/ethernet/mediatek/Kconfig
+++ b/drivers/net/ethernet/mediatek/Kconfig
@@ -7,11 +7,11 @@ config NET_VENDOR_MEDIATEK
 if NET_VENDOR_MEDIATEK
 
 config NET_MEDIATEK_SOC
-	tristate "MediaTek MT7623 Gigabit ethernet support"
-	depends on NET_VENDOR_MEDIATEK && (MACH_MT7623 || MACH_MT2701)
+	tristate "MediaTek SoC Gigabit Ethernet support"
+	depends on NET_VENDOR_MEDIATEK
 	select PHYLIB
 	---help---
 	  This driver supports the gigabit ethernet MACs in the
-	  MediaTek MT2701/MT7623 chipset family.
+	  MediaTek SoC family.
 
 endif #NET_VENDOR_MEDIATEK
diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
index e588a0cdb074..5e81a7263654 100644
--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
@@ -53,7 +53,8 @@ static const struct mtk_ethtool_stats {
 };
 
 static const char * const mtk_clks_source_name[] = {
-	"ethif", "esw", "gp1", "gp2", "trgpll"
+	"ethif", "esw", "gp0", "gp1", "gp2", "trgpll", "sgmii_tx250m",
+	"sgmii_rx250m", "sgmii_cdr_ref", "sgmii_cdr_fb", "sgmii_ck", "eth2pll"
 };
 
 void mtk_w32(struct mtk_eth *eth, u32 val, unsigned reg)
@@ -163,6 +164,47 @@ static void mtk_gmac0_rgmii_adjust(struct mtk_eth *eth, int speed)
 	mtk_w32(eth, val, TRGMII_TCK_CTRL);
 }
 
+static void mtk_gmac_sgmii_hw_setup(struct mtk_eth *eth, int mac_id)
+{
+	u32 val;
+
+	/* Setup the link timer and QPHY power up inside SGMIISYS */
+	regmap_write(eth->sgmiisys, SGMSYS_PCS_LINK_TIMER,
+		     SGMII_LINK_TIMER_DEFAULT);
+
+	regmap_read(eth->sgmiisys, SGMSYS_SGMII_MODE, &val);
+	val |= SGMII_REMOTE_FAULT_DIS;
+	regmap_write(eth->sgmiisys, SGMSYS_SGMII_MODE, val);
+
+	regmap_read(eth->sgmiisys, SGMSYS_PCS_CONTROL_1, &val);
+	val |= SGMII_AN_RESTART;
+	regmap_write(eth->sgmiisys, SGMSYS_PCS_CONTROL_1, val);
+
+	regmap_read(eth->sgmiisys, SGMSYS_QPHY_PWR_STATE_CTRL, &val);
+	val &= ~SGMII_PHYA_PWD;
+	regmap_write(eth->sgmiisys, SGMSYS_QPHY_PWR_STATE_CTRL, val);
+
+	/* Determine MUX for which GMAC uses the SGMII interface */
+	if (MTK_HAS_CAPS(eth->soc->caps, MTK_DUAL_GMAC_SHARED_SGMII)) {
+		regmap_read(eth->ethsys, ETHSYS_SYSCFG0, &val);
+		val &= ~SYSCFG0_SGMII_MASK;
+		val |= !mac_id ? SYSCFG0_SGMII_GMAC1 : SYSCFG0_SGMII_GMAC2;
+		regmap_write(eth->ethsys, ETHSYS_SYSCFG0, val);
+
+		dev_info(eth->dev, "setup shared sgmii for gmac=%d\n",
+			 mac_id);
+	}
+
+	/* Setup the GMAC1 going through SGMII path when SoC also support
+	 * ESW on GMAC1
+	 */
+	if (MTK_HAS_CAPS(eth->soc->caps, MTK_GMAC1_ESW | MTK_GMAC1_SGMII) &&
+	    !mac_id) {
+		mtk_w32(eth, 0, MTK_MAC_MISC);
+		dev_info(eth->dev, "setup gmac1 going through sgmii");
+	}
+}
+
 static void mtk_phy_link_adjust(struct net_device *dev)
 {
 	struct mtk_mac *mac = netdev_priv(dev);
@@ -185,7 +227,8 @@ static void mtk_phy_link_adjust(struct net_device *dev)
 		break;
 	};
 
-	if (mac->id == 0 && !mac->trgmii)
+	if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_GMAC1_TRGMII) &&
+	    !mac->id && !mac->trgmii)
 		mtk_gmac0_rgmii_adjust(mac->hw, dev->phydev->speed);
 
 	if (dev->phydev->link)
@@ -269,6 +312,7 @@ static int mtk_phy_connect(struct net_device *dev)
 	if (!np)
 		return -ENODEV;
 
+	mac->ge_mode = 0;
 	switch (of_get_phy_mode(np)) {
 	case PHY_INTERFACE_MODE_TRGMII:
 		mac->trgmii = true;
@@ -276,7 +320,10 @@ static int mtk_phy_connect(struct net_device *dev)
 	case PHY_INTERFACE_MODE_RGMII_RXID:
 	case PHY_INTERFACE_MODE_RGMII_ID:
 	case PHY_INTERFACE_MODE_RGMII:
-		mac->ge_mode = 0;
+		break;
+	case PHY_INTERFACE_MODE_SGMII:
+		if (MTK_HAS_CAPS(eth->soc->caps, MTK_SGMII))
+			mtk_gmac_sgmii_hw_setup(eth, mac->id);
 		break;
 	case PHY_INTERFACE_MODE_MII:
 		mac->ge_mode = 1;
@@ -1032,7 +1079,6 @@ static int mtk_poll_tx(struct mtk_eth *eth, int budget)
 	unsigned int done[MTK_MAX_DEVS];
 	unsigned int bytes[MTK_MAX_DEVS];
 	u32 cpu, dma;
-	static int condition;
 	int total = 0, i;
 
 	memset(done, 0, sizeof(done));
@@ -1056,10 +1102,8 @@ static int mtk_poll_tx(struct mtk_eth *eth, int budget)
 			mac = 1;
 
 		skb = tx_buf->skb;
-		if (!skb) {
-			condition = 1;
+		if (!skb)
 			break;
-		}
 
 		if (skb != (struct sk_buff *)MTK_DMA_DUMMY_DESC) {
 			bytes[mac] += skb->len;
@@ -1241,9 +1285,19 @@ static void mtk_tx_clean(struct mtk_eth *eth)
 
 static int mtk_rx_alloc(struct mtk_eth *eth, int ring_no, int rx_flag)
 {
-	struct mtk_rx_ring *ring = &eth->rx_ring[ring_no];
+	struct mtk_rx_ring *ring;
 	int rx_data_len, rx_dma_size;
 	int i;
+	u32 offset = 0;
+
+	if (rx_flag == MTK_RX_FLAGS_QDMA) {
+		if (ring_no)
+			return -EINVAL;
+		ring = &eth->rx_ring_qdma;
+		offset = 0x1000;
+	} else {
+		ring = &eth->rx_ring[ring_no];
+	}
 
 	if (rx_flag == MTK_RX_FLAGS_HWLRO) {
 		rx_data_len = MTK_MAX_LRO_RX_LENGTH;
@@ -1293,17 +1347,16 @@ static int mtk_rx_alloc(struct mtk_eth *eth, int ring_no, int rx_flag)
 	 */
 	wmb();
 
-	mtk_w32(eth, ring->phys, MTK_PRX_BASE_PTR_CFG(ring_no));
-	mtk_w32(eth, rx_dma_size, MTK_PRX_MAX_CNT_CFG(ring_no));
-	mtk_w32(eth, ring->calc_idx, ring->crx_idx_reg);
-	mtk_w32(eth, MTK_PST_DRX_IDX_CFG(ring_no), MTK_PDMA_RST_IDX);
+	mtk_w32(eth, ring->phys, MTK_PRX_BASE_PTR_CFG(ring_no) + offset);
+	mtk_w32(eth, rx_dma_size, MTK_PRX_MAX_CNT_CFG(ring_no) + offset);
+	mtk_w32(eth, ring->calc_idx, ring->crx_idx_reg + offset);
+	mtk_w32(eth, MTK_PST_DRX_IDX_CFG(ring_no), MTK_PDMA_RST_IDX + offset);
 
 	return 0;
 }
 
-static void mtk_rx_clean(struct mtk_eth *eth, int ring_no)
+static void mtk_rx_clean(struct mtk_eth *eth, struct mtk_rx_ring *ring)
 {
-	struct mtk_rx_ring *ring = &eth->rx_ring[ring_no];
 	int i;
 
 	if (ring->data && ring->dma) {
@@ -1629,6 +1682,10 @@ static int mtk_dma_init(struct mtk_eth *eth)
 	if (err)
 		return err;
 
+	err = mtk_rx_alloc(eth, 0, MTK_RX_FLAGS_QDMA);
+	if (err)
+		return err;
+
 	err = mtk_rx_alloc(eth, 0, MTK_RX_FLAGS_NORMAL);
 	if (err)
 		return err;
@@ -1668,12 +1725,13 @@ static void mtk_dma_free(struct mtk_eth *eth)
 		eth->phy_scratch_ring = 0;
 	}
 	mtk_tx_clean(eth);
-	mtk_rx_clean(eth, 0);
+	mtk_rx_clean(eth, &eth->rx_ring[0]);
+	mtk_rx_clean(eth, &eth->rx_ring_qdma);
 
 	if (eth->hwlro) {
 		mtk_hwlro_rx_uninit(eth);
 		for (i = 1; i < MTK_MAX_RX_RING_NUM; i++)
-			mtk_rx_clean(eth, i);
+			mtk_rx_clean(eth, &eth->rx_ring[i]);
 	}
 
 	kfree(eth->scratch_head);
@@ -1740,7 +1798,9 @@ static int mtk_start_dma(struct mtk_eth *eth)
 
 	mtk_w32(eth,
 		MTK_TX_WB_DDONE | MTK_TX_DMA_EN |
-		MTK_DMA_SIZE_16DWORDS | MTK_NDP_CO_PRO,
+		MTK_DMA_SIZE_16DWORDS | MTK_NDP_CO_PRO |
+		MTK_RX_DMA_EN | MTK_RX_2B_OFFSET |
+		MTK_RX_BT_32DWORDS,
 		MTK_QDMA_GLO_CFG);
 
 	mtk_w32(eth,
@@ -1837,9 +1897,36 @@ static void ethsys_reset(struct mtk_eth *eth, u32 reset_bits)
 	mdelay(10);
 }
 
+static void mtk_clk_disable(struct mtk_eth *eth)
+{
+	int clk;
+
+	for (clk = MTK_CLK_MAX - 1; clk >= 0; clk--)
+		clk_disable_unprepare(eth->clks[clk]);
+}
+
+static int mtk_clk_enable(struct mtk_eth *eth)
+{
+	int clk, ret;
+
+	for (clk = 0; clk < MTK_CLK_MAX ; clk++) {
+		ret = clk_prepare_enable(eth->clks[clk]);
+		if (ret)
+			goto err_disable_clks;
+	}
+
+	return 0;
+
+err_disable_clks:
+	while (--clk >= 0)
+		clk_disable_unprepare(eth->clks[clk]);
+
+	return ret;
+}
+
 static int mtk_hw_init(struct mtk_eth *eth)
 {
-	int i, val;
+	int i, val, ret;
 
 	if (test_and_set_bit(MTK_HW_INIT, &eth->state))
 		return 0;
@@ -1847,10 +1934,10 @@ static int mtk_hw_init(struct mtk_eth *eth)
 	pm_runtime_enable(eth->dev);
 	pm_runtime_get_sync(eth->dev);
 
-	clk_prepare_enable(eth->clks[MTK_CLK_ETHIF]);
-	clk_prepare_enable(eth->clks[MTK_CLK_ESW]);
-	clk_prepare_enable(eth->clks[MTK_CLK_GP1]);
-	clk_prepare_enable(eth->clks[MTK_CLK_GP2]);
+	ret = mtk_clk_enable(eth);
+	if (ret)
+		goto err_disable_pm;
+
 	ethsys_reset(eth, RSTCTRL_FE);
 	ethsys_reset(eth, RSTCTRL_PPE);
 
@@ -1918,6 +2005,12 @@ static int mtk_hw_init(struct mtk_eth *eth)
 	}
 
 	return 0;
+
+err_disable_pm:
+	pm_runtime_put_sync(eth->dev);
+	pm_runtime_disable(eth->dev);
+
+	return ret;
 }
 
 static int mtk_hw_deinit(struct mtk_eth *eth)
@@ -1925,10 +2018,7 @@ static int mtk_hw_deinit(struct mtk_eth *eth)
 	if (!test_and_clear_bit(MTK_HW_INIT, &eth->state))
 		return 0;
 
-	clk_disable_unprepare(eth->clks[MTK_CLK_GP2]);
-	clk_disable_unprepare(eth->clks[MTK_CLK_GP1]);
-	clk_disable_unprepare(eth->clks[MTK_CLK_ESW]);
-	clk_disable_unprepare(eth->clks[MTK_CLK_ETHIF]);
+	mtk_clk_disable(eth);
 
 	pm_runtime_put_sync(eth->dev);
 	pm_runtime_disable(eth->dev);
@@ -2395,6 +2485,7 @@ static int mtk_get_chip_id(struct mtk_eth *eth, u32 *chip_id)
 static bool mtk_is_hwlro_supported(struct mtk_eth *eth)
 {
 	switch (eth->chip_id) {
+	case MT7622_ETH:
 	case MT7623_ETH:
 		return true;
 	}
@@ -2406,6 +2497,7 @@ static int mtk_probe(struct platform_device *pdev)
 {
 	struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	struct device_node *mac_np;
+	const struct of_device_id *match;
 	struct mtk_eth *eth;
 	int err;
 	int i;
@@ -2414,6 +2506,9 @@ static int mtk_probe(struct platform_device *pdev)
 	if (!eth)
 		return -ENOMEM;
 
+	match = of_match_device(of_mtk_match, &pdev->dev);
+	eth->soc = (struct mtk_soc_data *)match->data;
+
 	eth->dev = &pdev->dev;
 	eth->base = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(eth->base))
@@ -2430,6 +2525,16 @@ static int mtk_probe(struct platform_device *pdev)
 		return PTR_ERR(eth->ethsys);
 	}
 
+	if (MTK_HAS_CAPS(eth->soc->caps, MTK_SGMII)) {
+		eth->sgmiisys =
+		syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
+						"mediatek,sgmiisys");
+		if (IS_ERR(eth->sgmiisys)) {
+			dev_err(&pdev->dev, "no sgmiisys regmap found\n");
+			return PTR_ERR(eth->sgmiisys);
+		}
+	}
+
 	eth->pctl = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
 						    "mediatek,pctl");
 	if (IS_ERR(eth->pctl)) {
@@ -2450,7 +2555,12 @@ static int mtk_probe(struct platform_device *pdev)
 		if (IS_ERR(eth->clks[i])) {
 			if (PTR_ERR(eth->clks[i]) == -EPROBE_DEFER)
 				return -EPROBE_DEFER;
-			return -ENODEV;
+			if (eth->soc->required_clks & BIT(i)) {
+				dev_err(&pdev->dev, "clock %s not found\n",
+					mtk_clks_source_name[i]);
+				return -EINVAL;
+			}
+			eth->clks[i] = NULL;
 		}
 	}
 
@@ -2553,8 +2663,25 @@ static int mtk_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static const struct mtk_soc_data mt2701_data = {
+	.caps = MTK_GMAC1_TRGMII,
+	.required_clks = MT7623_CLKS_BITMAP
+};
+
+static const struct mtk_soc_data mt7622_data = {
+	.caps = MTK_DUAL_GMAC_SHARED_SGMII | MTK_GMAC1_ESW,
+	.required_clks = MT7622_CLKS_BITMAP
+};
+
+static const struct mtk_soc_data mt7623_data = {
+	.caps = MTK_GMAC1_TRGMII,
+	.required_clks = MT7623_CLKS_BITMAP
+};
+
 const struct of_device_id of_mtk_match[] = {
-	{ .compatible = "mediatek,mt2701-eth" },
+	{ .compatible = "mediatek,mt2701-eth", .data = &mt2701_data},
+	{ .compatible = "mediatek,mt7622-eth", .data = &mt7622_data},
+	{ .compatible = "mediatek,mt7623-eth", .data = &mt7623_data},
 	{},
 };
 MODULE_DEVICE_TABLE(of, of_mtk_match);
diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.h b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
index 5868a09f623a..3d3c24a28112 100644
--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h
+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
@@ -302,6 +302,9 @@
 #define PHY_IAC_REG_SHIFT	25
 #define PHY_IAC_TIMEOUT		HZ
 
+#define MTK_MAC_MISC		0x1000c
+#define MTK_MUX_TO_ESW		BIT(0)
+
 /* Mac control registers */
 #define MTK_MAC_MCR(x)		(0x10100 + (x * 0x100))
 #define MAC_MCR_MAX_RX_1536	BIT(24)
@@ -357,11 +360,15 @@
 #define ETHSYS_CHIPID0_3	0x0
 #define ETHSYS_CHIPID4_7	0x4
 #define MT7623_ETH		7623
+#define MT7622_ETH		7622
 
 /* ethernet subsystem config register */
 #define ETHSYS_SYSCFG0		0x14
 #define SYSCFG0_GE_MASK		0x3
 #define SYSCFG0_GE_MODE(x, y)	(x << (12 + (y * 2)))
+#define SYSCFG0_SGMII_MASK	(3 << 8)
+#define SYSCFG0_SGMII_GMAC1	((2 << 8) & GENMASK(9, 8))
+#define SYSCFG0_SGMII_GMAC2	((3 << 8) & GENMASK(9, 8))
 
 /* ethernet subsystem clock register */
 #define ETHSYS_CLKCFG0		0x2c
@@ -372,6 +379,23 @@
 #define RSTCTRL_FE		BIT(6)
 #define RSTCTRL_PPE		BIT(31)
 
+/* SGMII subsystem config registers */
+/* Register to auto-negotiation restart */
+#define SGMSYS_PCS_CONTROL_1	0x0
+#define SGMII_AN_RESTART	BIT(9)
+
+/* Register to programmable link timer, the unit in 2 * 8ns */
+#define SGMSYS_PCS_LINK_TIMER	0x18
+#define SGMII_LINK_TIMER_DEFAULT	(0x186a0 & GENMASK(19, 0))
+
+/* Register to control remote fault */
+#define SGMSYS_SGMII_MODE	0x20
+#define SGMII_REMOTE_FAULT_DIS	BIT(8)
+
+/* Register to power up QPHY */
+#define SGMSYS_QPHY_PWR_STATE_CTRL 0xe8
+#define	SGMII_PHYA_PWD		BIT(4)
+
 struct mtk_rx_dma {
 	unsigned int rxd1;
 	unsigned int rxd2;
@@ -437,12 +461,31 @@ enum mtk_tx_flags {
 enum mtk_clks_map {
 	MTK_CLK_ETHIF,
 	MTK_CLK_ESW,
+	MTK_CLK_GP0,
 	MTK_CLK_GP1,
 	MTK_CLK_GP2,
 	MTK_CLK_TRGPLL,
+	MTK_CLK_SGMII_TX_250M,
+	MTK_CLK_SGMII_RX_250M,
+	MTK_CLK_SGMII_CDR_REF,
+	MTK_CLK_SGMII_CDR_FB,
+	MTK_CLK_SGMII_CK,
+	MTK_CLK_ETH2PLL,
 	MTK_CLK_MAX
 };
 
+#define MT7623_CLKS_BITMAP	(BIT(MTK_CLK_ETHIF) | BIT(MTK_CLK_ESW) |  \
+				 BIT(MTK_CLK_GP1) | BIT(MTK_CLK_GP2) | \
+				 BIT(MTK_CLK_TRGPLL))
+#define MT7622_CLKS_BITMAP	(BIT(MTK_CLK_ETHIF) | BIT(MTK_CLK_ESW) |  \
+				 BIT(MTK_CLK_GP0) | BIT(MTK_CLK_GP1) | \
+				 BIT(MTK_CLK_GP2) | \
+				 BIT(MTK_CLK_SGMII_TX_250M) | \
+				 BIT(MTK_CLK_SGMII_RX_250M) | \
+				 BIT(MTK_CLK_SGMII_CDR_REF) | \
+				 BIT(MTK_CLK_SGMII_CDR_FB) | \
+				 BIT(MTK_CLK_SGMII_CK) | \
+				 BIT(MTK_CLK_ETH2PLL))
 enum mtk_dev_state {
 	MTK_HW_INIT,
 	MTK_RESETTING
@@ -489,6 +532,7 @@ struct mtk_tx_ring {
 enum mtk_rx_flags {
 	MTK_RX_FLAGS_NORMAL = 0,
 	MTK_RX_FLAGS_HWLRO,
+	MTK_RX_FLAGS_QDMA,
 };
 
 /* struct mtk_rx_ring -	This struct holds info describing a RX ring
@@ -511,6 +555,28 @@ struct mtk_rx_ring {
 	u32 crx_idx_reg;
 };
 
+#define MTK_TRGMII			BIT(0)
+#define MTK_GMAC1_TRGMII		(BIT(1) | MTK_TRGMII)
+#define MTK_ESW				BIT(4)
+#define MTK_GMAC1_ESW			(BIT(5) | MTK_ESW)
+#define MTK_SGMII			BIT(8)
+#define MTK_GMAC1_SGMII			(BIT(9) | MTK_SGMII)
+#define MTK_GMAC2_SGMII			(BIT(10) | MTK_SGMII)
+#define MTK_DUAL_GMAC_SHARED_SGMII	(BIT(11) | MTK_GMAC1_SGMII | \
+					 MTK_GMAC2_SGMII)
+#define MTK_HAS_CAPS(caps, _x)		(((caps) & (_x)) == (_x))
+
+/* struct mtk_eth_data -	This is the structure holding all differences
+ *				among various plaforms
+ * @caps			Flags shown the extra capability for the SoC
+ * @required_clks		Flags shown the bitmap for required clocks on
+ *				the target SoC
+ */
+struct mtk_soc_data {
+	u32		caps;
+	u32		required_clks;
+};
+
 /* currently no SoC has more than 2 macs */
 #define MTK_MAX_DEVS			2
 
@@ -529,11 +595,14 @@ struct mtk_rx_ring {
  * @msg_enable:		Ethtool msg level
  * @ethsys:		The register map pointing at the range used to setup
  *			MII modes
+ * @sgmiisys:		The register map pointing at the range used to setup
+ *			SGMII modes
  * @pctl:		The register map pointing at the range used to setup
  *			GMAC port drive/slew values
  * @dma_refcnt:		track how many netdevs are using the DMA engine
- * @tx_ring:		Pointer to the memore holding info about the TX ring
- * @rx_ring:		Pointer to the memore holding info about the RX ring
+ * @tx_ring:		Pointer to the memory holding info about the TX ring
+ * @rx_ring:		Pointer to the memory holding info about the RX ring
+ * @rx_ring_qdma:	Pointer to the memory holding info about the QDMA RX ring
  * @tx_napi:		The TX NAPI struct
  * @rx_napi:		The RX NAPI struct
  * @scratch_ring:	Newer SoCs need memory for a second HW managed TX ring
@@ -542,7 +611,8 @@ struct mtk_rx_ring {
  * @clks:		clock array for all clocks required
  * @mii_bus:		If there is a bus we need to create an instance for it
  * @pending_work:	The workqueue used to reset the dma ring
- * @state               Initialization and runtime state of the device.
+ * @state:		Initialization and runtime state of the device
+ * @soc:		Holding specific data among vaious SoCs
  */
 
 struct mtk_eth {
@@ -558,12 +628,14 @@ struct mtk_eth {
 	u32				msg_enable;
 	unsigned long			sysclk;
 	struct regmap			*ethsys;
+	struct regmap			*sgmiisys;
 	struct regmap			*pctl;
 	u32				chip_id;
 	bool				hwlro;
 	atomic_t			dma_refcnt;
 	struct mtk_tx_ring		tx_ring;
 	struct mtk_rx_ring		rx_ring[MTK_MAX_RX_RING_NUM];
+	struct mtk_rx_ring		rx_ring_qdma;
 	struct napi_struct		tx_napi;
 	struct napi_struct		rx_napi;
 	struct mtk_tx_dma		*scratch_ring;
@@ -574,6 +646,8 @@ struct mtk_eth {
 	struct mii_bus			*mii_bus;
 	struct work_struct		pending_work;
 	unsigned long			state;
+
+	const struct mtk_soc_data	*soc;
 };
 
 /* struct mtk_mac -	the structure that holds the info about the MACs of the
diff --git a/drivers/net/ethernet/mellanox/Kconfig b/drivers/net/ethernet/mellanox/Kconfig
index 84a200764111..872548cd9431 100644
--- a/drivers/net/ethernet/mellanox/Kconfig
+++ b/drivers/net/ethernet/mellanox/Kconfig
@@ -5,9 +5,10 @@
 config NET_VENDOR_MELLANOX
 	bool "Mellanox devices"
 	default y
-	depends on PCI
+	depends on PCI || I2C
 	---help---
-	  If you have a network (Ethernet) card belonging to this class, say Y.
+	  If you have a network (Ethernet or RDMA) device belonging to this
+	  class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
 	  kernel: saying N will just cause the configurator to skip all
diff --git a/drivers/net/ethernet/mellanox/mlx4/alloc.c b/drivers/net/ethernet/mellanox/mlx4/alloc.c
index b651c1210555..6dabd983e7e0 100644
--- a/drivers/net/ethernet/mellanox/mlx4/alloc.c
+++ b/drivers/net/ethernet/mellanox/mlx4/alloc.c
@@ -186,7 +186,7 @@ int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask,
 	bitmap->effective_len = bitmap->avail;
 	spin_lock_init(&bitmap->lock);
 	bitmap->table = kzalloc(BITS_TO_LONGS(bitmap->max) *
-				sizeof (long), GFP_KERNEL);
+				sizeof(long), GFP_KERNEL);
 	if (!bitmap->table)
 		return -ENOMEM;
 
diff --git a/drivers/net/ethernet/mellanox/mlx4/cmd.c b/drivers/net/ethernet/mellanox/mlx4/cmd.c
index 674773b28b2e..6a9086dc1e92 100644
--- a/drivers/net/ethernet/mellanox/mlx4/cmd.c
+++ b/drivers/net/ethernet/mellanox/mlx4/cmd.c
@@ -1958,19 +1958,19 @@ static void mlx4_allocate_port_vpps(struct mlx4_dev *dev, int port)
 	int i;
 	int err;
 	int num_vfs;
-	u16 availible_vpp;
+	u16 available_vpp;
 	u8 vpp_param[MLX4_NUM_UP];
 	struct mlx4_qos_manager *port_qos;
 	struct mlx4_priv *priv = mlx4_priv(dev);
 
-	err = mlx4_ALLOCATE_VPP_get(dev, port, &availible_vpp, vpp_param);
+	err = mlx4_ALLOCATE_VPP_get(dev, port, &available_vpp, vpp_param);
 	if (err) {
-		mlx4_info(dev, "Failed query availible VPPs\n");
+		mlx4_info(dev, "Failed query available VPPs\n");
 		return;
 	}
 
 	port_qos = &priv->mfunc.master.qos_ctl[port];
-	num_vfs = (availible_vpp /
+	num_vfs = (available_vpp /
 		   bitmap_weight(port_qos->priority_bm, MLX4_NUM_UP));
 
 	for (i = 0; i < MLX4_NUM_UP; i++) {
@@ -1985,14 +1985,14 @@ static void mlx4_allocate_port_vpps(struct mlx4_dev *dev, int port)
 	}
 
 	/* Query actual allocated VPP, just to make sure */
-	err = mlx4_ALLOCATE_VPP_get(dev, port, &availible_vpp, vpp_param);
+	err = mlx4_ALLOCATE_VPP_get(dev, port, &available_vpp, vpp_param);
 	if (err) {
-		mlx4_info(dev, "Failed query availible VPPs\n");
+		mlx4_info(dev, "Failed query available VPPs\n");
 		return;
 	}
 
 	port_qos->num_of_qos_vfs = num_vfs;
-	mlx4_dbg(dev, "Port %d Availible VPPs %d\n", port, availible_vpp);
+	mlx4_dbg(dev, "Port %d Available VPPs %d\n", port, available_vpp);
 
 	for (i = 0; i < MLX4_NUM_UP; i++)
 		mlx4_dbg(dev, "Port %d UP %d Allocated %d VPPs\n", port, i,
@@ -2535,8 +2535,8 @@ int mlx4_cmd_init(struct mlx4_dev *dev)
 	}
 
 	if (!priv->cmd.pool) {
-		priv->cmd.pool = pci_pool_create("mlx4_cmd",
-						 dev->persist->pdev,
+		priv->cmd.pool = dma_pool_create("mlx4_cmd",
+						 &dev->persist->pdev->dev,
 						 MLX4_MAILBOX_SIZE,
 						 MLX4_MAILBOX_SIZE, 0);
 		if (!priv->cmd.pool)
@@ -2607,7 +2607,7 @@ void mlx4_cmd_cleanup(struct mlx4_dev *dev, int cleanup_mask)
 	struct mlx4_priv *priv = mlx4_priv(dev);
 
 	if (priv->cmd.pool && (cleanup_mask & MLX4_CMD_CLEANUP_POOL)) {
-		pci_pool_destroy(priv->cmd.pool);
+		dma_pool_destroy(priv->cmd.pool);
 		priv->cmd.pool = NULL;
 	}
 
@@ -2637,7 +2637,7 @@ int mlx4_cmd_use_events(struct mlx4_dev *dev)
 	int err = 0;
 
 	priv->cmd.context = kmalloc(priv->cmd.max_cmds *
-				   sizeof (struct mlx4_cmd_context),
+				   sizeof(struct mlx4_cmd_context),
 				   GFP_KERNEL);
 	if (!priv->cmd.context)
 		return -ENOMEM;
@@ -2695,11 +2695,11 @@ struct mlx4_cmd_mailbox *mlx4_alloc_cmd_mailbox(struct mlx4_dev *dev)
 {
 	struct mlx4_cmd_mailbox *mailbox;
 
-	mailbox = kmalloc(sizeof *mailbox, GFP_KERNEL);
+	mailbox = kmalloc(sizeof(*mailbox), GFP_KERNEL);
 	if (!mailbox)
 		return ERR_PTR(-ENOMEM);
 
-	mailbox->buf = pci_pool_zalloc(mlx4_priv(dev)->cmd.pool, GFP_KERNEL,
+	mailbox->buf = dma_pool_zalloc(mlx4_priv(dev)->cmd.pool, GFP_KERNEL,
 				       &mailbox->dma);
 	if (!mailbox->buf) {
 		kfree(mailbox);
@@ -2716,7 +2716,7 @@ void mlx4_free_cmd_mailbox(struct mlx4_dev *dev,
 	if (!mailbox)
 		return;
 
-	pci_pool_free(mlx4_priv(dev)->cmd.pool, mailbox->buf, mailbox->dma);
+	dma_pool_free(mlx4_priv(dev)->cmd.pool, mailbox->buf, mailbox->dma);
 	kfree(mailbox);
 }
 EXPORT_SYMBOL_GPL(mlx4_free_cmd_mailbox);
@@ -2891,7 +2891,7 @@ static int mlx4_set_vport_qos(struct mlx4_priv *priv, int slave, int port,
 	memset(vpp_qos, 0, sizeof(struct mlx4_vport_qos_param) * MLX4_NUM_UP);
 
 	if (slave > port_qos->num_of_qos_vfs) {
-		mlx4_info(dev, "No availible VPP resources for this VF\n");
+		mlx4_info(dev, "No available VPP resources for this VF\n");
 		return -EINVAL;
 	}
 
diff --git a/drivers/net/ethernet/mellanox/mlx4/cq.c b/drivers/net/ethernet/mellanox/mlx4/cq.c
index c56a511b918e..72eb50cd5ecd 100644
--- a/drivers/net/ethernet/mellanox/mlx4/cq.c
+++ b/drivers/net/ethernet/mellanox/mlx4/cq.c
@@ -241,13 +241,14 @@ err_out:
 	return err;
 }
 
-static int mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn)
+static int mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn, u8 usage)
 {
+	u32 in_modifier = RES_CQ | (((u32)usage & 3) << 30);
 	u64 out_param;
 	int err;
 
 	if (mlx4_is_mfunc(dev)) {
-		err = mlx4_cmd_imm(dev, 0, &out_param, RES_CQ,
+		err = mlx4_cmd_imm(dev, 0, &out_param, in_modifier,
 				   RES_OP_RESERVE_AND_MAP, MLX4_CMD_ALLOC_RES,
 				   MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
 		if (err)
@@ -303,7 +304,7 @@ int mlx4_cq_alloc(struct mlx4_dev *dev, int nent,
 
 	cq->vector = vector;
 
-	err = mlx4_cq_alloc_icm(dev, &cq->cqn);
+	err = mlx4_cq_alloc_icm(dev, &cq->cqn, cq->usage);
 	if (err)
 		return err;
 
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_cq.c b/drivers/net/ethernet/mellanox/mlx4/en_cq.c
index 85fe17e4dcfb..1e487acb4667 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_cq.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_cq.c
@@ -140,6 +140,7 @@ int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq,
 	    (cq->type == RX && priv->hwtstamp_config.rx_filter))
 		timestamp_en = 1;
 
+	cq->mcq.usage = MLX4_RES_USAGE_DRIVER;
 	err = mlx4_cq_alloc(mdev->dev, cq->size, &cq->wqres.mtt,
 			    &mdev->priv_uar, cq->wqres.db.dma, &cq->mcq,
 			    cq->vector, 0, timestamp_en);
@@ -208,12 +209,10 @@ int mlx4_en_set_cq_moder(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq)
 			      cq->moder_cnt, cq->moder_time);
 }
 
-int mlx4_en_arm_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq)
+void mlx4_en_arm_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq)
 {
 	mlx4_cq_arm(&cq->mcq, MLX4_CQ_DB_REQ_NOT, priv->mdev->uar_map,
 		    &priv->mdev->uar_lock);
-
-	return 0;
 }
 
 
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_main.c b/drivers/net/ethernet/mellanox/mlx4/en_main.c
index 2b0cbca4beb5..686e18de9a97 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_main.c
@@ -147,7 +147,7 @@ void mlx4_en_update_loopback_state(struct net_device *dev,
 	mutex_unlock(&priv->mdev->state_lock);
 }
 
-static int mlx4_en_get_profile(struct mlx4_en_dev *mdev)
+static void mlx4_en_get_profile(struct mlx4_en_dev *mdev)
 {
 	struct mlx4_en_profile *params = &mdev->profile;
 	int i;
@@ -176,8 +176,6 @@ static int mlx4_en_get_profile(struct mlx4_en_dev *mdev)
 		params->prof[i].rss_rings = 0;
 		params->prof[i].inline_thold = inline_thold;
 	}
-
-	return 0;
 }
 
 static void *mlx4_en_get_netdev(struct mlx4_dev *dev, void *ctx, u8 port)
@@ -309,10 +307,7 @@ static void *mlx4_en_add(struct mlx4_dev *dev)
 	}
 
 	/* Build device profile according to supplied module parameters */
-	if (mlx4_en_get_profile(mdev)) {
-		mlx4_err(mdev, "Bad module parameters, aborting\n");
-		goto err_mr;
-	}
+	mlx4_en_get_profile(mdev);
 
 	/* Configure which ports to start according to module parameters */
 	mdev->port_cnt = 0;
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
index 3a291fc1780a..9c218f1cfc6c 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
@@ -130,19 +130,20 @@ out:
 	return err;
 }
 
-static int __mlx4_en_setup_tc(struct net_device *dev, u32 handle,
-			      u32 chain_index, __be16 proto,
-			      struct tc_to_netdev *tc)
+static int __mlx4_en_setup_tc(struct net_device *dev, enum tc_setup_type type,
+			      void *type_data)
 {
-	if (tc->type != TC_SETUP_MQPRIO)
-		return -EINVAL;
+	struct tc_mqprio_qopt *mqprio = type_data;
+
+	if (type != TC_SETUP_MQPRIO)
+		return -EOPNOTSUPP;
 
-	if (tc->mqprio->num_tc && tc->mqprio->num_tc != MLX4_EN_NUM_UP_HIGH)
+	if (mqprio->num_tc && mqprio->num_tc != MLX4_EN_NUM_UP_HIGH)
 		return -EINVAL;
 
-	tc->mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
+	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
 
-	return mlx4_en_alloc_tx_queue_per_tc(dev, tc->mqprio->num_tc);
+	return mlx4_en_alloc_tx_queue_per_tc(dev, mqprio->num_tc);
 }
 
 #ifdef CONFIG_RFS_ACCEL
@@ -651,7 +652,8 @@ static int mlx4_en_get_qp(struct mlx4_en_priv *priv)
 		return 0;
 	}
 
-	err = mlx4_qp_reserve_range(dev, 1, 1, qpn, MLX4_RESERVE_A0_QP);
+	err = mlx4_qp_reserve_range(dev, 1, 1, qpn, MLX4_RESERVE_A0_QP,
+				    MLX4_RES_USAGE_DRIVER);
 	en_dbg(DRV, priv, "Reserved qp %d\n", *qpn);
 	if (err) {
 		en_err(priv, "Failed to reserve qp for mac registration\n");
@@ -731,6 +733,21 @@ static int mlx4_en_replace_mac(struct mlx4_en_priv *priv, int qpn,
 	return __mlx4_replace_mac(dev, priv->port, qpn, new_mac_u64);
 }
 
+static void mlx4_en_update_user_mac(struct mlx4_en_priv *priv,
+				    unsigned char new_mac[ETH_ALEN + 2])
+{
+	struct mlx4_en_dev *mdev = priv->mdev;
+	int err;
+
+	if (!(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_USER_MAC_EN))
+		return;
+
+	err = mlx4_SET_PORT_user_mac(mdev->dev, priv->port, new_mac);
+	if (err)
+		en_err(priv, "Failed to pass user MAC(%pM) to Firmware for port %d, with error %d\n",
+		       new_mac, priv->port, err);
+}
+
 static int mlx4_en_do_set_mac(struct mlx4_en_priv *priv,
 			      unsigned char new_mac[ETH_ALEN + 2])
 {
@@ -765,8 +782,12 @@ static int mlx4_en_set_mac(struct net_device *dev, void *addr)
 	mutex_lock(&mdev->state_lock);
 	memcpy(new_mac, saddr->sa_data, ETH_ALEN);
 	err = mlx4_en_do_set_mac(priv, new_mac);
-	if (!err)
-		memcpy(dev->dev_addr, saddr->sa_data, ETH_ALEN);
+	if (err)
+		goto out;
+
+	memcpy(dev->dev_addr, saddr->sa_data, ETH_ALEN);
+	mlx4_en_update_user_mac(priv, new_mac);
+out:
 	mutex_unlock(&mdev->state_lock);
 
 	return err;
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_resources.c b/drivers/net/ethernet/mellanox/mlx4/en_resources.c
index 86d2d42d658d..5a47f9669621 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_resources.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_resources.c
@@ -44,7 +44,7 @@ void mlx4_en_fill_qp_context(struct mlx4_en_priv *priv, int size, int stride,
 	struct mlx4_en_dev *mdev = priv->mdev;
 	struct net_device *dev = priv->dev;
 
-	memset(context, 0, sizeof *context);
+	memset(context, 0, sizeof(*context));
 	context->flags = cpu_to_be32(7 << 16 | rss << MLX4_RSS_QPC_FLAG_OFFSET);
 	context->pd = cpu_to_be32(mdev->priv_pdn);
 	context->mtu_msgmax = 0xff;
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
index bf1638044a7a..b97a55c827eb 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
@@ -1056,7 +1056,7 @@ static int mlx4_en_config_rss_qp(struct mlx4_en_priv *priv, int qpn,
 	}
 	qp->event = mlx4_en_sqp_event;
 
-	memset(context, 0, sizeof *context);
+	memset(context, 0, sizeof(*context));
 	mlx4_en_fill_qp_context(priv, ring->actual_size, ring->stride, 0, 0,
 				qpn, ring->cqn, -1, context);
 	context->db_rec_addr = cpu_to_be64(ring->wqres.db.dma);
@@ -1088,7 +1088,8 @@ int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv)
 	u32 qpn;
 
 	err = mlx4_qp_reserve_range(priv->mdev->dev, 1, 1, &qpn,
-				    MLX4_RESERVE_A0_QP);
+				    MLX4_RESERVE_A0_QP,
+				    MLX4_RES_USAGE_DRIVER);
 	if (err) {
 		en_err(priv, "Failed reserving drop qpn\n");
 		return err;
@@ -1134,7 +1135,8 @@ int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv)
 	flags = priv->rx_ring_num == 1 ? MLX4_RESERVE_A0_QP : 0;
 	err = mlx4_qp_reserve_range(mdev->dev, priv->rx_ring_num,
 				    priv->rx_ring_num,
-				    &rss_map->base_qpn, flags);
+				    &rss_map->base_qpn, flags,
+				    MLX4_RES_USAGE_DRIVER);
 	if (err) {
 		en_err(priv, "Failed reserving %d qps\n", priv->rx_ring_num);
 		return err;
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_tx.c b/drivers/net/ethernet/mellanox/mlx4/en_tx.c
index 73faa3d77921..8a32a8f7f9c0 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_tx.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_tx.c
@@ -105,7 +105,8 @@ int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
 	       (unsigned long long) ring->sp_wqres.buf.direct.map);
 
 	err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &ring->qpn,
-				    MLX4_RESERVE_ETH_BF_QP);
+				    MLX4_RESERVE_ETH_BF_QP,
+				    MLX4_RES_USAGE_DRIVER);
 	if (err) {
 		en_err(priv, "failed reserving qp for TX ring\n");
 		goto err_hwq_res;
@@ -643,7 +644,7 @@ static void build_inline_wqe(struct mlx4_en_tx_desc *tx_desc,
 			     void *fragptr)
 {
 	struct mlx4_wqe_inline_seg *inl = &tx_desc->inl;
-	int spc = MLX4_INLINE_ALIGN - CTRL_SIZE - sizeof *inl;
+	int spc = MLX4_INLINE_ALIGN - CTRL_SIZE - sizeof(*inl);
 	unsigned int hlen = skb_headlen(skb);
 
 	if (skb->len <= spc) {
diff --git a/drivers/net/ethernet/mellanox/mlx4/eq.c b/drivers/net/ethernet/mellanox/mlx4/eq.c
index 07406cf2eacd..6f57c052053e 100644
--- a/drivers/net/ethernet/mellanox/mlx4/eq.c
+++ b/drivers/net/ethernet/mellanox/mlx4/eq.c
@@ -259,7 +259,7 @@ int mlx4_gen_pkey_eqe(struct mlx4_dev *dev, int slave, u8 port)
 	if (!s_slave->active)
 		return 0;
 
-	memset(&eqe, 0, sizeof eqe);
+	memset(&eqe, 0, sizeof(eqe));
 
 	eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT;
 	eqe.subtype = MLX4_DEV_PMC_SUBTYPE_PKEY_TABLE;
@@ -276,7 +276,7 @@ int mlx4_gen_guid_change_eqe(struct mlx4_dev *dev, int slave, u8 port)
 	/*don't send if we don't have the that slave */
 	if (dev->persist->num_vfs < slave)
 		return 0;
-	memset(&eqe, 0, sizeof eqe);
+	memset(&eqe, 0, sizeof(eqe));
 
 	eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT;
 	eqe.subtype = MLX4_DEV_PMC_SUBTYPE_GUID_INFO;
@@ -295,7 +295,7 @@ int mlx4_gen_port_state_change_eqe(struct mlx4_dev *dev, int slave, u8 port,
 	/*don't send if we don't have the that slave */
 	if (dev->persist->num_vfs < slave)
 		return 0;
-	memset(&eqe, 0, sizeof eqe);
+	memset(&eqe, 0, sizeof(eqe));
 
 	eqe.type = MLX4_EVENT_TYPE_PORT_CHANGE;
 	eqe.subtype = port_subtype_change;
@@ -432,7 +432,7 @@ int mlx4_gen_slaves_port_mgt_ev(struct mlx4_dev *dev, u8 port, int attr)
 {
 	struct mlx4_eqe eqe;
 
-	memset(&eqe, 0, sizeof eqe);
+	memset(&eqe, 0, sizeof(eqe));
 
 	eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT;
 	eqe.subtype = MLX4_DEV_PMC_SUBTYPE_PORT_INFO;
@@ -726,7 +726,7 @@ static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq)
 			}
 			memcpy(&priv->mfunc.master.comm_arm_bit_vector,
 			       eqe->event.comm_channel_arm.bit_vec,
-			       sizeof eqe->event.comm_channel_arm.bit_vec);
+			       sizeof(eqe->event.comm_channel_arm.bit_vec));
 			queue_work(priv->mfunc.master.comm_wq,
 				   &priv->mfunc.master.comm_work);
 			break;
@@ -984,15 +984,15 @@ static int mlx4_create_eq(struct mlx4_dev *dev, int nent,
 	 */
 	npages = PAGE_ALIGN(eq->nent * dev->caps.eqe_size) / PAGE_SIZE;
 
-	eq->page_list = kmalloc(npages * sizeof *eq->page_list,
-				GFP_KERNEL);
+	eq->page_list = kmalloc_array(npages, sizeof(*eq->page_list),
+				      GFP_KERNEL);
 	if (!eq->page_list)
 		goto err_out;
 
 	for (i = 0; i < npages; ++i)
 		eq->page_list[i].buf = NULL;
 
-	dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL);
+	dma_list = kmalloc_array(npages, sizeof(*dma_list), GFP_KERNEL);
 	if (!dma_list)
 		goto err_out_free;
 
@@ -1161,7 +1161,7 @@ int mlx4_alloc_eq_table(struct mlx4_dev *dev)
 	struct mlx4_priv *priv = mlx4_priv(dev);
 
 	priv->eq_table.eq = kcalloc(dev->caps.num_eqs - dev->caps.reserved_eqs,
-				    sizeof *priv->eq_table.eq, GFP_KERNEL);
+				    sizeof(*priv->eq_table.eq), GFP_KERNEL);
 	if (!priv->eq_table.eq)
 		return -ENOMEM;
 
@@ -1180,7 +1180,7 @@ int mlx4_init_eq_table(struct mlx4_dev *dev)
 	int i;
 
 	priv->eq_table.uar_map = kcalloc(mlx4_num_eq_uar(dev),
-					 sizeof *priv->eq_table.uar_map,
+					 sizeof(*priv->eq_table.uar_map),
 					 GFP_KERNEL);
 	if (!priv->eq_table.uar_map) {
 		err = -ENOMEM;
diff --git a/drivers/net/ethernet/mellanox/mlx4/fw.c b/drivers/net/ethernet/mellanox/mlx4/fw.c
index 041c0ed65929..16c09949afd5 100644
--- a/drivers/net/ethernet/mellanox/mlx4/fw.c
+++ b/drivers/net/ethernet/mellanox/mlx4/fw.c
@@ -36,6 +36,7 @@
 #include <linux/mlx4/cmd.h>
 #include <linux/module.h>
 #include <linux/cache.h>
+#include <linux/kernel.h>
 
 #include "fw.h"
 #include "icm.h"
@@ -57,7 +58,7 @@ MODULE_PARM_DESC(enable_qos, "Enable Enhanced QoS support (default: off)");
 	do {							      \
 		void *__p = (char *) (source) + (offset);	      \
 		u64 val;                                              \
-		switch (sizeof (dest)) {			      \
+		switch (sizeof(dest)) {			      \
 		case 1: (dest) = *(u8 *) __p;	    break;	      \
 		case 2: (dest) = be16_to_cpup(__p); break;	      \
 		case 4: (dest) = be32_to_cpup(__p); break;	      \
@@ -162,6 +163,7 @@ static void dump_dev_cap_flags2(struct mlx4_dev *dev, u64 flags)
 		[35] = "Diag counters per port",
 		[36] = "QinQ VST mode support",
 		[37] = "sl to vl mapping table change event support",
+		[38] = "user MAC support",
 	};
 	int i;
 
@@ -679,22 +681,22 @@ int mlx4_QUERY_FUNC_CAP(struct mlx4_dev *dev, u8 gen_or_port,
 
 	if (func_cap->flags1 & QUERY_FUNC_CAP_VF_ENABLE_QP0) {
 		MLX4_GET(qkey, outbox, QUERY_FUNC_CAP_PRIV_VF_QKEY_OFFSET);
-		func_cap->qp0_qkey = qkey;
+		func_cap->spec_qps.qp0_qkey = qkey;
 	} else {
-		func_cap->qp0_qkey = 0;
+		func_cap->spec_qps.qp0_qkey = 0;
 	}
 
 	MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP0_TUNNEL);
-	func_cap->qp0_tunnel_qpn = size & 0xFFFFFF;
+	func_cap->spec_qps.qp0_tunnel = size & 0xFFFFFF;
 
 	MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP0_PROXY);
-	func_cap->qp0_proxy_qpn = size & 0xFFFFFF;
+	func_cap->spec_qps.qp0_proxy = size & 0xFFFFFF;
 
 	MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP1_TUNNEL);
-	func_cap->qp1_tunnel_qpn = size & 0xFFFFFF;
+	func_cap->spec_qps.qp1_tunnel = size & 0xFFFFFF;
 
 	MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP1_PROXY);
-	func_cap->qp1_proxy_qpn = size & 0xFFFFFF;
+	func_cap->spec_qps.qp1_proxy = size & 0xFFFFFF;
 
 	if (func_cap->flags1 & QUERY_FUNC_CAP_FLAGS1_NIC_INFO)
 		MLX4_GET(func_cap->phys_port_id, outbox,
@@ -778,6 +780,7 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
 #define QUERY_DEV_CAP_MAX_DESC_SZ_SQ_OFFSET	0x52
 #define QUERY_DEV_CAP_MAX_SG_RQ_OFFSET		0x55
 #define QUERY_DEV_CAP_MAX_DESC_SZ_RQ_OFFSET	0x56
+#define QUERY_DEV_CAP_USER_MAC_EN_OFFSET	0x5C
 #define QUERY_DEV_CAP_SVLAN_BY_QP_OFFSET	0x5D
 #define QUERY_DEV_CAP_MAX_QP_MCG_OFFSET		0x61
 #define QUERY_DEV_CAP_RSVD_MCG_OFFSET		0x62
@@ -949,6 +952,9 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
 	MLX4_GET(size, outbox, QUERY_DEV_CAP_MAX_DESC_SZ_SQ_OFFSET);
 	dev_cap->max_sq_desc_sz = size;
 
+	MLX4_GET(field, outbox, QUERY_DEV_CAP_USER_MAC_EN_OFFSET);
+	if (field & (1 << 2))
+		dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_USER_MAC_EN;
 	MLX4_GET(field, outbox, QUERY_DEV_CAP_SVLAN_BY_QP_OFFSET);
 	if (field & 0x1)
 		dev_cap->flags2 |= MLX4_DEV_CAP_FLAG2_SVLAN_BY_QP;
@@ -1534,7 +1540,7 @@ int mlx4_map_cmd(struct mlx4_dev *dev, u16 op, struct mlx4_icm *icm, u64 virt)
 		for (i = 0; i < mlx4_icm_size(&iter) >> lg; ++i) {
 			if (virt != -1) {
 				pages[nent * 2] = cpu_to_be64(virt);
-				virt += 1 << lg;
+				virt += 1ULL << lg;
 			}
 
 			pages[nent * 2 + 1] =
@@ -2450,14 +2456,14 @@ int mlx4_config_dev_retrieval(struct mlx4_dev *dev,
 	csum_mask = (config_dev.rx_checksum_val >> CONFIG_DEV_RX_CSUM_MODE_PORT1_BIT_OFFSET) &
 			CONFIG_DEV_RX_CSUM_MODE_MASK;
 
-	if (csum_mask >= sizeof(config_dev_csum_flags)/sizeof(config_dev_csum_flags[0]))
+	if (csum_mask >= ARRAY_SIZE(config_dev_csum_flags))
 		return -EINVAL;
 	params->rx_csum_flags_port_1 = config_dev_csum_flags[csum_mask];
 
 	csum_mask = (config_dev.rx_checksum_val >> CONFIG_DEV_RX_CSUM_MODE_PORT2_BIT_OFFSET) &
 			CONFIG_DEV_RX_CSUM_MODE_MASK;
 
-	if (csum_mask >= sizeof(config_dev_csum_flags)/sizeof(config_dev_csum_flags[0]))
+	if (csum_mask >= ARRAY_SIZE(config_dev_csum_flags))
 		return -EINVAL;
 	params->rx_csum_flags_port_2 = config_dev_csum_flags[csum_mask];
 
diff --git a/drivers/net/ethernet/mellanox/mlx4/fw.h b/drivers/net/ethernet/mellanox/mlx4/fw.h
index b52ba01aa486..cd6399c76bfd 100644
--- a/drivers/net/ethernet/mellanox/mlx4/fw.h
+++ b/drivers/net/ethernet/mellanox/mlx4/fw.h
@@ -144,11 +144,7 @@ struct mlx4_func_cap {
 	int	max_eq;
 	int	reserved_eq;
 	int	mcg_quota;
-	u32	qp0_qkey;
-	u32	qp0_tunnel_qpn;
-	u32	qp0_proxy_qpn;
-	u32	qp1_tunnel_qpn;
-	u32	qp1_proxy_qpn;
+	struct mlx4_spec_qps spec_qps;
 	u32	reserved_lkey;
 	u8	physical_port;
 	u8	flags0;
diff --git a/drivers/net/ethernet/mellanox/mlx4/fw_qos.c b/drivers/net/ethernet/mellanox/mlx4/fw_qos.c
index 8f2fde0487c4..3a09d7122d3b 100644
--- a/drivers/net/ethernet/mellanox/mlx4/fw_qos.c
+++ b/drivers/net/ethernet/mellanox/mlx4/fw_qos.c
@@ -65,7 +65,7 @@ struct mlx4_set_port_scheduler_context {
 
 /* Granular Qos (per VF) section */
 struct mlx4_alloc_vpp_param {
-	__be32 availible_vpp;
+	__be32 available_vpp;
 	__be32 vpp_p_up[MLX4_NUM_UP];
 };
 
@@ -157,7 +157,7 @@ int mlx4_SET_PORT_SCHEDULER(struct mlx4_dev *dev, u8 port, u8 *tc_tx_bw,
 EXPORT_SYMBOL(mlx4_SET_PORT_SCHEDULER);
 
 int mlx4_ALLOCATE_VPP_get(struct mlx4_dev *dev, u8 port,
-			  u16 *availible_vpp, u8 *vpp_p_up)
+			  u16 *available_vpp, u8 *vpp_p_up)
 {
 	int i;
 	int err;
@@ -179,7 +179,7 @@ int mlx4_ALLOCATE_VPP_get(struct mlx4_dev *dev, u8 port,
 		goto out;
 
 	/* Total number of supported VPPs */
-	*availible_vpp = (u16)be32_to_cpu(out_param->availible_vpp);
+	*available_vpp = (u16)be32_to_cpu(out_param->available_vpp);
 
 	for (i = 0; i < MLX4_NUM_UP; i++)
 		vpp_p_up[i] = (u8)be32_to_cpu(out_param->vpp_p_up[i]);
diff --git a/drivers/net/ethernet/mellanox/mlx4/fw_qos.h b/drivers/net/ethernet/mellanox/mlx4/fw_qos.h
index ac1f331878e6..582997577a04 100644
--- a/drivers/net/ethernet/mellanox/mlx4/fw_qos.h
+++ b/drivers/net/ethernet/mellanox/mlx4/fw_qos.h
@@ -84,23 +84,23 @@ int mlx4_SET_PORT_PRIO2TC(struct mlx4_dev *dev, u8 port, u8 *prio2tc);
 int mlx4_SET_PORT_SCHEDULER(struct mlx4_dev *dev, u8 port, u8 *tc_tx_bw,
 			    u8 *pg, u16 *ratelimit);
 /**
- * mlx4_ALLOCATE_VPP_get - Query port VPP availible resources and allocation.
- * Before distribution of VPPs to priorities, only availible_vpp is returned.
+ * mlx4_ALLOCATE_VPP_get - Query port VPP available resources and allocation.
+ * Before distribution of VPPs to priorities, only available_vpp is returned.
  * After initialization it returns the distribution of VPPs among priorities.
  *
  * @dev: mlx4_dev.
  * @port: Physical port number.
- * @availible_vpp: Pointer to variable where number of availible VPPs is stored
+ * @available_vpp: Pointer to variable where number of available VPPs is stored
  * @vpp_p_up: Distribution of VPPs to priorities is stored in this array
  *
  * Returns 0 on success or a negative mlx4_core errno code.
  **/
 int mlx4_ALLOCATE_VPP_get(struct mlx4_dev *dev, u8 port,
-			  u16 *availible_vpp, u8 *vpp_p_up);
+			  u16 *available_vpp, u8 *vpp_p_up);
 /**
  * mlx4_ALLOCATE_VPP_set - Distribution of VPPs among differnt priorities.
  * The total number of VPPs assigned to all for a port must not exceed
- * the value reported by availible_vpp in mlx4_ALLOCATE_VPP_get.
+ * the value reported by available_vpp in mlx4_ALLOCATE_VPP_get.
  * VPP allocation is allowed only after the port type has been set,
  * and while no QPs are open for this port.
  *
diff --git a/drivers/net/ethernet/mellanox/mlx4/icm.c b/drivers/net/ethernet/mellanox/mlx4/icm.c
index 5a7816e7c7b4..a822f7a56bc5 100644
--- a/drivers/net/ethernet/mellanox/mlx4/icm.c
+++ b/drivers/net/ethernet/mellanox/mlx4/icm.c
@@ -400,7 +400,7 @@ int mlx4_init_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table,
 	obj_per_chunk = MLX4_TABLE_CHUNK_SIZE / obj_size;
 	num_icm = (nobj + obj_per_chunk - 1) / obj_per_chunk;
 
-	table->icm      = kcalloc(num_icm, sizeof *table->icm, GFP_KERNEL);
+	table->icm      = kcalloc(num_icm, sizeof(*table->icm), GFP_KERNEL);
 	if (!table->icm)
 		return -ENOMEM;
 	table->virt     = virt;
diff --git a/drivers/net/ethernet/mellanox/mlx4/icm.h b/drivers/net/ethernet/mellanox/mlx4/icm.h
index dee67fa39107..c9169a490557 100644
--- a/drivers/net/ethernet/mellanox/mlx4/icm.h
+++ b/drivers/net/ethernet/mellanox/mlx4/icm.h
@@ -39,8 +39,8 @@
 #include <linux/mutex.h>
 
 #define MLX4_ICM_CHUNK_LEN						\
-	((256 - sizeof (struct list_head) - 2 * sizeof (int)) /		\
-	 (sizeof (struct scatterlist)))
+	((256 - sizeof(struct list_head) - 2 * sizeof(int)) /		\
+	 (sizeof(struct scatterlist)))
 
 enum {
 	MLX4_ICM_PAGE_SHIFT	= 12,
diff --git a/drivers/net/ethernet/mellanox/mlx4/intf.c b/drivers/net/ethernet/mellanox/mlx4/intf.c
index e00f627331cb..2edcce98ab2d 100644
--- a/drivers/net/ethernet/mellanox/mlx4/intf.c
+++ b/drivers/net/ethernet/mellanox/mlx4/intf.c
@@ -53,7 +53,7 @@ static void mlx4_add_device(struct mlx4_interface *intf, struct mlx4_priv *priv)
 {
 	struct mlx4_device_context *dev_ctx;
 
-	dev_ctx = kmalloc(sizeof *dev_ctx, GFP_KERNEL);
+	dev_ctx = kmalloc(sizeof(*dev_ctx), GFP_KERNEL);
 	if (!dev_ctx)
 		return;
 
diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c
index 09b9bc17bce9..e61c99ef741d 100644
--- a/drivers/net/ethernet/mellanox/mlx4/main.c
+++ b/drivers/net/ethernet/mellanox/mlx4/main.c
@@ -34,6 +34,7 @@
  */
 
 #include <linux/module.h>
+#include <linux/kernel.h>
 #include <linux/init.h>
 #include <linux/errno.h>
 #include <linux/pci.h>
@@ -121,7 +122,7 @@ static char mlx4_version[] =
 	DRV_NAME ": Mellanox ConnectX core driver v"
 	DRV_VERSION "\n";
 
-static struct mlx4_profile default_profile = {
+static const struct mlx4_profile default_profile = {
 	.num_qp		= 1 << 18,
 	.num_srq	= 1 << 16,
 	.rdmarc_per_qp	= 1 << 4,
@@ -131,7 +132,7 @@ static struct mlx4_profile default_profile = {
 	.num_mtt	= 1 << 20, /* It is really num mtt segements */
 };
 
-static struct mlx4_profile low_mem_profile = {
+static const struct mlx4_profile low_mem_profile = {
 	.num_qp		= 1 << 17,
 	.num_srq	= 1 << 6,
 	.rdmarc_per_qp	= 1 << 4,
@@ -432,7 +433,7 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
 		/* Virtual PCI function needs to determine UAR page size from
 		 * firmware. Only master PCI function can set the uar page size
 		 */
-		if (enable_4k_uar)
+		if (enable_4k_uar || !dev->persist->num_vfs)
 			dev->uar_page_shift = DEFAULT_UAR_PAGE_SHIFT;
 		else
 			dev->uar_page_shift = PAGE_SHIFT;
@@ -819,38 +820,93 @@ static void slave_adjust_steering_mode(struct mlx4_dev *dev,
 		 mlx4_steering_mode_str(dev->caps.steering_mode));
 }
 
+static void mlx4_slave_destroy_special_qp_cap(struct mlx4_dev *dev)
+{
+	kfree(dev->caps.spec_qps);
+	dev->caps.spec_qps = NULL;
+}
+
+static int mlx4_slave_special_qp_cap(struct mlx4_dev *dev)
+{
+	struct mlx4_func_cap *func_cap = NULL;
+	struct mlx4_caps *caps = &dev->caps;
+	int i, err = 0;
+
+	func_cap = kzalloc(sizeof(*func_cap), GFP_KERNEL);
+	caps->spec_qps = kcalloc(caps->num_ports, sizeof(*caps->spec_qps), GFP_KERNEL);
+
+	if (!func_cap || !caps->spec_qps) {
+		mlx4_err(dev, "Failed to allocate memory for special qps cap\n");
+		err = -ENOMEM;
+		goto err_mem;
+	}
+
+	for (i = 1; i <= caps->num_ports; ++i) {
+		err = mlx4_QUERY_FUNC_CAP(dev, i, func_cap);
+		if (err) {
+			mlx4_err(dev, "QUERY_FUNC_CAP port command failed for port %d, aborting (%d)\n",
+				 i, err);
+			goto err_mem;
+		}
+		caps->spec_qps[i - 1] = func_cap->spec_qps;
+		caps->port_mask[i] = caps->port_type[i];
+		caps->phys_port_id[i] = func_cap->phys_port_id;
+		err = mlx4_get_slave_pkey_gid_tbl_len(dev, i,
+						      &caps->gid_table_len[i],
+						      &caps->pkey_table_len[i]);
+		if (err) {
+			mlx4_err(dev, "QUERY_PORT command failed for port %d, aborting (%d)\n",
+				 i, err);
+			goto err_mem;
+		}
+	}
+
+err_mem:
+	if (err)
+		mlx4_slave_destroy_special_qp_cap(dev);
+	kfree(func_cap);
+	return err;
+}
+
 static int mlx4_slave_cap(struct mlx4_dev *dev)
 {
 	int			   err;
 	u32			   page_size;
-	struct mlx4_dev_cap	   dev_cap;
-	struct mlx4_func_cap	   func_cap;
-	struct mlx4_init_hca_param hca_param;
-	u8			   i;
+	struct mlx4_dev_cap	   *dev_cap = NULL;
+	struct mlx4_func_cap	   *func_cap = NULL;
+	struct mlx4_init_hca_param *hca_param = NULL;
+
+	hca_param = kzalloc(sizeof(*hca_param), GFP_KERNEL);
+	func_cap = kzalloc(sizeof(*func_cap), GFP_KERNEL);
+	dev_cap = kzalloc(sizeof(*dev_cap), GFP_KERNEL);
+	if (!hca_param || !func_cap || !dev_cap) {
+		mlx4_err(dev, "Failed to allocate memory for slave_cap\n");
+		err = -ENOMEM;
+		goto free_mem;
+	}
 
-	memset(&hca_param, 0, sizeof(hca_param));
-	err = mlx4_QUERY_HCA(dev, &hca_param);
+	err = mlx4_QUERY_HCA(dev, hca_param);
 	if (err) {
 		mlx4_err(dev, "QUERY_HCA command failed, aborting\n");
-		return err;
+		goto free_mem;
 	}
 
 	/* fail if the hca has an unknown global capability
 	 * at this time global_caps should be always zeroed
 	 */
-	if (hca_param.global_caps) {
+	if (hca_param->global_caps) {
 		mlx4_err(dev, "Unknown hca global capabilities\n");
-		return -EINVAL;
+		err = -EINVAL;
+		goto free_mem;
 	}
 
-	dev->caps.hca_core_clock = hca_param.hca_core_clock;
+	dev->caps.hca_core_clock = hca_param->hca_core_clock;
 
-	memset(&dev_cap, 0, sizeof(dev_cap));
-	dev->caps.max_qp_dest_rdma = 1 << hca_param.log_rd_per_qp;
-	err = mlx4_dev_cap(dev, &dev_cap);
+	dev->caps.max_qp_dest_rdma = 1 << hca_param->log_rd_per_qp;
+	err = mlx4_dev_cap(dev, dev_cap);
 	if (err) {
 		mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting\n");
-		return err;
+		goto free_mem;
 	}
 
 	err = mlx4_QUERY_FW(dev);
@@ -862,21 +918,23 @@ static int mlx4_slave_cap(struct mlx4_dev *dev)
 	if (page_size > PAGE_SIZE) {
 		mlx4_err(dev, "HCA minimum page size of %d bigger than kernel PAGE_SIZE of %ld, aborting\n",
 			 page_size, PAGE_SIZE);
-		return -ENODEV;
+		err = -ENODEV;
+		goto free_mem;
 	}
 
 	/* Set uar_page_shift for VF */
-	dev->uar_page_shift = hca_param.uar_page_sz + 12;
+	dev->uar_page_shift = hca_param->uar_page_sz + 12;
 
 	/* Make sure the master uar page size is valid */
 	if (dev->uar_page_shift > PAGE_SHIFT) {
 		mlx4_err(dev,
 			 "Invalid configuration: uar page size is larger than system page size\n");
-		return  -ENODEV;
+		err = -ENODEV;
+		goto free_mem;
 	}
 
 	/* Set reserved_uars based on the uar_page_shift */
-	mlx4_set_num_reserved_uars(dev, &dev_cap);
+	mlx4_set_num_reserved_uars(dev, dev_cap);
 
 	/* Although uar page size in FW differs from system page size,
 	 * upper software layers (mlx4_ib, mlx4_en and part of mlx4_core)
@@ -884,34 +942,35 @@ static int mlx4_slave_cap(struct mlx4_dev *dev)
 	 */
 	dev->caps.uar_page_size = PAGE_SIZE;
 
-	memset(&func_cap, 0, sizeof(func_cap));
-	err = mlx4_QUERY_FUNC_CAP(dev, 0, &func_cap);
+	err = mlx4_QUERY_FUNC_CAP(dev, 0, func_cap);
 	if (err) {
 		mlx4_err(dev, "QUERY_FUNC_CAP general command failed, aborting (%d)\n",
 			 err);
-		return err;
+		goto free_mem;
 	}
 
-	if ((func_cap.pf_context_behaviour | PF_CONTEXT_BEHAVIOUR_MASK) !=
+	if ((func_cap->pf_context_behaviour | PF_CONTEXT_BEHAVIOUR_MASK) !=
 	    PF_CONTEXT_BEHAVIOUR_MASK) {
 		mlx4_err(dev, "Unknown pf context behaviour %x known flags %x\n",
-			 func_cap.pf_context_behaviour, PF_CONTEXT_BEHAVIOUR_MASK);
-		return -EINVAL;
-	}
-
-	dev->caps.num_ports		= func_cap.num_ports;
-	dev->quotas.qp			= func_cap.qp_quota;
-	dev->quotas.srq			= func_cap.srq_quota;
-	dev->quotas.cq			= func_cap.cq_quota;
-	dev->quotas.mpt			= func_cap.mpt_quota;
-	dev->quotas.mtt			= func_cap.mtt_quota;
-	dev->caps.num_qps		= 1 << hca_param.log_num_qps;
-	dev->caps.num_srqs		= 1 << hca_param.log_num_srqs;
-	dev->caps.num_cqs		= 1 << hca_param.log_num_cqs;
-	dev->caps.num_mpts		= 1 << hca_param.log_mpt_sz;
-	dev->caps.num_eqs		= func_cap.max_eq;
-	dev->caps.reserved_eqs		= func_cap.reserved_eq;
-	dev->caps.reserved_lkey		= func_cap.reserved_lkey;
+			 func_cap->pf_context_behaviour,
+			 PF_CONTEXT_BEHAVIOUR_MASK);
+		err = -EINVAL;
+		goto free_mem;
+	}
+
+	dev->caps.num_ports		= func_cap->num_ports;
+	dev->quotas.qp			= func_cap->qp_quota;
+	dev->quotas.srq			= func_cap->srq_quota;
+	dev->quotas.cq			= func_cap->cq_quota;
+	dev->quotas.mpt			= func_cap->mpt_quota;
+	dev->quotas.mtt			= func_cap->mtt_quota;
+	dev->caps.num_qps		= 1 << hca_param->log_num_qps;
+	dev->caps.num_srqs		= 1 << hca_param->log_num_srqs;
+	dev->caps.num_cqs		= 1 << hca_param->log_num_cqs;
+	dev->caps.num_mpts		= 1 << hca_param->log_mpt_sz;
+	dev->caps.num_eqs		= func_cap->max_eq;
+	dev->caps.reserved_eqs		= func_cap->reserved_eq;
+	dev->caps.reserved_lkey		= func_cap->reserved_lkey;
 	dev->caps.num_pds               = MLX4_NUM_PDS;
 	dev->caps.num_mgms              = 0;
 	dev->caps.num_amgms             = 0;
@@ -919,43 +978,16 @@ static int mlx4_slave_cap(struct mlx4_dev *dev)
 	if (dev->caps.num_ports > MLX4_MAX_PORTS) {
 		mlx4_err(dev, "HCA has %d ports, but we only support %d, aborting\n",
 			 dev->caps.num_ports, MLX4_MAX_PORTS);
-		return -ENODEV;
+		err = -ENODEV;
+		goto free_mem;
 	}
 
 	mlx4_replace_zero_macs(dev);
 
-	dev->caps.qp0_qkey = kcalloc(dev->caps.num_ports, sizeof(u32), GFP_KERNEL);
-	dev->caps.qp0_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);
-	dev->caps.qp0_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);
-	dev->caps.qp1_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);
-	dev->caps.qp1_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);
-
-	if (!dev->caps.qp0_tunnel || !dev->caps.qp0_proxy ||
-	    !dev->caps.qp1_tunnel || !dev->caps.qp1_proxy ||
-	    !dev->caps.qp0_qkey) {
-		err = -ENOMEM;
-		goto err_mem;
-	}
-
-	for (i = 1; i <= dev->caps.num_ports; ++i) {
-		err = mlx4_QUERY_FUNC_CAP(dev, i, &func_cap);
-		if (err) {
-			mlx4_err(dev, "QUERY_FUNC_CAP port command failed for port %d, aborting (%d)\n",
-				 i, err);
-			goto err_mem;
-		}
-		dev->caps.qp0_qkey[i - 1] = func_cap.qp0_qkey;
-		dev->caps.qp0_tunnel[i - 1] = func_cap.qp0_tunnel_qpn;
-		dev->caps.qp0_proxy[i - 1] = func_cap.qp0_proxy_qpn;
-		dev->caps.qp1_tunnel[i - 1] = func_cap.qp1_tunnel_qpn;
-		dev->caps.qp1_proxy[i - 1] = func_cap.qp1_proxy_qpn;
-		dev->caps.port_mask[i] = dev->caps.port_type[i];
-		dev->caps.phys_port_id[i] = func_cap.phys_port_id;
-		err = mlx4_get_slave_pkey_gid_tbl_len(dev, i,
-						      &dev->caps.gid_table_len[i],
-						      &dev->caps.pkey_table_len[i]);
-		if (err)
-			goto err_mem;
+	err = mlx4_slave_special_qp_cap(dev);
+	if (err) {
+		mlx4_err(dev, "Set special QP caps failed. aborting\n");
+		goto free_mem;
 	}
 
 	if (dev->caps.uar_page_size * (dev->caps.num_uars -
@@ -970,7 +1002,7 @@ static int mlx4_slave_cap(struct mlx4_dev *dev)
 		goto err_mem;
 	}
 
-	if (hca_param.dev_cap_enabled & MLX4_DEV_CAP_64B_EQE_ENABLED) {
+	if (hca_param->dev_cap_enabled & MLX4_DEV_CAP_64B_EQE_ENABLED) {
 		dev->caps.eqe_size   = 64;
 		dev->caps.eqe_factor = 1;
 	} else {
@@ -978,20 +1010,20 @@ static int mlx4_slave_cap(struct mlx4_dev *dev)
 		dev->caps.eqe_factor = 0;
 	}
 
-	if (hca_param.dev_cap_enabled & MLX4_DEV_CAP_64B_CQE_ENABLED) {
+	if (hca_param->dev_cap_enabled & MLX4_DEV_CAP_64B_CQE_ENABLED) {
 		dev->caps.cqe_size   = 64;
 		dev->caps.userspace_caps |= MLX4_USER_DEV_CAP_LARGE_CQE;
 	} else {
 		dev->caps.cqe_size   = 32;
 	}
 
-	if (hca_param.dev_cap_enabled & MLX4_DEV_CAP_EQE_STRIDE_ENABLED) {
-		dev->caps.eqe_size = hca_param.eqe_size;
+	if (hca_param->dev_cap_enabled & MLX4_DEV_CAP_EQE_STRIDE_ENABLED) {
+		dev->caps.eqe_size = hca_param->eqe_size;
 		dev->caps.eqe_factor = 0;
 	}
 
-	if (hca_param.dev_cap_enabled & MLX4_DEV_CAP_CQE_STRIDE_ENABLED) {
-		dev->caps.cqe_size = hca_param.cqe_size;
+	if (hca_param->dev_cap_enabled & MLX4_DEV_CAP_CQE_STRIDE_ENABLED) {
+		dev->caps.cqe_size = hca_param->cqe_size;
 		/* User still need to know when CQE > 32B */
 		dev->caps.userspace_caps |= MLX4_USER_DEV_CAP_LARGE_CQE;
 	}
@@ -999,31 +1031,27 @@ static int mlx4_slave_cap(struct mlx4_dev *dev)
 	dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS;
 	mlx4_warn(dev, "Timestamping is not supported in slave mode\n");
 
-	slave_adjust_steering_mode(dev, &dev_cap, &hca_param);
+	dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_USER_MAC_EN;
+	mlx4_dbg(dev, "User MAC FW update is not supported in slave mode\n");
+
+	slave_adjust_steering_mode(dev, dev_cap, hca_param);
 	mlx4_dbg(dev, "RSS support for IP fragments is %s\n",
-		 hca_param.rss_ip_frags ? "on" : "off");
+		 hca_param->rss_ip_frags ? "on" : "off");
 
-	if (func_cap.extra_flags & MLX4_QUERY_FUNC_FLAGS_BF_RES_QP &&
+	if (func_cap->extra_flags & MLX4_QUERY_FUNC_FLAGS_BF_RES_QP &&
 	    dev->caps.bf_reg_size)
 		dev->caps.alloc_res_qp_mask |= MLX4_RESERVE_ETH_BF_QP;
 
-	if (func_cap.extra_flags & MLX4_QUERY_FUNC_FLAGS_A0_RES_QP)
+	if (func_cap->extra_flags & MLX4_QUERY_FUNC_FLAGS_A0_RES_QP)
 		dev->caps.alloc_res_qp_mask |= MLX4_RESERVE_A0_QP;
 
-	return 0;
-
 err_mem:
-	kfree(dev->caps.qp0_qkey);
-	kfree(dev->caps.qp0_tunnel);
-	kfree(dev->caps.qp0_proxy);
-	kfree(dev->caps.qp1_tunnel);
-	kfree(dev->caps.qp1_proxy);
-	dev->caps.qp0_qkey = NULL;
-	dev->caps.qp0_tunnel = NULL;
-	dev->caps.qp0_proxy = NULL;
-	dev->caps.qp1_tunnel = NULL;
-	dev->caps.qp1_proxy = NULL;
-
+	if (err)
+		mlx4_slave_destroy_special_qp_cap(dev);
+free_mem:
+	kfree(hca_param);
+	kfree(func_cap);
+	kfree(dev_cap);
 	return err;
 }
 
@@ -2277,7 +2305,7 @@ static int mlx4_init_hca(struct mlx4_dev *dev)
 
 		dev->caps.max_fmr_maps = (1 << (32 - ilog2(dev->caps.num_mpts))) - 1;
 
-		if (enable_4k_uar) {
+		if (enable_4k_uar || !dev->persist->num_vfs) {
 			init_hca.log_uar_sz = ilog2(dev->caps.num_uars) +
 						    PAGE_SHIFT - DEFAULT_UAR_PAGE_SHIFT;
 			init_hca.uar_page_sz = DEFAULT_UAR_PAGE_SHIFT - 12;
@@ -2399,7 +2427,7 @@ static int mlx4_init_hca(struct mlx4_dev *dev)
 		dev->caps.rx_checksum_flags_port[2] = params.rx_csum_flags_port_2;
 	}
 	priv->eq_table.inta_pin = adapter.inta_pin;
-	memcpy(dev->board_id, adapter.board_id, sizeof dev->board_id);
+	memcpy(dev->board_id, adapter.board_id, sizeof(dev->board_id));
 
 	return 0;
 
@@ -2407,13 +2435,8 @@ unmap_bf:
 	unmap_internal_clock(dev);
 	unmap_bf_area(dev);
 
-	if (mlx4_is_slave(dev)) {
-		kfree(dev->caps.qp0_qkey);
-		kfree(dev->caps.qp0_tunnel);
-		kfree(dev->caps.qp0_proxy);
-		kfree(dev->caps.qp1_tunnel);
-		kfree(dev->caps.qp1_proxy);
-	}
+	if (mlx4_is_slave(dev))
+		mlx4_slave_destroy_special_qp_cap(dev);
 
 err_close:
 	if (mlx4_is_slave(dev))
@@ -2477,7 +2500,7 @@ static int mlx4_allocate_default_counters(struct mlx4_dev *dev)
 		priv->def_counter[port] = -1;
 
 	for (port = 0; port < dev->caps.num_ports; port++) {
-		err = mlx4_counter_alloc(dev, &idx);
+		err = mlx4_counter_alloc(dev, &idx, MLX4_RES_USAGE_DRIVER);
 
 		if (!err || err == -ENOSPC) {
 			priv->def_counter[port] = idx;
@@ -2519,13 +2542,14 @@ int __mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx)
 	return 0;
 }
 
-int mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx)
+int mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx, u8 usage)
 {
+	u32 in_modifier = RES_COUNTER | (((u32)usage & 3) << 30);
 	u64 out_param;
 	int err;
 
 	if (mlx4_is_mfunc(dev)) {
-		err = mlx4_cmd_imm(dev, 0, &out_param, RES_COUNTER,
+		err = mlx4_cmd_imm(dev, 0, &out_param, in_modifier,
 				   RES_OP_RESERVE, MLX4_CMD_ALLOC_RES,
 				   MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
 		if (!err)
@@ -2869,7 +2893,7 @@ static void mlx4_enable_msi_x(struct mlx4_dev *dev)
 				dev->caps.num_eqs - dev->caps.reserved_eqs,
 				MAX_MSIX);
 
-		entries = kcalloc(nreq, sizeof *entries, GFP_KERNEL);
+		entries = kcalloc(nreq, sizeof(*entries), GFP_KERNEL);
 		if (!entries)
 			goto no_msi;
 
@@ -3596,13 +3620,8 @@ err_master_mfunc:
 		mlx4_multi_func_cleanup(dev);
 	}
 
-	if (mlx4_is_slave(dev)) {
-		kfree(dev->caps.qp0_qkey);
-		kfree(dev->caps.qp0_tunnel);
-		kfree(dev->caps.qp0_proxy);
-		kfree(dev->caps.qp1_tunnel);
-		kfree(dev->caps.qp1_proxy);
-	}
+	if (mlx4_is_slave(dev))
+		mlx4_slave_destroy_special_qp_cap(dev);
 
 err_close:
 	mlx4_close_hca(dev);
@@ -3658,7 +3677,7 @@ static int __mlx4_init_one(struct pci_dev *pdev, int pci_dev_data,
 	 * per port, we must limit the number of VFs to 63 (since their are
 	 * 128 MACs)
 	 */
-	for (i = 0; i < sizeof(nvfs)/sizeof(nvfs[0]) && i < num_vfs_argc;
+	for (i = 0; i < ARRAY_SIZE(nvfs) && i < num_vfs_argc;
 	     total_vfs += nvfs[param_map[num_vfs_argc - 1][i]], i++) {
 		nvfs[param_map[num_vfs_argc - 1][i]] = num_vfs[i];
 		if (nvfs[i] < 0) {
@@ -3667,7 +3686,7 @@ static int __mlx4_init_one(struct pci_dev *pdev, int pci_dev_data,
 			goto err_disable_pdev;
 		}
 	}
-	for (i = 0; i < sizeof(prb_vf)/sizeof(prb_vf[0]) && i < probe_vfs_argc;
+	for (i = 0; i < ARRAY_SIZE(prb_vf) && i < probe_vfs_argc;
 	     i++) {
 		prb_vf[param_map[probe_vfs_argc - 1][i]] = probe_vf[i];
 		if (prb_vf[i] < 0 || prb_vf[i] > nvfs[i]) {
@@ -3746,11 +3765,11 @@ static int __mlx4_init_one(struct pci_dev *pdev, int pci_dev_data,
 		if (total_vfs) {
 			unsigned vfs_offset = 0;
 
-			for (i = 0; i < sizeof(nvfs)/sizeof(nvfs[0]) &&
+			for (i = 0; i < ARRAY_SIZE(nvfs) &&
 			     vfs_offset + nvfs[i] < extended_func_num(pdev);
 			     vfs_offset += nvfs[i], i++)
 				;
-			if (i == sizeof(nvfs)/sizeof(nvfs[0])) {
+			if (i == ARRAY_SIZE(nvfs)) {
 				err = -ENODEV;
 				goto err_release_regions;
 			}
@@ -3782,7 +3801,6 @@ err_release_regions:
 
 err_disable_pdev:
 	mlx4_pci_disable_device(&priv->dev);
-	pci_set_drvdata(pdev, NULL);
 	return err;
 }
 
@@ -3943,11 +3961,7 @@ static void mlx4_unload_one(struct pci_dev *pdev)
 	if (!mlx4_is_slave(dev))
 		mlx4_free_ownership(dev);
 
-	kfree(dev->caps.qp0_qkey);
-	kfree(dev->caps.qp0_tunnel);
-	kfree(dev->caps.qp0_proxy);
-	kfree(dev->caps.qp1_tunnel);
-	kfree(dev->caps.qp1_proxy);
+	mlx4_slave_destroy_special_qp_cap(dev);
 	kfree(dev->dev_vfs);
 
 	mlx4_clean_dev(dev);
@@ -3997,7 +4011,6 @@ static void mlx4_remove_one(struct pci_dev *pdev)
 	devlink_unregister(devlink);
 	kfree(dev->persist);
 	devlink_free(devlink);
-	pci_set_drvdata(pdev, NULL);
 }
 
 static int restore_current_port_types(struct mlx4_dev *dev,
diff --git a/drivers/net/ethernet/mellanox/mlx4/mcg.c b/drivers/net/ethernet/mellanox/mlx4/mcg.c
index 0710b3677464..4c5306dbcf11 100644
--- a/drivers/net/ethernet/mellanox/mlx4/mcg.c
+++ b/drivers/net/ethernet/mellanox/mlx4/mcg.c
@@ -162,7 +162,7 @@ static int new_steering_entry(struct mlx4_dev *dev, u8 port,
 		return -EINVAL;
 
 	s_steer = &mlx4_priv(dev)->steer[port - 1];
-	new_entry = kzalloc(sizeof *new_entry, GFP_KERNEL);
+	new_entry = kzalloc(sizeof(*new_entry), GFP_KERNEL);
 	if (!new_entry)
 		return -ENOMEM;
 
@@ -175,7 +175,7 @@ static int new_steering_entry(struct mlx4_dev *dev, u8 port,
 	 */
 	pqp = get_promisc_qp(dev, port, steer, qpn);
 	if (pqp) {
-		dqp = kmalloc(sizeof *dqp, GFP_KERNEL);
+		dqp = kmalloc(sizeof(*dqp), GFP_KERNEL);
 		if (!dqp) {
 			err = -ENOMEM;
 			goto out_alloc;
@@ -274,7 +274,7 @@ static int existing_steering_entry(struct mlx4_dev *dev, u8 port,
 	}
 
 	/* add the qp as a duplicate on this index */
-	dqp = kmalloc(sizeof *dqp, GFP_KERNEL);
+	dqp = kmalloc(sizeof(*dqp), GFP_KERNEL);
 	if (!dqp)
 		return -ENOMEM;
 	dqp->qpn = qpn;
@@ -443,7 +443,7 @@ static int add_promisc_qp(struct mlx4_dev *dev, u8 port,
 		goto out_mutex;
 	}
 
-	pqp = kmalloc(sizeof *pqp, GFP_KERNEL);
+	pqp = kmalloc(sizeof(*pqp), GFP_KERNEL);
 	if (!pqp) {
 		err = -ENOMEM;
 		goto out_mutex;
@@ -514,7 +514,7 @@ static int add_promisc_qp(struct mlx4_dev *dev, u8 port,
 	/* add the new qpn to list of promisc qps */
 	list_add_tail(&pqp->list, &s_steer->promisc_qps[steer]);
 	/* now need to add all the promisc qps to default entry */
-	memset(mgm, 0, sizeof *mgm);
+	memset(mgm, 0, sizeof(*mgm));
 	members_count = 0;
 	list_for_each_entry(dqp, &s_steer->promisc_qps[steer], list) {
 		if (members_count == dev->caps.num_qp_per_mgm) {
@@ -1144,7 +1144,7 @@ int mlx4_qp_attach_common(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16],
 		index += dev->caps.num_mgms;
 
 		new_entry = 1;
-		memset(mgm, 0, sizeof *mgm);
+		memset(mgm, 0, sizeof(*mgm));
 		memcpy(mgm->gid, gid, 16);
 	}
 
diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4.h b/drivers/net/ethernet/mellanox/mlx4/mlx4.h
index 706d7f21ac5c..c68da1986e51 100644
--- a/drivers/net/ethernet/mellanox/mlx4/mlx4.h
+++ b/drivers/net/ethernet/mellanox/mlx4/mlx4.h
@@ -626,7 +626,7 @@ struct mlx4_mgm {
 };
 
 struct mlx4_cmd {
-	struct pci_pool	       *pool;
+	struct dma_pool	       *pool;
 	void __iomem	       *hcr;
 	struct mutex		slave_cmd_mutex;
 	struct semaphore	poll_sem;
@@ -807,6 +807,8 @@ struct mlx4_set_port_general_context {
 	u8 phv_en;
 	u8 reserved6[5];
 	__be16 user_mtu;
+	u16 reserved7;
+	u8 user_mac[6];
 };
 
 struct mlx4_set_port_rqp_calc_context {
diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
index d350b2158104..fdb3ad0cbe54 100644
--- a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
+++ b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
@@ -685,7 +685,7 @@ int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq,
 			int cq_idx);
 void mlx4_en_deactivate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq);
 int mlx4_en_set_cq_moder(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq);
-int mlx4_en_arm_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq);
+void mlx4_en_arm_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq);
 
 void mlx4_en_tx_irq(struct mlx4_cq *mcq);
 u16 mlx4_en_select_queue(struct net_device *dev, struct sk_buff *skb,
diff --git a/drivers/net/ethernet/mellanox/mlx4/mr.c b/drivers/net/ethernet/mellanox/mlx4/mr.c
index 24282cd017d3..c7c0764991c9 100644
--- a/drivers/net/ethernet/mellanox/mlx4/mr.c
+++ b/drivers/net/ethernet/mellanox/mlx4/mr.c
@@ -106,9 +106,9 @@ static int mlx4_buddy_init(struct mlx4_buddy *buddy, int max_order)
 	buddy->max_order = max_order;
 	spin_lock_init(&buddy->lock);
 
-	buddy->bits = kcalloc(buddy->max_order + 1, sizeof (long *),
+	buddy->bits = kcalloc(buddy->max_order + 1, sizeof(long *),
 			      GFP_KERNEL);
-	buddy->num_free = kcalloc((buddy->max_order + 1), sizeof *buddy->num_free,
+	buddy->num_free = kcalloc(buddy->max_order + 1, sizeof(*buddy->num_free),
 				  GFP_KERNEL);
 	if (!buddy->bits || !buddy->num_free)
 		goto err_out;
@@ -703,13 +703,13 @@ static int mlx4_write_mtt_chunk(struct mlx4_dev *dev, struct mlx4_mtt *mtt,
 		return -ENOMEM;
 
 	dma_sync_single_for_cpu(&dev->persist->pdev->dev, dma_handle,
-				npages * sizeof (u64), DMA_TO_DEVICE);
+				npages * sizeof(u64), DMA_TO_DEVICE);
 
 	for (i = 0; i < npages; ++i)
 		mtts[i] = cpu_to_be64(page_list[i] | MLX4_MTT_FLAG_PRESENT);
 
 	dma_sync_single_for_device(&dev->persist->pdev->dev, dma_handle,
-				   npages * sizeof (u64), DMA_TO_DEVICE);
+				   npages * sizeof(u64), DMA_TO_DEVICE);
 
 	return 0;
 }
@@ -1052,7 +1052,7 @@ int mlx4_fmr_alloc(struct mlx4_dev *dev, u32 pd, u32 access, int max_pages,
 		return -EINVAL;
 
 	/* All MTTs must fit in the same page */
-	if (max_pages * sizeof *fmr->mtts > PAGE_SIZE)
+	if (max_pages * sizeof(*fmr->mtts) > PAGE_SIZE)
 		return -EINVAL;
 
 	fmr->page_shift = page_shift;
diff --git a/drivers/net/ethernet/mellanox/mlx4/port.c b/drivers/net/ethernet/mellanox/mlx4/port.c
index 4e36e287d605..3ef3406ff4cb 100644
--- a/drivers/net/ethernet/mellanox/mlx4/port.c
+++ b/drivers/net/ethernet/mellanox/mlx4/port.c
@@ -52,6 +52,7 @@
 
 #define MLX4_FLAG2_V_IGNORE_FCS_MASK		BIT(1)
 #define MLX4_FLAG2_V_USER_MTU_MASK		BIT(5)
+#define MLX4_FLAG2_V_USER_MAC_MASK		BIT(6)
 #define MLX4_FLAG_V_MTU_MASK			BIT(0)
 #define MLX4_FLAG_V_PPRX_MASK			BIT(1)
 #define MLX4_FLAG_V_PPTX_MASK			BIT(2)
@@ -1700,6 +1701,30 @@ int mlx4_SET_PORT_user_mtu(struct mlx4_dev *dev, u8 port, u16 user_mtu)
 }
 EXPORT_SYMBOL(mlx4_SET_PORT_user_mtu);
 
+int mlx4_SET_PORT_user_mac(struct mlx4_dev *dev, u8 port, u8 *user_mac)
+{
+	struct mlx4_cmd_mailbox *mailbox;
+	struct mlx4_set_port_general_context *context;
+	u32 in_mod;
+	int err;
+
+	mailbox = mlx4_alloc_cmd_mailbox(dev);
+	if (IS_ERR(mailbox))
+		return PTR_ERR(mailbox);
+	context = mailbox->buf;
+	context->flags2 |= MLX4_FLAG2_V_USER_MAC_MASK;
+	memcpy(context->user_mac, user_mac, sizeof(context->user_mac));
+
+	in_mod = MLX4_SET_PORT_GENERAL << 8 | port;
+	err = mlx4_cmd(dev, mailbox->dma, in_mod, MLX4_SET_PORT_ETH_OPCODE,
+		       MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B,
+		       MLX4_CMD_NATIVE);
+
+	mlx4_free_cmd_mailbox(dev, mailbox);
+	return err;
+}
+EXPORT_SYMBOL(mlx4_SET_PORT_user_mac);
+
 int mlx4_SET_PORT_fcs_check(struct mlx4_dev *dev, u8 port, u8 ignore_fcs_value)
 {
 	struct mlx4_cmd_mailbox *mailbox;
diff --git a/drivers/net/ethernet/mellanox/mlx4/qp.c b/drivers/net/ethernet/mellanox/mlx4/qp.c
index 26747212526b..728a2fb1f5c0 100644
--- a/drivers/net/ethernet/mellanox/mlx4/qp.c
+++ b/drivers/net/ethernet/mellanox/mlx4/qp.c
@@ -174,7 +174,7 @@ static int __mlx4_qp_modify(struct mlx4_dev *dev, struct mlx4_mtt *mtt,
 			cpu_to_be16(mlx4_qp_roce_entropy(dev, qp->qpn));
 
 	*(__be32 *) mailbox->buf = cpu_to_be32(optpar);
-	memcpy(mailbox->buf + 8, context, sizeof *context);
+	memcpy(mailbox->buf + 8, context, sizeof(*context));
 
 	((struct mlx4_qp_context *) (mailbox->buf + 8))->local_qpn =
 		cpu_to_be32(qp->qpn);
@@ -245,8 +245,9 @@ int __mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align,
 }
 
 int mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align,
-			  int *base, u8 flags)
+			  int *base, u8 flags, u8 usage)
 {
+	u32 in_modifier = RES_QP | (((u32)usage & 3) << 30);
 	u64 in_param = 0;
 	u64 out_param;
 	int err;
@@ -258,7 +259,7 @@ int mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align,
 		set_param_l(&in_param, (((u32)flags) << 24) | (u32)cnt);
 		set_param_h(&in_param, align);
 		err = mlx4_cmd_imm(dev, in_param, &out_param,
-				   RES_QP, RES_OP_RESERVE,
+				   in_modifier, RES_OP_RESERVE,
 				   MLX4_CMD_ALLOC_RES,
 				   MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
 		if (err)
@@ -844,24 +845,21 @@ int mlx4_init_qp_table(struct mlx4_dev *dev)
 
 		/* In mfunc, calculate proxy and tunnel qp offsets for the PF here,
 		 * since the PF does not call mlx4_slave_caps */
-		dev->caps.qp0_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);
-		dev->caps.qp0_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);
-		dev->caps.qp1_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);
-		dev->caps.qp1_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);
-
-		if (!dev->caps.qp0_tunnel || !dev->caps.qp0_proxy ||
-		    !dev->caps.qp1_tunnel || !dev->caps.qp1_proxy) {
+		dev->caps.spec_qps = kcalloc(dev->caps.num_ports,
+					     sizeof(*dev->caps.spec_qps),
+					     GFP_KERNEL);
+		if (!dev->caps.spec_qps) {
 			err = -ENOMEM;
 			goto err_mem;
 		}
 
 		for (k = 0; k < dev->caps.num_ports; k++) {
-			dev->caps.qp0_proxy[k] = dev->phys_caps.base_proxy_sqpn +
+			dev->caps.spec_qps[k].qp0_proxy = dev->phys_caps.base_proxy_sqpn +
 				8 * mlx4_master_func_num(dev) + k;
-			dev->caps.qp0_tunnel[k] = dev->caps.qp0_proxy[k] + 8 * MLX4_MFUNC_MAX;
-			dev->caps.qp1_proxy[k] = dev->phys_caps.base_proxy_sqpn +
+			dev->caps.spec_qps[k].qp0_tunnel = dev->caps.spec_qps[k].qp0_proxy + 8 * MLX4_MFUNC_MAX;
+			dev->caps.spec_qps[k].qp1_proxy = dev->phys_caps.base_proxy_sqpn +
 				8 * mlx4_master_func_num(dev) + MLX4_MAX_PORTS + k;
-			dev->caps.qp1_tunnel[k] = dev->caps.qp1_proxy[k] + 8 * MLX4_MFUNC_MAX;
+			dev->caps.spec_qps[k].qp1_tunnel = dev->caps.spec_qps[k].qp1_proxy + 8 * MLX4_MFUNC_MAX;
 		}
 	}
 
@@ -873,12 +871,8 @@ int mlx4_init_qp_table(struct mlx4_dev *dev)
 	return err;
 
 err_mem:
-	kfree(dev->caps.qp0_tunnel);
-	kfree(dev->caps.qp0_proxy);
-	kfree(dev->caps.qp1_tunnel);
-	kfree(dev->caps.qp1_proxy);
-	dev->caps.qp0_tunnel = dev->caps.qp0_proxy =
-		dev->caps.qp1_tunnel = dev->caps.qp1_proxy = NULL;
+	kfree(dev->caps.spec_qps);
+	dev->caps.spec_qps = NULL;
 	mlx4_cleanup_qp_zones(dev);
 	return err;
 }
@@ -907,7 +901,7 @@ int mlx4_qp_query(struct mlx4_dev *dev, struct mlx4_qp *qp,
 			   MLX4_CMD_QUERY_QP, MLX4_CMD_TIME_CLASS_A,
 			   MLX4_CMD_WRAPPED);
 	if (!err)
-		memcpy(context, mailbox->buf + 8, sizeof *context);
+		memcpy(context, mailbox->buf + 8, sizeof(*context));
 
 	mlx4_free_cmd_mailbox(dev, mailbox);
 	return err;
diff --git a/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c b/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c
index 215e21c3dc8a..fabb53379727 100644
--- a/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c
+++ b/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c
@@ -1040,7 +1040,7 @@ static struct res_common *alloc_qp_tr(int id)
 {
 	struct res_qp *ret;
 
-	ret = kzalloc(sizeof *ret, GFP_KERNEL);
+	ret = kzalloc(sizeof(*ret), GFP_KERNEL);
 	if (!ret)
 		return NULL;
 
@@ -1058,7 +1058,7 @@ static struct res_common *alloc_mtt_tr(int id, int order)
 {
 	struct res_mtt *ret;
 
-	ret = kzalloc(sizeof *ret, GFP_KERNEL);
+	ret = kzalloc(sizeof(*ret), GFP_KERNEL);
 	if (!ret)
 		return NULL;
 
@@ -1074,7 +1074,7 @@ static struct res_common *alloc_mpt_tr(int id, int key)
 {
 	struct res_mpt *ret;
 
-	ret = kzalloc(sizeof *ret, GFP_KERNEL);
+	ret = kzalloc(sizeof(*ret), GFP_KERNEL);
 	if (!ret)
 		return NULL;
 
@@ -1089,7 +1089,7 @@ static struct res_common *alloc_eq_tr(int id)
 {
 	struct res_eq *ret;
 
-	ret = kzalloc(sizeof *ret, GFP_KERNEL);
+	ret = kzalloc(sizeof(*ret), GFP_KERNEL);
 	if (!ret)
 		return NULL;
 
@@ -1103,7 +1103,7 @@ static struct res_common *alloc_cq_tr(int id)
 {
 	struct res_cq *ret;
 
-	ret = kzalloc(sizeof *ret, GFP_KERNEL);
+	ret = kzalloc(sizeof(*ret), GFP_KERNEL);
 	if (!ret)
 		return NULL;
 
@@ -1118,7 +1118,7 @@ static struct res_common *alloc_srq_tr(int id)
 {
 	struct res_srq *ret;
 
-	ret = kzalloc(sizeof *ret, GFP_KERNEL);
+	ret = kzalloc(sizeof(*ret), GFP_KERNEL);
 	if (!ret)
 		return NULL;
 
@@ -1133,7 +1133,7 @@ static struct res_common *alloc_counter_tr(int id, int port)
 {
 	struct res_counter *ret;
 
-	ret = kzalloc(sizeof *ret, GFP_KERNEL);
+	ret = kzalloc(sizeof(*ret), GFP_KERNEL);
 	if (!ret)
 		return NULL;
 
@@ -1148,7 +1148,7 @@ static struct res_common *alloc_xrcdn_tr(int id)
 {
 	struct res_xrcdn *ret;
 
-	ret = kzalloc(sizeof *ret, GFP_KERNEL);
+	ret = kzalloc(sizeof(*ret), GFP_KERNEL);
 	if (!ret)
 		return NULL;
 
@@ -1162,7 +1162,7 @@ static struct res_common *alloc_fs_rule_tr(u64 id, int qpn)
 {
 	struct res_fs_rule *ret;
 
-	ret = kzalloc(sizeof *ret, GFP_KERNEL);
+	ret = kzalloc(sizeof(*ret), GFP_KERNEL);
 	if (!ret)
 		return NULL;
 
@@ -1274,7 +1274,7 @@ static int add_res_range(struct mlx4_dev *dev, int slave, u64 base, int count,
 	struct mlx4_resource_tracker *tracker = &priv->mfunc.master.res_tracker;
 	struct rb_root *root = &tracker->res_tree[type];
 
-	res_arr = kzalloc(count * sizeof *res_arr, GFP_KERNEL);
+	res_arr = kcalloc(count, sizeof(*res_arr), GFP_KERNEL);
 	if (!res_arr)
 		return -ENOMEM;
 
@@ -2027,7 +2027,7 @@ static int mac_add_to_slave(struct mlx4_dev *dev, int slave, u64 mac, int port,
 
 	if (mlx4_grant_resource(dev, slave, RES_MAC, 1, port))
 		return -EINVAL;
-	res = kzalloc(sizeof *res, GFP_KERNEL);
+	res = kzalloc(sizeof(*res), GFP_KERNEL);
 	if (!res) {
 		mlx4_release_resource(dev, slave, RES_MAC, 1, port);
 		return -ENOMEM;
@@ -4020,7 +4020,7 @@ static int add_mcg_res(struct mlx4_dev *dev, int slave, struct res_qp *rqp,
 	struct res_gid *res;
 	int err;
 
-	res = kzalloc(sizeof *res, GFP_KERNEL);
+	res = kzalloc(sizeof(*res), GFP_KERNEL);
 	if (!res)
 		return -ENOMEM;
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig
index 5aee05992f27..fdaef00465d7 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig
@@ -34,6 +34,27 @@ config MLX5_CORE_EN
 	---help---
 	  Ethernet support in Mellanox Technologies ConnectX-4 NIC.
 
+config MLX5_MPFS
+        bool "Mellanox Technologies MLX5 MPFS support"
+        depends on MLX5_CORE_EN
+	default y
+        ---help---
+	  Mellanox Technologies Ethernet Multi-Physical Function Switch (MPFS)
+          support in ConnectX NIC. MPFs is required for when multi-PF configuration
+          is enabled to allow passing user configured unicast MAC addresses to the
+          requesting PF.
+
+config MLX5_ESWITCH
+	bool "Mellanox Technologies MLX5 SRIOV E-Switch support"
+	depends on MLX5_CORE_EN
+	default y
+	---help---
+	  Mellanox Technologies Ethernet SRIOV E-Switch support in ConnectX NIC.
+          E-Switch provides internal SRIOV packet steering and switching for the
+          enabled VFs and PF in two available modes:
+                Legacy SRIOV mode (L2 mac vlan steering based).
+                Switchdev mode (eswitch offloads).
+
 config MLX5_CORE_EN_DCB
 	bool "Data Center Bridging (DCB) Support"
 	default y
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
index 9d17e4e76d3a..87a3099808f3 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
@@ -4,17 +4,21 @@ subdir-ccflags-y += -I$(src)
 mlx5_core-y :=	main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \
 		health.o mcg.o cq.o srq.o alloc.o qp.o port.o mr.o pd.o \
 		mad.o transobj.o vport.o sriov.o fs_cmd.o fs_core.o \
-		fs_counters.o rl.o lag.o dev.o wq.o lib/gid.o
+		fs_counters.o rl.o lag.o dev.o wq.o lib/gid.o \
+		diag/fs_tracepoint.o
 
 mlx5_core-$(CONFIG_MLX5_ACCEL) += accel/ipsec.o
 
 mlx5_core-$(CONFIG_MLX5_FPGA) += fpga/cmd.o fpga/core.o fpga/conn.o fpga/sdk.o \
 		fpga/ipsec.o
 
-mlx5_core-$(CONFIG_MLX5_CORE_EN) += eswitch.o eswitch_offloads.o \
-		en_main.o en_common.o en_fs.o en_ethtool.o en_tx.o \
-		en_rx.o en_rx_am.o en_txrx.o en_clock.o vxlan.o \
-		en_tc.o en_arfs.o en_rep.o en_fs_ethtool.o en_selftest.o
+mlx5_core-$(CONFIG_MLX5_CORE_EN) += en_main.o en_common.o en_fs.o en_ethtool.o \
+		en_tx.o en_rx.o en_rx_am.o en_txrx.o en_clock.o vxlan.o \
+		en_arfs.o en_fs_ethtool.o en_selftest.o
+
+mlx5_core-$(CONFIG_MLX5_MPFS) += lib/mpfs.o
+
+mlx5_core-$(CONFIG_MLX5_ESWITCH) += eswitch.o eswitch_offloads.o en_rep.o en_tc.o
 
 mlx5_core-$(CONFIG_MLX5_CORE_EN_DCB) +=  en_dcbnl.o
 
@@ -22,3 +26,5 @@ mlx5_core-$(CONFIG_MLX5_CORE_IPOIB) += ipoib/ipoib.o ipoib/ethtool.o
 
 mlx5_core-$(CONFIG_MLX5_EN_IPSEC) += en_accel/ipsec.o en_accel/ipsec_rxtx.o \
 		en_accel/ipsec_stats.o
+
+CFLAGS_tracepoint.o := -I$(src)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/alloc.c b/drivers/net/ethernet/mellanox/mlx5/core/alloc.c
index 3c95f7f53802..47239bf7bf43 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/alloc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/alloc.c
@@ -258,6 +258,7 @@ EXPORT_SYMBOL_GPL(mlx5_db_alloc);
 void mlx5_db_free(struct mlx5_core_dev *dev, struct mlx5_db *db)
 {
 	u32 db_per_page = PAGE_SIZE / cache_line_size();
+
 	mutex_lock(&dev->priv.pgdir_mutex);
 
 	__set_bit(db->index, db->u.pgdir->bitmap);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c
index 31cbe5e86a01..1fffdebbc9e8 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c
@@ -802,7 +802,6 @@ static void cmd_work_handler(struct work_struct *work)
 	bool poll_cmd = ent->polling;
 	int alloc_ret;
 
-
 	sem = ent->page_queue ? &cmd->pages_sem : &cmd->sem;
 	down(sem);
 	if (!ent->page_queue) {
@@ -1110,7 +1109,7 @@ static struct mlx5_cmd_mailbox *alloc_cmd_box(struct mlx5_core_dev *dev,
 	if (!mailbox)
 		return ERR_PTR(-ENOMEM);
 
-	mailbox->buf = pci_pool_zalloc(dev->cmd.pool, flags,
+	mailbox->buf = dma_pool_zalloc(dev->cmd.pool, flags,
 				       &mailbox->dma);
 	if (!mailbox->buf) {
 		mlx5_core_dbg(dev, "failed allocation\n");
@@ -1125,7 +1124,7 @@ static struct mlx5_cmd_mailbox *alloc_cmd_box(struct mlx5_core_dev *dev,
 static void free_cmd_box(struct mlx5_core_dev *dev,
 			 struct mlx5_cmd_mailbox *mailbox)
 {
-	pci_pool_free(dev->cmd.pool, mailbox->buf, mailbox->dma);
+	dma_pool_free(dev->cmd.pool, mailbox->buf, mailbox->dma);
 	kfree(mailbox);
 }
 
@@ -1776,7 +1775,8 @@ int mlx5_cmd_init(struct mlx5_core_dev *dev)
 		return -EINVAL;
 	}
 
-	cmd->pool = pci_pool_create("mlx5_cmd", dev->pdev, size, align, 0);
+	cmd->pool = dma_pool_create("mlx5_cmd", &dev->pdev->dev, size, align,
+				    0);
 	if (!cmd->pool)
 		return -ENOMEM;
 
@@ -1866,7 +1866,7 @@ err_free_page:
 	free_cmd_page(dev, cmd);
 
 err_free_pool:
-	pci_pool_destroy(cmd->pool);
+	dma_pool_destroy(cmd->pool);
 
 	return err;
 }
@@ -1880,6 +1880,6 @@ void mlx5_cmd_cleanup(struct mlx5_core_dev *dev)
 	destroy_workqueue(cmd->wq);
 	destroy_msg_cache(dev);
 	free_cmd_page(dev, cmd);
-	pci_pool_destroy(cmd->pool);
+	dma_pool_destroy(cmd->pool);
 }
 EXPORT_SYMBOL(mlx5_cmd_cleanup);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/dev.c b/drivers/net/ethernet/mellanox/mlx5/core/dev.c
index a62f4b6a21a5..ff60cf7342ca 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/dev.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/dev.c
@@ -45,11 +45,70 @@ struct mlx5_device_context {
 	unsigned long		state;
 };
 
+struct mlx5_delayed_event {
+	struct list_head	list;
+	struct mlx5_core_dev	*dev;
+	enum mlx5_dev_event	event;
+	unsigned long		param;
+};
+
 enum {
 	MLX5_INTERFACE_ADDED,
 	MLX5_INTERFACE_ATTACHED,
 };
 
+static void add_delayed_event(struct mlx5_priv *priv,
+			      struct mlx5_core_dev *dev,
+			      enum mlx5_dev_event event,
+			      unsigned long param)
+{
+	struct mlx5_delayed_event *delayed_event;
+
+	delayed_event = kzalloc(sizeof(*delayed_event), GFP_ATOMIC);
+	if (!delayed_event) {
+		mlx5_core_err(dev, "event %d is missed\n", event);
+		return;
+	}
+
+	mlx5_core_dbg(dev, "Accumulating event %d\n", event);
+	delayed_event->dev = dev;
+	delayed_event->event = event;
+	delayed_event->param = param;
+	list_add_tail(&delayed_event->list, &priv->waiting_events_list);
+}
+
+static void fire_delayed_event_locked(struct mlx5_device_context *dev_ctx,
+				      struct mlx5_core_dev *dev,
+				      struct mlx5_priv *priv)
+{
+	struct mlx5_delayed_event *de;
+	struct mlx5_delayed_event *n;
+
+	/* stop delaying events */
+	priv->is_accum_events = false;
+
+	/* fire all accumulated events before new event comes */
+	list_for_each_entry_safe(de, n, &priv->waiting_events_list, list) {
+		dev_ctx->intf->event(dev, dev_ctx->context, de->event, de->param);
+		list_del(&de->list);
+		kfree(de);
+	}
+}
+
+static void cleanup_delayed_evets(struct mlx5_priv *priv)
+{
+	struct mlx5_delayed_event *de;
+	struct mlx5_delayed_event *n;
+
+	spin_lock_irq(&priv->ctx_lock);
+	priv->is_accum_events = false;
+	list_for_each_entry_safe(de, n, &priv->waiting_events_list, list) {
+		list_del(&de->list);
+		kfree(de);
+	}
+	spin_unlock_irq(&priv->ctx_lock);
+}
+
 void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
 {
 	struct mlx5_device_context *dev_ctx;
@@ -63,6 +122,12 @@ void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
 		return;
 
 	dev_ctx->intf = intf;
+	/* accumulating events that can come after mlx5_ib calls to
+	 * ib_register_device, till adding that interface to the events list.
+	 */
+
+	priv->is_accum_events = true;
+
 	dev_ctx->context = intf->add(dev);
 	set_bit(MLX5_INTERFACE_ADDED, &dev_ctx->state);
 	if (intf->attach)
@@ -71,6 +136,9 @@ void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
 	if (dev_ctx->context) {
 		spin_lock_irq(&priv->ctx_lock);
 		list_add_tail(&dev_ctx->list, &priv->ctx_list);
+
+		fire_delayed_event_locked(dev_ctx, dev, priv);
+
 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
 		if (dev_ctx->intf->pfault) {
 			if (priv->pfault) {
@@ -84,6 +152,8 @@ void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
 		spin_unlock_irq(&priv->ctx_lock);
 	} else {
 		kfree(dev_ctx);
+		 /* delete all accumulated events */
+		cleanup_delayed_evets(priv);
 	}
 }
 
@@ -341,6 +411,9 @@ void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event,
 
 	spin_lock_irqsave(&priv->ctx_lock, flags);
 
+	if (priv->is_accum_events)
+		add_delayed_event(priv, dev, event, param);
+
 	list_for_each_entry(dev_ctx, &priv->ctx_list, list)
 		if (dev_ctx->intf->event)
 			dev_ctx->intf->event(dev, dev_ctx->context, event, param);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/diag/Makefile
new file mode 100644
index 000000000000..d8e17110f25d
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/Makefile
@@ -0,0 +1 @@
+subdir-ccflags-y += -I$(src)/..
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c b/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c
new file mode 100644
index 000000000000..0be4575b58a2
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c
@@ -0,0 +1,261 @@
+/*
+ * Copyright (c) 2017, Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#define CREATE_TRACE_POINTS
+
+#include "fs_tracepoint.h"
+#include <linux/stringify.h>
+
+#define DECLARE_MASK_VAL(type, name) struct {type m; type v; } name
+#define MASK_VAL(type, spec, name, mask, val, fld)	\
+		DECLARE_MASK_VAL(type, name) =		\
+			{.m = MLX5_GET(spec, mask, fld),\
+			 .v = MLX5_GET(spec, val, fld)}
+#define MASK_VAL_BE(type, spec, name, mask, val, fld)	\
+		    DECLARE_MASK_VAL(type, name) =	\
+			{.m = MLX5_GET_BE(type, spec, mask, fld),\
+			 .v = MLX5_GET_BE(type, spec, val, fld)}
+#define GET_MASKED_VAL(name) (name.m & name.v)
+
+#define GET_MASK_VAL(name, type, mask, val, fld)	\
+		(name.m = MLX5_GET(type, mask, fld),	\
+		 name.v = MLX5_GET(type, val, fld),	\
+		 name.m & name.v)
+#define PRINT_MASKED_VAL(name, p, format) {		\
+	if (name.m)			\
+		trace_seq_printf(p, __stringify(name) "=" format " ", name.v); \
+	}
+#define PRINT_MASKED_VALP(name, cast, p, format) {	\
+	if (name.m)			\
+		trace_seq_printf(p, __stringify(name) "=" format " ",	       \
+				 (cast)&name.v);\
+	}
+
+static void print_lyr_2_4_hdrs(struct trace_seq *p,
+			       const u32 *mask, const u32 *value)
+{
+#define MASK_VAL_L2(type, name, fld) \
+	MASK_VAL(type, fte_match_set_lyr_2_4, name, mask, value, fld)
+	DECLARE_MASK_VAL(u64, smac) = {
+		.m = MLX5_GET(fte_match_set_lyr_2_4, mask, smac_47_16) << 16 |
+		     MLX5_GET(fte_match_set_lyr_2_4, mask, smac_15_0),
+		.v = MLX5_GET(fte_match_set_lyr_2_4, value, smac_47_16) << 16 |
+		     MLX5_GET(fte_match_set_lyr_2_4, value, smac_15_0)};
+	DECLARE_MASK_VAL(u64, dmac) = {
+		.m = MLX5_GET(fte_match_set_lyr_2_4, mask, dmac_47_16) << 16 |
+		     MLX5_GET(fte_match_set_lyr_2_4, mask, dmac_15_0),
+		.v = MLX5_GET(fte_match_set_lyr_2_4, value, dmac_47_16) << 16 |
+		     MLX5_GET(fte_match_set_lyr_2_4, value, dmac_15_0)};
+	MASK_VAL_L2(u16, ethertype, ethertype);
+
+	PRINT_MASKED_VALP(smac, u8 *, p, "%pM");
+	PRINT_MASKED_VALP(dmac, u8 *, p, "%pM");
+	PRINT_MASKED_VAL(ethertype, p, "%04x");
+
+	if (ethertype.m == 0xffff) {
+		if (ethertype.v == ETH_P_IP) {
+#define MASK_VAL_L2_BE(type, name, fld) \
+	MASK_VAL_BE(type, fte_match_set_lyr_2_4, name, mask, value, fld)
+			MASK_VAL_L2_BE(u32, src_ipv4,
+				       src_ipv4_src_ipv6.ipv4_layout.ipv4);
+			MASK_VAL_L2_BE(u32, dst_ipv4,
+				       dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
+
+			PRINT_MASKED_VALP(src_ipv4, typeof(&src_ipv4.v), p,
+					  "%pI4");
+			PRINT_MASKED_VALP(dst_ipv4, typeof(&dst_ipv4.v), p,
+					  "%pI4");
+		} else if (ethertype.v == ETH_P_IPV6) {
+			static const struct in6_addr full_ones = {
+				.in6_u.u6_addr32 = {htonl(0xffffffff),
+						    htonl(0xffffffff),
+						    htonl(0xffffffff),
+						    htonl(0xffffffff)},
+			};
+			DECLARE_MASK_VAL(struct in6_addr, src_ipv6);
+			DECLARE_MASK_VAL(struct in6_addr, dst_ipv6);
+
+			memcpy(src_ipv6.m.in6_u.u6_addr8,
+			       MLX5_ADDR_OF(fte_match_set_lyr_2_4, mask,
+					    src_ipv4_src_ipv6.ipv6_layout.ipv6),
+			       sizeof(src_ipv6.m));
+			memcpy(dst_ipv6.m.in6_u.u6_addr8,
+			       MLX5_ADDR_OF(fte_match_set_lyr_2_4, mask,
+					    dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
+			       sizeof(dst_ipv6.m));
+			memcpy(src_ipv6.v.in6_u.u6_addr8,
+			       MLX5_ADDR_OF(fte_match_set_lyr_2_4, value,
+					    src_ipv4_src_ipv6.ipv6_layout.ipv6),
+			       sizeof(src_ipv6.v));
+			memcpy(dst_ipv6.v.in6_u.u6_addr8,
+			       MLX5_ADDR_OF(fte_match_set_lyr_2_4, value,
+					    dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
+			       sizeof(dst_ipv6.v));
+
+			if (!memcmp(&src_ipv6.m, &full_ones, sizeof(full_ones)))
+				trace_seq_printf(p, "src_ipv6=%pI6 ",
+						 &src_ipv6.v);
+			if (!memcmp(&dst_ipv6.m, &full_ones, sizeof(full_ones)))
+				trace_seq_printf(p, "dst_ipv6=%pI6 ",
+						 &dst_ipv6.v);
+		}
+	}
+
+#define PRINT_MASKED_VAL_L2(type, name, fld, p, format) {\
+	MASK_VAL_L2(type, name, fld);		         \
+	PRINT_MASKED_VAL(name, p, format);		 \
+}
+
+	PRINT_MASKED_VAL_L2(u8, ip_protocol, ip_protocol, p, "%02x");
+	PRINT_MASKED_VAL_L2(u16, tcp_flags, tcp_flags, p, "%x");
+	PRINT_MASKED_VAL_L2(u16, tcp_sport, tcp_sport, p, "%u");
+	PRINT_MASKED_VAL_L2(u16, tcp_dport, tcp_dport, p, "%u");
+	PRINT_MASKED_VAL_L2(u16, udp_sport, udp_sport, p, "%u");
+	PRINT_MASKED_VAL_L2(u16, udp_dport, udp_dport, p, "%u");
+	PRINT_MASKED_VAL_L2(u16, first_vid, first_vid, p, "%04x");
+	PRINT_MASKED_VAL_L2(u8, first_prio, first_prio, p, "%x");
+	PRINT_MASKED_VAL_L2(u8, first_cfi, first_cfi, p, "%d");
+	PRINT_MASKED_VAL_L2(u8, ip_dscp, ip_dscp, p, "%02x");
+	PRINT_MASKED_VAL_L2(u8, ip_ecn, ip_ecn, p, "%x");
+	PRINT_MASKED_VAL_L2(u8, cvlan_tag, cvlan_tag, p, "%d");
+	PRINT_MASKED_VAL_L2(u8, svlan_tag, svlan_tag, p, "%d");
+	PRINT_MASKED_VAL_L2(u8, frag, frag, p, "%d");
+}
+
+static void print_misc_parameters_hdrs(struct trace_seq *p,
+				       const u32 *mask, const u32 *value)
+{
+#define MASK_VAL_MISC(type, name, fld) \
+	MASK_VAL(type, fte_match_set_misc, name, mask, value, fld)
+#define PRINT_MASKED_VAL_MISC(type, name, fld, p, format) {\
+	MASK_VAL_MISC(type, name, fld);			   \
+	PRINT_MASKED_VAL(name, p, format);		   \
+}
+	DECLARE_MASK_VAL(u64, gre_key) = {
+		.m = MLX5_GET(fte_match_set_misc, mask, gre_key_h) << 8 |
+		     MLX5_GET(fte_match_set_misc, mask, gre_key_l),
+		.v = MLX5_GET(fte_match_set_misc, value, gre_key_h) << 8 |
+		     MLX5_GET(fte_match_set_misc, value, gre_key_l)};
+
+	PRINT_MASKED_VAL(gre_key, p, "%llu");
+	PRINT_MASKED_VAL_MISC(u32, source_sqn, source_sqn, p, "%u");
+	PRINT_MASKED_VAL_MISC(u16, source_port, source_port, p, "%u");
+	PRINT_MASKED_VAL_MISC(u8, outer_second_prio, outer_second_prio,
+			      p, "%u");
+	PRINT_MASKED_VAL_MISC(u8, outer_second_cfi, outer_second_cfi, p, "%u");
+	PRINT_MASKED_VAL_MISC(u16, outer_second_vid, outer_second_vid, p, "%u");
+	PRINT_MASKED_VAL_MISC(u8, inner_second_prio, inner_second_prio,
+			      p, "%u");
+	PRINT_MASKED_VAL_MISC(u8, inner_second_cfi, inner_second_cfi, p, "%u");
+	PRINT_MASKED_VAL_MISC(u16, inner_second_vid, inner_second_vid, p, "%u");
+
+	PRINT_MASKED_VAL_MISC(u8, outer_second_cvlan_tag,
+			      outer_second_cvlan_tag, p, "%u");
+	PRINT_MASKED_VAL_MISC(u8, inner_second_cvlan_tag,
+			      inner_second_cvlan_tag, p, "%u");
+	PRINT_MASKED_VAL_MISC(u8, outer_second_svlan_tag,
+			      outer_second_svlan_tag, p, "%u");
+	PRINT_MASKED_VAL_MISC(u8, inner_second_svlan_tag,
+			      inner_second_svlan_tag, p, "%u");
+
+	PRINT_MASKED_VAL_MISC(u8, gre_protocol, gre_protocol, p, "%u");
+
+	PRINT_MASKED_VAL_MISC(u32, vxlan_vni, vxlan_vni, p, "%u");
+	PRINT_MASKED_VAL_MISC(u32, outer_ipv6_flow_label, outer_ipv6_flow_label,
+			      p, "%x");
+	PRINT_MASKED_VAL_MISC(u32, inner_ipv6_flow_label, inner_ipv6_flow_label,
+			      p, "%x");
+}
+
+const char *parse_fs_hdrs(struct trace_seq *p,
+			  u8 match_criteria_enable,
+			  const u32 *mask_outer,
+			  const u32 *mask_misc,
+			  const u32 *mask_inner,
+			  const u32 *value_outer,
+			  const u32 *value_misc,
+			  const u32 *value_inner)
+{
+	const char *ret = trace_seq_buffer_ptr(p);
+
+	if (match_criteria_enable &
+	    1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS) {
+		trace_seq_printf(p, "[outer] ");
+		print_lyr_2_4_hdrs(p, mask_outer, value_outer);
+	}
+
+	if (match_criteria_enable &
+	    1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS) {
+		trace_seq_printf(p, "[misc] ");
+		print_misc_parameters_hdrs(p, mask_misc, value_misc);
+	}
+	if (match_criteria_enable &
+	    1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS) {
+		trace_seq_printf(p, "[inner] ");
+		print_lyr_2_4_hdrs(p, mask_inner, value_inner);
+	}
+	trace_seq_putc(p, 0);
+	return ret;
+}
+
+const char *parse_fs_dst(struct trace_seq *p,
+			 const struct mlx5_flow_destination *dst,
+			 u32 counter_id)
+{
+	const char *ret = trace_seq_buffer_ptr(p);
+
+	switch (dst->type) {
+	case MLX5_FLOW_DESTINATION_TYPE_VPORT:
+		trace_seq_printf(p, "vport=%u\n", dst->vport_num);
+		break;
+	case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE:
+		trace_seq_printf(p, "ft=%p\n", dst->ft);
+		break;
+	case MLX5_FLOW_DESTINATION_TYPE_TIR:
+		trace_seq_printf(p, "tir=%u\n", dst->tir_num);
+		break;
+	case MLX5_FLOW_DESTINATION_TYPE_COUNTER:
+		trace_seq_printf(p, "counter_id=%u\n", counter_id);
+		break;
+	}
+
+	trace_seq_putc(p, 0);
+	return ret;
+}
+
+EXPORT_TRACEPOINT_SYMBOL(mlx5_fs_add_fg);
+EXPORT_TRACEPOINT_SYMBOL(mlx5_fs_del_fg);
+EXPORT_TRACEPOINT_SYMBOL(mlx5_fs_set_fte);
+EXPORT_TRACEPOINT_SYMBOL(mlx5_fs_del_fte);
+EXPORT_TRACEPOINT_SYMBOL(mlx5_fs_add_rule);
+EXPORT_TRACEPOINT_SYMBOL(mlx5_fs_del_rule);
+
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.h b/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.h
new file mode 100644
index 000000000000..1e3a6c3e4132
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.h
@@ -0,0 +1,282 @@
+/*
+ * Copyright (c) 2017, Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#if !defined(_MLX5_FS_TP_) || defined(TRACE_HEADER_MULTI_READ)
+#define _MLX5_FS_TP_
+
+#include <linux/tracepoint.h>
+#include <linux/trace_seq.h>
+#include "../fs_core.h"
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM mlx5
+
+#define __parse_fs_hdrs(match_criteria_enable, mouter, mmisc, minner, vouter, \
+			vinner, vmisc)					      \
+	parse_fs_hdrs(p, match_criteria_enable, mouter, mmisc, minner, vouter,\
+		      vinner, vmisc)
+
+const char *parse_fs_hdrs(struct trace_seq *p,
+			  u8 match_criteria_enable,
+			  const u32 *mask_outer,
+			  const u32 *mask_misc,
+			  const u32 *mask_inner,
+			  const u32 *value_outer,
+			  const u32 *value_misc,
+			  const u32 *value_inner);
+
+#define __parse_fs_dst(dst, counter_id) \
+	parse_fs_dst(p, (const struct mlx5_flow_destination *)dst, counter_id)
+
+const char *parse_fs_dst(struct trace_seq *p,
+			 const struct mlx5_flow_destination *dst,
+			 u32 counter_id);
+
+TRACE_EVENT(mlx5_fs_add_fg,
+	    TP_PROTO(const struct mlx5_flow_group *fg),
+	    TP_ARGS(fg),
+	    TP_STRUCT__entry(
+		__field(const struct mlx5_flow_group *, fg)
+		__field(const struct mlx5_flow_table *, ft)
+		__field(u32, start_index)
+		__field(u32, end_index)
+		__field(u32, id)
+		__field(u8, mask_enable)
+		__array(u32, mask_outer, MLX5_ST_SZ_DW(fte_match_set_lyr_2_4))
+		__array(u32, mask_inner, MLX5_ST_SZ_DW(fte_match_set_lyr_2_4))
+		__array(u32, mask_misc, MLX5_ST_SZ_DW(fte_match_set_misc))
+	    ),
+	    TP_fast_assign(
+			   __entry->fg = fg;
+			   fs_get_obj(__entry->ft, fg->node.parent);
+			   __entry->start_index = fg->start_index;
+			   __entry->end_index = fg->start_index + fg->max_ftes;
+			   __entry->id = fg->id;
+			   __entry->mask_enable = fg->mask.match_criteria_enable;
+			   memcpy(__entry->mask_outer,
+				  MLX5_ADDR_OF(fte_match_param,
+					       &fg->mask.match_criteria,
+					       outer_headers),
+				  sizeof(__entry->mask_outer));
+			   memcpy(__entry->mask_inner,
+				  MLX5_ADDR_OF(fte_match_param,
+					       &fg->mask.match_criteria,
+					       inner_headers),
+				  sizeof(__entry->mask_inner));
+			   memcpy(__entry->mask_misc,
+				  MLX5_ADDR_OF(fte_match_param,
+					       &fg->mask.match_criteria,
+					       misc_parameters),
+				  sizeof(__entry->mask_misc));
+
+	    ),
+	    TP_printk("fg=%p ft=%p id=%u start=%u end=%u bit_mask=%02x %s\n",
+		      __entry->fg, __entry->ft, __entry->id,
+		      __entry->start_index, __entry->end_index,
+		      __entry->mask_enable,
+		      __parse_fs_hdrs(__entry->mask_enable,
+				      __entry->mask_outer,
+				      __entry->mask_misc,
+				      __entry->mask_inner,
+				      __entry->mask_outer,
+				      __entry->mask_misc,
+				      __entry->mask_inner))
+	    );
+
+TRACE_EVENT(mlx5_fs_del_fg,
+	    TP_PROTO(const struct mlx5_flow_group *fg),
+	    TP_ARGS(fg),
+	    TP_STRUCT__entry(
+		__field(const struct mlx5_flow_group *, fg)
+		__field(u32, id)
+	    ),
+	    TP_fast_assign(
+			   __entry->fg = fg;
+			   __entry->id = fg->id;
+
+	    ),
+	    TP_printk("fg=%p id=%u\n",
+		      __entry->fg, __entry->id)
+	    );
+
+#define ACTION_FLAGS \
+	{MLX5_FLOW_CONTEXT_ACTION_ALLOW,	 "ALLOW"},\
+	{MLX5_FLOW_CONTEXT_ACTION_DROP,		 "DROP"},\
+	{MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,	 "FWD"},\
+	{MLX5_FLOW_CONTEXT_ACTION_COUNT,	 "CNT"},\
+	{MLX5_FLOW_CONTEXT_ACTION_ENCAP,	 "ENCAP"},\
+	{MLX5_FLOW_CONTEXT_ACTION_DECAP,	 "DECAP"},\
+	{MLX5_FLOW_CONTEXT_ACTION_MOD_HDR,	 "MOD_HDR"},\
+	{MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO, "NEXT_PRIO"}
+
+TRACE_EVENT(mlx5_fs_set_fte,
+	    TP_PROTO(const struct fs_fte *fte, bool new_fte),
+	    TP_ARGS(fte, new_fte),
+	    TP_STRUCT__entry(
+		__field(const struct fs_fte *, fte)
+		__field(const struct mlx5_flow_group *, fg)
+		__field(u32, group_index)
+		__field(u32, index)
+		__field(u32, action)
+		__field(u32, flow_tag)
+		__field(u8,  mask_enable)
+		__field(bool, new_fte)
+		__array(u32, mask_outer, MLX5_ST_SZ_DW(fte_match_set_lyr_2_4))
+		__array(u32, mask_inner, MLX5_ST_SZ_DW(fte_match_set_lyr_2_4))
+		__array(u32, mask_misc, MLX5_ST_SZ_DW(fte_match_set_misc))
+		__array(u32, value_outer, MLX5_ST_SZ_DW(fte_match_set_lyr_2_4))
+		__array(u32, value_inner, MLX5_ST_SZ_DW(fte_match_set_lyr_2_4))
+		__array(u32, value_misc, MLX5_ST_SZ_DW(fte_match_set_misc))
+	    ),
+	    TP_fast_assign(
+			   __entry->fte = fte;
+			   __entry->new_fte = new_fte;
+			   fs_get_obj(__entry->fg, fte->node.parent);
+			   __entry->group_index = __entry->fg->id;
+			   __entry->index = fte->index;
+			   __entry->action = fte->action;
+			   __entry->mask_enable = __entry->fg->mask.match_criteria_enable;
+			   __entry->flow_tag = fte->flow_tag;
+			   memcpy(__entry->mask_outer,
+				  MLX5_ADDR_OF(fte_match_param,
+					       &__entry->fg->mask.match_criteria,
+					       outer_headers),
+				  sizeof(__entry->mask_outer));
+			   memcpy(__entry->mask_inner,
+				  MLX5_ADDR_OF(fte_match_param,
+					       &__entry->fg->mask.match_criteria,
+					       inner_headers),
+				  sizeof(__entry->mask_inner));
+			   memcpy(__entry->mask_misc,
+				  MLX5_ADDR_OF(fte_match_param,
+					       &__entry->fg->mask.match_criteria,
+					       misc_parameters),
+				  sizeof(__entry->mask_misc));
+			   memcpy(__entry->value_outer,
+				  MLX5_ADDR_OF(fte_match_param,
+					       &fte->val,
+					       outer_headers),
+				  sizeof(__entry->value_outer));
+			   memcpy(__entry->value_inner,
+				  MLX5_ADDR_OF(fte_match_param,
+					       &fte->val,
+					       inner_headers),
+				  sizeof(__entry->value_inner));
+			   memcpy(__entry->value_misc,
+				  MLX5_ADDR_OF(fte_match_param,
+					       &fte->val,
+					       misc_parameters),
+				  sizeof(__entry->value_misc));
+
+	    ),
+	    TP_printk("op=%s fte=%p fg=%p index=%u group_index=%u action=<%s> flow_tag=%x %s\n",
+		      __entry->new_fte ? "add" : "set",
+		      __entry->fte, __entry->fg, __entry->index,
+		      __entry->group_index, __print_flags(__entry->action, "|",
+							  ACTION_FLAGS),
+		      __entry->flow_tag,
+		      __parse_fs_hdrs(__entry->mask_enable,
+				      __entry->mask_outer,
+				      __entry->mask_misc,
+				      __entry->mask_inner,
+				      __entry->value_outer,
+				      __entry->value_misc,
+				      __entry->value_inner))
+	    );
+
+TRACE_EVENT(mlx5_fs_del_fte,
+	    TP_PROTO(const struct fs_fte *fte),
+	    TP_ARGS(fte),
+	    TP_STRUCT__entry(
+		__field(const struct fs_fte *, fte)
+		__field(u32, index)
+	    ),
+	    TP_fast_assign(
+			   __entry->fte = fte;
+			   __entry->index = fte->index;
+
+	    ),
+	    TP_printk("fte=%p index=%u\n",
+		      __entry->fte, __entry->index)
+	    );
+
+TRACE_EVENT(mlx5_fs_add_rule,
+	    TP_PROTO(const struct mlx5_flow_rule *rule),
+	    TP_ARGS(rule),
+	    TP_STRUCT__entry(
+		__field(const struct mlx5_flow_rule *, rule)
+		__field(const struct fs_fte *, fte)
+		__field(u32, sw_action)
+		__field(u32, index)
+		__field(u32, counter_id)
+		__array(u8, destination, sizeof(struct mlx5_flow_destination))
+	    ),
+	    TP_fast_assign(
+			   __entry->rule = rule;
+			   fs_get_obj(__entry->fte, rule->node.parent);
+			   __entry->index = __entry->fte->dests_size - 1;
+			   __entry->sw_action = rule->sw_action;
+			   memcpy(__entry->destination,
+				  &rule->dest_attr,
+				  sizeof(__entry->destination));
+			   if (rule->dest_attr.type & MLX5_FLOW_DESTINATION_TYPE_COUNTER &&
+			       rule->dest_attr.counter)
+				__entry->counter_id =
+				rule->dest_attr.counter->id;
+	    ),
+	    TP_printk("rule=%p fte=%p index=%u sw_action=<%s> [dst] %s\n",
+		      __entry->rule, __entry->fte, __entry->index,
+		      __print_flags(__entry->sw_action, "|", ACTION_FLAGS),
+		      __parse_fs_dst(__entry->destination, __entry->counter_id))
+	    );
+
+TRACE_EVENT(mlx5_fs_del_rule,
+	    TP_PROTO(const struct mlx5_flow_rule *rule),
+	    TP_ARGS(rule),
+	    TP_STRUCT__entry(
+		__field(const struct mlx5_flow_rule *, rule)
+		__field(const struct fs_fte *, fte)
+	    ),
+	    TP_fast_assign(
+			   __entry->rule = rule;
+			   fs_get_obj(__entry->fte, rule->node.parent);
+	    ),
+	    TP_printk("rule=%p fte=%p\n",
+		      __entry->rule, __entry->fte)
+	    );
+#endif
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH ./diag
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE fs_tracepoint
+#include <trace/define_trace.h>
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
index 0039b4725405..cc13d3dbd366 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -263,6 +263,7 @@ struct mlx5e_dcbx {
 
 	/* The only setting that cannot be read from FW */
 	u8                         tc_tsa[IEEE_8021QAZ_MAX_TCS];
+	u8                         cap;
 };
 #endif
 
@@ -290,10 +291,11 @@ struct mlx5e_tstamp {
 
 enum {
 	MLX5E_RQ_STATE_ENABLED,
-	MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS,
 	MLX5E_RQ_STATE_AM,
 };
 
+#define MLX5E_TEST_BIT(state, nr) (state & BIT(nr))
+
 struct mlx5e_cq {
 	/* data path - accessed per cqe */
 	struct mlx5_cqwq           wq;
@@ -341,7 +343,6 @@ enum {
 
 struct mlx5e_sq_wqe_info {
 	u8  opcode;
-	u8  num_wqebbs;
 };
 
 struct mlx5e_txqsq {
@@ -417,13 +418,8 @@ struct mlx5e_xdpsq {
 struct mlx5e_icosq {
 	/* data path */
 
-	/* dirtied @completion */
-	u16                        cc;
-
 	/* dirtied @xmit */
 	u16                        pc ____cacheline_aligned_in_smp;
-	u32                        dma_fifo_pc;
-	u16                        prev_cc;
 
 	struct mlx5e_cq            cq;
 
@@ -437,7 +433,6 @@ struct mlx5e_icosq {
 	void __iomem              *uar_map;
 	u32                        sqn;
 	u16                        edge;
-	struct device             *pdev;
 	__be32                     mkey_be;
 	unsigned long              state;
 
@@ -506,7 +501,7 @@ struct mlx5e_rx_am { /* Adaptive Moderation */
  */
 #define MLX5E_CACHE_UNIT	(MLX5_MPWRQ_PAGES_PER_WQE > NAPI_POLL_WEIGHT ? \
 				 MLX5_MPWRQ_PAGES_PER_WQE : NAPI_POLL_WEIGHT)
-#define MLX5E_CACHE_SIZE	(2 * roundup_pow_of_two(MLX5E_CACHE_UNIT))
+#define MLX5E_CACHE_SIZE	(4 * roundup_pow_of_two(MLX5E_CACHE_UNIT))
 struct mlx5e_page_cache {
 	u32 head;
 	u32 tail;
@@ -515,7 +510,7 @@ struct mlx5e_page_cache {
 
 struct mlx5e_rq;
 typedef void (*mlx5e_fp_handle_rx_cqe)(struct mlx5e_rq*, struct mlx5_cqe64*);
-typedef int (*mlx5e_fp_alloc_wqe)(struct mlx5e_rq*, struct mlx5e_rx_wqe*, u16);
+typedef bool (*mlx5e_fp_post_rx_wqes)(struct mlx5e_rq *rq);
 typedef void (*mlx5e_fp_dealloc_wqe)(struct mlx5e_rq*, u16);
 
 struct mlx5e_rq {
@@ -526,21 +521,26 @@ struct mlx5e_rq {
 		struct {
 			struct mlx5e_wqe_frag_info *frag_info;
 			u32 frag_sz;	/* max possible skb frag_sz */
-			bool page_reuse;
-			bool xdp_xmit;
+			union {
+				bool page_reuse;
+				bool xdp_xmit;
+			};
 		} wqe;
 		struct {
 			struct mlx5e_mpw_info *info;
 			void                  *mtt_no_align;
+			u16                    num_strides;
+			u8                     log_stride_sz;
+			bool                   umr_in_progress;
 		} mpwqe;
 	};
 	struct {
+		u16            headroom;
 		u8             page_order;
-		u32            wqe_sz;    /* wqe data buffer size */
 		u8             map_dir;   /* dma map direction */
 	} buff;
-	__be32                 mkey_be;
 
+	struct mlx5e_channel  *channel;
 	struct device         *pdev;
 	struct net_device     *netdev;
 	struct mlx5e_tstamp   *tstamp;
@@ -549,12 +549,11 @@ struct mlx5e_rq {
 	struct mlx5e_page_cache page_cache;
 
 	mlx5e_fp_handle_rx_cqe handle_rx_cqe;
-	mlx5e_fp_alloc_wqe     alloc_wqe;
+	mlx5e_fp_post_rx_wqes  post_wqes;
 	mlx5e_fp_dealloc_wqe   dealloc_wqe;
 
 	unsigned long          state;
 	int                    ix;
-	u16                    rx_headroom;
 
 	struct mlx5e_rx_am     am; /* Adaptive Moderation */
 
@@ -564,19 +563,13 @@ struct mlx5e_rq {
 
 	/* control */
 	struct mlx5_wq_ctrl    wq_ctrl;
+	__be32                 mkey_be;
 	u8                     wq_type;
-	u32                    mpwqe_stride_sz;
-	u32                    mpwqe_num_strides;
 	u32                    rqn;
-	struct mlx5e_channel  *channel;
 	struct mlx5_core_dev  *mdev;
 	struct mlx5_core_mkey  umr_mkey;
 } ____cacheline_aligned_in_smp;
 
-enum channel_flags {
-	MLX5E_CHANNEL_NAPI_SCHED = 1,
-};
-
 struct mlx5e_channel {
 	/* data path */
 	struct mlx5e_rq            rq;
@@ -588,14 +581,15 @@ struct mlx5e_channel {
 	struct net_device         *netdev;
 	__be32                     mkey_be;
 	u8                         num_tc;
-	unsigned long              flags;
+
+	/* data path - accessed per napi poll */
+	struct irq_desc *irq_desc;
 
 	/* control */
 	struct mlx5e_priv         *priv;
 	struct mlx5_core_dev      *mdev;
 	struct mlx5e_tstamp       *tstamp;
 	int                        ix;
-	int                        cpu;
 };
 
 struct mlx5e_channels {
@@ -620,6 +614,12 @@ enum mlx5e_traffic_types {
 	MLX5E_NUM_INDIR_TIRS = MLX5E_TT_ANY,
 };
 
+enum mlx5e_tunnel_types {
+	MLX5E_TT_IPV4_GRE,
+	MLX5E_TT_IPV6_GRE,
+	MLX5E_NUM_TUNNEL_TT,
+};
+
 enum {
 	MLX5E_STATE_ASYNC_EVENTS_ENABLED,
 	MLX5E_STATE_OPENED,
@@ -679,6 +679,7 @@ struct mlx5e_l2_table {
 struct mlx5e_ttc_table {
 	struct mlx5e_flow_table  ft;
 	struct mlx5_flow_handle	 *rules[MLX5E_NUM_TT];
+	struct mlx5_flow_handle  *tunnel_rules[MLX5E_NUM_TUNNEL_TT];
 };
 
 #define ARFS_HASH_SHIFT BITS_PER_BYTE
@@ -711,6 +712,7 @@ enum {
 	MLX5E_VLAN_FT_LEVEL = 0,
 	MLX5E_L2_FT_LEVEL,
 	MLX5E_TTC_FT_LEVEL,
+	MLX5E_INNER_TTC_FT_LEVEL,
 	MLX5E_ARFS_FT_LEVEL
 };
 
@@ -736,6 +738,7 @@ struct mlx5e_flow_steering {
 	struct mlx5e_vlan_table         vlan;
 	struct mlx5e_l2_table           l2;
 	struct mlx5e_ttc_table          ttc;
+	struct mlx5e_ttc_table          inner_ttc;
 	struct mlx5e_arfs_tables        arfs;
 };
 
@@ -769,6 +772,7 @@ struct mlx5e_priv {
 	u32                        tisn[MLX5E_MAX_NUM_TC];
 	struct mlx5e_rqt           indir_rqt;
 	struct mlx5e_tir           indir_tir[MLX5E_NUM_INDIR_TIRS];
+	struct mlx5e_tir           inner_indir_tir[MLX5E_NUM_INDIR_TIRS];
 	struct mlx5e_tir           direct_tir[MLX5E_MAX_NUM_CHANNELS];
 	u32                        tx_rates[MLX5E_MAX_NUM_SQS];
 	int                        hard_mtu;
@@ -839,11 +843,9 @@ void mlx5e_page_release(struct mlx5e_rq *rq, struct mlx5e_dma_info *dma_info,
 void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
 void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
 bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq);
-int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe *wqe, u16 ix);
-int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe *wqe,	u16 ix);
+bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq);
 void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix);
 void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix);
-void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq);
 void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi);
 
 void mlx5e_rx_am(struct mlx5e_rq *rq);
@@ -903,7 +905,7 @@ int mlx5e_redirect_rqt(struct mlx5e_priv *priv, u32 rqtn, int sz,
 		       struct mlx5e_redirect_rqt_param rrp);
 void mlx5e_build_indir_tir_ctx_hash(struct mlx5e_params *params,
 				    enum mlx5e_traffic_types tt,
-				    void *tirc);
+				    void *tirc, bool inner);
 
 int mlx5e_open_locked(struct net_device *netdev);
 int mlx5e_close_locked(struct net_device *netdev);
@@ -922,8 +924,7 @@ void mlx5e_switch_priv_channels(struct mlx5e_priv *priv,
 void mlx5e_activate_priv_channels(struct mlx5e_priv *priv);
 void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv);
 
-void mlx5e_build_default_indir_rqt(struct mlx5_core_dev *mdev,
-				   u32 *indirection_rqt, int len,
+void mlx5e_build_default_indir_rqt(u32 *indirection_rqt, int len,
 				   int num_channels);
 int mlx5e_get_max_linkspeed(struct mlx5_core_dev *mdev, u32 *speed);
 
@@ -932,6 +933,12 @@ void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params,
 void mlx5e_set_rq_type_params(struct mlx5_core_dev *mdev,
 			      struct mlx5e_params *params, u8 rq_type);
 
+static inline bool mlx5e_tunnel_inner_ft_supported(struct mlx5_core_dev *mdev)
+{
+	return (MLX5_CAP_ETH(mdev, tunnel_stateless_gre) &&
+		MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ft_field_support.inner_ip_version));
+}
+
 static inline
 struct mlx5e_tx_wqe *mlx5e_post_nop(struct mlx5_wq_cyc *wq, u32 sqn, u16 *pc)
 {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c b/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c
index 2eb54d36e16e..c1d384fca4dc 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c
@@ -288,13 +288,8 @@ static int mlx5e_dcbnl_ieee_setpfc(struct net_device *dev,
 static u8 mlx5e_dcbnl_getdcbx(struct net_device *dev)
 {
 	struct mlx5e_priv *priv = netdev_priv(dev);
-	struct mlx5e_dcbx *dcbx = &priv->dcbx;
-	u8 mode = DCB_CAP_DCBX_VER_IEEE | DCB_CAP_DCBX_VER_CEE;
-
-	if (dcbx->mode == MLX5E_DCBX_PARAM_VER_OPER_HOST)
-		mode |= DCB_CAP_DCBX_HOST;
 
-	return mode;
+	return priv->dcbx.cap;
 }
 
 static u8 mlx5e_dcbnl_setdcbx(struct net_device *dev, u8 mode)
@@ -312,6 +307,7 @@ static u8 mlx5e_dcbnl_setdcbx(struct net_device *dev, u8 mode)
 		/* set dcbx to fw controlled */
 		if (!mlx5e_dcbnl_set_dcbx_mode(priv, MLX5E_DCBX_PARAM_VER_OPER_AUTO)) {
 			dcbx->mode = MLX5E_DCBX_PARAM_VER_OPER_AUTO;
+			dcbx->cap &= ~DCB_CAP_DCBX_HOST;
 			return 0;
 		}
 
@@ -324,6 +320,8 @@ static u8 mlx5e_dcbnl_setdcbx(struct net_device *dev, u8 mode)
 	if (mlx5e_dcbnl_switch_to_host_mode(netdev_priv(dev)))
 		return 1;
 
+	dcbx->cap = mode;
+
 	return 0;
 }
 
@@ -628,9 +626,9 @@ static u8 mlx5e_dcbnl_getcap(struct net_device *netdev,
 		*cap = false;
 		break;
 	case DCB_CAP_ATTR_DCBX:
-		*cap = (DCB_CAP_DCBX_LLD_MANAGED |
-			DCB_CAP_DCBX_VER_CEE |
-			DCB_CAP_DCBX_STATIC);
+		*cap = priv->dcbx.cap |
+		       DCB_CAP_DCBX_VER_CEE |
+		       DCB_CAP_DCBX_VER_IEEE;
 		break;
 	default:
 		*cap = 0;
@@ -754,8 +752,16 @@ void mlx5e_dcbnl_initialize(struct mlx5e_priv *priv)
 {
 	struct mlx5e_dcbx *dcbx = &priv->dcbx;
 
+	if (!MLX5_CAP_GEN(priv->mdev, qos))
+		return;
+
 	if (MLX5_CAP_GEN(priv->mdev, dcbx))
 		mlx5e_dcbnl_query_dcbx_mode(priv, &dcbx->mode);
 
+	priv->dcbx.cap = DCB_CAP_DCBX_VER_CEE |
+			 DCB_CAP_DCBX_VER_IEEE;
+	if (priv->dcbx.mode == MLX5E_DCBX_PARAM_VER_OPER_HOST)
+		priv->dcbx.cap |= DCB_CAP_DCBX_HOST;
+
 	mlx5e_ets_init(priv);
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
index 917fade5f5d5..d12e9fc0d76b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
@@ -176,7 +176,6 @@ static bool mlx5e_query_global_pause_combined(struct mlx5e_priv *priv)
 
 int mlx5e_ethtool_get_sset_count(struct mlx5e_priv *priv, int sset)
 {
-
 	switch (sset) {
 	case ETH_SS_STATS:
 		return NUM_SW_COUNTERS +
@@ -207,7 +206,7 @@ static int mlx5e_get_sset_count(struct net_device *dev, int sset)
 	return mlx5e_ethtool_get_sset_count(priv, sset);
 }
 
-static void mlx5e_fill_stats_strings(struct mlx5e_priv *priv, uint8_t *data)
+static void mlx5e_fill_stats_strings(struct mlx5e_priv *priv, u8 *data)
 {
 	int i, j, tc, prio, idx = 0;
 	unsigned long pfc_combined;
@@ -242,10 +241,22 @@ static void mlx5e_fill_stats_strings(struct mlx5e_priv *priv, uint8_t *data)
 		strcpy(data + (idx++) * ETH_GSTRING_LEN,
 		       pport_phy_statistical_stats_desc[i].format);
 
+	for (i = 0; i < NUM_PPORT_ETH_EXT_COUNTERS(priv); i++)
+		strcpy(data + (idx++) * ETH_GSTRING_LEN,
+		       pport_eth_ext_stats_desc[i].format);
+
 	for (i = 0; i < NUM_PCIE_PERF_COUNTERS(priv); i++)
 		strcpy(data + (idx++) * ETH_GSTRING_LEN,
 		       pcie_perf_stats_desc[i].format);
 
+	for (i = 0; i < NUM_PCIE_PERF_COUNTERS64(priv); i++)
+		strcpy(data + (idx++) * ETH_GSTRING_LEN,
+		       pcie_perf_stats_desc64[i].format);
+
+	for (i = 0; i < NUM_PCIE_PERF_STALL_COUNTERS(priv); i++)
+		strcpy(data + (idx++) * ETH_GSTRING_LEN,
+		       pcie_perf_stall_stats_desc[i].format);
+
 	for (prio = 0; prio < NUM_PPORT_PRIO; prio++) {
 		for (i = 0; i < NUM_PPORT_PER_PRIO_TRAFFIC_COUNTERS; i++)
 			sprintf(data + (idx++) * ETH_GSTRING_LEN,
@@ -297,8 +308,7 @@ static void mlx5e_fill_stats_strings(struct mlx5e_priv *priv, uint8_t *data)
 					priv->channel_tc2txq[i][tc]);
 }
 
-void mlx5e_ethtool_get_strings(struct mlx5e_priv *priv,
-			       uint32_t stringset, uint8_t *data)
+void mlx5e_ethtool_get_strings(struct mlx5e_priv *priv, u32 stringset, u8 *data)
 {
 	int i;
 
@@ -320,8 +330,7 @@ void mlx5e_ethtool_get_strings(struct mlx5e_priv *priv,
 	}
 }
 
-static void mlx5e_get_strings(struct net_device *dev,
-			      uint32_t stringset, uint8_t *data)
+static void mlx5e_get_strings(struct net_device *dev, u32 stringset, u8 *data)
 {
 	struct mlx5e_priv *priv = netdev_priv(dev);
 
@@ -373,10 +382,22 @@ void mlx5e_ethtool_get_ethtool_stats(struct mlx5e_priv *priv,
 		data[idx++] = MLX5E_READ_CTR64_BE(&priv->stats.pport.phy_statistical_counters,
 						  pport_phy_statistical_stats_desc, i);
 
+	for (i = 0; i < NUM_PPORT_ETH_EXT_COUNTERS(priv); i++)
+		data[idx++] = MLX5E_READ_CTR64_BE(&priv->stats.pport.eth_ext_counters,
+						  pport_eth_ext_stats_desc, i);
+
 	for (i = 0; i < NUM_PCIE_PERF_COUNTERS(priv); i++)
 		data[idx++] = MLX5E_READ_CTR32_BE(&priv->stats.pcie.pcie_perf_counters,
 						  pcie_perf_stats_desc, i);
 
+	for (i = 0; i < NUM_PCIE_PERF_COUNTERS64(priv); i++)
+		data[idx++] = MLX5E_READ_CTR64_BE(&priv->stats.pcie.pcie_perf_counters,
+						  pcie_perf_stats_desc64, i);
+
+	for (i = 0; i < NUM_PCIE_PERF_STALL_COUNTERS(priv); i++)
+		data[idx++] = MLX5E_READ_CTR32_BE(&priv->stats.pcie.pcie_perf_counters,
+						  pcie_perf_stall_stats_desc, i);
+
 	for (prio = 0; prio < NUM_PPORT_PRIO; prio++) {
 		for (i = 0; i < NUM_PPORT_PER_PRIO_TRAFFIC_COUNTERS; i++)
 			data[idx++] = MLX5E_READ_CTR64_BE(&priv->stats.pport.per_prio_counters[prio],
@@ -641,8 +662,9 @@ int mlx5e_ethtool_set_channels(struct mlx5e_priv *priv,
 
 	new_channels.params = priv->channels.params;
 	new_channels.params.num_channels = count;
-	mlx5e_build_default_indir_rqt(priv->mdev, new_channels.params.indirection_rqt,
-				      MLX5E_INDIR_RQT_SIZE, count);
+	if (!netif_is_rxfh_configured(priv->netdev))
+		mlx5e_build_default_indir_rqt(new_channels.params.indirection_rqt,
+					      MLX5E_INDIR_RQT_SIZE, count);
 
 	if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
 		priv->channels.params = new_channels.params;
@@ -964,24 +986,27 @@ static u8 get_connector_port(u32 eth_proto, u8 connector_type)
 	if (connector_type && connector_type < MLX5E_CONNECTOR_TYPE_NUMBER)
 		return ptys2connector_type[connector_type];
 
-	if (eth_proto & (MLX5E_PROT_MASK(MLX5E_10GBASE_SR)
-			 | MLX5E_PROT_MASK(MLX5E_40GBASE_SR4)
-			 | MLX5E_PROT_MASK(MLX5E_100GBASE_SR4)
-			 | MLX5E_PROT_MASK(MLX5E_1000BASE_CX_SGMII))) {
-			return PORT_FIBRE;
+	if (eth_proto &
+	    (MLX5E_PROT_MASK(MLX5E_10GBASE_SR)   |
+	     MLX5E_PROT_MASK(MLX5E_40GBASE_SR4)  |
+	     MLX5E_PROT_MASK(MLX5E_100GBASE_SR4) |
+	     MLX5E_PROT_MASK(MLX5E_1000BASE_CX_SGMII))) {
+		return PORT_FIBRE;
 	}
 
-	if (eth_proto & (MLX5E_PROT_MASK(MLX5E_40GBASE_CR4)
-			 | MLX5E_PROT_MASK(MLX5E_10GBASE_CR)
-			 | MLX5E_PROT_MASK(MLX5E_100GBASE_CR4))) {
-			return PORT_DA;
+	if (eth_proto &
+	    (MLX5E_PROT_MASK(MLX5E_40GBASE_CR4) |
+	     MLX5E_PROT_MASK(MLX5E_10GBASE_CR)  |
+	     MLX5E_PROT_MASK(MLX5E_100GBASE_CR4))) {
+		return PORT_DA;
 	}
 
-	if (eth_proto & (MLX5E_PROT_MASK(MLX5E_10GBASE_KX4)
-			 | MLX5E_PROT_MASK(MLX5E_10GBASE_KR)
-			 | MLX5E_PROT_MASK(MLX5E_40GBASE_KR4)
-			 | MLX5E_PROT_MASK(MLX5E_100GBASE_KR4))) {
-			return PORT_NONE;
+	if (eth_proto &
+	    (MLX5E_PROT_MASK(MLX5E_10GBASE_KX4) |
+	     MLX5E_PROT_MASK(MLX5E_10GBASE_KR)  |
+	     MLX5E_PROT_MASK(MLX5E_40GBASE_KR4) |
+	     MLX5E_PROT_MASK(MLX5E_100GBASE_KR4))) {
+		return PORT_NONE;
 	}
 
 	return PORT_OTHER;
@@ -1188,9 +1213,18 @@ static void mlx5e_modify_tirs_hash(struct mlx5e_priv *priv, void *in, int inlen)
 
 	for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
 		memset(tirc, 0, ctxlen);
-		mlx5e_build_indir_tir_ctx_hash(&priv->channels.params, tt, tirc);
+		mlx5e_build_indir_tir_ctx_hash(&priv->channels.params, tt, tirc, false);
 		mlx5_core_modify_tir(mdev, priv->indir_tir[tt].tirn, in, inlen);
 	}
+
+	if (!mlx5e_tunnel_inner_ft_supported(priv->mdev))
+		return;
+
+	for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
+		memset(tirc, 0, ctxlen);
+		mlx5e_build_indir_tir_ctx_hash(&priv->channels.params, tt, tirc, true);
+		mlx5_core_modify_tir(mdev, priv->inner_indir_tir[tt].tirn, in, inlen);
+	}
 }
 
 static int mlx5e_set_rxfh(struct net_device *dev, const u32 *indir,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
index dfccb5305e9c..f11fd07ac4dd 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
@@ -36,6 +36,7 @@
 #include <linux/tcp.h>
 #include <linux/mlx5/fs.h>
 #include "en.h"
+#include "lib/mpfs.h"
 
 static int mlx5e_add_l2_flow_rule(struct mlx5e_priv *priv,
 				  struct mlx5e_l2_rule *ai, int type);
@@ -65,6 +66,7 @@ struct mlx5e_l2_hash_node {
 	struct hlist_node          hlist;
 	u8                         action;
 	struct mlx5e_l2_rule ai;
+	bool   mpfs;
 };
 
 static inline int mlx5e_hash_l2(u8 *addr)
@@ -362,17 +364,30 @@ static void mlx5e_del_vlan_rules(struct mlx5e_priv *priv)
 static void mlx5e_execute_l2_action(struct mlx5e_priv *priv,
 				    struct mlx5e_l2_hash_node *hn)
 {
-	switch (hn->action) {
+	u8 action = hn->action;
+	int l2_err = 0;
+
+	switch (action) {
 	case MLX5E_ACTION_ADD:
 		mlx5e_add_l2_flow_rule(priv, &hn->ai, MLX5E_FULLMATCH);
+		if (!is_multicast_ether_addr(hn->ai.addr)) {
+			l2_err = mlx5_mpfs_add_mac(priv->mdev, hn->ai.addr);
+			hn->mpfs = !l2_err;
+		}
 		hn->action = MLX5E_ACTION_NONE;
 		break;
 
 	case MLX5E_ACTION_DEL:
+		if (!is_multicast_ether_addr(hn->ai.addr) && hn->mpfs)
+			l2_err = mlx5_mpfs_del_mac(priv->mdev, hn->ai.addr);
 		mlx5e_del_l2_flow_rule(priv, &hn->ai);
 		mlx5e_del_l2_from_hash(hn);
 		break;
 	}
+
+	if (l2_err)
+		netdev_warn(priv->netdev, "MPFS, failed to %s mac %pM, err(%d)\n",
+			    action == MLX5E_ACTION_ADD ? "add" : "del", hn->ai.addr, l2_err);
 }
 
 static void mlx5e_sync_netdev_addr(struct mlx5e_priv *priv)
@@ -593,12 +608,21 @@ static void mlx5e_cleanup_ttc_rules(struct mlx5e_ttc_table *ttc)
 			ttc->rules[i] = NULL;
 		}
 	}
+
+	for (i = 0; i < MLX5E_NUM_TUNNEL_TT; i++) {
+		if (!IS_ERR_OR_NULL(ttc->tunnel_rules[i])) {
+			mlx5_del_flow_rules(ttc->tunnel_rules[i]);
+			ttc->tunnel_rules[i] = NULL;
+		}
+	}
 }
 
-static struct {
+struct mlx5e_etype_proto {
 	u16 etype;
 	u8 proto;
-} ttc_rules[] = {
+};
+
+static struct mlx5e_etype_proto ttc_rules[] = {
 	[MLX5E_TT_IPV4_TCP] = {
 		.etype = ETH_P_IP,
 		.proto = IPPROTO_TCP,
@@ -645,6 +669,28 @@ static struct {
 	},
 };
 
+static struct mlx5e_etype_proto ttc_tunnel_rules[] = {
+	[MLX5E_TT_IPV4_GRE] = {
+		.etype = ETH_P_IP,
+		.proto = IPPROTO_GRE,
+	},
+	[MLX5E_TT_IPV6_GRE] = {
+		.etype = ETH_P_IPV6,
+		.proto = IPPROTO_GRE,
+	},
+};
+
+static u8 mlx5e_etype_to_ipv(u16 ethertype)
+{
+	if (ethertype == ETH_P_IP)
+		return 4;
+
+	if (ethertype == ETH_P_IPV6)
+		return 6;
+
+	return 0;
+}
+
 static struct mlx5_flow_handle *
 mlx5e_generate_ttc_rule(struct mlx5e_priv *priv,
 			struct mlx5_flow_table *ft,
@@ -652,10 +698,12 @@ mlx5e_generate_ttc_rule(struct mlx5e_priv *priv,
 			u16 etype,
 			u8 proto)
 {
+	int match_ipv_outer = MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version);
 	MLX5_DECLARE_FLOW_ACT(flow_act);
 	struct mlx5_flow_handle *rule;
 	struct mlx5_flow_spec *spec;
 	int err = 0;
+	u8 ipv;
 
 	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
 	if (!spec)
@@ -666,7 +714,13 @@ mlx5e_generate_ttc_rule(struct mlx5e_priv *priv,
 		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
 		MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, proto);
 	}
-	if (etype) {
+
+	ipv = mlx5e_etype_to_ipv(etype);
+	if (match_ipv_outer && ipv) {
+		spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
+		MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, ipv);
+	} else if (etype) {
 		spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
 		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype);
 		MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, etype);
@@ -708,6 +762,20 @@ static int mlx5e_generate_ttc_table_rules(struct mlx5e_priv *priv)
 			goto del_rules;
 	}
 
+	if (!mlx5e_tunnel_inner_ft_supported(priv->mdev))
+		return 0;
+
+	rules     = ttc->tunnel_rules;
+	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+	dest.ft   = priv->fs.inner_ttc.ft.t;
+	for (tt = 0; tt < MLX5E_NUM_TUNNEL_TT; tt++) {
+		rules[tt] = mlx5e_generate_ttc_rule(priv, ft, &dest,
+						    ttc_tunnel_rules[tt].etype,
+						    ttc_tunnel_rules[tt].proto);
+		if (IS_ERR(rules[tt]))
+			goto del_rules;
+	}
+
 	return 0;
 
 del_rules:
@@ -718,13 +786,23 @@ del_rules:
 }
 
 #define MLX5E_TTC_NUM_GROUPS	3
-#define MLX5E_TTC_GROUP1_SIZE	BIT(3)
-#define MLX5E_TTC_GROUP2_SIZE	BIT(1)
-#define MLX5E_TTC_GROUP3_SIZE	BIT(0)
+#define MLX5E_TTC_GROUP1_SIZE	(BIT(3) + MLX5E_NUM_TUNNEL_TT)
+#define MLX5E_TTC_GROUP2_SIZE	 BIT(1)
+#define MLX5E_TTC_GROUP3_SIZE	 BIT(0)
 #define MLX5E_TTC_TABLE_SIZE	(MLX5E_TTC_GROUP1_SIZE +\
 				 MLX5E_TTC_GROUP2_SIZE +\
 				 MLX5E_TTC_GROUP3_SIZE)
-static int mlx5e_create_ttc_table_groups(struct mlx5e_ttc_table *ttc)
+
+#define MLX5E_INNER_TTC_NUM_GROUPS	3
+#define MLX5E_INNER_TTC_GROUP1_SIZE	BIT(3)
+#define MLX5E_INNER_TTC_GROUP2_SIZE	BIT(1)
+#define MLX5E_INNER_TTC_GROUP3_SIZE	BIT(0)
+#define MLX5E_INNER_TTC_TABLE_SIZE	(MLX5E_INNER_TTC_GROUP1_SIZE +\
+					 MLX5E_INNER_TTC_GROUP2_SIZE +\
+					 MLX5E_INNER_TTC_GROUP3_SIZE)
+
+static int mlx5e_create_ttc_table_groups(struct mlx5e_ttc_table *ttc,
+					 bool use_ipv)
 {
 	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
 	struct mlx5e_flow_table *ft = &ttc->ft;
@@ -746,7 +824,10 @@ static int mlx5e_create_ttc_table_groups(struct mlx5e_ttc_table *ttc)
 	/* L4 Group */
 	mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
 	MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol);
-	MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
+	if (use_ipv)
+		MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_version);
+	else
+		MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
 	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
 	MLX5_SET_CFG(in, start_flow_index, ix);
 	ix += MLX5E_TTC_GROUP1_SIZE;
@@ -787,6 +868,190 @@ err:
 	return err;
 }
 
+static struct mlx5_flow_handle *
+mlx5e_generate_inner_ttc_rule(struct mlx5e_priv *priv,
+			      struct mlx5_flow_table *ft,
+			      struct mlx5_flow_destination *dest,
+			      u16 etype, u8 proto)
+{
+	MLX5_DECLARE_FLOW_ACT(flow_act);
+	struct mlx5_flow_handle *rule;
+	struct mlx5_flow_spec *spec;
+	int err = 0;
+	u8 ipv;
+
+	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+	if (!spec)
+		return ERR_PTR(-ENOMEM);
+
+	ipv = mlx5e_etype_to_ipv(etype);
+	if (etype && ipv) {
+		spec->match_criteria_enable = MLX5_MATCH_INNER_HEADERS;
+		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, inner_headers.ip_version);
+		MLX5_SET(fte_match_param, spec->match_value, inner_headers.ip_version, ipv);
+	}
+
+	if (proto) {
+		spec->match_criteria_enable = MLX5_MATCH_INNER_HEADERS;
+		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, inner_headers.ip_protocol);
+		MLX5_SET(fte_match_param, spec->match_value, inner_headers.ip_protocol, proto);
+	}
+
+	rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, 1);
+	if (IS_ERR(rule)) {
+		err = PTR_ERR(rule);
+		netdev_err(priv->netdev, "%s: add rule failed\n", __func__);
+	}
+
+	kvfree(spec);
+	return err ? ERR_PTR(err) : rule;
+}
+
+static int mlx5e_generate_inner_ttc_table_rules(struct mlx5e_priv *priv)
+{
+	struct mlx5_flow_destination dest;
+	struct mlx5_flow_handle **rules;
+	struct mlx5e_ttc_table *ttc;
+	struct mlx5_flow_table *ft;
+	int err;
+	int tt;
+
+	ttc =  &priv->fs.inner_ttc;
+	ft = ttc->ft.t;
+	rules = ttc->rules;
+
+	dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
+	for (tt = 0; tt < MLX5E_NUM_TT; tt++) {
+		if (tt == MLX5E_TT_ANY)
+			dest.tir_num = priv->direct_tir[0].tirn;
+		else
+			dest.tir_num = priv->inner_indir_tir[tt].tirn;
+
+		rules[tt] = mlx5e_generate_inner_ttc_rule(priv, ft, &dest,
+							  ttc_rules[tt].etype,
+							  ttc_rules[tt].proto);
+		if (IS_ERR(rules[tt]))
+			goto del_rules;
+	}
+
+	return 0;
+
+del_rules:
+	err = PTR_ERR(rules[tt]);
+	rules[tt] = NULL;
+	mlx5e_cleanup_ttc_rules(ttc);
+	return err;
+}
+
+static int mlx5e_create_inner_ttc_table_groups(struct mlx5e_ttc_table *ttc)
+{
+	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+	struct mlx5e_flow_table *ft = &ttc->ft;
+	int ix = 0;
+	u32 *in;
+	int err;
+	u8 *mc;
+
+	ft->g = kcalloc(MLX5E_INNER_TTC_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL);
+	if (!ft->g)
+		return -ENOMEM;
+	in = kvzalloc(inlen, GFP_KERNEL);
+	if (!in) {
+		kfree(ft->g);
+		return -ENOMEM;
+	}
+
+	/* L4 Group */
+	mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
+	MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ip_protocol);
+	MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ip_version);
+	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_INNER_HEADERS);
+	MLX5_SET_CFG(in, start_flow_index, ix);
+	ix += MLX5E_INNER_TTC_GROUP1_SIZE;
+	MLX5_SET_CFG(in, end_flow_index, ix - 1);
+	ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
+	if (IS_ERR(ft->g[ft->num_groups]))
+		goto err;
+	ft->num_groups++;
+
+	/* L3 Group */
+	MLX5_SET(fte_match_param, mc, inner_headers.ip_protocol, 0);
+	MLX5_SET_CFG(in, start_flow_index, ix);
+	ix += MLX5E_INNER_TTC_GROUP2_SIZE;
+	MLX5_SET_CFG(in, end_flow_index, ix - 1);
+	ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
+	if (IS_ERR(ft->g[ft->num_groups]))
+		goto err;
+	ft->num_groups++;
+
+	/* Any Group */
+	memset(in, 0, inlen);
+	MLX5_SET_CFG(in, start_flow_index, ix);
+	ix += MLX5E_INNER_TTC_GROUP3_SIZE;
+	MLX5_SET_CFG(in, end_flow_index, ix - 1);
+	ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
+	if (IS_ERR(ft->g[ft->num_groups]))
+		goto err;
+	ft->num_groups++;
+
+	kvfree(in);
+	return 0;
+
+err:
+	err = PTR_ERR(ft->g[ft->num_groups]);
+	ft->g[ft->num_groups] = NULL;
+	kvfree(in);
+
+	return err;
+}
+
+static int mlx5e_create_inner_ttc_table(struct mlx5e_priv *priv)
+{
+	struct mlx5e_ttc_table *ttc = &priv->fs.inner_ttc;
+	struct mlx5_flow_table_attr ft_attr = {};
+	struct mlx5e_flow_table *ft = &ttc->ft;
+	int err;
+
+	if (!mlx5e_tunnel_inner_ft_supported(priv->mdev))
+		return 0;
+
+	ft_attr.max_fte = MLX5E_INNER_TTC_TABLE_SIZE;
+	ft_attr.level   = MLX5E_INNER_TTC_FT_LEVEL;
+	ft_attr.prio    = MLX5E_NIC_PRIO;
+
+	ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr);
+	if (IS_ERR(ft->t)) {
+		err = PTR_ERR(ft->t);
+		ft->t = NULL;
+		return err;
+	}
+
+	err = mlx5e_create_inner_ttc_table_groups(ttc);
+	if (err)
+		goto err;
+
+	err = mlx5e_generate_inner_ttc_table_rules(priv);
+	if (err)
+		goto err;
+
+	return 0;
+
+err:
+	mlx5e_destroy_flow_table(ft);
+	return err;
+}
+
+static void mlx5e_destroy_inner_ttc_table(struct mlx5e_priv *priv)
+{
+	struct mlx5e_ttc_table *ttc = &priv->fs.inner_ttc;
+
+	if (!mlx5e_tunnel_inner_ft_supported(priv->mdev))
+		return;
+
+	mlx5e_cleanup_ttc_rules(ttc);
+	mlx5e_destroy_flow_table(&ttc->ft);
+}
+
 void mlx5e_destroy_ttc_table(struct mlx5e_priv *priv)
 {
 	struct mlx5e_ttc_table *ttc = &priv->fs.ttc;
@@ -797,6 +1062,7 @@ void mlx5e_destroy_ttc_table(struct mlx5e_priv *priv)
 
 int mlx5e_create_ttc_table(struct mlx5e_priv *priv)
 {
+	bool match_ipv_outer = MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version);
 	struct mlx5e_ttc_table *ttc = &priv->fs.ttc;
 	struct mlx5_flow_table_attr ft_attr = {};
 	struct mlx5e_flow_table *ft = &ttc->ft;
@@ -813,7 +1079,7 @@ int mlx5e_create_ttc_table(struct mlx5e_priv *priv)
 		return err;
 	}
 
-	err = mlx5e_create_ttc_table_groups(ttc);
+	err = mlx5e_create_ttc_table_groups(ttc, match_ipv_outer);
 	if (err)
 		goto err;
 
@@ -1139,11 +1405,18 @@ int mlx5e_create_flow_steering(struct mlx5e_priv *priv)
 		priv->netdev->hw_features &= ~NETIF_F_NTUPLE;
 	}
 
+	err = mlx5e_create_inner_ttc_table(priv);
+	if (err) {
+		netdev_err(priv->netdev, "Failed to create inner ttc table, err=%d\n",
+			   err);
+		goto err_destroy_arfs_tables;
+	}
+
 	err = mlx5e_create_ttc_table(priv);
 	if (err) {
 		netdev_err(priv->netdev, "Failed to create ttc table, err=%d\n",
 			   err);
-		goto err_destroy_arfs_tables;
+		goto err_destroy_inner_ttc_table;
 	}
 
 	err = mlx5e_create_l2_table(priv);
@@ -1168,6 +1441,8 @@ err_destroy_l2_table:
 	mlx5e_destroy_l2_table(priv);
 err_destroy_ttc_table:
 	mlx5e_destroy_ttc_table(priv);
+err_destroy_inner_ttc_table:
+	mlx5e_destroy_inner_ttc_table(priv);
 err_destroy_arfs_tables:
 	mlx5e_arfs_destroy_tables(priv);
 
@@ -1179,6 +1454,7 @@ void mlx5e_destroy_flow_steering(struct mlx5e_priv *priv)
 	mlx5e_destroy_vlan_table(priv);
 	mlx5e_destroy_l2_table(priv);
 	mlx5e_destroy_ttc_table(priv);
+	mlx5e_destroy_inner_ttc_table(priv);
 	mlx5e_arfs_destroy_tables(priv);
 	mlx5e_ethtool_cleanup_steering(priv);
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index 57f31fa478ce..dfc29720ab77 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -71,6 +71,11 @@ struct mlx5e_channel_param {
 	struct mlx5e_cq_param      icosq_cq;
 };
 
+static int mlx5e_get_node(struct mlx5e_priv *priv, int ix)
+{
+	return pci_irq_get_node(priv->mdev->pdev, MLX5_EQ_VEC_COMP_BASE + ix);
+}
+
 static bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev)
 {
 	return MLX5_CAP_GEN(mdev, striding_rq) &&
@@ -208,6 +213,7 @@ static void mlx5e_update_sw_counters(struct mlx5e_priv *priv)
 		s->rx_cache_full  += rq_stats->cache_full;
 		s->rx_cache_empty += rq_stats->cache_empty;
 		s->rx_cache_busy  += rq_stats->cache_busy;
+		s->rx_cache_waive += rq_stats->cache_waive;
 
 		for (j = 0; j < priv->channels.params.num_tc; j++) {
 			sq_stats = &c->sq[j].stats;
@@ -288,6 +294,12 @@ static void mlx5e_update_pport_counters(struct mlx5e_priv *priv, bool full)
 		mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
 	}
 
+	if (MLX5_CAP_PCAM_FEATURE(mdev, rx_buffer_fullness_counters)) {
+		out = pstats->eth_ext_counters;
+		MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP);
+		mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
+	}
+
 	MLX5_SET(ppcnt_reg, in, grp, MLX5_PER_PRIORITY_COUNTERS_GROUP);
 	for (prio = 0; prio < NUM_PPORT_PRIO; prio++) {
 		out = pstats->per_prio_counters[prio];
@@ -396,7 +408,7 @@ static void mlx5e_enable_async_events(struct mlx5e_priv *priv)
 static void mlx5e_disable_async_events(struct mlx5e_priv *priv)
 {
 	clear_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED, &priv->state);
-	synchronize_irq(mlx5_get_msix_vec(priv->mdev, MLX5_EQ_VEC_ASYNC));
+	synchronize_irq(pci_irq_vector(priv->mdev->pdev, MLX5_EQ_VEC_ASYNC));
 }
 
 static inline int mlx5e_get_wqe_mtt_sz(void)
@@ -443,16 +455,17 @@ static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq,
 	int wq_sz = mlx5_wq_ll_get_size(&rq->wq);
 	int mtt_sz = mlx5e_get_wqe_mtt_sz();
 	int mtt_alloc = mtt_sz + MLX5_UMR_ALIGN - 1;
+	int node = mlx5e_get_node(c->priv, c->ix);
 	int i;
 
 	rq->mpwqe.info = kzalloc_node(wq_sz * sizeof(*rq->mpwqe.info),
-				      GFP_KERNEL, cpu_to_node(c->cpu));
+					GFP_KERNEL, node);
 	if (!rq->mpwqe.info)
 		goto err_out;
 
 	/* We allocate more than mtt_sz as we will align the pointer */
-	rq->mpwqe.mtt_no_align = kzalloc_node(mtt_alloc * wq_sz, GFP_KERNEL,
-					cpu_to_node(c->cpu));
+	rq->mpwqe.mtt_no_align = kzalloc_node(mtt_alloc * wq_sz,
+					GFP_KERNEL, node);
 	if (unlikely(!rq->mpwqe.mtt_no_align))
 		goto err_free_wqe_info;
 
@@ -560,7 +573,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
 	int err;
 	int i;
 
-	rqp->wq.db_numa_node = cpu_to_node(c->cpu);
+	rqp->wq.db_numa_node = mlx5e_get_node(c->priv, c->ix);
 
 	err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->wq,
 				&rq->wq_ctrl);
@@ -587,12 +600,12 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
 	}
 
 	rq->buff.map_dir = rq->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
-	rq->rx_headroom = params->rq_headroom;
+	rq->buff.headroom = params->rq_headroom;
 
 	switch (rq->wq_type) {
 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 
-		rq->alloc_wqe = mlx5e_alloc_rx_mpwqe;
+		rq->post_wqes = mlx5e_post_rx_mpwqes;
 		rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe;
 
 		rq->handle_rx_cqe = c->priv->profile->rx_handlers.handle_rx_cqe_mpwqe;
@@ -609,11 +622,10 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
 			goto err_rq_wq_destroy;
 		}
 
-		rq->mpwqe_stride_sz = BIT(params->mpwqe_log_stride_sz);
-		rq->mpwqe_num_strides = BIT(params->mpwqe_log_num_strides);
+		rq->mpwqe.log_stride_sz = params->mpwqe_log_stride_sz;
+		rq->mpwqe.num_strides = BIT(params->mpwqe_log_num_strides);
 
-		rq->buff.wqe_sz = rq->mpwqe_stride_sz * rq->mpwqe_num_strides;
-		byte_count = rq->buff.wqe_sz;
+		byte_count = rq->mpwqe.num_strides << rq->mpwqe.log_stride_sz;
 
 		err = mlx5e_create_rq_umr_mkey(mdev, rq);
 		if (err)
@@ -627,12 +639,13 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
 	default: /* MLX5_WQ_TYPE_LINKED_LIST */
 		rq->wqe.frag_info =
 			kzalloc_node(wq_sz * sizeof(*rq->wqe.frag_info),
-				     GFP_KERNEL, cpu_to_node(c->cpu));
+				     GFP_KERNEL,
+				     mlx5e_get_node(c->priv, c->ix));
 		if (!rq->wqe.frag_info) {
 			err = -ENOMEM;
 			goto err_rq_wq_destroy;
 		}
-		rq->alloc_wqe = mlx5e_alloc_rx_wqe;
+		rq->post_wqes = mlx5e_post_rx_wqes;
 		rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;
 
 #ifdef CONFIG_MLX5_EN_IPSEC
@@ -648,18 +661,17 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
 			goto err_rq_wq_destroy;
 		}
 
-		rq->buff.wqe_sz = params->lro_en  ?
+		byte_count = params->lro_en  ?
 				params->lro_wqe_sz :
 				MLX5E_SW2HW_MTU(c->priv, c->netdev->mtu);
 #ifdef CONFIG_MLX5_EN_IPSEC
 		if (MLX5_IPSEC_DEV(mdev))
-			rq->buff.wqe_sz += MLX5E_METADATA_ETHER_LEN;
+			byte_count += MLX5E_METADATA_ETHER_LEN;
 #endif
 		rq->wqe.page_reuse = !params->xdp_prog && !params->lro_en;
-		byte_count = rq->buff.wqe_sz;
 
 		/* calc the required page order */
-		rq->wqe.frag_sz = MLX5_SKB_FRAG_SZ(rq->rx_headroom + byte_count);
+		rq->wqe.frag_sz = MLX5_SKB_FRAG_SZ(rq->buff.headroom + byte_count);
 		npages = DIV_ROUND_UP(rq->wqe.frag_sz, PAGE_SIZE);
 		rq->buff.page_order = order_base_2(npages);
 
@@ -670,6 +682,12 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
 	for (i = 0; i < wq_sz; i++) {
 		struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i);
 
+		if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
+			u64 dma_offset = (u64)mlx5e_get_wqe_mtt_offset(rq, i) << PAGE_SHIFT;
+
+			wqe->data.addr = cpu_to_be64(dma_offset);
+		}
+
 		wqe->data.byte_count = cpu_to_be32(byte_count);
 		wqe->data.lkey = rq->mkey_be;
 	}
@@ -876,7 +894,8 @@ static void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
 	u16 wqe_ix;
 
 	/* UMR WQE (if in progress) is always at wq->head */
-	if (test_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state))
+	if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ &&
+	    rq->mpwqe.umr_in_progress)
 		mlx5e_free_rx_mpwqe(rq, &rq->mpwqe.info[wq->head]);
 
 	while (!mlx5_wq_ll_is_empty(wq)) {
@@ -919,7 +938,7 @@ static int mlx5e_open_rq(struct mlx5e_channel *c,
 		goto err_destroy_rq;
 
 	if (params->rx_am_enabled)
-		set_bit(MLX5E_RQ_STATE_AM, &c->rq.state);
+		c->rq.state |= BIT(MLX5E_RQ_STATE_AM);
 
 	return 0;
 
@@ -939,7 +958,6 @@ static void mlx5e_activate_rq(struct mlx5e_rq *rq)
 
 	set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
 	sq->db.ico_wqe[pi].opcode     = MLX5_OPCODE_NOP;
-	sq->db.ico_wqe[pi].num_wqebbs = 1;
 	nopwqe = mlx5e_post_nop(&sq->wq, sq->sqn, &sq->pc);
 	mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, &nopwqe->ctrl);
 }
@@ -992,13 +1010,13 @@ static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c,
 	sq->uar_map   = mdev->mlx5e_res.bfreg.map;
 	sq->min_inline_mode = params->tx_min_inline_mode;
 
-	param->wq.db_numa_node = cpu_to_node(c->cpu);
+	param->wq.db_numa_node = mlx5e_get_node(c->priv, c->ix);
 	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq, &sq->wq_ctrl);
 	if (err)
 		return err;
 	sq->wq.db = &sq->wq.db[MLX5_SND_DBR];
 
-	err = mlx5e_alloc_xdpsq_db(sq, cpu_to_node(c->cpu));
+	err = mlx5e_alloc_xdpsq_db(sq, mlx5e_get_node(c->priv, c->ix));
 	if (err)
 		goto err_sq_wq_destroy;
 
@@ -1041,18 +1059,17 @@ static int mlx5e_alloc_icosq(struct mlx5e_channel *c,
 	struct mlx5_core_dev *mdev = c->mdev;
 	int err;
 
-	sq->pdev      = c->pdev;
 	sq->mkey_be   = c->mkey_be;
 	sq->channel   = c;
 	sq->uar_map   = mdev->mlx5e_res.bfreg.map;
 
-	param->wq.db_numa_node = cpu_to_node(c->cpu);
+	param->wq.db_numa_node = mlx5e_get_node(c->priv, c->ix);
 	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq, &sq->wq_ctrl);
 	if (err)
 		return err;
 	sq->wq.db = &sq->wq.db[MLX5_SND_DBR];
 
-	err = mlx5e_alloc_icosq_db(sq, cpu_to_node(c->cpu));
+	err = mlx5e_alloc_icosq_db(sq, mlx5e_get_node(c->priv, c->ix));
 	if (err)
 		goto err_sq_wq_destroy;
 
@@ -1118,13 +1135,13 @@ static int mlx5e_alloc_txqsq(struct mlx5e_channel *c,
 	if (MLX5_IPSEC_DEV(c->priv->mdev))
 		set_bit(MLX5E_SQ_STATE_IPSEC, &sq->state);
 
-	param->wq.db_numa_node = cpu_to_node(c->cpu);
+	param->wq.db_numa_node = mlx5e_get_node(c->priv, c->ix);
 	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq, &sq->wq_ctrl);
 	if (err)
 		return err;
 	sq->wq.db    = &sq->wq.db[MLX5_SND_DBR];
 
-	err = mlx5e_alloc_txqsq_db(sq, cpu_to_node(c->cpu));
+	err = mlx5e_alloc_txqsq_db(sq, mlx5e_get_node(c->priv, c->ix));
 	if (err)
 		goto err_sq_wq_destroy;
 
@@ -1496,8 +1513,8 @@ static int mlx5e_alloc_cq(struct mlx5e_channel *c,
 	struct mlx5_core_dev *mdev = c->priv->mdev;
 	int err;
 
-	param->wq.buf_numa_node = cpu_to_node(c->cpu);
-	param->wq.db_numa_node  = cpu_to_node(c->cpu);
+	param->wq.buf_numa_node = mlx5e_get_node(c->priv, c->ix);
+	param->wq.db_numa_node  = mlx5e_get_node(c->priv, c->ix);
 	param->eq_ix   = c->ix;
 
 	err = mlx5e_alloc_cq_common(mdev, param, cq);
@@ -1596,11 +1613,6 @@ static void mlx5e_close_cq(struct mlx5e_cq *cq)
 	mlx5e_free_cq(cq);
 }
 
-static int mlx5e_get_cpu(struct mlx5e_priv *priv, int ix)
-{
-	return cpumask_first(priv->mdev->priv.irq_info[ix].mask);
-}
-
 static int mlx5e_open_tx_cqs(struct mlx5e_channel *c,
 			     struct mlx5e_params *params,
 			     struct mlx5e_channel_param *cparam)
@@ -1749,11 +1761,12 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
 {
 	struct mlx5e_cq_moder icocq_moder = {0, 0};
 	struct net_device *netdev = priv->netdev;
-	int cpu = mlx5e_get_cpu(priv, ix);
 	struct mlx5e_channel *c;
+	unsigned int irq;
 	int err;
+	int eqn;
 
-	c = kzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu));
+	c = kzalloc_node(sizeof(*c), GFP_KERNEL, mlx5e_get_node(priv, ix));
 	if (!c)
 		return -ENOMEM;
 
@@ -1761,13 +1774,15 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
 	c->mdev     = priv->mdev;
 	c->tstamp   = &priv->tstamp;
 	c->ix       = ix;
-	c->cpu      = cpu;
 	c->pdev     = &priv->mdev->pdev->dev;
 	c->netdev   = priv->netdev;
 	c->mkey_be  = cpu_to_be32(priv->mdev->mlx5e_res.mkey.key);
 	c->num_tc   = params->num_tc;
 	c->xdp      = !!params->xdp_prog;
 
+	mlx5_vector2eqn(priv->mdev, ix, &eqn, &irq);
+	c->irq_desc = irq_to_desc(irq);
+
 	netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);
 
 	err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->icosq.cq);
@@ -1847,7 +1862,8 @@ static void mlx5e_activate_channel(struct mlx5e_channel *c)
 	for (tc = 0; tc < c->num_tc; tc++)
 		mlx5e_activate_txqsq(&c->sq[tc]);
 	mlx5e_activate_rq(&c->rq);
-	netif_set_xps_queue(c->netdev, get_cpu_mask(c->cpu), c->ix);
+	netif_set_xps_queue(c->netdev,
+		mlx5_get_vector_affinity(c->priv->mdev, c->ix), c->ix);
 }
 
 static void mlx5e_deactivate_channel(struct mlx5e_channel *c)
@@ -1969,6 +1985,7 @@ static void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
 	}
 
 	mlx5e_build_common_cq_param(priv, param);
+	param->cq_period_mode = params->rx_cq_period_mode;
 }
 
 static void mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
@@ -2343,9 +2360,10 @@ static void mlx5e_build_tir_ctx_lro(struct mlx5e_params *params, void *tirc)
 
 void mlx5e_build_indir_tir_ctx_hash(struct mlx5e_params *params,
 				    enum mlx5e_traffic_types tt,
-				    void *tirc)
+				    void *tirc, bool inner)
 {
-	void *hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
+	void *hfso = inner ? MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner) :
+			     MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
 
 #define MLX5_HASH_IP            (MLX5_HASH_FIELD_SEL_SRC_IP   |\
 				 MLX5_HASH_FIELD_SEL_DST_IP)
@@ -2494,6 +2512,21 @@ free_in:
 	return err;
 }
 
+static void mlx5e_build_inner_indir_tir_ctx(struct mlx5e_priv *priv,
+					    enum mlx5e_traffic_types tt,
+					    u32 *tirc)
+{
+	MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn);
+
+	mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc);
+
+	MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
+	MLX5_SET(tirc, tirc, indirect_table, priv->indir_rqt.rqtn);
+	MLX5_SET(tirc, tirc, tunneled_offload_en, 0x1);
+
+	mlx5e_build_indir_tir_ctx_hash(&priv->channels.params, tt, tirc, true);
+}
+
 static int mlx5e_set_mtu(struct mlx5e_priv *priv, u16 mtu)
 {
 	struct mlx5_core_dev *mdev = priv->mdev;
@@ -2581,12 +2614,6 @@ static void mlx5e_build_channels_tx_maps(struct mlx5e_priv *priv)
 	}
 }
 
-static bool mlx5e_is_eswitch_vport_mngr(struct mlx5_core_dev *mdev)
-{
-	return (MLX5_CAP_GEN(mdev, vport_group_manager) &&
-		MLX5_CAP_GEN(mdev, port_type) == MLX5_CAP_PORT_TYPE_ETH);
-}
-
 void mlx5e_activate_priv_channels(struct mlx5e_priv *priv)
 {
 	int num_txqs = priv->channels.num * priv->channels.params.num_tc;
@@ -2600,7 +2627,7 @@ void mlx5e_activate_priv_channels(struct mlx5e_priv *priv)
 	mlx5e_activate_channels(&priv->channels);
 	netif_tx_start_all_queues(priv->netdev);
 
-	if (mlx5e_is_eswitch_vport_mngr(priv->mdev))
+	if (MLX5_VPORT_MANAGER(priv->mdev))
 		mlx5e_add_sqs_fwd_rules(priv);
 
 	mlx5e_wait_channels_min_rx_wqes(&priv->channels);
@@ -2611,7 +2638,7 @@ void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv)
 {
 	mlx5e_redirect_rqts_to_drop(priv);
 
-	if (mlx5e_is_eswitch_vport_mngr(priv->mdev))
+	if (MLX5_VPORT_MANAGER(priv->mdev))
 		mlx5e_remove_sqs_fwd_rules(priv);
 
 	/* FIXME: This is a W/A only for tx timeout watch dog false alarm when
@@ -2688,6 +2715,8 @@ int mlx5e_open(struct net_device *netdev)
 
 	mutex_lock(&priv->state_lock);
 	err = mlx5e_open_locked(netdev);
+	if (!err)
+		mlx5_set_port_admin_status(priv->mdev, MLX5_PORT_UP);
 	mutex_unlock(&priv->state_lock);
 
 	return err;
@@ -2722,6 +2751,7 @@ int mlx5e_close(struct net_device *netdev)
 		return -ENODEV;
 
 	mutex_lock(&priv->state_lock);
+	mlx5_set_port_admin_status(priv->mdev, MLX5_PORT_DOWN);
 	err = mlx5e_close_locked(netdev);
 	mutex_unlock(&priv->state_lock);
 
@@ -2862,7 +2892,7 @@ static void mlx5e_build_indir_tir_ctx(struct mlx5e_priv *priv,
 
 	MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
 	MLX5_SET(tirc, tirc, indirect_table, priv->indir_rqt.rqtn);
-	mlx5e_build_indir_tir_ctx_hash(&priv->channels.params, tt, tirc);
+	mlx5e_build_indir_tir_ctx_hash(&priv->channels.params, tt, tirc, false);
 }
 
 static void mlx5e_build_direct_tir_ctx(struct mlx5e_priv *priv, u32 rqtn, u32 *tirc)
@@ -2881,6 +2911,7 @@ int mlx5e_create_indirect_tirs(struct mlx5e_priv *priv)
 	struct mlx5e_tir *tir;
 	void *tirc;
 	int inlen;
+	int i = 0;
 	int err;
 	u32 *in;
 	int tt;
@@ -2896,16 +2927,36 @@ int mlx5e_create_indirect_tirs(struct mlx5e_priv *priv)
 		tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
 		mlx5e_build_indir_tir_ctx(priv, tt, tirc);
 		err = mlx5e_create_tir(priv->mdev, tir, in, inlen);
-		if (err)
-			goto err_destroy_tirs;
+		if (err) {
+			mlx5_core_warn(priv->mdev, "create indirect tirs failed, %d\n", err);
+			goto err_destroy_inner_tirs;
+		}
+	}
+
+	if (!mlx5e_tunnel_inner_ft_supported(priv->mdev))
+		goto out;
+
+	for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++) {
+		memset(in, 0, inlen);
+		tir = &priv->inner_indir_tir[i];
+		tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
+		mlx5e_build_inner_indir_tir_ctx(priv, i, tirc);
+		err = mlx5e_create_tir(priv->mdev, tir, in, inlen);
+		if (err) {
+			mlx5_core_warn(priv->mdev, "create inner indirect tirs failed, %d\n", err);
+			goto err_destroy_inner_tirs;
+		}
 	}
 
+out:
 	kvfree(in);
 
 	return 0;
 
-err_destroy_tirs:
-	mlx5_core_warn(priv->mdev, "create indirect tirs failed, %d\n", err);
+err_destroy_inner_tirs:
+	for (i--; i >= 0; i--)
+		mlx5e_destroy_tir(priv->mdev, &priv->inner_indir_tir[i]);
+
 	for (tt--; tt >= 0; tt--)
 		mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[tt]);
 
@@ -2959,6 +3010,12 @@ void mlx5e_destroy_indirect_tirs(struct mlx5e_priv *priv)
 
 	for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++)
 		mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[i]);
+
+	if (!mlx5e_tunnel_inner_ft_supported(priv->mdev))
+		return;
+
+	for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++)
+		mlx5e_destroy_tir(priv->mdev, &priv->inner_indir_tir[i]);
 }
 
 void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv)
@@ -2998,12 +3055,16 @@ static int mlx5e_modify_channels_vsd(struct mlx5e_channels *chs, bool vsd)
 	return 0;
 }
 
-static int mlx5e_setup_tc(struct net_device *netdev, u8 tc)
+static int mlx5e_setup_tc_mqprio(struct net_device *netdev,
+				 struct tc_mqprio_qopt *mqprio)
 {
 	struct mlx5e_priv *priv = netdev_priv(netdev);
 	struct mlx5e_channels new_channels = {};
+	u8 tc = mqprio->num_tc;
 	int err = 0;
 
+	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
+
 	if (tc && tc != MLX5E_MAX_NUM_TC)
 		return -EINVAL;
 
@@ -3027,39 +3088,42 @@ out:
 	return err;
 }
 
-static int mlx5e_ndo_setup_tc(struct net_device *dev, u32 handle,
-			      u32 chain_index, __be16 proto,
-			      struct tc_to_netdev *tc)
+#ifdef CONFIG_MLX5_ESWITCH
+static int mlx5e_setup_tc_cls_flower(struct net_device *dev,
+				     struct tc_cls_flower_offload *cls_flower)
 {
 	struct mlx5e_priv *priv = netdev_priv(dev);
 
-	if (TC_H_MAJ(handle) != TC_H_MAJ(TC_H_INGRESS))
-		goto mqprio;
+	if (!is_classid_clsact_ingress(cls_flower->common.classid) ||
+	    cls_flower->common.chain_index)
+		return -EOPNOTSUPP;
 
-	if (chain_index)
+	switch (cls_flower->command) {
+	case TC_CLSFLOWER_REPLACE:
+		return mlx5e_configure_flower(priv, cls_flower);
+	case TC_CLSFLOWER_DESTROY:
+		return mlx5e_delete_flower(priv, cls_flower);
+	case TC_CLSFLOWER_STATS:
+		return mlx5e_stats_flower(priv, cls_flower);
+	default:
 		return -EOPNOTSUPP;
+	}
+}
+#endif
 
-	switch (tc->type) {
+static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type,
+			  void *type_data)
+{
+	switch (type) {
+#ifdef CONFIG_MLX5_ESWITCH
 	case TC_SETUP_CLSFLOWER:
-		switch (tc->cls_flower->command) {
-		case TC_CLSFLOWER_REPLACE:
-			return mlx5e_configure_flower(priv, proto, tc->cls_flower);
-		case TC_CLSFLOWER_DESTROY:
-			return mlx5e_delete_flower(priv, tc->cls_flower);
-		case TC_CLSFLOWER_STATS:
-			return mlx5e_stats_flower(priv, tc->cls_flower);
-		}
+		return mlx5e_setup_tc_cls_flower(dev, type_data);
+#endif
+	case TC_SETUP_MQPRIO:
+		return mlx5e_setup_tc_mqprio(dev, type_data);
 	default:
 		return -EOPNOTSUPP;
 	}
-
-mqprio:
-	if (tc->type != TC_SETUP_MQPRIO)
-		return -EINVAL;
-
-	tc->mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
-
-	return mlx5e_setup_tc(dev, tc->mqprio->num_tc);
 }
 
 static void
@@ -3356,6 +3420,7 @@ static int mlx5e_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
 	}
 }
 
+#ifdef CONFIG_MLX5_ESWITCH
 static int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
 {
 	struct mlx5e_priv *priv = netdev_priv(dev);
@@ -3458,6 +3523,7 @@ static int mlx5e_get_vf_stats(struct net_device *dev,
 	return mlx5_eswitch_get_vport_stats(mdev->priv.eswitch, vf + 1,
 					    vf_stats);
 }
+#endif
 
 static void mlx5e_add_vxlan_port(struct net_device *netdev,
 				 struct udp_tunnel_info *ti)
@@ -3487,13 +3553,13 @@ static void mlx5e_del_vxlan_port(struct net_device *netdev,
 	mlx5e_vxlan_queue_work(priv, ti->sa_family, be16_to_cpu(ti->port), 0);
 }
 
-static netdev_features_t mlx5e_vxlan_features_check(struct mlx5e_priv *priv,
-						    struct sk_buff *skb,
-						    netdev_features_t features)
+static netdev_features_t mlx5e_tunnel_features_check(struct mlx5e_priv *priv,
+						     struct sk_buff *skb,
+						     netdev_features_t features)
 {
 	struct udphdr *udph;
-	u16 proto;
-	u16 port = 0;
+	u8 proto;
+	u16 port;
 
 	switch (vlan_get_protocol(skb)) {
 	case htons(ETH_P_IP):
@@ -3506,14 +3572,17 @@ static netdev_features_t mlx5e_vxlan_features_check(struct mlx5e_priv *priv,
 		goto out;
 	}
 
-	if (proto == IPPROTO_UDP) {
+	switch (proto) {
+	case IPPROTO_GRE:
+		return features;
+	case IPPROTO_UDP:
 		udph = udp_hdr(skb);
 		port = be16_to_cpu(udph->dest);
-	}
 
-	/* Verify if UDP port is being offloaded by HW */
-	if (port && mlx5e_vxlan_lookup_port(priv, port))
-		return features;
+		/* Verify if UDP port is being offloaded by HW */
+		if (mlx5e_vxlan_lookup_port(priv, port))
+			return features;
+	}
 
 out:
 	/* Disable CSUM and GSO if the udp dport is not offloaded by HW */
@@ -3537,7 +3606,7 @@ static netdev_features_t mlx5e_features_check(struct sk_buff *skb,
 	/* Validate if the tunneled packet is being offloaded by HW */
 	if (skb->encapsulation &&
 	    (features & NETIF_F_CSUM_MASK || features & NETIF_F_GSO_MASK))
-		return mlx5e_vxlan_features_check(priv, skb, features);
+		return mlx5e_tunnel_features_check(priv, skb, features);
 
 	return features;
 }
@@ -3634,7 +3703,6 @@ static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog)
 
 		set_bit(MLX5E_RQ_STATE_ENABLED, &c->rq.state);
 		/* napi_schedule in case we have missed anything */
-		set_bit(MLX5E_CHANNEL_NAPI_SCHED, &c->flags);
 		napi_schedule(&c->napi);
 
 		if (old_prog)
@@ -3691,11 +3759,11 @@ static void mlx5e_netpoll(struct net_device *dev)
 }
 #endif
 
-static const struct net_device_ops mlx5e_netdev_ops_basic = {
+static const struct net_device_ops mlx5e_netdev_ops = {
 	.ndo_open                = mlx5e_open,
 	.ndo_stop                = mlx5e_close,
 	.ndo_start_xmit          = mlx5e_xmit,
-	.ndo_setup_tc            = mlx5e_ndo_setup_tc,
+	.ndo_setup_tc            = mlx5e_setup_tc,
 	.ndo_select_queue        = mlx5e_select_queue,
 	.ndo_get_stats64         = mlx5e_get_stats,
 	.ndo_set_rx_mode         = mlx5e_set_rx_mode,
@@ -3706,6 +3774,9 @@ static const struct net_device_ops mlx5e_netdev_ops_basic = {
 	.ndo_change_mtu          = mlx5e_change_mtu,
 	.ndo_do_ioctl            = mlx5e_ioctl,
 	.ndo_set_tx_maxrate      = mlx5e_set_tx_maxrate,
+	.ndo_udp_tunnel_add      = mlx5e_add_vxlan_port,
+	.ndo_udp_tunnel_del      = mlx5e_del_vxlan_port,
+	.ndo_features_check      = mlx5e_features_check,
 #ifdef CONFIG_RFS_ACCEL
 	.ndo_rx_flow_steer	 = mlx5e_rx_flow_steer,
 #endif
@@ -3714,29 +3785,8 @@ static const struct net_device_ops mlx5e_netdev_ops_basic = {
 #ifdef CONFIG_NET_POLL_CONTROLLER
 	.ndo_poll_controller     = mlx5e_netpoll,
 #endif
-};
-
-static const struct net_device_ops mlx5e_netdev_ops_sriov = {
-	.ndo_open                = mlx5e_open,
-	.ndo_stop                = mlx5e_close,
-	.ndo_start_xmit          = mlx5e_xmit,
-	.ndo_setup_tc            = mlx5e_ndo_setup_tc,
-	.ndo_select_queue        = mlx5e_select_queue,
-	.ndo_get_stats64         = mlx5e_get_stats,
-	.ndo_set_rx_mode         = mlx5e_set_rx_mode,
-	.ndo_set_mac_address     = mlx5e_set_mac,
-	.ndo_vlan_rx_add_vid     = mlx5e_vlan_rx_add_vid,
-	.ndo_vlan_rx_kill_vid    = mlx5e_vlan_rx_kill_vid,
-	.ndo_set_features        = mlx5e_set_features,
-	.ndo_change_mtu          = mlx5e_change_mtu,
-	.ndo_do_ioctl            = mlx5e_ioctl,
-	.ndo_udp_tunnel_add	 = mlx5e_add_vxlan_port,
-	.ndo_udp_tunnel_del	 = mlx5e_del_vxlan_port,
-	.ndo_set_tx_maxrate      = mlx5e_set_tx_maxrate,
-	.ndo_features_check      = mlx5e_features_check,
-#ifdef CONFIG_RFS_ACCEL
-	.ndo_rx_flow_steer	 = mlx5e_rx_flow_steer,
-#endif
+#ifdef CONFIG_MLX5_ESWITCH
+	/* SRIOV E-Switch NDOs */
 	.ndo_set_vf_mac          = mlx5e_set_vf_mac,
 	.ndo_set_vf_vlan         = mlx5e_set_vf_vlan,
 	.ndo_set_vf_spoofchk     = mlx5e_set_vf_spoofchk,
@@ -3745,13 +3795,9 @@ static const struct net_device_ops mlx5e_netdev_ops_sriov = {
 	.ndo_get_vf_config       = mlx5e_get_vf_config,
 	.ndo_set_vf_link_state   = mlx5e_set_vf_link_state,
 	.ndo_get_vf_stats        = mlx5e_get_vf_stats,
-	.ndo_tx_timeout          = mlx5e_tx_timeout,
-	.ndo_xdp		 = mlx5e_xdp,
-#ifdef CONFIG_NET_POLL_CONTROLLER
-	.ndo_poll_controller     = mlx5e_netpoll,
-#endif
 	.ndo_has_offload_stats	 = mlx5e_has_offload_stats,
 	.ndo_get_offload_stats	 = mlx5e_get_offload_stats,
+#endif
 };
 
 static int mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev)
@@ -3788,22 +3834,11 @@ u16 mlx5e_get_max_inline_cap(struct mlx5_core_dev *mdev)
 	       2 /*sizeof(mlx5e_tx_wqe.inline_hdr_start)*/;
 }
 
-void mlx5e_build_default_indir_rqt(struct mlx5_core_dev *mdev,
-				   u32 *indirection_rqt, int len,
+void mlx5e_build_default_indir_rqt(u32 *indirection_rqt, int len,
 				   int num_channels)
 {
-	int node = mdev->priv.numa_node;
-	int node_num_of_cores;
 	int i;
 
-	if (node == -1)
-		node = first_online_node;
-
-	node_num_of_cores = cpumask_weight(cpumask_of_node(node));
-
-	if (node_num_of_cores)
-		num_channels = min_t(int, num_channels, node_num_of_cores);
-
 	for (i = 0; i < len; i++)
 		indirection_rqt[i] = i % num_channels;
 }
@@ -3942,7 +3977,7 @@ void mlx5e_build_nic_params(struct mlx5_core_dev *mdev,
 	/* RSS */
 	params->rss_hfunc = ETH_RSS_HASH_XOR;
 	netdev_rss_key_fill(params->toeplitz_hash_key, sizeof(params->toeplitz_hash_key));
-	mlx5e_build_default_indir_rqt(mdev, params->indirection_rqt,
+	mlx5e_build_default_indir_rqt(params->indirection_rqt,
 				      MLX5E_INDIR_RQT_SIZE, max_channels);
 }
 
@@ -3981,9 +4016,11 @@ static void mlx5e_set_netdev_dev_addr(struct net_device *netdev)
 	}
 }
 
+#if IS_ENABLED(CONFIG_NET_SWITCHDEV) && IS_ENABLED(CONFIG_MLX5_ESWITCH)
 static const struct switchdev_ops mlx5e_switchdev_ops = {
 	.switchdev_port_attr_get	= mlx5e_attr_get,
 };
+#endif
 
 static void mlx5e_build_nic_netdev(struct net_device *netdev)
 {
@@ -3994,15 +4031,12 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev)
 
 	SET_NETDEV_DEV(netdev, &mdev->pdev->dev);
 
-	if (MLX5_CAP_GEN(mdev, vport_group_manager)) {
-		netdev->netdev_ops = &mlx5e_netdev_ops_sriov;
+	netdev->netdev_ops = &mlx5e_netdev_ops;
+
 #ifdef CONFIG_MLX5_CORE_EN_DCB
-		if (MLX5_CAP_GEN(mdev, qos))
-			netdev->dcbnl_ops = &mlx5e_dcbnl_ops;
+	if (MLX5_CAP_GEN(mdev, vport_group_manager) && MLX5_CAP_GEN(mdev, qos))
+		netdev->dcbnl_ops = &mlx5e_dcbnl_ops;
 #endif
-	} else {
-		netdev->netdev_ops = &mlx5e_netdev_ops_basic;
-	}
 
 	netdev->watchdog_timeo    = 15 * HZ;
 
@@ -4025,20 +4059,32 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev)
 	netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_RX;
 	netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_FILTER;
 
-	if (mlx5e_vxlan_allowed(mdev)) {
-		netdev->hw_features     |= NETIF_F_GSO_UDP_TUNNEL |
-					   NETIF_F_GSO_UDP_TUNNEL_CSUM |
-					   NETIF_F_GSO_PARTIAL;
+	if (mlx5e_vxlan_allowed(mdev) || MLX5_CAP_ETH(mdev, tunnel_stateless_gre)) {
+		netdev->hw_features     |= NETIF_F_GSO_PARTIAL;
 		netdev->hw_enc_features |= NETIF_F_IP_CSUM;
 		netdev->hw_enc_features |= NETIF_F_IPV6_CSUM;
 		netdev->hw_enc_features |= NETIF_F_TSO;
 		netdev->hw_enc_features |= NETIF_F_TSO6;
-		netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;
-		netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM |
-					   NETIF_F_GSO_PARTIAL;
+		netdev->hw_enc_features |= NETIF_F_GSO_PARTIAL;
+	}
+
+	if (mlx5e_vxlan_allowed(mdev)) {
+		netdev->hw_features     |= NETIF_F_GSO_UDP_TUNNEL |
+					   NETIF_F_GSO_UDP_TUNNEL_CSUM;
+		netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL |
+					   NETIF_F_GSO_UDP_TUNNEL_CSUM;
 		netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM;
 	}
 
+	if (MLX5_CAP_ETH(mdev, tunnel_stateless_gre)) {
+		netdev->hw_features     |= NETIF_F_GSO_GRE |
+					   NETIF_F_GSO_GRE_CSUM;
+		netdev->hw_enc_features |= NETIF_F_GSO_GRE |
+					   NETIF_F_GSO_GRE_CSUM;
+		netdev->gso_partial_features |= NETIF_F_GSO_GRE |
+						NETIF_F_GSO_GRE_CSUM;
+	}
+
 	mlx5_query_port_fcs(mdev, &fcs_supported, &fcs_enabled);
 
 	if (fcs_supported)
@@ -4074,8 +4120,8 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev)
 
 	mlx5e_set_netdev_dev_addr(netdev);
 
-#ifdef CONFIG_NET_SWITCHDEV
-	if (MLX5_CAP_GEN(mdev, vport_group_manager))
+#if IS_ENABLED(CONFIG_NET_SWITCHDEV) && IS_ENABLED(CONFIG_MLX5_ESWITCH)
+	if (MLX5_VPORT_MANAGER(mdev))
 		netdev->switchdev_ops = &mlx5e_switchdev_ops;
 #endif
 
@@ -4207,6 +4253,10 @@ static void mlx5e_nic_enable(struct mlx5e_priv *priv)
 
 	mlx5e_init_l2_addr(priv);
 
+	/* Marking the link as currently not needed by the Driver */
+	if (!netif_running(netdev))
+		mlx5_set_port_admin_status(mdev, MLX5_PORT_DOWN);
+
 	/* MTU range: 68 - hw-specific max */
 	netdev->min_mtu = ETH_MIN_MTU;
 	mlx5_query_port_max_mtu(priv->mdev, &max_mtu, 1);
@@ -4217,7 +4267,7 @@ static void mlx5e_nic_enable(struct mlx5e_priv *priv)
 
 	mlx5e_enable_async_events(priv);
 
-	if (MLX5_CAP_GEN(mdev, vport_group_manager))
+	if (MLX5_VPORT_MANAGER(priv->mdev))
 		mlx5e_register_vport_reps(priv);
 
 	if (netdev->reg_state != NETREG_REGISTERED)
@@ -4251,7 +4301,7 @@ static void mlx5e_nic_disable(struct mlx5e_priv *priv)
 
 	queue_work(priv->wq, &priv->set_rx_mode_work);
 
-	if (MLX5_CAP_GEN(mdev, vport_group_manager))
+	if (MLX5_VPORT_MANAGER(priv->mdev))
 		mlx5e_unregister_vport_reps(priv);
 
 	mlx5e_disable_async_events(priv);
@@ -4424,32 +4474,29 @@ static void mlx5e_detach(struct mlx5_core_dev *mdev, void *vpriv)
 
 static void *mlx5e_add(struct mlx5_core_dev *mdev)
 {
-	struct mlx5_eswitch *esw = mdev->priv.eswitch;
-	int total_vfs = MLX5_TOTAL_VPORTS(mdev);
-	struct mlx5e_rep_priv *rpriv = NULL;
+	struct net_device *netdev;
+	void *rpriv = NULL;
 	void *priv;
-	int vport;
 	int err;
-	struct net_device *netdev;
 
 	err = mlx5e_check_required_hca_cap(mdev);
 	if (err)
 		return NULL;
 
-	if (MLX5_CAP_GEN(mdev, vport_group_manager)) {
-		rpriv = kzalloc(sizeof(*rpriv), GFP_KERNEL);
+#ifdef CONFIG_MLX5_ESWITCH
+	if (MLX5_VPORT_MANAGER(mdev)) {
+		rpriv = mlx5e_alloc_nic_rep_priv(mdev);
 		if (!rpriv) {
-			mlx5_core_warn(mdev,
-				       "Not creating net device, Failed to alloc rep priv data\n");
+			mlx5_core_warn(mdev, "Failed to alloc NIC rep priv data\n");
 			return NULL;
 		}
-		rpriv->rep = &esw->offloads.vport_reps[0];
 	}
+#endif
 
 	netdev = mlx5e_create_netdev(mdev, &mlx5e_nic_profile, rpriv);
 	if (!netdev) {
 		mlx5_core_err(mdev, "mlx5e_create_netdev failed\n");
-		goto err_unregister_reps;
+		goto err_free_rpriv;
 	}
 
 	priv = netdev_priv(netdev);
@@ -4470,14 +4517,9 @@ static void *mlx5e_add(struct mlx5_core_dev *mdev)
 
 err_detach:
 	mlx5e_detach(mdev, priv);
-
 err_destroy_netdev:
 	mlx5e_destroy_netdev(priv);
-
-err_unregister_reps:
-	for (vport = 1; vport < total_vfs; vport++)
-		mlx5_eswitch_unregister_vport_rep(esw, vport);
-
+err_free_rpriv:
 	kfree(rpriv);
 	return NULL;
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
index 45e60be9c277..45e03c427faf 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
@@ -613,15 +613,18 @@ static int mlx5e_rep_open(struct net_device *dev)
 	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 	int err;
 
-	err = mlx5e_open(dev);
+	mutex_lock(&priv->state_lock);
+	err = mlx5e_open_locked(dev);
 	if (err)
-		return err;
+		goto unlock;
 
-	err = mlx5_eswitch_set_vport_state(esw, rep->vport, MLX5_ESW_VPORT_ADMIN_STATE_UP);
-	if (!err)
+	if (!mlx5_eswitch_set_vport_state(esw, rep->vport,
+					  MLX5_ESW_VPORT_ADMIN_STATE_UP))
 		netif_carrier_on(dev);
 
-	return 0;
+unlock:
+	mutex_unlock(&priv->state_lock);
+	return err;
 }
 
 static int mlx5e_rep_close(struct net_device *dev)
@@ -630,10 +633,13 @@ static int mlx5e_rep_close(struct net_device *dev)
 	struct mlx5e_rep_priv *rpriv = priv->ppriv;
 	struct mlx5_eswitch_rep *rep = rpriv->rep;
 	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
+	int ret;
 
+	mutex_lock(&priv->state_lock);
 	(void)mlx5_eswitch_set_vport_state(esw, rep->vport, MLX5_ESW_VPORT_ADMIN_STATE_DOWN);
-
-	return mlx5e_close(dev);
+	ret = mlx5e_close_locked(dev);
+	mutex_unlock(&priv->state_lock);
+	return ret;
 }
 
 static int mlx5e_rep_get_phys_port_name(struct net_device *dev,
@@ -651,37 +657,42 @@ static int mlx5e_rep_get_phys_port_name(struct net_device *dev,
 	return 0;
 }
 
-static int mlx5e_rep_ndo_setup_tc(struct net_device *dev, u32 handle,
-				  u32 chain_index, __be16 proto,
-				  struct tc_to_netdev *tc)
+static int
+mlx5e_rep_setup_tc_cls_flower(struct net_device *dev,
+			      struct tc_cls_flower_offload *cls_flower)
 {
 	struct mlx5e_priv *priv = netdev_priv(dev);
 
-	if (TC_H_MAJ(handle) != TC_H_MAJ(TC_H_INGRESS))
+	if (!is_classid_clsact_ingress(cls_flower->common.classid) ||
+	    cls_flower->common.chain_index)
 		return -EOPNOTSUPP;
 
-	if (tc->egress_dev) {
+	if (cls_flower->egress_dev) {
 		struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
-		struct net_device *uplink_dev = mlx5_eswitch_get_uplink_netdev(esw);
 
-		return uplink_dev->netdev_ops->ndo_setup_tc(uplink_dev, handle,
-							    chain_index,
-							    proto, tc);
+		dev = mlx5_eswitch_get_uplink_netdev(esw);
+		return dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_CLSFLOWER,
+						     cls_flower);
 	}
 
-	if (chain_index)
+	switch (cls_flower->command) {
+	case TC_CLSFLOWER_REPLACE:
+		return mlx5e_configure_flower(priv, cls_flower);
+	case TC_CLSFLOWER_DESTROY:
+		return mlx5e_delete_flower(priv, cls_flower);
+	case TC_CLSFLOWER_STATS:
+		return mlx5e_stats_flower(priv, cls_flower);
+	default:
 		return -EOPNOTSUPP;
+	}
+}
 
-	switch (tc->type) {
+static int mlx5e_rep_setup_tc(struct net_device *dev, enum tc_setup_type type,
+			      void *type_data)
+{
+	switch (type) {
 	case TC_SETUP_CLSFLOWER:
-		switch (tc->cls_flower->command) {
-		case TC_CLSFLOWER_REPLACE:
-			return mlx5e_configure_flower(priv, proto, tc->cls_flower);
-		case TC_CLSFLOWER_DESTROY:
-			return mlx5e_delete_flower(priv, tc->cls_flower);
-		case TC_CLSFLOWER_STATS:
-			return mlx5e_stats_flower(priv, tc->cls_flower);
-		}
+		return mlx5e_rep_setup_tc_cls_flower(dev, type_data);
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -773,7 +784,7 @@ static const struct net_device_ops mlx5e_netdev_ops_rep = {
 	.ndo_stop                = mlx5e_rep_close,
 	.ndo_start_xmit          = mlx5e_xmit,
 	.ndo_get_phys_port_name  = mlx5e_rep_get_phys_port_name,
-	.ndo_setup_tc            = mlx5e_rep_ndo_setup_tc,
+	.ndo_setup_tc            = mlx5e_rep_setup_tc,
 	.ndo_get_stats64         = mlx5e_rep_get_stats,
 	.ndo_has_offload_stats	 = mlx5e_has_offload_stats,
 	.ndo_get_offload_stats	 = mlx5e_get_offload_stats,
@@ -913,7 +924,7 @@ static int mlx5e_get_rep_max_num_channels(struct mlx5_core_dev *mdev)
 	return MLX5E_PORT_REPRESENTOR_NCH;
 }
 
-static struct mlx5e_profile mlx5e_rep_profile = {
+static const struct mlx5e_profile mlx5e_rep_profile = {
 	.init			= mlx5e_init_rep,
 	.init_rx		= mlx5e_init_rep_rx,
 	.cleanup_rx		= mlx5e_cleanup_rep_rx,
@@ -1099,3 +1110,16 @@ void mlx5e_unregister_vport_reps(struct mlx5e_priv *priv)
 	mlx5e_rep_unregister_vf_vports(priv); /* VFs vports */
 	mlx5_eswitch_unregister_vport_rep(esw, 0); /* UPLINK PF*/
 }
+
+void *mlx5e_alloc_nic_rep_priv(struct mlx5_core_dev *mdev)
+{
+	struct mlx5_eswitch *esw = mdev->priv.eswitch;
+	struct mlx5e_rep_priv *rpriv;
+
+	rpriv = kzalloc(sizeof(*rpriv), GFP_KERNEL);
+	if (!rpriv)
+		return NULL;
+
+	rpriv->rep = &esw->offloads.vport_reps[0];
+	return rpriv;
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h
index a0a1a7a1d6c0..5659ed9f51e6 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h
@@ -38,6 +38,7 @@
 #include "eswitch.h"
 #include "en.h"
 
+#ifdef CONFIG_MLX5_ESWITCH
 struct mlx5e_neigh_update_table {
 	struct rhashtable       neigh_ht;
 	/* Save the neigh hash entries in a list in addition to the hash table
@@ -123,6 +124,7 @@ struct mlx5e_encap_entry {
 	int encap_size;
 };
 
+void *mlx5e_alloc_nic_rep_priv(struct mlx5_core_dev *mdev);
 void mlx5e_register_vport_reps(struct mlx5e_priv *priv);
 void mlx5e_unregister_vport_reps(struct mlx5e_priv *priv);
 bool mlx5e_is_uplink_rep(struct mlx5e_priv *priv);
@@ -141,5 +143,12 @@ void mlx5e_rep_encap_entry_detach(struct mlx5e_priv *priv,
 				  struct mlx5e_encap_entry *e);
 
 void mlx5e_rep_queue_neigh_stats_work(struct mlx5e_priv *priv);
+#else /* CONFIG_MLX5_ESWITCH */
+static inline void mlx5e_register_vport_reps(struct mlx5e_priv *priv) {}
+static inline void mlx5e_unregister_vport_reps(struct mlx5e_priv *priv) {}
+static inline bool mlx5e_is_uplink_rep(struct mlx5e_priv *priv) { return false; }
+static inline int mlx5e_add_sqs_fwd_rules(struct mlx5e_priv *priv) { return 0; }
+static inline void mlx5e_remove_sqs_fwd_rules(struct mlx5e_priv *priv) {}
+#endif
 
 #endif /* __MLX5E_REP_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
index 325b2c8c1c6d..f1dd638384d3 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -163,7 +163,7 @@ static inline u32 mlx5e_decompress_cqes_start(struct mlx5e_rq *rq,
 
 static inline bool mlx5e_page_is_reserved(struct page *page)
 {
-	return page_is_pfmemalloc(page) || page_to_nid(page) != numa_node_id();
+	return page_is_pfmemalloc(page) || page_to_nid(page) != numa_mem_id();
 }
 
 static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq,
@@ -177,8 +177,10 @@ static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq,
 		return false;
 	}
 
-	if (unlikely(page_is_pfmemalloc(dma_info->page)))
+	if (unlikely(mlx5e_page_is_reserved(dma_info->page))) {
+		rq->stats.cache_waive++;
 		return false;
+	}
 
 	cache->page_cache[cache->tail] = *dma_info;
 	cache->tail = tail_next;
@@ -222,13 +224,13 @@ static inline int mlx5e_page_alloc_mapped(struct mlx5e_rq *rq,
 	if (unlikely(!page))
 		return -ENOMEM;
 
-	dma_info->page = page;
 	dma_info->addr = dma_map_page(rq->pdev, page, 0,
 				      RQ_PAGE_SIZE(rq), rq->buff.map_dir);
 	if (unlikely(dma_mapping_error(rq->pdev, dma_info->addr))) {
 		put_page(page);
 		return -ENOMEM;
 	}
+	dma_info->page = page;
 
 	return 0;
 }
@@ -252,7 +254,7 @@ static inline bool mlx5e_page_reuse(struct mlx5e_rq *rq,
 		!mlx5e_page_is_reserved(wi->di.page);
 }
 
-int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe *wqe, u16 ix)
+static int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe *wqe, u16 ix)
 {
 	struct mlx5e_wqe_frag_info *wi = &rq->wqe.frag_info[ix];
 
@@ -263,8 +265,7 @@ int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe *wqe, u16 ix)
 		wi->offset = 0;
 	}
 
-	wqe->data.addr = cpu_to_be64(wi->di.addr + wi->offset +
-				     rq->rx_headroom);
+	wqe->data.addr = cpu_to_be64(wi->di.addr + wi->offset + rq->buff.headroom);
 	return 0;
 }
 
@@ -296,7 +297,7 @@ void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix)
 
 static inline int mlx5e_mpwqe_strides_per_page(struct mlx5e_rq *rq)
 {
-	return rq->mpwqe_num_strides >> MLX5_MPWRQ_WQE_PAGE_ORDER;
+	return rq->mpwqe.num_strides >> MLX5_MPWRQ_WQE_PAGE_ORDER;
 }
 
 static inline void mlx5e_add_skb_frag_mpwqe(struct mlx5e_rq *rq,
@@ -305,7 +306,7 @@ static inline void mlx5e_add_skb_frag_mpwqe(struct mlx5e_rq *rq,
 					    u32 page_idx, u32 frag_offset,
 					    u32 len)
 {
-	unsigned int truesize =	ALIGN(len, rq->mpwqe_stride_sz);
+	unsigned int truesize = ALIGN(len, BIT(rq->mpwqe.log_stride_sz));
 
 	dma_sync_single_for_cpu(rq->pdev,
 				wi->umr.dma_info[page_idx].addr + frag_offset,
@@ -358,7 +359,6 @@ static inline void mlx5e_post_umr_wqe(struct mlx5e_rq *rq, u16 ix)
 	/* fill sq edge with nops to avoid wqe wrap around */
 	while ((pi = (sq->pc & wq->sz_m1)) > sq->edge) {
 		sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_NOP;
-		sq->db.ico_wqe[pi].num_wqebbs = 1;
 		mlx5e_post_nop(wq, sq->sqn, &sq->pc);
 	}
 
@@ -369,41 +369,35 @@ static inline void mlx5e_post_umr_wqe(struct mlx5e_rq *rq, u16 ix)
 			    MLX5_OPCODE_UMR);
 
 	sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_UMR;
-	sq->db.ico_wqe[pi].num_wqebbs = num_wqebbs;
 	sq->pc += num_wqebbs;
 	mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, &wqe->ctrl);
 }
 
 static int mlx5e_alloc_rx_umr_mpwqe(struct mlx5e_rq *rq,
-				    struct mlx5e_rx_wqe *wqe,
 				    u16 ix)
 {
 	struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
-	u64 dma_offset = (u64)mlx5e_get_wqe_mtt_offset(rq, ix) << PAGE_SHIFT;
 	int pg_strides = mlx5e_mpwqe_strides_per_page(rq);
+	struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[0];
 	int err;
 	int i;
 
-	for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++) {
-		struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[i];
-
+	for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++, dma_info++) {
 		err = mlx5e_page_alloc_mapped(rq, dma_info);
 		if (unlikely(err))
 			goto err_unmap;
 		wi->umr.mtt[i] = cpu_to_be64(dma_info->addr | MLX5_EN_WR);
 		page_ref_add(dma_info->page, pg_strides);
-		wi->skbs_frags[i] = 0;
 	}
 
+	memset(wi->skbs_frags, 0, sizeof(*wi->skbs_frags) * MLX5_MPWRQ_PAGES_PER_WQE);
 	wi->consumed_strides = 0;
-	wqe->data.addr = cpu_to_be64(dma_offset);
 
 	return 0;
 
 err_unmap:
 	while (--i >= 0) {
-		struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[i];
-
+		dma_info--;
 		page_ref_sub(dma_info->page, pg_strides);
 		mlx5e_page_release(rq, dma_info, true);
 	}
@@ -414,27 +408,21 @@ err_unmap:
 void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi)
 {
 	int pg_strides = mlx5e_mpwqe_strides_per_page(rq);
+	struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[0];
 	int i;
 
-	for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++) {
-		struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[i];
-
+	for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++, dma_info++) {
 		page_ref_sub(dma_info->page, pg_strides - wi->skbs_frags[i]);
 		mlx5e_page_release(rq, dma_info, true);
 	}
 }
 
-void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq)
+static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq)
 {
 	struct mlx5_wq_ll *wq = &rq->wq;
 	struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(wq, wq->head);
 
-	clear_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state);
-
-	if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state))) {
-		mlx5e_free_rx_mpwqe(rq, &rq->mpwqe.info[wq->head]);
-		return;
-	}
+	rq->mpwqe.umr_in_progress = false;
 
 	mlx5_wq_ll_push(wq, be16_to_cpu(wqe->next.next_wqe_index));
 
@@ -444,16 +432,18 @@ void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq)
 	mlx5_wq_ll_update_db_record(wq);
 }
 
-int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe *wqe, u16 ix)
+static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 {
 	int err;
 
-	err = mlx5e_alloc_rx_umr_mpwqe(rq, wqe, ix);
-	if (unlikely(err))
+	err = mlx5e_alloc_rx_umr_mpwqe(rq, ix);
+	if (unlikely(err)) {
+		rq->stats.buff_alloc_err++;
 		return err;
-	set_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state);
+	}
+	rq->mpwqe.umr_in_progress = true;
 	mlx5e_post_umr_wqe(rq, ix);
-	return -EBUSY;
+	return 0;
 }
 
 void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
@@ -463,94 +453,150 @@ void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 	mlx5e_free_rx_mpwqe(rq, wi);
 }
 
-#define RQ_CANNOT_POST(rq) \
-	(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state) || \
-	 test_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state))
-
 bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
 {
 	struct mlx5_wq_ll *wq = &rq->wq;
+	int err;
+
+	if (unlikely(!MLX5E_TEST_BIT(rq->state, MLX5E_RQ_STATE_ENABLED)))
+		return false;
 
-	if (unlikely(RQ_CANNOT_POST(rq)))
+	if (mlx5_wq_ll_is_full(wq))
 		return false;
 
-	while (!mlx5_wq_ll_is_full(wq)) {
+	do {
 		struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(wq, wq->head);
-		int err;
 
-		err = rq->alloc_wqe(rq, wqe, wq->head);
-		if (err == -EBUSY)
-			return true;
+		err = mlx5e_alloc_rx_wqe(rq, wqe, wq->head);
 		if (unlikely(err)) {
 			rq->stats.buff_alloc_err++;
 			break;
 		}
 
 		mlx5_wq_ll_push(wq, be16_to_cpu(wqe->next.next_wqe_index));
-	}
+	} while (!mlx5_wq_ll_is_full(wq));
 
 	/* ensure wqes are visible to device before updating doorbell record */
 	dma_wmb();
 
 	mlx5_wq_ll_update_db_record(wq);
 
-	return !mlx5_wq_ll_is_full(wq);
+	return !!err;
+}
+
+static inline void mlx5e_poll_ico_single_cqe(struct mlx5e_cq *cq,
+					     struct mlx5e_icosq *sq,
+					     struct mlx5e_rq *rq,
+					     struct mlx5_cqe64 *cqe)
+{
+	struct mlx5_wq_cyc *wq = &sq->wq;
+	u16 ci = be16_to_cpu(cqe->wqe_counter) & wq->sz_m1;
+	struct mlx5e_sq_wqe_info *icowi = &sq->db.ico_wqe[ci];
+
+	mlx5_cqwq_pop(&cq->wq);
+
+	if (unlikely((cqe->op_own >> 4) != MLX5_CQE_REQ)) {
+		WARN_ONCE(true, "mlx5e: Bad OP in ICOSQ CQE: 0x%x\n",
+			  cqe->op_own);
+		return;
+	}
+
+	if (likely(icowi->opcode == MLX5_OPCODE_UMR)) {
+		mlx5e_post_rx_mpwqe(rq);
+		return;
+	}
+
+	if (unlikely(icowi->opcode != MLX5_OPCODE_NOP))
+		WARN_ONCE(true,
+			  "mlx5e: Bad OPCODE in ICOSQ WQE info: 0x%x\n",
+			  icowi->opcode);
+}
+
+static void mlx5e_poll_ico_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq)
+{
+	struct mlx5e_icosq *sq = container_of(cq, struct mlx5e_icosq, cq);
+	struct mlx5_cqe64 *cqe;
+
+	if (unlikely(!MLX5E_TEST_BIT(sq->state, MLX5E_SQ_STATE_ENABLED)))
+		return;
+
+	cqe = mlx5_cqwq_get_cqe(&cq->wq);
+	if (likely(!cqe))
+		return;
+
+	/* by design, there's only a single cqe */
+	mlx5e_poll_ico_single_cqe(cq, sq, rq, cqe);
+
+	mlx5_cqwq_update_db_record(&cq->wq);
+}
+
+bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq)
+{
+	struct mlx5_wq_ll *wq = &rq->wq;
+
+	if (unlikely(!MLX5E_TEST_BIT(rq->state, MLX5E_RQ_STATE_ENABLED)))
+		return false;
+
+	mlx5e_poll_ico_cq(&rq->channel->icosq.cq, rq);
+
+	if (mlx5_wq_ll_is_full(wq))
+		return false;
+
+	if (!rq->mpwqe.umr_in_progress)
+		mlx5e_alloc_rx_mpwqe(rq, wq->head);
+
+	return true;
 }
 
 static void mlx5e_lro_update_hdr(struct sk_buff *skb, struct mlx5_cqe64 *cqe,
 				 u32 cqe_bcnt)
 {
 	struct ethhdr	*eth = (struct ethhdr *)(skb->data);
-	struct iphdr	*ipv4;
-	struct ipv6hdr	*ipv6;
 	struct tcphdr	*tcp;
 	int network_depth = 0;
 	__be16 proto;
 	u16 tot_len;
+	void *ip_p;
 
 	u8 l4_hdr_type = get_cqe_l4_hdr_type(cqe);
-	int tcp_ack = ((CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA  == l4_hdr_type) ||
-		       (CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA == l4_hdr_type));
+	u8 tcp_ack = (l4_hdr_type == CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA) ||
+		(l4_hdr_type == CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA);
 
 	skb->mac_len = ETH_HLEN;
 	proto = __vlan_get_protocol(skb, eth->h_proto, &network_depth);
 
-	ipv4 = (struct iphdr *)(skb->data + network_depth);
-	ipv6 = (struct ipv6hdr *)(skb->data + network_depth);
 	tot_len = cqe_bcnt - network_depth;
+	ip_p = skb->data + network_depth;
 
 	if (proto == htons(ETH_P_IP)) {
-		tcp = (struct tcphdr *)(skb->data + network_depth +
-					sizeof(struct iphdr));
-		ipv6 = NULL;
-		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
-	} else {
-		tcp = (struct tcphdr *)(skb->data + network_depth +
-					sizeof(struct ipv6hdr));
-		ipv4 = NULL;
-		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
-	}
+		struct iphdr *ipv4 = ip_p;
 
-	if (get_cqe_lro_tcppsh(cqe))
-		tcp->psh                = 1;
-
-	if (tcp_ack) {
-		tcp->ack                = 1;
-		tcp->ack_seq            = cqe->lro_ack_seq_num;
-		tcp->window             = cqe->lro_tcp_win;
-	}
+		tcp = ip_p + sizeof(struct iphdr);
+		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
 
-	if (ipv4) {
 		ipv4->ttl               = cqe->lro_min_ttl;
 		ipv4->tot_len           = cpu_to_be16(tot_len);
 		ipv4->check             = 0;
 		ipv4->check             = ip_fast_csum((unsigned char *)ipv4,
 						       ipv4->ihl);
 	} else {
+		struct ipv6hdr *ipv6 = ip_p;
+
+		tcp = ip_p + sizeof(struct ipv6hdr);
+		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
+
 		ipv6->hop_limit         = cqe->lro_min_ttl;
 		ipv6->payload_len       = cpu_to_be16(tot_len -
 						      sizeof(struct ipv6hdr));
 	}
+
+	tcp->psh = get_cqe_lro_tcppsh(cqe);
+
+	if (tcp_ack) {
+		tcp->ack                = 1;
+		tcp->ack_seq            = cqe->lro_ack_seq_num;
+		tcp->window             = cqe->lro_tcp_win;
+	}
 }
 
 static inline void mlx5e_skb_set_hash(struct mlx5_cqe64 *cqe,
@@ -776,9 +822,9 @@ struct sk_buff *skb_from_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
 			     struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt)
 {
 	struct mlx5e_dma_info *di = &wi->di;
+	u16 rx_headroom = rq->buff.headroom;
 	struct sk_buff *skb;
 	void *va, *data;
-	u16 rx_headroom = rq->rx_headroom;
 	bool consumed;
 	u32 frag_size;
 
@@ -857,6 +903,7 @@ wq_ll_pop:
 		       &wqe->next.next_wqe_index);
 }
 
+#ifdef CONFIG_MLX5_ESWITCH
 void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
 {
 	struct net_device *netdev = rq->netdev;
@@ -901,6 +948,7 @@ wq_ll_pop:
 	mlx5_wq_ll_pop(&rq->wq, wqe_counter_be,
 		       &wqe->next.next_wqe_index);
 }
+#endif
 
 static inline void mlx5e_mpwqe_fill_rx_skb(struct mlx5e_rq *rq,
 					   struct mlx5_cqe64 *cqe,
@@ -909,7 +957,7 @@ static inline void mlx5e_mpwqe_fill_rx_skb(struct mlx5e_rq *rq,
 					   struct sk_buff *skb)
 {
 	u16 stride_ix      = mpwrq_get_cqe_stride_index(cqe);
-	u32 wqe_offset     = stride_ix * rq->mpwqe_stride_sz;
+	u32 wqe_offset     = stride_ix << rq->mpwqe.log_stride_sz;
 	u32 head_offset    = wqe_offset & (PAGE_SIZE - 1);
 	u32 page_idx       = wqe_offset >> PAGE_SHIFT;
 	u32 head_page_idx  = page_idx;
@@ -977,7 +1025,7 @@ void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
 	napi_gro_receive(rq->cq.napi, skb);
 
 mpwrq_cqe_out:
-	if (likely(wi->consumed_strides < rq->mpwqe_num_strides))
+	if (likely(wi->consumed_strides < rq->mpwqe.num_strides))
 		return;
 
 	mlx5e_free_rx_mpwqe(rq, wi);
@@ -987,21 +1035,23 @@ mpwrq_cqe_out:
 int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
 {
 	struct mlx5e_rq *rq = container_of(cq, struct mlx5e_rq, cq);
-	struct mlx5e_xdpsq *xdpsq = &rq->xdpsq;
+	struct mlx5e_xdpsq *xdpsq;
+	struct mlx5_cqe64 *cqe;
 	int work_done = 0;
 
-	if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state)))
+	if (unlikely(!MLX5E_TEST_BIT(rq->state, MLX5E_RQ_STATE_ENABLED)))
 		return 0;
 
 	if (cq->decmprs_left)
 		work_done += mlx5e_decompress_cqes_cont(rq, cq, 0, budget);
 
-	for (; work_done < budget; work_done++) {
-		struct mlx5_cqe64 *cqe = mlx5_cqwq_get_cqe(&cq->wq);
+	cqe = mlx5_cqwq_get_cqe(&cq->wq);
+	if (!cqe)
+		return 0;
 
-		if (!cqe)
-			break;
+	xdpsq = &rq->xdpsq;
 
+	do {
 		if (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED) {
 			work_done +=
 				mlx5e_decompress_cqes_start(rq, cq,
@@ -1012,7 +1062,7 @@ int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
 		mlx5_cqwq_pop(&cq->wq);
 
 		rq->handle_rx_cqe(rq, cqe);
-	}
+	} while ((++work_done < budget) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
 
 	if (xdpsq->db.doorbell) {
 		mlx5e_xmit_xdp_doorbell(xdpsq);
@@ -1030,13 +1080,18 @@ int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
 bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
 {
 	struct mlx5e_xdpsq *sq;
+	struct mlx5_cqe64 *cqe;
 	struct mlx5e_rq *rq;
 	u16 sqcc;
 	int i;
 
 	sq = container_of(cq, struct mlx5e_xdpsq, cq);
 
-	if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
+	if (unlikely(!MLX5E_TEST_BIT(sq->state, MLX5E_SQ_STATE_ENABLED)))
+		return false;
+
+	cqe = mlx5_cqwq_get_cqe(&cq->wq);
+	if (!cqe)
 		return false;
 
 	rq = container_of(sq, struct mlx5e_rq, xdpsq);
@@ -1046,15 +1101,11 @@ bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
 	 */
 	sqcc = sq->cc;
 
-	for (i = 0; i < MLX5E_TX_CQ_POLL_BUDGET; i++) {
-		struct mlx5_cqe64 *cqe;
+	i = 0;
+	do {
 		u16 wqe_counter;
 		bool last_wqe;
 
-		cqe = mlx5_cqwq_get_cqe(&cq->wq);
-		if (!cqe)
-			break;
-
 		mlx5_cqwq_pop(&cq->wq);
 
 		wqe_counter = be16_to_cpu(cqe->wqe_counter);
@@ -1072,7 +1123,7 @@ bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
 			/* Recycle RX page */
 			mlx5e_page_release(rq, di, true);
 		} while (!last_wqe);
-	}
+	} while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
 
 	mlx5_cqwq_update_db_record(&cq->wq);
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c b/drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c
index 898759fcf9ec..1f1f8af87d4d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c
@@ -189,6 +189,7 @@ struct mlx5e_lbt_priv {
 	struct packet_type pt;
 	struct completion comp;
 	bool loopback_ok;
+	bool local_lb;
 };
 
 static int
@@ -236,6 +237,13 @@ static int mlx5e_test_loopback_setup(struct mlx5e_priv *priv,
 {
 	int err = 0;
 
+	/* Temporarily enable local_lb */
+	if (MLX5_CAP_GEN(priv->mdev, disable_local_lb)) {
+		mlx5_nic_vport_query_local_lb(priv->mdev, &lbtp->local_lb);
+		if (!lbtp->local_lb)
+			mlx5_nic_vport_update_local_lb(priv->mdev, true);
+	}
+
 	err = mlx5e_refresh_tirs(priv, true);
 	if (err)
 		return err;
@@ -254,6 +262,11 @@ static int mlx5e_test_loopback_setup(struct mlx5e_priv *priv,
 static void mlx5e_test_loopback_cleanup(struct mlx5e_priv *priv,
 					struct mlx5e_lbt_priv *lbtp)
 {
+	if (MLX5_CAP_GEN(priv->mdev, disable_local_lb)) {
+		if (!lbtp->local_lb)
+			mlx5_nic_vport_update_local_lb(priv->mdev, false);
+	}
+
 	dev_remove_pack(&lbtp->pt);
 	mlx5e_refresh_tirs(priv, false);
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
index e65517eafc58..6d199ffb1c0b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
@@ -47,7 +47,7 @@
 
 struct counter_desc {
 	char		format[ETH_GSTRING_LEN];
-	int		offset; /* Byte offset */
+	size_t		offset; /* Byte offset */
 };
 
 struct mlx5e_sw_stats {
@@ -84,6 +84,7 @@ struct mlx5e_sw_stats {
 	u64 rx_cache_full;
 	u64 rx_cache_empty;
 	u64 rx_cache_busy;
+	u64 rx_cache_waive;
 
 	/* Special handling counters */
 	u64 link_down_events_phy;
@@ -123,6 +124,7 @@ static const struct counter_desc sw_stats_desc[] = {
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_full) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_empty) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_busy) },
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_waive) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, link_down_events_phy) },
 };
 
@@ -216,6 +218,12 @@ static const struct counter_desc vport_stats_desc[] = {
 	MLX5_GET64(ppcnt_reg, pstats->per_prio_counters[prio], \
 		   counter_set.eth_per_prio_grp_data_layout.c##_high)
 #define NUM_PPORT_PRIO				8
+#define PPORT_ETH_EXT_OFF(c) \
+	MLX5_BYTE_OFF(ppcnt_reg, \
+		      counter_set.eth_extended_cntrs_grp_data_layout.c##_high)
+#define PPORT_ETH_EXT_GET(pstats, c) \
+	MLX5_GET64(ppcnt_reg, (pstats)->eth_ext_counters, \
+		   counter_set.eth_extended_cntrs_grp_data_layout.c##_high)
 
 struct mlx5e_pport_stats {
 	__be64 IEEE_802_3_counters[MLX5_ST_SZ_QW(ppcnt_reg)];
@@ -224,6 +232,7 @@ struct mlx5e_pport_stats {
 	__be64 per_prio_counters[NUM_PPORT_PRIO][MLX5_ST_SZ_QW(ppcnt_reg)];
 	__be64 phy_counters[MLX5_ST_SZ_QW(ppcnt_reg)];
 	__be64 phy_statistical_counters[MLX5_ST_SZ_QW(ppcnt_reg)];
+	__be64 eth_ext_counters[MLX5_ST_SZ_QW(ppcnt_reg)];
 };
 
 static const struct counter_desc pport_802_3_stats_desc[] = {
@@ -290,12 +299,22 @@ static const struct counter_desc pport_per_prio_pfc_stats_desc[] = {
 	{ "rx_%s_pause_transition", PPORT_PER_PRIO_OFF(rx_pause_transition) },
 };
 
+static const struct counter_desc pport_eth_ext_stats_desc[] = {
+	{ "rx_buffer_passed_thres_phy", PPORT_ETH_EXT_OFF(rx_buffer_almost_full) },
+};
+
 #define PCIE_PERF_OFF(c) \
 	MLX5_BYTE_OFF(mpcnt_reg, counter_set.pcie_perf_cntrs_grp_data_layout.c)
 #define PCIE_PERF_GET(pcie_stats, c) \
 	MLX5_GET(mpcnt_reg, (pcie_stats)->pcie_perf_counters, \
 		 counter_set.pcie_perf_cntrs_grp_data_layout.c)
 
+#define PCIE_PERF_OFF64(c) \
+	MLX5_BYTE_OFF(mpcnt_reg, counter_set.pcie_perf_cntrs_grp_data_layout.c##_high)
+#define PCIE_PERF_GET64(pcie_stats, c) \
+	MLX5_GET64(mpcnt_reg, (pcie_stats)->pcie_perf_counters, \
+		   counter_set.pcie_perf_cntrs_grp_data_layout.c##_high)
+
 struct mlx5e_pcie_stats {
 	__be64 pcie_perf_counters[MLX5_ST_SZ_QW(mpcnt_reg)];
 };
@@ -305,6 +324,17 @@ static const struct counter_desc pcie_perf_stats_desc[] = {
 	{ "tx_pci_signal_integrity", PCIE_PERF_OFF(tx_errors) },
 };
 
+static const struct counter_desc pcie_perf_stats_desc64[] = {
+	{ "outbound_pci_buffer_overflow", PCIE_PERF_OFF64(tx_overflow_buffer_pkt) },
+};
+
+static const struct counter_desc pcie_perf_stall_stats_desc[] = {
+	{ "outbound_pci_stalled_rd", PCIE_PERF_OFF(outbound_stalled_reads) },
+	{ "outbound_pci_stalled_wr", PCIE_PERF_OFF(outbound_stalled_writes) },
+	{ "outbound_pci_stalled_rd_events", PCIE_PERF_OFF(outbound_stalled_reads_events) },
+	{ "outbound_pci_stalled_wr_events", PCIE_PERF_OFF(outbound_stalled_writes_events) },
+};
+
 struct mlx5e_rq_stats {
 	u64 packets;
 	u64 bytes;
@@ -326,6 +356,7 @@ struct mlx5e_rq_stats {
 	u64 cache_full;
 	u64 cache_empty;
 	u64 cache_busy;
+	u64 cache_waive;
 };
 
 static const struct counter_desc rq_stats_desc[] = {
@@ -349,6 +380,7 @@ static const struct counter_desc rq_stats_desc[] = {
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_full) },
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_empty) },
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_busy) },
+	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_waive) },
 };
 
 struct mlx5e_sq_stats {
@@ -397,17 +429,29 @@ static const struct counter_desc sq_stats_desc[] = {
 #define NUM_PCIE_PERF_COUNTERS(priv) \
 	(ARRAY_SIZE(pcie_perf_stats_desc) * \
 	 MLX5_CAP_MCAM_FEATURE((priv)->mdev, pcie_performance_group))
+#define NUM_PCIE_PERF_COUNTERS64(priv) \
+	(ARRAY_SIZE(pcie_perf_stats_desc64) * \
+	 MLX5_CAP_MCAM_FEATURE((priv)->mdev, tx_overflow_buffer_pkt))
+#define NUM_PCIE_PERF_STALL_COUNTERS(priv) \
+	(ARRAY_SIZE(pcie_perf_stall_stats_desc) * \
+	 MLX5_CAP_MCAM_FEATURE((priv)->mdev, pcie_outbound_stalled))
 #define NUM_PPORT_PER_PRIO_TRAFFIC_COUNTERS \
 	ARRAY_SIZE(pport_per_prio_traffic_stats_desc)
 #define NUM_PPORT_PER_PRIO_PFC_COUNTERS \
 	ARRAY_SIZE(pport_per_prio_pfc_stats_desc)
+#define NUM_PPORT_ETH_EXT_COUNTERS(priv) \
+	(ARRAY_SIZE(pport_eth_ext_stats_desc) * \
+	 MLX5_CAP_PCAM_FEATURE((priv)->mdev, rx_buffer_fullness_counters))
 #define NUM_PPORT_COUNTERS(priv)	(NUM_PPORT_802_3_COUNTERS + \
 					 NUM_PPORT_2863_COUNTERS  + \
 					 NUM_PPORT_2819_COUNTERS  + \
 					 NUM_PPORT_PHY_STATISTICAL_COUNTERS(priv) + \
 					 NUM_PPORT_PER_PRIO_TRAFFIC_COUNTERS * \
-					 NUM_PPORT_PRIO)
-#define NUM_PCIE_COUNTERS(priv)		NUM_PCIE_PERF_COUNTERS(priv)
+					 NUM_PPORT_PRIO + \
+					 NUM_PPORT_ETH_EXT_COUNTERS(priv))
+#define NUM_PCIE_COUNTERS(priv)		(NUM_PCIE_PERF_COUNTERS(priv) + \
+					 NUM_PCIE_PERF_COUNTERS64(priv) +\
+					 NUM_PCIE_PERF_STALL_COUNTERS(priv))
 #define NUM_RQ_STATS			ARRAY_SIZE(rq_stats_desc)
 #define NUM_SQ_STATS			ARRAY_SIZE(sq_stats_desc)
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
index 3c536f560dd2..da503e6411da 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
@@ -1326,7 +1326,7 @@ static int parse_tc_nic_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
 	LIST_HEAD(actions);
 	int err;
 
-	if (tc_no_actions(exts))
+	if (!tcf_exts_has_actions(exts))
 		return -EINVAL;
 
 	attr->flow_tag = MLX5_FS_DEFAULT_FLOW_TAG;
@@ -1443,12 +1443,10 @@ static int mlx5e_route_lookup_ipv6(struct mlx5e_priv *priv,
 	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 	int ret;
 
-	dst = ip6_route_output(dev_net(mirred_dev), NULL, fl6);
-	ret = dst->error;
-	if (ret) {
-		dst_release(dst);
+	ret = ipv6_stub->ipv6_dst_lookup(dev_net(mirred_dev), NULL, &dst,
+					 fl6);
+	if (ret < 0)
 		return ret;
-	}
 
 	*out_ttl = ip6_dst_hoplimit(dst);
 
@@ -1839,7 +1837,7 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
 	bool encap = false;
 	int err = 0;
 
-	if (tc_no_actions(exts))
+	if (!tcf_exts_has_actions(exts))
 		return -EINVAL;
 
 	memset(attr, 0, sizeof(*attr));
@@ -1939,7 +1937,7 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
 	return err;
 }
 
-int mlx5e_configure_flower(struct mlx5e_priv *priv, __be16 protocol,
+int mlx5e_configure_flower(struct mlx5e_priv *priv,
 			   struct tc_cls_flower_offload *f)
 {
 	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h
index ecbe30d808ae..c14c263a739b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h
@@ -33,12 +33,15 @@
 #ifndef __MLX5_EN_TC_H__
 #define __MLX5_EN_TC_H__
 
+#include <net/pkt_cls.h>
+
 #define MLX5E_TC_FLOW_ID_MASK 0x0000ffff
 
+#ifdef CONFIG_MLX5_ESWITCH
 int mlx5e_tc_init(struct mlx5e_priv *priv);
 void mlx5e_tc_cleanup(struct mlx5e_priv *priv);
 
-int mlx5e_configure_flower(struct mlx5e_priv *priv, __be16 protocol,
+int mlx5e_configure_flower(struct mlx5e_priv *priv,
 			   struct tc_cls_flower_offload *f);
 int mlx5e_delete_flower(struct mlx5e_priv *priv,
 			struct tc_cls_flower_offload *f);
@@ -60,4 +63,10 @@ static inline int mlx5e_tc_num_filters(struct mlx5e_priv *priv)
 	return atomic_read(&priv->fs.tc.ht.nelems);
 }
 
+#else /* CONFIG_MLX5_ESWITCH */
+static inline int  mlx5e_tc_init(struct mlx5e_priv *priv) { return 0; }
+static inline void mlx5e_tc_cleanup(struct mlx5e_priv *priv) {}
+static inline int  mlx5e_tc_num_filters(struct mlx5e_priv *priv) { return 0; }
+#endif
+
 #endif /* __MLX5_EN_TC_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
index aaa0f4ebba9a..fee43e40fa16 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
@@ -128,10 +128,10 @@ static inline int mlx5e_skb_l3_header_offset(struct sk_buff *skb)
 		return mlx5e_skb_l2_header_offset(skb);
 }
 
-static inline unsigned int mlx5e_calc_min_inline(enum mlx5_inline_modes mode,
-						 struct sk_buff *skb)
+static inline u16 mlx5e_calc_min_inline(enum mlx5_inline_modes mode,
+					struct sk_buff *skb)
 {
-	int hlen;
+	u16 hlen;
 
 	switch (mode) {
 	case MLX5_INLINE_MODE_NONE:
@@ -140,19 +140,22 @@ static inline unsigned int mlx5e_calc_min_inline(enum mlx5_inline_modes mode,
 		hlen = eth_get_headlen(skb->data, skb_headlen(skb));
 		if (hlen == ETH_HLEN && !skb_vlan_tag_present(skb))
 			hlen += VLAN_HLEN;
-		return hlen;
+		break;
 	case MLX5_INLINE_MODE_IP:
 		/* When transport header is set to zero, it means no transport
 		 * header. When transport header is set to 0xff's, it means
 		 * transport header wasn't set.
 		 */
-		if (skb_transport_offset(skb))
-			return mlx5e_skb_l3_header_offset(skb);
+		if (skb_transport_offset(skb)) {
+			hlen = mlx5e_skb_l3_header_offset(skb);
+			break;
+		}
 		/* fall through */
 	case MLX5_INLINE_MODE_L2:
 	default:
-		return mlx5e_skb_l2_header_offset(skb);
+		hlen = mlx5e_skb_l2_header_offset(skb);
 	}
+	return min_t(u16, hlen, skb->len);
 }
 
 static inline void mlx5e_tx_skb_pull_inline(unsigned char **skb_data,
@@ -391,6 +394,7 @@ netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev)
 bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget)
 {
 	struct mlx5e_txqsq *sq;
+	struct mlx5_cqe64 *cqe;
 	u32 dma_fifo_cc;
 	u32 nbytes;
 	u16 npkts;
@@ -399,7 +403,11 @@ bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget)
 
 	sq = container_of(cq, struct mlx5e_txqsq, cq);
 
-	if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
+	if (unlikely(!MLX5E_TEST_BIT(sq->state, MLX5E_SQ_STATE_ENABLED)))
+		return false;
+
+	cqe = mlx5_cqwq_get_cqe(&cq->wq);
+	if (!cqe)
 		return false;
 
 	npkts = 0;
@@ -413,15 +421,11 @@ bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget)
 	/* avoid dirtying sq cache line every cqe */
 	dma_fifo_cc = sq->dma_fifo_cc;
 
-	for (i = 0; i < MLX5E_TX_CQ_POLL_BUDGET; i++) {
-		struct mlx5_cqe64 *cqe;
+	i = 0;
+	do {
 		u16 wqe_counter;
 		bool last_wqe;
 
-		cqe = mlx5_cqwq_get_cqe(&cq->wq);
-		if (!cqe)
-			break;
-
 		mlx5_cqwq_pop(&cq->wq);
 
 		wqe_counter = be16_to_cpu(cqe->wqe_counter);
@@ -464,7 +468,8 @@ bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget)
 			sqcc += wi->num_wqebbs;
 			napi_consume_skb(skb, napi_budget);
 		} while (!last_wqe);
-	}
+
+	} while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
 
 	mlx5_cqwq_update_db_record(&cq->wq);
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
index 92db28a9ed43..e906b754415c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
@@ -30,66 +30,18 @@
  * SOFTWARE.
  */
 
+#include <linux/irq.h>
 #include "en.h"
 
-static inline void mlx5e_poll_ico_single_cqe(struct mlx5e_cq *cq,
-					     struct mlx5e_icosq *sq,
-					     struct mlx5_cqe64 *cqe,
-					     u16 *sqcc)
+static inline bool mlx5e_channel_no_affinity_change(struct mlx5e_channel *c)
 {
-	struct mlx5_wq_cyc *wq = &sq->wq;
-	u16 ci = be16_to_cpu(cqe->wqe_counter) & wq->sz_m1;
-	struct mlx5e_sq_wqe_info *icowi = &sq->db.ico_wqe[ci];
-	struct mlx5e_rq *rq = &sq->channel->rq;
-
-	prefetch(rq);
-	mlx5_cqwq_pop(&cq->wq);
-	*sqcc += icowi->num_wqebbs;
-
-	if (unlikely((cqe->op_own >> 4) != MLX5_CQE_REQ)) {
-		WARN_ONCE(true, "mlx5e: Bad OP in ICOSQ CQE: 0x%x\n",
-			  cqe->op_own);
-		return;
-	}
-
-	if (likely(icowi->opcode == MLX5_OPCODE_UMR)) {
-		mlx5e_post_rx_mpwqe(rq);
-		return;
-	}
-
-	if (unlikely(icowi->opcode != MLX5_OPCODE_NOP))
-		WARN_ONCE(true,
-			  "mlx5e: Bad OPCODE in ICOSQ WQE info: 0x%x\n",
-			  icowi->opcode);
-}
-
-static void mlx5e_poll_ico_cq(struct mlx5e_cq *cq)
-{
-	struct mlx5e_icosq *sq = container_of(cq, struct mlx5e_icosq, cq);
-	struct mlx5_cqe64 *cqe;
-	u16 sqcc;
-
-	if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
-		return;
-
-	cqe = mlx5_cqwq_get_cqe(&cq->wq);
-	if (likely(!cqe))
-		return;
+	int current_cpu = smp_processor_id();
+	const struct cpumask *aff;
+	struct irq_data *idata;
 
-	/* sq->cc must be updated only after mlx5_cqwq_update_db_record(),
-	 * otherwise a cq overrun may occur
-	 */
-	sqcc = sq->cc;
-
-	/* by design, there's only a single cqe */
-	mlx5e_poll_ico_single_cqe(cq, sq, cqe, &sqcc);
-
-	mlx5_cqwq_update_db_record(&cq->wq);
-
-	/* ensure cq space is freed before enabling more cqes */
-	wmb();
-
-	sq->cc = sqcc;
+	idata = irq_desc_get_irq_data(c->irq_desc);
+	aff = irq_data_get_affinity_mask(idata);
+	return cpumask_test_cpu(current_cpu, aff);
 }
 
 int mlx5e_napi_poll(struct napi_struct *napi, int budget)
@@ -100,8 +52,6 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget)
 	int work_done;
 	int i;
 
-	clear_bit(MLX5E_CHANNEL_NAPI_SCHED, &c->flags);
-
 	for (i = 0; i < c->num_tc; i++)
 		busy |= mlx5e_poll_tx_cq(&c->sq[i].cq, budget);
 
@@ -111,25 +61,22 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget)
 	work_done = mlx5e_poll_rx_cq(&c->rq.cq, budget);
 	busy |= work_done == budget;
 
-	mlx5e_poll_ico_cq(&c->icosq.cq);
+	busy |= c->rq.post_wqes(&c->rq);
 
-	busy |= mlx5e_post_rx_wqes(&c->rq);
-
-	if (busy)
-		return budget;
-
-	napi_complete_done(napi, work_done);
+	if (busy) {
+		if (likely(mlx5e_channel_no_affinity_change(c)))
+			return budget;
+		if (work_done == budget)
+			work_done--;
+	}
 
-	/* avoid losing completion event during/after polling cqs */
-	if (test_bit(MLX5E_CHANNEL_NAPI_SCHED, &c->flags)) {
-		napi_schedule(napi);
+	if (unlikely(!napi_complete_done(napi, work_done)))
 		return work_done;
-	}
 
 	for (i = 0; i < c->num_tc; i++)
 		mlx5e_cq_arm(&c->sq[i].cq);
 
-	if (test_bit(MLX5E_RQ_STATE_AM, &c->rq.state))
+	if (MLX5E_TEST_BIT(c->rq.state, MLX5E_RQ_STATE_AM))
 		mlx5e_rx_am(&c->rq);
 
 	mlx5e_cq_arm(&c->rq.cq);
@@ -143,7 +90,6 @@ void mlx5e_completion_event(struct mlx5_core_cq *mcq)
 	struct mlx5e_cq *cq = container_of(mcq, struct mlx5e_cq, mcq);
 
 	cq->event_ctr++;
-	set_bit(MLX5E_CHANNEL_NAPI_SCHED, &cq->channel->flags);
 	napi_schedule(cq->napi);
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
index 52b9a64cd3a2..fc606bfd1d6e 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
@@ -36,9 +36,7 @@
 #include <linux/mlx5/cmd.h>
 #include "mlx5_core.h"
 #include "fpga/core.h"
-#ifdef CONFIG_MLX5_CORE_EN
 #include "eswitch.h"
-#endif
 
 enum {
 	MLX5_EQE_SIZE		= sizeof(struct mlx5_eqe),
@@ -161,6 +159,8 @@ static const char *eqe_type_str(u8 type)
 		return "MLX5_EVENT_TYPE_NIC_VPORT_CHANGE";
 	case MLX5_EVENT_TYPE_FPGA_ERROR:
 		return "MLX5_EVENT_TYPE_FPGA_ERROR";
+	case MLX5_EVENT_TYPE_GENERAL_EVENT:
+		return "MLX5_EVENT_TYPE_GENERAL_EVENT";
 	default:
 		return "Unrecognized event";
 	}
@@ -191,6 +191,7 @@ static void eq_update_ci(struct mlx5_eq *eq, int arm)
 {
 	__be32 __iomem *addr = eq->doorbell + (arm ? 0 : 2);
 	u32 val = (eq->cons_index & 0xffffff) | (eq->eqn << 24);
+
 	__raw_writel((__force u32)cpu_to_be32(val), addr);
 	/* We still want ordering, just not swabbing, so add a barrier */
 	mb();
@@ -378,6 +379,20 @@ int mlx5_core_page_fault_resume(struct mlx5_core_dev *dev, u32 token,
 EXPORT_SYMBOL_GPL(mlx5_core_page_fault_resume);
 #endif
 
+static void general_event_handler(struct mlx5_core_dev *dev,
+				  struct mlx5_eqe *eqe)
+{
+	switch (eqe->sub_type) {
+	case MLX5_GENERAL_SUBTYPE_DELAY_DROP_TIMEOUT:
+		if (dev->event)
+			dev->event(dev, MLX5_DEV_EVENT_DELAY_DROP_TIMEOUT, 0);
+		break;
+	default:
+		mlx5_core_dbg(dev, "General event with unrecognized subtype: sub_type %d\n",
+			      eqe->sub_type);
+	}
+}
+
 static irqreturn_t mlx5_eq_int(int irq, void *eq_ptr)
 {
 	struct mlx5_eq *eq = eq_ptr;
@@ -467,11 +482,9 @@ static irqreturn_t mlx5_eq_int(int irq, void *eq_ptr)
 			}
 			break;
 
-#ifdef CONFIG_MLX5_CORE_EN
 		case MLX5_EVENT_TYPE_NIC_VPORT_CHANGE:
 			mlx5_eswitch_vport_event(dev->priv.eswitch, eqe);
 			break;
-#endif
 
 		case MLX5_EVENT_TYPE_PORT_MODULE_EVENT:
 			mlx5_port_module_event(dev, eqe);
@@ -486,6 +499,9 @@ static irqreturn_t mlx5_eq_int(int irq, void *eq_ptr)
 			mlx5_fpga_event(dev, eqe->type, &eqe->data.raw);
 			break;
 
+		case MLX5_EVENT_TYPE_GENERAL_EVENT:
+			general_event_handler(dev, eqe);
+			break;
 		default:
 			mlx5_core_warn(dev, "Unhandled event 0x%x on EQ 0x%x\n",
 				       eqe->type, eq->eqn);
@@ -585,7 +601,7 @@ int mlx5_create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, u8 vecidx,
 		 name, pci_name(dev->pdev));
 
 	eq->eqn = MLX5_GET(create_eq_out, out, eq_number);
-	eq->irqn = priv->msix_arr[vecidx].vector;
+	eq->irqn = pci_irq_vector(dev->pdev, vecidx);
 	eq->dev = dev;
 	eq->doorbell = priv->uar->map + MLX5_EQ_DOORBEL_OFFSET;
 	err = request_irq(eq->irqn, handler, 0,
@@ -620,7 +636,7 @@ int mlx5_create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, u8 vecidx,
 	return 0;
 
 err_irq:
-	free_irq(priv->msix_arr[vecidx].vector, eq);
+	free_irq(eq->irqn, eq);
 
 err_eq:
 	mlx5_cmd_destroy_eq(dev, eq->eqn);
@@ -661,11 +677,6 @@ int mlx5_destroy_unmap_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq)
 }
 EXPORT_SYMBOL_GPL(mlx5_destroy_unmap_eq);
 
-u32 mlx5_get_msix_vec(struct mlx5_core_dev *dev, int vecidx)
-{
-	return dev->priv.msix_arr[MLX5_EQ_VEC_ASYNC].vector;
-}
-
 int mlx5_eq_init(struct mlx5_core_dev *dev)
 {
 	int err;
@@ -688,11 +699,13 @@ int mlx5_start_eqs(struct mlx5_core_dev *dev)
 	u64 async_event_mask = MLX5_ASYNC_EVENT_MASK;
 	int err;
 
-	if (MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_ETH &&
-	    MLX5_CAP_GEN(dev, vport_group_manager) &&
-	    mlx5_core_is_pf(dev))
+	if (MLX5_VPORT_MANAGER(dev))
 		async_event_mask |= (1ull << MLX5_EVENT_TYPE_NIC_VPORT_CHANGE);
 
+	if (MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_ETH &&
+	    MLX5_CAP_GEN(dev, general_notification_event))
+		async_event_mask |= (1ull << MLX5_EVENT_TYPE_GENERAL_EVENT);
+
 	if (MLX5_CAP_GEN(dev, port_module_event))
 		async_event_mask |= (1ull << MLX5_EVENT_TYPE_PORT_MODULE_EVENT);
 	else
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
index 8b18cc9ec026..c77f4c0c7769 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
@@ -46,19 +46,13 @@ enum {
 	MLX5_ACTION_DEL  = 2,
 };
 
-/* E-Switch UC L2 table hash node */
-struct esw_uc_addr {
-	struct l2addr_node node;
-	u32                table_index;
-	u32                vport;
-};
-
 /* Vport UC/MC hash node */
 struct vport_addr {
 	struct l2addr_node     node;
 	u8                     action;
 	u32                    vport;
-	struct mlx5_flow_handle *flow_rule; /* SRIOV only */
+	struct mlx5_flow_handle *flow_rule;
+	bool mpfs; /* UC MAC was added to MPFs */
 	/* A flag indicating that mac was added due to mc promiscuous vport */
 	bool mc_promisc;
 };
@@ -154,81 +148,6 @@ static int modify_esw_vport_cvlan(struct mlx5_core_dev *dev, u32 vport,
 	return modify_esw_vport_context_cmd(dev, vport, in, sizeof(in));
 }
 
-/* HW L2 Table (MPFS) management */
-static int set_l2_table_entry_cmd(struct mlx5_core_dev *dev, u32 index,
-				  u8 *mac, u8 vlan_valid, u16 vlan)
-{
-	u32 in[MLX5_ST_SZ_DW(set_l2_table_entry_in)]   = {0};
-	u32 out[MLX5_ST_SZ_DW(set_l2_table_entry_out)] = {0};
-	u8 *in_mac_addr;
-
-	MLX5_SET(set_l2_table_entry_in, in, opcode,
-		 MLX5_CMD_OP_SET_L2_TABLE_ENTRY);
-	MLX5_SET(set_l2_table_entry_in, in, table_index, index);
-	MLX5_SET(set_l2_table_entry_in, in, vlan_valid, vlan_valid);
-	MLX5_SET(set_l2_table_entry_in, in, vlan, vlan);
-
-	in_mac_addr = MLX5_ADDR_OF(set_l2_table_entry_in, in, mac_address);
-	ether_addr_copy(&in_mac_addr[2], mac);
-
-	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
-}
-
-static int del_l2_table_entry_cmd(struct mlx5_core_dev *dev, u32 index)
-{
-	u32 in[MLX5_ST_SZ_DW(delete_l2_table_entry_in)]   = {0};
-	u32 out[MLX5_ST_SZ_DW(delete_l2_table_entry_out)] = {0};
-
-	MLX5_SET(delete_l2_table_entry_in, in, opcode,
-		 MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY);
-	MLX5_SET(delete_l2_table_entry_in, in, table_index, index);
-	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
-}
-
-static int alloc_l2_table_index(struct mlx5_l2_table *l2_table, u32 *ix)
-{
-	int err = 0;
-
-	*ix = find_first_zero_bit(l2_table->bitmap, l2_table->size);
-	if (*ix >= l2_table->size)
-		err = -ENOSPC;
-	else
-		__set_bit(*ix, l2_table->bitmap);
-
-	return err;
-}
-
-static void free_l2_table_index(struct mlx5_l2_table *l2_table, u32 ix)
-{
-	__clear_bit(ix, l2_table->bitmap);
-}
-
-static int set_l2_table_entry(struct mlx5_core_dev *dev, u8 *mac,
-			      u8 vlan_valid, u16 vlan,
-			      u32 *index)
-{
-	struct mlx5_l2_table *l2_table = &dev->priv.eswitch->l2_table;
-	int err;
-
-	err = alloc_l2_table_index(l2_table, index);
-	if (err)
-		return err;
-
-	err = set_l2_table_entry_cmd(dev, *index, mac, vlan_valid, vlan);
-	if (err)
-		free_l2_table_index(l2_table, *index);
-
-	return err;
-}
-
-static void del_l2_table_entry(struct mlx5_core_dev *dev, u32 index)
-{
-	struct mlx5_l2_table *l2_table = &dev->priv.eswitch->l2_table;
-
-	del_l2_table_entry_cmd(dev, index);
-	free_l2_table_index(l2_table, index);
-}
-
 /* E-Switch FDB */
 static struct mlx5_flow_handle *
 __esw_fdb_set_vport_rule(struct mlx5_eswitch *esw, u32 vport, bool rx_rule,
@@ -455,65 +374,60 @@ typedef int (*vport_addr_action)(struct mlx5_eswitch *esw,
 
 static int esw_add_uc_addr(struct mlx5_eswitch *esw, struct vport_addr *vaddr)
 {
-	struct hlist_head *hash = esw->l2_table.l2_hash;
-	struct esw_uc_addr *esw_uc;
 	u8 *mac = vaddr->node.addr;
 	u32 vport = vaddr->vport;
 	int err;
 
-	esw_uc = l2addr_hash_find(hash, mac, struct esw_uc_addr);
-	if (esw_uc) {
+	/* Skip mlx5_mpfs_add_mac for PFs,
+	 * it is already done by the PF netdev in mlx5e_execute_l2_action
+	 */
+	if (!vport)
+		goto fdb_add;
+
+	err = mlx5_mpfs_add_mac(esw->dev, mac);
+	if (err) {
 		esw_warn(esw->dev,
-			 "Failed to set L2 mac(%pM) for vport(%d), mac is already in use by vport(%d)\n",
-			 mac, vport, esw_uc->vport);
-		return -EEXIST;
+			 "Failed to add L2 table mac(%pM) for vport(%d), err(%d)\n",
+			 mac, vport, err);
+		return err;
 	}
+	vaddr->mpfs = true;
 
-	esw_uc = l2addr_hash_add(hash, mac, struct esw_uc_addr, GFP_KERNEL);
-	if (!esw_uc)
-		return -ENOMEM;
-	esw_uc->vport = vport;
-
-	err = set_l2_table_entry(esw->dev, mac, 0, 0, &esw_uc->table_index);
-	if (err)
-		goto abort;
-
+fdb_add:
 	/* SRIOV is enabled: Forward UC MAC to vport */
 	if (esw->fdb_table.fdb && esw->mode == SRIOV_LEGACY)
 		vaddr->flow_rule = esw_fdb_set_vport_rule(esw, mac, vport);
 
-	esw_debug(esw->dev, "\tADDED UC MAC: vport[%d] %pM index:%d fr(%p)\n",
-		  vport, mac, esw_uc->table_index, vaddr->flow_rule);
-	return err;
-abort:
-	l2addr_hash_del(esw_uc);
-	return err;
+	esw_debug(esw->dev, "\tADDED UC MAC: vport[%d] %pM fr(%p)\n",
+		  vport, mac, vaddr->flow_rule);
+
+	return 0;
 }
 
 static int esw_del_uc_addr(struct mlx5_eswitch *esw, struct vport_addr *vaddr)
 {
-	struct hlist_head *hash = esw->l2_table.l2_hash;
-	struct esw_uc_addr *esw_uc;
 	u8 *mac = vaddr->node.addr;
 	u32 vport = vaddr->vport;
+	int err = 0;
 
-	esw_uc = l2addr_hash_find(hash, mac, struct esw_uc_addr);
-	if (!esw_uc || esw_uc->vport != vport) {
-		esw_debug(esw->dev,
-			  "MAC(%pM) doesn't belong to vport (%d)\n",
-			  mac, vport);
-		return -EINVAL;
-	}
-	esw_debug(esw->dev, "\tDELETE UC MAC: vport[%d] %pM index:%d fr(%p)\n",
-		  vport, mac, esw_uc->table_index, vaddr->flow_rule);
+	/* Skip mlx5_mpfs_del_mac for PFs,
+	 * it is already done by the PF netdev in mlx5e_execute_l2_action
+	 */
+	if (!vport || !vaddr->mpfs)
+		goto fdb_del;
 
-	del_l2_table_entry(esw->dev, esw_uc->table_index);
+	err = mlx5_mpfs_del_mac(esw->dev, mac);
+	if (err)
+		esw_warn(esw->dev,
+			 "Failed to del L2 table mac(%pM) for vport(%d), err(%d)\n",
+			 mac, vport, err);
+	vaddr->mpfs = false;
 
+fdb_del:
 	if (vaddr->flow_rule)
 		mlx5_del_flow_rules(vaddr->flow_rule);
 	vaddr->flow_rule = NULL;
 
-	l2addr_hash_del(esw_uc);
 	return 0;
 }
 
@@ -1585,7 +1499,7 @@ static void esw_disable_vport(struct mlx5_eswitch *esw, int vport_num)
 	/* Mark this vport as disabled to discard new events */
 	vport->enabled = false;
 
-	synchronize_irq(mlx5_get_msix_vec(esw->dev, MLX5_EQ_VEC_ASYNC));
+	synchronize_irq(pci_irq_vector(esw->dev->pdev, MLX5_EQ_VEC_ASYNC));
 	/* Wait for current already scheduled events to complete */
 	flush_workqueue(esw->work_queue);
 	/* Disable events from this vport */
@@ -1611,13 +1525,14 @@ static void esw_disable_vport(struct mlx5_eswitch *esw, int vport_num)
 }
 
 /* Public E-Switch API */
+#define ESW_ALLOWED(esw) ((esw) && MLX5_VPORT_MANAGER((esw)->dev))
+
 int mlx5_eswitch_enable_sriov(struct mlx5_eswitch *esw, int nvfs, int mode)
 {
 	int err;
 	int i, enabled_events;
 
-	if (!esw || !MLX5_CAP_GEN(esw->dev, vport_group_manager) ||
-	    MLX5_CAP_GEN(esw->dev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
+	if (!ESW_ALLOWED(esw))
 		return 0;
 
 	if (!MLX5_CAP_GEN(esw->dev, eswitch_flow_table) ||
@@ -1634,7 +1549,6 @@ int mlx5_eswitch_enable_sriov(struct mlx5_eswitch *esw, int nvfs, int mode)
 
 	esw_info(esw->dev, "E-Switch enable SRIOV: nvfs(%d) mode (%d)\n", nvfs, mode);
 	esw->mode = mode;
-	esw_disable_vport(esw, 0);
 
 	if (mode == SRIOV_LEGACY)
 		err = esw_create_legacy_fdb_table(esw, nvfs + 1);
@@ -1647,7 +1561,11 @@ int mlx5_eswitch_enable_sriov(struct mlx5_eswitch *esw, int nvfs, int mode)
 	if (err)
 		esw_warn(esw->dev, "Failed to create eswitch TSAR");
 
-	enabled_events = (mode == SRIOV_LEGACY) ? SRIOV_VPORT_EVENTS : UC_ADDR_CHANGE;
+	/* Don't enable vport events when in SRIOV_OFFLOADS mode, since:
+	 * 1. L2 table (MPFS) is programmed by PF/VF representors netdevs set_rx_mode
+	 * 2. FDB/Eswitch is programmed by user space tools
+	 */
+	enabled_events = (mode == SRIOV_LEGACY) ? SRIOV_VPORT_EVENTS : 0;
 	for (i = 0; i <= nvfs; i++)
 		esw_enable_vport(esw, i, enabled_events);
 
@@ -1656,7 +1574,6 @@ int mlx5_eswitch_enable_sriov(struct mlx5_eswitch *esw, int nvfs, int mode)
 	return 0;
 
 abort:
-	esw_enable_vport(esw, 0, UC_ADDR_CHANGE);
 	esw->mode = SRIOV_NONE;
 	return err;
 }
@@ -1667,9 +1584,7 @@ void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw)
 	int nvports;
 	int i;
 
-	if (!esw || !MLX5_CAP_GEN(esw->dev, vport_group_manager) ||
-	    MLX5_CAP_GEN(esw->dev, port_type) != MLX5_CAP_PORT_TYPE_ETH ||
-	    esw->mode == SRIOV_NONE)
+	if (!ESW_ALLOWED(esw) || esw->mode == SRIOV_NONE)
 		return;
 
 	esw_info(esw->dev, "disable SRIOV: active vports(%d) mode(%d)\n",
@@ -1692,44 +1607,21 @@ void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw)
 		esw_offloads_cleanup(esw, nvports);
 
 	esw->mode = SRIOV_NONE;
-	/* VPORT 0 (PF) must be enabled back with non-sriov configuration */
-	esw_enable_vport(esw, 0, UC_ADDR_CHANGE);
-}
-
-void mlx5_eswitch_attach(struct mlx5_eswitch *esw)
-{
-	if (!esw || !MLX5_CAP_GEN(esw->dev, vport_group_manager) ||
-	    MLX5_CAP_GEN(esw->dev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
-		return;
-
-	esw_enable_vport(esw, 0, UC_ADDR_CHANGE);
-	/* VF Vports will be enabled when SRIOV is enabled */
-}
-
-void mlx5_eswitch_detach(struct mlx5_eswitch *esw)
-{
-	if (!esw || !MLX5_CAP_GEN(esw->dev, vport_group_manager) ||
-	    MLX5_CAP_GEN(esw->dev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
-		return;
-
-	esw_disable_vport(esw, 0);
 }
 
 int mlx5_eswitch_init(struct mlx5_core_dev *dev)
 {
-	int l2_table_size = 1 << MLX5_CAP_GEN(dev, log_max_l2_table);
 	int total_vports = MLX5_TOTAL_VPORTS(dev);
 	struct mlx5_eswitch *esw;
 	int vport_num;
 	int err;
 
-	if (!MLX5_CAP_GEN(dev, vport_group_manager) ||
-	    MLX5_CAP_GEN(dev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
+	if (!MLX5_VPORT_MANAGER(dev))
 		return 0;
 
 	esw_info(dev,
-		 "Total vports %d, l2 table size(%d), per vport: max uc(%d) max mc(%d)\n",
-		 total_vports, l2_table_size,
+		 "Total vports %d, per vport: max uc(%d) max mc(%d)\n",
+		 total_vports,
 		 MLX5_MAX_UC_PER_VPORT(dev),
 		 MLX5_MAX_MC_PER_VPORT(dev));
 
@@ -1739,14 +1631,6 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev)
 
 	esw->dev = dev;
 
-	esw->l2_table.bitmap = kcalloc(BITS_TO_LONGS(l2_table_size),
-				   sizeof(uintptr_t), GFP_KERNEL);
-	if (!esw->l2_table.bitmap) {
-		err = -ENOMEM;
-		goto abort;
-	}
-	esw->l2_table.size = l2_table_size;
-
 	esw->work_queue = create_singlethread_workqueue("mlx5_esw_wq");
 	if (!esw->work_queue) {
 		err = -ENOMEM;
@@ -1797,7 +1681,6 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev)
 abort:
 	if (esw->work_queue)
 		destroy_workqueue(esw->work_queue);
-	kfree(esw->l2_table.bitmap);
 	kfree(esw->vports);
 	kfree(esw->offloads.vport_reps);
 	kfree(esw);
@@ -1806,15 +1689,13 @@ abort:
 
 void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw)
 {
-	if (!esw || !MLX5_CAP_GEN(esw->dev, vport_group_manager) ||
-	    MLX5_CAP_GEN(esw->dev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
+	if (!esw || !MLX5_VPORT_MANAGER(esw->dev))
 		return;
 
 	esw_info(esw->dev, "cleanup\n");
 
 	esw->dev->priv.eswitch = NULL;
 	destroy_workqueue(esw->work_queue);
-	kfree(esw->l2_table.bitmap);
 	kfree(esw->offloads.vport_reps);
 	kfree(esw->vports);
 	kfree(esw);
@@ -1838,8 +1719,6 @@ void mlx5_eswitch_vport_event(struct mlx5_eswitch *esw, struct mlx5_eqe *eqe)
 }
 
 /* Vport Administration */
-#define ESW_ALLOWED(esw) \
-	(esw && MLX5_CAP_GEN(esw->dev, vport_group_manager) && mlx5_core_is_pf(esw->dev))
 #define LEGAL_VPORT(esw, vport) (vport >= 0 && vport < esw->total_vports)
 
 int mlx5_eswitch_set_vport_mac(struct mlx5_eswitch *esw,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
index 834a33050969..565c8b7a399a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
@@ -37,6 +37,15 @@
 #include <linux/if_link.h>
 #include <net/devlink.h>
 #include <linux/mlx5/device.h>
+#include "lib/mpfs.h"
+
+enum {
+	SRIOV_NONE,
+	SRIOV_LEGACY,
+	SRIOV_OFFLOADS
+};
+
+#ifdef CONFIG_MLX5_ESWITCH
 
 #define MLX5_MAX_UC_PER_VPORT(dev) \
 	(1 << MLX5_CAP_GEN(dev, log_max_current_uc_list))
@@ -44,9 +53,6 @@
 #define MLX5_MAX_MC_PER_VPORT(dev) \
 	(1 << MLX5_CAP_GEN(dev, log_max_current_mc_list))
 
-#define MLX5_L2_ADDR_HASH_SIZE (BIT(BITS_PER_BYTE))
-#define MLX5_L2_ADDR_HASH(addr) (addr[5])
-
 #define FDB_UPLINK_VPORT 0xffff
 
 #define MLX5_MIN_BW_SHARE 1
@@ -54,48 +60,6 @@
 #define MLX5_RATE_TO_BW_SHARE(rate, divider, limit) \
 	min_t(u32, max_t(u32, (rate) / (divider), MLX5_MIN_BW_SHARE), limit)
 
-/* L2 -mac address based- hash helpers */
-struct l2addr_node {
-	struct hlist_node hlist;
-	u8                addr[ETH_ALEN];
-};
-
-#define for_each_l2hash_node(hn, tmp, hash, i) \
-	for (i = 0; i < MLX5_L2_ADDR_HASH_SIZE; i++) \
-		hlist_for_each_entry_safe(hn, tmp, &hash[i], hlist)
-
-#define l2addr_hash_find(hash, mac, type) ({                \
-	int ix = MLX5_L2_ADDR_HASH(mac);                    \
-	bool found = false;                                 \
-	type *ptr = NULL;                                   \
-							    \
-	hlist_for_each_entry(ptr, &hash[ix], node.hlist)    \
-		if (ether_addr_equal(ptr->node.addr, mac)) {\
-			found = true;                       \
-			break;                              \
-		}                                           \
-	if (!found)                                         \
-		ptr = NULL;                                 \
-	ptr;                                                \
-})
-
-#define l2addr_hash_add(hash, mac, type, gfp) ({            \
-	int ix = MLX5_L2_ADDR_HASH(mac);                    \
-	type *ptr = NULL;                                   \
-							    \
-	ptr = kzalloc(sizeof(type), gfp);                   \
-	if (ptr) {                                          \
-		ether_addr_copy(ptr->node.addr, mac);       \
-		hlist_add_head(&ptr->node.hlist, &hash[ix]);\
-	}                                                   \
-	ptr;                                                \
-})
-
-#define l2addr_hash_del(ptr) ({                             \
-	hlist_del(&ptr->node.hlist);                        \
-	kfree(ptr);                                         \
-})
-
 struct vport_ingress {
 	struct mlx5_flow_table *acl;
 	struct mlx5_flow_group *allow_untagged_spoofchk_grp;
@@ -150,12 +114,6 @@ struct mlx5_vport {
 	u16                     enabled_events;
 };
 
-struct mlx5_l2_table {
-	struct hlist_head l2_hash[MLX5_L2_ADDR_HASH_SIZE];
-	u32                  size;
-	unsigned long        *bitmap;
-};
-
 struct mlx5_eswitch_fdb {
 	void *fdb;
 	union {
@@ -175,12 +133,6 @@ struct mlx5_eswitch_fdb {
 	};
 };
 
-enum {
-	SRIOV_NONE,
-	SRIOV_LEGACY,
-	SRIOV_OFFLOADS
-};
-
 struct mlx5_esw_sq {
 	struct mlx5_flow_handle	*send_to_vport_rule;
 	struct list_head	 list;
@@ -222,7 +174,6 @@ struct esw_mc_addr { /* SRIOV only */
 
 struct mlx5_eswitch {
 	struct mlx5_core_dev    *dev;
-	struct mlx5_l2_table    l2_table;
 	struct mlx5_eswitch_fdb fdb_table;
 	struct hlist_head       mc_table[MLX5_L2_ADDR_HASH_SIZE];
 	struct workqueue_struct *work_queue;
@@ -250,8 +201,6 @@ int esw_offloads_init(struct mlx5_eswitch *esw, int nvports);
 /* E-Switch API */
 int mlx5_eswitch_init(struct mlx5_core_dev *dev);
 void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw);
-void mlx5_eswitch_attach(struct mlx5_eswitch *esw);
-void mlx5_eswitch_detach(struct mlx5_eswitch *esw);
 void mlx5_eswitch_vport_event(struct mlx5_eswitch *esw, struct mlx5_eqe *eqe);
 int mlx5_eswitch_enable_sriov(struct mlx5_eswitch *esw, int nvfs, int mode);
 void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw);
@@ -345,4 +294,13 @@ int __mlx5_eswitch_set_vport_vlan(struct mlx5_eswitch *esw,
 
 #define esw_debug(dev, format, ...)				\
 	mlx5_core_dbg_mask(dev, MLX5_DEBUG_ESWITCH_MASK, format, ##__VA_ARGS__)
+#else  /* CONFIG_MLX5_ESWITCH */
+/* eswitch API stubs */
+static inline int  mlx5_eswitch_init(struct mlx5_core_dev *dev) { return 0; }
+static inline void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw) {}
+static inline void mlx5_eswitch_vport_event(struct mlx5_eswitch *esw, struct mlx5_eqe *eqe) {}
+static inline int  mlx5_eswitch_enable_sriov(struct mlx5_eswitch *esw, int nvfs, int mode) { return 0; }
+static inline void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw) {}
+#endif /* CONFIG_MLX5_ESWITCH */
+
 #endif /* __MLX5_ESWITCH_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
index 95b64025ce36..d9fd8570b07c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
@@ -433,6 +433,8 @@ static int esw_create_offloads_fast_fdb_table(struct mlx5_eswitch *esw)
 	struct mlx5_flow_table *fdb = NULL;
 	int esw_size, err = 0;
 	u32 flags = 0;
+	u32 max_flow_counter = (MLX5_CAP_GEN(dev, max_flow_counter_31_16) << 16) |
+				MLX5_CAP_GEN(dev, max_flow_counter_15_0);
 
 	root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB);
 	if (!root_ns) {
@@ -443,9 +445,9 @@ static int esw_create_offloads_fast_fdb_table(struct mlx5_eswitch *esw)
 
 	esw_debug(dev, "Create offloads FDB table, min (max esw size(2^%d), max counters(%d)*groups(%d))\n",
 		  MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size),
-		  MLX5_CAP_GEN(dev, max_flow_counter), ESW_OFFLOADS_NUM_GROUPS);
+		  max_flow_counter, ESW_OFFLOADS_NUM_GROUPS);
 
-	esw_size = min_t(int, MLX5_CAP_GEN(dev, max_flow_counter) * ESW_OFFLOADS_NUM_GROUPS,
+	esw_size = min_t(int, max_flow_counter * ESW_OFFLOADS_NUM_GROUPS,
 			 1 << MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size));
 
 	if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
@@ -815,7 +817,7 @@ void esw_offloads_cleanup(struct mlx5_eswitch *esw, int nvports)
 	struct mlx5_eswitch_rep *rep;
 	int vport;
 
-	for (vport = 0; vport < nvports; vport++) {
+	for (vport = nvports - 1; vport >= 0; vport--) {
 		rep = &esw->offloads.vport_reps[vport];
 		if (!rep->valid)
 			continue;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c
index e750f07793b8..e0d0efd903bc 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c
@@ -263,7 +263,7 @@ static int mlx5_cmd_set_fte(struct mlx5_core_dev *dev,
 	MLX5_SET(flow_context, in_flow_context, modify_header_id, fte->modify_id);
 	in_match_value = MLX5_ADDR_OF(flow_context, in_flow_context,
 				      match_value);
-	memcpy(in_match_value, &fte->val, MLX5_ST_SZ_BYTES(fte_match_param));
+	memcpy(in_match_value, &fte->val, sizeof(fte->val));
 
 	in_dests = MLX5_ADDR_OF(flow_context, in_flow_context, destination);
 	if (fte->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
@@ -359,7 +359,7 @@ int mlx5_cmd_delete_fte(struct mlx5_core_dev *dev,
 	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 
-int mlx5_cmd_fc_alloc(struct mlx5_core_dev *dev, u16 *id)
+int mlx5_cmd_fc_alloc(struct mlx5_core_dev *dev, u32 *id)
 {
 	u32 in[MLX5_ST_SZ_DW(alloc_flow_counter_in)]   = {0};
 	u32 out[MLX5_ST_SZ_DW(alloc_flow_counter_out)] = {0};
@@ -374,7 +374,7 @@ int mlx5_cmd_fc_alloc(struct mlx5_core_dev *dev, u16 *id)
 	return err;
 }
 
-int mlx5_cmd_fc_free(struct mlx5_core_dev *dev, u16 id)
+int mlx5_cmd_fc_free(struct mlx5_core_dev *dev, u32 id)
 {
 	u32 in[MLX5_ST_SZ_DW(dealloc_flow_counter_in)]   = {0};
 	u32 out[MLX5_ST_SZ_DW(dealloc_flow_counter_out)] = {0};
@@ -385,7 +385,7 @@ int mlx5_cmd_fc_free(struct mlx5_core_dev *dev, u16 id)
 	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 
-int mlx5_cmd_fc_query(struct mlx5_core_dev *dev, u16 id,
+int mlx5_cmd_fc_query(struct mlx5_core_dev *dev, u32 id,
 		      u64 *packets, u64 *bytes)
 {
 	u32 out[MLX5_ST_SZ_BYTES(query_flow_counter_out) +
@@ -409,14 +409,14 @@ int mlx5_cmd_fc_query(struct mlx5_core_dev *dev, u16 id,
 }
 
 struct mlx5_cmd_fc_bulk {
-	u16 id;
+	u32 id;
 	int num;
 	int outlen;
 	u32 out[0];
 };
 
 struct mlx5_cmd_fc_bulk *
-mlx5_cmd_fc_bulk_alloc(struct mlx5_core_dev *dev, u16 id, int num)
+mlx5_cmd_fc_bulk_alloc(struct mlx5_core_dev *dev, u32 id, int num)
 {
 	struct mlx5_cmd_fc_bulk *b;
 	int outlen =
@@ -453,7 +453,7 @@ mlx5_cmd_fc_bulk_query(struct mlx5_core_dev *dev, struct mlx5_cmd_fc_bulk *b)
 }
 
 void mlx5_cmd_fc_bulk_get(struct mlx5_core_dev *dev,
-			  struct mlx5_cmd_fc_bulk *b, u16 id,
+			  struct mlx5_cmd_fc_bulk *b, u32 id,
 			  u64 *packets, u64 *bytes)
 {
 	int index = id - b->id;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h
index 0f98a7cf4877..c6d7bdf255b6 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h
@@ -74,20 +74,20 @@ int mlx5_cmd_update_root_ft(struct mlx5_core_dev *dev,
 			    struct mlx5_flow_table *ft,
 			    u32 underlay_qpn);
 
-int mlx5_cmd_fc_alloc(struct mlx5_core_dev *dev, u16 *id);
-int mlx5_cmd_fc_free(struct mlx5_core_dev *dev, u16 id);
-int mlx5_cmd_fc_query(struct mlx5_core_dev *dev, u16 id,
+int mlx5_cmd_fc_alloc(struct mlx5_core_dev *dev, u32 *id);
+int mlx5_cmd_fc_free(struct mlx5_core_dev *dev, u32 id);
+int mlx5_cmd_fc_query(struct mlx5_core_dev *dev, u32 id,
 		      u64 *packets, u64 *bytes);
 
 struct mlx5_cmd_fc_bulk;
 
 struct mlx5_cmd_fc_bulk *
-mlx5_cmd_fc_bulk_alloc(struct mlx5_core_dev *dev, u16 id, int num);
+mlx5_cmd_fc_bulk_alloc(struct mlx5_core_dev *dev, u32 id, int num);
 void mlx5_cmd_fc_bulk_free(struct mlx5_cmd_fc_bulk *b);
 int
 mlx5_cmd_fc_bulk_query(struct mlx5_core_dev *dev, struct mlx5_cmd_fc_bulk *b);
 void mlx5_cmd_fc_bulk_get(struct mlx5_core_dev *dev,
-			  struct mlx5_cmd_fc_bulk *b, u16 id,
+			  struct mlx5_cmd_fc_bulk *b, u32 id,
 			  u64 *packets, u64 *bytes);
 
 #endif
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
index e8690fe46bf2..5a7bea688ec8 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
@@ -36,6 +36,7 @@
 #include "mlx5_core.h"
 #include "fs_core.h"
 #include "fs_cmd.h"
+#include "diag/fs_tracepoint.h"
 
 #define INIT_TREE_NODE_ARRAY_SIZE(...)	(sizeof((struct init_tree_node[]){__VA_ARGS__}) /\
 					 sizeof(struct init_tree_node))
@@ -82,8 +83,8 @@
 #define ETHTOOL_PRIO_NUM_LEVELS 1
 #define ETHTOOL_NUM_PRIOS 11
 #define ETHTOOL_MIN_LEVEL (KERNEL_MIN_LEVEL + ETHTOOL_NUM_PRIOS)
-/* Vlan, mac, ttc, aRFS */
-#define KERNEL_NIC_PRIO_NUM_LEVELS 4
+/* Vlan, mac, ttc, inner ttc, aRFS */
+#define KERNEL_NIC_PRIO_NUM_LEVELS 5
 #define KERNEL_NIC_NUM_PRIOS 1
 /* One more level for tc */
 #define KERNEL_MIN_LEVEL (KERNEL_NIC_PRIO_NUM_LEVELS + 1)
@@ -150,6 +151,23 @@ enum fs_i_mutex_lock_class {
 	FS_MUTEX_CHILD
 };
 
+static const struct rhashtable_params rhash_fte = {
+	.key_len = FIELD_SIZEOF(struct fs_fte, val),
+	.key_offset = offsetof(struct fs_fte, val),
+	.head_offset = offsetof(struct fs_fte, hash),
+	.automatic_shrinking = true,
+	.min_size = 1,
+};
+
+static const struct rhashtable_params rhash_fg = {
+	.key_len = FIELD_SIZEOF(struct mlx5_flow_group, mask),
+	.key_offset = offsetof(struct mlx5_flow_group, mask),
+	.head_offset = offsetof(struct mlx5_flow_group, hash),
+	.automatic_shrinking = true,
+	.min_size = 1,
+
+};
+
 static void del_rule(struct fs_node *node);
 static void del_flow_table(struct fs_node *node);
 static void del_flow_group(struct fs_node *node);
@@ -255,71 +273,77 @@ static struct fs_prio *find_prio(struct mlx5_flow_namespace *ns,
 	return NULL;
 }
 
-static bool masked_memcmp(void *mask, void *val1, void *val2, size_t size)
+static bool check_last_reserved(const u32 *match_criteria)
 {
-	unsigned int i;
+	char *match_criteria_reserved =
+		MLX5_ADDR_OF(fte_match_param, match_criteria, MLX5_FTE_MATCH_PARAM_RESERVED);
 
-	for (i = 0; i < size; i++, mask++, val1++, val2++)
-		if ((*((u8 *)val1) & (*(u8 *)mask)) !=
-		    ((*(u8 *)val2) & (*(u8 *)mask)))
-			return false;
-
-	return true;
+	return	!match_criteria_reserved[0] &&
+		!memcmp(match_criteria_reserved, match_criteria_reserved + 1,
+			MLX5_FLD_SZ_BYTES(fte_match_param,
+					  MLX5_FTE_MATCH_PARAM_RESERVED) - 1);
 }
 
-static bool compare_match_value(struct mlx5_flow_group_mask *mask,
-				void *fte_param1, void *fte_param2)
+static bool check_valid_mask(u8 match_criteria_enable, const u32 *match_criteria)
 {
-	if (mask->match_criteria_enable &
-	    1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS) {
-		void *fte_match1 = MLX5_ADDR_OF(fte_match_param,
-						fte_param1, outer_headers);
-		void *fte_match2 = MLX5_ADDR_OF(fte_match_param,
-						fte_param2, outer_headers);
-		void *fte_mask = MLX5_ADDR_OF(fte_match_param,
-					      mask->match_criteria, outer_headers);
+	if (match_criteria_enable & ~(
+		(1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS)   |
+		(1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS) |
+		(1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS)))
+		return false;
 
-		if (!masked_memcmp(fte_mask, fte_match1, fte_match2,
-				   MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4)))
+	if (!(match_criteria_enable &
+	      1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS)) {
+		char *fg_type_mask = MLX5_ADDR_OF(fte_match_param,
+						  match_criteria, outer_headers);
+
+		if (fg_type_mask[0] ||
+		    memcmp(fg_type_mask, fg_type_mask + 1,
+			   MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4) - 1))
 			return false;
 	}
 
-	if (mask->match_criteria_enable &
-	    1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS) {
-		void *fte_match1 = MLX5_ADDR_OF(fte_match_param,
-						fte_param1, misc_parameters);
-		void *fte_match2 = MLX5_ADDR_OF(fte_match_param,
-						fte_param2, misc_parameters);
-		void *fte_mask = MLX5_ADDR_OF(fte_match_param,
-					  mask->match_criteria, misc_parameters);
+	if (!(match_criteria_enable &
+	      1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS)) {
+		char *fg_type_mask = MLX5_ADDR_OF(fte_match_param,
+						  match_criteria, misc_parameters);
 
-		if (!masked_memcmp(fte_mask, fte_match1, fte_match2,
-				   MLX5_ST_SZ_BYTES(fte_match_set_misc)))
+		if (fg_type_mask[0] ||
+		    memcmp(fg_type_mask, fg_type_mask + 1,
+			   MLX5_ST_SZ_BYTES(fte_match_set_misc) - 1))
 			return false;
 	}
 
-	if (mask->match_criteria_enable &
-	    1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS) {
-		void *fte_match1 = MLX5_ADDR_OF(fte_match_param,
-						fte_param1, inner_headers);
-		void *fte_match2 = MLX5_ADDR_OF(fte_match_param,
-						fte_param2, inner_headers);
-		void *fte_mask = MLX5_ADDR_OF(fte_match_param,
-					  mask->match_criteria, inner_headers);
+	if (!(match_criteria_enable &
+	      1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS)) {
+		char *fg_type_mask = MLX5_ADDR_OF(fte_match_param,
+						  match_criteria, inner_headers);
 
-		if (!masked_memcmp(fte_mask, fte_match1, fte_match2,
-				   MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4)))
+		if (fg_type_mask[0] ||
+		    memcmp(fg_type_mask, fg_type_mask + 1,
+			   MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4) - 1))
 			return false;
 	}
-	return true;
+
+	return check_last_reserved(match_criteria);
 }
 
-static bool compare_match_criteria(u8 match_criteria_enable1,
-				   u8 match_criteria_enable2,
-				   void *mask1, void *mask2)
+static bool check_valid_spec(const struct mlx5_flow_spec *spec)
 {
-	return match_criteria_enable1 == match_criteria_enable2 &&
-		!memcmp(mask1, mask2, MLX5_ST_SZ_BYTES(fte_match_param));
+	int i;
+
+	if (!check_valid_mask(spec->match_criteria_enable, spec->match_criteria)) {
+		pr_warn("mlx5_core: Match criteria given mismatches match_criteria_enable\n");
+		return false;
+	}
+
+	for (i = 0; i < MLX5_ST_SZ_DW_MATCH_PARAM; i++)
+		if (spec->match_value[i] & ~spec->match_criteria[i]) {
+			pr_warn("mlx5_core: match_value differs from match_criteria\n");
+			return false;
+		}
+
+	return check_last_reserved(spec->match_value);
 }
 
 static struct mlx5_flow_root_namespace *find_root(struct fs_node *node)
@@ -360,6 +384,8 @@ static void del_flow_table(struct fs_node *node)
 	err = mlx5_cmd_destroy_flow_table(dev, ft);
 	if (err)
 		mlx5_core_warn(dev, "flow steering can't destroy ft\n");
+	ida_destroy(&ft->fte_allocator);
+	rhltable_destroy(&ft->fgs_hash);
 	fs_get_obj(prio, ft->node.parent);
 	prio->num_ft--;
 }
@@ -370,22 +396,16 @@ static void del_rule(struct fs_node *node)
 	struct mlx5_flow_table *ft;
 	struct mlx5_flow_group *fg;
 	struct fs_fte *fte;
-	u32	*match_value;
 	int modify_mask;
 	struct mlx5_core_dev *dev = get_dev(node);
-	int match_len = MLX5_ST_SZ_BYTES(fte_match_param);
 	int err;
 	bool update_fte = false;
 
-	match_value = kvzalloc(match_len, GFP_KERNEL);
-	if (!match_value)
-		return;
-
 	fs_get_obj(rule, node);
 	fs_get_obj(fte, rule->node.parent);
 	fs_get_obj(fg, fte->node.parent);
-	memcpy(match_value, fte->val, sizeof(fte->val));
 	fs_get_obj(ft, fg->node.parent);
+	trace_mlx5_fs_del_rule(rule);
 	list_del(&rule->node.list);
 	if (rule->sw_action == MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO) {
 		mutex_lock(&rule->dest_attr.ft->lock);
@@ -414,7 +434,18 @@ out:
 				       "%s can't del rule fg id=%d fte_index=%d\n",
 				       __func__, fg->id, fte->index);
 	}
-	kvfree(match_value);
+}
+
+static void destroy_fte(struct fs_fte *fte, struct mlx5_flow_group *fg)
+{
+	struct mlx5_flow_table *ft;
+	int ret;
+
+	ret = rhashtable_remove_fast(&fg->ftes_hash, &fte->hash, rhash_fte);
+	WARN_ON(ret);
+	fte->status = 0;
+	fs_get_obj(ft, fg->node.parent);
+	ida_simple_remove(&ft->fte_allocator, fte->index);
 }
 
 static void del_fte(struct fs_node *node)
@@ -428,6 +459,7 @@ static void del_fte(struct fs_node *node)
 	fs_get_obj(fte, node);
 	fs_get_obj(fg, fte->node.parent);
 	fs_get_obj(ft, fg->node.parent);
+	trace_mlx5_fs_del_fte(fte);
 
 	dev = get_dev(&ft->node);
 	err = mlx5_cmd_delete_fte(dev, ft,
@@ -437,8 +469,7 @@ static void del_fte(struct fs_node *node)
 			       "flow steering can't delete fte in index %d of flow group id %d\n",
 			       fte->index, fg->id);
 
-	fte->status = 0;
-	fg->num_ftes--;
+	destroy_fte(fte, fg);
 }
 
 static void del_flow_group(struct fs_node *node)
@@ -446,14 +477,21 @@ static void del_flow_group(struct fs_node *node)
 	struct mlx5_flow_group *fg;
 	struct mlx5_flow_table *ft;
 	struct mlx5_core_dev *dev;
+	int err;
 
 	fs_get_obj(fg, node);
 	fs_get_obj(ft, fg->node.parent);
 	dev = get_dev(&ft->node);
+	trace_mlx5_fs_del_fg(fg);
 
 	if (ft->autogroup.active)
 		ft->autogroup.num_groups--;
 
+	rhashtable_destroy(&fg->ftes_hash);
+	err = rhltable_remove(&ft->fgs_hash,
+			      &fg->hash,
+			      rhash_fg);
+	WARN_ON(err);
 	if (mlx5_cmd_destroy_flow_group(dev, ft, fg->id))
 		mlx5_core_warn(dev, "flow steering can't destroy fg %d of ft %d\n",
 			       fg->id, ft->id);
@@ -488,10 +526,17 @@ static struct mlx5_flow_group *alloc_flow_group(u32 *create_fg_in)
 	u8 match_criteria_enable = MLX5_GET(create_flow_group_in,
 					    create_fg_in,
 					    match_criteria_enable);
+	int ret;
+
 	fg = kzalloc(sizeof(*fg), GFP_KERNEL);
 	if (!fg)
 		return ERR_PTR(-ENOMEM);
 
+	ret = rhashtable_init(&fg->ftes_hash, &rhash_fte);
+	if (ret) {
+		kfree(fg);
+		return ERR_PTR(ret);
+	}
 	fg->mask.match_criteria_enable = match_criteria_enable;
 	memcpy(&fg->mask.match_criteria, match_criteria,
 	       sizeof(fg->mask.match_criteria));
@@ -509,10 +554,17 @@ static struct mlx5_flow_table *alloc_flow_table(int level, u16 vport, int max_ft
 						u32 flags)
 {
 	struct mlx5_flow_table *ft;
+	int ret;
 
 	ft  = kzalloc(sizeof(*ft), GFP_KERNEL);
 	if (!ft)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
+
+	ret = rhltable_init(&ft->fgs_hash, &rhash_fg);
+	if (ret) {
+		kfree(ft);
+		return ERR_PTR(ret);
+	}
 
 	ft->level = level;
 	ft->node.type = FS_TYPE_FLOW_TABLE;
@@ -523,6 +575,7 @@ static struct mlx5_flow_table *alloc_flow_table(int level, u16 vport, int max_ft
 	ft->flags = flags;
 	INIT_LIST_HEAD(&ft->fwd_rules);
 	mutex_init(&ft->lock);
+	ida_init(&ft->fte_allocator);
 
 	return ft;
 }
@@ -812,8 +865,8 @@ static struct mlx5_flow_table *__mlx5_create_flow_table(struct mlx5_flow_namespa
 			      ft_attr->max_fte ? roundup_pow_of_two(ft_attr->max_fte) : 0,
 			      root->table_type,
 			      op_mod, ft_attr->flags);
-	if (!ft) {
-		err = -ENOMEM;
+	if (IS_ERR(ft)) {
+		err = PTR_ERR(ft);
 		goto unlock_root;
 	}
 
@@ -839,6 +892,7 @@ static struct mlx5_flow_table *__mlx5_create_flow_table(struct mlx5_flow_namespa
 destroy_ft:
 	mlx5_cmd_destroy_flow_table(root->dev, ft);
 free_ft:
+	ida_destroy(&ft->fte_allocator);
 	kfree(ft);
 unlock_root:
 	mutex_unlock(&root->chain_lock);
@@ -924,11 +978,13 @@ static struct mlx5_flow_group *create_flow_group_common(struct mlx5_flow_table *
 	if (IS_ERR(fg))
 		return fg;
 
+	err = rhltable_insert(&ft->fgs_hash, &fg->hash, rhash_fg);
+	if (err)
+		goto err_free_fg;
+
 	err = mlx5_cmd_create_flow_group(dev, ft, fg_in, &fg->id);
-	if (err) {
-		kfree(fg);
-		return ERR_PTR(err);
-	}
+	if (err)
+		goto err_remove_fg;
 
 	if (ft->autogroup.active)
 		ft->autogroup.num_groups++;
@@ -938,14 +994,33 @@ static struct mlx5_flow_group *create_flow_group_common(struct mlx5_flow_table *
 	/* Add node to group list */
 	list_add(&fg->node.list, prev_fg);
 
+	trace_mlx5_fs_add_fg(fg);
 	return fg;
+
+err_remove_fg:
+	WARN_ON(rhltable_remove(&ft->fgs_hash,
+				&fg->hash,
+				rhash_fg));
+err_free_fg:
+	rhashtable_destroy(&fg->ftes_hash);
+	kfree(fg);
+
+	return ERR_PTR(err);
 }
 
 struct mlx5_flow_group *mlx5_create_flow_group(struct mlx5_flow_table *ft,
 					       u32 *fg_in)
 {
+	void *match_criteria = MLX5_ADDR_OF(create_flow_group_in,
+					    fg_in, match_criteria);
+	u8 match_criteria_enable = MLX5_GET(create_flow_group_in,
+					    fg_in,
+					    match_criteria_enable);
 	struct mlx5_flow_group *fg;
 
+	if (!check_valid_mask(match_criteria_enable, match_criteria))
+		return ERR_PTR(-EINVAL);
+
 	if (ft->autogroup.active)
 		return ERR_PTR(-EPERM);
 
@@ -1102,43 +1177,38 @@ free_handle:
 	return ERR_PTR(err);
 }
 
-/* Assumed fg is locked */
-static unsigned int get_free_fte_index(struct mlx5_flow_group *fg,
-				       struct list_head **prev)
-{
-	struct fs_fte *fte;
-	unsigned int start = fg->start_index;
-
-	if (prev)
-		*prev = &fg->node.children;
-
-	/* assumed list is sorted by index */
-	fs_for_each_fte(fte, fg) {
-		if (fte->index != start)
-			return start;
-		start++;
-		if (prev)
-			*prev = &fte->node.list;
-	}
-
-	return start;
-}
-
-/* prev is output, prev->next = new_fte */
 static struct fs_fte *create_fte(struct mlx5_flow_group *fg,
 				 u32 *match_value,
-				 struct mlx5_flow_act *flow_act,
-				 struct list_head **prev)
+				 struct mlx5_flow_act *flow_act)
 {
+	struct mlx5_flow_table *ft;
 	struct fs_fte *fte;
 	int index;
+	int ret;
+
+	fs_get_obj(ft, fg->node.parent);
+	index = ida_simple_get(&ft->fte_allocator, fg->start_index,
+			       fg->start_index + fg->max_ftes,
+			       GFP_KERNEL);
+	if (index < 0)
+		return ERR_PTR(index);
 
-	index = get_free_fte_index(fg, prev);
 	fte = alloc_fte(flow_act, match_value, index);
-	if (IS_ERR(fte))
-		return fte;
+	if (IS_ERR(fte)) {
+		ret = PTR_ERR(fte);
+		goto err_alloc;
+	}
+	ret = rhashtable_insert_fast(&fg->ftes_hash, &fte->hash, rhash_fte);
+	if (ret)
+		goto err_hash;
 
 	return fte;
+
+err_hash:
+	kfree(fte);
+err_alloc:
+	ida_simple_remove(&ft->fte_allocator, index);
+	return ERR_PTR(ret);
 }
 
 static struct mlx5_flow_group *create_autogroup(struct mlx5_flow_table *ft,
@@ -1226,79 +1296,104 @@ static struct mlx5_flow_rule *find_flow_rule(struct fs_fte *fte,
 	return NULL;
 }
 
+static bool check_conflicting_actions(u32 action1, u32 action2)
+{
+	u32 xored_actions = action1 ^ action2;
+
+	/* if one rule only wants to count, it's ok */
+	if (action1 == MLX5_FLOW_CONTEXT_ACTION_COUNT ||
+	    action2 == MLX5_FLOW_CONTEXT_ACTION_COUNT)
+		return false;
+
+	if (xored_actions & (MLX5_FLOW_CONTEXT_ACTION_DROP  |
+			     MLX5_FLOW_CONTEXT_ACTION_ENCAP |
+			     MLX5_FLOW_CONTEXT_ACTION_DECAP))
+		return true;
+
+	return false;
+}
+
+static int check_conflicting_ftes(struct fs_fte *fte, const struct mlx5_flow_act *flow_act)
+{
+	if (check_conflicting_actions(flow_act->action, fte->action)) {
+		mlx5_core_warn(get_dev(&fte->node),
+			       "Found two FTEs with conflicting actions\n");
+		return -EEXIST;
+	}
+
+	if (fte->flow_tag != flow_act->flow_tag) {
+		mlx5_core_warn(get_dev(&fte->node),
+			       "FTE flow tag %u already exists with different flow tag %u\n",
+			       fte->flow_tag,
+			       flow_act->flow_tag);
+		return -EEXIST;
+	}
+
+	return 0;
+}
+
 static struct mlx5_flow_handle *add_rule_fg(struct mlx5_flow_group *fg,
 					    u32 *match_value,
 					    struct mlx5_flow_act *flow_act,
 					    struct mlx5_flow_destination *dest,
-					    int dest_num)
+					    int dest_num,
+					    struct fs_fte *fte)
 {
 	struct mlx5_flow_handle *handle;
 	struct mlx5_flow_table *ft;
-	struct list_head *prev;
-	struct fs_fte *fte;
 	int i;
 
-	nested_lock_ref_node(&fg->node, FS_MUTEX_PARENT);
-	fs_for_each_fte(fte, fg) {
+	if (fte) {
+		int old_action;
+		int ret;
+
 		nested_lock_ref_node(&fte->node, FS_MUTEX_CHILD);
-		if (compare_match_value(&fg->mask, match_value, &fte->val) &&
-		    (flow_act->action & fte->action)) {
-			int old_action = fte->action;
-
-			if (fte->flow_tag != flow_act->flow_tag) {
-				mlx5_core_warn(get_dev(&fte->node),
-					       "FTE flow tag %u already exists with different flow tag %u\n",
-					       fte->flow_tag,
-					       flow_act->flow_tag);
-				handle = ERR_PTR(-EEXIST);
-				goto unlock_fte;
-			}
+		ret = check_conflicting_ftes(fte, flow_act);
+		if (ret) {
+			handle = ERR_PTR(ret);
+			goto unlock_fte;
+		}
 
-			fte->action |= flow_act->action;
-			handle = add_rule_fte(fte, fg, dest, dest_num,
-					      old_action != flow_act->action);
-			if (IS_ERR(handle)) {
-				fte->action = old_action;
-				goto unlock_fte;
-			} else {
-				goto add_rules;
-			}
+		old_action = fte->action;
+		fte->action |= flow_act->action;
+		handle = add_rule_fte(fte, fg, dest, dest_num,
+				      old_action != flow_act->action);
+		if (IS_ERR(handle)) {
+			fte->action = old_action;
+			goto unlock_fte;
+		} else {
+			trace_mlx5_fs_set_fte(fte, false);
+			goto add_rules;
 		}
-		unlock_ref_node(&fte->node);
 	}
 	fs_get_obj(ft, fg->node.parent);
-	if (fg->num_ftes >= fg->max_ftes) {
-		handle = ERR_PTR(-ENOSPC);
-		goto unlock_fg;
-	}
 
-	fte = create_fte(fg, match_value, flow_act, &prev);
-	if (IS_ERR(fte)) {
-		handle = (void *)fte;
-		goto unlock_fg;
-	}
+	fte = create_fte(fg, match_value, flow_act);
+	if (IS_ERR(fte))
+		return (void *)fte;
 	tree_init_node(&fte->node, 0, del_fte);
 	nested_lock_ref_node(&fte->node, FS_MUTEX_CHILD);
 	handle = add_rule_fte(fte, fg, dest, dest_num, false);
 	if (IS_ERR(handle)) {
 		unlock_ref_node(&fte->node);
+		destroy_fte(fte, fg);
 		kfree(fte);
-		goto unlock_fg;
+		return handle;
 	}
 
-	fg->num_ftes++;
-
 	tree_add_node(&fte->node, &fg->node);
-	list_add(&fte->node.list, prev);
+	/* fte list isn't sorted */
+	list_add_tail(&fte->node.list, &fg->node.children);
+	trace_mlx5_fs_set_fte(fte, true);
 add_rules:
 	for (i = 0; i < handle->num_rules; i++) {
-		if (atomic_read(&handle->rule[i]->node.refcount) == 1)
+		if (atomic_read(&handle->rule[i]->node.refcount) == 1) {
 			tree_add_node(&handle->rule[i]->node, &fte->node);
+			trace_mlx5_fs_add_rule(handle->rule[i]);
+		}
 	}
 unlock_fte:
 	unlock_ref_node(&fte->node);
-unlock_fg:
-	unlock_ref_node(&fg->node);
 	return handle;
 }
 
@@ -1347,6 +1442,96 @@ static bool dest_is_valid(struct mlx5_flow_destination *dest,
 }
 
 static struct mlx5_flow_handle *
+try_add_to_existing_fg(struct mlx5_flow_table *ft,
+		       struct mlx5_flow_spec *spec,
+		       struct mlx5_flow_act *flow_act,
+		       struct mlx5_flow_destination *dest,
+		       int dest_num)
+{
+	struct mlx5_flow_group *g;
+	struct mlx5_flow_handle *rule = ERR_PTR(-ENOENT);
+	struct rhlist_head *tmp, *list;
+	struct match_list {
+		struct list_head	list;
+		struct mlx5_flow_group *g;
+	} match_list, *iter;
+	LIST_HEAD(match_head);
+
+	rcu_read_lock();
+	/* Collect all fgs which has a matching match_criteria */
+	list = rhltable_lookup(&ft->fgs_hash, spec, rhash_fg);
+	rhl_for_each_entry_rcu(g, tmp, list, hash) {
+		struct match_list *curr_match;
+
+		if (likely(list_empty(&match_head))) {
+			match_list.g = g;
+			list_add_tail(&match_list.list, &match_head);
+			continue;
+		}
+		curr_match = kmalloc(sizeof(*curr_match), GFP_ATOMIC);
+
+		if (!curr_match) {
+			rcu_read_unlock();
+			rule = ERR_PTR(-ENOMEM);
+			goto free_list;
+		}
+		curr_match->g = g;
+		list_add_tail(&curr_match->list, &match_head);
+	}
+	rcu_read_unlock();
+
+	/* Try to find a fg that already contains a matching fte */
+	list_for_each_entry(iter, &match_head, list) {
+		struct fs_fte *fte;
+
+		g = iter->g;
+		nested_lock_ref_node(&g->node, FS_MUTEX_PARENT);
+		fte = rhashtable_lookup_fast(&g->ftes_hash, spec->match_value,
+					     rhash_fte);
+		if (fte) {
+			rule = add_rule_fg(g, spec->match_value,
+					   flow_act, dest, dest_num, fte);
+			unlock_ref_node(&g->node);
+			goto free_list;
+		}
+		unlock_ref_node(&g->node);
+	}
+
+	/* No group with matching fte found. Try to add a new fte to any
+	 * matching fg.
+	 */
+	list_for_each_entry(iter, &match_head, list) {
+		g = iter->g;
+
+		nested_lock_ref_node(&g->node, FS_MUTEX_PARENT);
+		rule = add_rule_fg(g, spec->match_value,
+				   flow_act, dest, dest_num, NULL);
+		if (!IS_ERR(rule) || PTR_ERR(rule) != -ENOSPC) {
+			unlock_ref_node(&g->node);
+			goto free_list;
+		}
+		unlock_ref_node(&g->node);
+	}
+
+free_list:
+	if (!list_empty(&match_head)) {
+		struct match_list *match_tmp;
+
+		/* The most common case is having one FG. Since we want to
+		 * optimize this case, we save the first on the stack.
+		 * Therefore, no need to free it.
+		 */
+		list_del(&list_first_entry(&match_head, typeof(*iter), list)->list);
+		list_for_each_entry_safe(iter, match_tmp, &match_head, list) {
+			list_del(&iter->list);
+			kfree(iter);
+		}
+	}
+
+	return rule;
+}
+
+static struct mlx5_flow_handle *
 _mlx5_add_flow_rules(struct mlx5_flow_table *ft,
 		     struct mlx5_flow_spec *spec,
 		     struct mlx5_flow_act *flow_act,
@@ -1358,22 +1543,18 @@ _mlx5_add_flow_rules(struct mlx5_flow_table *ft,
 	struct mlx5_flow_handle *rule;
 	int i;
 
+	if (!check_valid_spec(spec))
+		return ERR_PTR(-EINVAL);
+
 	for (i = 0; i < dest_num; i++) {
 		if (!dest_is_valid(&dest[i], flow_act->action, ft))
 			return ERR_PTR(-EINVAL);
 	}
 
 	nested_lock_ref_node(&ft->node, FS_MUTEX_GRANDPARENT);
-	fs_for_each_fg(g, ft)
-		if (compare_match_criteria(g->mask.match_criteria_enable,
-					   spec->match_criteria_enable,
-					   g->mask.match_criteria,
-					   spec->match_criteria)) {
-			rule = add_rule_fg(g, spec->match_value,
-					   flow_act, dest, dest_num);
-			if (!IS_ERR(rule) || PTR_ERR(rule) != -ENOSPC)
-				goto unlock;
-		}
+	rule = try_add_to_existing_fg(ft, spec, flow_act, dest, dest_num);
+	if (!IS_ERR(rule))
+		goto unlock;
 
 	g = create_autogroup(ft, spec->match_criteria_enable,
 			     spec->match_criteria);
@@ -1382,7 +1563,8 @@ _mlx5_add_flow_rules(struct mlx5_flow_table *ft,
 		goto unlock;
 	}
 
-	rule = add_rule_fg(g, spec->match_value, flow_act, dest, dest_num);
+	rule = add_rule_fg(g, spec->match_value, flow_act, dest,
+			   dest_num, NULL);
 	if (IS_ERR(rule)) {
 		/* Remove assumes refcount > 0 and autogroup creates a group
 		 * with a refcount = 0.
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h
index 990acee6fb09..5509a752f98e 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h
@@ -34,6 +34,7 @@
 #define _MLX5_FS_CORE_
 
 #include <linux/mlx5/fs.h>
+#include <linux/rhashtable.h>
 
 enum fs_node_type {
 	FS_TYPE_NAMESPACE,
@@ -118,6 +119,8 @@ struct mlx5_flow_table {
 	/* FWD rules that point on this flow table */
 	struct list_head		fwd_rules;
 	u32				flags;
+	struct ida			fte_allocator;
+	struct rhltable			fgs_hash;
 };
 
 struct mlx5_fc_cache {
@@ -136,17 +139,29 @@ struct mlx5_fc {
 	u64 lastpackets;
 	u64 lastbytes;
 
-	u16 id;
+	u32 id;
 	bool deleted;
 	bool aging;
 
 	struct mlx5_fc_cache cache ____cacheline_aligned_in_smp;
 };
 
+#define MLX5_FTE_MATCH_PARAM_RESERVED	reserved_at_600
+/* Calculate the fte_match_param length and without the reserved length.
+ * Make sure the reserved field is the last.
+ */
+#define MLX5_ST_SZ_DW_MATCH_PARAM					    \
+	((MLX5_BYTE_OFF(fte_match_param, MLX5_FTE_MATCH_PARAM_RESERVED) / sizeof(u32)) + \
+	 BUILD_BUG_ON_ZERO(MLX5_ST_SZ_BYTES(fte_match_param) !=		     \
+			   MLX5_FLD_SZ_BYTES(fte_match_param,		     \
+					     MLX5_FTE_MATCH_PARAM_RESERVED) +\
+			   MLX5_BYTE_OFF(fte_match_param,		     \
+					 MLX5_FTE_MATCH_PARAM_RESERVED)))
+
 /* Type of children is mlx5_flow_rule */
 struct fs_fte {
 	struct fs_node			node;
-	u32				val[MLX5_ST_SZ_DW(fte_match_param)];
+	u32				val[MLX5_ST_SZ_DW_MATCH_PARAM];
 	u32				dests_size;
 	u32				flow_tag;
 	u32				index;
@@ -155,6 +170,7 @@ struct fs_fte {
 	u32				modify_id;
 	enum fs_fte_status		status;
 	struct mlx5_fc			*counter;
+	struct rhash_head		hash;
 };
 
 /* Type of children is mlx5_flow_table/namespace */
@@ -174,7 +190,7 @@ struct mlx5_flow_namespace {
 
 struct mlx5_flow_group_mask {
 	u8	match_criteria_enable;
-	u32	match_criteria[MLX5_ST_SZ_DW(fte_match_param)];
+	u32	match_criteria[MLX5_ST_SZ_DW_MATCH_PARAM];
 };
 
 /* Type of children is fs_fte */
@@ -183,8 +199,9 @@ struct mlx5_flow_group {
 	struct mlx5_flow_group_mask	mask;
 	u32				start_index;
 	u32				max_ftes;
-	u32				num_ftes;
 	u32				id;
+	struct rhashtable		ftes_hash;
+	struct rhlist_head		hash;
 };
 
 struct mlx5_flow_root_namespace {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c
index 6507d8acc54d..89d1f8650033 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c
@@ -38,6 +38,8 @@
 #include "fs_cmd.h"
 
 #define MLX5_FC_STATS_PERIOD msecs_to_jiffies(1000)
+/* Max number of counters to query in bulk read is 32K */
+#define MLX5_SW_MAX_COUNTERS_BULK BIT(15)
 
 /* locking scheme:
  *
@@ -90,16 +92,21 @@ static void mlx5_fc_stats_insert(struct rb_root *root, struct mlx5_fc *counter)
 	rb_insert_color(&counter->node, root);
 }
 
+/* The function returns the last node that was queried so the caller
+ * function can continue calling it till all counters are queried.
+ */
 static struct rb_node *mlx5_fc_stats_query(struct mlx5_core_dev *dev,
 					   struct mlx5_fc *first,
-					   u16 last_id)
+					   u32 last_id)
 {
 	struct mlx5_cmd_fc_bulk *b;
 	struct rb_node *node = NULL;
-	u16 afirst_id;
+	u32 afirst_id;
 	int num;
 	int err;
-	int max_bulk = 1 << MLX5_CAP_GEN(dev, log_max_flow_counter_bulk);
+
+	int max_bulk = min_t(int, MLX5_SW_MAX_COUNTERS_BULK,
+			     (1 << MLX5_CAP_GEN(dev, log_max_flow_counter_bulk)));
 
 	/* first id must be aligned to 4 when using bulk query */
 	afirst_id = first->id & ~0x3;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c
index fa33d59ab485..2c71557d1cee 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c
@@ -120,6 +120,12 @@ int mlx5_query_hca_caps(struct mlx5_core_dev *dev)
 			return err;
 	}
 
+	if (MLX5_CAP_GEN(dev, ipoib_enhanced_offloads)) {
+		err = mlx5_core_get_caps(dev, MLX5_CAP_IPOIB_ENHANCED_OFFLOADS);
+		if (err)
+			return err;
+	}
+
 	if (MLX5_CAP_GEN(dev, pg)) {
 		err = mlx5_core_get_caps(dev, MLX5_CAP_ODP);
 		if (err)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c
index 4b6b03d6297f..8aea0a065e56 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/health.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c
@@ -81,7 +81,7 @@ static void trigger_cmd_completions(struct mlx5_core_dev *dev)
 	u64 vector;
 
 	/* wait for pending handlers to complete */
-	synchronize_irq(dev->priv.msix_arr[MLX5_EQ_VEC_CMD].vector);
+	synchronize_irq(pci_irq_vector(dev->pdev, MLX5_EQ_VEC_CMD));
 	spin_lock_irqsave(&dev->cmd.alloc_lock, flags);
 	vector = ~dev->cmd.bitmask & ((1ul << (1 << dev->cmd.log_sz)) - 1);
 	if (!vector)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c
index eb04e97d8765..43c126c63955 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c
@@ -39,10 +39,11 @@ static void mlx5i_get_drvinfo(struct net_device *dev,
 	struct mlx5e_priv *priv = mlx5i_epriv(dev);
 
 	mlx5e_ethtool_get_drvinfo(priv, drvinfo);
+	strlcpy(drvinfo->driver, DRIVER_NAME "[ib_ipoib]",
+		sizeof(drvinfo->driver));
 }
 
-static void mlx5i_get_strings(struct net_device *dev,
-			      uint32_t stringset, uint8_t *data)
+static void mlx5i_get_strings(struct net_device *dev, u32 stringset, u8 *data)
 {
 	struct mlx5e_priv *priv  = mlx5i_epriv(dev);
 
@@ -129,17 +130,123 @@ static int mlx5i_flash_device(struct net_device *netdev,
 	return mlx5e_ethtool_flash_device(priv, flash);
 }
 
+enum mlx5_ptys_width {
+	MLX5_PTYS_WIDTH_1X	= 1 << 0,
+	MLX5_PTYS_WIDTH_2X	= 1 << 1,
+	MLX5_PTYS_WIDTH_4X	= 1 << 2,
+	MLX5_PTYS_WIDTH_8X	= 1 << 3,
+	MLX5_PTYS_WIDTH_12X	= 1 << 4,
+};
+
+static inline int mlx5_ptys_width_enum_to_int(enum mlx5_ptys_width width)
+{
+	switch (width) {
+	case MLX5_PTYS_WIDTH_1X:  return  1;
+	case MLX5_PTYS_WIDTH_2X:  return  2;
+	case MLX5_PTYS_WIDTH_4X:  return  4;
+	case MLX5_PTYS_WIDTH_8X:  return  8;
+	case MLX5_PTYS_WIDTH_12X: return 12;
+	default:		  return -1;
+	}
+}
+
+enum mlx5_ptys_rate {
+	MLX5_PTYS_RATE_SDR	= 1 << 0,
+	MLX5_PTYS_RATE_DDR	= 1 << 1,
+	MLX5_PTYS_RATE_QDR	= 1 << 2,
+	MLX5_PTYS_RATE_FDR10	= 1 << 3,
+	MLX5_PTYS_RATE_FDR	= 1 << 4,
+	MLX5_PTYS_RATE_EDR	= 1 << 5,
+	MLX5_PTYS_RATE_HDR	= 1 << 6,
+};
+
+static inline int mlx5_ptys_rate_enum_to_int(enum mlx5_ptys_rate rate)
+{
+	switch (rate) {
+	case MLX5_PTYS_RATE_SDR:   return 2500;
+	case MLX5_PTYS_RATE_DDR:   return 5000;
+	case MLX5_PTYS_RATE_QDR:
+	case MLX5_PTYS_RATE_FDR10: return 10000;
+	case MLX5_PTYS_RATE_FDR:   return 14000;
+	case MLX5_PTYS_RATE_EDR:   return 25000;
+	case MLX5_PTYS_RATE_HDR:   return 50000;
+	default:		   return -1;
+	}
+}
+
+static int mlx5i_get_port_settings(struct net_device *netdev,
+				   u16 *ib_link_width_oper, u16 *ib_proto_oper)
+{
+	struct mlx5e_priv *priv    = mlx5i_epriv(netdev);
+	struct mlx5_core_dev *mdev = priv->mdev;
+	u32 out[MLX5_ST_SZ_DW(ptys_reg)] = {0};
+	int ret;
+
+	ret = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_IB, 1);
+	if (ret)
+		return ret;
+
+	*ib_link_width_oper = MLX5_GET(ptys_reg, out, ib_link_width_oper);
+	*ib_proto_oper      = MLX5_GET(ptys_reg, out, ib_proto_oper);
+
+	return 0;
+}
+
+static int mlx5i_get_speed_settings(u16 ib_link_width_oper, u16 ib_proto_oper)
+{
+	int rate, width;
+
+	rate = mlx5_ptys_rate_enum_to_int(ib_proto_oper);
+	if (rate < 0)
+		return -EINVAL;
+	width = mlx5_ptys_width_enum_to_int(ib_link_width_oper);
+	if (width < 0)
+		return -EINVAL;
+
+	return rate * width;
+}
+
+static int mlx5i_get_link_ksettings(struct net_device *netdev,
+				    struct ethtool_link_ksettings *link_ksettings)
+{
+	u16 ib_link_width_oper;
+	u16 ib_proto_oper;
+	int speed, ret;
+
+	ret = mlx5i_get_port_settings(netdev, &ib_link_width_oper, &ib_proto_oper);
+	if (ret)
+		return ret;
+
+	ethtool_link_ksettings_zero_link_mode(link_ksettings, supported);
+	ethtool_link_ksettings_zero_link_mode(link_ksettings, advertising);
+
+	speed = mlx5i_get_speed_settings(ib_link_width_oper, ib_proto_oper);
+	if (speed < 0)
+		return -EINVAL;
+
+	link_ksettings->base.duplex = DUPLEX_FULL;
+	link_ksettings->base.port = PORT_OTHER;
+
+	link_ksettings->base.autoneg = AUTONEG_DISABLE;
+
+	link_ksettings->base.speed = speed;
+
+	return 0;
+}
+
 const struct ethtool_ops mlx5i_ethtool_ops = {
-	.get_drvinfo       = mlx5i_get_drvinfo,
-	.get_strings       = mlx5i_get_strings,
-	.get_sset_count    = mlx5i_get_sset_count,
-	.get_ethtool_stats = mlx5i_get_ethtool_stats,
-	.get_ringparam     = mlx5i_get_ringparam,
-	.set_ringparam     = mlx5i_set_ringparam,
-	.flash_device      = mlx5i_flash_device,
-	.get_channels      = mlx5i_get_channels,
-	.set_channels      = mlx5i_set_channels,
-	.get_coalesce      = mlx5i_get_coalesce,
-	.set_coalesce      = mlx5i_set_coalesce,
-	.get_ts_info       = mlx5i_get_ts_info,
+	.get_drvinfo        = mlx5i_get_drvinfo,
+	.get_strings        = mlx5i_get_strings,
+	.get_sset_count     = mlx5i_get_sset_count,
+	.get_ethtool_stats  = mlx5i_get_ethtool_stats,
+	.get_ringparam      = mlx5i_get_ringparam,
+	.set_ringparam      = mlx5i_set_ringparam,
+	.flash_device       = mlx5i_flash_device,
+	.get_channels       = mlx5i_get_channels,
+	.set_channels       = mlx5i_set_channels,
+	.get_coalesce       = mlx5i_get_coalesce,
+	.set_coalesce       = mlx5i_set_coalesce,
+	.get_ts_info        = mlx5i_get_ts_info,
+	.get_link_ksettings = mlx5i_get_link_ksettings,
+	.get_link           = ethtool_op_get_link,
 };
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.c
new file mode 100644
index 000000000000..7cb67122e8b5
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.c
@@ -0,0 +1,201 @@
+/*
+ * Copyright (c) 2017, Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/etherdevice.h>
+#include <linux/mlx5/driver.h>
+#include <linux/mlx5/mlx5_ifc.h>
+#include "mlx5_core.h"
+#include "lib/mpfs.h"
+
+/* HW L2 Table (MPFS) management */
+static int set_l2table_entry_cmd(struct mlx5_core_dev *dev, u32 index, u8 *mac)
+{
+	u32 in[MLX5_ST_SZ_DW(set_l2_table_entry_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(set_l2_table_entry_out)] = {0};
+	u8 *in_mac_addr;
+
+	MLX5_SET(set_l2_table_entry_in, in, opcode, MLX5_CMD_OP_SET_L2_TABLE_ENTRY);
+	MLX5_SET(set_l2_table_entry_in, in, table_index, index);
+
+	in_mac_addr = MLX5_ADDR_OF(set_l2_table_entry_in, in, mac_address);
+	ether_addr_copy(&in_mac_addr[2], mac);
+
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+}
+
+static int del_l2table_entry_cmd(struct mlx5_core_dev *dev, u32 index)
+{
+	u32 in[MLX5_ST_SZ_DW(delete_l2_table_entry_in)]   = {0};
+	u32 out[MLX5_ST_SZ_DW(delete_l2_table_entry_out)] = {0};
+
+	MLX5_SET(delete_l2_table_entry_in, in, opcode, MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY);
+	MLX5_SET(delete_l2_table_entry_in, in, table_index, index);
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+}
+
+/* UC L2 table hash node */
+struct l2table_node {
+	struct l2addr_node node;
+	u32                index; /* index in HW l2 table */
+};
+
+struct mlx5_mpfs {
+	struct hlist_head    hash[MLX5_L2_ADDR_HASH_SIZE];
+	struct mutex         lock; /* Synchronize l2 table access */
+	u32                  size;
+	unsigned long        *bitmap;
+};
+
+static int alloc_l2table_index(struct mlx5_mpfs *l2table, u32 *ix)
+{
+	int err = 0;
+
+	*ix = find_first_zero_bit(l2table->bitmap, l2table->size);
+	if (*ix >= l2table->size)
+		err = -ENOSPC;
+	else
+		__set_bit(*ix, l2table->bitmap);
+
+	return err;
+}
+
+static void free_l2table_index(struct mlx5_mpfs *l2table, u32 ix)
+{
+	__clear_bit(ix, l2table->bitmap);
+}
+
+int mlx5_mpfs_init(struct mlx5_core_dev *dev)
+{
+	int l2table_size = 1 << MLX5_CAP_GEN(dev, log_max_l2_table);
+	struct mlx5_mpfs *mpfs;
+
+	if (!MLX5_VPORT_MANAGER(dev))
+		return 0;
+
+	mpfs = kzalloc(sizeof(*mpfs), GFP_KERNEL);
+	if (!mpfs)
+		return -ENOMEM;
+
+	mutex_init(&mpfs->lock);
+	mpfs->size   = l2table_size;
+	mpfs->bitmap = kcalloc(BITS_TO_LONGS(l2table_size),
+			       sizeof(uintptr_t), GFP_KERNEL);
+	if (!mpfs->bitmap) {
+		kfree(mpfs);
+		return -ENOMEM;
+	}
+
+	dev->priv.mpfs = mpfs;
+	return 0;
+}
+
+void mlx5_mpfs_cleanup(struct mlx5_core_dev *dev)
+{
+	struct mlx5_mpfs *mpfs = dev->priv.mpfs;
+
+	if (!MLX5_VPORT_MANAGER(dev))
+		return;
+
+	WARN_ON(!hlist_empty(mpfs->hash));
+	kfree(mpfs->bitmap);
+	kfree(mpfs);
+}
+
+int mlx5_mpfs_add_mac(struct mlx5_core_dev *dev, u8 *mac)
+{
+	struct mlx5_mpfs *mpfs = dev->priv.mpfs;
+	struct l2table_node *l2addr;
+	u32 index;
+	int err;
+
+	if (!MLX5_VPORT_MANAGER(dev))
+		return 0;
+
+	mutex_lock(&mpfs->lock);
+
+	l2addr = l2addr_hash_find(mpfs->hash, mac, struct l2table_node);
+	if (l2addr) {
+		err = -EEXIST;
+		goto abort;
+	}
+
+	err = alloc_l2table_index(mpfs, &index);
+	if (err)
+		goto abort;
+
+	l2addr = l2addr_hash_add(mpfs->hash, mac, struct l2table_node, GFP_KERNEL);
+	if (!l2addr) {
+		free_l2table_index(mpfs, index);
+		err = -ENOMEM;
+		goto abort;
+	}
+
+	l2addr->index = index;
+	err = set_l2table_entry_cmd(dev, index, mac);
+	if (err) {
+		l2addr_hash_del(l2addr);
+		free_l2table_index(mpfs, index);
+	}
+
+	mlx5_core_dbg(dev, "MPFS mac added %pM, index (%d)\n", mac, index);
+abort:
+	mutex_unlock(&mpfs->lock);
+	return err;
+}
+
+int mlx5_mpfs_del_mac(struct mlx5_core_dev *dev, u8 *mac)
+{
+	struct mlx5_mpfs *mpfs = dev->priv.mpfs;
+	struct l2table_node *l2addr;
+	int err = 0;
+	u32 index;
+
+	if (!MLX5_VPORT_MANAGER(dev))
+		return 0;
+
+	mutex_lock(&mpfs->lock);
+
+	l2addr = l2addr_hash_find(mpfs->hash, mac, struct l2table_node);
+	if (!l2addr) {
+		err = -ENOENT;
+		goto unlock;
+	}
+
+	index = l2addr->index;
+	del_l2table_entry_cmd(dev, index);
+	l2addr_hash_del(l2addr);
+	free_l2table_index(mpfs, index);
+	mlx5_core_dbg(dev, "MPFS mac deleted %pM, index (%d)\n", mac, index);
+unlock:
+	mutex_unlock(&mpfs->lock);
+	return err;
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.h
new file mode 100644
index 000000000000..4a7b2c3203a7
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.h
@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2017, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef __MLX5_MPFS_H__
+#define __MLX5_MPFS_H__
+
+#include <linux/if_ether.h>
+#include <linux/mlx5/device.h>
+
+/* L2 -mac address based- hash helpers */
+#define MLX5_L2_ADDR_HASH_SIZE (BIT(BITS_PER_BYTE))
+#define MLX5_L2_ADDR_HASH(addr) (addr[5])
+
+struct l2addr_node {
+	struct hlist_node hlist;
+	u8                addr[ETH_ALEN];
+};
+
+#define for_each_l2hash_node(hn, tmp, hash, i) \
+	for (i = 0; i < MLX5_L2_ADDR_HASH_SIZE; i++) \
+		hlist_for_each_entry_safe(hn, tmp, &(hash)[i], hlist)
+
+#define l2addr_hash_find(hash, mac, type) ({                \
+	int ix = MLX5_L2_ADDR_HASH(mac);                    \
+	bool found = false;                                 \
+	type *ptr = NULL;                                   \
+							    \
+	hlist_for_each_entry(ptr, &(hash)[ix], node.hlist)  \
+		if (ether_addr_equal(ptr->node.addr, mac)) {\
+			found = true;                       \
+			break;                              \
+		}                                           \
+	if (!found)                                         \
+		ptr = NULL;                                 \
+	ptr;                                                \
+})
+
+#define l2addr_hash_add(hash, mac, type, gfp) ({            \
+	int ix = MLX5_L2_ADDR_HASH(mac);                    \
+	type *ptr = NULL;                                   \
+							    \
+	ptr = kzalloc(sizeof(type), gfp);                   \
+	if (ptr) {                                          \
+		ether_addr_copy(ptr->node.addr, mac);       \
+		hlist_add_head(&ptr->node.hlist, &(hash)[ix]);\
+	}                                                   \
+	ptr;                                                \
+})
+
+#define l2addr_hash_del(ptr) ({                             \
+	hlist_del(&(ptr)->node.hlist);                      \
+	kfree(ptr);                                         \
+})
+
+#ifdef CONFIG_MLX5_MPFS
+int  mlx5_mpfs_init(struct mlx5_core_dev *dev);
+void mlx5_mpfs_cleanup(struct mlx5_core_dev *dev);
+int  mlx5_mpfs_add_mac(struct mlx5_core_dev *dev, u8 *mac);
+int  mlx5_mpfs_del_mac(struct mlx5_core_dev *dev, u8 *mac);
+#else /* #ifndef CONFIG_MLX5_MPFS */
+static inline int  mlx5_mpfs_init(struct mlx5_core_dev *dev) { return 0; }
+static inline void mlx5_mpfs_cleanup(struct mlx5_core_dev *dev) {}
+static inline int  mlx5_mpfs_add_mac(struct mlx5_core_dev *dev, u8 *mac) { return 0; }
+static inline int  mlx5_mpfs_del_mac(struct mlx5_core_dev *dev, u8 *mac) { return 0; }
+#endif
+#endif
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index c065132b956d..0d2c8dcd6eae 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -47,15 +47,15 @@
 #include <linux/debugfs.h>
 #include <linux/kmod.h>
 #include <linux/mlx5/mlx5_ifc.h>
+#include <linux/mlx5/vport.h>
 #ifdef CONFIG_RFS_ACCEL
 #include <linux/cpu_rmap.h>
 #endif
 #include <net/devlink.h>
 #include "mlx5_core.h"
 #include "fs_core.h"
-#ifdef CONFIG_MLX5_CORE_EN
+#include "lib/mpfs.h"
 #include "eswitch.h"
-#endif
 #include "lib/mlx5.h"
 #include "fpga/core.h"
 #include "accel/ipsec.h"
@@ -312,13 +312,15 @@ static void release_bar(struct pci_dev *pdev)
 	pci_release_regions(pdev);
 }
 
-static int mlx5_enable_msix(struct mlx5_core_dev *dev)
+static int mlx5_alloc_irq_vectors(struct mlx5_core_dev *dev)
 {
 	struct mlx5_priv *priv = &dev->priv;
 	struct mlx5_eq_table *table = &priv->eq_table;
+	struct irq_affinity irqdesc = {
+		.pre_vectors = MLX5_EQ_VEC_COMP_BASE,
+	};
 	int num_eqs = 1 << MLX5_CAP_GEN(dev, log_max_eq);
 	int nvec;
-	int i;
 
 	nvec = MLX5_CAP_GEN(dev, num_ports) * num_online_cpus() +
 	       MLX5_EQ_VEC_COMP_BASE;
@@ -326,17 +328,14 @@ static int mlx5_enable_msix(struct mlx5_core_dev *dev)
 	if (nvec <= MLX5_EQ_VEC_COMP_BASE)
 		return -ENOMEM;
 
-	priv->msix_arr = kcalloc(nvec, sizeof(*priv->msix_arr), GFP_KERNEL);
-
 	priv->irq_info = kcalloc(nvec, sizeof(*priv->irq_info), GFP_KERNEL);
-	if (!priv->msix_arr || !priv->irq_info)
+	if (!priv->irq_info)
 		goto err_free_msix;
 
-	for (i = 0; i < nvec; i++)
-		priv->msix_arr[i].entry = i;
-
-	nvec = pci_enable_msix_range(dev->pdev, priv->msix_arr,
-				     MLX5_EQ_VEC_COMP_BASE + 1, nvec);
+	nvec = pci_alloc_irq_vectors_affinity(dev->pdev,
+			MLX5_EQ_VEC_COMP_BASE + 1, nvec,
+			PCI_IRQ_MSIX | PCI_IRQ_AFFINITY,
+			&irqdesc);
 	if (nvec < 0)
 		return nvec;
 
@@ -346,17 +345,15 @@ static int mlx5_enable_msix(struct mlx5_core_dev *dev)
 
 err_free_msix:
 	kfree(priv->irq_info);
-	kfree(priv->msix_arr);
 	return -ENOMEM;
 }
 
-static void mlx5_disable_msix(struct mlx5_core_dev *dev)
+static void mlx5_free_irq_vectors(struct mlx5_core_dev *dev)
 {
 	struct mlx5_priv *priv = &dev->priv;
 
-	pci_disable_msix(dev->pdev);
+	pci_free_irq_vectors(dev->pdev);
 	kfree(priv->irq_info);
-	kfree(priv->msix_arr);
 }
 
 struct mlx5_reg_host_endianness {
@@ -579,6 +576,18 @@ static int set_hca_ctrl(struct mlx5_core_dev *dev)
 	return err;
 }
 
+static int mlx5_core_set_hca_defaults(struct mlx5_core_dev *dev)
+{
+	int ret = 0;
+
+	/* Disable local_lb by default */
+	if ((MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_ETH) &&
+	    MLX5_CAP_GEN(dev, disable_local_lb))
+		ret = mlx5_nic_vport_update_local_lb(dev, false);
+
+	return ret;
+}
+
 int mlx5_core_enable_hca(struct mlx5_core_dev *dev, u16 func_id)
 {
 	u32 out[MLX5_ST_SZ_DW(enable_hca_out)] = {0};
@@ -612,65 +621,6 @@ u64 mlx5_read_internal_timer(struct mlx5_core_dev *dev)
 	return (u64)timer_l | (u64)timer_h1 << 32;
 }
 
-static int mlx5_irq_set_affinity_hint(struct mlx5_core_dev *mdev, int i)
-{
-	struct mlx5_priv *priv  = &mdev->priv;
-	struct msix_entry *msix = priv->msix_arr;
-	int irq                 = msix[i + MLX5_EQ_VEC_COMP_BASE].vector;
-
-	if (!zalloc_cpumask_var(&priv->irq_info[i].mask, GFP_KERNEL)) {
-		mlx5_core_warn(mdev, "zalloc_cpumask_var failed");
-		return -ENOMEM;
-	}
-
-	cpumask_set_cpu(cpumask_local_spread(i, priv->numa_node),
-			priv->irq_info[i].mask);
-
-	if (IS_ENABLED(CONFIG_SMP) &&
-	    irq_set_affinity_hint(irq, priv->irq_info[i].mask))
-		mlx5_core_warn(mdev, "irq_set_affinity_hint failed, irq 0x%.4x", irq);
-
-	return 0;
-}
-
-static void mlx5_irq_clear_affinity_hint(struct mlx5_core_dev *mdev, int i)
-{
-	struct mlx5_priv *priv  = &mdev->priv;
-	struct msix_entry *msix = priv->msix_arr;
-	int irq                 = msix[i + MLX5_EQ_VEC_COMP_BASE].vector;
-
-	irq_set_affinity_hint(irq, NULL);
-	free_cpumask_var(priv->irq_info[i].mask);
-}
-
-static int mlx5_irq_set_affinity_hints(struct mlx5_core_dev *mdev)
-{
-	int err;
-	int i;
-
-	for (i = 0; i < mdev->priv.eq_table.num_comp_vectors; i++) {
-		err = mlx5_irq_set_affinity_hint(mdev, i);
-		if (err)
-			goto err_out;
-	}
-
-	return 0;
-
-err_out:
-	for (i--; i >= 0; i--)
-		mlx5_irq_clear_affinity_hint(mdev, i);
-
-	return err;
-}
-
-static void mlx5_irq_clear_affinity_hints(struct mlx5_core_dev *mdev)
-{
-	int i;
-
-	for (i = 0; i < mdev->priv.eq_table.num_comp_vectors; i++)
-		mlx5_irq_clear_affinity_hint(mdev, i);
-}
-
 int mlx5_vector2eqn(struct mlx5_core_dev *dev, int vector, int *eqn,
 		    unsigned int *irqn)
 {
@@ -760,8 +710,8 @@ static int alloc_comp_eqs(struct mlx5_core_dev *dev)
 		}
 
 #ifdef CONFIG_RFS_ACCEL
-		irq_cpu_rmap_add(dev->rmap,
-				 dev->priv.msix_arr[i + MLX5_EQ_VEC_COMP_BASE].vector);
+		irq_cpu_rmap_add(dev->rmap, pci_irq_vector(dev->pdev,
+				 MLX5_EQ_VEC_COMP_BASE + i));
 #endif
 		snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_comp%d", i);
 		err = mlx5_create_map_eq(dev, eq,
@@ -837,7 +787,6 @@ static int mlx5_core_set_issi(struct mlx5_core_dev *dev)
 	return -EOPNOTSUPP;
 }
 
-
 static int mlx5_pci_init(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
 {
 	struct pci_dev *pdev = dev->pdev;
@@ -946,13 +895,17 @@ static int mlx5_init_once(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
 		goto err_tables_cleanup;
 	}
 
-#ifdef CONFIG_MLX5_CORE_EN
+	err = mlx5_mpfs_init(dev);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to init l2 table %d\n", err);
+		goto err_rl_cleanup;
+	}
+
 	err = mlx5_eswitch_init(dev);
 	if (err) {
 		dev_err(&pdev->dev, "Failed to init eswitch %d\n", err);
-		goto err_rl_cleanup;
+		goto err_mpfs_cleanup;
 	}
-#endif
 
 	err = mlx5_sriov_init(dev);
 	if (err) {
@@ -971,13 +924,11 @@ static int mlx5_init_once(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
 err_sriov_cleanup:
 	mlx5_sriov_cleanup(dev);
 err_eswitch_cleanup:
-#ifdef CONFIG_MLX5_CORE_EN
 	mlx5_eswitch_cleanup(dev->priv.eswitch);
-
+err_mpfs_cleanup:
+	mlx5_mpfs_cleanup(dev);
 err_rl_cleanup:
-#endif
 	mlx5_cleanup_rl_table(dev);
-
 err_tables_cleanup:
 	mlx5_cleanup_mkey_table(dev);
 	mlx5_cleanup_srq_table(dev);
@@ -995,9 +946,8 @@ static void mlx5_cleanup_once(struct mlx5_core_dev *dev)
 {
 	mlx5_fpga_cleanup(dev);
 	mlx5_sriov_cleanup(dev);
-#ifdef CONFIG_MLX5_CORE_EN
 	mlx5_eswitch_cleanup(dev->priv.eswitch);
-#endif
+	mlx5_mpfs_cleanup(dev);
 	mlx5_cleanup_rl_table(dev);
 	mlx5_cleanup_reserved_gids(dev);
 	mlx5_cleanup_mkey_table(dev);
@@ -1119,9 +1069,9 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
 		goto err_stop_poll;
 	}
 
-	err = mlx5_enable_msix(dev);
+	err = mlx5_alloc_irq_vectors(dev);
 	if (err) {
-		dev_err(&pdev->dev, "enable msix failed\n");
+		dev_err(&pdev->dev, "alloc irq vectors failed\n");
 		goto err_cleanup_once;
 	}
 
@@ -1143,21 +1093,17 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
 		goto err_stop_eqs;
 	}
 
-	err = mlx5_irq_set_affinity_hints(dev);
-	if (err) {
-		dev_err(&pdev->dev, "Failed to alloc affinity hint cpumask\n");
-		goto err_affinity_hints;
-	}
-
 	err = mlx5_init_fs(dev);
 	if (err) {
 		dev_err(&pdev->dev, "Failed to init flow steering\n");
 		goto err_fs;
 	}
 
-#ifdef CONFIG_MLX5_CORE_EN
-	mlx5_eswitch_attach(dev->priv.eswitch);
-#endif
+	err = mlx5_core_set_hca_defaults(dev);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to set hca defaults\n");
+		goto err_fs;
+	}
 
 	err = mlx5_sriov_attach(dev);
 	if (err) {
@@ -1186,7 +1132,6 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
 		}
 	}
 
-	clear_bit(MLX5_INTERFACE_STATE_DOWN, &dev->intf_state);
 	set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
 out:
 	mutex_unlock(&dev->intf_state_mutex);
@@ -1202,15 +1147,9 @@ err_fpga_start:
 	mlx5_sriov_detach(dev);
 
 err_sriov:
-#ifdef CONFIG_MLX5_CORE_EN
-	mlx5_eswitch_detach(dev->priv.eswitch);
-#endif
 	mlx5_cleanup_fs(dev);
 
 err_fs:
-	mlx5_irq_clear_affinity_hints(dev);
-
-err_affinity_hints:
 	free_comp_eqs(dev);
 
 err_stop_eqs:
@@ -1220,7 +1159,7 @@ err_put_uars:
 	mlx5_put_uars_page(dev, priv->uar);
 
 err_disable_msix:
-	mlx5_disable_msix(dev);
+	mlx5_free_irq_vectors(dev);
 
 err_cleanup_once:
 	if (boot)
@@ -1261,7 +1200,7 @@ static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
 		mlx5_drain_health_recovery(dev);
 
 	mutex_lock(&dev->intf_state_mutex);
-	if (test_bit(MLX5_INTERFACE_STATE_DOWN, &dev->intf_state)) {
+	if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) {
 		dev_warn(&dev->pdev->dev, "%s: interface is down, NOP\n",
 			 __func__);
 		if (cleanup)
@@ -1270,7 +1209,6 @@ static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
 	}
 
 	clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
-	set_bit(MLX5_INTERFACE_STATE_DOWN, &dev->intf_state);
 
 	if (mlx5_device_registered(dev))
 		mlx5_detach_device(dev);
@@ -1279,15 +1217,11 @@ static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
 	mlx5_fpga_device_stop(dev);
 
 	mlx5_sriov_detach(dev);
-#ifdef CONFIG_MLX5_CORE_EN
-	mlx5_eswitch_detach(dev->priv.eswitch);
-#endif
 	mlx5_cleanup_fs(dev);
-	mlx5_irq_clear_affinity_hints(dev);
 	free_comp_eqs(dev);
 	mlx5_stop_eqs(dev);
 	mlx5_put_uars_page(dev, priv->uar);
-	mlx5_disable_msix(dev);
+	mlx5_free_irq_vectors(dev);
 	if (cleanup)
 		mlx5_cleanup_once(dev);
 	mlx5_stop_health_poll(dev);
@@ -1313,7 +1247,7 @@ struct mlx5_core_event_handler {
 };
 
 static const struct devlink_ops mlx5_devlink_ops = {
-#ifdef CONFIG_MLX5_CORE_EN
+#ifdef CONFIG_MLX5_ESWITCH
 	.eswitch_mode_set = mlx5_devlink_eswitch_mode_set,
 	.eswitch_mode_get = mlx5_devlink_eswitch_mode_get,
 	.eswitch_inline_mode_set = mlx5_devlink_eswitch_inline_mode_set,
@@ -1353,6 +1287,9 @@ static int init_one(struct pci_dev *pdev,
 	mutex_init(&dev->pci_status_mutex);
 	mutex_init(&dev->intf_state_mutex);
 
+	INIT_LIST_HEAD(&priv->waiting_events_list);
+	priv->is_accum_events = false;
+
 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
 	err = init_srcu_struct(&priv->pfault_srcu);
 	if (err) {
@@ -1407,7 +1344,6 @@ clean_srcu:
 	cleanup_srcu_struct(&priv->pfault_srcu);
 clean_dev:
 #endif
-	pci_set_drvdata(pdev, NULL);
 	devlink_free(devlink);
 
 	return err;
@@ -1434,7 +1370,6 @@ static void remove_one(struct pci_dev *pdev)
 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
 	cleanup_srcu_struct(&priv->pfault_srcu);
 #endif
-	pci_set_drvdata(pdev, NULL);
 	devlink_free(devlink);
 }
 
@@ -1565,8 +1500,6 @@ static void shutdown(struct pci_dev *pdev)
 	int err;
 
 	dev_info(&pdev->dev, "Shutdown was called\n");
-	/* Notify mlx5 clients that the kernel is being shut down */
-	set_bit(MLX5_INTERFACE_STATE_SHUTDOWN, &dev->intf_state);
 	err = mlx5_try_fast_unload(dev);
 	if (err)
 		mlx5_unload_one(dev, priv, false);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
index 6a263e8d883a..b7c2900b75f9 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
@@ -43,6 +43,10 @@
 #define DRIVER_VERSION "5.0-0"
 
 #define MLX5_TOTAL_VPORTS(mdev) (1 + pci_sriov_get_totalvfs(mdev->pdev))
+#define MLX5_VPORT_MANAGER(mdev) \
+	(MLX5_CAP_GEN(mdev, vport_group_manager) && \
+	(MLX5_CAP_GEN(mdev, port_type) == MLX5_CAP_PORT_TYPE_ETH) && \
+	 mlx5_core_is_pf(mdev))
 
 extern uint mlx5_core_debug_mask;
 
@@ -110,7 +114,6 @@ int mlx5_destroy_scheduling_element_cmd(struct mlx5_core_dev *dev, u8 hierarchy,
 					u32 element_id);
 int mlx5_wait_for_vf_pages(struct mlx5_core_dev *dev);
 u64 mlx5_read_internal_timer(struct mlx5_core_dev *dev);
-u32 mlx5_get_msix_vec(struct mlx5_core_dev *dev, int vecidx);
 struct mlx5_eq *mlx5_eqn2eq(struct mlx5_core_dev *dev, int eqn);
 void mlx5_cq_tasklet_cb(unsigned long data);
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/qp.c b/drivers/net/ethernet/mellanox/mlx5/core/qp.c
index 340f281c9801..db9e665ab104 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/qp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/qp.c
@@ -242,6 +242,20 @@ int mlx5_core_destroy_qp(struct mlx5_core_dev *dev,
 }
 EXPORT_SYMBOL_GPL(mlx5_core_destroy_qp);
 
+int mlx5_core_set_delay_drop(struct mlx5_core_dev *dev,
+			     u32 timeout_usec)
+{
+	u32 out[MLX5_ST_SZ_DW(set_delay_drop_params_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(set_delay_drop_params_in)]   = {0};
+
+	MLX5_SET(set_delay_drop_params_in, in, opcode,
+		 MLX5_CMD_OP_SET_DELAY_DROP_PARAMS);
+	MLX5_SET(set_delay_drop_params_in, in, delay_drop_timeout,
+		 timeout_usec / 100);
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+}
+EXPORT_SYMBOL_GPL(mlx5_core_set_delay_drop);
+
 struct mbox_info {
 	u32 *in;
 	u32 *out;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c
index bf99d40e30b4..6c48e9959b65 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c
@@ -32,10 +32,9 @@
 
 #include <linux/pci.h>
 #include <linux/mlx5/driver.h>
+#include <linux/mlx5/vport.h>
 #include "mlx5_core.h"
-#ifdef CONFIG_MLX5_CORE_EN
 #include "eswitch.h"
-#endif
 
 bool mlx5_sriov_is_enabled(struct mlx5_core_dev *dev)
 {
@@ -44,6 +43,38 @@ bool mlx5_sriov_is_enabled(struct mlx5_core_dev *dev)
 	return !!sriov->num_vfs;
 }
 
+static int sriov_restore_guids(struct mlx5_core_dev *dev, int vf)
+{
+	struct mlx5_core_sriov *sriov = &dev->priv.sriov;
+	struct mlx5_hca_vport_context *in;
+	int err = 0;
+
+	/* Restore sriov guid and policy settings */
+	if (sriov->vfs_ctx[vf].node_guid ||
+	    sriov->vfs_ctx[vf].port_guid ||
+	    sriov->vfs_ctx[vf].policy != MLX5_POLICY_INVALID) {
+		in = kzalloc(sizeof(*in), GFP_KERNEL);
+		if (!in)
+			return -ENOMEM;
+
+		in->node_guid = sriov->vfs_ctx[vf].node_guid;
+		in->port_guid = sriov->vfs_ctx[vf].port_guid;
+		in->policy = sriov->vfs_ctx[vf].policy;
+		in->field_select =
+			!!(in->port_guid) * MLX5_HCA_VPORT_SEL_PORT_GUID |
+			!!(in->node_guid) * MLX5_HCA_VPORT_SEL_NODE_GUID |
+			!!(in->policy) * MLX5_HCA_VPORT_SEL_STATE_POLICY;
+
+		err = mlx5_core_modify_hca_vport_context(dev, 1, 1, vf + 1, in);
+		if (err)
+			mlx5_core_warn(dev, "modify vport context failed, unable to restore VF %d settings\n", vf);
+
+		kfree(in);
+	}
+
+	return err;
+}
+
 static int mlx5_device_enable_sriov(struct mlx5_core_dev *dev, int num_vfs)
 {
 	struct mlx5_core_sriov *sriov = &dev->priv.sriov;
@@ -57,14 +88,12 @@ static int mlx5_device_enable_sriov(struct mlx5_core_dev *dev, int num_vfs)
 		return -EBUSY;
 	}
 
-#ifdef CONFIG_MLX5_CORE_EN
 	err = mlx5_eswitch_enable_sriov(dev->priv.eswitch, num_vfs, SRIOV_LEGACY);
 	if (err) {
 		mlx5_core_warn(dev,
 			       "failed to enable eswitch SRIOV (%d)\n", err);
 		return err;
 	}
-#endif
 
 	for (vf = 0; vf < num_vfs; vf++) {
 		err = mlx5_core_enable_hca(dev, vf + 1);
@@ -74,8 +103,16 @@ static int mlx5_device_enable_sriov(struct mlx5_core_dev *dev, int num_vfs)
 		}
 		sriov->vfs_ctx[vf].enabled = 1;
 		sriov->enabled_vfs++;
+		if (MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_IB) {
+			err = sriov_restore_guids(dev, vf);
+			if (err) {
+				mlx5_core_warn(dev,
+					       "failed to restore VF %d settings, err %d\n",
+					       vf, err);
+			continue;
+			}
+		}
 		mlx5_core_dbg(dev, "successfully enabled VF* %d\n", vf);
-
 	}
 
 	return 0;
@@ -88,11 +125,7 @@ static void mlx5_device_disable_sriov(struct mlx5_core_dev *dev)
 	int vf;
 
 	if (!sriov->enabled_vfs)
-#ifdef CONFIG_MLX5_CORE_EN
-		goto disable_sriov_resources;
-#else
-		return;
-#endif
+		goto out;
 
 	for (vf = 0; vf < sriov->num_vfs; vf++) {
 		if (!sriov->vfs_ctx[vf].enabled)
@@ -106,10 +139,8 @@ static void mlx5_device_disable_sriov(struct mlx5_core_dev *dev)
 		sriov->enabled_vfs--;
 	}
 
-#ifdef CONFIG_MLX5_CORE_EN
-disable_sriov_resources:
+out:
 	mlx5_eswitch_disable_sriov(dev->priv.eswitch);
-#endif
 
 	if (mlx5_wait_for_vf_pages(dev))
 		mlx5_core_warn(dev, "timeout reclaiming VFs pages\n");
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/srq.c b/drivers/net/ethernet/mellanox/mlx5/core/srq.c
index f774de6f5fcb..23cc337a96c9 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/srq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/srq.c
@@ -201,13 +201,13 @@ static int destroy_srq_cmd(struct mlx5_core_dev *dev,
 static int arm_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
 		       u16 lwm, int is_srq)
 {
-	/* arm_srq structs missing using identical xrc ones */
-	u32 srq_in[MLX5_ST_SZ_DW(arm_xrc_srq_in)] = {0};
-	u32 srq_out[MLX5_ST_SZ_DW(arm_xrc_srq_out)] = {0};
+	u32 srq_in[MLX5_ST_SZ_DW(arm_rq_in)] = {0};
+	u32 srq_out[MLX5_ST_SZ_DW(arm_rq_out)] = {0};
 
-	MLX5_SET(arm_xrc_srq_in, srq_in, opcode,   MLX5_CMD_OP_ARM_XRC_SRQ);
-	MLX5_SET(arm_xrc_srq_in, srq_in, xrc_srqn, srq->srqn);
-	MLX5_SET(arm_xrc_srq_in, srq_in, lwm,      lwm);
+	MLX5_SET(arm_rq_in, srq_in, opcode, MLX5_CMD_OP_ARM_RQ);
+	MLX5_SET(arm_rq_in, srq_in, op_mod, MLX5_ARM_RQ_IN_OP_MOD_SRQ);
+	MLX5_SET(arm_rq_in, srq_in, srq_number, srq->srqn);
+	MLX5_SET(arm_rq_in, srq_in, lwm,      lwm);
 
 	return  mlx5_cmd_exec(dev, srq_in, sizeof(srq_in),
 			      srq_out, sizeof(srq_out));
@@ -435,16 +435,128 @@ out:
 	return err;
 }
 
+static int create_xrq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
+			  struct mlx5_srq_attr *in)
+{
+	u32 create_out[MLX5_ST_SZ_DW(create_xrq_out)] = {0};
+	void *create_in;
+	void *xrqc;
+	void *wq;
+	int pas_size;
+	int inlen;
+	int err;
+
+	pas_size = get_pas_size(in);
+	inlen = MLX5_ST_SZ_BYTES(create_xrq_in) + pas_size;
+	create_in = kvzalloc(inlen, GFP_KERNEL);
+	if (!create_in)
+		return -ENOMEM;
+
+	xrqc = MLX5_ADDR_OF(create_xrq_in, create_in, xrq_context);
+	wq = MLX5_ADDR_OF(xrqc, xrqc, wq);
+
+	set_wq(wq, in);
+	memcpy(MLX5_ADDR_OF(xrqc, xrqc, wq.pas), in->pas, pas_size);
+
+	if (in->type == IB_SRQT_TM) {
+		MLX5_SET(xrqc, xrqc, topology, MLX5_XRQC_TOPOLOGY_TAG_MATCHING);
+		if (in->flags & MLX5_SRQ_FLAG_RNDV)
+			MLX5_SET(xrqc, xrqc, offload, MLX5_XRQC_OFFLOAD_RNDV);
+		MLX5_SET(xrqc, xrqc,
+			 tag_matching_topology_context.log_matching_list_sz,
+			 in->tm_log_list_size);
+	}
+	MLX5_SET(xrqc, xrqc, user_index, in->user_index);
+	MLX5_SET(xrqc, xrqc, cqn, in->cqn);
+	MLX5_SET(create_xrq_in, create_in, opcode, MLX5_CMD_OP_CREATE_XRQ);
+	err = mlx5_cmd_exec(dev, create_in, inlen, create_out,
+			    sizeof(create_out));
+	kvfree(create_in);
+	if (!err)
+		srq->srqn = MLX5_GET(create_xrq_out, create_out, xrqn);
+
+	return err;
+}
+
+static int destroy_xrq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq)
+{
+	u32 in[MLX5_ST_SZ_DW(destroy_xrq_in)] = {0};
+	u32 out[MLX5_ST_SZ_DW(destroy_xrq_out)] = {0};
+
+	MLX5_SET(destroy_xrq_in, in, opcode, MLX5_CMD_OP_DESTROY_XRQ);
+	MLX5_SET(destroy_xrq_in, in, xrqn,   srq->srqn);
+
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+}
+
+static int arm_xrq_cmd(struct mlx5_core_dev *dev,
+		       struct mlx5_core_srq *srq,
+		       u16 lwm)
+{
+	u32 out[MLX5_ST_SZ_DW(arm_rq_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(arm_rq_in)] = {0};
+
+	MLX5_SET(arm_rq_in, in, opcode,     MLX5_CMD_OP_ARM_RQ);
+	MLX5_SET(arm_rq_in, in, op_mod,     MLX5_ARM_RQ_IN_OP_MOD_XRQ);
+	MLX5_SET(arm_rq_in, in, srq_number, srq->srqn);
+	MLX5_SET(arm_rq_in, in, lwm,	    lwm);
+
+	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+}
+
+static int query_xrq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
+			 struct mlx5_srq_attr *out)
+{
+	u32 in[MLX5_ST_SZ_DW(query_xrq_in)] = {0};
+	u32 *xrq_out;
+	int outlen = MLX5_ST_SZ_BYTES(query_xrq_out);
+	void *xrqc;
+	int err;
+
+	xrq_out = kvzalloc(outlen, GFP_KERNEL);
+	if (!xrq_out)
+		return -ENOMEM;
+
+	MLX5_SET(query_xrq_in, in, opcode, MLX5_CMD_OP_QUERY_XRQ);
+	MLX5_SET(query_xrq_in, in, xrqn, srq->srqn);
+
+	err = mlx5_cmd_exec(dev, in, sizeof(in), xrq_out, outlen);
+	if (err)
+		goto out;
+
+	xrqc = MLX5_ADDR_OF(query_xrq_out, xrq_out, xrq_context);
+	get_wq(MLX5_ADDR_OF(xrqc, xrqc, wq), out);
+	if (MLX5_GET(xrqc, xrqc, state) != MLX5_XRQC_STATE_GOOD)
+		out->flags |= MLX5_SRQ_FLAG_ERR;
+	out->tm_next_tag =
+		MLX5_GET(xrqc, xrqc,
+			 tag_matching_topology_context.append_next_index);
+	out->tm_hw_phase_cnt =
+		MLX5_GET(xrqc, xrqc,
+			 tag_matching_topology_context.hw_phase_cnt);
+	out->tm_sw_phase_cnt =
+		MLX5_GET(xrqc, xrqc,
+			 tag_matching_topology_context.sw_phase_cnt);
+
+out:
+	kvfree(xrq_out);
+	return err;
+}
+
 static int create_srq_split(struct mlx5_core_dev *dev,
 			    struct mlx5_core_srq *srq,
 			    struct mlx5_srq_attr *in)
 {
 	if (!dev->issi)
 		return create_srq_cmd(dev, srq, in);
-	else if (srq->common.res == MLX5_RES_XSRQ)
+	switch (srq->common.res) {
+	case MLX5_RES_XSRQ:
 		return create_xrc_srq_cmd(dev, srq, in);
-	else
+	case MLX5_RES_XRQ:
+		return create_xrq_cmd(dev, srq, in);
+	default:
 		return create_rmp_cmd(dev, srq, in);
+	}
 }
 
 static int destroy_srq_split(struct mlx5_core_dev *dev,
@@ -452,10 +564,14 @@ static int destroy_srq_split(struct mlx5_core_dev *dev,
 {
 	if (!dev->issi)
 		return destroy_srq_cmd(dev, srq);
-	else if (srq->common.res == MLX5_RES_XSRQ)
+	switch (srq->common.res) {
+	case MLX5_RES_XSRQ:
 		return destroy_xrc_srq_cmd(dev, srq);
-	else
+	case MLX5_RES_XRQ:
+		return destroy_xrq_cmd(dev, srq);
+	default:
 		return destroy_rmp_cmd(dev, srq);
+	}
 }
 
 int mlx5_core_create_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
@@ -464,10 +580,16 @@ int mlx5_core_create_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
 	int err;
 	struct mlx5_srq_table *table = &dev->priv.srq_table;
 
-	if (in->type == IB_SRQT_XRC)
+	switch (in->type) {
+	case IB_SRQT_XRC:
 		srq->common.res = MLX5_RES_XSRQ;
-	else
+		break;
+	case IB_SRQT_TM:
+		srq->common.res = MLX5_RES_XRQ;
+		break;
+	default:
 		srq->common.res = MLX5_RES_SRQ;
+	}
 
 	err = create_srq_split(dev, srq, in);
 	if (err)
@@ -528,10 +650,14 @@ int mlx5_core_query_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
 {
 	if (!dev->issi)
 		return query_srq_cmd(dev, srq, out);
-	else if (srq->common.res == MLX5_RES_XSRQ)
+	switch (srq->common.res) {
+	case MLX5_RES_XSRQ:
 		return query_xrc_srq_cmd(dev, srq, out);
-	else
+	case MLX5_RES_XRQ:
+		return query_xrq_cmd(dev, srq, out);
+	default:
 		return query_rmp_cmd(dev, srq, out);
+	}
 }
 EXPORT_SYMBOL(mlx5_core_query_srq);
 
@@ -540,10 +666,14 @@ int mlx5_core_arm_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
 {
 	if (!dev->issi)
 		return arm_srq_cmd(dev, srq, lwm, is_srq);
-	else if (srq->common.res == MLX5_RES_XSRQ)
+	switch (srq->common.res) {
+	case MLX5_RES_XSRQ:
 		return arm_xrc_srq_cmd(dev, srq, lwm);
-	else
+	case MLX5_RES_XRQ:
+		return arm_xrq_cmd(dev, srq, lwm);
+	default:
 		return arm_rmp_cmd(dev, srq, lwm);
+	}
 }
 EXPORT_SYMBOL(mlx5_core_arm_srq);
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/vport.c b/drivers/net/ethernet/mellanox/mlx5/core/vport.c
index 5abfec1c3399..d653b0025b13 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/vport.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/vport.c
@@ -897,6 +897,68 @@ int mlx5_modify_nic_vport_promisc(struct mlx5_core_dev *mdev,
 }
 EXPORT_SYMBOL_GPL(mlx5_modify_nic_vport_promisc);
 
+enum {
+	UC_LOCAL_LB,
+	MC_LOCAL_LB
+};
+
+int mlx5_nic_vport_update_local_lb(struct mlx5_core_dev *mdev, bool enable)
+{
+	int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in);
+	void *in;
+	int err;
+
+	mlx5_core_dbg(mdev, "%s local_lb\n", enable ? "enable" : "disable");
+	in = kvzalloc(inlen, GFP_KERNEL);
+	if (!in)
+		return -ENOMEM;
+
+	MLX5_SET(modify_nic_vport_context_in, in,
+		 field_select.disable_mc_local_lb, 1);
+	MLX5_SET(modify_nic_vport_context_in, in,
+		 nic_vport_context.disable_mc_local_lb, !enable);
+
+	MLX5_SET(modify_nic_vport_context_in, in,
+		 field_select.disable_uc_local_lb, 1);
+	MLX5_SET(modify_nic_vport_context_in, in,
+		 nic_vport_context.disable_uc_local_lb, !enable);
+
+	err = mlx5_modify_nic_vport_context(mdev, in, inlen);
+
+	kvfree(in);
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_nic_vport_update_local_lb);
+
+int mlx5_nic_vport_query_local_lb(struct mlx5_core_dev *mdev, bool *status)
+{
+	int outlen = MLX5_ST_SZ_BYTES(query_nic_vport_context_out);
+	u32 *out;
+	int value;
+	int err;
+
+	out = kzalloc(outlen, GFP_KERNEL);
+	if (!out)
+		return -ENOMEM;
+
+	err = mlx5_query_nic_vport_context(mdev, 0, out, outlen);
+	if (err)
+		goto out;
+
+	value = MLX5_GET(query_nic_vport_context_out, out,
+			 nic_vport_context.disable_mc_local_lb) << MC_LOCAL_LB;
+
+	value |= MLX5_GET(query_nic_vport_context_out, out,
+			  nic_vport_context.disable_uc_local_lb) << UC_LOCAL_LB;
+
+	*status = !value;
+
+out:
+	kfree(out);
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_nic_vport_query_local_lb);
+
 enum mlx5_vport_roce_state {
 	MLX5_VPORT_ROCE_DISABLED = 0,
 	MLX5_VPORT_ROCE_ENABLED  = 1,
diff --git a/drivers/net/ethernet/mellanox/mlxsw/Kconfig b/drivers/net/ethernet/mellanox/mlxsw/Kconfig
index 695adff89d71..d56eea310509 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/Kconfig
+++ b/drivers/net/ethernet/mellanox/mlxsw/Kconfig
@@ -75,6 +75,7 @@ config MLXSW_SPECTRUM
 	depends on MLXSW_CORE && MLXSW_PCI && NET_SWITCHDEV && VLAN_8021Q
 	depends on PSAMPLE || PSAMPLE=n
 	depends on BRIDGE || BRIDGE=n
+	depends on IPV6 || IPV6=n
 	select PARMAN
 	select MLXFW
 	default m
diff --git a/drivers/net/ethernet/mellanox/mlxsw/Makefile b/drivers/net/ethernet/mellanox/mlxsw/Makefile
index 62fc42f396bb..891ff418bb5e 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/Makefile
+++ b/drivers/net/ethernet/mellanox/mlxsw/Makefile
@@ -16,8 +16,9 @@ mlxsw_spectrum-objs		:= spectrum.o spectrum_buffers.o \
 				   spectrum_switchdev.o spectrum_router.o \
 				   spectrum_kvdl.o spectrum_acl_tcam.o \
 				   spectrum_acl.o spectrum_flower.o \
-				   spectrum_cnt.o spectrum_dpipe.o \
-				   spectrum_fid.o
+				   spectrum_cnt.o spectrum_fid.o \
+				   spectrum_ipip.o
 mlxsw_spectrum-$(CONFIG_MLXSW_SPECTRUM_DCB)	+= spectrum_dcb.o
+mlxsw_spectrum-$(CONFIG_NET_DEVLINK) += spectrum_dpipe.o
 obj-$(CONFIG_MLXSW_MINIMAL)	+= mlxsw_minimal.o
 mlxsw_minimal-objs		:= minimal.o
diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.c b/drivers/net/ethernet/mellanox/mlxsw/core.c
index affe84eb4bff..9d5e7cf288be 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/core.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/core.c
@@ -667,7 +667,7 @@ static int mlxsw_emad_reg_access(struct mlxsw_core *mlxsw_core,
 	int err;
 
 	dev_dbg(mlxsw_core->bus_info->dev, "EMAD reg access (tid=%llx,reg_id=%x(%s),type=%s)\n",
-		trans->tid, reg->id, mlxsw_reg_id_str(reg->id),
+		tid, reg->id, mlxsw_reg_id_str(reg->id),
 		mlxsw_core_reg_access_type_str(type));
 
 	skb = mlxsw_emad_alloc(mlxsw_core, reg->len);
diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.h b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.h
index 9807ef814e42..f6963b0b4a55 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.h
@@ -57,6 +57,9 @@ enum mlxsw_afk_element {
 	MLXSW_AFK_ELEMENT_VID,
 	MLXSW_AFK_ELEMENT_PCP,
 	MLXSW_AFK_ELEMENT_TCP_FLAGS,
+	MLXSW_AFK_ELEMENT_IP_TTL_,
+	MLXSW_AFK_ELEMENT_IP_ECN,
+	MLXSW_AFK_ELEMENT_IP_DSCP,
 	MLXSW_AFK_ELEMENT_MAX,
 };
 
@@ -104,6 +107,9 @@ static const struct mlxsw_afk_element_info mlxsw_afk_element_infos[] = {
 	MLXSW_AFK_ELEMENT_INFO_U32(VID, 0x10, 8, 12),
 	MLXSW_AFK_ELEMENT_INFO_U32(PCP, 0x10, 20, 3),
 	MLXSW_AFK_ELEMENT_INFO_U32(TCP_FLAGS, 0x10, 23, 9),
+	MLXSW_AFK_ELEMENT_INFO_U32(IP_TTL_, 0x14, 0, 8),
+	MLXSW_AFK_ELEMENT_INFO_U32(IP_ECN, 0x14, 9, 2),
+	MLXSW_AFK_ELEMENT_INFO_U32(IP_DSCP, 0x14, 11, 6),
 	MLXSW_AFK_ELEMENT_INFO_U32(SRC_IP4, 0x18, 0, 32),
 	MLXSW_AFK_ELEMENT_INFO_U32(DST_IP4, 0x1C, 0, 32),
 	MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP6_HI, 0x18, 8),
diff --git a/drivers/net/ethernet/mellanox/mlxsw/reg.h b/drivers/net/ethernet/mellanox/mlxsw/reg.h
index 1bd34d9a7b9e..cc27c5de5a1d 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/reg.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/reg.h
@@ -5,6 +5,7 @@
  * Copyright (c) 2015 Elad Raz <eladr@mellanox.com>
  * Copyright (c) 2015-2017 Jiri Pirko <jiri@mellanox.com>
  * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
+ * Copyright (c) 2017 Petr Machata <petrm@mellanox.com>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -3679,15 +3680,17 @@ enum mlxsw_reg_htgt_trap_group {
 	MLXSW_REG_HTGT_TRAP_GROUP_SP_LACP,
 	MLXSW_REG_HTGT_TRAP_GROUP_SP_LLDP,
 	MLXSW_REG_HTGT_TRAP_GROUP_SP_IGMP,
-	MLXSW_REG_HTGT_TRAP_GROUP_SP_BGP_IPV4,
+	MLXSW_REG_HTGT_TRAP_GROUP_SP_BGP,
 	MLXSW_REG_HTGT_TRAP_GROUP_SP_OSPF,
 	MLXSW_REG_HTGT_TRAP_GROUP_SP_ARP,
-	MLXSW_REG_HTGT_TRAP_GROUP_SP_ARP_MISS,
+	MLXSW_REG_HTGT_TRAP_GROUP_SP_HOST_MISS,
 	MLXSW_REG_HTGT_TRAP_GROUP_SP_ROUTER_EXP,
 	MLXSW_REG_HTGT_TRAP_GROUP_SP_REMOTE_ROUTE,
 	MLXSW_REG_HTGT_TRAP_GROUP_SP_IP2ME,
 	MLXSW_REG_HTGT_TRAP_GROUP_SP_DHCP,
 	MLXSW_REG_HTGT_TRAP_GROUP_SP_EVENT,
+	MLXSW_REG_HTGT_TRAP_GROUP_SP_IPV6_MLD,
+	MLXSW_REG_HTGT_TRAP_GROUP_SP_IPV6_ND,
 };
 
 /* reg_htgt_trap_group
@@ -3952,10 +3955,12 @@ MLXSW_ITEM32(reg, rgcr, pcp_rw, 0x18, 16, 2);
  */
 MLXSW_ITEM32(reg, rgcr, activity_dis, 0x20, 0, 8);
 
-static inline void mlxsw_reg_rgcr_pack(char *payload, bool ipv4_en)
+static inline void mlxsw_reg_rgcr_pack(char *payload, bool ipv4_en,
+				       bool ipv6_en)
 {
 	MLXSW_REG_ZERO(rgcr, payload);
 	mlxsw_reg_rgcr_ipv4_en_set(payload, ipv4_en);
+	mlxsw_reg_rgcr_ipv6_en_set(payload, ipv6_en);
 }
 
 /* RITR - Router Interface Table Register
@@ -3988,16 +3993,18 @@ MLXSW_ITEM32(reg, ritr, ipv4, 0x00, 29, 1);
 MLXSW_ITEM32(reg, ritr, ipv6, 0x00, 28, 1);
 
 enum mlxsw_reg_ritr_if_type {
+	/* VLAN interface. */
 	MLXSW_REG_RITR_VLAN_IF,
+	/* FID interface. */
 	MLXSW_REG_RITR_FID_IF,
+	/* Sub-port interface. */
 	MLXSW_REG_RITR_SP_IF,
+	/* Loopback Interface. */
+	MLXSW_REG_RITR_LOOPBACK_IF,
 };
 
 /* reg_ritr_type
- * Router interface type.
- * 0 - VLAN interface.
- * 1 - FID interface.
- * 2 - Sub-port interface.
+ * Router interface type as per enum mlxsw_reg_ritr_if_type.
  * Access: RW
  */
 MLXSW_ITEM32(reg, ritr, type, 0x00, 23, 3);
@@ -4125,6 +4132,67 @@ MLXSW_ITEM32(reg, ritr, sp_if_system_port, 0x08, 0, 16);
  */
 MLXSW_ITEM32(reg, ritr, sp_if_vid, 0x18, 0, 12);
 
+/* Loopback Interface */
+
+enum mlxsw_reg_ritr_loopback_protocol {
+	/* IPinIP IPv4 underlay Unicast */
+	MLXSW_REG_RITR_LOOPBACK_PROTOCOL_IPIP_IPV4,
+	/* IPinIP IPv6 underlay Unicast */
+	MLXSW_REG_RITR_LOOPBACK_PROTOCOL_IPIP_IPV6,
+};
+
+/* reg_ritr_loopback_protocol
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, ritr, loopback_protocol, 0x08, 28, 4);
+
+enum mlxsw_reg_ritr_loopback_ipip_type {
+	/* Tunnel is IPinIP. */
+	MLXSW_REG_RITR_LOOPBACK_IPIP_TYPE_IP_IN_IP,
+	/* Tunnel is GRE, no key. */
+	MLXSW_REG_RITR_LOOPBACK_IPIP_TYPE_IP_IN_GRE_IN_IP,
+	/* Tunnel is GRE, with a key. */
+	MLXSW_REG_RITR_LOOPBACK_IPIP_TYPE_IP_IN_GRE_KEY_IN_IP,
+};
+
+/* reg_ritr_loopback_ipip_type
+ * Encapsulation type.
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, ritr, loopback_ipip_type, 0x10, 24, 4);
+
+enum mlxsw_reg_ritr_loopback_ipip_options {
+	/* The key is defined by gre_key. */
+	MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET,
+};
+
+/* reg_ritr_loopback_ipip_options
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, ritr, loopback_ipip_options, 0x10, 20, 4);
+
+/* reg_ritr_loopback_ipip_uvr
+ * Underlay Virtual Router ID.
+ * Range is 0..cap_max_virtual_routers-1.
+ * Reserved for Spectrum-2.
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, ritr, loopback_ipip_uvr, 0x10, 0, 16);
+
+/* reg_ritr_loopback_ipip_usip*
+ * Encapsulation Underlay source IP.
+ * Access: RW
+ */
+MLXSW_ITEM_BUF(reg, ritr, loopback_ipip_usip6, 0x18, 16);
+MLXSW_ITEM32(reg, ritr, loopback_ipip_usip4, 0x24, 0, 32);
+
+/* reg_ritr_loopback_ipip_gre_key
+ * GRE Key.
+ * Reserved when ipip_type is not IP_IN_GRE_KEY_IN_IP.
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, ritr, loopback_ipip_gre_key, 0x28, 0, 32);
+
 /* Shared between ingress/egress */
 enum mlxsw_reg_ritr_counter_set_type {
 	/* No Count. */
@@ -4195,24 +4263,54 @@ static inline void mlxsw_reg_ritr_sp_if_pack(char *payload, bool lag,
 
 static inline void mlxsw_reg_ritr_pack(char *payload, bool enable,
 				       enum mlxsw_reg_ritr_if_type type,
-				       u16 rif, u16 vr_id, u16 mtu,
-				       const char *mac)
+				       u16 rif, u16 vr_id, u16 mtu)
 {
 	bool op = enable ? MLXSW_REG_RITR_RIF_CREATE : MLXSW_REG_RITR_RIF_DEL;
 
 	MLXSW_REG_ZERO(ritr, payload);
 	mlxsw_reg_ritr_enable_set(payload, enable);
 	mlxsw_reg_ritr_ipv4_set(payload, 1);
+	mlxsw_reg_ritr_ipv6_set(payload, 1);
 	mlxsw_reg_ritr_type_set(payload, type);
 	mlxsw_reg_ritr_op_set(payload, op);
 	mlxsw_reg_ritr_rif_set(payload, rif);
 	mlxsw_reg_ritr_ipv4_fe_set(payload, 1);
+	mlxsw_reg_ritr_ipv6_fe_set(payload, 1);
 	mlxsw_reg_ritr_lb_en_set(payload, 1);
 	mlxsw_reg_ritr_virtual_router_set(payload, vr_id);
 	mlxsw_reg_ritr_mtu_set(payload, mtu);
+}
+
+static inline void mlxsw_reg_ritr_mac_pack(char *payload, const char *mac)
+{
 	mlxsw_reg_ritr_if_mac_memcpy_to(payload, mac);
 }
 
+static inline void
+mlxsw_reg_ritr_loopback_ipip_common_pack(char *payload,
+			    enum mlxsw_reg_ritr_loopback_ipip_type ipip_type,
+			    enum mlxsw_reg_ritr_loopback_ipip_options options,
+			    u16 uvr_id, u32 gre_key)
+{
+	mlxsw_reg_ritr_loopback_ipip_type_set(payload, ipip_type);
+	mlxsw_reg_ritr_loopback_ipip_options_set(payload, options);
+	mlxsw_reg_ritr_loopback_ipip_uvr_set(payload, uvr_id);
+	mlxsw_reg_ritr_loopback_ipip_gre_key_set(payload, gre_key);
+}
+
+static inline void
+mlxsw_reg_ritr_loopback_ipip4_pack(char *payload,
+			    enum mlxsw_reg_ritr_loopback_ipip_type ipip_type,
+			    enum mlxsw_reg_ritr_loopback_ipip_options options,
+			    u16 uvr_id, u32 usip, u32 gre_key)
+{
+	mlxsw_reg_ritr_loopback_protocol_set(payload,
+				    MLXSW_REG_RITR_LOOPBACK_PROTOCOL_IPIP_IPV4);
+	mlxsw_reg_ritr_loopback_ipip_common_pack(payload, ipip_type, options,
+						 uvr_id, gre_key);
+	mlxsw_reg_ritr_loopback_ipip_usip4_set(payload, usip);
+}
+
 /* RATR - Router Adjacency Table Register
  * --------------------------------------
  * The RATR register is used to configure the Router Adjacency (next-hop)
@@ -4268,6 +4366,38 @@ MLXSW_ITEM32(reg, ratr, v, 0x00, 24, 1);
  */
 MLXSW_ITEM32(reg, ratr, a, 0x00, 16, 1);
 
+enum mlxsw_reg_ratr_type {
+	/* Ethernet */
+	MLXSW_REG_RATR_TYPE_ETHERNET,
+	/* IPoIB Unicast without GRH.
+	 * Reserved for Spectrum.
+	 */
+	MLXSW_REG_RATR_TYPE_IPOIB_UC,
+	/* IPoIB Unicast with GRH. Supported only in table 0 (Ethernet unicast
+	 * adjacency).
+	 * Reserved for Spectrum.
+	 */
+	MLXSW_REG_RATR_TYPE_IPOIB_UC_W_GRH,
+	/* IPoIB Multicast.
+	 * Reserved for Spectrum.
+	 */
+	MLXSW_REG_RATR_TYPE_IPOIB_MC,
+	/* MPLS.
+	 * Reserved for SwitchX/-2.
+	 */
+	MLXSW_REG_RATR_TYPE_MPLS,
+	/* IPinIP Encap.
+	 * Reserved for SwitchX/-2.
+	 */
+	MLXSW_REG_RATR_TYPE_IPIP,
+};
+
+/* reg_ratr_type
+ * Adjacency entry type.
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, ratr, type, 0x04, 28, 4);
+
 /* reg_ratr_adjacency_index_low
  * Bits 15:0 of index into the adjacency table.
  * For SwitchX and SwitchX-2, the adjacency table is linear and
@@ -4297,17 +4427,17 @@ enum mlxsw_reg_ratr_trap_action {
  */
 MLXSW_ITEM32(reg, ratr, trap_action, 0x0C, 28, 4);
 
-enum mlxsw_reg_ratr_trap_id {
-	MLXSW_REG_RATR_TRAP_ID_RTR_EGRESS0 = 0,
-	MLXSW_REG_RATR_TRAP_ID_RTR_EGRESS1 = 1,
-};
-
 /* reg_ratr_adjacency_index_high
  * Bits 23:16 of the adjacency_index.
  * Access: Index
  */
 MLXSW_ITEM32(reg, ratr, adjacency_index_high, 0x0C, 16, 8);
 
+enum mlxsw_reg_ratr_trap_id {
+	MLXSW_REG_RATR_TRAP_ID_RTR_EGRESS0,
+	MLXSW_REG_RATR_TRAP_ID_RTR_EGRESS1,
+};
+
 /* reg_ratr_trap_id
  * Trap ID to be reported to CPU.
  * Trap-ID is RTR_EGRESS0 or RTR_EGRESS1.
@@ -4322,14 +4452,44 @@ MLXSW_ITEM32(reg, ratr, trap_id, 0x0C, 0, 8);
  */
 MLXSW_ITEM_BUF(reg, ratr, eth_destination_mac, 0x12, 6);
 
+enum mlxsw_reg_ratr_ipip_type {
+	/* IPv4, address set by mlxsw_reg_ratr_ipip_ipv4_udip. */
+	MLXSW_REG_RATR_IPIP_TYPE_IPV4,
+	/* IPv6, address set by mlxsw_reg_ratr_ipip_ipv6_ptr. */
+	MLXSW_REG_RATR_IPIP_TYPE_IPV6,
+};
+
+/* reg_ratr_ipip_type
+ * Underlay destination ip type.
+ * Note: the type field must match the protocol of the router interface.
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, ratr, ipip_type, 0x10, 16, 4);
+
+/* reg_ratr_ipip_ipv4_udip
+ * Underlay ipv4 dip.
+ * Reserved when ipip_type is IPv6.
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, ratr, ipip_ipv4_udip, 0x18, 0, 32);
+
+/* reg_ratr_ipip_ipv6_ptr
+ * Pointer to IPv6 underlay destination ip address.
+ * For Spectrum: Pointer to KVD linear space.
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, ratr, ipip_ipv6_ptr, 0x1C, 0, 24);
+
 static inline void
 mlxsw_reg_ratr_pack(char *payload,
 		    enum mlxsw_reg_ratr_op op, bool valid,
+		    enum mlxsw_reg_ratr_type type,
 		    u32 adjacency_index, u16 egress_rif)
 {
 	MLXSW_REG_ZERO(ratr, payload);
 	mlxsw_reg_ratr_op_set(payload, op);
 	mlxsw_reg_ratr_v_set(payload, valid);
+	mlxsw_reg_ratr_type_set(payload, type);
 	mlxsw_reg_ratr_adjacency_index_low_set(payload, adjacency_index);
 	mlxsw_reg_ratr_adjacency_index_high_set(payload, adjacency_index >> 16);
 	mlxsw_reg_ratr_egress_router_interface_set(payload, egress_rif);
@@ -4341,6 +4501,12 @@ static inline void mlxsw_reg_ratr_eth_entry_pack(char *payload,
 	mlxsw_reg_ratr_eth_destination_mac_memcpy_to(payload, dest_mac);
 }
 
+static inline void mlxsw_reg_ratr_ipip4_entry_pack(char *payload, u32 ipv4_udip)
+{
+	mlxsw_reg_ratr_ipip_type_set(payload, MLXSW_REG_RATR_IPIP_TYPE_IPV4);
+	mlxsw_reg_ratr_ipip_ipv4_udip_set(payload, ipv4_udip);
+}
+
 /* RICNT - Router Interface Counter Register
  * -----------------------------------------
  * The RICNT register retrieves per port performance counters
@@ -4712,12 +4878,13 @@ MLXSW_ITEM32(reg, ralue, prefix_len, 0x08, 0, 8);
 /* reg_ralue_dip*
  * The prefix of the route or of the marker that the object of the LPM
  * is compared with. The most significant bits of the dip are the prefix.
- * The list significant bits must be '0' if the prefix_len is smaller
+ * The least significant bits must be '0' if the prefix_len is smaller
  * than 128 for IPv6 or smaller than 32 for IPv4.
  * IPv4 address uses bits dip[31:0] and bits dip[127:32] are reserved.
  * Access: Index
  */
 MLXSW_ITEM32(reg, ralue, dip4, 0x18, 0, 32);
+MLXSW_ITEM_BUF(reg, ralue, dip6, 0x0C, 16);
 
 enum mlxsw_reg_ralue_entry_type {
 	MLXSW_REG_RALUE_ENTRY_TYPE_MARKER_ENTRY = 1,
@@ -4806,7 +4973,7 @@ MLXSW_ITEM32(reg, ralue, ecmp_size, 0x28, 0, 13);
  */
 MLXSW_ITEM32(reg, ralue, local_erif, 0x24, 0, 16);
 
-/* reg_ralue_v
+/* reg_ralue_ip2me_v
  * Valid bit for the tunnel_ptr field.
  * If valid = 0 then trap to CPU as IP2ME trap ID.
  * If valid = 1 and the packet format allows NVE or IPinIP tunnel
@@ -4816,15 +4983,15 @@ MLXSW_ITEM32(reg, ralue, local_erif, 0x24, 0, 16);
  * Only relevant in case of IP2ME action.
  * Access: RW
  */
-MLXSW_ITEM32(reg, ralue, v, 0x24, 31, 1);
+MLXSW_ITEM32(reg, ralue, ip2me_v, 0x24, 31, 1);
 
-/* reg_ralue_tunnel_ptr
+/* reg_ralue_ip2me_tunnel_ptr
  * Tunnel Pointer for NVE or IPinIP tunnel decapsulation.
  * For Spectrum, pointer to KVD Linear.
  * Only relevant in case of IP2ME action.
  * Access: RW
  */
-MLXSW_ITEM32(reg, ralue, tunnel_ptr, 0x24, 0, 24);
+MLXSW_ITEM32(reg, ralue, ip2me_tunnel_ptr, 0x24, 0, 24);
 
 static inline void mlxsw_reg_ralue_pack(char *payload,
 					enum mlxsw_reg_ralxx_protocol protocol,
@@ -4851,6 +5018,16 @@ static inline void mlxsw_reg_ralue_pack4(char *payload,
 	mlxsw_reg_ralue_dip4_set(payload, dip);
 }
 
+static inline void mlxsw_reg_ralue_pack6(char *payload,
+					 enum mlxsw_reg_ralxx_protocol protocol,
+					 enum mlxsw_reg_ralue_op op,
+					 u16 virtual_router, u8 prefix_len,
+					 const void *dip)
+{
+	mlxsw_reg_ralue_pack(payload, protocol, op, virtual_router, prefix_len);
+	mlxsw_reg_ralue_dip6_memcpy_to(payload, dip);
+}
+
 static inline void
 mlxsw_reg_ralue_act_remote_pack(char *payload,
 				enum mlxsw_reg_ralue_trap_action trap_action,
@@ -4883,6 +5060,15 @@ mlxsw_reg_ralue_act_ip2me_pack(char *payload)
 					MLXSW_REG_RALUE_ACTION_TYPE_IP2ME);
 }
 
+static inline void
+mlxsw_reg_ralue_act_ip2me_tun_pack(char *payload, u32 tunnel_ptr)
+{
+	mlxsw_reg_ralue_action_type_set(payload,
+					MLXSW_REG_RALUE_ACTION_TYPE_IP2ME);
+	mlxsw_reg_ralue_ip2me_v_set(payload, 1);
+	mlxsw_reg_ralue_ip2me_tunnel_ptr_set(payload, tunnel_ptr);
+}
+
 /* RAUHT - Router Algorithmic LPM Unicast Host Table Register
  * ----------------------------------------------------------
  * The RAUHT register is used to configure and query the Unicast Host table in
@@ -4954,6 +5140,7 @@ MLXSW_ITEM32(reg, rauht, rif, 0x00, 0, 16);
  * Access: Index
  */
 MLXSW_ITEM32(reg, rauht, dip4, 0x1C, 0x0, 32);
+MLXSW_ITEM_BUF(reg, rauht, dip6, 0x10, 16);
 
 enum mlxsw_reg_rauht_trap_action {
 	MLXSW_REG_RAUHT_TRAP_ACTION_NOP,
@@ -4982,6 +5169,15 @@ enum mlxsw_reg_rauht_trap_id {
  */
 MLXSW_ITEM32(reg, rauht, trap_id, 0x60, 0, 9);
 
+enum mlxsw_reg_flow_counter_set_type {
+	/* No count */
+	MLXSW_REG_FLOW_COUNTER_SET_TYPE_NO_COUNT = 0x00,
+	/* Count packets and bytes */
+	MLXSW_REG_FLOW_COUNTER_SET_TYPE_PACKETS_BYTES = 0x03,
+	/* Count only packets */
+	MLXSW_REG_FLOW_COUNTER_SET_TYPE_PACKETS = 0x05,
+};
+
 /* reg_rauht_counter_set_type
  * Counter set type for flow counters
  * Access: RW
@@ -5018,6 +5214,23 @@ static inline void mlxsw_reg_rauht_pack4(char *payload,
 	mlxsw_reg_rauht_dip4_set(payload, dip);
 }
 
+static inline void mlxsw_reg_rauht_pack6(char *payload,
+					 enum mlxsw_reg_rauht_op op, u16 rif,
+					 const char *mac, const char *dip)
+{
+	mlxsw_reg_rauht_pack(payload, op, rif, mac);
+	mlxsw_reg_rauht_type_set(payload, MLXSW_REG_RAUHT_TYPE_IPV6);
+	mlxsw_reg_rauht_dip6_memcpy_to(payload, dip);
+}
+
+static inline void mlxsw_reg_rauht_pack_counter(char *payload,
+						u64 counter_index)
+{
+	mlxsw_reg_rauht_counter_index_set(payload, counter_index);
+	mlxsw_reg_rauht_counter_set_type_set(payload,
+					     MLXSW_REG_FLOW_COUNTER_SET_TYPE_PACKETS_BYTES);
+}
+
 /* RALEU - Router Algorithmic LPM ECMP Update Register
  * ---------------------------------------------------
  * The register enables updating the ECMP section in the action for multiple
@@ -5216,6 +5429,30 @@ MLXSW_ITEM32_INDEXED(reg, rauhtd, ipv4_ent_rif, MLXSW_REG_RAUHTD_BASE_LEN, 0,
 MLXSW_ITEM32_INDEXED(reg, rauhtd, ipv4_ent_dip, MLXSW_REG_RAUHTD_BASE_LEN, 0,
 		     32, MLXSW_REG_RAUHTD_IPV4_ENT_LEN, 0x04, false);
 
+#define MLXSW_REG_RAUHTD_IPV6_ENT_LEN 0x20
+
+/* reg_rauhtd_ipv6_ent_a
+ * Activity. Set for new entries. Set if a packet lookup has hit on the
+ * specific entry.
+ * Access: RO
+ */
+MLXSW_ITEM32_INDEXED(reg, rauhtd, ipv6_ent_a, MLXSW_REG_RAUHTD_BASE_LEN, 16, 1,
+		     MLXSW_REG_RAUHTD_IPV6_ENT_LEN, 0x00, false);
+
+/* reg_rauhtd_ipv6_ent_rif
+ * Router interface.
+ * Access: RO
+ */
+MLXSW_ITEM32_INDEXED(reg, rauhtd, ipv6_ent_rif, MLXSW_REG_RAUHTD_BASE_LEN, 0,
+		     16, MLXSW_REG_RAUHTD_IPV6_ENT_LEN, 0x00, false);
+
+/* reg_rauhtd_ipv6_ent_dip
+ * Destination IPv6 address.
+ * Access: RO
+ */
+MLXSW_ITEM_BUF_INDEXED(reg, rauhtd, ipv6_ent_dip, MLXSW_REG_RAUHTD_BASE_LEN,
+		       16, MLXSW_REG_RAUHTD_IPV6_ENT_LEN, 0x10);
+
 static inline void mlxsw_reg_rauhtd_ent_ipv4_unpack(char *payload,
 						    int ent_index, u16 *p_rif,
 						    u32 *p_dip)
@@ -5224,6 +5461,141 @@ static inline void mlxsw_reg_rauhtd_ent_ipv4_unpack(char *payload,
 	*p_dip = mlxsw_reg_rauhtd_ipv4_ent_dip_get(payload, ent_index);
 }
 
+static inline void mlxsw_reg_rauhtd_ent_ipv6_unpack(char *payload,
+						    int rec_index, u16 *p_rif,
+						    char *p_dip)
+{
+	*p_rif = mlxsw_reg_rauhtd_ipv6_ent_rif_get(payload, rec_index);
+	mlxsw_reg_rauhtd_ipv6_ent_dip_memcpy_from(payload, rec_index, p_dip);
+}
+
+/* RTDP - Routing Tunnel Decap Properties Register
+ * -----------------------------------------------
+ * The RTDP register is used for configuring the tunnel decap properties of NVE
+ * and IPinIP.
+ */
+#define MLXSW_REG_RTDP_ID 0x8020
+#define MLXSW_REG_RTDP_LEN 0x44
+
+MLXSW_REG_DEFINE(rtdp, MLXSW_REG_RTDP_ID, MLXSW_REG_RTDP_LEN);
+
+enum mlxsw_reg_rtdp_type {
+	MLXSW_REG_RTDP_TYPE_NVE,
+	MLXSW_REG_RTDP_TYPE_IPIP,
+};
+
+/* reg_rtdp_type
+ * Type of the RTDP entry as per enum mlxsw_reg_rtdp_type.
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, rtdp, type, 0x00, 28, 4);
+
+/* reg_rtdp_tunnel_index
+ * Index to the Decap entry.
+ * For Spectrum, Index to KVD Linear.
+ * Access: Index
+ */
+MLXSW_ITEM32(reg, rtdp, tunnel_index, 0x00, 0, 24);
+
+/* IPinIP */
+
+/* reg_rtdp_ipip_irif
+ * Ingress Router Interface for the overlay router
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, rtdp, ipip_irif, 0x04, 16, 16);
+
+enum mlxsw_reg_rtdp_ipip_sip_check {
+	/* No sip checks. */
+	MLXSW_REG_RTDP_IPIP_SIP_CHECK_NO,
+	/* Filter packet if underlay is not IPv4 or if underlay SIP does not
+	 * equal ipv4_usip.
+	 */
+	MLXSW_REG_RTDP_IPIP_SIP_CHECK_FILTER_IPV4,
+	/* Filter packet if underlay is not IPv6 or if underlay SIP does not
+	 * equal ipv6_usip.
+	 */
+	MLXSW_REG_RTDP_IPIP_SIP_CHECK_FILTER_IPV6 = 3,
+};
+
+/* reg_rtdp_ipip_sip_check
+ * SIP check to perform. If decapsulation failed due to these configurations
+ * then trap_id is IPIP_DECAP_ERROR.
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, rtdp, ipip_sip_check, 0x04, 0, 3);
+
+/* If set, allow decapsulation of IPinIP (without GRE). */
+#define MLXSW_REG_RTDP_IPIP_TYPE_CHECK_ALLOW_IPIP	BIT(0)
+/* If set, allow decapsulation of IPinGREinIP without a key. */
+#define MLXSW_REG_RTDP_IPIP_TYPE_CHECK_ALLOW_GRE	BIT(1)
+/* If set, allow decapsulation of IPinGREinIP with a key. */
+#define MLXSW_REG_RTDP_IPIP_TYPE_CHECK_ALLOW_GRE_KEY	BIT(2)
+
+/* reg_rtdp_ipip_type_check
+ * Flags as per MLXSW_REG_RTDP_IPIP_TYPE_CHECK_*. If decapsulation failed due to
+ * these configurations then trap_id is IPIP_DECAP_ERROR.
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, rtdp, ipip_type_check, 0x08, 24, 3);
+
+/* reg_rtdp_ipip_gre_key_check
+ * Whether GRE key should be checked. When check is enabled:
+ * - A packet received as IPinIP (without GRE) will always pass.
+ * - A packet received as IPinGREinIP without a key will not pass the check.
+ * - A packet received as IPinGREinIP with a key will pass the check only if the
+ *   key in the packet is equal to expected_gre_key.
+ * If decapsulation failed due to GRE key then trap_id is IPIP_DECAP_ERROR.
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, rtdp, ipip_gre_key_check, 0x08, 23, 1);
+
+/* reg_rtdp_ipip_ipv4_usip
+ * Underlay IPv4 address for ipv4 source address check.
+ * Reserved when sip_check is not '1'.
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, rtdp, ipip_ipv4_usip, 0x0C, 0, 32);
+
+/* reg_rtdp_ipip_ipv6_usip_ptr
+ * This field is valid when sip_check is "sipv6 check explicitly". This is a
+ * pointer to the IPv6 DIP which is configured by RIPS. For Spectrum, the index
+ * is to the KVD linear.
+ * Reserved when sip_check is not MLXSW_REG_RTDP_IPIP_SIP_CHECK_FILTER_IPV6.
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, rtdp, ipip_ipv6_usip_ptr, 0x10, 0, 24);
+
+/* reg_rtdp_ipip_expected_gre_key
+ * GRE key for checking.
+ * Reserved when gre_key_check is '0'.
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, rtdp, ipip_expected_gre_key, 0x14, 0, 32);
+
+static inline void mlxsw_reg_rtdp_pack(char *payload,
+				       enum mlxsw_reg_rtdp_type type,
+				       u32 tunnel_index)
+{
+	MLXSW_REG_ZERO(rtdp, payload);
+	mlxsw_reg_rtdp_type_set(payload, type);
+	mlxsw_reg_rtdp_tunnel_index_set(payload, tunnel_index);
+}
+
+static inline void
+mlxsw_reg_rtdp_ipip4_pack(char *payload, u16 irif,
+			  enum mlxsw_reg_rtdp_ipip_sip_check sip_check,
+			  unsigned int type_check, bool gre_key_check,
+			  u32 ipv4_usip, u32 expected_gre_key)
+{
+	mlxsw_reg_rtdp_ipip_irif_set(payload, irif);
+	mlxsw_reg_rtdp_ipip_sip_check_set(payload, sip_check);
+	mlxsw_reg_rtdp_ipip_type_check_set(payload, type_check);
+	mlxsw_reg_rtdp_ipip_gre_key_check_set(payload, gre_key_check);
+	mlxsw_reg_rtdp_ipip_ipv4_usip_set(payload, ipv4_usip);
+	mlxsw_reg_rtdp_ipip_expected_gre_key_set(payload, expected_gre_key);
+}
+
 /* MFCR - Management Fan Control Register
  * --------------------------------------
  * This register controls the settings of the Fan Speed PWM mechanism.
@@ -5982,15 +6354,6 @@ static inline void mlxsw_reg_mpsc_pack(char *payload, u8 local_port, bool e,
 
 MLXSW_REG_DEFINE(mgpc, MLXSW_REG_MGPC_ID, MLXSW_REG_MGPC_LEN);
 
-enum mlxsw_reg_mgpc_counter_set_type {
-	/* No count */
-	MLXSW_REG_MGPC_COUNTER_SET_TYPE_NO_COUT = 0x00,
-	/* Count packets and bytes */
-	MLXSW_REG_MGPC_COUNTER_SET_TYPE_PACKETS_BYTES = 0x03,
-	/* Count only packets */
-	MLXSW_REG_MGPC_COUNTER_SET_TYPE_PACKETS = 0x05,
-};
-
 /* reg_mgpc_counter_set_type
  * Counter set type.
  * Access: OP
@@ -6030,7 +6393,7 @@ MLXSW_ITEM64(reg, mgpc, packet_counter, 0x10, 0, 64);
 
 static inline void mlxsw_reg_mgpc_pack(char *payload, u32 counter_index,
 				       enum mlxsw_reg_mgpc_opcode opcode,
-				       enum mlxsw_reg_mgpc_counter_set_type set_type)
+				       enum mlxsw_reg_flow_counter_set_type set_type)
 {
 	MLXSW_REG_ZERO(mgpc, payload);
 	mlxsw_reg_mgpc_counter_index_set(payload, counter_index);
@@ -6494,6 +6857,7 @@ static const struct mlxsw_reg_info *mlxsw_reg_infos[] = {
 	MLXSW_REG(rgcr),
 	MLXSW_REG(ritr),
 	MLXSW_REG(ratr),
+	MLXSW_REG(rtdp),
 	MLXSW_REG(ricnt),
 	MLXSW_REG(ralta),
 	MLXSW_REG(ralst),
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
index 60bf8f27cc00..696b99e65a5a 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
@@ -58,6 +58,7 @@
 #include <net/tc_act/tc_mirred.h>
 #include <net/netevent.h>
 #include <net/tc_act/tc_sample.h>
+#include <net/addrconf.h>
 
 #include "spectrum.h"
 #include "pci.h"
@@ -381,12 +382,14 @@ int mlxsw_sp_flow_counter_get(struct mlxsw_sp *mlxsw_sp,
 	int err;
 
 	mlxsw_reg_mgpc_pack(mgpc_pl, counter_index, MLXSW_REG_MGPC_OPCODE_NOP,
-			    MLXSW_REG_MGPC_COUNTER_SET_TYPE_PACKETS_BYTES);
+			    MLXSW_REG_FLOW_COUNTER_SET_TYPE_PACKETS_BYTES);
 	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(mgpc), mgpc_pl);
 	if (err)
 		return err;
-	*packets = mlxsw_reg_mgpc_packet_counter_get(mgpc_pl);
-	*bytes = mlxsw_reg_mgpc_byte_counter_get(mgpc_pl);
+	if (packets)
+		*packets = mlxsw_reg_mgpc_packet_counter_get(mgpc_pl);
+	if (bytes)
+		*bytes = mlxsw_reg_mgpc_byte_counter_get(mgpc_pl);
 	return 0;
 }
 
@@ -396,7 +399,7 @@ static int mlxsw_sp_flow_counter_clear(struct mlxsw_sp *mlxsw_sp,
 	char mgpc_pl[MLXSW_REG_MGPC_LEN];
 
 	mlxsw_reg_mgpc_pack(mgpc_pl, counter_index, MLXSW_REG_MGPC_OPCODE_CLEAR,
-			    MLXSW_REG_MGPC_COUNTER_SET_TYPE_PACKETS_BYTES);
+			    MLXSW_REG_FLOW_COUNTER_SET_TYPE_PACKETS_BYTES);
 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(mgpc), mgpc_pl);
 }
 
@@ -572,15 +575,14 @@ static void mlxsw_sp_span_entry_destroy(struct mlxsw_sp *mlxsw_sp,
 }
 
 static struct mlxsw_sp_span_entry *
-mlxsw_sp_span_entry_find(struct mlxsw_sp_port *port)
+mlxsw_sp_span_entry_find(struct mlxsw_sp *mlxsw_sp, u8 local_port)
 {
-	struct mlxsw_sp *mlxsw_sp = port->mlxsw_sp;
 	int i;
 
 	for (i = 0; i < mlxsw_sp->span.entries_count; i++) {
 		struct mlxsw_sp_span_entry *curr = &mlxsw_sp->span.entries[i];
 
-		if (curr->used && curr->local_port == port->local_port)
+		if (curr->used && curr->local_port == local_port)
 			return curr;
 	}
 	return NULL;
@@ -591,7 +593,8 @@ static struct mlxsw_sp_span_entry
 {
 	struct mlxsw_sp_span_entry *span_entry;
 
-	span_entry = mlxsw_sp_span_entry_find(port);
+	span_entry = mlxsw_sp_span_entry_find(port->mlxsw_sp,
+					      port->local_port);
 	if (span_entry) {
 		/* Already exists, just take a reference */
 		span_entry->ref_count++;
@@ -780,12 +783,13 @@ err_port_bind:
 }
 
 static void mlxsw_sp_span_mirror_remove(struct mlxsw_sp_port *from,
-					struct mlxsw_sp_port *to,
+					u8 destination_port,
 					enum mlxsw_sp_span_type type)
 {
 	struct mlxsw_sp_span_entry *span_entry;
 
-	span_entry = mlxsw_sp_span_entry_find(to);
+	span_entry = mlxsw_sp_span_entry_find(from->mlxsw_sp,
+					      destination_port);
 	if (!span_entry) {
 		netdev_err(from->dev, "no span entry found\n");
 		return;
@@ -1560,14 +1564,12 @@ static void
 mlxsw_sp_port_del_cls_matchall_mirror(struct mlxsw_sp_port *mlxsw_sp_port,
 				      struct mlxsw_sp_port_mall_mirror_tc_entry *mirror)
 {
-	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 	enum mlxsw_sp_span_type span_type;
-	struct mlxsw_sp_port *to_port;
 
-	to_port = mlxsw_sp->ports[mirror->to_local_port];
 	span_type = mirror->ingress ?
 			MLXSW_SP_SPAN_INGRESS : MLXSW_SP_SPAN_EGRESS;
-	mlxsw_sp_span_mirror_remove(mlxsw_sp_port, to_port, span_type);
+	mlxsw_sp_span_mirror_remove(mlxsw_sp_port, mirror->to_local_port,
+				    span_type);
 }
 
 static int
@@ -1616,16 +1618,16 @@ mlxsw_sp_port_del_cls_matchall_sample(struct mlxsw_sp_port *mlxsw_sp_port)
 }
 
 static int mlxsw_sp_port_add_cls_matchall(struct mlxsw_sp_port *mlxsw_sp_port,
-					  __be16 protocol,
-					  struct tc_cls_matchall_offload *cls,
+					  struct tc_cls_matchall_offload *f,
 					  bool ingress)
 {
 	struct mlxsw_sp_port_mall_tc_entry *mall_tc_entry;
+	__be16 protocol = f->common.protocol;
 	const struct tc_action *a;
 	LIST_HEAD(actions);
 	int err;
 
-	if (!tc_single_action(cls->exts)) {
+	if (!tcf_exts_has_one_action(f->exts)) {
 		netdev_err(mlxsw_sp_port->dev, "only singular actions are supported\n");
 		return -EOPNOTSUPP;
 	}
@@ -1633,9 +1635,9 @@ static int mlxsw_sp_port_add_cls_matchall(struct mlxsw_sp_port *mlxsw_sp_port,
 	mall_tc_entry = kzalloc(sizeof(*mall_tc_entry), GFP_KERNEL);
 	if (!mall_tc_entry)
 		return -ENOMEM;
-	mall_tc_entry->cookie = cls->cookie;
+	mall_tc_entry->cookie = f->cookie;
 
-	tcf_exts_to_list(cls->exts, &actions);
+	tcf_exts_to_list(f->exts, &actions);
 	a = list_first_entry(&actions, struct tc_action, list);
 
 	if (is_tcf_mirred_egress_mirror(a) && protocol == htons(ETH_P_ALL)) {
@@ -1647,7 +1649,7 @@ static int mlxsw_sp_port_add_cls_matchall(struct mlxsw_sp_port *mlxsw_sp_port,
 							    mirror, a, ingress);
 	} else if (is_tcf_sample(a) && protocol == htons(ETH_P_ALL)) {
 		mall_tc_entry->type = MLXSW_SP_PORT_MALL_SAMPLE;
-		err = mlxsw_sp_port_add_cls_matchall_sample(mlxsw_sp_port, cls,
+		err = mlxsw_sp_port_add_cls_matchall_sample(mlxsw_sp_port, f,
 							    a, ingress);
 	} else {
 		err = -EOPNOTSUPP;
@@ -1665,12 +1667,12 @@ err_add_action:
 }
 
 static void mlxsw_sp_port_del_cls_matchall(struct mlxsw_sp_port *mlxsw_sp_port,
-					   struct tc_cls_matchall_offload *cls)
+					   struct tc_cls_matchall_offload *f)
 {
 	struct mlxsw_sp_port_mall_tc_entry *mall_tc_entry;
 
 	mall_tc_entry = mlxsw_sp_port_mall_tc_entry_find(mlxsw_sp_port,
-							 cls->cookie);
+							 f->cookie);
 	if (!mall_tc_entry) {
 		netdev_dbg(mlxsw_sp_port->dev, "tc entry not found on port\n");
 		return;
@@ -1692,49 +1694,72 @@ static void mlxsw_sp_port_del_cls_matchall(struct mlxsw_sp_port *mlxsw_sp_port,
 	kfree(mall_tc_entry);
 }
 
-static int mlxsw_sp_setup_tc(struct net_device *dev, u32 handle,
-			     u32 chain_index, __be16 proto,
-			     struct tc_to_netdev *tc)
+static int mlxsw_sp_setup_tc_cls_matchall(struct mlxsw_sp_port *mlxsw_sp_port,
+					  struct tc_cls_matchall_offload *f)
 {
-	struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev);
-	bool ingress = TC_H_MAJ(handle) == TC_H_MAJ(TC_H_INGRESS);
+	bool ingress;
 
-	if (chain_index)
+	if (is_classid_clsact_ingress(f->common.classid))
+		ingress = true;
+	else if (is_classid_clsact_egress(f->common.classid))
+		ingress = false;
+	else
 		return -EOPNOTSUPP;
 
-	switch (tc->type) {
-	case TC_SETUP_MATCHALL:
-		switch (tc->cls_mall->command) {
-		case TC_CLSMATCHALL_REPLACE:
-			return mlxsw_sp_port_add_cls_matchall(mlxsw_sp_port,
-							      proto,
-							      tc->cls_mall,
-							      ingress);
-		case TC_CLSMATCHALL_DESTROY:
-			mlxsw_sp_port_del_cls_matchall(mlxsw_sp_port,
-						       tc->cls_mall);
-			return 0;
-		default:
-			return -EOPNOTSUPP;
-		}
-	case TC_SETUP_CLSFLOWER:
-		switch (tc->cls_flower->command) {
-		case TC_CLSFLOWER_REPLACE:
-			return mlxsw_sp_flower_replace(mlxsw_sp_port, ingress,
-						       proto, tc->cls_flower);
-		case TC_CLSFLOWER_DESTROY:
-			mlxsw_sp_flower_destroy(mlxsw_sp_port, ingress,
-						tc->cls_flower);
-			return 0;
-		case TC_CLSFLOWER_STATS:
-			return mlxsw_sp_flower_stats(mlxsw_sp_port, ingress,
-						     tc->cls_flower);
-		default:
-			return -EOPNOTSUPP;
-		}
+	if (f->common.chain_index)
+		return -EOPNOTSUPP;
+
+	switch (f->command) {
+	case TC_CLSMATCHALL_REPLACE:
+		return mlxsw_sp_port_add_cls_matchall(mlxsw_sp_port, f,
+						      ingress);
+	case TC_CLSMATCHALL_DESTROY:
+		mlxsw_sp_port_del_cls_matchall(mlxsw_sp_port, f);
+		return 0;
+	default:
+		return -EOPNOTSUPP;
 	}
+}
+
+static int
+mlxsw_sp_setup_tc_cls_flower(struct mlxsw_sp_port *mlxsw_sp_port,
+			     struct tc_cls_flower_offload *f)
+{
+	bool ingress;
+
+	if (is_classid_clsact_ingress(f->common.classid))
+		ingress = true;
+	else if (is_classid_clsact_egress(f->common.classid))
+		ingress = false;
+	else
+		return -EOPNOTSUPP;
 
-	return -EOPNOTSUPP;
+	switch (f->command) {
+	case TC_CLSFLOWER_REPLACE:
+		return mlxsw_sp_flower_replace(mlxsw_sp_port, ingress, f);
+	case TC_CLSFLOWER_DESTROY:
+		mlxsw_sp_flower_destroy(mlxsw_sp_port, ingress, f);
+		return 0;
+	case TC_CLSFLOWER_STATS:
+		return mlxsw_sp_flower_stats(mlxsw_sp_port, ingress, f);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int mlxsw_sp_setup_tc(struct net_device *dev, enum tc_setup_type type,
+			     void *type_data)
+{
+	struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev);
+
+	switch (type) {
+	case TC_SETUP_CLSMATCHALL:
+		return mlxsw_sp_setup_tc_cls_matchall(mlxsw_sp_port, type_data);
+	case TC_SETUP_CLSFLOWER:
+		return mlxsw_sp_setup_tc_cls_flower(mlxsw_sp_port, type_data);
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
 static const struct net_device_ops mlxsw_sp_port_netdev_ops = {
@@ -2519,7 +2544,9 @@ out:
 	return err;
 }
 
-#define MLXSW_SP_QSFP_I2C_ADDR 0x50
+#define MLXSW_SP_I2C_ADDR_LOW 0x50
+#define MLXSW_SP_I2C_ADDR_HIGH 0x51
+#define MLXSW_SP_EEPROM_PAGE_LENGTH 256
 
 static int mlxsw_sp_query_module_eeprom(struct mlxsw_sp_port *mlxsw_sp_port,
 					u16 offset, u16 size, void *data,
@@ -2528,12 +2555,25 @@ static int mlxsw_sp_query_module_eeprom(struct mlxsw_sp_port *mlxsw_sp_port,
 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 	char eeprom_tmp[MLXSW_SP_REG_MCIA_EEPROM_SIZE];
 	char mcia_pl[MLXSW_REG_MCIA_LEN];
+	u16 i2c_addr;
 	int status;
 	int err;
 
 	size = min_t(u16, size, MLXSW_SP_REG_MCIA_EEPROM_SIZE);
+
+	if (offset < MLXSW_SP_EEPROM_PAGE_LENGTH &&
+	    offset + size > MLXSW_SP_EEPROM_PAGE_LENGTH)
+		/* Cross pages read, read until offset 256 in low page */
+		size = MLXSW_SP_EEPROM_PAGE_LENGTH - offset;
+
+	i2c_addr = MLXSW_SP_I2C_ADDR_LOW;
+	if (offset >= MLXSW_SP_EEPROM_PAGE_LENGTH) {
+		i2c_addr = MLXSW_SP_I2C_ADDR_HIGH;
+		offset -= MLXSW_SP_EEPROM_PAGE_LENGTH;
+	}
+
 	mlxsw_reg_mcia_pack(mcia_pl, mlxsw_sp_port->mapping.module,
-			    0, 0, offset, size, MLXSW_SP_QSFP_I2C_ADDR);
+			    0, 0, offset, size, i2c_addr);
 
 	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(mcia), mcia_pl);
 	if (err)
@@ -3333,15 +3373,48 @@ static const struct mlxsw_listener mlxsw_sp_listener[] = {
 	MLXSW_SP_RXL_MARK(ARPBC, MIRROR_TO_CPU, ARP, false),
 	MLXSW_SP_RXL_MARK(ARPUC, MIRROR_TO_CPU, ARP, false),
 	MLXSW_SP_RXL_NO_MARK(FID_MISS, TRAP_TO_CPU, IP2ME, false),
+	MLXSW_SP_RXL_MARK(IPV6_MLDV12_LISTENER_QUERY, MIRROR_TO_CPU, IPV6_MLD,
+			  false),
+	MLXSW_SP_RXL_NO_MARK(IPV6_MLDV1_LISTENER_REPORT, TRAP_TO_CPU, IPV6_MLD,
+			     false),
+	MLXSW_SP_RXL_NO_MARK(IPV6_MLDV1_LISTENER_DONE, TRAP_TO_CPU, IPV6_MLD,
+			     false),
+	MLXSW_SP_RXL_NO_MARK(IPV6_MLDV2_LISTENER_REPORT, TRAP_TO_CPU, IPV6_MLD,
+			     false),
 	/* L3 traps */
-	MLXSW_SP_RXL_NO_MARK(MTUERROR, TRAP_TO_CPU, ROUTER_EXP, false),
-	MLXSW_SP_RXL_NO_MARK(TTLERROR, TRAP_TO_CPU, ROUTER_EXP, false),
-	MLXSW_SP_RXL_NO_MARK(LBERROR, TRAP_TO_CPU, ROUTER_EXP, false),
-	MLXSW_SP_RXL_MARK(OSPF, TRAP_TO_CPU, OSPF, false),
-	MLXSW_SP_RXL_NO_MARK(IP2ME, TRAP_TO_CPU, IP2ME, false),
-	MLXSW_SP_RXL_NO_MARK(RTR_INGRESS0, TRAP_TO_CPU, REMOTE_ROUTE, false),
-	MLXSW_SP_RXL_NO_MARK(HOST_MISS_IPV4, TRAP_TO_CPU, ARP_MISS, false),
-	MLXSW_SP_RXL_NO_MARK(BGP_IPV4, TRAP_TO_CPU, BGP_IPV4, false),
+	MLXSW_SP_RXL_MARK(MTUERROR, TRAP_TO_CPU, ROUTER_EXP, false),
+	MLXSW_SP_RXL_MARK(TTLERROR, TRAP_TO_CPU, ROUTER_EXP, false),
+	MLXSW_SP_RXL_MARK(LBERROR, TRAP_TO_CPU, ROUTER_EXP, false),
+	MLXSW_SP_RXL_MARK(IP2ME, TRAP_TO_CPU, IP2ME, false),
+	MLXSW_SP_RXL_MARK(IPV6_UNSPECIFIED_ADDRESS, TRAP_TO_CPU, ROUTER_EXP,
+			  false),
+	MLXSW_SP_RXL_MARK(IPV6_LINK_LOCAL_DEST, TRAP_TO_CPU, ROUTER_EXP, false),
+	MLXSW_SP_RXL_MARK(IPV6_LINK_LOCAL_SRC, TRAP_TO_CPU, ROUTER_EXP, false),
+	MLXSW_SP_RXL_MARK(IPV6_ALL_NODES_LINK, TRAP_TO_CPU, ROUTER_EXP, false),
+	MLXSW_SP_RXL_MARK(IPV6_ALL_ROUTERS_LINK, TRAP_TO_CPU, ROUTER_EXP,
+			  false),
+	MLXSW_SP_RXL_MARK(IPV4_OSPF, TRAP_TO_CPU, OSPF, false),
+	MLXSW_SP_RXL_MARK(IPV6_OSPF, TRAP_TO_CPU, OSPF, false),
+	MLXSW_SP_RXL_MARK(IPV6_DHCP, TRAP_TO_CPU, DHCP, false),
+	MLXSW_SP_RXL_MARK(RTR_INGRESS0, TRAP_TO_CPU, REMOTE_ROUTE, false),
+	MLXSW_SP_RXL_MARK(IPV4_BGP, TRAP_TO_CPU, BGP, false),
+	MLXSW_SP_RXL_MARK(IPV6_BGP, TRAP_TO_CPU, BGP, false),
+	MLXSW_SP_RXL_MARK(L3_IPV6_ROUTER_SOLICITATION, TRAP_TO_CPU, IPV6_ND,
+			  false),
+	MLXSW_SP_RXL_MARK(L3_IPV6_ROUTER_ADVERTISMENT, TRAP_TO_CPU, IPV6_ND,
+			  false),
+	MLXSW_SP_RXL_MARK(L3_IPV6_NEIGHBOR_SOLICITATION, TRAP_TO_CPU, IPV6_ND,
+			  false),
+	MLXSW_SP_RXL_MARK(L3_IPV6_NEIGHBOR_ADVERTISMENT, TRAP_TO_CPU, IPV6_ND,
+			  false),
+	MLXSW_SP_RXL_MARK(L3_IPV6_REDIRECTION, TRAP_TO_CPU, IPV6_ND, false),
+	MLXSW_SP_RXL_MARK(IPV6_MC_LINK_LOCAL_DEST, TRAP_TO_CPU, ROUTER_EXP,
+			  false),
+	MLXSW_SP_RXL_MARK(HOST_MISS_IPV4, TRAP_TO_CPU, HOST_MISS, false),
+	MLXSW_SP_RXL_MARK(HOST_MISS_IPV6, TRAP_TO_CPU, HOST_MISS, false),
+	MLXSW_SP_RXL_MARK(ROUTER_ALERT_IPV4, TRAP_TO_CPU, ROUTER_EXP, false),
+	MLXSW_SP_RXL_MARK(ROUTER_ALERT_IPV6, TRAP_TO_CPU, ROUTER_EXP, false),
+	MLXSW_SP_RXL_MARK(IPIP_DECAP_ERROR, TRAP_TO_CPU, ROUTER_EXP, false),
 	/* PKT Sample trap */
 	MLXSW_RXL(mlxsw_sp_rx_listener_sample_func, PKT_SAMPLE, MIRROR_TO_CPU,
 		  false, SP_IP2ME, DISCARD),
@@ -3376,15 +3449,17 @@ static int mlxsw_sp_cpu_policers_set(struct mlxsw_core *mlxsw_core)
 			burst_size = 7;
 			break;
 		case MLXSW_REG_HTGT_TRAP_GROUP_SP_IGMP:
+		case MLXSW_REG_HTGT_TRAP_GROUP_SP_IPV6_MLD:
 			rate = 16 * 1024;
 			burst_size = 10;
 			break;
-		case MLXSW_REG_HTGT_TRAP_GROUP_SP_BGP_IPV4:
+		case MLXSW_REG_HTGT_TRAP_GROUP_SP_BGP:
 		case MLXSW_REG_HTGT_TRAP_GROUP_SP_ARP:
 		case MLXSW_REG_HTGT_TRAP_GROUP_SP_DHCP:
-		case MLXSW_REG_HTGT_TRAP_GROUP_SP_ARP_MISS:
+		case MLXSW_REG_HTGT_TRAP_GROUP_SP_HOST_MISS:
 		case MLXSW_REG_HTGT_TRAP_GROUP_SP_ROUTER_EXP:
 		case MLXSW_REG_HTGT_TRAP_GROUP_SP_REMOTE_ROUTE:
+		case MLXSW_REG_HTGT_TRAP_GROUP_SP_IPV6_ND:
 			rate = 1024;
 			burst_size = 7;
 			break;
@@ -3433,21 +3508,23 @@ static int mlxsw_sp_trap_groups_set(struct mlxsw_core *mlxsw_core)
 			priority = 5;
 			tc = 5;
 			break;
-		case MLXSW_REG_HTGT_TRAP_GROUP_SP_BGP_IPV4:
+		case MLXSW_REG_HTGT_TRAP_GROUP_SP_BGP:
 		case MLXSW_REG_HTGT_TRAP_GROUP_SP_DHCP:
 			priority = 4;
 			tc = 4;
 			break;
 		case MLXSW_REG_HTGT_TRAP_GROUP_SP_IGMP:
 		case MLXSW_REG_HTGT_TRAP_GROUP_SP_IP2ME:
+		case MLXSW_REG_HTGT_TRAP_GROUP_SP_IPV6_MLD:
 			priority = 3;
 			tc = 3;
 			break;
 		case MLXSW_REG_HTGT_TRAP_GROUP_SP_ARP:
+		case MLXSW_REG_HTGT_TRAP_GROUP_SP_IPV6_ND:
 			priority = 2;
 			tc = 2;
 			break;
-		case MLXSW_REG_HTGT_TRAP_GROUP_SP_ARP_MISS:
+		case MLXSW_REG_HTGT_TRAP_GROUP_SP_HOST_MISS:
 		case MLXSW_REG_HTGT_TRAP_GROUP_SP_ROUTER_EXP:
 		case MLXSW_REG_HTGT_TRAP_GROUP_SP_REMOTE_ROUTE:
 			priority = 1;
@@ -3694,7 +3771,7 @@ static void mlxsw_sp_fini(struct mlxsw_core *mlxsw_core)
 	mlxsw_sp_fids_fini(mlxsw_sp);
 }
 
-static struct mlxsw_config_profile mlxsw_sp_config_profile = {
+static const struct mlxsw_config_profile mlxsw_sp_config_profile = {
 	.used_max_vepa_channels		= 1,
 	.max_vepa_channels		= 0,
 	.used_max_mid			= 1,
@@ -4139,6 +4216,8 @@ static int mlxsw_sp_netdevice_port_upper_event(struct net_device *lower_dev,
 			return -EINVAL;
 		if (!info->linking)
 			break;
+		if (netdev_has_any_upper_dev(upper_dev))
+			return -EINVAL;
 		if (netif_is_lag_master(upper_dev) &&
 		    !mlxsw_sp_master_lag_check(mlxsw_sp, upper_dev,
 					       info->upper_info))
@@ -4258,6 +4337,10 @@ static int mlxsw_sp_netdevice_port_vlan_event(struct net_device *vlan_dev,
 		upper_dev = info->upper_dev;
 		if (!netif_is_bridge_master(upper_dev))
 			return -EINVAL;
+		if (!info->linking)
+			break;
+		if (netdev_has_any_upper_dev(upper_dev))
+			return -EINVAL;
 		break;
 	case NETDEV_CHANGEUPPER:
 		upper_dev = info->upper_dev;
@@ -4357,6 +4440,10 @@ static struct notifier_block mlxsw_sp_inetaddr_nb __read_mostly = {
 	.priority = 10,	/* Must be called before FIB notifier block */
 };
 
+static struct notifier_block mlxsw_sp_inet6addr_nb __read_mostly = {
+	.notifier_call = mlxsw_sp_inet6addr_event,
+};
+
 static struct notifier_block mlxsw_sp_router_netevent_nb __read_mostly = {
 	.notifier_call = mlxsw_sp_router_netevent_event,
 };
@@ -4377,6 +4464,7 @@ static int __init mlxsw_sp_module_init(void)
 
 	register_netdevice_notifier(&mlxsw_sp_netdevice_nb);
 	register_inetaddr_notifier(&mlxsw_sp_inetaddr_nb);
+	register_inet6addr_notifier(&mlxsw_sp_inet6addr_nb);
 	register_netevent_notifier(&mlxsw_sp_router_netevent_nb);
 
 	err = mlxsw_core_driver_register(&mlxsw_sp_driver);
@@ -4393,6 +4481,7 @@ err_pci_driver_register:
 	mlxsw_core_driver_unregister(&mlxsw_sp_driver);
 err_core_driver_register:
 	unregister_netevent_notifier(&mlxsw_sp_router_netevent_nb);
+	unregister_inet6addr_notifier(&mlxsw_sp_inet6addr_nb);
 	unregister_inetaddr_notifier(&mlxsw_sp_inetaddr_nb);
 	unregister_netdevice_notifier(&mlxsw_sp_netdevice_nb);
 	return err;
@@ -4403,6 +4492,7 @@ static void __exit mlxsw_sp_module_exit(void)
 	mlxsw_pci_driver_unregister(&mlxsw_sp_pci_driver);
 	mlxsw_core_driver_unregister(&mlxsw_sp_driver);
 	unregister_netevent_notifier(&mlxsw_sp_router_netevent_nb);
+	unregister_inet6addr_notifier(&mlxsw_sp_inet6addr_nb);
 	unregister_inetaddr_notifier(&mlxsw_sp_inetaddr_nb);
 	unregister_netdevice_notifier(&mlxsw_sp_netdevice_nb);
 }
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h
index 5ef98d4d0ab6..84ce83acdc19 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h
@@ -77,6 +77,7 @@ enum mlxsw_sp_rif_type {
 	MLXSW_SP_RIF_TYPE_SUBPORT,
 	MLXSW_SP_RIF_TYPE_VLAN,
 	MLXSW_SP_RIF_TYPE_FID,
+	MLXSW_SP_RIF_TYPE_IPIP_LB, /* IP-in-IP loopback. */
 	MLXSW_SP_RIF_TYPE_MAX,
 };
 
@@ -384,6 +385,8 @@ int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev);
 int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
 			    unsigned long event, void *ptr);
+int mlxsw_sp_inet6addr_event(struct notifier_block *unused,
+			     unsigned long event, void *ptr);
 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
 				 struct netdev_notifier_changeupper_info *info);
 void
@@ -415,6 +418,7 @@ struct mlxsw_sp_acl_profile_ops {
 	int (*ruleset_bind)(struct mlxsw_sp *mlxsw_sp, void *ruleset_priv,
 			    struct net_device *dev, bool ingress);
 	void (*ruleset_unbind)(struct mlxsw_sp *mlxsw_sp, void *ruleset_priv);
+	u16 (*ruleset_group_id)(void *ruleset_priv);
 	size_t rule_priv_size;
 	int (*rule_add)(struct mlxsw_sp *mlxsw_sp,
 			void *ruleset_priv, void *rule_priv,
@@ -438,11 +442,16 @@ struct mlxsw_sp_acl_ruleset;
 /* spectrum_acl.c */
 struct mlxsw_afk *mlxsw_sp_acl_afk(struct mlxsw_sp_acl *acl);
 struct mlxsw_sp_acl_ruleset *
-mlxsw_sp_acl_ruleset_get(struct mlxsw_sp *mlxsw_sp,
-			 struct net_device *dev, bool ingress,
+mlxsw_sp_acl_ruleset_lookup(struct mlxsw_sp *mlxsw_sp, struct net_device *dev,
+			    bool ingress, u32 chain_index,
+			    enum mlxsw_sp_acl_profile profile);
+struct mlxsw_sp_acl_ruleset *
+mlxsw_sp_acl_ruleset_get(struct mlxsw_sp *mlxsw_sp, struct net_device *dev,
+			 bool ingress, u32 chain_index,
 			 enum mlxsw_sp_acl_profile profile);
 void mlxsw_sp_acl_ruleset_put(struct mlxsw_sp *mlxsw_sp,
 			      struct mlxsw_sp_acl_ruleset *ruleset);
+u16 mlxsw_sp_acl_ruleset_group_id(struct mlxsw_sp_acl_ruleset *ruleset);
 
 struct mlxsw_sp_acl_rule_info *
 mlxsw_sp_acl_rulei_create(struct mlxsw_sp_acl *acl);
@@ -506,7 +515,7 @@ extern const struct mlxsw_sp_acl_ops mlxsw_sp_acl_tcam_ops;
 
 /* spectrum_flower.c */
 int mlxsw_sp_flower_replace(struct mlxsw_sp_port *mlxsw_sp_port, bool ingress,
-			    __be16 protocol, struct tc_cls_flower_offload *f);
+			    struct tc_cls_flower_offload *f);
 void mlxsw_sp_flower_destroy(struct mlxsw_sp_port *mlxsw_sp_port, bool ingress,
 			     struct tc_cls_flower_offload *f);
 int mlxsw_sp_flower_stats(struct mlxsw_sp_port *mlxsw_sp_port, bool ingress,
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c
index 01a1501b56ca..4b2455e3e079 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c
@@ -74,6 +74,7 @@ struct mlxsw_afk *mlxsw_sp_acl_afk(struct mlxsw_sp_acl *acl)
 struct mlxsw_sp_acl_ruleset_ht_key {
 	struct net_device *dev; /* dev this ruleset is bound to */
 	bool ingress;
+	u32 chain_index;
 	const struct mlxsw_sp_acl_profile_ops *ops;
 };
 
@@ -163,7 +164,8 @@ static void mlxsw_sp_acl_ruleset_destroy(struct mlxsw_sp *mlxsw_sp,
 
 static int mlxsw_sp_acl_ruleset_bind(struct mlxsw_sp *mlxsw_sp,
 				     struct mlxsw_sp_acl_ruleset *ruleset,
-				     struct net_device *dev, bool ingress)
+				     struct net_device *dev, bool ingress,
+				     u32 chain_index)
 {
 	const struct mlxsw_sp_acl_profile_ops *ops = ruleset->ht_key.ops;
 	struct mlxsw_sp_acl *acl = mlxsw_sp->acl;
@@ -171,13 +173,20 @@ static int mlxsw_sp_acl_ruleset_bind(struct mlxsw_sp *mlxsw_sp,
 
 	ruleset->ht_key.dev = dev;
 	ruleset->ht_key.ingress = ingress;
+	ruleset->ht_key.chain_index = chain_index;
 	err = rhashtable_insert_fast(&acl->ruleset_ht, &ruleset->ht_node,
 				     mlxsw_sp_acl_ruleset_ht_params);
 	if (err)
 		return err;
-	err = ops->ruleset_bind(mlxsw_sp, ruleset->priv, dev, ingress);
-	if (err)
-		goto err_ops_ruleset_bind;
+	if (!ruleset->ht_key.chain_index) {
+		/* We only need ruleset with chain index 0, the implicit one,
+		 * to be directly bound to device. The rest of the rulesets
+		 * are bound by "Goto action set".
+		 */
+		err = ops->ruleset_bind(mlxsw_sp, ruleset->priv, dev, ingress);
+		if (err)
+			goto err_ops_ruleset_bind;
+	}
 	return 0;
 
 err_ops_ruleset_bind:
@@ -192,7 +201,8 @@ static void mlxsw_sp_acl_ruleset_unbind(struct mlxsw_sp *mlxsw_sp,
 	const struct mlxsw_sp_acl_profile_ops *ops = ruleset->ht_key.ops;
 	struct mlxsw_sp_acl *acl = mlxsw_sp->acl;
 
-	ops->ruleset_unbind(mlxsw_sp, ruleset->priv);
+	if (!ruleset->ht_key.chain_index)
+		ops->ruleset_unbind(mlxsw_sp, ruleset->priv);
 	rhashtable_remove_fast(&acl->ruleset_ht, &ruleset->ht_node,
 			       mlxsw_sp_acl_ruleset_ht_params);
 }
@@ -211,14 +221,48 @@ static void mlxsw_sp_acl_ruleset_ref_dec(struct mlxsw_sp *mlxsw_sp,
 	mlxsw_sp_acl_ruleset_destroy(mlxsw_sp, ruleset);
 }
 
+static struct mlxsw_sp_acl_ruleset *
+__mlxsw_sp_acl_ruleset_lookup(struct mlxsw_sp_acl *acl, struct net_device *dev,
+			      bool ingress, u32 chain_index,
+			      const struct mlxsw_sp_acl_profile_ops *ops)
+{
+	struct mlxsw_sp_acl_ruleset_ht_key ht_key;
+
+	memset(&ht_key, 0, sizeof(ht_key));
+	ht_key.dev = dev;
+	ht_key.ingress = ingress;
+	ht_key.chain_index = chain_index;
+	ht_key.ops = ops;
+	return rhashtable_lookup_fast(&acl->ruleset_ht, &ht_key,
+				      mlxsw_sp_acl_ruleset_ht_params);
+}
+
 struct mlxsw_sp_acl_ruleset *
-mlxsw_sp_acl_ruleset_get(struct mlxsw_sp *mlxsw_sp,
-			 struct net_device *dev, bool ingress,
+mlxsw_sp_acl_ruleset_lookup(struct mlxsw_sp *mlxsw_sp, struct net_device *dev,
+			    bool ingress, u32 chain_index,
+			    enum mlxsw_sp_acl_profile profile)
+{
+	const struct mlxsw_sp_acl_profile_ops *ops;
+	struct mlxsw_sp_acl *acl = mlxsw_sp->acl;
+	struct mlxsw_sp_acl_ruleset *ruleset;
+
+	ops = acl->ops->profile_ops(mlxsw_sp, profile);
+	if (!ops)
+		return ERR_PTR(-EINVAL);
+	ruleset = __mlxsw_sp_acl_ruleset_lookup(acl, dev, ingress,
+						chain_index, ops);
+	if (!ruleset)
+		return ERR_PTR(-ENOENT);
+	return ruleset;
+}
+
+struct mlxsw_sp_acl_ruleset *
+mlxsw_sp_acl_ruleset_get(struct mlxsw_sp *mlxsw_sp, struct net_device *dev,
+			 bool ingress, u32 chain_index,
 			 enum mlxsw_sp_acl_profile profile)
 {
 	const struct mlxsw_sp_acl_profile_ops *ops;
 	struct mlxsw_sp_acl *acl = mlxsw_sp->acl;
-	struct mlxsw_sp_acl_ruleset_ht_key ht_key;
 	struct mlxsw_sp_acl_ruleset *ruleset;
 	int err;
 
@@ -226,12 +270,8 @@ mlxsw_sp_acl_ruleset_get(struct mlxsw_sp *mlxsw_sp,
 	if (!ops)
 		return ERR_PTR(-EINVAL);
 
-	memset(&ht_key, 0, sizeof(ht_key));
-	ht_key.dev = dev;
-	ht_key.ingress = ingress;
-	ht_key.ops = ops;
-	ruleset = rhashtable_lookup_fast(&acl->ruleset_ht, &ht_key,
-					 mlxsw_sp_acl_ruleset_ht_params);
+	ruleset = __mlxsw_sp_acl_ruleset_lookup(acl, dev, ingress,
+						chain_index, ops);
 	if (ruleset) {
 		mlxsw_sp_acl_ruleset_ref_inc(ruleset);
 		return ruleset;
@@ -239,7 +279,8 @@ mlxsw_sp_acl_ruleset_get(struct mlxsw_sp *mlxsw_sp,
 	ruleset = mlxsw_sp_acl_ruleset_create(mlxsw_sp, ops);
 	if (IS_ERR(ruleset))
 		return ruleset;
-	err = mlxsw_sp_acl_ruleset_bind(mlxsw_sp, ruleset, dev, ingress);
+	err = mlxsw_sp_acl_ruleset_bind(mlxsw_sp, ruleset, dev,
+					ingress, chain_index);
 	if (err)
 		goto err_ruleset_bind;
 	return ruleset;
@@ -255,6 +296,13 @@ void mlxsw_sp_acl_ruleset_put(struct mlxsw_sp *mlxsw_sp,
 	mlxsw_sp_acl_ruleset_ref_dec(mlxsw_sp, ruleset);
 }
 
+u16 mlxsw_sp_acl_ruleset_group_id(struct mlxsw_sp_acl_ruleset *ruleset)
+{
+	const struct mlxsw_sp_acl_profile_ops *ops = ruleset->ht_key.ops;
+
+	return ops->ruleset_group_id(ruleset->priv);
+}
+
 static int
 mlxsw_sp_acl_rulei_counter_alloc(struct mlxsw_sp *mlxsw_sp,
 				 struct mlxsw_sp_acl_rule_info *rulei)
@@ -369,7 +417,7 @@ int mlxsw_sp_acl_rulei_act_fwd(struct mlxsw_sp *mlxsw_sp,
 		local_port = mlxsw_sp_port->local_port;
 		in_port = false;
 	} else {
-		/* If out_dev is NULL, the called wants to
+		/* If out_dev is NULL, the caller wants to
 		 * set forward to ingress port.
 		 */
 		local_port = 0;
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_keys.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_keys.h
index 85d5001a5818..fb8031828454 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_keys.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_keys.h
@@ -70,6 +70,9 @@ static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_ipv4_dip[] = {
 
 static struct mlxsw_afk_element_inst mlxsw_sp_afk_element_info_ipv4[] = {
 	MLXSW_AFK_ELEMENT_INST_U32(SRC_IP4, 0x00, 0, 32),
+	MLXSW_AFK_ELEMENT_INST_U32(IP_ECN, 0x04, 4, 2),
+	MLXSW_AFK_ELEMENT_INST_U32(IP_TTL_, 0x04, 24, 8),
+	MLXSW_AFK_ELEMENT_INST_U32(IP_DSCP, 0x08, 0, 6),
 	MLXSW_AFK_ELEMENT_INST_U32(TCP_FLAGS, 0x08, 8, 9), /* TCP_CONTROL+TCP_ECN */
 };
 
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.c
index 61a10f166f97..50b40de1fb91 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.c
@@ -295,6 +295,12 @@ mlxsw_sp_acl_tcam_group_unbind(struct mlxsw_sp *mlxsw_sp,
 	mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ppbt), ppbt_pl);
 }
 
+static u16
+mlxsw_sp_acl_tcam_group_id(struct mlxsw_sp_acl_tcam_group *group)
+{
+	return group->id;
+}
+
 static unsigned int
 mlxsw_sp_acl_tcam_region_prio(struct mlxsw_sp_acl_tcam_region *region)
 {
@@ -984,6 +990,9 @@ static const enum mlxsw_afk_element mlxsw_sp_acl_tcam_pattern_ipv4[] = {
 	MLXSW_AFK_ELEMENT_VID,
 	MLXSW_AFK_ELEMENT_PCP,
 	MLXSW_AFK_ELEMENT_TCP_FLAGS,
+	MLXSW_AFK_ELEMENT_IP_TTL_,
+	MLXSW_AFK_ELEMENT_IP_ECN,
+	MLXSW_AFK_ELEMENT_IP_DSCP,
 };
 
 static const enum mlxsw_afk_element mlxsw_sp_acl_tcam_pattern_ipv6[] = {
@@ -1060,6 +1069,14 @@ mlxsw_sp_acl_tcam_flower_ruleset_unbind(struct mlxsw_sp *mlxsw_sp,
 	mlxsw_sp_acl_tcam_group_unbind(mlxsw_sp, &ruleset->group);
 }
 
+static u16
+mlxsw_sp_acl_tcam_flower_ruleset_group_id(void *ruleset_priv)
+{
+	struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;
+
+	return mlxsw_sp_acl_tcam_group_id(&ruleset->group);
+}
+
 static int
 mlxsw_sp_acl_tcam_flower_rule_add(struct mlxsw_sp *mlxsw_sp,
 				  void *ruleset_priv, void *rule_priv,
@@ -1096,6 +1113,7 @@ static const struct mlxsw_sp_acl_profile_ops mlxsw_sp_acl_tcam_flower_ops = {
 	.ruleset_del		= mlxsw_sp_acl_tcam_flower_ruleset_del,
 	.ruleset_bind		= mlxsw_sp_acl_tcam_flower_ruleset_bind,
 	.ruleset_unbind		= mlxsw_sp_acl_tcam_flower_ruleset_unbind,
+	.ruleset_group_id	= mlxsw_sp_acl_tcam_flower_ruleset_group_id,
 	.rule_priv_size		= sizeof(struct mlxsw_sp_acl_tcam_flower_rule),
 	.rule_add		= mlxsw_sp_acl_tcam_flower_rule_add,
 	.rule_del		= mlxsw_sp_acl_tcam_flower_rule_del,
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_dpipe.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_dpipe.c
index af2c65a3fd9f..51e6846da72b 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_dpipe.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_dpipe.c
@@ -74,6 +74,9 @@ static struct devlink_dpipe_header mlxsw_sp_dpipe_header_metadata = {
 
 static struct devlink_dpipe_header *mlxsw_dpipe_headers[] = {
 	&mlxsw_sp_dpipe_header_metadata,
+	&devlink_dpipe_header_ethernet,
+	&devlink_dpipe_header_ipv4,
+	&devlink_dpipe_header_ipv6,
 };
 
 static struct devlink_dpipe_headers mlxsw_sp_dpipe_headers = {
@@ -114,26 +117,6 @@ static int mlxsw_sp_dpipe_table_erif_matches_dump(void *priv,
 	return devlink_dpipe_match_put(skb, &match);
 }
 
-static void mlxsw_sp_erif_entry_clear(struct devlink_dpipe_entry *entry)
-{
-	unsigned int value_count, value_index;
-	struct devlink_dpipe_value *value;
-
-	value = entry->action_values;
-	value_count = entry->action_values_count;
-	for (value_index = 0; value_index < value_count; value_index++) {
-		kfree(value[value_index].value);
-		kfree(value[value_index].mask);
-	}
-
-	value = entry->match_values;
-	value_count = entry->match_values_count;
-	for (value_index = 0; value_index < value_count; value_index++) {
-		kfree(value[value_index].value);
-		kfree(value[value_index].mask);
-	}
-}
-
 static void
 mlxsw_sp_erif_match_action_prepare(struct devlink_dpipe_match *match,
 				   struct devlink_dpipe_action *action)
@@ -215,8 +198,8 @@ static int mlxsw_sp_erif_entry_get(struct mlxsw_sp *mlxsw_sp,
 }
 
 static int
-mlxsw_sp_table_erif_entries_dump(void *priv, bool counters_enabled,
-				 struct devlink_dpipe_dump_ctx *dump_ctx)
+mlxsw_sp_dpipe_table_erif_entries_dump(void *priv, bool counters_enabled,
+				       struct devlink_dpipe_dump_ctx *dump_ctx)
 {
 	struct devlink_dpipe_value match_value, action_value;
 	struct devlink_dpipe_action action = {0};
@@ -270,16 +253,16 @@ start_again:
 		goto start_again;
 	rtnl_unlock();
 
-	mlxsw_sp_erif_entry_clear(&entry);
+	devlink_dpipe_entry_clear(&entry);
 	return 0;
 err_entry_append:
 err_entry_get:
 	rtnl_unlock();
-	mlxsw_sp_erif_entry_clear(&entry);
+	devlink_dpipe_entry_clear(&entry);
 	return err;
 }
 
-static int mlxsw_sp_table_erif_counters_update(void *priv, bool enable)
+static int mlxsw_sp_dpipe_table_erif_counters_update(void *priv, bool enable)
 {
 	struct mlxsw_sp *mlxsw_sp = priv;
 	int i;
@@ -301,24 +284,29 @@ static int mlxsw_sp_table_erif_counters_update(void *priv, bool enable)
 	return 0;
 }
 
+static u64 mlxsw_sp_dpipe_table_erif_size_get(void *priv)
+{
+	struct mlxsw_sp *mlxsw_sp = priv;
+
+	return MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
+}
+
 static struct devlink_dpipe_table_ops mlxsw_sp_erif_ops = {
 	.matches_dump = mlxsw_sp_dpipe_table_erif_matches_dump,
 	.actions_dump = mlxsw_sp_dpipe_table_erif_actions_dump,
-	.entries_dump = mlxsw_sp_table_erif_entries_dump,
-	.counters_set_update = mlxsw_sp_table_erif_counters_update,
+	.entries_dump = mlxsw_sp_dpipe_table_erif_entries_dump,
+	.counters_set_update = mlxsw_sp_dpipe_table_erif_counters_update,
+	.size_get = mlxsw_sp_dpipe_table_erif_size_get,
 };
 
 static int mlxsw_sp_dpipe_erif_table_init(struct mlxsw_sp *mlxsw_sp)
 {
 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
-	u64 table_size;
 
-	table_size = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
 	return devlink_dpipe_table_register(devlink,
 					    MLXSW_SP_DPIPE_TABLE_NAME_ERIF,
 					    &mlxsw_sp_erif_ops,
-					    mlxsw_sp, table_size,
-					    false);
+					    mlxsw_sp, false);
 }
 
 static void mlxsw_sp_dpipe_erif_table_fini(struct mlxsw_sp *mlxsw_sp)
@@ -328,6 +316,516 @@ static void mlxsw_sp_dpipe_erif_table_fini(struct mlxsw_sp *mlxsw_sp)
 	devlink_dpipe_table_unregister(devlink, MLXSW_SP_DPIPE_TABLE_NAME_ERIF);
 }
 
+static int mlxsw_sp_dpipe_table_host_matches_dump(struct sk_buff *skb, int type)
+{
+	struct devlink_dpipe_match match = {0};
+	int err;
+
+	match.type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
+	match.header = &mlxsw_sp_dpipe_header_metadata;
+	match.field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT;
+
+	err = devlink_dpipe_match_put(skb, &match);
+	if (err)
+		return err;
+
+	switch (type) {
+	case AF_INET:
+		match.type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
+		match.header = &devlink_dpipe_header_ipv4;
+		match.field_id = DEVLINK_DPIPE_FIELD_IPV4_DST_IP;
+		break;
+	case AF_INET6:
+		match.type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
+		match.header = &devlink_dpipe_header_ipv6;
+		match.field_id = DEVLINK_DPIPE_FIELD_IPV6_DST_IP;
+		break;
+	default:
+		WARN_ON(1);
+		return -EINVAL;
+	}
+
+	return devlink_dpipe_match_put(skb, &match);
+}
+
+static int
+mlxsw_sp_dpipe_table_host4_matches_dump(void *priv, struct sk_buff *skb)
+{
+	return mlxsw_sp_dpipe_table_host_matches_dump(skb, AF_INET);
+}
+
+static int
+mlxsw_sp_dpipe_table_host_actions_dump(void *priv, struct sk_buff *skb)
+{
+	struct devlink_dpipe_action action = {0};
+
+	action.type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
+	action.header = &devlink_dpipe_header_ethernet;
+	action.field_id = DEVLINK_DPIPE_FIELD_ETHERNET_DST_MAC;
+
+	return devlink_dpipe_action_put(skb, &action);
+}
+
+enum mlxsw_sp_dpipe_table_host_match {
+	MLXSW_SP_DPIPE_TABLE_HOST_MATCH_RIF,
+	MLXSW_SP_DPIPE_TABLE_HOST_MATCH_DIP,
+	MLXSW_SP_DPIPE_TABLE_HOST_MATCH_COUNT,
+};
+
+static void
+mlxsw_sp_dpipe_table_host_match_action_prepare(struct devlink_dpipe_match *matches,
+					       struct devlink_dpipe_action *action,
+					       int type)
+{
+	struct devlink_dpipe_match *match;
+
+	match = &matches[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_RIF];
+	match->type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
+	match->header = &mlxsw_sp_dpipe_header_metadata;
+	match->field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT;
+
+	match = &matches[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_DIP];
+	match->type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
+	switch (type) {
+	case AF_INET:
+		match->header = &devlink_dpipe_header_ipv4;
+		match->field_id = DEVLINK_DPIPE_FIELD_IPV4_DST_IP;
+		break;
+	case AF_INET6:
+		match->header = &devlink_dpipe_header_ipv6;
+		match->field_id = DEVLINK_DPIPE_FIELD_IPV6_DST_IP;
+		break;
+	default:
+		WARN_ON(1);
+		return;
+	}
+
+	action->type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
+	action->header = &devlink_dpipe_header_ethernet;
+	action->field_id = DEVLINK_DPIPE_FIELD_ETHERNET_DST_MAC;
+}
+
+static int
+mlxsw_sp_dpipe_table_host_entry_prepare(struct devlink_dpipe_entry *entry,
+					struct devlink_dpipe_value *match_values,
+					struct devlink_dpipe_match *matches,
+					struct devlink_dpipe_value *action_value,
+					struct devlink_dpipe_action *action,
+					int type)
+{
+	struct devlink_dpipe_value *match_value;
+	struct devlink_dpipe_match *match;
+
+	entry->match_values = match_values;
+	entry->match_values_count = MLXSW_SP_DPIPE_TABLE_HOST_MATCH_COUNT;
+
+	entry->action_values = action_value;
+	entry->action_values_count = 1;
+
+	match = &matches[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_RIF];
+	match_value = &match_values[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_RIF];
+
+	match_value->match = match;
+	match_value->value_size = sizeof(u32);
+	match_value->value = kmalloc(match_value->value_size, GFP_KERNEL);
+	if (!match_value->value)
+		return -ENOMEM;
+
+	match = &matches[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_DIP];
+	match_value = &match_values[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_DIP];
+
+	match_value->match = match;
+	switch (type) {
+	case AF_INET:
+		match_value->value_size = sizeof(u32);
+		break;
+	case AF_INET6:
+		match_value->value_size = sizeof(struct in6_addr);
+		break;
+	default:
+		WARN_ON(1);
+		return -EINVAL;
+	}
+
+	match_value->value = kmalloc(match_value->value_size, GFP_KERNEL);
+	if (!match_value->value)
+		return -ENOMEM;
+
+	action_value->action = action;
+	action_value->value_size = sizeof(u64);
+	action_value->value = kmalloc(action_value->value_size, GFP_KERNEL);
+	if (!action_value->value)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void
+__mlxsw_sp_dpipe_table_host_entry_fill(struct devlink_dpipe_entry *entry,
+				       struct mlxsw_sp_rif *rif,
+				       unsigned char *ha, void *dip)
+{
+	struct devlink_dpipe_value *value;
+	u32 *rif_value;
+	u8 *ha_value;
+
+	/* Set Match RIF index */
+	value = &entry->match_values[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_RIF];
+
+	rif_value = value->value;
+	*rif_value = mlxsw_sp_rif_index(rif);
+	value->mapping_value = mlxsw_sp_rif_dev_ifindex(rif);
+	value->mapping_valid = true;
+
+	/* Set Match DIP */
+	value = &entry->match_values[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_DIP];
+	memcpy(value->value, dip, value->value_size);
+
+	/* Set Action DMAC */
+	value = entry->action_values;
+	ha_value = value->value;
+	ether_addr_copy(ha_value, ha);
+}
+
+static void
+mlxsw_sp_dpipe_table_host4_entry_fill(struct devlink_dpipe_entry *entry,
+				      struct mlxsw_sp_neigh_entry *neigh_entry,
+				      struct mlxsw_sp_rif *rif)
+{
+	unsigned char *ha;
+	u32 dip;
+
+	ha = mlxsw_sp_neigh_entry_ha(neigh_entry);
+	dip = mlxsw_sp_neigh4_entry_dip(neigh_entry);
+	__mlxsw_sp_dpipe_table_host_entry_fill(entry, rif, ha, &dip);
+}
+
+static void
+mlxsw_sp_dpipe_table_host6_entry_fill(struct devlink_dpipe_entry *entry,
+				      struct mlxsw_sp_neigh_entry *neigh_entry,
+				      struct mlxsw_sp_rif *rif)
+{
+	struct in6_addr *dip;
+	unsigned char *ha;
+
+	ha = mlxsw_sp_neigh_entry_ha(neigh_entry);
+	dip = mlxsw_sp_neigh6_entry_dip(neigh_entry);
+
+	__mlxsw_sp_dpipe_table_host_entry_fill(entry, rif, ha, dip);
+}
+
+static void
+mlxsw_sp_dpipe_table_host_entry_fill(struct mlxsw_sp *mlxsw_sp,
+				     struct devlink_dpipe_entry *entry,
+				     struct mlxsw_sp_neigh_entry *neigh_entry,
+				     struct mlxsw_sp_rif *rif,
+				     int type)
+{
+	int err;
+
+	switch (type) {
+	case AF_INET:
+		mlxsw_sp_dpipe_table_host4_entry_fill(entry, neigh_entry, rif);
+		break;
+	case AF_INET6:
+		mlxsw_sp_dpipe_table_host6_entry_fill(entry, neigh_entry, rif);
+		break;
+	default:
+		WARN_ON(1);
+		return;
+	}
+
+	err = mlxsw_sp_neigh_counter_get(mlxsw_sp, neigh_entry,
+					 &entry->counter);
+	if (!err)
+		entry->counter_valid = true;
+}
+
+static int
+mlxsw_sp_dpipe_table_host_entries_get(struct mlxsw_sp *mlxsw_sp,
+				      struct devlink_dpipe_entry *entry,
+				      bool counters_enabled,
+				      struct devlink_dpipe_dump_ctx *dump_ctx,
+				      int type)
+{
+	int rif_neigh_count = 0;
+	int rif_neigh_skip = 0;
+	int neigh_count = 0;
+	int rif_count;
+	int i, j;
+	int err;
+
+	rtnl_lock();
+	i = 0;
+	rif_count = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
+start_again:
+	err = devlink_dpipe_entry_ctx_prepare(dump_ctx);
+	if (err)
+		goto err_ctx_prepare;
+	j = 0;
+	rif_neigh_skip = rif_neigh_count;
+	for (; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
+		struct mlxsw_sp_rif *rif = mlxsw_sp_rif_by_index(mlxsw_sp, i);
+		struct mlxsw_sp_neigh_entry *neigh_entry;
+
+		if (!rif)
+			continue;
+
+		rif_neigh_count = 0;
+		mlxsw_sp_rif_neigh_for_each(neigh_entry, rif) {
+			int neigh_type = mlxsw_sp_neigh_entry_type(neigh_entry);
+
+			if (neigh_type != type)
+				continue;
+
+			if (neigh_type == AF_INET6 &&
+			    mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
+				continue;
+
+			if (rif_neigh_count < rif_neigh_skip)
+				goto skip;
+
+			mlxsw_sp_dpipe_table_host_entry_fill(mlxsw_sp, entry,
+							     neigh_entry, rif,
+							     type);
+			entry->index = neigh_count;
+			err = devlink_dpipe_entry_ctx_append(dump_ctx, entry);
+			if (err) {
+				if (err == -EMSGSIZE) {
+					if (!j)
+						goto err_entry_append;
+					else
+						goto out;
+				}
+				goto err_entry_append;
+			}
+			neigh_count++;
+			j++;
+skip:
+			rif_neigh_count++;
+		}
+		rif_neigh_skip = 0;
+	}
+out:
+	devlink_dpipe_entry_ctx_close(dump_ctx);
+	if (i != rif_count)
+		goto start_again;
+
+	rtnl_unlock();
+	return 0;
+
+err_ctx_prepare:
+err_entry_append:
+	rtnl_unlock();
+	return err;
+}
+
+static int
+mlxsw_sp_dpipe_table_host_entries_dump(struct mlxsw_sp *mlxsw_sp,
+				       bool counters_enabled,
+				       struct devlink_dpipe_dump_ctx *dump_ctx,
+				       int type)
+{
+	struct devlink_dpipe_value match_values[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_COUNT];
+	struct devlink_dpipe_match matches[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_COUNT];
+	struct devlink_dpipe_value action_value;
+	struct devlink_dpipe_action action = {0};
+	struct devlink_dpipe_entry entry = {0};
+	int err;
+
+	memset(matches, 0, MLXSW_SP_DPIPE_TABLE_HOST_MATCH_COUNT *
+			   sizeof(matches[0]));
+	memset(match_values, 0, MLXSW_SP_DPIPE_TABLE_HOST_MATCH_COUNT *
+				sizeof(match_values[0]));
+	memset(&action_value, 0, sizeof(action_value));
+
+	mlxsw_sp_dpipe_table_host_match_action_prepare(matches, &action, type);
+	err = mlxsw_sp_dpipe_table_host_entry_prepare(&entry, match_values,
+						      matches, &action_value,
+						      &action, type);
+	if (err)
+		goto out;
+
+	err = mlxsw_sp_dpipe_table_host_entries_get(mlxsw_sp, &entry,
+						    counters_enabled, dump_ctx,
+						    type);
+out:
+	devlink_dpipe_entry_clear(&entry);
+	return err;
+}
+
+static int
+mlxsw_sp_dpipe_table_host4_entries_dump(void *priv, bool counters_enabled,
+					struct devlink_dpipe_dump_ctx *dump_ctx)
+{
+	struct mlxsw_sp *mlxsw_sp = priv;
+
+	return mlxsw_sp_dpipe_table_host_entries_dump(mlxsw_sp,
+						      counters_enabled,
+						      dump_ctx, AF_INET);
+}
+
+static void
+mlxsw_sp_dpipe_table_host_counters_update(struct mlxsw_sp *mlxsw_sp,
+					  bool enable, int type)
+{
+	int i;
+
+	rtnl_lock();
+	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
+		struct mlxsw_sp_rif *rif = mlxsw_sp_rif_by_index(mlxsw_sp, i);
+		struct mlxsw_sp_neigh_entry *neigh_entry;
+
+		if (!rif)
+			continue;
+		mlxsw_sp_rif_neigh_for_each(neigh_entry, rif) {
+			int neigh_type = mlxsw_sp_neigh_entry_type(neigh_entry);
+
+			if (neigh_type != type)
+				continue;
+
+			if (neigh_type == AF_INET6 &&
+			    mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
+				continue;
+
+			mlxsw_sp_neigh_entry_counter_update(mlxsw_sp,
+							    neigh_entry,
+							    enable);
+		}
+	}
+	rtnl_unlock();
+}
+
+static int mlxsw_sp_dpipe_table_host4_counters_update(void *priv, bool enable)
+{
+	struct mlxsw_sp *mlxsw_sp = priv;
+
+	mlxsw_sp_dpipe_table_host_counters_update(mlxsw_sp, enable, AF_INET);
+	return 0;
+}
+
+static u64
+mlxsw_sp_dpipe_table_host_size_get(struct mlxsw_sp *mlxsw_sp, int type)
+{
+	u64 size = 0;
+	int i;
+
+	rtnl_lock();
+	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
+		struct mlxsw_sp_rif *rif = mlxsw_sp_rif_by_index(mlxsw_sp, i);
+		struct mlxsw_sp_neigh_entry *neigh_entry;
+
+		if (!rif)
+			continue;
+		mlxsw_sp_rif_neigh_for_each(neigh_entry, rif) {
+			int neigh_type = mlxsw_sp_neigh_entry_type(neigh_entry);
+
+			if (neigh_type != type)
+				continue;
+
+			if (neigh_type == AF_INET6 &&
+			    mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
+				continue;
+
+			size++;
+		}
+	}
+	rtnl_unlock();
+
+	return size;
+}
+
+static u64 mlxsw_sp_dpipe_table_host4_size_get(void *priv)
+{
+	struct mlxsw_sp *mlxsw_sp = priv;
+
+	return mlxsw_sp_dpipe_table_host_size_get(mlxsw_sp, AF_INET);
+}
+
+static struct devlink_dpipe_table_ops mlxsw_sp_host4_ops = {
+	.matches_dump = mlxsw_sp_dpipe_table_host4_matches_dump,
+	.actions_dump = mlxsw_sp_dpipe_table_host_actions_dump,
+	.entries_dump = mlxsw_sp_dpipe_table_host4_entries_dump,
+	.counters_set_update = mlxsw_sp_dpipe_table_host4_counters_update,
+	.size_get = mlxsw_sp_dpipe_table_host4_size_get,
+};
+
+static int mlxsw_sp_dpipe_host4_table_init(struct mlxsw_sp *mlxsw_sp)
+{
+	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
+
+	return devlink_dpipe_table_register(devlink,
+					    MLXSW_SP_DPIPE_TABLE_NAME_HOST4,
+					    &mlxsw_sp_host4_ops,
+					    mlxsw_sp, false);
+}
+
+static void mlxsw_sp_dpipe_host4_table_fini(struct mlxsw_sp *mlxsw_sp)
+{
+	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
+
+	devlink_dpipe_table_unregister(devlink,
+				       MLXSW_SP_DPIPE_TABLE_NAME_HOST4);
+}
+
+static int
+mlxsw_sp_dpipe_table_host6_matches_dump(void *priv, struct sk_buff *skb)
+{
+	return mlxsw_sp_dpipe_table_host_matches_dump(skb, AF_INET6);
+}
+
+static int
+mlxsw_sp_dpipe_table_host6_entries_dump(void *priv, bool counters_enabled,
+					struct devlink_dpipe_dump_ctx *dump_ctx)
+{
+	struct mlxsw_sp *mlxsw_sp = priv;
+
+	return mlxsw_sp_dpipe_table_host_entries_dump(mlxsw_sp,
+						      counters_enabled,
+						      dump_ctx, AF_INET6);
+}
+
+static int mlxsw_sp_dpipe_table_host6_counters_update(void *priv, bool enable)
+{
+	struct mlxsw_sp *mlxsw_sp = priv;
+
+	mlxsw_sp_dpipe_table_host_counters_update(mlxsw_sp, enable, AF_INET6);
+	return 0;
+}
+
+static u64 mlxsw_sp_dpipe_table_host6_size_get(void *priv)
+{
+	struct mlxsw_sp *mlxsw_sp = priv;
+
+	return mlxsw_sp_dpipe_table_host_size_get(mlxsw_sp, AF_INET6);
+}
+
+static struct devlink_dpipe_table_ops mlxsw_sp_host6_ops = {
+	.matches_dump = mlxsw_sp_dpipe_table_host6_matches_dump,
+	.actions_dump = mlxsw_sp_dpipe_table_host_actions_dump,
+	.entries_dump = mlxsw_sp_dpipe_table_host6_entries_dump,
+	.counters_set_update = mlxsw_sp_dpipe_table_host6_counters_update,
+	.size_get = mlxsw_sp_dpipe_table_host6_size_get,
+};
+
+static int mlxsw_sp_dpipe_host6_table_init(struct mlxsw_sp *mlxsw_sp)
+{
+	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
+
+	return devlink_dpipe_table_register(devlink,
+					    MLXSW_SP_DPIPE_TABLE_NAME_HOST6,
+					    &mlxsw_sp_host6_ops,
+					    mlxsw_sp, false);
+}
+
+static void mlxsw_sp_dpipe_host6_table_fini(struct mlxsw_sp *mlxsw_sp)
+{
+	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
+
+	devlink_dpipe_table_unregister(devlink,
+				       MLXSW_SP_DPIPE_TABLE_NAME_HOST6);
+}
+
 int mlxsw_sp_dpipe_init(struct mlxsw_sp *mlxsw_sp)
 {
 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
@@ -339,10 +837,22 @@ int mlxsw_sp_dpipe_init(struct mlxsw_sp *mlxsw_sp)
 		return err;
 	err = mlxsw_sp_dpipe_erif_table_init(mlxsw_sp);
 	if (err)
-		goto err_erif_register;
+		goto err_erif_table_init;
+
+	err = mlxsw_sp_dpipe_host4_table_init(mlxsw_sp);
+	if (err)
+		goto err_host4_table_init;
+
+	err = mlxsw_sp_dpipe_host6_table_init(mlxsw_sp);
+	if (err)
+		goto err_host6_table_init;
 	return 0;
 
-err_erif_register:
+err_host6_table_init:
+	mlxsw_sp_dpipe_host4_table_fini(mlxsw_sp);
+err_host4_table_init:
+	mlxsw_sp_dpipe_erif_table_fini(mlxsw_sp);
+err_erif_table_init:
 	devlink_dpipe_headers_unregister(priv_to_devlink(mlxsw_sp->core));
 	return err;
 }
@@ -351,6 +861,8 @@ void mlxsw_sp_dpipe_fini(struct mlxsw_sp *mlxsw_sp)
 {
 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
 
+	mlxsw_sp_dpipe_host6_table_fini(mlxsw_sp);
+	mlxsw_sp_dpipe_host4_table_fini(mlxsw_sp);
 	mlxsw_sp_dpipe_erif_table_fini(mlxsw_sp);
 	devlink_dpipe_headers_unregister(devlink);
 }
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_dpipe.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum_dpipe.h
index d2089298cba3..283fde4e6783 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_dpipe.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_dpipe.h
@@ -35,9 +35,26 @@
 #ifndef _MLXSW_PIPELINE_H_
 #define _MLXSW_PIPELINE_H_
 
+#if IS_ENABLED(CONFIG_NET_DEVLINK)
+
 int mlxsw_sp_dpipe_init(struct mlxsw_sp *mlxsw_sp);
 void mlxsw_sp_dpipe_fini(struct mlxsw_sp *mlxsw_sp);
 
+#else
+
+static inline int mlxsw_sp_dpipe_init(struct mlxsw_sp *mlxsw_sp)
+{
+	return 0;
+}
+
+static inline void mlxsw_sp_dpipe_fini(struct mlxsw_sp *mlxsw_sp)
+{
+}
+
+#endif
+
 #define MLXSW_SP_DPIPE_TABLE_NAME_ERIF "mlxsw_erif"
+#define MLXSW_SP_DPIPE_TABLE_NAME_HOST4 "mlxsw_host4"
+#define MLXSW_SP_DPIPE_TABLE_NAME_HOST6 "mlxsw_host6"
 
 #endif /* _MLXSW_PIPELINE_H_*/
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_fid.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_fid.c
index 6afbe9ec64e2..bbd238e50f05 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_fid.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_fid.c
@@ -109,7 +109,6 @@ static const int mlxsw_sp_sfgc_uc_packet_types[MLXSW_REG_SFGC_TYPE_MAX] = {
 
 static const int mlxsw_sp_sfgc_bc_packet_types[MLXSW_REG_SFGC_TYPE_MAX] = {
 	[MLXSW_REG_SFGC_TYPE_BROADCAST]				= 1,
-	[MLXSW_REG_SFGC_TYPE_UNREGISTERED_MULTICAST_IPV6]	= 1,
 	[MLXSW_REG_SFGC_TYPE_UNREGISTERED_MULTICAST_NON_IP]	= 1,
 	[MLXSW_REG_SFGC_TYPE_IPV4_LINK_LOCAL]			= 1,
 	[MLXSW_REG_SFGC_TYPE_IPV6_ALL_HOST]			= 1,
@@ -117,6 +116,7 @@ static const int mlxsw_sp_sfgc_bc_packet_types[MLXSW_REG_SFGC_TYPE_MAX] = {
 
 static const int mlxsw_sp_sfgc_mc_packet_types[MLXSW_REG_SFGC_TYPE_MAX] = {
 	[MLXSW_REG_SFGC_TYPE_UNREGISTERED_MULTICAST_IPV4]	= 1,
+	[MLXSW_REG_SFGC_TYPE_UNREGISTERED_MULTICAST_IPV6]	= 1,
 };
 
 static const int *mlxsw_sp_packet_type_sfgc_types[] = {
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c
index 21bb2bf62d3e..8aace9a06a5d 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c
@@ -45,7 +45,7 @@
 #include "core_acl_flex_keys.h"
 
 static int mlxsw_sp_flower_parse_actions(struct mlxsw_sp *mlxsw_sp,
-					 struct net_device *dev,
+					 struct net_device *dev, bool ingress,
 					 struct mlxsw_sp_acl_rule_info *rulei,
 					 struct tcf_exts *exts)
 {
@@ -53,7 +53,7 @@ static int mlxsw_sp_flower_parse_actions(struct mlxsw_sp *mlxsw_sp,
 	LIST_HEAD(actions);
 	int err;
 
-	if (tc_no_actions(exts))
+	if (!tcf_exts_has_actions(exts))
 		return 0;
 
 	/* Count action is inserted first */
@@ -71,6 +71,20 @@ static int mlxsw_sp_flower_parse_actions(struct mlxsw_sp *mlxsw_sp,
 			err = mlxsw_sp_acl_rulei_act_trap(rulei);
 			if (err)
 				return err;
+		} else if (is_tcf_gact_goto_chain(a)) {
+			u32 chain_index = tcf_gact_goto_chain_index(a);
+			struct mlxsw_sp_acl_ruleset *ruleset;
+			u16 group_id;
+
+			ruleset = mlxsw_sp_acl_ruleset_lookup(mlxsw_sp, dev,
+							      ingress,
+							      chain_index,
+							      MLXSW_SP_ACL_PROFILE_FLOWER);
+			if (IS_ERR(ruleset))
+				return PTR_ERR(ruleset);
+
+			group_id = mlxsw_sp_acl_ruleset_group_id(ruleset);
+			mlxsw_sp_acl_rulei_act_jump(rulei, group_id);
 		} else if (is_tcf_mirred_egress_redirect(a)) {
 			int ifindex = tcf_mirred_ifindex(a);
 			struct net_device *out_dev;
@@ -212,11 +226,46 @@ static int mlxsw_sp_flower_parse_tcp(struct mlxsw_sp *mlxsw_sp,
 	return 0;
 }
 
+static int mlxsw_sp_flower_parse_ip(struct mlxsw_sp *mlxsw_sp,
+				    struct mlxsw_sp_acl_rule_info *rulei,
+				    struct tc_cls_flower_offload *f,
+				    u16 n_proto)
+{
+	struct flow_dissector_key_ip *key, *mask;
+
+	if (!dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_IP))
+		return 0;
+
+	if (n_proto != ETH_P_IP && n_proto != ETH_P_IPV6) {
+		dev_err(mlxsw_sp->bus_info->dev, "IP keys supported only for IPv4/6\n");
+		return -EINVAL;
+	}
+
+	key = skb_flow_dissector_target(f->dissector,
+					FLOW_DISSECTOR_KEY_IP,
+					f->key);
+	mask = skb_flow_dissector_target(f->dissector,
+					 FLOW_DISSECTOR_KEY_IP,
+					 f->mask);
+	mlxsw_sp_acl_rulei_keymask_u32(rulei, MLXSW_AFK_ELEMENT_IP_TTL_,
+				       key->ttl, mask->ttl);
+
+	mlxsw_sp_acl_rulei_keymask_u32(rulei, MLXSW_AFK_ELEMENT_IP_ECN,
+				       key->tos & 0x3, mask->tos & 0x3);
+
+	mlxsw_sp_acl_rulei_keymask_u32(rulei, MLXSW_AFK_ELEMENT_IP_DSCP,
+				       key->tos >> 6, mask->tos >> 6);
+
+	return 0;
+}
+
 static int mlxsw_sp_flower_parse(struct mlxsw_sp *mlxsw_sp,
-				 struct net_device *dev,
+				 struct net_device *dev, bool ingress,
 				 struct mlxsw_sp_acl_rule_info *rulei,
 				 struct tc_cls_flower_offload *f)
 {
+	u16 n_proto_mask = 0;
+	u16 n_proto_key = 0;
 	u16 addr_type = 0;
 	u8 ip_proto = 0;
 	int err;
@@ -229,12 +278,13 @@ static int mlxsw_sp_flower_parse(struct mlxsw_sp *mlxsw_sp,
 	      BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
 	      BIT(FLOW_DISSECTOR_KEY_PORTS) |
 	      BIT(FLOW_DISSECTOR_KEY_TCP) |
+	      BIT(FLOW_DISSECTOR_KEY_IP) |
 	      BIT(FLOW_DISSECTOR_KEY_VLAN))) {
 		dev_err(mlxsw_sp->bus_info->dev, "Unsupported key\n");
 		return -EOPNOTSUPP;
 	}
 
-	mlxsw_sp_acl_rulei_priority(rulei, f->prio);
+	mlxsw_sp_acl_rulei_priority(rulei, f->common.prio);
 
 	if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_CONTROL)) {
 		struct flow_dissector_key_control *key =
@@ -253,8 +303,8 @@ static int mlxsw_sp_flower_parse(struct mlxsw_sp *mlxsw_sp,
 			skb_flow_dissector_target(f->dissector,
 						  FLOW_DISSECTOR_KEY_BASIC,
 						  f->mask);
-		u16 n_proto_key = ntohs(key->n_proto);
-		u16 n_proto_mask = ntohs(mask->n_proto);
+		n_proto_key = ntohs(key->n_proto);
+		n_proto_mask = ntohs(mask->n_proto);
 
 		if (n_proto_key == ETH_P_ALL) {
 			n_proto_key = 0;
@@ -324,11 +374,16 @@ static int mlxsw_sp_flower_parse(struct mlxsw_sp *mlxsw_sp,
 	if (err)
 		return err;
 
-	return mlxsw_sp_flower_parse_actions(mlxsw_sp, dev, rulei, f->exts);
+	err = mlxsw_sp_flower_parse_ip(mlxsw_sp, rulei, f, n_proto_key & n_proto_mask);
+	if (err)
+		return err;
+
+	return mlxsw_sp_flower_parse_actions(mlxsw_sp, dev, ingress,
+					     rulei, f->exts);
 }
 
 int mlxsw_sp_flower_replace(struct mlxsw_sp_port *mlxsw_sp_port, bool ingress,
-			    __be16 protocol, struct tc_cls_flower_offload *f)
+			    struct tc_cls_flower_offload *f)
 {
 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 	struct net_device *dev = mlxsw_sp_port->dev;
@@ -338,6 +393,7 @@ int mlxsw_sp_flower_replace(struct mlxsw_sp_port *mlxsw_sp_port, bool ingress,
 	int err;
 
 	ruleset = mlxsw_sp_acl_ruleset_get(mlxsw_sp, dev, ingress,
+					   f->common.chain_index,
 					   MLXSW_SP_ACL_PROFILE_FLOWER);
 	if (IS_ERR(ruleset))
 		return PTR_ERR(ruleset);
@@ -349,7 +405,7 @@ int mlxsw_sp_flower_replace(struct mlxsw_sp_port *mlxsw_sp_port, bool ingress,
 	}
 
 	rulei = mlxsw_sp_acl_rule_rulei(rule);
-	err = mlxsw_sp_flower_parse(mlxsw_sp, dev, rulei, f);
+	err = mlxsw_sp_flower_parse(mlxsw_sp, dev, ingress, rulei, f);
 	if (err)
 		goto err_flower_parse;
 
@@ -381,7 +437,7 @@ void mlxsw_sp_flower_destroy(struct mlxsw_sp_port *mlxsw_sp_port, bool ingress,
 	struct mlxsw_sp_acl_rule *rule;
 
 	ruleset = mlxsw_sp_acl_ruleset_get(mlxsw_sp, mlxsw_sp_port->dev,
-					   ingress,
+					   ingress, f->common.chain_index,
 					   MLXSW_SP_ACL_PROFILE_FLOWER);
 	if (IS_ERR(ruleset))
 		return;
@@ -407,7 +463,7 @@ int mlxsw_sp_flower_stats(struct mlxsw_sp_port *mlxsw_sp_port, bool ingress,
 	int err;
 
 	ruleset = mlxsw_sp_acl_ruleset_get(mlxsw_sp, mlxsw_sp_port->dev,
-					   ingress,
+					   ingress, f->common.chain_index,
 					   MLXSW_SP_ACL_PROFILE_FLOWER);
 	if (WARN_ON(IS_ERR(ruleset)))
 		return -EINVAL;
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ipip.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ipip.c
new file mode 100644
index 000000000000..702fe945227c
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ipip.c
@@ -0,0 +1,214 @@
+/*
+ * drivers/net/ethernet/mellanox/mlxsw/spectrum_ipip.c
+ * Copyright (c) 2017 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2017 Petr Machata <petrm@mellanox.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the names of the copyright holders nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <net/ip_tunnels.h>
+
+#include "spectrum_ipip.h"
+
+static bool
+mlxsw_sp_ipip_netdev_has_ikey(const struct net_device *ol_dev)
+{
+	struct ip_tunnel *tun = netdev_priv(ol_dev);
+
+	return !!(tun->parms.i_flags & TUNNEL_KEY);
+}
+
+static bool
+mlxsw_sp_ipip_netdev_has_okey(const struct net_device *ol_dev)
+{
+	struct ip_tunnel *tun = netdev_priv(ol_dev);
+
+	return !!(tun->parms.o_flags & TUNNEL_KEY);
+}
+
+static u32 mlxsw_sp_ipip_netdev_ikey(const struct net_device *ol_dev)
+{
+	struct ip_tunnel *tun = netdev_priv(ol_dev);
+
+	return mlxsw_sp_ipip_netdev_has_ikey(ol_dev) ?
+		be32_to_cpu(tun->parms.i_key) : 0;
+}
+
+static u32 mlxsw_sp_ipip_netdev_okey(const struct net_device *ol_dev)
+{
+	struct ip_tunnel *tun = netdev_priv(ol_dev);
+
+	return mlxsw_sp_ipip_netdev_has_okey(ol_dev) ?
+		be32_to_cpu(tun->parms.o_key) : 0;
+}
+
+static int
+mlxsw_sp_ipip_nexthop_update_gre4(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
+				  struct mlxsw_sp_ipip_entry *ipip_entry)
+{
+	u16 rif_index = mlxsw_sp_ipip_lb_rif_index(ipip_entry->ol_lb);
+	__be32 daddr4 = mlxsw_sp_ipip_netdev_daddr4(ipip_entry->ol_dev);
+	char ratr_pl[MLXSW_REG_RATR_LEN];
+
+	mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
+			    true, MLXSW_REG_RATR_TYPE_IPIP,
+			    adj_index, rif_index);
+	mlxsw_reg_ratr_ipip4_entry_pack(ratr_pl, be32_to_cpu(daddr4));
+
+	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
+}
+
+static int
+mlxsw_sp_ipip_fib_entry_op_gre4_rtdp(struct mlxsw_sp *mlxsw_sp,
+				     u32 tunnel_index,
+				     struct mlxsw_sp_ipip_entry *ipip_entry)
+{
+	bool has_ikey = mlxsw_sp_ipip_netdev_has_ikey(ipip_entry->ol_dev);
+	u16 rif_index = mlxsw_sp_ipip_lb_rif_index(ipip_entry->ol_lb);
+	u32 ikey = mlxsw_sp_ipip_netdev_ikey(ipip_entry->ol_dev);
+	char rtdp_pl[MLXSW_REG_RTDP_LEN];
+	unsigned int type_check;
+	u32 daddr4;
+
+	mlxsw_reg_rtdp_pack(rtdp_pl, MLXSW_REG_RTDP_TYPE_IPIP, tunnel_index);
+
+	type_check = has_ikey ?
+		MLXSW_REG_RTDP_IPIP_TYPE_CHECK_ALLOW_GRE_KEY :
+		MLXSW_REG_RTDP_IPIP_TYPE_CHECK_ALLOW_GRE;
+
+	/* Linux demuxes tunnels based on packet SIP (which must match tunnel
+	 * remote IP). Thus configure decap so that it filters out packets that
+	 * are not IPv4 or have the wrong SIP. IPIP_DECAP_ERROR trap is
+	 * generated for packets that fail this criterion. Linux then handles
+	 * such packets in slow path and generates ICMP destination unreachable.
+	 */
+	daddr4 = be32_to_cpu(mlxsw_sp_ipip_netdev_daddr4(ipip_entry->ol_dev));
+	mlxsw_reg_rtdp_ipip4_pack(rtdp_pl, rif_index,
+				  MLXSW_REG_RTDP_IPIP_SIP_CHECK_FILTER_IPV4,
+				  type_check, has_ikey, daddr4, ikey);
+
+	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rtdp), rtdp_pl);
+}
+
+static int
+mlxsw_sp_ipip_fib_entry_op_gre4_ralue(struct mlxsw_sp *mlxsw_sp,
+				      u32 dip, u8 prefix_len, u16 ul_vr_id,
+				      enum mlxsw_reg_ralue_op op,
+				      u32 tunnel_index)
+{
+	char ralue_pl[MLXSW_REG_RALUE_LEN];
+
+	mlxsw_reg_ralue_pack4(ralue_pl, MLXSW_REG_RALXX_PROTOCOL_IPV4, op,
+			      ul_vr_id, prefix_len, dip);
+	mlxsw_reg_ralue_act_ip2me_tun_pack(ralue_pl, tunnel_index);
+	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
+}
+
+static int mlxsw_sp_ipip_fib_entry_op_gre4(struct mlxsw_sp *mlxsw_sp,
+					struct mlxsw_sp_ipip_entry *ipip_entry,
+					enum mlxsw_reg_ralue_op op,
+					u32 tunnel_index)
+{
+	u16 ul_vr_id = mlxsw_sp_ipip_lb_ul_vr_id(ipip_entry->ol_lb);
+	__be32 dip;
+	int err;
+
+	err = mlxsw_sp_ipip_fib_entry_op_gre4_rtdp(mlxsw_sp, tunnel_index,
+						   ipip_entry);
+	if (err)
+		return err;
+
+	dip = mlxsw_sp_ipip_netdev_saddr(MLXSW_SP_L3_PROTO_IPV4,
+					 ipip_entry->ol_dev).addr4;
+	return mlxsw_sp_ipip_fib_entry_op_gre4_ralue(mlxsw_sp, be32_to_cpu(dip),
+						     32, ul_vr_id, op,
+						     tunnel_index);
+}
+
+static bool mlxsw_sp_ipip_tunnel_complete(enum mlxsw_sp_l3proto proto,
+					  const struct net_device *ol_dev)
+{
+	union mlxsw_sp_l3addr saddr = mlxsw_sp_ipip_netdev_saddr(proto, ol_dev);
+	union mlxsw_sp_l3addr daddr = mlxsw_sp_ipip_netdev_daddr(proto, ol_dev);
+	union mlxsw_sp_l3addr naddr = {0};
+
+	/* Tunnels with unset local or remote address are valid in Linux and
+	 * used for lightweight tunnels (LWT) and Non-Broadcast Multi-Access
+	 * (NBMA) tunnels. In principle these can be offloaded, but the driver
+	 * currently doesn't support this. So punt.
+	 */
+	return memcmp(&saddr, &naddr, sizeof(naddr)) &&
+	       memcmp(&daddr, &naddr, sizeof(naddr));
+}
+
+static bool mlxsw_sp_ipip_can_offload_gre4(const struct mlxsw_sp *mlxsw_sp,
+					   const struct net_device *ol_dev,
+					   enum mlxsw_sp_l3proto ol_proto)
+{
+	struct ip_tunnel *tunnel = netdev_priv(ol_dev);
+	__be16 okflags = TUNNEL_KEY; /* We can't offload any other features. */
+	bool inherit_ttl = tunnel->parms.iph.ttl == 0;
+	bool inherit_tos = tunnel->parms.iph.tos & 0x1;
+
+	return (tunnel->parms.i_flags & ~okflags) == 0 &&
+	       (tunnel->parms.o_flags & ~okflags) == 0 &&
+	       inherit_ttl && inherit_tos &&
+	       mlxsw_sp_ipip_tunnel_complete(MLXSW_SP_L3_PROTO_IPV4, ol_dev);
+}
+
+static struct mlxsw_sp_rif_ipip_lb_config
+mlxsw_sp_ipip_ol_loopback_config_gre4(struct mlxsw_sp *mlxsw_sp,
+				      const struct net_device *ol_dev)
+{
+	enum mlxsw_reg_ritr_loopback_ipip_type lb_ipipt;
+
+	lb_ipipt = mlxsw_sp_ipip_netdev_has_okey(ol_dev) ?
+		MLXSW_REG_RITR_LOOPBACK_IPIP_TYPE_IP_IN_GRE_KEY_IN_IP :
+		MLXSW_REG_RITR_LOOPBACK_IPIP_TYPE_IP_IN_GRE_IN_IP;
+	return (struct mlxsw_sp_rif_ipip_lb_config){
+		.lb_ipipt = lb_ipipt,
+		.okey = mlxsw_sp_ipip_netdev_okey(ol_dev),
+		.ul_protocol = MLXSW_SP_L3_PROTO_IPV4,
+		.saddr = mlxsw_sp_ipip_netdev_saddr(MLXSW_SP_L3_PROTO_IPV4,
+						    ol_dev),
+	};
+}
+
+static const struct mlxsw_sp_ipip_ops mlxsw_sp_ipip_gre4_ops = {
+	.dev_type = ARPHRD_IPGRE,
+	.ul_proto = MLXSW_SP_L3_PROTO_IPV4,
+	.nexthop_update = mlxsw_sp_ipip_nexthop_update_gre4,
+	.fib_entry_op = mlxsw_sp_ipip_fib_entry_op_gre4,
+	.can_offload = mlxsw_sp_ipip_can_offload_gre4,
+	.ol_loopback_config = mlxsw_sp_ipip_ol_loopback_config_gre4,
+};
+
+const struct mlxsw_sp_ipip_ops *mlxsw_sp_ipip_ops_arr[] = {
+	[MLXSW_SP_IPIP_TYPE_GRE4] = &mlxsw_sp_ipip_gre4_ops,
+};
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ipip.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ipip.h
new file mode 100644
index 000000000000..1c2db831d83b
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ipip.h
@@ -0,0 +1,79 @@
+/*
+ * drivers/net/ethernet/mellanox/mlxsw/spectrum_ipip.h
+ * Copyright (c) 2017 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2017 Petr Machata <petrm@mellanox.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the names of the copyright holders nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _MLXSW_IPIP_H_
+#define _MLXSW_IPIP_H_
+
+#include "spectrum_router.h"
+#include <net/ip_fib.h>
+
+enum mlxsw_sp_ipip_type {
+	MLXSW_SP_IPIP_TYPE_GRE4,
+	MLXSW_SP_IPIP_TYPE_MAX,
+};
+
+struct mlxsw_sp_ipip_entry {
+	enum mlxsw_sp_ipip_type ipipt;
+	struct net_device *ol_dev; /* Overlay. */
+	struct mlxsw_sp_rif_ipip_lb *ol_lb;
+	unsigned int ref_count; /* Number of next hops using the tunnel. */
+	struct mlxsw_sp_fib_entry *decap_fib_entry;
+	struct list_head ipip_list_node;
+};
+
+struct mlxsw_sp_ipip_ops {
+	int dev_type;
+	enum mlxsw_sp_l3proto ul_proto; /* Underlay. */
+
+	int (*nexthop_update)(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
+			      struct mlxsw_sp_ipip_entry *ipip_entry);
+
+	bool (*can_offload)(const struct mlxsw_sp *mlxsw_sp,
+			    const struct net_device *ol_dev,
+			    enum mlxsw_sp_l3proto ol_proto);
+
+	/* Return a configuration for creating an overlay loopback RIF. */
+	struct mlxsw_sp_rif_ipip_lb_config
+	(*ol_loopback_config)(struct mlxsw_sp *mlxsw_sp,
+			      const struct net_device *ol_dev);
+
+	int (*fib_entry_op)(struct mlxsw_sp *mlxsw_sp,
+			    struct mlxsw_sp_ipip_entry *ipip_entry,
+			    enum mlxsw_reg_ralue_op op,
+			    u32 tunnel_index);
+};
+
+extern const struct mlxsw_sp_ipip_ops *mlxsw_sp_ipip_ops_arr[];
+
+#endif /* _MLXSW_IPIP_H_*/
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
index 4b2e0fd7d51e..2cfb3f5d092d 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
@@ -1,9 +1,10 @@
 /*
  * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
- * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2016-2017 Mellanox Technologies. All rights reserved.
  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
  * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
  * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
+ * Copyright (c) 2017 Petr Machata <petrm@mellanox.com>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -43,18 +44,27 @@
 #include <linux/inetdevice.h>
 #include <linux/netdevice.h>
 #include <linux/if_bridge.h>
+#include <linux/socket.h>
+#include <linux/route.h>
 #include <net/netevent.h>
 #include <net/neighbour.h>
 #include <net/arp.h>
 #include <net/ip_fib.h>
+#include <net/ip6_fib.h>
 #include <net/fib_rules.h>
+#include <net/ip_tunnels.h>
 #include <net/l3mdev.h>
+#include <net/addrconf.h>
+#include <net/ndisc.h>
+#include <net/ipv6.h>
+#include <net/fib_notifier.h>
 
 #include "spectrum.h"
 #include "core.h"
 #include "reg.h"
 #include "spectrum_cnt.h"
 #include "spectrum_dpipe.h"
+#include "spectrum_ipip.h"
 #include "spectrum_router.h"
 
 struct mlxsw_sp_vr;
@@ -79,9 +89,11 @@ struct mlxsw_sp_router {
 	struct delayed_work nexthop_probe_dw;
 #define MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL 5000 /* ms */
 	struct list_head nexthop_neighs_list;
+	struct list_head ipip_list;
 	bool aborted;
 	struct notifier_block fib_nb;
 	const struct mlxsw_sp_rif_ops **rif_ops_arr;
+	const struct mlxsw_sp_ipip_ops **ipip_ops_arr;
 };
 
 struct mlxsw_sp_rif {
@@ -122,6 +134,17 @@ struct mlxsw_sp_rif_subport {
 	bool lag;
 };
 
+struct mlxsw_sp_rif_ipip_lb {
+	struct mlxsw_sp_rif common;
+	struct mlxsw_sp_rif_ipip_lb_config lb_config;
+	u16 ul_vr_id; /* Reserved for Spectrum-2. */
+};
+
+struct mlxsw_sp_rif_params_ipip_lb {
+	struct mlxsw_sp_rif_params common;
+	struct mlxsw_sp_rif_ipip_lb_config lb_config;
+};
+
 struct mlxsw_sp_rif_ops {
 	enum mlxsw_sp_rif_type type;
 	size_t rif_size;
@@ -304,7 +327,7 @@ static struct mlxsw_sp_rif *
 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
 			 const struct net_device *dev);
 
-#define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE)
+#define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1)
 
 struct mlxsw_sp_prefix_usage {
 	DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
@@ -314,19 +337,6 @@ struct mlxsw_sp_prefix_usage {
 	for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
 
 static bool
-mlxsw_sp_prefix_usage_subset(struct mlxsw_sp_prefix_usage *prefix_usage1,
-			     struct mlxsw_sp_prefix_usage *prefix_usage2)
-{
-	unsigned char prefix;
-
-	mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage1) {
-		if (!test_bit(prefix, prefix_usage2->b))
-			return false;
-	}
-	return true;
-}
-
-static bool
 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
 			 struct mlxsw_sp_prefix_usage *prefix_usage2)
 {
@@ -371,6 +381,14 @@ enum mlxsw_sp_fib_entry_type {
 	MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
 	MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
 	MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
+
+	/* This is a special case of local delivery, where a packet should be
+	 * decapsulated on reception. Note that there is no corresponding ENCAP,
+	 * because that's a type of next hop, not of FIB entry. (There can be
+	 * several next hops in a REMOTE entry, and some of them may be
+	 * encapsulating entries.)
+	 */
+	MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP,
 };
 
 struct mlxsw_sp_nexthop_group;
@@ -384,11 +402,9 @@ struct mlxsw_sp_fib_node {
 	struct mlxsw_sp_fib_key key;
 };
 
-struct mlxsw_sp_fib_entry_params {
-	u32 tb_id;
-	u32 prio;
-	u8 tos;
-	u8 type;
+struct mlxsw_sp_fib_entry_decap {
+	struct mlxsw_sp_ipip_entry *ipip_entry;
+	u32 tunnel_index;
 };
 
 struct mlxsw_sp_fib_entry {
@@ -397,13 +413,26 @@ struct mlxsw_sp_fib_entry {
 	enum mlxsw_sp_fib_entry_type type;
 	struct list_head nexthop_group_node;
 	struct mlxsw_sp_nexthop_group *nh_group;
-	struct mlxsw_sp_fib_entry_params params;
-	bool offloaded;
+	struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */
+};
+
+struct mlxsw_sp_fib4_entry {
+	struct mlxsw_sp_fib_entry common;
+	u32 tb_id;
+	u32 prio;
+	u8 tos;
+	u8 type;
+};
+
+struct mlxsw_sp_fib6_entry {
+	struct mlxsw_sp_fib_entry common;
+	struct list_head rt6_list;
+	unsigned int nrt6;
 };
 
-enum mlxsw_sp_l3proto {
-	MLXSW_SP_L3_PROTO_IPV4,
-	MLXSW_SP_L3_PROTO_IPV6,
+struct mlxsw_sp_rt6 {
+	struct list_head list;
+	struct rt6_info *rt;
 };
 
 struct mlxsw_sp_lpm_tree {
@@ -428,6 +457,7 @@ struct mlxsw_sp_vr {
 	u32 tb_id; /* kernel fib table id */
 	unsigned int rif_count;
 	struct mlxsw_sp_fib *fib4;
+	struct mlxsw_sp_fib *fib6;
 };
 
 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
@@ -487,15 +517,15 @@ static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 }
 
-static int mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
-				  struct mlxsw_sp_lpm_tree *lpm_tree)
+static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
+				   struct mlxsw_sp_lpm_tree *lpm_tree)
 {
 	char ralta_pl[MLXSW_REG_RALTA_LEN];
 
 	mlxsw_reg_ralta_pack(ralta_pl, false,
 			     (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 			     lpm_tree->id);
-	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
+	mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 }
 
 static int
@@ -551,10 +581,10 @@ err_left_struct_set:
 	return ERR_PTR(err);
 }
 
-static int mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
-				     struct mlxsw_sp_lpm_tree *lpm_tree)
+static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
+				      struct mlxsw_sp_lpm_tree *lpm_tree)
 {
-	return mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
+	mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 }
 
 static struct mlxsw_sp_lpm_tree *
@@ -571,24 +601,21 @@ mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 		    lpm_tree->proto == proto &&
 		    mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
 					     prefix_usage))
-			goto inc_ref_count;
+			return lpm_tree;
 	}
-	lpm_tree = mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage,
-					    proto);
-	if (IS_ERR(lpm_tree))
-		return lpm_tree;
+	return mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, proto);
+}
 
-inc_ref_count:
+static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree)
+{
 	lpm_tree->ref_count++;
-	return lpm_tree;
 }
 
-static int mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
-				 struct mlxsw_sp_lpm_tree *lpm_tree)
+static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
+				  struct mlxsw_sp_lpm_tree *lpm_tree)
 {
 	if (--lpm_tree->ref_count == 0)
-		return mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
-	return 0;
+		mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
 }
 
 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
@@ -625,7 +652,7 @@ static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
 
 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
 {
-	return !!vr->fib4;
+	return !!vr->fib4 || !!vr->fib6;
 }
 
 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
@@ -642,13 +669,13 @@ static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 }
 
 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
-				     const struct mlxsw_sp_fib *fib)
+				     const struct mlxsw_sp_fib *fib, u8 tree_id)
 {
 	char raltb_pl[MLXSW_REG_RALTB_LEN];
 
 	mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 			     (enum mlxsw_reg_ralxx_protocol) fib->proto,
-			     fib->lpm_tree->id);
+			     tree_id);
 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 }
 
@@ -694,7 +721,7 @@ static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
 	case MLXSW_SP_L3_PROTO_IPV4:
 		return vr->fib4;
 	case MLXSW_SP_L3_PROTO_IPV6:
-		BUG_ON(1);
+		return vr->fib6;
 	}
 	return NULL;
 }
@@ -703,6 +730,7 @@ static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 					      u32 tb_id)
 {
 	struct mlxsw_sp_vr *vr;
+	int err;
 
 	vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
 	if (!vr)
@@ -710,54 +738,26 @@ static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 	vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4);
 	if (IS_ERR(vr->fib4))
 		return ERR_CAST(vr->fib4);
+	vr->fib6 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV6);
+	if (IS_ERR(vr->fib6)) {
+		err = PTR_ERR(vr->fib6);
+		goto err_fib6_create;
+	}
 	vr->tb_id = tb_id;
 	return vr;
-}
 
-static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
-{
+err_fib6_create:
 	mlxsw_sp_fib_destroy(vr->fib4);
 	vr->fib4 = NULL;
+	return ERR_PTR(err);
 }
 
-static int
-mlxsw_sp_vr_lpm_tree_check(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_fib *fib,
-			   struct mlxsw_sp_prefix_usage *req_prefix_usage)
+static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
 {
-	struct mlxsw_sp_lpm_tree *lpm_tree = fib->lpm_tree;
-	struct mlxsw_sp_lpm_tree *new_tree;
-	int err;
-
-	if (mlxsw_sp_prefix_usage_eq(req_prefix_usage, &lpm_tree->prefix_usage))
-		return 0;
-
-	new_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, req_prefix_usage,
-					 fib->proto);
-	if (IS_ERR(new_tree)) {
-		/* We failed to get a tree according to the required
-		 * prefix usage. However, the current tree might be still good
-		 * for us if our requirement is subset of the prefixes used
-		 * in the tree.
-		 */
-		if (mlxsw_sp_prefix_usage_subset(req_prefix_usage,
-						 &lpm_tree->prefix_usage))
-			return 0;
-		return PTR_ERR(new_tree);
-	}
-
-	/* Prevent packet loss by overwriting existing binding */
-	fib->lpm_tree = new_tree;
-	err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
-	if (err)
-		goto err_tree_bind;
-	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
-
-	return 0;
-
-err_tree_bind:
-	fib->lpm_tree = lpm_tree;
-	mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
-	return err;
+	mlxsw_sp_fib_destroy(vr->fib6);
+	vr->fib6 = NULL;
+	mlxsw_sp_fib_destroy(vr->fib4);
+	vr->fib4 = NULL;
 }
 
 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id)
@@ -773,10 +773,105 @@ static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id)
 
 static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr)
 {
-	if (!vr->rif_count && list_empty(&vr->fib4->node_list))
+	if (!vr->rif_count && list_empty(&vr->fib4->node_list) &&
+	    list_empty(&vr->fib6->node_list))
 		mlxsw_sp_vr_destroy(vr);
 }
 
+static bool
+mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr,
+				    enum mlxsw_sp_l3proto proto, u8 tree_id)
+{
+	struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
+
+	if (!mlxsw_sp_vr_is_used(vr))
+		return false;
+	if (fib->lpm_tree && fib->lpm_tree->id == tree_id)
+		return true;
+	return false;
+}
+
+static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
+					struct mlxsw_sp_fib *fib,
+					struct mlxsw_sp_lpm_tree *new_tree)
+{
+	struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
+	int err;
+
+	err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
+	if (err)
+		return err;
+	fib->lpm_tree = new_tree;
+	mlxsw_sp_lpm_tree_hold(new_tree);
+	mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
+	return 0;
+}
+
+static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
+					 struct mlxsw_sp_fib *fib,
+					 struct mlxsw_sp_lpm_tree *new_tree)
+{
+	struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
+	enum mlxsw_sp_l3proto proto = fib->proto;
+	u8 old_id, new_id = new_tree->id;
+	struct mlxsw_sp_vr *vr;
+	int i, err;
+
+	if (!old_tree)
+		goto no_replace;
+	old_id = old_tree->id;
+
+	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
+		vr = &mlxsw_sp->router->vrs[i];
+		if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id))
+			continue;
+		err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
+						   mlxsw_sp_vr_fib(vr, proto),
+						   new_tree);
+		if (err)
+			goto err_tree_replace;
+	}
+
+	return 0;
+
+err_tree_replace:
+	for (i--; i >= 0; i--) {
+		if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id))
+			continue;
+		mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
+					     mlxsw_sp_vr_fib(vr, proto),
+					     old_tree);
+	}
+	return err;
+
+no_replace:
+	err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
+	if (err)
+		return err;
+	fib->lpm_tree = new_tree;
+	mlxsw_sp_lpm_tree_hold(new_tree);
+	return 0;
+}
+
+static void
+mlxsw_sp_vrs_prefixes(struct mlxsw_sp *mlxsw_sp,
+		      enum mlxsw_sp_l3proto proto,
+		      struct mlxsw_sp_prefix_usage *req_prefix_usage)
+{
+	int i;
+
+	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
+		struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
+		struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
+		unsigned char prefix;
+
+		if (!mlxsw_sp_vr_is_used(vr))
+			continue;
+		mlxsw_sp_prefix_usage_for_each(prefix, &fib->prefix_usage)
+			mlxsw_sp_prefix_usage_set(req_prefix_usage, prefix);
+	}
+}
+
 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
 {
 	struct mlxsw_sp_vr *vr;
@@ -816,6 +911,374 @@ static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
 	kfree(mlxsw_sp->router->vrs);
 }
 
+static struct net_device *
+__mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev)
+{
+	struct ip_tunnel *tun = netdev_priv(ol_dev);
+	struct net *net = dev_net(ol_dev);
+
+	return __dev_get_by_index(net, tun->parms.link);
+}
+
+static u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev)
+{
+	struct net_device *d = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
+
+	if (d)
+		return l3mdev_fib_table(d) ? : RT_TABLE_MAIN;
+	else
+		return l3mdev_fib_table(ol_dev) ? : RT_TABLE_MAIN;
+}
+
+static struct mlxsw_sp_rif *
+mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
+		    const struct mlxsw_sp_rif_params *params);
+
+static struct mlxsw_sp_rif_ipip_lb *
+mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp,
+				enum mlxsw_sp_ipip_type ipipt,
+				struct net_device *ol_dev)
+{
+	struct mlxsw_sp_rif_params_ipip_lb lb_params;
+	const struct mlxsw_sp_ipip_ops *ipip_ops;
+	struct mlxsw_sp_rif *rif;
+
+	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
+	lb_params = (struct mlxsw_sp_rif_params_ipip_lb) {
+		.common.dev = ol_dev,
+		.common.lag = false,
+		.lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev),
+	};
+
+	rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common);
+	if (IS_ERR(rif))
+		return ERR_CAST(rif);
+	return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
+}
+
+static struct mlxsw_sp_ipip_entry *
+mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp,
+			  enum mlxsw_sp_ipip_type ipipt,
+			  struct net_device *ol_dev)
+{
+	struct mlxsw_sp_ipip_entry *ipip_entry;
+	struct mlxsw_sp_ipip_entry *ret = NULL;
+
+	ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL);
+	if (!ipip_entry)
+		return ERR_PTR(-ENOMEM);
+
+	ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt,
+							    ol_dev);
+	if (IS_ERR(ipip_entry->ol_lb)) {
+		ret = ERR_CAST(ipip_entry->ol_lb);
+		goto err_ol_ipip_lb_create;
+	}
+
+	ipip_entry->ipipt = ipipt;
+	ipip_entry->ol_dev = ol_dev;
+
+	return ipip_entry;
+
+err_ol_ipip_lb_create:
+	kfree(ipip_entry);
+	return ret;
+}
+
+static void
+mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp_ipip_entry *ipip_entry)
+{
+	WARN_ON(ipip_entry->ref_count > 0);
+	mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common);
+	kfree(ipip_entry);
+}
+
+static __be32
+mlxsw_sp_ipip_netdev_saddr4(const struct net_device *ol_dev)
+{
+	struct ip_tunnel *tun = netdev_priv(ol_dev);
+
+	return tun->parms.iph.saddr;
+}
+
+union mlxsw_sp_l3addr
+mlxsw_sp_ipip_netdev_saddr(enum mlxsw_sp_l3proto proto,
+			   const struct net_device *ol_dev)
+{
+	switch (proto) {
+	case MLXSW_SP_L3_PROTO_IPV4:
+		return (union mlxsw_sp_l3addr) {
+			.addr4 = mlxsw_sp_ipip_netdev_saddr4(ol_dev),
+		};
+	case MLXSW_SP_L3_PROTO_IPV6:
+		break;
+	};
+
+	WARN_ON(1);
+	return (union mlxsw_sp_l3addr) {
+		.addr4 = 0,
+	};
+}
+
+__be32 mlxsw_sp_ipip_netdev_daddr4(const struct net_device *ol_dev)
+{
+	struct ip_tunnel *tun = netdev_priv(ol_dev);
+
+	return tun->parms.iph.daddr;
+}
+
+union mlxsw_sp_l3addr
+mlxsw_sp_ipip_netdev_daddr(enum mlxsw_sp_l3proto proto,
+			   const struct net_device *ol_dev)
+{
+	switch (proto) {
+	case MLXSW_SP_L3_PROTO_IPV4:
+		return (union mlxsw_sp_l3addr) {
+			.addr4 = mlxsw_sp_ipip_netdev_daddr4(ol_dev),
+		};
+	case MLXSW_SP_L3_PROTO_IPV6:
+		break;
+	};
+
+	WARN_ON(1);
+	return (union mlxsw_sp_l3addr) {
+		.addr4 = 0,
+	};
+}
+
+static bool mlxsw_sp_l3addr_eq(const union mlxsw_sp_l3addr *addr1,
+			       const union mlxsw_sp_l3addr *addr2)
+{
+	return !memcmp(addr1, addr2, sizeof(*addr1));
+}
+
+static bool
+mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp,
+				  const enum mlxsw_sp_l3proto ul_proto,
+				  union mlxsw_sp_l3addr saddr,
+				  u32 ul_tb_id,
+				  struct mlxsw_sp_ipip_entry *ipip_entry)
+{
+	u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
+	enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
+	union mlxsw_sp_l3addr tun_saddr;
+
+	if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
+		return false;
+
+	tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
+	return tun_ul_tb_id == ul_tb_id &&
+	       mlxsw_sp_l3addr_eq(&tun_saddr, &saddr);
+}
+
+static int
+mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp,
+			      struct mlxsw_sp_fib_entry *fib_entry,
+			      struct mlxsw_sp_ipip_entry *ipip_entry)
+{
+	u32 tunnel_index;
+	int err;
+
+	err = mlxsw_sp_kvdl_alloc(mlxsw_sp, 1, &tunnel_index);
+	if (err)
+		return err;
+
+	ipip_entry->decap_fib_entry = fib_entry;
+	fib_entry->decap.ipip_entry = ipip_entry;
+	fib_entry->decap.tunnel_index = tunnel_index;
+	return 0;
+}
+
+static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp,
+					  struct mlxsw_sp_fib_entry *fib_entry)
+{
+	/* Unlink this node from the IPIP entry that it's the decap entry of. */
+	fib_entry->decap.ipip_entry->decap_fib_entry = NULL;
+	fib_entry->decap.ipip_entry = NULL;
+	mlxsw_sp_kvdl_free(mlxsw_sp, fib_entry->decap.tunnel_index);
+}
+
+static struct mlxsw_sp_fib_node *
+mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
+			 size_t addr_len, unsigned char prefix_len);
+static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
+				     struct mlxsw_sp_fib_entry *fib_entry);
+
+static void
+mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp,
+				 struct mlxsw_sp_ipip_entry *ipip_entry)
+{
+	struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry;
+
+	mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
+	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
+
+	mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
+}
+
+static void
+mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp,
+				  struct mlxsw_sp_ipip_entry *ipip_entry,
+				  struct mlxsw_sp_fib_entry *decap_fib_entry)
+{
+	if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry,
+					  ipip_entry))
+		return;
+	decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
+
+	if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry))
+		mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
+}
+
+/* Given an IPIP entry, find the corresponding decap route. */
+static struct mlxsw_sp_fib_entry *
+mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp,
+			       struct mlxsw_sp_ipip_entry *ipip_entry)
+{
+	static struct mlxsw_sp_fib_node *fib_node;
+	const struct mlxsw_sp_ipip_ops *ipip_ops;
+	struct mlxsw_sp_fib_entry *fib_entry;
+	unsigned char saddr_prefix_len;
+	union mlxsw_sp_l3addr saddr;
+	struct mlxsw_sp_fib *ul_fib;
+	struct mlxsw_sp_vr *ul_vr;
+	const void *saddrp;
+	size_t saddr_len;
+	u32 ul_tb_id;
+	u32 saddr4;
+
+	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
+
+	ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
+	ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id);
+	if (!ul_vr)
+		return NULL;
+
+	ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto);
+	saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto,
+					   ipip_entry->ol_dev);
+
+	switch (ipip_ops->ul_proto) {
+	case MLXSW_SP_L3_PROTO_IPV4:
+		saddr4 = be32_to_cpu(saddr.addr4);
+		saddrp = &saddr4;
+		saddr_len = 4;
+		saddr_prefix_len = 32;
+		break;
+	case MLXSW_SP_L3_PROTO_IPV6:
+		WARN_ON(1);
+		return NULL;
+	}
+
+	fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len,
+					    saddr_prefix_len);
+	if (!fib_node || list_empty(&fib_node->entry_list))
+		return NULL;
+
+	fib_entry = list_first_entry(&fib_node->entry_list,
+				     struct mlxsw_sp_fib_entry, list);
+	if (fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP)
+		return NULL;
+
+	return fib_entry;
+}
+
+static struct mlxsw_sp_ipip_entry *
+mlxsw_sp_ipip_entry_get(struct mlxsw_sp *mlxsw_sp,
+			enum mlxsw_sp_ipip_type ipipt,
+			struct net_device *ol_dev)
+{
+	u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev);
+	struct mlxsw_sp_router *router = mlxsw_sp->router;
+	struct mlxsw_sp_fib_entry *decap_fib_entry;
+	struct mlxsw_sp_ipip_entry *ipip_entry;
+	enum mlxsw_sp_l3proto ul_proto;
+	union mlxsw_sp_l3addr saddr;
+
+	list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
+			    ipip_list_node) {
+		if (ipip_entry->ol_dev == ol_dev)
+			goto inc_ref_count;
+
+		/* The configuration where several tunnels have the same local
+		 * address in the same underlay table needs special treatment in
+		 * the HW. That is currently not implemented in the driver.
+		 */
+		ul_proto = router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto;
+		saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev);
+		if (mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr,
+						      ul_tb_id, ipip_entry))
+			return ERR_PTR(-EEXIST);
+	}
+
+	ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev);
+	if (IS_ERR(ipip_entry))
+		return ipip_entry;
+
+	decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry);
+	if (decap_fib_entry)
+		mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry,
+						  decap_fib_entry);
+
+	list_add_tail(&ipip_entry->ipip_list_node,
+		      &mlxsw_sp->router->ipip_list);
+
+inc_ref_count:
+	++ipip_entry->ref_count;
+	return ipip_entry;
+}
+
+static void
+mlxsw_sp_ipip_entry_put(struct mlxsw_sp *mlxsw_sp,
+			struct mlxsw_sp_ipip_entry *ipip_entry)
+{
+	if (--ipip_entry->ref_count == 0) {
+		list_del(&ipip_entry->ipip_list_node);
+		if (ipip_entry->decap_fib_entry)
+			mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
+		mlxsw_sp_ipip_entry_destroy(ipip_entry);
+	}
+}
+
+static bool
+mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp,
+				  const struct net_device *ul_dev,
+				  enum mlxsw_sp_l3proto ul_proto,
+				  union mlxsw_sp_l3addr ul_dip,
+				  struct mlxsw_sp_ipip_entry *ipip_entry)
+{
+	u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
+	enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
+	struct net_device *ipip_ul_dev;
+
+	if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
+		return false;
+
+	ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ipip_entry->ol_dev);
+	return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip,
+						 ul_tb_id, ipip_entry) &&
+	       (!ipip_ul_dev || ipip_ul_dev == ul_dev);
+}
+
+/* Given decap parameters, find the corresponding IPIP entry. */
+static struct mlxsw_sp_ipip_entry *
+mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp,
+				  const struct net_device *ul_dev,
+				  enum mlxsw_sp_l3proto ul_proto,
+				  union mlxsw_sp_l3addr ul_dip)
+{
+	struct mlxsw_sp_ipip_entry *ipip_entry;
+
+	list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
+			    ipip_list_node)
+		if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev,
+						      ul_proto, ul_dip,
+						      ipip_entry))
+			return ipip_entry;
+
+	return NULL;
+}
+
 struct mlxsw_sp_neigh_key {
 	struct neighbour *n;
 };
@@ -831,6 +1294,8 @@ struct mlxsw_sp_neigh_entry {
 					* this neigh entry
 					*/
 	struct list_head nexthop_neighs_list_node;
+	unsigned int counter_index;
+	bool counter_valid;
 };
 
 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
@@ -839,6 +1304,62 @@ static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
 	.key_len = sizeof(struct mlxsw_sp_neigh_key),
 };
 
+struct mlxsw_sp_neigh_entry *
+mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif,
+			struct mlxsw_sp_neigh_entry *neigh_entry)
+{
+	if (!neigh_entry) {
+		if (list_empty(&rif->neigh_list))
+			return NULL;
+		else
+			return list_first_entry(&rif->neigh_list,
+						typeof(*neigh_entry),
+						rif_list_node);
+	}
+	if (neigh_entry->rif_list_node.next == &rif->neigh_list)
+		return NULL;
+	return list_next_entry(neigh_entry, rif_list_node);
+}
+
+int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry)
+{
+	return neigh_entry->key.n->tbl->family;
+}
+
+unsigned char *
+mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry)
+{
+	return neigh_entry->ha;
+}
+
+u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
+{
+	struct neighbour *n;
+
+	n = neigh_entry->key.n;
+	return ntohl(*((__be32 *) n->primary_key));
+}
+
+struct in6_addr *
+mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
+{
+	struct neighbour *n;
+
+	n = neigh_entry->key.n;
+	return (struct in6_addr *) &n->primary_key;
+}
+
+int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp,
+			       struct mlxsw_sp_neigh_entry *neigh_entry,
+			       u64 *p_counter)
+{
+	if (!neigh_entry->counter_valid)
+		return -EINVAL;
+
+	return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index,
+					 p_counter, NULL);
+}
+
 static struct mlxsw_sp_neigh_entry *
 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
 			   u16 rif)
@@ -879,6 +1400,53 @@ mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
 			       mlxsw_sp_neigh_ht_params);
 }
 
+static bool
+mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp,
+				    struct mlxsw_sp_neigh_entry *neigh_entry)
+{
+	struct devlink *devlink;
+	const char *table_name;
+
+	switch (mlxsw_sp_neigh_entry_type(neigh_entry)) {
+	case AF_INET:
+		table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4;
+		break;
+	case AF_INET6:
+		table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6;
+		break;
+	default:
+		WARN_ON(1);
+		return false;
+	}
+
+	devlink = priv_to_devlink(mlxsw_sp->core);
+	return devlink_dpipe_table_counter_enabled(devlink, table_name);
+}
+
+static void
+mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp,
+			     struct mlxsw_sp_neigh_entry *neigh_entry)
+{
+	if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry))
+		return;
+
+	if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index))
+		return;
+
+	neigh_entry->counter_valid = true;
+}
+
+static void
+mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp,
+			    struct mlxsw_sp_neigh_entry *neigh_entry)
+{
+	if (!neigh_entry->counter_valid)
+		return;
+	mlxsw_sp_flow_counter_free(mlxsw_sp,
+				   neigh_entry->counter_index);
+	neigh_entry->counter_valid = false;
+}
+
 static struct mlxsw_sp_neigh_entry *
 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
 {
@@ -898,6 +1466,7 @@ mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
 	if (err)
 		goto err_neigh_entry_insert;
 
+	mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
 	list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
 
 	return neigh_entry;
@@ -912,6 +1481,7 @@ mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
 			     struct mlxsw_sp_neigh_entry *neigh_entry)
 {
 	list_del(&neigh_entry->rif_list_node);
+	mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
 	mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
 	mlxsw_sp_neigh_entry_free(neigh_entry);
 }
@@ -929,8 +1499,15 @@ mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
 static void
 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
 {
-	unsigned long interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
+	unsigned long interval;
 
+#if IS_ENABLED(CONFIG_IPV6)
+	interval = min_t(unsigned long,
+			 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
+			 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
+#else
+	interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
+#endif
 	mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
 }
 
@@ -965,6 +1542,44 @@ static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 	neigh_release(n);
 }
 
+#if IS_ENABLED(CONFIG_IPV6)
+static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
+						   char *rauhtd_pl,
+						   int rec_index)
+{
+	struct net_device *dev;
+	struct neighbour *n;
+	struct in6_addr dip;
+	u16 rif;
+
+	mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif,
+					 (char *) &dip);
+
+	if (!mlxsw_sp->router->rifs[rif]) {
+		dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
+		return;
+	}
+
+	dev = mlxsw_sp->router->rifs[rif]->dev;
+	n = neigh_lookup(&nd_tbl, &dip, dev);
+	if (!n) {
+		netdev_err(dev, "Failed to find matching neighbour for IP=%pI6c\n",
+			   &dip);
+		return;
+	}
+
+	netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip);
+	neigh_event_send(n, NULL);
+	neigh_release(n);
+}
+#else
+static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
+						   char *rauhtd_pl,
+						   int rec_index)
+{
+}
+#endif
+
 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 						   char *rauhtd_pl,
 						   int rec_index)
@@ -988,6 +1603,15 @@ static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 
 }
 
+static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp,
+						   char *rauhtd_pl,
+						   int rec_index)
+{
+	/* One record contains one entry. */
+	mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl,
+					       rec_index);
+}
+
 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
 					      char *rauhtd_pl, int rec_index)
 {
@@ -997,7 +1621,8 @@ static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
 						       rec_index);
 		break;
 	case MLXSW_REG_RAUHTD_TYPE_IPV6:
-		WARN_ON_ONCE(1);
+		mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl,
+						       rec_index);
 		break;
 	}
 }
@@ -1022,22 +1647,20 @@ static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
 	return false;
 }
 
-static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
+static int
+__mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp,
+				       char *rauhtd_pl,
+				       enum mlxsw_reg_rauhtd_type type)
 {
-	char *rauhtd_pl;
-	u8 num_rec;
-	int i, err;
-
-	rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
-	if (!rauhtd_pl)
-		return -ENOMEM;
+	int i, num_rec;
+	int err;
 
 	/* Make sure the neighbour's netdev isn't removed in the
 	 * process.
 	 */
 	rtnl_lock();
 	do {
-		mlxsw_reg_rauhtd_pack(rauhtd_pl, MLXSW_REG_RAUHTD_TYPE_IPV4);
+		mlxsw_reg_rauhtd_pack(rauhtd_pl, type);
 		err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
 				      rauhtd_pl);
 		if (err) {
@@ -1051,6 +1674,27 @@ static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
 	} while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
 	rtnl_unlock();
 
+	return err;
+}
+
+static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
+{
+	enum mlxsw_reg_rauhtd_type type;
+	char *rauhtd_pl;
+	int err;
+
+	rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
+	if (!rauhtd_pl)
+		return -ENOMEM;
+
+	type = MLXSW_REG_RAUHTD_TYPE_IPV4;
+	err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
+	if (err)
+		goto out;
+
+	type = MLXSW_REG_RAUHTD_TYPE_IPV6;
+	err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
+out:
 	kfree(rauhtd_pl);
 	return err;
 }
@@ -1143,9 +1787,43 @@ mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
 
 	mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
 			      dip);
+	if (neigh_entry->counter_valid)
+		mlxsw_reg_rauht_pack_counter(rauht_pl,
+					     neigh_entry->counter_index);
+	mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
+}
+
+static void
+mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp,
+				struct mlxsw_sp_neigh_entry *neigh_entry,
+				enum mlxsw_reg_rauht_op op)
+{
+	struct neighbour *n = neigh_entry->key.n;
+	char rauht_pl[MLXSW_REG_RAUHT_LEN];
+	const char *dip = n->primary_key;
+
+	mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
+			      dip);
+	if (neigh_entry->counter_valid)
+		mlxsw_reg_rauht_pack_counter(rauht_pl,
+					     neigh_entry->counter_index);
 	mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
 }
 
+bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry)
+{
+	struct neighbour *n = neigh_entry->key.n;
+
+	/* Packets with a link-local destination address are trapped
+	 * after LPM lookup and never reach the neighbour table, so
+	 * there is no need to program such neighbours to the device.
+	 */
+	if (ipv6_addr_type((struct in6_addr *) &n->primary_key) &
+	    IPV6_ADDR_LINKLOCAL)
+		return true;
+	return false;
+}
+
 static void
 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
 			    struct mlxsw_sp_neigh_entry *neigh_entry,
@@ -1154,11 +1832,29 @@ mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
 	if (!adding && !neigh_entry->connected)
 		return;
 	neigh_entry->connected = adding;
-	if (neigh_entry->key.n->tbl == &arp_tbl)
+	if (neigh_entry->key.n->tbl->family == AF_INET) {
 		mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
 						mlxsw_sp_rauht_op(adding));
-	else
+	} else if (neigh_entry->key.n->tbl->family == AF_INET6) {
+		if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
+			return;
+		mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry,
+						mlxsw_sp_rauht_op(adding));
+	} else {
 		WARN_ON_ONCE(1);
+	}
+}
+
+void
+mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp,
+				    struct mlxsw_sp_neigh_entry *neigh_entry,
+				    bool adding)
+{
+	if (adding)
+		mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
+	else
+		mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
+	mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true);
 }
 
 struct mlxsw_sp_neigh_event_work {
@@ -1227,7 +1923,8 @@ int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
 		p = ptr;
 
 		/* We don't care about changes in the default table. */
-		if (!p->dev || p->tbl != &arp_tbl)
+		if (!p->dev || (p->tbl->family != AF_INET &&
+				p->tbl->family != AF_INET6))
 			return NOTIFY_DONE;
 
 		/* We are in atomic context and can't take RTNL mutex,
@@ -1246,7 +1943,7 @@ int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
 	case NETEVENT_NEIGH_UPDATE:
 		n = ptr;
 
-		if (n->tbl != &arp_tbl)
+		if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6)
 			return NOTIFY_DONE;
 
 		mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
@@ -1307,27 +2004,23 @@ static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
 	rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
 }
 
-static int mlxsw_sp_neigh_rif_flush(struct mlxsw_sp *mlxsw_sp,
-				    const struct mlxsw_sp_rif *rif)
-{
-	char rauht_pl[MLXSW_REG_RAUHT_LEN];
-
-	mlxsw_reg_rauht_pack(rauht_pl, MLXSW_REG_RAUHT_OP_WRITE_DELETE_ALL,
-			     rif->rif_index, rif->addr);
-	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
-}
-
 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
 					 struct mlxsw_sp_rif *rif)
 {
 	struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
 
-	mlxsw_sp_neigh_rif_flush(mlxsw_sp, rif);
 	list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
-				 rif_list_node)
+				 rif_list_node) {
+		mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false);
 		mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
+	}
 }
 
+enum mlxsw_sp_nexthop_type {
+	MLXSW_SP_NEXTHOP_TYPE_ETH,
+	MLXSW_SP_NEXTHOP_TYPE_IPIP,
+};
+
 struct mlxsw_sp_nexthop_key {
 	struct fib_nh *fib_nh;
 };
@@ -1340,6 +2033,8 @@ struct mlxsw_sp_nexthop {
 						*/
 	struct rhash_head ht_node;
 	struct mlxsw_sp_nexthop_key key;
+	unsigned char gw_addr[sizeof(struct in6_addr)];
+	int ifindex;
 	struct mlxsw_sp_rif *rif;
 	u8 should_offload:1, /* set indicates this neigh is connected and
 			      * should be put to KVD linear area of this group.
@@ -1350,17 +2045,18 @@ struct mlxsw_sp_nexthop {
 	   update:1; /* set indicates that MAC of this neigh should be
 		      * updated in HW
 		      */
-	struct mlxsw_sp_neigh_entry *neigh_entry;
-};
-
-struct mlxsw_sp_nexthop_group_key {
-	struct fib_info *fi;
+	enum mlxsw_sp_nexthop_type type;
+	union {
+		struct mlxsw_sp_neigh_entry *neigh_entry;
+		struct mlxsw_sp_ipip_entry *ipip_entry;
+	};
 };
 
 struct mlxsw_sp_nexthop_group {
+	void *priv;
 	struct rhash_head ht_node;
 	struct list_head fib_list; /* list of fib entries that use this group */
-	struct mlxsw_sp_nexthop_group_key key;
+	struct neigh_table *neigh_tbl;
 	u8 adj_index_valid:1,
 	   gateway:1; /* routes using the group use a gateway */
 	u32 adj_index;
@@ -1370,15 +2066,154 @@ struct mlxsw_sp_nexthop_group {
 #define nh_rif	nexthops[0].rif
 };
 
+static struct fib_info *
+mlxsw_sp_nexthop4_group_fi(const struct mlxsw_sp_nexthop_group *nh_grp)
+{
+	return nh_grp->priv;
+}
+
+struct mlxsw_sp_nexthop_group_cmp_arg {
+	enum mlxsw_sp_l3proto proto;
+	union {
+		struct fib_info *fi;
+		struct mlxsw_sp_fib6_entry *fib6_entry;
+	};
+};
+
+static bool
+mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp,
+				    const struct in6_addr *gw, int ifindex)
+{
+	int i;
+
+	for (i = 0; i < nh_grp->count; i++) {
+		const struct mlxsw_sp_nexthop *nh;
+
+		nh = &nh_grp->nexthops[i];
+		if (nh->ifindex == ifindex &&
+		    ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr))
+			return true;
+	}
+
+	return false;
+}
+
+static bool
+mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp,
+			    const struct mlxsw_sp_fib6_entry *fib6_entry)
+{
+	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
+
+	if (nh_grp->count != fib6_entry->nrt6)
+		return false;
+
+	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
+		struct in6_addr *gw;
+		int ifindex;
+
+		ifindex = mlxsw_sp_rt6->rt->dst.dev->ifindex;
+		gw = &mlxsw_sp_rt6->rt->rt6i_gateway;
+		if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex))
+			return false;
+	}
+
+	return true;
+}
+
+static int
+mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr)
+{
+	const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key;
+	const struct mlxsw_sp_nexthop_group *nh_grp = ptr;
+
+	switch (cmp_arg->proto) {
+	case MLXSW_SP_L3_PROTO_IPV4:
+		return cmp_arg->fi != mlxsw_sp_nexthop4_group_fi(nh_grp);
+	case MLXSW_SP_L3_PROTO_IPV6:
+		return !mlxsw_sp_nexthop6_group_cmp(nh_grp,
+						    cmp_arg->fib6_entry);
+	default:
+		WARN_ON(1);
+		return 1;
+	}
+}
+
+static int
+mlxsw_sp_nexthop_group_type(const struct mlxsw_sp_nexthop_group *nh_grp)
+{
+	return nh_grp->neigh_tbl->family;
+}
+
+static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed)
+{
+	const struct mlxsw_sp_nexthop_group *nh_grp = data;
+	const struct mlxsw_sp_nexthop *nh;
+	struct fib_info *fi;
+	unsigned int val;
+	int i;
+
+	switch (mlxsw_sp_nexthop_group_type(nh_grp)) {
+	case AF_INET:
+		fi = mlxsw_sp_nexthop4_group_fi(nh_grp);
+		return jhash(&fi, sizeof(fi), seed);
+	case AF_INET6:
+		val = nh_grp->count;
+		for (i = 0; i < nh_grp->count; i++) {
+			nh = &nh_grp->nexthops[i];
+			val ^= nh->ifindex;
+		}
+		return jhash(&val, sizeof(val), seed);
+	default:
+		WARN_ON(1);
+		return 0;
+	}
+}
+
+static u32
+mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed)
+{
+	unsigned int val = fib6_entry->nrt6;
+	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
+	struct net_device *dev;
+
+	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
+		dev = mlxsw_sp_rt6->rt->dst.dev;
+		val ^= dev->ifindex;
+	}
+
+	return jhash(&val, sizeof(val), seed);
+}
+
+static u32
+mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed)
+{
+	const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data;
+
+	switch (cmp_arg->proto) {
+	case MLXSW_SP_L3_PROTO_IPV4:
+		return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed);
+	case MLXSW_SP_L3_PROTO_IPV6:
+		return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed);
+	default:
+		WARN_ON(1);
+		return 0;
+	}
+}
+
 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
-	.key_offset = offsetof(struct mlxsw_sp_nexthop_group, key),
 	.head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
-	.key_len = sizeof(struct mlxsw_sp_nexthop_group_key),
+	.hashfn	     = mlxsw_sp_nexthop_group_hash,
+	.obj_hashfn  = mlxsw_sp_nexthop_group_hash_obj,
+	.obj_cmpfn   = mlxsw_sp_nexthop_group_cmp,
 };
 
 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
 					 struct mlxsw_sp_nexthop_group *nh_grp)
 {
+	if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
+	    !nh_grp->gateway)
+		return 0;
+
 	return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
 				      &nh_grp->ht_node,
 				      mlxsw_sp_nexthop_group_ht_params);
@@ -1387,16 +2222,38 @@ static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
 					  struct mlxsw_sp_nexthop_group *nh_grp)
 {
+	if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
+	    !nh_grp->gateway)
+		return;
+
 	rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
 			       &nh_grp->ht_node,
 			       mlxsw_sp_nexthop_group_ht_params);
 }
 
 static struct mlxsw_sp_nexthop_group *
-mlxsw_sp_nexthop_group_lookup(struct mlxsw_sp *mlxsw_sp,
-			      struct mlxsw_sp_nexthop_group_key key)
+mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp,
+			       struct fib_info *fi)
+{
+	struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
+
+	cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV4;
+	cmp_arg.fi = fi;
+	return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
+				      &cmp_arg,
+				      mlxsw_sp_nexthop_group_ht_params);
+}
+
+static struct mlxsw_sp_nexthop_group *
+mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp,
+			       struct mlxsw_sp_fib6_entry *fib6_entry)
 {
-	return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, &key,
+	struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
+
+	cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV6;
+	cmp_arg.fib6_entry = fib6_entry;
+	return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
+				      &cmp_arg,
 				      mlxsw_sp_nexthop_group_ht_params);
 }
 
@@ -1473,15 +2330,26 @@ static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
 	char ratr_pl[MLXSW_REG_RATR_LEN];
 
 	mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
-			    true, adj_index, neigh_entry->rif);
+			    true, MLXSW_REG_RATR_TYPE_ETHERNET,
+			    adj_index, neigh_entry->rif);
 	mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
 }
 
+static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
+					u32 adj_index,
+					struct mlxsw_sp_nexthop *nh)
+{
+	const struct mlxsw_sp_ipip_ops *ipip_ops;
+
+	ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt];
+	return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry);
+}
+
 static int
-mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp,
-				  struct mlxsw_sp_nexthop_group *nh_grp,
-				  bool reallocate)
+mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp,
+			      struct mlxsw_sp_nexthop_group *nh_grp,
+			      bool reallocate)
 {
 	u32 adj_index = nh_grp->adj_index; /* base */
 	struct mlxsw_sp_nexthop *nh;
@@ -1497,8 +2365,16 @@ mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp,
 		}
 
 		if (nh->update || reallocate) {
-			err = mlxsw_sp_nexthop_mac_update(mlxsw_sp,
-							  adj_index, nh);
+			switch (nh->type) {
+			case MLXSW_SP_NEXTHOP_TYPE_ETH:
+				err = mlxsw_sp_nexthop_mac_update
+					    (mlxsw_sp, adj_index, nh);
+				break;
+			case MLXSW_SP_NEXTHOP_TYPE_IPIP:
+				err = mlxsw_sp_nexthop_ipip_update
+					    (mlxsw_sp, adj_index, nh);
+				break;
+			}
 			if (err)
 				return err;
 			nh->update = 0;
@@ -1509,9 +2385,6 @@ mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp,
 	return 0;
 }
 
-static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
-				     struct mlxsw_sp_fib_entry *fib_entry);
-
 static bool
 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
 				 const struct mlxsw_sp_fib_entry *fib_entry);
@@ -1535,6 +2408,24 @@ mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
 }
 
 static void
+mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
+				   enum mlxsw_reg_ralue_op op, int err);
+
+static void
+mlxsw_sp_nexthop_fib_entries_refresh(struct mlxsw_sp_nexthop_group *nh_grp)
+{
+	enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_WRITE;
+	struct mlxsw_sp_fib_entry *fib_entry;
+
+	list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
+		if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
+						      fib_entry))
+			continue;
+		mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
+	}
+}
+
+static void
 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
 			       struct mlxsw_sp_nexthop_group *nh_grp)
 {
@@ -1556,7 +2447,7 @@ mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
 	for (i = 0; i < nh_grp->count; i++) {
 		nh = &nh_grp->nexthops[i];
 
-		if (nh->should_offload ^ nh->offloaded) {
+		if (nh->should_offload != nh->offloaded) {
 			offload_change = true;
 			if (nh->should_offload)
 				nh->update = 1;
@@ -1568,8 +2459,7 @@ mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
 		/* Nothing was added or removed, so no need to reallocate. Just
 		 * update MAC on existing adjacency indexes.
 		 */
-		err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp,
-							false);
+		err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, false);
 		if (err) {
 			dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
 			goto set_trap;
@@ -1596,7 +2486,7 @@ mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
 	nh_grp->adj_index_valid = 1;
 	nh_grp->adj_index = adj_index;
 	nh_grp->ecmp_size = ecmp_size;
-	err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp, true);
+	err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, true);
 	if (err) {
 		dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
 		goto set_trap;
@@ -1621,6 +2511,10 @@ mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
 		dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
 		goto set_trap;
 	}
+
+	/* Offload state within the group changed, so update the flags. */
+	mlxsw_sp_nexthop_fib_entries_refresh(nh_grp);
+
 	return;
 
 set_trap:
@@ -1640,9 +2534,9 @@ set_trap:
 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
 					    bool removing)
 {
-	if (!removing && !nh->should_offload)
+	if (!removing)
 		nh->should_offload = 1;
-	else if (removing && nh->offloaded)
+	else if (nh->offloaded)
 		nh->should_offload = 0;
 	nh->update = 1;
 }
@@ -1684,7 +2578,6 @@ static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
 				       struct mlxsw_sp_nexthop *nh)
 {
 	struct mlxsw_sp_neigh_entry *neigh_entry;
-	struct fib_nh *fib_nh = nh->key.fib_nh;
 	struct neighbour *n;
 	u8 nud_state, dead;
 	int err;
@@ -1693,13 +2586,14 @@ static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
 		return 0;
 
 	/* Take a reference of neigh here ensuring that neigh would
-	 * not be detructed before the nexthop entry is finished.
+	 * not be destructed before the nexthop entry is finished.
 	 * The reference is taken either in neigh_lookup() or
 	 * in neigh_create() in case n is not found.
 	 */
-	n = neigh_lookup(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
+	n = neigh_lookup(nh->nh_grp->neigh_tbl, &nh->gw_addr, nh->rif->dev);
 	if (!n) {
-		n = neigh_create(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
+		n = neigh_create(nh->nh_grp->neigh_tbl, &nh->gw_addr,
+				 nh->rif->dev);
 		if (IS_ERR(n))
 			return PTR_ERR(n);
 		neigh_event_send(n, NULL);
@@ -1761,18 +2655,131 @@ static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
 	neigh_release(n);
 }
 
-static int mlxsw_sp_nexthop_init(struct mlxsw_sp *mlxsw_sp,
-				 struct mlxsw_sp_nexthop_group *nh_grp,
-				 struct mlxsw_sp_nexthop *nh,
-				 struct fib_nh *fib_nh)
+static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp,
+				      const struct net_device *dev,
+				      enum mlxsw_sp_ipip_type *p_type)
+{
+	struct mlxsw_sp_router *router = mlxsw_sp->router;
+	const struct mlxsw_sp_ipip_ops *ipip_ops;
+	enum mlxsw_sp_ipip_type ipipt;
+
+	for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) {
+		ipip_ops = router->ipip_ops_arr[ipipt];
+		if (dev->type == ipip_ops->dev_type) {
+			if (p_type)
+				*p_type = ipipt;
+			return true;
+		}
+	}
+	return false;
+}
+
+static int mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp,
+				      enum mlxsw_sp_ipip_type ipipt,
+				      struct mlxsw_sp_nexthop *nh,
+				      struct net_device *ol_dev)
+{
+	if (!nh->nh_grp->gateway || nh->ipip_entry)
+		return 0;
+
+	nh->ipip_entry = mlxsw_sp_ipip_entry_get(mlxsw_sp, ipipt, ol_dev);
+	if (IS_ERR(nh->ipip_entry))
+		return PTR_ERR(nh->ipip_entry);
+
+	__mlxsw_sp_nexthop_neigh_update(nh, false);
+	return 0;
+}
+
+static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp,
+				       struct mlxsw_sp_nexthop *nh)
+{
+	struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry;
+
+	if (!ipip_entry)
+		return;
+
+	__mlxsw_sp_nexthop_neigh_update(nh, true);
+	mlxsw_sp_ipip_entry_put(mlxsw_sp, ipip_entry);
+	nh->ipip_entry = NULL;
+}
+
+static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp,
+					const struct fib_nh *fib_nh,
+					enum mlxsw_sp_ipip_type *p_ipipt)
 {
 	struct net_device *dev = fib_nh->nh_dev;
-	struct in_device *in_dev;
+
+	return dev &&
+	       fib_nh->nh_parent->fib_type == RTN_UNICAST &&
+	       mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt);
+}
+
+static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp,
+				       struct mlxsw_sp_nexthop *nh)
+{
+	switch (nh->type) {
+	case MLXSW_SP_NEXTHOP_TYPE_ETH:
+		mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
+		mlxsw_sp_nexthop_rif_fini(nh);
+		break;
+	case MLXSW_SP_NEXTHOP_TYPE_IPIP:
+		mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh);
+		break;
+	}
+}
+
+static int mlxsw_sp_nexthop4_type_init(struct mlxsw_sp *mlxsw_sp,
+				       struct mlxsw_sp_nexthop *nh,
+				       struct fib_nh *fib_nh)
+{
+	struct mlxsw_sp_router *router = mlxsw_sp->router;
+	struct net_device *dev = fib_nh->nh_dev;
+	enum mlxsw_sp_ipip_type ipipt;
 	struct mlxsw_sp_rif *rif;
 	int err;
 
+	if (mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, fib_nh, &ipipt) &&
+	    router->ipip_ops_arr[ipipt]->can_offload(mlxsw_sp, dev,
+						     MLXSW_SP_L3_PROTO_IPV4)) {
+		nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
+		return mlxsw_sp_nexthop_ipip_init(mlxsw_sp, ipipt, nh, dev);
+	}
+
+	nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
+	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
+	if (!rif)
+		return 0;
+
+	mlxsw_sp_nexthop_rif_init(nh, rif);
+	err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
+	if (err)
+		goto err_neigh_init;
+
+	return 0;
+
+err_neigh_init:
+	mlxsw_sp_nexthop_rif_fini(nh);
+	return err;
+}
+
+static void mlxsw_sp_nexthop4_type_fini(struct mlxsw_sp *mlxsw_sp,
+					struct mlxsw_sp_nexthop *nh)
+{
+	mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
+}
+
+static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp,
+				  struct mlxsw_sp_nexthop_group *nh_grp,
+				  struct mlxsw_sp_nexthop *nh,
+				  struct fib_nh *fib_nh)
+{
+	struct net_device *dev = fib_nh->nh_dev;
+	struct in_device *in_dev;
+	int err;
+
 	nh->nh_grp = nh_grp;
 	nh->key.fib_nh = fib_nh;
+	memcpy(&nh->gw_addr, &fib_nh->nh_gw, sizeof(fib_nh->nh_gw));
 	err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
 	if (err)
 		return err;
@@ -1785,37 +2792,29 @@ static int mlxsw_sp_nexthop_init(struct mlxsw_sp *mlxsw_sp,
 	    fib_nh->nh_flags & RTNH_F_LINKDOWN)
 		return 0;
 
-	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
-	if (!rif)
-		return 0;
-	mlxsw_sp_nexthop_rif_init(nh, rif);
-
-	err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
+	err = mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh);
 	if (err)
 		goto err_nexthop_neigh_init;
 
 	return 0;
 
 err_nexthop_neigh_init:
-	mlxsw_sp_nexthop_rif_fini(nh);
 	mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
 	return err;
 }
 
-static void mlxsw_sp_nexthop_fini(struct mlxsw_sp *mlxsw_sp,
-				  struct mlxsw_sp_nexthop *nh)
+static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp,
+				   struct mlxsw_sp_nexthop *nh)
 {
-	mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
-	mlxsw_sp_nexthop_rif_fini(nh);
+	mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh);
 	mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
 }
 
-static void mlxsw_sp_nexthop_event(struct mlxsw_sp *mlxsw_sp,
-				   unsigned long event, struct fib_nh *fib_nh)
+static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp,
+				    unsigned long event, struct fib_nh *fib_nh)
 {
 	struct mlxsw_sp_nexthop_key key;
 	struct mlxsw_sp_nexthop *nh;
-	struct mlxsw_sp_rif *rif;
 
 	if (mlxsw_sp->router->aborted)
 		return;
@@ -1825,18 +2824,12 @@ static void mlxsw_sp_nexthop_event(struct mlxsw_sp *mlxsw_sp,
 	if (WARN_ON_ONCE(!nh))
 		return;
 
-	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, fib_nh->nh_dev);
-	if (!rif)
-		return;
-
 	switch (event) {
 	case FIB_EVENT_NH_ADD:
-		mlxsw_sp_nexthop_rif_init(nh, rif);
-		mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
+		mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh);
 		break;
 	case FIB_EVENT_NH_DEL:
-		mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
-		mlxsw_sp_nexthop_rif_fini(nh);
+		mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh);
 		break;
 	}
 
@@ -1849,14 +2842,20 @@ static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
 	struct mlxsw_sp_nexthop *nh, *tmp;
 
 	list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
-		mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
-		mlxsw_sp_nexthop_rif_fini(nh);
+		mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
 		mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
 	}
 }
 
+static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp,
+				   const struct fib_info *fi)
+{
+	return fi->fib_nh->nh_scope == RT_SCOPE_LINK ||
+	       mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, fi->fib_nh, NULL);
+}
+
 static struct mlxsw_sp_nexthop_group *
-mlxsw_sp_nexthop_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
+mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
 {
 	struct mlxsw_sp_nexthop_group *nh_grp;
 	struct mlxsw_sp_nexthop *nh;
@@ -1870,17 +2869,19 @@ mlxsw_sp_nexthop_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
 	nh_grp = kzalloc(alloc_size, GFP_KERNEL);
 	if (!nh_grp)
 		return ERR_PTR(-ENOMEM);
+	nh_grp->priv = fi;
 	INIT_LIST_HEAD(&nh_grp->fib_list);
-	nh_grp->gateway = fi->fib_nh->nh_scope == RT_SCOPE_LINK;
+	nh_grp->neigh_tbl = &arp_tbl;
+
+	nh_grp->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, fi);
 	nh_grp->count = fi->fib_nhs;
-	nh_grp->key.fi = fi;
 	fib_info_hold(fi);
 	for (i = 0; i < nh_grp->count; i++) {
 		nh = &nh_grp->nexthops[i];
 		fib_nh = &fi->fib_nh[i];
-		err = mlxsw_sp_nexthop_init(mlxsw_sp, nh_grp, nh, fib_nh);
+		err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh);
 		if (err)
-			goto err_nexthop_init;
+			goto err_nexthop4_init;
 	}
 	err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
 	if (err)
@@ -1889,19 +2890,19 @@ mlxsw_sp_nexthop_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
 	return nh_grp;
 
 err_nexthop_group_insert:
-err_nexthop_init:
+err_nexthop4_init:
 	for (i--; i >= 0; i--) {
 		nh = &nh_grp->nexthops[i];
-		mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
+		mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
 	}
-	fib_info_put(nh_grp->key.fi);
+	fib_info_put(fi);
 	kfree(nh_grp);
 	return ERR_PTR(err);
 }
 
 static void
-mlxsw_sp_nexthop_group_destroy(struct mlxsw_sp *mlxsw_sp,
-			       struct mlxsw_sp_nexthop_group *nh_grp)
+mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp,
+				struct mlxsw_sp_nexthop_group *nh_grp)
 {
 	struct mlxsw_sp_nexthop *nh;
 	int i;
@@ -1909,25 +2910,23 @@ mlxsw_sp_nexthop_group_destroy(struct mlxsw_sp *mlxsw_sp,
 	mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
 	for (i = 0; i < nh_grp->count; i++) {
 		nh = &nh_grp->nexthops[i];
-		mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
+		mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
 	}
 	mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
 	WARN_ON_ONCE(nh_grp->adj_index_valid);
-	fib_info_put(nh_grp->key.fi);
+	fib_info_put(mlxsw_sp_nexthop4_group_fi(nh_grp));
 	kfree(nh_grp);
 }
 
-static int mlxsw_sp_nexthop_group_get(struct mlxsw_sp *mlxsw_sp,
-				      struct mlxsw_sp_fib_entry *fib_entry,
-				      struct fib_info *fi)
+static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp,
+				       struct mlxsw_sp_fib_entry *fib_entry,
+				       struct fib_info *fi)
 {
-	struct mlxsw_sp_nexthop_group_key key;
 	struct mlxsw_sp_nexthop_group *nh_grp;
 
-	key.fi = fi;
-	nh_grp = mlxsw_sp_nexthop_group_lookup(mlxsw_sp, key);
+	nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi);
 	if (!nh_grp) {
-		nh_grp = mlxsw_sp_nexthop_group_create(mlxsw_sp, fi);
+		nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi);
 		if (IS_ERR(nh_grp))
 			return PTR_ERR(nh_grp);
 	}
@@ -1936,15 +2935,25 @@ static int mlxsw_sp_nexthop_group_get(struct mlxsw_sp *mlxsw_sp,
 	return 0;
 }
 
-static void mlxsw_sp_nexthop_group_put(struct mlxsw_sp *mlxsw_sp,
-				       struct mlxsw_sp_fib_entry *fib_entry)
+static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp,
+					struct mlxsw_sp_fib_entry *fib_entry)
 {
 	struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
 
 	list_del(&fib_entry->nexthop_group_node);
 	if (!list_empty(&nh_grp->fib_list))
 		return;
-	mlxsw_sp_nexthop_group_destroy(mlxsw_sp, nh_grp);
+	mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp);
+}
+
+static bool
+mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
+{
+	struct mlxsw_sp_fib4_entry *fib4_entry;
+
+	fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
+				  common);
+	return !fib4_entry->tos;
 }
 
 static bool
@@ -1952,29 +2961,133 @@ mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
 {
 	struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
 
-	if (fib_entry->params.tos)
-		return false;
+	switch (fib_entry->fib_node->fib->proto) {
+	case MLXSW_SP_L3_PROTO_IPV4:
+		if (!mlxsw_sp_fib4_entry_should_offload(fib_entry))
+			return false;
+		break;
+	case MLXSW_SP_L3_PROTO_IPV6:
+		break;
+	}
 
 	switch (fib_entry->type) {
 	case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
 		return !!nh_group->adj_index_valid;
 	case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
 		return !!nh_group->nh_rif;
+	case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
+		return true;
 	default:
 		return false;
 	}
 }
 
-static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
+static struct mlxsw_sp_nexthop *
+mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
+		     const struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
+{
+	int i;
+
+	for (i = 0; i < nh_grp->count; i++) {
+		struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
+		struct rt6_info *rt = mlxsw_sp_rt6->rt;
+
+		if (nh->rif && nh->rif->dev == rt->dst.dev &&
+		    ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr,
+				    &rt->rt6i_gateway))
+			return nh;
+		continue;
+	}
+
+	return NULL;
+}
+
+static void
+mlxsw_sp_fib4_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
+{
+	struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
+	int i;
+
+	if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL ||
+	    fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP) {
+		nh_grp->nexthops->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
+		return;
+	}
+
+	for (i = 0; i < nh_grp->count; i++) {
+		struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
+
+		if (nh->offloaded)
+			nh->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
+		else
+			nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
+	}
+}
+
+static void
+mlxsw_sp_fib4_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
 {
-	fib_entry->offloaded = true;
+	struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
+	int i;
+
+	for (i = 0; i < nh_grp->count; i++) {
+		struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
+
+		nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
+	}
+}
+
+static void
+mlxsw_sp_fib6_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
+{
+	struct mlxsw_sp_fib6_entry *fib6_entry;
+	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
+
+	fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
+				  common);
+
+	if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL) {
+		list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
+				 list)->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD;
+		return;
+	}
 
+	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
+		struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
+		struct mlxsw_sp_nexthop *nh;
+
+		nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6);
+		if (nh && nh->offloaded)
+			mlxsw_sp_rt6->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD;
+		else
+			mlxsw_sp_rt6->rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD;
+	}
+}
+
+static void
+mlxsw_sp_fib6_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
+{
+	struct mlxsw_sp_fib6_entry *fib6_entry;
+	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
+
+	fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
+				  common);
+	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
+		struct rt6_info *rt = mlxsw_sp_rt6->rt;
+
+		rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD;
+	}
+}
+
+static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
+{
 	switch (fib_entry->fib_node->fib->proto) {
 	case MLXSW_SP_L3_PROTO_IPV4:
-		fib_info_offload_inc(fib_entry->nh_group->key.fi);
+		mlxsw_sp_fib4_entry_offload_set(fib_entry);
 		break;
 	case MLXSW_SP_L3_PROTO_IPV6:
-		WARN_ON_ONCE(1);
+		mlxsw_sp_fib6_entry_offload_set(fib_entry);
+		break;
 	}
 }
 
@@ -1983,13 +3096,12 @@ mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
 {
 	switch (fib_entry->fib_node->fib->proto) {
 	case MLXSW_SP_L3_PROTO_IPV4:
-		fib_info_offload_dec(fib_entry->nh_group->key.fi);
+		mlxsw_sp_fib4_entry_offload_unset(fib_entry);
 		break;
 	case MLXSW_SP_L3_PROTO_IPV6:
-		WARN_ON_ONCE(1);
+		mlxsw_sp_fib6_entry_offload_unset(fib_entry);
+		break;
 	}
-
-	fib_entry->offloaded = false;
 }
 
 static void
@@ -1998,17 +3110,13 @@ mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
 {
 	switch (op) {
 	case MLXSW_REG_RALUE_OP_WRITE_DELETE:
-		if (!fib_entry->offloaded)
-			return;
 		return mlxsw_sp_fib_entry_offload_unset(fib_entry);
 	case MLXSW_REG_RALUE_OP_WRITE_WRITE:
 		if (err)
 			return;
-		if (mlxsw_sp_fib_entry_should_offload(fib_entry) &&
-		    !fib_entry->offloaded)
+		if (mlxsw_sp_fib_entry_should_offload(fib_entry))
 			mlxsw_sp_fib_entry_offload_set(fib_entry);
-		else if (!mlxsw_sp_fib_entry_should_offload(fib_entry) &&
-			 fib_entry->offloaded)
+		else if (!mlxsw_sp_fib_entry_should_offload(fib_entry))
 			mlxsw_sp_fib_entry_offload_unset(fib_entry);
 		return;
 	default:
@@ -2016,13 +3124,37 @@ mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
 	}
 }
 
-static int mlxsw_sp_fib_entry_op4_remote(struct mlxsw_sp *mlxsw_sp,
-					 struct mlxsw_sp_fib_entry *fib_entry,
-					 enum mlxsw_reg_ralue_op op)
+static void
+mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl,
+			      const struct mlxsw_sp_fib_entry *fib_entry,
+			      enum mlxsw_reg_ralue_op op)
 {
-	char ralue_pl[MLXSW_REG_RALUE_LEN];
 	struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
-	u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
+	enum mlxsw_reg_ralxx_protocol proto;
+	u32 *p_dip;
+
+	proto = (enum mlxsw_reg_ralxx_protocol) fib->proto;
+
+	switch (fib->proto) {
+	case MLXSW_SP_L3_PROTO_IPV4:
+		p_dip = (u32 *) fib_entry->fib_node->key.addr;
+		mlxsw_reg_ralue_pack4(ralue_pl, proto, op, fib->vr->id,
+				      fib_entry->fib_node->key.prefix_len,
+				      *p_dip);
+		break;
+	case MLXSW_SP_L3_PROTO_IPV6:
+		mlxsw_reg_ralue_pack6(ralue_pl, proto, op, fib->vr->id,
+				      fib_entry->fib_node->key.prefix_len,
+				      fib_entry->fib_node->key.addr);
+		break;
+	}
+}
+
+static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp,
+					struct mlxsw_sp_fib_entry *fib_entry,
+					enum mlxsw_reg_ralue_op op)
+{
+	char ralue_pl[MLXSW_REG_RALUE_LEN];
 	enum mlxsw_reg_ralue_trap_action trap_action;
 	u16 trap_id = 0;
 	u32 adjacency_index = 0;
@@ -2041,24 +3173,19 @@ static int mlxsw_sp_fib_entry_op4_remote(struct mlxsw_sp *mlxsw_sp,
 		trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
 	}
 
-	mlxsw_reg_ralue_pack4(ralue_pl,
-			      (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
-			      fib->vr->id, fib_entry->fib_node->key.prefix_len,
-			      *p_dip);
+	mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
 	mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
 					adjacency_index, ecmp_size);
 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
 }
 
-static int mlxsw_sp_fib_entry_op4_local(struct mlxsw_sp *mlxsw_sp,
-					struct mlxsw_sp_fib_entry *fib_entry,
-					enum mlxsw_reg_ralue_op op)
+static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp,
+				       struct mlxsw_sp_fib_entry *fib_entry,
+				       enum mlxsw_reg_ralue_op op)
 {
 	struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif;
-	struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
 	enum mlxsw_reg_ralue_trap_action trap_action;
 	char ralue_pl[MLXSW_REG_RALUE_LEN];
-	u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
 	u16 trap_id = 0;
 	u16 rif_index = 0;
 
@@ -2070,42 +3197,53 @@ static int mlxsw_sp_fib_entry_op4_local(struct mlxsw_sp *mlxsw_sp,
 		trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
 	}
 
-	mlxsw_reg_ralue_pack4(ralue_pl,
-			      (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
-			      fib->vr->id, fib_entry->fib_node->key.prefix_len,
-			      *p_dip);
+	mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
 	mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
 				       rif_index);
 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
 }
 
-static int mlxsw_sp_fib_entry_op4_trap(struct mlxsw_sp *mlxsw_sp,
-				       struct mlxsw_sp_fib_entry *fib_entry,
-				       enum mlxsw_reg_ralue_op op)
+static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp,
+				      struct mlxsw_sp_fib_entry *fib_entry,
+				      enum mlxsw_reg_ralue_op op)
 {
-	struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
 	char ralue_pl[MLXSW_REG_RALUE_LEN];
-	u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
 
-	mlxsw_reg_ralue_pack4(ralue_pl,
-			      (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
-			      fib->vr->id, fib_entry->fib_node->key.prefix_len,
-			      *p_dip);
+	mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
 	mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
 }
 
-static int mlxsw_sp_fib_entry_op4(struct mlxsw_sp *mlxsw_sp,
-				  struct mlxsw_sp_fib_entry *fib_entry,
-				  enum mlxsw_reg_ralue_op op)
+static int
+mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp,
+				 struct mlxsw_sp_fib_entry *fib_entry,
+				 enum mlxsw_reg_ralue_op op)
+{
+	struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry;
+	const struct mlxsw_sp_ipip_ops *ipip_ops;
+
+	if (WARN_ON(!ipip_entry))
+		return -EINVAL;
+
+	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
+	return ipip_ops->fib_entry_op(mlxsw_sp, ipip_entry, op,
+				      fib_entry->decap.tunnel_index);
+}
+
+static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
+				   struct mlxsw_sp_fib_entry *fib_entry,
+				   enum mlxsw_reg_ralue_op op)
 {
 	switch (fib_entry->type) {
 	case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
-		return mlxsw_sp_fib_entry_op4_remote(mlxsw_sp, fib_entry, op);
+		return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, fib_entry, op);
 	case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
-		return mlxsw_sp_fib_entry_op4_local(mlxsw_sp, fib_entry, op);
+		return mlxsw_sp_fib_entry_op_local(mlxsw_sp, fib_entry, op);
 	case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
-		return mlxsw_sp_fib_entry_op4_trap(mlxsw_sp, fib_entry, op);
+		return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, fib_entry, op);
+	case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
+		return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp,
+							fib_entry, op);
 	}
 	return -EINVAL;
 }
@@ -2114,16 +3252,10 @@ static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
 				 struct mlxsw_sp_fib_entry *fib_entry,
 				 enum mlxsw_reg_ralue_op op)
 {
-	int err = -EINVAL;
+	int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op);
 
-	switch (fib_entry->fib_node->fib->proto) {
-	case MLXSW_SP_L3_PROTO_IPV4:
-		err = mlxsw_sp_fib_entry_op4(mlxsw_sp, fib_entry, op);
-		break;
-	case MLXSW_SP_L3_PROTO_IPV6:
-		return err;
-	}
 	mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
+
 	return err;
 }
 
@@ -2146,11 +3278,23 @@ mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
 			     const struct fib_entry_notifier_info *fen_info,
 			     struct mlxsw_sp_fib_entry *fib_entry)
 {
+	union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) };
+	struct net_device *dev = fen_info->fi->fib_dev;
+	struct mlxsw_sp_ipip_entry *ipip_entry;
 	struct fib_info *fi = fen_info->fi;
 
 	switch (fen_info->type) {
-	case RTN_BROADCAST: /* fall through */
 	case RTN_LOCAL:
+		ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, dev,
+						 MLXSW_SP_L3_PROTO_IPV4, dip);
+		if (ipip_entry) {
+			fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
+			return mlxsw_sp_fib_entry_decap_init(mlxsw_sp,
+							     fib_entry,
+							     ipip_entry);
+		}
+		/* fall through */
+	case RTN_BROADCAST:
 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
 		return 0;
 	case RTN_UNREACHABLE: /* fall through */
@@ -2163,82 +3307,87 @@ mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
 		return 0;
 	case RTN_UNICAST:
-		if (fi->fib_nh->nh_scope != RT_SCOPE_LINK)
-			fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
-		else
+		if (mlxsw_sp_fi_is_gateway(mlxsw_sp, fi))
 			fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
+		else
+			fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
 		return 0;
 	default:
 		return -EINVAL;
 	}
 }
 
-static struct mlxsw_sp_fib_entry *
+static struct mlxsw_sp_fib4_entry *
 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
 			   struct mlxsw_sp_fib_node *fib_node,
 			   const struct fib_entry_notifier_info *fen_info)
 {
+	struct mlxsw_sp_fib4_entry *fib4_entry;
 	struct mlxsw_sp_fib_entry *fib_entry;
 	int err;
 
-	fib_entry = kzalloc(sizeof(*fib_entry), GFP_KERNEL);
-	if (!fib_entry) {
-		err = -ENOMEM;
-		goto err_fib_entry_alloc;
-	}
+	fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL);
+	if (!fib4_entry)
+		return ERR_PTR(-ENOMEM);
+	fib_entry = &fib4_entry->common;
 
 	err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
 	if (err)
 		goto err_fib4_entry_type_set;
 
-	err = mlxsw_sp_nexthop_group_get(mlxsw_sp, fib_entry, fen_info->fi);
+	err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi);
 	if (err)
-		goto err_nexthop_group_get;
+		goto err_nexthop4_group_get;
 
-	fib_entry->params.prio = fen_info->fi->fib_priority;
-	fib_entry->params.tb_id = fen_info->tb_id;
-	fib_entry->params.type = fen_info->type;
-	fib_entry->params.tos = fen_info->tos;
+	fib4_entry->prio = fen_info->fi->fib_priority;
+	fib4_entry->tb_id = fen_info->tb_id;
+	fib4_entry->type = fen_info->type;
+	fib4_entry->tos = fen_info->tos;
 
 	fib_entry->fib_node = fib_node;
 
-	return fib_entry;
+	return fib4_entry;
 
-err_nexthop_group_get:
+err_nexthop4_group_get:
 err_fib4_entry_type_set:
-	kfree(fib_entry);
-err_fib_entry_alloc:
+	kfree(fib4_entry);
 	return ERR_PTR(err);
 }
 
 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
-					struct mlxsw_sp_fib_entry *fib_entry)
+					struct mlxsw_sp_fib4_entry *fib4_entry)
 {
-	mlxsw_sp_nexthop_group_put(mlxsw_sp, fib_entry);
-	kfree(fib_entry);
+	mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
+	kfree(fib4_entry);
 }
 
-static struct mlxsw_sp_fib_node *
-mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
-		       const struct fib_entry_notifier_info *fen_info);
-
-static struct mlxsw_sp_fib_entry *
+static struct mlxsw_sp_fib4_entry *
 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
 			   const struct fib_entry_notifier_info *fen_info)
 {
-	struct mlxsw_sp_fib_entry *fib_entry;
+	struct mlxsw_sp_fib4_entry *fib4_entry;
 	struct mlxsw_sp_fib_node *fib_node;
+	struct mlxsw_sp_fib *fib;
+	struct mlxsw_sp_vr *vr;
 
-	fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
-	if (IS_ERR(fib_node))
+	vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id);
+	if (!vr)
 		return NULL;
+	fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
 
-	list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
-		if (fib_entry->params.tb_id == fen_info->tb_id &&
-		    fib_entry->params.tos == fen_info->tos &&
-		    fib_entry->params.type == fen_info->type &&
-		    fib_entry->nh_group->key.fi == fen_info->fi) {
-			return fib_entry;
+	fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
+					    sizeof(fen_info->dst),
+					    fen_info->dst_len);
+	if (!fib_node)
+		return NULL;
+
+	list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
+		if (fib4_entry->tb_id == fen_info->tb_id &&
+		    fib4_entry->tos == fen_info->tos &&
+		    fib4_entry->type == fen_info->type &&
+		    mlxsw_sp_nexthop4_group_fi(fib4_entry->common.nh_group) ==
+		    fen_info->fi) {
+			return fib4_entry;
 		}
 	}
 
@@ -2311,6 +3460,67 @@ mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
 				struct mlxsw_sp_fib_entry, list) == fib_entry;
 }
 
+static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp,
+				      struct mlxsw_sp_fib *fib,
+				      struct mlxsw_sp_fib_node *fib_node)
+{
+	struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
+	struct mlxsw_sp_lpm_tree *lpm_tree;
+	int err;
+
+	/* Since the tree is shared between all virtual routers we must
+	 * make sure it contains all the required prefix lengths. This
+	 * can be computed by either adding the new prefix length to the
+	 * existing prefix usage of a bound tree, or by aggregating the
+	 * prefix lengths across all virtual routers and adding the new
+	 * one as well.
+	 */
+	if (fib->lpm_tree)
+		mlxsw_sp_prefix_usage_cpy(&req_prefix_usage,
+					  &fib->lpm_tree->prefix_usage);
+	else
+		mlxsw_sp_vrs_prefixes(mlxsw_sp, fib->proto, &req_prefix_usage);
+	mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
+
+	lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
+					 fib->proto);
+	if (IS_ERR(lpm_tree))
+		return PTR_ERR(lpm_tree);
+
+	if (fib->lpm_tree && fib->lpm_tree->id == lpm_tree->id)
+		return 0;
+
+	err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
+	if (err)
+		return err;
+
+	return 0;
+}
+
+static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp,
+					 struct mlxsw_sp_fib *fib)
+{
+	struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
+	struct mlxsw_sp_lpm_tree *lpm_tree;
+
+	/* Aggregate prefix lengths across all virtual routers to make
+	 * sure we only have used prefix lengths in the LPM tree.
+	 */
+	mlxsw_sp_vrs_prefixes(mlxsw_sp, fib->proto, &req_prefix_usage);
+	lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
+					 fib->proto);
+	if (IS_ERR(lpm_tree))
+		goto err_tree_get;
+	mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
+
+err_tree_get:
+	if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage))
+		return;
+	mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
+	mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree);
+	fib->lpm_tree = NULL;
+}
+
 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
 {
 	unsigned char prefix_len = fib_node->key.prefix_len;
@@ -2333,8 +3543,6 @@ static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
 				  struct mlxsw_sp_fib_node *fib_node,
 				  struct mlxsw_sp_fib *fib)
 {
-	struct mlxsw_sp_prefix_usage req_prefix_usage;
-	struct mlxsw_sp_lpm_tree *lpm_tree;
 	int err;
 
 	err = mlxsw_sp_fib_node_insert(fib, fib_node);
@@ -2342,33 +3550,15 @@ static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
 		return err;
 	fib_node->fib = fib;
 
-	mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &fib->prefix_usage);
-	mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
-
-	if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
-		err = mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib,
-						 &req_prefix_usage);
-		if (err)
-			goto err_tree_check;
-	} else {
-		lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
-						 fib->proto);
-		if (IS_ERR(lpm_tree))
-			return PTR_ERR(lpm_tree);
-		fib->lpm_tree = lpm_tree;
-		err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
-		if (err)
-			goto err_tree_bind;
-	}
+	err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib, fib_node);
+	if (err)
+		goto err_fib_lpm_tree_link;
 
 	mlxsw_sp_fib_node_prefix_inc(fib_node);
 
 	return 0;
 
-err_tree_bind:
-	fib->lpm_tree = NULL;
-	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
-err_tree_check:
+err_fib_lpm_tree_link:
 	fib_node->fib = NULL;
 	mlxsw_sp_fib_node_remove(fib, fib_node);
 	return err;
@@ -2377,46 +3567,34 @@ err_tree_check:
 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
 				   struct mlxsw_sp_fib_node *fib_node)
 {
-	struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
 	struct mlxsw_sp_fib *fib = fib_node->fib;
 
 	mlxsw_sp_fib_node_prefix_dec(fib_node);
-
-	if (mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
-		mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
-		fib->lpm_tree = NULL;
-		mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
-	} else {
-		mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib, &fib->prefix_usage);
-	}
-
+	mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib);
 	fib_node->fib = NULL;
 	mlxsw_sp_fib_node_remove(fib, fib_node);
 }
 
 static struct mlxsw_sp_fib_node *
-mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
-		       const struct fib_entry_notifier_info *fen_info)
+mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr,
+		      size_t addr_len, unsigned char prefix_len,
+		      enum mlxsw_sp_l3proto proto)
 {
 	struct mlxsw_sp_fib_node *fib_node;
 	struct mlxsw_sp_fib *fib;
 	struct mlxsw_sp_vr *vr;
 	int err;
 
-	vr = mlxsw_sp_vr_get(mlxsw_sp, fen_info->tb_id);
+	vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id);
 	if (IS_ERR(vr))
 		return ERR_CAST(vr);
-	fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
+	fib = mlxsw_sp_vr_fib(vr, proto);
 
-	fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
-					    sizeof(fen_info->dst),
-					    fen_info->dst_len);
+	fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len);
 	if (fib_node)
 		return fib_node;
 
-	fib_node = mlxsw_sp_fib_node_create(fib, &fen_info->dst,
-					    sizeof(fen_info->dst),
-					    fen_info->dst_len);
+	fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len);
 	if (!fib_node) {
 		err = -ENOMEM;
 		goto err_fib_node_create;
@@ -2435,8 +3613,8 @@ err_fib_node_create:
 	return ERR_PTR(err);
 }
 
-static void mlxsw_sp_fib4_node_put(struct mlxsw_sp *mlxsw_sp,
-				   struct mlxsw_sp_fib_node *fib_node)
+static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp,
+				  struct mlxsw_sp_fib_node *fib_node)
 {
 	struct mlxsw_sp_vr *vr = fib_node->fib->vr;
 
@@ -2447,95 +3625,100 @@ static void mlxsw_sp_fib4_node_put(struct mlxsw_sp *mlxsw_sp,
 	mlxsw_sp_vr_put(vr);
 }
 
-static struct mlxsw_sp_fib_entry *
+static struct mlxsw_sp_fib4_entry *
 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
-			      const struct mlxsw_sp_fib_entry_params *params)
+			      const struct mlxsw_sp_fib4_entry *new4_entry)
 {
-	struct mlxsw_sp_fib_entry *fib_entry;
+	struct mlxsw_sp_fib4_entry *fib4_entry;
 
-	list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
-		if (fib_entry->params.tb_id > params->tb_id)
+	list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
+		if (fib4_entry->tb_id > new4_entry->tb_id)
 			continue;
-		if (fib_entry->params.tb_id != params->tb_id)
+		if (fib4_entry->tb_id != new4_entry->tb_id)
 			break;
-		if (fib_entry->params.tos > params->tos)
+		if (fib4_entry->tos > new4_entry->tos)
 			continue;
-		if (fib_entry->params.prio >= params->prio ||
-		    fib_entry->params.tos < params->tos)
-			return fib_entry;
+		if (fib4_entry->prio >= new4_entry->prio ||
+		    fib4_entry->tos < new4_entry->tos)
+			return fib4_entry;
 	}
 
 	return NULL;
 }
 
-static int mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib_entry *fib_entry,
-					  struct mlxsw_sp_fib_entry *new_entry)
+static int
+mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib4_entry *fib4_entry,
+			       struct mlxsw_sp_fib4_entry *new4_entry)
 {
 	struct mlxsw_sp_fib_node *fib_node;
 
-	if (WARN_ON(!fib_entry))
+	if (WARN_ON(!fib4_entry))
 		return -EINVAL;
 
-	fib_node = fib_entry->fib_node;
-	list_for_each_entry_from(fib_entry, &fib_node->entry_list, list) {
-		if (fib_entry->params.tb_id != new_entry->params.tb_id ||
-		    fib_entry->params.tos != new_entry->params.tos ||
-		    fib_entry->params.prio != new_entry->params.prio)
+	fib_node = fib4_entry->common.fib_node;
+	list_for_each_entry_from(fib4_entry, &fib_node->entry_list,
+				 common.list) {
+		if (fib4_entry->tb_id != new4_entry->tb_id ||
+		    fib4_entry->tos != new4_entry->tos ||
+		    fib4_entry->prio != new4_entry->prio)
 			break;
 	}
 
-	list_add_tail(&new_entry->list, &fib_entry->list);
+	list_add_tail(&new4_entry->common.list, &fib4_entry->common.list);
 	return 0;
 }
 
 static int
-mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib_node *fib_node,
-			       struct mlxsw_sp_fib_entry *new_entry,
+mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib4_entry *new4_entry,
 			       bool replace, bool append)
 {
-	struct mlxsw_sp_fib_entry *fib_entry;
+	struct mlxsw_sp_fib_node *fib_node = new4_entry->common.fib_node;
+	struct mlxsw_sp_fib4_entry *fib4_entry;
 
-	fib_entry = mlxsw_sp_fib4_node_entry_find(fib_node, &new_entry->params);
+	fib4_entry = mlxsw_sp_fib4_node_entry_find(fib_node, new4_entry);
 
 	if (append)
-		return mlxsw_sp_fib4_node_list_append(fib_entry, new_entry);
-	if (replace && WARN_ON(!fib_entry))
+		return mlxsw_sp_fib4_node_list_append(fib4_entry, new4_entry);
+	if (replace && WARN_ON(!fib4_entry))
 		return -EINVAL;
 
 	/* Insert new entry before replaced one, so that we can later
 	 * remove the second.
 	 */
-	if (fib_entry) {
-		list_add_tail(&new_entry->list, &fib_entry->list);
+	if (fib4_entry) {
+		list_add_tail(&new4_entry->common.list,
+			      &fib4_entry->common.list);
 	} else {
-		struct mlxsw_sp_fib_entry *last;
+		struct mlxsw_sp_fib4_entry *last;
 
-		list_for_each_entry(last, &fib_node->entry_list, list) {
-			if (new_entry->params.tb_id > last->params.tb_id)
+		list_for_each_entry(last, &fib_node->entry_list, common.list) {
+			if (new4_entry->tb_id > last->tb_id)
 				break;
-			fib_entry = last;
+			fib4_entry = last;
 		}
 
-		if (fib_entry)
-			list_add(&new_entry->list, &fib_entry->list);
+		if (fib4_entry)
+			list_add(&new4_entry->common.list,
+				 &fib4_entry->common.list);
 		else
-			list_add(&new_entry->list, &fib_node->entry_list);
+			list_add(&new4_entry->common.list,
+				 &fib_node->entry_list);
 	}
 
 	return 0;
 }
 
 static void
-mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib_entry *fib_entry)
+mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib4_entry *fib4_entry)
 {
-	list_del(&fib_entry->list);
+	list_del(&fib4_entry->common.list);
 }
 
-static int
-mlxsw_sp_fib4_node_entry_add(struct mlxsw_sp *mlxsw_sp,
-			     const struct mlxsw_sp_fib_node *fib_node,
-			     struct mlxsw_sp_fib_entry *fib_entry)
+static int mlxsw_sp_fib_node_entry_add(struct mlxsw_sp *mlxsw_sp,
+				       struct mlxsw_sp_fib_entry *fib_entry)
 {
+	struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
+
 	if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
 		return 0;
 
@@ -2552,11 +3735,11 @@ mlxsw_sp_fib4_node_entry_add(struct mlxsw_sp *mlxsw_sp,
 	return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
 }
 
-static void
-mlxsw_sp_fib4_node_entry_del(struct mlxsw_sp *mlxsw_sp,
-			     const struct mlxsw_sp_fib_node *fib_node,
-			     struct mlxsw_sp_fib_entry *fib_entry)
+static void mlxsw_sp_fib_node_entry_del(struct mlxsw_sp *mlxsw_sp,
+					struct mlxsw_sp_fib_entry *fib_entry)
 {
+	struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
+
 	if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
 		return;
 
@@ -2574,54 +3757,53 @@ mlxsw_sp_fib4_node_entry_del(struct mlxsw_sp *mlxsw_sp,
 }
 
 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
-					 struct mlxsw_sp_fib_entry *fib_entry,
+					 struct mlxsw_sp_fib4_entry *fib4_entry,
 					 bool replace, bool append)
 {
-	struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
 	int err;
 
-	err = mlxsw_sp_fib4_node_list_insert(fib_node, fib_entry, replace,
-					     append);
+	err = mlxsw_sp_fib4_node_list_insert(fib4_entry, replace, append);
 	if (err)
 		return err;
 
-	err = mlxsw_sp_fib4_node_entry_add(mlxsw_sp, fib_node, fib_entry);
+	err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib4_entry->common);
 	if (err)
-		goto err_fib4_node_entry_add;
+		goto err_fib_node_entry_add;
 
 	return 0;
 
-err_fib4_node_entry_add:
-	mlxsw_sp_fib4_node_list_remove(fib_entry);
+err_fib_node_entry_add:
+	mlxsw_sp_fib4_node_list_remove(fib4_entry);
 	return err;
 }
 
 static void
 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
-				struct mlxsw_sp_fib_entry *fib_entry)
+				struct mlxsw_sp_fib4_entry *fib4_entry)
 {
-	struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
+	mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib4_entry->common);
+	mlxsw_sp_fib4_node_list_remove(fib4_entry);
 
-	mlxsw_sp_fib4_node_entry_del(mlxsw_sp, fib_node, fib_entry);
-	mlxsw_sp_fib4_node_list_remove(fib_entry);
+	if (fib4_entry->common.type == MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP)
+		mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, &fib4_entry->common);
 }
 
 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
-					struct mlxsw_sp_fib_entry *fib_entry,
+					struct mlxsw_sp_fib4_entry *fib4_entry,
 					bool replace)
 {
-	struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
-	struct mlxsw_sp_fib_entry *replaced;
+	struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
+	struct mlxsw_sp_fib4_entry *replaced;
 
 	if (!replace)
 		return;
 
 	/* We inserted the new entry before replaced one */
-	replaced = list_next_entry(fib_entry, list);
+	replaced = list_next_entry(fib4_entry, common.list);
 
 	mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
 	mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
-	mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
+	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
 }
 
 static int
@@ -2629,76 +3811,774 @@ mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
 			 const struct fib_entry_notifier_info *fen_info,
 			 bool replace, bool append)
 {
-	struct mlxsw_sp_fib_entry *fib_entry;
+	struct mlxsw_sp_fib4_entry *fib4_entry;
 	struct mlxsw_sp_fib_node *fib_node;
 	int err;
 
 	if (mlxsw_sp->router->aborted)
 		return 0;
 
-	fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
+	fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id,
+					 &fen_info->dst, sizeof(fen_info->dst),
+					 fen_info->dst_len,
+					 MLXSW_SP_L3_PROTO_IPV4);
 	if (IS_ERR(fib_node)) {
 		dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
 		return PTR_ERR(fib_node);
 	}
 
-	fib_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
-	if (IS_ERR(fib_entry)) {
+	fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
+	if (IS_ERR(fib4_entry)) {
 		dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
-		err = PTR_ERR(fib_entry);
+		err = PTR_ERR(fib4_entry);
 		goto err_fib4_entry_create;
 	}
 
-	err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib_entry, replace,
+	err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib4_entry, replace,
 					    append);
 	if (err) {
 		dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
 		goto err_fib4_node_entry_link;
 	}
 
-	mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib_entry, replace);
+	mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib4_entry, replace);
 
 	return 0;
 
 err_fib4_node_entry_link:
-	mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
+	mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
 err_fib4_entry_create:
-	mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
+	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
 	return err;
 }
 
 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
 				     struct fib_entry_notifier_info *fen_info)
 {
+	struct mlxsw_sp_fib4_entry *fib4_entry;
+	struct mlxsw_sp_fib_node *fib_node;
+
+	if (mlxsw_sp->router->aborted)
+		return;
+
+	fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
+	if (WARN_ON(!fib4_entry))
+		return;
+	fib_node = fib4_entry->common.fib_node;
+
+	mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
+	mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
+	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
+}
+
+static bool mlxsw_sp_fib6_rt_should_ignore(const struct rt6_info *rt)
+{
+	/* Packets with link-local destination IP arriving to the router
+	 * are trapped to the CPU, so no need to program specific routes
+	 * for them.
+	 */
+	if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_LINKLOCAL)
+		return true;
+
+	/* Multicast routes aren't supported, so ignore them. Neighbour
+	 * Discovery packets are specifically trapped.
+	 */
+	if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_MULTICAST)
+		return true;
+
+	/* Cloned routes are irrelevant in the forwarding path. */
+	if (rt->rt6i_flags & RTF_CACHE)
+		return true;
+
+	return false;
+}
+
+static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct rt6_info *rt)
+{
+	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
+
+	mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL);
+	if (!mlxsw_sp_rt6)
+		return ERR_PTR(-ENOMEM);
+
+	/* In case of route replace, replaced route is deleted with
+	 * no notification. Take reference to prevent accessing freed
+	 * memory.
+	 */
+	mlxsw_sp_rt6->rt = rt;
+	rt6_hold(rt);
+
+	return mlxsw_sp_rt6;
+}
+
+#if IS_ENABLED(CONFIG_IPV6)
+static void mlxsw_sp_rt6_release(struct rt6_info *rt)
+{
+	rt6_release(rt);
+}
+#else
+static void mlxsw_sp_rt6_release(struct rt6_info *rt)
+{
+}
+#endif
+
+static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
+{
+	mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt);
+	kfree(mlxsw_sp_rt6);
+}
+
+static bool mlxsw_sp_fib6_rt_can_mp(const struct rt6_info *rt)
+{
+	/* RTF_CACHE routes are ignored */
+	return (rt->rt6i_flags & (RTF_GATEWAY | RTF_ADDRCONF)) == RTF_GATEWAY;
+}
+
+static struct rt6_info *
+mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry)
+{
+	return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
+				list)->rt;
+}
+
+static struct mlxsw_sp_fib6_entry *
+mlxsw_sp_fib6_node_mp_entry_find(const struct mlxsw_sp_fib_node *fib_node,
+				 const struct rt6_info *nrt, bool replace)
+{
+	struct mlxsw_sp_fib6_entry *fib6_entry;
+
+	if (!mlxsw_sp_fib6_rt_can_mp(nrt) || replace)
+		return NULL;
+
+	list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
+		struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
+
+		/* RT6_TABLE_LOCAL and RT6_TABLE_MAIN share the same
+		 * virtual router.
+		 */
+		if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id)
+			continue;
+		if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id)
+			break;
+		if (rt->rt6i_metric < nrt->rt6i_metric)
+			continue;
+		if (rt->rt6i_metric == nrt->rt6i_metric &&
+		    mlxsw_sp_fib6_rt_can_mp(rt))
+			return fib6_entry;
+		if (rt->rt6i_metric > nrt->rt6i_metric)
+			break;
+	}
+
+	return NULL;
+}
+
+static struct mlxsw_sp_rt6 *
+mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry,
+			    const struct rt6_info *rt)
+{
+	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
+
+	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
+		if (mlxsw_sp_rt6->rt == rt)
+			return mlxsw_sp_rt6;
+	}
+
+	return NULL;
+}
+
+static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp,
+					const struct rt6_info *rt,
+					enum mlxsw_sp_ipip_type *ret)
+{
+	return rt->dst.dev &&
+	       mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->dst.dev, ret);
+}
+
+static int mlxsw_sp_nexthop6_type_init(struct mlxsw_sp *mlxsw_sp,
+				       struct mlxsw_sp_nexthop_group *nh_grp,
+				       struct mlxsw_sp_nexthop *nh,
+				       const struct rt6_info *rt)
+{
+	struct mlxsw_sp_router *router = mlxsw_sp->router;
+	struct net_device *dev = rt->dst.dev;
+	enum mlxsw_sp_ipip_type ipipt;
+	struct mlxsw_sp_rif *rif;
+	int err;
+
+	if (mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, &ipipt) &&
+	    router->ipip_ops_arr[ipipt]->can_offload(mlxsw_sp, dev,
+						     MLXSW_SP_L3_PROTO_IPV6)) {
+		nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
+		return mlxsw_sp_nexthop_ipip_init(mlxsw_sp, ipipt, nh, dev);
+	}
+
+	nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
+	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
+	if (!rif)
+		return 0;
+	mlxsw_sp_nexthop_rif_init(nh, rif);
+
+	err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
+	if (err)
+		goto err_nexthop_neigh_init;
+
+	return 0;
+
+err_nexthop_neigh_init:
+	mlxsw_sp_nexthop_rif_fini(nh);
+	return err;
+}
+
+static void mlxsw_sp_nexthop6_type_fini(struct mlxsw_sp *mlxsw_sp,
+					struct mlxsw_sp_nexthop *nh)
+{
+	mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
+}
+
+static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
+				  struct mlxsw_sp_nexthop_group *nh_grp,
+				  struct mlxsw_sp_nexthop *nh,
+				  const struct rt6_info *rt)
+{
+	struct net_device *dev = rt->dst.dev;
+
+	nh->nh_grp = nh_grp;
+	memcpy(&nh->gw_addr, &rt->rt6i_gateway, sizeof(nh->gw_addr));
+
+	if (!dev)
+		return 0;
+	nh->ifindex = dev->ifindex;
+
+	return mlxsw_sp_nexthop6_type_init(mlxsw_sp, nh_grp, nh, rt);
+}
+
+static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp,
+				   struct mlxsw_sp_nexthop *nh)
+{
+	mlxsw_sp_nexthop6_type_fini(mlxsw_sp, nh);
+}
+
+static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp,
+				    const struct rt6_info *rt)
+{
+	return rt->rt6i_flags & RTF_GATEWAY ||
+	       mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL);
+}
+
+static struct mlxsw_sp_nexthop_group *
+mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp,
+			       struct mlxsw_sp_fib6_entry *fib6_entry)
+{
+	struct mlxsw_sp_nexthop_group *nh_grp;
+	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
+	struct mlxsw_sp_nexthop *nh;
+	size_t alloc_size;
+	int i = 0;
+	int err;
+
+	alloc_size = sizeof(*nh_grp) +
+		     fib6_entry->nrt6 * sizeof(struct mlxsw_sp_nexthop);
+	nh_grp = kzalloc(alloc_size, GFP_KERNEL);
+	if (!nh_grp)
+		return ERR_PTR(-ENOMEM);
+	INIT_LIST_HEAD(&nh_grp->fib_list);
+#if IS_ENABLED(CONFIG_IPV6)
+	nh_grp->neigh_tbl = &nd_tbl;
+#endif
+	mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list,
+					struct mlxsw_sp_rt6, list);
+	nh_grp->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt);
+	nh_grp->count = fib6_entry->nrt6;
+	for (i = 0; i < nh_grp->count; i++) {
+		struct rt6_info *rt = mlxsw_sp_rt6->rt;
+
+		nh = &nh_grp->nexthops[i];
+		err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt);
+		if (err)
+			goto err_nexthop6_init;
+		mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list);
+	}
+
+	err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
+	if (err)
+		goto err_nexthop_group_insert;
+
+	mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
+	return nh_grp;
+
+err_nexthop_group_insert:
+err_nexthop6_init:
+	for (i--; i >= 0; i--) {
+		nh = &nh_grp->nexthops[i];
+		mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
+	}
+	kfree(nh_grp);
+	return ERR_PTR(err);
+}
+
+static void
+mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp,
+				struct mlxsw_sp_nexthop_group *nh_grp)
+{
+	struct mlxsw_sp_nexthop *nh;
+	int i = nh_grp->count;
+
+	mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
+	for (i--; i >= 0; i--) {
+		nh = &nh_grp->nexthops[i];
+		mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
+	}
+	mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
+	WARN_ON(nh_grp->adj_index_valid);
+	kfree(nh_grp);
+}
+
+static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp,
+				       struct mlxsw_sp_fib6_entry *fib6_entry)
+{
+	struct mlxsw_sp_nexthop_group *nh_grp;
+
+	nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry);
+	if (!nh_grp) {
+		nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry);
+		if (IS_ERR(nh_grp))
+			return PTR_ERR(nh_grp);
+	}
+
+	list_add_tail(&fib6_entry->common.nexthop_group_node,
+		      &nh_grp->fib_list);
+	fib6_entry->common.nh_group = nh_grp;
+
+	return 0;
+}
+
+static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp,
+					struct mlxsw_sp_fib_entry *fib_entry)
+{
+	struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
+
+	list_del(&fib_entry->nexthop_group_node);
+	if (!list_empty(&nh_grp->fib_list))
+		return;
+	mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp);
+}
+
+static int
+mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp,
+			       struct mlxsw_sp_fib6_entry *fib6_entry)
+{
+	struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group;
+	int err;
+
+	fib6_entry->common.nh_group = NULL;
+	list_del(&fib6_entry->common.nexthop_group_node);
+
+	err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
+	if (err)
+		goto err_nexthop6_group_get;
+
+	/* In case this entry is offloaded, then the adjacency index
+	 * currently associated with it in the device's table is that
+	 * of the old group. Start using the new one instead.
+	 */
+	err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
+	if (err)
+		goto err_fib_node_entry_add;
+
+	if (list_empty(&old_nh_grp->fib_list))
+		mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp);
+
+	return 0;
+
+err_fib_node_entry_add:
+	mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
+err_nexthop6_group_get:
+	list_add_tail(&fib6_entry->common.nexthop_group_node,
+		      &old_nh_grp->fib_list);
+	fib6_entry->common.nh_group = old_nh_grp;
+	return err;
+}
+
+static int
+mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp,
+				struct mlxsw_sp_fib6_entry *fib6_entry,
+				struct rt6_info *rt)
+{
+	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
+	int err;
+
+	mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
+	if (IS_ERR(mlxsw_sp_rt6))
+		return PTR_ERR(mlxsw_sp_rt6);
+
+	list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
+	fib6_entry->nrt6++;
+
+	err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
+	if (err)
+		goto err_nexthop6_group_update;
+
+	return 0;
+
+err_nexthop6_group_update:
+	fib6_entry->nrt6--;
+	list_del(&mlxsw_sp_rt6->list);
+	mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
+	return err;
+}
+
+static void
+mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp,
+				struct mlxsw_sp_fib6_entry *fib6_entry,
+				struct rt6_info *rt)
+{
+	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
+
+	mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt);
+	if (WARN_ON(!mlxsw_sp_rt6))
+		return;
+
+	fib6_entry->nrt6--;
+	list_del(&mlxsw_sp_rt6->list);
+	mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
+	mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
+}
+
+static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp,
+					 struct mlxsw_sp_fib_entry *fib_entry,
+					 const struct rt6_info *rt)
+{
+	/* Packets hitting RTF_REJECT routes need to be discarded by the
+	 * stack. We can rely on their destination device not having a
+	 * RIF (it's the loopback device) and can thus use action type
+	 * local, which will cause them to be trapped with a lower
+	 * priority than packets that need to be locally received.
+	 */
+	if (rt->rt6i_flags & (RTF_LOCAL | RTF_ANYCAST))
+		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
+	else if (rt->rt6i_flags & RTF_REJECT)
+		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
+	else if (mlxsw_sp_rt6_is_gateway(mlxsw_sp, rt))
+		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
+	else
+		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
+}
+
+static void
+mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry)
+{
+	struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp;
+
+	list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list,
+				 list) {
+		fib6_entry->nrt6--;
+		list_del(&mlxsw_sp_rt6->list);
+		mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
+	}
+}
+
+static struct mlxsw_sp_fib6_entry *
+mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp,
+			   struct mlxsw_sp_fib_node *fib_node,
+			   struct rt6_info *rt)
+{
+	struct mlxsw_sp_fib6_entry *fib6_entry;
 	struct mlxsw_sp_fib_entry *fib_entry;
+	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
+	int err;
+
+	fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL);
+	if (!fib6_entry)
+		return ERR_PTR(-ENOMEM);
+	fib_entry = &fib6_entry->common;
+
+	mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
+	if (IS_ERR(mlxsw_sp_rt6)) {
+		err = PTR_ERR(mlxsw_sp_rt6);
+		goto err_rt6_create;
+	}
+
+	mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, mlxsw_sp_rt6->rt);
+
+	INIT_LIST_HEAD(&fib6_entry->rt6_list);
+	list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
+	fib6_entry->nrt6 = 1;
+	err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
+	if (err)
+		goto err_nexthop6_group_get;
+
+	fib_entry->fib_node = fib_node;
+
+	return fib6_entry;
+
+err_nexthop6_group_get:
+	list_del(&mlxsw_sp_rt6->list);
+	mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
+err_rt6_create:
+	kfree(fib6_entry);
+	return ERR_PTR(err);
+}
+
+static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp,
+					struct mlxsw_sp_fib6_entry *fib6_entry)
+{
+	mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
+	mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry);
+	WARN_ON(fib6_entry->nrt6);
+	kfree(fib6_entry);
+}
+
+static struct mlxsw_sp_fib6_entry *
+mlxsw_sp_fib6_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
+			      const struct rt6_info *nrt, bool replace)
+{
+	struct mlxsw_sp_fib6_entry *fib6_entry, *fallback = NULL;
+
+	list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
+		struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
+
+		if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id)
+			continue;
+		if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id)
+			break;
+		if (replace && rt->rt6i_metric == nrt->rt6i_metric) {
+			if (mlxsw_sp_fib6_rt_can_mp(rt) ==
+			    mlxsw_sp_fib6_rt_can_mp(nrt))
+				return fib6_entry;
+			if (mlxsw_sp_fib6_rt_can_mp(nrt))
+				fallback = fallback ?: fib6_entry;
+		}
+		if (rt->rt6i_metric > nrt->rt6i_metric)
+			return fallback ?: fib6_entry;
+	}
+
+	return fallback;
+}
+
+static int
+mlxsw_sp_fib6_node_list_insert(struct mlxsw_sp_fib6_entry *new6_entry,
+			       bool replace)
+{
+	struct mlxsw_sp_fib_node *fib_node = new6_entry->common.fib_node;
+	struct rt6_info *nrt = mlxsw_sp_fib6_entry_rt(new6_entry);
+	struct mlxsw_sp_fib6_entry *fib6_entry;
+
+	fib6_entry = mlxsw_sp_fib6_node_entry_find(fib_node, nrt, replace);
+
+	if (replace && WARN_ON(!fib6_entry))
+		return -EINVAL;
+
+	if (fib6_entry) {
+		list_add_tail(&new6_entry->common.list,
+			      &fib6_entry->common.list);
+	} else {
+		struct mlxsw_sp_fib6_entry *last;
+
+		list_for_each_entry(last, &fib_node->entry_list, common.list) {
+			struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(last);
+
+			if (nrt->rt6i_table->tb6_id > rt->rt6i_table->tb6_id)
+				break;
+			fib6_entry = last;
+		}
+
+		if (fib6_entry)
+			list_add(&new6_entry->common.list,
+				 &fib6_entry->common.list);
+		else
+			list_add(&new6_entry->common.list,
+				 &fib_node->entry_list);
+	}
+
+	return 0;
+}
+
+static void
+mlxsw_sp_fib6_node_list_remove(struct mlxsw_sp_fib6_entry *fib6_entry)
+{
+	list_del(&fib6_entry->common.list);
+}
+
+static int mlxsw_sp_fib6_node_entry_link(struct mlxsw_sp *mlxsw_sp,
+					 struct mlxsw_sp_fib6_entry *fib6_entry,
+					 bool replace)
+{
+	int err;
+
+	err = mlxsw_sp_fib6_node_list_insert(fib6_entry, replace);
+	if (err)
+		return err;
+
+	err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
+	if (err)
+		goto err_fib_node_entry_add;
+
+	return 0;
+
+err_fib_node_entry_add:
+	mlxsw_sp_fib6_node_list_remove(fib6_entry);
+	return err;
+}
+
+static void
+mlxsw_sp_fib6_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
+				struct mlxsw_sp_fib6_entry *fib6_entry)
+{
+	mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib6_entry->common);
+	mlxsw_sp_fib6_node_list_remove(fib6_entry);
+}
+
+static struct mlxsw_sp_fib6_entry *
+mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp,
+			   const struct rt6_info *rt)
+{
+	struct mlxsw_sp_fib6_entry *fib6_entry;
+	struct mlxsw_sp_fib_node *fib_node;
+	struct mlxsw_sp_fib *fib;
+	struct mlxsw_sp_vr *vr;
+
+	vr = mlxsw_sp_vr_find(mlxsw_sp, rt->rt6i_table->tb6_id);
+	if (!vr)
+		return NULL;
+	fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6);
+
+	fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->rt6i_dst.addr,
+					    sizeof(rt->rt6i_dst.addr),
+					    rt->rt6i_dst.plen);
+	if (!fib_node)
+		return NULL;
+
+	list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
+		struct rt6_info *iter_rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
+
+		if (rt->rt6i_table->tb6_id == iter_rt->rt6i_table->tb6_id &&
+		    rt->rt6i_metric == iter_rt->rt6i_metric &&
+		    mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt))
+			return fib6_entry;
+	}
+
+	return NULL;
+}
+
+static void mlxsw_sp_fib6_entry_replace(struct mlxsw_sp *mlxsw_sp,
+					struct mlxsw_sp_fib6_entry *fib6_entry,
+					bool replace)
+{
+	struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
+	struct mlxsw_sp_fib6_entry *replaced;
+
+	if (!replace)
+		return;
+
+	replaced = list_next_entry(fib6_entry, common.list);
+
+	mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, replaced);
+	mlxsw_sp_fib6_entry_destroy(mlxsw_sp, replaced);
+	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
+}
+
+static int mlxsw_sp_router_fib6_add(struct mlxsw_sp *mlxsw_sp,
+				    struct rt6_info *rt, bool replace)
+{
+	struct mlxsw_sp_fib6_entry *fib6_entry;
+	struct mlxsw_sp_fib_node *fib_node;
+	int err;
+
+	if (mlxsw_sp->router->aborted)
+		return 0;
+
+	if (rt->rt6i_src.plen)
+		return -EINVAL;
+
+	if (mlxsw_sp_fib6_rt_should_ignore(rt))
+		return 0;
+
+	fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->rt6i_table->tb6_id,
+					 &rt->rt6i_dst.addr,
+					 sizeof(rt->rt6i_dst.addr),
+					 rt->rt6i_dst.plen,
+					 MLXSW_SP_L3_PROTO_IPV6);
+	if (IS_ERR(fib_node))
+		return PTR_ERR(fib_node);
+
+	/* Before creating a new entry, try to append route to an existing
+	 * multipath entry.
+	 */
+	fib6_entry = mlxsw_sp_fib6_node_mp_entry_find(fib_node, rt, replace);
+	if (fib6_entry) {
+		err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, fib6_entry, rt);
+		if (err)
+			goto err_fib6_entry_nexthop_add;
+		return 0;
+	}
+
+	fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt);
+	if (IS_ERR(fib6_entry)) {
+		err = PTR_ERR(fib6_entry);
+		goto err_fib6_entry_create;
+	}
+
+	err = mlxsw_sp_fib6_node_entry_link(mlxsw_sp, fib6_entry, replace);
+	if (err)
+		goto err_fib6_node_entry_link;
+
+	mlxsw_sp_fib6_entry_replace(mlxsw_sp, fib6_entry, replace);
+
+	return 0;
+
+err_fib6_node_entry_link:
+	mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
+err_fib6_entry_create:
+err_fib6_entry_nexthop_add:
+	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
+	return err;
+}
+
+static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp,
+				     struct rt6_info *rt)
+{
+	struct mlxsw_sp_fib6_entry *fib6_entry;
 	struct mlxsw_sp_fib_node *fib_node;
 
 	if (mlxsw_sp->router->aborted)
 		return;
 
-	fib_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
-	if (WARN_ON(!fib_entry))
+	if (mlxsw_sp_fib6_rt_should_ignore(rt))
 		return;
-	fib_node = fib_entry->fib_node;
 
-	mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
-	mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
-	mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
+	fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt);
+	if (WARN_ON(!fib6_entry))
+		return;
+
+	/* If route is part of a multipath entry, but not the last one
+	 * removed, then only reduce its nexthop group.
+	 */
+	if (!list_is_singular(&fib6_entry->rt6_list)) {
+		mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, fib6_entry, rt);
+		return;
+	}
+
+	fib_node = fib6_entry->common.fib_node;
+
+	mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
+	mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
+	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
 }
 
-static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
+static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp,
+					    enum mlxsw_reg_ralxx_protocol proto,
+					    u8 tree_id)
 {
 	char ralta_pl[MLXSW_REG_RALTA_LEN];
 	char ralst_pl[MLXSW_REG_RALST_LEN];
 	int i, err;
 
-	mlxsw_reg_ralta_pack(ralta_pl, true, MLXSW_REG_RALXX_PROTOCOL_IPV4,
-			     MLXSW_SP_LPM_TREE_MIN);
+	mlxsw_reg_ralta_pack(ralta_pl, true, proto, tree_id);
 	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 	if (err)
 		return err;
 
-	mlxsw_reg_ralst_pack(ralst_pl, 0xff, MLXSW_SP_LPM_TREE_MIN);
+	mlxsw_reg_ralst_pack(ralst_pl, 0xff, tree_id);
 	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
 	if (err)
 		return err;
@@ -2708,20 +4588,14 @@ static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
 		char raltb_pl[MLXSW_REG_RALTB_LEN];
 		char ralue_pl[MLXSW_REG_RALUE_LEN];
 
-		if (!mlxsw_sp_vr_is_used(vr))
-			continue;
-
-		mlxsw_reg_raltb_pack(raltb_pl, vr->id,
-				     MLXSW_REG_RALXX_PROTOCOL_IPV4,
-				     MLXSW_SP_LPM_TREE_MIN);
+		mlxsw_reg_raltb_pack(raltb_pl, vr->id, proto, tree_id);
 		err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
 				      raltb_pl);
 		if (err)
 			return err;
 
-		mlxsw_reg_ralue_pack4(ralue_pl, MLXSW_SP_L3_PROTO_IPV4,
-				      MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0,
-				      0);
+		mlxsw_reg_ralue_pack(ralue_pl, proto,
+				     MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0);
 		mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
 		err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
 				      ralue_pl);
@@ -2732,17 +4606,33 @@ static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
 	return 0;
 }
 
+static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
+{
+	enum mlxsw_reg_ralxx_protocol proto = MLXSW_REG_RALXX_PROTOCOL_IPV4;
+	int err;
+
+	err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
+					       MLXSW_SP_LPM_TREE_MIN);
+	if (err)
+		return err;
+
+	proto = MLXSW_REG_RALXX_PROTOCOL_IPV6;
+	return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
+						MLXSW_SP_LPM_TREE_MIN + 1);
+}
+
 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
 				     struct mlxsw_sp_fib_node *fib_node)
 {
-	struct mlxsw_sp_fib_entry *fib_entry, *tmp;
+	struct mlxsw_sp_fib4_entry *fib4_entry, *tmp;
 
-	list_for_each_entry_safe(fib_entry, tmp, &fib_node->entry_list, list) {
-		bool do_break = &tmp->list == &fib_node->entry_list;
+	list_for_each_entry_safe(fib4_entry, tmp, &fib_node->entry_list,
+				 common.list) {
+		bool do_break = &tmp->common.list == &fib_node->entry_list;
 
-		mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
-		mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
-		mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
+		mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
+		mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
+		mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
 		/* Break when entry list is empty and node was freed.
 		 * Otherwise, we'll access freed memory in the next
 		 * iteration.
@@ -2752,6 +4642,23 @@ static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
 	}
 }
 
+static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp,
+				     struct mlxsw_sp_fib_node *fib_node)
+{
+	struct mlxsw_sp_fib6_entry *fib6_entry, *tmp;
+
+	list_for_each_entry_safe(fib6_entry, tmp, &fib_node->entry_list,
+				 common.list) {
+		bool do_break = &tmp->common.list == &fib_node->entry_list;
+
+		mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
+		mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
+		mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
+		if (do_break)
+			break;
+	}
+}
+
 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
 				    struct mlxsw_sp_fib_node *fib_node)
 {
@@ -2760,7 +4667,7 @@ static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
 		mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
 		break;
 	case MLXSW_SP_L3_PROTO_IPV6:
-		WARN_ON_ONCE(1);
+		mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node);
 		break;
 	}
 }
@@ -2791,10 +4698,17 @@ static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
 		if (!mlxsw_sp_vr_is_used(vr))
 			continue;
 		mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
+
+		/* If virtual router was only used for IPv4, then it's no
+		 * longer used.
+		 */
+		if (!mlxsw_sp_vr_is_used(vr))
+			continue;
+		mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
 	}
 }
 
-static void mlxsw_sp_router_fib4_abort(struct mlxsw_sp *mlxsw_sp)
+static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp)
 {
 	int err;
 
@@ -2811,6 +4725,7 @@ static void mlxsw_sp_router_fib4_abort(struct mlxsw_sp *mlxsw_sp)
 struct mlxsw_sp_fib_event_work {
 	struct work_struct work;
 	union {
+		struct fib6_entry_notifier_info fen6_info;
 		struct fib_entry_notifier_info fen_info;
 		struct fib_rule_notifier_info fr_info;
 		struct fib_nh_notifier_info fnh_info;
@@ -2819,7 +4734,7 @@ struct mlxsw_sp_fib_event_work {
 	unsigned long event;
 };
 
-static void mlxsw_sp_router_fib_event_work(struct work_struct *work)
+static void mlxsw_sp_router_fib4_event_work(struct work_struct *work)
 {
 	struct mlxsw_sp_fib_event_work *fib_work =
 		container_of(work, struct mlxsw_sp_fib_event_work, work);
@@ -2839,7 +4754,7 @@ static void mlxsw_sp_router_fib_event_work(struct work_struct *work)
 		err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
 					       replace, append);
 		if (err)
-			mlxsw_sp_router_fib4_abort(mlxsw_sp);
+			mlxsw_sp_router_fib_abort(mlxsw_sp);
 		fib_info_put(fib_work->fen_info.fi);
 		break;
 	case FIB_EVENT_ENTRY_DEL:
@@ -2850,13 +4765,13 @@ static void mlxsw_sp_router_fib_event_work(struct work_struct *work)
 	case FIB_EVENT_RULE_DEL:
 		rule = fib_work->fr_info.rule;
 		if (!fib4_rule_default(rule) && !rule->l3mdev)
-			mlxsw_sp_router_fib4_abort(mlxsw_sp);
+			mlxsw_sp_router_fib_abort(mlxsw_sp);
 		fib_rule_put(rule);
 		break;
 	case FIB_EVENT_NH_ADD: /* fall through */
 	case FIB_EVENT_NH_DEL:
-		mlxsw_sp_nexthop_event(mlxsw_sp, fib_work->event,
-				       fib_work->fnh_info.fib_nh);
+		mlxsw_sp_nexthop4_event(mlxsw_sp, fib_work->event,
+					fib_work->fnh_info.fib_nh);
 		fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
 		break;
 	}
@@ -2864,32 +4779,51 @@ static void mlxsw_sp_router_fib_event_work(struct work_struct *work)
 	kfree(fib_work);
 }
 
-/* Called with rcu_read_lock() */
-static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
-				     unsigned long event, void *ptr)
+static void mlxsw_sp_router_fib6_event_work(struct work_struct *work)
 {
-	struct mlxsw_sp_fib_event_work *fib_work;
-	struct fib_notifier_info *info = ptr;
-	struct mlxsw_sp_router *router;
-
-	if (!net_eq(info->net, &init_net))
-		return NOTIFY_DONE;
-
-	fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
-	if (WARN_ON(!fib_work))
-		return NOTIFY_BAD;
+	struct mlxsw_sp_fib_event_work *fib_work =
+		container_of(work, struct mlxsw_sp_fib_event_work, work);
+	struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
+	struct fib_rule *rule;
+	bool replace;
+	int err;
 
-	INIT_WORK(&fib_work->work, mlxsw_sp_router_fib_event_work);
-	router = container_of(nb, struct mlxsw_sp_router, fib_nb);
-	fib_work->mlxsw_sp = router->mlxsw_sp;
-	fib_work->event = event;
+	rtnl_lock();
+	switch (fib_work->event) {
+	case FIB_EVENT_ENTRY_REPLACE: /* fall through */
+	case FIB_EVENT_ENTRY_ADD:
+		replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
+		err = mlxsw_sp_router_fib6_add(mlxsw_sp,
+					       fib_work->fen6_info.rt, replace);
+		if (err)
+			mlxsw_sp_router_fib_abort(mlxsw_sp);
+		mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
+		break;
+	case FIB_EVENT_ENTRY_DEL:
+		mlxsw_sp_router_fib6_del(mlxsw_sp, fib_work->fen6_info.rt);
+		mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
+		break;
+	case FIB_EVENT_RULE_ADD: /* fall through */
+	case FIB_EVENT_RULE_DEL:
+		rule = fib_work->fr_info.rule;
+		if (!fib6_rule_default(rule) && !rule->l3mdev)
+			mlxsw_sp_router_fib_abort(mlxsw_sp);
+		fib_rule_put(rule);
+		break;
+	}
+	rtnl_unlock();
+	kfree(fib_work);
+}
 
-	switch (event) {
+static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work,
+				       struct fib_notifier_info *info)
+{
+	switch (fib_work->event) {
 	case FIB_EVENT_ENTRY_REPLACE: /* fall through */
 	case FIB_EVENT_ENTRY_APPEND: /* fall through */
 	case FIB_EVENT_ENTRY_ADD: /* fall through */
 	case FIB_EVENT_ENTRY_DEL:
-		memcpy(&fib_work->fen_info, ptr, sizeof(fib_work->fen_info));
+		memcpy(&fib_work->fen_info, info, sizeof(fib_work->fen_info));
 		/* Take referece on fib_info to prevent it from being
 		 * freed while work is queued. Release it afterwards.
 		 */
@@ -2897,15 +4831,65 @@ static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
 		break;
 	case FIB_EVENT_RULE_ADD: /* fall through */
 	case FIB_EVENT_RULE_DEL:
-		memcpy(&fib_work->fr_info, ptr, sizeof(fib_work->fr_info));
+		memcpy(&fib_work->fr_info, info, sizeof(fib_work->fr_info));
 		fib_rule_get(fib_work->fr_info.rule);
 		break;
 	case FIB_EVENT_NH_ADD: /* fall through */
 	case FIB_EVENT_NH_DEL:
-		memcpy(&fib_work->fnh_info, ptr, sizeof(fib_work->fnh_info));
+		memcpy(&fib_work->fnh_info, info, sizeof(fib_work->fnh_info));
 		fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
 		break;
 	}
+}
+
+static void mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work,
+				       struct fib_notifier_info *info)
+{
+	switch (fib_work->event) {
+	case FIB_EVENT_ENTRY_REPLACE: /* fall through */
+	case FIB_EVENT_ENTRY_ADD: /* fall through */
+	case FIB_EVENT_ENTRY_DEL:
+		memcpy(&fib_work->fen6_info, info, sizeof(fib_work->fen6_info));
+		rt6_hold(fib_work->fen6_info.rt);
+		break;
+	case FIB_EVENT_RULE_ADD: /* fall through */
+	case FIB_EVENT_RULE_DEL:
+		memcpy(&fib_work->fr_info, info, sizeof(fib_work->fr_info));
+		fib_rule_get(fib_work->fr_info.rule);
+		break;
+	}
+}
+
+/* Called with rcu_read_lock() */
+static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
+				     unsigned long event, void *ptr)
+{
+	struct mlxsw_sp_fib_event_work *fib_work;
+	struct fib_notifier_info *info = ptr;
+	struct mlxsw_sp_router *router;
+
+	if (!net_eq(info->net, &init_net) ||
+	    (info->family != AF_INET && info->family != AF_INET6))
+		return NOTIFY_DONE;
+
+	fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
+	if (WARN_ON(!fib_work))
+		return NOTIFY_BAD;
+
+	router = container_of(nb, struct mlxsw_sp_router, fib_nb);
+	fib_work->mlxsw_sp = router->mlxsw_sp;
+	fib_work->event = event;
+
+	switch (info->family) {
+	case AF_INET:
+		INIT_WORK(&fib_work->work, mlxsw_sp_router_fib4_event_work);
+		mlxsw_sp_router_fib4_event(fib_work, info);
+		break;
+	case AF_INET6:
+		INIT_WORK(&fib_work->work, mlxsw_sp_router_fib6_event_work);
+		mlxsw_sp_router_fib6_event(fib_work, info);
+		break;
+	}
 
 	mlxsw_core_schedule_work(&fib_work->work);
 
@@ -2948,17 +4932,28 @@ static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
 	mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
 }
 
-static bool mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif,
-				       const struct in_device *in_dev,
-				       unsigned long event)
+static bool
+mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev,
+			   unsigned long event)
 {
+	struct inet6_dev *inet6_dev;
+	bool addr_list_empty = true;
+	struct in_device *idev;
+
 	switch (event) {
 	case NETDEV_UP:
-		if (!rif)
-			return true;
-		return false;
+		return rif == NULL;
 	case NETDEV_DOWN:
-		if (rif && !in_dev->ifa_list &&
+		idev = __in_dev_get_rtnl(dev);
+		if (idev && idev->ifa_list)
+			addr_list_empty = false;
+
+		inet6_dev = __in6_dev_get(dev);
+		if (addr_list_empty && inet6_dev &&
+		    !list_empty(&inet6_dev->addr_list))
+			addr_list_empty = false;
+
+		if (rif && addr_list_empty &&
 		    !netif_is_l3_slave(rif->dev))
 			return true;
 		/* It is possible we already removed the RIF ourselves
@@ -2977,7 +4972,10 @@ mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
 {
 	enum mlxsw_sp_fid_type type;
 
-	/* RIF type is derived from the type of the underlying FID */
+	if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL))
+		return MLXSW_SP_RIF_TYPE_IPIP_LB;
+
+	/* Otherwise RIF type is derived from the type of the underlying FID. */
 	if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
 		type = MLXSW_SP_FID_TYPE_8021Q;
 	else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
@@ -3036,6 +5034,16 @@ u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
 	return rif->rif_index;
 }
 
+u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
+{
+	return lb_rif->common.rif_index;
+}
+
+u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
+{
+	return lb_rif->ul_vr_id;
+}
+
 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
 {
 	return rif->dev->ifindex;
@@ -3047,9 +5055,9 @@ mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
 {
 	u32 tb_id = l3mdev_fib_table(params->dev);
 	const struct mlxsw_sp_rif_ops *ops;
+	struct mlxsw_sp_fid *fid = NULL;
 	enum mlxsw_sp_rif_type type;
 	struct mlxsw_sp_rif *rif;
-	struct mlxsw_sp_fid *fid;
 	struct mlxsw_sp_vr *vr;
 	u16 rif_index;
 	int err;
@@ -3073,12 +5081,14 @@ mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
 	rif->mlxsw_sp = mlxsw_sp;
 	rif->ops = ops;
 
-	fid = ops->fid_get(rif);
-	if (IS_ERR(fid)) {
-		err = PTR_ERR(fid);
-		goto err_fid_get;
+	if (ops->fid_get) {
+		fid = ops->fid_get(rif);
+		if (IS_ERR(fid)) {
+			err = PTR_ERR(fid);
+			goto err_fid_get;
+		}
+		rif->fid = fid;
 	}
-	rif->fid = fid;
 
 	if (ops->setup)
 		ops->setup(rif, params);
@@ -3087,22 +5097,15 @@ mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
 	if (err)
 		goto err_configure;
 
-	err = mlxsw_sp_rif_fdb_op(mlxsw_sp, params->dev->dev_addr,
-				  mlxsw_sp_fid_index(fid), true);
-	if (err)
-		goto err_rif_fdb_op;
-
 	mlxsw_sp_rif_counters_alloc(rif);
-	mlxsw_sp_fid_rif_set(fid, rif);
 	mlxsw_sp->router->rifs[rif_index] = rif;
 	vr->rif_count++;
 
 	return rif;
 
-err_rif_fdb_op:
-	ops->deconfigure(rif);
 err_configure:
-	mlxsw_sp_fid_put(fid);
+	if (fid)
+		mlxsw_sp_fid_put(fid);
 err_fid_get:
 	kfree(rif);
 err_rif_alloc:
@@ -3123,12 +5126,11 @@ void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
 
 	vr->rif_count--;
 	mlxsw_sp->router->rifs[rif->rif_index] = NULL;
-	mlxsw_sp_fid_rif_set(fid, NULL);
 	mlxsw_sp_rif_counters_free(rif);
-	mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->dev->dev_addr,
-			    mlxsw_sp_fid_index(fid), false);
 	ops->deconfigure(rif);
-	mlxsw_sp_fid_put(fid);
+	if (fid)
+		/* Loopback RIFs are not associated with a FID. */
+		mlxsw_sp_fid_put(fid);
 	kfree(rif);
 	mlxsw_sp_vr_put(vr);
 }
@@ -3356,7 +5358,7 @@ int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
 		goto out;
 
 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
-	if (!mlxsw_sp_rif_should_config(rif, ifa->ifa_dev, event))
+	if (!mlxsw_sp_rif_should_config(rif, dev, event))
 		goto out;
 
 	err = __mlxsw_sp_inetaddr_event(dev, event);
@@ -3364,6 +5366,61 @@ out:
 	return notifier_from_errno(err);
 }
 
+struct mlxsw_sp_inet6addr_event_work {
+	struct work_struct work;
+	struct net_device *dev;
+	unsigned long event;
+};
+
+static void mlxsw_sp_inet6addr_event_work(struct work_struct *work)
+{
+	struct mlxsw_sp_inet6addr_event_work *inet6addr_work =
+		container_of(work, struct mlxsw_sp_inet6addr_event_work, work);
+	struct net_device *dev = inet6addr_work->dev;
+	unsigned long event = inet6addr_work->event;
+	struct mlxsw_sp *mlxsw_sp;
+	struct mlxsw_sp_rif *rif;
+
+	rtnl_lock();
+	mlxsw_sp = mlxsw_sp_lower_get(dev);
+	if (!mlxsw_sp)
+		goto out;
+
+	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
+	if (!mlxsw_sp_rif_should_config(rif, dev, event))
+		goto out;
+
+	__mlxsw_sp_inetaddr_event(dev, event);
+out:
+	rtnl_unlock();
+	dev_put(dev);
+	kfree(inet6addr_work);
+}
+
+/* Called with rcu_read_lock() */
+int mlxsw_sp_inet6addr_event(struct notifier_block *unused,
+			     unsigned long event, void *ptr)
+{
+	struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr;
+	struct mlxsw_sp_inet6addr_event_work *inet6addr_work;
+	struct net_device *dev = if6->idev->dev;
+
+	if (!mlxsw_sp_port_dev_lower_find_rcu(dev))
+		return NOTIFY_DONE;
+
+	inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC);
+	if (!inet6addr_work)
+		return NOTIFY_BAD;
+
+	INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work);
+	inet6addr_work->dev = dev;
+	inet6addr_work->event = event;
+	dev_hold(dev);
+	mlxsw_core_schedule_work(&inet6addr_work->work);
+
+	return NOTIFY_DONE;
+}
+
 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
 			     const char *mac, int mtu)
 {
@@ -3501,8 +5558,8 @@ static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
 
 	rif_subport = mlxsw_sp_rif_subport_rif(rif);
 	mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
-			    rif->rif_index, rif->vr_id, rif->dev->mtu,
-			    rif->dev->dev_addr);
+			    rif->rif_index, rif->vr_id, rif->dev->mtu);
+	mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
 	mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
 				  rif_subport->lag ? rif_subport->lag_id :
 						     rif_subport->system_port,
@@ -3513,11 +5570,32 @@ static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
 
 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif)
 {
-	return mlxsw_sp_rif_subport_op(rif, true);
+	int err;
+
+	err = mlxsw_sp_rif_subport_op(rif, true);
+	if (err)
+		return err;
+
+	err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
+				  mlxsw_sp_fid_index(rif->fid), true);
+	if (err)
+		goto err_rif_fdb_op;
+
+	mlxsw_sp_fid_rif_set(rif->fid, rif);
+	return 0;
+
+err_rif_fdb_op:
+	mlxsw_sp_rif_subport_op(rif, false);
+	return err;
 }
 
 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
 {
+	struct mlxsw_sp_fid *fid = rif->fid;
+
+	mlxsw_sp_fid_rif_set(fid, NULL);
+	mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
+			    mlxsw_sp_fid_index(fid), false);
 	mlxsw_sp_rif_subport_op(rif, false);
 }
 
@@ -3544,7 +5622,8 @@ static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif,
 	char ritr_pl[MLXSW_REG_RITR_LEN];
 
 	mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
-			    rif->dev->mtu, rif->dev->dev_addr);
+			    rif->dev->mtu);
+	mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
 	mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid);
 
 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
@@ -3565,25 +5644,48 @@ static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif)
 	if (err)
 		return err;
 
+	err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
+				     mlxsw_sp_router_port(mlxsw_sp), true);
+	if (err)
+		goto err_fid_mc_flood_set;
+
 	err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
 				     mlxsw_sp_router_port(mlxsw_sp), true);
 	if (err)
 		goto err_fid_bc_flood_set;
 
+	err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
+				  mlxsw_sp_fid_index(rif->fid), true);
+	if (err)
+		goto err_rif_fdb_op;
+
+	mlxsw_sp_fid_rif_set(rif->fid, rif);
 	return 0;
 
+err_rif_fdb_op:
+	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
+			       mlxsw_sp_router_port(mlxsw_sp), false);
 err_fid_bc_flood_set:
+	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
+			       mlxsw_sp_router_port(mlxsw_sp), false);
+err_fid_mc_flood_set:
 	mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
 	return err;
 }
 
 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif)
 {
-	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 	u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
+	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
+	struct mlxsw_sp_fid *fid = rif->fid;
 
+	mlxsw_sp_fid_rif_set(fid, NULL);
+	mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
+			    mlxsw_sp_fid_index(fid), false);
 	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
 			       mlxsw_sp_router_port(mlxsw_sp), false);
+	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
+			       mlxsw_sp_router_port(mlxsw_sp), false);
 	mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
 }
 
@@ -3614,25 +5716,48 @@ static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif)
 	if (err)
 		return err;
 
+	err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
+				     mlxsw_sp_router_port(mlxsw_sp), true);
+	if (err)
+		goto err_fid_mc_flood_set;
+
 	err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
 				     mlxsw_sp_router_port(mlxsw_sp), true);
 	if (err)
 		goto err_fid_bc_flood_set;
 
+	err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
+				  mlxsw_sp_fid_index(rif->fid), true);
+	if (err)
+		goto err_rif_fdb_op;
+
+	mlxsw_sp_fid_rif_set(rif->fid, rif);
 	return 0;
 
+err_rif_fdb_op:
+	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
+			       mlxsw_sp_router_port(mlxsw_sp), false);
 err_fid_bc_flood_set:
+	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
+			       mlxsw_sp_router_port(mlxsw_sp), false);
+err_fid_mc_flood_set:
 	mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
 	return err;
 }
 
 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
 {
-	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 	u16 fid_index = mlxsw_sp_fid_index(rif->fid);
+	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
+	struct mlxsw_sp_fid *fid = rif->fid;
 
+	mlxsw_sp_fid_rif_set(fid, NULL);
+	mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
+			    mlxsw_sp_fid_index(fid), false);
 	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
 			       mlxsw_sp_router_port(mlxsw_sp), false);
+	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
+			       mlxsw_sp_router_port(mlxsw_sp), false);
 	mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
 }
 
@@ -3650,10 +5775,104 @@ static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
 	.fid_get		= mlxsw_sp_rif_fid_fid_get,
 };
 
+static struct mlxsw_sp_rif_ipip_lb *
+mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif)
+{
+	return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
+}
+
+static void
+mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif,
+			   const struct mlxsw_sp_rif_params *params)
+{
+	struct mlxsw_sp_rif_params_ipip_lb *params_lb;
+	struct mlxsw_sp_rif_ipip_lb *rif_lb;
+
+	params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb,
+				 common);
+	rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif);
+	rif_lb->lb_config = params_lb->lb_config;
+}
+
+static int
+mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif,
+			struct mlxsw_sp_vr *ul_vr, bool enable)
+{
+	struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config;
+	struct mlxsw_sp_rif *rif = &lb_rif->common;
+	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
+	char ritr_pl[MLXSW_REG_RITR_LEN];
+	u32 saddr4;
+
+	switch (lb_cf.ul_protocol) {
+	case MLXSW_SP_L3_PROTO_IPV4:
+		saddr4 = be32_to_cpu(lb_cf.saddr.addr4);
+		mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
+				    rif->rif_index, rif->vr_id, rif->dev->mtu);
+		mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt,
+			    MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET,
+			    ul_vr->id, saddr4, lb_cf.okey);
+		break;
+
+	case MLXSW_SP_L3_PROTO_IPV6:
+		return -EAFNOSUPPORT;
+	}
+
+	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
+}
+
+static int
+mlxsw_sp_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif)
+{
+	struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
+	u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
+	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
+	struct mlxsw_sp_vr *ul_vr;
+	int err;
+
+	ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id);
+	if (IS_ERR(ul_vr))
+		return PTR_ERR(ul_vr);
+
+	err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, true);
+	if (err)
+		goto err_loopback_op;
+
+	lb_rif->ul_vr_id = ul_vr->id;
+	++ul_vr->rif_count;
+	return 0;
+
+err_loopback_op:
+	mlxsw_sp_vr_put(ul_vr);
+	return err;
+}
+
+static void mlxsw_sp_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
+{
+	struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
+	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
+	struct mlxsw_sp_vr *ul_vr;
+
+	ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id];
+	mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, false);
+
+	--ul_vr->rif_count;
+	mlxsw_sp_vr_put(ul_vr);
+}
+
+static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_ipip_lb_ops = {
+	.type			= MLXSW_SP_RIF_TYPE_IPIP_LB,
+	.rif_size		= sizeof(struct mlxsw_sp_rif_ipip_lb),
+	.setup                  = mlxsw_sp_rif_ipip_lb_setup,
+	.configure		= mlxsw_sp_rif_ipip_lb_configure,
+	.deconfigure		= mlxsw_sp_rif_ipip_lb_deconfigure,
+};
+
 static const struct mlxsw_sp_rif_ops *mlxsw_sp_rif_ops_arr[] = {
 	[MLXSW_SP_RIF_TYPE_SUBPORT]	= &mlxsw_sp_rif_subport_ops,
 	[MLXSW_SP_RIF_TYPE_VLAN]	= &mlxsw_sp_rif_vlan_ops,
 	[MLXSW_SP_RIF_TYPE_FID]		= &mlxsw_sp_rif_fid_ops,
+	[MLXSW_SP_RIF_TYPE_IPIP_LB]	= &mlxsw_sp_rif_ipip_lb_ops,
 };
 
 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
@@ -3681,6 +5900,18 @@ static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
 	kfree(mlxsw_sp->router->rifs);
 }
 
+static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp)
+{
+	mlxsw_sp->router->ipip_ops_arr = mlxsw_sp_ipip_ops_arr;
+	INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list);
+	return 0;
+}
+
+static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp)
+{
+	WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list));
+}
+
 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
 {
 	struct mlxsw_sp_router *router;
@@ -3704,7 +5935,7 @@ static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
 		return -EIO;
 	max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
 
-	mlxsw_reg_rgcr_pack(rgcr_pl, true);
+	mlxsw_reg_rgcr_pack(rgcr_pl, true, true);
 	mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
 	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
 	if (err)
@@ -3716,7 +5947,7 @@ static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
 {
 	char rgcr_pl[MLXSW_REG_RGCR_LEN];
 
-	mlxsw_reg_rgcr_pack(rgcr_pl, false);
+	mlxsw_reg_rgcr_pack(rgcr_pl, false, false);
 	mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
 }
 
@@ -3740,6 +5971,10 @@ int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
 	if (err)
 		goto err_rifs_init;
 
+	err = mlxsw_sp_ipips_init(mlxsw_sp);
+	if (err)
+		goto err_ipips_init;
+
 	err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
 			      &mlxsw_sp_nexthop_ht_params);
 	if (err)
@@ -3781,6 +6016,8 @@ err_lpm_init:
 err_nexthop_group_ht_init:
 	rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
 err_nexthop_ht_init:
+	mlxsw_sp_ipips_fini(mlxsw_sp);
+err_ipips_init:
 	mlxsw_sp_rifs_fini(mlxsw_sp);
 err_rifs_init:
 	__mlxsw_sp_router_fini(mlxsw_sp);
@@ -3797,6 +6034,7 @@ void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
 	mlxsw_sp_lpm_fini(mlxsw_sp);
 	rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
 	rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
+	mlxsw_sp_ipips_fini(mlxsw_sp);
 	mlxsw_sp_rifs_fini(mlxsw_sp);
 	__mlxsw_sp_router_fini(mlxsw_sp);
 	kfree(mlxsw_sp->router);
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.h
index a3e8d2b25148..345fcc4f38e9 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.h
@@ -36,15 +36,38 @@
 #define _MLXSW_ROUTER_H_
 
 #include "spectrum.h"
+#include "reg.h"
+
+enum mlxsw_sp_l3proto {
+	MLXSW_SP_L3_PROTO_IPV4,
+	MLXSW_SP_L3_PROTO_IPV6,
+};
+
+union mlxsw_sp_l3addr {
+	__be32 addr4;
+	struct in6_addr addr6;
+};
+
+struct mlxsw_sp_rif_ipip_lb;
+struct mlxsw_sp_rif_ipip_lb_config {
+	enum mlxsw_reg_ritr_loopback_ipip_type lb_ipipt;
+	u32 okey;
+	enum mlxsw_sp_l3proto ul_protocol; /* Underlay. */
+	union mlxsw_sp_l3addr saddr;
+};
 
 enum mlxsw_sp_rif_counter_dir {
 	MLXSW_SP_RIF_COUNTER_INGRESS,
 	MLXSW_SP_RIF_COUNTER_EGRESS,
 };
 
+struct mlxsw_sp_neigh_entry;
+
 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
 					   u16 rif_index);
 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif);
+u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *rif);
+u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *rif);
 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif);
 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
 				   struct mlxsw_sp_rif *rif,
@@ -56,5 +79,33 @@ void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp,
 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp,
 			       struct mlxsw_sp_rif *rif,
 			       enum mlxsw_sp_rif_counter_dir dir);
+struct mlxsw_sp_neigh_entry *
+mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif,
+			struct mlxsw_sp_neigh_entry *neigh_entry);
+int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry);
+unsigned char *
+mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry);
+u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry);
+struct in6_addr *
+mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry);
+
+#define mlxsw_sp_rif_neigh_for_each(neigh_entry, rif)				\
+	for (neigh_entry = mlxsw_sp_rif_neigh_next(rif, NULL); neigh_entry;	\
+	     neigh_entry = mlxsw_sp_rif_neigh_next(rif, neigh_entry))
+int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp,
+			       struct mlxsw_sp_neigh_entry *neigh_entry,
+			       u64 *p_counter);
+void
+mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp,
+				    struct mlxsw_sp_neigh_entry *neigh_entry,
+				    bool adding);
+bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry);
+union mlxsw_sp_l3addr
+mlxsw_sp_ipip_netdev_saddr(enum mlxsw_sp_l3proto proto,
+			   const struct net_device *ol_dev);
+union mlxsw_sp_l3addr
+mlxsw_sp_ipip_netdev_daddr(enum mlxsw_sp_l3proto proto,
+			   const struct net_device *ol_dev);
+__be32 mlxsw_sp_ipip_netdev_daddr4(const struct net_device *ol_dev);
 
 #endif /* _MLXSW_ROUTER_H_*/
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c
index 5eb1606765c5..d39ffbfcc436 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c
@@ -705,6 +705,7 @@ static int mlxsw_sp_port_attr_mc_router_set(struct mlxsw_sp_port *mlxsw_sp_port,
 					    bool is_port_mc_router)
 {
 	struct mlxsw_sp_bridge_port *bridge_port;
+	int err;
 
 	if (switchdev_trans_ph_prepare(trans))
 		return 0;
@@ -715,11 +716,17 @@ static int mlxsw_sp_port_attr_mc_router_set(struct mlxsw_sp_port *mlxsw_sp_port,
 		return 0;
 
 	if (!bridge_port->bridge_device->multicast_enabled)
-		return 0;
+		goto out;
 
-	return mlxsw_sp_bridge_port_flood_table_set(mlxsw_sp_port, bridge_port,
-						    MLXSW_SP_FLOOD_TYPE_MC,
-						    is_port_mc_router);
+	err = mlxsw_sp_bridge_port_flood_table_set(mlxsw_sp_port, bridge_port,
+						   MLXSW_SP_FLOOD_TYPE_MC,
+						   is_port_mc_router);
+	if (err)
+		return err;
+
+out:
+	bridge_port->mrouter = is_port_mc_router;
+	return 0;
 }
 
 static int mlxsw_sp_port_mc_disabled_set(struct mlxsw_sp_port *mlxsw_sp_port,
diff --git a/drivers/net/ethernet/mellanox/mlxsw/switchib.c b/drivers/net/ethernet/mellanox/mlxsw/switchib.c
index 74341fe0eb25..ab7a29846bfa 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/switchib.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/switchib.c
@@ -497,7 +497,7 @@ static void mlxsw_sib_fini(struct mlxsw_core *mlxsw_core)
 	mlxsw_sib_ports_remove(mlxsw_sib);
 }
 
-static struct mlxsw_config_profile mlxsw_sib_config_profile = {
+static const struct mlxsw_config_profile mlxsw_sib_config_profile = {
 	.used_max_system_port		= 1,
 	.max_system_port		= 48000,
 	.used_max_ib_mc			= 1,
diff --git a/drivers/net/ethernet/mellanox/mlxsw/switchx2.c b/drivers/net/ethernet/mellanox/mlxsw/switchx2.c
index 3b0f72455681..f3c29bbf07e2 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/switchx2.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/switchx2.c
@@ -1674,7 +1674,7 @@ static void mlxsw_sx_fini(struct mlxsw_core *mlxsw_core)
 	mlxsw_sx_ports_remove(mlxsw_sx);
 }
 
-static struct mlxsw_config_profile mlxsw_sx_config_profile = {
+static const struct mlxsw_config_profile mlxsw_sx_config_profile = {
 	.used_max_vepa_channels		= 1,
 	.max_vepa_channels		= 0,
 	.used_max_mid			= 1,
diff --git a/drivers/net/ethernet/mellanox/mlxsw/trap.h b/drivers/net/ethernet/mellanox/mlxsw/trap.h
index 12b5ed58f3eb..f396a1fef633 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/trap.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/trap.h
@@ -61,11 +61,33 @@ enum {
 	MLXSW_TRAP_ID_MTUERROR = 0x52,
 	MLXSW_TRAP_ID_TTLERROR = 0x53,
 	MLXSW_TRAP_ID_LBERROR = 0x54,
-	MLXSW_TRAP_ID_OSPF = 0x55,
+	MLXSW_TRAP_ID_IPV4_OSPF = 0x55,
 	MLXSW_TRAP_ID_IP2ME = 0x5F,
+	MLXSW_TRAP_ID_IPV6_UNSPECIFIED_ADDRESS = 0x60,
+	MLXSW_TRAP_ID_IPV6_LINK_LOCAL_DEST = 0x61,
+	MLXSW_TRAP_ID_IPV6_LINK_LOCAL_SRC = 0x62,
+	MLXSW_TRAP_ID_IPV6_ALL_NODES_LINK = 0x63,
+	MLXSW_TRAP_ID_IPV6_OSPF = 0x64,
+	MLXSW_TRAP_ID_IPV6_MLDV12_LISTENER_QUERY = 0x65,
+	MLXSW_TRAP_ID_IPV6_MLDV1_LISTENER_REPORT = 0x66,
+	MLXSW_TRAP_ID_IPV6_MLDV1_LISTENER_DONE = 0x67,
+	MLXSW_TRAP_ID_IPV6_MLDV2_LISTENER_REPORT = 0x68,
+	MLXSW_TRAP_ID_IPV6_DHCP = 0x69,
+	MLXSW_TRAP_ID_IPV6_ALL_ROUTERS_LINK = 0x6F,
 	MLXSW_TRAP_ID_RTR_INGRESS0 = 0x70,
-	MLXSW_TRAP_ID_BGP_IPV4 = 0x88,
+	MLXSW_TRAP_ID_IPV4_BGP = 0x88,
+	MLXSW_TRAP_ID_IPV6_BGP = 0x89,
+	MLXSW_TRAP_ID_L3_IPV6_ROUTER_SOLICITATION = 0x8A,
+	MLXSW_TRAP_ID_L3_IPV6_ROUTER_ADVERTISMENT = 0x8B,
+	MLXSW_TRAP_ID_L3_IPV6_NEIGHBOR_SOLICITATION = 0x8C,
+	MLXSW_TRAP_ID_L3_IPV6_NEIGHBOR_ADVERTISMENT = 0x8D,
+	MLXSW_TRAP_ID_L3_IPV6_REDIRECTION = 0x8E,
 	MLXSW_TRAP_ID_HOST_MISS_IPV4 = 0x90,
+	MLXSW_TRAP_ID_IPV6_MC_LINK_LOCAL_DEST = 0x91,
+	MLXSW_TRAP_ID_HOST_MISS_IPV6 = 0x92,
+	MLXSW_TRAP_ID_IPIP_DECAP_ERROR = 0xB1,
+	MLXSW_TRAP_ID_ROUTER_ALERT_IPV4 = 0xD6,
+	MLXSW_TRAP_ID_ROUTER_ALERT_IPV6 = 0xD7,
 	MLXSW_TRAP_ID_ACL0 = 0x1C0,
 
 	MLXSW_TRAP_ID_MAX = 0x1FF
diff --git a/drivers/net/ethernet/moxa/moxart_ether.c b/drivers/net/ethernet/moxa/moxart_ether.c
index c0d7d5eec7e7..2e4effa9fe45 100644
--- a/drivers/net/ethernet/moxa/moxart_ether.c
+++ b/drivers/net/ethernet/moxa/moxart_ether.c
@@ -161,7 +161,7 @@ static void moxart_mac_setup_desc_ring(struct net_device *ndev)
 
 	priv->rx_head = 0;
 
-	/* reset the MAC controller TX/RX desciptor base address */
+	/* reset the MAC controller TX/RX descriptor base address */
 	writel(priv->tx_base, priv->base + REG_TXR_BASE_ADDRESS);
 	writel(priv->rx_base, priv->base + REG_RXR_BASE_ADDRESS);
 }
@@ -269,9 +269,8 @@ rx_next:
 		priv->rx_head = rx_head;
 	}
 
-	if (rx < budget) {
+	if (rx < budget)
 		napi_complete_done(napi, rx);
-	}
 
 	priv->reg_imr |= RPKT_FINISH_M;
 	writel(priv->reg_imr, priv->base + REG_INTERRUPT_MASK);
@@ -289,8 +288,8 @@ static int moxart_tx_queue_space(struct net_device *ndev)
 static void moxart_tx_finished(struct net_device *ndev)
 {
 	struct moxart_mac_priv_t *priv = netdev_priv(ndev);
-	unsigned tx_head = priv->tx_head;
-	unsigned tx_tail = priv->tx_tail;
+	unsigned int tx_head = priv->tx_head;
+	unsigned int tx_tail = priv->tx_tail;
 
 	while (tx_tail != tx_head) {
 		dma_unmap_single(&ndev->dev, priv->tx_mapping[tx_tail],
@@ -312,7 +311,7 @@ static void moxart_tx_finished(struct net_device *ndev)
 
 static irqreturn_t moxart_mac_interrupt(int irq, void *dev_id)
 {
-	struct net_device *ndev = (struct net_device *) dev_id;
+	struct net_device *ndev = (struct net_device *)dev_id;
 	struct moxart_mac_priv_t *priv = netdev_priv(ndev);
 	unsigned int ists = readl(priv->base + REG_INTERRUPT_STATUS);
 
@@ -495,7 +494,7 @@ static int moxart_mac_probe(struct platform_device *pdev)
 	priv->tx_desc_base = dma_alloc_coherent(NULL, TX_REG_DESC_SIZE *
 						TX_DESC_NUM, &priv->tx_base,
 						GFP_DMA | GFP_KERNEL);
-	if (priv->tx_desc_base == NULL) {
+	if (!priv->tx_desc_base) {
 		ret = -ENOMEM;
 		goto init_fail;
 	}
@@ -503,7 +502,7 @@ static int moxart_mac_probe(struct platform_device *pdev)
 	priv->rx_desc_base = dma_alloc_coherent(NULL, RX_REG_DESC_SIZE *
 						RX_DESC_NUM, &priv->rx_base,
 						GFP_DMA | GFP_KERNEL);
-	if (priv->rx_desc_base == NULL) {
+	if (!priv->rx_desc_base) {
 		ret = -ENOMEM;
 		goto init_fail;
 	}
diff --git a/drivers/net/ethernet/moxa/moxart_ether.h b/drivers/net/ethernet/moxa/moxart_ether.h
index 686b8957d5cf..bee608b547d1 100644
--- a/drivers/net/ethernet/moxa/moxart_ether.h
+++ b/drivers/net/ethernet/moxa/moxart_ether.h
@@ -55,17 +55,17 @@
 #define RX_DESC2_ADDRESS_VIRT	4
 
 #define TX_DESC_NUM		64
-#define TX_DESC_NUM_MASK	(TX_DESC_NUM-1)
+#define TX_DESC_NUM_MASK	(TX_DESC_NUM - 1)
 #define TX_NEXT(N)		(((N) + 1) & (TX_DESC_NUM_MASK))
 #define TX_BUF_SIZE		1600
-#define TX_BUF_SIZE_MAX		(TX_DESC1_BUF_SIZE_MASK+1)
+#define TX_BUF_SIZE_MAX		(TX_DESC1_BUF_SIZE_MASK + 1)
 #define TX_WAKE_THRESHOLD	16
 
 #define RX_DESC_NUM		64
-#define RX_DESC_NUM_MASK	(RX_DESC_NUM-1)
+#define RX_DESC_NUM_MASK	(RX_DESC_NUM - 1)
 #define RX_NEXT(N)		(((N) + 1) & (RX_DESC_NUM_MASK))
 #define RX_BUF_SIZE		1600
-#define RX_BUF_SIZE_MAX		(RX_DESC1_BUF_SIZE_MASK+1)
+#define RX_BUF_SIZE_MAX		(RX_DESC1_BUF_SIZE_MASK + 1)
 
 #define REG_INTERRUPT_STATUS	0
 #define REG_INTERRUPT_MASK	4
diff --git a/drivers/net/ethernet/neterion/s2io.c b/drivers/net/ethernet/neterion/s2io.c
index fd2ec36c6fa1..462eda926b1c 100644
--- a/drivers/net/ethernet/neterion/s2io.c
+++ b/drivers/net/ethernet/neterion/s2io.c
@@ -42,8 +42,6 @@
  *     aggregated as a single large packet
  * napi: This parameter used to enable/disable NAPI (polling Rx)
  *     Possible values '1' for enable and '0' for disable. Default is '1'
- * ufo: This parameter used to enable/disable UDP Fragmentation Offload(UFO)
- *      Possible values '1' for enable and '0' for disable. Default is '0'
  * vlan_tag_strip: This can be used to enable or disable vlan stripping.
  *                 Possible values '1' for enable , '0' for disable.
  *                 Default is '2' - which means disable in promisc mode
@@ -453,7 +451,6 @@ S2IO_PARM_INT(lro_max_pkts, 0xFFFF);
 S2IO_PARM_INT(indicate_max_pkts, 0);
 
 S2IO_PARM_INT(napi, 1);
-S2IO_PARM_INT(ufo, 0);
 S2IO_PARM_INT(vlan_tag_strip, NO_STRIP_IN_PROMISC);
 
 static unsigned int tx_fifo_len[MAX_TX_FIFOS] =
@@ -4128,32 +4125,6 @@ static netdev_tx_t s2io_xmit(struct sk_buff *skb, struct net_device *dev)
 	}
 
 	frg_len = skb_headlen(skb);
-	if (offload_type == SKB_GSO_UDP) {
-		int ufo_size;
-
-		ufo_size = s2io_udp_mss(skb);
-		ufo_size &= ~7;
-		txdp->Control_1 |= TXD_UFO_EN;
-		txdp->Control_1 |= TXD_UFO_MSS(ufo_size);
-		txdp->Control_1 |= TXD_BUFFER0_SIZE(8);
-#ifdef __BIG_ENDIAN
-		/* both variants do cpu_to_be64(be32_to_cpu(...)) */
-		fifo->ufo_in_band_v[put_off] =
-			(__force u64)skb_shinfo(skb)->ip6_frag_id;
-#else
-		fifo->ufo_in_band_v[put_off] =
-			(__force u64)skb_shinfo(skb)->ip6_frag_id << 32;
-#endif
-		txdp->Host_Control = (unsigned long)fifo->ufo_in_band_v;
-		txdp->Buffer_Pointer = pci_map_single(sp->pdev,
-						      fifo->ufo_in_band_v,
-						      sizeof(u64),
-						      PCI_DMA_TODEVICE);
-		if (pci_dma_mapping_error(sp->pdev, txdp->Buffer_Pointer))
-			goto pci_map_failed;
-		txdp++;
-	}
-
 	txdp->Buffer_Pointer = pci_map_single(sp->pdev, skb->data,
 					      frg_len, PCI_DMA_TODEVICE);
 	if (pci_dma_mapping_error(sp->pdev, txdp->Buffer_Pointer))
@@ -4161,8 +4132,6 @@ static netdev_tx_t s2io_xmit(struct sk_buff *skb, struct net_device *dev)
 
 	txdp->Host_Control = (unsigned long)skb;
 	txdp->Control_1 |= TXD_BUFFER0_SIZE(frg_len);
-	if (offload_type == SKB_GSO_UDP)
-		txdp->Control_1 |= TXD_UFO_EN;
 
 	frg_cnt = skb_shinfo(skb)->nr_frags;
 	/* For fragmented SKB. */
@@ -4177,14 +4146,9 @@ static netdev_tx_t s2io_xmit(struct sk_buff *skb, struct net_device *dev)
 							     skb_frag_size(frag),
 							     DMA_TO_DEVICE);
 		txdp->Control_1 = TXD_BUFFER0_SIZE(skb_frag_size(frag));
-		if (offload_type == SKB_GSO_UDP)
-			txdp->Control_1 |= TXD_UFO_EN;
 	}
 	txdp->Control_1 |= TXD_GATHER_CODE_LAST;
 
-	if (offload_type == SKB_GSO_UDP)
-		frg_cnt++; /* as Txd0 was used for inband header */
-
 	tx_fifo = mac_control->tx_FIFO_start[queue];
 	val64 = fifo->list_info[put_off].list_phy_addr;
 	writeq(val64, &tx_fifo->TxDL_Pointer);
@@ -7910,11 +7874,6 @@ s2io_init_nic(struct pci_dev *pdev, const struct pci_device_id *pre)
 		NETIF_F_RXCSUM | NETIF_F_LRO;
 	dev->features |= dev->hw_features |
 		NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
-	if (sp->device_type & XFRAME_II_DEVICE) {
-		dev->hw_features |= NETIF_F_UFO;
-		if (ufo)
-			dev->features |= NETIF_F_UFO;
-	}
 	if (sp->high_dma_flag == true)
 		dev->features |= NETIF_F_HIGHDMA;
 	dev->watchdog_timeo = WATCH_DOG_TIMEOUT;
@@ -8147,10 +8106,6 @@ s2io_init_nic(struct pci_dev *pdev, const struct pci_device_id *pre)
 
 	DBG_PRINT(ERR_DBG, "%s: Large receive offload enabled\n",
 		  dev->name);
-	if (ufo)
-		DBG_PRINT(ERR_DBG,
-			  "%s: UDP Fragmentation Offload(UFO) enabled\n",
-			  dev->name);
 	/* Initialize device name */
 	snprintf(sp->name, sizeof(sp->name), "%s Neterion %s", dev->name,
 		 sp->product_name);
diff --git a/drivers/net/ethernet/netronome/Kconfig b/drivers/net/ethernet/netronome/Kconfig
index 0e331e2f685a..ae0c46ba7546 100644
--- a/drivers/net/ethernet/netronome/Kconfig
+++ b/drivers/net/ethernet/netronome/Kconfig
@@ -29,6 +29,7 @@ config NFP_APP_FLOWER
 	bool "NFP4000/NFP6000 TC Flower offload support"
 	depends on NFP
 	depends on NET_SWITCHDEV
+	default y
 	---help---
 	  Enable driver support for TC Flower offload on NFP4000 and NFP6000.
 	  Say Y, if you are planning to make use of TC Flower offload
diff --git a/drivers/net/ethernet/netronome/nfp/Makefile b/drivers/net/ethernet/netronome/nfp/Makefile
index b8e1358868bd..96e579a15cbe 100644
--- a/drivers/net/ethernet/netronome/nfp/Makefile
+++ b/drivers/net/ethernet/netronome/nfp/Makefile
@@ -23,6 +23,7 @@ nfp-objs := \
 	    nfp_net_ethtool.o \
 	    nfp_net_main.o \
 	    nfp_net_repr.o \
+	    nfp_net_sriov.o \
 	    nfp_netvf_main.o \
 	    nfp_port.o \
 	    bpf/main.o \
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/jit.c b/drivers/net/ethernet/netronome/nfp/bpf/jit.c
index 8e57fda6b8b5..239dfbe8a0a1 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/jit.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/jit.c
@@ -1238,6 +1238,16 @@ static int jge_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 	return wrp_cmp_imm(nfp_prog, meta, BR_BHS, true);
 }
 
+static int jlt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_cmp_imm(nfp_prog, meta, BR_BHS, false);
+}
+
+static int jle_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_cmp_imm(nfp_prog, meta, BR_BLO, true);
+}
+
 static int jset_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
 	const struct bpf_insn *insn = &meta->insn;
@@ -1325,6 +1335,16 @@ static int jge_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 	return wrp_cmp_reg(nfp_prog, meta, BR_BHS, true);
 }
 
+static int jlt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_cmp_reg(nfp_prog, meta, BR_BHS, false);
+}
+
+static int jle_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_cmp_reg(nfp_prog, meta, BR_BLO, true);
+}
+
 static int jset_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
 	return wrp_test_reg(nfp_prog, meta, ALU_OP_AND, BR_BNE);
@@ -1383,11 +1403,15 @@ static const instr_cb_t instr_cb[256] = {
 	[BPF_JMP | BPF_JEQ | BPF_K] =	jeq_imm,
 	[BPF_JMP | BPF_JGT | BPF_K] =	jgt_imm,
 	[BPF_JMP | BPF_JGE | BPF_K] =	jge_imm,
+	[BPF_JMP | BPF_JLT | BPF_K] =	jlt_imm,
+	[BPF_JMP | BPF_JLE | BPF_K] =	jle_imm,
 	[BPF_JMP | BPF_JSET | BPF_K] =	jset_imm,
 	[BPF_JMP | BPF_JNE | BPF_K] =	jne_imm,
 	[BPF_JMP | BPF_JEQ | BPF_X] =	jeq_reg,
 	[BPF_JMP | BPF_JGT | BPF_X] =	jgt_reg,
 	[BPF_JMP | BPF_JGE | BPF_X] =	jge_reg,
+	[BPF_JMP | BPF_JLT | BPF_X] =	jlt_reg,
+	[BPF_JMP | BPF_JLE | BPF_X] =	jle_reg,
 	[BPF_JMP | BPF_JSET | BPF_X] =	jset_reg,
 	[BPF_JMP | BPF_JNE | BPF_X] =	jne_reg,
 	[BPF_JMP | BPF_EXIT] =		goto_out,
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/main.c b/drivers/net/ethernet/netronome/nfp/bpf/main.c
index afbdf5fd4e4f..be2cf10a2cd7 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/main.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/main.c
@@ -84,7 +84,7 @@ static const char *nfp_bpf_extra_cap(struct nfp_app *app, struct nfp_net *nn)
 }
 
 static int
-nfp_bpf_vnic_init(struct nfp_app *app, struct nfp_net *nn, unsigned int id)
+nfp_bpf_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id)
 {
 	struct nfp_net_bpf_priv *priv;
 	int ret;
@@ -106,14 +106,14 @@ nfp_bpf_vnic_init(struct nfp_app *app, struct nfp_net *nn, unsigned int id)
 	setup_timer(&priv->rx_filter_stats_timer,
 		    nfp_net_filter_stats_timer, (unsigned long)nn);
 
-	ret = nfp_app_nic_vnic_init(app, nn, id);
+	ret = nfp_app_nic_vnic_alloc(app, nn, id);
 	if (ret)
 		kfree(priv);
 
 	return ret;
 }
 
-static void nfp_bpf_vnic_clean(struct nfp_app *app, struct nfp_net *nn)
+static void nfp_bpf_vnic_free(struct nfp_app *app, struct nfp_net *nn)
 {
 	if (nn->dp.bpf_offload_xdp)
 		nfp_bpf_xdp_offload(app, nn, NULL);
@@ -121,23 +121,21 @@ static void nfp_bpf_vnic_clean(struct nfp_app *app, struct nfp_net *nn)
 }
 
 static int nfp_bpf_setup_tc(struct nfp_app *app, struct net_device *netdev,
-			    u32 handle, __be16 proto, struct tc_to_netdev *tc)
+			    enum tc_setup_type type, void *type_data)
 {
+	struct tc_cls_bpf_offload *cls_bpf = type_data;
 	struct nfp_net *nn = netdev_priv(netdev);
 
-	if (TC_H_MAJ(handle) != TC_H_MAJ(TC_H_INGRESS))
-		return -EOPNOTSUPP;
-	if (proto != htons(ETH_P_ALL))
+	if (type != TC_SETUP_CLSBPF || !nfp_net_ebpf_capable(nn) ||
+	    !is_classid_clsact_ingress(cls_bpf->common.classid) ||
+	    cls_bpf->common.protocol != htons(ETH_P_ALL) ||
+	    cls_bpf->common.chain_index)
 		return -EOPNOTSUPP;
 
-	if (tc->type == TC_SETUP_CLSBPF && nfp_net_ebpf_capable(nn)) {
-		if (!nn->dp.bpf_offload_xdp)
-			return nfp_net_bpf_offload(nn, tc->cls_bpf);
-		else
-			return -EBUSY;
-	}
+	if (nn->dp.bpf_offload_xdp)
+		return -EBUSY;
 
-	return -EINVAL;
+	return nfp_net_bpf_offload(nn, cls_bpf);
 }
 
 static bool nfp_bpf_tc_busy(struct nfp_app *app, struct nfp_net *nn)
@@ -151,8 +149,8 @@ const struct nfp_app_type app_bpf = {
 
 	.extra_cap	= nfp_bpf_extra_cap,
 
-	.vnic_init	= nfp_bpf_vnic_init,
-	.vnic_clean	= nfp_bpf_vnic_clean,
+	.vnic_alloc	= nfp_bpf_vnic_alloc,
+	.vnic_free	= nfp_bpf_vnic_free,
 
 	.setup_tc	= nfp_bpf_setup_tc,
 	.tc_busy	= nfp_bpf_tc_busy,
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/offload.c b/drivers/net/ethernet/netronome/nfp/bpf/offload.c
index 78d80a364edb..a88bb5bc0082 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/offload.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/offload.c
@@ -115,14 +115,14 @@ nfp_net_bpf_get_act(struct nfp_net *nn, struct tc_cls_bpf_offload *cls_bpf)
 
 	/* TC direct action */
 	if (cls_bpf->exts_integrated) {
-		if (tc_no_actions(cls_bpf->exts))
+		if (!tcf_exts_has_actions(cls_bpf->exts))
 			return NN_ACT_DIRECT;
 
 		return -EOPNOTSUPP;
 	}
 
 	/* TC legacy mode */
-	if (!tc_single_action(cls_bpf->exts))
+	if (!tcf_exts_has_one_action(cls_bpf->exts))
 		return -EOPNOTSUPP;
 
 	tcf_exts_to_list(cls_bpf->exts, &actions);
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
index d696ba46f70a..5b783a91b115 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
@@ -79,28 +79,32 @@ nfp_bpf_check_exit(struct nfp_prog *nfp_prog,
 		   const struct bpf_verifier_env *env)
 {
 	const struct bpf_reg_state *reg0 = &env->cur_state.regs[0];
+	u64 imm;
 
 	if (nfp_prog->act == NN_ACT_XDP)
 		return 0;
 
-	if (reg0->type != CONST_IMM) {
-		pr_info("unsupported exit state: %d, imm: %llx\n",
-			reg0->type, reg0->imm);
+	if (!(reg0->type == SCALAR_VALUE && tnum_is_const(reg0->var_off))) {
+		char tn_buf[48];
+
+		tnum_strn(tn_buf, sizeof(tn_buf), reg0->var_off);
+		pr_info("unsupported exit state: %d, var_off: %s\n",
+			reg0->type, tn_buf);
 		return -EINVAL;
 	}
 
-	if (nfp_prog->act != NN_ACT_DIRECT &&
-	    reg0->imm != 0 && (reg0->imm & ~0U) != ~0U) {
+	imm = reg0->var_off.value;
+	if (nfp_prog->act != NN_ACT_DIRECT && imm != 0 && (imm & ~0U) != ~0U) {
 		pr_info("unsupported exit state: %d, imm: %llx\n",
-			reg0->type, reg0->imm);
+			reg0->type, imm);
 		return -EINVAL;
 	}
 
-	if (nfp_prog->act == NN_ACT_DIRECT && reg0->imm <= TC_ACT_REDIRECT &&
-	    reg0->imm != TC_ACT_SHOT && reg0->imm != TC_ACT_STOLEN &&
-	    reg0->imm != TC_ACT_QUEUED) {
+	if (nfp_prog->act == NN_ACT_DIRECT && imm <= TC_ACT_REDIRECT &&
+	    imm != TC_ACT_SHOT && imm != TC_ACT_STOLEN &&
+	    imm != TC_ACT_QUEUED) {
 		pr_info("unsupported exit state: %d, imm: %llx\n",
-			reg0->type, reg0->imm);
+			reg0->type, imm);
 		return -EINVAL;
 	}
 
diff --git a/drivers/net/ethernet/netronome/nfp/flower/cmsg.c b/drivers/net/ethernet/netronome/nfp/flower/cmsg.c
index dd7fa9cf225f..c3ca05d10fe1 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/cmsg.c
+++ b/drivers/net/ethernet/netronome/nfp/flower/cmsg.c
@@ -34,10 +34,12 @@
 #include <linux/bitfield.h>
 #include <linux/netdevice.h>
 #include <linux/skbuff.h>
+#include <linux/workqueue.h>
 #include <net/dst_metadata.h>
 
 #include "main.h"
 #include "../nfpcore/nfp_cpp.h"
+#include "../nfp_net.h"
 #include "../nfp_net_repr.h"
 #include "./cmsg.h"
 
@@ -75,6 +77,39 @@ nfp_flower_cmsg_alloc(struct nfp_app *app, unsigned int size,
 	return skb;
 }
 
+struct sk_buff *
+nfp_flower_cmsg_mac_repr_start(struct nfp_app *app, unsigned int num_ports)
+{
+	struct nfp_flower_cmsg_mac_repr *msg;
+	struct sk_buff *skb;
+	unsigned int size;
+
+	size = sizeof(*msg) + num_ports * sizeof(msg->ports[0]);
+	skb = nfp_flower_cmsg_alloc(app, size, NFP_FLOWER_CMSG_TYPE_MAC_REPR);
+	if (!skb)
+		return NULL;
+
+	msg = nfp_flower_cmsg_get_data(skb);
+	memset(msg->reserved, 0, sizeof(msg->reserved));
+	msg->num_ports = num_ports;
+
+	return skb;
+}
+
+void
+nfp_flower_cmsg_mac_repr_add(struct sk_buff *skb, unsigned int idx,
+			     unsigned int nbi, unsigned int nbi_port,
+			     unsigned int phys_port)
+{
+	struct nfp_flower_cmsg_mac_repr *msg;
+
+	msg = nfp_flower_cmsg_get_data(skb);
+	msg->ports[idx].idx = idx;
+	msg->ports[idx].info = nbi & NFP_FLOWER_CMSG_MAC_REPR_NBI;
+	msg->ports[idx].nbi_port = nbi_port;
+	msg->ports[idx].phys_port = phys_port;
+}
+
 int nfp_flower_cmsg_portmod(struct nfp_repr *repr, bool carrier_ok)
 {
 	struct nfp_flower_cmsg_portmod *msg;
@@ -106,27 +141,33 @@ nfp_flower_cmsg_portmod_rx(struct nfp_app *app, struct sk_buff *skb)
 	msg = nfp_flower_cmsg_get_data(skb);
 	link = msg->info & NFP_FLOWER_CMSG_PORTMOD_INFO_LINK;
 
+	rtnl_lock();
 	rcu_read_lock();
 	netdev = nfp_app_repr_get(app, be32_to_cpu(msg->portnum));
+	rcu_read_unlock();
 	if (!netdev) {
 		nfp_flower_cmsg_warn(app, "ctrl msg for unknown port 0x%08x\n",
 				     be32_to_cpu(msg->portnum));
-		rcu_read_unlock();
+		rtnl_unlock();
 		return;
 	}
 
 	if (link) {
+		u16 mtu = be16_to_cpu(msg->mtu);
+
 		netif_carrier_on(netdev);
-		rtnl_lock();
-		dev_set_mtu(netdev, be16_to_cpu(msg->mtu));
-		rtnl_unlock();
+
+		/* An MTU of 0 from the firmware should be ignored */
+		if (mtu)
+			dev_set_mtu(netdev, mtu);
 	} else {
 		netif_carrier_off(netdev);
 	}
-	rcu_read_unlock();
+	rtnl_unlock();
 }
 
-void nfp_flower_cmsg_rx(struct nfp_app *app, struct sk_buff *skb)
+static void
+nfp_flower_cmsg_process_one_rx(struct nfp_app *app, struct sk_buff *skb)
 {
 	struct nfp_flower_cmsg_hdr *cmsg_hdr;
 	enum nfp_flower_cmsg_type_port type;
@@ -150,8 +191,30 @@ void nfp_flower_cmsg_rx(struct nfp_app *app, struct sk_buff *skb)
 	default:
 		nfp_flower_cmsg_warn(app, "Cannot handle invalid repr control type %u\n",
 				     type);
+		goto out;
 	}
 
+	dev_consume_skb_any(skb);
+	return;
 out:
 	dev_kfree_skb_any(skb);
 }
+
+void nfp_flower_cmsg_process_rx(struct work_struct *work)
+{
+	struct nfp_flower_priv *priv;
+	struct sk_buff *skb;
+
+	priv = container_of(work, struct nfp_flower_priv, cmsg_work);
+
+	while ((skb = skb_dequeue(&priv->cmsg_skbs)))
+		nfp_flower_cmsg_process_one_rx(priv->app, skb);
+}
+
+void nfp_flower_cmsg_rx(struct nfp_app *app, struct sk_buff *skb)
+{
+	struct nfp_flower_priv *priv = app->priv;
+
+	skb_queue_tail(&priv->cmsg_skbs, skb);
+	schedule_work(&priv->cmsg_work);
+}
diff --git a/drivers/net/ethernet/netronome/nfp/flower/cmsg.h b/drivers/net/ethernet/netronome/nfp/flower/cmsg.h
index cf738de170ab..a2ec60344236 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/cmsg.h
+++ b/drivers/net/ethernet/netronome/nfp/flower/cmsg.h
@@ -247,12 +247,27 @@ struct nfp_flower_cmsg_hdr {
 enum nfp_flower_cmsg_type_port {
 	NFP_FLOWER_CMSG_TYPE_FLOW_ADD =		0,
 	NFP_FLOWER_CMSG_TYPE_FLOW_DEL =		2,
+	NFP_FLOWER_CMSG_TYPE_MAC_REPR =		7,
 	NFP_FLOWER_CMSG_TYPE_PORT_MOD =		8,
 	NFP_FLOWER_CMSG_TYPE_FLOW_STATS =	15,
 	NFP_FLOWER_CMSG_TYPE_PORT_ECHO =	16,
 	NFP_FLOWER_CMSG_TYPE_MAX =		32,
 };
 
+/* NFP_FLOWER_CMSG_TYPE_MAC_REPR */
+struct nfp_flower_cmsg_mac_repr {
+	u8 reserved[3];
+	u8 num_ports;
+	struct {
+		u8 idx;
+		u8 info;
+		u8 nbi_port;
+		u8 phys_port;
+	} ports[0];
+};
+
+#define NFP_FLOWER_CMSG_MAC_REPR_NBI		GENMASK(1, 0)
+
 /* NFP_FLOWER_CMSG_TYPE_PORT_MOD */
 struct nfp_flower_cmsg_portmod {
 	__be32 portnum;
@@ -308,7 +323,14 @@ static inline void *nfp_flower_cmsg_get_data(struct sk_buff *skb)
 	return (unsigned char *)skb->data + NFP_FLOWER_CMSG_HLEN;
 }
 
+struct sk_buff *
+nfp_flower_cmsg_mac_repr_start(struct nfp_app *app, unsigned int num_ports);
+void
+nfp_flower_cmsg_mac_repr_add(struct sk_buff *skb, unsigned int idx,
+			     unsigned int nbi, unsigned int nbi_port,
+			     unsigned int phys_port);
 int nfp_flower_cmsg_portmod(struct nfp_repr *repr, bool carrier_ok);
+void nfp_flower_cmsg_process_rx(struct work_struct *work);
 void nfp_flower_cmsg_rx(struct nfp_app *app, struct sk_buff *skb);
 struct sk_buff *
 nfp_flower_cmsg_alloc(struct nfp_app *app, unsigned int size,
diff --git a/drivers/net/ethernet/netronome/nfp/flower/main.c b/drivers/net/ethernet/netronome/nfp/flower/main.c
index 6a65c8b33807..91fe03617106 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/main.c
+++ b/drivers/net/ethernet/netronome/nfp/flower/main.c
@@ -127,6 +127,11 @@ nfp_flower_repr_netdev_stop(struct nfp_app *app, struct nfp_repr *repr)
 
 static void nfp_flower_sriov_disable(struct nfp_app *app)
 {
+	struct nfp_flower_priv *priv = app->priv;
+
+	if (!priv->nn)
+		return;
+
 	nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_VF);
 }
 
@@ -159,12 +164,18 @@ nfp_flower_spawn_vnic_reprs(struct nfp_app *app,
 			goto err_reprs_clean;
 		}
 
+		/* For now we only support 1 PF */
+		WARN_ON(repr_type == NFP_REPR_TYPE_PF && i);
+
 		port = nfp_port_alloc(app, port_type, reprs->reprs[i]);
 		if (repr_type == NFP_REPR_TYPE_PF) {
 			port->pf_id = i;
+			port->vnic = priv->nn->dp.ctrl_bar;
 		} else {
-			port->pf_id = 0; /* For now we only support 1 PF */
+			port->pf_id = 0;
 			port->vf_id = i;
+			port->vnic =
+				app->pf->vf_cfg_mem + i * NFP_NET_CFG_BAR_SZ;
 		}
 
 		eth_hw_addr_random(reprs->reprs[i]);
@@ -197,32 +208,37 @@ err_reprs_clean:
 
 static int nfp_flower_sriov_enable(struct nfp_app *app, int num_vfs)
 {
+	struct nfp_flower_priv *priv = app->priv;
+
+	if (!priv->nn)
+		return 0;
+
 	return nfp_flower_spawn_vnic_reprs(app,
 					   NFP_FLOWER_CMSG_PORT_VNIC_TYPE_VF,
 					   NFP_REPR_TYPE_VF, num_vfs);
 }
 
-static void nfp_flower_stop(struct nfp_app *app)
-{
-	nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PF);
-	nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT);
-
-}
-
 static int
 nfp_flower_spawn_phy_reprs(struct nfp_app *app, struct nfp_flower_priv *priv)
 {
 	struct nfp_eth_table *eth_tbl = app->pf->eth_tbl;
 	struct nfp_reprs *reprs, *old_reprs;
+	struct sk_buff *ctrl_skb;
 	unsigned int i;
 	int err;
 
-	reprs = nfp_reprs_alloc(eth_tbl->max_index + 1);
-	if (!reprs)
+	ctrl_skb = nfp_flower_cmsg_mac_repr_start(app, eth_tbl->count);
+	if (!ctrl_skb)
 		return -ENOMEM;
 
+	reprs = nfp_reprs_alloc(eth_tbl->max_index + 1);
+	if (!reprs) {
+		err = -ENOMEM;
+		goto err_free_ctrl_skb;
+	}
+
 	for (i = 0; i < eth_tbl->count; i++) {
-		int phys_port = eth_tbl->ports[i].index;
+		unsigned int phys_port = eth_tbl->ports[i].index;
 		struct nfp_port *port;
 		u32 cmsg_port_id;
 
@@ -255,6 +271,11 @@ nfp_flower_spawn_phy_reprs(struct nfp_app *app, struct nfp_flower_priv *priv)
 			goto err_reprs_clean;
 		}
 
+		nfp_flower_cmsg_mac_repr_add(ctrl_skb, i,
+					     eth_tbl->ports[i].nbi,
+					     eth_tbl->ports[i].base,
+					     phys_port);
+
 		nfp_info(app->cpp, "Phys Port %d Representor(%s) created\n",
 			 phys_port, reprs->reprs[phys_port]->name);
 	}
@@ -265,37 +286,31 @@ nfp_flower_spawn_phy_reprs(struct nfp_app *app, struct nfp_flower_priv *priv)
 		goto err_reprs_clean;
 	}
 
+	/* The MAC_REPR control message should be sent after the MAC
+	 * representors are registered using nfp_app_reprs_set().  This is
+	 * because the firmware may respond with control messages for the
+	 * MAC representors, f.e. to provide the driver with information
+	 * about their state, and without registration the driver will drop
+	 * any such messages.
+	 */
+	nfp_ctrl_tx(app->ctrl, ctrl_skb);
+
 	return 0;
 err_reprs_clean:
 	nfp_reprs_clean_and_free(reprs);
+err_free_ctrl_skb:
+	kfree_skb(ctrl_skb);
 	return err;
 }
 
-static int nfp_flower_start(struct nfp_app *app)
-{
-	int err;
-
-	err = nfp_flower_spawn_phy_reprs(app, app->priv);
-	if (err)
-		return err;
-
-	return nfp_flower_spawn_vnic_reprs(app,
-					   NFP_FLOWER_CMSG_PORT_VNIC_TYPE_PF,
-					   NFP_REPR_TYPE_PF, 1);
-}
-
-static int nfp_flower_vnic_init(struct nfp_app *app, struct nfp_net *nn,
-				unsigned int id)
+static int nfp_flower_vnic_alloc(struct nfp_app *app, struct nfp_net *nn,
+				 unsigned int id)
 {
-	struct nfp_flower_priv *priv = app->priv;
-
 	if (id > 0) {
 		nfp_warn(app->cpp, "FlowerNIC doesn't support more than one data vNIC\n");
 		goto err_invalid_port;
 	}
 
-	priv->nn = nn;
-
 	eth_hw_addr_random(nn->dp.netdev);
 	netif_keep_dst(nn->dp.netdev);
 
@@ -306,9 +321,59 @@ err_invalid_port:
 	return PTR_ERR_OR_ZERO(nn->port);
 }
 
+static void nfp_flower_vnic_clean(struct nfp_app *app, struct nfp_net *nn)
+{
+	struct nfp_flower_priv *priv = app->priv;
+
+	if (app->pf->num_vfs)
+		nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_VF);
+	nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PF);
+	nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT);
+
+	priv->nn = NULL;
+}
+
+static int nfp_flower_vnic_init(struct nfp_app *app, struct nfp_net *nn)
+{
+	struct nfp_flower_priv *priv = app->priv;
+	int err;
+
+	priv->nn = nn;
+
+	err = nfp_flower_spawn_phy_reprs(app, app->priv);
+	if (err)
+		goto err_clear_nn;
+
+	err = nfp_flower_spawn_vnic_reprs(app,
+					  NFP_FLOWER_CMSG_PORT_VNIC_TYPE_PF,
+					  NFP_REPR_TYPE_PF, 1);
+	if (err)
+		goto err_destroy_reprs_phy;
+
+	if (app->pf->num_vfs) {
+		err = nfp_flower_spawn_vnic_reprs(app,
+						  NFP_FLOWER_CMSG_PORT_VNIC_TYPE_VF,
+						  NFP_REPR_TYPE_VF,
+						  app->pf->num_vfs);
+		if (err)
+			goto err_destroy_reprs_pf;
+	}
+
+	return 0;
+
+err_destroy_reprs_pf:
+	nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PF);
+err_destroy_reprs_phy:
+	nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT);
+err_clear_nn:
+	priv->nn = NULL;
+	return err;
+}
+
 static int nfp_flower_init(struct nfp_app *app)
 {
 	const struct nfp_pf *pf = app->pf;
+	struct nfp_flower_priv *app_priv;
 	u64 version;
 	int err;
 
@@ -339,10 +404,15 @@ static int nfp_flower_init(struct nfp_app *app)
 		return -EINVAL;
 	}
 
-	app->priv = vzalloc(sizeof(struct nfp_flower_priv));
-	if (!app->priv)
+	app_priv = vzalloc(sizeof(struct nfp_flower_priv));
+	if (!app_priv)
 		return -ENOMEM;
 
+	app->priv = app_priv;
+	app_priv->app = app;
+	skb_queue_head_init(&app_priv->cmsg_skbs);
+	INIT_WORK(&app_priv->cmsg_work, nfp_flower_cmsg_process_rx);
+
 	err = nfp_flower_metadata_init(app);
 	if (err)
 		goto err_free_app_priv;
@@ -356,6 +426,11 @@ err_free_app_priv:
 
 static void nfp_flower_clean(struct nfp_app *app)
 {
+	struct nfp_flower_priv *app_priv = app->priv;
+
+	skb_queue_purge(&app_priv->cmsg_skbs);
+	flush_work(&app_priv->cmsg_work);
+
 	nfp_flower_metadata_cleanup(app);
 	vfree(app->priv);
 	app->priv = NULL;
@@ -371,14 +446,13 @@ const struct nfp_app_type app_flower = {
 	.init		= nfp_flower_init,
 	.clean		= nfp_flower_clean,
 
+	.vnic_alloc	= nfp_flower_vnic_alloc,
 	.vnic_init	= nfp_flower_vnic_init,
+	.vnic_clean	= nfp_flower_vnic_clean,
 
 	.repr_open	= nfp_flower_repr_netdev_open,
 	.repr_stop	= nfp_flower_repr_netdev_stop,
 
-	.start		= nfp_flower_start,
-	.stop		= nfp_flower_stop,
-
 	.ctrl_msg_rx	= nfp_flower_cmsg_rx,
 
 	.sriov_enable	= nfp_flower_sriov_enable,
diff --git a/drivers/net/ethernet/netronome/nfp/flower/main.h b/drivers/net/ethernet/netronome/nfp/flower/main.h
index 9e64c048e83f..c20dd00a1cae 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/main.h
+++ b/drivers/net/ethernet/netronome/nfp/flower/main.h
@@ -38,8 +38,9 @@
 #include <linux/hashtable.h>
 #include <linux/time64.h>
 #include <linux/types.h>
+#include <net/pkt_cls.h>
+#include <linux/workqueue.h>
 
-struct tc_to_netdev;
 struct net_device;
 struct nfp_app;
 
@@ -71,6 +72,7 @@ struct nfp_fl_stats_id {
 
 /**
  * struct nfp_flower_priv - Flower APP per-vNIC priv data
+ * @app:		Back pointer to app
  * @nn:			Pointer to vNIC
  * @mask_id_seed:	Seed used for mask hash table
  * @flower_version:	HW version of flower
@@ -78,8 +80,11 @@ struct nfp_fl_stats_id {
  * @mask_ids:		List of free mask ids
  * @mask_table:		Hash table used to store masks
  * @flow_table:		Hash table used to store flower rules
+ * @cmsg_work:		Workqueue for control messages processing
+ * @cmsg_skbs:		List of skbs for control message processing
  */
 struct nfp_flower_priv {
+	struct nfp_app *app;
 	struct nfp_net *nn;
 	u32 mask_id_seed;
 	u64 flower_version;
@@ -87,6 +92,8 @@ struct nfp_flower_priv {
 	struct nfp_fl_mask_id mask_ids;
 	DECLARE_HASHTABLE(mask_table, NFP_FLOWER_MASK_HASH_BITS);
 	DECLARE_HASHTABLE(flow_table, NFP_FLOWER_HASH_BITS);
+	struct work_struct cmsg_work;
+	struct sk_buff_head cmsg_skbs;
 };
 
 struct nfp_fl_key_ls {
@@ -135,7 +142,7 @@ int nfp_flower_metadata_init(struct nfp_app *app);
 void nfp_flower_metadata_cleanup(struct nfp_app *app);
 
 int nfp_flower_setup_tc(struct nfp_app *app, struct net_device *netdev,
-			u32 handle, __be16 proto, struct tc_to_netdev *tc);
+			enum tc_setup_type type, void *type_data);
 int nfp_flower_compile_flow_match(struct tc_cls_flower_offload *flow,
 				  struct nfp_fl_key_ls *key_ls,
 				  struct net_device *netdev,
diff --git a/drivers/net/ethernet/netronome/nfp/flower/match.c b/drivers/net/ethernet/netronome/nfp/flower/match.c
index 0e08404480ef..d25b5038c3a2 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/match.c
+++ b/drivers/net/ethernet/netronome/nfp/flower/match.c
@@ -42,33 +42,29 @@ nfp_flower_compile_meta_tci(struct nfp_flower_meta_two *frame,
 			    struct tc_cls_flower_offload *flow, u8 key_type,
 			    bool mask_version)
 {
+	struct fl_flow_key *target = mask_version ? flow->mask : flow->key;
 	struct flow_dissector_key_vlan *flow_vlan;
 	u16 tmp_tci;
 
+	memset(frame, 0, sizeof(struct nfp_flower_meta_two));
 	/* Populate the metadata frame. */
 	frame->nfp_flow_key_layer = key_type;
 	frame->mask_id = ~0;
 
-	if (mask_version) {
-		frame->tci = cpu_to_be16(~0);
-		return;
-	}
-
-	flow_vlan = skb_flow_dissector_target(flow->dissector,
-					      FLOW_DISSECTOR_KEY_VLAN,
-					      flow->key);
-
-	/* Populate the tci field. */
-	if (!flow_vlan->vlan_id) {
-		tmp_tci = 0;
-	} else {
-		tmp_tci = FIELD_PREP(NFP_FLOWER_MASK_VLAN_PRIO,
-				     flow_vlan->vlan_priority) |
-			  FIELD_PREP(NFP_FLOWER_MASK_VLAN_VID,
-				     flow_vlan->vlan_id) |
-			  NFP_FLOWER_MASK_VLAN_CFI;
+	if (dissector_uses_key(flow->dissector, FLOW_DISSECTOR_KEY_VLAN)) {
+		flow_vlan = skb_flow_dissector_target(flow->dissector,
+						      FLOW_DISSECTOR_KEY_VLAN,
+						      target);
+		/* Populate the tci field. */
+		if (flow_vlan->vlan_id) {
+			tmp_tci = FIELD_PREP(NFP_FLOWER_MASK_VLAN_PRIO,
+					     flow_vlan->vlan_priority) |
+				  FIELD_PREP(NFP_FLOWER_MASK_VLAN_VID,
+					     flow_vlan->vlan_id) |
+				  NFP_FLOWER_MASK_VLAN_CFI;
+			frame->tci = cpu_to_be16(tmp_tci);
+		}
 	}
-	frame->tci = cpu_to_be16(tmp_tci);
 }
 
 static void
@@ -99,17 +95,18 @@ nfp_flower_compile_mac(struct nfp_flower_mac_mpls *frame,
 		       bool mask_version)
 {
 	struct fl_flow_key *target = mask_version ? flow->mask : flow->key;
-	struct flow_dissector_key_eth_addrs *flow_mac;
-
-	flow_mac = skb_flow_dissector_target(flow->dissector,
-					     FLOW_DISSECTOR_KEY_ETH_ADDRS,
-					     target);
+	struct flow_dissector_key_eth_addrs *addr;
 
 	memset(frame, 0, sizeof(struct nfp_flower_mac_mpls));
 
-	/* Populate mac frame. */
-	ether_addr_copy(frame->mac_dst, &flow_mac->dst[0]);
-	ether_addr_copy(frame->mac_src, &flow_mac->src[0]);
+	if (dissector_uses_key(flow->dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
+		addr = skb_flow_dissector_target(flow->dissector,
+						 FLOW_DISSECTOR_KEY_ETH_ADDRS,
+						 target);
+		/* Populate mac frame. */
+		ether_addr_copy(frame->mac_dst, &addr->dst[0]);
+		ether_addr_copy(frame->mac_src, &addr->src[0]);
+	}
 
 	if (mask_version)
 		frame->mpls_lse = cpu_to_be32(~0);
@@ -121,14 +118,17 @@ nfp_flower_compile_tport(struct nfp_flower_tp_ports *frame,
 			 bool mask_version)
 {
 	struct fl_flow_key *target = mask_version ? flow->mask : flow->key;
-	struct flow_dissector_key_ports *flow_tp;
+	struct flow_dissector_key_ports *tp;
 
-	flow_tp = skb_flow_dissector_target(flow->dissector,
-					    FLOW_DISSECTOR_KEY_PORTS,
-					    target);
+	memset(frame, 0, sizeof(struct nfp_flower_tp_ports));
 
-	frame->port_src = flow_tp->src;
-	frame->port_dst = flow_tp->dst;
+	if (dissector_uses_key(flow->dissector, FLOW_DISSECTOR_KEY_PORTS)) {
+		tp = skb_flow_dissector_target(flow->dissector,
+					       FLOW_DISSECTOR_KEY_PORTS,
+					       target);
+		frame->port_src = tp->src;
+		frame->port_dst = tp->dst;
+	}
 }
 
 static void
@@ -137,25 +137,27 @@ nfp_flower_compile_ipv4(struct nfp_flower_ipv4 *frame,
 			bool mask_version)
 {
 	struct fl_flow_key *target = mask_version ? flow->mask : flow->key;
-	struct flow_dissector_key_ipv4_addrs *flow_ipv4;
-	struct flow_dissector_key_basic *flow_basic;
-
-	flow_ipv4 = skb_flow_dissector_target(flow->dissector,
-					      FLOW_DISSECTOR_KEY_IPV4_ADDRS,
-					      target);
-
-	flow_basic = skb_flow_dissector_target(flow->dissector,
-					       FLOW_DISSECTOR_KEY_BASIC,
-					       target);
+	struct flow_dissector_key_ipv4_addrs *addr;
+	struct flow_dissector_key_basic *basic;
 
-	/* Populate IPv4 frame. */
-	frame->reserved = 0;
-	frame->ipv4_src = flow_ipv4->src;
-	frame->ipv4_dst = flow_ipv4->dst;
-	frame->proto = flow_basic->ip_proto;
 	/* Wildcard TOS/TTL for now. */
-	frame->tos = 0;
-	frame->ttl = 0;
+	memset(frame, 0, sizeof(struct nfp_flower_ipv4));
+
+	if (dissector_uses_key(flow->dissector,
+			       FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
+		addr = skb_flow_dissector_target(flow->dissector,
+						 FLOW_DISSECTOR_KEY_IPV4_ADDRS,
+						 target);
+		frame->ipv4_src = addr->src;
+		frame->ipv4_dst = addr->dst;
+	}
+
+	if (dissector_uses_key(flow->dissector, FLOW_DISSECTOR_KEY_BASIC)) {
+		basic = skb_flow_dissector_target(flow->dissector,
+						  FLOW_DISSECTOR_KEY_BASIC,
+						  target);
+		frame->proto = basic->ip_proto;
+	}
 }
 
 static void
@@ -164,26 +166,27 @@ nfp_flower_compile_ipv6(struct nfp_flower_ipv6 *frame,
 			bool mask_version)
 {
 	struct fl_flow_key *target = mask_version ? flow->mask : flow->key;
-	struct flow_dissector_key_ipv6_addrs *flow_ipv6;
-	struct flow_dissector_key_basic *flow_basic;
-
-	flow_ipv6 = skb_flow_dissector_target(flow->dissector,
-					      FLOW_DISSECTOR_KEY_IPV6_ADDRS,
-					      target);
+	struct flow_dissector_key_ipv6_addrs *addr;
+	struct flow_dissector_key_basic *basic;
 
-	flow_basic = skb_flow_dissector_target(flow->dissector,
-					       FLOW_DISSECTOR_KEY_BASIC,
-					       target);
-
-	/* Populate IPv6 frame. */
-	frame->reserved = 0;
-	frame->ipv6_src = flow_ipv6->src;
-	frame->ipv6_dst = flow_ipv6->dst;
-	frame->proto = flow_basic->ip_proto;
 	/* Wildcard LABEL/TOS/TTL for now. */
-	frame->ipv6_flow_label_exthdr = 0;
-	frame->tos = 0;
-	frame->ttl = 0;
+	memset(frame, 0, sizeof(struct nfp_flower_ipv6));
+
+	if (dissector_uses_key(flow->dissector,
+			       FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
+		addr = skb_flow_dissector_target(flow->dissector,
+						 FLOW_DISSECTOR_KEY_IPV6_ADDRS,
+						 target);
+		frame->ipv6_src = addr->src;
+		frame->ipv6_dst = addr->dst;
+	}
+
+	if (dissector_uses_key(flow->dissector, FLOW_DISSECTOR_KEY_BASIC)) {
+		basic = skb_flow_dissector_target(flow->dissector,
+						  FLOW_DISSECTOR_KEY_BASIC,
+						  target);
+		frame->proto = basic->ip_proto;
+	}
 }
 
 int nfp_flower_compile_flow_match(struct tc_cls_flower_offload *flow,
diff --git a/drivers/net/ethernet/netronome/nfp/flower/offload.c b/drivers/net/ethernet/netronome/nfp/flower/offload.c
index 4ad10bd5e139..a18b4d2b1d3e 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/offload.c
+++ b/drivers/net/ethernet/netronome/nfp/flower/offload.c
@@ -44,6 +44,16 @@
 #include "../nfp_net.h"
 #include "../nfp_port.h"
 
+#define NFP_FLOWER_WHITELIST_DISSECTOR \
+	(BIT(FLOW_DISSECTOR_KEY_CONTROL) | \
+	 BIT(FLOW_DISSECTOR_KEY_BASIC) | \
+	 BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | \
+	 BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | \
+	 BIT(FLOW_DISSECTOR_KEY_PORTS) | \
+	 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | \
+	 BIT(FLOW_DISSECTOR_KEY_VLAN) | \
+	 BIT(FLOW_DISSECTOR_KEY_IP))
+
 static int
 nfp_flower_xmit_flow(struct net_device *netdev,
 		     struct nfp_fl_payload *nfp_flow, u8 mtype)
@@ -105,43 +115,65 @@ static int
 nfp_flower_calculate_key_layers(struct nfp_fl_key_ls *ret_key_ls,
 				struct tc_cls_flower_offload *flow)
 {
-	struct flow_dissector_key_control *mask_enc_ctl;
-	struct flow_dissector_key_basic *mask_basic;
-	struct flow_dissector_key_basic *key_basic;
+	struct flow_dissector_key_basic *mask_basic = NULL;
+	struct flow_dissector_key_basic *key_basic = NULL;
+	struct flow_dissector_key_ip *mask_ip = NULL;
 	u32 key_layer_two;
 	u8 key_layer;
 	int key_size;
 
-	mask_enc_ctl = skb_flow_dissector_target(flow->dissector,
-						 FLOW_DISSECTOR_KEY_ENC_CONTROL,
-						 flow->mask);
+	if (flow->dissector->used_keys & ~NFP_FLOWER_WHITELIST_DISSECTOR)
+		return -EOPNOTSUPP;
+
+	if (dissector_uses_key(flow->dissector,
+			       FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
+		struct flow_dissector_key_control *mask_enc_ctl =
+			skb_flow_dissector_target(flow->dissector,
+						  FLOW_DISSECTOR_KEY_ENC_CONTROL,
+						  flow->mask);
+		/* We are expecting a tunnel. For now we ignore offloading. */
+		if (mask_enc_ctl->addr_type)
+			return -EOPNOTSUPP;
+	}
+
+	if (dissector_uses_key(flow->dissector, FLOW_DISSECTOR_KEY_BASIC)) {
+		mask_basic = skb_flow_dissector_target(flow->dissector,
+						       FLOW_DISSECTOR_KEY_BASIC,
+						       flow->mask);
+
+		key_basic = skb_flow_dissector_target(flow->dissector,
+						      FLOW_DISSECTOR_KEY_BASIC,
+						      flow->key);
+	}
 
-	mask_basic = skb_flow_dissector_target(flow->dissector,
-					       FLOW_DISSECTOR_KEY_BASIC,
-					       flow->mask);
+	if (dissector_uses_key(flow->dissector, FLOW_DISSECTOR_KEY_IP))
+		mask_ip = skb_flow_dissector_target(flow->dissector,
+						    FLOW_DISSECTOR_KEY_IP,
+						    flow->mask);
 
-	key_basic = skb_flow_dissector_target(flow->dissector,
-					      FLOW_DISSECTOR_KEY_BASIC,
-					      flow->key);
 	key_layer_two = 0;
 	key_layer = NFP_FLOWER_LAYER_PORT | NFP_FLOWER_LAYER_MAC;
 	key_size = sizeof(struct nfp_flower_meta_one) +
 		   sizeof(struct nfp_flower_in_port) +
 		   sizeof(struct nfp_flower_mac_mpls);
 
-	/* We are expecting a tunnel. For now we ignore offloading. */
-	if (mask_enc_ctl->addr_type)
-		return -EOPNOTSUPP;
-
-	if (mask_basic->n_proto) {
+	if (mask_basic && mask_basic->n_proto) {
 		/* Ethernet type is present in the key. */
 		switch (key_basic->n_proto) {
 		case cpu_to_be16(ETH_P_IP):
+			if (mask_ip && mask_ip->tos)
+				return -EOPNOTSUPP;
+			if (mask_ip && mask_ip->ttl)
+				return -EOPNOTSUPP;
 			key_layer |= NFP_FLOWER_LAYER_IPV4;
 			key_size += sizeof(struct nfp_flower_ipv4);
 			break;
 
 		case cpu_to_be16(ETH_P_IPV6):
+			if (mask_ip && mask_ip->tos)
+				return -EOPNOTSUPP;
+			if (mask_ip && mask_ip->ttl)
+				return -EOPNOTSUPP;
 			key_layer |= NFP_FLOWER_LAYER_IPV6;
 			key_size += sizeof(struct nfp_flower_ipv6);
 			break;
@@ -152,6 +184,11 @@ nfp_flower_calculate_key_layers(struct nfp_fl_key_ls *ret_key_ls,
 		case cpu_to_be16(ETH_P_ARP):
 			return -EOPNOTSUPP;
 
+		/* Currently we do not offload MPLS. */
+		case cpu_to_be16(ETH_P_MPLS_UC):
+		case cpu_to_be16(ETH_P_MPLS_MC):
+			return -EOPNOTSUPP;
+
 		/* Will be included in layer 2. */
 		case cpu_to_be16(ETH_P_8021Q):
 			break;
@@ -166,7 +203,7 @@ nfp_flower_calculate_key_layers(struct nfp_fl_key_ls *ret_key_ls,
 		}
 	}
 
-	if (mask_basic->ip_proto) {
+	if (mask_basic && mask_basic->ip_proto) {
 		/* Ethernet type is present in the key. */
 		switch (key_basic->ip_proto) {
 		case IPPROTO_TCP:
@@ -385,16 +422,15 @@ nfp_flower_repr_offload(struct nfp_app *app, struct net_device *netdev,
 }
 
 int nfp_flower_setup_tc(struct nfp_app *app, struct net_device *netdev,
-			u32 handle, __be16 proto, struct tc_to_netdev *tc)
+			enum tc_setup_type type, void *type_data)
 {
-	if (TC_H_MAJ(handle) != TC_H_MAJ(TC_H_INGRESS))
-		return -EOPNOTSUPP;
+	struct tc_cls_flower_offload *cls_flower = type_data;
 
-	if (!eth_proto_is_802_3(proto))
+	if (type != TC_SETUP_CLSFLOWER ||
+	    !is_classid_clsact_ingress(cls_flower->common.classid) ||
+	    !eth_proto_is_802_3(cls_flower->common.protocol) ||
+	    cls_flower->common.chain_index)
 		return -EOPNOTSUPP;
 
-	if (tc->type != TC_SETUP_CLSFLOWER)
-		return -EINVAL;
-
-	return nfp_flower_repr_offload(app, netdev, tc->cls_flower);
+	return nfp_flower_repr_offload(app, netdev, cls_flower);
 }
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_app.c b/drivers/net/ethernet/netronome/nfp/nfp_app.c
index c704c022574f..82c290763529 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_app.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_app.c
@@ -38,6 +38,7 @@
 #include "nfpcore/nfp_nffw.h"
 #include "nfp_app.h"
 #include "nfp_main.h"
+#include "nfp_net.h"
 #include "nfp_net_repr.h"
 
 static const struct nfp_app_type *apps[] = {
@@ -48,6 +49,25 @@ static const struct nfp_app_type *apps[] = {
 #endif
 };
 
+struct nfp_app *nfp_app_from_netdev(struct net_device *netdev)
+{
+	if (nfp_netdev_is_nfp_net(netdev)) {
+		struct nfp_net *nn = netdev_priv(netdev);
+
+		return nn->app;
+	}
+
+	if (nfp_netdev_is_nfp_repr(netdev)) {
+		struct nfp_repr *repr = netdev_priv(netdev);
+
+		return repr->app;
+	}
+
+	WARN(1, "Unknown netdev type for nfp_app\n");
+
+	return NULL;
+}
+
 const char *nfp_app_mip_name(struct nfp_app *app)
 {
 	if (!app || !app->pf->mip)
@@ -105,7 +125,7 @@ struct nfp_app *nfp_app_alloc(struct nfp_pf *pf, enum nfp_app_id id)
 		return ERR_PTR(-EINVAL);
 	}
 
-	if (WARN_ON(!apps[i]->name || !apps[i]->vnic_init))
+	if (WARN_ON(!apps[i]->name || !apps[i]->vnic_alloc))
 		return ERR_PTR(-EINVAL);
 
 	app = kzalloc(sizeof(*app), GFP_KERNEL);
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_app.h b/drivers/net/ethernet/netronome/nfp/nfp_app.h
index 5d714e10d9a9..af640b5c2108 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_app.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_app.h
@@ -42,7 +42,6 @@ struct bpf_prog;
 struct net_device;
 struct pci_dev;
 struct sk_buff;
-struct tc_to_netdev;
 struct sk_buff;
 struct nfp_app;
 struct nfp_cpp;
@@ -70,8 +69,10 @@ extern const struct nfp_app_type app_flower;
  * @init:	perform basic app checks and init
  * @clean:	clean app state
  * @extra_cap:	extra capabilities string
- * @vnic_init:	init vNICs (assign port types, etc.)
- * @vnic_clean:	clean up app's vNIC state
+ * @vnic_alloc:	allocate vNICs (assign port types, etc.)
+ * @vnic_free:	free up app's vNIC state
+ * @vnic_init:	vNIC netdev was registered
+ * @vnic_clean:	vNIC netdev about to be unregistered
  * @repr_open:	representor netdev open callback
  * @repr_stop:	representor netdev stop callback
  * @start:	start application logic
@@ -96,8 +97,10 @@ struct nfp_app_type {
 
 	const char *(*extra_cap)(struct nfp_app *app, struct nfp_net *nn);
 
-	int (*vnic_init)(struct nfp_app *app, struct nfp_net *nn,
-			 unsigned int id);
+	int (*vnic_alloc)(struct nfp_app *app, struct nfp_net *nn,
+			  unsigned int id);
+	void (*vnic_free)(struct nfp_app *app, struct nfp_net *nn);
+	int (*vnic_init)(struct nfp_app *app, struct nfp_net *nn);
 	void (*vnic_clean)(struct nfp_app *app, struct nfp_net *nn);
 
 	int (*repr_open)(struct nfp_app *app, struct nfp_repr *repr);
@@ -109,7 +112,7 @@ struct nfp_app_type {
 	void (*ctrl_msg_rx)(struct nfp_app *app, struct sk_buff *skb);
 
 	int (*setup_tc)(struct nfp_app *app, struct net_device *netdev,
-			u32 handle, __be16 proto, struct tc_to_netdev *tc);
+			enum tc_setup_type type, void *type_data);
 	bool (*tc_busy)(struct nfp_app *app, struct nfp_net *nn);
 	int (*xdp_offload)(struct nfp_app *app, struct nfp_net *nn,
 			   struct bpf_prog *prog);
@@ -158,10 +161,23 @@ static inline void nfp_app_clean(struct nfp_app *app)
 		app->type->clean(app);
 }
 
-static inline int nfp_app_vnic_init(struct nfp_app *app, struct nfp_net *nn,
-				    unsigned int id)
+static inline int nfp_app_vnic_alloc(struct nfp_app *app, struct nfp_net *nn,
+				     unsigned int id)
+{
+	return app->type->vnic_alloc(app, nn, id);
+}
+
+static inline void nfp_app_vnic_free(struct nfp_app *app, struct nfp_net *nn)
+{
+	if (app->type->vnic_free)
+		app->type->vnic_free(app, nn);
+}
+
+static inline int nfp_app_vnic_init(struct nfp_app *app, struct nfp_net *nn)
 {
-	return app->type->vnic_init(app, nn, id);
+	if (!app->type->vnic_init)
+		return 0;
+	return app->type->vnic_init(app, nn);
 }
 
 static inline void nfp_app_vnic_clean(struct nfp_app *app, struct nfp_net *nn)
@@ -238,12 +254,11 @@ static inline bool nfp_app_tc_busy(struct nfp_app *app, struct nfp_net *nn)
 
 static inline int nfp_app_setup_tc(struct nfp_app *app,
 				   struct net_device *netdev,
-				   u32 handle, __be16 proto,
-				   struct tc_to_netdev *tc)
+				   enum tc_setup_type type, void *type_data)
 {
 	if (!app || !app->type->setup_tc)
 		return -EOPNOTSUPP;
-	return app->type->setup_tc(app, netdev, handle, proto, tc);
+	return app->type->setup_tc(app, netdev, type, type_data);
 }
 
 static inline int nfp_app_xdp_offload(struct nfp_app *app, struct nfp_net *nn,
@@ -295,6 +310,8 @@ static inline struct net_device *nfp_app_repr_get(struct nfp_app *app, u32 id)
 	return app->type->repr_get(app, id);
 }
 
+struct nfp_app *nfp_app_from_netdev(struct net_device *netdev);
+
 struct nfp_reprs *
 nfp_app_reprs_set(struct nfp_app *app, enum nfp_repr_type type,
 		  struct nfp_reprs *reprs);
@@ -308,7 +325,7 @@ void nfp_app_free(struct nfp_app *app);
 
 /* Callbacks shared between apps */
 
-int nfp_app_nic_vnic_init(struct nfp_app *app, struct nfp_net *nn,
-			  unsigned int id);
+int nfp_app_nic_vnic_alloc(struct nfp_app *app, struct nfp_net *nn,
+			   unsigned int id);
 
 #endif
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_app_nic.c b/drivers/net/ethernet/netronome/nfp/nfp_app_nic.c
index 4e37c81f9eaf..2a2f2fbc8850 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_app_nic.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_app_nic.c
@@ -60,8 +60,8 @@ nfp_app_nic_vnic_init_phy_port(struct nfp_pf *pf, struct nfp_app *app,
 	return nn->port->type == NFP_PORT_INVALID;
 }
 
-int nfp_app_nic_vnic_init(struct nfp_app *app, struct nfp_net *nn,
-			  unsigned int id)
+int nfp_app_nic_vnic_alloc(struct nfp_app *app, struct nfp_net *nn,
+			   unsigned int id)
 {
 	int err;
 
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_main.c b/drivers/net/ethernet/netronome/nfp/nfp_main.c
index d67969d3e484..f8fa63b66739 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_main.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_main.c
@@ -74,6 +74,45 @@ static const struct pci_device_id nfp_pci_device_ids[] = {
 };
 MODULE_DEVICE_TABLE(pci, nfp_pci_device_ids);
 
+static bool nfp_board_ready(struct nfp_pf *pf)
+{
+	const char *cp;
+	long state;
+	int err;
+
+	cp = nfp_hwinfo_lookup(pf->hwinfo, "board.state");
+	if (!cp)
+		return false;
+
+	err = kstrtol(cp, 0, &state);
+	if (err < 0)
+		return false;
+
+	return state == 15;
+}
+
+static int nfp_pf_board_state_wait(struct nfp_pf *pf)
+{
+	const unsigned long wait_until = jiffies + 10 * HZ;
+
+	while (!nfp_board_ready(pf)) {
+		if (time_is_before_eq_jiffies(wait_until)) {
+			nfp_err(pf->cpp, "NFP board initialization timeout\n");
+			return -EINVAL;
+		}
+
+		nfp_info(pf->cpp, "waiting for board initialization\n");
+		if (msleep_interruptible(500))
+			return -ERESTARTSYS;
+
+		/* Refresh cached information */
+		kfree(pf->hwinfo);
+		pf->hwinfo = nfp_hwinfo_read(pf->cpp);
+	}
+
+	return 0;
+}
+
 static int nfp_pcie_sriov_read_nfd_limit(struct nfp_pf *pf)
 {
 	int err;
@@ -98,21 +137,20 @@ static int nfp_pcie_sriov_enable(struct pci_dev *pdev, int num_vfs)
 	struct nfp_pf *pf = pci_get_drvdata(pdev);
 	int err;
 
-	mutex_lock(&pf->lock);
-
 	if (num_vfs > pf->limit_vfs) {
 		nfp_info(pf->cpp, "Firmware limits number of VFs to %u\n",
 			 pf->limit_vfs);
-		err = -EINVAL;
-		goto err_unlock;
+		return -EINVAL;
 	}
 
 	err = pci_enable_sriov(pdev, num_vfs);
 	if (err) {
 		dev_warn(&pdev->dev, "Failed to enable PCI SR-IOV: %d\n", err);
-		goto err_unlock;
+		return err;
 	}
 
+	mutex_lock(&pf->lock);
+
 	err = nfp_app_sriov_enable(pf->app, num_vfs);
 	if (err) {
 		dev_warn(&pdev->dev,
@@ -129,9 +167,8 @@ static int nfp_pcie_sriov_enable(struct pci_dev *pdev, int num_vfs)
 	return num_vfs;
 
 err_sriov_disable:
-	pci_disable_sriov(pdev);
-err_unlock:
 	mutex_unlock(&pf->lock);
+	pci_disable_sriov(pdev);
 	return err;
 #endif
 	return 0;
@@ -158,10 +195,10 @@ static int nfp_pcie_sriov_disable(struct pci_dev *pdev)
 
 	pf->num_vfs = 0;
 
+	mutex_unlock(&pf->lock);
+
 	pci_disable_sriov(pdev);
 	dev_dbg(&pdev->dev, "Removed VFs.\n");
-
-	mutex_unlock(&pf->lock);
 #endif
 	return 0;
 }
@@ -174,6 +211,21 @@ static int nfp_pcie_sriov_configure(struct pci_dev *pdev, int num_vfs)
 		return nfp_pcie_sriov_enable(pdev, num_vfs);
 }
 
+static const struct firmware *
+nfp_net_fw_request(struct pci_dev *pdev, struct nfp_pf *pf, const char *name)
+{
+	const struct firmware *fw = NULL;
+	int err;
+
+	err = request_firmware_direct(&fw, name, &pdev->dev);
+	nfp_info(pf->cpp, "  %s: %s\n",
+		 name, err ? "not found" : "found, loading...");
+	if (err)
+		return NULL;
+
+	return fw;
+}
+
 /**
  * nfp_net_fw_find() - Find the correct firmware image for netdev mode
  * @pdev:	PCI Device structure
@@ -184,13 +236,32 @@ static int nfp_pcie_sriov_configure(struct pci_dev *pdev, int num_vfs)
 static const struct firmware *
 nfp_net_fw_find(struct pci_dev *pdev, struct nfp_pf *pf)
 {
-	const struct firmware *fw = NULL;
 	struct nfp_eth_table_port *port;
+	const struct firmware *fw;
 	const char *fw_model;
 	char fw_name[256];
-	int spc, err = 0;
-	int i, j;
+	const u8 *serial;
+	u16 interface;
+	int spc, i, j;
 
+	nfp_info(pf->cpp, "Looking for firmware file in order of priority:\n");
+
+	/* First try to find a firmware image specific for this device */
+	interface = nfp_cpp_interface(pf->cpp);
+	nfp_cpp_serial(pf->cpp, &serial);
+	sprintf(fw_name, "netronome/serial-%pMF-%02hhx-%02hhx.nffw",
+		serial, interface >> 8, interface & 0xff);
+	fw = nfp_net_fw_request(pdev, pf, fw_name);
+	if (fw)
+		return fw;
+
+	/* Then try the PCI name */
+	sprintf(fw_name, "netronome/pci-%s.nffw", pci_name(pdev));
+	fw = nfp_net_fw_request(pdev, pf, fw_name);
+	if (fw)
+		return fw;
+
+	/* Finally try the card type and media */
 	if (!pf->eth_tbl) {
 		dev_err(&pdev->dev, "Error: can't identify media config\n");
 		return NULL;
@@ -223,13 +294,7 @@ nfp_net_fw_find(struct pci_dev *pdev, struct nfp_pf *pf)
 	if (spc <= 0)
 		return NULL;
 
-	err = request_firmware(&fw, fw_name, &pdev->dev);
-	if (err)
-		return NULL;
-
-	dev_info(&pdev->dev, "Loading FW image: %s\n", fw_name);
-
-	return fw;
+	return nfp_net_fw_request(pdev, pf, fw_name);
 }
 
 /**
@@ -286,6 +351,10 @@ static int nfp_nsp_init(struct pci_dev *pdev, struct nfp_pf *pf)
 	struct nfp_nsp *nsp;
 	int err;
 
+	err = nfp_resource_wait(pf->cpp, NFP_RESOURCE_NSP, 30);
+	if (err)
+		return err;
+
 	nsp = nfp_nsp_open(pf->cpp);
 	if (IS_ERR(nsp)) {
 		err = PTR_ERR(nsp);
@@ -399,6 +468,10 @@ static int nfp_pci_probe(struct pci_dev *pdev,
 		 nfp_hwinfo_lookup(pf->hwinfo, "assembly.revision"),
 		 nfp_hwinfo_lookup(pf->hwinfo, "cpld.version"));
 
+	err = nfp_pf_board_state_wait(pf);
+	if (err)
+		goto err_hwinfo_free;
+
 	err = devlink_register(devlink, &pdev->dev);
 	if (err)
 		goto err_hwinfo_free;
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_main.h b/drivers/net/ethernet/netronome/nfp/nfp_main.h
index 6922410806db..be0ee59f2eb9 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_main.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_main.h
@@ -73,6 +73,8 @@ struct nfp_rtsym_table;
  * @mac_stats_mem:	Pointer to mapped MAC stats area
  * @vf_cfg_bar:		Pointer to the CPP area for the VF configuration BAR
  * @vf_cfg_mem:		Pointer to mapped VF configuration area
+ * @vfcfg_tbl2_area:	Pointer to the CPP area for the VF config table
+ * @vfcfg_tbl2:		Pointer to mapped VF config table
  * @irq_entries:	Array of MSI-X entries for all vNICs
  * @limit_vfs:		Number of VFs supported by firmware (~0 for PCI limit)
  * @num_vfs:		Number of SR-IOV VFs enabled
@@ -107,6 +109,8 @@ struct nfp_pf {
 	u8 __iomem *mac_stats_mem;
 	struct nfp_cpp_area *vf_cfg_bar;
 	u8 __iomem *vf_cfg_mem;
+	struct nfp_cpp_area *vfcfg_tbl2_area;
+	u8 __iomem *vfcfg_tbl2;
 
 	struct msix_entry *irq_entries;
 
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net.h b/drivers/net/ethernet/netronome/nfp/nfp_net.h
index b1fa77bd708b..d51d8237b984 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net.h
@@ -573,7 +573,6 @@ struct nfp_net_dp {
  * @tx_bar:             Pointer to mapped TX queues
  * @rx_bar:             Pointer to mapped FL/RX queues
  * @debugfs_dir:	Device directory in debugfs
- * @ethtool_dump_flag:	Ethtool dump flag
  * @vnic_list:		Entry on device vNIC list
  * @pdev:		Backpointer to PCI device
  * @app:		APP handle if available
@@ -640,7 +639,6 @@ struct nfp_net {
 	u8 __iomem *rx_bar;
 
 	struct dentry *debugfs_dir;
-	u32 ethtool_dump_flag;
 
 	struct list_head vnic_list;
 
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
index 4631ca8b8eb2..1c0187f0af51 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
@@ -71,6 +71,7 @@
 #include "nfp_app.h"
 #include "nfp_net_ctrl.h"
 #include "nfp_net.h"
+#include "nfp_net_sriov.h"
 #include "nfp_port.h"
 
 /**
@@ -895,6 +896,8 @@ static int nfp_net_tx(struct sk_buff *skb, struct net_device *netdev)
 
 	netdev_tx_sent_queue(nd_q, txbuf->real_len);
 
+	skb_tx_timestamp(skb);
+
 	tx_ring->wr_p += nr_frags + 1;
 	if (nfp_net_tx_ring_should_stop(tx_ring))
 		nfp_net_tx_ring_stop(nd_q, tx_ring);
@@ -903,13 +906,10 @@ static int nfp_net_tx(struct sk_buff *skb, struct net_device *netdev)
 	if (!skb->xmit_more || netif_xmit_stopped(nd_q))
 		nfp_net_tx_xmit_more_flush(tx_ring);
 
-	skb_tx_timestamp(skb);
-
 	return NETDEV_TX_OK;
 
 err_unmap:
-	--f;
-	while (f >= 0) {
+	while (--f >= 0) {
 		frag = &skb_shinfo(skb)->frags[f];
 		dma_unmap_page(dp->dev, tx_ring->txbufs[wr_idx].dma_addr,
 			       skb_frag_size(frag), DMA_TO_DEVICE);
@@ -991,7 +991,7 @@ static void nfp_net_tx_complete(struct nfp_net_tx_ring *tx_ring)
 
 		/* check for last gather fragment */
 		if (fidx == nr_frags - 1)
-			dev_kfree_skb_any(skb);
+			dev_consume_skb_any(skb);
 
 		tx_ring->txbufs[idx].dma_addr = 0;
 		tx_ring->txbufs[idx].skb = NULL;
@@ -1752,6 +1752,10 @@ static int nfp_net_rx(struct nfp_net_rx_ring *rx_ring, int budget)
 			continue;
 		}
 
+		nfp_net_dma_unmap_rx(dp, rxbuf->dma_addr);
+
+		nfp_net_rx_give_one(dp, rx_ring, new_frag, new_dma_addr);
+
 		if (likely(!meta.portid)) {
 			netdev = dp->netdev;
 		} else {
@@ -1760,16 +1764,12 @@ static int nfp_net_rx(struct nfp_net_rx_ring *rx_ring, int budget)
 			nn = netdev_priv(dp->netdev);
 			netdev = nfp_app_repr_get(nn->app, meta.portid);
 			if (unlikely(!netdev)) {
-				nfp_net_rx_drop(dp, r_vec, rx_ring, rxbuf, skb);
+				nfp_net_rx_drop(dp, r_vec, rx_ring, NULL, skb);
 				continue;
 			}
 			nfp_repr_inc_rx_stats(netdev, pkt_len);
 		}
 
-		nfp_net_dma_unmap_rx(dp, rxbuf->dma_addr);
-
-		nfp_net_rx_give_one(dp, rx_ring, new_frag, new_dma_addr);
-
 		skb_reserve(skb, pkt_off);
 		skb_put(skb, pkt_len);
 
@@ -2660,6 +2660,7 @@ static int nfp_net_netdev_close(struct net_device *netdev)
 	/* Step 2: Tell NFP
 	 */
 	nfp_net_clear_config_and_disable(nn);
+	nfp_port_configure(netdev, false);
 
 	/* Step 3: Free resources
 	 */
@@ -2777,16 +2778,21 @@ static int nfp_net_netdev_open(struct net_device *netdev)
 		goto err_free_all;
 
 	/* Step 2: Configure the NFP
+	 * - Ifup the physical interface if it exists
 	 * - Enable rings from 0 to tx_rings/rx_rings - 1.
 	 * - Write MAC address (in case it changed)
 	 * - Set the MTU
 	 * - Set the Freelist buffer size
 	 * - Enable the FW
 	 */
-	err = nfp_net_set_config_and_enable(nn);
+	err = nfp_port_configure(netdev, true);
 	if (err)
 		goto err_free_all;
 
+	err = nfp_net_set_config_and_enable(nn);
+	if (err)
+		goto err_port_disable;
+
 	/* Step 3: Enable for kernel
 	 * - put some freelist descriptors on each RX ring
 	 * - enable NAPI on each ring
@@ -2797,6 +2803,8 @@ static int nfp_net_netdev_open(struct net_device *netdev)
 
 	return 0;
 
+err_port_disable:
+	nfp_port_configure(netdev, false);
 err_free_all:
 	nfp_net_close_free_all(nn);
 	return err;
@@ -3414,6 +3422,11 @@ const struct net_device_ops nfp_net_netdev_ops = {
 	.ndo_get_stats64	= nfp_net_stat64,
 	.ndo_vlan_rx_add_vid	= nfp_net_vlan_rx_add_vid,
 	.ndo_vlan_rx_kill_vid	= nfp_net_vlan_rx_kill_vid,
+	.ndo_set_vf_mac         = nfp_app_set_vf_mac,
+	.ndo_set_vf_vlan        = nfp_app_set_vf_vlan,
+	.ndo_set_vf_spoofchk    = nfp_app_set_vf_spoofchk,
+	.ndo_get_vf_config	= nfp_app_get_vf_config,
+	.ndo_set_vf_link_state  = nfp_app_set_vf_link_state,
 	.ndo_setup_tc		= nfp_port_setup_tc,
 	.ndo_tx_timeout		= nfp_net_tx_timeout,
 	.ndo_set_rx_mode	= nfp_net_set_rx_mode,
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h b/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h
index e5e94e0746ec..b0a452ba9039 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h
@@ -164,6 +164,7 @@
 #define   NFP_NET_CFG_UPDATE_BPF	  (0x1 << 10) /* BPF program load */
 #define   NFP_NET_CFG_UPDATE_MACADDR	  (0x1 << 11) /* MAC address change */
 #define   NFP_NET_CFG_UPDATE_MBOX	  (0x1 << 12) /* Mailbox update */
+#define   NFP_NET_CFG_UPDATE_VF		  (0x1 << 13) /* VF settings change */
 #define   NFP_NET_CFG_UPDATE_ERR          (0x1 << 31) /* A error occurred */
 #define NFP_NET_CFG_TXRS_ENABLE         0x0008
 #define NFP_NET_CFG_RXRS_ENABLE         0x0010
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_debugfs.c b/drivers/net/ethernet/netronome/nfp/nfp_net_debugfs.c
index 40217ece5fcb..cf81cf95d1d8 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_debugfs.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_debugfs.c
@@ -125,7 +125,6 @@ static int nfp_net_debugfs_tx_q_read(struct seq_file *file, void *data)
 	struct nfp_net_tx_ring *tx_ring;
 	struct nfp_net_tx_desc *txd;
 	int d_rd_p, d_wr_p, txd_cnt;
-	struct sk_buff *skb;
 	struct nfp_net *nn;
 	int i;
 
@@ -158,13 +157,15 @@ static int nfp_net_debugfs_tx_q_read(struct seq_file *file, void *data)
 			   txd->vals[0], txd->vals[1],
 			   txd->vals[2], txd->vals[3]);
 
-		skb = READ_ONCE(tx_ring->txbufs[i].skb);
-		if (skb) {
-			if (tx_ring == r_vec->tx_ring)
+		if (tx_ring == r_vec->tx_ring) {
+			struct sk_buff *skb = READ_ONCE(tx_ring->txbufs[i].skb);
+
+			if (skb)
 				seq_printf(file, " skb->head=%p skb->data=%p",
 					   skb->head, skb->data);
-			else
-				seq_printf(file, " frag=%p", skb);
+		} else {
+			seq_printf(file, " frag=%p",
+				   READ_ONCE(tx_ring->txbufs[i].frag));
 		}
 
 		if (tx_ring->txbufs[i].dma_addr)
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c
index 6e31355c3567..07969f06df10 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c
@@ -59,82 +59,129 @@ enum nfp_dump_diag {
 	NFP_DUMP_NSP_DIAG = 0,
 };
 
-/* Support for stats. Returns netdev, driver, and device stats */
-enum { NETDEV_ET_STATS, NFP_NET_DRV_ET_STATS, NFP_NET_DEV_ET_STATS };
-struct _nfp_net_et_stats {
+struct nfp_et_stat {
 	char name[ETH_GSTRING_LEN];
-	int type;
-	int sz;
 	int off;
 };
 
-#define NN_ET_NETDEV_STAT(m) NETDEV_ET_STATS,			\
-		FIELD_SIZEOF(struct net_device_stats, m),	\
-		offsetof(struct net_device_stats, m)
-/* For stats in the control BAR (other than Q stats) */
-#define NN_ET_DEV_STAT(m) NFP_NET_DEV_ET_STATS,			\
-		sizeof(u64),					\
-		(m)
-static const struct _nfp_net_et_stats nfp_net_et_stats[] = {
-	/* netdev stats */
-	{"rx_packets", NN_ET_NETDEV_STAT(rx_packets)},
-	{"tx_packets", NN_ET_NETDEV_STAT(tx_packets)},
-	{"rx_bytes", NN_ET_NETDEV_STAT(rx_bytes)},
-	{"tx_bytes", NN_ET_NETDEV_STAT(tx_bytes)},
-	{"rx_errors", NN_ET_NETDEV_STAT(rx_errors)},
-	{"tx_errors", NN_ET_NETDEV_STAT(tx_errors)},
-	{"rx_dropped", NN_ET_NETDEV_STAT(rx_dropped)},
-	{"tx_dropped", NN_ET_NETDEV_STAT(tx_dropped)},
-	{"multicast", NN_ET_NETDEV_STAT(multicast)},
-	{"collisions", NN_ET_NETDEV_STAT(collisions)},
-	{"rx_over_errors", NN_ET_NETDEV_STAT(rx_over_errors)},
-	{"rx_crc_errors", NN_ET_NETDEV_STAT(rx_crc_errors)},
-	{"rx_frame_errors", NN_ET_NETDEV_STAT(rx_frame_errors)},
-	{"rx_fifo_errors", NN_ET_NETDEV_STAT(rx_fifo_errors)},
-	{"rx_missed_errors", NN_ET_NETDEV_STAT(rx_missed_errors)},
-	{"tx_aborted_errors", NN_ET_NETDEV_STAT(tx_aborted_errors)},
-	{"tx_carrier_errors", NN_ET_NETDEV_STAT(tx_carrier_errors)},
-	{"tx_fifo_errors", NN_ET_NETDEV_STAT(tx_fifo_errors)},
+static const struct nfp_et_stat nfp_net_et_stats[] = {
 	/* Stats from the device */
-	{"dev_rx_discards", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_RX_DISCARDS)},
-	{"dev_rx_errors", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_RX_ERRORS)},
-	{"dev_rx_bytes", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_RX_OCTETS)},
-	{"dev_rx_uc_bytes", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_RX_UC_OCTETS)},
-	{"dev_rx_mc_bytes", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_RX_MC_OCTETS)},
-	{"dev_rx_bc_bytes", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_RX_BC_OCTETS)},
-	{"dev_rx_pkts", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_RX_FRAMES)},
-	{"dev_rx_mc_pkts", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_RX_MC_FRAMES)},
-	{"dev_rx_bc_pkts", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_RX_BC_FRAMES)},
-
-	{"dev_tx_discards", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_TX_DISCARDS)},
-	{"dev_tx_errors", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_TX_ERRORS)},
-	{"dev_tx_bytes", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_TX_OCTETS)},
-	{"dev_tx_uc_bytes", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_TX_UC_OCTETS)},
-	{"dev_tx_mc_bytes", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_TX_MC_OCTETS)},
-	{"dev_tx_bc_bytes", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_TX_BC_OCTETS)},
-	{"dev_tx_pkts", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_TX_FRAMES)},
-	{"dev_tx_mc_pkts", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_TX_MC_FRAMES)},
-	{"dev_tx_bc_pkts", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_TX_BC_FRAMES)},
-
-	{"bpf_pass_pkts", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_APP0_FRAMES)},
-	{"bpf_pass_bytes", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_APP0_BYTES)},
+	{ "dev_rx_discards",	NFP_NET_CFG_STATS_RX_DISCARDS },
+	{ "dev_rx_errors",	NFP_NET_CFG_STATS_RX_ERRORS },
+	{ "dev_rx_bytes",	NFP_NET_CFG_STATS_RX_OCTETS },
+	{ "dev_rx_uc_bytes",	NFP_NET_CFG_STATS_RX_UC_OCTETS },
+	{ "dev_rx_mc_bytes",	NFP_NET_CFG_STATS_RX_MC_OCTETS },
+	{ "dev_rx_bc_bytes",	NFP_NET_CFG_STATS_RX_BC_OCTETS },
+	{ "dev_rx_pkts",	NFP_NET_CFG_STATS_RX_FRAMES },
+	{ "dev_rx_mc_pkts",	NFP_NET_CFG_STATS_RX_MC_FRAMES },
+	{ "dev_rx_bc_pkts",	NFP_NET_CFG_STATS_RX_BC_FRAMES },
+
+	{ "dev_tx_discards",	NFP_NET_CFG_STATS_TX_DISCARDS },
+	{ "dev_tx_errors",	NFP_NET_CFG_STATS_TX_ERRORS },
+	{ "dev_tx_bytes",	NFP_NET_CFG_STATS_TX_OCTETS },
+	{ "dev_tx_uc_bytes",	NFP_NET_CFG_STATS_TX_UC_OCTETS },
+	{ "dev_tx_mc_bytes",	NFP_NET_CFG_STATS_TX_MC_OCTETS },
+	{ "dev_tx_bc_bytes",	NFP_NET_CFG_STATS_TX_BC_OCTETS },
+	{ "dev_tx_pkts",	NFP_NET_CFG_STATS_TX_FRAMES },
+	{ "dev_tx_mc_pkts",	NFP_NET_CFG_STATS_TX_MC_FRAMES },
+	{ "dev_tx_bc_pkts",	NFP_NET_CFG_STATS_TX_BC_FRAMES },
+
+	{ "bpf_pass_pkts",	NFP_NET_CFG_STATS_APP0_FRAMES },
+	{ "bpf_pass_bytes",	NFP_NET_CFG_STATS_APP0_BYTES },
 	/* see comments in outro functions in nfp_bpf_jit.c to find out
 	 * how different BPF modes use app-specific counters
 	 */
-	{"bpf_app1_pkts", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_APP1_FRAMES)},
-	{"bpf_app1_bytes", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_APP1_BYTES)},
-	{"bpf_app2_pkts", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_APP2_FRAMES)},
-	{"bpf_app2_bytes", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_APP2_BYTES)},
-	{"bpf_app3_pkts", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_APP3_FRAMES)},
-	{"bpf_app3_bytes", NN_ET_DEV_STAT(NFP_NET_CFG_STATS_APP3_BYTES)},
+	{ "bpf_app1_pkts",	NFP_NET_CFG_STATS_APP1_FRAMES },
+	{ "bpf_app1_bytes",	NFP_NET_CFG_STATS_APP1_BYTES },
+	{ "bpf_app2_pkts",	NFP_NET_CFG_STATS_APP2_FRAMES },
+	{ "bpf_app2_bytes",	NFP_NET_CFG_STATS_APP2_BYTES },
+	{ "bpf_app3_pkts",	NFP_NET_CFG_STATS_APP3_FRAMES },
+	{ "bpf_app3_bytes",	NFP_NET_CFG_STATS_APP3_BYTES },
+};
+
+static const struct nfp_et_stat nfp_mac_et_stats[] = {
+	{ "rx_octets",			NFP_MAC_STATS_RX_IN_OCTETS, },
+	{ "rx_frame_too_long_errors",
+			NFP_MAC_STATS_RX_FRAME_TOO_LONG_ERRORS, },
+	{ "rx_range_length_errors",	NFP_MAC_STATS_RX_RANGE_LENGTH_ERRORS, },
+	{ "rx_vlan_reveive_ok",		NFP_MAC_STATS_RX_VLAN_REVEIVE_OK, },
+	{ "rx_errors",			NFP_MAC_STATS_RX_IN_ERRORS, },
+	{ "rx_broadcast_pkts",		NFP_MAC_STATS_RX_IN_BROADCAST_PKTS, },
+	{ "rx_drop_events",		NFP_MAC_STATS_RX_DROP_EVENTS, },
+	{ "rx_alignment_errors",	NFP_MAC_STATS_RX_ALIGNMENT_ERRORS, },
+	{ "rx_pause_mac_ctrl_frames",
+			NFP_MAC_STATS_RX_PAUSE_MAC_CTRL_FRAMES, },
+	{ "rx_frames_received_ok",	NFP_MAC_STATS_RX_FRAMES_RECEIVED_OK, },
+	{ "rx_frame_check_sequence_errors",
+			NFP_MAC_STATS_RX_FRAME_CHECK_SEQUENCE_ERRORS, },
+	{ "rx_unicast_pkts",		NFP_MAC_STATS_RX_UNICAST_PKTS, },
+	{ "rx_multicast_pkts",		NFP_MAC_STATS_RX_MULTICAST_PKTS, },
+	{ "rx_pkts",			NFP_MAC_STATS_RX_PKTS, },
+	{ "rx_undersize_pkts",		NFP_MAC_STATS_RX_UNDERSIZE_PKTS, },
+	{ "rx_pkts_64_octets",		NFP_MAC_STATS_RX_PKTS_64_OCTETS, },
+	{ "rx_pkts_65_to_127_octets",
+			NFP_MAC_STATS_RX_PKTS_65_TO_127_OCTETS, },
+	{ "rx_pkts_128_to_255_octets",
+			NFP_MAC_STATS_RX_PKTS_128_TO_255_OCTETS, },
+	{ "rx_pkts_256_to_511_octets",
+			NFP_MAC_STATS_RX_PKTS_256_TO_511_OCTETS, },
+	{ "rx_pkts_512_to_1023_octets",
+			NFP_MAC_STATS_RX_PKTS_512_TO_1023_OCTETS, },
+	{ "rx_pkts_1024_to_1518_octets",
+			NFP_MAC_STATS_RX_PKTS_1024_TO_1518_OCTETS, },
+	{ "rx_pkts_1519_to_max_octets",
+			NFP_MAC_STATS_RX_PKTS_1519_TO_MAX_OCTETS, },
+	{ "rx_jabbers",			NFP_MAC_STATS_RX_JABBERS, },
+	{ "rx_fragments",		NFP_MAC_STATS_RX_FRAGMENTS, },
+	{ "rx_oversize_pkts",		NFP_MAC_STATS_RX_OVERSIZE_PKTS, },
+	{ "rx_pause_frames_class0",	NFP_MAC_STATS_RX_PAUSE_FRAMES_CLASS0, },
+	{ "rx_pause_frames_class1",	NFP_MAC_STATS_RX_PAUSE_FRAMES_CLASS1, },
+	{ "rx_pause_frames_class2",	NFP_MAC_STATS_RX_PAUSE_FRAMES_CLASS2, },
+	{ "rx_pause_frames_class3",	NFP_MAC_STATS_RX_PAUSE_FRAMES_CLASS3, },
+	{ "rx_pause_frames_class4",	NFP_MAC_STATS_RX_PAUSE_FRAMES_CLASS4, },
+	{ "rx_pause_frames_class5",	NFP_MAC_STATS_RX_PAUSE_FRAMES_CLASS5, },
+	{ "rx_pause_frames_class6",	NFP_MAC_STATS_RX_PAUSE_FRAMES_CLASS6, },
+	{ "rx_pause_frames_class7",	NFP_MAC_STATS_RX_PAUSE_FRAMES_CLASS7, },
+	{ "rx_mac_ctrl_frames_received",
+			NFP_MAC_STATS_RX_MAC_CTRL_FRAMES_RECEIVED, },
+	{ "rx_mac_head_drop",		NFP_MAC_STATS_RX_MAC_HEAD_DROP, },
+	{ "tx_queue_drop",		NFP_MAC_STATS_TX_QUEUE_DROP, },
+	{ "tx_octets",			NFP_MAC_STATS_TX_OUT_OCTETS, },
+	{ "tx_vlan_transmitted_ok",	NFP_MAC_STATS_TX_VLAN_TRANSMITTED_OK, },
+	{ "tx_errors",			NFP_MAC_STATS_TX_OUT_ERRORS, },
+	{ "tx_broadcast_pkts",		NFP_MAC_STATS_TX_BROADCAST_PKTS, },
+	{ "tx_pause_mac_ctrl_frames",
+			NFP_MAC_STATS_TX_PAUSE_MAC_CTRL_FRAMES, },
+	{ "tx_frames_transmitted_ok",
+			NFP_MAC_STATS_TX_FRAMES_TRANSMITTED_OK, },
+	{ "tx_unicast_pkts",		NFP_MAC_STATS_TX_UNICAST_PKTS, },
+	{ "tx_multicast_pkts",		NFP_MAC_STATS_TX_MULTICAST_PKTS, },
+	{ "tx_pkts_64_octets",		NFP_MAC_STATS_TX_PKTS_64_OCTETS, },
+	{ "tx_pkts_65_to_127_octets",
+			NFP_MAC_STATS_TX_PKTS_65_TO_127_OCTETS, },
+	{ "tx_pkts_128_to_255_octets",
+			NFP_MAC_STATS_TX_PKTS_128_TO_255_OCTETS, },
+	{ "tx_pkts_256_to_511_octets",
+			NFP_MAC_STATS_TX_PKTS_256_TO_511_OCTETS, },
+	{ "tx_pkts_512_to_1023_octets",
+			NFP_MAC_STATS_TX_PKTS_512_TO_1023_OCTETS, },
+	{ "tx_pkts_1024_to_1518_octets",
+			NFP_MAC_STATS_TX_PKTS_1024_TO_1518_OCTETS, },
+	{ "tx_pkts_1519_to_max_octets",
+			NFP_MAC_STATS_TX_PKTS_1519_TO_MAX_OCTETS, },
+	{ "tx_pause_frames_class0",	NFP_MAC_STATS_TX_PAUSE_FRAMES_CLASS0, },
+	{ "tx_pause_frames_class1",	NFP_MAC_STATS_TX_PAUSE_FRAMES_CLASS1, },
+	{ "tx_pause_frames_class2",	NFP_MAC_STATS_TX_PAUSE_FRAMES_CLASS2, },
+	{ "tx_pause_frames_class3",	NFP_MAC_STATS_TX_PAUSE_FRAMES_CLASS3, },
+	{ "tx_pause_frames_class4",	NFP_MAC_STATS_TX_PAUSE_FRAMES_CLASS4, },
+	{ "tx_pause_frames_class5",	NFP_MAC_STATS_TX_PAUSE_FRAMES_CLASS5, },
+	{ "tx_pause_frames_class6",	NFP_MAC_STATS_TX_PAUSE_FRAMES_CLASS6, },
+	{ "tx_pause_frames_class7",	NFP_MAC_STATS_TX_PAUSE_FRAMES_CLASS7, },
 };
 
 #define NN_ET_GLOBAL_STATS_LEN ARRAY_SIZE(nfp_net_et_stats)
-#define NN_ET_RVEC_STATS_LEN (nn->dp.num_r_vecs * 3)
+#define NN_ET_SWITCH_STATS_LEN 9
 #define NN_ET_RVEC_GATHER_STATS 7
-#define NN_ET_QUEUE_STATS_LEN ((nn->dp.num_tx_rings + nn->dp.num_rx_rings) * 2)
-#define NN_ET_STATS_LEN (NN_ET_GLOBAL_STATS_LEN + NN_ET_RVEC_GATHER_STATS + \
-			 NN_ET_RVEC_STATS_LEN + NN_ET_QUEUE_STATS_LEN)
 
 static void nfp_net_get_nspinfo(struct nfp_app *app, char *version)
 {
@@ -147,34 +194,53 @@ static void nfp_net_get_nspinfo(struct nfp_app *app, char *version)
 	if (IS_ERR(nsp))
 		return;
 
-	snprintf(version, ETHTOOL_FWVERS_LEN, "sp:%hu.%hu",
+	snprintf(version, ETHTOOL_FWVERS_LEN, "%hu.%hu",
 		 nfp_nsp_get_abi_ver_major(nsp),
 		 nfp_nsp_get_abi_ver_minor(nsp));
 
 	nfp_nsp_close(nsp);
 }
 
-static void nfp_net_get_drvinfo(struct net_device *netdev,
-				struct ethtool_drvinfo *drvinfo)
+static void
+nfp_get_drvinfo(struct nfp_app *app, struct pci_dev *pdev,
+		const char *vnic_version, struct ethtool_drvinfo *drvinfo)
 {
 	char nsp_version[ETHTOOL_FWVERS_LEN] = {};
-	struct nfp_net *nn = netdev_priv(netdev);
 
-	strlcpy(drvinfo->driver, nn->pdev->driver->name,
-		sizeof(drvinfo->driver));
+	strlcpy(drvinfo->driver, pdev->driver->name, sizeof(drvinfo->driver));
 	strlcpy(drvinfo->version, nfp_driver_version, sizeof(drvinfo->version));
 
-	nfp_net_get_nspinfo(nn->app, nsp_version);
+	nfp_net_get_nspinfo(app, nsp_version);
 	snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version),
-		 "%d.%d.%d.%d %s %s %s",
+		 "%s %s %s %s", vnic_version, nsp_version,
+		 nfp_app_mip_name(app), nfp_app_name(app));
+}
+
+static void
+nfp_net_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo)
+{
+	char vnic_version[ETHTOOL_FWVERS_LEN] = {};
+	struct nfp_net *nn = netdev_priv(netdev);
+
+	snprintf(vnic_version, sizeof(vnic_version), "%d.%d.%d.%d",
 		 nn->fw_ver.resv, nn->fw_ver.class,
-		 nn->fw_ver.major, nn->fw_ver.minor, nsp_version,
-		 nfp_app_mip_name(nn->app), nfp_app_name(nn->app));
+		 nn->fw_ver.major, nn->fw_ver.minor);
 	strlcpy(drvinfo->bus_info, pci_name(nn->pdev),
 		sizeof(drvinfo->bus_info));
 
-	drvinfo->n_stats = NN_ET_STATS_LEN;
-	drvinfo->regdump_len = NFP_NET_CFG_BAR_SZ;
+	nfp_get_drvinfo(nn->app, nn->pdev, vnic_version, drvinfo);
+}
+
+static void
+nfp_app_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo)
+{
+	struct nfp_app *app;
+
+	app = nfp_app_from_netdev(netdev);
+	if (!app)
+		return;
+
+	nfp_get_drvinfo(app, app->pdev, "*", drvinfo);
 }
 
 /**
@@ -346,132 +412,270 @@ static int nfp_net_set_ringparam(struct net_device *netdev,
 	return nfp_net_set_ring_size(nn, rxd_cnt, txd_cnt);
 }
 
-static void nfp_net_get_strings(struct net_device *netdev,
-				u32 stringset, u8 *data)
+static __printf(2, 3) u8 *nfp_pr_et(u8 *data, const char *fmt, ...)
+{
+	va_list args;
+
+	va_start(args, fmt);
+	vsnprintf(data, ETH_GSTRING_LEN, fmt, args);
+	va_end(args);
+
+	return data + ETH_GSTRING_LEN;
+}
+
+static unsigned int nfp_vnic_get_sw_stats_count(struct net_device *netdev)
+{
+	struct nfp_net *nn = netdev_priv(netdev);
+
+	return NN_ET_RVEC_GATHER_STATS + nn->dp.num_r_vecs * 3;
+}
+
+static u8 *nfp_vnic_get_sw_stats_strings(struct net_device *netdev, u8 *data)
 {
 	struct nfp_net *nn = netdev_priv(netdev);
-	u8 *p = data;
 	int i;
 
-	switch (stringset) {
-	case ETH_SS_STATS:
-		for (i = 0; i < NN_ET_GLOBAL_STATS_LEN; i++) {
-			memcpy(p, nfp_net_et_stats[i].name, ETH_GSTRING_LEN);
-			p += ETH_GSTRING_LEN;
-		}
-		for (i = 0; i < nn->dp.num_r_vecs; i++) {
-			sprintf(p, "rvec_%u_rx_pkts", i);
-			p += ETH_GSTRING_LEN;
-			sprintf(p, "rvec_%u_tx_pkts", i);
-			p += ETH_GSTRING_LEN;
-			sprintf(p, "rvec_%u_tx_busy", i);
-			p += ETH_GSTRING_LEN;
-		}
-		strncpy(p, "hw_rx_csum_ok", ETH_GSTRING_LEN);
-		p += ETH_GSTRING_LEN;
-		strncpy(p, "hw_rx_csum_inner_ok", ETH_GSTRING_LEN);
-		p += ETH_GSTRING_LEN;
-		strncpy(p, "hw_rx_csum_err", ETH_GSTRING_LEN);
-		p += ETH_GSTRING_LEN;
-		strncpy(p, "hw_tx_csum", ETH_GSTRING_LEN);
-		p += ETH_GSTRING_LEN;
-		strncpy(p, "hw_tx_inner_csum", ETH_GSTRING_LEN);
-		p += ETH_GSTRING_LEN;
-		strncpy(p, "tx_gather", ETH_GSTRING_LEN);
-		p += ETH_GSTRING_LEN;
-		strncpy(p, "tx_lso", ETH_GSTRING_LEN);
-		p += ETH_GSTRING_LEN;
-		for (i = 0; i < nn->dp.num_tx_rings; i++) {
-			sprintf(p, "txq_%u_pkts", i);
-			p += ETH_GSTRING_LEN;
-			sprintf(p, "txq_%u_bytes", i);
-			p += ETH_GSTRING_LEN;
-		}
-		for (i = 0; i < nn->dp.num_rx_rings; i++) {
-			sprintf(p, "rxq_%u_pkts", i);
-			p += ETH_GSTRING_LEN;
-			sprintf(p, "rxq_%u_bytes", i);
-			p += ETH_GSTRING_LEN;
-		}
-		break;
+	for (i = 0; i < nn->dp.num_r_vecs; i++) {
+		data = nfp_pr_et(data, "rvec_%u_rx_pkts", i);
+		data = nfp_pr_et(data, "rvec_%u_tx_pkts", i);
+		data = nfp_pr_et(data, "rvec_%u_tx_busy", i);
 	}
+
+	data = nfp_pr_et(data, "hw_rx_csum_ok");
+	data = nfp_pr_et(data, "hw_rx_csum_inner_ok");
+	data = nfp_pr_et(data, "hw_rx_csum_err");
+	data = nfp_pr_et(data, "hw_tx_csum");
+	data = nfp_pr_et(data, "hw_tx_inner_csum");
+	data = nfp_pr_et(data, "tx_gather");
+	data = nfp_pr_et(data, "tx_lso");
+
+	return data;
 }
 
-static void nfp_net_get_stats(struct net_device *netdev,
-			      struct ethtool_stats *stats, u64 *data)
+static u64 *nfp_vnic_get_sw_stats(struct net_device *netdev, u64 *data)
 {
 	u64 gathered_stats[NN_ET_RVEC_GATHER_STATS] = {};
 	struct nfp_net *nn = netdev_priv(netdev);
-	struct rtnl_link_stats64 *netdev_stats;
-	struct rtnl_link_stats64 temp = {};
 	u64 tmp[NN_ET_RVEC_GATHER_STATS];
-	u8 __iomem *io_p;
-	int i, j, k;
-	u8 *p;
-
-	netdev_stats = dev_get_stats(netdev, &temp);
-
-	for (i = 0; i < NN_ET_GLOBAL_STATS_LEN; i++) {
-		switch (nfp_net_et_stats[i].type) {
-		case NETDEV_ET_STATS:
-			p = (char *)netdev_stats + nfp_net_et_stats[i].off;
-			data[i] = nfp_net_et_stats[i].sz == sizeof(u64) ?
-				*(u64 *)p : *(u32 *)p;
-			break;
-
-		case NFP_NET_DEV_ET_STATS:
-			io_p = nn->dp.ctrl_bar + nfp_net_et_stats[i].off;
-			data[i] = readq(io_p);
-			break;
-		}
-	}
-	for (j = 0; j < nn->dp.num_r_vecs; j++) {
+	unsigned int i, j;
+
+	for (i = 0; i < nn->dp.num_r_vecs; i++) {
 		unsigned int start;
 
 		do {
-			start = u64_stats_fetch_begin(&nn->r_vecs[j].rx_sync);
-			data[i++] = nn->r_vecs[j].rx_pkts;
-			tmp[0] = nn->r_vecs[j].hw_csum_rx_ok;
-			tmp[1] = nn->r_vecs[j].hw_csum_rx_inner_ok;
-			tmp[2] = nn->r_vecs[j].hw_csum_rx_error;
-		} while (u64_stats_fetch_retry(&nn->r_vecs[j].rx_sync, start));
+			start = u64_stats_fetch_begin(&nn->r_vecs[i].rx_sync);
+			*data++ = nn->r_vecs[i].rx_pkts;
+			tmp[0] = nn->r_vecs[i].hw_csum_rx_ok;
+			tmp[1] = nn->r_vecs[i].hw_csum_rx_inner_ok;
+			tmp[2] = nn->r_vecs[i].hw_csum_rx_error;
+		} while (u64_stats_fetch_retry(&nn->r_vecs[i].rx_sync, start));
 
 		do {
-			start = u64_stats_fetch_begin(&nn->r_vecs[j].tx_sync);
-			data[i++] = nn->r_vecs[j].tx_pkts;
-			data[i++] = nn->r_vecs[j].tx_busy;
-			tmp[3] = nn->r_vecs[j].hw_csum_tx;
-			tmp[4] = nn->r_vecs[j].hw_csum_tx_inner;
-			tmp[5] = nn->r_vecs[j].tx_gather;
-			tmp[6] = nn->r_vecs[j].tx_lso;
-		} while (u64_stats_fetch_retry(&nn->r_vecs[j].tx_sync, start));
-
-		for (k = 0; k < NN_ET_RVEC_GATHER_STATS; k++)
-			gathered_stats[k] += tmp[k];
+			start = u64_stats_fetch_begin(&nn->r_vecs[i].tx_sync);
+			*data++ = nn->r_vecs[i].tx_pkts;
+			*data++ = nn->r_vecs[i].tx_busy;
+			tmp[3] = nn->r_vecs[i].hw_csum_tx;
+			tmp[4] = nn->r_vecs[i].hw_csum_tx_inner;
+			tmp[5] = nn->r_vecs[i].tx_gather;
+			tmp[6] = nn->r_vecs[i].tx_lso;
+		} while (u64_stats_fetch_retry(&nn->r_vecs[i].tx_sync, start));
+
+		for (j = 0; j < NN_ET_RVEC_GATHER_STATS; j++)
+			gathered_stats[j] += tmp[j];
 	}
+
 	for (j = 0; j < NN_ET_RVEC_GATHER_STATS; j++)
-		data[i++] = gathered_stats[j];
-	for (j = 0; j < nn->dp.num_tx_rings; j++) {
-		io_p = nn->dp.ctrl_bar + NFP_NET_CFG_TXR_STATS(j);
-		data[i++] = readq(io_p);
-		io_p = nn->dp.ctrl_bar + NFP_NET_CFG_TXR_STATS(j) + 8;
-		data[i++] = readq(io_p);
+		*data++ = gathered_stats[j];
+
+	return data;
+}
+
+static unsigned int
+nfp_vnic_get_hw_stats_count(unsigned int rx_rings, unsigned int tx_rings)
+{
+	return NN_ET_GLOBAL_STATS_LEN + (rx_rings + tx_rings) * 2;
+}
+
+static u8 *
+nfp_vnic_get_hw_stats_strings(u8 *data, unsigned int rx_rings,
+			      unsigned int tx_rings, bool repr)
+{
+	int swap_off, i;
+
+	BUILD_BUG_ON(NN_ET_GLOBAL_STATS_LEN < NN_ET_SWITCH_STATS_LEN * 2);
+	/* If repr is true first add SWITCH_STATS_LEN and then subtract it
+	 * effectively swapping the RX and TX statistics (giving us the RX
+	 * and TX from perspective of the switch).
+	 */
+	swap_off = repr * NN_ET_SWITCH_STATS_LEN;
+
+	for (i = 0; i < NN_ET_SWITCH_STATS_LEN; i++)
+		data = nfp_pr_et(data, nfp_net_et_stats[i + swap_off].name);
+
+	for (i = NN_ET_SWITCH_STATS_LEN; i < NN_ET_SWITCH_STATS_LEN * 2; i++)
+		data = nfp_pr_et(data, nfp_net_et_stats[i - swap_off].name);
+
+	for (i = NN_ET_SWITCH_STATS_LEN * 2; i < NN_ET_GLOBAL_STATS_LEN; i++)
+		data = nfp_pr_et(data, nfp_net_et_stats[i].name);
+
+	for (i = 0; i < tx_rings; i++) {
+		data = nfp_pr_et(data, "txq_%u_pkts", i);
+		data = nfp_pr_et(data, "txq_%u_bytes", i);
+	}
+
+	for (i = 0; i < rx_rings; i++) {
+		data = nfp_pr_et(data, "rxq_%u_pkts", i);
+		data = nfp_pr_et(data, "rxq_%u_bytes", i);
+	}
+
+	return data;
+}
+
+static u64 *
+nfp_vnic_get_hw_stats(u64 *data, u8 __iomem *mem,
+		      unsigned int rx_rings, unsigned int tx_rings)
+{
+	unsigned int i;
+
+	for (i = 0; i < NN_ET_GLOBAL_STATS_LEN; i++)
+		*data++ = readq(mem + nfp_net_et_stats[i].off);
+
+	for (i = 0; i < tx_rings; i++) {
+		*data++ = readq(mem + NFP_NET_CFG_TXR_STATS(i));
+		*data++ = readq(mem + NFP_NET_CFG_TXR_STATS(i) + 8);
+	}
+
+	for (i = 0; i < rx_rings; i++) {
+		*data++ = readq(mem + NFP_NET_CFG_RXR_STATS(i));
+		*data++ = readq(mem + NFP_NET_CFG_RXR_STATS(i) + 8);
 	}
-	for (j = 0; j < nn->dp.num_rx_rings; j++) {
-		io_p = nn->dp.ctrl_bar + NFP_NET_CFG_RXR_STATS(j);
-		data[i++] = readq(io_p);
-		io_p = nn->dp.ctrl_bar + NFP_NET_CFG_RXR_STATS(j) + 8;
-		data[i++] = readq(io_p);
+
+	return data;
+}
+
+static unsigned int nfp_mac_get_stats_count(struct net_device *netdev)
+{
+	struct nfp_port *port;
+
+	port = nfp_port_from_netdev(netdev);
+	if (!__nfp_port_get_eth_port(port) || !port->eth_stats)
+		return 0;
+
+	return ARRAY_SIZE(nfp_mac_et_stats);
+}
+
+static u8 *nfp_mac_get_stats_strings(struct net_device *netdev, u8 *data)
+{
+	struct nfp_port *port;
+	unsigned int i;
+
+	port = nfp_port_from_netdev(netdev);
+	if (!__nfp_port_get_eth_port(port) || !port->eth_stats)
+		return data;
+
+	for (i = 0; i < ARRAY_SIZE(nfp_mac_et_stats); i++)
+		data = nfp_pr_et(data, "mac.%s", nfp_mac_et_stats[i].name);
+
+	return data;
+}
+
+static u64 *nfp_mac_get_stats(struct net_device *netdev, u64 *data)
+{
+	struct nfp_port *port;
+	unsigned int i;
+
+	port = nfp_port_from_netdev(netdev);
+	if (!__nfp_port_get_eth_port(port) || !port->eth_stats)
+		return data;
+
+	for (i = 0; i < ARRAY_SIZE(nfp_mac_et_stats); i++)
+		*data++ = readq(port->eth_stats + nfp_mac_et_stats[i].off);
+
+	return data;
+}
+
+static void nfp_net_get_strings(struct net_device *netdev,
+				u32 stringset, u8 *data)
+{
+	struct nfp_net *nn = netdev_priv(netdev);
+
+	switch (stringset) {
+	case ETH_SS_STATS:
+		data = nfp_vnic_get_sw_stats_strings(netdev, data);
+		data = nfp_vnic_get_hw_stats_strings(data, nn->dp.num_rx_rings,
+						     nn->dp.num_tx_rings,
+						     false);
+		data = nfp_mac_get_stats_strings(netdev, data);
+		break;
 	}
 }
 
+static void
+nfp_net_get_stats(struct net_device *netdev, struct ethtool_stats *stats,
+		  u64 *data)
+{
+	struct nfp_net *nn = netdev_priv(netdev);
+
+	data = nfp_vnic_get_sw_stats(netdev, data);
+	data = nfp_vnic_get_hw_stats(data, nn->dp.ctrl_bar,
+				     nn->dp.num_rx_rings, nn->dp.num_tx_rings);
+	data = nfp_mac_get_stats(netdev, data);
+}
+
 static int nfp_net_get_sset_count(struct net_device *netdev, int sset)
 {
 	struct nfp_net *nn = netdev_priv(netdev);
 
 	switch (sset) {
 	case ETH_SS_STATS:
-		return NN_ET_STATS_LEN;
+		return nfp_vnic_get_sw_stats_count(netdev) +
+		       nfp_vnic_get_hw_stats_count(nn->dp.num_rx_rings,
+						   nn->dp.num_tx_rings) +
+		       nfp_mac_get_stats_count(netdev);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static void nfp_port_get_strings(struct net_device *netdev,
+				 u32 stringset, u8 *data)
+{
+	struct nfp_port *port = nfp_port_from_netdev(netdev);
+
+	switch (stringset) {
+	case ETH_SS_STATS:
+		if (nfp_port_is_vnic(port))
+			data = nfp_vnic_get_hw_stats_strings(data, 0, 0, true);
+		else
+			data = nfp_mac_get_stats_strings(netdev, data);
+		break;
+	}
+}
+
+static void
+nfp_port_get_stats(struct net_device *netdev, struct ethtool_stats *stats,
+		   u64 *data)
+{
+	struct nfp_port *port = nfp_port_from_netdev(netdev);
+
+	if (nfp_port_is_vnic(port))
+		data = nfp_vnic_get_hw_stats(data, port->vnic, 0, 0);
+	else
+		data = nfp_mac_get_stats(netdev, data);
+}
+
+static int nfp_port_get_sset_count(struct net_device *netdev, int sset)
+{
+	struct nfp_port *port = nfp_port_from_netdev(netdev);
+	unsigned int count;
+
+	switch (sset) {
+	case ETH_SS_STATS:
+		if (nfp_port_is_vnic(port))
+			count = nfp_vnic_get_hw_stats_count(0, 0);
+		else
+			count = nfp_mac_get_stats_count(netdev);
+		return count;
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -708,18 +912,18 @@ static int nfp_net_get_coalesce(struct net_device *netdev,
 /* Other debug dumps
  */
 static int
-nfp_dump_nsp_diag(struct nfp_net *nn, struct ethtool_dump *dump, void *buffer)
+nfp_dump_nsp_diag(struct nfp_app *app, struct ethtool_dump *dump, void *buffer)
 {
 	struct nfp_resource *res;
 	int ret;
 
-	if (!nn->app)
+	if (!app)
 		return -EOPNOTSUPP;
 
 	dump->version = 1;
 	dump->flag = NFP_DUMP_NSP_DIAG;
 
-	res = nfp_resource_acquire(nn->app->cpp, NFP_RESOURCE_NSP_DIAG);
+	res = nfp_resource_acquire(app->cpp, NFP_RESOURCE_NSP_DIAG);
 	if (IS_ERR(res))
 		return PTR_ERR(res);
 
@@ -729,7 +933,7 @@ nfp_dump_nsp_diag(struct nfp_net *nn, struct ethtool_dump *dump, void *buffer)
 			goto exit_release;
 		}
 
-		ret = nfp_cpp_read(nn->app->cpp, nfp_resource_cpp_id(res),
+		ret = nfp_cpp_read(app->cpp, nfp_resource_cpp_id(res),
 				   nfp_resource_address(res),
 				   buffer, dump->len);
 		if (ret != dump->len)
@@ -746,32 +950,30 @@ exit_release:
 	return ret;
 }
 
-static int nfp_net_set_dump(struct net_device *netdev, struct ethtool_dump *val)
+static int nfp_app_set_dump(struct net_device *netdev, struct ethtool_dump *val)
 {
-	struct nfp_net *nn = netdev_priv(netdev);
+	struct nfp_app *app = nfp_app_from_netdev(netdev);
 
-	if (!nn->app)
+	if (!app)
 		return -EOPNOTSUPP;
 
 	if (val->flag != NFP_DUMP_NSP_DIAG)
 		return -EINVAL;
 
-	nn->ethtool_dump_flag = val->flag;
-
 	return 0;
 }
 
 static int
-nfp_net_get_dump_flag(struct net_device *netdev, struct ethtool_dump *dump)
+nfp_app_get_dump_flag(struct net_device *netdev, struct ethtool_dump *dump)
 {
-	return nfp_dump_nsp_diag(netdev_priv(netdev), dump, NULL);
+	return nfp_dump_nsp_diag(nfp_app_from_netdev(netdev), dump, NULL);
 }
 
 static int
-nfp_net_get_dump_data(struct net_device *netdev, struct ethtool_dump *dump,
+nfp_app_get_dump_data(struct net_device *netdev, struct ethtool_dump *dump,
 		      void *buffer)
 {
-	return nfp_dump_nsp_diag(netdev_priv(netdev), dump, buffer);
+	return nfp_dump_nsp_diag(nfp_app_from_netdev(netdev), dump, buffer);
 }
 
 static int nfp_net_set_coalesce(struct net_device *netdev,
@@ -928,9 +1130,9 @@ static const struct ethtool_ops nfp_net_ethtool_ops = {
 	.set_rxfh		= nfp_net_set_rxfh,
 	.get_regs_len		= nfp_net_get_regs_len,
 	.get_regs		= nfp_net_get_regs,
-	.set_dump		= nfp_net_set_dump,
-	.get_dump_flag		= nfp_net_get_dump_flag,
-	.get_dump_data		= nfp_net_get_dump_data,
+	.set_dump		= nfp_app_set_dump,
+	.get_dump_flag		= nfp_app_get_dump_flag,
+	.get_dump_data		= nfp_app_get_dump_data,
 	.get_coalesce           = nfp_net_get_coalesce,
 	.set_coalesce           = nfp_net_set_coalesce,
 	.get_channels		= nfp_net_get_channels,
@@ -939,6 +1141,17 @@ static const struct ethtool_ops nfp_net_ethtool_ops = {
 	.set_link_ksettings	= nfp_net_set_link_ksettings,
 };
 
+const struct ethtool_ops nfp_port_ethtool_ops = {
+	.get_drvinfo		= nfp_app_get_drvinfo,
+	.get_link		= ethtool_op_get_link,
+	.get_strings		= nfp_port_get_strings,
+	.get_ethtool_stats	= nfp_port_get_stats,
+	.get_sset_count		= nfp_port_get_sset_count,
+	.set_dump		= nfp_app_set_dump,
+	.get_dump_flag		= nfp_app_get_dump_flag,
+	.get_dump_data		= nfp_app_get_dump_data,
+};
+
 void nfp_net_set_ethtool_ops(struct net_device *netdev)
 {
 	netdev->ethtool_ops = &nfp_net_ethtool_ops;
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_main.c b/drivers/net/ethernet/netronome/nfp/nfp_net_main.c
index 5797dbf2b507..ff373acd28f3 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_main.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_main.c
@@ -57,29 +57,13 @@
 #include "nfpcore/nfp6000_pcie.h"
 #include "nfp_app.h"
 #include "nfp_net_ctrl.h"
+#include "nfp_net_sriov.h"
 #include "nfp_net.h"
 #include "nfp_main.h"
 #include "nfp_port.h"
 
 #define NFP_PF_CSR_SLICE_SIZE	(32 * 1024)
 
-static int nfp_is_ready(struct nfp_pf *pf)
-{
-	const char *cp;
-	long state;
-	int err;
-
-	cp = nfp_hwinfo_lookup(pf->hwinfo, "board.state");
-	if (!cp)
-		return 0;
-
-	err = kstrtol(cp, 0, &state);
-	if (err < 0)
-		return 0;
-
-	return state == 15;
-}
-
 /**
  * nfp_net_get_mac_addr() - Get the MAC address.
  * @pf:       NFP PF handle
@@ -160,6 +144,8 @@ nfp_net_pf_map_rtsym(struct nfp_pf *pf, const char *name, const char *sym_fmt,
 
 static void nfp_net_pf_free_vnic(struct nfp_pf *pf, struct nfp_net *nn)
 {
+	if (nfp_net_is_data_vnic(nn))
+		nfp_app_vnic_free(pf->app, nn);
 	nfp_port_free(nn->port);
 	list_del(&nn->vnic_list);
 	pf->num_vnics--;
@@ -204,7 +190,7 @@ nfp_net_pf_alloc_vnic(struct nfp_pf *pf, bool needs_netdev,
 	nn->stride_tx = stride;
 
 	if (needs_netdev) {
-		err = nfp_app_vnic_init(pf->app, nn, id);
+		err = nfp_app_vnic_alloc(pf->app, nn, id);
 		if (err) {
 			nfp_net_free(nn);
 			return ERR_PTR(err);
@@ -242,8 +228,17 @@ nfp_net_pf_init_vnic(struct nfp_pf *pf, struct nfp_net *nn, unsigned int id)
 
 	nfp_net_info(nn);
 
+	if (nfp_net_is_data_vnic(nn)) {
+		err = nfp_app_vnic_init(pf->app, nn);
+		if (err)
+			goto err_devlink_port_clean;
+	}
+
 	return 0;
 
+err_devlink_port_clean:
+	if (nn->port)
+		nfp_devlink_port_unregister(nn->port);
 err_dfs_clean:
 	nfp_net_debugfs_dir_clean(&nn->debugfs_dir);
 	nfp_net_clean(nn);
@@ -287,11 +282,12 @@ err_free_prev:
 
 static void nfp_net_pf_clean_vnic(struct nfp_pf *pf, struct nfp_net *nn)
 {
+	if (nfp_net_is_data_vnic(nn))
+		nfp_app_vnic_clean(pf->app, nn);
 	if (nn->port)
 		nfp_devlink_port_unregister(nn->port);
 	nfp_net_debugfs_dir_clean(&nn->debugfs_dir);
 	nfp_net_clean(nn);
-	nfp_app_vnic_clean(pf->app, nn);
 }
 
 static int nfp_net_pf_alloc_irqs(struct nfp_pf *pf)
@@ -388,7 +384,7 @@ nfp_net_pf_app_init(struct nfp_pf *pf, u8 __iomem *qc_bar, unsigned int stride)
 					NFP_PF_CSR_SLICE_SIZE,
 					&pf->ctrl_vnic_bar);
 	if (IS_ERR(ctrl_bar)) {
-		nfp_err(pf->cpp, "Failed to find data vNIC memory symbol\n");
+		nfp_err(pf->cpp, "Failed to find ctrl vNIC memory symbol\n");
 		err = PTR_ERR(ctrl_bar);
 		goto err_app_clean;
 	}
@@ -489,6 +485,8 @@ static void nfp_net_pf_app_stop(struct nfp_pf *pf)
 
 static void nfp_net_pci_unmap_mem(struct nfp_pf *pf)
 {
+	if (pf->vfcfg_tbl2_area)
+		nfp_cpp_area_release_free(pf->vfcfg_tbl2_area);
 	if (pf->vf_cfg_bar)
 		nfp_cpp_area_release_free(pf->vf_cfg_bar);
 	if (pf->mac_stats_bar)
@@ -504,7 +502,7 @@ static int nfp_net_pci_map_mem(struct nfp_pf *pf)
 	int err;
 
 	min_size = pf->max_data_vnics * NFP_PF_CSR_SLICE_SIZE;
-	mem = nfp_net_pf_map_rtsym(pf, "net.ctrl", "_pf%d_net_bar0",
+	mem = nfp_net_pf_map_rtsym(pf, "net.bar0", "_pf%d_net_bar0",
 				   min_size, &pf->data_vnic_bar);
 	if (IS_ERR(mem)) {
 		nfp_err(pf->cpp, "Failed to find data vNIC memory symbol\n");
@@ -535,17 +533,32 @@ static int nfp_net_pci_map_mem(struct nfp_pf *pf)
 		pf->vf_cfg_mem = NULL;
 	}
 
+	min_size = NFP_NET_VF_CFG_SZ * pf->limit_vfs + NFP_NET_VF_CFG_MB_SZ;
+	pf->vfcfg_tbl2 = nfp_net_pf_map_rtsym(pf, "net.vfcfg_tbl2",
+					      "_pf%d_net_vf_cfg2",
+					      min_size, &pf->vfcfg_tbl2_area);
+	if (IS_ERR(pf->vfcfg_tbl2)) {
+		if (PTR_ERR(pf->vfcfg_tbl2) != -ENOENT) {
+			err = PTR_ERR(pf->vfcfg_tbl2);
+			goto err_unmap_vf_cfg;
+		}
+		pf->vfcfg_tbl2 = NULL;
+	}
+
 	mem = nfp_cpp_map_area(pf->cpp, "net.qc", 0, 0,
 			       NFP_PCIE_QUEUE(0), NFP_QCP_QUEUE_AREA_SZ,
 			       &pf->qc_area);
 	if (IS_ERR(mem)) {
 		nfp_err(pf->cpp, "Failed to map Queue Controller area.\n");
 		err = PTR_ERR(mem);
-		goto err_unmap_vf_cfg;
+		goto err_unmap_vfcfg_tbl2;
 	}
 
 	return 0;
 
+err_unmap_vfcfg_tbl2:
+	if (pf->vfcfg_tbl2_area)
+		nfp_cpp_area_release_free(pf->vfcfg_tbl2_area);
 err_unmap_vf_cfg:
 	if (pf->vf_cfg_bar)
 		nfp_cpp_area_release_free(pf->vf_cfg_bar);
@@ -695,16 +708,10 @@ int nfp_net_pci_probe(struct nfp_pf *pf)
 
 	INIT_WORK(&pf->port_refresh_work, nfp_net_refresh_vnics);
 
-	/* Verify that the board has completed initialization */
-	if (!nfp_is_ready(pf)) {
-		nfp_err(pf->cpp, "NFP is not ready for NIC operation.\n");
-		return -EINVAL;
-	}
-
 	if (!pf->rtbl) {
 		nfp_err(pf->cpp, "No %s, giving up.\n",
 			pf->fw_loaded ? "symbol table" : "firmware found");
-		return -EPROBE_DEFER;
+		return -EINVAL;
 	}
 
 	mutex_lock(&pf->lock);
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c
index 8ec5474f4b18..d540a9dc77b3 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c
@@ -43,6 +43,7 @@
 #include "nfp_main.h"
 #include "nfp_net_ctrl.h"
 #include "nfp_net_repr.h"
+#include "nfp_net_sriov.h"
 #include "nfp_port.h"
 
 static void
@@ -78,12 +79,10 @@ void nfp_repr_inc_rx_stats(struct net_device *netdev, unsigned int len)
 }
 
 static void
-nfp_repr_phy_port_get_stats64(const struct nfp_app *app, u8 phy_port,
+nfp_repr_phy_port_get_stats64(struct nfp_port *port,
 			      struct rtnl_link_stats64 *stats)
 {
-	u8 __iomem *mem;
-
-	mem = app->pf->mac_stats_mem + phy_port * NFP_MAC_STATS_SIZE;
+	u8 __iomem *mem = port->eth_stats;
 
 	/* TX and RX stats are flipped as we are returning the stats as seen
 	 * at the switch port corresponding to the phys port.
@@ -98,67 +97,38 @@ nfp_repr_phy_port_get_stats64(const struct nfp_app *app, u8 phy_port,
 }
 
 static void
-nfp_repr_vf_get_stats64(const struct nfp_app *app, u8 vf,
-			struct rtnl_link_stats64 *stats)
+nfp_repr_vnic_get_stats64(struct nfp_port *port,
+			  struct rtnl_link_stats64 *stats)
 {
-	u8 __iomem *mem;
-
-	mem = app->pf->vf_cfg_mem + vf * NFP_NET_CFG_BAR_SZ;
-
 	/* TX and RX stats are flipped as we are returning the stats as seen
 	 * at the switch port corresponding to the VF.
 	 */
-	stats->tx_packets = readq(mem + NFP_NET_CFG_STATS_RX_FRAMES);
-	stats->tx_bytes = readq(mem + NFP_NET_CFG_STATS_RX_OCTETS);
-	stats->tx_dropped = readq(mem + NFP_NET_CFG_STATS_RX_DISCARDS);
-
-	stats->rx_packets = readq(mem + NFP_NET_CFG_STATS_TX_FRAMES);
-	stats->rx_bytes = readq(mem + NFP_NET_CFG_STATS_TX_OCTETS);
-	stats->rx_dropped = readq(mem + NFP_NET_CFG_STATS_TX_DISCARDS);
-}
-
-static void
-nfp_repr_pf_get_stats64(const struct nfp_app *app, u8 pf,
-			struct rtnl_link_stats64 *stats)
-{
-	u8 __iomem *mem;
-
-	if (pf)
-		return;
+	stats->tx_packets = readq(port->vnic + NFP_NET_CFG_STATS_RX_FRAMES);
+	stats->tx_bytes = readq(port->vnic + NFP_NET_CFG_STATS_RX_OCTETS);
+	stats->tx_dropped = readq(port->vnic + NFP_NET_CFG_STATS_RX_DISCARDS);
 
-	mem = nfp_cpp_area_iomem(app->pf->data_vnic_bar);
-
-	stats->tx_packets = readq(mem + NFP_NET_CFG_STATS_RX_FRAMES);
-	stats->tx_bytes = readq(mem + NFP_NET_CFG_STATS_RX_OCTETS);
-	stats->tx_dropped = readq(mem + NFP_NET_CFG_STATS_RX_DISCARDS);
-
-	stats->rx_packets = readq(mem + NFP_NET_CFG_STATS_TX_FRAMES);
-	stats->rx_bytes = readq(mem + NFP_NET_CFG_STATS_TX_OCTETS);
-	stats->rx_dropped = readq(mem + NFP_NET_CFG_STATS_TX_DISCARDS);
+	stats->rx_packets = readq(port->vnic + NFP_NET_CFG_STATS_TX_FRAMES);
+	stats->rx_bytes = readq(port->vnic + NFP_NET_CFG_STATS_TX_OCTETS);
+	stats->rx_dropped = readq(port->vnic + NFP_NET_CFG_STATS_TX_DISCARDS);
 }
 
 static void
 nfp_repr_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats)
 {
 	struct nfp_repr *repr = netdev_priv(netdev);
-	struct nfp_eth_table_port *eth_port;
-	struct nfp_app *app = repr->app;
 
 	if (WARN_ON(!repr->port))
 		return;
 
 	switch (repr->port->type) {
 	case NFP_PORT_PHYS_PORT:
-		eth_port = __nfp_port_get_eth_port(repr->port);
-		if (!eth_port)
+		if (!__nfp_port_get_eth_port(repr->port))
 			break;
-		nfp_repr_phy_port_get_stats64(app, eth_port->index, stats);
+		nfp_repr_phy_port_get_stats64(repr->port, stats);
 		break;
 	case NFP_PORT_PF_PORT:
-		nfp_repr_pf_get_stats64(app, repr->port->pf_id, stats);
-		break;
 	case NFP_PORT_VF_PORT:
-		nfp_repr_vf_get_stats64(app, repr->port->vf_id, stats);
+		nfp_repr_vnic_get_stats64(repr->port, stats);
 	default:
 		break;
 	}
@@ -239,15 +209,34 @@ static netdev_tx_t nfp_repr_xmit(struct sk_buff *skb, struct net_device *netdev)
 static int nfp_repr_stop(struct net_device *netdev)
 {
 	struct nfp_repr *repr = netdev_priv(netdev);
+	int err;
+
+	err = nfp_app_repr_stop(repr->app, repr);
+	if (err)
+		return err;
 
-	return nfp_app_repr_stop(repr->app, repr);
+	nfp_port_configure(netdev, false);
+	return 0;
 }
 
 static int nfp_repr_open(struct net_device *netdev)
 {
 	struct nfp_repr *repr = netdev_priv(netdev);
+	int err;
+
+	err = nfp_port_configure(netdev, true);
+	if (err)
+		return err;
 
-	return nfp_app_repr_open(repr->app, repr);
+	err = nfp_app_repr_open(repr->app, repr);
+	if (err)
+		goto err_port_disable;
+
+	return 0;
+
+err_port_disable:
+	nfp_port_configure(netdev, false);
+	return err;
 }
 
 const struct net_device_ops nfp_repr_netdev_ops = {
@@ -259,6 +248,11 @@ const struct net_device_ops nfp_repr_netdev_ops = {
 	.ndo_get_offload_stats	= nfp_repr_get_offload_stats,
 	.ndo_get_phys_port_name	= nfp_port_get_phys_port_name,
 	.ndo_setup_tc		= nfp_port_setup_tc,
+	.ndo_set_vf_mac		= nfp_app_set_vf_mac,
+	.ndo_set_vf_vlan	= nfp_app_set_vf_vlan,
+	.ndo_set_vf_spoofchk	= nfp_app_set_vf_spoofchk,
+	.ndo_get_vf_config	= nfp_app_get_vf_config,
+	.ndo_set_vf_link_state	= nfp_app_set_vf_link_state,
 };
 
 static void nfp_repr_clean(struct nfp_repr *repr)
@@ -301,6 +295,8 @@ int nfp_repr_init(struct nfp_app *app, struct net_device *netdev,
 	repr->dst->u.port_info.lower_dev = pf_netdev;
 
 	netdev->netdev_ops = &nfp_repr_netdev_ops;
+	netdev->ethtool_ops = &nfp_port_ethtool_ops;
+
 	SWITCHDEV_SET_OPS(netdev, &nfp_port_switchdev_ops);
 
 	if (nfp_app_has_tc(app)) {
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_sriov.c b/drivers/net/ethernet/netronome/nfp/nfp_net_sriov.c
new file mode 100644
index 000000000000..e6d2e06b050c
--- /dev/null
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_sriov.c
@@ -0,0 +1,243 @@
+/*
+ * Copyright (C) 2017 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below.  You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      1. Redistributions of source code must retain the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer.
+ *
+ *      2. Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials
+ *         provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/bitfield.h>
+#include <linux/errno.h>
+#include <linux/etherdevice.h>
+#include <linux/if_link.h>
+#include <linux/if_ether.h>
+
+#include "nfpcore/nfp_cpp.h"
+#include "nfp_app.h"
+#include "nfp_main.h"
+#include "nfp_net_ctrl.h"
+#include "nfp_net.h"
+#include "nfp_net_sriov.h"
+
+static int
+nfp_net_sriov_check(struct nfp_app *app, int vf, u16 cap, const char *msg)
+{
+	u16 cap_vf;
+
+	if (!app || !app->pf->vfcfg_tbl2)
+		return -EOPNOTSUPP;
+
+	cap_vf = readw(app->pf->vfcfg_tbl2 + NFP_NET_VF_CFG_MB_CAP);
+	if ((cap_vf & cap) != cap) {
+		nfp_warn(app->pf->cpp, "ndo_set_vf_%s not supported\n", msg);
+		return -EOPNOTSUPP;
+	}
+
+	if (vf < 0 || vf >= app->pf->num_vfs) {
+		nfp_warn(app->pf->cpp, "invalid VF id %d\n", vf);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+nfp_net_sriov_update(struct nfp_app *app, int vf, u16 update, const char *msg)
+{
+	struct nfp_net *nn;
+	int ret;
+
+	/* Write update info to mailbox in VF config symbol */
+	writeb(vf, app->pf->vfcfg_tbl2 + NFP_NET_VF_CFG_MB_VF_NUM);
+	writew(update, app->pf->vfcfg_tbl2 + NFP_NET_VF_CFG_MB_UPD);
+
+	nn = list_first_entry(&app->pf->vnics, struct nfp_net, vnic_list);
+	/* Signal VF reconfiguration */
+	ret = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_VF);
+	if (ret)
+		return ret;
+
+	ret = readw(app->pf->vfcfg_tbl2 + NFP_NET_VF_CFG_MB_RET);
+	if (ret)
+		nfp_warn(app->pf->cpp,
+			 "FW refused VF %s update with errno: %d\n", msg, ret);
+	return -ret;
+}
+
+int nfp_app_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
+{
+	struct nfp_app *app = nfp_app_from_netdev(netdev);
+	unsigned int vf_offset;
+	int err;
+
+	err = nfp_net_sriov_check(app, vf, NFP_NET_VF_CFG_MB_CAP_MAC, "mac");
+	if (err)
+		return err;
+
+	if (is_multicast_ether_addr(mac)) {
+		nfp_warn(app->pf->cpp,
+			 "invalid Ethernet address %pM for VF id %d\n",
+			 mac, vf);
+		return -EINVAL;
+	}
+
+	/* Write MAC to VF entry in VF config symbol */
+	vf_offset = NFP_NET_VF_CFG_MB_SZ + vf * NFP_NET_VF_CFG_SZ;
+	writel(get_unaligned_be32(mac), app->pf->vfcfg_tbl2 + vf_offset);
+	writew(get_unaligned_be16(mac + 4),
+	       app->pf->vfcfg_tbl2 + vf_offset + NFP_NET_VF_CFG_MAC_LO);
+
+	return nfp_net_sriov_update(app, vf, NFP_NET_VF_CFG_MB_UPD_MAC, "MAC");
+}
+
+int nfp_app_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos,
+			__be16 vlan_proto)
+{
+	struct nfp_app *app = nfp_app_from_netdev(netdev);
+	unsigned int vf_offset;
+	u16 vlan_tci;
+	int err;
+
+	err = nfp_net_sriov_check(app, vf, NFP_NET_VF_CFG_MB_CAP_VLAN, "vlan");
+	if (err)
+		return err;
+
+	if (vlan_proto != htons(ETH_P_8021Q))
+		return -EOPNOTSUPP;
+
+	if (vlan > 4095 || qos > 7) {
+		nfp_warn(app->pf->cpp,
+			 "invalid vlan id or qos for VF id %d\n", vf);
+		return -EINVAL;
+	}
+
+	/* Write VLAN tag to VF entry in VF config symbol */
+	vlan_tci = FIELD_PREP(NFP_NET_VF_CFG_VLAN_VID, vlan) |
+		FIELD_PREP(NFP_NET_VF_CFG_VLAN_QOS, qos);
+	vf_offset = NFP_NET_VF_CFG_MB_SZ + vf * NFP_NET_VF_CFG_SZ;
+	writew(vlan_tci, app->pf->vfcfg_tbl2 + vf_offset + NFP_NET_VF_CFG_VLAN);
+
+	return nfp_net_sriov_update(app, vf, NFP_NET_VF_CFG_MB_UPD_VLAN,
+				    "vlan");
+}
+
+int nfp_app_set_vf_spoofchk(struct net_device *netdev, int vf, bool enable)
+{
+	struct nfp_app *app = nfp_app_from_netdev(netdev);
+	unsigned int vf_offset;
+	u8 vf_ctrl;
+	int err;
+
+	err = nfp_net_sriov_check(app, vf, NFP_NET_VF_CFG_MB_CAP_SPOOF,
+				  "spoofchk");
+	if (err)
+		return err;
+
+	/* Write spoof check control bit to VF entry in VF config symbol */
+	vf_offset = NFP_NET_VF_CFG_MB_SZ + vf * NFP_NET_VF_CFG_SZ +
+		NFP_NET_VF_CFG_CTRL;
+	vf_ctrl = readb(app->pf->vfcfg_tbl2 + vf_offset);
+	vf_ctrl &= ~NFP_NET_VF_CFG_CTRL_SPOOF;
+	vf_ctrl |= FIELD_PREP(NFP_NET_VF_CFG_CTRL_SPOOF, enable);
+	writeb(vf_ctrl, app->pf->vfcfg_tbl2 + vf_offset);
+
+	return nfp_net_sriov_update(app, vf, NFP_NET_VF_CFG_MB_UPD_SPOOF,
+				    "spoofchk");
+}
+
+int nfp_app_set_vf_link_state(struct net_device *netdev, int vf,
+			      int link_state)
+{
+	struct nfp_app *app = nfp_app_from_netdev(netdev);
+	unsigned int vf_offset;
+	u8 vf_ctrl;
+	int err;
+
+	err = nfp_net_sriov_check(app, vf, NFP_NET_VF_CFG_MB_CAP_LINK_STATE,
+				  "link_state");
+	if (err)
+		return err;
+
+	switch (link_state) {
+	case IFLA_VF_LINK_STATE_AUTO:
+	case IFLA_VF_LINK_STATE_ENABLE:
+	case IFLA_VF_LINK_STATE_DISABLE:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* Write link state to VF entry in VF config symbol */
+	vf_offset = NFP_NET_VF_CFG_MB_SZ + vf * NFP_NET_VF_CFG_SZ +
+		NFP_NET_VF_CFG_CTRL;
+	vf_ctrl = readb(app->pf->vfcfg_tbl2 + vf_offset);
+	vf_ctrl &= ~NFP_NET_VF_CFG_CTRL_LINK_STATE;
+	vf_ctrl |= FIELD_PREP(NFP_NET_VF_CFG_CTRL_LINK_STATE, link_state);
+	writeb(vf_ctrl, app->pf->vfcfg_tbl2 + vf_offset);
+
+	return nfp_net_sriov_update(app, vf, NFP_NET_VF_CFG_MB_UPD_LINK_STATE,
+				    "link state");
+}
+
+int nfp_app_get_vf_config(struct net_device *netdev, int vf,
+			  struct ifla_vf_info *ivi)
+{
+	struct nfp_app *app = nfp_app_from_netdev(netdev);
+	unsigned int vf_offset;
+	u16 vlan_tci;
+	u32 mac_hi;
+	u16 mac_lo;
+	u8 flags;
+	int err;
+
+	err = nfp_net_sriov_check(app, vf, 0, "");
+	if (err)
+		return err;
+
+	vf_offset = NFP_NET_VF_CFG_MB_SZ + vf * NFP_NET_VF_CFG_SZ;
+
+	mac_hi = readl(app->pf->vfcfg_tbl2 + vf_offset);
+	mac_lo = readw(app->pf->vfcfg_tbl2 + vf_offset + NFP_NET_VF_CFG_MAC_LO);
+
+	flags = readb(app->pf->vfcfg_tbl2 + vf_offset + NFP_NET_VF_CFG_CTRL);
+	vlan_tci = readw(app->pf->vfcfg_tbl2 + vf_offset + NFP_NET_VF_CFG_VLAN);
+
+	memset(ivi, 0, sizeof(*ivi));
+	ivi->vf = vf;
+
+	put_unaligned_be32(mac_hi, &ivi->mac[0]);
+	put_unaligned_be16(mac_lo, &ivi->mac[4]);
+
+	ivi->vlan = FIELD_GET(NFP_NET_VF_CFG_VLAN_VID, vlan_tci);
+	ivi->qos = FIELD_GET(NFP_NET_VF_CFG_VLAN_QOS, vlan_tci);
+
+	ivi->spoofchk = FIELD_GET(NFP_NET_VF_CFG_CTRL_SPOOF, flags);
+	ivi->linkstate = FIELD_GET(NFP_NET_VF_CFG_CTRL_LINK_STATE, flags);
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_sriov.h b/drivers/net/ethernet/netronome/nfp/nfp_net_sriov.h
new file mode 100644
index 000000000000..e9df9d1eab8e
--- /dev/null
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_sriov.h
@@ -0,0 +1,86 @@
+/*
+ * Copyright (C) 2017 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below.  You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      1. Redistributions of source code must retain the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer.
+ *
+ *      2. Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials
+ *         provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef _NFP_NET_SRIOV_H_
+#define _NFP_NET_SRIOV_H_
+
+/**
+ * SRIOV VF configuration.
+ * The configuration memory begins with a mailbox region for communication with
+ * the firmware followed by individual VF entries.
+ */
+#define NFP_NET_VF_CFG_SZ		16
+#define NFP_NET_VF_CFG_MB_SZ		16
+
+/* VF config mailbox */
+#define NFP_NET_VF_CFG_MB				0x0
+#define NFP_NET_VF_CFG_MB_CAP				0x0
+#define   NFP_NET_VF_CFG_MB_CAP_MAC			  (0x1 << 0)
+#define   NFP_NET_VF_CFG_MB_CAP_VLAN			  (0x1 << 1)
+#define   NFP_NET_VF_CFG_MB_CAP_SPOOF			  (0x1 << 2)
+#define   NFP_NET_VF_CFG_MB_CAP_LINK_STATE		  (0x1 << 3)
+#define NFP_NET_VF_CFG_MB_RET				0x2
+#define NFP_NET_VF_CFG_MB_UPD				0x4
+#define   NFP_NET_VF_CFG_MB_UPD_MAC			  (0x1 << 0)
+#define   NFP_NET_VF_CFG_MB_UPD_VLAN			  (0x1 << 1)
+#define   NFP_NET_VF_CFG_MB_UPD_SPOOF			  (0x1 << 2)
+#define   NFP_NET_VF_CFG_MB_UPD_LINK_STATE		  (0x1 << 3)
+#define NFP_NET_VF_CFG_MB_VF_NUM			0x7
+
+/* VF config entry
+ * MAC_LO is set that the MAC address can be read in a single 6 byte read
+ * by the NFP
+ */
+#define NFP_NET_VF_CFG_MAC				0x0
+#define   NFP_NET_VF_CFG_MAC_HI				  0x0
+#define   NFP_NET_VF_CFG_MAC_LO				  0x6
+#define NFP_NET_VF_CFG_CTRL				0x4
+#define   NFP_NET_VF_CFG_CTRL_SPOOF			  0x4
+#define   NFP_NET_VF_CFG_CTRL_LINK_STATE		  0x3
+#define     NFP_NET_VF_CFG_LS_MODE_AUTO			    0
+#define     NFP_NET_VF_CFG_LS_MODE_ENABLE		    1
+#define     NFP_NET_VF_CFG_LS_MODE_DISABLE		    2
+#define NFP_NET_VF_CFG_VLAN				0x8
+#define   NFP_NET_VF_CFG_VLAN_QOS			  0xe000
+#define   NFP_NET_VF_CFG_VLAN_VID			  0x0fff
+
+int nfp_app_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
+int nfp_app_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos,
+			__be16 vlan_proto);
+int nfp_app_set_vf_spoofchk(struct net_device *netdev, int vf, bool setting);
+int nfp_app_set_vf_link_state(struct net_device *netdev, int vf,
+			      int link_state);
+int nfp_app_get_vf_config(struct net_device *netdev, int vf,
+			  struct ifla_vf_info *ivi);
+
+#endif /* _NFP_NET_SRIOV_H_ */
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_port.c b/drivers/net/ethernet/netronome/nfp/nfp_port.c
index e42644dbb865..34a6e035fe9a 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_port.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_port.c
@@ -88,19 +88,16 @@ const struct switchdev_ops nfp_port_switchdev_ops = {
 	.switchdev_port_attr_get	= nfp_port_attr_get,
 };
 
-int nfp_port_setup_tc(struct net_device *netdev, u32 handle, u32 chain_index,
-		      __be16 proto, struct tc_to_netdev *tc)
+int nfp_port_setup_tc(struct net_device *netdev, enum tc_setup_type type,
+		      void *type_data)
 {
 	struct nfp_port *port;
 
-	if (chain_index)
-		return -EOPNOTSUPP;
-
 	port = nfp_port_from_netdev(netdev);
 	if (!port)
 		return -EOPNOTSUPP;
 
-	return nfp_app_setup_tc(port->app, netdev, handle, proto, tc);
+	return nfp_app_setup_tc(port->app, netdev, type, type_data);
 }
 
 struct nfp_port *
@@ -181,6 +178,33 @@ nfp_port_get_phys_port_name(struct net_device *netdev, char *name, size_t len)
 	return 0;
 }
 
+/**
+ * nfp_port_configure() - helper to set the interface configured bit
+ * @netdev:	net_device instance
+ * @configed:	Desired state
+ *
+ * Helper to set the ifup/ifdown state on the PHY only if there is a physical
+ * interface associated with the netdev.
+ *
+ * Return:
+ * 0 - configuration successful (or no change);
+ * -ERRNO - configuration failed.
+ */
+int nfp_port_configure(struct net_device *netdev, bool configed)
+{
+	struct nfp_eth_table_port *eth_port;
+	struct nfp_port *port;
+	int err;
+
+	port = nfp_port_from_netdev(netdev);
+	eth_port = __nfp_port_get_eth_port(port);
+	if (!eth_port)
+		return 0;
+
+	err = nfp_eth_set_configured(port->app->cpp, eth_port->index, configed);
+	return err < 0 && err != -EOPNOTSUPP ? err : 0;
+}
+
 int nfp_port_init_phy_port(struct nfp_pf *pf, struct nfp_app *app,
 			   struct nfp_port *port, unsigned int id)
 {
@@ -201,6 +225,9 @@ int nfp_port_init_phy_port(struct nfp_pf *pf, struct nfp_app *app,
 
 	port->eth_port = &pf->eth_tbl->ports[id];
 	port->eth_id = pf->eth_tbl->ports[id].index;
+	if (pf->mac_stats_mem)
+		port->eth_stats =
+			pf->mac_stats_mem + port->eth_id * NFP_MAC_STATS_SIZE;
 
 	return 0;
 }
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_port.h b/drivers/net/ethernet/netronome/nfp/nfp_port.h
index a33d22e18f94..51dcb9c603ee 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_port.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_port.h
@@ -36,7 +36,6 @@
 
 #include <net/devlink.h>
 
-struct tc_to_netdev;
 struct net_device;
 struct nfp_app;
 struct nfp_pf;
@@ -77,8 +76,10 @@ enum nfp_port_flags {
  * @dl_port:	devlink port structure
  * @eth_id:	for %NFP_PORT_PHYS_PORT port ID in NFP enumeration scheme
  * @eth_port:	for %NFP_PORT_PHYS_PORT translated ETH Table port entry
+ * @eth_stats:	for %NFP_PORT_PHYS_PORT MAC stats if available
  * @pf_id:	for %NFP_PORT_PF_PORT, %NFP_PORT_VF_PORT ID of the PCI PF (0-3)
  * @vf_id:	for %NFP_PORT_VF_PORT ID of the PCI VF within @pf_id
+ * @vnic:	for %NFP_PORT_PF_PORT, %NFP_PORT_VF_PORT vNIC ctrl memory
  * @port_list:	entry on pf's list of ports
  */
 struct nfp_port {
@@ -96,21 +97,29 @@ struct nfp_port {
 		struct {
 			unsigned int eth_id;
 			struct nfp_eth_table_port *eth_port;
+			u8 __iomem *eth_stats;
 		};
 		/* NFP_PORT_PF_PORT, NFP_PORT_VF_PORT */
 		struct {
 			unsigned int pf_id;
 			unsigned int vf_id;
+			u8 __iomem *vnic;
 		};
 	};
 
 	struct list_head port_list;
 };
 
+extern const struct ethtool_ops nfp_port_ethtool_ops;
 extern const struct switchdev_ops nfp_port_switchdev_ops;
 
-int nfp_port_setup_tc(struct net_device *netdev, u32 handle, u32 chain_index,
-		      __be16 proto, struct tc_to_netdev *tc);
+int nfp_port_setup_tc(struct net_device *netdev, enum tc_setup_type type,
+		      void *type_data);
+
+static inline bool nfp_port_is_vnic(const struct nfp_port *port)
+{
+	return port->type == NFP_PORT_PF_PORT || port->type == NFP_PORT_VF_PORT;
+}
 
 struct nfp_port *nfp_port_from_netdev(struct net_device *netdev);
 struct nfp_port *
@@ -120,6 +129,7 @@ struct nfp_eth_table_port *nfp_port_get_eth_port(struct nfp_port *port);
 
 int
 nfp_port_get_phys_port_name(struct net_device *netdev, char *name, size_t len);
+int nfp_port_configure(struct net_device *netdev, bool configed);
 
 struct nfp_port *
 nfp_port_alloc(struct nfp_app *app, enum nfp_port_type type,
@@ -144,31 +154,32 @@ void nfp_devlink_port_unregister(struct nfp_port *port);
 #define NFP_MAC_STATS_SIZE                0x0200
 
 #define NFP_MAC_STATS_RX_IN_OCTETS			(NFP_MAC_STATS_BASE + 0x000)
+							/* unused 0x008 */
 #define NFP_MAC_STATS_RX_FRAME_TOO_LONG_ERRORS		(NFP_MAC_STATS_BASE + 0x010)
 #define NFP_MAC_STATS_RX_RANGE_LENGTH_ERRORS		(NFP_MAC_STATS_BASE + 0x018)
 #define NFP_MAC_STATS_RX_VLAN_REVEIVE_OK		(NFP_MAC_STATS_BASE + 0x020)
 #define NFP_MAC_STATS_RX_IN_ERRORS			(NFP_MAC_STATS_BASE + 0x028)
 #define NFP_MAC_STATS_RX_IN_BROADCAST_PKTS		(NFP_MAC_STATS_BASE + 0x030)
-#define NFP_MAC_STATS_RX_STATS_DROP_EVENTS		(NFP_MAC_STATS_BASE + 0x038)
+#define NFP_MAC_STATS_RX_DROP_EVENTS			(NFP_MAC_STATS_BASE + 0x038)
 #define NFP_MAC_STATS_RX_ALIGNMENT_ERRORS		(NFP_MAC_STATS_BASE + 0x040)
 #define NFP_MAC_STATS_RX_PAUSE_MAC_CTRL_FRAMES		(NFP_MAC_STATS_BASE + 0x048)
 #define NFP_MAC_STATS_RX_FRAMES_RECEIVED_OK		(NFP_MAC_STATS_BASE + 0x050)
 #define NFP_MAC_STATS_RX_FRAME_CHECK_SEQUENCE_ERRORS	(NFP_MAC_STATS_BASE + 0x058)
 #define NFP_MAC_STATS_RX_UNICAST_PKTS			(NFP_MAC_STATS_BASE + 0x060)
 #define NFP_MAC_STATS_RX_MULTICAST_PKTS			(NFP_MAC_STATS_BASE + 0x068)
-#define NFP_MAC_STATS_RX_STATS_PKTS			(NFP_MAC_STATS_BASE + 0x070)
-#define NFP_MAC_STATS_RX_STATS_UNDERSIZE_PKTS		(NFP_MAC_STATS_BASE + 0x078)
-#define NFP_MAC_STATS_RX_STATS_PKTS_64_OCTETS		(NFP_MAC_STATS_BASE + 0x080)
-#define NFP_MAC_STATS_RX_STATS_PKTS_65_TO_127_OCTETS	(NFP_MAC_STATS_BASE + 0x088)
-#define NFP_MAC_STATS_RX_STATS_PKTS_512_TO_1023_OCTETS	(NFP_MAC_STATS_BASE + 0x090)
-#define NFP_MAC_STATS_RX_STATS_PKTS_1024_TO_1518_OCTETS	(NFP_MAC_STATS_BASE + 0x098)
-#define NFP_MAC_STATS_RX_STATS_JABBERS			(NFP_MAC_STATS_BASE + 0x0a0)
-#define NFP_MAC_STATS_RX_STATS_FRAGMENTS		(NFP_MAC_STATS_BASE + 0x0a8)
+#define NFP_MAC_STATS_RX_PKTS				(NFP_MAC_STATS_BASE + 0x070)
+#define NFP_MAC_STATS_RX_UNDERSIZE_PKTS			(NFP_MAC_STATS_BASE + 0x078)
+#define NFP_MAC_STATS_RX_PKTS_64_OCTETS			(NFP_MAC_STATS_BASE + 0x080)
+#define NFP_MAC_STATS_RX_PKTS_65_TO_127_OCTETS		(NFP_MAC_STATS_BASE + 0x088)
+#define NFP_MAC_STATS_RX_PKTS_512_TO_1023_OCTETS	(NFP_MAC_STATS_BASE + 0x090)
+#define NFP_MAC_STATS_RX_PKTS_1024_TO_1518_OCTETS	(NFP_MAC_STATS_BASE + 0x098)
+#define NFP_MAC_STATS_RX_JABBERS			(NFP_MAC_STATS_BASE + 0x0a0)
+#define NFP_MAC_STATS_RX_FRAGMENTS			(NFP_MAC_STATS_BASE + 0x0a8)
 #define NFP_MAC_STATS_RX_PAUSE_FRAMES_CLASS2		(NFP_MAC_STATS_BASE + 0x0b0)
 #define NFP_MAC_STATS_RX_PAUSE_FRAMES_CLASS3		(NFP_MAC_STATS_BASE + 0x0b8)
-#define NFP_MAC_STATS_RX_STATS_PKTS_128_TO_255_OCTETS	(NFP_MAC_STATS_BASE + 0x0c0)
-#define NFP_MAC_STATS_RX_STATS_PKTS_256_TO_511_OCTETS	(NFP_MAC_STATS_BASE + 0x0c8)
-#define NFP_MAC_STATS_RX_STATS_PKTS_1519_TO_MAX_OCTETS	(NFP_MAC_STATS_BASE + 0x0d0)
+#define NFP_MAC_STATS_RX_PKTS_128_TO_255_OCTETS		(NFP_MAC_STATS_BASE + 0x0c0)
+#define NFP_MAC_STATS_RX_PKTS_256_TO_511_OCTETS		(NFP_MAC_STATS_BASE + 0x0c8)
+#define NFP_MAC_STATS_RX_PKTS_1519_TO_MAX_OCTETS	(NFP_MAC_STATS_BASE + 0x0d0)
 #define NFP_MAC_STATS_RX_OVERSIZE_PKTS			(NFP_MAC_STATS_BASE + 0x0d8)
 #define NFP_MAC_STATS_RX_PAUSE_FRAMES_CLASS0		(NFP_MAC_STATS_BASE + 0x0e0)
 #define NFP_MAC_STATS_RX_PAUSE_FRAMES_CLASS1		(NFP_MAC_STATS_BASE + 0x0e8)
@@ -178,9 +189,12 @@ void nfp_devlink_port_unregister(struct nfp_port *port);
 #define NFP_MAC_STATS_RX_PAUSE_FRAMES_CLASS7		(NFP_MAC_STATS_BASE + 0x108)
 #define NFP_MAC_STATS_RX_MAC_CTRL_FRAMES_RECEIVED	(NFP_MAC_STATS_BASE + 0x110)
 #define NFP_MAC_STATS_RX_MAC_HEAD_DROP			(NFP_MAC_STATS_BASE + 0x118)
-
+							/* unused 0x120 */
+							/* unused 0x128 */
+							/* unused 0x130 */
 #define NFP_MAC_STATS_TX_QUEUE_DROP			(NFP_MAC_STATS_BASE + 0x138)
 #define NFP_MAC_STATS_TX_OUT_OCTETS			(NFP_MAC_STATS_BASE + 0x140)
+							/* unused 0x148 */
 #define NFP_MAC_STATS_TX_VLAN_TRANSMITTED_OK		(NFP_MAC_STATS_BASE + 0x150)
 #define NFP_MAC_STATS_TX_OUT_ERRORS			(NFP_MAC_STATS_BASE + 0x158)
 #define NFP_MAC_STATS_TX_BROADCAST_PKTS			(NFP_MAC_STATS_BASE + 0x160)
@@ -192,8 +206,16 @@ void nfp_devlink_port_unregister(struct nfp_port *port);
 #define NFP_MAC_STATS_TX_UNICAST_PKTS			(NFP_MAC_STATS_BASE + 0x190)
 #define NFP_MAC_STATS_TX_MULTICAST_PKTS			(NFP_MAC_STATS_BASE + 0x198)
 #define NFP_MAC_STATS_TX_PKTS_65_TO_127_OCTETS		(NFP_MAC_STATS_BASE + 0x1a0)
-#define NFP_MAC_STATS_TX_PKTS_127_TO_512_OCTETS		(NFP_MAC_STATS_BASE + 0x1a8)
-#define NFP_MAC_STATS_TX_PKTS_128_TO_1518_OCTETS	(NFP_MAC_STATS_BASE + 0x1b0)
-#define NFP_MAC_STATS_TX_PKTS_1518_TO_MAX_OCTETS	(NFP_MAC_STATS_BASE + 0x1b8)
+#define NFP_MAC_STATS_TX_PKTS_128_TO_255_OCTETS		(NFP_MAC_STATS_BASE + 0x1a8)
+#define NFP_MAC_STATS_TX_PKTS_1024_TO_1518_OCTETS	(NFP_MAC_STATS_BASE + 0x1b0)
+#define NFP_MAC_STATS_TX_PKTS_1519_TO_MAX_OCTETS	(NFP_MAC_STATS_BASE + 0x1b8)
+#define NFP_MAC_STATS_TX_PAUSE_FRAMES_CLASS0		(NFP_MAC_STATS_BASE + 0x1c0)
+#define NFP_MAC_STATS_TX_PAUSE_FRAMES_CLASS1		(NFP_MAC_STATS_BASE + 0x1c8)
+#define NFP_MAC_STATS_TX_PAUSE_FRAMES_CLASS4		(NFP_MAC_STATS_BASE + 0x1d0)
+#define NFP_MAC_STATS_TX_PAUSE_FRAMES_CLASS5		(NFP_MAC_STATS_BASE + 0x1d8)
+#define NFP_MAC_STATS_TX_PAUSE_FRAMES_CLASS2		(NFP_MAC_STATS_BASE + 0x1e0)
+#define NFP_MAC_STATS_TX_PAUSE_FRAMES_CLASS3		(NFP_MAC_STATS_BASE + 0x1e8)
+#define NFP_MAC_STATS_TX_PAUSE_FRAMES_CLASS6		(NFP_MAC_STATS_BASE + 0x1f0)
+#define NFP_MAC_STATS_TX_PAUSE_FRAMES_CLASS7		(NFP_MAC_STATS_BASE + 0x1f8)
 
 #endif
diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp.h b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp.h
index 1a8d04a1e113..3ce51f03126f 100644
--- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp.h
+++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp.h
@@ -97,6 +97,8 @@ nfp_resource_acquire(struct nfp_cpp *cpp, const char *name);
 
 void nfp_resource_release(struct nfp_resource *res);
 
+int nfp_resource_wait(struct nfp_cpp *cpp, const char *name, unsigned int secs);
+
 u32 nfp_resource_cpp_id(struct nfp_resource *res);
 
 const char *nfp_resource_name(struct nfp_resource *res);
diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp_eth.c b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp_eth.c
index c2bc36e8649f..f6f7c085f8e0 100644
--- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp_eth.c
+++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp_eth.c
@@ -391,7 +391,10 @@ int nfp_eth_config_commit_end(struct nfp_nsp *nsp)
  * Enable or disable PHY module (this usually means setting the TX lanes
  * disable bits).
  *
- * Return: 0 or -ERRNO.
+ * Return:
+ * 0 - configuration successful;
+ * 1 - no changes were needed;
+ * -ERRNO - configuration failed.
  */
 int nfp_eth_set_mod_enable(struct nfp_cpp *cpp, unsigned int idx, bool enable)
 {
@@ -427,7 +430,10 @@ int nfp_eth_set_mod_enable(struct nfp_cpp *cpp, unsigned int idx, bool enable)
  *
  * Set the ifup/ifdown state on the PHY.
  *
- * Return: 0 or -ERRNO.
+ * Return:
+ * 0 - configuration successful;
+ * 1 - no changes were needed;
+ * -ERRNO - configuration failed.
  */
 int nfp_eth_set_configured(struct nfp_cpp *cpp, unsigned int idx, bool configed)
 {
@@ -439,6 +445,14 @@ int nfp_eth_set_configured(struct nfp_cpp *cpp, unsigned int idx, bool configed)
 	if (IS_ERR(nsp))
 		return PTR_ERR(nsp);
 
+	/* Older ABI versions did support this feature, however this has only
+	 * been reliable since ABI 20.
+	 */
+	if (nfp_nsp_get_abi_ver_minor(nsp) < 20) {
+		nfp_eth_config_cleanup_end(nsp);
+		return -EOPNOTSUPP;
+	}
+
 	entries = nfp_nsp_config_entries(nsp);
 
 	/* Check if we are already in requested state */
diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_resource.c b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_resource.c
index 072612263dab..b1dd13ff282b 100644
--- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_resource.c
+++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_resource.c
@@ -250,6 +250,51 @@ void nfp_resource_release(struct nfp_resource *res)
 }
 
 /**
+ * nfp_resource_wait() - Wait for resource to appear
+ * @cpp:	NFP CPP handle
+ * @name:	Name of the resource
+ * @secs:	Number of seconds to wait
+ *
+ * Wait for resource to appear in the resource table, grab and release
+ * its lock.  The wait is jiffies-based, don't expect fine granularity.
+ *
+ * Return: 0 on success, errno otherwise.
+ */
+int nfp_resource_wait(struct nfp_cpp *cpp, const char *name, unsigned int secs)
+{
+	unsigned long warn_at = jiffies + NFP_MUTEX_WAIT_FIRST_WARN * HZ;
+	unsigned long err_at = jiffies + secs * HZ;
+	struct nfp_resource *res;
+
+	while (true) {
+		res = nfp_resource_acquire(cpp, name);
+		if (!IS_ERR(res)) {
+			nfp_resource_release(res);
+			return 0;
+		}
+
+		if (PTR_ERR(res) != -ENOENT) {
+			nfp_err(cpp, "error waiting for resource %s: %ld\n",
+				name, PTR_ERR(res));
+			return PTR_ERR(res);
+		}
+		if (time_is_before_eq_jiffies(err_at)) {
+			nfp_err(cpp, "timeout waiting for resource %s\n", name);
+			return -ETIMEDOUT;
+		}
+		if (time_is_before_eq_jiffies(warn_at)) {
+			warn_at = jiffies + NFP_MUTEX_WAIT_NEXT_WARN * HZ;
+			nfp_info(cpp, "waiting for NFP resource %s\n", name);
+		}
+		if (msleep_interruptible(10)) {
+			nfp_err(cpp, "wait for resource %s interrupted\n",
+				name);
+			return -ERESTARTSYS;
+		}
+	}
+}
+
+/**
  * nfp_resource_cpp_id() - Return the cpp_id of a resource handle
  * @res:	NFP Resource handle
  *
diff --git a/drivers/net/ethernet/netronome/nfp/nic/main.c b/drivers/net/ethernet/netronome/nfp/nic/main.c
index 520684242b7d..d5b587fccaa3 100644
--- a/drivers/net/ethernet/netronome/nfp/nic/main.c
+++ b/drivers/net/ethernet/netronome/nfp/nic/main.c
@@ -49,10 +49,22 @@ static int nfp_nic_init(struct nfp_app *app)
 	return 0;
 }
 
+static int nfp_nic_sriov_enable(struct nfp_app *app, int num_vfs)
+{
+	return 0;
+}
+
+static void nfp_nic_sriov_disable(struct nfp_app *app)
+{
+}
+
 const struct nfp_app_type app_nic = {
 	.id		= NFP_APP_CORE_NIC,
 	.name		= "nic",
 
 	.init		= nfp_nic_init,
-	.vnic_init	= nfp_app_nic_vnic_init,
+	.vnic_alloc	= nfp_app_nic_vnic_alloc,
+
+	.sriov_enable	= nfp_nic_sriov_enable,
+	.sriov_disable	= nfp_nic_sriov_disable,
 };
diff --git a/drivers/net/ethernet/nuvoton/w90p910_ether.c b/drivers/net/ethernet/nuvoton/w90p910_ether.c
index 89ab786da25f..4a67c55aa9f1 100644
--- a/drivers/net/ethernet/nuvoton/w90p910_ether.c
+++ b/drivers/net/ethernet/nuvoton/w90p910_ether.c
@@ -11,6 +11,7 @@
 
 #include <linux/module.h>
 #include <linux/init.h>
+#include <linux/interrupt.h>
 #include <linux/mii.h>
 #include <linux/netdevice.h>
 #include <linux/etherdevice.h>
diff --git a/drivers/net/ethernet/nvidia/forcedeth.c b/drivers/net/ethernet/nvidia/forcedeth.c
index aa912f43e15f..994a83a1f0a5 100644
--- a/drivers/net/ethernet/nvidia/forcedeth.c
+++ b/drivers/net/ethernet/nvidia/forcedeth.c
@@ -5629,9 +5629,8 @@ static int nv_probe(struct pci_dev *pci_dev, const struct pci_device_id *id)
 
 	setup_timer(&np->oom_kick, nv_do_rx_refill, (unsigned long)dev);
 	setup_timer(&np->nic_poll, nv_do_nic_poll, (unsigned long)dev);
-	init_timer_deferrable(&np->stats_poll);
-	np->stats_poll.data = (unsigned long) dev;
-	np->stats_poll.function = nv_do_stats_poll;	/* timer handler */
+	setup_deferrable_timer(&np->stats_poll, nv_do_stats_poll,
+			       (unsigned long)dev);
 
 	err = pci_enable_device(pci_dev);
 	if (err)
diff --git a/drivers/net/ethernet/qlogic/netxen/netxen_nic_hw.c b/drivers/net/ethernet/qlogic/netxen/netxen_nic_hw.c
index 66ff15d08bad..0a66389c06c2 100644
--- a/drivers/net/ethernet/qlogic/netxen/netxen_nic_hw.c
+++ b/drivers/net/ethernet/qlogic/netxen/netxen_nic_hw.c
@@ -2311,7 +2311,7 @@ netxen_md_rdqueue(struct netxen_adapter *adapter,
 				 loop_cnt++) {
 		NX_WR_DUMP_REG(select_addr, adapter->ahw.pci_base0, queue_id);
 		read_addr = queueEntry->read_addr;
-		for (k = 0; k < read_cnt; k--) {
+		for (k = 0; k < read_cnt; k++) {
 			NX_RD_DUMP_REG(read_addr, adapter->ahw.pci_base0,
 							&read_value);
 			*data_buff++ = read_value;
diff --git a/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c b/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c
index 827de838389f..f2e8de607119 100644
--- a/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c
+++ b/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c
@@ -2828,7 +2828,7 @@ netxen_show_bridged_mode(struct device *dev,
 	return sprintf(buf, "%d\n", bridged_mode);
 }
 
-static struct device_attribute dev_attr_bridged_mode = {
+static const struct device_attribute dev_attr_bridged_mode = {
        .attr = {.name = "bridged_mode", .mode = (S_IRUGO | S_IWUSR)},
        .show = netxen_show_bridged_mode,
        .store = netxen_store_bridged_mode,
@@ -2860,7 +2860,7 @@ netxen_show_diag_mode(struct device *dev,
 			!!(adapter->flags & NETXEN_NIC_DIAG_ENABLED));
 }
 
-static struct device_attribute dev_attr_diag_mode = {
+static const struct device_attribute dev_attr_diag_mode = {
 	.attr = {.name = "diag_mode", .mode = (S_IRUGO | S_IWUSR)},
 	.show = netxen_show_diag_mode,
 	.store = netxen_store_diag_mode,
diff --git a/drivers/net/ethernet/qlogic/qed/qed_dcbx.c b/drivers/net/ethernet/qlogic/qed/qed_dcbx.c
index eaca4578435d..8f6ccc0c39e5 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_dcbx.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_dcbx.c
@@ -1244,7 +1244,6 @@ int qed_dcbx_get_config_params(struct qed_hwfn *p_hwfn,
 	if (!dcbx_info)
 		return -ENOMEM;
 
-	memset(dcbx_info, 0, sizeof(*dcbx_info));
 	rc = qed_dcbx_query_params(p_hwfn, dcbx_info, QED_DCBX_OPERATIONAL_MIB);
 	if (rc) {
 		kfree(dcbx_info);
diff --git a/drivers/net/ethernet/qlogic/qed/qed_dev.c b/drivers/net/ethernet/qlogic/qed/qed_dev.c
index 6c87bed13bd2..58a689fb04db 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_dev.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_dev.c
@@ -1684,6 +1684,8 @@ int qed_hw_init(struct qed_dev *cdev, struct qed_hw_init_params *p_params)
 			   "Load request was sent. Load code: 0x%x\n",
 			   load_code);
 
+		qed_mcp_set_capabilities(p_hwfn, p_hwfn->p_main_ptt);
+
 		qed_reset_mb_shadow(p_hwfn, p_hwfn->p_main_ptt);
 
 		p_hwfn->first_on_engine = (load_code ==
@@ -2472,6 +2474,7 @@ static int qed_hw_get_nvm_info(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 {
 	u32 port_cfg_addr, link_temp, nvm_cfg_addr, device_capabilities;
 	u32 nvm_cfg1_offset, mf_mode, addr, generic_cont0, core_cfg;
+	struct qed_mcp_link_capabilities *p_caps;
 	struct qed_mcp_link_params *link;
 
 	/* Read global nvm_cfg address */
@@ -2534,6 +2537,7 @@ static int qed_hw_get_nvm_info(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 
 	/* Read default link configuration */
 	link = &p_hwfn->mcp_info->link_input;
+	p_caps = &p_hwfn->mcp_info->link_capabilities;
 	port_cfg_addr = MCP_REG_SCRATCH + nvm_cfg1_offset +
 			offsetof(struct nvm_cfg1, port[MFW_PORT(p_hwfn)]);
 	link_temp = qed_rd(p_hwfn, p_ptt,
@@ -2588,10 +2592,45 @@ static int qed_hw_get_nvm_info(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 				   NVM_CFG1_PORT_DRV_FLOW_CONTROL_TX);
 	link->loopback_mode = 0;
 
-	DP_VERBOSE(p_hwfn, NETIF_MSG_LINK,
-		   "Read default link: Speed 0x%08x, Adv. Speed 0x%08x, AN: 0x%02x, PAUSE AN: 0x%02x\n",
-		   link->speed.forced_speed, link->speed.advertised_speeds,
-		   link->speed.autoneg, link->pause.autoneg);
+	if (p_hwfn->mcp_info->capabilities & FW_MB_PARAM_FEATURE_SUPPORT_EEE) {
+		link_temp = qed_rd(p_hwfn, p_ptt, port_cfg_addr +
+				   offsetof(struct nvm_cfg1_port, ext_phy));
+		link_temp &= NVM_CFG1_PORT_EEE_POWER_SAVING_MODE_MASK;
+		link_temp >>= NVM_CFG1_PORT_EEE_POWER_SAVING_MODE_OFFSET;
+		p_caps->default_eee = QED_MCP_EEE_ENABLED;
+		link->eee.enable = true;
+		switch (link_temp) {
+		case NVM_CFG1_PORT_EEE_POWER_SAVING_MODE_DISABLED:
+			p_caps->default_eee = QED_MCP_EEE_DISABLED;
+			link->eee.enable = false;
+			break;
+		case NVM_CFG1_PORT_EEE_POWER_SAVING_MODE_BALANCED:
+			p_caps->eee_lpi_timer = EEE_TX_TIMER_USEC_BALANCED_TIME;
+			break;
+		case NVM_CFG1_PORT_EEE_POWER_SAVING_MODE_AGGRESSIVE:
+			p_caps->eee_lpi_timer =
+			    EEE_TX_TIMER_USEC_AGGRESSIVE_TIME;
+			break;
+		case NVM_CFG1_PORT_EEE_POWER_SAVING_MODE_LOW_LATENCY:
+			p_caps->eee_lpi_timer = EEE_TX_TIMER_USEC_LATENCY_TIME;
+			break;
+		}
+
+		link->eee.tx_lpi_timer = p_caps->eee_lpi_timer;
+		link->eee.tx_lpi_enable = link->eee.enable;
+		link->eee.adv_caps = QED_EEE_1G_ADV | QED_EEE_10G_ADV;
+	} else {
+		p_caps->default_eee = QED_MCP_EEE_UNSUPPORTED;
+	}
+
+	DP_VERBOSE(p_hwfn,
+		   NETIF_MSG_LINK,
+		   "Read default link: Speed 0x%08x, Adv. Speed 0x%08x, AN: 0x%02x, PAUSE AN: 0x%02x EEE: %02x [%08x usec]\n",
+		   link->speed.forced_speed,
+		   link->speed.advertised_speeds,
+		   link->speed.autoneg,
+		   link->pause.autoneg,
+		   p_caps->default_eee, p_caps->eee_lpi_timer);
 
 	/* Read Multi-function information from shmem */
 	addr = MCP_REG_SCRATCH + nvm_cfg1_offset +
@@ -2751,6 +2790,27 @@ static void qed_hw_info_port_num(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 		qed_hw_info_port_num_ah(p_hwfn, p_ptt);
 }
 
+static void qed_get_eee_caps(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
+{
+	struct qed_mcp_link_capabilities *p_caps;
+	u32 eee_status;
+
+	p_caps = &p_hwfn->mcp_info->link_capabilities;
+	if (p_caps->default_eee == QED_MCP_EEE_UNSUPPORTED)
+		return;
+
+	p_caps->eee_speed_caps = 0;
+	eee_status = qed_rd(p_hwfn, p_ptt, p_hwfn->mcp_info->port_addr +
+			    offsetof(struct public_port, eee_status));
+	eee_status = (eee_status & EEE_SUPPORTED_SPEED_MASK) >>
+			EEE_SUPPORTED_SPEED_OFFSET;
+
+	if (eee_status & EEE_1G_SUPPORTED)
+		p_caps->eee_speed_caps |= QED_EEE_1G_ADV;
+	if (eee_status & EEE_10G_ADV)
+		p_caps->eee_speed_caps |= QED_EEE_10G_ADV;
+}
+
 static int
 qed_get_hw_info(struct qed_hwfn *p_hwfn,
 		struct qed_ptt *p_ptt,
@@ -2767,6 +2827,8 @@ qed_get_hw_info(struct qed_hwfn *p_hwfn,
 
 	qed_hw_info_port_num(p_hwfn, p_ptt);
 
+	qed_mcp_get_capabilities(p_hwfn, p_ptt);
+
 	qed_hw_get_nvm_info(p_hwfn, p_ptt);
 
 	rc = qed_int_igu_read_cam(p_hwfn, p_ptt);
@@ -2785,6 +2847,8 @@ qed_get_hw_info(struct qed_hwfn *p_hwfn,
 				p_hwfn->mcp_info->func_info.ovlan;
 
 		qed_mcp_cmd_port_init(p_hwfn, p_ptt);
+
+		qed_get_eee_caps(p_hwfn, p_ptt);
 	}
 
 	if (qed_mcp_is_init(p_hwfn)) {
@@ -3630,7 +3694,7 @@ static int qed_set_coalesce(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
 	}
 
 	p_coal_timeset = p_eth_qzone;
-	memset(p_coal_timeset, 0, eth_qzone_size);
+	memset(p_eth_qzone, 0, eth_qzone_size);
 	SET_FIELD(p_coal_timeset->value, COALESCING_TIMESET_TIMESET, timeset);
 	SET_FIELD(p_coal_timeset->value, COALESCING_TIMESET_VALID, 1);
 	qed_memcpy_to(p_hwfn, p_ptt, hw_addr, p_eth_qzone, eth_qzone_size);
@@ -3638,12 +3702,46 @@ static int qed_set_coalesce(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
 	return 0;
 }
 
-int qed_set_rxq_coalesce(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
-			 u16 coalesce, u16 qid, u16 sb_id)
+int qed_set_queue_coalesce(u16 rx_coal, u16 tx_coal, void *p_handle)
+{
+	struct qed_queue_cid *p_cid = p_handle;
+	struct qed_hwfn *p_hwfn;
+	struct qed_ptt *p_ptt;
+	int rc = 0;
+
+	p_hwfn = p_cid->p_owner;
+
+	if (IS_VF(p_hwfn->cdev))
+		return qed_vf_pf_set_coalesce(p_hwfn, rx_coal, tx_coal, p_cid);
+
+	p_ptt = qed_ptt_acquire(p_hwfn);
+	if (!p_ptt)
+		return -EAGAIN;
+
+	if (rx_coal) {
+		rc = qed_set_rxq_coalesce(p_hwfn, p_ptt, rx_coal, p_cid);
+		if (rc)
+			goto out;
+		p_hwfn->cdev->rx_coalesce_usecs = rx_coal;
+	}
+
+	if (tx_coal) {
+		rc = qed_set_txq_coalesce(p_hwfn, p_ptt, tx_coal, p_cid);
+		if (rc)
+			goto out;
+		p_hwfn->cdev->tx_coalesce_usecs = tx_coal;
+	}
+out:
+	qed_ptt_release(p_hwfn, p_ptt);
+	return rc;
+}
+
+int qed_set_rxq_coalesce(struct qed_hwfn *p_hwfn,
+			 struct qed_ptt *p_ptt,
+			 u16 coalesce, struct qed_queue_cid *p_cid)
 {
 	struct ustorm_eth_queue_zone eth_qzone;
 	u8 timeset, timer_res;
-	u16 fw_qid = 0;
 	u32 address;
 	int rc;
 
@@ -3660,32 +3758,29 @@ int qed_set_rxq_coalesce(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
 	}
 	timeset = (u8)(coalesce >> timer_res);
 
-	rc = qed_fw_l2_queue(p_hwfn, qid, &fw_qid);
-	if (rc)
-		return rc;
-
-	rc = qed_int_set_timer_res(p_hwfn, p_ptt, timer_res, sb_id, false);
+	rc = qed_int_set_timer_res(p_hwfn, p_ptt, timer_res,
+				   p_cid->sb_igu_id, false);
 	if (rc)
 		goto out;
 
-	address = BAR0_MAP_REG_USDM_RAM + USTORM_ETH_QUEUE_ZONE_OFFSET(fw_qid);
+	address = BAR0_MAP_REG_USDM_RAM +
+		  USTORM_ETH_QUEUE_ZONE_OFFSET(p_cid->abs.queue_id);
 
 	rc = qed_set_coalesce(p_hwfn, p_ptt, address, &eth_qzone,
 			      sizeof(struct ustorm_eth_queue_zone), timeset);
 	if (rc)
 		goto out;
 
-	p_hwfn->cdev->rx_coalesce_usecs = coalesce;
 out:
 	return rc;
 }
 
-int qed_set_txq_coalesce(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
-			 u16 coalesce, u16 qid, u16 sb_id)
+int qed_set_txq_coalesce(struct qed_hwfn *p_hwfn,
+			 struct qed_ptt *p_ptt,
+			 u16 coalesce, struct qed_queue_cid *p_cid)
 {
 	struct xstorm_eth_queue_zone eth_qzone;
 	u8 timeset, timer_res;
-	u16 fw_qid = 0;
 	u32 address;
 	int rc;
 
@@ -3702,22 +3797,16 @@ int qed_set_txq_coalesce(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
 	}
 	timeset = (u8)(coalesce >> timer_res);
 
-	rc = qed_fw_l2_queue(p_hwfn, qid, &fw_qid);
-	if (rc)
-		return rc;
-
-	rc = qed_int_set_timer_res(p_hwfn, p_ptt, timer_res, sb_id, true);
+	rc = qed_int_set_timer_res(p_hwfn, p_ptt, timer_res,
+				   p_cid->sb_igu_id, true);
 	if (rc)
 		goto out;
 
-	address = BAR0_MAP_REG_XSDM_RAM + XSTORM_ETH_QUEUE_ZONE_OFFSET(fw_qid);
+	address = BAR0_MAP_REG_XSDM_RAM +
+		  XSTORM_ETH_QUEUE_ZONE_OFFSET(p_cid->abs.queue_id);
 
 	rc = qed_set_coalesce(p_hwfn, p_ptt, address, &eth_qzone,
 			      sizeof(struct xstorm_eth_queue_zone), timeset);
-	if (rc)
-		goto out;
-
-	p_hwfn->cdev->tx_coalesce_usecs = coalesce;
 out:
 	return rc;
 }
diff --git a/drivers/net/ethernet/qlogic/qed/qed_dev_api.h b/drivers/net/ethernet/qlogic/qed/qed_dev_api.h
index 1f1df1bf127c..defdda1ffaa2 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_dev_api.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_dev_api.h
@@ -443,38 +443,35 @@ int qed_final_cleanup(struct qed_hwfn *p_hwfn,
 		      struct qed_ptt *p_ptt, u16 id, bool is_vf);
 
 /**
- * @brief qed_set_rxq_coalesce - Configure coalesce parameters for an Rx queue
- * The fact that we can configure coalescing to up to 511, but on varying
- * accuracy [the bigger the value the less accurate] up to a mistake of 3usec
- * for the highest values.
+ * @brief qed_get_queue_coalesce - Retrieve coalesce value for a given queue.
  *
  * @param p_hwfn
- * @param p_ptt
- * @param coalesce - Coalesce value in micro seconds.
- * @param qid - Queue index.
- * @param qid - SB Id
+ * @param p_coal - store coalesce value read from the hardware.
+ * @param p_handle
  *
  * @return int
- */
-int qed_set_rxq_coalesce(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
-			 u16 coalesce, u16 qid, u16 sb_id);
+ **/
+int qed_get_queue_coalesce(struct qed_hwfn *p_hwfn, u16 *coal, void *handle);
 
 /**
- * @brief qed_set_txq_coalesce - Configure coalesce parameters for a Tx queue
- * While the API allows setting coalescing per-qid, all tx queues sharing a
- * SB should be in same range [i.e., either 0-0x7f, 0x80-0xff or 0x100-0x1ff]
- * otherwise configuration would break.
+ * @brief qed_set_queue_coalesce - Configure coalesce parameters for Rx and
+ *    Tx queue. The fact that we can configure coalescing to up to 511, but on
+ *    varying accuracy [the bigger the value the less accurate] up to a mistake
+ *    of 3usec for the highest values.
+ *    While the API allows setting coalescing per-qid, all queues sharing a SB
+ *    should be in same range [i.e., either 0-0x7f, 0x80-0xff or 0x100-0x1ff]
+ *    otherwise configuration would break.
  *
- * @param p_hwfn
- * @param p_ptt
- * @param coalesce - Coalesce value in micro seconds.
- * @param qid - Queue index.
- * @param qid - SB Id
+ *
+ * @param rx_coal - Rx Coalesce value in micro seconds.
+ * @param tx_coal - TX Coalesce value in micro seconds.
+ * @param p_handle
  *
  * @return int
- */
-int qed_set_txq_coalesce(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
-			 u16 coalesce, u16 qid, u16 sb_id);
+ **/
+int
+qed_set_queue_coalesce(u16 rx_coal, u16 tx_coal, void *p_handle);
+
 
 const char *qed_hw_get_resc_name(enum qed_resources res_id);
 #endif
diff --git a/drivers/net/ethernet/qlogic/qed/qed_hsi.h b/drivers/net/ethernet/qlogic/qed/qed_hsi.h
index 31fb0bffa098..3427fe7049b5 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_hsi.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_hsi.h
@@ -10825,6 +10825,17 @@ struct eth_phy_cfg {
 #define ETH_LOOPBACK_EXT		(3)
 #define ETH_LOOPBACK_MAC		(4)
 
+	u32 eee_cfg;
+#define EEE_CFG_EEE_ENABLED			BIT(0)
+#define EEE_CFG_TX_LPI				BIT(1)
+#define EEE_CFG_ADV_SPEED_1G			BIT(2)
+#define EEE_CFG_ADV_SPEED_10G			BIT(3)
+#define EEE_TX_TIMER_USEC_MASK			(0xfffffff0)
+#define EEE_TX_TIMER_USEC_OFFSET		4
+#define EEE_TX_TIMER_USEC_BALANCED_TIME		(0xa00)
+#define EEE_TX_TIMER_USEC_AGGRESSIVE_TIME	(0x100)
+#define EEE_TX_TIMER_USEC_LATENCY_TIME		(0x6000)
+
 	u32 feature_config_flags;
 #define ETH_EEE_MODE_ADV_LPI		(1 << 0)
 };
@@ -11242,6 +11253,25 @@ struct public_port {
 	u32 wol_pkt_len;
 	u32 wol_pkt_details;
 	struct dcb_dscp_map dcb_dscp_map;
+
+	u32 eee_status;
+#define EEE_ACTIVE_BIT			BIT(0)
+#define EEE_LD_ADV_STATUS_MASK		0x000000f0
+#define EEE_LD_ADV_STATUS_OFFSET	4
+#define EEE_1G_ADV			BIT(1)
+#define EEE_10G_ADV			BIT(2)
+#define EEE_LP_ADV_STATUS_MASK		0x00000f00
+#define EEE_LP_ADV_STATUS_OFFSET	8
+#define EEE_SUPPORTED_SPEED_MASK	0x0000f000
+#define EEE_SUPPORTED_SPEED_OFFSET	12
+#define EEE_1G_SUPPORTED		BIT(1)
+#define EEE_10G_SUPPORTED		BIT(2)
+
+	u32 eee_remote;
+#define EEE_REMOTE_TW_TX_MASK   0x0000ffff
+#define EEE_REMOTE_TW_TX_OFFSET 0
+#define EEE_REMOTE_TW_RX_MASK   0xffff0000
+#define EEE_REMOTE_TW_RX_OFFSET 16
 };
 
 struct public_func {
@@ -11570,6 +11600,9 @@ struct public_drv_mb {
 #define DRV_MSG_CODE_GET_PF_RDMA_PROTOCOL	0x002b0000
 #define DRV_MSG_CODE_OS_WOL			0x002e0000
 
+#define DRV_MSG_CODE_FEATURE_SUPPORT		0x00300000
+#define DRV_MSG_CODE_GET_MFW_FEATURE_SUPPORT	0x00310000
+
 #define DRV_MSG_SEQ_NUMBER_MASK			0x0000ffff
 
 	u32 drv_mb_param;
@@ -11653,6 +11686,10 @@ struct public_drv_mb {
 #define DRV_MB_PARAM_BIST_TEST_IMAGE_INDEX_SHIFT	8
 #define DRV_MB_PARAM_BIST_TEST_IMAGE_INDEX_MASK		0x0000FF00
 
+#define DRV_MB_PARAM_FEATURE_SUPPORT_PORT_MASK		0x0000FFFF
+#define DRV_MB_PARAM_FEATURE_SUPPORT_PORT_OFFSET	0
+#define DRV_MB_PARAM_FEATURE_SUPPORT_PORT_EEE		0x00000002
+
 	u32 fw_mb_header;
 #define FW_MSG_CODE_MASK			0xffff0000
 #define FW_MSG_CODE_UNSUPPORTED                 0x00000000
@@ -11696,6 +11733,9 @@ struct public_drv_mb {
 #define FW_MB_PARAM_GET_PF_RDMA_IWARP		0x2
 #define FW_MB_PARAM_GET_PF_RDMA_BOTH		0x3
 
+/* get MFW feature support response */
+#define FW_MB_PARAM_FEATURE_SUPPORT_EEE		0x00000002
+
 #define FW_MB_PARAM_LOAD_DONE_DID_EFUSE_ERROR	(1 << 0)
 
 	u32 drv_pulse_mb;
@@ -11891,7 +11931,16 @@ struct nvm_cfg1_port {
 #define NVM_CFG1_PORT_DRV_FLOW_CONTROL_TX			0x4
 	u32 phy_cfg;
 	u32 mgmt_traffic;
+
 	u32 ext_phy;
+	/* EEE power saving mode */
+#define NVM_CFG1_PORT_EEE_POWER_SAVING_MODE_MASK		0x00FF0000
+#define NVM_CFG1_PORT_EEE_POWER_SAVING_MODE_OFFSET		16
+#define NVM_CFG1_PORT_EEE_POWER_SAVING_MODE_DISABLED		0x0
+#define NVM_CFG1_PORT_EEE_POWER_SAVING_MODE_BALANCED		0x1
+#define NVM_CFG1_PORT_EEE_POWER_SAVING_MODE_AGGRESSIVE		0x2
+#define NVM_CFG1_PORT_EEE_POWER_SAVING_MODE_LOW_LATENCY		0x3
+
 	u32 mba_cfg1;
 	u32 mba_cfg2;
 	u32 vf_cfg;
diff --git a/drivers/net/ethernet/qlogic/qed/qed_l2.c b/drivers/net/ethernet/qlogic/qed/qed_l2.c
index 0ba5ec8a9814..085338990f49 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_l2.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_l2.c
@@ -2047,6 +2047,106 @@ qed_configure_rfs_ntuple_filter(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
 	return qed_spq_post(p_hwfn, p_ent, NULL);
 }
 
+int qed_get_rxq_coalesce(struct qed_hwfn *p_hwfn,
+			 struct qed_ptt *p_ptt,
+			 struct qed_queue_cid *p_cid, u16 *p_rx_coal)
+{
+	u32 coalesce, address, is_valid;
+	struct cau_sb_entry sb_entry;
+	u8 timer_res;
+	int rc;
+
+	rc = qed_dmae_grc2host(p_hwfn, p_ptt, CAU_REG_SB_VAR_MEMORY +
+			       p_cid->sb_igu_id * sizeof(u64),
+			       (u64)(uintptr_t)&sb_entry, 2, 0);
+	if (rc) {
+		DP_ERR(p_hwfn, "dmae_grc2host failed %d\n", rc);
+		return rc;
+	}
+
+	timer_res = GET_FIELD(sb_entry.params, CAU_SB_ENTRY_TIMER_RES0);
+
+	address = BAR0_MAP_REG_USDM_RAM +
+		  USTORM_ETH_QUEUE_ZONE_OFFSET(p_cid->abs.queue_id);
+	coalesce = qed_rd(p_hwfn, p_ptt, address);
+
+	is_valid = GET_FIELD(coalesce, COALESCING_TIMESET_VALID);
+	if (!is_valid)
+		return -EINVAL;
+
+	coalesce = GET_FIELD(coalesce, COALESCING_TIMESET_TIMESET);
+	*p_rx_coal = (u16)(coalesce << timer_res);
+
+	return 0;
+}
+
+int qed_get_txq_coalesce(struct qed_hwfn *p_hwfn,
+			 struct qed_ptt *p_ptt,
+			 struct qed_queue_cid *p_cid, u16 *p_tx_coal)
+{
+	u32 coalesce, address, is_valid;
+	struct cau_sb_entry sb_entry;
+	u8 timer_res;
+	int rc;
+
+	rc = qed_dmae_grc2host(p_hwfn, p_ptt, CAU_REG_SB_VAR_MEMORY +
+			       p_cid->sb_igu_id * sizeof(u64),
+			       (u64)(uintptr_t)&sb_entry, 2, 0);
+	if (rc) {
+		DP_ERR(p_hwfn, "dmae_grc2host failed %d\n", rc);
+		return rc;
+	}
+
+	timer_res = GET_FIELD(sb_entry.params, CAU_SB_ENTRY_TIMER_RES1);
+
+	address = BAR0_MAP_REG_XSDM_RAM +
+		  XSTORM_ETH_QUEUE_ZONE_OFFSET(p_cid->abs.queue_id);
+	coalesce = qed_rd(p_hwfn, p_ptt, address);
+
+	is_valid = GET_FIELD(coalesce, COALESCING_TIMESET_VALID);
+	if (!is_valid)
+		return -EINVAL;
+
+	coalesce = GET_FIELD(coalesce, COALESCING_TIMESET_TIMESET);
+	*p_tx_coal = (u16)(coalesce << timer_res);
+
+	return 0;
+}
+
+int qed_get_queue_coalesce(struct qed_hwfn *p_hwfn, u16 *p_coal, void *handle)
+{
+	struct qed_queue_cid *p_cid = handle;
+	struct qed_ptt *p_ptt;
+	int rc = 0;
+
+	if (IS_VF(p_hwfn->cdev)) {
+		rc = qed_vf_pf_get_coalesce(p_hwfn, p_coal, p_cid);
+		if (rc)
+			DP_NOTICE(p_hwfn, "Unable to read queue coalescing\n");
+
+		return rc;
+	}
+
+	p_ptt = qed_ptt_acquire(p_hwfn);
+	if (!p_ptt)
+		return -EAGAIN;
+
+	if (p_cid->b_is_rx) {
+		rc = qed_get_rxq_coalesce(p_hwfn, p_ptt, p_cid, p_coal);
+		if (rc)
+			goto out;
+	} else {
+		rc = qed_get_txq_coalesce(p_hwfn, p_ptt, p_cid, p_coal);
+		if (rc)
+			goto out;
+	}
+
+out:
+	qed_ptt_release(p_hwfn, p_ptt);
+
+	return rc;
+}
+
 static int qed_fill_eth_dev_info(struct qed_dev *cdev,
 				 struct qed_dev_eth_info *info)
 {
@@ -2696,6 +2796,20 @@ static int qed_ntuple_arfs_filter_config(struct qed_dev *cdev, void *cookie,
 	return rc;
 }
 
+static int qed_get_coalesce(struct qed_dev *cdev, u16 *coal, void *handle)
+{
+	struct qed_queue_cid *p_cid = handle;
+	struct qed_hwfn *p_hwfn;
+	int rc;
+
+	p_hwfn = p_cid->p_owner;
+	rc = qed_get_queue_coalesce(p_hwfn, coal, handle);
+	if (rc)
+		DP_NOTICE(p_hwfn, "Unable to read queue coalescing\n");
+
+	return rc;
+}
+
 static int qed_fp_cqe_completion(struct qed_dev *dev,
 				 u8 rss_id, struct eth_slow_path_rx_cqe *cqe)
 {
@@ -2739,6 +2853,7 @@ static const struct qed_eth_ops qed_eth_ops_pass = {
 	.tunn_config = &qed_tunn_configure,
 	.ntuple_filter_config = &qed_ntuple_arfs_filter_config,
 	.configure_arfs_searcher = &qed_configure_arfs_searcher,
+	.get_coalesce = &qed_get_coalesce,
 };
 
 const struct qed_eth_ops *qed_get_eth_ops(void)
diff --git a/drivers/net/ethernet/qlogic/qed/qed_l2.h b/drivers/net/ethernet/qlogic/qed/qed_l2.h
index f8f09aadced7..cc1f248551c9 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_l2.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_l2.h
@@ -400,4 +400,20 @@ qed_eth_txq_start_ramrod(struct qed_hwfn *p_hwfn,
 
 u8 qed_mcast_bin_from_mac(u8 *mac);
 
-#endif /* _QED_L2_H */
+int qed_set_rxq_coalesce(struct qed_hwfn *p_hwfn,
+			 struct qed_ptt *p_ptt,
+			 u16 coalesce, struct qed_queue_cid *p_cid);
+
+int qed_set_txq_coalesce(struct qed_hwfn *p_hwfn,
+			 struct qed_ptt *p_ptt,
+			 u16 coalesce, struct qed_queue_cid *p_cid);
+
+int qed_get_rxq_coalesce(struct qed_hwfn *p_hwfn,
+			 struct qed_ptt *p_ptt,
+			 struct qed_queue_cid *p_cid, u16 *p_hw_coal);
+
+int qed_get_txq_coalesce(struct qed_hwfn *p_hwfn,
+			 struct qed_ptt *p_ptt,
+			 struct qed_queue_cid *p_cid, u16 *p_hw_coal);
+
+#endif
diff --git a/drivers/net/ethernet/qlogic/qed/qed_main.c b/drivers/net/ethernet/qlogic/qed/qed_main.c
index b11399606990..27832885a87f 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_main.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_main.c
@@ -954,9 +954,7 @@ static int qed_slowpath_start(struct qed_dev *cdev,
 	struct qed_tunnel_info tunn_info;
 	const u8 *data = NULL;
 	struct qed_hwfn *hwfn;
-#ifdef CONFIG_RFS_ACCEL
 	struct qed_ptt *p_ptt;
-#endif
 	int rc = -EINVAL;
 
 	if (qed_iov_wq_start(cdev))
@@ -972,7 +970,6 @@ static int qed_slowpath_start(struct qed_dev *cdev,
 			goto err;
 		}
 
-#ifdef CONFIG_RFS_ACCEL
 		if (cdev->num_hwfns == 1) {
 			p_ptt = qed_ptt_acquire(QED_LEADING_HWFN(cdev));
 			if (p_ptt) {
@@ -983,7 +980,6 @@ static int qed_slowpath_start(struct qed_dev *cdev,
 				goto err;
 			}
 		}
-#endif
 	}
 
 	cdev->rx_coalesce_usecs = QED_DEFAULT_RX_USECS;
@@ -1091,12 +1087,10 @@ err:
 	if (IS_PF(cdev))
 		release_firmware(cdev->firmware);
 
-#ifdef CONFIG_RFS_ACCEL
 	if (IS_PF(cdev) && (cdev->num_hwfns == 1) &&
 	    QED_LEADING_HWFN(cdev)->p_arfs_ptt)
 		qed_ptt_release(QED_LEADING_HWFN(cdev),
 				QED_LEADING_HWFN(cdev)->p_arfs_ptt);
-#endif
 
 	qed_iov_wq_stop(cdev, false);
 
@@ -1111,11 +1105,9 @@ static int qed_slowpath_stop(struct qed_dev *cdev)
 	qed_ll2_dealloc_if(cdev);
 
 	if (IS_PF(cdev)) {
-#ifdef CONFIG_RFS_ACCEL
 		if (cdev->num_hwfns == 1)
 			qed_ptt_release(QED_LEADING_HWFN(cdev),
 					QED_LEADING_HWFN(cdev)->p_arfs_ptt);
-#endif
 		qed_free_stream_mem(cdev);
 		if (IS_QED_ETH_IF(cdev))
 			qed_sriov_disable(cdev, true);
@@ -1305,6 +1297,10 @@ static int qed_set_link(struct qed_dev *cdev, struct qed_link_params *params)
 		}
 	}
 
+	if (params->override_flags & QED_LINK_OVERRIDE_EEE_CONFIG)
+		memcpy(&link_params->eee, &params->eee,
+		       sizeof(link_params->eee));
+
 	rc = qed_mcp_set_link(hwfn, ptt, params->link_up);
 
 	qed_ptt_release(hwfn, ptt);
@@ -1491,6 +1487,21 @@ static void qed_fill_link(struct qed_hwfn *hwfn,
 	if (link.partner_adv_pause == QED_LINK_PARTNER_ASYMMETRIC_PAUSE ||
 	    link.partner_adv_pause == QED_LINK_PARTNER_BOTH_PAUSE)
 		if_link->lp_caps |= QED_LM_Asym_Pause_BIT;
+
+	if (link_caps.default_eee == QED_MCP_EEE_UNSUPPORTED) {
+		if_link->eee_supported = false;
+	} else {
+		if_link->eee_supported = true;
+		if_link->eee_active = link.eee_active;
+		if_link->sup_caps = link_caps.eee_speed_caps;
+		/* MFW clears adv_caps on eee disable; use configured value */
+		if_link->eee.adv_caps = link.eee_adv_caps ? link.eee_adv_caps :
+					params.eee.adv_caps;
+		if_link->eee.lp_adv_caps = link.eee_lp_adv_caps;
+		if_link->eee.enable = params.eee.enable;
+		if_link->eee.tx_lpi_enable = params.eee.tx_lpi_enable;
+		if_link->eee.tx_lpi_timer = params.eee.tx_lpi_timer;
+	}
 }
 
 static void qed_get_current_link(struct qed_dev *cdev,
@@ -1557,36 +1568,10 @@ static int qed_nvm_get_image(struct qed_dev *cdev, enum qed_nvm_images type,
 	return rc;
 }
 
-static void qed_get_coalesce(struct qed_dev *cdev, u16 *rx_coal, u16 *tx_coal)
-{
-	*rx_coal = cdev->rx_coalesce_usecs;
-	*tx_coal = cdev->tx_coalesce_usecs;
-}
-
 static int qed_set_coalesce(struct qed_dev *cdev, u16 rx_coal, u16 tx_coal,
-			    u16 qid, u16 sb_id)
+			    void *handle)
 {
-	struct qed_hwfn *hwfn;
-	struct qed_ptt *ptt;
-	int hwfn_index;
-	int status = 0;
-
-	hwfn_index = qid % cdev->num_hwfns;
-	hwfn = &cdev->hwfns[hwfn_index];
-	ptt = qed_ptt_acquire(hwfn);
-	if (!ptt)
-		return -EAGAIN;
-
-	status = qed_set_rxq_coalesce(hwfn, ptt, rx_coal,
-				      qid / cdev->num_hwfns, sb_id);
-	if (status)
-		goto out;
-	status = qed_set_txq_coalesce(hwfn, ptt, tx_coal,
-				      qid / cdev->num_hwfns, sb_id);
-out:
-	qed_ptt_release(hwfn, ptt);
-
-	return status;
+		return qed_set_queue_coalesce(rx_coal, tx_coal, handle);
 }
 
 static int qed_set_led(struct qed_dev *cdev, enum qed_led_mode mode)
@@ -1735,7 +1720,6 @@ const struct qed_common_ops qed_common_ops_pass = {
 	.chain_alloc = &qed_chain_alloc,
 	.chain_free = &qed_chain_free,
 	.nvm_get_image = &qed_nvm_get_image,
-	.get_coalesce = &qed_get_coalesce,
 	.set_coalesce = &qed_set_coalesce,
 	.set_led = &qed_set_led,
 	.update_drv_state = &qed_update_drv_state,
diff --git a/drivers/net/ethernet/qlogic/qed/qed_mcp.c b/drivers/net/ethernet/qlogic/qed/qed_mcp.c
index 3eb241657368..376485d99357 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_mcp.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_mcp.c
@@ -1097,6 +1097,31 @@ static void qed_mcp_handle_transceiver_change(struct qed_hwfn *p_hwfn,
 		DP_NOTICE(p_hwfn, "Transceiver is unplugged.\n");
 }
 
+static void qed_mcp_read_eee_config(struct qed_hwfn *p_hwfn,
+				    struct qed_ptt *p_ptt,
+				    struct qed_mcp_link_state *p_link)
+{
+	u32 eee_status, val;
+
+	p_link->eee_adv_caps = 0;
+	p_link->eee_lp_adv_caps = 0;
+	eee_status = qed_rd(p_hwfn,
+			    p_ptt,
+			    p_hwfn->mcp_info->port_addr +
+			    offsetof(struct public_port, eee_status));
+	p_link->eee_active = !!(eee_status & EEE_ACTIVE_BIT);
+	val = (eee_status & EEE_LD_ADV_STATUS_MASK) >> EEE_LD_ADV_STATUS_OFFSET;
+	if (val & EEE_1G_ADV)
+		p_link->eee_adv_caps |= QED_EEE_1G_ADV;
+	if (val & EEE_10G_ADV)
+		p_link->eee_adv_caps |= QED_EEE_10G_ADV;
+	val = (eee_status & EEE_LP_ADV_STATUS_MASK) >> EEE_LP_ADV_STATUS_OFFSET;
+	if (val & EEE_1G_ADV)
+		p_link->eee_lp_adv_caps |= QED_EEE_1G_ADV;
+	if (val & EEE_10G_ADV)
+		p_link->eee_lp_adv_caps |= QED_EEE_10G_ADV;
+}
+
 static void qed_mcp_handle_link_change(struct qed_hwfn *p_hwfn,
 				       struct qed_ptt *p_ptt, bool b_reset)
 {
@@ -1228,6 +1253,9 @@ static void qed_mcp_handle_link_change(struct qed_hwfn *p_hwfn,
 
 	p_link->sfp_tx_fault = !!(status & LINK_STATUS_SFP_TX_FAULT);
 
+	if (p_hwfn->mcp_info->capabilities & FW_MB_PARAM_FEATURE_SUPPORT_EEE)
+		qed_mcp_read_eee_config(p_hwfn, p_ptt, p_link);
+
 	qed_link_update(p_hwfn);
 out:
 	spin_unlock_bh(&p_hwfn->mcp_info->link_lock);
@@ -1251,6 +1279,19 @@ int qed_mcp_set_link(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, bool b_up)
 	phy_cfg.pause |= (params->pause.forced_tx) ? ETH_PAUSE_TX : 0;
 	phy_cfg.adv_speed = params->speed.advertised_speeds;
 	phy_cfg.loopback_mode = params->loopback_mode;
+	if (p_hwfn->mcp_info->capabilities & FW_MB_PARAM_FEATURE_SUPPORT_EEE) {
+		if (params->eee.enable)
+			phy_cfg.eee_cfg |= EEE_CFG_EEE_ENABLED;
+		if (params->eee.tx_lpi_enable)
+			phy_cfg.eee_cfg |= EEE_CFG_TX_LPI;
+		if (params->eee.adv_caps & QED_EEE_1G_ADV)
+			phy_cfg.eee_cfg |= EEE_CFG_ADV_SPEED_1G;
+		if (params->eee.adv_caps & QED_EEE_10G_ADV)
+			phy_cfg.eee_cfg |= EEE_CFG_ADV_SPEED_10G;
+		phy_cfg.eee_cfg |= (params->eee.tx_lpi_timer <<
+				    EEE_TX_TIMER_USEC_OFFSET) &
+				   EEE_TX_TIMER_USEC_MASK;
+	}
 
 	p_hwfn->b_drv_link_init = b_up;
 
@@ -2822,3 +2863,28 @@ void qed_mcp_resc_lock_default_init(struct qed_resc_lock_params *p_lock,
 		p_unlock->resource = resource;
 	}
 }
+
+int qed_mcp_get_capabilities(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
+{
+	u32 mcp_resp;
+	int rc;
+
+	rc = qed_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_GET_MFW_FEATURE_SUPPORT,
+			 0, &mcp_resp, &p_hwfn->mcp_info->capabilities);
+	if (!rc)
+		DP_VERBOSE(p_hwfn, (QED_MSG_SP | NETIF_MSG_PROBE),
+			   "MFW supported features: %08x\n",
+			   p_hwfn->mcp_info->capabilities);
+
+	return rc;
+}
+
+int qed_mcp_set_capabilities(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
+{
+	u32 mcp_resp, mcp_param, features;
+
+	features = DRV_MB_PARAM_FEATURE_SUPPORT_PORT_EEE;
+
+	return qed_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_FEATURE_SUPPORT,
+			   features, &mcp_resp, &mcp_param);
+}
diff --git a/drivers/net/ethernet/qlogic/qed/qed_mcp.h b/drivers/net/ethernet/qlogic/qed/qed_mcp.h
index af03b3651411..c7ec2395d1ce 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_mcp.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_mcp.h
@@ -53,15 +53,25 @@ struct qed_mcp_link_pause_params {
 	bool    forced_tx;
 };
 
+enum qed_mcp_eee_mode {
+	QED_MCP_EEE_DISABLED,
+	QED_MCP_EEE_ENABLED,
+	QED_MCP_EEE_UNSUPPORTED
+};
+
 struct qed_mcp_link_params {
-	struct qed_mcp_link_speed_params	speed;
-	struct qed_mcp_link_pause_params	pause;
-	u32				     loopback_mode;
+	struct qed_mcp_link_speed_params speed;
+	struct qed_mcp_link_pause_params pause;
+	u32 loopback_mode;
+	struct qed_link_eee_params eee;
 };
 
 struct qed_mcp_link_capabilities {
 	u32 speed_capabilities;
 	bool default_speed_autoneg;
+	enum qed_mcp_eee_mode default_eee;
+	u32 eee_lpi_timer;
+	u8 eee_speed_caps;
 };
 
 struct qed_mcp_link_state {
@@ -102,6 +112,9 @@ struct qed_mcp_link_state {
 	u8      partner_adv_pause;
 
 	bool    sfp_tx_fault;
+	bool    eee_active;
+	u8      eee_adv_caps;
+	u8      eee_lp_adv_caps;
 };
 
 struct qed_mcp_function_info {
@@ -546,6 +559,9 @@ struct qed_mcp_info {
 	u8					*mfw_mb_shadow;
 	u16					mfw_mb_length;
 	u32					mcp_hist;
+
+	/* Capabilties negotiated with the MFW */
+	u32					capabilities;
 };
 
 struct qed_mcp_mb_params {
@@ -925,5 +941,20 @@ void qed_mcp_resc_lock_default_init(struct qed_resc_lock_params *p_lock,
 				    struct qed_resc_unlock_params *p_unlock,
 				    enum qed_resc_lock
 				    resource, bool b_is_permanent);
+/**
+ * @brief Learn of supported MFW features; To be done during early init
+ *
+ * @param p_hwfn
+ * @param p_ptt
+ */
+int qed_mcp_get_capabilities(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt);
 
+/**
+ * @brief Inform MFW of set of features supported by driver. Should be done
+ * inside the content of the LOAD_REQ.
+ *
+ * @param p_hwfn
+ * @param p_ptt
+ */
+int qed_mcp_set_capabilities(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt);
 #endif
diff --git a/drivers/net/ethernet/qlogic/qed/qed_sriov.c b/drivers/net/ethernet/qlogic/qed/qed_sriov.c
index 2cfd3bd9a031..3f40b1de7957 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_sriov.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_sriov.c
@@ -3400,6 +3400,157 @@ static void qed_iov_vf_mbx_release(struct qed_hwfn *p_hwfn,
 			     length, status);
 }
 
+static void qed_iov_vf_pf_get_coalesce(struct qed_hwfn *p_hwfn,
+				       struct qed_ptt *p_ptt,
+				       struct qed_vf_info *p_vf)
+{
+	struct qed_iov_vf_mbx *mbx = &p_vf->vf_mbx;
+	struct pfvf_read_coal_resp_tlv *p_resp;
+	struct vfpf_read_coal_req_tlv *req;
+	u8 status = PFVF_STATUS_FAILURE;
+	struct qed_vf_queue *p_queue;
+	struct qed_queue_cid *p_cid;
+	u16 coal = 0, qid, i;
+	bool b_is_rx;
+	int rc = 0;
+
+	mbx->offset = (u8 *)mbx->reply_virt;
+	req = &mbx->req_virt->read_coal_req;
+
+	qid = req->qid;
+	b_is_rx = req->is_rx ? true : false;
+
+	if (b_is_rx) {
+		if (!qed_iov_validate_rxq(p_hwfn, p_vf, qid,
+					  QED_IOV_VALIDATE_Q_ENABLE)) {
+			DP_VERBOSE(p_hwfn, QED_MSG_IOV,
+				   "VF[%d]: Invalid Rx queue_id = %d\n",
+				   p_vf->abs_vf_id, qid);
+			goto send_resp;
+		}
+
+		p_cid = qed_iov_get_vf_rx_queue_cid(&p_vf->vf_queues[qid]);
+		rc = qed_get_rxq_coalesce(p_hwfn, p_ptt, p_cid, &coal);
+		if (rc)
+			goto send_resp;
+	} else {
+		if (!qed_iov_validate_txq(p_hwfn, p_vf, qid,
+					  QED_IOV_VALIDATE_Q_ENABLE)) {
+			DP_VERBOSE(p_hwfn, QED_MSG_IOV,
+				   "VF[%d]: Invalid Tx queue_id = %d\n",
+				   p_vf->abs_vf_id, qid);
+			goto send_resp;
+		}
+		for (i = 0; i < MAX_QUEUES_PER_QZONE; i++) {
+			p_queue = &p_vf->vf_queues[qid];
+			if ((!p_queue->cids[i].p_cid) ||
+			    (!p_queue->cids[i].b_is_tx))
+				continue;
+
+			p_cid = p_queue->cids[i].p_cid;
+
+			rc = qed_get_txq_coalesce(p_hwfn, p_ptt, p_cid, &coal);
+			if (rc)
+				goto send_resp;
+			break;
+		}
+	}
+
+	status = PFVF_STATUS_SUCCESS;
+
+send_resp:
+	p_resp = qed_add_tlv(p_hwfn, &mbx->offset, CHANNEL_TLV_COALESCE_READ,
+			     sizeof(*p_resp));
+	p_resp->coal = coal;
+
+	qed_add_tlv(p_hwfn, &mbx->offset, CHANNEL_TLV_LIST_END,
+		    sizeof(struct channel_list_end_tlv));
+
+	qed_iov_send_response(p_hwfn, p_ptt, p_vf, sizeof(*p_resp), status);
+}
+
+static void qed_iov_vf_pf_set_coalesce(struct qed_hwfn *p_hwfn,
+				       struct qed_ptt *p_ptt,
+				       struct qed_vf_info *vf)
+{
+	struct qed_iov_vf_mbx *mbx = &vf->vf_mbx;
+	struct vfpf_update_coalesce *req;
+	u8 status = PFVF_STATUS_FAILURE;
+	struct qed_queue_cid *p_cid;
+	u16 rx_coal, tx_coal;
+	int rc = 0, i;
+	u16 qid;
+
+	req = &mbx->req_virt->update_coalesce;
+
+	rx_coal = req->rx_coal;
+	tx_coal = req->tx_coal;
+	qid = req->qid;
+
+	if (!qed_iov_validate_rxq(p_hwfn, vf, qid,
+				  QED_IOV_VALIDATE_Q_ENABLE) && rx_coal) {
+		DP_VERBOSE(p_hwfn, QED_MSG_IOV,
+			   "VF[%d]: Invalid Rx queue_id = %d\n",
+			   vf->abs_vf_id, qid);
+		goto out;
+	}
+
+	if (!qed_iov_validate_txq(p_hwfn, vf, qid,
+				  QED_IOV_VALIDATE_Q_ENABLE) && tx_coal) {
+		DP_VERBOSE(p_hwfn, QED_MSG_IOV,
+			   "VF[%d]: Invalid Tx queue_id = %d\n",
+			   vf->abs_vf_id, qid);
+		goto out;
+	}
+
+	DP_VERBOSE(p_hwfn,
+		   QED_MSG_IOV,
+		   "VF[%d]: Setting coalesce for VF rx_coal = %d, tx_coal = %d at queue = %d\n",
+		   vf->abs_vf_id, rx_coal, tx_coal, qid);
+
+	if (rx_coal) {
+		p_cid = qed_iov_get_vf_rx_queue_cid(&vf->vf_queues[qid]);
+
+		rc = qed_set_rxq_coalesce(p_hwfn, p_ptt, rx_coal, p_cid);
+		if (rc) {
+			DP_VERBOSE(p_hwfn,
+				   QED_MSG_IOV,
+				   "VF[%d]: Unable to set rx queue = %d coalesce\n",
+				   vf->abs_vf_id, vf->vf_queues[qid].fw_rx_qid);
+			goto out;
+		}
+		vf->rx_coal = rx_coal;
+	}
+
+	if (tx_coal) {
+		struct qed_vf_queue *p_queue = &vf->vf_queues[qid];
+
+		for (i = 0; i < MAX_QUEUES_PER_QZONE; i++) {
+			if (!p_queue->cids[i].p_cid)
+				continue;
+
+			if (!p_queue->cids[i].b_is_tx)
+				continue;
+
+			rc = qed_set_txq_coalesce(p_hwfn, p_ptt, tx_coal,
+						  p_queue->cids[i].p_cid);
+
+			if (rc) {
+				DP_VERBOSE(p_hwfn,
+					   QED_MSG_IOV,
+					   "VF[%d]: Unable to set tx queue coalesce\n",
+					   vf->abs_vf_id);
+				goto out;
+			}
+		}
+		vf->tx_coal = tx_coal;
+	}
+
+	status = PFVF_STATUS_SUCCESS;
+out:
+	qed_iov_prepare_resp(p_hwfn, p_ptt, vf, CHANNEL_TLV_COALESCE_UPDATE,
+			     sizeof(struct pfvf_def_resp_tlv), status);
+}
 static int
 qed_iov_vf_flr_poll_dorq(struct qed_hwfn *p_hwfn,
 			 struct qed_vf_info *p_vf, struct qed_ptt *p_ptt)
@@ -3725,6 +3876,12 @@ static void qed_iov_process_mbx_req(struct qed_hwfn *p_hwfn,
 		case CHANNEL_TLV_UPDATE_TUNN_PARAM:
 			qed_iov_vf_mbx_update_tunn_param(p_hwfn, p_ptt, p_vf);
 			break;
+		case CHANNEL_TLV_COALESCE_UPDATE:
+			qed_iov_vf_pf_set_coalesce(p_hwfn, p_ptt, p_vf);
+			break;
+		case CHANNEL_TLV_COALESCE_READ:
+			qed_iov_vf_pf_get_coalesce(p_hwfn, p_ptt, p_vf);
+			break;
 		}
 	} else if (qed_iov_tlv_supported(mbx->first_tlv.tl.type)) {
 		DP_VERBOSE(p_hwfn, QED_MSG_IOV,
diff --git a/drivers/net/ethernet/qlogic/qed/qed_sriov.h b/drivers/net/ethernet/qlogic/qed/qed_sriov.h
index c2e44bce398c..3955929ba892 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_sriov.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_sriov.h
@@ -217,6 +217,9 @@ struct qed_vf_info {
 	u8 num_rxqs;
 	u8 num_txqs;
 
+	u16 rx_coal;
+	u16 tx_coal;
+
 	u8 num_sbs;
 
 	u8 num_mac_filters;
diff --git a/drivers/net/ethernet/qlogic/qed/qed_vf.c b/drivers/net/ethernet/qlogic/qed/qed_vf.c
index 1926d1ed439f..91b5e9f02a62 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_vf.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_vf.c
@@ -1343,6 +1343,81 @@ exit:
 	return rc;
 }
 
+int qed_vf_pf_get_coalesce(struct qed_hwfn *p_hwfn,
+			   u16 *p_coal, struct qed_queue_cid *p_cid)
+{
+	struct qed_vf_iov *p_iov = p_hwfn->vf_iov_info;
+	struct pfvf_read_coal_resp_tlv *resp;
+	struct vfpf_read_coal_req_tlv *req;
+	int rc;
+
+	/* clear mailbox and prep header tlv */
+	req = qed_vf_pf_prep(p_hwfn, CHANNEL_TLV_COALESCE_READ, sizeof(*req));
+	req->qid = p_cid->rel.queue_id;
+	req->is_rx = p_cid->b_is_rx ? 1 : 0;
+
+	qed_add_tlv(p_hwfn, &p_iov->offset, CHANNEL_TLV_LIST_END,
+		    sizeof(struct channel_list_end_tlv));
+	resp = &p_iov->pf2vf_reply->read_coal_resp;
+
+	rc = qed_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp));
+	if (rc)
+		goto exit;
+
+	if (resp->hdr.status != PFVF_STATUS_SUCCESS)
+		goto exit;
+
+	*p_coal = resp->coal;
+exit:
+	qed_vf_pf_req_end(p_hwfn, rc);
+
+	return rc;
+}
+
+int
+qed_vf_pf_set_coalesce(struct qed_hwfn *p_hwfn,
+		       u16 rx_coal, u16 tx_coal, struct qed_queue_cid *p_cid)
+{
+	struct qed_vf_iov *p_iov = p_hwfn->vf_iov_info;
+	struct vfpf_update_coalesce *req;
+	struct pfvf_def_resp_tlv *resp;
+	int rc;
+
+	/* clear mailbox and prep header tlv */
+	req = qed_vf_pf_prep(p_hwfn, CHANNEL_TLV_COALESCE_UPDATE, sizeof(*req));
+
+	req->rx_coal = rx_coal;
+	req->tx_coal = tx_coal;
+	req->qid = p_cid->rel.queue_id;
+
+	DP_VERBOSE(p_hwfn,
+		   QED_MSG_IOV,
+		   "Setting coalesce rx_coal = %d, tx_coal = %d at queue = %d\n",
+		   rx_coal, tx_coal, req->qid);
+
+	/* add list termination tlv */
+	qed_add_tlv(p_hwfn, &p_iov->offset, CHANNEL_TLV_LIST_END,
+		    sizeof(struct channel_list_end_tlv));
+
+	resp = &p_iov->pf2vf_reply->default_resp;
+	rc = qed_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp));
+	if (rc)
+		goto exit;
+
+	if (resp->hdr.status != PFVF_STATUS_SUCCESS)
+		goto exit;
+
+	if (rx_coal)
+		p_hwfn->cdev->rx_coalesce_usecs = rx_coal;
+
+	if (tx_coal)
+		p_hwfn->cdev->tx_coalesce_usecs = tx_coal;
+
+exit:
+	qed_vf_pf_req_end(p_hwfn, rc);
+	return rc;
+}
+
 u16 qed_vf_get_igu_sb_id(struct qed_hwfn *p_hwfn, u16 sb_id)
 {
 	struct qed_vf_iov *p_iov = p_hwfn->vf_iov_info;
diff --git a/drivers/net/ethernet/qlogic/qed/qed_vf.h b/drivers/net/ethernet/qlogic/qed/qed_vf.h
index 34d9b882a780..97d44dfb38ca 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_vf.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_vf.h
@@ -497,6 +497,27 @@ struct tlv_buffer_size {
 	u8 tlv_buffer[TLV_BUFFER_SIZE];
 };
 
+struct vfpf_update_coalesce {
+	struct vfpf_first_tlv first_tlv;
+	u16 rx_coal;
+	u16 tx_coal;
+	u16 qid;
+	u8 padding[2];
+};
+
+struct vfpf_read_coal_req_tlv {
+	struct vfpf_first_tlv first_tlv;
+	u16 qid;
+	u8 is_rx;
+	u8 padding[5];
+};
+
+struct pfvf_read_coal_resp_tlv {
+	struct pfvf_tlv hdr;
+	u16 coal;
+	u8 padding[6];
+};
+
 union vfpf_tlvs {
 	struct vfpf_first_tlv first_tlv;
 	struct vfpf_acquire_tlv acquire;
@@ -509,7 +530,8 @@ union vfpf_tlvs {
 	struct vfpf_vport_update_tlv vport_update;
 	struct vfpf_ucast_filter_tlv ucast_filter;
 	struct vfpf_update_tunn_param_tlv tunn_param_update;
-	struct channel_list_end_tlv list_end;
+	struct vfpf_update_coalesce update_coalesce;
+	struct vfpf_read_coal_req_tlv read_coal_req;
 	struct tlv_buffer_size tlv_buf_size;
 };
 
@@ -519,6 +541,7 @@ union pfvf_tlvs {
 	struct tlv_buffer_size tlv_buf_size;
 	struct pfvf_start_queue_resp_tlv queue_start;
 	struct pfvf_update_tunn_param_tlv tunn_param_resp;
+	struct pfvf_read_coal_resp_tlv read_coal_resp;
 };
 
 enum qed_bulletin_bit {
@@ -624,8 +647,9 @@ enum {
 	CHANNEL_TLV_VPORT_UPDATE_ACCEPT_ANY_VLAN,
 	CHANNEL_TLV_VPORT_UPDATE_SGE_TPA,
 	CHANNEL_TLV_UPDATE_TUNN_PARAM,
-	CHANNEL_TLV_RESERVED,
+	CHANNEL_TLV_COALESCE_UPDATE,
 	CHANNEL_TLV_QID,
+	CHANNEL_TLV_COALESCE_READ,
 	CHANNEL_TLV_MAX,
 
 	/* Required for iterating over vport-update tlvs.
@@ -677,6 +701,31 @@ struct qed_vf_iov {
 	bool b_doorbell_bar;
 };
 
+/**
+ * @brief VF - Set Rx/Tx coalesce per VF's relative queue.
+ *             Coalesce value '0' will omit the configuration.
+ *
+ * @param p_hwfn
+ * @param rx_coal - coalesce value in micro second for rx queue
+ * @param tx_coal - coalesce value in micro second for tx queue
+ * @param p_cid   - queue cid
+ *
+ **/
+int qed_vf_pf_set_coalesce(struct qed_hwfn *p_hwfn,
+			   u16 rx_coal,
+			   u16 tx_coal, struct qed_queue_cid *p_cid);
+
+/**
+ * @brief VF - Get coalesce per VF's relative queue.
+ *
+ * @param p_hwfn
+ * @param p_coal - coalesce value in micro second for VF queues.
+ * @param p_cid  - queue cid
+ *
+ **/
+int qed_vf_pf_get_coalesce(struct qed_hwfn *p_hwfn,
+			   u16 *p_coal, struct qed_queue_cid *p_cid);
+
 #ifdef CONFIG_QED_SRIOV
 /**
  * @brief Read the VF bulletin and act on it if needed
diff --git a/drivers/net/ethernet/qlogic/qede/qede.h b/drivers/net/ethernet/qlogic/qede/qede.h
index 4dfb238221f9..adb700512baa 100644
--- a/drivers/net/ethernet/qlogic/qede/qede.h
+++ b/drivers/net/ethernet/qlogic/qede/qede.h
@@ -160,6 +160,8 @@ struct qede_rdma_dev {
 
 struct qede_ptp;
 
+#define QEDE_RFS_MAX_FLTR	256
+
 struct qede_dev {
 	struct qed_dev			*cdev;
 	struct net_device		*ndev;
@@ -241,9 +243,7 @@ struct qede_dev {
 	u16				vxlan_dst_port;
 	u16				geneve_dst_port;
 
-#ifdef CONFIG_RFS_ACCEL
 	struct qede_arfs		*arfs;
-#endif
 	bool				wol_enabled;
 
 	struct qede_rdma_dev		rdma_info;
@@ -447,16 +447,21 @@ struct qede_fastpath {
 #ifdef CONFIG_RFS_ACCEL
 int qede_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
 		       u16 rxq_index, u32 flow_id);
+#define QEDE_SP_ARFS_CONFIG	4
+#define QEDE_SP_TASK_POLL_DELAY	(5 * HZ)
+#endif
+
 void qede_process_arfs_filters(struct qede_dev *edev, bool free_fltr);
 void qede_poll_for_freeing_arfs_filters(struct qede_dev *edev);
 void qede_arfs_filter_op(void *dev, void *filter, u8 fw_rc);
 void qede_free_arfs(struct qede_dev *edev);
 int qede_alloc_arfs(struct qede_dev *edev);
-
-#define QEDE_SP_ARFS_CONFIG	4
-#define QEDE_SP_TASK_POLL_DELAY	(5 * HZ)
-#define QEDE_RFS_MAX_FLTR	256
-#endif
+int qede_add_cls_rule(struct qede_dev *edev, struct ethtool_rxnfc *info);
+int qede_del_cls_rule(struct qede_dev *edev, struct ethtool_rxnfc *info);
+int qede_get_cls_rule_entry(struct qede_dev *edev, struct ethtool_rxnfc *cmd);
+int qede_get_cls_rule_all(struct qede_dev *edev, struct ethtool_rxnfc *info,
+			  u32 *rule_locs);
+int qede_get_arfs_filter_count(struct qede_dev *edev);
 
 struct qede_reload_args {
 	void (*func)(struct qede_dev *edev, struct qede_reload_args *args);
diff --git a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c
index 6a03d3e66cff..dae741270022 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c
@@ -702,24 +702,62 @@ static u32 qede_get_link(struct net_device *dev)
 static int qede_get_coalesce(struct net_device *dev,
 			     struct ethtool_coalesce *coal)
 {
+	void *rx_handle = NULL, *tx_handle = NULL;
 	struct qede_dev *edev = netdev_priv(dev);
-	u16 rxc, txc;
+	u16 rx_coal, tx_coal, i, rc = 0;
+	struct qede_fastpath *fp;
+
+	rx_coal = QED_DEFAULT_RX_USECS;
+	tx_coal = QED_DEFAULT_TX_USECS;
 
 	memset(coal, 0, sizeof(struct ethtool_coalesce));
-	edev->ops->common->get_coalesce(edev->cdev, &rxc, &txc);
 
-	coal->rx_coalesce_usecs = rxc;
-	coal->tx_coalesce_usecs = txc;
+	__qede_lock(edev);
+	if (edev->state == QEDE_STATE_OPEN) {
+		for_each_queue(i) {
+			fp = &edev->fp_array[i];
 
-	return 0;
+			if (fp->type & QEDE_FASTPATH_RX) {
+				rx_handle = fp->rxq->handle;
+				break;
+			}
+		}
+
+		rc = edev->ops->get_coalesce(edev->cdev, &rx_coal, rx_handle);
+		if (rc) {
+			DP_INFO(edev, "Read Rx coalesce error\n");
+			goto out;
+		}
+
+		for_each_queue(i) {
+			fp = &edev->fp_array[i];
+			if (fp->type & QEDE_FASTPATH_TX) {
+				tx_handle = fp->txq->handle;
+				break;
+			}
+		}
+
+		rc = edev->ops->get_coalesce(edev->cdev, &tx_coal, tx_handle);
+		if (rc)
+			DP_INFO(edev, "Read Tx coalesce error\n");
+	}
+
+out:
+	__qede_unlock(edev);
+
+	coal->rx_coalesce_usecs = rx_coal;
+	coal->tx_coalesce_usecs = tx_coal;
+
+	return rc;
 }
 
 static int qede_set_coalesce(struct net_device *dev,
 			     struct ethtool_coalesce *coal)
 {
 	struct qede_dev *edev = netdev_priv(dev);
+	struct qede_fastpath *fp;
 	int i, rc = 0;
-	u16 rxc, txc, sb_id;
+	u16 rxc, txc;
 
 	if (!netif_running(dev)) {
 		DP_INFO(edev, "Interface is down\n");
@@ -730,21 +768,36 @@ static int qede_set_coalesce(struct net_device *dev,
 	    coal->tx_coalesce_usecs > QED_COALESCE_MAX) {
 		DP_INFO(edev,
 			"Can't support requested %s coalesce value [max supported value %d]\n",
-			coal->rx_coalesce_usecs > QED_COALESCE_MAX ? "rx"
-								   : "tx",
-			QED_COALESCE_MAX);
+			coal->rx_coalesce_usecs > QED_COALESCE_MAX ? "rx" :
+			"tx", QED_COALESCE_MAX);
 		return -EINVAL;
 	}
 
 	rxc = (u16)coal->rx_coalesce_usecs;
 	txc = (u16)coal->tx_coalesce_usecs;
 	for_each_queue(i) {
-		sb_id = edev->fp_array[i].sb_info->igu_sb_id;
-		rc = edev->ops->common->set_coalesce(edev->cdev, rxc, txc,
-						     (u16)i, sb_id);
-		if (rc) {
-			DP_INFO(edev, "Set coalesce error, rc = %d\n", rc);
-			return rc;
+		fp = &edev->fp_array[i];
+
+		if (edev->fp_array[i].type & QEDE_FASTPATH_RX) {
+			rc = edev->ops->common->set_coalesce(edev->cdev,
+							     rxc, 0,
+							     fp->rxq->handle);
+			if (rc) {
+				DP_INFO(edev,
+					"Set RX coalesce error, rc = %d\n", rc);
+				return rc;
+			}
+		}
+
+		if (edev->fp_array[i].type & QEDE_FASTPATH_TX) {
+			rc = edev->ops->common->set_coalesce(edev->cdev,
+							     0, txc,
+							     fp->txq->handle);
+			if (rc) {
+				DP_INFO(edev,
+					"Set TX coalesce error, rc = %d\n", rc);
+				return rc;
+			}
 		}
 	}
 
@@ -1045,20 +1098,34 @@ static int qede_get_rss_flags(struct qede_dev *edev, struct ethtool_rxnfc *info)
 }
 
 static int qede_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
-			  u32 *rules __always_unused)
+			  u32 *rule_locs)
 {
 	struct qede_dev *edev = netdev_priv(dev);
+	int rc = 0;
 
 	switch (info->cmd) {
 	case ETHTOOL_GRXRINGS:
 		info->data = QEDE_RSS_COUNT(edev);
-		return 0;
+		break;
 	case ETHTOOL_GRXFH:
-		return qede_get_rss_flags(edev, info);
+		rc = qede_get_rss_flags(edev, info);
+		break;
+	case ETHTOOL_GRXCLSRLCNT:
+		info->rule_cnt = qede_get_arfs_filter_count(edev);
+		info->data = QEDE_RFS_MAX_FLTR;
+		break;
+	case ETHTOOL_GRXCLSRULE:
+		rc = qede_get_cls_rule_entry(edev, info);
+		break;
+	case ETHTOOL_GRXCLSRLALL:
+		rc = qede_get_cls_rule_all(edev, info, rule_locs);
+		break;
 	default:
 		DP_ERR(edev, "Command parameters not supported\n");
-		return -EOPNOTSUPP;
+		rc = -EOPNOTSUPP;
 	}
+
+	return rc;
 }
 
 static int qede_set_rss_flags(struct qede_dev *edev, struct ethtool_rxnfc *info)
@@ -1168,14 +1235,24 @@ static int qede_set_rss_flags(struct qede_dev *edev, struct ethtool_rxnfc *info)
 static int qede_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info)
 {
 	struct qede_dev *edev = netdev_priv(dev);
+	int rc;
 
 	switch (info->cmd) {
 	case ETHTOOL_SRXFH:
-		return qede_set_rss_flags(edev, info);
+		rc = qede_set_rss_flags(edev, info);
+		break;
+	case ETHTOOL_SRXCLSRLINS:
+		rc = qede_add_cls_rule(edev, info);
+		break;
+	case ETHTOOL_SRXCLSRLDEL:
+		rc = qede_del_cls_rule(edev, info);
+		break;
 	default:
 		DP_INFO(edev, "Command parameters not supported\n");
-		return -EOPNOTSUPP;
+		rc = -EOPNOTSUPP;
 	}
+
+	return rc;
 }
 
 static u32 qede_get_rxfh_indir_size(struct net_device *dev)
@@ -1607,6 +1684,87 @@ static int qede_get_tunable(struct net_device *dev,
 	return 0;
 }
 
+static int qede_get_eee(struct net_device *dev, struct ethtool_eee *edata)
+{
+	struct qede_dev *edev = netdev_priv(dev);
+	struct qed_link_output current_link;
+
+	memset(&current_link, 0, sizeof(current_link));
+	edev->ops->common->get_link(edev->cdev, &current_link);
+
+	if (!current_link.eee_supported) {
+		DP_INFO(edev, "EEE is not supported\n");
+		return -EOPNOTSUPP;
+	}
+
+	if (current_link.eee.adv_caps & QED_EEE_1G_ADV)
+		edata->advertised = ADVERTISED_1000baseT_Full;
+	if (current_link.eee.adv_caps & QED_EEE_10G_ADV)
+		edata->advertised |= ADVERTISED_10000baseT_Full;
+	if (current_link.sup_caps & QED_EEE_1G_ADV)
+		edata->supported = ADVERTISED_1000baseT_Full;
+	if (current_link.sup_caps & QED_EEE_10G_ADV)
+		edata->supported |= ADVERTISED_10000baseT_Full;
+	if (current_link.eee.lp_adv_caps & QED_EEE_1G_ADV)
+		edata->lp_advertised = ADVERTISED_1000baseT_Full;
+	if (current_link.eee.lp_adv_caps & QED_EEE_10G_ADV)
+		edata->lp_advertised |= ADVERTISED_10000baseT_Full;
+
+	edata->tx_lpi_timer = current_link.eee.tx_lpi_timer;
+	edata->eee_enabled = current_link.eee.enable;
+	edata->tx_lpi_enabled = current_link.eee.tx_lpi_enable;
+	edata->eee_active = current_link.eee_active;
+
+	return 0;
+}
+
+static int qede_set_eee(struct net_device *dev, struct ethtool_eee *edata)
+{
+	struct qede_dev *edev = netdev_priv(dev);
+	struct qed_link_output current_link;
+	struct qed_link_params params;
+
+	if (!edev->ops->common->can_link_change(edev->cdev)) {
+		DP_INFO(edev, "Link settings are not allowed to be changed\n");
+		return -EOPNOTSUPP;
+	}
+
+	memset(&current_link, 0, sizeof(current_link));
+	edev->ops->common->get_link(edev->cdev, &current_link);
+
+	if (!current_link.eee_supported) {
+		DP_INFO(edev, "EEE is not supported\n");
+		return -EOPNOTSUPP;
+	}
+
+	memset(&params, 0, sizeof(params));
+	params.override_flags |= QED_LINK_OVERRIDE_EEE_CONFIG;
+
+	if (!(edata->advertised & (ADVERTISED_1000baseT_Full |
+				   ADVERTISED_10000baseT_Full)) ||
+	    ((edata->advertised & (ADVERTISED_1000baseT_Full |
+				   ADVERTISED_10000baseT_Full)) !=
+	     edata->advertised)) {
+		DP_VERBOSE(edev, QED_MSG_DEBUG,
+			   "Invalid advertised capabilities %d\n",
+			   edata->advertised);
+		return -EINVAL;
+	}
+
+	if (edata->advertised & ADVERTISED_1000baseT_Full)
+		params.eee.adv_caps = QED_EEE_1G_ADV;
+	if (edata->advertised & ADVERTISED_10000baseT_Full)
+		params.eee.adv_caps |= QED_EEE_10G_ADV;
+	params.eee.enable = edata->eee_enabled;
+	params.eee.tx_lpi_enable = edata->tx_lpi_enabled;
+	params.eee.tx_lpi_timer = edata->tx_lpi_timer;
+
+	params.link_up = true;
+	edev->ops->common->set_link(edev->cdev, &params);
+
+	return 0;
+}
+
 static const struct ethtool_ops qede_ethtool_ops = {
 	.get_link_ksettings = qede_get_link_ksettings,
 	.set_link_ksettings = qede_set_link_ksettings,
@@ -1640,6 +1798,9 @@ static const struct ethtool_ops qede_ethtool_ops = {
 	.get_channels = qede_get_channels,
 	.set_channels = qede_set_channels,
 	.self_test = qede_self_test,
+	.get_eee = qede_get_eee,
+	.set_eee = qede_set_eee,
+
 	.get_tunable = qede_get_tunable,
 	.set_tunable = qede_set_tunable,
 };
@@ -1650,6 +1811,8 @@ static const struct ethtool_ops qede_vf_ethtool_ops = {
 	.get_msglevel = qede_get_msglevel,
 	.set_msglevel = qede_set_msglevel,
 	.get_link = qede_get_link,
+	.get_coalesce = qede_get_coalesce,
+	.set_coalesce = qede_set_coalesce,
 	.get_ringparam = qede_get_ringparam,
 	.set_ringparam = qede_set_ringparam,
 	.get_strings = qede_get_strings,
diff --git a/drivers/net/ethernet/qlogic/qede/qede_filter.c b/drivers/net/ethernet/qlogic/qede/qede_filter.c
index f939db5bac5f..f79e36e4060a 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_filter.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_filter.c
@@ -38,7 +38,6 @@
 #include <linux/qed/qed_if.h>
 #include "qede.h"
 
-#ifdef CONFIG_RFS_ACCEL
 struct qede_arfs_tuple {
 	union {
 		__be32 src_ipv4;
@@ -76,10 +75,12 @@ struct qede_arfs_fltr_node {
 	u16 next_rxq_id;
 	bool filter_op;
 	bool used;
+	u8 fw_rc;
 	struct hlist_node node;
 };
 
 struct qede_arfs {
+#define QEDE_ARFS_BUCKET_HEAD(edev, idx) (&(edev)->arfs->arfs_hl_head[idx])
 #define QEDE_ARFS_POLL_COUNT	100
 #define QEDE_RFS_FLW_BITSHIFT	(4)
 #define QEDE_RFS_FLW_MASK	((1 << QEDE_RFS_FLW_BITSHIFT) - 1)
@@ -121,11 +122,56 @@ qede_free_arfs_filter(struct qede_dev *edev,  struct qede_arfs_fltr_node *fltr)
 	kfree(fltr);
 }
 
+static int
+qede_enqueue_fltr_and_config_searcher(struct qede_dev *edev,
+				      struct qede_arfs_fltr_node *fltr,
+				      u16 bucket_idx)
+{
+	fltr->mapping = dma_map_single(&edev->pdev->dev, fltr->data,
+				       fltr->buf_len, DMA_TO_DEVICE);
+	if (dma_mapping_error(&edev->pdev->dev, fltr->mapping)) {
+		DP_NOTICE(edev, "Failed to map DMA memory for rule\n");
+		qede_free_arfs_filter(edev, fltr);
+		return -ENOMEM;
+	}
+
+	INIT_HLIST_NODE(&fltr->node);
+	hlist_add_head(&fltr->node,
+		       QEDE_ARFS_BUCKET_HEAD(edev, bucket_idx));
+	edev->arfs->filter_count++;
+
+	if (edev->arfs->filter_count == 1 && !edev->arfs->enable) {
+		edev->ops->configure_arfs_searcher(edev->cdev, true);
+		edev->arfs->enable = true;
+	}
+
+	return 0;
+}
+
+static void
+qede_dequeue_fltr_and_config_searcher(struct qede_dev *edev,
+				      struct qede_arfs_fltr_node *fltr)
+{
+	hlist_del(&fltr->node);
+	dma_unmap_single(&edev->pdev->dev, fltr->mapping,
+			 fltr->buf_len, DMA_TO_DEVICE);
+
+	qede_free_arfs_filter(edev, fltr);
+	edev->arfs->filter_count--;
+
+	if (!edev->arfs->filter_count && edev->arfs->enable) {
+		edev->arfs->enable = false;
+		edev->ops->configure_arfs_searcher(edev->cdev, false);
+	}
+}
+
 void qede_arfs_filter_op(void *dev, void *filter, u8 fw_rc)
 {
 	struct qede_arfs_fltr_node *fltr = filter;
 	struct qede_dev *edev = dev;
 
+	fltr->fw_rc = fw_rc;
+
 	if (fw_rc) {
 		DP_NOTICE(edev,
 			  "Failed arfs filter configuration fw_rc=%d, flow_id=%d, sw_id=%d, src_port=%d, dst_port=%d, rxq=%d\n",
@@ -185,18 +231,17 @@ void qede_process_arfs_filters(struct qede_dev *edev, bool free_fltr)
 
 			if ((!test_bit(QEDE_FLTR_VALID, &fltr->state) &&
 			     !fltr->used) || free_fltr) {
-				hlist_del(&fltr->node);
-				dma_unmap_single(&edev->pdev->dev,
-						 fltr->mapping,
-						 fltr->buf_len, DMA_TO_DEVICE);
-				qede_free_arfs_filter(edev, fltr);
-				edev->arfs->filter_count--;
+				qede_dequeue_fltr_and_config_searcher(edev,
+								      fltr);
 			} else {
-				if ((rps_may_expire_flow(edev->ndev,
-							 fltr->rxq_id,
-							 fltr->flow_id,
-							 fltr->sw_id) || del) &&
-							 !free_fltr)
+				bool flow_exp = false;
+#ifdef CONFIG_RFS_ACCEL
+				flow_exp = rps_may_expire_flow(edev->ndev,
+							       fltr->rxq_id,
+							       fltr->flow_id,
+							       fltr->sw_id);
+#endif
+				if ((flow_exp || del) && !free_fltr)
 					qede_configure_arfs_fltr(edev, fltr,
 								 fltr->rxq_id,
 								 false);
@@ -213,10 +258,12 @@ void qede_process_arfs_filters(struct qede_dev *edev, bool free_fltr)
 			edev->arfs->enable = false;
 			edev->ops->configure_arfs_searcher(edev->cdev, false);
 		}
+#ifdef CONFIG_RFS_ACCEL
 	} else {
 		set_bit(QEDE_SP_ARFS_CONFIG, &edev->sp_flags);
 		schedule_delayed_work(&edev->sp_task,
 				      QEDE_SP_TASK_POLL_DELAY);
+#endif
 	}
 
 	spin_unlock_bh(&edev->arfs->arfs_list_lock);
@@ -258,25 +305,26 @@ int qede_alloc_arfs(struct qede_dev *edev)
 	spin_lock_init(&edev->arfs->arfs_list_lock);
 
 	for (i = 0; i <= QEDE_RFS_FLW_MASK; i++)
-		INIT_HLIST_HEAD(&edev->arfs->arfs_hl_head[i]);
+		INIT_HLIST_HEAD(QEDE_ARFS_BUCKET_HEAD(edev, i));
 
-	edev->ndev->rx_cpu_rmap = alloc_irq_cpu_rmap(QEDE_RSS_COUNT(edev));
-	if (!edev->ndev->rx_cpu_rmap) {
+	edev->arfs->arfs_fltr_bmap = vzalloc(BITS_TO_LONGS(QEDE_RFS_MAX_FLTR) *
+					     sizeof(long));
+	if (!edev->arfs->arfs_fltr_bmap) {
 		vfree(edev->arfs);
 		edev->arfs = NULL;
 		return -ENOMEM;
 	}
 
-	edev->arfs->arfs_fltr_bmap = vzalloc(BITS_TO_LONGS(QEDE_RFS_MAX_FLTR) *
-					     sizeof(long));
-	if (!edev->arfs->arfs_fltr_bmap) {
-		free_irq_cpu_rmap(edev->ndev->rx_cpu_rmap);
-		edev->ndev->rx_cpu_rmap = NULL;
+#ifdef CONFIG_RFS_ACCEL
+	edev->ndev->rx_cpu_rmap = alloc_irq_cpu_rmap(QEDE_RSS_COUNT(edev));
+	if (!edev->ndev->rx_cpu_rmap) {
+		vfree(edev->arfs->arfs_fltr_bmap);
+		edev->arfs->arfs_fltr_bmap = NULL;
 		vfree(edev->arfs);
 		edev->arfs = NULL;
 		return -ENOMEM;
 	}
-
+#endif
 	return 0;
 }
 
@@ -285,16 +333,19 @@ void qede_free_arfs(struct qede_dev *edev)
 	if (!edev->arfs)
 		return;
 
+#ifdef CONFIG_RFS_ACCEL
 	if (edev->ndev->rx_cpu_rmap)
 		free_irq_cpu_rmap(edev->ndev->rx_cpu_rmap);
 
 	edev->ndev->rx_cpu_rmap = NULL;
+#endif
 	vfree(edev->arfs->arfs_fltr_bmap);
 	edev->arfs->arfs_fltr_bmap = NULL;
 	vfree(edev->arfs);
 	edev->arfs = NULL;
 }
 
+#ifdef CONFIG_RFS_ACCEL
 static bool qede_compare_ip_addr(struct qede_arfs_fltr_node *tpos,
 				 const struct sk_buff *skb)
 {
@@ -394,9 +445,8 @@ int qede_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
 
 	spin_lock_bh(&edev->arfs->arfs_list_lock);
 
-	n = qede_arfs_htbl_key_search(&edev->arfs->arfs_hl_head[tbl_idx],
+	n = qede_arfs_htbl_key_search(QEDE_ARFS_BUCKET_HEAD(edev, tbl_idx),
 				      skb, ports[0], ports[1], ip_proto);
-
 	if (n) {
 		/* Filter match */
 		n->next_rxq_id = rxq_index;
@@ -448,23 +498,9 @@ int qede_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
 	n->tuple.ip_proto = ip_proto;
 	memcpy(n->data + ETH_HLEN, skb->data, skb_headlen(skb));
 
-	n->mapping = dma_map_single(&edev->pdev->dev, n->data,
-				    n->buf_len, DMA_TO_DEVICE);
-	if (dma_mapping_error(&edev->pdev->dev, n->mapping)) {
-		DP_NOTICE(edev, "Failed to map DMA memory for arfs\n");
-		qede_free_arfs_filter(edev, n);
-		rc = -ENOMEM;
+	rc = qede_enqueue_fltr_and_config_searcher(edev, n, tbl_idx);
+	if (rc)
 		goto ret_unlock;
-	}
-
-	INIT_HLIST_NODE(&n->node);
-	hlist_add_head(&n->node, &edev->arfs->arfs_hl_head[tbl_idx]);
-	edev->arfs->filter_count++;
-
-	if (edev->arfs->filter_count == 1 && !edev->arfs->enable) {
-		edev->ops->configure_arfs_searcher(edev->cdev, true);
-		edev->arfs->enable = true;
-	}
 
 	qede_configure_arfs_fltr(edev, n, n->rxq_id, true);
 
@@ -472,6 +508,7 @@ int qede_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
 
 	set_bit(QEDE_SP_ARFS_CONFIG, &edev->sp_flags);
 	schedule_delayed_work(&edev->sp_task, 0);
+
 	return n->sw_id;
 
 ret_unlock:
@@ -1263,3 +1300,371 @@ void qede_config_rx_mode(struct net_device *ndev)
 out:
 	kfree(uc_macs);
 }
+
+static struct qede_arfs_fltr_node *
+qede_get_arfs_fltr_by_loc(struct hlist_head *head, u32 location)
+{
+	struct qede_arfs_fltr_node *fltr;
+
+	hlist_for_each_entry(fltr, head, node)
+		if (location == fltr->sw_id)
+			return fltr;
+
+	return NULL;
+}
+
+static bool
+qede_compare_user_flow_ips(struct qede_arfs_fltr_node *tpos,
+			   struct ethtool_rx_flow_spec *fsp,
+			   __be16 proto)
+{
+	if (proto == htons(ETH_P_IP)) {
+		struct ethtool_tcpip4_spec *ip;
+
+		ip = &fsp->h_u.tcp_ip4_spec;
+
+		if (tpos->tuple.src_ipv4 == ip->ip4src &&
+		    tpos->tuple.dst_ipv4 == ip->ip4dst)
+			return true;
+		else
+			return false;
+	} else {
+		struct ethtool_tcpip6_spec *ip6;
+		struct in6_addr *src;
+
+		ip6 = &fsp->h_u.tcp_ip6_spec;
+		src = &tpos->tuple.src_ipv6;
+
+		if (!memcmp(src, &ip6->ip6src, sizeof(struct in6_addr)) &&
+		    !memcmp(&tpos->tuple.dst_ipv6, &ip6->ip6dst,
+			    sizeof(struct in6_addr)))
+			return true;
+		else
+			return false;
+	}
+	return false;
+}
+
+int qede_get_cls_rule_all(struct qede_dev *edev, struct ethtool_rxnfc *info,
+			  u32 *rule_locs)
+{
+	struct qede_arfs_fltr_node *fltr;
+	struct hlist_head *head;
+	int cnt = 0, rc = 0;
+
+	info->data = QEDE_RFS_MAX_FLTR;
+
+	__qede_lock(edev);
+
+	if (!edev->arfs) {
+		rc = -EPERM;
+		goto unlock;
+	}
+
+	head = QEDE_ARFS_BUCKET_HEAD(edev, 0);
+
+	hlist_for_each_entry(fltr, head, node) {
+		if (cnt == info->rule_cnt) {
+			rc = -EMSGSIZE;
+			goto unlock;
+		}
+
+		rule_locs[cnt] = fltr->sw_id;
+		cnt++;
+	}
+
+	info->rule_cnt = cnt;
+
+unlock:
+	__qede_unlock(edev);
+	return rc;
+}
+
+int qede_get_cls_rule_entry(struct qede_dev *edev, struct ethtool_rxnfc *cmd)
+{
+	struct ethtool_rx_flow_spec *fsp = &cmd->fs;
+	struct qede_arfs_fltr_node *fltr = NULL;
+	int rc = 0;
+
+	cmd->data = QEDE_RFS_MAX_FLTR;
+
+	__qede_lock(edev);
+
+	if (!edev->arfs) {
+		rc = -EPERM;
+		goto unlock;
+	}
+
+	fltr = qede_get_arfs_fltr_by_loc(QEDE_ARFS_BUCKET_HEAD(edev, 0),
+					 fsp->location);
+	if (!fltr) {
+		DP_NOTICE(edev, "Rule not found - location=0x%x\n",
+			  fsp->location);
+		rc = -EINVAL;
+		goto unlock;
+	}
+
+	if (fltr->tuple.eth_proto == htons(ETH_P_IP)) {
+		if (fltr->tuple.ip_proto == IPPROTO_TCP)
+			fsp->flow_type = TCP_V4_FLOW;
+		else
+			fsp->flow_type = UDP_V4_FLOW;
+
+		fsp->h_u.tcp_ip4_spec.psrc = fltr->tuple.src_port;
+		fsp->h_u.tcp_ip4_spec.pdst = fltr->tuple.dst_port;
+		fsp->h_u.tcp_ip4_spec.ip4src = fltr->tuple.src_ipv4;
+		fsp->h_u.tcp_ip4_spec.ip4dst = fltr->tuple.dst_ipv4;
+	} else {
+		if (fltr->tuple.ip_proto == IPPROTO_TCP)
+			fsp->flow_type = TCP_V6_FLOW;
+		else
+			fsp->flow_type = UDP_V6_FLOW;
+		fsp->h_u.tcp_ip6_spec.psrc = fltr->tuple.src_port;
+		fsp->h_u.tcp_ip6_spec.pdst = fltr->tuple.dst_port;
+		memcpy(&fsp->h_u.tcp_ip6_spec.ip6src,
+		       &fltr->tuple.src_ipv6, sizeof(struct in6_addr));
+		memcpy(&fsp->h_u.tcp_ip6_spec.ip6dst,
+		       &fltr->tuple.dst_ipv6, sizeof(struct in6_addr));
+	}
+
+	fsp->ring_cookie = fltr->rxq_id;
+
+unlock:
+	__qede_unlock(edev);
+	return rc;
+}
+
+static int
+qede_validate_and_check_flow_exist(struct qede_dev *edev,
+				   struct ethtool_rx_flow_spec *fsp,
+				   int *min_hlen)
+{
+	__be16 src_port = 0x0, dst_port = 0x0;
+	struct qede_arfs_fltr_node *fltr;
+	struct hlist_node *temp;
+	struct hlist_head *head;
+	__be16 eth_proto;
+	u8 ip_proto;
+
+	if (fsp->location >= QEDE_RFS_MAX_FLTR ||
+	    fsp->ring_cookie >= QEDE_RSS_COUNT(edev))
+		return -EINVAL;
+
+	if (fsp->flow_type == TCP_V4_FLOW) {
+		*min_hlen += sizeof(struct iphdr) +
+				sizeof(struct tcphdr);
+		eth_proto = htons(ETH_P_IP);
+		ip_proto = IPPROTO_TCP;
+	} else if (fsp->flow_type == UDP_V4_FLOW) {
+		*min_hlen += sizeof(struct iphdr) +
+				sizeof(struct udphdr);
+		eth_proto = htons(ETH_P_IP);
+		ip_proto = IPPROTO_UDP;
+	} else if (fsp->flow_type == TCP_V6_FLOW) {
+		*min_hlen += sizeof(struct ipv6hdr) +
+				sizeof(struct tcphdr);
+		eth_proto = htons(ETH_P_IPV6);
+		ip_proto = IPPROTO_TCP;
+	} else if (fsp->flow_type == UDP_V6_FLOW) {
+		*min_hlen += sizeof(struct ipv6hdr) +
+				sizeof(struct udphdr);
+		eth_proto = htons(ETH_P_IPV6);
+		ip_proto = IPPROTO_UDP;
+	} else {
+		DP_NOTICE(edev, "Unsupported flow type = 0x%x\n",
+			  fsp->flow_type);
+		return -EPROTONOSUPPORT;
+	}
+
+	if (eth_proto == htons(ETH_P_IP)) {
+		src_port = fsp->h_u.tcp_ip4_spec.psrc;
+		dst_port = fsp->h_u.tcp_ip4_spec.pdst;
+	} else {
+		src_port = fsp->h_u.tcp_ip6_spec.psrc;
+		dst_port = fsp->h_u.tcp_ip6_spec.pdst;
+	}
+
+	head = QEDE_ARFS_BUCKET_HEAD(edev, 0);
+	hlist_for_each_entry_safe(fltr, temp, head, node) {
+		if ((fltr->tuple.ip_proto == ip_proto &&
+		     fltr->tuple.eth_proto == eth_proto &&
+		     qede_compare_user_flow_ips(fltr, fsp, eth_proto) &&
+		     fltr->tuple.src_port == src_port &&
+		     fltr->tuple.dst_port == dst_port) ||
+		    fltr->sw_id == fsp->location)
+			return -EEXIST;
+	}
+
+	return 0;
+}
+
+static int
+qede_poll_arfs_filter_config(struct qede_dev *edev,
+			     struct qede_arfs_fltr_node *fltr)
+{
+	int count = QEDE_ARFS_POLL_COUNT;
+
+	while (fltr->used && count) {
+		msleep(20);
+		count--;
+	}
+
+	if (count == 0 || fltr->fw_rc) {
+		qede_dequeue_fltr_and_config_searcher(edev, fltr);
+		return -EIO;
+	}
+
+	return fltr->fw_rc;
+}
+
+int qede_add_cls_rule(struct qede_dev *edev, struct ethtool_rxnfc *info)
+{
+	struct ethtool_rx_flow_spec *fsp = &info->fs;
+	struct qede_arfs_fltr_node *n;
+	int min_hlen = ETH_HLEN, rc;
+	struct ethhdr *eth;
+	struct iphdr *ip;
+	__be16 *ports;
+
+	__qede_lock(edev);
+
+	if (!edev->arfs) {
+		rc = -EPERM;
+		goto unlock;
+	}
+
+	rc = qede_validate_and_check_flow_exist(edev, fsp, &min_hlen);
+	if (rc)
+		goto unlock;
+
+	n = kzalloc(sizeof(*n), GFP_KERNEL);
+	if (!n) {
+		rc = -ENOMEM;
+		goto unlock;
+	}
+
+	n->data = kzalloc(min_hlen, GFP_KERNEL);
+	if (!n->data) {
+		kfree(n);
+		rc = -ENOMEM;
+		goto unlock;
+	}
+
+	n->sw_id = fsp->location;
+	set_bit(n->sw_id, edev->arfs->arfs_fltr_bmap);
+	n->buf_len = min_hlen;
+	n->rxq_id = fsp->ring_cookie;
+	n->next_rxq_id = n->rxq_id;
+	eth = (struct ethhdr *)n->data;
+
+	if (info->fs.flow_type == TCP_V4_FLOW ||
+	    info->fs.flow_type == UDP_V4_FLOW) {
+		ports = (__be16 *)(n->data + ETH_HLEN +
+					sizeof(struct iphdr));
+		eth->h_proto = htons(ETH_P_IP);
+		n->tuple.eth_proto = htons(ETH_P_IP);
+		n->tuple.src_ipv4 = info->fs.h_u.tcp_ip4_spec.ip4src;
+		n->tuple.dst_ipv4 = info->fs.h_u.tcp_ip4_spec.ip4dst;
+		n->tuple.src_port = info->fs.h_u.tcp_ip4_spec.psrc;
+		n->tuple.dst_port = info->fs.h_u.tcp_ip4_spec.pdst;
+		ports[0] = n->tuple.src_port;
+		ports[1] = n->tuple.dst_port;
+		ip = (struct iphdr *)(n->data + ETH_HLEN);
+		ip->saddr = info->fs.h_u.tcp_ip4_spec.ip4src;
+		ip->daddr = info->fs.h_u.tcp_ip4_spec.ip4dst;
+		ip->version = 0x4;
+		ip->ihl = 0x5;
+
+		if (info->fs.flow_type == TCP_V4_FLOW) {
+			n->tuple.ip_proto = IPPROTO_TCP;
+			ip->protocol = IPPROTO_TCP;
+		} else {
+			n->tuple.ip_proto = IPPROTO_UDP;
+			ip->protocol = IPPROTO_UDP;
+		}
+		ip->tot_len = cpu_to_be16(min_hlen - ETH_HLEN);
+	} else {
+		struct ipv6hdr *ip6;
+
+		ip6 = (struct ipv6hdr *)(n->data + ETH_HLEN);
+		ports = (__be16 *)(n->data + ETH_HLEN +
+					sizeof(struct ipv6hdr));
+		eth->h_proto = htons(ETH_P_IPV6);
+		n->tuple.eth_proto = htons(ETH_P_IPV6);
+		memcpy(&n->tuple.src_ipv6, &info->fs.h_u.tcp_ip6_spec.ip6src,
+		       sizeof(struct in6_addr));
+		memcpy(&n->tuple.dst_ipv6, &info->fs.h_u.tcp_ip6_spec.ip6dst,
+		       sizeof(struct in6_addr));
+		n->tuple.src_port = info->fs.h_u.tcp_ip6_spec.psrc;
+		n->tuple.dst_port = info->fs.h_u.tcp_ip6_spec.pdst;
+		ports[0] = n->tuple.src_port;
+		ports[1] = n->tuple.dst_port;
+		memcpy(&ip6->saddr, &n->tuple.src_ipv6,
+		       sizeof(struct in6_addr));
+		memcpy(&ip6->daddr, &n->tuple.dst_ipv6,
+		       sizeof(struct in6_addr));
+		ip6->version = 0x6;
+
+		if (info->fs.flow_type == TCP_V6_FLOW) {
+			n->tuple.ip_proto = IPPROTO_TCP;
+			ip6->nexthdr = NEXTHDR_TCP;
+			ip6->payload_len = cpu_to_be16(sizeof(struct tcphdr));
+		} else {
+			n->tuple.ip_proto = IPPROTO_UDP;
+			ip6->nexthdr = NEXTHDR_UDP;
+			ip6->payload_len = cpu_to_be16(sizeof(struct udphdr));
+		}
+	}
+
+	rc = qede_enqueue_fltr_and_config_searcher(edev, n, 0);
+	if (rc)
+		goto unlock;
+
+	qede_configure_arfs_fltr(edev, n, n->rxq_id, true);
+	rc = qede_poll_arfs_filter_config(edev, n);
+unlock:
+	__qede_unlock(edev);
+	return rc;
+}
+
+int qede_del_cls_rule(struct qede_dev *edev, struct ethtool_rxnfc *info)
+{
+	struct ethtool_rx_flow_spec *fsp = &info->fs;
+	struct qede_arfs_fltr_node *fltr = NULL;
+	int rc = -EPERM;
+
+	__qede_lock(edev);
+	if (!edev->arfs)
+		goto unlock;
+
+	fltr = qede_get_arfs_fltr_by_loc(QEDE_ARFS_BUCKET_HEAD(edev, 0),
+					 fsp->location);
+	if (!fltr)
+		goto unlock;
+
+	qede_configure_arfs_fltr(edev, fltr, fltr->rxq_id, false);
+
+	rc = qede_poll_arfs_filter_config(edev, fltr);
+	if (rc == 0)
+		qede_dequeue_fltr_and_config_searcher(edev, fltr);
+
+unlock:
+	__qede_unlock(edev);
+	return rc;
+}
+
+int qede_get_arfs_filter_count(struct qede_dev *edev)
+{
+	int count = 0;
+
+	__qede_lock(edev);
+
+	if (!edev->arfs)
+		goto unlock;
+
+	count = edev->arfs->filter_count;
+
+unlock:
+	__qede_unlock(edev);
+	return count;
+}
diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c
index 06ca13dd9ddb..e5ee9f274a71 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_main.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_main.c
@@ -873,9 +873,7 @@ static void qede_update_pf_params(struct qed_dev *cdev)
 	 */
 	pf_params.eth_pf_params.num_vf_cons = 48;
 
-#ifdef CONFIG_RFS_ACCEL
 	pf_params.eth_pf_params.num_arfs_filters = QEDE_RFS_MAX_FLTR;
-#endif
 	qed_ops->common->update_pf_params(cdev, &pf_params);
 }
 
@@ -1984,12 +1982,12 @@ static void qede_unload(struct qede_dev *edev, enum qede_unload_mode mode,
 
 	qede_vlan_mark_nonconfigured(edev);
 	edev->ops->fastpath_stop(edev->cdev);
-#ifdef CONFIG_RFS_ACCEL
+
 	if (!IS_VF(edev) && edev->dev_info.common.num_hwfns == 1) {
 		qede_poll_for_freeing_arfs_filters(edev);
 		qede_free_arfs(edev);
 	}
-#endif
+
 	/* Release the interrupts */
 	qede_sync_free_irqs(edev);
 	edev->ops->common->set_fp_int(edev->cdev, 0);
@@ -2041,13 +2039,12 @@ static int qede_load(struct qede_dev *edev, enum qede_load_mode mode,
 	if (rc)
 		goto err2;
 
-#ifdef CONFIG_RFS_ACCEL
 	if (!IS_VF(edev) && edev->dev_info.common.num_hwfns == 1) {
 		rc = qede_alloc_arfs(edev);
 		if (rc)
 			DP_NOTICE(edev, "aRFS memory allocation failed\n");
 	}
-#endif
+
 	qede_napi_add_enable(edev);
 	DP_INFO(edev, "Napi added and enabled\n");
 
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_init.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_init.c
index be41e4c77b65..c48a0e2d4d7e 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_init.c
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_init.c
@@ -592,13 +592,9 @@ qlcnic_receive_peg_ready(struct qlcnic_adapter *adapter)
 
 	} while (--retries);
 
-	if (!retries) {
-		dev_err(&adapter->pdev->dev, "Receive Peg initialization not "
-			      "complete, state: 0x%x.\n", val);
-		return -EIO;
-	}
-
-	return 0;
+	dev_err(&adapter->pdev->dev, "Receive Peg initialization not complete, state: 0x%x.\n",
+		val);
+	return -EIO;
 }
 
 int
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_minidump.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_minidump.c
index 0844b7c75767..afa10a163da1 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_minidump.c
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_minidump.c
@@ -1285,7 +1285,7 @@ flash_temp:
 int qlcnic_dump_fw(struct qlcnic_adapter *adapter)
 {
 	struct qlcnic_fw_dump *fw_dump = &adapter->ahw->fw_dump;
-	static const struct qlcnic_dump_operations *fw_dump_ops;
+	const struct qlcnic_dump_operations *fw_dump_ops;
 	struct qlcnic_83xx_dump_template_hdr *hdr_83xx;
 	u32 entry_offset, dump, no_entries, buf_offset = 0;
 	int i, k, ops_cnt, ops_index, dump_size = 0;
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sysfs.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sysfs.c
index 73027a6c06c7..287d89dd086f 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sysfs.c
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sysfs.c
@@ -1174,19 +1174,19 @@ static ssize_t qlcnic_83xx_sysfs_flash_write_handler(struct file *filp,
 	return size;
 }
 
-static struct device_attribute dev_attr_bridged_mode = {
+static const struct device_attribute dev_attr_bridged_mode = {
        .attr = {.name = "bridged_mode", .mode = (S_IRUGO | S_IWUSR)},
        .show = qlcnic_show_bridged_mode,
        .store = qlcnic_store_bridged_mode,
 };
 
-static struct device_attribute dev_attr_diag_mode = {
+static const struct device_attribute dev_attr_diag_mode = {
 	.attr = {.name = "diag_mode", .mode = (S_IRUGO | S_IWUSR)},
 	.show = qlcnic_show_diag_mode,
 	.store = qlcnic_store_diag_mode,
 };
 
-static struct device_attribute dev_attr_beacon = {
+static const struct device_attribute dev_attr_beacon = {
 	.attr = {.name = "beacon", .mode = (S_IRUGO | S_IWUSR)},
 	.show = qlcnic_show_beacon,
 	.store = qlcnic_store_beacon,
@@ -1248,7 +1248,7 @@ static const struct bin_attribute bin_attr_pm_config = {
 	.write = qlcnic_sysfs_write_pm_config,
 };
 
-static struct bin_attribute bin_attr_flash = {
+static const struct bin_attribute bin_attr_flash = {
 	.attr = {.name = "flash", .mode = (S_IRUGO | S_IWUSR)},
 	.size = 0,
 	.read = qlcnic_83xx_sysfs_flash_read_handler,
diff --git a/drivers/net/ethernet/qlogic/qlge/qlge_dbg.c b/drivers/net/ethernet/qlogic/qlge/qlge_dbg.c
index 28ea0af89aef..fe2599b83d09 100644
--- a/drivers/net/ethernet/qlogic/qlge/qlge_dbg.c
+++ b/drivers/net/ethernet/qlogic/qlge/qlge_dbg.c
@@ -144,42 +144,23 @@ static int ql_get_serdes_regs(struct ql_adapter *qdev,
 	xaui_direct_valid = xaui_indirect_valid = 1;
 
 	/* The XAUI needs to be read out per port */
-	if (qdev->func & 1) {
-		/* We are NIC 2	*/
-		status = ql_read_other_func_serdes_reg(qdev,
-				XG_SERDES_XAUI_HSS_PCS_START, &temp);
-		if (status)
-			temp = XG_SERDES_ADDR_XAUI_PWR_DOWN;
-		if ((temp & XG_SERDES_ADDR_XAUI_PWR_DOWN) ==
-					XG_SERDES_ADDR_XAUI_PWR_DOWN)
-			xaui_indirect_valid = 0;
+	status = ql_read_other_func_serdes_reg(qdev,
+			XG_SERDES_XAUI_HSS_PCS_START, &temp);
+	if (status)
+		temp = XG_SERDES_ADDR_XAUI_PWR_DOWN;
 
-		status = ql_read_serdes_reg(qdev,
-				XG_SERDES_XAUI_HSS_PCS_START, &temp);
-		if (status)
-			temp = XG_SERDES_ADDR_XAUI_PWR_DOWN;
+	if ((temp & XG_SERDES_ADDR_XAUI_PWR_DOWN) ==
+				XG_SERDES_ADDR_XAUI_PWR_DOWN)
+		xaui_indirect_valid = 0;
 
-		if ((temp & XG_SERDES_ADDR_XAUI_PWR_DOWN) ==
-					XG_SERDES_ADDR_XAUI_PWR_DOWN)
-			xaui_direct_valid = 0;
-	} else {
-		/* We are NIC 1	*/
-		status = ql_read_other_func_serdes_reg(qdev,
-				XG_SERDES_XAUI_HSS_PCS_START, &temp);
-		if (status)
-			temp = XG_SERDES_ADDR_XAUI_PWR_DOWN;
-		if ((temp & XG_SERDES_ADDR_XAUI_PWR_DOWN) ==
-					XG_SERDES_ADDR_XAUI_PWR_DOWN)
-			xaui_indirect_valid = 0;
+	status = ql_read_serdes_reg(qdev, XG_SERDES_XAUI_HSS_PCS_START, &temp);
 
-		status = ql_read_serdes_reg(qdev,
-				XG_SERDES_XAUI_HSS_PCS_START, &temp);
-		if (status)
-			temp = XG_SERDES_ADDR_XAUI_PWR_DOWN;
-		if ((temp & XG_SERDES_ADDR_XAUI_PWR_DOWN) ==
-					XG_SERDES_ADDR_XAUI_PWR_DOWN)
-			xaui_direct_valid = 0;
-	}
+	if (status)
+		temp = XG_SERDES_ADDR_XAUI_PWR_DOWN;
+
+	if ((temp & XG_SERDES_ADDR_XAUI_PWR_DOWN) ==
+				XG_SERDES_ADDR_XAUI_PWR_DOWN)
+		xaui_direct_valid = 0;
 
 	/*
 	 * XFI register is shared so only need to read one
@@ -724,7 +705,7 @@ static void ql_build_coredump_seg_header(
 	seg_hdr->cookie = MPI_COREDUMP_COOKIE;
 	seg_hdr->segNum = seg_number;
 	seg_hdr->segSize = seg_size;
-	memcpy(seg_hdr->description, desc, (sizeof(seg_hdr->description)) - 1);
+	strncpy(seg_hdr->description, desc, (sizeof(seg_hdr->description)) - 1);
 }
 
 /*
diff --git a/drivers/net/ethernet/qualcomm/Kconfig b/drivers/net/ethernet/qualcomm/Kconfig
index 877675a27b9f..f5200712718d 100644
--- a/drivers/net/ethernet/qualcomm/Kconfig
+++ b/drivers/net/ethernet/qualcomm/Kconfig
@@ -59,4 +59,6 @@ config QCOM_EMAC
 	  low power, Receive-Side Scaling (RSS), and IEEE 1588-2008
 	  Precision Clock Synchronization Protocol.
 
+source "drivers/net/ethernet/qualcomm/rmnet/Kconfig"
+
 endif # NET_VENDOR_QUALCOMM
diff --git a/drivers/net/ethernet/qualcomm/Makefile b/drivers/net/ethernet/qualcomm/Makefile
index 92fa7c4da90a..1847350f48a7 100644
--- a/drivers/net/ethernet/qualcomm/Makefile
+++ b/drivers/net/ethernet/qualcomm/Makefile
@@ -9,3 +9,5 @@ obj-$(CONFIG_QCA7000_UART) += qcauart.o
 qcauart-objs := qca_uart.o
 
 obj-y += emac/
+
+obj-$(CONFIG_RMNET) += rmnet/
diff --git a/drivers/net/ethernet/qualcomm/rmnet/Kconfig b/drivers/net/ethernet/qualcomm/rmnet/Kconfig
new file mode 100644
index 000000000000..6e2587af47a4
--- /dev/null
+++ b/drivers/net/ethernet/qualcomm/rmnet/Kconfig
@@ -0,0 +1,12 @@
+#
+# RMNET MAP driver
+#
+
+menuconfig RMNET
+	tristate "RmNet MAP driver"
+	default n
+	---help---
+	  If you select this, you will enable the RMNET module which is used
+	  for handling data in the multiplexing and aggregation protocol (MAP)
+	  format in the embedded data path. RMNET devices can be attached to
+	  any IP mode physical device.
diff --git a/drivers/net/ethernet/qualcomm/rmnet/Makefile b/drivers/net/ethernet/qualcomm/rmnet/Makefile
new file mode 100644
index 000000000000..01bddf207cac
--- /dev/null
+++ b/drivers/net/ethernet/qualcomm/rmnet/Makefile
@@ -0,0 +1,10 @@
+#
+# Makefile for the RMNET module
+#
+
+rmnet-y		 := rmnet_config.o
+rmnet-y		 += rmnet_vnd.o
+rmnet-y		 += rmnet_handlers.o
+rmnet-y		 += rmnet_map_data.o
+rmnet-y		 += rmnet_map_command.o
+obj-$(CONFIG_RMNET) += rmnet.o
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c
new file mode 100644
index 000000000000..98f22551eb45
--- /dev/null
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c
@@ -0,0 +1,356 @@
+/* Copyright (c) 2013-2017, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * RMNET configuration engine
+ *
+ */
+
+#include <net/sock.h>
+#include <linux/module.h>
+#include <linux/netlink.h>
+#include <linux/netdevice.h>
+#include "rmnet_config.h"
+#include "rmnet_handlers.h"
+#include "rmnet_vnd.h"
+#include "rmnet_private.h"
+
+/* Locking scheme -
+ * The shared resource which needs to be protected is realdev->rx_handler_data.
+ * For the writer path, this is using rtnl_lock(). The writer paths are
+ * rmnet_newlink(), rmnet_dellink() and rmnet_force_unassociate_device(). These
+ * paths are already called with rtnl_lock() acquired in. There is also an
+ * ASSERT_RTNL() to ensure that we are calling with rtnl acquired. For
+ * dereference here, we will need to use rtnl_dereference(). Dev list writing
+ * needs to happen with rtnl_lock() acquired for netdev_master_upper_dev_link().
+ * For the reader path, the real_dev->rx_handler_data is called in the TX / RX
+ * path. We only need rcu_read_lock() for these scenarios. In these cases,
+ * the rcu_read_lock() is held in __dev_queue_xmit() and
+ * netif_receive_skb_internal(), so readers need to use rcu_dereference_rtnl()
+ * to get the relevant information. For dev list reading, we again acquire
+ * rcu_read_lock() in rmnet_dellink() for netdev_master_upper_dev_get_rcu().
+ * We also use unregister_netdevice_many() to free all rmnet devices in
+ * rmnet_force_unassociate_device() so we dont lose the rtnl_lock() and free in
+ * same context.
+ */
+
+/* Local Definitions and Declarations */
+
+struct rmnet_walk_data {
+	struct net_device *real_dev;
+	struct list_head *head;
+	struct rmnet_port *port;
+};
+
+static int rmnet_is_real_dev_registered(const struct net_device *real_dev)
+{
+	rx_handler_func_t *rx_handler;
+
+	rx_handler = rcu_dereference(real_dev->rx_handler);
+	return (rx_handler == rmnet_rx_handler);
+}
+
+/* Needs rtnl lock */
+static struct rmnet_port*
+rmnet_get_port_rtnl(const struct net_device *real_dev)
+{
+	return rtnl_dereference(real_dev->rx_handler_data);
+}
+
+static struct rmnet_endpoint*
+rmnet_get_endpoint(struct net_device *dev, int config_id)
+{
+	struct rmnet_endpoint *ep;
+	struct rmnet_port *port;
+
+	if (!rmnet_is_real_dev_registered(dev)) {
+		ep = rmnet_vnd_get_endpoint(dev);
+	} else {
+		port = rmnet_get_port_rtnl(dev);
+
+		ep = &port->muxed_ep[config_id];
+	}
+
+	return ep;
+}
+
+static int rmnet_unregister_real_device(struct net_device *real_dev,
+					struct rmnet_port *port)
+{
+	if (port->nr_rmnet_devs)
+		return -EINVAL;
+
+	kfree(port);
+
+	netdev_rx_handler_unregister(real_dev);
+
+	/* release reference on real_dev */
+	dev_put(real_dev);
+
+	netdev_dbg(real_dev, "Removed from rmnet\n");
+	return 0;
+}
+
+static int rmnet_register_real_device(struct net_device *real_dev)
+{
+	struct rmnet_port *port;
+	int rc;
+
+	ASSERT_RTNL();
+
+	if (rmnet_is_real_dev_registered(real_dev))
+		return 0;
+
+	port = kzalloc(sizeof(*port), GFP_ATOMIC);
+	if (!port)
+		return -ENOMEM;
+
+	port->dev = real_dev;
+	rc = netdev_rx_handler_register(real_dev, rmnet_rx_handler, port);
+	if (rc) {
+		kfree(port);
+		return -EBUSY;
+	}
+
+	/* hold on to real dev for MAP data */
+	dev_hold(real_dev);
+
+	netdev_dbg(real_dev, "registered with rmnet\n");
+	return 0;
+}
+
+static void rmnet_set_endpoint_config(struct net_device *dev,
+				      u8 mux_id, u8 rmnet_mode,
+				      struct net_device *egress_dev)
+{
+	struct rmnet_endpoint *ep;
+
+	netdev_dbg(dev, "id %d mode %d dev %s\n",
+		   mux_id, rmnet_mode, egress_dev->name);
+
+	ep = rmnet_get_endpoint(dev, mux_id);
+	/* This config is cleared on every set, so its ok to not
+	 * clear it on a device delete.
+	 */
+	memset(ep, 0, sizeof(struct rmnet_endpoint));
+	ep->rmnet_mode = rmnet_mode;
+	ep->egress_dev = egress_dev;
+	ep->mux_id = mux_id;
+}
+
+static int rmnet_newlink(struct net *src_net, struct net_device *dev,
+			 struct nlattr *tb[], struct nlattr *data[],
+			 struct netlink_ext_ack *extack)
+{
+	int ingress_format = RMNET_INGRESS_FORMAT_DEMUXING |
+			     RMNET_INGRESS_FORMAT_DEAGGREGATION |
+			     RMNET_INGRESS_FORMAT_MAP;
+	int egress_format = RMNET_EGRESS_FORMAT_MUXING |
+			    RMNET_EGRESS_FORMAT_MAP;
+	struct net_device *real_dev;
+	int mode = RMNET_EPMODE_VND;
+	struct rmnet_port *port;
+	int err = 0;
+	u16 mux_id;
+
+	real_dev = __dev_get_by_index(src_net, nla_get_u32(tb[IFLA_LINK]));
+	if (!real_dev || !dev)
+		return -ENODEV;
+
+	if (!data[IFLA_VLAN_ID])
+		return -EINVAL;
+
+	mux_id = nla_get_u16(data[IFLA_VLAN_ID]);
+
+	err = rmnet_register_real_device(real_dev);
+	if (err)
+		goto err0;
+
+	port = rmnet_get_port_rtnl(real_dev);
+	err = rmnet_vnd_newlink(mux_id, dev, port, real_dev);
+	if (err)
+		goto err1;
+
+	err = netdev_master_upper_dev_link(dev, real_dev, NULL, NULL);
+	if (err)
+		goto err2;
+
+	netdev_dbg(dev, "data format [ingress 0x%08X] [egress 0x%08X]\n",
+		   ingress_format, egress_format);
+	port->egress_data_format = egress_format;
+	port->ingress_data_format = ingress_format;
+
+	rmnet_set_endpoint_config(real_dev, mux_id, mode, dev);
+	rmnet_set_endpoint_config(dev, mux_id, mode, real_dev);
+	return 0;
+
+err2:
+	rmnet_vnd_dellink(mux_id, port);
+err1:
+	rmnet_unregister_real_device(real_dev, port);
+err0:
+	return err;
+}
+
+static void rmnet_dellink(struct net_device *dev, struct list_head *head)
+{
+	struct net_device *real_dev;
+	struct rmnet_port *port;
+	u8 mux_id;
+
+	rcu_read_lock();
+	real_dev = netdev_master_upper_dev_get_rcu(dev);
+	rcu_read_unlock();
+
+	if (!real_dev || !rmnet_is_real_dev_registered(real_dev))
+		return;
+
+	port = rmnet_get_port_rtnl(real_dev);
+
+	mux_id = rmnet_vnd_get_mux(dev);
+	rmnet_vnd_dellink(mux_id, port);
+	netdev_upper_dev_unlink(dev, real_dev);
+	rmnet_unregister_real_device(real_dev, port);
+
+	unregister_netdevice_queue(dev, head);
+}
+
+static int rmnet_dev_walk_unreg(struct net_device *rmnet_dev, void *data)
+{
+	struct rmnet_walk_data *d = data;
+	u8 mux_id;
+
+	mux_id = rmnet_vnd_get_mux(rmnet_dev);
+
+	rmnet_vnd_dellink(mux_id, d->port);
+	netdev_upper_dev_unlink(rmnet_dev, d->real_dev);
+	unregister_netdevice_queue(rmnet_dev, d->head);
+
+	return 0;
+}
+
+static void rmnet_force_unassociate_device(struct net_device *dev)
+{
+	struct net_device *real_dev = dev;
+	struct rmnet_walk_data d;
+	struct rmnet_port *port;
+	LIST_HEAD(list);
+
+	if (!rmnet_is_real_dev_registered(real_dev))
+		return;
+
+	ASSERT_RTNL();
+
+	d.real_dev = real_dev;
+	d.head = &list;
+
+	port = rmnet_get_port_rtnl(dev);
+	d.port = port;
+
+	rcu_read_lock();
+	netdev_walk_all_lower_dev_rcu(real_dev, rmnet_dev_walk_unreg, &d);
+	rcu_read_unlock();
+	unregister_netdevice_many(&list);
+
+	rmnet_unregister_real_device(real_dev, port);
+}
+
+static int rmnet_config_notify_cb(struct notifier_block *nb,
+				  unsigned long event, void *data)
+{
+	struct net_device *dev = netdev_notifier_info_to_dev(data);
+
+	if (!dev)
+		return NOTIFY_DONE;
+
+	switch (event) {
+	case NETDEV_UNREGISTER:
+		netdev_dbg(dev, "Kernel unregister\n");
+		rmnet_force_unassociate_device(dev);
+		break;
+
+	default:
+		break;
+	}
+
+	return NOTIFY_DONE;
+}
+
+static struct notifier_block rmnet_dev_notifier __read_mostly = {
+	.notifier_call = rmnet_config_notify_cb,
+};
+
+static int rmnet_rtnl_validate(struct nlattr *tb[], struct nlattr *data[],
+			       struct netlink_ext_ack *extack)
+{
+	u16 mux_id;
+
+	if (!data || !data[IFLA_VLAN_ID])
+		return -EINVAL;
+
+	mux_id = nla_get_u16(data[IFLA_VLAN_ID]);
+	if (mux_id > (RMNET_MAX_LOGICAL_EP - 1))
+		return -ERANGE;
+
+	return 0;
+}
+
+static size_t rmnet_get_size(const struct net_device *dev)
+{
+	return nla_total_size(2); /* IFLA_VLAN_ID */
+}
+
+struct rtnl_link_ops rmnet_link_ops __read_mostly = {
+	.kind		= "rmnet",
+	.maxtype	= __IFLA_VLAN_MAX,
+	.priv_size	= sizeof(struct rmnet_priv),
+	.setup		= rmnet_vnd_setup,
+	.validate	= rmnet_rtnl_validate,
+	.newlink	= rmnet_newlink,
+	.dellink	= rmnet_dellink,
+	.get_size	= rmnet_get_size,
+};
+
+/* Needs either rcu_read_lock() or rtnl lock */
+struct rmnet_port *rmnet_get_port(struct net_device *real_dev)
+{
+	if (rmnet_is_real_dev_registered(real_dev))
+		return rcu_dereference_rtnl(real_dev->rx_handler_data);
+	else
+		return NULL;
+}
+
+/* Startup/Shutdown */
+
+static int __init rmnet_init(void)
+{
+	int rc;
+
+	rc = register_netdevice_notifier(&rmnet_dev_notifier);
+	if (rc != 0)
+		return rc;
+
+	rc = rtnl_link_register(&rmnet_link_ops);
+	if (rc != 0) {
+		unregister_netdevice_notifier(&rmnet_dev_notifier);
+		return rc;
+	}
+	return rc;
+}
+
+static void __exit rmnet_exit(void)
+{
+	unregister_netdevice_notifier(&rmnet_dev_notifier);
+	rtnl_link_unregister(&rmnet_link_ops);
+}
+
+module_init(rmnet_init)
+module_exit(rmnet_exit)
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h
new file mode 100644
index 000000000000..dde4e9f14f4a
--- /dev/null
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h
@@ -0,0 +1,55 @@
+/* Copyright (c) 2013-2014, 2016-2017 The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * RMNET Data configuration engine
+ *
+ */
+
+#include <linux/skbuff.h>
+
+#ifndef _RMNET_CONFIG_H_
+#define _RMNET_CONFIG_H_
+
+#define RMNET_MAX_LOGICAL_EP 255
+
+/* Information about the next device to deliver the packet to.
+ * Exact usage of this parameter depends on the rmnet_mode.
+ */
+struct rmnet_endpoint {
+	u8 rmnet_mode;
+	u8 mux_id;
+	struct net_device *egress_dev;
+};
+
+/* One instance of this structure is instantiated for each real_dev associated
+ * with rmnet.
+ */
+struct rmnet_port {
+	struct net_device *dev;
+	struct rmnet_endpoint local_ep;
+	struct rmnet_endpoint muxed_ep[RMNET_MAX_LOGICAL_EP];
+	u32 ingress_data_format;
+	u32 egress_data_format;
+	struct net_device *rmnet_devices[RMNET_MAX_LOGICAL_EP];
+	u8 nr_rmnet_devs;
+};
+
+extern struct rtnl_link_ops rmnet_link_ops;
+
+struct rmnet_priv {
+	struct rmnet_endpoint local_ep;
+	u8 mux_id;
+	struct net_device *real_dev;
+};
+
+struct rmnet_port *rmnet_get_port(struct net_device *real_dev);
+
+#endif /* _RMNET_CONFIG_H_ */
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c
new file mode 100644
index 000000000000..540c7622dcb1
--- /dev/null
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c
@@ -0,0 +1,271 @@
+/* Copyright (c) 2013-2017, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * RMNET Data ingress/egress handler
+ *
+ */
+
+#include <linux/netdevice.h>
+#include <linux/netdev_features.h>
+#include "rmnet_private.h"
+#include "rmnet_config.h"
+#include "rmnet_vnd.h"
+#include "rmnet_map.h"
+#include "rmnet_handlers.h"
+
+#define RMNET_IP_VERSION_4 0x40
+#define RMNET_IP_VERSION_6 0x60
+
+/* Helper Functions */
+
+static void rmnet_set_skb_proto(struct sk_buff *skb)
+{
+	switch (skb->data[0] & 0xF0) {
+	case RMNET_IP_VERSION_4:
+		skb->protocol = htons(ETH_P_IP);
+		break;
+	case RMNET_IP_VERSION_6:
+		skb->protocol = htons(ETH_P_IPV6);
+		break;
+	default:
+		skb->protocol = htons(ETH_P_MAP);
+		break;
+	}
+}
+
+/* Generic handler */
+
+static rx_handler_result_t
+rmnet_bridge_handler(struct sk_buff *skb, struct rmnet_endpoint *ep)
+{
+	if (!ep->egress_dev)
+		kfree_skb(skb);
+	else
+		rmnet_egress_handler(skb, ep);
+
+	return RX_HANDLER_CONSUMED;
+}
+
+static rx_handler_result_t
+rmnet_deliver_skb(struct sk_buff *skb, struct rmnet_endpoint *ep)
+{
+	switch (ep->rmnet_mode) {
+	case RMNET_EPMODE_NONE:
+		return RX_HANDLER_PASS;
+
+	case RMNET_EPMODE_BRIDGE:
+		return rmnet_bridge_handler(skb, ep);
+
+	case RMNET_EPMODE_VND:
+		skb_reset_transport_header(skb);
+		skb_reset_network_header(skb);
+		rmnet_vnd_rx_fixup(skb, skb->dev);
+
+		skb->pkt_type = PACKET_HOST;
+		skb_set_mac_header(skb, 0);
+		netif_receive_skb(skb);
+		return RX_HANDLER_CONSUMED;
+
+	default:
+		kfree_skb(skb);
+		return RX_HANDLER_CONSUMED;
+	}
+}
+
+static rx_handler_result_t
+rmnet_ingress_deliver_packet(struct sk_buff *skb,
+			     struct rmnet_port *port)
+{
+	if (!port) {
+		kfree_skb(skb);
+		return RX_HANDLER_CONSUMED;
+	}
+
+	skb->dev = port->local_ep.egress_dev;
+
+	return rmnet_deliver_skb(skb, &port->local_ep);
+}
+
+/* MAP handler */
+
+static rx_handler_result_t
+__rmnet_map_ingress_handler(struct sk_buff *skb,
+			    struct rmnet_port *port)
+{
+	struct rmnet_endpoint *ep;
+	u8 mux_id;
+	u16 len;
+
+	if (RMNET_MAP_GET_CD_BIT(skb)) {
+		if (port->ingress_data_format
+		    & RMNET_INGRESS_FORMAT_MAP_COMMANDS)
+			return rmnet_map_command(skb, port);
+
+		kfree_skb(skb);
+		return RX_HANDLER_CONSUMED;
+	}
+
+	mux_id = RMNET_MAP_GET_MUX_ID(skb);
+	len = RMNET_MAP_GET_LENGTH(skb) - RMNET_MAP_GET_PAD(skb);
+
+	if (mux_id >= RMNET_MAX_LOGICAL_EP) {
+		kfree_skb(skb);
+		return RX_HANDLER_CONSUMED;
+	}
+
+	ep = &port->muxed_ep[mux_id];
+
+	if (port->ingress_data_format & RMNET_INGRESS_FORMAT_DEMUXING)
+		skb->dev = ep->egress_dev;
+
+	/* Subtract MAP header */
+	skb_pull(skb, sizeof(struct rmnet_map_header));
+	skb_trim(skb, len);
+	rmnet_set_skb_proto(skb);
+	return rmnet_deliver_skb(skb, ep);
+}
+
+static rx_handler_result_t
+rmnet_map_ingress_handler(struct sk_buff *skb,
+			  struct rmnet_port *port)
+{
+	struct sk_buff *skbn;
+	int rc;
+
+	if (port->ingress_data_format & RMNET_INGRESS_FORMAT_DEAGGREGATION) {
+		while ((skbn = rmnet_map_deaggregate(skb)) != NULL)
+			__rmnet_map_ingress_handler(skbn, port);
+
+		consume_skb(skb);
+		rc = RX_HANDLER_CONSUMED;
+	} else {
+		rc = __rmnet_map_ingress_handler(skb, port);
+	}
+
+	return rc;
+}
+
+static int rmnet_map_egress_handler(struct sk_buff *skb,
+				    struct rmnet_port *port,
+				    struct rmnet_endpoint *ep,
+				    struct net_device *orig_dev)
+{
+	int required_headroom, additional_header_len;
+	struct rmnet_map_header *map_header;
+
+	additional_header_len = 0;
+	required_headroom = sizeof(struct rmnet_map_header);
+
+	if (skb_headroom(skb) < required_headroom) {
+		if (pskb_expand_head(skb, required_headroom, 0, GFP_KERNEL))
+			return RMNET_MAP_CONSUMED;
+	}
+
+	map_header = rmnet_map_add_map_header(skb, additional_header_len, 0);
+	if (!map_header)
+		return RMNET_MAP_CONSUMED;
+
+	if (port->egress_data_format & RMNET_EGRESS_FORMAT_MUXING) {
+		if (ep->mux_id == 0xff)
+			map_header->mux_id = 0;
+		else
+			map_header->mux_id = ep->mux_id;
+	}
+
+	skb->protocol = htons(ETH_P_MAP);
+
+	return RMNET_MAP_SUCCESS;
+}
+
+/* Ingress / Egress Entry Points */
+
+/* Processes packet as per ingress data format for receiving device. Logical
+ * endpoint is determined from packet inspection. Packet is then sent to the
+ * egress device listed in the logical endpoint configuration.
+ */
+rx_handler_result_t rmnet_rx_handler(struct sk_buff **pskb)
+{
+	struct rmnet_port *port;
+	struct sk_buff *skb = *pskb;
+	struct net_device *dev;
+	int rc;
+
+	if (!skb)
+		return RX_HANDLER_CONSUMED;
+
+	dev = skb->dev;
+	port = rmnet_get_port(dev);
+
+	if (port->ingress_data_format & RMNET_INGRESS_FORMAT_MAP) {
+		rc = rmnet_map_ingress_handler(skb, port);
+	} else {
+		switch (ntohs(skb->protocol)) {
+		case ETH_P_MAP:
+			if (port->local_ep.rmnet_mode ==
+				RMNET_EPMODE_BRIDGE) {
+				rc = rmnet_ingress_deliver_packet(skb, port);
+			} else {
+				kfree_skb(skb);
+				rc = RX_HANDLER_CONSUMED;
+			}
+			break;
+
+		case ETH_P_IP:
+		case ETH_P_IPV6:
+			rc = rmnet_ingress_deliver_packet(skb, port);
+			break;
+
+		default:
+			rc = RX_HANDLER_PASS;
+		}
+	}
+
+	return rc;
+}
+
+/* Modifies packet as per logical endpoint configuration and egress data format
+ * for egress device configured in logical endpoint. Packet is then transmitted
+ * on the egress device.
+ */
+void rmnet_egress_handler(struct sk_buff *skb,
+			  struct rmnet_endpoint *ep)
+{
+	struct net_device *orig_dev;
+	struct rmnet_port *port;
+
+	orig_dev = skb->dev;
+	skb->dev = ep->egress_dev;
+
+	port = rmnet_get_port(skb->dev);
+	if (!port) {
+		kfree_skb(skb);
+		return;
+	}
+
+	if (port->egress_data_format & RMNET_EGRESS_FORMAT_MAP) {
+		switch (rmnet_map_egress_handler(skb, port, ep, orig_dev)) {
+		case RMNET_MAP_CONSUMED:
+			return;
+
+		case RMNET_MAP_SUCCESS:
+			break;
+
+		default:
+			kfree_skb(skb);
+			return;
+		}
+	}
+
+	if (ep->rmnet_mode == RMNET_EPMODE_VND)
+		rmnet_vnd_tx_fixup(skb, orig_dev);
+
+	dev_queue_xmit(skb);
+}
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.h
new file mode 100644
index 000000000000..f2638cf5693c
--- /dev/null
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.h
@@ -0,0 +1,26 @@
+/* Copyright (c) 2013, 2016-2017 The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * RMNET Data ingress/egress handler
+ *
+ */
+
+#ifndef _RMNET_HANDLERS_H_
+#define _RMNET_HANDLERS_H_
+
+#include "rmnet_config.h"
+
+void rmnet_egress_handler(struct sk_buff *skb,
+			  struct rmnet_endpoint *ep);
+
+rx_handler_result_t rmnet_rx_handler(struct sk_buff **pskb);
+
+#endif /* _RMNET_HANDLERS_H_ */
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h
new file mode 100644
index 000000000000..ce2302c25b12
--- /dev/null
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h
@@ -0,0 +1,86 @@
+/* Copyright (c) 2013-2017, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _RMNET_MAP_H_
+#define _RMNET_MAP_H_
+
+struct rmnet_map_control_command {
+	u8  command_name;
+	u8  cmd_type:2;
+	u8  reserved:6;
+	u16 reserved2;
+	u32 transaction_id;
+	union {
+		struct {
+			u16 ip_family:2;
+			u16 reserved:14;
+			u16 flow_control_seq_num;
+			u32 qos_id;
+		} flow_control;
+		u8 data[0];
+	};
+}  __aligned(1);
+
+enum rmnet_map_results {
+	RMNET_MAP_SUCCESS,
+	RMNET_MAP_CONSUMED,
+	RMNET_MAP_GENERAL_FAILURE,
+	RMNET_MAP_NOT_ENABLED,
+	RMNET_MAP_FAILED_AGGREGATION,
+	RMNET_MAP_FAILED_MUX
+};
+
+enum rmnet_map_commands {
+	RMNET_MAP_COMMAND_NONE,
+	RMNET_MAP_COMMAND_FLOW_DISABLE,
+	RMNET_MAP_COMMAND_FLOW_ENABLE,
+	/* These should always be the last 2 elements */
+	RMNET_MAP_COMMAND_UNKNOWN,
+	RMNET_MAP_COMMAND_ENUM_LENGTH
+};
+
+struct rmnet_map_header {
+	u8  pad_len:6;
+	u8  reserved_bit:1;
+	u8  cd_bit:1;
+	u8  mux_id;
+	u16 pkt_len;
+}  __aligned(1);
+
+#define RMNET_MAP_GET_MUX_ID(Y) (((struct rmnet_map_header *) \
+				 (Y)->data)->mux_id)
+#define RMNET_MAP_GET_CD_BIT(Y) (((struct rmnet_map_header *) \
+				(Y)->data)->cd_bit)
+#define RMNET_MAP_GET_PAD(Y) (((struct rmnet_map_header *) \
+				(Y)->data)->pad_len)
+#define RMNET_MAP_GET_CMD_START(Y) ((struct rmnet_map_control_command *) \
+				    ((Y)->data + \
+				      sizeof(struct rmnet_map_header)))
+#define RMNET_MAP_GET_LENGTH(Y) (ntohs(((struct rmnet_map_header *) \
+					(Y)->data)->pkt_len))
+
+#define RMNET_MAP_COMMAND_REQUEST     0
+#define RMNET_MAP_COMMAND_ACK         1
+#define RMNET_MAP_COMMAND_UNSUPPORTED 2
+#define RMNET_MAP_COMMAND_INVALID     3
+
+#define RMNET_MAP_NO_PAD_BYTES        0
+#define RMNET_MAP_ADD_PAD_BYTES       1
+
+u8 rmnet_map_demultiplex(struct sk_buff *skb);
+struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb);
+struct rmnet_map_header *rmnet_map_add_map_header(struct sk_buff *skb,
+						  int hdrlen, int pad);
+rx_handler_result_t rmnet_map_command(struct sk_buff *skb,
+				      struct rmnet_port *port);
+
+#endif /* _RMNET_MAP_H_ */
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_command.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_command.c
new file mode 100644
index 000000000000..d1ea5e21b982
--- /dev/null
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_command.c
@@ -0,0 +1,106 @@
+/* Copyright (c) 2013-2017, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/netdevice.h>
+#include "rmnet_config.h"
+#include "rmnet_map.h"
+#include "rmnet_private.h"
+#include "rmnet_vnd.h"
+
+static u8 rmnet_map_do_flow_control(struct sk_buff *skb,
+				    struct rmnet_port *rdinfo,
+				    int enable)
+{
+	struct rmnet_map_control_command *cmd;
+	struct rmnet_endpoint *ep;
+	struct net_device *vnd;
+	u16 ip_family;
+	u16 fc_seq;
+	u32 qos_id;
+	u8 mux_id;
+	int r;
+
+	mux_id = RMNET_MAP_GET_MUX_ID(skb);
+	cmd = RMNET_MAP_GET_CMD_START(skb);
+
+	if (mux_id >= RMNET_MAX_LOGICAL_EP) {
+		kfree_skb(skb);
+		return RX_HANDLER_CONSUMED;
+	}
+
+	ep = &rdinfo->muxed_ep[mux_id];
+	vnd = ep->egress_dev;
+
+	ip_family = cmd->flow_control.ip_family;
+	fc_seq = ntohs(cmd->flow_control.flow_control_seq_num);
+	qos_id = ntohl(cmd->flow_control.qos_id);
+
+	/* Ignore the ip family and pass the sequence number for both v4 and v6
+	 * sequence. User space does not support creating dedicated flows for
+	 * the 2 protocols
+	 */
+	r = rmnet_vnd_do_flow_control(vnd, enable);
+	if (r) {
+		kfree_skb(skb);
+		return RMNET_MAP_COMMAND_UNSUPPORTED;
+	} else {
+		return RMNET_MAP_COMMAND_ACK;
+	}
+}
+
+static void rmnet_map_send_ack(struct sk_buff *skb,
+			       unsigned char type)
+{
+	struct rmnet_map_control_command *cmd;
+	int xmit_status;
+
+	skb->protocol = htons(ETH_P_MAP);
+
+	cmd = RMNET_MAP_GET_CMD_START(skb);
+	cmd->cmd_type = type & 0x03;
+
+	netif_tx_lock(skb->dev);
+	xmit_status = skb->dev->netdev_ops->ndo_start_xmit(skb, skb->dev);
+	netif_tx_unlock(skb->dev);
+}
+
+/* Process MAP command frame and send N/ACK message as appropriate. Message cmd
+ * name is decoded here and appropriate handler is called.
+ */
+rx_handler_result_t rmnet_map_command(struct sk_buff *skb,
+				      struct rmnet_port *port)
+{
+	struct rmnet_map_control_command *cmd;
+	unsigned char command_name;
+	unsigned char rc = 0;
+
+	cmd = RMNET_MAP_GET_CMD_START(skb);
+	command_name = cmd->command_name;
+
+	switch (command_name) {
+	case RMNET_MAP_COMMAND_FLOW_ENABLE:
+		rc = rmnet_map_do_flow_control(skb, port, 1);
+		break;
+
+	case RMNET_MAP_COMMAND_FLOW_DISABLE:
+		rc = rmnet_map_do_flow_control(skb, port, 0);
+		break;
+
+	default:
+		rc = RMNET_MAP_COMMAND_UNSUPPORTED;
+		kfree_skb(skb);
+		break;
+	}
+	if (rc == RMNET_MAP_COMMAND_ACK)
+		rmnet_map_send_ack(skb, rc);
+	return RX_HANDLER_CONSUMED;
+}
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c
new file mode 100644
index 000000000000..86b8c758f94e
--- /dev/null
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c
@@ -0,0 +1,102 @@
+/* Copyright (c) 2013-2017, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * RMNET Data MAP protocol
+ *
+ */
+
+#include <linux/netdevice.h>
+#include "rmnet_config.h"
+#include "rmnet_map.h"
+#include "rmnet_private.h"
+
+#define RMNET_MAP_DEAGGR_SPACING  64
+#define RMNET_MAP_DEAGGR_HEADROOM (RMNET_MAP_DEAGGR_SPACING / 2)
+
+/* Adds MAP header to front of skb->data
+ * Padding is calculated and set appropriately in MAP header. Mux ID is
+ * initialized to 0.
+ */
+struct rmnet_map_header *rmnet_map_add_map_header(struct sk_buff *skb,
+						  int hdrlen, int pad)
+{
+	struct rmnet_map_header *map_header;
+	u32 padding, map_datalen;
+	u8 *padbytes;
+
+	if (skb_headroom(skb) < sizeof(struct rmnet_map_header))
+		return NULL;
+
+	map_datalen = skb->len - hdrlen;
+	map_header = (struct rmnet_map_header *)
+			skb_push(skb, sizeof(struct rmnet_map_header));
+	memset(map_header, 0, sizeof(struct rmnet_map_header));
+
+	if (pad == RMNET_MAP_NO_PAD_BYTES) {
+		map_header->pkt_len = htons(map_datalen);
+		return map_header;
+	}
+
+	padding = ALIGN(map_datalen, 4) - map_datalen;
+
+	if (padding == 0)
+		goto done;
+
+	if (skb_tailroom(skb) < padding)
+		return NULL;
+
+	padbytes = (u8 *)skb_put(skb, padding);
+	memset(padbytes, 0, padding);
+
+done:
+	map_header->pkt_len = htons(map_datalen + padding);
+	map_header->pad_len = padding & 0x3F;
+
+	return map_header;
+}
+
+/* Deaggregates a single packet
+ * A whole new buffer is allocated for each portion of an aggregated frame.
+ * Caller should keep calling deaggregate() on the source skb until 0 is
+ * returned, indicating that there are no more packets to deaggregate. Caller
+ * is responsible for freeing the original skb.
+ */
+struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb)
+{
+	struct rmnet_map_header *maph;
+	struct sk_buff *skbn;
+	u32 packet_len;
+
+	if (skb->len == 0)
+		return NULL;
+
+	maph = (struct rmnet_map_header *)skb->data;
+	packet_len = ntohs(maph->pkt_len) + sizeof(struct rmnet_map_header);
+
+	if (((int)skb->len - (int)packet_len) < 0)
+		return NULL;
+
+	/* Some hardware can send us empty frames. Catch them */
+	if (ntohs(maph->pkt_len) == 0)
+		return NULL;
+
+	skbn = alloc_skb(packet_len + RMNET_MAP_DEAGGR_SPACING, GFP_ATOMIC);
+	if (!skbn)
+		return NULL;
+
+	skbn->dev = skb->dev;
+	skb_reserve(skbn, RMNET_MAP_DEAGGR_HEADROOM);
+	skb_put(skbn, packet_len);
+	memcpy(skbn->data, skb->data, packet_len);
+	skb_pull(skb, packet_len);
+
+	return skbn;
+}
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_private.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_private.h
new file mode 100644
index 000000000000..7967198fdd90
--- /dev/null
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_private.h
@@ -0,0 +1,44 @@
+/* Copyright (c) 2013-2014, 2016-2017 The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _RMNET_PRIVATE_H_
+#define _RMNET_PRIVATE_H_
+
+#define RMNET_MAX_PACKET_SIZE      16384
+#define RMNET_DFLT_PACKET_SIZE     1500
+#define RMNET_NEEDED_HEADROOM      16
+#define RMNET_TX_QUEUE_LEN         1000
+
+/* Constants */
+#define RMNET_EGRESS_FORMAT__RESERVED__         BIT(0)
+#define RMNET_EGRESS_FORMAT_MAP                 BIT(1)
+#define RMNET_EGRESS_FORMAT_AGGREGATION         BIT(2)
+#define RMNET_EGRESS_FORMAT_MUXING              BIT(3)
+#define RMNET_EGRESS_FORMAT_MAP_CKSUMV3         BIT(4)
+#define RMNET_EGRESS_FORMAT_MAP_CKSUMV4         BIT(5)
+
+#define RMNET_INGRESS_FIX_ETHERNET              BIT(0)
+#define RMNET_INGRESS_FORMAT_MAP                BIT(1)
+#define RMNET_INGRESS_FORMAT_DEAGGREGATION      BIT(2)
+#define RMNET_INGRESS_FORMAT_DEMUXING           BIT(3)
+#define RMNET_INGRESS_FORMAT_MAP_COMMANDS       BIT(4)
+#define RMNET_INGRESS_FORMAT_MAP_CKSUMV3        BIT(5)
+#define RMNET_INGRESS_FORMAT_MAP_CKSUMV4        BIT(6)
+
+/* Pass the frame up the stack with no modifications to skb->dev */
+#define RMNET_EPMODE_NONE (0)
+/* Replace skb->dev to a virtual rmnet device and pass up the stack */
+#define RMNET_EPMODE_VND (1)
+/* Pass the frame directly to another device with dev_queue_xmit() */
+#define RMNET_EPMODE_BRIDGE (2)
+
+#endif /* _RMNET_PRIVATE_H_ */
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c
new file mode 100644
index 000000000000..7f90d5587653
--- /dev/null
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c
@@ -0,0 +1,174 @@
+/* Copyright (c) 2013-2017, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ *
+ * RMNET Data virtual network driver
+ *
+ */
+
+#include <linux/etherdevice.h>
+#include <linux/if_arp.h>
+#include <net/pkt_sched.h>
+#include "rmnet_config.h"
+#include "rmnet_handlers.h"
+#include "rmnet_private.h"
+#include "rmnet_map.h"
+#include "rmnet_vnd.h"
+
+/* RX/TX Fixup */
+
+void rmnet_vnd_rx_fixup(struct sk_buff *skb, struct net_device *dev)
+{
+	dev->stats.rx_packets++;
+	dev->stats.rx_bytes += skb->len;
+}
+
+void rmnet_vnd_tx_fixup(struct sk_buff *skb, struct net_device *dev)
+{
+	dev->stats.tx_packets++;
+	dev->stats.tx_bytes += skb->len;
+}
+
+/* Network Device Operations */
+
+static netdev_tx_t rmnet_vnd_start_xmit(struct sk_buff *skb,
+					struct net_device *dev)
+{
+	struct rmnet_priv *priv;
+
+	priv = netdev_priv(dev);
+	if (priv->local_ep.egress_dev) {
+		rmnet_egress_handler(skb, &priv->local_ep);
+	} else {
+		dev->stats.tx_dropped++;
+		kfree_skb(skb);
+	}
+	return NETDEV_TX_OK;
+}
+
+static int rmnet_vnd_change_mtu(struct net_device *rmnet_dev, int new_mtu)
+{
+	if (new_mtu < 0 || new_mtu > RMNET_MAX_PACKET_SIZE)
+		return -EINVAL;
+
+	rmnet_dev->mtu = new_mtu;
+	return 0;
+}
+
+static int rmnet_vnd_get_iflink(const struct net_device *dev)
+{
+	struct rmnet_priv *priv = netdev_priv(dev);
+
+	return priv->real_dev->ifindex;
+}
+
+static const struct net_device_ops rmnet_vnd_ops = {
+	.ndo_start_xmit = rmnet_vnd_start_xmit,
+	.ndo_change_mtu = rmnet_vnd_change_mtu,
+	.ndo_get_iflink = rmnet_vnd_get_iflink,
+};
+
+/* Called by kernel whenever a new rmnet<n> device is created. Sets MTU,
+ * flags, ARP type, needed headroom, etc...
+ */
+void rmnet_vnd_setup(struct net_device *rmnet_dev)
+{
+	rmnet_dev->netdev_ops = &rmnet_vnd_ops;
+	rmnet_dev->mtu = RMNET_DFLT_PACKET_SIZE;
+	rmnet_dev->needed_headroom = RMNET_NEEDED_HEADROOM;
+	random_ether_addr(rmnet_dev->dev_addr);
+	rmnet_dev->tx_queue_len = RMNET_TX_QUEUE_LEN;
+
+	/* Raw IP mode */
+	rmnet_dev->header_ops = NULL;  /* No header */
+	rmnet_dev->type = ARPHRD_RAWIP;
+	rmnet_dev->hard_header_len = 0;
+	rmnet_dev->flags &= ~(IFF_BROADCAST | IFF_MULTICAST);
+
+	rmnet_dev->needs_free_netdev = true;
+}
+
+/* Exposed API */
+
+int rmnet_vnd_newlink(u8 id, struct net_device *rmnet_dev,
+		      struct rmnet_port *port,
+		      struct net_device *real_dev)
+{
+	struct rmnet_priv *priv;
+	int rc;
+
+	if (port->rmnet_devices[id])
+		return -EINVAL;
+
+	rc = register_netdevice(rmnet_dev);
+	if (!rc) {
+		port->rmnet_devices[id] = rmnet_dev;
+		port->nr_rmnet_devs++;
+
+		rmnet_dev->rtnl_link_ops = &rmnet_link_ops;
+
+		priv = netdev_priv(rmnet_dev);
+		priv->mux_id = id;
+		priv->real_dev = real_dev;
+
+		netdev_dbg(rmnet_dev, "rmnet dev created\n");
+	}
+
+	return rc;
+}
+
+int rmnet_vnd_dellink(u8 id, struct rmnet_port *port)
+{
+	if (id >= RMNET_MAX_LOGICAL_EP || !port->rmnet_devices[id])
+		return -EINVAL;
+
+	port->rmnet_devices[id] = NULL;
+	port->nr_rmnet_devs--;
+	return 0;
+}
+
+u8 rmnet_vnd_get_mux(struct net_device *rmnet_dev)
+{
+	struct rmnet_priv *priv;
+
+	priv = netdev_priv(rmnet_dev);
+	return priv->mux_id;
+}
+
+/* Gets the logical endpoint configuration for a RmNet virtual network device
+ * node. Caller should confirm that devices is a RmNet VND before calling.
+ */
+struct rmnet_endpoint *rmnet_vnd_get_endpoint(struct net_device *rmnet_dev)
+{
+	struct rmnet_priv *priv;
+
+	if (!rmnet_dev)
+		return NULL;
+
+	priv = netdev_priv(rmnet_dev);
+
+	return &priv->local_ep;
+}
+
+int rmnet_vnd_do_flow_control(struct net_device *rmnet_dev, int enable)
+{
+	netdev_dbg(rmnet_dev, "Setting VND TX queue state to %d\n", enable);
+	/* Although we expect similar number of enable/disable
+	 * commands, optimize for the disable. That is more
+	 * latency sensitive than enable
+	 */
+	if (unlikely(enable))
+		netif_wake_queue(rmnet_dev);
+	else
+		netif_stop_queue(rmnet_dev);
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.h
new file mode 100644
index 000000000000..8a4042f0f6bf
--- /dev/null
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.h
@@ -0,0 +1,29 @@
+/* Copyright (c) 2013-2017, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * RMNET Data Virtual Network Device APIs
+ *
+ */
+
+#ifndef _RMNET_VND_H_
+#define _RMNET_VND_H_
+
+int rmnet_vnd_do_flow_control(struct net_device *dev, int enable);
+struct rmnet_endpoint *rmnet_vnd_get_endpoint(struct net_device *dev);
+int rmnet_vnd_newlink(u8 id, struct net_device *rmnet_dev,
+		      struct rmnet_port *port,
+		      struct net_device *real_dev);
+int rmnet_vnd_dellink(u8 id, struct rmnet_port *port);
+void rmnet_vnd_rx_fixup(struct sk_buff *skb, struct net_device *dev);
+void rmnet_vnd_tx_fixup(struct sk_buff *skb, struct net_device *dev);
+u8 rmnet_vnd_get_mux(struct net_device *rmnet_dev);
+void rmnet_vnd_setup(struct net_device *dev);
+#endif /* _RMNET_VND_H_ */
diff --git a/drivers/net/ethernet/realtek/r8169.c b/drivers/net/ethernet/realtek/r8169.c
index bd07a15d3b7c..e03fcf914690 100644
--- a/drivers/net/ethernet/realtek/r8169.c
+++ b/drivers/net/ethernet/realtek/r8169.c
@@ -6863,8 +6863,7 @@ static void rtl8169_tx_clear_range(struct rtl8169_private *tp, u32 start,
 			rtl8169_unmap_tx_skb(&tp->pci_dev->dev, tx_skb,
 					     tp->TxDescArray + entry);
 			if (skb) {
-				tp->dev->stats.tx_dropped++;
-				dev_kfree_skb_any(skb);
+				dev_consume_skb_any(skb);
 				tx_skb->skb = NULL;
 			}
 		}
@@ -7319,7 +7318,7 @@ static void rtl_tx(struct net_device *dev, struct rtl8169_private *tp)
 			tp->tx_stats.packets++;
 			tp->tx_stats.bytes += tx_skb->skb->len;
 			u64_stats_update_end(&tp->tx_stats.syncp);
-			dev_kfree_skb_any(tx_skb->skb);
+			dev_consume_skb_any(tx_skb->skb);
 			tx_skb->skb = NULL;
 		}
 		dirty_tx++;
diff --git a/drivers/net/ethernet/renesas/ravb.h b/drivers/net/ethernet/renesas/ravb.h
index 0525bd696d5d..96a27b00c90e 100644
--- a/drivers/net/ethernet/renesas/ravb.h
+++ b/drivers/net/ethernet/renesas/ravb.h
@@ -991,6 +991,7 @@ struct ravb_private {
 	struct net_device *ndev;
 	struct platform_device *pdev;
 	void __iomem *addr;
+	struct clk *clk;
 	struct mdiobb_ctrl mdiobb;
 	u32 num_rx_ring[NUM_RX_QUEUE];
 	u32 num_tx_ring[NUM_TX_QUEUE];
@@ -1033,6 +1034,7 @@ struct ravb_private {
 
 	unsigned no_avb_link:1;
 	unsigned avb_link_active_low:1;
+	unsigned wol_enabled:1;
 };
 
 static inline u32 ravb_read(struct net_device *ndev, enum ravb_reg reg)
diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c
index 5931e859876c..fdf30bfa403b 100644
--- a/drivers/net/ethernet/renesas/ravb_main.c
+++ b/drivers/net/ethernet/renesas/ravb_main.c
@@ -680,6 +680,9 @@ static void ravb_emac_interrupt_unlocked(struct net_device *ndev)
 
 	ecsr = ravb_read(ndev, ECSR);
 	ravb_write(ndev, ecsr, ECSR);	/* clear interrupt */
+
+	if (ecsr & ECSR_MPD)
+		pm_wakeup_event(&priv->pdev->dev, 0);
 	if (ecsr & ECSR_ICD)
 		ndev->stats.tx_carrier_errors++;
 	if (ecsr & ECSR_LCHNG) {
@@ -1330,6 +1333,33 @@ static int ravb_get_ts_info(struct net_device *ndev,
 	return 0;
 }
 
+static void ravb_get_wol(struct net_device *ndev, struct ethtool_wolinfo *wol)
+{
+	struct ravb_private *priv = netdev_priv(ndev);
+
+	wol->supported = 0;
+	wol->wolopts = 0;
+
+	if (priv->clk) {
+		wol->supported = WAKE_MAGIC;
+		wol->wolopts = priv->wol_enabled ? WAKE_MAGIC : 0;
+	}
+}
+
+static int ravb_set_wol(struct net_device *ndev, struct ethtool_wolinfo *wol)
+{
+	struct ravb_private *priv = netdev_priv(ndev);
+
+	if (!priv->clk || wol->wolopts & ~WAKE_MAGIC)
+		return -EOPNOTSUPP;
+
+	priv->wol_enabled = !!(wol->wolopts & WAKE_MAGIC);
+
+	device_set_wakeup_enable(&priv->pdev->dev, priv->wol_enabled);
+
+	return 0;
+}
+
 static const struct ethtool_ops ravb_ethtool_ops = {
 	.nway_reset		= ravb_nway_reset,
 	.get_msglevel		= ravb_get_msglevel,
@@ -1343,6 +1373,8 @@ static const struct ethtool_ops ravb_ethtool_ops = {
 	.get_ts_info		= ravb_get_ts_info,
 	.get_link_ksettings	= ravb_get_link_ksettings,
 	.set_link_ksettings	= ravb_set_link_ksettings,
+	.get_wol		= ravb_get_wol,
+	.set_wol		= ravb_set_wol,
 };
 
 static inline int ravb_hook_irq(unsigned int irq, irq_handler_t handler,
@@ -2041,6 +2073,11 @@ static int ravb_probe(struct platform_device *pdev)
 
 	priv->chip_id = chip_id;
 
+	/* Get clock, if not found that's OK but Wake-On-Lan is unavailable */
+	priv->clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(priv->clk))
+		priv->clk = NULL;
+
 	/* Set function */
 	ndev->netdev_ops = &ravb_netdev_ops;
 	ndev->ethtool_ops = &ravb_ethtool_ops;
@@ -2107,6 +2144,9 @@ static int ravb_probe(struct platform_device *pdev)
 	if (error)
 		goto out_napi_del;
 
+	if (priv->clk)
+		device_set_wakeup_capable(&pdev->dev, 1);
+
 	/* Print device information */
 	netdev_info(ndev, "Base address at %#x, %pM, IRQ %d.\n",
 		    (u32)ndev->base_addr, ndev->dev_addr, ndev->irq);
@@ -2160,15 +2200,66 @@ static int ravb_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static int ravb_wol_setup(struct net_device *ndev)
+{
+	struct ravb_private *priv = netdev_priv(ndev);
+
+	/* Disable interrupts by clearing the interrupt masks. */
+	ravb_write(ndev, 0, RIC0);
+	ravb_write(ndev, 0, RIC2);
+	ravb_write(ndev, 0, TIC);
+
+	/* Only allow ECI interrupts */
+	synchronize_irq(priv->emac_irq);
+	napi_disable(&priv->napi[RAVB_NC]);
+	napi_disable(&priv->napi[RAVB_BE]);
+	ravb_write(ndev, ECSIPR_MPDIP, ECSIPR);
+
+	/* Enable MagicPacket */
+	ravb_modify(ndev, ECMR, ECMR_MPDE, ECMR_MPDE);
+
+	/* Increased clock usage so device won't be suspended */
+	clk_enable(priv->clk);
+
+	return enable_irq_wake(priv->emac_irq);
+}
+
+static int ravb_wol_restore(struct net_device *ndev)
+{
+	struct ravb_private *priv = netdev_priv(ndev);
+	int ret;
+
+	napi_enable(&priv->napi[RAVB_NC]);
+	napi_enable(&priv->napi[RAVB_BE]);
+
+	/* Disable MagicPacket */
+	ravb_modify(ndev, ECMR, ECMR_MPDE, 0);
+
+	ret = ravb_close(ndev);
+	if (ret < 0)
+		return ret;
+
+	/* Restore clock usage count */
+	clk_disable(priv->clk);
+
+	return disable_irq_wake(priv->emac_irq);
+}
+
 static int __maybe_unused ravb_suspend(struct device *dev)
 {
 	struct net_device *ndev = dev_get_drvdata(dev);
-	int ret = 0;
+	struct ravb_private *priv = netdev_priv(ndev);
+	int ret;
 
-	if (netif_running(ndev)) {
-		netif_device_detach(ndev);
+	if (!netif_running(ndev))
+		return 0;
+
+	netif_device_detach(ndev);
+
+	if (priv->wol_enabled)
+		ret = ravb_wol_setup(ndev);
+	else
 		ret = ravb_close(ndev);
-	}
 
 	return ret;
 }
@@ -2179,6 +2270,33 @@ static int __maybe_unused ravb_resume(struct device *dev)
 	struct ravb_private *priv = netdev_priv(ndev);
 	int ret = 0;
 
+	if (priv->wol_enabled) {
+		/* Reduce the usecount of the clock to zero and then
+		 * restore it to its original value. This is done to force
+		 * the clock to be re-enabled which is a workaround
+		 * for renesas-cpg-mssr driver which do not enable clocks
+		 * when resuming from PSCI suspend/resume.
+		 *
+		 * Without this workaround the driver fails to communicate
+		 * with the hardware if WoL was enabled when the system
+		 * entered PSCI suspend. This is due to that if WoL is enabled
+		 * we explicitly keep the clock from being turned off when
+		 * suspending, but in PSCI sleep power is cut so the clock
+		 * is disabled anyhow, the clock driver is not aware of this
+		 * so the clock is not turned back on when resuming.
+		 *
+		 * TODO: once the renesas-cpg-mssr suspend/resume is working
+		 *       this clock dance should be removed.
+		 */
+		clk_disable(priv->clk);
+		clk_disable(priv->clk);
+		clk_enable(priv->clk);
+		clk_enable(priv->clk);
+
+		/* Set reset mode to rearm the WoL logic */
+		ravb_write(ndev, CCC_OPC_RESET, CCC);
+	}
+
 	/* All register have been reset to default values.
 	 * Restore all registers which where setup at probe time and
 	 * reopen device if it was running before system suspended.
@@ -2202,6 +2320,11 @@ static int __maybe_unused ravb_resume(struct device *dev)
 	ravb_write(ndev, priv->desc_bat_dma, DBAT);
 
 	if (netif_running(ndev)) {
+		if (priv->wol_enabled) {
+			ret = ravb_wol_restore(ndev);
+			if (ret)
+				return ret;
+		}
 		ret = ravb_open(ndev);
 		if (ret < 0)
 			return ret;
diff --git a/drivers/net/ethernet/renesas/sh_eth.c b/drivers/net/ethernet/renesas/sh_eth.c
index d2dc0a8ef305..d2e88a30f57b 100644
--- a/drivers/net/ethernet/renesas/sh_eth.c
+++ b/drivers/net/ethernet/renesas/sh_eth.c
@@ -3402,7 +3402,7 @@ static const struct dev_pm_ops sh_eth_dev_pm_ops = {
 #define SH_ETH_PM_OPS NULL
 #endif
 
-static struct platform_device_id sh_eth_id_table[] = {
+static const struct platform_device_id sh_eth_id_table[] = {
 	{ "sh7619-ether", (kernel_ulong_t)&sh7619_data },
 	{ "sh771x-ether", (kernel_ulong_t)&sh771x_data },
 	{ "sh7724-ether", (kernel_ulong_t)&sh7724_data },
diff --git a/drivers/net/ethernet/rocker/rocker_main.c b/drivers/net/ethernet/rocker/rocker_main.c
index b1e5c07099fa..fc8f8bdf6579 100644
--- a/drivers/net/ethernet/rocker/rocker_main.c
+++ b/drivers/net/ethernet/rocker/rocker_main.c
@@ -34,6 +34,7 @@
 #include <net/netevent.h>
 #include <net/arp.h>
 #include <net/fib_rules.h>
+#include <net/fib_notifier.h>
 #include <linux/io-64-nonatomic-lo-hi.h>
 #include <generated/utsrelease.h>
 
@@ -2191,6 +2192,10 @@ static int rocker_router_fib_event(struct notifier_block *nb,
 {
 	struct rocker *rocker = container_of(nb, struct rocker, fib_nb);
 	struct rocker_fib_event_work *fib_work;
+	struct fib_notifier_info *info = ptr;
+
+	if (info->family != AF_INET)
+		return NOTIFY_DONE;
 
 	fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
 	if (WARN_ON(!fib_work))
diff --git a/drivers/net/ethernet/rocker/rocker_ofdpa.c b/drivers/net/ethernet/rocker/rocker_ofdpa.c
index 600e30e8f0be..0653b70723a3 100644
--- a/drivers/net/ethernet/rocker/rocker_ofdpa.c
+++ b/drivers/net/ethernet/rocker/rocker_ofdpa.c
@@ -1177,7 +1177,7 @@ static int ofdpa_group_l2_fan_out(struct ofdpa_port *ofdpa_port,
 	entry->group_id = group_id;
 	entry->group_count = group_count;
 
-	entry->group_ids = kcalloc(flags, group_count, sizeof(u32));
+	entry->group_ids = kcalloc(group_count, sizeof(u32), GFP_KERNEL);
 	if (!entry->group_ids) {
 		kfree(entry);
 		return -ENOMEM;
@@ -1456,7 +1456,7 @@ static int ofdpa_port_vlan_flood_group(struct ofdpa_port *ofdpa_port,
 	int err = 0;
 	int i;
 
-	group_ids = kcalloc(flags, port_count, sizeof(u32));
+	group_ids = kcalloc(port_count, sizeof(u32), GFP_KERNEL);
 	if (!group_ids)
 		return -ENOMEM;
 
@@ -2761,7 +2761,7 @@ static int ofdpa_fib4_add(struct rocker *rocker,
 				  fen_info->tb_id, 0);
 	if (err)
 		return err;
-	fib_info_offload_inc(fen_info->fi);
+	fen_info->fi->fib_nh->nh_flags |= RTNH_F_OFFLOAD;
 	return 0;
 }
 
@@ -2776,7 +2776,7 @@ static int ofdpa_fib4_del(struct rocker *rocker,
 	ofdpa_port = ofdpa_port_dev_lower_find(fen_info->fi->fib_dev, rocker);
 	if (!ofdpa_port)
 		return 0;
-	fib_info_offload_dec(fen_info->fi);
+	fen_info->fi->fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
 	return ofdpa_port_fib_ipv4(ofdpa_port, htonl(fen_info->dst),
 				   fen_info->dst_len, fen_info->fi,
 				   fen_info->tb_id, OFDPA_OP_FLAG_REMOVE);
@@ -2803,7 +2803,7 @@ static void ofdpa_fib4_abort(struct rocker *rocker)
 						       rocker);
 		if (!ofdpa_port)
 			continue;
-		fib_info_offload_dec(flow_entry->fi);
+		flow_entry->fi->fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
 		ofdpa_flow_tbl_del(ofdpa_port, OFDPA_OP_FLAG_REMOVE,
 				   flow_entry);
 	}
diff --git a/drivers/net/ethernet/samsung/sxgbe/sxgbe_platform.c b/drivers/net/ethernet/samsung/sxgbe/sxgbe_platform.c
index 73427e29df2a..fbd00cb0cb7d 100644
--- a/drivers/net/ethernet/samsung/sxgbe/sxgbe_platform.c
+++ b/drivers/net/ethernet/samsung/sxgbe/sxgbe_platform.c
@@ -47,6 +47,8 @@ static int sxgbe_probe_config_dt(struct platform_device *pdev,
 	plat->mdio_bus_data = devm_kzalloc(&pdev->dev,
 					   sizeof(*plat->mdio_bus_data),
 					   GFP_KERNEL);
+	if (!plat->mdio_bus_data)
+		return -ENOMEM;
 
 	dma_cfg = devm_kzalloc(&pdev->dev, sizeof(*dma_cfg), GFP_KERNEL);
 	if (!dma_cfg)
diff --git a/drivers/net/ethernet/seeq/sgiseeq.c b/drivers/net/ethernet/seeq/sgiseeq.c
index 70347720fdf9..573691bc3b71 100644
--- a/drivers/net/ethernet/seeq/sgiseeq.c
+++ b/drivers/net/ethernet/seeq/sgiseeq.c
@@ -737,8 +737,8 @@ static int sgiseeq_probe(struct platform_device *pdev)
 	sp = netdev_priv(dev);
 
 	/* Make private data page aligned */
-	sr = dma_alloc_noncoherent(&pdev->dev, sizeof(*sp->srings),
-				&sp->srings_dma, GFP_KERNEL);
+	sr = dma_alloc_attrs(&pdev->dev, sizeof(*sp->srings), &sp->srings_dma,
+			     GFP_KERNEL, DMA_ATTR_NON_CONSISTENT);
 	if (!sr) {
 		printk(KERN_ERR "Sgiseeq: Page alloc failed, aborting.\n");
 		err = -ENOMEM;
@@ -813,8 +813,8 @@ static int sgiseeq_remove(struct platform_device *pdev)
 	struct sgiseeq_private *sp = netdev_priv(dev);
 
 	unregister_netdev(dev);
-	dma_free_noncoherent(&pdev->dev, sizeof(*sp->srings), sp->srings,
-			     sp->srings_dma);
+	dma_free_attrs(&pdev->dev, sizeof(*sp->srings), sp->srings,
+		       sp->srings_dma, DMA_ATTR_NON_CONSISTENT);
 	free_netdev(dev);
 
 	return 0;
diff --git a/drivers/net/ethernet/sfc/efx.h b/drivers/net/ethernet/sfc/efx.h
index fcea9371ab7f..d407adf59610 100644
--- a/drivers/net/ethernet/sfc/efx.h
+++ b/drivers/net/ethernet/sfc/efx.h
@@ -32,8 +32,8 @@ netdev_tx_t efx_hard_start_xmit(struct sk_buff *skb,
 				struct net_device *net_dev);
 netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb);
 void efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index);
-int efx_setup_tc(struct net_device *net_dev, u32 handle, u32 chain_index,
-		 __be16 proto, struct tc_to_netdev *tc);
+int efx_setup_tc(struct net_device *net_dev, enum tc_setup_type type,
+		 void *type_data);
 unsigned int efx_tx_max_skb_descs(struct efx_nic *efx);
 extern unsigned int efx_piobuf_size;
 extern bool efx_separate_tx_channels;
diff --git a/drivers/net/ethernet/sfc/falcon/efx.h b/drivers/net/ethernet/sfc/falcon/efx.h
index e5a7a40cc8b6..4f3bb30661ea 100644
--- a/drivers/net/ethernet/sfc/falcon/efx.h
+++ b/drivers/net/ethernet/sfc/falcon/efx.h
@@ -32,8 +32,8 @@ netdev_tx_t ef4_hard_start_xmit(struct sk_buff *skb,
 				struct net_device *net_dev);
 netdev_tx_t ef4_enqueue_skb(struct ef4_tx_queue *tx_queue, struct sk_buff *skb);
 void ef4_xmit_done(struct ef4_tx_queue *tx_queue, unsigned int index);
-int ef4_setup_tc(struct net_device *net_dev, u32 handle, u32 chain_index,
-		 __be16 proto, struct tc_to_netdev *tc);
+int ef4_setup_tc(struct net_device *net_dev, enum tc_setup_type type,
+		 void *type_data);
 unsigned int ef4_tx_max_skb_descs(struct ef4_nic *efx);
 extern bool ef4_separate_tx_channels;
 
diff --git a/drivers/net/ethernet/sfc/falcon/tx.c b/drivers/net/ethernet/sfc/falcon/tx.c
index f1520a404ac6..6a75f4140a4b 100644
--- a/drivers/net/ethernet/sfc/falcon/tx.c
+++ b/drivers/net/ethernet/sfc/falcon/tx.c
@@ -425,24 +425,25 @@ void ef4_init_tx_queue_core_txq(struct ef4_tx_queue *tx_queue)
 				     efx->n_tx_channels : 0));
 }
 
-int ef4_setup_tc(struct net_device *net_dev, u32 handle, u32 chain_index,
-		 __be16 proto, struct tc_to_netdev *ntc)
+int ef4_setup_tc(struct net_device *net_dev, enum tc_setup_type type,
+		 void *type_data)
 {
 	struct ef4_nic *efx = netdev_priv(net_dev);
+	struct tc_mqprio_qopt *mqprio = type_data;
 	struct ef4_channel *channel;
 	struct ef4_tx_queue *tx_queue;
 	unsigned tc, num_tc;
 	int rc;
 
-	if (ntc->type != TC_SETUP_MQPRIO)
-		return -EINVAL;
+	if (type != TC_SETUP_MQPRIO)
+		return -EOPNOTSUPP;
 
-	num_tc = ntc->mqprio->num_tc;
+	num_tc = mqprio->num_tc;
 
 	if (ef4_nic_rev(efx) < EF4_REV_FALCON_B0 || num_tc > EF4_MAX_TX_TC)
 		return -EINVAL;
 
-	ntc->mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
+	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
 
 	if (num_tc == net_dev->num_tc)
 		return 0;
diff --git a/drivers/net/ethernet/sfc/mcdi_port.c b/drivers/net/ethernet/sfc/mcdi_port.c
index c905971c5f3a..c7407d129c7d 100644
--- a/drivers/net/ethernet/sfc/mcdi_port.c
+++ b/drivers/net/ethernet/sfc/mcdi_port.c
@@ -746,59 +746,171 @@ static const char *efx_mcdi_phy_test_name(struct efx_nic *efx,
 	return NULL;
 }
 
-#define SFP_PAGE_SIZE	128
-#define SFP_NUM_PAGES	2
-static int efx_mcdi_phy_get_module_eeprom(struct efx_nic *efx,
-					  struct ethtool_eeprom *ee, u8 *data)
+#define SFP_PAGE_SIZE		128
+#define SFF_DIAG_TYPE_OFFSET	92
+#define SFF_DIAG_ADDR_CHANGE	BIT(2)
+#define SFF_8079_NUM_PAGES	2
+#define SFF_8472_NUM_PAGES	4
+#define SFF_8436_NUM_PAGES	5
+#define SFF_DMT_LEVEL_OFFSET	94
+
+/** efx_mcdi_phy_get_module_eeprom_page() - Get a single page of module eeprom
+ * @efx:	NIC context
+ * @page:	EEPROM page number
+ * @data:	Destination data pointer
+ * @offset:	Offset in page to copy from in to data
+ * @space:	Space available in data
+ *
+ * Return:
+ *   >=0 - amount of data copied
+ *   <0  - error
+ */
+static int efx_mcdi_phy_get_module_eeprom_page(struct efx_nic *efx,
+					       unsigned int page,
+					       u8 *data, ssize_t offset,
+					       ssize_t space)
 {
 	MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_PHY_MEDIA_INFO_OUT_LENMAX);
 	MCDI_DECLARE_BUF(inbuf, MC_CMD_GET_PHY_MEDIA_INFO_IN_LEN);
 	size_t outlen;
-	int rc;
 	unsigned int payload_len;
-	unsigned int space_remaining = ee->len;
-	unsigned int page;
-	unsigned int page_off;
 	unsigned int to_copy;
-	u8 *user_data = data;
+	int rc;
 
-	BUILD_BUG_ON(SFP_PAGE_SIZE * SFP_NUM_PAGES != ETH_MODULE_SFF_8079_LEN);
+	if (offset > SFP_PAGE_SIZE)
+		return -EINVAL;
 
-	page_off = ee->offset % SFP_PAGE_SIZE;
-	page = ee->offset / SFP_PAGE_SIZE;
+	to_copy = min(space, SFP_PAGE_SIZE - offset);
 
-	while (space_remaining && (page < SFP_NUM_PAGES)) {
-		MCDI_SET_DWORD(inbuf, GET_PHY_MEDIA_INFO_IN_PAGE, page);
+	MCDI_SET_DWORD(inbuf, GET_PHY_MEDIA_INFO_IN_PAGE, page);
+	rc = efx_mcdi_rpc_quiet(efx, MC_CMD_GET_PHY_MEDIA_INFO,
+				inbuf, sizeof(inbuf),
+				outbuf, sizeof(outbuf),
+				&outlen);
 
-		rc = efx_mcdi_rpc(efx, MC_CMD_GET_PHY_MEDIA_INFO,
-				  inbuf, sizeof(inbuf),
-				  outbuf, sizeof(outbuf),
-				  &outlen);
-		if (rc)
-			return rc;
+	if (rc)
+		return rc;
+
+	if (outlen < (MC_CMD_GET_PHY_MEDIA_INFO_OUT_DATA_OFST +
+			SFP_PAGE_SIZE))
+		return -EIO;
+
+	payload_len = MCDI_DWORD(outbuf, GET_PHY_MEDIA_INFO_OUT_DATALEN);
+	if (payload_len != SFP_PAGE_SIZE)
+		return -EIO;
+
+	memcpy(data, MCDI_PTR(outbuf, GET_PHY_MEDIA_INFO_OUT_DATA) + offset,
+	       to_copy);
+
+	return to_copy;
+}
+
+static int efx_mcdi_phy_get_module_eeprom_byte(struct efx_nic *efx,
+					       unsigned int page,
+					       u8 byte)
+{
+	int rc;
+	u8 data;
+
+	rc = efx_mcdi_phy_get_module_eeprom_page(efx, page, &data, byte, 1);
+	if (rc == 1)
+		return data;
+
+	return rc;
+}
+
+static int efx_mcdi_phy_diag_type(struct efx_nic *efx)
+{
+	/* Page zero of the EEPROM includes the diagnostic type at byte 92. */
+	return efx_mcdi_phy_get_module_eeprom_byte(efx, 0,
+						   SFF_DIAG_TYPE_OFFSET);
+}
+
+static int efx_mcdi_phy_sff_8472_level(struct efx_nic *efx)
+{
+	/* Page zero of the EEPROM includes the DMT level at byte 94. */
+	return efx_mcdi_phy_get_module_eeprom_byte(efx, 0,
+						   SFF_DMT_LEVEL_OFFSET);
+}
+
+static u32 efx_mcdi_phy_module_type(struct efx_nic *efx)
+{
+	struct efx_mcdi_phy_data *phy_data = efx->phy_data;
 
-		if (outlen < (MC_CMD_GET_PHY_MEDIA_INFO_OUT_DATA_OFST +
-			      SFP_PAGE_SIZE))
-			return -EIO;
+	if (phy_data->media != MC_CMD_MEDIA_QSFP_PLUS)
+		return phy_data->media;
 
-		payload_len = MCDI_DWORD(outbuf,
-					 GET_PHY_MEDIA_INFO_OUT_DATALEN);
-		if (payload_len != SFP_PAGE_SIZE)
-			return -EIO;
+	/* A QSFP+ NIC may actually have an SFP+ module attached.
+	 * The ID is page 0, byte 0.
+	 */
+	switch (efx_mcdi_phy_get_module_eeprom_byte(efx, 0, 0)) {
+	case 0x3:
+		return MC_CMD_MEDIA_SFP_PLUS;
+	case 0xc:
+	case 0xd:
+		return MC_CMD_MEDIA_QSFP_PLUS;
+	default:
+		return 0;
+	}
+}
 
-		/* Copy as much as we can into data */
-		payload_len -= page_off;
-		to_copy = (space_remaining < payload_len) ?
-			space_remaining : payload_len;
+static int efx_mcdi_phy_get_module_eeprom(struct efx_nic *efx,
+					  struct ethtool_eeprom *ee, u8 *data)
+{
+	int rc;
+	ssize_t space_remaining = ee->len;
+	unsigned int page_off;
+	bool ignore_missing;
+	int num_pages;
+	int page;
 
-		memcpy(user_data,
-		       MCDI_PTR(outbuf, GET_PHY_MEDIA_INFO_OUT_DATA) + page_off,
-		       to_copy);
+	switch (efx_mcdi_phy_module_type(efx)) {
+	case MC_CMD_MEDIA_SFP_PLUS:
+		num_pages = efx_mcdi_phy_sff_8472_level(efx) > 0 ?
+				SFF_8472_NUM_PAGES : SFF_8079_NUM_PAGES;
+		page = 0;
+		ignore_missing = false;
+		break;
+	case MC_CMD_MEDIA_QSFP_PLUS:
+		num_pages = SFF_8436_NUM_PAGES;
+		page = -1; /* We obtain the lower page by asking for -1. */
+		ignore_missing = true; /* Ignore missing pages after page 0. */
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
 
-		space_remaining -= to_copy;
-		user_data += to_copy;
-		page_off = 0;
-		page++;
+	page_off = ee->offset % SFP_PAGE_SIZE;
+	page += ee->offset / SFP_PAGE_SIZE;
+
+	while (space_remaining && (page < num_pages)) {
+		rc = efx_mcdi_phy_get_module_eeprom_page(efx, page,
+							 data, page_off,
+							 space_remaining);
+
+		if (rc > 0) {
+			space_remaining -= rc;
+			data += rc;
+			page_off = 0;
+			page++;
+		} else if (rc == 0) {
+			space_remaining = 0;
+		} else if (ignore_missing && (page > 0)) {
+			int intended_size = SFP_PAGE_SIZE - page_off;
+
+			space_remaining -= intended_size;
+			if (space_remaining < 0) {
+				space_remaining = 0;
+			} else {
+				memset(data, 0, intended_size);
+				data += intended_size;
+				page_off = 0;
+				page++;
+				rc = 0;
+			}
+		} else {
+			return rc;
+		}
 	}
 
 	return 0;
@@ -807,16 +919,42 @@ static int efx_mcdi_phy_get_module_eeprom(struct efx_nic *efx,
 static int efx_mcdi_phy_get_module_info(struct efx_nic *efx,
 					struct ethtool_modinfo *modinfo)
 {
-	struct efx_mcdi_phy_data *phy_cfg = efx->phy_data;
+	int sff_8472_level;
+	int diag_type;
 
-	switch (phy_cfg->media) {
+	switch (efx_mcdi_phy_module_type(efx)) {
 	case MC_CMD_MEDIA_SFP_PLUS:
-		modinfo->type = ETH_MODULE_SFF_8079;
-		modinfo->eeprom_len = ETH_MODULE_SFF_8079_LEN;
-		return 0;
+		sff_8472_level = efx_mcdi_phy_sff_8472_level(efx);
+
+		/* If we can't read the diagnostics level we have none. */
+		if (sff_8472_level < 0)
+			return -EOPNOTSUPP;
+
+		/* Check if this module requires the (unsupported) address
+		 * change operation.
+		 */
+		diag_type = efx_mcdi_phy_diag_type(efx);
+
+		if ((sff_8472_level == 0) ||
+		    (diag_type & SFF_DIAG_ADDR_CHANGE)) {
+			modinfo->type = ETH_MODULE_SFF_8079;
+			modinfo->eeprom_len = ETH_MODULE_SFF_8079_LEN;
+		} else {
+			modinfo->type = ETH_MODULE_SFF_8472;
+			modinfo->eeprom_len = ETH_MODULE_SFF_8472_LEN;
+		}
+		break;
+
+	case MC_CMD_MEDIA_QSFP_PLUS:
+		modinfo->type = ETH_MODULE_SFF_8436;
+		modinfo->eeprom_len = ETH_MODULE_SFF_8436_LEN;
+		break;
+
 	default:
 		return -EOPNOTSUPP;
 	}
+
+	return 0;
 }
 
 static const struct efx_phy_operations efx_mcdi_phy_ops = {
@@ -938,7 +1076,6 @@ enum efx_stats_action {
 static int efx_mcdi_mac_stats(struct efx_nic *efx,
 			      enum efx_stats_action action, int clear)
 {
-	struct efx_ef10_nic_data *nic_data = efx->nic_data;
 	MCDI_DECLARE_BUF(inbuf, MC_CMD_MAC_STATS_IN_LEN);
 	int rc;
 	int change = action == EFX_STATS_PULL ? 0 : 1;
@@ -960,7 +1097,12 @@ static int efx_mcdi_mac_stats(struct efx_nic *efx,
 			      MAC_STATS_IN_PERIODIC_NOEVENT, 1,
 			      MAC_STATS_IN_PERIOD_MS, period);
 	MCDI_SET_DWORD(inbuf, MAC_STATS_IN_DMA_LEN, dma_len);
-	MCDI_SET_DWORD(inbuf, MAC_STATS_IN_PORT_ID, nic_data->vport_id);
+
+	if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0) {
+		struct efx_ef10_nic_data *nic_data = efx->nic_data;
+
+		MCDI_SET_DWORD(inbuf, MAC_STATS_IN_PORT_ID, nic_data->vport_id);
+	}
 
 	rc = efx_mcdi_rpc_quiet(efx, MC_CMD_MAC_STATS, inbuf, sizeof(inbuf),
 				NULL, 0, NULL);
diff --git a/drivers/net/ethernet/sfc/tx.c b/drivers/net/ethernet/sfc/tx.c
index 02d41eb4a8e9..32bf1fecf864 100644
--- a/drivers/net/ethernet/sfc/tx.c
+++ b/drivers/net/ethernet/sfc/tx.c
@@ -653,24 +653,25 @@ void efx_init_tx_queue_core_txq(struct efx_tx_queue *tx_queue)
 				     efx->n_tx_channels : 0));
 }
 
-int efx_setup_tc(struct net_device *net_dev, u32 handle, u32 chain_index,
-		 __be16 proto, struct tc_to_netdev *ntc)
+int efx_setup_tc(struct net_device *net_dev, enum tc_setup_type type,
+		 void *type_data)
 {
 	struct efx_nic *efx = netdev_priv(net_dev);
+	struct tc_mqprio_qopt *mqprio = type_data;
 	struct efx_channel *channel;
 	struct efx_tx_queue *tx_queue;
 	unsigned tc, num_tc;
 	int rc;
 
-	if (ntc->type != TC_SETUP_MQPRIO)
-		return -EINVAL;
+	if (type != TC_SETUP_MQPRIO)
+		return -EOPNOTSUPP;
 
-	num_tc = ntc->mqprio->num_tc;
+	num_tc = mqprio->num_tc;
 
 	if (num_tc > EFX_MAX_TX_TC)
 		return -EINVAL;
 
-	ntc->mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
+	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
 
 	if (num_tc == net_dev->num_tc)
 		return 0;
diff --git a/drivers/net/ethernet/smsc/smsc911x.c b/drivers/net/ethernet/smsc/smsc911x.c
index 0b6a39b003a4..012fb66eed8d 100644
--- a/drivers/net/ethernet/smsc/smsc911x.c
+++ b/drivers/net/ethernet/smsc/smsc911x.c
@@ -2595,6 +2595,11 @@ static int smsc911x_suspend(struct device *dev)
 	struct net_device *ndev = dev_get_drvdata(dev);
 	struct smsc911x_data *pdata = netdev_priv(ndev);
 
+	if (netif_running(ndev)) {
+		netif_stop_queue(ndev);
+		netif_device_detach(ndev);
+	}
+
 	/* enable wake on LAN, energy detection and the external PME
 	 * signal. */
 	smsc911x_reg_write(pdata, PMT_CTRL,
@@ -2628,7 +2633,15 @@ static int smsc911x_resume(struct device *dev)
 	while (!(smsc911x_reg_read(pdata, PMT_CTRL) & PMT_CTRL_READY_) && --to)
 		udelay(1000);
 
-	return (to == 0) ? -EIO : 0;
+	if (to == 0)
+		return -EIO;
+
+	if (netif_running(ndev)) {
+		netif_device_attach(ndev);
+		netif_start_queue(ndev);
+	}
+
+	return 0;
 }
 
 static const struct dev_pm_ops smsc911x_pm_ops = {
diff --git a/drivers/net/ethernet/stmicro/stmmac/Kconfig b/drivers/net/ethernet/stmicro/stmmac/Kconfig
index 85c0e41f8021..97035766c291 100644
--- a/drivers/net/ethernet/stmicro/stmmac/Kconfig
+++ b/drivers/net/ethernet/stmicro/stmmac/Kconfig
@@ -45,6 +45,15 @@ config DWMAC_GENERIC
 	  platform specific code to function or is using platform
 	  data for setup.
 
+config DWMAC_ANARION
+	tristate "Adaptrum Anarion GMAC support"
+	default ARC
+	depends on OF && (ARC || COMPILE_TEST)
+	help
+	  Support for Adaptrum Anarion GMAC Ethernet controller.
+
+	  This selects the Anarion SoC glue layer support for the stmmac driver.
+
 config DWMAC_IPQ806X
 	tristate "QCA IPQ806x DWMAC support"
 	default ARCH_QCOM
diff --git a/drivers/net/ethernet/stmicro/stmmac/Makefile b/drivers/net/ethernet/stmicro/stmmac/Makefile
index fd4937a7fcab..238307fadcdb 100644
--- a/drivers/net/ethernet/stmicro/stmmac/Makefile
+++ b/drivers/net/ethernet/stmicro/stmmac/Makefile
@@ -7,6 +7,7 @@ stmmac-objs:= stmmac_main.o stmmac_ethtool.o stmmac_mdio.o ring_mode.o	\
 
 # Ordering matters. Generic driver must be last.
 obj-$(CONFIG_STMMAC_PLATFORM)	+= stmmac-platform.o
+obj-$(CONFIG_DWMAC_ANARION)	+= dwmac-anarion.o
 obj-$(CONFIG_DWMAC_IPQ806X)	+= dwmac-ipq806x.o
 obj-$(CONFIG_DWMAC_LPC18XX)	+= dwmac-lpc18xx.o
 obj-$(CONFIG_DWMAC_MESON)	+= dwmac-meson.o dwmac-meson8b.o
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-anarion.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-anarion.c
new file mode 100644
index 000000000000..85ce80c600c7
--- /dev/null
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-anarion.c
@@ -0,0 +1,152 @@
+/*
+ * Adaptrum Anarion DWMAC glue layer
+ *
+ * Copyright (C) 2017, Adaptrum, Inc.
+ * (Written by Alexandru Gagniuc <alex.g at adaptrum.com> for Adaptrum, Inc.)
+ * Licensed under the GPLv2 or (at your option) any later version.
+ */
+
+#include <linux/io.h>
+#include <linux/of.h>
+#include <linux/of_net.h>
+#include <linux/stmmac.h>
+
+#include "stmmac.h"
+#include "stmmac_platform.h"
+
+#define GMAC_RESET_CONTROL_REG		0
+#define GMAC_SW_CONFIG_REG		4
+#define  GMAC_CONFIG_INTF_SEL_MASK	(0x7 << 0)
+#define  GMAC_CONFIG_INTF_RGMII		(0x1 << 0)
+
+struct anarion_gmac {
+	uintptr_t ctl_block;
+	uint32_t phy_intf_sel;
+};
+
+static uint32_t gmac_read_reg(struct anarion_gmac *gmac, uint8_t reg)
+{
+	return readl((void *)(gmac->ctl_block + reg));
+};
+
+static void gmac_write_reg(struct anarion_gmac *gmac, uint8_t reg, uint32_t val)
+{
+	writel(val, (void *)(gmac->ctl_block + reg));
+}
+
+static int anarion_gmac_init(struct platform_device *pdev, void *priv)
+{
+	uint32_t sw_config;
+	struct anarion_gmac *gmac = priv;
+
+	/* Reset logic, configure interface mode, then release reset. SIMPLE! */
+	gmac_write_reg(gmac, GMAC_RESET_CONTROL_REG, 1);
+
+	sw_config = gmac_read_reg(gmac, GMAC_SW_CONFIG_REG);
+	sw_config &= ~GMAC_CONFIG_INTF_SEL_MASK;
+	sw_config |= (gmac->phy_intf_sel & GMAC_CONFIG_INTF_SEL_MASK);
+	gmac_write_reg(gmac, GMAC_SW_CONFIG_REG, sw_config);
+
+	gmac_write_reg(gmac, GMAC_RESET_CONTROL_REG, 0);
+
+	return 0;
+}
+
+static void anarion_gmac_exit(struct platform_device *pdev, void *priv)
+{
+	struct anarion_gmac *gmac = priv;
+
+	gmac_write_reg(gmac, GMAC_RESET_CONTROL_REG, 1);
+}
+
+static struct anarion_gmac *anarion_config_dt(struct platform_device *pdev)
+{
+	int phy_mode;
+	struct resource *res;
+	void __iomem *ctl_block;
+	struct anarion_gmac *gmac;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+	ctl_block = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(ctl_block)) {
+		dev_err(&pdev->dev, "Cannot get reset region (%ld)!\n",
+			PTR_ERR(ctl_block));
+		return ctl_block;
+	}
+
+	gmac = devm_kzalloc(&pdev->dev, sizeof(*gmac), GFP_KERNEL);
+	if (!gmac)
+		return ERR_PTR(-ENOMEM);
+
+	gmac->ctl_block = (uintptr_t)ctl_block;
+
+	phy_mode = of_get_phy_mode(pdev->dev.of_node);
+	switch (phy_mode) {
+	case PHY_INTERFACE_MODE_RGMII:		/* Fall through */
+	case PHY_INTERFACE_MODE_RGMII_ID	/* Fall through */:
+	case PHY_INTERFACE_MODE_RGMII_RXID:	/* Fall through */
+	case PHY_INTERFACE_MODE_RGMII_TXID:
+		gmac->phy_intf_sel = GMAC_CONFIG_INTF_RGMII;
+		break;
+	default:
+		dev_err(&pdev->dev, "Unsupported phy-mode (%d)\n",
+			phy_mode);
+		return ERR_PTR(-ENOTSUPP);
+	}
+
+	return gmac;
+}
+
+static int anarion_dwmac_probe(struct platform_device *pdev)
+{
+	int ret;
+	struct anarion_gmac *gmac;
+	struct plat_stmmacenet_data *plat_dat;
+	struct stmmac_resources stmmac_res;
+
+	ret = stmmac_get_platform_resources(pdev, &stmmac_res);
+	if (ret)
+		return ret;
+
+	gmac = anarion_config_dt(pdev);
+	if (IS_ERR(gmac))
+		return PTR_ERR(gmac);
+
+	plat_dat = stmmac_probe_config_dt(pdev, &stmmac_res.mac);
+	if (IS_ERR(plat_dat))
+		return PTR_ERR(plat_dat);
+
+	plat_dat->init = anarion_gmac_init;
+	plat_dat->exit = anarion_gmac_exit;
+	anarion_gmac_init(pdev, gmac);
+	plat_dat->bsp_priv = gmac;
+
+	ret = stmmac_dvr_probe(&pdev->dev, plat_dat, &stmmac_res);
+	if (ret) {
+		stmmac_remove_config_dt(pdev, plat_dat);
+		return ret;
+	}
+
+	return 0;
+}
+
+static const struct of_device_id anarion_dwmac_match[] = {
+	{ .compatible = "adaptrum,anarion-gmac" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, anarion_dwmac_match);
+
+static struct platform_driver anarion_dwmac_driver = {
+	.probe  = anarion_dwmac_probe,
+	.remove = stmmac_pltfr_remove,
+	.driver = {
+		.name           = "anarion-dwmac",
+		.pm		= &stmmac_pltfr_pm_ops,
+		.of_match_table = anarion_dwmac_match,
+	},
+};
+module_platform_driver(anarion_dwmac_driver);
+
+MODULE_DESCRIPTION("Adaptrum Anarion DWMAC specific glue layer");
+MODULE_AUTHOR("Alexandru Gagniuc <mr.nuke.me@gmail.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c
index 9685555932ea..4404650b32c5 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c
@@ -89,7 +89,7 @@ static int meson8b_init_clk(struct meson8b_dwmac *dwmac)
 	char clk_name[32];
 	const char *clk_div_parents[1];
 	const char *mux_parent_names[MUX_CLK_NUM_PARENTS];
-	static struct clk_div_table clk_25m_div_table[] = {
+	static const struct clk_div_table clk_25m_div_table[] = {
 		{ .val = 0, .div = 5 },
 		{ .val = 1, .div = 10 },
 		{ /* sentinel */ },
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c
index f0df5193f047..99823f54696a 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c
@@ -41,6 +41,7 @@ struct rk_gmac_ops {
 	void (*set_to_rmii)(struct rk_priv_data *bsp_priv);
 	void (*set_rgmii_speed)(struct rk_priv_data *bsp_priv, int speed);
 	void (*set_rmii_speed)(struct rk_priv_data *bsp_priv, int speed);
+	void (*integrated_phy_powerup)(struct rk_priv_data *bsp_priv);
 };
 
 struct rk_priv_data {
@@ -52,6 +53,7 @@ struct rk_priv_data {
 
 	bool clk_enabled;
 	bool clock_input;
+	bool integrated_phy;
 
 	struct clk *clk_mac;
 	struct clk *gmac_clkin;
@@ -61,6 +63,9 @@ struct rk_priv_data {
 	struct clk *clk_mac_refout;
 	struct clk *aclk_mac;
 	struct clk *pclk_mac;
+	struct clk *clk_phy;
+
+	struct reset_control *phy_reset;
 
 	int tx_delay;
 	int rx_delay;
@@ -81,6 +86,8 @@ struct rk_priv_data {
 #define RK3228_GRF_MAC_CON0	0x0900
 #define RK3228_GRF_MAC_CON1	0x0904
 
+#define RK3228_GRF_CON_MUX	0x50
+
 /* RK3228_GRF_MAC_CON0 */
 #define RK3228_GMAC_CLK_RX_DL_CFG(val)	HIWORD_UPDATE(val, 0x7F, 7)
 #define RK3228_GMAC_CLK_TX_DL_CFG(val)	HIWORD_UPDATE(val, 0x7F, 0)
@@ -106,6 +113,9 @@ struct rk_priv_data {
 #define RK3228_GMAC_RXCLK_DLY_ENABLE	GRF_BIT(1)
 #define RK3228_GMAC_RXCLK_DLY_DISABLE	GRF_CLR_BIT(1)
 
+/* RK3228_GRF_COM_MUX */
+#define RK3228_GRF_CON_MUX_GMAC_INTEGRATED_PHY	GRF_BIT(15)
+
 static void rk3228_set_to_rgmii(struct rk_priv_data *bsp_priv,
 				int tx_delay, int rx_delay)
 {
@@ -186,11 +196,18 @@ static void rk3228_set_rmii_speed(struct rk_priv_data *bsp_priv, int speed)
 		dev_err(dev, "unknown speed value for RMII! speed=%d", speed);
 }
 
+static void rk3228_integrated_phy_powerup(struct rk_priv_data *priv)
+{
+	regmap_write(priv->grf, RK3228_GRF_CON_MUX,
+		     RK3228_GRF_CON_MUX_GMAC_INTEGRATED_PHY);
+}
+
 static const struct rk_gmac_ops rk3228_ops = {
 	.set_to_rgmii = rk3228_set_to_rgmii,
 	.set_to_rmii = rk3228_set_to_rmii,
 	.set_rgmii_speed = rk3228_set_rgmii_speed,
 	.set_rmii_speed = rk3228_set_rmii_speed,
+	.integrated_phy_powerup =  rk3228_integrated_phy_powerup,
 };
 
 #define RK3288_GRF_SOC_CON1	0x0248
@@ -306,6 +323,8 @@ static const struct rk_gmac_ops rk3288_ops = {
 
 #define RK3328_GRF_MAC_CON0	0x0900
 #define RK3328_GRF_MAC_CON1	0x0904
+#define RK3328_GRF_MAC_CON2	0x0908
+#define RK3328_GRF_MACPHY_CON1	0xb04
 
 /* RK3328_GRF_MAC_CON0 */
 #define RK3328_GMAC_CLK_RX_DL_CFG(val)	HIWORD_UPDATE(val, 0x7F, 7)
@@ -332,6 +351,9 @@ static const struct rk_gmac_ops rk3288_ops = {
 #define RK3328_GMAC_RXCLK_DLY_ENABLE	GRF_BIT(1)
 #define RK3328_GMAC_RXCLK_DLY_DISABLE	GRF_CLR_BIT(0)
 
+/* RK3328_GRF_MACPHY_CON1 */
+#define RK3328_MACPHY_RMII_MODE		GRF_BIT(9)
+
 static void rk3328_set_to_rgmii(struct rk_priv_data *bsp_priv,
 				int tx_delay, int rx_delay)
 {
@@ -356,18 +378,19 @@ static void rk3328_set_to_rgmii(struct rk_priv_data *bsp_priv,
 static void rk3328_set_to_rmii(struct rk_priv_data *bsp_priv)
 {
 	struct device *dev = &bsp_priv->pdev->dev;
+	unsigned int reg;
 
 	if (IS_ERR(bsp_priv->grf)) {
 		dev_err(dev, "Missing rockchip,grf property\n");
 		return;
 	}
 
-	regmap_write(bsp_priv->grf, RK3328_GRF_MAC_CON1,
+	reg = bsp_priv->integrated_phy ? RK3328_GRF_MAC_CON2 :
+		  RK3328_GRF_MAC_CON1;
+
+	regmap_write(bsp_priv->grf, reg,
 		     RK3328_GMAC_PHY_INTF_SEL_RMII |
 		     RK3328_GMAC_RMII_MODE);
-
-	/* set MAC to RMII mode */
-	regmap_write(bsp_priv->grf, RK3328_GRF_MAC_CON1, GRF_BIT(11));
 }
 
 static void rk3328_set_rgmii_speed(struct rk_priv_data *bsp_priv, int speed)
@@ -395,29 +418,40 @@ static void rk3328_set_rgmii_speed(struct rk_priv_data *bsp_priv, int speed)
 static void rk3328_set_rmii_speed(struct rk_priv_data *bsp_priv, int speed)
 {
 	struct device *dev = &bsp_priv->pdev->dev;
+	unsigned int reg;
 
 	if (IS_ERR(bsp_priv->grf)) {
 		dev_err(dev, "Missing rockchip,grf property\n");
 		return;
 	}
 
+	reg = bsp_priv->integrated_phy ? RK3328_GRF_MAC_CON2 :
+		  RK3328_GRF_MAC_CON1;
+
 	if (speed == 10)
-		regmap_write(bsp_priv->grf, RK3328_GRF_MAC_CON1,
+		regmap_write(bsp_priv->grf, reg,
 			     RK3328_GMAC_RMII_CLK_2_5M |
 			     RK3328_GMAC_SPEED_10M);
 	else if (speed == 100)
-		regmap_write(bsp_priv->grf, RK3328_GRF_MAC_CON1,
+		regmap_write(bsp_priv->grf, reg,
 			     RK3328_GMAC_RMII_CLK_25M |
 			     RK3328_GMAC_SPEED_100M);
 	else
 		dev_err(dev, "unknown speed value for RMII! speed=%d", speed);
 }
 
+static void rk3328_integrated_phy_powerup(struct rk_priv_data *priv)
+{
+	regmap_write(priv->grf, RK3328_GRF_MACPHY_CON1,
+		     RK3328_MACPHY_RMII_MODE);
+}
+
 static const struct rk_gmac_ops rk3328_ops = {
 	.set_to_rgmii = rk3328_set_to_rgmii,
 	.set_to_rmii = rk3328_set_to_rmii,
 	.set_rgmii_speed = rk3328_set_rgmii_speed,
 	.set_rmii_speed = rk3328_set_rmii_speed,
+	.integrated_phy_powerup =  rk3328_integrated_phy_powerup,
 };
 
 #define RK3366_GRF_SOC_CON6	0x0418
@@ -753,9 +787,107 @@ static const struct rk_gmac_ops rk3399_ops = {
 	.set_rmii_speed = rk3399_set_rmii_speed,
 };
 
-static int gmac_clk_init(struct rk_priv_data *bsp_priv)
+#define RV1108_GRF_GMAC_CON0		0X0900
+
+/* RV1108_GRF_GMAC_CON0 */
+#define RV1108_GMAC_PHY_INTF_SEL_RMII	(GRF_CLR_BIT(4) | GRF_CLR_BIT(5) | \
+					GRF_BIT(6))
+#define RV1108_GMAC_FLOW_CTRL		GRF_BIT(3)
+#define RV1108_GMAC_FLOW_CTRL_CLR	GRF_CLR_BIT(3)
+#define RV1108_GMAC_SPEED_10M		GRF_CLR_BIT(2)
+#define RV1108_GMAC_SPEED_100M		GRF_BIT(2)
+#define RV1108_GMAC_RMII_CLK_25M	GRF_BIT(7)
+#define RV1108_GMAC_RMII_CLK_2_5M	GRF_CLR_BIT(7)
+
+static void rv1108_set_to_rmii(struct rk_priv_data *bsp_priv)
+{
+	struct device *dev = &bsp_priv->pdev->dev;
+
+	if (IS_ERR(bsp_priv->grf)) {
+		dev_err(dev, "%s: Missing rockchip,grf property\n", __func__);
+		return;
+	}
+
+	regmap_write(bsp_priv->grf, RV1108_GRF_GMAC_CON0,
+		     RV1108_GMAC_PHY_INTF_SEL_RMII);
+}
+
+static void rv1108_set_rmii_speed(struct rk_priv_data *bsp_priv, int speed)
+{
+	struct device *dev = &bsp_priv->pdev->dev;
+
+	if (IS_ERR(bsp_priv->grf)) {
+		dev_err(dev, "%s: Missing rockchip,grf property\n", __func__);
+		return;
+	}
+
+	if (speed == 10) {
+		regmap_write(bsp_priv->grf, RV1108_GRF_GMAC_CON0,
+			     RV1108_GMAC_RMII_CLK_2_5M |
+			     RV1108_GMAC_SPEED_10M);
+	} else if (speed == 100) {
+		regmap_write(bsp_priv->grf, RV1108_GRF_GMAC_CON0,
+			     RV1108_GMAC_RMII_CLK_25M |
+			     RV1108_GMAC_SPEED_100M);
+	} else {
+		dev_err(dev, "unknown speed value for RMII! speed=%d", speed);
+	}
+}
+
+static const struct rk_gmac_ops rv1108_ops = {
+	.set_to_rmii = rv1108_set_to_rmii,
+	.set_rmii_speed = rv1108_set_rmii_speed,
+};
+
+#define RK_GRF_MACPHY_CON0		0xb00
+#define RK_GRF_MACPHY_CON1		0xb04
+#define RK_GRF_MACPHY_CON2		0xb08
+#define RK_GRF_MACPHY_CON3		0xb0c
+
+#define RK_MACPHY_ENABLE		GRF_BIT(0)
+#define RK_MACPHY_DISABLE		GRF_CLR_BIT(0)
+#define RK_MACPHY_CFG_CLK_50M		GRF_BIT(14)
+#define RK_GMAC2PHY_RMII_MODE		(GRF_BIT(6) | GRF_CLR_BIT(7))
+#define RK_GRF_CON2_MACPHY_ID		HIWORD_UPDATE(0x1234, 0xffff, 0)
+#define RK_GRF_CON3_MACPHY_ID		HIWORD_UPDATE(0x35, 0x3f, 0)
+
+static void rk_gmac_integrated_phy_powerup(struct rk_priv_data *priv)
+{
+	if (priv->ops->integrated_phy_powerup)
+		priv->ops->integrated_phy_powerup(priv);
+
+	regmap_write(priv->grf, RK_GRF_MACPHY_CON0, RK_MACPHY_CFG_CLK_50M);
+	regmap_write(priv->grf, RK_GRF_MACPHY_CON0, RK_GMAC2PHY_RMII_MODE);
+
+	regmap_write(priv->grf, RK_GRF_MACPHY_CON2, RK_GRF_CON2_MACPHY_ID);
+	regmap_write(priv->grf, RK_GRF_MACPHY_CON3, RK_GRF_CON3_MACPHY_ID);
+
+	if (priv->phy_reset) {
+		/* PHY needs to be disabled before trying to reset it */
+		regmap_write(priv->grf, RK_GRF_MACPHY_CON0, RK_MACPHY_DISABLE);
+		if (priv->phy_reset)
+			reset_control_assert(priv->phy_reset);
+		usleep_range(10, 20);
+		if (priv->phy_reset)
+			reset_control_deassert(priv->phy_reset);
+		usleep_range(10, 20);
+		regmap_write(priv->grf, RK_GRF_MACPHY_CON0, RK_MACPHY_ENABLE);
+		msleep(30);
+	}
+}
+
+static void rk_gmac_integrated_phy_powerdown(struct rk_priv_data *priv)
+{
+	regmap_write(priv->grf, RK_GRF_MACPHY_CON0, RK_MACPHY_DISABLE);
+	if (priv->phy_reset)
+		reset_control_assert(priv->phy_reset);
+}
+
+static int rk_gmac_clk_init(struct plat_stmmacenet_data *plat)
 {
+	struct rk_priv_data *bsp_priv = plat->bsp_priv;
 	struct device *dev = &bsp_priv->pdev->dev;
+	int ret;
 
 	bsp_priv->clk_enabled = false;
 
@@ -806,6 +938,16 @@ static int gmac_clk_init(struct rk_priv_data *bsp_priv)
 			clk_set_rate(bsp_priv->clk_mac, 50000000);
 	}
 
+	if (plat->phy_node && bsp_priv->integrated_phy) {
+		bsp_priv->clk_phy = of_clk_get(plat->phy_node, 0);
+		if (IS_ERR(bsp_priv->clk_phy)) {
+			ret = PTR_ERR(bsp_priv->clk_phy);
+			dev_err(dev, "Cannot get PHY clock: %d\n", ret);
+			return -EINVAL;
+		}
+		clk_set_rate(bsp_priv->clk_phy, 50000000);
+	}
+
 	return 0;
 }
 
@@ -829,6 +971,9 @@ static int gmac_clk_enable(struct rk_priv_data *bsp_priv, bool enable)
 						bsp_priv->clk_mac_refout);
 			}
 
+			if (!IS_ERR(bsp_priv->clk_phy))
+				clk_prepare_enable(bsp_priv->clk_phy);
+
 			if (!IS_ERR(bsp_priv->aclk_mac))
 				clk_prepare_enable(bsp_priv->aclk_mac);
 
@@ -861,6 +1006,9 @@ static int gmac_clk_enable(struct rk_priv_data *bsp_priv, bool enable)
 						bsp_priv->clk_mac_refout);
 			}
 
+			if (!IS_ERR(bsp_priv->clk_phy))
+				clk_disable_unprepare(bsp_priv->clk_phy);
+
 			if (!IS_ERR(bsp_priv->aclk_mac))
 				clk_disable_unprepare(bsp_priv->aclk_mac);
 
@@ -905,6 +1053,7 @@ static int phy_power_on(struct rk_priv_data *bsp_priv, bool enable)
 }
 
 static struct rk_priv_data *rk_gmac_setup(struct platform_device *pdev,
+					  struct plat_stmmacenet_data *plat,
 					  const struct rk_gmac_ops *ops)
 {
 	struct rk_priv_data *bsp_priv;
@@ -967,9 +1116,22 @@ static struct rk_priv_data *rk_gmac_setup(struct platform_device *pdev,
 
 	bsp_priv->grf = syscon_regmap_lookup_by_phandle(dev->of_node,
 							"rockchip,grf");
-	bsp_priv->pdev = pdev;
 
-	gmac_clk_init(bsp_priv);
+	if (plat->phy_node) {
+		bsp_priv->integrated_phy = of_property_read_bool(plat->phy_node,
+								 "phy-is-integrated");
+		if (bsp_priv->integrated_phy) {
+			bsp_priv->phy_reset = of_reset_control_get(plat->phy_node, NULL);
+			if (IS_ERR(bsp_priv->phy_reset)) {
+				dev_err(&pdev->dev, "No PHY reset control found.\n");
+				bsp_priv->phy_reset = NULL;
+			}
+		}
+	}
+	dev_info(dev, "integrated PHY? (%s).\n",
+		 bsp_priv->integrated_phy ? "yes" : "no");
+
+	bsp_priv->pdev = pdev;
 
 	return bsp_priv;
 }
@@ -1017,6 +1179,9 @@ static int rk_gmac_powerup(struct rk_priv_data *bsp_priv)
 	pm_runtime_enable(dev);
 	pm_runtime_get_sync(dev);
 
+	if (bsp_priv->integrated_phy)
+		rk_gmac_integrated_phy_powerup(bsp_priv);
+
 	return 0;
 }
 
@@ -1024,6 +1189,9 @@ static void rk_gmac_powerdown(struct rk_priv_data *gmac)
 {
 	struct device *dev = &gmac->pdev->dev;
 
+	if (gmac->integrated_phy)
+		rk_gmac_integrated_phy_powerdown(gmac);
+
 	pm_runtime_put_sync(dev);
 	pm_runtime_disable(dev);
 
@@ -1075,12 +1243,16 @@ static int rk_gmac_probe(struct platform_device *pdev)
 	plat_dat->has_gmac = true;
 	plat_dat->fix_mac_speed = rk_fix_speed;
 
-	plat_dat->bsp_priv = rk_gmac_setup(pdev, data);
+	plat_dat->bsp_priv = rk_gmac_setup(pdev, plat_dat, data);
 	if (IS_ERR(plat_dat->bsp_priv)) {
 		ret = PTR_ERR(plat_dat->bsp_priv);
 		goto err_remove_config_dt;
 	}
 
+	ret = rk_gmac_clk_init(plat_dat);
+	if (ret)
+		return ret;
+
 	ret = rk_gmac_powerup(plat_dat->bsp_priv);
 	if (ret)
 		goto err_remove_config_dt;
@@ -1147,6 +1319,7 @@ static const struct of_device_id rk_gmac_dwmac_match[] = {
 	{ .compatible = "rockchip,rk3366-gmac", .data = &rk3366_ops },
 	{ .compatible = "rockchip,rk3368-gmac", .data = &rk3368_ops },
 	{ .compatible = "rockchip,rk3399-gmac", .data = &rk3399_ops },
+	{ .compatible = "rockchip,rv1108-gmac", .data = &rv1108_ops },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, rk_gmac_dwmac_match);
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c
index 17d4bbaeb65c..6e359572b9f0 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c
@@ -269,7 +269,10 @@ static int socfpga_dwmac_set_phy_mode(struct socfpga_dwmac *dwmac)
 	ctrl &= ~(SYSMGR_EMACGRP_CTRL_PHYSEL_MASK << reg_shift);
 	ctrl |= val << reg_shift;
 
-	if (dwmac->f2h_ptp_ref_clk) {
+	if (dwmac->f2h_ptp_ref_clk ||
+	    phymode == PHY_INTERFACE_MODE_MII ||
+	    phymode == PHY_INTERFACE_MODE_GMII ||
+	    phymode == PHY_INTERFACE_MODE_SGMII) {
 		ctrl |= SYSMGR_EMACGRP_CTRL_PTP_REF_CLK_MASK << (reg_shift / 2);
 		regmap_read(sys_mgr_base_addr, SYSMGR_FPGAGRP_MODULE_REG,
 			    &module);
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c
index fffd6d5fc907..39c2122a4f26 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c
@@ -979,14 +979,6 @@ static int sun8i_dwmac_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id sun8i_dwmac_match[] = {
-	{ .compatible = "allwinner,sun8i-h3-emac",
-		.data = &emac_variant_h3 },
-	{ .compatible = "allwinner,sun8i-v3s-emac",
-		.data = &emac_variant_v3s },
-	{ .compatible = "allwinner,sun8i-a83t-emac",
-		.data = &emac_variant_a83t },
-	{ .compatible = "allwinner,sun50i-a64-emac",
-		.data = &emac_variant_a64 },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, sun8i_dwmac_match);
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c
index db157a47000c..f5f37bfa1d58 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c
@@ -204,6 +204,7 @@ int stmmac_mdio_register(struct net_device *ndev)
 	struct stmmac_priv *priv = netdev_priv(ndev);
 	struct stmmac_mdio_bus_data *mdio_bus_data = priv->plat->mdio_bus_data;
 	struct device_node *mdio_node = priv->plat->mdio_node;
+	struct device *dev = ndev->dev.parent;
 	int addr, found;
 
 	if (!mdio_bus_data)
@@ -237,7 +238,7 @@ int stmmac_mdio_register(struct net_device *ndev)
 	else
 		err = mdiobus_register(new_bus);
 	if (err != 0) {
-		netdev_err(ndev, "Cannot register the MDIO bus\n");
+		dev_err(dev, "Cannot register the MDIO bus\n");
 		goto bus_register_fail;
 	}
 
@@ -247,9 +248,6 @@ int stmmac_mdio_register(struct net_device *ndev)
 	found = 0;
 	for (addr = 0; addr < PHY_MAX_ADDR; addr++) {
 		struct phy_device *phydev = mdiobus_get_phy(new_bus, addr);
-		int act = 0;
-		char irq_num[4];
-		char *irq_str;
 
 		if (!phydev)
 			continue;
@@ -272,27 +270,12 @@ int stmmac_mdio_register(struct net_device *ndev)
 		if (priv->plat->phy_addr == -1)
 			priv->plat->phy_addr = addr;
 
-		act = (priv->plat->phy_addr == addr);
-		switch (phydev->irq) {
-		case PHY_POLL:
-			irq_str = "POLL";
-			break;
-		case PHY_IGNORE_INTERRUPT:
-			irq_str = "IGNORE";
-			break;
-		default:
-			sprintf(irq_num, "%d", phydev->irq);
-			irq_str = irq_num;
-			break;
-		}
-		netdev_info(ndev, "PHY ID %08x at %d IRQ %s (%s)%s\n",
-			    phydev->phy_id, addr, irq_str, phydev_name(phydev),
-			    act ? " active" : "");
+		phy_attached_info(phydev);
 		found = 1;
 	}
 
 	if (!found && !mdio_node) {
-		netdev_warn(ndev, "No PHY found\n");
+		dev_warn(dev, "No PHY found\n");
 		mdiobus_unregister(new_bus);
 		mdiobus_free(new_bus);
 		return -ENODEV;
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c
index d71bd80c5b5b..e471a903c654 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c
@@ -152,7 +152,7 @@ static int stmmac_enable(struct ptp_clock_info *ptp,
 }
 
 /* structure describing a PTP hardware clock */
-static struct ptp_clock_info stmmac_ptp_clock_ops = {
+static const struct ptp_clock_info stmmac_ptp_clock_ops = {
 	.owner = THIS_MODULE,
 	.name = "stmmac_ptp_clock",
 	.max_adj = 62500000,
diff --git a/drivers/net/ethernet/sun/ldmvsw.c b/drivers/net/ethernet/sun/ldmvsw.c
index 8603e397097e..5b56c24b6ed2 100644
--- a/drivers/net/ethernet/sun/ldmvsw.c
+++ b/drivers/net/ethernet/sun/ldmvsw.c
@@ -248,7 +248,7 @@ static struct net_device *vsw_alloc_netdev(u8 hwaddr[],
 	dev->ethtool_ops = &vsw_ethtool_ops;
 	dev->watchdog_timeo = VSW_TX_TIMEOUT;
 
-	dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_SG;
+	dev->hw_features = NETIF_F_HW_CSUM | NETIF_F_SG;
 	dev->features = dev->hw_features;
 
 	/* MTU range: 68 - 65535 */
diff --git a/drivers/net/ethernet/sun/niu.c b/drivers/net/ethernet/sun/niu.c
index 4bb04aaf9650..6a4e8e1bbd90 100644
--- a/drivers/net/ethernet/sun/niu.c
+++ b/drivers/net/ethernet/sun/niu.c
@@ -9221,8 +9221,7 @@ static int niu_get_of_props(struct niu *np)
 
 	phy_type = of_get_property(dp, "phy-type", &prop_len);
 	if (!phy_type) {
-		netdev_err(dev, "%s: OF node lacks phy-type property\n",
-			   dp->full_name);
+		netdev_err(dev, "%pOF: OF node lacks phy-type property\n", dp);
 		return -EINVAL;
 	}
 
@@ -9232,26 +9231,25 @@ static int niu_get_of_props(struct niu *np)
 	strcpy(np->vpd.phy_type, phy_type);
 
 	if (niu_phy_type_prop_decode(np, np->vpd.phy_type)) {
-		netdev_err(dev, "%s: Illegal phy string [%s]\n",
-			   dp->full_name, np->vpd.phy_type);
+		netdev_err(dev, "%pOF: Illegal phy string [%s]\n",
+			   dp, np->vpd.phy_type);
 		return -EINVAL;
 	}
 
 	mac_addr = of_get_property(dp, "local-mac-address", &prop_len);
 	if (!mac_addr) {
-		netdev_err(dev, "%s: OF node lacks local-mac-address property\n",
-			   dp->full_name);
+		netdev_err(dev, "%pOF: OF node lacks local-mac-address property\n",
+			   dp);
 		return -EINVAL;
 	}
 	if (prop_len != dev->addr_len) {
-		netdev_err(dev, "%s: OF MAC address prop len (%d) is wrong\n",
-			   dp->full_name, prop_len);
+		netdev_err(dev, "%pOF: OF MAC address prop len (%d) is wrong\n",
+			   dp, prop_len);
 	}
 	memcpy(dev->dev_addr, mac_addr, dev->addr_len);
 	if (!is_valid_ether_addr(&dev->dev_addr[0])) {
-		netdev_err(dev, "%s: OF MAC address is invalid\n",
-			   dp->full_name);
-		netdev_err(dev, "%s: [ %pM ]\n", dp->full_name, dev->dev_addr);
+		netdev_err(dev, "%pOF: OF MAC address is invalid\n", dp);
+		netdev_err(dev, "%pOF: [ %pM ]\n", dp, dev->dev_addr);
 		return -EINVAL;
 	}
 
@@ -10027,8 +10025,8 @@ static int niu_of_probe(struct platform_device *op)
 
 	reg = of_get_property(op->dev.of_node, "reg", NULL);
 	if (!reg) {
-		dev_err(&op->dev, "%s: No 'reg' property, aborting\n",
-			op->dev.of_node->full_name);
+		dev_err(&op->dev, "%pOF: No 'reg' property, aborting\n",
+			op->dev.of_node);
 		return -ENODEV;
 	}
 
diff --git a/drivers/net/ethernet/sun/sunvnet.c b/drivers/net/ethernet/sun/sunvnet.c
index 75b167e3fe98..0b95105f7060 100644
--- a/drivers/net/ethernet/sun/sunvnet.c
+++ b/drivers/net/ethernet/sun/sunvnet.c
@@ -312,7 +312,7 @@ static struct vnet *vnet_new(const u64 *local_mac,
 	dev->watchdog_timeo = VNET_TX_TIMEOUT;
 
 	dev->hw_features = NETIF_F_TSO | NETIF_F_GSO | NETIF_F_GSO_SOFTWARE |
-			   NETIF_F_IP_CSUM | NETIF_F_SG;
+			   NETIF_F_HW_CSUM | NETIF_F_SG;
 	dev->features = dev->hw_features;
 
 	/* MTU range: 68 - 65535 */
diff --git a/drivers/net/ethernet/sun/sunvnet_common.c b/drivers/net/ethernet/sun/sunvnet_common.c
index 9e86833249d4..ecf456c7b6d1 100644
--- a/drivers/net/ethernet/sun/sunvnet_common.c
+++ b/drivers/net/ethernet/sun/sunvnet_common.c
@@ -303,7 +303,7 @@ static struct sk_buff *alloc_and_align_skb(struct net_device *dev,
 	return skb;
 }
 
-static inline void vnet_fullcsum(struct sk_buff *skb)
+static inline void vnet_fullcsum_ipv4(struct sk_buff *skb)
 {
 	struct iphdr *iph = ip_hdr(skb);
 	int offset = skb_transport_offset(skb);
@@ -335,6 +335,40 @@ static inline void vnet_fullcsum(struct sk_buff *skb)
 	}
 }
 
+#if IS_ENABLED(CONFIG_IPV6)
+static inline void vnet_fullcsum_ipv6(struct sk_buff *skb)
+{
+	struct ipv6hdr *ip6h = ipv6_hdr(skb);
+	int offset = skb_transport_offset(skb);
+
+	if (skb->protocol != htons(ETH_P_IPV6))
+		return;
+	if (ip6h->nexthdr != IPPROTO_TCP &&
+	    ip6h->nexthdr != IPPROTO_UDP)
+		return;
+	skb->ip_summed = CHECKSUM_NONE;
+	skb->csum_level = 1;
+	skb->csum = 0;
+	if (ip6h->nexthdr == IPPROTO_TCP) {
+		struct tcphdr *ptcp = tcp_hdr(skb);
+
+		ptcp->check = 0;
+		skb->csum = skb_checksum(skb, offset, skb->len - offset, 0);
+		ptcp->check = csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr,
+					      skb->len - offset, IPPROTO_TCP,
+					      skb->csum);
+	} else if (ip6h->nexthdr == IPPROTO_UDP) {
+		struct udphdr *pudp = udp_hdr(skb);
+
+		pudp->check = 0;
+		skb->csum = skb_checksum(skb, offset, skb->len - offset, 0);
+		pudp->check = csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr,
+					      skb->len - offset, IPPROTO_UDP,
+					      skb->csum);
+	}
+}
+#endif
+
 static int vnet_rx_one(struct vnet_port *port, struct vio_net_desc *desc)
 {
 	struct net_device *dev = VNET_PORT_TO_NET_DEVICE(port);
@@ -394,9 +428,14 @@ static int vnet_rx_one(struct vnet_port *port, struct vio_net_desc *desc)
 				struct iphdr *iph = ip_hdr(skb);
 				int ihl = iph->ihl * 4;
 
-				skb_reset_transport_header(skb);
 				skb_set_transport_header(skb, ihl);
-				vnet_fullcsum(skb);
+				vnet_fullcsum_ipv4(skb);
+#if IS_ENABLED(CONFIG_IPV6)
+			} else if (skb->protocol == htons(ETH_P_IPV6)) {
+				skb_set_transport_header(skb,
+							 sizeof(struct ipv6hdr));
+				vnet_fullcsum_ipv6(skb);
+#endif
 			}
 		}
 		if (dext->flags & VNET_PKT_HCK_IPV4_HDRCKSUM_OK) {
@@ -1115,24 +1154,47 @@ static inline struct sk_buff *vnet_skb_shape(struct sk_buff *skb, int ncookies)
 		if (skb->ip_summed == CHECKSUM_PARTIAL)
 			start = skb_checksum_start_offset(skb);
 		if (start) {
-			struct iphdr *iph = ip_hdr(nskb);
 			int offset = start + nskb->csum_offset;
 
+			/* copy the headers, no csum here */
 			if (skb_copy_bits(skb, 0, nskb->data, start)) {
 				dev_kfree_skb(nskb);
 				dev_kfree_skb(skb);
 				return NULL;
 			}
+
+			/* copy the rest, with csum calculation */
 			*(__sum16 *)(skb->data + offset) = 0;
 			csum = skb_copy_and_csum_bits(skb, start,
 						      nskb->data + start,
 						      skb->len - start, 0);
-			if (iph->protocol == IPPROTO_TCP ||
-			    iph->protocol == IPPROTO_UDP) {
-				csum = csum_tcpudp_magic(iph->saddr, iph->daddr,
-							 skb->len - start,
-							 iph->protocol, csum);
+
+			/* add in the header checksums */
+			if (skb->protocol == htons(ETH_P_IP)) {
+				struct iphdr *iph = ip_hdr(nskb);
+
+				if (iph->protocol == IPPROTO_TCP ||
+				    iph->protocol == IPPROTO_UDP) {
+					csum = csum_tcpudp_magic(iph->saddr,
+								 iph->daddr,
+								 skb->len - start,
+								 iph->protocol,
+								 csum);
+				}
+			} else if (skb->protocol == htons(ETH_P_IPV6)) {
+				struct ipv6hdr *ip6h = ipv6_hdr(nskb);
+
+				if (ip6h->nexthdr == IPPROTO_TCP ||
+				    ip6h->nexthdr == IPPROTO_UDP) {
+					csum = csum_ipv6_magic(&ip6h->saddr,
+							       &ip6h->daddr,
+							       skb->len - start,
+							       ip6h->nexthdr,
+							       csum);
+				}
 			}
+
+			/* save the final result */
 			*(__sum16 *)(nskb->data + offset) = csum;
 
 			nskb->ip_summed = CHECKSUM_NONE;
@@ -1318,8 +1380,14 @@ int sunvnet_start_xmit_common(struct sk_buff *skb, struct net_device *dev,
 	if (unlikely(!skb))
 		goto out_dropped;
 
-	if (skb->ip_summed == CHECKSUM_PARTIAL)
-		vnet_fullcsum(skb);
+	if (skb->ip_summed == CHECKSUM_PARTIAL) {
+		if (skb->protocol == htons(ETH_P_IP))
+			vnet_fullcsum_ipv4(skb);
+#if IS_ENABLED(CONFIG_IPV6)
+		else if (skb->protocol == htons(ETH_P_IPV6))
+			vnet_fullcsum_ipv6(skb);
+#endif
+	}
 
 	dr = &port->vio.drings[VIO_DRIVER_TX_RING];
 	i = skb_get_queue_mapping(skb);
diff --git a/drivers/net/ethernet/synopsys/dwc-xlgmac-net.c b/drivers/net/ethernet/synopsys/dwc-xlgmac-net.c
index 3b91257683bc..e1b55b8fb8e0 100644
--- a/drivers/net/ethernet/synopsys/dwc-xlgmac-net.c
+++ b/drivers/net/ethernet/synopsys/dwc-xlgmac-net.c
@@ -17,6 +17,7 @@
 
 #include <linux/netdevice.h>
 #include <linux/tcp.h>
+#include <linux/interrupt.h>
 
 #include "dwc-xlgmac.h"
 #include "dwc-xlgmac-reg.h"
diff --git a/drivers/net/ethernet/ti/cpsw-common.c b/drivers/net/ethernet/ti/cpsw-common.c
index 56ba411421f0..38d1cc557c11 100644
--- a/drivers/net/ethernet/ti/cpsw-common.c
+++ b/drivers/net/ethernet/ti/cpsw-common.c
@@ -96,7 +96,7 @@ int ti_cm_get_macid(struct device *dev, int slave, u8 *mac_addr)
 	if (of_machine_is_compatible("ti,dra7"))
 		return davinci_emac_3517_get_macid(dev, 0x514, slave, mac_addr);
 
-	dev_err(dev, "incompatible machine/device type for reading mac address\n");
+	dev_info(dev, "incompatible machine/device type for reading mac address\n");
 	return -ENOENT;
 }
 EXPORT_SYMBOL_GPL(ti_cm_get_macid);
diff --git a/drivers/net/ethernet/ti/cpsw.c b/drivers/net/ethernet/ti/cpsw.c
index badd0a8caeb9..db8a4bcfc6c7 100644
--- a/drivers/net/ethernet/ti/cpsw.c
+++ b/drivers/net/ethernet/ti/cpsw.c
@@ -1321,8 +1321,8 @@ static void cpsw_slave_open(struct cpsw_slave *slave, struct cpsw_priv *priv)
 		phy = of_phy_connect(priv->ndev, slave->data->phy_node,
 				 &cpsw_adjust_link, 0, slave->data->phy_if);
 		if (!phy) {
-			dev_err(priv->dev, "phy \"%s\" not found on slave %d\n",
-				slave->data->phy_node->full_name,
+			dev_err(priv->dev, "phy \"%pOF\" not found on slave %d\n",
+				slave->data->phy_node,
 				slave->slave_num);
 			return;
 		}
@@ -2670,8 +2670,8 @@ static int cpsw_probe_dt(struct cpsw_platform_data *data,
 		parp = of_get_property(slave_node, "phy_id", &lenp);
 		if (slave_data->phy_node) {
 			dev_dbg(&pdev->dev,
-				"slave[%d] using phy-handle=\"%s\"\n",
-				i, slave_data->phy_node->full_name);
+				"slave[%d] using phy-handle=\"%pOF\"\n",
+				i, slave_data->phy_node);
 		} else if (of_phy_is_fixed_link(slave_node)) {
 			/* In the case of a fixed PHY, the DT node associated
 			 * to the PHY is the Ethernet MAC DT node.
@@ -2827,7 +2827,7 @@ static int cpsw_probe_dual_emac(struct cpsw_priv *priv)
 
 #define CPSW_QUIRK_IRQ		BIT(0)
 
-static struct platform_device_id cpsw_devtype[] = {
+static const struct platform_device_id cpsw_devtype[] = {
 	{
 		/* keep it for existing comaptibles */
 		.name = "cpsw",
diff --git a/drivers/net/ethernet/ti/cpts.c b/drivers/net/ethernet/ti/cpts.c
index c2121d214f08..e7b76f6b4f67 100644
--- a/drivers/net/ethernet/ti/cpts.c
+++ b/drivers/net/ethernet/ti/cpts.c
@@ -298,7 +298,7 @@ static long cpts_overflow_check(struct ptp_clock_info *ptp)
 	return (long)delay;
 }
 
-static struct ptp_clock_info cpts_info = {
+static const struct ptp_clock_info cpts_info = {
 	.owner		= THIS_MODULE,
 	.name		= "CTPS timer",
 	.max_adj	= 1000000,
diff --git a/drivers/net/ethernet/ti/davinci_emac.c b/drivers/net/ethernet/ti/davinci_emac.c
index 64d5527feb2a..4bb561856af5 100644
--- a/drivers/net/ethernet/ti/davinci_emac.c
+++ b/drivers/net/ethernet/ti/davinci_emac.c
@@ -1480,8 +1480,8 @@ static int emac_dev_open(struct net_device *ndev)
 		phydev = of_phy_connect(ndev, priv->phy_node,
 					&emac_adjust_link, 0, 0);
 		if (!phydev) {
-			dev_err(emac_dev, "could not connect to phy %s\n",
-				priv->phy_node->full_name);
+			dev_err(emac_dev, "could not connect to phy %pOF\n",
+				priv->phy_node);
 			ret = -ENODEV;
 			goto err;
 		}
diff --git a/drivers/net/ethernet/ti/davinci_mdio.c b/drivers/net/ethernet/ti/davinci_mdio.c
index 33df340db1f1..3c33f4504d8e 100644
--- a/drivers/net/ethernet/ti/davinci_mdio.c
+++ b/drivers/net/ethernet/ti/davinci_mdio.c
@@ -159,8 +159,10 @@ static int davinci_mdio_reset(struct mii_bus *bus)
 
 	/* dump hardware version info */
 	ver = __raw_readl(&data->regs->version);
-	dev_info(data->dev, "davinci mdio revision %d.%d\n",
-		 (ver >> 8) & 0xff, ver & 0xff);
+	dev_info(data->dev,
+		 "davinci mdio revision %d.%d, bus freq %ld\n",
+		 (ver >> 8) & 0xff, ver & 0xff,
+		 data->pdata.bus_freq);
 
 	if (data->skip_scan)
 		goto done;
@@ -198,8 +200,10 @@ static inline int wait_for_user_access(struct davinci_mdio_data *data)
 			return 0;
 
 		reg = __raw_readl(&regs->control);
-		if ((reg & CONTROL_IDLE) == 0)
+		if ((reg & CONTROL_IDLE) == 0) {
+			usleep_range(100, 200);
 			continue;
+		}
 
 		/*
 		 * An emac soft_reset may have clobbered the mdio controller's
diff --git a/drivers/net/ethernet/ti/netcp_core.c b/drivers/net/ethernet/ti/netcp_core.c
index 9d52c3a78621..437d36289786 100644
--- a/drivers/net/ethernet/ti/netcp_core.c
+++ b/drivers/net/ethernet/ti/netcp_core.c
@@ -1877,20 +1877,21 @@ static u16 netcp_select_queue(struct net_device *dev, struct sk_buff *skb,
 	return 0;
 }
 
-static int netcp_setup_tc(struct net_device *dev, u32 handle, u32 chain_index,
-			  __be16 proto, struct tc_to_netdev *tc)
+static int netcp_setup_tc(struct net_device *dev, enum tc_setup_type type,
+			  void *type_data)
 {
+	struct tc_mqprio_qopt *mqprio = type_data;
 	u8 num_tc;
 	int i;
 
 	/* setup tc must be called under rtnl lock */
 	ASSERT_RTNL();
 
-	if (tc->type != TC_SETUP_MQPRIO)
-		return -EINVAL;
+	if (type != TC_SETUP_MQPRIO)
+		return -EOPNOTSUPP;
 
-	tc->mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
-	num_tc = tc->mqprio->num_tc;
+	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
+	num_tc = mqprio->num_tc;
 
 	/* Sanity-check the number of traffic classes requested */
 	if ((dev->real_num_tx_queues <= 1) ||
@@ -2144,7 +2145,6 @@ static void netcp_delete_interface(struct netcp_device *netcp_device,
 
 	of_node_put(netcp->node_interface);
 	unregister_netdev(ndev);
-	netif_napi_del(&netcp->rx_napi);
 	free_netdev(ndev);
 }
 
diff --git a/drivers/net/ethernet/tile/tilegx.c b/drivers/net/ethernet/tile/tilegx.c
index aec95382ea5c..c00102b8145a 100644
--- a/drivers/net/ethernet/tile/tilegx.c
+++ b/drivers/net/ethernet/tile/tilegx.c
@@ -873,7 +873,7 @@ static int ptp_mpipe_enable(struct ptp_clock_info *ptp,
 	return -EOPNOTSUPP;
 }
 
-static struct ptp_clock_info ptp_mpipe_caps = {
+static const struct ptp_clock_info ptp_mpipe_caps = {
 	.owner		= THIS_MODULE,
 	.name		= "mPIPE clock",
 	.max_adj	= 999999999,
diff --git a/drivers/net/ethernet/via/via-rhine.c b/drivers/net/ethernet/via/via-rhine.c
index acd29d60174a..83e6f76eb965 100644
--- a/drivers/net/ethernet/via/via-rhine.c
+++ b/drivers/net/ethernet/via/via-rhine.c
@@ -2598,7 +2598,7 @@ static struct platform_driver rhine_driver_platform = {
 	}
 };
 
-static struct dmi_system_id rhine_dmi_table[] __initdata = {
+static const struct dmi_system_id rhine_dmi_table[] __initconst = {
 	{
 		.ident = "EPIA-M",
 		.matches = {
diff --git a/drivers/net/ethernet/xilinx/ll_temac_main.c b/drivers/net/ethernet/xilinx/ll_temac_main.c
index d73da8afe08e..60abc9250f56 100644
--- a/drivers/net/ethernet/xilinx/ll_temac_main.c
+++ b/drivers/net/ethernet/xilinx/ll_temac_main.c
@@ -1089,7 +1089,7 @@ static int temac_of_probe(struct platform_device *op)
 
 	lp->phy_node = of_parse_phandle(op->dev.of_node, "phy-handle", 0);
 	if (lp->phy_node)
-		dev_dbg(lp->dev, "using PHY node %s (%p)\n", np->full_name, np);
+		dev_dbg(lp->dev, "using PHY node %pOF (%p)\n", np, np);
 
 	/* Add the device attributes */
 	rc = sysfs_create_group(&lp->dev->kobj, &temac_attr_group);
diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet.h b/drivers/net/ethernet/xilinx/xilinx_axienet.h
index af27f7d1cbf3..5ef626331f85 100644
--- a/drivers/net/ethernet/xilinx/xilinx_axienet.h
+++ b/drivers/net/ethernet/xilinx/xilinx_axienet.h
@@ -389,7 +389,7 @@ struct axidma_bd {
  * @dma_err_tasklet: Tasklet structure to process Axi DMA errors
  * @tx_irq:	Axidma TX IRQ number
  * @rx_irq:	Axidma RX IRQ number
- * @phy_type:	Phy type to identify between MII/GMII/RGMII/SGMII/1000 Base-X
+ * @phy_mode:	Phy type to identify between MII/GMII/RGMII/SGMII/1000 Base-X
  * @options:	AxiEthernet option word
  * @last_link:	Phy link state in which the PHY was negotiated earlier
  * @features:	Stores the extended features supported by the axienet hw
@@ -432,7 +432,7 @@ struct axienet_local {
 
 	int tx_irq;
 	int rx_irq;
-	u32 phy_type;
+	phy_interface_t phy_mode;
 
 	u32 options;			/* Current options word */
 	u32 last_link;
diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c
index 33c595f4691d..e74e1e897864 100644
--- a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c
+++ b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c
@@ -531,11 +531,11 @@ static void axienet_adjust_link(struct net_device *ndev)
 	link_state = phy->speed | (phy->duplex << 1) | phy->link;
 	if (lp->last_link != link_state) {
 		if ((phy->speed == SPEED_10) || (phy->speed == SPEED_100)) {
-			if (lp->phy_type == XAE_PHY_TYPE_1000BASE_X)
+			if (lp->phy_mode == PHY_INTERFACE_MODE_1000BASEX)
 				setspeed = 0;
 		} else {
 			if ((phy->speed == SPEED_1000) &&
-			    (lp->phy_type == XAE_PHY_TYPE_MII))
+			    (lp->phy_mode == PHY_INTERFACE_MODE_MII))
 				setspeed = 0;
 		}
 
@@ -935,15 +935,8 @@ static int axienet_open(struct net_device *ndev)
 		return ret;
 
 	if (lp->phy_node) {
-		if (lp->phy_type == XAE_PHY_TYPE_GMII) {
-			phydev = of_phy_connect(lp->ndev, lp->phy_node,
-						axienet_adjust_link, 0,
-						PHY_INTERFACE_MODE_GMII);
-		} else if (lp->phy_type == XAE_PHY_TYPE_RGMII_2_0) {
-			phydev = of_phy_connect(lp->ndev, lp->phy_node,
-						axienet_adjust_link, 0,
-						PHY_INTERFACE_MODE_RGMII_ID);
-		}
+		phydev = of_phy_connect(lp->ndev, lp->phy_node,
+					axienet_adjust_link, 0, lp->phy_mode);
 
 		if (!phydev)
 			dev_err(lp->dev, "of_phy_connect() failed\n");
@@ -1539,7 +1532,38 @@ static int axienet_probe(struct platform_device *pdev)
 	 * the device-tree and accordingly set flags.
 	 */
 	of_property_read_u32(pdev->dev.of_node, "xlnx,rxmem", &lp->rxmem);
-	of_property_read_u32(pdev->dev.of_node, "xlnx,phy-type", &lp->phy_type);
+
+	/* Start with the proprietary, and broken phy_type */
+	ret = of_property_read_u32(pdev->dev.of_node, "xlnx,phy-type", &value);
+	if (!ret) {
+		netdev_warn(ndev, "Please upgrade your device tree binary blob to use phy-mode");
+		switch (value) {
+		case XAE_PHY_TYPE_MII:
+			lp->phy_mode = PHY_INTERFACE_MODE_MII;
+			break;
+		case XAE_PHY_TYPE_GMII:
+			lp->phy_mode = PHY_INTERFACE_MODE_GMII;
+			break;
+		case XAE_PHY_TYPE_RGMII_2_0:
+			lp->phy_mode = PHY_INTERFACE_MODE_RGMII_ID;
+			break;
+		case XAE_PHY_TYPE_SGMII:
+			lp->phy_mode = PHY_INTERFACE_MODE_SGMII;
+			break;
+		case XAE_PHY_TYPE_1000BASE_X:
+			lp->phy_mode = PHY_INTERFACE_MODE_1000BASEX;
+			break;
+		default:
+			ret = -EINVAL;
+			goto free_netdev;
+		}
+	} else {
+		lp->phy_mode = of_get_phy_mode(pdev->dev.of_node);
+		if (lp->phy_mode < 0) {
+			ret = -EINVAL;
+			goto free_netdev;
+		}
+	}
 
 	/* Find the DMA node, map the DMA registers, and decode the DMA IRQs */
 	np = of_parse_phandle(pdev->dev.of_node, "axistream-connected", 0);
diff --git a/drivers/net/ethernet/xircom/xirc2ps_cs.c b/drivers/net/ethernet/xircom/xirc2ps_cs.c
index f71883264cc0..fd5288ff53b5 100644
--- a/drivers/net/ethernet/xircom/xirc2ps_cs.c
+++ b/drivers/net/ethernet/xircom/xirc2ps_cs.c
@@ -1781,7 +1781,7 @@ static int __init setup_xirc2ps_cs(char *str)
 	 */
 	int ints[10] = { -1 };
 
-	str = get_options(str, 9, ints);
+	str = get_options(str, ARRAY_SIZE(ints), ints);
 
 #define MAYBE_SET(X,Y) if (ints[0] >= Y && ints[Y] != -1) { X = ints[Y]; }
 	MAYBE_SET(if_port, 3);
diff --git a/drivers/net/fddi/defxx.c b/drivers/net/fddi/defxx.c
index f4a816cf012a..61fceee73c1b 100644
--- a/drivers/net/fddi/defxx.c
+++ b/drivers/net/fddi/defxx.c
@@ -3767,7 +3767,7 @@ static void dfx_pci_unregister(struct pci_dev *pdev)
 #endif /* CONFIG_PCI */
 
 #ifdef CONFIG_EISA
-static struct eisa_device_id dfx_eisa_table[] = {
+static const struct eisa_device_id dfx_eisa_table[] = {
         { "DEC3001", DEFEA_PROD_ID_1 },
         { "DEC3002", DEFEA_PROD_ID_2 },
         { "DEC3003", DEFEA_PROD_ID_3 },
diff --git a/drivers/net/geneve.c b/drivers/net/geneve.c
index 2bbda71818ad..f6404074b7b0 100644
--- a/drivers/net/geneve.c
+++ b/drivers/net/geneve.c
@@ -715,6 +715,7 @@ free_dst:
 
 static struct rtable *geneve_get_v4_rt(struct sk_buff *skb,
 				       struct net_device *dev,
+				       struct geneve_sock *gs4,
 				       struct flowi4 *fl4,
 				       const struct ip_tunnel_info *info)
 {
@@ -724,7 +725,7 @@ static struct rtable *geneve_get_v4_rt(struct sk_buff *skb,
 	struct rtable *rt = NULL;
 	__u8 tos;
 
-	if (!rcu_dereference(geneve->sock4))
+	if (!gs4)
 		return ERR_PTR(-EIO);
 
 	memset(fl4, 0, sizeof(*fl4));
@@ -764,6 +765,7 @@ static struct rtable *geneve_get_v4_rt(struct sk_buff *skb,
 #if IS_ENABLED(CONFIG_IPV6)
 static struct dst_entry *geneve_get_v6_dst(struct sk_buff *skb,
 					   struct net_device *dev,
+					   struct geneve_sock *gs6,
 					   struct flowi6 *fl6,
 					   const struct ip_tunnel_info *info)
 {
@@ -771,10 +773,8 @@ static struct dst_entry *geneve_get_v6_dst(struct sk_buff *skb,
 	struct geneve_dev *geneve = netdev_priv(dev);
 	struct dst_entry *dst = NULL;
 	struct dst_cache *dst_cache;
-	struct geneve_sock *gs6;
 	__u8 prio;
 
-	gs6 = rcu_dereference(geneve->sock6);
 	if (!gs6)
 		return ERR_PTR(-EIO);
 
@@ -827,7 +827,7 @@ static int geneve_xmit_skb(struct sk_buff *skb, struct net_device *dev,
 	__be16 df;
 	int err;
 
-	rt = geneve_get_v4_rt(skb, dev, &fl4, info);
+	rt = geneve_get_v4_rt(skb, dev, gs4, &fl4, info);
 	if (IS_ERR(rt))
 		return PTR_ERR(rt);
 
@@ -866,7 +866,7 @@ static int geneve6_xmit_skb(struct sk_buff *skb, struct net_device *dev,
 	__be16 sport;
 	int err;
 
-	dst = geneve_get_v6_dst(skb, dev, &fl6, info);
+	dst = geneve_get_v6_dst(skb, dev, gs6, &fl6, info);
 	if (IS_ERR(dst))
 		return PTR_ERR(dst);
 
@@ -951,8 +951,9 @@ static int geneve_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
 	if (ip_tunnel_info_af(info) == AF_INET) {
 		struct rtable *rt;
 		struct flowi4 fl4;
+		struct geneve_sock *gs4 = rcu_dereference(geneve->sock4);
 
-		rt = geneve_get_v4_rt(skb, dev, &fl4, info);
+		rt = geneve_get_v4_rt(skb, dev, gs4, &fl4, info);
 		if (IS_ERR(rt))
 			return PTR_ERR(rt);
 
@@ -962,8 +963,9 @@ static int geneve_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
 	} else if (ip_tunnel_info_af(info) == AF_INET6) {
 		struct dst_entry *dst;
 		struct flowi6 fl6;
+		struct geneve_sock *gs6 = rcu_dereference(geneve->sock6);
 
-		dst = geneve_get_v6_dst(skb, dev, &fl6, info);
+		dst = geneve_get_v6_dst(skb, dev, gs6, &fl6, info);
 		if (IS_ERR(dst))
 			return PTR_ERR(dst);
 
@@ -1014,16 +1016,22 @@ static struct device_type geneve_type = {
  * supply the listening GENEVE udp ports. Callers are expected
  * to implement the ndo_udp_tunnel_add.
  */
-static void geneve_push_rx_ports(struct net_device *dev)
+static void geneve_offload_rx_ports(struct net_device *dev, bool push)
 {
 	struct net *net = dev_net(dev);
 	struct geneve_net *gn = net_generic(net, geneve_net_id);
 	struct geneve_sock *gs;
 
 	rcu_read_lock();
-	list_for_each_entry_rcu(gs, &gn->sock_list, list)
-		udp_tunnel_push_rx_port(dev, gs->sock,
-					UDP_TUNNEL_TYPE_GENEVE);
+	list_for_each_entry_rcu(gs, &gn->sock_list, list) {
+		if (push) {
+			udp_tunnel_push_rx_port(dev, gs->sock,
+						UDP_TUNNEL_TYPE_GENEVE);
+		} else {
+			udp_tunnel_drop_rx_port(dev, gs->sock,
+						UDP_TUNNEL_TYPE_GENEVE);
+		}
+	}
 	rcu_read_unlock();
 }
 
@@ -1078,21 +1086,33 @@ static int geneve_validate(struct nlattr *tb[], struct nlattr *data[],
 			   struct netlink_ext_ack *extack)
 {
 	if (tb[IFLA_ADDRESS]) {
-		if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN)
+		if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN) {
+			NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_ADDRESS],
+					    "Provided link layer address is not Ethernet");
 			return -EINVAL;
+		}
 
-		if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS])))
+		if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS]))) {
+			NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_ADDRESS],
+					    "Provided Ethernet address is not unicast");
 			return -EADDRNOTAVAIL;
+		}
 	}
 
-	if (!data)
+	if (!data) {
+		NL_SET_ERR_MSG(extack,
+			       "Not enough attributes provided to perform the operation");
 		return -EINVAL;
+	}
 
 	if (data[IFLA_GENEVE_ID]) {
 		__u32 vni =  nla_get_u32(data[IFLA_GENEVE_ID]);
 
-		if (vni >= GENEVE_N_VID)
+		if (vni >= GENEVE_N_VID) {
+			NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_ID],
+					    "Geneve ID must be lower than 16777216");
 			return -ERANGE;
+		}
 	}
 
 	return 0;
@@ -1140,7 +1160,17 @@ static bool is_tnl_info_zero(const struct ip_tunnel_info *info)
 		return true;
 }
 
+static bool geneve_dst_addr_equal(struct ip_tunnel_info *a,
+				  struct ip_tunnel_info *b)
+{
+	if (ip_tunnel_info_af(a) == AF_INET)
+		return a->key.u.ipv4.dst == b->key.u.ipv4.dst;
+	else
+		return ipv6_addr_equal(&a->key.u.ipv6.dst, &b->key.u.ipv6.dst);
+}
+
 static int geneve_configure(struct net *net, struct net_device *dev,
+			    struct netlink_ext_ack *extack,
 			    const struct ip_tunnel_info *info,
 			    bool metadata, bool ipv6_rx_csum)
 {
@@ -1149,8 +1179,11 @@ static int geneve_configure(struct net *net, struct net_device *dev,
 	bool tun_collect_md, tun_on_same_port;
 	int err, encap_len;
 
-	if (metadata && !is_tnl_info_zero(info))
+	if (metadata && !is_tnl_info_zero(info)) {
+		NL_SET_ERR_MSG(extack,
+			       "Device is externally controlled, so attributes (VNI, Port, and so on) must not be specified");
 		return -EINVAL;
+	}
 
 	geneve->net = net;
 	geneve->dev = dev;
@@ -1171,11 +1204,17 @@ static int geneve_configure(struct net *net, struct net_device *dev,
 	dev->needed_headroom = encap_len + ETH_HLEN;
 
 	if (metadata) {
-		if (tun_on_same_port)
+		if (tun_on_same_port) {
+			NL_SET_ERR_MSG(extack,
+				       "There can be only one externally controlled device on a destination port");
 			return -EPERM;
+		}
 	} else {
-		if (tun_collect_md)
+		if (tun_collect_md) {
+			NL_SET_ERR_MSG(extack,
+				       "There already exists an externally controlled device on this destination port");
 			return -EPERM;
+		}
 	}
 
 	dst_cache_reset(&geneve->info.dst_cache);
@@ -1197,47 +1236,62 @@ static void init_tnl_info(struct ip_tunnel_info *info, __u16 dst_port)
 	info->key.tp_dst = htons(dst_port);
 }
 
-static int geneve_newlink(struct net *net, struct net_device *dev,
-			  struct nlattr *tb[], struct nlattr *data[],
-			  struct netlink_ext_ack *extack)
+static int geneve_nl2info(struct nlattr *tb[], struct nlattr *data[],
+			  struct netlink_ext_ack *extack,
+			  struct ip_tunnel_info *info, bool *metadata,
+			  bool *use_udp6_rx_checksums, bool changelink)
 {
-	bool use_udp6_rx_checksums = false;
-	struct ip_tunnel_info info;
-	bool metadata = false;
+	int attrtype;
 
-	init_tnl_info(&info, GENEVE_UDP_PORT);
-
-	if (data[IFLA_GENEVE_REMOTE] && data[IFLA_GENEVE_REMOTE6])
+	if (data[IFLA_GENEVE_REMOTE] && data[IFLA_GENEVE_REMOTE6]) {
+		NL_SET_ERR_MSG(extack,
+			       "Cannot specify both IPv4 and IPv6 Remote addresses");
 		return -EINVAL;
+	}
 
 	if (data[IFLA_GENEVE_REMOTE]) {
-		info.key.u.ipv4.dst =
+		if (changelink && (ip_tunnel_info_af(info) == AF_INET6)) {
+			attrtype = IFLA_GENEVE_REMOTE;
+			goto change_notsup;
+		}
+
+		info->key.u.ipv4.dst =
 			nla_get_in_addr(data[IFLA_GENEVE_REMOTE]);
 
-		if (IN_MULTICAST(ntohl(info.key.u.ipv4.dst))) {
-			netdev_dbg(dev, "multicast remote is unsupported\n");
+		if (IN_MULTICAST(ntohl(info->key.u.ipv4.dst))) {
+			NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_REMOTE],
+					    "Remote IPv4 address cannot be Multicast");
 			return -EINVAL;
 		}
 	}
 
 	if (data[IFLA_GENEVE_REMOTE6]) {
  #if IS_ENABLED(CONFIG_IPV6)
-		info.mode = IP_TUNNEL_INFO_IPV6;
-		info.key.u.ipv6.dst =
+		if (changelink && (ip_tunnel_info_af(info) == AF_INET)) {
+			attrtype = IFLA_GENEVE_REMOTE6;
+			goto change_notsup;
+		}
+
+		info->mode = IP_TUNNEL_INFO_IPV6;
+		info->key.u.ipv6.dst =
 			nla_get_in6_addr(data[IFLA_GENEVE_REMOTE6]);
 
-		if (ipv6_addr_type(&info.key.u.ipv6.dst) &
+		if (ipv6_addr_type(&info->key.u.ipv6.dst) &
 		    IPV6_ADDR_LINKLOCAL) {
-			netdev_dbg(dev, "link-local remote is unsupported\n");
+			NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_REMOTE6],
+					    "Remote IPv6 address cannot be link-local");
 			return -EINVAL;
 		}
-		if (ipv6_addr_is_multicast(&info.key.u.ipv6.dst)) {
-			netdev_dbg(dev, "multicast remote is unsupported\n");
+		if (ipv6_addr_is_multicast(&info->key.u.ipv6.dst)) {
+			NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_REMOTE6],
+					    "Remote IPv6 address cannot be Multicast");
 			return -EINVAL;
 		}
-		info.key.tun_flags |= TUNNEL_CSUM;
-		use_udp6_rx_checksums = true;
+		info->key.tun_flags |= TUNNEL_CSUM;
+		*use_udp6_rx_checksums = true;
 #else
+		NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_REMOTE6],
+				    "IPv6 support not enabled in the kernel");
 		return -EPFNOSUPPORT;
 #endif
 	}
@@ -1245,46 +1299,187 @@ static int geneve_newlink(struct net *net, struct net_device *dev,
 	if (data[IFLA_GENEVE_ID]) {
 		__u32 vni;
 		__u8 tvni[3];
+		__be64 tunid;
 
 		vni = nla_get_u32(data[IFLA_GENEVE_ID]);
 		tvni[0] = (vni & 0x00ff0000) >> 16;
 		tvni[1] = (vni & 0x0000ff00) >> 8;
 		tvni[2] =  vni & 0x000000ff;
 
-		info.key.tun_id = vni_to_tunnel_id(tvni);
+		tunid = vni_to_tunnel_id(tvni);
+		if (changelink && (tunid != info->key.tun_id)) {
+			attrtype = IFLA_GENEVE_ID;
+			goto change_notsup;
+		}
+		info->key.tun_id = tunid;
 	}
+
 	if (data[IFLA_GENEVE_TTL])
-		info.key.ttl = nla_get_u8(data[IFLA_GENEVE_TTL]);
+		info->key.ttl = nla_get_u8(data[IFLA_GENEVE_TTL]);
 
 	if (data[IFLA_GENEVE_TOS])
-		info.key.tos = nla_get_u8(data[IFLA_GENEVE_TOS]);
+		info->key.tos = nla_get_u8(data[IFLA_GENEVE_TOS]);
 
 	if (data[IFLA_GENEVE_LABEL]) {
-		info.key.label = nla_get_be32(data[IFLA_GENEVE_LABEL]) &
+		info->key.label = nla_get_be32(data[IFLA_GENEVE_LABEL]) &
 				  IPV6_FLOWLABEL_MASK;
-		if (info.key.label && (!(info.mode & IP_TUNNEL_INFO_IPV6)))
+		if (info->key.label && (!(info->mode & IP_TUNNEL_INFO_IPV6))) {
+			NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_LABEL],
+					    "Label attribute only applies for IPv6 Geneve devices");
 			return -EINVAL;
+		}
+	}
+
+	if (data[IFLA_GENEVE_PORT]) {
+		if (changelink) {
+			attrtype = IFLA_GENEVE_PORT;
+			goto change_notsup;
+		}
+		info->key.tp_dst = nla_get_be16(data[IFLA_GENEVE_PORT]);
+	}
+
+	if (data[IFLA_GENEVE_COLLECT_METADATA]) {
+		if (changelink) {
+			attrtype = IFLA_GENEVE_COLLECT_METADATA;
+			goto change_notsup;
+		}
+		*metadata = true;
+	}
+
+	if (data[IFLA_GENEVE_UDP_CSUM]) {
+		if (changelink) {
+			attrtype = IFLA_GENEVE_UDP_CSUM;
+			goto change_notsup;
+		}
+		if (nla_get_u8(data[IFLA_GENEVE_UDP_CSUM]))
+			info->key.tun_flags |= TUNNEL_CSUM;
+	}
+
+	if (data[IFLA_GENEVE_UDP_ZERO_CSUM6_TX]) {
+		if (changelink) {
+			attrtype = IFLA_GENEVE_UDP_ZERO_CSUM6_TX;
+			goto change_notsup;
+		}
+		if (nla_get_u8(data[IFLA_GENEVE_UDP_ZERO_CSUM6_TX]))
+			info->key.tun_flags &= ~TUNNEL_CSUM;
+	}
+
+	if (data[IFLA_GENEVE_UDP_ZERO_CSUM6_RX]) {
+		if (changelink) {
+			attrtype = IFLA_GENEVE_UDP_ZERO_CSUM6_RX;
+			goto change_notsup;
+		}
+		if (nla_get_u8(data[IFLA_GENEVE_UDP_ZERO_CSUM6_RX]))
+			*use_udp6_rx_checksums = false;
 	}
 
-	if (data[IFLA_GENEVE_PORT])
-		info.key.tp_dst = nla_get_be16(data[IFLA_GENEVE_PORT]);
+	return 0;
+change_notsup:
+	NL_SET_ERR_MSG_ATTR(extack, data[attrtype],
+			    "Changing VNI, Port, endpoint IP address family, external, and UDP checksum attributes are not supported");
+	return -EOPNOTSUPP;
+}
 
-	if (data[IFLA_GENEVE_COLLECT_METADATA])
-		metadata = true;
+static int geneve_newlink(struct net *net, struct net_device *dev,
+			  struct nlattr *tb[], struct nlattr *data[],
+			  struct netlink_ext_ack *extack)
+{
+	bool use_udp6_rx_checksums = false;
+	struct ip_tunnel_info info;
+	bool metadata = false;
+	int err;
 
-	if (data[IFLA_GENEVE_UDP_CSUM] &&
-	    nla_get_u8(data[IFLA_GENEVE_UDP_CSUM]))
-		info.key.tun_flags |= TUNNEL_CSUM;
+	init_tnl_info(&info, GENEVE_UDP_PORT);
+	err = geneve_nl2info(tb, data, extack, &info, &metadata,
+			     &use_udp6_rx_checksums, false);
+	if (err)
+		return err;
 
-	if (data[IFLA_GENEVE_UDP_ZERO_CSUM6_TX] &&
-	    nla_get_u8(data[IFLA_GENEVE_UDP_ZERO_CSUM6_TX]))
-		info.key.tun_flags &= ~TUNNEL_CSUM;
+	return geneve_configure(net, dev, extack, &info, metadata,
+				use_udp6_rx_checksums);
+}
 
-	if (data[IFLA_GENEVE_UDP_ZERO_CSUM6_RX] &&
-	    nla_get_u8(data[IFLA_GENEVE_UDP_ZERO_CSUM6_RX]))
-		use_udp6_rx_checksums = false;
+/* Quiesces the geneve device data path for both TX and RX.
+ *
+ * On transmit geneve checks for non-NULL geneve_sock before it proceeds.
+ * So, if we set that socket to NULL under RCU and wait for synchronize_net()
+ * to complete for the existing set of in-flight packets to be transmitted,
+ * then we would have quiesced the transmit data path. All the future packets
+ * will get dropped until we unquiesce the data path.
+ *
+ * On receive geneve dereference the geneve_sock stashed in the socket. So,
+ * if we set that to NULL under RCU and wait for synchronize_net() to
+ * complete, then we would have quiesced the receive data path.
+ */
+static void geneve_quiesce(struct geneve_dev *geneve, struct geneve_sock **gs4,
+			   struct geneve_sock **gs6)
+{
+	*gs4 = rtnl_dereference(geneve->sock4);
+	rcu_assign_pointer(geneve->sock4, NULL);
+	if (*gs4)
+		rcu_assign_sk_user_data((*gs4)->sock->sk, NULL);
+#if IS_ENABLED(CONFIG_IPV6)
+	*gs6 = rtnl_dereference(geneve->sock6);
+	rcu_assign_pointer(geneve->sock6, NULL);
+	if (*gs6)
+		rcu_assign_sk_user_data((*gs6)->sock->sk, NULL);
+#else
+	*gs6 = NULL;
+#endif
+	synchronize_net();
+}
+
+/* Resumes the geneve device data path for both TX and RX. */
+static void geneve_unquiesce(struct geneve_dev *geneve, struct geneve_sock *gs4,
+			     struct geneve_sock __maybe_unused *gs6)
+{
+	rcu_assign_pointer(geneve->sock4, gs4);
+	if (gs4)
+		rcu_assign_sk_user_data(gs4->sock->sk, gs4);
+#if IS_ENABLED(CONFIG_IPV6)
+	rcu_assign_pointer(geneve->sock6, gs6);
+	if (gs6)
+		rcu_assign_sk_user_data(gs6->sock->sk, gs6);
+#endif
+	synchronize_net();
+}
+
+static int geneve_changelink(struct net_device *dev, struct nlattr *tb[],
+			     struct nlattr *data[],
+			     struct netlink_ext_ack *extack)
+{
+	struct geneve_dev *geneve = netdev_priv(dev);
+	struct geneve_sock *gs4, *gs6;
+	struct ip_tunnel_info info;
+	bool metadata;
+	bool use_udp6_rx_checksums;
+	int err;
+
+	/* If the geneve device is configured for metadata (or externally
+	 * controlled, for example, OVS), then nothing can be changed.
+	 */
+	if (geneve->collect_md)
+		return -EOPNOTSUPP;
+
+	/* Start with the existing info. */
+	memcpy(&info, &geneve->info, sizeof(info));
+	metadata = geneve->collect_md;
+	use_udp6_rx_checksums = geneve->use_udp6_rx_checksums;
+	err = geneve_nl2info(tb, data, extack, &info, &metadata,
+			     &use_udp6_rx_checksums, true);
+	if (err)
+		return err;
 
-	return geneve_configure(net, dev, &info, metadata, use_udp6_rx_checksums);
+	if (!geneve_dst_addr_equal(&geneve->info, &info))
+		dst_cache_reset(&info.dst_cache);
+
+	geneve_quiesce(geneve, &gs4, &gs6);
+	geneve->info = info;
+	geneve->collect_md = metadata;
+	geneve->use_udp6_rx_checksums = use_udp6_rx_checksums;
+	geneve_unquiesce(geneve, gs4, gs6);
+
+	return 0;
 }
 
 static void geneve_dellink(struct net_device *dev, struct list_head *head)
@@ -1375,6 +1570,7 @@ static struct rtnl_link_ops geneve_link_ops __read_mostly = {
 	.setup		= geneve_setup,
 	.validate	= geneve_validate,
 	.newlink	= geneve_newlink,
+	.changelink	= geneve_changelink,
 	.dellink	= geneve_dellink,
 	.get_size	= geneve_get_size,
 	.fill_info	= geneve_fill_info,
@@ -1396,7 +1592,7 @@ struct net_device *geneve_dev_create_fb(struct net *net, const char *name,
 		return dev;
 
 	init_tnl_info(&info, dst_port);
-	err = geneve_configure(net, dev, &info, true, true);
+	err = geneve_configure(net, dev, NULL, &info, true, true);
 	if (err) {
 		free_netdev(dev);
 		return ERR_PTR(err);
@@ -1426,8 +1622,14 @@ static int geneve_netdevice_event(struct notifier_block *unused,
 {
 	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
 
-	if (event == NETDEV_UDP_TUNNEL_PUSH_INFO)
-		geneve_push_rx_ports(dev);
+	if (event == NETDEV_UDP_TUNNEL_PUSH_INFO ||
+	    event == NETDEV_UDP_TUNNEL_DROP_INFO) {
+		geneve_offload_rx_ports(dev, event == NETDEV_UDP_TUNNEL_PUSH_INFO);
+	} else if (event == NETDEV_UNREGISTER) {
+		geneve_offload_rx_ports(dev, false);
+	} else if (event == NETDEV_REGISTER) {
+		geneve_offload_rx_ports(dev, true);
+	}
 
 	return NOTIFY_DONE;
 }
diff --git a/drivers/net/hamradio/baycom_par.c b/drivers/net/hamradio/baycom_par.c
index 92b13b39f426..e1783832d304 100644
--- a/drivers/net/hamradio/baycom_par.c
+++ b/drivers/net/hamradio/baycom_par.c
@@ -386,7 +386,7 @@ static int baycom_ioctl(struct net_device *dev, struct ifreq *ifr,
 
 /* --------------------------------------------------------------------- */
 
-static struct hdlcdrv_ops par96_ops = {
+static const struct hdlcdrv_ops par96_ops = {
 	.drvname = bc_drvname,
 	.drvinfo = bc_drvinfo,
 	.open    = par96_open,
diff --git a/drivers/net/hamradio/baycom_ser_fdx.c b/drivers/net/hamradio/baycom_ser_fdx.c
index d9a646acca20..190f66c88479 100644
--- a/drivers/net/hamradio/baycom_ser_fdx.c
+++ b/drivers/net/hamradio/baycom_ser_fdx.c
@@ -508,7 +508,7 @@ static int baycom_ioctl(struct net_device *dev, struct ifreq *ifr,
 
 /* --------------------------------------------------------------------- */
 
-static struct hdlcdrv_ops ser12_ops = {
+static const struct hdlcdrv_ops ser12_ops = {
 	.drvname = bc_drvname,
 	.drvinfo = bc_drvinfo,
 	.open    = ser12_open,
diff --git a/drivers/net/hamradio/baycom_ser_hdx.c b/drivers/net/hamradio/baycom_ser_hdx.c
index f1c8a9ff3891..3c823c648cf5 100644
--- a/drivers/net/hamradio/baycom_ser_hdx.c
+++ b/drivers/net/hamradio/baycom_ser_hdx.c
@@ -542,7 +542,7 @@ static int baycom_ioctl(struct net_device *dev, struct ifreq *ifr,
 
 /* --------------------------------------------------------------------- */
 
-static struct hdlcdrv_ops ser12_ops = {
+static const struct hdlcdrv_ops ser12_ops = {
 	.drvname = bc_drvname,
 	.drvinfo = bc_drvinfo,
 	.open    = ser12_open,
diff --git a/drivers/net/hamradio/dmascc.c b/drivers/net/hamradio/dmascc.c
index dec6b76bc0fb..cde41200f40a 100644
--- a/drivers/net/hamradio/dmascc.c
+++ b/drivers/net/hamradio/dmascc.c
@@ -581,7 +581,7 @@ static int __init setup_adapter(int card_base, int type, int n)
 		priv->param.dma = -1;
 		INIT_WORK(&priv->rx_work, rx_bh);
 		dev->ml_priv = priv;
-		sprintf(dev->name, "dmascc%i", 2 * n + i);
+		snprintf(dev->name, sizeof(dev->name), "dmascc%i", 2 * n + i);
 		dev->base_addr = card_base;
 		dev->irq = irq;
 		dev->netdev_ops = &scc_netdev_ops;
diff --git a/drivers/net/hyperv/hyperv_net.h b/drivers/net/hyperv/hyperv_net.h
index 12cc64bfcff8..d98cdfb1536b 100644
--- a/drivers/net/hyperv/hyperv_net.h
+++ b/drivers/net/hyperv/hyperv_net.h
@@ -147,8 +147,9 @@ struct hv_netvsc_packet {
 struct netvsc_device_info {
 	unsigned char mac_adr[ETH_ALEN];
 	int  ring_size;
-	u32  max_num_vrss_chns;
 	u32  num_chn;
+	u32  send_sections;
+	u32  recv_sections;
 };
 
 enum rndis_device_state {
@@ -183,13 +184,16 @@ struct rndis_device {
 /* Interface */
 struct rndis_message;
 struct netvsc_device;
-int netvsc_device_add(struct hv_device *device,
-		      const struct netvsc_device_info *info);
+struct net_device_context;
+
+struct netvsc_device *netvsc_device_add(struct hv_device *device,
+					const struct netvsc_device_info *info);
+int netvsc_alloc_recv_comp_ring(struct netvsc_device *net_device, u32 q_idx);
 void netvsc_device_remove(struct hv_device *device);
-int netvsc_send(struct hv_device *device,
+int netvsc_send(struct net_device_context *ndc,
 		struct hv_netvsc_packet *packet,
 		struct rndis_message *rndis_msg,
-		struct hv_page_buffer **page_buffer,
+		struct hv_page_buffer *page_buffer,
 		struct sk_buff *skb);
 void netvsc_linkstatus_callback(struct hv_device *device_obj,
 				struct rndis_message *resp);
@@ -200,22 +204,26 @@ int netvsc_recv_callback(struct net_device *net,
 			 const struct ndis_pkt_8021q_info *vlan);
 void netvsc_channel_cb(void *context);
 int netvsc_poll(struct napi_struct *napi, int budget);
+
+void rndis_set_subchannel(struct work_struct *w);
+bool rndis_filter_opened(const struct netvsc_device *nvdev);
 int rndis_filter_open(struct netvsc_device *nvdev);
 int rndis_filter_close(struct netvsc_device *nvdev);
-int rndis_filter_device_add(struct hv_device *dev,
-			    struct netvsc_device_info *info);
+struct netvsc_device *rndis_filter_device_add(struct hv_device *dev,
+					      struct netvsc_device_info *info);
 void rndis_filter_update(struct netvsc_device *nvdev);
 void rndis_filter_device_remove(struct hv_device *dev,
 				struct netvsc_device *nvdev);
 int rndis_filter_set_rss_param(struct rndis_device *rdev,
-			       const u8 *key, int num_queue);
+			       const u8 *key);
 int rndis_filter_receive(struct net_device *ndev,
 			 struct netvsc_device *net_dev,
 			 struct hv_device *dev,
 			 struct vmbus_channel *channel,
 			 void *data, u32 buflen);
 
-int rndis_filter_set_device_mac(struct net_device *ndev, char *mac);
+int rndis_filter_set_device_mac(struct netvsc_device *ndev,
+				const char *mac);
 
 void netvsc_switch_datapath(struct net_device *nv_dev, bool vf);
 
@@ -630,12 +638,12 @@ struct nvsp_message {
 #define NETVSC_SEND_BUFFER_SIZE			(1024 * 1024 * 15)   /* 15MB */
 #define NETVSC_INVALID_INDEX			-1
 
+#define NETVSC_SEND_SECTION_SIZE		6144
+#define NETVSC_RECV_SECTION_SIZE		1728
 
 #define NETVSC_RECEIVE_BUFFER_ID		0xcafe
 #define NETVSC_SEND_BUFFER_ID			0
 
-#define NETVSC_PACKET_SIZE                      4096
-
 #define VRSS_SEND_TAB_SIZE 16  /* must be power of 2 */
 #define VRSS_CHANNEL_MAX 64
 #define VRSS_CHANNEL_DEFAULT 8
@@ -654,13 +662,10 @@ struct recv_comp_data {
 	u32 status;
 };
 
-/* Netvsc Receive Slots Max */
-#define NETVSC_RECVSLOT_MAX (NETVSC_RECEIVE_BUFFER_SIZE / ETH_DATA_LEN + 1)
-
 struct multi_recv_comp {
-	void *buf; /* queued receive completions */
-	u32 first; /* first data entry */
-	u32 next; /* next entry for writing */
+	struct recv_comp_data *slots;
+	u32 first;	/* first data entry */
+	u32 next;	/* next entry for writing */
 };
 
 struct netvsc_stats {
@@ -677,6 +682,17 @@ struct netvsc_ethtool_stats {
 	unsigned long tx_no_space;
 	unsigned long tx_too_big;
 	unsigned long tx_busy;
+	unsigned long tx_send_full;
+	unsigned long rx_comp_busy;
+};
+
+struct netvsc_vf_pcpu_stats {
+	u64     rx_packets;
+	u64     rx_bytes;
+	u64     tx_packets;
+	u64     tx_bytes;
+	struct u64_stats_sync   syncp;
+	u32	tx_dropped;
 };
 
 struct netvsc_reconfig {
@@ -706,24 +722,27 @@ struct net_device_context {
 	u32 tx_send_table[VRSS_SEND_TAB_SIZE];
 
 	/* Ethtool settings */
+	bool udp4_l4_hash;
+	bool udp6_l4_hash;
 	u8 duplex;
 	u32 speed;
 	struct netvsc_ethtool_stats eth_stats;
 
 	/* State to manage the associated VF interface. */
 	struct net_device __rcu *vf_netdev;
+	struct netvsc_vf_pcpu_stats __percpu *vf_stats;
+	struct delayed_work vf_takeover;
 
 	/* 1: allocated, serial number is valid. 0: not allocated */
 	u32 vf_alloc;
 	/* Serial number of the VF to team with */
 	u32 vf_serial;
-
-	bool datapath;	/* 0 - synthetic, 1 - VF nic */
 };
 
 /* Per channel data */
 struct netvsc_channel {
 	struct vmbus_channel *channel;
+	struct netvsc_device *net_device;
 	const struct vmpacket_descriptor *desc;
 	struct napi_struct napi;
 	struct multi_send_data msd;
@@ -743,14 +762,13 @@ struct netvsc_device {
 
 	/* Receive buffer allocated by us but manages by NetVSP */
 	void *recv_buf;
-	u32 recv_buf_size;
 	u32 recv_buf_gpadl_handle;
 	u32 recv_section_cnt;
-	struct nvsp_1_receive_buffer_section *recv_section;
+	u32 recv_section_size;
+	u32 recv_completion_cnt;
 
 	/* Send buffer allocated by us */
 	void *send_buf;
-	u32 send_buf_size;
 	u32 send_buf_gpadl_handle;
 	u32 send_section_cnt;
 	u32 send_section_size;
@@ -766,6 +784,7 @@ struct netvsc_device {
 	u32 num_chn;
 
 	atomic_t open_chn;
+	struct work_struct subchan_work;
 	wait_queue_head_t subchan_open;
 
 	struct rndis_device *extension;
@@ -775,8 +794,6 @@ struct netvsc_device {
 	u32 max_pkt; /* max number of pkt in one send, e.g. 8 */
 	u32 pkt_align; /* alignment bytes, e.g. 8 */
 
-	atomic_t num_outstanding_recvs;
-
 	atomic_t open_cnt;
 
 	struct netvsc_channel chan_table[VRSS_CHANNEL_MAX];
@@ -784,18 +801,6 @@ struct netvsc_device {
 	struct rcu_head rcu;
 };
 
-static inline struct netvsc_device *
-net_device_to_netvsc_device(struct net_device *ndev)
-{
-	return ((struct net_device_context *)netdev_priv(ndev))->nvdev;
-}
-
-static inline struct netvsc_device *
-hv_device_to_netvsc_device(struct hv_device *device)
-{
-	return net_device_to_netvsc_device(hv_get_drvdata(device));
-}
-
 /* NdisInitialize message */
 struct rndis_initialize_request {
 	u32 req_id;
diff --git a/drivers/net/hyperv/netvsc.c b/drivers/net/hyperv/netvsc.c
index d18c3326a1f7..a5511b7326af 100644
--- a/drivers/net/hyperv/netvsc.c
+++ b/drivers/net/hyperv/netvsc.c
@@ -29,6 +29,9 @@
 #include <linux/netdevice.h>
 #include <linux/if_ether.h>
 #include <linux/vmalloc.h>
+#include <linux/rtnetlink.h>
+#include <linux/prefetch.h>
+
 #include <asm/sync_bitops.h>
 
 #include "hyperv_net.h"
@@ -41,7 +44,7 @@ void netvsc_switch_datapath(struct net_device *ndev, bool vf)
 {
 	struct net_device_context *net_device_ctx = netdev_priv(ndev);
 	struct hv_device *dev = net_device_ctx->device_ctx;
-	struct netvsc_device *nv_dev = net_device_ctx->nvdev;
+	struct netvsc_device *nv_dev = rtnl_dereference(net_device_ctx->nvdev);
 	struct nvsp_message *init_pkt = &nv_dev->channel_init_pkt;
 
 	memset(init_pkt, 0, sizeof(struct nvsp_message));
@@ -57,8 +60,6 @@ void netvsc_switch_datapath(struct net_device *ndev, bool vf)
 			       sizeof(struct nvsp_message),
 			       (unsigned long)init_pkt,
 			       VM_PKT_DATA_INBAND, 0);
-
-	net_device_ctx->datapath = vf;
 }
 
 static struct netvsc_device *alloc_net_device(void)
@@ -69,16 +70,18 @@ static struct netvsc_device *alloc_net_device(void)
 	if (!net_device)
 		return NULL;
 
-	net_device->chan_table[0].mrc.buf
-		= vzalloc(NETVSC_RECVSLOT_MAX * sizeof(struct recv_comp_data));
-
 	init_waitqueue_head(&net_device->wait_drain);
 	net_device->destroy = false;
 	atomic_set(&net_device->open_cnt, 0);
 	net_device->max_pkt = RNDIS_MAX_PKT_DEFAULT;
 	net_device->pkt_align = RNDIS_PKT_ALIGN_DEFAULT;
+
+	net_device->recv_section_size = NETVSC_RECV_SECTION_SIZE;
+	net_device->send_section_size = NETVSC_SEND_SECTION_SIZE;
+
 	init_completion(&net_device->channel_init_wait);
 	init_waitqueue_head(&net_device->subchan_open);
+	INIT_WORK(&net_device->subchan_work, rndis_set_subchannel);
 
 	return net_device;
 }
@@ -90,7 +93,7 @@ static void free_netvsc_device(struct rcu_head *head)
 	int i;
 
 	for (i = 0; i < VRSS_CHANNEL_MAX; i++)
-		vfree(nvdev->chan_table[i].mrc.buf);
+		vfree(nvdev->chan_table[i].mrc.slots);
 
 	kfree(nvdev);
 }
@@ -104,7 +107,8 @@ static void netvsc_destroy_buf(struct hv_device *device)
 {
 	struct nvsp_message *revoke_packet;
 	struct net_device *ndev = hv_get_drvdata(device);
-	struct netvsc_device *net_device = net_device_to_netvsc_device(ndev);
+	struct net_device_context *ndc = netdev_priv(ndev);
+	struct netvsc_device *net_device = rtnl_dereference(ndc->nvdev);
 	int ret;
 
 	/*
@@ -144,6 +148,7 @@ static void netvsc_destroy_buf(struct hv_device *device)
 				"revoke receive buffer to netvsp\n");
 			return;
 		}
+		net_device->recv_section_cnt = 0;
 	}
 
 	/* Teardown the gpadl on the vsp end */
@@ -168,19 +173,13 @@ static void netvsc_destroy_buf(struct hv_device *device)
 		net_device->recv_buf = NULL;
 	}
 
-	if (net_device->recv_section) {
-		net_device->recv_section_cnt = 0;
-		kfree(net_device->recv_section);
-		net_device->recv_section = NULL;
-	}
-
 	/* Deal with the send buffer we may have setup.
 	 * If we got a  send section size, it means we received a
 	 * NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE msg (ie sent
 	 * NVSP_MSG1_TYPE_SEND_SEND_BUF msg) therefore, we need
 	 * to send a revoke msg here
 	 */
-	if (net_device->send_section_size) {
+	if (net_device->send_section_cnt) {
 		/* Send the revoke receive buffer */
 		revoke_packet = &net_device->revoke_packet;
 		memset(revoke_packet, 0, sizeof(struct nvsp_message));
@@ -212,6 +211,7 @@ static void netvsc_destroy_buf(struct hv_device *device)
 				   "revoke send buffer to netvsp\n");
 			return;
 		}
+		net_device->send_section_cnt = 0;
 	}
 	/* Teardown the gpadl on the vsp end */
 	if (net_device->send_buf_gpadl_handle) {
@@ -236,25 +236,40 @@ static void netvsc_destroy_buf(struct hv_device *device)
 	kfree(net_device->send_section_map);
 }
 
+int netvsc_alloc_recv_comp_ring(struct netvsc_device *net_device, u32 q_idx)
+{
+	struct netvsc_channel *nvchan = &net_device->chan_table[q_idx];
+	int node = cpu_to_node(nvchan->channel->target_cpu);
+	size_t size;
+
+	size = net_device->recv_completion_cnt * sizeof(struct recv_comp_data);
+	nvchan->mrc.slots = vzalloc_node(size, node);
+	if (!nvchan->mrc.slots)
+		nvchan->mrc.slots = vzalloc(size);
+
+	return nvchan->mrc.slots ? 0 : -ENOMEM;
+}
+
 static int netvsc_init_buf(struct hv_device *device,
-			   struct netvsc_device *net_device)
+			   struct netvsc_device *net_device,
+			   const struct netvsc_device_info *device_info)
 {
-	int ret = 0;
+	struct nvsp_1_message_send_receive_buffer_complete *resp;
+	struct net_device *ndev = hv_get_drvdata(device);
 	struct nvsp_message *init_packet;
-	struct net_device *ndev;
+	unsigned int buf_size;
 	size_t map_words;
-	int node;
-
-	ndev = hv_get_drvdata(device);
+	int ret = 0;
 
-	node = cpu_to_node(device->channel->target_cpu);
-	net_device->recv_buf = vzalloc_node(net_device->recv_buf_size, node);
-	if (!net_device->recv_buf)
-		net_device->recv_buf = vzalloc(net_device->recv_buf_size);
+	/* Get receive buffer area. */
+	buf_size = device_info->recv_sections * net_device->recv_section_size;
+	buf_size = roundup(buf_size, PAGE_SIZE);
 
+	net_device->recv_buf = vzalloc(buf_size);
 	if (!net_device->recv_buf) {
-		netdev_err(ndev, "unable to allocate receive "
-			"buffer of size %d\n", net_device->recv_buf_size);
+		netdev_err(ndev,
+			   "unable to allocate receive buffer of size %u\n",
+			   buf_size);
 		ret = -ENOMEM;
 		goto cleanup;
 	}
@@ -265,7 +280,7 @@ static int netvsc_init_buf(struct hv_device *device,
 	 * than the channel to establish the gpadl handle.
 	 */
 	ret = vmbus_establish_gpadl(device->channel, net_device->recv_buf,
-				    net_device->recv_buf_size,
+				    buf_size,
 				    &net_device->recv_buf_gpadl_handle);
 	if (ret != 0) {
 		netdev_err(ndev,
@@ -297,49 +312,45 @@ static int netvsc_init_buf(struct hv_device *device,
 	wait_for_completion(&net_device->channel_init_wait);
 
 	/* Check the response */
-	if (init_packet->msg.v1_msg.
-	    send_recv_buf_complete.status != NVSP_STAT_SUCCESS) {
-		netdev_err(ndev, "Unable to complete receive buffer "
-			   "initialization with NetVsp - status %d\n",
-			   init_packet->msg.v1_msg.
-			   send_recv_buf_complete.status);
+	resp = &init_packet->msg.v1_msg.send_recv_buf_complete;
+	if (resp->status != NVSP_STAT_SUCCESS) {
+		netdev_err(ndev,
+			   "Unable to complete receive buffer initialization with NetVsp - status %d\n",
+			   resp->status);
 		ret = -EINVAL;
 		goto cleanup;
 	}
 
 	/* Parse the response */
+	netdev_dbg(ndev, "Receive sections: %u sub_allocs: size %u count: %u\n",
+		   resp->num_sections, resp->sections[0].sub_alloc_size,
+		   resp->sections[0].num_sub_allocs);
 
-	net_device->recv_section_cnt = init_packet->msg.
-		v1_msg.send_recv_buf_complete.num_sections;
-
-	net_device->recv_section = kmemdup(
-		init_packet->msg.v1_msg.send_recv_buf_complete.sections,
-		net_device->recv_section_cnt *
-		sizeof(struct nvsp_1_receive_buffer_section),
-		GFP_KERNEL);
-	if (net_device->recv_section == NULL) {
+	/* There should only be one section for the entire receive buffer */
+	if (resp->num_sections != 1 || resp->sections[0].offset != 0) {
 		ret = -EINVAL;
 		goto cleanup;
 	}
 
-	/*
-	 * For 1st release, there should only be 1 section that represents the
-	 * entire receive buffer
-	 */
-	if (net_device->recv_section_cnt != 1 ||
-	    net_device->recv_section->offset != 0) {
-		ret = -EINVAL;
+	net_device->recv_section_size = resp->sections[0].sub_alloc_size;
+	net_device->recv_section_cnt = resp->sections[0].num_sub_allocs;
+
+	/* Setup receive completion ring */
+	net_device->recv_completion_cnt
+		= round_up(net_device->recv_section_cnt + 1,
+			   PAGE_SIZE / sizeof(u64));
+	ret = netvsc_alloc_recv_comp_ring(net_device, 0);
+	if (ret)
 		goto cleanup;
-	}
 
-	/* Now setup the send buffer.
-	 */
-	net_device->send_buf = vzalloc_node(net_device->send_buf_size, node);
-	if (!net_device->send_buf)
-		net_device->send_buf = vzalloc(net_device->send_buf_size);
+	/* Now setup the send buffer. */
+	buf_size = device_info->send_sections * net_device->send_section_size;
+	buf_size = round_up(buf_size, PAGE_SIZE);
+
+	net_device->send_buf = vzalloc(buf_size);
 	if (!net_device->send_buf) {
-		netdev_err(ndev, "unable to allocate send "
-			   "buffer of size %d\n", net_device->send_buf_size);
+		netdev_err(ndev, "unable to allocate send buffer of size %u\n",
+			   buf_size);
 		ret = -ENOMEM;
 		goto cleanup;
 	}
@@ -349,7 +360,7 @@ static int netvsc_init_buf(struct hv_device *device,
 	 * than the channel to establish the gpadl handle.
 	 */
 	ret = vmbus_establish_gpadl(device->channel, net_device->send_buf,
-				    net_device->send_buf_size,
+				    buf_size,
 				    &net_device->send_buf_gpadl_handle);
 	if (ret != 0) {
 		netdev_err(ndev,
@@ -394,10 +405,8 @@ static int netvsc_init_buf(struct hv_device *device,
 	net_device->send_section_size = init_packet->msg.
 				v1_msg.send_send_buf_complete.section_size;
 
-	/* Section count is simply the size divided by the section size.
-	 */
-	net_device->send_section_cnt =
-		net_device->send_buf_size / net_device->send_section_size;
+	/* Section count is simply the size divided by the section size. */
+	net_device->send_section_cnt = buf_size / net_device->send_section_size;
 
 	netdev_dbg(ndev, "Send section size: %d, Section count:%d\n",
 		   net_device->send_section_size, net_device->send_section_cnt);
@@ -475,7 +484,8 @@ static int negotiate_nvsp_ver(struct hv_device *device,
 }
 
 static int netvsc_connect_vsp(struct hv_device *device,
-			      struct netvsc_device *net_device)
+			      struct netvsc_device *net_device,
+			      const struct netvsc_device_info *device_info)
 {
 	const u32 ver_list[] = {
 		NVSP_PROTOCOL_VERSION_1, NVSP_PROTOCOL_VERSION_2,
@@ -525,14 +535,8 @@ static int netvsc_connect_vsp(struct hv_device *device,
 	if (ret != 0)
 		goto cleanup;
 
-	/* Post the big receive buffer to NetVSP */
-	if (net_device->nvsp_version <= NVSP_PROTOCOL_VERSION_2)
-		net_device->recv_buf_size = NETVSC_RECEIVE_BUFFER_SIZE_LEGACY;
-	else
-		net_device->recv_buf_size = NETVSC_RECEIVE_BUFFER_SIZE;
-	net_device->send_buf_size = NETVSC_SEND_BUFFER_SIZE;
 
-	ret = netvsc_init_buf(device, net_device);
+	ret = netvsc_init_buf(device, net_device, device_info);
 
 cleanup:
 	return ret;
@@ -550,9 +554,12 @@ void netvsc_device_remove(struct hv_device *device)
 {
 	struct net_device *ndev = hv_get_drvdata(device);
 	struct net_device_context *net_device_ctx = netdev_priv(ndev);
-	struct netvsc_device *net_device = net_device_ctx->nvdev;
+	struct netvsc_device *net_device
+		= rtnl_dereference(net_device_ctx->nvdev);
 	int i;
 
+	cancel_work_sync(&net_device->subchan_work);
+
 	netvsc_disconnect_vsp(device);
 
 	RCU_INIT_POINTER(net_device_ctx->nvdev, NULL);
@@ -693,7 +700,7 @@ static u32 netvsc_copy_to_send_buf(struct netvsc_device *net_device,
 				   u32 pend_size,
 				   struct hv_netvsc_packet *packet,
 				   struct rndis_message *rndis_msg,
-				   struct hv_page_buffer **pb,
+				   struct hv_page_buffer *pb,
 				   struct sk_buff *skb)
 {
 	char *start = net_device->send_buf;
@@ -714,9 +721,9 @@ static u32 netvsc_copy_to_send_buf(struct netvsc_device *net_device,
 	}
 
 	for (i = 0; i < page_count; i++) {
-		char *src = phys_to_virt((*pb)[i].pfn << PAGE_SHIFT);
-		u32 offset = (*pb)[i].offset;
-		u32 len = (*pb)[i].len;
+		char *src = phys_to_virt(pb[i].pfn << PAGE_SHIFT);
+		u32 offset = pb[i].offset;
+		u32 len = pb[i].len;
 
 		memcpy(dest, (src + offset), len);
 		msg_size += len;
@@ -735,36 +742,32 @@ static inline int netvsc_send_pkt(
 	struct hv_device *device,
 	struct hv_netvsc_packet *packet,
 	struct netvsc_device *net_device,
-	struct hv_page_buffer **pb,
+	struct hv_page_buffer *pb,
 	struct sk_buff *skb)
 {
 	struct nvsp_message nvmsg;
-	struct netvsc_channel *nvchan
-		= &net_device->chan_table[packet->q_idx];
+	struct nvsp_1_message_send_rndis_packet * const rpkt =
+		&nvmsg.msg.v1_msg.send_rndis_pkt;
+	struct netvsc_channel * const nvchan =
+		&net_device->chan_table[packet->q_idx];
 	struct vmbus_channel *out_channel = nvchan->channel;
 	struct net_device *ndev = hv_get_drvdata(device);
 	struct netdev_queue *txq = netdev_get_tx_queue(ndev, packet->q_idx);
 	u64 req_id;
 	int ret;
-	struct hv_page_buffer *pgbuf;
 	u32 ring_avail = hv_ringbuf_avail_percent(&out_channel->outbound);
 
 	nvmsg.hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT;
-	if (skb != NULL) {
-		/* 0 is RMC_DATA; */
-		nvmsg.msg.v1_msg.send_rndis_pkt.channel_type = 0;
-	} else {
-		/* 1 is RMC_CONTROL; */
-		nvmsg.msg.v1_msg.send_rndis_pkt.channel_type = 1;
-	}
+	if (skb)
+		rpkt->channel_type = 0;		/* 0 is RMC_DATA */
+	else
+		rpkt->channel_type = 1;		/* 1 is RMC_CONTROL */
 
-	nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_index =
-		packet->send_buf_index;
+	rpkt->send_buf_section_index = packet->send_buf_index;
 	if (packet->send_buf_index == NETVSC_INVALID_INDEX)
-		nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_size = 0;
+		rpkt->send_buf_section_size = 0;
 	else
-		nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_size =
-			packet->total_data_buflen;
+		rpkt->send_buf_section_size = packet->total_data_buflen;
 
 	req_id = (ulong)skb;
 
@@ -772,21 +775,18 @@ static inline int netvsc_send_pkt(
 		return -ENODEV;
 
 	if (packet->page_buf_cnt) {
-		pgbuf = packet->cp_partial ? (*pb) +
-			packet->rmsg_pgcnt : (*pb);
-		ret = vmbus_sendpacket_pagebuffer_ctl(out_channel,
-						      pgbuf,
-						      packet->page_buf_cnt,
-						      &nvmsg,
-						      sizeof(struct nvsp_message),
-						      req_id,
-						      VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
+		if (packet->cp_partial)
+			pb += packet->rmsg_pgcnt;
+
+		ret = vmbus_sendpacket_pagebuffer(out_channel,
+						  pb, packet->page_buf_cnt,
+						  &nvmsg, sizeof(nvmsg),
+						  req_id);
 	} else {
-		ret = vmbus_sendpacket_ctl(out_channel, &nvmsg,
-					   sizeof(struct nvsp_message),
-					   req_id,
-					   VM_PKT_DATA_INBAND,
-					   VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
+		ret = vmbus_sendpacket(out_channel,
+				       &nvmsg, sizeof(nvmsg),
+				       req_id, VM_PKT_DATA_INBAND,
+				       VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 	}
 
 	if (ret == 0) {
@@ -801,8 +801,10 @@ static inline int netvsc_send_pkt(
 			ret = -ENOSPC;
 		}
 	} else {
-		netdev_err(ndev, "Unable to send packet %p ret %d\n",
-			   packet, ret);
+		netdev_err(ndev,
+			   "Unable to send packet pages %u len %u, ret %d\n",
+			   packet->page_buf_cnt, packet->total_data_buflen,
+			   ret);
 	}
 
 	return ret;
@@ -820,13 +822,16 @@ static inline void move_pkt_msd(struct hv_netvsc_packet **msd_send,
 	msdp->count = 0;
 }
 
-int netvsc_send(struct hv_device *device,
+/* RCU already held by caller */
+int netvsc_send(struct net_device_context *ndev_ctx,
 		struct hv_netvsc_packet *packet,
 		struct rndis_message *rndis_msg,
-		struct hv_page_buffer **pb,
+		struct hv_page_buffer *pb,
 		struct sk_buff *skb)
 {
-	struct netvsc_device *net_device = hv_device_to_netvsc_device(device);
+	struct netvsc_device *net_device
+		= rcu_dereference_bh(ndev_ctx->nvdev);
+	struct hv_device *device = ndev_ctx->device_ctx;
 	int ret = 0;
 	struct netvsc_channel *nvchan;
 	u32 pktlen = packet->total_data_buflen, msd_len = 0;
@@ -838,7 +843,7 @@ int netvsc_send(struct hv_device *device,
 	bool xmit_more = (skb != NULL) ? skb->xmit_more : false;
 
 	/* If device is rescinded, return error and packet will get dropped. */
-	if (unlikely(net_device->destroy))
+	if (unlikely(!net_device || net_device->destroy))
 		return -ENODEV;
 
 	/* We may race with netvsc_connect_vsp()/netvsc_init_buf() and get
@@ -878,7 +883,9 @@ int netvsc_send(struct hv_device *device,
 	} else if (pktlen + net_device->pkt_align <
 		   net_device->send_section_size) {
 		section_index = netvsc_get_next_send_section(net_device);
-		if (section_index != NETVSC_INVALID_INDEX) {
+		if (unlikely(section_index == NETVSC_INVALID_INDEX)) {
+			++ndev_ctx->eth_stats.tx_send_full;
+		} else {
 			move_pkt_msd(&msd_send, &msd_skb, msdp);
 			msd_len = 0;
 		}
@@ -943,130 +950,99 @@ send_now:
 	return ret;
 }
 
-static int netvsc_send_recv_completion(struct vmbus_channel *channel,
-				       u64 transaction_id, u32 status)
+/* Send pending recv completions */
+static int send_recv_completions(struct net_device *ndev,
+				 struct netvsc_device *nvdev,
+				 struct netvsc_channel *nvchan)
 {
-	struct nvsp_message recvcompMessage;
+	struct multi_recv_comp *mrc = &nvchan->mrc;
+	struct recv_comp_msg {
+		struct nvsp_message_header hdr;
+		u32 status;
+	}  __packed;
+	struct recv_comp_msg msg = {
+		.hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE,
+	};
 	int ret;
 
-	recvcompMessage.hdr.msg_type =
-				NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE;
+	while (mrc->first != mrc->next) {
+		const struct recv_comp_data *rcd
+			= mrc->slots + mrc->first;
 
-	recvcompMessage.msg.v1_msg.send_rndis_pkt_complete.status = status;
-
-	/* Send the completion */
-	ret = vmbus_sendpacket(channel, &recvcompMessage,
-			       sizeof(struct nvsp_message_header) + sizeof(u32),
-			       transaction_id, VM_PKT_COMP, 0);
-
-	return ret;
-}
-
-static inline void count_recv_comp_slot(struct netvsc_device *nvdev, u16 q_idx,
-					u32 *filled, u32 *avail)
-{
-	struct multi_recv_comp *mrc = &nvdev->chan_table[q_idx].mrc;
-	u32 first = mrc->first;
-	u32 next = mrc->next;
+		msg.status = rcd->status;
+		ret = vmbus_sendpacket(nvchan->channel, &msg, sizeof(msg),
+				       rcd->tid, VM_PKT_COMP, 0);
+		if (unlikely(ret)) {
+			struct net_device_context *ndev_ctx = netdev_priv(ndev);
 
-	*filled = (first > next) ? NETVSC_RECVSLOT_MAX - first + next :
-		  next - first;
-
-	*avail = NETVSC_RECVSLOT_MAX - *filled - 1;
-}
-
-/* Read the first filled slot, no change to index */
-static inline struct recv_comp_data *read_recv_comp_slot(struct netvsc_device
-							 *nvdev, u16 q_idx)
-{
-	struct multi_recv_comp *mrc = &nvdev->chan_table[q_idx].mrc;
-	u32 filled, avail;
+			++ndev_ctx->eth_stats.rx_comp_busy;
+			return ret;
+		}
 
-	if (unlikely(!mrc->buf))
-		return NULL;
+		if (++mrc->first == nvdev->recv_completion_cnt)
+			mrc->first = 0;
+	}
 
-	count_recv_comp_slot(nvdev, q_idx, &filled, &avail);
-	if (!filled)
-		return NULL;
+	/* receive completion ring has been emptied */
+	if (unlikely(nvdev->destroy))
+		wake_up(&nvdev->wait_drain);
 
-	return mrc->buf + mrc->first * sizeof(struct recv_comp_data);
+	return 0;
 }
 
-/* Put the first filled slot back to available pool */
-static inline void put_recv_comp_slot(struct netvsc_device *nvdev, u16 q_idx)
+/* Count how many receive completions are outstanding */
+static void recv_comp_slot_avail(const struct netvsc_device *nvdev,
+				 const struct multi_recv_comp *mrc,
+				 u32 *filled, u32 *avail)
 {
-	struct multi_recv_comp *mrc = &nvdev->chan_table[q_idx].mrc;
-	int num_recv;
-
-	mrc->first = (mrc->first + 1) % NETVSC_RECVSLOT_MAX;
+	u32 count = nvdev->recv_completion_cnt;
 
-	num_recv = atomic_dec_return(&nvdev->num_outstanding_recvs);
+	if (mrc->next >= mrc->first)
+		*filled = mrc->next - mrc->first;
+	else
+		*filled = (count - mrc->first) + mrc->next;
 
-	if (nvdev->destroy && num_recv == 0)
-		wake_up(&nvdev->wait_drain);
+	*avail = count - *filled - 1;
 }
 
-/* Check and send pending recv completions */
-static void netvsc_chk_recv_comp(struct netvsc_device *nvdev,
-				 struct vmbus_channel *channel, u16 q_idx)
+/* Add receive complete to ring to send to host. */
+static void enq_receive_complete(struct net_device *ndev,
+				 struct netvsc_device *nvdev, u16 q_idx,
+				 u64 tid, u32 status)
 {
+	struct netvsc_channel *nvchan = &nvdev->chan_table[q_idx];
+	struct multi_recv_comp *mrc = &nvchan->mrc;
 	struct recv_comp_data *rcd;
-	int ret;
-
-	while (true) {
-		rcd = read_recv_comp_slot(nvdev, q_idx);
-		if (!rcd)
-			break;
+	u32 filled, avail;
 
-		ret = netvsc_send_recv_completion(channel, rcd->tid,
-						  rcd->status);
-		if (ret)
-			break;
+	recv_comp_slot_avail(nvdev, mrc, &filled, &avail);
 
-		put_recv_comp_slot(nvdev, q_idx);
+	if (unlikely(filled > NAPI_POLL_WEIGHT)) {
+		send_recv_completions(ndev, nvdev, nvchan);
+		recv_comp_slot_avail(nvdev, mrc, &filled, &avail);
 	}
-}
-
-#define NETVSC_RCD_WATERMARK 80
-
-/* Get next available slot */
-static inline struct recv_comp_data *get_recv_comp_slot(
-	struct netvsc_device *nvdev, struct vmbus_channel *channel, u16 q_idx)
-{
-	struct multi_recv_comp *mrc = &nvdev->chan_table[q_idx].mrc;
-	u32 filled, avail, next;
-	struct recv_comp_data *rcd;
 
-	if (unlikely(!nvdev->recv_section))
-		return NULL;
-
-	if (unlikely(!mrc->buf))
-		return NULL;
-
-	if (atomic_read(&nvdev->num_outstanding_recvs) >
-	    nvdev->recv_section->num_sub_allocs * NETVSC_RCD_WATERMARK / 100)
-		netvsc_chk_recv_comp(nvdev, channel, q_idx);
-
-	count_recv_comp_slot(nvdev, q_idx, &filled, &avail);
-	if (!avail)
-		return NULL;
-
-	next = mrc->next;
-	rcd = mrc->buf + next * sizeof(struct recv_comp_data);
-	mrc->next = (next + 1) % NETVSC_RECVSLOT_MAX;
+	if (unlikely(!avail)) {
+		netdev_err(ndev, "Recv_comp full buf q:%hd, tid:%llx\n",
+			   q_idx, tid);
+		return;
+	}
 
-	atomic_inc(&nvdev->num_outstanding_recvs);
+	rcd = mrc->slots + mrc->next;
+	rcd->tid = tid;
+	rcd->status = status;
 
-	return rcd;
+	if (++mrc->next == nvdev->recv_completion_cnt)
+		mrc->next = 0;
 }
 
 static int netvsc_receive(struct net_device *ndev,
-		   struct netvsc_device *net_device,
-		   struct net_device_context *net_device_ctx,
-		   struct hv_device *device,
-		   struct vmbus_channel *channel,
-		   const struct vmpacket_descriptor *desc,
-		   struct nvsp_message *nvsp)
+			  struct netvsc_device *net_device,
+			  struct net_device_context *net_device_ctx,
+			  struct hv_device *device,
+			  struct vmbus_channel *channel,
+			  const struct vmpacket_descriptor *desc,
+			  struct nvsp_message *nvsp)
 {
 	const struct vmtransfer_page_packet_header *vmxferpage_packet
 		= container_of(desc, const struct vmtransfer_page_packet_header, d);
@@ -1075,7 +1051,6 @@ static int netvsc_receive(struct net_device *ndev,
 	u32 status = NVSP_STAT_SUCCESS;
 	int i;
 	int count = 0;
-	int ret;
 
 	/* Make sure this is a valid nvsp packet */
 	if (unlikely(nvsp->hdr.msg_type != NVSP_MSG1_TYPE_SEND_RNDIS_PKT)) {
@@ -1106,25 +1081,9 @@ static int netvsc_receive(struct net_device *ndev,
 					      channel, data, buflen);
 	}
 
-	if (net_device->chan_table[q_idx].mrc.buf) {
-		struct recv_comp_data *rcd;
+	enq_receive_complete(ndev, net_device, q_idx,
+			     vmxferpage_packet->d.trans_id, status);
 
-		rcd = get_recv_comp_slot(net_device, channel, q_idx);
-		if (rcd) {
-			rcd->tid = vmxferpage_packet->d.trans_id;
-			rcd->status = status;
-		} else {
-			netdev_err(ndev, "Recv_comp full buf q:%hd, tid:%llx\n",
-				   q_idx, vmxferpage_packet->d.trans_id);
-		}
-	} else {
-		ret = netvsc_send_recv_completion(channel,
-						  vmxferpage_packet->d.trans_id,
-						  status);
-		if (ret)
-			netdev_err(ndev, "Recv_comp q:%hd, tid:%llx, err:%d\n",
-				   q_idx, vmxferpage_packet->d.trans_id, ret);
-	}
 	return count;
 }
 
@@ -1220,11 +1179,10 @@ int netvsc_poll(struct napi_struct *napi, int budget)
 {
 	struct netvsc_channel *nvchan
 		= container_of(napi, struct netvsc_channel, napi);
+	struct netvsc_device *net_device = nvchan->net_device;
 	struct vmbus_channel *channel = nvchan->channel;
 	struct hv_device *device = netvsc_channel_to_device(channel);
-	u16 q_idx = channel->offermsg.offer.sub_channel_index;
 	struct net_device *ndev = hv_get_drvdata(device);
-	struct netvsc_device *net_device = net_device_to_netvsc_device(ndev);
 	int work_done = 0;
 
 	/* If starting a new interval */
@@ -1237,17 +1195,19 @@ int netvsc_poll(struct napi_struct *napi, int budget)
 		nvchan->desc = hv_pkt_iter_next(channel, nvchan->desc);
 	}
 
-	/* If receive ring was exhausted
-	 * and not doing busy poll
+	/* If send of pending receive completions suceeded
+	 *   and did not exhaust NAPI budget this time
+	 *   and not doing busy poll
 	 * then re-enable host interrupts
-	 *  and reschedule if ring is not empty.
+	 *     and reschedule if ring is not empty.
 	 */
-	if (work_done < budget &&
+	if (send_recv_completions(ndev, net_device, nvchan) == 0 &&
+	    work_done < budget &&
 	    napi_complete_done(napi, work_done) &&
-	    hv_end_read(&channel->inbound) != 0)
+	    hv_end_read(&channel->inbound)) {
+		hv_begin_read(&channel->inbound);
 		napi_reschedule(napi);
-
-	netvsc_chk_recv_comp(net_device, channel, q_idx);
+	}
 
 	/* Driver may overshoot since multiple packets per descriptor */
 	return min(work_done, budget);
@@ -1259,10 +1219,15 @@ int netvsc_poll(struct napi_struct *napi, int budget)
 void netvsc_channel_cb(void *context)
 {
 	struct netvsc_channel *nvchan = context;
+	struct vmbus_channel *channel = nvchan->channel;
+	struct hv_ring_buffer_info *rbi = &channel->inbound;
+
+	/* preload first vmpacket descriptor */
+	prefetch(hv_get_ring_buffer(rbi) + rbi->priv_read_index);
 
 	if (napi_schedule_prep(&nvchan->napi)) {
 		/* disable interupts from host */
-		hv_begin_read(&nvchan->channel->inbound);
+		hv_begin_read(rbi);
 
 		__napi_schedule(&nvchan->napi);
 	}
@@ -1272,8 +1237,8 @@ void netvsc_channel_cb(void *context)
  * netvsc_device_add - Callback when the device belonging to this
  * driver is added
  */
-int netvsc_device_add(struct hv_device *device,
-		      const struct netvsc_device_info *device_info)
+struct netvsc_device *netvsc_device_add(struct hv_device *device,
+				const struct netvsc_device_info *device_info)
 {
 	int i, ret = 0;
 	int ring_size = device_info->ring_size;
@@ -1283,7 +1248,7 @@ int netvsc_device_add(struct hv_device *device,
 
 	net_device = alloc_net_device();
 	if (!net_device)
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
 
 	net_device->ring_size = ring_size;
 
@@ -1303,6 +1268,7 @@ int netvsc_device_add(struct hv_device *device,
 		struct netvsc_channel *nvchan = &net_device->chan_table[i];
 
 		nvchan->channel = device->channel;
+		nvchan->net_device = net_device;
 		u64_stats_init(&nvchan->tx_stats.syncp);
 		u64_stats_init(&nvchan->rx_stats.syncp);
 	}
@@ -1334,17 +1300,18 @@ int netvsc_device_add(struct hv_device *device,
 	rcu_assign_pointer(net_device_ctx->nvdev, net_device);
 
 	/* Connect with the NetVsp */
-	ret = netvsc_connect_vsp(device, net_device);
+	ret = netvsc_connect_vsp(device, net_device, device_info);
 	if (ret != 0) {
 		netdev_err(ndev,
 			"unable to connect to NetVSP - %d\n", ret);
 		goto close;
 	}
 
-	return ret;
+	return net_device;
 
 close:
-	netif_napi_del(&net_device->chan_table[0].napi);
+	RCU_INIT_POINTER(net_device_ctx->nvdev, NULL);
+	napi_disable(&net_device->chan_table[0].napi);
 
 	/* Now, we can close the channel safely */
 	vmbus_close(device->channel);
@@ -1352,6 +1319,5 @@ close:
 cleanup:
 	free_netvsc_device(&net_device->rcu);
 
-	return ret;
-
+	return ERR_PTR(ret);
 }
diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c
index 0d78727f1a14..d4902ee5f260 100644
--- a/drivers/net/hyperv/netvsc_drv.c
+++ b/drivers/net/hyperv/netvsc_drv.c
@@ -33,6 +33,9 @@
 #include <linux/if_vlan.h>
 #include <linux/in.h>
 #include <linux/slab.h>
+#include <linux/rtnetlink.h>
+#include <linux/netpoll.h>
+
 #include <net/arp.h>
 #include <net/route.h>
 #include <net/sock.h>
@@ -42,8 +45,14 @@
 
 #include "hyperv_net.h"
 
-#define RING_SIZE_MIN 64
+#define RING_SIZE_MIN		64
+#define NETVSC_MIN_TX_SECTIONS	10
+#define NETVSC_DEFAULT_TX	192	/* ~1M */
+#define NETVSC_MIN_RX_SECTIONS	10	/* ~64K */
+#define NETVSC_DEFAULT_RX	10485   /* Max ~16M */
+
 #define LINKCHANGE_INT (2 * HZ)
+#define VF_TAKEOVER_INT (HZ / 10)
 
 static int ring_size = 128;
 module_param(ring_size, int, S_IRUGO);
@@ -69,7 +78,8 @@ static void netvsc_set_multicast_list(struct net_device *net)
 static int netvsc_open(struct net_device *net)
 {
 	struct net_device_context *ndev_ctx = netdev_priv(net);
-	struct netvsc_device *nvdev = ndev_ctx->nvdev;
+	struct net_device *vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
+	struct netvsc_device *nvdev = rtnl_dereference(ndev_ctx->nvdev);
 	struct rndis_device *rdev;
 	int ret = 0;
 
@@ -85,22 +95,40 @@ static int netvsc_open(struct net_device *net)
 	netif_tx_wake_all_queues(net);
 
 	rdev = nvdev->extension;
-	if (!rdev->link_state && !ndev_ctx->datapath)
+
+	if (!rdev->link_state)
 		netif_carrier_on(net);
 
-	return ret;
+	if (vf_netdev) {
+		/* Setting synthetic device up transparently sets
+		 * slave as up. If open fails, then slave will be
+		 * still be offline (and not used).
+		 */
+		ret = dev_open(vf_netdev);
+		if (ret)
+			netdev_warn(net,
+				    "unable to open slave: %s: %d\n",
+				    vf_netdev->name, ret);
+	}
+	return 0;
 }
 
 static int netvsc_close(struct net_device *net)
 {
 	struct net_device_context *net_device_ctx = netdev_priv(net);
+	struct net_device *vf_netdev
+		= rtnl_dereference(net_device_ctx->vf_netdev);
 	struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
-	int ret;
+	int ret = 0;
 	u32 aread, i, msec = 10, retry = 0, retry_max = 20;
 	struct vmbus_channel *chn;
 
 	netif_tx_disable(net);
 
+	/* No need to close rndis filter if it is removed already */
+	if (!nvdev)
+		goto out;
+
 	ret = rndis_filter_close(nvdev);
 	if (ret != 0) {
 		netdev_err(net, "unable to close device (ret %d).\n", ret);
@@ -139,11 +167,15 @@ static int netvsc_close(struct net_device *net)
 		ret = -ETIMEDOUT;
 	}
 
+out:
+	if (vf_netdev)
+		dev_close(vf_netdev);
+
 	return ret;
 }
 
 static void *init_ppi_data(struct rndis_message *msg, u32 ppi_size,
-				int pkt_type)
+			   int pkt_type)
 {
 	struct rndis_packet *rndis_pkt;
 	struct rndis_per_packet_info *ppi;
@@ -163,10 +195,12 @@ static void *init_ppi_data(struct rndis_message *msg, u32 ppi_size,
 	return ppi;
 }
 
-/* Azure hosts don't support non-TCP port numbers in hashing yet. We compute
- * hash for non-TCP traffic with only IP numbers.
+/* Azure hosts don't support non-TCP port numbers in hashing for fragmented
+ * packets. We can use ethtool to change UDP hash level when necessary.
  */
-static inline u32 netvsc_get_hash(struct sk_buff *skb, struct sock *sk)
+static inline u32 netvsc_get_hash(
+	struct sk_buff *skb,
+	const struct net_device_context *ndc)
 {
 	struct flow_keys flow;
 	u32 hash;
@@ -177,7 +211,11 @@ static inline u32 netvsc_get_hash(struct sk_buff *skb, struct sock *sk)
 	if (!skb_flow_dissect_flow_keys(skb, &flow, 0))
 		return 0;
 
-	if (flow.basic.ip_proto == IPPROTO_TCP) {
+	if (flow.basic.ip_proto == IPPROTO_TCP ||
+	    (flow.basic.ip_proto == IPPROTO_UDP &&
+	     ((flow.basic.n_proto == htons(ETH_P_IP) && ndc->udp4_l4_hash) ||
+	      (flow.basic.n_proto == htons(ETH_P_IPV6) &&
+	       ndc->udp6_l4_hash)))) {
 		return skb_get_hash(skb);
 	} else {
 		if (flow.basic.n_proto == htons(ETH_P_IP))
@@ -200,7 +238,7 @@ static inline int netvsc_get_tx_queue(struct net_device *ndev,
 	struct sock *sk = skb->sk;
 	int q_idx;
 
-	q_idx = ndc->tx_send_table[netvsc_get_hash(skb, sk) &
+	q_idx = ndc->tx_send_table[netvsc_get_hash(skb, ndc) &
 				   (VRSS_SEND_TAB_SIZE - 1)];
 
 	/* If queue index changed record the new value */
@@ -222,13 +260,11 @@ static inline int netvsc_get_tx_queue(struct net_device *ndev,
  *
  * TODO support XPS - but get_xps_queue not exported
  */
-static u16 netvsc_select_queue(struct net_device *ndev, struct sk_buff *skb,
-			void *accel_priv, select_queue_fallback_t fallback)
+static u16 netvsc_pick_tx(struct net_device *ndev, struct sk_buff *skb)
 {
-	unsigned int num_tx_queues = ndev->real_num_tx_queues;
 	int q_idx = sk_tx_queue_get(skb->sk);
 
-	if (q_idx < 0 || skb->ooo_okay) {
+	if (q_idx < 0 || skb->ooo_okay || q_idx >= ndev->real_num_tx_queues) {
 		/* If forwarding a packet, we use the recorded queue when
 		 * available for better cache locality.
 		 */
@@ -238,14 +274,35 @@ static u16 netvsc_select_queue(struct net_device *ndev, struct sk_buff *skb,
 			q_idx = netvsc_get_tx_queue(ndev, skb, q_idx);
 	}
 
-	while (unlikely(q_idx >= num_tx_queues))
-		q_idx -= num_tx_queues;
-
 	return q_idx;
 }
 
+static u16 netvsc_select_queue(struct net_device *ndev, struct sk_buff *skb,
+			       void *accel_priv,
+			       select_queue_fallback_t fallback)
+{
+	struct net_device_context *ndc = netdev_priv(ndev);
+	struct net_device *vf_netdev;
+	u16 txq;
+
+	rcu_read_lock();
+	vf_netdev = rcu_dereference(ndc->vf_netdev);
+	if (vf_netdev) {
+		txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : 0;
+		qdisc_skb_cb(skb)->slave_dev_queue_mapping = skb->queue_mapping;
+	} else {
+		txq = netvsc_pick_tx(ndev, skb);
+	}
+	rcu_read_unlock();
+
+	while (unlikely(txq >= ndev->real_num_tx_queues))
+		txq -= ndev->real_num_tx_queues;
+
+	return txq;
+}
+
 static u32 fill_pg_buf(struct page *page, u32 offset, u32 len,
-			struct hv_page_buffer *pb)
+		       struct hv_page_buffer *pb)
 {
 	int j = 0;
 
@@ -280,9 +337,8 @@ static u32 fill_pg_buf(struct page *page, u32 offset, u32 len,
 
 static u32 init_page_array(void *hdr, u32 len, struct sk_buff *skb,
 			   struct hv_netvsc_packet *packet,
-			   struct hv_page_buffer **page_buf)
+			   struct hv_page_buffer *pb)
 {
-	struct hv_page_buffer *pb = *page_buf;
 	u32 slots_used = 0;
 	char *data = skb->data;
 	int frags = skb_shinfo(skb)->nr_frags;
@@ -293,10 +349,9 @@ static u32 init_page_array(void *hdr, u32 len, struct sk_buff *skb,
 	 * 2. skb linear data
 	 * 3. skb fragment data
 	 */
-	if (hdr != NULL)
-		slots_used += fill_pg_buf(virt_to_page(hdr),
-					offset_in_page(hdr),
-					len, &pb[slots_used]);
+	slots_used += fill_pg_buf(virt_to_page(hdr),
+				  offset_in_page(hdr),
+				  len, &pb[slots_used]);
 
 	packet->rmsg_size = len;
 	packet->rmsg_pgcnt = slots_used;
@@ -359,13 +414,40 @@ static u32 net_checksum_info(struct sk_buff *skb)
 
 		if (ip6->nexthdr == IPPROTO_TCP)
 			return TRANSPORT_INFO_IPV6_TCP;
-		else if (ipv6_hdr(skb)->nexthdr == IPPROTO_UDP)
+		else if (ip6->nexthdr == IPPROTO_UDP)
 			return TRANSPORT_INFO_IPV6_UDP;
 	}
 
 	return TRANSPORT_INFO_NOT_IP;
 }
 
+/* Send skb on the slave VF device. */
+static int netvsc_vf_xmit(struct net_device *net, struct net_device *vf_netdev,
+			  struct sk_buff *skb)
+{
+	struct net_device_context *ndev_ctx = netdev_priv(net);
+	unsigned int len = skb->len;
+	int rc;
+
+	skb->dev = vf_netdev;
+	skb->queue_mapping = qdisc_skb_cb(skb)->slave_dev_queue_mapping;
+
+	rc = dev_queue_xmit(skb);
+	if (likely(rc == NET_XMIT_SUCCESS || rc == NET_XMIT_CN)) {
+		struct netvsc_vf_pcpu_stats *pcpu_stats
+			= this_cpu_ptr(ndev_ctx->vf_stats);
+
+		u64_stats_update_begin(&pcpu_stats->syncp);
+		pcpu_stats->tx_packets++;
+		pcpu_stats->tx_bytes += len;
+		u64_stats_update_end(&pcpu_stats->syncp);
+	} else {
+		this_cpu_inc(ndev_ctx->vf_stats->tx_dropped);
+	}
+
+	return rc;
+}
+
 static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net)
 {
 	struct net_device_context *net_device_ctx = netdev_priv(net);
@@ -374,11 +456,19 @@ static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net)
 	unsigned int num_data_pgs;
 	struct rndis_message *rndis_msg;
 	struct rndis_packet *rndis_pkt;
+	struct net_device *vf_netdev;
 	u32 rndis_msg_size;
 	struct rndis_per_packet_info *ppi;
 	u32 hash;
-	struct hv_page_buffer page_buf[MAX_PAGE_BUFFER_COUNT];
-	struct hv_page_buffer *pb = page_buf;
+	struct hv_page_buffer pb[MAX_PAGE_BUFFER_COUNT];
+
+	/* if VF is present and up then redirect packets
+	 * already called with rcu_read_lock_bh
+	 */
+	vf_netdev = rcu_dereference_bh(net_device_ctx->vf_netdev);
+	if (vf_netdev && netif_running(vf_netdev) &&
+	    !netpoll_tx_running(net))
+		return netvsc_vf_xmit(net, vf_netdev, skb);
 
 	/* We will atmost need two pages to describe the rndis
 	 * header. We can only transmit MAX_PAGE_BUFFER_COUNT number
@@ -448,9 +538,9 @@ static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net)
 
 		rndis_msg_size += NDIS_VLAN_PPI_SIZE;
 		ppi = init_ppi_data(rndis_msg, NDIS_VLAN_PPI_SIZE,
-					IEEE_8021Q_INFO);
-		vlan = (struct ndis_pkt_8021q_info *)((void *)ppi +
-						ppi->ppi_offset);
+				    IEEE_8021Q_INFO);
+
+		vlan = (void *)ppi + ppi->ppi_offset;
 		vlan->vlanid = skb->vlan_tci & VLAN_VID_MASK;
 		vlan->pri = (skb->vlan_tci & VLAN_PRIO_MASK) >>
 				VLAN_PRIO_SHIFT;
@@ -463,8 +553,7 @@ static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net)
 		ppi = init_ppi_data(rndis_msg, NDIS_LSO_PPI_SIZE,
 				    TCP_LARGESEND_PKTINFO);
 
-		lso_info = (struct ndis_tcp_lso_info *)((void *)ppi +
-							ppi->ppi_offset);
+		lso_info = (void *)ppi + ppi->ppi_offset;
 
 		lso_info->lso_v2_transmit.type = NDIS_TCP_LARGE_SEND_OFFLOAD_V2_TYPE;
 		if (skb->protocol == htons(ETH_P_IP)) {
@@ -524,12 +613,12 @@ static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net)
 	rndis_msg->msg_len += rndis_msg_size;
 	packet->total_data_buflen = rndis_msg->msg_len;
 	packet->page_buf_cnt = init_page_array(rndis_msg, rndis_msg_size,
-					       skb, packet, &pb);
+					       skb, packet, pb);
 
 	/* timestamp packet in software */
 	skb_tx_timestamp(skb);
-	ret = netvsc_send(net_device_ctx->device_ctx, packet,
-			  rndis_msg, &pb, skb);
+
+	ret = netvsc_send(net_device_ctx, packet, rndis_msg, pb, skb);
 	if (likely(ret == 0))
 		return NETDEV_TX_OK;
 
@@ -551,6 +640,7 @@ no_memory:
 	++net_device_ctx->eth_stats.tx_no_memory;
 	goto drop;
 }
+
 /*
  * netvsc_linkstatus_callback - Link up/down notification
  */
@@ -574,8 +664,8 @@ void netvsc_linkstatus_callback(struct hv_device *device_obj,
 	if (indicate->status == RNDIS_STATUS_LINK_SPEED_CHANGE) {
 		u32 speed;
 
-		speed = *(u32 *)((void *)indicate + indicate->
-				 status_buf_offset) / 10000;
+		speed = *(u32 *)((void *)indicate
+				 + indicate->status_buf_offset) / 10000;
 		ndev_ctx->speed = speed;
 		return;
 	}
@@ -658,29 +748,18 @@ int netvsc_recv_callback(struct net_device *net,
 	struct netvsc_device *net_device;
 	u16 q_idx = channel->offermsg.offer.sub_channel_index;
 	struct netvsc_channel *nvchan;
-	struct net_device *vf_netdev;
 	struct sk_buff *skb;
 	struct netvsc_stats *rx_stats;
 
 	if (net->reg_state != NETREG_REGISTERED)
 		return NVSP_STAT_FAIL;
 
-	/*
-	 * If necessary, inject this packet into the VF interface.
-	 * On Hyper-V, multicast and brodcast packets are only delivered
-	 * to the synthetic interface (after subjecting these to
-	 * policy filters on the host). Deliver these via the VF
-	 * interface in the guest.
-	 */
 	rcu_read_lock();
 	net_device = rcu_dereference(net_device_ctx->nvdev);
 	if (unlikely(!net_device))
 		goto drop;
 
 	nvchan = &net_device->chan_table[q_idx];
-	vf_netdev = rcu_dereference(net_device_ctx->vf_netdev);
-	if (vf_netdev && (vf_netdev->flags & IFF_UP))
-		net = vf_netdev;
 
 	/* Allocate a skb - TODO direct I/O to pages? */
 	skb = netvsc_alloc_recv_skb(net, &nvchan->napi,
@@ -692,8 +771,7 @@ drop:
 		return NVSP_STAT_FAIL;
 	}
 
-	if (net != vf_netdev)
-		skb_record_rx_queue(skb, q_idx);
+	skb_record_rx_queue(skb, q_idx);
 
 	/*
 	 * Even if injecting the packet, record the statistics
@@ -736,48 +814,22 @@ static void netvsc_get_channels(struct net_device *net,
 	}
 }
 
-static int netvsc_set_queues(struct net_device *net, struct hv_device *dev,
-			     u32 num_chn)
-{
-	struct netvsc_device_info device_info;
-	int ret;
-
-	memset(&device_info, 0, sizeof(device_info));
-	device_info.num_chn = num_chn;
-	device_info.ring_size = ring_size;
-	device_info.max_num_vrss_chns = num_chn;
-
-	ret = rndis_filter_device_add(dev, &device_info);
-	if (ret)
-		return ret;
-
-	ret = netif_set_real_num_tx_queues(net, num_chn);
-	if (ret)
-		return ret;
-
-	ret = netif_set_real_num_rx_queues(net, num_chn);
-
-	return ret;
-}
-
 static int netvsc_set_channels(struct net_device *net,
 			       struct ethtool_channels *channels)
 {
 	struct net_device_context *net_device_ctx = netdev_priv(net);
 	struct hv_device *dev = net_device_ctx->device_ctx;
 	struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
-	unsigned int count = channels->combined_count;
-	bool was_running;
-	int ret;
+	unsigned int orig, count = channels->combined_count;
+	struct netvsc_device_info device_info;
+	bool was_opened;
+	int ret = 0;
 
 	/* We do not support separate count for rx, tx, or other */
 	if (count == 0 ||
 	    channels->rx_count || channels->tx_count || channels->other_count)
 		return -EINVAL;
 
-	if (count > net->num_tx_queues || count > VRSS_CHANNEL_MAX)
-		return -EINVAL;
-
 	if (!nvdev || nvdev->destroy)
 		return -ENODEV;
 
@@ -787,25 +839,37 @@ static int netvsc_set_channels(struct net_device *net,
 	if (count > nvdev->max_chn)
 		return -EINVAL;
 
-	was_running = netif_running(net);
-	if (was_running) {
-		ret = netvsc_close(net);
-		if (ret)
-			return ret;
-	}
+	orig = nvdev->num_chn;
+	was_opened = rndis_filter_opened(nvdev);
+	if (was_opened)
+		rndis_filter_close(nvdev);
+
+	memset(&device_info, 0, sizeof(device_info));
+	device_info.num_chn = count;
+	device_info.ring_size = ring_size;
+	device_info.send_sections = nvdev->send_section_cnt;
+	device_info.recv_sections = nvdev->recv_section_cnt;
 
 	rndis_filter_device_remove(dev, nvdev);
 
-	ret = netvsc_set_queues(net, dev, count);
-	if (ret == 0)
-		nvdev->num_chn = count;
-	else
-		netvsc_set_queues(net, dev, nvdev->num_chn);
+	nvdev = rndis_filter_device_add(dev, &device_info);
+	if (IS_ERR(nvdev)) {
+		ret = PTR_ERR(nvdev);
+		device_info.num_chn = orig;
+		nvdev = rndis_filter_device_add(dev, &device_info);
+
+		if (IS_ERR(nvdev)) {
+			netdev_err(net, "restoring channel setting failed: %ld\n",
+				   PTR_ERR(nvdev));
+			return ret;
+		}
+	}
 
-	if (was_running)
-		ret = netvsc_open(net);
+	if (was_opened)
+		rndis_filter_open(nvdev);
 
 	/* We may have missed link change notifications */
+	net_device_ctx->last_reconfig = 0;
 	schedule_delayed_work(&net_device_ctx->dwork, 0);
 
 	return ret;
@@ -832,6 +896,9 @@ static void netvsc_init_settings(struct net_device *dev)
 {
 	struct net_device_context *ndc = netdev_priv(dev);
 
+	ndc->udp4_l4_hash = true;
+	ndc->udp6_l4_hash = true;
+
 	ndc->speed = SPEED_UNKNOWN;
 	ndc->duplex = DUPLEX_FULL;
 }
@@ -869,41 +936,61 @@ static int netvsc_set_link_ksettings(struct net_device *dev,
 static int netvsc_change_mtu(struct net_device *ndev, int mtu)
 {
 	struct net_device_context *ndevctx = netdev_priv(ndev);
+	struct net_device *vf_netdev = rtnl_dereference(ndevctx->vf_netdev);
 	struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
 	struct hv_device *hdev = ndevctx->device_ctx;
+	int orig_mtu = ndev->mtu;
 	struct netvsc_device_info device_info;
-	bool was_running;
+	bool was_opened;
 	int ret = 0;
 
 	if (!nvdev || nvdev->destroy)
 		return -ENODEV;
 
-	was_running = netif_running(ndev);
-	if (was_running) {
-		ret = netvsc_close(ndev);
+	/* Change MTU of underlying VF netdev first. */
+	if (vf_netdev) {
+		ret = dev_set_mtu(vf_netdev, mtu);
 		if (ret)
 			return ret;
 	}
 
+	netif_device_detach(ndev);
+	was_opened = rndis_filter_opened(nvdev);
+	if (was_opened)
+		rndis_filter_close(nvdev);
+
 	memset(&device_info, 0, sizeof(device_info));
 	device_info.ring_size = ring_size;
 	device_info.num_chn = nvdev->num_chn;
-	device_info.max_num_vrss_chns = nvdev->num_chn;
+	device_info.send_sections = nvdev->send_section_cnt;
+	device_info.recv_sections = nvdev->recv_section_cnt;
 
 	rndis_filter_device_remove(hdev, nvdev);
 
-	/* 'nvdev' has been freed in rndis_filter_device_remove() ->
-	 * netvsc_device_remove () -> free_netvsc_device().
-	 * We mustn't access it before it's re-created in
-	 * rndis_filter_device_add() -> netvsc_device_add().
-	 */
-
 	ndev->mtu = mtu;
 
-	rndis_filter_device_add(hdev, &device_info);
+	nvdev = rndis_filter_device_add(hdev, &device_info);
+	if (IS_ERR(nvdev)) {
+		ret = PTR_ERR(nvdev);
+
+		/* Attempt rollback to original MTU */
+		ndev->mtu = orig_mtu;
+		nvdev = rndis_filter_device_add(hdev, &device_info);
 
-	if (was_running)
-		ret = netvsc_open(ndev);
+		if (vf_netdev)
+			dev_set_mtu(vf_netdev, orig_mtu);
+
+		if (IS_ERR(nvdev)) {
+			netdev_err(ndev, "restoring mtu failed: %ld\n",
+				   PTR_ERR(nvdev));
+			return ret;
+		}
+	}
+
+	if (was_opened)
+		rndis_filter_open(nvdev);
+
+	netif_device_attach(ndev);
 
 	/* We may have missed link change notifications */
 	schedule_delayed_work(&ndevctx->dwork, 0);
@@ -911,16 +998,56 @@ static int netvsc_change_mtu(struct net_device *ndev, int mtu)
 	return ret;
 }
 
+static void netvsc_get_vf_stats(struct net_device *net,
+				struct netvsc_vf_pcpu_stats *tot)
+{
+	struct net_device_context *ndev_ctx = netdev_priv(net);
+	int i;
+
+	memset(tot, 0, sizeof(*tot));
+
+	for_each_possible_cpu(i) {
+		const struct netvsc_vf_pcpu_stats *stats
+			= per_cpu_ptr(ndev_ctx->vf_stats, i);
+		u64 rx_packets, rx_bytes, tx_packets, tx_bytes;
+		unsigned int start;
+
+		do {
+			start = u64_stats_fetch_begin_irq(&stats->syncp);
+			rx_packets = stats->rx_packets;
+			tx_packets = stats->tx_packets;
+			rx_bytes = stats->rx_bytes;
+			tx_bytes = stats->tx_bytes;
+		} while (u64_stats_fetch_retry_irq(&stats->syncp, start));
+
+		tot->rx_packets += rx_packets;
+		tot->tx_packets += tx_packets;
+		tot->rx_bytes   += rx_bytes;
+		tot->tx_bytes   += tx_bytes;
+		tot->tx_dropped += stats->tx_dropped;
+	}
+}
+
 static void netvsc_get_stats64(struct net_device *net,
 			       struct rtnl_link_stats64 *t)
 {
 	struct net_device_context *ndev_ctx = netdev_priv(net);
 	struct netvsc_device *nvdev = rcu_dereference_rtnl(ndev_ctx->nvdev);
+	struct netvsc_vf_pcpu_stats vf_tot;
 	int i;
 
 	if (!nvdev)
 		return;
 
+	netdev_stats_to_stats64(t, &net->stats);
+
+	netvsc_get_vf_stats(net, &vf_tot);
+	t->rx_packets += vf_tot.rx_packets;
+	t->tx_packets += vf_tot.tx_packets;
+	t->rx_bytes   += vf_tot.rx_bytes;
+	t->tx_bytes   += vf_tot.tx_bytes;
+	t->tx_dropped += vf_tot.tx_dropped;
+
 	for (i = 0; i < nvdev->num_chn; i++) {
 		const struct netvsc_channel *nvchan = &nvdev->chan_table[i];
 		const struct netvsc_stats *stats;
@@ -949,33 +1076,36 @@ static void netvsc_get_stats64(struct net_device *net,
 		t->rx_packets	+= packets;
 		t->multicast	+= multicast;
 	}
-
-	t->tx_dropped	= net->stats.tx_dropped;
-	t->tx_errors	= net->stats.tx_errors;
-
-	t->rx_dropped	= net->stats.rx_dropped;
-	t->rx_errors	= net->stats.rx_errors;
 }
 
 static int netvsc_set_mac_addr(struct net_device *ndev, void *p)
 {
+	struct net_device_context *ndc = netdev_priv(ndev);
+	struct net_device *vf_netdev = rtnl_dereference(ndc->vf_netdev);
+	struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
 	struct sockaddr *addr = p;
-	char save_adr[ETH_ALEN];
-	unsigned char save_aatype;
 	int err;
 
-	memcpy(save_adr, ndev->dev_addr, ETH_ALEN);
-	save_aatype = ndev->addr_assign_type;
-
-	err = eth_mac_addr(ndev, p);
-	if (err != 0)
+	err = eth_prepare_mac_addr_change(ndev, p);
+	if (err)
 		return err;
 
-	err = rndis_filter_set_device_mac(ndev, addr->sa_data);
-	if (err != 0) {
-		/* roll back to saved MAC */
-		memcpy(ndev->dev_addr, save_adr, ETH_ALEN);
-		ndev->addr_assign_type = save_aatype;
+	if (!nvdev)
+		return -ENODEV;
+
+	if (vf_netdev) {
+		err = dev_set_mac_address(vf_netdev, addr);
+		if (err)
+			return err;
+	}
+
+	err = rndis_filter_set_device_mac(nvdev, addr->sa_data);
+	if (!err) {
+		eth_commit_mac_addr_change(ndev, p);
+	} else if (vf_netdev) {
+		/* rollback change on VF */
+		memcpy(addr->sa_data, ndev->dev_addr, ETH_ALEN);
+		dev_set_mac_address(vf_netdev, addr);
 	}
 
 	return err;
@@ -990,9 +1120,18 @@ static const struct {
 	{ "tx_no_space",  offsetof(struct netvsc_ethtool_stats, tx_no_space) },
 	{ "tx_too_big",	  offsetof(struct netvsc_ethtool_stats, tx_too_big) },
 	{ "tx_busy",	  offsetof(struct netvsc_ethtool_stats, tx_busy) },
+	{ "tx_send_full", offsetof(struct netvsc_ethtool_stats, tx_send_full) },
+	{ "rx_comp_busy", offsetof(struct netvsc_ethtool_stats, rx_comp_busy) },
+}, vf_stats[] = {
+	{ "vf_rx_packets", offsetof(struct netvsc_vf_pcpu_stats, rx_packets) },
+	{ "vf_rx_bytes",   offsetof(struct netvsc_vf_pcpu_stats, rx_bytes) },
+	{ "vf_tx_packets", offsetof(struct netvsc_vf_pcpu_stats, tx_packets) },
+	{ "vf_tx_bytes",   offsetof(struct netvsc_vf_pcpu_stats, tx_bytes) },
+	{ "vf_tx_dropped", offsetof(struct netvsc_vf_pcpu_stats, tx_dropped) },
 };
 
 #define NETVSC_GLOBAL_STATS_LEN	ARRAY_SIZE(netvsc_stats)
+#define NETVSC_VF_STATS_LEN	ARRAY_SIZE(vf_stats)
 
 /* 4 statistics per queue (rx/tx packets/bytes) */
 #define NETVSC_QUEUE_STATS_LEN(dev) ((dev)->num_chn * 4)
@@ -1007,7 +1146,9 @@ static int netvsc_get_sset_count(struct net_device *dev, int string_set)
 
 	switch (string_set) {
 	case ETH_SS_STATS:
-		return NETVSC_GLOBAL_STATS_LEN + NETVSC_QUEUE_STATS_LEN(nvdev);
+		return NETVSC_GLOBAL_STATS_LEN
+			+ NETVSC_VF_STATS_LEN
+			+ NETVSC_QUEUE_STATS_LEN(nvdev);
 	default:
 		return -EINVAL;
 	}
@@ -1017,9 +1158,10 @@ static void netvsc_get_ethtool_stats(struct net_device *dev,
 				     struct ethtool_stats *stats, u64 *data)
 {
 	struct net_device_context *ndc = netdev_priv(dev);
-	struct netvsc_device *nvdev = rcu_dereference(ndc->nvdev);
+	struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
 	const void *nds = &ndc->eth_stats;
 	const struct netvsc_stats *qstats;
+	struct netvsc_vf_pcpu_stats sum;
 	unsigned int start;
 	u64 packets, bytes;
 	int i, j;
@@ -1030,6 +1172,10 @@ static void netvsc_get_ethtool_stats(struct net_device *dev,
 	for (i = 0; i < NETVSC_GLOBAL_STATS_LEN; i++)
 		data[i] = *(unsigned long *)(nds + netvsc_stats[i].offset);
 
+	netvsc_get_vf_stats(dev, &sum);
+	for (j = 0; j < NETVSC_VF_STATS_LEN; j++)
+		data[i++] = *(u64 *)((void *)&sum + vf_stats[j].offset);
+
 	for (j = 0; j < nvdev->num_chn; j++) {
 		qstats = &nvdev->chan_table[j].tx_stats;
 
@@ -1055,7 +1201,7 @@ static void netvsc_get_ethtool_stats(struct net_device *dev,
 static void netvsc_get_strings(struct net_device *dev, u32 stringset, u8 *data)
 {
 	struct net_device_context *ndc = netdev_priv(dev);
-	struct netvsc_device *nvdev = rcu_dereference(ndc->nvdev);
+	struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
 	u8 *p = data;
 	int i;
 
@@ -1064,11 +1210,16 @@ static void netvsc_get_strings(struct net_device *dev, u32 stringset, u8 *data)
 
 	switch (stringset) {
 	case ETH_SS_STATS:
-		for (i = 0; i < ARRAY_SIZE(netvsc_stats); i++)
-			memcpy(p + i * ETH_GSTRING_LEN,
-			       netvsc_stats[i].name, ETH_GSTRING_LEN);
+		for (i = 0; i < ARRAY_SIZE(netvsc_stats); i++) {
+			memcpy(p, netvsc_stats[i].name, ETH_GSTRING_LEN);
+			p += ETH_GSTRING_LEN;
+		}
+
+		for (i = 0; i < ARRAY_SIZE(vf_stats); i++) {
+			memcpy(p, vf_stats[i].name, ETH_GSTRING_LEN);
+			p += ETH_GSTRING_LEN;
+		}
 
-		p += i * ETH_GSTRING_LEN;
 		for (i = 0; i < nvdev->num_chn; i++) {
 			sprintf(p, "tx_queue_%u_packets", i);
 			p += ETH_GSTRING_LEN;
@@ -1085,7 +1236,7 @@ static void netvsc_get_strings(struct net_device *dev, u32 stringset, u8 *data)
 }
 
 static int
-netvsc_get_rss_hash_opts(struct netvsc_device *nvdev,
+netvsc_get_rss_hash_opts(struct net_device_context *ndc,
 			 struct ethtool_rxnfc *info)
 {
 	info->data = RXH_IP_SRC | RXH_IP_DST;
@@ -1094,9 +1245,20 @@ netvsc_get_rss_hash_opts(struct netvsc_device *nvdev,
 	case TCP_V4_FLOW:
 	case TCP_V6_FLOW:
 		info->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3;
-		/* fallthrough */
+		break;
+
 	case UDP_V4_FLOW:
+		if (ndc->udp4_l4_hash)
+			info->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3;
+
+		break;
+
 	case UDP_V6_FLOW:
+		if (ndc->udp6_l4_hash)
+			info->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3;
+
+		break;
+
 	case IPV4_FLOW:
 	case IPV6_FLOW:
 		break;
@@ -1113,7 +1275,7 @@ netvsc_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
 		 u32 *rules)
 {
 	struct net_device_context *ndc = netdev_priv(dev);
-	struct netvsc_device *nvdev = rcu_dereference(ndc->nvdev);
+	struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
 
 	if (!nvdev)
 		return -ENODEV;
@@ -1124,8 +1286,48 @@ netvsc_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
 		return 0;
 
 	case ETHTOOL_GRXFH:
-		return netvsc_get_rss_hash_opts(nvdev, info);
+		return netvsc_get_rss_hash_opts(ndc, info);
+	}
+	return -EOPNOTSUPP;
+}
+
+static int netvsc_set_rss_hash_opts(struct net_device_context *ndc,
+				    struct ethtool_rxnfc *info)
+{
+	if (info->data == (RXH_IP_SRC | RXH_IP_DST |
+			   RXH_L4_B_0_1 | RXH_L4_B_2_3)) {
+		if (info->flow_type == UDP_V4_FLOW)
+			ndc->udp4_l4_hash = true;
+		else if (info->flow_type == UDP_V6_FLOW)
+			ndc->udp6_l4_hash = true;
+		else
+			return -EOPNOTSUPP;
+
+		return 0;
 	}
+
+	if (info->data == (RXH_IP_SRC | RXH_IP_DST)) {
+		if (info->flow_type == UDP_V4_FLOW)
+			ndc->udp4_l4_hash = false;
+		else if (info->flow_type == UDP_V6_FLOW)
+			ndc->udp6_l4_hash = false;
+		else
+			return -EOPNOTSUPP;
+
+		return 0;
+	}
+
+	return -EOPNOTSUPP;
+}
+
+static int
+netvsc_set_rxnfc(struct net_device *ndev, struct ethtool_rxnfc *info)
+{
+	struct net_device_context *ndc = netdev_priv(ndev);
+
+	if (info->cmd == ETHTOOL_SRXFH)
+		return netvsc_set_rss_hash_opts(ndc, info);
+
 	return -EOPNOTSUPP;
 }
 
@@ -1163,7 +1365,7 @@ static int netvsc_get_rxfh(struct net_device *dev, u32 *indir, u8 *key,
 			   u8 *hfunc)
 {
 	struct net_device_context *ndc = netdev_priv(dev);
-	struct netvsc_device *ndev = rcu_dereference(ndc->nvdev);
+	struct netvsc_device *ndev = rtnl_dereference(ndc->nvdev);
 	struct rndis_device *rndis_dev;
 	int i;
 
@@ -1202,7 +1404,7 @@ static int netvsc_set_rxfh(struct net_device *dev, const u32 *indir,
 	rndis_dev = ndev->extension;
 	if (indir) {
 		for (i = 0; i < ITAB_NUM; i++)
-			if (indir[i] >= VRSS_CHANNEL_MAX)
+			if (indir[i] >= ndev->num_chn)
 				return -EINVAL;
 
 		for (i = 0; i < ITAB_NUM; i++)
@@ -1216,7 +1418,105 @@ static int netvsc_set_rxfh(struct net_device *dev, const u32 *indir,
 		key = rndis_dev->rss_key;
 	}
 
-	return rndis_filter_set_rss_param(rndis_dev, key, ndev->num_chn);
+	return rndis_filter_set_rss_param(rndis_dev, key);
+}
+
+/* Hyper-V RNDIS protocol does not have ring in the HW sense.
+ * It does have pre-allocated receive area which is divided into sections.
+ */
+static void __netvsc_get_ringparam(struct netvsc_device *nvdev,
+				   struct ethtool_ringparam *ring)
+{
+	u32 max_buf_size;
+
+	ring->rx_pending = nvdev->recv_section_cnt;
+	ring->tx_pending = nvdev->send_section_cnt;
+
+	if (nvdev->nvsp_version <= NVSP_PROTOCOL_VERSION_2)
+		max_buf_size = NETVSC_RECEIVE_BUFFER_SIZE_LEGACY;
+	else
+		max_buf_size = NETVSC_RECEIVE_BUFFER_SIZE;
+
+	ring->rx_max_pending = max_buf_size / nvdev->recv_section_size;
+	ring->tx_max_pending = NETVSC_SEND_BUFFER_SIZE
+		/ nvdev->send_section_size;
+}
+
+static void netvsc_get_ringparam(struct net_device *ndev,
+				 struct ethtool_ringparam *ring)
+{
+	struct net_device_context *ndevctx = netdev_priv(ndev);
+	struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
+
+	if (!nvdev)
+		return;
+
+	__netvsc_get_ringparam(nvdev, ring);
+}
+
+static int netvsc_set_ringparam(struct net_device *ndev,
+				struct ethtool_ringparam *ring)
+{
+	struct net_device_context *ndevctx = netdev_priv(ndev);
+	struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
+	struct hv_device *hdev = ndevctx->device_ctx;
+	struct netvsc_device_info device_info;
+	struct ethtool_ringparam orig;
+	u32 new_tx, new_rx;
+	bool was_opened;
+	int ret = 0;
+
+	if (!nvdev || nvdev->destroy)
+		return -ENODEV;
+
+	memset(&orig, 0, sizeof(orig));
+	__netvsc_get_ringparam(nvdev, &orig);
+
+	new_tx = clamp_t(u32, ring->tx_pending,
+			 NETVSC_MIN_TX_SECTIONS, orig.tx_max_pending);
+	new_rx = clamp_t(u32, ring->rx_pending,
+			 NETVSC_MIN_RX_SECTIONS, orig.rx_max_pending);
+
+	if (new_tx == orig.tx_pending &&
+	    new_rx == orig.rx_pending)
+		return 0;	 /* no change */
+
+	memset(&device_info, 0, sizeof(device_info));
+	device_info.num_chn = nvdev->num_chn;
+	device_info.ring_size = ring_size;
+	device_info.send_sections = new_tx;
+	device_info.recv_sections = new_rx;
+
+	netif_device_detach(ndev);
+	was_opened = rndis_filter_opened(nvdev);
+	if (was_opened)
+		rndis_filter_close(nvdev);
+
+	rndis_filter_device_remove(hdev, nvdev);
+
+	nvdev = rndis_filter_device_add(hdev, &device_info);
+	if (IS_ERR(nvdev)) {
+		ret = PTR_ERR(nvdev);
+
+		device_info.send_sections = orig.tx_pending;
+		device_info.recv_sections = orig.rx_pending;
+		nvdev = rndis_filter_device_add(hdev, &device_info);
+		if (IS_ERR(nvdev)) {
+			netdev_err(ndev, "restoring ringparam failed: %ld\n",
+				   PTR_ERR(nvdev));
+			return ret;
+		}
+	}
+
+	if (was_opened)
+		rndis_filter_open(nvdev);
+	netif_device_attach(ndev);
+
+	/* We may have missed link change notifications */
+	ndevctx->last_reconfig = 0;
+	schedule_delayed_work(&ndevctx->dwork, 0);
+
+	return ret;
 }
 
 static const struct ethtool_ops ethtool_ops = {
@@ -1229,12 +1529,15 @@ static const struct ethtool_ops ethtool_ops = {
 	.set_channels   = netvsc_set_channels,
 	.get_ts_info	= ethtool_op_get_ts_info,
 	.get_rxnfc	= netvsc_get_rxnfc,
+	.set_rxnfc	= netvsc_set_rxnfc,
 	.get_rxfh_key_size = netvsc_get_rxfh_key_size,
 	.get_rxfh_indir_size = netvsc_rss_indir_size,
 	.get_rxfh	= netvsc_get_rxfh,
 	.set_rxfh	= netvsc_set_rxfh,
 	.get_link_ksettings = netvsc_get_link_ksettings,
 	.set_link_ksettings = netvsc_set_link_ksettings,
+	.get_ringparam	= netvsc_get_ringparam,
+	.set_ringparam	= netvsc_set_ringparam,
 };
 
 static const struct net_device_ops device_ops = {
@@ -1269,7 +1572,12 @@ static void netvsc_link_change(struct work_struct *w)
 	bool notify = false, reschedule = false;
 	unsigned long flags, next_reconfig, delay;
 
-	rtnl_lock();
+	/* if changes are happening, comeback later */
+	if (!rtnl_trylock()) {
+		schedule_delayed_work(&ndev_ctx->dwork, LINKCHANGE_INT);
+		return;
+	}
+
 	net_device = rtnl_dereference(ndev_ctx->nvdev);
 	if (!net_device)
 		goto out_unlock;
@@ -1308,8 +1616,7 @@ static void netvsc_link_change(struct work_struct *w)
 	case RNDIS_STATUS_MEDIA_CONNECT:
 		if (rdev->link_state) {
 			rdev->link_state = false;
-			if (!ndev_ctx->datapath)
-				netif_carrier_on(net);
+			netif_carrier_on(net);
 			netif_tx_wake_all_queues(net);
 		} else {
 			notify = true;
@@ -1386,7 +1693,7 @@ static struct net_device *get_netvsc_byref(struct net_device *vf_netdev)
 			continue;	/* not a netvsc device */
 
 		net_device_ctx = netdev_priv(dev);
-		if (net_device_ctx->nvdev == NULL)
+		if (!rtnl_dereference(net_device_ctx->nvdev))
 			continue;	/* device is removed */
 
 		if (rtnl_dereference(net_device_ctx->vf_netdev) == vf_netdev)
@@ -1396,6 +1703,108 @@ static struct net_device *get_netvsc_byref(struct net_device *vf_netdev)
 	return NULL;
 }
 
+/* Called when VF is injecting data into network stack.
+ * Change the associated network device from VF to netvsc.
+ * note: already called with rcu_read_lock
+ */
+static rx_handler_result_t netvsc_vf_handle_frame(struct sk_buff **pskb)
+{
+	struct sk_buff *skb = *pskb;
+	struct net_device *ndev = rcu_dereference(skb->dev->rx_handler_data);
+	struct net_device_context *ndev_ctx = netdev_priv(ndev);
+	struct netvsc_vf_pcpu_stats *pcpu_stats
+		 = this_cpu_ptr(ndev_ctx->vf_stats);
+
+	skb->dev = ndev;
+
+	u64_stats_update_begin(&pcpu_stats->syncp);
+	pcpu_stats->rx_packets++;
+	pcpu_stats->rx_bytes += skb->len;
+	u64_stats_update_end(&pcpu_stats->syncp);
+
+	return RX_HANDLER_ANOTHER;
+}
+
+static int netvsc_vf_join(struct net_device *vf_netdev,
+			  struct net_device *ndev)
+{
+	struct net_device_context *ndev_ctx = netdev_priv(ndev);
+	int ret;
+
+	ret = netdev_rx_handler_register(vf_netdev,
+					 netvsc_vf_handle_frame, ndev);
+	if (ret != 0) {
+		netdev_err(vf_netdev,
+			   "can not register netvsc VF receive handler (err = %d)\n",
+			   ret);
+		goto rx_handler_failed;
+	}
+
+	ret = netdev_upper_dev_link(vf_netdev, ndev);
+	if (ret != 0) {
+		netdev_err(vf_netdev,
+			   "can not set master device %s (err = %d)\n",
+			   ndev->name, ret);
+		goto upper_link_failed;
+	}
+
+	/* set slave flag before open to prevent IPv6 addrconf */
+	vf_netdev->flags |= IFF_SLAVE;
+
+	schedule_delayed_work(&ndev_ctx->vf_takeover, VF_TAKEOVER_INT);
+
+	call_netdevice_notifiers(NETDEV_JOIN, vf_netdev);
+
+	netdev_info(vf_netdev, "joined to %s\n", ndev->name);
+	return 0;
+
+upper_link_failed:
+	netdev_rx_handler_unregister(vf_netdev);
+rx_handler_failed:
+	return ret;
+}
+
+static void __netvsc_vf_setup(struct net_device *ndev,
+			      struct net_device *vf_netdev)
+{
+	int ret;
+
+	/* Align MTU of VF with master */
+	ret = dev_set_mtu(vf_netdev, ndev->mtu);
+	if (ret)
+		netdev_warn(vf_netdev,
+			    "unable to change mtu to %u\n", ndev->mtu);
+
+	if (netif_running(ndev)) {
+		ret = dev_open(vf_netdev);
+		if (ret)
+			netdev_warn(vf_netdev,
+				    "unable to open: %d\n", ret);
+	}
+}
+
+/* Setup VF as slave of the synthetic device.
+ * Runs in workqueue to avoid recursion in netlink callbacks.
+ */
+static void netvsc_vf_setup(struct work_struct *w)
+{
+	struct net_device_context *ndev_ctx
+		= container_of(w, struct net_device_context, vf_takeover.work);
+	struct net_device *ndev = hv_get_drvdata(ndev_ctx->device_ctx);
+	struct net_device *vf_netdev;
+
+	if (!rtnl_trylock()) {
+		schedule_delayed_work(&ndev_ctx->vf_takeover, 0);
+		return;
+	}
+
+	vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
+	if (vf_netdev)
+		__netvsc_vf_setup(ndev, vf_netdev);
+
+	rtnl_unlock();
+}
+
 static int netvsc_register_vf(struct net_device *vf_netdev)
 {
 	struct net_device *ndev;
@@ -1419,56 +1828,23 @@ static int netvsc_register_vf(struct net_device *vf_netdev)
 	if (!netvsc_dev || rtnl_dereference(net_device_ctx->vf_netdev))
 		return NOTIFY_DONE;
 
+	if (netvsc_vf_join(vf_netdev, ndev) != 0)
+		return NOTIFY_DONE;
+
 	netdev_info(ndev, "VF registering: %s\n", vf_netdev->name);
-	/*
-	 * Take a reference on the module.
-	 */
-	try_module_get(THIS_MODULE);
 
 	dev_hold(vf_netdev);
 	rcu_assign_pointer(net_device_ctx->vf_netdev, vf_netdev);
 	return NOTIFY_OK;
 }
 
-static int netvsc_vf_up(struct net_device *vf_netdev)
+/* VF up/down change detected, schedule to change data path */
+static int netvsc_vf_changed(struct net_device *vf_netdev)
 {
-	struct net_device *ndev;
-	struct netvsc_device *netvsc_dev;
 	struct net_device_context *net_device_ctx;
-
-	ndev = get_netvsc_byref(vf_netdev);
-	if (!ndev)
-		return NOTIFY_DONE;
-
-	net_device_ctx = netdev_priv(ndev);
-	netvsc_dev = rtnl_dereference(net_device_ctx->nvdev);
-
-	netdev_info(ndev, "VF up: %s\n", vf_netdev->name);
-
-	/*
-	 * Open the device before switching data path.
-	 */
-	rndis_filter_open(netvsc_dev);
-
-	/*
-	 * notify the host to switch the data path.
-	 */
-	netvsc_switch_datapath(ndev, true);
-	netdev_info(ndev, "Data path switched to VF: %s\n", vf_netdev->name);
-
-	netif_carrier_off(ndev);
-
-	/* Now notify peers through VF device. */
-	call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, vf_netdev);
-
-	return NOTIFY_OK;
-}
-
-static int netvsc_vf_down(struct net_device *vf_netdev)
-{
-	struct net_device *ndev;
 	struct netvsc_device *netvsc_dev;
-	struct net_device_context *net_device_ctx;
+	struct net_device *ndev;
+	bool vf_is_up = netif_running(vf_netdev);
 
 	ndev = get_netvsc_byref(vf_netdev);
 	if (!ndev)
@@ -1476,15 +1852,12 @@ static int netvsc_vf_down(struct net_device *vf_netdev)
 
 	net_device_ctx = netdev_priv(ndev);
 	netvsc_dev = rtnl_dereference(net_device_ctx->nvdev);
+	if (!netvsc_dev)
+		return NOTIFY_DONE;
 
-	netdev_info(ndev, "VF down: %s\n", vf_netdev->name);
-	netvsc_switch_datapath(ndev, false);
-	netdev_info(ndev, "Data path switched from VF: %s\n", vf_netdev->name);
-	rndis_filter_close(netvsc_dev);
-	netif_carrier_on(ndev);
-
-	/* Now notify peers through netvsc device. */
-	call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, ndev);
+	netvsc_switch_datapath(ndev, vf_is_up);
+	netdev_info(ndev, "Data path switched %s VF: %s\n",
+		    vf_is_up ? "to" : "from", vf_netdev->name);
 
 	return NOTIFY_OK;
 }
@@ -1499,12 +1872,15 @@ static int netvsc_unregister_vf(struct net_device *vf_netdev)
 		return NOTIFY_DONE;
 
 	net_device_ctx = netdev_priv(ndev);
+	cancel_delayed_work_sync(&net_device_ctx->vf_takeover);
 
 	netdev_info(ndev, "VF unregistering: %s\n", vf_netdev->name);
 
+	netdev_rx_handler_unregister(vf_netdev);
+	netdev_upper_dev_unlink(vf_netdev, ndev);
 	RCU_INIT_POINTER(net_device_ctx->vf_netdev, NULL);
 	dev_put(vf_netdev);
-	module_put(THIS_MODULE);
+
 	return NOTIFY_OK;
 }
 
@@ -1515,12 +1891,12 @@ static int netvsc_probe(struct hv_device *dev,
 	struct net_device_context *net_device_ctx;
 	struct netvsc_device_info device_info;
 	struct netvsc_device *nvdev;
-	int ret;
+	int ret = -ENOMEM;
 
 	net = alloc_etherdev_mq(sizeof(struct net_device_context),
 				VRSS_CHANNEL_MAX);
 	if (!net)
-		return -ENOMEM;
+		goto no_net;
 
 	netif_carrier_off(net);
 
@@ -1539,6 +1915,12 @@ static int netvsc_probe(struct hv_device *dev,
 
 	spin_lock_init(&net_device_ctx->lock);
 	INIT_LIST_HEAD(&net_device_ctx->reconfig_events);
+	INIT_DELAYED_WORK(&net_device_ctx->vf_takeover, netvsc_vf_setup);
+
+	net_device_ctx->vf_stats
+		= netdev_alloc_pcpu_stats(struct netvsc_vf_pcpu_stats);
+	if (!net_device_ctx->vf_stats)
+		goto no_stats;
 
 	net->netdev_ops = &device_ops;
 	net->ethtool_ops = &ethtool_ops;
@@ -1551,13 +1933,16 @@ static int netvsc_probe(struct hv_device *dev,
 	memset(&device_info, 0, sizeof(device_info));
 	device_info.ring_size = ring_size;
 	device_info.num_chn = VRSS_CHANNEL_DEFAULT;
-	ret = rndis_filter_device_add(dev, &device_info);
-	if (ret != 0) {
+	device_info.send_sections = NETVSC_DEFAULT_TX;
+	device_info.recv_sections = NETVSC_DEFAULT_RX;
+
+	nvdev = rndis_filter_device_add(dev, &device_info);
+	if (IS_ERR(nvdev)) {
+		ret = PTR_ERR(nvdev);
 		netdev_err(net, "unable to add netvsc device (ret %d)\n", ret);
-		free_netdev(net);
-		hv_set_drvdata(dev, NULL);
-		return ret;
+		goto rndis_failed;
 	}
+
 	memcpy(net->dev_addr, device_info.mac_adr, ETH_ALEN);
 
 	/* hw_features computed in rndis_filter_device_add */
@@ -1566,10 +1951,7 @@ static int netvsc_probe(struct hv_device *dev,
 		NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
 	net->vlan_features = net->features;
 
-	/* RCU not necessary here, device not registered */
-	nvdev = net_device_ctx->nvdev;
-	netif_set_real_num_tx_queues(net, nvdev->num_chn);
-	netif_set_real_num_rx_queues(net, nvdev->num_chn);
+	netdev_lockdep_set_classes(net);
 
 	/* MTU range: 68 - 1500 or 65521 */
 	net->min_mtu = NETVSC_MTU_MIN;
@@ -1581,20 +1963,29 @@ static int netvsc_probe(struct hv_device *dev,
 	ret = register_netdev(net);
 	if (ret != 0) {
 		pr_err("Unable to register netdev.\n");
-		rndis_filter_device_remove(dev, nvdev);
-		free_netdev(net);
+		goto register_failed;
 	}
 
 	return ret;
+
+register_failed:
+	rndis_filter_device_remove(dev, nvdev);
+rndis_failed:
+	free_percpu(net_device_ctx->vf_stats);
+no_stats:
+	hv_set_drvdata(dev, NULL);
+	free_netdev(net);
+no_net:
+	return ret;
 }
 
 static int netvsc_remove(struct hv_device *dev)
 {
-	struct net_device *net;
 	struct net_device_context *ndev_ctx;
+	struct net_device *vf_netdev;
+	struct net_device *net;
 
 	net = hv_get_drvdata(dev);
-
 	if (net == NULL) {
 		dev_err(&dev->device, "No net device to remove\n");
 		return 0;
@@ -1611,13 +2002,19 @@ static int netvsc_remove(struct hv_device *dev)
 	 * removed. Also blocks mtu and channel changes.
 	 */
 	rtnl_lock();
-	rndis_filter_device_remove(dev, ndev_ctx->nvdev);
-	rtnl_unlock();
+	vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
+	if (vf_netdev)
+		netvsc_unregister_vf(vf_netdev);
 
-	unregister_netdev(net);
+	unregister_netdevice(net);
+
+	rndis_filter_device_remove(dev,
+				   rtnl_dereference(ndev_ctx->nvdev));
+	rtnl_unlock();
 
 	hv_set_drvdata(dev, NULL);
 
+	free_percpu(ndev_ctx->vf_stats);
 	free_netdev(net);
 	return 0;
 }
@@ -1672,9 +2069,8 @@ static int netvsc_netdev_event(struct notifier_block *this,
 	case NETDEV_UNREGISTER:
 		return netvsc_unregister_vf(event_dev);
 	case NETDEV_UP:
-		return netvsc_vf_up(event_dev);
 	case NETDEV_DOWN:
-		return netvsc_vf_down(event_dev);
+		return netvsc_vf_changed(event_dev);
 	default:
 		return NOTIFY_DONE;
 	}
diff --git a/drivers/net/hyperv/rndis_filter.c b/drivers/net/hyperv/rndis_filter.c
index d6308ffda53e..065b204d8e17 100644
--- a/drivers/net/hyperv/rndis_filter.c
+++ b/drivers/net/hyperv/rndis_filter.c
@@ -28,6 +28,7 @@
 #include <linux/if_vlan.h>
 #include <linux/nls.h>
 #include <linux/vmalloc.h>
+#include <linux/rtnetlink.h>
 
 #include "hyperv_net.h"
 
@@ -213,11 +214,11 @@ static void dump_rndis_message(struct hv_device *hv_dev,
 static int rndis_filter_send_request(struct rndis_device *dev,
 				  struct rndis_request *req)
 {
-	int ret;
 	struct hv_netvsc_packet *packet;
 	struct hv_page_buffer page_buf[2];
 	struct hv_page_buffer *pb = page_buf;
 	struct net_device_context *net_device_ctx = netdev_priv(dev->ndev);
+	int ret;
 
 	/* Setup the packet to send it */
 	packet = &req->pkt;
@@ -243,7 +244,10 @@ static int rndis_filter_send_request(struct rndis_device *dev,
 			pb[0].len;
 	}
 
-	ret = netvsc_send(net_device_ctx->device_ctx, packet, NULL, &pb, NULL);
+	rcu_read_lock_bh();
+	ret = netvsc_send(net_device_ctx, packet, NULL, pb, NULL);
+	rcu_read_unlock_bh();
+
 	return ret;
 }
 
@@ -443,8 +447,9 @@ int rndis_filter_receive(struct net_device *ndev,
 	return 0;
 }
 
-static int rndis_filter_query_device(struct rndis_device *dev, u32 oid,
-				  void *result, u32 *result_size)
+static int rndis_filter_query_device(struct rndis_device *dev,
+				     struct netvsc_device *nvdev,
+				     u32 oid, void *result, u32 *result_size)
 {
 	struct rndis_request *request;
 	u32 inresult_size = *result_size;
@@ -471,8 +476,6 @@ static int rndis_filter_query_device(struct rndis_device *dev, u32 oid,
 	query->dev_vc_handle = 0;
 
 	if (oid == OID_TCP_OFFLOAD_HARDWARE_CAPABILITIES) {
-		struct net_device_context *ndevctx = netdev_priv(dev->ndev);
-		struct netvsc_device *nvdev = ndevctx->nvdev;
 		struct ndis_offload *hwcaps;
 		u32 nvsp_version = nvdev->nvsp_version;
 		u8 ndis_rev;
@@ -541,14 +544,15 @@ cleanup:
 
 /* Get the hardware offload capabilities */
 static int
-rndis_query_hwcaps(struct rndis_device *dev, struct ndis_offload *caps)
+rndis_query_hwcaps(struct rndis_device *dev, struct netvsc_device *net_device,
+		   struct ndis_offload *caps)
 {
 	u32 caps_len = sizeof(*caps);
 	int ret;
 
 	memset(caps, 0, sizeof(*caps));
 
-	ret = rndis_filter_query_device(dev,
+	ret = rndis_filter_query_device(dev, net_device,
 					OID_TCP_OFFLOAD_HARDWARE_CAPABILITIES,
 					caps, &caps_len);
 	if (ret)
@@ -577,11 +581,12 @@ rndis_query_hwcaps(struct rndis_device *dev, struct ndis_offload *caps)
 	return 0;
 }
 
-static int rndis_filter_query_device_mac(struct rndis_device *dev)
+static int rndis_filter_query_device_mac(struct rndis_device *dev,
+					 struct netvsc_device *net_device)
 {
 	u32 size = ETH_ALEN;
 
-	return rndis_filter_query_device(dev,
+	return rndis_filter_query_device(dev, net_device,
 				      RNDIS_OID_802_3_PERMANENT_ADDRESS,
 				      dev->hw_mac_adr, &size);
 }
@@ -589,9 +594,9 @@ static int rndis_filter_query_device_mac(struct rndis_device *dev)
 #define NWADR_STR "NetworkAddress"
 #define NWADR_STRLEN 14
 
-int rndis_filter_set_device_mac(struct net_device *ndev, char *mac)
+int rndis_filter_set_device_mac(struct netvsc_device *nvdev,
+				const char *mac)
 {
-	struct netvsc_device *nvdev = net_device_to_netvsc_device(ndev);
 	struct rndis_device *rdev = nvdev->extension;
 	struct rndis_request *request;
 	struct rndis_set_request *set;
@@ -645,11 +650,8 @@ int rndis_filter_set_device_mac(struct net_device *ndev, char *mac)
 	wait_for_completion(&request->wait_event);
 
 	set_complete = &request->response_msg.msg.set_complete;
-	if (set_complete->status != RNDIS_STATUS_SUCCESS) {
-		netdev_err(ndev, "Fail to set MAC on host side:0x%x\n",
-			   set_complete->status);
-		ret = -EINVAL;
-	}
+	if (set_complete->status != RNDIS_STATUS_SUCCESS)
+		ret = -EIO;
 
 cleanup:
 	put_rndis_request(rdev, request);
@@ -658,9 +660,9 @@ cleanup:
 
 static int
 rndis_filter_set_offload_params(struct net_device *ndev,
+				struct netvsc_device *nvdev,
 				struct ndis_offload_params *req_offloads)
 {
-	struct netvsc_device *nvdev = net_device_to_netvsc_device(ndev);
 	struct rndis_device *rdev = nvdev->extension;
 	struct rndis_request *request;
 	struct rndis_set_request *set;
@@ -715,7 +717,7 @@ cleanup:
 }
 
 int rndis_filter_set_rss_param(struct rndis_device *rdev,
-			       const u8 *rss_key, int num_queue)
+			       const u8 *rss_key)
 {
 	struct net_device *ndev = rdev->ndev;
 	struct rndis_request *request;
@@ -782,27 +784,27 @@ cleanup:
 	return ret;
 }
 
-static int rndis_filter_query_device_link_status(struct rndis_device *dev)
+static int rndis_filter_query_device_link_status(struct rndis_device *dev,
+						 struct netvsc_device *net_device)
 {
 	u32 size = sizeof(u32);
 	u32 link_status;
-	int ret;
-
-	ret = rndis_filter_query_device(dev,
-				      RNDIS_OID_GEN_MEDIA_CONNECT_STATUS,
-				      &link_status, &size);
 
-	return ret;
+	return rndis_filter_query_device(dev, net_device,
+					 RNDIS_OID_GEN_MEDIA_CONNECT_STATUS,
+					 &link_status, &size);
 }
 
-static int rndis_filter_query_link_speed(struct rndis_device *dev)
+static int rndis_filter_query_link_speed(struct rndis_device *dev,
+					 struct netvsc_device *net_device)
 {
 	u32 size = sizeof(u32);
 	u32 link_speed;
 	struct net_device_context *ndc;
 	int ret;
 
-	ret = rndis_filter_query_device(dev, RNDIS_OID_GEN_LINK_SPEED,
+	ret = rndis_filter_query_device(dev, net_device,
+					RNDIS_OID_GEN_LINK_SPEED,
 					&link_speed, &size);
 
 	if (!ret) {
@@ -871,14 +873,14 @@ void rndis_filter_update(struct netvsc_device *nvdev)
 	schedule_work(&rdev->mcast_work);
 }
 
-static int rndis_filter_init_device(struct rndis_device *dev)
+static int rndis_filter_init_device(struct rndis_device *dev,
+				    struct netvsc_device *nvdev)
 {
 	struct rndis_request *request;
 	struct rndis_initialize_request *init;
 	struct rndis_initialize_complete *init_complete;
 	u32 status;
 	int ret;
-	struct netvsc_device *nvdev = net_device_to_netvsc_device(dev->ndev);
 
 	request = get_rndis_request(dev, RNDIS_MSG_INIT,
 			RNDIS_MESSAGE_SIZE(struct rndis_initialize_request));
@@ -926,12 +928,12 @@ static bool netvsc_device_idle(const struct netvsc_device *nvdev)
 {
 	int i;
 
-	if (atomic_read(&nvdev->num_outstanding_recvs) > 0)
-		return false;
-
 	for (i = 0; i < nvdev->num_chn; i++) {
 		const struct netvsc_channel *nvchan = &nvdev->chan_table[i];
 
+		if (nvchan->mrc.first != nvchan->mrc.next)
+			return false;
+
 		if (atomic_read(&nvchan->queue_sends) > 0)
 			return false;
 	}
@@ -944,7 +946,7 @@ static void rndis_filter_halt_device(struct rndis_device *dev)
 	struct rndis_request *request;
 	struct rndis_halt_request *halt;
 	struct net_device_context *net_device_ctx = netdev_priv(dev->ndev);
-	struct netvsc_device *nvdev = net_device_ctx->nvdev;
+	struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
 
 	/* Attempt to do a rndis device halt */
 	request = get_rndis_request(dev, RNDIS_MSG_HALT,
@@ -1015,20 +1017,20 @@ static void netvsc_sc_open(struct vmbus_channel *new_sc)
 {
 	struct net_device *ndev =
 		hv_get_drvdata(new_sc->primary_channel->device_obj);
-	struct netvsc_device *nvscdev = net_device_to_netvsc_device(ndev);
+	struct net_device_context *ndev_ctx = netdev_priv(ndev);
+	struct netvsc_device *nvscdev;
 	u16 chn_index = new_sc->offermsg.offer.sub_channel_index;
 	struct netvsc_channel *nvchan;
 	int ret;
 
-	if (chn_index >= nvscdev->num_chn)
+	/* This is safe because this callback only happens when
+	 * new device is being setup and waiting on the channel_init_wait.
+	 */
+	nvscdev = rcu_dereference_raw(ndev_ctx->nvdev);
+	if (!nvscdev || chn_index >= nvscdev->num_chn)
 		return;
 
 	nvchan = nvscdev->chan_table + chn_index;
-	nvchan->mrc.buf
-		= vzalloc(NETVSC_RECVSLOT_MAX * sizeof(struct recv_comp_data));
-
-	if (!nvchan->mrc.buf)
-		return;
 
 	/* Because the device uses NAPI, all the interrupt batching and
 	 * control is done via Net softirq, not the channel handling
@@ -1037,8 +1039,6 @@ static void netvsc_sc_open(struct vmbus_channel *new_sc)
 
 	/* Set the channel before opening.*/
 	nvchan->channel = new_sc;
-	netif_napi_add(ndev, &nvchan->napi,
-		       netvsc_poll, NAPI_POLL_WEIGHT);
 
 	ret = vmbus_open(new_sc, nvscdev->ring_size * PAGE_SIZE,
 			 nvscdev->ring_size * PAGE_SIZE, NULL, 0,
@@ -1046,14 +1046,90 @@ static void netvsc_sc_open(struct vmbus_channel *new_sc)
 	if (ret == 0)
 		napi_enable(&nvchan->napi);
 	else
-		netif_napi_del(&nvchan->napi);
+		netdev_notice(ndev, "sub channel open failed: %d\n", ret);
 
-	atomic_inc(&nvscdev->open_chn);
-	wake_up(&nvscdev->subchan_open);
+	if (atomic_inc_return(&nvscdev->open_chn) == nvscdev->num_chn)
+		wake_up(&nvscdev->subchan_open);
 }
 
-int rndis_filter_device_add(struct hv_device *dev,
-			    struct netvsc_device_info *device_info)
+/* Open sub-channels after completing the handling of the device probe.
+ * This breaks overlap of processing the host message for the
+ * new primary channel with the initialization of sub-channels.
+ */
+void rndis_set_subchannel(struct work_struct *w)
+{
+	struct netvsc_device *nvdev
+		= container_of(w, struct netvsc_device, subchan_work);
+	struct nvsp_message *init_packet = &nvdev->channel_init_pkt;
+	struct net_device_context *ndev_ctx;
+	struct rndis_device *rdev;
+	struct net_device *ndev;
+	struct hv_device *hv_dev;
+	int i, ret;
+
+	if (!rtnl_trylock()) {
+		schedule_work(w);
+		return;
+	}
+
+	rdev = nvdev->extension;
+	if (!rdev)
+		goto unlock;	/* device was removed */
+
+	ndev = rdev->ndev;
+	ndev_ctx = netdev_priv(ndev);
+	hv_dev = ndev_ctx->device_ctx;
+
+	memset(init_packet, 0, sizeof(struct nvsp_message));
+	init_packet->hdr.msg_type = NVSP_MSG5_TYPE_SUBCHANNEL;
+	init_packet->msg.v5_msg.subchn_req.op = NVSP_SUBCHANNEL_ALLOCATE;
+	init_packet->msg.v5_msg.subchn_req.num_subchannels =
+						nvdev->num_chn - 1;
+	ret = vmbus_sendpacket(hv_dev->channel, init_packet,
+			       sizeof(struct nvsp_message),
+			       (unsigned long)init_packet,
+			       VM_PKT_DATA_INBAND,
+			       VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
+	if (ret) {
+		netdev_err(ndev, "sub channel allocate send failed: %d\n", ret);
+		goto failed;
+	}
+
+	wait_for_completion(&nvdev->channel_init_wait);
+	if (init_packet->msg.v5_msg.subchn_comp.status != NVSP_STAT_SUCCESS) {
+		netdev_err(ndev, "sub channel request failed\n");
+		goto failed;
+	}
+
+	nvdev->num_chn = 1 +
+		init_packet->msg.v5_msg.subchn_comp.num_subchannels;
+
+	/* wait for all sub channels to open */
+	wait_event(nvdev->subchan_open,
+		   atomic_read(&nvdev->open_chn) == nvdev->num_chn);
+
+	/* ignore failues from setting rss parameters, still have channels */
+	rndis_filter_set_rss_param(rdev, netvsc_hash_key);
+
+	netif_set_real_num_tx_queues(ndev, nvdev->num_chn);
+	netif_set_real_num_rx_queues(ndev, nvdev->num_chn);
+
+	rtnl_unlock();
+	return;
+
+failed:
+	/* fallback to only primary channel */
+	for (i = 1; i < nvdev->num_chn; i++)
+		netif_napi_del(&nvdev->chan_table[i].napi);
+
+	nvdev->max_chn = 1;
+	nvdev->num_chn = 1;
+unlock:
+	rtnl_unlock();
+}
+
+struct netvsc_device *rndis_filter_device_add(struct hv_device *dev,
+				      struct netvsc_device_info *device_info)
 {
 	struct net_device *net = hv_get_drvdata(dev);
 	struct net_device_context *net_device_ctx = netdev_priv(net);
@@ -1061,32 +1137,30 @@ int rndis_filter_device_add(struct hv_device *dev,
 	struct rndis_device *rndis_device;
 	struct ndis_offload hwcaps;
 	struct ndis_offload_params offloads;
-	struct nvsp_message *init_packet;
 	struct ndis_recv_scale_cap rsscap;
 	u32 rsscap_size = sizeof(struct ndis_recv_scale_cap);
 	unsigned int gso_max_size = GSO_MAX_SIZE;
-	u32 mtu, size, num_rss_qs;
+	u32 mtu, size;
 	const struct cpumask *node_cpu_mask;
 	u32 num_possible_rss_qs;
 	int i, ret;
 
 	rndis_device = get_rndis_device();
 	if (!rndis_device)
-		return -ENODEV;
+		return ERR_PTR(-ENODEV);
 
 	/*
 	 * Let the inner driver handle this first to create the netvsc channel
 	 * NOTE! Once the channel is created, we may get a receive callback
 	 * (RndisFilterOnReceive()) before this call is completed
 	 */
-	ret = netvsc_device_add(dev, device_info);
-	if (ret != 0) {
+	net_device = netvsc_device_add(dev, device_info);
+	if (IS_ERR(net_device)) {
 		kfree(rndis_device);
-		return ret;
+		return net_device;
 	}
 
 	/* Initialize the rndis device */
-	net_device = net_device_ctx->nvdev;
 	net_device->max_chn = 1;
 	net_device->num_chn = 1;
 
@@ -1094,35 +1168,29 @@ int rndis_filter_device_add(struct hv_device *dev,
 	rndis_device->ndev = net;
 
 	/* Send the rndis initialization message */
-	ret = rndis_filter_init_device(rndis_device);
-	if (ret != 0) {
-		rndis_filter_device_remove(dev, net_device);
-		return ret;
-	}
+	ret = rndis_filter_init_device(rndis_device, net_device);
+	if (ret != 0)
+		goto err_dev_remv;
 
 	/* Get the MTU from the host */
 	size = sizeof(u32);
-	ret = rndis_filter_query_device(rndis_device,
+	ret = rndis_filter_query_device(rndis_device, net_device,
 					RNDIS_OID_GEN_MAXIMUM_FRAME_SIZE,
 					&mtu, &size);
 	if (ret == 0 && size == sizeof(u32) && mtu < net->mtu)
 		net->mtu = mtu;
 
 	/* Get the mac address */
-	ret = rndis_filter_query_device_mac(rndis_device);
-	if (ret != 0) {
-		rndis_filter_device_remove(dev, net_device);
-		return ret;
-	}
+	ret = rndis_filter_query_device_mac(rndis_device, net_device);
+	if (ret != 0)
+		goto err_dev_remv;
 
 	memcpy(device_info->mac_adr, rndis_device->hw_mac_adr, ETH_ALEN);
 
 	/* Find HW offload capabilities */
-	ret = rndis_query_hwcaps(rndis_device, &hwcaps);
-	if (ret != 0) {
-		rndis_filter_device_remove(dev, net_device);
-		return ret;
-	}
+	ret = rndis_query_hwcaps(rndis_device, net_device, &hwcaps);
+	if (ret != 0)
+		goto err_dev_remv;
 
 	/* A value of zero means "no change"; now turn on what we want. */
 	memset(&offloads, 0, sizeof(struct ndis_offload_params));
@@ -1177,24 +1245,24 @@ int rndis_filter_device_add(struct hv_device *dev,
 
 	netif_set_gso_max_size(net, gso_max_size);
 
-	ret = rndis_filter_set_offload_params(net, &offloads);
+	ret = rndis_filter_set_offload_params(net, net_device, &offloads);
 	if (ret)
 		goto err_dev_remv;
 
-	rndis_filter_query_device_link_status(rndis_device);
+	rndis_filter_query_device_link_status(rndis_device, net_device);
 
 	netdev_dbg(net, "Device MAC %pM link state %s\n",
 		   rndis_device->hw_mac_adr,
 		   rndis_device->link_state ? "down" : "up");
 
 	if (net_device->nvsp_version < NVSP_PROTOCOL_VERSION_5)
-		return 0;
+		return net_device;
 
-	rndis_filter_query_link_speed(rndis_device);
+	rndis_filter_query_link_speed(rndis_device, net_device);
 
 	/* vRSS setup */
 	memset(&rsscap, 0, rsscap_size);
-	ret = rndis_filter_query_device(rndis_device,
+	ret = rndis_filter_query_device(rndis_device, net_device,
 					OID_GEN_RECEIVE_SCALE_CAPABILITIES,
 					&rsscap, &rsscap_size);
 	if (ret || rsscap.num_recv_que < 2)
@@ -1220,53 +1288,36 @@ int rndis_filter_device_add(struct hv_device *dev,
 							net_device->num_chn);
 
 	atomic_set(&net_device->open_chn, 1);
-	num_rss_qs = net_device->num_chn - 1;
-	if (num_rss_qs == 0)
-		return 0;
-
 	vmbus_set_sc_create_callback(dev->channel, netvsc_sc_open);
 
-	init_packet = &net_device->channel_init_pkt;
-	memset(init_packet, 0, sizeof(struct nvsp_message));
-	init_packet->hdr.msg_type = NVSP_MSG5_TYPE_SUBCHANNEL;
-	init_packet->msg.v5_msg.subchn_req.op = NVSP_SUBCHANNEL_ALLOCATE;
-	init_packet->msg.v5_msg.subchn_req.num_subchannels =
-						net_device->num_chn - 1;
-	ret = vmbus_sendpacket(dev->channel, init_packet,
-			       sizeof(struct nvsp_message),
-			       (unsigned long)init_packet,
-			       VM_PKT_DATA_INBAND,
-			       VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
-	if (ret)
-		goto out;
-
-	wait_for_completion(&net_device->channel_init_wait);
-	if (init_packet->msg.v5_msg.subchn_comp.status != NVSP_STAT_SUCCESS) {
-		ret = -ENODEV;
-		goto out;
+	for (i = 1; i < net_device->num_chn; i++) {
+		ret = netvsc_alloc_recv_comp_ring(net_device, i);
+		if (ret) {
+			while (--i != 0)
+				vfree(net_device->chan_table[i].mrc.slots);
+			goto out;
+		}
 	}
 
-	net_device->num_chn = 1 +
-		init_packet->msg.v5_msg.subchn_comp.num_subchannels;
+	for (i = 1; i < net_device->num_chn; i++)
+		netif_napi_add(net, &net_device->chan_table[i].napi,
+			       netvsc_poll, NAPI_POLL_WEIGHT);
 
-	/* wait for all sub channels to open */
-	wait_event(net_device->subchan_open,
-		   atomic_read(&net_device->open_chn) == net_device->num_chn);
+	if (net_device->num_chn > 1)
+		schedule_work(&net_device->subchan_work);
 
-	/* ignore failues from setting rss parameters, still have channels */
-	rndis_filter_set_rss_param(rndis_device, netvsc_hash_key,
-				   net_device->num_chn);
 out:
+	/* if unavailable, just proceed with one queue */
 	if (ret) {
 		net_device->max_chn = 1;
 		net_device->num_chn = 1;
 	}
 
-	return 0; /* return 0 because primary channel can be used alone */
+	return net_device;
 
 err_dev_remv:
 	rndis_filter_device_remove(dev, net_device);
-	return ret;
+	return ERR_PTR(ret);
 }
 
 void rndis_filter_device_remove(struct hv_device *dev,
@@ -1277,10 +1328,10 @@ void rndis_filter_device_remove(struct hv_device *dev,
 	/* Halt and release the rndis device */
 	rndis_filter_halt_device(rndis_dev);
 
-	kfree(rndis_dev);
 	net_dev->extension = NULL;
 
 	netvsc_device_remove(dev);
+	kfree(rndis_dev);
 }
 
 int rndis_filter_open(struct netvsc_device *nvdev)
@@ -1304,3 +1355,8 @@ int rndis_filter_close(struct netvsc_device *nvdev)
 
 	return rndis_filter_close_device(nvdev->extension);
 }
+
+bool rndis_filter_opened(const struct netvsc_device *nvdev)
+{
+	return atomic_read(&nvdev->open_cnt) > 0;
+}
diff --git a/drivers/net/ieee802154/ca8210.c b/drivers/net/ieee802154/ca8210.c
index a626c539fb17..24a1eabbbc9d 100644
--- a/drivers/net/ieee802154/ca8210.c
+++ b/drivers/net/ieee802154/ca8210.c
@@ -66,6 +66,7 @@
 #include <linux/spinlock.h>
 #include <linux/string.h>
 #include <linux/workqueue.h>
+#include <linux/interrupt.h>
 
 #include <net/ieee802154_netdev.h>
 #include <net/mac802154.h>
@@ -916,10 +917,7 @@ static int ca8210_spi_transfer(
 	struct cas_control *cas_ctl;
 
 	if (!spi) {
-		dev_crit(
-			&spi->dev,
-			"NULL spi device passed to ca8210_spi_transfer\n"
-		);
+		pr_crit("NULL spi device passed to %s\n", __func__);
 		return -ENODEV;
 	}
 
diff --git a/drivers/net/ieee802154/mrf24j40.c b/drivers/net/ieee802154/mrf24j40.c
index 7d334963dc08..ee7084b2d52d 100644
--- a/drivers/net/ieee802154/mrf24j40.c
+++ b/drivers/net/ieee802154/mrf24j40.c
@@ -1330,7 +1330,8 @@ static int mrf24j40_probe(struct spi_device *spi)
 	if (spi->max_speed_hz > MAX_SPI_SPEED_HZ) {
 		dev_warn(&spi->dev, "spi clock above possible maximum: %d",
 			 MAX_SPI_SPEED_HZ);
-		return -EINVAL;
+		ret = -EINVAL;
+		goto err_register_device;
 	}
 
 	ret = mrf24j40_hw_init(devrec);
diff --git a/drivers/net/ipvlan/ipvlan_main.c b/drivers/net/ipvlan/ipvlan_main.c
index 8dab74a81303..c74893c1e620 100644
--- a/drivers/net/ipvlan/ipvlan_main.c
+++ b/drivers/net/ipvlan/ipvlan_main.c
@@ -15,7 +15,7 @@ struct ipvlan_netns {
 	unsigned int ipvl_nf_hook_refcnt;
 };
 
-static struct nf_hook_ops ipvl_nfops[] __read_mostly = {
+static const struct nf_hook_ops ipvl_nfops[] = {
 	{
 		.hook     = ipvlan_nf_input,
 		.pf       = NFPROTO_IPV4,
@@ -169,7 +169,7 @@ static void ipvlan_port_destroy(struct net_device *dev)
 
 #define IPVLAN_FEATURES \
 	(NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST | \
-	 NETIF_F_GSO | NETIF_F_TSO | NETIF_F_UFO | NETIF_F_GSO_ROBUST | \
+	 NETIF_F_GSO | NETIF_F_TSO | NETIF_F_GSO_ROBUST | \
 	 NETIF_F_TSO_ECN | NETIF_F_TSO6 | NETIF_F_GRO | NETIF_F_RXCSUM | \
 	 NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_STAG_FILTER)
 
diff --git a/drivers/net/ipvlan/ipvtap.c b/drivers/net/ipvlan/ipvtap.c
index 22f133ea8d7b..5dea2063dbc8 100644
--- a/drivers/net/ipvlan/ipvtap.c
+++ b/drivers/net/ipvlan/ipvtap.c
@@ -24,7 +24,7 @@
 #include <linux/virtio_net.h>
 
 #define TUN_OFFLOADS (NETIF_F_HW_CSUM | NETIF_F_TSO_ECN | NETIF_F_TSO | \
-		      NETIF_F_TSO6 | NETIF_F_UFO)
+		      NETIF_F_TSO6)
 
 static dev_t ipvtap_major;
 static struct cdev ipvtap_cdev;
diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c
index 5e1ab1160856..98e4deaa3a6a 100644
--- a/drivers/net/macsec.c
+++ b/drivers/net/macsec.c
@@ -3521,6 +3521,7 @@ module_init(macsec_init);
 module_exit(macsec_exit);
 
 MODULE_ALIAS_RTNL_LINK("macsec");
+MODULE_ALIAS_GENL_FAMILY("macsec");
 
 MODULE_DESCRIPTION("MACsec IEEE 802.1AE");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c
index 0f581ee74fe4..d2aea961e0f4 100644
--- a/drivers/net/macvlan.c
+++ b/drivers/net/macvlan.c
@@ -835,13 +835,13 @@ static struct lock_class_key macvlan_netdev_addr_lock_key;
 
 #define ALWAYS_ON_OFFLOADS \
 	(NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE | \
-	 NETIF_F_GSO_ROBUST)
+	 NETIF_F_GSO_ROBUST | NETIF_F_GSO_ENCAP_ALL)
 
 #define ALWAYS_ON_FEATURES (ALWAYS_ON_OFFLOADS | NETIF_F_LLTX)
 
 #define MACVLAN_FEATURES \
 	(NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST | \
-	 NETIF_F_GSO | NETIF_F_TSO | NETIF_F_UFO | NETIF_F_LRO | \
+	 NETIF_F_GSO | NETIF_F_TSO | NETIF_F_LRO | \
 	 NETIF_F_TSO_ECN | NETIF_F_TSO6 | NETIF_F_GRO | NETIF_F_RXCSUM | \
 	 NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_STAG_FILTER)
 
@@ -874,6 +874,7 @@ static int macvlan_init(struct net_device *dev)
 	dev->hw_features	|= NETIF_F_LRO;
 	dev->vlan_features	= lowerdev->vlan_features & MACVLAN_FEATURES;
 	dev->vlan_features	|= ALWAYS_ON_OFFLOADS;
+	dev->hw_enc_features    |= dev->features;
 	dev->gso_max_size	= lowerdev->gso_max_size;
 	dev->gso_max_segs	= lowerdev->gso_max_segs;
 	dev->hard_header_len	= lowerdev->hard_header_len;
diff --git a/drivers/net/macvtap.c b/drivers/net/macvtap.c
index 91e7b19bbf86..c2d0ea2fb019 100644
--- a/drivers/net/macvtap.c
+++ b/drivers/net/macvtap.c
@@ -49,7 +49,7 @@ static struct class macvtap_class = {
 static struct cdev macvtap_cdev;
 
 #define TUN_OFFLOADS (NETIF_F_HW_CSUM | NETIF_F_TSO_ECN | NETIF_F_TSO | \
-		      NETIF_F_TSO6 | NETIF_F_UFO)
+		      NETIF_F_TSO6)
 
 static void macvtap_count_tx_dropped(struct tap_dev *tap)
 {
diff --git a/drivers/net/phy/Kconfig b/drivers/net/phy/Kconfig
index 928fd892f167..a9d16a3af514 100644
--- a/drivers/net/phy/Kconfig
+++ b/drivers/net/phy/Kconfig
@@ -5,7 +5,7 @@
 menuconfig MDIO_DEVICE
 	tristate "MDIO bus device drivers"
 	help
-	   MDIO devices and driver infrastructure code.
+	  MDIO devices and driver infrastructure code.
 
 config MDIO_BUS
 	tristate
@@ -85,7 +85,7 @@ config MDIO_BUS_MUX_MMIOREG
 	  parent bus.  Child bus selection is under the control of one of
 	  the FPGA's registers.
 
-	  Currently, only 8-bit registers are supported.
+	  Currently, only 8/16/32 bits registers are supported.
 
 config MDIO_CAVIUM
 	tristate
@@ -106,12 +106,22 @@ config MDIO_HISI_FEMAC
 	  This module provides a driver for the MDIO busses found in the
 	  Hisilicon SoC that have an Fast Ethernet MAC.
 
+config MDIO_I2C
+	tristate
+	depends on I2C
+	help
+	  Support I2C based PHYs.  This provides a MDIO bus bridged
+	  to I2C to allow PHYs connected in I2C mode to be accessed
+	  using the existing infrastructure.
+
+	  This is library mode.
+
 config MDIO_MOXART
-        tristate "MOXA ART MDIO interface support"
-        depends on ARCH_MOXART
-        help
-          This driver supports the MDIO interface found in the network
-          interface units of the MOXA ART SoC
+	tristate "MOXA ART MDIO interface support"
+	depends on ARCH_MOXART
+	help
+	  This driver supports the MDIO interface found in the network
+	  interface units of the MOXA ART SoC
 
 config MDIO_OCTEON
 	tristate "Octeon and some ThunderX SOCs MDIO buses"
@@ -159,6 +169,16 @@ menuconfig PHYLIB
 	  devices.  This option provides infrastructure for
 	  managing PHY devices.
 
+config PHYLINK
+	tristate
+	depends on NETDEVICES
+	select PHYLIB
+	select SWPHY
+	help
+	  PHYlink models the link between the PHY and MAC, allowing fixed
+	  configuration links, PHYs, and Serdes links with MAC level
+	  autonegotiation modes.
+
 if PHYLIB
 
 config SWPHY
@@ -172,7 +192,7 @@ config LED_TRIGGER_PHY
 	  state change will trigger the events, for consumption by an
 	  LED class driver.  There are triggers for each link speed currently
 	  supported by the phy, and are of the form:
-	       <mii bus id>:<phy>:<speed>
+	      <mii bus id>:<phy>:<speed>
 
 	  Where speed is in the form:
 		<Speed in megabits>Mbps or <Speed in gigabits>Gbps
@@ -180,15 +200,20 @@ config LED_TRIGGER_PHY
 
 comment "MII PHY device drivers"
 
+config SFP
+	tristate "SFP cage support"
+	depends on I2C && PHYLINK
+	select MDIO_I2C
+
 config AMD_PHY
 	tristate "AMD PHYs"
 	---help---
 	  Currently supports the am79c874
 
 config AQUANTIA_PHY
-        tristate "Aquantia PHYs"
-        ---help---
-          Currently supports the Aquantia AQ1202, AQ2104, AQR105, AQR405
+	tristate "Aquantia PHYs"
+	---help---
+	  Currently supports the Aquantia AQ1202, AQ2104, AQR105, AQR405
 
 config AT803X_PHY
 	tristate "AT803X PHYs"
@@ -341,6 +366,11 @@ config REALTEK_PHY
 	---help---
 	  Supports the Realtek 821x PHY.
 
+config ROCKCHIP_PHY
+        tristate "Driver for Rockchip Ethernet PHYs"
+        ---help---
+          Currently supports the integrated Ethernet PHY.
+
 config SMSC_PHY
 	tristate "SMSC PHYs"
 	---help---
@@ -352,21 +382,21 @@ config STE10XP
 	  This is the driver for the STe100p and STe101p PHYs.
 
 config TERANETICS_PHY
-        tristate "Teranetics PHYs"
-        ---help---
-          Currently supports the Teranetics TN2020
+	tristate "Teranetics PHYs"
+	---help---
+	  Currently supports the Teranetics TN2020
 
 config VITESSE_PHY
-        tristate "Vitesse PHYs"
-        ---help---
-          Currently supports the vsc8244
+	tristate "Vitesse PHYs"
+	---help---
+	  Currently supports the vsc8244
 
 config XILINX_GMII2RGMII
-       tristate "Xilinx GMII2RGMII converter driver"
-       ---help---
-         This driver support xilinx GMII to RGMII IP core it provides
-         the Reduced Gigabit Media Independent Interface(RGMII) between
-         Ethernet physical media devices and the Gigabit Ethernet controller.
+	tristate "Xilinx GMII2RGMII converter driver"
+	---help---
+	  This driver support xilinx GMII to RGMII IP core it provides
+	  the Reduced Gigabit Media Independent Interface(RGMII) between
+	  Ethernet physical media devices and the Gigabit Ethernet controller.
 
 endif # PHYLIB
 
diff --git a/drivers/net/phy/Makefile b/drivers/net/phy/Makefile
index 8e9b9f349384..416df92fbf4f 100644
--- a/drivers/net/phy/Makefile
+++ b/drivers/net/phy/Makefile
@@ -18,6 +18,7 @@ endif
 libphy-$(CONFIG_SWPHY)		+= swphy.o
 libphy-$(CONFIG_LED_TRIGGER_PHY)	+= phy_led_triggers.o
 
+obj-$(CONFIG_PHYLINK)		+= phylink.o
 obj-$(CONFIG_PHYLIB)		+= libphy.o
 
 obj-$(CONFIG_MDIO_BCM_IPROC)	+= mdio-bcm-iproc.o
@@ -30,12 +31,17 @@ obj-$(CONFIG_MDIO_BUS_MUX_MMIOREG) += mdio-mux-mmioreg.o
 obj-$(CONFIG_MDIO_CAVIUM)	+= mdio-cavium.o
 obj-$(CONFIG_MDIO_GPIO)		+= mdio-gpio.o
 obj-$(CONFIG_MDIO_HISI_FEMAC)	+= mdio-hisi-femac.o
+obj-$(CONFIG_MDIO_I2C)		+= mdio-i2c.o
 obj-$(CONFIG_MDIO_MOXART)	+= mdio-moxart.o
 obj-$(CONFIG_MDIO_OCTEON)	+= mdio-octeon.o
 obj-$(CONFIG_MDIO_SUN4I)	+= mdio-sun4i.o
 obj-$(CONFIG_MDIO_THUNDER)	+= mdio-thunder.o
 obj-$(CONFIG_MDIO_XGENE)	+= mdio-xgene.o
 
+obj-$(CONFIG_SFP)		+= sfp.o
+sfp-obj-$(CONFIG_SFP)		+= sfp-bus.o
+obj-y				+= $(sfp-obj-y) $(sfp-obj-m)
+
 obj-$(CONFIG_AMD_PHY)		+= amd.o
 obj-$(CONFIG_AQUANTIA_PHY)	+= aquantia.o
 obj-$(CONFIG_AT803X_PHY)	+= at803x.o
@@ -66,6 +72,7 @@ obj-$(CONFIG_MICROSEMI_PHY)	+= mscc.o
 obj-$(CONFIG_NATIONAL_PHY)	+= national.o
 obj-$(CONFIG_QSEMI_PHY)		+= qsemi.o
 obj-$(CONFIG_REALTEK_PHY)	+= realtek.o
+obj-$(CONFIG_ROCKCHIP_PHY)	+= rockchip.o
 obj-$(CONFIG_SMSC_PHY)		+= smsc.o
 obj-$(CONFIG_STE10XP)		+= ste10Xp.o
 obj-$(CONFIG_TERANETICS_PHY)	+= teranetics.o
diff --git a/drivers/net/phy/bcm7xxx.c b/drivers/net/phy/bcm7xxx.c
index caa9f6e17f34..8b33f688ac8a 100644
--- a/drivers/net/phy/bcm7xxx.c
+++ b/drivers/net/phy/bcm7xxx.c
@@ -511,7 +511,7 @@ static int bcm7xxx_config_init(struct phy_device *phydev)
 static int bcm7xxx_suspend(struct phy_device *phydev)
 {
 	int ret;
-	const struct bcm7xxx_regs {
+	static const struct bcm7xxx_regs {
 		int reg;
 		u16 value;
 	} bcm7xxx_suspend_cfg[] = {
diff --git a/drivers/net/phy/dp83640.c b/drivers/net/phy/dp83640.c
index c3065236ffcc..cbd629822f04 100644
--- a/drivers/net/phy/dp83640.c
+++ b/drivers/net/phy/dp83640.c
@@ -874,7 +874,6 @@ static void decode_rxts(struct dp83640_private *dp83640,
 			shhwtstamps = skb_hwtstamps(skb);
 			memset(shhwtstamps, 0, sizeof(*shhwtstamps));
 			shhwtstamps->hwtstamp = ns_to_ktime(rxts->ns);
-			netif_rx_ni(skb);
 			list_add(&rxts->list, &dp83640->rxpool);
 			break;
 		}
@@ -885,6 +884,9 @@ static void decode_rxts(struct dp83640_private *dp83640,
 		list_add_tail(&rxts->list, &dp83640->rxts);
 out:
 	spin_unlock_irqrestore(&dp83640->rx_lock, flags);
+
+	if (shhwtstamps)
+		netif_rx_ni(skb);
 }
 
 static void decode_txts(struct dp83640_private *dp83640,
@@ -1425,7 +1427,6 @@ static bool dp83640_rxtstamp(struct phy_device *phydev,
 			shhwtstamps = skb_hwtstamps(skb);
 			memset(shhwtstamps, 0, sizeof(*shhwtstamps));
 			shhwtstamps->hwtstamp = ns_to_ktime(rxts->ns);
-			netif_rx_ni(skb);
 			list_del_init(&rxts->list);
 			list_add(&rxts->list, &dp83640->rxpool);
 			break;
@@ -1438,6 +1439,8 @@ static bool dp83640_rxtstamp(struct phy_device *phydev,
 		skb_info->tmo = jiffies + SKB_TIMESTAMP_TIMEOUT;
 		skb_queue_tail(&dp83640->rx_queue, skb);
 		schedule_delayed_work(&dp83640->ts_work, SKB_TIMESTAMP_TIMEOUT);
+	} else {
+		netif_rx_ni(skb);
 	}
 
 	return true;
diff --git a/drivers/net/phy/marvell.c b/drivers/net/phy/marvell.c
index 5d314f143aea..15cbcdba618a 100644
--- a/drivers/net/phy/marvell.c
+++ b/drivers/net/phy/marvell.c
@@ -55,43 +55,35 @@
 #define MII_M1011_IMASK_INIT		0x6400
 #define MII_M1011_IMASK_CLEAR		0x0000
 
-#define MII_M1011_PHY_SCR		0x10
-#define MII_M1011_PHY_SCR_MDI		0x0000
-#define MII_M1011_PHY_SCR_MDI_X		0x0020
-#define MII_M1011_PHY_SCR_AUTO_CROSS	0x0060
-
-#define MII_M1145_PHY_EXT_SR		0x1b
-#define MII_M1145_PHY_EXT_CR		0x14
-#define MII_M1145_RGMII_RX_DELAY	0x0080
-#define MII_M1145_RGMII_TX_DELAY	0x0002
-#define MII_M1145_HWCFG_MODE_SGMII_NO_CLK	0x4
-#define MII_M1145_HWCFG_MODE_MASK		0xf
-#define MII_M1145_HWCFG_FIBER_COPPER_AUTO	0x8000
-
-#define MII_M1145_HWCFG_MODE_SGMII_NO_CLK	0x4
-#define MII_M1145_HWCFG_MODE_MASK		0xf
-#define MII_M1145_HWCFG_FIBER_COPPER_AUTO	0x8000
+#define MII_M1011_PHY_SCR			0x10
+#define MII_M1011_PHY_SCR_DOWNSHIFT_EN		BIT(11)
+#define MII_M1011_PHY_SCR_DOWNSHIFT_SHIFT	12
+#define MII_M1011_PHY_SRC_DOWNSHIFT_MASK	0x7800
+#define MII_M1011_PHY_SCR_MDI			(0x0 << 5)
+#define MII_M1011_PHY_SCR_MDI_X			(0x1 << 5)
+#define MII_M1011_PHY_SCR_AUTO_CROSS		(0x3 << 5)
 
 #define MII_M1111_PHY_LED_CONTROL	0x18
 #define MII_M1111_PHY_LED_DIRECT	0x4100
 #define MII_M1111_PHY_LED_COMBINE	0x411c
 #define MII_M1111_PHY_EXT_CR		0x14
-#define MII_M1111_RX_DELAY		0x80
-#define MII_M1111_TX_DELAY		0x2
+#define MII_M1111_RGMII_RX_DELAY	BIT(7)
+#define MII_M1111_RGMII_TX_DELAY	BIT(1)
 #define MII_M1111_PHY_EXT_SR		0x1b
 
 #define MII_M1111_HWCFG_MODE_MASK		0xf
-#define MII_M1111_HWCFG_MODE_COPPER_RGMII	0xb
 #define MII_M1111_HWCFG_MODE_FIBER_RGMII	0x3
 #define MII_M1111_HWCFG_MODE_SGMII_NO_CLK	0x4
+#define MII_M1111_HWCFG_MODE_RTBI		0x7
 #define MII_M1111_HWCFG_MODE_COPPER_RTBI	0x9
-#define MII_M1111_HWCFG_FIBER_COPPER_AUTO	0x8000
-#define MII_M1111_HWCFG_FIBER_COPPER_RES	0x2000
+#define MII_M1111_HWCFG_MODE_COPPER_RGMII	0xb
+#define MII_M1111_HWCFG_FIBER_COPPER_RES	BIT(13)
+#define MII_M1111_HWCFG_FIBER_COPPER_AUTO	BIT(15)
 
 #define MII_88E1121_PHY_MSCR_REG	21
 #define MII_88E1121_PHY_MSCR_RX_DELAY	BIT(5)
 #define MII_88E1121_PHY_MSCR_TX_DELAY	BIT(4)
-#define MII_88E1121_PHY_MSCR_DELAY_MASK	(~(0x3 << 4))
+#define MII_88E1121_PHY_MSCR_DELAY_MASK	(~(BIT(5) | BIT(4)))
 
 #define MII_88E1121_MISC_TEST				0x1a
 #define MII_88E1510_MISC_TEST_TEMP_THRESHOLD_MASK	0x1f00
@@ -108,24 +100,24 @@
 #define MII_88E1318S_PHY_MSCR1_PAD_ODD	BIT(6)
 
 /* Copper Specific Interrupt Enable Register */
-#define MII_88E1318S_PHY_CSIER                              0x12
+#define MII_88E1318S_PHY_CSIER				0x12
 /* WOL Event Interrupt Enable */
-#define MII_88E1318S_PHY_CSIER_WOL_EIE                      BIT(7)
+#define MII_88E1318S_PHY_CSIER_WOL_EIE			BIT(7)
 
 /* LED Timer Control Register */
-#define MII_88E1318S_PHY_LED_TCR                            0x12
-#define MII_88E1318S_PHY_LED_TCR_FORCE_INT                  BIT(15)
-#define MII_88E1318S_PHY_LED_TCR_INTn_ENABLE                BIT(7)
-#define MII_88E1318S_PHY_LED_TCR_INT_ACTIVE_LOW             BIT(11)
+#define MII_88E1318S_PHY_LED_TCR			0x12
+#define MII_88E1318S_PHY_LED_TCR_FORCE_INT		BIT(15)
+#define MII_88E1318S_PHY_LED_TCR_INTn_ENABLE		BIT(7)
+#define MII_88E1318S_PHY_LED_TCR_INT_ACTIVE_LOW		BIT(11)
 
 /* Magic Packet MAC address registers */
-#define MII_88E1318S_PHY_MAGIC_PACKET_WORD2                 0x17
-#define MII_88E1318S_PHY_MAGIC_PACKET_WORD1                 0x18
-#define MII_88E1318S_PHY_MAGIC_PACKET_WORD0                 0x19
+#define MII_88E1318S_PHY_MAGIC_PACKET_WORD2		0x17
+#define MII_88E1318S_PHY_MAGIC_PACKET_WORD1		0x18
+#define MII_88E1318S_PHY_MAGIC_PACKET_WORD0		0x19
 
-#define MII_88E1318S_PHY_WOL_CTRL                           0x10
-#define MII_88E1318S_PHY_WOL_CTRL_CLEAR_WOL_STATUS          BIT(12)
-#define MII_88E1318S_PHY_WOL_CTRL_MAGIC_PACKET_MATCH_ENABLE BIT(14)
+#define MII_88E1318S_PHY_WOL_CTRL				0x10
+#define MII_88E1318S_PHY_WOL_CTRL_CLEAR_WOL_STATUS		BIT(12)
+#define MII_88E1318S_PHY_WOL_CTRL_MAGIC_PACKET_MATCH_ENABLE	BIT(14)
 
 #define MII_88E1121_PHY_LED_CTRL	16
 #define MII_88E1121_PHY_LED_DEF		0x0030
@@ -138,8 +130,6 @@
 #define MII_M1011_PHY_STATUS_RESOLVED	0x0800
 #define MII_M1011_PHY_STATUS_LINK	0x0400
 
-#define MII_M1116R_CONTROL_REG_MAC	21
-
 #define MII_88E3016_PHY_SPEC_CTRL	0x10
 #define MII_88E3016_DISABLE_SCRAMBLER	0x0200
 #define MII_88E3016_AUTO_MDIX_CROSSOVER	0x0030
@@ -152,7 +142,7 @@
 #define LPA_FIBER_1000HALF	0x40
 #define LPA_FIBER_1000FULL	0x20
 
-#define LPA_PAUSE_FIBER	0x180
+#define LPA_PAUSE_FIBER		0x180
 #define LPA_PAUSE_ASYM_FIBER	0x100
 
 #define ADVERTISE_FIBER_1000HALF	0x40
@@ -274,6 +264,23 @@ static int marvell_set_polarity(struct phy_device *phydev, int polarity)
 	return 0;
 }
 
+static int marvell_set_downshift(struct phy_device *phydev, bool enable,
+				 u8 retries)
+{
+	int reg;
+
+	reg = phy_read(phydev, MII_M1011_PHY_SCR);
+	if (reg < 0)
+		return reg;
+
+	reg &= MII_M1011_PHY_SRC_DOWNSHIFT_MASK;
+	reg |= ((retries - 1) << MII_M1011_PHY_SCR_DOWNSHIFT_SHIFT);
+	if (enable)
+		reg |= MII_M1011_PHY_SCR_DOWNSHIFT_EN;
+
+	return phy_write(phydev, MII_M1011_PHY_SCR, reg);
+}
+
 static int marvell_config_aneg(struct phy_device *phydev)
 {
 	int err;
@@ -292,17 +299,11 @@ static int marvell_config_aneg(struct phy_device *phydev)
 		return err;
 
 	if (phydev->autoneg != AUTONEG_ENABLE) {
-		int bmcr;
-
 		/* A write to speed/duplex bits (that is performed by
 		 * genphy_config_aneg() call above) must be followed by
 		 * a software reset. Otherwise, the write has no effect.
 		 */
-		bmcr = phy_read(phydev, MII_BMCR);
-		if (bmcr < 0)
-			return bmcr;
-
-		err = phy_write(phydev, MII_BMCR, bmcr | BMCR_RESET);
+		err = genphy_soft_reset(phydev);
 		if (err < 0)
 			return err;
 	}
@@ -318,8 +319,7 @@ static int m88e1101_config_aneg(struct phy_device *phydev)
 	 * that certain registers get written in order
 	 * to restart autonegotiation
 	 */
-	err = phy_write(phydev, MII_BMCR, BMCR_RESET);
-
+	err = genphy_soft_reset(phydev);
 	if (err < 0)
 		return err;
 
@@ -354,7 +354,7 @@ static int m88e1111_config_aneg(struct phy_device *phydev)
 	 * that certain registers get written in order
 	 * to restart autonegotiation
 	 */
-	err = phy_write(phydev, MII_BMCR, BMCR_RESET);
+	err = genphy_soft_reset(phydev);
 
 	err = marvell_set_polarity(phydev, phydev->mdix_ctrl);
 	if (err < 0)
@@ -370,17 +370,11 @@ static int m88e1111_config_aneg(struct phy_device *phydev)
 		return err;
 
 	if (phydev->autoneg != AUTONEG_ENABLE) {
-		int bmcr;
-
 		/* A write to speed/duplex bits (that is performed by
 		 * genphy_config_aneg() call above) must be followed by
 		 * a software reset. Otherwise, the write has no effect.
 		 */
-		bmcr = phy_read(phydev, MII_BMCR);
-		if (bmcr < 0)
-			return bmcr;
-
-		err = phy_write(phydev, MII_BMCR, bmcr | BMCR_RESET);
+		err = genphy_soft_reset(phydev);
 		if (err < 0)
 			return err;
 	}
@@ -466,7 +460,7 @@ static int marvell_of_reg_init(struct phy_device *phydev)
 }
 #endif /* CONFIG_OF_MDIO */
 
-static int m88e1121_config_aneg(struct phy_device *phydev)
+static int m88e1121_config_aneg_rgmii_delays(struct phy_device *phydev)
 {
 	int err, oldpage, mscr;
 
@@ -474,31 +468,45 @@ static int m88e1121_config_aneg(struct phy_device *phydev)
 	if (oldpage < 0)
 		return oldpage;
 
-	if (phy_interface_is_rgmii(phydev)) {
-		mscr = phy_read(phydev, MII_88E1121_PHY_MSCR_REG) &
-			MII_88E1121_PHY_MSCR_DELAY_MASK;
-
-		if (phydev->interface == PHY_INTERFACE_MODE_RGMII_ID)
-			mscr |= (MII_88E1121_PHY_MSCR_RX_DELAY |
-				 MII_88E1121_PHY_MSCR_TX_DELAY);
-		else if (phydev->interface == PHY_INTERFACE_MODE_RGMII_RXID)
-			mscr |= MII_88E1121_PHY_MSCR_RX_DELAY;
-		else if (phydev->interface == PHY_INTERFACE_MODE_RGMII_TXID)
-			mscr |= MII_88E1121_PHY_MSCR_TX_DELAY;
-
-		err = phy_write(phydev, MII_88E1121_PHY_MSCR_REG, mscr);
-		if (err < 0)
-			return err;
+	mscr = phy_read(phydev, MII_88E1121_PHY_MSCR_REG);
+	if (mscr < 0) {
+		err = mscr;
+		goto out;
 	}
 
+	mscr &= MII_88E1121_PHY_MSCR_DELAY_MASK;
+
+	if (phydev->interface == PHY_INTERFACE_MODE_RGMII_ID)
+		mscr |= (MII_88E1121_PHY_MSCR_RX_DELAY |
+			 MII_88E1121_PHY_MSCR_TX_DELAY);
+	else if (phydev->interface == PHY_INTERFACE_MODE_RGMII_RXID)
+		mscr |= MII_88E1121_PHY_MSCR_RX_DELAY;
+	else if (phydev->interface == PHY_INTERFACE_MODE_RGMII_TXID)
+		mscr |= MII_88E1121_PHY_MSCR_TX_DELAY;
+
+	err = phy_write(phydev, MII_88E1121_PHY_MSCR_REG, mscr);
+
+out:
 	marvell_set_page(phydev, oldpage);
 
-	err = phy_write(phydev, MII_BMCR, BMCR_RESET);
+	return err;
+}
+
+static int m88e1121_config_aneg(struct phy_device *phydev)
+{
+	int err = 0;
+
+	if (phy_interface_is_rgmii(phydev)) {
+		err = m88e1121_config_aneg_rgmii_delays(phydev);
+		if (err)
+			return err;
+	}
+
+	err = genphy_soft_reset(phydev);
 	if (err < 0)
 		return err;
 
-	err = phy_write(phydev, MII_M1011_PHY_SCR,
-			MII_M1011_PHY_SCR_AUTO_CROSS);
+	err = marvell_set_polarity(phydev, phydev->mdix_ctrl);
 	if (err < 0)
 		return err;
 
@@ -596,7 +604,7 @@ static int marvell_config_aneg_fiber(struct phy_device *phydev)
 
 	if (changed == 0) {
 		/* Advertisement hasn't changed, but maybe aneg was never on to
-		 * begin with?  Or maybe phy was isolated?
+		 * begin with?	Or maybe phy was isolated?
 		 */
 		int ctl = phy_read(phydev, MII_BMCR);
 
@@ -653,12 +661,9 @@ static int marvell_config_init(struct phy_device *phydev)
 
 static int m88e1116r_config_init(struct phy_device *phydev)
 {
-	int temp;
 	int err;
 
-	temp = phy_read(phydev, MII_BMCR);
-	temp |= BMCR_RESET;
-	err = phy_write(phydev, MII_BMCR, temp);
+	err = genphy_soft_reset(phydev);
 	if (err < 0)
 		return err;
 
@@ -668,35 +673,22 @@ static int m88e1116r_config_init(struct phy_device *phydev)
 	if (err < 0)
 		return err;
 
-	temp = phy_read(phydev, MII_M1011_PHY_SCR);
-	temp |= (7 << 12);	/* max number of gigabit attempts */
-	temp |= (1 << 11);	/* enable downshift */
-	temp |= MII_M1011_PHY_SCR_AUTO_CROSS;
-	err = phy_write(phydev, MII_M1011_PHY_SCR, temp);
+	err = marvell_set_polarity(phydev, phydev->mdix_ctrl);
 	if (err < 0)
 		return err;
 
-	err = marvell_set_page(phydev, MII_MARVELL_MSCR_PAGE);
-	if (err < 0)
-		return err;
-	temp = phy_read(phydev, MII_M1116R_CONTROL_REG_MAC);
-	temp |= (1 << 5);
-	temp |= (1 << 4);
-	err = phy_write(phydev, MII_M1116R_CONTROL_REG_MAC, temp);
+	err = marvell_set_downshift(phydev, true, 8);
 	if (err < 0)
 		return err;
-	err = marvell_set_page(phydev, MII_MARVELL_COPPER_PAGE);
+
+	err = m88e1121_config_aneg_rgmii_delays(phydev);
 	if (err < 0)
 		return err;
 
-	temp = phy_read(phydev, MII_BMCR);
-	temp |= BMCR_RESET;
-	err = phy_write(phydev, MII_BMCR, temp);
+	err = genphy_soft_reset(phydev);
 	if (err < 0)
 		return err;
 
-	mdelay(500);
-
 	return marvell_config_init(phydev);
 }
 
@@ -719,9 +711,29 @@ static int m88e3016_config_init(struct phy_device *phydev)
 	return marvell_config_init(phydev);
 }
 
-static int m88e1111_config_init_rgmii(struct phy_device *phydev)
+static int m88e1111_config_init_hwcfg_mode(struct phy_device *phydev,
+					   u16 mode,
+					   int fibre_copper_auto)
+{
+	int temp;
+
+	temp = phy_read(phydev, MII_M1111_PHY_EXT_SR);
+	if (temp < 0)
+		return temp;
+
+	temp &= ~(MII_M1111_HWCFG_MODE_MASK |
+		  MII_M1111_HWCFG_FIBER_COPPER_AUTO |
+		  MII_M1111_HWCFG_FIBER_COPPER_RES);
+	temp |= mode;
+
+	if (fibre_copper_auto)
+		temp |= MII_M1111_HWCFG_FIBER_COPPER_AUTO;
+
+	return phy_write(phydev, MII_M1111_PHY_EXT_SR, temp);
+}
+
+static int m88e1111_config_init_rgmii_delays(struct phy_device *phydev)
 {
-	int err;
 	int temp;
 
 	temp = phy_read(phydev, MII_M1111_PHY_EXT_CR);
@@ -729,16 +741,24 @@ static int m88e1111_config_init_rgmii(struct phy_device *phydev)
 		return temp;
 
 	if (phydev->interface == PHY_INTERFACE_MODE_RGMII_ID) {
-		temp |= (MII_M1111_RX_DELAY | MII_M1111_TX_DELAY);
+		temp |= (MII_M1111_RGMII_RX_DELAY | MII_M1111_RGMII_TX_DELAY);
 	} else if (phydev->interface == PHY_INTERFACE_MODE_RGMII_RXID) {
-		temp &= ~MII_M1111_TX_DELAY;
-		temp |= MII_M1111_RX_DELAY;
+		temp &= ~MII_M1111_RGMII_TX_DELAY;
+		temp |= MII_M1111_RGMII_RX_DELAY;
 	} else if (phydev->interface == PHY_INTERFACE_MODE_RGMII_TXID) {
-		temp &= ~MII_M1111_RX_DELAY;
-		temp |= MII_M1111_TX_DELAY;
+		temp &= ~MII_M1111_RGMII_RX_DELAY;
+		temp |= MII_M1111_RGMII_TX_DELAY;
 	}
 
-	err = phy_write(phydev, MII_M1111_PHY_EXT_CR, temp);
+	return phy_write(phydev, MII_M1111_PHY_EXT_CR, temp);
+}
+
+static int m88e1111_config_init_rgmii(struct phy_device *phydev)
+{
+	int temp;
+	int err;
+
+	err = m88e1111_config_init_rgmii_delays(phydev);
 	if (err < 0)
 		return err;
 
@@ -759,17 +779,11 @@ static int m88e1111_config_init_rgmii(struct phy_device *phydev)
 static int m88e1111_config_init_sgmii(struct phy_device *phydev)
 {
 	int err;
-	int temp;
-
-	temp = phy_read(phydev, MII_M1111_PHY_EXT_SR);
-	if (temp < 0)
-		return temp;
-
-	temp &= ~(MII_M1111_HWCFG_MODE_MASK);
-	temp |= MII_M1111_HWCFG_MODE_SGMII_NO_CLK;
-	temp |= MII_M1111_HWCFG_FIBER_COPPER_AUTO;
 
-	err = phy_write(phydev, MII_M1111_PHY_EXT_SR, temp);
+	err = m88e1111_config_init_hwcfg_mode(
+		phydev,
+		MII_M1111_HWCFG_MODE_SGMII_NO_CLK,
+		MII_M1111_HWCFG_FIBER_COPPER_AUTO);
 	if (err < 0)
 		return err;
 
@@ -780,48 +794,27 @@ static int m88e1111_config_init_sgmii(struct phy_device *phydev)
 static int m88e1111_config_init_rtbi(struct phy_device *phydev)
 {
 	int err;
-	int temp;
 
-	temp = phy_read(phydev, MII_M1111_PHY_EXT_CR);
-	if (temp < 0)
-		return temp;
-
-	temp |= (MII_M1111_RX_DELAY | MII_M1111_TX_DELAY);
-	err = phy_write(phydev, MII_M1111_PHY_EXT_CR, temp);
-	if (err < 0)
+	err = m88e1111_config_init_rgmii_delays(phydev);
+	if (err)
 		return err;
 
-	temp = phy_read(phydev, MII_M1111_PHY_EXT_SR);
-	if (temp < 0)
-		return temp;
-
-	temp &= ~(MII_M1111_HWCFG_MODE_MASK |
-		  MII_M1111_HWCFG_FIBER_COPPER_RES);
-	temp |= 0x7 | MII_M1111_HWCFG_FIBER_COPPER_AUTO;
-
-	err = phy_write(phydev, MII_M1111_PHY_EXT_SR, temp);
+	err = m88e1111_config_init_hwcfg_mode(
+		phydev,
+		MII_M1111_HWCFG_MODE_RTBI,
+		MII_M1111_HWCFG_FIBER_COPPER_AUTO);
 	if (err < 0)
 		return err;
 
 	/* soft reset */
-	err = phy_write(phydev, MII_BMCR, BMCR_RESET);
+	err = genphy_soft_reset(phydev);
 	if (err < 0)
 		return err;
 
-	do
-		temp = phy_read(phydev, MII_BMCR);
-	while (temp & BMCR_RESET);
-
-	temp = phy_read(phydev, MII_M1111_PHY_EXT_SR);
-	if (temp < 0)
-		return temp;
-
-	temp &= ~(MII_M1111_HWCFG_MODE_MASK |
-		  MII_M1111_HWCFG_FIBER_COPPER_RES);
-	temp |= MII_M1111_HWCFG_MODE_COPPER_RTBI |
-		MII_M1111_HWCFG_FIBER_COPPER_AUTO;
-
-	return phy_write(phydev, MII_M1111_PHY_EXT_SR, temp);
+	return m88e1111_config_init_hwcfg_mode(
+		phydev,
+		MII_M1111_HWCFG_MODE_RTBI,
+		MII_M1111_HWCFG_FIBER_COPPER_AUTO);
 }
 
 static int m88e1111_config_init(struct phy_device *phydev)
@@ -850,7 +843,7 @@ static int m88e1111_config_init(struct phy_device *phydev)
 	if (err < 0)
 		return err;
 
-	return phy_write(phydev, MII_BMCR, BMCR_RESET);
+	return genphy_soft_reset(phydev);
 }
 
 static int m88e1121_config_init(struct phy_device *phydev)
@@ -912,12 +905,11 @@ static int m88e1118_config_aneg(struct phy_device *phydev)
 {
 	int err;
 
-	err = phy_write(phydev, MII_BMCR, BMCR_RESET);
+	err = genphy_soft_reset(phydev);
 	if (err < 0)
 		return err;
 
-	err = phy_write(phydev, MII_M1011_PHY_SCR,
-			MII_M1011_PHY_SCR_AUTO_CROSS);
+	err = marvell_set_polarity(phydev, phydev->mdix_ctrl);
 	if (err < 0)
 		return err;
 
@@ -961,7 +953,7 @@ static int m88e1118_config_init(struct phy_device *phydev)
 	if (err < 0)
 		return err;
 
-	return phy_write(phydev, MII_BMCR, BMCR_RESET);
+	return genphy_soft_reset(phydev);
 }
 
 static int m88e1149_config_init(struct phy_device *phydev)
@@ -987,20 +979,15 @@ static int m88e1149_config_init(struct phy_device *phydev)
 	if (err < 0)
 		return err;
 
-	return phy_write(phydev, MII_BMCR, BMCR_RESET);
+	return genphy_soft_reset(phydev);
 }
 
 static int m88e1145_config_init_rgmii(struct phy_device *phydev)
 {
+	int temp;
 	int err;
-	int temp = phy_read(phydev, MII_M1145_PHY_EXT_CR);
-
-	if (temp < 0)
-		return temp;
 
-	temp |= (MII_M1145_RGMII_RX_DELAY | MII_M1145_RGMII_TX_DELAY);
-
-	err = phy_write(phydev, MII_M1145_PHY_EXT_CR, temp);
+	err = m88e1111_config_init_rgmii_delays(phydev);
 	if (err < 0)
 		return err;
 
@@ -1032,16 +1019,9 @@ static int m88e1145_config_init_rgmii(struct phy_device *phydev)
 
 static int m88e1145_config_init_sgmii(struct phy_device *phydev)
 {
-	int temp = phy_read(phydev, MII_M1145_PHY_EXT_SR);
-
-	if (temp < 0)
-		return temp;
-
-	temp &= ~MII_M1145_HWCFG_MODE_MASK;
-	temp |= MII_M1145_HWCFG_MODE_SGMII_NO_CLK;
-	temp |= MII_M1145_HWCFG_FIBER_COPPER_AUTO;
-
-	return phy_write(phydev, MII_M1145_PHY_EXT_SR, temp);
+	return m88e1111_config_init_hwcfg_mode(
+		phydev, MII_M1111_HWCFG_MODE_SGMII_NO_CLK,
+		MII_M1111_HWCFG_FIBER_COPPER_AUTO);
 }
 
 static int m88e1145_config_init(struct phy_device *phydev)
@@ -1515,7 +1495,7 @@ static void marvell_get_strings(struct phy_device *phydev, u8 *data)
 }
 
 #ifndef UINT64_MAX
-#define UINT64_MAX              (u64)(~((u64)0))
+#define UINT64_MAX		(u64)(~((u64)0))
 #endif
 static u64 marvell_get_stat(struct phy_device *phydev, int i)
 {
diff --git a/drivers/net/phy/mdio-bcm-unimac.c b/drivers/net/phy/mdio-bcm-unimac.c
index 34395230ce70..08e0647b85e2 100644
--- a/drivers/net/phy/mdio-bcm-unimac.c
+++ b/drivers/net/phy/mdio-bcm-unimac.c
@@ -21,6 +21,8 @@
 #include <linux/of_platform.h>
 #include <linux/of_mdio.h>
 
+#include <linux/platform_data/mdio-bcm-unimac.h>
+
 #define MDIO_CMD		0x00
 #define  MDIO_START_BUSY	(1 << 29)
 #define  MDIO_READ_FAIL		(1 << 28)
@@ -41,46 +43,80 @@
 struct unimac_mdio_priv {
 	struct mii_bus		*mii_bus;
 	void __iomem		*base;
+	int (*wait_func)	(void *wait_func_data);
+	void			*wait_func_data;
 };
 
+static inline u32 unimac_mdio_readl(struct unimac_mdio_priv *priv, u32 offset)
+{
+	/* MIPS chips strapped for BE will automagically configure the
+	 * peripheral registers for CPU-native byte order.
+	 */
+	if (IS_ENABLED(CONFIG_MIPS) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
+		return __raw_readl(priv->base + offset);
+	else
+		return readl_relaxed(priv->base + offset);
+}
+
+static inline void unimac_mdio_writel(struct unimac_mdio_priv *priv, u32 val,
+				      u32 offset)
+{
+	if (IS_ENABLED(CONFIG_MIPS) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
+		__raw_writel(val, priv->base + offset);
+	else
+		writel_relaxed(val, priv->base + offset);
+}
+
 static inline void unimac_mdio_start(struct unimac_mdio_priv *priv)
 {
 	u32 reg;
 
-	reg = __raw_readl(priv->base + MDIO_CMD);
+	reg = unimac_mdio_readl(priv, MDIO_CMD);
 	reg |= MDIO_START_BUSY;
-	__raw_writel(reg, priv->base + MDIO_CMD);
+	unimac_mdio_writel(priv, reg, MDIO_CMD);
 }
 
 static inline unsigned int unimac_mdio_busy(struct unimac_mdio_priv *priv)
 {
-	return __raw_readl(priv->base + MDIO_CMD) & MDIO_START_BUSY;
+	return unimac_mdio_readl(priv, MDIO_CMD) & MDIO_START_BUSY;
+}
+
+static int unimac_mdio_poll(void *wait_func_data)
+{
+	struct unimac_mdio_priv *priv = wait_func_data;
+	unsigned int timeout = 1000;
+
+	do {
+		if (!unimac_mdio_busy(priv))
+			return 0;
+
+		usleep_range(1000, 2000);
+	} while (--timeout);
+
+	if (!timeout)
+		return -ETIMEDOUT;
+
+	return 0;
 }
 
 static int unimac_mdio_read(struct mii_bus *bus, int phy_id, int reg)
 {
 	struct unimac_mdio_priv *priv = bus->priv;
-	unsigned int timeout = 1000;
+	int ret;
 	u32 cmd;
 
 	/* Prepare the read operation */
 	cmd = MDIO_RD | (phy_id << MDIO_PMD_SHIFT) | (reg << MDIO_REG_SHIFT);
-	__raw_writel(cmd, priv->base + MDIO_CMD);
+	unimac_mdio_writel(priv, cmd, MDIO_CMD);
 
 	/* Start MDIO transaction */
 	unimac_mdio_start(priv);
 
-	do {
-		if (!unimac_mdio_busy(priv))
-			break;
+	ret = priv->wait_func(priv->wait_func_data);
+	if (ret)
+		return ret;
 
-		usleep_range(1000, 2000);
-	} while (timeout--);
-
-	if (!timeout)
-		return -ETIMEDOUT;
-
-	cmd = __raw_readl(priv->base + MDIO_CMD);
+	cmd = unimac_mdio_readl(priv, MDIO_CMD);
 
 	/* Some broken devices are known not to release the line during
 	 * turn-around, e.g: Broadcom BCM53125 external switches, so check for
@@ -97,27 +133,16 @@ static int unimac_mdio_write(struct mii_bus *bus, int phy_id,
 			     int reg, u16 val)
 {
 	struct unimac_mdio_priv *priv = bus->priv;
-	unsigned int timeout = 1000;
 	u32 cmd;
 
 	/* Prepare the write operation */
 	cmd = MDIO_WR | (phy_id << MDIO_PMD_SHIFT) |
 		(reg << MDIO_REG_SHIFT) | (0xffff & val);
-	__raw_writel(cmd, priv->base + MDIO_CMD);
+	unimac_mdio_writel(priv, cmd, MDIO_CMD);
 
 	unimac_mdio_start(priv);
 
-	do {
-		if (!unimac_mdio_busy(priv))
-			break;
-
-		usleep_range(1000, 2000);
-	} while (timeout--);
-
-	if (!timeout)
-		return -ETIMEDOUT;
-
-	return 0;
+	return priv->wait_func(priv->wait_func_data);
 }
 
 /* Workaround for integrated BCM7xxx Gigabit PHYs which have a problem with
@@ -155,8 +180,10 @@ static int unimac_mdio_reset(struct mii_bus *bus)
 	}
 
 	for (addr = 0; addr < PHY_MAX_ADDR; addr++) {
-		if (read_mask & 1 << addr)
+		if (read_mask & 1 << addr) {
+			dev_dbg(&bus->dev, "Workaround for PHY @ %d\n", addr);
 			mdiobus_read(bus, addr, MII_BMSR);
+		}
 	}
 
 	return 0;
@@ -164,6 +191,7 @@ static int unimac_mdio_reset(struct mii_bus *bus)
 
 static int unimac_mdio_probe(struct platform_device *pdev)
 {
+	struct unimac_mdio_pdata *pdata = pdev->dev.platform_data;
 	struct unimac_mdio_priv *priv;
 	struct device_node *np;
 	struct mii_bus *bus;
@@ -193,12 +221,21 @@ static int unimac_mdio_probe(struct platform_device *pdev)
 
 	bus = priv->mii_bus;
 	bus->priv = priv;
-	bus->name = "unimac MII bus";
+	if (pdata) {
+		bus->name = pdata->bus_name;
+		priv->wait_func = pdata->wait_func;
+		priv->wait_func_data = pdata->wait_func_data;
+		bus->phy_mask = ~pdata->phy_mask;
+	} else {
+		bus->name = "unimac MII bus";
+		priv->wait_func_data = priv;
+		priv->wait_func = unimac_mdio_poll;
+	}
 	bus->parent = &pdev->dev;
 	bus->read = unimac_mdio_read;
 	bus->write = unimac_mdio_write;
 	bus->reset = unimac_mdio_reset;
-	snprintf(bus->id, MII_BUS_ID_SIZE, "%s", pdev->name);
+	snprintf(bus->id, MII_BUS_ID_SIZE, "%s-%d", pdev->name, pdev->id);
 
 	ret = of_mdiobus_register(bus, np);
 	if (ret) {
@@ -240,7 +277,7 @@ MODULE_DEVICE_TABLE(of, unimac_mdio_ids);
 
 static struct platform_driver unimac_mdio_driver = {
 	.driver = {
-		.name = "unimac-mdio",
+		.name = UNIMAC_MDIO_DRV_NAME,
 		.of_match_table = unimac_mdio_ids,
 	},
 	.probe	= unimac_mdio_probe,
@@ -251,4 +288,4 @@ module_platform_driver(unimac_mdio_driver);
 MODULE_AUTHOR("Broadcom Corporation");
 MODULE_DESCRIPTION("Broadcom UniMAC MDIO bus controller");
 MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:unimac-mdio");
+MODULE_ALIAS("platform:" UNIMAC_MDIO_DRV_NAME);
diff --git a/drivers/net/phy/mdio-gpio.c b/drivers/net/phy/mdio-gpio.c
index 7faa79b254ef..4333c6e14742 100644
--- a/drivers/net/phy/mdio-gpio.c
+++ b/drivers/net/phy/mdio-gpio.c
@@ -116,7 +116,7 @@ static void mdc_set(struct mdiobb_ctrl *ctrl, int what)
 	gpiod_set_value(bitbang->mdc, what);
 }
 
-static struct mdiobb_ops mdio_gpio_ops = {
+static const struct mdiobb_ops mdio_gpio_ops = {
 	.owner = THIS_MODULE,
 	.set_mdc = mdc_set,
 	.set_mdio_dir = mdio_dir,
diff --git a/drivers/net/phy/mdio-i2c.c b/drivers/net/phy/mdio-i2c.c
new file mode 100644
index 000000000000..6d24fd13ca86
--- /dev/null
+++ b/drivers/net/phy/mdio-i2c.c
@@ -0,0 +1,109 @@
+/*
+ * MDIO I2C bridge
+ *
+ * Copyright (C) 2015-2016 Russell King
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Network PHYs can appear on I2C buses when they are part of SFP module.
+ * This driver exposes these PHYs to the networking PHY code, allowing
+ * our PHY drivers access to these PHYs, and so allowing configuration
+ * of their settings.
+ */
+#include <linux/i2c.h>
+#include <linux/phy.h>
+
+#include "mdio-i2c.h"
+
+/*
+ * I2C bus addresses 0x50 and 0x51 are normally an EEPROM, which is
+ * specified to be present in SFP modules.  These correspond with PHY
+ * addresses 16 and 17.  Disallow access to these "phy" addresses.
+ */
+static bool i2c_mii_valid_phy_id(int phy_id)
+{
+	return phy_id != 0x10 && phy_id != 0x11;
+}
+
+static unsigned int i2c_mii_phy_addr(int phy_id)
+{
+	return phy_id + 0x40;
+}
+
+static int i2c_mii_read(struct mii_bus *bus, int phy_id, int reg)
+{
+	struct i2c_adapter *i2c = bus->priv;
+	struct i2c_msg msgs[2];
+	u8 data[2], dev_addr = reg;
+	int bus_addr, ret;
+
+	if (!i2c_mii_valid_phy_id(phy_id))
+		return 0xffff;
+
+	bus_addr = i2c_mii_phy_addr(phy_id);
+	msgs[0].addr = bus_addr;
+	msgs[0].flags = 0;
+	msgs[0].len = 1;
+	msgs[0].buf = &dev_addr;
+	msgs[1].addr = bus_addr;
+	msgs[1].flags = I2C_M_RD;
+	msgs[1].len = sizeof(data);
+	msgs[1].buf = data;
+
+	ret = i2c_transfer(i2c, msgs, ARRAY_SIZE(msgs));
+	if (ret != ARRAY_SIZE(msgs))
+		return 0xffff;
+
+	return data[0] << 8 | data[1];
+}
+
+static int i2c_mii_write(struct mii_bus *bus, int phy_id, int reg, u16 val)
+{
+	struct i2c_adapter *i2c = bus->priv;
+	struct i2c_msg msg;
+	int ret;
+	u8 data[3];
+
+	if (!i2c_mii_valid_phy_id(phy_id))
+		return 0;
+
+	data[0] = reg;
+	data[1] = val >> 8;
+	data[2] = val;
+
+	msg.addr = i2c_mii_phy_addr(phy_id);
+	msg.flags = 0;
+	msg.len = 3;
+	msg.buf = data;
+
+	ret = i2c_transfer(i2c, &msg, 1);
+
+	return ret < 0 ? ret : 0;
+}
+
+struct mii_bus *mdio_i2c_alloc(struct device *parent, struct i2c_adapter *i2c)
+{
+	struct mii_bus *mii;
+
+	if (!i2c_check_functionality(i2c, I2C_FUNC_I2C))
+		return ERR_PTR(-EINVAL);
+
+	mii = mdiobus_alloc();
+	if (!mii)
+		return ERR_PTR(-ENOMEM);
+
+	snprintf(mii->id, MII_BUS_ID_SIZE, "i2c:%s", dev_name(parent));
+	mii->parent = parent;
+	mii->read = i2c_mii_read;
+	mii->write = i2c_mii_write;
+	mii->priv = i2c;
+
+	return mii;
+}
+EXPORT_SYMBOL_GPL(mdio_i2c_alloc);
+
+MODULE_AUTHOR("Russell King");
+MODULE_DESCRIPTION("MDIO I2C bridge library");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/net/phy/mdio-i2c.h b/drivers/net/phy/mdio-i2c.h
new file mode 100644
index 000000000000..889ab57d7f3e
--- /dev/null
+++ b/drivers/net/phy/mdio-i2c.h
@@ -0,0 +1,19 @@
+/*
+ * MDIO I2C bridge
+ *
+ * Copyright (C) 2015 Russell King
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#ifndef MDIO_I2C_H
+#define MDIO_I2C_H
+
+struct device;
+struct i2c_adapter;
+struct mii_bus;
+
+struct mii_bus *mdio_i2c_alloc(struct device *parent, struct i2c_adapter *i2c);
+
+#endif
diff --git a/drivers/net/phy/mdio-mux-bcm-iproc.c b/drivers/net/phy/mdio-mux-bcm-iproc.c
index 0a5f62e0efcc..0831b7142df7 100644
--- a/drivers/net/phy/mdio-mux-bcm-iproc.c
+++ b/drivers/net/phy/mdio-mux-bcm-iproc.c
@@ -199,7 +199,7 @@ static int mdio_mux_iproc_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, md);
 
-	rc = mdio_mux_init(md->dev, mdio_mux_iproc_switch_fn,
+	rc = mdio_mux_init(md->dev, md->dev->of_node, mdio_mux_iproc_switch_fn,
 			   &md->mux_handle, md, md->mii_bus);
 	if (rc) {
 		dev_info(md->dev, "mdiomux initialization failed\n");
diff --git a/drivers/net/phy/mdio-mux-gpio.c b/drivers/net/phy/mdio-mux-gpio.c
index 919949960a10..082ffef0dec4 100644
--- a/drivers/net/phy/mdio-mux-gpio.c
+++ b/drivers/net/phy/mdio-mux-gpio.c
@@ -54,7 +54,7 @@ static int mdio_mux_gpio_probe(struct platform_device *pdev)
 	if (IS_ERR(s->gpios))
 		return PTR_ERR(s->gpios);
 
-	r = mdio_mux_init(&pdev->dev,
+	r = mdio_mux_init(&pdev->dev, pdev->dev.of_node,
 			  mdio_mux_gpio_switch_fn, &s->mux_handle, s, NULL);
 
 	if (r != 0) {
diff --git a/drivers/net/phy/mdio-mux-mmioreg.c b/drivers/net/phy/mdio-mux-mmioreg.c
index 6a33646bdf05..2573ab012f16 100644
--- a/drivers/net/phy/mdio-mux-mmioreg.c
+++ b/drivers/net/phy/mdio-mux-mmioreg.c
@@ -105,7 +105,7 @@ static int mdio_mux_mmioreg_probe(struct platform_device *pdev)
 	const __be32 *iprop;
 	int len, ret;
 
-	dev_dbg(&pdev->dev, "probing node %s\n", np->full_name);
+	dev_dbg(&pdev->dev, "probing node %pOF\n", np);
 
 	s = devm_kzalloc(&pdev->dev, sizeof(*s), GFP_KERNEL);
 	if (!s)
@@ -113,8 +113,8 @@ static int mdio_mux_mmioreg_probe(struct platform_device *pdev)
 
 	ret = of_address_to_resource(np, 0, &res);
 	if (ret) {
-		dev_err(&pdev->dev, "could not obtain memory map for node %s\n",
-			np->full_name);
+		dev_err(&pdev->dev, "could not obtain memory map for node %pOF\n",
+			np);
 		return ret;
 	}
 	s->phys = res.start;
@@ -145,25 +145,26 @@ static int mdio_mux_mmioreg_probe(struct platform_device *pdev)
 	for_each_available_child_of_node(np, np2) {
 		iprop = of_get_property(np2, "reg", &len);
 		if (!iprop || len != sizeof(uint32_t)) {
-			dev_err(&pdev->dev, "mdio-mux child node %s is "
-				"missing a 'reg' property\n", np2->full_name);
+			dev_err(&pdev->dev, "mdio-mux child node %pOF is "
+				"missing a 'reg' property\n", np2);
 			of_node_put(np2);
 			return -ENODEV;
 		}
 		if (be32_to_cpup(iprop) & ~s->mask) {
-			dev_err(&pdev->dev, "mdio-mux child node %s has "
+			dev_err(&pdev->dev, "mdio-mux child node %pOF has "
 				"a 'reg' value with unmasked bits\n",
-				np2->full_name);
+				np2);
 			of_node_put(np2);
 			return -ENODEV;
 		}
 	}
 
-	ret = mdio_mux_init(&pdev->dev, mdio_mux_mmioreg_switch_fn,
+	ret = mdio_mux_init(&pdev->dev, pdev->dev.of_node,
+			    mdio_mux_mmioreg_switch_fn,
 			    &s->mux_handle, s, NULL);
 	if (ret) {
-		dev_err(&pdev->dev, "failed to register mdio-mux bus %s\n",
-			np->full_name);
+		dev_err(&pdev->dev, "failed to register mdio-mux bus %pOF\n",
+			np);
 		return ret;
 	}
 
diff --git a/drivers/net/phy/mdio-mux.c b/drivers/net/phy/mdio-mux.c
index c608e1dfaf09..0a86f1e4c02f 100644
--- a/drivers/net/phy/mdio-mux.c
+++ b/drivers/net/phy/mdio-mux.c
@@ -13,7 +13,6 @@
 #include <linux/module.h>
 #include <linux/phy.h>
 
-#define DRV_VERSION "1.0"
 #define DRV_DESCRIPTION "MDIO bus multiplexer driver"
 
 struct mdio_mux_child_bus;
@@ -87,6 +86,7 @@ out:
 static int parent_count;
 
 int mdio_mux_init(struct device *dev,
+		  struct device_node *mux_node,
 		  int (*switch_fn)(int cur, int desired, void *data),
 		  void **mux_handle,
 		  void *data,
@@ -99,11 +99,11 @@ int mdio_mux_init(struct device *dev,
 	struct mdio_mux_parent_bus *pb;
 	struct mdio_mux_child_bus *cb;
 
-	if (!dev->of_node)
+	if (!mux_node)
 		return -ENODEV;
 
 	if (!mux_bus) {
-		parent_bus_node = of_parse_phandle(dev->of_node,
+		parent_bus_node = of_parse_phandle(mux_node,
 						   "mdio-parent-bus", 0);
 
 		if (!parent_bus_node)
@@ -117,10 +117,11 @@ int mdio_mux_init(struct device *dev,
 	} else {
 		parent_bus_node = NULL;
 		parent_bus = mux_bus;
+		get_device(&parent_bus->dev);
 	}
 
 	pb = devm_kzalloc(dev, sizeof(*pb), GFP_KERNEL);
-	if (pb == NULL) {
+	if (!pb) {
 		ret_val = -ENOMEM;
 		goto err_pb_kz;
 	}
@@ -132,22 +133,19 @@ int mdio_mux_init(struct device *dev,
 	pb->mii_bus = parent_bus;
 
 	ret_val = -ENODEV;
-	for_each_available_child_of_node(dev->of_node, child_bus_node) {
+	for_each_available_child_of_node(mux_node, child_bus_node) {
 		int v;
 
 		r = of_property_read_u32(child_bus_node, "reg", &v);
 		if (r) {
 			dev_err(dev,
-				"Error: Failed to find reg for child %s\n",
-				of_node_full_name(child_bus_node));
+				"Error: Failed to find reg for child %pOF\n",
+				child_bus_node);
 			continue;
 		}
 
 		cb = devm_kzalloc(dev, sizeof(*cb), GFP_KERNEL);
-		if (cb == NULL) {
-			dev_err(dev,
-				"Error: Failed to allocate memory for child %s\n",
-				of_node_full_name(child_bus_node));
+		if (!cb) {
 			ret_val = -ENOMEM;
 			continue;
 		}
@@ -156,9 +154,6 @@ int mdio_mux_init(struct device *dev,
 
 		cb->mii_bus = mdiobus_alloc();
 		if (!cb->mii_bus) {
-			dev_err(dev,
-				"Error: Failed to allocate MDIO bus for child %s\n",
-				of_node_full_name(child_bus_node));
 			ret_val = -ENOMEM;
 			devm_kfree(dev, cb);
 			continue;
@@ -174,8 +169,8 @@ int mdio_mux_init(struct device *dev,
 		r = of_mdiobus_register(cb->mii_bus, child_bus_node);
 		if (r) {
 			dev_err(dev,
-				"Error: Failed to register MDIO bus for child %s\n",
-				of_node_full_name(child_bus_node));
+				"Error: Failed to register MDIO bus for child %pOF\n",
+				child_bus_node);
 			mdiobus_free(cb->mii_bus);
 			devm_kfree(dev, cb);
 		} else {
@@ -185,16 +180,13 @@ int mdio_mux_init(struct device *dev,
 	}
 	if (pb->children) {
 		*mux_handle = pb;
-		dev_info(dev, "Version " DRV_VERSION "\n");
 		return 0;
 	}
 
 	dev_err(dev, "Error: No acceptable child buses found\n");
 	devm_kfree(dev, pb);
 err_pb_kz:
-	/* balance the reference of_mdio_find_bus() took */
-	if (!mux_bus)
-		put_device(&parent_bus->dev);
+	put_device(&parent_bus->dev);
 err_parent_bus:
 	of_node_put(parent_bus_node);
 	return ret_val;
@@ -212,12 +204,10 @@ void mdio_mux_uninit(void *mux_handle)
 		cb = cb->next;
 	}
 
-	/* balance the reference of_mdio_find_bus() in mdio_mux_init() took */
 	put_device(&pb->mii_bus->dev);
 }
 EXPORT_SYMBOL_GPL(mdio_mux_uninit);
 
 MODULE_DESCRIPTION(DRV_DESCRIPTION);
-MODULE_VERSION(DRV_VERSION);
 MODULE_AUTHOR("David Daney");
 MODULE_LICENSE("GPL");
diff --git a/drivers/net/phy/phy-core.c b/drivers/net/phy/phy-core.c
index 6739b738bbaf..21f75ae244b3 100644
--- a/drivers/net/phy/phy-core.c
+++ b/drivers/net/phy/phy-core.c
@@ -9,6 +9,186 @@
 #include <linux/export.h>
 #include <linux/phy.h>
 
+const char *phy_speed_to_str(int speed)
+{
+	switch (speed) {
+	case SPEED_10:
+		return "10Mbps";
+	case SPEED_100:
+		return "100Mbps";
+	case SPEED_1000:
+		return "1Gbps";
+	case SPEED_2500:
+		return "2.5Gbps";
+	case SPEED_5000:
+		return "5Gbps";
+	case SPEED_10000:
+		return "10Gbps";
+	case SPEED_14000:
+		return "14Gbps";
+	case SPEED_20000:
+		return "20Gbps";
+	case SPEED_25000:
+		return "25Gbps";
+	case SPEED_40000:
+		return "40Gbps";
+	case SPEED_50000:
+		return "50Gbps";
+	case SPEED_56000:
+		return "56Gbps";
+	case SPEED_100000:
+		return "100Gbps";
+	case SPEED_UNKNOWN:
+		return "Unknown";
+	default:
+		return "Unsupported (update phy-core.c)";
+	}
+}
+EXPORT_SYMBOL_GPL(phy_speed_to_str);
+
+const char *phy_duplex_to_str(unsigned int duplex)
+{
+	if (duplex == DUPLEX_HALF)
+		return "Half";
+	if (duplex == DUPLEX_FULL)
+		return "Full";
+	if (duplex == DUPLEX_UNKNOWN)
+		return "Unknown";
+	return "Unsupported (update phy-core.c)";
+}
+EXPORT_SYMBOL_GPL(phy_duplex_to_str);
+
+/* A mapping of all SUPPORTED settings to speed/duplex.  This table
+ * must be grouped by speed and sorted in descending match priority
+ * - iow, descending speed. */
+static const struct phy_setting settings[] = {
+	{
+		.speed = SPEED_10000,
+		.duplex = DUPLEX_FULL,
+		.bit = ETHTOOL_LINK_MODE_10000baseKR_Full_BIT,
+	},
+	{
+		.speed = SPEED_10000,
+		.duplex = DUPLEX_FULL,
+		.bit = ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT,
+	},
+	{
+		.speed = SPEED_10000,
+		.duplex = DUPLEX_FULL,
+		.bit = ETHTOOL_LINK_MODE_10000baseT_Full_BIT,
+	},
+	{
+		.speed = SPEED_2500,
+		.duplex = DUPLEX_FULL,
+		.bit = ETHTOOL_LINK_MODE_2500baseX_Full_BIT,
+	},
+	{
+		.speed = SPEED_1000,
+		.duplex = DUPLEX_FULL,
+		.bit = ETHTOOL_LINK_MODE_1000baseKX_Full_BIT,
+	},
+	{
+		.speed = SPEED_1000,
+		.duplex = DUPLEX_FULL,
+		.bit = ETHTOOL_LINK_MODE_1000baseX_Full_BIT,
+	},
+	{
+		.speed = SPEED_1000,
+		.duplex = DUPLEX_FULL,
+		.bit = ETHTOOL_LINK_MODE_1000baseT_Full_BIT,
+	},
+	{
+		.speed = SPEED_1000,
+		.duplex = DUPLEX_HALF,
+		.bit = ETHTOOL_LINK_MODE_1000baseT_Half_BIT,
+	},
+	{
+		.speed = SPEED_100,
+		.duplex = DUPLEX_FULL,
+		.bit = ETHTOOL_LINK_MODE_100baseT_Full_BIT,
+	},
+	{
+		.speed = SPEED_100,
+		.duplex = DUPLEX_HALF,
+		.bit = ETHTOOL_LINK_MODE_100baseT_Half_BIT,
+	},
+	{
+		.speed = SPEED_10,
+		.duplex = DUPLEX_FULL,
+		.bit = ETHTOOL_LINK_MODE_10baseT_Full_BIT,
+	},
+	{
+		.speed = SPEED_10,
+		.duplex = DUPLEX_HALF,
+		.bit = ETHTOOL_LINK_MODE_10baseT_Half_BIT,
+	},
+};
+
+/**
+ * phy_lookup_setting - lookup a PHY setting
+ * @speed: speed to match
+ * @duplex: duplex to match
+ * @mask: allowed link modes
+ * @maxbit: bit size of link modes
+ * @exact: an exact match is required
+ *
+ * Search the settings array for a setting that matches the speed and
+ * duplex, and which is supported.
+ *
+ * If @exact is unset, either an exact match or %NULL for no match will
+ * be returned.
+ *
+ * If @exact is set, an exact match, the fastest supported setting at
+ * or below the specified speed, the slowest supported setting, or if
+ * they all fail, %NULL will be returned.
+ */
+const struct phy_setting *
+phy_lookup_setting(int speed, int duplex, const unsigned long *mask,
+		   size_t maxbit, bool exact)
+{
+	const struct phy_setting *p, *match = NULL, *last = NULL;
+	int i;
+
+	for (i = 0, p = settings; i < ARRAY_SIZE(settings); i++, p++) {
+		if (p->bit < maxbit && test_bit(p->bit, mask)) {
+			last = p;
+			if (p->speed == speed && p->duplex == duplex) {
+				/* Exact match for speed and duplex */
+				match = p;
+				break;
+			} else if (!exact) {
+				if (!match && p->speed <= speed)
+					/* Candidate */
+					match = p;
+
+				if (p->speed < speed)
+					break;
+			}
+		}
+	}
+
+	if (!match && !exact)
+		match = last;
+
+	return match;
+}
+EXPORT_SYMBOL_GPL(phy_lookup_setting);
+
+size_t phy_speeds(unsigned int *speeds, size_t size,
+		  unsigned long *mask, size_t maxbit)
+{
+	size_t count;
+	int i;
+
+	for (i = 0, count = 0; i < ARRAY_SIZE(settings) && count < size; i++)
+		if (settings[i].bit < maxbit &&
+		    test_bit(settings[i].bit, mask) &&
+		    (count == 0 || speeds[count - 1] != settings[i].speed))
+			speeds[count++] = settings[i].speed;
+
+	return count;
+}
+
 static void mmd_phy_indirect(struct mii_bus *bus, int phy_addr, int devad,
 			     u16 regnum)
 {
diff --git a/drivers/net/phy/phy.c b/drivers/net/phy/phy.c
index 5068c582d502..e842d2cd1ee7 100644
--- a/drivers/net/phy/phy.c
+++ b/drivers/net/phy/phy.c
@@ -30,7 +30,6 @@
 #include <linux/ethtool.h>
 #include <linux/phy.h>
 #include <linux/phy_led_triggers.h>
-#include <linux/timer.h>
 #include <linux/workqueue.h>
 #include <linux/mdio.h>
 #include <linux/io.h>
@@ -39,42 +38,6 @@
 
 #include <asm/irq.h>
 
-static const char *phy_speed_to_str(int speed)
-{
-	switch (speed) {
-	case SPEED_10:
-		return "10Mbps";
-	case SPEED_100:
-		return "100Mbps";
-	case SPEED_1000:
-		return "1Gbps";
-	case SPEED_2500:
-		return "2.5Gbps";
-	case SPEED_5000:
-		return "5Gbps";
-	case SPEED_10000:
-		return "10Gbps";
-	case SPEED_14000:
-		return "14Gbps";
-	case SPEED_20000:
-		return "20Gbps";
-	case SPEED_25000:
-		return "25Gbps";
-	case SPEED_40000:
-		return "40Gbps";
-	case SPEED_50000:
-		return "50Gbps";
-	case SPEED_56000:
-		return "56Gbps";
-	case SPEED_100000:
-		return "100Gbps";
-	case SPEED_UNKNOWN:
-		return "Unknown";
-	default:
-		return "Unsupported (update phy.c)";
-	}
-}
-
 #define PHY_STATE_STR(_state)			\
 	case PHY_##_state:			\
 		return __stringify(_state);	\
@@ -110,7 +73,7 @@ void phy_print_status(struct phy_device *phydev)
 		netdev_info(phydev->attached_dev,
 			"Link is Up - %s/%s - flow control %s\n",
 			phy_speed_to_str(phydev->speed),
-			DUPLEX_FULL == phydev->duplex ? "Full" : "Half",
+			phy_duplex_to_str(phydev->duplex),
 			phydev->pause ? "rx/tx" : "off");
 	} else	{
 		netdev_info(phydev->attached_dev, "Link is Down\n");
@@ -194,123 +157,6 @@ int phy_aneg_done(struct phy_device *phydev)
 }
 EXPORT_SYMBOL(phy_aneg_done);
 
-/* A structure for mapping a particular speed and duplex
- * combination to a particular SUPPORTED and ADVERTISED value
- */
-struct phy_setting {
-	int speed;
-	int duplex;
-	u32 setting;
-};
-
-/* A mapping of all SUPPORTED settings to speed/duplex.  This table
- * must be grouped by speed and sorted in descending match priority
- * - iow, descending speed. */
-static const struct phy_setting settings[] = {
-	{
-		.speed = SPEED_10000,
-		.duplex = DUPLEX_FULL,
-		.setting = SUPPORTED_10000baseKR_Full,
-	},
-	{
-		.speed = SPEED_10000,
-		.duplex = DUPLEX_FULL,
-		.setting = SUPPORTED_10000baseKX4_Full,
-	},
-	{
-		.speed = SPEED_10000,
-		.duplex = DUPLEX_FULL,
-		.setting = SUPPORTED_10000baseT_Full,
-	},
-	{
-		.speed = SPEED_2500,
-		.duplex = DUPLEX_FULL,
-		.setting = SUPPORTED_2500baseX_Full,
-	},
-	{
-		.speed = SPEED_1000,
-		.duplex = DUPLEX_FULL,
-		.setting = SUPPORTED_1000baseKX_Full,
-	},
-	{
-		.speed = SPEED_1000,
-		.duplex = DUPLEX_FULL,
-		.setting = SUPPORTED_1000baseT_Full,
-	},
-	{
-		.speed = SPEED_1000,
-		.duplex = DUPLEX_HALF,
-		.setting = SUPPORTED_1000baseT_Half,
-	},
-	{
-		.speed = SPEED_100,
-		.duplex = DUPLEX_FULL,
-		.setting = SUPPORTED_100baseT_Full,
-	},
-	{
-		.speed = SPEED_100,
-		.duplex = DUPLEX_HALF,
-		.setting = SUPPORTED_100baseT_Half,
-	},
-	{
-		.speed = SPEED_10,
-		.duplex = DUPLEX_FULL,
-		.setting = SUPPORTED_10baseT_Full,
-	},
-	{
-		.speed = SPEED_10,
-		.duplex = DUPLEX_HALF,
-		.setting = SUPPORTED_10baseT_Half,
-	},
-};
-
-/**
- * phy_lookup_setting - lookup a PHY setting
- * @speed: speed to match
- * @duplex: duplex to match
- * @features: allowed link modes
- * @exact: an exact match is required
- *
- * Search the settings array for a setting that matches the speed and
- * duplex, and which is supported.
- *
- * If @exact is unset, either an exact match or %NULL for no match will
- * be returned.
- *
- * If @exact is set, an exact match, the fastest supported setting at
- * or below the specified speed, the slowest supported setting, or if
- * they all fail, %NULL will be returned.
- */
-static const struct phy_setting *
-phy_lookup_setting(int speed, int duplex, u32 features, bool exact)
-{
-	const struct phy_setting *p, *match = NULL, *last = NULL;
-	int i;
-
-	for (i = 0, p = settings; i < ARRAY_SIZE(settings); i++, p++) {
-		if (p->setting & features) {
-			last = p;
-			if (p->speed == speed && p->duplex == duplex) {
-				/* Exact match for speed and duplex */
-				match = p;
-				break;
-			} else if (!exact) {
-				if (!match && p->speed <= speed)
-					/* Candidate */
-					match = p;
-
-				if (p->speed < speed)
-					break;
-			}
-		}
-	}
-
-	if (!match && !exact)
-		match = last;
-
-	return match;
-}
-
 /**
  * phy_find_valid - find a PHY setting that matches the requested parameters
  * @speed: desired speed
@@ -327,7 +173,9 @@ phy_lookup_setting(int speed, int duplex, u32 features, bool exact)
 static const struct phy_setting *
 phy_find_valid(int speed, int duplex, u32 supported)
 {
-	return phy_lookup_setting(speed, duplex, supported, false);
+	unsigned long mask = supported;
+
+	return phy_lookup_setting(speed, duplex, &mask, BITS_PER_LONG, false);
 }
 
 /**
@@ -344,16 +192,9 @@ unsigned int phy_supported_speeds(struct phy_device *phy,
 				  unsigned int *speeds,
 				  unsigned int size)
 {
-	unsigned int count = 0;
-	unsigned int idx = 0;
-
-	for (idx = 0; idx < ARRAY_SIZE(settings) && count < size; idx++)
-		/* Assumes settings are grouped by speed */
-		if ((settings[idx].setting & phy->supported) &&
-		    (count == 0 || speeds[count - 1] != settings[idx].speed))
-			speeds[count++] = settings[idx].speed;
+	unsigned long supported = phy->supported;
 
-	return count;
+	return phy_speeds(speeds, size, &supported, BITS_PER_LONG);
 }
 
 /**
@@ -367,7 +208,9 @@ unsigned int phy_supported_speeds(struct phy_device *phy,
  */
 static inline bool phy_check_valid(int speed, int duplex, u32 features)
 {
-	return !!phy_lookup_setting(speed, duplex, features, true);
+	unsigned long mask = features;
+
+	return !!phy_lookup_setting(speed, duplex, &mask, BITS_PER_LONG, true);
 }
 
 /**
@@ -705,14 +548,15 @@ EXPORT_SYMBOL(phy_start_aneg);
  *
  * Description: The PHY infrastructure can run a state machine
  *   which tracks whether the PHY is starting up, negotiating,
- *   etc.  This function starts the timer which tracks the state
- *   of the PHY.  If you want to maintain your own state machine,
+ *   etc.  This function starts the delayed workqueue which tracks
+ *   the state of the PHY. If you want to maintain your own state machine,
  *   do not call this function.
  */
 void phy_start_machine(struct phy_device *phydev)
 {
 	queue_delayed_work(system_power_efficient_wq, &phydev->state_queue, HZ);
 }
+EXPORT_SYMBOL_GPL(phy_start_machine);
 
 /**
  * phy_trigger_machine - trigger the state machine to run
@@ -737,9 +581,9 @@ void phy_trigger_machine(struct phy_device *phydev, bool sync)
  * phy_stop_machine - stop the PHY state machine tracking
  * @phydev: target phy_device struct
  *
- * Description: Stops the state machine timer, sets the state to UP
- *   (unless it wasn't up yet). This function must be called BEFORE
- *   phy_detach.
+ * Description: Stops the state machine delayed workqueue, sets the
+ *   state to UP (unless it wasn't up yet). This function must be
+ *   called BEFORE phy_detach.
  */
 void phy_stop_machine(struct phy_device *phydev)
 {
@@ -749,9 +593,6 @@ void phy_stop_machine(struct phy_device *phydev)
 	if (phydev->state > PHY_UP && phydev->state != PHY_HALTED)
 		phydev->state = PHY_UP;
 	mutex_unlock(&phydev->lock);
-
-	/* Now we can run the state machine synchronously */
-	phy_state_machine(&phydev->state_queue.work);
 }
 
 /**
@@ -1022,9 +863,15 @@ void phy_start(struct phy_device *phydev)
 }
 EXPORT_SYMBOL(phy_start);
 
-static void phy_adjust_link(struct phy_device *phydev)
+static void phy_link_up(struct phy_device *phydev)
 {
-	phydev->adjust_link(phydev->attached_dev);
+	phydev->phy_link_change(phydev, true, true);
+	phy_led_trigger_change_speed(phydev);
+}
+
+static void phy_link_down(struct phy_device *phydev, bool do_carrier)
+{
+	phydev->phy_link_change(phydev, false, do_carrier);
 	phy_led_trigger_change_speed(phydev);
 }
 
@@ -1069,8 +916,7 @@ void phy_state_machine(struct work_struct *work)
 		/* If the link is down, give up on negotiation for now */
 		if (!phydev->link) {
 			phydev->state = PHY_NOLINK;
-			netif_carrier_off(phydev->attached_dev);
-			phy_adjust_link(phydev);
+			phy_link_down(phydev, true);
 			break;
 		}
 
@@ -1082,9 +928,7 @@ void phy_state_machine(struct work_struct *work)
 		/* If AN is done, we're running */
 		if (err > 0) {
 			phydev->state = PHY_RUNNING;
-			netif_carrier_on(phydev->attached_dev);
-			phy_adjust_link(phydev);
-
+			phy_link_up(phydev);
 		} else if (0 == phydev->link_timeout--)
 			needs_aneg = true;
 		break;
@@ -1109,8 +953,7 @@ void phy_state_machine(struct work_struct *work)
 				}
 			}
 			phydev->state = PHY_RUNNING;
-			netif_carrier_on(phydev->attached_dev);
-			phy_adjust_link(phydev);
+			phy_link_up(phydev);
 		}
 		break;
 	case PHY_FORCING:
@@ -1120,13 +963,12 @@ void phy_state_machine(struct work_struct *work)
 
 		if (phydev->link) {
 			phydev->state = PHY_RUNNING;
-			netif_carrier_on(phydev->attached_dev);
+			phy_link_up(phydev);
 		} else {
 			if (0 == phydev->link_timeout--)
 				needs_aneg = true;
+			phy_link_down(phydev, false);
 		}
-
-		phy_adjust_link(phydev);
 		break;
 	case PHY_RUNNING:
 		/* Only register a CHANGE if we are polling and link changed
@@ -1158,14 +1000,12 @@ void phy_state_machine(struct work_struct *work)
 
 		if (phydev->link) {
 			phydev->state = PHY_RUNNING;
-			netif_carrier_on(phydev->attached_dev);
+			phy_link_up(phydev);
 		} else {
 			phydev->state = PHY_NOLINK;
-			netif_carrier_off(phydev->attached_dev);
+			phy_link_down(phydev, true);
 		}
 
-		phy_adjust_link(phydev);
-
 		if (phy_interrupt_is_valid(phydev))
 			err = phy_config_interrupt(phydev,
 						   PHY_INTERRUPT_ENABLED);
@@ -1173,8 +1013,7 @@ void phy_state_machine(struct work_struct *work)
 	case PHY_HALTED:
 		if (phydev->link) {
 			phydev->link = 0;
-			netif_carrier_off(phydev->attached_dev);
-			phy_adjust_link(phydev);
+			phy_link_down(phydev, true);
 			do_suspend = true;
 		}
 		break;
@@ -1194,11 +1033,11 @@ void phy_state_machine(struct work_struct *work)
 
 				if (phydev->link) {
 					phydev->state = PHY_RUNNING;
-					netif_carrier_on(phydev->attached_dev);
+					phy_link_up(phydev);
 				} else	{
 					phydev->state = PHY_NOLINK;
+					phy_link_down(phydev, false);
 				}
-				phy_adjust_link(phydev);
 			} else {
 				phydev->state = PHY_AN;
 				phydev->link_timeout = PHY_AN_TIMEOUT;
@@ -1210,11 +1049,11 @@ void phy_state_machine(struct work_struct *work)
 
 			if (phydev->link) {
 				phydev->state = PHY_RUNNING;
-				netif_carrier_on(phydev->attached_dev);
+				phy_link_up(phydev);
 			} else	{
 				phydev->state = PHY_NOLINK;
+				phy_link_down(phydev, false);
 			}
-			phy_adjust_link(phydev);
 		}
 		break;
 	}
@@ -1229,9 +1068,10 @@ void phy_state_machine(struct work_struct *work)
 	if (err < 0)
 		phy_error(phydev);
 
-	phydev_dbg(phydev, "PHY state change %s -> %s\n",
-		   phy_state_to_str(old_state),
-		   phy_state_to_str(phydev->state));
+	if (old_state != phydev->state)
+		phydev_dbg(phydev, "PHY state change %s -> %s\n",
+			   phy_state_to_str(old_state),
+			   phy_state_to_str(phydev->state));
 
 	/* Only re-schedule a PHY state machine change if we are polling the
 	 * PHY, if PHY_IGNORE_INTERRUPT is set, then we will be moving
diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c
index 1790f7fec125..8cf0c5901f95 100644
--- a/drivers/net/phy/phy_device.c
+++ b/drivers/net/phy/phy_device.c
@@ -688,6 +688,19 @@ struct phy_device *phy_find_first(struct mii_bus *bus)
 }
 EXPORT_SYMBOL(phy_find_first);
 
+static void phy_link_change(struct phy_device *phydev, bool up, bool do_carrier)
+{
+	struct net_device *netdev = phydev->attached_dev;
+
+	if (do_carrier) {
+		if (up)
+			netif_carrier_on(netdev);
+		else
+			netif_carrier_off(netdev);
+	}
+	phydev->adjust_link(netdev);
+}
+
 /**
  * phy_prepare_link - prepares the PHY layer to monitor link status
  * @phydev: target phy_device struct
@@ -861,19 +874,37 @@ void phy_attached_info(struct phy_device *phydev)
 }
 EXPORT_SYMBOL(phy_attached_info);
 
-#define ATTACHED_FMT "attached PHY driver [%s] (mii_bus:phy_addr=%s, irq=%d)"
+#define ATTACHED_FMT "attached PHY driver [%s] (mii_bus:phy_addr=%s, irq=%s)"
 void phy_attached_print(struct phy_device *phydev, const char *fmt, ...)
 {
+	const char *drv_name = phydev->drv ? phydev->drv->name : "unbound";
+	char *irq_str;
+	char irq_num[4];
+
+	switch(phydev->irq) {
+	case PHY_POLL:
+		irq_str = "POLL";
+		break;
+	case PHY_IGNORE_INTERRUPT:
+		irq_str = "IGNORE";
+		break;
+	default:
+		snprintf(irq_num, sizeof(irq_num), "%d", phydev->irq);
+		irq_str = irq_num;
+		break;
+	}
+
+
 	if (!fmt) {
 		dev_info(&phydev->mdio.dev, ATTACHED_FMT "\n",
-			 phydev->drv->name, phydev_name(phydev),
-			 phydev->irq);
+			 drv_name, phydev_name(phydev),
+			 irq_str);
 	} else {
 		va_list ap;
 
 		dev_info(&phydev->mdio.dev, ATTACHED_FMT,
-			 phydev->drv->name, phydev_name(phydev),
-			 phydev->irq);
+			 drv_name, phydev_name(phydev),
+			 irq_str);
 
 		va_start(ap, fmt);
 		vprintk(fmt, ap);
@@ -951,6 +982,7 @@ int phy_attach_direct(struct net_device *dev, struct phy_device *phydev,
 		goto error;
 	}
 
+	phydev->phy_link_change = phy_link_change;
 	phydev->attached_dev = dev;
 	dev->phydev = phydev;
 
@@ -1070,6 +1102,7 @@ void phy_detach(struct phy_device *phydev)
 	phydev->attached_dev->phydev = NULL;
 	phydev->attached_dev = NULL;
 	phy_suspend(phydev);
+	phydev->phylink = NULL;
 
 	phy_led_triggers_unregister(phydev);
 
diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c
new file mode 100644
index 000000000000..bcb4755bcd95
--- /dev/null
+++ b/drivers/net/phy/phylink.c
@@ -0,0 +1,1462 @@
+/*
+ * phylink models the MAC to optional PHY connection, supporting
+ * technologies such as SFP cages where the PHY is hot-pluggable.
+ *
+ * Copyright (C) 2015 Russell King
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <linux/ethtool.h>
+#include <linux/export.h>
+#include <linux/gpio/consumer.h>
+#include <linux/netdevice.h>
+#include <linux/of.h>
+#include <linux/of_mdio.h>
+#include <linux/phy.h>
+#include <linux/phy_fixed.h>
+#include <linux/phylink.h>
+#include <linux/rtnetlink.h>
+#include <linux/spinlock.h>
+#include <linux/workqueue.h>
+
+#include "sfp.h"
+#include "swphy.h"
+
+#define SUPPORTED_INTERFACES \
+	(SUPPORTED_TP | SUPPORTED_MII | SUPPORTED_FIBRE | \
+	 SUPPORTED_BNC | SUPPORTED_AUI | SUPPORTED_Backplane)
+#define ADVERTISED_INTERFACES \
+	(ADVERTISED_TP | ADVERTISED_MII | ADVERTISED_FIBRE | \
+	 ADVERTISED_BNC | ADVERTISED_AUI | ADVERTISED_Backplane)
+
+enum {
+	PHYLINK_DISABLE_STOPPED,
+	PHYLINK_DISABLE_LINK,
+};
+
+struct phylink {
+	struct net_device *netdev;
+	const struct phylink_mac_ops *ops;
+
+	unsigned long phylink_disable_state; /* bitmask of disables */
+	struct phy_device *phydev;
+	phy_interface_t link_interface;	/* PHY_INTERFACE_xxx */
+	u8 link_an_mode;		/* MLO_AN_xxx */
+	u8 link_port;			/* The current non-phy ethtool port */
+	__ETHTOOL_DECLARE_LINK_MODE_MASK(supported);
+
+	/* The link configuration settings */
+	struct phylink_link_state link_config;
+	struct gpio_desc *link_gpio;
+
+	struct mutex state_mutex;
+	struct phylink_link_state phy_state;
+	struct work_struct resolve;
+
+	bool mac_link_dropped;
+
+	struct sfp_bus *sfp_bus;
+};
+
+static inline void linkmode_zero(unsigned long *dst)
+{
+	bitmap_zero(dst, __ETHTOOL_LINK_MODE_MASK_NBITS);
+}
+
+static inline void linkmode_copy(unsigned long *dst, const unsigned long *src)
+{
+	bitmap_copy(dst, src, __ETHTOOL_LINK_MODE_MASK_NBITS);
+}
+
+static inline void linkmode_and(unsigned long *dst, const unsigned long *a,
+				const unsigned long *b)
+{
+	bitmap_and(dst, a, b, __ETHTOOL_LINK_MODE_MASK_NBITS);
+}
+
+static inline void linkmode_or(unsigned long *dst, const unsigned long *a,
+				const unsigned long *b)
+{
+	bitmap_or(dst, a, b, __ETHTOOL_LINK_MODE_MASK_NBITS);
+}
+
+static inline bool linkmode_empty(const unsigned long *src)
+{
+	return bitmap_empty(src, __ETHTOOL_LINK_MODE_MASK_NBITS);
+}
+
+void phylink_set_port_modes(unsigned long *mask)
+{
+	phylink_set(mask, TP);
+	phylink_set(mask, AUI);
+	phylink_set(mask, MII);
+	phylink_set(mask, FIBRE);
+	phylink_set(mask, BNC);
+	phylink_set(mask, Backplane);
+}
+EXPORT_SYMBOL_GPL(phylink_set_port_modes);
+
+static int phylink_is_empty_linkmode(const unsigned long *linkmode)
+{
+	__ETHTOOL_DECLARE_LINK_MODE_MASK(tmp) = { 0, };
+
+	phylink_set_port_modes(tmp);
+	phylink_set(tmp, Autoneg);
+	phylink_set(tmp, Pause);
+	phylink_set(tmp, Asym_Pause);
+
+	bitmap_andnot(tmp, linkmode, tmp, __ETHTOOL_LINK_MODE_MASK_NBITS);
+
+	return linkmode_empty(tmp);
+}
+
+static const char *phylink_an_mode_str(unsigned int mode)
+{
+	static const char *modestr[] = {
+		[MLO_AN_PHY] = "phy",
+		[MLO_AN_FIXED] = "fixed",
+		[MLO_AN_SGMII] = "SGMII",
+		[MLO_AN_8023Z] = "802.3z",
+	};
+
+	return mode < ARRAY_SIZE(modestr) ? modestr[mode] : "unknown";
+}
+
+static int phylink_validate(struct phylink *pl, unsigned long *supported,
+			    struct phylink_link_state *state)
+{
+	pl->ops->validate(pl->netdev, supported, state);
+
+	return phylink_is_empty_linkmode(supported) ? -EINVAL : 0;
+}
+
+static int phylink_parse_fixedlink(struct phylink *pl, struct device_node *np)
+{
+	struct device_node *fixed_node;
+	const struct phy_setting *s;
+	struct gpio_desc *desc;
+	const __be32 *fixed_prop;
+	u32 speed;
+	int ret, len;
+
+	fixed_node = of_get_child_by_name(np, "fixed-link");
+	if (fixed_node) {
+		ret = of_property_read_u32(fixed_node, "speed", &speed);
+
+		pl->link_config.speed = speed;
+		pl->link_config.duplex = DUPLEX_HALF;
+
+		if (of_property_read_bool(fixed_node, "full-duplex"))
+			pl->link_config.duplex = DUPLEX_FULL;
+
+		/* We treat the "pause" and "asym-pause" terminology as
+		 * defining the link partner's ability. */
+		if (of_property_read_bool(fixed_node, "pause"))
+			pl->link_config.pause |= MLO_PAUSE_SYM;
+		if (of_property_read_bool(fixed_node, "asym-pause"))
+			pl->link_config.pause |= MLO_PAUSE_ASYM;
+
+		if (ret == 0) {
+			desc = fwnode_get_named_gpiod(&fixed_node->fwnode,
+						      "link-gpios", 0,
+						      GPIOD_IN, "?");
+
+			if (!IS_ERR(desc))
+				pl->link_gpio = desc;
+			else if (desc == ERR_PTR(-EPROBE_DEFER))
+				ret = -EPROBE_DEFER;
+		}
+		of_node_put(fixed_node);
+
+		if (ret)
+			return ret;
+	} else {
+		fixed_prop = of_get_property(np, "fixed-link", &len);
+		if (!fixed_prop) {
+			netdev_err(pl->netdev, "broken fixed-link?\n");
+			return -EINVAL;
+		}
+		if (len == 5 * sizeof(*fixed_prop)) {
+			pl->link_config.duplex = be32_to_cpu(fixed_prop[1]) ?
+						DUPLEX_FULL : DUPLEX_HALF;
+			pl->link_config.speed = be32_to_cpu(fixed_prop[2]);
+			if (be32_to_cpu(fixed_prop[3]))
+				pl->link_config.pause |= MLO_PAUSE_SYM;
+			if (be32_to_cpu(fixed_prop[4]))
+				pl->link_config.pause |= MLO_PAUSE_ASYM;
+		}
+	}
+
+	if (pl->link_config.speed > SPEED_1000 &&
+	    pl->link_config.duplex != DUPLEX_FULL)
+		netdev_warn(pl->netdev, "fixed link specifies half duplex for %dMbps link?\n",
+			    pl->link_config.speed);
+
+	bitmap_fill(pl->supported, __ETHTOOL_LINK_MODE_MASK_NBITS);
+	linkmode_copy(pl->link_config.advertising, pl->supported);
+	phylink_validate(pl, pl->supported, &pl->link_config);
+
+	s = phy_lookup_setting(pl->link_config.speed, pl->link_config.duplex,
+			       pl->supported,
+			       __ETHTOOL_LINK_MODE_MASK_NBITS, true);
+	linkmode_zero(pl->supported);
+	phylink_set(pl->supported, MII);
+	if (s) {
+		__set_bit(s->bit, pl->supported);
+	} else {
+		netdev_warn(pl->netdev, "fixed link %s duplex %dMbps not recognised\n",
+			    pl->link_config.duplex == DUPLEX_FULL ? "full" : "half",
+			    pl->link_config.speed);
+	}
+
+	linkmode_and(pl->link_config.advertising, pl->link_config.advertising,
+		     pl->supported);
+
+	pl->link_config.link = 1;
+	pl->link_config.an_complete = 1;
+
+	return 0;
+}
+
+static int phylink_parse_mode(struct phylink *pl, struct device_node *np)
+{
+	struct device_node *dn;
+	const char *managed;
+
+	dn = of_get_child_by_name(np, "fixed-link");
+	if (dn || of_find_property(np, "fixed-link", NULL))
+		pl->link_an_mode = MLO_AN_FIXED;
+	of_node_put(dn);
+
+	if (of_property_read_string(np, "managed", &managed) == 0 &&
+	    strcmp(managed, "in-band-status") == 0) {
+		if (pl->link_an_mode == MLO_AN_FIXED) {
+			netdev_err(pl->netdev,
+				   "can't use both fixed-link and in-band-status\n");
+			return -EINVAL;
+		}
+
+		linkmode_zero(pl->supported);
+		phylink_set(pl->supported, MII);
+		phylink_set(pl->supported, Autoneg);
+		phylink_set(pl->supported, Asym_Pause);
+		phylink_set(pl->supported, Pause);
+		pl->link_config.an_enabled = true;
+
+		switch (pl->link_config.interface) {
+		case PHY_INTERFACE_MODE_SGMII:
+			phylink_set(pl->supported, 10baseT_Half);
+			phylink_set(pl->supported, 10baseT_Full);
+			phylink_set(pl->supported, 100baseT_Half);
+			phylink_set(pl->supported, 100baseT_Full);
+			phylink_set(pl->supported, 1000baseT_Half);
+			phylink_set(pl->supported, 1000baseT_Full);
+			pl->link_an_mode = MLO_AN_SGMII;
+			break;
+
+		case PHY_INTERFACE_MODE_1000BASEX:
+			phylink_set(pl->supported, 1000baseX_Full);
+			pl->link_an_mode = MLO_AN_8023Z;
+			break;
+
+		case PHY_INTERFACE_MODE_2500BASEX:
+			phylink_set(pl->supported, 2500baseX_Full);
+			pl->link_an_mode = MLO_AN_8023Z;
+			break;
+
+		case PHY_INTERFACE_MODE_10GKR:
+			phylink_set(pl->supported, 10baseT_Half);
+			phylink_set(pl->supported, 10baseT_Full);
+			phylink_set(pl->supported, 100baseT_Half);
+			phylink_set(pl->supported, 100baseT_Full);
+			phylink_set(pl->supported, 1000baseT_Half);
+			phylink_set(pl->supported, 1000baseT_Full);
+			phylink_set(pl->supported, 1000baseX_Full);
+			phylink_set(pl->supported, 10000baseKR_Full);
+			phylink_set(pl->supported, 10000baseCR_Full);
+			phylink_set(pl->supported, 10000baseSR_Full);
+			phylink_set(pl->supported, 10000baseLR_Full);
+			phylink_set(pl->supported, 10000baseLRM_Full);
+			phylink_set(pl->supported, 10000baseER_Full);
+			pl->link_an_mode = MLO_AN_SGMII;
+			break;
+
+		default:
+			netdev_err(pl->netdev,
+				   "incorrect link mode %s for in-band status\n",
+				   phy_modes(pl->link_config.interface));
+			return -EINVAL;
+		}
+
+		linkmode_copy(pl->link_config.advertising, pl->supported);
+
+		if (phylink_validate(pl, pl->supported, &pl->link_config)) {
+			netdev_err(pl->netdev,
+				   "failed to validate link configuration for in-band status\n");
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static void phylink_mac_config(struct phylink *pl,
+			       const struct phylink_link_state *state)
+{
+	netdev_dbg(pl->netdev,
+		   "%s: mode=%s/%s/%s/%s adv=%*pb pause=%02x link=%u an=%u\n",
+		   __func__, phylink_an_mode_str(pl->link_an_mode),
+		   phy_modes(state->interface),
+		   phy_speed_to_str(state->speed),
+		   phy_duplex_to_str(state->duplex),
+		   __ETHTOOL_LINK_MODE_MASK_NBITS, state->advertising,
+		   state->pause, state->link, state->an_enabled);
+
+	pl->ops->mac_config(pl->netdev, pl->link_an_mode, state);
+}
+
+static void phylink_mac_an_restart(struct phylink *pl)
+{
+	if (pl->link_config.an_enabled &&
+	    (pl->link_config.interface == PHY_INTERFACE_MODE_1000BASEX ||
+	     pl->link_config.interface == PHY_INTERFACE_MODE_2500BASEX))
+		pl->ops->mac_an_restart(pl->netdev);
+}
+
+static int phylink_get_mac_state(struct phylink *pl, struct phylink_link_state *state)
+{
+	struct net_device *ndev = pl->netdev;
+
+	linkmode_copy(state->advertising, pl->link_config.advertising);
+	linkmode_zero(state->lp_advertising);
+	state->interface = pl->link_config.interface;
+	state->an_enabled = pl->link_config.an_enabled;
+	state->link = 1;
+
+	return pl->ops->mac_link_state(ndev, state);
+}
+
+/* The fixed state is... fixed except for the link state,
+ * which may be determined by a GPIO.
+ */
+static void phylink_get_fixed_state(struct phylink *pl, struct phylink_link_state *state)
+{
+	*state = pl->link_config;
+	if (pl->link_gpio)
+		state->link = !!gpiod_get_value(pl->link_gpio);
+}
+
+/* Flow control is resolved according to our and the link partners
+ * advertisments using the following drawn from the 802.3 specs:
+ *  Local device  Link partner
+ *  Pause AsymDir Pause AsymDir Result
+ *    1     X       1     X     TX+RX
+ *    0     1       1     1     RX
+ *    1     1       0     1     TX
+ */
+static void phylink_resolve_flow(struct phylink *pl,
+	struct phylink_link_state *state)
+{
+	int new_pause = 0;
+
+	if (pl->link_config.pause & MLO_PAUSE_AN) {
+		int pause = 0;
+
+		if (phylink_test(pl->link_config.advertising, Pause))
+			pause |= MLO_PAUSE_SYM;
+		if (phylink_test(pl->link_config.advertising, Asym_Pause))
+			pause |= MLO_PAUSE_ASYM;
+
+		pause &= state->pause;
+
+		if (pause & MLO_PAUSE_SYM)
+			new_pause = MLO_PAUSE_TX | MLO_PAUSE_RX;
+		else if (pause & MLO_PAUSE_ASYM)
+			new_pause = state->pause & MLO_PAUSE_SYM ?
+				 MLO_PAUSE_RX : MLO_PAUSE_TX;
+	} else {
+		new_pause = pl->link_config.pause & MLO_PAUSE_TXRX_MASK;
+	}
+
+	state->pause &= ~MLO_PAUSE_TXRX_MASK;
+	state->pause |= new_pause;
+}
+
+static const char *phylink_pause_to_str(int pause)
+{
+	switch (pause & MLO_PAUSE_TXRX_MASK) {
+	case MLO_PAUSE_TX | MLO_PAUSE_RX:
+		return "rx/tx";
+	case MLO_PAUSE_TX:
+		return "tx";
+	case MLO_PAUSE_RX:
+		return "rx";
+	default:
+		return "off";
+	}
+}
+
+static void phylink_resolve(struct work_struct *w)
+{
+	struct phylink *pl = container_of(w, struct phylink, resolve);
+	struct phylink_link_state link_state;
+	struct net_device *ndev = pl->netdev;
+
+	mutex_lock(&pl->state_mutex);
+	if (pl->phylink_disable_state) {
+		pl->mac_link_dropped = false;
+		link_state.link = false;
+	} else if (pl->mac_link_dropped) {
+		link_state.link = false;
+	} else {
+		switch (pl->link_an_mode) {
+		case MLO_AN_PHY:
+			link_state = pl->phy_state;
+			phylink_resolve_flow(pl, &link_state);
+			phylink_mac_config(pl, &link_state);
+			break;
+
+		case MLO_AN_FIXED:
+			phylink_get_fixed_state(pl, &link_state);
+			phylink_mac_config(pl, &link_state);
+			break;
+
+		case MLO_AN_SGMII:
+			phylink_get_mac_state(pl, &link_state);
+			if (pl->phydev) {
+				bool changed = false;
+
+				link_state.link = link_state.link &&
+						  pl->phy_state.link;
+
+				if (pl->phy_state.interface !=
+				    link_state.interface) {
+					link_state.interface = pl->phy_state.interface;
+					changed = true;
+				}
+
+				/* Propagate the flow control from the PHY
+				 * to the MAC. Also propagate the interface
+				 * if changed.
+				 */
+				if (pl->phy_state.link || changed) {
+					link_state.pause |= pl->phy_state.pause;
+					phylink_resolve_flow(pl, &link_state);
+
+					phylink_mac_config(pl, &link_state);
+				}
+			}
+			break;
+
+		case MLO_AN_8023Z:
+			phylink_get_mac_state(pl, &link_state);
+			break;
+		}
+	}
+
+	if (link_state.link != netif_carrier_ok(ndev)) {
+		if (!link_state.link) {
+			netif_carrier_off(ndev);
+			pl->ops->mac_link_down(ndev, pl->link_an_mode);
+			netdev_info(ndev, "Link is Down\n");
+		} else {
+			pl->ops->mac_link_up(ndev, pl->link_an_mode,
+					     pl->phydev);
+
+			netif_carrier_on(ndev);
+
+			netdev_info(ndev,
+				    "Link is Up - %s/%s - flow control %s\n",
+				    phy_speed_to_str(link_state.speed),
+				    phy_duplex_to_str(link_state.duplex),
+				    phylink_pause_to_str(link_state.pause));
+		}
+	}
+	if (!link_state.link && pl->mac_link_dropped) {
+		pl->mac_link_dropped = false;
+		queue_work(system_power_efficient_wq, &pl->resolve);
+	}
+	mutex_unlock(&pl->state_mutex);
+}
+
+static void phylink_run_resolve(struct phylink *pl)
+{
+	if (!pl->phylink_disable_state)
+		queue_work(system_power_efficient_wq, &pl->resolve);
+}
+
+static const struct sfp_upstream_ops sfp_phylink_ops;
+
+static int phylink_register_sfp(struct phylink *pl, struct device_node *np)
+{
+	struct device_node *sfp_np;
+
+	sfp_np = of_parse_phandle(np, "sfp", 0);
+	if (!sfp_np)
+		return 0;
+
+	pl->sfp_bus = sfp_register_upstream(sfp_np, pl->netdev, pl,
+					    &sfp_phylink_ops);
+	if (!pl->sfp_bus)
+		return -ENOMEM;
+
+	return 0;
+}
+
+struct phylink *phylink_create(struct net_device *ndev, struct device_node *np,
+	phy_interface_t iface, const struct phylink_mac_ops *ops)
+{
+	struct phylink *pl;
+	int ret;
+
+	pl = kzalloc(sizeof(*pl), GFP_KERNEL);
+	if (!pl)
+		return ERR_PTR(-ENOMEM);
+
+	mutex_init(&pl->state_mutex);
+	INIT_WORK(&pl->resolve, phylink_resolve);
+	pl->netdev = ndev;
+	pl->phy_state.interface = iface;
+	pl->link_interface = iface;
+	pl->link_port = PORT_MII;
+	pl->link_config.interface = iface;
+	pl->link_config.pause = MLO_PAUSE_AN;
+	pl->link_config.speed = SPEED_UNKNOWN;
+	pl->link_config.duplex = DUPLEX_UNKNOWN;
+	pl->ops = ops;
+	__set_bit(PHYLINK_DISABLE_STOPPED, &pl->phylink_disable_state);
+
+	bitmap_fill(pl->supported, __ETHTOOL_LINK_MODE_MASK_NBITS);
+	linkmode_copy(pl->link_config.advertising, pl->supported);
+	phylink_validate(pl, pl->supported, &pl->link_config);
+
+	ret = phylink_parse_mode(pl, np);
+	if (ret < 0) {
+		kfree(pl);
+		return ERR_PTR(ret);
+	}
+
+	if (pl->link_an_mode == MLO_AN_FIXED) {
+		ret = phylink_parse_fixedlink(pl, np);
+		if (ret < 0) {
+			kfree(pl);
+			return ERR_PTR(ret);
+		}
+	}
+
+	ret = phylink_register_sfp(pl, np);
+	if (ret < 0) {
+		kfree(pl);
+		return ERR_PTR(ret);
+	}
+
+	return pl;
+}
+EXPORT_SYMBOL_GPL(phylink_create);
+
+void phylink_destroy(struct phylink *pl)
+{
+	if (pl->sfp_bus)
+		sfp_unregister_upstream(pl->sfp_bus);
+
+	cancel_work_sync(&pl->resolve);
+	kfree(pl);
+}
+EXPORT_SYMBOL_GPL(phylink_destroy);
+
+void phylink_phy_change(struct phy_device *phydev, bool up, bool do_carrier)
+{
+	struct phylink *pl = phydev->phylink;
+
+	mutex_lock(&pl->state_mutex);
+	pl->phy_state.speed = phydev->speed;
+	pl->phy_state.duplex = phydev->duplex;
+	pl->phy_state.pause = MLO_PAUSE_NONE;
+	if (phydev->pause)
+		pl->phy_state.pause |= MLO_PAUSE_SYM;
+	if (phydev->asym_pause)
+		pl->phy_state.pause |= MLO_PAUSE_ASYM;
+	pl->phy_state.interface = phydev->interface;
+	pl->phy_state.link = up;
+	mutex_unlock(&pl->state_mutex);
+
+	phylink_run_resolve(pl);
+
+	netdev_dbg(pl->netdev, "phy link %s %s/%s/%s\n", up ? "up" : "down",
+	           phy_modes(phydev->interface),
+		   phy_speed_to_str(phydev->speed),
+		   phy_duplex_to_str(phydev->duplex));
+}
+
+static int phylink_bringup_phy(struct phylink *pl, struct phy_device *phy)
+{
+	struct phylink_link_state config;
+	__ETHTOOL_DECLARE_LINK_MODE_MASK(supported);
+	u32 advertising;
+	int ret;
+
+	memset(&config, 0, sizeof(config));
+	ethtool_convert_legacy_u32_to_link_mode(supported, phy->supported);
+	ethtool_convert_legacy_u32_to_link_mode(config.advertising,
+						phy->advertising);
+	config.interface = pl->link_config.interface;
+
+	/*
+	 * This is the new way of dealing with flow control for PHYs,
+	 * as described by Timur Tabi in commit 529ed1275263 ("net: phy:
+	 * phy drivers should not set SUPPORTED_[Asym_]Pause") except
+	 * using our validate call to the MAC, we rely upon the MAC
+	 * clearing the bits from both supported and advertising fields.
+	 */
+	if (phylink_test(supported, Pause))
+		phylink_set(config.advertising, Pause);
+	if (phylink_test(supported, Asym_Pause))
+		phylink_set(config.advertising, Asym_Pause);
+
+	ret = phylink_validate(pl, supported, &config);
+	if (ret)
+		return ret;
+
+	phy->phylink = pl;
+	phy->phy_link_change = phylink_phy_change;
+
+	netdev_info(pl->netdev,
+		    "PHY [%s] driver [%s]\n", dev_name(&phy->mdio.dev),
+		    phy->drv->name);
+
+	mutex_lock(&phy->lock);
+	mutex_lock(&pl->state_mutex);
+	pl->netdev->phydev = phy;
+	pl->phydev = phy;
+	linkmode_copy(pl->supported, supported);
+	linkmode_copy(pl->link_config.advertising, config.advertising);
+
+	/* Restrict the phy advertisment according to the MAC support. */
+	ethtool_convert_link_mode_to_legacy_u32(&advertising, config.advertising);
+	phy->advertising = advertising;
+	mutex_unlock(&pl->state_mutex);
+	mutex_unlock(&phy->lock);
+
+	netdev_dbg(pl->netdev,
+		   "phy: setting supported %*pb advertising 0x%08x\n",
+		   __ETHTOOL_LINK_MODE_MASK_NBITS, pl->supported,
+		   phy->advertising);
+
+	phy_start_machine(phy);
+	if (phy->irq > 0)
+		phy_start_interrupts(phy);
+
+	return 0;
+}
+
+int phylink_connect_phy(struct phylink *pl, struct phy_device *phy)
+{
+	int ret;
+
+	ret = phy_attach_direct(pl->netdev, phy, 0, pl->link_interface);
+	if (ret)
+		return ret;
+
+	ret = phylink_bringup_phy(pl, phy);
+	if (ret)
+		phy_detach(phy);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(phylink_connect_phy);
+
+int phylink_of_phy_connect(struct phylink *pl, struct device_node *dn)
+{
+	struct device_node *phy_node;
+	struct phy_device *phy_dev;
+	int ret;
+
+	/* Fixed links are handled without needing a PHY */
+	if (pl->link_an_mode == MLO_AN_FIXED)
+		return 0;
+
+	phy_node = of_parse_phandle(dn, "phy-handle", 0);
+	if (!phy_node)
+		phy_node = of_parse_phandle(dn, "phy", 0);
+	if (!phy_node)
+		phy_node = of_parse_phandle(dn, "phy-device", 0);
+
+	if (!phy_node) {
+		if (pl->link_an_mode == MLO_AN_PHY) {
+			netdev_err(pl->netdev, "unable to find PHY node\n");
+			return -ENODEV;
+		}
+		return 0;
+	}
+
+	phy_dev = of_phy_attach(pl->netdev, phy_node, 0, pl->link_interface);
+	/* We're done with the phy_node handle */
+	of_node_put(phy_node);
+
+	if (!phy_dev)
+		return -ENODEV;
+
+	ret = phylink_bringup_phy(pl, phy_dev);
+	if (ret)
+		phy_detach(phy_dev);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(phylink_of_phy_connect);
+
+void phylink_disconnect_phy(struct phylink *pl)
+{
+	struct phy_device *phy;
+
+	WARN_ON(!lockdep_rtnl_is_held());
+
+	phy = pl->phydev;
+	if (phy) {
+		mutex_lock(&phy->lock);
+		mutex_lock(&pl->state_mutex);
+		pl->netdev->phydev = NULL;
+		pl->phydev = NULL;
+		mutex_unlock(&pl->state_mutex);
+		mutex_unlock(&phy->lock);
+		flush_work(&pl->resolve);
+
+		phy_disconnect(phy);
+	}
+}
+EXPORT_SYMBOL_GPL(phylink_disconnect_phy);
+
+void phylink_mac_change(struct phylink *pl, bool up)
+{
+	if (!up)
+		pl->mac_link_dropped = true;
+	phylink_run_resolve(pl);
+	netdev_dbg(pl->netdev, "mac link %s\n", up ? "up" : "down");
+}
+EXPORT_SYMBOL_GPL(phylink_mac_change);
+
+void phylink_start(struct phylink *pl)
+{
+	WARN_ON(!lockdep_rtnl_is_held());
+
+	netdev_info(pl->netdev, "configuring for %s/%s link mode\n",
+		    phylink_an_mode_str(pl->link_an_mode),
+		    phy_modes(pl->link_config.interface));
+
+	/* Apply the link configuration to the MAC when starting. This allows
+	 * a fixed-link to start with the correct parameters, and also
+	 * ensures that we set the appropriate advertisment for Serdes links.
+	 */
+	phylink_resolve_flow(pl, &pl->link_config);
+	phylink_mac_config(pl, &pl->link_config);
+
+	clear_bit(PHYLINK_DISABLE_STOPPED, &pl->phylink_disable_state);
+	phylink_run_resolve(pl);
+
+	if (pl->sfp_bus)
+		sfp_upstream_start(pl->sfp_bus);
+	if (pl->phydev)
+		phy_start(pl->phydev);
+}
+EXPORT_SYMBOL_GPL(phylink_start);
+
+void phylink_stop(struct phylink *pl)
+{
+	WARN_ON(!lockdep_rtnl_is_held());
+
+	if (pl->phydev)
+		phy_stop(pl->phydev);
+	if (pl->sfp_bus)
+		sfp_upstream_stop(pl->sfp_bus);
+
+	set_bit(PHYLINK_DISABLE_STOPPED, &pl->phylink_disable_state);
+	flush_work(&pl->resolve);
+}
+EXPORT_SYMBOL_GPL(phylink_stop);
+
+void phylink_ethtool_get_wol(struct phylink *pl, struct ethtool_wolinfo *wol)
+{
+	WARN_ON(!lockdep_rtnl_is_held());
+
+	wol->supported = 0;
+	wol->wolopts = 0;
+
+	if (pl->phydev)
+		phy_ethtool_get_wol(pl->phydev, wol);
+}
+EXPORT_SYMBOL_GPL(phylink_ethtool_get_wol);
+
+int phylink_ethtool_set_wol(struct phylink *pl, struct ethtool_wolinfo *wol)
+{
+	int ret = -EOPNOTSUPP;
+
+	WARN_ON(!lockdep_rtnl_is_held());
+
+	if (pl->phydev)
+		ret = phy_ethtool_set_wol(pl->phydev, wol);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(phylink_ethtool_set_wol);
+
+static void phylink_merge_link_mode(unsigned long *dst, const unsigned long *b)
+{
+	__ETHTOOL_DECLARE_LINK_MODE_MASK(mask);
+
+	linkmode_zero(mask);
+	phylink_set_port_modes(mask);
+
+	linkmode_and(dst, dst, mask);
+	linkmode_or(dst, dst, b);
+}
+
+static void phylink_get_ksettings(const struct phylink_link_state *state,
+				  struct ethtool_link_ksettings *kset)
+{
+	phylink_merge_link_mode(kset->link_modes.advertising, state->advertising);
+	linkmode_copy(kset->link_modes.lp_advertising, state->lp_advertising);
+	kset->base.speed = state->speed;
+	kset->base.duplex = state->duplex;
+	kset->base.autoneg = state->an_enabled ? AUTONEG_ENABLE :
+				AUTONEG_DISABLE;
+}
+
+int phylink_ethtool_ksettings_get(struct phylink *pl,
+	struct ethtool_link_ksettings *kset)
+{
+	struct phylink_link_state link_state;
+
+	WARN_ON(!lockdep_rtnl_is_held());
+
+	if (pl->phydev) {
+		phy_ethtool_ksettings_get(pl->phydev, kset);
+	} else {
+		kset->base.port = pl->link_port;
+	}
+
+	linkmode_copy(kset->link_modes.supported, pl->supported);
+
+	switch (pl->link_an_mode) {
+	case MLO_AN_FIXED:
+		/* We are using fixed settings. Report these as the
+		 * current link settings - and note that these also
+		 * represent the supported speeds/duplex/pause modes.
+		 */
+		phylink_get_fixed_state(pl, &link_state);
+		phylink_get_ksettings(&link_state, kset);
+		break;
+
+	case MLO_AN_SGMII:
+		/* If there is a phy attached, then use the reported
+		 * settings from the phy with no modification.
+		 */
+		if (pl->phydev)
+			break;
+
+	case MLO_AN_8023Z:
+		phylink_get_mac_state(pl, &link_state);
+
+		/* The MAC is reporting the link results from its own PCS
+		 * layer via in-band status. Report these as the current
+		 * link settings.
+		 */
+		phylink_get_ksettings(&link_state, kset);
+		break;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(phylink_ethtool_ksettings_get);
+
+int phylink_ethtool_ksettings_set(struct phylink *pl,
+	const struct ethtool_link_ksettings *kset)
+{
+	struct ethtool_link_ksettings our_kset;
+	struct phylink_link_state config;
+	int ret;
+
+	WARN_ON(!lockdep_rtnl_is_held());
+
+	if (kset->base.autoneg != AUTONEG_DISABLE &&
+	    kset->base.autoneg != AUTONEG_ENABLE)
+		return -EINVAL;
+
+	config = pl->link_config;
+
+	/* Mask out unsupported advertisments */
+	linkmode_and(config.advertising, kset->link_modes.advertising,
+		     pl->supported);
+
+	/* FIXME: should we reject autoneg if phy/mac does not support it? */
+	if (kset->base.autoneg == AUTONEG_DISABLE) {
+		const struct phy_setting *s;
+
+		/* Autonegotiation disabled, select a suitable speed and
+		 * duplex.
+		 */
+		s = phy_lookup_setting(kset->base.speed, kset->base.duplex,
+				       pl->supported,
+				       __ETHTOOL_LINK_MODE_MASK_NBITS, false);
+		if (!s)
+			return -EINVAL;
+
+		/* If we have a fixed link (as specified by firmware), refuse
+		 * to change link parameters.
+		 */
+		if (pl->link_an_mode == MLO_AN_FIXED &&
+		    (s->speed != pl->link_config.speed ||
+		     s->duplex != pl->link_config.duplex))
+			return -EINVAL;
+
+		config.speed = s->speed;
+		config.duplex = s->duplex;
+		config.an_enabled = false;
+
+		__clear_bit(ETHTOOL_LINK_MODE_Autoneg_BIT, config.advertising);
+	} else {
+		/* If we have a fixed link, refuse to enable autonegotiation */
+		if (pl->link_an_mode == MLO_AN_FIXED)
+			return -EINVAL;
+
+		config.speed = SPEED_UNKNOWN;
+		config.duplex = DUPLEX_UNKNOWN;
+		config.an_enabled = true;
+
+		__set_bit(ETHTOOL_LINK_MODE_Autoneg_BIT, config.advertising);
+	}
+
+	if (phylink_validate(pl, pl->supported, &config))
+		return -EINVAL;
+
+	/* If autonegotiation is enabled, we must have an advertisment */
+	if (config.an_enabled && phylink_is_empty_linkmode(config.advertising))
+		return -EINVAL;
+
+	our_kset = *kset;
+	linkmode_copy(our_kset.link_modes.advertising, config.advertising);
+	our_kset.base.speed = config.speed;
+	our_kset.base.duplex = config.duplex;
+
+	/* If we have a PHY, configure the phy */
+	if (pl->phydev) {
+		ret = phy_ethtool_ksettings_set(pl->phydev, &our_kset);
+		if (ret)
+			return ret;
+	}
+
+	mutex_lock(&pl->state_mutex);
+	/* Configure the MAC to match the new settings */
+	linkmode_copy(pl->link_config.advertising, our_kset.link_modes.advertising);
+	pl->link_config.speed = our_kset.base.speed;
+	pl->link_config.duplex = our_kset.base.duplex;
+	pl->link_config.an_enabled = our_kset.base.autoneg != AUTONEG_DISABLE;
+
+	if (!test_bit(PHYLINK_DISABLE_STOPPED, &pl->phylink_disable_state)) {
+		phylink_mac_config(pl, &pl->link_config);
+		phylink_mac_an_restart(pl);
+	}
+	mutex_unlock(&pl->state_mutex);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(phylink_ethtool_ksettings_set);
+
+int phylink_ethtool_nway_reset(struct phylink *pl)
+{
+	int ret = 0;
+
+	WARN_ON(!lockdep_rtnl_is_held());
+
+	if (pl->phydev)
+		ret = phy_restart_aneg(pl->phydev);
+	phylink_mac_an_restart(pl);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(phylink_ethtool_nway_reset);
+
+void phylink_ethtool_get_pauseparam(struct phylink *pl,
+				    struct ethtool_pauseparam *pause)
+{
+	WARN_ON(!lockdep_rtnl_is_held());
+
+	pause->autoneg = !!(pl->link_config.pause & MLO_PAUSE_AN);
+	pause->rx_pause = !!(pl->link_config.pause & MLO_PAUSE_RX);
+	pause->tx_pause = !!(pl->link_config.pause & MLO_PAUSE_TX);
+}
+EXPORT_SYMBOL_GPL(phylink_ethtool_get_pauseparam);
+
+int phylink_ethtool_set_pauseparam(struct phylink *pl,
+				   struct ethtool_pauseparam *pause)
+{
+	struct phylink_link_state *config = &pl->link_config;
+
+	WARN_ON(!lockdep_rtnl_is_held());
+
+	if (!phylink_test(pl->supported, Pause) &&
+	    !phylink_test(pl->supported, Asym_Pause))
+		return -EOPNOTSUPP;
+
+	if (!phylink_test(pl->supported, Asym_Pause) &&
+	    !pause->autoneg && pause->rx_pause != pause->tx_pause)
+		return -EINVAL;
+
+	config->pause &= ~(MLO_PAUSE_AN | MLO_PAUSE_TXRX_MASK);
+
+	if (pause->autoneg)
+		config->pause |= MLO_PAUSE_AN;
+	if (pause->rx_pause)
+		config->pause |= MLO_PAUSE_RX;
+	if (pause->tx_pause)
+		config->pause |= MLO_PAUSE_TX;
+
+	if (!test_bit(PHYLINK_DISABLE_STOPPED, &pl->phylink_disable_state)) {
+		switch (pl->link_an_mode) {
+		case MLO_AN_PHY:
+			/* Silently mark the carrier down, and then trigger a resolve */
+			netif_carrier_off(pl->netdev);
+			phylink_run_resolve(pl);
+			break;
+
+		case MLO_AN_FIXED:
+			/* Should we allow fixed links to change against the config? */
+			phylink_resolve_flow(pl, config);
+			phylink_mac_config(pl, config);
+			break;
+
+		case MLO_AN_SGMII:
+		case MLO_AN_8023Z:
+			phylink_mac_config(pl, config);
+			phylink_mac_an_restart(pl);
+			break;
+		}
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(phylink_ethtool_set_pauseparam);
+
+int phylink_ethtool_get_module_info(struct phylink *pl,
+				    struct ethtool_modinfo *modinfo)
+{
+	int ret = -EOPNOTSUPP;
+
+	WARN_ON(!lockdep_rtnl_is_held());
+
+	if (pl->sfp_bus)
+		ret = sfp_get_module_info(pl->sfp_bus, modinfo);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(phylink_ethtool_get_module_info);
+
+int phylink_ethtool_get_module_eeprom(struct phylink *pl,
+				      struct ethtool_eeprom *ee, u8 *buf)
+{
+	int ret = -EOPNOTSUPP;
+
+	WARN_ON(!lockdep_rtnl_is_held());
+
+	if (pl->sfp_bus)
+		ret = sfp_get_module_eeprom(pl->sfp_bus, ee, buf);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(phylink_ethtool_get_module_eeprom);
+
+int phylink_init_eee(struct phylink *pl, bool clk_stop_enable)
+{
+	int ret = -EPROTONOSUPPORT;
+
+	WARN_ON(!lockdep_rtnl_is_held());
+
+	if (pl->phydev)
+		ret = phy_init_eee(pl->phydev, clk_stop_enable);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(phylink_init_eee);
+
+int phylink_get_eee_err(struct phylink *pl)
+{
+	int ret = 0;
+
+	WARN_ON(!lockdep_rtnl_is_held());
+
+	if (pl->phydev)
+		ret = phy_get_eee_err(pl->phydev);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(phylink_get_eee_err);
+
+int phylink_ethtool_get_eee(struct phylink *pl, struct ethtool_eee *eee)
+{
+	int ret = -EOPNOTSUPP;
+
+	WARN_ON(!lockdep_rtnl_is_held());
+
+	if (pl->phydev)
+		ret = phy_ethtool_get_eee(pl->phydev, eee);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(phylink_ethtool_get_eee);
+
+int phylink_ethtool_set_eee(struct phylink *pl, struct ethtool_eee *eee)
+{
+	int ret = -EOPNOTSUPP;
+
+	WARN_ON(!lockdep_rtnl_is_held());
+
+	if (pl->phydev)
+		ret = phy_ethtool_set_eee(pl->phydev, eee);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(phylink_ethtool_set_eee);
+
+/* This emulates MII registers for a fixed-mode phy operating as per the
+ * passed in state. "aneg" defines if we report negotiation is possible.
+ *
+ * FIXME: should deal with negotiation state too.
+ */
+static int phylink_mii_emul_read(struct net_device *ndev, unsigned int reg,
+				 struct phylink_link_state *state, bool aneg)
+{
+	struct fixed_phy_status fs;
+	int val;
+
+	fs.link = state->link;
+	fs.speed = state->speed;
+	fs.duplex = state->duplex;
+	fs.pause = state->pause & MLO_PAUSE_SYM;
+	fs.asym_pause = state->pause & MLO_PAUSE_ASYM;
+
+	val = swphy_read_reg(reg, &fs);
+	if (reg == MII_BMSR) {
+		if (!state->an_complete)
+			val &= ~BMSR_ANEGCOMPLETE;
+		if (!aneg)
+			val &= ~BMSR_ANEGCAPABLE;
+	}
+	return val;
+}
+
+static int phylink_phy_read(struct phylink *pl, unsigned int phy_id,
+			    unsigned int reg)
+{
+	struct phy_device *phydev = pl->phydev;
+	int prtad, devad;
+
+	if (mdio_phy_id_is_c45(phy_id)) {
+		prtad = mdio_phy_id_prtad(phy_id);
+		devad = mdio_phy_id_devad(phy_id);
+		devad = MII_ADDR_C45 | devad << 16 | reg;
+	} else if (phydev->is_c45) {
+		switch (reg) {
+		case MII_BMCR:
+		case MII_BMSR:
+		case MII_PHYSID1:
+		case MII_PHYSID2:
+			devad = __ffs(phydev->c45_ids.devices_in_package);
+			break;
+		case MII_ADVERTISE:
+		case MII_LPA:
+			if (!(phydev->c45_ids.devices_in_package & MDIO_DEVS_AN))
+				return -EINVAL;
+			devad = MDIO_MMD_AN;
+			if (reg == MII_ADVERTISE)
+				reg = MDIO_AN_ADVERTISE;
+			else
+				reg = MDIO_AN_LPA;
+			break;
+		default:
+			return -EINVAL;
+		}
+		prtad = phy_id;
+		devad = MII_ADDR_C45 | devad << 16 | reg;
+	} else {
+		prtad = phy_id;
+		devad = reg;
+	}
+	return mdiobus_read(pl->phydev->mdio.bus, prtad, devad);
+}
+
+static int phylink_phy_write(struct phylink *pl, unsigned int phy_id,
+			     unsigned int reg, unsigned int val)
+{
+	struct phy_device *phydev = pl->phydev;
+	int prtad, devad;
+
+	if (mdio_phy_id_is_c45(phy_id)) {
+		prtad = mdio_phy_id_prtad(phy_id);
+		devad = mdio_phy_id_devad(phy_id);
+		devad = MII_ADDR_C45 | devad << 16 | reg;
+	} else if (phydev->is_c45) {
+		switch (reg) {
+		case MII_BMCR:
+		case MII_BMSR:
+		case MII_PHYSID1:
+		case MII_PHYSID2:
+			devad = __ffs(phydev->c45_ids.devices_in_package);
+			break;
+		case MII_ADVERTISE:
+		case MII_LPA:
+			if (!(phydev->c45_ids.devices_in_package & MDIO_DEVS_AN))
+				return -EINVAL;
+			devad = MDIO_MMD_AN;
+			if (reg == MII_ADVERTISE)
+				reg = MDIO_AN_ADVERTISE;
+			else
+				reg = MDIO_AN_LPA;
+			break;
+		default:
+			return -EINVAL;
+		}
+		prtad = phy_id;
+		devad = MII_ADDR_C45 | devad << 16 | reg;
+	} else {
+		prtad = phy_id;
+		devad = reg;
+	}
+
+	return mdiobus_write(phydev->mdio.bus, prtad, devad, val);
+}
+
+static int phylink_mii_read(struct phylink *pl, unsigned int phy_id,
+			    unsigned int reg)
+{
+	struct phylink_link_state state;
+	int val = 0xffff;
+
+	switch (pl->link_an_mode) {
+	case MLO_AN_FIXED:
+		if (phy_id == 0) {
+			phylink_get_fixed_state(pl, &state);
+			val = phylink_mii_emul_read(pl->netdev, reg, &state,
+						    true);
+		}
+		break;
+
+	case MLO_AN_PHY:
+		return -EOPNOTSUPP;
+
+	case MLO_AN_SGMII:
+		/* No phy, fall through to 8023z method */
+	case MLO_AN_8023Z:
+		if (phy_id == 0) {
+			val = phylink_get_mac_state(pl, &state);
+			if (val < 0)
+				return val;
+
+			val = phylink_mii_emul_read(pl->netdev, reg, &state,
+						    true);
+		}
+		break;
+	}
+
+	return val & 0xffff;
+}
+
+static int phylink_mii_write(struct phylink *pl, unsigned int phy_id,
+			     unsigned int reg, unsigned int val)
+{
+	switch (pl->link_an_mode) {
+	case MLO_AN_FIXED:
+		break;
+
+	case MLO_AN_PHY:
+		return -EOPNOTSUPP;
+
+	case MLO_AN_SGMII:
+		/* No phy, fall through to 8023z method */
+	case MLO_AN_8023Z:
+		break;
+	}
+
+	return 0;
+}
+
+int phylink_mii_ioctl(struct phylink *pl, struct ifreq *ifr, int cmd)
+{
+	struct mii_ioctl_data *mii = if_mii(ifr);
+	int  ret;
+
+	WARN_ON(!lockdep_rtnl_is_held());
+
+	if (pl->phydev) {
+		/* PHYs only exist for MLO_AN_PHY and MLO_AN_SGMII */
+		switch (cmd) {
+		case SIOCGMIIPHY:
+			mii->phy_id = pl->phydev->mdio.addr;
+
+		case SIOCGMIIREG:
+			ret = phylink_phy_read(pl, mii->phy_id, mii->reg_num);
+			if (ret >= 0) {
+				mii->val_out = ret;
+				ret = 0;
+			}
+			break;
+
+		case SIOCSMIIREG:
+			ret = phylink_phy_write(pl, mii->phy_id, mii->reg_num,
+						mii->val_in);
+			break;
+
+		default:
+			ret = phy_mii_ioctl(pl->phydev, ifr, cmd);
+			break;
+		}
+	} else {
+		switch (cmd) {
+		case SIOCGMIIPHY:
+			mii->phy_id = 0;
+
+		case SIOCGMIIREG:
+			ret = phylink_mii_read(pl, mii->phy_id, mii->reg_num);
+			if (ret >= 0) {
+				mii->val_out = ret;
+				ret = 0;
+			}
+			break;
+
+		case SIOCSMIIREG:
+			ret = phylink_mii_write(pl, mii->phy_id, mii->reg_num,
+						mii->val_in);
+			break;
+
+		default:
+			ret = -EOPNOTSUPP;
+			break;
+		}
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(phylink_mii_ioctl);
+
+
+
+static int phylink_sfp_module_insert(void *upstream,
+				     const struct sfp_eeprom_id *id)
+{
+	struct phylink *pl = upstream;
+	__ETHTOOL_DECLARE_LINK_MODE_MASK(support) = { 0, };
+	struct phylink_link_state config;
+	phy_interface_t iface;
+	int mode, ret = 0;
+	bool changed;
+	u8 port;
+
+	sfp_parse_support(pl->sfp_bus, id, support);
+	port = sfp_parse_port(pl->sfp_bus, id, support);
+	iface = sfp_parse_interface(pl->sfp_bus, id);
+
+	WARN_ON(!lockdep_rtnl_is_held());
+
+	switch (iface) {
+	case PHY_INTERFACE_MODE_SGMII:
+		mode = MLO_AN_SGMII;
+		break;
+	case PHY_INTERFACE_MODE_1000BASEX:
+		mode = MLO_AN_8023Z;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	memset(&config, 0, sizeof(config));
+	linkmode_copy(config.advertising, support);
+	config.interface = iface;
+	config.speed = SPEED_UNKNOWN;
+	config.duplex = DUPLEX_UNKNOWN;
+	config.pause = MLO_PAUSE_AN;
+	config.an_enabled = pl->link_config.an_enabled;
+
+	/* Ignore errors if we're expecting a PHY to attach later */
+	ret = phylink_validate(pl, support, &config);
+	if (ret) {
+		netdev_err(pl->netdev, "validation of %s/%s with support %*pb failed: %d\n",
+			   phylink_an_mode_str(mode), phy_modes(config.interface),
+			   __ETHTOOL_LINK_MODE_MASK_NBITS, support, ret);
+		return ret;
+	}
+
+	netdev_dbg(pl->netdev, "requesting link mode %s/%s with support %*pb\n",
+		   phylink_an_mode_str(mode), phy_modes(config.interface),
+		   __ETHTOOL_LINK_MODE_MASK_NBITS, support);
+
+	if (mode == MLO_AN_8023Z && pl->phydev)
+		return -EINVAL;
+
+	changed = !bitmap_equal(pl->supported, support,
+				__ETHTOOL_LINK_MODE_MASK_NBITS);
+	if (changed) {
+		linkmode_copy(pl->supported, support);
+		linkmode_copy(pl->link_config.advertising, config.advertising);
+	}
+
+	if (pl->link_an_mode != mode ||
+	    pl->link_config.interface != config.interface) {
+		pl->link_config.interface = config.interface;
+		pl->link_an_mode = mode;
+
+		changed = true;
+
+		netdev_info(pl->netdev, "switched to %s/%s link mode\n",
+			    phylink_an_mode_str(mode),
+			    phy_modes(config.interface));
+	}
+
+	pl->link_port = port;
+
+	if (changed && !test_bit(PHYLINK_DISABLE_STOPPED,
+				 &pl->phylink_disable_state))
+		phylink_mac_config(pl, &pl->link_config);
+
+	return ret;
+}
+
+static void phylink_sfp_link_down(void *upstream)
+{
+	struct phylink *pl = upstream;
+
+	WARN_ON(!lockdep_rtnl_is_held());
+
+	set_bit(PHYLINK_DISABLE_LINK, &pl->phylink_disable_state);
+	flush_work(&pl->resolve);
+
+	netif_carrier_off(pl->netdev);
+}
+
+static void phylink_sfp_link_up(void *upstream)
+{
+	struct phylink *pl = upstream;
+
+	WARN_ON(!lockdep_rtnl_is_held());
+
+	clear_bit(PHYLINK_DISABLE_LINK, &pl->phylink_disable_state);
+	phylink_run_resolve(pl);
+}
+
+static int phylink_sfp_connect_phy(void *upstream, struct phy_device *phy)
+{
+	return phylink_connect_phy(upstream, phy);
+}
+
+static void phylink_sfp_disconnect_phy(void *upstream)
+{
+	phylink_disconnect_phy(upstream);
+}
+
+static const struct sfp_upstream_ops sfp_phylink_ops = {
+	.module_insert = phylink_sfp_module_insert,
+	.link_up = phylink_sfp_link_up,
+	.link_down = phylink_sfp_link_down,
+	.connect_phy = phylink_sfp_connect_phy,
+	.disconnect_phy = phylink_sfp_disconnect_phy,
+};
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/phy/rockchip.c b/drivers/net/phy/rockchip.c
new file mode 100644
index 000000000000..c092af137056
--- /dev/null
+++ b/drivers/net/phy/rockchip.c
@@ -0,0 +1,233 @@
+/**
+ * drivers/net/phy/rockchip.c
+ *
+ * Driver for ROCKCHIP Ethernet PHYs
+ *
+ * Copyright (c) 2017, Fuzhou Rockchip Electronics Co., Ltd
+ *
+ * David Wu <david.wu@rock-chips.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ */
+
+#include <linux/ethtool.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mii.h>
+#include <linux/netdevice.h>
+#include <linux/phy.h>
+
+#define INTERNAL_EPHY_ID			0x1234d400
+
+#define MII_INTERNAL_CTRL_STATUS		17
+#define SMI_ADDR_TSTCNTL			20
+#define SMI_ADDR_TSTREAD1			21
+#define SMI_ADDR_TSTREAD2			22
+#define SMI_ADDR_TSTWRITE			23
+#define MII_SPECIAL_CONTROL_STATUS		31
+
+#define MII_AUTO_MDIX_EN			BIT(7)
+#define MII_MDIX_EN				BIT(6)
+
+#define MII_SPEED_10				BIT(2)
+#define MII_SPEED_100				BIT(3)
+
+#define TSTCNTL_RD				(BIT(15) | BIT(10))
+#define TSTCNTL_WR				(BIT(14) | BIT(10))
+
+#define TSTMODE_ENABLE				0x400
+#define TSTMODE_DISABLE				0x0
+
+#define WR_ADDR_A7CFG				0x18
+
+static int rockchip_init_tstmode(struct phy_device *phydev)
+{
+	int ret;
+
+	/* Enable access to Analog and DSP register banks */
+	ret = phy_write(phydev, SMI_ADDR_TSTCNTL, TSTMODE_ENABLE);
+	if (ret)
+		return ret;
+
+	ret = phy_write(phydev, SMI_ADDR_TSTCNTL, TSTMODE_DISABLE);
+	if (ret)
+		return ret;
+
+	return phy_write(phydev, SMI_ADDR_TSTCNTL, TSTMODE_ENABLE);
+}
+
+static int rockchip_close_tstmode(struct phy_device *phydev)
+{
+	/* Back to basic register bank */
+	return phy_write(phydev, SMI_ADDR_TSTCNTL, TSTMODE_DISABLE);
+}
+
+static int rockchip_integrated_phy_analog_init(struct phy_device *phydev)
+{
+	int ret;
+
+	ret = rockchip_init_tstmode(phydev);
+	if (ret)
+		return ret;
+
+	/*
+	 * Adjust tx amplitude to make sginal better,
+	 * the default value is 0x8.
+	 */
+	ret = phy_write(phydev, SMI_ADDR_TSTWRITE, 0xB);
+	if (ret)
+		return ret;
+	ret = phy_write(phydev, SMI_ADDR_TSTCNTL, TSTCNTL_WR | WR_ADDR_A7CFG);
+	if (ret)
+		return ret;
+
+	return rockchip_close_tstmode(phydev);
+}
+
+static int rockchip_integrated_phy_config_init(struct phy_device *phydev)
+{
+	int val, ret;
+
+	/*
+	 * The auto MIDX has linked problem on some board,
+	 * workround to disable auto MDIX.
+	 */
+	val = phy_read(phydev, MII_INTERNAL_CTRL_STATUS);
+	if (val < 0)
+		return val;
+	val &= ~MII_AUTO_MDIX_EN;
+	ret = phy_write(phydev, MII_INTERNAL_CTRL_STATUS, val);
+	if (ret)
+		return ret;
+
+	return rockchip_integrated_phy_analog_init(phydev);
+}
+
+static void rockchip_link_change_notify(struct phy_device *phydev)
+{
+	int speed = SPEED_10;
+
+	if (phydev->autoneg == AUTONEG_ENABLE) {
+		int reg = phy_read(phydev, MII_SPECIAL_CONTROL_STATUS);
+
+		if (reg < 0) {
+			phydev_err(phydev, "phy_read err: %d.\n", reg);
+			return;
+		}
+
+		if (reg & MII_SPEED_100)
+			speed = SPEED_100;
+		else if (reg & MII_SPEED_10)
+			speed = SPEED_10;
+	} else {
+		int bmcr = phy_read(phydev, MII_BMCR);
+
+		if (bmcr < 0) {
+			phydev_err(phydev, "phy_read err: %d.\n", bmcr);
+			return;
+		}
+
+		if (bmcr & BMCR_SPEED100)
+			speed = SPEED_100;
+		else
+			speed = SPEED_10;
+	}
+
+	/*
+	 * If mode switch happens from 10BT to 100BT, all DSP/AFE
+	 * registers are set to default values. So any AFE/DSP
+	 * registers have to be re-initialized in this case.
+	 */
+	if ((phydev->speed == SPEED_10) && (speed == SPEED_100)) {
+		int ret = rockchip_integrated_phy_analog_init(phydev);
+		if (ret)
+			phydev_err(phydev, "rockchip_integrated_phy_analog_init err: %d.\n",
+				   ret);
+	}
+}
+
+static int rockchip_set_polarity(struct phy_device *phydev, int polarity)
+{
+	int reg, err, val;
+
+	/* get the current settings */
+	reg = phy_read(phydev, MII_INTERNAL_CTRL_STATUS);
+	if (reg < 0)
+		return reg;
+
+	reg &= ~MII_AUTO_MDIX_EN;
+	val = reg;
+	switch (polarity) {
+	case ETH_TP_MDI:
+		val &= ~MII_MDIX_EN;
+		break;
+	case ETH_TP_MDI_X:
+		val |= MII_MDIX_EN;
+		break;
+	case ETH_TP_MDI_AUTO:
+	case ETH_TP_MDI_INVALID:
+	default:
+		return 0;
+	}
+
+	if (val != reg) {
+		/* Set the new polarity value in the register */
+		err = phy_write(phydev, MII_INTERNAL_CTRL_STATUS, val);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+static int rockchip_config_aneg(struct phy_device *phydev)
+{
+	int err;
+
+	err = rockchip_set_polarity(phydev, phydev->mdix);
+	if (err < 0)
+		return err;
+
+	return genphy_config_aneg(phydev);
+}
+
+static int rockchip_phy_resume(struct phy_device *phydev)
+{
+	genphy_resume(phydev);
+
+	return rockchip_integrated_phy_config_init(phydev);
+}
+
+static struct phy_driver rockchip_phy_driver[] = {
+{
+	.phy_id			= INTERNAL_EPHY_ID,
+	.phy_id_mask		= 0xfffffff0,
+	.name			= "Rockchip integrated EPHY",
+	.features		= PHY_BASIC_FEATURES,
+	.flags			= 0,
+	.link_change_notify	= rockchip_link_change_notify,
+	.soft_reset		= genphy_soft_reset,
+	.config_init		= rockchip_integrated_phy_config_init,
+	.config_aneg		= rockchip_config_aneg,
+	.read_status		= genphy_read_status,
+	.suspend		= genphy_suspend,
+	.resume			= rockchip_phy_resume,
+},
+};
+
+module_phy_driver(rockchip_phy_driver);
+
+static struct mdio_device_id __maybe_unused rockchip_phy_tbl[] = {
+	{ INTERNAL_EPHY_ID, 0xfffffff0 },
+	{ }
+};
+
+MODULE_DEVICE_TABLE(mdio, rockchip_phy_tbl);
+
+MODULE_AUTHOR("David Wu <david.wu@rock-chips.com>");
+MODULE_DESCRIPTION("Rockchip Ethernet PHY driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/net/phy/sfp-bus.c b/drivers/net/phy/sfp-bus.c
new file mode 100644
index 000000000000..5cb5384697ea
--- /dev/null
+++ b/drivers/net/phy/sfp-bus.c
@@ -0,0 +1,475 @@
+#include <linux/export.h>
+#include <linux/kref.h>
+#include <linux/list.h>
+#include <linux/mutex.h>
+#include <linux/phylink.h>
+#include <linux/rtnetlink.h>
+#include <linux/slab.h>
+
+#include "sfp.h"
+
+struct sfp_bus {
+	struct kref kref;
+	struct list_head node;
+	struct device_node *device_node;
+
+	const struct sfp_socket_ops *socket_ops;
+	struct device *sfp_dev;
+	struct sfp *sfp;
+
+	const struct sfp_upstream_ops *upstream_ops;
+	void *upstream;
+	struct net_device *netdev;
+	struct phy_device *phydev;
+
+	bool registered;
+	bool started;
+};
+
+
+int sfp_parse_port(struct sfp_bus *bus, const struct sfp_eeprom_id *id,
+		   unsigned long *support)
+{
+	int port;
+
+	/* port is the physical connector, set this from the connector field. */
+	switch (id->base.connector) {
+	case SFP_CONNECTOR_SC:
+	case SFP_CONNECTOR_FIBERJACK:
+	case SFP_CONNECTOR_LC:
+	case SFP_CONNECTOR_MT_RJ:
+	case SFP_CONNECTOR_MU:
+	case SFP_CONNECTOR_OPTICAL_PIGTAIL:
+		if (support)
+			phylink_set(support, FIBRE);
+		port = PORT_FIBRE;
+		break;
+
+	case SFP_CONNECTOR_RJ45:
+		if (support)
+			phylink_set(support, TP);
+		port = PORT_TP;
+		break;
+
+	case SFP_CONNECTOR_UNSPEC:
+		if (id->base.e1000_base_t) {
+			if (support)
+				phylink_set(support, TP);
+			port = PORT_TP;
+			break;
+		}
+		/* fallthrough */
+	case SFP_CONNECTOR_SG: /* guess */
+	case SFP_CONNECTOR_MPO_1X12:
+	case SFP_CONNECTOR_MPO_2X16:
+	case SFP_CONNECTOR_HSSDC_II:
+	case SFP_CONNECTOR_COPPER_PIGTAIL:
+	case SFP_CONNECTOR_NOSEPARATE:
+	case SFP_CONNECTOR_MXC_2X16:
+		port = PORT_OTHER;
+		break;
+	default:
+		dev_warn(bus->sfp_dev, "SFP: unknown connector id 0x%02x\n",
+			 id->base.connector);
+		port = PORT_OTHER;
+		break;
+	}
+
+	return port;
+}
+EXPORT_SYMBOL_GPL(sfp_parse_port);
+
+phy_interface_t sfp_parse_interface(struct sfp_bus *bus,
+				    const struct sfp_eeprom_id *id)
+{
+	phy_interface_t iface;
+
+	/* Setting the serdes link mode is guesswork: there's no field in
+	 * the EEPROM which indicates what mode should be used.
+	 *
+	 * If the module wants 64b66b, then it must be >= 10G.
+	 *
+	 * If it's a gigabit-only fiber module, it probably does not have
+	 * a PHY, so switch to 802.3z negotiation mode. Otherwise, switch
+	 * to SGMII mode (which is required to support non-gigabit speeds).
+	 */
+	switch (id->base.encoding) {
+	case SFP_ENCODING_8472_64B66B:
+		iface = PHY_INTERFACE_MODE_10GKR;
+		break;
+
+	case SFP_ENCODING_8B10B:
+		if (!id->base.e1000_base_t &&
+		    !id->base.e100_base_lx &&
+		    !id->base.e100_base_fx)
+			iface = PHY_INTERFACE_MODE_1000BASEX;
+		else
+			iface = PHY_INTERFACE_MODE_SGMII;
+		break;
+
+	default:
+		iface = PHY_INTERFACE_MODE_NA;
+		dev_err(bus->sfp_dev,
+			"SFP module encoding does not support 8b10b nor 64b66b\n");
+		break;
+	}
+
+	return iface;
+}
+EXPORT_SYMBOL_GPL(sfp_parse_interface);
+
+void sfp_parse_support(struct sfp_bus *bus, const struct sfp_eeprom_id *id,
+		       unsigned long *support)
+{
+	phylink_set(support, Autoneg);
+	phylink_set(support, Pause);
+	phylink_set(support, Asym_Pause);
+
+	/* Set ethtool support from the compliance fields. */
+	if (id->base.e10g_base_sr)
+		phylink_set(support, 10000baseSR_Full);
+	if (id->base.e10g_base_lr)
+		phylink_set(support, 10000baseLR_Full);
+	if (id->base.e10g_base_lrm)
+		phylink_set(support, 10000baseLRM_Full);
+	if (id->base.e10g_base_er)
+		phylink_set(support, 10000baseER_Full);
+	if (id->base.e1000_base_sx ||
+	    id->base.e1000_base_lx ||
+	    id->base.e1000_base_cx)
+		phylink_set(support, 1000baseX_Full);
+	if (id->base.e1000_base_t) {
+		phylink_set(support, 1000baseT_Half);
+		phylink_set(support, 1000baseT_Full);
+	}
+
+	switch (id->base.extended_cc) {
+	case 0x00: /* Unspecified */
+		break;
+	case 0x02: /* 100Gbase-SR4 or 25Gbase-SR */
+		phylink_set(support, 100000baseSR4_Full);
+		phylink_set(support, 25000baseSR_Full);
+		break;
+	case 0x03: /* 100Gbase-LR4 or 25Gbase-LR */
+	case 0x04: /* 100Gbase-ER4 or 25Gbase-ER */
+		phylink_set(support, 100000baseLR4_ER4_Full);
+		break;
+	case 0x0b: /* 100Gbase-CR4 or 25Gbase-CR CA-L */
+	case 0x0c: /* 25Gbase-CR CA-S */
+	case 0x0d: /* 25Gbase-CR CA-N */
+		phylink_set(support, 100000baseCR4_Full);
+		phylink_set(support, 25000baseCR_Full);
+		break;
+	default:
+		dev_warn(bus->sfp_dev,
+			 "Unknown/unsupported extended compliance code: 0x%02x\n",
+			 id->base.extended_cc);
+		break;
+	}
+
+	/* For fibre channel SFP, derive possible BaseX modes */
+	if (id->base.fc_speed_100 ||
+	    id->base.fc_speed_200 ||
+	    id->base.fc_speed_400) {
+		if (id->base.br_nominal >= 31)
+			phylink_set(support, 2500baseX_Full);
+		if (id->base.br_nominal >= 12)
+			phylink_set(support, 1000baseX_Full);
+	}
+
+	switch (id->base.connector) {
+	case SFP_CONNECTOR_SC:
+	case SFP_CONNECTOR_FIBERJACK:
+	case SFP_CONNECTOR_LC:
+	case SFP_CONNECTOR_MT_RJ:
+	case SFP_CONNECTOR_MU:
+	case SFP_CONNECTOR_OPTICAL_PIGTAIL:
+		break;
+
+	case SFP_CONNECTOR_UNSPEC:
+		if (id->base.e1000_base_t)
+			break;
+
+	case SFP_CONNECTOR_SG: /* guess */
+	case SFP_CONNECTOR_MPO_1X12:
+	case SFP_CONNECTOR_MPO_2X16:
+	case SFP_CONNECTOR_HSSDC_II:
+	case SFP_CONNECTOR_COPPER_PIGTAIL:
+	case SFP_CONNECTOR_NOSEPARATE:
+	case SFP_CONNECTOR_MXC_2X16:
+	default:
+		/* a guess at the supported link modes */
+		dev_warn(bus->sfp_dev,
+			 "Guessing link modes, please report...\n");
+		phylink_set(support, 1000baseT_Half);
+		phylink_set(support, 1000baseT_Full);
+		break;
+	}
+}
+EXPORT_SYMBOL_GPL(sfp_parse_support);
+
+
+static LIST_HEAD(sfp_buses);
+static DEFINE_MUTEX(sfp_mutex);
+
+static const struct sfp_upstream_ops *sfp_get_upstream_ops(struct sfp_bus *bus)
+{
+	return bus->registered ? bus->upstream_ops : NULL;
+}
+
+static struct sfp_bus *sfp_bus_get(struct device_node *np)
+{
+	struct sfp_bus *sfp, *new, *found = NULL;
+
+	new = kzalloc(sizeof(*new), GFP_KERNEL);
+
+	mutex_lock(&sfp_mutex);
+
+	list_for_each_entry(sfp, &sfp_buses, node) {
+		if (sfp->device_node == np) {
+			kref_get(&sfp->kref);
+			found = sfp;
+			break;
+		}
+	}
+
+	if (!found && new) {
+		kref_init(&new->kref);
+		new->device_node = np;
+		list_add(&new->node, &sfp_buses);
+		found = new;
+		new = NULL;
+	}
+
+	mutex_unlock(&sfp_mutex);
+
+	kfree(new);
+
+	return found;
+}
+
+static void sfp_bus_release(struct kref *kref) __releases(sfp_mutex)
+{
+	struct sfp_bus *bus = container_of(kref, struct sfp_bus, kref);
+
+	list_del(&bus->node);
+	mutex_unlock(&sfp_mutex);
+	kfree(bus);
+}
+
+static void sfp_bus_put(struct sfp_bus *bus)
+{
+	kref_put_mutex(&bus->kref, sfp_bus_release, &sfp_mutex);
+}
+
+static int sfp_register_bus(struct sfp_bus *bus)
+{
+	const struct sfp_upstream_ops *ops = bus->upstream_ops;
+	int ret;
+
+	if (ops) {
+		if (ops->link_down)
+			ops->link_down(bus->upstream);
+		if (ops->connect_phy && bus->phydev) {
+			ret = ops->connect_phy(bus->upstream, bus->phydev);
+			if (ret)
+				return ret;
+		}
+	}
+	if (bus->started)
+		bus->socket_ops->start(bus->sfp);
+	bus->registered = true;
+	return 0;
+}
+
+static void sfp_unregister_bus(struct sfp_bus *bus)
+{
+	const struct sfp_upstream_ops *ops = bus->upstream_ops;
+
+	if (bus->registered) {
+		if (bus->started)
+			bus->socket_ops->stop(bus->sfp);
+		if (bus->phydev && ops && ops->disconnect_phy)
+			ops->disconnect_phy(bus->upstream);
+	}
+	bus->registered = false;
+}
+
+
+int sfp_get_module_info(struct sfp_bus *bus, struct ethtool_modinfo *modinfo)
+{
+	if (!bus->registered)
+		return -ENOIOCTLCMD;
+	return bus->socket_ops->module_info(bus->sfp, modinfo);
+}
+EXPORT_SYMBOL_GPL(sfp_get_module_info);
+
+int sfp_get_module_eeprom(struct sfp_bus *bus, struct ethtool_eeprom *ee,
+	u8 *data)
+{
+	if (!bus->registered)
+		return -ENOIOCTLCMD;
+	return bus->socket_ops->module_eeprom(bus->sfp, ee, data);
+}
+EXPORT_SYMBOL_GPL(sfp_get_module_eeprom);
+
+void sfp_upstream_start(struct sfp_bus *bus)
+{
+	if (bus->registered)
+		bus->socket_ops->start(bus->sfp);
+	bus->started = true;
+}
+EXPORT_SYMBOL_GPL(sfp_upstream_start);
+
+void sfp_upstream_stop(struct sfp_bus *bus)
+{
+	if (bus->registered)
+		bus->socket_ops->stop(bus->sfp);
+	bus->started = false;
+}
+EXPORT_SYMBOL_GPL(sfp_upstream_stop);
+
+struct sfp_bus *sfp_register_upstream(struct device_node *np,
+	struct net_device *ndev, void *upstream,
+	const struct sfp_upstream_ops *ops)
+{
+	struct sfp_bus *bus = sfp_bus_get(np);
+	int ret = 0;
+
+	if (bus) {
+		rtnl_lock();
+		bus->upstream_ops = ops;
+		bus->upstream = upstream;
+		bus->netdev = ndev;
+
+		if (bus->sfp)
+			ret = sfp_register_bus(bus);
+		rtnl_unlock();
+	}
+
+	if (ret) {
+		sfp_bus_put(bus);
+		bus = NULL;
+	}
+
+	return bus;
+}
+EXPORT_SYMBOL_GPL(sfp_register_upstream);
+
+void sfp_unregister_upstream(struct sfp_bus *bus)
+{
+	rtnl_lock();
+	sfp_unregister_bus(bus);
+	bus->upstream = NULL;
+	bus->netdev = NULL;
+	rtnl_unlock();
+
+	sfp_bus_put(bus);
+}
+EXPORT_SYMBOL_GPL(sfp_unregister_upstream);
+
+
+/* Socket driver entry points */
+int sfp_add_phy(struct sfp_bus *bus, struct phy_device *phydev)
+{
+	const struct sfp_upstream_ops *ops = sfp_get_upstream_ops(bus);
+	int ret = 0;
+
+	if (ops && ops->connect_phy)
+		ret = ops->connect_phy(bus->upstream, phydev);
+
+	if (ret == 0)
+		bus->phydev = phydev;
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(sfp_add_phy);
+
+void sfp_remove_phy(struct sfp_bus *bus)
+{
+	const struct sfp_upstream_ops *ops = sfp_get_upstream_ops(bus);
+
+	if (ops && ops->disconnect_phy)
+		ops->disconnect_phy(bus->upstream);
+	bus->phydev = NULL;
+}
+EXPORT_SYMBOL_GPL(sfp_remove_phy);
+
+
+void sfp_link_up(struct sfp_bus *bus)
+{
+	const struct sfp_upstream_ops *ops = sfp_get_upstream_ops(bus);
+
+	if (ops && ops->link_up)
+		ops->link_up(bus->upstream);
+}
+EXPORT_SYMBOL_GPL(sfp_link_up);
+
+void sfp_link_down(struct sfp_bus *bus)
+{
+	const struct sfp_upstream_ops *ops = sfp_get_upstream_ops(bus);
+
+	if (ops && ops->link_down)
+		ops->link_down(bus->upstream);
+}
+EXPORT_SYMBOL_GPL(sfp_link_down);
+
+int sfp_module_insert(struct sfp_bus *bus, const struct sfp_eeprom_id *id)
+{
+	const struct sfp_upstream_ops *ops = sfp_get_upstream_ops(bus);
+	int ret = 0;
+
+	if (ops && ops->module_insert)
+		ret = ops->module_insert(bus->upstream, id);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(sfp_module_insert);
+
+void sfp_module_remove(struct sfp_bus *bus)
+{
+	const struct sfp_upstream_ops *ops = sfp_get_upstream_ops(bus);
+
+	if (ops && ops->module_remove)
+		ops->module_remove(bus->upstream);
+}
+EXPORT_SYMBOL_GPL(sfp_module_remove);
+
+struct sfp_bus *sfp_register_socket(struct device *dev, struct sfp *sfp,
+				    const struct sfp_socket_ops *ops)
+{
+	struct sfp_bus *bus = sfp_bus_get(dev->of_node);
+	int ret = 0;
+
+	if (bus) {
+		rtnl_lock();
+		bus->sfp_dev = dev;
+		bus->sfp = sfp;
+		bus->socket_ops = ops;
+
+		if (bus->netdev)
+			ret = sfp_register_bus(bus);
+		rtnl_unlock();
+	}
+
+	if (ret) {
+		sfp_bus_put(bus);
+		bus = NULL;
+	}
+
+	return bus;
+}
+EXPORT_SYMBOL_GPL(sfp_register_socket);
+
+void sfp_unregister_socket(struct sfp_bus *bus)
+{
+	rtnl_lock();
+	sfp_unregister_bus(bus);
+	bus->sfp_dev = NULL;
+	bus->sfp = NULL;
+	bus->socket_ops = NULL;
+	rtnl_unlock();
+
+	sfp_bus_put(bus);
+}
+EXPORT_SYMBOL_GPL(sfp_unregister_socket);
diff --git a/drivers/net/phy/sfp.c b/drivers/net/phy/sfp.c
new file mode 100644
index 000000000000..baee371bf767
--- /dev/null
+++ b/drivers/net/phy/sfp.c
@@ -0,0 +1,915 @@
+#include <linux/delay.h>
+#include <linux/gpio.h>
+#include <linux/i2c.h>
+#include <linux/interrupt.h>
+#include <linux/jiffies.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/phy.h>
+#include <linux/platform_device.h>
+#include <linux/rtnetlink.h>
+#include <linux/slab.h>
+#include <linux/workqueue.h>
+
+#include "mdio-i2c.h"
+#include "sfp.h"
+#include "swphy.h"
+
+enum {
+	GPIO_MODDEF0,
+	GPIO_LOS,
+	GPIO_TX_FAULT,
+	GPIO_TX_DISABLE,
+	GPIO_RATE_SELECT,
+	GPIO_MAX,
+
+	SFP_F_PRESENT = BIT(GPIO_MODDEF0),
+	SFP_F_LOS = BIT(GPIO_LOS),
+	SFP_F_TX_FAULT = BIT(GPIO_TX_FAULT),
+	SFP_F_TX_DISABLE = BIT(GPIO_TX_DISABLE),
+	SFP_F_RATE_SELECT = BIT(GPIO_RATE_SELECT),
+
+	SFP_E_INSERT = 0,
+	SFP_E_REMOVE,
+	SFP_E_DEV_DOWN,
+	SFP_E_DEV_UP,
+	SFP_E_TX_FAULT,
+	SFP_E_TX_CLEAR,
+	SFP_E_LOS_HIGH,
+	SFP_E_LOS_LOW,
+	SFP_E_TIMEOUT,
+
+	SFP_MOD_EMPTY = 0,
+	SFP_MOD_PROBE,
+	SFP_MOD_PRESENT,
+	SFP_MOD_ERROR,
+
+	SFP_DEV_DOWN = 0,
+	SFP_DEV_UP,
+
+	SFP_S_DOWN = 0,
+	SFP_S_INIT,
+	SFP_S_WAIT_LOS,
+	SFP_S_LINK_UP,
+	SFP_S_TX_FAULT,
+	SFP_S_REINIT,
+	SFP_S_TX_DISABLE,
+};
+
+static const char *gpio_of_names[] = {
+	"mod-def0",
+	"los",
+	"tx-fault",
+	"tx-disable",
+	"rate-select0",
+};
+
+static const enum gpiod_flags gpio_flags[] = {
+	GPIOD_IN,
+	GPIOD_IN,
+	GPIOD_IN,
+	GPIOD_ASIS,
+	GPIOD_ASIS,
+};
+
+#define T_INIT_JIFFIES	msecs_to_jiffies(300)
+#define T_RESET_US	10
+#define T_FAULT_RECOVER	msecs_to_jiffies(1000)
+
+/* SFP module presence detection is poor: the three MOD DEF signals are
+ * the same length on the PCB, which means it's possible for MOD DEF 0 to
+ * connect before the I2C bus on MOD DEF 1/2.
+ *
+ * The SFP MSA specifies 300ms as t_init (the time taken for TX_FAULT to
+ * be deasserted) but makes no mention of the earliest time before we can
+ * access the I2C EEPROM.  However, Avago modules require 300ms.
+ */
+#define T_PROBE_INIT	msecs_to_jiffies(300)
+#define T_PROBE_RETRY	msecs_to_jiffies(100)
+
+/*
+ * SFP modules appear to always have their PHY configured for bus address
+ * 0x56 (which with mdio-i2c, translates to a PHY address of 22).
+ */
+#define SFP_PHY_ADDR	22
+
+/*
+ * Give this long for the PHY to reset.
+ */
+#define T_PHY_RESET_MS	50
+
+static DEFINE_MUTEX(sfp_mutex);
+
+struct sfp {
+	struct device *dev;
+	struct i2c_adapter *i2c;
+	struct mii_bus *i2c_mii;
+	struct sfp_bus *sfp_bus;
+	struct phy_device *mod_phy;
+
+	unsigned int (*get_state)(struct sfp *);
+	void (*set_state)(struct sfp *, unsigned int);
+	int (*read)(struct sfp *, bool, u8, void *, size_t);
+
+	struct gpio_desc *gpio[GPIO_MAX];
+
+	unsigned int state;
+	struct delayed_work poll;
+	struct delayed_work timeout;
+	struct mutex sm_mutex;
+	unsigned char sm_mod_state;
+	unsigned char sm_dev_state;
+	unsigned short sm_state;
+	unsigned int sm_retries;
+
+	struct sfp_eeprom_id id;
+};
+
+static unsigned long poll_jiffies;
+
+static unsigned int sfp_gpio_get_state(struct sfp *sfp)
+{
+	unsigned int i, state, v;
+
+	for (i = state = 0; i < GPIO_MAX; i++) {
+		if (gpio_flags[i] != GPIOD_IN || !sfp->gpio[i])
+			continue;
+
+		v = gpiod_get_value_cansleep(sfp->gpio[i]);
+		if (v)
+			state |= BIT(i);
+	}
+
+	return state;
+}
+
+static void sfp_gpio_set_state(struct sfp *sfp, unsigned int state)
+{
+	if (state & SFP_F_PRESENT) {
+		/* If the module is present, drive the signals */
+		if (sfp->gpio[GPIO_TX_DISABLE])
+			gpiod_direction_output(sfp->gpio[GPIO_TX_DISABLE],
+						state & SFP_F_TX_DISABLE);
+		if (state & SFP_F_RATE_SELECT)
+			gpiod_direction_output(sfp->gpio[GPIO_RATE_SELECT],
+						state & SFP_F_RATE_SELECT);
+	} else {
+		/* Otherwise, let them float to the pull-ups */
+		if (sfp->gpio[GPIO_TX_DISABLE])
+			gpiod_direction_input(sfp->gpio[GPIO_TX_DISABLE]);
+		if (state & SFP_F_RATE_SELECT)
+			gpiod_direction_input(sfp->gpio[GPIO_RATE_SELECT]);
+	}
+}
+
+static int sfp__i2c_read(struct i2c_adapter *i2c, u8 bus_addr, u8 dev_addr,
+	void *buf, size_t len)
+{
+	struct i2c_msg msgs[2];
+	int ret;
+
+	msgs[0].addr = bus_addr;
+	msgs[0].flags = 0;
+	msgs[0].len = 1;
+	msgs[0].buf = &dev_addr;
+	msgs[1].addr = bus_addr;
+	msgs[1].flags = I2C_M_RD;
+	msgs[1].len = len;
+	msgs[1].buf = buf;
+
+	ret = i2c_transfer(i2c, msgs, ARRAY_SIZE(msgs));
+	if (ret < 0)
+		return ret;
+
+	return ret == ARRAY_SIZE(msgs) ? len : 0;
+}
+
+static int sfp_i2c_read(struct sfp *sfp, bool a2, u8 addr, void *buf,
+	size_t len)
+{
+	return sfp__i2c_read(sfp->i2c, a2 ? 0x51 : 0x50, addr, buf, len);
+}
+
+static int sfp_i2c_configure(struct sfp *sfp, struct i2c_adapter *i2c)
+{
+	struct mii_bus *i2c_mii;
+	int ret;
+
+	if (!i2c_check_functionality(i2c, I2C_FUNC_I2C))
+		return -EINVAL;
+
+	sfp->i2c = i2c;
+	sfp->read = sfp_i2c_read;
+
+	i2c_mii = mdio_i2c_alloc(sfp->dev, i2c);
+	if (IS_ERR(i2c_mii))
+		return PTR_ERR(i2c_mii);
+
+	i2c_mii->name = "SFP I2C Bus";
+	i2c_mii->phy_mask = ~0;
+
+	ret = mdiobus_register(i2c_mii);
+	if (ret < 0) {
+		mdiobus_free(i2c_mii);
+		return ret;
+	}
+
+	sfp->i2c_mii = i2c_mii;
+
+	return 0;
+}
+
+
+/* Interface */
+static unsigned int sfp_get_state(struct sfp *sfp)
+{
+	return sfp->get_state(sfp);
+}
+
+static void sfp_set_state(struct sfp *sfp, unsigned int state)
+{
+	sfp->set_state(sfp, state);
+}
+
+static int sfp_read(struct sfp *sfp, bool a2, u8 addr, void *buf, size_t len)
+{
+	return sfp->read(sfp, a2, addr, buf, len);
+}
+
+static unsigned int sfp_check(void *buf, size_t len)
+{
+	u8 *p, check;
+
+	for (p = buf, check = 0; len; p++, len--)
+		check += *p;
+
+	return check;
+}
+
+/* Helpers */
+static void sfp_module_tx_disable(struct sfp *sfp)
+{
+	dev_dbg(sfp->dev, "tx disable %u -> %u\n",
+		sfp->state & SFP_F_TX_DISABLE ? 1 : 0, 1);
+	sfp->state |= SFP_F_TX_DISABLE;
+	sfp_set_state(sfp, sfp->state);
+}
+
+static void sfp_module_tx_enable(struct sfp *sfp)
+{
+	dev_dbg(sfp->dev, "tx disable %u -> %u\n",
+		sfp->state & SFP_F_TX_DISABLE ? 1 : 0, 0);
+	sfp->state &= ~SFP_F_TX_DISABLE;
+	sfp_set_state(sfp, sfp->state);
+}
+
+static void sfp_module_tx_fault_reset(struct sfp *sfp)
+{
+	unsigned int state = sfp->state;
+
+	if (state & SFP_F_TX_DISABLE)
+		return;
+
+	sfp_set_state(sfp, state | SFP_F_TX_DISABLE);
+
+	udelay(T_RESET_US);
+
+	sfp_set_state(sfp, state);
+}
+
+/* SFP state machine */
+static void sfp_sm_set_timer(struct sfp *sfp, unsigned int timeout)
+{
+	if (timeout)
+		mod_delayed_work(system_power_efficient_wq, &sfp->timeout,
+				 timeout);
+	else
+		cancel_delayed_work(&sfp->timeout);
+}
+
+static void sfp_sm_next(struct sfp *sfp, unsigned int state,
+			unsigned int timeout)
+{
+	sfp->sm_state = state;
+	sfp_sm_set_timer(sfp, timeout);
+}
+
+static void sfp_sm_ins_next(struct sfp *sfp, unsigned int state, unsigned int timeout)
+{
+	sfp->sm_mod_state = state;
+	sfp_sm_set_timer(sfp, timeout);
+}
+
+static void sfp_sm_phy_detach(struct sfp *sfp)
+{
+	phy_stop(sfp->mod_phy);
+	sfp_remove_phy(sfp->sfp_bus);
+	phy_device_remove(sfp->mod_phy);
+	phy_device_free(sfp->mod_phy);
+	sfp->mod_phy = NULL;
+}
+
+static void sfp_sm_probe_phy(struct sfp *sfp)
+{
+	struct phy_device *phy;
+	int err;
+
+	msleep(T_PHY_RESET_MS);
+
+	phy = mdiobus_scan(sfp->i2c_mii, SFP_PHY_ADDR);
+	if (IS_ERR(phy)) {
+		dev_err(sfp->dev, "mdiobus scan returned %ld\n", PTR_ERR(phy));
+		return;
+	}
+	if (!phy) {
+		dev_info(sfp->dev, "no PHY detected\n");
+		return;
+	}
+
+	err = sfp_add_phy(sfp->sfp_bus, phy);
+	if (err) {
+		phy_device_remove(phy);
+		phy_device_free(phy);
+		dev_err(sfp->dev, "sfp_add_phy failed: %d\n", err);
+		return;
+	}
+
+	sfp->mod_phy = phy;
+	phy_start(phy);
+}
+
+static void sfp_sm_link_up(struct sfp *sfp)
+{
+	sfp_link_up(sfp->sfp_bus);
+	sfp_sm_next(sfp, SFP_S_LINK_UP, 0);
+}
+
+static void sfp_sm_link_down(struct sfp *sfp)
+{
+	sfp_link_down(sfp->sfp_bus);
+}
+
+static void sfp_sm_link_check_los(struct sfp *sfp)
+{
+	unsigned int los = sfp->state & SFP_F_LOS;
+
+	/* FIXME: what if neither SFP_OPTIONS_LOS_INVERTED nor
+	 * SFP_OPTIONS_LOS_NORMAL are set?  For now, we assume
+	 * the same as SFP_OPTIONS_LOS_NORMAL set.
+	 */
+	if (sfp->id.ext.options & SFP_OPTIONS_LOS_INVERTED)
+		los ^= SFP_F_LOS;
+
+	if (los)
+		sfp_sm_next(sfp, SFP_S_WAIT_LOS, 0);
+	else
+		sfp_sm_link_up(sfp);
+}
+
+static void sfp_sm_fault(struct sfp *sfp, bool warn)
+{
+	if (sfp->sm_retries && !--sfp->sm_retries) {
+		dev_err(sfp->dev, "module persistently indicates fault, disabling\n");
+		sfp_sm_next(sfp, SFP_S_TX_DISABLE, 0);
+	} else {
+		if (warn)
+			dev_err(sfp->dev, "module transmit fault indicated\n");
+
+		sfp_sm_next(sfp, SFP_S_TX_FAULT, T_FAULT_RECOVER);
+	}
+}
+
+static void sfp_sm_mod_init(struct sfp *sfp)
+{
+	sfp_module_tx_enable(sfp);
+
+	/* Wait t_init before indicating that the link is up, provided the
+	 * current state indicates no TX_FAULT.  If TX_FAULT clears before
+	 * this time, that's fine too.
+	 */
+	sfp_sm_next(sfp, SFP_S_INIT, T_INIT_JIFFIES);
+	sfp->sm_retries = 5;
+
+	/* Setting the serdes link mode is guesswork: there's no
+	 * field in the EEPROM which indicates what mode should
+	 * be used.
+	 *
+	 * If it's a gigabit-only fiber module, it probably does
+	 * not have a PHY, so switch to 802.3z negotiation mode.
+	 * Otherwise, switch to SGMII mode (which is required to
+	 * support non-gigabit speeds) and probe for a PHY.
+	 */
+	if (sfp->id.base.e1000_base_t ||
+	    sfp->id.base.e100_base_lx ||
+	    sfp->id.base.e100_base_fx)
+		sfp_sm_probe_phy(sfp);
+}
+
+static int sfp_sm_mod_probe(struct sfp *sfp)
+{
+	/* SFP module inserted - read I2C data */
+	struct sfp_eeprom_id id;
+	char vendor[17];
+	char part[17];
+	char sn[17];
+	char date[9];
+	char rev[5];
+	u8 check;
+	int err;
+
+	err = sfp_read(sfp, false, 0, &id, sizeof(id));
+	if (err < 0) {
+		dev_err(sfp->dev, "failed to read EEPROM: %d\n", err);
+		return -EAGAIN;
+	}
+
+	if (err != sizeof(id)) {
+		dev_err(sfp->dev, "EEPROM short read: %d\n", err);
+		return -EAGAIN;
+	}
+
+	/* Validate the checksum over the base structure */
+	check = sfp_check(&id.base, sizeof(id.base) - 1);
+	if (check != id.base.cc_base) {
+		dev_err(sfp->dev,
+			"EEPROM base structure checksum failure: 0x%02x\n",
+			check);
+		print_hex_dump(KERN_ERR, "sfp EE: ", DUMP_PREFIX_OFFSET,
+			       16, 1, &id, sizeof(id.base) - 1, true);
+		return -EINVAL;
+	}
+
+	check = sfp_check(&id.ext, sizeof(id.ext) - 1);
+	if (check != id.ext.cc_ext) {
+		dev_err(sfp->dev,
+			"EEPROM extended structure checksum failure: 0x%02x\n",
+			check);
+		memset(&id.ext, 0, sizeof(id.ext));
+	}
+
+	sfp->id = id;
+
+	memcpy(vendor, sfp->id.base.vendor_name, 16);
+	vendor[16] = '\0';
+	memcpy(part, sfp->id.base.vendor_pn, 16);
+	part[16] = '\0';
+	memcpy(rev, sfp->id.base.vendor_rev, 4);
+	rev[4] = '\0';
+	memcpy(sn, sfp->id.ext.vendor_sn, 16);
+	sn[16] = '\0';
+	memcpy(date, sfp->id.ext.datecode, 8);
+	date[8] = '\0';
+
+	dev_info(sfp->dev, "module %s %s rev %s sn %s dc %s\n", vendor, part, rev, sn, date);
+
+	/* We only support SFP modules, not the legacy GBIC modules. */
+	if (sfp->id.base.phys_id != SFP_PHYS_ID_SFP ||
+	    sfp->id.base.phys_ext_id != SFP_PHYS_EXT_ID_SFP) {
+		dev_err(sfp->dev, "module is not SFP - phys id 0x%02x 0x%02x\n",
+			sfp->id.base.phys_id, sfp->id.base.phys_ext_id);
+		return -EINVAL;
+	}
+
+	return sfp_module_insert(sfp->sfp_bus, &sfp->id);
+}
+
+static void sfp_sm_mod_remove(struct sfp *sfp)
+{
+	sfp_module_remove(sfp->sfp_bus);
+
+	if (sfp->mod_phy)
+		sfp_sm_phy_detach(sfp);
+
+	sfp_module_tx_disable(sfp);
+
+	memset(&sfp->id, 0, sizeof(sfp->id));
+
+	dev_info(sfp->dev, "module removed\n");
+}
+
+static void sfp_sm_event(struct sfp *sfp, unsigned int event)
+{
+	mutex_lock(&sfp->sm_mutex);
+
+	dev_dbg(sfp->dev, "SM: enter %u:%u:%u event %u\n",
+		sfp->sm_mod_state, sfp->sm_dev_state, sfp->sm_state, event);
+
+	/* This state machine tracks the insert/remove state of
+	 * the module, and handles probing the on-board EEPROM.
+	 */
+	switch (sfp->sm_mod_state) {
+	default:
+		if (event == SFP_E_INSERT) {
+			sfp_module_tx_disable(sfp);
+			sfp_sm_ins_next(sfp, SFP_MOD_PROBE, T_PROBE_INIT);
+		}
+		break;
+
+	case SFP_MOD_PROBE:
+		if (event == SFP_E_REMOVE) {
+			sfp_sm_ins_next(sfp, SFP_MOD_EMPTY, 0);
+		} else if (event == SFP_E_TIMEOUT) {
+			int err = sfp_sm_mod_probe(sfp);
+
+			if (err == 0)
+				sfp_sm_ins_next(sfp, SFP_MOD_PRESENT, 0);
+			else if (err == -EAGAIN)
+				sfp_sm_set_timer(sfp, T_PROBE_RETRY);
+			else
+				sfp_sm_ins_next(sfp, SFP_MOD_ERROR, 0);
+		}
+		break;
+
+	case SFP_MOD_PRESENT:
+	case SFP_MOD_ERROR:
+		if (event == SFP_E_REMOVE) {
+			sfp_sm_mod_remove(sfp);
+			sfp_sm_ins_next(sfp, SFP_MOD_EMPTY, 0);
+		}
+		break;
+	}
+
+	/* This state machine tracks the netdev up/down state */
+	switch (sfp->sm_dev_state) {
+	default:
+		if (event == SFP_E_DEV_UP)
+			sfp->sm_dev_state = SFP_DEV_UP;
+		break;
+
+	case SFP_DEV_UP:
+		if (event == SFP_E_DEV_DOWN) {
+			/* If the module has a PHY, avoid raising TX disable
+			 * as this resets the PHY. Otherwise, raise it to
+			 * turn the laser off.
+			 */
+			if (!sfp->mod_phy)
+				sfp_module_tx_disable(sfp);
+			sfp->sm_dev_state = SFP_DEV_DOWN;
+		}
+		break;
+	}
+
+	/* Some events are global */
+	if (sfp->sm_state != SFP_S_DOWN &&
+	    (sfp->sm_mod_state != SFP_MOD_PRESENT ||
+	     sfp->sm_dev_state != SFP_DEV_UP)) {
+		if (sfp->sm_state == SFP_S_LINK_UP &&
+		    sfp->sm_dev_state == SFP_DEV_UP)
+			sfp_sm_link_down(sfp);
+		if (sfp->mod_phy)
+			sfp_sm_phy_detach(sfp);
+		sfp_sm_next(sfp, SFP_S_DOWN, 0);
+		mutex_unlock(&sfp->sm_mutex);
+		return;
+	}
+
+	/* The main state machine */
+	switch (sfp->sm_state) {
+	case SFP_S_DOWN:
+		if (sfp->sm_mod_state == SFP_MOD_PRESENT &&
+		    sfp->sm_dev_state == SFP_DEV_UP)
+			sfp_sm_mod_init(sfp);
+		break;
+
+	case SFP_S_INIT:
+		if (event == SFP_E_TIMEOUT && sfp->state & SFP_F_TX_FAULT)
+			sfp_sm_fault(sfp, true);
+		else if (event == SFP_E_TIMEOUT || event == SFP_E_TX_CLEAR)
+			sfp_sm_link_check_los(sfp);
+		break;
+
+	case SFP_S_WAIT_LOS:
+		if (event == SFP_E_TX_FAULT)
+			sfp_sm_fault(sfp, true);
+		else if (event ==
+			 (sfp->id.ext.options & SFP_OPTIONS_LOS_INVERTED ?
+			  SFP_E_LOS_HIGH : SFP_E_LOS_LOW))
+			sfp_sm_link_up(sfp);
+		break;
+
+	case SFP_S_LINK_UP:
+		if (event == SFP_E_TX_FAULT) {
+			sfp_sm_link_down(sfp);
+			sfp_sm_fault(sfp, true);
+		} else if (event ==
+			   (sfp->id.ext.options & SFP_OPTIONS_LOS_INVERTED ?
+			    SFP_E_LOS_LOW : SFP_E_LOS_HIGH)) {
+			sfp_sm_link_down(sfp);
+			sfp_sm_next(sfp, SFP_S_WAIT_LOS, 0);
+		}
+		break;
+
+	case SFP_S_TX_FAULT:
+		if (event == SFP_E_TIMEOUT) {
+			sfp_module_tx_fault_reset(sfp);
+			sfp_sm_next(sfp, SFP_S_REINIT, T_INIT_JIFFIES);
+		}
+		break;
+
+	case SFP_S_REINIT:
+		if (event == SFP_E_TIMEOUT && sfp->state & SFP_F_TX_FAULT) {
+			sfp_sm_fault(sfp, false);
+		} else if (event == SFP_E_TIMEOUT || event == SFP_E_TX_CLEAR) {
+			dev_info(sfp->dev, "module transmit fault recovered\n");
+			sfp_sm_link_check_los(sfp);
+		}
+		break;
+
+	case SFP_S_TX_DISABLE:
+		break;
+	}
+
+	dev_dbg(sfp->dev, "SM: exit %u:%u:%u\n",
+		sfp->sm_mod_state, sfp->sm_dev_state, sfp->sm_state);
+
+	mutex_unlock(&sfp->sm_mutex);
+}
+
+static void sfp_start(struct sfp *sfp)
+{
+	sfp_sm_event(sfp, SFP_E_DEV_UP);
+}
+
+static void sfp_stop(struct sfp *sfp)
+{
+	sfp_sm_event(sfp, SFP_E_DEV_DOWN);
+}
+
+static int sfp_module_info(struct sfp *sfp, struct ethtool_modinfo *modinfo)
+{
+	/* locking... and check module is present */
+
+	if (sfp->id.ext.sff8472_compliance) {
+		modinfo->type = ETH_MODULE_SFF_8472;
+		modinfo->eeprom_len = ETH_MODULE_SFF_8472_LEN;
+	} else {
+		modinfo->type = ETH_MODULE_SFF_8079;
+		modinfo->eeprom_len = ETH_MODULE_SFF_8079_LEN;
+	}
+	return 0;
+}
+
+static int sfp_module_eeprom(struct sfp *sfp, struct ethtool_eeprom *ee,
+	u8 *data)
+{
+	unsigned int first, last, len;
+	int ret;
+
+	if (ee->len == 0)
+		return -EINVAL;
+
+	first = ee->offset;
+	last = ee->offset + ee->len;
+	if (first < ETH_MODULE_SFF_8079_LEN) {
+		len = min_t(unsigned int, last, ETH_MODULE_SFF_8079_LEN);
+		len -= first;
+
+		ret = sfp->read(sfp, false, first, data, len);
+		if (ret < 0)
+			return ret;
+
+		first += len;
+		data += len;
+	}
+	if (first >= ETH_MODULE_SFF_8079_LEN &&
+	    first < ETH_MODULE_SFF_8472_LEN) {
+		len = min_t(unsigned int, last, ETH_MODULE_SFF_8472_LEN);
+		len -= first;
+		first -= ETH_MODULE_SFF_8079_LEN;
+
+		ret = sfp->read(sfp, true, first, data, len);
+		if (ret < 0)
+			return ret;
+	}
+	return 0;
+}
+
+static const struct sfp_socket_ops sfp_module_ops = {
+	.start = sfp_start,
+	.stop = sfp_stop,
+	.module_info = sfp_module_info,
+	.module_eeprom = sfp_module_eeprom,
+};
+
+static void sfp_timeout(struct work_struct *work)
+{
+	struct sfp *sfp = container_of(work, struct sfp, timeout.work);
+
+	rtnl_lock();
+	sfp_sm_event(sfp, SFP_E_TIMEOUT);
+	rtnl_unlock();
+}
+
+static void sfp_check_state(struct sfp *sfp)
+{
+	unsigned int state, i, changed;
+
+	state = sfp_get_state(sfp);
+	changed = state ^ sfp->state;
+	changed &= SFP_F_PRESENT | SFP_F_LOS | SFP_F_TX_FAULT;
+
+	for (i = 0; i < GPIO_MAX; i++)
+		if (changed & BIT(i))
+			dev_dbg(sfp->dev, "%s %u -> %u\n", gpio_of_names[i],
+				!!(sfp->state & BIT(i)), !!(state & BIT(i)));
+
+	state |= sfp->state & (SFP_F_TX_DISABLE | SFP_F_RATE_SELECT);
+	sfp->state = state;
+
+	rtnl_lock();
+	if (changed & SFP_F_PRESENT)
+		sfp_sm_event(sfp, state & SFP_F_PRESENT ?
+				SFP_E_INSERT : SFP_E_REMOVE);
+
+	if (changed & SFP_F_TX_FAULT)
+		sfp_sm_event(sfp, state & SFP_F_TX_FAULT ?
+				SFP_E_TX_FAULT : SFP_E_TX_CLEAR);
+
+	if (changed & SFP_F_LOS)
+		sfp_sm_event(sfp, state & SFP_F_LOS ?
+				SFP_E_LOS_HIGH : SFP_E_LOS_LOW);
+	rtnl_unlock();
+}
+
+static irqreturn_t sfp_irq(int irq, void *data)
+{
+	struct sfp *sfp = data;
+
+	sfp_check_state(sfp);
+
+	return IRQ_HANDLED;
+}
+
+static void sfp_poll(struct work_struct *work)
+{
+	struct sfp *sfp = container_of(work, struct sfp, poll.work);
+
+	sfp_check_state(sfp);
+	mod_delayed_work(system_wq, &sfp->poll, poll_jiffies);
+}
+
+static struct sfp *sfp_alloc(struct device *dev)
+{
+	struct sfp *sfp;
+
+	sfp = kzalloc(sizeof(*sfp), GFP_KERNEL);
+	if (!sfp)
+		return ERR_PTR(-ENOMEM);
+
+	sfp->dev = dev;
+
+	mutex_init(&sfp->sm_mutex);
+	INIT_DELAYED_WORK(&sfp->poll, sfp_poll);
+	INIT_DELAYED_WORK(&sfp->timeout, sfp_timeout);
+
+	return sfp;
+}
+
+static void sfp_cleanup(void *data)
+{
+	struct sfp *sfp = data;
+
+	cancel_delayed_work_sync(&sfp->poll);
+	cancel_delayed_work_sync(&sfp->timeout);
+	if (sfp->i2c_mii) {
+		mdiobus_unregister(sfp->i2c_mii);
+		mdiobus_free(sfp->i2c_mii);
+	}
+	if (sfp->i2c)
+		i2c_put_adapter(sfp->i2c);
+	kfree(sfp);
+}
+
+static int sfp_probe(struct platform_device *pdev)
+{
+	struct sfp *sfp;
+	bool poll = false;
+	int irq, err, i;
+
+	sfp = sfp_alloc(&pdev->dev);
+	if (IS_ERR(sfp))
+		return PTR_ERR(sfp);
+
+	platform_set_drvdata(pdev, sfp);
+
+	err = devm_add_action(sfp->dev, sfp_cleanup, sfp);
+	if (err < 0)
+		return err;
+
+	if (pdev->dev.of_node) {
+		struct device_node *node = pdev->dev.of_node;
+		struct device_node *np;
+
+		np = of_parse_phandle(node, "i2c-bus", 0);
+		if (np) {
+			struct i2c_adapter *i2c;
+
+			i2c = of_find_i2c_adapter_by_node(np);
+			of_node_put(np);
+			if (!i2c)
+				return -EPROBE_DEFER;
+
+			err = sfp_i2c_configure(sfp, i2c);
+			if (err < 0) {
+				i2c_put_adapter(i2c);
+				return err;
+			}
+		}
+
+		for (i = 0; i < GPIO_MAX; i++) {
+			sfp->gpio[i] = devm_gpiod_get_optional(sfp->dev,
+					   gpio_of_names[i], gpio_flags[i]);
+			if (IS_ERR(sfp->gpio[i]))
+				return PTR_ERR(sfp->gpio[i]);
+		}
+
+		sfp->get_state = sfp_gpio_get_state;
+		sfp->set_state = sfp_gpio_set_state;
+	}
+
+	sfp->sfp_bus = sfp_register_socket(sfp->dev, sfp, &sfp_module_ops);
+	if (!sfp->sfp_bus)
+		return -ENOMEM;
+
+	/* Get the initial state, and always signal TX disable,
+	 * since the network interface will not be up.
+	 */
+	sfp->state = sfp_get_state(sfp) | SFP_F_TX_DISABLE;
+
+	if (sfp->gpio[GPIO_RATE_SELECT] &&
+	    gpiod_get_value_cansleep(sfp->gpio[GPIO_RATE_SELECT]))
+		sfp->state |= SFP_F_RATE_SELECT;
+	sfp_set_state(sfp, sfp->state);
+	sfp_module_tx_disable(sfp);
+	rtnl_lock();
+	if (sfp->state & SFP_F_PRESENT)
+		sfp_sm_event(sfp, SFP_E_INSERT);
+	rtnl_unlock();
+
+	for (i = 0; i < GPIO_MAX; i++) {
+		if (gpio_flags[i] != GPIOD_IN || !sfp->gpio[i])
+			continue;
+
+		irq = gpiod_to_irq(sfp->gpio[i]);
+		if (!irq) {
+			poll = true;
+			continue;
+		}
+
+		err = devm_request_threaded_irq(sfp->dev, irq, NULL, sfp_irq,
+						IRQF_ONESHOT |
+						IRQF_TRIGGER_RISING |
+						IRQF_TRIGGER_FALLING,
+						dev_name(sfp->dev), sfp);
+		if (err)
+			poll = true;
+	}
+
+	if (poll)
+		mod_delayed_work(system_wq, &sfp->poll, poll_jiffies);
+
+	return 0;
+}
+
+static int sfp_remove(struct platform_device *pdev)
+{
+	struct sfp *sfp = platform_get_drvdata(pdev);
+
+	sfp_unregister_socket(sfp->sfp_bus);
+
+	return 0;
+}
+
+static const struct of_device_id sfp_of_match[] = {
+	{ .compatible = "sff,sfp", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, sfp_of_match);
+
+static struct platform_driver sfp_driver = {
+	.probe = sfp_probe,
+	.remove = sfp_remove,
+	.driver = {
+		.name = "sfp",
+		.of_match_table = sfp_of_match,
+	},
+};
+
+static int sfp_init(void)
+{
+	poll_jiffies = msecs_to_jiffies(100);
+
+	return platform_driver_register(&sfp_driver);
+}
+module_init(sfp_init);
+
+static void sfp_exit(void)
+{
+	platform_driver_unregister(&sfp_driver);
+}
+module_exit(sfp_exit);
+
+MODULE_ALIAS("platform:sfp");
+MODULE_AUTHOR("Russell King");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/net/phy/sfp.h b/drivers/net/phy/sfp.h
new file mode 100644
index 000000000000..31b0acf337e2
--- /dev/null
+++ b/drivers/net/phy/sfp.h
@@ -0,0 +1,28 @@
+#ifndef SFP_H
+#define SFP_H
+
+#include <linux/ethtool.h>
+#include <linux/sfp.h>
+
+struct sfp;
+
+struct sfp_socket_ops {
+	void (*start)(struct sfp *sfp);
+	void (*stop)(struct sfp *sfp);
+	int (*module_info)(struct sfp *sfp, struct ethtool_modinfo *modinfo);
+	int (*module_eeprom)(struct sfp *sfp, struct ethtool_eeprom *ee,
+			     u8 *data);
+};
+
+int sfp_add_phy(struct sfp_bus *bus, struct phy_device *phydev);
+void sfp_remove_phy(struct sfp_bus *bus);
+void sfp_link_up(struct sfp_bus *bus);
+void sfp_link_down(struct sfp_bus *bus);
+int sfp_module_insert(struct sfp_bus *bus, const struct sfp_eeprom_id *id);
+void sfp_module_remove(struct sfp_bus *bus);
+int sfp_link_configure(struct sfp_bus *bus, const struct sfp_eeprom_id *id);
+struct sfp_bus *sfp_register_socket(struct device *dev, struct sfp *sfp,
+				    const struct sfp_socket_ops *ops);
+void sfp_unregister_socket(struct sfp_bus *bus);
+
+#endif
diff --git a/drivers/net/tap.c b/drivers/net/tap.c
index 3570c7576993..21b71ae947fd 100644
--- a/drivers/net/tap.c
+++ b/drivers/net/tap.c
@@ -943,9 +943,6 @@ static int set_offload(struct tap_queue *q, unsigned long arg)
 			if (arg & TUN_F_TSO6)
 				feature_mask |= NETIF_F_TSO6;
 		}
-
-		if (arg & TUN_F_UFO)
-			feature_mask |= NETIF_F_UFO;
 	}
 
 	/* tun/tap driver inverts the usage for TSO offloads, where
@@ -956,7 +953,7 @@ static int set_offload(struct tap_queue *q, unsigned long arg)
 	 * When user space turns off TSO, we turn off GSO/LRO so that
 	 * user-space will not receive TSO frames.
 	 */
-	if (feature_mask & (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_UFO))
+	if (feature_mask & (NETIF_F_TSO | NETIF_F_TSO6))
 		features |= RX_OFFLOADS;
 	else
 		features &= ~RX_OFFLOADS;
@@ -1078,7 +1075,7 @@ static long tap_ioctl(struct file *file, unsigned int cmd,
 	case TUNSETOFFLOAD:
 		/* let the user check for future flags */
 		if (arg & ~(TUN_F_CSUM | TUN_F_TSO4 | TUN_F_TSO6 |
-			    TUN_F_TSO_ECN | TUN_F_UFO))
+			    TUN_F_TSO_ECN))
 			return -EINVAL;
 
 		rtnl_lock();
@@ -1130,7 +1127,7 @@ static long tap_compat_ioctl(struct file *file, unsigned int cmd,
 }
 #endif
 
-const struct file_operations tap_fops = {
+static const struct file_operations tap_fops = {
 	.owner		= THIS_MODULE,
 	.open		= tap_open,
 	.release	= tap_release,
@@ -1218,7 +1215,7 @@ int tap_queue_resize(struct tap_dev *tap)
 	int n = tap->numqueues;
 	int ret, i = 0;
 
-	arrays = kmalloc(sizeof *arrays * n, GFP_KERNEL);
+	arrays = kmalloc_array(n, sizeof(*arrays), GFP_KERNEL);
 	if (!arrays)
 		return -ENOMEM;
 
diff --git a/drivers/net/tun.c b/drivers/net/tun.c
index 32ad87345f57..3c9985f29950 100644
--- a/drivers/net/tun.c
+++ b/drivers/net/tun.c
@@ -73,6 +73,8 @@
 #include <linux/seq_file.h>
 #include <linux/uio.h>
 #include <linux/skb_array.h>
+#include <linux/bpf.h>
+#include <linux/bpf_trace.h>
 
 #include <linux/uaccess.h>
 
@@ -105,6 +107,9 @@ do {								\
 } while (0)
 #endif
 
+#define TUN_HEADROOM 256
+#define TUN_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)
+
 /* TUN device flags */
 
 /* IFF_ATTACH_QUEUE is never stored in device flags,
@@ -199,7 +204,7 @@ struct tun_struct {
 	struct net_device	*dev;
 	netdev_features_t	set_features;
 #define TUN_USER_FEATURES (NETIF_F_HW_CSUM|NETIF_F_TSO_ECN|NETIF_F_TSO| \
-			  NETIF_F_TSO6|NETIF_F_UFO)
+			  NETIF_F_TSO6)
 
 	int			align;
 	int			vnet_hdr_sz;
@@ -221,6 +226,7 @@ struct tun_struct {
 	u32 flow_count;
 	u32 rx_batched;
 	struct tun_pcpu_stats __percpu *pcpu_stats;
+	struct bpf_prog __rcu *xdp_prog;
 };
 
 #ifdef CONFIG_TUN_VNET_CROSS_LE
@@ -585,6 +591,7 @@ static void tun_detach(struct tun_file *tfile, bool clean)
 static void tun_detach_all(struct net_device *dev)
 {
 	struct tun_struct *tun = netdev_priv(dev);
+	struct bpf_prog *xdp_prog = rtnl_dereference(tun->xdp_prog);
 	struct tun_file *tfile, *tmp;
 	int i, n = tun->numqueues;
 
@@ -617,6 +624,9 @@ static void tun_detach_all(struct net_device *dev)
 	}
 	BUG_ON(tun->numdisabled != 0);
 
+	if (xdp_prog)
+		bpf_prog_put(xdp_prog);
+
 	if (tun->flags & IFF_PERSIST)
 		module_put(THIS_MODULE);
 }
@@ -892,7 +902,7 @@ static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev)
 	    sk_filter(tfile->socket.sk, skb))
 		goto drop;
 
-	if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
+	if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
 		goto drop;
 
 	skb_tx_timestamp(skb);
@@ -1003,6 +1013,46 @@ tun_net_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
 	stats->tx_dropped = tx_dropped;
 }
 
+static int tun_xdp_set(struct net_device *dev, struct bpf_prog *prog,
+		       struct netlink_ext_ack *extack)
+{
+	struct tun_struct *tun = netdev_priv(dev);
+	struct bpf_prog *old_prog;
+
+	old_prog = rtnl_dereference(tun->xdp_prog);
+	rcu_assign_pointer(tun->xdp_prog, prog);
+	if (old_prog)
+		bpf_prog_put(old_prog);
+
+	return 0;
+}
+
+static u32 tun_xdp_query(struct net_device *dev)
+{
+	struct tun_struct *tun = netdev_priv(dev);
+	const struct bpf_prog *xdp_prog;
+
+	xdp_prog = rtnl_dereference(tun->xdp_prog);
+	if (xdp_prog)
+		return xdp_prog->aux->id;
+
+	return 0;
+}
+
+static int tun_xdp(struct net_device *dev, struct netdev_xdp *xdp)
+{
+	switch (xdp->command) {
+	case XDP_SETUP_PROG:
+		return tun_xdp_set(dev, xdp->prog, xdp->extack);
+	case XDP_QUERY_PROG:
+		xdp->prog_id = tun_xdp_query(dev);
+		xdp->prog_attached = !!xdp->prog_id;
+		return 0;
+	default:
+		return -EINVAL;
+	}
+}
+
 static const struct net_device_ops tun_netdev_ops = {
 	.ndo_uninit		= tun_net_uninit,
 	.ndo_open		= tun_net_open,
@@ -1033,6 +1083,7 @@ static const struct net_device_ops tap_netdev_ops = {
 	.ndo_features_check	= passthru_features_check,
 	.ndo_set_rx_headroom	= tun_set_headroom,
 	.ndo_get_stats64	= tun_net_get_stats64,
+	.ndo_xdp		= tun_xdp,
 };
 
 static void tun_flow_init(struct tun_struct *tun)
@@ -1190,6 +1241,138 @@ static void tun_rx_batched(struct tun_struct *tun, struct tun_file *tfile,
 	}
 }
 
+static bool tun_can_build_skb(struct tun_struct *tun, struct tun_file *tfile,
+			      int len, int noblock, bool zerocopy)
+{
+	if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
+		return false;
+
+	if (tfile->socket.sk->sk_sndbuf != INT_MAX)
+		return false;
+
+	if (!noblock)
+		return false;
+
+	if (zerocopy)
+		return false;
+
+	if (SKB_DATA_ALIGN(len + TUN_RX_PAD) +
+	    SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) > PAGE_SIZE)
+		return false;
+
+	return true;
+}
+
+static struct sk_buff *tun_build_skb(struct tun_struct *tun,
+				     struct tun_file *tfile,
+				     struct iov_iter *from,
+				     struct virtio_net_hdr *hdr,
+				     int len, int *skb_xdp)
+{
+	struct page_frag *alloc_frag = &current->task_frag;
+	struct sk_buff *skb;
+	struct bpf_prog *xdp_prog;
+	int buflen = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+	unsigned int delta = 0;
+	char *buf;
+	size_t copied;
+	bool xdp_xmit = false;
+	int err, pad = TUN_RX_PAD;
+
+	rcu_read_lock();
+	xdp_prog = rcu_dereference(tun->xdp_prog);
+	if (xdp_prog)
+		pad += TUN_HEADROOM;
+	buflen += SKB_DATA_ALIGN(len + pad);
+	rcu_read_unlock();
+
+	if (unlikely(!skb_page_frag_refill(buflen, alloc_frag, GFP_KERNEL)))
+		return ERR_PTR(-ENOMEM);
+
+	buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
+	copied = copy_page_from_iter(alloc_frag->page,
+				     alloc_frag->offset + pad,
+				     len, from);
+	if (copied != len)
+		return ERR_PTR(-EFAULT);
+
+	/* There's a small window that XDP may be set after the check
+	 * of xdp_prog above, this should be rare and for simplicity
+	 * we do XDP on skb in case the headroom is not enough.
+	 */
+	if (hdr->gso_type || !xdp_prog)
+		*skb_xdp = 1;
+	else
+		*skb_xdp = 0;
+
+	rcu_read_lock();
+	xdp_prog = rcu_dereference(tun->xdp_prog);
+	if (xdp_prog && !*skb_xdp) {
+		struct xdp_buff xdp;
+		void *orig_data;
+		u32 act;
+
+		xdp.data_hard_start = buf;
+		xdp.data = buf + pad;
+		xdp.data_end = xdp.data + len;
+		orig_data = xdp.data;
+		act = bpf_prog_run_xdp(xdp_prog, &xdp);
+
+		switch (act) {
+		case XDP_REDIRECT:
+			get_page(alloc_frag->page);
+			alloc_frag->offset += buflen;
+			err = xdp_do_redirect(tun->dev, &xdp, xdp_prog);
+			if (err)
+				goto err_redirect;
+			return NULL;
+		case XDP_TX:
+			xdp_xmit = true;
+			/* fall through */
+		case XDP_PASS:
+			delta = orig_data - xdp.data;
+			break;
+		default:
+			bpf_warn_invalid_xdp_action(act);
+			/* fall through */
+		case XDP_ABORTED:
+			trace_xdp_exception(tun->dev, xdp_prog, act);
+			/* fall through */
+		case XDP_DROP:
+			goto err_xdp;
+		}
+	}
+
+	skb = build_skb(buf, buflen);
+	if (!skb) {
+		rcu_read_unlock();
+		return ERR_PTR(-ENOMEM);
+	}
+
+	skb_reserve(skb, pad - delta);
+	skb_put(skb, len + delta);
+	get_page(alloc_frag->page);
+	alloc_frag->offset += buflen;
+
+	if (xdp_xmit) {
+		skb->dev = tun->dev;
+		generic_xdp_tx(skb, xdp_prog);
+		rcu_read_lock();
+		return NULL;
+	}
+
+	rcu_read_unlock();
+
+	return skb;
+
+err_redirect:
+	put_page(alloc_frag->page);
+err_xdp:
+	rcu_read_unlock();
+	this_cpu_inc(tun->pcpu_stats->rx_dropped);
+	return NULL;
+}
+
 /* Get packet from user space buffer */
 static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
 			    void *msg_control, struct iov_iter *from,
@@ -1206,6 +1389,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
 	bool zerocopy = false;
 	int err;
 	u32 rxhash;
+	int skb_xdp = 1;
 
 	if (!(tun->dev->flags & IFF_UP))
 		return -EIO;
@@ -1263,30 +1447,44 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
 			zerocopy = true;
 	}
 
-	if (!zerocopy) {
-		copylen = len;
-		if (tun16_to_cpu(tun, gso.hdr_len) > good_linear)
-			linear = good_linear;
-		else
-			linear = tun16_to_cpu(tun, gso.hdr_len);
-	}
-
-	skb = tun_alloc_skb(tfile, align, copylen, linear, noblock);
-	if (IS_ERR(skb)) {
-		if (PTR_ERR(skb) != -EAGAIN)
+	if (tun_can_build_skb(tun, tfile, len, noblock, zerocopy)) {
+		/* For the packet that is not easy to be processed
+		 * (e.g gso or jumbo packet), we will do it at after
+		 * skb was created with generic XDP routine.
+		 */
+		skb = tun_build_skb(tun, tfile, from, &gso, len, &skb_xdp);
+		if (IS_ERR(skb)) {
 			this_cpu_inc(tun->pcpu_stats->rx_dropped);
-		return PTR_ERR(skb);
-	}
+			return PTR_ERR(skb);
+		}
+		if (!skb)
+			return total_len;
+	} else {
+		if (!zerocopy) {
+			copylen = len;
+			if (tun16_to_cpu(tun, gso.hdr_len) > good_linear)
+				linear = good_linear;
+			else
+				linear = tun16_to_cpu(tun, gso.hdr_len);
+		}
 
-	if (zerocopy)
-		err = zerocopy_sg_from_iter(skb, from);
-	else
-		err = skb_copy_datagram_from_iter(skb, 0, from, len);
+		skb = tun_alloc_skb(tfile, align, copylen, linear, noblock);
+		if (IS_ERR(skb)) {
+			if (PTR_ERR(skb) != -EAGAIN)
+				this_cpu_inc(tun->pcpu_stats->rx_dropped);
+			return PTR_ERR(skb);
+		}
 
-	if (err) {
-		this_cpu_inc(tun->pcpu_stats->rx_dropped);
-		kfree_skb(skb);
-		return -EFAULT;
+		if (zerocopy)
+			err = zerocopy_sg_from_iter(skb, from);
+		else
+			err = skb_copy_datagram_from_iter(skb, 0, from, len);
+
+		if (err) {
+			this_cpu_inc(tun->pcpu_stats->rx_dropped);
+			kfree_skb(skb);
+			return -EFAULT;
+		}
 	}
 
 	if (virtio_net_hdr_to_skb(skb, &gso, tun_is_little_endian(tun))) {
@@ -1334,6 +1532,22 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
 	skb_reset_network_header(skb);
 	skb_probe_transport_header(skb, 0);
 
+	if (skb_xdp) {
+		struct bpf_prog *xdp_prog;
+		int ret;
+
+		rcu_read_lock();
+		xdp_prog = rcu_dereference(tun->xdp_prog);
+		if (xdp_prog) {
+			ret = do_xdp_generic(xdp_prog, skb);
+			if (ret != XDP_PASS) {
+				rcu_read_unlock();
+				return total_len;
+			}
+		}
+		rcu_read_unlock();
+	}
+
 	rxhash = __skb_get_hash_symmetric(skb);
 #ifndef CONFIG_4KSTACKS
 	tun_rx_batched(tun, tfile, skb, more);
@@ -1879,6 +2093,9 @@ static int tun_set_iff(struct net *net, struct file *file, struct ifreq *ifr)
 
 err_detach:
 	tun_detach_all(dev);
+	/* register_netdevice() already called tun_free_netdev() */
+	goto err_free_dev;
+
 err_free_flow:
 	tun_flow_uninit(tun);
 	security_tun_dev_free_security(tun->security);
@@ -1921,11 +2138,6 @@ static int set_offload(struct tun_struct *tun, unsigned long arg)
 				features |= NETIF_F_TSO6;
 			arg &= ~(TUN_F_TSO4|TUN_F_TSO6);
 		}
-
-		if (arg & TUN_F_UFO) {
-			features |= NETIF_F_UFO;
-			arg &= ~TUN_F_UFO;
-		}
 	}
 
 	/* This gives the user a way to test for new features in future by
@@ -2537,7 +2749,7 @@ static int tun_queue_resize(struct tun_struct *tun)
 	int n = tun->numqueues + tun->numdisabled;
 	int ret, i;
 
-	arrays = kmalloc(sizeof *arrays * n, GFP_KERNEL);
+	arrays = kmalloc_array(n, sizeof(*arrays), GFP_KERNEL);
 	if (!arrays)
 		return -ENOMEM;
 
diff --git a/drivers/net/usb/catc.c b/drivers/net/usb/catc.c
index fce92f0e5abd..dbc90313f472 100644
--- a/drivers/net/usb/catc.c
+++ b/drivers/net/usb/catc.c
@@ -961,7 +961,7 @@ static void catc_disconnect(struct usb_interface *intf)
  * Module functions and tables.
  */
 
-static struct usb_device_id catc_id_table [] = {
+static const struct usb_device_id catc_id_table[] = {
 	{ USB_DEVICE(0x0423, 0xa) },	/* CATC Netmate, Belkin F5U011 */
 	{ USB_DEVICE(0x0423, 0xc) },	/* CATC Netmate II, Belkin F5U111 */
 	{ USB_DEVICE(0x08d1, 0x1) },	/* smartBridges smartNIC */
diff --git a/drivers/net/usb/cdc-phonet.c b/drivers/net/usb/cdc-phonet.c
index 2952cb570996..288ecd999171 100644
--- a/drivers/net/usb/cdc-phonet.c
+++ b/drivers/net/usb/cdc-phonet.c
@@ -304,7 +304,7 @@ static void usbpn_setup(struct net_device *dev)
 /*
  * USB driver callbacks
  */
-static struct usb_device_id usbpn_ids[] = {
+static const struct usb_device_id usbpn_ids[] = {
 	{
 		.match_flags = USB_DEVICE_ID_MATCH_VENDOR
 			| USB_DEVICE_ID_MATCH_INT_CLASS
diff --git a/drivers/net/usb/cdc_ncm.c b/drivers/net/usb/cdc_ncm.c
index 8f572b9f3625..47cab1bde065 100644
--- a/drivers/net/usb/cdc_ncm.c
+++ b/drivers/net/usb/cdc_ncm.c
@@ -367,7 +367,7 @@ static struct attribute *cdc_ncm_sysfs_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group cdc_ncm_sysfs_attr_group = {
+static const struct attribute_group cdc_ncm_sysfs_attr_group = {
 	.name = "cdc_ncm",
 	.attrs = cdc_ncm_sysfs_attrs,
 };
@@ -1758,6 +1758,13 @@ static const struct usb_device_id cdc_devs[] = {
 	  .driver_info = (unsigned long)&wwan_noarp_info,
 	},
 
+	/* u-blox TOBY-L4 */
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x1546, 0x1010,
+		USB_CLASS_COMM,
+		USB_CDC_SUBCLASS_NCM, USB_CDC_PROTO_NONE),
+	  .driver_info = (unsigned long)&wwan_info,
+	},
+
 	/* Generic CDC-NCM devices */
 	{ USB_INTERFACE_INFO(USB_CLASS_COMM,
 		USB_CDC_SUBCLASS_NCM, USB_CDC_PROTO_NONE),
diff --git a/drivers/net/usb/ipheth.c b/drivers/net/usb/ipheth.c
index 0f213ea22c75..d49c7103085e 100644
--- a/drivers/net/usb/ipheth.c
+++ b/drivers/net/usb/ipheth.c
@@ -87,7 +87,7 @@
 #define IPHETH_CARRIER_CHECK_TIMEOUT round_jiffies_relative(1 * HZ)
 #define IPHETH_CARRIER_ON       0x04
 
-static struct usb_device_id ipheth_table[] = {
+static const struct usb_device_id ipheth_table[] = {
 	{ USB_DEVICE_AND_INTERFACE_INFO(
 		USB_VENDOR_APPLE, USB_PRODUCT_IPHONE,
 		IPHETH_USBINTF_CLASS, IPHETH_USBINTF_SUBCLASS,
diff --git a/drivers/net/usb/kaweth.c b/drivers/net/usb/kaweth.c
index 92e4fd29ae44..f1605833c5cf 100644
--- a/drivers/net/usb/kaweth.c
+++ b/drivers/net/usb/kaweth.c
@@ -125,7 +125,7 @@ static int kaweth_resume(struct usb_interface *intf);
 /****************************************************************
  *     usb_device_id
  ****************************************************************/
-static struct usb_device_id usb_klsi_table[] = {
+static const struct usb_device_id usb_klsi_table[] = {
 	{ USB_DEVICE(0x03e8, 0x0008) }, /* AOX Endpoints USB Ethernet */
 	{ USB_DEVICE(0x04bb, 0x0901) }, /* I-O DATA USB-ET/T */
 	{ USB_DEVICE(0x0506, 0x03e8) }, /* 3Com 3C19250 */
diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c
index 6cfffeff6108..ceb78e2ea4f0 100644
--- a/drivers/net/usb/r8152.c
+++ b/drivers/net/usb/r8152.c
@@ -5303,7 +5303,7 @@ static void rtl8152_disconnect(struct usb_interface *intf)
 	.bInterfaceProtocol = USB_CDC_PROTO_NONE
 
 /* table of devices that work with this driver */
-static struct usb_device_id rtl8152_table[] = {
+static const struct usb_device_id rtl8152_table[] = {
 	{REALTEK_USB_DEVICE(VENDOR_ID_REALTEK, 0x8050)},
 	{REALTEK_USB_DEVICE(VENDOR_ID_REALTEK, 0x8152)},
 	{REALTEK_USB_DEVICE(VENDOR_ID_REALTEK, 0x8153)},
diff --git a/drivers/net/usb/rtl8150.c b/drivers/net/usb/rtl8150.c
index daaa88a66f40..5f565bd574da 100644
--- a/drivers/net/usb/rtl8150.c
+++ b/drivers/net/usb/rtl8150.c
@@ -112,7 +112,7 @@
 #undef	EEPROM_WRITE
 
 /* table of devices that work with this driver */
-static struct usb_device_id rtl8150_table[] = {
+static const struct usb_device_id rtl8150_table[] = {
 	{USB_DEVICE(VENDOR_ID_REALTEK, PRODUCT_ID_RTL8150)},
 	{USB_DEVICE(VENDOR_ID_MELCO, PRODUCT_ID_LUAKTX)},
 	{USB_DEVICE(VENDOR_ID_MICRONET, PRODUCT_ID_SP128AR)},
diff --git a/drivers/net/usb/smsc95xx.c b/drivers/net/usb/smsc95xx.c
index 340c13484e5c..309b88acd3d0 100644
--- a/drivers/net/usb/smsc95xx.c
+++ b/drivers/net/usb/smsc95xx.c
@@ -526,7 +526,7 @@ static void smsc95xx_set_multicast(struct net_device *netdev)
 static int smsc95xx_phy_update_flowcontrol(struct usbnet *dev, u8 duplex,
 					   u16 lcladv, u16 rmtadv)
 {
-	u32 flow, afc_cfg = 0;
+	u32 flow = 0, afc_cfg;
 
 	int ret = smsc95xx_read_reg(dev, AFC_CFG, &afc_cfg);
 	if (ret < 0)
@@ -537,20 +537,19 @@ static int smsc95xx_phy_update_flowcontrol(struct usbnet *dev, u8 duplex,
 
 		if (cap & FLOW_CTRL_RX)
 			flow = 0xFFFF0002;
-		else
-			flow = 0;
 
-		if (cap & FLOW_CTRL_TX)
+		if (cap & FLOW_CTRL_TX) {
 			afc_cfg |= 0xF;
-		else
+			flow |= 0xFFFF0000;
+		} else {
 			afc_cfg &= ~0xF;
+		}
 
 		netif_dbg(dev, link, dev->net, "rx pause %s, tx pause %s\n",
 				   cap & FLOW_CTRL_RX ? "enabled" : "disabled",
 				   cap & FLOW_CTRL_TX ? "enabled" : "disabled");
 	} else {
 		netif_dbg(dev, link, dev->net, "half duplex\n");
-		flow = 0;
 		afc_cfg |= 0xF;
 	}
 
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index 98f17b05c68b..511f8339fa96 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -57,6 +57,13 @@ DECLARE_EWMA(pkt_len, 0, 64)
 
 #define VIRTNET_DRIVER_VERSION "1.0.0"
 
+static const unsigned long guest_offloads[] = {
+	VIRTIO_NET_F_GUEST_TSO4,
+	VIRTIO_NET_F_GUEST_TSO6,
+	VIRTIO_NET_F_GUEST_ECN,
+	VIRTIO_NET_F_GUEST_UFO
+};
+
 struct virtnet_stats {
 	struct u64_stats_sync tx_syncp;
 	struct u64_stats_sync rx_syncp;
@@ -164,10 +171,13 @@ struct virtnet_info {
 	u8 ctrl_promisc;
 	u8 ctrl_allmulti;
 	u16 ctrl_vid;
+	u64 ctrl_offloads;
 
 	/* Ethtool settings */
 	u8 duplex;
 	u32 speed;
+
+	unsigned long guest_offloads;
 };
 
 struct padded_vnet_hdr {
@@ -270,6 +280,23 @@ static void skb_xmit_done(struct virtqueue *vq)
 		netif_wake_subqueue(vi->dev, vq2txq(vq));
 }
 
+#define MRG_CTX_HEADER_SHIFT 22
+static void *mergeable_len_to_ctx(unsigned int truesize,
+				  unsigned int headroom)
+{
+	return (void *)(unsigned long)((headroom << MRG_CTX_HEADER_SHIFT) | truesize);
+}
+
+static unsigned int mergeable_ctx_to_headroom(void *mrg_ctx)
+{
+	return (unsigned long)mrg_ctx >> MRG_CTX_HEADER_SHIFT;
+}
+
+static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx)
+{
+	return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1);
+}
+
 /* Called from bottom half context */
 static struct sk_buff *page_to_skb(struct virtnet_info *vi,
 				   struct receive_queue *rq,
@@ -292,7 +319,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,
 
 	hdr_len = vi->hdr_len;
 	if (vi->mergeable_rx_bufs)
-		hdr_padded_len = sizeof *hdr;
+		hdr_padded_len = sizeof(*hdr);
 	else
 		hdr_padded_len = sizeof(struct padded_vnet_hdr);
 
@@ -390,19 +417,85 @@ static unsigned int virtnet_get_headroom(struct virtnet_info *vi)
 	return vi->xdp_queue_pairs ? VIRTIO_XDP_HEADROOM : 0;
 }
 
+/* We copy the packet for XDP in the following cases:
+ *
+ * 1) Packet is scattered across multiple rx buffers.
+ * 2) Headroom space is insufficient.
+ *
+ * This is inefficient but it's a temporary condition that
+ * we hit right after XDP is enabled and until queue is refilled
+ * with large buffers with sufficient headroom - so it should affect
+ * at most queue size packets.
+ * Afterwards, the conditions to enable
+ * XDP should preclude the underlying device from sending packets
+ * across multiple buffers (num_buf > 1), and we make sure buffers
+ * have enough headroom.
+ */
+static struct page *xdp_linearize_page(struct receive_queue *rq,
+				       u16 *num_buf,
+				       struct page *p,
+				       int offset,
+				       int page_off,
+				       unsigned int *len)
+{
+	struct page *page = alloc_page(GFP_ATOMIC);
+
+	if (!page)
+		return NULL;
+
+	memcpy(page_address(page) + page_off, page_address(p) + offset, *len);
+	page_off += *len;
+
+	while (--*num_buf) {
+		unsigned int buflen;
+		void *buf;
+		int off;
+
+		buf = virtqueue_get_buf(rq->vq, &buflen);
+		if (unlikely(!buf))
+			goto err_buf;
+
+		p = virt_to_head_page(buf);
+		off = buf - page_address(p);
+
+		/* guard against a misconfigured or uncooperative backend that
+		 * is sending packet larger than the MTU.
+		 */
+		if ((page_off + buflen) > PAGE_SIZE) {
+			put_page(p);
+			goto err_buf;
+		}
+
+		memcpy(page_address(page) + page_off,
+		       page_address(p) + off, buflen);
+		page_off += buflen;
+		put_page(p);
+	}
+
+	/* Headroom does not contribute to packet length */
+	*len = page_off - VIRTIO_XDP_HEADROOM;
+	return page;
+err_buf:
+	__free_pages(page, 0);
+	return NULL;
+}
+
 static struct sk_buff *receive_small(struct net_device *dev,
 				     struct virtnet_info *vi,
 				     struct receive_queue *rq,
-				     void *buf, unsigned int len)
+				     void *buf, void *ctx,
+				     unsigned int len)
 {
 	struct sk_buff *skb;
 	struct bpf_prog *xdp_prog;
-	unsigned int xdp_headroom = virtnet_get_headroom(vi);
+	unsigned int xdp_headroom = (unsigned long)ctx;
 	unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom;
 	unsigned int headroom = vi->hdr_len + header_offset;
 	unsigned int buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
 			      SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+	struct page *page = virt_to_head_page(buf);
 	unsigned int delta = 0;
+	struct page *xdp_page;
 	len -= vi->hdr_len;
 
 	rcu_read_lock();
@@ -416,6 +509,27 @@ static struct sk_buff *receive_small(struct net_device *dev,
 		if (unlikely(hdr->hdr.gso_type || hdr->hdr.flags))
 			goto err_xdp;
 
+		if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) {
+			int offset = buf - page_address(page) + header_offset;
+			unsigned int tlen = len + vi->hdr_len;
+			u16 num_buf = 1;
+
+			xdp_headroom = virtnet_get_headroom(vi);
+			header_offset = VIRTNET_RX_PAD + xdp_headroom;
+			headroom = vi->hdr_len + header_offset;
+			buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
+				 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+			xdp_page = xdp_linearize_page(rq, &num_buf, page,
+						      offset, header_offset,
+						      &tlen);
+			if (!xdp_page)
+				goto err_xdp;
+
+			buf = page_address(xdp_page);
+			put_page(page);
+			page = xdp_page;
+		}
+
 		xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len;
 		xdp.data = xdp.data_hard_start + xdp_headroom;
 		xdp.data_end = xdp.data + len;
@@ -444,7 +558,7 @@ static struct sk_buff *receive_small(struct net_device *dev,
 
 	skb = build_skb(buf, buflen);
 	if (!skb) {
-		put_page(virt_to_head_page(buf));
+		put_page(page);
 		goto err;
 	}
 	skb_reserve(skb, headroom - delta);
@@ -460,7 +574,7 @@ err:
 err_xdp:
 	rcu_read_unlock();
 	dev->stats.rx_dropped++;
-	put_page(virt_to_head_page(buf));
+	put_page(page);
 xdp_xmit:
 	return NULL;
 }
@@ -485,66 +599,6 @@ err:
 	return NULL;
 }
 
-/* The conditions to enable XDP should preclude the underlying device from
- * sending packets across multiple buffers (num_buf > 1). However per spec
- * it does not appear to be illegal to do so but rather just against convention.
- * So in order to avoid making a system unresponsive the packets are pushed
- * into a page and the XDP program is run. This will be extremely slow and we
- * push a warning to the user to fix this as soon as possible. Fixing this may
- * require resolving the underlying hardware to determine why multiple buffers
- * are being received or simply loading the XDP program in the ingress stack
- * after the skb is built because there is no advantage to running it here
- * anymore.
- */
-static struct page *xdp_linearize_page(struct receive_queue *rq,
-				       u16 *num_buf,
-				       struct page *p,
-				       int offset,
-				       unsigned int *len)
-{
-	struct page *page = alloc_page(GFP_ATOMIC);
-	unsigned int page_off = VIRTIO_XDP_HEADROOM;
-
-	if (!page)
-		return NULL;
-
-	memcpy(page_address(page) + page_off, page_address(p) + offset, *len);
-	page_off += *len;
-
-	while (--*num_buf) {
-		unsigned int buflen;
-		void *buf;
-		int off;
-
-		buf = virtqueue_get_buf(rq->vq, &buflen);
-		if (unlikely(!buf))
-			goto err_buf;
-
-		p = virt_to_head_page(buf);
-		off = buf - page_address(p);
-
-		/* guard against a misconfigured or uncooperative backend that
-		 * is sending packet larger than the MTU.
-		 */
-		if ((page_off + buflen) > PAGE_SIZE) {
-			put_page(p);
-			goto err_buf;
-		}
-
-		memcpy(page_address(page) + page_off,
-		       page_address(p) + off, buflen);
-		page_off += buflen;
-		put_page(p);
-	}
-
-	/* Headroom does not contribute to packet length */
-	*len = page_off - VIRTIO_XDP_HEADROOM;
-	return page;
-err_buf:
-	__free_pages(page, 0);
-	return NULL;
-}
-
 static struct sk_buff *receive_mergeable(struct net_device *dev,
 					 struct virtnet_info *vi,
 					 struct receive_queue *rq,
@@ -559,6 +613,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 	struct sk_buff *head_skb, *curr_skb;
 	struct bpf_prog *xdp_prog;
 	unsigned int truesize;
+	unsigned int headroom = mergeable_ctx_to_headroom(ctx);
 
 	head_skb = NULL;
 
@@ -571,10 +626,13 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 		u32 act;
 
 		/* This happens when rx buffer size is underestimated */
-		if (unlikely(num_buf > 1)) {
+		if (unlikely(num_buf > 1 ||
+			     headroom < virtnet_get_headroom(vi))) {
 			/* linearize data for XDP */
 			xdp_page = xdp_linearize_page(rq, &num_buf,
-						      page, offset, &len);
+						      page, offset,
+						      VIRTIO_XDP_HEADROOM,
+						      &len);
 			if (!xdp_page)
 				goto err_xdp;
 			offset = VIRTIO_XDP_HEADROOM;
@@ -639,13 +697,14 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 	}
 	rcu_read_unlock();
 
-	if (unlikely(len > (unsigned long)ctx)) {
+	truesize = mergeable_ctx_to_truesize(ctx);
+	if (unlikely(len > truesize)) {
 		pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
 			 dev->name, len, (unsigned long)ctx);
 		dev->stats.rx_length_errors++;
 		goto err_skb;
 	}
-	truesize = (unsigned long)ctx;
+
 	head_skb = page_to_skb(vi, rq, page, offset, len, truesize);
 	curr_skb = head_skb;
 
@@ -665,13 +724,14 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 		}
 
 		page = virt_to_head_page(buf);
-		if (unlikely(len > (unsigned long)ctx)) {
+
+		truesize = mergeable_ctx_to_truesize(ctx);
+		if (unlikely(len > truesize)) {
 			pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
 				 dev->name, len, (unsigned long)ctx);
 			dev->stats.rx_length_errors++;
 			goto err_skb;
 		}
-		truesize = (unsigned long)ctx;
 
 		num_skb_frags = skb_shinfo(curr_skb)->nr_frags;
 		if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) {
@@ -754,7 +814,7 @@ static int receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
 	else if (vi->big_packets)
 		skb = receive_big(dev, vi, rq, buf, len);
 	else
-		skb = receive_small(dev, vi, rq, buf, len);
+		skb = receive_small(dev, vi, rq, buf, ctx, len);
 
 	if (unlikely(!skb))
 		return 0;
@@ -787,12 +847,18 @@ frame_err:
 	return 0;
 }
 
+/* Unlike mergeable buffers, all buffers are allocated to the
+ * same size, except for the headroom. For this reason we do
+ * not need to use  mergeable_len_to_ctx here - it is enough
+ * to store the headroom as the context ignoring the truesize.
+ */
 static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
 			     gfp_t gfp)
 {
 	struct page_frag *alloc_frag = &rq->alloc_frag;
 	char *buf;
 	unsigned int xdp_headroom = virtnet_get_headroom(vi);
+	void *ctx = (void *)(unsigned long)xdp_headroom;
 	int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom;
 	int err;
 
@@ -806,10 +872,9 @@ static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
 	alloc_frag->offset += len;
 	sg_init_one(rq->sg, buf + VIRTNET_RX_PAD + xdp_headroom,
 		    vi->hdr_len + GOOD_PACKET_LEN);
-	err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, buf, gfp);
+	err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
 	if (err < 0)
 		put_page(virt_to_head_page(buf));
-
 	return err;
 }
 
@@ -902,7 +967,7 @@ static int add_recvbuf_mergeable(struct virtnet_info *vi,
 	}
 
 	sg_init_one(rq->sg, buf, len);
-	ctx = (void *)(unsigned long)len;
+	ctx = mergeable_len_to_ctx(len, headroom);
 	err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp);
 	if (err < 0)
 		put_page(virt_to_head_page(buf));
@@ -1014,7 +1079,7 @@ static int virtnet_receive(struct receive_queue *rq, int budget)
 	void *buf;
 	struct virtnet_stats *stats = this_cpu_ptr(vi->stats);
 
-	if (vi->mergeable_rx_bufs) {
+	if (!vi->big_packets || vi->mergeable_rx_bufs) {
 		void *ctx;
 
 		while (received < budget &&
@@ -1058,7 +1123,7 @@ static void free_old_xmit_skbs(struct send_queue *sq)
 		bytes += skb->len;
 		packets++;
 
-		dev_kfree_skb_any(skb);
+		dev_consume_skb_any(skb);
 	}
 
 	/* Avoid overhead when no packets have been processed
@@ -1813,7 +1878,6 @@ static void virtnet_freeze_down(struct virtio_device *vdev)
 }
 
 static int init_vqs(struct virtnet_info *vi);
-static void _remove_vq_common(struct virtnet_info *vi);
 
 static int virtnet_restore_up(struct virtio_device *vdev)
 {
@@ -1842,37 +1906,45 @@ static int virtnet_restore_up(struct virtio_device *vdev)
 	return err;
 }
 
-static int virtnet_reset(struct virtnet_info *vi, int curr_qp, int xdp_qp)
+static int virtnet_set_guest_offloads(struct virtnet_info *vi, u64 offloads)
 {
-	struct virtio_device *dev = vi->vdev;
-	int ret;
+	struct scatterlist sg;
+	vi->ctrl_offloads = cpu_to_virtio64(vi->vdev, offloads);
 
-	virtio_config_disable(dev);
-	dev->failed = dev->config->get_status(dev) & VIRTIO_CONFIG_S_FAILED;
-	virtnet_freeze_down(dev);
-	_remove_vq_common(vi);
+	sg_init_one(&sg, &vi->ctrl_offloads, sizeof(vi->ctrl_offloads));
 
-	virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
-	virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER);
+	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_GUEST_OFFLOADS,
+				  VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET, &sg)) {
+		dev_warn(&vi->dev->dev, "Fail to set guest offload. \n");
+		return -EINVAL;
+	}
 
-	ret = virtio_finalize_features(dev);
-	if (ret)
-		goto err;
+	return 0;
+}
 
-	vi->xdp_queue_pairs = xdp_qp;
-	ret = virtnet_restore_up(dev);
-	if (ret)
-		goto err;
-	ret = _virtnet_set_queues(vi, curr_qp);
-	if (ret)
-		goto err;
+static int virtnet_clear_guest_offloads(struct virtnet_info *vi)
+{
+	u64 offloads = 0;
 
-	virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);
-	virtio_config_enable(dev);
-	return 0;
-err:
-	virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
-	return ret;
+	if (!vi->guest_offloads)
+		return 0;
+
+	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM))
+		offloads = 1ULL << VIRTIO_NET_F_GUEST_CSUM;
+
+	return virtnet_set_guest_offloads(vi, offloads);
+}
+
+static int virtnet_restore_guest_offloads(struct virtnet_info *vi)
+{
+	u64 offloads = vi->guest_offloads;
+
+	if (!vi->guest_offloads)
+		return 0;
+	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM))
+		offloads |= 1ULL << VIRTIO_NET_F_GUEST_CSUM;
+
+	return virtnet_set_guest_offloads(vi, offloads);
 }
 
 static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog,
@@ -1884,10 +1956,11 @@ static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog,
 	u16 xdp_qp = 0, curr_qp;
 	int i, err;
 
-	if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) ||
-	    virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) ||
-	    virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) ||
-	    virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO)) {
+	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)
+	    && (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) ||
+	        virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) ||
+	        virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) ||
+		virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO))) {
 		NL_SET_ERR_MSG_MOD(extack, "Can't set XDP while host is implementing LRO, disable LRO first");
 		return -EOPNOTSUPP;
 	}
@@ -1921,35 +1994,35 @@ static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog,
 			return PTR_ERR(prog);
 	}
 
-	/* Changing the headroom in buffers is a disruptive operation because
-	 * existing buffers must be flushed and reallocated. This will happen
-	 * when a xdp program is initially added or xdp is disabled by removing
-	 * the xdp program resulting in number of XDP queues changing.
-	 */
-	if (vi->xdp_queue_pairs != xdp_qp) {
-		err = virtnet_reset(vi, curr_qp + xdp_qp, xdp_qp);
-		if (err) {
-			dev_warn(&dev->dev, "XDP reset failure.\n");
-			goto virtio_reset_err;
-		}
-	}
+	/* Make sure NAPI is not using any XDP TX queues for RX. */
+	for (i = 0; i < vi->max_queue_pairs; i++)
+		napi_disable(&vi->rq[i].napi);
 
 	netif_set_real_num_rx_queues(dev, curr_qp + xdp_qp);
+	err = _virtnet_set_queues(vi, curr_qp + xdp_qp);
+	if (err)
+		goto err;
+	vi->xdp_queue_pairs = xdp_qp;
 
 	for (i = 0; i < vi->max_queue_pairs; i++) {
 		old_prog = rtnl_dereference(vi->rq[i].xdp_prog);
 		rcu_assign_pointer(vi->rq[i].xdp_prog, prog);
+		if (i == 0) {
+			if (!old_prog)
+				virtnet_clear_guest_offloads(vi);
+			if (!prog)
+				virtnet_restore_guest_offloads(vi);
+		}
 		if (old_prog)
 			bpf_prog_put(old_prog);
+		virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
 	}
 
 	return 0;
 
-virtio_reset_err:
-	/* On reset error do our best to unwind XDP changes inflight and return
-	 * error up to user space for resolution. The underlying reset hung on
-	 * us so not much we can do here.
-	 */
+err:
+	for (i = 0; i < vi->max_queue_pairs; i++)
+		virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
 	if (prog)
 		bpf_prog_sub(prog, vi->max_queue_pairs - 1);
 	return err;
@@ -2182,7 +2255,7 @@ static int virtnet_find_vqs(struct virtnet_info *vi)
 	names = kmalloc(total_vqs * sizeof(*names), GFP_KERNEL);
 	if (!names)
 		goto err_names;
-	if (vi->mergeable_rx_bufs) {
+	if (!vi->big_packets || vi->mergeable_rx_bufs) {
 		ctx = kzalloc(total_vqs * sizeof(*ctx), GFP_KERNEL);
 		if (!ctx)
 			goto err_ctx;
@@ -2303,7 +2376,7 @@ err:
 
 #ifdef CONFIG_SYSFS
 static ssize_t mergeable_rx_buffer_size_show(struct netdev_rx_queue *queue,
-		struct rx_queue_attribute *attribute, char *buf)
+		char *buf)
 {
 	struct virtnet_info *vi = netdev_priv(queue->dev);
 	unsigned int queue_index = get_netdev_rx_queue_index(queue);
@@ -2428,7 +2501,7 @@ static int virtnet_probe(struct virtio_device *vdev)
 			dev->features |= NETIF_F_HW_CSUM | NETIF_F_SG;
 
 		if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
-			dev->hw_features |= NETIF_F_TSO | NETIF_F_UFO
+			dev->hw_features |= NETIF_F_TSO
 				| NETIF_F_TSO_ECN | NETIF_F_TSO6;
 		}
 		/* Individual feature bits: what can host handle? */
@@ -2438,13 +2511,11 @@ static int virtnet_probe(struct virtio_device *vdev)
 			dev->hw_features |= NETIF_F_TSO6;
 		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
 			dev->hw_features |= NETIF_F_TSO_ECN;
-		if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_UFO))
-			dev->hw_features |= NETIF_F_UFO;
 
 		dev->features |= NETIF_F_GSO_ROBUST;
 
 		if (gso)
-			dev->features |= dev->hw_features & (NETIF_F_ALL_TSO|NETIF_F_UFO);
+			dev->features |= dev->hw_features & NETIF_F_ALL_TSO;
 		/* (!csum && gso) case will be fixed by register_netdev() */
 	}
 	if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_CSUM))
@@ -2577,6 +2648,10 @@ static int virtnet_probe(struct virtio_device *vdev)
 		netif_carrier_on(dev);
 	}
 
+	for (i = 0; i < ARRAY_SIZE(guest_offloads); i++)
+		if (virtio_has_feature(vi->vdev, guest_offloads[i]))
+			set_bit(guest_offloads[i], &vi->guest_offloads);
+
 	pr_debug("virtnet: registered device %s with %d RX and TX vq's\n",
 		 dev->name, max_queue_pairs);
 
@@ -2597,15 +2672,6 @@ free:
 	return err;
 }
 
-static void _remove_vq_common(struct virtnet_info *vi)
-{
-	vi->vdev->config->reset(vi->vdev);
-	free_unused_bufs(vi);
-	_free_receive_bufs(vi);
-	free_receive_page_frags(vi);
-	virtnet_del_vqs(vi);
-}
-
 static void remove_vq_common(struct virtnet_info *vi)
 {
 	vi->vdev->config->reset(vi->vdev);
@@ -2637,8 +2703,7 @@ static void virtnet_remove(struct virtio_device *vdev)
 	free_netdev(vi->dev);
 }
 
-#ifdef CONFIG_PM_SLEEP
-static int virtnet_freeze(struct virtio_device *vdev)
+static __maybe_unused int virtnet_freeze(struct virtio_device *vdev)
 {
 	struct virtnet_info *vi = vdev->priv;
 
@@ -2649,7 +2714,7 @@ static int virtnet_freeze(struct virtio_device *vdev)
 	return 0;
 }
 
-static int virtnet_restore(struct virtio_device *vdev)
+static __maybe_unused int virtnet_restore(struct virtio_device *vdev)
 {
 	struct virtnet_info *vi = vdev->priv;
 	int err;
@@ -2665,7 +2730,6 @@ static int virtnet_restore(struct virtio_device *vdev)
 
 	return 0;
 }
-#endif
 
 static struct virtio_device_id id_table[] = {
 	{ VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
@@ -2682,7 +2746,7 @@ static struct virtio_device_id id_table[] = {
 	VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN, \
 	VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \
 	VIRTIO_NET_F_CTRL_MAC_ADDR, \
-	VIRTIO_NET_F_MTU
+	VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
 
 static unsigned int features[] = {
 	VIRTNET_FEATURES,
diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c
index 8a1eaf3c302a..9b243e6f3008 100644
--- a/drivers/net/vrf.c
+++ b/drivers/net/vrf.c
@@ -47,9 +47,7 @@ static unsigned int vrf_net_id;
 
 struct net_vrf {
 	struct rtable __rcu	*rth;
-	struct rtable __rcu	*rth_local;
 	struct rt6_info	__rcu	*rt6;
-	struct rt6_info	__rcu	*rt6_local;
 	u32                     tb_id;
 };
 
@@ -194,42 +192,10 @@ static netdev_tx_t vrf_process_v6_outbound(struct sk_buff *skb,
 
 	/* if dst.dev is loopback or the VRF device again this is locally
 	 * originated traffic destined to a local address. Short circuit
-	 * to Rx path using our local dst
+	 * to Rx path
 	 */
-	if (dst->dev == net->loopback_dev || dst->dev == dev) {
-		struct net_vrf *vrf = netdev_priv(dev);
-		struct rt6_info *rt6_local;
-
-		/* release looked up dst and use cached local dst */
-		dst_release(dst);
-
-		rcu_read_lock();
-
-		rt6_local = rcu_dereference(vrf->rt6_local);
-		if (unlikely(!rt6_local)) {
-			rcu_read_unlock();
-			goto err;
-		}
-
-		/* Ordering issue: cached local dst is created on newlink
-		 * before the IPv6 initialization. Using the local dst
-		 * requires rt6i_idev to be set so make sure it is.
-		 */
-		if (unlikely(!rt6_local->rt6i_idev)) {
-			rt6_local->rt6i_idev = in6_dev_get(dev);
-			if (!rt6_local->rt6i_idev) {
-				rcu_read_unlock();
-				goto err;
-			}
-		}
-
-		dst = &rt6_local->dst;
-		dst_hold(dst);
-
-		rcu_read_unlock();
-
-		return vrf_local_xmit(skb, dev, &rt6_local->dst);
-	}
+	if (dst->dev == dev)
+		return vrf_local_xmit(skb, dev, dst);
 
 	skb_dst_set(skb, dst);
 
@@ -296,30 +262,10 @@ static netdev_tx_t vrf_process_v4_outbound(struct sk_buff *skb,
 
 	/* if dst.dev is loopback or the VRF device again this is locally
 	 * originated traffic destined to a local address. Short circuit
-	 * to Rx path using our local dst
+	 * to Rx path
 	 */
-	if (rt->dst.dev == net->loopback_dev || rt->dst.dev == vrf_dev) {
-		struct net_vrf *vrf = netdev_priv(vrf_dev);
-		struct rtable *rth_local;
-		struct dst_entry *dst = NULL;
-
-		ip_rt_put(rt);
-
-		rcu_read_lock();
-
-		rth_local = rcu_dereference(vrf->rth_local);
-		if (likely(rth_local)) {
-			dst = &rth_local->dst;
-			dst_hold(dst);
-		}
-
-		rcu_read_unlock();
-
-		if (unlikely(!dst))
-			goto err;
-
-		return vrf_local_xmit(skb, vrf_dev, dst);
-	}
+	if (rt->dst.dev == vrf_dev)
+		return vrf_local_xmit(skb, vrf_dev, &rt->dst);
 
 	skb_dst_set(skb, &rt->dst);
 
@@ -528,12 +474,10 @@ static struct sk_buff *vrf_ip6_out(struct net_device *vrf_dev,
 static void vrf_rt6_release(struct net_device *dev, struct net_vrf *vrf)
 {
 	struct rt6_info *rt6 = rtnl_dereference(vrf->rt6);
-	struct rt6_info *rt6_local = rtnl_dereference(vrf->rt6_local);
 	struct net *net = dev_net(dev);
 	struct dst_entry *dst;
 
 	RCU_INIT_POINTER(vrf->rt6, NULL);
-	RCU_INIT_POINTER(vrf->rt6_local, NULL);
 	synchronize_rcu();
 
 	/* move dev in dst's to loopback so this VRF device can be deleted
@@ -546,19 +490,6 @@ static void vrf_rt6_release(struct net_device *dev, struct net_vrf *vrf)
 		dev_hold(dst->dev);
 		dst_release(dst);
 	}
-
-	if (rt6_local) {
-		if (rt6_local->rt6i_idev) {
-			in6_dev_put(rt6_local->rt6i_idev);
-			rt6_local->rt6i_idev = NULL;
-		}
-
-		dst = &rt6_local->dst;
-		dev_put(dst->dev);
-		dst->dev = net->loopback_dev;
-		dev_hold(dst->dev);
-		dst_release(dst);
-	}
 }
 
 static int vrf_rt6_create(struct net_device *dev)
@@ -567,7 +498,7 @@ static int vrf_rt6_create(struct net_device *dev)
 	struct net_vrf *vrf = netdev_priv(dev);
 	struct net *net = dev_net(dev);
 	struct fib6_table *rt6i_table;
-	struct rt6_info *rt6, *rt6_local;
+	struct rt6_info *rt6;
 	int rc = -ENOMEM;
 
 	/* IPv6 can be CONFIG enabled and then disabled runtime */
@@ -586,22 +517,7 @@ static int vrf_rt6_create(struct net_device *dev)
 	rt6->rt6i_table = rt6i_table;
 	rt6->dst.output	= vrf_output6;
 
-	/* create a dst for local routing - packets sent locally
-	 * to local address via the VRF device as a loopback
-	 */
-	rt6_local = ip6_dst_alloc(net, dev, flags);
-	if (!rt6_local) {
-		dst_release(&rt6->dst);
-		goto out;
-	}
-
-	rt6_local->rt6i_idev  = in6_dev_get(dev);
-	rt6_local->rt6i_flags = RTF_UP | RTF_NONEXTHOP | RTF_LOCAL;
-	rt6_local->rt6i_table = rt6i_table;
-	rt6_local->dst.input  = ip6_input;
-
 	rcu_assign_pointer(vrf->rt6, rt6);
-	rcu_assign_pointer(vrf->rt6_local, rt6_local);
 
 	rc = 0;
 out:
@@ -788,12 +704,10 @@ static struct sk_buff *vrf_l3_out(struct net_device *vrf_dev,
 static void vrf_rtable_release(struct net_device *dev, struct net_vrf *vrf)
 {
 	struct rtable *rth = rtnl_dereference(vrf->rth);
-	struct rtable *rth_local = rtnl_dereference(vrf->rth_local);
 	struct net *net = dev_net(dev);
 	struct dst_entry *dst;
 
 	RCU_INIT_POINTER(vrf->rth, NULL);
-	RCU_INIT_POINTER(vrf->rth_local, NULL);
 	synchronize_rcu();
 
 	/* move dev in dst's to loopback so this VRF device can be deleted
@@ -806,20 +720,12 @@ static void vrf_rtable_release(struct net_device *dev, struct net_vrf *vrf)
 		dev_hold(dst->dev);
 		dst_release(dst);
 	}
-
-	if (rth_local) {
-		dst = &rth_local->dst;
-		dev_put(dst->dev);
-		dst->dev = net->loopback_dev;
-		dev_hold(dst->dev);
-		dst_release(dst);
-	}
 }
 
 static int vrf_rtable_create(struct net_device *dev)
 {
 	struct net_vrf *vrf = netdev_priv(dev);
-	struct rtable *rth, *rth_local;
+	struct rtable *rth;
 
 	if (!fib_new_table(dev_net(dev), vrf->tb_id))
 		return -ENOMEM;
@@ -829,22 +735,10 @@ static int vrf_rtable_create(struct net_device *dev)
 	if (!rth)
 		return -ENOMEM;
 
-	/* create a dst for local ingress routing - packets sent locally
-	 * to local address via the VRF device as a loopback
-	 */
-	rth_local = rt_dst_alloc(dev, RTCF_LOCAL, RTN_LOCAL, 1, 1, 0);
-	if (!rth_local) {
-		dst_release(&rth->dst);
-		return -ENOMEM;
-	}
-
 	rth->dst.output	= vrf_output;
 	rth->rt_table_id = vrf->tb_id;
 
-	rth_local->rt_table_id = vrf->tb_id;
-
 	rcu_assign_pointer(vrf->rth, rth);
-	rcu_assign_pointer(vrf->rth_local, rth_local);
 
 	return 0;
 }
@@ -1063,12 +957,12 @@ static void vrf_ip6_input_dst(struct sk_buff *skb, struct net_device *vrf_dev,
 {
 	const struct ipv6hdr *iph = ipv6_hdr(skb);
 	struct flowi6 fl6 = {
+		.flowi6_iif     = ifindex,
+		.flowi6_mark    = skb->mark,
+		.flowi6_proto   = iph->nexthdr,
 		.daddr          = iph->daddr,
 		.saddr          = iph->saddr,
 		.flowlabel      = ip6_flowinfo(iph),
-		.flowi6_mark    = skb->mark,
-		.flowi6_proto   = iph->nexthdr,
-		.flowi6_iif     = ifindex,
 	};
 	struct net *net = dev_net(vrf_dev);
 	struct rt6_info *rt6;
@@ -1371,10 +1265,14 @@ static int vrf_validate(struct nlattr *tb[], struct nlattr *data[],
 			struct netlink_ext_ack *extack)
 {
 	if (tb[IFLA_ADDRESS]) {
-		if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN)
+		if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN) {
+			NL_SET_ERR_MSG(extack, "Invalid hardware address");
 			return -EINVAL;
-		if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS])))
+		}
+		if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS]))) {
+			NL_SET_ERR_MSG(extack, "Invalid hardware address");
 			return -EADDRNOTAVAIL;
+		}
 	}
 	return 0;
 }
@@ -1399,12 +1297,17 @@ static int vrf_newlink(struct net *src_net, struct net_device *dev,
 	struct net *net;
 	int err;
 
-	if (!data || !data[IFLA_VRF_TABLE])
+	if (!data || !data[IFLA_VRF_TABLE]) {
+		NL_SET_ERR_MSG(extack, "VRF table id is missing");
 		return -EINVAL;
+	}
 
 	vrf->tb_id = nla_get_u32(data[IFLA_VRF_TABLE]);
-	if (vrf->tb_id == RT_TABLE_UNSPEC)
+	if (vrf->tb_id == RT_TABLE_UNSPEC) {
+		NL_SET_ERR_MSG_ATTR(extack, data[IFLA_VRF_TABLE],
+				    "Invalid VRF table id");
 		return -EINVAL;
+	}
 
 	dev->priv_flags |= IFF_L3MDEV_MASTER;
 
diff --git a/drivers/net/vxlan.c b/drivers/net/vxlan.c
index e17baac70f43..d7c49cf1d5e9 100644
--- a/drivers/net/vxlan.c
+++ b/drivers/net/vxlan.c
@@ -26,6 +26,7 @@
 #include <net/inet_ecn.h>
 #include <net/net_namespace.h>
 #include <net/netns/generic.h>
+#include <net/tun_proto.h>
 #include <net/vxlan.h>
 
 #if IS_ENABLED(CONFIG_IPV6)
@@ -1261,19 +1262,9 @@ static bool vxlan_parse_gpe_hdr(struct vxlanhdr *unparsed,
 	if (gpe->oam_flag)
 		return false;
 
-	switch (gpe->next_protocol) {
-	case VXLAN_GPE_NP_IPV4:
-		*protocol = htons(ETH_P_IP);
-		break;
-	case VXLAN_GPE_NP_IPV6:
-		*protocol = htons(ETH_P_IPV6);
-		break;
-	case VXLAN_GPE_NP_ETHERNET:
-		*protocol = htons(ETH_P_TEB);
-		break;
-	default:
+	*protocol = tun_p_to_eth_p(gpe->next_protocol);
+	if (!*protocol)
 		return false;
-	}
 
 	unparsed->vx_flags &= ~VXLAN_GPE_USED_BITS;
 	return true;
@@ -1799,19 +1790,10 @@ static int vxlan_build_gpe_hdr(struct vxlanhdr *vxh, u32 vxflags,
 	struct vxlanhdr_gpe *gpe = (struct vxlanhdr_gpe *)vxh;
 
 	gpe->np_applied = 1;
-
-	switch (protocol) {
-	case htons(ETH_P_IP):
-		gpe->next_protocol = VXLAN_GPE_NP_IPV4;
-		return 0;
-	case htons(ETH_P_IPV6):
-		gpe->next_protocol = VXLAN_GPE_NP_IPV6;
-		return 0;
-	case htons(ETH_P_TEB):
-		gpe->next_protocol = VXLAN_GPE_NP_ETHERNET;
-		return 0;
-	}
-	return -EPFNOSUPPORT;
+	gpe->next_protocol = tun_p_from_eth_p(protocol);
+	if (!gpe->next_protocol)
+		return -EPFNOSUPPORT;
+	return 0;
 }
 
 static int vxlan_build_skb(struct sk_buff *skb, struct dst_entry *dst,
@@ -2609,7 +2591,7 @@ static struct device_type vxlan_type = {
  * supply the listening VXLAN udp ports. Callers are expected
  * to implement the ndo_udp_tunnel_add.
  */
-static void vxlan_push_rx_ports(struct net_device *dev)
+static void vxlan_offload_rx_ports(struct net_device *dev, bool push)
 {
 	struct vxlan_sock *vs;
 	struct net *net = dev_net(dev);
@@ -2618,11 +2600,19 @@ static void vxlan_push_rx_ports(struct net_device *dev)
 
 	spin_lock(&vn->sock_lock);
 	for (i = 0; i < PORT_HASH_SIZE; ++i) {
-		hlist_for_each_entry_rcu(vs, &vn->sock_list[i], hlist)
-			udp_tunnel_push_rx_port(dev, vs->sock,
-						(vs->flags & VXLAN_F_GPE) ?
-						UDP_TUNNEL_TYPE_VXLAN_GPE :
-						UDP_TUNNEL_TYPE_VXLAN);
+		hlist_for_each_entry_rcu(vs, &vn->sock_list[i], hlist) {
+			unsigned short type;
+
+			if (vs->flags & VXLAN_F_GPE)
+				type = UDP_TUNNEL_TYPE_VXLAN_GPE;
+			else
+				type = UDP_TUNNEL_TYPE_VXLAN;
+
+			if (push)
+				udp_tunnel_push_rx_port(dev, vs->sock, type);
+			else
+				udp_tunnel_drop_rx_port(dev, vs->sock, type);
+		}
 	}
 	spin_unlock(&vn->sock_lock);
 }
@@ -2721,12 +2711,14 @@ static int vxlan_validate(struct nlattr *tb[], struct nlattr *data[],
 {
 	if (tb[IFLA_ADDRESS]) {
 		if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN) {
-			pr_debug("invalid link address (not ethernet)\n");
+			NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_ADDRESS],
+					    "Provided link layer address is not Ethernet");
 			return -EINVAL;
 		}
 
 		if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS]))) {
-			pr_debug("invalid all zero ethernet address\n");
+			NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_ADDRESS],
+					    "Provided Ethernet address is not unicast");
 			return -EADDRNOTAVAIL;
 		}
 	}
@@ -2734,18 +2726,27 @@ static int vxlan_validate(struct nlattr *tb[], struct nlattr *data[],
 	if (tb[IFLA_MTU]) {
 		u32 mtu = nla_get_u32(tb[IFLA_MTU]);
 
-		if (mtu < ETH_MIN_MTU || mtu > ETH_MAX_MTU)
+		if (mtu < ETH_MIN_MTU || mtu > ETH_MAX_MTU) {
+			NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_MTU],
+					    "MTU must be between 68 and 65535");
 			return -EINVAL;
+		}
 	}
 
-	if (!data)
+	if (!data) {
+		NL_SET_ERR_MSG(extack,
+			       "Required attributes not provided to perform the operation");
 		return -EINVAL;
+	}
 
 	if (data[IFLA_VXLAN_ID]) {
 		u32 id = nla_get_u32(data[IFLA_VXLAN_ID]);
 
-		if (id >= VXLAN_N_VID)
+		if (id >= VXLAN_N_VID) {
+			NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_VXLAN_ID],
+					    "VXLAN ID must be lower than 16777216");
 			return -ERANGE;
+		}
 	}
 
 	if (data[IFLA_VXLAN_PORT_RANGE]) {
@@ -2753,8 +2754,8 @@ static int vxlan_validate(struct nlattr *tb[], struct nlattr *data[],
 			= nla_data(data[IFLA_VXLAN_PORT_RANGE]);
 
 		if (ntohs(p->high) < ntohs(p->low)) {
-			pr_debug("port range %u .. %u not valid\n",
-				 ntohs(p->low), ntohs(p->high));
+			NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_VXLAN_PORT_RANGE],
+					    "Invalid source port range");
 			return -EINVAL;
 		}
 	}
@@ -2911,7 +2912,8 @@ static int vxlan_sock_add(struct vxlan_dev *vxlan)
 
 static int vxlan_config_validate(struct net *src_net, struct vxlan_config *conf,
 				 struct net_device **lower,
-				 struct vxlan_dev *old)
+				 struct vxlan_dev *old,
+				 struct netlink_ext_ack *extack)
 {
 	struct vxlan_net *vn = net_generic(src_net, vxlan_net_id);
 	struct vxlan_dev *tmp;
@@ -2925,6 +2927,8 @@ static int vxlan_config_validate(struct net *src_net, struct vxlan_config *conf,
 		 */
 		if ((conf->flags & ~VXLAN_F_ALLOWED_GPE) ||
 		    !(conf->flags & VXLAN_F_COLLECT_METADATA)) {
+			NL_SET_ERR_MSG(extack,
+				       "VXLAN GPE does not support this combination of attributes");
 			return -EINVAL;
 		}
 	}
@@ -2939,15 +2943,23 @@ static int vxlan_config_validate(struct net *src_net, struct vxlan_config *conf,
 		conf->saddr.sa.sa_family = conf->remote_ip.sa.sa_family;
 	}
 
-	if (conf->saddr.sa.sa_family != conf->remote_ip.sa.sa_family)
+	if (conf->saddr.sa.sa_family != conf->remote_ip.sa.sa_family) {
+		NL_SET_ERR_MSG(extack,
+			       "Local and remote address must be from the same family");
 		return -EINVAL;
+	}
 
-	if (vxlan_addr_multicast(&conf->saddr))
+	if (vxlan_addr_multicast(&conf->saddr)) {
+		NL_SET_ERR_MSG(extack, "Local address cannot be multicast");
 		return -EINVAL;
+	}
 
 	if (conf->saddr.sa.sa_family == AF_INET6) {
-		if (!IS_ENABLED(CONFIG_IPV6))
+		if (!IS_ENABLED(CONFIG_IPV6)) {
+			NL_SET_ERR_MSG(extack,
+				       "IPv6 support not enabled in the kernel");
 			return -EPFNOSUPPORT;
+		}
 		use_ipv6 = true;
 		conf->flags |= VXLAN_F_IPV6;
 
@@ -2959,46 +2971,68 @@ static int vxlan_config_validate(struct net *src_net, struct vxlan_config *conf,
 
 			if (local_type & IPV6_ADDR_LINKLOCAL) {
 				if (!(remote_type & IPV6_ADDR_LINKLOCAL) &&
-				    (remote_type != IPV6_ADDR_ANY))
+				    (remote_type != IPV6_ADDR_ANY)) {
+					NL_SET_ERR_MSG(extack,
+						       "Invalid combination of local and remote address scopes");
 					return -EINVAL;
+				}
 
 				conf->flags |= VXLAN_F_IPV6_LINKLOCAL;
 			} else {
 				if (remote_type ==
-				    (IPV6_ADDR_UNICAST | IPV6_ADDR_LINKLOCAL))
+				    (IPV6_ADDR_UNICAST | IPV6_ADDR_LINKLOCAL)) {
+					NL_SET_ERR_MSG(extack,
+						       "Invalid combination of local and remote address scopes");
 					return -EINVAL;
+				}
 
 				conf->flags &= ~VXLAN_F_IPV6_LINKLOCAL;
 			}
 		}
 	}
 
-	if (conf->label && !use_ipv6)
+	if (conf->label && !use_ipv6) {
+		NL_SET_ERR_MSG(extack,
+			       "Label attribute only applies to IPv6 VXLAN devices");
 		return -EINVAL;
+	}
 
 	if (conf->remote_ifindex) {
 		struct net_device *lowerdev;
 
 		lowerdev = __dev_get_by_index(src_net, conf->remote_ifindex);
-		if (!lowerdev)
+		if (!lowerdev) {
+			NL_SET_ERR_MSG(extack,
+				       "Invalid local interface, device not found");
 			return -ENODEV;
+		}
 
 #if IS_ENABLED(CONFIG_IPV6)
 		if (use_ipv6) {
 			struct inet6_dev *idev = __in6_dev_get(lowerdev);
-			if (idev && idev->cnf.disable_ipv6)
+			if (idev && idev->cnf.disable_ipv6) {
+				NL_SET_ERR_MSG(extack,
+					       "IPv6 support disabled by administrator");
 				return -EPERM;
+			}
 		}
 #endif
 
 		*lower = lowerdev;
 	} else {
-		if (vxlan_addr_multicast(&conf->remote_ip))
+		if (vxlan_addr_multicast(&conf->remote_ip)) {
+			NL_SET_ERR_MSG(extack,
+				       "Local interface required for multicast remote destination");
+
 			return -EINVAL;
+		}
 
 #if IS_ENABLED(CONFIG_IPV6)
-		if (conf->flags & VXLAN_F_IPV6_LINKLOCAL)
+		if (conf->flags & VXLAN_F_IPV6_LINKLOCAL) {
+			NL_SET_ERR_MSG(extack,
+				       "Local interface required for link-local local/remote addresses");
 			return -EINVAL;
+		}
 #endif
 
 		*lower = NULL;
@@ -3030,6 +3064,8 @@ static int vxlan_config_validate(struct net *src_net, struct vxlan_config *conf,
 		    tmp->cfg.remote_ifindex != conf->remote_ifindex)
 			continue;
 
+		NL_SET_ERR_MSG(extack,
+			       "A VXLAN device with the specified VNI already exists");
 		return -EEXIST;
 	}
 
@@ -3089,14 +3125,14 @@ static void vxlan_config_apply(struct net_device *dev,
 }
 
 static int vxlan_dev_configure(struct net *src_net, struct net_device *dev,
-			       struct vxlan_config *conf,
-			       bool changelink)
+			       struct vxlan_config *conf, bool changelink,
+			       struct netlink_ext_ack *extack)
 {
 	struct vxlan_dev *vxlan = netdev_priv(dev);
 	struct net_device *lowerdev;
 	int ret;
 
-	ret = vxlan_config_validate(src_net, conf, &lowerdev, vxlan);
+	ret = vxlan_config_validate(src_net, conf, &lowerdev, vxlan, extack);
 	if (ret)
 		return ret;
 
@@ -3106,13 +3142,14 @@ static int vxlan_dev_configure(struct net *src_net, struct net_device *dev,
 }
 
 static int __vxlan_dev_create(struct net *net, struct net_device *dev,
-			      struct vxlan_config *conf)
+			      struct vxlan_config *conf,
+			      struct netlink_ext_ack *extack)
 {
 	struct vxlan_net *vn = net_generic(net, vxlan_net_id);
 	struct vxlan_dev *vxlan = netdev_priv(dev);
 	int err;
 
-	err = vxlan_dev_configure(net, dev, conf, false);
+	err = vxlan_dev_configure(net, dev, conf, false, extack);
 	if (err)
 		return err;
 
@@ -3358,7 +3395,7 @@ static int vxlan_newlink(struct net *src_net, struct net_device *dev,
 	if (err)
 		return err;
 
-	return __vxlan_dev_create(src_net, dev, &conf);
+	return __vxlan_dev_create(src_net, dev, &conf, extack);
 }
 
 static int vxlan_changelink(struct net_device *dev, struct nlattr *tb[],
@@ -3378,7 +3415,7 @@ static int vxlan_changelink(struct net_device *dev, struct nlattr *tb[],
 
 	memcpy(&old_dst, dst, sizeof(struct vxlan_rdst));
 
-	err = vxlan_dev_configure(vxlan->net, dev, &conf, true);
+	err = vxlan_dev_configure(vxlan->net, dev, &conf, true, extack);
 	if (err)
 		return err;
 
@@ -3584,7 +3621,7 @@ struct net_device *vxlan_dev_create(struct net *net, const char *name,
 	if (IS_ERR(dev))
 		return dev;
 
-	err = __vxlan_dev_create(net, dev, conf);
+	err = __vxlan_dev_create(net, dev, conf, NULL);
 	if (err < 0) {
 		free_netdev(dev);
 		return ERR_PTR(err);
@@ -3631,10 +3668,15 @@ static int vxlan_netdevice_event(struct notifier_block *unused,
 	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
 	struct vxlan_net *vn = net_generic(dev_net(dev), vxlan_net_id);
 
-	if (event == NETDEV_UNREGISTER)
+	if (event == NETDEV_UNREGISTER) {
+		vxlan_offload_rx_ports(dev, false);
 		vxlan_handle_lowerdev_unregister(vn, dev);
-	else if (event == NETDEV_UDP_TUNNEL_PUSH_INFO)
-		vxlan_push_rx_ports(dev);
+	} else if (event == NETDEV_REGISTER) {
+		vxlan_offload_rx_ports(dev, true);
+	} else if (event == NETDEV_UDP_TUNNEL_PUSH_INFO ||
+		   event == NETDEV_UDP_TUNNEL_DROP_INFO) {
+		vxlan_offload_rx_ports(dev, event == NETDEV_UDP_TUNNEL_PUSH_INFO);
+	}
 
 	return NOTIFY_DONE;
 }
diff --git a/drivers/net/wan/dscc4.c b/drivers/net/wan/dscc4.c
index 799830ffcae2..a043fb1367bd 100644
--- a/drivers/net/wan/dscc4.c
+++ b/drivers/net/wan/dscc4.c
@@ -483,20 +483,20 @@ static void dscc4_tx_print(struct net_device *dev,
 
 static void dscc4_release_ring(struct dscc4_dev_priv *dpriv)
 {
-	struct pci_dev *pdev = dpriv->pci_priv->pdev;
+	struct device *d = &dpriv->pci_priv->pdev->dev;
 	struct TxFD *tx_fd = dpriv->tx_fd;
 	struct RxFD *rx_fd = dpriv->rx_fd;
 	struct sk_buff **skbuff;
 	int i;
 
-	pci_free_consistent(pdev, TX_TOTAL_SIZE, tx_fd, dpriv->tx_fd_dma);
-	pci_free_consistent(pdev, RX_TOTAL_SIZE, rx_fd, dpriv->rx_fd_dma);
+	dma_free_coherent(d, TX_TOTAL_SIZE, tx_fd, dpriv->tx_fd_dma);
+	dma_free_coherent(d, RX_TOTAL_SIZE, rx_fd, dpriv->rx_fd_dma);
 
 	skbuff = dpriv->tx_skbuff;
 	for (i = 0; i < TX_RING_SIZE; i++) {
 		if (*skbuff) {
-			pci_unmap_single(pdev, le32_to_cpu(tx_fd->data),
-				(*skbuff)->len, PCI_DMA_TODEVICE);
+			dma_unmap_single(d, le32_to_cpu(tx_fd->data),
+					 (*skbuff)->len, DMA_TO_DEVICE);
 			dev_kfree_skb(*skbuff);
 		}
 		skbuff++;
@@ -506,8 +506,9 @@ static void dscc4_release_ring(struct dscc4_dev_priv *dpriv)
 	skbuff = dpriv->rx_skbuff;
 	for (i = 0; i < RX_RING_SIZE; i++) {
 		if (*skbuff) {
-			pci_unmap_single(pdev, le32_to_cpu(rx_fd->data),
-				RX_MAX(HDLC_MAX_MRU), PCI_DMA_FROMDEVICE);
+			dma_unmap_single(d, le32_to_cpu(rx_fd->data),
+					 RX_MAX(HDLC_MAX_MRU),
+					 DMA_FROM_DEVICE);
 			dev_kfree_skb(*skbuff);
 		}
 		skbuff++;
@@ -519,22 +520,30 @@ static inline int try_get_rx_skb(struct dscc4_dev_priv *dpriv,
 				 struct net_device *dev)
 {
 	unsigned int dirty = dpriv->rx_dirty%RX_RING_SIZE;
+	struct device *d = &dpriv->pci_priv->pdev->dev;
 	struct RxFD *rx_fd = dpriv->rx_fd + dirty;
 	const int len = RX_MAX(HDLC_MAX_MRU);
 	struct sk_buff *skb;
-	int ret = 0;
+	dma_addr_t addr;
 
 	skb = dev_alloc_skb(len);
+	if (!skb)
+		goto err_out;
+
+	skb->protocol = hdlc_type_trans(skb, dev);
+	addr = dma_map_single(d, skb->data, len, DMA_FROM_DEVICE);
+	if (dma_mapping_error(d, addr))
+		goto err_free_skb;
+
 	dpriv->rx_skbuff[dirty] = skb;
-	if (skb) {
-		skb->protocol = hdlc_type_trans(skb, dev);
-		rx_fd->data = cpu_to_le32(pci_map_single(dpriv->pci_priv->pdev,
-					  skb->data, len, PCI_DMA_FROMDEVICE));
-	} else {
-		rx_fd->data = 0;
-		ret = -1;
-	}
-	return ret;
+	rx_fd->data = cpu_to_le32(addr);
+	return 0;
+
+err_free_skb:
+	dev_kfree_skb_any(skb);
+err_out:
+	rx_fd->data = 0;
+	return -1;
 }
 
 /*
@@ -646,7 +655,7 @@ static inline void dscc4_rx_skb(struct dscc4_dev_priv *dpriv,
 				struct net_device *dev)
 {
 	struct RxFD *rx_fd = dpriv->rx_fd + dpriv->rx_current%RX_RING_SIZE;
-	struct pci_dev *pdev = dpriv->pci_priv->pdev;
+	struct device *d = &dpriv->pci_priv->pdev->dev;
 	struct sk_buff *skb;
 	int pkt_len;
 
@@ -656,8 +665,8 @@ static inline void dscc4_rx_skb(struct dscc4_dev_priv *dpriv,
 		goto refill;
 	}
 	pkt_len = TO_SIZE(le32_to_cpu(rx_fd->state2));
-	pci_unmap_single(pdev, le32_to_cpu(rx_fd->data),
-			 RX_MAX(HDLC_MAX_MRU), PCI_DMA_FROMDEVICE);
+	dma_unmap_single(d, le32_to_cpu(rx_fd->data),
+			 RX_MAX(HDLC_MAX_MRU), DMA_FROM_DEVICE);
 	if ((skb->data[--pkt_len] & FrameOk) == FrameOk) {
 		dev->stats.rx_packets++;
 		dev->stats.rx_bytes += pkt_len;
@@ -774,8 +783,8 @@ static int dscc4_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	rc = -ENOMEM;
 
-	priv->iqcfg = (__le32 *) pci_alloc_consistent(pdev,
-		IRQ_RING_SIZE*sizeof(__le32), &priv->iqcfg_dma);
+	priv->iqcfg = (__le32 *)dma_alloc_coherent(&pdev->dev,
+		IRQ_RING_SIZE*sizeof(__le32), &priv->iqcfg_dma, GFP_KERNEL);
 	if (!priv->iqcfg)
 		goto err_free_irq_5;
 	writel(priv->iqcfg_dma, ioaddr + IQCFG);
@@ -786,16 +795,18 @@ static int dscc4_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	 */
 	for (i = 0; i < dev_per_card; i++) {
 		dpriv = priv->root + i;
-		dpriv->iqtx = (__le32 *) pci_alloc_consistent(pdev,
-			IRQ_RING_SIZE*sizeof(u32), &dpriv->iqtx_dma);
+		dpriv->iqtx = (__le32 *)dma_alloc_coherent(&pdev->dev,
+			IRQ_RING_SIZE*sizeof(u32), &dpriv->iqtx_dma,
+			GFP_KERNEL);
 		if (!dpriv->iqtx)
 			goto err_free_iqtx_6;
 		writel(dpriv->iqtx_dma, ioaddr + IQTX0 + i*4);
 	}
 	for (i = 0; i < dev_per_card; i++) {
 		dpriv = priv->root + i;
-		dpriv->iqrx = (__le32 *) pci_alloc_consistent(pdev,
-			IRQ_RING_SIZE*sizeof(u32), &dpriv->iqrx_dma);
+		dpriv->iqrx = (__le32 *)dma_alloc_coherent(&pdev->dev,
+			IRQ_RING_SIZE*sizeof(u32), &dpriv->iqrx_dma,
+			GFP_KERNEL);
 		if (!dpriv->iqrx)
 			goto err_free_iqrx_7;
 		writel(dpriv->iqrx_dma, ioaddr + IQRX0 + i*4);
@@ -819,18 +830,18 @@ out:
 err_free_iqrx_7:
 	while (--i >= 0) {
 		dpriv = priv->root + i;
-		pci_free_consistent(pdev, IRQ_RING_SIZE*sizeof(u32),
-				    dpriv->iqrx, dpriv->iqrx_dma);
+		dma_free_coherent(&pdev->dev, IRQ_RING_SIZE*sizeof(u32),
+				  dpriv->iqrx, dpriv->iqrx_dma);
 	}
 	i = dev_per_card;
 err_free_iqtx_6:
 	while (--i >= 0) {
 		dpriv = priv->root + i;
-		pci_free_consistent(pdev, IRQ_RING_SIZE*sizeof(u32),
-				    dpriv->iqtx, dpriv->iqtx_dma);
+		dma_free_coherent(&pdev->dev, IRQ_RING_SIZE*sizeof(u32),
+				  dpriv->iqtx, dpriv->iqtx_dma);
 	}
-	pci_free_consistent(pdev, IRQ_RING_SIZE*sizeof(u32), priv->iqcfg,
-			    priv->iqcfg_dma);
+	dma_free_coherent(&pdev->dev, IRQ_RING_SIZE*sizeof(u32), priv->iqcfg,
+			  priv->iqcfg_dma);
 err_free_irq_5:
 	free_irq(pdev->irq, priv->root);
 err_release_4:
@@ -1145,16 +1156,23 @@ static netdev_tx_t dscc4_start_xmit(struct sk_buff *skb,
 					  struct net_device *dev)
 {
 	struct dscc4_dev_priv *dpriv = dscc4_priv(dev);
-	struct dscc4_pci_priv *ppriv = dpriv->pci_priv;
+	struct device *d = &dpriv->pci_priv->pdev->dev;
 	struct TxFD *tx_fd;
+	dma_addr_t addr;
 	int next;
 
+	addr = dma_map_single(d, skb->data, skb->len, DMA_TO_DEVICE);
+	if (dma_mapping_error(d, addr)) {
+		dev_kfree_skb_any(skb);
+		dev->stats.tx_dropped++;
+		return NETDEV_TX_OK;
+	}
+
 	next = dpriv->tx_current%TX_RING_SIZE;
 	dpriv->tx_skbuff[next] = skb;
 	tx_fd = dpriv->tx_fd + next;
 	tx_fd->state = FrameEnd | TO_STATE_TX(skb->len);
-	tx_fd->data = cpu_to_le32(pci_map_single(ppriv->pdev, skb->data, skb->len,
-				     PCI_DMA_TODEVICE));
+	tx_fd->data = cpu_to_le32(addr);
 	tx_fd->complete = 0x00000000;
 	tx_fd->jiffies = jiffies;
 	mb();
@@ -1572,8 +1590,9 @@ try:
 			tx_fd = dpriv->tx_fd + cur;
 			skb = dpriv->tx_skbuff[cur];
 			if (skb) {
-				pci_unmap_single(ppriv->pdev, le32_to_cpu(tx_fd->data),
-						 skb->len, PCI_DMA_TODEVICE);
+				dma_unmap_single(&ppriv->pdev->dev,
+						 le32_to_cpu(tx_fd->data),
+						 skb->len, DMA_TO_DEVICE);
 				if (tx_fd->state & FrameEnd) {
 					dev->stats.tx_packets++;
 					dev->stats.tx_bytes += skb->len;
@@ -1887,16 +1906,22 @@ static struct sk_buff *dscc4_init_dummy_skb(struct dscc4_dev_priv *dpriv)
 
 	skb = dev_alloc_skb(DUMMY_SKB_SIZE);
 	if (skb) {
+		struct device *d = &dpriv->pci_priv->pdev->dev;
 		int last = dpriv->tx_dirty%TX_RING_SIZE;
 		struct TxFD *tx_fd = dpriv->tx_fd + last;
+		dma_addr_t addr;
 
 		skb->len = DUMMY_SKB_SIZE;
 		skb_copy_to_linear_data(skb, version,
 					strlen(version) % DUMMY_SKB_SIZE);
+		addr = dma_map_single(d, skb->data, DUMMY_SKB_SIZE,
+				      DMA_TO_DEVICE);
+		if (dma_mapping_error(d, addr)) {
+			dev_kfree_skb_any(skb);
+			return NULL;
+		}
 		tx_fd->state = FrameEnd | TO_STATE_TX(DUMMY_SKB_SIZE);
-		tx_fd->data = cpu_to_le32(pci_map_single(dpriv->pci_priv->pdev,
-					     skb->data, DUMMY_SKB_SIZE,
-					     PCI_DMA_TODEVICE));
+		tx_fd->data = cpu_to_le32(addr);
 		dpriv->tx_skbuff[last] = skb;
 	}
 	return skb;
@@ -1905,18 +1930,20 @@ static struct sk_buff *dscc4_init_dummy_skb(struct dscc4_dev_priv *dpriv)
 static int dscc4_init_ring(struct net_device *dev)
 {
 	struct dscc4_dev_priv *dpriv = dscc4_priv(dev);
-	struct pci_dev *pdev = dpriv->pci_priv->pdev;
+	struct device *d = &dpriv->pci_priv->pdev->dev;
 	struct TxFD *tx_fd;
 	struct RxFD *rx_fd;
 	void *ring;
 	int i;
 
-	ring = pci_alloc_consistent(pdev, RX_TOTAL_SIZE, &dpriv->rx_fd_dma);
+	ring = dma_alloc_coherent(d, RX_TOTAL_SIZE, &dpriv->rx_fd_dma,
+				  GFP_KERNEL);
 	if (!ring)
 		goto err_out;
 	dpriv->rx_fd = rx_fd = (struct RxFD *) ring;
 
-	ring = pci_alloc_consistent(pdev, TX_TOTAL_SIZE, &dpriv->tx_fd_dma);
+	ring = dma_alloc_coherent(d, TX_TOTAL_SIZE, &dpriv->tx_fd_dma,
+				  GFP_KERNEL);
 	if (!ring)
 		goto err_free_dma_rx;
 	dpriv->tx_fd = tx_fd = (struct TxFD *) ring;
@@ -1954,9 +1981,9 @@ static int dscc4_init_ring(struct net_device *dev)
 	return 0;
 
 err_free_dma_tx:
-	pci_free_consistent(pdev, TX_TOTAL_SIZE, ring, dpriv->tx_fd_dma);
+	dma_free_coherent(d, TX_TOTAL_SIZE, ring, dpriv->tx_fd_dma);
 err_free_dma_rx:
-	pci_free_consistent(pdev, RX_TOTAL_SIZE, rx_fd, dpriv->rx_fd_dma);
+	dma_free_coherent(d, RX_TOTAL_SIZE, rx_fd, dpriv->rx_fd_dma);
 err_out:
 	return -ENOMEM;
 }
@@ -1976,16 +2003,16 @@ static void dscc4_remove_one(struct pci_dev *pdev)
 	dscc4_pci_reset(pdev, ioaddr);
 
 	free_irq(pdev->irq, root);
-	pci_free_consistent(pdev, IRQ_RING_SIZE*sizeof(u32), ppriv->iqcfg,
-			    ppriv->iqcfg_dma);
+	dma_free_coherent(&pdev->dev, IRQ_RING_SIZE*sizeof(u32), ppriv->iqcfg,
+			  ppriv->iqcfg_dma);
 	for (i = 0; i < dev_per_card; i++) {
 		struct dscc4_dev_priv *dpriv = root + i;
 
 		dscc4_release_ring(dpriv);
-		pci_free_consistent(pdev, IRQ_RING_SIZE*sizeof(u32),
-				    dpriv->iqrx, dpriv->iqrx_dma);
-		pci_free_consistent(pdev, IRQ_RING_SIZE*sizeof(u32),
-				    dpriv->iqtx, dpriv->iqtx_dma);
+		dma_free_coherent(&pdev->dev, IRQ_RING_SIZE*sizeof(u32),
+				  dpriv->iqrx, dpriv->iqrx_dma);
+		dma_free_coherent(&pdev->dev, IRQ_RING_SIZE*sizeof(u32),
+				  dpriv->iqtx, dpriv->iqtx_dma);
 	}
 
 	dscc4_free1(pdev);
diff --git a/drivers/net/wan/z85230.c b/drivers/net/wan/z85230.c
index 2f0bd6955f33..deea41e96f01 100644
--- a/drivers/net/wan/z85230.c
+++ b/drivers/net/wan/z85230.c
@@ -483,11 +483,10 @@ static void z8530_status(struct z8530_channel *chan)
 	write_zsctrl(chan, RES_H_IUS);
 }
 
-struct z8530_irqhandler z8530_sync =
-{
-	z8530_rx,
-	z8530_tx,
-	z8530_status
+struct z8530_irqhandler z8530_sync = {
+	.rx = z8530_rx,
+	.tx = z8530_tx,
+	.status = z8530_status,
 };
 
 EXPORT_SYMBOL(z8530_sync);
@@ -605,15 +604,15 @@ static void z8530_dma_status(struct z8530_channel *chan)
 }
 
 static struct z8530_irqhandler z8530_dma_sync = {
-	z8530_dma_rx,
-	z8530_dma_tx,
-	z8530_dma_status
+	.rx = z8530_dma_rx,
+	.tx = z8530_dma_tx,
+	.status = z8530_dma_status,
 };
 
 static struct z8530_irqhandler z8530_txdma_sync = {
-	z8530_rx,
-	z8530_dma_tx,
-	z8530_dma_status
+	.rx = z8530_rx,
+	.tx = z8530_dma_tx,
+	.status = z8530_dma_status,
 };
 
 /**
@@ -678,11 +677,10 @@ static void z8530_status_clear(struct z8530_channel *chan)
 	write_zsctrl(chan, RES_H_IUS);
 }
 
-struct z8530_irqhandler z8530_nop=
-{
-	z8530_rx_clear,
-	z8530_tx_clear,
-	z8530_status_clear
+struct z8530_irqhandler z8530_nop = {
+	.rx = z8530_rx_clear,
+	.tx = z8530_tx_clear,
+	.status = z8530_status_clear,
 };
 
 
diff --git a/drivers/net/wireless/ath/ar5523/ar5523.c b/drivers/net/wireless/ath/ar5523/ar5523.c
index 106d6f8d471a..68f0463ed8df 100644
--- a/drivers/net/wireless/ath/ar5523/ar5523.c
+++ b/drivers/net/wireless/ath/ar5523/ar5523.c
@@ -1749,7 +1749,7 @@ static void ar5523_disconnect(struct usb_interface *intf)
 	{ USB_DEVICE((vendor), (device) + 1), \
 		.driver_info = AR5523_FLAG_ABG|AR5523_FLAG_PRE_FIRMWARE }
 
-static struct usb_device_id ar5523_id_table[] = {
+static const struct usb_device_id ar5523_id_table[] = {
 	AR5523_DEVICE_UG(0x168c, 0x0001),	/* Atheros / AR5523 */
 	AR5523_DEVICE_UG(0x0cf3, 0x0001),	/* Atheros2 / AR5523_1 */
 	AR5523_DEVICE_UG(0x0cf3, 0x0003),	/* Atheros2 / AR5523_2 */
diff --git a/drivers/net/wireless/ath/ath10k/Kconfig b/drivers/net/wireless/ath/ath10k/Kconfig
index 412eb1380dcc..87f56d0e17a6 100644
--- a/drivers/net/wireless/ath/ath10k/Kconfig
+++ b/drivers/net/wireless/ath/ath10k/Kconfig
@@ -29,6 +29,13 @@ config ATH10K_SDIO
 	  This module adds experimental support for SDIO/MMC bus. Currently
 	  work in progress and will not fully work.
 
+config ATH10K_USB
+	tristate "Atheros ath10k USB support (EXPERIMENTAL)"
+	depends on ATH10K && USB
+	---help---
+	  This module adds experimental support for USB bus. Currently
+	  work in progress and will not fully work.
+
 config ATH10K_DEBUG
 	bool "Atheros ath10k debugging"
 	depends on ATH10K
diff --git a/drivers/net/wireless/ath/ath10k/Makefile b/drivers/net/wireless/ath/ath10k/Makefile
index b0b19a7eb98b..899b9b79f4ce 100644
--- a/drivers/net/wireless/ath/ath10k/Makefile
+++ b/drivers/net/wireless/ath/ath10k/Makefile
@@ -30,5 +30,8 @@ ath10k_pci-$(CONFIG_ATH10K_AHB) += ahb.o
 obj-$(CONFIG_ATH10K_SDIO) += ath10k_sdio.o
 ath10k_sdio-y += sdio.o
 
+obj-$(CONFIG_ATH10K_USB) += ath10k_usb.o
+ath10k_usb-y += usb.o
+
 # for tracing framework to find trace.h
 CFLAGS_trace.o := -I$(src)
diff --git a/drivers/net/wireless/ath/ath10k/ahb.c b/drivers/net/wireless/ath/ath10k/ahb.c
index da770af83036..ff6815e95684 100644
--- a/drivers/net/wireless/ath/ath10k/ahb.c
+++ b/drivers/net/wireless/ath/ath10k/ahb.c
@@ -197,35 +197,40 @@ static int ath10k_ahb_rst_ctrl_init(struct ath10k *ar)
 
 	dev = &ar_ahb->pdev->dev;
 
-	ar_ahb->core_cold_rst = devm_reset_control_get(dev, "wifi_core_cold");
+	ar_ahb->core_cold_rst = devm_reset_control_get_exclusive(dev,
+								 "wifi_core_cold");
 	if (IS_ERR(ar_ahb->core_cold_rst)) {
 		ath10k_err(ar, "failed to get core cold rst ctrl: %ld\n",
 			   PTR_ERR(ar_ahb->core_cold_rst));
 		return PTR_ERR(ar_ahb->core_cold_rst);
 	}
 
-	ar_ahb->radio_cold_rst = devm_reset_control_get(dev, "wifi_radio_cold");
+	ar_ahb->radio_cold_rst = devm_reset_control_get_exclusive(dev,
+								  "wifi_radio_cold");
 	if (IS_ERR(ar_ahb->radio_cold_rst)) {
 		ath10k_err(ar, "failed to get radio cold rst ctrl: %ld\n",
 			   PTR_ERR(ar_ahb->radio_cold_rst));
 		return PTR_ERR(ar_ahb->radio_cold_rst);
 	}
 
-	ar_ahb->radio_warm_rst = devm_reset_control_get(dev, "wifi_radio_warm");
+	ar_ahb->radio_warm_rst = devm_reset_control_get_exclusive(dev,
+								  "wifi_radio_warm");
 	if (IS_ERR(ar_ahb->radio_warm_rst)) {
 		ath10k_err(ar, "failed to get radio warm rst ctrl: %ld\n",
 			   PTR_ERR(ar_ahb->radio_warm_rst));
 		return PTR_ERR(ar_ahb->radio_warm_rst);
 	}
 
-	ar_ahb->radio_srif_rst = devm_reset_control_get(dev, "wifi_radio_srif");
+	ar_ahb->radio_srif_rst = devm_reset_control_get_exclusive(dev,
+								  "wifi_radio_srif");
 	if (IS_ERR(ar_ahb->radio_srif_rst)) {
 		ath10k_err(ar, "failed to get radio srif rst ctrl: %ld\n",
 			   PTR_ERR(ar_ahb->radio_srif_rst));
 		return PTR_ERR(ar_ahb->radio_srif_rst);
 	}
 
-	ar_ahb->cpu_init_rst = devm_reset_control_get(dev, "wifi_cpu_init");
+	ar_ahb->cpu_init_rst = devm_reset_control_get_exclusive(dev,
+								"wifi_cpu_init");
 	if (IS_ERR(ar_ahb->cpu_init_rst)) {
 		ath10k_err(ar, "failed to get cpu init rst ctrl: %ld\n",
 			   PTR_ERR(ar_ahb->cpu_init_rst));
@@ -787,8 +792,9 @@ static int ath10k_ahb_probe(struct platform_device *pdev)
 	ar_pci->mem = ar_ahb->mem;
 	ar_pci->mem_len = ar_ahb->mem_len;
 	ar_pci->ar = ar;
-	ar_pci->bus_ops = &ath10k_ahb_bus_ops;
+	ar_pci->ce.bus_ops = &ath10k_ahb_bus_ops;
 	ar_pci->targ_cpu_to_ce_addr = ath10k_ahb_qca4019_targ_cpu_to_ce_addr;
+	ar->ce_priv = &ar_pci->ce;
 
 	ret = ath10k_pci_setup_resource(ar);
 	if (ret) {
diff --git a/drivers/net/wireless/ath/ath10k/ce.c b/drivers/net/wireless/ath/ath10k/ce.c
index 08b84c8c3614..a8afd690290f 100644
--- a/drivers/net/wireless/ath/ath10k/ce.c
+++ b/drivers/net/wireless/ath/ath10k/ce.c
@@ -16,7 +16,6 @@
  */
 
 #include "hif.h"
-#include "pci.h"
 #include "ce.h"
 #include "debug.h"
 
@@ -33,7 +32,7 @@
  * Each ring consists of a number of descriptors which specify
  * an address, length, and meta-data.
  *
- * Typically, one side of the PCIe interconnect (Host or Target)
+ * Typically, one side of the PCIe/AHB/SNOC interconnect (Host or Target)
  * controls one ring and the other side controls the other ring.
  * The source side chooses when to initiate a transfer and it
  * chooses what to send (buffer address, length). The destination
@@ -73,57 +72,71 @@ ath10k_get_ring_byte(unsigned int offset,
 	return ((offset & addr_map->mask) >> (addr_map->lsb));
 }
 
+static inline u32 ath10k_ce_read32(struct ath10k *ar, u32 offset)
+{
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
+
+	return ce->bus_ops->read32(ar, offset);
+}
+
+static inline void ath10k_ce_write32(struct ath10k *ar, u32 offset, u32 value)
+{
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
+
+	ce->bus_ops->write32(ar, offset, value);
+}
+
 static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar,
 						       u32 ce_ctrl_addr,
 						       unsigned int n)
 {
-	ath10k_pci_write32(ar, ce_ctrl_addr +
-			ar->hw_ce_regs->dst_wr_index_addr, n);
+	ath10k_ce_write32(ar, ce_ctrl_addr +
+			  ar->hw_ce_regs->dst_wr_index_addr, n);
 }
 
 static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar,
 						      u32 ce_ctrl_addr)
 {
-	return ath10k_pci_read32(ar, ce_ctrl_addr +
-			ar->hw_ce_regs->dst_wr_index_addr);
+	return ath10k_ce_read32(ar, ce_ctrl_addr +
+				ar->hw_ce_regs->dst_wr_index_addr);
 }
 
 static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar,
 						      u32 ce_ctrl_addr,
 						      unsigned int n)
 {
-	ath10k_pci_write32(ar, ce_ctrl_addr +
-			ar->hw_ce_regs->sr_wr_index_addr, n);
+	ath10k_ce_write32(ar, ce_ctrl_addr +
+			  ar->hw_ce_regs->sr_wr_index_addr, n);
 }
 
 static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar,
 						     u32 ce_ctrl_addr)
 {
-	return ath10k_pci_read32(ar, ce_ctrl_addr +
-			ar->hw_ce_regs->sr_wr_index_addr);
+	return ath10k_ce_read32(ar, ce_ctrl_addr +
+				ar->hw_ce_regs->sr_wr_index_addr);
 }
 
 static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar,
 						    u32 ce_ctrl_addr)
 {
-	return ath10k_pci_read32(ar, ce_ctrl_addr +
-			ar->hw_ce_regs->current_srri_addr);
+	return ath10k_ce_read32(ar, ce_ctrl_addr +
+				ar->hw_ce_regs->current_srri_addr);
 }
 
 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar,
 						    u32 ce_ctrl_addr,
 						    unsigned int addr)
 {
-	ath10k_pci_write32(ar, ce_ctrl_addr +
-			ar->hw_ce_regs->sr_base_addr, addr);
+	ath10k_ce_write32(ar, ce_ctrl_addr +
+			  ar->hw_ce_regs->sr_base_addr, addr);
 }
 
 static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar,
 					       u32 ce_ctrl_addr,
 					       unsigned int n)
 {
-	ath10k_pci_write32(ar, ce_ctrl_addr +
-			ar->hw_ce_regs->sr_size_addr, n);
+	ath10k_ce_write32(ar, ce_ctrl_addr +
+			  ar->hw_ce_regs->sr_size_addr, n);
 }
 
 static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar,
@@ -131,12 +144,13 @@ static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar,
 					       unsigned int n)
 {
 	struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
-	u32 ctrl1_addr = ath10k_pci_read32(ar,
-					   ce_ctrl_addr + ctrl_regs->addr);
 
-	ath10k_pci_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
-			   (ctrl1_addr &  ~(ctrl_regs->dmax->mask)) |
-			   ath10k_set_ring_byte(n, ctrl_regs->dmax));
+	u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
+					  ctrl_regs->addr);
+
+	ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
+			  (ctrl1_addr &  ~(ctrl_regs->dmax->mask)) |
+			  ath10k_set_ring_byte(n, ctrl_regs->dmax));
 }
 
 static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar,
@@ -144,11 +158,13 @@ static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar,
 						    unsigned int n)
 {
 	struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
-	u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + ctrl_regs->addr);
 
-	ath10k_pci_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
-			   (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) |
-			   ath10k_set_ring_byte(n, ctrl_regs->src_ring));
+	u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
+					  ctrl_regs->addr);
+
+	ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
+			  (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) |
+			  ath10k_set_ring_byte(n, ctrl_regs->src_ring));
 }
 
 static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar,
@@ -156,34 +172,36 @@ static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar,
 						     unsigned int n)
 {
 	struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
-	u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + ctrl_regs->addr);
 
-	ath10k_pci_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
-			   (ctrl1_addr & ~(ctrl_regs->dst_ring->mask)) |
-			   ath10k_set_ring_byte(n, ctrl_regs->dst_ring));
+	u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
+					  ctrl_regs->addr);
+
+	ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
+			  (ctrl1_addr & ~(ctrl_regs->dst_ring->mask)) |
+			  ath10k_set_ring_byte(n, ctrl_regs->dst_ring));
 }
 
 static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar,
 						     u32 ce_ctrl_addr)
 {
-	return ath10k_pci_read32(ar, ce_ctrl_addr +
-				 ar->hw_ce_regs->current_drri_addr);
+	return ath10k_ce_read32(ar, ce_ctrl_addr +
+				ar->hw_ce_regs->current_drri_addr);
 }
 
 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar,
 						     u32 ce_ctrl_addr,
 						     u32 addr)
 {
-	ath10k_pci_write32(ar, ce_ctrl_addr +
-			ar->hw_ce_regs->dr_base_addr, addr);
+	ath10k_ce_write32(ar, ce_ctrl_addr +
+			  ar->hw_ce_regs->dr_base_addr, addr);
 }
 
 static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar,
 						u32 ce_ctrl_addr,
 						unsigned int n)
 {
-	ath10k_pci_write32(ar, ce_ctrl_addr +
-			ar->hw_ce_regs->dr_size_addr, n);
+	ath10k_ce_write32(ar, ce_ctrl_addr +
+			  ar->hw_ce_regs->dr_size_addr, n);
 }
 
 static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar,
@@ -191,11 +209,11 @@ static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar,
 						   unsigned int n)
 {
 	struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
-	u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + srcr_wm->addr);
+	u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
 
-	ath10k_pci_write32(ar, ce_ctrl_addr + srcr_wm->addr,
-			   (addr & ~(srcr_wm->wm_high->mask)) |
-			   (ath10k_set_ring_byte(n, srcr_wm->wm_high)));
+	ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
+			  (addr & ~(srcr_wm->wm_high->mask)) |
+			  (ath10k_set_ring_byte(n, srcr_wm->wm_high)));
 }
 
 static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar,
@@ -203,11 +221,11 @@ static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar,
 						  unsigned int n)
 {
 	struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
-	u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + srcr_wm->addr);
+	u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
 
-	ath10k_pci_write32(ar, ce_ctrl_addr + srcr_wm->addr,
-			   (addr & ~(srcr_wm->wm_low->mask)) |
-			   (ath10k_set_ring_byte(n, srcr_wm->wm_low)));
+	ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
+			  (addr & ~(srcr_wm->wm_low->mask)) |
+			  (ath10k_set_ring_byte(n, srcr_wm->wm_low)));
 }
 
 static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar,
@@ -215,11 +233,11 @@ static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar,
 						    unsigned int n)
 {
 	struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
-	u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + dstr_wm->addr);
+	u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
 
-	ath10k_pci_write32(ar, ce_ctrl_addr + dstr_wm->addr,
-			   (addr & ~(dstr_wm->wm_high->mask)) |
-			   (ath10k_set_ring_byte(n, dstr_wm->wm_high)));
+	ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
+			  (addr & ~(dstr_wm->wm_high->mask)) |
+			  (ath10k_set_ring_byte(n, dstr_wm->wm_high)));
 }
 
 static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar,
@@ -227,66 +245,73 @@ static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar,
 						   unsigned int n)
 {
 	struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
-	u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + dstr_wm->addr);
+	u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
 
-	ath10k_pci_write32(ar, ce_ctrl_addr + dstr_wm->addr,
-			   (addr & ~(dstr_wm->wm_low->mask)) |
-			   (ath10k_set_ring_byte(n, dstr_wm->wm_low)));
+	ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
+			  (addr & ~(dstr_wm->wm_low->mask)) |
+			  (ath10k_set_ring_byte(n, dstr_wm->wm_low)));
 }
 
 static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar,
 							u32 ce_ctrl_addr)
 {
 	struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
-	u32 host_ie_addr = ath10k_pci_read32(ar, ce_ctrl_addr +
-					ar->hw_ce_regs->host_ie_addr);
 
-	ath10k_pci_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
-			   host_ie_addr | host_ie->copy_complete->mask);
+	u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
+					    ar->hw_ce_regs->host_ie_addr);
+
+	ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
+			  host_ie_addr | host_ie->copy_complete->mask);
 }
 
 static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar,
 							u32 ce_ctrl_addr)
 {
 	struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
-	u32 host_ie_addr = ath10k_pci_read32(ar, ce_ctrl_addr +
-					ar->hw_ce_regs->host_ie_addr);
 
-	ath10k_pci_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
-			   host_ie_addr & ~(host_ie->copy_complete->mask));
+	u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
+					    ar->hw_ce_regs->host_ie_addr);
+
+	ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
+			  host_ie_addr & ~(host_ie->copy_complete->mask));
 }
 
 static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar,
 						    u32 ce_ctrl_addr)
 {
 	struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
-	u32 host_ie_addr = ath10k_pci_read32(ar, ce_ctrl_addr +
-					ar->hw_ce_regs->host_ie_addr);
 
-	ath10k_pci_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
-			   host_ie_addr & ~(wm_regs->wm_mask));
+	u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
+					    ar->hw_ce_regs->host_ie_addr);
+
+	ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
+			  host_ie_addr & ~(wm_regs->wm_mask));
 }
 
 static inline void ath10k_ce_error_intr_enable(struct ath10k *ar,
 					       u32 ce_ctrl_addr)
 {
 	struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;
-	u32 misc_ie_addr = ath10k_pci_read32(ar, ce_ctrl_addr +
-					ar->hw_ce_regs->misc_ie_addr);
 
-	ath10k_pci_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
-			   misc_ie_addr | misc_regs->err_mask);
+	u32 misc_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
+					    ar->hw_ce_regs->misc_ie_addr);
+
+	ath10k_ce_write32(ar,
+			  ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
+			  misc_ie_addr | misc_regs->err_mask);
 }
 
 static inline void ath10k_ce_error_intr_disable(struct ath10k *ar,
 						u32 ce_ctrl_addr)
 {
 	struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;
-	u32 misc_ie_addr = ath10k_pci_read32(ar, ce_ctrl_addr +
-					ar->hw_ce_regs->misc_ie_addr);
 
-	ath10k_pci_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
-			   misc_ie_addr & ~(misc_regs->err_mask));
+	u32 misc_ie_addr = ath10k_ce_read32(ar,
+			ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr);
+
+	ath10k_ce_write32(ar,
+			  ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
+			  misc_ie_addr & ~(misc_regs->err_mask));
 }
 
 static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar,
@@ -295,7 +320,7 @@ static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar,
 {
 	struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
 
-	ath10k_pci_write32(ar, ce_ctrl_addr + wm_regs->addr, mask);
+	ath10k_ce_write32(ar, ce_ctrl_addr + wm_regs->addr, mask);
 }
 
 /*
@@ -362,11 +387,11 @@ exit:
 void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe)
 {
 	struct ath10k *ar = pipe->ar;
-	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 	struct ath10k_ce_ring *src_ring = pipe->src_ring;
 	u32 ctrl_addr = pipe->ctrl_addr;
 
-	lockdep_assert_held(&ar_pci->ce_lock);
+	lockdep_assert_held(&ce->ce_lock);
 
 	/*
 	 * This function must be called only if there is an incomplete
@@ -394,13 +419,13 @@ int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,
 		   unsigned int flags)
 {
 	struct ath10k *ar = ce_state->ar;
-	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 	int ret;
 
-	spin_lock_bh(&ar_pci->ce_lock);
+	spin_lock_bh(&ce->ce_lock);
 	ret = ath10k_ce_send_nolock(ce_state, per_transfer_context,
 				    buffer, nbytes, transfer_id, flags);
-	spin_unlock_bh(&ar_pci->ce_lock);
+	spin_unlock_bh(&ce->ce_lock);
 
 	return ret;
 }
@@ -408,14 +433,14 @@ int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,
 int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe)
 {
 	struct ath10k *ar = pipe->ar;
-	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 	int delta;
 
-	spin_lock_bh(&ar_pci->ce_lock);
+	spin_lock_bh(&ce->ce_lock);
 	delta = CE_RING_DELTA(pipe->src_ring->nentries_mask,
 			      pipe->src_ring->write_index,
 			      pipe->src_ring->sw_index - 1);
-	spin_unlock_bh(&ar_pci->ce_lock);
+	spin_unlock_bh(&ce->ce_lock);
 
 	return delta;
 }
@@ -423,13 +448,13 @@ int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe)
 int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe)
 {
 	struct ath10k *ar = pipe->ar;
-	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
 	unsigned int nentries_mask = dest_ring->nentries_mask;
 	unsigned int write_index = dest_ring->write_index;
 	unsigned int sw_index = dest_ring->sw_index;
 
-	lockdep_assert_held(&ar_pci->ce_lock);
+	lockdep_assert_held(&ce->ce_lock);
 
 	return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1);
 }
@@ -437,7 +462,7 @@ int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe)
 int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, u32 paddr)
 {
 	struct ath10k *ar = pipe->ar;
-	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
 	unsigned int nentries_mask = dest_ring->nentries_mask;
 	unsigned int write_index = dest_ring->write_index;
@@ -446,7 +471,7 @@ int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, u32 paddr)
 	struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index);
 	u32 ctrl_addr = pipe->ctrl_addr;
 
-	lockdep_assert_held(&ar_pci->ce_lock);
+	lockdep_assert_held(&ce->ce_lock);
 
 	if ((pipe->id != 5) &&
 	    CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
@@ -486,12 +511,12 @@ void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries)
 int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, u32 paddr)
 {
 	struct ath10k *ar = pipe->ar;
-	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 	int ret;
 
-	spin_lock_bh(&ar_pci->ce_lock);
+	spin_lock_bh(&ce->ce_lock);
 	ret = __ath10k_ce_rx_post_buf(pipe, ctx, paddr);
-	spin_unlock_bh(&ar_pci->ce_lock);
+	spin_unlock_bh(&ce->ce_lock);
 
 	return ret;
 }
@@ -554,14 +579,14 @@ int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state,
 				  unsigned int *nbytesp)
 {
 	struct ath10k *ar = ce_state->ar;
-	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 	int ret;
 
-	spin_lock_bh(&ar_pci->ce_lock);
+	spin_lock_bh(&ce->ce_lock);
 	ret = ath10k_ce_completed_recv_next_nolock(ce_state,
 						   per_transfer_contextp,
 						   nbytesp);
-	spin_unlock_bh(&ar_pci->ce_lock);
+	spin_unlock_bh(&ce->ce_lock);
 
 	return ret;
 }
@@ -576,7 +601,7 @@ int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
 	unsigned int write_index;
 	int ret;
 	struct ath10k *ar;
-	struct ath10k_pci *ar_pci;
+	struct ath10k_ce *ce;
 
 	dest_ring = ce_state->dest_ring;
 
@@ -584,9 +609,9 @@ int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
 		return -EIO;
 
 	ar = ce_state->ar;
-	ar_pci = ath10k_pci_priv(ar);
+	ce = ath10k_ce_priv(ar);
 
-	spin_lock_bh(&ar_pci->ce_lock);
+	spin_lock_bh(&ce->ce_lock);
 
 	nentries_mask = dest_ring->nentries_mask;
 	sw_index = dest_ring->sw_index;
@@ -614,7 +639,7 @@ int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
 		ret = -EIO;
 	}
 
-	spin_unlock_bh(&ar_pci->ce_lock);
+	spin_unlock_bh(&ce->ce_lock);
 
 	return ret;
 }
@@ -686,7 +711,7 @@ int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,
 	unsigned int write_index;
 	int ret;
 	struct ath10k *ar;
-	struct ath10k_pci *ar_pci;
+	struct ath10k_ce *ce;
 
 	src_ring = ce_state->src_ring;
 
@@ -694,9 +719,9 @@ int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,
 		return -EIO;
 
 	ar = ce_state->ar;
-	ar_pci = ath10k_pci_priv(ar);
+	ce = ath10k_ce_priv(ar);
 
-	spin_lock_bh(&ar_pci->ce_lock);
+	spin_lock_bh(&ce->ce_lock);
 
 	nentries_mask = src_ring->nentries_mask;
 	sw_index = src_ring->sw_index;
@@ -727,7 +752,7 @@ int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,
 		ret = -EIO;
 	}
 
-	spin_unlock_bh(&ar_pci->ce_lock);
+	spin_unlock_bh(&ce->ce_lock);
 
 	return ret;
 }
@@ -736,13 +761,13 @@ int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state,
 				  void **per_transfer_contextp)
 {
 	struct ath10k *ar = ce_state->ar;
-	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 	int ret;
 
-	spin_lock_bh(&ar_pci->ce_lock);
+	spin_lock_bh(&ce->ce_lock);
 	ret = ath10k_ce_completed_send_next_nolock(ce_state,
 						   per_transfer_contextp);
-	spin_unlock_bh(&ar_pci->ce_lock);
+	spin_unlock_bh(&ce->ce_lock);
 
 	return ret;
 }
@@ -755,17 +780,18 @@ int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state,
  */
 void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id)
 {
-	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
-	struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id];
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
+	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 	struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
 	u32 ctrl_addr = ce_state->ctrl_addr;
 
-	spin_lock_bh(&ar_pci->ce_lock);
+	spin_lock_bh(&ce->ce_lock);
 
 	/* Clear the copy-complete interrupts that will be handled here. */
-	ath10k_ce_engine_int_status_clear(ar, ctrl_addr, wm_regs->cc_mask);
+	ath10k_ce_engine_int_status_clear(ar, ctrl_addr,
+					  wm_regs->cc_mask);
 
-	spin_unlock_bh(&ar_pci->ce_lock);
+	spin_unlock_bh(&ce->ce_lock);
 
 	if (ce_state->recv_cb)
 		ce_state->recv_cb(ce_state);
@@ -773,7 +799,7 @@ void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id)
 	if (ce_state->send_cb)
 		ce_state->send_cb(ce_state);
 
-	spin_lock_bh(&ar_pci->ce_lock);
+	spin_lock_bh(&ce->ce_lock);
 
 	/*
 	 * Misc CE interrupts are not being handled, but still need
@@ -781,7 +807,7 @@ void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id)
 	 */
 	ath10k_ce_engine_int_status_clear(ar, ctrl_addr, wm_regs->wm_mask);
 
-	spin_unlock_bh(&ar_pci->ce_lock);
+	spin_unlock_bh(&ce->ce_lock);
 }
 
 /*
@@ -795,7 +821,7 @@ void ath10k_ce_per_engine_service_any(struct ath10k *ar)
 	int ce_id;
 	u32 intr_summary;
 
-	intr_summary = CE_INTERRUPT_SUMMARY(ar);
+	intr_summary = ath10k_ce_interrupt_summary(ar);
 
 	for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) {
 		if (intr_summary & (1 << ce_id))
@@ -847,22 +873,25 @@ int ath10k_ce_disable_interrupts(struct ath10k *ar)
 
 void ath10k_ce_enable_interrupts(struct ath10k *ar)
 {
-	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 	int ce_id;
+	struct ath10k_ce_pipe *ce_state;
 
 	/* Skip the last copy engine, CE7 the diagnostic window, as that
 	 * uses polling and isn't initialized for interrupts.
 	 */
-	for (ce_id = 0; ce_id < CE_COUNT - 1; ce_id++)
-		ath10k_ce_per_engine_handler_adjust(&ar_pci->ce_states[ce_id]);
+	for (ce_id = 0; ce_id < CE_COUNT - 1; ce_id++) {
+		ce_state  = &ce->ce_states[ce_id];
+		ath10k_ce_per_engine_handler_adjust(ce_state);
+	}
 }
 
 static int ath10k_ce_init_src_ring(struct ath10k *ar,
 				   unsigned int ce_id,
 				   const struct ce_attr *attr)
 {
-	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
-	struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id];
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
+	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;
 	u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
 
@@ -898,8 +927,8 @@ static int ath10k_ce_init_dest_ring(struct ath10k *ar,
 				    unsigned int ce_id,
 				    const struct ce_attr *attr)
 {
-	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
-	struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id];
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
+	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
 	u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
 
@@ -1081,8 +1110,8 @@ void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id)
 int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id,
 			 const struct ce_attr *attr)
 {
-	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
-	struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id];
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
+	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 	int ret;
 
 	/*
@@ -1138,8 +1167,8 @@ int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id,
 
 void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
 {
-	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
-	struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id];
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
+	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 
 	if (ce_state->src_ring) {
 		dma_free_coherent(ar->dev,
@@ -1168,38 +1197,38 @@ void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
 void ath10k_ce_dump_registers(struct ath10k *ar,
 			      struct ath10k_fw_crash_data *crash_data)
 {
-	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
-	struct ath10k_ce_crash_data ce;
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
+	struct ath10k_ce_crash_data ce_data;
 	u32 addr, id;
 
 	lockdep_assert_held(&ar->data_lock);
 
 	ath10k_err(ar, "Copy Engine register dump:\n");
 
-	spin_lock_bh(&ar_pci->ce_lock);
+	spin_lock_bh(&ce->ce_lock);
 	for (id = 0; id < CE_COUNT; id++) {
 		addr = ath10k_ce_base_address(ar, id);
-		ce.base_addr = cpu_to_le32(addr);
+		ce_data.base_addr = cpu_to_le32(addr);
 
-		ce.src_wr_idx =
+		ce_data.src_wr_idx =
 			cpu_to_le32(ath10k_ce_src_ring_write_index_get(ar, addr));
-		ce.src_r_idx =
+		ce_data.src_r_idx =
 			cpu_to_le32(ath10k_ce_src_ring_read_index_get(ar, addr));
-		ce.dst_wr_idx =
+		ce_data.dst_wr_idx =
 			cpu_to_le32(ath10k_ce_dest_ring_write_index_get(ar, addr));
-		ce.dst_r_idx =
+		ce_data.dst_r_idx =
 			cpu_to_le32(ath10k_ce_dest_ring_read_index_get(ar, addr));
 
 		if (crash_data)
-			crash_data->ce_crash_data[id] = ce;
+			crash_data->ce_crash_data[id] = ce_data;
 
 		ath10k_err(ar, "[%02d]: 0x%08x %3u %3u %3u %3u", id,
-			   le32_to_cpu(ce.base_addr),
-			   le32_to_cpu(ce.src_wr_idx),
-			   le32_to_cpu(ce.src_r_idx),
-			   le32_to_cpu(ce.dst_wr_idx),
-			   le32_to_cpu(ce.dst_r_idx));
+			   le32_to_cpu(ce_data.base_addr),
+			   le32_to_cpu(ce_data.src_wr_idx),
+			   le32_to_cpu(ce_data.src_r_idx),
+			   le32_to_cpu(ce_data.dst_wr_idx),
+			   le32_to_cpu(ce_data.dst_r_idx));
 	}
 
-	spin_unlock_bh(&ar_pci->ce_lock);
+	spin_unlock_bh(&ce->ce_lock);
 }
diff --git a/drivers/net/wireless/ath/ath10k/ce.h b/drivers/net/wireless/ath/ath10k/ce.h
index 95743a57525d..bdec794704d9 100644
--- a/drivers/net/wireless/ath/ath10k/ce.h
+++ b/drivers/net/wireless/ath/ath10k/ce.h
@@ -122,6 +122,24 @@ struct ath10k_ce_pipe {
 /* Copy Engine settable attributes */
 struct ce_attr;
 
+struct ath10k_bus_ops {
+	u32 (*read32)(struct ath10k *ar, u32 offset);
+	void (*write32)(struct ath10k *ar, u32 offset, u32 value);
+	int (*get_num_banks)(struct ath10k *ar);
+};
+
+static inline struct ath10k_ce *ath10k_ce_priv(struct ath10k *ar)
+{
+	return (struct ath10k_ce *)ar->ce_priv;
+}
+
+struct ath10k_ce {
+	/* protects CE info */
+	spinlock_t ce_lock;
+	const struct ath10k_bus_ops *bus_ops;
+	struct ath10k_ce_pipe ce_states[CE_COUNT_MAX];
+};
+
 /*==================Send====================*/
 
 /* ath10k_ce_send flags */
@@ -291,9 +309,13 @@ static inline u32 ath10k_ce_base_address(struct ath10k *ar, unsigned int ce_id)
 		CE_WRAPPER_INTERRUPT_SUMMARY_HOST_MSI_LSB)
 #define CE_WRAPPER_INTERRUPT_SUMMARY_ADDRESS			0x0000
 
-#define CE_INTERRUPT_SUMMARY(ar) \
-	CE_WRAPPER_INTERRUPT_SUMMARY_HOST_MSI_GET( \
-		ath10k_pci_read32((ar), CE_WRAPPER_BASE_ADDRESS + \
-		CE_WRAPPER_INTERRUPT_SUMMARY_ADDRESS))
+static inline u32 ath10k_ce_interrupt_summary(struct ath10k *ar)
+{
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
+
+	return CE_WRAPPER_INTERRUPT_SUMMARY_HOST_MSI_GET(
+		ce->bus_ops->read32((ar), CE_WRAPPER_BASE_ADDRESS +
+		CE_WRAPPER_INTERRUPT_SUMMARY_ADDRESS));
+}
 
 #endif /* _CE_H_ */
diff --git a/drivers/net/wireless/ath/ath10k/core.c b/drivers/net/wireless/ath/ath10k/core.c
index 75c5c903c8a6..a4f635820f35 100644
--- a/drivers/net/wireless/ath/ath10k/core.c
+++ b/drivers/net/wireless/ath/ath10k/core.c
@@ -519,7 +519,7 @@ static const struct firmware *ath10k_fetch_fw_file(struct ath10k *ar,
 		dir = ".";
 
 	snprintf(filename, sizeof(filename), "%s/%s", dir, file);
-	ret = request_firmware_direct(&fw, filename, ar->dev);
+	ret = request_firmware(&fw, filename, ar->dev);
 	ath10k_dbg(ar, ATH10K_DBG_BOOT, "boot fw request '%s': %d\n",
 		   filename, ret);
 
@@ -1454,6 +1454,7 @@ static void ath10k_core_get_fw_name(struct ath10k *ar, char *fw_name,
 {
 	switch (ar->hif.bus) {
 	case ATH10K_BUS_SDIO:
+	case ATH10K_BUS_USB:
 		scnprintf(fw_name, fw_name_len, "%s-%s-%d.bin",
 			  ATH10K_FW_FILE_BASE, ath10k_bus_str(ar->hif.bus),
 			  fw_api);
@@ -1885,6 +1886,7 @@ static int ath10k_core_init_firmware_features(struct ath10k *ar)
 		ar->fw_stats_req_mask = WMI_10_4_STAT_PEER |
 					WMI_10_4_STAT_PEER_EXTD;
 		ar->max_spatial_stream = ar->hw_params.max_spatial_stream;
+		ar->max_num_tdls_vdevs = TARGET_10_4_NUM_TDLS_VDEVS;
 
 		if (test_bit(ATH10K_FW_FEATURE_PEER_FLOW_CONTROL,
 			     fw_file->fw_features))
@@ -2055,6 +2057,12 @@ int ath10k_core_start(struct ath10k *ar, enum ath10k_firmware_mode mode,
 		goto err_wmi_detach;
 	}
 
+	/* If firmware indicates Full Rx Reorder support it must be used in a
+	 * slightly different manner. Let HTT code know.
+	 */
+	ar->htt.rx_ring.in_ord_rx = !!(test_bit(WMI_SERVICE_RX_FULL_REORDER,
+						ar->wmi.svc_map));
+
 	status = ath10k_htt_rx_alloc(&ar->htt);
 	if (status) {
 		ath10k_err(ar, "failed to alloc htt rx: %d\n", status);
@@ -2123,6 +2131,14 @@ int ath10k_core_start(struct ath10k *ar, enum ath10k_firmware_mode mode,
 			     ar->running_fw->fw_file.fw_features))
 			val |= WMI_10_4_COEX_GPIO_SUPPORT;
 
+		if (test_bit(WMI_SERVICE_TDLS_EXPLICIT_MODE_ONLY,
+			     ar->wmi.svc_map))
+			val |= WMI_10_4_TDLS_EXPLICIT_MODE_ONLY;
+
+		if (test_bit(WMI_SERVICE_TDLS_UAPSD_BUFFER_STA,
+			     ar->wmi.svc_map))
+			val |= WMI_10_4_TDLS_UAPSD_BUFFER_STA;
+
 		status = ath10k_mac_ext_resource_config(ar, val);
 		if (status) {
 			ath10k_err(ar,
@@ -2167,12 +2183,6 @@ int ath10k_core_start(struct ath10k *ar, enum ath10k_firmware_mode mode,
 		}
 	}
 
-	/* If firmware indicates Full Rx Reorder support it must be used in a
-	 * slightly different manner. Let HTT code know.
-	 */
-	ar->htt.rx_ring.in_ord_rx = !!(test_bit(WMI_SERVICE_RX_FULL_REORDER,
-						ar->wmi.svc_map));
-
 	status = ath10k_htt_rx_ring_refill(ar);
 	if (status) {
 		ath10k_err(ar, "failed to refill htt rx ring: %d\n", status);
@@ -2516,6 +2526,11 @@ struct ath10k *ath10k_core_create(size_t priv_size, struct device *dev,
 		ar->hw_ce_regs = &qcax_ce_regs;
 		ar->hw_values = &qca4019_values;
 		break;
+	case ATH10K_HW_WCN3990:
+		ar->regs = &wcn3990_regs;
+		ar->hw_ce_regs = &wcn3990_ce_regs;
+		ar->hw_values = &wcn3990_values;
+		break;
 	default:
 		ath10k_err(ar, "unsupported core hardware revision %d\n",
 			   hw_rev);
diff --git a/drivers/net/wireless/ath/ath10k/core.h b/drivers/net/wireless/ath/ath10k/core.h
index 1aa5cf12fce0..949ebb3e967b 100644
--- a/drivers/net/wireless/ath/ath10k/core.h
+++ b/drivers/net/wireless/ath/ath10k/core.h
@@ -92,6 +92,7 @@ enum ath10k_bus {
 	ATH10K_BUS_PCI,
 	ATH10K_BUS_AHB,
 	ATH10K_BUS_SDIO,
+	ATH10K_BUS_USB,
 };
 
 static inline const char *ath10k_bus_str(enum ath10k_bus bus)
@@ -103,6 +104,8 @@ static inline const char *ath10k_bus_str(enum ath10k_bus bus)
 		return "ahb";
 	case ATH10K_BUS_SDIO:
 		return "sdio";
+	case ATH10K_BUS_USB:
+		return "usb";
 	}
 
 	return "unknown";
@@ -459,7 +462,7 @@ struct ath10k_ce_crash_hdr {
 struct ath10k_fw_crash_data {
 	bool crashed_since_read;
 
-	uuid_le uuid;
+	guid_t guid;
 	struct timespec timestamp;
 	__le32 registers[REG_DUMP_COUNT_QCA988X];
 	struct ath10k_ce_crash_data ce_crash_data[CE_COUNT_MAX];
@@ -993,6 +996,10 @@ struct ath10k {
 		u32 reg_ack_cts_timeout_orig;
 	} fw_coverage;
 
+	u32 ampdu_reference;
+
+	void *ce_priv;
+
 	/* must be last */
 	u8 drv_priv[0] __aligned(sizeof(void *));
 };
diff --git a/drivers/net/wireless/ath/ath10k/debug.c b/drivers/net/wireless/ath/ath10k/debug.c
index 389fcb7a9fd0..df514507d3f1 100644
--- a/drivers/net/wireless/ath/ath10k/debug.c
+++ b/drivers/net/wireless/ath/ath10k/debug.c
@@ -70,7 +70,7 @@ struct ath10k_dump_file_data {
 
 	/* some info we can get from ath10k struct that might help */
 
-	u8 uuid[16];
+	guid_t guid;
 
 	__le32 chip_id;
 
@@ -237,7 +237,7 @@ static ssize_t ath10k_read_wmi_services(struct file *file,
 {
 	struct ath10k *ar = file->private_data;
 	char *buf;
-	size_t len = 0, buf_len = 4096;
+	size_t len = 0, buf_len = 8192;
 	const char *name;
 	ssize_t ret_cnt;
 	bool enabled;
@@ -719,7 +719,7 @@ ath10k_debug_get_new_fw_crash_data(struct ath10k *ar)
 	lockdep_assert_held(&ar->data_lock);
 
 	crash_data->crashed_since_read = true;
-	uuid_le_gen(&crash_data->uuid);
+	guid_gen(&crash_data->guid);
 	getnstimeofday(&crash_data->timestamp);
 
 	return crash_data;
@@ -766,7 +766,7 @@ static struct ath10k_dump_file_data *ath10k_build_dump_file(struct ath10k *ar,
 
 	dump_data->version = cpu_to_le32(ATH10K_FW_CRASH_DUMP_VERSION);
 
-	memcpy(dump_data->uuid, &crash_data->uuid, sizeof(dump_data->uuid));
+	guid_copy(&dump_data->guid, &crash_data->guid);
 	dump_data->chip_id = cpu_to_le32(ar->chip_id);
 	dump_data->bus_type = cpu_to_le32(0);
 	dump_data->target_version = cpu_to_le32(ar->target_version);
diff --git a/drivers/net/wireless/ath/ath10k/debug.h b/drivers/net/wireless/ath/ath10k/debug.h
index 257d10985c6e..548ad5483a4a 100644
--- a/drivers/net/wireless/ath/ath10k/debug.h
+++ b/drivers/net/wireless/ath/ath10k/debug.h
@@ -40,6 +40,8 @@ enum ath10k_debug_mask {
 	ATH10K_DBG_AHB		= 0x00008000,
 	ATH10K_DBG_SDIO		= 0x00010000,
 	ATH10K_DBG_SDIO_DUMP	= 0x00020000,
+	ATH10K_DBG_USB		= 0x00040000,
+	ATH10K_DBG_USB_BULK	= 0x00080000,
 	ATH10K_DBG_ANY		= 0xffffffff,
 };
 
diff --git a/drivers/net/wireless/ath/ath10k/htt_rx.c b/drivers/net/wireless/ath/ath10k/htt_rx.c
index 398dda978d6e..a3f5dc78353f 100644
--- a/drivers/net/wireless/ath/ath10k/htt_rx.c
+++ b/drivers/net/wireless/ath/ath10k/htt_rx.c
@@ -890,16 +890,26 @@ static void ath10k_htt_rx_h_ppdu(struct ath10k *ar,
 		status->nss = 0;
 		status->encoding = RX_ENC_LEGACY;
 		status->bw = RATE_INFO_BW_20;
+
 		status->flag &= ~RX_FLAG_MACTIME_END;
 		status->flag |= RX_FLAG_NO_SIGNAL_VAL;
 
+		status->flag &= ~(RX_FLAG_AMPDU_IS_LAST);
+		status->flag |= RX_FLAG_AMPDU_DETAILS | RX_FLAG_AMPDU_LAST_KNOWN;
+		status->ampdu_reference = ar->ampdu_reference;
+
 		ath10k_htt_rx_h_signal(ar, status, rxd);
 		ath10k_htt_rx_h_channel(ar, status, rxd, vdev_id);
 		ath10k_htt_rx_h_rates(ar, status, rxd);
 	}
 
-	if (is_last_ppdu)
+	if (is_last_ppdu) {
 		ath10k_htt_rx_h_mactime(ar, status, rxd);
+
+		/* set ampdu last segment flag */
+		status->flag |= RX_FLAG_AMPDU_IS_LAST;
+		ar->ampdu_reference++;
+	}
 }
 
 static const char * const tid_to_ac[] = {
@@ -1514,7 +1524,7 @@ static bool ath10k_htt_rx_amsdu_allowed(struct ath10k *ar,
 	 */
 
 	if (!rx_status->freq) {
-		ath10k_warn(ar, "no channel configured; ignoring frame(s)!\n");
+		ath10k_dbg(ar, ATH10K_DBG_HTT, "no channel configured; ignoring frame(s)!\n");
 		return false;
 	}
 
@@ -1735,7 +1745,8 @@ static void ath10k_htt_rx_delba(struct ath10k *ar, struct htt_resp *resp)
 }
 
 static int ath10k_htt_rx_extract_amsdu(struct sk_buff_head *list,
-				       struct sk_buff_head *amsdu)
+				       struct sk_buff_head *amsdu,
+				       int budget_left)
 {
 	struct sk_buff *msdu;
 	struct htt_rx_desc *rxd;
@@ -1746,8 +1757,9 @@ static int ath10k_htt_rx_extract_amsdu(struct sk_buff_head *list,
 	if (WARN_ON(!skb_queue_empty(amsdu)))
 		return -EINVAL;
 
-	while ((msdu = __skb_dequeue(list))) {
+	while ((msdu = __skb_dequeue(list)) && budget_left) {
 		__skb_queue_tail(amsdu, msdu);
+		budget_left--;
 
 		rxd = (void *)msdu->data - sizeof(*rxd);
 		if (rxd->msdu_end.common.info0 &
@@ -1838,7 +1850,8 @@ static int ath10k_htt_rx_h_rx_offload(struct ath10k *ar,
 	return num_msdu;
 }
 
-static int ath10k_htt_rx_in_ord_ind(struct ath10k *ar, struct sk_buff *skb)
+static int ath10k_htt_rx_in_ord_ind(struct ath10k *ar, struct sk_buff *skb,
+				    int budget_left)
 {
 	struct ath10k_htt *htt = &ar->htt;
 	struct htt_resp *resp = (void *)skb->data;
@@ -1895,9 +1908,9 @@ static int ath10k_htt_rx_in_ord_ind(struct ath10k *ar, struct sk_buff *skb)
 	if (offload)
 		num_msdus = ath10k_htt_rx_h_rx_offload(ar, &list);
 
-	while (!skb_queue_empty(&list)) {
+	while (!skb_queue_empty(&list) && budget_left) {
 		__skb_queue_head_init(&amsdu);
-		ret = ath10k_htt_rx_extract_amsdu(&list, &amsdu);
+		ret = ath10k_htt_rx_extract_amsdu(&list, &amsdu, budget_left);
 		switch (ret) {
 		case 0:
 			/* Note: The in-order indication may report interleaved
@@ -1907,6 +1920,7 @@ static int ath10k_htt_rx_in_ord_ind(struct ath10k *ar, struct sk_buff *skb)
 			 * should still give an idea about rx rate to the user.
 			 */
 			num_msdus += skb_queue_len(&amsdu);
+			budget_left -= skb_queue_len(&amsdu);
 			ath10k_htt_rx_h_ppdu(ar, &amsdu, status, vdev_id);
 			ath10k_htt_rx_h_filter(ar, &amsdu, status);
 			ath10k_htt_rx_h_mpdu(ar, &amsdu, status);
@@ -2549,7 +2563,8 @@ int ath10k_htt_txrx_compl_task(struct ath10k *ar, int budget)
 		}
 
 		spin_lock_bh(&htt->rx_ring.lock);
-		num_rx_msdus = ath10k_htt_rx_in_ord_ind(ar, skb);
+		num_rx_msdus = ath10k_htt_rx_in_ord_ind(ar, skb,
+							(budget - quota));
 		spin_unlock_bh(&htt->rx_ring.lock);
 		if (num_rx_msdus < 0) {
 			resched_napi = true;
diff --git a/drivers/net/wireless/ath/ath10k/hw.c b/drivers/net/wireless/ath/ath10k/hw.c
index afb0c01cbb55..a860691d635d 100644
--- a/drivers/net/wireless/ath/ath10k/hw.c
+++ b/drivers/net/wireless/ath/ath10k/hw.c
@@ -192,6 +192,156 @@ const struct ath10k_hw_values qca4019_values = {
 	.ce_desc_meta_data_lsb          = 4,
 };
 
+const struct ath10k_hw_regs wcn3990_regs = {
+	.rtc_soc_base_address			= 0x00000000,
+	.rtc_wmac_base_address			= 0x00000000,
+	.soc_core_base_address			= 0x00000000,
+	.ce_wrapper_base_address		= 0x0024C000,
+	.ce0_base_address			= 0x00240000,
+	.ce1_base_address			= 0x00241000,
+	.ce2_base_address			= 0x00242000,
+	.ce3_base_address			= 0x00243000,
+	.ce4_base_address			= 0x00244000,
+	.ce5_base_address			= 0x00245000,
+	.ce6_base_address			= 0x00246000,
+	.ce7_base_address			= 0x00247000,
+	.ce8_base_address			= 0x00248000,
+	.ce9_base_address			= 0x00249000,
+	.ce10_base_address			= 0x0024A000,
+	.ce11_base_address			= 0x0024B000,
+	.soc_chip_id_address			= 0x000000f0,
+	.soc_reset_control_si0_rst_mask		= 0x00000001,
+	.soc_reset_control_ce_rst_mask		= 0x00000100,
+	.ce_wrap_intr_sum_host_msi_lsb		= 0x0000000c,
+	.ce_wrap_intr_sum_host_msi_mask		= 0x00fff000,
+	.pcie_intr_fw_mask			= 0x00100000,
+};
+
+static struct ath10k_hw_ce_regs_addr_map wcn3990_src_ring = {
+	.msb	= 0x00000010,
+	.lsb	= 0x00000010,
+	.mask	= GENMASK(17, 17),
+};
+
+static struct ath10k_hw_ce_regs_addr_map wcn3990_dst_ring = {
+	.msb	= 0x00000012,
+	.lsb	= 0x00000012,
+	.mask	= GENMASK(18, 18),
+};
+
+static struct ath10k_hw_ce_regs_addr_map wcn3990_dmax = {
+	.msb	= 0x00000000,
+	.lsb	= 0x00000000,
+	.mask	= GENMASK(15, 0),
+};
+
+static struct ath10k_hw_ce_ctrl1 wcn3990_ctrl1 = {
+	.addr		= 0x00000018,
+	.src_ring	= &wcn3990_src_ring,
+	.dst_ring	= &wcn3990_dst_ring,
+	.dmax		= &wcn3990_dmax,
+};
+
+static struct ath10k_hw_ce_regs_addr_map wcn3990_host_ie_cc = {
+	.mask	= GENMASK(0, 0),
+};
+
+static struct ath10k_hw_ce_host_ie wcn3990_host_ie = {
+	.copy_complete	= &wcn3990_host_ie_cc,
+};
+
+static struct ath10k_hw_ce_host_wm_regs wcn3990_wm_reg = {
+	.dstr_lmask	= 0x00000010,
+	.dstr_hmask	= 0x00000008,
+	.srcr_lmask	= 0x00000004,
+	.srcr_hmask	= 0x00000002,
+	.cc_mask	= 0x00000001,
+	.wm_mask	= 0x0000001E,
+	.addr		= 0x00000030,
+};
+
+static struct ath10k_hw_ce_misc_regs wcn3990_misc_reg = {
+	.axi_err	= 0x00000100,
+	.dstr_add_err	= 0x00000200,
+	.srcr_len_err	= 0x00000100,
+	.dstr_mlen_vio	= 0x00000080,
+	.dstr_overflow	= 0x00000040,
+	.srcr_overflow	= 0x00000020,
+	.err_mask	= 0x000003E0,
+	.addr		= 0x00000038,
+};
+
+static struct ath10k_hw_ce_regs_addr_map wcn3990_src_wm_low = {
+	.msb	= 0x00000000,
+	.lsb	= 0x00000010,
+	.mask	= GENMASK(31, 16),
+};
+
+static struct ath10k_hw_ce_regs_addr_map wcn3990_src_wm_high = {
+	.msb	= 0x0000000f,
+	.lsb	= 0x00000000,
+	.mask	= GENMASK(15, 0),
+};
+
+static struct ath10k_hw_ce_dst_src_wm_regs wcn3990_wm_src_ring = {
+	.addr		= 0x0000004c,
+	.low_rst	= 0x00000000,
+	.high_rst	= 0x00000000,
+	.wm_low		= &wcn3990_src_wm_low,
+	.wm_high	= &wcn3990_src_wm_high,
+};
+
+static struct ath10k_hw_ce_regs_addr_map wcn3990_dst_wm_low = {
+	.lsb	= 0x00000010,
+	.mask	= GENMASK(31, 16),
+};
+
+static struct ath10k_hw_ce_regs_addr_map wcn3990_dst_wm_high = {
+	.msb	= 0x0000000f,
+	.lsb	= 0x00000000,
+	.mask	= GENMASK(15, 0),
+};
+
+static struct ath10k_hw_ce_dst_src_wm_regs wcn3990_wm_dst_ring = {
+	.addr		= 0x00000050,
+	.low_rst	= 0x00000000,
+	.high_rst	= 0x00000000,
+	.wm_low		= &wcn3990_dst_wm_low,
+	.wm_high	= &wcn3990_dst_wm_high,
+};
+
+struct ath10k_hw_ce_regs wcn3990_ce_regs = {
+	.sr_base_addr		= 0x00000000,
+	.sr_size_addr		= 0x00000008,
+	.dr_base_addr		= 0x0000000c,
+	.dr_size_addr		= 0x00000014,
+	.misc_ie_addr		= 0x00000034,
+	.sr_wr_index_addr	= 0x0000003c,
+	.dst_wr_index_addr	= 0x00000040,
+	.current_srri_addr	= 0x00000044,
+	.current_drri_addr	= 0x00000048,
+	.ddr_addr_for_rri_low	= 0x00000004,
+	.ddr_addr_for_rri_high	= 0x00000008,
+	.ce_rri_low		= 0x0024C004,
+	.ce_rri_high		= 0x0024C008,
+	.host_ie_addr		= 0x0000002c,
+	.ctrl1_regs		= &wcn3990_ctrl1,
+	.host_ie		= &wcn3990_host_ie,
+	.wm_regs		= &wcn3990_wm_reg,
+	.misc_regs		= &wcn3990_misc_reg,
+	.wm_srcr		= &wcn3990_wm_src_ring,
+	.wm_dstr		= &wcn3990_wm_dst_ring,
+};
+
+const struct ath10k_hw_values wcn3990_values = {
+	.rtc_state_val_on		= 5,
+	.ce_count			= 12,
+	.msi_assign_ce_max		= 12,
+	.num_target_ce_config_wlan	= 12,
+	.ce_desc_meta_data_mask		= 0xFFF0,
+	.ce_desc_meta_data_lsb		= 4,
+};
+
 static struct ath10k_hw_ce_regs_addr_map qcax_src_ring = {
 	.msb	= 0x00000010,
 	.lsb	= 0x00000010,
diff --git a/drivers/net/wireless/ath/ath10k/hw.h b/drivers/net/wireless/ath/ath10k/hw.h
index 97dc1479f44e..0c089f6dd3d9 100644
--- a/drivers/net/wireless/ath/ath10k/hw.h
+++ b/drivers/net/wireless/ath/ath10k/hw.h
@@ -231,6 +231,7 @@ enum ath10k_hw_rev {
 	ATH10K_HW_QCA9377,
 	ATH10K_HW_QCA4019,
 	ATH10K_HW_QCA9887,
+	ATH10K_HW_WCN3990,
 };
 
 struct ath10k_hw_regs {
@@ -247,6 +248,10 @@ struct ath10k_hw_regs {
 	u32 ce5_base_address;
 	u32 ce6_base_address;
 	u32 ce7_base_address;
+	u32 ce8_base_address;
+	u32 ce9_base_address;
+	u32 ce10_base_address;
+	u32 ce11_base_address;
 	u32 soc_reset_control_si0_rst_mask;
 	u32 soc_reset_control_ce_rst_mask;
 	u32 soc_chip_id_address;
@@ -267,6 +272,7 @@ extern const struct ath10k_hw_regs qca988x_regs;
 extern const struct ath10k_hw_regs qca6174_regs;
 extern const struct ath10k_hw_regs qca99x0_regs;
 extern const struct ath10k_hw_regs qca4019_regs;
+extern const struct ath10k_hw_regs wcn3990_regs;
 
 struct ath10k_hw_ce_regs_addr_map {
 	u32 msb;
@@ -362,6 +368,8 @@ extern const struct ath10k_hw_values qca6174_values;
 extern const struct ath10k_hw_values qca99x0_values;
 extern const struct ath10k_hw_values qca9888_values;
 extern const struct ath10k_hw_values qca4019_values;
+extern const struct ath10k_hw_values wcn3990_values;
+extern struct ath10k_hw_ce_regs wcn3990_ce_regs;
 extern struct ath10k_hw_ce_regs qcax_ce_regs;
 
 void ath10k_hw_fill_survey_time(struct ath10k *ar, struct survey_info *survey,
@@ -375,6 +383,7 @@ void ath10k_hw_fill_survey_time(struct ath10k *ar, struct survey_info *survey,
 #define QCA_REV_9984(ar) ((ar)->hw_rev == ATH10K_HW_QCA9984)
 #define QCA_REV_9377(ar) ((ar)->hw_rev == ATH10K_HW_QCA9377)
 #define QCA_REV_40XX(ar) ((ar)->hw_rev == ATH10K_HW_QCA4019)
+#define QCA_REV_WCN3990(ar) ((ar)->hw_rev == ATH10K_HW_WCN3990)
 
 /* Known peculiarities:
  *  - raw appears in nwifi decap, raw and nwifi appear in ethernet decap
@@ -711,6 +720,11 @@ ath10k_rx_desc_get_l3_pad_bytes(struct ath10k_hw_params *hw,
 #define TARGET_10_4_IPHDR_PAD_CONFIG		1
 #define TARGET_10_4_QWRAP_CONFIG		0
 
+/* TDLS config */
+#define TARGET_10_4_NUM_TDLS_VDEVS		1
+#define TARGET_10_4_NUM_TDLS_BUFFER_STA		1
+#define TARGET_10_4_NUM_TDLS_SLEEP_STA		1
+
 /* Maximum number of Copy Engine's supported */
 #define CE_COUNT_MAX 12
 
diff --git a/drivers/net/wireless/ath/ath10k/mac.c b/drivers/net/wireless/ath/ath10k/mac.c
index 55c808f03a84..5683f1a5330e 100644
--- a/drivers/net/wireless/ath/ath10k/mac.c
+++ b/drivers/net/wireless/ath/ath10k/mac.c
@@ -7644,6 +7644,7 @@ static const struct ieee80211_ops ath10k_ops = {
 #ifdef CONFIG_PM
 	.suspend			= ath10k_wow_op_suspend,
 	.resume				= ath10k_wow_op_resume,
+	.set_wakeup			= ath10k_wow_op_set_wakeup,
 #endif
 #ifdef CONFIG_MAC80211_DEBUGFS
 	.sta_add_debugfs		= ath10k_sta_add_debugfs,
@@ -8197,8 +8198,11 @@ int ath10k_mac_register(struct ath10k *ar)
 			NL80211_PROBE_RESP_OFFLOAD_SUPPORT_P2P;
 	}
 
-	if (test_bit(WMI_SERVICE_TDLS, ar->wmi.svc_map))
+	if (test_bit(WMI_SERVICE_TDLS, ar->wmi.svc_map) ||
+	    test_bit(WMI_SERVICE_TDLS_EXPLICIT_MODE_ONLY, ar->wmi.svc_map)) {
 		ar->hw->wiphy->flags |= WIPHY_FLAG_SUPPORTS_TDLS;
+		ieee80211_hw_set(ar->hw, TDLS_WIDER_BW);
+	}
 
 	ar->hw->wiphy->flags |= WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL;
 	ar->hw->wiphy->flags |= WIPHY_FLAG_HAS_CHANNEL_SWITCH;
diff --git a/drivers/net/wireless/ath/ath10k/pci.c b/drivers/net/wireless/ath/ath10k/pci.c
index 7ebfc409018d..bc1633945a56 100644
--- a/drivers/net/wireless/ath/ath10k/pci.c
+++ b/drivers/net/wireless/ath/ath10k/pci.c
@@ -672,16 +672,16 @@ static u32 ath10k_bus_pci_read32(struct ath10k *ar, u32 offset)
 
 inline void ath10k_pci_write32(struct ath10k *ar, u32 offset, u32 value)
 {
-	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 
-	ar_pci->bus_ops->write32(ar, offset, value);
+	ce->bus_ops->write32(ar, offset, value);
 }
 
 inline u32 ath10k_pci_read32(struct ath10k *ar, u32 offset)
 {
-	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 
-	return ar_pci->bus_ops->read32(ar, offset);
+	return ce->bus_ops->read32(ar, offset);
 }
 
 u32 ath10k_pci_soc_read32(struct ath10k *ar, u32 addr)
@@ -761,7 +761,7 @@ static inline const char *ath10k_pci_get_irq_method(struct ath10k *ar)
 static int __ath10k_pci_rx_post_buf(struct ath10k_pci_pipe *pipe)
 {
 	struct ath10k *ar = pipe->hif_ce_state;
-	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 	struct ath10k_ce_pipe *ce_pipe = pipe->ce_hdl;
 	struct sk_buff *skb;
 	dma_addr_t paddr;
@@ -784,9 +784,9 @@ static int __ath10k_pci_rx_post_buf(struct ath10k_pci_pipe *pipe)
 
 	ATH10K_SKB_RXCB(skb)->paddr = paddr;
 
-	spin_lock_bh(&ar_pci->ce_lock);
+	spin_lock_bh(&ce->ce_lock);
 	ret = __ath10k_ce_rx_post_buf(ce_pipe, skb, paddr);
-	spin_unlock_bh(&ar_pci->ce_lock);
+	spin_unlock_bh(&ce->ce_lock);
 	if (ret) {
 		dma_unmap_single(ar->dev, paddr, skb->len + skb_tailroom(skb),
 				 DMA_FROM_DEVICE);
@@ -801,6 +801,7 @@ static void ath10k_pci_rx_post_pipe(struct ath10k_pci_pipe *pipe)
 {
 	struct ath10k *ar = pipe->hif_ce_state;
 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 	struct ath10k_ce_pipe *ce_pipe = pipe->ce_hdl;
 	int ret, num;
 
@@ -810,9 +811,9 @@ static void ath10k_pci_rx_post_pipe(struct ath10k_pci_pipe *pipe)
 	if (!ce_pipe->dest_ring)
 		return;
 
-	spin_lock_bh(&ar_pci->ce_lock);
+	spin_lock_bh(&ce->ce_lock);
 	num = __ath10k_ce_rx_num_free_bufs(ce_pipe);
-	spin_unlock_bh(&ar_pci->ce_lock);
+	spin_unlock_bh(&ce->ce_lock);
 
 	while (num >= 0) {
 		ret = __ath10k_pci_rx_post_buf(pipe);
@@ -882,6 +883,7 @@ static int ath10k_pci_diag_read_mem(struct ath10k *ar, u32 address, void *data,
 				    int nbytes)
 {
 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 	int ret = 0;
 	u32 *buf;
 	unsigned int completed_nbytes, alloc_nbytes, remaining_bytes;
@@ -892,7 +894,7 @@ static int ath10k_pci_diag_read_mem(struct ath10k *ar, u32 address, void *data,
 	void *data_buf = NULL;
 	int i;
 
-	spin_lock_bh(&ar_pci->ce_lock);
+	spin_lock_bh(&ce->ce_lock);
 
 	ce_diag = ar_pci->ce_diag;
 
@@ -986,7 +988,7 @@ done:
 		dma_free_coherent(ar->dev, alloc_nbytes, data_buf,
 				  ce_data_base);
 
-	spin_unlock_bh(&ar_pci->ce_lock);
+	spin_unlock_bh(&ce->ce_lock);
 
 	return ret;
 }
@@ -1034,6 +1036,7 @@ int ath10k_pci_diag_write_mem(struct ath10k *ar, u32 address,
 			      const void *data, int nbytes)
 {
 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 	int ret = 0;
 	u32 *buf;
 	unsigned int completed_nbytes, orig_nbytes, remaining_bytes;
@@ -1043,7 +1046,7 @@ int ath10k_pci_diag_write_mem(struct ath10k *ar, u32 address,
 	dma_addr_t ce_data_base = 0;
 	int i;
 
-	spin_lock_bh(&ar_pci->ce_lock);
+	spin_lock_bh(&ce->ce_lock);
 
 	ce_diag = ar_pci->ce_diag;
 
@@ -1147,7 +1150,7 @@ done:
 		ath10k_warn(ar, "failed to write diag value at 0x%x: %d\n",
 			    address, ret);
 
-	spin_unlock_bh(&ar_pci->ce_lock);
+	spin_unlock_bh(&ce->ce_lock);
 
 	return ret;
 }
@@ -1342,6 +1345,7 @@ int ath10k_pci_hif_tx_sg(struct ath10k *ar, u8 pipe_id,
 			 struct ath10k_hif_sg_item *items, int n_items)
 {
 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 	struct ath10k_pci_pipe *pci_pipe = &ar_pci->pipe_info[pipe_id];
 	struct ath10k_ce_pipe *ce_pipe = pci_pipe->ce_hdl;
 	struct ath10k_ce_ring *src_ring = ce_pipe->src_ring;
@@ -1350,7 +1354,7 @@ int ath10k_pci_hif_tx_sg(struct ath10k *ar, u8 pipe_id,
 	unsigned int write_index;
 	int err, i = 0;
 
-	spin_lock_bh(&ar_pci->ce_lock);
+	spin_lock_bh(&ce->ce_lock);
 
 	nentries_mask = src_ring->nentries_mask;
 	sw_index = src_ring->sw_index;
@@ -1396,14 +1400,14 @@ int ath10k_pci_hif_tx_sg(struct ath10k *ar, u8 pipe_id,
 	if (err)
 		goto err;
 
-	spin_unlock_bh(&ar_pci->ce_lock);
+	spin_unlock_bh(&ce->ce_lock);
 	return 0;
 
 err:
 	for (; i > 0; i--)
 		__ath10k_ce_send_revert(ce_pipe);
 
-	spin_unlock_bh(&ar_pci->ce_lock);
+	spin_unlock_bh(&ce->ce_lock);
 	return err;
 }
 
@@ -1459,7 +1463,7 @@ static void ath10k_pci_dump_registers(struct ath10k *ar,
 static void ath10k_pci_fw_crashed_dump(struct ath10k *ar)
 {
 	struct ath10k_fw_crash_data *crash_data;
-	char uuid[50];
+	char guid[UUID_STRING_LEN + 1];
 
 	spin_lock_bh(&ar->data_lock);
 
@@ -1468,11 +1472,11 @@ static void ath10k_pci_fw_crashed_dump(struct ath10k *ar)
 	crash_data = ath10k_debug_get_new_fw_crash_data(ar);
 
 	if (crash_data)
-		scnprintf(uuid, sizeof(uuid), "%pUl", &crash_data->uuid);
+		scnprintf(guid, sizeof(guid), "%pUl", &crash_data->guid);
 	else
-		scnprintf(uuid, sizeof(uuid), "n/a");
+		scnprintf(guid, sizeof(guid), "n/a");
 
-	ath10k_err(ar, "firmware crashed! (uuid %s)\n", uuid);
+	ath10k_err(ar, "firmware crashed! (guid %s)\n", guid);
 	ath10k_print_driver_info(ar);
 	ath10k_pci_dump_registers(ar, crash_data);
 	ath10k_ce_dump_registers(ar, crash_data);
@@ -1593,6 +1597,8 @@ void ath10k_pci_irq_msi_fw_mask(struct ath10k *ar)
 		 *  to mask irq/MSI.
 		 */
 		break;
+	case ATH10K_HW_WCN3990:
+		break;
 	}
 }
 
@@ -1619,6 +1625,8 @@ static void ath10k_pci_irq_msi_fw_unmask(struct ath10k *ar)
 		 *  to unmask irq/MSI.
 		 */
 		break;
+	case ATH10K_HW_WCN3990:
+		break;
 	}
 }
 
@@ -2000,9 +2008,9 @@ static int ath10k_pci_get_num_banks(struct ath10k *ar)
 
 static int ath10k_bus_get_num_banks(struct ath10k *ar)
 {
-	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 
-	return ar_pci->bus_ops->get_num_banks(ar);
+	return ce->bus_ops->get_num_banks(ar);
 }
 
 int ath10k_pci_init_config(struct ath10k *ar)
@@ -2173,11 +2181,12 @@ int ath10k_pci_alloc_pipes(struct ath10k *ar)
 {
 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
 	struct ath10k_pci_pipe *pipe;
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 	int i, ret;
 
 	for (i = 0; i < CE_COUNT; i++) {
 		pipe = &ar_pci->pipe_info[i];
-		pipe->ce_hdl = &ar_pci->ce_states[i];
+		pipe->ce_hdl = &ce->ce_states[i];
 		pipe->pipe_num = i;
 		pipe->hif_ce_state = ar;
 
@@ -2825,7 +2834,7 @@ static int ath10k_pci_napi_poll(struct napi_struct *ctx, int budget)
 		 * interrupts safer to check for pending interrupts for
 		 * immediate servicing.
 		 */
-		if (CE_INTERRUPT_SUMMARY(ar)) {
+		if (ath10k_ce_interrupt_summary(ar)) {
 			napi_reschedule(ctx);
 			goto out;
 		}
@@ -3142,9 +3151,10 @@ static bool ath10k_pci_chip_is_supported(u32 dev_id, u32 chip_id)
 int ath10k_pci_setup_resource(struct ath10k *ar)
 {
 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 	int ret;
 
-	spin_lock_init(&ar_pci->ce_lock);
+	spin_lock_init(&ce->ce_lock);
 	spin_lock_init(&ar_pci->ps_lock);
 
 	setup_timer(&ar_pci->rx_post_retry, ath10k_pci_rx_replenish_retry,
@@ -3263,10 +3273,11 @@ static int ath10k_pci_probe(struct pci_dev *pdev,
 	ar_pci->ar = ar;
 	ar->dev_id = pci_dev->device;
 	ar_pci->pci_ps = pci_ps;
-	ar_pci->bus_ops = &ath10k_pci_bus_ops;
+	ar_pci->ce.bus_ops = &ath10k_pci_bus_ops;
 	ar_pci->pci_soft_reset = pci_soft_reset;
 	ar_pci->pci_hard_reset = pci_hard_reset;
 	ar_pci->targ_cpu_to_ce_addr = targ_cpu_to_ce_addr;
+	ar->ce_priv = &ar_pci->ce;
 
 	ar->id.vendor = pdev->vendor;
 	ar->id.device = pdev->device;
@@ -3385,11 +3396,53 @@ static void ath10k_pci_remove(struct pci_dev *pdev)
 
 MODULE_DEVICE_TABLE(pci, ath10k_pci_id_table);
 
+#ifdef CONFIG_PM
+
+static int ath10k_pci_pm_suspend(struct device *dev)
+{
+	struct ath10k *ar = dev_get_drvdata(dev);
+	int ret;
+
+	if (test_bit(ATH10K_FW_FEATURE_WOWLAN_SUPPORT,
+		     ar->running_fw->fw_file.fw_features))
+		return 0;
+
+	ret = ath10k_hif_suspend(ar);
+	if (ret)
+		ath10k_warn(ar, "failed to suspend hif: %d\n", ret);
+
+	return ret;
+}
+
+static int ath10k_pci_pm_resume(struct device *dev)
+{
+	struct ath10k *ar = dev_get_drvdata(dev);
+	int ret;
+
+	if (test_bit(ATH10K_FW_FEATURE_WOWLAN_SUPPORT,
+		     ar->running_fw->fw_file.fw_features))
+		return 0;
+
+	ret = ath10k_hif_resume(ar);
+	if (ret)
+		ath10k_warn(ar, "failed to resume hif: %d\n", ret);
+
+	return ret;
+}
+
+static SIMPLE_DEV_PM_OPS(ath10k_pci_pm_ops,
+			 ath10k_pci_pm_suspend,
+			 ath10k_pci_pm_resume);
+#endif
+
 static struct pci_driver ath10k_pci_driver = {
 	.name = "ath10k_pci",
 	.id_table = ath10k_pci_id_table,
 	.probe = ath10k_pci_probe,
 	.remove = ath10k_pci_remove,
+#ifdef CONFIG_PM
+	.driver.pm = &ath10k_pci_pm_ops,
+#endif
 };
 
 static int __init ath10k_pci_init(void)
diff --git a/drivers/net/wireless/ath/ath10k/pci.h b/drivers/net/wireless/ath/ath10k/pci.h
index c1e08ad63940..424ff323b2dc 100644
--- a/drivers/net/wireless/ath/ath10k/pci.h
+++ b/drivers/net/wireless/ath/ath10k/pci.h
@@ -150,12 +150,6 @@ struct ath10k_pci_supp_chip {
 	u32 rev_id;
 };
 
-struct ath10k_bus_ops {
-	u32 (*read32)(struct ath10k *ar, u32 offset);
-	void (*write32)(struct ath10k *ar, u32 offset, u32 value);
-	int (*get_num_banks)(struct ath10k *ar);
-};
-
 enum ath10k_pci_irq_mode {
 	ATH10K_PCI_IRQ_AUTO = 0,
 	ATH10K_PCI_IRQ_LEGACY = 1,
@@ -177,11 +171,7 @@ struct ath10k_pci {
 	/* Copy Engine used for Diagnostic Accesses */
 	struct ath10k_ce_pipe *ce_diag;
 
-	/* FIXME: document what this really protects */
-	spinlock_t ce_lock;
-
-	/* Map CE id to ce_state */
-	struct ath10k_ce_pipe ce_states[CE_COUNT_MAX];
+	struct ath10k_ce ce;
 	struct timer_list rx_post_retry;
 
 	/* Due to HW quirks it is recommended to disable ASPM during device
@@ -225,8 +215,6 @@ struct ath10k_pci {
 	 */
 	bool pci_ps;
 
-	const struct ath10k_bus_ops *bus_ops;
-
 	/* Chip specific pci reset routine used to do a safe reset */
 	int (*pci_soft_reset)(struct ath10k *ar);
 
diff --git a/drivers/net/wireless/ath/ath10k/sdio.c b/drivers/net/wireless/ath/ath10k/sdio.c
index 859ed870bd97..03a69e5b1116 100644
--- a/drivers/net/wireless/ath/ath10k/sdio.c
+++ b/drivers/net/wireless/ath/ath10k/sdio.c
@@ -683,7 +683,7 @@ static int ath10k_sdio_mbox_rxmsg_pending_handler(struct ath10k *ar,
 	lookaheads[0] = msg_lookahead;
 
 	timeout = jiffies + SDIO_MBOX_PROCESSING_TIMEOUT_HZ;
-	while (time_before(jiffies, timeout)) {
+	do {
 		/* Try to allocate as many HTC RX packets indicated by
 		 * n_lookaheads.
 		 */
@@ -719,7 +719,7 @@ static int ath10k_sdio_mbox_rxmsg_pending_handler(struct ath10k *ar,
 		 * performance in high throughput situations.
 		 */
 		*done = false;
-	}
+	} while (time_before(jiffies, timeout));
 
 	if (ret && (ret != -ECANCELED))
 		ath10k_warn(ar, "failed to get pending recv messages: %d\n",
@@ -1336,16 +1336,14 @@ static void ath10k_sdio_irq_handler(struct sdio_func *func)
 	sdio_release_host(ar_sdio->func);
 
 	timeout = jiffies + ATH10K_SDIO_HIF_COMMUNICATION_TIMEOUT_HZ;
-	while (time_before(jiffies, timeout) && !done) {
+	do {
 		ret = ath10k_sdio_mbox_proc_pending_irqs(ar, &done);
 		if (ret)
 			break;
-	}
+	} while (time_before(jiffies, timeout) && !done);
 
 	sdio_claim_host(ar_sdio->func);
 
-	wake_up(&ar_sdio->irq_wq);
-
 	if (ret && ret != -ECANCELED)
 		ath10k_warn(ar, "failed to process pending SDIO interrupts: %d\n",
 			    ret);
@@ -2000,8 +1998,6 @@ static int ath10k_sdio_probe(struct sdio_func *func,
 		goto err_free_bmi_buf;
 	}
 
-	init_waitqueue_head(&ar_sdio->irq_wq);
-
 	for (i = 0; i < ATH10K_SDIO_BUS_REQUEST_MAX_NUM; i++)
 		ath10k_sdio_free_bus_req(ar, &ar_sdio->bus_req[i]);
 
diff --git a/drivers/net/wireless/ath/ath10k/sdio.h b/drivers/net/wireless/ath/ath10k/sdio.h
index 3f61c67c601d..4ff7b545293b 100644
--- a/drivers/net/wireless/ath/ath10k/sdio.h
+++ b/drivers/net/wireless/ath/ath10k/sdio.h
@@ -210,8 +210,6 @@ struct ath10k_sdio {
 	/* temporary buffer for BMI requests */
 	u8 *bmi_buf;
 
-	wait_queue_head_t irq_wq;
-
 	bool is_disabled;
 
 	struct workqueue_struct *workqueue;
diff --git a/drivers/net/wireless/ath/ath10k/usb.c b/drivers/net/wireless/ath/ath10k/usb.c
new file mode 100644
index 000000000000..d4803ff5a78a
--- /dev/null
+++ b/drivers/net/wireless/ath/ath10k/usb.c
@@ -0,0 +1,1106 @@
+/*
+ * Copyright (c) 2007-2011 Atheros Communications Inc.
+ * Copyright (c) 2011-2012,2017 Qualcomm Atheros, Inc.
+ * Copyright (c) 2016-2017 Erik Stromdahl <erik.stromdahl@gmail.com>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <linux/module.h>
+#include <linux/usb.h>
+
+#include "debug.h"
+#include "core.h"
+#include "bmi.h"
+#include "hif.h"
+#include "htc.h"
+#include "usb.h"
+
+static void ath10k_usb_post_recv_transfers(struct ath10k *ar,
+					   struct ath10k_usb_pipe *recv_pipe);
+
+/* inlined helper functions */
+
+static inline enum ath10k_htc_ep_id
+eid_from_htc_hdr(struct ath10k_htc_hdr *htc_hdr)
+{
+	return (enum ath10k_htc_ep_id)htc_hdr->eid;
+}
+
+static inline bool is_trailer_only_msg(struct ath10k_htc_hdr *htc_hdr)
+{
+	return __le16_to_cpu(htc_hdr->len) == htc_hdr->trailer_len;
+}
+
+/* pipe/urb operations */
+static struct ath10k_urb_context *
+ath10k_usb_alloc_urb_from_pipe(struct ath10k_usb_pipe *pipe)
+{
+	struct ath10k_urb_context *urb_context = NULL;
+	unsigned long flags;
+
+	spin_lock_irqsave(&pipe->ar_usb->cs_lock, flags);
+	if (!list_empty(&pipe->urb_list_head)) {
+		urb_context = list_first_entry(&pipe->urb_list_head,
+					       struct ath10k_urb_context, link);
+		list_del(&urb_context->link);
+		pipe->urb_cnt--;
+	}
+	spin_unlock_irqrestore(&pipe->ar_usb->cs_lock, flags);
+
+	return urb_context;
+}
+
+static void ath10k_usb_free_urb_to_pipe(struct ath10k_usb_pipe *pipe,
+					struct ath10k_urb_context *urb_context)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&pipe->ar_usb->cs_lock, flags);
+
+	pipe->urb_cnt++;
+	list_add(&urb_context->link, &pipe->urb_list_head);
+
+	spin_unlock_irqrestore(&pipe->ar_usb->cs_lock, flags);
+}
+
+static void ath10k_usb_cleanup_recv_urb(struct ath10k_urb_context *urb_context)
+{
+	dev_kfree_skb(urb_context->skb);
+	urb_context->skb = NULL;
+
+	ath10k_usb_free_urb_to_pipe(urb_context->pipe, urb_context);
+}
+
+static void ath10k_usb_free_pipe_resources(struct ath10k *ar,
+					   struct ath10k_usb_pipe *pipe)
+{
+	struct ath10k_urb_context *urb_context;
+
+	if (!pipe->ar_usb) {
+		/* nothing allocated for this pipe */
+		return;
+	}
+
+	ath10k_dbg(ar, ATH10K_DBG_USB,
+		   "usb free resources lpipe %d hpipe 0x%x urbs %d avail %d\n",
+		   pipe->logical_pipe_num, pipe->usb_pipe_handle,
+		   pipe->urb_alloc, pipe->urb_cnt);
+
+	if (pipe->urb_alloc != pipe->urb_cnt) {
+		ath10k_dbg(ar, ATH10K_DBG_USB,
+			   "usb urb leak lpipe %d hpipe 0x%x urbs %d avail %d\n",
+			   pipe->logical_pipe_num, pipe->usb_pipe_handle,
+			   pipe->urb_alloc, pipe->urb_cnt);
+	}
+
+	for (;;) {
+		urb_context = ath10k_usb_alloc_urb_from_pipe(pipe);
+
+		if (!urb_context)
+			break;
+
+		kfree(urb_context);
+	}
+}
+
+static void ath10k_usb_cleanup_pipe_resources(struct ath10k *ar)
+{
+	struct ath10k_usb *ar_usb = ath10k_usb_priv(ar);
+	int i;
+
+	for (i = 0; i < ATH10K_USB_PIPE_MAX; i++)
+		ath10k_usb_free_pipe_resources(ar, &ar_usb->pipes[i]);
+}
+
+/* hif usb rx/tx completion functions */
+
+static void ath10k_usb_recv_complete(struct urb *urb)
+{
+	struct ath10k_urb_context *urb_context = urb->context;
+	struct ath10k_usb_pipe *pipe = urb_context->pipe;
+	struct ath10k *ar = pipe->ar_usb->ar;
+	struct sk_buff *skb;
+	int status = 0;
+
+	ath10k_dbg(ar, ATH10K_DBG_USB_BULK,
+		   "usb recv pipe %d stat %d len %d urb 0x%pK\n",
+		   pipe->logical_pipe_num, urb->status, urb->actual_length,
+		   urb);
+
+	if (urb->status != 0) {
+		status = -EIO;
+		switch (urb->status) {
+		case -ECONNRESET:
+		case -ENOENT:
+		case -ESHUTDOWN:
+			/* no need to spew these errors when device
+			 * removed or urb killed due to driver shutdown
+			 */
+			status = -ECANCELED;
+			break;
+		default:
+			ath10k_dbg(ar, ATH10K_DBG_USB_BULK,
+				   "usb recv pipe %d ep 0x%2.2x failed: %d\n",
+				   pipe->logical_pipe_num,
+				   pipe->ep_address, urb->status);
+			break;
+		}
+		goto cleanup_recv_urb;
+	}
+
+	if (urb->actual_length == 0)
+		goto cleanup_recv_urb;
+
+	skb = urb_context->skb;
+
+	/* we are going to pass it up */
+	urb_context->skb = NULL;
+	skb_put(skb, urb->actual_length);
+
+	/* note: queue implements a lock */
+	skb_queue_tail(&pipe->io_comp_queue, skb);
+	schedule_work(&pipe->io_complete_work);
+
+cleanup_recv_urb:
+	ath10k_usb_cleanup_recv_urb(urb_context);
+
+	if (status == 0 &&
+	    pipe->urb_cnt >= pipe->urb_cnt_thresh) {
+		/* our free urbs are piling up, post more transfers */
+		ath10k_usb_post_recv_transfers(ar, pipe);
+	}
+}
+
+static void ath10k_usb_transmit_complete(struct urb *urb)
+{
+	struct ath10k_urb_context *urb_context = urb->context;
+	struct ath10k_usb_pipe *pipe = urb_context->pipe;
+	struct ath10k *ar = pipe->ar_usb->ar;
+	struct sk_buff *skb;
+
+	if (urb->status != 0) {
+		ath10k_dbg(ar, ATH10K_DBG_USB_BULK,
+			   "pipe: %d, failed:%d\n",
+			   pipe->logical_pipe_num, urb->status);
+	}
+
+	skb = urb_context->skb;
+	urb_context->skb = NULL;
+	ath10k_usb_free_urb_to_pipe(urb_context->pipe, urb_context);
+
+	/* note: queue implements a lock */
+	skb_queue_tail(&pipe->io_comp_queue, skb);
+	schedule_work(&pipe->io_complete_work);
+}
+
+/* pipe operations */
+static void ath10k_usb_post_recv_transfers(struct ath10k *ar,
+					   struct ath10k_usb_pipe *recv_pipe)
+{
+	struct ath10k_urb_context *urb_context;
+	struct urb *urb;
+	int usb_status;
+
+	for (;;) {
+		urb_context = ath10k_usb_alloc_urb_from_pipe(recv_pipe);
+		if (!urb_context)
+			break;
+
+		urb_context->skb = dev_alloc_skb(ATH10K_USB_RX_BUFFER_SIZE);
+		if (!urb_context->skb)
+			goto err;
+
+		urb = usb_alloc_urb(0, GFP_ATOMIC);
+		if (!urb)
+			goto err;
+
+		usb_fill_bulk_urb(urb,
+				  recv_pipe->ar_usb->udev,
+				  recv_pipe->usb_pipe_handle,
+				  urb_context->skb->data,
+				  ATH10K_USB_RX_BUFFER_SIZE,
+				  ath10k_usb_recv_complete, urb_context);
+
+		ath10k_dbg(ar, ATH10K_DBG_USB_BULK,
+			   "usb bulk recv submit %d 0x%x ep 0x%2.2x len %d buf 0x%pK\n",
+			   recv_pipe->logical_pipe_num,
+			   recv_pipe->usb_pipe_handle, recv_pipe->ep_address,
+			   ATH10K_USB_RX_BUFFER_SIZE, urb_context->skb);
+
+		usb_anchor_urb(urb, &recv_pipe->urb_submitted);
+		usb_status = usb_submit_urb(urb, GFP_ATOMIC);
+
+		if (usb_status) {
+			ath10k_dbg(ar, ATH10K_DBG_USB_BULK,
+				   "usb bulk recv failed: %d\n",
+				   usb_status);
+			usb_unanchor_urb(urb);
+			usb_free_urb(urb);
+			goto err;
+		}
+		usb_free_urb(urb);
+	}
+
+	return;
+
+err:
+	ath10k_usb_cleanup_recv_urb(urb_context);
+}
+
+static void ath10k_usb_flush_all(struct ath10k *ar)
+{
+	struct ath10k_usb *ar_usb = ath10k_usb_priv(ar);
+	int i;
+
+	for (i = 0; i < ATH10K_USB_PIPE_MAX; i++) {
+		if (ar_usb->pipes[i].ar_usb) {
+			usb_kill_anchored_urbs(&ar_usb->pipes[i].urb_submitted);
+			cancel_work_sync(&ar_usb->pipes[i].io_complete_work);
+		}
+	}
+}
+
+static void ath10k_usb_start_recv_pipes(struct ath10k *ar)
+{
+	struct ath10k_usb *ar_usb = ath10k_usb_priv(ar);
+
+	ar_usb->pipes[ATH10K_USB_PIPE_RX_DATA].urb_cnt_thresh = 1;
+
+	ath10k_usb_post_recv_transfers(ar,
+				       &ar_usb->pipes[ATH10K_USB_PIPE_RX_DATA]);
+}
+
+static void ath10k_usb_tx_complete(struct ath10k *ar, struct sk_buff *skb)
+{
+	struct ath10k_htc_hdr *htc_hdr;
+	struct ath10k_htc_ep *ep;
+
+	htc_hdr = (struct ath10k_htc_hdr *)skb->data;
+	ep = &ar->htc.endpoint[htc_hdr->eid];
+	ath10k_htc_notify_tx_completion(ep, skb);
+	/* The TX complete handler now owns the skb... */
+}
+
+static void ath10k_usb_rx_complete(struct ath10k *ar, struct sk_buff *skb)
+{
+	struct ath10k_htc *htc = &ar->htc;
+	struct ath10k_htc_hdr *htc_hdr;
+	enum ath10k_htc_ep_id eid;
+	struct ath10k_htc_ep *ep;
+	u16 payload_len;
+	u8 *trailer;
+	int ret;
+
+	htc_hdr = (struct ath10k_htc_hdr *)skb->data;
+	eid = eid_from_htc_hdr(htc_hdr);
+	ep = &ar->htc.endpoint[eid];
+
+	if (ep->service_id == 0) {
+		ath10k_warn(ar, "ep %d is not connected\n", eid);
+		goto out_free_skb;
+	}
+
+	payload_len = le16_to_cpu(htc_hdr->len);
+	if (!payload_len) {
+		ath10k_warn(ar, "zero length frame received, firmware crashed?\n");
+		goto out_free_skb;
+	}
+
+	if (payload_len < htc_hdr->trailer_len) {
+		ath10k_warn(ar, "malformed frame received, firmware crashed?\n");
+		goto out_free_skb;
+	}
+
+	if (htc_hdr->flags & ATH10K_HTC_FLAG_TRAILER_PRESENT) {
+		trailer = skb->data + sizeof(*htc_hdr) + payload_len -
+			  htc_hdr->trailer_len;
+
+		ret = ath10k_htc_process_trailer(htc,
+						 trailer,
+						 htc_hdr->trailer_len,
+						 eid,
+						 NULL,
+						 NULL);
+		if (ret)
+			goto out_free_skb;
+
+		if (is_trailer_only_msg(htc_hdr))
+			goto out_free_skb;
+
+		/* strip off the trailer from the skb since it should not
+		 * be passed on to upper layers
+		 */
+		skb_trim(skb, skb->len - htc_hdr->trailer_len);
+	}
+
+	skb_pull(skb, sizeof(*htc_hdr));
+	ep->ep_ops.ep_rx_complete(ar, skb);
+	/* The RX complete handler now owns the skb... */
+
+	return;
+
+out_free_skb:
+	dev_kfree_skb(skb);
+}
+
+static void ath10k_usb_io_comp_work(struct work_struct *work)
+{
+	struct ath10k_usb_pipe *pipe = container_of(work,
+						    struct ath10k_usb_pipe,
+						    io_complete_work);
+	struct ath10k *ar = pipe->ar_usb->ar;
+	struct sk_buff *skb;
+
+	while ((skb = skb_dequeue(&pipe->io_comp_queue))) {
+		if (pipe->flags & ATH10K_USB_PIPE_FLAG_TX)
+			ath10k_usb_tx_complete(ar, skb);
+		else
+			ath10k_usb_rx_complete(ar, skb);
+	}
+}
+
+#define ATH10K_USB_MAX_DIAG_CMD (sizeof(struct ath10k_usb_ctrl_diag_cmd_write))
+#define ATH10K_USB_MAX_DIAG_RESP (sizeof(struct ath10k_usb_ctrl_diag_resp_read))
+
+static void ath10k_usb_destroy(struct ath10k *ar)
+{
+	struct ath10k_usb *ar_usb = ath10k_usb_priv(ar);
+
+	ath10k_usb_flush_all(ar);
+	ath10k_usb_cleanup_pipe_resources(ar);
+	usb_set_intfdata(ar_usb->interface, NULL);
+
+	kfree(ar_usb->diag_cmd_buffer);
+	kfree(ar_usb->diag_resp_buffer);
+}
+
+static int ath10k_usb_hif_start(struct ath10k *ar)
+{
+	int i;
+	struct ath10k_usb *ar_usb = ath10k_usb_priv(ar);
+
+	ath10k_usb_start_recv_pipes(ar);
+
+	/* set the TX resource avail threshold for each TX pipe */
+	for (i = ATH10K_USB_PIPE_TX_CTRL;
+	     i <= ATH10K_USB_PIPE_TX_DATA_HP; i++) {
+		ar_usb->pipes[i].urb_cnt_thresh =
+		    ar_usb->pipes[i].urb_alloc / 2;
+	}
+
+	return 0;
+}
+
+static int ath10k_usb_hif_tx_sg(struct ath10k *ar, u8 pipe_id,
+				struct ath10k_hif_sg_item *items, int n_items)
+{
+	struct ath10k_usb *ar_usb = ath10k_usb_priv(ar);
+	struct ath10k_usb_pipe *pipe = &ar_usb->pipes[pipe_id];
+	struct ath10k_urb_context *urb_context;
+	struct sk_buff *skb;
+	struct urb *urb;
+	int ret, i;
+
+	for (i = 0; i < n_items; i++) {
+		urb_context = ath10k_usb_alloc_urb_from_pipe(pipe);
+		if (!urb_context) {
+			ret = -ENOMEM;
+			goto err;
+		}
+
+		skb = items[i].transfer_context;
+		urb_context->skb = skb;
+
+		urb = usb_alloc_urb(0, GFP_ATOMIC);
+		if (!urb) {
+			ret = -ENOMEM;
+			goto err_free_urb_to_pipe;
+		}
+
+		usb_fill_bulk_urb(urb,
+				  ar_usb->udev,
+				  pipe->usb_pipe_handle,
+				  skb->data,
+				  skb->len,
+				  ath10k_usb_transmit_complete, urb_context);
+
+		if (!(skb->len % pipe->max_packet_size)) {
+			/* hit a max packet boundary on this pipe */
+			urb->transfer_flags |= URB_ZERO_PACKET;
+		}
+
+		usb_anchor_urb(urb, &pipe->urb_submitted);
+		ret = usb_submit_urb(urb, GFP_ATOMIC);
+		if (ret) {
+			ath10k_dbg(ar, ATH10K_DBG_USB_BULK,
+				   "usb bulk transmit failed: %d\n", ret);
+			usb_unanchor_urb(urb);
+			ret = -EINVAL;
+			goto err_free_urb_to_pipe;
+		}
+
+		usb_free_urb(urb);
+	}
+
+	return 0;
+
+err_free_urb_to_pipe:
+	ath10k_usb_free_urb_to_pipe(urb_context->pipe, urb_context);
+err:
+	return ret;
+}
+
+static void ath10k_usb_hif_stop(struct ath10k *ar)
+{
+	ath10k_usb_flush_all(ar);
+}
+
+static u16 ath10k_usb_hif_get_free_queue_number(struct ath10k *ar, u8 pipe_id)
+{
+	struct ath10k_usb *ar_usb = ath10k_usb_priv(ar);
+
+	return ar_usb->pipes[pipe_id].urb_cnt;
+}
+
+static int ath10k_usb_submit_ctrl_out(struct ath10k *ar,
+				      u8 req, u16 value, u16 index, void *data,
+				      u32 size)
+{
+	struct ath10k_usb *ar_usb = ath10k_usb_priv(ar);
+	u8 *buf = NULL;
+	int ret;
+
+	if (size > 0) {
+		buf = kmemdup(data, size, GFP_KERNEL);
+		if (!buf)
+			return -ENOMEM;
+	}
+
+	/* note: if successful returns number of bytes transferred */
+	ret = usb_control_msg(ar_usb->udev,
+			      usb_sndctrlpipe(ar_usb->udev, 0),
+			      req,
+			      USB_DIR_OUT | USB_TYPE_VENDOR |
+			      USB_RECIP_DEVICE, value, index, buf,
+			      size, 1000);
+
+	if (ret < 0) {
+		ath10k_warn(ar, "Failed to submit usb control message: %d\n",
+			    ret);
+		kfree(buf);
+		return ret;
+	}
+
+	kfree(buf);
+
+	return 0;
+}
+
+static int ath10k_usb_submit_ctrl_in(struct ath10k *ar,
+				     u8 req, u16 value, u16 index, void *data,
+				     u32 size)
+{
+	struct ath10k_usb *ar_usb = ath10k_usb_priv(ar);
+	u8 *buf = NULL;
+	int ret;
+
+	if (size > 0) {
+		buf = kmalloc(size, GFP_KERNEL);
+		if (!buf)
+			return -ENOMEM;
+	}
+
+	/* note: if successful returns number of bytes transferred */
+	ret = usb_control_msg(ar_usb->udev,
+			      usb_rcvctrlpipe(ar_usb->udev, 0),
+			      req,
+			      USB_DIR_IN | USB_TYPE_VENDOR |
+			      USB_RECIP_DEVICE, value, index, buf,
+			      size, 2 * HZ);
+
+	if (ret < 0) {
+		ath10k_warn(ar, "Failed to read usb control message: %d\n",
+			    ret);
+		kfree(buf);
+		return ret;
+	}
+
+	memcpy((u8 *)data, buf, size);
+
+	kfree(buf);
+
+	return 0;
+}
+
+static int ath10k_usb_ctrl_msg_exchange(struct ath10k *ar,
+					u8 req_val, u8 *req_buf, u32 req_len,
+					u8 resp_val, u8 *resp_buf,
+					u32 *resp_len)
+{
+	int ret;
+
+	/* send command */
+	ret = ath10k_usb_submit_ctrl_out(ar, req_val, 0, 0,
+					 req_buf, req_len);
+	if (ret)
+		goto err;
+
+	/* get response */
+	if (resp_buf) {
+		ret = ath10k_usb_submit_ctrl_in(ar, resp_val, 0, 0,
+						resp_buf, *resp_len);
+		if (ret)
+			goto err;
+	}
+
+	return 0;
+err:
+	return ret;
+}
+
+static int ath10k_usb_hif_diag_read(struct ath10k *ar, u32 address, void *buf,
+				    size_t buf_len)
+{
+	struct ath10k_usb *ar_usb = ath10k_usb_priv(ar);
+	struct ath10k_usb_ctrl_diag_cmd_read *cmd;
+	u32 resp_len;
+	int ret;
+
+	if (buf_len < sizeof(struct ath10k_usb_ctrl_diag_resp_read))
+		return -EINVAL;
+
+	cmd = (struct ath10k_usb_ctrl_diag_cmd_read *)ar_usb->diag_cmd_buffer;
+	memset(cmd, 0, sizeof(*cmd));
+	cmd->cmd = ATH10K_USB_CTRL_DIAG_CC_READ;
+	cmd->address = cpu_to_le32(address);
+	resp_len = sizeof(struct ath10k_usb_ctrl_diag_resp_read);
+
+	ret = ath10k_usb_ctrl_msg_exchange(ar,
+					   ATH10K_USB_CONTROL_REQ_DIAG_CMD,
+					   (u8 *)cmd,
+					   sizeof(*cmd),
+					   ATH10K_USB_CONTROL_REQ_DIAG_RESP,
+					   ar_usb->diag_resp_buffer, &resp_len);
+	if (ret)
+		return ret;
+
+	if (resp_len != sizeof(struct ath10k_usb_ctrl_diag_resp_read))
+		return -EMSGSIZE;
+
+	memcpy(buf, ar_usb->diag_resp_buffer,
+	       sizeof(struct ath10k_usb_ctrl_diag_resp_read));
+
+	return 0;
+}
+
+static int ath10k_usb_hif_diag_write(struct ath10k *ar, u32 address,
+				     const void *data, int nbytes)
+{
+	struct ath10k_usb *ar_usb = ath10k_usb_priv(ar);
+	struct ath10k_usb_ctrl_diag_cmd_write *cmd;
+	int ret;
+
+	if (nbytes != sizeof(cmd->value))
+		return -EINVAL;
+
+	cmd = (struct ath10k_usb_ctrl_diag_cmd_write *)ar_usb->diag_cmd_buffer;
+	memset(cmd, 0, sizeof(*cmd));
+	cmd->cmd = cpu_to_le32(ATH10K_USB_CTRL_DIAG_CC_WRITE);
+	cmd->address = cpu_to_le32(address);
+	memcpy(&cmd->value, data, nbytes);
+
+	ret = ath10k_usb_ctrl_msg_exchange(ar,
+					   ATH10K_USB_CONTROL_REQ_DIAG_CMD,
+					   (u8 *)cmd,
+					   sizeof(*cmd),
+					   0, NULL, NULL);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int ath10k_usb_bmi_exchange_msg(struct ath10k *ar,
+				       void *req, u32 req_len,
+				       void *resp, u32 *resp_len)
+{
+	int ret;
+
+	if (req) {
+		ret = ath10k_usb_submit_ctrl_out(ar,
+						 ATH10K_USB_CONTROL_REQ_SEND_BMI_CMD,
+						 0, 0, req, req_len);
+		if (ret) {
+			ath10k_warn(ar,
+				    "unable to send the bmi data to the device: %d\n",
+				    ret);
+			return ret;
+		}
+	}
+
+	if (resp) {
+		ret = ath10k_usb_submit_ctrl_in(ar,
+						ATH10K_USB_CONTROL_REQ_RECV_BMI_RESP,
+						0, 0, resp, *resp_len);
+		if (ret) {
+			ath10k_warn(ar,
+				    "Unable to read the bmi data from the device: %d\n",
+				    ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static void ath10k_usb_hif_get_default_pipe(struct ath10k *ar,
+					    u8 *ul_pipe, u8 *dl_pipe)
+{
+	*ul_pipe = ATH10K_USB_PIPE_TX_CTRL;
+	*dl_pipe = ATH10K_USB_PIPE_RX_CTRL;
+}
+
+static int ath10k_usb_hif_map_service_to_pipe(struct ath10k *ar, u16 svc_id,
+					      u8 *ul_pipe, u8 *dl_pipe)
+{
+	switch (svc_id) {
+	case ATH10K_HTC_SVC_ID_RSVD_CTRL:
+	case ATH10K_HTC_SVC_ID_WMI_CONTROL:
+		*ul_pipe = ATH10K_USB_PIPE_TX_CTRL;
+		/* due to large control packets, shift to data pipe */
+		*dl_pipe = ATH10K_USB_PIPE_RX_DATA;
+		break;
+	case ATH10K_HTC_SVC_ID_HTT_DATA_MSG:
+		*ul_pipe = ATH10K_USB_PIPE_TX_DATA_LP;
+		/* Disable rxdata2 directly, it will be enabled
+		 * if FW enable rxdata2
+		 */
+		*dl_pipe = ATH10K_USB_PIPE_RX_DATA;
+		break;
+	default:
+		return -EPERM;
+	}
+
+	return 0;
+}
+
+/* This op is currently only used by htc_wait_target if the HTC ready
+ * message times out. It is not applicable for USB since there is nothing
+ * we can do if the HTC ready message does not arrive in time.
+ * TODO: Make this op non mandatory by introducing a NULL check in the
+ * hif op wrapper.
+ */
+static void ath10k_usb_hif_send_complete_check(struct ath10k *ar,
+					       u8 pipe, int force)
+{
+}
+
+static int ath10k_usb_hif_power_up(struct ath10k *ar)
+{
+	return 0;
+}
+
+static void ath10k_usb_hif_power_down(struct ath10k *ar)
+{
+	ath10k_usb_flush_all(ar);
+}
+
+#ifdef CONFIG_PM
+
+static int ath10k_usb_hif_suspend(struct ath10k *ar)
+{
+	return -EOPNOTSUPP;
+}
+
+static int ath10k_usb_hif_resume(struct ath10k *ar)
+{
+	return -EOPNOTSUPP;
+}
+#endif
+
+static const struct ath10k_hif_ops ath10k_usb_hif_ops = {
+	.tx_sg			= ath10k_usb_hif_tx_sg,
+	.diag_read		= ath10k_usb_hif_diag_read,
+	.diag_write		= ath10k_usb_hif_diag_write,
+	.exchange_bmi_msg	= ath10k_usb_bmi_exchange_msg,
+	.start			= ath10k_usb_hif_start,
+	.stop			= ath10k_usb_hif_stop,
+	.map_service_to_pipe	= ath10k_usb_hif_map_service_to_pipe,
+	.get_default_pipe	= ath10k_usb_hif_get_default_pipe,
+	.send_complete_check	= ath10k_usb_hif_send_complete_check,
+	.get_free_queue_number	= ath10k_usb_hif_get_free_queue_number,
+	.power_up		= ath10k_usb_hif_power_up,
+	.power_down		= ath10k_usb_hif_power_down,
+#ifdef CONFIG_PM
+	.suspend		= ath10k_usb_hif_suspend,
+	.resume			= ath10k_usb_hif_resume,
+#endif
+};
+
+static u8 ath10k_usb_get_logical_pipe_num(u8 ep_address, int *urb_count)
+{
+	u8 pipe_num = ATH10K_USB_PIPE_INVALID;
+
+	switch (ep_address) {
+	case ATH10K_USB_EP_ADDR_APP_CTRL_IN:
+		pipe_num = ATH10K_USB_PIPE_RX_CTRL;
+		*urb_count = RX_URB_COUNT;
+		break;
+	case ATH10K_USB_EP_ADDR_APP_DATA_IN:
+		pipe_num = ATH10K_USB_PIPE_RX_DATA;
+		*urb_count = RX_URB_COUNT;
+		break;
+	case ATH10K_USB_EP_ADDR_APP_INT_IN:
+		pipe_num = ATH10K_USB_PIPE_RX_INT;
+		*urb_count = RX_URB_COUNT;
+		break;
+	case ATH10K_USB_EP_ADDR_APP_DATA2_IN:
+		pipe_num = ATH10K_USB_PIPE_RX_DATA2;
+		*urb_count = RX_URB_COUNT;
+		break;
+	case ATH10K_USB_EP_ADDR_APP_CTRL_OUT:
+		pipe_num = ATH10K_USB_PIPE_TX_CTRL;
+		*urb_count = TX_URB_COUNT;
+		break;
+	case ATH10K_USB_EP_ADDR_APP_DATA_LP_OUT:
+		pipe_num = ATH10K_USB_PIPE_TX_DATA_LP;
+		*urb_count = TX_URB_COUNT;
+		break;
+	case ATH10K_USB_EP_ADDR_APP_DATA_MP_OUT:
+		pipe_num = ATH10K_USB_PIPE_TX_DATA_MP;
+		*urb_count = TX_URB_COUNT;
+		break;
+	case ATH10K_USB_EP_ADDR_APP_DATA_HP_OUT:
+		pipe_num = ATH10K_USB_PIPE_TX_DATA_HP;
+		*urb_count = TX_URB_COUNT;
+		break;
+	default:
+		/* note: there may be endpoints not currently used */
+		break;
+	}
+
+	return pipe_num;
+}
+
+static int ath10k_usb_alloc_pipe_resources(struct ath10k *ar,
+					   struct ath10k_usb_pipe *pipe,
+					   int urb_cnt)
+{
+	struct ath10k_urb_context *urb_context;
+	int i;
+
+	INIT_LIST_HEAD(&pipe->urb_list_head);
+	init_usb_anchor(&pipe->urb_submitted);
+
+	for (i = 0; i < urb_cnt; i++) {
+		urb_context = kzalloc(sizeof(*urb_context), GFP_KERNEL);
+		if (!urb_context)
+			return -ENOMEM;
+
+		urb_context->pipe = pipe;
+
+		/* we are only allocate the urb contexts here, the actual URB
+		 * is allocated from the kernel as needed to do a transaction
+		 */
+		pipe->urb_alloc++;
+		ath10k_usb_free_urb_to_pipe(pipe, urb_context);
+	}
+
+	ath10k_dbg(ar, ATH10K_DBG_USB,
+		   "usb alloc resources lpipe %d hpipe 0x%x urbs %d\n",
+		   pipe->logical_pipe_num, pipe->usb_pipe_handle,
+		   pipe->urb_alloc);
+
+	return 0;
+}
+
+static int ath10k_usb_setup_pipe_resources(struct ath10k *ar,
+					   struct usb_interface *interface)
+{
+	struct ath10k_usb *ar_usb = ath10k_usb_priv(ar);
+	struct usb_host_interface *iface_desc = interface->cur_altsetting;
+	struct usb_endpoint_descriptor *endpoint;
+	struct ath10k_usb_pipe *pipe;
+	int ret, i, urbcount;
+	u8 pipe_num;
+
+	ath10k_dbg(ar, ATH10K_DBG_USB, "usb setting up pipes using interface\n");
+
+	/* walk decriptors and setup pipes */
+	for (i = 0; i < iface_desc->desc.bNumEndpoints; ++i) {
+		endpoint = &iface_desc->endpoint[i].desc;
+
+		if (ATH10K_USB_IS_BULK_EP(endpoint->bmAttributes)) {
+			ath10k_dbg(ar, ATH10K_DBG_USB,
+				   "usb %s bulk ep 0x%2.2x maxpktsz %d\n",
+				   ATH10K_USB_IS_DIR_IN
+				   (endpoint->bEndpointAddress) ?
+				   "rx" : "tx", endpoint->bEndpointAddress,
+				   le16_to_cpu(endpoint->wMaxPacketSize));
+		} else if (ATH10K_USB_IS_INT_EP(endpoint->bmAttributes)) {
+			ath10k_dbg(ar, ATH10K_DBG_USB,
+				   "usb %s int ep 0x%2.2x maxpktsz %d interval %d\n",
+				   ATH10K_USB_IS_DIR_IN
+				   (endpoint->bEndpointAddress) ?
+				   "rx" : "tx", endpoint->bEndpointAddress,
+				   le16_to_cpu(endpoint->wMaxPacketSize),
+				   endpoint->bInterval);
+		} else if (ATH10K_USB_IS_ISOC_EP(endpoint->bmAttributes)) {
+			/* TODO for ISO */
+			ath10k_dbg(ar, ATH10K_DBG_USB,
+				   "usb %s isoc ep 0x%2.2x maxpktsz %d interval %d\n",
+				   ATH10K_USB_IS_DIR_IN
+				   (endpoint->bEndpointAddress) ?
+				   "rx" : "tx", endpoint->bEndpointAddress,
+				   le16_to_cpu(endpoint->wMaxPacketSize),
+				   endpoint->bInterval);
+		}
+		urbcount = 0;
+
+		pipe_num =
+		    ath10k_usb_get_logical_pipe_num(endpoint->bEndpointAddress,
+						    &urbcount);
+		if (pipe_num == ATH10K_USB_PIPE_INVALID)
+			continue;
+
+		pipe = &ar_usb->pipes[pipe_num];
+		if (pipe->ar_usb)
+			/* hmmm..pipe was already setup */
+			continue;
+
+		pipe->ar_usb = ar_usb;
+		pipe->logical_pipe_num = pipe_num;
+		pipe->ep_address = endpoint->bEndpointAddress;
+		pipe->max_packet_size = le16_to_cpu(endpoint->wMaxPacketSize);
+
+		if (ATH10K_USB_IS_BULK_EP(endpoint->bmAttributes)) {
+			if (ATH10K_USB_IS_DIR_IN(pipe->ep_address)) {
+				pipe->usb_pipe_handle =
+				    usb_rcvbulkpipe(ar_usb->udev,
+						    pipe->ep_address);
+			} else {
+				pipe->usb_pipe_handle =
+				    usb_sndbulkpipe(ar_usb->udev,
+						    pipe->ep_address);
+			}
+		} else if (ATH10K_USB_IS_INT_EP(endpoint->bmAttributes)) {
+			if (ATH10K_USB_IS_DIR_IN(pipe->ep_address)) {
+				pipe->usb_pipe_handle =
+				    usb_rcvintpipe(ar_usb->udev,
+						   pipe->ep_address);
+			} else {
+				pipe->usb_pipe_handle =
+				    usb_sndintpipe(ar_usb->udev,
+						   pipe->ep_address);
+			}
+		} else if (ATH10K_USB_IS_ISOC_EP(endpoint->bmAttributes)) {
+			/* TODO for ISO */
+			if (ATH10K_USB_IS_DIR_IN(pipe->ep_address)) {
+				pipe->usb_pipe_handle =
+				    usb_rcvisocpipe(ar_usb->udev,
+						    pipe->ep_address);
+			} else {
+				pipe->usb_pipe_handle =
+				    usb_sndisocpipe(ar_usb->udev,
+						    pipe->ep_address);
+			}
+		}
+
+		pipe->ep_desc = endpoint;
+
+		if (!ATH10K_USB_IS_DIR_IN(pipe->ep_address))
+			pipe->flags |= ATH10K_USB_PIPE_FLAG_TX;
+
+		ret = ath10k_usb_alloc_pipe_resources(ar, pipe, urbcount);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int ath10k_usb_create(struct ath10k *ar,
+			     struct usb_interface *interface)
+{
+	struct ath10k_usb *ar_usb = ath10k_usb_priv(ar);
+	struct usb_device *dev = interface_to_usbdev(interface);
+	struct ath10k_usb_pipe *pipe;
+	int ret, i;
+
+	usb_set_intfdata(interface, ar_usb);
+	spin_lock_init(&ar_usb->cs_lock);
+	ar_usb->udev = dev;
+	ar_usb->interface = interface;
+
+	for (i = 0; i < ATH10K_USB_PIPE_MAX; i++) {
+		pipe = &ar_usb->pipes[i];
+		INIT_WORK(&pipe->io_complete_work,
+			  ath10k_usb_io_comp_work);
+		skb_queue_head_init(&pipe->io_comp_queue);
+	}
+
+	ar_usb->diag_cmd_buffer = kzalloc(ATH10K_USB_MAX_DIAG_CMD, GFP_KERNEL);
+	if (!ar_usb->diag_cmd_buffer) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	ar_usb->diag_resp_buffer = kzalloc(ATH10K_USB_MAX_DIAG_RESP,
+					   GFP_KERNEL);
+	if (!ar_usb->diag_resp_buffer) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	ret = ath10k_usb_setup_pipe_resources(ar, interface);
+	if (ret)
+		goto err;
+
+	return 0;
+
+err:
+	ath10k_usb_destroy(ar);
+	return ret;
+}
+
+/* ath10k usb driver registered functions */
+static int ath10k_usb_probe(struct usb_interface *interface,
+			    const struct usb_device_id *id)
+{
+	struct ath10k *ar;
+	struct ath10k_usb *ar_usb;
+	struct usb_device *dev = interface_to_usbdev(interface);
+	int ret, vendor_id, product_id;
+	enum ath10k_hw_rev hw_rev;
+	u32 chip_id;
+
+	/* Assumption: All USB based chipsets (so far) are QCA9377 based.
+	 * If there will be newer chipsets that does not use the hw reg
+	 * setup as defined in qca6174_regs and qca6174_values, this
+	 * assumption is no longer valid and hw_rev must be setup differently
+	 * depending on chipset.
+	 */
+	hw_rev = ATH10K_HW_QCA9377;
+
+	ar = ath10k_core_create(sizeof(*ar_usb), &dev->dev, ATH10K_BUS_USB,
+				hw_rev, &ath10k_usb_hif_ops);
+	if (!ar) {
+		dev_err(&dev->dev, "failed to allocate core\n");
+		return -ENOMEM;
+	}
+
+	usb_get_dev(dev);
+	vendor_id = le16_to_cpu(dev->descriptor.idVendor);
+	product_id = le16_to_cpu(dev->descriptor.idProduct);
+
+	ath10k_dbg(ar, ATH10K_DBG_BOOT,
+		   "usb new func vendor 0x%04x product 0x%04x\n",
+		   vendor_id, product_id);
+
+	ar_usb = ath10k_usb_priv(ar);
+	ret = ath10k_usb_create(ar, interface);
+	ar_usb->ar = ar;
+
+	ar->dev_id = product_id;
+	ar->id.vendor = vendor_id;
+	ar->id.device = product_id;
+
+	/* TODO: don't know yet how to get chip_id with USB */
+	chip_id = 0;
+	ret = ath10k_core_register(ar, chip_id);
+	if (ret) {
+		ath10k_warn(ar, "failed to register driver core: %d\n", ret);
+		goto err;
+	}
+
+	/* TODO: remove this once USB support is fully implemented */
+	ath10k_warn(ar, "WARNING: ath10k USB support is incomplete, don't expect anything to work!\n");
+
+	return 0;
+
+err:
+	ath10k_core_destroy(ar);
+
+	usb_put_dev(dev);
+
+	return ret;
+}
+
+static void ath10k_usb_remove(struct usb_interface *interface)
+{
+	struct ath10k_usb *ar_usb;
+
+	ar_usb = usb_get_intfdata(interface);
+	if (!ar_usb)
+		return;
+
+	ath10k_core_unregister(ar_usb->ar);
+	ath10k_usb_destroy(ar_usb->ar);
+	usb_put_dev(interface_to_usbdev(interface));
+	ath10k_core_destroy(ar_usb->ar);
+}
+
+#ifdef CONFIG_PM
+
+static int ath10k_usb_pm_suspend(struct usb_interface *interface,
+				 pm_message_t message)
+{
+	struct ath10k_usb *ar_usb = usb_get_intfdata(interface);
+
+	ath10k_usb_flush_all(ar_usb->ar);
+	return 0;
+}
+
+static int ath10k_usb_pm_resume(struct usb_interface *interface)
+{
+	struct ath10k_usb *ar_usb = usb_get_intfdata(interface);
+	struct ath10k *ar = ar_usb->ar;
+
+	ath10k_usb_post_recv_transfers(ar,
+				       &ar_usb->pipes[ATH10K_USB_PIPE_RX_DATA]);
+
+	return 0;
+}
+
+#else
+
+#define ath10k_usb_pm_suspend NULL
+#define ath10k_usb_pm_resume NULL
+
+#endif
+
+/* table of devices that work with this driver */
+static struct usb_device_id ath10k_usb_ids[] = {
+	{USB_DEVICE(0x13b1, 0x0042)}, /* Linksys WUSB6100M */
+	{ /* Terminating entry */ },
+};
+
+MODULE_DEVICE_TABLE(usb, ath10k_usb_ids);
+
+static struct usb_driver ath10k_usb_driver = {
+	.name = "ath10k_usb",
+	.probe = ath10k_usb_probe,
+	.suspend = ath10k_usb_pm_suspend,
+	.resume = ath10k_usb_pm_resume,
+	.disconnect = ath10k_usb_remove,
+	.id_table = ath10k_usb_ids,
+	.supports_autosuspend = true,
+	.disable_hub_initiated_lpm = 1,
+};
+
+module_usb_driver(ath10k_usb_driver);
+
+MODULE_AUTHOR("Atheros Communications, Inc.");
+MODULE_DESCRIPTION("Driver support for Qualcomm Atheros 802.11ac WLAN USB devices");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/net/wireless/ath/ath10k/usb.h b/drivers/net/wireless/ath/ath10k/usb.h
new file mode 100644
index 000000000000..f60a3cc7d712
--- /dev/null
+++ b/drivers/net/wireless/ath/ath10k/usb.h
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2004-2011 Atheros Communications Inc.
+ * Copyright (c) 2011-2012 Qualcomm Atheros, Inc.
+ * Copyright (c) 2016-2017 Erik Stromdahl <erik.stromdahl@gmail.com>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef _USB_H_
+#define _USB_H_
+
+/* constants */
+#define TX_URB_COUNT               32
+#define RX_URB_COUNT               32
+#define ATH10K_USB_RX_BUFFER_SIZE  4096
+
+#define ATH10K_USB_PIPE_INVALID ATH10K_USB_PIPE_MAX
+
+/* USB endpoint definitions */
+#define ATH10K_USB_EP_ADDR_APP_CTRL_IN          0x81
+#define ATH10K_USB_EP_ADDR_APP_DATA_IN          0x82
+#define ATH10K_USB_EP_ADDR_APP_DATA2_IN         0x83
+#define ATH10K_USB_EP_ADDR_APP_INT_IN           0x84
+
+#define ATH10K_USB_EP_ADDR_APP_CTRL_OUT         0x01
+#define ATH10K_USB_EP_ADDR_APP_DATA_LP_OUT      0x02
+#define ATH10K_USB_EP_ADDR_APP_DATA_MP_OUT      0x03
+#define ATH10K_USB_EP_ADDR_APP_DATA_HP_OUT      0x04
+
+/* diagnostic command defnitions */
+#define ATH10K_USB_CONTROL_REQ_SEND_BMI_CMD        1
+#define ATH10K_USB_CONTROL_REQ_RECV_BMI_RESP       2
+#define ATH10K_USB_CONTROL_REQ_DIAG_CMD            3
+#define ATH10K_USB_CONTROL_REQ_DIAG_RESP           4
+
+#define ATH10K_USB_CTRL_DIAG_CC_READ               0
+#define ATH10K_USB_CTRL_DIAG_CC_WRITE              1
+
+#define ATH10K_USB_IS_BULK_EP(attr) (((attr) & 3) == 0x02)
+#define ATH10K_USB_IS_INT_EP(attr)  (((attr) & 3) == 0x03)
+#define ATH10K_USB_IS_ISOC_EP(attr) (((attr) & 3) == 0x01)
+#define ATH10K_USB_IS_DIR_IN(addr)  ((addr) & 0x80)
+
+struct ath10k_usb_ctrl_diag_cmd_write {
+	__le32 cmd;
+	__le32 address;
+	__le32 value;
+	__le32 padding;
+} __packed;
+
+struct ath10k_usb_ctrl_diag_cmd_read {
+	__le32 cmd;
+	__le32 address;
+} __packed;
+
+struct ath10k_usb_ctrl_diag_resp_read {
+	u8 value[4];
+} __packed;
+
+/* tx/rx pipes for usb */
+enum ath10k_usb_pipe_id {
+	ATH10K_USB_PIPE_TX_CTRL = 0,
+	ATH10K_USB_PIPE_TX_DATA_LP,
+	ATH10K_USB_PIPE_TX_DATA_MP,
+	ATH10K_USB_PIPE_TX_DATA_HP,
+	ATH10K_USB_PIPE_RX_CTRL,
+	ATH10K_USB_PIPE_RX_DATA,
+	ATH10K_USB_PIPE_RX_DATA2,
+	ATH10K_USB_PIPE_RX_INT,
+	ATH10K_USB_PIPE_MAX
+};
+
+struct ath10k_usb_pipe {
+	struct list_head urb_list_head;
+	struct usb_anchor urb_submitted;
+	u32 urb_alloc;
+	u32 urb_cnt;
+	u32 urb_cnt_thresh;
+	unsigned int usb_pipe_handle;
+	u32 flags;
+	u8 ep_address;
+	u8 logical_pipe_num;
+	struct ath10k_usb *ar_usb;
+	u16 max_packet_size;
+	struct work_struct io_complete_work;
+	struct sk_buff_head io_comp_queue;
+	struct usb_endpoint_descriptor *ep_desc;
+};
+
+#define ATH10K_USB_PIPE_FLAG_TX BIT(0)
+
+/* usb device object */
+struct ath10k_usb {
+	/* protects pipe->urb_list_head and  pipe->urb_cnt */
+	spinlock_t cs_lock;
+
+	struct usb_device *udev;
+	struct usb_interface *interface;
+	struct ath10k_usb_pipe pipes[ATH10K_USB_PIPE_MAX];
+	u8 *diag_cmd_buffer;
+	u8 *diag_resp_buffer;
+	struct ath10k *ar;
+};
+
+/* usb urb object */
+struct ath10k_urb_context {
+	struct list_head link;
+	struct ath10k_usb_pipe *pipe;
+	struct sk_buff *skb;
+	struct ath10k *ar;
+};
+
+static inline struct ath10k_usb *ath10k_usb_priv(struct ath10k *ar)
+{
+	return (struct ath10k_usb *)ar->drv_priv;
+}
+
+#endif
diff --git a/drivers/net/wireless/ath/ath10k/wmi.c b/drivers/net/wireless/ath/ath10k/wmi.c
index 3efb404b83c0..38a97086708b 100644
--- a/drivers/net/wireless/ath/ath10k/wmi.c
+++ b/drivers/net/wireless/ath/ath10k/wmi.c
@@ -651,8 +651,6 @@ static struct wmi_cmd_map wmi_10_4_cmd_map = {
 	.gpio_output_cmdid = WMI_10_4_GPIO_OUTPUT_CMDID,
 	.pdev_get_temperature_cmdid = WMI_10_4_PDEV_GET_TEMPERATURE_CMDID,
 	.vdev_set_wmm_params_cmdid = WMI_CMD_UNSUPPORTED,
-	.tdls_set_state_cmdid = WMI_CMD_UNSUPPORTED,
-	.tdls_peer_update_cmdid = WMI_CMD_UNSUPPORTED,
 	.adaptive_qcs_cmdid = WMI_CMD_UNSUPPORTED,
 	.scan_update_request_cmdid = WMI_10_4_SCAN_UPDATE_REQUEST_CMDID,
 	.vdev_standby_response_cmdid = WMI_10_4_VDEV_STANDBY_RESPONSE_CMDID,
@@ -711,6 +709,33 @@ static struct wmi_cmd_map wmi_10_4_cmd_map = {
 	.pdev_bss_chan_info_request_cmdid =
 			WMI_10_4_PDEV_BSS_CHAN_INFO_REQUEST_CMDID,
 	.ext_resource_cfg_cmdid = WMI_10_4_EXT_RESOURCE_CFG_CMDID,
+	.vdev_set_ie_cmdid = WMI_10_4_VDEV_SET_IE_CMDID,
+	.set_lteu_config_cmdid = WMI_10_4_SET_LTEU_CONFIG_CMDID,
+	.atf_ssid_grouping_request_cmdid =
+			WMI_10_4_ATF_SSID_GROUPING_REQUEST_CMDID,
+	.peer_atf_ext_request_cmdid = WMI_10_4_PEER_ATF_EXT_REQUEST_CMDID,
+	.set_periodic_channel_stats_cfg_cmdid =
+			WMI_10_4_SET_PERIODIC_CHANNEL_STATS_CONFIG,
+	.peer_bwf_request_cmdid = WMI_10_4_PEER_BWF_REQUEST_CMDID,
+	.btcoex_cfg_cmdid = WMI_10_4_BTCOEX_CFG_CMDID,
+	.peer_tx_mu_txmit_count_cmdid = WMI_10_4_PEER_TX_MU_TXMIT_COUNT_CMDID,
+	.peer_tx_mu_txmit_rstcnt_cmdid = WMI_10_4_PEER_TX_MU_TXMIT_RSTCNT_CMDID,
+	.peer_gid_userpos_list_cmdid = WMI_10_4_PEER_GID_USERPOS_LIST_CMDID,
+	.pdev_check_cal_version_cmdid = WMI_10_4_PDEV_CHECK_CAL_VERSION_CMDID,
+	.coex_version_cfg_cmid = WMI_10_4_COEX_VERSION_CFG_CMID,
+	.pdev_get_rx_filter_cmdid = WMI_10_4_PDEV_GET_RX_FILTER_CMDID,
+	.pdev_extended_nss_cfg_cmdid = WMI_10_4_PDEV_EXTENDED_NSS_CFG_CMDID,
+	.vdev_set_scan_nac_rssi_cmdid = WMI_10_4_VDEV_SET_SCAN_NAC_RSSI_CMDID,
+	.prog_gpio_band_select_cmdid = WMI_10_4_PROG_GPIO_BAND_SELECT_CMDID,
+	.config_smart_logging_cmdid = WMI_10_4_CONFIG_SMART_LOGGING_CMDID,
+	.debug_fatal_condition_cmdid = WMI_10_4_DEBUG_FATAL_CONDITION_CMDID,
+	.get_tsf_timer_cmdid = WMI_10_4_GET_TSF_TIMER_CMDID,
+	.pdev_get_tpc_table_cmdid = WMI_10_4_PDEV_GET_TPC_TABLE_CMDID,
+	.vdev_sifs_trigger_time_cmdid = WMI_10_4_VDEV_SIFS_TRIGGER_TIME_CMDID,
+	.pdev_wds_entry_list_cmdid = WMI_10_4_PDEV_WDS_ENTRY_LIST_CMDID,
+	.tdls_set_state_cmdid = WMI_10_4_TDLS_SET_STATE_CMDID,
+	.tdls_peer_update_cmdid = WMI_10_4_TDLS_PEER_UPDATE_CMDID,
+	.tdls_set_offchan_mode_cmdid = WMI_10_4_TDLS_SET_OFFCHAN_MODE_CMDID,
 };
 
 /* MAIN WMI VDEV param map */
@@ -3305,7 +3330,7 @@ static void ath10k_wmi_update_noa(struct ath10k *ar, struct ath10k_vif *arvif,
 	if (arvif->u.ap.noa_data)
 		if (!pskb_expand_head(bcn, 0, arvif->u.ap.noa_len, GFP_ATOMIC))
 			skb_put_data(bcn, arvif->u.ap.noa_data,
-			             arvif->u.ap.noa_len);
+				     arvif->u.ap.noa_len);
 }
 
 static int ath10k_wmi_op_pull_swba_ev(struct ath10k *ar, struct sk_buff *skb,
@@ -6473,6 +6498,7 @@ ath10k_wmi_op_gen_peer_create(struct ath10k *ar, u32 vdev_id,
 	cmd = (struct wmi_peer_create_cmd *)skb->data;
 	cmd->vdev_id = __cpu_to_le32(vdev_id);
 	ether_addr_copy(cmd->peer_macaddr.addr, peer_addr);
+	cmd->peer_type = __cpu_to_le32(peer_type);
 
 	ath10k_dbg(ar, ATH10K_DBG_WMI,
 		   "wmi peer create vdev_id %d peer_addr %pM\n",
@@ -7803,14 +7829,28 @@ ath10k_wmi_10_4_ext_resource_config(struct ath10k *ar,
 {
 	struct wmi_ext_resource_config_10_4_cmd *cmd;
 	struct sk_buff *skb;
+	u32 num_tdls_sleep_sta = 0;
 
 	skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd));
 	if (!skb)
 		return ERR_PTR(-ENOMEM);
 
+	if (test_bit(WMI_SERVICE_TDLS_UAPSD_SLEEP_STA, ar->wmi.svc_map))
+		num_tdls_sleep_sta = TARGET_10_4_NUM_TDLS_SLEEP_STA;
+
 	cmd = (struct wmi_ext_resource_config_10_4_cmd *)skb->data;
 	cmd->host_platform_config = __cpu_to_le32(type);
 	cmd->fw_feature_bitmap = __cpu_to_le32(fw_feature_bitmap);
+	cmd->wlan_gpio_priority = __cpu_to_le32(-1);
+	cmd->coex_version = __cpu_to_le32(WMI_NO_COEX_VERSION_SUPPORT);
+	cmd->coex_gpio_pin1 = __cpu_to_le32(-1);
+	cmd->coex_gpio_pin2 = __cpu_to_le32(-1);
+	cmd->coex_gpio_pin3 = __cpu_to_le32(-1);
+	cmd->num_tdls_vdevs = __cpu_to_le32(TARGET_10_4_NUM_TDLS_VDEVS);
+	cmd->num_tdls_conn_table_entries = __cpu_to_le32(20);
+	cmd->max_tdls_concurrent_sleep_sta = __cpu_to_le32(num_tdls_sleep_sta);
+	cmd->max_tdls_concurrent_buffer_sta =
+			__cpu_to_le32(TARGET_10_4_NUM_TDLS_BUFFER_STA);
 
 	ath10k_dbg(ar, ATH10K_DBG_WMI,
 		   "wmi ext resource config host type %d firmware feature bitmap %08x\n",
@@ -7819,6 +7859,124 @@ ath10k_wmi_10_4_ext_resource_config(struct ath10k *ar,
 }
 
 static struct sk_buff *
+ath10k_wmi_10_4_gen_update_fw_tdls_state(struct ath10k *ar, u32 vdev_id,
+					 enum wmi_tdls_state state)
+{
+	struct wmi_10_4_tdls_set_state_cmd *cmd;
+	struct sk_buff *skb;
+	u32 options = 0;
+
+	skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd));
+	if (!skb)
+		return ERR_PTR(-ENOMEM);
+
+	if (test_bit(WMI_SERVICE_TDLS_EXPLICIT_MODE_ONLY, ar->wmi.svc_map))
+		state = WMI_TDLS_ENABLE_PASSIVE;
+
+	if (test_bit(WMI_SERVICE_TDLS_UAPSD_BUFFER_STA, ar->wmi.svc_map))
+		options |= WMI_TDLS_BUFFER_STA_EN;
+
+	cmd = (struct wmi_10_4_tdls_set_state_cmd *)skb->data;
+	cmd->vdev_id = __cpu_to_le32(vdev_id);
+	cmd->state = __cpu_to_le32(state);
+	cmd->notification_interval_ms = __cpu_to_le32(5000);
+	cmd->tx_discovery_threshold = __cpu_to_le32(100);
+	cmd->tx_teardown_threshold = __cpu_to_le32(5);
+	cmd->rssi_teardown_threshold = __cpu_to_le32(-75);
+	cmd->rssi_delta = __cpu_to_le32(-20);
+	cmd->tdls_options = __cpu_to_le32(options);
+	cmd->tdls_peer_traffic_ind_window = __cpu_to_le32(2);
+	cmd->tdls_peer_traffic_response_timeout_ms = __cpu_to_le32(5000);
+	cmd->tdls_puapsd_mask = __cpu_to_le32(0xf);
+	cmd->tdls_puapsd_inactivity_time_ms = __cpu_to_le32(0);
+	cmd->tdls_puapsd_rx_frame_threshold = __cpu_to_le32(10);
+	cmd->teardown_notification_ms = __cpu_to_le32(10);
+	cmd->tdls_peer_kickout_threshold = __cpu_to_le32(96);
+
+	ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi update fw tdls state %d for vdev %i\n",
+		   state, vdev_id);
+	return skb;
+}
+
+static u32 ath10k_wmi_prepare_peer_qos(u8 uapsd_queues, u8 sp)
+{
+	u32 peer_qos = 0;
+
+	if (uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_VO)
+		peer_qos |= WMI_TDLS_PEER_QOS_AC_VO;
+	if (uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_VI)
+		peer_qos |= WMI_TDLS_PEER_QOS_AC_VI;
+	if (uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_BK)
+		peer_qos |= WMI_TDLS_PEER_QOS_AC_BK;
+	if (uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_BE)
+		peer_qos |= WMI_TDLS_PEER_QOS_AC_BE;
+
+	peer_qos |= SM(sp, WMI_TDLS_PEER_SP);
+
+	return peer_qos;
+}
+
+static struct sk_buff *
+ath10k_wmi_10_4_gen_tdls_peer_update(struct ath10k *ar,
+				     const struct wmi_tdls_peer_update_cmd_arg *arg,
+				     const struct wmi_tdls_peer_capab_arg *cap,
+				     const struct wmi_channel_arg *chan_arg)
+{
+	struct wmi_10_4_tdls_peer_update_cmd *cmd;
+	struct wmi_tdls_peer_capabilities *peer_cap;
+	struct wmi_channel *chan;
+	struct sk_buff *skb;
+	u32 peer_qos;
+	int len, chan_len;
+	int i;
+
+	/* tdls peer update cmd has place holder for one channel*/
+	chan_len = cap->peer_chan_len ? (cap->peer_chan_len - 1) : 0;
+
+	len = sizeof(*cmd) + chan_len * sizeof(*chan);
+
+	skb = ath10k_wmi_alloc_skb(ar, len);
+	if (!skb)
+		return ERR_PTR(-ENOMEM);
+
+	memset(skb->data, 0, sizeof(*cmd));
+
+	cmd = (struct wmi_10_4_tdls_peer_update_cmd *)skb->data;
+	cmd->vdev_id = __cpu_to_le32(arg->vdev_id);
+	ether_addr_copy(cmd->peer_macaddr.addr, arg->addr);
+	cmd->peer_state = __cpu_to_le32(arg->peer_state);
+
+	peer_qos = ath10k_wmi_prepare_peer_qos(cap->peer_uapsd_queues,
+					       cap->peer_max_sp);
+
+	peer_cap = &cmd->peer_capab;
+	peer_cap->peer_qos = __cpu_to_le32(peer_qos);
+	peer_cap->buff_sta_support = __cpu_to_le32(cap->buff_sta_support);
+	peer_cap->off_chan_support = __cpu_to_le32(cap->off_chan_support);
+	peer_cap->peer_curr_operclass = __cpu_to_le32(cap->peer_curr_operclass);
+	peer_cap->self_curr_operclass = __cpu_to_le32(cap->self_curr_operclass);
+	peer_cap->peer_chan_len = __cpu_to_le32(cap->peer_chan_len);
+	peer_cap->peer_operclass_len = __cpu_to_le32(cap->peer_operclass_len);
+
+	for (i = 0; i < WMI_TDLS_MAX_SUPP_OPER_CLASSES; i++)
+		peer_cap->peer_operclass[i] = cap->peer_operclass[i];
+
+	peer_cap->is_peer_responder = __cpu_to_le32(cap->is_peer_responder);
+	peer_cap->pref_offchan_num = __cpu_to_le32(cap->pref_offchan_num);
+	peer_cap->pref_offchan_bw = __cpu_to_le32(cap->pref_offchan_bw);
+
+	for (i = 0; i < cap->peer_chan_len; i++) {
+		chan = (struct wmi_channel *)&peer_cap->peer_chan_list[i];
+		ath10k_wmi_put_wmi_channel(chan, &chan_arg[i]);
+	}
+
+	ath10k_dbg(ar, ATH10K_DBG_WMI,
+		   "wmi tdls peer update vdev %i state %d n_chans %u\n",
+		   arg->vdev_id, arg->peer_state, cap->peer_chan_len);
+	return skb;
+}
+
+static struct sk_buff *
 ath10k_wmi_op_gen_echo(struct ath10k *ar, u32 value)
 {
 	struct wmi_echo_cmd *cmd;
@@ -8197,6 +8355,8 @@ static const struct wmi_ops wmi_10_4_ops = {
 	.gen_delba_send = ath10k_wmi_op_gen_delba_send,
 	.fw_stats_fill = ath10k_wmi_10_4_op_fw_stats_fill,
 	.ext_resource_config = ath10k_wmi_10_4_ext_resource_config,
+	.gen_update_fw_tdls_state = ath10k_wmi_10_4_gen_update_fw_tdls_state,
+	.gen_tdls_peer_update = ath10k_wmi_10_4_gen_tdls_peer_update,
 
 	/* shared with 10.2 */
 	.pull_echo_ev = ath10k_wmi_op_pull_echo_ev,
diff --git a/drivers/net/wireless/ath/ath10k/wmi.h b/drivers/net/wireless/ath/ath10k/wmi.h
index baa38c8f847c..7a3606dde227 100644
--- a/drivers/net/wireless/ath/ath10k/wmi.h
+++ b/drivers/net/wireless/ath/ath10k/wmi.h
@@ -184,6 +184,17 @@ enum wmi_service {
 	WMI_SERVICE_TX_MODE_PUSH_ONLY,
 	WMI_SERVICE_TX_MODE_PUSH_PULL,
 	WMI_SERVICE_TX_MODE_DYNAMIC,
+	WMI_SERVICE_VDEV_RX_FILTER,
+	WMI_SERVICE_BTCOEX,
+	WMI_SERVICE_CHECK_CAL_VERSION,
+	WMI_SERVICE_DBGLOG_WARN2,
+	WMI_SERVICE_BTCOEX_DUTY_CYCLE,
+	WMI_SERVICE_4_WIRE_COEX_SUPPORT,
+	WMI_SERVICE_EXTENDED_NSS_SUPPORT,
+	WMI_SERVICE_PROG_GPIO_BAND_SELECT,
+	WMI_SERVICE_SMART_LOGGING_SUPPORT,
+	WMI_SERVICE_TDLS_CONN_TRACKER_IN_HOST_MODE,
+	WMI_SERVICE_TDLS_EXPLICIT_MODE_ONLY,
 
 	/* keep last */
 	WMI_SERVICE_MAX,
@@ -310,6 +321,21 @@ enum wmi_10_4_service {
 	WMI_10_4_SERVICE_TX_MODE_PUSH_ONLY,
 	WMI_10_4_SERVICE_TX_MODE_PUSH_PULL,
 	WMI_10_4_SERVICE_TX_MODE_DYNAMIC,
+	WMI_10_4_SERVICE_VDEV_RX_FILTER,
+	WMI_10_4_SERVICE_BTCOEX,
+	WMI_10_4_SERVICE_CHECK_CAL_VERSION,
+	WMI_10_4_SERVICE_DBGLOG_WARN2,
+	WMI_10_4_SERVICE_BTCOEX_DUTY_CYCLE,
+	WMI_10_4_SERVICE_4_WIRE_COEX_SUPPORT,
+	WMI_10_4_SERVICE_EXTENDED_NSS_SUPPORT,
+	WMI_10_4_SERVICE_PROG_GPIO_BAND_SELECT,
+	WMI_10_4_SERVICE_SMART_LOGGING_SUPPORT,
+	WMI_10_4_SERVICE_TDLS,
+	WMI_10_4_SERVICE_TDLS_OFFCHAN,
+	WMI_10_4_SERVICE_TDLS_UAPSD_BUFFER_STA,
+	WMI_10_4_SERVICE_TDLS_UAPSD_SLEEP_STA,
+	WMI_10_4_SERVICE_TDLS_CONN_TRACKER_IN_HOST_MODE,
+	WMI_10_4_SERVICE_TDLS_EXPLICIT_MODE_ONLY,
 };
 
 static inline char *wmi_service_name(int service_id)
@@ -408,6 +434,16 @@ static inline char *wmi_service_name(int service_id)
 	SVCSTR(WMI_SERVICE_TX_MODE_PUSH_ONLY);
 	SVCSTR(WMI_SERVICE_TX_MODE_PUSH_PULL);
 	SVCSTR(WMI_SERVICE_TX_MODE_DYNAMIC);
+	SVCSTR(WMI_SERVICE_VDEV_RX_FILTER);
+	SVCSTR(WMI_SERVICE_CHECK_CAL_VERSION);
+	SVCSTR(WMI_SERVICE_DBGLOG_WARN2);
+	SVCSTR(WMI_SERVICE_BTCOEX_DUTY_CYCLE);
+	SVCSTR(WMI_SERVICE_4_WIRE_COEX_SUPPORT);
+	SVCSTR(WMI_SERVICE_EXTENDED_NSS_SUPPORT);
+	SVCSTR(WMI_SERVICE_PROG_GPIO_BAND_SELECT);
+	SVCSTR(WMI_SERVICE_SMART_LOGGING_SUPPORT);
+	SVCSTR(WMI_SERVICE_TDLS_CONN_TRACKER_IN_HOST_MODE);
+	SVCSTR(WMI_SERVICE_TDLS_EXPLICIT_MODE_ONLY);
 	default:
 		return NULL;
 	}
@@ -420,9 +456,20 @@ static inline char *wmi_service_name(int service_id)
 	 __le32_to_cpu((wmi_svc_bmap)[(svc_id) / (sizeof(u32))]) & \
 	 BIT((svc_id) % (sizeof(u32))))
 
+/* This extension is required to accommodate new services, current limit
+ * for wmi_services is 64 as target is using only 4-bits of each 32-bit
+ * wmi_service word. Extending this to make use of remaining unused bits
+ * for new services.
+ */
+#define WMI_EXT_SERVICE_IS_ENABLED(wmi_svc_bmap, svc_id, len) \
+	((svc_id) >= (len) && \
+	__le32_to_cpu((wmi_svc_bmap)[((svc_id) - (len)) / 28]) & \
+	BIT(((((svc_id) - (len)) % 28) & 0x1f) + 4))
+
 #define SVCMAP(x, y, len) \
 	do { \
-		if (WMI_SERVICE_IS_ENABLED((in), (x), (len))) \
+		if ((WMI_SERVICE_IS_ENABLED((in), (x), (len))) || \
+		    (WMI_EXT_SERVICE_IS_ENABLED((in), (x), (len)))) \
 			__set_bit(y, out); \
 	} while (0)
 
@@ -663,6 +710,36 @@ static inline void wmi_10_4_svc_map(const __le32 *in, unsigned long *out,
 	       WMI_SERVICE_TX_MODE_PUSH_PULL, len);
 	SVCMAP(WMI_10_4_SERVICE_TX_MODE_DYNAMIC,
 	       WMI_SERVICE_TX_MODE_DYNAMIC, len);
+	SVCMAP(WMI_10_4_SERVICE_VDEV_RX_FILTER,
+	       WMI_SERVICE_VDEV_RX_FILTER, len);
+	SVCMAP(WMI_10_4_SERVICE_BTCOEX,
+	       WMI_SERVICE_BTCOEX, len);
+	SVCMAP(WMI_10_4_SERVICE_CHECK_CAL_VERSION,
+	       WMI_SERVICE_CHECK_CAL_VERSION, len);
+	SVCMAP(WMI_10_4_SERVICE_DBGLOG_WARN2,
+	       WMI_SERVICE_DBGLOG_WARN2, len);
+	SVCMAP(WMI_10_4_SERVICE_BTCOEX_DUTY_CYCLE,
+	       WMI_SERVICE_BTCOEX_DUTY_CYCLE, len);
+	SVCMAP(WMI_10_4_SERVICE_4_WIRE_COEX_SUPPORT,
+	       WMI_SERVICE_4_WIRE_COEX_SUPPORT, len);
+	SVCMAP(WMI_10_4_SERVICE_EXTENDED_NSS_SUPPORT,
+	       WMI_SERVICE_EXTENDED_NSS_SUPPORT, len);
+	SVCMAP(WMI_10_4_SERVICE_PROG_GPIO_BAND_SELECT,
+	       WMI_SERVICE_PROG_GPIO_BAND_SELECT, len);
+	SVCMAP(WMI_10_4_SERVICE_SMART_LOGGING_SUPPORT,
+	       WMI_SERVICE_SMART_LOGGING_SUPPORT, len);
+	SVCMAP(WMI_10_4_SERVICE_TDLS,
+	       WMI_SERVICE_TDLS, len);
+	SVCMAP(WMI_10_4_SERVICE_TDLS_OFFCHAN,
+	       WMI_SERVICE_TDLS_OFFCHAN, len);
+	SVCMAP(WMI_10_4_SERVICE_TDLS_UAPSD_BUFFER_STA,
+	       WMI_SERVICE_TDLS_UAPSD_BUFFER_STA, len);
+	SVCMAP(WMI_10_4_SERVICE_TDLS_UAPSD_SLEEP_STA,
+	       WMI_SERVICE_TDLS_UAPSD_SLEEP_STA, len);
+	SVCMAP(WMI_10_4_SERVICE_TDLS_CONN_TRACKER_IN_HOST_MODE,
+	       WMI_SERVICE_TDLS_CONN_TRACKER_IN_HOST_MODE, len);
+	SVCMAP(WMI_10_4_SERVICE_TDLS_EXPLICIT_MODE_ONLY,
+	       WMI_SERVICE_TDLS_EXPLICIT_MODE_ONLY, len);
 }
 
 #undef SVCMAP
@@ -837,6 +914,29 @@ struct wmi_cmd_map {
 	u32 pdev_bss_chan_info_request_cmdid;
 	u32 pdev_enable_adaptive_cca_cmdid;
 	u32 ext_resource_cfg_cmdid;
+	u32 vdev_set_ie_cmdid;
+	u32 set_lteu_config_cmdid;
+	u32 atf_ssid_grouping_request_cmdid;
+	u32 peer_atf_ext_request_cmdid;
+	u32 set_periodic_channel_stats_cfg_cmdid;
+	u32 peer_bwf_request_cmdid;
+	u32 btcoex_cfg_cmdid;
+	u32 peer_tx_mu_txmit_count_cmdid;
+	u32 peer_tx_mu_txmit_rstcnt_cmdid;
+	u32 peer_gid_userpos_list_cmdid;
+	u32 pdev_check_cal_version_cmdid;
+	u32 coex_version_cfg_cmid;
+	u32 pdev_get_rx_filter_cmdid;
+	u32 pdev_extended_nss_cfg_cmdid;
+	u32 vdev_set_scan_nac_rssi_cmdid;
+	u32 prog_gpio_band_select_cmdid;
+	u32 config_smart_logging_cmdid;
+	u32 debug_fatal_condition_cmdid;
+	u32 get_tsf_timer_cmdid;
+	u32 pdev_get_tpc_table_cmdid;
+	u32 vdev_sifs_trigger_time_cmdid;
+	u32 pdev_wds_entry_list_cmdid;
+	u32 tdls_set_offchan_mode_cmdid;
 };
 
 /*
@@ -1647,6 +1747,29 @@ enum wmi_10_4_cmd_id {
 	WMI_10_4_EXT_RESOURCE_CFG_CMDID,
 	WMI_10_4_VDEV_SET_IE_CMDID,
 	WMI_10_4_SET_LTEU_CONFIG_CMDID,
+	WMI_10_4_ATF_SSID_GROUPING_REQUEST_CMDID,
+	WMI_10_4_PEER_ATF_EXT_REQUEST_CMDID,
+	WMI_10_4_SET_PERIODIC_CHANNEL_STATS_CONFIG,
+	WMI_10_4_PEER_BWF_REQUEST_CMDID,
+	WMI_10_4_BTCOEX_CFG_CMDID,
+	WMI_10_4_PEER_TX_MU_TXMIT_COUNT_CMDID,
+	WMI_10_4_PEER_TX_MU_TXMIT_RSTCNT_CMDID,
+	WMI_10_4_PEER_GID_USERPOS_LIST_CMDID,
+	WMI_10_4_PDEV_CHECK_CAL_VERSION_CMDID,
+	WMI_10_4_COEX_VERSION_CFG_CMID,
+	WMI_10_4_PDEV_GET_RX_FILTER_CMDID,
+	WMI_10_4_PDEV_EXTENDED_NSS_CFG_CMDID,
+	WMI_10_4_VDEV_SET_SCAN_NAC_RSSI_CMDID,
+	WMI_10_4_PROG_GPIO_BAND_SELECT_CMDID,
+	WMI_10_4_CONFIG_SMART_LOGGING_CMDID,
+	WMI_10_4_DEBUG_FATAL_CONDITION_CMDID,
+	WMI_10_4_GET_TSF_TIMER_CMDID,
+	WMI_10_4_PDEV_GET_TPC_TABLE_CMDID,
+	WMI_10_4_VDEV_SIFS_TRIGGER_TIME_CMDID,
+	WMI_10_4_PDEV_WDS_ENTRY_LIST_CMDID,
+	WMI_10_4_TDLS_SET_STATE_CMDID,
+	WMI_10_4_TDLS_PEER_UPDATE_CMDID,
+	WMI_10_4_TDLS_SET_OFFCHAN_MODE_CMDID,
 	WMI_10_4_PDEV_UTF_CMDID = WMI_10_4_END_CMDID - 1,
 };
 
@@ -1710,6 +1833,18 @@ enum wmi_10_4_event_id {
 	WMI_10_4_PDEV_NFCAL_POWER_ALL_CHANNELS_EVENTID,
 	WMI_10_4_PDEV_BSS_CHAN_INFO_EVENTID,
 	WMI_10_4_MU_REPORT_EVENTID,
+	WMI_10_4_TX_DATA_TRAFFIC_CTRL_EVENTID,
+	WMI_10_4_PEER_TX_MU_TXMIT_COUNT_EVENTID,
+	WMI_10_4_PEER_GID_USERPOS_LIST_EVENTID,
+	WMI_10_4_PDEV_CHECK_CAL_VERSION_EVENTID,
+	WMI_10_4_ATF_PEER_STATS_EVENTID,
+	WMI_10_4_PDEV_GET_RX_FILTER_EVENTID,
+	WMI_10_4_NAC_RSSI_EVENTID,
+	WMI_10_4_DEBUG_FATAL_CONDITION_EVENTID,
+	WMI_10_4_GET_TSF_TIMER_RESP_EVENTID,
+	WMI_10_4_PDEV_TPC_TABLE_EVENTID,
+	WMI_10_4_PDEV_WDS_ENTRY_LIST_EVENTID,
+	WMI_10_4_TDLS_PEER_EVENTID,
 	WMI_10_4_PDEV_UTF_EVENTID = WMI_10_4_END_EVENTID - 1,
 };
 
@@ -2718,6 +2853,18 @@ struct wmi_resource_config_10_4 {
 	__le32 qwrap_config;
 } __packed;
 
+enum wmi_coex_version {
+	WMI_NO_COEX_VERSION_SUPPORT	= 0,
+	/* 3 wire coex support*/
+	WMI_COEX_VERSION_1		= 1,
+	/* 2.5 wire coex support*/
+	WMI_COEX_VERSION_2		= 2,
+	/* 2.5 wire coex with duty cycle support */
+	WMI_COEX_VERSION_3		= 3,
+	/* 4 wire coex support*/
+	WMI_COEX_VERSION_4		= 4,
+};
+
 /**
  * enum wmi_10_4_feature_mask - WMI 10.4 feature enable/disable flags
  * @WMI_10_4_LTEU_SUPPORT: LTEU config
@@ -2726,6 +2873,14 @@ struct wmi_resource_config_10_4 {
  * @WMI_10_4_AUX_RADIO_CHAN_LOAD_INTF: AUX Radio Enhancement for chan load scan
  * @WMI_10_4_BSS_CHANNEL_INFO_64: BSS channel info stats
  * @WMI_10_4_PEER_STATS: Per station stats
+ * @WMI_10_4_VDEV_STATS: Per vdev stats
+ * @WMI_10_4_TDLS: Implicit TDLS support in firmware enable/disable
+ * @WMI_10_4_TDLS_OFFCHAN: TDLS offchannel support enable/disable
+ * @WMI_10_4_TDLS_UAPSD_BUFFER_STA: TDLS buffer sta support enable/disable
+ * @WMI_10_4_TDLS_UAPSD_SLEEP_STA: TDLS sleep sta support enable/disable
+ * @WMI_10_4_TDLS_CONN_TRACKER_IN_HOST_MODE: TDLS connection tracker in host
+ *	enable/disable
+ * @WMI_10_4_TDLS_EXPLICIT_MODE_ONLY:Explicit TDLS mode enable/disable
  */
 enum wmi_10_4_feature_mask {
 	WMI_10_4_LTEU_SUPPORT			= BIT(0),
@@ -2734,6 +2889,14 @@ enum wmi_10_4_feature_mask {
 	WMI_10_4_AUX_RADIO_CHAN_LOAD_INTF	= BIT(3),
 	WMI_10_4_BSS_CHANNEL_INFO_64		= BIT(4),
 	WMI_10_4_PEER_STATS			= BIT(5),
+	WMI_10_4_VDEV_STATS			= BIT(6),
+	WMI_10_4_TDLS				= BIT(7),
+	WMI_10_4_TDLS_OFFCHAN			= BIT(8),
+	WMI_10_4_TDLS_UAPSD_BUFFER_STA		= BIT(9),
+	WMI_10_4_TDLS_UAPSD_SLEEP_STA		= BIT(10),
+	WMI_10_4_TDLS_CONN_TRACKER_IN_HOST_MODE = BIT(11),
+	WMI_10_4_TDLS_EXPLICIT_MODE_ONLY	= BIT(12),
+
 };
 
 struct wmi_ext_resource_config_10_4_cmd {
@@ -2741,6 +2904,22 @@ struct wmi_ext_resource_config_10_4_cmd {
 	__le32 host_platform_config;
 	/* see enum wmi_10_4_feature_mask */
 	__le32 fw_feature_bitmap;
+	/* WLAN priority GPIO number */
+	__le32 wlan_gpio_priority;
+	/* see enum wmi_coex_version */
+	__le32 coex_version;
+	/* COEX GPIO config */
+	__le32 coex_gpio_pin1;
+	__le32 coex_gpio_pin2;
+	__le32 coex_gpio_pin3;
+	/* number of vdevs allowed to perform tdls */
+	__le32 num_tdls_vdevs;
+	/* number of peers to track per TDLS vdev */
+	__le32 num_tdls_conn_table_entries;
+	/* number of tdls sleep sta supported */
+	__le32 max_tdls_concurrent_sleep_sta;
+	/* number of tdls buffer sta supported */
+	__le32 max_tdls_concurrent_buffer_sta;
 };
 
 /* strucutre describing host memory chunk. */
@@ -5698,6 +5877,7 @@ struct wmi_tbtt_offset_event {
 struct wmi_peer_create_cmd {
 	__le32 vdev_id;
 	struct wmi_mac_addr peer_macaddr;
+	__le32 peer_type;
 } __packed;
 
 enum wmi_peer_type {
@@ -6556,6 +6736,22 @@ struct wmi_tdls_peer_update_cmd_arg {
 
 #define WMI_TDLS_MAX_SUPP_OPER_CLASSES 32
 
+#define WMI_TDLS_PEER_SP_MASK	0x60
+#define WMI_TDLS_PEER_SP_LSB	5
+
+enum wmi_tdls_options {
+	WMI_TDLS_OFFCHAN_EN = BIT(0),
+	WMI_TDLS_BUFFER_STA_EN = BIT(1),
+	WMI_TDLS_SLEEP_STA_EN = BIT(2),
+};
+
+enum {
+	WMI_TDLS_PEER_QOS_AC_VO = BIT(0),
+	WMI_TDLS_PEER_QOS_AC_VI = BIT(1),
+	WMI_TDLS_PEER_QOS_AC_BK = BIT(2),
+	WMI_TDLS_PEER_QOS_AC_BE = BIT(3),
+};
+
 struct wmi_tdls_peer_capab_arg {
 	u8 peer_uapsd_queues;
 	u8 peer_max_sp;
@@ -6571,6 +6767,79 @@ struct wmi_tdls_peer_capab_arg {
 	u32 pref_offchan_bw;
 };
 
+struct wmi_10_4_tdls_set_state_cmd {
+	__le32 vdev_id;
+	__le32 state;
+	__le32 notification_interval_ms;
+	__le32 tx_discovery_threshold;
+	__le32 tx_teardown_threshold;
+	__le32 rssi_teardown_threshold;
+	__le32 rssi_delta;
+	__le32 tdls_options;
+	__le32 tdls_peer_traffic_ind_window;
+	__le32 tdls_peer_traffic_response_timeout_ms;
+	__le32 tdls_puapsd_mask;
+	__le32 tdls_puapsd_inactivity_time_ms;
+	__le32 tdls_puapsd_rx_frame_threshold;
+	__le32 teardown_notification_ms;
+	__le32 tdls_peer_kickout_threshold;
+} __packed;
+
+struct wmi_tdls_peer_capabilities {
+	__le32 peer_qos;
+	__le32 buff_sta_support;
+	__le32 off_chan_support;
+	__le32 peer_curr_operclass;
+	__le32 self_curr_operclass;
+	__le32 peer_chan_len;
+	__le32 peer_operclass_len;
+	u8 peer_operclass[WMI_TDLS_MAX_SUPP_OPER_CLASSES];
+	__le32 is_peer_responder;
+	__le32 pref_offchan_num;
+	__le32 pref_offchan_bw;
+	struct wmi_channel peer_chan_list[1];
+} __packed;
+
+struct wmi_10_4_tdls_peer_update_cmd {
+	__le32 vdev_id;
+	struct wmi_mac_addr peer_macaddr;
+	__le32 peer_state;
+	__le32 reserved[4];
+	struct wmi_tdls_peer_capabilities peer_capab;
+} __packed;
+
+enum wmi_tdls_peer_reason {
+	WMI_TDLS_TEARDOWN_REASON_TX,
+	WMI_TDLS_TEARDOWN_REASON_RSSI,
+	WMI_TDLS_TEARDOWN_REASON_SCAN,
+	WMI_TDLS_DISCONNECTED_REASON_PEER_DELETE,
+	WMI_TDLS_TEARDOWN_REASON_PTR_TIMEOUT,
+	WMI_TDLS_TEARDOWN_REASON_BAD_PTR,
+	WMI_TDLS_TEARDOWN_REASON_NO_RESPONSE,
+	WMI_TDLS_ENTER_BUF_STA,
+	WMI_TDLS_EXIT_BUF_STA,
+	WMI_TDLS_ENTER_BT_BUSY_MODE,
+	WMI_TDLS_EXIT_BT_BUSY_MODE,
+	WMI_TDLS_SCAN_STARTED_EVENT,
+	WMI_TDLS_SCAN_COMPLETED_EVENT,
+};
+
+enum wmi_tdls_peer_notification {
+	WMI_TDLS_SHOULD_DISCOVER,
+	WMI_TDLS_SHOULD_TEARDOWN,
+	WMI_TDLS_PEER_DISCONNECTED,
+	WMI_TDLS_CONNECTION_TRACKER_NOTIFICATION,
+};
+
+struct wmi_tdls_peer_event {
+	struct wmi_mac_addr peer_macaddr;
+	/* see enum wmi_tdls_peer_notification*/
+	__le32 peer_status;
+	/* see enum wmi_tdls_peer_reason */
+	__le32 peer_reason;
+	__le32 vdev_id;
+} __packed;
+
 enum wmi_txbf_conf {
 	WMI_TXBF_CONF_UNSUPPORTED,
 	WMI_TXBF_CONF_BEFORE_ASSOC,
diff --git a/drivers/net/wireless/ath/ath10k/wow.c b/drivers/net/wireless/ath/ath10k/wow.c
index 77100d42f401..0d46d6dc7578 100644
--- a/drivers/net/wireless/ath/ath10k/wow.c
+++ b/drivers/net/wireless/ath/ath10k/wow.c
@@ -277,6 +277,18 @@ exit:
 	return ret ? 1 : 0;
 }
 
+void ath10k_wow_op_set_wakeup(struct ieee80211_hw *hw, bool enabled)
+{
+	struct ath10k *ar = hw->priv;
+
+	mutex_lock(&ar->conf_mutex);
+	if (test_bit(ATH10K_FW_FEATURE_WOWLAN_SUPPORT,
+		     ar->running_fw->fw_file.fw_features)) {
+		device_set_wakeup_enable(ar->dev, enabled);
+	}
+	mutex_unlock(&ar->conf_mutex);
+}
+
 int ath10k_wow_op_resume(struct ieee80211_hw *hw)
 {
 	struct ath10k *ar = hw->priv;
@@ -336,5 +348,7 @@ int ath10k_wow_init(struct ath10k *ar)
 	ar->wow.wowlan_support.n_patterns = ar->wow.max_num_patterns;
 	ar->hw->wiphy->wowlan = &ar->wow.wowlan_support;
 
+	device_set_wakeup_capable(ar->dev, true);
+
 	return 0;
 }
diff --git a/drivers/net/wireless/ath/ath10k/wow.h b/drivers/net/wireless/ath/ath10k/wow.h
index abbb04b6d1bd..9745b9ddc7f5 100644
--- a/drivers/net/wireless/ath/ath10k/wow.h
+++ b/drivers/net/wireless/ath/ath10k/wow.h
@@ -28,6 +28,7 @@ int ath10k_wow_init(struct ath10k *ar);
 int ath10k_wow_op_suspend(struct ieee80211_hw *hw,
 			  struct cfg80211_wowlan *wowlan);
 int ath10k_wow_op_resume(struct ieee80211_hw *hw);
+void ath10k_wow_op_set_wakeup(struct ieee80211_hw *hw, bool enabled);
 
 #else
 
diff --git a/drivers/net/wireless/ath/ath6kl/usb.c b/drivers/net/wireless/ath/ath6kl/usb.c
index 9da3594fd010..4defb7a0330f 100644
--- a/drivers/net/wireless/ath/ath6kl/usb.c
+++ b/drivers/net/wireless/ath/ath6kl/usb.c
@@ -1201,7 +1201,7 @@ static int ath6kl_usb_pm_resume(struct usb_interface *interface)
 #endif
 
 /* table of devices that work with this driver */
-static struct usb_device_id ath6kl_usb_ids[] = {
+static const struct usb_device_id ath6kl_usb_ids[] = {
 	{USB_DEVICE(0x0cf3, 0x9375)},
 	{USB_DEVICE(0x0cf3, 0x9374)},
 	{ /* Terminating entry */ },
diff --git a/drivers/net/wireless/ath/ath9k/debug.c b/drivers/net/wireless/ath/ath9k/debug.c
index 2e64977a8ab6..01fa30117288 100644
--- a/drivers/net/wireless/ath/ath9k/debug.c
+++ b/drivers/net/wireless/ath/ath9k/debug.c
@@ -1452,7 +1452,7 @@ int ath9k_init_debug(struct ath_hw *ah)
 #endif
 
 #ifdef CONFIG_ATH9K_DYNACK
-	debugfs_create_file("ack_to", S_IRUSR | S_IWUSR, sc->debug.debugfs_phy,
+	debugfs_create_file("ack_to", S_IRUSR, sc->debug.debugfs_phy,
 			    sc, &fops_ackto);
 #endif
 	debugfs_create_file("tpc", S_IRUSR | S_IWUSR,
diff --git a/drivers/net/wireless/ath/ath9k/hif_usb.c b/drivers/net/wireless/ath/ath9k/hif_usb.c
index 0d9687a2aa98..c5f4dd808745 100644
--- a/drivers/net/wireless/ath/ath9k/hif_usb.c
+++ b/drivers/net/wireless/ath/ath9k/hif_usb.c
@@ -20,7 +20,7 @@
 MODULE_FIRMWARE(HTC_7010_MODULE_FW);
 MODULE_FIRMWARE(HTC_9271_MODULE_FW);
 
-static struct usb_device_id ath9k_hif_usb_ids[] = {
+static const struct usb_device_id ath9k_hif_usb_ids[] = {
 	{ USB_DEVICE(0x0cf3, 0x9271) }, /* Atheros */
 	{ USB_DEVICE(0x0cf3, 0x1006) }, /* Atheros */
 	{ USB_DEVICE(0x0846, 0x9030) }, /* Netgear N150 */
diff --git a/drivers/net/wireless/ath/ath9k/htc_drv_init.c b/drivers/net/wireless/ath/ath9k/htc_drv_init.c
index defacc6c9c99..da2164b0cccc 100644
--- a/drivers/net/wireless/ath/ath9k/htc_drv_init.c
+++ b/drivers/net/wireless/ath/ath9k/htc_drv_init.c
@@ -71,7 +71,7 @@ static void ath9k_htc_op_ps_restore(struct ath_common *common)
 	ath9k_htc_ps_restore((struct ath9k_htc_priv *) common->priv);
 }
 
-static struct ath_ps_ops ath9k_htc_ps_ops = {
+static const struct ath_ps_ops ath9k_htc_ps_ops = {
 	.wakeup = ath9k_htc_op_ps_wakeup,
 	.restore = ath9k_htc_op_ps_restore,
 };
diff --git a/drivers/net/wireless/ath/ath9k/init.c b/drivers/net/wireless/ath/ath9k/init.c
index fd9a61834c17..bb7936090b91 100644
--- a/drivers/net/wireless/ath/ath9k/init.c
+++ b/drivers/net/wireless/ath/ath9k/init.c
@@ -104,7 +104,7 @@ static void ath9k_op_ps_restore(struct ath_common *common)
 	ath9k_ps_restore((struct ath_softc *) common->priv);
 }
 
-static struct ath_ps_ops ath9k_ps_ops = {
+static const struct ath_ps_ops ath9k_ps_ops = {
 	.wakeup = ath9k_op_ps_wakeup,
 	.restore = ath9k_op_ps_restore,
 };
diff --git a/drivers/net/wireless/ath/ath9k/pci.c b/drivers/net/wireless/ath/ath9k/pci.c
index 7b7627f85d3a..223606311261 100644
--- a/drivers/net/wireless/ath/ath9k/pci.c
+++ b/drivers/net/wireless/ath/ath9k/pci.c
@@ -388,6 +388,11 @@ static const struct pci_device_id ath_pci_id_table[] = {
 			 PCI_VENDOR_ID_DELL,
 			 0x020B),
 	  .driver_data = ATH9K_PCI_WOW },
+	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_ATHEROS,
+			 0x0034,
+			 PCI_VENDOR_ID_DELL,
+			 0x0300),
+	  .driver_data = ATH9K_PCI_WOW },
 
 	/* Killer Wireless (2x2) */
 	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_ATHEROS,
diff --git a/drivers/net/wireless/ath/carl9170/usb.c b/drivers/net/wireless/ath/carl9170/usb.c
index 99ab20334d21..e7c3f3b8457d 100644
--- a/drivers/net/wireless/ath/carl9170/usb.c
+++ b/drivers/net/wireless/ath/carl9170/usb.c
@@ -64,7 +64,7 @@ MODULE_ALIAS("arusb_lnx");
  * http://wireless.kernel.org/en/users/Drivers/ar9170/devices ),
  * whenever you add a new device.
  */
-static struct usb_device_id carl9170_usb_ids[] = {
+static const struct usb_device_id carl9170_usb_ids[] = {
 	/* Atheros 9170 */
 	{ USB_DEVICE(0x0cf3, 0x9170) },
 	/* Atheros TG121N */
diff --git a/drivers/net/wireless/ath/wcn36xx/dxe.c b/drivers/net/wireless/ath/wcn36xx/dxe.c
index 87dfdaf9044c..d5c810a8cc52 100644
--- a/drivers/net/wireless/ath/wcn36xx/dxe.c
+++ b/drivers/net/wireless/ath/wcn36xx/dxe.c
@@ -289,6 +289,11 @@ static int wcn36xx_dxe_fill_skb(struct device *dev, struct wcn36xx_dxe_ctl *ctl)
 					 skb_tail_pointer(skb),
 					 WCN36XX_PKT_SIZE,
 					 DMA_FROM_DEVICE);
+	if (dma_mapping_error(dev, dxe->dst_addr_l)) {
+		dev_err(dev, "unable to map skb\n");
+		kfree_skb(skb);
+		return -ENOMEM;
+	}
 	ctl->skb = skb;
 
 	return 0;
diff --git a/drivers/net/wireless/ath/wcn36xx/main.c b/drivers/net/wireless/ath/wcn36xx/main.c
index 517a315e259b..35bd50bcbbd5 100644
--- a/drivers/net/wireless/ath/wcn36xx/main.c
+++ b/drivers/net/wireless/ath/wcn36xx/main.c
@@ -372,6 +372,8 @@ static int wcn36xx_config(struct ieee80211_hw *hw, u32 changed)
 
 	wcn36xx_dbg(WCN36XX_DBG_MAC, "mac config changed 0x%08x\n", changed);
 
+	mutex_lock(&wcn->conf_mutex);
+
 	if (changed & IEEE80211_CONF_CHANGE_CHANNEL) {
 		int ch = WCN36XX_HW_CHANNEL(wcn);
 		wcn36xx_dbg(WCN36XX_DBG_MAC, "wcn36xx_config channel switch=%d\n",
@@ -382,6 +384,8 @@ static int wcn36xx_config(struct ieee80211_hw *hw, u32 changed)
 		}
 	}
 
+	mutex_unlock(&wcn->conf_mutex);
+
 	return 0;
 }
 
@@ -396,6 +400,8 @@ static void wcn36xx_configure_filter(struct ieee80211_hw *hw,
 
 	wcn36xx_dbg(WCN36XX_DBG_MAC, "mac configure filter\n");
 
+	mutex_lock(&wcn->conf_mutex);
+
 	*total &= FIF_ALLMULTI;
 
 	fp = (void *)(unsigned long)multicast;
@@ -408,6 +414,8 @@ static void wcn36xx_configure_filter(struct ieee80211_hw *hw,
 		else if (NL80211_IFTYPE_STATION == vif->type && tmp->sta_assoc)
 			wcn36xx_smd_set_mc_list(wcn, vif, fp);
 	}
+
+	mutex_unlock(&wcn->conf_mutex);
 	kfree(fp);
 }
 
@@ -471,6 +479,8 @@ static int wcn36xx_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
 			 key_conf->key,
 			 key_conf->keylen);
 
+	mutex_lock(&wcn->conf_mutex);
+
 	switch (key_conf->cipher) {
 	case WLAN_CIPHER_SUITE_WEP40:
 		vif_priv->encrypt_type = WCN36XX_HAL_ED_WEP40;
@@ -565,6 +575,8 @@ static int wcn36xx_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
 	}
 
 out:
+	mutex_unlock(&wcn->conf_mutex);
+
 	return ret;
 }
 
@@ -725,6 +737,8 @@ static void wcn36xx_bss_info_changed(struct ieee80211_hw *hw,
 	wcn36xx_dbg(WCN36XX_DBG_MAC, "mac bss info changed vif %p changed 0x%08x\n",
 		    vif, changed);
 
+	mutex_lock(&wcn->conf_mutex);
+
 	if (changed & BSS_CHANGED_BEACON_INFO) {
 		wcn36xx_dbg(WCN36XX_DBG_MAC,
 			    "mac bss changed dtim period %d\n",
@@ -787,7 +801,13 @@ static void wcn36xx_bss_info_changed(struct ieee80211_hw *hw,
 				     bss_conf->aid);
 
 			vif_priv->sta_assoc = true;
-			rcu_read_lock();
+
+			/*
+			 * Holding conf_mutex ensures mutal exclusion with
+			 * wcn36xx_sta_remove() and as such ensures that sta
+			 * won't be freed while we're operating on it. As such
+			 * we do not need to hold the rcu_read_lock().
+			 */
 			sta = ieee80211_find_sta(vif, bss_conf->bssid);
 			if (!sta) {
 				wcn36xx_err("sta %pM is not found\n",
@@ -811,7 +831,6 @@ static void wcn36xx_bss_info_changed(struct ieee80211_hw *hw,
 			 * place where AID is available.
 			 */
 			wcn36xx_smd_config_sta(wcn, vif, sta);
-			rcu_read_unlock();
 		} else {
 			wcn36xx_dbg(WCN36XX_DBG_MAC,
 				    "disassociated bss %pM vif %pM AID=%d\n",
@@ -873,6 +892,9 @@ static void wcn36xx_bss_info_changed(struct ieee80211_hw *hw,
 		}
 	}
 out:
+
+	mutex_unlock(&wcn->conf_mutex);
+
 	return;
 }
 
@@ -882,7 +904,10 @@ static int wcn36xx_set_rts_threshold(struct ieee80211_hw *hw, u32 value)
 	struct wcn36xx *wcn = hw->priv;
 	wcn36xx_dbg(WCN36XX_DBG_MAC, "mac set RTS threshold %d\n", value);
 
+	mutex_lock(&wcn->conf_mutex);
 	wcn36xx_smd_update_cfg(wcn, WCN36XX_HAL_CFG_RTS_THRESHOLD, value);
+	mutex_unlock(&wcn->conf_mutex);
+
 	return 0;
 }
 
@@ -893,8 +918,12 @@ static void wcn36xx_remove_interface(struct ieee80211_hw *hw,
 	struct wcn36xx_vif *vif_priv = wcn36xx_vif_to_priv(vif);
 	wcn36xx_dbg(WCN36XX_DBG_MAC, "mac remove interface vif %p\n", vif);
 
+	mutex_lock(&wcn->conf_mutex);
+
 	list_del(&vif_priv->list);
 	wcn36xx_smd_delete_sta_self(wcn, vif->addr);
+
+	mutex_unlock(&wcn->conf_mutex);
 }
 
 static int wcn36xx_add_interface(struct ieee80211_hw *hw,
@@ -915,9 +944,13 @@ static int wcn36xx_add_interface(struct ieee80211_hw *hw,
 		return -EOPNOTSUPP;
 	}
 
+	mutex_lock(&wcn->conf_mutex);
+
 	list_add(&vif_priv->list, &wcn->vif_list);
 	wcn36xx_smd_add_sta_self(wcn, vif);
 
+	mutex_unlock(&wcn->conf_mutex);
+
 	return 0;
 }
 
@@ -930,6 +963,8 @@ static int wcn36xx_sta_add(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	wcn36xx_dbg(WCN36XX_DBG_MAC, "mac sta add vif %p sta %pM\n",
 		    vif, sta->addr);
 
+	mutex_lock(&wcn->conf_mutex);
+
 	spin_lock_init(&sta_priv->ampdu_lock);
 	sta_priv->vif = vif_priv;
 	/*
@@ -941,6 +976,9 @@ static int wcn36xx_sta_add(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 		sta_priv->aid = sta->aid;
 		wcn36xx_smd_config_sta(wcn, vif, sta);
 	}
+
+	mutex_unlock(&wcn->conf_mutex);
+
 	return 0;
 }
 
@@ -954,8 +992,13 @@ static int wcn36xx_sta_remove(struct ieee80211_hw *hw,
 	wcn36xx_dbg(WCN36XX_DBG_MAC, "mac sta remove vif %p sta %pM index %d\n",
 		    vif, sta->addr, sta_priv->sta_index);
 
+	mutex_lock(&wcn->conf_mutex);
+
 	wcn36xx_smd_delete_sta(wcn, sta_priv->sta_index);
 	sta_priv->vif = NULL;
+
+	mutex_unlock(&wcn->conf_mutex);
+
 	return 0;
 }
 
@@ -999,6 +1042,8 @@ static int wcn36xx_ampdu_action(struct ieee80211_hw *hw,
 	wcn36xx_dbg(WCN36XX_DBG_MAC, "mac ampdu action action %d tid %d\n",
 		    action, tid);
 
+	mutex_lock(&wcn->conf_mutex);
+
 	switch (action) {
 	case IEEE80211_AMPDU_RX_START:
 		sta_priv->tid = tid;
@@ -1038,6 +1083,8 @@ static int wcn36xx_ampdu_action(struct ieee80211_hw *hw,
 		wcn36xx_err("Unknown AMPDU action\n");
 	}
 
+	mutex_unlock(&wcn->conf_mutex);
+
 	return 0;
 }
 
@@ -1216,6 +1263,7 @@ static int wcn36xx_probe(struct platform_device *pdev)
 	wcn = hw->priv;
 	wcn->hw = hw;
 	wcn->dev = &pdev->dev;
+	mutex_init(&wcn->conf_mutex);
 	mutex_init(&wcn->hal_mutex);
 	mutex_init(&wcn->scan_lock);
 
diff --git a/drivers/net/wireless/ath/wcn36xx/wcn36xx.h b/drivers/net/wireless/ath/wcn36xx/wcn36xx.h
index b52b4da9a967..6aefba4c0cda 100644
--- a/drivers/net/wireless/ath/wcn36xx/wcn36xx.h
+++ b/drivers/net/wireless/ath/wcn36xx/wcn36xx.h
@@ -202,6 +202,9 @@ struct wcn36xx {
 	struct qcom_smem_state	*tx_rings_empty_state;
 	unsigned		tx_rings_empty_state_bit;
 
+	/* prevents concurrent FW reconfiguration */
+	struct mutex		conf_mutex;
+
 	/*
 	 * smd_buf must be protected with smd_mutex to garantee
 	 * that all messages are sent one after another
diff --git a/drivers/net/wireless/ath/wil6210/Kconfig b/drivers/net/wireless/ath/wil6210/Kconfig
index 6dfedc8bd6a3..b448926b0c0f 100644
--- a/drivers/net/wireless/ath/wil6210/Kconfig
+++ b/drivers/net/wireless/ath/wil6210/Kconfig
@@ -40,3 +40,15 @@ config WIL6210_TRACING
 	  option if you are interested in debugging the driver.
 
 	  If unsure, say Y to make it easier to debug problems.
+
+config WIL6210_DEBUGFS
+	bool "wil6210 debugfs support"
+	depends on WIL6210
+	depends on DEBUG_FS
+	default y
+	---help---
+	  Say Y here to enable wil6210 debugfs support, using the
+	  kernel debugfs infrastructure. Select this
+	  option if you are interested in debugging the driver.
+
+	  If unsure, say Y to make it easier to debug problems.
diff --git a/drivers/net/wireless/ath/wil6210/Makefile b/drivers/net/wireless/ath/wil6210/Makefile
index 4ae21da78e9e..d27efe83748b 100644
--- a/drivers/net/wireless/ath/wil6210/Makefile
+++ b/drivers/net/wireless/ath/wil6210/Makefile
@@ -4,7 +4,7 @@ wil6210-y := main.o
 wil6210-y += netdev.o
 wil6210-y += cfg80211.o
 wil6210-y += pcie_bus.o
-wil6210-y += debugfs.o
+wil6210-$(CONFIG_WIL6210_DEBUGFS) += debugfs.o
 wil6210-y += wmi.o
 wil6210-y += interrupt.o
 wil6210-y += txrx.o
diff --git a/drivers/net/wireless/ath/wil6210/cfg80211.c b/drivers/net/wireless/ath/wil6210/cfg80211.c
index 0b5383a62d42..85d5c04618eb 100644
--- a/drivers/net/wireless/ath/wil6210/cfg80211.c
+++ b/drivers/net/wireless/ath/wil6210/cfg80211.c
@@ -26,6 +26,12 @@ bool disable_ap_sme;
 module_param(disable_ap_sme, bool, 0444);
 MODULE_PARM_DESC(disable_ap_sme, " let user space handle AP mode SME");
 
+#ifdef CONFIG_PM
+static struct wiphy_wowlan_support wil_wowlan_support = {
+	.flags = WIPHY_WOWLAN_ANY | WIPHY_WOWLAN_DISCONNECT,
+};
+#endif
+
 #define CHAN60G(_channel, _flags) {				\
 	.band			= NL80211_BAND_60GHZ,		\
 	.center_freq		= 56160 + (2160 * (_channel)),	\
@@ -273,12 +279,12 @@ int wil_cid_fill_sinfo(struct wil6210_priv *wil, int cid,
 
 	wil_dbg_wmi(wil, "Link status for CID %d: {\n"
 		    "  MCS %d TSF 0x%016llx\n"
-		    "  BF status 0x%08x SNR 0x%08x SQI %d%%\n"
+		    "  BF status 0x%08x RSSI %d SQI %d%%\n"
 		    "  Tx Tpt %d goodput %d Rx goodput %d\n"
 		    "  Sectors(rx:tx) my %d:%d peer %d:%d\n""}\n",
 		    cid, le16_to_cpu(reply.evt.bf_mcs),
 		    le64_to_cpu(reply.evt.tsf), reply.evt.status,
-		    le32_to_cpu(reply.evt.snr_val),
+		    reply.evt.rssi,
 		    reply.evt.sqi,
 		    le32_to_cpu(reply.evt.tx_tpt),
 		    le32_to_cpu(reply.evt.tx_goodput),
@@ -311,7 +317,11 @@ int wil_cid_fill_sinfo(struct wil6210_priv *wil, int cid,
 
 	if (test_bit(wil_status_fwconnected, wil->status)) {
 		sinfo->filled |= BIT(NL80211_STA_INFO_SIGNAL);
-		sinfo->signal = reply.evt.sqi;
+		if (test_bit(WMI_FW_CAPABILITY_RSSI_REPORTING,
+			     wil->fw_capabilities))
+			sinfo->signal = reply.evt.rssi;
+		else
+			sinfo->signal = reply.evt.sqi;
 	}
 
 	return rc;
@@ -372,6 +382,34 @@ static int wil_cfg80211_dump_station(struct wiphy *wiphy,
 	return rc;
 }
 
+static int wil_cfg80211_start_p2p_device(struct wiphy *wiphy,
+					 struct wireless_dev *wdev)
+{
+	struct wil6210_priv *wil = wiphy_to_wil(wiphy);
+
+	wil_dbg_misc(wil, "start_p2p_device: entered\n");
+	wil->p2p.p2p_dev_started = 1;
+	return 0;
+}
+
+static void wil_cfg80211_stop_p2p_device(struct wiphy *wiphy,
+					 struct wireless_dev *wdev)
+{
+	struct wil6210_priv *wil = wiphy_to_wil(wiphy);
+	struct wil_p2p_info *p2p = &wil->p2p;
+
+	if (!p2p->p2p_dev_started)
+		return;
+
+	wil_dbg_misc(wil, "stop_p2p_device: entered\n");
+	mutex_lock(&wil->mutex);
+	mutex_lock(&wil->p2p_wdev_mutex);
+	wil_p2p_stop_radio_operations(wil);
+	p2p->p2p_dev_started = 0;
+	mutex_unlock(&wil->p2p_wdev_mutex);
+	mutex_unlock(&wil->mutex);
+}
+
 static struct wireless_dev *
 wil_cfg80211_add_iface(struct wiphy *wiphy, const char *name,
 		       unsigned char name_assign_type,
@@ -420,6 +458,7 @@ static int wil_cfg80211_del_iface(struct wiphy *wiphy,
 		return -EINVAL;
 	}
 
+	wil_cfg80211_stop_p2p_device(wiphy, wdev);
 	wil_p2p_wdev_free(wil);
 
 	return 0;
@@ -801,7 +840,7 @@ static int wil_cfg80211_connect(struct wiphy *wiphy,
 		wil->bss = bss;
 		/* Connect can take lots of time */
 		mod_timer(&wil->connect_timer,
-			  jiffies + msecs_to_jiffies(2000));
+			  jiffies + msecs_to_jiffies(5000));
 	} else {
 		clear_bit(wil_status_fwconnecting, wil->status);
 	}
@@ -884,6 +923,9 @@ int wil_cfg80211_mgmt_tx(struct wiphy *wiphy, struct wireless_dev *wdev,
 	wil_hex_dump_misc("mgmt tx frame ", DUMP_PREFIX_OFFSET, 16, 1, buf,
 			  len, true);
 
+	if (len < sizeof(struct ieee80211_hdr_3addr))
+		return -EINVAL;
+
 	cmd = kmalloc(sizeof(*cmd) + len, GFP_KERNEL);
 	if (!cmd) {
 		rc = -ENOMEM;
@@ -1648,34 +1690,6 @@ static int wil_cfg80211_change_bss(struct wiphy *wiphy,
 	return 0;
 }
 
-static int wil_cfg80211_start_p2p_device(struct wiphy *wiphy,
-					 struct wireless_dev *wdev)
-{
-	struct wil6210_priv *wil = wiphy_to_wil(wiphy);
-
-	wil_dbg_misc(wil, "start_p2p_device: entered\n");
-	wil->p2p.p2p_dev_started = 1;
-	return 0;
-}
-
-static void wil_cfg80211_stop_p2p_device(struct wiphy *wiphy,
-					 struct wireless_dev *wdev)
-{
-	struct wil6210_priv *wil = wiphy_to_wil(wiphy);
-	struct wil_p2p_info *p2p = &wil->p2p;
-
-	if (!p2p->p2p_dev_started)
-		return;
-
-	wil_dbg_misc(wil, "stop_p2p_device: entered\n");
-	mutex_lock(&wil->mutex);
-	mutex_lock(&wil->p2p_wdev_mutex);
-	wil_p2p_stop_radio_operations(wil);
-	p2p->p2p_dev_started = 0;
-	mutex_unlock(&wil->p2p_wdev_mutex);
-	mutex_unlock(&wil->mutex);
-}
-
 static int wil_cfg80211_set_power_mgmt(struct wiphy *wiphy,
 				       struct net_device *dev,
 				       bool enabled, int timeout)
@@ -1791,7 +1805,7 @@ static void wil_wiphy_init(struct wiphy *wiphy)
 
 	wiphy->bands[NL80211_BAND_60GHZ] = &wil_band_60ghz;
 
-	/* TODO: figure this out */
+	/* may change after reading FW capabilities */
 	wiphy->signal_type = CFG80211_SIGNAL_TYPE_UNSPEC;
 
 	wiphy->cipher_suites = wil_cipher_suites;
@@ -1801,6 +1815,10 @@ static void wil_wiphy_init(struct wiphy *wiphy)
 
 	wiphy->n_vendor_commands = ARRAY_SIZE(wil_nl80211_vendor_commands);
 	wiphy->vendor_commands = wil_nl80211_vendor_commands;
+
+#ifdef CONFIG_PM
+	wiphy->wowlan = &wil_wowlan_support;
+#endif
 }
 
 struct wireless_dev *wil_cfg80211_init(struct device *dev)
diff --git a/drivers/net/wireless/ath/wil6210/debugfs.c b/drivers/net/wireless/ath/wil6210/debugfs.c
index f82506d276d3..6db00c167d2e 100644
--- a/drivers/net/wireless/ath/wil6210/debugfs.c
+++ b/drivers/net/wireless/ath/wil6210/debugfs.c
@@ -20,7 +20,6 @@
 #include <linux/pci.h>
 #include <linux/rtnetlink.h>
 #include <linux/power_supply.h>
-
 #include "wil6210.h"
 #include "wmi.h"
 #include "txrx.h"
@@ -30,7 +29,6 @@
 static u32 mem_addr;
 static u32 dbg_txdesc_index;
 static u32 dbg_vring_index; /* 24+ for Rx, 0..23 for Tx */
-u32 vring_idle_trsh = 16; /* HW fetches up to 16 descriptors at once */
 
 enum dbg_off_type {
 	doff_u32 = 0,
@@ -801,6 +799,9 @@ static ssize_t wil_write_file_txmgmt(struct file *file, const char __user *buf,
 	int rc;
 	void *frame;
 
+	if (!len)
+		return -EINVAL;
+
 	frame = memdup_user(buf, len);
 	if (IS_ERR(frame))
 		return PTR_ERR(frame);
@@ -1013,6 +1014,7 @@ static int wil_bf_debugfs_show(struct seq_file *s, void *data)
 			   "  TSF = 0x%016llx\n"
 			   "  TxMCS = %2d TxTpt = %4d\n"
 			   "  SQI = %4d\n"
+			   "  RSSI = %4d\n"
 			   "  Status = 0x%08x %s\n"
 			   "  Sectors(rx:tx) my %2d:%2d peer %2d:%2d\n"
 			   "  Goodput(rx:tx) %4d:%4d\n"
@@ -1022,6 +1024,7 @@ static int wil_bf_debugfs_show(struct seq_file *s, void *data)
 			   le16_to_cpu(reply.evt.bf_mcs),
 			   le32_to_cpu(reply.evt.tx_tpt),
 			   reply.evt.sqi,
+			   reply.evt.rssi,
 			   status, wil_bfstatus_str(status),
 			   le16_to_cpu(reply.evt.my_rx_sector),
 			   le16_to_cpu(reply.evt.my_tx_sector),
@@ -1612,6 +1615,8 @@ static ssize_t wil_write_suspend_stats(struct file *file,
 	struct wil6210_priv *wil = file->private_data;
 
 	memset(&wil->suspend_stats, 0, sizeof(wil->suspend_stats));
+	wil->suspend_stats.min_suspend_time = ULONG_MAX;
+	wil->suspend_stats.collection_start = ktime_get();
 
 	return len;
 }
@@ -1623,18 +1628,27 @@ static ssize_t wil_read_suspend_stats(struct file *file,
 	struct wil6210_priv *wil = file->private_data;
 	static char text[400];
 	int n;
+	unsigned long long stats_collection_time =
+		ktime_to_us(ktime_sub(ktime_get(),
+				      wil->suspend_stats.collection_start));
 
 	n = snprintf(text, sizeof(text),
 		     "Suspend statistics:\n"
 		     "successful suspends:%ld failed suspends:%ld\n"
 		     "successful resumes:%ld failed resumes:%ld\n"
-		     "rejected by host:%ld rejected by device:%ld\n",
+		     "rejected by host:%ld rejected by device:%ld\n"
+		     "total suspend time:%lld min suspend time:%lld\n"
+		     "max suspend time:%lld stats collection time: %lld\n",
 		     wil->suspend_stats.successful_suspends,
 		     wil->suspend_stats.failed_suspends,
 		     wil->suspend_stats.successful_resumes,
 		     wil->suspend_stats.failed_resumes,
 		     wil->suspend_stats.rejected_by_host,
-		     wil->suspend_stats.rejected_by_device);
+		     wil->suspend_stats.rejected_by_device,
+		     wil->suspend_stats.total_suspend_time,
+		     wil->suspend_stats.min_suspend_time,
+		     wil->suspend_stats.max_suspend_time,
+		     stats_collection_time);
 
 	n = min_t(int, n, sizeof(text));
 
@@ -1747,6 +1761,7 @@ static const struct dbg_off dbg_wil_off[] = {
 	WIL_FIELD(chip_revision, 0444,	doff_u8),
 	WIL_FIELD(abft_len, 0644,		doff_u8),
 	WIL_FIELD(wakeup_trigger, 0644,		doff_u8),
+	WIL_FIELD(vring_idle_trsh, 0644,	doff_u32),
 	{},
 };
 
@@ -1762,8 +1777,6 @@ static const struct dbg_off dbg_statics[] = {
 	{"desc_index",	0644, (ulong)&dbg_txdesc_index, doff_u32},
 	{"vring_index",	0644, (ulong)&dbg_vring_index, doff_u32},
 	{"mem_addr",	0644, (ulong)&mem_addr, doff_u32},
-	{"vring_idle_trsh", 0644, (ulong)&vring_idle_trsh,
-	 doff_u32},
 	{"led_polarity", 0644, (ulong)&led_polarity, doff_u8},
 	{},
 };
@@ -1790,6 +1803,8 @@ int wil6210_debugfs_init(struct wil6210_priv *wil)
 
 	wil6210_debugfs_create_ITR_CNT(wil, dbg);
 
+	wil->suspend_stats.collection_start = ktime_get();
+
 	return 0;
 }
 
diff --git a/drivers/net/wireless/ath/wil6210/interrupt.c b/drivers/net/wireless/ath/wil6210/interrupt.c
index cad8a95c4e4e..59def4f3fcf3 100644
--- a/drivers/net/wireless/ath/wil6210/interrupt.c
+++ b/drivers/net/wireless/ath/wil6210/interrupt.c
@@ -244,7 +244,7 @@ static irqreturn_t wil6210_irq_rx(int irq, void *cookie)
 	wil_dbg_irq(wil, "ISR RX 0x%08x\n", isr);
 
 	if (unlikely(!isr)) {
-		wil_err(wil, "spurious IRQ: RX\n");
+		wil_err_ratelimited(wil, "spurious IRQ: RX\n");
 		return IRQ_NONE;
 	}
 
@@ -269,11 +269,12 @@ static irqreturn_t wil6210_irq_rx(int irq, void *cookie)
 				need_unmask = false;
 				napi_schedule(&wil->napi_rx);
 			} else {
-				wil_err(wil,
+				wil_err_ratelimited(
+					wil,
 					"Got Rx interrupt while stopping interface\n");
 			}
 		} else {
-			wil_err(wil, "Got Rx interrupt while in reset\n");
+			wil_err_ratelimited(wil, "Got Rx interrupt while in reset\n");
 		}
 	}
 
@@ -302,7 +303,7 @@ static irqreturn_t wil6210_irq_tx(int irq, void *cookie)
 	wil_dbg_irq(wil, "ISR TX 0x%08x\n", isr);
 
 	if (unlikely(!isr)) {
-		wil_err(wil, "spurious IRQ: TX\n");
+		wil_err_ratelimited(wil, "spurious IRQ: TX\n");
 		return IRQ_NONE;
 	}
 
@@ -318,12 +319,13 @@ static irqreturn_t wil6210_irq_tx(int irq, void *cookie)
 			need_unmask = false;
 			napi_schedule(&wil->napi_tx);
 		} else {
-			wil_err(wil, "Got Tx interrupt while in reset\n");
+			wil_err_ratelimited(wil, "Got Tx interrupt while in reset\n");
 		}
 	}
 
 	if (unlikely(isr))
-		wil_err(wil, "un-handled TX ISR bits 0x%08x\n", isr);
+		wil_err_ratelimited(wil, "un-handled TX ISR bits 0x%08x\n",
+				    isr);
 
 	/* Tx IRQ will be enabled when NAPI processing finished */
 
diff --git a/drivers/net/wireless/ath/wil6210/main.c b/drivers/net/wireless/ath/wil6210/main.c
index daf944a71901..bac829aa950d 100644
--- a/drivers/net/wireless/ath/wil6210/main.c
+++ b/drivers/net/wireless/ath/wil6210/main.c
@@ -394,10 +394,11 @@ static void wil_fw_error_worker(struct work_struct *work)
 	struct wil6210_priv *wil = container_of(work, struct wil6210_priv,
 						fw_error_worker);
 	struct wireless_dev *wdev = wil->wdev;
+	struct net_device *ndev = wil_to_ndev(wil);
 
 	wil_dbg_misc(wil, "fw error worker\n");
 
-	if (!netif_running(wil_to_ndev(wil))) {
+	if (!(ndev->flags & IFF_UP)) {
 		wil_info(wil, "No recovery - interface is down\n");
 		return;
 	}
@@ -578,6 +579,9 @@ int wil_priv_init(struct wil6210_priv *wil)
 
 	wil->wakeup_trigger = WMI_WAKEUP_TRIGGER_UCAST |
 			      WMI_WAKEUP_TRIGGER_BCAST;
+	memset(&wil->suspend_stats, 0, sizeof(wil->suspend_stats));
+	wil->suspend_stats.min_suspend_time = ULONG_MAX;
+	wil->vring_idle_trsh = 16;
 
 	return 0;
 
@@ -926,6 +930,29 @@ int wil_ps_update(struct wil6210_priv *wil, enum wmi_ps_profile_type ps_profile)
 	return rc;
 }
 
+static void wil_pre_fw_config(struct wil6210_priv *wil)
+{
+	/* Mark FW as loaded from host */
+	wil_s(wil, RGF_USER_USAGE_6, 1);
+
+	/* clear any interrupts which on-card-firmware
+	 * may have set
+	 */
+	wil6210_clear_irq(wil);
+	/* CAF_ICR - clear and mask */
+	/* it is W1C, clear by writing back same value */
+	wil_s(wil, RGF_CAF_ICR + offsetof(struct RGF_ICR, ICR), 0);
+	wil_w(wil, RGF_CAF_ICR + offsetof(struct RGF_ICR, IMV), ~0);
+	/* clear PAL_UNIT_ICR (potential D0->D3 leftover) */
+	wil_s(wil, RGF_PAL_UNIT_ICR + offsetof(struct RGF_ICR, ICR), 0);
+
+	if (wil->fw_calib_result > 0) {
+		__le32 val = cpu_to_le32(wil->fw_calib_result |
+						(CALIB_RESULT_SIGNATURE << 8));
+		wil_w(wil, RGF_USER_FW_CALIB_RESULT, (u32 __force)val);
+	}
+}
+
 /*
  * We reset all the structures, and we reset the UMAC.
  * After calling this routine, you're expected to reload
@@ -1019,18 +1046,7 @@ int wil_reset(struct wil6210_priv *wil, bool load_fw)
 		if (rc)
 			return rc;
 
-		/* Mark FW as loaded from host */
-		wil_s(wil, RGF_USER_USAGE_6, 1);
-
-		/* clear any interrupts which on-card-firmware
-		 * may have set
-		 */
-		wil6210_clear_irq(wil);
-		/* CAF_ICR - clear and mask */
-		/* it is W1C, clear by writing back same value */
-		wil_s(wil, RGF_CAF_ICR + offsetof(struct RGF_ICR, ICR), 0);
-		wil_w(wil, RGF_CAF_ICR + offsetof(struct RGF_ICR, IMV), ~0);
-
+		wil_pre_fw_config(wil);
 		wil_release_cpu(wil);
 	}
 
diff --git a/drivers/net/wireless/ath/wil6210/pcie_bus.c b/drivers/net/wireless/ath/wil6210/pcie_bus.c
index d571feb2370e..6a3ab4bf916d 100644
--- a/drivers/net/wireless/ath/wil6210/pcie_bus.c
+++ b/drivers/net/wireless/ath/wil6210/pcie_bus.c
@@ -84,6 +84,9 @@ void wil_set_capabilities(struct wil6210_priv *wil)
 
 	/* extract FW capabilities from file without loading the FW */
 	wil_request_firmware(wil, wil->wil_fw_name, false);
+
+	if (test_bit(WMI_FW_CAPABILITY_RSSI_REPORTING, wil->fw_capabilities))
+		wil_to_wiphy(wil)->signal_type = CFG80211_SIGNAL_TYPE_MBM;
 }
 
 void wil_disable_irq(struct wil6210_priv *wil)
diff --git a/drivers/net/wireless/ath/wil6210/pm.c b/drivers/net/wireless/ath/wil6210/pm.c
index ce1f384e7f8e..8f5d1b447aaa 100644
--- a/drivers/net/wireless/ath/wil6210/pm.c
+++ b/drivers/net/wireless/ath/wil6210/pm.c
@@ -21,10 +21,11 @@ int wil_can_suspend(struct wil6210_priv *wil, bool is_runtime)
 {
 	int rc = 0;
 	struct wireless_dev *wdev = wil->wdev;
+	struct net_device *ndev = wil_to_ndev(wil);
 
 	wil_dbg_pm(wil, "can_suspend: %s\n", is_runtime ? "runtime" : "system");
 
-	if (!netif_running(wil_to_ndev(wil))) {
+	if (!(ndev->flags & IFF_UP)) {
 		/* can always sleep when down */
 		wil_dbg_pm(wil, "Interface is down\n");
 		goto out;
@@ -85,7 +86,9 @@ static int wil_resume_keep_radio_on(struct wil6210_priv *wil)
 	/* Send WMI resume request to the device */
 	rc = wmi_resume(wil);
 	if (rc) {
-		wil_err(wil, "device failed to resume (%d), resetting\n", rc);
+		wil_err(wil, "device failed to resume (%d)\n", rc);
+		if (no_fw_recovery)
+			goto out;
 		rc = wil_down(wil);
 		if (rc) {
 			wil_err(wil, "wil_down failed (%d)\n", rc);
@@ -298,6 +301,9 @@ int wil_suspend(struct wil6210_priv *wil, bool is_runtime)
 	wil_dbg_pm(wil, "suspend: %s => %d\n",
 		   is_runtime ? "runtime" : "system", rc);
 
+	if (!rc)
+		wil->suspend_stats.suspend_start_time = ktime_get();
+
 	return rc;
 }
 
@@ -307,6 +313,7 @@ int wil_resume(struct wil6210_priv *wil, bool is_runtime)
 	struct net_device *ndev = wil_to_ndev(wil);
 	bool keep_radio_on = ndev->flags & IFF_UP &&
 			     wil->keep_radio_on_during_sleep;
+	unsigned long long suspend_time_usec = 0;
 
 	wil_dbg_pm(wil, "resume: %s\n", is_runtime ? "runtime" : "system");
 
@@ -324,8 +331,20 @@ int wil_resume(struct wil6210_priv *wil, bool is_runtime)
 	else
 		rc = wil_resume_radio_off(wil);
 
+	if (rc)
+		goto out;
+
+	suspend_time_usec =
+		ktime_to_us(ktime_sub(ktime_get(),
+				      wil->suspend_stats.suspend_start_time));
+	wil->suspend_stats.total_suspend_time += suspend_time_usec;
+	if (suspend_time_usec < wil->suspend_stats.min_suspend_time)
+		wil->suspend_stats.min_suspend_time = suspend_time_usec;
+	if (suspend_time_usec > wil->suspend_stats.max_suspend_time)
+		wil->suspend_stats.max_suspend_time = suspend_time_usec;
+
 out:
-	wil_dbg_pm(wil, "resume: %s => %d\n",
-		   is_runtime ? "runtime" : "system", rc);
+	wil_dbg_pm(wil, "resume: %s => %d, suspend time %lld usec\n",
+		   is_runtime ? "runtime" : "system", rc, suspend_time_usec);
 	return rc;
 }
diff --git a/drivers/net/wireless/ath/wil6210/txrx.c b/drivers/net/wireless/ath/wil6210/txrx.c
index ec57bcce9601..389c718cd257 100644
--- a/drivers/net/wireless/ath/wil6210/txrx.c
+++ b/drivers/net/wireless/ath/wil6210/txrx.c
@@ -1666,7 +1666,7 @@ static int __wil_tx_vring_tso(struct wil6210_priv *wil, struct vring *vring,
 
 	/* performance monitoring */
 	used = wil_vring_used_tx(vring);
-	if (wil_val_in_range(vring_idle_trsh,
+	if (wil_val_in_range(wil->vring_idle_trsh,
 			     used, used + descs_used)) {
 		txdata->idle += get_cycles() - txdata->last_idle;
 		wil_dbg_txrx(wil,  "Ring[%2d] not idle %d -> %d\n",
@@ -1813,7 +1813,7 @@ static int __wil_tx_vring(struct wil6210_priv *wil, struct vring *vring,
 
 	/* performance monitoring */
 	used = wil_vring_used_tx(vring);
-	if (wil_val_in_range(vring_idle_trsh,
+	if (wil_val_in_range(wil->vring_idle_trsh,
 			     used, used + nr_frags + 1)) {
 		txdata->idle += get_cycles() - txdata->last_idle;
 		wil_dbg_txrx(wil,  "Ring[%2d] not idle %d -> %d\n",
@@ -2175,7 +2175,7 @@ int wil_tx_complete(struct wil6210_priv *wil, int ringid)
 
 	/* performance monitoring */
 	used_new = wil_vring_used_tx(vring);
-	if (wil_val_in_range(vring_idle_trsh,
+	if (wil_val_in_range(wil->vring_idle_trsh,
 			     used_new, used_before_complete)) {
 		wil_dbg_txrx(wil, "Ring[%2d] idle %d -> %d\n",
 			     ringid, used_before_complete, used_new);
diff --git a/drivers/net/wireless/ath/wil6210/wil6210.h b/drivers/net/wireless/ath/wil6210/wil6210.h
index d085ccfc7228..315ec8b59662 100644
--- a/drivers/net/wireless/ath/wil6210/wil6210.h
+++ b/drivers/net/wireless/ath/wil6210/wil6210.h
@@ -30,7 +30,6 @@ extern bool no_fw_recovery;
 extern unsigned int mtu_max;
 extern unsigned short rx_ring_overflow_thrsh;
 extern int agg_wsize;
-extern u32 vring_idle_trsh;
 extern bool rx_align_2;
 extern bool rx_large_buf;
 extern bool debug_fw;
@@ -90,6 +89,11 @@ struct wil_suspend_stats {
 	unsigned long failed_resumes;
 	unsigned long rejected_by_device;
 	unsigned long rejected_by_host;
+	unsigned long long total_suspend_time;
+	unsigned long long min_suspend_time;
+	unsigned long long max_suspend_time;
+	ktime_t collection_start;
+	ktime_t suspend_start_time;
 };
 
 /* Calculate MAC buffer size for the firmware. It includes all overhead,
@@ -166,6 +170,10 @@ struct RGF_ICR {
 #define RGF_USER_USER_SCRATCH_PAD	(0x8802bc)
 #define RGF_USER_BL			(0x880A3C) /* Boot Loader */
 #define RGF_USER_FW_REV_ID		(0x880a8c) /* chip revision */
+#define RGF_USER_FW_CALIB_RESULT	(0x880a90) /* b0-7:result
+						    * b8-15:signature
+						    */
+	#define CALIB_RESULT_SIGNATURE	(0x11)
 #define RGF_USER_CLKS_CTL_0		(0x880abc)
 	#define BIT_USER_CLKS_CAR_AHB_SW_SEL	BIT(1) /* ref clk/PLL */
 	#define BIT_USER_CLKS_RST_PWGD	BIT(11) /* reset on "power good" */
@@ -260,6 +268,7 @@ struct RGF_ICR {
 	#define BIT_DMA_PSEUDO_CAUSE_MISC	BIT(2)
 
 #define RGF_HP_CTRL			(0x88265c)
+#define RGF_PAL_UNIT_ICR		(0x88266c) /* struct RGF_ICR */
 #define RGF_PCIE_LOS_COUNTER_CTL	(0x882dc4)
 
 /* MAC timer, usec, for packet lifetime */
@@ -684,6 +693,7 @@ struct wil6210_priv {
 	u8 vring2cid_tid[WIL6210_MAX_TX_RINGS][2]; /* [0] - CID, [1] - TID */
 	struct wil_sta_info sta[WIL6210_MAX_CID];
 	int bcast_vring;
+	u32 vring_idle_trsh; /* HW fetches up to 16 descriptors at once  */
 	bool use_extended_dma_addr; /* indicates whether we are using 48 bits */
 	/* scan */
 	struct cfg80211_scan_request *scan_request;
@@ -719,6 +729,8 @@ struct wil6210_priv {
 
 	enum wmi_ps_profile_type ps_profile;
 
+	int fw_calib_result;
+
 #ifdef CONFIG_PM
 #ifdef CONFIG_PM_SLEEP
 	struct notifier_block pm_notify;
@@ -929,8 +941,14 @@ int wil_cfg80211_mgmt_tx(struct wiphy *wiphy, struct wireless_dev *wdev,
 			 struct cfg80211_mgmt_tx_params *params,
 			 u64 *cookie);
 
+#if defined(CONFIG_WIL6210_DEBUGFS)
 int wil6210_debugfs_init(struct wil6210_priv *wil);
 void wil6210_debugfs_remove(struct wil6210_priv *wil);
+#else
+static inline int wil6210_debugfs_init(struct wil6210_priv *wil) { return 0; }
+static inline void wil6210_debugfs_remove(struct wil6210_priv *wil) {}
+#endif
+
 int wil_cid_fill_sinfo(struct wil6210_priv *wil, int cid,
 		       struct station_info *sinfo);
 
diff --git a/drivers/net/wireless/ath/wil6210/wmi.c b/drivers/net/wireless/ath/wil6210/wmi.c
index 65ef67321fc0..ffdd2fa401b1 100644
--- a/drivers/net/wireless/ath/wil6210/wmi.c
+++ b/drivers/net/wireless/ath/wil6210/wmi.c
@@ -344,6 +344,11 @@ static void wmi_evt_ready(struct wil6210_priv *wil, int id, void *d, int len)
 	strlcpy(wdev->wiphy->fw_version, wil->fw_version,
 		sizeof(wdev->wiphy->fw_version));
 
+	if (len > offsetof(struct wmi_ready_event, rfc_read_calib_result)) {
+		wil_dbg_wmi(wil, "rfc calibration result %d\n",
+			    evt->rfc_read_calib_result);
+		wil->fw_calib_result = evt->rfc_read_calib_result;
+	}
 	wil_set_recovery_state(wil, fw_recovery_idle);
 	set_bit(wil_status_fwready, wil->status);
 	/* let the reset sequence continue */
@@ -381,12 +386,15 @@ static void wmi_evt_rx_mgmt(struct wil6210_priv *wil, int id, void *d, int len)
 	ch_no = data->info.channel + 1;
 	freq = ieee80211_channel_to_frequency(ch_no, NL80211_BAND_60GHZ);
 	channel = ieee80211_get_channel(wiphy, freq);
-	signal = data->info.sqi;
+	if (test_bit(WMI_FW_CAPABILITY_RSSI_REPORTING, wil->fw_capabilities))
+		signal = 100 * data->info.rssi;
+	else
+		signal = data->info.sqi;
 	d_status = le16_to_cpu(data->info.status);
 	fc = rx_mgmt_frame->frame_control;
 
-	wil_dbg_wmi(wil, "MGMT Rx: channel %d MCS %d SNR %d SQI %d%%\n",
-		    data->info.channel, data->info.mcs, data->info.snr,
+	wil_dbg_wmi(wil, "MGMT Rx: channel %d MCS %d RSSI %d SQI %d%%\n",
+		    data->info.channel, data->info.mcs, data->info.rssi,
 		    data->info.sqi);
 	wil_dbg_wmi(wil, "status 0x%04x len %d fc 0x%04x\n", d_status, d_len,
 		    le16_to_cpu(fc));
diff --git a/drivers/net/wireless/ath/wil6210/wmi.h b/drivers/net/wireless/ath/wil6210/wmi.h
index 256f63c57da0..5263ee717a4f 100644
--- a/drivers/net/wireless/ath/wil6210/wmi.h
+++ b/drivers/net/wireless/ath/wil6210/wmi.h
@@ -36,6 +36,11 @@
 #define WMI_PROX_RANGE_NUM		(3)
 #define WMI_MAX_LOSS_DMG_BEACONS	(20)
 #define MAX_NUM_OF_SECTORS		(128)
+#define WMI_SCHED_MAX_ALLOCS_PER_CMD	(4)
+#define WMI_RF_DTYPE_LENGTH		(3)
+#define WMI_RF_ETYPE_LENGTH		(3)
+#define WMI_RF_RX2TX_LENGTH		(3)
+#define WMI_RF_ETYPE_VAL_PER_RANGE	(5)
 
 /* Mailbox interface
  * used for commands and events
@@ -52,14 +57,20 @@ enum wmi_mid {
  * the host
  */
 enum wmi_fw_capability {
-	WMI_FW_CAPABILITY_FTM			= 0,
-	WMI_FW_CAPABILITY_PS_CONFIG		= 1,
-	WMI_FW_CAPABILITY_RF_SECTORS		= 2,
-	WMI_FW_CAPABILITY_MGMT_RETRY_LIMIT	= 3,
-	WMI_FW_CAPABILITY_DISABLE_AP_SME	= 4,
-	WMI_FW_CAPABILITY_WMI_ONLY		= 5,
-	WMI_FW_CAPABILITY_THERMAL_THROTTLING	= 7,
-	WMI_FW_CAPABILITY_D3_SUSPEND		= 8,
+	WMI_FW_CAPABILITY_FTM				= 0,
+	WMI_FW_CAPABILITY_PS_CONFIG			= 1,
+	WMI_FW_CAPABILITY_RF_SECTORS			= 2,
+	WMI_FW_CAPABILITY_MGMT_RETRY_LIMIT		= 3,
+	WMI_FW_CAPABILITY_DISABLE_AP_SME		= 4,
+	WMI_FW_CAPABILITY_WMI_ONLY			= 5,
+	WMI_FW_CAPABILITY_THERMAL_THROTTLING		= 7,
+	WMI_FW_CAPABILITY_D3_SUSPEND			= 8,
+	WMI_FW_CAPABILITY_LONG_RANGE			= 9,
+	WMI_FW_CAPABILITY_FIXED_SCHEDULING		= 10,
+	WMI_FW_CAPABILITY_MULTI_DIRECTED_OMNIS		= 11,
+	WMI_FW_CAPABILITY_RSSI_REPORTING		= 12,
+	WMI_FW_CAPABILITY_SET_SILENT_RSSI_TABLE		= 13,
+	WMI_FW_CAPABILITY_LO_POWER_CALIB_FROM_OTP	= 14,
 	WMI_FW_CAPABILITY_MAX,
 };
 
@@ -79,6 +90,7 @@ enum wmi_command_id {
 	WMI_START_SCAN_CMDID				= 0x07,
 	WMI_SET_BSS_FILTER_CMDID			= 0x09,
 	WMI_SET_PROBED_SSID_CMDID			= 0x0A,
+	/* deprecated */
 	WMI_SET_LISTEN_INT_CMDID			= 0x0B,
 	WMI_BCON_CTRL_CMDID				= 0x0F,
 	WMI_ADD_CIPHER_KEY_CMDID			= 0x16,
@@ -93,26 +105,28 @@ enum wmi_command_id {
 	WMI_ECHO_CMDID					= 0x803,
 	WMI_DEEP_ECHO_CMDID				= 0x804,
 	WMI_CONFIG_MAC_CMDID				= 0x805,
+	/* deprecated */
 	WMI_CONFIG_PHY_DEBUG_CMDID			= 0x806,
 	WMI_ADD_DEBUG_TX_PCKT_CMDID			= 0x808,
 	WMI_PHY_GET_STATISTICS_CMDID			= 0x809,
+	/* deprecated */
 	WMI_FS_TUNE_CMDID				= 0x80A,
+	/* deprecated */
 	WMI_CORR_MEASURE_CMDID				= 0x80B,
 	WMI_READ_RSSI_CMDID				= 0x80C,
 	WMI_TEMP_SENSE_CMDID				= 0x80E,
 	WMI_DC_CALIB_CMDID				= 0x80F,
+	/* deprecated */
 	WMI_SEND_TONE_CMDID				= 0x810,
+	/* deprecated */
 	WMI_IQ_TX_CALIB_CMDID				= 0x811,
+	/* deprecated */
 	WMI_IQ_RX_CALIB_CMDID				= 0x812,
-	WMI_SET_UCODE_IDLE_CMDID			= 0x813,
 	WMI_SET_WORK_MODE_CMDID				= 0x815,
 	WMI_LO_LEAKAGE_CALIB_CMDID			= 0x816,
-	WMI_MARLON_R_READ_CMDID				= 0x818,
-	WMI_MARLON_R_WRITE_CMDID			= 0x819,
-	WMI_MARLON_R_TXRX_SEL_CMDID			= 0x81A,
-	MAC_IO_STATIC_PARAMS_CMDID			= 0x81B,
-	MAC_IO_DYNAMIC_PARAMS_CMDID			= 0x81C,
+	WMI_LO_POWER_CALIB_FROM_OTP_CMDID		= 0x817,
 	WMI_SILENT_RSSI_CALIB_CMDID			= 0x81D,
+	/* deprecated */
 	WMI_RF_RX_TEST_CMDID				= 0x81E,
 	WMI_CFG_RX_CHAIN_CMDID				= 0x820,
 	WMI_VRING_CFG_CMDID				= 0x821,
@@ -126,11 +140,6 @@ enum wmi_command_id {
 	WMI_SET_PCP_CHANNEL_CMDID			= 0x829,
 	WMI_GET_PCP_CHANNEL_CMDID			= 0x82A,
 	WMI_SW_TX_REQ_CMDID				= 0x82B,
-	WMI_READ_MAC_RXQ_CMDID				= 0x830,
-	WMI_READ_MAC_TXQ_CMDID				= 0x831,
-	WMI_WRITE_MAC_RXQ_CMDID				= 0x832,
-	WMI_WRITE_MAC_TXQ_CMDID				= 0x833,
-	WMI_WRITE_MAC_XQ_FIELD_CMDID			= 0x834,
 	WMI_MLME_PUSH_CMDID				= 0x835,
 	WMI_BEAMFORMING_MGMT_CMDID			= 0x836,
 	WMI_BF_TXSS_MGMT_CMDID				= 0x837,
@@ -144,9 +153,13 @@ enum wmi_command_id {
 	WMI_MAINTAIN_RESUME_CMDID			= 0x851,
 	WMI_RS_MGMT_CMDID				= 0x852,
 	WMI_RF_MGMT_CMDID				= 0x853,
-	WMI_OTP_READ_CMDID				= 0x856,
-	WMI_OTP_WRITE_CMDID				= 0x857,
+	WMI_RF_XPM_READ_CMDID				= 0x856,
+	WMI_RF_XPM_WRITE_CMDID				= 0x857,
 	WMI_LED_CFG_CMDID				= 0x858,
+	WMI_SET_CONNECT_SNR_THR_CMDID			= 0x85B,
+	WMI_SET_ACTIVE_SILENT_RSSI_TABLE_CMDID		= 0x85C,
+	WMI_RF_PWR_ON_DELAY_CMDID			= 0x85D,
+	WMI_SET_HIGH_POWER_TABLE_PARAMS_CMDID		= 0x85E,
 	/* Performance monitoring commands */
 	WMI_BF_CTRL_CMDID				= 0x862,
 	WMI_NOTIFY_REQ_CMDID				= 0x863,
@@ -154,7 +167,6 @@ enum wmi_command_id {
 	WMI_GET_RF_STATUS_CMDID				= 0x866,
 	WMI_GET_BASEBAND_TYPE_CMDID			= 0x867,
 	WMI_UNIT_TEST_CMDID				= 0x900,
-	WMI_HICCUP_CMDID				= 0x901,
 	WMI_FLASH_READ_CMDID				= 0x902,
 	WMI_FLASH_WRITE_CMDID				= 0x903,
 	/* Power management */
@@ -174,16 +186,6 @@ enum wmi_command_id {
 	WMI_GET_PCP_FACTOR_CMDID			= 0x91B,
 	/* Power Save Configuration Commands */
 	WMI_PS_DEV_PROFILE_CFG_CMDID			= 0x91C,
-	/* Not supported yet */
-	WMI_PS_DEV_CFG_CMDID				= 0x91D,
-	/* Not supported yet */
-	WMI_PS_DEV_CFG_READ_CMDID			= 0x91E,
-	/* Per MAC Power Save Configuration commands
-	 * Not supported yet
-	 */
-	WMI_PS_MID_CFG_CMDID				= 0x91F,
-	/* Not supported yet */
-	WMI_PS_MID_CFG_READ_CMDID			= 0x920,
 	WMI_RS_CFG_CMDID				= 0x921,
 	WMI_GET_DETAILED_RS_RES_CMDID			= 0x922,
 	WMI_AOA_MEAS_CMDID				= 0x923,
@@ -194,13 +196,16 @@ enum wmi_command_id {
 	WMI_DEL_STA_CMDID				= 0x936,
 	WMI_SET_THERMAL_THROTTLING_CFG_CMDID		= 0x940,
 	WMI_GET_THERMAL_THROTTLING_CFG_CMDID		= 0x941,
+	/* Read Power Save profile type */
+	WMI_PS_DEV_PROFILE_CFG_READ_CMDID		= 0x942,
 	WMI_TOF_SESSION_START_CMDID			= 0x991,
 	WMI_TOF_GET_CAPABILITIES_CMDID			= 0x992,
 	WMI_TOF_SET_LCR_CMDID				= 0x993,
 	WMI_TOF_SET_LCI_CMDID				= 0x994,
-	WMI_TOF_CHANNEL_INFO_CMDID			= 0x995,
+	WMI_TOF_CFG_RESPONDER_CMDID			= 0x996,
 	WMI_TOF_SET_TX_RX_OFFSET_CMDID			= 0x997,
 	WMI_TOF_GET_TX_RX_OFFSET_CMDID			= 0x998,
+	WMI_TOF_CHANNEL_INFO_CMDID			= 0x999,
 	WMI_GET_RF_SECTOR_PARAMS_CMDID			= 0x9A0,
 	WMI_SET_RF_SECTOR_PARAMS_CMDID			= 0x9A1,
 	WMI_GET_SELECTED_RF_SECTOR_INDEX_CMDID		= 0x9A2,
@@ -209,12 +214,20 @@ enum wmi_command_id {
 	WMI_PRIO_TX_SECTORS_ORDER_CMDID			= 0x9A5,
 	WMI_PRIO_TX_SECTORS_NUMBER_CMDID		= 0x9A6,
 	WMI_PRIO_TX_SECTORS_SET_DEFAULT_CFG_CMDID	= 0x9A7,
+	WMI_SCHEDULING_SCHEME_CMDID			= 0xA01,
+	WMI_FIXED_SCHEDULING_CONFIG_CMDID		= 0xA02,
+	WMI_ENABLE_FIXED_SCHEDULING_CMDID		= 0xA03,
+	WMI_SET_MULTI_DIRECTED_OMNIS_CONFIG_CMDID	= 0xA04,
+	WMI_SET_LONG_RANGE_CONFIG_CMDID			= 0xA05,
 	WMI_SET_MAC_ADDRESS_CMDID			= 0xF003,
 	WMI_ABORT_SCAN_CMDID				= 0xF007,
 	WMI_SET_PROMISCUOUS_MODE_CMDID			= 0xF041,
+	/* deprecated */
 	WMI_GET_PMK_CMDID				= 0xF048,
 	WMI_SET_PASSPHRASE_CMDID			= 0xF049,
+	/* deprecated */
 	WMI_SEND_ASSOC_RES_CMDID			= 0xF04A,
+	/* deprecated */
 	WMI_SET_ASSOC_REQ_RELAY_CMDID			= 0xF04B,
 	WMI_MAC_ADDR_REQ_CMDID				= 0xF04D,
 	WMI_FW_VER_CMDID				= 0xF04E,
@@ -440,11 +453,6 @@ struct wmi_rf_mgmt_cmd {
 	__le32 rf_mgmt_type;
 } __packed;
 
-/* WMI_RF_RX_TEST_CMDID */
-struct wmi_rf_rx_test_cmd {
-	__le32 sector;
-} __packed;
-
 /* WMI_CORR_MEASURE_CMDID */
 struct wmi_corr_measure_cmd {
 	__le32 freq_mhz;
@@ -657,6 +665,20 @@ struct wmi_bcast_vring_cfg_cmd {
 	struct wmi_bcast_vring_cfg vring_cfg;
 } __packed;
 
+/* WMI_LO_POWER_CALIB_FROM_OTP_CMDID */
+struct wmi_lo_power_calib_from_otp_cmd {
+	/* index to read from OTP. zero based */
+	u8 index;
+	u8 reserved[3];
+} __packed;
+
+/* WMI_LO_POWER_CALIB_FROM_OTP_EVENTID */
+struct wmi_lo_power_calib_from_otp_event {
+	/* wmi_fw_status */
+	u8 status;
+	u8 reserved[3];
+} __packed;
+
 /* WMI_VRING_BA_EN_CMDID */
 struct wmi_vring_ba_en_cmd {
 	u8 ringid;
@@ -692,6 +714,24 @@ enum wmi_sniffer_cfg_mode {
 	WMI_SNIFFER_ON	= 0x01,
 };
 
+/* WMI_SILENT_RSSI_TABLE */
+enum wmi_silent_rssi_table {
+	RF_TEMPERATURE_CALIB_DEFAULT_DB		= 0x00,
+	RF_TEMPERATURE_CALIB_HIGH_POWER_DB	= 0x01,
+};
+
+/* WMI_SILENT_RSSI_STATUS */
+enum wmi_silent_rssi_status {
+	SILENT_RSSI_SUCCESS	= 0x00,
+	SILENT_RSSI_FAILURE	= 0x01,
+};
+
+/* WMI_SET_ACTIVE_SILENT_RSSI_TABLE_CMDID */
+struct wmi_set_active_silent_rssi_table_cmd {
+	/* enum wmi_silent_rssi_table */
+	__le32 table;
+} __packed;
+
 enum wmi_sniffer_cfg_phy_info_mode {
 	WMI_SNIFFER_PHY_INFO_DISABLED	= 0x00,
 	WMI_SNIFFER_PHY_INFO_ENABLED	= 0x01,
@@ -835,18 +875,85 @@ struct wmi_echo_cmd {
 	__le32 value;
 } __packed;
 
-/* WMI_OTP_READ_CMDID */
-struct wmi_otp_read_cmd {
-	__le32 addr;
-	__le32 size;
-	__le32 values;
+/* WMI_RF_PWR_ON_DELAY_CMDID
+ * set FW time parameters used through RF resetting
+ *  RF reset consists of bringing its power down for a period of time, then
+ * bringing the power up
+ * Returned event: WMI_RF_PWR_ON_DELAY_RSP_EVENTID
+ */
+struct wmi_rf_pwr_on_delay_cmd {
+	/* time in usec the FW waits after bringing the RF PWR down,
+	 * set 0 for default
+	 */
+	__le16 down_delay_usec;
+	/* time in usec the FW waits after bringing the RF PWR up,
+	 * set 0 for default
+	 */
+	__le16 up_delay_usec;
+} __packed;
+
+/* \WMI_SET_HIGH_POWER_TABLE_PARAMS_CMDID
+ * This API controls the Tx and Rx gain over temperature.
+ * It controls the Tx D-type, Rx D-type and Rx E-type amplifiers.
+ * It also controls the Tx gain index, by controlling the Rx to Tx gain index
+ * offset.
+ * The control is divided by 3 temperature values to 4 temperature ranges.
+ * Each parameter uses its own temperature values.
+ * Returned event: WMI_SET_HIGH_POWER_TABLE_PARAMS_EVENTID
+ */
+struct wmi_set_high_power_table_params_cmd {
+	/* Temperature range for Tx D-type parameters */
+	u8 tx_dtype_temp[WMI_RF_DTYPE_LENGTH];
+	u8 reserved0;
+	/* Tx D-type values to be used for each temperature range */
+	__le32 tx_dtype_conf[WMI_RF_DTYPE_LENGTH + 1];
+	/* Temperature range for Rx D-type parameters */
+	u8 rx_dtype_temp[WMI_RF_DTYPE_LENGTH];
+	u8 reserved1;
+	/* Rx D-type values to be used for each temperature range */
+	__le32 rx_dtype_conf[WMI_RF_DTYPE_LENGTH + 1];
+	/* Temperature range for Rx E-type parameters */
+	u8 rx_etype_temp[WMI_RF_ETYPE_LENGTH];
+	u8 reserved2;
+	/* Rx E-type values to be used for each temperature range.
+	 * The last 4 values of any range are the first 4 values of the next
+	 * range and so on
+	 */
+	__le32 rx_etype_conf[WMI_RF_ETYPE_VAL_PER_RANGE + WMI_RF_ETYPE_LENGTH];
+	/* Temperature range for rx_2_tx_offs parameters */
+	u8 rx_2_tx_temp[WMI_RF_RX2TX_LENGTH];
+	u8 reserved3;
+	/* Rx to Tx gain index offset */
+	s8 rx_2_tx_offs[WMI_RF_RX2TX_LENGTH + 1];
+} __packed;
+
+/* CMD: WMI_RF_XPM_READ_CMDID */
+struct wmi_rf_xpm_read_cmd {
+	u8 rf_id;
+	u8 reserved[3];
+	/* XPM bit start address in range [0,8191]bits - rounded by FW to
+	 * multiple of 8bits
+	 */
+	__le32 xpm_bit_address;
+	__le32 num_bytes;
 } __packed;
 
-/* WMI_OTP_WRITE_CMDID */
-struct wmi_otp_write_cmd {
-	__le32 addr;
-	__le32 size;
-	__le32 values;
+/* CMD: WMI_RF_XPM_WRITE_CMDID */
+struct wmi_rf_xpm_write_cmd {
+	u8 rf_id;
+	u8 reserved0[3];
+	/* XPM bit start address in range [0,8191]bits - rounded by FW to
+	 * multiple of 8bits
+	 */
+	__le32 xpm_bit_address;
+	__le32 num_bytes;
+	/* boolean flag indicating whether FW should verify the write
+	 * operation
+	 */
+	u8 verify;
+	u8 reserved1[3];
+	/* actual size=num_bytes */
+	u8 data_bytes[0];
 } __packed;
 
 /* WMI_TEMP_SENSE_CMDID
@@ -989,19 +1096,26 @@ struct wmi_ftm_dest_info {
 	 */
 	__le16 burst_period;
 	u8 dst_mac[WMI_MAC_LEN];
-	__le16 reserved;
+	u8 reserved;
+	u8 num_burst_per_aoa_meas;
 } __packed;
 
 /* WMI_TOF_SESSION_START_CMDID */
 struct wmi_tof_session_start_cmd {
 	__le32 session_id;
-	u8 num_of_aoa_measures;
+	u8 reserved1;
 	u8 aoa_type;
 	__le16 num_of_dest;
 	u8 reserved[4];
 	struct wmi_ftm_dest_info ftm_dest_info[0];
 } __packed;
 
+/* WMI_TOF_CFG_RESPONDER_CMDID */
+struct wmi_tof_cfg_responder_cmd {
+	u8 enable;
+	u8 reserved[3];
+} __packed;
+
 enum wmi_tof_channel_info_report_type {
 	WMI_TOF_CHANNEL_INFO_TYPE_CIR			= 0x1,
 	WMI_TOF_CHANNEL_INFO_TYPE_RSSI			= 0x2,
@@ -1022,7 +1136,99 @@ struct wmi_tof_set_tx_rx_offset_cmd {
 	__le32 tx_offset;
 	/* RX delay offset */
 	__le32 rx_offset;
-	__le32 reserved[2];
+	/* Mask to define which RFs to configure. 0 means all RFs */
+	__le32 rf_mask;
+	/* Offset to strongest tap of CIR */
+	__le32 precursor;
+} __packed;
+
+/* WMI_TOF_GET_TX_RX_OFFSET_CMDID */
+struct wmi_tof_get_tx_rx_offset_cmd {
+	/* rf index to read offsets from */
+	u8 rf_index;
+	u8 reserved[3];
+} __packed;
+
+/* WMI_FIXED_SCHEDULING_CONFIG_CMDID */
+struct wmi_map_mcs_to_schd_params {
+	u8 mcs;
+	/* time in usec from start slot to start tx flow - default 15 */
+	u8 time_in_usec_before_initiate_tx;
+	/* RD enable - if yes consider RD according to STA mcs */
+	u8 rd_enabled;
+	u8 reserved;
+	/* time in usec from start slot to stop vring */
+	__le16 time_in_usec_to_stop_vring;
+	/* timeout to force flush from start of slot */
+	__le16 flush_to_in_usec;
+	/* per mcs the mac buffer limit size in bytes */
+	__le32 mac_buff_size_in_bytes;
+} __packed;
+
+/* WMI_FIXED_SCHEDULING_CONFIG_COMPLETE_EVENTID */
+struct wmi_fixed_scheduling_config_complete_event {
+	/* wmi_fw_status */
+	u8 status;
+	u8 reserved[3];
+} __packed;
+
+#define WMI_NUM_MCS	(13)
+
+/* WMI_FIXED_SCHEDULING_CONFIG_CMDID */
+struct wmi_fixed_scheduling_config_cmd {
+	/* defaults in the SAS table */
+	struct wmi_map_mcs_to_schd_params mcs_to_schd_params_map[WMI_NUM_MCS];
+	/* default 150 uSec */
+	__le16 max_sta_rd_ppdu_duration_in_usec;
+	/* default 300 uSec */
+	__le16 max_sta_grant_ppdu_duration_in_usec;
+	/* default 1000 uSec */
+	__le16 assoc_slot_duration_in_usec;
+	/* default 360 uSec */
+	__le16 virtual_slot_duration_in_usec;
+	/* each this field value slots start with grant frame to the station
+	 * - default 2
+	 */
+	u8 number_of_ap_slots_for_initiate_grant;
+	u8 reserved[3];
+} __packed;
+
+/* WMI_ENABLE_FIXED_SCHEDULING_CMDID */
+struct wmi_enable_fixed_scheduling_cmd {
+	__le32 reserved;
+} __packed;
+
+/* WMI_ENABLE_FIXED_SCHEDULING_COMPLETE_EVENTID */
+struct wmi_enable_fixed_scheduling_complete_event {
+	/* wmi_fw_status */
+	u8 status;
+	u8 reserved[3];
+} __packed;
+
+/* WMI_SET_MULTI_DIRECTED_OMNIS_CONFIG_CMDID */
+struct wmi_set_multi_directed_omnis_config_cmd {
+	/* number of directed omnis at destination AP */
+	u8 dest_ap_num_directed_omnis;
+	u8 reserved[3];
+} __packed;
+
+/* WMI_SET_MULTI_DIRECTED_OMNIS_CONFIG_EVENTID */
+struct wmi_set_multi_directed_omnis_config_event {
+	/* wmi_fw_status */
+	u8 status;
+	u8 reserved[3];
+} __packed;
+
+/* WMI_SET_LONG_RANGE_CONFIG_CMDID */
+struct wmi_set_long_range_config_cmd {
+	__le32 reserved;
+} __packed;
+
+/* WMI_SET_LONG_RANGE_CONFIG_COMPLETE_EVENTID */
+struct wmi_set_long_range_config_complete_event {
+	/* wmi_fw_status */
+	u8 status;
+	u8 reserved[3];
 } __packed;
 
 /* WMI Events
@@ -1038,19 +1244,22 @@ enum wmi_event_id {
 	WMI_FW_READY_EVENTID				= 0x1801,
 	WMI_EXIT_FAST_MEM_ACC_MODE_EVENTID		= 0x200,
 	WMI_ECHO_RSP_EVENTID				= 0x1803,
+	/* deprecated */
 	WMI_FS_TUNE_DONE_EVENTID			= 0x180A,
+	/* deprecated */
 	WMI_CORR_MEASURE_EVENTID			= 0x180B,
 	WMI_READ_RSSI_EVENTID				= 0x180C,
 	WMI_TEMP_SENSE_DONE_EVENTID			= 0x180E,
 	WMI_DC_CALIB_DONE_EVENTID			= 0x180F,
+	/* deprecated */
 	WMI_IQ_TX_CALIB_DONE_EVENTID			= 0x1811,
+	/* deprecated */
 	WMI_IQ_RX_CALIB_DONE_EVENTID			= 0x1812,
 	WMI_SET_WORK_MODE_DONE_EVENTID			= 0x1815,
 	WMI_LO_LEAKAGE_CALIB_DONE_EVENTID		= 0x1816,
-	WMI_MARLON_R_READ_DONE_EVENTID			= 0x1818,
-	WMI_MARLON_R_WRITE_DONE_EVENTID			= 0x1819,
-	WMI_MARLON_R_TXRX_SEL_DONE_EVENTID		= 0x181A,
+	WMI_LO_POWER_CALIB_FROM_OTP_EVENTID		= 0x1817,
 	WMI_SILENT_RSSI_CALIB_DONE_EVENTID		= 0x181D,
+	/* deprecated */
 	WMI_RF_RX_TEST_DONE_EVENTID			= 0x181E,
 	WMI_CFG_RX_CHAIN_DONE_EVENTID			= 0x1820,
 	WMI_VRING_CFG_DONE_EVENTID			= 0x1821,
@@ -1061,11 +1270,6 @@ enum wmi_event_id {
 	WMI_GET_SSID_EVENTID				= 0x1828,
 	WMI_GET_PCP_CHANNEL_EVENTID			= 0x182A,
 	WMI_SW_TX_COMPLETE_EVENTID			= 0x182B,
-	WMI_READ_MAC_RXQ_EVENTID			= 0x1830,
-	WMI_READ_MAC_TXQ_EVENTID			= 0x1831,
-	WMI_WRITE_MAC_RXQ_EVENTID			= 0x1832,
-	WMI_WRITE_MAC_TXQ_EVENTID			= 0x1833,
-	WMI_WRITE_MAC_XQ_FIELD_EVENTID			= 0x1834,
 	WMI_BEAMFORMING_MGMT_DONE_EVENTID		= 0x1836,
 	WMI_BF_TXSS_MGMT_DONE_EVENTID			= 0x1837,
 	WMI_BF_RXSS_MGMT_DONE_EVENTID			= 0x1839,
@@ -1076,8 +1280,12 @@ enum wmi_event_id {
 	WMI_TX_MGMT_PACKET_EVENTID			= 0x1841,
 	WMI_LINK_MAINTAIN_CFG_WRITE_DONE_EVENTID	= 0x1842,
 	WMI_LINK_MAINTAIN_CFG_READ_DONE_EVENTID		= 0x1843,
-	WMI_OTP_READ_RESULT_EVENTID			= 0x1856,
+	WMI_RF_XPM_READ_RESULT_EVENTID			= 0x1856,
+	WMI_RF_XPM_WRITE_RESULT_EVENTID			= 0x1857,
 	WMI_LED_CFG_DONE_EVENTID			= 0x1858,
+	WMI_SET_SILENT_RSSI_TABLE_DONE_EVENTID		= 0x185C,
+	WMI_RF_PWR_ON_DELAY_RSP_EVENTID			= 0x185D,
+	WMI_SET_HIGH_POWER_TABLE_PARAMS_EVENTID		= 0x185E,
 	/* Performance monitoring events */
 	WMI_DATA_PORT_OPEN_EVENTID			= 0x1860,
 	WMI_WBE_LINK_DOWN_EVENTID			= 0x1861,
@@ -1106,14 +1314,6 @@ enum wmi_event_id {
 	WMI_PCP_FACTOR_EVENTID				= 0x191A,
 	/* Power Save Configuration Events */
 	WMI_PS_DEV_PROFILE_CFG_EVENTID			= 0x191C,
-	/* Not supported yet */
-	WMI_PS_DEV_CFG_EVENTID				= 0x191D,
-	/* Not supported yet */
-	WMI_PS_DEV_CFG_READ_EVENTID			= 0x191E,
-	/* Not supported yet */
-	WMI_PS_MID_CFG_EVENTID				= 0x191F,
-	/* Not supported yet */
-	WMI_PS_MID_CFG_READ_EVENTID			= 0x1920,
 	WMI_RS_CFG_DONE_EVENTID				= 0x1921,
 	WMI_GET_DETAILED_RS_RES_EVENTID			= 0x1922,
 	WMI_AOA_MEAS_EVENTID				= 0x1923,
@@ -1122,14 +1322,17 @@ enum wmi_event_id {
 	WMI_GET_MGMT_RETRY_LIMIT_EVENTID		= 0x1931,
 	WMI_SET_THERMAL_THROTTLING_CFG_EVENTID		= 0x1940,
 	WMI_GET_THERMAL_THROTTLING_CFG_EVENTID		= 0x1941,
+	/* return the Power Save profile */
+	WMI_PS_DEV_PROFILE_CFG_READ_EVENTID		= 0x1942,
 	WMI_TOF_SESSION_END_EVENTID			= 0x1991,
 	WMI_TOF_GET_CAPABILITIES_EVENTID		= 0x1992,
 	WMI_TOF_SET_LCR_EVENTID				= 0x1993,
 	WMI_TOF_SET_LCI_EVENTID				= 0x1994,
 	WMI_TOF_FTM_PER_DEST_RES_EVENTID		= 0x1995,
-	WMI_TOF_CHANNEL_INFO_EVENTID			= 0x1996,
+	WMI_TOF_CFG_RESPONDER_EVENTID			= 0x1996,
 	WMI_TOF_SET_TX_RX_OFFSET_EVENTID		= 0x1997,
 	WMI_TOF_GET_TX_RX_OFFSET_EVENTID		= 0x1998,
+	WMI_TOF_CHANNEL_INFO_EVENTID			= 0x1999,
 	WMI_GET_RF_SECTOR_PARAMS_DONE_EVENTID		= 0x19A0,
 	WMI_SET_RF_SECTOR_PARAMS_DONE_EVENTID		= 0x19A1,
 	WMI_GET_SELECTED_RF_SECTOR_INDEX_DONE_EVENTID	= 0x19A2,
@@ -1138,12 +1341,18 @@ enum wmi_event_id {
 	WMI_PRIO_TX_SECTORS_ORDER_EVENTID		= 0x19A5,
 	WMI_PRIO_TX_SECTORS_NUMBER_EVENTID		= 0x19A6,
 	WMI_PRIO_TX_SECTORS_SET_DEFAULT_CFG_EVENTID	= 0x19A7,
+	WMI_SCHEDULING_SCHEME_EVENTID			= 0x1A01,
+	WMI_FIXED_SCHEDULING_CONFIG_COMPLETE_EVENTID	= 0x1A02,
+	WMI_ENABLE_FIXED_SCHEDULING_COMPLETE_EVENTID	= 0x1A03,
+	WMI_SET_MULTI_DIRECTED_OMNIS_CONFIG_EVENTID	= 0x1A04,
+	WMI_SET_LONG_RANGE_CONFIG_COMPLETE_EVENTID	= 0x1A05,
 	WMI_SET_CHANNEL_EVENTID				= 0x9000,
 	WMI_ASSOC_REQ_EVENTID				= 0x9001,
 	WMI_EAPOL_RX_EVENTID				= 0x9002,
 	WMI_MAC_ADDR_RESP_EVENTID			= 0x9003,
 	WMI_FW_VER_EVENTID				= 0x9004,
 	WMI_ACS_PASSIVE_SCAN_COMPLETE_EVENTID		= 0x9005,
+	WMI_COMMAND_NOT_SUPPORTED_EVENTID		= 0xFFFF,
 };
 
 /* Events data structures */
@@ -1200,7 +1409,7 @@ struct wmi_fw_ver_event {
 	__le32 bl_minor;
 	__le32 bl_subminor;
 	__le32 bl_build;
-	/* The number of entries in the FW capabilies array */
+	/* The number of entries in the FW capabilities array */
 	u8 fw_capabilities_len;
 	u8 reserved[3];
 	/* FW capabilities info
@@ -1245,7 +1454,9 @@ struct wmi_get_rf_status_event {
 	__le32 board_file_platform_type;
 	/* board file version */
 	__le32 board_file_version;
-	__le32 reserved[2];
+	/* enabled XIFs bit vector */
+	__le32 enabled_xif_vector;
+	__le32 reserved;
 } __packed;
 
 /* WMI_GET_BASEBAND_TYPE_EVENTID */
@@ -1299,6 +1510,9 @@ struct wmi_ready_event {
 	/* enum wmi_phy_capability */
 	u8 phy_capability;
 	u8 numof_additional_mids;
+	/* rfc read calibration result. 5..15 */
+	u8 rfc_read_calib_result;
+	u8 reserved[3];
 } __packed;
 
 /* WMI_NOTIFY_REQ_DONE_EVENTID */
@@ -1306,7 +1520,8 @@ struct wmi_notify_req_done_event {
 	/* beamforming status, 0: fail; 1: OK; 2: retrying */
 	__le32 status;
 	__le64 tsf;
-	__le32 snr_val;
+	s8 rssi;
+	u8 reserved0[3];
 	__le32 tx_tpt;
 	__le32 tx_goodput;
 	__le32 rx_goodput;
@@ -1576,7 +1791,7 @@ struct wmi_sw_tx_complete_event {
 	u8 reserved[3];
 } __packed;
 
-/* WMI_CORR_MEASURE_EVENTID */
+/* WMI_CORR_MEASURE_EVENTID - deprecated */
 struct wmi_corr_measure_event {
 	/* signed */
 	__le32 i;
@@ -1602,31 +1817,35 @@ struct wmi_get_ssid_event {
 /* wmi_rx_mgmt_info */
 struct wmi_rx_mgmt_info {
 	u8 mcs;
-	s8 snr;
+	s8 rssi;
 	u8 range;
 	u8 sqi;
 	__le16 stype;
 	__le16 status;
 	__le32 len;
-	/* Not resolved when == 0xFFFFFFFF  ==> Broadcast to all MIDS */
+	/* Not resolved when == 0xFFFFFFFF == > Broadcast to all MIDS */
 	u8 qid;
-	/* Not resolved when == 0xFFFFFFFF  ==> Broadcast to all MIDS */
+	/* Not resolved when == 0xFFFFFFFF == > Broadcast to all MIDS */
 	u8 mid;
 	u8 cid;
 	/* From Radio MNGR */
 	u8 channel;
 } __packed;
 
-/* wmi_otp_read_write_cmd */
-struct wmi_otp_read_write_cmd {
-	__le32 addr;
-	__le32 size;
-	u8 values[0];
+/* EVENT: WMI_RF_XPM_READ_RESULT_EVENTID */
+struct wmi_rf_xpm_read_result_event {
+	/* enum wmi_fw_status_e - success=0 or fail=1 */
+	u8 status;
+	u8 reserved[3];
+	/* requested num_bytes of data */
+	u8 data_bytes[0];
 } __packed;
 
-/* WMI_OTP_READ_RESULT_EVENTID */
-struct wmi_otp_read_result_event {
-	u8 payload[0];
+/* EVENT: WMI_RF_XPM_WRITE_RESULT_EVENTID */
+struct wmi_rf_xpm_write_result_event {
+	/* enum wmi_fw_status_e - success=0 or fail=1 */
+	u8 status;
+	u8 reserved[3];
 } __packed;
 
 /* WMI_TX_MGMT_PACKET_EVENTID */
@@ -1645,6 +1864,20 @@ struct wmi_echo_rsp_event {
 	__le32 echoed_value;
 } __packed;
 
+/* WMI_RF_PWR_ON_DELAY_RSP_EVENTID */
+struct wmi_rf_pwr_on_delay_rsp_event {
+	/* wmi_fw_status */
+	u8 status;
+	u8 reserved[3];
+} __packed;
+
+/* WMI_SET_HIGH_POWER_TABLE_PARAMS_EVENTID */
+struct wmi_set_high_power_table_params_event {
+	/* wmi_fw_status */
+	u8 status;
+	u8 reserved[3];
+} __packed;
+
 /* WMI_TEMP_SENSE_DONE_EVENTID
  *
  * Measure MAC and radio temperatures
@@ -1722,14 +1955,22 @@ struct wmi_led_cfg_cmd {
 	u8 reserved;
 } __packed;
 
+/* \WMI_SET_CONNECT_SNR_THR_CMDID */
+struct wmi_set_connect_snr_thr_cmd {
+	u8 enable;
+	u8 reserved;
+	/* 1/4 Db units */
+	__le16 omni_snr_thr;
+	/* 1/4 Db units */
+	__le16 direct_snr_thr;
+} __packed;
+
 /* WMI_LED_CFG_DONE_EVENTID */
 struct wmi_led_cfg_done_event {
 	/* led config status */
 	__le32 status;
 } __packed;
 
-#define WMI_NUM_MCS	(13)
-
 /* Rate search parameters configuration per connection */
 struct wmi_rs_cfg {
 	/* The maximal allowed PER for each MCS
@@ -1754,6 +1995,98 @@ struct wmi_rs_cfg {
 	__le32 mcs_en_vec;
 } __packed;
 
+/* Slot types */
+enum wmi_sched_scheme_slot_type {
+	WMI_SCHED_SLOT_SP		= 0x0,
+	WMI_SCHED_SLOT_CBAP		= 0x1,
+	WMI_SCHED_SLOT_IDLE		= 0x2,
+	WMI_SCHED_SLOT_ANNOUNCE_NO_ACK	= 0x3,
+	WMI_SCHED_SLOT_DISCOVERY	= 0x4,
+};
+
+enum wmi_sched_scheme_slot_flags {
+	WMI_SCHED_SCHEME_SLOT_PERIODIC	= 0x1,
+};
+
+struct wmi_sched_scheme_slot {
+	/* in microsecond */
+	__le32 tbtt_offset;
+	/* wmi_sched_scheme_slot_flags */
+	u8 flags;
+	/* wmi_sched_scheme_slot_type */
+	u8 type;
+	/* in microsecond */
+	__le16 duration;
+	/* frame_exchange_sequence_duration */
+	__le16 tx_op;
+	/* time in microseconds between two consecutive slots
+	 * relevant only if flag WMI_SCHED_SCHEME_SLOT_PERIODIC set
+	 */
+	__le16 period;
+	/* relevant only if flag WMI_SCHED_SCHEME_SLOT_PERIODIC set
+	 * number of times to repeat allocation
+	 */
+	u8 num_of_blocks;
+	/* relevant only if flag WMI_SCHED_SCHEME_SLOT_PERIODIC set
+	 * every idle_period allocation will be idle
+	 */
+	u8 idle_period;
+	u8 src_aid;
+	u8 dest_aid;
+	__le32 reserved;
+} __packed;
+
+enum wmi_sched_scheme_flags {
+	/* should not be set when clearing scheduling scheme */
+	WMI_SCHED_SCHEME_ENABLE		= 0x01,
+	WMI_SCHED_PROTECTED_SP		= 0x02,
+	/* should be set only on first WMI fragment of scheme */
+	WMI_SCHED_FIRST			= 0x04,
+	/* should be set only on last WMI fragment of scheme */
+	WMI_SCHED_LAST			= 0x08,
+	WMI_SCHED_IMMEDIATE_START	= 0x10,
+};
+
+enum wmi_sched_scheme_advertisment {
+	/* ESE is not advertised at all, STA has to be configured with WMI
+	 * also
+	 */
+	WMI_ADVERTISE_ESE_DISABLED		= 0x0,
+	WMI_ADVERTISE_ESE_IN_BEACON		= 0x1,
+	WMI_ADVERTISE_ESE_IN_ANNOUNCE_FRAME	= 0x2,
+};
+
+/* WMI_SCHEDULING_SCHEME_CMD */
+struct wmi_scheduling_scheme_cmd {
+	u8 serial_num;
+	/* wmi_sched_scheme_advertisment */
+	u8 ese_advertisment;
+	/* wmi_sched_scheme_flags */
+	__le16 flags;
+	u8 num_allocs;
+	u8 reserved[3];
+	__le64 start_tbtt;
+	/* allocations list */
+	struct wmi_sched_scheme_slot allocs[WMI_SCHED_MAX_ALLOCS_PER_CMD];
+} __packed;
+
+enum wmi_sched_scheme_failure_type {
+	WMI_SCHED_SCHEME_FAILURE_NO_ERROR		= 0x00,
+	WMI_SCHED_SCHEME_FAILURE_OLD_START_TSF_ERR	= 0x01,
+};
+
+/* WMI_SCHEDULING_SCHEME_EVENTID */
+struct wmi_scheduling_scheme_event {
+	/* wmi_fw_status_e */
+	u8 status;
+	/* serial number given in command */
+	u8 serial_num;
+	/* wmi_sched_scheme_failure_type */
+	u8 failure_type;
+	/* alignment to 32b */
+	u8 reserved[1];
+} __packed;
+
 /* WMI_RS_CFG_CMDID */
 struct wmi_rs_cfg_cmd {
 	/* connection id */
@@ -1971,6 +2304,19 @@ enum wmi_ps_profile_type {
 	WMI_PS_PROFILE_TYPE_LOW_LATENCY_PS	= 0x03,
 };
 
+/* WMI_PS_DEV_PROFILE_CFG_READ_CMDID */
+struct wmi_ps_dev_profile_cfg_read_cmd {
+	/* reserved */
+	__le32 reserved;
+} __packed;
+
+/* WMI_PS_DEV_PROFILE_CFG_READ_EVENTID */
+struct wmi_ps_dev_profile_cfg_read_event {
+	/* wmi_ps_profile_type_e */
+	u8 ps_profile;
+	u8 reserved[3];
+} __packed;
+
 /* WMI_PS_DEV_PROFILE_CFG_CMDID
  *
  * Power save profile to be used by the device
@@ -2019,157 +2365,6 @@ enum wmi_ps_d3_resp_policy {
 	WMI_PS_D3_RESP_POLICY_APPROVED	= 0x02,
 };
 
-/* Device common power save configurations */
-struct wmi_ps_dev_cfg {
-	/* lowest level of PS allowed while unassociated, enum wmi_ps_level_e
-	 */
-	u8 ps_unassoc_min_level;
-	/* lowest deep sleep clock level while nonassoc, enum
-	 * wmi_ps_deep_sleep_clk_level_e
-	 */
-	u8 ps_unassoc_deep_sleep_min_level;
-	/* lowest level of PS allowed while associated, enum wmi_ps_level_e */
-	u8 ps_assoc_min_level;
-	/* lowest deep sleep clock level while assoc, enum
-	 * wmi_ps_deep_sleep_clk_level_e
-	 */
-	u8 ps_assoc_deep_sleep_min_level;
-	/* enum wmi_ps_deep_sleep_clk_level_e */
-	u8 ps_assoc_low_latency_ds_min_level;
-	/* enum wmi_ps_d3_resp_policy_e */
-	u8 ps_D3_response_policy;
-	/* BOOL */
-	u8 ps_D3_pm_pme_enabled;
-	/* BOOL */
-	u8 ps_halp_enable;
-	u8 ps_deep_sleep_enter_thresh_msec;
-	/* BOOL */
-	u8 ps_voltage_scaling_en;
-} __packed;
-
-/* WMI_PS_DEV_CFG_CMDID
- *
- * Configure common Power Save parameters of the device and all MIDs.
- *
- * Returned event:
- * - WMI_PS_DEV_CFG_EVENTID
- */
-struct wmi_ps_dev_cfg_cmd {
-	/* Device Power Save configuration to be applied */
-	struct wmi_ps_dev_cfg ps_dev_cfg;
-	/* alignment to 32b */
-	u8 reserved[2];
-} __packed;
-
-/* WMI_PS_DEV_CFG_EVENTID */
-struct wmi_ps_dev_cfg_event {
-	/* wmi_ps_cfg_cmd_status_e */
-	__le32 status;
-} __packed;
-
-/* WMI_PS_DEV_CFG_READ_CMDID
- *
- * request to retrieve  device Power Save configuration
- * (WMI_PS_DEV_CFG_CMD params)
- *
- * Returned event:
- * - WMI_PS_DEV_CFG_READ_EVENTID
- */
-struct wmi_ps_dev_cfg_read_cmd {
-	__le32 reserved;
-} __packed;
-
-/* WMI_PS_DEV_CFG_READ_EVENTID */
-struct wmi_ps_dev_cfg_read_event {
-	/* wmi_ps_cfg_cmd_status_e */
-	__le32 status;
-	/* Retrieved device Power Save configuration (WMI_PS_DEV_CFG_CMD
-	 * params)
-	 */
-	struct wmi_ps_dev_cfg dev_ps_cfg;
-	/* alignment to 32b */
-	u8 reserved[2];
-} __packed;
-
-/* Per Mac Power Save configurations */
-struct wmi_ps_mid_cfg {
-	/* Low power RX in BTI is enabled, BOOL */
-	u8 beacon_lprx_enable;
-	/* Sync to sector ID enabled, BOOL */
-	u8 beacon_sync_to_sectorId_enable;
-	/* Low power RX in DTI is enabled, BOOL */
-	u8 frame_exchange_lprx_enable;
-	/* Sleep Cycle while in scheduled PS, 1-31 */
-	u8 scheduled_sleep_cycle_pow2;
-	/* Stay Awake for k BIs every (sleep_cycle - k) BIs, 1-31 */
-	u8 scheduled_num_of_awake_bis;
-	u8 am_to_traffic_load_thresh_mbp;
-	u8 traffic_to_am_load_thresh_mbps;
-	u8 traffic_to_am_num_of_no_traffic_bis;
-	/* BOOL */
-	u8 continuous_traffic_psm;
-	__le16 no_traffic_to_min_usec;
-	__le16 no_traffic_to_max_usec;
-	__le16 snoozing_sleep_interval_milisec;
-	u8 max_no_data_awake_events;
-	/* Trigger WEB after k failed beacons */
-	u8 num_of_failed_beacons_rx_to_trigger_web;
-	/* Trigger BF after k failed beacons */
-	u8 num_of_failed_beacons_rx_to_trigger_bf;
-	/* Trigger SOB after k successful beacons */
-	u8 num_of_successful_beacons_rx_to_trigger_sob;
-} __packed;
-
-/* WMI_PS_MID_CFG_CMDID
- *
- * Configure Power Save parameters of a specific MID.
- * These parameters are relevant for the specific BSS this MID belongs to.
- *
- * Returned event:
- * - WMI_PS_MID_CFG_EVENTID
- */
-struct wmi_ps_mid_cfg_cmd {
-	/* MAC ID */
-	u8 mid;
-	/* mid PS configuration to be applied */
-	struct wmi_ps_mid_cfg ps_mid_cfg;
-} __packed;
-
-/* WMI_PS_MID_CFG_EVENTID */
-struct wmi_ps_mid_cfg_event {
-	/* MAC ID */
-	u8 mid;
-	/* alignment to 32b */
-	u8 reserved[3];
-	/* wmi_ps_cfg_cmd_status_e */
-	__le32 status;
-} __packed;
-
-/* WMI_PS_MID_CFG_READ_CMDID
- *
- * request to retrieve Power Save configuration of mid
- * (WMI_PS_MID_CFG_CMD params)
- *
- * Returned event:
- * - WMI_PS_MID_CFG_READ_EVENTID
- */
-struct wmi_ps_mid_cfg_read_cmd {
-	/* MAC ID */
-	u8 mid;
-	/* alignment to 32b */
-	u8 reserved[3];
-} __packed;
-
-/* WMI_PS_MID_CFG_READ_EVENTID */
-struct wmi_ps_mid_cfg_read_event {
-	/* MAC ID */
-	u8 mid;
-	/* Retrieved MID Power Save configuration(WMI_PS_MID_CFG_CMD params) */
-	struct wmi_ps_mid_cfg mid_ps_cfg;
-	/* wmi_ps_cfg_cmd_status_e */
-	__le32 status;
-} __packed;
-
 #define WMI_AOA_MAX_DATA_SIZE	(128)
 
 enum wmi_aoa_meas_status {
@@ -2260,6 +2455,20 @@ struct wmi_tof_session_end_event {
 	u8 reserved[3];
 } __packed;
 
+/* WMI_TOF_SET_LCI_EVENTID */
+struct wmi_tof_set_lci_event {
+	/* enum wmi_fw_status */
+	u8 status;
+	u8 reserved[3];
+} __packed;
+
+/* WMI_TOF_SET_LCR_EVENTID */
+struct wmi_tof_set_lcr_event {
+	/* enum wmi_fw_status */
+	u8 status;
+	u8 reserved[3];
+} __packed;
+
 /* Responder FTM Results */
 struct wmi_responder_ftm_res {
 	u8 t1[6];
@@ -2313,10 +2522,19 @@ struct wmi_tof_ftm_per_dest_res_event {
 	__le32 tsf_sync;
 	/* actual received ftm per burst */
 	u8 actual_ftm_per_burst;
-	u8 reserved0[7];
+	/* Measurments are from RFs, defined by the mask */
+	__le32 meas_rf_mask;
+	u8 reserved0[3];
 	struct wmi_responder_ftm_res responder_ftm_res[0];
 } __packed;
 
+/* WMI_TOF_CFG_RESPONDER_EVENTID */
+struct wmi_tof_cfg_responder_event {
+	/* enum wmi_fw_status */
+	u8 status;
+	u8 reserved[3];
+} __packed;
+
 enum wmi_tof_channel_info_type {
 	WMI_TOF_CHANNEL_INFO_AOA		= 0x00,
 	WMI_TOF_CHANNEL_INFO_LCI		= 0x01,
@@ -2353,12 +2571,15 @@ struct wmi_tof_set_tx_rx_offset_event {
 struct wmi_tof_get_tx_rx_offset_event {
 	/* enum wmi_fw_status */
 	u8 status;
-	u8 reserved1[3];
+	/* RF index used to read the offsets */
+	u8 rf_index;
+	u8 reserved1[2];
 	/* TX delay offset */
 	__le32 tx_offset;
 	/* RX delay offset */
 	__le32 rx_offset;
-	__le32 reserved2[2];
+	/* Offset to strongest tap of CIR */
+	__le32 precursor;
 } __packed;
 
 /* Result status codes for WMI commands */
@@ -2621,4 +2842,23 @@ struct wmi_prio_tx_sectors_set_default_cfg_event {
 	u8 reserved[3];
 } __packed;
 
+/* WMI_SET_SILENT_RSSI_TABLE_DONE_EVENTID */
+struct wmi_set_silent_rssi_table_done_event {
+	/* enum wmi_silent_rssi_status */
+	__le32 status;
+	/* enum wmi_silent_rssi_table */
+	__le32 table;
+} __packed;
+
+/* \WMI_COMMAND_NOT_SUPPORTED_EVENTID */
+struct wmi_command_not_supported_event {
+	/* device id */
+	u8 mid;
+	u8 reserved0;
+	__le16 command_id;
+	/* for UT command only, otherwise reserved */
+	__le16 command_subtype;
+	__le16 reserved1;
+} __packed;
+
 #endif /* __WILOCITY_WMI_H__ */
diff --git a/drivers/net/wireless/atmel/at76c50x-usb.c b/drivers/net/wireless/atmel/at76c50x-usb.c
index 09defbcedd5e..94bf01f8b2a8 100644
--- a/drivers/net/wireless/atmel/at76c50x-usb.c
+++ b/drivers/net/wireless/atmel/at76c50x-usb.c
@@ -130,7 +130,7 @@ MODULE_FIRMWARE("atmel_at76c505amx-rfmd.bin");
 
 #define USB_DEVICE_DATA(__ops)	.driver_info = (kernel_ulong_t)(__ops)
 
-static struct usb_device_id dev_table[] = {
+static const struct usb_device_id dev_table[] = {
 	/*
 	 * at76c503-i3861
 	 */
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c
index 984c1d0560b1..cd587325e286 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c
@@ -1105,6 +1105,7 @@ static const struct sdio_device_id brcmf_sdmmc_ids[] = {
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_43455),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4354),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4356),
+	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_CYPRESS_4373),
 	{ /* end: all zeroes */ }
 };
 MODULE_DEVICE_TABLE(sdio, brcmf_sdmmc_ids);
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
index 7e689c86d565..aaed4ab503ad 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
@@ -3940,6 +3940,7 @@ brcmf_cfg80211_flush_pmksa(struct wiphy *wiphy, struct net_device *ndev)
 static s32 brcmf_configure_opensecurity(struct brcmf_if *ifp)
 {
 	s32 err;
+	s32 wpa_val;
 
 	/* set auth */
 	err = brcmf_fil_bsscfg_int_set(ifp, "auth", 0);
@@ -3954,7 +3955,11 @@ static s32 brcmf_configure_opensecurity(struct brcmf_if *ifp)
 		return err;
 	}
 	/* set upper-layer auth */
-	err = brcmf_fil_bsscfg_int_set(ifp, "wpa_auth", WPA_AUTH_NONE);
+	if (brcmf_is_ibssmode(ifp->vif))
+		wpa_val = WPA_AUTH_NONE;
+	else
+		wpa_val = WPA_AUTH_DISABLED;
+	err = brcmf_fil_bsscfg_int_set(ifp, "wpa_auth", wpa_val);
 	if (err < 0) {
 		brcmf_err("wpa_auth error %d\n", err);
 		return err;
@@ -5693,10 +5698,13 @@ brcmf_notify_roaming_status(struct brcmf_if *ifp,
 	u32 status = e->status;
 
 	if (event == BRCMF_E_ROAM && status == BRCMF_E_STATUS_SUCCESS) {
-		if (test_bit(BRCMF_VIF_STATUS_CONNECTED, &ifp->vif->sme_state))
+		if (test_bit(BRCMF_VIF_STATUS_CONNECTED,
+			     &ifp->vif->sme_state)) {
 			brcmf_bss_roaming_done(cfg, ifp->ndev, e);
-		else
+		} else {
 			brcmf_bss_connect_done(cfg, ifp->ndev, e, true);
+			brcmf_net_setcarrier(ifp, true);
+		}
 	}
 
 	return 0;
@@ -6456,6 +6464,8 @@ static int brcmf_setup_ifmodes(struct wiphy *wiphy, struct brcmf_if *ifp)
 	if (p2p) {
 		if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MCHAN))
 			combo[c].num_different_channels = 2;
+		else
+			combo[c].num_different_channels = 1;
 		wiphy->interface_modes |= BIT(NL80211_IFTYPE_P2P_CLIENT) |
 					  BIT(NL80211_IFTYPE_P2P_GO) |
 					  BIT(NL80211_IFTYPE_P2P_DEVICE);
@@ -6465,10 +6475,10 @@ static int brcmf_setup_ifmodes(struct wiphy *wiphy, struct brcmf_if *ifp)
 		c0_limits[i++].types = BIT(NL80211_IFTYPE_P2P_CLIENT) |
 				       BIT(NL80211_IFTYPE_P2P_GO);
 	} else {
+		combo[c].num_different_channels = 1;
 		c0_limits[i].max = 1;
 		c0_limits[i++].types = BIT(NL80211_IFTYPE_AP);
 	}
-	combo[c].num_different_channels = 1;
 	combo[c].max_interfaces = i;
 	combo[c].n_limits = i;
 	combo[c].limits = c0_limits;
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c
index 05f22ff81d60..c5d1a1cbf601 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c
@@ -690,6 +690,8 @@ static u32 brcmf_chip_tcm_rambase(struct brcmf_chip_priv *ci)
 	case BRCM_CC_4365_CHIP_ID:
 	case BRCM_CC_4366_CHIP_ID:
 		return 0x200000;
+	case CY_CC_4373_CHIP_ID:
+		return 0x160000;
 	default:
 		brcmf_err("unknown chip: %s\n", ci->pub.name);
 		break;
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c
index 1447a8352383..2d3e5e263a32 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c
@@ -78,10 +78,7 @@ int brcmf_debug_attach(struct brcmf_pub *drvr)
 		return -ENODEV;
 
 	drvr->dbgfs_dir = debugfs_create_dir(dev_name(dev), root_folder);
-	if (IS_ERR(drvr->dbgfs_dir))
-		return PTR_ERR(drvr->dbgfs_dir);
-
-	return 0;
+	return PTR_ERR_OR_ZERO(drvr->dbgfs_dir);
 }
 
 void brcmf_debug_detach(struct brcmf_pub *drvr)
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c
index d21258d277ce..53ae30259989 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c
@@ -159,8 +159,11 @@ void brcmf_feat_attach(struct brcmf_pub *drvr)
 
 	brcmf_feat_firmware_capabilities(ifp);
 	memset(&gscan_cfg, 0, sizeof(gscan_cfg));
-	brcmf_feat_iovar_data_set(ifp, BRCMF_FEAT_GSCAN, "pfn_gscan_cfg",
-				  &gscan_cfg, sizeof(gscan_cfg));
+	if (drvr->bus_if->chip != BRCM_CC_43430_CHIP_ID &&
+	    drvr->bus_if->chip != BRCM_CC_4345_CHIP_ID)
+		brcmf_feat_iovar_data_set(ifp, BRCMF_FEAT_GSCAN,
+					  "pfn_gscan_cfg",
+					  &gscan_cfg, sizeof(gscan_cfg));
 	brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_PNO, "pfn");
 	if (drvr->bus_if->wowl_supported)
 		brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_WOWL, "wowl");
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c
index d231042f19d6..091b52979e03 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c
@@ -601,6 +601,9 @@ int brcmf_fw_map_chip_to_name(u32 chip, u32 chiprev,
 	if ((nvram_name) && (mapping_table[i].nvram))
 		strlcat(nvram_name, mapping_table[i].nvram, BRCMF_FW_NAME_LEN);
 
+	brcmf_info("using %s for chip %#08x(%d) rev %#08x\n",
+		   fw_name, chip, chip, chiprev);
+
 	return 0;
 }
 
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
index f878706613e6..e6e9b00b79d7 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
@@ -1951,7 +1951,7 @@ static const struct dev_pm_ops brcmf_pciedrvr_pm = {
 	BRCM_PCIE_VENDOR_ID_BROADCOM, dev_id,\
 	subvend, subdev, PCI_CLASS_NETWORK_OTHER << 8, 0xffff00, 0 }
 
-static struct pci_device_id brcmf_pcie_devid_table[] = {
+static const struct pci_device_id brcmf_pcie_devid_table[] = {
 	BRCMF_PCIE_DEVICE(BRCM_PCIE_4350_DEVICE_ID),
 	BRCMF_PCIE_DEVICE(BRCM_PCIE_4356_DEVICE_ID),
 	BRCMF_PCIE_DEVICE(BRCM_PCIE_43567_DEVICE_ID),
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
index f3556122c6ac..613caca7dc02 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
@@ -618,6 +618,7 @@ BRCMF_FW_NVRAM_DEF(43430A1, "brcmfmac43430-sdio.bin", "brcmfmac43430-sdio.txt");
 BRCMF_FW_NVRAM_DEF(43455, "brcmfmac43455-sdio.bin", "brcmfmac43455-sdio.txt");
 BRCMF_FW_NVRAM_DEF(4354, "brcmfmac4354-sdio.bin", "brcmfmac4354-sdio.txt");
 BRCMF_FW_NVRAM_DEF(4356, "brcmfmac4356-sdio.bin", "brcmfmac4356-sdio.txt");
+BRCMF_FW_NVRAM_DEF(4373, "brcmfmac4373-sdio.bin", "brcmfmac4373-sdio.txt");
 
 static struct brcmf_firmware_mapping brcmf_sdio_fwnames[] = {
 	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43143_CHIP_ID, 0xFFFFFFFF, 43143),
@@ -636,7 +637,8 @@ static struct brcmf_firmware_mapping brcmf_sdio_fwnames[] = {
 	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43430_CHIP_ID, 0xFFFFFFFE, 43430A1),
 	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4345_CHIP_ID, 0xFFFFFFC0, 43455),
 	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4354_CHIP_ID, 0xFFFFFFFF, 4354),
-	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4356_CHIP_ID, 0xFFFFFFFF, 4356)
+	BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4356_CHIP_ID, 0xFFFFFFFF, 4356),
+	BRCMF_FW_NVRAM_ENTRY(CY_CC_4373_CHIP_ID, 0xFFFFFFFF, 4373)
 };
 
 static void pkt_align(struct sk_buff *p, int len, int align)
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c
index 0eea48e73331..11ffaa01599e 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c
@@ -50,6 +50,7 @@ BRCMF_FW_DEF(43143, "brcmfmac43143.bin");
 BRCMF_FW_DEF(43236B, "brcmfmac43236b.bin");
 BRCMF_FW_DEF(43242A, "brcmfmac43242a.bin");
 BRCMF_FW_DEF(43569, "brcmfmac43569.bin");
+BRCMF_FW_DEF(4373, "brcmfmac4373.bin");
 
 static struct brcmf_firmware_mapping brcmf_usb_fwnames[] = {
 	BRCMF_FW_ENTRY(BRCM_CC_43143_CHIP_ID, 0xFFFFFFFF, 43143),
@@ -58,7 +59,8 @@ static struct brcmf_firmware_mapping brcmf_usb_fwnames[] = {
 	BRCMF_FW_ENTRY(BRCM_CC_43238_CHIP_ID, 0x00000008, 43236B),
 	BRCMF_FW_ENTRY(BRCM_CC_43242_CHIP_ID, 0xFFFFFFFF, 43242A),
 	BRCMF_FW_ENTRY(BRCM_CC_43566_CHIP_ID, 0xFFFFFFFF, 43569),
-	BRCMF_FW_ENTRY(BRCM_CC_43569_CHIP_ID, 0xFFFFFFFF, 43569)
+	BRCMF_FW_ENTRY(BRCM_CC_43569_CHIP_ID, 0xFFFFFFFF, 43569),
+	BRCMF_FW_ENTRY(CY_CC_4373_CHIP_ID, 0xFFFFFFFF, 4373)
 };
 
 #define TRX_MAGIC		0x30524448	/* "HDR0" */
@@ -1463,15 +1465,20 @@ static int brcmf_usb_reset_resume(struct usb_interface *intf)
 #define LINKSYS_USB_DEVICE(dev_id)	\
 	{ USB_DEVICE(BRCM_USB_VENDOR_ID_LINKSYS, dev_id) }
 
-static struct usb_device_id brcmf_usb_devid_table[] = {
+#define CYPRESS_USB_DEVICE(dev_id)	\
+	{ USB_DEVICE(CY_USB_VENDOR_ID_CYPRESS, dev_id) }
+
+static const struct usb_device_id brcmf_usb_devid_table[] = {
 	BRCMF_USB_DEVICE(BRCM_USB_43143_DEVICE_ID),
 	BRCMF_USB_DEVICE(BRCM_USB_43236_DEVICE_ID),
 	BRCMF_USB_DEVICE(BRCM_USB_43242_DEVICE_ID),
 	BRCMF_USB_DEVICE(BRCM_USB_43569_DEVICE_ID),
 	LINKSYS_USB_DEVICE(BRCM_USB_43235_LINKSYS_DEVICE_ID),
+	CYPRESS_USB_DEVICE(CY_USB_4373_DEVICE_ID),
 	{ USB_DEVICE(BRCM_USB_VENDOR_ID_LG, BRCM_USB_43242_LG_DEVICE_ID) },
 	/* special entry for device with firmware loaded and running */
 	BRCMF_USB_DEVICE(BRCM_USB_BCMFW_DEVICE_ID),
+	CYPRESS_USB_DEVICE(BRCM_USB_BCMFW_DEVICE_ID),
 	{ /* end: all zeroes */ }
 };
 
diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h
index f1fb8a3c7a32..57544a3a3ce4 100644
--- a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h
+++ b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h
@@ -23,6 +23,7 @@
 #define BRCM_USB_VENDOR_ID_BROADCOM	0x0a5c
 #define BRCM_USB_VENDOR_ID_LG		0x043e
 #define BRCM_USB_VENDOR_ID_LINKSYS	0x13b1
+#define CY_USB_VENDOR_ID_CYPRESS	0x04b4
 #define BRCM_PCIE_VENDOR_ID_BROADCOM	PCI_VENDOR_ID_BROADCOM
 
 /* Chipcommon Core Chip IDs */
@@ -57,6 +58,7 @@
 #define BRCM_CC_4365_CHIP_ID		0x4365
 #define BRCM_CC_4366_CHIP_ID		0x4366
 #define BRCM_CC_4371_CHIP_ID		0x4371
+#define CY_CC_4373_CHIP_ID		0x4373
 
 /* USB Device IDs */
 #define BRCM_USB_43143_DEVICE_ID	0xbd1e
@@ -66,6 +68,7 @@
 #define BRCM_USB_43242_LG_DEVICE_ID	0x3101
 #define BRCM_USB_43569_DEVICE_ID	0xbd27
 #define BRCM_USB_BCMFW_DEVICE_ID	0x0bdc
+#define CY_USB_4373_DEVICE_ID		0xbd29
 
 /* PCIE Device IDs */
 #define BRCM_PCIE_4350_DEVICE_ID	0x43a3
diff --git a/drivers/net/wireless/cisco/airo.c b/drivers/net/wireless/cisco/airo.c
index 84143a02adce..54201c02fdb8 100644
--- a/drivers/net/wireless/cisco/airo.c
+++ b/drivers/net/wireless/cisco/airo.c
@@ -7837,7 +7837,7 @@ static int writerids(struct net_device *dev, aironet_ioctl *comp) {
 	struct airo_info *ai = dev->ml_priv;
 	int  ridcode;
         int  enabled;
-	static int (* writer)(struct airo_info *, u16 rid, const void *, int, int);
+	int (*writer)(struct airo_info *, u16 rid, const void *, int, int);
 	unsigned char *iobuf;
 
 	/* Only super-user can write RIDs */
diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2100.c b/drivers/net/wireless/intel/ipw2x00/ipw2100.c
index aaaca4d08e2b..19c442cb93e4 100644
--- a/drivers/net/wireless/intel/ipw2x00/ipw2100.c
+++ b/drivers/net/wireless/intel/ipw2x00/ipw2100.c
@@ -340,7 +340,7 @@ static int ipw2100_ucode_download(struct ipw2100_priv *priv,
 				  struct ipw2100_fw *fw);
 static void ipw2100_wx_event_work(struct work_struct *work);
 static struct iw_statistics *ipw2100_wx_wireless_stats(struct net_device *dev);
-static struct iw_handler_def ipw2100_wx_handler_def;
+static const struct iw_handler_def ipw2100_wx_handler_def;
 
 static inline void read_register(struct net_device *dev, u32 reg, u32 * val)
 {
@@ -1724,7 +1724,7 @@ static const struct libipw_geo ipw_geos[] = {
 static int ipw2100_up(struct ipw2100_priv *priv, int deferred)
 {
 	unsigned long flags;
-	int rc = 0;
+	int err = 0;
 	u32 lock;
 	u32 ord_len = sizeof(lock);
 
@@ -1757,33 +1757,33 @@ static int ipw2100_up(struct ipw2100_priv *priv, int deferred)
 	if (priv->status & STATUS_POWERED ||
 	    (priv->status & STATUS_RESET_PENDING)) {
 		/* Power cycle the card ... */
-		if (ipw2100_power_cycle_adapter(priv)) {
+		err = ipw2100_power_cycle_adapter(priv);
+		if (err) {
 			printk(KERN_WARNING DRV_NAME
 			       ": %s: Could not cycle adapter.\n",
 			       priv->net_dev->name);
-			rc = 1;
 			goto exit;
 		}
 	} else
 		priv->status |= STATUS_POWERED;
 
 	/* Load the firmware, start the clocks, etc. */
-	if (ipw2100_start_adapter(priv)) {
+	err = ipw2100_start_adapter(priv);
+	if (err) {
 		printk(KERN_ERR DRV_NAME
 		       ": %s: Failed to start the firmware.\n",
 		       priv->net_dev->name);
-		rc = 1;
 		goto exit;
 	}
 
 	ipw2100_initialize_ordinals(priv);
 
 	/* Determine capabilities of this particular HW configuration */
-	if (ipw2100_get_hw_features(priv)) {
+	err = ipw2100_get_hw_features(priv);
+	if (err) {
 		printk(KERN_ERR DRV_NAME
 		       ": %s: Failed to determine HW features.\n",
 		       priv->net_dev->name);
-		rc = 1;
 		goto exit;
 	}
 
@@ -1792,11 +1792,11 @@ static int ipw2100_up(struct ipw2100_priv *priv, int deferred)
 	priv->ieee->freq_band = LIBIPW_24GHZ_BAND;
 
 	lock = LOCK_NONE;
-	if (ipw2100_set_ordinal(priv, IPW_ORD_PERS_DB_LOCK, &lock, &ord_len)) {
+	err = ipw2100_set_ordinal(priv, IPW_ORD_PERS_DB_LOCK, &lock, &ord_len);
+	if (err) {
 		printk(KERN_ERR DRV_NAME
 		       ": %s: Failed to clear ordinal lock.\n",
 		       priv->net_dev->name);
-		rc = 1;
 		goto exit;
 	}
 
@@ -1820,21 +1820,21 @@ static int ipw2100_up(struct ipw2100_priv *priv, int deferred)
 
 	/* Send all of the commands that must be sent prior to
 	 * HOST_COMPLETE */
-	if (ipw2100_adapter_setup(priv)) {
+	err = ipw2100_adapter_setup(priv);
+	if (err) {
 		printk(KERN_ERR DRV_NAME ": %s: Failed to start the card.\n",
 		       priv->net_dev->name);
-		rc = 1;
 		goto exit;
 	}
 
 	if (!deferred) {
 		/* Enable the adapter - sends HOST_COMPLETE */
-		if (ipw2100_enable_adapter(priv)) {
+		err = ipw2100_enable_adapter(priv);
+		if (err) {
 			printk(KERN_ERR DRV_NAME ": "
 			       "%s: failed in call to enable adapter.\n",
 			       priv->net_dev->name);
 			ipw2100_hw_stop_adapter(priv);
-			rc = 1;
 			goto exit;
 		}
 
@@ -1844,7 +1844,7 @@ static int ipw2100_up(struct ipw2100_priv *priv, int deferred)
 	}
 
       exit:
-	return rc;
+	return err;
 }
 
 static void ipw2100_down(struct ipw2100_priv *priv)
@@ -4324,7 +4324,7 @@ static struct attribute *ipw2100_sysfs_entries[] = {
 	NULL,
 };
 
-static struct attribute_group ipw2100_attribute_group = {
+static const struct attribute_group ipw2100_attribute_group = {
 	.attrs = ipw2100_sysfs_entries,
 };
 
@@ -8273,7 +8273,7 @@ static struct iw_statistics *ipw2100_wx_wireless_stats(struct net_device *dev)
 	return (struct iw_statistics *)NULL;
 }
 
-static struct iw_handler_def ipw2100_wx_handler_def = {
+static const struct iw_handler_def ipw2100_wx_handler_def = {
 	.standard = ipw2100_wx_handlers,
 	.num_standard = ARRAY_SIZE(ipw2100_wx_handlers),
 	.num_private = ARRAY_SIZE(ipw2100_private_handler),
diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2200.c b/drivers/net/wireless/intel/ipw2x00/ipw2200.c
index 9368abdf18e2..8da87496cb58 100644
--- a/drivers/net/wireless/intel/ipw2x00/ipw2200.c
+++ b/drivers/net/wireless/intel/ipw2x00/ipw2200.c
@@ -3209,7 +3209,7 @@ static int ipw_load_firmware(struct ipw_priv *priv, u8 * data, size_t len)
 	struct fw_chunk *chunk;
 	int total_nr = 0;
 	int i;
-	struct pci_pool *pool;
+	struct dma_pool *pool;
 	void **virts;
 	dma_addr_t *phys;
 
@@ -3226,9 +3226,10 @@ static int ipw_load_firmware(struct ipw_priv *priv, u8 * data, size_t len)
 		kfree(virts);
 		return -ENOMEM;
 	}
-	pool = pci_pool_create("ipw2200", priv->pci_dev, CB_MAX_LENGTH, 0, 0);
+	pool = dma_pool_create("ipw2200", &priv->pci_dev->dev, CB_MAX_LENGTH, 0,
+			       0);
 	if (!pool) {
-		IPW_ERROR("pci_pool_create failed\n");
+		IPW_ERROR("dma_pool_create failed\n");
 		kfree(phys);
 		kfree(virts);
 		return -ENOMEM;
@@ -3253,7 +3254,7 @@ static int ipw_load_firmware(struct ipw_priv *priv, u8 * data, size_t len)
 
 		nr = (chunk_len + CB_MAX_LENGTH - 1) / CB_MAX_LENGTH;
 		for (i = 0; i < nr; i++) {
-			virts[total_nr] = pci_pool_alloc(pool, GFP_KERNEL,
+			virts[total_nr] = dma_pool_alloc(pool, GFP_KERNEL,
 							 &phys[total_nr]);
 			if (!virts[total_nr]) {
 				ret = -ENOMEM;
@@ -3297,9 +3298,9 @@ static int ipw_load_firmware(struct ipw_priv *priv, u8 * data, size_t len)
 	}
  out:
 	for (i = 0; i < total_nr; i++)
-		pci_pool_free(pool, virts[i], phys[i]);
+		dma_pool_free(pool, virts[i], phys[i]);
 
-	pci_pool_destroy(pool);
+	dma_pool_destroy(pool);
 	kfree(phys);
 	kfree(virts);
 
@@ -10008,7 +10009,7 @@ static iw_handler ipw_priv_handler[] = {
 #endif
 };
 
-static struct iw_handler_def ipw_wx_handler_def = {
+static const struct iw_handler_def ipw_wx_handler_def = {
 	.standard = ipw_wx_handlers,
 	.num_standard = ARRAY_SIZE(ipw_wx_handlers),
 	.num_private = ARRAY_SIZE(ipw_priv_handler),
@@ -11500,7 +11501,7 @@ static struct attribute *ipw_sysfs_entries[] = {
 	NULL
 };
 
-static struct attribute_group ipw_attribute_group = {
+static const struct attribute_group ipw_attribute_group = {
 	.name = NULL,		/* put in device directory */
 	.attrs = ipw_sysfs_entries,
 };
diff --git a/drivers/net/wireless/intel/iwlegacy/3945-mac.c b/drivers/net/wireless/intel/iwlegacy/3945-mac.c
index 38bf403bb1e1..329f3a63dadd 100644
--- a/drivers/net/wireless/intel/iwlegacy/3945-mac.c
+++ b/drivers/net/wireless/intel/iwlegacy/3945-mac.c
@@ -3464,7 +3464,7 @@ static struct attribute *il3945_sysfs_entries[] = {
 	NULL
 };
 
-static struct attribute_group il3945_attribute_group = {
+static const struct attribute_group il3945_attribute_group = {
 	.name = NULL,		/* put in device directory */
 	.attrs = il3945_sysfs_entries,
 };
diff --git a/drivers/net/wireless/intel/iwlegacy/4965-mac.c b/drivers/net/wireless/intel/iwlegacy/4965-mac.c
index 5b51fba75595..de9b6522c43f 100644
--- a/drivers/net/wireless/intel/iwlegacy/4965-mac.c
+++ b/drivers/net/wireless/intel/iwlegacy/4965-mac.c
@@ -4654,7 +4654,7 @@ static struct attribute *il_sysfs_entries[] = {
 	NULL
 };
 
-static struct attribute_group il_attribute_group = {
+static const struct attribute_group il_attribute_group = {
 	.name = NULL,		/* put in device directory */
 	.attrs = il_sysfs_entries,
 };
diff --git a/drivers/net/wireless/intel/iwlwifi/Makefile b/drivers/net/wireless/intel/iwlwifi/Makefile
index 20bd261223af..35a32a3ec882 100644
--- a/drivers/net/wireless/intel/iwlwifi/Makefile
+++ b/drivers/net/wireless/intel/iwlwifi/Makefile
@@ -11,6 +11,8 @@ iwlwifi-$(CONFIG_IWLDVM) += cfg/1000.o cfg/2000.o cfg/5000.o cfg/6000.o
 iwlwifi-$(CONFIG_IWLMVM) += cfg/7000.o cfg/8000.o cfg/9000.o cfg/a000.o
 iwlwifi-objs		+= iwl-trans.o
 iwlwifi-objs		+= fw/notif-wait.o
+iwlwifi-$(CONFIG_IWLMVM) += fw/paging.o fw/smem.o fw/init.o fw/dbg.o
+iwlwifi-$(CONFIG_IWLMVM) += fw/common_rx.o fw/nvm.o
 
 iwlwifi-objs += $(iwlwifi-m)
 
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/8000.c b/drivers/net/wireless/intel/iwlwifi/cfg/8000.c
index 5081720608af..2e6c52664cee 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/8000.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/8000.c
@@ -70,8 +70,8 @@
 #include "iwl-agn-hw.h"
 
 /* Highest firmware API version supported */
-#define IWL8000_UCODE_API_MAX	33
-#define IWL8265_UCODE_API_MAX	33
+#define IWL8000_UCODE_API_MAX	34
+#define IWL8265_UCODE_API_MAX	34
 
 /* Lowest firmware API version supported */
 #define IWL8000_UCODE_API_MIN	22
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/9000.c b/drivers/net/wireless/intel/iwlwifi/cfg/9000.c
index b4ecd1fe1374..2babe0a1f18b 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/9000.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/9000.c
@@ -55,7 +55,7 @@
 #include "iwl-agn-hw.h"
 
 /* Highest firmware API version supported */
-#define IWL9000_UCODE_API_MAX	33
+#define IWL9000_UCODE_API_MAX	34
 
 /* Lowest firmware API version supported */
 #define IWL9000_UCODE_API_MIN	30
@@ -154,7 +154,7 @@ static const struct iwl_tt_params iwl9000_tt_params = {
 const struct iwl_cfg iwl9160_2ac_cfg = {
 	.name = "Intel(R) Dual Band Wireless AC 9160",
 	.fw_name_pre = IWL9260A_FW_PRE,
-	.fw_name_pre_next_step = IWL9260B_FW_PRE,
+	.fw_name_pre_b_or_c_step = IWL9260B_FW_PRE,
 	IWL_DEVICE_9000,
 	.ht_params = &iwl9000_ht_params,
 	.nvm_ver = IWL9000_NVM_VERSION,
@@ -165,7 +165,7 @@ const struct iwl_cfg iwl9160_2ac_cfg = {
 const struct iwl_cfg iwl9260_2ac_cfg = {
 	.name = "Intel(R) Dual Band Wireless AC 9260",
 	.fw_name_pre = IWL9260A_FW_PRE,
-	.fw_name_pre_next_step = IWL9260B_FW_PRE,
+	.fw_name_pre_b_or_c_step = IWL9260B_FW_PRE,
 	IWL_DEVICE_9000,
 	.ht_params = &iwl9000_ht_params,
 	.nvm_ver = IWL9000_NVM_VERSION,
@@ -176,7 +176,7 @@ const struct iwl_cfg iwl9260_2ac_cfg = {
 const struct iwl_cfg iwl9270_2ac_cfg = {
 	.name = "Intel(R) Dual Band Wireless AC 9270",
 	.fw_name_pre = IWL9260A_FW_PRE,
-	.fw_name_pre_next_step = IWL9260B_FW_PRE,
+	.fw_name_pre_b_or_c_step = IWL9260B_FW_PRE,
 	IWL_DEVICE_9000,
 	.ht_params = &iwl9000_ht_params,
 	.nvm_ver = IWL9000_NVM_VERSION,
@@ -186,8 +186,8 @@ const struct iwl_cfg iwl9270_2ac_cfg = {
 
 const struct iwl_cfg iwl9460_2ac_cfg = {
 	.name = "Intel(R) Dual Band Wireless AC 9460",
-	.fw_name_pre = IWL9000_FW_PRE,
-	.fw_name_pre_rf_next_step = IWL9000RFB_FW_PRE,
+	.fw_name_pre = IWL9260A_FW_PRE,
+	.fw_name_pre_b_or_c_step = IWL9260B_FW_PRE,
 	IWL_DEVICE_9000,
 	.ht_params = &iwl9000_ht_params,
 	.nvm_ver = IWL9000_NVM_VERSION,
@@ -198,8 +198,8 @@ const struct iwl_cfg iwl9460_2ac_cfg = {
 
 const struct iwl_cfg iwl9560_2ac_cfg = {
 	.name = "Intel(R) Dual Band Wireless AC 9560",
-	.fw_name_pre = IWL9000_FW_PRE,
-	.fw_name_pre_rf_next_step = IWL9000RFB_FW_PRE,
+	.fw_name_pre = IWL9260A_FW_PRE,
+	.fw_name_pre_b_or_c_step = IWL9260B_FW_PRE,
 	IWL_DEVICE_9000,
 	.ht_params = &iwl9000_ht_params,
 	.nvm_ver = IWL9000_NVM_VERSION,
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/a000.c b/drivers/net/wireless/intel/iwlwifi/cfg/a000.c
index 98f24cd1b44f..76ba1f8bc72f 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/a000.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/a000.c
@@ -55,7 +55,7 @@
 #include "iwl-agn-hw.h"
 
 /* Highest firmware API version supported */
-#define IWL_A000_UCODE_API_MAX	33
+#define IWL_A000_UCODE_API_MAX	34
 
 /* Lowest firmware API version supported */
 #define IWL_A000_UCODE_API_MIN	24
@@ -75,11 +75,20 @@
 #define IWL_A000_JF_FW_PRE	"iwlwifi-Qu-a0-jf-b0-"
 #define IWL_A000_HR_FW_PRE	"iwlwifi-Qu-a0-hr-a0-"
 #define IWL_A000_HR_CDB_FW_PRE	"iwlwifi-QuIcp-z0-hrcdb-a0-"
+#define IWL_A000_HR_F0_FW_PRE	"iwlwifi-QuQnj-f0-hr-a0-"
+#define IWL_A000_JF_B0_FW_PRE	"iwlwifi-QuQnj-a0-jf-b0-"
+#define IWL_A000_HR_A0_FW_PRE	"iwlwifi-QuQnj-a0-hr-a0-"
 
 #define IWL_A000_HR_MODULE_FIRMWARE(api) \
 	IWL_A000_HR_FW_PRE "-" __stringify(api) ".ucode"
 #define IWL_A000_JF_MODULE_FIRMWARE(api) \
 	IWL_A000_JF_FW_PRE "-" __stringify(api) ".ucode"
+#define IWL_A000_HR_F0_QNJ_MODULE_FIRMWARE(api) \
+	IWL_A000_HR_F0_FW_PRE "-" __stringify(api) ".ucode"
+#define IWL_A000_JF_B0_QNJ_MODULE_FIRMWARE(api) \
+	IWL_A000_JF_B0_FW_PRE "-" __stringify(api) ".ucode"
+#define IWL_A000_HR_A0_QNJ_MODULE_FIRMWARE(api) \
+	IWL_A000_HR_A0_FW_PRE "-" __stringify(api) ".ucode"
 
 #define NVM_HW_SECTION_NUM_FAMILY_A000		10
 
@@ -168,5 +177,38 @@ const struct iwl_cfg iwla000_2ax_cfg_hr = {
 		.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
 };
 
+const struct iwl_cfg iwla000_2ax_cfg_qnj_hr_f0 = {
+		.name = "Intel(R) Dual Band Wireless AX a000",
+		.fw_name_pre = IWL_A000_HR_F0_FW_PRE,
+		IWL_DEVICE_A000,
+		.ht_params = &iwl_a000_ht_params,
+		.nvm_ver = IWL_A000_NVM_VERSION,
+		.nvm_calib_ver = IWL_A000_TX_POWER_VERSION,
+		.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
+};
+
+const struct iwl_cfg iwla000_2ax_cfg_qnj_jf_b0 = {
+		.name = "Intel(R) Dual Band Wireless AX a000",
+		.fw_name_pre = IWL_A000_JF_B0_FW_PRE,
+		IWL_DEVICE_A000,
+		.ht_params = &iwl_a000_ht_params,
+		.nvm_ver = IWL_A000_NVM_VERSION,
+		.nvm_calib_ver = IWL_A000_TX_POWER_VERSION,
+		.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
+};
+
+const struct iwl_cfg iwla000_2ax_cfg_qnj_hr_a0 = {
+		.name = "Intel(R) Dual Band Wireless AX a000",
+		.fw_name_pre = IWL_A000_HR_A0_FW_PRE,
+		IWL_DEVICE_A000,
+		.ht_params = &iwl_a000_ht_params,
+		.nvm_ver = IWL_A000_NVM_VERSION,
+		.nvm_calib_ver = IWL_A000_TX_POWER_VERSION,
+		.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
+};
+
 MODULE_FIRMWARE(IWL_A000_HR_MODULE_FIRMWARE(IWL_A000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_A000_JF_MODULE_FIRMWARE(IWL_A000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_A000_HR_F0_QNJ_MODULE_FIRMWARE(IWL_A000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_A000_JF_B0_QNJ_MODULE_FIRMWARE(IWL_A000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_A000_HR_A0_QNJ_MODULE_FIRMWARE(IWL_A000_UCODE_API_MAX));
diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/commands.h b/drivers/net/wireless/intel/iwlwifi/dvm/commands.h
index 2ab2773655a8..f89736d60a3d 100644
--- a/drivers/net/wireless/intel/iwlwifi/dvm/commands.h
+++ b/drivers/net/wireless/intel/iwlwifi/dvm/commands.h
@@ -311,11 +311,6 @@ enum {
 
 /**
  * rate_n_flags Tx antenna masks
- * 4965 has 2 transmitters
- * 5100 has 1 transmitter B
- * 5150 has 1 transmitter A
- * 5300 has 3 transmitters
- * 5350 has 3 transmitters
  * bit14:16
  */
 #define RATE_MCS_ANT_POS	14
@@ -1230,7 +1225,6 @@ struct iwl_rx_mpdu_res_start {
  */
 
 /*
- * 4965 uCode updates these Tx attempt count values in host DRAM.
  * Used for managing Tx retries when expecting block-acks.
  * Driver should set these fields to 0.
  */
@@ -1437,22 +1431,6 @@ struct agg_tx_status {
 	__le16 sequence;
 } __packed;
 
-/*
- * definitions for initial rate index field
- * bits [3:0] initial rate index
- * bits [6:4] rate table color, used for the initial rate
- * bit-7 invalid rate indication
- *   i.e. rate was not chosen from rate table
- *   or rate table color was changed during frame retries
- * refer tlc rate info
- */
-
-#define IWL50_TX_RES_INIT_RATE_INDEX_POS	0
-#define IWL50_TX_RES_INIT_RATE_INDEX_MSK	0x0f
-#define IWL50_TX_RES_RATE_TABLE_COLOR_POS	4
-#define IWL50_TX_RES_RATE_TABLE_COLOR_MSK	0x70
-#define IWL50_TX_RES_INV_RATE_INDEX_MSK	0x80
-
 /* refer to ra_tid */
 #define IWLAGN_TX_RES_TID_POS	0
 #define IWLAGN_TX_RES_TID_MSK	0x0f
@@ -1556,7 +1534,7 @@ struct iwl_link_qual_general_params {
 	/* Best single antenna to use for single stream (legacy, SISO). */
 	u8 single_stream_ant_msk;	/* LINK_QUAL_ANT_* */
 
-	/* Best antennas to use for MIMO (unused for 4965, assumes both). */
+	/* Best antennas to use for MIMO */
 	u8 dual_stream_ant_msk;		/* LINK_QUAL_ANT_* */
 
 	/*
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/alive.h b/drivers/net/wireless/intel/iwlwifi/fw/api/alive.h
new file mode 100644
index 000000000000..3684a3e180e5
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/alive.h
@@ -0,0 +1,206 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+
+#ifndef __iwl_fw_api_alive_h__
+#define __iwl_fw_api_alive_h__
+
+/* alive response is_valid values */
+#define ALIVE_RESP_UCODE_OK	BIT(0)
+#define ALIVE_RESP_RFKILL	BIT(1)
+
+/* alive response ver_type values */
+enum {
+	FW_TYPE_HW = 0,
+	FW_TYPE_PROT = 1,
+	FW_TYPE_AP = 2,
+	FW_TYPE_WOWLAN = 3,
+	FW_TYPE_TIMING = 4,
+	FW_TYPE_WIPAN = 5
+};
+
+/* alive response ver_subtype values */
+enum {
+	FW_SUBTYPE_FULL_FEATURE = 0,
+	FW_SUBTYPE_BOOTSRAP = 1, /* Not valid */
+	FW_SUBTYPE_REDUCED = 2,
+	FW_SUBTYPE_ALIVE_ONLY = 3,
+	FW_SUBTYPE_WOWLAN = 4,
+	FW_SUBTYPE_AP_SUBTYPE = 5,
+	FW_SUBTYPE_WIPAN = 6,
+	FW_SUBTYPE_INITIALIZE = 9
+};
+
+#define IWL_ALIVE_STATUS_ERR 0xDEAD
+#define IWL_ALIVE_STATUS_OK 0xCAFE
+
+#define IWL_ALIVE_FLG_RFKILL	BIT(0)
+
+struct iwl_lmac_alive {
+	__le32 ucode_minor;
+	__le32 ucode_major;
+	u8 ver_subtype;
+	u8 ver_type;
+	u8 mac;
+	u8 opt;
+	__le32 timestamp;
+	__le32 error_event_table_ptr;	/* SRAM address for error log */
+	__le32 log_event_table_ptr;	/* SRAM address for LMAC event log */
+	__le32 cpu_register_ptr;
+	__le32 dbgm_config_ptr;
+	__le32 alive_counter_ptr;
+	__le32 scd_base_ptr;		/* SRAM address for SCD */
+	__le32 st_fwrd_addr;		/* pointer to Store and forward */
+	__le32 st_fwrd_size;
+} __packed; /* UCODE_ALIVE_NTFY_API_S_VER_3 */
+
+struct iwl_umac_alive {
+	__le32 umac_minor;		/* UMAC version: minor */
+	__le32 umac_major;		/* UMAC version: major */
+	__le32 error_info_addr;		/* SRAM address for UMAC error log */
+	__le32 dbg_print_buff_addr;
+} __packed; /* UMAC_ALIVE_DATA_API_S_VER_2 */
+
+struct mvm_alive_resp_v3 {
+	__le16 status;
+	__le16 flags;
+	struct iwl_lmac_alive lmac_data;
+	struct iwl_umac_alive umac_data;
+} __packed; /* ALIVE_RES_API_S_VER_3 */
+
+struct mvm_alive_resp {
+	__le16 status;
+	__le16 flags;
+	struct iwl_lmac_alive lmac_data[2];
+	struct iwl_umac_alive umac_data;
+} __packed; /* ALIVE_RES_API_S_VER_4 */
+
+/**
+ * enum iwl_extended_cfg_flag - commands driver may send before
+ *	finishing init flow
+ * @IWL_INIT_DEBUG_CFG: driver is going to send debug config command
+ * @IWL_INIT_NVM: driver is going to send NVM_ACCESS commands
+ * @IWL_INIT_PHY: driver is going to send PHY_DB commands
+ */
+enum iwl_extended_cfg_flags {
+	IWL_INIT_DEBUG_CFG,
+	IWL_INIT_NVM,
+	IWL_INIT_PHY,
+};
+
+/**
+ * struct iwl_extended_cfg_cmd - mark what commands ucode should wait for
+ * before finishing init flows
+ * @init_flags: values from iwl_extended_cfg_flags
+ */
+struct iwl_init_extended_cfg_cmd {
+	__le32 init_flags;
+} __packed; /* INIT_EXTENDED_CFG_CMD_API_S_VER_1 */
+
+/**
+ * struct iwl_radio_version_notif - information on the radio version
+ * ( RADIO_VERSION_NOTIFICATION = 0x68 )
+ * @radio_flavor: radio flavor
+ * @radio_step: radio version step
+ * @radio_dash: radio version dash
+ */
+struct iwl_radio_version_notif {
+	__le32 radio_flavor;
+	__le32 radio_step;
+	__le32 radio_dash;
+} __packed; /* RADIO_VERSION_NOTOFICATION_S_VER_1 */
+
+enum iwl_card_state_flags {
+	CARD_ENABLED		= 0x00,
+	HW_CARD_DISABLED	= 0x01,
+	SW_CARD_DISABLED	= 0x02,
+	CT_KILL_CARD_DISABLED	= 0x04,
+	HALT_CARD_DISABLED	= 0x08,
+	CARD_DISABLED_MSK	= 0x0f,
+	CARD_IS_RX_ON		= 0x10,
+};
+
+/**
+ * struct iwl_radio_version_notif - information on the card state
+ * ( CARD_STATE_NOTIFICATION = 0xa1 )
+ * @flags: &enum iwl_card_state_flags
+ */
+struct iwl_card_state_notif {
+	__le32 flags;
+} __packed; /* CARD_STATE_NTFY_API_S_VER_1 */
+
+/**
+ * struct iwl_fseq_ver_mismatch_nty - Notification about version
+ *
+ * This notification does not have a direct impact on the init flow.
+ * It means that another core (not WiFi) has initiated the FSEQ flow
+ * and updated the FSEQ version.  The driver only prints an error when
+ * this occurs.
+ *
+ * @aux_read_fseq_ver: auxiliary read FSEQ version
+ * @wifi_fseq_ver: FSEQ version (embedded in WiFi)
+ */
+struct iwl_fseq_ver_mismatch_ntf {
+	__le32 aux_read_fseq_ver;
+	__le32 wifi_fseq_ver;
+} __packed; /* FSEQ_VER_MISMATCH_NTFY_API_S_VER_1 */
+
+#endif /* __iwl_fw_api_alive_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/binding.h b/drivers/net/wireless/intel/iwlwifi/fw/api/binding.h
new file mode 100644
index 000000000000..d2717fafdf5b
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/binding.h
@@ -0,0 +1,144 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+
+#ifndef __iwl_fw_api_binding_h__
+#define __iwl_fw_api_binding_h__
+
+#define MAX_MACS_IN_BINDING	(3)
+#define MAX_BINDINGS		(4)
+
+/**
+ * struct iwl_binding_cmd_v1 - configuring bindings
+ * ( BINDING_CONTEXT_CMD = 0x2b )
+ * @id_and_color: ID and color of the relevant Binding,
+ *	&enum iwl_ctxt_id_and_color
+ * @action: action to perform, one of FW_CTXT_ACTION_*
+ * @macs: array of MAC id and colors which belong to the binding,
+ *	&enum iwl_ctxt_id_and_color
+ * @phy: PHY id and color which belongs to the binding,
+ *	&enum iwl_ctxt_id_and_color
+ */
+struct iwl_binding_cmd_v1 {
+	/* COMMON_INDEX_HDR_API_S_VER_1 */
+	__le32 id_and_color;
+	__le32 action;
+	/* BINDING_DATA_API_S_VER_1 */
+	__le32 macs[MAX_MACS_IN_BINDING];
+	__le32 phy;
+} __packed; /* BINDING_CMD_API_S_VER_1 */
+
+/**
+ * struct iwl_binding_cmd - configuring bindings
+ * ( BINDING_CONTEXT_CMD = 0x2b )
+ * @id_and_color: ID and color of the relevant Binding,
+ *	&enum iwl_ctxt_id_and_color
+ * @action: action to perform, one of FW_CTXT_ACTION_*
+ * @macs: array of MAC id and colors which belong to the binding
+ *	&enum iwl_ctxt_id_and_color
+ * @phy: PHY id and color which belongs to the binding
+ *	&enum iwl_ctxt_id_and_color
+ * @lmac_id: the lmac id the binding belongs to
+ */
+struct iwl_binding_cmd {
+	/* COMMON_INDEX_HDR_API_S_VER_1 */
+	__le32 id_and_color;
+	__le32 action;
+	/* BINDING_DATA_API_S_VER_1 */
+	__le32 macs[MAX_MACS_IN_BINDING];
+	__le32 phy;
+	__le32 lmac_id;
+} __packed; /* BINDING_CMD_API_S_VER_2 */
+
+#define IWL_BINDING_CMD_SIZE_V1	sizeof(struct iwl_binding_cmd_v1)
+#define IWL_LMAC_24G_INDEX		0
+#define IWL_LMAC_5G_INDEX		1
+
+/* The maximal number of fragments in the FW's schedule session */
+#define IWL_MVM_MAX_QUOTA 128
+
+/**
+ * struct iwl_time_quota_data - configuration of time quota per binding
+ * @id_and_color: ID and color of the relevant Binding,
+ *	&enum iwl_ctxt_id_and_color
+ * @quota: absolute time quota in TU. The scheduler will try to divide the
+ *	remainig quota (after Time Events) according to this quota.
+ * @max_duration: max uninterrupted context duration in TU
+ */
+struct iwl_time_quota_data {
+	__le32 id_and_color;
+	__le32 quota;
+	__le32 max_duration;
+} __packed; /* TIME_QUOTA_DATA_API_S_VER_1 */
+
+/**
+ * struct iwl_time_quota_cmd - configuration of time quota between bindings
+ * ( TIME_QUOTA_CMD = 0x2c )
+ * @quotas: allocations per binding
+ * Note: on non-CDB the fourth one is the auxilary mac and is
+ *	essentially zero.
+ *	On CDB the fourth one is a regular binding.
+ */
+struct iwl_time_quota_cmd {
+	struct iwl_time_quota_data quotas[MAX_BINDINGS];
+} __packed; /* TIME_QUOTA_ALLOCATION_CMD_API_S_VER_1 */
+
+#endif /* __iwl_fw_api_binding_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api.h b/drivers/net/wireless/intel/iwlwifi/fw/api/cmdhdr.h
index 0e107f916ce3..ea4a3f04a83a 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/cmdhdr.h
@@ -59,8 +59,8 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  *****************************************************************************/
-#ifndef __iwl_fw_api_h__
-#define __iwl_fw_api_h__
+#ifndef __iwl_fw_api_cmdhdr_h__
+#define __iwl_fw_api_cmdhdr_h__
 
 /**
  * DOC: Host command section
@@ -112,15 +112,24 @@ static inline u32 iwl_cmd_id(u8 opcode, u8 groupid, u8 version)
 #define IWL_ALWAYS_LONG_GROUP	1
 
 /**
- * struct iwl_cmd_header
+ * struct iwl_cmd_header - (short) command header format
  *
  * This header format appears in the beginning of each command sent from the
  * driver, and each response/notification received from uCode.
  */
 struct iwl_cmd_header {
-	u8 cmd;		/* Command ID:  REPLY_RXON, etc. */
+	/**
+	 * @cmd: Command ID: REPLY_RXON, etc.
+	 */
+	u8 cmd;
+	/**
+	 * @group_id: group ID, for commands with groups
+	 */
 	u8 group_id;
-	/*
+	/**
+	 * @sequence:
+	 * Sequence number for the command.
+	 *
 	 * The driver sets up the sequence number to values of its choosing.
 	 * uCode does not use this value, but passes it back to the driver
 	 * when sending the response to each driver-originated command, so
@@ -150,6 +159,13 @@ struct iwl_cmd_header {
  * driver, and each response/notification received from uCode.
  * this is the wide version that contains more information about the command
  * like length, version and command type
+ *
+ * @cmd: command ID, like in &struct iwl_cmd_header
+ * @group_id: group ID, like in &struct iwl_cmd_header
+ * @sequence: sequence, like in &struct iwl_cmd_header
+ * @length: length of the command
+ * @reserved: reserved
+ * @version: command version
  */
 struct iwl_cmd_header_wide {
 	u8 cmd;
@@ -161,48 +177,6 @@ struct iwl_cmd_header_wide {
 } __packed;
 
 /**
- * iwl_tx_queue_cfg_actions - TXQ config options
- * @TX_QUEUE_CFG_ENABLE_QUEUE: enable a queue
- * @TX_QUEUE_CFG_TFD_SHORT_FORMAT: use short TFD format
- */
-enum iwl_tx_queue_cfg_actions {
-	TX_QUEUE_CFG_ENABLE_QUEUE		= BIT(0),
-	TX_QUEUE_CFG_TFD_SHORT_FORMAT		= BIT(1),
-};
-
-/**
- * struct iwl_tx_queue_cfg_cmd - txq hw scheduler config command
- * @sta_id: station id
- * @tid: tid of the queue
- * @flags: see &enum iwl_tx_queue_cfg_actions
- * @cb_size: size of TFD cyclic buffer. Value is exponent - 3.
- *	Minimum value 0 (8 TFDs), maximum value 5 (256 TFDs)
- * @byte_cnt_addr: address of byte count table
- * @tfdq_addr: address of TFD circular buffer
- */
-struct iwl_tx_queue_cfg_cmd {
-	u8 sta_id;
-	u8 tid;
-	__le16 flags;
-	__le32 cb_size;
-	__le64 byte_cnt_addr;
-	__le64 tfdq_addr;
-} __packed; /* TX_QUEUE_CFG_CMD_API_S_VER_2 */
-
-/**
- * struct iwl_tx_queue_cfg_rsp - response to txq hw scheduler config
- * @queue_number: queue number assigned to this RA -TID
- * @flags: set on failure
- * @write_pointer: initial value for write pointer
- */
-struct iwl_tx_queue_cfg_rsp {
-	__le16 queue_number;
-	__le16 flags;
-	__le16 write_pointer;
-	__le16 reserved;
-} __packed; /* TX_QUEUE_CFG_RSP_API_S_VER_2 */
-
-/**
  * struct iwl_calib_res_notif_phy_db - Receive phy db chunk after calibrations
  * @type: type of the result - mostly ignored
  * @length: length of the data
@@ -226,4 +200,12 @@ struct iwl_phy_db_cmd {
 	u8 data[];
 } __packed;
 
-#endif /* __iwl_fw_api_h__*/
+/**
+ * struct iwl_cmd_response - generic response struct for most commands
+ * @status: status of the command asked, changes for each one
+ */
+struct iwl_cmd_response {
+	__le32 status;
+};
+
+#endif /* __iwl_fw_api_cmdhdr_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-coex.h b/drivers/net/wireless/intel/iwlwifi/fw/api/coex.h
index 8cd06aaa1f54..d09555afe2c5 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-coex.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/coex.h
@@ -18,11 +18,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
- * USA
- *
  * The full GNU General Public License is included in this distribution
  * in the file called COPYING.
  *
@@ -64,8 +59,8 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *****************************************************************************/
 
-#ifndef __fw_api_bt_coex_h__
-#define __fw_api_bt_coex_h__
+#ifndef __iwl_fw_api_coex_h__
+#define __iwl_fw_api_coex_h__
 
 #include <linux/types.h>
 #include <linux/bitops.h>
@@ -81,7 +76,6 @@ enum iwl_bt_coex_lut_type {
 	BT_COEX_INVALID_LUT = 0xff,
 }; /* BT_COEX_DECISION_LUT_INDEX_API_E_VER_1 */
 
-#define BT_COEX_CORUN_LUT_SIZE (32)
 #define BT_REDUCED_TX_POWER_BIT BIT(7)
 
 enum iwl_bt_coex_mode {
@@ -112,18 +106,6 @@ struct iwl_bt_coex_cmd {
 } __packed; /* BT_COEX_CMD_API_S_VER_6 */
 
 /**
- * struct iwl_bt_coex_corun_lut_update - bt coex update the corun lut
- * @corun_lut20: co-running 20 MHz LUT configuration
- * @corun_lut40: co-running 40 MHz LUT configuration
- *
- * The structure is used for the BT_COEX_UPDATE_CORUN_LUT command.
- */
-struct iwl_bt_coex_corun_lut_update_cmd {
-	__le32 corun_lut20[BT_COEX_CORUN_LUT_SIZE];
-	__le32 corun_lut40[BT_COEX_CORUN_LUT_SIZE];
-} __packed; /* BT_COEX_UPDATE_CORUN_LUT_API_S_VER_1 */
-
-/**
  * struct iwl_bt_coex_reduced_txp_update_cmd
  * @reduced_txp: bit BT_REDUCED_TX_POWER_BIT to enable / disable, rest of the
  *	bits are the sta_id (value)
@@ -196,6 +178,7 @@ enum iwl_bt_mxbox_dw3 {
 	BT_MBOX(3, ACL_STATE, 3, 1),
 	BT_MBOX(3, MSTR_STATE, 4, 1),
 	BT_MBOX(3, OBX_STATE, 5, 1),
+	BT_MBOX(3, A2DP_SRC, 6, 1),
 	BT_MBOX(3, OPEN_CON_2, 8, 2),
 	BT_MBOX(3, TRAFFIC_LOAD, 10, 2),
 	BT_MBOX(3, CHL_SEQN_LSB, 12, 1),
@@ -205,10 +188,21 @@ enum iwl_bt_mxbox_dw3 {
 	BT_MBOX(3, UPDATE_REQUEST, 21, 1),
 };
 
+enum iwl_bt_mxbox_dw4 {
+	BT_MBOX(4, ATS_BT_INTERVAL, 0, 7),
+	BT_MBOX(4, ATS_BT_ACTIVE_MAX_TH, 7, 7),
+};
+
 #define BT_MBOX_MSG(_notif, _num, _field)				     \
 	((le32_to_cpu((_notif)->mbox_msg[(_num)]) & BT_MBOX##_num##_##_field)\
 	>> BT_MBOX##_num##_##_field##_POS)
 
+#define BT_MBOX_PRINT(_num, _field, _end)				    \
+			pos += scnprintf(buf + pos, bufsz - pos,	    \
+					 "\t%s: %d%s",			    \
+					 #_field,			    \
+					 BT_MBOX_MSG(notif, _num, _field),  \
+					 true ? "\n" : ", ");
 enum iwl_bt_activity_grading {
 	BT_OFF			= 0,
 	BT_ON_NO_CONNECTION	= 1,
@@ -225,11 +219,30 @@ enum iwl_bt_ci_compliance {
 	BT_CI_COMPLIANCE_BOTH		= 3,
 }; /* BT_COEX_CI_COMPLIENCE_E_VER_1 */
 
-#define IWL_COEX_IS_TTC_ON(_ttc_rrc_status, _phy_id)	\
-		(_ttc_rrc_status & BIT(_phy_id))
+/**
+ * struct iwl_bt_coex_profile_notif - notification about BT coex
+ * @mbox_msg: message from BT to WiFi
+ * @msg_idx: the index of the message
+ * @bt_ci_compliance: enum %iwl_bt_ci_compliance
+ * @primary_ch_lut: LUT used for primary channel &enum iwl_bt_coex_lut_type
+ * @secondary_ch_lut: LUT used for secondary channel &enum iwl_bt_coex_lut_type
+ * @bt_activity_grading: the activity of BT &enum iwl_bt_activity_grading
+ * @ttc_status: is TTC enabled - one bit per PHY
+ * @rrc_status: is RRC enabled - one bit per PHY
+ * @reserved: reserved
+ */
+struct iwl_bt_coex_profile_notif {
+	__le32 mbox_msg[8];
+	__le32 msg_idx;
+	__le32 bt_ci_compliance;
 
-#define IWL_COEX_IS_RRC_ON(_ttc_rrc_status, _phy_id)	\
-		((_ttc_rrc_status >> 4) & BIT(_phy_id))
+	__le32 primary_ch_lut;
+	__le32 secondary_ch_lut;
+	__le32 bt_activity_grading;
+	u8 ttc_status;
+	u8 rrc_status;
+	__le16 reserved;
+} __packed; /* BT_COEX_PROFILE_NTFY_API_S_VER_5 */
 
 /**
  * struct iwl_bt_coex_profile_notif - notification about BT coex
@@ -239,10 +252,11 @@ enum iwl_bt_ci_compliance {
  * @primary_ch_lut: LUT used for primary channel &enum iwl_bt_coex_lut_type
  * @secondary_ch_lut: LUT used for secondary channel &enum iwl_bt_coex_lut_type
  * @bt_activity_grading: the activity of BT &enum iwl_bt_activity_grading
- * @ttc_rrc_status: is TTC or RRC enabled - one bit per PHY
+ * @ttc_status: is TTC enabled - one bit per PHY
+ * @rrc_status: is RRC enabled - one bit per PHY
  * @reserved: reserved
  */
-struct iwl_bt_coex_profile_notif {
+struct iwl_bt_coex_profile_notif_v4 {
 	__le32 mbox_msg[4];
 	__le32 msg_idx;
 	__le32 bt_ci_compliance;
@@ -250,8 +264,9 @@ struct iwl_bt_coex_profile_notif {
 	__le32 primary_ch_lut;
 	__le32 secondary_ch_lut;
 	__le32 bt_activity_grading;
-	u8 ttc_rrc_status;
-	u8 reserved[3];
+	u8 ttc_status;
+	u8 rrc_status;
+	__le16 reserved;
 } __packed; /* BT_COEX_PROFILE_NTFY_API_S_VER_4 */
 
-#endif /* __fw_api_bt_coex_h__ */
+#endif /* __iwl_fw_api_coex_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h b/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h
new file mode 100644
index 000000000000..074868394427
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h
@@ -0,0 +1,657 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+
+#ifndef __iwl_fw_api_commands_h__
+#define __iwl_fw_api_commands_h__
+
+/**
+ * enum iwl_mvm_command_groups - command groups for the firmware
+ * @LEGACY_GROUP: legacy group, uses command IDs from &enum iwl_legacy_cmds
+ * @LONG_GROUP: legacy group with long header, also uses command IDs
+ *	from &enum iwl_legacy_cmds
+ * @SYSTEM_GROUP: system group, uses command IDs from
+ *	&enum iwl_system_subcmd_ids
+ * @MAC_CONF_GROUP: MAC configuration group, uses command IDs from
+ *	&enum iwl_mac_conf_subcmd_ids
+ * @PHY_OPS_GROUP: PHY operations group, uses command IDs from
+ *	&enum iwl_phy_ops_subcmd_ids
+ * @DATA_PATH_GROUP: data path group, uses command IDs from
+ *	&enum iwl_data_path_subcmd_ids
+ * @NAN_GROUP: NAN group, uses command IDs from &enum iwl_nan_subcmd_ids
+ * @TOF_GROUP: TOF group, uses command IDs from &enum iwl_tof_subcmd_ids
+ * @PROT_OFFLOAD_GROUP: protocol offload group, uses command IDs from
+ *	&enum iwl_prot_offload_subcmd_ids
+ * @REGULATORY_AND_NVM_GROUP: regulatory/NVM group, uses command IDs from
+ *	&enum iwl_regulatory_and_nvm_subcmd_ids
+ * @DEBUG_GROUP: Debug group, uses command IDs from &enum iwl_debug_cmds
+ */
+enum iwl_mvm_command_groups {
+	LEGACY_GROUP = 0x0,
+	LONG_GROUP = 0x1,
+	SYSTEM_GROUP = 0x2,
+	MAC_CONF_GROUP = 0x3,
+	PHY_OPS_GROUP = 0x4,
+	DATA_PATH_GROUP = 0x5,
+	NAN_GROUP = 0x7,
+	TOF_GROUP = 0x8,
+	PROT_OFFLOAD_GROUP = 0xb,
+	REGULATORY_AND_NVM_GROUP = 0xc,
+	DEBUG_GROUP = 0xf,
+};
+
+/**
+ * enum iwl_legacy_cmds - legacy group command IDs
+ */
+enum iwl_legacy_cmds {
+	/**
+	 * @MVM_ALIVE:
+	 * Alive data from the firmware, as described in
+	 * &struct mvm_alive_resp_v3 or &struct mvm_alive_resp.
+	 */
+	MVM_ALIVE = 0x1,
+
+	/**
+	 * @REPLY_ERROR: Cause an error in the firmware, for testing purposes.
+	 */
+	REPLY_ERROR = 0x2,
+
+	/**
+	 * @ECHO_CMD: Send data to the device to have it returned immediately.
+	 */
+	ECHO_CMD = 0x3,
+
+	/**
+	 * @INIT_COMPLETE_NOTIF: Notification that initialization is complete.
+	 */
+	INIT_COMPLETE_NOTIF = 0x4,
+
+	/**
+	 * @PHY_CONTEXT_CMD:
+	 * Add/modify/remove a PHY context, using &struct iwl_phy_context_cmd.
+	 */
+	PHY_CONTEXT_CMD = 0x8,
+
+	/**
+	 * @DBG_CFG: Debug configuration command.
+	 */
+	DBG_CFG = 0x9,
+
+	/**
+	 * @SCAN_ITERATION_COMPLETE_UMAC:
+	 * Firmware indicates a scan iteration completed, using
+	 * &struct iwl_umac_scan_iter_complete_notif.
+	 */
+	SCAN_ITERATION_COMPLETE_UMAC = 0xb5,
+
+	/**
+	 * @SCAN_CFG_CMD:
+	 * uses &struct iwl_scan_config_v1 or &struct iwl_scan_config
+	 */
+	SCAN_CFG_CMD = 0xc,
+
+	/**
+	 * @SCAN_REQ_UMAC: uses &struct iwl_scan_req_umac
+	 */
+	SCAN_REQ_UMAC = 0xd,
+
+	/**
+	 * @SCAN_ABORT_UMAC: uses &struct iwl_umac_scan_abort
+	 */
+	SCAN_ABORT_UMAC = 0xe,
+
+	/**
+	 * @SCAN_COMPLETE_UMAC: uses &struct iwl_umac_scan_complete
+	 */
+	SCAN_COMPLETE_UMAC = 0xf,
+
+	/**
+	 * @BA_WINDOW_STATUS_NOTIFICATION_ID:
+	 * uses &struct iwl_ba_window_status_notif
+	 */
+	BA_WINDOW_STATUS_NOTIFICATION_ID = 0x13,
+
+	/**
+	 * @ADD_STA_KEY:
+	 * &struct iwl_mvm_add_sta_key_cmd_v1 or
+	 * &struct iwl_mvm_add_sta_key_cmd.
+	 */
+	ADD_STA_KEY = 0x17,
+
+	/**
+	 * @ADD_STA:
+	 * &struct iwl_mvm_add_sta_cmd or &struct iwl_mvm_add_sta_cmd_v7.
+	 */
+	ADD_STA = 0x18,
+
+	/**
+	 * @REMOVE_STA: &struct iwl_mvm_rm_sta_cmd
+	 */
+	REMOVE_STA = 0x19,
+
+	/**
+	 * @FW_GET_ITEM_CMD: uses &struct iwl_fw_get_item_cmd
+	 */
+	FW_GET_ITEM_CMD = 0x1a,
+
+	/**
+	 * @TX_CMD: uses &struct iwl_tx_cmd or &struct iwl_tx_cmd_gen2,
+	 *	response in &struct iwl_mvm_tx_resp or
+	 *	&struct iwl_mvm_tx_resp_v3
+	 */
+	TX_CMD = 0x1c,
+
+	/**
+	 * @TXPATH_FLUSH: &struct iwl_tx_path_flush_cmd
+	 */
+	TXPATH_FLUSH = 0x1e,
+
+	/**
+	 * @MGMT_MCAST_KEY:
+	 * &struct iwl_mvm_mgmt_mcast_key_cmd or
+	 * &struct iwl_mvm_mgmt_mcast_key_cmd_v1
+	 */
+	MGMT_MCAST_KEY = 0x1f,
+
+	/* scheduler config */
+	/**
+	 * @SCD_QUEUE_CFG: &struct iwl_scd_txq_cfg_cmd for older hardware,
+	 *	&struct iwl_tx_queue_cfg_cmd with &struct iwl_tx_queue_cfg_rsp
+	 *	for newer (A000) hardware.
+	 */
+	SCD_QUEUE_CFG = 0x1d,
+
+	/**
+	 * @WEP_KEY: uses &struct iwl_mvm_wep_key_cmd
+	 */
+	WEP_KEY = 0x20,
+
+	/**
+	 * @SHARED_MEM_CFG:
+	 * retrieve shared memory configuration - response in
+	 * &struct iwl_shared_mem_cfg
+	 */
+	SHARED_MEM_CFG = 0x25,
+
+	/**
+	 * @TDLS_CHANNEL_SWITCH_CMD: uses &struct iwl_tdls_channel_switch_cmd
+	 */
+	TDLS_CHANNEL_SWITCH_CMD = 0x27,
+
+	/**
+	 * @TDLS_CHANNEL_SWITCH_NOTIFICATION:
+	 * uses &struct iwl_tdls_channel_switch_notif
+	 */
+	TDLS_CHANNEL_SWITCH_NOTIFICATION = 0xaa,
+
+	/**
+	 * @TDLS_CONFIG_CMD:
+	 * &struct iwl_tdls_config_cmd, response in &struct iwl_tdls_config_res
+	 */
+	TDLS_CONFIG_CMD = 0xa7,
+
+	/**
+	 * @MAC_CONTEXT_CMD: &struct iwl_mac_ctx_cmd
+	 */
+	MAC_CONTEXT_CMD = 0x28,
+
+	/**
+	 * @TIME_EVENT_CMD:
+	 * &struct iwl_time_event_cmd, response in &struct iwl_time_event_resp
+	 */
+	TIME_EVENT_CMD = 0x29, /* both CMD and response */
+
+	/**
+	 * @TIME_EVENT_NOTIFICATION: &struct iwl_time_event_notif
+	 */
+	TIME_EVENT_NOTIFICATION = 0x2a,
+
+	/**
+	 * @BINDING_CONTEXT_CMD:
+	 * &struct iwl_binding_cmd or &struct iwl_binding_cmd_v1
+	 */
+	BINDING_CONTEXT_CMD = 0x2b,
+
+	/**
+	 * @TIME_QUOTA_CMD: &struct iwl_time_quota_cmd
+	 */
+	TIME_QUOTA_CMD = 0x2c,
+
+	/**
+	 * @NON_QOS_TX_COUNTER_CMD:
+	 * command is &struct iwl_nonqos_seq_query_cmd
+	 */
+	NON_QOS_TX_COUNTER_CMD = 0x2d,
+
+	/**
+	 * @LEDS_CMD: command is &struct iwl_led_cmd
+	 */
+	LEDS_CMD = 0x48,
+
+	/**
+	 * @LQ_CMD: using &struct iwl_lq_cmd
+	 */
+	LQ_CMD = 0x4e,
+
+	/**
+	 * @FW_PAGING_BLOCK_CMD:
+	 * &struct iwl_fw_paging_cmd
+	 */
+	FW_PAGING_BLOCK_CMD = 0x4f,
+
+	/**
+	 * @SCAN_OFFLOAD_REQUEST_CMD: uses &struct iwl_scan_req_lmac
+	 */
+	SCAN_OFFLOAD_REQUEST_CMD = 0x51,
+
+	/**
+	 * @SCAN_OFFLOAD_ABORT_CMD: abort the scan - no further contents
+	 */
+	SCAN_OFFLOAD_ABORT_CMD = 0x52,
+
+	/**
+	 * @HOT_SPOT_CMD: uses &struct iwl_hs20_roc_req
+	 */
+	HOT_SPOT_CMD = 0x53,
+
+	/**
+	 * @SCAN_OFFLOAD_COMPLETE:
+	 * notification, &struct iwl_periodic_scan_complete
+	 */
+	SCAN_OFFLOAD_COMPLETE = 0x6D,
+
+	/**
+	 * @SCAN_OFFLOAD_UPDATE_PROFILES_CMD:
+	 * update scan offload (scheduled scan) profiles/blacklist/etc.
+	 */
+	SCAN_OFFLOAD_UPDATE_PROFILES_CMD = 0x6E,
+
+	/**
+	 * @MATCH_FOUND_NOTIFICATION: scan match found
+	 */
+	MATCH_FOUND_NOTIFICATION = 0xd9,
+
+	/**
+	 * @SCAN_ITERATION_COMPLETE:
+	 * uses &struct iwl_lmac_scan_complete_notif
+	 */
+	SCAN_ITERATION_COMPLETE = 0xe7,
+
+	/* Phy */
+	/**
+	 * @PHY_CONFIGURATION_CMD: &struct iwl_phy_cfg_cmd
+	 */
+	PHY_CONFIGURATION_CMD = 0x6a,
+
+	/**
+	 * @CALIB_RES_NOTIF_PHY_DB: &struct iwl_calib_res_notif_phy_db
+	 */
+	CALIB_RES_NOTIF_PHY_DB = 0x6b,
+
+	/**
+	 * @PHY_DB_CMD: &struct iwl_phy_db_cmd
+	 */
+	PHY_DB_CMD = 0x6c,
+
+	/**
+	 * @TOF_CMD: &struct iwl_tof_config_cmd
+	 */
+	TOF_CMD = 0x10,
+
+	/**
+	 * @TOF_NOTIFICATION: &struct iwl_tof_gen_resp_cmd
+	 */
+	TOF_NOTIFICATION = 0x11,
+
+	/**
+	 * @POWER_TABLE_CMD: &struct iwl_device_power_cmd
+	 */
+	POWER_TABLE_CMD = 0x77,
+
+	/**
+	 * @PSM_UAPSD_AP_MISBEHAVING_NOTIFICATION:
+	 * &struct iwl_uapsd_misbehaving_ap_notif
+	 */
+	PSM_UAPSD_AP_MISBEHAVING_NOTIFICATION = 0x78,
+
+	/**
+	 * @LTR_CONFIG: &struct iwl_ltr_config_cmd
+	 */
+	LTR_CONFIG = 0xee,
+
+	/**
+	 * @REPLY_THERMAL_MNG_BACKOFF:
+	 * Thermal throttling command
+	 */
+	REPLY_THERMAL_MNG_BACKOFF = 0x7e,
+
+	/**
+	 * @DC2DC_CONFIG_CMD:
+	 * Set/Get DC2DC frequency tune
+	 * Command is &struct iwl_dc2dc_config_cmd,
+	 * response is &struct iwl_dc2dc_config_resp
+	 */
+	DC2DC_CONFIG_CMD = 0x83,
+
+	/**
+	 * @NVM_ACCESS_CMD: using &struct iwl_nvm_access_cmd
+	 */
+	NVM_ACCESS_CMD = 0x88,
+
+	/**
+	 * @BEACON_NOTIFICATION: &struct iwl_extended_beacon_notif
+	 */
+	BEACON_NOTIFICATION = 0x90,
+
+	/**
+	 * @BEACON_TEMPLATE_CMD:
+	 *	Uses one of &struct iwl_mac_beacon_cmd_v6,
+	 *	&struct iwl_mac_beacon_cmd_v7 or &struct iwl_mac_beacon_cmd
+	 *	depending on the device version.
+	 */
+	BEACON_TEMPLATE_CMD = 0x91,
+	/**
+	 * @TX_ANT_CONFIGURATION_CMD: &struct iwl_tx_ant_cfg_cmd
+	 */
+	TX_ANT_CONFIGURATION_CMD = 0x98,
+
+	/**
+	 * @STATISTICS_CMD: &struct iwl_statistics_cmd
+	 */
+	STATISTICS_CMD = 0x9c,
+
+	/**
+	 * @STATISTICS_NOTIFICATION:
+	 * one of &struct iwl_notif_statistics_v10,
+	 * &struct iwl_notif_statistics_v11,
+	 * &struct iwl_notif_statistics_cdb
+	 */
+	STATISTICS_NOTIFICATION = 0x9d,
+
+	/**
+	 * @EOSP_NOTIFICATION:
+	 * Notify that a service period ended,
+	 * &struct iwl_mvm_eosp_notification
+	 */
+	EOSP_NOTIFICATION = 0x9e,
+
+	/**
+	 * @REDUCE_TX_POWER_CMD:
+	 * &struct iwl_dev_tx_power_cmd_v3 or &struct iwl_dev_tx_power_cmd
+	 */
+	REDUCE_TX_POWER_CMD = 0x9f,
+
+	/**
+	 * @CARD_STATE_NOTIFICATION:
+	 * Card state (RF/CT kill) notification,
+	 * uses &struct iwl_card_state_notif
+	 */
+	CARD_STATE_NOTIFICATION = 0xa1,
+
+	/**
+	 * @MISSED_BEACONS_NOTIFICATION: &struct iwl_missed_beacons_notif
+	 */
+	MISSED_BEACONS_NOTIFICATION = 0xa2,
+
+	/**
+	 * @MAC_PM_POWER_TABLE: using &struct iwl_mac_power_cmd
+	 */
+	MAC_PM_POWER_TABLE = 0xa9,
+
+	/**
+	 * @MFUART_LOAD_NOTIFICATION: &struct iwl_mfuart_load_notif
+	 */
+	MFUART_LOAD_NOTIFICATION = 0xb1,
+
+	/**
+	 * @RSS_CONFIG_CMD: &struct iwl_rss_config_cmd
+	 */
+	RSS_CONFIG_CMD = 0xb3,
+
+	/**
+	 * @REPLY_RX_PHY_CMD: &struct iwl_rx_phy_info
+	 */
+	REPLY_RX_PHY_CMD = 0xc0,
+
+	/**
+	 * @REPLY_RX_MPDU_CMD:
+	 * &struct iwl_rx_mpdu_res_start or &struct iwl_rx_mpdu_desc
+	 */
+	REPLY_RX_MPDU_CMD = 0xc1,
+
+	/**
+	 * @FRAME_RELEASE:
+	 * Frame release (reorder helper) notification, uses
+	 * &struct iwl_frame_release
+	 */
+	FRAME_RELEASE = 0xc3,
+
+	/**
+	 * @BA_NOTIF:
+	 * BlockAck notification, uses &struct iwl_mvm_compressed_ba_notif
+	 * or &struct iwl_mvm_ba_notif depending on the HW
+	 */
+	BA_NOTIF = 0xc5,
+
+	/* Location Aware Regulatory */
+	/**
+	 * @MCC_UPDATE_CMD: using &struct iwl_mcc_update_cmd
+	 */
+	MCC_UPDATE_CMD = 0xc8,
+
+	/**
+	 * @MCC_CHUB_UPDATE_CMD: using &struct iwl_mcc_chub_notif
+	 */
+	MCC_CHUB_UPDATE_CMD = 0xc9,
+
+	/**
+	 * @MARKER_CMD: trace marker command, uses &struct iwl_mvm_marker
+	 */
+	MARKER_CMD = 0xcb,
+
+	/**
+	 * @BT_PROFILE_NOTIFICATION: &struct iwl_bt_coex_profile_notif
+	 */
+	BT_PROFILE_NOTIFICATION = 0xce,
+
+	/**
+	 * @BT_CONFIG: &struct iwl_bt_coex_cmd
+	 */
+	BT_CONFIG = 0x9b,
+
+	/**
+	 * @BT_COEX_UPDATE_REDUCED_TXP:
+	 * &struct iwl_bt_coex_reduced_txp_update_cmd
+	 */
+	BT_COEX_UPDATE_REDUCED_TXP = 0x5c,
+
+	/**
+	 * @BT_COEX_CI: &struct iwl_bt_coex_ci_cmd
+	 */
+	BT_COEX_CI = 0x5d,
+
+	/**
+	 * @REPLY_SF_CFG_CMD: &struct iwl_sf_cfg_cmd
+	 */
+	REPLY_SF_CFG_CMD = 0xd1,
+	/**
+	 * @REPLY_BEACON_FILTERING_CMD: &struct iwl_beacon_filter_cmd
+	 */
+	REPLY_BEACON_FILTERING_CMD = 0xd2,
+
+	/**
+	 * @DTS_MEASUREMENT_NOTIFICATION:
+	 * &struct iwl_dts_measurement_notif_v1 or
+	 * &struct iwl_dts_measurement_notif_v2
+	 */
+	DTS_MEASUREMENT_NOTIFICATION = 0xdd,
+
+	/**
+	 * @LDBG_CONFIG_CMD: configure continuous trace recording
+	 */
+	LDBG_CONFIG_CMD = 0xf6,
+
+	/**
+	 * @DEBUG_LOG_MSG: Debugging log data from firmware
+	 */
+	DEBUG_LOG_MSG = 0xf7,
+
+	/**
+	 * @BCAST_FILTER_CMD: &struct iwl_bcast_filter_cmd
+	 */
+	BCAST_FILTER_CMD = 0xcf,
+
+	/**
+	 * @MCAST_FILTER_CMD: &struct iwl_mcast_filter_cmd
+	 */
+	MCAST_FILTER_CMD = 0xd0,
+
+	/**
+	 * @D3_CONFIG_CMD: &struct iwl_d3_manager_config
+	 */
+	D3_CONFIG_CMD = 0xd3,
+
+	/**
+	 * @PROT_OFFLOAD_CONFIG_CMD: Depending on firmware, uses one of
+	 * &struct iwl_proto_offload_cmd_v1, &struct iwl_proto_offload_cmd_v2,
+	 * &struct iwl_proto_offload_cmd_v3_small,
+	 * &struct iwl_proto_offload_cmd_v3_large
+	 */
+	PROT_OFFLOAD_CONFIG_CMD = 0xd4,
+
+	/**
+	 * @OFFLOADS_QUERY_CMD:
+	 * No data in command, response in &struct iwl_wowlan_status
+	 */
+	OFFLOADS_QUERY_CMD = 0xd5,
+
+	/**
+	 * @REMOTE_WAKE_CONFIG_CMD: &struct iwl_wowlan_remote_wake_config
+	 */
+	REMOTE_WAKE_CONFIG_CMD = 0xd6,
+
+	/**
+	 * @D0I3_END_CMD: End D0i3/D3 state, no command data
+	 */
+	D0I3_END_CMD = 0xed,
+
+	/**
+	 * @WOWLAN_PATTERNS: &struct iwl_wowlan_patterns_cmd
+	 */
+	WOWLAN_PATTERNS = 0xe0,
+
+	/**
+	 * @WOWLAN_CONFIGURATION: &struct iwl_wowlan_config_cmd
+	 */
+	WOWLAN_CONFIGURATION = 0xe1,
+
+	/**
+	 * @WOWLAN_TSC_RSC_PARAM: &struct iwl_wowlan_rsc_tsc_params_cmd
+	 */
+	WOWLAN_TSC_RSC_PARAM = 0xe2,
+
+	/**
+	 * @WOWLAN_TKIP_PARAM: &struct iwl_wowlan_tkip_params_cmd
+	 */
+	WOWLAN_TKIP_PARAM = 0xe3,
+
+	/**
+	 * @WOWLAN_KEK_KCK_MATERIAL: &struct iwl_wowlan_kek_kck_material_cmd
+	 */
+	WOWLAN_KEK_KCK_MATERIAL = 0xe4,
+
+	/**
+	 * @WOWLAN_GET_STATUSES: response in &struct iwl_wowlan_status
+	 */
+	WOWLAN_GET_STATUSES = 0xe5,
+
+	/**
+	 * @SCAN_OFFLOAD_PROFILES_QUERY_CMD:
+	 * No command data, response is &struct iwl_scan_offload_profiles_query
+	 */
+	SCAN_OFFLOAD_PROFILES_QUERY_CMD = 0x56,
+};
+
+/**
+ * enum iwl_system_subcmd_ids - system group command IDs
+ */
+enum iwl_system_subcmd_ids {
+	/**
+	 * @SHARED_MEM_CFG_CMD:
+	 * response in &struct iwl_shared_mem_cfg or
+	 * &struct iwl_shared_mem_cfg_v2
+	 */
+	SHARED_MEM_CFG_CMD = 0x0,
+
+	/**
+	 * @INIT_EXTENDED_CFG_CMD: &struct iwl_init_extended_cfg_cmd
+	 */
+	INIT_EXTENDED_CFG_CMD = 0x03,
+
+	/**
+	 * @FSEQ_VER_MISMATCH_NTF: Notification about fseq version
+	 *	mismatch during init.  The format is specified in
+	 *	&struct iwl_fseq_ver_mismatch_ntf.
+	 */
+	FSEQ_VER_MISMATCH_NTF = 0xFF,
+};
+
+#endif /* __iwl_fw_api_commands_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/config.h b/drivers/net/wireless/intel/iwlwifi/fw/api/config.h
new file mode 100644
index 000000000000..7f645b62804e
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/config.h
@@ -0,0 +1,184 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+
+#ifndef __iwl_fw_api_config_h__
+#define __iwl_fw_api_config_h__
+
+/*
+ * struct iwl_dqa_enable_cmd
+ * @cmd_queue: the TXQ number of the command queue
+ */
+struct iwl_dqa_enable_cmd {
+	__le32 cmd_queue;
+} __packed; /* DQA_CONTROL_CMD_API_S_VER_1 */
+
+/*
+ * struct iwl_tx_ant_cfg_cmd
+ * @valid: valid antenna configuration
+ */
+struct iwl_tx_ant_cfg_cmd {
+	__le32 valid;
+} __packed;
+
+/**
+ * struct iwl_calib_ctrl - Calibration control struct.
+ * Sent as part of the phy configuration command.
+ * @flow_trigger: bitmap for which calibrations to perform according to
+ *		flow triggers, using &enum iwl_calib_cfg
+ * @event_trigger: bitmap for which calibrations to perform according to
+ *		event triggers, using &enum iwl_calib_cfg
+ */
+struct iwl_calib_ctrl {
+	__le32 flow_trigger;
+	__le32 event_trigger;
+} __packed;
+
+/* This enum defines the bitmap of various calibrations to enable in both
+ * init ucode and runtime ucode through CALIBRATION_CFG_CMD.
+ */
+enum iwl_calib_cfg {
+	IWL_CALIB_CFG_XTAL_IDX			= BIT(0),
+	IWL_CALIB_CFG_TEMPERATURE_IDX		= BIT(1),
+	IWL_CALIB_CFG_VOLTAGE_READ_IDX		= BIT(2),
+	IWL_CALIB_CFG_PAPD_IDX			= BIT(3),
+	IWL_CALIB_CFG_TX_PWR_IDX		= BIT(4),
+	IWL_CALIB_CFG_DC_IDX			= BIT(5),
+	IWL_CALIB_CFG_BB_FILTER_IDX		= BIT(6),
+	IWL_CALIB_CFG_LO_LEAKAGE_IDX		= BIT(7),
+	IWL_CALIB_CFG_TX_IQ_IDX			= BIT(8),
+	IWL_CALIB_CFG_TX_IQ_SKEW_IDX		= BIT(9),
+	IWL_CALIB_CFG_RX_IQ_IDX			= BIT(10),
+	IWL_CALIB_CFG_RX_IQ_SKEW_IDX		= BIT(11),
+	IWL_CALIB_CFG_SENSITIVITY_IDX		= BIT(12),
+	IWL_CALIB_CFG_CHAIN_NOISE_IDX		= BIT(13),
+	IWL_CALIB_CFG_DISCONNECTED_ANT_IDX	= BIT(14),
+	IWL_CALIB_CFG_ANT_COUPLING_IDX		= BIT(15),
+	IWL_CALIB_CFG_DAC_IDX			= BIT(16),
+	IWL_CALIB_CFG_ABS_IDX			= BIT(17),
+	IWL_CALIB_CFG_AGC_IDX			= BIT(18),
+};
+
+/**
+ * struct iwl_phy_cfg_cmd - Phy configuration command
+ * @phy_cfg: PHY configuration value, uses &enum iwl_fw_phy_cfg
+ * @calib_control: calibration control data
+ */
+struct iwl_phy_cfg_cmd {
+	__le32	phy_cfg;
+	struct iwl_calib_ctrl calib_control;
+} __packed;
+
+#define PHY_CFG_RADIO_TYPE	(BIT(0) | BIT(1))
+#define PHY_CFG_RADIO_STEP	(BIT(2) | BIT(3))
+#define PHY_CFG_RADIO_DASH	(BIT(4) | BIT(5))
+#define PHY_CFG_PRODUCT_NUMBER	(BIT(6) | BIT(7))
+#define PHY_CFG_TX_CHAIN_A	BIT(8)
+#define PHY_CFG_TX_CHAIN_B	BIT(9)
+#define PHY_CFG_TX_CHAIN_C	BIT(10)
+#define PHY_CFG_RX_CHAIN_A	BIT(12)
+#define PHY_CFG_RX_CHAIN_B	BIT(13)
+#define PHY_CFG_RX_CHAIN_C	BIT(14)
+
+/*
+ * enum iwl_dc2dc_config_id - flag ids
+ *
+ * Ids of dc2dc configuration flags
+ */
+enum iwl_dc2dc_config_id {
+	DCDC_LOW_POWER_MODE_MSK_SET  = 0x1, /* not used */
+	DCDC_FREQ_TUNE_SET = 0x2,
+}; /* MARKER_ID_API_E_VER_1 */
+
+/**
+ * struct iwl_dc2dc_config_cmd - configure dc2dc values
+ *
+ * (DC2DC_CONFIG_CMD = 0x83)
+ *
+ * Set/Get & configure dc2dc values.
+ * The command always returns the current dc2dc values.
+ *
+ * @flags: set/get dc2dc
+ * @enable_low_power_mode: not used.
+ * @dc2dc_freq_tune0: frequency divider - digital domain
+ * @dc2dc_freq_tune1: frequency divider - analog domain
+ */
+struct iwl_dc2dc_config_cmd {
+	__le32 flags;
+	__le32 enable_low_power_mode; /* not used */
+	__le32 dc2dc_freq_tune0;
+	__le32 dc2dc_freq_tune1;
+} __packed; /* DC2DC_CONFIG_CMD_API_S_VER_1 */
+
+/**
+ * struct iwl_dc2dc_config_resp - response for iwl_dc2dc_config_cmd
+ *
+ * Current dc2dc values returned by the FW.
+ *
+ * @dc2dc_freq_tune0: frequency divider - digital domain
+ * @dc2dc_freq_tune1: frequency divider - analog domain
+ */
+struct iwl_dc2dc_config_resp {
+	__le32 dc2dc_freq_tune0;
+	__le32 dc2dc_freq_tune1;
+} __packed; /* DC2DC_CONFIG_RESP_API_S_VER_1 */
+
+#endif /* __iwl_fw_api_config_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/context.h b/drivers/net/wireless/intel/iwlwifi/fw/api/context.h
new file mode 100644
index 000000000000..2f0d7c498b3e
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/context.h
@@ -0,0 +1,94 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+
+#ifndef __iwl_fw_api_context_h__
+#define __iwl_fw_api_context_h__
+
+/**
+ * enum iwl_ctxt_id_and_color - ID and color fields in context dword
+ * @FW_CTXT_ID_POS: position of the ID
+ * @FW_CTXT_ID_MSK: mask of the ID
+ * @FW_CTXT_COLOR_POS: position of the color
+ * @FW_CTXT_COLOR_MSK: mask of the color
+ * @FW_CTXT_INVALID: value used to indicate unused/invalid
+ */
+enum iwl_ctxt_id_and_color {
+	FW_CTXT_ID_POS		= 0,
+	FW_CTXT_ID_MSK		= 0xff << FW_CTXT_ID_POS,
+	FW_CTXT_COLOR_POS	= 8,
+	FW_CTXT_COLOR_MSK	= 0xff << FW_CTXT_COLOR_POS,
+	FW_CTXT_INVALID		= 0xffffffff,
+};
+
+#define FW_CMD_ID_AND_COLOR(_id, _color) (((_id) << FW_CTXT_ID_POS) |\
+					  ((_color) << FW_CTXT_COLOR_POS))
+
+/* Possible actions on PHYs, MACs and Bindings */
+enum iwl_ctxt_action {
+	FW_CTXT_ACTION_STUB = 0,
+	FW_CTXT_ACTION_ADD,
+	FW_CTXT_ACTION_MODIFY,
+	FW_CTXT_ACTION_REMOVE,
+	FW_CTXT_ACTION_NUM
+}; /* COMMON_CONTEXT_ACTION_API_E_VER_1 */
+
+#endif /* __iwl_fw_api_context_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-d3.h b/drivers/net/wireless/intel/iwlwifi/fw/api/d3.h
index d4a4c28b7192..57f4bc242023 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-d3.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/d3.h
@@ -18,11 +18,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
- * USA
- *
  * The full GNU General Public License is included in this distribution
  * in the file called COPYING.
  *
@@ -64,8 +59,8 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *****************************************************************************/
 
-#ifndef __fw_api_d3_h__
-#define __fw_api_d3_h__
+#ifndef __iwl_fw_api_d3_h__
+#define __iwl_fw_api_d3_h__
 
 /**
  * enum iwl_d3_wakeup_flags - D3 manager wakeup flags
@@ -468,4 +463,4 @@ struct iwl_wowlan_remote_wake_config {
 
 /* TODO: NetDetect API */
 
-#endif /* __fw_api_d3_h__ */
+#endif /* __iwl_fw_api_d3_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/datapath.h b/drivers/net/wireless/intel/iwlwifi/fw/api/datapath.h
new file mode 100644
index 000000000000..aa76dcc148bd
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/datapath.h
@@ -0,0 +1,127 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+
+#ifndef __iwl_fw_api_datapath_h__
+#define __iwl_fw_api_datapath_h__
+
+/**
+ * enum iwl_data_path_subcmd_ids - data path group commands
+ */
+enum iwl_data_path_subcmd_ids {
+	/**
+	 * @DQA_ENABLE_CMD: &struct iwl_dqa_enable_cmd
+	 */
+	DQA_ENABLE_CMD = 0x0,
+
+	/**
+	 * @UPDATE_MU_GROUPS_CMD: &struct iwl_mu_group_mgmt_cmd
+	 */
+	UPDATE_MU_GROUPS_CMD = 0x1,
+
+	/**
+	 * @TRIGGER_RX_QUEUES_NOTIF_CMD: &struct iwl_rxq_sync_cmd
+	 */
+	TRIGGER_RX_QUEUES_NOTIF_CMD = 0x2,
+
+	/**
+	 * @STA_PM_NOTIF: &struct iwl_mvm_pm_state_notification
+	 */
+	STA_PM_NOTIF = 0xFD,
+
+	/**
+	 * @MU_GROUP_MGMT_NOTIF: &struct iwl_mu_group_mgmt_notif
+	 */
+	MU_GROUP_MGMT_NOTIF = 0xFE,
+
+	/**
+	 * @RX_QUEUES_NOTIFICATION: &struct iwl_rxq_sync_notification
+	 */
+	RX_QUEUES_NOTIFICATION = 0xFF,
+};
+
+/**
+ * struct iwl_mu_group_mgmt_cmd - VHT MU-MIMO group configuration
+ *
+ * @reserved: reserved
+ * @membership_status: a bitmap of MU groups
+ * @user_position:the position of station in a group. If the station is in the
+ *	group then bits (group * 2) is the position -1
+ */
+struct iwl_mu_group_mgmt_cmd {
+	__le32 reserved;
+	__le32 membership_status[2];
+	__le32 user_position[4];
+} __packed; /* MU_GROUP_ID_MNG_TABLE_API_S_VER_1 */
+
+/**
+ * struct iwl_mu_group_mgmt_notif - VHT MU-MIMO group id notification
+ *
+ * @membership_status: a bitmap of MU groups
+ * @user_position: the position of station in a group. If the station is in the
+ *	group then bits (group * 2) is the position -1
+ */
+struct iwl_mu_group_mgmt_notif {
+	__le32 membership_status[2];
+	__le32 user_position[4];
+} __packed; /* MU_GROUP_MNG_NTFY_API_S_VER_1 */
+
+#endif /* __iwl_fw_api_datapath_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/debug.h b/drivers/net/wireless/intel/iwlwifi/fw/api/debug.h
new file mode 100644
index 000000000000..9f88b61536bc
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/debug.h
@@ -0,0 +1,345 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2007 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2005 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+#ifndef __iwl_fw_api_debug_h__
+#define __iwl_fw_api_debug_h__
+
+/**
+ * enum iwl_debug_cmds - debug commands
+ */
+enum iwl_debug_cmds {
+	/**
+	 * @LMAC_RD_WR:
+	 * LMAC memory read/write, using &struct iwl_dbg_mem_access_cmd and
+	 * &struct iwl_dbg_mem_access_rsp
+	 */
+	LMAC_RD_WR = 0x0,
+	/**
+	 * @UMAC_RD_WR:
+	 * UMAC memory read/write, using &struct iwl_dbg_mem_access_cmd and
+	 * &struct iwl_dbg_mem_access_rsp
+	 */
+	UMAC_RD_WR = 0x1,
+	/**
+	 * @MFU_ASSERT_DUMP_NTF:
+	 * &struct iwl_mfu_assert_dump_notif
+	 */
+	MFU_ASSERT_DUMP_NTF = 0xFE,
+};
+
+/* Error response/notification */
+enum {
+	FW_ERR_UNKNOWN_CMD = 0x0,
+	FW_ERR_INVALID_CMD_PARAM = 0x1,
+	FW_ERR_SERVICE = 0x2,
+	FW_ERR_ARC_MEMORY = 0x3,
+	FW_ERR_ARC_CODE = 0x4,
+	FW_ERR_WATCH_DOG = 0x5,
+	FW_ERR_WEP_GRP_KEY_INDX = 0x10,
+	FW_ERR_WEP_KEY_SIZE = 0x11,
+	FW_ERR_OBSOLETE_FUNC = 0x12,
+	FW_ERR_UNEXPECTED = 0xFE,
+	FW_ERR_FATAL = 0xFF
+};
+
+/**
+ * struct iwl_error_resp - FW error indication
+ * ( REPLY_ERROR = 0x2 )
+ * @error_type: one of FW_ERR_*
+ * @cmd_id: the command ID for which the error occurred
+ * @reserved1: reserved
+ * @bad_cmd_seq_num: sequence number of the erroneous command
+ * @error_service: which service created the error, applicable only if
+ *     error_type = 2, otherwise 0
+ * @timestamp: TSF in usecs.
+ */
+struct iwl_error_resp {
+	__le32 error_type;
+	u8 cmd_id;
+	u8 reserved1;
+	__le16 bad_cmd_seq_num;
+	__le32 error_service;
+	__le64 timestamp;
+} __packed;
+
+#define TX_FIFO_MAX_NUM_9000		8
+#define TX_FIFO_MAX_NUM			15
+#define RX_FIFO_MAX_NUM			2
+#define TX_FIFO_INTERNAL_MAX_NUM	6
+
+/**
+ * struct iwl_shared_mem_cfg_v2 - Shared memory configuration information
+ *
+ * @shared_mem_addr: shared memory addr (pre 8000 HW set to 0x0 as MARBH is not
+ *	accessible)
+ * @shared_mem_size: shared memory size
+ * @sample_buff_addr: internal sample (mon/adc) buff addr (pre 8000 HW set to
+ *	0x0 as accessible only via DBGM RDAT)
+ * @sample_buff_size: internal sample buff size
+ * @txfifo_addr: start addr of TXF0 (excluding the context table 0.5KB), (pre
+ *	8000 HW set to 0x0 as not accessible)
+ * @txfifo_size: size of TXF0 ... TXF7
+ * @rxfifo_size: RXF1, RXF2 sizes. If there is no RXF2, it'll have a value of 0
+ * @page_buff_addr: used by UMAC and performance debug (page miss analysis),
+ *	when paging is not supported this should be 0
+ * @page_buff_size: size of %page_buff_addr
+ * @rxfifo_addr: Start address of rxFifo
+ * @internal_txfifo_addr: start address of internalFifo
+ * @internal_txfifo_size: internal fifos' size
+ *
+ * NOTE: on firmware that don't have IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG
+ *	 set, the last 3 members don't exist.
+ */
+struct iwl_shared_mem_cfg_v2 {
+	__le32 shared_mem_addr;
+	__le32 shared_mem_size;
+	__le32 sample_buff_addr;
+	__le32 sample_buff_size;
+	__le32 txfifo_addr;
+	__le32 txfifo_size[TX_FIFO_MAX_NUM_9000];
+	__le32 rxfifo_size[RX_FIFO_MAX_NUM];
+	__le32 page_buff_addr;
+	__le32 page_buff_size;
+	__le32 rxfifo_addr;
+	__le32 internal_txfifo_addr;
+	__le32 internal_txfifo_size[TX_FIFO_INTERNAL_MAX_NUM];
+} __packed; /* SHARED_MEM_ALLOC_API_S_VER_2 */
+
+/**
+ * struct iwl_shared_mem_lmac_cfg - LMAC shared memory configuration
+ *
+ * @txfifo_addr: start addr of TXF0 (excluding the context table 0.5KB)
+ * @txfifo_size: size of TX FIFOs
+ * @rxfifo1_addr: RXF1 addr
+ * @rxfifo1_size: RXF1 size
+ */
+struct iwl_shared_mem_lmac_cfg {
+	__le32 txfifo_addr;
+	__le32 txfifo_size[TX_FIFO_MAX_NUM];
+	__le32 rxfifo1_addr;
+	__le32 rxfifo1_size;
+
+} __packed; /* SHARED_MEM_ALLOC_LMAC_API_S_VER_1 */
+
+/**
+ * struct iwl_shared_mem_cfg - Shared memory configuration information
+ *
+ * @shared_mem_addr: shared memory address
+ * @shared_mem_size: shared memory size
+ * @sample_buff_addr: internal sample (mon/adc) buff addr
+ * @sample_buff_size: internal sample buff size
+ * @rxfifo2_addr: start addr of RXF2
+ * @rxfifo2_size: size of RXF2
+ * @page_buff_addr: used by UMAC and performance debug (page miss analysis),
+ *	when paging is not supported this should be 0
+ * @page_buff_size: size of %page_buff_addr
+ * @lmac_num: number of LMACs (1 or 2)
+ * @lmac_smem: per - LMAC smem data
+ */
+struct iwl_shared_mem_cfg {
+	__le32 shared_mem_addr;
+	__le32 shared_mem_size;
+	__le32 sample_buff_addr;
+	__le32 sample_buff_size;
+	__le32 rxfifo2_addr;
+	__le32 rxfifo2_size;
+	__le32 page_buff_addr;
+	__le32 page_buff_size;
+	__le32 lmac_num;
+	struct iwl_shared_mem_lmac_cfg lmac_smem[2];
+} __packed; /* SHARED_MEM_ALLOC_API_S_VER_3 */
+
+/**
+ * struct iwl_mfuart_load_notif - mfuart image version & status
+ * ( MFUART_LOAD_NOTIFICATION = 0xb1 )
+ * @installed_ver: installed image version
+ * @external_ver: external image version
+ * @status: MFUART loading status
+ * @duration: MFUART loading time
+ * @image_size: MFUART image size in bytes
+*/
+struct iwl_mfuart_load_notif {
+	__le32 installed_ver;
+	__le32 external_ver;
+	__le32 status;
+	__le32 duration;
+	/* image size valid only in v2 of the command */
+	__le32 image_size;
+} __packed; /* MFU_LOADER_NTFY_API_S_VER_2 */
+
+/**
+ * struct iwl_mfu_assert_dump_notif - mfuart dump logs
+ * ( MFU_ASSERT_DUMP_NTF = 0xfe )
+ * @assert_id: mfuart assert id that cause the notif
+ * @curr_reset_num: number of asserts since uptime
+ * @index_num: current chunk id
+ * @parts_num: total number of chunks
+ * @data_size: number of data bytes sent
+ * @data: data buffer
+ */
+struct iwl_mfu_assert_dump_notif {
+	__le32   assert_id;
+	__le32   curr_reset_num;
+	__le16   index_num;
+	__le16   parts_num;
+	__le32   data_size;
+	__le32   data[0];
+} __packed; /* MFU_DUMP_ASSERT_API_S_VER_1 */
+
+/**
+ * enum iwl_mvm_marker_id - marker ids
+ *
+ * The ids for different type of markers to insert into the usniffer logs
+ *
+ * @MARKER_ID_TX_FRAME_LATENCY: TX latency marker
+ */
+enum iwl_mvm_marker_id {
+	MARKER_ID_TX_FRAME_LATENCY = 1,
+}; /* MARKER_ID_API_E_VER_1 */
+
+/**
+ * struct iwl_mvm_marker - mark info into the usniffer logs
+ *
+ * (MARKER_CMD = 0xcb)
+ *
+ * Mark the UTC time stamp into the usniffer logs together with additional
+ * metadata, so the usniffer output can be parsed.
+ * In the command response the ucode will return the GP2 time.
+ *
+ * @dw_len: The amount of dwords following this byte including this byte.
+ * @marker_id: A unique marker id (iwl_mvm_marker_id).
+ * @reserved: reserved.
+ * @timestamp: in milliseconds since 1970-01-01 00:00:00 UTC
+ * @metadata: additional meta data that will be written to the unsiffer log
+ */
+struct iwl_mvm_marker {
+	u8 dw_len;
+	u8 marker_id;
+	__le16 reserved;
+	__le64 timestamp;
+	__le32 metadata[0];
+} __packed; /* MARKER_API_S_VER_1 */
+
+/* Operation types for the debug mem access */
+enum {
+	DEBUG_MEM_OP_READ = 0,
+	DEBUG_MEM_OP_WRITE = 1,
+	DEBUG_MEM_OP_WRITE_BYTES = 2,
+};
+
+#define DEBUG_MEM_MAX_SIZE_DWORDS 32
+
+/**
+ * struct iwl_dbg_mem_access_cmd - Request the device to read/write memory
+ * @op: DEBUG_MEM_OP_*
+ * @addr: address to read/write from/to
+ * @len: in dwords, to read/write
+ * @data: for write opeations, contains the source buffer
+ */
+struct iwl_dbg_mem_access_cmd {
+	__le32 op;
+	__le32 addr;
+	__le32 len;
+	__le32 data[];
+} __packed; /* DEBUG_(U|L)MAC_RD_WR_CMD_API_S_VER_1 */
+
+/* Status responses for the debug mem access */
+enum {
+	DEBUG_MEM_STATUS_SUCCESS = 0x0,
+	DEBUG_MEM_STATUS_FAILED = 0x1,
+	DEBUG_MEM_STATUS_LOCKED = 0x2,
+	DEBUG_MEM_STATUS_HIDDEN = 0x3,
+	DEBUG_MEM_STATUS_LENGTH = 0x4,
+};
+
+/**
+ * struct iwl_dbg_mem_access_rsp - Response to debug mem commands
+ * @status: DEBUG_MEM_STATUS_*
+ * @len: read dwords (0 for write operations)
+ * @data: contains the read DWs
+ */
+struct iwl_dbg_mem_access_rsp {
+	__le32 status;
+	__le32 len;
+	__le32 data[];
+} __packed; /* DEBUG_(U|L)MAC_RD_WR_RSP_API_S_VER_1 */
+
+#define CONT_REC_COMMAND_SIZE	80
+#define ENABLE_CONT_RECORDING	0x15
+#define DISABLE_CONT_RECORDING	0x16
+
+/*
+ * struct iwl_continuous_record_mode - recording mode
+ */
+struct iwl_continuous_record_mode {
+	__le16 enable_recording;
+} __packed;
+
+/*
+ * struct iwl_continuous_record_cmd - enable/disable continuous recording
+ */
+struct iwl_continuous_record_cmd {
+	struct iwl_continuous_record_mode record_mode;
+	u8 pad[CONT_REC_COMMAND_SIZE -
+		sizeof(struct iwl_continuous_record_mode)];
+} __packed;
+
+#endif /* __iwl_fw_api_debug_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/filter.h b/drivers/net/wireless/intel/iwlwifi/fw/api/filter.h
new file mode 100644
index 000000000000..befc3b126041
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/filter.h
@@ -0,0 +1,183 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+
+#ifndef __iwl_fw_api_filter_h__
+#define __iwl_fw_api_filter_h__
+
+#include "fw/api/mac.h"
+
+#define MAX_PORT_ID_NUM	2
+#define MAX_MCAST_FILTERING_ADDRESSES 256
+
+/**
+ * struct iwl_mcast_filter_cmd - configure multicast filter.
+ * @filter_own: Set 1 to filter out multicast packets sent by station itself
+ * @port_id:	Multicast MAC addresses array specifier. This is a strange way
+ *		to identify network interface adopted in host-device IF.
+ *		It is used by FW as index in array of addresses. This array has
+ *		MAX_PORT_ID_NUM members.
+ * @count:	Number of MAC addresses in the array
+ * @pass_all:	Set 1 to pass all multicast packets.
+ * @bssid:	current association BSSID.
+ * @reserved:	reserved
+ * @addr_list:	Place holder for array of MAC addresses.
+ *		IMPORTANT: add padding if necessary to ensure DWORD alignment.
+ */
+struct iwl_mcast_filter_cmd {
+	u8 filter_own;
+	u8 port_id;
+	u8 count;
+	u8 pass_all;
+	u8 bssid[6];
+	u8 reserved[2];
+	u8 addr_list[0];
+} __packed; /* MCAST_FILTERING_CMD_API_S_VER_1 */
+
+#define MAX_BCAST_FILTERS 8
+#define MAX_BCAST_FILTER_ATTRS 2
+
+/**
+ * enum iwl_mvm_bcast_filter_attr_offset - written by fw for each Rx packet
+ * @BCAST_FILTER_OFFSET_PAYLOAD_START: offset is from payload start.
+ * @BCAST_FILTER_OFFSET_IP_END: offset is from ip header end (i.e.
+ *	start of ip payload).
+ */
+enum iwl_mvm_bcast_filter_attr_offset {
+	BCAST_FILTER_OFFSET_PAYLOAD_START = 0,
+	BCAST_FILTER_OFFSET_IP_END = 1,
+};
+
+/**
+ * struct iwl_fw_bcast_filter_attr - broadcast filter attribute
+ * @offset_type:	&enum iwl_mvm_bcast_filter_attr_offset.
+ * @offset:	starting offset of this pattern.
+ * @reserved1:	reserved
+ * @val:	value to match - big endian (MSB is the first
+ *		byte to match from offset pos).
+ * @mask:	mask to match (big endian).
+ */
+struct iwl_fw_bcast_filter_attr {
+	u8 offset_type;
+	u8 offset;
+	__le16 reserved1;
+	__be32 val;
+	__be32 mask;
+} __packed; /* BCAST_FILTER_ATT_S_VER_1 */
+
+/**
+ * enum iwl_mvm_bcast_filter_frame_type - filter frame type
+ * @BCAST_FILTER_FRAME_TYPE_ALL: consider all frames.
+ * @BCAST_FILTER_FRAME_TYPE_IPV4: consider only ipv4 frames
+ */
+enum iwl_mvm_bcast_filter_frame_type {
+	BCAST_FILTER_FRAME_TYPE_ALL = 0,
+	BCAST_FILTER_FRAME_TYPE_IPV4 = 1,
+};
+
+/**
+ * struct iwl_fw_bcast_filter - broadcast filter
+ * @discard: discard frame (1) or let it pass (0).
+ * @frame_type: &enum iwl_mvm_bcast_filter_frame_type.
+ * @reserved1: reserved
+ * @num_attrs: number of valid attributes in this filter.
+ * @attrs: attributes of this filter. a filter is considered matched
+ *	only when all its attributes are matched (i.e. AND relationship)
+ */
+struct iwl_fw_bcast_filter {
+	u8 discard;
+	u8 frame_type;
+	u8 num_attrs;
+	u8 reserved1;
+	struct iwl_fw_bcast_filter_attr attrs[MAX_BCAST_FILTER_ATTRS];
+} __packed; /* BCAST_FILTER_S_VER_1 */
+
+/**
+ * struct iwl_fw_bcast_mac - per-mac broadcast filtering configuration.
+ * @default_discard: default action for this mac (discard (1) / pass (0)).
+ * @reserved1: reserved
+ * @attached_filters: bitmap of relevant filters for this mac.
+ */
+struct iwl_fw_bcast_mac {
+	u8 default_discard;
+	u8 reserved1;
+	__le16 attached_filters;
+} __packed; /* BCAST_MAC_CONTEXT_S_VER_1 */
+
+/**
+ * struct iwl_bcast_filter_cmd - broadcast filtering configuration
+ * @disable: enable (0) / disable (1)
+ * @max_bcast_filters: max number of filters (MAX_BCAST_FILTERS)
+ * @max_macs: max number of macs (NUM_MAC_INDEX_DRIVER)
+ * @reserved1: reserved
+ * @filters: broadcast filters
+ * @macs: broadcast filtering configuration per-mac
+ */
+struct iwl_bcast_filter_cmd {
+	u8 disable;
+	u8 max_bcast_filters;
+	u8 max_macs;
+	u8 reserved1;
+	struct iwl_fw_bcast_filter filters[MAX_BCAST_FILTERS];
+	struct iwl_fw_bcast_mac macs[NUM_MAC_INDEX_DRIVER];
+} __packed; /* BCAST_FILTERING_HCMD_API_S_VER_1 */
+
+#endif /* __iwl_fw_api_filter_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/led.h b/drivers/net/wireless/intel/iwlwifi/fw/api/led.h
new file mode 100644
index 000000000000..b30c9d229d6e
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/led.h
@@ -0,0 +1,71 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+
+#ifndef __iwl_fw_api_led_h__
+#define __iwl_fw_api_led_h__
+
+/**
+ * struct iwl_led_cmd - LED switching command
+ *
+ * @status: LED status (on/off)
+ */
+struct iwl_led_cmd {
+	__le32 status;
+} __packed; /* LEDS_CMD_API_S_VER_2 */
+
+#endif /* __iwl_fw_api_led_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h b/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h
new file mode 100644
index 000000000000..39c89e85fd2f
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h
@@ -0,0 +1,152 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+
+#ifndef __iwl_fw_api_mac_cfg_h__
+#define __iwl_fw_api_mac_cfg_h__
+
+/**
+ * enum iwl_mac_conf_subcmd_ids - mac configuration command IDs
+ */
+enum iwl_mac_conf_subcmd_ids {
+	/**
+	 * @LINK_QUALITY_MEASUREMENT_CMD: &struct iwl_link_qual_msrmnt_cmd
+	 */
+	LINK_QUALITY_MEASUREMENT_CMD = 0x1,
+
+	/**
+	 * @LINK_QUALITY_MEASUREMENT_COMPLETE_NOTIF:
+	 * &struct iwl_link_qual_msrmnt_notif
+	 */
+	LINK_QUALITY_MEASUREMENT_COMPLETE_NOTIF = 0xFE,
+
+	/**
+	 * @CHANNEL_SWITCH_NOA_NOTIF: &struct iwl_channel_switch_noa_notif
+	 */
+	CHANNEL_SWITCH_NOA_NOTIF = 0xFF,
+};
+
+#define LQM_NUMBER_OF_STATIONS_IN_REPORT 16
+
+enum iwl_lqm_cmd_operatrions {
+	LQM_CMD_OPERATION_START_MEASUREMENT = 0x01,
+	LQM_CMD_OPERATION_STOP_MEASUREMENT = 0x02,
+};
+
+enum iwl_lqm_status {
+	LQM_STATUS_SUCCESS = 0,
+	LQM_STATUS_TIMEOUT = 1,
+	LQM_STATUS_ABORT = 2,
+};
+
+/**
+ * struct iwl_link_qual_msrmnt_cmd - Link Quality Measurement command
+ * @cmd_operation: command operation to be performed (start or stop)
+ *	as defined above.
+ * @mac_id: MAC ID the measurement applies to.
+ * @measurement_time: time of the total measurement to be performed, in uSec.
+ * @timeout: maximum time allowed until a response is sent, in uSec.
+ */
+struct iwl_link_qual_msrmnt_cmd {
+	__le32 cmd_operation;
+	__le32 mac_id;
+	__le32 measurement_time;
+	__le32 timeout;
+} __packed /* LQM_CMD_API_S_VER_1 */;
+
+/**
+ * struct iwl_link_qual_msrmnt_notif - Link Quality Measurement notification
+ *
+ * @frequent_stations_air_time: an array containing the total air time
+ *	(in uSec) used by the most frequently transmitting stations.
+ * @number_of_stations: the number of uniqe stations included in the array
+ *	(a number between 0 to 16)
+ * @total_air_time_other_stations: the total air time (uSec) used by all the
+ *	stations which are not included in the above report.
+ * @time_in_measurement_window: the total time in uSec in which a measurement
+ *	took place.
+ * @tx_frame_dropped: the number of TX frames dropped due to retry limit during
+ *	measurement
+ * @mac_id: MAC ID the measurement applies to.
+ * @status: return status. may be one of the LQM_STATUS_* defined above.
+ * @reserved: reserved.
+ */
+struct iwl_link_qual_msrmnt_notif {
+	__le32 frequent_stations_air_time[LQM_NUMBER_OF_STATIONS_IN_REPORT];
+	__le32 number_of_stations;
+	__le32 total_air_time_other_stations;
+	__le32 time_in_measurement_window;
+	__le32 tx_frame_dropped;
+	__le32 mac_id;
+	__le32 status;
+	u8 reserved[12];
+} __packed; /* LQM_MEASUREMENT_COMPLETE_NTF_API_S_VER1 */
+
+/**
+ * struct iwl_channel_switch_noa_notif - Channel switch NOA notification
+ *
+ * @id_and_color: ID and color of the MAC
+ */
+struct iwl_channel_switch_noa_notif {
+	__le32 id_and_color;
+} __packed; /* CHANNEL_SWITCH_START_NTFY_API_S_VER_1 */
+
+#endif /* __iwl_fw_api_mac_cfg_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-mac.h b/drivers/net/wireless/intel/iwlwifi/fw/api/mac.h
index 0c3350ad2f2f..f2e31e040a7b 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-mac.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/mac.h
@@ -6,6 +6,7 @@
  * GPL LICENSE SUMMARY
  *
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2017        Intel Deutschland GmbH
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -16,11 +17,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
- * USA
- *
  * The full GNU General Public License is included in this distribution
  * in the file called COPYING.
  *
@@ -31,6 +27,7 @@
  * BSD LICENSE
  *
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2017        Intel Deutschland GmbH
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -60,8 +57,8 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *****************************************************************************/
 
-#ifndef __fw_api_mac_h__
-#define __fw_api_mac_h__
+#ifndef __iwl_fw_api_mac_h__
+#define __iwl_fw_api_mac_h__
 
 /*
  * The first MAC indices (starting from 0) are available to the driver,
@@ -76,8 +73,6 @@
 #define IWL_MVM_STATION_COUNT		16
 #define IWL_MVM_INVALID_STA		0xFF
 
-#define IWL_MVM_TDLS_STA_COUNT	4
-
 enum iwl_ac {
 	AC_BK,
 	AC_BE,
@@ -393,4 +388,22 @@ struct iwl_nonqos_seq_query_cmd {
 	__le16 reserved;
 } __packed; /* NON_QOS_TX_COUNTER_GET_SET_API_S_VER_1 */
 
-#endif /* __fw_api_mac_h__ */
+/**
+ * struct iwl_missed_beacons_notif - information on missed beacons
+ * ( MISSED_BEACONS_NOTIFICATION = 0xa2 )
+ * @mac_id: interface ID
+ * @consec_missed_beacons_since_last_rx: number of consecutive missed
+ *	beacons since last RX.
+ * @consec_missed_beacons: number of consecutive missed beacons
+ * @num_expected_beacons: number of expected beacons
+ * @num_recvd_beacons: number of received beacons
+ */
+struct iwl_missed_beacons_notif {
+	__le32 mac_id;
+	__le32 consec_missed_beacons_since_last_rx;
+	__le32 consec_missed_beacons;
+	__le32 num_expected_beacons;
+	__le32 num_recvd_beacons;
+} __packed; /* MISSED_BEACON_NTFY_API_S_VER_3 */
+
+#endif /* __iwl_fw_api_mac_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/nvm-reg.h b/drivers/net/wireless/intel/iwlwifi/fw/api/nvm-reg.h
new file mode 100644
index 000000000000..00bc7a25dece
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/nvm-reg.h
@@ -0,0 +1,386 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+
+#ifndef __iwl_fw_api_nvm_reg_h__
+#define __iwl_fw_api_nvm_reg_h__
+
+/**
+ * enum iwl_regulatory_and_nvm_subcmd_ids - regulatory/NVM commands
+ */
+enum iwl_regulatory_and_nvm_subcmd_ids {
+	/**
+	 * @NVM_ACCESS_COMPLETE: &struct iwl_nvm_access_complete_cmd
+	 */
+	NVM_ACCESS_COMPLETE = 0x0,
+
+	/**
+	 * @NVM_GET_INFO:
+	 * Command is &struct iwl_nvm_get_info,
+	 * response is &struct iwl_nvm_get_info_rsp
+	 */
+	NVM_GET_INFO = 0x2,
+};
+
+/**
+ * enum iwl_nvm_access_op - NVM access opcode
+ * @IWL_NVM_READ: read NVM
+ * @IWL_NVM_WRITE: write NVM
+ */
+enum iwl_nvm_access_op {
+	IWL_NVM_READ	= 0,
+	IWL_NVM_WRITE	= 1,
+};
+
+/**
+ * enum iwl_nvm_access_target - target of the NVM_ACCESS_CMD
+ * @NVM_ACCESS_TARGET_CACHE: access the cache
+ * @NVM_ACCESS_TARGET_OTP: access the OTP
+ * @NVM_ACCESS_TARGET_EEPROM: access the EEPROM
+ */
+enum iwl_nvm_access_target {
+	NVM_ACCESS_TARGET_CACHE = 0,
+	NVM_ACCESS_TARGET_OTP = 1,
+	NVM_ACCESS_TARGET_EEPROM = 2,
+};
+
+/**
+ * enum iwl_nvm_section_type - section types for NVM_ACCESS_CMD
+ * @NVM_SECTION_TYPE_SW: software section
+ * @NVM_SECTION_TYPE_REGULATORY: regulatory section
+ * @NVM_SECTION_TYPE_CALIBRATION: calibration section
+ * @NVM_SECTION_TYPE_PRODUCTION: production section
+ * @NVM_SECTION_TYPE_MAC_OVERRIDE: MAC override section
+ * @NVM_SECTION_TYPE_PHY_SKU: PHY SKU section
+ * @NVM_MAX_NUM_SECTIONS: number of sections
+ */
+enum iwl_nvm_section_type {
+	NVM_SECTION_TYPE_SW = 1,
+	NVM_SECTION_TYPE_REGULATORY = 3,
+	NVM_SECTION_TYPE_CALIBRATION = 4,
+	NVM_SECTION_TYPE_PRODUCTION = 5,
+	NVM_SECTION_TYPE_MAC_OVERRIDE = 11,
+	NVM_SECTION_TYPE_PHY_SKU = 12,
+	NVM_MAX_NUM_SECTIONS = 13,
+};
+
+/**
+ * struct iwl_nvm_access_cmd - Request the device to send an NVM section
+ * @op_code: &enum iwl_nvm_access_op
+ * @target: &enum iwl_nvm_access_target
+ * @type: &enum iwl_nvm_section_type
+ * @offset: offset in bytes into the section
+ * @length: in bytes, to read/write
+ * @data: if write operation, the data to write. On read its empty
+ */
+struct iwl_nvm_access_cmd {
+	u8 op_code;
+	u8 target;
+	__le16 type;
+	__le16 offset;
+	__le16 length;
+	u8 data[];
+} __packed; /* NVM_ACCESS_CMD_API_S_VER_2 */
+
+/**
+ * struct iwl_nvm_access_resp_ver2 - response to NVM_ACCESS_CMD
+ * @offset: offset in bytes into the section
+ * @length: in bytes, either how much was written or read
+ * @type: NVM_SECTION_TYPE_*
+ * @status: 0 for success, fail otherwise
+ * @data: if read operation, the data returned. Empty on write.
+ */
+struct iwl_nvm_access_resp {
+	__le16 offset;
+	__le16 length;
+	__le16 type;
+	__le16 status;
+	u8 data[];
+} __packed; /* NVM_ACCESS_CMD_RESP_API_S_VER_2 */
+
+/*
+ * struct iwl_nvm_get_info - request to get NVM data
+ */
+struct iwl_nvm_get_info {
+	__le32 reserved;
+} __packed; /* GRP_REGULATORY_NVM_GET_INFO_CMD_S_VER_1 */
+
+/**
+ * enum iwl_nvm_info_general_flags - flags in NVM_GET_INFO resp
+ * @NVM_GENERAL_FLAGS_EMPTY_OTP: 1 if OTP is empty
+ */
+enum iwl_nvm_info_general_flags {
+	NVM_GENERAL_FLAGS_EMPTY_OTP	= BIT(0),
+};
+
+/**
+ * struct iwl_nvm_get_info_general - general NVM data
+ * @flags: bit 0: 1 - empty, 0 - non-empty
+ * @nvm_version: nvm version
+ * @board_type: board type
+ * @reserved: reserved
+ */
+struct iwl_nvm_get_info_general {
+	__le32 flags;
+	__le16 nvm_version;
+	u8 board_type;
+	u8 reserved;
+} __packed; /* GRP_REGULATORY_NVM_GET_INFO_GENERAL_S_VER_1 */
+
+/**
+ * struct iwl_nvm_get_info_sku - mac information
+ * @enable_24g: band 2.4G enabled
+ * @enable_5g: band 5G enabled
+ * @enable_11n: 11n enabled
+ * @enable_11ac: 11ac enabled
+ * @mimo_disable: MIMO enabled
+ * @ext_crypto: Extended crypto enabled
+ */
+struct iwl_nvm_get_info_sku {
+	__le32 enable_24g;
+	__le32 enable_5g;
+	__le32 enable_11n;
+	__le32 enable_11ac;
+	__le32 mimo_disable;
+	__le32 ext_crypto;
+} __packed; /* GRP_REGULATORY_NVM_GET_INFO_MAC_SKU_SECTION_S_VER_1 */
+
+/**
+ * struct iwl_nvm_get_info_phy - phy information
+ * @tx_chains: BIT 0 chain A, BIT 1 chain B
+ * @rx_chains: BIT 0 chain A, BIT 1 chain B
+ */
+struct iwl_nvm_get_info_phy {
+	__le32 tx_chains;
+	__le32 rx_chains;
+} __packed; /* GRP_REGULATORY_NVM_GET_INFO_PHY_SKU_SECTION_S_VER_1 */
+
+#define IWL_NUM_CHANNELS (51)
+
+/**
+ * struct iwl_nvm_get_info_regulatory - regulatory information
+ * @lar_enabled: is LAR enabled
+ * @channel_profile: regulatory data of this channel
+ * @reserved: reserved
+ */
+struct iwl_nvm_get_info_regulatory {
+	__le32 lar_enabled;
+	__le16 channel_profile[IWL_NUM_CHANNELS];
+	__le16 reserved;
+} __packed; /* GRP_REGULATORY_NVM_GET_INFO_REGULATORY_S_VER_1 */
+
+/**
+ * struct iwl_nvm_get_info_rsp - response to get NVM data
+ * @general: general NVM data
+ * @mac_sku: data relating to MAC sku
+ * @phy_sku: data relating to PHY sku
+ * @regulatory: regulatory data
+ */
+struct iwl_nvm_get_info_rsp {
+	struct iwl_nvm_get_info_general general;
+	struct iwl_nvm_get_info_sku mac_sku;
+	struct iwl_nvm_get_info_phy phy_sku;
+	struct iwl_nvm_get_info_regulatory regulatory;
+} __packed; /* GRP_REGULATORY_NVM_GET_INFO_CMD_RSP_S_VER_1 */
+
+/**
+ * struct iwl_nvm_access_complete_cmd - NVM_ACCESS commands are completed
+ * @reserved: reserved
+ */
+struct iwl_nvm_access_complete_cmd {
+	__le32 reserved;
+} __packed; /* NVM_ACCESS_COMPLETE_CMD_API_S_VER_1 */
+
+/**
+ * struct iwl_mcc_update_cmd_v1 - Request the device to update geographic
+ * regulatory profile according to the given MCC (Mobile Country Code).
+ * The MCC is two letter-code, ascii upper case[A-Z] or '00' for world domain.
+ * 'ZZ' MCC will be used to switch to NVM default profile; in this case, the
+ * MCC in the cmd response will be the relevant MCC in the NVM.
+ * @mcc: given mobile country code
+ * @source_id: the source from where we got the MCC, see iwl_mcc_source
+ * @reserved: reserved for alignment
+ */
+struct iwl_mcc_update_cmd_v1 {
+	__le16 mcc;
+	u8 source_id;
+	u8 reserved;
+} __packed; /* LAR_UPDATE_MCC_CMD_API_S_VER_1 */
+
+/**
+ * struct iwl_mcc_update_cmd - Request the device to update geographic
+ * regulatory profile according to the given MCC (Mobile Country Code).
+ * The MCC is two letter-code, ascii upper case[A-Z] or '00' for world domain.
+ * 'ZZ' MCC will be used to switch to NVM default profile; in this case, the
+ * MCC in the cmd response will be the relevant MCC in the NVM.
+ * @mcc: given mobile country code
+ * @source_id: the source from where we got the MCC, see iwl_mcc_source
+ * @reserved: reserved for alignment
+ * @key: integrity key for MCC API OEM testing
+ * @reserved2: reserved
+ */
+struct iwl_mcc_update_cmd {
+	__le16 mcc;
+	u8 source_id;
+	u8 reserved;
+	__le32 key;
+	u8 reserved2[20];
+} __packed; /* LAR_UPDATE_MCC_CMD_API_S_VER_2 */
+
+/**
+ * struct iwl_mcc_update_resp_v1  - response to MCC_UPDATE_CMD.
+ * Contains the new channel control profile map, if changed, and the new MCC
+ * (mobile country code).
+ * The new MCC may be different than what was requested in MCC_UPDATE_CMD.
+ * @status: see &enum iwl_mcc_update_status
+ * @mcc: the new applied MCC
+ * @cap: capabilities for all channels which matches the MCC
+ * @source_id: the MCC source, see iwl_mcc_source
+ * @n_channels: number of channels in @channels_data (may be 14, 39, 50 or 51
+ *		channels, depending on platform)
+ * @channels: channel control data map, DWORD for each channel. Only the first
+ *	16bits are used.
+ */
+struct iwl_mcc_update_resp_v1  {
+	__le32 status;
+	__le16 mcc;
+	u8 cap;
+	u8 source_id;
+	__le32 n_channels;
+	__le32 channels[0];
+} __packed; /* LAR_UPDATE_MCC_CMD_RESP_S_VER_1 */
+
+/**
+ * struct iwl_mcc_update_resp - response to MCC_UPDATE_CMD.
+ * Contains the new channel control profile map, if changed, and the new MCC
+ * (mobile country code).
+ * The new MCC may be different than what was requested in MCC_UPDATE_CMD.
+ * @status: see &enum iwl_mcc_update_status
+ * @mcc: the new applied MCC
+ * @cap: capabilities for all channels which matches the MCC
+ * @source_id: the MCC source, see iwl_mcc_source
+ * @time: time elapsed from the MCC test start (in 30 seconds TU)
+ * @reserved: reserved.
+ * @n_channels: number of channels in @channels_data (may be 14, 39, 50 or 51
+ *		channels, depending on platform)
+ * @channels: channel control data map, DWORD for each channel. Only the first
+ *	16bits are used.
+ */
+struct iwl_mcc_update_resp {
+	__le32 status;
+	__le16 mcc;
+	u8 cap;
+	u8 source_id;
+	__le16 time;
+	__le16 reserved;
+	__le32 n_channels;
+	__le32 channels[0];
+} __packed; /* LAR_UPDATE_MCC_CMD_RESP_S_VER_2 */
+
+/**
+ * struct iwl_mcc_chub_notif - chub notifies of mcc change
+ * (MCC_CHUB_UPDATE_CMD = 0xc9)
+ * The Chub (Communication Hub, CommsHUB) is a HW component that connects to
+ * the cellular and connectivity cores that gets updates of the mcc, and
+ * notifies the ucode directly of any mcc change.
+ * The ucode requests the driver to request the device to update geographic
+ * regulatory  profile according to the given MCC (Mobile Country Code).
+ * The MCC is two letter-code, ascii upper case[A-Z] or '00' for world domain.
+ * 'ZZ' MCC will be used to switch to NVM default profile; in this case, the
+ * MCC in the cmd response will be the relevant MCC in the NVM.
+ * @mcc: given mobile country code
+ * @source_id: identity of the change originator, see iwl_mcc_source
+ * @reserved1: reserved for alignment
+ */
+struct iwl_mcc_chub_notif {
+	__le16 mcc;
+	u8 source_id;
+	u8 reserved1;
+} __packed; /* LAR_MCC_NOTIFY_S */
+
+enum iwl_mcc_update_status {
+	MCC_RESP_NEW_CHAN_PROFILE,
+	MCC_RESP_SAME_CHAN_PROFILE,
+	MCC_RESP_INVALID,
+	MCC_RESP_NVM_DISABLED,
+	MCC_RESP_ILLEGAL,
+	MCC_RESP_LOW_PRIORITY,
+	MCC_RESP_TEST_MODE_ACTIVE,
+	MCC_RESP_TEST_MODE_NOT_ACTIVE,
+	MCC_RESP_TEST_MODE_DENIAL_OF_SERVICE,
+};
+
+enum iwl_mcc_source {
+	MCC_SOURCE_OLD_FW = 0,
+	MCC_SOURCE_ME = 1,
+	MCC_SOURCE_BIOS = 2,
+	MCC_SOURCE_3G_LTE_HOST = 3,
+	MCC_SOURCE_3G_LTE_DEVICE = 4,
+	MCC_SOURCE_WIFI = 5,
+	MCC_SOURCE_RESERVED = 6,
+	MCC_SOURCE_DEFAULT = 7,
+	MCC_SOURCE_UNINITIALIZED = 8,
+	MCC_SOURCE_MCC_API = 9,
+	MCC_SOURCE_GET_CURRENT = 0x10,
+	MCC_SOURCE_GETTING_MCC_TEST_MODE = 0x11,
+};
+
+#endif /* __iwl_fw_api_nvm_reg_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/offload.h b/drivers/net/wireless/intel/iwlwifi/fw/api/offload.h
new file mode 100644
index 000000000000..53cab993068f
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/offload.h
@@ -0,0 +1,101 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+
+#ifndef __iwl_fw_api_offload_h__
+#define __iwl_fw_api_offload_h__
+
+/**
+ * enum iwl_prot_offload_subcmd_ids - protocol offload commands
+ */
+enum iwl_prot_offload_subcmd_ids {
+	/**
+	 * @STORED_BEACON_NTF: &struct iwl_stored_beacon_notif
+	 */
+	STORED_BEACON_NTF = 0xFF,
+};
+
+#define MAX_STORED_BEACON_SIZE 600
+
+/**
+ * struct iwl_stored_beacon_notif - Stored beacon notification
+ *
+ * @system_time: system time on air rise
+ * @tsf: TSF on air rise
+ * @beacon_timestamp: beacon on air rise
+ * @band: band, matches &RX_RES_PHY_FLAGS_BAND_24 definition
+ * @channel: channel this beacon was received on
+ * @rates: rate in ucode internal format
+ * @byte_count: frame's byte count
+ * @data: beacon data, length in @byte_count
+ */
+struct iwl_stored_beacon_notif {
+	__le32 system_time;
+	__le64 tsf;
+	__le32 beacon_timestamp;
+	__le16 band;
+	__le16 channel;
+	__le32 rates;
+	__le32 byte_count;
+	u8 data[MAX_STORED_BEACON_SIZE];
+} __packed; /* WOWLAN_STROED_BEACON_INFO_S_VER_2 */
+
+#endif /* __iwl_fw_api_offload_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/paging.h b/drivers/net/wireless/intel/iwlwifi/fw/api/paging.h
new file mode 100644
index 000000000000..e76f9cd4473d
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/paging.h
@@ -0,0 +1,108 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2007 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2005 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+#ifndef __iwl_fw_api_paging_h__
+#define __iwl_fw_api_paging_h__
+
+#define NUM_OF_FW_PAGING_BLOCKS	33 /* 32 for data and 1 block for CSS */
+
+/**
+ * struct iwl_fw_paging_cmd - paging layout
+ *
+ * Send to FW the paging layout in the driver.
+ *
+ * @flags: various flags for the command
+ * @block_size: the block size in powers of 2
+ * @block_num: number of blocks specified in the command.
+ * @device_phy_addr: virtual addresses from device side
+ */
+struct iwl_fw_paging_cmd {
+	__le32 flags;
+	__le32 block_size;
+	__le32 block_num;
+	__le32 device_phy_addr[NUM_OF_FW_PAGING_BLOCKS];
+} __packed; /* FW_PAGING_BLOCK_CMD_API_S_VER_1 */
+
+/**
+ * enum iwl_fw_item_id - FW item IDs
+ *
+ * @IWL_FW_ITEM_ID_PAGING: Address of the pages that the FW will upload
+ *	download
+ */
+enum iwl_fw_item_id {
+	IWL_FW_ITEM_ID_PAGING = 3,
+};
+
+/**
+ * struct iwl_fw_get_item_cmd - get an item from the fw
+ * @item_id: ID of item to obtain, see &enum iwl_fw_item_id
+ */
+struct iwl_fw_get_item_cmd {
+	__le32 item_id;
+} __packed; /* FW_GET_ITEM_CMD_API_S_VER_1 */
+
+struct iwl_fw_get_item_resp {
+	__le32 item_id;
+	__le32 item_byte_cnt;
+	__le32 item_val;
+} __packed; /* FW_GET_ITEM_RSP_S_VER_1 */
+
+#endif /* __iwl_fw_api_paging_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/phy-ctxt.h b/drivers/net/wireless/intel/iwlwifi/fw/api/phy-ctxt.h
new file mode 100644
index 000000000000..45f61c6af14e
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/phy-ctxt.h
@@ -0,0 +1,164 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+
+#ifndef __iwl_fw_api_phy_ctxt_h__
+#define __iwl_fw_api_phy_ctxt_h__
+
+/* Supported bands */
+#define PHY_BAND_5  (0)
+#define PHY_BAND_24 (1)
+
+/* Supported channel width, vary if there is VHT support */
+#define PHY_VHT_CHANNEL_MODE20	(0x0)
+#define PHY_VHT_CHANNEL_MODE40	(0x1)
+#define PHY_VHT_CHANNEL_MODE80	(0x2)
+#define PHY_VHT_CHANNEL_MODE160	(0x3)
+
+/*
+ * Control channel position:
+ * For legacy set bit means upper channel, otherwise lower.
+ * For VHT - bit-2 marks if the control is lower/upper relative to center-freq
+ *   bits-1:0 mark the distance from the center freq. for 20Mhz, offset is 0.
+ *                                   center_freq
+ *                                        |
+ * 40Mhz                          |_______|_______|
+ * 80Mhz                  |_______|_______|_______|_______|
+ * 160Mhz |_______|_______|_______|_______|_______|_______|_______|_______|
+ * code      011     010     001     000  |  100     101     110    111
+ */
+#define PHY_VHT_CTRL_POS_1_BELOW  (0x0)
+#define PHY_VHT_CTRL_POS_2_BELOW  (0x1)
+#define PHY_VHT_CTRL_POS_3_BELOW  (0x2)
+#define PHY_VHT_CTRL_POS_4_BELOW  (0x3)
+#define PHY_VHT_CTRL_POS_1_ABOVE  (0x4)
+#define PHY_VHT_CTRL_POS_2_ABOVE  (0x5)
+#define PHY_VHT_CTRL_POS_3_ABOVE  (0x6)
+#define PHY_VHT_CTRL_POS_4_ABOVE  (0x7)
+
+/*
+ * @band: PHY_BAND_*
+ * @channel: channel number
+ * @width: PHY_[VHT|LEGACY]_CHANNEL_*
+ * @ctrl channel: PHY_[VHT|LEGACY]_CTRL_*
+ */
+struct iwl_fw_channel_info {
+	u8 band;
+	u8 channel;
+	u8 width;
+	u8 ctrl_pos;
+} __packed;
+
+#define PHY_RX_CHAIN_DRIVER_FORCE_POS	(0)
+#define PHY_RX_CHAIN_DRIVER_FORCE_MSK \
+	(0x1 << PHY_RX_CHAIN_DRIVER_FORCE_POS)
+#define PHY_RX_CHAIN_VALID_POS		(1)
+#define PHY_RX_CHAIN_VALID_MSK \
+	(0x7 << PHY_RX_CHAIN_VALID_POS)
+#define PHY_RX_CHAIN_FORCE_SEL_POS	(4)
+#define PHY_RX_CHAIN_FORCE_SEL_MSK \
+	(0x7 << PHY_RX_CHAIN_FORCE_SEL_POS)
+#define PHY_RX_CHAIN_FORCE_MIMO_SEL_POS	(7)
+#define PHY_RX_CHAIN_FORCE_MIMO_SEL_MSK \
+	(0x7 << PHY_RX_CHAIN_FORCE_MIMO_SEL_POS)
+#define PHY_RX_CHAIN_CNT_POS		(10)
+#define PHY_RX_CHAIN_CNT_MSK \
+	(0x3 << PHY_RX_CHAIN_CNT_POS)
+#define PHY_RX_CHAIN_MIMO_CNT_POS	(12)
+#define PHY_RX_CHAIN_MIMO_CNT_MSK \
+	(0x3 << PHY_RX_CHAIN_MIMO_CNT_POS)
+#define PHY_RX_CHAIN_MIMO_FORCE_POS	(14)
+#define PHY_RX_CHAIN_MIMO_FORCE_MSK \
+	(0x1 << PHY_RX_CHAIN_MIMO_FORCE_POS)
+
+/* TODO: fix the value, make it depend on firmware at runtime? */
+#define NUM_PHY_CTX	3
+
+/* TODO: complete missing documentation */
+/**
+ * struct iwl_phy_context_cmd - config of the PHY context
+ * ( PHY_CONTEXT_CMD = 0x8 )
+ * @id_and_color: ID and color of the relevant Binding
+ * @action: action to perform, one of FW_CTXT_ACTION_*
+ * @apply_time: 0 means immediate apply and context switch.
+ *	other value means apply new params after X usecs
+ * @tx_param_color: ???
+ * @ci: channel info
+ * @txchain_info: ???
+ * @rxchain_info: ???
+ * @acquisition_data: ???
+ * @dsp_cfg_flags: set to 0
+ */
+struct iwl_phy_context_cmd {
+	/* COMMON_INDEX_HDR_API_S_VER_1 */
+	__le32 id_and_color;
+	__le32 action;
+	/* PHY_CONTEXT_DATA_API_S_VER_1 */
+	__le32 apply_time;
+	__le32 tx_param_color;
+	struct iwl_fw_channel_info ci;
+	__le32 txchain_info;
+	__le32 rxchain_info;
+	__le32 acquisition_data;
+	__le32 dsp_cfg_flags;
+} __packed; /* PHY_CONTEXT_CMD_API_VER_1 */
+
+#endif /* __iwl_fw_api_phy_ctxt_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/phy.h b/drivers/net/wireless/intel/iwlwifi/fw/api/phy.h
new file mode 100644
index 000000000000..9cc59e00bd95
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/phy.h
@@ -0,0 +1,258 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+
+#ifndef __iwl_fw_api_phy_h__
+#define __iwl_fw_api_phy_h__
+
+/**
+ * enum iwl_phy_ops_subcmd_ids - PHY group commands
+ */
+enum iwl_phy_ops_subcmd_ids {
+	/**
+	 * @CMD_DTS_MEASUREMENT_TRIGGER_WIDE:
+	 * Uses either &struct iwl_dts_measurement_cmd or
+	 * &struct iwl_ext_dts_measurement_cmd
+	 */
+	CMD_DTS_MEASUREMENT_TRIGGER_WIDE = 0x0,
+
+	/**
+	 * @CTDP_CONFIG_CMD: &struct iwl_mvm_ctdp_cmd
+	 */
+	CTDP_CONFIG_CMD = 0x03,
+
+	/**
+	 * @TEMP_REPORTING_THRESHOLDS_CMD: &struct temp_report_ths_cmd
+	 */
+	TEMP_REPORTING_THRESHOLDS_CMD = 0x04,
+
+	/**
+	 * @GEO_TX_POWER_LIMIT: &struct iwl_geo_tx_power_profiles_cmd
+	 */
+	GEO_TX_POWER_LIMIT = 0x05,
+
+	/**
+	 * @CT_KILL_NOTIFICATION: &struct ct_kill_notif
+	 */
+	CT_KILL_NOTIFICATION = 0xFE,
+
+	/**
+	 * @DTS_MEASUREMENT_NOTIF_WIDE:
+	 * &struct iwl_dts_measurement_notif_v1 or
+	 * &struct iwl_dts_measurement_notif_v2
+	 */
+	DTS_MEASUREMENT_NOTIF_WIDE = 0xFF,
+};
+
+/* DTS measurements */
+
+enum iwl_dts_measurement_flags {
+	DTS_TRIGGER_CMD_FLAGS_TEMP	= BIT(0),
+	DTS_TRIGGER_CMD_FLAGS_VOLT	= BIT(1),
+};
+
+/**
+ * struct iwl_dts_measurement_cmd - request DTS temp and/or voltage measurements
+ *
+ * @flags: indicates which measurements we want as specified in
+ *	&enum iwl_dts_measurement_flags
+ */
+struct iwl_dts_measurement_cmd {
+	__le32 flags;
+} __packed; /* TEMPERATURE_MEASUREMENT_TRIGGER_CMD_S */
+
+/**
+* enum iwl_dts_control_measurement_mode - DTS measurement type
+* @DTS_AUTOMATIC: Automatic mode (full SW control). Provide temperature read
+*                 back (latest value. Not waiting for new value). Use automatic
+*                 SW DTS configuration.
+* @DTS_REQUEST_READ: Request DTS read. Configure DTS with manual settings,
+*                    trigger DTS reading and provide read back temperature read
+*                    when available.
+* @DTS_OVER_WRITE: over-write the DTS temperatures in the SW until next read
+* @DTS_DIRECT_WITHOUT_MEASURE: DTS returns its latest temperature result,
+*                              without measurement trigger.
+*/
+enum iwl_dts_control_measurement_mode {
+	DTS_AUTOMATIC			= 0,
+	DTS_REQUEST_READ		= 1,
+	DTS_OVER_WRITE			= 2,
+	DTS_DIRECT_WITHOUT_MEASURE	= 3,
+};
+
+/**
+* enum iwl_dts_used - DTS to use or used for measurement in the DTS request
+* @DTS_USE_TOP: Top
+* @DTS_USE_CHAIN_A: chain A
+* @DTS_USE_CHAIN_B: chain B
+* @DTS_USE_CHAIN_C: chain C
+* @XTAL_TEMPERATURE: read temperature from xtal
+*/
+enum iwl_dts_used {
+	DTS_USE_TOP		= 0,
+	DTS_USE_CHAIN_A		= 1,
+	DTS_USE_CHAIN_B		= 2,
+	DTS_USE_CHAIN_C		= 3,
+	XTAL_TEMPERATURE	= 4,
+};
+
+/**
+* enum iwl_dts_bit_mode - bit-mode to use in DTS request read mode
+* @DTS_BIT6_MODE: bit 6 mode
+* @DTS_BIT8_MODE: bit 8 mode
+*/
+enum iwl_dts_bit_mode {
+	DTS_BIT6_MODE	= 0,
+	DTS_BIT8_MODE	= 1,
+};
+
+/**
+ * struct iwl_ext_dts_measurement_cmd - request extended DTS temp measurements
+ * @control_mode: see &enum iwl_dts_control_measurement_mode
+ * @temperature: used when over write DTS mode is selected
+ * @sensor: set temperature sensor to use. See &enum iwl_dts_used
+ * @avg_factor: average factor to DTS in request DTS read mode
+ * @bit_mode: value defines the DTS bit mode to use. See &enum iwl_dts_bit_mode
+ * @step_duration: step duration for the DTS
+ */
+struct iwl_ext_dts_measurement_cmd {
+	__le32 control_mode;
+	__le32 temperature;
+	__le32 sensor;
+	__le32 avg_factor;
+	__le32 bit_mode;
+	__le32 step_duration;
+} __packed; /* XVT_FW_DTS_CONTROL_MEASUREMENT_REQUEST_API_S */
+
+/**
+ * struct iwl_dts_measurement_notif_v1 - measurements notification
+ *
+ * @temp: the measured temperature
+ * @voltage: the measured voltage
+ */
+struct iwl_dts_measurement_notif_v1 {
+	__le32 temp;
+	__le32 voltage;
+} __packed; /* TEMPERATURE_MEASUREMENT_TRIGGER_NTFY_S_VER_1*/
+
+/**
+ * struct iwl_dts_measurement_notif_v2 - measurements notification
+ *
+ * @temp: the measured temperature
+ * @voltage: the measured voltage
+ * @threshold_idx: the trip index that was crossed
+ */
+struct iwl_dts_measurement_notif_v2 {
+	__le32 temp;
+	__le32 voltage;
+	__le32 threshold_idx;
+} __packed; /* TEMPERATURE_MEASUREMENT_TRIGGER_NTFY_S_VER_2 */
+
+/**
+ * struct ct_kill_notif - CT-kill entry notification
+ *
+ * @temperature: the current temperature in celsius
+ * @reserved: reserved
+ */
+struct ct_kill_notif {
+	__le16 temperature;
+	__le16 reserved;
+} __packed; /* GRP_PHY_CT_KILL_NTF */
+
+/**
+* enum ctdp_cmd_operation - CTDP command operations
+* @CTDP_CMD_OPERATION_START: update the current budget
+* @CTDP_CMD_OPERATION_STOP: stop ctdp
+* @CTDP_CMD_OPERATION_REPORT: get the average budget
+*/
+enum iwl_mvm_ctdp_cmd_operation {
+	CTDP_CMD_OPERATION_START	= 0x1,
+	CTDP_CMD_OPERATION_STOP		= 0x2,
+	CTDP_CMD_OPERATION_REPORT	= 0x4,
+};/* CTDP_CMD_OPERATION_TYPE_E */
+
+/**
+ * struct iwl_mvm_ctdp_cmd - track and manage the FW power consumption budget
+ *
+ * @operation: see &enum iwl_mvm_ctdp_cmd_operation
+ * @budget: the budget in milliwatt
+ * @window_size: defined in API but not used
+ */
+struct iwl_mvm_ctdp_cmd {
+	__le32 operation;
+	__le32 budget;
+	__le32 window_size;
+} __packed;
+
+#define IWL_MAX_DTS_TRIPS	8
+
+/**
+ * struct temp_report_ths_cmd - set temperature thresholds
+ *
+ * @num_temps: number of temperature thresholds passed
+ * @thresholds: array with the thresholds to be configured
+ */
+struct temp_report_ths_cmd {
+	__le32 num_temps;
+	__le16 thresholds[IWL_MAX_DTS_TRIPS];
+} __packed; /* GRP_PHY_TEMP_REPORTING_THRESHOLDS_CMD */
+
+#endif /* __iwl_fw_api_phy_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-power.h b/drivers/net/wireless/intel/iwlwifi/fw/api/power.h
index 7da57ef2454e..a06afb5605d2 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-power.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/power.h
@@ -18,11 +18,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
- * USA
- *
  * The full GNU General Public License is included in this distribution
  * in the file called COPYING.
  *
@@ -65,8 +60,8 @@
  *
  *****************************************************************************/
 
-#ifndef __fw_api_power_h__
-#define __fw_api_power_h__
+#ifndef __iwl_fw_api_power_h__
+#define __iwl_fw_api_power_h__
 
 /* Power Management Commands, Responses, Notifications */
 
@@ -224,7 +219,7 @@ struct iwl_device_power_cmd {
 /**
  * struct iwl_mac_power_cmd - New power command containing uAPSD support
  * MAC_PM_POWER_TABLE = 0xA9 (command, has simple generic response)
- * @id_and_color:	MAC contex identifier, &enum iwl_mvm_id_and_color
+ * @id_and_color:	MAC contex identifier, &enum iwl_ctxt_id_and_color
  * @flags:		Power table command flags from POWER_FLAGS_*
  * @keep_alive_seconds:	Keep alive period in seconds. Default - 25 sec.
  *			Minimum allowed:- 3 * DTIM. Keep alive period must be
@@ -528,4 +523,4 @@ struct iwl_beacon_filter_cmd {
 
 #define IWL_BF_CMD_CONFIG_DEFAULTS IWL_BF_CMD_CONFIG(_DEFAULT)
 #define IWL_BF_CMD_CONFIG_D0I3 IWL_BF_CMD_CONFIG(_D0I3)
-#endif
+#endif /* __iwl_fw_api_power_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-rs.h b/drivers/net/wireless/intel/iwlwifi/fw/api/rs.h
index bdf1228d050b..a13fd8a1be62 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-rs.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/rs.h
@@ -17,11 +17,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
- * USA
- *
  * The full GNU General Public License is included in this distribution
  * in the file called COPYING.
  *
@@ -62,10 +57,10 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *****************************************************************************/
 
-#ifndef __fw_api_rs_h__
-#define __fw_api_rs_h__
+#ifndef __iwl_fw_api_rs_h__
+#define __iwl_fw_api_rs_h__
 
-#include "fw-api-mac.h"
+#include "mac.h"
 
 /*
  * These serve as indexes into
@@ -410,4 +405,4 @@ struct iwl_lq_cmd {
 	__le32 ss_params;
 }; /* LINK_QUALITY_CMD_API_S_VER_1 */
 
-#endif /* __fw_api_rs_h__ */
+#endif /* __iwl_fw_api_rs_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-rx.h b/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h
index 59038ade08d8..e7565f37ece9 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-rx.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h
@@ -18,11 +18,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
- * USA
- *
  * The full GNU General Public License is included in this distribution
  * in the file called COPYING.
  *
@@ -65,8 +60,8 @@
  *
  *****************************************************************************/
 
-#ifndef __fw_api_rx_h__
-#define __fw_api_rx_h__
+#ifndef __iwl_fw_api_rx_h__
+#define __iwl_fw_api_rx_h__
 
 /* API for pre-9000 hardware */
 
@@ -571,4 +566,24 @@ struct iwl_mvm_pm_state_notification {
 	__le16 reserved;
 } __packed; /* PEER_PM_NTFY_API_S_VER_1 */
 
-#endif /* __fw_api_rx_h__ */
+#define BA_WINDOW_STREAMS_MAX		16
+#define BA_WINDOW_STATUS_TID_MSK	0x000F
+#define BA_WINDOW_STATUS_STA_ID_POS	4
+#define BA_WINDOW_STATUS_STA_ID_MSK	0x01F0
+#define BA_WINDOW_STATUS_VALID_MSK	BIT(9)
+
+/**
+ * struct iwl_ba_window_status_notif - reordering window's status notification
+ * @bitmap: bitmap of received frames [start_seq_num + 0]..[start_seq_num + 63]
+ * @ra_tid: bit 3:0 - TID, bit 8:4 - STA_ID, bit 9 - valid
+ * @start_seq_num: the start sequence number of the bitmap
+ * @mpdu_rx_count: the number of received MPDUs since entering D0i3
+ */
+struct iwl_ba_window_status_notif {
+	__le64 bitmap[BA_WINDOW_STREAMS_MAX];
+	__le16 ra_tid[BA_WINDOW_STREAMS_MAX];
+	__le32 start_seq_num[BA_WINDOW_STREAMS_MAX];
+	__le16 mpdu_rx_count[BA_WINDOW_STREAMS_MAX];
+} __packed; /* BA_WINDOW_STATUS_NTFY_API_S_VER_1 */
+
+#endif /* __iwl_fw_api_rx_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-scan.h b/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h
index 1cd7cc087936..5a40092febfb 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-scan.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h
@@ -18,11 +18,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
- * USA
- *
  * The full GNU General Public License is included in this distribution
  * in the file called COPYING.
  *
@@ -65,8 +60,8 @@
  *
  *****************************************************************************/
 
-#ifndef __fw_api_scan_h__
-#define __fw_api_scan_h__
+#ifndef __iwl_fw_api_scan_h__
+#define __iwl_fw_api_scan_h__
 
 /* Scan Commands, Responses, Notifications */
 
@@ -789,4 +784,4 @@ struct iwl_umac_scan_iter_complete_notif {
 	struct iwl_scan_results_notif results[];
 } __packed; /* SCAN_ITER_COMPLETE_NTF_UMAC_API_S_VER_2 */
 
-#endif
+#endif /* __iwl_fw_api_scan_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/sf.h b/drivers/net/wireless/intel/iwlwifi/fw/api/sf.h
new file mode 100644
index 000000000000..e517b55f1bc6
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/sf.h
@@ -0,0 +1,138 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+
+#ifndef __iwl_fw_api_sf_h__
+#define __iwl_fw_api_sf_h__
+
+/* Smart Fifo state */
+enum iwl_sf_state {
+	SF_LONG_DELAY_ON = 0, /* should never be called by driver */
+	SF_FULL_ON,
+	SF_UNINIT,
+	SF_INIT_OFF,
+	SF_HW_NUM_STATES
+};
+
+/* Smart Fifo possible scenario */
+enum iwl_sf_scenario {
+	SF_SCENARIO_SINGLE_UNICAST,
+	SF_SCENARIO_AGG_UNICAST,
+	SF_SCENARIO_MULTICAST,
+	SF_SCENARIO_BA_RESP,
+	SF_SCENARIO_TX_RESP,
+	SF_NUM_SCENARIO
+};
+
+#define SF_TRANSIENT_STATES_NUMBER 2	/* SF_LONG_DELAY_ON and SF_FULL_ON */
+#define SF_NUM_TIMEOUT_TYPES 2		/* Aging timer and Idle timer */
+
+/* smart FIFO default values */
+#define SF_W_MARK_SISO 6144
+#define SF_W_MARK_MIMO2 8192
+#define SF_W_MARK_MIMO3 6144
+#define SF_W_MARK_LEGACY 4096
+#define SF_W_MARK_SCAN 4096
+
+/* SF Scenarios timers for default configuration (aligned to 32 uSec) */
+#define SF_SINGLE_UNICAST_IDLE_TIMER_DEF 160	/* 150 uSec  */
+#define SF_SINGLE_UNICAST_AGING_TIMER_DEF 400	/* 0.4 mSec */
+#define SF_AGG_UNICAST_IDLE_TIMER_DEF 160		/* 150 uSec */
+#define SF_AGG_UNICAST_AGING_TIMER_DEF 400		/* 0.4 mSec */
+#define SF_MCAST_IDLE_TIMER_DEF 160		/* 150 mSec */
+#define SF_MCAST_AGING_TIMER_DEF 400		/* 0.4 mSec */
+#define SF_BA_IDLE_TIMER_DEF 160			/* 150 uSec */
+#define SF_BA_AGING_TIMER_DEF 400			/* 0.4 mSec */
+#define SF_TX_RE_IDLE_TIMER_DEF 160			/* 150 uSec */
+#define SF_TX_RE_AGING_TIMER_DEF 400		/* 0.4 mSec */
+
+/* SF Scenarios timers for BSS MAC configuration (aligned to 32 uSec) */
+#define SF_SINGLE_UNICAST_IDLE_TIMER 320	/* 300 uSec  */
+#define SF_SINGLE_UNICAST_AGING_TIMER 2016	/* 2 mSec */
+#define SF_AGG_UNICAST_IDLE_TIMER 320		/* 300 uSec */
+#define SF_AGG_UNICAST_AGING_TIMER 2016		/* 2 mSec */
+#define SF_MCAST_IDLE_TIMER 2016		/* 2 mSec */
+#define SF_MCAST_AGING_TIMER 10016		/* 10 mSec */
+#define SF_BA_IDLE_TIMER 320			/* 300 uSec */
+#define SF_BA_AGING_TIMER 2016			/* 2 mSec */
+#define SF_TX_RE_IDLE_TIMER 320			/* 300 uSec */
+#define SF_TX_RE_AGING_TIMER 2016		/* 2 mSec */
+
+#define SF_LONG_DELAY_AGING_TIMER 1000000	/* 1 Sec */
+
+#define SF_CFG_DUMMY_NOTIF_OFF	BIT(16)
+
+/**
+ * struct iwl_sf_cfg_cmd - Smart Fifo configuration command.
+ * @state: smart fifo state, types listed in &enum iwl_sf_state.
+ * @watermark: Minimum allowed available free space in RXF for transient state.
+ * @long_delay_timeouts: aging and idle timer values for each scenario
+ * in long delay state.
+ * @full_on_timeouts: timer values for each scenario in full on state.
+ */
+struct iwl_sf_cfg_cmd {
+	__le32 state;
+	__le32 watermark[SF_TRANSIENT_STATES_NUMBER];
+	__le32 long_delay_timeouts[SF_NUM_SCENARIO][SF_NUM_TIMEOUT_TYPES];
+	__le32 full_on_timeouts[SF_NUM_SCENARIO][SF_NUM_TIMEOUT_TYPES];
+} __packed; /* SF_CFG_API_S_VER_2 */
+
+#endif /* __iwl_fw_api_sf_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-sta.h b/drivers/net/wireless/intel/iwlwifi/fw/api/sta.h
index 81f0a3463bac..af369eba3795 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-sta.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/sta.h
@@ -18,11 +18,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
- * USA
- *
  * The full GNU General Public License is included in this distribution
  * in the file called COPYING.
  *
@@ -64,8 +59,8 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *****************************************************************************/
 
-#ifndef __fw_api_sta_h__
-#define __fw_api_sta_h__
+#ifndef __iwl_fw_api_sta_h__
+#define __iwl_fw_api_sta_h__
 
 /**
  * enum iwl_sta_flags - flags for the ADD_STA host command
@@ -291,7 +286,7 @@ struct iwl_mvm_keyinfo {
  * @tid_disable_tx: is tid BIT(tid) enabled for Tx. Clear BIT(x) to enable
  *	AMPDU for tid x. Set %STA_MODIFY_TID_DISABLE_TX to change this field.
  * @mac_id_n_color: the Mac context this station belongs to,
- *	see &enum iwl_mvm_id_and_color
+ *	see &enum iwl_ctxt_id_and_color
  * @addr: station's MAC address
  * @reserved2: reserved
  * @sta_id: index of station in uCode's station table
@@ -372,7 +367,7 @@ enum iwl_sta_type {
  * @tid_disable_tx: is tid BIT(tid) enabled for Tx. Clear BIT(x) to enable
  *	AMPDU for tid x. Set %STA_MODIFY_TID_DISABLE_TX to change this field.
  * @mac_id_n_color: the Mac context this station belongs to,
- *	see &enum iwl_mvm_id_and_color
+ *	see &enum iwl_ctxt_id_and_color
  * @addr: station's MAC address
  * @reserved2: reserved
  * @sta_id: index of station in uCode's station table
@@ -575,4 +570,4 @@ struct iwl_mvm_eosp_notification {
 	__le32 sta_id;
 } __packed; /* UAPSD_EOSP_NTFY_API_S_VER_1 */
 
-#endif /* __fw_api_sta_h__ */
+#endif /* __iwl_fw_api_sta_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-stats.h b/drivers/net/wireless/intel/iwlwifi/fw/api/stats.h
index c7531da508fd..53cb622aa9ab 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-stats.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/stats.h
@@ -18,11 +18,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
- * USA
- *
  * The full GNU General Public License is included in this distribution
  * in the file called COPYING.
  *
@@ -64,9 +59,9 @@
  *
  *****************************************************************************/
 
-#ifndef __fw_api_stats_h__
-#define __fw_api_stats_h__
-#include "fw-api-mac.h"
+#ifndef __iwl_fw_api_stats_h__
+#define __iwl_fw_api_stats_h__
+#include "mac.h"
 
 struct mvm_statistics_dbg {
 	__le32 burst_check;
@@ -476,4 +471,4 @@ struct iwl_statistics_cmd {
 	__le32 flags;
 } __packed; /* STATISTICS_CMD_API_S_VER_1 */
 
-#endif /* __fw_api_stats_h__ */
+#endif /* __iwl_fw_api_stats_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/tdls.h b/drivers/net/wireless/intel/iwlwifi/fw/api/tdls.h
new file mode 100644
index 000000000000..7c6c2462d0e8
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/tdls.h
@@ -0,0 +1,208 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+
+#ifndef __iwl_fw_api_tdls_h__
+#define __iwl_fw_api_tdls_h__
+
+#include "fw/api/tx.h"
+#include "fw/api/phy-ctxt.h"
+
+#define IWL_MVM_TDLS_STA_COUNT	4
+
+/* Type of TDLS request */
+enum iwl_tdls_channel_switch_type {
+	TDLS_SEND_CHAN_SW_REQ = 0,
+	TDLS_SEND_CHAN_SW_RESP_AND_MOVE_CH,
+	TDLS_MOVE_CH,
+}; /* TDLS_STA_CHANNEL_SWITCH_CMD_TYPE_API_E_VER_1 */
+
+/**
+ * struct iwl_tdls_channel_switch_timing - Switch timing in TDLS channel-switch
+ * @frame_timestamp: GP2 timestamp of channel-switch request/response packet
+ *	received from peer
+ * @max_offchan_duration: What amount of microseconds out of a DTIM is given
+ *	to the TDLS off-channel communication. For instance if the DTIM is
+ *	200TU and the TDLS peer is to be given 25% of the time, the value
+ *	given will be 50TU, or 50 * 1024 if translated into microseconds.
+ * @switch_time: switch time the peer sent in its channel switch timing IE
+ * @switch_timeout: switch timeout the peer sent in its channel switch timing IE
+ */
+struct iwl_tdls_channel_switch_timing {
+	__le32 frame_timestamp; /* GP2 time of peer packet Rx */
+	__le32 max_offchan_duration; /* given in micro-seconds */
+	__le32 switch_time; /* given in micro-seconds */
+	__le32 switch_timeout; /* given in micro-seconds */
+} __packed; /* TDLS_STA_CHANNEL_SWITCH_TIMING_DATA_API_S_VER_1 */
+
+#define IWL_TDLS_CH_SW_FRAME_MAX_SIZE 200
+
+/**
+ * struct iwl_tdls_channel_switch_frame - TDLS channel switch frame template
+ *
+ * A template representing a TDLS channel-switch request or response frame
+ *
+ * @switch_time_offset: offset to the channel switch timing IE in the template
+ * @tx_cmd: Tx parameters for the frame
+ * @data: frame data
+ */
+struct iwl_tdls_channel_switch_frame {
+	__le32 switch_time_offset;
+	struct iwl_tx_cmd tx_cmd;
+	u8 data[IWL_TDLS_CH_SW_FRAME_MAX_SIZE];
+} __packed; /* TDLS_STA_CHANNEL_SWITCH_FRAME_API_S_VER_1 */
+
+/**
+ * struct iwl_tdls_channel_switch_cmd - TDLS channel switch command
+ *
+ * The command is sent to initiate a channel switch and also in response to
+ * incoming TDLS channel-switch request/response packets from remote peers.
+ *
+ * @switch_type: see &enum iwl_tdls_channel_switch_type
+ * @peer_sta_id: station id of TDLS peer
+ * @ci: channel we switch to
+ * @timing: timing related data for command
+ * @frame: channel-switch request/response template, depending to switch_type
+ */
+struct iwl_tdls_channel_switch_cmd {
+	u8 switch_type;
+	__le32 peer_sta_id;
+	struct iwl_fw_channel_info ci;
+	struct iwl_tdls_channel_switch_timing timing;
+	struct iwl_tdls_channel_switch_frame frame;
+} __packed; /* TDLS_STA_CHANNEL_SWITCH_CMD_API_S_VER_1 */
+
+/**
+ * struct iwl_tdls_channel_switch_notif - TDLS channel switch start notification
+ *
+ * @status: non-zero on success
+ * @offchannel_duration: duration given in microseconds
+ * @sta_id: peer currently performing the channel-switch with
+ */
+struct iwl_tdls_channel_switch_notif {
+	__le32 status;
+	__le32 offchannel_duration;
+	__le32 sta_id;
+} __packed; /* TDLS_STA_CHANNEL_SWITCH_NTFY_API_S_VER_1 */
+
+/**
+ * struct iwl_tdls_sta_info - TDLS station info
+ *
+ * @sta_id: station id of the TDLS peer
+ * @tx_to_peer_tid: TID reserved vs. the peer for FW based Tx
+ * @tx_to_peer_ssn: initial SSN the FW should use for Tx on its TID vs the peer
+ * @is_initiator: 1 if the peer is the TDLS link initiator, 0 otherwise
+ */
+struct iwl_tdls_sta_info {
+	u8 sta_id;
+	u8 tx_to_peer_tid;
+	__le16 tx_to_peer_ssn;
+	__le32 is_initiator;
+} __packed; /* TDLS_STA_INFO_VER_1 */
+
+/**
+ * struct iwl_tdls_config_cmd - TDLS basic config command
+ *
+ * @id_and_color: MAC id and color being configured
+ * @tdls_peer_count: amount of currently connected TDLS peers
+ * @tx_to_ap_tid: TID reverved vs. the AP for FW based Tx
+ * @tx_to_ap_ssn: initial SSN the FW should use for Tx on its TID vs. the AP
+ * @sta_info: per-station info. Only the first tdls_peer_count entries are set
+ * @pti_req_data_offset: offset of network-level data for the PTI template
+ * @pti_req_tx_cmd: Tx parameters for PTI request template
+ * @pti_req_template: PTI request template data
+ */
+struct iwl_tdls_config_cmd {
+	__le32 id_and_color; /* mac id and color */
+	u8 tdls_peer_count;
+	u8 tx_to_ap_tid;
+	__le16 tx_to_ap_ssn;
+	struct iwl_tdls_sta_info sta_info[IWL_MVM_TDLS_STA_COUNT];
+
+	__le32 pti_req_data_offset;
+	struct iwl_tx_cmd pti_req_tx_cmd;
+	u8 pti_req_template[0];
+} __packed; /* TDLS_CONFIG_CMD_API_S_VER_1 */
+
+/**
+ * struct iwl_tdls_config_sta_info_res - TDLS per-station config information
+ *
+ * @sta_id: station id of the TDLS peer
+ * @tx_to_peer_last_seq: last sequence number used by FW during FW-based Tx to
+ *	the peer
+ */
+struct iwl_tdls_config_sta_info_res {
+	__le16 sta_id;
+	__le16 tx_to_peer_last_seq;
+} __packed; /* TDLS_STA_INFO_RSP_VER_1 */
+
+/**
+ * struct iwl_tdls_config_res - TDLS config information from FW
+ *
+ * @tx_to_ap_last_seq: last sequence number used by FW during FW-based Tx to AP
+ * @sta_info: per-station TDLS config information
+ */
+struct iwl_tdls_config_res {
+	__le32 tx_to_ap_last_seq;
+	struct iwl_tdls_config_sta_info_res sta_info[IWL_MVM_TDLS_STA_COUNT];
+} __packed; /* TDLS_CONFIG_RSP_API_S_VER_1 */
+
+#endif /* __iwl_fw_api_tdls_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/time-event.h b/drivers/net/wireless/intel/iwlwifi/fw/api/time-event.h
new file mode 100644
index 000000000000..3721a3ed358b
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/time-event.h
@@ -0,0 +1,386 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+
+#ifndef __iwl_fw_api_time_event_h__
+#define __iwl_fw_api_time_event_h__
+
+#include "fw/api/phy-ctxt.h"
+
+/* Time Event types, according to MAC type */
+enum iwl_time_event_type {
+	/* BSS Station Events */
+	TE_BSS_STA_AGGRESSIVE_ASSOC,
+	TE_BSS_STA_ASSOC,
+	TE_BSS_EAP_DHCP_PROT,
+	TE_BSS_QUIET_PERIOD,
+
+	/* P2P Device Events */
+	TE_P2P_DEVICE_DISCOVERABLE,
+	TE_P2P_DEVICE_LISTEN,
+	TE_P2P_DEVICE_ACTION_SCAN,
+	TE_P2P_DEVICE_FULL_SCAN,
+
+	/* P2P Client Events */
+	TE_P2P_CLIENT_AGGRESSIVE_ASSOC,
+	TE_P2P_CLIENT_ASSOC,
+	TE_P2P_CLIENT_QUIET_PERIOD,
+
+	/* P2P GO Events */
+	TE_P2P_GO_ASSOC_PROT,
+	TE_P2P_GO_REPETITIVET_NOA,
+	TE_P2P_GO_CT_WINDOW,
+
+	/* WiDi Sync Events */
+	TE_WIDI_TX_SYNC,
+
+	/* Channel Switch NoA */
+	TE_CHANNEL_SWITCH_PERIOD,
+
+	TE_MAX
+}; /* MAC_EVENT_TYPE_API_E_VER_1 */
+
+/* Time event - defines for command API v1 */
+
+/*
+ * @TE_V1_FRAG_NONE: fragmentation of the time event is NOT allowed.
+ * @TE_V1_FRAG_SINGLE: fragmentation of the time event is allowed, but only
+ *	the first fragment is scheduled.
+ * @TE_V1_FRAG_DUAL: fragmentation of the time event is allowed, but only
+ *	the first 2 fragments are scheduled.
+ * @TE_V1_FRAG_ENDLESS: fragmentation of the time event is allowed, and any
+ *	number of fragments are valid.
+ *
+ * Other than the constant defined above, specifying a fragmentation value 'x'
+ * means that the event can be fragmented but only the first 'x' will be
+ * scheduled.
+ */
+enum {
+	TE_V1_FRAG_NONE = 0,
+	TE_V1_FRAG_SINGLE = 1,
+	TE_V1_FRAG_DUAL = 2,
+	TE_V1_FRAG_ENDLESS = 0xffffffff
+};
+
+/* If a Time Event can be fragmented, this is the max number of fragments */
+#define TE_V1_FRAG_MAX_MSK	0x0fffffff
+/* Repeat the time event endlessly (until removed) */
+#define TE_V1_REPEAT_ENDLESS	0xffffffff
+/* If a Time Event has bounded repetitions, this is the maximal value */
+#define TE_V1_REPEAT_MAX_MSK_V1	0x0fffffff
+
+/* Time Event dependencies: none, on another TE, or in a specific time */
+enum {
+	TE_V1_INDEPENDENT		= 0,
+	TE_V1_DEP_OTHER			= BIT(0),
+	TE_V1_DEP_TSF			= BIT(1),
+	TE_V1_EVENT_SOCIOPATHIC		= BIT(2),
+}; /* MAC_EVENT_DEPENDENCY_POLICY_API_E_VER_2 */
+
+/*
+ * @TE_V1_NOTIF_NONE: no notifications
+ * @TE_V1_NOTIF_HOST_EVENT_START: request/receive notification on event start
+ * @TE_V1_NOTIF_HOST_EVENT_END:request/receive notification on event end
+ * @TE_V1_NOTIF_INTERNAL_EVENT_START: internal FW use
+ * @TE_V1_NOTIF_INTERNAL_EVENT_END: internal FW use.
+ * @TE_V1_NOTIF_HOST_FRAG_START: request/receive notification on frag start
+ * @TE_V1_NOTIF_HOST_FRAG_END:request/receive notification on frag end
+ * @TE_V1_NOTIF_INTERNAL_FRAG_START: internal FW use.
+ * @TE_V1_NOTIF_INTERNAL_FRAG_END: internal FW use.
+ *
+ * Supported Time event notifications configuration.
+ * A notification (both event and fragment) includes a status indicating weather
+ * the FW was able to schedule the event or not. For fragment start/end
+ * notification the status is always success. There is no start/end fragment
+ * notification for monolithic events.
+ */
+enum {
+	TE_V1_NOTIF_NONE = 0,
+	TE_V1_NOTIF_HOST_EVENT_START = BIT(0),
+	TE_V1_NOTIF_HOST_EVENT_END = BIT(1),
+	TE_V1_NOTIF_INTERNAL_EVENT_START = BIT(2),
+	TE_V1_NOTIF_INTERNAL_EVENT_END = BIT(3),
+	TE_V1_NOTIF_HOST_FRAG_START = BIT(4),
+	TE_V1_NOTIF_HOST_FRAG_END = BIT(5),
+	TE_V1_NOTIF_INTERNAL_FRAG_START = BIT(6),
+	TE_V1_NOTIF_INTERNAL_FRAG_END = BIT(7),
+}; /* MAC_EVENT_ACTION_API_E_VER_2 */
+
+/* Time event - defines for command API */
+
+/*
+ * @TE_V2_FRAG_NONE: fragmentation of the time event is NOT allowed.
+ * @TE_V2_FRAG_SINGLE: fragmentation of the time event is allowed, but only
+ *  the first fragment is scheduled.
+ * @TE_V2_FRAG_DUAL: fragmentation of the time event is allowed, but only
+ *  the first 2 fragments are scheduled.
+ * @TE_V2_FRAG_ENDLESS: fragmentation of the time event is allowed, and any
+ *  number of fragments are valid.
+ *
+ * Other than the constant defined above, specifying a fragmentation value 'x'
+ * means that the event can be fragmented but only the first 'x' will be
+ * scheduled.
+ */
+enum {
+	TE_V2_FRAG_NONE = 0,
+	TE_V2_FRAG_SINGLE = 1,
+	TE_V2_FRAG_DUAL = 2,
+	TE_V2_FRAG_MAX = 0xfe,
+	TE_V2_FRAG_ENDLESS = 0xff
+};
+
+/* Repeat the time event endlessly (until removed) */
+#define TE_V2_REPEAT_ENDLESS	0xff
+/* If a Time Event has bounded repetitions, this is the maximal value */
+#define TE_V2_REPEAT_MAX	0xfe
+
+#define TE_V2_PLACEMENT_POS	12
+#define TE_V2_ABSENCE_POS	15
+
+/**
+ * enum iwl_time_event_policy - Time event policy values
+ * A notification (both event and fragment) includes a status indicating weather
+ * the FW was able to schedule the event or not. For fragment start/end
+ * notification the status is always success. There is no start/end fragment
+ * notification for monolithic events.
+ *
+ * @TE_V2_DEFAULT_POLICY: independent, social, present, unoticable
+ * @TE_V2_NOTIF_HOST_EVENT_START: request/receive notification on event start
+ * @TE_V2_NOTIF_HOST_EVENT_END:request/receive notification on event end
+ * @TE_V2_NOTIF_INTERNAL_EVENT_START: internal FW use
+ * @TE_V2_NOTIF_INTERNAL_EVENT_END: internal FW use.
+ * @TE_V2_NOTIF_HOST_FRAG_START: request/receive notification on frag start
+ * @TE_V2_NOTIF_HOST_FRAG_END:request/receive notification on frag end
+ * @TE_V2_NOTIF_INTERNAL_FRAG_START: internal FW use.
+ * @TE_V2_NOTIF_INTERNAL_FRAG_END: internal FW use.
+ * @T2_V2_START_IMMEDIATELY: start time event immediately
+ * @TE_V2_DEP_OTHER: depends on another time event
+ * @TE_V2_DEP_TSF: depends on a specific time
+ * @TE_V2_EVENT_SOCIOPATHIC: can't co-exist with other events of tha same MAC
+ * @TE_V2_ABSENCE: are we present or absent during the Time Event.
+ */
+enum iwl_time_event_policy {
+	TE_V2_DEFAULT_POLICY = 0x0,
+
+	/* notifications (event start/stop, fragment start/stop) */
+	TE_V2_NOTIF_HOST_EVENT_START = BIT(0),
+	TE_V2_NOTIF_HOST_EVENT_END = BIT(1),
+	TE_V2_NOTIF_INTERNAL_EVENT_START = BIT(2),
+	TE_V2_NOTIF_INTERNAL_EVENT_END = BIT(3),
+
+	TE_V2_NOTIF_HOST_FRAG_START = BIT(4),
+	TE_V2_NOTIF_HOST_FRAG_END = BIT(5),
+	TE_V2_NOTIF_INTERNAL_FRAG_START = BIT(6),
+	TE_V2_NOTIF_INTERNAL_FRAG_END = BIT(7),
+	T2_V2_START_IMMEDIATELY = BIT(11),
+
+	/* placement characteristics */
+	TE_V2_DEP_OTHER = BIT(TE_V2_PLACEMENT_POS),
+	TE_V2_DEP_TSF = BIT(TE_V2_PLACEMENT_POS + 1),
+	TE_V2_EVENT_SOCIOPATHIC = BIT(TE_V2_PLACEMENT_POS + 2),
+
+	/* are we present or absent during the Time Event. */
+	TE_V2_ABSENCE = BIT(TE_V2_ABSENCE_POS),
+};
+
+/**
+ * struct iwl_time_event_cmd - configuring Time Events
+ * with struct MAC_TIME_EVENT_DATA_API_S_VER_2 (see also
+ * with version 1. determined by IWL_UCODE_TLV_FLAGS)
+ * ( TIME_EVENT_CMD = 0x29 )
+ * @id_and_color: ID and color of the relevant MAC,
+ *	&enum iwl_ctxt_id_and_color
+ * @action: action to perform, one of &enum iwl_ctxt_action
+ * @id: this field has two meanings, depending on the action:
+ *	If the action is ADD, then it means the type of event to add.
+ *	For all other actions it is the unique event ID assigned when the
+ *	event was added by the FW.
+ * @apply_time: When to start the Time Event (in GP2)
+ * @max_delay: maximum delay to event's start (apply time), in TU
+ * @depends_on: the unique ID of the event we depend on (if any)
+ * @interval: interval between repetitions, in TU
+ * @duration: duration of event in TU
+ * @repeat: how many repetitions to do, can be TE_REPEAT_ENDLESS
+ * @max_frags: maximal number of fragments the Time Event can be divided to
+ * @policy: defines whether uCode shall notify the host or other uCode modules
+ *	on event and/or fragment start and/or end
+ *	using one of TE_INDEPENDENT, TE_DEP_OTHER, TE_DEP_TSF
+ *	TE_EVENT_SOCIOPATHIC
+ *	using TE_ABSENCE and using TE_NOTIF_*,
+ *	&enum iwl_time_event_policy
+ */
+struct iwl_time_event_cmd {
+	/* COMMON_INDEX_HDR_API_S_VER_1 */
+	__le32 id_and_color;
+	__le32 action;
+	__le32 id;
+	/* MAC_TIME_EVENT_DATA_API_S_VER_2 */
+	__le32 apply_time;
+	__le32 max_delay;
+	__le32 depends_on;
+	__le32 interval;
+	__le32 duration;
+	u8 repeat;
+	u8 max_frags;
+	__le16 policy;
+} __packed; /* MAC_TIME_EVENT_CMD_API_S_VER_2 */
+
+/**
+ * struct iwl_time_event_resp - response structure to iwl_time_event_cmd
+ * @status: bit 0 indicates success, all others specify errors
+ * @id: the Time Event type
+ * @unique_id: the unique ID assigned (in ADD) or given (others) to the TE
+ * @id_and_color: ID and color of the relevant MAC,
+ *	&enum iwl_ctxt_id_and_color
+ */
+struct iwl_time_event_resp {
+	__le32 status;
+	__le32 id;
+	__le32 unique_id;
+	__le32 id_and_color;
+} __packed; /* MAC_TIME_EVENT_RSP_API_S_VER_1 */
+
+/**
+ * struct iwl_time_event_notif - notifications of time event start/stop
+ * ( TIME_EVENT_NOTIFICATION = 0x2a )
+ * @timestamp: action timestamp in GP2
+ * @session_id: session's unique id
+ * @unique_id: unique id of the Time Event itself
+ * @id_and_color: ID and color of the relevant MAC
+ * @action: &enum iwl_time_event_policy
+ * @status: true if scheduled, false otherwise (not executed)
+ */
+struct iwl_time_event_notif {
+	__le32 timestamp;
+	__le32 session_id;
+	__le32 unique_id;
+	__le32 id_and_color;
+	__le32 action;
+	__le32 status;
+} __packed; /* MAC_TIME_EVENT_NTFY_API_S_VER_1 */
+
+/*
+ * Aux ROC command
+ *
+ * Command requests the firmware to create a time event for a certain duration
+ * and remain on the given channel. This is done by using the Aux framework in
+ * the FW.
+ * The command was first used for Hot Spot issues - but can be used regardless
+ * to Hot Spot.
+ *
+ * ( HOT_SPOT_CMD 0x53 )
+ *
+ * @id_and_color: ID and color of the MAC
+ * @action: action to perform, one of FW_CTXT_ACTION_*
+ * @event_unique_id: If the action FW_CTXT_ACTION_REMOVE then the
+ *	event_unique_id should be the id of the time event assigned by ucode.
+ *	Otherwise ignore the event_unique_id.
+ * @sta_id_and_color: station id and color, resumed during "Remain On Channel"
+ *	activity.
+ * @channel_info: channel info
+ * @node_addr: Our MAC Address
+ * @reserved: reserved for alignment
+ * @apply_time: GP2 value to start (should always be the current GP2 value)
+ * @apply_time_max_delay: Maximum apply time delay value in TU. Defines max
+ *	time by which start of the event is allowed to be postponed.
+ * @duration: event duration in TU To calculate event duration:
+ *	timeEventDuration = min(duration, remainingQuota)
+ */
+struct iwl_hs20_roc_req {
+	/* COMMON_INDEX_HDR_API_S_VER_1 hdr */
+	__le32 id_and_color;
+	__le32 action;
+	__le32 event_unique_id;
+	__le32 sta_id_and_color;
+	struct iwl_fw_channel_info channel_info;
+	u8 node_addr[ETH_ALEN];
+	__le16 reserved;
+	__le32 apply_time;
+	__le32 apply_time_max_delay;
+	__le32 duration;
+} __packed; /* HOT_SPOT_CMD_API_S_VER_1 */
+
+/*
+ * values for AUX ROC result values
+ */
+enum iwl_mvm_hot_spot {
+	HOT_SPOT_RSP_STATUS_OK,
+	HOT_SPOT_RSP_STATUS_TOO_MANY_EVENTS,
+	HOT_SPOT_MAX_NUM_OF_SESSIONS,
+};
+
+/*
+ * Aux ROC command response
+ *
+ * In response to iwl_hs20_roc_req the FW sends this command to notify the
+ * driver the uid of the timevent.
+ *
+ * ( HOT_SPOT_CMD 0x53 )
+ *
+ * @event_unique_id: Unique ID of time event assigned by ucode
+ * @status: Return status 0 is success, all the rest used for specific errors
+ */
+struct iwl_hs20_roc_res {
+	__le32 event_unique_id;
+	__le32 status;
+} __packed; /* HOT_SPOT_RSP_API_S_VER_1 */
+
+#endif /* __iwl_fw_api_time_event_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-tof.h b/drivers/net/wireless/intel/iwlwifi/fw/api/tof.h
index 8658a983c463..7328a1606146 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-tof.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/tof.h
@@ -16,11 +16,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
- * USA
- *
  * The full GNU General Public License is included in this distribution
  * in the file called COPYING.
  *
@@ -60,8 +55,8 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  *****************************************************************************/
-#ifndef __fw_api_tof_h__
-#define __fw_api_tof_h__
+#ifndef __iwl_fw_api_tof_h__
+#define __iwl_fw_api_tof_h__
 
 /* ToF sub-group command IDs */
 enum iwl_mvm_tof_sub_grp_ids {
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-tx.h b/drivers/net/wireless/intel/iwlwifi/fw/api/tx.h
index 97d7eed32622..14ad9fb895f9 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api-tx.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/tx.h
@@ -17,11 +17,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
- * USA
- *
  * The full GNU General Public License is included in this distribution
  * in the file called COPYING.
  *
@@ -62,8 +57,8 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *****************************************************************************/
 
-#ifndef __fw_api_tx_h__
-#define __fw_api_tx_h__
+#ifndef __iwl_fw_api_tx_h__
+#define __iwl_fw_api_tx_h__
 
 /**
  * enum iwl_tx_flags - bitmasks for tx_flags in TX command
@@ -414,7 +409,8 @@ enum iwl_tx_status {
  * @AGG_TX_STATE_BT_PRIO:
  * @AGG_TX_STATE_FEW_BYTES:
  * @AGG_TX_STATE_ABORT:
- * @AGG_TX_STATE_LAST_SENT_TTL:
+ * @AGG_TX_STATE_TX_ON_AIR_DROP: TX_ON_AIR signal drop without underrun or
+ *	BT detection
  * @AGG_TX_STATE_LAST_SENT_TRY_CNT:
  * @AGG_TX_STATE_LAST_SENT_BT_KILL:
  * @AGG_TX_STATE_SCD_QUERY:
@@ -426,7 +422,7 @@ enum iwl_tx_status {
  *	occur if tx failed for this frame when it was a member of a previous
  *	aggregation block). If rate scaling is used, retry count indicates the
  *	rate table entry used for all frames in the new agg.
- *@ AGG_TX_STATE_SEQ_NUM_MSK: Command ID and sequence number of Tx command for
+ * @AGG_TX_STATE_SEQ_NUM_MSK: Command ID and sequence number of Tx command for
  *	this frame
  *
  * TODO: complete documentation
@@ -438,7 +434,7 @@ enum iwl_tx_agg_status {
 	AGG_TX_STATE_BT_PRIO = 0x002,
 	AGG_TX_STATE_FEW_BYTES = 0x004,
 	AGG_TX_STATE_ABORT = 0x008,
-	AGG_TX_STATE_LAST_SENT_TTL = 0x010,
+	AGG_TX_STATE_TX_ON_AIR_DROP = 0x010,
 	AGG_TX_STATE_LAST_SENT_TRY_CNT = 0x020,
 	AGG_TX_STATE_LAST_SENT_BT_KILL = 0x040,
 	AGG_TX_STATE_SCD_QUERY = 0x080,
@@ -450,10 +446,6 @@ enum iwl_tx_agg_status {
 	AGG_TX_STATE_TRY_CNT_MSK = 0xf << AGG_TX_STATE_TRY_CNT_POS,
 };
 
-#define AGG_TX_STATE_LAST_SENT_MSK  (AGG_TX_STATE_LAST_SENT_TTL| \
-				     AGG_TX_STATE_LAST_SENT_TRY_CNT| \
-				     AGG_TX_STATE_LAST_SENT_BT_KILL)
-
 /*
  * The mask below describes a status where we are absolutely sure that the MPDU
  * wasn't sent. For BA/Underrun we cannot be that sure. All we know that we've
@@ -771,7 +763,8 @@ struct iwl_mac_beacon_cmd_v6 {
 } __packed; /* BEACON_TEMPLATE_CMD_API_S_VER_6 */
 
 /**
- * struct iwl_mac_beacon_cmd_data - data of beacon template with offloaded CSA
+ * struct iwl_mac_beacon_cmd_v7 - beacon template command with offloaded CSA
+ * @tx: the tx commands associated with the beacon frame
  * @template_id: currently equal to the mac context id of the coresponding
  *  mac.
  * @tim_idx: the offset of the tim IE in the beacon
@@ -780,38 +773,47 @@ struct iwl_mac_beacon_cmd_v6 {
  * @csa_offset: offset to the CSA IE if present
  * @frame: the template of the beacon frame
  */
-struct iwl_mac_beacon_cmd_data {
+struct iwl_mac_beacon_cmd_v7 {
+	struct iwl_tx_cmd tx;
 	__le32 template_id;
 	__le32 tim_idx;
 	__le32 tim_size;
 	__le32 ecsa_offset;
 	__le32 csa_offset;
 	struct ieee80211_hdr frame[0];
-};
-
-/**
- * struct iwl_mac_beacon_cmd_v7 - beacon template command with offloaded CSA
- * @tx: the tx commands associated with the beacon frame
- * @data: see &iwl_mac_beacon_cmd_data
- */
-struct iwl_mac_beacon_cmd_v7 {
-	struct iwl_tx_cmd tx;
-	struct iwl_mac_beacon_cmd_data data;
 } __packed; /* BEACON_TEMPLATE_CMD_API_S_VER_7 */
 
+enum iwl_mac_beacon_flags {
+	IWL_MAC_BEACON_CCK	= BIT(8),
+	IWL_MAC_BEACON_ANT_A	= BIT(9),
+	IWL_MAC_BEACON_ANT_B	= BIT(10),
+	IWL_MAC_BEACON_ANT_C	= BIT(11),
+};
+
 /**
  * struct iwl_mac_beacon_cmd - beacon template command with offloaded CSA
- * @byte_cnt: byte count of the beacon frame
- * @flags: for future use
+ * @byte_cnt: byte count of the beacon frame.
+ * @flags: least significant byte for rate code. The most significant byte
+ *	is &enum iwl_mac_beacon_flags.
  * @reserved: reserved
- * @data: see &iwl_mac_beacon_cmd_data
+ * @template_id: currently equal to the mac context id of the coresponding mac.
+ * @tim_idx: the offset of the tim IE in the beacon
+ * @tim_size: the length of the tim IE
+ * @ecsa_offset: offset to the ECSA IE if present
+ * @csa_offset: offset to the CSA IE if present
+ * @frame: the template of the beacon frame
  */
 struct iwl_mac_beacon_cmd {
 	__le16 byte_cnt;
 	__le16 flags;
 	__le64 reserved;
-	struct iwl_mac_beacon_cmd_data data;
-} __packed; /* BEACON_TEMPLATE_CMD_API_S_VER_8 */
+	__le32 template_id;
+	__le32 tim_idx;
+	__le32 tim_size;
+	__le32 ecsa_offset;
+	__le32 csa_offset;
+	struct ieee80211_hdr frame[0];
+} __packed; /* BEACON_TEMPLATE_CMD_API_S_VER_9 */
 
 struct iwl_beacon_notif {
 	struct iwl_mvm_tx_resp beacon_notify_hdr;
@@ -914,4 +916,4 @@ struct iwl_scd_txq_cfg_rsp {
 	u8 scd_queue;
 } __packed; /* SCD_QUEUE_CFG_RSP_API_S_VER_1 */
 
-#endif /* __fw_api_tx_h__ */
+#endif /* __iwl_fw_api_tx_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/txq.h b/drivers/net/wireless/intel/iwlwifi/fw/api/txq.h
new file mode 100644
index 000000000000..87b4434224a1
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/txq.h
@@ -0,0 +1,163 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2007 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2005 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+#ifndef __iwl_fw_api_txq_h__
+#define __iwl_fw_api_txq_h__
+
+/*
+ * DQA queue numbers
+ *
+ * @IWL_MVM_DQA_CMD_QUEUE: a queue reserved for sending HCMDs to the FW
+ * @IWL_MVM_DQA_AUX_QUEUE: a queue reserved for aux frames
+ * @IWL_MVM_DQA_P2P_DEVICE_QUEUE: a queue reserved for P2P device frames
+ * @IWL_MVM_DQA_GCAST_QUEUE: a queue reserved for P2P GO/SoftAP GCAST frames
+ * @IWL_MVM_DQA_BSS_CLIENT_QUEUE: a queue reserved for BSS activity, to ensure
+ *	that we are never left without the possibility to connect to an AP.
+ * @IWL_MVM_DQA_MIN_MGMT_QUEUE: first TXQ in pool for MGMT and non-QOS frames.
+ *	Each MGMT queue is mapped to a single STA
+ *	MGMT frames are frames that return true on ieee80211_is_mgmt()
+ * @IWL_MVM_DQA_MAX_MGMT_QUEUE: last TXQ in pool for MGMT frames
+ * @IWL_MVM_DQA_AP_PROBE_RESP_QUEUE: a queue reserved for P2P GO/SoftAP probe
+ *	responses
+ * @IWL_MVM_DQA_MIN_DATA_QUEUE: first TXQ in pool for DATA frames.
+ *	DATA frames are intended for !ieee80211_is_mgmt() frames, but if
+ *	the MGMT TXQ pool is exhausted, mgmt frames can be sent on DATA queues
+ *	as well
+ * @IWL_MVM_DQA_MAX_DATA_QUEUE: last TXQ in pool for DATA frames
+ */
+enum iwl_mvm_dqa_txq {
+	IWL_MVM_DQA_CMD_QUEUE = 0,
+	IWL_MVM_DQA_AUX_QUEUE = 1,
+	IWL_MVM_DQA_P2P_DEVICE_QUEUE = 2,
+	IWL_MVM_DQA_GCAST_QUEUE = 3,
+	IWL_MVM_DQA_BSS_CLIENT_QUEUE = 4,
+	IWL_MVM_DQA_MIN_MGMT_QUEUE = 5,
+	IWL_MVM_DQA_MAX_MGMT_QUEUE = 8,
+	IWL_MVM_DQA_AP_PROBE_RESP_QUEUE = 9,
+	IWL_MVM_DQA_MIN_DATA_QUEUE = 10,
+	IWL_MVM_DQA_MAX_DATA_QUEUE = 31,
+};
+
+enum iwl_mvm_tx_fifo {
+	IWL_MVM_TX_FIFO_BK = 0,
+	IWL_MVM_TX_FIFO_BE,
+	IWL_MVM_TX_FIFO_VI,
+	IWL_MVM_TX_FIFO_VO,
+	IWL_MVM_TX_FIFO_MCAST = 5,
+	IWL_MVM_TX_FIFO_CMD = 7,
+};
+
+enum iwl_gen2_tx_fifo {
+	IWL_GEN2_TX_FIFO_CMD = 0,
+	IWL_GEN2_EDCA_TX_FIFO_BK,
+	IWL_GEN2_EDCA_TX_FIFO_BE,
+	IWL_GEN2_EDCA_TX_FIFO_VI,
+	IWL_GEN2_EDCA_TX_FIFO_VO,
+	IWL_GEN2_TRIG_TX_FIFO_BK,
+	IWL_GEN2_TRIG_TX_FIFO_BE,
+	IWL_GEN2_TRIG_TX_FIFO_VI,
+	IWL_GEN2_TRIG_TX_FIFO_VO,
+};
+
+/**
+ * enum iwl_tx_queue_cfg_actions - TXQ config options
+ * @TX_QUEUE_CFG_ENABLE_QUEUE: enable a queue
+ * @TX_QUEUE_CFG_TFD_SHORT_FORMAT: use short TFD format
+ */
+enum iwl_tx_queue_cfg_actions {
+	TX_QUEUE_CFG_ENABLE_QUEUE		= BIT(0),
+	TX_QUEUE_CFG_TFD_SHORT_FORMAT		= BIT(1),
+};
+
+/**
+ * struct iwl_tx_queue_cfg_cmd - txq hw scheduler config command
+ * @sta_id: station id
+ * @tid: tid of the queue
+ * @flags: see &enum iwl_tx_queue_cfg_actions
+ * @cb_size: size of TFD cyclic buffer. Value is exponent - 3.
+ *	Minimum value 0 (8 TFDs), maximum value 5 (256 TFDs)
+ * @byte_cnt_addr: address of byte count table
+ * @tfdq_addr: address of TFD circular buffer
+ */
+struct iwl_tx_queue_cfg_cmd {
+	u8 sta_id;
+	u8 tid;
+	__le16 flags;
+	__le32 cb_size;
+	__le64 byte_cnt_addr;
+	__le64 tfdq_addr;
+} __packed; /* TX_QUEUE_CFG_CMD_API_S_VER_2 */
+
+/**
+ * struct iwl_tx_queue_cfg_rsp - response to txq hw scheduler config
+ * @queue_number: queue number assigned to this RA -TID
+ * @flags: set on failure
+ * @write_pointer: initial value for write pointer
+ * @reserved: reserved
+ */
+struct iwl_tx_queue_cfg_rsp {
+	__le16 queue_number;
+	__le16 flags;
+	__le16 write_pointer;
+	__le16 reserved;
+} __packed; /* TX_QUEUE_CFG_RSP_API_S_VER_2 */
+
+#endif /* __iwl_fw_api_txq_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/common_rx.c b/drivers/net/wireless/intel/iwlwifi/fw/common_rx.c
new file mode 100644
index 000000000000..6f75985eea66
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/common_rx.c
@@ -0,0 +1,88 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+#include "iwl-drv.h"
+#include "runtime.h"
+#include "fw/api/commands.h"
+#include "fw/api/alive.h"
+
+static void iwl_fwrt_fseq_ver_mismatch(struct iwl_fw_runtime *fwrt,
+				       struct iwl_rx_cmd_buffer *rxb)
+{
+	struct iwl_rx_packet *pkt = rxb_addr(rxb);
+	struct iwl_fseq_ver_mismatch_ntf *fseq = (void *)pkt->data;
+
+	IWL_ERR(fwrt, "FSEQ version mismatch (aux: %d, wifi: %d)\n",
+		__le32_to_cpu(fseq->aux_read_fseq_ver),
+		__le32_to_cpu(fseq->wifi_fseq_ver));
+}
+
+void iwl_fwrt_handle_notification(struct iwl_fw_runtime *fwrt,
+				  struct iwl_rx_cmd_buffer *rxb)
+{
+	struct iwl_rx_packet *pkt = rxb_addr(rxb);
+	u32 cmd = WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd);
+
+	switch (cmd) {
+	case WIDE_ID(SYSTEM_GROUP, FSEQ_VER_MISMATCH_NTF):
+		iwl_fwrt_fseq_ver_mismatch(fwrt, rxb);
+		break;
+	default:
+		break;
+	}
+}
+IWL_EXPORT_SYMBOL(iwl_fwrt_handle_notification);
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw-dbg.c b/drivers/net/wireless/intel/iwlwifi/fw/dbg.c
index 1602b360353c..6afc7a799892 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw-dbg.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/dbg.c
@@ -63,22 +63,37 @@
  *
  *****************************************************************************/
 #include <linux/devcoredump.h>
-
-#include "fw-dbg.h"
+#include "iwl-drv.h"
+#include "runtime.h"
+#include "dbg.h"
 #include "iwl-io.h"
-#include "mvm.h"
 #include "iwl-prph.h"
 #include "iwl-csr.h"
 
+/**
+ * struct iwl_fw_dump_ptrs - set of pointers needed for the fw-error-dump
+ *
+ * @fwrt_ptr: pointer to the buffer coming from fwrt
+ * @trans_ptr: pointer to struct %iwl_trans_dump_data which contains the
+ *	transport's data.
+ * @trans_len: length of the valid data in trans_ptr
+ * @fwrt_len: length of the valid data in fwrt_ptr
+ */
+struct iwl_fw_dump_ptrs {
+	struct iwl_trans_dump_data *trans_ptr;
+	void *fwrt_ptr;
+	u32 fwrt_len;
+};
+
 #define RADIO_REG_MAX_READ 0x2ad
-static void iwl_mvm_read_radio_reg(struct iwl_mvm *mvm,
-				   struct iwl_fw_error_dump_data **dump_data)
+static void iwl_read_radio_regs(struct iwl_fw_runtime *fwrt,
+				struct iwl_fw_error_dump_data **dump_data)
 {
 	u8 *pos = (void *)(*dump_data)->data;
 	unsigned long flags;
 	int i;
 
-	if (!iwl_trans_grab_nic_access(mvm->trans, &flags))
+	if (!iwl_trans_grab_nic_access(fwrt->trans, &flags))
 		return;
 
 	(*dump_data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_RADIO_REG);
@@ -88,20 +103,20 @@ static void iwl_mvm_read_radio_reg(struct iwl_mvm *mvm,
 		u32 rd_cmd = RADIO_RSP_RD_CMD;
 
 		rd_cmd |= i << RADIO_RSP_ADDR_POS;
-		iwl_write_prph_no_grab(mvm->trans, RSP_RADIO_CMD, rd_cmd);
-		*pos =  (u8)iwl_read_prph_no_grab(mvm->trans, RSP_RADIO_RDDAT);
+		iwl_write_prph_no_grab(fwrt->trans, RSP_RADIO_CMD, rd_cmd);
+		*pos = (u8)iwl_read_prph_no_grab(fwrt->trans, RSP_RADIO_RDDAT);
 
 		pos++;
 	}
 
 	*dump_data = iwl_fw_error_next_data(*dump_data);
 
-	iwl_trans_release_nic_access(mvm->trans, &flags);
+	iwl_trans_release_nic_access(fwrt->trans, &flags);
 }
 
-static void iwl_mvm_dump_rxf(struct iwl_mvm *mvm,
-			     struct iwl_fw_error_dump_data **dump_data,
-			     int size, u32 offset, int fifo_num)
+static void iwl_fwrt_dump_rxf(struct iwl_fw_runtime *fwrt,
+			      struct iwl_fw_error_dump_data **dump_data,
+			      int size, u32 offset, int fifo_num)
 {
 	struct iwl_fw_error_dump_fifo *fifo_hdr;
 	u32 *fifo_data;
@@ -122,41 +137,41 @@ static void iwl_mvm_dump_rxf(struct iwl_mvm *mvm,
 
 	fifo_hdr->fifo_num = cpu_to_le32(fifo_num);
 	fifo_hdr->available_bytes =
-		cpu_to_le32(iwl_trans_read_prph(mvm->trans,
+		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 						RXF_RD_D_SPACE + offset));
 	fifo_hdr->wr_ptr =
-		cpu_to_le32(iwl_trans_read_prph(mvm->trans,
+		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 						RXF_RD_WR_PTR + offset));
 	fifo_hdr->rd_ptr =
-		cpu_to_le32(iwl_trans_read_prph(mvm->trans,
+		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 						RXF_RD_RD_PTR + offset));
 	fifo_hdr->fence_ptr =
-		cpu_to_le32(iwl_trans_read_prph(mvm->trans,
+		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 						RXF_RD_FENCE_PTR + offset));
 	fifo_hdr->fence_mode =
-		cpu_to_le32(iwl_trans_read_prph(mvm->trans,
+		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 						RXF_SET_FENCE_MODE + offset));
 
 	/* Lock fence */
-	iwl_trans_write_prph(mvm->trans, RXF_SET_FENCE_MODE + offset, 0x1);
+	iwl_trans_write_prph(fwrt->trans, RXF_SET_FENCE_MODE + offset, 0x1);
 	/* Set fence pointer to the same place like WR pointer */
-	iwl_trans_write_prph(mvm->trans, RXF_LD_WR2FENCE + offset, 0x1);
+	iwl_trans_write_prph(fwrt->trans, RXF_LD_WR2FENCE + offset, 0x1);
 	/* Set fence offset */
-	iwl_trans_write_prph(mvm->trans,
+	iwl_trans_write_prph(fwrt->trans,
 			     RXF_LD_FENCE_OFFSET_ADDR + offset, 0x0);
 
 	/* Read FIFO */
 	fifo_len /= sizeof(u32); /* Size in DWORDS */
 	for (i = 0; i < fifo_len; i++)
-		fifo_data[i] = iwl_trans_read_prph(mvm->trans,
+		fifo_data[i] = iwl_trans_read_prph(fwrt->trans,
 						 RXF_FIFO_RD_FENCE_INC +
 						 offset);
 	*dump_data = iwl_fw_error_next_data(*dump_data);
 }
 
-static void iwl_mvm_dump_txf(struct iwl_mvm *mvm,
-			     struct iwl_fw_error_dump_data **dump_data,
-			     int size, u32 offset, int fifo_num)
+static void iwl_fwrt_dump_txf(struct iwl_fw_runtime *fwrt,
+			      struct iwl_fw_error_dump_data **dump_data,
+			      int size, u32 offset, int fifo_num)
 {
 	struct iwl_fw_error_dump_fifo *fifo_hdr;
 	u32 *fifo_data;
@@ -177,91 +192,91 @@ static void iwl_mvm_dump_txf(struct iwl_mvm *mvm,
 
 	fifo_hdr->fifo_num = cpu_to_le32(fifo_num);
 	fifo_hdr->available_bytes =
-		cpu_to_le32(iwl_trans_read_prph(mvm->trans,
+		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 						TXF_FIFO_ITEM_CNT + offset));
 	fifo_hdr->wr_ptr =
-		cpu_to_le32(iwl_trans_read_prph(mvm->trans,
+		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 						TXF_WR_PTR + offset));
 	fifo_hdr->rd_ptr =
-		cpu_to_le32(iwl_trans_read_prph(mvm->trans,
+		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 						TXF_RD_PTR + offset));
 	fifo_hdr->fence_ptr =
-		cpu_to_le32(iwl_trans_read_prph(mvm->trans,
+		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 						TXF_FENCE_PTR + offset));
 	fifo_hdr->fence_mode =
-		cpu_to_le32(iwl_trans_read_prph(mvm->trans,
+		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 						TXF_LOCK_FENCE + offset));
 
 	/* Set the TXF_READ_MODIFY_ADDR to TXF_WR_PTR */
-	iwl_trans_write_prph(mvm->trans, TXF_READ_MODIFY_ADDR + offset,
+	iwl_trans_write_prph(fwrt->trans, TXF_READ_MODIFY_ADDR + offset,
 			     TXF_WR_PTR + offset);
 
 	/* Dummy-read to advance the read pointer to the head */
-	iwl_trans_read_prph(mvm->trans, TXF_READ_MODIFY_DATA + offset);
+	iwl_trans_read_prph(fwrt->trans, TXF_READ_MODIFY_DATA + offset);
 
 	/* Read FIFO */
 	fifo_len /= sizeof(u32); /* Size in DWORDS */
 	for (i = 0; i < fifo_len; i++)
-		fifo_data[i] = iwl_trans_read_prph(mvm->trans,
+		fifo_data[i] = iwl_trans_read_prph(fwrt->trans,
 						  TXF_READ_MODIFY_DATA +
 						  offset);
 	*dump_data = iwl_fw_error_next_data(*dump_data);
 }
 
-static void iwl_mvm_dump_fifos(struct iwl_mvm *mvm,
-			       struct iwl_fw_error_dump_data **dump_data)
+static void iwl_fw_dump_fifos(struct iwl_fw_runtime *fwrt,
+			      struct iwl_fw_error_dump_data **dump_data)
 {
 	struct iwl_fw_error_dump_fifo *fifo_hdr;
-	struct iwl_mvm_shared_mem_cfg *cfg = &mvm->smem_cfg;
+	struct iwl_fwrt_shared_mem_cfg *cfg = &fwrt->smem_cfg;
 	u32 *fifo_data;
 	u32 fifo_len;
 	unsigned long flags;
 	int i, j;
 
-	if (!iwl_trans_grab_nic_access(mvm->trans, &flags))
+	if (!iwl_trans_grab_nic_access(fwrt->trans, &flags))
 		return;
 
 	/* Pull RXF1 */
-	iwl_mvm_dump_rxf(mvm, dump_data, cfg->lmac[0].rxfifo1_size, 0, 0);
+	iwl_fwrt_dump_rxf(fwrt, dump_data, cfg->lmac[0].rxfifo1_size, 0, 0);
 	/* Pull RXF2 */
-	iwl_mvm_dump_rxf(mvm, dump_data, cfg->rxfifo2_size,
-			 RXF_DIFF_FROM_PREV, 1);
+	iwl_fwrt_dump_rxf(fwrt, dump_data, cfg->rxfifo2_size,
+			  RXF_DIFF_FROM_PREV, 1);
 	/* Pull LMAC2 RXF1 */
-	if (mvm->smem_cfg.num_lmacs > 1)
-		iwl_mvm_dump_rxf(mvm, dump_data, cfg->lmac[1].rxfifo1_size,
-				 LMAC2_PRPH_OFFSET, 2);
+	if (fwrt->smem_cfg.num_lmacs > 1)
+		iwl_fwrt_dump_rxf(fwrt, dump_data, cfg->lmac[1].rxfifo1_size,
+				  LMAC2_PRPH_OFFSET, 2);
 
 	/* Pull TXF data from LMAC1 */
-	for (i = 0; i < mvm->smem_cfg.num_txfifo_entries; i++) {
+	for (i = 0; i < fwrt->smem_cfg.num_txfifo_entries; i++) {
 		/* Mark the number of TXF we're pulling now */
-		iwl_trans_write_prph(mvm->trans, TXF_LARC_NUM, i);
-		iwl_mvm_dump_txf(mvm, dump_data, cfg->lmac[0].txfifo_size[i],
-				 0, i);
+		iwl_trans_write_prph(fwrt->trans, TXF_LARC_NUM, i);
+		iwl_fwrt_dump_txf(fwrt, dump_data, cfg->lmac[0].txfifo_size[i],
+				  0, i);
 	}
 
 	/* Pull TXF data from LMAC2 */
-	if (mvm->smem_cfg.num_lmacs > 1) {
-		for (i = 0; i < mvm->smem_cfg.num_txfifo_entries; i++) {
+	if (fwrt->smem_cfg.num_lmacs > 1) {
+		for (i = 0; i < fwrt->smem_cfg.num_txfifo_entries; i++) {
 			/* Mark the number of TXF we're pulling now */
-			iwl_trans_write_prph(mvm->trans,
+			iwl_trans_write_prph(fwrt->trans,
 					     TXF_LARC_NUM + LMAC2_PRPH_OFFSET,
 					     i);
-			iwl_mvm_dump_txf(mvm, dump_data,
-					 cfg->lmac[1].txfifo_size[i],
-					 LMAC2_PRPH_OFFSET,
-					 i + cfg->num_txfifo_entries);
+			iwl_fwrt_dump_txf(fwrt, dump_data,
+					  cfg->lmac[1].txfifo_size[i],
+					  LMAC2_PRPH_OFFSET,
+					  i + cfg->num_txfifo_entries);
 		}
 	}
 
-	if (fw_has_capa(&mvm->fw->ucode_capa,
+	if (fw_has_capa(&fwrt->fw->ucode_capa,
 			IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG)) {
 		/* Pull UMAC internal TXF data from all TXFs */
 		for (i = 0;
-		     i < ARRAY_SIZE(mvm->smem_cfg.internal_txfifo_size);
+		     i < ARRAY_SIZE(fwrt->smem_cfg.internal_txfifo_size);
 		     i++) {
 			fifo_hdr = (void *)(*dump_data)->data;
 			fifo_data = (void *)fifo_hdr->data;
-			fifo_len = mvm->smem_cfg.internal_txfifo_size[i];
+			fifo_len = fwrt->smem_cfg.internal_txfifo_size[i];
 
 			/* No need to try to read the data if the length is 0 */
 			if (fifo_len == 0)
@@ -276,52 +291,45 @@ static void iwl_mvm_dump_fifos(struct iwl_mvm *mvm,
 			fifo_hdr->fifo_num = cpu_to_le32(i);
 
 			/* Mark the number of TXF we're pulling now */
-			iwl_trans_write_prph(mvm->trans, TXF_CPU2_NUM, i +
-				mvm->smem_cfg.num_txfifo_entries);
+			iwl_trans_write_prph(fwrt->trans, TXF_CPU2_NUM, i +
+				fwrt->smem_cfg.num_txfifo_entries);
 
 			fifo_hdr->available_bytes =
-				cpu_to_le32(iwl_trans_read_prph(mvm->trans,
+				cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 								TXF_CPU2_FIFO_ITEM_CNT));
 			fifo_hdr->wr_ptr =
-				cpu_to_le32(iwl_trans_read_prph(mvm->trans,
+				cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 								TXF_CPU2_WR_PTR));
 			fifo_hdr->rd_ptr =
-				cpu_to_le32(iwl_trans_read_prph(mvm->trans,
+				cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 								TXF_CPU2_RD_PTR));
 			fifo_hdr->fence_ptr =
-				cpu_to_le32(iwl_trans_read_prph(mvm->trans,
+				cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 								TXF_CPU2_FENCE_PTR));
 			fifo_hdr->fence_mode =
-				cpu_to_le32(iwl_trans_read_prph(mvm->trans,
+				cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 								TXF_CPU2_LOCK_FENCE));
 
 			/* Set TXF_CPU2_READ_MODIFY_ADDR to TXF_CPU2_WR_PTR */
-			iwl_trans_write_prph(mvm->trans,
+			iwl_trans_write_prph(fwrt->trans,
 					     TXF_CPU2_READ_MODIFY_ADDR,
 					     TXF_CPU2_WR_PTR);
 
 			/* Dummy-read to advance the read pointer to head */
-			iwl_trans_read_prph(mvm->trans,
+			iwl_trans_read_prph(fwrt->trans,
 					    TXF_CPU2_READ_MODIFY_DATA);
 
 			/* Read FIFO */
 			fifo_len /= sizeof(u32); /* Size in DWORDS */
 			for (j = 0; j < fifo_len; j++)
 				fifo_data[j] =
-					iwl_trans_read_prph(mvm->trans,
+					iwl_trans_read_prph(fwrt->trans,
 							    TXF_CPU2_READ_MODIFY_DATA);
 			*dump_data = iwl_fw_error_next_data(*dump_data);
 		}
 	}
 
-	iwl_trans_release_nic_access(mvm->trans, &flags);
-}
-
-void iwl_mvm_free_fw_dump_desc(struct iwl_mvm *mvm)
-{
-	if (mvm->fw_dump_desc != &iwl_mvm_dump_desc_assert)
-		kfree(mvm->fw_dump_desc);
-	mvm->fw_dump_desc = NULL;
+	iwl_trans_release_nic_access(fwrt->trans, &flags);
 }
 
 #define IWL8260_ICCM_OFFSET		0x44000 /* Only for B-step */
@@ -531,37 +539,34 @@ static struct scatterlist *alloc_sgtable(int size)
 	return table;
 }
 
-void iwl_mvm_fw_error_dump(struct iwl_mvm *mvm)
+void iwl_fw_error_dump(struct iwl_fw_runtime *fwrt)
 {
 	struct iwl_fw_error_dump_file *dump_file;
 	struct iwl_fw_error_dump_data *dump_data;
 	struct iwl_fw_error_dump_info *dump_info;
 	struct iwl_fw_error_dump_mem *dump_mem;
+	struct iwl_fw_error_dump_smem_cfg *dump_smem_cfg;
 	struct iwl_fw_error_dump_trigger_desc *dump_trig;
-	struct iwl_mvm_dump_ptrs *fw_error_dump;
+	struct iwl_fw_dump_ptrs *fw_error_dump;
 	struct scatterlist *sg_dump_data;
 	u32 sram_len, sram_ofs;
-	const struct iwl_fw_dbg_mem_seg_tlv *fw_dbg_mem = mvm->fw->dbg_mem_tlv;
+	const struct iwl_fw_dbg_mem_seg_tlv *fw_dbg_mem = fwrt->fw->dbg_mem_tlv;
+	struct iwl_fwrt_shared_mem_cfg *mem_cfg = &fwrt->smem_cfg;
 	u32 file_len, fifo_data_len = 0, prph_len = 0, radio_len = 0;
-	u32 smem_len = mvm->fw->n_dbg_mem_tlv ? 0 : mvm->cfg->smem_len;
-	u32 sram2_len = mvm->fw->n_dbg_mem_tlv ? 0 : mvm->cfg->dccm2_len;
+	u32 smem_len = fwrt->fw->n_dbg_mem_tlv ? 0 : fwrt->trans->cfg->smem_len;
+	u32 sram2_len = fwrt->fw->n_dbg_mem_tlv ?
+				0 : fwrt->trans->cfg->dccm2_len;
 	bool monitor_dump_only = false;
 	int i;
 
-	if (!IWL_MVM_COLLECT_FW_ERR_DUMP &&
-	    !mvm->trans->dbg_dest_tlv)
-		return;
-
-	lockdep_assert_held(&mvm->mutex);
-
 	/* there's no point in fw dump if the bus is dead */
-	if (test_bit(STATUS_TRANS_DEAD, &mvm->trans->status)) {
-		IWL_ERR(mvm, "Skip fw error dump since bus is dead\n");
+	if (test_bit(STATUS_TRANS_DEAD, &fwrt->trans->status)) {
+		IWL_ERR(fwrt, "Skip fw error dump since bus is dead\n");
 		goto out;
 	}
 
-	if (mvm->fw_dump_trig &&
-	    mvm->fw_dump_trig->mode & IWL_FW_DBG_TRIGGER_MONITOR_ONLY)
+	if (fwrt->dump.trig &&
+	    fwrt->dump.trig->mode & IWL_FW_DBG_TRIGGER_MONITOR_ONLY)
 		monitor_dump_only = true;
 
 	fw_error_dump = kzalloc(sizeof(*fw_error_dump), GFP_KERNEL);
@@ -569,21 +574,19 @@ void iwl_mvm_fw_error_dump(struct iwl_mvm *mvm)
 		goto out;
 
 	/* SRAM - include stack CCM if driver knows the values for it */
-	if (!mvm->cfg->dccm_offset || !mvm->cfg->dccm_len) {
+	if (!fwrt->trans->cfg->dccm_offset || !fwrt->trans->cfg->dccm_len) {
 		const struct fw_img *img;
 
-		img = &mvm->fw->img[mvm->cur_ucode];
+		img = &fwrt->fw->img[fwrt->cur_fw_img];
 		sram_ofs = img->sec[IWL_UCODE_SECTION_DATA].offset;
 		sram_len = img->sec[IWL_UCODE_SECTION_DATA].len;
 	} else {
-		sram_ofs = mvm->cfg->dccm_offset;
-		sram_len = mvm->cfg->dccm_len;
+		sram_ofs = fwrt->trans->cfg->dccm_offset;
+		sram_len = fwrt->trans->cfg->dccm_len;
 	}
 
 	/* reading RXF/TXF sizes */
-	if (test_bit(STATUS_FW_ERROR, &mvm->trans->status)) {
-		struct iwl_mvm_shared_mem_cfg *mem_cfg = &mvm->smem_cfg;
-
+	if (test_bit(STATUS_FW_ERROR, &fwrt->trans->status)) {
 		fifo_data_len = 0;
 
 		/* Count RXF2 size */
@@ -621,7 +624,7 @@ void iwl_mvm_fw_error_dump(struct iwl_mvm *mvm)
 			}
 		}
 
-		if (fw_has_capa(&mvm->fw->ucode_capa,
+		if (fw_has_capa(&fwrt->fw->ucode_capa,
 				IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG)) {
 			for (i = 0;
 			     i < ARRAY_SIZE(mem_cfg->internal_txfifo_size);
@@ -638,7 +641,7 @@ void iwl_mvm_fw_error_dump(struct iwl_mvm *mvm)
 		}
 
 		/* Make room for PRPH registers */
-		if (!mvm->trans->cfg->gen2) {
+		if (!fwrt->trans->cfg->gen2) {
 			for (i = 0; i < ARRAY_SIZE(iwl_prph_dump_addr_comm);
 			     i++) {
 				/* The range includes both boundaries */
@@ -652,7 +655,8 @@ void iwl_mvm_fw_error_dump(struct iwl_mvm *mvm)
 			}
 		}
 
-		if (!mvm->trans->cfg->gen2 && mvm->cfg->mq_rx_supported) {
+		if (!fwrt->trans->cfg->gen2 &&
+		    fwrt->trans->cfg->mq_rx_supported) {
 			for (i = 0; i <
 				ARRAY_SIZE(iwl_prph_dump_addr_9000); i++) {
 				/* The range includes both boundaries */
@@ -666,12 +670,13 @@ void iwl_mvm_fw_error_dump(struct iwl_mvm *mvm)
 			}
 		}
 
-		if (mvm->cfg->device_family == IWL_DEVICE_FAMILY_7000)
+		if (fwrt->trans->cfg->device_family == IWL_DEVICE_FAMILY_7000)
 			radio_len = sizeof(*dump_data) + RADIO_REG_MAX_READ;
 	}
 
 	file_len = sizeof(*dump_file) +
-		   sizeof(*dump_data) * 2 +
+		   sizeof(*dump_data) * 3 +
+		   sizeof(*dump_smem_cfg) +
 		   fifo_data_len +
 		   prph_len +
 		   radio_len +
@@ -686,31 +691,31 @@ void iwl_mvm_fw_error_dump(struct iwl_mvm *mvm)
 		file_len += sizeof(*dump_data) + sizeof(*dump_mem) + sram2_len;
 
 	/* Make room for MEM segments */
-	for (i = 0; i < mvm->fw->n_dbg_mem_tlv; i++) {
+	for (i = 0; i < fwrt->fw->n_dbg_mem_tlv; i++) {
 		file_len += sizeof(*dump_data) + sizeof(*dump_mem) +
 			    le32_to_cpu(fw_dbg_mem[i].len);
 	}
 
 	/* Make room for fw's virtual image pages, if it exists */
-	if (!mvm->trans->cfg->gen2 &&
-	    mvm->fw->img[mvm->cur_ucode].paging_mem_size &&
-	    mvm->fw_paging_db[0].fw_paging_block)
-		file_len += mvm->num_of_paging_blk *
+	if (!fwrt->trans->cfg->gen2 &&
+	    fwrt->fw->img[fwrt->cur_fw_img].paging_mem_size &&
+	    fwrt->fw_paging_db[0].fw_paging_block)
+		file_len += fwrt->num_of_paging_blk *
 			(sizeof(*dump_data) +
 			 sizeof(struct iwl_fw_error_dump_paging) +
 			 PAGING_BLOCK_SIZE);
 
 	/* If we only want a monitor dump, reset the file length */
 	if (monitor_dump_only) {
-		file_len = sizeof(*dump_file) + sizeof(*dump_data) +
-			   sizeof(*dump_info);
+		file_len = sizeof(*dump_file) + sizeof(*dump_data) * 2 +
+			   sizeof(*dump_info) + sizeof(*dump_smem_cfg);
 	}
 
-	if (mvm->fw_dump_desc)
+	if (fwrt->dump.desc)
 		file_len += sizeof(*dump_data) + sizeof(*dump_trig) +
-			    mvm->fw_dump_desc->len;
+			    fwrt->dump.desc->len;
 
-	if (!mvm->fw->n_dbg_mem_tlv)
+	if (!fwrt->fw->n_dbg_mem_tlv)
 		file_len += sram_len + sizeof(*dump_mem);
 
 	dump_file = vzalloc(file_len);
@@ -719,7 +724,7 @@ void iwl_mvm_fw_error_dump(struct iwl_mvm *mvm)
 		goto out;
 	}
 
-	fw_error_dump->op_mode_ptr = dump_file;
+	fw_error_dump->fwrt_ptr = dump_file;
 
 	dump_file->barker = cpu_to_le32(IWL_FW_ERROR_DUMP_BARKER);
 	dump_data = (void *)dump_file->data;
@@ -728,32 +733,59 @@ void iwl_mvm_fw_error_dump(struct iwl_mvm *mvm)
 	dump_data->len = cpu_to_le32(sizeof(*dump_info));
 	dump_info = (void *)dump_data->data;
 	dump_info->device_family =
-		mvm->cfg->device_family == IWL_DEVICE_FAMILY_7000 ?
+		fwrt->trans->cfg->device_family == IWL_DEVICE_FAMILY_7000 ?
 			cpu_to_le32(IWL_FW_ERROR_DUMP_FAMILY_7) :
 			cpu_to_le32(IWL_FW_ERROR_DUMP_FAMILY_8);
-	dump_info->hw_step = cpu_to_le32(CSR_HW_REV_STEP(mvm->trans->hw_rev));
-	memcpy(dump_info->fw_human_readable, mvm->fw->human_readable,
+	dump_info->hw_step = cpu_to_le32(CSR_HW_REV_STEP(fwrt->trans->hw_rev));
+	memcpy(dump_info->fw_human_readable, fwrt->fw->human_readable,
 	       sizeof(dump_info->fw_human_readable));
-	strncpy(dump_info->dev_human_readable, mvm->cfg->name,
+	strncpy(dump_info->dev_human_readable, fwrt->trans->cfg->name,
 		sizeof(dump_info->dev_human_readable));
-	strncpy(dump_info->bus_human_readable, mvm->dev->bus->name,
+	strncpy(dump_info->bus_human_readable, fwrt->dev->bus->name,
 		sizeof(dump_info->bus_human_readable));
 
 	dump_data = iwl_fw_error_next_data(dump_data);
+
+	/* Dump shared memory configuration */
+	dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_MEM_CFG);
+	dump_data->len = cpu_to_le32(sizeof(*dump_smem_cfg));
+	dump_smem_cfg = (void *)dump_data->data;
+	dump_smem_cfg->num_lmacs = cpu_to_le32(mem_cfg->num_lmacs);
+	dump_smem_cfg->num_txfifo_entries =
+		cpu_to_le32(mem_cfg->num_txfifo_entries);
+	for (i = 0; i < MAX_NUM_LMAC; i++) {
+		int j;
+
+		for (j = 0; j < TX_FIFO_MAX_NUM; j++)
+			dump_smem_cfg->lmac[i].txfifo_size[j] =
+				cpu_to_le32(mem_cfg->lmac[i].txfifo_size[j]);
+		dump_smem_cfg->lmac[i].rxfifo1_size =
+			cpu_to_le32(mem_cfg->lmac[i].rxfifo1_size);
+	}
+	dump_smem_cfg->rxfifo2_size = cpu_to_le32(mem_cfg->rxfifo2_size);
+	dump_smem_cfg->internal_txfifo_addr =
+		cpu_to_le32(mem_cfg->internal_txfifo_addr);
+	for (i = 0; i < TX_FIFO_INTERNAL_MAX_NUM; i++) {
+		dump_smem_cfg->internal_txfifo_size[i] =
+			cpu_to_le32(mem_cfg->internal_txfifo_size[i]);
+	}
+
+	dump_data = iwl_fw_error_next_data(dump_data);
+
 	/* We only dump the FIFOs if the FW is in error state */
-	if (test_bit(STATUS_FW_ERROR, &mvm->trans->status)) {
-		iwl_mvm_dump_fifos(mvm, &dump_data);
+	if (test_bit(STATUS_FW_ERROR, &fwrt->trans->status)) {
+		iwl_fw_dump_fifos(fwrt, &dump_data);
 		if (radio_len)
-			iwl_mvm_read_radio_reg(mvm, &dump_data);
+			iwl_read_radio_regs(fwrt, &dump_data);
 	}
 
-	if (mvm->fw_dump_desc) {
+	if (fwrt->dump.desc) {
 		dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_ERROR_INFO);
 		dump_data->len = cpu_to_le32(sizeof(*dump_trig) +
-					     mvm->fw_dump_desc->len);
+					     fwrt->dump.desc->len);
 		dump_trig = (void *)dump_data->data;
-		memcpy(dump_trig, &mvm->fw_dump_desc->trig_desc,
-		       sizeof(*dump_trig) + mvm->fw_dump_desc->len);
+		memcpy(dump_trig, &fwrt->dump.desc->trig_desc,
+		       sizeof(*dump_trig) + fwrt->dump.desc->len);
 
 		dump_data = iwl_fw_error_next_data(dump_data);
 	}
@@ -762,18 +794,18 @@ void iwl_mvm_fw_error_dump(struct iwl_mvm *mvm)
 	if (monitor_dump_only)
 		goto dump_trans_data;
 
-	if (!mvm->fw->n_dbg_mem_tlv) {
+	if (!fwrt->fw->n_dbg_mem_tlv) {
 		dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_MEM);
 		dump_data->len = cpu_to_le32(sram_len + sizeof(*dump_mem));
 		dump_mem = (void *)dump_data->data;
 		dump_mem->type = cpu_to_le32(IWL_FW_ERROR_DUMP_MEM_SRAM);
 		dump_mem->offset = cpu_to_le32(sram_ofs);
-		iwl_trans_read_mem_bytes(mvm->trans, sram_ofs, dump_mem->data,
+		iwl_trans_read_mem_bytes(fwrt->trans, sram_ofs, dump_mem->data,
 					 sram_len);
 		dump_data = iwl_fw_error_next_data(dump_data);
 	}
 
-	for (i = 0; i < mvm->fw->n_dbg_mem_tlv; i++) {
+	for (i = 0; i < fwrt->fw->n_dbg_mem_tlv; i++) {
 		u32 len = le32_to_cpu(fw_dbg_mem[i].len);
 		u32 ofs = le32_to_cpu(fw_dbg_mem[i].ofs);
 		bool success;
@@ -786,13 +818,13 @@ void iwl_mvm_fw_error_dump(struct iwl_mvm *mvm)
 
 		switch (dump_mem->type & cpu_to_le32(FW_DBG_MEM_TYPE_MASK)) {
 		case cpu_to_le32(FW_DBG_MEM_TYPE_REGULAR):
-			iwl_trans_read_mem_bytes(mvm->trans, ofs,
+			iwl_trans_read_mem_bytes(fwrt->trans, ofs,
 						 dump_mem->data,
 						 len);
 			success = true;
 			break;
 		case cpu_to_le32(FW_DBG_MEM_TYPE_PRPH):
-			success = iwl_read_prph_block(mvm->trans, ofs, len,
+			success = iwl_read_prph_block(fwrt->trans, ofs, len,
 						      (void *)dump_mem->data);
 			break;
 		default:
@@ -813,8 +845,9 @@ void iwl_mvm_fw_error_dump(struct iwl_mvm *mvm)
 		dump_data->len = cpu_to_le32(smem_len + sizeof(*dump_mem));
 		dump_mem = (void *)dump_data->data;
 		dump_mem->type = cpu_to_le32(IWL_FW_ERROR_DUMP_MEM_SMEM);
-		dump_mem->offset = cpu_to_le32(mvm->cfg->smem_offset);
-		iwl_trans_read_mem_bytes(mvm->trans, mvm->cfg->smem_offset,
+		dump_mem->offset = cpu_to_le32(fwrt->trans->cfg->smem_offset);
+		iwl_trans_read_mem_bytes(fwrt->trans,
+					 fwrt->trans->cfg->smem_offset,
 					 dump_mem->data, smem_len);
 		dump_data = iwl_fw_error_next_data(dump_data);
 	}
@@ -824,28 +857,29 @@ void iwl_mvm_fw_error_dump(struct iwl_mvm *mvm)
 		dump_data->len = cpu_to_le32(sram2_len + sizeof(*dump_mem));
 		dump_mem = (void *)dump_data->data;
 		dump_mem->type = cpu_to_le32(IWL_FW_ERROR_DUMP_MEM_SRAM);
-		dump_mem->offset = cpu_to_le32(mvm->cfg->dccm2_offset);
-		iwl_trans_read_mem_bytes(mvm->trans, mvm->cfg->dccm2_offset,
+		dump_mem->offset = cpu_to_le32(fwrt->trans->cfg->dccm2_offset);
+		iwl_trans_read_mem_bytes(fwrt->trans,
+					 fwrt->trans->cfg->dccm2_offset,
 					 dump_mem->data, sram2_len);
 		dump_data = iwl_fw_error_next_data(dump_data);
 	}
 
 	/* Dump fw's virtual image */
-	if (!mvm->trans->cfg->gen2 &&
-	    mvm->fw->img[mvm->cur_ucode].paging_mem_size &&
-	    mvm->fw_paging_db[0].fw_paging_block) {
-		for (i = 1; i < mvm->num_of_paging_blk + 1; i++) {
+	if (!fwrt->trans->cfg->gen2 &&
+	    fwrt->fw->img[fwrt->cur_fw_img].paging_mem_size &&
+	    fwrt->fw_paging_db[0].fw_paging_block) {
+		for (i = 1; i < fwrt->num_of_paging_blk + 1; i++) {
 			struct iwl_fw_error_dump_paging *paging;
 			struct page *pages =
-				mvm->fw_paging_db[i].fw_paging_block;
-			dma_addr_t addr = mvm->fw_paging_db[i].fw_paging_phys;
+				fwrt->fw_paging_db[i].fw_paging_block;
+			dma_addr_t addr = fwrt->fw_paging_db[i].fw_paging_phys;
 
 			dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_PAGING);
 			dump_data->len = cpu_to_le32(sizeof(*paging) +
 						     PAGING_BLOCK_SIZE);
 			paging = (void *)dump_data->data;
 			paging->index = cpu_to_le32(i);
-			dma_sync_single_for_cpu(mvm->trans->dev, addr,
+			dma_sync_single_for_cpu(fwrt->trans->dev, addr,
 						PAGING_BLOCK_SIZE,
 						DMA_BIDIRECTIONAL);
 			memcpy(paging->data, page_address(pages),
@@ -855,20 +889,20 @@ void iwl_mvm_fw_error_dump(struct iwl_mvm *mvm)
 	}
 
 	if (prph_len) {
-		iwl_dump_prph(mvm->trans, &dump_data,
+		iwl_dump_prph(fwrt->trans, &dump_data,
 			      iwl_prph_dump_addr_comm,
 			      ARRAY_SIZE(iwl_prph_dump_addr_comm));
 
-		if (mvm->cfg->mq_rx_supported)
-			iwl_dump_prph(mvm->trans, &dump_data,
+		if (fwrt->trans->cfg->mq_rx_supported)
+			iwl_dump_prph(fwrt->trans, &dump_data,
 				      iwl_prph_dump_addr_9000,
 				      ARRAY_SIZE(iwl_prph_dump_addr_9000));
 	}
 
 dump_trans_data:
-	fw_error_dump->trans_ptr = iwl_trans_dump_data(mvm->trans,
-						       mvm->fw_dump_trig);
-	fw_error_dump->op_mode_len = file_len;
+	fw_error_dump->trans_ptr = iwl_trans_dump_data(fwrt->trans,
+						       fwrt->dump.trig);
+	fw_error_dump->fwrt_len = file_len;
 	if (fw_error_dump->trans_ptr)
 		file_len += fw_error_dump->trans_ptr->len;
 	dump_file->file_len = cpu_to_le32(file_len);
@@ -877,68 +911,72 @@ dump_trans_data:
 	if (sg_dump_data) {
 		sg_pcopy_from_buffer(sg_dump_data,
 				     sg_nents(sg_dump_data),
-				     fw_error_dump->op_mode_ptr,
-				     fw_error_dump->op_mode_len, 0);
+				     fw_error_dump->fwrt_ptr,
+				     fw_error_dump->fwrt_len, 0);
 		if (fw_error_dump->trans_ptr)
 			sg_pcopy_from_buffer(sg_dump_data,
 					     sg_nents(sg_dump_data),
 					     fw_error_dump->trans_ptr->data,
 					     fw_error_dump->trans_ptr->len,
-					     fw_error_dump->op_mode_len);
-		dev_coredumpsg(mvm->trans->dev, sg_dump_data, file_len,
+					     fw_error_dump->fwrt_len);
+		dev_coredumpsg(fwrt->trans->dev, sg_dump_data, file_len,
 			       GFP_KERNEL);
 	}
-	vfree(fw_error_dump->op_mode_ptr);
+	vfree(fw_error_dump->fwrt_ptr);
 	vfree(fw_error_dump->trans_ptr);
 	kfree(fw_error_dump);
 
 out:
-	iwl_mvm_free_fw_dump_desc(mvm);
-	mvm->fw_dump_trig = NULL;
-	clear_bit(IWL_MVM_STATUS_DUMPING_FW_LOG, &mvm->status);
+	iwl_fw_free_dump_desc(fwrt);
+	fwrt->dump.trig = NULL;
+	clear_bit(IWL_FWRT_STATUS_DUMPING, &fwrt->status);
 }
+IWL_EXPORT_SYMBOL(iwl_fw_error_dump);
 
-const struct iwl_mvm_dump_desc iwl_mvm_dump_desc_assert = {
+const struct iwl_fw_dump_desc iwl_dump_desc_assert = {
 	.trig_desc = {
 		.type = cpu_to_le32(FW_DBG_TRIGGER_FW_ASSERT),
 	},
 };
+IWL_EXPORT_SYMBOL(iwl_dump_desc_assert);
 
-int iwl_mvm_fw_dbg_collect_desc(struct iwl_mvm *mvm,
-				const struct iwl_mvm_dump_desc *desc,
-				const struct iwl_fw_dbg_trigger_tlv *trigger)
+int iwl_fw_dbg_collect_desc(struct iwl_fw_runtime *fwrt,
+			    const struct iwl_fw_dump_desc *desc,
+			    const struct iwl_fw_dbg_trigger_tlv *trigger)
 {
 	unsigned int delay = 0;
 
 	if (trigger)
 		delay = msecs_to_jiffies(le32_to_cpu(trigger->stop_delay));
 
-	if (WARN(mvm->trans->state == IWL_TRANS_NO_FW,
+	if (WARN(fwrt->trans->state == IWL_TRANS_NO_FW,
 		 "Can't collect dbg data when FW isn't alive\n"))
 		return -EIO;
 
-	if (test_and_set_bit(IWL_MVM_STATUS_DUMPING_FW_LOG, &mvm->status))
+	if (test_and_set_bit(IWL_FWRT_STATUS_DUMPING, &fwrt->status))
 		return -EBUSY;
 
-	if (WARN_ON(mvm->fw_dump_desc))
-		iwl_mvm_free_fw_dump_desc(mvm);
+	if (WARN_ON(fwrt->dump.desc))
+		iwl_fw_free_dump_desc(fwrt);
 
-	IWL_WARN(mvm, "Collecting data: trigger %d fired.\n",
+	IWL_WARN(fwrt, "Collecting data: trigger %d fired.\n",
 		 le32_to_cpu(desc->trig_desc.type));
 
-	mvm->fw_dump_desc = desc;
-	mvm->fw_dump_trig = trigger;
+	fwrt->dump.desc = desc;
+	fwrt->dump.trig = trigger;
 
-	schedule_delayed_work(&mvm->fw_dump_wk, delay);
+	schedule_delayed_work(&fwrt->dump.wk, delay);
 
 	return 0;
 }
+IWL_EXPORT_SYMBOL(iwl_fw_dbg_collect_desc);
 
-int iwl_mvm_fw_dbg_collect(struct iwl_mvm *mvm, enum iwl_fw_dbg_trigger trig,
-			   const char *str, size_t len,
-			   const struct iwl_fw_dbg_trigger_tlv *trigger)
+int iwl_fw_dbg_collect(struct iwl_fw_runtime *fwrt,
+		       enum iwl_fw_dbg_trigger trig,
+		       const char *str, size_t len,
+		       const struct iwl_fw_dbg_trigger_tlv *trigger)
 {
-	struct iwl_mvm_dump_desc *desc;
+	struct iwl_fw_dump_desc *desc;
 
 	desc = kzalloc(sizeof(*desc) + len, GFP_ATOMIC);
 	if (!desc)
@@ -948,12 +986,13 @@ int iwl_mvm_fw_dbg_collect(struct iwl_mvm *mvm, enum iwl_fw_dbg_trigger trig,
 	desc->trig_desc.type = cpu_to_le32(trig);
 	memcpy(desc->trig_desc.data, str, len);
 
-	return iwl_mvm_fw_dbg_collect_desc(mvm, desc, trigger);
+	return iwl_fw_dbg_collect_desc(fwrt, desc, trigger);
 }
+IWL_EXPORT_SYMBOL(iwl_fw_dbg_collect);
 
-int iwl_mvm_fw_dbg_collect_trig(struct iwl_mvm *mvm,
-				struct iwl_fw_dbg_trigger_tlv *trigger,
-				const char *fmt, ...)
+int iwl_fw_dbg_collect_trig(struct iwl_fw_runtime *fwrt,
+			    struct iwl_fw_dbg_trigger_tlv *trigger,
+			    const char *fmt, ...)
 {
 	u16 occurrences = le16_to_cpu(trigger->occurrences);
 	int ret, len = 0;
@@ -978,8 +1017,8 @@ int iwl_mvm_fw_dbg_collect_trig(struct iwl_mvm *mvm,
 		len = strlen(buf) + 1;
 	}
 
-	ret = iwl_mvm_fw_dbg_collect(mvm, le32_to_cpu(trigger->id), buf, len,
-				     trigger);
+	ret = iwl_fw_dbg_collect(fwrt, le32_to_cpu(trigger->id), buf, len,
+				 trigger);
 
 	if (ret)
 		return ret;
@@ -987,37 +1026,42 @@ int iwl_mvm_fw_dbg_collect_trig(struct iwl_mvm *mvm,
 	trigger->occurrences = cpu_to_le16(occurrences - 1);
 	return 0;
 }
+IWL_EXPORT_SYMBOL(iwl_fw_dbg_collect_trig);
 
-int iwl_mvm_start_fw_dbg_conf(struct iwl_mvm *mvm, u8 conf_id)
+int iwl_fw_start_dbg_conf(struct iwl_fw_runtime *fwrt, u8 conf_id)
 {
 	u8 *ptr;
 	int ret;
 	int i;
 
-	if (WARN_ONCE(conf_id >= ARRAY_SIZE(mvm->fw->dbg_conf_tlv),
+	if (WARN_ONCE(conf_id >= ARRAY_SIZE(fwrt->fw->dbg_conf_tlv),
 		      "Invalid configuration %d\n", conf_id))
 		return -EINVAL;
 
 	/* EARLY START - firmware's configuration is hard coded */
-	if ((!mvm->fw->dbg_conf_tlv[conf_id] ||
-	     !mvm->fw->dbg_conf_tlv[conf_id]->num_of_hcmds) &&
+	if ((!fwrt->fw->dbg_conf_tlv[conf_id] ||
+	     !fwrt->fw->dbg_conf_tlv[conf_id]->num_of_hcmds) &&
 	    conf_id == FW_DBG_START_FROM_ALIVE)
 		return 0;
 
-	if (!mvm->fw->dbg_conf_tlv[conf_id])
+	if (!fwrt->fw->dbg_conf_tlv[conf_id])
 		return -EINVAL;
 
-	if (mvm->fw_dbg_conf != FW_DBG_INVALID)
-		IWL_WARN(mvm, "FW already configured (%d) - re-configuring\n",
-			 mvm->fw_dbg_conf);
+	if (fwrt->dump.conf != FW_DBG_INVALID)
+		IWL_WARN(fwrt, "FW already configured (%d) - re-configuring\n",
+			 fwrt->dump.conf);
 
 	/* Send all HCMDs for configuring the FW debug */
-	ptr = (void *)&mvm->fw->dbg_conf_tlv[conf_id]->hcmd;
-	for (i = 0; i < mvm->fw->dbg_conf_tlv[conf_id]->num_of_hcmds; i++) {
+	ptr = (void *)&fwrt->fw->dbg_conf_tlv[conf_id]->hcmd;
+	for (i = 0; i < fwrt->fw->dbg_conf_tlv[conf_id]->num_of_hcmds; i++) {
 		struct iwl_fw_dbg_conf_hcmd *cmd = (void *)ptr;
+		struct iwl_host_cmd hcmd = {
+			.id = cmd->id,
+			.len = { le16_to_cpu(cmd->len), },
+			.data = { cmd->data, },
+		};
 
-		ret = iwl_mvm_send_cmd_pdu(mvm, cmd->id, 0,
-					   le16_to_cpu(cmd->len), cmd->data);
+		ret = iwl_trans_send_cmd(fwrt->trans, &hcmd);
 		if (ret)
 			return ret;
 
@@ -1025,7 +1069,59 @@ int iwl_mvm_start_fw_dbg_conf(struct iwl_mvm *mvm, u8 conf_id)
 		ptr += le16_to_cpu(cmd->len);
 	}
 
-	mvm->fw_dbg_conf = conf_id;
+	fwrt->dump.conf = conf_id;
 
 	return 0;
 }
+IWL_EXPORT_SYMBOL(iwl_fw_start_dbg_conf);
+
+void iwl_fw_error_dump_wk(struct work_struct *work)
+{
+	struct iwl_fw_runtime *fwrt =
+		container_of(work, struct iwl_fw_runtime, dump.wk.work);
+
+	if (fwrt->ops && fwrt->ops->dump_start &&
+	    fwrt->ops->dump_start(fwrt->ops_ctx))
+		return;
+
+	if (fwrt->trans->cfg->device_family == IWL_DEVICE_FAMILY_7000) {
+		/* stop recording */
+		iwl_set_bits_prph(fwrt->trans, MON_BUFF_SAMPLE_CTL, 0x100);
+
+		iwl_fw_error_dump(fwrt);
+
+		/* start recording again if the firmware is not crashed */
+		if (!test_bit(STATUS_FW_ERROR, &fwrt->trans->status) &&
+		    fwrt->fw->dbg_dest_tlv) {
+			iwl_clear_bits_prph(fwrt->trans,
+					    MON_BUFF_SAMPLE_CTL, 0x100);
+			iwl_clear_bits_prph(fwrt->trans,
+					    MON_BUFF_SAMPLE_CTL, 0x1);
+			iwl_set_bits_prph(fwrt->trans,
+					  MON_BUFF_SAMPLE_CTL, 0x1);
+		}
+	} else {
+		u32 in_sample = iwl_read_prph(fwrt->trans, DBGC_IN_SAMPLE);
+		u32 out_ctrl = iwl_read_prph(fwrt->trans, DBGC_OUT_CTRL);
+
+		/* stop recording */
+		iwl_write_prph(fwrt->trans, DBGC_IN_SAMPLE, 0);
+		udelay(100);
+		iwl_write_prph(fwrt->trans, DBGC_OUT_CTRL, 0);
+		/* wait before we collect the data till the DBGC stop */
+		udelay(500);
+
+		iwl_fw_error_dump(fwrt);
+
+		/* start recording again if the firmware is not crashed */
+		if (!test_bit(STATUS_FW_ERROR, &fwrt->trans->status) &&
+		    fwrt->fw->dbg_dest_tlv) {
+			iwl_write_prph(fwrt->trans, DBGC_IN_SAMPLE, in_sample);
+			iwl_write_prph(fwrt->trans, DBGC_OUT_CTRL, out_ctrl);
+		}
+	}
+
+	if (fwrt->ops && fwrt->ops->dump_end)
+		fwrt->ops->dump_end(fwrt->ops_ctx);
+}
+
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw-dbg.h b/drivers/net/wireless/intel/iwlwifi/fw/dbg.h
index 4a5287a0c617..0f810ea89d31 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw-dbg.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/dbg.h
@@ -7,7 +7,7 @@
  *
  * Copyright(c) 2008 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
- * Copyright(c) 2015        Intel Deutschland GmbH
+ * Copyright(c) 2015 - 2017 Intel Deutschland GmbH
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -32,7 +32,7 @@
  *
  * Copyright(c) 2005 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
- * Copyright(c) 2015        Intel Deutschland GmbH
+ * Copyright(c) 2015 - 2017 Intel Deutschland GmbH
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -63,24 +63,46 @@
  *
  *****************************************************************************/
 
-#ifndef __mvm_fw_dbg_h__
-#define __mvm_fw_dbg_h__
-#include "fw/file.h"
-#include "fw/error-dump.h"
-#include "mvm.h"
-
-void iwl_mvm_fw_error_dump(struct iwl_mvm *mvm);
-void iwl_mvm_free_fw_dump_desc(struct iwl_mvm *mvm);
-int iwl_mvm_fw_dbg_collect_desc(struct iwl_mvm *mvm,
-				const struct iwl_mvm_dump_desc *desc,
-				const struct iwl_fw_dbg_trigger_tlv *trigger);
-int iwl_mvm_fw_dbg_collect(struct iwl_mvm *mvm, enum iwl_fw_dbg_trigger trig,
-			   const char *str, size_t len,
-			   const struct iwl_fw_dbg_trigger_tlv *trigger);
-int iwl_mvm_fw_dbg_collect_trig(struct iwl_mvm *mvm,
-				struct iwl_fw_dbg_trigger_tlv *trigger,
-				const char *fmt, ...) __printf(3, 4);
-int iwl_mvm_start_fw_dbg_conf(struct iwl_mvm *mvm, u8 id);
+#ifndef __iwl_fw_dbg_h__
+#define __iwl_fw_dbg_h__
+#include <linux/workqueue.h>
+#include <net/cfg80211.h>
+#include "runtime.h"
+#include "file.h"
+#include "error-dump.h"
+
+/**
+ * struct iwl_fw_dump_desc - describes the dump
+ * @len: length of trig_desc->data
+ * @trig_desc: the description of the dump
+ */
+struct iwl_fw_dump_desc {
+	size_t len;
+	/* must be last */
+	struct iwl_fw_error_dump_trigger_desc trig_desc;
+};
+
+extern const struct iwl_fw_dump_desc iwl_dump_desc_assert;
+
+static inline void iwl_fw_free_dump_desc(struct iwl_fw_runtime *fwrt)
+{
+	if (fwrt->dump.desc != &iwl_dump_desc_assert)
+		kfree(fwrt->dump.desc);
+	fwrt->dump.desc = NULL;
+}
+
+void iwl_fw_error_dump(struct iwl_fw_runtime *fwrt);
+int iwl_fw_dbg_collect_desc(struct iwl_fw_runtime *fwrt,
+			    const struct iwl_fw_dump_desc *desc,
+			    const struct iwl_fw_dbg_trigger_tlv *trigger);
+int iwl_fw_dbg_collect(struct iwl_fw_runtime *fwrt,
+		       enum iwl_fw_dbg_trigger trig,
+		       const char *str, size_t len,
+		       const struct iwl_fw_dbg_trigger_tlv *trigger);
+int iwl_fw_dbg_collect_trig(struct iwl_fw_runtime *fwrt,
+			    struct iwl_fw_dbg_trigger_tlv *trigger,
+			    const char *fmt, ...) __printf(3, 4);
+int iwl_fw_start_dbg_conf(struct iwl_fw_runtime *fwrt, u8 id);
 
 #define iwl_fw_dbg_trigger_enabled(fw, id) ({			\
 	void *__dbg_trigger = (fw)->dbg_trigger_tlv[(id)];	\
@@ -101,25 +123,25 @@ _iwl_fw_dbg_get_trigger(const struct iwl_fw *fw, enum iwl_fw_dbg_trigger id)
 
 static inline bool
 iwl_fw_dbg_trigger_vif_match(struct iwl_fw_dbg_trigger_tlv *trig,
-			     struct ieee80211_vif *vif)
+			     struct wireless_dev *wdev)
 {
 	u32 trig_vif = le32_to_cpu(trig->vif_type);
 
 	return trig_vif == IWL_FW_DBG_CONF_VIF_ANY ||
-	       ieee80211_vif_type_p2p(vif) == trig_vif;
+	       wdev->iftype == trig_vif;
 }
 
 static inline bool
-iwl_fw_dbg_trigger_stop_conf_match(struct iwl_mvm *mvm,
+iwl_fw_dbg_trigger_stop_conf_match(struct iwl_fw_runtime *fwrt,
 				   struct iwl_fw_dbg_trigger_tlv *trig)
 {
 	return ((trig->mode & IWL_FW_DBG_TRIGGER_STOP) &&
-		(mvm->fw_dbg_conf == FW_DBG_INVALID ||
-		(BIT(mvm->fw_dbg_conf) & le32_to_cpu(trig->stop_conf_ids))));
+		(fwrt->dump.conf == FW_DBG_INVALID ||
+		(BIT(fwrt->dump.conf) & le32_to_cpu(trig->stop_conf_ids))));
 }
 
 static inline bool
-iwl_fw_dbg_no_trig_window(struct iwl_mvm *mvm,
+iwl_fw_dbg_no_trig_window(struct iwl_fw_runtime *fwrt,
 			  struct iwl_fw_dbg_trigger_tlv *trig)
 {
 	unsigned long wind_jiff =
@@ -127,49 +149,66 @@ iwl_fw_dbg_no_trig_window(struct iwl_mvm *mvm,
 	u32 id = le32_to_cpu(trig->id);
 
 	/* If this is the first event checked, jump to update start ts */
-	if (mvm->fw_dbg_non_collect_ts_start[id] &&
-	    (time_after(mvm->fw_dbg_non_collect_ts_start[id] + wind_jiff,
+	if (fwrt->dump.non_collect_ts_start[id] &&
+	    (time_after(fwrt->dump.non_collect_ts_start[id] + wind_jiff,
 			jiffies)))
 		return true;
 
-	mvm->fw_dbg_non_collect_ts_start[id] = jiffies;
+	fwrt->dump.non_collect_ts_start[id] = jiffies;
 	return false;
 }
 
 static inline bool
-iwl_fw_dbg_trigger_check_stop(struct iwl_mvm *mvm,
-			      struct ieee80211_vif *vif,
+iwl_fw_dbg_trigger_check_stop(struct iwl_fw_runtime *fwrt,
+			      struct wireless_dev *wdev,
 			      struct iwl_fw_dbg_trigger_tlv *trig)
 {
-	if (vif && !iwl_fw_dbg_trigger_vif_match(trig, vif))
+	if (wdev && !iwl_fw_dbg_trigger_vif_match(trig, wdev))
 		return false;
 
-	if (iwl_fw_dbg_no_trig_window(mvm, trig)) {
-		IWL_WARN(mvm, "Trigger %d occurred while no-collect window.\n",
+	if (iwl_fw_dbg_no_trig_window(fwrt, trig)) {
+		IWL_WARN(fwrt, "Trigger %d occurred while no-collect window.\n",
 			 trig->id);
 		return false;
 	}
 
-	return iwl_fw_dbg_trigger_stop_conf_match(mvm, trig);
+	return iwl_fw_dbg_trigger_stop_conf_match(fwrt, trig);
 }
 
 static inline void
-_iwl_fw_dbg_trigger_simple_stop(struct iwl_mvm *mvm,
-				struct ieee80211_vif *vif,
+_iwl_fw_dbg_trigger_simple_stop(struct iwl_fw_runtime *fwrt,
+				struct wireless_dev *wdev,
 				struct iwl_fw_dbg_trigger_tlv *trigger)
 {
 	if (!trigger)
 		return;
 
-	if (!iwl_fw_dbg_trigger_check_stop(mvm, vif, trigger))
+	if (!iwl_fw_dbg_trigger_check_stop(fwrt, wdev, trigger))
 		return;
 
-	iwl_mvm_fw_dbg_collect_trig(mvm, trigger, NULL);
+	iwl_fw_dbg_collect_trig(fwrt, trigger, NULL);
 }
 
-#define iwl_fw_dbg_trigger_simple_stop(mvm, vif, trig)	\
-	_iwl_fw_dbg_trigger_simple_stop((mvm), (vif),	\
-					iwl_fw_dbg_get_trigger((mvm)->fw,\
+#define iwl_fw_dbg_trigger_simple_stop(fwrt, wdev, trig)	\
+	_iwl_fw_dbg_trigger_simple_stop((fwrt), (wdev),		\
+					iwl_fw_dbg_get_trigger((fwrt)->fw,\
 							       (trig)))
 
-#endif  /* __mvm_fw_dbg_h__ */
+static inline void iwl_fw_dump_conf_clear(struct iwl_fw_runtime *fwrt)
+{
+	fwrt->dump.conf = FW_DBG_INVALID;
+}
+
+void iwl_fw_error_dump_wk(struct work_struct *work);
+
+static inline void iwl_fw_flush_dump(struct iwl_fw_runtime *fwrt)
+{
+	flush_delayed_work(&fwrt->dump.wk);
+}
+
+static inline void iwl_fw_cancel_dump(struct iwl_fw_runtime *fwrt)
+{
+	cancel_delayed_work_sync(&fwrt->dump.wk);
+}
+
+#endif  /* __iwl_fw_dbg_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/error-dump.h b/drivers/net/wireless/intel/iwlwifi/fw/error-dump.h
index cfebde68a391..ed7beca8817e 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/error-dump.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/error-dump.h
@@ -7,6 +7,7 @@
  *
  * Copyright(c) 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2014 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -33,6 +34,7 @@
  *
  * Copyright(c) 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2014 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -92,6 +94,9 @@
  * @IWL_FW_ERROR_DUMP_EXTERNAL: used only by external code utilities, and
  *	for that reason is not in use in any other place in the Linux Wi-Fi
  *	stack.
+ * @IWL_FW_ERROR_DUMP_MEM_CFG: the addresses and sizes of fifos in the smem,
+ *	which we get from the fw after ALIVE. The content is structured as
+ *	&struct iwl_fw_error_dump_smem_cfg.
  */
 enum iwl_fw_error_dump_type {
 	/* 0 is deprecated */
@@ -110,6 +115,7 @@ enum iwl_fw_error_dump_type {
 	IWL_FW_ERROR_DUMP_RADIO_REG = 13,
 	IWL_FW_ERROR_DUMP_INTERNAL_TXF = 14,
 	IWL_FW_ERROR_DUMP_EXTERNAL = 15, /* Do not move */
+	IWL_FW_ERROR_DUMP_MEM_CFG = 16,
 
 	IWL_FW_ERROR_DUMP_MAX,
 };
@@ -208,6 +214,30 @@ struct iwl_fw_error_dump_fw_mon {
 	u8 data[];
 } __packed;
 
+#define MAX_NUM_LMAC 2
+#define TX_FIFO_INTERNAL_MAX_NUM	6
+#define TX_FIFO_MAX_NUM			15
+/**
+ * struct iwl_fw_error_dump_smem_cfg - Dump SMEM configuration
+ *	This must follow &struct iwl_fwrt_shared_mem_cfg.
+ * @num_lmacs: number of lmacs
+ * @num_txfifo_entries: number of tx fifos
+ * @lmac: sizes of lmacs txfifos and rxfifo1
+ * @rxfifo2_size: size of rxfifo2
+ * @internal_txfifo_addr: address of internal tx fifo
+ * @internal_txfifo_size: size of internal tx fifo
+ */
+struct iwl_fw_error_dump_smem_cfg {
+	__le32 num_lmacs;
+	__le32 num_txfifo_entries;
+	struct {
+		__le32 txfifo_size[TX_FIFO_MAX_NUM];
+		__le32 rxfifo1_size;
+	} lmac[MAX_NUM_LMAC];
+	__le32 rxfifo2_size;
+	__le32 internal_txfifo_addr;
+	__le32 internal_txfifo_size[TX_FIFO_INTERNAL_MAX_NUM];
+} __packed;
 /**
  * struct iwl_fw_error_dump_prph - periphery registers data
  * @prph_start: address of the first register in this chunk
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/file.h b/drivers/net/wireless/intel/iwlwifi/fw/file.h
index 0fa8c473f1e2..279248cd9cfb 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/file.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/file.h
@@ -246,6 +246,8 @@ typedef unsigned int __bitwise iwl_ucode_tlv_api_t;
  * @IWL_UCODE_TLV_API_STA_TYPE: This ucode supports station type assignement.
  * @IWL_UCODE_TLV_API_NAN2_VER2: This ucode supports NAN API version 2
  * @IWL_UCODE_TLV_API_NEW_RX_STATS: should new RX STATISTICS API be used
+ * @IWL_UCODE_TLV_API_ATS_COEX_EXTERNAL: the coex notification is enlared to
+ *	include information about ACL time sharing.
  *
  * @NUM_IWL_UCODE_TLV_API: number of bits used
  */
@@ -260,7 +262,9 @@ enum iwl_ucode_tlv_api {
 	IWL_UCODE_TLV_API_STA_TYPE		= (__force iwl_ucode_tlv_api_t)30,
 	IWL_UCODE_TLV_API_NAN2_VER2		= (__force iwl_ucode_tlv_api_t)31,
 	/* API Set 1 */
+	IWL_UCODE_TLV_API_NEW_BEACON_TEMPLATE	= (__force iwl_ucode_tlv_api_t)34,
 	IWL_UCODE_TLV_API_NEW_RX_STATS		= (__force iwl_ucode_tlv_api_t)35,
+	IWL_UCODE_TLV_API_COEX_ATS_EXTERNAL	= (__force iwl_ucode_tlv_api_t)37,
 
 	NUM_IWL_UCODE_TLV_API
 #ifdef __CHECKER__
@@ -328,6 +332,7 @@ typedef unsigned int __bitwise iwl_ucode_tlv_capa_t;
  * @IWL_UCODE_TLV_CAPA_TX_POWER_ACK: reduced TX power API has larger
  *	command size (command version 4) that supports toggling ACK TX
  *	power reduction.
+ * @IWL_UCODE_TLV_CAPA_MLME_OFFLOAD: supports MLME offload
  *
  * @NUM_IWL_UCODE_TLV_CAPA: number of bits used
  */
@@ -373,6 +378,8 @@ enum iwl_ucode_tlv_capa {
 	IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG	= (__force iwl_ucode_tlv_capa_t)80,
 	IWL_UCODE_TLV_CAPA_LQM_SUPPORT			= (__force iwl_ucode_tlv_capa_t)81,
 	IWL_UCODE_TLV_CAPA_TX_POWER_ACK			= (__force iwl_ucode_tlv_capa_t)84,
+	IWL_UCODE_TLV_CAPA_LED_CMD_SUPPORT		= (__force iwl_ucode_tlv_capa_t)86,
+	IWL_UCODE_TLV_CAPA_MLME_OFFLOAD			= (__force iwl_ucode_tlv_capa_t)96,
 
 	NUM_IWL_UCODE_TLV_CAPA
 #ifdef __CHECKER__
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/init.c b/drivers/net/wireless/intel/iwlwifi/fw/init.c
new file mode 100644
index 000000000000..bfe5316bbb6a
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/init.c
@@ -0,0 +1,75 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+#include "iwl-drv.h"
+#include "runtime.h"
+#include "dbg.h"
+
+void iwl_fw_runtime_init(struct iwl_fw_runtime *fwrt, struct iwl_trans *trans,
+			 const struct iwl_fw *fw,
+			 const struct iwl_fw_runtime_ops *ops, void *ops_ctx)
+{
+	memset(fwrt, 0, sizeof(*fwrt));
+	fwrt->trans = trans;
+	fwrt->fw = fw;
+	fwrt->dev = trans->dev;
+	fwrt->dump.conf = FW_DBG_INVALID;
+	fwrt->ops = ops;
+	fwrt->ops_ctx = ops_ctx;
+	INIT_DELAYED_WORK(&fwrt->dump.wk, iwl_fw_error_dump_wk);
+}
+IWL_EXPORT_SYMBOL(iwl_fw_runtime_init);
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/notif-wait.c b/drivers/net/wireless/intel/iwlwifi/fw/notif-wait.c
index 29bb92e3df59..1096c945a68b 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/notif-wait.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/notif-wait.c
@@ -6,7 +6,7 @@
  * GPL LICENSE SUMMARY
  *
  * Copyright(c) 2007 - 2014 Intel Corporation. All rights reserved.
- * Copyright(c) 2015 Intel Deutschland GmbH
+ * Copyright(c) 2015 - 2017 Intel Deutschland GmbH
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -32,6 +32,7 @@
  * BSD LICENSE
  *
  * Copyright(c) 2005 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2015 - 2017 Intel Deutschland GmbH
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -161,6 +162,15 @@ iwl_init_notification_wait(struct iwl_notif_wait_data *notif_wait,
 }
 IWL_EXPORT_SYMBOL(iwl_init_notification_wait);
 
+void iwl_remove_notification(struct iwl_notif_wait_data *notif_wait,
+			     struct iwl_notification_wait *wait_entry)
+{
+	spin_lock_bh(&notif_wait->notif_wait_lock);
+	list_del(&wait_entry->list);
+	spin_unlock_bh(&notif_wait->notif_wait_lock);
+}
+IWL_EXPORT_SYMBOL(iwl_remove_notification);
+
 int iwl_wait_notification(struct iwl_notif_wait_data *notif_wait,
 			  struct iwl_notification_wait *wait_entry,
 			  unsigned long timeout)
@@ -171,9 +181,7 @@ int iwl_wait_notification(struct iwl_notif_wait_data *notif_wait,
 				 wait_entry->triggered || wait_entry->aborted,
 				 timeout);
 
-	spin_lock_bh(&notif_wait->notif_wait_lock);
-	list_del(&wait_entry->list);
-	spin_unlock_bh(&notif_wait->notif_wait_lock);
+	iwl_remove_notification(notif_wait, wait_entry);
 
 	if (wait_entry->aborted)
 		return -EIO;
@@ -184,12 +192,3 @@ int iwl_wait_notification(struct iwl_notif_wait_data *notif_wait,
 	return 0;
 }
 IWL_EXPORT_SYMBOL(iwl_wait_notification);
-
-void iwl_remove_notification(struct iwl_notif_wait_data *notif_wait,
-			     struct iwl_notification_wait *wait_entry)
-{
-	spin_lock_bh(&notif_wait->notif_wait_lock);
-	list_del(&wait_entry->list);
-	spin_unlock_bh(&notif_wait->notif_wait_lock);
-}
-IWL_EXPORT_SYMBOL(iwl_remove_notification);
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/nvm.c b/drivers/net/wireless/intel/iwlwifi/fw/nvm.c
new file mode 100644
index 000000000000..bd2e1fb43f5a
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/nvm.c
@@ -0,0 +1,162 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
+ * USA
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+#include "iwl-drv.h"
+#include "runtime.h"
+#include "fw/api/nvm-reg.h"
+#include "fw/api/commands.h"
+#include "iwl-nvm-parse.h"
+
+struct iwl_nvm_data *iwl_fw_get_nvm(struct iwl_fw_runtime *fwrt)
+{
+	struct iwl_nvm_get_info cmd = {};
+	struct iwl_nvm_get_info_rsp *rsp;
+	struct iwl_trans *trans = fwrt->trans;
+	struct iwl_nvm_data *nvm;
+	struct iwl_host_cmd hcmd = {
+		.flags = CMD_WANT_SKB | CMD_SEND_IN_RFKILL,
+		.data = { &cmd, },
+		.len = { sizeof(cmd) },
+		.id = WIDE_ID(REGULATORY_AND_NVM_GROUP, NVM_GET_INFO)
+	};
+	int  ret;
+	bool lar_fw_supported = !iwlwifi_mod_params.lar_disable &&
+				fw_has_capa(&fwrt->fw->ucode_capa,
+					    IWL_UCODE_TLV_CAPA_LAR_SUPPORT);
+
+	ret = iwl_trans_send_cmd(trans, &hcmd);
+	if (ret)
+		return ERR_PTR(ret);
+
+	if (WARN(iwl_rx_packet_payload_len(hcmd.resp_pkt) != sizeof(*rsp),
+		 "Invalid payload len in NVM response from FW %d",
+		 iwl_rx_packet_payload_len(hcmd.resp_pkt))) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	rsp = (void *)hcmd.resp_pkt->data;
+	if (le32_to_cpu(rsp->general.flags) & NVM_GENERAL_FLAGS_EMPTY_OTP)
+		IWL_INFO(fwrt, "OTP is empty\n");
+
+	nvm = kzalloc(sizeof(*nvm) +
+		      sizeof(struct ieee80211_channel) * IWL_NUM_CHANNELS,
+		      GFP_KERNEL);
+	if (!nvm) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	iwl_set_hw_address_from_csr(trans, nvm);
+	/* TODO: if platform NVM has MAC address - override it here */
+
+	if (!is_valid_ether_addr(nvm->hw_addr)) {
+		IWL_ERR(fwrt, "no valid mac address was found\n");
+		ret = -EINVAL;
+		goto err_free;
+	}
+
+	IWL_INFO(trans, "base HW address: %pM\n", nvm->hw_addr);
+
+	/* Initialize general data */
+	nvm->nvm_version = le16_to_cpu(rsp->general.nvm_version);
+
+	/* Initialize MAC sku data */
+	nvm->sku_cap_11ac_enable =
+		le32_to_cpu(rsp->mac_sku.enable_11ac);
+	nvm->sku_cap_11n_enable =
+		le32_to_cpu(rsp->mac_sku.enable_11n);
+	nvm->sku_cap_band_24GHz_enable =
+		le32_to_cpu(rsp->mac_sku.enable_24g);
+	nvm->sku_cap_band_52GHz_enable =
+		le32_to_cpu(rsp->mac_sku.enable_5g);
+	nvm->sku_cap_mimo_disabled =
+		le32_to_cpu(rsp->mac_sku.mimo_disable);
+
+	/* Initialize PHY sku data */
+	nvm->valid_tx_ant = (u8)le32_to_cpu(rsp->phy_sku.tx_chains);
+	nvm->valid_rx_ant = (u8)le32_to_cpu(rsp->phy_sku.rx_chains);
+
+	/* Initialize regulatory data */
+	nvm->lar_enabled =
+		le32_to_cpu(rsp->regulatory.lar_enabled) && lar_fw_supported;
+
+	iwl_init_sbands(trans->dev, trans->cfg, nvm,
+			rsp->regulatory.channel_profile,
+			nvm->valid_tx_ant & fwrt->fw->valid_tx_ant,
+			nvm->valid_rx_ant & fwrt->fw->valid_rx_ant,
+			nvm->lar_enabled, false);
+
+	iwl_free_resp(&hcmd);
+	return nvm;
+
+err_free:
+	kfree(nvm);
+out:
+	iwl_free_resp(&hcmd);
+	return ERR_PTR(ret);
+}
+IWL_EXPORT_SYMBOL(iwl_fw_get_nvm);
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/paging.c b/drivers/net/wireless/intel/iwlwifi/fw/paging.c
new file mode 100644
index 000000000000..1610722b8099
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/paging.c
@@ -0,0 +1,414 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+#include "iwl-drv.h"
+#include "runtime.h"
+#include "fw/api/commands.h"
+
+void iwl_free_fw_paging(struct iwl_fw_runtime *fwrt)
+{
+	int i;
+
+	if (!fwrt->fw_paging_db[0].fw_paging_block)
+		return;
+
+	for (i = 0; i < NUM_OF_FW_PAGING_BLOCKS; i++) {
+		struct iwl_fw_paging *paging = &fwrt->fw_paging_db[i];
+
+		if (!paging->fw_paging_block) {
+			IWL_DEBUG_FW(fwrt,
+				     "Paging: block %d already freed, continue to next page\n",
+				     i);
+
+			continue;
+		}
+		dma_unmap_page(fwrt->trans->dev, paging->fw_paging_phys,
+			       paging->fw_paging_size, DMA_BIDIRECTIONAL);
+
+		__free_pages(paging->fw_paging_block,
+			     get_order(paging->fw_paging_size));
+		paging->fw_paging_block = NULL;
+	}
+	kfree(fwrt->trans->paging_download_buf);
+	fwrt->trans->paging_download_buf = NULL;
+	fwrt->trans->paging_db = NULL;
+
+	memset(fwrt->fw_paging_db, 0, sizeof(fwrt->fw_paging_db));
+}
+IWL_EXPORT_SYMBOL(iwl_free_fw_paging);
+
+static int iwl_alloc_fw_paging_mem(struct iwl_fw_runtime *fwrt,
+				   const struct fw_img *image)
+{
+	struct page *block;
+	dma_addr_t phys = 0;
+	int blk_idx, order, num_of_pages, size, dma_enabled;
+
+	if (fwrt->fw_paging_db[0].fw_paging_block)
+		return 0;
+
+	dma_enabled = is_device_dma_capable(fwrt->trans->dev);
+
+	/* ensure BLOCK_2_EXP_SIZE is power of 2 of PAGING_BLOCK_SIZE */
+	BUILD_BUG_ON(BIT(BLOCK_2_EXP_SIZE) != PAGING_BLOCK_SIZE);
+
+	num_of_pages = image->paging_mem_size / FW_PAGING_SIZE;
+	fwrt->num_of_paging_blk =
+		DIV_ROUND_UP(num_of_pages, NUM_OF_PAGE_PER_GROUP);
+	fwrt->num_of_pages_in_last_blk =
+		num_of_pages -
+		NUM_OF_PAGE_PER_GROUP * (fwrt->num_of_paging_blk - 1);
+
+	IWL_DEBUG_FW(fwrt,
+		     "Paging: allocating mem for %d paging blocks, each block holds 8 pages, last block holds %d pages\n",
+		     fwrt->num_of_paging_blk,
+		     fwrt->num_of_pages_in_last_blk);
+
+	/*
+	 * Allocate CSS and paging blocks in dram.
+	 */
+	for (blk_idx = 0; blk_idx < fwrt->num_of_paging_blk + 1; blk_idx++) {
+		/* For CSS allocate 4KB, for others PAGING_BLOCK_SIZE (32K) */
+		size = blk_idx ? PAGING_BLOCK_SIZE : FW_PAGING_SIZE;
+		order = get_order(size);
+		block = alloc_pages(GFP_KERNEL, order);
+		if (!block) {
+			/* free all the previous pages since we failed */
+			iwl_free_fw_paging(fwrt);
+			return -ENOMEM;
+		}
+
+		fwrt->fw_paging_db[blk_idx].fw_paging_block = block;
+		fwrt->fw_paging_db[blk_idx].fw_paging_size = size;
+
+		if (dma_enabled) {
+			phys = dma_map_page(fwrt->trans->dev, block, 0,
+					    PAGE_SIZE << order,
+					    DMA_BIDIRECTIONAL);
+			if (dma_mapping_error(fwrt->trans->dev, phys)) {
+				/*
+				 * free the previous pages and the current one
+				 * since we failed to map_page.
+				 */
+				iwl_free_fw_paging(fwrt);
+				return -ENOMEM;
+			}
+			fwrt->fw_paging_db[blk_idx].fw_paging_phys = phys;
+		} else {
+			fwrt->fw_paging_db[blk_idx].fw_paging_phys =
+				PAGING_ADDR_SIG |
+				blk_idx << BLOCK_2_EXP_SIZE;
+		}
+
+		if (!blk_idx)
+			IWL_DEBUG_FW(fwrt,
+				     "Paging: allocated 4K(CSS) bytes (order %d) for firmware paging.\n",
+				     order);
+		else
+			IWL_DEBUG_FW(fwrt,
+				     "Paging: allocated 32K bytes (order %d) for firmware paging.\n",
+				     order);
+	}
+
+	return 0;
+}
+
+static int iwl_fill_paging_mem(struct iwl_fw_runtime *fwrt,
+			       const struct fw_img *image)
+{
+	int sec_idx, idx;
+	u32 offset = 0;
+
+	/*
+	 * find where is the paging image start point:
+	 * if CPU2 exist and it's in paging format, then the image looks like:
+	 * CPU1 sections (2 or more)
+	 * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between CPU1 to CPU2
+	 * CPU2 sections (not paged)
+	 * PAGING_SEPARATOR_SECTION delimiter - separate between CPU2
+	 * non paged to CPU2 paging sec
+	 * CPU2 paging CSS
+	 * CPU2 paging image (including instruction and data)
+	 */
+	for (sec_idx = 0; sec_idx < image->num_sec; sec_idx++) {
+		if (image->sec[sec_idx].offset == PAGING_SEPARATOR_SECTION) {
+			sec_idx++;
+			break;
+		}
+	}
+
+	/*
+	 * If paging is enabled there should be at least 2 more sections left
+	 * (one for CSS and one for Paging data)
+	 */
+	if (sec_idx >= image->num_sec - 1) {
+		IWL_ERR(fwrt, "Paging: Missing CSS and/or paging sections\n");
+		iwl_free_fw_paging(fwrt);
+		return -EINVAL;
+	}
+
+	/* copy the CSS block to the dram */
+	IWL_DEBUG_FW(fwrt, "Paging: load paging CSS to FW, sec = %d\n",
+		     sec_idx);
+
+	memcpy(page_address(fwrt->fw_paging_db[0].fw_paging_block),
+	       image->sec[sec_idx].data,
+	       fwrt->fw_paging_db[0].fw_paging_size);
+	dma_sync_single_for_device(fwrt->trans->dev,
+				   fwrt->fw_paging_db[0].fw_paging_phys,
+				   fwrt->fw_paging_db[0].fw_paging_size,
+				   DMA_BIDIRECTIONAL);
+
+	IWL_DEBUG_FW(fwrt,
+		     "Paging: copied %d CSS bytes to first block\n",
+		     fwrt->fw_paging_db[0].fw_paging_size);
+
+	sec_idx++;
+
+	/*
+	 * copy the paging blocks to the dram
+	 * loop index start from 1 since that CSS block already copied to dram
+	 * and CSS index is 0.
+	 * loop stop at num_of_paging_blk since that last block is not full.
+	 */
+	for (idx = 1; idx < fwrt->num_of_paging_blk; idx++) {
+		struct iwl_fw_paging *block = &fwrt->fw_paging_db[idx];
+
+		memcpy(page_address(block->fw_paging_block),
+		       image->sec[sec_idx].data + offset,
+		       block->fw_paging_size);
+		dma_sync_single_for_device(fwrt->trans->dev,
+					   block->fw_paging_phys,
+					   block->fw_paging_size,
+					   DMA_BIDIRECTIONAL);
+
+		IWL_DEBUG_FW(fwrt,
+			     "Paging: copied %d paging bytes to block %d\n",
+			     fwrt->fw_paging_db[idx].fw_paging_size,
+			     idx);
+
+		offset += fwrt->fw_paging_db[idx].fw_paging_size;
+	}
+
+	/* copy the last paging block */
+	if (fwrt->num_of_pages_in_last_blk > 0) {
+		struct iwl_fw_paging *block = &fwrt->fw_paging_db[idx];
+
+		memcpy(page_address(block->fw_paging_block),
+		       image->sec[sec_idx].data + offset,
+		       FW_PAGING_SIZE * fwrt->num_of_pages_in_last_blk);
+		dma_sync_single_for_device(fwrt->trans->dev,
+					   block->fw_paging_phys,
+					   block->fw_paging_size,
+					   DMA_BIDIRECTIONAL);
+
+		IWL_DEBUG_FW(fwrt,
+			     "Paging: copied %d pages in the last block %d\n",
+			     fwrt->num_of_pages_in_last_blk, idx);
+	}
+
+	return 0;
+}
+
+static int iwl_save_fw_paging(struct iwl_fw_runtime *fwrt,
+			      const struct fw_img *fw)
+{
+	int ret;
+
+	ret = iwl_alloc_fw_paging_mem(fwrt, fw);
+	if (ret)
+		return ret;
+
+	return iwl_fill_paging_mem(fwrt, fw);
+}
+
+/* send paging cmd to FW in case CPU2 has paging image */
+static int iwl_send_paging_cmd(struct iwl_fw_runtime *fwrt,
+			       const struct fw_img *fw)
+{
+	struct iwl_fw_paging_cmd paging_cmd = {
+		.flags = cpu_to_le32(PAGING_CMD_IS_SECURED |
+				     PAGING_CMD_IS_ENABLED |
+				     (fwrt->num_of_pages_in_last_blk <<
+				      PAGING_CMD_NUM_OF_PAGES_IN_LAST_GRP_POS)),
+		.block_size = cpu_to_le32(BLOCK_2_EXP_SIZE),
+		.block_num = cpu_to_le32(fwrt->num_of_paging_blk),
+	};
+	struct iwl_host_cmd hcmd = {
+		.id = iwl_cmd_id(FW_PAGING_BLOCK_CMD, IWL_ALWAYS_LONG_GROUP, 0),
+		.len = { sizeof(paging_cmd), },
+		.data = { &paging_cmd, },
+	};
+	int blk_idx;
+
+	/* loop for for all paging blocks + CSS block */
+	for (blk_idx = 0; blk_idx < fwrt->num_of_paging_blk + 1; blk_idx++) {
+		dma_addr_t addr = fwrt->fw_paging_db[blk_idx].fw_paging_phys;
+		__le32 phy_addr;
+
+		addr = addr >> PAGE_2_EXP_SIZE;
+		phy_addr = cpu_to_le32(addr);
+		paging_cmd.device_phy_addr[blk_idx] = phy_addr;
+	}
+
+	return iwl_trans_send_cmd(fwrt->trans, &hcmd);
+}
+
+/*
+ * Send paging item cmd to FW in case CPU2 has paging image
+ */
+static int iwl_trans_get_paging_item(struct iwl_fw_runtime *fwrt)
+{
+	int ret;
+	struct iwl_fw_get_item_cmd fw_get_item_cmd = {
+		.item_id = cpu_to_le32(IWL_FW_ITEM_ID_PAGING),
+	};
+	struct iwl_fw_get_item_resp *item_resp;
+	struct iwl_host_cmd cmd = {
+		.id = iwl_cmd_id(FW_GET_ITEM_CMD, IWL_ALWAYS_LONG_GROUP, 0),
+		.flags = CMD_WANT_SKB | CMD_SEND_IN_RFKILL,
+		.data = { &fw_get_item_cmd, },
+		.len = { sizeof(fw_get_item_cmd), },
+	};
+
+	ret = iwl_trans_send_cmd(fwrt->trans, &cmd);
+	if (ret) {
+		IWL_ERR(fwrt,
+			"Paging: Failed to send FW_GET_ITEM_CMD cmd (err = %d)\n",
+			ret);
+		return ret;
+	}
+
+	item_resp = (void *)((struct iwl_rx_packet *)cmd.resp_pkt)->data;
+	if (item_resp->item_id != cpu_to_le32(IWL_FW_ITEM_ID_PAGING)) {
+		IWL_ERR(fwrt,
+			"Paging: got wrong item in FW_GET_ITEM_CMD resp (item_id = %u)\n",
+			le32_to_cpu(item_resp->item_id));
+		ret = -EIO;
+		goto exit;
+	}
+
+	/* Add an extra page for headers */
+	fwrt->trans->paging_download_buf = kzalloc(PAGING_BLOCK_SIZE +
+						  FW_PAGING_SIZE,
+						  GFP_KERNEL);
+	if (!fwrt->trans->paging_download_buf) {
+		ret = -ENOMEM;
+		goto exit;
+	}
+	fwrt->trans->paging_req_addr = le32_to_cpu(item_resp->item_val);
+	fwrt->trans->paging_db = fwrt->fw_paging_db;
+	IWL_DEBUG_FW(fwrt,
+		     "Paging: got paging request address (paging_req_addr 0x%08x)\n",
+		     fwrt->trans->paging_req_addr);
+
+exit:
+	iwl_free_resp(&cmd);
+
+	return ret;
+}
+
+int iwl_init_paging(struct iwl_fw_runtime *fwrt, enum iwl_ucode_type type)
+{
+	const struct fw_img *fw = &fwrt->fw->img[type];
+	int ret;
+
+	if (fwrt->trans->cfg->gen2)
+		return 0;
+
+	/*
+	 * Configure and operate fw paging mechanism.
+	 * The driver configures the paging flow only once.
+	 * The CPU2 paging image is included in the IWL_UCODE_INIT image.
+	 */
+	if (!fw->paging_mem_size)
+		return 0;
+
+	/*
+	 * When dma is not enabled, the driver needs to copy / write
+	 * the downloaded / uploaded page to / from the smem.
+	 * This gets the location of the place were the pages are
+	 * stored.
+	 */
+	if (!is_device_dma_capable(fwrt->trans->dev)) {
+		ret = iwl_trans_get_paging_item(fwrt);
+		if (ret) {
+			IWL_ERR(fwrt, "failed to get FW paging item\n");
+			return ret;
+		}
+	}
+
+	ret = iwl_save_fw_paging(fwrt, fw);
+	if (ret) {
+		IWL_ERR(fwrt, "failed to save the FW paging image\n");
+		return ret;
+	}
+
+	ret = iwl_send_paging_cmd(fwrt, fw);
+	if (ret) {
+		IWL_ERR(fwrt, "failed to send the paging cmd\n");
+		iwl_free_fw_paging(fwrt);
+		return ret;
+	}
+
+	return 0;
+}
+IWL_EXPORT_SYMBOL(iwl_init_paging);
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/runtime.h b/drivers/net/wireless/intel/iwlwifi/fw/runtime.h
new file mode 100644
index 000000000000..50cfb6d795a5
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/runtime.h
@@ -0,0 +1,158 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+#ifndef __iwl_fw_runtime_h__
+#define __iwl_fw_runtime_h__
+
+#include "iwl-config.h"
+#include "iwl-trans.h"
+#include "img.h"
+#include "fw/api/debug.h"
+#include "fw/api/paging.h"
+#include "iwl-eeprom-parse.h"
+
+struct iwl_fw_runtime_ops {
+	int (*dump_start)(void *ctx);
+	void (*dump_end)(void *ctx);
+};
+
+#define MAX_NUM_LMAC 2
+struct iwl_fwrt_shared_mem_cfg {
+	int num_lmacs;
+	int num_txfifo_entries;
+	struct {
+		u32 txfifo_size[TX_FIFO_MAX_NUM];
+		u32 rxfifo1_size;
+	} lmac[MAX_NUM_LMAC];
+	u32 rxfifo2_size;
+	u32 internal_txfifo_addr;
+	u32 internal_txfifo_size[TX_FIFO_INTERNAL_MAX_NUM];
+};
+
+enum iwl_fw_runtime_status {
+	IWL_FWRT_STATUS_DUMPING = 0,
+};
+
+/**
+ * struct iwl_fw_runtime - runtime data for firmware
+ * @fw: firmware image
+ * @cfg: NIC configuration
+ * @dev: device pointer
+ * @ops: user ops
+ * @ops_ctx: user ops context
+ * @status: status flags
+ * @fw_paging_db: paging database
+ * @num_of_paging_blk: number of paging blocks
+ * @num_of_pages_in_last_blk: number of pages in the last block
+ * @smem_cfg: saved firmware SMEM configuration
+ * @cur_fw_img: current firmware image, must be maintained by
+ *	the driver by calling &iwl_fw_set_current_image()
+ * @dump: debug dump data
+ */
+struct iwl_fw_runtime {
+	struct iwl_trans *trans;
+	const struct iwl_fw *fw;
+	struct device *dev;
+
+	const struct iwl_fw_runtime_ops *ops;
+	void *ops_ctx;
+
+	unsigned long status;
+
+	/* Paging */
+	struct iwl_fw_paging fw_paging_db[NUM_OF_FW_PAGING_BLOCKS];
+	u16 num_of_paging_blk;
+	u16 num_of_pages_in_last_blk;
+
+	enum iwl_ucode_type cur_fw_img;
+
+	/* memory configuration */
+	struct iwl_fwrt_shared_mem_cfg smem_cfg;
+
+	/* debug */
+	struct {
+		const struct iwl_fw_dump_desc *desc;
+		const struct iwl_fw_dbg_trigger_tlv *trig;
+		struct delayed_work wk;
+
+		u8 conf;
+
+		/* ts of the beginning of a non-collect fw dbg data period */
+		unsigned long non_collect_ts_start[FW_DBG_TRIGGER_MAX - 1];
+	} dump;
+};
+
+void iwl_fw_runtime_init(struct iwl_fw_runtime *fwrt, struct iwl_trans *trans,
+			 const struct iwl_fw *fw,
+			 const struct iwl_fw_runtime_ops *ops, void *ops_ctx);
+
+static inline void iwl_fw_set_current_image(struct iwl_fw_runtime *fwrt,
+					    enum iwl_ucode_type cur_fw_img)
+{
+	fwrt->cur_fw_img = cur_fw_img;
+}
+
+int iwl_init_paging(struct iwl_fw_runtime *fwrt, enum iwl_ucode_type type);
+void iwl_free_fw_paging(struct iwl_fw_runtime *fwrt);
+
+void iwl_get_shared_mem_conf(struct iwl_fw_runtime *fwrt);
+
+void iwl_fwrt_handle_notification(struct iwl_fw_runtime *fwrt,
+				  struct iwl_rx_cmd_buffer *rxb);
+struct iwl_nvm_data *iwl_fw_get_nvm(struct iwl_fw_runtime *fwrt);
+
+#endif /* __iwl_fw_runtime_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/smem.c b/drivers/net/wireless/intel/iwlwifi/fw/smem.c
new file mode 100644
index 000000000000..76675736ba4f
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/smem.c
@@ -0,0 +1,155 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+#include "iwl-drv.h"
+#include "runtime.h"
+#include "fw/api/commands.h"
+
+static void iwl_parse_shared_mem_a000(struct iwl_fw_runtime *fwrt,
+				      struct iwl_rx_packet *pkt)
+{
+	struct iwl_shared_mem_cfg *mem_cfg = (void *)pkt->data;
+	int i, lmac;
+	int lmac_num = le32_to_cpu(mem_cfg->lmac_num);
+
+	if (WARN_ON(lmac_num > ARRAY_SIZE(mem_cfg->lmac_smem)))
+		return;
+
+	fwrt->smem_cfg.num_lmacs = lmac_num;
+	fwrt->smem_cfg.num_txfifo_entries =
+		ARRAY_SIZE(mem_cfg->lmac_smem[0].txfifo_size);
+	fwrt->smem_cfg.rxfifo2_size = le32_to_cpu(mem_cfg->rxfifo2_size);
+
+	for (lmac = 0; lmac < lmac_num; lmac++) {
+		struct iwl_shared_mem_lmac_cfg *lmac_cfg =
+			&mem_cfg->lmac_smem[lmac];
+
+		for (i = 0; i < ARRAY_SIZE(lmac_cfg->txfifo_size); i++)
+			fwrt->smem_cfg.lmac[lmac].txfifo_size[i] =
+				le32_to_cpu(lmac_cfg->txfifo_size[i]);
+		fwrt->smem_cfg.lmac[lmac].rxfifo1_size =
+			le32_to_cpu(lmac_cfg->rxfifo1_size);
+	}
+}
+
+static void iwl_parse_shared_mem(struct iwl_fw_runtime *fwrt,
+				 struct iwl_rx_packet *pkt)
+{
+	struct iwl_shared_mem_cfg_v2 *mem_cfg = (void *)pkt->data;
+	int i;
+
+	fwrt->smem_cfg.num_lmacs = 1;
+
+	fwrt->smem_cfg.num_txfifo_entries = ARRAY_SIZE(mem_cfg->txfifo_size);
+	for (i = 0; i < ARRAY_SIZE(mem_cfg->txfifo_size); i++)
+		fwrt->smem_cfg.lmac[0].txfifo_size[i] =
+			le32_to_cpu(mem_cfg->txfifo_size[i]);
+
+	fwrt->smem_cfg.lmac[0].rxfifo1_size =
+		le32_to_cpu(mem_cfg->rxfifo_size[0]);
+	fwrt->smem_cfg.rxfifo2_size = le32_to_cpu(mem_cfg->rxfifo_size[1]);
+
+	/* new API has more data, from rxfifo_addr field and on */
+	if (fw_has_capa(&fwrt->fw->ucode_capa,
+			IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG)) {
+		BUILD_BUG_ON(sizeof(fwrt->smem_cfg.internal_txfifo_size) !=
+			     sizeof(mem_cfg->internal_txfifo_size));
+
+		fwrt->smem_cfg.internal_txfifo_addr =
+			le32_to_cpu(mem_cfg->internal_txfifo_addr);
+
+		for (i = 0;
+		     i < ARRAY_SIZE(fwrt->smem_cfg.internal_txfifo_size);
+		     i++)
+			fwrt->smem_cfg.internal_txfifo_size[i] =
+				le32_to_cpu(mem_cfg->internal_txfifo_size[i]);
+	}
+}
+
+void iwl_get_shared_mem_conf(struct iwl_fw_runtime *fwrt)
+{
+	struct iwl_host_cmd cmd = {
+		.flags = CMD_WANT_SKB,
+		.data = { NULL, },
+		.len = { 0, },
+	};
+	struct iwl_rx_packet *pkt;
+
+	if (fw_has_capa(&fwrt->fw->ucode_capa,
+			IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG))
+		cmd.id = iwl_cmd_id(SHARED_MEM_CFG_CMD, SYSTEM_GROUP, 0);
+	else
+		cmd.id = SHARED_MEM_CFG;
+
+	if (WARN_ON(iwl_trans_send_cmd(fwrt->trans, &cmd)))
+		return;
+
+	pkt = cmd.resp_pkt;
+	if (fwrt->trans->cfg->device_family == IWL_DEVICE_FAMILY_A000)
+		iwl_parse_shared_mem_a000(fwrt, pkt);
+	else
+		iwl_parse_shared_mem(fwrt, pkt);
+
+	IWL_DEBUG_INFO(fwrt, "SHARED MEM CFG: got memory offsets/sizes\n");
+
+	iwl_free_resp(&cmd);
+}
+IWL_EXPORT_SYMBOL(iwl_get_shared_mem_conf);
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-config.h b/drivers/net/wireless/intel/iwlwifi/iwl-config.h
index c52623cb7c2a..3e057b539d5b 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-config.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-config.h
@@ -276,10 +276,10 @@ struct iwl_pwr_tx_backoff {
  * @fw_name_pre: Firmware filename prefix. The api version and extension
  *	(.ucode) will be added to filename before loading from disk. The
  *	filename is constructed as fw_name_pre<api>.ucode.
- * @fw_name_pre_next_step: same as @fw_name_pre, only for next step
+ * @fw_name_pre_b_or_c_step: same as @fw_name_pre, only for b or c steps
  *	(if supported)
- * @fw_name_pre_rf_next_step: same as @fw_name_pre_next_step, only for rf next
- *	step. Supported only in integrated solutions.
+ * @fw_name_pre_rf_next_step: same as @fw_name_pre_b_or_c_step, only for rf
+ *	next step. Supported only in integrated solutions.
  * @ucode_api_max: Highest version of uCode API supported by driver.
  * @ucode_api_min: Lowest version of uCode API supported by driver.
  * @max_inst_size: The maximal length of the fw inst section
@@ -330,7 +330,7 @@ struct iwl_cfg {
 	/* params specific to an individual device within a device family */
 	const char *name;
 	const char *fw_name_pre;
-	const char *fw_name_pre_next_step;
+	const char *fw_name_pre_b_or_c_step;
 	const char *fw_name_pre_rf_next_step;
 	/* params not likely to change within a device family */
 	const struct iwl_base_params *base_params;
@@ -463,6 +463,9 @@ extern const struct iwl_cfg iwla000_2ac_cfg_hr;
 extern const struct iwl_cfg iwla000_2ac_cfg_hr_cdb;
 extern const struct iwl_cfg iwla000_2ac_cfg_jf;
 extern const struct iwl_cfg iwla000_2ax_cfg_hr;
+extern const struct iwl_cfg iwla000_2ax_cfg_qnj_hr_f0;
+extern const struct iwl_cfg iwla000_2ax_cfg_qnj_jf_b0;
+extern const struct iwl_cfg iwla000_2ax_cfg_qnj_hr_a0;
 #endif /* CONFIG_IWLMVM */
 
 #endif /* __IWL_CONFIG_H__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-csr.h b/drivers/net/wireless/intel/iwlwifi/iwl-csr.h
index c6c1876c1ad4..b03e0f975b5a 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-csr.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-csr.h
@@ -169,7 +169,7 @@
 
 /*
  * CSR Hardware Revision Workaround Register.  Indicates hardware rev;
- * "step" determines CCK backoff for txpower calculation.  Used for 4965 only.
+ * "step" determines CCK backoff for txpower calculation.
  * See also CSR_HW_REV register.
  * Bit fields:
  *  3-2:  0 = A, 1 = B, 2 = C, 3 = D step
@@ -354,11 +354,16 @@ enum {
 #define CSR_HW_REV_TYPE_135		(0x0000120)
 #define CSR_HW_REV_TYPE_7265D		(0x0000210)
 #define CSR_HW_REV_TYPE_NONE		(0x00001F0)
+#define CSR_HW_REV_TYPE_QNJ		(0x0000360)
+#define CSR_HW_REV_TYPE_HR_CDB		(0x0000340)
 
 /* RF_ID value */
-#define CSR_HW_RF_ID_TYPE_JF		(0x00105000)
+#define CSR_HW_RF_ID_TYPE_JF		(0x00105100)
 #define CSR_HW_RF_ID_TYPE_HR		(0x0010A000)
-#define CSR_HW_RF_ID_TYPE_HRCDB		(0x00109000)
+#define CSR_HW_RF_ID_TYPE_HRCDB		(0x00109F00)
+
+/* HW_RF CHIP ID  */
+#define CSR_HW_RF_ID_TYPE_CHIP_ID(_val) (((_val) >> 12) & 0xFFF)
 
 /* EEPROM REG */
 #define CSR_EEPROM_REG_READ_VALID_MSK	(0x00000001)
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-drv.c b/drivers/net/wireless/intel/iwlwifi/iwl-drv.c
index 6fdb5921e17f..99676d6c4713 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-drv.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-drv.c
@@ -216,8 +216,9 @@ static int iwl_request_firmware(struct iwl_drv *drv, bool first)
 	const char *fw_pre_name;
 
 	if (drv->trans->cfg->device_family == IWL_DEVICE_FAMILY_9000 &&
-	    CSR_HW_REV_STEP(drv->trans->hw_rev) == SILICON_B_STEP)
-		fw_pre_name = cfg->fw_name_pre_next_step;
+	    (CSR_HW_REV_STEP(drv->trans->hw_rev) == SILICON_B_STEP ||
+	     CSR_HW_REV_STEP(drv->trans->hw_rev) == SILICON_C_STEP))
+		fw_pre_name = cfg->fw_name_pre_b_or_c_step;
 	else if (drv->trans->cfg->integrated &&
 		 CSR_HW_RFID_STEP(drv->trans->hw_rf_id) == SILICON_B_STEP &&
 		 cfg->fw_name_pre_rf_next_step)
@@ -478,8 +479,8 @@ static int iwl_set_default_calib(struct iwl_drv *drv, const u8 *data)
 	return 0;
 }
 
-static int iwl_set_ucode_api_flags(struct iwl_drv *drv, const u8 *data,
-				   struct iwl_ucode_capabilities *capa)
+static void iwl_set_ucode_api_flags(struct iwl_drv *drv, const u8 *data,
+				    struct iwl_ucode_capabilities *capa)
 {
 	const struct iwl_ucode_api *ucode_api = (void *)data;
 	u32 api_index = le32_to_cpu(ucode_api->api_index);
@@ -487,23 +488,20 @@ static int iwl_set_ucode_api_flags(struct iwl_drv *drv, const u8 *data,
 	int i;
 
 	if (api_index >= DIV_ROUND_UP(NUM_IWL_UCODE_TLV_API, 32)) {
-		IWL_ERR(drv,
-			"api flags index %d larger than supported by driver\n",
-			api_index);
-		/* don't return an error so we can load FW that has more bits */
-		return 0;
+		IWL_WARN(drv,
+			 "api flags index %d larger than supported by driver\n",
+			 api_index);
+		return;
 	}
 
 	for (i = 0; i < 32; i++) {
 		if (api_flags & BIT(i))
 			__set_bit(i + 32 * api_index, capa->_api);
 	}
-
-	return 0;
 }
 
-static int iwl_set_ucode_capabilities(struct iwl_drv *drv, const u8 *data,
-				      struct iwl_ucode_capabilities *capa)
+static void iwl_set_ucode_capabilities(struct iwl_drv *drv, const u8 *data,
+				       struct iwl_ucode_capabilities *capa)
 {
 	const struct iwl_ucode_capa *ucode_capa = (void *)data;
 	u32 api_index = le32_to_cpu(ucode_capa->api_index);
@@ -511,19 +509,16 @@ static int iwl_set_ucode_capabilities(struct iwl_drv *drv, const u8 *data,
 	int i;
 
 	if (api_index >= DIV_ROUND_UP(NUM_IWL_UCODE_TLV_CAPA, 32)) {
-		IWL_ERR(drv,
-			"capa flags index %d larger than supported by driver\n",
-			api_index);
-		/* don't return an error so we can load FW that has more bits */
-		return 0;
+		IWL_WARN(drv,
+			 "capa flags index %d larger than supported by driver\n",
+			 api_index);
+		return;
 	}
 
 	for (i = 0; i < 32; i++) {
 		if (api_flags & BIT(i))
 			__set_bit(i + 32 * api_index, capa->_capa);
 	}
-
-	return 0;
 }
 
 static int iwl_parse_v1_v2_firmware(struct iwl_drv *drv,
@@ -765,14 +760,12 @@ static int iwl_parse_tlv_firmware(struct iwl_drv *drv,
 		case IWL_UCODE_TLV_API_CHANGES_SET:
 			if (tlv_len != sizeof(struct iwl_ucode_api))
 				goto invalid_tlv_len;
-			if (iwl_set_ucode_api_flags(drv, tlv_data, capa))
-				goto tlv_error;
+			iwl_set_ucode_api_flags(drv, tlv_data, capa);
 			break;
 		case IWL_UCODE_TLV_ENABLED_CAPABILITIES:
 			if (tlv_len != sizeof(struct iwl_ucode_capa))
 				goto invalid_tlv_len;
-			if (iwl_set_ucode_capabilities(drv, tlv_data, capa))
-				goto tlv_error;
+			iwl_set_ucode_capabilities(drv, tlv_data, capa);
 			break;
 		case IWL_UCODE_TLV_INIT_EVTLOG_PTR:
 			if (tlv_len != sizeof(u32))
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-io.c b/drivers/net/wireless/intel/iwlwifi/iwl-io.c
index c527b8c10370..efb1998dcabd 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-io.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-io.c
@@ -241,20 +241,12 @@ IWL_EXPORT_SYMBOL(iwl_clear_bits_prph);
 
 void iwl_force_nmi(struct iwl_trans *trans)
 {
-	if (trans->cfg->device_family < IWL_DEVICE_FAMILY_8000) {
+	if (trans->cfg->device_family < IWL_DEVICE_FAMILY_9000)
 		iwl_write_prph(trans, DEVICE_SET_NMI_REG,
 			       DEVICE_SET_NMI_VAL_DRV);
-		iwl_write_prph(trans, DEVICE_SET_NMI_REG,
-			       DEVICE_SET_NMI_VAL_HW);
-	} else if (trans->cfg->device_family == IWL_DEVICE_FAMILY_A000) {
+	else
 		iwl_write_prph(trans, UREG_NIC_SET_NMI_DRIVER,
-			       DEVICE_SET_NMI_8000_VAL);
-	} else {
-		iwl_write_prph(trans, DEVICE_SET_NMI_8000_REG,
-			       DEVICE_SET_NMI_8000_VAL);
-		iwl_write_prph(trans, DEVICE_SET_NMI_REG,
-			       DEVICE_SET_NMI_VAL_DRV);
-	}
+			       UREG_NIC_SET_NMI_DRIVER_NMI_FROM_DRIVER_MSK);
 }
 IWL_EXPORT_SYMBOL(iwl_force_nmi);
 
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c
index 5c08f4d40f6a..3014beef4873 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c
@@ -79,6 +79,7 @@
 /* NVM offsets (in words) definitions */
 enum wkp_nvm_offsets {
 	/* NVM HW-Section offset (in words) definitions */
+	SUBSYSTEM_ID = 0x0A,
 	HW_ADDR = 0x15,
 
 	/* NVM SW-Section offset (in words) definitions */
@@ -183,22 +184,26 @@ static struct ieee80211_rate iwl_cfg80211_rates[] = {
  * @NVM_CHANNEL_INDOOR_ONLY: only indoor use is allowed
  * @NVM_CHANNEL_GO_CONCURRENT: GO operation is allowed when connected to BSS
  *	on same channel on 2.4 or same UNII band on 5.2
- * @NVM_CHANNEL_WIDE: 20 MHz channel okay (?)
- * @NVM_CHANNEL_40MHZ: 40 MHz channel okay (?)
- * @NVM_CHANNEL_80MHZ: 80 MHz channel okay (?)
- * @NVM_CHANNEL_160MHZ: 160 MHz channel okay (?)
+ * @NVM_CHANNEL_UNIFORM: uniform spreading required
+ * @NVM_CHANNEL_20MHZ: 20 MHz channel okay
+ * @NVM_CHANNEL_40MHZ: 40 MHz channel okay
+ * @NVM_CHANNEL_80MHZ: 80 MHz channel okay
+ * @NVM_CHANNEL_160MHZ: 160 MHz channel okay
+ * @NVM_CHANNEL_DC_HIGH: DC HIGH required/allowed (?)
  */
 enum iwl_nvm_channel_flags {
-	NVM_CHANNEL_VALID = BIT(0),
-	NVM_CHANNEL_IBSS = BIT(1),
-	NVM_CHANNEL_ACTIVE = BIT(3),
-	NVM_CHANNEL_RADAR = BIT(4),
-	NVM_CHANNEL_INDOOR_ONLY = BIT(5),
-	NVM_CHANNEL_GO_CONCURRENT = BIT(6),
-	NVM_CHANNEL_WIDE = BIT(8),
-	NVM_CHANNEL_40MHZ = BIT(9),
-	NVM_CHANNEL_80MHZ = BIT(10),
-	NVM_CHANNEL_160MHZ = BIT(11),
+	NVM_CHANNEL_VALID		= BIT(0),
+	NVM_CHANNEL_IBSS		= BIT(1),
+	NVM_CHANNEL_ACTIVE		= BIT(3),
+	NVM_CHANNEL_RADAR		= BIT(4),
+	NVM_CHANNEL_INDOOR_ONLY		= BIT(5),
+	NVM_CHANNEL_GO_CONCURRENT	= BIT(6),
+	NVM_CHANNEL_UNIFORM		= BIT(7),
+	NVM_CHANNEL_20MHZ		= BIT(8),
+	NVM_CHANNEL_40MHZ		= BIT(9),
+	NVM_CHANNEL_80MHZ		= BIT(10),
+	NVM_CHANNEL_160MHZ		= BIT(11),
+	NVM_CHANNEL_DC_HIGH		= BIT(12),
 };
 
 #define CHECK_AND_PRINT_I(x)	\
@@ -254,13 +259,12 @@ static u32 iwl_get_channel_flags(u8 ch_num, int ch_idx, bool is_5ghz,
 static int iwl_init_channel_map(struct device *dev, const struct iwl_cfg *cfg,
 				struct iwl_nvm_data *data,
 				const __le16 * const nvm_ch_flags,
-				bool lar_supported)
+				bool lar_supported, bool no_wide_in_5ghz)
 {
 	int ch_idx;
 	int n_channels = 0;
 	struct ieee80211_channel *channel;
 	u16 ch_flags;
-	bool is_5ghz;
 	int num_of_ch, num_2ghz_channels;
 	const u8 *nvm_chan;
 
@@ -275,12 +279,20 @@ static int iwl_init_channel_map(struct device *dev, const struct iwl_cfg *cfg,
 	}
 
 	for (ch_idx = 0; ch_idx < num_of_ch; ch_idx++) {
+		bool is_5ghz = (ch_idx >= num_2ghz_channels);
+
 		ch_flags = __le16_to_cpup(nvm_ch_flags + ch_idx);
 
-		if (ch_idx >= num_2ghz_channels &&
-		    !data->sku_cap_band_52GHz_enable)
+		if (is_5ghz && !data->sku_cap_band_52GHz_enable)
 			continue;
 
+		/* workaround to disable wide channels in 5GHz */
+		if (no_wide_in_5ghz && is_5ghz) {
+			ch_flags &= ~(NVM_CHANNEL_40MHZ |
+				     NVM_CHANNEL_80MHZ |
+				     NVM_CHANNEL_160MHZ);
+		}
+
 		if (ch_flags & NVM_CHANNEL_160MHZ)
 			data->vht160_supported = true;
 
@@ -303,8 +315,8 @@ static int iwl_init_channel_map(struct device *dev, const struct iwl_cfg *cfg,
 		n_channels++;
 
 		channel->hw_value = nvm_chan[ch_idx];
-		channel->band = (ch_idx < num_2ghz_channels) ?
-				NL80211_BAND_2GHZ : NL80211_BAND_5GHZ;
+		channel->band = is_5ghz ?
+				NL80211_BAND_5GHZ : NL80211_BAND_2GHZ;
 		channel->center_freq =
 			ieee80211_channel_to_frequency(
 				channel->hw_value, channel->band);
@@ -316,7 +328,6 @@ static int iwl_init_channel_map(struct device *dev, const struct iwl_cfg *cfg,
 		 * is not used in mvm, and is used for backwards compatibility
 		 */
 		channel->max_power = IWL_DEFAULT_MAX_TX_POWER;
-		is_5ghz = channel->band == NL80211_BAND_5GHZ;
 
 		/* don't put limitations in case we're using LAR */
 		if (!lar_supported)
@@ -327,7 +338,7 @@ static int iwl_init_channel_map(struct device *dev, const struct iwl_cfg *cfg,
 			channel->flags = 0;
 
 		IWL_DEBUG_EEPROM(dev,
-				 "Ch. %d [%sGHz] flags 0x%x %s%s%s%s%s%s%s%s%s%s(%ddBm): Ad-Hoc %ssupported\n",
+				 "Ch. %d [%sGHz] flags 0x%x %s%s%s%s%s%s%s%s%s%s%s%s(%ddBm): Ad-Hoc %ssupported\n",
 				 channel->hw_value,
 				 is_5ghz ? "5.2" : "2.4",
 				 ch_flags,
@@ -337,10 +348,12 @@ static int iwl_init_channel_map(struct device *dev, const struct iwl_cfg *cfg,
 				 CHECK_AND_PRINT_I(RADAR),
 				 CHECK_AND_PRINT_I(INDOOR_ONLY),
 				 CHECK_AND_PRINT_I(GO_CONCURRENT),
-				 CHECK_AND_PRINT_I(WIDE),
+				 CHECK_AND_PRINT_I(UNIFORM),
+				 CHECK_AND_PRINT_I(20MHZ),
 				 CHECK_AND_PRINT_I(40MHZ),
 				 CHECK_AND_PRINT_I(80MHZ),
 				 CHECK_AND_PRINT_I(160MHZ),
+				 CHECK_AND_PRINT_I(DC_HIGH),
 				 channel->max_power,
 				 ((ch_flags & NVM_CHANNEL_IBSS) &&
 				  !(ch_flags & NVM_CHANNEL_RADAR))
@@ -432,14 +445,15 @@ static void iwl_init_vht_hw_capab(const struct iwl_cfg *cfg,
 
 void iwl_init_sbands(struct device *dev, const struct iwl_cfg *cfg,
 		     struct iwl_nvm_data *data, const __le16 *nvm_ch_flags,
-		     u8 tx_chains, u8 rx_chains, bool lar_supported)
+		     u8 tx_chains, u8 rx_chains, bool lar_supported,
+		     bool no_wide_in_5ghz)
 {
 	int n_channels;
 	int n_used = 0;
 	struct ieee80211_supported_band *sband;
 
 	n_channels = iwl_init_channel_map(dev, cfg, data, nvm_ch_flags,
-					  lar_supported);
+					  lar_supported, no_wide_in_5ghz);
 	sband = &data->bands[NL80211_BAND_2GHZ];
 	sband->band = NL80211_BAND_2GHZ;
 	sband->bitrates = &iwl_cfg80211_rates[RATES_24_OFFS];
@@ -568,7 +582,7 @@ static void iwl_set_hw_address_family_8000(struct iwl_trans *trans,
 					   const struct iwl_cfg *cfg,
 					   struct iwl_nvm_data *data,
 					   const __le16 *mac_override,
-					   const __le16 *nvm_hw)
+					   const __be16 *nvm_hw)
 {
 	const u8 *hw_addr;
 
@@ -615,7 +629,7 @@ static void iwl_set_hw_address_family_8000(struct iwl_trans *trans,
 
 static int iwl_set_hw_address(struct iwl_trans *trans,
 			      const struct iwl_cfg *cfg,
-			      struct iwl_nvm_data *data, const __le16 *nvm_hw,
+			      struct iwl_nvm_data *data, const __be16 *nvm_hw,
 			      const __le16 *mac_override)
 {
 	if (cfg->mac_addr_from_csr) {
@@ -645,9 +659,41 @@ static int iwl_set_hw_address(struct iwl_trans *trans,
 	return 0;
 }
 
+static bool
+iwl_nvm_no_wide_in_5ghz(struct device *dev, const struct iwl_cfg *cfg,
+			const __be16 *nvm_hw)
+{
+	/*
+	 * Workaround a bug in Indonesia SKUs where the regulatory in
+	 * some 7000-family OTPs erroneously allow wide channels in
+	 * 5GHz.  To check for Indonesia, we take the SKU value from
+	 * bits 1-4 in the subsystem ID and check if it is either 5 or
+	 * 9.  In those cases, we need to force-disable wide channels
+	 * in 5GHz otherwise the FW will throw a sysassert when we try
+	 * to use them.
+	 */
+	if (cfg->device_family == IWL_DEVICE_FAMILY_7000) {
+		/*
+		 * Unlike the other sections in the NVM, the hw
+		 * section uses big-endian.
+		 */
+		u16 subsystem_id = be16_to_cpup(nvm_hw + SUBSYSTEM_ID);
+		u8 sku = (subsystem_id & 0x1e) >> 1;
+
+		if (sku == 5 || sku == 9) {
+			IWL_DEBUG_EEPROM(dev,
+					 "disabling wide channels in 5GHz (0x%0x %d)\n",
+					 subsystem_id, sku);
+			return true;
+		}
+	}
+
+	return false;
+}
+
 struct iwl_nvm_data *
 iwl_parse_nvm_data(struct iwl_trans *trans, const struct iwl_cfg *cfg,
-		   const __le16 *nvm_hw, const __le16 *nvm_sw,
+		   const __be16 *nvm_hw, const __le16 *nvm_sw,
 		   const __le16 *nvm_calib, const __le16 *regulatory,
 		   const __le16 *mac_override, const __le16 *phy_sku,
 		   u8 tx_chains, u8 rx_chains, bool lar_fw_supported)
@@ -655,6 +701,7 @@ iwl_parse_nvm_data(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 	struct device *dev = trans->dev;
 	struct iwl_nvm_data *data;
 	bool lar_enabled;
+	bool no_wide_in_5ghz = iwl_nvm_no_wide_in_5ghz(dev, cfg, nvm_hw);
 	u32 sku, radio_cfg;
 	u16 lar_config;
 	const __le16 *ch_section;
@@ -725,7 +772,7 @@ iwl_parse_nvm_data(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 	}
 
 	iwl_init_sbands(dev, cfg, data, ch_section, tx_chains, rx_chains,
-			lar_fw_supported && lar_enabled);
+			lar_fw_supported && lar_enabled, no_wide_in_5ghz);
 	data->calib_version = 255;
 
 	return data;
@@ -785,7 +832,8 @@ iwl_parse_nvm_mcc_info(struct device *dev, const struct iwl_cfg *cfg,
 		       int num_of_ch, __le32 *channels, u16 fw_mcc)
 {
 	int ch_idx;
-	u16 ch_flags, prev_ch_flags = 0;
+	u16 ch_flags;
+	u32 reg_rule_flags, prev_reg_rule_flags = 0;
 	const u8 *nvm_chan = cfg->ext_nvm ?
 			     iwl_ext_nvm_channels : iwl_nvm_channels;
 	struct ieee80211_regdomain *regd;
@@ -834,8 +882,11 @@ iwl_parse_nvm_mcc_info(struct device *dev, const struct iwl_cfg *cfg,
 			continue;
 		}
 
+		reg_rule_flags = iwl_nvm_get_regdom_bw_flags(nvm_chan, ch_idx,
+							     ch_flags, cfg);
+
 		/* we can't continue the same rule */
-		if (ch_idx == 0 || prev_ch_flags != ch_flags ||
+		if (ch_idx == 0 || prev_reg_rule_flags != reg_rule_flags ||
 		    center_freq - prev_center_freq > 20) {
 			valid_rules++;
 			new_rule = true;
@@ -854,33 +905,40 @@ iwl_parse_nvm_mcc_info(struct device *dev, const struct iwl_cfg *cfg,
 		rule->power_rule.max_eirp =
 			DBM_TO_MBM(IWL_DEFAULT_MAX_TX_POWER);
 
-		rule->flags = iwl_nvm_get_regdom_bw_flags(nvm_chan, ch_idx,
-							  ch_flags, cfg);
+		rule->flags = reg_rule_flags;
 
 		/* rely on auto-calculation to merge BW of contiguous chans */
 		rule->flags |= NL80211_RRF_AUTO_BW;
 		rule->freq_range.max_bandwidth_khz = 0;
 
-		prev_ch_flags = ch_flags;
 		prev_center_freq = center_freq;
+		prev_reg_rule_flags = reg_rule_flags;
 
 		IWL_DEBUG_DEV(dev, IWL_DL_LAR,
-			      "Ch. %d [%sGHz] %s%s%s%s%s%s%s%s%s(0x%02x): Ad-Hoc %ssupported\n",
+			      "Ch. %d [%sGHz] %s%s%s%s%s%s%s%s%s%s%s%s(0x%02x)\n",
 			      center_freq,
 			      band == NL80211_BAND_5GHZ ? "5.2" : "2.4",
 			      CHECK_AND_PRINT_I(VALID),
+			      CHECK_AND_PRINT_I(IBSS),
 			      CHECK_AND_PRINT_I(ACTIVE),
 			      CHECK_AND_PRINT_I(RADAR),
-			      CHECK_AND_PRINT_I(WIDE),
+			      CHECK_AND_PRINT_I(INDOOR_ONLY),
+			      CHECK_AND_PRINT_I(GO_CONCURRENT),
+			      CHECK_AND_PRINT_I(UNIFORM),
+			      CHECK_AND_PRINT_I(20MHZ),
 			      CHECK_AND_PRINT_I(40MHZ),
 			      CHECK_AND_PRINT_I(80MHZ),
 			      CHECK_AND_PRINT_I(160MHZ),
-			      CHECK_AND_PRINT_I(INDOOR_ONLY),
-			      CHECK_AND_PRINT_I(GO_CONCURRENT),
-			      ch_flags,
+			      CHECK_AND_PRINT_I(DC_HIGH),
+			      ch_flags);
+		IWL_DEBUG_DEV(dev, IWL_DL_LAR,
+			      "Ch. %d [%sGHz] reg_flags 0x%x: %s\n",
+			      center_freq,
+			      band == NL80211_BAND_5GHZ ? "5.2" : "2.4",
+			      reg_rule_flags,
 			      ((ch_flags & NVM_CHANNEL_ACTIVE) &&
 			       !(ch_flags & NVM_CHANNEL_RADAR))
-					 ? "" : "not ");
+					 ? "Ad-Hoc" : "");
 	}
 
 	regd->n_reg_rules = valid_rules;
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h
index 3fd6506a02ab..2d1a24dd8410 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h
@@ -77,7 +77,7 @@
  */
 struct iwl_nvm_data *
 iwl_parse_nvm_data(struct iwl_trans *trans, const struct iwl_cfg *cfg,
-		   const __le16 *nvm_hw, const __le16 *nvm_sw,
+		   const __be16 *nvm_hw, const __le16 *nvm_sw,
 		   const __le16 *nvm_calib, const __le16 *regulatory,
 		   const __le16 *mac_override, const __le16 *phy_sku,
 		   u8 tx_chains, u8 rx_chains, bool lar_fw_supported);
@@ -93,7 +93,8 @@ void iwl_set_hw_address_from_csr(struct iwl_trans *trans,
  */
 void iwl_init_sbands(struct device *dev, const struct iwl_cfg *cfg,
 		     struct iwl_nvm_data *data, const __le16 *nvm_ch_flags,
-		     u8 tx_chains, u8 rx_chains, bool lar_supported);
+		     u8 tx_chains, u8 rx_chains, bool lar_supported,
+		     bool no_wide_in_5ghz);
 
 /**
  * iwl_parse_mcc_info - parse MCC (mobile country code) info coming from FW
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-prph.h b/drivers/net/wireless/intel/iwlwifi/iwl-prph.h
index 6772c59b7764..421a869633a3 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-prph.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-prph.h
@@ -109,13 +109,12 @@
 /* Device system time */
 #define DEVICE_SYSTEM_TIME_REG 0xA0206C
 
-/* Device NMI register */
+/* Device NMI register and value for 8000 family and lower hw's */
 #define DEVICE_SET_NMI_REG 0x00a01c30
-#define DEVICE_SET_NMI_VAL_HW BIT(0)
 #define DEVICE_SET_NMI_VAL_DRV BIT(7)
-#define DEVICE_SET_NMI_8000_REG 0x00a01c24
-#define DEVICE_SET_NMI_8000_VAL 0x1000000
+/* Device NMI register and value for 9000 family and above hw's */
 #define UREG_NIC_SET_NMI_DRIVER 0x00a05c10
+#define UREG_NIC_SET_NMI_DRIVER_NMI_FROM_DRIVER_MSK 0xff000000
 
 /* Shared registers (0x0..0x3ff, via target indirect or periphery */
 #define SHR_BASE	0x00a10000
@@ -404,6 +403,12 @@ enum aux_misc_master1_en {
 #define SB_CPU_2_STATUS			0xA01E34
 #define UMAG_SB_CPU_1_STATUS		0xA038C0
 #define UMAG_SB_CPU_2_STATUS		0xA038C4
+#define UMAG_GEN_HW_STATUS		0xA038C8
+
+/* For UMAG_GEN_HW_STATUS reg check */
+enum {
+	UMAG_GEN_HW_IS_FPGA = BIT(1),
+};
 
 /* FW chicken bits */
 #define LMPM_CHICK			0xA01FF8
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h
index eb6842abb4c7..e90abbfba718 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h
@@ -76,7 +76,8 @@
 #include "iwl-config.h"
 #include "fw/img.h"
 #include "iwl-op-mode.h"
-#include "fw/api.h"
+#include "fw/api/cmdhdr.h"
+#include "fw/api/txq.h"
 
 /**
  * DOC: Transport layer - what is it ?
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/Makefile b/drivers/net/wireless/intel/iwlwifi/mvm/Makefile
index 83ac807e547d..00e6737dda72 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/Makefile
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/Makefile
@@ -6,7 +6,7 @@ iwlmvm-y += power.o coex.o
 iwlmvm-y += tt.o offloading.o tdls.o
 iwlmvm-$(CONFIG_IWLWIFI_DEBUGFS) += debugfs.o debugfs-vif.o
 iwlmvm-$(CONFIG_IWLWIFI_LEDS) += led.o
-iwlmvm-y += tof.o fw-dbg.o
+iwlmvm-y += tof.o
 iwlmvm-$(CONFIG_PM) += d3.o
 
 ccflags-y += -I$(src)/../
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/coex.c b/drivers/net/wireless/intel/iwlwifi/mvm/coex.c
index 34dd5c40ce77..79c80f181f7d 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/coex.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/coex.c
@@ -7,6 +7,7 @@
  *
  * Copyright(c) 2013 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2017        Intel Deutschland GmbH
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -33,6 +34,7 @@
  *
  * Copyright(c) 2013 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2017        Intel Deutschland GmbH
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -67,7 +69,7 @@
 #include <linux/etherdevice.h>
 #include <net/mac80211.h>
 
-#include "fw-api-coex.h"
+#include "fw/api/coex.h"
 #include "iwl-modparams.h"
 #include "mvm.h"
 #include "iwl-debug.h"
@@ -148,215 +150,6 @@ static const __le64 iwl_ci_mask[][3] = {
 	},
 };
 
-struct corunning_block_luts {
-	u8 range;
-	__le32 lut20[BT_COEX_CORUN_LUT_SIZE];
-};
-
-/*
- * Ranges for the antenna coupling calibration / co-running block LUT:
- *		LUT0: [ 0, 12[
- *		LUT1: [12, 20[
- *		LUT2: [20, 21[
- *		LUT3: [21, 23[
- *		LUT4: [23, 27[
- *		LUT5: [27, 30[
- *		LUT6: [30, 32[
- *		LUT7: [32, 33[
- *		LUT8: [33, - [
- */
-static const struct corunning_block_luts antenna_coupling_ranges[] = {
-	{
-		.range = 0,
-		.lut20 = {
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-		},
-	},
-	{
-		.range = 12,
-		.lut20 = {
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-		},
-	},
-	{
-		.range = 20,
-		.lut20 = {
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-		},
-	},
-	{
-		.range = 21,
-		.lut20 = {
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-		},
-	},
-	{
-		.range = 23,
-		.lut20 = {
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-		},
-	},
-	{
-		.range = 27,
-		.lut20 = {
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-		},
-	},
-	{
-		.range = 30,
-		.lut20 = {
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-		},
-	},
-	{
-		.range = 32,
-		.lut20 = {
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-		},
-	},
-	{
-		.range = 33,
-		.lut20 = {
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-			cpu_to_le32(0x00000000),  cpu_to_le32(0x00000000),
-		},
-	},
-};
-
 static enum iwl_bt_coex_lut_type
 iwl_get_coex_type(struct iwl_mvm *mvm, const struct ieee80211_vif *vif)
 {
@@ -437,9 +230,6 @@ int iwl_mvm_send_bt_init_conf(struct iwl_mvm *mvm)
 		bt_cmd.enabled_modules |=
 			cpu_to_le32(BT_COEX_SYNC2SCO_ENABLED);
 
-	if (iwl_mvm_bt_is_plcr_supported(mvm))
-		bt_cmd.enabled_modules |= cpu_to_le32(BT_COEX_CORUN_ENABLED);
-
 	if (iwl_mvm_is_mplut_supported(mvm))
 		bt_cmd.enabled_modules |= cpu_to_le32(BT_COEX_MPLUT_ENABLED);
 
@@ -560,8 +350,7 @@ static void iwl_mvm_bt_notif_iterator(void *_data, u8 *mac,
 		smps_mode = IEEE80211_SMPS_AUTOMATIC;
 
 	if (mvmvif->phy_ctxt &&
-	    IWL_COEX_IS_RRC_ON(mvm->last_bt_notif.ttc_rrc_status,
-			       mvmvif->phy_ctxt->id))
+	    (mvm->last_bt_notif.rrc_status & BIT(mvmvif->phy_ctxt->id)))
 		smps_mode = IEEE80211_SMPS_AUTOMATIC;
 
 	IWL_DEBUG_COEX(data->mvm,
@@ -725,17 +514,36 @@ void iwl_mvm_rx_bt_coex_notif(struct iwl_mvm *mvm,
 	struct iwl_rx_packet *pkt = rxb_addr(rxb);
 	struct iwl_bt_coex_profile_notif *notif = (void *)pkt->data;
 
+	if (!iwl_mvm_has_new_ats_coex_api(mvm)) {
+		struct iwl_bt_coex_profile_notif_v4 *v4 = (void *)pkt->data;
+
+		mvm->last_bt_notif.mbox_msg[0] = v4->mbox_msg[0];
+		mvm->last_bt_notif.mbox_msg[1] = v4->mbox_msg[1];
+		mvm->last_bt_notif.mbox_msg[2] = v4->mbox_msg[2];
+		mvm->last_bt_notif.mbox_msg[3] = v4->mbox_msg[3];
+		mvm->last_bt_notif.msg_idx = v4->msg_idx;
+		mvm->last_bt_notif.bt_ci_compliance = v4->bt_ci_compliance;
+		mvm->last_bt_notif.primary_ch_lut = v4->primary_ch_lut;
+		mvm->last_bt_notif.secondary_ch_lut = v4->secondary_ch_lut;
+		mvm->last_bt_notif.bt_activity_grading =
+			v4->bt_activity_grading;
+		mvm->last_bt_notif.ttc_status = v4->ttc_status;
+		mvm->last_bt_notif.rrc_status = v4->rrc_status;
+	} else {
+		/* save this notification for future use: rssi fluctuations */
+		memcpy(&mvm->last_bt_notif, notif, sizeof(mvm->last_bt_notif));
+	}
+
 	IWL_DEBUG_COEX(mvm, "BT Coex Notification received\n");
-	IWL_DEBUG_COEX(mvm, "\tBT ci compliance %d\n", notif->bt_ci_compliance);
+	IWL_DEBUG_COEX(mvm, "\tBT ci compliance %d\n",
+		       mvm->last_bt_notif.bt_ci_compliance);
 	IWL_DEBUG_COEX(mvm, "\tBT primary_ch_lut %d\n",
-		       le32_to_cpu(notif->primary_ch_lut));
+		       le32_to_cpu(mvm->last_bt_notif.primary_ch_lut));
 	IWL_DEBUG_COEX(mvm, "\tBT secondary_ch_lut %d\n",
-		       le32_to_cpu(notif->secondary_ch_lut));
+		       le32_to_cpu(mvm->last_bt_notif.secondary_ch_lut));
 	IWL_DEBUG_COEX(mvm, "\tBT activity grading %d\n",
-		       le32_to_cpu(notif->bt_activity_grading));
+		       le32_to_cpu(mvm->last_bt_notif.bt_activity_grading));
 
-	/* remember this notification for future use: rssi fluctuations */
-	memcpy(&mvm->last_bt_notif, notif, sizeof(mvm->last_bt_notif));
 
 	iwl_mvm_bt_coex_notif_handle(mvm);
 }
@@ -792,7 +600,7 @@ u16 iwl_mvm_coex_agg_time_limit(struct iwl_mvm *mvm,
 	struct iwl_mvm_phy_ctxt *phy_ctxt = mvmvif->phy_ctxt;
 	enum iwl_bt_coex_lut_type lut_type;
 
-	if (IWL_COEX_IS_TTC_ON(mvm->last_bt_notif.ttc_rrc_status, phy_ctxt->id))
+	if (mvm->last_bt_notif.ttc_status & BIT(phy_ctxt->id))
 		return LINK_QUAL_AGG_TIME_LIMIT_DEF;
 
 	if (le32_to_cpu(mvm->last_bt_notif.bt_activity_grading) <
@@ -816,7 +624,7 @@ bool iwl_mvm_bt_coex_is_mimo_allowed(struct iwl_mvm *mvm,
 	struct iwl_mvm_phy_ctxt *phy_ctxt = mvmvif->phy_ctxt;
 	enum iwl_bt_coex_lut_type lut_type;
 
-	if (IWL_COEX_IS_TTC_ON(mvm->last_bt_notif.ttc_rrc_status, phy_ctxt->id))
+	if (mvm->last_bt_notif.ttc_status & BIT(phy_ctxt->id))
 		return true;
 
 	if (le32_to_cpu(mvm->last_bt_notif.bt_activity_grading) <
@@ -909,59 +717,3 @@ void iwl_mvm_bt_coex_vif_change(struct iwl_mvm *mvm)
 {
 	iwl_mvm_bt_coex_notif_handle(mvm);
 }
-
-void iwl_mvm_rx_ant_coupling_notif(struct iwl_mvm *mvm,
-				   struct iwl_rx_cmd_buffer *rxb)
-{
-	struct iwl_rx_packet *pkt = rxb_addr(rxb);
-	struct iwl_mvm_antenna_coupling_notif *notif = (void *)pkt->data;
-	u32 ant_isolation = le32_to_cpu(notif->isolation);
-	struct iwl_bt_coex_corun_lut_update_cmd cmd = {};
-	u8 __maybe_unused lower_bound, upper_bound;
-	u8 lut;
-
-	if (!iwl_mvm_bt_is_plcr_supported(mvm))
-		return;
-
-	lockdep_assert_held(&mvm->mutex);
-
-	/* Ignore updates if we are in force mode */
-	if (unlikely(mvm->bt_force_ant_mode != BT_FORCE_ANT_DIS))
-		return;
-
-	if (ant_isolation ==  mvm->last_ant_isol)
-		return;
-
-	for (lut = 0; lut < ARRAY_SIZE(antenna_coupling_ranges) - 1; lut++)
-		if (ant_isolation < antenna_coupling_ranges[lut + 1].range)
-			break;
-
-	lower_bound = antenna_coupling_ranges[lut].range;
-
-	if (lut < ARRAY_SIZE(antenna_coupling_ranges) - 1)
-		upper_bound = antenna_coupling_ranges[lut + 1].range;
-	else
-		upper_bound = antenna_coupling_ranges[lut].range;
-
-	IWL_DEBUG_COEX(mvm, "Antenna isolation=%d in range [%d,%d[, lut=%d\n",
-		       ant_isolation, lower_bound, upper_bound, lut);
-
-	mvm->last_ant_isol = ant_isolation;
-
-	if (mvm->last_corun_lut == lut)
-		return;
-
-	mvm->last_corun_lut = lut;
-
-	/* For the moment, use the same LUT for 20GHz and 40GHz */
-	memcpy(&cmd.corun_lut20, antenna_coupling_ranges[lut].lut20,
-	       sizeof(cmd.corun_lut20));
-
-	memcpy(&cmd.corun_lut40, antenna_coupling_ranges[lut].lut20,
-	       sizeof(cmd.corun_lut40));
-
-	if (iwl_mvm_send_cmd_pdu(mvm, BT_COEX_UPDATE_CORUN_LUT, 0,
-				 sizeof(cmd), &cmd))
-		IWL_ERR(mvm,
-			"failed to send BT_COEX_UPDATE_CORUN_LUT command\n");
-}
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/constants.h b/drivers/net/wireless/intel/iwlwifi/mvm/constants.h
index 6fda8627b726..976640fed334 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/constants.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/constants.h
@@ -95,7 +95,6 @@
 #define IWL_MVM_BT_COEX_EN_RED_TXP_THRESH	62
 #define IWL_MVM_BT_COEX_DIS_RED_TXP_THRESH	65
 #define IWL_MVM_BT_COEX_SYNC2SCO		1
-#define IWL_MVM_BT_COEX_CORUNNING		0
 #define IWL_MVM_BT_COEX_MPLUT			1
 #define IWL_MVM_BT_COEX_RRC			1
 #define IWL_MVM_BT_COEX_TTC			1
@@ -111,7 +110,6 @@
 #define IWL_MVM_SW_TX_CSUM_OFFLOAD		0
 #define IWL_MVM_HW_CSUM_DISABLE			0
 #define IWL_MVM_PARSE_NVM			0
-#define IWL_MVM_COLLECT_FW_ERR_DUMP		1
 #define IWL_MVM_RS_NUM_TRY_BEFORE_ANT_TOGGLE    1
 #define IWL_MVM_RS_HT_VHT_RETRIES_PER_RATE      2
 #define IWL_MVM_RS_HT_VHT_RETRIES_PER_RATE_TW   1
@@ -138,8 +136,10 @@
 #define IWL_MVM_RS_SR_NO_DECREASE		85	/* percent */
 #define IWL_MVM_RS_AGG_TIME_LIMIT	        4000    /* 4 msecs. valid 100-8000 */
 #define IWL_MVM_RS_AGG_DISABLE_START	        3
+#define IWL_MVM_RS_AGG_START_THRESHOLD	        10	/* num frames per second */
 #define IWL_MVM_RS_TPC_SR_FORCE_INCREASE	75	/* percent */
 #define IWL_MVM_RS_TPC_SR_NO_INCREASE		85	/* percent */
 #define IWL_MVM_RS_TPC_TX_POWER_STEP		3
+#define IWL_MVM_ENABLE_EBS			1
 
 #endif /* __MVM_CONSTANTS_H */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs-vif.c b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs-vif.c
index a7ac281e5cde..71a01df96f8b 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs-vif.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs-vif.c
@@ -65,7 +65,7 @@
  *
  *****************************************************************************/
 #include "mvm.h"
-#include "fw-api-tof.h"
+#include "fw/api/tof.h"
 #include "debugfs.h"
 
 static void iwl_dbgfs_update_pm(struct iwl_mvm *mvm,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
index c1c9c489edc9..e97904c2c4d4 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
@@ -69,7 +69,6 @@
 #include <linux/netdevice.h>
 
 #include "mvm.h"
-#include "fw-dbg.h"
 #include "sta.h"
 #include "iwl-io.h"
 #include "debugfs.h"
@@ -83,8 +82,11 @@ static ssize_t iwl_dbgfs_ctdp_budget_read(struct file *file,
 	char buf[16];
 	int pos, budget;
 
+	if (!iwl_mvm_is_ctdp_supported(mvm))
+		return -EOPNOTSUPP;
+
 	if (!iwl_mvm_firmware_running(mvm) ||
-	    mvm->cur_ucode != IWL_UCODE_REGULAR)
+	    mvm->fwrt.cur_fw_img != IWL_UCODE_REGULAR)
 		return -EIO;
 
 	mutex_lock(&mvm->mutex);
@@ -104,8 +106,11 @@ static ssize_t iwl_dbgfs_stop_ctdp_write(struct iwl_mvm *mvm, char *buf,
 {
 	int ret;
 
+	if (!iwl_mvm_is_ctdp_supported(mvm))
+		return -EOPNOTSUPP;
+
 	if (!iwl_mvm_firmware_running(mvm) ||
-	    mvm->cur_ucode != IWL_UCODE_REGULAR)
+	    mvm->fwrt.cur_fw_img != IWL_UCODE_REGULAR)
 		return -EIO;
 
 	mutex_lock(&mvm->mutex);
@@ -115,6 +120,18 @@ static ssize_t iwl_dbgfs_stop_ctdp_write(struct iwl_mvm *mvm, char *buf,
 	return ret ?: count;
 }
 
+static ssize_t iwl_dbgfs_force_ctkill_write(struct iwl_mvm *mvm, char *buf,
+					    size_t count, loff_t *ppos)
+{
+	if (!iwl_mvm_firmware_running(mvm) ||
+	    mvm->fwrt.cur_fw_img != IWL_UCODE_REGULAR)
+		return -EIO;
+
+	iwl_mvm_enter_ctkill(mvm);
+
+	return count;
+}
+
 static ssize_t iwl_dbgfs_tx_flush_write(struct iwl_mvm *mvm, char *buf,
 					size_t count, loff_t *ppos)
 {
@@ -122,7 +139,7 @@ static ssize_t iwl_dbgfs_tx_flush_write(struct iwl_mvm *mvm, char *buf,
 	u32 flush_arg;
 
 	if (!iwl_mvm_firmware_running(mvm) ||
-	    mvm->cur_ucode != IWL_UCODE_REGULAR)
+	    mvm->fwrt.cur_fw_img != IWL_UCODE_REGULAR)
 		return -EIO;
 
 	if (kstrtou32(buf, 0, &flush_arg))
@@ -155,7 +172,7 @@ static ssize_t iwl_dbgfs_sta_drain_write(struct iwl_mvm *mvm, char *buf,
 	int sta_id, drain, ret;
 
 	if (!iwl_mvm_firmware_running(mvm) ||
-	    mvm->cur_ucode != IWL_UCODE_REGULAR)
+	    mvm->fwrt.cur_fw_img != IWL_UCODE_REGULAR)
 		return -EIO;
 
 	if (sscanf(buf, "%d %d", &sta_id, &drain) != 2)
@@ -192,7 +209,7 @@ static ssize_t iwl_dbgfs_sram_read(struct file *file, char __user *user_buf,
 		return -EINVAL;
 
 	/* default is to dump the entire data segment */
-	img = &mvm->fw->img[mvm->cur_ucode];
+	img = &mvm->fw->img[mvm->fwrt.cur_fw_img];
 	ofs = img->sec[IWL_UCODE_SECTION_DATA].offset;
 	len = img->sec[IWL_UCODE_SECTION_DATA].len;
 
@@ -224,7 +241,7 @@ static ssize_t iwl_dbgfs_sram_write(struct iwl_mvm *mvm, char *buf,
 	if (!iwl_mvm_firmware_running(mvm))
 		return -EINVAL;
 
-	img = &mvm->fw->img[mvm->cur_ucode];
+	img = &mvm->fw->img[mvm->fwrt.cur_fw_img];
 	img_offset = img->sec[IWL_UCODE_SECTION_DATA].offset;
 	img_len = img->sec[IWL_UCODE_SECTION_DATA].len;
 
@@ -452,20 +469,9 @@ static ssize_t iwl_dbgfs_disable_power_off_write(struct iwl_mvm *mvm, char *buf,
 	return ret ?: count;
 }
 
-#define BT_MBOX_MSG(_notif, _num, _field)				     \
-	((le32_to_cpu((_notif)->mbox_msg[(_num)]) & BT_MBOX##_num##_##_field)\
-	>> BT_MBOX##_num##_##_field##_POS)
-
-
-#define BT_MBOX_PRINT(_num, _field, _end)				    \
-			pos += scnprintf(buf + pos, bufsz - pos,	    \
-					 "\t%s: %d%s",			    \
-					 #_field,			    \
-					 BT_MBOX_MSG(notif, _num, _field),  \
-					 true ? "\n" : ", ");
-
 static
-int iwl_mvm_coex_dump_mbox(struct iwl_bt_coex_profile_notif *notif, char *buf,
+int iwl_mvm_coex_dump_mbox(struct iwl_mvm *mvm,
+			   struct iwl_bt_coex_profile_notif *notif, char *buf,
 			   int pos, int bufsz)
 {
 	pos += scnprintf(buf+pos, bufsz-pos, "MBOX dw0:\n");
@@ -509,6 +515,7 @@ int iwl_mvm_coex_dump_mbox(struct iwl_bt_coex_profile_notif *notif, char *buf,
 	BT_MBOX_PRINT(3, SCO_STATE, false);
 	BT_MBOX_PRINT(3, SNIFF_STATE, false);
 	BT_MBOX_PRINT(3, A2DP_STATE, false);
+	BT_MBOX_PRINT(3, A2DP_SRC, false);
 	BT_MBOX_PRINT(3, ACL_STATE, false);
 	BT_MBOX_PRINT(3, MSTR_STATE, false);
 	BT_MBOX_PRINT(3, OBX_STATE, false);
@@ -518,7 +525,12 @@ int iwl_mvm_coex_dump_mbox(struct iwl_bt_coex_profile_notif *notif, char *buf,
 	BT_MBOX_PRINT(3, INBAND_P, false);
 	BT_MBOX_PRINT(3, MSG_TYPE_2, false);
 	BT_MBOX_PRINT(3, SSN_2, false);
-	BT_MBOX_PRINT(3, UPDATE_REQUEST, true);
+	BT_MBOX_PRINT(3, UPDATE_REQUEST, !iwl_mvm_has_new_ats_coex_api(mvm));
+
+	if (iwl_mvm_has_new_ats_coex_api(mvm)) {
+		BT_MBOX_PRINT(4, ATS_BT_INTERVAL, false);
+		BT_MBOX_PRINT(4, ATS_BT_ACTIVE_MAX_TH, true);
+	}
 
 	return pos;
 }
@@ -537,7 +549,7 @@ static ssize_t iwl_dbgfs_bt_notif_read(struct file *file, char __user *user_buf,
 
 	mutex_lock(&mvm->mutex);
 
-	pos += iwl_mvm_coex_dump_mbox(notif, buf, pos, bufsz);
+	pos += iwl_mvm_coex_dump_mbox(mvm, notif, buf, pos, bufsz);
 
 	pos += scnprintf(buf + pos, bufsz - pos, "bt_ci_compliance = %d\n",
 			 notif->bt_ci_compliance);
@@ -548,20 +560,15 @@ static ssize_t iwl_dbgfs_bt_notif_read(struct file *file, char __user *user_buf,
 	pos += scnprintf(buf + pos,
 			 bufsz - pos, "bt_activity_grading = %d\n",
 			 le32_to_cpu(notif->bt_activity_grading));
-	pos += scnprintf(buf + pos, bufsz - pos,
-			 "antenna isolation = %d CORUN LUT index = %d\n",
-			 mvm->last_ant_isol, mvm->last_corun_lut);
 	pos += scnprintf(buf + pos, bufsz - pos, "bt_rrc = %d\n",
-			 (notif->ttc_rrc_status >> 4) & 0xF);
+			 notif->rrc_status & 0xF);
 	pos += scnprintf(buf + pos, bufsz - pos, "bt_ttc = %d\n",
-			 notif->ttc_rrc_status & 0xF);
+			 notif->ttc_status & 0xF);
 
 	pos += scnprintf(buf + pos, bufsz - pos, "sync_sco = %d\n",
 			 IWL_MVM_BT_COEX_SYNC2SCO);
 	pos += scnprintf(buf + pos, bufsz - pos, "mplut = %d\n",
 			 IWL_MVM_BT_COEX_MPLUT);
-	pos += scnprintf(buf + pos, bufsz - pos, "corunning = %d\n",
-			 IWL_MVM_BT_COEX_CORUNNING);
 
 	mutex_unlock(&mvm->mutex);
 
@@ -1123,7 +1130,7 @@ static ssize_t iwl_dbgfs_fw_dbg_conf_read(struct file *file,
 	int pos = 0;
 
 	mutex_lock(&mvm->mutex);
-	conf = mvm->fw_dbg_conf;
+	conf = mvm->fwrt.dump.conf;
 	mutex_unlock(&mvm->mutex);
 
 	pos += scnprintf(buf + pos, bufsz - pos, "%d\n", conf);
@@ -1190,7 +1197,7 @@ static ssize_t iwl_dbgfs_fw_dbg_conf_write(struct iwl_mvm *mvm,
 		return -EINVAL;
 
 	mutex_lock(&mvm->mutex);
-	ret = iwl_mvm_start_fw_dbg_conf(mvm, conf_id);
+	ret = iwl_fw_start_dbg_conf(&mvm->fwrt, conf_id);
 	mutex_unlock(&mvm->mutex);
 
 	return ret ?: count;
@@ -1211,8 +1218,8 @@ static ssize_t iwl_dbgfs_fw_dbg_collect_write(struct iwl_mvm *mvm,
 	if (count == 0)
 		return 0;
 
-	iwl_mvm_fw_dbg_collect(mvm, FW_DBG_TRIGGER_USER, buf,
-			       (count - 1), NULL);
+	iwl_fw_dbg_collect(&mvm->fwrt, FW_DBG_TRIGGER_USER, buf,
+			   (count - 1), NULL);
 
 	iwl_mvm_unref(mvm, IWL_MVM_REF_PRPH_WRITE);
 
@@ -1642,6 +1649,7 @@ MVM_DEBUGFS_READ_WRITE_FILE_OPS(prph_reg, 64);
 /* Device wide debugfs entries */
 MVM_DEBUGFS_READ_FILE_OPS(ctdp_budget);
 MVM_DEBUGFS_WRITE_FILE_OPS(stop_ctdp, 8);
+MVM_DEBUGFS_WRITE_FILE_OPS(force_ctkill, 8);
 MVM_DEBUGFS_WRITE_FILE_OPS(tx_flush, 16);
 MVM_DEBUGFS_WRITE_FILE_OPS(sta_drain, 8);
 MVM_DEBUGFS_WRITE_FILE_OPS(send_echo_cmd, 8);
@@ -1829,6 +1837,7 @@ int iwl_mvm_dbgfs_register(struct iwl_mvm *mvm, struct dentry *dbgfs_dir)
 	MVM_DEBUGFS_ADD_FILE(nic_temp, dbgfs_dir, S_IRUSR);
 	MVM_DEBUGFS_ADD_FILE(ctdp_budget, dbgfs_dir, S_IRUSR);
 	MVM_DEBUGFS_ADD_FILE(stop_ctdp, dbgfs_dir, S_IWUSR);
+	MVM_DEBUGFS_ADD_FILE(force_ctkill, dbgfs_dir, S_IWUSR);
 	MVM_DEBUGFS_ADD_FILE(stations, dbgfs_dir, S_IRUSR);
 	MVM_DEBUGFS_ADD_FILE(bt_notif, dbgfs_dir, S_IRUSR);
 	MVM_DEBUGFS_ADD_FILE(bt_cmd, dbgfs_dir, S_IRUSR);
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api.h b/drivers/net/wireless/intel/iwlwifi/mvm/fw-api.h
index aad265dcfaf5..e8e74dd558f7 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw-api.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/fw-api.h
@@ -68,2823 +68,33 @@
 #ifndef __fw_api_h__
 #define __fw_api_h__
 
-#include "fw-api-rs.h"
-#include "fw-api-rx.h"
-#include "fw-api-tx.h"
-#include "fw-api-sta.h"
-#include "fw-api-mac.h"
-#include "fw-api-power.h"
-#include "fw-api-d3.h"
-#include "fw-api-coex.h"
-#include "fw-api-scan.h"
-#include "fw-api-stats.h"
-#include "fw-api-tof.h"
-
-/* Tx queue numbers for non-DQA mode */
-enum {
-	IWL_MVM_OFFCHANNEL_QUEUE = 8,
-	IWL_MVM_CMD_QUEUE = 9,
-};
-
-/*
- * DQA queue numbers
- *
- * @IWL_MVM_DQA_CMD_QUEUE: a queue reserved for sending HCMDs to the FW
- * @IWL_MVM_DQA_AUX_QUEUE: a queue reserved for aux frames
- * @IWL_MVM_DQA_P2P_DEVICE_QUEUE: a queue reserved for P2P device frames
- * @IWL_MVM_DQA_GCAST_QUEUE: a queue reserved for P2P GO/SoftAP GCAST frames
- * @IWL_MVM_DQA_BSS_CLIENT_QUEUE: a queue reserved for BSS activity, to ensure
- *	that we are never left without the possibility to connect to an AP.
- * @IWL_MVM_DQA_MIN_MGMT_QUEUE: first TXQ in pool for MGMT and non-QOS frames.
- *	Each MGMT queue is mapped to a single STA
- *	MGMT frames are frames that return true on ieee80211_is_mgmt()
- * @IWL_MVM_DQA_MAX_MGMT_QUEUE: last TXQ in pool for MGMT frames
- * @IWL_MVM_DQA_AP_PROBE_RESP_QUEUE: a queue reserved for P2P GO/SoftAP probe
- *	responses
- * @IWL_MVM_DQA_MIN_DATA_QUEUE: first TXQ in pool for DATA frames.
- *	DATA frames are intended for !ieee80211_is_mgmt() frames, but if
- *	the MGMT TXQ pool is exhausted, mgmt frames can be sent on DATA queues
- *	as well
- * @IWL_MVM_DQA_MAX_DATA_QUEUE: last TXQ in pool for DATA frames
- */
-enum iwl_mvm_dqa_txq {
-	IWL_MVM_DQA_CMD_QUEUE = 0,
-	IWL_MVM_DQA_AUX_QUEUE = 1,
-	IWL_MVM_DQA_P2P_DEVICE_QUEUE = 2,
-	IWL_MVM_DQA_GCAST_QUEUE = 3,
-	IWL_MVM_DQA_BSS_CLIENT_QUEUE = 4,
-	IWL_MVM_DQA_MIN_MGMT_QUEUE = 5,
-	IWL_MVM_DQA_MAX_MGMT_QUEUE = 8,
-	IWL_MVM_DQA_AP_PROBE_RESP_QUEUE = 9,
-	IWL_MVM_DQA_MIN_DATA_QUEUE = 10,
-	IWL_MVM_DQA_MAX_DATA_QUEUE = 31,
-};
-
-enum iwl_mvm_tx_fifo {
-	IWL_MVM_TX_FIFO_BK = 0,
-	IWL_MVM_TX_FIFO_BE,
-	IWL_MVM_TX_FIFO_VI,
-	IWL_MVM_TX_FIFO_VO,
-	IWL_MVM_TX_FIFO_MCAST = 5,
-	IWL_MVM_TX_FIFO_CMD = 7,
-};
-
-
-/**
- * enum iwl_legacy_cmds - legacy group command IDs
- */
-enum iwl_legacy_cmds {
-	/**
-	 * @MVM_ALIVE:
-	 * Alive data from the firmware, as described in
-	 * &struct mvm_alive_resp_v3 or &struct mvm_alive_resp.
-	 */
-	MVM_ALIVE = 0x1,
-
-	/**
-	 * @REPLY_ERROR: Cause an error in the firmware, for testing purposes.
-	 */
-	REPLY_ERROR = 0x2,
-
-	/**
-	 * @ECHO_CMD: Send data to the device to have it returned immediately.
-	 */
-	ECHO_CMD = 0x3,
-
-	/**
-	 * @INIT_COMPLETE_NOTIF: Notification that initialization is complete.
-	 */
-	INIT_COMPLETE_NOTIF = 0x4,
-
-	/**
-	 * @PHY_CONTEXT_CMD:
-	 * Add/modify/remove a PHY context, using &struct iwl_phy_context_cmd.
-	 */
-	PHY_CONTEXT_CMD = 0x8,
-
-	/**
-	 * @DBG_CFG: Debug configuration command.
-	 */
-	DBG_CFG = 0x9,
-
-	/**
-	 * @ANTENNA_COUPLING_NOTIFICATION:
-	 * Antenna coupling data, &struct iwl_mvm_antenna_coupling_notif
-	 */
-	ANTENNA_COUPLING_NOTIFICATION = 0xa,
-
-	/**
-	 * @SCAN_ITERATION_COMPLETE_UMAC:
-	 * Firmware indicates a scan iteration completed, using
-	 * &struct iwl_umac_scan_iter_complete_notif.
-	 */
-	SCAN_ITERATION_COMPLETE_UMAC = 0xb5,
-
-	/**
-	 * @SCAN_CFG_CMD:
-	 * uses &struct iwl_scan_config_v1 or &struct iwl_scan_config
-	 */
-	SCAN_CFG_CMD = 0xc,
-
-	/**
-	 * @SCAN_REQ_UMAC: uses &struct iwl_scan_req_umac
-	 */
-	SCAN_REQ_UMAC = 0xd,
-
-	/**
-	 * @SCAN_ABORT_UMAC: uses &struct iwl_umac_scan_abort
-	 */
-	SCAN_ABORT_UMAC = 0xe,
-
-	/**
-	 * @SCAN_COMPLETE_UMAC: uses &struct iwl_umac_scan_complete
-	 */
-	SCAN_COMPLETE_UMAC = 0xf,
-
-	/**
-	 * @BA_WINDOW_STATUS_NOTIFICATION_ID:
-	 * uses &struct iwl_ba_window_status_notif
-	 */
-	BA_WINDOW_STATUS_NOTIFICATION_ID = 0x13,
-
-	/**
-	 * @ADD_STA_KEY:
-	 * &struct iwl_mvm_add_sta_key_cmd_v1 or
-	 * &struct iwl_mvm_add_sta_key_cmd.
-	 */
-	ADD_STA_KEY = 0x17,
-
-	/**
-	 * @ADD_STA:
-	 * &struct iwl_mvm_add_sta_cmd or &struct iwl_mvm_add_sta_cmd_v7.
-	 */
-	ADD_STA = 0x18,
-
-	/**
-	 * @REMOVE_STA: &struct iwl_mvm_rm_sta_cmd
-	 */
-	REMOVE_STA = 0x19,
-
-	/**
-	 * @FW_GET_ITEM_CMD: uses &struct iwl_fw_get_item_cmd
-	 */
-	FW_GET_ITEM_CMD = 0x1a,
-
-	/**
-	 * @TX_CMD: uses &struct iwl_tx_cmd or &struct iwl_tx_cmd_gen2,
-	 *	response in &struct iwl_mvm_tx_resp or
-	 *	&struct iwl_mvm_tx_resp_v3
-	 */
-	TX_CMD = 0x1c,
-
-	/**
-	 * @TXPATH_FLUSH: &struct iwl_tx_path_flush_cmd
-	 */
-	TXPATH_FLUSH = 0x1e,
-
-	/**
-	 * @MGMT_MCAST_KEY:
-	 * &struct iwl_mvm_mgmt_mcast_key_cmd or
-	 * &struct iwl_mvm_mgmt_mcast_key_cmd_v1
-	 */
-	MGMT_MCAST_KEY = 0x1f,
-
-	/* scheduler config */
-	/**
-	 * @SCD_QUEUE_CFG: &struct iwl_scd_txq_cfg_cmd for older hardware,
-	 *	&struct iwl_tx_queue_cfg_cmd with &struct iwl_tx_queue_cfg_rsp
-	 *	for newer (A000) hardware.
-	 */
-	SCD_QUEUE_CFG = 0x1d,
-
-	/**
-	 * @WEP_KEY: uses &struct iwl_mvm_wep_key_cmd
-	 */
-	WEP_KEY = 0x20,
-
-	/**
-	 * @SHARED_MEM_CFG:
-	 * retrieve shared memory configuration - response in
-	 * &struct iwl_shared_mem_cfg
-	 */
-	SHARED_MEM_CFG = 0x25,
-
-	/**
-	 * @TDLS_CHANNEL_SWITCH_CMD: uses &struct iwl_tdls_channel_switch_cmd
-	 */
-	TDLS_CHANNEL_SWITCH_CMD = 0x27,
-
-	/**
-	 * @TDLS_CHANNEL_SWITCH_NOTIFICATION:
-	 * uses &struct iwl_tdls_channel_switch_notif
-	 */
-	TDLS_CHANNEL_SWITCH_NOTIFICATION = 0xaa,
-
-	/**
-	 * @TDLS_CONFIG_CMD:
-	 * &struct iwl_tdls_config_cmd, response in &struct iwl_tdls_config_res
-	 */
-	TDLS_CONFIG_CMD = 0xa7,
-
-	/**
-	 * @MAC_CONTEXT_CMD: &struct iwl_mac_ctx_cmd
-	 */
-	MAC_CONTEXT_CMD = 0x28,
-
-	/**
-	 * @TIME_EVENT_CMD:
-	 * &struct iwl_time_event_cmd, response in &struct iwl_time_event_resp
-	 */
-	TIME_EVENT_CMD = 0x29, /* both CMD and response */
-
-	/**
-	 * @TIME_EVENT_NOTIFICATION: &struct iwl_time_event_notif
-	 */
-	TIME_EVENT_NOTIFICATION = 0x2a,
-
-	/**
-	 * @BINDING_CONTEXT_CMD:
-	 * &struct iwl_binding_cmd or &struct iwl_binding_cmd_v1
-	 */
-	BINDING_CONTEXT_CMD = 0x2b,
-
-	/**
-	 * @TIME_QUOTA_CMD: &struct iwl_time_quota_cmd
-	 */
-	TIME_QUOTA_CMD = 0x2c,
-
-	/**
-	 * @NON_QOS_TX_COUNTER_CMD:
-	 * command is &struct iwl_nonqos_seq_query_cmd
-	 */
-	NON_QOS_TX_COUNTER_CMD = 0x2d,
-
-	/**
-	 * @LQ_CMD: using &struct iwl_lq_cmd
-	 */
-	LQ_CMD = 0x4e,
-
-	/**
-	 * @FW_PAGING_BLOCK_CMD:
-	 * &struct iwl_fw_paging_cmd
-	 */
-	FW_PAGING_BLOCK_CMD = 0x4f,
-
-	/**
-	 * @SCAN_OFFLOAD_REQUEST_CMD: uses &struct iwl_scan_req_lmac
-	 */
-	SCAN_OFFLOAD_REQUEST_CMD = 0x51,
-
-	/**
-	 * @SCAN_OFFLOAD_ABORT_CMD: abort the scan - no further contents
-	 */
-	SCAN_OFFLOAD_ABORT_CMD = 0x52,
-
-	/**
-	 * @HOT_SPOT_CMD: uses &struct iwl_hs20_roc_req
-	 */
-	HOT_SPOT_CMD = 0x53,
-
-	/**
-	 * @SCAN_OFFLOAD_COMPLETE:
-	 * notification, &struct iwl_periodic_scan_complete
-	 */
-	SCAN_OFFLOAD_COMPLETE = 0x6D,
-
-	/**
-	 * @SCAN_OFFLOAD_UPDATE_PROFILES_CMD:
-	 * update scan offload (scheduled scan) profiles/blacklist/etc.
-	 */
-	SCAN_OFFLOAD_UPDATE_PROFILES_CMD = 0x6E,
-
-	/**
-	 * @MATCH_FOUND_NOTIFICATION: scan match found
-	 */
-	MATCH_FOUND_NOTIFICATION = 0xd9,
-
-	/**
-	 * @SCAN_ITERATION_COMPLETE:
-	 * uses &struct iwl_lmac_scan_complete_notif
-	 */
-	SCAN_ITERATION_COMPLETE = 0xe7,
-
-	/* Phy */
-	/**
-	 * @PHY_CONFIGURATION_CMD: &struct iwl_phy_cfg_cmd
-	 */
-	PHY_CONFIGURATION_CMD = 0x6a,
-
-	/**
-	 * @CALIB_RES_NOTIF_PHY_DB: &struct iwl_calib_res_notif_phy_db
-	 */
-	CALIB_RES_NOTIF_PHY_DB = 0x6b,
-
-	/**
-	 * @PHY_DB_CMD: &struct iwl_phy_db_cmd
-	 */
-	PHY_DB_CMD = 0x6c,
-
-	/**
-	 * @TOF_CMD: &struct iwl_tof_config_cmd
-	 */
-	TOF_CMD = 0x10,
-
-	/**
-	 * @TOF_NOTIFICATION: &struct iwl_tof_gen_resp_cmd
-	 */
-	TOF_NOTIFICATION = 0x11,
-
-	/**
-	 * @POWER_TABLE_CMD: &struct iwl_device_power_cmd
-	 */
-	POWER_TABLE_CMD = 0x77,
-
-	/**
-	 * @PSM_UAPSD_AP_MISBEHAVING_NOTIFICATION:
-	 * &struct iwl_uapsd_misbehaving_ap_notif
-	 */
-	PSM_UAPSD_AP_MISBEHAVING_NOTIFICATION = 0x78,
-
-	/**
-	 * @LTR_CONFIG: &struct iwl_ltr_config_cmd
-	 */
-	LTR_CONFIG = 0xee,
-
-	/**
-	 * @REPLY_THERMAL_MNG_BACKOFF:
-	 * Thermal throttling command
-	 */
-	REPLY_THERMAL_MNG_BACKOFF = 0x7e,
-
-	/**
-	 * @DC2DC_CONFIG_CMD:
-	 * Set/Get DC2DC frequency tune
-	 * Command is &struct iwl_dc2dc_config_cmd,
-	 * response is &struct iwl_dc2dc_config_resp
-	 */
-	DC2DC_CONFIG_CMD = 0x83,
-
-	/**
-	 * @NVM_ACCESS_CMD: using &struct iwl_nvm_access_cmd
-	 */
-	NVM_ACCESS_CMD = 0x88,
-
-	/**
-	 * @BEACON_NOTIFICATION: &struct iwl_extended_beacon_notif
-	 */
-	BEACON_NOTIFICATION = 0x90,
-
-	/**
-	 * @BEACON_TEMPLATE_CMD:
-	 *	Uses one of &struct iwl_mac_beacon_cmd_v6,
-	 *	&struct iwl_mac_beacon_cmd_v7 or &struct iwl_mac_beacon_cmd
-	 *	depending on the device version.
-	 */
-	BEACON_TEMPLATE_CMD = 0x91,
-	/**
-	 * @TX_ANT_CONFIGURATION_CMD: &struct iwl_tx_ant_cfg_cmd
-	 */
-	TX_ANT_CONFIGURATION_CMD = 0x98,
-
-	/**
-	 * @STATISTICS_CMD:
-	 * one of &struct iwl_statistics_cmd,
-	 * &struct iwl_notif_statistics_v11,
-	 * &struct iwl_notif_statistics_v10,
-	 * &struct iwl_notif_statistics_cdb
-	 */
-	STATISTICS_CMD = 0x9c,
-
-	/**
-	 * @STATISTICS_NOTIFICATION:
-	 * one of &struct iwl_notif_statistics_v10,
-	 * &struct iwl_notif_statistics_v11,
-	 * &struct iwl_notif_statistics_cdb
-	 */
-	STATISTICS_NOTIFICATION = 0x9d,
-
-	/**
-	 * @EOSP_NOTIFICATION:
-	 * Notify that a service period ended,
-	 * &struct iwl_mvm_eosp_notification
-	 */
-	EOSP_NOTIFICATION = 0x9e,
-
-	/**
-	 * @REDUCE_TX_POWER_CMD:
-	 * &struct iwl_dev_tx_power_cmd_v3 or &struct iwl_dev_tx_power_cmd
-	 */
-	REDUCE_TX_POWER_CMD = 0x9f,
-
-	/**
-	 * @CARD_STATE_NOTIFICATION:
-	 * Card state (RF/CT kill) notification,
-	 * uses &struct iwl_card_state_notif
-	 */
-	CARD_STATE_NOTIFICATION = 0xa1,
-
-	/**
-	 * @MISSED_BEACONS_NOTIFICATION: &struct iwl_missed_beacons_notif
-	 */
-	MISSED_BEACONS_NOTIFICATION = 0xa2,
-
-	/**
-	 * @MAC_PM_POWER_TABLE: using &struct iwl_mac_power_cmd
-	 */
-	MAC_PM_POWER_TABLE = 0xa9,
-
-	/**
-	 * @MFUART_LOAD_NOTIFICATION: &struct iwl_mfuart_load_notif
-	 */
-	MFUART_LOAD_NOTIFICATION = 0xb1,
-
-	/**
-	 * @RSS_CONFIG_CMD: &struct iwl_rss_config_cmd
-	 */
-	RSS_CONFIG_CMD = 0xb3,
-
-	/**
-	 * @REPLY_RX_PHY_CMD: &struct iwl_rx_phy_info
-	 */
-	REPLY_RX_PHY_CMD = 0xc0,
-
-	/**
-	 * @REPLY_RX_MPDU_CMD:
-	 * &struct iwl_rx_mpdu_res_start or &struct iwl_rx_mpdu_desc
-	 */
-	REPLY_RX_MPDU_CMD = 0xc1,
-
-	/**
-	 * @FRAME_RELEASE:
-	 * Frame release (reorder helper) notification, uses
-	 * &struct iwl_frame_release
-	 */
-	FRAME_RELEASE = 0xc3,
-
-	/**
-	 * @BA_NOTIF:
-	 * BlockAck notification, uses &struct iwl_mvm_compressed_ba_notif
-	 * or &struct iwl_mvm_ba_notif depending on the HW
-	 */
-	BA_NOTIF = 0xc5,
-
-	/* Location Aware Regulatory */
-	/**
-	 * @MCC_UPDATE_CMD: using &struct iwl_mcc_update_cmd
-	 */
-	MCC_UPDATE_CMD = 0xc8,
-
-	/**
-	 * @MCC_CHUB_UPDATE_CMD: using &struct iwl_mcc_chub_notif
-	 */
-	MCC_CHUB_UPDATE_CMD = 0xc9,
-
-	/**
-	 * @MARKER_CMD: trace marker command, uses &struct iwl_mvm_marker
-	 */
-	MARKER_CMD = 0xcb,
-
-	/**
-	 * @BT_PROFILE_NOTIFICATION: &struct iwl_bt_coex_profile_notif
-	 */
-	BT_PROFILE_NOTIFICATION = 0xce,
-
-	/**
-	 * @BT_CONFIG: &struct iwl_bt_coex_cmd
-	 */
-	BT_CONFIG = 0x9b,
-
-	/**
-	 * @BT_COEX_UPDATE_CORUN_LUT:
-	 * &struct iwl_bt_coex_corun_lut_update_cmd
-	 */
-	BT_COEX_UPDATE_CORUN_LUT = 0x5b,
-
-	/**
-	 * @BT_COEX_UPDATE_REDUCED_TXP:
-	 * &struct iwl_bt_coex_reduced_txp_update_cmd
-	 */
-	BT_COEX_UPDATE_REDUCED_TXP = 0x5c,
-
-	/**
-	 * @BT_COEX_CI: &struct iwl_bt_coex_ci_cmd
-	 */
-	BT_COEX_CI = 0x5d,
-
-	/**
-	 * @REPLY_SF_CFG_CMD: &struct iwl_sf_cfg_cmd
-	 */
-	REPLY_SF_CFG_CMD = 0xd1,
-	/**
-	 * @REPLY_BEACON_FILTERING_CMD: &struct iwl_beacon_filter_cmd
-	 */
-	REPLY_BEACON_FILTERING_CMD = 0xd2,
-
-	/**
-	 * @DTS_MEASUREMENT_NOTIFICATION:
-	 * &struct iwl_dts_measurement_notif_v1 or
-	 * &struct iwl_dts_measurement_notif_v2
-	 */
-	DTS_MEASUREMENT_NOTIFICATION = 0xdd,
-
-	/**
-	 * @LDBG_CONFIG_CMD: configure continuous trace recording
-	 */
-	LDBG_CONFIG_CMD = 0xf6,
-
-	/**
-	 * @DEBUG_LOG_MSG: Debugging log data from firmware
-	 */
-	DEBUG_LOG_MSG = 0xf7,
-
-	/**
-	 * @BCAST_FILTER_CMD: &struct iwl_bcast_filter_cmd
-	 */
-	BCAST_FILTER_CMD = 0xcf,
-
-	/**
-	 * @MCAST_FILTER_CMD: &struct iwl_mcast_filter_cmd
-	 */
-	MCAST_FILTER_CMD = 0xd0,
-
-	/**
-	 * @D3_CONFIG_CMD: &struct iwl_d3_manager_config
-	 */
-	D3_CONFIG_CMD = 0xd3,
-
-	/**
-	 * @PROT_OFFLOAD_CONFIG_CMD: Depending on firmware, uses one of
-	 * &struct iwl_proto_offload_cmd_v1, &struct iwl_proto_offload_cmd_v2,
-	 * &struct iwl_proto_offload_cmd_v3_small,
-	 * &struct iwl_proto_offload_cmd_v3_large
-	 */
-	PROT_OFFLOAD_CONFIG_CMD = 0xd4,
-
-	/**
-	 * @OFFLOADS_QUERY_CMD:
-	 * No data in command, response in &struct iwl_wowlan_status
-	 */
-	OFFLOADS_QUERY_CMD = 0xd5,
-
-	/**
-	 * @REMOTE_WAKE_CONFIG_CMD: &struct iwl_wowlan_remote_wake_config
-	 */
-	REMOTE_WAKE_CONFIG_CMD = 0xd6,
-
-	/**
-	 * @D0I3_END_CMD: End D0i3/D3 state, no command data
-	 */
-	D0I3_END_CMD = 0xed,
-
-	/**
-	 * @WOWLAN_PATTERNS: &struct iwl_wowlan_patterns_cmd
-	 */
-	WOWLAN_PATTERNS = 0xe0,
-
-	/**
-	 * @WOWLAN_CONFIGURATION: &struct iwl_wowlan_config_cmd
-	 */
-	WOWLAN_CONFIGURATION = 0xe1,
-
-	/**
-	 * @WOWLAN_TSC_RSC_PARAM: &struct iwl_wowlan_rsc_tsc_params_cmd
-	 */
-	WOWLAN_TSC_RSC_PARAM = 0xe2,
-
-	/**
-	 * @WOWLAN_TKIP_PARAM: &struct iwl_wowlan_tkip_params_cmd
-	 */
-	WOWLAN_TKIP_PARAM = 0xe3,
-
-	/**
-	 * @WOWLAN_KEK_KCK_MATERIAL: &struct iwl_wowlan_kek_kck_material_cmd
-	 */
-	WOWLAN_KEK_KCK_MATERIAL = 0xe4,
-
-	/**
-	 * @WOWLAN_GET_STATUSES: response in &struct iwl_wowlan_status
-	 */
-	WOWLAN_GET_STATUSES = 0xe5,
-
-	/**
-	 * @SCAN_OFFLOAD_PROFILES_QUERY_CMD:
-	 * No command data, response is &struct iwl_scan_offload_profiles_query
-	 */
-	SCAN_OFFLOAD_PROFILES_QUERY_CMD = 0x56,
-};
-
-/* Please keep this enum *SORTED* by hex value.
- * Needed for binary search, otherwise a warning will be triggered.
- */
-enum iwl_mac_conf_subcmd_ids {
-	LINK_QUALITY_MEASUREMENT_CMD = 0x1,
-	LINK_QUALITY_MEASUREMENT_COMPLETE_NOTIF = 0xFE,
-	CHANNEL_SWITCH_NOA_NOTIF = 0xFF,
-};
-
-/**
- * enum iwl_phy_ops_subcmd_ids - PHY group commands
- */
-enum iwl_phy_ops_subcmd_ids {
-	/**
-	 * @CMD_DTS_MEASUREMENT_TRIGGER_WIDE:
-	 * Uses either &struct iwl_dts_measurement_cmd or
-	 * &struct iwl_ext_dts_measurement_cmd
-	 */
-	CMD_DTS_MEASUREMENT_TRIGGER_WIDE = 0x0,
-
-	/**
-	 * @CTDP_CONFIG_CMD: &struct iwl_mvm_ctdp_cmd
-	 */
-	CTDP_CONFIG_CMD = 0x03,
-
-	/**
-	 * @TEMP_REPORTING_THRESHOLDS_CMD: &struct temp_report_ths_cmd
-	 */
-	TEMP_REPORTING_THRESHOLDS_CMD = 0x04,
-
-	/**
-	 * @GEO_TX_POWER_LIMIT: &struct iwl_geo_tx_power_profiles_cmd
-	 */
-	GEO_TX_POWER_LIMIT = 0x05,
-
-	/**
-	 * @CT_KILL_NOTIFICATION: &struct ct_kill_notif
-	 */
-	CT_KILL_NOTIFICATION = 0xFE,
-
-	/**
-	 * @DTS_MEASUREMENT_NOTIF_WIDE:
-	 * &struct iwl_dts_measurement_notif_v1 or
-	 * &struct iwl_dts_measurement_notif_v2
-	 */
-	DTS_MEASUREMENT_NOTIF_WIDE = 0xFF,
-};
-
-/**
- * enum iwl_system_subcmd_ids - system group command IDs
- */
-enum iwl_system_subcmd_ids {
-	/**
-	 * @SHARED_MEM_CFG_CMD:
-	 * response in &struct iwl_shared_mem_cfg or
-	 * &struct iwl_shared_mem_cfg_v2
-	 */
-	SHARED_MEM_CFG_CMD = 0x0,
-
-	/**
-	 * @INIT_EXTENDED_CFG_CMD: &struct iwl_init_extended_cfg_cmd
-	 */
-	INIT_EXTENDED_CFG_CMD = 0x03,
-};
-
-/**
- * enum iwl_data_path_subcmd_ids - data path group commands
- */
-enum iwl_data_path_subcmd_ids {
-	/**
-	 * @DQA_ENABLE_CMD: &struct iwl_dqa_enable_cmd
-	 */
-	DQA_ENABLE_CMD = 0x0,
-
-	/**
-	 * @UPDATE_MU_GROUPS_CMD: &struct iwl_mu_group_mgmt_cmd
-	 */
-	UPDATE_MU_GROUPS_CMD = 0x1,
-
-	/**
-	 * @TRIGGER_RX_QUEUES_NOTIF_CMD: &struct iwl_rxq_sync_cmd
-	 */
-	TRIGGER_RX_QUEUES_NOTIF_CMD = 0x2,
-
-	/**
-	 * @STA_PM_NOTIF: &struct iwl_mvm_pm_state_notification
-	 */
-	STA_PM_NOTIF = 0xFD,
-
-	/**
-	 * @MU_GROUP_MGMT_NOTIF: &struct iwl_mu_group_mgmt_notif
-	 */
-	MU_GROUP_MGMT_NOTIF = 0xFE,
-
-	/**
-	 * @RX_QUEUES_NOTIFICATION: &struct iwl_rxq_sync_notification
-	 */
-	RX_QUEUES_NOTIFICATION = 0xFF,
-};
-
-/**
- * enum iwl_prot_offload_subcmd_ids - protocol offload commands
- */
-enum iwl_prot_offload_subcmd_ids {
-	/**
-	 * @STORED_BEACON_NTF: &struct iwl_stored_beacon_notif
-	 */
-	STORED_BEACON_NTF = 0xFF,
-};
-
-/**
- * enum iwl_regulatory_and_nvm_subcmd_ids - regulatory/NVM commands
- */
-enum iwl_regulatory_and_nvm_subcmd_ids {
-	/**
-	 * @NVM_ACCESS_COMPLETE: &struct iwl_nvm_access_complete_cmd
-	 */
-	NVM_ACCESS_COMPLETE = 0x0,
-
-	/**
-	 * @NVM_GET_INFO:
-	 * Command is &struct iwl_nvm_get_info,
-	 * response is &struct iwl_nvm_get_info_rsp
-	 */
-	NVM_GET_INFO = 0x2,
-};
-
-/**
- * enum iwl_debug_cmds - debug commands
- */
-enum iwl_debug_cmds {
-	/**
-	 * @LMAC_RD_WR:
-	 * LMAC memory read/write, using &struct iwl_dbg_mem_access_cmd and
-	 * &struct iwl_dbg_mem_access_rsp
-	 */
-	LMAC_RD_WR = 0x0,
-	/**
-	 * @UMAC_RD_WR:
-	 * UMAC memory read/write, using &struct iwl_dbg_mem_access_cmd and
-	 * &struct iwl_dbg_mem_access_rsp
-	 */
-	UMAC_RD_WR = 0x1,
-	/**
-	 * @MFU_ASSERT_DUMP_NTF:
-	 * &struct iwl_mfu_assert_dump_notif
-	 */
-	MFU_ASSERT_DUMP_NTF = 0xFE,
-};
-
-/**
- * enum iwl_mvm_command_groups - command groups for the firmware
- * @LEGACY_GROUP: legacy group, uses command IDs from &enum iwl_legacy_cmds
- * @LONG_GROUP: legacy group with long header, also uses command IDs
- *	from &enum iwl_legacy_cmds
- * @SYSTEM_GROUP: system group, uses command IDs from
- *	&enum iwl_system_subcmd_ids
- * @MAC_CONF_GROUP: MAC configuration group, uses command IDs from
- *	&enum iwl_mac_conf_subcmd_ids
- * @PHY_OPS_GROUP: PHY operations group, uses command IDs from
- *	&enum iwl_phy_ops_subcmd_ids
- * @DATA_PATH_GROUP: data path group, uses command IDs from
- *	&enum iwl_data_path_subcmd_ids
- * @NAN_GROUP: NAN group, uses command IDs from &enum iwl_nan_subcmd_ids
- * @TOF_GROUP: TOF group, uses command IDs from &enum iwl_tof_subcmd_ids
- * @PROT_OFFLOAD_GROUP: protocol offload group, uses command IDs from
- *	&enum iwl_prot_offload_subcmd_ids
- * @REGULATORY_AND_NVM_GROUP: regulatory/NVM group, uses command IDs from
- *	&enum iwl_regulatory_and_nvm_subcmd_ids
- * @DEBUG_GROUP: Debug group, uses command IDs from &enum iwl_debug_cmds
- */
-enum iwl_mvm_command_groups {
-	LEGACY_GROUP = 0x0,
-	LONG_GROUP = 0x1,
-	SYSTEM_GROUP = 0x2,
-	MAC_CONF_GROUP = 0x3,
-	PHY_OPS_GROUP = 0x4,
-	DATA_PATH_GROUP = 0x5,
-	PROT_OFFLOAD_GROUP = 0xb,
-	REGULATORY_AND_NVM_GROUP = 0xc,
-	DEBUG_GROUP = 0xf,
-};
-
-/**
- * struct iwl_cmd_response - generic response struct for most commands
- * @status: status of the command asked, changes for each one
- */
-struct iwl_cmd_response {
-	__le32 status;
-};
-
-/*
- * struct iwl_dqa_enable_cmd
- * @cmd_queue: the TXQ number of the command queue
- */
-struct iwl_dqa_enable_cmd {
-	__le32 cmd_queue;
-} __packed; /* DQA_CONTROL_CMD_API_S_VER_1 */
-
-/*
- * struct iwl_tx_ant_cfg_cmd
- * @valid: valid antenna configuration
- */
-struct iwl_tx_ant_cfg_cmd {
-	__le32 valid;
-} __packed;
-
-/**
- * struct iwl_calib_ctrl - Calibration control struct.
- * Sent as part of the phy configuration command.
- * @flow_trigger: bitmap for which calibrations to perform according to
- *		flow triggers, using &enum iwl_calib_cfg
- * @event_trigger: bitmap for which calibrations to perform according to
- *		event triggers, using &enum iwl_calib_cfg
- */
-struct iwl_calib_ctrl {
-	__le32 flow_trigger;
-	__le32 event_trigger;
-} __packed;
-
-/* This enum defines the bitmap of various calibrations to enable in both
- * init ucode and runtime ucode through CALIBRATION_CFG_CMD.
- */
-enum iwl_calib_cfg {
-	IWL_CALIB_CFG_XTAL_IDX			= BIT(0),
-	IWL_CALIB_CFG_TEMPERATURE_IDX		= BIT(1),
-	IWL_CALIB_CFG_VOLTAGE_READ_IDX		= BIT(2),
-	IWL_CALIB_CFG_PAPD_IDX			= BIT(3),
-	IWL_CALIB_CFG_TX_PWR_IDX		= BIT(4),
-	IWL_CALIB_CFG_DC_IDX			= BIT(5),
-	IWL_CALIB_CFG_BB_FILTER_IDX		= BIT(6),
-	IWL_CALIB_CFG_LO_LEAKAGE_IDX		= BIT(7),
-	IWL_CALIB_CFG_TX_IQ_IDX			= BIT(8),
-	IWL_CALIB_CFG_TX_IQ_SKEW_IDX		= BIT(9),
-	IWL_CALIB_CFG_RX_IQ_IDX			= BIT(10),
-	IWL_CALIB_CFG_RX_IQ_SKEW_IDX		= BIT(11),
-	IWL_CALIB_CFG_SENSITIVITY_IDX		= BIT(12),
-	IWL_CALIB_CFG_CHAIN_NOISE_IDX		= BIT(13),
-	IWL_CALIB_CFG_DISCONNECTED_ANT_IDX	= BIT(14),
-	IWL_CALIB_CFG_ANT_COUPLING_IDX		= BIT(15),
-	IWL_CALIB_CFG_DAC_IDX			= BIT(16),
-	IWL_CALIB_CFG_ABS_IDX			= BIT(17),
-	IWL_CALIB_CFG_AGC_IDX			= BIT(18),
-};
-
-/**
- * struct iwl_phy_cfg_cmd - Phy configuration command
- * @phy_cfg: PHY configuration value, uses &enum iwl_fw_phy_cfg
- * @calib_control: calibration control data
- */
-struct iwl_phy_cfg_cmd {
-	__le32	phy_cfg;
-	struct iwl_calib_ctrl calib_control;
-} __packed;
-
-#define PHY_CFG_RADIO_TYPE	(BIT(0) | BIT(1))
-#define PHY_CFG_RADIO_STEP	(BIT(2) | BIT(3))
-#define PHY_CFG_RADIO_DASH	(BIT(4) | BIT(5))
-#define PHY_CFG_PRODUCT_NUMBER	(BIT(6) | BIT(7))
-#define PHY_CFG_TX_CHAIN_A	BIT(8)
-#define PHY_CFG_TX_CHAIN_B	BIT(9)
-#define PHY_CFG_TX_CHAIN_C	BIT(10)
-#define PHY_CFG_RX_CHAIN_A	BIT(12)
-#define PHY_CFG_RX_CHAIN_B	BIT(13)
-#define PHY_CFG_RX_CHAIN_C	BIT(14)
-
-
-/**
- * enum iwl_nvm_access_op - NVM access opcode
- * @IWL_NVM_READ: read NVM
- * @IWL_NVM_WRITE: write NVM
- */
-enum iwl_nvm_access_op {
-	IWL_NVM_READ	= 0,
-	IWL_NVM_WRITE	= 1,
-};
-
-/**
- * enum iwl_nvm_access_target - target of the NVM_ACCESS_CMD
- * @NVM_ACCESS_TARGET_CACHE: access the cache
- * @NVM_ACCESS_TARGET_OTP: access the OTP
- * @NVM_ACCESS_TARGET_EEPROM: access the EEPROM
- */
-enum iwl_nvm_access_target {
-	NVM_ACCESS_TARGET_CACHE = 0,
-	NVM_ACCESS_TARGET_OTP = 1,
-	NVM_ACCESS_TARGET_EEPROM = 2,
-};
-
-/**
- * enum iwl_nvm_section_type - section types for NVM_ACCESS_CMD
- * @NVM_SECTION_TYPE_SW: software section
- * @NVM_SECTION_TYPE_REGULATORY: regulatory section
- * @NVM_SECTION_TYPE_CALIBRATION: calibration section
- * @NVM_SECTION_TYPE_PRODUCTION: production section
- * @NVM_SECTION_TYPE_MAC_OVERRIDE: MAC override section
- * @NVM_SECTION_TYPE_PHY_SKU: PHY SKU section
- * @NVM_MAX_NUM_SECTIONS: number of sections
- */
-enum iwl_nvm_section_type {
-	NVM_SECTION_TYPE_SW = 1,
-	NVM_SECTION_TYPE_REGULATORY = 3,
-	NVM_SECTION_TYPE_CALIBRATION = 4,
-	NVM_SECTION_TYPE_PRODUCTION = 5,
-	NVM_SECTION_TYPE_MAC_OVERRIDE = 11,
-	NVM_SECTION_TYPE_PHY_SKU = 12,
-	NVM_MAX_NUM_SECTIONS = 13,
-};
-
-/**
- * struct iwl_nvm_access_cmd - Request the device to send an NVM section
- * @op_code: &enum iwl_nvm_access_op
- * @target: &enum iwl_nvm_access_target
- * @type: &enum iwl_nvm_section_type
- * @offset: offset in bytes into the section
- * @length: in bytes, to read/write
- * @data: if write operation, the data to write. On read its empty
- */
-struct iwl_nvm_access_cmd {
-	u8 op_code;
-	u8 target;
-	__le16 type;
-	__le16 offset;
-	__le16 length;
-	u8 data[];
-} __packed; /* NVM_ACCESS_CMD_API_S_VER_2 */
-
-#define NUM_OF_FW_PAGING_BLOCKS	33 /* 32 for data and 1 block for CSS */
-
-/**
- * struct iwl_fw_paging_cmd - paging layout
- *
- * (FW_PAGING_BLOCK_CMD = 0x4f)
- *
- * Send to FW the paging layout in the driver.
- *
- * @flags: various flags for the command
- * @block_size: the block size in powers of 2
- * @block_num: number of blocks specified in the command.
- * @device_phy_addr: virtual addresses from device side
- */
-struct iwl_fw_paging_cmd {
-	__le32 flags;
-	__le32 block_size;
-	__le32 block_num;
-	__le32 device_phy_addr[NUM_OF_FW_PAGING_BLOCKS];
-} __packed; /* FW_PAGING_BLOCK_CMD_API_S_VER_1 */
-
-/*
- * Fw items ID's
- *
- * @IWL_FW_ITEM_ID_PAGING: Address of the pages that the FW will upload
- *	download
- */
-enum iwl_fw_item_id {
-	IWL_FW_ITEM_ID_PAGING = 3,
-};
-
-/*
- * struct iwl_fw_get_item_cmd - get an item from the fw
- */
-struct iwl_fw_get_item_cmd {
-	__le32 item_id;
-} __packed; /* FW_GET_ITEM_CMD_API_S_VER_1 */
-
-#define CONT_REC_COMMAND_SIZE	80
-#define ENABLE_CONT_RECORDING	0x15
-#define DISABLE_CONT_RECORDING	0x16
-
-/*
- * struct iwl_continuous_record_mode - recording mode
- */
-struct iwl_continuous_record_mode {
-	__le16 enable_recording;
-} __packed;
-
-/*
- * struct iwl_continuous_record_cmd - enable/disable continuous recording
- */
-struct iwl_continuous_record_cmd {
-	struct iwl_continuous_record_mode record_mode;
-	u8 pad[CONT_REC_COMMAND_SIZE -
-		sizeof(struct iwl_continuous_record_mode)];
-} __packed;
-
-struct iwl_fw_get_item_resp {
-	__le32 item_id;
-	__le32 item_byte_cnt;
-	__le32 item_val;
-} __packed; /* FW_GET_ITEM_RSP_S_VER_1 */
-
-/**
- * struct iwl_nvm_access_resp_ver2 - response to NVM_ACCESS_CMD
- * @offset: offset in bytes into the section
- * @length: in bytes, either how much was written or read
- * @type: NVM_SECTION_TYPE_*
- * @status: 0 for success, fail otherwise
- * @data: if read operation, the data returned. Empty on write.
- */
-struct iwl_nvm_access_resp {
-	__le16 offset;
-	__le16 length;
-	__le16 type;
-	__le16 status;
-	u8 data[];
-} __packed; /* NVM_ACCESS_CMD_RESP_API_S_VER_2 */
-
-/* MVM_ALIVE 0x1 */
-
-/* alive response is_valid values */
-#define ALIVE_RESP_UCODE_OK	BIT(0)
-#define ALIVE_RESP_RFKILL	BIT(1)
-
-/* alive response ver_type values */
-enum {
-	FW_TYPE_HW = 0,
-	FW_TYPE_PROT = 1,
-	FW_TYPE_AP = 2,
-	FW_TYPE_WOWLAN = 3,
-	FW_TYPE_TIMING = 4,
-	FW_TYPE_WIPAN = 5
-};
-
-/* alive response ver_subtype values */
-enum {
-	FW_SUBTYPE_FULL_FEATURE = 0,
-	FW_SUBTYPE_BOOTSRAP = 1, /* Not valid */
-	FW_SUBTYPE_REDUCED = 2,
-	FW_SUBTYPE_ALIVE_ONLY = 3,
-	FW_SUBTYPE_WOWLAN = 4,
-	FW_SUBTYPE_AP_SUBTYPE = 5,
-	FW_SUBTYPE_WIPAN = 6,
-	FW_SUBTYPE_INITIALIZE = 9
-};
-
-#define IWL_ALIVE_STATUS_ERR 0xDEAD
-#define IWL_ALIVE_STATUS_OK 0xCAFE
-
-#define IWL_ALIVE_FLG_RFKILL	BIT(0)
-
-struct iwl_lmac_alive {
-	__le32 ucode_minor;
-	__le32 ucode_major;
-	u8 ver_subtype;
-	u8 ver_type;
-	u8 mac;
-	u8 opt;
-	__le32 timestamp;
-	__le32 error_event_table_ptr;	/* SRAM address for error log */
-	__le32 log_event_table_ptr;	/* SRAM address for LMAC event log */
-	__le32 cpu_register_ptr;
-	__le32 dbgm_config_ptr;
-	__le32 alive_counter_ptr;
-	__le32 scd_base_ptr;		/* SRAM address for SCD */
-	__le32 st_fwrd_addr;		/* pointer to Store and forward */
-	__le32 st_fwrd_size;
-} __packed; /* UCODE_ALIVE_NTFY_API_S_VER_3 */
-
-struct iwl_umac_alive {
-	__le32 umac_minor;		/* UMAC version: minor */
-	__le32 umac_major;		/* UMAC version: major */
-	__le32 error_info_addr;		/* SRAM address for UMAC error log */
-	__le32 dbg_print_buff_addr;
-} __packed; /* UMAC_ALIVE_DATA_API_S_VER_2 */
-
-struct mvm_alive_resp_v3 {
-	__le16 status;
-	__le16 flags;
-	struct iwl_lmac_alive lmac_data;
-	struct iwl_umac_alive umac_data;
-} __packed; /* ALIVE_RES_API_S_VER_3 */
-
-struct mvm_alive_resp {
-	__le16 status;
-	__le16 flags;
-	struct iwl_lmac_alive lmac_data[2];
-	struct iwl_umac_alive umac_data;
-} __packed; /* ALIVE_RES_API_S_VER_4 */
-
-/* Error response/notification */
-enum {
-	FW_ERR_UNKNOWN_CMD = 0x0,
-	FW_ERR_INVALID_CMD_PARAM = 0x1,
-	FW_ERR_SERVICE = 0x2,
-	FW_ERR_ARC_MEMORY = 0x3,
-	FW_ERR_ARC_CODE = 0x4,
-	FW_ERR_WATCH_DOG = 0x5,
-	FW_ERR_WEP_GRP_KEY_INDX = 0x10,
-	FW_ERR_WEP_KEY_SIZE = 0x11,
-	FW_ERR_OBSOLETE_FUNC = 0x12,
-	FW_ERR_UNEXPECTED = 0xFE,
-	FW_ERR_FATAL = 0xFF
-};
-
-/**
- * struct iwl_error_resp - FW error indication
- * ( REPLY_ERROR = 0x2 )
- * @error_type: one of FW_ERR_*
- * @cmd_id: the command ID for which the error occured
- * @reserved1: reserved
- * @bad_cmd_seq_num: sequence number of the erroneous command
- * @error_service: which service created the error, applicable only if
- *	error_type = 2, otherwise 0
- * @timestamp: TSF in usecs.
- */
-struct iwl_error_resp {
-	__le32 error_type;
-	u8 cmd_id;
-	u8 reserved1;
-	__le16 bad_cmd_seq_num;
-	__le32 error_service;
-	__le64 timestamp;
-} __packed;
-
-
-/* Common PHY, MAC and Bindings definitions */
-#define MAX_MACS_IN_BINDING	(3)
-#define MAX_BINDINGS		(4)
-
-/**
- * enum iwl_mvm_id_and_color - ID and color fields in context dword
- * @FW_CTXT_ID_POS: position of the ID
- * @FW_CTXT_ID_MSK: mask of the ID
- * @FW_CTXT_COLOR_POS: position of the color
- * @FW_CTXT_COLOR_MSK: mask of the color
- * @FW_CTXT_INVALID: value used to indicate unused/invalid
- */
-enum iwl_mvm_id_and_color {
-	FW_CTXT_ID_POS		= 0,
-	FW_CTXT_ID_MSK		= 0xff << FW_CTXT_ID_POS,
-	FW_CTXT_COLOR_POS	= 8,
-	FW_CTXT_COLOR_MSK	= 0xff << FW_CTXT_COLOR_POS,
-	FW_CTXT_INVALID		= 0xffffffff,
-};
-
-#define FW_CMD_ID_AND_COLOR(_id, _color) ((_id << FW_CTXT_ID_POS) |\
-					  (_color << FW_CTXT_COLOR_POS))
-
-/* Possible actions on PHYs, MACs and Bindings */
-enum iwl_phy_ctxt_action {
-	FW_CTXT_ACTION_STUB = 0,
-	FW_CTXT_ACTION_ADD,
-	FW_CTXT_ACTION_MODIFY,
-	FW_CTXT_ACTION_REMOVE,
-	FW_CTXT_ACTION_NUM
-}; /* COMMON_CONTEXT_ACTION_API_E_VER_1 */
-
-/* Time Events */
-
-/* Time Event types, according to MAC type */
-enum iwl_time_event_type {
-	/* BSS Station Events */
-	TE_BSS_STA_AGGRESSIVE_ASSOC,
-	TE_BSS_STA_ASSOC,
-	TE_BSS_EAP_DHCP_PROT,
-	TE_BSS_QUIET_PERIOD,
-
-	/* P2P Device Events */
-	TE_P2P_DEVICE_DISCOVERABLE,
-	TE_P2P_DEVICE_LISTEN,
-	TE_P2P_DEVICE_ACTION_SCAN,
-	TE_P2P_DEVICE_FULL_SCAN,
-
-	/* P2P Client Events */
-	TE_P2P_CLIENT_AGGRESSIVE_ASSOC,
-	TE_P2P_CLIENT_ASSOC,
-	TE_P2P_CLIENT_QUIET_PERIOD,
-
-	/* P2P GO Events */
-	TE_P2P_GO_ASSOC_PROT,
-	TE_P2P_GO_REPETITIVET_NOA,
-	TE_P2P_GO_CT_WINDOW,
-
-	/* WiDi Sync Events */
-	TE_WIDI_TX_SYNC,
-
-	/* Channel Switch NoA */
-	TE_CHANNEL_SWITCH_PERIOD,
-
-	TE_MAX
-}; /* MAC_EVENT_TYPE_API_E_VER_1 */
-
-
-
-/* Time event - defines for command API v1 */
-
-/*
- * @TE_V1_FRAG_NONE: fragmentation of the time event is NOT allowed.
- * @TE_V1_FRAG_SINGLE: fragmentation of the time event is allowed, but only
- *	the first fragment is scheduled.
- * @TE_V1_FRAG_DUAL: fragmentation of the time event is allowed, but only
- *	the first 2 fragments are scheduled.
- * @TE_V1_FRAG_ENDLESS: fragmentation of the time event is allowed, and any
- *	number of fragments are valid.
- *
- * Other than the constant defined above, specifying a fragmentation value 'x'
- * means that the event can be fragmented but only the first 'x' will be
- * scheduled.
- */
-enum {
-	TE_V1_FRAG_NONE = 0,
-	TE_V1_FRAG_SINGLE = 1,
-	TE_V1_FRAG_DUAL = 2,
-	TE_V1_FRAG_ENDLESS = 0xffffffff
-};
-
-/* If a Time Event can be fragmented, this is the max number of fragments */
-#define TE_V1_FRAG_MAX_MSK	0x0fffffff
-/* Repeat the time event endlessly (until removed) */
-#define TE_V1_REPEAT_ENDLESS	0xffffffff
-/* If a Time Event has bounded repetitions, this is the maximal value */
-#define TE_V1_REPEAT_MAX_MSK_V1	0x0fffffff
-
-/* Time Event dependencies: none, on another TE, or in a specific time */
-enum {
-	TE_V1_INDEPENDENT		= 0,
-	TE_V1_DEP_OTHER			= BIT(0),
-	TE_V1_DEP_TSF			= BIT(1),
-	TE_V1_EVENT_SOCIOPATHIC		= BIT(2),
-}; /* MAC_EVENT_DEPENDENCY_POLICY_API_E_VER_2 */
-
-/*
- * @TE_V1_NOTIF_NONE: no notifications
- * @TE_V1_NOTIF_HOST_EVENT_START: request/receive notification on event start
- * @TE_V1_NOTIF_HOST_EVENT_END:request/receive notification on event end
- * @TE_V1_NOTIF_INTERNAL_EVENT_START: internal FW use
- * @TE_V1_NOTIF_INTERNAL_EVENT_END: internal FW use.
- * @TE_V1_NOTIF_HOST_FRAG_START: request/receive notification on frag start
- * @TE_V1_NOTIF_HOST_FRAG_END:request/receive notification on frag end
- * @TE_V1_NOTIF_INTERNAL_FRAG_START: internal FW use.
- * @TE_V1_NOTIF_INTERNAL_FRAG_END: internal FW use.
- *
- * Supported Time event notifications configuration.
- * A notification (both event and fragment) includes a status indicating weather
- * the FW was able to schedule the event or not. For fragment start/end
- * notification the status is always success. There is no start/end fragment
- * notification for monolithic events.
- */
-enum {
-	TE_V1_NOTIF_NONE = 0,
-	TE_V1_NOTIF_HOST_EVENT_START = BIT(0),
-	TE_V1_NOTIF_HOST_EVENT_END = BIT(1),
-	TE_V1_NOTIF_INTERNAL_EVENT_START = BIT(2),
-	TE_V1_NOTIF_INTERNAL_EVENT_END = BIT(3),
-	TE_V1_NOTIF_HOST_FRAG_START = BIT(4),
-	TE_V1_NOTIF_HOST_FRAG_END = BIT(5),
-	TE_V1_NOTIF_INTERNAL_FRAG_START = BIT(6),
-	TE_V1_NOTIF_INTERNAL_FRAG_END = BIT(7),
-}; /* MAC_EVENT_ACTION_API_E_VER_2 */
-
-/* Time event - defines for command API */
-
-/*
- * @TE_V2_FRAG_NONE: fragmentation of the time event is NOT allowed.
- * @TE_V2_FRAG_SINGLE: fragmentation of the time event is allowed, but only
- *  the first fragment is scheduled.
- * @TE_V2_FRAG_DUAL: fragmentation of the time event is allowed, but only
- *  the first 2 fragments are scheduled.
- * @TE_V2_FRAG_ENDLESS: fragmentation of the time event is allowed, and any
- *  number of fragments are valid.
- *
- * Other than the constant defined above, specifying a fragmentation value 'x'
- * means that the event can be fragmented but only the first 'x' will be
- * scheduled.
- */
-enum {
-	TE_V2_FRAG_NONE = 0,
-	TE_V2_FRAG_SINGLE = 1,
-	TE_V2_FRAG_DUAL = 2,
-	TE_V2_FRAG_MAX = 0xfe,
-	TE_V2_FRAG_ENDLESS = 0xff
-};
-
-/* Repeat the time event endlessly (until removed) */
-#define TE_V2_REPEAT_ENDLESS	0xff
-/* If a Time Event has bounded repetitions, this is the maximal value */
-#define TE_V2_REPEAT_MAX	0xfe
-
-#define TE_V2_PLACEMENT_POS	12
-#define TE_V2_ABSENCE_POS	15
-
-/**
- * enum iwl_time_event_policy - Time event policy values
- * A notification (both event and fragment) includes a status indicating weather
- * the FW was able to schedule the event or not. For fragment start/end
- * notification the status is always success. There is no start/end fragment
- * notification for monolithic events.
- *
- * @TE_V2_DEFAULT_POLICY: independent, social, present, unoticable
- * @TE_V2_NOTIF_HOST_EVENT_START: request/receive notification on event start
- * @TE_V2_NOTIF_HOST_EVENT_END:request/receive notification on event end
- * @TE_V2_NOTIF_INTERNAL_EVENT_START: internal FW use
- * @TE_V2_NOTIF_INTERNAL_EVENT_END: internal FW use.
- * @TE_V2_NOTIF_HOST_FRAG_START: request/receive notification on frag start
- * @TE_V2_NOTIF_HOST_FRAG_END:request/receive notification on frag end
- * @TE_V2_NOTIF_INTERNAL_FRAG_START: internal FW use.
- * @TE_V2_NOTIF_INTERNAL_FRAG_END: internal FW use.
- * @T2_V2_START_IMMEDIATELY: start time event immediately
- * @TE_V2_DEP_OTHER: depends on another time event
- * @TE_V2_DEP_TSF: depends on a specific time
- * @TE_V2_EVENT_SOCIOPATHIC: can't co-exist with other events of tha same MAC
- * @TE_V2_ABSENCE: are we present or absent during the Time Event.
- */
-enum iwl_time_event_policy {
-	TE_V2_DEFAULT_POLICY = 0x0,
-
-	/* notifications (event start/stop, fragment start/stop) */
-	TE_V2_NOTIF_HOST_EVENT_START = BIT(0),
-	TE_V2_NOTIF_HOST_EVENT_END = BIT(1),
-	TE_V2_NOTIF_INTERNAL_EVENT_START = BIT(2),
-	TE_V2_NOTIF_INTERNAL_EVENT_END = BIT(3),
-
-	TE_V2_NOTIF_HOST_FRAG_START = BIT(4),
-	TE_V2_NOTIF_HOST_FRAG_END = BIT(5),
-	TE_V2_NOTIF_INTERNAL_FRAG_START = BIT(6),
-	TE_V2_NOTIF_INTERNAL_FRAG_END = BIT(7),
-	T2_V2_START_IMMEDIATELY = BIT(11),
-
-	/* placement characteristics */
-	TE_V2_DEP_OTHER = BIT(TE_V2_PLACEMENT_POS),
-	TE_V2_DEP_TSF = BIT(TE_V2_PLACEMENT_POS + 1),
-	TE_V2_EVENT_SOCIOPATHIC = BIT(TE_V2_PLACEMENT_POS + 2),
-
-	/* are we present or absent during the Time Event. */
-	TE_V2_ABSENCE = BIT(TE_V2_ABSENCE_POS),
-};
-
-/**
- * struct iwl_time_event_cmd - configuring Time Events
- * with struct MAC_TIME_EVENT_DATA_API_S_VER_2 (see also
- * with version 1. determined by IWL_UCODE_TLV_FLAGS)
- * ( TIME_EVENT_CMD = 0x29 )
- * @id_and_color: ID and color of the relevant MAC,
- *	&enum iwl_mvm_id_and_color
- * @action: action to perform, one of &enum iwl_phy_ctxt_action
- * @id: this field has two meanings, depending on the action:
- *	If the action is ADD, then it means the type of event to add.
- *	For all other actions it is the unique event ID assigned when the
- *	event was added by the FW.
- * @apply_time: When to start the Time Event (in GP2)
- * @max_delay: maximum delay to event's start (apply time), in TU
- * @depends_on: the unique ID of the event we depend on (if any)
- * @interval: interval between repetitions, in TU
- * @duration: duration of event in TU
- * @repeat: how many repetitions to do, can be TE_REPEAT_ENDLESS
- * @max_frags: maximal number of fragments the Time Event can be divided to
- * @policy: defines whether uCode shall notify the host or other uCode modules
- *	on event and/or fragment start and/or end
- *	using one of TE_INDEPENDENT, TE_DEP_OTHER, TE_DEP_TSF
- *	TE_EVENT_SOCIOPATHIC
- *	using TE_ABSENCE and using TE_NOTIF_*,
- *	&enum iwl_time_event_policy
- */
-struct iwl_time_event_cmd {
-	/* COMMON_INDEX_HDR_API_S_VER_1 */
-	__le32 id_and_color;
-	__le32 action;
-	__le32 id;
-	/* MAC_TIME_EVENT_DATA_API_S_VER_2 */
-	__le32 apply_time;
-	__le32 max_delay;
-	__le32 depends_on;
-	__le32 interval;
-	__le32 duration;
-	u8 repeat;
-	u8 max_frags;
-	__le16 policy;
-} __packed; /* MAC_TIME_EVENT_CMD_API_S_VER_2 */
-
-/**
- * struct iwl_time_event_resp - response structure to iwl_time_event_cmd
- * @status: bit 0 indicates success, all others specify errors
- * @id: the Time Event type
- * @unique_id: the unique ID assigned (in ADD) or given (others) to the TE
- * @id_and_color: ID and color of the relevant MAC,
- *	&enum iwl_mvm_id_and_color
- */
-struct iwl_time_event_resp {
-	__le32 status;
-	__le32 id;
-	__le32 unique_id;
-	__le32 id_and_color;
-} __packed; /* MAC_TIME_EVENT_RSP_API_S_VER_1 */
-
-/**
- * struct iwl_time_event_notif - notifications of time event start/stop
- * ( TIME_EVENT_NOTIFICATION = 0x2a )
- * @timestamp: action timestamp in GP2
- * @session_id: session's unique id
- * @unique_id: unique id of the Time Event itself
- * @id_and_color: ID and color of the relevant MAC
- * @action: &enum iwl_time_event_policy
- * @status: true if scheduled, false otherwise (not executed)
- */
-struct iwl_time_event_notif {
-	__le32 timestamp;
-	__le32 session_id;
-	__le32 unique_id;
-	__le32 id_and_color;
-	__le32 action;
-	__le32 status;
-} __packed; /* MAC_TIME_EVENT_NTFY_API_S_VER_1 */
-
-
-/* Bindings and Time Quota */
-
-/**
- * struct iwl_binding_cmd_v1 - configuring bindings
- * ( BINDING_CONTEXT_CMD = 0x2b )
- * @id_and_color: ID and color of the relevant Binding,
- *	&enum iwl_mvm_id_and_color
- * @action: action to perform, one of FW_CTXT_ACTION_*
- * @macs: array of MAC id and colors which belong to the binding,
- *	&enum iwl_mvm_id_and_color
- * @phy: PHY id and color which belongs to the binding,
- *	&enum iwl_mvm_id_and_color
- */
-struct iwl_binding_cmd_v1 {
-	/* COMMON_INDEX_HDR_API_S_VER_1 */
-	__le32 id_and_color;
-	__le32 action;
-	/* BINDING_DATA_API_S_VER_1 */
-	__le32 macs[MAX_MACS_IN_BINDING];
-	__le32 phy;
-} __packed; /* BINDING_CMD_API_S_VER_1 */
-
-/**
- * struct iwl_binding_cmd - configuring bindings
- * ( BINDING_CONTEXT_CMD = 0x2b )
- * @id_and_color: ID and color of the relevant Binding,
- *	&enum iwl_mvm_id_and_color
- * @action: action to perform, one of FW_CTXT_ACTION_*
- * @macs: array of MAC id and colors which belong to the binding
- *	&enum iwl_mvm_id_and_color
- * @phy: PHY id and color which belongs to the binding
- *	&enum iwl_mvm_id_and_color
- * @lmac_id: the lmac id the binding belongs to
- */
-struct iwl_binding_cmd {
-	/* COMMON_INDEX_HDR_API_S_VER_1 */
-	__le32 id_and_color;
-	__le32 action;
-	/* BINDING_DATA_API_S_VER_1 */
-	__le32 macs[MAX_MACS_IN_BINDING];
-	__le32 phy;
-	__le32 lmac_id;
-} __packed; /* BINDING_CMD_API_S_VER_2 */
-
-#define IWL_BINDING_CMD_SIZE_V1	sizeof(struct iwl_binding_cmd_v1)
-#define IWL_LMAC_24G_INDEX		0
-#define IWL_LMAC_5G_INDEX		1
-
-/* The maximal number of fragments in the FW's schedule session */
-#define IWL_MVM_MAX_QUOTA 128
-
-/**
- * struct iwl_time_quota_data - configuration of time quota per binding
- * @id_and_color: ID and color of the relevant Binding,
- *	&enum iwl_mvm_id_and_color
- * @quota: absolute time quota in TU. The scheduler will try to divide the
- *	remainig quota (after Time Events) according to this quota.
- * @max_duration: max uninterrupted context duration in TU
- */
-struct iwl_time_quota_data {
-	__le32 id_and_color;
-	__le32 quota;
-	__le32 max_duration;
-} __packed; /* TIME_QUOTA_DATA_API_S_VER_1 */
-
-/**
- * struct iwl_time_quota_cmd - configuration of time quota between bindings
- * ( TIME_QUOTA_CMD = 0x2c )
- * @quotas: allocations per binding
- * Note: on non-CDB the fourth one is the auxilary mac and is
- *	essentially zero.
- *	On CDB the fourth one is a regular binding.
- */
-struct iwl_time_quota_cmd {
-	struct iwl_time_quota_data quotas[MAX_BINDINGS];
-} __packed; /* TIME_QUOTA_ALLOCATION_CMD_API_S_VER_1 */
-
-
-/* PHY context */
-
-/* Supported bands */
-#define PHY_BAND_5  (0)
-#define PHY_BAND_24 (1)
-
-/* Supported channel width, vary if there is VHT support */
-#define PHY_VHT_CHANNEL_MODE20	(0x0)
-#define PHY_VHT_CHANNEL_MODE40	(0x1)
-#define PHY_VHT_CHANNEL_MODE80	(0x2)
-#define PHY_VHT_CHANNEL_MODE160	(0x3)
-
-/*
- * Control channel position:
- * For legacy set bit means upper channel, otherwise lower.
- * For VHT - bit-2 marks if the control is lower/upper relative to center-freq
- *   bits-1:0 mark the distance from the center freq. for 20Mhz, offset is 0.
- *                                   center_freq
- *                                        |
- * 40Mhz                          |_______|_______|
- * 80Mhz                  |_______|_______|_______|_______|
- * 160Mhz |_______|_______|_______|_______|_______|_______|_______|_______|
- * code      011     010     001     000  |  100     101     110    111
- */
-#define PHY_VHT_CTRL_POS_1_BELOW  (0x0)
-#define PHY_VHT_CTRL_POS_2_BELOW  (0x1)
-#define PHY_VHT_CTRL_POS_3_BELOW  (0x2)
-#define PHY_VHT_CTRL_POS_4_BELOW  (0x3)
-#define PHY_VHT_CTRL_POS_1_ABOVE  (0x4)
-#define PHY_VHT_CTRL_POS_2_ABOVE  (0x5)
-#define PHY_VHT_CTRL_POS_3_ABOVE  (0x6)
-#define PHY_VHT_CTRL_POS_4_ABOVE  (0x7)
-
-/*
- * @band: PHY_BAND_*
- * @channel: channel number
- * @width: PHY_[VHT|LEGACY]_CHANNEL_*
- * @ctrl channel: PHY_[VHT|LEGACY]_CTRL_*
- */
-struct iwl_fw_channel_info {
-	u8 band;
-	u8 channel;
-	u8 width;
-	u8 ctrl_pos;
-} __packed;
-
-#define PHY_RX_CHAIN_DRIVER_FORCE_POS	(0)
-#define PHY_RX_CHAIN_DRIVER_FORCE_MSK \
-	(0x1 << PHY_RX_CHAIN_DRIVER_FORCE_POS)
-#define PHY_RX_CHAIN_VALID_POS		(1)
-#define PHY_RX_CHAIN_VALID_MSK \
-	(0x7 << PHY_RX_CHAIN_VALID_POS)
-#define PHY_RX_CHAIN_FORCE_SEL_POS	(4)
-#define PHY_RX_CHAIN_FORCE_SEL_MSK \
-	(0x7 << PHY_RX_CHAIN_FORCE_SEL_POS)
-#define PHY_RX_CHAIN_FORCE_MIMO_SEL_POS	(7)
-#define PHY_RX_CHAIN_FORCE_MIMO_SEL_MSK \
-	(0x7 << PHY_RX_CHAIN_FORCE_MIMO_SEL_POS)
-#define PHY_RX_CHAIN_CNT_POS		(10)
-#define PHY_RX_CHAIN_CNT_MSK \
-	(0x3 << PHY_RX_CHAIN_CNT_POS)
-#define PHY_RX_CHAIN_MIMO_CNT_POS	(12)
-#define PHY_RX_CHAIN_MIMO_CNT_MSK \
-	(0x3 << PHY_RX_CHAIN_MIMO_CNT_POS)
-#define PHY_RX_CHAIN_MIMO_FORCE_POS	(14)
-#define PHY_RX_CHAIN_MIMO_FORCE_MSK \
-	(0x1 << PHY_RX_CHAIN_MIMO_FORCE_POS)
-
-/* TODO: fix the value, make it depend on firmware at runtime? */
-#define NUM_PHY_CTX	3
-
-/* TODO: complete missing documentation */
-/**
- * struct iwl_phy_context_cmd - config of the PHY context
- * ( PHY_CONTEXT_CMD = 0x8 )
- * @id_and_color: ID and color of the relevant Binding
- * @action: action to perform, one of FW_CTXT_ACTION_*
- * @apply_time: 0 means immediate apply and context switch.
- *	other value means apply new params after X usecs
- * @tx_param_color: ???
- * @ci: channel info
- * @txchain_info: ???
- * @rxchain_info: ???
- * @acquisition_data: ???
- * @dsp_cfg_flags: set to 0
- */
-struct iwl_phy_context_cmd {
-	/* COMMON_INDEX_HDR_API_S_VER_1 */
-	__le32 id_and_color;
-	__le32 action;
-	/* PHY_CONTEXT_DATA_API_S_VER_1 */
-	__le32 apply_time;
-	__le32 tx_param_color;
-	struct iwl_fw_channel_info ci;
-	__le32 txchain_info;
-	__le32 rxchain_info;
-	__le32 acquisition_data;
-	__le32 dsp_cfg_flags;
-} __packed; /* PHY_CONTEXT_CMD_API_VER_1 */
-
-/*
- * Aux ROC command
- *
- * Command requests the firmware to create a time event for a certain duration
- * and remain on the given channel. This is done by using the Aux framework in
- * the FW.
- * The command was first used for Hot Spot issues - but can be used regardless
- * to Hot Spot.
- *
- * ( HOT_SPOT_CMD 0x53 )
- *
- * @id_and_color: ID and color of the MAC
- * @action: action to perform, one of FW_CTXT_ACTION_*
- * @event_unique_id: If the action FW_CTXT_ACTION_REMOVE then the
- *	event_unique_id should be the id of the time event assigned by ucode.
- *	Otherwise ignore the event_unique_id.
- * @sta_id_and_color: station id and color, resumed during "Remain On Channel"
- *	activity.
- * @channel_info: channel info
- * @node_addr: Our MAC Address
- * @reserved: reserved for alignment
- * @apply_time: GP2 value to start (should always be the current GP2 value)
- * @apply_time_max_delay: Maximum apply time delay value in TU. Defines max
- *	time by which start of the event is allowed to be postponed.
- * @duration: event duration in TU To calculate event duration:
- *	timeEventDuration = min(duration, remainingQuota)
- */
-struct iwl_hs20_roc_req {
-	/* COMMON_INDEX_HDR_API_S_VER_1 hdr */
-	__le32 id_and_color;
-	__le32 action;
-	__le32 event_unique_id;
-	__le32 sta_id_and_color;
-	struct iwl_fw_channel_info channel_info;
-	u8 node_addr[ETH_ALEN];
-	__le16 reserved;
-	__le32 apply_time;
-	__le32 apply_time_max_delay;
-	__le32 duration;
-} __packed; /* HOT_SPOT_CMD_API_S_VER_1 */
-
-/*
- * values for AUX ROC result values
- */
-enum iwl_mvm_hot_spot {
-	HOT_SPOT_RSP_STATUS_OK,
-	HOT_SPOT_RSP_STATUS_TOO_MANY_EVENTS,
-	HOT_SPOT_MAX_NUM_OF_SESSIONS,
-};
-
-/*
- * Aux ROC command response
- *
- * In response to iwl_hs20_roc_req the FW sends this command to notify the
- * driver the uid of the timevent.
- *
- * ( HOT_SPOT_CMD 0x53 )
- *
- * @event_unique_id: Unique ID of time event assigned by ucode
- * @status: Return status 0 is success, all the rest used for specific errors
- */
-struct iwl_hs20_roc_res {
-	__le32 event_unique_id;
-	__le32 status;
-} __packed; /* HOT_SPOT_RSP_API_S_VER_1 */
-
-/**
- * struct iwl_radio_version_notif - information on the radio version
- * ( RADIO_VERSION_NOTIFICATION = 0x68 )
- * @radio_flavor: radio flavor
- * @radio_step: radio version step
- * @radio_dash: radio version dash
- */
-struct iwl_radio_version_notif {
-	__le32 radio_flavor;
-	__le32 radio_step;
-	__le32 radio_dash;
-} __packed; /* RADIO_VERSION_NOTOFICATION_S_VER_1 */
-
-enum iwl_card_state_flags {
-	CARD_ENABLED		= 0x00,
-	HW_CARD_DISABLED	= 0x01,
-	SW_CARD_DISABLED	= 0x02,
-	CT_KILL_CARD_DISABLED	= 0x04,
-	HALT_CARD_DISABLED	= 0x08,
-	CARD_DISABLED_MSK	= 0x0f,
-	CARD_IS_RX_ON		= 0x10,
-};
-
-/**
- * struct iwl_radio_version_notif - information on the radio version
- * ( CARD_STATE_NOTIFICATION = 0xa1 )
- * @flags: %iwl_card_state_flags
- */
-struct iwl_card_state_notif {
-	__le32 flags;
-} __packed; /* CARD_STATE_NTFY_API_S_VER_1 */
-
-/**
- * struct iwl_missed_beacons_notif - information on missed beacons
- * ( MISSED_BEACONS_NOTIFICATION = 0xa2 )
- * @mac_id: interface ID
- * @consec_missed_beacons_since_last_rx: number of consecutive missed
- *	beacons since last RX.
- * @consec_missed_beacons: number of consecutive missed beacons
- * @num_expected_beacons: number of expected beacons
- * @num_recvd_beacons: number of received beacons
- */
-struct iwl_missed_beacons_notif {
-	__le32 mac_id;
-	__le32 consec_missed_beacons_since_last_rx;
-	__le32 consec_missed_beacons;
-	__le32 num_expected_beacons;
-	__le32 num_recvd_beacons;
-} __packed; /* MISSED_BEACON_NTFY_API_S_VER_3 */
-
-/**
- * struct iwl_mfuart_load_notif - mfuart image version & status
- * ( MFUART_LOAD_NOTIFICATION = 0xb1 )
- * @installed_ver: installed image version
- * @external_ver: external image version
- * @status: MFUART loading status
- * @duration: MFUART loading time
- * @image_size: MFUART image size in bytes
-*/
-struct iwl_mfuart_load_notif {
-	__le32 installed_ver;
-	__le32 external_ver;
-	__le32 status;
-	__le32 duration;
-	/* image size valid only in v2 of the command */
-	__le32 image_size;
-} __packed; /*MFU_LOADER_NTFY_API_S_VER_2*/
-
-/**
- * struct iwl_mfu_assert_dump_notif - mfuart dump logs
- * ( MFU_ASSERT_DUMP_NTF = 0xfe )
- * @assert_id: mfuart assert id that cause the notif
- * @curr_reset_num: number of asserts since uptime
- * @index_num: current chunk id
- * @parts_num: total number of chunks
- * @data_size: number of data bytes sent
- * @data: data buffer
- */
-struct iwl_mfu_assert_dump_notif {
-	__le32   assert_id;
-	__le32   curr_reset_num;
-	__le16   index_num;
-	__le16   parts_num;
-	__le32   data_size;
-	__le32   data[0];
-} __packed; /*MFU_DUMP_ASSERT_API_S_VER_1*/
-
-#define MAX_PORT_ID_NUM	2
-#define MAX_MCAST_FILTERING_ADDRESSES 256
-
-/**
- * struct iwl_mcast_filter_cmd - configure multicast filter.
- * @filter_own: Set 1 to filter out multicast packets sent by station itself
- * @port_id:	Multicast MAC addresses array specifier. This is a strange way
- *		to identify network interface adopted in host-device IF.
- *		It is used by FW as index in array of addresses. This array has
- *		MAX_PORT_ID_NUM members.
- * @count:	Number of MAC addresses in the array
- * @pass_all:	Set 1 to pass all multicast packets.
- * @bssid:	current association BSSID.
- * @reserved:	reserved
- * @addr_list:	Place holder for array of MAC addresses.
- *		IMPORTANT: add padding if necessary to ensure DWORD alignment.
- */
-struct iwl_mcast_filter_cmd {
-	u8 filter_own;
-	u8 port_id;
-	u8 count;
-	u8 pass_all;
-	u8 bssid[6];
-	u8 reserved[2];
-	u8 addr_list[0];
-} __packed; /* MCAST_FILTERING_CMD_API_S_VER_1 */
-
-#define MAX_BCAST_FILTERS 8
-#define MAX_BCAST_FILTER_ATTRS 2
-
-/**
- * enum iwl_mvm_bcast_filter_attr_offset - written by fw for each Rx packet
- * @BCAST_FILTER_OFFSET_PAYLOAD_START: offset is from payload start.
- * @BCAST_FILTER_OFFSET_IP_END: offset is from ip header end (i.e.
- *	start of ip payload).
- */
-enum iwl_mvm_bcast_filter_attr_offset {
-	BCAST_FILTER_OFFSET_PAYLOAD_START = 0,
-	BCAST_FILTER_OFFSET_IP_END = 1,
-};
-
-/**
- * struct iwl_fw_bcast_filter_attr - broadcast filter attribute
- * @offset_type:	&enum iwl_mvm_bcast_filter_attr_offset.
- * @offset:	starting offset of this pattern.
- * @reserved1:	reserved
- * @val:	value to match - big endian (MSB is the first
- *		byte to match from offset pos).
- * @mask:	mask to match (big endian).
- */
-struct iwl_fw_bcast_filter_attr {
-	u8 offset_type;
-	u8 offset;
-	__le16 reserved1;
-	__be32 val;
-	__be32 mask;
-} __packed; /* BCAST_FILTER_ATT_S_VER_1 */
-
-/**
- * enum iwl_mvm_bcast_filter_frame_type - filter frame type
- * @BCAST_FILTER_FRAME_TYPE_ALL: consider all frames.
- * @BCAST_FILTER_FRAME_TYPE_IPV4: consider only ipv4 frames
- */
-enum iwl_mvm_bcast_filter_frame_type {
-	BCAST_FILTER_FRAME_TYPE_ALL = 0,
-	BCAST_FILTER_FRAME_TYPE_IPV4 = 1,
-};
-
-/**
- * struct iwl_fw_bcast_filter - broadcast filter
- * @discard: discard frame (1) or let it pass (0).
- * @frame_type: &enum iwl_mvm_bcast_filter_frame_type.
- * @reserved1: reserved
- * @num_attrs: number of valid attributes in this filter.
- * @attrs: attributes of this filter. a filter is considered matched
- *	only when all its attributes are matched (i.e. AND relationship)
- */
-struct iwl_fw_bcast_filter {
-	u8 discard;
-	u8 frame_type;
-	u8 num_attrs;
-	u8 reserved1;
-	struct iwl_fw_bcast_filter_attr attrs[MAX_BCAST_FILTER_ATTRS];
-} __packed; /* BCAST_FILTER_S_VER_1 */
-
-#define BA_WINDOW_STREAMS_MAX		16
-#define BA_WINDOW_STATUS_TID_MSK	0x000F
-#define BA_WINDOW_STATUS_STA_ID_POS	4
-#define BA_WINDOW_STATUS_STA_ID_MSK	0x01F0
-#define BA_WINDOW_STATUS_VALID_MSK	BIT(9)
-
-/**
- * struct iwl_ba_window_status_notif - reordering window's status notification
- * @bitmap: bitmap of received frames [start_seq_num + 0]..[start_seq_num + 63]
- * @ra_tid: bit 3:0 - TID, bit 8:4 - STA_ID, bit 9 - valid
- * @start_seq_num: the start sequence number of the bitmap
- * @mpdu_rx_count: the number of received MPDUs since entering D0i3
- */
-struct iwl_ba_window_status_notif {
-	__le64 bitmap[BA_WINDOW_STREAMS_MAX];
-	__le16 ra_tid[BA_WINDOW_STREAMS_MAX];
-	__le32 start_seq_num[BA_WINDOW_STREAMS_MAX];
-	__le16 mpdu_rx_count[BA_WINDOW_STREAMS_MAX];
-} __packed; /* BA_WINDOW_STATUS_NTFY_API_S_VER_1 */
-
-/**
- * struct iwl_fw_bcast_mac - per-mac broadcast filtering configuration.
- * @default_discard: default action for this mac (discard (1) / pass (0)).
- * @reserved1: reserved
- * @attached_filters: bitmap of relevant filters for this mac.
- */
-struct iwl_fw_bcast_mac {
-	u8 default_discard;
-	u8 reserved1;
-	__le16 attached_filters;
-} __packed; /* BCAST_MAC_CONTEXT_S_VER_1 */
-
-/**
- * struct iwl_bcast_filter_cmd - broadcast filtering configuration
- * @disable: enable (0) / disable (1)
- * @max_bcast_filters: max number of filters (MAX_BCAST_FILTERS)
- * @max_macs: max number of macs (NUM_MAC_INDEX_DRIVER)
- * @reserved1: reserved
- * @filters: broadcast filters
- * @macs: broadcast filtering configuration per-mac
- */
-struct iwl_bcast_filter_cmd {
-	u8 disable;
-	u8 max_bcast_filters;
-	u8 max_macs;
-	u8 reserved1;
-	struct iwl_fw_bcast_filter filters[MAX_BCAST_FILTERS];
-	struct iwl_fw_bcast_mac macs[NUM_MAC_INDEX_DRIVER];
-} __packed; /* BCAST_FILTERING_HCMD_API_S_VER_1 */
-
-/*
- * enum iwl_mvm_marker_id - maker ids
- *
- * The ids for different type of markers to insert into the usniffer logs
- */
-enum iwl_mvm_marker_id {
-	MARKER_ID_TX_FRAME_LATENCY = 1,
-}; /* MARKER_ID_API_E_VER_1 */
-
-/**
- * struct iwl_mvm_marker - mark info into the usniffer logs
- *
- * (MARKER_CMD = 0xcb)
- *
- * Mark the UTC time stamp into the usniffer logs together with additional
- * metadata, so the usniffer output can be parsed.
- * In the command response the ucode will return the GP2 time.
- *
- * @dw_len: The amount of dwords following this byte including this byte.
- * @marker_id: A unique marker id (iwl_mvm_marker_id).
- * @reserved: reserved.
- * @timestamp: in milliseconds since 1970-01-01 00:00:00 UTC
- * @metadata: additional meta data that will be written to the unsiffer log
- */
-struct iwl_mvm_marker {
-	u8 dw_len;
-	u8 marker_id;
-	__le16 reserved;
-	__le64 timestamp;
-	__le32 metadata[0];
-} __packed; /* MARKER_API_S_VER_1 */
-
-/*
- * enum iwl_dc2dc_config_id - flag ids
- *
- * Ids of dc2dc configuration flags
- */
-enum iwl_dc2dc_config_id {
-	DCDC_LOW_POWER_MODE_MSK_SET  = 0x1, /* not used */
-	DCDC_FREQ_TUNE_SET = 0x2,
-}; /* MARKER_ID_API_E_VER_1 */
-
-/**
- * struct iwl_dc2dc_config_cmd - configure dc2dc values
- *
- * (DC2DC_CONFIG_CMD = 0x83)
- *
- * Set/Get & configure dc2dc values.
- * The command always returns the current dc2dc values.
- *
- * @flags: set/get dc2dc
- * @enable_low_power_mode: not used.
- * @dc2dc_freq_tune0: frequency divider - digital domain
- * @dc2dc_freq_tune1: frequency divider - analog domain
- */
-struct iwl_dc2dc_config_cmd {
-	__le32 flags;
-	__le32 enable_low_power_mode; /* not used */
-	__le32 dc2dc_freq_tune0;
-	__le32 dc2dc_freq_tune1;
-} __packed; /* DC2DC_CONFIG_CMD_API_S_VER_1 */
-
-/**
- * struct iwl_dc2dc_config_resp - response for iwl_dc2dc_config_cmd
- *
- * Current dc2dc values returned by the FW.
- *
- * @dc2dc_freq_tune0: frequency divider - digital domain
- * @dc2dc_freq_tune1: frequency divider - analog domain
- */
-struct iwl_dc2dc_config_resp {
-	__le32 dc2dc_freq_tune0;
-	__le32 dc2dc_freq_tune1;
-} __packed; /* DC2DC_CONFIG_RESP_API_S_VER_1 */
-
-/***********************************
- * Smart Fifo API
- ***********************************/
-/* Smart Fifo state */
-enum iwl_sf_state {
-	SF_LONG_DELAY_ON = 0, /* should never be called by driver */
-	SF_FULL_ON,
-	SF_UNINIT,
-	SF_INIT_OFF,
-	SF_HW_NUM_STATES
-};
-
-/* Smart Fifo possible scenario */
-enum iwl_sf_scenario {
-	SF_SCENARIO_SINGLE_UNICAST,
-	SF_SCENARIO_AGG_UNICAST,
-	SF_SCENARIO_MULTICAST,
-	SF_SCENARIO_BA_RESP,
-	SF_SCENARIO_TX_RESP,
-	SF_NUM_SCENARIO
-};
-
-#define SF_TRANSIENT_STATES_NUMBER 2	/* SF_LONG_DELAY_ON and SF_FULL_ON */
-#define SF_NUM_TIMEOUT_TYPES 2		/* Aging timer and Idle timer */
-
-/* smart FIFO default values */
-#define SF_W_MARK_SISO 6144
-#define SF_W_MARK_MIMO2 8192
-#define SF_W_MARK_MIMO3 6144
-#define SF_W_MARK_LEGACY 4096
-#define SF_W_MARK_SCAN 4096
-
-/* SF Scenarios timers for default configuration (aligned to 32 uSec) */
-#define SF_SINGLE_UNICAST_IDLE_TIMER_DEF 160	/* 150 uSec  */
-#define SF_SINGLE_UNICAST_AGING_TIMER_DEF 400	/* 0.4 mSec */
-#define SF_AGG_UNICAST_IDLE_TIMER_DEF 160		/* 150 uSec */
-#define SF_AGG_UNICAST_AGING_TIMER_DEF 400		/* 0.4 mSec */
-#define SF_MCAST_IDLE_TIMER_DEF 160		/* 150 mSec */
-#define SF_MCAST_AGING_TIMER_DEF 400		/* 0.4 mSec */
-#define SF_BA_IDLE_TIMER_DEF 160			/* 150 uSec */
-#define SF_BA_AGING_TIMER_DEF 400			/* 0.4 mSec */
-#define SF_TX_RE_IDLE_TIMER_DEF 160			/* 150 uSec */
-#define SF_TX_RE_AGING_TIMER_DEF 400		/* 0.4 mSec */
-
-/* SF Scenarios timers for BSS MAC configuration (aligned to 32 uSec) */
-#define SF_SINGLE_UNICAST_IDLE_TIMER 320	/* 300 uSec  */
-#define SF_SINGLE_UNICAST_AGING_TIMER 2016	/* 2 mSec */
-#define SF_AGG_UNICAST_IDLE_TIMER 320		/* 300 uSec */
-#define SF_AGG_UNICAST_AGING_TIMER 2016		/* 2 mSec */
-#define SF_MCAST_IDLE_TIMER 2016		/* 2 mSec */
-#define SF_MCAST_AGING_TIMER 10016		/* 10 mSec */
-#define SF_BA_IDLE_TIMER 320			/* 300 uSec */
-#define SF_BA_AGING_TIMER 2016			/* 2 mSec */
-#define SF_TX_RE_IDLE_TIMER 320			/* 300 uSec */
-#define SF_TX_RE_AGING_TIMER 2016		/* 2 mSec */
-
-#define SF_LONG_DELAY_AGING_TIMER 1000000	/* 1 Sec */
-
-#define SF_CFG_DUMMY_NOTIF_OFF	BIT(16)
-
-/**
- * struct iwl_sf_cfg_cmd - Smart Fifo configuration command.
- * @state: smart fifo state, types listed in &enum iwl_sf_state.
- * @watermark: Minimum allowed availabe free space in RXF for transient state.
- * @long_delay_timeouts: aging and idle timer values for each scenario
- * in long delay state.
- * @full_on_timeouts: timer values for each scenario in full on state.
- */
-struct iwl_sf_cfg_cmd {
-	__le32 state;
-	__le32 watermark[SF_TRANSIENT_STATES_NUMBER];
-	__le32 long_delay_timeouts[SF_NUM_SCENARIO][SF_NUM_TIMEOUT_TYPES];
-	__le32 full_on_timeouts[SF_NUM_SCENARIO][SF_NUM_TIMEOUT_TYPES];
-} __packed; /* SF_CFG_API_S_VER_2 */
-
-/***********************************
- * Location Aware Regulatory (LAR) API - MCC updates
- ***********************************/
-
-/**
- * struct iwl_mcc_update_cmd_v1 - Request the device to update geographic
- * regulatory profile according to the given MCC (Mobile Country Code).
- * The MCC is two letter-code, ascii upper case[A-Z] or '00' for world domain.
- * 'ZZ' MCC will be used to switch to NVM default profile; in this case, the
- * MCC in the cmd response will be the relevant MCC in the NVM.
- * @mcc: given mobile country code
- * @source_id: the source from where we got the MCC, see iwl_mcc_source
- * @reserved: reserved for alignment
- */
-struct iwl_mcc_update_cmd_v1 {
-	__le16 mcc;
-	u8 source_id;
-	u8 reserved;
-} __packed; /* LAR_UPDATE_MCC_CMD_API_S_VER_1 */
-
-/**
- * struct iwl_mcc_update_cmd - Request the device to update geographic
- * regulatory profile according to the given MCC (Mobile Country Code).
- * The MCC is two letter-code, ascii upper case[A-Z] or '00' for world domain.
- * 'ZZ' MCC will be used to switch to NVM default profile; in this case, the
- * MCC in the cmd response will be the relevant MCC in the NVM.
- * @mcc: given mobile country code
- * @source_id: the source from where we got the MCC, see iwl_mcc_source
- * @reserved: reserved for alignment
- * @key: integrity key for MCC API OEM testing
- * @reserved2: reserved
- */
-struct iwl_mcc_update_cmd {
-	__le16 mcc;
-	u8 source_id;
-	u8 reserved;
-	__le32 key;
-	u8 reserved2[20];
-} __packed; /* LAR_UPDATE_MCC_CMD_API_S_VER_2 */
-
-/**
- * struct iwl_mcc_update_resp_v1  - response to MCC_UPDATE_CMD.
- * Contains the new channel control profile map, if changed, and the new MCC
- * (mobile country code).
- * The new MCC may be different than what was requested in MCC_UPDATE_CMD.
- * @status: see &enum iwl_mcc_update_status
- * @mcc: the new applied MCC
- * @cap: capabilities for all channels which matches the MCC
- * @source_id: the MCC source, see iwl_mcc_source
- * @n_channels: number of channels in @channels_data (may be 14, 39, 50 or 51
- *		channels, depending on platform)
- * @channels: channel control data map, DWORD for each channel. Only the first
- *	16bits are used.
- */
-struct iwl_mcc_update_resp_v1  {
-	__le32 status;
-	__le16 mcc;
-	u8 cap;
-	u8 source_id;
-	__le32 n_channels;
-	__le32 channels[0];
-} __packed; /* LAR_UPDATE_MCC_CMD_RESP_S_VER_1 */
-
-/**
- * struct iwl_mcc_update_resp - response to MCC_UPDATE_CMD.
- * Contains the new channel control profile map, if changed, and the new MCC
- * (mobile country code).
- * The new MCC may be different than what was requested in MCC_UPDATE_CMD.
- * @status: see &enum iwl_mcc_update_status
- * @mcc: the new applied MCC
- * @cap: capabilities for all channels which matches the MCC
- * @source_id: the MCC source, see iwl_mcc_source
- * @time: time elapsed from the MCC test start (in 30 seconds TU)
- * @reserved: reserved.
- * @n_channels: number of channels in @channels_data (may be 14, 39, 50 or 51
- *		channels, depending on platform)
- * @channels: channel control data map, DWORD for each channel. Only the first
- *	16bits are used.
- */
-struct iwl_mcc_update_resp {
-	__le32 status;
-	__le16 mcc;
-	u8 cap;
-	u8 source_id;
-	__le16 time;
-	__le16 reserved;
-	__le32 n_channels;
-	__le32 channels[0];
-} __packed; /* LAR_UPDATE_MCC_CMD_RESP_S_VER_2 */
-
-/**
- * struct iwl_mcc_chub_notif - chub notifies of mcc change
- * (MCC_CHUB_UPDATE_CMD = 0xc9)
- * The Chub (Communication Hub, CommsHUB) is a HW component that connects to
- * the cellular and connectivity cores that gets updates of the mcc, and
- * notifies the ucode directly of any mcc change.
- * The ucode requests the driver to request the device to update geographic
- * regulatory  profile according to the given MCC (Mobile Country Code).
- * The MCC is two letter-code, ascii upper case[A-Z] or '00' for world domain.
- * 'ZZ' MCC will be used to switch to NVM default profile; in this case, the
- * MCC in the cmd response will be the relevant MCC in the NVM.
- * @mcc: given mobile country code
- * @source_id: identity of the change originator, see iwl_mcc_source
- * @reserved1: reserved for alignment
- */
-struct iwl_mcc_chub_notif {
-	__le16 mcc;
-	u8 source_id;
-	u8 reserved1;
-} __packed; /* LAR_MCC_NOTIFY_S */
-
-enum iwl_mcc_update_status {
-	MCC_RESP_NEW_CHAN_PROFILE,
-	MCC_RESP_SAME_CHAN_PROFILE,
-	MCC_RESP_INVALID,
-	MCC_RESP_NVM_DISABLED,
-	MCC_RESP_ILLEGAL,
-	MCC_RESP_LOW_PRIORITY,
-	MCC_RESP_TEST_MODE_ACTIVE,
-	MCC_RESP_TEST_MODE_NOT_ACTIVE,
-	MCC_RESP_TEST_MODE_DENIAL_OF_SERVICE,
-};
-
-enum iwl_mcc_source {
-	MCC_SOURCE_OLD_FW = 0,
-	MCC_SOURCE_ME = 1,
-	MCC_SOURCE_BIOS = 2,
-	MCC_SOURCE_3G_LTE_HOST = 3,
-	MCC_SOURCE_3G_LTE_DEVICE = 4,
-	MCC_SOURCE_WIFI = 5,
-	MCC_SOURCE_RESERVED = 6,
-	MCC_SOURCE_DEFAULT = 7,
-	MCC_SOURCE_UNINITIALIZED = 8,
-	MCC_SOURCE_MCC_API = 9,
-	MCC_SOURCE_GET_CURRENT = 0x10,
-	MCC_SOURCE_GETTING_MCC_TEST_MODE = 0x11,
-};
-
-/* DTS measurements */
-
-enum iwl_dts_measurement_flags {
-	DTS_TRIGGER_CMD_FLAGS_TEMP	= BIT(0),
-	DTS_TRIGGER_CMD_FLAGS_VOLT	= BIT(1),
-};
-
-/**
- * struct iwl_dts_measurement_cmd - request DTS temp and/or voltage measurements
- *
- * @flags: indicates which measurements we want as specified in
- *	&enum iwl_dts_measurement_flags
- */
-struct iwl_dts_measurement_cmd {
-	__le32 flags;
-} __packed; /* TEMPERATURE_MEASUREMENT_TRIGGER_CMD_S */
-
-/**
-* enum iwl_dts_control_measurement_mode - DTS measurement type
-* @DTS_AUTOMATIC: Automatic mode (full SW control). Provide temperature read
-*                 back (latest value. Not waiting for new value). Use automatic
-*                 SW DTS configuration.
-* @DTS_REQUEST_READ: Request DTS read. Configure DTS with manual settings,
-*                    trigger DTS reading and provide read back temperature read
-*                    when available.
-* @DTS_OVER_WRITE: over-write the DTS temperatures in the SW until next read
-* @DTS_DIRECT_WITHOUT_MEASURE: DTS returns its latest temperature result,
-*                              without measurement trigger.
-*/
-enum iwl_dts_control_measurement_mode {
-	DTS_AUTOMATIC			= 0,
-	DTS_REQUEST_READ		= 1,
-	DTS_OVER_WRITE			= 2,
-	DTS_DIRECT_WITHOUT_MEASURE	= 3,
-};
-
-/**
-* enum iwl_dts_used - DTS to use or used for measurement in the DTS request
-* @DTS_USE_TOP: Top
-* @DTS_USE_CHAIN_A: chain A
-* @DTS_USE_CHAIN_B: chain B
-* @DTS_USE_CHAIN_C: chain C
-* @XTAL_TEMPERATURE: read temperature from xtal
-*/
-enum iwl_dts_used {
-	DTS_USE_TOP		= 0,
-	DTS_USE_CHAIN_A		= 1,
-	DTS_USE_CHAIN_B		= 2,
-	DTS_USE_CHAIN_C		= 3,
-	XTAL_TEMPERATURE	= 4,
-};
-
-/**
-* enum iwl_dts_bit_mode - bit-mode to use in DTS request read mode
-* @DTS_BIT6_MODE: bit 6 mode
-* @DTS_BIT8_MODE: bit 8 mode
-*/
-enum iwl_dts_bit_mode {
-	DTS_BIT6_MODE	= 0,
-	DTS_BIT8_MODE	= 1,
-};
-
-/**
- * struct iwl_ext_dts_measurement_cmd - request extended DTS temp measurements
- * @control_mode: see &enum iwl_dts_control_measurement_mode
- * @temperature: used when over write DTS mode is selected
- * @sensor: set temperature sensor to use. See &enum iwl_dts_used
- * @avg_factor: average factor to DTS in request DTS read mode
- * @bit_mode: value defines the DTS bit mode to use. See &enum iwl_dts_bit_mode
- * @step_duration: step duration for the DTS
- */
-struct iwl_ext_dts_measurement_cmd {
-	__le32 control_mode;
-	__le32 temperature;
-	__le32 sensor;
-	__le32 avg_factor;
-	__le32 bit_mode;
-	__le32 step_duration;
-} __packed; /* XVT_FW_DTS_CONTROL_MEASUREMENT_REQUEST_API_S */
-
-/**
- * struct iwl_dts_measurement_notif_v1 - measurements notification
- *
- * @temp: the measured temperature
- * @voltage: the measured voltage
- */
-struct iwl_dts_measurement_notif_v1 {
-	__le32 temp;
-	__le32 voltage;
-} __packed; /* TEMPERATURE_MEASUREMENT_TRIGGER_NTFY_S_VER_1*/
-
-/**
- * struct iwl_dts_measurement_notif_v2 - measurements notification
- *
- * @temp: the measured temperature
- * @voltage: the measured voltage
- * @threshold_idx: the trip index that was crossed
- */
-struct iwl_dts_measurement_notif_v2 {
-	__le32 temp;
-	__le32 voltage;
-	__le32 threshold_idx;
-} __packed; /* TEMPERATURE_MEASUREMENT_TRIGGER_NTFY_S_VER_2 */
-
-/**
- * struct ct_kill_notif - CT-kill entry notification
- *
- * @temperature: the current temperature in celsius
- * @reserved: reserved
- */
-struct ct_kill_notif {
-	__le16 temperature;
-	__le16 reserved;
-} __packed; /* GRP_PHY_CT_KILL_NTF */
-
-/**
-* enum ctdp_cmd_operation - CTDP command operations
-* @CTDP_CMD_OPERATION_START: update the current budget
-* @CTDP_CMD_OPERATION_STOP: stop ctdp
-* @CTDP_CMD_OPERATION_REPORT: get the average budget
-*/
-enum iwl_mvm_ctdp_cmd_operation {
-	CTDP_CMD_OPERATION_START	= 0x1,
-	CTDP_CMD_OPERATION_STOP		= 0x2,
-	CTDP_CMD_OPERATION_REPORT	= 0x4,
-};/* CTDP_CMD_OPERATION_TYPE_E */
-
-/**
- * struct iwl_mvm_ctdp_cmd - track and manage the FW power consumption budget
- *
- * @operation: see &enum iwl_mvm_ctdp_cmd_operation
- * @budget: the budget in milliwatt
- * @window_size: defined in API but not used
- */
-struct iwl_mvm_ctdp_cmd {
-	__le32 operation;
-	__le32 budget;
-	__le32 window_size;
-} __packed;
-
-#define IWL_MAX_DTS_TRIPS	8
-
-/**
- * struct temp_report_ths_cmd - set temperature thresholds
- *
- * @num_temps: number of temperature thresholds passed
- * @thresholds: array with the thresholds to be configured
- */
-struct temp_report_ths_cmd {
-	__le32 num_temps;
-	__le16 thresholds[IWL_MAX_DTS_TRIPS];
-} __packed; /* GRP_PHY_TEMP_REPORTING_THRESHOLDS_CMD */
-
-/***********************************
- * TDLS API
- ***********************************/
-
-/* Type of TDLS request */
-enum iwl_tdls_channel_switch_type {
-	TDLS_SEND_CHAN_SW_REQ = 0,
-	TDLS_SEND_CHAN_SW_RESP_AND_MOVE_CH,
-	TDLS_MOVE_CH,
-}; /* TDLS_STA_CHANNEL_SWITCH_CMD_TYPE_API_E_VER_1 */
-
-/**
- * struct iwl_tdls_channel_switch_timing - Switch timing in TDLS channel-switch
- * @frame_timestamp: GP2 timestamp of channel-switch request/response packet
- *	received from peer
- * @max_offchan_duration: What amount of microseconds out of a DTIM is given
- *	to the TDLS off-channel communication. For instance if the DTIM is
- *	200TU and the TDLS peer is to be given 25% of the time, the value
- *	given will be 50TU, or 50 * 1024 if translated into microseconds.
- * @switch_time: switch time the peer sent in its channel switch timing IE
- * @switch_timeout: switch timeout the peer sent in its channel switch timing IE
- */
-struct iwl_tdls_channel_switch_timing {
-	__le32 frame_timestamp; /* GP2 time of peer packet Rx */
-	__le32 max_offchan_duration; /* given in micro-seconds */
-	__le32 switch_time; /* given in micro-seconds */
-	__le32 switch_timeout; /* given in micro-seconds */
-} __packed; /* TDLS_STA_CHANNEL_SWITCH_TIMING_DATA_API_S_VER_1 */
-
-#define IWL_TDLS_CH_SW_FRAME_MAX_SIZE 200
-
-/**
- * struct iwl_tdls_channel_switch_frame - TDLS channel switch frame template
- *
- * A template representing a TDLS channel-switch request or response frame
- *
- * @switch_time_offset: offset to the channel switch timing IE in the template
- * @tx_cmd: Tx parameters for the frame
- * @data: frame data
- */
-struct iwl_tdls_channel_switch_frame {
-	__le32 switch_time_offset;
-	struct iwl_tx_cmd tx_cmd;
-	u8 data[IWL_TDLS_CH_SW_FRAME_MAX_SIZE];
-} __packed; /* TDLS_STA_CHANNEL_SWITCH_FRAME_API_S_VER_1 */
-
-/**
- * struct iwl_tdls_channel_switch_cmd - TDLS channel switch command
- *
- * The command is sent to initiate a channel switch and also in response to
- * incoming TDLS channel-switch request/response packets from remote peers.
- *
- * @switch_type: see &enum iwl_tdls_channel_switch_type
- * @peer_sta_id: station id of TDLS peer
- * @ci: channel we switch to
- * @timing: timing related data for command
- * @frame: channel-switch request/response template, depending to switch_type
- */
-struct iwl_tdls_channel_switch_cmd {
-	u8 switch_type;
-	__le32 peer_sta_id;
-	struct iwl_fw_channel_info ci;
-	struct iwl_tdls_channel_switch_timing timing;
-	struct iwl_tdls_channel_switch_frame frame;
-} __packed; /* TDLS_STA_CHANNEL_SWITCH_CMD_API_S_VER_1 */
-
-/**
- * struct iwl_tdls_channel_switch_notif - TDLS channel switch start notification
- *
- * @status: non-zero on success
- * @offchannel_duration: duration given in microseconds
- * @sta_id: peer currently performing the channel-switch with
- */
-struct iwl_tdls_channel_switch_notif {
-	__le32 status;
-	__le32 offchannel_duration;
-	__le32 sta_id;
-} __packed; /* TDLS_STA_CHANNEL_SWITCH_NTFY_API_S_VER_1 */
-
-/**
- * struct iwl_tdls_sta_info - TDLS station info
- *
- * @sta_id: station id of the TDLS peer
- * @tx_to_peer_tid: TID reserved vs. the peer for FW based Tx
- * @tx_to_peer_ssn: initial SSN the FW should use for Tx on its TID vs the peer
- * @is_initiator: 1 if the peer is the TDLS link initiator, 0 otherwise
- */
-struct iwl_tdls_sta_info {
-	u8 sta_id;
-	u8 tx_to_peer_tid;
-	__le16 tx_to_peer_ssn;
-	__le32 is_initiator;
-} __packed; /* TDLS_STA_INFO_VER_1 */
-
-/**
- * struct iwl_tdls_config_cmd - TDLS basic config command
- *
- * @id_and_color: MAC id and color being configured
- * @tdls_peer_count: amount of currently connected TDLS peers
- * @tx_to_ap_tid: TID reverved vs. the AP for FW based Tx
- * @tx_to_ap_ssn: initial SSN the FW should use for Tx on its TID vs. the AP
- * @sta_info: per-station info. Only the first tdls_peer_count entries are set
- * @pti_req_data_offset: offset of network-level data for the PTI template
- * @pti_req_tx_cmd: Tx parameters for PTI request template
- * @pti_req_template: PTI request template data
- */
-struct iwl_tdls_config_cmd {
-	__le32 id_and_color; /* mac id and color */
-	u8 tdls_peer_count;
-	u8 tx_to_ap_tid;
-	__le16 tx_to_ap_ssn;
-	struct iwl_tdls_sta_info sta_info[IWL_MVM_TDLS_STA_COUNT];
-
-	__le32 pti_req_data_offset;
-	struct iwl_tx_cmd pti_req_tx_cmd;
-	u8 pti_req_template[0];
-} __packed; /* TDLS_CONFIG_CMD_API_S_VER_1 */
-
-/**
- * struct iwl_tdls_config_sta_info_res - TDLS per-station config information
- *
- * @sta_id: station id of the TDLS peer
- * @tx_to_peer_last_seq: last sequence number used by FW during FW-based Tx to
- *	the peer
- */
-struct iwl_tdls_config_sta_info_res {
-	__le16 sta_id;
-	__le16 tx_to_peer_last_seq;
-} __packed; /* TDLS_STA_INFO_RSP_VER_1 */
-
-/**
- * struct iwl_tdls_config_res - TDLS config information from FW
- *
- * @tx_to_ap_last_seq: last sequence number used by FW during FW-based Tx to AP
- * @sta_info: per-station TDLS config information
- */
-struct iwl_tdls_config_res {
-	__le32 tx_to_ap_last_seq;
-	struct iwl_tdls_config_sta_info_res sta_info[IWL_MVM_TDLS_STA_COUNT];
-} __packed; /* TDLS_CONFIG_RSP_API_S_VER_1 */
-
-#define TX_FIFO_MAX_NUM_9000		8
-#define TX_FIFO_MAX_NUM			15
-#define RX_FIFO_MAX_NUM			2
-#define TX_FIFO_INTERNAL_MAX_NUM	6
-
-/**
- * struct iwl_shared_mem_cfg_v2 - Shared memory configuration information
- *
- * @shared_mem_addr: shared memory addr (pre 8000 HW set to 0x0 as MARBH is not
- *	accessible)
- * @shared_mem_size: shared memory size
- * @sample_buff_addr: internal sample (mon/adc) buff addr (pre 8000 HW set to
- *	0x0 as accessible only via DBGM RDAT)
- * @sample_buff_size: internal sample buff size
- * @txfifo_addr: start addr of TXF0 (excluding the context table 0.5KB), (pre
- *	8000 HW set to 0x0 as not accessible)
- * @txfifo_size: size of TXF0 ... TXF7
- * @rxfifo_size: RXF1, RXF2 sizes. If there is no RXF2, it'll have a value of 0
- * @page_buff_addr: used by UMAC and performance debug (page miss analysis),
- *	when paging is not supported this should be 0
- * @page_buff_size: size of %page_buff_addr
- * @rxfifo_addr: Start address of rxFifo
- * @internal_txfifo_addr: start address of internalFifo
- * @internal_txfifo_size: internal fifos' size
- *
- * NOTE: on firmware that don't have IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG
- *	 set, the last 3 members don't exist.
- */
-struct iwl_shared_mem_cfg_v2 {
-	__le32 shared_mem_addr;
-	__le32 shared_mem_size;
-	__le32 sample_buff_addr;
-	__le32 sample_buff_size;
-	__le32 txfifo_addr;
-	__le32 txfifo_size[TX_FIFO_MAX_NUM_9000];
-	__le32 rxfifo_size[RX_FIFO_MAX_NUM];
-	__le32 page_buff_addr;
-	__le32 page_buff_size;
-	__le32 rxfifo_addr;
-	__le32 internal_txfifo_addr;
-	__le32 internal_txfifo_size[TX_FIFO_INTERNAL_MAX_NUM];
-} __packed; /* SHARED_MEM_ALLOC_API_S_VER_2 */
-
-/**
- * struct iwl_shared_mem_lmac_cfg - LMAC shared memory configuration
- *
- * @txfifo_addr: start addr of TXF0 (excluding the context table 0.5KB)
- * @txfifo_size: size of TX FIFOs
- * @rxfifo1_addr: RXF1 addr
- * @rxfifo1_size: RXF1 size
- */
-struct iwl_shared_mem_lmac_cfg {
-	__le32 txfifo_addr;
-	__le32 txfifo_size[TX_FIFO_MAX_NUM];
-	__le32 rxfifo1_addr;
-	__le32 rxfifo1_size;
-
-} __packed; /* SHARED_MEM_ALLOC_LMAC_API_S_VER_1 */
-
-/**
- * struct iwl_shared_mem_cfg - Shared memory configuration information
- *
- * @shared_mem_addr: shared memory address
- * @shared_mem_size: shared memory size
- * @sample_buff_addr: internal sample (mon/adc) buff addr
- * @sample_buff_size: internal sample buff size
- * @rxfifo2_addr: start addr of RXF2
- * @rxfifo2_size: size of RXF2
- * @page_buff_addr: used by UMAC and performance debug (page miss analysis),
- *	when paging is not supported this should be 0
- * @page_buff_size: size of %page_buff_addr
- * @lmac_num: number of LMACs (1 or 2)
- * @lmac_smem: per - LMAC smem data
- */
-struct iwl_shared_mem_cfg {
-	__le32 shared_mem_addr;
-	__le32 shared_mem_size;
-	__le32 sample_buff_addr;
-	__le32 sample_buff_size;
-	__le32 rxfifo2_addr;
-	__le32 rxfifo2_size;
-	__le32 page_buff_addr;
-	__le32 page_buff_size;
-	__le32 lmac_num;
-	struct iwl_shared_mem_lmac_cfg lmac_smem[2];
-} __packed; /* SHARED_MEM_ALLOC_API_S_VER_3 */
-
-/**
- * struct iwl_mu_group_mgmt_cmd - VHT MU-MIMO group configuration
- *
- * @reserved: reserved
- * @membership_status: a bitmap of MU groups
- * @user_position:the position of station in a group. If the station is in the
- *	group then bits (group * 2) is the position -1
- */
-struct iwl_mu_group_mgmt_cmd {
-	__le32 reserved;
-	__le32 membership_status[2];
-	__le32 user_position[4];
-} __packed; /* MU_GROUP_ID_MNG_TABLE_API_S_VER_1 */
-
-/**
- * struct iwl_mu_group_mgmt_notif - VHT MU-MIMO group id notification
- *
- * @membership_status: a bitmap of MU groups
- * @user_position: the position of station in a group. If the station is in the
- *	group then bits (group * 2) is the position -1
- */
-struct iwl_mu_group_mgmt_notif {
-	__le32 membership_status[2];
-	__le32 user_position[4];
-} __packed; /* MU_GROUP_MNG_NTFY_API_S_VER_1 */
-
-#define MAX_STORED_BEACON_SIZE 600
-
-/**
- * struct iwl_stored_beacon_notif - Stored beacon notification
- *
- * @system_time: system time on air rise
- * @tsf: TSF on air rise
- * @beacon_timestamp: beacon on air rise
- * @band: band, matches &RX_RES_PHY_FLAGS_BAND_24 definition
- * @channel: channel this beacon was received on
- * @rates: rate in ucode internal format
- * @byte_count: frame's byte count
- * @data: beacon data, length in @byte_count
- */
-struct iwl_stored_beacon_notif {
-	__le32 system_time;
-	__le64 tsf;
-	__le32 beacon_timestamp;
-	__le16 band;
-	__le16 channel;
-	__le32 rates;
-	__le32 byte_count;
-	u8 data[MAX_STORED_BEACON_SIZE];
-} __packed; /* WOWLAN_STROED_BEACON_INFO_S_VER_2 */
-
-#define LQM_NUMBER_OF_STATIONS_IN_REPORT 16
-
-enum iwl_lqm_cmd_operatrions {
-	LQM_CMD_OPERATION_START_MEASUREMENT = 0x01,
-	LQM_CMD_OPERATION_STOP_MEASUREMENT = 0x02,
-};
-
-enum iwl_lqm_status {
-	LQM_STATUS_SUCCESS = 0,
-	LQM_STATUS_TIMEOUT = 1,
-	LQM_STATUS_ABORT = 2,
-};
-
-/**
- * struct iwl_link_qual_msrmnt_cmd - Link Quality Measurement command
- * @cmd_operation: command operation to be performed (start or stop)
- *	as defined above.
- * @mac_id: MAC ID the measurement applies to.
- * @measurement_time: time of the total measurement to be performed, in uSec.
- * @timeout: maximum time allowed until a response is sent, in uSec.
- */
-struct iwl_link_qual_msrmnt_cmd {
-	__le32 cmd_operation;
-	__le32 mac_id;
-	__le32 measurement_time;
-	__le32 timeout;
-} __packed /* LQM_CMD_API_S_VER_1 */;
-
-/**
- * struct iwl_link_qual_msrmnt_notif - Link Quality Measurement notification
- *
- * @frequent_stations_air_time: an array containing the total air time
- *	(in uSec) used by the most frequently transmitting stations.
- * @number_of_stations: the number of uniqe stations included in the array
- *	(a number between 0 to 16)
- * @total_air_time_other_stations: the total air time (uSec) used by all the
- *	stations which are not included in the above report.
- * @time_in_measurement_window: the total time in uSec in which a measurement
- *	took place.
- * @tx_frame_dropped: the number of TX frames dropped due to retry limit during
- *	measurement
- * @mac_id: MAC ID the measurement applies to.
- * @status: return status. may be one of the LQM_STATUS_* defined above.
- * @reserved: reserved.
- */
-struct iwl_link_qual_msrmnt_notif {
-	__le32 frequent_stations_air_time[LQM_NUMBER_OF_STATIONS_IN_REPORT];
-	__le32 number_of_stations;
-	__le32 total_air_time_other_stations;
-	__le32 time_in_measurement_window;
-	__le32 tx_frame_dropped;
-	__le32 mac_id;
-	__le32 status;
-	u8 reserved[12];
-} __packed; /* LQM_MEASUREMENT_COMPLETE_NTF_API_S_VER1 */
-
-/**
- * struct iwl_channel_switch_noa_notif - Channel switch NOA notification
- *
- * @id_and_color: ID and color of the MAC
- */
-struct iwl_channel_switch_noa_notif {
-	__le32 id_and_color;
-} __packed; /* CHANNEL_SWITCH_START_NTFY_API_S_VER_1 */
-
-/* Operation types for the debug mem access */
-enum {
-	DEBUG_MEM_OP_READ = 0,
-	DEBUG_MEM_OP_WRITE = 1,
-	DEBUG_MEM_OP_WRITE_BYTES = 2,
-};
-
-#define DEBUG_MEM_MAX_SIZE_DWORDS 32
-
-/**
- * struct iwl_dbg_mem_access_cmd - Request the device to read/write memory
- * @op: DEBUG_MEM_OP_*
- * @addr: address to read/write from/to
- * @len: in dwords, to read/write
- * @data: for write opeations, contains the source buffer
- */
-struct iwl_dbg_mem_access_cmd {
-	__le32 op;
-	__le32 addr;
-	__le32 len;
-	__le32 data[];
-} __packed; /* DEBUG_(U|L)MAC_RD_WR_CMD_API_S_VER_1 */
-
-/* Status responses for the debug mem access */
-enum {
-	DEBUG_MEM_STATUS_SUCCESS = 0x0,
-	DEBUG_MEM_STATUS_FAILED = 0x1,
-	DEBUG_MEM_STATUS_LOCKED = 0x2,
-	DEBUG_MEM_STATUS_HIDDEN = 0x3,
-	DEBUG_MEM_STATUS_LENGTH = 0x4,
-};
-
-/**
- * struct iwl_dbg_mem_access_rsp - Response to debug mem commands
- * @status: DEBUG_MEM_STATUS_*
- * @len: read dwords (0 for write operations)
- * @data: contains the read DWs
- */
-struct iwl_dbg_mem_access_rsp {
-	__le32 status;
-	__le32 len;
-	__le32 data[];
-} __packed; /* DEBUG_(U|L)MAC_RD_WR_RSP_API_S_VER_1 */
-
-/**
- * struct iwl_nvm_access_complete_cmd - NVM_ACCESS commands are completed
- * @reserved: reserved
- */
-struct iwl_nvm_access_complete_cmd {
-	__le32 reserved;
-} __packed; /* NVM_ACCESS_COMPLETE_CMD_API_S_VER_1 */
-
-/**
- * enum iwl_extended_cfg_flag - commands driver may send before
- *	finishing init flow
- * @IWL_INIT_DEBUG_CFG: driver is going to send debug config command
- * @IWL_INIT_NVM: driver is going to send NVM_ACCESS commands
- * @IWL_INIT_PHY: driver is going to send PHY_DB commands
- */
-enum iwl_extended_cfg_flags {
-	IWL_INIT_DEBUG_CFG,
-	IWL_INIT_NVM,
-	IWL_INIT_PHY,
-};
-
-/**
- * struct iwl_extended_cfg_cmd - mark what commands ucode should wait for
- * before finishing init flows
- * @init_flags: values from iwl_extended_cfg_flags
- */
-struct iwl_init_extended_cfg_cmd {
-	__le32 init_flags;
-} __packed; /* INIT_EXTENDED_CFG_CMD_API_S_VER_1 */
-
-/*
- * struct iwl_nvm_get_info - request to get NVM data
- */
-struct iwl_nvm_get_info {
-	__le32 reserved;
-} __packed; /* GRP_REGULATORY_NVM_GET_INFO_CMD_S_VER_1 */
-
-/**
- * struct iwl_nvm_get_info_general - general NVM data
- * @flags: 1 - empty, 0 - valid
- * @nvm_version: nvm version
- * @board_type: board type
- * @reserved: reserved
- */
-struct iwl_nvm_get_info_general {
-	__le32 flags;
-	__le16 nvm_version;
-	u8 board_type;
-	u8 reserved;
-} __packed; /* GRP_REGULATORY_NVM_GET_INFO_GENERAL_S_VER_1 */
-
-/**
- * struct iwl_nvm_get_info_sku - mac information
- * @enable_24g: band 2.4G enabled
- * @enable_5g: band 5G enabled
- * @enable_11n: 11n enabled
- * @enable_11ac: 11ac enabled
- * @mimo_disable: MIMO enabled
- * @ext_crypto: Extended crypto enabled
- */
-struct iwl_nvm_get_info_sku {
-	__le32 enable_24g;
-	__le32 enable_5g;
-	__le32 enable_11n;
-	__le32 enable_11ac;
-	__le32 mimo_disable;
-	__le32 ext_crypto;
-} __packed; /* GRP_REGULATORY_NVM_GET_INFO_MAC_SKU_SECTION_S_VER_1 */
-
-/**
- * struct iwl_nvm_get_info_phy - phy information
- * @tx_chains: BIT 0 chain A, BIT 1 chain B
- * @rx_chains: BIT 0 chain A, BIT 1 chain B
- */
-struct iwl_nvm_get_info_phy {
-	__le32 tx_chains;
-	__le32 rx_chains;
-} __packed; /* GRP_REGULATORY_NVM_GET_INFO_PHY_SKU_SECTION_S_VER_1 */
-
-#define IWL_NUM_CHANNELS (51)
-
-/**
- * struct iwl_nvm_get_info_regulatory - regulatory information
- * @lar_enabled: is LAR enabled
- * @channel_profile: regulatory data of this channel
- * @reserved: reserved
- */
-struct iwl_nvm_get_info_regulatory {
-	__le32 lar_enabled;
-	__le16 channel_profile[IWL_NUM_CHANNELS];
-	__le16 reserved;
-} __packed; /* GRP_REGULATORY_NVM_GET_INFO_REGULATORY_S_VER_1 */
-
-/**
- * struct iwl_nvm_get_info_rsp - response to get NVM data
- * @general: general NVM data
- * @mac_sku: data relating to MAC sku
- * @phy_sku: data relating to PHY sku
- * @regulatory: regulatory data
- */
-struct iwl_nvm_get_info_rsp {
-	struct iwl_nvm_get_info_general general;
-	struct iwl_nvm_get_info_sku mac_sku;
-	struct iwl_nvm_get_info_phy phy_sku;
-	struct iwl_nvm_get_info_regulatory regulatory;
-} __packed; /* GRP_REGULATORY_NVM_GET_INFO_CMD_RSP_S_VER_1 */
-
-/**
- * struct iwl_mvm_antenna_coupling_notif - antenna coupling notification
- * @isolation: antenna isolation value
- */
-struct iwl_mvm_antenna_coupling_notif {
-	__le32 isolation;
-} __packed;
+#include "fw/api/tdls.h"
+#include "fw/api/mac-cfg.h"
+#include "fw/api/offload.h"
+#include "fw/api/context.h"
+#include "fw/api/time-event.h"
+#include "fw/api/datapath.h"
+#include "fw/api/phy.h"
+#include "fw/api/config.h"
+#include "fw/api/alive.h"
+#include "fw/api/binding.h"
+#include "fw/api/cmdhdr.h"
+#include "fw/api/coex.h"
+#include "fw/api/commands.h"
+#include "fw/api/d3.h"
+#include "fw/api/filter.h"
+#include "fw/api/led.h"
+#include "fw/api/mac.h"
+#include "fw/api/nvm-reg.h"
+#include "fw/api/phy-ctxt.h"
+#include "fw/api/power.h"
+#include "fw/api/rs.h"
+#include "fw/api/rx.h"
+#include "fw/api/scan.h"
+#include "fw/api/sf.h"
+#include "fw/api/sta.h"
+#include "fw/api/stats.h"
+#include "fw/api/tof.h"
+#include "fw/api/tx.h"
 
 #endif /* __fw_api_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
index 79e7a7a285dc..83485493a79a 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
@@ -78,7 +78,7 @@
 #include "iwl-eeprom-parse.h"
 
 #include "mvm.h"
-#include "fw-dbg.h"
+#include "fw/dbg.h"
 #include "iwl-phy-db.h"
 
 #define MVM_UCODE_ALIVE_TIMEOUT	HZ
@@ -144,134 +144,6 @@ static int iwl_mvm_send_dqa_cmd(struct iwl_mvm *mvm)
 	return ret;
 }
 
-void iwl_free_fw_paging(struct iwl_mvm *mvm)
-{
-	int i;
-
-	if (!mvm->fw_paging_db[0].fw_paging_block)
-		return;
-
-	for (i = 0; i < NUM_OF_FW_PAGING_BLOCKS; i++) {
-		struct iwl_fw_paging *paging = &mvm->fw_paging_db[i];
-
-		if (!paging->fw_paging_block) {
-			IWL_DEBUG_FW(mvm,
-				     "Paging: block %d already freed, continue to next page\n",
-				     i);
-
-			continue;
-		}
-		dma_unmap_page(mvm->trans->dev, paging->fw_paging_phys,
-			       paging->fw_paging_size, DMA_BIDIRECTIONAL);
-
-		__free_pages(paging->fw_paging_block,
-			     get_order(paging->fw_paging_size));
-		paging->fw_paging_block = NULL;
-	}
-	kfree(mvm->trans->paging_download_buf);
-	mvm->trans->paging_download_buf = NULL;
-	mvm->trans->paging_db = NULL;
-
-	memset(mvm->fw_paging_db, 0, sizeof(mvm->fw_paging_db));
-}
-
-static int iwl_fill_paging_mem(struct iwl_mvm *mvm, const struct fw_img *image)
-{
-	int sec_idx, idx;
-	u32 offset = 0;
-
-	/*
-	 * find where is the paging image start point:
-	 * if CPU2 exist and it's in paging format, then the image looks like:
-	 * CPU1 sections (2 or more)
-	 * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between CPU1 to CPU2
-	 * CPU2 sections (not paged)
-	 * PAGING_SEPARATOR_SECTION delimiter - separate between CPU2
-	 * non paged to CPU2 paging sec
-	 * CPU2 paging CSS
-	 * CPU2 paging image (including instruction and data)
-	 */
-	for (sec_idx = 0; sec_idx < image->num_sec; sec_idx++) {
-		if (image->sec[sec_idx].offset == PAGING_SEPARATOR_SECTION) {
-			sec_idx++;
-			break;
-		}
-	}
-
-	/*
-	 * If paging is enabled there should be at least 2 more sections left
-	 * (one for CSS and one for Paging data)
-	 */
-	if (sec_idx >= image->num_sec - 1) {
-		IWL_ERR(mvm, "Paging: Missing CSS and/or paging sections\n");
-		iwl_free_fw_paging(mvm);
-		return -EINVAL;
-	}
-
-	/* copy the CSS block to the dram */
-	IWL_DEBUG_FW(mvm, "Paging: load paging CSS to FW, sec = %d\n",
-		     sec_idx);
-
-	memcpy(page_address(mvm->fw_paging_db[0].fw_paging_block),
-	       image->sec[sec_idx].data,
-	       mvm->fw_paging_db[0].fw_paging_size);
-	dma_sync_single_for_device(mvm->trans->dev,
-				   mvm->fw_paging_db[0].fw_paging_phys,
-				   mvm->fw_paging_db[0].fw_paging_size,
-				   DMA_BIDIRECTIONAL);
-
-	IWL_DEBUG_FW(mvm,
-		     "Paging: copied %d CSS bytes to first block\n",
-		     mvm->fw_paging_db[0].fw_paging_size);
-
-	sec_idx++;
-
-	/*
-	 * copy the paging blocks to the dram
-	 * loop index start from 1 since that CSS block already copied to dram
-	 * and CSS index is 0.
-	 * loop stop at num_of_paging_blk since that last block is not full.
-	 */
-	for (idx = 1; idx < mvm->num_of_paging_blk; idx++) {
-		struct iwl_fw_paging *block = &mvm->fw_paging_db[idx];
-
-		memcpy(page_address(block->fw_paging_block),
-		       image->sec[sec_idx].data + offset,
-		       block->fw_paging_size);
-		dma_sync_single_for_device(mvm->trans->dev,
-					   block->fw_paging_phys,
-					   block->fw_paging_size,
-					   DMA_BIDIRECTIONAL);
-
-
-		IWL_DEBUG_FW(mvm,
-			     "Paging: copied %d paging bytes to block %d\n",
-			     mvm->fw_paging_db[idx].fw_paging_size,
-			     idx);
-
-		offset += mvm->fw_paging_db[idx].fw_paging_size;
-	}
-
-	/* copy the last paging block */
-	if (mvm->num_of_pages_in_last_blk > 0) {
-		struct iwl_fw_paging *block = &mvm->fw_paging_db[idx];
-
-		memcpy(page_address(block->fw_paging_block),
-		       image->sec[sec_idx].data + offset,
-		       FW_PAGING_SIZE * mvm->num_of_pages_in_last_blk);
-		dma_sync_single_for_device(mvm->trans->dev,
-					   block->fw_paging_phys,
-					   block->fw_paging_size,
-					   DMA_BIDIRECTIONAL);
-
-		IWL_DEBUG_FW(mvm,
-			     "Paging: copied %d pages in the last block %d\n",
-			     mvm->num_of_pages_in_last_blk, idx);
-	}
-
-	return 0;
-}
-
 void iwl_mvm_mfu_assert_dump_notif(struct iwl_mvm *mvm,
 				   struct iwl_rx_cmd_buffer *rxb)
 {
@@ -293,178 +165,6 @@ void iwl_mvm_mfu_assert_dump_notif(struct iwl_mvm *mvm,
 			       le32_to_cpu(dump_data[i]));
 }
 
-static int iwl_alloc_fw_paging_mem(struct iwl_mvm *mvm,
-				   const struct fw_img *image)
-{
-	struct page *block;
-	dma_addr_t phys = 0;
-	int blk_idx, order, num_of_pages, size, dma_enabled;
-
-	if (mvm->fw_paging_db[0].fw_paging_block)
-		return 0;
-
-	dma_enabled = is_device_dma_capable(mvm->trans->dev);
-
-	/* ensure BLOCK_2_EXP_SIZE is power of 2 of PAGING_BLOCK_SIZE */
-	BUILD_BUG_ON(BIT(BLOCK_2_EXP_SIZE) != PAGING_BLOCK_SIZE);
-
-	num_of_pages = image->paging_mem_size / FW_PAGING_SIZE;
-	mvm->num_of_paging_blk =
-		DIV_ROUND_UP(num_of_pages, NUM_OF_PAGE_PER_GROUP);
-	mvm->num_of_pages_in_last_blk =
-		num_of_pages -
-		NUM_OF_PAGE_PER_GROUP * (mvm->num_of_paging_blk - 1);
-
-	IWL_DEBUG_FW(mvm,
-		     "Paging: allocating mem for %d paging blocks, each block holds 8 pages, last block holds %d pages\n",
-		     mvm->num_of_paging_blk,
-		     mvm->num_of_pages_in_last_blk);
-
-	/*
-	 * Allocate CSS and paging blocks in dram.
-	 */
-	for (blk_idx = 0; blk_idx < mvm->num_of_paging_blk + 1; blk_idx++) {
-		/* For CSS allocate 4KB, for others PAGING_BLOCK_SIZE (32K) */
-		size = blk_idx ? PAGING_BLOCK_SIZE : FW_PAGING_SIZE;
-		order = get_order(size);
-		block = alloc_pages(GFP_KERNEL, order);
-		if (!block) {
-			/* free all the previous pages since we failed */
-			iwl_free_fw_paging(mvm);
-			return -ENOMEM;
-		}
-
-		mvm->fw_paging_db[blk_idx].fw_paging_block = block;
-		mvm->fw_paging_db[blk_idx].fw_paging_size = size;
-
-		if (dma_enabled) {
-			phys = dma_map_page(mvm->trans->dev, block, 0,
-					    PAGE_SIZE << order,
-					    DMA_BIDIRECTIONAL);
-			if (dma_mapping_error(mvm->trans->dev, phys)) {
-				/*
-				 * free the previous pages and the current one
-				 * since we failed to map_page.
-				 */
-				iwl_free_fw_paging(mvm);
-				return -ENOMEM;
-			}
-			mvm->fw_paging_db[blk_idx].fw_paging_phys = phys;
-		} else {
-			mvm->fw_paging_db[blk_idx].fw_paging_phys =
-				PAGING_ADDR_SIG |
-				blk_idx << BLOCK_2_EXP_SIZE;
-		}
-
-		if (!blk_idx)
-			IWL_DEBUG_FW(mvm,
-				     "Paging: allocated 4K(CSS) bytes (order %d) for firmware paging.\n",
-				     order);
-		else
-			IWL_DEBUG_FW(mvm,
-				     "Paging: allocated 32K bytes (order %d) for firmware paging.\n",
-				     order);
-	}
-
-	return 0;
-}
-
-static int iwl_save_fw_paging(struct iwl_mvm *mvm,
-			      const struct fw_img *fw)
-{
-	int ret;
-
-	ret = iwl_alloc_fw_paging_mem(mvm, fw);
-	if (ret)
-		return ret;
-
-	return iwl_fill_paging_mem(mvm, fw);
-}
-
-/* send paging cmd to FW in case CPU2 has paging image */
-static int iwl_send_paging_cmd(struct iwl_mvm *mvm, const struct fw_img *fw)
-{
-	struct iwl_fw_paging_cmd paging_cmd = {
-		.flags = cpu_to_le32(PAGING_CMD_IS_SECURED |
-				     PAGING_CMD_IS_ENABLED |
-				     (mvm->num_of_pages_in_last_blk <<
-				      PAGING_CMD_NUM_OF_PAGES_IN_LAST_GRP_POS)),
-		.block_size = cpu_to_le32(BLOCK_2_EXP_SIZE),
-		.block_num = cpu_to_le32(mvm->num_of_paging_blk),
-	};
-	int blk_idx;
-
-	/* loop for for all paging blocks + CSS block */
-	for (blk_idx = 0; blk_idx < mvm->num_of_paging_blk + 1; blk_idx++) {
-		dma_addr_t addr = mvm->fw_paging_db[blk_idx].fw_paging_phys;
-		__le32 phy_addr;
-
-		addr = addr >> PAGE_2_EXP_SIZE;
-		phy_addr = cpu_to_le32(addr);
-		paging_cmd.device_phy_addr[blk_idx] = phy_addr;
-	}
-
-	return iwl_mvm_send_cmd_pdu(mvm, iwl_cmd_id(FW_PAGING_BLOCK_CMD,
-						    IWL_ALWAYS_LONG_GROUP, 0),
-				    0, sizeof(paging_cmd), &paging_cmd);
-}
-
-/*
- * Send paging item cmd to FW in case CPU2 has paging image
- */
-static int iwl_trans_get_paging_item(struct iwl_mvm *mvm)
-{
-	int ret;
-	struct iwl_fw_get_item_cmd fw_get_item_cmd = {
-		.item_id = cpu_to_le32(IWL_FW_ITEM_ID_PAGING),
-	};
-
-	struct iwl_fw_get_item_resp *item_resp;
-	struct iwl_host_cmd cmd = {
-		.id = iwl_cmd_id(FW_GET_ITEM_CMD, IWL_ALWAYS_LONG_GROUP, 0),
-		.flags = CMD_WANT_SKB | CMD_SEND_IN_RFKILL,
-		.data = { &fw_get_item_cmd, },
-	};
-
-	cmd.len[0] = sizeof(struct iwl_fw_get_item_cmd);
-
-	ret = iwl_mvm_send_cmd(mvm, &cmd);
-	if (ret) {
-		IWL_ERR(mvm,
-			"Paging: Failed to send FW_GET_ITEM_CMD cmd (err = %d)\n",
-			ret);
-		return ret;
-	}
-
-	item_resp = (void *)((struct iwl_rx_packet *)cmd.resp_pkt)->data;
-	if (item_resp->item_id != cpu_to_le32(IWL_FW_ITEM_ID_PAGING)) {
-		IWL_ERR(mvm,
-			"Paging: got wrong item in FW_GET_ITEM_CMD resp (item_id = %u)\n",
-			le32_to_cpu(item_resp->item_id));
-		ret = -EIO;
-		goto exit;
-	}
-
-	/* Add an extra page for headers */
-	mvm->trans->paging_download_buf = kzalloc(PAGING_BLOCK_SIZE +
-						  FW_PAGING_SIZE,
-						  GFP_KERNEL);
-	if (!mvm->trans->paging_download_buf) {
-		ret = -ENOMEM;
-		goto exit;
-	}
-	mvm->trans->paging_req_addr = le32_to_cpu(item_resp->item_val);
-	mvm->trans->paging_db = mvm->fw_paging_db;
-	IWL_DEBUG_FW(mvm,
-		     "Paging: got paging request address (paging_req_addr 0x%08x)\n",
-		     mvm->trans->paging_req_addr);
-
-exit:
-	iwl_free_resp(&cmd);
-
-	return ret;
-}
-
 static bool iwl_alive_fn(struct iwl_notif_wait_data *notif_wait,
 			 struct iwl_rx_packet *pkt, void *data)
 {
@@ -544,48 +244,6 @@ static bool iwl_wait_phy_db_entry(struct iwl_notif_wait_data *notif_wait,
 	return false;
 }
 
-static int iwl_mvm_init_paging(struct iwl_mvm *mvm)
-{
-	const struct fw_img *fw = &mvm->fw->img[mvm->cur_ucode];
-	int ret;
-
-	/*
-	 * Configure and operate fw paging mechanism.
-	 * The driver configures the paging flow only once.
-	 * The CPU2 paging image is included in the IWL_UCODE_INIT image.
-	 */
-	if (!fw->paging_mem_size)
-		return 0;
-
-	/*
-	 * When dma is not enabled, the driver needs to copy / write
-	 * the downloaded / uploaded page to / from the smem.
-	 * This gets the location of the place were the pages are
-	 * stored.
-	 */
-	if (!is_device_dma_capable(mvm->trans->dev)) {
-		ret = iwl_trans_get_paging_item(mvm);
-		if (ret) {
-			IWL_ERR(mvm, "failed to get FW paging item\n");
-			return ret;
-		}
-	}
-
-	ret = iwl_save_fw_paging(mvm, fw);
-	if (ret) {
-		IWL_ERR(mvm, "failed to save the FW paging image\n");
-		return ret;
-	}
-
-	ret = iwl_send_paging_cmd(mvm, fw);
-	if (ret) {
-		IWL_ERR(mvm, "failed to send the paging cmd\n");
-		iwl_free_fw_paging(mvm);
-		return ret;
-	}
-
-	return 0;
-}
 static int iwl_mvm_load_ucode_wait_alive(struct iwl_mvm *mvm,
 					 enum iwl_ucode_type ucode_type)
 {
@@ -593,7 +251,7 @@ static int iwl_mvm_load_ucode_wait_alive(struct iwl_mvm *mvm,
 	struct iwl_mvm_alive_data alive_data;
 	const struct fw_img *fw;
 	int ret, i;
-	enum iwl_ucode_type old_type = mvm->cur_ucode;
+	enum iwl_ucode_type old_type = mvm->fwrt.cur_fw_img;
 	static const u16 alive_cmd[] = { MVM_ALIVE };
 	struct iwl_sf_region st_fwrd_space;
 
@@ -606,7 +264,7 @@ static int iwl_mvm_load_ucode_wait_alive(struct iwl_mvm *mvm,
 		fw = iwl_get_ucode_image(mvm->fw, ucode_type);
 	if (WARN_ON(!fw))
 		return -EINVAL;
-	mvm->cur_ucode = ucode_type;
+	iwl_fw_set_current_image(&mvm->fwrt, ucode_type);
 	clear_bit(IWL_MVM_STATUS_FIRMWARE_RUNNING, &mvm->status);
 
 	iwl_init_notification_wait(&mvm->notif_wait, &alive_wait,
@@ -615,7 +273,7 @@ static int iwl_mvm_load_ucode_wait_alive(struct iwl_mvm *mvm,
 
 	ret = iwl_trans_start_fw(mvm->trans, fw, ucode_type == IWL_UCODE_INIT);
 	if (ret) {
-		mvm->cur_ucode = old_type;
+		iwl_fw_set_current_image(&mvm->fwrt, old_type);
 		iwl_remove_notification(&mvm->notif_wait, &alive_wait);
 		return ret;
 	}
@@ -639,13 +297,13 @@ static int iwl_mvm_load_ucode_wait_alive(struct iwl_mvm *mvm,
 				"SecBoot CPU1 Status: 0x%x, CPU2 Status: 0x%x\n",
 				iwl_read_prph(trans, SB_CPU_1_STATUS),
 				iwl_read_prph(trans, SB_CPU_2_STATUS));
-		mvm->cur_ucode = old_type;
+		iwl_fw_set_current_image(&mvm->fwrt, old_type);
 		return ret;
 	}
 
 	if (!alive_data.valid) {
 		IWL_ERR(mvm, "Loaded ucode is not valid!\n");
-		mvm->cur_ucode = old_type;
+		iwl_fw_set_current_image(&mvm->fwrt, old_type);
 		return -EIO;
 	}
 
@@ -673,10 +331,7 @@ static int iwl_mvm_load_ucode_wait_alive(struct iwl_mvm *mvm,
 	 */
 
 	memset(&mvm->queue_info, 0, sizeof(mvm->queue_info));
-	if (iwl_mvm_is_dqa_supported(mvm))
-		mvm->queue_info[IWL_MVM_DQA_CMD_QUEUE].hw_queue_refcount = 1;
-	else
-		mvm->queue_info[IWL_MVM_CMD_QUEUE].hw_queue_refcount = 1;
+	mvm->queue_info[IWL_MVM_DQA_CMD_QUEUE].hw_queue_refcount = 1;
 
 	for (i = 0; i < IEEE80211_MAX_QUEUES; i++)
 		atomic_set(&mvm->mac80211_queue_stop_count[i], 0);
@@ -733,7 +388,7 @@ static int iwl_run_unified_mvm_ucode(struct iwl_mvm *mvm, bool read_nvm)
 	}
 
 	if (IWL_MVM_PARSE_NVM && read_nvm) {
-		ret = iwl_nvm_init(mvm, true);
+		ret = iwl_nvm_init(mvm);
 		if (ret) {
 			IWL_ERR(mvm, "Failed to read NVM: %d\n", ret);
 			goto error;
@@ -757,8 +412,10 @@ static int iwl_run_unified_mvm_ucode(struct iwl_mvm *mvm, bool read_nvm)
 
 	/* Read the NVM only at driver load time, no need to do this twice */
 	if (!IWL_MVM_PARSE_NVM && read_nvm) {
-		ret = iwl_mvm_nvm_get_from_fw(mvm);
-		if (ret) {
+		mvm->nvm_data = iwl_fw_get_nvm(&mvm->fwrt);
+		if (IS_ERR(mvm->nvm_data)) {
+			ret = PTR_ERR(mvm->nvm_data);
+			mvm->nvm_data = NULL;
 			IWL_ERR(mvm, "Failed to read NVM: %d\n", ret);
 			return ret;
 		}
@@ -774,7 +431,7 @@ error:
 static int iwl_send_phy_cfg_cmd(struct iwl_mvm *mvm)
 {
 	struct iwl_phy_cfg_cmd phy_cfg_cmd;
-	enum iwl_ucode_type ucode_type = mvm->cur_ucode;
+	enum iwl_ucode_type ucode_type = mvm->fwrt.cur_fw_img;
 
 	/* Set parameters */
 	phy_cfg_cmd.phy_cfg = cpu_to_le32(iwl_mvm_get_phy_config(mvm));
@@ -799,7 +456,7 @@ int iwl_run_init_mvm_ucode(struct iwl_mvm *mvm, bool read_nvm)
 	};
 	int ret;
 
-	if (iwl_mvm_has_new_tx_api(mvm))
+	if (iwl_mvm_has_unified_ucode(mvm))
 		return iwl_run_unified_mvm_ucode(mvm, true);
 
 	lockdep_assert_held(&mvm->mutex);
@@ -818,22 +475,21 @@ int iwl_run_init_mvm_ucode(struct iwl_mvm *mvm, bool read_nvm)
 	ret = iwl_mvm_load_ucode_wait_alive(mvm, IWL_UCODE_INIT);
 	if (ret) {
 		IWL_ERR(mvm, "Failed to start INIT ucode: %d\n", ret);
-		goto error;
+		goto remove_notif;
 	}
 
 	if (mvm->cfg->device_family < IWL_DEVICE_FAMILY_8000) {
 		ret = iwl_mvm_send_bt_init_conf(mvm);
 		if (ret)
-			goto error;
+			goto remove_notif;
 	}
 
 	/* Read the NVM only at driver load time, no need to do this twice */
 	if (read_nvm) {
-		/* Read nvm */
-		ret = iwl_nvm_init(mvm, true);
+		ret = iwl_nvm_init(mvm);
 		if (ret) {
 			IWL_ERR(mvm, "Failed to read NVM: %d\n", ret);
-			goto error;
+			goto remove_notif;
 		}
 	}
 
@@ -841,8 +497,7 @@ int iwl_run_init_mvm_ucode(struct iwl_mvm *mvm, bool read_nvm)
 	if (mvm->nvm_file_name)
 		iwl_mvm_load_nvm_to_nic(mvm);
 
-	ret = iwl_nvm_check_version(mvm->nvm_data, mvm->trans);
-	WARN_ON(ret);
+	WARN_ON(iwl_nvm_check_version(mvm->nvm_data, mvm->trans));
 
 	/*
 	 * abort after reading the nvm in case RF Kill is on, we will complete
@@ -851,9 +506,7 @@ int iwl_run_init_mvm_ucode(struct iwl_mvm *mvm, bool read_nvm)
 	if (iwl_mvm_is_radio_hw_killed(mvm)) {
 		IWL_DEBUG_RF_KILL(mvm,
 				  "jump over all phy activities due to RF kill\n");
-		iwl_remove_notification(&mvm->notif_wait, &calib_wait);
-		ret = 1;
-		goto out;
+		goto remove_notif;
 	}
 
 	mvm->calibrating = true;
@@ -861,17 +514,13 @@ int iwl_run_init_mvm_ucode(struct iwl_mvm *mvm, bool read_nvm)
 	/* Send TX valid antennas before triggering calibrations */
 	ret = iwl_send_tx_ant_cfg(mvm, iwl_mvm_get_valid_tx_ant(mvm));
 	if (ret)
-		goto error;
+		goto remove_notif;
 
-	/*
-	 * Send phy configurations command to init uCode
-	 * to start the 16.0 uCode init image internal calibrations.
-	 */
 	ret = iwl_send_phy_cfg_cmd(mvm);
 	if (ret) {
 		IWL_ERR(mvm, "Failed to run INIT calibrations: %d\n",
 			ret);
-		goto error;
+		goto remove_notif;
 	}
 
 	/*
@@ -879,15 +528,21 @@ int iwl_run_init_mvm_ucode(struct iwl_mvm *mvm, bool read_nvm)
 	 * just wait for the calibration complete notification.
 	 */
 	ret = iwl_wait_notification(&mvm->notif_wait, &calib_wait,
-			MVM_UCODE_CALIB_TIMEOUT);
+				    MVM_UCODE_CALIB_TIMEOUT);
+	if (!ret)
+		goto out;
 
-	if (ret && iwl_mvm_is_radio_hw_killed(mvm)) {
+	if (iwl_mvm_is_radio_hw_killed(mvm)) {
 		IWL_DEBUG_RF_KILL(mvm, "RFKILL while calibrating.\n");
-		ret = 1;
+		ret = 0;
+	} else {
+		IWL_ERR(mvm, "Failed to run INIT calibrations: %d\n",
+			ret);
 	}
+
 	goto out;
 
-error:
+remove_notif:
 	iwl_remove_notification(&mvm->notif_wait, &calib_wait);
 out:
 	mvm->calibrating = false;
@@ -910,95 +565,6 @@ out:
 	return ret;
 }
 
-static void iwl_mvm_parse_shared_mem_a000(struct iwl_mvm *mvm,
-					  struct iwl_rx_packet *pkt)
-{
-	struct iwl_shared_mem_cfg *mem_cfg = (void *)pkt->data;
-	int i, lmac;
-	int lmac_num = le32_to_cpu(mem_cfg->lmac_num);
-
-	if (WARN_ON(lmac_num > ARRAY_SIZE(mem_cfg->lmac_smem)))
-		return;
-
-	mvm->smem_cfg.num_lmacs = lmac_num;
-	mvm->smem_cfg.num_txfifo_entries =
-		ARRAY_SIZE(mem_cfg->lmac_smem[0].txfifo_size);
-	mvm->smem_cfg.rxfifo2_size = le32_to_cpu(mem_cfg->rxfifo2_size);
-
-	for (lmac = 0; lmac < lmac_num; lmac++) {
-		struct iwl_shared_mem_lmac_cfg *lmac_cfg =
-			&mem_cfg->lmac_smem[lmac];
-
-		for (i = 0; i < ARRAY_SIZE(lmac_cfg->txfifo_size); i++)
-			mvm->smem_cfg.lmac[lmac].txfifo_size[i] =
-				le32_to_cpu(lmac_cfg->txfifo_size[i]);
-		mvm->smem_cfg.lmac[lmac].rxfifo1_size =
-			le32_to_cpu(lmac_cfg->rxfifo1_size);
-	}
-}
-
-static void iwl_mvm_parse_shared_mem(struct iwl_mvm *mvm,
-				     struct iwl_rx_packet *pkt)
-{
-	struct iwl_shared_mem_cfg_v2 *mem_cfg = (void *)pkt->data;
-	int i;
-
-	mvm->smem_cfg.num_lmacs = 1;
-
-	mvm->smem_cfg.num_txfifo_entries = ARRAY_SIZE(mem_cfg->txfifo_size);
-	for (i = 0; i < ARRAY_SIZE(mem_cfg->txfifo_size); i++)
-		mvm->smem_cfg.lmac[0].txfifo_size[i] =
-			le32_to_cpu(mem_cfg->txfifo_size[i]);
-
-	mvm->smem_cfg.lmac[0].rxfifo1_size =
-		le32_to_cpu(mem_cfg->rxfifo_size[0]);
-	mvm->smem_cfg.rxfifo2_size = le32_to_cpu(mem_cfg->rxfifo_size[1]);
-
-	/* new API has more data, from rxfifo_addr field and on */
-	if (fw_has_capa(&mvm->fw->ucode_capa,
-			IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG)) {
-		BUILD_BUG_ON(sizeof(mvm->smem_cfg.internal_txfifo_size) !=
-			     sizeof(mem_cfg->internal_txfifo_size));
-
-		for (i = 0;
-		     i < ARRAY_SIZE(mvm->smem_cfg.internal_txfifo_size);
-		     i++)
-			mvm->smem_cfg.internal_txfifo_size[i] =
-				le32_to_cpu(mem_cfg->internal_txfifo_size[i]);
-	}
-}
-
-static void iwl_mvm_get_shared_mem_conf(struct iwl_mvm *mvm)
-{
-	struct iwl_host_cmd cmd = {
-		.flags = CMD_WANT_SKB,
-		.data = { NULL, },
-		.len = { 0, },
-	};
-	struct iwl_rx_packet *pkt;
-
-	lockdep_assert_held(&mvm->mutex);
-
-	if (fw_has_capa(&mvm->fw->ucode_capa,
-			IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG))
-		cmd.id = iwl_cmd_id(SHARED_MEM_CFG_CMD, SYSTEM_GROUP, 0);
-	else
-		cmd.id = SHARED_MEM_CFG;
-
-	if (WARN_ON(iwl_mvm_send_cmd(mvm, &cmd)))
-		return;
-
-	pkt = cmd.resp_pkt;
-	if (iwl_mvm_has_new_tx_api(mvm))
-		iwl_mvm_parse_shared_mem_a000(mvm, pkt);
-	else
-		iwl_mvm_parse_shared_mem(mvm, pkt);
-
-	IWL_DEBUG_INFO(mvm, "SHARED MEM CFG: got memory offsets/sizes\n");
-
-	iwl_free_resp(&cmd);
-}
-
 static int iwl_mvm_config_ltr(struct iwl_mvm *mvm)
 {
 	struct iwl_ltr_config_cmd cmd = {
@@ -1048,8 +614,8 @@ static union acpi_object *iwl_mvm_sar_find_wifi_pkg(struct iwl_mvm *mvm,
 						    union acpi_object *data,
 						    int data_size)
 {
+	union acpi_object *wifi_pkg = NULL;
 	int i;
-	union acpi_object *wifi_pkg;
 
 	/*
 	 * We need at least two packages, one for the revision and one
@@ -1275,8 +841,10 @@ static int iwl_mvm_sar_get_wgds_table(struct iwl_mvm *mvm)
 
 			entry = &wifi_pkg->package.elements[idx++];
 			if ((entry->type != ACPI_TYPE_INTEGER) ||
-			    (entry->integer.value > U8_MAX))
-				return -EINVAL;
+			    (entry->integer.value > U8_MAX)) {
+				ret = -EINVAL;
+				goto out_free;
+			}
 
 			mvm->geo_profiles[i].values[j] = entry->integer.value;
 		}
@@ -1429,6 +997,17 @@ static int iwl_mvm_sar_geo_init(struct iwl_mvm *mvm)
 {
 	return 0;
 }
+
+int iwl_mvm_sar_select_profile(struct iwl_mvm *mvm, int prof_a,
+			       int prof_b)
+{
+	return -ENOENT;
+}
+
+int iwl_mvm_get_sar_geo_profile(struct iwl_mvm *mvm)
+{
+	return -ENOENT;
+}
 #endif /* CONFIG_ACPI */
 
 static int iwl_mvm_sar_init(struct iwl_mvm *mvm)
@@ -1465,7 +1044,7 @@ static int iwl_mvm_load_rt_fw(struct iwl_mvm *mvm)
 {
 	int ret;
 
-	if (iwl_mvm_has_new_tx_api(mvm))
+	if (iwl_mvm_has_unified_ucode(mvm))
 		return iwl_run_unified_mvm_ucode(mvm, false);
 
 	ret = iwl_run_init_mvm_ucode(mvm, false);
@@ -1475,9 +1054,6 @@ static int iwl_mvm_load_rt_fw(struct iwl_mvm *mvm)
 
 	if (ret) {
 		IWL_ERR(mvm, "Failed to run INIT ucode: %d\n", ret);
-		/* this can't happen */
-		if (WARN_ON(ret > 0))
-			ret = -ERFKILL;
 		return ret;
 	}
 
@@ -1495,7 +1071,7 @@ static int iwl_mvm_load_rt_fw(struct iwl_mvm *mvm)
 	if (ret)
 		return ret;
 
-	return iwl_mvm_init_paging(mvm);
+	return iwl_init_paging(&mvm->fwrt, mvm->fwrt.cur_fw_img);
 }
 
 int iwl_mvm_up(struct iwl_mvm *mvm)
@@ -1516,24 +1092,24 @@ int iwl_mvm_up(struct iwl_mvm *mvm)
 		goto error;
 	}
 
-	iwl_mvm_get_shared_mem_conf(mvm);
+	iwl_get_shared_mem_conf(&mvm->fwrt);
 
 	ret = iwl_mvm_sf_update(mvm, NULL, false);
 	if (ret)
 		IWL_ERR(mvm, "Failed to initialize Smart Fifo\n");
 
-	mvm->fw_dbg_conf = FW_DBG_INVALID;
+	mvm->fwrt.dump.conf = FW_DBG_INVALID;
 	/* if we have a destination, assume EARLY START */
 	if (mvm->fw->dbg_dest_tlv)
-		mvm->fw_dbg_conf = FW_DBG_START_FROM_ALIVE;
-	iwl_mvm_start_fw_dbg_conf(mvm, FW_DBG_START_FROM_ALIVE);
+		mvm->fwrt.dump.conf = FW_DBG_START_FROM_ALIVE;
+	iwl_fw_start_dbg_conf(&mvm->fwrt, FW_DBG_START_FROM_ALIVE);
 
 	ret = iwl_send_tx_ant_cfg(mvm, iwl_mvm_get_valid_tx_ant(mvm));
 	if (ret)
 		goto error;
 
-	/* Send phy db control command and then phy db calibration*/
-	if (!iwl_mvm_has_new_tx_api(mvm)) {
+	if (!iwl_mvm_has_unified_ucode(mvm)) {
+		/* Send phy db control command and then phy db calibration */
 		ret = iwl_send_phy_db_data(mvm->phy_db);
 		if (ret)
 			goto error;
@@ -1549,7 +1125,8 @@ int iwl_mvm_up(struct iwl_mvm *mvm)
 
 	/* Init RSS configuration */
 	/* TODO - remove a000 disablement when we have RXQ config API */
-	if (iwl_mvm_has_new_rx_api(mvm) && !iwl_mvm_has_new_tx_api(mvm)) {
+	if (iwl_mvm_has_new_rx_api(mvm) &&
+	    mvm->trans->cfg->device_family != IWL_DEVICE_FAMILY_A000) {
 		ret = iwl_send_rss_cfg_cmd(mvm);
 		if (ret) {
 			IWL_ERR(mvm, "Failed to configure RSS queues: %d\n",
@@ -1567,14 +1144,9 @@ int iwl_mvm_up(struct iwl_mvm *mvm)
 	/* reset quota debouncing buffer - 0xff will yield invalid data */
 	memset(&mvm->last_quota_cmd, 0xff, sizeof(mvm->last_quota_cmd));
 
-	/* Enable DQA-mode if required */
-	if (iwl_mvm_is_dqa_supported(mvm)) {
-		ret = iwl_mvm_send_dqa_cmd(mvm);
-		if (ret)
-			goto error;
-	} else {
-		IWL_DEBUG_FW(mvm, "Working in non-DQA mode\n");
-	}
+	ret = iwl_mvm_send_dqa_cmd(mvm);
+	if (ret)
+		goto error;
 
 	/* Add auxiliary station for scanning */
 	ret = iwl_mvm_add_aux_sta(mvm);
@@ -1609,7 +1181,12 @@ int iwl_mvm_up(struct iwl_mvm *mvm)
 	}
 
 	/* TODO: read the budget from BIOS / Platform NVM */
-	if (iwl_mvm_is_ctdp_supported(mvm) && mvm->cooling_dev.cur_state > 0) {
+
+	/*
+	 * In case there is no budget from BIOS / Platform NVM the default
+	 * budget should be 2000mW (cooling state 0).
+	 */
+	if (iwl_mvm_is_ctdp_supported(mvm)) {
 		ret = iwl_mvm_ctdp_command(mvm, CTDP_CMD_OPERATION_START,
 					   mvm->cooling_dev.cur_state);
 		if (ret)
@@ -1655,6 +1232,8 @@ int iwl_mvm_up(struct iwl_mvm *mvm)
 	if (ret)
 		goto error;
 
+	iwl_mvm_leds_sync(mvm);
+
 	IWL_DEBUG_INFO(mvm, "RT uCode started.\n");
 	return 0;
  error:
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/led.c b/drivers/net/wireless/intel/iwlwifi/mvm/led.c
index 3cac4278a5fd..b27269504a62 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/led.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/led.c
@@ -6,6 +6,7 @@
  * GPL LICENSE SUMMARY
  *
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2017        Intel Deutschland GmbH
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -31,6 +32,7 @@
  * BSD LICENSE
  *
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2017        Intel Deutschland GmbH
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -66,26 +68,46 @@
 #include "iwl-csr.h"
 #include "mvm.h"
 
-/* Set led register on */
-static void iwl_mvm_led_enable(struct iwl_mvm *mvm)
+static void iwl_mvm_send_led_fw_cmd(struct iwl_mvm *mvm, bool on)
 {
-	iwl_write32(mvm->trans, CSR_LED_REG, CSR_LED_REG_TURN_ON);
+	struct iwl_led_cmd led_cmd = {
+		.status = cpu_to_le32(on),
+	};
+	struct iwl_host_cmd cmd = {
+		.id = WIDE_ID(LONG_GROUP, LEDS_CMD),
+		.len = { sizeof(led_cmd), },
+		.data = { &led_cmd, },
+		.flags = CMD_ASYNC,
+	};
+	int err;
+
+	if (!iwl_mvm_firmware_running(mvm))
+		return;
+
+	err = iwl_mvm_send_cmd(mvm, &cmd);
+
+	if (err)
+		IWL_WARN(mvm, "LED command failed: %d\n", err);
 }
 
-/* Set led register off */
-static void iwl_mvm_led_disable(struct iwl_mvm *mvm)
+static void iwl_mvm_led_set(struct iwl_mvm *mvm, bool on)
 {
-	iwl_write32(mvm->trans, CSR_LED_REG, CSR_LED_REG_TURN_OFF);
+	if (fw_has_capa(&mvm->fw->ucode_capa,
+			IWL_UCODE_TLV_CAPA_LED_CMD_SUPPORT)) {
+		iwl_mvm_send_led_fw_cmd(mvm, on);
+		return;
+	}
+
+	iwl_write32(mvm->trans, CSR_LED_REG,
+		    on ? CSR_LED_REG_TURN_ON : CSR_LED_REG_TURN_OFF);
 }
 
 static void iwl_led_brightness_set(struct led_classdev *led_cdev,
 				   enum led_brightness brightness)
 {
 	struct iwl_mvm *mvm = container_of(led_cdev, struct iwl_mvm, led);
-	if (brightness > 0)
-		iwl_mvm_led_enable(mvm);
-	else
-		iwl_mvm_led_disable(mvm);
+
+	iwl_mvm_led_set(mvm, brightness > 0);
 }
 
 int iwl_mvm_leds_init(struct iwl_mvm *mvm)
@@ -127,10 +149,24 @@ int iwl_mvm_leds_init(struct iwl_mvm *mvm)
 	return 0;
 }
 
+void iwl_mvm_leds_sync(struct iwl_mvm *mvm)
+{
+	if (!(mvm->init_status & IWL_MVM_INIT_STATUS_LEDS_INIT_COMPLETE))
+		return;
+
+	/*
+	 * if we control through the register, we're doing it
+	 * even when the firmware isn't up, so no need to sync
+	 */
+	if (mvm->cfg->device_family < IWL_DEVICE_FAMILY_8000)
+		return;
+
+	iwl_mvm_led_set(mvm, mvm->led.brightness > 0);
+}
+
 void iwl_mvm_leds_exit(struct iwl_mvm *mvm)
 {
-	if (iwlwifi_mod_params.led_mode == IWL_LED_DISABLE ||
-	    !(mvm->init_status & IWL_MVM_INIT_STATUS_LEDS_INIT_COMPLETE))
+	if (!(mvm->init_status & IWL_MVM_INIT_STATUS_LEDS_INIT_COMPLETE))
 		return;
 
 	led_classdev_unregister(&mvm->led);
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c
index dc631b23e189..a2bf530eeae4 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c
@@ -7,7 +7,7 @@
  *
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2014 Intel Mobile Communications GmbH
- * Copyright(c) 2015 - 2016 Intel Deutschland GmbH
+ * Copyright(c) 2015 - 2017 Intel Deutschland GmbH
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -34,7 +34,7 @@
  *
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2014 Intel Mobile Communications GmbH
- * Copyright(c) 2015 - 2016 Intel Deutschland GmbH
+ * Copyright(c) 2015 - 2017 Intel Deutschland GmbH
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -72,7 +72,6 @@
 #include "fw-api.h"
 #include "mvm.h"
 #include "time-event.h"
-#include "fw-dbg.h"
 
 const u8 iwl_mvm_ac_to_tx_fifo[] = {
 	IWL_MVM_TX_FIFO_VO,
@@ -81,6 +80,13 @@ const u8 iwl_mvm_ac_to_tx_fifo[] = {
 	IWL_MVM_TX_FIFO_BK,
 };
 
+const u8 iwl_mvm_ac_to_gen2_tx_fifo[] = {
+	IWL_GEN2_EDCA_TX_FIFO_VO,
+	IWL_GEN2_EDCA_TX_FIFO_VI,
+	IWL_GEN2_EDCA_TX_FIFO_BE,
+	IWL_GEN2_EDCA_TX_FIFO_BK,
+};
+
 struct iwl_mvm_mac_iface_iterator_data {
 	struct iwl_mvm *mvm;
 	struct ieee80211_vif *vif;
@@ -235,32 +241,17 @@ static void iwl_mvm_iface_hw_queues_iter(void *_data, u8 *mac,
 	data->used_hw_queues |= iwl_mvm_mac_get_queues_mask(vif);
 }
 
-static void iwl_mvm_mac_sta_hw_queues_iter(void *_data,
-					   struct ieee80211_sta *sta)
-{
-	struct iwl_mvm_hw_queues_iface_iterator_data *data = _data;
-	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
-
-	/* Mark the queues used by the sta */
-	data->used_hw_queues |= mvmsta->tfd_queue_msk;
-}
-
 unsigned long iwl_mvm_get_used_hw_queues(struct iwl_mvm *mvm,
 					 struct ieee80211_vif *exclude_vif)
 {
-	u8 sta_id;
 	struct iwl_mvm_hw_queues_iface_iterator_data data = {
 		.exclude_vif = exclude_vif,
 		.used_hw_queues =
 			BIT(IWL_MVM_OFFCHANNEL_QUEUE) |
-			BIT(mvm->aux_queue),
+			BIT(mvm->aux_queue) |
+			BIT(IWL_MVM_DQA_GCAST_QUEUE),
 	};
 
-	if (iwl_mvm_is_dqa_supported(mvm))
-		data.used_hw_queues |= BIT(IWL_MVM_DQA_GCAST_QUEUE);
-	else
-		data.used_hw_queues |= BIT(IWL_MVM_CMD_QUEUE);
-
 	lockdep_assert_held(&mvm->mutex);
 
 	/* mark all VIF used hw queues */
@@ -268,26 +259,6 @@ unsigned long iwl_mvm_get_used_hw_queues(struct iwl_mvm *mvm,
 		mvm->hw, IEEE80211_IFACE_ITER_RESUME_ALL,
 		iwl_mvm_iface_hw_queues_iter, &data);
 
-	/*
-	 * for DQA, the hw_queue in mac80211 is never really used for
-	 * real traffic (only the few queue IDs covered above), so
-	 * we can reuse the real HW queue IDs the stations use
-	 */
-	if (iwl_mvm_is_dqa_supported(mvm))
-		return data.used_hw_queues;
-
-	/* don't assign the same hw queues as TDLS stations */
-	ieee80211_iterate_stations_atomic(mvm->hw,
-					  iwl_mvm_mac_sta_hw_queues_iter,
-					  &data);
-
-	/*
-	 * Some TDLS stations may be removed but are in the process of being
-	 * drained. Don't touch their queues.
-	 */
-	for_each_set_bit(sta_id, mvm->sta_drained, IWL_MVM_STATION_COUNT)
-		data.used_hw_queues |= mvm->tfd_drained[sta_id];
-
 	return data.used_hw_queues;
 }
 
@@ -338,8 +309,7 @@ void iwl_mvm_mac_ctxt_recalc_tsf_id(struct iwl_mvm *mvm,
 						NUM_TSF_IDS);
 }
 
-static int iwl_mvm_mac_ctxt_allocate_resources(struct iwl_mvm *mvm,
-					       struct ieee80211_vif *vif)
+int iwl_mvm_mac_ctxt_init(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	struct iwl_mvm_mac_iface_iterator_data data = {
@@ -355,6 +325,8 @@ static int iwl_mvm_mac_ctxt_allocate_resources(struct iwl_mvm *mvm,
 	int ret, i, queue_limit;
 	unsigned long used_hw_queues;
 
+	lockdep_assert_held(&mvm->mutex);
+
 	/*
 	 * Allocate a MAC ID and a TSF for this MAC, along with the queues
 	 * and other resources.
@@ -438,19 +410,14 @@ static int iwl_mvm_mac_ctxt_allocate_resources(struct iwl_mvm *mvm,
 		return 0;
 	}
 
-	if (iwl_mvm_is_dqa_supported(mvm)) {
-		/*
-		 * queues in mac80211 almost entirely independent of
-		 * the ones here - no real limit
-		 */
-		queue_limit = IEEE80211_MAX_QUEUES;
-		BUILD_BUG_ON(IEEE80211_MAX_QUEUES >
-			     BITS_PER_BYTE *
-			     sizeof(mvm->hw_queue_to_mac80211[0]));
-	} else {
-		/* need to not use too many in this case */
-		queue_limit = mvm->first_agg_queue;
-	}
+	/*
+	 * queues in mac80211 almost entirely independent of
+	 * the ones here - no real limit
+	 */
+	queue_limit = IEEE80211_MAX_QUEUES;
+	BUILD_BUG_ON(IEEE80211_MAX_QUEUES >
+		     BITS_PER_BYTE *
+		     sizeof(mvm->hw_queue_to_mac80211[0]));
 
 	/*
 	 * Find available queues, and allocate them to the ACs. When in
@@ -472,27 +439,12 @@ static int iwl_mvm_mac_ctxt_allocate_resources(struct iwl_mvm *mvm,
 
 	/* Allocate the CAB queue for softAP and GO interfaces */
 	if (vif->type == NL80211_IFTYPE_AP) {
-		u8 queue;
-
-		if (!iwl_mvm_is_dqa_supported(mvm)) {
-			queue = find_first_zero_bit(&used_hw_queues,
-						    mvm->first_agg_queue);
-
-			if (queue >= mvm->first_agg_queue) {
-				IWL_ERR(mvm, "Failed to allocate cab queue\n");
-				ret = -EIO;
-				goto exit_fail;
-			}
-		} else {
-			queue = IWL_MVM_DQA_GCAST_QUEUE;
-		}
-
 		/*
 		 * For TVQM this will be overwritten later with the FW assigned
 		 * queue value (when queue is enabled).
 		 */
-		mvmvif->cab_queue = queue;
-		vif->cab_queue = queue;
+		mvmvif->cab_queue = IWL_MVM_DQA_GCAST_QUEUE;
+		vif->cab_queue = IWL_MVM_DQA_GCAST_QUEUE;
 	} else {
 		vif->cab_queue = IEEE80211_INVAL_HW_QUEUE;
 	}
@@ -513,78 +465,6 @@ exit_fail:
 	return ret;
 }
 
-int iwl_mvm_mac_ctxt_init(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
-{
-	unsigned int wdg_timeout =
-		iwl_mvm_get_wd_timeout(mvm, vif, false, false);
-	u32 ac;
-	int ret;
-
-	lockdep_assert_held(&mvm->mutex);
-
-	ret = iwl_mvm_mac_ctxt_allocate_resources(mvm, vif);
-	if (ret)
-		return ret;
-
-	/* If DQA is supported - queues will be enabled when needed */
-	if (iwl_mvm_is_dqa_supported(mvm))
-		return 0;
-
-	switch (vif->type) {
-	case NL80211_IFTYPE_P2P_DEVICE:
-		iwl_mvm_enable_ac_txq(mvm, IWL_MVM_OFFCHANNEL_QUEUE,
-				      IWL_MVM_OFFCHANNEL_QUEUE,
-				      IWL_MVM_TX_FIFO_VO, 0, wdg_timeout);
-		break;
-	case NL80211_IFTYPE_AP:
-		iwl_mvm_enable_ac_txq(mvm, vif->cab_queue, vif->cab_queue,
-				      IWL_MVM_TX_FIFO_MCAST, 0, wdg_timeout);
-		/* fall through */
-	default:
-		for (ac = 0; ac < IEEE80211_NUM_ACS; ac++)
-			iwl_mvm_enable_ac_txq(mvm, vif->hw_queue[ac],
-					      vif->hw_queue[ac],
-					      iwl_mvm_ac_to_tx_fifo[ac], 0,
-					      wdg_timeout);
-		break;
-	}
-
-	return 0;
-}
-
-void iwl_mvm_mac_ctxt_release(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
-{
-	int ac;
-
-	lockdep_assert_held(&mvm->mutex);
-
-	/*
-	 * If DQA is supported - queues were already disabled, since in
-	 * DQA-mode the queues are a property of the STA and not of the
-	 * vif, and at this point the STA was already deleted
-	 */
-	if (iwl_mvm_is_dqa_supported(mvm))
-		return;
-
-	switch (vif->type) {
-	case NL80211_IFTYPE_P2P_DEVICE:
-		iwl_mvm_disable_txq(mvm, IWL_MVM_OFFCHANNEL_QUEUE,
-				    IWL_MVM_OFFCHANNEL_QUEUE,
-				    IWL_MAX_TID_COUNT, 0);
-
-		break;
-	case NL80211_IFTYPE_AP:
-		iwl_mvm_disable_txq(mvm, vif->cab_queue, vif->cab_queue,
-				    IWL_MAX_TID_COUNT, 0);
-		/* fall through */
-	default:
-		for (ac = 0; ac < IEEE80211_NUM_ACS; ac++)
-			iwl_mvm_disable_txq(mvm, vif->hw_queue[ac],
-					    vif->hw_queue[ac],
-					    IWL_MAX_TID_COUNT, 0);
-	}
-}
-
 static void iwl_mvm_ack_rates(struct iwl_mvm *mvm,
 			      struct ieee80211_vif *vif,
 			      enum nl80211_band band,
@@ -775,7 +655,7 @@ static void iwl_mvm_mac_ctxt_cmd_common(struct iwl_mvm *mvm,
 	cmd->filter_flags = cpu_to_le32(MAC_FILTER_ACCEPT_GRP);
 
 	for (i = 0; i < IEEE80211_NUM_ACS; i++) {
-		u8 txf = iwl_mvm_ac_to_tx_fifo[i];
+		u8 txf = iwl_mvm_mac_ac_to_tx_fifo(mvm, i);
 
 		cmd->ac[txf].cw_min =
 			cpu_to_le16(mvmvif->queue_params[i].cw_min);
@@ -908,18 +788,12 @@ static int iwl_mvm_mac_ctxt_cmd_listener(struct iwl_mvm *mvm,
 {
 	struct iwl_mac_ctx_cmd cmd = {};
 	u32 tfd_queue_msk = 0;
-	int ret, i;
+	int ret;
 
 	WARN_ON(vif->type != NL80211_IFTYPE_MONITOR);
 
 	iwl_mvm_mac_ctxt_cmd_common(mvm, vif, &cmd, NULL, action);
 
-	if (!iwl_mvm_is_dqa_supported(mvm)) {
-		for (i = 0; i < IEEE80211_NUM_ACS; i++)
-			if (vif->hw_queue[i] != IEEE80211_INVAL_HW_QUEUE)
-				tfd_queue_msk |= BIT(vif->hw_queue[i]);
-	}
-
 	cmd.filter_flags = cpu_to_le32(MAC_FILTER_IN_PROMISC |
 				       MAC_FILTER_IN_CONTROL_AND_MGMT |
 				       MAC_FILTER_IN_BEACON |
@@ -1049,83 +923,40 @@ static u32 iwl_mvm_find_ie_offset(u8 *beacon, u8 eid, u32 frame_size)
 	return ie - beacon;
 }
 
-static int iwl_mvm_mac_ctxt_send_beacon(struct iwl_mvm *mvm,
-					struct ieee80211_vif *vif,
-					struct sk_buff *beacon)
+static u8 iwl_mvm_mac_ctxt_get_lowest_rate(struct ieee80211_tx_info *info,
+					   struct ieee80211_vif *vif)
 {
-	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
-	struct iwl_host_cmd cmd = {
-		.id = BEACON_TEMPLATE_CMD,
-		.flags = CMD_ASYNC,
-	};
-	union {
-		struct iwl_mac_beacon_cmd_v6 beacon_cmd_v6;
-		struct iwl_mac_beacon_cmd_v7 beacon_cmd;
-	} u = {};
-	struct iwl_mac_beacon_cmd beacon_cmd = {};
-	struct ieee80211_tx_info *info;
-	u32 beacon_skb_len;
-	u32 rate, tx_flags;
+	u8 rate;
 
-	if (WARN_ON(!beacon))
-		return -EINVAL;
+	if (info->band == NL80211_BAND_5GHZ || vif->p2p)
+		rate = IWL_FIRST_OFDM_RATE;
+	else
+		rate = IWL_FIRST_CCK_RATE;
 
-	beacon_skb_len = beacon->len;
-
-	if (fw_has_capa(&mvm->fw->ucode_capa,
-			IWL_UCODE_TLV_CAPA_CSA_AND_TBTT_OFFLOAD)) {
-		u32 csa_offset, ecsa_offset;
-
-		csa_offset = iwl_mvm_find_ie_offset(beacon->data,
-						    WLAN_EID_CHANNEL_SWITCH,
-						    beacon_skb_len);
-		ecsa_offset =
-			iwl_mvm_find_ie_offset(beacon->data,
-					       WLAN_EID_EXT_CHANSWITCH_ANN,
-					       beacon_skb_len);
-
-		if (iwl_mvm_has_new_tx_api(mvm)) {
-			beacon_cmd.data.template_id =
-				cpu_to_le32((u32)mvmvif->id);
-			beacon_cmd.data.ecsa_offset = cpu_to_le32(ecsa_offset);
-			beacon_cmd.data.csa_offset = cpu_to_le32(csa_offset);
-			beacon_cmd.byte_cnt = cpu_to_le16((u16)beacon_skb_len);
-			if (vif->type == NL80211_IFTYPE_AP)
-				iwl_mvm_mac_ctxt_set_tim(mvm,
-							 &beacon_cmd.data.tim_idx,
-							 &beacon_cmd.data.tim_size,
-							 beacon->data,
-							 beacon_skb_len);
-			cmd.len[0] = sizeof(beacon_cmd);
-			cmd.data[0] = &beacon_cmd;
-			goto send;
+	return rate;
+}
 
-		} else {
-			u.beacon_cmd.data.ecsa_offset =
-				cpu_to_le32(ecsa_offset);
-			u.beacon_cmd.data.csa_offset = cpu_to_le32(csa_offset);
-			cmd.len[0] = sizeof(u.beacon_cmd);
-			cmd.data[0] = &u;
-		}
-	} else {
-		cmd.len[0] = sizeof(u.beacon_cmd_v6);
-		cmd.data[0] = &u;
-	}
+static void iwl_mvm_mac_ctxt_set_tx(struct iwl_mvm *mvm,
+				    struct ieee80211_vif *vif,
+				    struct sk_buff *beacon,
+				    struct iwl_tx_cmd *tx)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct ieee80211_tx_info *info;
+	u8 rate;
+	u32 tx_flags;
 
-	/* TODO: for now the beacon template id is set to be the mac context id.
-	 * Might be better to handle it as another resource ... */
-	u.beacon_cmd_v6.template_id = cpu_to_le32((u32)mvmvif->id);
 	info = IEEE80211_SKB_CB(beacon);
 
 	/* Set up TX command fields */
-	u.beacon_cmd_v6.tx.len = cpu_to_le16((u16)beacon_skb_len);
-	u.beacon_cmd_v6.tx.sta_id = mvmvif->bcast_sta.sta_id;
-	u.beacon_cmd_v6.tx.life_time = cpu_to_le32(TX_CMD_LIFE_TIME_INFINITE);
+	tx->len = cpu_to_le16((u16)beacon->len);
+	tx->sta_id = mvmvif->bcast_sta.sta_id;
+	tx->life_time = cpu_to_le32(TX_CMD_LIFE_TIME_INFINITE);
 	tx_flags = TX_CMD_FLG_SEQ_CTL | TX_CMD_FLG_TSF;
 	tx_flags |=
 		iwl_mvm_bt_coex_tx_prio(mvm, (void *)beacon->data, info, 0) <<
 						TX_CMD_FLG_BT_PRIO_POS;
-	u.beacon_cmd_v6.tx.tx_flags = cpu_to_le32(tx_flags);
+	tx->tx_flags = cpu_to_le32(tx_flags);
 
 	if (!fw_has_capa(&mvm->fw->ucode_capa,
 			 IWL_UCODE_TLV_CAPA_BEACON_ANT_SELECTION)) {
@@ -1134,37 +965,141 @@ static int iwl_mvm_mac_ctxt_send_beacon(struct iwl_mvm *mvm,
 					     mvm->mgmt_last_antenna_idx);
 	}
 
-	u.beacon_cmd_v6.tx.rate_n_flags =
+	tx->rate_n_flags =
 		cpu_to_le32(BIT(mvm->mgmt_last_antenna_idx) <<
 			    RATE_MCS_ANT_POS);
 
-	if (info->band == NL80211_BAND_5GHZ || vif->p2p) {
-		rate = IWL_FIRST_OFDM_RATE;
-	} else {
-		rate = IWL_FIRST_CCK_RATE;
-		u.beacon_cmd_v6.tx.rate_n_flags |=
-					cpu_to_le32(RATE_MCS_CCK_MSK);
-	}
-	u.beacon_cmd_v6.tx.rate_n_flags |=
-		cpu_to_le32(iwl_mvm_mac80211_idx_to_hwrate(rate));
+	rate = iwl_mvm_mac_ctxt_get_lowest_rate(info, vif);
 
-	/* Set up TX beacon command fields */
-	if (vif->type == NL80211_IFTYPE_AP)
-		iwl_mvm_mac_ctxt_set_tim(mvm, &u.beacon_cmd_v6.tim_idx,
-					 &u.beacon_cmd_v6.tim_size,
-					 beacon->data,
-					 beacon_skb_len);
+	tx->rate_n_flags |= cpu_to_le32(iwl_mvm_mac80211_idx_to_hwrate(rate));
+	if (rate == IWL_FIRST_CCK_RATE)
+		tx->rate_n_flags |= cpu_to_le32(RATE_MCS_CCK_MSK);
+
+}
 
-send:
-	/* Submit command */
+static int iwl_mvm_mac_ctxt_send_beacon_cmd(struct iwl_mvm *mvm,
+					    struct sk_buff *beacon,
+					    void *data, int len)
+{
+	struct iwl_host_cmd cmd = {
+		.id = BEACON_TEMPLATE_CMD,
+		.flags = CMD_ASYNC,
+	};
+
+	cmd.len[0] = len;
+	cmd.data[0] = data;
 	cmd.dataflags[0] = 0;
-	cmd.len[1] = beacon_skb_len;
+	cmd.len[1] = beacon->len;
 	cmd.data[1] = beacon->data;
 	cmd.dataflags[1] = IWL_HCMD_DFL_DUP;
 
 	return iwl_mvm_send_cmd(mvm, &cmd);
 }
 
+static int iwl_mvm_mac_ctxt_send_beacon_v6(struct iwl_mvm *mvm,
+					   struct ieee80211_vif *vif,
+					   struct sk_buff *beacon)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_mac_beacon_cmd_v6 beacon_cmd = {};
+
+	iwl_mvm_mac_ctxt_set_tx(mvm, vif, beacon, &beacon_cmd.tx);
+
+	beacon_cmd.template_id = cpu_to_le32((u32)mvmvif->id);
+
+	if (vif->type == NL80211_IFTYPE_AP)
+		iwl_mvm_mac_ctxt_set_tim(mvm, &beacon_cmd.tim_idx,
+					 &beacon_cmd.tim_size,
+					 beacon->data, beacon->len);
+
+	return iwl_mvm_mac_ctxt_send_beacon_cmd(mvm, beacon, &beacon_cmd,
+						sizeof(beacon_cmd));
+}
+
+static int iwl_mvm_mac_ctxt_send_beacon_v7(struct iwl_mvm *mvm,
+					   struct ieee80211_vif *vif,
+					   struct sk_buff *beacon)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct iwl_mac_beacon_cmd_v7 beacon_cmd = {};
+
+	iwl_mvm_mac_ctxt_set_tx(mvm, vif, beacon, &beacon_cmd.tx);
+
+	beacon_cmd.template_id = cpu_to_le32((u32)mvmvif->id);
+
+	if (vif->type == NL80211_IFTYPE_AP)
+		iwl_mvm_mac_ctxt_set_tim(mvm, &beacon_cmd.tim_idx,
+					 &beacon_cmd.tim_size,
+					 beacon->data, beacon->len);
+
+	beacon_cmd.csa_offset =
+		cpu_to_le32(iwl_mvm_find_ie_offset(beacon->data,
+						   WLAN_EID_CHANNEL_SWITCH,
+						   beacon->len));
+	beacon_cmd.ecsa_offset =
+		cpu_to_le32(iwl_mvm_find_ie_offset(beacon->data,
+						   WLAN_EID_EXT_CHANSWITCH_ANN,
+						   beacon->len));
+
+	return iwl_mvm_mac_ctxt_send_beacon_cmd(mvm, beacon, &beacon_cmd,
+						sizeof(beacon_cmd));
+}
+
+static int iwl_mvm_mac_ctxt_send_beacon_v9(struct iwl_mvm *mvm,
+					   struct ieee80211_vif *vif,
+					   struct sk_buff *beacon)
+{
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(beacon);
+	struct iwl_mac_beacon_cmd beacon_cmd = {};
+	u8 rate = iwl_mvm_mac_ctxt_get_lowest_rate(info, vif);
+	u16 flags;
+
+	flags = iwl_mvm_mac80211_idx_to_hwrate(rate);
+
+	if (rate == IWL_FIRST_CCK_RATE)
+		flags |= IWL_MAC_BEACON_CCK;
+
+	beacon_cmd.flags = cpu_to_le16(flags);
+	beacon_cmd.byte_cnt = cpu_to_le16((u16)beacon->len);
+	beacon_cmd.template_id = cpu_to_le32((u32)mvmvif->id);
+
+	if (vif->type == NL80211_IFTYPE_AP)
+		iwl_mvm_mac_ctxt_set_tim(mvm, &beacon_cmd.tim_idx,
+					 &beacon_cmd.tim_size,
+					 beacon->data, beacon->len);
+
+	beacon_cmd.csa_offset =
+		cpu_to_le32(iwl_mvm_find_ie_offset(beacon->data,
+						   WLAN_EID_CHANNEL_SWITCH,
+						   beacon->len));
+	beacon_cmd.ecsa_offset =
+		cpu_to_le32(iwl_mvm_find_ie_offset(beacon->data,
+						   WLAN_EID_EXT_CHANSWITCH_ANN,
+						   beacon->len));
+
+	return iwl_mvm_mac_ctxt_send_beacon_cmd(mvm, beacon, &beacon_cmd,
+						sizeof(beacon_cmd));
+}
+
+static int iwl_mvm_mac_ctxt_send_beacon(struct iwl_mvm *mvm,
+					struct ieee80211_vif *vif,
+					struct sk_buff *beacon)
+{
+	if (WARN_ON(!beacon))
+		return -EINVAL;
+
+	if (!fw_has_capa(&mvm->fw->ucode_capa,
+			 IWL_UCODE_TLV_CAPA_CSA_AND_TBTT_OFFLOAD))
+		return iwl_mvm_mac_ctxt_send_beacon_v6(mvm, vif, beacon);
+
+	if (fw_has_api(&mvm->fw->ucode_capa,
+		       IWL_UCODE_TLV_API_NEW_BEACON_TEMPLATE))
+		return iwl_mvm_mac_ctxt_send_beacon_v9(mvm, vif, beacon);
+
+	return iwl_mvm_mac_ctxt_send_beacon_v7(mvm, vif, beacon);
+}
+
 /* The beacon template for the AP/GO/IBSS has changed and needs update */
 int iwl_mvm_mac_ctxt_beacon_changed(struct iwl_mvm *mvm,
 				    struct ieee80211_vif *vif)
@@ -1559,12 +1494,14 @@ static void iwl_mvm_beacon_loss_iterator(void *_data, u8 *mac,
 
 	/* TODO: implement start trigger */
 
-	if (!iwl_fw_dbg_trigger_check_stop(mvm, vif, trigger))
+	if (!iwl_fw_dbg_trigger_check_stop(&mvm->fwrt,
+					   ieee80211_vif_to_wdev(vif),
+					   trigger))
 		return;
 
 	if (rx_missed_bcon_since_rx >= stop_trig_missed_bcon_since_rx ||
 	    rx_missed_bcon >= stop_trig_missed_bcon)
-		iwl_mvm_fw_dbg_collect_trig(mvm, trigger, NULL);
+		iwl_fw_dbg_collect_trig(&mvm->fwrt, trigger, NULL);
 }
 
 void iwl_mvm_rx_missed_beacons_notif(struct iwl_mvm *mvm,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
index c7b1e58e3384..15f2d826bb4b 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
@@ -87,7 +87,6 @@
 #include "fw/error-dump.h"
 #include "iwl-prph.h"
 #include "iwl-nvm-parse.h"
-#include "fw-dbg.h"
 
 static const struct ieee80211_iface_limit iwl_mvm_limits[] = {
 	{
@@ -446,8 +445,18 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 	ieee80211_hw_set(hw, NEEDS_UNIQUE_STA_ADDR);
 	if (iwl_mvm_has_new_rx_api(mvm))
 		ieee80211_hw_set(hw, SUPPORTS_REORDERING_BUFFER);
-	if (fw_has_capa(&mvm->fw->ucode_capa, IWL_UCODE_TLV_CAPA_STA_PM_NOTIF))
+
+	if (fw_has_capa(&mvm->fw->ucode_capa,
+			IWL_UCODE_TLV_CAPA_STA_PM_NOTIF)) {
 		ieee80211_hw_set(hw, AP_LINK_PS);
+	} else if (WARN_ON(iwl_mvm_has_new_tx_api(mvm))) {
+		/*
+		 * we absolutely need this for the new TX API since that comes
+		 * with many more queues than the current code can deal with
+		 * for station powersave
+		 */
+		return -EINVAL;
+	}
 
 	if (mvm->trans->num_rx_queues > 1)
 		ieee80211_hw_set(hw, USES_RSS);
@@ -455,10 +464,7 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 	if (mvm->trans->max_skb_frags)
 		hw->netdev_features = NETIF_F_HIGHDMA | NETIF_F_SG;
 
-	if (!iwl_mvm_is_dqa_supported(mvm))
-		hw->queues = mvm->first_agg_queue;
-	else
-		hw->queues = IEEE80211_MAX_QUEUES;
+	hw->queues = IEEE80211_MAX_QUEUES;
 	hw->offchannel_tx_hw_queue = IWL_MVM_OFFCHANNEL_QUEUE;
 	hw->radiotap_mcs_details |= IEEE80211_RADIOTAP_MCS_HAVE_FEC |
 				    IEEE80211_RADIOTAP_MCS_HAVE_STBC;
@@ -799,17 +805,16 @@ static void iwl_mvm_mac_tx(struct ieee80211_hw *hw,
 		goto drop;
 	}
 
-	if (IEEE80211_SKB_CB(skb)->hw_queue == IWL_MVM_OFFCHANNEL_QUEUE &&
+	if (info->hw_queue == IWL_MVM_OFFCHANNEL_QUEUE &&
 	    !test_bit(IWL_MVM_STATUS_ROC_RUNNING, &mvm->status) &&
 	    !test_bit(IWL_MVM_STATUS_ROC_AUX_RUNNING, &mvm->status))
 		goto drop;
 
-	/* treat non-bufferable MMPDUs as broadcast if sta is sleeping */
-	if (unlikely(info->flags & IEEE80211_TX_CTL_NO_PS_BUFFER &&
-		     ieee80211_is_mgmt(hdr->frame_control) &&
-		     !ieee80211_is_deauth(hdr->frame_control) &&
-		     !ieee80211_is_disassoc(hdr->frame_control) &&
-		     !ieee80211_is_action(hdr->frame_control)))
+	/* treat non-bufferable MMPDUs on AP interfaces as broadcast */
+	if ((info->control.vif->type == NL80211_IFTYPE_AP ||
+	     info->control.vif->type == NL80211_IFTYPE_ADHOC) &&
+	    ieee80211_is_mgmt(hdr->frame_control) &&
+	    !ieee80211_is_bufferable_mmpdu(hdr->frame_control))
 		sta = NULL;
 
 	if (sta) {
@@ -845,11 +850,11 @@ static inline bool iwl_enable_tx_ampdu(const struct iwl_cfg *cfg)
 	return true;
 }
 
-#define CHECK_BA_TRIGGER(_mvm, _trig, _tid_bm, _tid, _fmt...)	\
-	do {							\
-		if (!(le16_to_cpu(_tid_bm) & BIT(_tid)))	\
-			break;					\
-		iwl_mvm_fw_dbg_collect_trig(_mvm, _trig, _fmt);	\
+#define CHECK_BA_TRIGGER(_mvm, _trig, _tid_bm, _tid, _fmt...)		\
+	do {								\
+		if (!(le16_to_cpu(_tid_bm) & BIT(_tid)))		\
+			break;						\
+		iwl_fw_dbg_collect_trig(&(_mvm)->fwrt, _trig, _fmt);	\
 	} while (0)
 
 static void
@@ -866,7 +871,8 @@ iwl_mvm_ampdu_check_trigger(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	trig = iwl_fw_dbg_get_trigger(mvm->fw, FW_DBG_TRIGGER_BA);
 	ba_trig = (void *)trig->data;
 
-	if (!iwl_fw_dbg_trigger_check_stop(mvm, vif, trig))
+	if (!iwl_fw_dbg_trigger_check_stop(&mvm->fwrt,
+					   ieee80211_vif_to_wdev(vif), trig))
 		return;
 
 	switch (action) {
@@ -1029,8 +1035,8 @@ static void iwl_mvm_restart_cleanup(struct iwl_mvm *mvm)
 	 * on D3->D0 transition
 	 */
 	if (!test_and_clear_bit(IWL_MVM_STATUS_D3_RECONFIG, &mvm->status)) {
-		mvm->fw_dump_desc = &iwl_mvm_dump_desc_assert;
-		iwl_mvm_fw_error_dump(mvm);
+		mvm->fwrt.dump.desc = &iwl_dump_desc_assert;
+		iwl_fw_error_dump(&mvm->fwrt);
 	}
 
 	/* cleanup all stale references (scan, roc), but keep the
@@ -1059,9 +1065,7 @@ static void iwl_mvm_restart_cleanup(struct iwl_mvm *mvm)
 
 	iwl_mvm_reset_phy_ctxts(mvm);
 	memset(mvm->fw_key_table, 0, sizeof(mvm->fw_key_table));
-	memset(mvm->sta_drained, 0, sizeof(mvm->sta_drained));
 	memset(mvm->sta_deferred_frames, 0, sizeof(mvm->sta_deferred_frames));
-	memset(mvm->tfd_drained, 0, sizeof(mvm->tfd_drained));
 	memset(&mvm->last_bt_notif, 0, sizeof(mvm->last_bt_notif));
 	memset(&mvm->last_bt_ci_cmd, 0, sizeof(mvm->last_bt_ci_cmd));
 
@@ -1072,7 +1076,7 @@ static void iwl_mvm_restart_cleanup(struct iwl_mvm *mvm)
 
 	mvm->vif_count = 0;
 	mvm->rx_ba_sessions = 0;
-	mvm->fw_dbg_conf = FW_DBG_INVALID;
+	mvm->fwrt.dump.conf = FW_DBG_INVALID;
 
 	/* keep statistics ticking */
 	iwl_mvm_accu_radio_stats(mvm);
@@ -1255,16 +1259,16 @@ static void iwl_mvm_mac_stop(struct ieee80211_hw *hw)
 	 * Lock and clear the firmware running bit here already, so that
 	 * new commands coming in elsewhere, e.g. from debugfs, will not
 	 * be able to proceed. This is important here because one of those
-	 * debugfs files causes the fw_dump_wk to be triggered, and if we
+	 * debugfs files causes the firmware dump to be triggered, and if we
 	 * don't stop debugfs accesses before canceling that it could be
 	 * retriggered after we flush it but before we've cleared the bit.
 	 */
 	clear_bit(IWL_MVM_STATUS_FIRMWARE_RUNNING, &mvm->status);
 
-	cancel_delayed_work_sync(&mvm->fw_dump_wk);
+	iwl_fw_cancel_dump(&mvm->fwrt);
 	cancel_delayed_work_sync(&mvm->cs_tx_unblock_dwork);
 	cancel_delayed_work_sync(&mvm->scan_timeout_dwork);
-	iwl_mvm_free_fw_dump_desc(mvm);
+	iwl_fw_free_dump_desc(&mvm->fwrt);
 
 	mutex_lock(&mvm->mutex);
 	__iwl_mvm_mac_stop(mvm);
@@ -1370,17 +1374,15 @@ static int iwl_mvm_mac_add_interface(struct ieee80211_hw *hw,
 			goto out_release;
 		}
 
-		if (iwl_mvm_is_dqa_supported(mvm)) {
-			/*
-			 * Only queue for this station is the mcast queue,
-			 * which shouldn't be in TFD mask anyway
-			 */
-			ret = iwl_mvm_allocate_int_sta(mvm, &mvmvif->mcast_sta,
-						       0, vif->type,
-						       IWL_STA_MULTICAST);
-			if (ret)
-				goto out_release;
-		}
+		/*
+		 * Only queue for this station is the mcast queue,
+		 * which shouldn't be in TFD mask anyway
+		 */
+		ret = iwl_mvm_allocate_int_sta(mvm, &mvmvif->mcast_sta,
+					       0, vif->type,
+					       IWL_STA_MULTICAST);
+		if (ret)
+			goto out_release;
 
 		iwl_mvm_vif_dbgfs_register(mvm, vif);
 		goto out_unlock;
@@ -1426,7 +1428,7 @@ static int iwl_mvm_mac_add_interface(struct ieee80211_hw *hw,
 		if (ret)
 			goto out_unref_phy;
 
-		ret = iwl_mvm_add_bcast_sta(mvm, vif);
+		ret = iwl_mvm_add_p2p_bcast_sta(mvm, vif);
 		if (ret)
 			goto out_unbind;
 
@@ -1454,8 +1456,6 @@ static int iwl_mvm_mac_add_interface(struct ieee80211_hw *hw,
  out_release:
 	if (vif->type != NL80211_IFTYPE_P2P_DEVICE)
 		mvm->vif_count--;
-
-	iwl_mvm_mac_ctxt_release(mvm, vif);
  out_unlock:
 	mutex_unlock(&mvm->mutex);
 
@@ -1467,40 +1467,6 @@ static int iwl_mvm_mac_add_interface(struct ieee80211_hw *hw,
 static void iwl_mvm_prepare_mac_removal(struct iwl_mvm *mvm,
 					struct ieee80211_vif *vif)
 {
-	u32 tfd_msk = iwl_mvm_mac_get_queues_mask(vif);
-
-	if (tfd_msk && !iwl_mvm_is_dqa_supported(mvm)) {
-		/*
-		 * mac80211 first removes all the stations of the vif and
-		 * then removes the vif. When it removes a station it also
-		 * flushes the AMPDU session. So by now, all the AMPDU sessions
-		 * of all the stations of this vif are closed, and the queues
-		 * of these AMPDU sessions are properly closed.
-		 * We still need to take care of the shared queues of the vif.
-		 * Flush them here.
-		 * For DQA mode there is no need - broacast and multicast queue
-		 * are flushed separately.
-		 */
-		mutex_lock(&mvm->mutex);
-		iwl_mvm_flush_tx_path(mvm, tfd_msk, 0);
-		mutex_unlock(&mvm->mutex);
-
-		/*
-		 * There are transports that buffer a few frames in the host.
-		 * For these, the flush above isn't enough since while we were
-		 * flushing, the transport might have sent more frames to the
-		 * device. To solve this, wait here until the transport is
-		 * empty. Technically, this could have replaced the flush
-		 * above, but flush is much faster than draining. So flush
-		 * first, and drain to make sure we have no frames in the
-		 * transport anymore.
-		 * If a station still had frames on the shared queues, it is
-		 * already marked as draining, so to complete the draining, we
-		 * just need to wait until the transport is empty.
-		 */
-		iwl_trans_wait_tx_queues_empty(mvm->trans, tfd_msk);
-	}
-
 	if (vif->type == NL80211_IFTYPE_P2P_DEVICE) {
 		/*
 		 * Flush the ROC worker which will flush the OFFCHANNEL queue.
@@ -1508,14 +1474,6 @@ static void iwl_mvm_prepare_mac_removal(struct iwl_mvm *mvm,
 		 * queue are sent in ROC session.
 		 */
 		flush_work(&mvm->roc_done_wk);
-	} else {
-		/*
-		 * By now, all the AC queues are empty. The AGG queues are
-		 * empty too. We already got all the Tx responses for all the
-		 * packets in the queues. The drain work can have been
-		 * triggered. Flush it.
-		 */
-		flush_work(&mvm->sta_drained_wk);
 	}
 }
 
@@ -1556,7 +1514,7 @@ static void iwl_mvm_mac_remove_interface(struct ieee80211_hw *hw,
 
 	if (vif->type == NL80211_IFTYPE_P2P_DEVICE) {
 		mvm->p2p_device_vif = NULL;
-		iwl_mvm_rm_bcast_sta(mvm, vif);
+		iwl_mvm_rm_p2p_bcast_sta(mvm, vif);
 		iwl_mvm_binding_remove_vif(mvm, vif);
 		iwl_mvm_phy_ctxt_unref(mvm, mvmvif->phy_ctxt);
 		mvmvif->phy_ctxt = NULL;
@@ -1569,7 +1527,6 @@ static void iwl_mvm_mac_remove_interface(struct ieee80211_hw *hw,
 	iwl_mvm_mac_ctxt_remove(mvm, vif);
 
 out_release:
-	iwl_mvm_mac_ctxt_release(mvm, vif);
 	mutex_unlock(&mvm->mutex);
 }
 
@@ -2067,8 +2024,7 @@ static void iwl_mvm_bss_info_changed_station(struct iwl_mvm *mvm,
 		 * We received a beacon from the associated AP so
 		 * remove the session protection.
 		 */
-		iwl_mvm_remove_time_event(mvm, mvmvif,
-					  &mvmvif->time_event_data);
+		iwl_mvm_stop_session_protection(mvm, vif);
 
 		iwl_mvm_sf_update(mvm, vif, false);
 		WARN_ON(iwl_mvm_enable_beacon_filter(mvm, vif, 0));
@@ -2405,15 +2361,18 @@ static void __iwl_mvm_mac_sta_notify(struct ieee80211_hw *hw,
 	unsigned long txqs = 0, tids = 0;
 	int tid;
 
+	/*
+	 * If we have TVQM then we get too high queue numbers - luckily
+	 * we really shouldn't get here with that because such hardware
+	 * should have firmware supporting buffer station offload.
+	 */
+	if (WARN_ON(iwl_mvm_has_new_tx_api(mvm)))
+		return;
+
 	spin_lock_bh(&mvmsta->lock);
 	for (tid = 0; tid < IWL_MAX_TID_COUNT; tid++) {
 		struct iwl_mvm_tid_data *tid_data = &mvmsta->tid_data[tid];
 
-		if (!iwl_mvm_is_dqa_supported(mvm) &&
-		    tid_data->state != IWL_AGG_ON &&
-		    tid_data->state != IWL_EMPTYING_HW_QUEUE_DELBA)
-			continue;
-
 		if (tid_data->txq_id == IWL_MVM_INVALID_QUEUE)
 			continue;
 
@@ -2427,9 +2386,6 @@ static void __iwl_mvm_mac_sta_notify(struct ieee80211_hw *hw,
 
 	switch (cmd) {
 	case STA_NOTIFY_SLEEP:
-		if (atomic_read(&mvm->pending_frames[mvmsta->sta_id]) > 0)
-			ieee80211_sta_block_awake(hw, sta, true);
-
 		for_each_set_bit(tid, &tids, IWL_MAX_TID_COUNT)
 			ieee80211_sta_set_buffered(sta, tid, true);
 
@@ -2572,7 +2528,8 @@ iwl_mvm_tdls_check_trigger(struct iwl_mvm *mvm,
 
 	trig = iwl_fw_dbg_get_trigger(mvm->fw, FW_DBG_TRIGGER_TDLS);
 	tdls_trig = (void *)trig->data;
-	if (!iwl_fw_dbg_trigger_check_stop(mvm, vif, trig))
+	if (!iwl_fw_dbg_trigger_check_stop(&mvm->fwrt,
+					   ieee80211_vif_to_wdev(vif), trig))
 		return;
 
 	if (!(tdls_trig->action_bitmap & BIT(action)))
@@ -2582,9 +2539,9 @@ iwl_mvm_tdls_check_trigger(struct iwl_mvm *mvm,
 	    memcmp(tdls_trig->peer, peer_addr, ETH_ALEN) != 0)
 		return;
 
-	iwl_mvm_fw_dbg_collect_trig(mvm, trig,
-				    "TDLS event occurred, peer %pM, action %d",
-				    peer_addr, action);
+	iwl_fw_dbg_collect_trig(&mvm->fwrt, trig,
+				"TDLS event occurred, peer %pM, action %d",
+				peer_addr, action);
 }
 
 static void iwl_mvm_purge_deferred_tx_frames(struct iwl_mvm *mvm,
@@ -2597,8 +2554,18 @@ static void iwl_mvm_purge_deferred_tx_frames(struct iwl_mvm *mvm,
 	spin_lock_bh(&mvm_sta->lock);
 	for (i = 0; i <= IWL_MAX_TID_COUNT; i++) {
 		tid_data = &mvm_sta->tid_data[i];
-		while ((skb = __skb_dequeue(&tid_data->deferred_tx_frames)))
+
+		while ((skb = __skb_dequeue(&tid_data->deferred_tx_frames))) {
+			struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+
+			/*
+			 * The first deferred frame should've stopped the MAC
+			 * queues, so we should never get a second deferred
+			 * frame for the RA/TID.
+			 */
+			iwl_mvm_start_mac_queues(mvm, info->hw_queue);
 			ieee80211_free_txskb(mvm->hw, skb);
+		}
 	}
 	spin_unlock_bh(&mvm_sta->lock);
 }
@@ -2621,9 +2588,6 @@ static int iwl_mvm_mac_sta_state(struct ieee80211_hw *hw,
 	if (WARN_ON_ONCE(!mvmvif->phy_ctxt))
 		return -EINVAL;
 
-	/* if a STA is being removed, reuse its ID */
-	flush_work(&mvm->sta_drained_wk);
-
 	/*
 	 * If we are in a STA removal flow and in DQA mode:
 	 *
@@ -2638,8 +2602,7 @@ static int iwl_mvm_mac_sta_state(struct ieee80211_hw *hw,
 	 * make sure the worker is no longer handling frames for this STA.
 	 */
 	if (old_state == IEEE80211_STA_NONE &&
-	    new_state == IEEE80211_STA_NOTEXIST &&
-	    iwl_mvm_is_dqa_supported(mvm)) {
+	    new_state == IEEE80211_STA_NOTEXIST) {
 		iwl_mvm_purge_deferred_tx_frames(mvm, mvm_sta);
 		flush_work(&mvm->add_stream_wk);
 
@@ -3882,7 +3845,9 @@ static int iwl_mvm_pre_channel_switch(struct ieee80211_hw *hw,
 	IWL_DEBUG_MAC80211(mvm, "pre CSA to freq %d\n",
 			   chsw->chandef.center_freq1);
 
-	iwl_fw_dbg_trigger_simple_stop(mvm, vif, FW_DBG_TRIGGER_CHANNEL_SWITCH);
+	iwl_fw_dbg_trigger_simple_stop(&mvm->fwrt,
+				       ieee80211_vif_to_wdev(vif),
+				       FW_DBG_TRIGGER_CHANNEL_SWITCH);
 
 	switch (vif->type) {
 	case NL80211_IFTYPE_AP:
@@ -3921,11 +3886,16 @@ static int iwl_mvm_pre_channel_switch(struct ieee80211_hw *hw,
 
 		/* Schedule the time event to a bit before beacon 1,
 		 * to make sure we're in the new channel when the
-		 * GO/AP arrives.
+		 * GO/AP arrives. In case count <= 1 immediately schedule the
+		 * TE (this might result with some packet loss or connection
+		 * loss).
 		 */
-		apply_time = chsw->device_timestamp +
-			((vif->bss_conf.beacon_int * (chsw->count - 1) -
-			  IWL_MVM_CHANNEL_SWITCH_TIME_CLIENT) * 1024);
+		if (chsw->count <= 1)
+			apply_time = 0;
+		else
+			apply_time = chsw->device_timestamp +
+				((vif->bss_conf.beacon_int * (chsw->count - 1) -
+				  IWL_MVM_CHANNEL_SWITCH_TIME_CLIENT) * 1024);
 
 		if (chsw->block_tx)
 			iwl_mvm_csa_client_absent(mvm, vif);
@@ -4019,8 +3989,7 @@ static void iwl_mvm_mac_flush(struct ieee80211_hw *hw,
 		return;
 
 	/* Make sure we're done with the deferred traffic before flushing */
-	if (iwl_mvm_is_dqa_supported(mvm))
-		flush_work(&mvm->add_stream_wk);
+	flush_work(&mvm->add_stream_wk);
 
 	mutex_lock(&mvm->mutex);
 	mvmvif = iwl_mvm_vif_from_mac80211(vif);
@@ -4157,11 +4126,11 @@ static void iwl_mvm_event_mlme_callback(struct iwl_mvm *mvm,
 					struct ieee80211_vif *vif,
 					const struct ieee80211_event *event)
 {
-#define CHECK_MLME_TRIGGER(_cnt, _fmt...)			\
-	do {							\
-		if ((trig_mlme->_cnt) && --(trig_mlme->_cnt))	\
-			break;					\
-		iwl_mvm_fw_dbg_collect_trig(mvm, trig, _fmt);	\
+#define CHECK_MLME_TRIGGER(_cnt, _fmt...)				\
+	do {								\
+		if ((trig_mlme->_cnt) && --(trig_mlme->_cnt))		\
+			break;						\
+		iwl_fw_dbg_collect_trig(&(mvm)->fwrt, trig, _fmt);	\
 	} while (0)
 
 	struct iwl_fw_dbg_trigger_tlv *trig;
@@ -4172,7 +4141,8 @@ static void iwl_mvm_event_mlme_callback(struct iwl_mvm *mvm,
 
 	trig = iwl_fw_dbg_get_trigger(mvm->fw, FW_DBG_TRIGGER_MLME);
 	trig_mlme = (void *)trig->data;
-	if (!iwl_fw_dbg_trigger_check_stop(mvm, vif, trig))
+	if (!iwl_fw_dbg_trigger_check_stop(&mvm->fwrt,
+					   ieee80211_vif_to_wdev(vif), trig))
 		return;
 
 	if (event->u.mlme.data == ASSOC_EVENT) {
@@ -4213,16 +4183,17 @@ static void iwl_mvm_event_bar_rx_callback(struct iwl_mvm *mvm,
 
 	trig = iwl_fw_dbg_get_trigger(mvm->fw, FW_DBG_TRIGGER_BA);
 	ba_trig = (void *)trig->data;
-	if (!iwl_fw_dbg_trigger_check_stop(mvm, vif, trig))
+	if (!iwl_fw_dbg_trigger_check_stop(&mvm->fwrt,
+					   ieee80211_vif_to_wdev(vif), trig))
 		return;
 
 	if (!(le16_to_cpu(ba_trig->rx_bar) & BIT(event->u.ba.tid)))
 		return;
 
-	iwl_mvm_fw_dbg_collect_trig(mvm, trig,
-				    "BAR received from %pM, tid %d, ssn %d",
-				    event->u.ba.sta->addr, event->u.ba.tid,
-				    event->u.ba.ssn);
+	iwl_fw_dbg_collect_trig(&mvm->fwrt, trig,
+				"BAR received from %pM, tid %d, ssn %d",
+				event->u.ba.sta->addr, event->u.ba.tid,
+				event->u.ba.ssn);
 }
 
 static void
@@ -4238,15 +4209,16 @@ iwl_mvm_event_frame_timeout_callback(struct iwl_mvm *mvm,
 
 	trig = iwl_fw_dbg_get_trigger(mvm->fw, FW_DBG_TRIGGER_BA);
 	ba_trig = (void *)trig->data;
-	if (!iwl_fw_dbg_trigger_check_stop(mvm, vif, trig))
+	if (!iwl_fw_dbg_trigger_check_stop(&mvm->fwrt,
+					   ieee80211_vif_to_wdev(vif), trig))
 		return;
 
 	if (!(le16_to_cpu(ba_trig->frame_timeout) & BIT(event->u.ba.tid)))
 		return;
 
-	iwl_mvm_fw_dbg_collect_trig(mvm, trig,
-				    "Frame from %pM timed out, tid %d",
-				    event->u.ba.sta->addr, event->u.ba.tid);
+	iwl_fw_dbg_collect_trig(&mvm->fwrt, trig,
+				"Frame from %pM timed out, tid %d",
+				event->u.ba.sta->addr, event->u.ba.tid);
 }
 
 static void iwl_mvm_mac_event_callback(struct ieee80211_hw *hw,
@@ -4280,7 +4252,8 @@ void iwl_mvm_sync_rx_queues_internal(struct iwl_mvm *mvm,
 	lockdep_assert_held(&mvm->mutex);
 
 	/* TODO - remove a000 disablement when we have RXQ config API */
-	if (!iwl_mvm_has_new_rx_api(mvm) || iwl_mvm_has_new_tx_api(mvm))
+	if (!iwl_mvm_has_new_rx_api(mvm) ||
+	    mvm->trans->cfg->device_family == IWL_DEVICE_FAMILY_A000)
 		return;
 
 	notif->cookie = mvm->queue_sync_cookie;
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
index ddd8719f27b8..83303bac0e4b 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
@@ -87,6 +87,8 @@
 #include "fw-api.h"
 #include "constants.h"
 #include "tof.h"
+#include "fw/runtime.h"
+#include "fw/dbg.h"
 
 #define IWL_MVM_MAX_ADDRESSES		5
 /* RSSI offset for WkP */
@@ -119,6 +121,9 @@
  */
 #define IWL_MVM_CS_UNBLOCK_TX_TIMEOUT 3
 
+/* offchannel queue towards mac80211 */
+#define IWL_MVM_OFFCHANNEL_QUEUE 0
+
 extern const struct ieee80211_ops iwl_mvm_hw_ops;
 
 /**
@@ -137,34 +142,6 @@ struct iwl_mvm_mod_params {
 };
 extern struct iwl_mvm_mod_params iwlmvm_mod_params;
 
-/**
- * struct iwl_mvm_dump_ptrs - set of pointers needed for the fw-error-dump
- *
- * @op_mode_ptr: pointer to the buffer coming from the mvm op_mode
- * @trans_ptr: pointer to struct %iwl_trans_dump_data which contains the
- *	transport's data.
- * @trans_len: length of the valid data in trans_ptr
- * @op_mode_len: length of the valid data in op_mode_ptr
- */
-struct iwl_mvm_dump_ptrs {
-	struct iwl_trans_dump_data *trans_ptr;
-	void *op_mode_ptr;
-	u32 op_mode_len;
-};
-
-/**
- * struct iwl_mvm_dump_desc - describes the dump
- * @len: length of trig_desc->data
- * @trig_desc: the description of the dump
- */
-struct iwl_mvm_dump_desc {
-	size_t len;
-	/* must be last */
-	struct iwl_fw_error_dump_trigger_desc trig_desc;
-};
-
-extern const struct iwl_mvm_dump_desc iwl_mvm_dump_desc_assert;
-
 struct iwl_mvm_phy_ctxt {
 	u16 id;
 	u16 color;
@@ -606,19 +583,6 @@ enum iwl_mvm_tdls_cs_state {
 	IWL_MVM_TDLS_SW_ACTIVE,
 };
 
-#define MAX_NUM_LMAC 2
-struct iwl_mvm_shared_mem_cfg {
-	int num_lmacs;
-	int num_txfifo_entries;
-	struct {
-		u32 txfifo_size[TX_FIFO_MAX_NUM];
-		u32 rxfifo1_size;
-	} lmac[MAX_NUM_LMAC];
-	u32 rxfifo2_size;
-	u32 internal_txfifo_addr;
-	u32 internal_txfifo_size[TX_FIFO_INTERNAL_MAX_NUM];
-};
-
 /**
  * struct iwl_mvm_reorder_buffer - per ra/tid/queue reorder buffer
  * @head_sn: reorder window head sn
@@ -766,7 +730,6 @@ struct iwl_mvm {
 	 */
 	struct iwl_mvm_vif *bf_allowed_vif;
 
-	enum iwl_ucode_type cur_ucode;
 	bool hw_registered;
 	bool calibrating;
 	u32 error_event_table[2];
@@ -815,10 +778,7 @@ struct iwl_mvm {
 	/* NVM sections */
 	struct iwl_nvm_section nvm_sections[NVM_MAX_NUM_SECTIONS];
 
-	/* Paging section */
-	struct iwl_fw_paging fw_paging_db[NUM_OF_FW_PAGING_BLOCKS];
-	u16 num_of_paging_blk;
-	u16 num_of_pages_in_last_blk;
+	struct iwl_fw_runtime fwrt;
 
 	/* EEPROM MAC addresses */
 	struct mac_address addresses[IWL_MVM_MAX_ADDRESSES];
@@ -826,11 +786,7 @@ struct iwl_mvm {
 	/* data related to data path */
 	struct iwl_rx_phy_info last_phy_info;
 	struct ieee80211_sta __rcu *fw_id_to_mac_id[IWL_MVM_STATION_COUNT];
-	struct work_struct sta_drained_wk;
 	unsigned long sta_deferred_frames[BITS_TO_LONGS(IWL_MVM_STATION_COUNT)];
-	unsigned long sta_drained[BITS_TO_LONGS(IWL_MVM_STATION_COUNT)];
-	atomic_t pending_frames[IWL_MVM_STATION_COUNT];
-	u32 tfd_drained[IWL_MVM_STATION_COUNT];
 	u8 rx_ba_sessions;
 
 	/* configured by mac80211 */
@@ -847,9 +803,6 @@ struct iwl_mvm {
 	/* max number of simultaneous scans the FW supports */
 	unsigned int max_scans;
 
-	/* ts of the beginning of a non-collect fw dbg data period */
-	unsigned long fw_dbg_non_collect_ts_start[FW_DBG_TRIGGER_MAX - 1];
-
 	/* UMAC scan tracking */
 	u32 scan_uid_status[IWL_MVM_MAX_UMAC_SCANS];
 
@@ -925,10 +878,6 @@ struct iwl_mvm {
 
 	/* -1 for always, 0 for never, >0 for that many times */
 	s8 fw_restart;
-	u8 fw_dbg_conf;
-	struct delayed_work fw_dump_wk;
-	const struct iwl_mvm_dump_desc *fw_dump_desc;
-	const struct iwl_fw_dbg_trigger_tlv *fw_dump_trig;
 
 #ifdef CONFIG_IWLWIFI_LEDS
 	struct led_classdev led;
@@ -975,8 +924,6 @@ struct iwl_mvm {
 	struct iwl_bt_coex_profile_notif last_bt_notif;
 	struct iwl_bt_coex_ci_cmd last_bt_ci_cmd;
 
-	u32 last_ant_isol;
-	u8 last_corun_lut;
 	u8 bt_tx_prio;
 	enum iwl_bt_force_ant_mode bt_force_ant_mode;
 
@@ -1010,9 +957,6 @@ struct iwl_mvm {
 	u16 probe_queue;
 	u16 p2p_dev_queue;
 
-	u8 first_agg_queue;
-	u8 last_agg_queue;
-
 	/* Indicate if device power save is allowed */
 	u8 ps_disabled; /* u8 instead of bool to ease debugfs_create_* usage */
 	unsigned int max_amsdu_len; /* used for debugfs only */
@@ -1055,7 +999,6 @@ struct iwl_mvm {
 		} peer;
 	} tdls_cs;
 
-	struct iwl_mvm_shared_mem_cfg smem_cfg;
 
 	u32 ciphers[IWL_MVM_NUM_CIPHERS];
 	struct ieee80211_cipher_scheme cs[IWL_UCODE_MAX_CS];
@@ -1095,7 +1038,6 @@ struct iwl_mvm {
  * @IWL_MVM_STATUS_IN_D0I3: NIC is in D0i3
  * @IWL_MVM_STATUS_ROC_AUX_RUNNING: AUX remain-on-channel is running
  * @IWL_MVM_STATUS_D3_RECONFIG: D3 reconfiguration is being done
- * @IWL_MVM_STATUS_DUMPING_FW_LOG: FW log is being dumped
  * @IWL_MVM_STATUS_FIRMWARE_RUNNING: firmware is running
  */
 enum iwl_mvm_status {
@@ -1107,7 +1049,6 @@ enum iwl_mvm_status {
 	IWL_MVM_STATUS_IN_D0I3,
 	IWL_MVM_STATUS_ROC_AUX_RUNNING,
 	IWL_MVM_STATUS_D3_RECONFIG,
-	IWL_MVM_STATUS_DUMPING_FW_LOG,
 	IWL_MVM_STATUS_FIRMWARE_RUNNING,
 };
 
@@ -1180,12 +1121,6 @@ static inline bool iwl_mvm_is_d0i3_supported(struct iwl_mvm *mvm)
 			    IWL_UCODE_TLV_CAPA_D0I3_SUPPORT);
 }
 
-static inline bool iwl_mvm_is_dqa_supported(struct iwl_mvm *mvm)
-{
-	return fw_has_capa(&mvm->fw->ucode_capa,
-			   IWL_UCODE_TLV_CAPA_DQA_SUPPORT);
-}
-
 static inline bool iwl_mvm_enter_d0i3_on_suspend(struct iwl_mvm *mvm)
 {
 	/* For now we only use this mode to differentiate between
@@ -1238,13 +1173,6 @@ static inline bool iwl_mvm_is_wifi_mcc_supported(struct iwl_mvm *mvm)
 			   IWL_UCODE_TLV_CAPA_LAR_MULTI_MCC);
 }
 
-static inline bool iwl_mvm_bt_is_plcr_supported(struct iwl_mvm *mvm)
-{
-	return fw_has_capa(&mvm->fw->ucode_capa,
-			   IWL_UCODE_TLV_CAPA_BT_COEX_PLCR) &&
-		IWL_MVM_BT_COEX_CORUNNING;
-}
-
 static inline bool iwl_mvm_bt_is_rrc_supported(struct iwl_mvm *mvm)
 {
 	return fw_has_capa(&mvm->fw->ucode_capa,
@@ -1287,6 +1215,12 @@ static inline bool iwl_mvm_has_new_tx_api(struct iwl_mvm *mvm)
 	return mvm->trans->cfg->use_tfh;
 }
 
+static inline bool iwl_mvm_has_unified_ucode(struct iwl_mvm *mvm)
+{
+	/* TODO - better define this */
+	return mvm->trans->cfg->device_family >= IWL_DEVICE_FAMILY_A000;
+}
+
 static inline bool iwl_mvm_is_cdb_supported(struct iwl_mvm *mvm)
 {
 	/*
@@ -1308,6 +1242,12 @@ static inline bool iwl_mvm_has_new_rx_stats_api(struct iwl_mvm *mvm)
 			  IWL_UCODE_TLV_API_NEW_RX_STATS);
 }
 
+static inline bool iwl_mvm_has_new_ats_coex_api(struct iwl_mvm *mvm)
+{
+	return fw_has_api(&mvm->fw->ucode_capa,
+			  IWL_UCODE_TLV_API_COEX_ATS_EXTERNAL);
+}
+
 static inline struct agg_tx_status *
 iwl_mvm_get_agg_status(struct iwl_mvm *mvm, void *tx_resp)
 {
@@ -1340,6 +1280,14 @@ static inline bool iwl_mvm_is_ctdp_supported(struct iwl_mvm *mvm)
 }
 
 extern const u8 iwl_mvm_ac_to_tx_fifo[];
+extern const u8 iwl_mvm_ac_to_gen2_tx_fifo[];
+
+static inline u8 iwl_mvm_mac_ac_to_tx_fifo(struct iwl_mvm *mvm,
+					   enum ieee80211_ac_numbers ac)
+{
+	return iwl_mvm_has_new_tx_api(mvm) ?
+		iwl_mvm_ac_to_gen2_tx_fifo[ac] : iwl_mvm_ac_to_tx_fifo[ac];
+}
 
 struct iwl_rate_info {
 	u8 plcp;	/* uCode API:  IWL_RATE_6M_PLCP, etc. */
@@ -1425,8 +1373,7 @@ int iwl_mvm_request_statistics(struct iwl_mvm *mvm, bool clear);
 void iwl_mvm_accu_radio_stats(struct iwl_mvm *mvm);
 
 /* NVM */
-int iwl_nvm_init(struct iwl_mvm *mvm, bool read_nvm_from_nic);
-int iwl_mvm_nvm_get_from_fw(struct iwl_mvm *mvm);
+int iwl_nvm_init(struct iwl_mvm *mvm);
 int iwl_mvm_load_nvm_to_nic(struct iwl_mvm *mvm);
 int iwl_mvm_read_external_nvm(struct iwl_mvm *mvm);
 
@@ -1510,7 +1457,6 @@ u8 iwl_mvm_get_ctrl_pos(struct cfg80211_chan_def *chandef);
 
 /* MAC (virtual interface) programming */
 int iwl_mvm_mac_ctxt_init(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
-void iwl_mvm_mac_ctxt_release(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 int iwl_mvm_mac_ctxt_add(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 int iwl_mvm_mac_ctxt_changed(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 			     bool force_assoc_off, const u8 *bssid_override);
@@ -1573,9 +1519,6 @@ void iwl_mvm_rx_umac_scan_complete_notif(struct iwl_mvm *mvm,
 void iwl_mvm_rx_umac_scan_iter_complete_notif(struct iwl_mvm *mvm,
 					      struct iwl_rx_cmd_buffer *rxb);
 
-/* Paging */
-void iwl_free_fw_paging(struct iwl_mvm *mvm);
-
 /* MVM debugfs */
 #ifdef CONFIG_IWLWIFI_DEBUGFS
 int iwl_mvm_dbgfs_register(struct iwl_mvm *mvm, struct dentry *dbgfs_dir);
@@ -1619,6 +1562,7 @@ void iwl_mvm_power_uapsd_misbehaving_ap_notif(struct iwl_mvm *mvm,
 #ifdef CONFIG_IWLWIFI_LEDS
 int iwl_mvm_leds_init(struct iwl_mvm *mvm);
 void iwl_mvm_leds_exit(struct iwl_mvm *mvm);
+void iwl_mvm_leds_sync(struct iwl_mvm *mvm);
 #else
 static inline int iwl_mvm_leds_init(struct iwl_mvm *mvm)
 {
@@ -1627,6 +1571,9 @@ static inline int iwl_mvm_leds_init(struct iwl_mvm *mvm)
 static inline void iwl_mvm_leds_exit(struct iwl_mvm *mvm)
 {
 }
+static inline void iwl_mvm_leds_sync(struct iwl_mvm *mvm)
+{
+}
 #endif
 
 /* D3 (WoWLAN, NetDetect) */
@@ -1764,10 +1711,6 @@ bool iwl_mvm_enable_txq(struct iwl_mvm *mvm, int queue, int mac80211_queue,
 int iwl_mvm_tvqm_enable_txq(struct iwl_mvm *mvm, int mac80211_queue,
 			    u8 sta_id, u8 tid, unsigned int timeout);
 
-/*
- * Disable a TXQ.
- * Note that in non-DQA mode the %mac80211_queue and %tid params are ignored.
- */
 int iwl_mvm_disable_txq(struct iwl_mvm *mvm, int queue, int mac80211_queue,
 			u8 tid, u8 flags);
 int iwl_mvm_find_free_queue(struct iwl_mvm *mvm, u8 sta_id, u8 minq, u8 maxq);
@@ -1777,33 +1720,15 @@ int iwl_mvm_find_free_queue(struct iwl_mvm *mvm, u8 sta_id, u8 minq, u8 maxq);
  */
 static inline u32 iwl_mvm_flushable_queues(struct iwl_mvm *mvm)
 {
-	u32 cmd_queue = iwl_mvm_is_dqa_supported(mvm) ? IWL_MVM_DQA_CMD_QUEUE :
-		IWL_MVM_CMD_QUEUE;
-
 	return ((BIT(mvm->cfg->base_params->num_of_queues) - 1) &
-		~BIT(cmd_queue));
-}
-
-static inline
-void iwl_mvm_enable_ac_txq(struct iwl_mvm *mvm, int queue, int mac80211_queue,
-			   u8 fifo, u16 ssn, unsigned int wdg_timeout)
-{
-	struct iwl_trans_txq_scd_cfg cfg = {
-		.fifo = fifo,
-		.tid = IWL_MAX_TID_COUNT,
-		.aggregate = false,
-		.frame_limit = IWL_FRAME_LIMIT,
-	};
-
-	iwl_mvm_enable_txq(mvm, queue, mac80211_queue, ssn, &cfg, wdg_timeout);
+		~BIT(IWL_MVM_DQA_CMD_QUEUE));
 }
 
 static inline void iwl_mvm_stop_device(struct iwl_mvm *mvm)
 {
-	if (!iwl_mvm_has_new_tx_api(mvm))
-		iwl_free_fw_paging(mvm);
+	iwl_free_fw_paging(&mvm->fwrt);
 	clear_bit(IWL_MVM_STATUS_FIRMWARE_RUNNING, &mvm->status);
-	mvm->fw_dbg_conf = FW_DBG_INVALID;
+	iwl_fw_dump_conf_clear(&mvm->fwrt);
 	iwl_trans_stop_device(mvm->trans);
 }
 
@@ -1826,6 +1751,7 @@ void iwl_mvm_thermal_exit(struct iwl_mvm *mvm);
 void iwl_mvm_set_hw_ctkill_state(struct iwl_mvm *mvm, bool state);
 int iwl_mvm_get_temp(struct iwl_mvm *mvm, s32 *temp);
 void iwl_mvm_ct_kill_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb);
+void iwl_mvm_enter_ctkill(struct iwl_mvm *mvm);
 int iwl_mvm_send_temp_report_ths_cmd(struct iwl_mvm *mvm);
 int iwl_mvm_ctdp_command(struct iwl_mvm *mvm, u32 op, u32 budget);
 
@@ -1899,21 +1825,7 @@ int iwl_mvm_send_lqm_cmd(struct ieee80211_vif *vif,
 			 u32 duration, u32 timeout);
 bool iwl_mvm_lqm_active(struct iwl_mvm *mvm);
 
-#ifdef CONFIG_ACPI
 int iwl_mvm_sar_select_profile(struct iwl_mvm *mvm, int prof_a, int prof_b);
 int iwl_mvm_get_sar_geo_profile(struct iwl_mvm *mvm);
-#else
-static inline
-int iwl_mvm_sar_select_profile(struct iwl_mvm *mvm, int prof_a, int prof_b)
-{
-	return -ENOENT;
-}
-
-static inline
-int iwl_mvm_get_sar_geo_profile(struct iwl_mvm *mvm)
-{
-	return -ENOENT;
-}
-#endif /* CONFIG_ACPI */
 
 #endif /* __IWL_MVM_H__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/nvm.c b/drivers/net/wireless/intel/iwlwifi/mvm/nvm.c
index dac7e542a190..422aa6be9932 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/nvm.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/nvm.c
@@ -292,7 +292,8 @@ static struct iwl_nvm_data *
 iwl_parse_nvm_sections(struct iwl_mvm *mvm)
 {
 	struct iwl_nvm_section *sections = mvm->nvm_sections;
-	const __le16 *hw, *sw, *calib, *regulatory, *mac_override, *phy_sku;
+	const __be16 *hw;
+	const __le16 *sw, *calib, *regulatory, *mac_override, *phy_sku;
 	bool lar_enabled;
 
 	/* Checking for required sections */
@@ -326,10 +327,7 @@ iwl_parse_nvm_sections(struct iwl_mvm *mvm)
 		}
 	}
 
-	if (WARN_ON(!mvm->cfg))
-		return NULL;
-
-	hw = (const __le16 *)sections[mvm->cfg->nvm_hw_section_num].data;
+	hw = (const __be16 *)sections[mvm->cfg->nvm_hw_section_num].data;
 	sw = (const __le16 *)sections[NVM_SECTION_TYPE_SW].data;
 	calib = (const __le16 *)sections[NVM_SECTION_TYPE_CALIBRATION].data;
 	regulatory = (const __le16 *)sections[NVM_SECTION_TYPE_REGULATORY].data;
@@ -546,101 +544,7 @@ int iwl_mvm_load_nvm_to_nic(struct iwl_mvm *mvm)
 	return ret;
 }
 
-int iwl_mvm_nvm_get_from_fw(struct iwl_mvm *mvm)
-{
-	struct iwl_nvm_get_info cmd = {};
-	struct iwl_nvm_get_info_rsp *rsp;
-	struct iwl_trans *trans = mvm->trans;
-	struct iwl_host_cmd hcmd = {
-		.flags = CMD_WANT_SKB | CMD_SEND_IN_RFKILL,
-		.data = { &cmd, },
-		.len = { sizeof(cmd) },
-		.id = WIDE_ID(REGULATORY_AND_NVM_GROUP, NVM_GET_INFO)
-	};
-	int  ret;
-	bool lar_fw_supported = !iwlwifi_mod_params.lar_disable &&
-				fw_has_capa(&mvm->fw->ucode_capa,
-					    IWL_UCODE_TLV_CAPA_LAR_SUPPORT);
-
-	lockdep_assert_held(&mvm->mutex);
-
-	ret = iwl_mvm_send_cmd(mvm, &hcmd);
-	if (ret)
-		return ret;
-
-	if (WARN(iwl_rx_packet_payload_len(hcmd.resp_pkt) != sizeof(*rsp),
-		 "Invalid payload len in NVM response from FW %d",
-		 iwl_rx_packet_payload_len(hcmd.resp_pkt))) {
-		ret = -EINVAL;
-		goto out;
-	}
-
-	rsp = (void *)hcmd.resp_pkt->data;
-	if (le32_to_cpu(rsp->general.flags)) {
-		IWL_ERR(mvm, "Invalid NVM data from FW\n");
-		ret = -EINVAL;
-		goto out;
-	}
-
-	mvm->nvm_data = kzalloc(sizeof(*mvm->nvm_data) +
-				sizeof(struct ieee80211_channel) *
-				IWL_NUM_CHANNELS, GFP_KERNEL);
-	if (!mvm->nvm_data) {
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	iwl_set_hw_address_from_csr(trans, mvm->nvm_data);
-	/* TODO: if platform NVM has MAC address - override it here */
-
-	if (!is_valid_ether_addr(mvm->nvm_data->hw_addr)) {
-		IWL_ERR(trans, "no valid mac address was found\n");
-		ret = -EINVAL;
-		goto err_free;
-	}
-
-	IWL_INFO(trans, "base HW address: %pM\n", mvm->nvm_data->hw_addr);
-
-	/* Initialize general data */
-	mvm->nvm_data->nvm_version = le16_to_cpu(rsp->general.nvm_version);
-
-	/* Initialize MAC sku data */
-	mvm->nvm_data->sku_cap_11ac_enable =
-		le32_to_cpu(rsp->mac_sku.enable_11ac);
-	mvm->nvm_data->sku_cap_11n_enable =
-		le32_to_cpu(rsp->mac_sku.enable_11n);
-	mvm->nvm_data->sku_cap_band_24GHz_enable =
-		le32_to_cpu(rsp->mac_sku.enable_24g);
-	mvm->nvm_data->sku_cap_band_52GHz_enable =
-		le32_to_cpu(rsp->mac_sku.enable_5g);
-	mvm->nvm_data->sku_cap_mimo_disabled =
-		le32_to_cpu(rsp->mac_sku.mimo_disable);
-
-	/* Initialize PHY sku data */
-	mvm->nvm_data->valid_tx_ant = (u8)le32_to_cpu(rsp->phy_sku.tx_chains);
-	mvm->nvm_data->valid_rx_ant = (u8)le32_to_cpu(rsp->phy_sku.rx_chains);
-
-	/* Initialize regulatory data */
-	mvm->nvm_data->lar_enabled =
-		le32_to_cpu(rsp->regulatory.lar_enabled) && lar_fw_supported;
-
-	iwl_init_sbands(trans->dev, trans->cfg, mvm->nvm_data,
-			rsp->regulatory.channel_profile,
-			mvm->nvm_data->valid_tx_ant & mvm->fw->valid_tx_ant,
-			mvm->nvm_data->valid_rx_ant & mvm->fw->valid_rx_ant,
-			rsp->regulatory.lar_enabled && lar_fw_supported);
-
-	iwl_free_resp(&hcmd);
-	return 0;
-
-err_free:
-	kfree(mvm->nvm_data);
-out:
-	iwl_free_resp(&hcmd);
-	return ret;
-}
-
-int iwl_nvm_init(struct iwl_mvm *mvm, bool read_nvm_from_nic)
+int iwl_nvm_init(struct iwl_mvm *mvm)
 {
 	int ret, section;
 	u32 size_read = 0;
@@ -651,63 +555,61 @@ int iwl_nvm_init(struct iwl_mvm *mvm, bool read_nvm_from_nic)
 		return -EINVAL;
 
 	/* load NVM values from nic */
-	if (read_nvm_from_nic) {
-		/* Read From FW NVM */
-		IWL_DEBUG_EEPROM(mvm->trans->dev, "Read from NVM\n");
-
-		nvm_buffer = kmalloc(mvm->cfg->base_params->eeprom_size,
-				     GFP_KERNEL);
-		if (!nvm_buffer)
-			return -ENOMEM;
-		for (section = 0; section < NVM_MAX_NUM_SECTIONS; section++) {
-			/* we override the constness for initial read */
-			ret = iwl_nvm_read_section(mvm, section, nvm_buffer,
-						   size_read);
-			if (ret < 0)
-				continue;
-			size_read += ret;
-			temp = kmemdup(nvm_buffer, ret, GFP_KERNEL);
-			if (!temp) {
-				ret = -ENOMEM;
-				break;
-			}
+	/* Read From FW NVM */
+	IWL_DEBUG_EEPROM(mvm->trans->dev, "Read from NVM\n");
+
+	nvm_buffer = kmalloc(mvm->cfg->base_params->eeprom_size,
+			     GFP_KERNEL);
+	if (!nvm_buffer)
+		return -ENOMEM;
+	for (section = 0; section < NVM_MAX_NUM_SECTIONS; section++) {
+		/* we override the constness for initial read */
+		ret = iwl_nvm_read_section(mvm, section, nvm_buffer,
+					   size_read);
+		if (ret < 0)
+			continue;
+		size_read += ret;
+		temp = kmemdup(nvm_buffer, ret, GFP_KERNEL);
+		if (!temp) {
+			ret = -ENOMEM;
+			break;
+		}
 
-			iwl_mvm_nvm_fixups(mvm, section, temp, ret);
+		iwl_mvm_nvm_fixups(mvm, section, temp, ret);
 
-			mvm->nvm_sections[section].data = temp;
-			mvm->nvm_sections[section].length = ret;
+		mvm->nvm_sections[section].data = temp;
+		mvm->nvm_sections[section].length = ret;
 
 #ifdef CONFIG_IWLWIFI_DEBUGFS
-			switch (section) {
-			case NVM_SECTION_TYPE_SW:
-				mvm->nvm_sw_blob.data = temp;
-				mvm->nvm_sw_blob.size  = ret;
-				break;
-			case NVM_SECTION_TYPE_CALIBRATION:
-				mvm->nvm_calib_blob.data = temp;
-				mvm->nvm_calib_blob.size  = ret;
-				break;
-			case NVM_SECTION_TYPE_PRODUCTION:
-				mvm->nvm_prod_blob.data = temp;
-				mvm->nvm_prod_blob.size  = ret;
-				break;
-			case NVM_SECTION_TYPE_PHY_SKU:
-				mvm->nvm_phy_sku_blob.data = temp;
-				mvm->nvm_phy_sku_blob.size  = ret;
+		switch (section) {
+		case NVM_SECTION_TYPE_SW:
+			mvm->nvm_sw_blob.data = temp;
+			mvm->nvm_sw_blob.size  = ret;
+			break;
+		case NVM_SECTION_TYPE_CALIBRATION:
+			mvm->nvm_calib_blob.data = temp;
+			mvm->nvm_calib_blob.size  = ret;
+			break;
+		case NVM_SECTION_TYPE_PRODUCTION:
+			mvm->nvm_prod_blob.data = temp;
+			mvm->nvm_prod_blob.size  = ret;
+			break;
+		case NVM_SECTION_TYPE_PHY_SKU:
+			mvm->nvm_phy_sku_blob.data = temp;
+			mvm->nvm_phy_sku_blob.size  = ret;
+			break;
+		default:
+			if (section == mvm->cfg->nvm_hw_section_num) {
+				mvm->nvm_hw_blob.data = temp;
+				mvm->nvm_hw_blob.size = ret;
 				break;
-			default:
-				if (section == mvm->cfg->nvm_hw_section_num) {
-					mvm->nvm_hw_blob.data = temp;
-					mvm->nvm_hw_blob.size = ret;
-					break;
-				}
 			}
-#endif
 		}
-		if (!size_read)
-			IWL_ERR(mvm, "OTP is blank\n");
-		kfree(nvm_buffer);
+#endif
 	}
+	if (!size_read)
+		IWL_ERR(mvm, "OTP is blank\n");
+	kfree(nvm_buffer);
 
 	/* Only if PNVM selected in the mod param - load external NVM  */
 	if (mvm->nvm_file_name) {
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
index 9c175d5e9d67..231878969332 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
@@ -82,11 +82,10 @@
 #include "iwl-io.h"
 #include "iwl-prph.h"
 #include "rs.h"
-#include "fw-api-scan.h"
+#include "fw/api/scan.h"
 #include "time-event.h"
-#include "fw-dbg.h"
 #include "fw-api.h"
-#include "fw-api-scan.h"
+#include "fw/api/scan.h"
 
 #define DRV_DESCRIPTION	"The new Intel(R) wireless AGN driver for Linux"
 MODULE_DESCRIPTION(DRV_DESCRIPTION);
@@ -257,8 +256,6 @@ static const struct iwl_rx_handlers iwl_mvm_rx_handlers[] = {
 		   RX_HANDLER_ASYNC_LOCKED),
 	RX_HANDLER(STATISTICS_NOTIFICATION, iwl_mvm_rx_statistics,
 		   RX_HANDLER_ASYNC_LOCKED),
-	RX_HANDLER(ANTENNA_COUPLING_NOTIFICATION,
-		   iwl_mvm_rx_ant_coupling_notif, RX_HANDLER_ASYNC_LOCKED),
 
 	RX_HANDLER(BA_WINDOW_STATUS_NOTIFICATION_ID,
 		   iwl_mvm_window_status_notif, RX_HANDLER_SYNC),
@@ -326,7 +323,6 @@ static const struct iwl_hcmd_names iwl_mvm_legacy_names[] = {
 	HCMD_NAME(INIT_COMPLETE_NOTIF),
 	HCMD_NAME(PHY_CONTEXT_CMD),
 	HCMD_NAME(DBG_CFG),
-	HCMD_NAME(ANTENNA_COUPLING_NOTIFICATION),
 	HCMD_NAME(SCAN_CFG_CMD),
 	HCMD_NAME(SCAN_REQ_UMAC),
 	HCMD_NAME(SCAN_ABORT_UMAC),
@@ -351,13 +347,13 @@ static const struct iwl_hcmd_names iwl_mvm_legacy_names[] = {
 	HCMD_NAME(BINDING_CONTEXT_CMD),
 	HCMD_NAME(TIME_QUOTA_CMD),
 	HCMD_NAME(NON_QOS_TX_COUNTER_CMD),
+	HCMD_NAME(LEDS_CMD),
 	HCMD_NAME(LQ_CMD),
 	HCMD_NAME(FW_PAGING_BLOCK_CMD),
 	HCMD_NAME(SCAN_OFFLOAD_REQUEST_CMD),
 	HCMD_NAME(SCAN_OFFLOAD_ABORT_CMD),
 	HCMD_NAME(HOT_SPOT_CMD),
 	HCMD_NAME(SCAN_OFFLOAD_PROFILES_QUERY_CMD),
-	HCMD_NAME(BT_COEX_UPDATE_CORUN_LUT),
 	HCMD_NAME(BT_COEX_UPDATE_REDUCED_TXP),
 	HCMD_NAME(BT_COEX_CI),
 	HCMD_NAME(PHY_CONFIGURATION_CMD),
@@ -388,6 +384,7 @@ static const struct iwl_hcmd_names iwl_mvm_legacy_names[] = {
 	HCMD_NAME(SCAN_ITERATION_COMPLETE_UMAC),
 	HCMD_NAME(REPLY_RX_PHY_CMD),
 	HCMD_NAME(REPLY_RX_MPDU_CMD),
+	HCMD_NAME(FRAME_RELEASE),
 	HCMD_NAME(BA_NOTIF),
 	HCMD_NAME(MCC_UPDATE_CMD),
 	HCMD_NAME(MCC_CHUB_UPDATE_CMD),
@@ -510,8 +507,6 @@ static u32 calc_min_backoff(struct iwl_trans *trans, const struct iwl_cfg *cfg)
 	return 0;
 }
 
-static void iwl_mvm_fw_error_dump_wk(struct work_struct *work);
-
 static void iwl_mvm_tx_unblock_dwork(struct work_struct *work)
 {
 	struct iwl_mvm *mvm =
@@ -535,6 +530,34 @@ unlock:
 	mutex_unlock(&mvm->mutex);
 }
 
+static int iwl_mvm_fwrt_dump_start(void *ctx)
+{
+	struct iwl_mvm *mvm = ctx;
+	int ret;
+
+	ret = iwl_mvm_ref_sync(mvm, IWL_MVM_REF_FW_DBG_COLLECT);
+	if (ret)
+		return ret;
+
+	mutex_lock(&mvm->mutex);
+
+	return 0;
+}
+
+static void iwl_mvm_fwrt_dump_end(void *ctx)
+{
+	struct iwl_mvm *mvm = ctx;
+
+	mutex_unlock(&mvm->mutex);
+
+	iwl_mvm_unref(mvm, IWL_MVM_REF_FW_DBG_COLLECT);
+}
+
+static const struct iwl_fw_runtime_ops iwl_mvm_fwrt_ops = {
+	.dump_start = iwl_mvm_fwrt_dump_start,
+	.dump_end = iwl_mvm_fwrt_dump_end,
+};
+
 static struct iwl_op_mode *
 iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 		      const struct iwl_fw *fw, struct dentry *dbgfs_dir)
@@ -580,6 +603,8 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 	mvm->fw = fw;
 	mvm->hw = hw;
 
+	iwl_fw_runtime_init(&mvm->fwrt, trans, fw, &iwl_mvm_fwrt_ops, mvm);
+
 	mvm->init_status = 0;
 
 	if (iwl_mvm_has_new_rx_api(mvm)) {
@@ -596,32 +621,15 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 
 	mvm->fw_restart = iwlwifi_mod_params.fw_restart ? -1 : 0;
 
-	if (!iwl_mvm_is_dqa_supported(mvm)) {
-		mvm->last_agg_queue = mvm->cfg->base_params->num_of_queues - 1;
-
-		if (mvm->cfg->base_params->num_of_queues == 16) {
-			mvm->aux_queue = 11;
-			mvm->first_agg_queue = 12;
-			BUILD_BUG_ON(BITS_PER_BYTE *
-				     sizeof(mvm->hw_queue_to_mac80211[0]) < 12);
-		} else {
-			mvm->aux_queue = 15;
-			mvm->first_agg_queue = 16;
-			BUILD_BUG_ON(BITS_PER_BYTE *
-				     sizeof(mvm->hw_queue_to_mac80211[0]) < 16);
-		}
-	} else {
-		mvm->aux_queue = IWL_MVM_DQA_AUX_QUEUE;
-		mvm->probe_queue = IWL_MVM_DQA_AP_PROBE_RESP_QUEUE;
-		mvm->p2p_dev_queue = IWL_MVM_DQA_P2P_DEVICE_QUEUE;
-		mvm->first_agg_queue = IWL_MVM_DQA_MIN_DATA_QUEUE;
-		mvm->last_agg_queue = IWL_MVM_DQA_MAX_DATA_QUEUE;
-	}
+	mvm->aux_queue = IWL_MVM_DQA_AUX_QUEUE;
+	mvm->probe_queue = IWL_MVM_DQA_AP_PROBE_RESP_QUEUE;
+	mvm->p2p_dev_queue = IWL_MVM_DQA_P2P_DEVICE_QUEUE;
+
 	mvm->sf_state = SF_UNINIT;
-	if (iwl_mvm_has_new_tx_api(mvm))
-		mvm->cur_ucode = IWL_UCODE_REGULAR;
+	if (iwl_mvm_has_unified_ucode(mvm))
+		iwl_fw_set_current_image(&mvm->fwrt, IWL_UCODE_REGULAR);
 	else
-		mvm->cur_ucode = IWL_UCODE_INIT;
+		iwl_fw_set_current_image(&mvm->fwrt, IWL_UCODE_INIT);
 	mvm->drop_bcn_ap_mode = true;
 
 	mutex_init(&mvm->mutex);
@@ -635,9 +643,7 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 
 	INIT_WORK(&mvm->async_handlers_wk, iwl_mvm_async_handlers_wk);
 	INIT_WORK(&mvm->roc_done_wk, iwl_mvm_roc_done_wk);
-	INIT_WORK(&mvm->sta_drained_wk, iwl_mvm_sta_drained_wk);
 	INIT_WORK(&mvm->d0i3_exit_work, iwl_mvm_d0i3_exit_work);
-	INIT_DELAYED_WORK(&mvm->fw_dump_wk, iwl_mvm_fw_error_dump_wk);
 	INIT_DELAYED_WORK(&mvm->tdls_cs.dwork, iwl_mvm_tdls_ch_switch_work);
 	INIT_DELAYED_WORK(&mvm->scan_timeout_dwork, iwl_mvm_scan_timeout_wk);
 	INIT_WORK(&mvm->add_stream_wk, iwl_mvm_add_new_dqa_stream_wk);
@@ -688,10 +694,7 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 	trans_cfg.command_groups = iwl_mvm_groups;
 	trans_cfg.command_groups_size = ARRAY_SIZE(iwl_mvm_groups);
 
-	if (iwl_mvm_is_dqa_supported(mvm))
-		trans_cfg.cmd_queue = IWL_MVM_DQA_CMD_QUEUE;
-	else
-		trans_cfg.cmd_queue = IWL_MVM_CMD_QUEUE;
+	trans_cfg.cmd_queue = IWL_MVM_DQA_CMD_QUEUE;
 	trans_cfg.cmd_fifo = IWL_MVM_TX_FIFO_CMD;
 	trans_cfg.scd_set_active = true;
 
@@ -749,7 +752,6 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 		iwl_mvm_stop_device(mvm);
 	iwl_mvm_unref(mvm, IWL_MVM_REF_INIT_UCODE);
 	mutex_unlock(&mvm->mutex);
-	/* returns 0 if successful, 1 if success but in rfkill */
 	if (err < 0) {
 		IWL_ERR(mvm, "Failed to run INIT ucode: %d\n", err);
 		goto out_free;
@@ -800,7 +802,7 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 	iwl_mvm_leds_exit(mvm);
 	iwl_mvm_thermal_exit(mvm);
  out_free:
-	flush_delayed_work(&mvm->fw_dump_wk);
+	iwl_fw_flush_dump(&mvm->fwrt);
 
 	if (iwlmvm_mod_params.init_dbg)
 		return op_mode;
@@ -920,7 +922,7 @@ static inline void iwl_mvm_rx_check_trigger(struct iwl_mvm *mvm,
 	trig = iwl_fw_dbg_get_trigger(mvm->fw, FW_DBG_TRIGGER_FW_NOTIF);
 	cmds_trig = (void *)trig->data;
 
-	if (!iwl_fw_dbg_trigger_check_stop(mvm, NULL, trig))
+	if (!iwl_fw_dbg_trigger_check_stop(&mvm->fwrt, NULL, trig))
 		return;
 
 	for (i = 0; i < ARRAY_SIZE(cmds_trig->cmds); i++) {
@@ -932,9 +934,9 @@ static inline void iwl_mvm_rx_check_trigger(struct iwl_mvm *mvm,
 		    cmds_trig->cmds[i].group_id != pkt->hdr.group_id)
 			continue;
 
-		iwl_mvm_fw_dbg_collect_trig(mvm, trig,
-					    "CMD 0x%02x.%02x received",
-					    pkt->hdr.group_id, pkt->hdr.cmd);
+		iwl_fw_dbg_collect_trig(&mvm->fwrt, trig,
+					"CMD 0x%02x.%02x received",
+					pkt->hdr.group_id, pkt->hdr.cmd);
 		break;
 	}
 }
@@ -980,8 +982,10 @@ static void iwl_mvm_rx_common(struct iwl_mvm *mvm,
 		list_add_tail(&entry->list, &mvm->async_handlers_list);
 		spin_unlock(&mvm->async_handlers_lock);
 		schedule_work(&mvm->async_handlers_wk);
-		break;
+		return;
 	}
+
+	iwl_fwrt_handle_notification(&mvm->fwrt, rxb);
 }
 
 static void iwl_mvm_rx(struct iwl_op_mode *op_mode,
@@ -1131,7 +1135,7 @@ static bool iwl_mvm_set_hw_rfkill_state(struct iwl_op_mode *op_mode, bool state)
 	 * Stop the device if we run OPERATIONAL firmware or if we are in the
 	 * middle of the calibrations.
 	 */
-	return state && (mvm->cur_ucode != IWL_UCODE_INIT || calibrating);
+	return state && (mvm->fwrt.cur_fw_img != IWL_UCODE_INIT || calibrating);
 }
 
 static void iwl_mvm_free_skb(struct iwl_op_mode *op_mode, struct sk_buff *skb)
@@ -1160,57 +1164,6 @@ static void iwl_mvm_reprobe_wk(struct work_struct *wk)
 	module_put(THIS_MODULE);
 }
 
-static void iwl_mvm_fw_error_dump_wk(struct work_struct *work)
-{
-	struct iwl_mvm *mvm =
-		container_of(work, struct iwl_mvm, fw_dump_wk.work);
-
-	if (iwl_mvm_ref_sync(mvm, IWL_MVM_REF_FW_DBG_COLLECT))
-		return;
-
-	mutex_lock(&mvm->mutex);
-
-	if (mvm->cfg->device_family == IWL_DEVICE_FAMILY_7000) {
-		/* stop recording */
-		iwl_set_bits_prph(mvm->trans, MON_BUFF_SAMPLE_CTL, 0x100);
-
-		iwl_mvm_fw_error_dump(mvm);
-
-		/* start recording again if the firmware is not crashed */
-		if (!test_bit(STATUS_FW_ERROR, &mvm->trans->status) &&
-		    mvm->fw->dbg_dest_tlv) {
-			iwl_clear_bits_prph(mvm->trans,
-					    MON_BUFF_SAMPLE_CTL, 0x100);
-			iwl_clear_bits_prph(mvm->trans,
-					    MON_BUFF_SAMPLE_CTL, 0x1);
-			iwl_set_bits_prph(mvm->trans, MON_BUFF_SAMPLE_CTL, 0x1);
-		}
-	} else {
-		u32 in_sample = iwl_read_prph(mvm->trans, DBGC_IN_SAMPLE);
-		u32 out_ctrl = iwl_read_prph(mvm->trans, DBGC_OUT_CTRL);
-
-		/* stop recording */
-		iwl_write_prph(mvm->trans, DBGC_IN_SAMPLE, 0);
-		udelay(100);
-		iwl_write_prph(mvm->trans, DBGC_OUT_CTRL, 0);
-		/* wait before we collect the data till the DBGC stop */
-		udelay(500);
-
-		iwl_mvm_fw_error_dump(mvm);
-
-		/* start recording again if the firmware is not crashed */
-		if (!test_bit(STATUS_FW_ERROR, &mvm->trans->status) &&
-		    mvm->fw->dbg_dest_tlv) {
-			iwl_write_prph(mvm->trans, DBGC_IN_SAMPLE, in_sample);
-			iwl_write_prph(mvm->trans, DBGC_OUT_CTRL, out_ctrl);
-		}
-	}
-
-	mutex_unlock(&mvm->mutex);
-
-	iwl_mvm_unref(mvm, IWL_MVM_REF_FW_DBG_COLLECT);
-}
-
 void iwl_mvm_nic_restart(struct iwl_mvm *mvm, bool fw_error)
 {
 	iwl_abort_notification_waits(&mvm->notif_wait);
@@ -1234,7 +1187,7 @@ void iwl_mvm_nic_restart(struct iwl_mvm *mvm, bool fw_error)
 	 * can't recover this since we're already half suspended.
 	 */
 	if (!mvm->fw_restart && fw_error) {
-		iwl_mvm_fw_dbg_collect_desc(mvm, &iwl_mvm_dump_desc_assert,
+		iwl_fw_dbg_collect_desc(&mvm->fwrt, &iwl_dump_desc_assert,
 					NULL);
 	} else if (test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status)) {
 		struct iwl_mvm_reprobe *reprobe;
@@ -1260,7 +1213,7 @@ void iwl_mvm_nic_restart(struct iwl_mvm *mvm, bool fw_error)
 		reprobe->dev = mvm->trans->dev;
 		INIT_WORK(&reprobe->work, iwl_mvm_reprobe_wk);
 		schedule_work(&reprobe->work);
-	} else if (mvm->cur_ucode == IWL_UCODE_REGULAR &&
+	} else if (mvm->fwrt.cur_fw_img == IWL_UCODE_REGULAR &&
 		   mvm->hw_registered) {
 		/* don't let the transport/FW power down */
 		iwl_mvm_ref(mvm, IWL_MVM_REF_UCODE_DOWN);
@@ -1439,7 +1392,7 @@ int iwl_mvm_enter_d0i3(struct iwl_op_mode *op_mode)
 
 	IWL_DEBUG_RPM(mvm, "MVM entering D0i3\n");
 
-	if (WARN_ON_ONCE(mvm->cur_ucode != IWL_UCODE_REGULAR))
+	if (WARN_ON_ONCE(mvm->fwrt.cur_fw_img != IWL_UCODE_REGULAR))
 		return -EINVAL;
 
 	set_bit(IWL_MVM_STATUS_IN_D0I3, &mvm->status);
@@ -1665,7 +1618,7 @@ int _iwl_mvm_exit_d0i3(struct iwl_mvm *mvm)
 
 	IWL_DEBUG_RPM(mvm, "MVM exiting D0i3\n");
 
-	if (WARN_ON_ONCE(mvm->cur_ucode != IWL_UCODE_REGULAR))
+	if (WARN_ON_ONCE(mvm->fwrt.cur_fw_img != IWL_UCODE_REGULAR))
 		return -EINVAL;
 
 	mutex_lock(&mvm->d0i3_suspend_mutex);
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/phy-ctxt.c b/drivers/net/wireless/intel/iwlwifi/mvm/phy-ctxt.c
index fb9eaf003ea5..7ee8e9077baf 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/phy-ctxt.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/phy-ctxt.c
@@ -251,7 +251,7 @@ int iwl_mvm_phy_ctxt_changed(struct iwl_mvm *mvm, struct iwl_mvm_phy_ctxt *ctxt,
 			     struct cfg80211_chan_def *chandef,
 			     u8 chains_static, u8 chains_dynamic)
 {
-	enum iwl_phy_ctxt_action action = FW_CTXT_ACTION_MODIFY;
+	enum iwl_ctxt_action action = FW_CTXT_ACTION_MODIFY;
 
 	lockdep_assert_held(&mvm->mutex);
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/power.c b/drivers/net/wireless/intel/iwlwifi/mvm/power.c
index e684811f8e8b..c11fe2621d51 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/power.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/power.c
@@ -7,7 +7,7 @@
  *
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2014 Intel Mobile Communications GmbH
- * Copyright(c) 2015 - 2016 Intel Deutschland GmbH
+ * Copyright(c) 2015 - 2017 Intel Deutschland GmbH
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -34,7 +34,7 @@
  *
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2014 Intel Mobile Communications GmbH
- * Copyright(c) 2015 - 2016 Intel Deutschland GmbH
+ * Copyright(c) 2015 - 2017 Intel Deutschland GmbH
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -75,7 +75,7 @@
 #include "iwl-debug.h"
 #include "mvm.h"
 #include "iwl-modparams.h"
-#include "fw-api-power.h"
+#include "fw/api/power.h"
 
 #define POWER_KEEP_ALIVE_PERIOD_SEC    25
 
@@ -186,7 +186,7 @@ static void iwl_mvm_power_configure_uapsd(struct iwl_mvm *mvm,
 		if (!mvmvif->queue_params[ac].uapsd)
 			continue;
 
-		if (mvm->cur_ucode != IWL_UCODE_WOWLAN)
+		if (mvm->fwrt.cur_fw_img != IWL_UCODE_WOWLAN)
 			cmd->flags |=
 				cpu_to_le16(POWER_FLAGS_ADVANCE_PM_ENA_MSK);
 
@@ -220,14 +220,15 @@ static void iwl_mvm_power_configure_uapsd(struct iwl_mvm *mvm,
 				    BIT(IEEE80211_AC_BK))) {
 		cmd->flags |= cpu_to_le16(POWER_FLAGS_SNOOZE_ENA_MSK);
 		cmd->snooze_interval = cpu_to_le16(IWL_MVM_PS_SNOOZE_INTERVAL);
-		cmd->snooze_window = (mvm->cur_ucode == IWL_UCODE_WOWLAN) ?
-			cpu_to_le16(IWL_MVM_WOWLAN_PS_SNOOZE_WINDOW) :
-			cpu_to_le16(IWL_MVM_PS_SNOOZE_WINDOW);
+		cmd->snooze_window =
+			(mvm->fwrt.cur_fw_img == IWL_UCODE_WOWLAN) ?
+				cpu_to_le16(IWL_MVM_WOWLAN_PS_SNOOZE_WINDOW) :
+				cpu_to_le16(IWL_MVM_PS_SNOOZE_WINDOW);
 	}
 
 	cmd->uapsd_max_sp = mvm->hw->uapsd_max_sp_len;
 
-	if (mvm->cur_ucode == IWL_UCODE_WOWLAN || cmd->flags &
+	if (mvm->fwrt.cur_fw_img == IWL_UCODE_WOWLAN || cmd->flags &
 	    cpu_to_le16(POWER_FLAGS_SNOOZE_ENA_MSK)) {
 		cmd->rx_data_timeout_uapsd =
 			cpu_to_le32(IWL_MVM_WOWLAN_PS_RX_DATA_TIMEOUT);
@@ -502,7 +503,7 @@ static int iwl_mvm_power_send_cmd(struct iwl_mvm *mvm,
 	struct iwl_mac_power_cmd cmd = {};
 
 	iwl_mvm_power_build_cmd(mvm, vif, &cmd,
-				mvm->cur_ucode != IWL_UCODE_WOWLAN);
+				mvm->fwrt.cur_fw_img != IWL_UCODE_WOWLAN);
 	iwl_mvm_power_log(mvm, &cmd);
 #ifdef CONFIG_IWLWIFI_DEBUGFS
 	memcpy(&iwl_mvm_vif_from_mac80211(vif)->mac_pwr_cmd, &cmd, sizeof(cmd));
@@ -525,8 +526,8 @@ int iwl_mvm_power_update_device(struct iwl_mvm *mvm)
 		cmd.flags |= cpu_to_le16(DEVICE_POWER_FLAGS_POWER_SAVE_ENA_MSK);
 
 #ifdef CONFIG_IWLWIFI_DEBUGFS
-	if ((mvm->cur_ucode == IWL_UCODE_WOWLAN) ? mvm->disable_power_off_d3 :
-	    mvm->disable_power_off)
+	if ((mvm->fwrt.cur_fw_img == IWL_UCODE_WOWLAN) ?
+			mvm->disable_power_off_d3 : mvm->disable_power_off)
 		cmd.flags &=
 			cpu_to_le16(~DEVICE_POWER_FLAGS_POWER_SAVE_ENA_MSK);
 #endif
@@ -933,7 +934,7 @@ static int iwl_mvm_power_set_ba(struct iwl_mvm *mvm,
 	if (!mvmvif->bf_data.bf_enabled)
 		return 0;
 
-	if (mvm->cur_ucode == IWL_UCODE_WOWLAN)
+	if (mvm->fwrt.cur_fw_img == IWL_UCODE_WOWLAN)
 		cmd.ba_escape_timer = cpu_to_le32(IWL_BA_ESCAPE_TIMER_D3);
 
 	mvmvif->bf_data.ba_enabled = !(!mvmvif->pm_enabled ||
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rs.c b/drivers/net/wireless/intel/iwlwifi/mvm/rs.c
index 65beca3a457a..ba7bd049d3d4 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rs.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rs.c
@@ -622,7 +622,9 @@ static int rs_tl_turn_on_agg_for_tid(struct iwl_mvm *mvm,
 
 	IWL_DEBUG_HT(mvm, "Starting Tx agg: STA: %pM tid: %d\n",
 		     sta->addr, tid);
-	ret = ieee80211_start_tx_ba_session(sta, tid, 5000);
+
+	/* start BA session until the peer sends del BA */
+	ret = ieee80211_start_tx_ba_session(sta, tid, 0);
 	if (ret == -EAGAIN) {
 		/*
 		 * driver and mac80211 is out of sync
@@ -636,15 +638,31 @@ static int rs_tl_turn_on_agg_for_tid(struct iwl_mvm *mvm,
 	return ret;
 }
 
-static void rs_tl_turn_on_agg(struct iwl_mvm *mvm, u8 tid,
-			      struct iwl_lq_sta *lq_data,
+static void rs_tl_turn_on_agg(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvmsta,
+			      u8 tid, struct iwl_lq_sta *lq_sta,
 			      struct ieee80211_sta *sta)
 {
-	if (tid < IWL_MAX_TID_COUNT)
-		rs_tl_turn_on_agg_for_tid(mvm, lq_data, tid, sta);
-	else
+	struct iwl_mvm_tid_data *tid_data;
+
+	/*
+	 * In AP mode, tid can be equal to IWL_MAX_TID_COUNT
+	 * when the frame is not QoS
+	 */
+	if (WARN_ON_ONCE(tid > IWL_MAX_TID_COUNT)) {
 		IWL_ERR(mvm, "tid exceeds max TID count: %d/%d\n",
 			tid, IWL_MAX_TID_COUNT);
+		return;
+	} else if (tid == IWL_MAX_TID_COUNT) {
+		return;
+	}
+
+	tid_data = &mvmsta->tid_data[tid];
+	if ((tid_data->state == IWL_AGG_OFF) &&
+	    (lq_sta->tx_agg_tid_en & BIT(tid)) &&
+	    (tid_data->tx_count_last >= IWL_MVM_RS_AGG_START_THRESHOLD)) {
+		IWL_DEBUG_RATE(mvm, "try to aggregate tid %d\n", tid);
+		rs_tl_turn_on_agg_for_tid(mvm, lq_sta, tid, sta);
+	}
 }
 
 static inline int get_num_of_ant_from_rate(u32 rate_n_flags)
@@ -753,8 +771,38 @@ static int rs_collect_tpc_data(struct iwl_mvm *mvm,
 				    window);
 }
 
+static void rs_update_tid_tpt_stats(struct iwl_mvm *mvm,
+				    struct iwl_mvm_sta *mvmsta,
+				    u8 tid, int successes)
+{
+	struct iwl_mvm_tid_data *tid_data;
+
+	if (tid >= IWL_MAX_TID_COUNT)
+		return;
+
+	tid_data = &mvmsta->tid_data[tid];
+
+	/*
+	 * Measure if there're enough successful transmits per second.
+	 * These statistics are used only to decide if we can start a
+	 * BA session, so it should be updated only when A-MPDU is
+	 * off.
+	 */
+	if (tid_data->state != IWL_AGG_OFF)
+		return;
+
+	if (time_is_before_jiffies(tid_data->tpt_meas_start + HZ) ||
+	    (tid_data->tx_count >= IWL_MVM_RS_AGG_START_THRESHOLD)) {
+		tid_data->tx_count_last = tid_data->tx_count;
+		tid_data->tx_count = 0;
+		tid_data->tpt_meas_start = jiffies;
+	} else {
+		tid_data->tx_count += successes;
+	}
+}
+
 static int rs_collect_tlc_data(struct iwl_mvm *mvm,
-			       struct iwl_lq_sta *lq_sta,
+			       struct iwl_mvm_sta *mvmsta, u8 tid,
 			       struct iwl_scale_tbl_info *tbl,
 			       int scale_index, int attempts, int successes)
 {
@@ -764,12 +812,14 @@ static int rs_collect_tlc_data(struct iwl_mvm *mvm,
 		return -EINVAL;
 
 	if (tbl->column != RS_COLUMN_INVALID) {
-		struct lq_sta_pers *pers = &lq_sta->pers;
+		struct lq_sta_pers *pers = &mvmsta->lq_sta.pers;
 
 		pers->tx_stats[tbl->column][scale_index].total += attempts;
 		pers->tx_stats[tbl->column][scale_index].success += successes;
 	}
 
+	rs_update_tid_tpt_stats(mvm, mvmsta, tid, successes);
+
 	/* Select window for current tx bit rate */
 	window = &(tbl->win[scale_index]);
 	return _rs_collect_tx_data(mvm, tbl, scale_index, attempts, successes,
@@ -1211,12 +1261,7 @@ void iwl_mvm_rs_tx_status(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 	if (time_after(jiffies,
 		       (unsigned long)(lq_sta->last_tx +
 				       (IWL_MVM_RS_IDLE_TIMEOUT * HZ)))) {
-		int t;
-
 		IWL_DEBUG_RATE(mvm, "Tx idle for too long. reinit rs\n");
-		for (t = 0; t < IWL_MAX_TID_COUNT; t++)
-			ieee80211_stop_tx_ba_session(sta, t);
-
 		iwl_mvm_rs_rate_init(mvm, sta, info->band, false);
 		return;
 	}
@@ -1291,7 +1336,7 @@ void iwl_mvm_rs_tx_status(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 	 * first index into rate scale table.
 	 */
 	if (info->flags & IEEE80211_TX_STAT_AMPDU) {
-		rs_collect_tpc_data(mvm, lq_sta, curr_tbl, lq_rate.index,
+		rs_collect_tpc_data(mvm, lq_sta, curr_tbl, tx_resp_rate.index,
 				    info->status.ampdu_len,
 				    info->status.ampdu_ack_len,
 				    reduced_txp);
@@ -1312,7 +1357,7 @@ void iwl_mvm_rs_tx_status(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 		if (info->status.ampdu_ack_len == 0)
 			info->status.ampdu_len = 1;
 
-		rs_collect_tlc_data(mvm, lq_sta, curr_tbl, lq_rate.index,
+		rs_collect_tlc_data(mvm, mvmsta, tid, curr_tbl, tx_resp_rate.index,
 				    info->status.ampdu_len,
 				    info->status.ampdu_ack_len);
 
@@ -1348,11 +1393,11 @@ void iwl_mvm_rs_tx_status(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 				continue;
 
 			rs_collect_tpc_data(mvm, lq_sta, tmp_tbl,
-					    lq_rate.index, 1,
+					    tx_resp_rate.index, 1,
 					    i < retries ? 0 : legacy_success,
 					    reduced_txp);
-			rs_collect_tlc_data(mvm, lq_sta, tmp_tbl,
-					    lq_rate.index, 1,
+			rs_collect_tlc_data(mvm, mvmsta, tid, tmp_tbl,
+					    tx_resp_rate.index, 1,
 					    i < retries ? 0 : legacy_success);
 		}
 
@@ -1673,14 +1718,14 @@ static void rs_set_amsdu_len(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 			     struct iwl_scale_tbl_info *tbl,
 			     enum rs_action scale_action)
 {
-	struct iwl_mvm_sta *sta_priv = iwl_mvm_sta_from_mac80211(sta);
+	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
 
 	if ((!is_vht(&tbl->rate) && !is_ht(&tbl->rate)) ||
 	    tbl->rate.index < IWL_RATE_MCS_5_INDEX ||
 	    scale_action == RS_ACTION_DOWNSCALE)
-		sta_priv->tlc_amsdu = false;
+		mvmsta->tlc_amsdu = false;
 	else
-		sta_priv->tlc_amsdu = true;
+		mvmsta->tlc_amsdu = true;
 }
 
 /*
@@ -2228,11 +2273,10 @@ static void rs_rate_scale_perform(struct iwl_mvm *mvm,
 	u16 high_low;
 	s32 sr;
 	u8 prev_agg = lq_sta->is_agg;
-	struct iwl_mvm_sta *sta_priv = iwl_mvm_sta_from_mac80211(sta);
-	struct iwl_mvm_tid_data *tid_data;
+	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
 	struct rs_rate *rate;
 
-	lq_sta->is_agg = !!sta_priv->agg_tids;
+	lq_sta->is_agg = !!mvmsta->agg_tids;
 
 	/*
 	 * Select rate-scale / modulation-mode table to work with in
@@ -2480,44 +2524,12 @@ lq_update:
 		}
 	}
 
+	if (!ndp)
+		rs_tl_turn_on_agg(mvm, mvmsta, tid, lq_sta, sta);
+
 	if (done_search && lq_sta->rs_state == RS_STATE_SEARCH_CYCLE_ENDED) {
-		/* If the "active" (non-search) mode was legacy,
-		 * and we've tried switching antennas,
-		 * but we haven't been able to try HT modes (not available),
-		 * stay with best antenna legacy modulation for a while
-		 * before next round of mode comparisons. */
 		tbl1 = &(lq_sta->lq_info[lq_sta->active_tbl]);
-		if (is_legacy(&tbl1->rate)) {
-			IWL_DEBUG_RATE(mvm, "LQ: STAY in legacy table\n");
-
-			if (tid != IWL_MAX_TID_COUNT) {
-				tid_data = &sta_priv->tid_data[tid];
-				if (tid_data->state != IWL_AGG_OFF) {
-					IWL_DEBUG_RATE(mvm,
-						       "Stop aggregation on tid %d\n",
-						       tid);
-					ieee80211_stop_tx_ba_session(sta, tid);
-				}
-			}
-			rs_set_stay_in_table(mvm, 1, lq_sta);
-		} else {
-		/* If we're in an HT mode, and all 3 mode switch actions
-		 * have been tried and compared, stay in this best modulation
-		 * mode for a while before next round of mode comparisons. */
-			if ((lq_sta->last_tpt > IWL_AGG_TPT_THREHOLD) &&
-			    (lq_sta->tx_agg_tid_en & (1 << tid)) &&
-			    (tid != IWL_MAX_TID_COUNT)) {
-				tid_data = &sta_priv->tid_data[tid];
-				if (tid_data->state == IWL_AGG_OFF && !ndp) {
-					IWL_DEBUG_RATE(mvm,
-						       "try to aggregate tid %d\n",
-						       tid);
-					rs_tl_turn_on_agg(mvm, tid,
-							  lq_sta, sta);
-				}
-			}
-			rs_set_stay_in_table(mvm, 0, lq_sta);
-		}
+		rs_set_stay_in_table(mvm, is_legacy(&tbl1->rate), lq_sta);
 	}
 }
 
@@ -2900,10 +2912,10 @@ static void rs_get_rate(void *mvm_r, struct ieee80211_sta *sta, void *mvm_sta,
 static void *rs_alloc_sta(void *mvm_rate, struct ieee80211_sta *sta,
 			  gfp_t gfp)
 {
-	struct iwl_mvm_sta *sta_priv = iwl_mvm_sta_from_mac80211(sta);
+	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
 	struct iwl_op_mode *op_mode = (struct iwl_op_mode *)mvm_rate;
 	struct iwl_mvm *mvm  = IWL_OP_MODE_GET_MVM(op_mode);
-	struct iwl_lq_sta *lq_sta = &sta_priv->lq_sta;
+	struct iwl_lq_sta *lq_sta = &mvmsta->lq_sta;
 
 	IWL_DEBUG_RATE(mvm, "create station rate scale window\n");
 
@@ -2917,7 +2929,7 @@ static void *rs_alloc_sta(void *mvm_rate, struct ieee80211_sta *sta,
 	memset(lq_sta->pers.chain_signal, 0, sizeof(lq_sta->pers.chain_signal));
 	lq_sta->pers.last_rssi = S8_MIN;
 
-	return &sta_priv->lq_sta;
+	return &mvmsta->lq_sta;
 }
 
 static int rs_vht_highest_rx_mcs_index(struct ieee80211_sta_vht_cap *vht_cap,
@@ -3109,8 +3121,8 @@ void iwl_mvm_rs_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 	struct ieee80211_hw *hw = mvm->hw;
 	struct ieee80211_sta_ht_cap *ht_cap = &sta->ht_cap;
 	struct ieee80211_sta_vht_cap *vht_cap = &sta->vht_cap;
-	struct iwl_mvm_sta *sta_priv = iwl_mvm_sta_from_mac80211(sta);
-	struct iwl_lq_sta *lq_sta = &sta_priv->lq_sta;
+	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
+	struct iwl_lq_sta *lq_sta = &mvmsta->lq_sta;
 	struct ieee80211_supported_band *sband;
 	unsigned long supp; /* must be unsigned long for for_each_set_bit */
 
@@ -3119,8 +3131,8 @@ void iwl_mvm_rs_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 
 	sband = hw->wiphy->bands[band];
 
-	lq_sta->lq.sta_id = sta_priv->sta_id;
-	sta_priv->tlc_amsdu = false;
+	lq_sta->lq.sta_id = mvmsta->sta_id;
+	mvmsta->tlc_amsdu = false;
 
 	for (j = 0; j < LQ_SIZE; j++)
 		rs_rate_scale_clear_tbl_windows(mvm, &lq_sta->lq_info[j]);
@@ -3130,7 +3142,7 @@ void iwl_mvm_rs_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 
 	IWL_DEBUG_RATE(mvm,
 		       "LQ: *** rate scale station global init for station %d ***\n",
-		       sta_priv->sta_id);
+		       mvmsta->sta_id);
 	/* TODO: what is a good starting rate for STA? About middle? Maybe not
 	 * the lowest or the highest rate.. Could consider using RSSI from
 	 * previous packets? Need to have IEEE 802.1X auth succeed immediately
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rx.c b/drivers/net/wireless/intel/iwlwifi/mvm/rx.c
index 622d543abb70..184c749766f2 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rx.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rx.c
@@ -67,7 +67,6 @@
 #include "iwl-trans.h"
 #include "mvm.h"
 #include "fw-api.h"
-#include "fw-dbg.h"
 
 /*
  * iwl_mvm_rx_rx_phy_cmd - REPLY_RX_PHY_CMD handler
@@ -397,10 +396,12 @@ void iwl_mvm_rx_rx_mpdu(struct iwl_mvm *mvm, struct napi_struct *napi,
 			rssi = le32_to_cpu(rssi_trig->rssi);
 
 			trig_check =
-				iwl_fw_dbg_trigger_check_stop(mvm, mvmsta->vif,
+				iwl_fw_dbg_trigger_check_stop(&mvm->fwrt,
+							      ieee80211_vif_to_wdev(mvmsta->vif),
 							      trig);
 			if (trig_check && rx_status->signal < rssi)
-				iwl_mvm_fw_dbg_collect_trig(mvm, trig, NULL);
+				iwl_fw_dbg_collect_trig(&mvm->fwrt, trig,
+							NULL);
 		}
 
 		if (ieee80211_is_data(hdr->frame_control))
@@ -624,7 +625,7 @@ iwl_mvm_rx_stats_check_trigger(struct iwl_mvm *mvm, struct iwl_rx_packet *pkt)
 	trig = iwl_fw_dbg_get_trigger(mvm->fw, FW_DBG_TRIGGER_STATS);
 	trig_stats = (void *)trig->data;
 
-	if (!iwl_fw_dbg_trigger_check_stop(mvm, NULL, trig))
+	if (!iwl_fw_dbg_trigger_check_stop(&mvm->fwrt, NULL, trig))
 		return;
 
 	trig_offset = le32_to_cpu(trig_stats->stop_offset);
@@ -636,7 +637,7 @@ iwl_mvm_rx_stats_check_trigger(struct iwl_mvm *mvm, struct iwl_rx_packet *pkt)
 	if (le32_to_cpup((__le32 *) (pkt->data + trig_offset)) < trig_thold)
 		return;
 
-	iwl_mvm_fw_dbg_collect_trig(mvm, trig, NULL);
+	iwl_fw_dbg_collect_trig(&mvm->fwrt, trig, NULL);
 }
 
 void iwl_mvm_handle_rx_statistics(struct iwl_mvm *mvm,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c b/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c
index f3e608196369..67ffd9774712 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c
@@ -63,7 +63,6 @@
 #include "iwl-trans.h"
 #include "mvm.h"
 #include "fw-api.h"
-#include "fw-dbg.h"
 
 static inline int iwl_mvm_check_pn(struct iwl_mvm *mvm, struct sk_buff *skb,
 				   int queue, struct ieee80211_sta *sta)
@@ -636,9 +635,9 @@ static bool iwl_mvm_reorder(struct iwl_mvm *mvm,
 
 	baid_data = rcu_dereference(mvm->baid_map[baid]);
 	if (!baid_data) {
-		WARN(!(reorder & IWL_RX_MPDU_REORDER_BA_OLD_SN),
-		     "Received baid %d, but no data exists for this BAID\n",
-		     baid);
+		IWL_DEBUG_RX(mvm,
+			     "Got valid BAID but no baid allocated, bypass the re-ordering buffer. Baid %d reorder 0x%x\n",
+			      baid, reorder);
 		return false;
 	}
 
@@ -759,7 +758,9 @@ static void iwl_mvm_agg_rx_received(struct iwl_mvm *mvm,
 
 	data = rcu_dereference(mvm->baid_map[baid]);
 	if (!data) {
-		WARN_ON(!(reorder_data & IWL_RX_MPDU_REORDER_BA_OLD_SN));
+		IWL_DEBUG_RX(mvm,
+			     "Got valid BAID but no baid allocated, bypass the re-ordering buffer. Baid %d reorder 0x%x\n",
+			      baid, reorder_data);
 		goto out;
 	}
 
@@ -852,7 +853,7 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 
 	rcu_read_lock();
 
-	if (le16_to_cpu(desc->status) & IWL_RX_MPDU_STATUS_SRC_STA_FOUND) {
+	if (desc->status & cpu_to_le16(IWL_RX_MPDU_STATUS_SRC_STA_FOUND)) {
 		u8 id = desc->sta_id_flags & IWL_RX_MPDU_SIF_STA_ID_MASK;
 
 		if (!WARN_ON_ONCE(id >= ARRAY_SIZE(mvm->fw_id_to_mac_id))) {
@@ -906,10 +907,12 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 			rssi = le32_to_cpu(rssi_trig->rssi);
 
 			trig_check =
-				iwl_fw_dbg_trigger_check_stop(mvm, mvmsta->vif,
+				iwl_fw_dbg_trigger_check_stop(&mvm->fwrt,
+							      ieee80211_vif_to_wdev(mvmsta->vif),
 							      trig);
 			if (trig_check && rx_status->signal < rssi)
-				iwl_mvm_fw_dbg_collect_trig(mvm, trig, NULL);
+				iwl_fw_dbg_collect_trig(&mvm->fwrt, trig,
+							NULL);
 		}
 
 		if (ieee80211_is_data(hdr->frame_control))
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
index 35e813bdfbe5..50983615dce6 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
@@ -69,7 +69,7 @@
 #include <net/mac80211.h>
 
 #include "mvm.h"
-#include "fw-api-scan.h"
+#include "fw/api/scan.h"
 #include "iwl-io.h"
 
 #define IWL_DENSE_EBS_SCAN_RATIO 5
@@ -743,7 +743,7 @@ static inline bool iwl_mvm_scan_use_ebs(struct iwl_mvm *mvm,
 	 *	4. it's not a p2p find operation.
 	 */
 	return ((capa->flags & IWL_UCODE_TLV_FLAGS_EBS_SUPPORT) &&
-		mvm->last_ebs_successful &&
+		mvm->last_ebs_successful && IWL_MVM_ENABLE_EBS &&
 		vif->type != NL80211_IFTYPE_P2P_DEVICE);
 }
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/sta.c b/drivers/net/wireless/intel/iwlwifi/mvm/sta.c
index ab66b4394dfc..411a2055dc45 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/sta.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/sta.c
@@ -121,7 +121,8 @@ int iwl_mvm_sta_send_to_fw(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 		.mac_id_n_color = cpu_to_le32(mvm_sta->mac_id_n_color),
 		.add_modify = update ? 1 : 0,
 		.station_flags_msk = cpu_to_le32(STA_FLG_FAT_EN_MSK |
-						 STA_FLG_MIMO_EN_MSK),
+						 STA_FLG_MIMO_EN_MSK |
+						 STA_FLG_RTS_MIMO_PROT),
 		.tid_disable_tx = cpu_to_le16(mvm_sta->tid_disable_agg),
 	};
 	int ret;
@@ -290,66 +291,12 @@ static void iwl_mvm_rx_agg_session_expired(unsigned long data)
 		goto unlock;
 
 	mvm_sta = iwl_mvm_sta_from_mac80211(sta);
-	ieee80211_stop_rx_ba_session_offl(mvm_sta->vif,
-					  sta->addr, ba_data->tid);
+	ieee80211_rx_ba_timer_expired(mvm_sta->vif,
+				      sta->addr, ba_data->tid);
 unlock:
 	rcu_read_unlock();
 }
 
-static int iwl_mvm_tdls_sta_init(struct iwl_mvm *mvm,
-				 struct ieee80211_sta *sta)
-{
-	unsigned long used_hw_queues;
-	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
-	unsigned int wdg_timeout =
-		iwl_mvm_get_wd_timeout(mvm, NULL, true, false);
-	u32 ac;
-
-	lockdep_assert_held(&mvm->mutex);
-
-	used_hw_queues = iwl_mvm_get_used_hw_queues(mvm, NULL);
-
-	/* Find available queues, and allocate them to the ACs */
-	for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) {
-		u8 queue = find_first_zero_bit(&used_hw_queues,
-					       mvm->first_agg_queue);
-
-		if (queue >= mvm->first_agg_queue) {
-			IWL_ERR(mvm, "Failed to allocate STA queue\n");
-			return -EBUSY;
-		}
-
-		__set_bit(queue, &used_hw_queues);
-		mvmsta->hw_queue[ac] = queue;
-	}
-
-	/* Found a place for all queues - enable them */
-	for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) {
-		iwl_mvm_enable_ac_txq(mvm, mvmsta->hw_queue[ac],
-				      mvmsta->hw_queue[ac],
-				      iwl_mvm_ac_to_tx_fifo[ac], 0,
-				      wdg_timeout);
-		mvmsta->tfd_queue_msk |= BIT(mvmsta->hw_queue[ac]);
-	}
-
-	return 0;
-}
-
-static void iwl_mvm_tdls_sta_deinit(struct iwl_mvm *mvm,
-				    struct ieee80211_sta *sta)
-{
-	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
-	unsigned long sta_msk;
-	int i;
-
-	lockdep_assert_held(&mvm->mutex);
-
-	/* disable the TDLS STA-specific queues */
-	sta_msk = mvmsta->tfd_queue_msk;
-	for_each_set_bit(i, &sta_msk, sizeof(sta_msk) * BITS_PER_BYTE)
-		iwl_mvm_disable_txq(mvm, i, i, IWL_MAX_TID_COUNT, 0);
-}
-
 /* Disable aggregations for a bitmap of TIDs for a given station */
 static int iwl_mvm_invalidate_sta_queue(struct iwl_mvm *mvm, int queue,
 					unsigned long disable_agg_tids,
@@ -757,7 +704,7 @@ static int iwl_mvm_sta_alloc_queue(struct iwl_mvm *mvm,
 {
 	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
 	struct iwl_trans_txq_scd_cfg cfg = {
-		.fifo = iwl_mvm_ac_to_tx_fifo[ac],
+		.fifo = iwl_mvm_mac_ac_to_tx_fifo(mvm, ac),
 		.sta_id = mvmsta->sta_id,
 		.tid = tid,
 		.frame_limit = IWL_FRAME_LIMIT,
@@ -1315,7 +1262,7 @@ static void iwl_mvm_realloc_queues_after_restart(struct iwl_mvm *mvm,
 			u16 seq = IEEE80211_SEQ_TO_SN(tid_data->seq_number);
 
 			cfg.tid = i;
-			cfg.fifo = iwl_mvm_ac_to_tx_fifo[ac];
+			cfg.fifo = iwl_mvm_mac_ac_to_tx_fifo(mvm, ac);
 			cfg.aggregate = (txq_id >= IWL_MVM_DQA_MIN_DATA_QUEUE ||
 					 txq_id ==
 					 IWL_MVM_DQA_BSS_CLIENT_QUEUE);
@@ -1329,8 +1276,50 @@ static void iwl_mvm_realloc_queues_after_restart(struct iwl_mvm *mvm,
 			mvm->queue_info[txq_id].status = IWL_MVM_QUEUE_READY;
 		}
 	}
+}
 
-	atomic_set(&mvm->pending_frames[mvm_sta->sta_id], 0);
+static int iwl_mvm_add_int_sta_common(struct iwl_mvm *mvm,
+				      struct iwl_mvm_int_sta *sta,
+				      const u8 *addr,
+				      u16 mac_id, u16 color)
+{
+	struct iwl_mvm_add_sta_cmd cmd;
+	int ret;
+	u32 status;
+
+	lockdep_assert_held(&mvm->mutex);
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.sta_id = sta->sta_id;
+	cmd.mac_id_n_color = cpu_to_le32(FW_CMD_ID_AND_COLOR(mac_id,
+							     color));
+	if (fw_has_api(&mvm->fw->ucode_capa, IWL_UCODE_TLV_API_STA_TYPE))
+		cmd.station_type = sta->type;
+
+	if (!iwl_mvm_has_new_tx_api(mvm))
+		cmd.tfd_queue_msk = cpu_to_le32(sta->tfd_queue_msk);
+	cmd.tid_disable_tx = cpu_to_le16(0xffff);
+
+	if (addr)
+		memcpy(cmd.addr, addr, ETH_ALEN);
+
+	ret = iwl_mvm_send_cmd_pdu_status(mvm, ADD_STA,
+					  iwl_mvm_add_sta_cmd_size(mvm),
+					  &cmd, &status);
+	if (ret)
+		return ret;
+
+	switch (status & IWL_ADD_STA_STATUS_MASK) {
+	case ADD_STA_SUCCESS:
+		IWL_DEBUG_INFO(mvm, "Internal station added.\n");
+		return 0;
+	default:
+		ret = -EIO;
+		IWL_ERR(mvm, "Add internal station failed, status=0x%x\n",
+			status);
+		break;
+	}
+	return ret;
 }
 
 int iwl_mvm_add_sta(struct iwl_mvm *mvm,
@@ -1341,6 +1330,8 @@ int iwl_mvm_add_sta(struct iwl_mvm *mvm,
 	struct iwl_mvm_sta *mvm_sta = iwl_mvm_sta_from_mac80211(sta);
 	struct iwl_mvm_rxq_dup_data *dup_data;
 	int i, ret, sta_id;
+	bool sta_update = false;
+	unsigned int sta_flags = 0;
 
 	lockdep_assert_held(&mvm->mutex);
 
@@ -1355,10 +1346,25 @@ int iwl_mvm_add_sta(struct iwl_mvm *mvm,
 
 	spin_lock_init(&mvm_sta->lock);
 
-	/* In DQA mode, if this is a HW restart, re-alloc existing queues */
-	if (iwl_mvm_is_dqa_supported(mvm) &&
-	    test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status)) {
+	/* if this is a HW restart re-alloc existing queues */
+	if (test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status)) {
+		struct iwl_mvm_int_sta tmp_sta = {
+			.sta_id = sta_id,
+			.type = mvm_sta->sta_type,
+		};
+
+		/*
+		 * First add an empty station since allocating
+		 * a queue requires a valid station
+		 */
+		ret = iwl_mvm_add_int_sta_common(mvm, &tmp_sta, sta->addr,
+						 mvmvif->id, mvmvif->color);
+		if (ret)
+			goto err;
+
 		iwl_mvm_realloc_queues_after_restart(mvm, mvm_sta);
+		sta_update = true;
+		sta_flags = iwl_mvm_has_new_tx_api(mvm) ? 0 : STA_MODIFY_QUEUES;
 		goto update_fw;
 	}
 
@@ -1375,33 +1381,15 @@ int iwl_mvm_add_sta(struct iwl_mvm *mvm,
 	mvm_sta->sta_type = sta->tdls ? IWL_STA_TDLS_LINK : IWL_STA_LINK;
 
 	/* HW restart, don't assume the memory has been zeroed */
-	atomic_set(&mvm->pending_frames[sta_id], 0);
 	mvm_sta->tid_disable_agg = 0xffff; /* No aggs at first */
 	mvm_sta->tfd_queue_msk = 0;
 
-	/*
-	 * Allocate new queues for a TDLS station, unless we're in DQA mode,
-	 * and then they'll be allocated dynamically
-	 */
-	if (!iwl_mvm_is_dqa_supported(mvm) && sta->tdls) {
-		ret = iwl_mvm_tdls_sta_init(mvm, sta);
-		if (ret)
-			return ret;
-	} else if (!iwl_mvm_is_dqa_supported(mvm)) {
-		for (i = 0; i < IEEE80211_NUM_ACS; i++)
-			if (vif->hw_queue[i] != IEEE80211_INVAL_HW_QUEUE)
-				mvm_sta->tfd_queue_msk |= BIT(vif->hw_queue[i]);
-	}
-
 	/* for HW restart - reset everything but the sequence number */
 	for (i = 0; i <= IWL_MAX_TID_COUNT; i++) {
 		u16 seq = mvm_sta->tid_data[i].seq_number;
 		memset(&mvm_sta->tid_data[i], 0, sizeof(mvm_sta->tid_data[i]));
 		mvm_sta->tid_data[i].seq_number = seq;
 
-		if (!iwl_mvm_is_dqa_supported(mvm))
-			continue;
-
 		/*
 		 * Mark all queues for this STA as unallocated and defer TX
 		 * frames until the queue is allocated
@@ -1435,7 +1423,7 @@ int iwl_mvm_add_sta(struct iwl_mvm *mvm,
 		mvm_sta->dup_data = dup_data;
 	}
 
-	if (iwl_mvm_is_dqa_supported(mvm) && !iwl_mvm_has_new_tx_api(mvm)) {
+	if (!iwl_mvm_has_new_tx_api(mvm)) {
 		ret = iwl_mvm_reserve_sta_stream(mvm, sta,
 						 ieee80211_vif_type_p2p(vif));
 		if (ret)
@@ -1443,7 +1431,7 @@ int iwl_mvm_add_sta(struct iwl_mvm *mvm,
 	}
 
 update_fw:
-	ret = iwl_mvm_sta_send_to_fw(mvm, sta, false, 0);
+	ret = iwl_mvm_sta_send_to_fw(mvm, sta, sta_update, sta_flags);
 	if (ret)
 		goto err;
 
@@ -1461,8 +1449,6 @@ update_fw:
 	return 0;
 
 err:
-	if (!iwl_mvm_is_dqa_supported(mvm) && sta->tdls)
-		iwl_mvm_tdls_sta_deinit(mvm, sta);
 	return ret;
 }
 
@@ -1535,79 +1521,6 @@ static int iwl_mvm_rm_sta_common(struct iwl_mvm *mvm, u8 sta_id)
 	return 0;
 }
 
-void iwl_mvm_sta_drained_wk(struct work_struct *wk)
-{
-	struct iwl_mvm *mvm = container_of(wk, struct iwl_mvm, sta_drained_wk);
-	u8 sta_id;
-
-	/*
-	 * The mutex is needed because of the SYNC cmd, but not only: if the
-	 * work would run concurrently with iwl_mvm_rm_sta, it would run before
-	 * iwl_mvm_rm_sta sets the station as busy, and exit. Then
-	 * iwl_mvm_rm_sta would set the station as busy, and nobody will clean
-	 * that later.
-	 */
-	mutex_lock(&mvm->mutex);
-
-	for_each_set_bit(sta_id, mvm->sta_drained, IWL_MVM_STATION_COUNT) {
-		int ret;
-		struct ieee80211_sta *sta =
-			rcu_dereference_protected(mvm->fw_id_to_mac_id[sta_id],
-						  lockdep_is_held(&mvm->mutex));
-
-		/*
-		 * This station is in use or RCU-removed; the latter happens in
-		 * managed mode, where mac80211 removes the station before we
-		 * can remove it from firmware (we can only do that after the
-		 * MAC is marked unassociated), and possibly while the deauth
-		 * frame to disconnect from the AP is still queued. Then, the
-		 * station pointer is -ENOENT when the last skb is reclaimed.
-		 */
-		if (!IS_ERR(sta) || PTR_ERR(sta) == -ENOENT)
-			continue;
-
-		if (PTR_ERR(sta) == -EINVAL) {
-			IWL_ERR(mvm, "Drained sta %d, but it is internal?\n",
-				sta_id);
-			continue;
-		}
-
-		if (!sta) {
-			IWL_ERR(mvm, "Drained sta %d, but it was NULL?\n",
-				sta_id);
-			continue;
-		}
-
-		WARN_ON(PTR_ERR(sta) != -EBUSY);
-		/* This station was removed and we waited until it got drained,
-		 * we can now proceed and remove it.
-		 */
-		ret = iwl_mvm_rm_sta_common(mvm, sta_id);
-		if (ret) {
-			IWL_ERR(mvm,
-				"Couldn't remove sta %d after it was drained\n",
-				sta_id);
-			continue;
-		}
-		RCU_INIT_POINTER(mvm->fw_id_to_mac_id[sta_id], NULL);
-		clear_bit(sta_id, mvm->sta_drained);
-
-		if (mvm->tfd_drained[sta_id]) {
-			unsigned long i, msk = mvm->tfd_drained[sta_id];
-
-			for_each_set_bit(i, &msk, sizeof(msk) * BITS_PER_BYTE)
-				iwl_mvm_disable_txq(mvm, i, i,
-						    IWL_MAX_TID_COUNT, 0);
-
-			mvm->tfd_drained[sta_id] = 0;
-			IWL_DEBUG_TDLS(mvm, "Drained sta %d, with queues %ld\n",
-				       sta_id, msk);
-		}
-	}
-
-	mutex_unlock(&mvm->mutex);
-}
-
 static void iwl_mvm_disable_sta_queues(struct iwl_mvm *mvm,
 				       struct ieee80211_vif *vif,
 				       struct iwl_mvm_sta *mvm_sta)
@@ -1631,10 +1544,11 @@ static void iwl_mvm_disable_sta_queues(struct iwl_mvm *mvm,
 int iwl_mvm_wait_sta_queues_empty(struct iwl_mvm *mvm,
 				  struct iwl_mvm_sta *mvm_sta)
 {
-	int i, ret;
+	int i;
 
 	for (i = 0; i < ARRAY_SIZE(mvm_sta->tid_data); i++) {
 		u16 txq_id;
+		int ret;
 
 		spin_lock_bh(&mvm_sta->lock);
 		txq_id = mvm_sta->tid_data[i].txq_id;
@@ -1645,10 +1559,10 @@ int iwl_mvm_wait_sta_queues_empty(struct iwl_mvm *mvm,
 
 		ret = iwl_trans_wait_txq_empty(mvm->trans, txq_id);
 		if (ret)
-			break;
+			return ret;
 	}
 
-	return ret;
+	return 0;
 }
 
 int iwl_mvm_rm_sta(struct iwl_mvm *mvm,
@@ -1665,79 +1579,65 @@ int iwl_mvm_rm_sta(struct iwl_mvm *mvm,
 	if (iwl_mvm_has_new_rx_api(mvm))
 		kfree(mvm_sta->dup_data);
 
-	if ((vif->type == NL80211_IFTYPE_STATION &&
-	     mvmvif->ap_sta_id == sta_id) ||
-	    iwl_mvm_is_dqa_supported(mvm)){
-		ret = iwl_mvm_drain_sta(mvm, mvm_sta, true);
-		if (ret)
-			return ret;
-		/* flush its queues here since we are freeing mvm_sta */
-		ret = iwl_mvm_flush_sta(mvm, mvm_sta, false, 0);
-		if (ret)
-			return ret;
-		if (iwl_mvm_has_new_tx_api(mvm)) {
-			ret = iwl_mvm_wait_sta_queues_empty(mvm, mvm_sta);
-		} else {
-			u32 q_mask = mvm_sta->tfd_queue_msk;
+	ret = iwl_mvm_drain_sta(mvm, mvm_sta, true);
+	if (ret)
+		return ret;
 
-			ret = iwl_trans_wait_tx_queues_empty(mvm->trans,
-							     q_mask);
-		}
-		if (ret)
-			return ret;
-		ret = iwl_mvm_drain_sta(mvm, mvm_sta, false);
-
-		/* If DQA is supported - the queues can be disabled now */
-		if (iwl_mvm_is_dqa_supported(mvm)) {
-			iwl_mvm_disable_sta_queues(mvm, vif, mvm_sta);
-			/*
-			 * If pending_frames is set at this point - it must be
-			 * driver internal logic error, since queues are empty
-			 * and removed successuly.
-			 * warn on it but set it to 0 anyway to avoid station
-			 * not being removed later in the function
-			 */
-			WARN_ON(atomic_xchg(&mvm->pending_frames[sta_id], 0));
-		}
+	/* flush its queues here since we are freeing mvm_sta */
+	ret = iwl_mvm_flush_sta(mvm, mvm_sta, false, 0);
+	if (ret)
+		return ret;
+	if (iwl_mvm_has_new_tx_api(mvm)) {
+		ret = iwl_mvm_wait_sta_queues_empty(mvm, mvm_sta);
+	} else {
+		u32 q_mask = mvm_sta->tfd_queue_msk;
 
-		/* If there is a TXQ still marked as reserved - free it */
-		if (iwl_mvm_is_dqa_supported(mvm) &&
-		    mvm_sta->reserved_queue != IEEE80211_INVAL_HW_QUEUE) {
-			u8 reserved_txq = mvm_sta->reserved_queue;
-			enum iwl_mvm_queue_status *status;
-
-			/*
-			 * If no traffic has gone through the reserved TXQ - it
-			 * is still marked as IWL_MVM_QUEUE_RESERVED, and
-			 * should be manually marked as free again
-			 */
-			spin_lock_bh(&mvm->queue_info_lock);
-			status = &mvm->queue_info[reserved_txq].status;
-			if (WARN((*status != IWL_MVM_QUEUE_RESERVED) &&
-				 (*status != IWL_MVM_QUEUE_FREE),
-				 "sta_id %d reserved txq %d status %d",
-				 sta_id, reserved_txq, *status)) {
-				spin_unlock_bh(&mvm->queue_info_lock);
-				return -EINVAL;
-			}
+		ret = iwl_trans_wait_tx_queues_empty(mvm->trans,
+						     q_mask);
+	}
+	if (ret)
+		return ret;
 
-			*status = IWL_MVM_QUEUE_FREE;
+	ret = iwl_mvm_drain_sta(mvm, mvm_sta, false);
+
+	iwl_mvm_disable_sta_queues(mvm, vif, mvm_sta);
+
+	/* If there is a TXQ still marked as reserved - free it */
+	if (mvm_sta->reserved_queue != IEEE80211_INVAL_HW_QUEUE) {
+		u8 reserved_txq = mvm_sta->reserved_queue;
+		enum iwl_mvm_queue_status *status;
+
+		/*
+		 * If no traffic has gone through the reserved TXQ - it
+		 * is still marked as IWL_MVM_QUEUE_RESERVED, and
+		 * should be manually marked as free again
+		 */
+		spin_lock_bh(&mvm->queue_info_lock);
+		status = &mvm->queue_info[reserved_txq].status;
+		if (WARN((*status != IWL_MVM_QUEUE_RESERVED) &&
+			 (*status != IWL_MVM_QUEUE_FREE),
+			 "sta_id %d reserved txq %d status %d",
+			 sta_id, reserved_txq, *status)) {
 			spin_unlock_bh(&mvm->queue_info_lock);
+			return -EINVAL;
 		}
 
-		if (vif->type == NL80211_IFTYPE_STATION &&
-		    mvmvif->ap_sta_id == sta_id) {
-			/* if associated - we can't remove the AP STA now */
-			if (vif->bss_conf.assoc)
-				return ret;
+		*status = IWL_MVM_QUEUE_FREE;
+		spin_unlock_bh(&mvm->queue_info_lock);
+	}
 
-			/* unassoc - go ahead - remove the AP STA now */
-			mvmvif->ap_sta_id = IWL_MVM_INVALID_STA;
+	if (vif->type == NL80211_IFTYPE_STATION &&
+	    mvmvif->ap_sta_id == sta_id) {
+		/* if associated - we can't remove the AP STA now */
+		if (vif->bss_conf.assoc)
+			return ret;
 
-			/* clear d0i3_ap_sta_id if no longer relevant */
-			if (mvm->d0i3_ap_sta_id == sta_id)
-				mvm->d0i3_ap_sta_id = IWL_MVM_INVALID_STA;
-		}
+		/* unassoc - go ahead - remove the AP STA now */
+		mvmvif->ap_sta_id = IWL_MVM_INVALID_STA;
+
+		/* clear d0i3_ap_sta_id if no longer relevant */
+		if (mvm->d0i3_ap_sta_id == sta_id)
+			mvm->d0i3_ap_sta_id = IWL_MVM_INVALID_STA;
 	}
 
 	/*
@@ -1754,32 +1654,10 @@ int iwl_mvm_rm_sta(struct iwl_mvm *mvm,
 	 * calls the drain worker.
 	 */
 	spin_lock_bh(&mvm_sta->lock);
+	spin_unlock_bh(&mvm_sta->lock);
 
-	/*
-	 * There are frames pending on the AC queues for this station.
-	 * We need to wait until all the frames are drained...
-	 */
-	if (atomic_read(&mvm->pending_frames[sta_id])) {
-		rcu_assign_pointer(mvm->fw_id_to_mac_id[sta_id],
-				   ERR_PTR(-EBUSY));
-		spin_unlock_bh(&mvm_sta->lock);
-
-		/* disable TDLS sta queues on drain complete */
-		if (sta->tdls) {
-			mvm->tfd_drained[sta_id] = mvm_sta->tfd_queue_msk;
-			IWL_DEBUG_TDLS(mvm, "Draining TDLS sta %d\n", sta_id);
-		}
-
-		ret = iwl_mvm_drain_sta(mvm, mvm_sta, true);
-	} else {
-		spin_unlock_bh(&mvm_sta->lock);
-
-		if (!iwl_mvm_is_dqa_supported(mvm) && sta->tdls)
-			iwl_mvm_tdls_sta_deinit(mvm, sta);
-
-		ret = iwl_mvm_rm_sta_common(mvm, mvm_sta->sta_id);
-		RCU_INIT_POINTER(mvm->fw_id_to_mac_id[mvm_sta->sta_id], NULL);
-	}
+	ret = iwl_mvm_rm_sta_common(mvm, mvm_sta->sta_id);
+	RCU_INIT_POINTER(mvm->fw_id_to_mac_id[mvm_sta->sta_id], NULL);
 
 	return ret;
 }
@@ -1822,50 +1700,6 @@ void iwl_mvm_dealloc_int_sta(struct iwl_mvm *mvm, struct iwl_mvm_int_sta *sta)
 	sta->sta_id = IWL_MVM_INVALID_STA;
 }
 
-static int iwl_mvm_add_int_sta_common(struct iwl_mvm *mvm,
-				      struct iwl_mvm_int_sta *sta,
-				      const u8 *addr,
-				      u16 mac_id, u16 color)
-{
-	struct iwl_mvm_add_sta_cmd cmd;
-	int ret;
-	u32 status;
-
-	lockdep_assert_held(&mvm->mutex);
-
-	memset(&cmd, 0, sizeof(cmd));
-	cmd.sta_id = sta->sta_id;
-	cmd.mac_id_n_color = cpu_to_le32(FW_CMD_ID_AND_COLOR(mac_id,
-							     color));
-	if (fw_has_api(&mvm->fw->ucode_capa, IWL_UCODE_TLV_API_STA_TYPE))
-		cmd.station_type = sta->type;
-
-	if (!iwl_mvm_has_new_tx_api(mvm))
-		cmd.tfd_queue_msk = cpu_to_le32(sta->tfd_queue_msk);
-	cmd.tid_disable_tx = cpu_to_le16(0xffff);
-
-	if (addr)
-		memcpy(cmd.addr, addr, ETH_ALEN);
-
-	ret = iwl_mvm_send_cmd_pdu_status(mvm, ADD_STA,
-					  iwl_mvm_add_sta_cmd_size(mvm),
-					  &cmd, &status);
-	if (ret)
-		return ret;
-
-	switch (status & IWL_ADD_STA_STATUS_MASK) {
-	case ADD_STA_SUCCESS:
-		IWL_DEBUG_INFO(mvm, "Internal station added.\n");
-		return 0;
-	default:
-		ret = -EIO;
-		IWL_ERR(mvm, "Add internal station failed, status=0x%x\n",
-			status);
-		break;
-	}
-	return ret;
-}
-
 static void iwl_mvm_enable_aux_queue(struct iwl_mvm *mvm)
 {
 	unsigned int wdg_timeout = iwlmvm_mod_params.tfd_q_hang_detect ?
@@ -1878,7 +1712,7 @@ static void iwl_mvm_enable_aux_queue(struct iwl_mvm *mvm)
 						    IWL_MAX_TID_COUNT,
 						    wdg_timeout);
 		mvm->aux_queue = queue;
-	} else if (iwl_mvm_is_dqa_supported(mvm)) {
+	} else {
 		struct iwl_trans_txq_scd_cfg cfg = {
 			.fifo = IWL_MVM_TX_FIFO_MCAST,
 			.sta_id = mvm->aux_sta.sta_id,
@@ -1889,9 +1723,6 @@ static void iwl_mvm_enable_aux_queue(struct iwl_mvm *mvm)
 
 		iwl_mvm_enable_txq(mvm, mvm->aux_queue, mvm->aux_queue, 0, &cfg,
 				   wdg_timeout);
-	} else {
-		iwl_mvm_enable_ac_txq(mvm, mvm->aux_queue, mvm->aux_queue,
-				      IWL_MVM_TX_FIFO_MCAST, 0, wdg_timeout);
 	}
 }
 
@@ -1991,7 +1822,7 @@ int iwl_mvm_send_add_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 
 	lockdep_assert_held(&mvm->mutex);
 
-	if (iwl_mvm_is_dqa_supported(mvm) && !iwl_mvm_has_new_tx_api(mvm)) {
+	if (!iwl_mvm_has_new_tx_api(mvm)) {
 		if (vif->type == NL80211_IFTYPE_AP ||
 		    vif->type == NL80211_IFTYPE_ADHOC)
 			queue = mvm->probe_queue;
@@ -2078,8 +1909,7 @@ int iwl_mvm_send_rm_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 
 	lockdep_assert_held(&mvm->mutex);
 
-	if (iwl_mvm_is_dqa_supported(mvm))
-		iwl_mvm_free_bcast_sta_queues(mvm, vif);
+	iwl_mvm_free_bcast_sta_queues(mvm, vif);
 
 	ret = iwl_mvm_rm_sta_common(mvm, mvmvif->bcast_sta.sta_id);
 	if (ret)
@@ -2090,23 +1920,10 @@ int iwl_mvm_send_rm_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 int iwl_mvm_alloc_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
-	u32 qmask = 0;
 
 	lockdep_assert_held(&mvm->mutex);
 
-	if (!iwl_mvm_is_dqa_supported(mvm)) {
-		qmask = iwl_mvm_mac_get_queues_mask(vif);
-
-		/*
-		 * The firmware defines the TFD queue mask to only be relevant
-		 * for *unicast* queues, so the multicast (CAB) queue shouldn't
-		 * be included. This only happens in NL80211_IFTYPE_AP vif type,
-		 * so the next line will only have an effect there.
-		 */
-		qmask &= ~BIT(vif->cab_queue);
-	}
-
-	return iwl_mvm_allocate_int_sta(mvm, &mvmvif->bcast_sta, qmask,
+	return iwl_mvm_allocate_int_sta(mvm, &mvmvif->bcast_sta, 0,
 					ieee80211_vif_type_p2p(vif),
 					IWL_STA_GENERAL_PURPOSE);
 }
@@ -2118,7 +1935,7 @@ int iwl_mvm_alloc_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
  * @mvm: the mvm component
  * @vif: the interface to which the broadcast station is added
  * @bsta: the broadcast station to add. */
-int iwl_mvm_add_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
+int iwl_mvm_add_p2p_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	struct iwl_mvm_int_sta *bsta = &mvmvif->bcast_sta;
@@ -2149,7 +1966,7 @@ void iwl_mvm_dealloc_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
  * Send the FW a request to remove the station from it's internal data
  * structures, and in addition remove it from the local data structure.
  */
-int iwl_mvm_rm_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
+int iwl_mvm_rm_p2p_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 {
 	int ret;
 
@@ -2188,9 +2005,6 @@ int iwl_mvm_add_mcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 
 	lockdep_assert_held(&mvm->mutex);
 
-	if (!iwl_mvm_is_dqa_supported(mvm))
-		return 0;
-
 	if (WARN_ON(vif->type != NL80211_IFTYPE_AP &&
 		    vif->type != NL80211_IFTYPE_ADHOC))
 		return -ENOTSUPP;
@@ -2255,9 +2069,6 @@ int iwl_mvm_rm_mcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 
 	lockdep_assert_held(&mvm->mutex);
 
-	if (!iwl_mvm_is_dqa_supported(mvm))
-		return 0;
-
 	iwl_mvm_flush_sta(mvm, &mvmvif->mcast_sta, true, 0);
 
 	iwl_mvm_disable_txq(mvm, mvmvif->cab_queue, vif->cab_queue,
@@ -2507,8 +2318,6 @@ int iwl_mvm_sta_tx_agg(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 		mvm_sta->tid_disable_agg &= ~BIT(tid);
 	} else {
 		/* In DQA-mode the queue isn't removed on agg termination */
-		if (!iwl_mvm_is_dqa_supported(mvm))
-			mvm_sta->tfd_queue_msk &= ~BIT(queue);
 		mvm_sta->tid_disable_agg |= BIT(tid);
 	}
 
@@ -2611,19 +2420,17 @@ int iwl_mvm_sta_tx_agg_start(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 			ret = -ENXIO;
 			goto release_locks;
 		}
-	} else if (iwl_mvm_is_dqa_supported(mvm) &&
-		   unlikely(mvm->queue_info[txq_id].status ==
+	} else if (unlikely(mvm->queue_info[txq_id].status ==
 			    IWL_MVM_QUEUE_SHARED)) {
 		ret = -ENXIO;
 		IWL_DEBUG_TX_QUEUES(mvm,
 				    "Can't start tid %d agg on shared queue!\n",
 				    tid);
 		goto release_locks;
-	} else if (!iwl_mvm_is_dqa_supported(mvm) ||
-	    mvm->queue_info[txq_id].status != IWL_MVM_QUEUE_READY) {
+	} else if (mvm->queue_info[txq_id].status != IWL_MVM_QUEUE_READY) {
 		txq_id = iwl_mvm_find_free_queue(mvm, mvmsta->sta_id,
-						 mvm->first_agg_queue,
-						 mvm->last_agg_queue);
+						 IWL_MVM_DQA_MIN_DATA_QUEUE,
+						 IWL_MVM_DQA_MAX_DATA_QUEUE);
 		if (txq_id < 0) {
 			ret = txq_id;
 			IWL_ERR(mvm, "Failed to allocate agg queue\n");
@@ -2741,37 +2548,34 @@ int iwl_mvm_sta_tx_agg_oper(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	queue_status = mvm->queue_info[queue].status;
 	spin_unlock_bh(&mvm->queue_info_lock);
 
-	/* In DQA mode, the existing queue might need to be reconfigured */
-	if (iwl_mvm_is_dqa_supported(mvm)) {
-		/* Maybe there is no need to even alloc a queue... */
-		if (mvm->queue_info[queue].status == IWL_MVM_QUEUE_READY)
-			alloc_queue = false;
+	/* Maybe there is no need to even alloc a queue... */
+	if (mvm->queue_info[queue].status == IWL_MVM_QUEUE_READY)
+		alloc_queue = false;
 
+	/*
+	 * Only reconfig the SCD for the queue if the window size has
+	 * changed from current (become smaller)
+	 */
+	if (!alloc_queue && buf_size < mvmsta->max_agg_bufsize) {
 		/*
-		 * Only reconfig the SCD for the queue if the window size has
-		 * changed from current (become smaller)
+		 * If reconfiguring an existing queue, it first must be
+		 * drained
 		 */
-		if (!alloc_queue && buf_size < mvmsta->max_agg_bufsize) {
-			/*
-			 * If reconfiguring an existing queue, it first must be
-			 * drained
-			 */
-			ret = iwl_trans_wait_tx_queues_empty(mvm->trans,
-							     BIT(queue));
-			if (ret) {
-				IWL_ERR(mvm,
-					"Error draining queue before reconfig\n");
-				return ret;
-			}
+		ret = iwl_trans_wait_tx_queues_empty(mvm->trans,
+						     BIT(queue));
+		if (ret) {
+			IWL_ERR(mvm,
+				"Error draining queue before reconfig\n");
+			return ret;
+		}
 
-			ret = iwl_mvm_reconfig_scd(mvm, queue, cfg.fifo,
-						   mvmsta->sta_id, tid,
-						   buf_size, ssn);
-			if (ret) {
-				IWL_ERR(mvm,
-					"Error reconfiguring TXQ #%d\n", queue);
-				return ret;
-			}
+		ret = iwl_mvm_reconfig_scd(mvm, queue, cfg.fifo,
+					   mvmsta->sta_id, tid,
+					   buf_size, ssn);
+		if (ret) {
+			IWL_ERR(mvm,
+				"Error reconfiguring TXQ #%d\n", queue);
+			return ret;
 		}
 	}
 
@@ -2867,18 +2671,6 @@ int iwl_mvm_sta_tx_agg_stop(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 				    "ssn = %d, next_recl = %d\n",
 				    tid_data->ssn, tid_data->next_reclaimed);
 
-		/*
-		 * There are still packets for this RA / TID in the HW.
-		 * Not relevant for DQA mode, since there is no need to disable
-		 * the queue.
-		 */
-		if (!iwl_mvm_is_dqa_supported(mvm) &&
-		    tid_data->ssn != tid_data->next_reclaimed) {
-			tid_data->state = IWL_EMPTYING_HW_QUEUE_DELBA;
-			err = 0;
-			break;
-		}
-
 		tid_data->ssn = 0xffff;
 		tid_data->state = IWL_AGG_OFF;
 		spin_unlock_bh(&mvmsta->lock);
@@ -2886,12 +2678,6 @@ int iwl_mvm_sta_tx_agg_stop(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 		ieee80211_stop_tx_ba_cb_irqsafe(vif, sta->addr, tid);
 
 		iwl_mvm_sta_tx_agg(mvm, sta, tid, txq_id, false);
-
-		if (!iwl_mvm_is_dqa_supported(mvm)) {
-			int mac_queue = vif->hw_queue[tid_to_mac80211_ac[tid]];
-
-			iwl_mvm_disable_txq(mvm, txq_id, mac_queue, tid, 0);
-		}
 		return 0;
 	case IWL_AGG_STARTING:
 	case IWL_EMPTYING_HW_QUEUE_ADDBA:
@@ -2961,13 +2747,6 @@ int iwl_mvm_sta_tx_agg_flush(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 		iwl_mvm_drain_sta(mvm, mvmsta, false);
 
 		iwl_mvm_sta_tx_agg(mvm, sta, tid, txq_id, false);
-
-		if (!iwl_mvm_is_dqa_supported(mvm)) {
-			int mac_queue = vif->hw_queue[tid_to_mac80211_ac[tid]];
-
-			iwl_mvm_disable_txq(mvm, tid_data->txq_id, mac_queue,
-					    tid, 0);
-		}
 	}
 
 	return 0;
@@ -3586,15 +3365,6 @@ void iwl_mvm_sta_modify_sleep_tx_count(struct iwl_mvm *mvm,
 			u16 n_queued;
 
 			tid_data = &mvmsta->tid_data[tid];
-			if (WARN(!iwl_mvm_is_dqa_supported(mvm) &&
-				 tid_data->state != IWL_AGG_ON &&
-				 tid_data->state != IWL_EMPTYING_HW_QUEUE_DELBA,
-				 "TID %d state is %d\n",
-				 tid, tid_data->state)) {
-				spin_unlock_bh(&mvmsta->lock);
-				ieee80211_sta_eosp(sta);
-				return;
-			}
 
 			n_queued = iwl_mvm_tid_queued(mvm, tid_data);
 			if (n_queued > remaining) {
@@ -3688,13 +3458,8 @@ void iwl_mvm_sta_modify_disable_tx_ap(struct iwl_mvm *mvm,
 
 	mvm_sta->disable_tx = disable;
 
-	/*
-	 * Tell mac80211 to start/stop queuing tx for this station,
-	 * but don't stop queuing if there are still pending frames
-	 * for this station.
-	 */
-	if (disable || !atomic_read(&mvm->pending_frames[mvm_sta->sta_id]))
-		ieee80211_sta_block_awake(mvm->hw, sta, disable);
+	/* Tell mac80211 to start/stop queuing tx for this station */
+	ieee80211_sta_block_awake(mvm->hw, sta, disable);
 
 	iwl_mvm_sta_modify_disable_tx(mvm, mvm_sta, disable);
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/sta.h b/drivers/net/wireless/intel/iwlwifi/mvm/sta.h
index 05fecbe87da4..d13893806513 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/sta.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/sta.h
@@ -222,16 +222,7 @@ struct iwl_mvm_vif;
  * we remove the STA of the AP. The flush can be done synchronously against the
  * fw.
  * Drain means that the fw will drop all the frames sent to a specific station.
- * This is useful when a client (if we are IBSS / GO or AP) disassociates. In
- * that case, we need to drain all the frames for that client from the AC queues
- * that are shared with the other clients. Only then, we can remove the STA in
- * the fw. In order to do so, we track the non-AMPDU packets for each station.
- * If mac80211 removes a STA and if it still has non-AMPDU packets pending in
- * the queues, we mark this station as %EBUSY in %fw_id_to_mac_id, and drop all
- * the frames for this STA (%iwl_mvm_rm_sta). When the last frame is dropped
- * (we know about it with its Tx response), we remove the station in fw and set
- * it as %NULL in %fw_id_to_mac_id: this is the purpose of
- * %iwl_mvm_sta_drained_wk.
+ * This is useful when a client (if we are IBSS / GO or AP) disassociates.
  */
 
 /**
@@ -325,6 +316,10 @@ enum iwl_mvm_agg_state {
  * @is_tid_active: has this TID sent traffic in the last
  *	%IWL_MVM_DQA_QUEUE_TIMEOUT time period. If %txq_id is invalid, this
  *	field should be ignored.
+ * @tpt_meas_start: time of the throughput measurements start, is reset every HZ
+ * @tx_count_last: number of frames transmitted during the last second
+ * @tx_count: counts the number of frames transmitted since the last reset of
+ *	 tpt_meas_start
  */
 struct iwl_mvm_tid_data {
 	struct sk_buff_head deferred_tx_frames;
@@ -339,6 +334,9 @@ struct iwl_mvm_tid_data {
 	u16 ssn;
 	u16 tx_time;
 	bool is_tid_active;
+	unsigned long tpt_meas_start;
+	u32 tx_count_last;
+	u32 tx_count;
 };
 
 struct iwl_mvm_key_pn {
@@ -371,7 +369,6 @@ struct iwl_mvm_rxq_dup_data {
  * struct iwl_mvm_sta - representation of a station in the driver
  * @sta_id: the index of the station in the fw (will be replaced by id_n_color)
  * @tfd_queue_msk: the tfd queues used by the station
- * @hw_queue: per-AC mapping of the TFD queues used by station
  * @mac_id_n_color: the MAC context this station is linked to
  * @tid_disable_agg: bitmap: if bit(tid) is set, the fw won't send ampdus for
  *	tid.
@@ -409,7 +406,6 @@ struct iwl_mvm_rxq_dup_data {
 struct iwl_mvm_sta {
 	u32 sta_id;
 	u32 tfd_queue_msk;
-	u8 hw_queue[IEEE80211_NUM_ACS];
 	u32 mac_id_n_color;
 	u16 tid_disable_agg;
 	u8 max_agg_bufsize;
@@ -533,9 +529,9 @@ void iwl_mvm_del_aux_sta(struct iwl_mvm *mvm);
 
 int iwl_mvm_alloc_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 int iwl_mvm_send_add_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
-int iwl_mvm_add_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
+int iwl_mvm_add_p2p_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 int iwl_mvm_send_rm_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
-int iwl_mvm_rm_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
+int iwl_mvm_rm_p2p_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 int iwl_mvm_add_mcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 int iwl_mvm_rm_mcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 int iwl_mvm_allocate_int_sta(struct iwl_mvm *mvm,
@@ -548,7 +544,6 @@ int iwl_mvm_add_snif_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 int iwl_mvm_rm_snif_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif);
 void iwl_mvm_dealloc_snif_sta(struct iwl_mvm *mvm);
 
-void iwl_mvm_sta_drained_wk(struct work_struct *wk);
 void iwl_mvm_sta_modify_ps_wake(struct iwl_mvm *mvm,
 				struct ieee80211_sta *sta);
 void iwl_mvm_sta_modify_sleep_tx_count(struct iwl_mvm *mvm,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/time-event.c b/drivers/net/wireless/intel/iwlwifi/mvm/time-event.c
index 5a682722adce..4d0314912e94 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/time-event.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/time-event.c
@@ -7,6 +7,7 @@
  *
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2017 Intel Deutschland GmbH
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -33,6 +34,7 @@
  *
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2017 Intel Deutschland GmbH
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -73,7 +75,6 @@
 #include "mvm.h"
 #include "iwl-io.h"
 #include "iwl-prph.h"
-#include "fw-dbg.h"
 
 /*
  * For the high priority TE use a time event type that has similar priority to
@@ -130,10 +131,7 @@ void iwl_mvm_roc_done_wk(struct work_struct *wk)
 	 * issue as it will have to complete before the next command is
 	 * executed, and a new time event means a new command.
 	 */
-	if (iwl_mvm_is_dqa_supported(mvm))
-		iwl_mvm_flush_sta(mvm, &mvm->aux_sta, true, CMD_ASYNC);
-	else
-		iwl_mvm_flush_tx_path(mvm, queues, CMD_ASYNC);
+	iwl_mvm_flush_sta(mvm, &mvm->aux_sta, true, CMD_ASYNC);
 }
 
 static void iwl_mvm_roc_finished(struct iwl_mvm *mvm)
@@ -248,7 +246,9 @@ static void iwl_mvm_te_check_trigger(struct iwl_mvm *mvm,
 	trig = iwl_fw_dbg_get_trigger(mvm->fw, FW_DBG_TRIGGER_TIME_EVENT);
 	te_trig = (void *)trig->data;
 
-	if (!iwl_fw_dbg_trigger_check_stop(mvm, te_data->vif, trig))
+	if (!iwl_fw_dbg_trigger_check_stop(&mvm->fwrt,
+					   ieee80211_vif_to_wdev(te_data->vif),
+					   trig))
 		return;
 
 	for (i = 0; i < ARRAY_SIZE(te_trig->time_events); i++) {
@@ -263,11 +263,11 @@ static void iwl_mvm_te_check_trigger(struct iwl_mvm *mvm,
 		    !(trig_status_bitmap & BIT(le32_to_cpu(notif->status))))
 			continue;
 
-		iwl_mvm_fw_dbg_collect_trig(mvm, trig,
-					    "Time event %d Action 0x%x received status: %d",
-					    te_data->id,
-					    le32_to_cpu(notif->action),
-					    le32_to_cpu(notif->status));
+		iwl_fw_dbg_collect_trig(&mvm->fwrt, trig,
+					"Time event %d Action 0x%x received status: %d",
+					te_data->id,
+					le32_to_cpu(notif->action),
+					le32_to_cpu(notif->status));
 		break;
 	}
 }
@@ -728,8 +728,21 @@ void iwl_mvm_stop_session_protection(struct iwl_mvm *mvm,
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	struct iwl_mvm_time_event_data *te_data = &mvmvif->time_event_data;
+	u32 id;
 
 	lockdep_assert_held(&mvm->mutex);
+
+	spin_lock_bh(&mvm->time_event_lock);
+	id = te_data->id;
+	spin_unlock_bh(&mvm->time_event_lock);
+
+	if (id != TE_BSS_STA_AGGRESSIVE_ASSOC) {
+		IWL_DEBUG_TE(mvm,
+			     "don't remove TE with id=%u (not session protection)\n",
+			     id);
+		return;
+	}
+
 	iwl_mvm_remove_time_event(mvm, mvmvif, te_data);
 }
 
@@ -861,8 +874,23 @@ int iwl_mvm_schedule_csa_period(struct iwl_mvm *mvm,
 	lockdep_assert_held(&mvm->mutex);
 
 	if (te_data->running) {
-		IWL_DEBUG_TE(mvm, "CS period is already scheduled\n");
-		return -EBUSY;
+		u32 id;
+
+		spin_lock_bh(&mvm->time_event_lock);
+		id = te_data->id;
+		spin_unlock_bh(&mvm->time_event_lock);
+
+		if (id == TE_CHANNEL_SWITCH_PERIOD) {
+			IWL_DEBUG_TE(mvm, "CS period is already scheduled\n");
+			return -EBUSY;
+		}
+
+		/*
+		 * Remove the session protection time event to allow the
+		 * channel switch. If we got here, we just heard a beacon so
+		 * the session protection is not needed anymore anyway.
+		 */
+		iwl_mvm_remove_time_event(mvm, mvmvif, te_data);
 	}
 
 	time_cmd.action = cpu_to_le32(FW_CTXT_ACTION_ADD);
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tof.c b/drivers/net/wireless/intel/iwlwifi/mvm/tof.c
index 634175b2480c..2d0b8a391308 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/tof.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/tof.c
@@ -61,7 +61,7 @@
  *
  *****************************************************************************/
 #include "mvm.h"
-#include "fw-api-tof.h"
+#include "fw/api/tof.h"
 
 #define IWL_MVM_TOF_RANGE_REQ_MAX_ID 256
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tof.h b/drivers/net/wireless/intel/iwlwifi/mvm/tof.h
index 8c3421c9991d..2ff560aa1a82 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/tof.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/tof.h
@@ -63,7 +63,7 @@
 #ifndef __tof_h__
 #define __tof_h__
 
-#include "fw-api-tof.h"
+#include "fw/api/tof.h"
 
 struct iwl_mvm_tof_data {
 	struct iwl_tof_config_cmd tof_cfg;
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tt.c b/drivers/net/wireless/intel/iwlwifi/mvm/tt.c
index 453a785a3ea5..8876c2abc440 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/tt.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/tt.c
@@ -71,7 +71,7 @@
 
 #define IWL_MVM_TEMP_NOTIF_WAIT_TIMEOUT	HZ
 
-static void iwl_mvm_enter_ctkill(struct iwl_mvm *mvm)
+void iwl_mvm_enter_ctkill(struct iwl_mvm *mvm)
 {
 	struct iwl_mvm_tt_mgmt *tt = &mvm->thermal_throttle;
 	u32 duration = tt->params.ct_kill_duration;
@@ -629,7 +629,7 @@ static int iwl_mvm_tzone_get_temp(struct thermal_zone_device *device,
 	mutex_lock(&mvm->mutex);
 
 	if (!iwl_mvm_firmware_running(mvm) ||
-	    mvm->cur_ucode != IWL_UCODE_REGULAR) {
+	    mvm->fwrt.cur_fw_img != IWL_UCODE_REGULAR) {
 		ret = -EIO;
 		goto out;
 	}
@@ -680,7 +680,7 @@ static int iwl_mvm_tzone_set_trip_temp(struct thermal_zone_device *device,
 	mutex_lock(&mvm->mutex);
 
 	if (!iwl_mvm_firmware_running(mvm) ||
-	    mvm->cur_ucode != IWL_UCODE_REGULAR) {
+	    mvm->fwrt.cur_fw_img != IWL_UCODE_REGULAR) {
 		ret = -EIO;
 		goto out;
 	}
@@ -795,7 +795,7 @@ static int iwl_mvm_tcool_set_cur_state(struct thermal_cooling_device *cdev,
 	mutex_lock(&mvm->mutex);
 
 	if (!iwl_mvm_firmware_running(mvm) ||
-	    mvm->cur_ucode != IWL_UCODE_REGULAR) {
+	    mvm->fwrt.cur_fw_img != IWL_UCODE_REGULAR) {
 		ret = -EIO;
 		goto unlock;
 	}
@@ -813,7 +813,7 @@ unlock:
 	return ret;
 }
 
-static struct thermal_cooling_device_ops tcooling_ops = {
+static const struct thermal_cooling_device_ops tcooling_ops = {
 	.get_max_state = iwl_mvm_tcool_get_max_state,
 	.get_cur_state = iwl_mvm_tcool_get_cur_state,
 	.set_cur_state = iwl_mvm_tcool_set_cur_state,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c
index 60360ed73f26..172b5e63d3fb 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c
@@ -74,7 +74,6 @@
 #include "iwl-eeprom-parse.h"
 #include "mvm.h"
 #include "sta.h"
-#include "fw-dbg.h"
 
 static void
 iwl_mvm_bar_check_trigger(struct iwl_mvm *mvm, const u8 *addr,
@@ -89,15 +88,15 @@ iwl_mvm_bar_check_trigger(struct iwl_mvm *mvm, const u8 *addr,
 	trig = iwl_fw_dbg_get_trigger(mvm->fw, FW_DBG_TRIGGER_BA);
 	ba_trig = (void *)trig->data;
 
-	if (!iwl_fw_dbg_trigger_check_stop(mvm, NULL, trig))
+	if (!iwl_fw_dbg_trigger_check_stop(&mvm->fwrt, NULL, trig))
 		return;
 
 	if (!(le16_to_cpu(ba_trig->tx_bar) & BIT(tid)))
 		return;
 
-	iwl_mvm_fw_dbg_collect_trig(mvm, trig,
-				    "BAR sent to %pM, tid %d, ssn %d",
-				    addr, tid, ssn);
+	iwl_fw_dbg_collect_trig(&mvm->fwrt, trig,
+				"BAR sent to %pM, tid %d, ssn %d",
+				addr, tid, ssn);
 }
 
 #define OPT_HDR(type, skb, off) \
@@ -185,8 +184,14 @@ static u16 iwl_mvm_tx_csum(struct iwl_mvm *mvm, struct sk_buff *skb,
 	else
 		udp_hdr(skb)->check = 0;
 
-	/* mac header len should include IV, size is in words */
-	if (info->control.hw_key)
+	/*
+	 * mac header len should include IV, size is in words unless
+	 * the IV is added by the firmware like in WEP.
+	 * In new Tx API, the IV is always added by the firmware.
+	 */
+	if (!iwl_mvm_has_new_tx_api(mvm) && info->control.hw_key &&
+	    info->control.hw_key->cipher != WLAN_CIPHER_SUITE_WEP40 &&
+	    info->control.hw_key->cipher != WLAN_CIPHER_SUITE_WEP104)
 		mh_len += info->control.hw_key->iv_len;
 	mh_len /= 2;
 	offload_assist |= mh_len << TX_CMD_OFFLD_MH_SIZE;
@@ -553,9 +558,6 @@ static int iwl_mvm_get_ctrl_vif_queue(struct iwl_mvm *mvm,
 {
 	struct iwl_mvm_vif *mvmvif;
 
-	if (!iwl_mvm_is_dqa_supported(mvm))
-		return info->hw_queue;
-
 	mvmvif = iwl_mvm_vif_from_mac80211(info->control.vif);
 
 	switch (info->control.vif->type) {
@@ -654,8 +656,7 @@ int iwl_mvm_tx_skb_non_sta(struct iwl_mvm *mvm, struct sk_buff *skb)
 
 			if (ap_sta_id != IWL_MVM_INVALID_STA)
 				sta_id = ap_sta_id;
-		} else if (iwl_mvm_is_dqa_supported(mvm) &&
-			   info.control.vif->type == NL80211_IFTYPE_MONITOR) {
+		} else if (info.control.vif->type == NL80211_IFTYPE_MONITOR) {
 			queue = mvm->aux_queue;
 		}
 	}
@@ -674,17 +675,6 @@ int iwl_mvm_tx_skb_non_sta(struct iwl_mvm *mvm, struct sk_buff *skb)
 		return -1;
 	}
 
-	/*
-	 * Increase the pending frames counter, so that later when a reply comes
-	 * in and the counter is decreased - we don't start getting negative
-	 * values.
-	 * Note that we don't need to make sure it isn't agg'd, since we're
-	 * TXing non-sta
-	 * For DQA mode - we shouldn't increase it though
-	 */
-	if (!iwl_mvm_is_dqa_supported(mvm))
-		atomic_inc(&mvm->pending_frames[sta_id]);
-
 	return 0;
 }
 
@@ -752,7 +742,7 @@ static int iwl_mvm_tx_tso(struct iwl_mvm *mvm, struct sk_buff *skb,
 	max_amsdu_len = sta->max_amsdu_len;
 
 	/* the Tx FIFO to which this A-MSDU will be routed */
-	txf = iwl_mvm_ac_to_tx_fifo[tid_to_mac80211_ac[tid]];
+	txf = iwl_mvm_mac_ac_to_tx_fifo(mvm, tid_to_mac80211_ac[tid]);
 
 	/*
 	 * Don't send an AMSDU that will be longer than the TXF.
@@ -761,7 +751,8 @@ static int iwl_mvm_tx_tso(struct iwl_mvm *mvm, struct sk_buff *skb,
 	 * fifo to be able to send bursts.
 	 */
 	max_amsdu_len = min_t(unsigned int, max_amsdu_len,
-			      mvm->smem_cfg.lmac[0].txfifo_size[txf] - 256);
+			      mvm->fwrt.smem_cfg.lmac[0].txfifo_size[txf] -
+			      256);
 
 	if (unlikely(dbg_max_amsdu_len))
 		max_amsdu_len = min_t(unsigned int, max_amsdu_len,
@@ -994,22 +985,13 @@ static int iwl_mvm_tx_mpdu(struct iwl_mvm *mvm, struct sk_buff *skb,
 		}
 	}
 
-	if (iwl_mvm_is_dqa_supported(mvm) || is_ampdu)
-		txq_id = mvmsta->tid_data[tid].txq_id;
-
-	if (sta->tdls && !iwl_mvm_is_dqa_supported(mvm)) {
-		/* default to TID 0 for non-QoS packets */
-		u8 tdls_tid = tid == IWL_MAX_TID_COUNT ? 0 : tid;
-
-		txq_id = mvmsta->hw_queue[tid_to_mac80211_ac[tdls_tid]];
-	}
+	txq_id = mvmsta->tid_data[tid].txq_id;
 
 	WARN_ON_ONCE(info->flags & IEEE80211_TX_CTL_SEND_AFTER_DTIM);
 
 	/* Check if TXQ needs to be allocated or re-activated */
 	if (unlikely(txq_id == IWL_MVM_INVALID_QUEUE ||
-		     !mvmsta->tid_data[tid].is_tid_active) &&
-	    iwl_mvm_is_dqa_supported(mvm)) {
+		     !mvmsta->tid_data[tid].is_tid_active)) {
 		/* If TXQ needs to be allocated... */
 		if (txq_id == IWL_MVM_INVALID_QUEUE) {
 			iwl_mvm_tx_add_stream(mvm, mvmsta, tid, skb);
@@ -1036,7 +1018,7 @@ static int iwl_mvm_tx_mpdu(struct iwl_mvm *mvm, struct sk_buff *skb,
 				    txq_id);
 	}
 
-	if (iwl_mvm_is_dqa_supported(mvm) && !iwl_mvm_has_new_tx_api(mvm)) {
+	if (!iwl_mvm_has_new_tx_api(mvm)) {
 		/* Keep track of the time of the last frame for this RA/TID */
 		mvm->queue_info[txq_id].last_frame_time[tid] = jiffies;
 
@@ -1070,10 +1052,6 @@ static int iwl_mvm_tx_mpdu(struct iwl_mvm *mvm, struct sk_buff *skb,
 
 	spin_unlock(&mvmsta->lock);
 
-	/* Increase pending frames count if this isn't AMPDU or DQA queue */
-	if (!iwl_mvm_is_dqa_supported(mvm) && !is_ampdu)
-		atomic_inc(&mvm->pending_frames[mvmsta->sta_id]);
-
 	return 0;
 
 drop_unlock_sta:
@@ -1142,8 +1120,7 @@ static void iwl_mvm_check_ratid_empty(struct iwl_mvm *mvm,
 	lockdep_assert_held(&mvmsta->lock);
 
 	if ((tid_data->state == IWL_AGG_ON ||
-	     tid_data->state == IWL_EMPTYING_HW_QUEUE_DELBA ||
-	     iwl_mvm_is_dqa_supported(mvm)) &&
+	     tid_data->state == IWL_EMPTYING_HW_QUEUE_DELBA) &&
 	    iwl_mvm_tid_queued(mvm, tid_data) == 0) {
 		/*
 		 * Now that this aggregation or DQA queue is empty tell
@@ -1177,13 +1154,6 @@ static void iwl_mvm_check_ratid_empty(struct iwl_mvm *mvm,
 		IWL_DEBUG_TX_QUEUES(mvm,
 				    "Can continue DELBA flow ssn = next_recl = %d\n",
 				    tid_data->next_reclaimed);
-		if (!iwl_mvm_is_dqa_supported(mvm)) {
-			u8 mac80211_ac = tid_to_mac80211_ac[tid];
-
-			iwl_mvm_disable_txq(mvm, tid_data->txq_id,
-					    vif->hw_queue[mac80211_ac], tid,
-					    CMD_ASYNC);
-		}
 		tid_data->state = IWL_AGG_OFF;
 		ieee80211_stop_tx_ba_cb_irqsafe(vif, sta->addr, tid);
 		break;
@@ -1295,7 +1265,7 @@ static void iwl_mvm_tx_status_check_trigger(struct iwl_mvm *mvm,
 	trig = iwl_fw_dbg_get_trigger(mvm->fw, FW_DBG_TRIGGER_TX_STATUS);
 	status_trig = (void *)trig->data;
 
-	if (!iwl_fw_dbg_trigger_check_stop(mvm, NULL, trig))
+	if (!iwl_fw_dbg_trigger_check_stop(&mvm->fwrt, NULL, trig))
 		return;
 
 	for (i = 0; i < ARRAY_SIZE(status_trig->statuses); i++) {
@@ -1306,9 +1276,9 @@ static void iwl_mvm_tx_status_check_trigger(struct iwl_mvm *mvm,
 		if (status_trig->statuses[i].status != (status & TX_STATUS_MSK))
 			continue;
 
-		iwl_mvm_fw_dbg_collect_trig(mvm, trig,
-					    "Tx status %d was received",
-					    status & TX_STATUS_MSK);
+		iwl_fw_dbg_collect_trig(&mvm->fwrt, trig,
+					"Tx status %d was received",
+					status & TX_STATUS_MSK);
 		break;
 	}
 }
@@ -1367,6 +1337,7 @@ static void iwl_mvm_rx_tx_cmd_single(struct iwl_mvm *mvm,
 	while (!skb_queue_empty(&skbs)) {
 		struct sk_buff *skb = __skb_dequeue(&skbs);
 		struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+		bool flushed = false;
 
 		skb_freed++;
 
@@ -1380,11 +1351,15 @@ static void iwl_mvm_rx_tx_cmd_single(struct iwl_mvm *mvm,
 		case TX_STATUS_DIRECT_DONE:
 			info->flags |= IEEE80211_TX_STAT_ACK;
 			break;
+		case TX_STATUS_FAIL_FIFO_FLUSHED:
+		case TX_STATUS_FAIL_DRAIN_FLOW:
+			flushed = true;
+			break;
 		case TX_STATUS_FAIL_DEST_PS:
-			/* In DQA, the FW should have stopped the queue and not
+			/* the FW should have stopped the queue and not
 			 * return this status
 			 */
-			WARN_ON(iwl_mvm_is_dqa_supported(mvm));
+			WARN_ON(1);
 			info->flags |= IEEE80211_TX_STAT_TX_FILTERED;
 			break;
 		default:
@@ -1402,7 +1377,7 @@ static void iwl_mvm_rx_tx_cmd_single(struct iwl_mvm *mvm,
 		/* Single frame failure in an AMPDU queue => send BAR */
 		if (info->flags & IEEE80211_TX_CTL_AMPDU &&
 		    !(info->flags & IEEE80211_TX_STAT_ACK) &&
-		    !(info->flags & IEEE80211_TX_STAT_TX_FILTERED))
+		    !(info->flags & IEEE80211_TX_STAT_TX_FILTERED) && !flushed)
 			info->flags |= IEEE80211_TX_STAT_AMPDU_NO_BACK;
 		info->flags &= ~IEEE80211_TX_CTL_AMPDU;
 
@@ -1440,26 +1415,21 @@ static void iwl_mvm_rx_tx_cmd_single(struct iwl_mvm *mvm,
 		ieee80211_tx_status(mvm->hw, skb);
 	}
 
-	if (iwl_mvm_is_dqa_supported(mvm) || txq_id >= mvm->first_agg_queue) {
-		/* If this is an aggregation queue, we use the ssn since:
-		 * ssn = wifi seq_num % 256.
-		 * The seq_ctl is the sequence control of the packet to which
-		 * this Tx response relates. But if there is a hole in the
-		 * bitmap of the BA we received, this Tx response may allow to
-		 * reclaim the hole and all the subsequent packets that were
-		 * already acked. In that case, seq_ctl != ssn, and the next
-		 * packet to be reclaimed will be ssn and not seq_ctl. In that
-		 * case, several packets will be reclaimed even if
-		 * frame_count = 1.
-		 *
-		 * The ssn is the index (% 256) of the latest packet that has
-		 * treated (acked / dropped) + 1.
-		 */
-		next_reclaimed = ssn;
-	} else {
-		/* The next packet to be reclaimed is the one after this one */
-		next_reclaimed = IEEE80211_SEQ_TO_SN(seq_ctl + 0x10);
-	}
+	/* This is an aggregation queue or might become one, so we use
+	 * the ssn since: ssn = wifi seq_num % 256.
+	 * The seq_ctl is the sequence control of the packet to which
+	 * this Tx response relates. But if there is a hole in the
+	 * bitmap of the BA we received, this Tx response may allow to
+	 * reclaim the hole and all the subsequent packets that were
+	 * already acked. In that case, seq_ctl != ssn, and the next
+	 * packet to be reclaimed will be ssn and not seq_ctl. In that
+	 * case, several packets will be reclaimed even if
+	 * frame_count = 1.
+	 *
+	 * The ssn is the index (% 256) of the latest packet that has
+	 * treated (acked / dropped) + 1.
+	 */
+	next_reclaimed = ssn;
 
 	IWL_DEBUG_TX_REPLY(mvm,
 			   "TXQ %d status %s (0x%08x)\n",
@@ -1542,49 +1512,6 @@ static void iwl_mvm_rx_tx_cmd_single(struct iwl_mvm *mvm,
 		mvmsta = NULL;
 	}
 
-	/*
-	 * If the txq is not an AMPDU queue, there is no chance we freed
-	 * several skbs. Check that out...
-	 */
-	if (iwl_mvm_is_dqa_supported(mvm) || txq_id >= mvm->first_agg_queue)
-		goto out;
-
-	/* We can't free more than one frame at once on a shared queue */
-	WARN_ON(skb_freed > 1);
-
-	/* If we have still frames for this STA nothing to do here */
-	if (!atomic_sub_and_test(skb_freed, &mvm->pending_frames[sta_id]))
-		goto out;
-
-	if (mvmsta && mvmsta->vif->type == NL80211_IFTYPE_AP) {
-
-		/*
-		 * If there are no pending frames for this STA and
-		 * the tx to this station is not disabled, notify
-		 * mac80211 that this station can now wake up in its
-		 * STA table.
-		 * If mvmsta is not NULL, sta is valid.
-		 */
-
-		spin_lock_bh(&mvmsta->lock);
-
-		if (!mvmsta->disable_tx)
-			ieee80211_sta_block_awake(mvm->hw, sta, false);
-
-		spin_unlock_bh(&mvmsta->lock);
-	}
-
-	if (PTR_ERR(sta) == -EBUSY || PTR_ERR(sta) == -ENOENT) {
-		/*
-		 * We are draining and this was the last packet - pre_rcu_remove
-		 * has been called already. We might be after the
-		 * synchronize_net already.
-		 * Don't rely on iwl_mvm_rm_sta to see the empty Tx queues.
-		 */
-		set_bit(sta_id, mvm->sta_drained);
-		schedule_work(&mvm->sta_drained_wk);
-	}
-
 out:
 	rcu_read_unlock();
 }
@@ -1599,7 +1526,7 @@ static const char *iwl_get_agg_tx_status(u16 status)
 	AGG_TX_STATE_(BT_PRIO);
 	AGG_TX_STATE_(FEW_BYTES);
 	AGG_TX_STATE_(ABORT);
-	AGG_TX_STATE_(LAST_SENT_TTL);
+	AGG_TX_STATE_(TX_ON_AIR_DROP);
 	AGG_TX_STATE_(LAST_SENT_TRY_CNT);
 	AGG_TX_STATE_(LAST_SENT_BT_KILL);
 	AGG_TX_STATE_(SCD_QUERY);
@@ -1648,9 +1575,8 @@ static void iwl_mvm_rx_tx_cmd_agg(struct iwl_mvm *mvm,
 	struct iwl_mvm_sta *mvmsta;
 	int queue = SEQ_TO_QUEUE(sequence);
 
-	if (WARN_ON_ONCE(queue < mvm->first_agg_queue &&
-			 (!iwl_mvm_is_dqa_supported(mvm) ||
-			  (queue != IWL_MVM_DQA_BSS_CLIENT_QUEUE))))
+	if (WARN_ON_ONCE(queue < IWL_MVM_DQA_MIN_DATA_QUEUE &&
+			 (queue != IWL_MVM_DQA_BSS_CLIENT_QUEUE)))
 		return;
 
 	if (WARN_ON_ONCE(tid == IWL_TID_NON_QOS))
@@ -1815,6 +1741,8 @@ void iwl_mvm_rx_ba_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
 	struct iwl_mvm_tid_data *tid_data;
 	struct iwl_mvm_sta *mvmsta;
 
+	ba_info.flags = IEEE80211_TX_STAT_AMPDU;
+
 	if (iwl_mvm_has_new_tx_api(mvm)) {
 		struct iwl_mvm_compressed_ba_notif *ba_res =
 			(void *)pkt->data;
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/utils.c b/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
index fc5a490880d0..2ea74abad73d 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
@@ -70,9 +70,8 @@
 #include "iwl-io.h"
 #include "iwl-prph.h"
 #include "iwl-csr.h"
-#include "fw-dbg.h"
 #include "mvm.h"
-#include "fw-api-rs.h"
+#include "fw/api/rs.h"
 
 /*
  * Will return 0 even if the cmd failed when RFKILL is asserted unless
@@ -464,8 +463,8 @@ static void iwl_mvm_dump_umac_error_log(struct iwl_mvm *mvm)
 		IWL_ERR(mvm,
 			"Not valid error log pointer 0x%08X for %s uCode\n",
 			base,
-			(mvm->cur_ucode == IWL_UCODE_INIT)
-					? "Init" : "RT");
+			(mvm->fwrt.cur_fw_img == IWL_UCODE_INIT)
+			? "Init" : "RT");
 		return;
 	}
 
@@ -500,7 +499,7 @@ static void iwl_mvm_dump_lmac_error_log(struct iwl_mvm *mvm, u32 base)
 	struct iwl_error_event_table table;
 	u32 val;
 
-	if (mvm->cur_ucode == IWL_UCODE_INIT) {
+	if (mvm->fwrt.cur_fw_img == IWL_UCODE_INIT) {
 		if (!base)
 			base = mvm->fw->init_errlog_ptr;
 	} else {
@@ -512,8 +511,8 @@ static void iwl_mvm_dump_lmac_error_log(struct iwl_mvm *mvm, u32 base)
 		IWL_ERR(mvm,
 			"Not valid error log pointer 0x%08X for %s uCode\n",
 			base,
-			(mvm->cur_ucode == IWL_UCODE_INIT)
-					? "Init" : "RT");
+			(mvm->fwrt.cur_fw_img == IWL_UCODE_INIT)
+			? "Init" : "RT");
 		return;
 	}
 
@@ -1190,14 +1189,15 @@ void iwl_mvm_connection_loss(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 
 	trig = iwl_fw_dbg_get_trigger(mvm->fw, FW_DBG_TRIGGER_MLME);
 	trig_mlme = (void *)trig->data;
-	if (!iwl_fw_dbg_trigger_check_stop(mvm, vif, trig))
+	if (!iwl_fw_dbg_trigger_check_stop(&mvm->fwrt,
+					   ieee80211_vif_to_wdev(vif), trig))
 		goto out;
 
 	if (trig_mlme->stop_connection_loss &&
 	    --trig_mlme->stop_connection_loss)
 		goto out;
 
-	iwl_mvm_fw_dbg_collect_trig(mvm, trig, "%s", errmsg);
+	iwl_fw_dbg_collect_trig(&mvm->fwrt, trig, "%s", errmsg);
 
 out:
 	ieee80211_connection_loss(vif);
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/ctxt-info.c b/drivers/net/wireless/intel/iwlwifi/pcie/ctxt-info.c
index eddaca76d514..3fc4343581ee 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/ctxt-info.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/ctxt-info.c
@@ -244,7 +244,7 @@ int iwl_pcie_ctxt_info_init(struct iwl_trans *trans,
 	ctxt_info->hcmd_cfg.cmd_queue_addr =
 		cpu_to_le64(trans_pcie->txq[trans_pcie->cmd_queue]->dma_addr);
 	ctxt_info->hcmd_cfg.cmd_queue_size =
-		TFD_QUEUE_CB_SIZE(TFD_QUEUE_SIZE_MAX);
+		TFD_QUEUE_CB_SIZE(TFD_CMD_SLOTS);
 
 	/* allocate ucode sections in dram and set addresses */
 	ret = iwl_pcie_ctxt_info_init_fw_sec(trans, fw, ctxt_info);
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
index f16c1bb9bf94..858765fed8f8 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
@@ -430,6 +430,7 @@ static const struct pci_device_id iwl_hw_card_ids[] = {
 	{IWL_PCI_DEVICE(0x095B, 0x520A, iwl7265_2ac_cfg)},
 	{IWL_PCI_DEVICE(0x095A, 0x9000, iwl7265_2ac_cfg)},
 	{IWL_PCI_DEVICE(0x095A, 0x9400, iwl7265_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x095A, 0x9E10, iwl7265_2ac_cfg)},
 
 /* 8000 Series */
 	{IWL_PCI_DEVICE(0x24F3, 0x0010, iwl8260_2ac_cfg)},
@@ -510,9 +511,17 @@ static const struct pci_device_id iwl_hw_card_ids[] = {
 
 /* 9000 Series */
 	{IWL_PCI_DEVICE(0x271B, 0x0010, iwl9160_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x271B, 0x0014, iwl9160_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x271B, 0x0210, iwl9160_2ac_cfg)},
 	{IWL_PCI_DEVICE(0x2526, 0x0000, iwl9260_2ac_cfg)},
 	{IWL_PCI_DEVICE(0x2526, 0x0010, iwl9260_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x2526, 0x0014, iwl9260_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x2526, 0xA014, iwl9260_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x2526, 0x4010, iwl9260_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x2526, 0x0210, iwl9260_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x2526, 0x0214, iwl9260_2ac_cfg)},
 	{IWL_PCI_DEVICE(0x2526, 0x1410, iwl9270_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x2526, 0x1610, iwl9270_2ac_cfg)},
 	{IWL_PCI_DEVICE(0x9DF0, 0x0A10, iwl9460_2ac_cfg)},
 	{IWL_PCI_DEVICE(0x9DF0, 0x0010, iwl9460_2ac_cfg)},
 	{IWL_PCI_DEVICE(0x9DF0, 0x0210, iwl9460_2ac_cfg)},
@@ -527,10 +536,22 @@ static const struct pci_device_id iwl_hw_card_ids[] = {
 	{IWL_PCI_DEVICE(0x9DF0, 0x2A10, iwl9460_2ac_cfg)},
 	{IWL_PCI_DEVICE(0x30DC, 0x0060, iwl9460_2ac_cfg)},
 	{IWL_PCI_DEVICE(0x2526, 0x0060, iwl9460_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x2526, 0x0260, iwl9460_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x2526, 0x0064, iwl9460_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x2526, 0x00A4, iwl9460_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x2526, 0x40A4, iwl9460_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x2526, 0x02A4, iwl9460_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x2526, 0x00A0, iwl9460_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x2526, 0x02A0, iwl9460_2ac_cfg)},
 	{IWL_PCI_DEVICE(0x9DF0, 0x0060, iwl9460_2ac_cfg)},
 	{IWL_PCI_DEVICE(0xA370, 0x0060, iwl9460_2ac_cfg)},
 	{IWL_PCI_DEVICE(0x31DC, 0x0060, iwl9460_2ac_cfg)},
 	{IWL_PCI_DEVICE(0x2526, 0x0030, iwl9560_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x2526, 0x4030, iwl9560_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x2526, 0x0230, iwl9560_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x2526, 0x0234, iwl9560_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x2526, 0x0238, iwl9560_2ac_cfg)},
+	{IWL_PCI_DEVICE(0x2526, 0x023C, iwl9560_2ac_cfg)},
 	{IWL_PCI_DEVICE(0x9DF0, 0x0030, iwl9560_2ac_cfg)},
 	{IWL_PCI_DEVICE(0xA370, 0x0030, iwl9560_2ac_cfg)},
 	{IWL_PCI_DEVICE(0x31DC, 0x0030, iwl9560_2ac_cfg)},
@@ -690,12 +711,23 @@ static int iwl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		iwl_trans->cfg = cfg_7265d;
 	}
 
-	if (iwl_trans->cfg->rf_id && cfg == &iwla000_2ac_cfg_hr_cdb) {
-		if (iwl_trans->hw_rf_id == CSR_HW_RF_ID_TYPE_JF)
-			cfg = &iwla000_2ac_cfg_jf;
-		else if (iwl_trans->hw_rf_id == CSR_HW_RF_ID_TYPE_HR)
-			cfg = &iwla000_2ac_cfg_hr;
-
+	if (iwl_trans->cfg->rf_id && cfg == &iwla000_2ac_cfg_hr_cdb &&
+	    iwl_trans->hw_rev != CSR_HW_REV_TYPE_HR_CDB) {
+		u32 rf_id_chp = CSR_HW_RF_ID_TYPE_CHIP_ID(iwl_trans->hw_rf_id);
+		u32 jf_chp_id = CSR_HW_RF_ID_TYPE_CHIP_ID(CSR_HW_RF_ID_TYPE_JF);
+		u32 hr_chp_id = CSR_HW_RF_ID_TYPE_CHIP_ID(CSR_HW_RF_ID_TYPE_HR);
+
+		if (rf_id_chp == jf_chp_id) {
+			if (iwl_trans->hw_rev == CSR_HW_REV_TYPE_QNJ)
+				cfg = &iwla000_2ax_cfg_qnj_jf_b0;
+			else
+				cfg = &iwla000_2ac_cfg_jf;
+		} else if (rf_id_chp == hr_chp_id) {
+			if (iwl_trans->hw_rev == CSR_HW_REV_TYPE_QNJ)
+				cfg = &iwla000_2ax_cfg_qnj_hr_a0;
+			else
+				cfg = &iwla000_2ac_cfg_hr;
+		}
 		iwl_trans->cfg = cfg;
 	}
 #endif
@@ -805,11 +837,11 @@ static int iwl_pci_resume(struct device *device)
 	/*
 	 * Enable rfkill interrupt (in order to keep track of the rfkill
 	 * status). Must be locked to avoid processing a possible rfkill
-	 * interrupt while in iwl_trans_check_hw_rf_kill().
+	 * interrupt while in iwl_pcie_check_hw_rf_kill().
 	 */
 	mutex_lock(&trans_pcie->mutex);
 	iwl_enable_rfkill_int(trans);
-	iwl_trans_check_hw_rf_kill(trans);
+	iwl_pcie_check_hw_rf_kill(trans);
 	mutex_unlock(&trans_pcie->mutex);
 
 	return 0;
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/internal.h b/drivers/net/wireless/intel/iwlwifi/pcie/internal.h
index fa315d84e98e..4fb7647995c3 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/internal.h
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/internal.h
@@ -661,10 +661,16 @@ static inline void iwl_pcie_sw_reset(struct iwl_trans *trans)
 	usleep_range(5000, 6000);
 }
 
+static inline u8 iwl_pcie_get_cmd_index(struct iwl_txq *q, u32 index)
+{
+	return index & (q->n_window - 1);
+}
+
 static inline void *iwl_pcie_get_tfd(struct iwl_trans_pcie *trans_pcie,
 				     struct iwl_txq *txq, int idx)
 {
-	return txq->tfds + trans_pcie->tfd_size * idx;
+	return txq->tfds + trans_pcie->tfd_size * iwl_pcie_get_cmd_index(txq,
+									 idx);
 }
 
 static inline void iwl_enable_rfkill_int(struct iwl_trans *trans)
@@ -726,11 +732,6 @@ static inline bool iwl_queue_used(const struct iwl_txq *q, int i)
 		!(i < q->read_ptr && i >= q->write_ptr);
 }
 
-static inline u8 get_cmd_index(struct iwl_txq *q, u32 index)
-{
-	return index & (q->n_window - 1);
-}
-
 static inline bool iwl_is_rfkill_set(struct iwl_trans *trans)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
@@ -787,11 +788,13 @@ int iwl_pci_fw_enter_d0i3(struct iwl_trans *trans);
 
 void iwl_pcie_enable_rx_wake(struct iwl_trans *trans, bool enable);
 
+void iwl_pcie_rx_allocator_work(struct work_struct *data);
+
 /* common functions that are used by gen2 transport */
 void iwl_pcie_apm_config(struct iwl_trans *trans);
 int iwl_pcie_prepare_card_hw(struct iwl_trans *trans);
 void iwl_pcie_synchronize_irqs(struct iwl_trans *trans);
-bool iwl_trans_check_hw_rf_kill(struct iwl_trans *trans);
+bool iwl_pcie_check_hw_rf_kill(struct iwl_trans *trans);
 void iwl_trans_pcie_handle_stop_rfkill(struct iwl_trans *trans,
 				       bool was_in_rfkill);
 void iwl_pcie_txq_free_tfd(struct iwl_trans *trans, struct iwl_txq *txq);
@@ -806,6 +809,8 @@ int iwl_pcie_alloc_dma_ptr(struct iwl_trans *trans,
 			   struct iwl_dma_ptr *ptr, size_t size);
 void iwl_pcie_free_dma_ptr(struct iwl_trans *trans, struct iwl_dma_ptr *ptr);
 void iwl_pcie_apply_destination(struct iwl_trans *trans);
+void iwl_pcie_free_tso_page(struct iwl_trans_pcie *trans_pcie,
+			    struct sk_buff *skb);
 #ifdef CONFIG_INET
 struct iwl_tso_hdr_page *get_page_hdr(struct iwl_trans *trans, size_t len);
 #endif
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/rx.c b/drivers/net/wireless/intel/iwlwifi/pcie/rx.c
index 351c4423125a..a06b6612b658 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/rx.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/rx.c
@@ -597,7 +597,7 @@ static void iwl_pcie_rx_allocator_get(struct iwl_trans *trans,
 	rxq->free_count += RX_CLAIM_REQ_ALLOC;
 }
 
-static void iwl_pcie_rx_allocator_work(struct work_struct *data)
+void iwl_pcie_rx_allocator_work(struct work_struct *data)
 {
 	struct iwl_rb_allocator *rba_p =
 		container_of(data, struct iwl_rb_allocator, rx_alloc);
@@ -900,10 +900,6 @@ static int _iwl_pcie_rx_init(struct iwl_trans *trans)
 			return err;
 	}
 	def_rxq = trans_pcie->rxq;
-	if (!rba->alloc_wq)
-		rba->alloc_wq = alloc_workqueue("rb_allocator",
-						WQ_HIGHPRI | WQ_UNBOUND, 1);
-	INIT_WORK(&rba->rx_alloc, iwl_pcie_rx_allocator_work);
 
 	spin_lock(&rba->lock);
 	atomic_set(&rba->req_pending, 0);
@@ -1017,10 +1013,6 @@ void iwl_pcie_rx_free(struct iwl_trans *trans)
 	}
 
 	cancel_work_sync(&rba->rx_alloc);
-	if (rba->alloc_wq) {
-		destroy_workqueue(rba->alloc_wq);
-		rba->alloc_wq = NULL;
-	}
 
 	iwl_pcie_free_rbs_pool(trans);
 
@@ -1176,7 +1168,7 @@ static void iwl_pcie_rx_handle_rb(struct iwl_trans *trans,
 
 		sequence = le16_to_cpu(pkt->hdr.sequence);
 		index = SEQ_TO_INDEX(sequence);
-		cmd_index = get_cmd_index(txq, index);
+		cmd_index = iwl_pcie_get_cmd_index(txq, index);
 
 		if (rxq->id == 0)
 			iwl_op_mode_rx(trans->op_mode, &rxq->napi,
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c b/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c
index b84b78293e7b..c59f4581e972 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c
@@ -307,7 +307,7 @@ int iwl_trans_pcie_gen2_start_fw(struct iwl_trans *trans,
 	mutex_lock(&trans_pcie->mutex);
 
 	/* If platform's RF_KILL switch is NOT set to KILL */
-	hw_rfkill = iwl_trans_check_hw_rf_kill(trans);
+	hw_rfkill = iwl_pcie_check_hw_rf_kill(trans);
 	if (hw_rfkill && !run_in_rfkill) {
 		ret = -ERFKILL;
 		goto out;
@@ -340,7 +340,7 @@ int iwl_trans_pcie_gen2_start_fw(struct iwl_trans *trans,
 		goto out;
 
 	/* re-check RF-Kill state since we may have missed the interrupt */
-	hw_rfkill = iwl_trans_check_hw_rf_kill(trans);
+	hw_rfkill = iwl_pcie_check_hw_rf_kill(trans);
 	if (hw_rfkill && !run_in_rfkill)
 		ret = -ERFKILL;
 
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c
index f95eec52508e..2e3e013ec95a 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c
@@ -986,7 +986,7 @@ static int iwl_pcie_load_given_ucode_8000(struct iwl_trans *trans,
 					       &first_ucode_section);
 }
 
-bool iwl_trans_check_hw_rf_kill(struct iwl_trans *trans)
+bool iwl_pcie_check_hw_rf_kill(struct iwl_trans *trans)
 {
 	struct iwl_trans_pcie *trans_pcie =  IWL_TRANS_GET_PCIE_TRANS(trans);
 	bool hw_rfkill = iwl_is_rfkill_set(trans);
@@ -1252,7 +1252,7 @@ static int iwl_trans_pcie_start_fw(struct iwl_trans *trans,
 	mutex_lock(&trans_pcie->mutex);
 
 	/* If platform's RF_KILL switch is NOT set to KILL */
-	hw_rfkill = iwl_trans_check_hw_rf_kill(trans);
+	hw_rfkill = iwl_pcie_check_hw_rf_kill(trans);
 	if (hw_rfkill && !run_in_rfkill) {
 		ret = -ERFKILL;
 		goto out;
@@ -1300,7 +1300,7 @@ static int iwl_trans_pcie_start_fw(struct iwl_trans *trans,
 		ret = iwl_pcie_load_given_ucode(trans, fw);
 
 	/* re-check RF-Kill state since we may have missed the interrupt */
-	hw_rfkill = iwl_trans_check_hw_rf_kill(trans);
+	hw_rfkill = iwl_pcie_check_hw_rf_kill(trans);
 	if (hw_rfkill && !run_in_rfkill)
 		ret = -ERFKILL;
 
@@ -1663,7 +1663,7 @@ static int _iwl_trans_pcie_start_hw(struct iwl_trans *trans, bool low_power)
 	trans_pcie->is_down = false;
 
 	/* ...rfkill can call stop_device and set it false if needed */
-	iwl_trans_check_hw_rf_kill(trans);
+	iwl_pcie_check_hw_rf_kill(trans);
 
 	/* Make sure we sync here, because we'll need full access later */
 	if (low_power)
@@ -1786,6 +1786,11 @@ void iwl_trans_pcie_free(struct iwl_trans *trans)
 		iwl_pcie_tx_free(trans);
 	iwl_pcie_rx_free(trans);
 
+	if (trans_pcie->rba.alloc_wq) {
+		destroy_workqueue(trans_pcie->rba.alloc_wq);
+		trans_pcie->rba.alloc_wq = NULL;
+	}
+
 	if (trans_pcie->msix_enabled) {
 		for (i = 0; i < trans_pcie->alloc_vecs; i++) {
 			irq_set_affinity_hint(
@@ -1842,8 +1847,8 @@ static bool iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans,
 	 * These bits say the device is running, and should keep running for
 	 * at least a short while (at least as long as MAC_ACCESS_REQ stays 1),
 	 * but they do not indicate that embedded SRAM is restored yet;
-	 * 3945 and 4965 have volatile SRAM, and must save/restore contents
-	 * to/from host DRAM when sleeping/waking for power-saving.
+	 * HW with volatile SRAM must save/restore contents to/from
+	 * host DRAM when sleeping/waking for power-saving.
 	 * Each direction takes approximately 1/4 millisecond; with this
 	 * overhead, it's a good idea to grab and hold MAC_ACCESS_REQUEST if a
 	 * series of register accesses are expected (e.g. reading Event Log),
@@ -1851,8 +1856,9 @@ static bool iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans,
 	 *
 	 * CSR_UCODE_DRV_GP1 register bit MAC_SLEEP == 0 indicates that
 	 * SRAM is okay/restored.  We don't check that here because this call
-	 * is just for hardware register access; but GP1 MAC_SLEEP check is a
-	 * good idea before accessing 3945/4965 SRAM (e.g. reading Event Log).
+	 * is just for hardware register access; but GP1 MAC_SLEEP
+	 * check is a good idea before accessing the SRAM of HW with
+	 * volatile SRAM (e.g. reading Event Log).
 	 *
 	 * 5000 series and later (including 1000 series) have non-volatile SRAM,
 	 * and do not save/restore SRAM when power cycling.
@@ -2834,7 +2840,7 @@ static struct iwl_trans_dump_data
 	spin_lock_bh(&cmdq->lock);
 	ptr = cmdq->write_ptr;
 	for (i = 0; i < cmdq->n_window; i++) {
-		u8 idx = get_cmd_index(cmdq, ptr);
+		u8 idx = iwl_pcie_get_cmd_index(cmdq, ptr);
 		u32 caplen, cmdlen;
 
 		cmdlen = iwl_trans_pcie_get_cmdlen(trans, cmdq->tfds +
@@ -3137,7 +3143,18 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
 		iwl_set_bit(trans, CSR_HOST_CHICKEN,
 			    CSR_HOST_CHICKEN_PM_IDLE_SRC_DIS_SB_PME);
 
+#if IS_ENABLED(CONFIG_IWLMVM)
 	trans->hw_rf_id = iwl_read32(trans, CSR_HW_RF_ID);
+	if (trans->hw_rf_id == CSR_HW_RF_ID_TYPE_HR) {
+		u32 hw_status;
+
+		hw_status = iwl_read_prph(trans, UMAG_GEN_HW_STATUS);
+		if (hw_status & UMAG_GEN_HW_IS_FPGA)
+			trans->cfg = &iwla000_2ax_cfg_qnj_hr_f0;
+		else
+			trans->cfg = &iwla000_2ac_cfg_hr;
+	}
+#endif
 
 	iwl_pcie_set_interrupt_capa(pdev, trans);
 	trans->hw_id = (pdev->device << 16) + pdev->subsystem_device;
@@ -3169,6 +3186,10 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
 		trans_pcie->inta_mask = CSR_INI_SET_MASK;
 	 }
 
+	trans_pcie->rba.alloc_wq = alloc_workqueue("rb_allocator",
+						   WQ_HIGHPRI | WQ_UNBOUND, 1);
+	INIT_WORK(&trans_pcie->rba.rx_alloc, iwl_pcie_rx_allocator_work);
+
 #ifdef CONFIG_IWLWIFI_PCIE_RTPM
 	trans->runtime_pm_mode = IWL_PLAT_PM_MODE_D0I3;
 #else
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/tx-gen2.c b/drivers/net/wireless/intel/iwlwifi/pcie/tx-gen2.c
index a3795ba0d7b9..d74613fcb756 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/tx-gen2.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/tx-gen2.c
@@ -55,7 +55,7 @@
 #include "iwl-csr.h"
 #include "iwl-io.h"
 #include "internal.h"
-#include "mvm/fw-api.h"
+#include "fw/api/tx.h"
 
  /*
  * iwl_pcie_gen2_tx_stop - Stop all Tx DMA channels
@@ -88,14 +88,14 @@ static void iwl_pcie_gen2_update_byte_tbl(struct iwl_txq *txq, u16 byte_cnt,
 					  int num_tbs)
 {
 	struct iwlagn_scd_bc_tbl *scd_bc_tbl = txq->bc_tbl.addr;
-	int write_ptr = txq->write_ptr;
+	int idx = iwl_pcie_get_cmd_index(txq, txq->write_ptr);
 	u8 filled_tfd_size, num_fetch_chunks;
 	u16 len = byte_cnt;
 	__le16 bc_ent;
 
 	len = DIV_ROUND_UP(len, 4);
 
-	if (WARN_ON(len > 0xFFF || write_ptr >= TFD_QUEUE_SIZE_MAX))
+	if (WARN_ON(len > 0xFFF || idx >= txq->n_window))
 		return;
 
 	filled_tfd_size = offsetof(struct iwl_tfh_tfd, tbs) +
@@ -111,7 +111,7 @@ static void iwl_pcie_gen2_update_byte_tbl(struct iwl_txq *txq, u16 byte_cnt,
 	num_fetch_chunks = DIV_ROUND_UP(filled_tfd_size, 64) - 1;
 
 	bc_ent = cpu_to_le16(len | (num_fetch_chunks << 12));
-	scd_bc_tbl->tfd_offset[write_ptr] = bc_ent;
+	scd_bc_tbl->tfd_offset[idx] = bc_ent;
 }
 
 /*
@@ -176,16 +176,12 @@ static void iwl_pcie_gen2_free_tfd(struct iwl_trans *trans, struct iwl_txq *txq)
 	/* rd_ptr is bounded by TFD_QUEUE_SIZE_MAX and
 	 * idx is bounded by n_window
 	 */
-	int rd_ptr = txq->read_ptr;
-	int idx = get_cmd_index(txq, rd_ptr);
+	int idx = iwl_pcie_get_cmd_index(txq, txq->read_ptr);
 
 	lockdep_assert_held(&txq->lock);
 
-	/* We have only q->n_window txq->entries, but we use
-	 * TFD_QUEUE_SIZE_MAX tfds
-	 */
 	iwl_pcie_gen2_tfd_unmap(trans, &txq->entries[idx].meta,
-				iwl_pcie_get_tfd(trans_pcie, txq, rd_ptr));
+				iwl_pcie_get_tfd(trans_pcie, txq, idx));
 
 	/* free SKB */
 	if (txq->entries) {
@@ -373,8 +369,9 @@ struct iwl_tfh_tfd *iwl_pcie_gen2_build_tfd(struct iwl_trans *trans,
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
+	int idx = iwl_pcie_get_cmd_index(txq, txq->write_ptr);
 	struct iwl_tfh_tfd *tfd =
-		iwl_pcie_get_tfd(trans_pcie, txq, txq->write_ptr);
+		iwl_pcie_get_tfd(trans_pcie, txq, idx);
 	dma_addr_t tb_phys;
 	bool amsdu;
 	int i, len, tb1_len, tb2_len, hdr_len;
@@ -386,10 +383,10 @@ struct iwl_tfh_tfd *iwl_pcie_gen2_build_tfd(struct iwl_trans *trans,
 		(*ieee80211_get_qos_ctl(hdr) &
 		 IEEE80211_QOS_CTL_A_MSDU_PRESENT);
 
-	tb_phys = iwl_pcie_get_first_tb_dma(txq, txq->write_ptr);
+	tb_phys = iwl_pcie_get_first_tb_dma(txq, idx);
 	/* The first TB points to bi-directional DMA data */
 	if (!amsdu)
-		memcpy(&txq->first_tb_bufs[txq->write_ptr], &dev_cmd->hdr,
+		memcpy(&txq->first_tb_bufs[idx], &dev_cmd->hdr,
 		       IWL_FIRST_TB_SIZE);
 
 	iwl_pcie_gen2_set_tb(trans, tfd, tb_phys, IWL_FIRST_TB_SIZE);
@@ -422,16 +419,16 @@ struct iwl_tfh_tfd *iwl_pcie_gen2_build_tfd(struct iwl_trans *trans,
 	hdr_len = ieee80211_hdrlen(hdr->frame_control);
 
 	if (amsdu) {
-		if (!iwl_pcie_gen2_build_amsdu(trans, skb, tfd,
-					       tb1_len + IWL_FIRST_TB_SIZE,
-					       hdr_len, dev_cmd))
+		if (iwl_pcie_gen2_build_amsdu(trans, skb, tfd,
+					      tb1_len + IWL_FIRST_TB_SIZE,
+					      hdr_len, dev_cmd))
 			goto out_err;
 
 		/*
 		 * building the A-MSDU might have changed this data, so memcpy
 		 * it now
 		 */
-		memcpy(&txq->first_tb_bufs[txq->write_ptr], &dev_cmd->hdr,
+		memcpy(&txq->first_tb_bufs[idx], &dev_cmd->hdr,
 		       IWL_FIRST_TB_SIZE);
 		return tfd;
 	}
@@ -484,6 +481,7 @@ int iwl_trans_pcie_gen2_tx(struct iwl_trans *trans, struct sk_buff *skb,
 	struct iwl_tx_cmd_gen2 *tx_cmd = (void *)dev_cmd->payload;
 	struct iwl_cmd_meta *out_meta;
 	struct iwl_txq *txq = trans_pcie->txq[txq_id];
+	int idx;
 	void *tfd;
 
 	if (WARN_ONCE(!test_bit(txq_id, trans_pcie->queue_used),
@@ -497,16 +495,18 @@ int iwl_trans_pcie_gen2_tx(struct iwl_trans *trans, struct sk_buff *skb,
 
 	spin_lock(&txq->lock);
 
+	idx = iwl_pcie_get_cmd_index(txq, txq->write_ptr);
+
 	/* Set up driver data for this TFD */
-	txq->entries[txq->write_ptr].skb = skb;
-	txq->entries[txq->write_ptr].cmd = dev_cmd;
+	txq->entries[idx].skb = skb;
+	txq->entries[idx].cmd = dev_cmd;
 
 	dev_cmd->hdr.sequence =
 		cpu_to_le16((u16)(QUEUE_TO_SEQ(txq_id) |
-			    INDEX_TO_SEQ(txq->write_ptr)));
+			    INDEX_TO_SEQ(idx)));
 
 	/* Set up first empty entry in queue's array of Tx/cmd buffers */
-	out_meta = &txq->entries[txq->write_ptr].meta;
+	out_meta = &txq->entries[idx].meta;
 	out_meta->flags = 0;
 
 	tfd = iwl_pcie_gen2_build_tfd(trans, txq, dev_cmd, skb, out_meta);
@@ -562,7 +562,7 @@ static int iwl_pcie_gen2_enqueue_hcmd(struct iwl_trans *trans,
 	unsigned long flags;
 	void *dup_buf = NULL;
 	dma_addr_t phys_addr;
-	int idx, i, cmd_pos;
+	int i, cmd_pos, idx = iwl_pcie_get_cmd_index(txq, txq->write_ptr);
 	u16 copy_size, cmd_size, tb0_size;
 	bool had_nocopy = false;
 	u8 group_id = iwl_cmd_groupid(cmd->id);
@@ -651,7 +651,6 @@ static int iwl_pcie_gen2_enqueue_hcmd(struct iwl_trans *trans,
 		goto free_dup_buf;
 	}
 
-	idx = get_cmd_index(txq, txq->write_ptr);
 	out_cmd = txq->entries[idx].cmd;
 	out_meta = &txq->entries[idx].meta;
 
@@ -937,6 +936,15 @@ void iwl_pcie_gen2_txq_unmap(struct iwl_trans *trans, int txq_id)
 		IWL_DEBUG_TX_REPLY(trans, "Q %d Free %d\n",
 				   txq_id, txq->read_ptr);
 
+		if (txq_id != trans_pcie->cmd_queue) {
+			int idx = iwl_pcie_get_cmd_index(txq, txq->read_ptr);
+			struct sk_buff *skb = txq->entries[idx].skb;
+
+			if (WARN_ON_ONCE(!skb))
+				continue;
+
+			iwl_pcie_free_tso_page(trans_pcie, skb);
+		}
 		iwl_pcie_gen2_free_tfd(trans, txq);
 		txq->read_ptr = iwl_queue_inc_wrap(txq->read_ptr);
 
@@ -1033,6 +1041,7 @@ int iwl_trans_pcie_dyn_txq_alloc(struct iwl_trans *trans,
 		.flags = CMD_WANT_SKB,
 	};
 	int ret, qid;
+	u32 wr_ptr;
 
 	txq = kzalloc(sizeof(*txq), GFP_KERNEL);
 	if (!txq)
@@ -1060,7 +1069,7 @@ int iwl_trans_pcie_dyn_txq_alloc(struct iwl_trans *trans,
 
 	cmd->tfdq_addr = cpu_to_le64(txq->dma_addr);
 	cmd->byte_cnt_addr = cpu_to_le64(txq->bc_tbl.dma);
-	cmd->cb_size = cpu_to_le32(TFD_QUEUE_CB_SIZE(TFD_QUEUE_SIZE_MAX));
+	cmd->cb_size = cpu_to_le32(TFD_QUEUE_CB_SIZE(TFD_TX_CMD_SLOTS));
 
 	ret = iwl_trans_send_cmd(trans, &hcmd);
 	if (ret)
@@ -1073,6 +1082,7 @@ int iwl_trans_pcie_dyn_txq_alloc(struct iwl_trans *trans,
 
 	rsp = (void *)hcmd.resp_pkt->data;
 	qid = le16_to_cpu(rsp->queue_number);
+	wr_ptr = le16_to_cpu(rsp->write_pointer);
 
 	if (qid >= ARRAY_SIZE(trans_pcie->txq)) {
 		WARN_ONCE(1, "queue index %d unsupported", qid);
@@ -1088,10 +1098,11 @@ int iwl_trans_pcie_dyn_txq_alloc(struct iwl_trans *trans,
 
 	txq->id = qid;
 	trans_pcie->txq[qid] = txq;
+	wr_ptr &= (TFD_QUEUE_SIZE_MAX - 1);
 
 	/* Place first TFD at index corresponding to start sequence number */
-	txq->read_ptr = le16_to_cpu(rsp->write_pointer);
-	txq->write_ptr = le16_to_cpu(rsp->write_pointer);
+	txq->read_ptr = wr_ptr;
+	txq->write_ptr = wr_ptr;
 	iwl_write_direct32(trans, HBUS_TARG_WRPTR,
 			   (txq->write_ptr) | (qid << 16));
 	IWL_DEBUG_TX_QUEUES(trans, "Activate queue %d\n", qid);
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/tx.c b/drivers/net/wireless/intel/iwlwifi/pcie/tx.c
index 034bdb4a0b06..c645d10d3707 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/tx.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/tx.c
@@ -43,8 +43,7 @@
 #include "iwl-scd.h"
 #include "iwl-op-mode.h"
 #include "internal.h"
-/* FIXME: need to abstract out TX command (once we know what it looks like) */
-#include "dvm/commands.h"
+#include "fw/api/tx.h"
 
 #define IWL_TX_CRC_SIZE 4
 #define IWL_TX_DELIMITER_SIZE 4
@@ -107,7 +106,7 @@ static int iwl_queue_init(struct iwl_txq *q, int slots_num)
 	q->n_window = slots_num;
 
 	/* slots_num must be power-of-two size, otherwise
-	 * get_cmd_index is broken. */
+	 * iwl_pcie_get_cmd_index is broken. */
 	if (WARN_ON(!is_power_of_2(slots_num)))
 		return -EINVAL;
 
@@ -429,7 +428,7 @@ void iwl_pcie_txq_free_tfd(struct iwl_trans *trans, struct iwl_txq *txq)
 	 * idx is bounded by n_window
 	 */
 	int rd_ptr = txq->read_ptr;
-	int idx = get_cmd_index(txq, rd_ptr);
+	int idx = iwl_pcie_get_cmd_index(txq, rd_ptr);
 
 	lockdep_assert_held(&txq->lock);
 
@@ -578,8 +577,8 @@ int iwl_pcie_txq_init(struct iwl_trans *trans, struct iwl_txq *txq,
 	return 0;
 }
 
-static void iwl_pcie_free_tso_page(struct iwl_trans_pcie *trans_pcie,
-				   struct sk_buff *skb)
+void iwl_pcie_free_tso_page(struct iwl_trans_pcie *trans_pcie,
+			    struct sk_buff *skb)
 {
 	struct page **page_ptr;
 
@@ -1101,7 +1100,8 @@ void iwl_trans_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn,
 	for (;
 	     txq->read_ptr != tfd_num;
 	     txq->read_ptr = iwl_queue_inc_wrap(txq->read_ptr)) {
-		struct sk_buff *skb = txq->entries[txq->read_ptr].skb;
+		int idx = iwl_pcie_get_cmd_index(txq, txq->read_ptr);
+		struct sk_buff *skb = txq->entries[idx].skb;
 
 		if (WARN_ON_ONCE(!skb))
 			continue;
@@ -1110,7 +1110,7 @@ void iwl_trans_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn,
 
 		__skb_queue_tail(skbs, skb);
 
-		txq->entries[txq->read_ptr].skb = NULL;
+		txq->entries[idx].skb = NULL;
 
 		if (!trans->cfg->use_tfh)
 			iwl_pcie_txq_inval_byte_cnt_tbl(trans, txq);
@@ -1560,7 +1560,7 @@ static int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans,
 		goto free_dup_buf;
 	}
 
-	idx = get_cmd_index(txq, txq->write_ptr);
+	idx = iwl_pcie_get_cmd_index(txq, txq->write_ptr);
 	out_cmd = txq->entries[idx].cmd;
 	out_meta = &txq->entries[idx].meta;
 
@@ -1752,7 +1752,7 @@ void iwl_pcie_hcmd_complete(struct iwl_trans *trans,
 
 	spin_lock_bh(&txq->lock);
 
-	cmd_index = get_cmd_index(txq, index);
+	cmd_index = iwl_pcie_get_cmd_index(txq, index);
 	cmd = txq->entries[cmd_index].cmd;
 	meta = &txq->entries[cmd_index].meta;
 	group_id = cmd->hdr.group_id;
@@ -2370,7 +2370,7 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
 		tb1_len = ALIGN(len, 4);
 		/* Tell NIC about any 2-byte padding after MAC header */
 		if (tb1_len != len)
-			tx_cmd->tx_flags |= TX_CMD_FLG_MH_PAD_MSK;
+			tx_cmd->tx_flags |= cpu_to_le32(TX_CMD_FLG_MH_PAD);
 	} else {
 		tb1_len = len;
 	}
diff --git a/drivers/net/wireless/intersil/hostap/hostap_main.c b/drivers/net/wireless/intersil/hostap/hostap_main.c
index a3c066f90afc..012930d35434 100644
--- a/drivers/net/wireless/intersil/hostap/hostap_main.c
+++ b/drivers/net/wireless/intersil/hostap/hostap_main.c
@@ -125,8 +125,8 @@ void hostap_remove_interface(struct net_device *dev, int rtnl_locked,
 	else
 		unregister_netdev(dev);
 
-	/* dev->destructor = free_netdev() will free the device data, including
-	 * private data, when removing the device */
+	/* 'dev->needs_free_netdev = true' implies device data, including
+	 * private data, will be freed when the device is removed */
 }
 
 
diff --git a/drivers/net/wireless/intersil/orinoco/orinoco_usb.c b/drivers/net/wireless/intersil/orinoco/orinoco_usb.c
index c84fd8490601..56f6e3b71f48 100644
--- a/drivers/net/wireless/intersil/orinoco/orinoco_usb.c
+++ b/drivers/net/wireless/intersil/orinoco/orinoco_usb.c
@@ -210,7 +210,7 @@ struct ezusb_packet {
 } __packed;
 
 /* Table of devices that work or may work with this driver */
-static struct usb_device_id ezusb_table[] = {
+static const struct usb_device_id ezusb_table[] = {
 	{USB_DEVICE(USB_COMPAQ_VENDOR_ID, USB_COMPAQ_WL215_ID)},
 	{USB_DEVICE(USB_COMPAQ_VENDOR_ID, USB_HP_WL215_ID)},
 	{USB_DEVICE(USB_COMPAQ_VENDOR_ID, USB_COMPAQ_W200_ID)},
diff --git a/drivers/net/wireless/intersil/p54/p54usb.c b/drivers/net/wireless/intersil/p54/p54usb.c
index 043bd1c23c19..b0b86f701061 100644
--- a/drivers/net/wireless/intersil/p54/p54usb.c
+++ b/drivers/net/wireless/intersil/p54/p54usb.c
@@ -41,7 +41,7 @@ MODULE_FIRMWARE("isl3887usb");
  * whenever you add a new device.
  */
 
-static struct usb_device_id p54u_table[] = {
+static const struct usb_device_id p54u_table[] = {
 	/* Version 1 devices (pci chip + net2280) */
 	{USB_DEVICE(0x0411, 0x0050)},	/* Buffalo WLI2-USB2-G54 */
 	{USB_DEVICE(0x045e, 0x00c2)},	/* Microsoft MN-710 */
diff --git a/drivers/net/wireless/mac80211_hwsim.c b/drivers/net/wireless/mac80211_hwsim.c
index c8852acc1462..6467ffac9811 100644
--- a/drivers/net/wireless/mac80211_hwsim.c
+++ b/drivers/net/wireless/mac80211_hwsim.c
@@ -1362,8 +1362,6 @@ static void mac80211_hwsim_tx(struct ieee80211_hw *hw,
 				       txi->control.rates,
 				       ARRAY_SIZE(txi->control.rates));
 
-	txi->rate_driver_data[0] = channel;
-
 	if (skb->len >= 24 + 8 &&
 	    ieee80211_is_probe_resp(hdr->frame_control)) {
 		/* fake header transmission time */
diff --git a/drivers/net/wireless/marvell/libertas/if_usb.c b/drivers/net/wireless/marvell/libertas/if_usb.c
index e53025ea6689..16e54c757dd0 100644
--- a/drivers/net/wireless/marvell/libertas/if_usb.c
+++ b/drivers/net/wireless/marvell/libertas/if_usb.c
@@ -52,7 +52,7 @@ static const struct lbs_fw_table fw_table[] = {
 	{ MODEL_8682, "libertas/usb8682.bin", NULL }
 };
 
-static struct usb_device_id if_usb_table[] = {
+static const struct usb_device_id if_usb_table[] = {
 	/* Enter the device signature inside */
 	{ USB_DEVICE(0x1286, 0x2001), .driver_info = MODEL_8388 },
 	{ USB_DEVICE(0x05a3, 0x8388), .driver_info = MODEL_8388 },
diff --git a/drivers/net/wireless/marvell/libertas_tf/if_usb.c b/drivers/net/wireless/marvell/libertas_tf/if_usb.c
index e0ade40d9497..e9104eca327b 100644
--- a/drivers/net/wireless/marvell/libertas_tf/if_usb.c
+++ b/drivers/net/wireless/marvell/libertas_tf/if_usb.c
@@ -31,7 +31,7 @@ module_param_named(fw_name, lbtf_fw_name, charp, 0644);
 
 MODULE_FIRMWARE("lbtf_usb.bin");
 
-static struct usb_device_id if_usb_table[] = {
+static const struct usb_device_id if_usb_table[] = {
 	/* Enter the device signature inside */
 	{ USB_DEVICE(0x1286, 0x2001) },
 	{ USB_DEVICE(0x05a3, 0x8388) },
diff --git a/drivers/net/wireless/marvell/mwifiex/11n.c b/drivers/net/wireless/marvell/mwifiex/11n.c
index 16c77c27f1b6..725206914911 100644
--- a/drivers/net/wireless/marvell/mwifiex/11n.c
+++ b/drivers/net/wireless/marvell/mwifiex/11n.c
@@ -572,6 +572,8 @@ int mwifiex_send_addba(struct mwifiex_private *priv, int tid, u8 *peer_mac)
 
 	mwifiex_dbg(priv->adapter, CMD, "cmd: %s: tid %d\n", __func__, tid);
 
+	memset(&add_ba_req, 0, sizeof(add_ba_req));
+
 	if ((GET_BSS_ROLE(priv) == MWIFIEX_BSS_ROLE_STA) &&
 	    ISSUPP_TDLS_ENABLED(priv->adapter->fw_cap_info) &&
 	    priv->adapter->is_hw_11ac_capable &&
diff --git a/drivers/net/wireless/marvell/mwifiex/cfg80211.c b/drivers/net/wireless/marvell/mwifiex/cfg80211.c
index 06ad2d50f9b0..32c5074da84c 100644
--- a/drivers/net/wireless/marvell/mwifiex/cfg80211.c
+++ b/drivers/net/wireless/marvell/mwifiex/cfg80211.c
@@ -889,23 +889,15 @@ mwifiex_init_new_priv_params(struct mwifiex_private *priv,
 	switch (type) {
 	case NL80211_IFTYPE_STATION:
 	case NL80211_IFTYPE_ADHOC:
-		priv->bss_num = mwifiex_get_unused_bss_num(adapter,
-			 MWIFIEX_BSS_TYPE_STA);
 		priv->bss_role =  MWIFIEX_BSS_ROLE_STA;
 		break;
 	case NL80211_IFTYPE_P2P_CLIENT:
-		priv->bss_num = mwifiex_get_unused_bss_num(adapter,
-			 MWIFIEX_BSS_TYPE_P2P);
 		priv->bss_role =  MWIFIEX_BSS_ROLE_STA;
 		break;
 	case NL80211_IFTYPE_P2P_GO:
-		priv->bss_num = mwifiex_get_unused_bss_num(adapter,
-			 MWIFIEX_BSS_TYPE_P2P);
 		priv->bss_role =  MWIFIEX_BSS_ROLE_UAP;
 		break;
 	case NL80211_IFTYPE_AP:
-		priv->bss_num = mwifiex_get_unused_bss_num(adapter,
-			 MWIFIEX_BSS_TYPE_UAP);
 		priv->bss_role = MWIFIEX_BSS_ROLE_UAP;
 		break;
 	default:
@@ -923,6 +915,8 @@ mwifiex_init_new_priv_params(struct mwifiex_private *priv,
 	adapter->rx_locked = false;
 	spin_unlock_irqrestore(&adapter->rx_proc_lock, flags);
 
+	mwifiex_set_mac_address(priv, dev);
+
 	return 0;
 }
 
@@ -2012,6 +2006,8 @@ static int mwifiex_cfg80211_start_ap(struct wiphy *wiphy,
 		priv->state_11h.is_11h_active = false;
 	}
 
+	mwifiex_config_uap_11d(priv, &params->beacon);
+
 	if (mwifiex_config_start_uap(priv, bss_cfg)) {
 		mwifiex_dbg(priv->adapter, ERROR,
 			    "Failed to start AP\n");
@@ -2963,6 +2959,8 @@ struct wireless_dev *mwifiex_add_virtual_intf(struct wiphy *wiphy,
 	}
 
 	mwifiex_init_priv_params(priv, dev);
+	mwifiex_set_mac_address(priv, dev);
+
 	priv->netdev = dev;
 
 	ret = mwifiex_send_cmd(priv, HostCmd_CMD_SET_BSS_MODE,
@@ -2990,7 +2988,6 @@ struct wireless_dev *mwifiex_add_virtual_intf(struct wiphy *wiphy,
 	dev_net_set(dev, wiphy_net(wiphy));
 	dev->ieee80211_ptr = &priv->wdev;
 	dev->ieee80211_ptr->iftype = priv->bss_mode;
-	memcpy(dev->dev_addr, wiphy->perm_addr, ETH_ALEN);
 	SET_NETDEV_DEV(dev, wiphy_dev(wiphy));
 
 	dev->flags |= IFF_BROADCAST | IFF_MULTICAST;
@@ -3123,11 +3120,7 @@ int mwifiex_del_virtual_intf(struct wiphy *wiphy, struct wireless_dev *wdev)
 		priv->dfs_chan_sw_workqueue = NULL;
 	}
 	/* Clear the priv in adapter */
-	priv->netdev->ieee80211_ptr = NULL;
 	priv->netdev = NULL;
-	priv->wdev.iftype = NL80211_IFTYPE_UNSPECIFIED;
-
-	priv->media_connected = false;
 
 	switch (priv->bss_mode) {
 	case NL80211_IFTYPE_UNSPECIFIED:
@@ -3395,11 +3388,8 @@ static int mwifiex_cfg80211_suspend(struct wiphy *wiphy,
 
 	for (i = 0; i < adapter->priv_num; i++) {
 		priv = adapter->priv[i];
-		if (priv && priv->netdev) {
-			mwifiex_stop_net_dev_queue(priv->netdev, adapter);
-			if (netif_carrier_ok(priv->netdev))
-				netif_carrier_off(priv->netdev);
-		}
+		if (priv && priv->netdev)
+			netif_device_detach(priv->netdev);
 	}
 
 	for (i = 0; i < retry_num; i++) {
@@ -3470,11 +3460,8 @@ static int mwifiex_cfg80211_resume(struct wiphy *wiphy)
 
 	for (i = 0; i < adapter->priv_num; i++) {
 		priv = adapter->priv[i];
-		if (priv && priv->netdev) {
-			if (!netif_carrier_ok(priv->netdev))
-				netif_carrier_on(priv->netdev);
-			mwifiex_wake_up_net_dev_queue(priv->netdev, adapter);
-		}
+		if (priv && priv->netdev)
+			netif_device_attach(priv->netdev);
 	}
 
 	if (!wiphy->wowlan_config)
@@ -4215,7 +4202,7 @@ int mwifiex_init_channel_scan_gap(struct mwifiex_adapter *adapter)
 	if (adapter->config_bands & BAND_A)
 		n_channels_a = mwifiex_band_5ghz.n_channels;
 
-	adapter->num_in_chan_stats = max_t(u32, n_channels_bg, n_channels_a);
+	adapter->num_in_chan_stats = n_channels_bg + n_channels_a;
 	adapter->chan_stats = vmalloc(sizeof(*adapter->chan_stats) *
 				      adapter->num_in_chan_stats);
 
diff --git a/drivers/net/wireless/marvell/mwifiex/cfp.c b/drivers/net/wireless/marvell/mwifiex/cfp.c
index 6e2994308526..bfe84e55df77 100644
--- a/drivers/net/wireless/marvell/mwifiex/cfp.c
+++ b/drivers/net/wireless/marvell/mwifiex/cfp.c
@@ -180,11 +180,9 @@ static struct region_code_mapping region_code_mapping_t[] = {
 u8 *mwifiex_11d_code_2_region(u8 code)
 {
 	u8 i;
-	u8 size = sizeof(region_code_mapping_t)/
-				sizeof(struct region_code_mapping);
 
 	/* Look for code in mapping table */
-	for (i = 0; i < size; i++)
+	for (i = 0; i < ARRAY_SIZE(region_code_mapping_t); i++)
 		if (region_code_mapping_t[i].code == code)
 			return region_code_mapping_t[i].region;
 
diff --git a/drivers/net/wireless/marvell/mwifiex/cmdevt.c b/drivers/net/wireless/marvell/mwifiex/cmdevt.c
index 8dad52886034..0edc5d621304 100644
--- a/drivers/net/wireless/marvell/mwifiex/cmdevt.c
+++ b/drivers/net/wireless/marvell/mwifiex/cmdevt.c
@@ -26,6 +26,8 @@
 #include "11n.h"
 #include "11ac.h"
 
+static void mwifiex_cancel_pending_ioctl(struct mwifiex_adapter *adapter);
+
 /*
  * This function initializes a command node.
  *
@@ -427,7 +429,7 @@ int mwifiex_alloc_cmd_buffer(struct mwifiex_adapter *adapter)
  * The function calls the completion callback for all the command
  * buffers that still have response buffers associated with them.
  */
-int mwifiex_free_cmd_buffer(struct mwifiex_adapter *adapter)
+void mwifiex_free_cmd_buffer(struct mwifiex_adapter *adapter)
 {
 	struct cmd_ctrl_node *cmd_array;
 	u32 i;
@@ -436,7 +438,7 @@ int mwifiex_free_cmd_buffer(struct mwifiex_adapter *adapter)
 	if (!adapter->cmd_pool) {
 		mwifiex_dbg(adapter, FATAL,
 			    "info: FREE_CMD_BUF: cmd_pool is null\n");
-		return 0;
+		return;
 	}
 
 	cmd_array = adapter->cmd_pool;
@@ -464,8 +466,6 @@ int mwifiex_free_cmd_buffer(struct mwifiex_adapter *adapter)
 		kfree(adapter->cmd_pool);
 		adapter->cmd_pool = NULL;
 	}
-
-	return 0;
 }
 
 /*
@@ -666,7 +666,7 @@ int mwifiex_send_cmd(struct mwifiex_private *priv, u16 cmd_no,
 	    cmd_no == HostCmd_CMD_802_11_SCAN_EXT) {
 		mwifiex_queue_scan_cmd(priv, cmd_node);
 	} else {
-		mwifiex_insert_cmd_to_pending_q(adapter, cmd_node, true);
+		mwifiex_insert_cmd_to_pending_q(adapter, cmd_node);
 		queue_work(adapter->workqueue, &adapter->main_work);
 		if (cmd_node->wait_q_enabled)
 			ret = mwifiex_wait_queue_complete(adapter, cmd_node);
@@ -684,11 +684,12 @@ int mwifiex_send_cmd(struct mwifiex_private *priv, u16 cmd_no,
  */
 void
 mwifiex_insert_cmd_to_pending_q(struct mwifiex_adapter *adapter,
-				struct cmd_ctrl_node *cmd_node, u32 add_tail)
+				struct cmd_ctrl_node *cmd_node)
 {
 	struct host_cmd_ds_command *host_cmd = NULL;
 	u16 command;
 	unsigned long flags;
+	bool add_tail = true;
 
 	host_cmd = (struct host_cmd_ds_command *) (cmd_node->cmd_skb->data);
 	if (!host_cmd) {
@@ -1075,7 +1076,7 @@ mwifiex_cancel_all_pending_cmd(struct mwifiex_adapter *adapter)
  * In case of scan commands, all pending commands in scan pending queue
  * are cancelled.
  */
-void
+static void
 mwifiex_cancel_pending_ioctl(struct mwifiex_adapter *adapter)
 {
 	struct cmd_ctrl_node *cmd_node = NULL;
diff --git a/drivers/net/wireless/marvell/mwifiex/debugfs.c b/drivers/net/wireless/marvell/mwifiex/debugfs.c
index f6f105a7d3ff..6f4239be609d 100644
--- a/drivers/net/wireless/marvell/mwifiex/debugfs.c
+++ b/drivers/net/wireless/marvell/mwifiex/debugfs.c
@@ -940,8 +940,6 @@ mwifiex_reset_write(struct file *file,
 
 	if (adapter->if_ops.card_reset) {
 		dev_info(adapter->dev, "Resetting per request\n");
-		adapter->hw_status = MWIFIEX_HW_STATUS_RESET;
-		mwifiex_cancel_all_pending_cmd(adapter);
 		adapter->if_ops.card_reset(adapter);
 	}
 
diff --git a/drivers/net/wireless/marvell/mwifiex/init.c b/drivers/net/wireless/marvell/mwifiex/init.c
index 3ecb59f7405b..e11919db7818 100644
--- a/drivers/net/wireless/marvell/mwifiex/init.c
+++ b/drivers/net/wireless/marvell/mwifiex/init.c
@@ -337,17 +337,9 @@ void mwifiex_wake_up_net_dev_queue(struct net_device *netdev,
 					struct mwifiex_adapter *adapter)
 {
 	unsigned long dev_queue_flags;
-	unsigned int i;
 
 	spin_lock_irqsave(&adapter->queue_lock, dev_queue_flags);
-
-	for (i = 0; i < netdev->num_tx_queues; i++) {
-		struct netdev_queue *txq = netdev_get_tx_queue(netdev, i);
-
-		if (netif_tx_queue_stopped(txq))
-			netif_tx_wake_queue(txq);
-	}
-
+	netif_tx_wake_all_queues(netdev);
 	spin_unlock_irqrestore(&adapter->queue_lock, dev_queue_flags);
 }
 
@@ -358,30 +350,20 @@ void mwifiex_stop_net_dev_queue(struct net_device *netdev,
 					struct mwifiex_adapter *adapter)
 {
 	unsigned long dev_queue_flags;
-	unsigned int i;
 
 	spin_lock_irqsave(&adapter->queue_lock, dev_queue_flags);
-
-	for (i = 0; i < netdev->num_tx_queues; i++) {
-		struct netdev_queue *txq = netdev_get_tx_queue(netdev, i);
-
-		if (!netif_tx_queue_stopped(txq))
-			netif_tx_stop_queue(txq);
-	}
-
+	netif_tx_stop_all_queues(netdev);
 	spin_unlock_irqrestore(&adapter->queue_lock, dev_queue_flags);
 }
 
 /*
- *  This function releases the lock variables and frees the locks and
- *  associated locks.
+ * This function invalidates the list heads.
  */
-static void mwifiex_free_lock_list(struct mwifiex_adapter *adapter)
+static void mwifiex_invalidate_lists(struct mwifiex_adapter *adapter)
 {
 	struct mwifiex_private *priv;
 	s32 i, j;
 
-	/* Free lists */
 	list_del(&adapter->cmd_free_q);
 	list_del(&adapter->cmd_pending_q);
 	list_del(&adapter->scan_pending_q);
@@ -418,9 +400,11 @@ mwifiex_adapter_cleanup(struct mwifiex_adapter *adapter)
 	mwifiex_cancel_all_pending_cmd(adapter);
 	wake_up_interruptible(&adapter->cmd_wait_q.wait);
 	wake_up_interruptible(&adapter->hs_activate_wait_q);
+}
 
-	/* Free lock variables */
-	mwifiex_free_lock_list(adapter);
+void mwifiex_free_cmd_buffers(struct mwifiex_adapter *adapter)
+{
+	mwifiex_invalidate_lists(adapter);
 
 	/* Free command buffer */
 	mwifiex_dbg(adapter, INFO, "info: free cmd buffer\n");
diff --git a/drivers/net/wireless/marvell/mwifiex/join.c b/drivers/net/wireless/marvell/mwifiex/join.c
index b89596c18b41..d87aeff70cef 100644
--- a/drivers/net/wireless/marvell/mwifiex/join.c
+++ b/drivers/net/wireless/marvell/mwifiex/join.c
@@ -253,7 +253,7 @@ mwifiex_cmd_append_wps_ie(struct mwifiex_private *priv, u8 **buffer)
 			    priv->wps_ie_len, *buffer);
 
 		/* Wrap the generic IE buffer with a pass through TLV type */
-		ie_header.type = cpu_to_le16(TLV_TYPE_MGMT_IE);
+		ie_header.type = cpu_to_le16(TLV_TYPE_PASSTHROUGH);
 		ie_header.len = cpu_to_le16(priv->wps_ie_len);
 		memcpy(*buffer, &ie_header, sizeof(ie_header));
 		*buffer += sizeof(ie_header);
diff --git a/drivers/net/wireless/marvell/mwifiex/main.c b/drivers/net/wireless/marvell/mwifiex/main.c
index f2600b827e81..ee40b739b289 100644
--- a/drivers/net/wireless/marvell/mwifiex/main.c
+++ b/drivers/net/wireless/marvell/mwifiex/main.c
@@ -46,7 +46,7 @@ MODULE_PARM_DESC(mfg_mode, "manufacturing mode enable:1, disable:0");
 
 bool aggr_ctrl;
 module_param(aggr_ctrl, bool, 0000);
-MODULE_PARM_DESC(aggr_ctrl, "usb tx aggreataon enable:1, disable:0");
+MODULE_PARM_DESC(aggr_ctrl, "usb tx aggregation enable:1, disable:0");
 
 /*
  * This function registers the device and performs all the necessary
@@ -588,7 +588,7 @@ static int _mwifiex_fw_dpc(const struct firmware *firmware, void *context)
 	if (mwifiex_init_channel_scan_gap(adapter)) {
 		mwifiex_dbg(adapter, ERROR,
 			    "could not init channel stats table\n");
-		goto err_init_fw;
+		goto err_init_chan_scan;
 	}
 
 	if (driver_mode) {
@@ -636,6 +636,7 @@ static int _mwifiex_fw_dpc(const struct firmware *firmware, void *context)
 
 err_add_intf:
 	vfree(adapter->chan_stats);
+err_init_chan_scan:
 	wiphy_unregister(adapter->wiphy);
 	wiphy_free(adapter->wiphy);
 err_init_fw:
@@ -653,6 +654,7 @@ err_dnld_fw:
 	if (adapter->hw_status == MWIFIEX_HW_STATUS_READY) {
 		pr_debug("info: %s: shutdown mwifiex\n", __func__);
 		mwifiex_shutdown_drv(adapter);
+		mwifiex_free_cmd_buffers(adapter);
 	}
 
 	init_failed = true;
@@ -665,8 +667,11 @@ done:
 		release_firmware(adapter->firmware);
 		adapter->firmware = NULL;
 	}
-	if (init_failed)
+	if (init_failed) {
+		if (adapter->irq_wakeup >= 0)
+			device_init_wakeup(adapter->dev, false);
 		mwifiex_free_adapter(adapter);
+	}
 	/* Tell all current and future waiters we're finished */
 	complete_all(fw_done);
 
@@ -935,31 +940,44 @@ mwifiex_hard_start_xmit(struct sk_buff *skb, struct net_device *dev)
 	return 0;
 }
 
-/*
- * CFG802.11 network device handler for setting MAC address.
- */
-static int
-mwifiex_set_mac_address(struct net_device *dev, void *addr)
+int mwifiex_set_mac_address(struct mwifiex_private *priv,
+			    struct net_device *dev)
 {
-	struct mwifiex_private *priv = mwifiex_netdev_get_priv(dev);
-	struct sockaddr *hw_addr = addr;
 	int ret;
+	u64 mac_addr;
 
-	memcpy(priv->curr_addr, hw_addr->sa_data, ETH_ALEN);
+	if (priv->bss_type != MWIFIEX_BSS_TYPE_P2P)
+		goto done;
+
+	mac_addr = ether_addr_to_u64(priv->curr_addr);
+	mac_addr |= BIT_ULL(MWIFIEX_MAC_LOCAL_ADMIN_BIT);
+	u64_to_ether_addr(mac_addr, priv->curr_addr);
 
 	/* Send request to firmware */
 	ret = mwifiex_send_cmd(priv, HostCmd_CMD_802_11_MAC_ADDRESS,
 			       HostCmd_ACT_GEN_SET, 0, NULL, true);
 
-	if (!ret)
-		memcpy(priv->netdev->dev_addr, priv->curr_addr, ETH_ALEN);
-	else
+	if (ret) {
 		mwifiex_dbg(priv->adapter, ERROR,
 			    "set mac address failed: ret=%d\n", ret);
+		return ret;
+	}
 
+done:
 	memcpy(dev->dev_addr, priv->curr_addr, ETH_ALEN);
+	return 0;
+}
 
-	return ret;
+/* CFG802.11 network device handler for setting MAC address.
+ */
+static int
+mwifiex_ndo_set_mac_address(struct net_device *dev, void *addr)
+{
+	struct mwifiex_private *priv = mwifiex_netdev_get_priv(dev);
+	struct sockaddr *hw_addr = addr;
+
+	memcpy(priv->curr_addr, hw_addr->sa_data, ETH_ALEN);
+	return mwifiex_set_mac_address(priv, dev);
 }
 
 /*
@@ -1252,7 +1270,7 @@ static const struct net_device_ops mwifiex_netdev_ops = {
 	.ndo_open = mwifiex_open,
 	.ndo_stop = mwifiex_close,
 	.ndo_start_xmit = mwifiex_hard_start_xmit,
-	.ndo_set_mac_address = mwifiex_set_mac_address,
+	.ndo_set_mac_address = mwifiex_ndo_set_mac_address,
 	.ndo_validate_addr = eth_validate_addr,
 	.ndo_tx_timeout = mwifiex_tx_timeout,
 	.ndo_get_stats = mwifiex_get_stats,
@@ -1296,7 +1314,6 @@ void mwifiex_init_priv_params(struct mwifiex_private *priv,
 	priv->gen_idx = MWIFIEX_AUTO_IDX_MASK;
 	priv->num_tx_timeout = 0;
 	ether_addr_copy(priv->curr_addr, priv->adapter->perm_addr);
-	memcpy(dev->dev_addr, priv->curr_addr, ETH_ALEN);
 
 	if (GET_BSS_ROLE(priv) == MWIFIEX_BSS_ROLE_STA ||
 	    GET_BSS_ROLE(priv) == MWIFIEX_BSS_ROLE_UAP) {
@@ -1352,26 +1369,12 @@ static void mwifiex_main_work_queue(struct work_struct *work)
 	mwifiex_main_process(adapter);
 }
 
-/*
- * This function gets called during PCIe function level reset. Required
- * code is extracted from mwifiex_remove_card()
- */
-int
-mwifiex_shutdown_sw(struct mwifiex_adapter *adapter)
+/* Common teardown code used for both device removal and reset */
+static void mwifiex_uninit_sw(struct mwifiex_adapter *adapter)
 {
 	struct mwifiex_private *priv;
 	int i;
 
-	if (!adapter)
-		goto exit_return;
-
-	wait_for_completion(adapter->fw_done);
-	/* Caller should ensure we aren't suspending while this happens */
-	reinit_completion(adapter->fw_done);
-
-	priv = mwifiex_get_priv(adapter, MWIFIEX_BSS_ROLE_ANY);
-	mwifiex_deauthenticate(priv, NULL);
-
 	/* We can no longer handle interrupts once we start doing the teardown
 	 * below.
 	 */
@@ -1380,6 +1383,7 @@ mwifiex_shutdown_sw(struct mwifiex_adapter *adapter)
 
 	adapter->surprise_removed = true;
 	mwifiex_terminate_workqueue(adapter);
+	adapter->int_status = 0;
 
 	/* Stop data */
 	for (i = 0; i < adapter->priv_num; i++) {
@@ -1393,12 +1397,9 @@ mwifiex_shutdown_sw(struct mwifiex_adapter *adapter)
 	}
 
 	mwifiex_dbg(adapter, CMD, "cmd: calling mwifiex_shutdown_drv...\n");
-
 	mwifiex_shutdown_drv(adapter);
-	if (adapter->if_ops.down_dev)
-		adapter->if_ops.down_dev(adapter);
-
 	mwifiex_dbg(adapter, CMD, "cmd: mwifiex_shutdown_drv done\n");
+
 	if (atomic_read(&adapter->rx_pending) ||
 	    atomic_read(&adapter->tx_pending) ||
 	    atomic_read(&adapter->cmd_pending)) {
@@ -1420,10 +1421,37 @@ mwifiex_shutdown_sw(struct mwifiex_adapter *adapter)
 			mwifiex_del_virtual_intf(adapter->wiphy, &priv->wdev);
 		rtnl_unlock();
 	}
+
+	wiphy_unregister(adapter->wiphy);
+	wiphy_free(adapter->wiphy);
+	adapter->wiphy = NULL;
+
 	vfree(adapter->chan_stats);
+	mwifiex_free_cmd_buffers(adapter);
+}
+
+/*
+ * This function gets called during PCIe function level reset.
+ */
+int mwifiex_shutdown_sw(struct mwifiex_adapter *adapter)
+{
+	struct mwifiex_private *priv;
+
+	if (!adapter)
+		return 0;
+
+	wait_for_completion(adapter->fw_done);
+	/* Caller should ensure we aren't suspending while this happens */
+	reinit_completion(adapter->fw_done);
+
+	priv = mwifiex_get_priv(adapter, MWIFIEX_BSS_ROLE_ANY);
+	mwifiex_deauthenticate(priv, NULL);
+
+	mwifiex_uninit_sw(adapter);
+
+	if (adapter->if_ops.down_dev)
+		adapter->if_ops.down_dev(adapter);
 
-	mwifiex_dbg(adapter, INFO, "%s, successful\n", __func__);
-exit_return:
 	return 0;
 }
 EXPORT_SYMBOL_GPL(mwifiex_shutdown_sw);
@@ -1506,6 +1534,7 @@ err_kmalloc:
 		mwifiex_dbg(adapter, ERROR,
 			    "info: %s: shutdown mwifiex\n", __func__);
 		mwifiex_shutdown_drv(adapter);
+		mwifiex_free_cmd_buffers(adapter);
 	}
 
 	complete_all(adapter->fw_done);
@@ -1605,10 +1634,8 @@ mwifiex_add_card(void *card, struct completion *fw_done,
 	adapter->cmd_wait_q.status = 0;
 	adapter->scan_wait_q_woken = false;
 
-	if ((num_possible_cpus() > 1) || adapter->iface_type == MWIFIEX_USB) {
+	if ((num_possible_cpus() > 1) || adapter->iface_type == MWIFIEX_USB)
 		adapter->rx_work_enabled = true;
-		pr_notice("rx work enabled, cpus %d\n", num_possible_cpus());
-	}
 
 	adapter->workqueue =
 		alloc_workqueue("MWIFIEX_WORK_QUEUE",
@@ -1653,8 +1680,11 @@ err_registerdev:
 	if (adapter->hw_status == MWIFIEX_HW_STATUS_READY) {
 		pr_debug("info: %s: shutdown mwifiex\n", __func__);
 		mwifiex_shutdown_drv(adapter);
+		mwifiex_free_cmd_buffers(adapter);
 	}
 err_kmalloc:
+	if (adapter->irq_wakeup >= 0)
+		device_init_wakeup(adapter->dev, false);
 	mwifiex_free_adapter(adapter);
 
 err_init_sw:
@@ -1676,64 +1706,10 @@ EXPORT_SYMBOL_GPL(mwifiex_add_card);
  */
 int mwifiex_remove_card(struct mwifiex_adapter *adapter)
 {
-	struct mwifiex_private *priv = NULL;
-	int i;
-
 	if (!adapter)
-		goto exit_remove;
-
-	/* We can no longer handle interrupts once we start doing the teardown
-	 * below. */
-	if (adapter->if_ops.disable_int)
-		adapter->if_ops.disable_int(adapter);
-
-	adapter->surprise_removed = true;
-
-	mwifiex_terminate_workqueue(adapter);
-
-	/* Stop data */
-	for (i = 0; i < adapter->priv_num; i++) {
-		priv = adapter->priv[i];
-		if (priv && priv->netdev) {
-			mwifiex_stop_net_dev_queue(priv->netdev, adapter);
-			if (netif_carrier_ok(priv->netdev))
-				netif_carrier_off(priv->netdev);
-		}
-	}
-
-	mwifiex_dbg(adapter, CMD,
-		    "cmd: calling mwifiex_shutdown_drv...\n");
-
-	mwifiex_shutdown_drv(adapter);
-	mwifiex_dbg(adapter, CMD,
-		    "cmd: mwifiex_shutdown_drv done\n");
-	if (atomic_read(&adapter->rx_pending) ||
-	    atomic_read(&adapter->tx_pending) ||
-	    atomic_read(&adapter->cmd_pending)) {
-		mwifiex_dbg(adapter, ERROR,
-			    "rx_pending=%d, tx_pending=%d,\t"
-			    "cmd_pending=%d\n",
-			    atomic_read(&adapter->rx_pending),
-			    atomic_read(&adapter->tx_pending),
-			    atomic_read(&adapter->cmd_pending));
-	}
-
-	for (i = 0; i < adapter->priv_num; i++) {
-		priv = adapter->priv[i];
-
-		if (!priv)
-			continue;
-
-		rtnl_lock();
-		if (priv->netdev &&
-		    priv->wdev.iftype != NL80211_IFTYPE_UNSPECIFIED)
-			mwifiex_del_virtual_intf(adapter->wiphy, &priv->wdev);
-		rtnl_unlock();
-	}
-	vfree(adapter->chan_stats);
+		return 0;
 
-	wiphy_unregister(adapter->wiphy);
-	wiphy_free(adapter->wiphy);
+	mwifiex_uninit_sw(adapter);
 
 	if (adapter->irq_wakeup >= 0)
 		device_init_wakeup(adapter->dev, false);
@@ -1748,7 +1724,6 @@ int mwifiex_remove_card(struct mwifiex_adapter *adapter)
 		    "info: free adapter\n");
 	mwifiex_free_adapter(adapter);
 
-exit_remove:
 	return 0;
 }
 EXPORT_SYMBOL_GPL(mwifiex_remove_card);
diff --git a/drivers/net/wireless/marvell/mwifiex/main.h b/drivers/net/wireless/marvell/mwifiex/main.h
index f8cf3079ac7d..a76bd797e454 100644
--- a/drivers/net/wireless/marvell/mwifiex/main.h
+++ b/drivers/net/wireless/marvell/mwifiex/main.h
@@ -165,6 +165,8 @@ enum {
 /* Address alignment */
 #define MWIFIEX_ALIGN_ADDR(p, a) (((long)(p) + (a) - 1) & ~((a) - 1))
 
+#define MWIFIEX_MAC_LOCAL_ADMIN_BIT		41
+
 /**
  *enum mwifiex_debug_level  -  marvell wifi debug level
  */
@@ -1077,9 +1079,9 @@ int mwifiex_get_debug_info(struct mwifiex_private *,
 			   struct mwifiex_debug_info *);
 
 int mwifiex_alloc_cmd_buffer(struct mwifiex_adapter *adapter);
-int mwifiex_free_cmd_buffer(struct mwifiex_adapter *adapter);
+void mwifiex_free_cmd_buffer(struct mwifiex_adapter *adapter);
+void mwifiex_free_cmd_buffers(struct mwifiex_adapter *adapter);
 void mwifiex_cancel_all_pending_cmd(struct mwifiex_adapter *adapter);
-void mwifiex_cancel_pending_ioctl(struct mwifiex_adapter *adapter);
 void mwifiex_cancel_pending_scan_cmd(struct mwifiex_adapter *adapter);
 void mwifiex_cancel_scan(struct mwifiex_adapter *adapter);
 
@@ -1087,8 +1089,7 @@ void mwifiex_recycle_cmd_node(struct mwifiex_adapter *adapter,
 			      struct cmd_ctrl_node *cmd_node);
 
 void mwifiex_insert_cmd_to_pending_q(struct mwifiex_adapter *adapter,
-				     struct cmd_ctrl_node *cmd_node,
-				     u32 addtail);
+				     struct cmd_ctrl_node *cmd_node);
 
 int mwifiex_exec_next_cmd(struct mwifiex_adapter *adapter);
 int mwifiex_process_cmdresp(struct mwifiex_adapter *adapter);
@@ -1563,6 +1564,9 @@ int mwifiex_config_start_uap(struct mwifiex_private *priv,
 void mwifiex_uap_del_sta_data(struct mwifiex_private *priv,
 			      struct mwifiex_sta_node *node);
 
+void mwifiex_config_uap_11d(struct mwifiex_private *priv,
+			    struct cfg80211_beacon_data *beacon_data);
+
 void mwifiex_init_11h_params(struct mwifiex_private *priv);
 int mwifiex_is_11h_active(struct mwifiex_private *priv);
 int mwifiex_11h_activate(struct mwifiex_private *priv, bool flag);
@@ -1672,6 +1676,8 @@ void mwifiex_process_tx_pause_event(struct mwifiex_private *priv,
 void mwifiex_process_multi_chan_event(struct mwifiex_private *priv,
 				      struct sk_buff *event_skb);
 void mwifiex_multi_chan_resync(struct mwifiex_adapter *adapter);
+int mwifiex_set_mac_address(struct mwifiex_private *priv,
+			    struct net_device *dev);
 
 #ifdef CONFIG_DEBUG_FS
 void mwifiex_debugfs_init(void);
diff --git a/drivers/net/wireless/marvell/mwifiex/pcie.c b/drivers/net/wireless/marvell/mwifiex/pcie.c
index 21f2201405d1..cd314946452c 100644
--- a/drivers/net/wireless/marvell/mwifiex/pcie.c
+++ b/drivers/net/wireless/marvell/mwifiex/pcie.c
@@ -1043,12 +1043,14 @@ static int mwifiex_pcie_delete_cmdrsp_buf(struct mwifiex_adapter *adapter)
 		mwifiex_unmap_pci_memory(adapter, card->cmdrsp_buf,
 					 PCI_DMA_FROMDEVICE);
 		dev_kfree_skb_any(card->cmdrsp_buf);
+		card->cmdrsp_buf = NULL;
 	}
 
 	if (card && card->cmd_buf) {
 		mwifiex_unmap_pci_memory(adapter, card->cmd_buf,
 					 PCI_DMA_TODEVICE);
 		dev_kfree_skb_any(card->cmd_buf);
+		card->cmd_buf = NULL;
 	}
 	return 0;
 }
@@ -1983,7 +1985,8 @@ static int mwifiex_pcie_event_complete(struct mwifiex_adapter *adapter,
  * (3) wifi image.
  *
  * This function bypass the header and bluetooth part, return
- * the offset of tail wifi-only part.
+ * the offset of tail wifi-only part. If the image is already wifi-only,
+ * that is start with CMD1, return 0.
  */
 
 static int mwifiex_extract_wifi_fw(struct mwifiex_adapter *adapter,
@@ -1991,7 +1994,7 @@ static int mwifiex_extract_wifi_fw(struct mwifiex_adapter *adapter,
 	const struct mwifiex_fw_data *fwdata;
 	u32 offset = 0, data_len, dnld_cmd;
 	int ret = 0;
-	bool cmd7_before = false;
+	bool cmd7_before = false, first_cmd = false;
 
 	while (1) {
 		/* Check for integer and buffer overflow */
@@ -2012,20 +2015,29 @@ static int mwifiex_extract_wifi_fw(struct mwifiex_adapter *adapter,
 
 		switch (dnld_cmd) {
 		case MWIFIEX_FW_DNLD_CMD_1:
-			if (!cmd7_before) {
-				mwifiex_dbg(adapter, ERROR,
-					    "no cmd7 before cmd1!\n");
+			if (offset + data_len < data_len) {
+				mwifiex_dbg(adapter, ERROR, "bad FW parse\n");
 				ret = -1;
 				goto done;
 			}
-			if (offset + data_len < data_len) {
-				mwifiex_dbg(adapter, ERROR, "bad FW parse\n");
+
+			/* Image start with cmd1, already wifi-only firmware */
+			if (!first_cmd) {
+				mwifiex_dbg(adapter, MSG,
+					    "input wifi-only firmware\n");
+				return 0;
+			}
+
+			if (!cmd7_before) {
+				mwifiex_dbg(adapter, ERROR,
+					    "no cmd7 before cmd1!\n");
 				ret = -1;
 				goto done;
 			}
 			offset += data_len;
 			break;
 		case MWIFIEX_FW_DNLD_CMD_5:
+			first_cmd = true;
 			/* Check for integer overflow */
 			if (offset + data_len < data_len) {
 				mwifiex_dbg(adapter, ERROR, "bad FW parse\n");
@@ -2035,6 +2047,7 @@ static int mwifiex_extract_wifi_fw(struct mwifiex_adapter *adapter,
 			offset += data_len;
 			break;
 		case MWIFIEX_FW_DNLD_CMD_6:
+			first_cmd = true;
 			/* Check for integer overflow */
 			if (offset + data_len < data_len) {
 				mwifiex_dbg(adapter, ERROR, "bad FW parse\n");
@@ -2051,6 +2064,7 @@ static int mwifiex_extract_wifi_fw(struct mwifiex_adapter *adapter,
 			}
 			goto done;
 		case MWIFIEX_FW_DNLD_CMD_7:
+			first_cmd = true;
 			cmd7_before = true;
 			break;
 		default:
@@ -2428,7 +2442,7 @@ exit:
  * In case of Rx packets received, the packets are uploaded from card to
  * host and processed accordingly.
  */
-static int mwifiex_process_pcie_int(struct mwifiex_adapter *adapter)
+static int mwifiex_process_int_status(struct mwifiex_adapter *adapter)
 {
 	int ret;
 	u32 pcie_ireg = 0;
@@ -2471,28 +2485,24 @@ static int mwifiex_process_pcie_int(struct mwifiex_adapter *adapter)
 	}
 
 	if (pcie_ireg & HOST_INTR_DNLD_DONE) {
-		pcie_ireg &= ~HOST_INTR_DNLD_DONE;
 		mwifiex_dbg(adapter, INTR, "info: TX DNLD Done\n");
 		ret = mwifiex_pcie_send_data_complete(adapter);
 		if (ret)
 			return ret;
 	}
 	if (pcie_ireg & HOST_INTR_UPLD_RDY) {
-		pcie_ireg &= ~HOST_INTR_UPLD_RDY;
 		mwifiex_dbg(adapter, INTR, "info: Rx DATA\n");
 		ret = mwifiex_pcie_process_recv_data(adapter);
 		if (ret)
 			return ret;
 	}
 	if (pcie_ireg & HOST_INTR_EVENT_RDY) {
-		pcie_ireg &= ~HOST_INTR_EVENT_RDY;
 		mwifiex_dbg(adapter, INTR, "info: Rx EVENT\n");
 		ret = mwifiex_pcie_process_event_ready(adapter);
 		if (ret)
 			return ret;
 	}
 	if (pcie_ireg & HOST_INTR_CMD_DONE) {
-		pcie_ireg &= ~HOST_INTR_CMD_DONE;
 		if (adapter->cmd_sent) {
 			mwifiex_dbg(adapter, INTR,
 				    "info: CMD sent Interrupt\n");
@@ -2507,75 +2517,13 @@ static int mwifiex_process_pcie_int(struct mwifiex_adapter *adapter)
 	mwifiex_dbg(adapter, INTR,
 		    "info: cmd_sent=%d data_sent=%d\n",
 		    adapter->cmd_sent, adapter->data_sent);
-	if (!card->msi_enable && adapter->ps_state != PS_STATE_SLEEP)
+	if (!card->msi_enable && !card->msix_enable &&
+				 adapter->ps_state != PS_STATE_SLEEP)
 		mwifiex_pcie_enable_host_int(adapter);
 
 	return 0;
 }
 
-static int mwifiex_process_msix_int(struct mwifiex_adapter *adapter)
-{
-	int ret;
-	u32 pcie_ireg;
-	unsigned long flags;
-
-	spin_lock_irqsave(&adapter->int_lock, flags);
-	/* Clear out unused interrupts */
-	pcie_ireg = adapter->int_status;
-	adapter->int_status = 0;
-	spin_unlock_irqrestore(&adapter->int_lock, flags);
-
-	if (pcie_ireg & HOST_INTR_DNLD_DONE) {
-		mwifiex_dbg(adapter, INTR,
-			    "info: TX DNLD Done\n");
-		ret = mwifiex_pcie_send_data_complete(adapter);
-		if (ret)
-			return ret;
-	}
-	if (pcie_ireg & HOST_INTR_UPLD_RDY) {
-		mwifiex_dbg(adapter, INTR,
-			    "info: Rx DATA\n");
-		ret = mwifiex_pcie_process_recv_data(adapter);
-		if (ret)
-			return ret;
-	}
-	if (pcie_ireg & HOST_INTR_EVENT_RDY) {
-		mwifiex_dbg(adapter, INTR,
-			    "info: Rx EVENT\n");
-		ret = mwifiex_pcie_process_event_ready(adapter);
-		if (ret)
-			return ret;
-	}
-
-	if (pcie_ireg & HOST_INTR_CMD_DONE) {
-		if (adapter->cmd_sent) {
-			mwifiex_dbg(adapter, INTR,
-				    "info: CMD sent Interrupt\n");
-			adapter->cmd_sent = false;
-		}
-		/* Handle command response */
-		ret = mwifiex_pcie_process_cmd_complete(adapter);
-		if (ret)
-			return ret;
-	}
-
-	mwifiex_dbg(adapter, INTR,
-		    "info: cmd_sent=%d data_sent=%d\n",
-		    adapter->cmd_sent, adapter->data_sent);
-
-	return 0;
-}
-
-static int mwifiex_process_int_status(struct mwifiex_adapter *adapter)
-{
-	struct pcie_service_card *card = adapter->card;
-
-	if (card->msix_enable)
-		return mwifiex_process_msix_int(adapter);
-	else
-		return mwifiex_process_pcie_int(adapter);
-}
-
 /*
  * This function downloads data from driver to card.
  *
@@ -2934,7 +2882,6 @@ static void mwifiex_pcie_free_buffers(struct mwifiex_adapter *adapter)
 	mwifiex_pcie_delete_evtbd_ring(adapter);
 	mwifiex_pcie_delete_rxbd_ring(adapter);
 	mwifiex_pcie_delete_txbd_ring(adapter);
-	card->cmdrsp_buf = NULL;
 }
 
 /*
@@ -3036,15 +2983,14 @@ static void mwifiex_cleanup_pcie(struct mwifiex_adapter *adapter)
 				    "Failed to write driver not-ready signature\n");
 	}
 
-	mwifiex_pcie_free_buffers(adapter);
+	pci_disable_device(pdev);
 
-	if (pdev) {
-		pci_iounmap(pdev, card->pci_mmap);
-		pci_iounmap(pdev, card->pci_mmap1);
-		pci_disable_device(pdev);
-		pci_release_region(pdev, 2);
-		pci_release_region(pdev, 0);
-	}
+	pci_iounmap(pdev, card->pci_mmap);
+	pci_iounmap(pdev, card->pci_mmap1);
+	pci_release_region(pdev, 2);
+	pci_release_region(pdev, 0);
+
+	mwifiex_pcie_free_buffers(adapter);
 }
 
 static int mwifiex_pcie_request_irq(struct mwifiex_adapter *adapter)
@@ -3220,7 +3166,6 @@ static void mwifiex_unregister_dev(struct mwifiex_adapter *adapter)
 static void mwifiex_pcie_up_dev(struct mwifiex_adapter *adapter)
 {
 	struct pcie_service_card *card = adapter->card;
-	int ret;
 	struct pci_dev *pdev = card->dev;
 
 	/* tx_buf_size might be changed to 3584 by firmware during
@@ -3228,11 +3173,9 @@ static void mwifiex_pcie_up_dev(struct mwifiex_adapter *adapter)
 	 */
 	adapter->tx_buf_size = card->pcie.tx_buf_size;
 
-	ret = mwifiex_pcie_alloc_buffers(adapter);
-	if (!ret)
-		return;
+	mwifiex_pcie_alloc_buffers(adapter);
 
-	pci_iounmap(pdev, card->pci_mmap1);
+	pci_set_master(pdev);
 }
 
 /* This function cleans up the PCI-E host memory space. */
@@ -3240,10 +3183,13 @@ static void mwifiex_pcie_down_dev(struct mwifiex_adapter *adapter)
 {
 	struct pcie_service_card *card = adapter->card;
 	const struct mwifiex_pcie_card_reg *reg = card->pcie.reg;
+	struct pci_dev *pdev = card->dev;
 
 	if (mwifiex_write_reg(adapter, reg->drv_rdy, 0x00000000))
 		mwifiex_dbg(adapter, ERROR, "Failed to write driver not-ready signature\n");
 
+	pci_clear_master(pdev);
+
 	adapter->seq_num = 0;
 
 	mwifiex_pcie_free_buffers(adapter);
diff --git a/drivers/net/wireless/marvell/mwifiex/scan.c b/drivers/net/wireless/marvell/mwifiex/scan.c
index ae9630b49342..c9d41ed77fc7 100644
--- a/drivers/net/wireless/marvell/mwifiex/scan.c
+++ b/drivers/net/wireless/marvell/mwifiex/scan.c
@@ -1534,8 +1534,7 @@ int mwifiex_scan_networks(struct mwifiex_private *priv,
 			list_del(&cmd_node->list);
 			spin_unlock_irqrestore(&adapter->scan_pending_q_lock,
 					       flags);
-			mwifiex_insert_cmd_to_pending_q(adapter, cmd_node,
-							true);
+			mwifiex_insert_cmd_to_pending_q(adapter, cmd_node);
 			queue_work(adapter->workqueue, &adapter->main_work);
 
 			/* Perform internal scan synchronously */
@@ -1948,7 +1947,8 @@ mwifiex_active_scan_req_for_passive_chan(struct mwifiex_private *priv)
 	}
 
 	adapter->active_scan_triggered = true;
-	ether_addr_copy(user_scan_cfg->random_mac, priv->random_mac);
+	if (priv->scan_request->flags & NL80211_SCAN_FLAG_RANDOM_ADDR)
+		ether_addr_copy(user_scan_cfg->random_mac, priv->random_mac);
 	user_scan_cfg->num_ssids = priv->scan_request->n_ssids;
 	user_scan_cfg->ssid_list = priv->scan_request->ssids;
 
@@ -2033,7 +2033,7 @@ static void mwifiex_check_next_scan_command(struct mwifiex_private *priv)
 					    struct cmd_ctrl_node, list);
 		list_del(&cmd_node->list);
 		spin_unlock_irqrestore(&adapter->scan_pending_q_lock, flags);
-		mwifiex_insert_cmd_to_pending_q(adapter, cmd_node, true);
+		mwifiex_insert_cmd_to_pending_q(adapter, cmd_node);
 	}
 
 	return;
@@ -2492,6 +2492,12 @@ mwifiex_update_chan_statistics(struct mwifiex_private *priv,
 					      sizeof(struct mwifiex_chan_stats);
 
 	for (i = 0 ; i < num_chan; i++) {
+		if (adapter->survey_idx >= adapter->num_in_chan_stats) {
+			mwifiex_dbg(adapter, WARN,
+				    "FW reported too many channel results (max %d)\n",
+				    adapter->num_in_chan_stats);
+			return;
+		}
 		chan_stats.chan_num = fw_chan_stats->chan_num;
 		chan_stats.bandcfg = fw_chan_stats->bandcfg;
 		chan_stats.flags = fw_chan_stats->flags;
@@ -2785,7 +2791,6 @@ static int mwifiex_scan_specific_ssid(struct mwifiex_private *priv,
 	if (!scan_cfg)
 		return -ENOMEM;
 
-	ether_addr_copy(scan_cfg->random_mac, priv->random_mac);
 	scan_cfg->ssid_list = req_ssid;
 	scan_cfg->num_ssids = 1;
 
diff --git a/drivers/net/wireless/marvell/mwifiex/sdio.c b/drivers/net/wireless/marvell/mwifiex/sdio.c
index f81a006668f3..fd5183c10c4e 100644
--- a/drivers/net/wireless/marvell/mwifiex/sdio.c
+++ b/drivers/net/wireless/marvell/mwifiex/sdio.c
@@ -390,7 +390,8 @@ mwifiex_sdio_remove(struct sdio_func *func)
 	mwifiex_dbg(adapter, INFO, "info: SDIO func num=%d\n", func->num);
 
 	ret = mwifiex_sdio_read_fw_status(adapter, &firmware_stat);
-	if (firmware_stat == FIRMWARE_READY_SDIO && !adapter->mfg_mode) {
+	if (!ret && firmware_stat == FIRMWARE_READY_SDIO &&
+	    !adapter->mfg_mode) {
 		mwifiex_deauthenticate_all(adapter);
 
 		priv = mwifiex_get_priv(adapter, MWIFIEX_BSS_ROLE_ANY);
diff --git a/drivers/net/wireless/marvell/mwifiex/sta_cmd.c b/drivers/net/wireless/marvell/mwifiex/sta_cmd.c
index 534d94a206a5..fb090144a6d8 100644
--- a/drivers/net/wireless/marvell/mwifiex/sta_cmd.c
+++ b/drivers/net/wireless/marvell/mwifiex/sta_cmd.c
@@ -189,9 +189,7 @@ static int mwifiex_cmd_tx_rate_cfg(struct mwifiex_private *priv,
 	if (pbitmap_rates != NULL) {
 		rate_scope->hr_dsss_rate_bitmap = cpu_to_le16(pbitmap_rates[0]);
 		rate_scope->ofdm_rate_bitmap = cpu_to_le16(pbitmap_rates[1]);
-		for (i = 0;
-		     i < sizeof(rate_scope->ht_mcs_rate_bitmap) / sizeof(u16);
-		     i++)
+		for (i = 0; i < ARRAY_SIZE(rate_scope->ht_mcs_rate_bitmap); i++)
 			rate_scope->ht_mcs_rate_bitmap[i] =
 				cpu_to_le16(pbitmap_rates[2 + i]);
 		if (priv->adapter->fw_api_ver == MWIFIEX_FW_V15) {
@@ -206,9 +204,7 @@ static int mwifiex_cmd_tx_rate_cfg(struct mwifiex_private *priv,
 			cpu_to_le16(priv->bitmap_rates[0]);
 		rate_scope->ofdm_rate_bitmap =
 			cpu_to_le16(priv->bitmap_rates[1]);
-		for (i = 0;
-		     i < sizeof(rate_scope->ht_mcs_rate_bitmap) / sizeof(u16);
-		     i++)
+		for (i = 0; i < ARRAY_SIZE(rate_scope->ht_mcs_rate_bitmap); i++)
 			rate_scope->ht_mcs_rate_bitmap[i] =
 				cpu_to_le16(priv->bitmap_rates[2 + i]);
 		if (priv->adapter->fw_api_ver == MWIFIEX_FW_V15) {
@@ -1755,7 +1751,7 @@ mwifiex_cmd_tdls_oper(struct mwifiex_private *priv,
 	struct mwifiex_ie_types_vhtcap *vht_capab;
 	struct mwifiex_ie_types_aid *aid;
 	struct mwifiex_ie_types_tdls_idle_timeout *timeout;
-	u8 *pos, qos_info;
+	u8 *pos;
 	u16 config_len = 0;
 	struct station_parameters *params = priv->sta_params;
 
@@ -1789,12 +1785,11 @@ mwifiex_cmd_tdls_oper(struct mwifiex_private *priv,
 		put_unaligned_le16(params->capability, pos);
 		config_len += sizeof(params->capability);
 
-		qos_info = params->uapsd_queues | (params->max_sp << 5);
-		wmm_qos_info = (struct mwifiex_ie_types_qos_info *)(pos +
-								    config_len);
+		wmm_qos_info = (void *)(pos + config_len);
 		wmm_qos_info->header.type = cpu_to_le16(WLAN_EID_QOS_CAPA);
-		wmm_qos_info->header.len = cpu_to_le16(sizeof(qos_info));
-		wmm_qos_info->qos_info = qos_info;
+		wmm_qos_info->header.len =
+				cpu_to_le16(sizeof(wmm_qos_info->qos_info));
+		wmm_qos_info->qos_info = 0;
 		config_len += sizeof(struct mwifiex_ie_types_qos_info);
 
 		if (params->ht_capa) {
diff --git a/drivers/net/wireless/marvell/mwifiex/sta_cmdresp.c b/drivers/net/wireless/marvell/mwifiex/sta_cmdresp.c
index 2945775e83c5..0fba5b10ef2d 100644
--- a/drivers/net/wireless/marvell/mwifiex/sta_cmdresp.c
+++ b/drivers/net/wireless/marvell/mwifiex/sta_cmdresp.c
@@ -298,9 +298,8 @@ static int mwifiex_ret_tx_rate_cfg(struct mwifiex_private *priv,
 			priv->bitmap_rates[1] =
 				le16_to_cpu(rate_scope->ofdm_rate_bitmap);
 			for (i = 0;
-			     i <
-			     sizeof(rate_scope->ht_mcs_rate_bitmap) /
-			     sizeof(u16); i++)
+			     i < ARRAY_SIZE(rate_scope->ht_mcs_rate_bitmap);
+			     i++)
 				priv->bitmap_rates[2 + i] =
 					le16_to_cpu(rate_scope->
 						    ht_mcs_rate_bitmap[i]);
diff --git a/drivers/net/wireless/marvell/mwifiex/sta_ioctl.c b/drivers/net/wireless/marvell/mwifiex/sta_ioctl.c
index 42997e05d90f..a6077ab3efc3 100644
--- a/drivers/net/wireless/marvell/mwifiex/sta_ioctl.c
+++ b/drivers/net/wireless/marvell/mwifiex/sta_ioctl.c
@@ -654,9 +654,9 @@ int mwifiex_get_bss_info(struct mwifiex_private *priv,
  */
 int mwifiex_disable_auto_ds(struct mwifiex_private *priv)
 {
-	struct mwifiex_ds_auto_ds auto_ds;
-
-	auto_ds.auto_ds = DEEP_SLEEP_OFF;
+	struct mwifiex_ds_auto_ds auto_ds = {
+		.auto_ds = DEEP_SLEEP_OFF,
+	};
 
 	return mwifiex_send_cmd(priv, HostCmd_CMD_802_11_PS_MODE_ENH,
 				DIS_AUTO_PS, BITMAP_AUTO_DS, &auto_ds, true);
@@ -811,8 +811,8 @@ int mwifiex_drv_set_power(struct mwifiex_private *priv, u32 *ps_mode)
  * is checked to determine WPA version. If buffer length is zero, the existing
  * WPA IE is reset.
  */
-static int mwifiex_set_wpa_ie_helper(struct mwifiex_private *priv,
-				     u8 *ie_data_ptr, u16 ie_len)
+static int mwifiex_set_wpa_ie(struct mwifiex_private *priv,
+			      u8 *ie_data_ptr, u16 ie_len)
 {
 	if (ie_len) {
 		if (ie_len > sizeof(priv->wpa_ie)) {
@@ -1351,101 +1351,96 @@ static int
 mwifiex_set_gen_ie_helper(struct mwifiex_private *priv, u8 *ie_data_ptr,
 			  u16 ie_len)
 {
-	int ret = 0;
 	struct ieee_types_vendor_header *pvendor_ie;
 	const u8 wpa_oui[] = { 0x00, 0x50, 0xf2, 0x01 };
 	const u8 wps_oui[] = { 0x00, 0x50, 0xf2, 0x04 };
-	u16 unparsed_len = ie_len;
-	int find_wpa_ie = 0;
+	u16 unparsed_len = ie_len, cur_ie_len;
 
 	/* If the passed length is zero, reset the buffer */
 	if (!ie_len) {
 		priv->gen_ie_buf_len = 0;
 		priv->wps.session_enable = false;
-
 		return 0;
-	} else if (!ie_data_ptr) {
+	} else if (!ie_data_ptr ||
+		   ie_len <= sizeof(struct ieee_types_header)) {
 		return -1;
 	}
 	pvendor_ie = (struct ieee_types_vendor_header *) ie_data_ptr;
 
 	while (pvendor_ie) {
+		cur_ie_len = pvendor_ie->len + sizeof(struct ieee_types_header);
+
+		if (pvendor_ie->element_id == WLAN_EID_RSN) {
+			/* IE is a WPA/WPA2 IE so call set_wpa function */
+			mwifiex_set_wpa_ie(priv, (u8 *)pvendor_ie, cur_ie_len);
+			priv->wps.session_enable = false;
+			goto next_ie;
+		}
+
+		if (pvendor_ie->element_id == WLAN_EID_BSS_AC_ACCESS_DELAY) {
+			/* IE is a WAPI IE so call set_wapi function */
+			mwifiex_set_wapi_ie(priv, (u8 *)pvendor_ie,
+					    cur_ie_len);
+			goto next_ie;
+		}
+
 		if (pvendor_ie->element_id == WLAN_EID_VENDOR_SPECIFIC) {
-			/* Test to see if it is a WPA IE, if not, then it is a
-			 * gen IE
+			/* Test to see if it is a WPA IE, if not, then
+			 * it is a gen IE
 			 */
 			if (!memcmp(pvendor_ie->oui, wpa_oui,
 				    sizeof(wpa_oui))) {
-				find_wpa_ie = 1;
-				break;
+				/* IE is a WPA/WPA2 IE so call set_wpa function
+				 */
+				mwifiex_set_wpa_ie(priv, (u8 *)pvendor_ie,
+						   cur_ie_len);
+				priv->wps.session_enable = false;
+				goto next_ie;
 			}
 
-			/* Test to see if it is a WPS IE, if so, enable
-			 * wps session flag
-			 */
 			if (!memcmp(pvendor_ie->oui, wps_oui,
 				    sizeof(wps_oui))) {
+				/* Test to see if it is a WPS IE,
+				 * if so, enable wps session flag
+				 */
 				priv->wps.session_enable = true;
 				mwifiex_dbg(priv->adapter, MSG,
-					    "info: WPS Session Enabled.\n");
-				ret = mwifiex_set_wps_ie(priv,
-							 (u8 *)pvendor_ie,
-							 unparsed_len);
+					    "WPS Session Enabled.\n");
+				mwifiex_set_wps_ie(priv, (u8 *)pvendor_ie,
+						   cur_ie_len);
+				goto next_ie;
 			}
 		}
 
-		if (pvendor_ie->element_id == WLAN_EID_RSN) {
-			find_wpa_ie = 1;
-			break;
-		}
+		/* Saved in gen_ie, such as P2P IE.etc.*/
 
-		if (pvendor_ie->element_id == WLAN_EID_BSS_AC_ACCESS_DELAY) {
-		/* IE is a WAPI IE so call set_wapi function */
-			ret = mwifiex_set_wapi_ie(priv, (u8 *)pvendor_ie,
-						  unparsed_len);
-			return ret;
+		/* Verify that the passed length is not larger than the
+		 * available space remaining in the buffer
+		 */
+		if (cur_ie_len <
+		    (sizeof(priv->gen_ie_buf) - priv->gen_ie_buf_len)) {
+			/* Append the passed data to the end
+			 * of the genIeBuffer
+			 */
+			memcpy(priv->gen_ie_buf + priv->gen_ie_buf_len,
+			       (u8 *)pvendor_ie, cur_ie_len);
+			/* Increment the stored buffer length by the
+			 * size passed
+			 */
+			priv->gen_ie_buf_len += cur_ie_len;
 		}
 
-		unparsed_len -= (pvendor_ie->len +
-				 sizeof(struct ieee_types_header));
+next_ie:
+		unparsed_len -= cur_ie_len;
 
 		if (unparsed_len <= sizeof(struct ieee_types_header))
 			pvendor_ie = NULL;
 		else
 			pvendor_ie = (struct ieee_types_vendor_header *)
-				(((u8 *)pvendor_ie) + pvendor_ie->len +
-				 sizeof(struct ieee_types_header));
+				(((u8 *)pvendor_ie) + cur_ie_len);
 	}
 
-	if (find_wpa_ie) {
-		/* IE is a WPA/WPA2 IE so call set_wpa function */
-		ret = mwifiex_set_wpa_ie_helper(priv, (u8 *)pvendor_ie,
-						unparsed_len);
-		priv->wps.session_enable = false;
-		return ret;
-	}
-
-	/*
-	 * Verify that the passed length is not larger than the
-	 * available space remaining in the buffer
-	 */
-	if (ie_len < (sizeof(priv->gen_ie_buf) - priv->gen_ie_buf_len)) {
-
-		/* Append the passed data to the end of the
-		   genIeBuffer */
-		memcpy(priv->gen_ie_buf + priv->gen_ie_buf_len, ie_data_ptr,
-		       ie_len);
-		/* Increment the stored buffer length by the
-		   size passed */
-		priv->gen_ie_buf_len += ie_len;
-	} else {
-		/* Passed data does not fit in the remaining
-		   buffer space */
-		ret = -1;
-	}
-
-	/* Return 0, or -1 for error case */
-	return ret;
+	return 0;
 }
 
 /*
diff --git a/drivers/net/wireless/marvell/mwifiex/tdls.c b/drivers/net/wireless/marvell/mwifiex/tdls.c
index 39cd677d4159..e76af2866a19 100644
--- a/drivers/net/wireless/marvell/mwifiex/tdls.c
+++ b/drivers/net/wireless/marvell/mwifiex/tdls.c
@@ -130,7 +130,7 @@ mwifiex_tdls_append_rates_ie(struct mwifiex_private *priv,
 
 	if (skb_tailroom(skb) < rates_size + 4) {
 		mwifiex_dbg(priv->adapter, ERROR,
-			    "Insuffient space while adding rates\n");
+			    "Insufficient space while adding rates\n");
 		return -ENOMEM;
 	}
 
diff --git a/drivers/net/wireless/marvell/mwifiex/uap_cmd.c b/drivers/net/wireless/marvell/mwifiex/uap_cmd.c
index 477c29c9f5d9..18f7d9bf30b2 100644
--- a/drivers/net/wireless/marvell/mwifiex/uap_cmd.c
+++ b/drivers/net/wireless/marvell/mwifiex/uap_cmd.c
@@ -444,6 +444,28 @@ mwifiex_uap_bss_wep(u8 **tlv_buf, void *cmd_buf, u16 *param_size)
 	return;
 }
 
+/* This function enable 11D if userspace set the country IE.
+ */
+void mwifiex_config_uap_11d(struct mwifiex_private *priv,
+			    struct cfg80211_beacon_data *beacon_data)
+{
+	enum state_11d_t state_11d;
+	const u8 *country_ie;
+
+	country_ie = cfg80211_find_ie(WLAN_EID_COUNTRY, beacon_data->tail,
+				      beacon_data->tail_len);
+	if (country_ie) {
+		/* Send cmd to FW to enable 11D function */
+		state_11d = ENABLE_11D;
+		if (mwifiex_send_cmd(priv, HostCmd_CMD_802_11_SNMP_MIB,
+				     HostCmd_ACT_GEN_SET, DOT11D_I,
+				     &state_11d, true)) {
+			mwifiex_dbg(priv->adapter, ERROR,
+				    "11D: failed to enable 11D\n");
+		}
+	}
+}
+
 /* This function parses BSS related parameters from structure
  * and prepares TLVs. These TLVs are appended to command buffer.
 */
@@ -848,8 +870,6 @@ void mwifiex_uap_set_channel(struct mwifiex_private *priv,
 int mwifiex_config_start_uap(struct mwifiex_private *priv,
 			     struct mwifiex_uap_bss_param *bss_cfg)
 {
-	enum state_11d_t state_11d;
-
 	if (mwifiex_send_cmd(priv, HostCmd_CMD_UAP_SYS_CONFIG,
 			     HostCmd_ACT_GEN_SET,
 			     UAP_BSS_PARAMS_I, bss_cfg, true)) {
@@ -858,16 +878,6 @@ int mwifiex_config_start_uap(struct mwifiex_private *priv,
 		return -1;
 	}
 
-	/* Send cmd to FW to enable 11D function */
-	state_11d = ENABLE_11D;
-	if (mwifiex_send_cmd(priv, HostCmd_CMD_802_11_SNMP_MIB,
-			     HostCmd_ACT_GEN_SET, DOT11D_I,
-			     &state_11d, true)) {
-		mwifiex_dbg(priv->adapter, ERROR,
-			    "11D: failed to enable 11D\n");
-		return -1;
-	}
-
 	if (mwifiex_send_cmd(priv, HostCmd_CMD_UAP_BSS_START,
 			     HostCmd_ACT_GEN_SET, 0, NULL, true)) {
 		mwifiex_dbg(priv->adapter, ERROR,
diff --git a/drivers/net/wireless/marvell/mwifiex/usb.c b/drivers/net/wireless/marvell/mwifiex/usb.c
index cb1753e43ef4..f4f2b9b27e32 100644
--- a/drivers/net/wireless/marvell/mwifiex/usb.c
+++ b/drivers/net/wireless/marvell/mwifiex/usb.c
@@ -24,7 +24,7 @@
 
 static struct mwifiex_if_ops usb_ops;
 
-static struct usb_device_id mwifiex_usb_table[] = {
+static const struct usb_device_id mwifiex_usb_table[] = {
 	/* 8766 */
 	{USB_DEVICE(USB8XXX_VID, USB8766_PID_1)},
 	{USB_DEVICE_AND_INTERFACE_INFO(USB8XXX_VID, USB8766_PID_2,
@@ -1112,7 +1112,7 @@ static void mwifiex_usb_tx_aggr_tmo(unsigned long context)
 	if (err) {
 		mwifiex_dbg(adapter, ERROR,
 			    "prepare tx aggr skb failed, err=%d\n", err);
-		return;
+		goto unlock;
 	}
 
 	if (atomic_read(&port->tx_data_urb_pending) >=
@@ -1133,6 +1133,7 @@ static void mwifiex_usb_tx_aggr_tmo(unsigned long context)
 done:
 	if (err == -1)
 		mwifiex_write_data_complete(adapter, skb_send, 0, -1);
+unlock:
 	spin_unlock_irqrestore(&port->tx_aggr_lock, flags);
 }
 
diff --git a/drivers/net/wireless/mediatek/mt7601u/dma.c b/drivers/net/wireless/mediatek/mt7601u/dma.c
index 660267b359e4..7f3e3983b781 100644
--- a/drivers/net/wireless/mediatek/mt7601u/dma.c
+++ b/drivers/net/wireless/mediatek/mt7601u/dma.c
@@ -457,6 +457,9 @@ static void mt7601u_free_tx(struct mt7601u_dev *dev)
 {
 	int i;
 
+	if (!dev->tx_q)
+		return;
+
 	for (i = 0; i < __MT_EP_OUT_MAX; i++)
 		mt7601u_free_tx_queue(&dev->tx_q[i]);
 }
@@ -484,6 +487,8 @@ static int mt7601u_alloc_tx(struct mt7601u_dev *dev)
 
 	dev->tx_q = devm_kcalloc(dev->dev, __MT_EP_OUT_MAX,
 				 sizeof(*dev->tx_q), GFP_KERNEL);
+	if (!dev->tx_q)
+		return -ENOMEM;
 
 	for (i = 0; i < __MT_EP_OUT_MAX; i++)
 		if (mt7601u_alloc_tx_queue(dev, &dev->tx_q[i]))
diff --git a/drivers/net/wireless/mediatek/mt7601u/usb.c b/drivers/net/wireless/mediatek/mt7601u/usb.c
index 416c6045ff31..b9e4f6793138 100644
--- a/drivers/net/wireless/mediatek/mt7601u/usb.c
+++ b/drivers/net/wireless/mediatek/mt7601u/usb.c
@@ -19,7 +19,7 @@
 #include "usb.h"
 #include "trace.h"
 
-static struct usb_device_id mt7601u_device_table[] = {
+static const struct usb_device_id mt7601u_device_table[] = {
 	{ USB_DEVICE(0x0b05, 0x17d3) },
 	{ USB_DEVICE(0x0e8d, 0x760a) },
 	{ USB_DEVICE(0x0e8d, 0x760b) },
diff --git a/drivers/net/wireless/quantenna/qtnfmac/Makefile b/drivers/net/wireless/quantenna/qtnfmac/Makefile
index 0d618e5e5f5b..f236b7dc2be3 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/Makefile
+++ b/drivers/net/wireless/quantenna/qtnfmac/Makefile
@@ -25,7 +25,3 @@ qtnfmac_pearl_pcie-objs += \
 	pearl/pcie.o
 
 qtnfmac_pearl_pcie-$(CONFIG_DEBUG_FS) += debug.o
-
-#
-
-ccflags-y += -D__CHECK_ENDIAN
diff --git a/drivers/net/wireless/quantenna/qtnfmac/bus.h b/drivers/net/wireless/quantenna/qtnfmac/bus.h
index dda05003d522..56e5fed92a2a 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/bus.h
+++ b/drivers/net/wireless/quantenna/qtnfmac/bus.h
@@ -130,7 +130,6 @@ static __always_inline void qtnf_bus_unlock(struct qtnf_bus *bus)
 
 /* interface functions from common layer */
 
-void qtnf_rx_frame(struct device *dev, struct sk_buff *rxp);
 int qtnf_core_attach(struct qtnf_bus *bus);
 void qtnf_core_detach(struct qtnf_bus *bus);
 void qtnf_txflowblock(struct device *dev, bool state);
diff --git a/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c b/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c
index e3c090008125..856fa6e8327e 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c
@@ -266,11 +266,19 @@ static int qtnf_start_ap(struct wiphy *wiphy, struct net_device *dev,
 			 struct cfg80211_ap_settings *settings)
 {
 	struct qtnf_vif *vif = qtnf_netdev_get_priv(dev);
+	struct qtnf_wmac *mac = wiphy_priv(wiphy);
 	struct qtnf_bss_config *bss_cfg;
 	int ret;
 
-	bss_cfg = &vif->bss_cfg;
+	if (!cfg80211_chandef_identical(&mac->chandef, &settings->chandef)) {
+		memcpy(&mac->chandef, &settings->chandef, sizeof(mac->chandef));
+		if (vif->vifid != 0)
+			pr_warn("%s: unexpected chan %u (%u MHz)\n", dev->name,
+				settings->chandef.chan->hw_value,
+				settings->chandef.chan->center_freq);
+	}
 
+	bss_cfg = &vif->bss_cfg;
 	memset(bss_cfg, 0, sizeof(*bss_cfg));
 
 	bss_cfg->bcn_period = settings->beacon_interval;
@@ -281,8 +289,6 @@ static int qtnf_start_ap(struct wiphy *wiphy, struct net_device *dev,
 	bss_cfg->ssid_len = settings->ssid_len;
 	memcpy(&bss_cfg->ssid, settings->ssid, bss_cfg->ssid_len);
 
-	memcpy(&bss_cfg->chandef, &settings->chandef,
-	       sizeof(struct cfg80211_chan_def));
 	memcpy(&bss_cfg->crypto, &settings->crypto,
 	       sizeof(struct cfg80211_crypto_settings));
 
@@ -573,19 +579,33 @@ qtnf_del_station(struct wiphy *wiphy, struct net_device *dev,
 	return ret;
 }
 
+static void qtnf_scan_timeout(unsigned long data)
+{
+	struct qtnf_wmac *mac = (struct qtnf_wmac *)data;
+
+	pr_warn("mac%d scan timed out\n", mac->macid);
+	qtnf_scan_done(mac, true);
+}
+
 static int
 qtnf_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request)
 {
 	struct qtnf_wmac *mac = wiphy_priv(wiphy);
-	int ret;
 
 	mac->scan_req = request;
 
-	ret = qtnf_cmd_send_scan(mac);
-	if (ret)
+	if (qtnf_cmd_send_scan(mac)) {
 		pr_err("MAC%u: failed to start scan\n", mac->macid);
+		mac->scan_req = NULL;
+		return -EFAULT;
+	}
 
-	return ret;
+	mac->scan_timeout.data = (unsigned long)mac;
+	mac->scan_timeout.function = qtnf_scan_timeout;
+	mod_timer(&mac->scan_timeout,
+		  jiffies + QTNF_SCAN_TIMEOUT_SEC * HZ);
+
+	return 0;
 }
 
 static int
@@ -593,6 +613,8 @@ qtnf_connect(struct wiphy *wiphy, struct net_device *dev,
 	     struct cfg80211_connect_params *sme)
 {
 	struct qtnf_vif *vif = qtnf_netdev_get_priv(dev);
+	struct qtnf_wmac *mac = wiphy_priv(wiphy);
+	struct cfg80211_chan_def chandef;
 	struct qtnf_bss_config *bss_cfg;
 	int ret;
 
@@ -605,9 +627,20 @@ qtnf_connect(struct wiphy *wiphy, struct net_device *dev,
 	bss_cfg = &vif->bss_cfg;
 	memset(bss_cfg, 0, sizeof(*bss_cfg));
 
+	if (sme->channel) {
+		/* FIXME: need to set proper nl80211_channel_type value */
+		cfg80211_chandef_create(&chandef, sme->channel,
+					NL80211_CHAN_HT20);
+		/* fall-back to minimal safe chandef description */
+		if (!cfg80211_chandef_valid(&chandef))
+			cfg80211_chandef_create(&chandef, sme->channel,
+						NL80211_CHAN_HT20);
+
+		memcpy(&mac->chandef, &chandef, sizeof(mac->chandef));
+	}
+
 	bss_cfg->ssid_len = sme->ssid_len;
 	memcpy(&bss_cfg->ssid, sme->ssid, bss_cfg->ssid_len);
-	bss_cfg->chandef.chan = sme->channel;
 	bss_cfg->auth_type = sme->auth_type;
 	bss_cfg->privacy = sme->privacy;
 	bss_cfg->mfp = sme->mfp;
@@ -677,6 +710,175 @@ qtnf_disconnect(struct wiphy *wiphy, struct net_device *dev,
 	return 0;
 }
 
+static int
+qtnf_dump_survey(struct wiphy *wiphy, struct net_device *dev,
+		 int idx, struct survey_info *survey)
+{
+	struct qtnf_wmac *mac = wiphy_priv(wiphy);
+	struct ieee80211_supported_band *sband;
+	struct cfg80211_chan_def *chandef;
+	struct ieee80211_channel *chan;
+	struct qtnf_chan_stats stats;
+	struct qtnf_vif *vif;
+	int ret;
+
+	vif = qtnf_netdev_get_priv(dev);
+	chandef = &mac->chandef;
+
+	sband = wiphy->bands[NL80211_BAND_2GHZ];
+	if (sband && idx >= sband->n_channels) {
+		idx -= sband->n_channels;
+		sband = NULL;
+	}
+
+	if (!sband)
+		sband = wiphy->bands[NL80211_BAND_5GHZ];
+
+	if (!sband || idx >= sband->n_channels)
+		return -ENOENT;
+
+	chan = &sband->channels[idx];
+	memset(&stats, 0, sizeof(stats));
+
+	survey->channel = chan;
+	survey->filled = 0x0;
+
+	if (chandef->chan) {
+		if (chan->hw_value == chandef->chan->hw_value)
+			survey->filled = SURVEY_INFO_IN_USE;
+	}
+
+	ret = qtnf_cmd_get_chan_stats(mac, chan->hw_value, &stats);
+	switch (ret) {
+	case 0:
+		if (unlikely(stats.chan_num != chan->hw_value)) {
+			pr_err("received stats for channel %d instead of %d\n",
+			       stats.chan_num, chan->hw_value);
+			ret = -EINVAL;
+			break;
+		}
+
+		survey->filled |= SURVEY_INFO_TIME |
+				 SURVEY_INFO_TIME_SCAN |
+				 SURVEY_INFO_TIME_BUSY |
+				 SURVEY_INFO_TIME_RX |
+				 SURVEY_INFO_TIME_TX |
+				 SURVEY_INFO_NOISE_DBM;
+
+		survey->time_scan = stats.cca_try;
+		survey->time = stats.cca_try;
+		survey->time_tx = stats.cca_tx;
+		survey->time_rx = stats.cca_rx;
+		survey->time_busy = stats.cca_busy;
+		survey->noise = stats.chan_noise;
+		break;
+	case -ENOENT:
+		pr_debug("no stats for channel %u\n", chan->hw_value);
+		ret = 0;
+		break;
+	default:
+		pr_debug("failed to get chan(%d) stats from card\n",
+			 chan->hw_value);
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+static int
+qtnf_get_channel(struct wiphy *wiphy, struct wireless_dev *wdev,
+		 struct cfg80211_chan_def *chandef)
+{
+	struct qtnf_wmac *mac = wiphy_priv(wiphy);
+	struct net_device *ndev = wdev->netdev;
+	struct qtnf_vif *vif;
+
+	if (!ndev)
+		return -ENODEV;
+
+	vif = qtnf_netdev_get_priv(wdev->netdev);
+
+	switch (vif->wdev.iftype) {
+	case NL80211_IFTYPE_STATION:
+		if (vif->sta_state == QTNF_STA_DISCONNECTED) {
+			pr_warn("%s: STA disconnected\n", ndev->name);
+			return -ENODATA;
+		}
+		break;
+	case NL80211_IFTYPE_AP:
+		if (!(vif->bss_status & QTNF_STATE_AP_START)) {
+			pr_warn("%s: AP not started\n", ndev->name);
+			return -ENODATA;
+		}
+		break;
+	default:
+		pr_err("unsupported vif type (%d)\n", vif->wdev.iftype);
+		return -ENODATA;
+	}
+
+	if (!cfg80211_chandef_valid(&mac->chandef)) {
+		pr_err("invalid channel settings on %s\n", ndev->name);
+		return -ENODATA;
+	}
+
+	memcpy(chandef, &mac->chandef, sizeof(*chandef));
+	return 0;
+}
+
+static int qtnf_channel_switch(struct wiphy *wiphy, struct net_device *dev,
+			       struct cfg80211_csa_settings *params)
+{
+	struct qtnf_wmac *mac = wiphy_priv(wiphy);
+	struct qtnf_vif *vif = qtnf_netdev_get_priv(dev);
+	int ret;
+
+	pr_debug("%s: chan(%u) count(%u) radar(%u) block_tx(%u)\n", dev->name,
+		 params->chandef.chan->hw_value, params->count,
+		 params->radar_required, params->block_tx);
+
+	switch (vif->wdev.iftype) {
+	case NL80211_IFTYPE_AP:
+		if (!(vif->bss_status & QTNF_STATE_AP_START)) {
+			pr_warn("AP not started on %s\n", dev->name);
+			return -ENOTCONN;
+		}
+		break;
+	default:
+		pr_err("unsupported vif type (%d) on %s\n",
+		       vif->wdev.iftype, dev->name);
+		return -EOPNOTSUPP;
+	}
+
+	if (vif->vifid != 0) {
+		if (!(mac->status & QTNF_MAC_CSA_ACTIVE))
+			return -EOPNOTSUPP;
+
+		if (!cfg80211_chandef_identical(&params->chandef,
+						&mac->csa_chandef))
+			return -EINVAL;
+
+		return 0;
+	}
+
+	if (!cfg80211_chandef_valid(&params->chandef)) {
+		pr_err("%s: invalid channel\n", dev->name);
+		return -EINVAL;
+	}
+
+	if (cfg80211_chandef_identical(&params->chandef, &mac->chandef)) {
+		pr_err("%s: switch request to the same channel\n", dev->name);
+		return -EALREADY;
+	}
+
+	ret = qtnf_cmd_send_chan_switch(mac, params);
+	if (ret)
+		pr_warn("%s: failed to switch to channel (%u)\n",
+			dev->name, params->chandef.chan->hw_value);
+
+	return ret;
+}
+
 static struct cfg80211_ops qtn_cfg80211_ops = {
 	.add_virtual_intf	= qtnf_add_virtual_intf,
 	.change_virtual_intf	= qtnf_change_virtual_intf,
@@ -697,69 +899,49 @@ static struct cfg80211_ops qtn_cfg80211_ops = {
 	.set_default_mgmt_key	= qtnf_set_default_mgmt_key,
 	.scan			= qtnf_scan,
 	.connect		= qtnf_connect,
-	.disconnect		= qtnf_disconnect
+	.disconnect		= qtnf_disconnect,
+	.dump_survey		= qtnf_dump_survey,
+	.get_channel		= qtnf_get_channel,
+	.channel_switch		= qtnf_channel_switch
 };
 
-static void qtnf_cfg80211_reg_notifier(struct wiphy *wiphy,
+static void qtnf_cfg80211_reg_notifier(struct wiphy *wiphy_in,
 				       struct regulatory_request *req)
 {
-	struct qtnf_wmac *mac = wiphy_priv(wiphy);
-	struct qtnf_bus *bus;
-	struct qtnf_vif *vif;
-	struct qtnf_wmac *chan_mac;
-	int i;
+	struct qtnf_wmac *mac = wiphy_priv(wiphy_in);
+	struct qtnf_bus *bus = mac->bus;
+	struct wiphy *wiphy;
+	unsigned int mac_idx;
 	enum nl80211_band band;
-
-	bus = mac->bus;
+	int ret;
 
 	pr_debug("MAC%u: initiator=%d alpha=%c%c\n", mac->macid, req->initiator,
 		 req->alpha2[0], req->alpha2[1]);
 
-	vif = qtnf_mac_get_base_vif(mac);
-	if (!vif) {
-		pr_err("MAC%u: primary VIF is not configured\n", mac->macid);
-		return;
-	}
-
-	/* ignore non-ISO3166 country codes */
-	for (i = 0; i < sizeof(req->alpha2); i++) {
-		if (req->alpha2[i] < 'A' || req->alpha2[i] > 'Z') {
-			pr_err("MAC%u: not an ISO3166 code\n", mac->macid);
-			return;
-		}
-	}
-	if (!strncasecmp(req->alpha2, bus->hw_info.alpha2_code,
-			 sizeof(req->alpha2))) {
-		pr_warn("MAC%u: unchanged country code\n", mac->macid);
-		return;
-	}
-
-	if (qtnf_cmd_send_regulatory_config(mac, req->alpha2)) {
-		pr_err("MAC%u: failed to configure regulatory\n", mac->macid);
+	ret = qtnf_cmd_reg_notify(bus, req);
+	if (ret) {
+		if (ret != -EOPNOTSUPP && ret != -EALREADY)
+			pr_err("failed to update reg domain to %c%c\n",
+			       req->alpha2[0], req->alpha2[1]);
 		return;
 	}
 
-	for (i = 0; i < bus->hw_info.num_mac; i++) {
-		chan_mac = bus->mac[i];
-
-		if (!chan_mac)
+	for (mac_idx = 0; mac_idx < QTNF_MAX_MAC; ++mac_idx) {
+		if (!(bus->hw_info.mac_bitmap & (1 << mac_idx)))
 			continue;
 
-		if (!(bus->hw_info.mac_bitmap & BIT(i)))
-			continue;
+		mac = bus->mac[mac_idx];
+		wiphy = priv_to_wiphy(mac);
 
 		for (band = 0; band < NUM_NL80211_BANDS; ++band) {
 			if (!wiphy->bands[band])
 				continue;
 
-			if (qtnf_cmd_get_mac_chan_info(chan_mac,
-						       wiphy->bands[band])) {
-				pr_err("MAC%u: can't get channel info\n",
-				       chan_mac->macid);
-				qtnf_core_detach(bus);
-
-				return;
-			}
+			ret = qtnf_cmd_get_mac_chan_info(mac,
+							 wiphy->bands[band]);
+			if (ret)
+				pr_err("failed to get chan info for mac %u band %u\n",
+				       mac_idx, band);
 		}
 	}
 }
@@ -844,10 +1026,8 @@ int qtnf_wiphy_register(struct qtnf_hw_info *hw_info, struct qtnf_wmac *mac)
 	}
 
 	iface_comb = kzalloc(sizeof(*iface_comb), GFP_KERNEL);
-	if (!iface_comb) {
-		ret = -ENOMEM;
-		goto out;
-	}
+	if (!iface_comb)
+		return -ENOMEM;
 
 	ret = qtnf_wiphy_setup_if_comb(wiphy, iface_comb, &mac->macinfo);
 	if (ret)
@@ -869,6 +1049,7 @@ int qtnf_wiphy_register(struct qtnf_hw_info *hw_info, struct qtnf_wmac *mac)
 
 	wiphy->iface_combinations = iface_comb;
 	wiphy->n_iface_combinations = 1;
+	wiphy->max_num_csa_counters = 2;
 
 	/* Initialize cipher suits */
 	wiphy->cipher_suites = qtnf_cipher_suites;
@@ -876,7 +1057,8 @@ int qtnf_wiphy_register(struct qtnf_hw_info *hw_info, struct qtnf_wmac *mac)
 	wiphy->signal_type = CFG80211_SIGNAL_TYPE_MBM;
 	wiphy->flags |= WIPHY_FLAG_HAVE_AP_SME |
 			WIPHY_FLAG_AP_PROBE_RESP_OFFLOAD |
-			WIPHY_FLAG_AP_UAPSD;
+			WIPHY_FLAG_AP_UAPSD |
+			WIPHY_FLAG_HAS_CHANNEL_SWITCH;
 
 	wiphy->probe_resp_offload = NL80211_PROBE_RESP_OFFLOAD_SUPPORT_WPS |
 				    NL80211_PROBE_RESP_OFFLOAD_SUPPORT_WPS2;
@@ -889,21 +1071,26 @@ int qtnf_wiphy_register(struct qtnf_hw_info *hw_info, struct qtnf_wmac *mac)
 	ether_addr_copy(wiphy->perm_addr, mac->macaddr);
 
 	if (hw_info->hw_capab & QLINK_HW_SUPPORTS_REG_UPDATE) {
-		pr_debug("device supports REG_UPDATE\n");
+		wiphy->regulatory_flags |= REGULATORY_STRICT_REG |
+			REGULATORY_CUSTOM_REG;
 		wiphy->reg_notifier = qtnf_cfg80211_reg_notifier;
-		pr_debug("hint regulatory about EP region: %c%c\n",
-			 hw_info->alpha2_code[0],
-			 hw_info->alpha2_code[1]);
-		regulatory_hint(wiphy, hw_info->alpha2_code);
+		wiphy_apply_custom_regulatory(wiphy, hw_info->rd);
 	} else {
-		pr_debug("device doesn't support REG_UPDATE\n");
 		wiphy->regulatory_flags |= REGULATORY_WIPHY_SELF_MANAGED;
 	}
 
 	ret = wiphy_register(wiphy);
+	if (ret < 0)
+		goto out;
+
+	if (wiphy->regulatory_flags & REGULATORY_WIPHY_SELF_MANAGED)
+		ret = regulatory_set_wiphy_regd(wiphy, hw_info->rd);
+	else if (isalpha(hw_info->rd->alpha2[0]) &&
+		 isalpha(hw_info->rd->alpha2[1]))
+		ret = regulatory_hint(wiphy, hw_info->rd->alpha2);
 
 out:
-	if (ret < 0) {
+	if (ret) {
 		kfree(iface_comb);
 		return ret;
 	}
diff --git a/drivers/net/wireless/quantenna/qtnfmac/cfg80211.h b/drivers/net/wireless/quantenna/qtnfmac/cfg80211.h
index 5bd33124a7c8..6a4af52522b8 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/cfg80211.h
+++ b/drivers/net/wireless/quantenna/qtnfmac/cfg80211.h
@@ -34,10 +34,14 @@ static inline void qtnf_scan_done(struct qtnf_wmac *mac, bool aborted)
 		.aborted = aborted,
 	};
 
+	mutex_lock(&mac->mac_lock);
+
 	if (mac->scan_req) {
 		cfg80211_scan_done(mac->scan_req, &info);
 		mac->scan_req = NULL;
 	}
+
+	mutex_unlock(&mac->mac_lock);
 }
 
 #endif /* _QTN_FMAC_CFG80211_H_ */
diff --git a/drivers/net/wireless/quantenna/qtnfmac/commands.c b/drivers/net/wireless/quantenna/qtnfmac/commands.c
index b39dbc3d3c1f..4206886b110c 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/commands.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/commands.c
@@ -181,43 +181,11 @@ out:
 	return ret;
 }
 
-int qtnf_cmd_send_regulatory_config(struct qtnf_wmac *mac, const char *alpha2)
-{
-	struct sk_buff *cmd_skb;
-	u16 res_code = QLINK_CMD_RESULT_OK;
-	int ret;
-
-	cmd_skb = qtnf_cmd_alloc_new_cmdskb(mac->macid, QLINK_VIFID_RSVD,
-					    QLINK_CMD_REG_REGION,
-					    sizeof(struct qlink_cmd));
-	if (unlikely(!cmd_skb))
-		return -ENOMEM;
-
-	qtnf_cmd_skb_put_tlv_arr(cmd_skb, WLAN_EID_COUNTRY, alpha2,
-				 QTNF_MAX_ALPHA_LEN);
-
-	ret = qtnf_cmd_send(mac->bus, cmd_skb, &res_code);
-
-	if (unlikely(ret))
-		goto out;
-
-	if (unlikely(res_code != QLINK_CMD_RESULT_OK)) {
-		pr_err("MAC%u: CMD failed: %u\n", mac->macid, res_code);
-		ret = -EFAULT;
-		goto out;
-	}
-
-	memcpy(mac->bus->hw_info.alpha2_code, alpha2,
-	       sizeof(mac->bus->hw_info.alpha2_code));
-out:
-	return ret;
-}
-
 int qtnf_cmd_send_config_ap(struct qtnf_vif *vif)
 {
 	struct sk_buff *cmd_skb;
 	struct qtnf_bss_config *bss_cfg = &vif->bss_cfg;
-	struct cfg80211_chan_def *chandef = &bss_cfg->chandef;
+	struct cfg80211_chan_def *chandef = &vif->mac->chandef;
 	struct qlink_tlv_channel *qchan;
 	struct qlink_auth_encr aen;
 	u16 res_code = QLINK_CMD_RESULT_OK;
@@ -848,25 +816,168 @@ out:
 	return ret;
 }
 
+static u32 qtnf_cmd_resp_reg_rule_flags_parse(u32 qflags)
+{
+	u32 flags = 0;
+
+	if (qflags & QLINK_RRF_NO_OFDM)
+		flags |= NL80211_RRF_NO_OFDM;
+
+	if (qflags & QLINK_RRF_NO_CCK)
+		flags |= NL80211_RRF_NO_CCK;
+
+	if (qflags & QLINK_RRF_NO_INDOOR)
+		flags |= NL80211_RRF_NO_INDOOR;
+
+	if (qflags & QLINK_RRF_NO_OUTDOOR)
+		flags |= NL80211_RRF_NO_OUTDOOR;
+
+	if (qflags & QLINK_RRF_DFS)
+		flags |= NL80211_RRF_DFS;
+
+	if (qflags & QLINK_RRF_PTP_ONLY)
+		flags |= NL80211_RRF_PTP_ONLY;
+
+	if (qflags & QLINK_RRF_PTMP_ONLY)
+		flags |= NL80211_RRF_PTMP_ONLY;
+
+	if (qflags & QLINK_RRF_NO_IR)
+		flags |= NL80211_RRF_NO_IR;
+
+	if (qflags & QLINK_RRF_AUTO_BW)
+		flags |= NL80211_RRF_AUTO_BW;
+
+	if (qflags & QLINK_RRF_IR_CONCURRENT)
+		flags |= NL80211_RRF_IR_CONCURRENT;
+
+	if (qflags & QLINK_RRF_NO_HT40MINUS)
+		flags |= NL80211_RRF_NO_HT40MINUS;
+
+	if (qflags & QLINK_RRF_NO_HT40PLUS)
+		flags |= NL80211_RRF_NO_HT40PLUS;
+
+	if (qflags & QLINK_RRF_NO_80MHZ)
+		flags |= NL80211_RRF_NO_80MHZ;
+
+	if (qflags & QLINK_RRF_NO_160MHZ)
+		flags |= NL80211_RRF_NO_160MHZ;
+
+	return flags;
+}
+
 static int
 qtnf_cmd_resp_proc_hw_info(struct qtnf_bus *bus,
-			   const struct qlink_resp_get_hw_info *resp)
+			   const struct qlink_resp_get_hw_info *resp,
+			   size_t info_len)
 {
 	struct qtnf_hw_info *hwinfo = &bus->hw_info;
+	const struct qlink_tlv_hdr *tlv;
+	const struct qlink_tlv_reg_rule *tlv_rule;
+	struct ieee80211_reg_rule *rule;
+	u16 tlv_type;
+	u16 tlv_value_len;
+	unsigned int rule_idx = 0;
+
+	if (WARN_ON(resp->n_reg_rules > NL80211_MAX_SUPP_REG_RULES))
+		return -E2BIG;
+
+	hwinfo->rd = kzalloc(sizeof(*hwinfo->rd)
+			     + sizeof(struct ieee80211_reg_rule)
+			     * resp->n_reg_rules, GFP_KERNEL);
+
+	if (!hwinfo->rd)
+		return -ENOMEM;
 
 	hwinfo->num_mac = resp->num_mac;
 	hwinfo->mac_bitmap = resp->mac_bitmap;
 	hwinfo->fw_ver = le32_to_cpu(resp->fw_ver);
 	hwinfo->ql_proto_ver = le16_to_cpu(resp->ql_proto_ver);
-	memcpy(hwinfo->alpha2_code, resp->alpha2_code,
-	       sizeof(hwinfo->alpha2_code));
 	hwinfo->total_tx_chain = resp->total_tx_chain;
 	hwinfo->total_rx_chain = resp->total_rx_chain;
 	hwinfo->hw_capab = le32_to_cpu(resp->hw_capab);
+	hwinfo->rd->n_reg_rules = resp->n_reg_rules;
+	hwinfo->rd->alpha2[0] = resp->alpha2[0];
+	hwinfo->rd->alpha2[1] = resp->alpha2[1];
+
+	switch (resp->dfs_region) {
+	case QLINK_DFS_FCC:
+		hwinfo->rd->dfs_region = NL80211_DFS_FCC;
+		break;
+	case QLINK_DFS_ETSI:
+		hwinfo->rd->dfs_region = NL80211_DFS_ETSI;
+		break;
+	case QLINK_DFS_JP:
+		hwinfo->rd->dfs_region = NL80211_DFS_JP;
+		break;
+	case QLINK_DFS_UNSET:
+	default:
+		hwinfo->rd->dfs_region = NL80211_DFS_UNSET;
+		break;
+	}
+
+	tlv = (const struct qlink_tlv_hdr *)resp->info;
+
+	while (info_len >= sizeof(*tlv)) {
+		tlv_type = le16_to_cpu(tlv->type);
+		tlv_value_len = le16_to_cpu(tlv->len);
+
+		if (tlv_value_len + sizeof(*tlv) > info_len) {
+			pr_warn("malformed TLV 0x%.2X; LEN: %u\n",
+				tlv_type, tlv_value_len);
+			return -EINVAL;
+		}
+
+		switch (tlv_type) {
+		case QTN_TLV_ID_REG_RULE:
+			if (rule_idx >= resp->n_reg_rules) {
+				pr_warn("unexpected number of rules: %u\n",
+					resp->n_reg_rules);
+				return -EINVAL;
+			}
+
+			if (tlv_value_len != sizeof(*tlv_rule) - sizeof(*tlv)) {
+				pr_warn("malformed TLV 0x%.2X; LEN: %u\n",
+					tlv_type, tlv_value_len);
+				return -EINVAL;
+			}
+
+			tlv_rule = (const struct qlink_tlv_reg_rule *)tlv;
+			rule = &hwinfo->rd->reg_rules[rule_idx++];
+
+			rule->freq_range.start_freq_khz =
+				le32_to_cpu(tlv_rule->start_freq_khz);
+			rule->freq_range.end_freq_khz =
+				le32_to_cpu(tlv_rule->end_freq_khz);
+			rule->freq_range.max_bandwidth_khz =
+				le32_to_cpu(tlv_rule->max_bandwidth_khz);
+			rule->power_rule.max_antenna_gain =
+				le32_to_cpu(tlv_rule->max_antenna_gain);
+			rule->power_rule.max_eirp =
+				le32_to_cpu(tlv_rule->max_eirp);
+			rule->dfs_cac_ms =
+				le32_to_cpu(tlv_rule->dfs_cac_ms);
+			rule->flags = qtnf_cmd_resp_reg_rule_flags_parse(
+					le32_to_cpu(tlv_rule->flags));
+			break;
+		default:
+			break;
+		}
+
+		info_len -= tlv_value_len + sizeof(*tlv);
+		tlv = (struct qlink_tlv_hdr *)(tlv->val + tlv_value_len);
+	}
+
+	if (rule_idx != resp->n_reg_rules) {
+		pr_warn("unexpected number of rules: expected %u got %u\n",
+			resp->n_reg_rules, rule_idx);
+		kfree(hwinfo->rd);
+		hwinfo->rd = NULL;
+		return -EINVAL;
+	}
 
 	pr_info("fw_version=%d, MACs map %#x, alpha2=\"%c%c\", chains Tx=%u Rx=%u\n",
 		hwinfo->fw_ver, hwinfo->mac_bitmap,
-		hwinfo->alpha2_code[0], hwinfo->alpha2_code[1],
+		hwinfo->rd->alpha2[0], hwinfo->rd->alpha2[1],
 		hwinfo->total_tx_chain, hwinfo->total_rx_chain);
 
 	return 0;
@@ -878,7 +989,7 @@ static int qtnf_parse_variable_mac_info(struct qtnf_wmac *mac,
 	struct ieee80211_iface_limit *limits = NULL;
 	const struct qlink_iface_limit *limit_record;
 	size_t record_count = 0, rec = 0;
-	u16 tlv_type, tlv_value_len, mask;
+	u16 tlv_type, tlv_value_len;
 	struct qlink_iface_comb_num *comb;
 	size_t tlv_full_len;
 	const struct qlink_tlv_hdr *tlv;
@@ -931,10 +1042,12 @@ static int qtnf_parse_variable_mac_info(struct qtnf_wmac *mac,
 
 			limit_record = (void *)tlv->val;
 			limits[rec].max = le16_to_cpu(limit_record->max_num);
-			mask = le16_to_cpu(limit_record->type_mask);
-			limits[rec].types = qlink_iface_type_mask_to_nl(mask);
-			/* only AP and STA modes are supported */
+			limits[rec].types = qlink_iface_type_to_nl_mask(
+				le16_to_cpu(limit_record->type));
+
+			/* supported modes: STA, AP */
 			limits[rec].types &= BIT(NL80211_IFTYPE_AP) |
+					     BIT(NL80211_IFTYPE_AP_VLAN) |
 					     BIT(NL80211_IFTYPE_STATION);
 
 			pr_debug("MAC%u: MAX: %u; TYPES: %.4X\n", mac->macid,
@@ -946,6 +1059,7 @@ static int qtnf_parse_variable_mac_info(struct qtnf_wmac *mac,
 		default:
 			break;
 		}
+
 		tlv_buf_size -= tlv_full_len;
 		tlv = (struct qlink_tlv_hdr *)(tlv->val + tlv_value_len);
 	}
@@ -1013,14 +1127,24 @@ qtnf_cmd_resp_fill_channels_info(struct ieee80211_supported_band *band,
 	unsigned int chidx = 0;
 	u32 qflags;
 
-	kfree(band->channels);
-	band->channels = NULL;
+	if (band->channels) {
+		if (band->n_channels == resp->num_chans) {
+			memset(band->channels, 0,
+			       sizeof(*band->channels) * band->n_channels);
+		} else {
+			kfree(band->channels);
+			band->n_channels = 0;
+			band->channels = NULL;
+		}
+	}
 
 	band->n_channels = resp->num_chans;
 	if (band->n_channels == 0)
 		return 0;
 
-	band->channels = kcalloc(band->n_channels, sizeof(*chan), GFP_KERNEL);
+	if (!band->channels)
+		band->channels = kcalloc(band->n_channels, sizeof(*chan),
+					 GFP_KERNEL);
 	if (!band->channels) {
 		band->n_channels = 0;
 		return -ENOMEM;
@@ -1212,6 +1336,62 @@ static int qtnf_cmd_resp_proc_phy_params(struct qtnf_wmac *mac,
 	return 0;
 }
 
+static int
+qtnf_cmd_resp_proc_chan_stat_info(struct qtnf_chan_stats *stats,
+				  const u8 *payload, size_t payload_len)
+{
+	struct qlink_chan_stats *qlink_stats;
+	const struct qlink_tlv_hdr *tlv;
+	size_t tlv_full_len;
+	u16 tlv_value_len;
+	u16 tlv_type;
+
+	tlv = (struct qlink_tlv_hdr *)payload;
+	while (payload_len >= sizeof(struct qlink_tlv_hdr)) {
+		tlv_type = le16_to_cpu(tlv->type);
+		tlv_value_len = le16_to_cpu(tlv->len);
+		tlv_full_len = tlv_value_len + sizeof(struct qlink_tlv_hdr);
+		if (tlv_full_len > payload_len) {
+			pr_warn("malformed TLV 0x%.2X; LEN: %u\n",
+				tlv_type, tlv_value_len);
+			return -EINVAL;
+		}
+		switch (tlv_type) {
+		case QTN_TLV_ID_CHANNEL_STATS:
+			if (unlikely(tlv_value_len != sizeof(*qlink_stats))) {
+				pr_err("invalid CHANNEL_STATS entry size\n");
+				return -EINVAL;
+			}
+
+			qlink_stats = (void *)tlv->val;
+
+			stats->chan_num = le32_to_cpu(qlink_stats->chan_num);
+			stats->cca_tx = le32_to_cpu(qlink_stats->cca_tx);
+			stats->cca_rx = le32_to_cpu(qlink_stats->cca_rx);
+			stats->cca_busy = le32_to_cpu(qlink_stats->cca_busy);
+			stats->cca_try = le32_to_cpu(qlink_stats->cca_try);
+			stats->chan_noise = qlink_stats->chan_noise;
+
+			pr_debug("chan(%u) try(%u) busy(%u) noise(%d)\n",
+				 stats->chan_num, stats->cca_try,
+				 stats->cca_busy, stats->chan_noise);
+			break;
+		default:
+			pr_warn("Unknown TLV type: %#x\n",
+				le16_to_cpu(tlv->type));
+		}
+		payload_len -= tlv_full_len;
+		tlv = (struct qlink_tlv_hdr *)(tlv->val + tlv_value_len);
+	}
+
+	if (payload_len) {
+		pr_warn("malformed TLV buf; bytes left: %zu\n", payload_len);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 int qtnf_cmd_get_mac_info(struct qtnf_wmac *mac)
 {
 	struct sk_buff *cmd_skb, *resp_skb = NULL;
@@ -1256,6 +1436,7 @@ int qtnf_cmd_get_hw_info(struct qtnf_bus *bus)
 	const struct qlink_resp_get_hw_info *resp;
 	u16 res_code = QLINK_CMD_RESULT_OK;
 	int ret = 0;
+	size_t info_len;
 
 	cmd_skb = qtnf_cmd_alloc_new_cmdskb(QLINK_MACID_RSVD, QLINK_VIFID_RSVD,
 					    QLINK_CMD_GET_HW_INFO,
@@ -1266,7 +1447,7 @@ int qtnf_cmd_get_hw_info(struct qtnf_bus *bus)
 	qtnf_bus_lock(bus);
 
 	ret = qtnf_cmd_send_with_reply(bus, cmd_skb, &resp_skb, &res_code,
-				       sizeof(*resp), NULL);
+				       sizeof(*resp), &info_len);
 
 	if (unlikely(ret))
 		goto out;
@@ -1278,7 +1459,7 @@ int qtnf_cmd_get_hw_info(struct qtnf_bus *bus)
 	}
 
 	resp = (const struct qlink_resp_get_hw_info *)resp_skb->data;
-	ret = qtnf_cmd_resp_proc_hw_info(bus, resp);
+	ret = qtnf_cmd_resp_proc_hw_info(bus, resp, info_len);
 
 out:
 	qtnf_bus_unlock(bus);
@@ -1320,6 +1501,9 @@ int qtnf_cmd_get_mac_chan_info(struct qtnf_wmac *mac,
 
 	cmd = (struct qlink_cmd_chans_info_get *)cmd_skb->data;
 	cmd->band = qband;
+
+	qtnf_bus_lock(mac->bus);
+
 	ret = qtnf_cmd_send_with_reply(mac->bus, cmd_skb, &resp_skb, &res_code,
 				       sizeof(*resp), &info_len);
 
@@ -1343,6 +1527,7 @@ int qtnf_cmd_get_mac_chan_info(struct qtnf_wmac *mac,
 	ret = qtnf_cmd_resp_fill_channels_info(band, resp, info_len);
 
 out:
+	qtnf_bus_unlock(mac->bus);
 	consume_skb(resp_skb);
 
 	return ret;
@@ -1676,10 +1861,27 @@ int qtnf_cmd_send_change_sta(struct qtnf_vif *vif, const u8 *mac,
 
 	cmd = (struct qlink_cmd_change_sta *)cmd_skb->data;
 	ether_addr_copy(cmd->sta_addr, mac);
-	cmd->sta_flags_mask = cpu_to_le32(qtnf_encode_sta_flags(
-					  params->sta_flags_mask));
-	cmd->sta_flags_set = cpu_to_le32(qtnf_encode_sta_flags(
-					 params->sta_flags_set));
+
+	switch (vif->wdev.iftype) {
+	case NL80211_IFTYPE_AP:
+		cmd->if_type = cpu_to_le16(QLINK_IFTYPE_AP);
+		cmd->sta_flags_mask = cpu_to_le32(qtnf_encode_sta_flags(
+						  params->sta_flags_mask));
+		cmd->sta_flags_set = cpu_to_le32(qtnf_encode_sta_flags(
+						 params->sta_flags_set));
+		break;
+	case NL80211_IFTYPE_STATION:
+		cmd->if_type = cpu_to_le16(QLINK_IFTYPE_STATION);
+		cmd->sta_flags_mask = cpu_to_le32(qtnf_encode_sta_flags(
+						  params->sta_flags_mask));
+		cmd->sta_flags_set = cpu_to_le32(qtnf_encode_sta_flags(
+						 params->sta_flags_set));
+		break;
+	default:
+		pr_err("unsupported iftype %d\n", vif->wdev.iftype);
+		ret = -EINVAL;
+		goto out;
+	}
 
 	ret = qtnf_cmd_send(vif->mac->bus, cmd_skb, &res_code);
 	if (unlikely(ret))
@@ -1853,8 +2055,8 @@ int qtnf_cmd_send_connect(struct qtnf_vif *vif,
 
 	ether_addr_copy(cmd->bssid, bss_cfg->bssid);
 
-	if (bss_cfg->chandef.chan)
-		cmd->freq = cpu_to_le16(bss_cfg->chandef.chan->center_freq);
+	if (vif->mac->chandef.chan)
+		cmd->channel = cpu_to_le16(vif->mac->chandef.chan->hw_value);
 
 	cmd->bg_scan_period = cpu_to_le16(bss_cfg->bg_scan_period);
 
@@ -1976,3 +2178,183 @@ out:
 	qtnf_bus_unlock(vif->mac->bus);
 	return ret;
 }
+
+int qtnf_cmd_reg_notify(struct qtnf_bus *bus, struct regulatory_request *req)
+{
+	struct sk_buff *cmd_skb;
+	int ret;
+	u16 res_code;
+	struct qlink_cmd_reg_notify *cmd;
+
+	cmd_skb = qtnf_cmd_alloc_new_cmdskb(QLINK_MACID_RSVD, QLINK_VIFID_RSVD,
+					    QLINK_CMD_REG_NOTIFY,
+					    sizeof(*cmd));
+	if (!cmd_skb)
+		return -ENOMEM;
+
+	cmd = (struct qlink_cmd_reg_notify *)cmd_skb->data;
+	cmd->alpha2[0] = req->alpha2[0];
+	cmd->alpha2[1] = req->alpha2[1];
+
+	switch (req->initiator) {
+	case NL80211_REGDOM_SET_BY_CORE:
+		cmd->initiator = QLINK_REGDOM_SET_BY_CORE;
+		break;
+	case NL80211_REGDOM_SET_BY_USER:
+		cmd->initiator = QLINK_REGDOM_SET_BY_USER;
+		break;
+	case NL80211_REGDOM_SET_BY_DRIVER:
+		cmd->initiator = QLINK_REGDOM_SET_BY_DRIVER;
+		break;
+	case NL80211_REGDOM_SET_BY_COUNTRY_IE:
+		cmd->initiator = QLINK_REGDOM_SET_BY_COUNTRY_IE;
+		break;
+	}
+
+	switch (req->user_reg_hint_type) {
+	case NL80211_USER_REG_HINT_USER:
+		cmd->user_reg_hint_type = QLINK_USER_REG_HINT_USER;
+		break;
+	case NL80211_USER_REG_HINT_CELL_BASE:
+		cmd->user_reg_hint_type = QLINK_USER_REG_HINT_CELL_BASE;
+		break;
+	case NL80211_USER_REG_HINT_INDOOR:
+		cmd->user_reg_hint_type = QLINK_USER_REG_HINT_INDOOR;
+		break;
+	}
+
+	qtnf_bus_lock(bus);
+
+	ret = qtnf_cmd_send(bus, cmd_skb, &res_code);
+	if (ret)
+		goto out;
+
+	switch (res_code) {
+	case QLINK_CMD_RESULT_ENOTSUPP:
+		pr_warn("reg update not supported\n");
+		ret = -EOPNOTSUPP;
+		break;
+	case QLINK_CMD_RESULT_EALREADY:
+		pr_info("regulatory domain is already set to %c%c",
+			req->alpha2[0], req->alpha2[1]);
+		ret = -EALREADY;
+		break;
+	case QLINK_CMD_RESULT_OK:
+		ret = 0;
+		break;
+	default:
+		ret = -EFAULT;
+		break;
+	}
+
+out:
+	qtnf_bus_unlock(bus);
+
+	return ret;
+}
+
+int qtnf_cmd_get_chan_stats(struct qtnf_wmac *mac, u16 channel,
+			    struct qtnf_chan_stats *stats)
+{
+	struct sk_buff *cmd_skb, *resp_skb = NULL;
+	struct qlink_cmd_get_chan_stats *cmd;
+	struct qlink_resp_get_chan_stats *resp;
+	size_t var_data_len;
+	u16 res_code = QLINK_CMD_RESULT_OK;
+	int ret = 0;
+
+	cmd_skb = qtnf_cmd_alloc_new_cmdskb(mac->macid, QLINK_VIFID_RSVD,
+					    QLINK_CMD_CHAN_STATS,
+					    sizeof(*cmd));
+	if (!cmd_skb)
+		return -ENOMEM;
+
+	qtnf_bus_lock(mac->bus);
+
+	cmd = (struct qlink_cmd_get_chan_stats *)cmd_skb->data;
+	cmd->channel = cpu_to_le16(channel);
+
+	ret = qtnf_cmd_send_with_reply(mac->bus, cmd_skb, &resp_skb, &res_code,
+				       sizeof(*resp), &var_data_len);
+	if (unlikely(ret)) {
+		qtnf_bus_unlock(mac->bus);
+		return ret;
+	}
+
+	if (unlikely(res_code != QLINK_CMD_RESULT_OK)) {
+		switch (res_code) {
+		case QLINK_CMD_RESULT_ENOTFOUND:
+			ret = -ENOENT;
+			break;
+		default:
+			pr_err("cmd exec failed: 0x%.4X\n", res_code);
+			ret = -EFAULT;
+			break;
+		}
+		goto out;
+	}
+
+	resp = (struct qlink_resp_get_chan_stats *)resp_skb->data;
+	ret = qtnf_cmd_resp_proc_chan_stat_info(stats, resp->info,
+						var_data_len);
+
+out:
+	qtnf_bus_unlock(mac->bus);
+	consume_skb(resp_skb);
+	return ret;
+}
+
+int qtnf_cmd_send_chan_switch(struct qtnf_wmac *mac,
+			      struct cfg80211_csa_settings *params)
+{
+	struct qlink_cmd_chan_switch *cmd;
+	struct sk_buff *cmd_skb;
+	u16 res_code = QLINK_CMD_RESULT_OK;
+	int ret;
+
+	cmd_skb = qtnf_cmd_alloc_new_cmdskb(mac->macid, 0x0,
+					    QLINK_CMD_CHAN_SWITCH,
+					    sizeof(*cmd));
+
+	if (unlikely(!cmd_skb))
+		return -ENOMEM;
+
+	qtnf_bus_lock(mac->bus);
+
+	cmd = (struct qlink_cmd_chan_switch *)cmd_skb->data;
+	cmd->channel = cpu_to_le16(params->chandef.chan->hw_value);
+	cmd->radar_required = params->radar_required;
+	cmd->block_tx = params->block_tx;
+	cmd->beacon_count = params->count;
+
+	ret = qtnf_cmd_send(mac->bus, cmd_skb, &res_code);
+
+	if (unlikely(ret))
+		goto out;
+
+	switch (res_code) {
+	case QLINK_CMD_RESULT_OK:
+		memcpy(&mac->csa_chandef, &params->chandef,
+		       sizeof(mac->csa_chandef));
+		mac->status |= QTNF_MAC_CSA_ACTIVE;
+		ret = 0;
+		break;
+	case QLINK_CMD_RESULT_ENOTFOUND:
+		ret = -ENOENT;
+		break;
+	case QLINK_CMD_RESULT_ENOTSUPP:
+		ret = -EOPNOTSUPP;
+		break;
+	case QLINK_CMD_RESULT_EALREADY:
+		ret = -EALREADY;
+		break;
+	case QLINK_CMD_RESULT_INVALID:
+	default:
+		ret = -EFAULT;
+		break;
+	}
+
+out:
+	qtnf_bus_unlock(mac->bus);
+	return ret;
+}
diff --git a/drivers/net/wireless/quantenna/qtnfmac/commands.h b/drivers/net/wireless/quantenna/qtnfmac/commands.h
index 6c51854ef5e7..783b20364296 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/commands.h
+++ b/drivers/net/wireless/quantenna/qtnfmac/commands.h
@@ -70,5 +70,10 @@ int qtnf_cmd_send_disconnect(struct qtnf_vif *vif,
 			     u16 reason_code);
 int qtnf_cmd_send_updown_intf(struct qtnf_vif *vif,
 			      bool up);
+int qtnf_cmd_reg_notify(struct qtnf_bus *bus, struct regulatory_request *req);
+int qtnf_cmd_get_chan_stats(struct qtnf_wmac *mac, u16 channel,
+			    struct qtnf_chan_stats *stats);
+int qtnf_cmd_send_chan_switch(struct qtnf_wmac *mac,
+			      struct cfg80211_csa_settings *params);
 
 #endif /* QLINK_COMMANDS_H_ */
diff --git a/drivers/net/wireless/quantenna/qtnfmac/core.c b/drivers/net/wireless/quantenna/qtnfmac/core.c
index f053532c0e87..5e60180482d1 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/core.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/core.c
@@ -288,6 +288,8 @@ static struct qtnf_wmac *qtnf_core_mac_alloc(struct qtnf_bus *bus,
 		mac->iflist[i].mac = mac;
 		mac->iflist[i].vifid = i;
 		qtnf_sta_list_init(&mac->iflist[i].sta_list);
+		mutex_init(&mac->mac_lock);
+		init_timer(&mac->scan_timeout);
 	}
 
 	qtnf_mac_init_primary_intf(mac);
@@ -549,6 +551,9 @@ void qtnf_core_detach(struct qtnf_bus *bus)
 		destroy_workqueue(bus->workqueue);
 	}
 
+	kfree(bus->hw_info.rd);
+	bus->hw_info.rd = NULL;
+
 	qtnf_trans_free(bus);
 }
 EXPORT_SYMBOL_GPL(qtnf_core_detach);
diff --git a/drivers/net/wireless/quantenna/qtnfmac/core.h b/drivers/net/wireless/quantenna/qtnfmac/core.h
index a616434281cf..066fcd1095a0 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/core.h
+++ b/drivers/net/wireless/quantenna/qtnfmac/core.h
@@ -42,11 +42,11 @@
 
 #define QTNF_MAX_SSID_LIST_LENGTH	2
 #define QTNF_MAX_VSIE_LEN		255
-#define QTNF_MAX_ALPHA_LEN		2
 #define QTNF_MAX_INTF			8
 #define QTNF_MAX_EVENT_QUEUE_LEN	255
 #define QTNF_DEFAULT_BG_SCAN_PERIOD	300
 #define QTNF_MAX_BG_SCAN_PERIOD		0xffff
+#define QTNF_SCAN_TIMEOUT_SEC		15
 
 #define QTNF_DEF_BSS_PRIORITY		0
 #define QTNF_DEF_WDOG_TIMEOUT		5
@@ -68,7 +68,6 @@ struct qtnf_bss_config {
 	u16 auth_type;
 	bool privacy;
 	enum nl80211_mfp mfp;
-	struct cfg80211_chan_def chandef;
 	struct cfg80211_crypto_settings crypto;
 	u16 bg_scan_period;
 	u32 connect_flags;
@@ -90,6 +89,10 @@ enum qtnf_sta_state {
 	QTNF_STA_CONNECTED
 };
 
+enum qtnf_mac_status {
+	QTNF_MAC_CSA_ACTIVE	= BIT(0)
+};
+
 struct qtnf_vif {
 	struct wireless_dev wdev;
 	u8 vifid;
@@ -125,25 +128,39 @@ struct qtnf_mac_info {
 	size_t n_limits;
 };
 
+struct qtnf_chan_stats {
+	u32 chan_num;
+	u32 cca_tx;
+	u32 cca_rx;
+	u32 cca_busy;
+	u32 cca_try;
+	s8 chan_noise;
+};
+
 struct qtnf_wmac {
 	u8 macid;
 	u8 wiphy_registered;
 	u8 macaddr[ETH_ALEN];
+	u32 status;
 	struct qtnf_bus *bus;
 	struct qtnf_mac_info macinfo;
 	struct qtnf_vif iflist[QTNF_MAX_INTF];
 	struct cfg80211_scan_request *scan_req;
+	struct cfg80211_chan_def chandef;
+	struct cfg80211_chan_def csa_chandef;
+	struct mutex mac_lock;	/* lock during wmac speicific ops */
+	struct timer_list scan_timeout;
 };
 
 struct qtnf_hw_info {
+	u16 ql_proto_ver;
 	u8 num_mac;
 	u8 mac_bitmap;
-	u8 alpha2_code[QTNF_MAX_ALPHA_LEN];
 	u32 fw_ver;
-	u16 ql_proto_ver;
+	u32 hw_capab;
+	struct ieee80211_regdomain *rd;
 	u8 total_tx_chain;
 	u8 total_rx_chain;
-	u32 hw_capab;
 };
 
 struct qtnf_vif *qtnf_mac_get_free_vif(struct qtnf_wmac *mac);
diff --git a/drivers/net/wireless/quantenna/qtnfmac/event.c b/drivers/net/wireless/quantenna/qtnfmac/event.c
index 9b61e9a83670..0fc2814eafad 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/event.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/event.c
@@ -211,8 +211,8 @@ qtnf_event_handle_bss_leave(struct qtnf_vif *vif,
 
 	pr_debug("VIF%u.%u: disconnected\n", vif->mac->macid, vif->vifid);
 
-	cfg80211_disconnected(vif->netdev, leave_info->reason, NULL, 0, 0,
-			      GFP_KERNEL);
+	cfg80211_disconnected(vif->netdev, le16_to_cpu(leave_info->reason),
+			      NULL, 0, 0, GFP_KERNEL);
 
 	vif->sta_state = QTNF_STA_DISCONNECTED;
 	netif_carrier_off(vif->netdev);
@@ -345,11 +345,70 @@ qtnf_event_handle_scan_complete(struct qtnf_wmac *mac,
 		return -EINVAL;
 	}
 
+	if (timer_pending(&mac->scan_timeout))
+		del_timer_sync(&mac->scan_timeout);
 	qtnf_scan_done(mac, le32_to_cpu(status->flags) & QLINK_SCAN_ABORTED);
 
 	return 0;
 }
 
+static int
+qtnf_event_handle_freq_change(struct qtnf_wmac *mac,
+			      const struct qlink_event_freq_change *data,
+			      u16 len)
+{
+	struct wiphy *wiphy = priv_to_wiphy(mac);
+	struct cfg80211_chan_def chandef;
+	struct ieee80211_channel *chan;
+	struct qtnf_vif *vif;
+	int freq;
+	int i;
+
+	if (len < sizeof(*data)) {
+		pr_err("payload is too short\n");
+		return -EINVAL;
+	}
+
+	freq = le32_to_cpu(data->freq);
+	chan = ieee80211_get_channel(wiphy, freq);
+	if (!chan) {
+		pr_err("channel at %d MHz not found\n", freq);
+		return -EINVAL;
+	}
+
+	pr_debug("MAC%d switch to new channel %u MHz\n", mac->macid, freq);
+
+	if (mac->status & QTNF_MAC_CSA_ACTIVE) {
+		mac->status &= ~QTNF_MAC_CSA_ACTIVE;
+		if (chan->hw_value != mac->csa_chandef.chan->hw_value)
+			pr_warn("unexpected switch to %u during CSA to %u\n",
+				chan->hw_value,
+				mac->csa_chandef.chan->hw_value);
+	}
+
+	/* FIXME: need to figure out proper nl80211_channel_type value */
+	cfg80211_chandef_create(&chandef, chan, NL80211_CHAN_HT20);
+	/* fall-back to minimal safe chandef description */
+	if (!cfg80211_chandef_valid(&chandef))
+		cfg80211_chandef_create(&chandef, chan, NL80211_CHAN_HT20);
+
+	memcpy(&mac->chandef, &chandef, sizeof(mac->chandef));
+
+	for (i = 0; i < QTNF_MAX_INTF; i++) {
+		vif = &mac->iflist[i];
+		if (vif->wdev.iftype == NL80211_IFTYPE_UNSPECIFIED)
+			continue;
+
+		if (vif->netdev) {
+			mutex_lock(&vif->wdev.mtx);
+			cfg80211_ch_switch_notify(vif->netdev, &chandef);
+			mutex_unlock(&vif->wdev.mtx);
+		}
+	}
+
+	return 0;
+}
+
 static int qtnf_event_parse(struct qtnf_wmac *mac,
 			    const struct sk_buff *event_skb)
 {
@@ -400,6 +459,10 @@ static int qtnf_event_parse(struct qtnf_wmac *mac,
 		ret = qtnf_event_handle_bss_leave(vif, (const void *)event,
 						  event_len);
 		break;
+	case QLINK_EVENT_FREQ_CHANGE:
+		ret = qtnf_event_handle_freq_change(mac, (const void *)event,
+						    event_len);
+		break;
 	default:
 		pr_warn("unknown event type: %x\n", event_id);
 		break;
diff --git a/drivers/net/wireless/quantenna/qtnfmac/pearl/pcie.c b/drivers/net/wireless/quantenna/qtnfmac/pearl/pcie.c
index 7fc4f0d6a9ad..502e72b7cdcc 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/pearl/pcie.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/pearl/pcie.c
@@ -25,6 +25,8 @@
 #include <linux/completion.h>
 #include <linux/crc32.h>
 #include <linux/spinlock.h>
+#include <linux/circ_buf.h>
+#include <linux/log2.h>
 
 #include "qtn_hw_ids.h"
 #include "pcie_bus_priv.h"
@@ -36,17 +38,13 @@ static bool use_msi = true;
 module_param(use_msi, bool, 0644);
 MODULE_PARM_DESC(use_msi, "set 0 to use legacy interrupt");
 
-static unsigned int tx_bd_size_param = 256;
+static unsigned int tx_bd_size_param = 32;
 module_param(tx_bd_size_param, uint, 0644);
-MODULE_PARM_DESC(tx_bd_size_param, "Tx descriptors queue size");
+MODULE_PARM_DESC(tx_bd_size_param, "Tx descriptors queue size, power of two");
 
 static unsigned int rx_bd_size_param = 256;
 module_param(rx_bd_size_param, uint, 0644);
-MODULE_PARM_DESC(rx_bd_size_param, "Rx descriptors queue size");
-
-static unsigned int rx_bd_reserved_param = 16;
-module_param(rx_bd_reserved_param, uint, 0644);
-MODULE_PARM_DESC(rx_bd_reserved_param, "Reserved RX descriptors");
+MODULE_PARM_DESC(rx_bd_size_param, "Rx descriptors queue size, power of two");
 
 static u8 flashboot = 1;
 module_param(flashboot, byte, 0644);
@@ -186,8 +184,10 @@ static void __iomem *qtnf_map_bar(struct qtnf_pcie_bus_priv *priv, u8 index)
 		return IOMEM_ERR_PTR(ret);
 
 	busaddr = pci_resource_start(priv->pdev, index);
-	vaddr = pcim_iomap_table(priv->pdev)[index];
 	len = pci_resource_len(priv->pdev, index);
+	vaddr = pcim_iomap_table(priv->pdev)[index];
+	if (!vaddr)
+		return IOMEM_ERR_PTR(-ENOMEM);
 
 	pr_debug("BAR%u vaddr=0x%p busaddr=%pad len=%u\n",
 		 index, vaddr, &busaddr, (int)len);
@@ -250,19 +250,19 @@ static int qtnf_pcie_init_memory(struct qtnf_pcie_bus_priv *priv)
 	int ret = -ENOMEM;
 
 	priv->sysctl_bar = qtnf_map_bar(priv, QTN_SYSCTL_BAR);
-	if (IS_ERR_OR_NULL(priv->sysctl_bar)) {
+	if (IS_ERR(priv->sysctl_bar)) {
 		pr_err("failed to map BAR%u\n", QTN_SYSCTL_BAR);
 		return ret;
 	}
 
 	priv->dmareg_bar = qtnf_map_bar(priv, QTN_DMA_BAR);
-	if (IS_ERR_OR_NULL(priv->dmareg_bar)) {
+	if (IS_ERR(priv->dmareg_bar)) {
 		pr_err("failed to map BAR%u\n", QTN_DMA_BAR);
 		return ret;
 	}
 
 	priv->epmem_bar = qtnf_map_bar(priv, QTN_SHMEM_BAR);
-	if (IS_ERR_OR_NULL(priv->epmem_bar)) {
+	if (IS_ERR(priv->epmem_bar)) {
 		pr_err("failed to map BAR%u\n", QTN_SHMEM_BAR);
 		return ret;
 	}
@@ -274,32 +274,6 @@ static int qtnf_pcie_init_memory(struct qtnf_pcie_bus_priv *priv)
 	return 0;
 }
 
-static int
-qtnf_pcie_init_dma_mask(struct qtnf_pcie_bus_priv *priv, u64 dma_mask)
-{
-	int ret;
-
-	ret = dma_supported(&priv->pdev->dev, dma_mask);
-	if (!ret) {
-		pr_err("DMA mask %llu not supported\n", dma_mask);
-		return ret;
-	}
-
-	ret = pci_set_dma_mask(priv->pdev, dma_mask);
-	if (ret) {
-		pr_err("failed to set DMA mask %llu\n", dma_mask);
-		return ret;
-	}
-
-	ret = pci_set_consistent_dma_mask(priv->pdev, dma_mask);
-	if (ret) {
-		pr_err("failed to set consistent DMA mask %llu\n", dma_mask);
-		return ret;
-	}
-
-	return ret;
-}
-
 static void qtnf_tune_pcie_mps(struct qtnf_pcie_bus_priv *priv)
 {
 	struct pci_dev *pdev = priv->pdev;
@@ -418,9 +392,8 @@ static int alloc_bd_table(struct qtnf_pcie_bus_priv *priv)
 
 	pr_debug("TX descriptor table: vaddr=0x%p paddr=%pad\n", vaddr, &paddr);
 
-	priv->tx_bd_reclaim_start = 0;
-	priv->tx_bd_index = 0;
-	priv->tx_queue_len = 0;
+	priv->tx_bd_r_index = 0;
+	priv->tx_bd_w_index = 0;
 
 	/* rx bd */
 
@@ -430,43 +403,34 @@ static int alloc_bd_table(struct qtnf_pcie_bus_priv *priv)
 	priv->rx_bd_vbase = vaddr;
 	priv->rx_bd_pbase = paddr;
 
-	writel(QTN_HOST_LO32(paddr),
-	       PCIE_HDP_TX_HOST_Q_BASE_L(priv->pcie_reg_base));
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
 	writel(QTN_HOST_HI32(paddr),
 	       PCIE_HDP_TX_HOST_Q_BASE_H(priv->pcie_reg_base));
+#endif
+	writel(QTN_HOST_LO32(paddr),
+	       PCIE_HDP_TX_HOST_Q_BASE_L(priv->pcie_reg_base));
 	writel(priv->rx_bd_num | (sizeof(struct qtnf_rx_bd)) << 16,
 	       PCIE_HDP_TX_HOST_Q_SZ_CTRL(priv->pcie_reg_base));
 
-	priv->hw_txproc_wr_ptr = priv->rx_bd_num - rx_bd_reserved_param;
-
-	writel(priv->hw_txproc_wr_ptr,
-	       PCIE_HDP_TX_HOST_Q_WR_PTR(priv->pcie_reg_base));
-
 	pr_debug("RX descriptor table: vaddr=0x%p paddr=%pad\n", vaddr, &paddr);
 
-	priv->rx_bd_index = 0;
-
 	return 0;
 }
 
-static int skb2rbd_attach(struct qtnf_pcie_bus_priv *priv, u16 rx_bd_index)
+static int skb2rbd_attach(struct qtnf_pcie_bus_priv *priv, u16 index)
 {
 	struct qtnf_rx_bd *rxbd;
 	struct sk_buff *skb;
 	dma_addr_t paddr;
 
-	skb = __dev_alloc_skb(SKB_BUF_SIZE + NET_IP_ALIGN,
-			      GFP_ATOMIC);
+	skb = __netdev_alloc_skb_ip_align(NULL, SKB_BUF_SIZE, GFP_ATOMIC);
 	if (!skb) {
-		priv->rx_skb[rx_bd_index] = NULL;
+		priv->rx_skb[index] = NULL;
 		return -ENOMEM;
 	}
 
-	priv->rx_skb[rx_bd_index] = skb;
-
-	skb_reserve(skb, NET_IP_ALIGN);
-
-	rxbd = &priv->rx_bd_vbase[rx_bd_index];
+	priv->rx_skb[index] = skb;
+	rxbd = &priv->rx_bd_vbase[index];
 
 	paddr = pci_map_single(priv->pdev, skb->data,
 			       SKB_BUF_SIZE, PCI_DMA_FROMDEVICE);
@@ -475,17 +439,24 @@ static int skb2rbd_attach(struct qtnf_pcie_bus_priv *priv, u16 rx_bd_index)
 		return -ENOMEM;
 	}
 
-	writel(QTN_HOST_LO32(paddr),
-	       PCIE_HDP_HHBM_BUF_PTR(priv->pcie_reg_base));
-	writel(QTN_HOST_HI32(paddr),
-	       PCIE_HDP_HHBM_BUF_PTR_H(priv->pcie_reg_base));
-
 	/* keep rx skb paddrs in rx buffer descriptors for cleanup purposes */
 	rxbd->addr = cpu_to_le32(QTN_HOST_LO32(paddr));
 	rxbd->addr_h = cpu_to_le32(QTN_HOST_HI32(paddr));
-
 	rxbd->info = 0x0;
 
+	priv->rx_bd_w_index = index;
+
+	/* sync up all descriptor updates */
+	wmb();
+
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+	writel(QTN_HOST_HI32(paddr),
+	       PCIE_HDP_HHBM_BUF_PTR_H(priv->pcie_reg_base));
+#endif
+	writel(QTN_HOST_LO32(paddr),
+	       PCIE_HDP_HHBM_BUF_PTR(priv->pcie_reg_base));
+
+	writel(index, PCIE_HDP_TX_HOST_Q_WR_PTR(priv->pcie_reg_base));
 	return 0;
 }
 
@@ -516,7 +487,7 @@ static void free_xfer_buffers(void *data)
 
 	/* free rx buffers */
 	for (i = 0; i < priv->rx_bd_num; i++) {
-		if (priv->rx_skb[i]) {
+		if (priv->rx_skb && priv->rx_skb[i]) {
 			rxbd = &priv->rx_bd_vbase[i];
 			paddr = QTN_HOST_ADDR(le32_to_cpu(rxbd->addr_h),
 					      le32_to_cpu(rxbd->addr));
@@ -529,19 +500,72 @@ static void free_xfer_buffers(void *data)
 
 	/* free tx buffers */
 	for (i = 0; i < priv->tx_bd_num; i++) {
-		if (priv->tx_skb[i]) {
+		if (priv->tx_skb && priv->tx_skb[i]) {
 			dev_kfree_skb_any(priv->tx_skb[i]);
 			priv->tx_skb[i] = NULL;
 		}
 	}
 }
 
+static int qtnf_hhbm_init(struct qtnf_pcie_bus_priv *priv)
+{
+	u32 val;
+
+	val = readl(PCIE_HHBM_CONFIG(priv->pcie_reg_base));
+	val |= HHBM_CONFIG_SOFT_RESET;
+	writel(val, PCIE_HHBM_CONFIG(priv->pcie_reg_base));
+	usleep_range(50, 100);
+	val &= ~HHBM_CONFIG_SOFT_RESET;
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+	val |= HHBM_64BIT;
+#endif
+	writel(val, PCIE_HHBM_CONFIG(priv->pcie_reg_base));
+	writel(priv->rx_bd_num, PCIE_HHBM_Q_LIMIT_REG(priv->pcie_reg_base));
+
+	return 0;
+}
+
 static int qtnf_pcie_init_xfer(struct qtnf_pcie_bus_priv *priv)
 {
 	int ret;
+	u32 val;
 
 	priv->tx_bd_num = tx_bd_size_param;
 	priv->rx_bd_num = rx_bd_size_param;
+	priv->rx_bd_w_index = 0;
+	priv->rx_bd_r_index = 0;
+
+	if (!priv->tx_bd_num || !is_power_of_2(priv->tx_bd_num)) {
+		pr_err("tx_bd_size_param %u is not power of two\n",
+		       priv->tx_bd_num);
+		return -EINVAL;
+	}
+
+	val = priv->tx_bd_num * sizeof(struct qtnf_tx_bd);
+	if (val > PCIE_HHBM_MAX_SIZE) {
+		pr_err("tx_bd_size_param %u is too large\n",
+		       priv->tx_bd_num);
+		return -EINVAL;
+	}
+
+	if (!priv->rx_bd_num || !is_power_of_2(priv->rx_bd_num)) {
+		pr_err("rx_bd_size_param %u is not power of two\n",
+		       priv->rx_bd_num);
+		return -EINVAL;
+	}
+
+	val = priv->rx_bd_num * sizeof(dma_addr_t);
+	if (val > PCIE_HHBM_MAX_SIZE) {
+		pr_err("rx_bd_size_param %u is too large\n",
+		       priv->rx_bd_num);
+		return -EINVAL;
+	}
+
+	ret = qtnf_hhbm_init(priv);
+	if (ret) {
+		pr_err("failed to init h/w queues\n");
+		return ret;
+	}
 
 	ret = alloc_skb_array(priv);
 	if (ret) {
@@ -564,67 +588,72 @@ static int qtnf_pcie_init_xfer(struct qtnf_pcie_bus_priv *priv)
 	return ret;
 }
 
-static int qtnf_pcie_data_tx_reclaim(struct qtnf_pcie_bus_priv *priv)
+static void qtnf_pcie_data_tx_reclaim(struct qtnf_pcie_bus_priv *priv)
 {
 	struct qtnf_tx_bd *txbd;
 	struct sk_buff *skb;
+	unsigned long flags;
 	dma_addr_t paddr;
-	int last_sent;
-	int count;
+	u32 tx_done_index;
+	int count = 0;
 	int i;
 
-	last_sent = readl(PCIE_HDP_RX0DMA_CNT(priv->pcie_reg_base))
-			% priv->tx_bd_num;
-	i = priv->tx_bd_reclaim_start;
-	count = 0;
+	spin_lock_irqsave(&priv->tx_reclaim_lock, flags);
 
-	while (i != last_sent) {
-		skb = priv->tx_skb[i];
-		if (!skb)
-			break;
+	tx_done_index = readl(PCIE_HDP_RX0DMA_CNT(priv->pcie_reg_base))
+			& (priv->tx_bd_num - 1);
 
-		txbd = &priv->tx_bd_vbase[i];
-		paddr = QTN_HOST_ADDR(le32_to_cpu(txbd->addr_h),
-				      le32_to_cpu(txbd->addr));
-		pci_unmap_single(priv->pdev, paddr, skb->len, PCI_DMA_TODEVICE);
+	i = priv->tx_bd_r_index;
 
-		if (skb->dev) {
-			skb->dev->stats.tx_packets++;
-			skb->dev->stats.tx_bytes += skb->len;
+	while (CIRC_CNT(tx_done_index, i, priv->tx_bd_num)) {
+		skb = priv->tx_skb[i];
+		if (likely(skb)) {
+			txbd = &priv->tx_bd_vbase[i];
+			paddr = QTN_HOST_ADDR(le32_to_cpu(txbd->addr_h),
+					      le32_to_cpu(txbd->addr));
+			pci_unmap_single(priv->pdev, paddr, skb->len,
+					 PCI_DMA_TODEVICE);
+
+			if (skb->dev) {
+				skb->dev->stats.tx_packets++;
+				skb->dev->stats.tx_bytes += skb->len;
+
+				if (netif_queue_stopped(skb->dev))
+					netif_wake_queue(skb->dev);
+			}
 
-			if (netif_queue_stopped(skb->dev))
-				netif_wake_queue(skb->dev);
+			dev_kfree_skb_any(skb);
 		}
 
-		dev_kfree_skb_any(skb);
 		priv->tx_skb[i] = NULL;
-		priv->tx_queue_len--;
 		count++;
 
 		if (++i >= priv->tx_bd_num)
 			i = 0;
 	}
 
-	priv->tx_bd_reclaim_start = i;
 	priv->tx_reclaim_done += count;
 	priv->tx_reclaim_req++;
+	priv->tx_bd_r_index = i;
 
-	return count;
+	spin_unlock_irqrestore(&priv->tx_reclaim_lock, flags);
 }
 
-static bool qtnf_tx_queue_ready(struct qtnf_pcie_bus_priv *priv)
+static int qtnf_tx_queue_ready(struct qtnf_pcie_bus_priv *priv)
 {
-	if (priv->tx_queue_len >= priv->tx_bd_num - 1) {
+	if (!CIRC_SPACE(priv->tx_bd_w_index, priv->tx_bd_r_index,
+			priv->tx_bd_num)) {
 		pr_err_ratelimited("reclaim full Tx queue\n");
 		qtnf_pcie_data_tx_reclaim(priv);
 
-		if (priv->tx_queue_len >= priv->tx_bd_num - 1) {
+		if (!CIRC_SPACE(priv->tx_bd_w_index, priv->tx_bd_r_index,
+				priv->tx_bd_num)) {
 			priv->tx_full_count++;
-			return false;
+			return 0;
 		}
 	}
 
-	return true;
+	return 1;
 }
 
 static int qtnf_pcie_data_tx(struct qtnf_bus *bus, struct sk_buff *skb)
@@ -632,24 +661,18 @@ static int qtnf_pcie_data_tx(struct qtnf_bus *bus, struct sk_buff *skb)
 	struct qtnf_pcie_bus_priv *priv = (void *)get_bus_priv(bus);
 	dma_addr_t txbd_paddr, skb_paddr;
 	struct qtnf_tx_bd *txbd;
-	unsigned long flags;
 	int len, i;
 	u32 info;
 	int ret = 0;
 
-	spin_lock_irqsave(&priv->tx_lock, flags);
-
-	priv->tx_done_count++;
-
 	if (!qtnf_tx_queue_ready(priv)) {
 		if (skb->dev)
 			netif_stop_queue(skb->dev);
 
-		spin_unlock_irqrestore(&priv->tx_lock, flags);
 		return NETDEV_TX_BUSY;
 	}
 
-	i = priv->tx_bd_index;
+	i = priv->tx_bd_w_index;
 	priv->tx_skb[i] = skb;
 	len = skb->len;
 
@@ -673,16 +696,18 @@ static int qtnf_pcie_data_tx(struct qtnf_bus *bus, struct sk_buff *skb)
 
 	/* write new TX descriptor to PCIE_RX_FIFO on EP */
 	txbd_paddr = priv->tx_bd_pbase + i * sizeof(struct qtnf_tx_bd);
-	writel(QTN_HOST_LO32(txbd_paddr),
-	       PCIE_HDP_HOST_WR_DESC0(priv->pcie_reg_base));
+
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
 	writel(QTN_HOST_HI32(txbd_paddr),
 	       PCIE_HDP_HOST_WR_DESC0_H(priv->pcie_reg_base));
+#endif
+	writel(QTN_HOST_LO32(txbd_paddr),
+	       PCIE_HDP_HOST_WR_DESC0(priv->pcie_reg_base));
 
 	if (++i >= priv->tx_bd_num)
 		i = 0;
 
-	priv->tx_bd_index = i;
-	priv->tx_queue_len++;
+	priv->tx_bd_w_index = i;
 
 tx_done:
 	if (ret && skb) {
@@ -692,7 +717,8 @@ tx_done:
 		dev_kfree_skb_any(skb);
 	}
 
-	spin_unlock_irqrestore(&priv->tx_lock, flags);
+	qtnf_pcie_data_tx_reclaim(priv);
+	priv->tx_done_count++;
 
 	return NETDEV_TX_OK;
 }
@@ -719,14 +745,21 @@ static irqreturn_t qtnf_interrupt(int irq, void *data)
 	if (!(status & priv->pcie_irq_mask))
 		goto irq_done;
 
-	if (status & PCIE_HDP_INT_RX_BITS) {
+	if (status & PCIE_HDP_INT_RX_BITS)
 		priv->pcie_irq_rx_count++;
+
+	if (status & PCIE_HDP_INT_TX_BITS)
+		priv->pcie_irq_tx_count++;
+
+	if (status & PCIE_HDP_INT_HHBM_UF)
+		priv->pcie_irq_uf_count++;
+
+	if (status & PCIE_HDP_INT_RX_BITS) {
 		qtnf_dis_rxdone_irq(priv);
 		napi_schedule(&bus->mux_napi);
 	}
 
 	if (status & PCIE_HDP_INT_TX_BITS) {
-		priv->pcie_irq_tx_count++;
 		qtnf_dis_txdone_irq(priv);
 		tasklet_hi_schedule(&priv->reclaim_tq);
 	}
@@ -741,16 +774,19 @@ irq_done:
 	return IRQ_HANDLED;
 }
 
-static inline void hw_txproc_wr_ptr_inc(struct qtnf_pcie_bus_priv *priv)
+static int qtnf_rx_data_ready(struct qtnf_pcie_bus_priv *priv)
 {
-	u32 index;
+	u16 index = priv->rx_bd_r_index;
+	struct qtnf_rx_bd *rxbd;
+	u32 descw;
 
-	index = priv->hw_txproc_wr_ptr;
+	rxbd = &priv->rx_bd_vbase[index];
+	descw = le32_to_cpu(rxbd->info);
 
-	if (++index >= priv->rx_bd_num)
-		index = 0;
+	if (descw & QTN_TXDONE_MASK)
+		return 1;
 
-	priv->hw_txproc_wr_ptr = index;
+	return 0;
 }
 
 static int qtnf_rx_poll(struct napi_struct *napi, int budget)
@@ -762,64 +798,96 @@ static int qtnf_rx_poll(struct napi_struct *napi, int budget)
 	int processed = 0;
 	struct qtnf_rx_bd *rxbd;
 	dma_addr_t skb_paddr;
+	int consume;
 	u32 descw;
-	u16 index;
+	u32 psize;
+	u16 r_idx;
+	u16 w_idx;
 	int ret;
 
-	index = priv->rx_bd_index;
-	rxbd = &priv->rx_bd_vbase[index];
+	while (processed < budget) {
 
-	descw = le32_to_cpu(rxbd->info);
 
-	while ((descw & QTN_TXDONE_MASK) && (processed < budget)) {
-		skb = priv->rx_skb[index];
+		if (!qtnf_rx_data_ready(priv))
+			goto rx_out;
 
-		if (likely(skb)) {
-			skb_put(skb, QTN_GET_LEN(descw));
+		r_idx = priv->rx_bd_r_index;
+		rxbd = &priv->rx_bd_vbase[r_idx];
+		descw = le32_to_cpu(rxbd->info);
+
+		skb = priv->rx_skb[r_idx];
+		psize = QTN_GET_LEN(descw);
+		consume = 1;
+
+		if (!(descw & QTN_TXDONE_MASK)) {
+			pr_warn("skip invalid rxbd[%d]\n", r_idx);
+			consume = 0;
+		}
+
+		if (!skb) {
+			pr_warn("skip missing rx_skb[%d]\n", r_idx);
+			consume = 0;
+		}
 
+		if (skb && (skb_tailroom(skb) <  psize)) {
+			pr_err("skip packet with invalid length: %u > %u\n",
+			       psize, skb_tailroom(skb));
+			consume = 0;
+		}
+
+		if (skb) {
 			skb_paddr = QTN_HOST_ADDR(le32_to_cpu(rxbd->addr_h),
 						  le32_to_cpu(rxbd->addr));
 			pci_unmap_single(priv->pdev, skb_paddr, SKB_BUF_SIZE,
 					 PCI_DMA_FROMDEVICE);
+		}
 
+		if (consume) {
+			skb_put(skb, psize);
 			ndev = qtnf_classify_skb(bus, skb);
 			if (likely(ndev)) {
 				ndev->stats.rx_packets++;
 				ndev->stats.rx_bytes += skb->len;
 
 				skb->protocol = eth_type_trans(skb, ndev);
-				netif_receive_skb(skb);
+				napi_gro_receive(napi, skb);
 			} else {
 				pr_debug("drop untagged skb\n");
 				bus->mux_dev.stats.rx_dropped++;
 				dev_kfree_skb_any(skb);
 			}
-
-			processed++;
 		} else {
-			pr_err("missing rx_skb[%d]\n", index);
+			if (skb) {
+				bus->mux_dev.stats.rx_dropped++;
+				dev_kfree_skb_any(skb);
+			}
 		}
 
-		/* attached rx buffer is passed upstream: map a new one */
-		ret = skb2rbd_attach(priv, index);
-		if (likely(!ret)) {
-			if (++index >= priv->rx_bd_num)
-				index = 0;
+		priv->rx_skb[r_idx] = NULL;
+		if (++r_idx >= priv->rx_bd_num)
+			r_idx = 0;
 
-			priv->rx_bd_index = index;
-			hw_txproc_wr_ptr_inc(priv);
+		priv->rx_bd_r_index = r_idx;
 
-			rxbd = &priv->rx_bd_vbase[index];
-			descw = le32_to_cpu(rxbd->info);
-		} else {
-			pr_err("failed to allocate new rx_skb[%d]\n", index);
-			break;
+		/* repalce processed buffer by a new one */
+		w_idx = priv->rx_bd_w_index;
+		while (CIRC_SPACE(priv->rx_bd_w_index, priv->rx_bd_r_index,
+				  priv->rx_bd_num) > 0) {
+			if (++w_idx >= priv->rx_bd_num)
+				w_idx = 0;
+
+			ret = skb2rbd_attach(priv, w_idx);
+			if (ret) {
+				pr_err("failed to allocate new rx_skb[%d]\n",
+				       w_idx);
+				break;
+			}
 		}
 
-		writel(priv->hw_txproc_wr_ptr,
-		       PCIE_HDP_TX_HOST_Q_WR_PTR(priv->pcie_reg_base));
+		processed++;
 	}
 
+rx_out:
 	if (processed < budget) {
 		napi_complete(napi);
 		qtnf_en_rxdone_irq(priv);
@@ -1058,11 +1126,8 @@ static int qtnf_bringup_fw(struct qtnf_bus *bus)
 static void qtnf_reclaim_tasklet_fn(unsigned long data)
 {
 	struct qtnf_pcie_bus_priv *priv = (void *)data;
-	unsigned long flags;
 
-	spin_lock_irqsave(&priv->tx_lock, flags);
 	qtnf_pcie_data_tx_reclaim(priv);
-	spin_unlock_irqrestore(&priv->tx_lock, flags);
 	qtnf_en_txdone_irq(priv);
 }
 
@@ -1090,10 +1155,22 @@ static int qtnf_dbg_irq_stats(struct seq_file *s, void *data)
 {
 	struct qtnf_bus *bus = dev_get_drvdata(s->private);
 	struct qtnf_pcie_bus_priv *priv = get_bus_priv(bus);
+	u32 reg = readl(PCIE_HDP_INT_EN(priv->pcie_reg_base));
+	u32 status;
 
 	seq_printf(s, "pcie_irq_count(%u)\n", priv->pcie_irq_count);
 	seq_printf(s, "pcie_irq_tx_count(%u)\n", priv->pcie_irq_tx_count);
+	status = reg &  PCIE_HDP_INT_TX_BITS;
+	seq_printf(s, "pcie_irq_tx_status(%s)\n",
+		   (status == PCIE_HDP_INT_TX_BITS) ? "EN" : "DIS");
 	seq_printf(s, "pcie_irq_rx_count(%u)\n", priv->pcie_irq_rx_count);
+	status = reg &  PCIE_HDP_INT_RX_BITS;
+	seq_printf(s, "pcie_irq_rx_status(%s)\n",
+		   (status == PCIE_HDP_INT_RX_BITS) ? "EN" : "DIS");
+	seq_printf(s, "pcie_irq_uf_count(%u)\n", priv->pcie_irq_uf_count);
+	status = reg &  PCIE_HDP_INT_HHBM_UF;
+	seq_printf(s, "pcie_irq_hhbm_uf_status(%s)\n",
+		   (status == PCIE_HDP_INT_HHBM_UF) ? "EN" : "DIS");
 
 	return 0;
 }
@@ -1107,10 +1184,24 @@ static int qtnf_dbg_hdp_stats(struct seq_file *s, void *data)
 	seq_printf(s, "tx_done_count(%u)\n", priv->tx_done_count);
 	seq_printf(s, "tx_reclaim_done(%u)\n", priv->tx_reclaim_done);
 	seq_printf(s, "tx_reclaim_req(%u)\n", priv->tx_reclaim_req);
-	seq_printf(s, "tx_bd_reclaim_start(%u)\n", priv->tx_bd_reclaim_start);
-	seq_printf(s, "tx_bd_index(%u)\n", priv->tx_bd_index);
-	seq_printf(s, "rx_bd_index(%u)\n", priv->rx_bd_index);
-	seq_printf(s, "tx_queue_len(%u)\n", priv->tx_queue_len);
+
+	seq_printf(s, "tx_bd_r_index(%u)\n", priv->tx_bd_r_index);
+	seq_printf(s, "tx_bd_p_index(%u)\n",
+		   readl(PCIE_HDP_RX0DMA_CNT(priv->pcie_reg_base))
+			& (priv->tx_bd_num - 1));
+	seq_printf(s, "tx_bd_w_index(%u)\n", priv->tx_bd_w_index);
+	seq_printf(s, "tx queue len(%u)\n",
+		   CIRC_CNT(priv->tx_bd_w_index, priv->tx_bd_r_index,
+			    priv->tx_bd_num));
+
+	seq_printf(s, "rx_bd_r_index(%u)\n", priv->rx_bd_r_index);
+	seq_printf(s, "rx_bd_p_index(%u)\n",
+		   readl(PCIE_HDP_TX0DMA_CNT(priv->pcie_reg_base))
+			& (priv->rx_bd_num - 1));
+	seq_printf(s, "rx_bd_w_index(%u)\n", priv->rx_bd_w_index);
+	seq_printf(s, "rx alloc queue len(%u)\n",
+		   CIRC_SPACE(priv->rx_bd_w_index, priv->rx_bd_r_index,
+			      priv->rx_bd_num));
 
 	return 0;
 }
@@ -1157,7 +1248,7 @@ static int qtnf_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	init_completion(&bus->request_firmware_complete);
 	mutex_init(&bus->bus_lock);
 	spin_lock_init(&pcie_priv->irq_lock);
-	spin_lock_init(&pcie_priv->tx_lock);
+	spin_lock_init(&pcie_priv->tx_reclaim_lock);
 
 	/* init stats */
 	pcie_priv->tx_full_count = 0;
@@ -1165,6 +1256,7 @@ static int qtnf_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	pcie_priv->pcie_irq_count = 0;
 	pcie_priv->pcie_irq_rx_count = 0;
 	pcie_priv->pcie_irq_tx_count = 0;
+	pcie_priv->pcie_irq_uf_count = 0;
 	pcie_priv->tx_reclaim_done = 0;
 	pcie_priv->tx_reclaim_req = 0;
 
@@ -1191,6 +1283,16 @@ static int qtnf_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		pr_debug("successful init of PCI device %x\n", pdev->device);
 	}
 
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+#else
+	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
+#endif
+	if (ret) {
+		pr_err("PCIE DMA coherent mask init failed\n");
+		goto err_base;
+	}
+
 	pcim_pin_device(pdev);
 	pci_set_master(pdev);
 
@@ -1212,12 +1314,6 @@ static int qtnf_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		goto err_base;
 	}
 
-	ret = qtnf_pcie_init_dma_mask(pcie_priv, DMA_BIT_MASK(32));
-	if (ret) {
-		pr_err("PCIE DMA mask init failed\n");
-		goto err_base;
-	}
-
 	ret = devm_add_action(&pdev->dev, free_xfer_buffers, (void *)pcie_priv);
 	if (ret) {
 		pr_err("custom release callback init failed\n");
@@ -1336,7 +1432,7 @@ static SIMPLE_DEV_PM_OPS(qtnf_pcie_pm_ops, qtnf_pcie_suspend,
 			 qtnf_pcie_resume);
 #endif
 
-static struct pci_device_id qtnf_pcie_devid_table[] = {
+static const struct pci_device_id qtnf_pcie_devid_table[] = {
 	{
 		PCIE_VENDOR_ID_QUANTENNA, PCIE_DEVICE_ID_QTN_PEARL,
 		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
diff --git a/drivers/net/wireless/quantenna/qtnfmac/pearl/pcie_bus_priv.h b/drivers/net/wireless/quantenna/qtnfmac/pearl/pcie_bus_priv.h
index 2a897db2bd79..e76a23716ee0 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/pearl/pcie_bus_priv.h
+++ b/drivers/net/wireless/quantenna/qtnfmac/pearl/pcie_bus_priv.h
@@ -32,8 +32,8 @@ struct qtnf_pcie_bus_priv {
 	/* lock for irq configuration changes */
 	spinlock_t irq_lock;
 
-	/* lock for tx operations */
-	spinlock_t tx_lock;
+	/* lock for tx reclaim operations */
+	spinlock_t tx_reclaim_lock;
 	u8 msi_enabled;
 	int mps;
 
@@ -66,13 +66,11 @@ struct qtnf_pcie_bus_priv {
 	void *bd_table_vaddr;
 	u32 bd_table_len;
 
-	u32 hw_txproc_wr_ptr;
+	u32 rx_bd_w_index;
+	u32 rx_bd_r_index;
 
-	u16 tx_bd_reclaim_start;
-	u16 tx_bd_index;
-	u32 tx_queue_len;
-
-	u16 rx_bd_index;
+	u32 tx_bd_w_index;
+	u32 tx_bd_r_index;
 
 	u32 pcie_irq_mask;
 
@@ -80,6 +78,7 @@ struct qtnf_pcie_bus_priv {
 	u32 pcie_irq_count;
 	u32 pcie_irq_rx_count;
 	u32 pcie_irq_tx_count;
+	u32 pcie_irq_uf_count;
 	u32 tx_full_count;
 	u32 tx_done_count;
 	u32 tx_reclaim_done;
diff --git a/drivers/net/wireless/quantenna/qtnfmac/pearl/pcie_ipc.h b/drivers/net/wireless/quantenna/qtnfmac/pearl/pcie_ipc.h
index e00d508fbcf0..c5a4e46d26ef 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/pearl/pcie_ipc.h
+++ b/drivers/net/wireless/quantenna/qtnfmac/pearl/pcie_ipc.h
@@ -50,22 +50,21 @@
 #define PCIE_HDP_INT_RX_BITS (0		\
 	| PCIE_HDP_INT_EP_TXDMA		\
 	| PCIE_HDP_INT_EP_TXEMPTY	\
+	| PCIE_HDP_INT_HHBM_UF		\
 	)
 
 #define PCIE_HDP_INT_TX_BITS (0		\
 	| PCIE_HDP_INT_EP_RXDMA		\
 	)
 
-#if BITS_PER_LONG == 64
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
 #define QTN_HOST_HI32(a)	((u32)(((u64)a) >> 32))
 #define QTN_HOST_LO32(a)	((u32)(((u64)a) & 0xffffffffUL))
 #define QTN_HOST_ADDR(h, l)	((((u64)h) << 32) | ((u64)l))
-#elif BITS_PER_LONG == 32
+#else
 #define QTN_HOST_HI32(a)	0
 #define QTN_HOST_LO32(a)	((u32)(((u32)a) & 0xffffffffUL))
 #define QTN_HOST_ADDR(h, l)	((u32)l)
-#else
-#error Unexpected BITS_PER_LONG value
 #endif
 
 #define QTN_SYSCTL_BAR	0
@@ -75,7 +74,7 @@
 #define QTN_PCIE_BDA_VERSION		0x1002
 
 #define PCIE_BDA_NAMELEN		32
-#define PCIE_HHBM_MAX_SIZE		512
+#define PCIE_HHBM_MAX_SIZE		2048
 
 #define SKB_BUF_SIZE		2048
 
@@ -112,7 +111,7 @@ struct qtnf_pcie_bda {
 	__le32 bda_flashsz;
 	u8 bda_boardname[PCIE_BDA_NAMELEN];
 	__le32 bda_rc_msi_enabled;
-	__le32 bda_hhbm_list[PCIE_HHBM_MAX_SIZE];
+	u8 bda_hhbm_list[PCIE_HHBM_MAX_SIZE];
 	__le32 bda_dsbw_start_index;
 	__le32 bda_dsbw_end_index;
 	__le32 bda_dsbw_total_bytes;
diff --git a/drivers/net/wireless/quantenna/qtnfmac/pearl/pcie_regs_pearl.h b/drivers/net/wireless/quantenna/qtnfmac/pearl/pcie_regs_pearl.h
index 78715b8a8ef9..5b48b425fa7f 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/pearl/pcie_regs_pearl.h
+++ b/drivers/net/wireless/quantenna/qtnfmac/pearl/pcie_regs_pearl.h
@@ -109,6 +109,7 @@
 #define HHBM_WR_REQ				(BIT(0))
 #define HHBM_RD_REQ				(BIT(1))
 #define HHBM_DONE				(BIT(31))
+#define HHBM_64BIT				(BIT(10))
 
 /* offsets for dual PCIE */
 #define PCIE_PORT_LINK_CTL(base)		((base) + 0x0710)
@@ -333,6 +334,7 @@
 #define PCIE_HDP_INT_RX_LEN_ERR		(BIT(2))
 #define PCIE_HDP_INT_RX_HDR_LEN_ERR	(BIT(3))
 #define PCIE_HDP_INT_EP_TXDMA		(BIT(12))
+#define PCIE_HDP_INT_HHBM_UF		(BIT(13))
 #define PCIE_HDP_INT_EP_TXEMPTY		(BIT(15))
 #define PCIE_HDP_INT_IPC		(BIT(29))
 
diff --git a/drivers/net/wireless/quantenna/qtnfmac/qlink.h b/drivers/net/wireless/quantenna/qtnfmac/qlink.h
index 6eafc15e0065..a8242f678496 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/qlink.h
+++ b/drivers/net/wireless/quantenna/qtnfmac/qlink.h
@@ -19,7 +19,7 @@
 
 #include <linux/ieee80211.h>
 
-#define QLINK_PROTO_VER		3
+#define QLINK_PROTO_VER		5
 
 #define QLINK_MACID_RSVD		0xFF
 #define QLINK_VIFID_RSVD		0xFF
@@ -77,6 +77,7 @@ enum qlink_iface_type {
 	QLINK_IFTYPE_ADHOC	= 3,
 	QLINK_IFTYPE_MONITOR	= 4,
 	QLINK_IFTYPE_WDS	= 5,
+	QLINK_IFTYPE_AP_VLAN	= 6,
 };
 
 /**
@@ -85,12 +86,12 @@ enum qlink_iface_type {
  * Data describing a single virtual interface.
  *
  * @if_type: Mode of interface operation, one of &enum qlink_iface_type
- * @flags: interface flagsmap.
+ * @vlanid: VLAN ID for AP_VLAN interface type
  * @mac_addr: MAC address of virtual interface.
  */
 struct qlink_intf_info {
 	__le16 if_type;
-	__le16 flags;
+	__le16 vlanid;
 	u8 mac_addr[ETH_ALEN];
 	u8 rsvd[2];
 } __packed;
@@ -133,6 +134,9 @@ enum qlink_channel_width {
  *	number of operational channels and information on each of the channel.
  *	This command is generic to a specified MAC, interface index must be set
  *	to QLINK_VIFID_RSVD in command header.
+ * @QLINK_CMD_REG_NOTIFY: notify device about regulatory domain change. This
+ *	command is supported only if device reports QLINK_HW_SUPPORTS_REG_UPDATE
+ *	capability.
  */
 enum qlink_cmd_type {
 	QLINK_CMD_FW_INIT		= 0x0001,
@@ -148,8 +152,9 @@ enum qlink_cmd_type {
 	QLINK_CMD_DEL_INTF		= 0x0016,
 	QLINK_CMD_CHANGE_INTF		= 0x0017,
 	QLINK_CMD_UPDOWN_INTF		= 0x0018,
-	QLINK_CMD_REG_REGION		= 0x0019,
+	QLINK_CMD_REG_NOTIFY		= 0x0019,
 	QLINK_CMD_CHANS_INFO_GET	= 0x001A,
+	QLINK_CMD_CHAN_SWITCH		= 0x001B,
 	QLINK_CMD_CONFIG_AP		= 0x0020,
 	QLINK_CMD_START_AP		= 0x0021,
 	QLINK_CMD_STOP_AP		= 0x0022,
@@ -161,6 +166,7 @@ enum qlink_cmd_type {
 	QLINK_CMD_CHANGE_STA		= 0x0051,
 	QLINK_CMD_DEL_STA		= 0x0052,
 	QLINK_CMD_SCAN			= 0x0053,
+	QLINK_CMD_CHAN_STATS		= 0x0054,
 	QLINK_CMD_CONNECT		= 0x0060,
 	QLINK_CMD_DISCONNECT		= 0x0061,
 };
@@ -287,6 +293,7 @@ struct qlink_cmd_get_sta_info {
  * @pairwise: whether to use pairwise key.
  * @addr: MAC address of a STA key is being installed to.
  * @cipher: cipher suite.
+ * @vlanid: VLAN ID for AP_VLAN interface type
  * @key_data: key data itself.
  */
 struct qlink_cmd_add_key {
@@ -295,6 +302,7 @@ struct qlink_cmd_add_key {
 	u8 pairwise;
 	u8 addr[ETH_ALEN];
 	__le32 cipher;
+	__le16 vlanid;
 	u8 key_data[0];
 } __packed;
 
@@ -341,12 +349,16 @@ struct qlink_cmd_set_def_mgmt_key {
  *
  * @sta_flags_mask: STA flags mask, bitmap of &enum qlink_sta_flags
  * @sta_flags_set: STA flags values, bitmap of &enum qlink_sta_flags
+ * @if_type: Mode of interface operation, one of &enum qlink_iface_type
+ * @vlanid: VLAN ID to assign to specific STA
  * @sta_addr: address of the STA for which parameters are set.
  */
 struct qlink_cmd_change_sta {
 	struct qlink_cmd chdr;
 	__le32 sta_flags_mask;
 	__le32 sta_flags_set;
+	__le16 if_type;
+	__le16 vlanid;
 	u8 sta_addr[ETH_ALEN];
 } __packed;
 
@@ -380,7 +392,7 @@ enum qlink_sta_connect_flags {
 struct qlink_cmd_connect {
 	struct qlink_cmd chdr;
 	__le32 flags;
-	__le16 freq;
+	__le16 channel;
 	__le16 bg_scan_period;
 	u8 bssid[ETH_ALEN];
 	u8 payload[0];
@@ -430,6 +442,70 @@ struct qlink_cmd_chans_info_get {
 	u8 band;
 } __packed;
 
+/**
+ * struct qlink_cmd_get_chan_stats - data for QLINK_CMD_CHAN_STATS command
+ *
+ * @channel: channel number according to 802.11 17.3.8.3.2 and Annex J
+ */
+struct qlink_cmd_get_chan_stats {
+	struct qlink_cmd chdr;
+	__le16 channel;
+} __packed;
+
+/**
+ * enum qlink_reg_initiator - Indicates the initiator of a reg domain request
+ *
+ * See &enum nl80211_reg_initiator for more info.
+ */
+enum qlink_reg_initiator {
+	QLINK_REGDOM_SET_BY_CORE,
+	QLINK_REGDOM_SET_BY_USER,
+	QLINK_REGDOM_SET_BY_DRIVER,
+	QLINK_REGDOM_SET_BY_COUNTRY_IE,
+};
+
+/**
+ * enum qlink_user_reg_hint_type - type of user regulatory hint
+ *
+ * See &enum nl80211_user_reg_hint_type for more info.
+ */
+enum qlink_user_reg_hint_type {
+	QLINK_USER_REG_HINT_USER	= 0,
+	QLINK_USER_REG_HINT_CELL_BASE	= 1,
+	QLINK_USER_REG_HINT_INDOOR	= 2,
+};
+
+/**
+ * struct qlink_cmd_reg_notify - data for QLINK_CMD_REG_NOTIFY command
+ *
+ * @alpha2: the ISO / IEC 3166 alpha2 country code.
+ * @initiator: which entity sent the request, one of &enum qlink_reg_initiator.
+ * @user_reg_hint_type: type of hint for QLINK_REGDOM_SET_BY_USER request, one
+ *	of &enum qlink_user_reg_hint_type.
+ */
+struct qlink_cmd_reg_notify {
+	struct qlink_cmd chdr;
+	u8 alpha2[2];
+	u8 initiator;
+	u8 user_reg_hint_type;
+} __packed;
+
+/**
+ * struct qlink_cmd_chan_switch - data for QLINK_CMD_CHAN_SWITCH command
+ *
+ * @channel: channel number according to 802.11 17.3.8.3.2 and Annex J
+ * @radar_required: whether radar detection is required on the new channel
+ * @block_tx: whether transmissions should be blocked while changing
+ * @beacon_count: number of beacons until switch
+ */
+struct qlink_cmd_chan_switch {
+	struct qlink_cmd chdr;
+	__le16 channel;
+	u8 radar_required;
+	u8 block_tx;
+	u8 beacon_count;
+} __packed;
+
 /* QLINK Command Responses messages related definitions
  */
 
@@ -438,6 +514,7 @@ enum qlink_cmd_result {
 	QLINK_CMD_RESULT_INVALID,
 	QLINK_CMD_RESULT_ENOTSUPP,
 	QLINK_CMD_RESULT_ENOTFOUND,
+	QLINK_CMD_RESULT_EALREADY,
 };
 
 /**
@@ -497,6 +574,18 @@ struct qlink_resp_get_mac_info {
 } __packed;
 
 /**
+ * enum qlink_dfs_regions - regulatory DFS regions
+ *
+ * Corresponds to &enum nl80211_dfs_regions.
+ */
+enum qlink_dfs_regions {
+	QLINK_DFS_UNSET	= 0,
+	QLINK_DFS_FCC	= 1,
+	QLINK_DFS_ETSI	= 2,
+	QLINK_DFS_JP	= 3,
+};
+
+/**
  * struct qlink_resp_get_hw_info - response for QLINK_CMD_GET_HW_INFO command
  *
  * Description of wireless hardware capabilities and features.
@@ -504,22 +593,29 @@ struct qlink_resp_get_mac_info {
  * @fw_ver: wireless hardware firmware version.
  * @hw_capab: Bitmap of capabilities supported by firmware.
  * @ql_proto_ver: Version of QLINK protocol used by firmware.
- * @country_code: country code ID firmware is configured to.
  * @num_mac: Number of separate physical radio devices provided by hardware.
  * @mac_bitmap: Bitmap of MAC IDs that are active and can be used in firmware.
  * @total_tx_chains: total number of transmit chains used by device.
  * @total_rx_chains: total number of receive chains.
+ * @alpha2: country code ID firmware is configured to.
+ * @n_reg_rules: number of regulatory rules TLVs in variable portion of the
+ *	message.
+ * @dfs_region: regulatory DFS region, one of @enum qlink_dfs_region.
+ * @info: variable-length HW info, can contain QTN_TLV_ID_REG_RULE.
  */
 struct qlink_resp_get_hw_info {
 	struct qlink_resp rhdr;
 	__le32 fw_ver;
 	__le32 hw_capab;
 	__le16 ql_proto_ver;
-	u8 alpha2_code[2];
 	u8 num_mac;
 	u8 mac_bitmap;
 	u8 total_tx_chain;
 	u8 total_rx_chain;
+	u8 alpha2[2];
+	u8 n_reg_rules;
+	u8 dfs_region;
+	u8 info[0];
 } __packed;
 
 /**
@@ -574,6 +670,16 @@ struct qlink_resp_phy_params {
 	u8 info[0];
 } __packed;
 
+/**
+ * struct qlink_resp_get_chan_stats - response for QLINK_CMD_CHAN_STATS cmd
+ *
+ * @info: variable-length channel info.
+ */
+struct qlink_resp_get_chan_stats {
+	struct qlink_cmd rhdr;
+	u8 info[0];
+} __packed;
+
 /* QLINK Events messages related definitions
  */
 
@@ -585,6 +691,7 @@ enum qlink_event_type {
 	QLINK_EVENT_SCAN_COMPLETE	= 0x0025,
 	QLINK_EVENT_BSS_JOIN		= 0x0026,
 	QLINK_EVENT_BSS_LEAVE		= 0x0027,
+	QLINK_EVENT_FREQ_CHANGE		= 0x0028,
 };
 
 /**
@@ -651,7 +758,17 @@ struct qlink_event_bss_join {
  */
 struct qlink_event_bss_leave {
 	struct qlink_event ehdr;
-	u16 reason;
+	__le16 reason;
+} __packed;
+
+/**
+ * struct qlink_event_freq_change - data for QLINK_EVENT_FREQ_CHANGE event
+ *
+ * @freq: new operating frequency in MHz
+ */
+struct qlink_event_freq_change {
+	struct qlink_event ehdr;
+	__le32 freq;
 } __packed;
 
 enum qlink_rxmgmt_flags {
@@ -741,10 +858,12 @@ enum qlink_tlv_id {
 	QTN_TLV_ID_LRETRY_LIMIT		= 0x0204,
 	QTN_TLV_ID_BCN_PERIOD		= 0x0205,
 	QTN_TLV_ID_DTIM			= 0x0206,
+	QTN_TLV_ID_REG_RULE		= 0x0207,
 	QTN_TLV_ID_CHANNEL		= 0x020F,
 	QTN_TLV_ID_COVERAGE_CLASS	= 0x0213,
 	QTN_TLV_ID_IFACE_LIMIT		= 0x0214,
 	QTN_TLV_ID_NUM_IFACE_COMB	= 0x0215,
+	QTN_TLV_ID_CHANNEL_STATS	= 0x0216,
 	QTN_TLV_ID_STA_BASIC_COUNTERS	= 0x0300,
 	QTN_TLV_ID_STA_GENERIC_INFO	= 0x0301,
 	QTN_TLV_ID_KEY			= 0x0302,
@@ -761,7 +880,7 @@ struct qlink_tlv_hdr {
 
 struct qlink_iface_limit {
 	__le16 max_num;
-	__le16 type_mask;
+	__le16 type;
 } __packed;
 
 struct qlink_iface_comb_num {
@@ -844,12 +963,54 @@ struct qlink_tlv_cclass {
 	u8 cclass;
 } __packed;
 
-enum qlink_dfs_state {
-	QLINK_DFS_USABLE,
-	QLINK_DFS_UNAVAILABLE,
-	QLINK_DFS_AVAILABLE,
+/**
+ * enum qlink_reg_rule_flags - regulatory rule flags
+ *
+ * See description of &enum nl80211_reg_rule_flags
+ */
+enum qlink_reg_rule_flags {
+	QLINK_RRF_NO_OFDM	= BIT(0),
+	QLINK_RRF_NO_CCK	= BIT(1),
+	QLINK_RRF_NO_INDOOR	= BIT(2),
+	QLINK_RRF_NO_OUTDOOR	= BIT(3),
+	QLINK_RRF_DFS		= BIT(4),
+	QLINK_RRF_PTP_ONLY	= BIT(5),
+	QLINK_RRF_PTMP_ONLY	= BIT(6),
+	QLINK_RRF_NO_IR		= BIT(7),
+	QLINK_RRF_AUTO_BW	= BIT(8),
+	QLINK_RRF_IR_CONCURRENT	= BIT(9),
+	QLINK_RRF_NO_HT40MINUS	= BIT(10),
+	QLINK_RRF_NO_HT40PLUS	= BIT(11),
+	QLINK_RRF_NO_80MHZ	= BIT(12),
+	QLINK_RRF_NO_160MHZ	= BIT(13),
 };
 
+/**
+ * struct qlink_tlv_reg_rule - data for QTN_TLV_ID_REG_RULE TLV
+ *
+ * Regulatory rule description.
+ *
+ * @start_freq_khz: start frequency of the range the rule is attributed to.
+ * @end_freq_khz: end frequency of the range the rule is attributed to.
+ * @max_bandwidth_khz: max bandwidth that channels in specified range can be
+ *	configured to.
+ * @max_antenna_gain: max antenna gain that can be used in the specified
+ *	frequency range, dBi.
+ * @max_eirp: maximum EIRP.
+ * @flags: regulatory rule flags in &enum qlink_reg_rule_flags.
+ * @dfs_cac_ms: DFS CAC period.
+ */
+struct qlink_tlv_reg_rule {
+	struct qlink_tlv_hdr hdr;
+	__le32 start_freq_khz;
+	__le32 end_freq_khz;
+	__le32 max_bandwidth_khz;
+	__le32 max_antenna_gain;
+	__le32 max_eirp;
+	__le32 flags;
+	__le32 dfs_cac_ms;
+} __packed;
+
 enum qlink_channel_flags {
 	QLINK_CHAN_DISABLED		= BIT(0),
 	QLINK_CHAN_NO_IR		= BIT(1),
@@ -865,6 +1026,12 @@ enum qlink_channel_flags {
 	QLINK_CHAN_NO_10MHZ		= BIT(12),
 };
 
+enum qlink_dfs_state {
+	QLINK_DFS_USABLE,
+	QLINK_DFS_UNAVAILABLE,
+	QLINK_DFS_AVAILABLE,
+};
+
 struct qlink_tlv_channel {
 	struct qlink_tlv_hdr hdr;
 	__le16 hw_value;
@@ -898,4 +1065,13 @@ struct qlink_auth_encr {
 	u8 control_port_no_encrypt;
 } __packed;
 
+struct qlink_chan_stats {
+	__le32 chan_num;
+	__le32 cca_tx;
+	__le32 cca_rx;
+	__le32 cca_busy;
+	__le32 cca_try;
+	s8 chan_noise;
+} __packed;
+
 #endif /* _QTN_QLINK_H_ */
diff --git a/drivers/net/wireless/quantenna/qtnfmac/qlink_util.c b/drivers/net/wireless/quantenna/qtnfmac/qlink_util.c
index 49ae652ad9a3..cf024c995fd6 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/qlink_util.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/qlink_util.c
@@ -17,24 +17,30 @@
 
 #include "qlink_util.h"
 
-u16 qlink_iface_type_mask_to_nl(u16 qlink_mask)
+u16 qlink_iface_type_to_nl_mask(u16 qlink_type)
 {
 	u16 result = 0;
 
-	if (qlink_mask & QLINK_IFTYPE_AP)
+	switch (qlink_type) {
+	case QLINK_IFTYPE_AP:
 		result |= BIT(NL80211_IFTYPE_AP);
-
-	if (qlink_mask & QLINK_IFTYPE_STATION)
+		break;
+	case QLINK_IFTYPE_STATION:
 		result |= BIT(NL80211_IFTYPE_STATION);
-
-	if (qlink_mask & QLINK_IFTYPE_ADHOC)
+		break;
+	case QLINK_IFTYPE_ADHOC:
 		result |= BIT(NL80211_IFTYPE_ADHOC);
-
-	if (qlink_mask & QLINK_IFTYPE_MONITOR)
+		break;
+	case QLINK_IFTYPE_MONITOR:
 		result |= BIT(NL80211_IFTYPE_MONITOR);
-
-	if (qlink_mask & QLINK_IFTYPE_WDS)
+		break;
+	case QLINK_IFTYPE_WDS:
 		result |= BIT(NL80211_IFTYPE_WDS);
+		break;
+	case QLINK_IFTYPE_AP_VLAN:
+		result |= BIT(NL80211_IFTYPE_AP_VLAN);
+		break;
+	}
 
 	return result;
 }
diff --git a/drivers/net/wireless/quantenna/qtnfmac/qlink_util.h b/drivers/net/wireless/quantenna/qtnfmac/qlink_util.h
index 90d7d09a6c63..de06c1e20b5b 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/qlink_util.h
+++ b/drivers/net/wireless/quantenna/qtnfmac/qlink_util.h
@@ -22,14 +22,6 @@
 
 #include "qlink.h"
 
-static inline void qtnf_cmd_skb_put_action(struct sk_buff *skb, u16 action)
-{
-	__le16 *buf_ptr;
-
-	buf_ptr = skb_put(skb, sizeof(action));
-	*buf_ptr = cpu_to_le16(action);
-}
-
 static inline void
 qtnf_cmd_skb_put_buffer(struct sk_buff *skb, const u8 *buf_src, size_t len)
 {
@@ -68,7 +60,7 @@ static inline void qtnf_cmd_skb_put_tlv_u16(struct sk_buff *skb,
 	memcpy(hdr->val, &tmp, sizeof(tmp));
 }
 
-u16 qlink_iface_type_mask_to_nl(u16 qlink_mask);
+u16 qlink_iface_type_to_nl_mask(u16 qlink_type);
 u8 qlink_chan_width_mask_to_nl(u16 qlink_mask);
 
 #endif /* _QTN_FMAC_QLINK_UTIL_H_ */
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2500usb.c b/drivers/net/wireless/ralink/rt2x00/rt2500usb.c
index 529e05999abb..f4b48b77c491 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2500usb.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2500usb.c
@@ -1911,7 +1911,7 @@ static const struct rt2x00_ops rt2500usb_ops = {
 /*
  * rt2500usb module information.
  */
-static struct usb_device_id rt2500usb_device_table[] = {
+static const struct usb_device_id rt2500usb_device_table[] = {
 	/* ASUS */
 	{ USB_DEVICE(0x0b05, 0x1706) },
 	{ USB_DEVICE(0x0b05, 0x1707) },
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2800lib.c b/drivers/net/wireless/ralink/rt2x00/rt2800lib.c
index 0b75def39c6c..d2c289446c00 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2800lib.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2800lib.c
@@ -3702,7 +3702,10 @@ static void rt2800_config_channel(struct rt2x00_dev *rt2x00dev,
 	if (rt2x00_rt(rt2x00dev, RT3572))
 		rt2800_rfcsr_write(rt2x00dev, 8, 0);
 
-	tx_pin = rt2800_register_read(rt2x00dev, TX_PIN_CFG);
+	if (rt2x00_rt(rt2x00dev, RT6352))
+		tx_pin = rt2800_register_read(rt2x00dev, TX_PIN_CFG);
+	else
+		tx_pin = 0;
 
 	switch (rt2x00dev->default_ant.tx_chain_num) {
 	case 3:
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2800mmio.c b/drivers/net/wireless/ralink/rt2x00/rt2800mmio.c
index ee5276e233fa..1123e2bed803 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2800mmio.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2800mmio.c
@@ -136,10 +136,19 @@ void rt2800mmio_fill_rxdone(struct queue_entry *entry,
 		 */
 		rxdesc->flags |= RX_FLAG_MMIC_STRIPPED;
 
-		if (rxdesc->cipher_status == RX_CRYPTO_SUCCESS)
+		if (rxdesc->cipher_status == RX_CRYPTO_SUCCESS) {
 			rxdesc->flags |= RX_FLAG_DECRYPTED;
-		else if (rxdesc->cipher_status == RX_CRYPTO_FAIL_MIC)
+		} else if (rxdesc->cipher_status == RX_CRYPTO_FAIL_MIC) {
+			/*
+			 * In order to check the Michael Mic, the packet must have
+			 * been decrypted.  Mac80211 doesnt check the MMIC failure 
+			 * flag to initiate MMIC countermeasures if the decoded flag
+			 * has not been set.
+			 */
+			rxdesc->flags |= RX_FLAG_DECRYPTED;
+
 			rxdesc->flags |= RX_FLAG_MMIC_ERROR;
+		}
 	}
 
 	if (rt2x00_get_field32(word, RXD_W3_MY_BSS))
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2800usb.c b/drivers/net/wireless/ralink/rt2x00/rt2800usb.c
index 685b8e0cd67d..24fc6d2045ef 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2800usb.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2800usb.c
@@ -697,11 +697,20 @@ static void rt2800usb_fill_rxdone(struct queue_entry *entry,
 		 * stripped it from the frame. Signal this to mac80211.
 		 */
 		rxdesc->flags |= RX_FLAG_MMIC_STRIPPED;
-
-		if (rxdesc->cipher_status == RX_CRYPTO_SUCCESS)
+        
+		if (rxdesc->cipher_status == RX_CRYPTO_SUCCESS) {
+			rxdesc->flags |= RX_FLAG_DECRYPTED;
+		} else if (rxdesc->cipher_status == RX_CRYPTO_FAIL_MIC) {
+			/*
+			 * In order to check the Michael Mic, the packet must have
+			 * been decrypted.  Mac80211 doesnt check the MMIC failure 
+			 * flag to initiate MMIC countermeasures if the decoded flag
+			 * has not been set.
+			 */
 			rxdesc->flags |= RX_FLAG_DECRYPTED;
-		else if (rxdesc->cipher_status == RX_CRYPTO_FAIL_MIC)
+
 			rxdesc->flags |= RX_FLAG_MMIC_ERROR;
+		}
 	}
 
 	if (rt2x00_get_field32(word, RXD_W0_MY_BSS))
@@ -915,7 +924,7 @@ static const struct rt2x00_ops rt2800usb_ops = {
 /*
  * rt2800usb module information.
  */
-static struct usb_device_id rt2800usb_device_table[] = {
+static const struct usb_device_id rt2800usb_device_table[] = {
 	/* Abocom */
 	{ USB_DEVICE(0x07b8, 0x2870) },
 	{ USB_DEVICE(0x07b8, 0x2770) },
diff --git a/drivers/net/wireless/ralink/rt2x00/rt73usb.c b/drivers/net/wireless/ralink/rt2x00/rt73usb.c
index fd913222abd1..9a212823f42c 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt73usb.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt73usb.c
@@ -2408,7 +2408,7 @@ static const struct rt2x00_ops rt73usb_ops = {
 /*
  * rt73usb module information.
  */
-static struct usb_device_id rt73usb_device_table[] = {
+static const struct usb_device_id rt73usb_device_table[] = {
 	/* AboCom */
 	{ USB_DEVICE(0x07b8, 0xb21b) },
 	{ USB_DEVICE(0x07b8, 0xb21c) },
diff --git a/drivers/net/wireless/realtek/rtl818x/rtl8187/dev.c b/drivers/net/wireless/realtek/rtl818x/rtl8187/dev.c
index 55198ac2b755..121b94f09714 100644
--- a/drivers/net/wireless/realtek/rtl818x/rtl8187/dev.c
+++ b/drivers/net/wireless/realtek/rtl818x/rtl8187/dev.c
@@ -43,7 +43,7 @@ MODULE_AUTHOR("Larry Finger <Larry.Finger@lwfinger.net>");
 MODULE_DESCRIPTION("RTL8187/RTL8187B USB wireless driver");
 MODULE_LICENSE("GPL");
 
-static struct usb_device_id rtl8187_table[] = {
+static const struct usb_device_id rtl8187_table[] = {
 	/* Asus */
 	{USB_DEVICE(0x0b05, 0x171d), .driver_info = DEVICE_RTL8187},
 	/* Belkin */
diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c
index 21e5ef021260..7806a4d2b1fc 100644
--- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c
+++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c
@@ -6190,7 +6190,7 @@ static void rtl8xxxu_disconnect(struct usb_interface *interface)
 	ieee80211_free_hw(hw);
 }
 
-static struct usb_device_id dev_table[] = {
+static const struct usb_device_id dev_table[] = {
 {USB_DEVICE_AND_INTERFACE_INFO(USB_VENDOR_ID_REALTEK, 0x8724, 0xff, 0xff, 0xff),
 	.driver_info = (unsigned long)&rtl8723au_fops},
 {USB_DEVICE_AND_INTERFACE_INFO(USB_VENDOR_ID_REALTEK, 0x1724, 0xff, 0xff, 0xff),
diff --git a/drivers/net/wireless/realtek/rtlwifi/base.c b/drivers/net/wireless/realtek/rtlwifi/base.c
index e36ee592c660..ea18aa7afecb 100644
--- a/drivers/net/wireless/realtek/rtlwifi/base.c
+++ b/drivers/net/wireless/realtek/rtlwifi/base.c
@@ -426,9 +426,8 @@ static void _rtl_init_mac80211(struct ieee80211_hw *hw)
 	hw->extra_tx_headroom = RTL_TX_HEADER_SIZE;
 
 	/* TODO: Correct this value for our hw */
-	/* TODO: define these hard code value */
-	hw->max_listen_interval = 10;
-	hw->max_rate_tries = 4;
+	hw->max_listen_interval = MAX_LISTEN_INTERVAL;
+	hw->max_rate_tries = MAX_RATE_TRIES;
 	/* hw->max_rates = 1; */
 	hw->sta_data_size = sizeof(struct rtl_sta_info);
 
@@ -1166,9 +1165,9 @@ void rtl_get_tcb_desc(struct ieee80211_hw *hw,
 			}
 		}
 
-		if (is_multicast_ether_addr(ieee80211_get_DA(hdr)))
+		if (is_multicast_ether_addr(hdr->addr1))
 			tcb_desc->multicast = 1;
-		else if (is_broadcast_ether_addr(ieee80211_get_DA(hdr)))
+		else if (is_broadcast_ether_addr(hdr->addr1))
 			tcb_desc->broadcast = 1;
 
 		_rtl_txrate_selectmode(hw, sta, tcb_desc);
@@ -1408,6 +1407,11 @@ u8 rtl_is_special_data(struct ieee80211_hw *hw, struct sk_buff *skb, u8 is_tx,
 
 		return true;
 	} else if (ETH_P_PAE == ether_type) {
+		/* EAPOL is seens as in-4way */
+		rtlpriv->btcoexist.btc_info.in_4way = true;
+		rtlpriv->btcoexist.btc_info.in_4way_ts = jiffies;
+	rtlpriv->btcoexist.btc_info.in_4way_ts = jiffies;
+
 		RT_TRACE(rtlpriv, (COMP_SEND | COMP_RECV), DBG_DMESG,
 			 "802.1X %s EAPOL pkt!!\n", (is_tx) ? "Tx" : "Rx");
 
@@ -1735,12 +1739,12 @@ void rtl_scan_list_expire(struct ieee80211_hw *hw)
 			continue;
 
 		list_del(&entry->list);
-		kfree(entry);
 		rtlpriv->scan_list.num--;
 
 		RT_TRACE(rtlpriv, COMP_SCAN, DBG_LOUD,
 			 "BSSID=%pM is expire in scan list (total=%d)\n",
 			 entry->bssid, rtlpriv->scan_list.num);
+		kfree(entry);
 	}
 
 	spin_unlock_irqrestore(&rtlpriv->locks.scan_list_lock, flags);
@@ -1959,6 +1963,12 @@ label_lps_done:
 	if (rtlpriv->cfg->ops->get_btc_status())
 		rtlpriv->btcoexist.btc_ops->btc_periodical(rtlpriv);
 
+	if (rtlpriv->btcoexist.btc_info.in_4way) {
+		if (time_after(jiffies, rtlpriv->btcoexist.btc_info.in_4way_ts +
+			       msecs_to_jiffies(IN_4WAY_TIMEOUT_TIME)))
+			rtlpriv->btcoexist.btc_info.in_4way = false;
+	}
+
 	rtlpriv->link_info.bcn_rx_inperiod = 0;
 
 	/* <6> scan list */
diff --git a/drivers/net/wireless/realtek/rtlwifi/base.h b/drivers/net/wireless/realtek/rtlwifi/base.h
index ab7d81904d25..b56d1b7f5567 100644
--- a/drivers/net/wireless/realtek/rtlwifi/base.h
+++ b/drivers/net/wireless/realtek/rtlwifi/base.h
@@ -65,6 +65,8 @@ enum ap_peer {
 #define FRAME_OFFSET_ADDRESS3		16
 #define FRAME_OFFSET_SEQUENCE		22
 #define FRAME_OFFSET_ADDRESS4		24
+#define MAX_LISTEN_INTERVAL		10
+#define MAX_RATE_TRIES			4
 
 #define SET_80211_HDR_FRAME_CONTROL(_hdr, _val)		\
 	WRITEEF2BYTE(_hdr, _val)
diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbt_precomp.h b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbt_precomp.h
index 2ac989a4b2bb..02dff4c3f664 100644
--- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbt_precomp.h
+++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbt_precomp.h
@@ -43,22 +43,6 @@
 #define RT_SDIO_INTERFACE	3
 #define DEV_BUS_TYPE		RT_PCI_INTERFACE
 
-/* IC type */
-#define RTL_HW_TYPE(adapter)	(rtl_hal((struct rtl_priv *)adapter)->hw_type)
-
-#define IS_NEW_GENERATION_IC(adapter)		\
-			(RTL_HW_TYPE(adapter) >= HARDWARE_TYPE_RTL8192EE)
-#define IS_HARDWARE_TYPE_8812(adapter)		\
-			(RTL_HW_TYPE(adapter) == HARDWARE_TYPE_RTL8812AE)
-#define IS_HARDWARE_TYPE_8821(adapter)		\
-			(RTL_HW_TYPE(adapter) == HARDWARE_TYPE_RTL8821AE)
-#define IS_HARDWARE_TYPE_8723A(adapter)	\
-			(RTL_HW_TYPE(adapter) == HARDWARE_TYPE_RTL8723AE)
-#define IS_HARDWARE_TYPE_8723B(adapter)	\
-			(RTL_HW_TYPE(adapter) == HARDWARE_TYPE_RTL8723BE)
-#define IS_HARDWARE_TYPE_8192E(adapter)	\
-			(RTL_HW_TYPE(adapter) == HARDWARE_TYPE_RTL8192EE)
-
 #include "halbtc8192e2ant.h"
 #include "halbtc8723b1ant.h"
 #include "halbtc8723b2ant.h"
diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b1ant.c b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b1ant.c
index 03998d2e9eb8..c04425236ce4 100644
--- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b1ant.c
+++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b1ant.c
@@ -600,14 +600,8 @@ static void halbtc8723b1ant_coex_table_with_type(struct btc_coexist *btcoexist,
 						   0xffffff, 0x3);
 		break;
 	case 5:
-		if ((coex_sta->cck_ever_lock) && (coex_sta->scan_ap_num <= 5))
-			halbtc8723b1ant_coex_table(btcoexist, force_exec,
-						   0x5a5a5a5a, 0x5aaa5a5a,
-						   0xffffff, 0x3);
-		else
-			halbtc8723b1ant_coex_table(btcoexist, force_exec,
-						   0x5a5a5a5a, 0x5aaa5a5a,
-						   0xffffff, 0x3);
+		halbtc8723b1ant_coex_table(btcoexist, force_exec, 0x5a5a5a5a,
+					   0x5aaa5a5a, 0xffffff, 0x3);
 		break;
 	case 6:
 		halbtc8723b1ant_coex_table(btcoexist, force_exec, 0x55555555,
diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b2ant.c b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b2ant.c
index 31965f0ef69d..e8f07573aed9 100644
--- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b2ant.c
+++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b2ant.c
@@ -1183,7 +1183,10 @@ static void btc8723b2ant_set_ant_path(struct btc_coexist *btcoexist,
 		}
 
 		/* fixed internal switch S1->WiFi, S0->BT */
-		btcoexist->btc_write_4byte(btcoexist, 0x948, 0x0);
+		if (board_info->btdm_ant_pos == BTC_ANTENNA_AT_MAIN_PORT)
+			btcoexist->btc_write_2byte(btcoexist, 0x948, 0x0);
+		else
+			btcoexist->btc_write_2byte(btcoexist, 0x948, 0x280);
 
 		switch (antpos_type) {
 		case BTC_ANT_WIFI_AT_MAIN:
diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtcoutsrc.c b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtcoutsrc.c
index e6024b013ca5..b5e9877d935c 100644
--- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtcoutsrc.c
+++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtcoutsrc.c
@@ -173,6 +173,16 @@ static u8 halbtc_get_wifi_central_chnl(struct btc_coexist *btcoexist)
 
 u8 rtl_get_hwpg_single_ant_path(struct rtl_priv *rtlpriv)
 {
+	struct rtl_mod_params *mod_params = rtlpriv->cfg->mod_params;
+
+	/* override ant_num / ant_path */
+	if (mod_params->ant_sel) {
+		rtlpriv->btcoexist.btc_info.ant_num =
+			(mod_params->ant_sel == 1 ? ANT_X2 : ANT_X1);
+
+		rtlpriv->btcoexist.btc_info.single_ant_path =
+			(mod_params->ant_sel == 1 ? 0 : 1);
+	}
 	return rtlpriv->btcoexist.btc_info.single_ant_path;
 }
 
@@ -183,6 +193,7 @@ u8 rtl_get_hwpg_bt_type(struct rtl_priv *rtlpriv)
 
 u8 rtl_get_hwpg_ant_num(struct rtl_priv *rtlpriv)
 {
+	struct rtl_mod_params *mod_params = rtlpriv->cfg->mod_params;
 	u8 num;
 
 	if (rtlpriv->btcoexist.btc_info.ant_num == ANT_X2)
@@ -190,6 +201,10 @@ u8 rtl_get_hwpg_ant_num(struct rtl_priv *rtlpriv)
 	else
 		num = 1;
 
+	/* override ant_num / ant_path */
+	if (mod_params->ant_sel)
+		num = (mod_params->ant_sel == 1 ? ANT_X2 : ANT_X1) + 1;
+
 	return num;
 }
 
@@ -327,7 +342,22 @@ static void halbtc_aggregation_check(struct btc_coexist *btcoexist)
 
 static u32 halbtc_get_bt_patch_version(struct btc_coexist *btcoexist)
 {
-	return 0;
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	u8 cmd_buffer[4] = {0};
+	u8 oper_ver = 0;
+	u8 req_num = 0x0E;
+
+	if (btcoexist->bt_info.bt_real_fw_ver)
+		goto label_done;
+
+	cmd_buffer[0] |= (oper_ver & 0x0f);	/* Set OperVer */
+	cmd_buffer[0] |= ((req_num << 4) & 0xf0);	/* Set ReqNum */
+	cmd_buffer[1] = 0; /* BT_OP_GET_BT_VERSION = 0 */
+	rtlpriv->cfg->ops->fill_h2c_cmd(rtlpriv->mac80211.hw, 0x67, 4,
+					&cmd_buffer[0]);
+
+label_done:
+	return btcoexist->bt_info.bt_real_fw_ver;
 }
 
 u32 halbtc_get_wifi_link_status(struct btc_coexist *btcoexist)
@@ -861,7 +891,7 @@ bool exhalbtc_bind_bt_coex_withadapter(void *adapter)
 {
 	struct btc_coexist *btcoexist = &gl_bt_coexist;
 	struct rtl_priv *rtlpriv = adapter;
-	u8 ant_num = 2, chip_type, single_ant_path = 0;
+	u8 ant_num = 2, chip_type;
 
 	if (btcoexist->binded)
 		return false;
@@ -896,12 +926,6 @@ bool exhalbtc_bind_bt_coex_withadapter(void *adapter)
 	ant_num = rtl_get_hwpg_ant_num(rtlpriv);
 	exhalbtc_set_ant_num(rtlpriv, BT_COEX_ANT_TYPE_PG, ant_num);
 
-	/* set default antenna position to main  port */
-	btcoexist->board_info.btdm_ant_pos = BTC_ANTENNA_AT_MAIN_PORT;
-
-	single_ant_path = rtl_get_hwpg_single_ant_path(rtlpriv);
-	exhalbtc_set_single_ant_path(single_ant_path);
-
 	if (rtl_get_hwpg_package_type(rtlpriv) == 0)
 		btcoexist->board_info.tfbga_package = false;
 	else if (rtl_get_hwpg_package_type(rtlpriv) == 1)
diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/rtl_btc.c b/drivers/net/wireless/realtek/rtlwifi/btcoexist/rtl_btc.c
index 4366c9817e1e..7d296a401b6f 100644
--- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/rtl_btc.c
+++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/rtl_btc.c
@@ -41,6 +41,7 @@ static struct rtl_btc_ops rtl_btc_operation = {
 	.btc_periodical = rtl_btc_periodical,
 	.btc_halt_notify = rtl_btc_halt_notify,
 	.btc_btinfo_notify = rtl_btc_btinfo_notify,
+	.btc_btmpinfo_notify = rtl_btc_btmpinfo_notify,
 	.btc_is_limited_dig = rtl_btc_is_limited_dig,
 	.btc_is_disable_edca_turbo = rtl_btc_is_disable_edca_turbo,
 	.btc_is_bt_disabled = rtl_btc_is_bt_disabled,
@@ -165,6 +166,33 @@ void rtl_btc_btinfo_notify(struct rtl_priv *rtlpriv, u8 *tmp_buf, u8 length)
 	exhalbtc_bt_info_notify(&gl_bt_coexist, tmp_buf, length);
 }
 
+void rtl_btc_btmpinfo_notify(struct rtl_priv *rtlpriv, u8 *tmp_buf, u8 length)
+{
+	u8 extid, seq, len;
+	u16 bt_real_fw_ver;
+	u8 bt_fw_ver;
+
+	if ((length < 4) || (!tmp_buf))
+		return;
+
+	extid = tmp_buf[0];
+	/* not response from BT FW then exit*/
+	if (extid != 1) /* C2H_TRIG_BY_BT_FW = 1 */
+		return;
+
+	len = tmp_buf[1] >> 4;
+	seq = tmp_buf[2] >> 4;
+
+	/* BT Firmware version response */
+	if (seq == 0x0E) {
+		bt_real_fw_ver = tmp_buf[3] | (tmp_buf[4] << 8);
+		bt_fw_ver = tmp_buf[5];
+
+		gl_bt_coexist.bt_info.bt_real_fw_ver = bt_real_fw_ver;
+		gl_bt_coexist.bt_info.bt_fw_ver = bt_fw_ver;
+	}
+}
+
 bool rtl_btc_is_limited_dig(struct rtl_priv *rtlpriv)
 {
 	return gl_bt_coexist.bt_info.limited_dig;
diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/rtl_btc.h b/drivers/net/wireless/realtek/rtlwifi/btcoexist/rtl_btc.h
index 6fe521cbe7f0..ac1253c46f44 100644
--- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/rtl_btc.h
+++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/rtl_btc.h
@@ -39,6 +39,7 @@ void rtl_btc_mediastatus_notify(struct rtl_priv *rtlpriv,
 void rtl_btc_periodical(struct rtl_priv *rtlpriv);
 void rtl_btc_halt_notify(void);
 void rtl_btc_btinfo_notify(struct rtl_priv *rtlpriv, u8 *tmpbuf, u8 length);
+void rtl_btc_btmpinfo_notify(struct rtl_priv *rtlpriv, u8 *tmp_buf, u8 length);
 bool rtl_btc_is_limited_dig(struct rtl_priv *rtlpriv);
 bool rtl_btc_is_disable_edca_turbo(struct rtl_priv *rtlpriv);
 bool rtl_btc_is_bt_disabled(struct rtl_priv *rtlpriv);
diff --git a/drivers/net/wireless/realtek/rtlwifi/core.c b/drivers/net/wireless/realtek/rtlwifi/core.c
index b0ad061048c5..c53cbf3d52bd 100644
--- a/drivers/net/wireless/realtek/rtlwifi/core.c
+++ b/drivers/net/wireless/realtek/rtlwifi/core.c
@@ -1505,6 +1505,8 @@ static int rtl_op_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
 	u8 mac_addr[ETH_ALEN];
 	u8 bcast_addr[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
 
+	rtlpriv->btcoexist.btc_info.in_4way = false;
+
 	if (rtlpriv->cfg->mod_params->sw_crypto || rtlpriv->sec.use_sw_sec) {
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING,
 			 "not open hw encryption\n");
diff --git a/drivers/net/wireless/realtek/rtlwifi/pci.c b/drivers/net/wireless/realtek/rtlwifi/pci.c
index 032b6317690d..08dc8919ef60 100644
--- a/drivers/net/wireless/realtek/rtlwifi/pci.c
+++ b/drivers/net/wireless/realtek/rtlwifi/pci.c
@@ -2257,7 +2257,7 @@ int rtl_pci_probe(struct pci_dev *pdev,
 	/* find adapter */
 	if (!_rtl_pci_find_adapter(pdev, hw)) {
 		err = -ENODEV;
-		goto fail3;
+		goto fail2;
 	}
 
 	/* Init IO handler */
@@ -2318,10 +2318,10 @@ fail3:
 	pci_set_drvdata(pdev, NULL);
 	rtl_deinit_core(hw);
 
+fail2:
 	if (rtlpriv->io.pci_mem_start != 0)
 		pci_iounmap(pdev, (void __iomem *)rtlpriv->io.pci_mem_start);
 
-fail2:
 	pci_release_regions(pdev);
 	complete(&rtlpriv->firmware_loading_complete);
 
diff --git a/drivers/net/wireless/realtek/rtlwifi/rc.c b/drivers/net/wireless/realtek/rtlwifi/rc.c
index 951d257cd4c0..02811eda57cd 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rc.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rc.c
@@ -283,7 +283,7 @@ static void rtl_rate_free_sta(void *rtlpriv,
 	kfree(rate_priv);
 }
 
-static struct rate_control_ops rtl_rate_ops = {
+static const struct rate_control_ops rtl_rate_ops = {
 	.name = "rtl_rc",
 	.alloc = rtl_rate_alloc,
 	.free = rtl_rate_free,
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c
index 774e72058d24..57e5d5c1d24b 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c
@@ -175,6 +175,8 @@ int rtl88e_init_sw_vars(struct ieee80211_hw *hw)
 				      rtl_fw_cb);
 	if (err) {
 		pr_info("Failed to request firmware!\n");
+		vfree(rtlpriv->rtlhal.pfirmware);
+		rtlpriv->rtlhal.pfirmware = NULL;
 		return 1;
 	}
 
@@ -376,7 +378,7 @@ static const struct rtl_hal_cfg rtl88ee_hal_cfg = {
 	.maps[RTL_RC_HT_RATEMCS15] = DESC92C_RATEMCS15,
 };
 
-static struct pci_device_id rtl88ee_pci_ids[] = {
+static const struct pci_device_id rtl88ee_pci_ids[] = {
 	{RTL_PCI_DEVICE(PCI_VENDOR_ID_REALTEK, 0x8179, rtl88ee_hal_cfg)},
 	{},
 };
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c
index bcbb0c60f1f1..38f85bfdf0c7 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c
@@ -176,6 +176,8 @@ int rtl92c_init_sw_vars(struct ieee80211_hw *hw)
 				      rtl_fw_cb);
 	if (err) {
 		pr_err("Failed to request firmware!\n");
+		vfree(rtlpriv->rtlhal.pfirmware);
+		rtlpriv->rtlhal.pfirmware = NULL;
 		return 1;
 	}
 
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/hw.c
index f95a64507f17..530e80f0ef0b 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/hw.c
@@ -777,10 +777,6 @@ static void _rtl92cu_init_queue_priority(struct ieee80211_hw *hw,
 						   queue_sel);
 }
 
-static void _rtl92cu_init_usb_aggregation(struct ieee80211_hw *hw)
-{
-}
-
 static void _rtl92cu_init_wmac_setting(struct ieee80211_hw *hw)
 {
 	u16 value16;
@@ -870,7 +866,6 @@ static int _rtl92cu_init_mac(struct ieee80211_hw *hw)
 	rtl92c_init_edca(hw);
 	rtl92c_init_rate_fallback(hw);
 	rtl92c_init_retry_function(hw);
-	_rtl92cu_init_usb_aggregation(hw);
 	rtlpriv->cfg->ops->set_bw_mode(hw, NL80211_CHAN_HT20);
 	rtl92c_set_min_space(hw, IS_92C_SERIAL(rtlhal->version));
 	_rtl92cu_init_beacon_parameters(hw);
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/mac.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/mac.c
index 1b124eade846..5657b1e34ad0 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/mac.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/mac.c
@@ -352,11 +352,10 @@ u32 rtl92c_get_txdma_status(struct ieee80211_hw *hw)
 void rtl92c_enable_interrupt(struct ieee80211_hw *hw)
 {
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
-	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
 	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
 	struct rtl_usb *rtlusb = rtl_usbdev(rtl_usbpriv(hw));
 
-	if (IS_HARDWARE_TYPE_8192CE(rtlhal)) {
+	if (IS_HARDWARE_TYPE_8192CE(rtlpriv)) {
 		rtl_write_dword(rtlpriv, REG_HIMR, rtlpci->irq_mask[0] &
 				0xFFFFFFFF);
 		rtl_write_dword(rtlpriv, REG_HIMRE, rtlpci->irq_mask[1] &
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/sw.c
index 96c923b3feb4..43e021b49260 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/sw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/sw.c
@@ -85,6 +85,10 @@ static int rtl92cu_init_sw_vars(struct ieee80211_hw *hw)
 	err = request_firmware_nowait(THIS_MODULE, 1,
 				      fw_name, rtlpriv->io.dev,
 				      GFP_KERNEL, hw, rtl_fw_cb);
+	if (err) {
+		vfree(rtlpriv->rtlhal.pfirmware);
+		rtlpriv->rtlhal.pfirmware = NULL;
+	}
 	return err;
 }
 
@@ -173,7 +177,7 @@ static struct rtl_hal_usbint_cfg rtl92cu_interface_cfg = {
 	.rx_urb_num = RTL92C_NUM_RX_URBS,
 	.rx_max_size = RTL92C_SIZE_MAX_RX_BUFFER,
 	.usb_rx_hdl = rtl8192cu_rx_hdl,
-	.usb_rx_segregate_hdl = NULL, /* rtl8192c_rx_segregate_hdl; */
+	.usb_rx_segregate_hdl = NULL,
 	/* tx */
 	.usb_tx_cleanup = rtl8192c_tx_cleanup,
 	.usb_tx_post_hdl = rtl8192c_tx_post_hdl,
@@ -275,7 +279,7 @@ static struct rtl_hal_cfg rtl92cu_hal_cfg = {
 #define USB_VENDER_ID_REALTEK		0x0bda
 
 /* 2010-10-19 DID_USB_V3.4 */
-static struct usb_device_id rtl8192c_usb_ids[] = {
+static const struct usb_device_id rtl8192c_usb_ids[] = {
 
 	/*=== Realtek demoboard ===*/
 	/* Default ID */
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/trx.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/trx.c
index de6c3428f7c6..ac4a82de40c7 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/trx.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/trx.c
@@ -436,13 +436,6 @@ void  rtl8192cu_rx_hdl(struct ieee80211_hw *hw, struct sk_buff * skb)
 	_rtl_rx_process(hw, skb);
 }
 
-void rtl8192c_rx_segregate_hdl(
-	struct ieee80211_hw *hw,
-	struct sk_buff *skb,
-	struct sk_buff_head *skb_list)
-{
-}
-
 /*----------------------------------------------------------------------
  *
  *	Tx handler
@@ -675,8 +668,3 @@ void rtl92cu_tx_fill_cmddesc(struct ieee80211_hw *hw,
 	RT_PRINT_DATA(rtlpriv, COMP_CMD, DBG_LOUD, "H2C Tx Cmd Content",
 		      pdesc, RTL_TX_DESC_SIZE);
 }
-
-bool rtl92cu_cmd_send_packet(struct ieee80211_hw *hw, struct sk_buff *skb)
-{
-	return true;
-}
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/trx.h b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/trx.h
index 487eec89bc29..15a66c547287 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/trx.h
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/trx.h
@@ -385,8 +385,6 @@ bool rtl92cu_rx_query_desc(struct ieee80211_hw *hw,
 			   struct ieee80211_rx_status *rx_status,
 			   u8 *p_desc, struct sk_buff *skb);
 void  rtl8192cu_rx_hdl(struct ieee80211_hw *hw, struct sk_buff * skb);
-void rtl8192c_rx_segregate_hdl(struct ieee80211_hw *, struct sk_buff *,
-			       struct sk_buff_head *);
 void rtl8192c_tx_cleanup(struct ieee80211_hw *hw, struct sk_buff  *skb);
 int rtl8192c_tx_post_hdl(struct ieee80211_hw *hw, struct urb *urb,
 			 struct sk_buff *skb);
@@ -404,6 +402,5 @@ void rtl92cu_fill_fake_txdesc(struct ieee80211_hw *hw, u8 * pDesc,
 void rtl92cu_tx_fill_cmddesc(struct ieee80211_hw *hw,
 			     u8 *pdesc, bool b_firstseg,
 			     bool b_lastseg, struct sk_buff *skb);
-bool rtl92cu_cmd_send_packet(struct ieee80211_hw *hw, struct sk_buff *skb);
 
 #endif
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/sw.c
index 16132c66e5e1..a6549f5f6c59 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/sw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/sw.c
@@ -183,6 +183,8 @@ static int rtl92d_init_sw_vars(struct ieee80211_hw *hw)
 				      rtl_fw_cb);
 	if (err) {
 		pr_err("Failed to request firmware!\n");
+		vfree(rtlpriv->rtlhal.pfirmware);
+		rtlpriv->rtlhal.pfirmware = NULL;
 		return 1;
 	}
 
@@ -347,7 +349,7 @@ static const struct rtl_hal_cfg rtl92de_hal_cfg = {
 	.maps[RTL_RC_HT_RATEMCS15] = DESC_RATEMCS15,
 };
 
-static struct pci_device_id rtl92de_pci_ids[] = {
+static const struct pci_device_id rtl92de_pci_ids[] = {
 	{RTL_PCI_DEVICE(PCI_VENDOR_ID_REALTEK, 0x8193, rtl92de_hal_cfg)},
 	{RTL_PCI_DEVICE(PCI_VENDOR_ID_REALTEK, 0x002B, rtl92de_hal_cfg)},
 	{},
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/fw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/fw.c
index f5d4df985c37..7eae27f8e173 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/fw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/fw.c
@@ -887,6 +887,7 @@ void rtl92ee_c2h_content_parsing(struct ieee80211_hw *hw, u8 c2h_cmd_id,
 				 u8 c2h_cmd_len, u8 *tmp_buf)
 {
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_btc_ops *btc_ops = rtlpriv->btcoexist.btc_ops;
 
 	switch (c2h_cmd_id) {
 	case C2H_8192E_DBG:
@@ -905,12 +906,16 @@ void rtl92ee_c2h_content_parsing(struct ieee80211_hw *hw, u8 c2h_cmd_id,
 	case C2H_8192E_BT_INFO:
 		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
 			 "[C2H], C2H_8723BE_BT_INFO!!\n");
-		rtlpriv->btcoexist.btc_ops->btc_btinfo_notify(rtlpriv, tmp_buf,
-							      c2h_cmd_len);
+		if (rtlpriv->cfg->ops->get_btc_status())
+			btc_ops->btc_btinfo_notify(rtlpriv, tmp_buf,
+						   c2h_cmd_len);
 		break;
 	case C2H_8192E_BT_MP:
 		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
 			 "[C2H], C2H_8723BE_BT_MP!!\n");
+		if (rtlpriv->cfg->ops->get_btc_status())
+			btc_ops->btc_btmpinfo_notify(rtlpriv, tmp_buf,
+						     c2h_cmd_len);
 		break;
 	case C2H_8192E_RA_RPT:
 		_rtl92ee_c2h_ra_report_handler(hw, tmp_buf, c2h_cmd_len);
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.c
index d84ac7adfd82..ef9394be7016 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.c
@@ -2133,7 +2133,12 @@ static void _rtl92ee_read_adapter_info(struct ieee80211_hw *hw)
 	if ((*(u8 *)&hwinfo[EEPROM_RF_BOARD_OPTION_92E]) == 0xFF)
 		rtlefuse->board_type = 0;
 
+	if (rtlpriv->btcoexist.btc_info.btcoexist == 1)
+		rtlefuse->board_type |= BIT(2); /* ODM_BOARD_BT */
+
 	rtlhal->board_type = rtlefuse->board_type;
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+		 "board_type = 0x%x\n", rtlefuse->board_type);
 	/*parse xtal*/
 	rtlefuse->crystalcap = hwinfo[EEPROM_XTAL_92E];
 	if (hwinfo[EEPROM_XTAL_92E] == 0xFF)
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c
index eaa503b7c4b4..a3490080d066 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c
@@ -177,6 +177,8 @@ int rtl92ee_init_sw_vars(struct ieee80211_hw *hw)
 				      rtl_fw_cb);
 	if (err) {
 		pr_err("Failed to request firmware!\n");
+		vfree(rtlpriv->rtlhal.pfirmware);
+		rtlpriv->rtlhal.pfirmware = NULL;
 		return 1;
 	}
 
@@ -354,7 +356,7 @@ static const struct rtl_hal_cfg rtl92ee_hal_cfg = {
 	.maps[RTL_RC_HT_RATEMCS15] = DESC92C_RATEMCS15,
 };
 
-static struct pci_device_id rtl92ee_pci_ids[] = {
+static const struct pci_device_id rtl92ee_pci_ids[] = {
 	{RTL_PCI_DEVICE(PCI_VENDOR_ID_REALTEK, 0x818B, rtl92ee_hal_cfg)},
 	{},
 };
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/trx.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/trx.c
index 55f238a2a310..c58393eab6a1 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/trx.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/trx.c
@@ -478,7 +478,6 @@ u16 rtl92ee_rx_desc_buff_remained_cnt(struct ieee80211_hw *hw, u8 queue_index)
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	u16 read_point = 0, write_point = 0, remind_cnt = 0;
 	u32 tmp_4byte = 0;
-	static u16 last_read_point;
 	static bool start_rx;
 
 	tmp_4byte = rtl_read_dword(rtlpriv, REG_RXQ_TXBD_IDX);
@@ -506,7 +505,6 @@ u16 rtl92ee_rx_desc_buff_remained_cnt(struct ieee80211_hw *hw, u8 queue_index)
 
 	rtlpci->rx_ring[queue_index].next_rx_rp = write_point;
 
-	last_read_point = read_point;
 	return remind_cnt;
 }
 
@@ -917,7 +915,6 @@ void rtl92ee_set_desc(struct ieee80211_hw *hw, u8 *pdesc, bool istx,
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	u16 cur_tx_rp = 0;
 	u16 cur_tx_wp = 0;
-	static u16 last_txw_point;
 	static bool over_run;
 	u32 tmp = 0;
 	u8 q_idx = *val;
@@ -951,9 +948,6 @@ void rtl92ee_set_desc(struct ieee80211_hw *hw, u8 *pdesc, bool istx,
 				rtl_write_word(rtlpriv,
 					       get_desc_addr_fr_q_idx(q_idx),
 					       ring->cur_tx_wp);
-
-				if (q_idx == 1)
-					last_txw_point = cur_tx_wp;
 			}
 
 			if (ring->avl_desc < (max_tx_desc - 15)) {
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/sw.c
index 2006b09ea74f..d7945b9db493 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/sw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/sw.c
@@ -216,6 +216,8 @@ static int rtl92s_init_sw_vars(struct ieee80211_hw *hw)
 				      rtl92se_fw_cb);
 	if (err) {
 		pr_err("Failed to request firmware!\n");
+		vfree(rtlpriv->rtlhal.pfirmware);
+		rtlpriv->rtlhal.pfirmware = NULL;
 		return 1;
 	}
 
@@ -396,7 +398,7 @@ static const struct rtl_hal_cfg rtl92se_hal_cfg = {
 	.maps[RTL_RC_HT_RATEMCS15] = DESC_RATEMCS15,
 };
 
-static struct pci_device_id rtl92se_pci_ids[] = {
+static const struct pci_device_id rtl92se_pci_ids[] = {
 	{RTL_PCI_DEVICE(PCI_VENDOR_ID_REALTEK, 0x8192, rtl92se_hal_cfg)},
 	{RTL_PCI_DEVICE(PCI_VENDOR_ID_REALTEK, 0x8171, rtl92se_hal_cfg)},
 	{RTL_PCI_DEVICE(PCI_VENDOR_ID_REALTEK, 0x8172, rtl92se_hal_cfg)},
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c
index 7bf9f2557920..97b8bd294aa8 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c
@@ -184,6 +184,8 @@ int rtl8723e_init_sw_vars(struct ieee80211_hw *hw)
 				      rtl_fw_cb);
 	if (err) {
 		pr_err("Failed to request firmware!\n");
+		vfree(rtlpriv->rtlhal.pfirmware);
+		rtlpriv->rtlhal.pfirmware = NULL;
 		return 1;
 	}
 	return 0;
@@ -367,7 +369,7 @@ static const struct rtl_hal_cfg rtl8723e_hal_cfg = {
 	.maps[RTL_RC_HT_RATEMCS15] = DESC92C_RATEMCS15,
 };
 
-static struct pci_device_id rtl8723e_pci_ids[] = {
+static const struct pci_device_id rtl8723e_pci_ids[] = {
 	{RTL_PCI_DEVICE(PCI_VENDOR_ID_REALTEK, 0x8723, rtl8723e_hal_cfg)},
 	{},
 };
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/dm.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/dm.c
index 131c0d1d633e..15c117e95a99 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/dm.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/dm.c
@@ -883,12 +883,8 @@ static void rtl8723be_dm_txpower_tracking_callback_thermalmeter(
 	if ((rtldm->power_index_offset[RF90_PATH_A] != 0) &&
 	    (rtldm->txpower_track_control)) {
 		rtldm->done_txpower = true;
-		if (thermalvalue > rtlefuse->eeprom_thermalmeter)
-			rtl8723be_dm_tx_power_track_set_power(hw, BBSWING, 0,
-							     index_for_channel);
-		else
-			rtl8723be_dm_tx_power_track_set_power(hw, BBSWING, 0,
-							     index_for_channel);
+		rtl8723be_dm_tx_power_track_set_power(hw, BBSWING, 0,
+						      index_for_channel);
 
 		rtldm->swing_idx_cck_base = rtldm->swing_idx_cck;
 		rtldm->swing_idx_ofdm_base[RF90_PATH_A] =
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/fw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/fw.c
index dd6f95cfaec9..4b963fd27d64 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/fw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/fw.c
@@ -709,6 +709,7 @@ void rtl8723be_c2h_content_parsing(struct ieee80211_hw *hw,
 				   u8 c2h_cmd_len, u8 *tmp_buf)
 {
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_btc_ops *btc_ops = rtlpriv->btcoexist.btc_ops;
 
 	switch (c2h_cmd_id) {
 	case C2H_8723B_DBG:
@@ -723,12 +724,16 @@ void rtl8723be_c2h_content_parsing(struct ieee80211_hw *hw,
 	case C2H_8723B_BT_INFO:
 		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
 			 "[C2H], C2H_8723BE_BT_INFO!!\n");
-		rtlpriv->btcoexist.btc_ops->btc_btinfo_notify(rtlpriv, tmp_buf,
-							      c2h_cmd_len);
+		if (rtlpriv->cfg->ops->get_btc_status())
+			btc_ops->btc_btinfo_notify(rtlpriv, tmp_buf,
+						   c2h_cmd_len);
 		break;
 	case C2H_8723B_BT_MP:
 		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
 			 "[C2H], C2H_8723BE_BT_MP!!\n");
+		if (rtlpriv->cfg->ops->get_btc_status())
+			btc_ops->btc_btmpinfo_notify(rtlpriv, tmp_buf,
+						     c2h_cmd_len);
 		break;
 	default:
 		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/hw.c
index cd5dc6dcb19f..4d47b97adfed 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/hw.c
@@ -2111,6 +2111,13 @@ static void _rtl8723be_read_adapter_info(struct ieee80211_hw *hw,
 						 rtlefuse->autoload_failflag,
 						 hwinfo);
 
+	if (rtlpriv->btcoexist.btc_info.btcoexist == 1)
+		rtlefuse->board_type |= BIT(2); /* ODM_BOARD_BT */
+
+	rtlhal->board_type = rtlefuse->board_type;
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+		 "board_type = 0x%x\n", rtlefuse->board_type);
+
 	rtlhal->package_type = _rtl8723be_read_package_type(hw);
 
 	/* set channel plan from efuse */
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/phy.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/phy.c
index 9752175cc466..9606641519e7 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/phy.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/phy.c
@@ -152,33 +152,86 @@ bool rtl8723be_phy_rf_config(struct ieee80211_hw *hw)
 	return rtl8723be_phy_rf6052_config(hw);
 }
 
-static bool _rtl8723be_check_condition(struct ieee80211_hw *hw,
-				       const u32  condition)
+static bool _rtl8723be_check_positive(struct ieee80211_hw *hw,
+				      const u32 condition1,
+				      const u32 condition2)
 {
-	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
-	struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
-	u32 _board = rtlefuse->board_type; /*need efuse define*/
-	u32 _interface = rtlhal->interface;
-	u32 _platform = 0x08;/*SupportPlatform */
-	u32 cond = condition;
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+	u32 cut_ver = ((rtlhal->version & CHIP_VER_RTL_MASK)
+					>> CHIP_VER_RTL_SHIFT);
+	u32 intf = (rtlhal->interface == INTF_USB ? BIT(1) : BIT(0));
+
+	u8  board_type = ((rtlhal->board_type & BIT(4)) >> 4) << 0 | /* _GLNA */
+			 ((rtlhal->board_type & BIT(3)) >> 3) << 1 | /* _GPA  */
+			 ((rtlhal->board_type & BIT(7)) >> 7) << 2 | /* _ALNA */
+			 ((rtlhal->board_type & BIT(6)) >> 6) << 3 | /* _APA  */
+			 ((rtlhal->board_type & BIT(2)) >> 2) << 4;  /* _BT   */
+
+	u32 cond1 = condition1, cond2 = condition2;
+	u32 driver1 = cut_ver << 24 |	/* CUT ver */
+		      0 << 20 |			/* interface 2/2 */
+		      0x04 << 16 |		/* platform */
+		      rtlhal->package_type << 12 |
+		      intf << 8 |			/* interface 1/2 */
+		      board_type;
+
+	u32 driver2 = rtlhal->type_glna <<  0 |
+		      rtlhal->type_gpa  <<  8 |
+		      rtlhal->type_alna << 16 |
+		      rtlhal->type_apa  << 24;
 
-	if (condition == 0xCDCDCDCD)
-		return true;
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE,
+		 "===> [8812A] CheckPositive (cond1, cond2) = (0x%X 0x%X)\n",
+		 cond1, cond2);
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE,
+		 "===> [8812A] CheckPositive (driver1, driver2) = (0x%X 0x%X)\n",
+		 driver1, driver2);
 
-	cond = condition & 0xFF;
-	if ((_board & cond) == 0 && cond != 0x1F)
-		return false;
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE,
+		 "	(Platform, Interface) = (0x%X, 0x%X)\n", 0x04, intf);
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE,
+		 "	(Board, Package) = (0x%X, 0x%X)\n",
+		 rtlhal->board_type, rtlhal->package_type);
 
-	cond = condition & 0xFF00;
-	cond = cond >> 8;
-	if ((_interface & cond) == 0 && cond != 0x07)
-		return false;
+	/*============== Value Defined Check ===============*/
+	/*QFN Type [15:12] and Cut Version [27:24] need to do value check*/
 
-	cond = condition & 0xFF0000;
-	cond = cond >> 16;
-	if ((_platform & cond) == 0 && cond != 0x0F)
+	if (((cond1 & 0x0000F000) != 0) && ((cond1 & 0x0000F000) !=
+		(driver1 & 0x0000F000)))
 		return false;
-	return true;
+	if (((cond1 & 0x0F000000) != 0) && ((cond1 & 0x0F000000) !=
+		(driver1 & 0x0F000000)))
+		return false;
+
+	/*=============== Bit Defined Check ================*/
+	/* We don't care [31:28] */
+
+	cond1   &= 0x00FF0FFF;
+	driver1 &= 0x00FF0FFF;
+
+	if ((cond1 & driver1) == cond1) {
+		u32 mask = 0;
+
+		if ((cond1 & 0x0F) == 0) /* BoardType is DONTCARE*/
+			return true;
+
+		if ((cond1 & BIT(0)) != 0) /*GLNA*/
+			mask |= 0x000000FF;
+		if ((cond1 & BIT(1)) != 0) /*GPA*/
+			mask |= 0x0000FF00;
+		if ((cond1 & BIT(2)) != 0) /*ALNA*/
+			mask |= 0x00FF0000;
+		if ((cond1 & BIT(3)) != 0) /*APA*/
+			mask |= 0xFF000000;
+
+		/* BoardType of each RF path is matched*/
+		if ((cond2 & mask) == (driver2 & mask))
+			return true;
+		else
+			return false;
+	}
+	return false;
 }
 
 static void _rtl8723be_config_rf_reg(struct ieee80211_hw *hw, u32 addr,
@@ -464,6 +517,16 @@ static bool _rtl8723be_phy_bb8723b_config_parafile(struct ieee80211_hw *hw)
 	struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
 	bool rtstatus;
 
+	/* switch ant to BT */
+	if (rtlpriv->rtlhal.interface == INTF_USB) {
+		rtl_write_dword(rtlpriv, 0x948, 0x0);
+	} else {
+		if (rtlpriv->btcoexist.btc_info.single_ant_path == 0)
+			rtl_write_dword(rtlpriv, 0x948, 0x280);
+		else
+			rtl_write_dword(rtlpriv, 0x948, 0x0);
+	}
+
 	rtstatus = _rtl8723be_phy_config_bb_with_headerfile(hw,
 						BASEBAND_CONFIG_PHY_REG);
 	if (!rtstatus) {
@@ -493,142 +556,84 @@ static bool _rtl8723be_phy_bb8723b_config_parafile(struct ieee80211_hw *hw)
 	return true;
 }
 
+static bool rtl8723be_phy_config_with_headerfile(struct ieee80211_hw *hw,
+						 u32 *array_table,
+						 u16 arraylen,
+		void (*set_reg)(struct ieee80211_hw *hw, u32 regaddr, u32 data))
+{
+	#define COND_ELSE  2
+	#define COND_ENDIF 3
+
+	int i = 0;
+	u8 cond;
+	bool matched = true, skipped = false;
+
+	while ((i + 1) < arraylen) {
+		u32 v1 = array_table[i];
+		u32 v2 = array_table[i + 1];
+
+		if (v1 & (BIT(31) | BIT(30))) {/*positive & negative condition*/
+			if (v1 & BIT(31)) {/* positive condition*/
+				cond  = (u8)((v1 & (BIT(29) | BIT(28))) >> 28);
+				if (cond == COND_ENDIF) { /*end*/
+					matched = true;
+					skipped = false;
+				} else if (cond == COND_ELSE) { /*else*/
+					matched = skipped ? false : true;
+				} else {/*if , else if*/
+					if (skipped) {
+						matched = false;
+					} else {
+						if (_rtl8723be_check_positive(
+								hw, v1, v2)) {
+							matched = true;
+							skipped = true;
+						} else {
+							matched = false;
+							skipped = false;
+						}
+					}
+				}
+			} else if (v1 & BIT(30)) { /*negative condition*/
+			/*do nothing*/
+			}
+		} else {
+			if (matched)
+				set_reg(hw, v1, v2);
+		}
+		i = i + 2;
+	}
+
+	return true;
+}
+
 static bool _rtl8723be_phy_config_mac_with_headerfile(struct ieee80211_hw *hw)
 {
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
-	u32 i;
-	u32 arraylength;
-	u32 *ptrarray;
 
 	RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE, "Read rtl8723beMACPHY_Array\n");
-	arraylength = RTL8723BEMAC_1T_ARRAYLEN;
-	ptrarray = RTL8723BEMAC_1T_ARRAY;
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
-		 "Img:RTL8723bEMAC_1T_ARRAY LEN %d\n", arraylength);
-	for (i = 0; i < arraylength; i = i + 2)
-		rtl_write_byte(rtlpriv, ptrarray[i], (u8)ptrarray[i + 1]);
-	return true;
+
+	return rtl8723be_phy_config_with_headerfile(hw,
+			RTL8723BEMAC_1T_ARRAY, RTL8723BEMAC_1T_ARRAYLEN,
+			rtl_write_byte_with_val32);
 }
 
 static bool _rtl8723be_phy_config_bb_with_headerfile(struct ieee80211_hw *hw,
 						     u8 configtype)
 {
-	#define READ_NEXT_PAIR(v1, v2, i) \
-		do { \
-			i += 2; \
-			v1 = array_table[i];\
-			v2 = array_table[i+1]; \
-		} while (0)
-
-	int i;
-	u32 *array_table;
-	u16 arraylen;
-	struct rtl_priv *rtlpriv = rtl_priv(hw);
-	u32 v1 = 0, v2 = 0;
-
-	if (configtype == BASEBAND_CONFIG_PHY_REG) {
-		arraylen = RTL8723BEPHY_REG_1TARRAYLEN;
-		array_table = RTL8723BEPHY_REG_1TARRAY;
-
-		for (i = 0; i < arraylen; i = i + 2) {
-			v1 = array_table[i];
-			v2 = array_table[i+1];
-			if (v1 < 0xcdcdcdcd) {
-				_rtl8723be_config_bb_reg(hw, v1, v2);
-			} else {/*This line is the start line of branch.*/
-				/* to protect READ_NEXT_PAIR not overrun */
-				if (i >= arraylen - 2)
-					break;
-
-				if (!_rtl8723be_check_condition(hw,
-						array_table[i])) {
-					/*Discard the following
-					 *(offset, data) pairs
-					 */
-					READ_NEXT_PAIR(v1, v2, i);
-					while (v2 != 0xDEAD &&
-					       v2 != 0xCDEF &&
-					       v2 != 0xCDCD &&
-					       i < arraylen - 2) {
-						READ_NEXT_PAIR(v1, v2, i);
-					}
-					i -= 2; /* prevent from for-loop += 2*/
-				/*Configure matched pairs and
-				 *skip to end of if-else.
-				 */
-				} else {
-					READ_NEXT_PAIR(v1, v2, i);
-					while (v2 != 0xDEAD &&
-					       v2 != 0xCDEF &&
-					       v2 != 0xCDCD &&
-					       i < arraylen - 2) {
-						_rtl8723be_config_bb_reg(hw,
-								    v1, v2);
-						READ_NEXT_PAIR(v1, v2, i);
-					}
 
-					while (v2 != 0xDEAD && i < arraylen - 2)
-						READ_NEXT_PAIR(v1, v2, i);
-				}
-			}
-		}
-	} else if (configtype == BASEBAND_CONFIG_AGC_TAB) {
-		arraylen = RTL8723BEAGCTAB_1TARRAYLEN;
-		array_table = RTL8723BEAGCTAB_1TARRAY;
-
-		for (i = 0; i < arraylen; i = i + 2) {
-			v1 = array_table[i];
-			v2 = array_table[i+1];
-			if (v1 < 0xCDCDCDCD) {
-				rtl_set_bbreg(hw, array_table[i],
-					      MASKDWORD,
-					      array_table[i + 1]);
-				udelay(1);
-				continue;
-			} else {/*This line is the start line of branch.*/
-				/* to protect READ_NEXT_PAIR not overrun */
-				if (i >= arraylen - 2)
-					break;
-
-				if (!_rtl8723be_check_condition(hw,
-					array_table[i])) {
-					/*Discard the following
-					 *(offset, data) pairs
-					 */
-					READ_NEXT_PAIR(v1, v2, i);
-					while (v2 != 0xDEAD &&
-					       v2 != 0xCDEF &&
-					       v2 != 0xCDCD &&
-					       i < arraylen - 2) {
-						READ_NEXT_PAIR(v1, v2, i);
-					}
-					i -= 2; /* prevent from for-loop += 2*/
-				/*Configure matched pairs and
-				 *skip to end of if-else.
-				 */
-				} else {
-					READ_NEXT_PAIR(v1, v2, i);
-					while (v2 != 0xDEAD &&
-					       v2 != 0xCDEF &&
-					       v2 != 0xCDCD &&
-					       i < arraylen - 2) {
-						rtl_set_bbreg(hw, array_table[i],
-							      MASKDWORD,
-							      array_table[i + 1]);
-						udelay(1);
-						READ_NEXT_PAIR(v1, v2, i);
-					}
+	if (configtype == BASEBAND_CONFIG_PHY_REG)
+		return rtl8723be_phy_config_with_headerfile(hw,
+				RTL8723BEPHY_REG_1TARRAY,
+				RTL8723BEPHY_REG_1TARRAYLEN,
+				_rtl8723be_config_bb_reg);
+	else if (configtype == BASEBAND_CONFIG_AGC_TAB)
+		return rtl8723be_phy_config_with_headerfile(hw,
+				RTL8723BEAGCTAB_1TARRAY,
+				RTL8723BEAGCTAB_1TARRAYLEN,
+				rtl_set_bbreg_with_dwmask);
 
-					while (v2 != 0xDEAD && i < arraylen - 2)
-						READ_NEXT_PAIR(v1, v2, i);
-				}
-			}
-			RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE,
-				 "The agctab_array_table[0] is %x Rtl818EEPHY_REGArray[1] is %x\n",
-				 array_table[i], array_table[i + 1]);
-		}
-	}
-	return true;
+	return false;
 }
 
 static u8 _rtl8723be_get_rate_section_index(u32 regaddr)
@@ -761,73 +766,17 @@ static bool _rtl8723be_phy_config_bb_with_pgheaderfile(struct ieee80211_hw *hw,
 bool rtl8723be_phy_config_rf_with_headerfile(struct ieee80211_hw *hw,
 					     enum radio_path rfpath)
 {
-	#define READ_NEXT_RF_PAIR(v1, v2, i) \
-		do { \
-			i += 2; \
-			v1 = radioa_array_table[i]; \
-			v2 = radioa_array_table[i+1]; \
-		} while (0)
-
-	int i;
-	bool rtstatus = true;
-	u32 *radioa_array_table;
-	u16 radioa_arraylen;
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
-	u32 v1 = 0, v2 = 0;
+	bool ret = true;
 
-	radioa_arraylen = RTL8723BE_RADIOA_1TARRAYLEN;
-	radioa_array_table = RTL8723BE_RADIOA_1TARRAY;
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
-		 "Radio_A:RTL8723BE_RADIOA_1TARRAY %d\n", radioa_arraylen);
 	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD, "Radio No %x\n", rfpath);
-	rtstatus = true;
 	switch (rfpath) {
 	case RF90_PATH_A:
-		for (i = 0; i < radioa_arraylen; i = i + 2) {
-			v1 = radioa_array_table[i];
-			v2 = radioa_array_table[i+1];
-			if (v1 < 0xcdcdcdcd) {
-				_rtl8723be_config_rf_radio_a(hw, v1, v2);
-			} else {/*This line is the start line of branch.*/
-				/* to protect READ_NEXT_PAIR not overrun */
-				if (i >= radioa_arraylen - 2)
-					break;
-
-				if (!_rtl8723be_check_condition(hw,
-						radioa_array_table[i])) {
-					/*Discard the following
-					 *(offset, data) pairs
-					 */
-					READ_NEXT_RF_PAIR(v1, v2, i);
-					while (v2 != 0xDEAD &&
-					       v2 != 0xCDEF &&
-					       v2 != 0xCDCD &&
-					       i < radioa_arraylen - 2) {
-						READ_NEXT_RF_PAIR(v1, v2, i);
-					}
-					i -= 2; /* prevent from for-loop += 2*/
-				} else {
-					/*Configure matched pairs
-					 *and skip to end of if-else.
-					 */
-					READ_NEXT_RF_PAIR(v1, v2, i);
-					while (v2 != 0xDEAD &&
-					       v2 != 0xCDEF &&
-					       v2 != 0xCDCD &&
-					       i < radioa_arraylen - 2) {
-						_rtl8723be_config_rf_radio_a(hw,
-									v1, v2);
-						READ_NEXT_RF_PAIR(v1, v2, i);
-					}
-
-					while (v2 != 0xDEAD &&
-					       i < radioa_arraylen - 2) {
-						READ_NEXT_RF_PAIR(v1, v2, i);
-					}
-				}
-			}
-		}
+		ret =  rtl8723be_phy_config_with_headerfile(hw,
+				RTL8723BE_RADIOA_1TARRAY,
+				RTL8723BE_RADIOA_1TARRAYLEN,
+				_rtl8723be_config_rf_radio_a);
 
 		if (rtlhal->oem_id == RT_CID_819X_HP)
 			_rtl8723be_config_rf_radio_a(hw, 0x52, 0x7E4BD);
@@ -840,7 +789,7 @@ bool rtl8723be_phy_config_rf_with_headerfile(struct ieee80211_hw *hw,
 			 "switch case %#x not processed\n", rfpath);
 		break;
 	}
-	return true;
+	return ret;
 }
 
 void rtl8723be_phy_get_hw_reg_originalvalue(struct ieee80211_hw *hw)
@@ -1350,7 +1299,7 @@ void rtl8723be_phy_sw_chnl_callback(struct ieee80211_hw *hw)
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
 	struct rtl_phy *rtlphy = &rtlpriv->phy;
-	u32 delay;
+	u32 delay = 0;
 
 	RT_TRACE(rtlpriv, COMP_SCAN, DBG_TRACE,
 		 "switch to channel%d\n", rtlphy->current_channel);
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c
index f9d10f1e7cf8..2b16a1467e78 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c
@@ -187,16 +187,10 @@ int rtl8723be_init_sw_vars(struct ieee80211_hw *hw)
 				      rtlpriv->io.dev, GFP_KERNEL, hw,
 				      rtl_fw_cb);
 	if (err) {
-		/* Failed to get firmware. Check if old version available */
-		fw_name = "rtlwifi/rtl8723befw.bin";
-		pr_info("Using firmware %s\n", fw_name);
-		err = request_firmware_nowait(THIS_MODULE, 1, fw_name,
-					      rtlpriv->io.dev, GFP_KERNEL, hw,
-					      rtl_fw_cb);
-		if (err) {
-			pr_err("Failed to request firmware!\n");
-			return 1;
-		}
+		pr_err("Failed to request firmware!\n");
+		vfree(rtlpriv->rtlhal.pfirmware);
+		rtlpriv->rtlhal.pfirmware = NULL;
+		return 1;
 	}
 	return 0;
 }
@@ -287,6 +281,7 @@ static const struct rtl_hal_cfg rtl8723be_hal_cfg = {
 	.bar_id = 2,
 	.write_readback = true,
 	.name = "rtl8723be_pci",
+	.alt_fw_name = "rtlwifi/rtl8723befw.bin",
 	.ops = &rtl8723be_hal_ops,
 	.mod_params = &rtl8723be_mod_params,
 	.maps[SYS_ISO_CTRL] = REG_SYS_ISO_CTRL,
@@ -380,7 +375,7 @@ static const struct rtl_hal_cfg rtl8723be_hal_cfg = {
 	.maps[RTL_RC_HT_RATEMCS15] = DESC92C_RATEMCS15,
 };
 
-static struct pci_device_id rtl8723be_pci_ids[] = {
+static const struct pci_device_id rtl8723be_pci_ids[] = {
 	{RTL_PCI_DEVICE(PCI_VENDOR_ID_REALTEK, 0xB723, rtl8723be_hal_cfg)},
 	{},
 };
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/table.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/table.c
index a180761e8810..381c16b9b3a9 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/table.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/table.c
@@ -26,6 +26,7 @@
  *****************************************************************************/
 
 #include "table.h"
+
 u32 RTL8723BEPHY_REG_1TARRAY[] = {
 		0x800, 0x80040000,
 		0x804, 0x00000003,
@@ -36,7 +37,7 @@ u32 RTL8723BEPHY_REG_1TARRAY[] = {
 		0x818, 0x02200385,
 		0x81C, 0x00000000,
 		0x820, 0x01000100,
-		0x824, 0x00390204,
+		0x824, 0x00190204,
 		0x828, 0x00000000,
 		0x82C, 0x00000000,
 		0x830, 0x00000000,
@@ -73,9 +74,8 @@ u32 RTL8723BEPHY_REG_1TARRAY[] = {
 		0x90C, 0x81121111,
 		0x910, 0x00000002,
 		0x914, 0x00000201,
-		0x948, 0x00000280,
 		0xA00, 0x00D047C8,
-		0xA04, 0x80FF000C,
+		0xA04, 0x80FF800C,
 		0xA08, 0x8C838300,
 		0xA0C, 0x2E7F120F,
 		0xA10, 0x9500BB78,
@@ -114,7 +114,7 @@ u32 RTL8723BEPHY_REG_1TARRAY[] = {
 		0xC4C, 0x007F037F,
 		0xC50, 0x69553420,
 		0xC54, 0x43BC0094,
-		0xC58, 0x00023169,
+		0xC58, 0x00013147,
 		0xC5C, 0x00250492,
 		0xC60, 0x00000000,
 		0xC64, 0x7112848B,
@@ -125,7 +125,7 @@ u32 RTL8723BEPHY_REG_1TARRAY[] = {
 		0xC78, 0x0000001F,
 		0xC7C, 0x00B91612,
 		0xC80, 0x390000E4,
-		0xC84, 0x20F60000,
+		0xC84, 0x21F60000,
 		0xC88, 0x40000100,
 		0xC8C, 0x20200000,
 		0xC90, 0x00020E1A,
@@ -224,15 +224,21 @@ u32 RTL8723BEPHY_REG_1TARRAY[] = {
 
 };
 
+u32 RTL8723BEPHY_REG_1TARRAYLEN =
+	sizeof(RTL8723BEPHY_REG_1TARRAY) / sizeof(u32);
+
 u32 RTL8723BEPHY_REG_ARRAY_PG[] = {
-	0, 0, 0, 0x00000e08, 0x0000ff00, 0x00004000,
-	0, 0, 0, 0x0000086c, 0xffffff00, 0x34363800,
-	0, 0, 0, 0x00000e00, 0xffffffff, 0x42444646,
-	0, 0, 0, 0x00000e04, 0xffffffff, 0x30343840,
+	0, 0, 0, 0x00000e08, 0x0000ff00, 0x00003800,
+	0, 0, 0, 0x0000086c, 0xffffff00, 0x32343600,
+	0, 0, 0, 0x00000e00, 0xffffffff, 0x40424444,
+	0, 0, 0, 0x00000e04, 0xffffffff, 0x28323638,
 	0, 0, 0, 0x00000e10, 0xffffffff, 0x38404244,
 	0, 0, 0, 0x00000e14, 0xffffffff, 0x26303436
 };
 
+u32 RTL8723BEPHY_REG_ARRAY_PGLEN =
+		sizeof(RTL8723BEPHY_REG_ARRAY_PG) / sizeof(u32);
+
 u32 RTL8723BE_RADIOA_1TARRAY[] = {
 		0x000, 0x00010000,
 		0x0B0, 0x000DFFE0,
@@ -257,15 +263,37 @@ u32 RTL8723BE_RADIOA_1TARRAY[] = {
 		0x01E, 0x00000000,
 		0x0DF, 0x00000780,
 		0x050, 0x00067435,
+	0x80002000,	0x00000000,	0x40000000,	0x00000000,
+		0x051, 0x0006F10E,
+		0x052, 0x000007D3,
+	0x90003000,	0x00000000,	0x40000000,	0x00000000,
+		0x051, 0x0006F10E,
+		0x052, 0x000007D3,
+	0x90004000,	0x00000000,	0x40000000,	0x00000000,
+		0x051, 0x0006F10E,
+		0x052, 0x000007D3,
+	0xA0000000,	0x00000000,
 		0x051, 0x0006B04E,
 		0x052, 0x000007D2,
+	0xB0000000,	0x00000000,
 		0x053, 0x00000000,
 		0x054, 0x00050400,
 		0x055, 0x0004026E,
 		0x0DD, 0x0000004C,
 		0x070, 0x00067435,
+	0x80002000,	0x00000000,	0x40000000,	0x00000000,
+		0x071, 0x0006F10E,
+		0x072, 0x000007D3,
+	0x90003000,	0x00000000,	0x40000000,	0x00000000,
+		0x071, 0x0006F10E,
+		0x072, 0x000007D3,
+	0x90004000,	0x00000000,	0x40000000,	0x00000000,
+		0x071, 0x0006F10E,
+		0x072, 0x000007D3,
+	0xA0000000,	0x00000000,
 		0x071, 0x0006B04E,
 		0x072, 0x000007D2,
+	0xB0000000,	0x00000000,
 		0x073, 0x00000000,
 		0x074, 0x00050400,
 		0x075, 0x0004026E,
@@ -308,6 +336,7 @@ u32 RTL8723BE_RADIOA_1TARRAY[] = {
 		0x044, 0x00000051,
 		0x0EF, 0x00000000,
 		0x0ED, 0x00000000,
+		0x07F, 0x00020080,
 		0x0EF, 0x00002000,
 		0x03B, 0x000380EF,
 		0x03B, 0x000302FE,
@@ -336,14 +365,24 @@ u32 RTL8723BE_RADIOA_1TARRAY[] = {
 		0x0A3, 0x00008000,
 		0x0A4, 0x00048D80,
 		0x0A5, 0x00068000,
-		0x000, 0x00033D80,
+		0x0ED, 0x00000002,
+		0x0EF, 0x00000002,
+		0x056, 0x00000032,
+		0x076, 0x00000032,
+		0x001, 0x00000780,
 
 };
 
+u32 RTL8723BE_RADIOA_1TARRAYLEN =
+	sizeof(RTL8723BE_RADIOA_1TARRAY) / sizeof(u32);
+
 u32 RTL8723BEMAC_1T_ARRAY[] = {
 		0x02F, 0x00000030,
 		0x035, 0x00000000,
+		0x039, 0x00000008,
+		0x064, 0x00000000,
 		0x067, 0x00000020,
+		0x421, 0x0000000F,
 		0x428, 0x0000000A,
 		0x429, 0x00000010,
 		0x430, 0x00000000,
@@ -439,9 +478,13 @@ u32 RTL8723BEMAC_1T_ARRAY[] = {
 		0x709, 0x00000043,
 		0x70A, 0x00000065,
 		0x70B, 0x00000087,
+		0x765, 0x00000018,
+		0x76E, 0x00000004,
 
 };
 
+u32 RTL8723BEMAC_1T_ARRAYLEN = sizeof(RTL8723BEMAC_1T_ARRAY) / sizeof(u32);
+
 u32 RTL8723BEAGCTAB_1TARRAY[] = {
 		0xC78, 0xFD000001,
 		0xC78, 0xFC010001,
@@ -466,21 +509,21 @@ u32 RTL8723BEAGCTAB_1TARRAY[] = {
 		0xC78, 0xE9140001,
 		0xC78, 0xE8150001,
 		0xC78, 0xE7160001,
-		0xC78, 0xAA170001,
-		0xC78, 0xA9180001,
-		0xC78, 0xA8190001,
-		0xC78, 0xA71A0001,
-		0xC78, 0xA61B0001,
-		0xC78, 0xA51C0001,
-		0xC78, 0xA41D0001,
-		0xC78, 0xA31E0001,
-		0xC78, 0x671F0001,
-		0xC78, 0x66200001,
-		0xC78, 0x65210001,
-		0xC78, 0x64220001,
-		0xC78, 0x63230001,
-		0xC78, 0x62240001,
-		0xC78, 0x61250001,
+		0xC78, 0xE6170001,
+		0xC78, 0xE5180001,
+		0xC78, 0xE4190001,
+		0xC78, 0xE31A0001,
+		0xC78, 0xA51B0001,
+		0xC78, 0xA41C0001,
+		0xC78, 0xA31D0001,
+		0xC78, 0x671E0001,
+		0xC78, 0x661F0001,
+		0xC78, 0x65200001,
+		0xC78, 0x64210001,
+		0xC78, 0x63220001,
+		0xC78, 0x4A230001,
+		0xC78, 0x49240001,
+		0xC78, 0x48250001,
 		0xC78, 0x47260001,
 		0xC78, 0x46270001,
 		0xC78, 0x45280001,
@@ -491,22 +534,22 @@ u32 RTL8723BEAGCTAB_1TARRAY[] = {
 		0xC78, 0x282D0001,
 		0xC78, 0x272E0001,
 		0xC78, 0x262F0001,
-		0xC78, 0x25300001,
-		0xC78, 0x24310001,
-		0xC78, 0x09320001,
-		0xC78, 0x08330001,
-		0xC78, 0x07340001,
-		0xC78, 0x06350001,
-		0xC78, 0x05360001,
-		0xC78, 0x04370001,
-		0xC78, 0x03380001,
-		0xC78, 0x02390001,
+		0xC78, 0x0A300001,
+		0xC78, 0x09310001,
+		0xC78, 0x08320001,
+		0xC78, 0x07330001,
+		0xC78, 0x06340001,
+		0xC78, 0x05350001,
+		0xC78, 0x04360001,
+		0xC78, 0x03370001,
+		0xC78, 0x02380001,
+		0xC78, 0x01390001,
 		0xC78, 0x013A0001,
-		0xC78, 0x003B0001,
-		0xC78, 0x003C0001,
-		0xC78, 0x003D0001,
-		0xC78, 0x003E0001,
-		0xC78, 0x003F0001,
+		0xC78, 0x013B0001,
+		0xC78, 0x013C0001,
+		0xC78, 0x013D0001,
+		0xC78, 0x013E0001,
+		0xC78, 0x013F0001,
 		0xC78, 0xFC400001,
 		0xC78, 0xFB410001,
 		0xC78, 0xFA420001,
@@ -531,47 +574,50 @@ u32 RTL8723BEAGCTAB_1TARRAY[] = {
 		0xC78, 0xE7550001,
 		0xC78, 0xE6560001,
 		0xC78, 0xE5570001,
-		0xC78, 0xAA580001,
-		0xC78, 0xA9590001,
-		0xC78, 0xA85A0001,
-		0xC78, 0xA75B0001,
-		0xC78, 0xA65C0001,
-		0xC78, 0xA55D0001,
-		0xC78, 0xA45E0001,
-		0xC78, 0x675F0001,
-		0xC78, 0x66600001,
-		0xC78, 0x65610001,
-		0xC78, 0x64620001,
-		0xC78, 0x63630001,
-		0xC78, 0x62640001,
-		0xC78, 0x61650001,
+		0xC78, 0xE4580001,
+		0xC78, 0xE3590001,
+		0xC78, 0xA65A0001,
+		0xC78, 0xA55B0001,
+		0xC78, 0xA45C0001,
+		0xC78, 0xA35D0001,
+		0xC78, 0x675E0001,
+		0xC78, 0x665F0001,
+		0xC78, 0x65600001,
+		0xC78, 0x64610001,
+		0xC78, 0x63620001,
+		0xC78, 0x62630001,
+		0xC78, 0x61640001,
+		0xC78, 0x48650001,
 		0xC78, 0x47660001,
 		0xC78, 0x46670001,
 		0xC78, 0x45680001,
 		0xC78, 0x44690001,
 		0xC78, 0x436A0001,
 		0xC78, 0x426B0001,
-		0xC78, 0x296C0001,
-		0xC78, 0x286D0001,
-		0xC78, 0x276E0001,
-		0xC78, 0x266F0001,
-		0xC78, 0x25700001,
-		0xC78, 0x24710001,
-		0xC78, 0x09720001,
-		0xC78, 0x08730001,
-		0xC78, 0x07740001,
-		0xC78, 0x06750001,
-		0xC78, 0x05760001,
-		0xC78, 0x04770001,
-		0xC78, 0x03780001,
-		0xC78, 0x02790001,
+		0xC78, 0x286C0001,
+		0xC78, 0x276D0001,
+		0xC78, 0x266E0001,
+		0xC78, 0x256F0001,
+		0xC78, 0x24700001,
+		0xC78, 0x09710001,
+		0xC78, 0x08720001,
+		0xC78, 0x07730001,
+		0xC78, 0x06740001,
+		0xC78, 0x05750001,
+		0xC78, 0x04760001,
+		0xC78, 0x03770001,
+		0xC78, 0x02780001,
+		0xC78, 0x01790001,
 		0xC78, 0x017A0001,
-		0xC78, 0x007B0001,
-		0xC78, 0x007C0001,
-		0xC78, 0x007D0001,
-		0xC78, 0x007E0001,
-		0xC78, 0x007F0001,
+		0xC78, 0x017B0001,
+		0xC78, 0x017C0001,
+		0xC78, 0x017D0001,
+		0xC78, 0x017E0001,
+		0xC78, 0x017F0001,
 		0xC50, 0x69553422,
 		0xC50, 0x69553420,
+		0x824, 0x00390204,
 
 };
+
+u32 RTL8723BEAGCTAB_1TARRAYLEN = sizeof(RTL8723BEAGCTAB_1TARRAY) / sizeof(u32);
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/table.h b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/table.h
index dc17001632f7..1deaffe22251 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/table.h
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/table.h
@@ -29,15 +29,15 @@
 #define __RTL8723BE_TABLE__H_
 
 #include <linux/types.h>
-#define  RTL8723BEPHY_REG_1TARRAYLEN	388
+extern u32 RTL8723BEPHY_REG_1TARRAYLEN;
 extern u32 RTL8723BEPHY_REG_1TARRAY[];
-#define RTL8723BEPHY_REG_ARRAY_PGLEN	36
+extern u32 RTL8723BEPHY_REG_ARRAY_PGLEN;
 extern u32 RTL8723BEPHY_REG_ARRAY_PG[];
-#define	RTL8723BE_RADIOA_1TARRAYLEN	206
+extern u32 RTL8723BE_RADIOA_1TARRAYLEN;
 extern u32 RTL8723BE_RADIOA_1TARRAY[];
-#define RTL8723BEMAC_1T_ARRAYLEN	196
+extern u32 RTL8723BEMAC_1T_ARRAYLEN;
 extern u32 RTL8723BEMAC_1T_ARRAY[];
-#define RTL8723BEAGCTAB_1TARRAYLEN	260
+extern u32 RTL8723BEAGCTAB_1TARRAYLEN;
 extern u32 RTL8723BEAGCTAB_1TARRAY[];
 
 #endif
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/fw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/fw.c
index 03259aa150fd..f2b2c549e5b2 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/fw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/fw.c
@@ -98,7 +98,7 @@ static int _rtl8821ae_fw_free_to_go(struct ieee80211_hw *hw)
 
 	if (counter >= FW_8821AE_POLLING_TIMEOUT_COUNT) {
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
-			 "chksum report faill ! REG_MCUFWDL:0x%08x .\n",
+			 "chksum report fail! REG_MCUFWDL:0x%08x .\n",
 			  value32);
 		goto exit;
 	}
@@ -1923,6 +1923,7 @@ void rtl8821ae_c2h_content_parsing(struct ieee80211_hw *hw,
 				   u8 *tmp_buf)
 {
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_btc_ops *btc_ops = rtlpriv->btcoexist.btc_ops;
 
 	switch (c2h_cmd_id) {
 	case C2H_8812_DBG:
@@ -1938,9 +1939,15 @@ void rtl8821ae_c2h_content_parsing(struct ieee80211_hw *hw,
 		RT_TRACE(rtlpriv, COMP_FW, DBG_LOUD,
 			 "[C2H], C2H_8812_BT_INFO!!\n");
 		if (rtlpriv->cfg->ops->get_btc_status())
-			rtlpriv->btcoexist.btc_ops->btc_btinfo_notify(rtlpriv,
-								      tmp_buf,
-								      c2h_cmd_len);
+			btc_ops->btc_btinfo_notify(rtlpriv, tmp_buf,
+						   c2h_cmd_len);
+		break;
+	case C2H_8812_BT_MP:
+		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
+			 "[C2H], C2H_8812_BT_MP!!\n");
+		if (rtlpriv->cfg->ops->get_btc_status())
+			btc_ops->btc_btmpinfo_notify(rtlpriv, tmp_buf,
+						     c2h_cmd_len);
 		break;
 	default:
 		break;
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/hw.c
index 2bc6bace069c..4f73012978e9 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/hw.c
@@ -779,7 +779,7 @@ void rtl8821ae_set_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
 			_rtl8821ae_resume_tx_beacon(hw);
 		break; }
 	case HW_VAR_NAV_UPPER: {
-		u32	us_nav_upper = ((u32)*val);
+		u32	us_nav_upper = *(u32 *)val;
 
 		if (us_nav_upper > HAL_92C_NAV_UPPER_UNIT * 0xFF) {
 			RT_TRACE(rtlpriv, COMP_INIT , DBG_WARNING,
@@ -2966,6 +2966,44 @@ static void _rtl8812ae_read_pa_type(struct ieee80211_hw *hw, u8 *hwinfo,
 	}
 }
 
+static void _rtl8812ae_read_amplifier_type(struct ieee80211_hw *hw, u8 *hwinfo,
+					   bool autoload_fail)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+
+	u8 ext_type_pa_2g_a  = (hwinfo[0xBD] & BIT(2))      >> 2; /* 0xBD[2] */
+	u8 ext_type_pa_2g_b  = (hwinfo[0xBD] & BIT(6))      >> 6; /* 0xBD[6] */
+	u8 ext_type_pa_5g_a  = (hwinfo[0xBF] & BIT(2))      >> 2; /* 0xBF[2] */
+	u8 ext_type_pa_5g_b  = (hwinfo[0xBF] & BIT(6))      >> 6; /* 0xBF[6] */
+	/* 0xBD[1:0] */
+	u8 ext_type_lna_2g_a = (hwinfo[0xBD] & (BIT(1) | BIT(0))) >> 0;
+	/* 0xBD[5:4] */
+	u8 ext_type_lna_2g_b = (hwinfo[0xBD] & (BIT(5) | BIT(4))) >> 4;
+	/* 0xBF[1:0] */
+	u8 ext_type_lna_5g_a = (hwinfo[0xBF] & (BIT(1) | BIT(0))) >> 0;
+	/* 0xBF[5:4] */
+	u8 ext_type_lna_5g_b = (hwinfo[0xBF] & (BIT(5) | BIT(4))) >> 4;
+
+	_rtl8812ae_read_pa_type(hw, hwinfo, autoload_fail);
+
+	/* [2.4G] Path A and B are both extPA */
+	if ((rtlhal->pa_type_2g & (BIT(5) | BIT(4))) == (BIT(5) | BIT(4)))
+		rtlhal->type_gpa  = ext_type_pa_2g_b  << 2 | ext_type_pa_2g_a;
+
+	/* [5G] Path A and B are both extPA */
+	if ((rtlhal->pa_type_5g & (BIT(1) | BIT(0))) == (BIT(1) | BIT(0)))
+		rtlhal->type_apa  = ext_type_pa_5g_b  << 2 | ext_type_pa_5g_a;
+
+	/* [2.4G] Path A and B are both extLNA */
+	if ((rtlhal->lna_type_2g & (BIT(7) | BIT(3))) == (BIT(7) | BIT(3)))
+		rtlhal->type_glna = ext_type_lna_2g_b << 2 | ext_type_lna_2g_a;
+
+	/* [5G] Path A and B are both extLNA */
+	if ((rtlhal->lna_type_5g & (BIT(7) | BIT(3))) == (BIT(7) | BIT(3)))
+		rtlhal->type_alna = ext_type_lna_5g_b << 2 | ext_type_lna_5g_a;
+}
+
 static void _rtl8821ae_read_pa_type(struct ieee80211_hw *hw, u8 *hwinfo,
 				    bool autoload_fail)
 {
@@ -3114,7 +3152,8 @@ static void _rtl8821ae_read_adapter_info(struct ieee80211_hw *hw, bool b_pseudo_
 					       hwinfo);
 
 	if (rtlhal->hw_type == HARDWARE_TYPE_RTL8812AE) {
-		_rtl8812ae_read_pa_type(hw, hwinfo, rtlefuse->autoload_failflag);
+		_rtl8812ae_read_amplifier_type(hw, hwinfo,
+					       rtlefuse->autoload_failflag);
 		_rtl8812ae_read_bt_coexist_info_from_hwpg(hw,
 				rtlefuse->autoload_failflag, hwinfo);
 	} else {
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/phy.c b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/phy.c
index aa3ccc740521..176deb2b5386 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/phy.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/phy.c
@@ -3773,10 +3773,11 @@ static void _rtl8821ae_iqk_tx(struct ieee80211_hw *hw, enum radio_path path)
 	u32	tx_fail, rx_fail, delay_count, iqk_ready, cal_retry, cal = 0, temp_reg65;
 	int	tx_x = 0, tx_y = 0, rx_x = 0, rx_y = 0, tx_average = 0, rx_average = 0;
 	int	tx_x0[cal_num], tx_y0[cal_num], tx_x0_rxk[cal_num],
-		tx_y0_rxk[cal_num], rx_x0[cal_num], rx_y0[cal_num];
+		tx_y0_rxk[cal_num], rx_x0[cal_num], rx_y0[cal_num],
+		tx_dt[cal_num], rx_dt[cal_num];
 	bool	tx0iqkok = false, rx0iqkok = false;
 	bool	vdf_enable = false;
-	int	i, k, vdf_y[3], vdf_x[3], tx_dt[3], rx_dt[3],
+	int	i, k, vdf_y[3], vdf_x[3],
 		ii, dx = 0, dy = 0, tx_finish = 0, rx_finish = 0;
 
 	RT_TRACE(rtlpriv, COMP_IQK, DBG_LOUD,
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c
index d71d2776ca03..0894ef48ab87 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c
@@ -196,6 +196,8 @@ int rtl8821ae_init_sw_vars(struct ieee80211_hw *hw)
 	rtlpriv->rtlhal.wowlan_firmware = vzalloc(0x8000);
 	if (!rtlpriv->rtlhal.wowlan_firmware) {
 		pr_err("Can't alloc buffer for wowlan fw.\n");
+		vfree(rtlpriv->rtlhal.pfirmware);
+		rtlpriv->rtlhal.pfirmware = NULL;
 		return 1;
 	}
 
@@ -214,16 +216,10 @@ int rtl8821ae_init_sw_vars(struct ieee80211_hw *hw)
 				      rtlpriv->io.dev, GFP_KERNEL, hw,
 				      rtl_fw_cb);
 	if (err) {
-		/* Failed to get firmware. Check if old version available */
-		fw_name = "rtlwifi/rtl8821aefw.bin";
-		pr_info("Using firmware %s\n", fw_name);
-		err = request_firmware_nowait(THIS_MODULE, 1, fw_name,
-					      rtlpriv->io.dev, GFP_KERNEL, hw,
-					      rtl_fw_cb);
-		if (err) {
-			pr_err("Failed to request normal firmware!\n");
-			return 1;
-		}
+		pr_err("Failed to request normal firmware!\n");
+		vfree(rtlpriv->rtlhal.wowlan_firmware);
+		vfree(rtlpriv->rtlhal.pfirmware);
+		return 1;
 	}
 	/*load wowlan firmware*/
 	pr_info("Using firmware %s\n", wowlan_fw_name);
@@ -233,6 +229,8 @@ int rtl8821ae_init_sw_vars(struct ieee80211_hw *hw)
 				      rtl_wowlan_fw_cb);
 	if (err) {
 		pr_err("Failed to request wowlan firmware!\n");
+		vfree(rtlpriv->rtlhal.wowlan_firmware);
+		vfree(rtlpriv->rtlhal.pfirmware);
 		return 1;
 	}
 	return 0;
@@ -325,6 +323,7 @@ static const struct rtl_hal_cfg rtl8821ae_hal_cfg = {
 	.bar_id = 2,
 	.write_readback = true,
 	.name = "rtl8821ae_pci",
+	.alt_fw_name = "rtlwifi/rtl8821aefw.bin",
 	.ops = &rtl8821ae_hal_ops,
 	.mod_params = &rtl8821ae_mod_params,
 	.maps[SYS_ISO_CTRL] = REG_SYS_ISO_CTRL,
@@ -424,7 +423,7 @@ static const struct rtl_hal_cfg rtl8821ae_hal_cfg = {
 	.maps[RTL_RC_VHT_RATE_2SS_MCS9] = DESC_RATEVHT2SS_MCS9,
 };
 
-static struct pci_device_id rtl8821ae_pci_ids[] = {
+static const struct pci_device_id rtl8821ae_pci_ids[] = {
 	{RTL_PCI_DEVICE(PCI_VENDOR_ID_REALTEK, 0x8812, rtl8821ae_hal_cfg)},
 	{RTL_PCI_DEVICE(PCI_VENDOR_ID_REALTEK, 0x8821, rtl8821ae_hal_cfg)},
 	{},
diff --git a/drivers/net/wireless/realtek/rtlwifi/wifi.h b/drivers/net/wireless/realtek/rtlwifi/wifi.h
index 70723e67b7d7..1ab1024330fb 100644
--- a/drivers/net/wireless/realtek/rtlwifi/wifi.h
+++ b/drivers/net/wireless/realtek/rtlwifi/wifi.h
@@ -314,35 +314,29 @@ enum hardware_type {
 	HARDWARE_TYPE_RTL8192EE,
 	HARDWARE_TYPE_RTL8821AE,
 	HARDWARE_TYPE_RTL8812AE,
+	HARDWARE_TYPE_RTL8822BE,
 
 	/* keep it last */
 	HARDWARE_TYPE_NUM
 };
 
-#define IS_HARDWARE_TYPE_8192SU(rtlhal)			\
-	(rtlhal->hw_type == HARDWARE_TYPE_RTL8192SU)
-#define IS_HARDWARE_TYPE_8192SE(rtlhal)			\
-	(rtlhal->hw_type == HARDWARE_TYPE_RTL8192SE)
-#define IS_HARDWARE_TYPE_8192CE(rtlhal)			\
-	(rtlhal->hw_type == HARDWARE_TYPE_RTL8192CE)
-#define IS_HARDWARE_TYPE_8192CU(rtlhal)			\
-	(rtlhal->hw_type == HARDWARE_TYPE_RTL8192CU)
-#define IS_HARDWARE_TYPE_8192DE(rtlhal)			\
-	(rtlhal->hw_type == HARDWARE_TYPE_RTL8192DE)
-#define IS_HARDWARE_TYPE_8192DU(rtlhal)			\
-	(rtlhal->hw_type == HARDWARE_TYPE_RTL8192DU)
-#define IS_HARDWARE_TYPE_8723E(rtlhal)			\
-	(rtlhal->hw_type == HARDWARE_TYPE_RTL8723E)
-#define IS_HARDWARE_TYPE_8723U(rtlhal)			\
-	(rtlhal->hw_type == HARDWARE_TYPE_RTL8723U)
-#define	IS_HARDWARE_TYPE_8192S(rtlhal)			\
-(IS_HARDWARE_TYPE_8192SE(rtlhal) || IS_HARDWARE_TYPE_8192SU(rtlhal))
-#define	IS_HARDWARE_TYPE_8192C(rtlhal)			\
-(IS_HARDWARE_TYPE_8192CE(rtlhal) || IS_HARDWARE_TYPE_8192CU(rtlhal))
-#define	IS_HARDWARE_TYPE_8192D(rtlhal)			\
-(IS_HARDWARE_TYPE_8192DE(rtlhal) || IS_HARDWARE_TYPE_8192DU(rtlhal))
-#define	IS_HARDWARE_TYPE_8723(rtlhal)			\
-(IS_HARDWARE_TYPE_8723E(rtlhal) || IS_HARDWARE_TYPE_8723U(rtlhal))
+#define RTL_HW_TYPE(rtlpriv)	(rtl_hal((struct rtl_priv *)rtlpriv)->hw_type)
+#define IS_NEW_GENERATION_IC(rtlpriv)			\
+			(RTL_HW_TYPE(rtlpriv) >= HARDWARE_TYPE_RTL8192EE)
+#define IS_HARDWARE_TYPE_8192CE(rtlpriv)		\
+			(RTL_HW_TYPE(rtlpriv) == HARDWARE_TYPE_RTL8192CE)
+#define IS_HARDWARE_TYPE_8812(rtlpriv)			\
+			(RTL_HW_TYPE(rtlpriv) == HARDWARE_TYPE_RTL8812AE)
+#define IS_HARDWARE_TYPE_8821(rtlpriv)			\
+			(RTL_HW_TYPE(rtlpriv) == HARDWARE_TYPE_RTL8821AE)
+#define IS_HARDWARE_TYPE_8723A(rtlpriv)			\
+			(RTL_HW_TYPE(rtlpriv) == HARDWARE_TYPE_RTL8723AE)
+#define IS_HARDWARE_TYPE_8723B(rtlpriv)			\
+			(RTL_HW_TYPE(rtlpriv) == HARDWARE_TYPE_RTL8723BE)
+#define IS_HARDWARE_TYPE_8192E(rtlpriv)			\
+			(RTL_HW_TYPE(rtlpriv) == HARDWARE_TYPE_RTL8192EE)
+#define IS_HARDWARE_TYPE_8822B(rtlpriv)			\
+			(RTL_HW_TYPE(rtlpriv) == HARDWARE_TYPE_RTL8822BE)
 
 #define RX_HAL_IS_CCK_RATE(rxmcs)			\
 	((rxmcs) == DESC_RATE1M ||			\
@@ -592,7 +586,7 @@ enum rtl_hal_state {
 	_HAL_STATE_START = 1,
 };
 
-enum rtl_desc92_rate {
+enum rtl_desc_rate {
 	DESC_RATE1M = 0x00,
 	DESC_RATE2M = 0x01,
 	DESC_RATE5_5M = 0x02,
@@ -2477,6 +2471,8 @@ struct rtl_global_var {
 	spinlock_t glb_list_lock;
 };
 
+#define IN_4WAY_TIMEOUT_TIME	(30 * MSEC_PER_SEC)	/* 30 seconds */
+
 struct rtl_btc_info {
 	u8 bt_type;
 	u8 btcoexist;
@@ -2485,6 +2481,7 @@ struct rtl_btc_info {
 
 	u8 ap_num;
 	bool in_4way;
+	unsigned long in_4way_ts;
 };
 
 struct bt_coexist_info {
@@ -2558,6 +2555,8 @@ struct rtl_btc_ops {
 	void (*btc_halt_notify) (void);
 	void (*btc_btinfo_notify) (struct rtl_priv *rtlpriv,
 				   u8 *tmp_buf, u8 length);
+	void (*btc_btmpinfo_notify)(struct rtl_priv *rtlpriv,
+				    u8 *tmp_buf, u8 length);
 	bool (*btc_is_limited_dig) (struct rtl_priv *rtlpriv);
 	bool (*btc_is_disable_edca_turbo) (struct rtl_priv *rtlpriv);
 	bool (*btc_is_bt_disabled) (struct rtl_priv *rtlpriv);
diff --git a/drivers/net/wireless/rsi/Makefile b/drivers/net/wireless/rsi/Makefile
index a475c813674a..ebb89965997a 100644
--- a/drivers/net/wireless/rsi/Makefile
+++ b/drivers/net/wireless/rsi/Makefile
@@ -3,6 +3,7 @@ rsi_91x-y			+= rsi_91x_core.o
 rsi_91x-y			+= rsi_91x_mac80211.o
 rsi_91x-y			+= rsi_91x_mgmt.o
 rsi_91x-y			+= rsi_91x_hal.o
+rsi_91x-y			+= rsi_91x_ps.o
 rsi_91x-$(CONFIG_RSI_DEBUGFS)	+= rsi_91x_debugfs.o
 
 rsi_usb-y			+= rsi_91x_usb.o rsi_91x_usb_ops.o
diff --git a/drivers/net/wireless/rsi/rsi_91x_core.c b/drivers/net/wireless/rsi/rsi_91x_core.c
index 68f04a76769e..2b0516d2f63d 100644
--- a/drivers/net/wireless/rsi/rsi_91x_core.c
+++ b/drivers/net/wireless/rsi/rsi_91x_core.c
@@ -16,6 +16,7 @@
 
 #include "rsi_mgmt.h"
 #include "rsi_common.h"
+#include "rsi_hal.h"
 
 /**
  * rsi_determine_min_weight_queue() - This function determines the queue with
@@ -136,6 +137,10 @@ static u8 rsi_core_determine_hal_queue(struct rsi_common *common)
 	u8 q_num = INVALID_QUEUE;
 	u8 ii = 0;
 
+	if (skb_queue_len(&common->tx_queue[MGMT_BEACON_Q])) {
+		q_num = MGMT_BEACON_Q;
+		return q_num;
+	}
 	if (skb_queue_len(&common->tx_queue[MGMT_SOFT_Q])) {
 		if (!common->mgmt_q_block)
 			q_num = MGMT_SOFT_Q;
@@ -268,11 +273,11 @@ void rsi_core_qos_processor(struct rsi_common *common)
 			break;
 		}
 
-		mutex_lock(&common->tx_rxlock);
+		mutex_lock(&common->tx_lock);
 
 		status = adapter->check_hw_queue_status(adapter, q_num);
 		if ((status <= 0)) {
-			mutex_unlock(&common->tx_rxlock);
+			mutex_unlock(&common->tx_lock);
 			break;
 		}
 
@@ -287,30 +292,48 @@ void rsi_core_qos_processor(struct rsi_common *common)
 		skb = rsi_core_dequeue_pkt(common, q_num);
 		if (skb == NULL) {
 			rsi_dbg(ERR_ZONE, "skb null\n");
-			mutex_unlock(&common->tx_rxlock);
+			mutex_unlock(&common->tx_lock);
 			break;
 		}
 
-		if (q_num == MGMT_SOFT_Q)
+		if (q_num == MGMT_SOFT_Q) {
 			status = rsi_send_mgmt_pkt(common, skb);
-		else
+		} else if (q_num == MGMT_BEACON_Q) {
+			status = rsi_send_pkt_to_bus(common, skb);
+			dev_kfree_skb(skb);
+		} else {
 			status = rsi_send_data_pkt(common, skb);
+		}
 
 		if (status) {
-			mutex_unlock(&common->tx_rxlock);
+			mutex_unlock(&common->tx_lock);
 			break;
 		}
 
 		common->tx_stats.total_tx_pkt_send[q_num]++;
 
 		tstamp_2 = jiffies;
-		mutex_unlock(&common->tx_rxlock);
+		mutex_unlock(&common->tx_lock);
 
 		if (time_after(tstamp_2, tstamp_1 + (300 * HZ) / 1000))
 			schedule();
 	}
 }
 
+struct rsi_sta *rsi_find_sta(struct rsi_common *common, u8 *mac_addr)
+{
+	int i;
+
+	for (i = 0; i < common->max_stations; i++) {
+		if (!common->stations[i].sta)
+			continue;
+		if (!(memcmp(common->stations[i].sta->addr,
+			     mac_addr, ETH_ALEN)))
+			return &common->stations[i];
+	}
+	return NULL;
+}
+
 /**
  * rsi_core_xmit() - This function transmits the packets received from mac80211
  * @common: Pointer to the driver private structure.
@@ -323,42 +346,63 @@ void rsi_core_xmit(struct rsi_common *common, struct sk_buff *skb)
 	struct rsi_hw *adapter = common->priv;
 	struct ieee80211_tx_info *info;
 	struct skb_info *tx_params;
-	struct ieee80211_hdr *tmp_hdr = NULL;
+	struct ieee80211_hdr *wh;
+	struct ieee80211_vif *vif = adapter->vifs[0];
 	u8 q_num, tid = 0;
+	struct rsi_sta *rsta = NULL;
 
 	if ((!skb) || (!skb->len)) {
 		rsi_dbg(ERR_ZONE, "%s: Null skb/zero Length packet\n",
 			__func__);
 		goto xmit_fail;
 	}
-	info = IEEE80211_SKB_CB(skb);
-	tx_params = (struct skb_info *)info->driver_data;
-	tmp_hdr = (struct ieee80211_hdr *)&skb->data[0];
-
 	if (common->fsm_state != FSM_MAC_INIT_DONE) {
 		rsi_dbg(ERR_ZONE, "%s: FSM state not open\n", __func__);
 		goto xmit_fail;
 	}
 
-	if ((ieee80211_is_mgmt(tmp_hdr->frame_control)) ||
-	    (ieee80211_is_ctl(tmp_hdr->frame_control)) ||
-	    (ieee80211_is_qos_nullfunc(tmp_hdr->frame_control))) {
+	info = IEEE80211_SKB_CB(skb);
+	tx_params = (struct skb_info *)info->driver_data;
+	wh = (struct ieee80211_hdr *)&skb->data[0];
+	tx_params->sta_id = 0;
+
+	if ((ieee80211_is_mgmt(wh->frame_control)) ||
+	    (ieee80211_is_ctl(wh->frame_control)) ||
+	    (ieee80211_is_qos_nullfunc(wh->frame_control))) {
 		q_num = MGMT_SOFT_Q;
 		skb->priority = q_num;
 	} else {
-		if (ieee80211_is_data_qos(tmp_hdr->frame_control)) {
+		if (ieee80211_is_data_qos(wh->frame_control)) {
 			tid = (skb->data[24] & IEEE80211_QOS_TID);
 			skb->priority = TID_TO_WME_AC(tid);
 		} else {
 			tid = IEEE80211_NONQOS_TID;
 			skb->priority = BE_Q;
 		}
+
 		q_num = skb->priority;
 		tx_params->tid = tid;
-		tx_params->sta_id = 0;
+
+		if ((vif->type == NL80211_IFTYPE_AP) &&
+		    (!is_broadcast_ether_addr(wh->addr1)) &&
+		    (!is_multicast_ether_addr(wh->addr1))) {
+			rsta = rsi_find_sta(common, wh->addr1);
+			if (!rsta)
+				goto xmit_fail;
+			tx_params->sta_id = rsta->sta_id;
+		}
+
+		if (rsta) {
+			/* Start aggregation if not done for this tid */
+			if (!rsta->start_tx_aggr[tid]) {
+				rsta->start_tx_aggr[tid] = true;
+				ieee80211_start_tx_ba_session(rsta->sta,
+							      tid, 0);
+			}
+		}
 	}
 
-	if ((q_num != MGMT_SOFT_Q) &&
+	if ((q_num < MGMT_SOFT_Q) &&
 	    ((skb_queue_len(&common->tx_queue[q_num]) + 1) >=
 	     DATA_QUEUE_WATER_MARK)) {
 		rsi_dbg(ERR_ZONE, "%s: sw queue full\n", __func__);
diff --git a/drivers/net/wireless/rsi/rsi_91x_debugfs.c b/drivers/net/wireless/rsi/rsi_91x_debugfs.c
index 4c0a493bd44e..e98eb55c26cc 100644
--- a/drivers/net/wireless/rsi/rsi_91x_debugfs.c
+++ b/drivers/net/wireless/rsi/rsi_91x_debugfs.c
@@ -130,6 +130,7 @@ static int rsi_stats_read(struct seq_file *seq, void *data)
 		"FSM_COMMON_DEV_PARAMS_SENT",
 		"FSM_BOOT_PARAMS_SENT",
 		"FSM_EEPROM_READ_MAC_ADDR",
+		"FSM_EEPROM_READ_RF_TYPE",
 		"FSM_RESET_MAC_SENT",
 		"FSM_RADIO_CAPS_SENT",
 		"FSM_BB_RF_PROG_SENT",
@@ -138,6 +139,8 @@ static int rsi_stats_read(struct seq_file *seq, void *data)
 	seq_puts(seq, "==> RSI STA DRIVER STATUS <==\n");
 	seq_puts(seq, "DRIVER_FSM_STATE: ");
 
+	BUILD_BUG_ON(ARRAY_SIZE(fsm_state) != NUM_FSM_STATES);
+
 	if (common->fsm_state <= FSM_MAC_INIT_DONE)
 		seq_printf(seq, "%s", fsm_state[common->fsm_state]);
 
diff --git a/drivers/net/wireless/rsi/rsi_91x_hal.c b/drivers/net/wireless/rsi/rsi_91x_hal.c
index c2303599c12e..070dfd68bb83 100644
--- a/drivers/net/wireless/rsi/rsi_91x_hal.c
+++ b/drivers/net/wireless/rsi/rsi_91x_hal.c
@@ -18,6 +18,7 @@
 #include "rsi_mgmt.h"
 #include "rsi_hal.h"
 #include "rsi_sdio.h"
+#include "rsi_common.h"
 
 /* FLASH Firmware */
 static struct ta_metadata metadata_flash_content[] = {
@@ -25,99 +26,268 @@ static struct ta_metadata metadata_flash_content[] = {
 	{"rsi/rs9113_wlan_qspi.rps", 0x00010000},
 };
 
-/**
- * rsi_send_data_pkt() - This function sends the recieved data packet from
- *			 driver to device.
- * @common: Pointer to the driver private structure.
- * @skb: Pointer to the socket buffer structure.
- *
- * Return: status: 0 on success, -1 on failure.
- */
-int rsi_send_data_pkt(struct rsi_common *common, struct sk_buff *skb)
+int rsi_send_pkt_to_bus(struct rsi_common *common, struct sk_buff *skb)
+{
+	struct rsi_hw *adapter = common->priv;
+	int status;
+
+	status = adapter->host_intf_ops->write_pkt(common->priv,
+						   skb->data, skb->len);
+	return status;
+}
+
+static int rsi_prepare_mgmt_desc(struct rsi_common *common, struct sk_buff *skb)
 {
 	struct rsi_hw *adapter = common->priv;
-	struct ieee80211_hdr *tmp_hdr;
+	struct ieee80211_hdr *wh = NULL;
+	struct ieee80211_tx_info *info;
+	struct ieee80211_conf *conf = &adapter->hw->conf;
+	struct ieee80211_vif *vif = adapter->vifs[0];
+	struct rsi_mgmt_desc *mgmt_desc;
+	struct skb_info *tx_params;
+	struct ieee80211_bss_conf *bss = NULL;
+	struct xtended_desc *xtend_desc = NULL;
+	u8 header_size;
+	u32 dword_align_bytes = 0;
+
+	if (skb->len > MAX_MGMT_PKT_SIZE) {
+		rsi_dbg(INFO_ZONE, "%s: Dropping mgmt pkt > 512\n", __func__);
+		return -EINVAL;
+	}
+
+	info = IEEE80211_SKB_CB(skb);
+	tx_params = (struct skb_info *)info->driver_data;
+
+	/* Update header size */
+	header_size = FRAME_DESC_SZ + sizeof(struct xtended_desc);
+	if (header_size > skb_headroom(skb)) {
+		rsi_dbg(ERR_ZONE,
+			"%s: Failed to add extended descriptor\n",
+			__func__);
+		return -ENOSPC;
+	}
+	skb_push(skb, header_size);
+	dword_align_bytes = ((unsigned long)skb->data & 0x3f);
+	if (dword_align_bytes > skb_headroom(skb)) {
+		rsi_dbg(ERR_ZONE,
+			"%s: Failed to add dword align\n", __func__);
+		return -ENOSPC;
+	}
+	skb_push(skb, dword_align_bytes);
+	header_size += dword_align_bytes;
+
+	tx_params->internal_hdr_size = header_size;
+	memset(&skb->data[0], 0, header_size);
+	bss = &info->control.vif->bss_conf;
+	wh = (struct ieee80211_hdr *)&skb->data[header_size];
+
+	mgmt_desc = (struct rsi_mgmt_desc *)skb->data;
+	xtend_desc = (struct xtended_desc *)&skb->data[FRAME_DESC_SZ];
+
+	rsi_set_len_qno(&mgmt_desc->len_qno, (skb->len - FRAME_DESC_SZ),
+			RSI_WIFI_MGMT_Q);
+	mgmt_desc->frame_type = TX_DOT11_MGMT;
+	mgmt_desc->header_len = MIN_802_11_HDR_LEN;
+	mgmt_desc->xtend_desc_size = header_size - FRAME_DESC_SZ;
+	mgmt_desc->frame_info |= cpu_to_le16(RATE_INFO_ENABLE);
+	if (is_broadcast_ether_addr(wh->addr1))
+		mgmt_desc->frame_info |= cpu_to_le16(RSI_BROADCAST_PKT);
+
+	mgmt_desc->seq_ctrl =
+		cpu_to_le16(IEEE80211_SEQ_TO_SN(le16_to_cpu(wh->seq_ctrl)));
+	if (common->band == NL80211_BAND_2GHZ)
+		mgmt_desc->rate_info = RSI_RATE_1;
+	else
+		mgmt_desc->rate_info = RSI_RATE_6;
+
+	if (conf_is_ht40(conf))
+		mgmt_desc->bbp_info = cpu_to_le16(FULL40M_ENABLE);
+
+	if (ieee80211_is_probe_req(wh->frame_control)) {
+		if (!bss->assoc) {
+			rsi_dbg(INFO_ZONE,
+				"%s: blocking mgmt queue\n", __func__);
+			mgmt_desc->misc_flags = RSI_DESC_REQUIRE_CFM_TO_HOST;
+			xtend_desc->confirm_frame_type = PROBEREQ_CONFIRM;
+			common->mgmt_q_block = true;
+			rsi_dbg(INFO_ZONE, "Mgmt queue blocked\n");
+		}
+	}
+
+	if (ieee80211_is_probe_resp(wh->frame_control)) {
+		mgmt_desc->misc_flags |= (RSI_ADD_DELTA_TSF_VAP_ID |
+					  RSI_FETCH_RETRY_CNT_FRM_HST);
+#define PROBE_RESP_RETRY_CNT	3
+		xtend_desc->retry_cnt = PROBE_RESP_RETRY_CNT;
+	}
+
+	if ((vif->type == NL80211_IFTYPE_AP) &&
+	    (ieee80211_is_action(wh->frame_control))) {
+		struct rsi_sta *rsta = rsi_find_sta(common, wh->addr1);
+
+		if (rsta)
+			mgmt_desc->sta_id = tx_params->sta_id;
+		else
+			return -EINVAL;
+	}
+	return 0;
+}
+
+/* This function prepares descriptor for given data packet */
+static int rsi_prepare_data_desc(struct rsi_common *common, struct sk_buff *skb)
+{
+	struct rsi_hw *adapter = common->priv;
+	struct ieee80211_vif *vif;
+	struct ieee80211_hdr *wh = NULL;
 	struct ieee80211_tx_info *info;
 	struct skb_info *tx_params;
 	struct ieee80211_bss_conf *bss;
-	int status;
+	struct rsi_data_desc *data_desc;
+	struct xtended_desc *xtend_desc;
 	u8 ieee80211_size = MIN_802_11_HDR_LEN;
-	u8 extnd_size;
-	__le16 *frame_desc;
+	u8 header_size;
+	u8 vap_id = 0;
+	u8 dword_align_bytes;
 	u16 seq_num;
 
 	info = IEEE80211_SKB_CB(skb);
 	bss = &info->control.vif->bss_conf;
 	tx_params = (struct skb_info *)info->driver_data;
 
-	if (!bss->assoc) {
-		status = -EINVAL;
-		goto err;
+	header_size = FRAME_DESC_SZ + sizeof(struct xtended_desc);
+	if (header_size > skb_headroom(skb)) {
+		rsi_dbg(ERR_ZONE, "%s: Unable to send pkt\n", __func__);
+		return -ENOSPC;
 	}
+	skb_push(skb, header_size);
+	dword_align_bytes = ((unsigned long)skb->data & 0x3f);
+	if (header_size > skb_headroom(skb)) {
+		rsi_dbg(ERR_ZONE, "%s: Not enough headroom\n", __func__);
+		return -ENOSPC;
+	}
+	skb_push(skb, dword_align_bytes);
+	header_size += dword_align_bytes;
 
-	tmp_hdr = (struct ieee80211_hdr *)&skb->data[0];
-	seq_num = (le16_to_cpu(tmp_hdr->seq_ctrl) >> 4);
+	tx_params->internal_hdr_size = header_size;
+	data_desc = (struct rsi_data_desc *)skb->data;
+	memset(data_desc, 0, header_size);
 
-	extnd_size = ((uintptr_t)skb->data & 0x3);
+	xtend_desc = (struct xtended_desc *)&skb->data[FRAME_DESC_SZ];
+	wh = (struct ieee80211_hdr *)&skb->data[header_size];
+	seq_num = IEEE80211_SEQ_TO_SN(le16_to_cpu(wh->seq_ctrl));
+	vif = adapter->vifs[0];
 
-	if ((FRAME_DESC_SZ + extnd_size) > skb_headroom(skb)) {
-		rsi_dbg(ERR_ZONE, "%s: Unable to send pkt\n", __func__);
-		status = -ENOSPC;
-		goto err;
-	}
+	data_desc->xtend_desc_size = header_size - FRAME_DESC_SZ;
 
-	skb_push(skb, (FRAME_DESC_SZ + extnd_size));
-	frame_desc = (__le16 *)&skb->data[0];
-	memset((u8 *)frame_desc, 0, FRAME_DESC_SZ);
-
-	if (ieee80211_is_data_qos(tmp_hdr->frame_control)) {
+	if (ieee80211_is_data_qos(wh->frame_control)) {
 		ieee80211_size += 2;
-		frame_desc[6] |= cpu_to_le16(BIT(12));
+		data_desc->mac_flags |= cpu_to_le16(RSI_QOS_ENABLE);
 	}
 
+	if ((vif->type == NL80211_IFTYPE_STATION) &&
+	    (adapter->ps_state == PS_ENABLED))
+		wh->frame_control |= cpu_to_le16(RSI_SET_PS_ENABLE);
+
 	if ((!(info->flags & IEEE80211_TX_INTFL_DONT_ENCRYPT)) &&
 	    (common->secinfo.security_enable)) {
 		if (rsi_is_cipher_wep(common))
 			ieee80211_size += 4;
 		else
 			ieee80211_size += 8;
-		frame_desc[6] |= cpu_to_le16(BIT(15));
+		data_desc->mac_flags |= cpu_to_le16(RSI_ENCRYPT_PKT);
 	}
+	rsi_set_len_qno(&data_desc->len_qno, (skb->len - FRAME_DESC_SZ),
+			RSI_WIFI_DATA_Q);
+	data_desc->header_len = ieee80211_size;
 
-	frame_desc[0] = cpu_to_le16((skb->len - FRAME_DESC_SZ) |
-				    (RSI_WIFI_DATA_Q << 12));
-	frame_desc[2] = cpu_to_le16((extnd_size) | (ieee80211_size) << 8);
-
-	if (common->min_rate != 0xffff) {
+	if (common->min_rate != RSI_RATE_AUTO) {
 		/* Send fixed rate */
-		frame_desc[3] = cpu_to_le16(RATE_INFO_ENABLE);
-		frame_desc[4] = cpu_to_le16(common->min_rate);
+		data_desc->frame_info = cpu_to_le16(RATE_INFO_ENABLE);
+		data_desc->rate_info = cpu_to_le16(common->min_rate);
 
 		if (conf_is_ht40(&common->priv->hw->conf))
-			frame_desc[5] = cpu_to_le16(FULL40M_ENABLE);
+			data_desc->bbp_info = cpu_to_le16(FULL40M_ENABLE);
 
-		if (common->vif_info[0].sgi) {
-			if (common->min_rate & 0x100) /* Only MCS rates */
-				frame_desc[4] |=
-					cpu_to_le16(ENABLE_SHORTGI_RATE);
+		if ((common->vif_info[0].sgi) && (common->min_rate & 0x100)) {
+		       /* Only MCS rates */
+			data_desc->rate_info |=
+				cpu_to_le16(ENABLE_SHORTGI_RATE);
 		}
+	}
 
+	if (skb->protocol == cpu_to_be16(ETH_P_PAE)) {
+		rsi_dbg(INFO_ZONE, "*** Tx EAPOL ***\n");
+
+		data_desc->frame_info = cpu_to_le16(RATE_INFO_ENABLE);
+		if (common->band == NL80211_BAND_5GHZ)
+			data_desc->rate_info = cpu_to_le16(RSI_RATE_6);
+		else
+			data_desc->rate_info = cpu_to_le16(RSI_RATE_1);
+		data_desc->mac_flags |= cpu_to_le16(RSI_REKEY_PURPOSE);
+		data_desc->misc_flags |= RSI_FETCH_RETRY_CNT_FRM_HST;
+#define EAPOL_RETRY_CNT 15
+		xtend_desc->retry_cnt = EAPOL_RETRY_CNT;
+	}
+
+	data_desc->mac_flags = cpu_to_le16(seq_num & 0xfff);
+	data_desc->qid_tid = ((skb->priority & 0xf) |
+			      ((tx_params->tid & 0xf) << 4));
+	data_desc->sta_id = tx_params->sta_id;
+
+	if ((is_broadcast_ether_addr(wh->addr1)) ||
+	    (is_multicast_ether_addr(wh->addr1))) {
+		data_desc->frame_info = cpu_to_le16(RATE_INFO_ENABLE);
+		data_desc->frame_info |= cpu_to_le16(RSI_BROADCAST_PKT);
+		data_desc->sta_id = vap_id;
+
+		if (vif->type == NL80211_IFTYPE_AP) {
+			if (common->band == NL80211_BAND_5GHZ)
+				data_desc->rate_info = cpu_to_le16(RSI_RATE_6);
+			else
+				data_desc->rate_info = cpu_to_le16(RSI_RATE_1);
+		}
 	}
+	if ((vif->type == NL80211_IFTYPE_AP) &&
+	    (ieee80211_has_moredata(wh->frame_control)))
+		data_desc->frame_info |= cpu_to_le16(MORE_DATA_PRESENT);
+
+	return 0;
+}
+
+/* This function sends received data packet from driver to device */
+int rsi_send_data_pkt(struct rsi_common *common, struct sk_buff *skb)
+{
+	struct rsi_hw *adapter = common->priv;
+	struct ieee80211_vif *vif = adapter->vifs[0];
+	struct ieee80211_tx_info *info;
+	struct ieee80211_bss_conf *bss;
+	int status = -EINVAL;
+
+	if (!skb)
+		return 0;
+	if (common->iface_down)
+		goto err;
 
-	frame_desc[6] |= cpu_to_le16(seq_num & 0xfff);
-	frame_desc[7] = cpu_to_le16(((tx_params->tid & 0xf) << 4) |
-				    (skb->priority & 0xf) |
-				    (tx_params->sta_id << 8));
+	info = IEEE80211_SKB_CB(skb);
+	if (!info->control.vif)
+		goto err;
+	bss = &info->control.vif->bss_conf;
+
+	if ((vif->type == NL80211_IFTYPE_STATION) && (!bss->assoc))
+		goto err;
+
+	status = rsi_prepare_data_desc(common, skb);
+	if (status)
+		goto err;
 
 	status = adapter->host_intf_ops->write_pkt(common->priv, skb->data,
 						   skb->len);
 	if (status)
-		rsi_dbg(ERR_ZONE, "%s: Failed to write pkt\n",
-			__func__);
+		rsi_dbg(ERR_ZONE, "%s: Failed to write pkt\n", __func__);
 
 err:
 	++common->tx_stats.total_tx_pkt_freed[skb->priority];
-	rsi_indicate_tx_status(common->priv, skb, status);
+	rsi_indicate_tx_status(adapter, skb, status);
 	return status;
 }
 
@@ -133,22 +303,17 @@ int rsi_send_mgmt_pkt(struct rsi_common *common,
 		      struct sk_buff *skb)
 {
 	struct rsi_hw *adapter = common->priv;
-	struct ieee80211_hdr *wh;
 	struct ieee80211_tx_info *info;
-	struct ieee80211_bss_conf *bss;
-	struct ieee80211_hw *hw = adapter->hw;
-	struct ieee80211_conf *conf = &hw->conf;
 	struct skb_info *tx_params;
 	int status = -E2BIG;
-	__le16 *msg;
 	u8 extnd_size;
-	u8 vap_id = 0;
 
 	info = IEEE80211_SKB_CB(skb);
 	tx_params = (struct skb_info *)info->driver_data;
 	extnd_size = ((uintptr_t)skb->data & 0x3);
 
 	if (tx_params->flags & INTERNAL_MGMT_PKT) {
+		skb->data[1] |= BIT(7); /* Immediate Wakeup bit*/
 		if ((extnd_size) > skb_headroom(skb)) {
 			rsi_dbg(ERR_ZONE, "%s: Unable to send pkt\n", __func__);
 			dev_kfree_skb(skb);
@@ -167,58 +332,73 @@ int rsi_send_mgmt_pkt(struct rsi_common *common,
 		return status;
 	}
 
-	bss = &info->control.vif->bss_conf;
-	wh = (struct ieee80211_hdr *)&skb->data[0];
-
 	if (FRAME_DESC_SZ > skb_headroom(skb))
 		goto err;
 
-	skb_push(skb, FRAME_DESC_SZ);
-	memset(skb->data, 0, FRAME_DESC_SZ);
-	msg = (__le16 *)skb->data;
+	rsi_prepare_mgmt_desc(common, skb);
+	status = adapter->host_intf_ops->write_pkt(common->priv,
+						   (u8 *)skb->data, skb->len);
+	if (status)
+		rsi_dbg(ERR_ZONE, "%s: Failed to write the packet\n", __func__);
 
-	if (skb->len > MAX_MGMT_PKT_SIZE) {
-		rsi_dbg(INFO_ZONE, "%s: Dropping mgmt pkt > 512\n", __func__);
-		goto err;
+err:
+	rsi_indicate_tx_status(common->priv, skb, status);
+	return status;
+}
+
+int rsi_prepare_beacon(struct rsi_common *common, struct sk_buff *skb)
+{
+	struct rsi_hw *adapter = (struct rsi_hw *)common->priv;
+	struct rsi_data_desc *bcn_frm;
+	struct ieee80211_hw *hw = common->priv->hw;
+	struct ieee80211_conf *conf = &hw->conf;
+	struct sk_buff *mac_bcn;
+	u8 vap_id = 0;
+	u16 tim_offset;
+
+	mac_bcn = ieee80211_beacon_get_tim(adapter->hw,
+					   adapter->vifs[adapter->sc_nvifs - 1],
+					   &tim_offset, NULL);
+	if (!mac_bcn) {
+		rsi_dbg(ERR_ZONE, "Failed to get beacon from mac80211\n");
+		return -EINVAL;
 	}
 
-	msg[0] = cpu_to_le16((skb->len - FRAME_DESC_SZ) |
-			    (RSI_WIFI_MGMT_Q << 12));
-	msg[1] = cpu_to_le16(TX_DOT11_MGMT);
-	msg[2] = cpu_to_le16(MIN_802_11_HDR_LEN << 8);
-	msg[3] = cpu_to_le16(RATE_INFO_ENABLE);
-	msg[6] = cpu_to_le16(le16_to_cpu(wh->seq_ctrl) >> 4);
+	common->beacon_cnt++;
+	bcn_frm = (struct rsi_data_desc *)skb->data;
+	rsi_set_len_qno(&bcn_frm->len_qno, mac_bcn->len, RSI_WIFI_DATA_Q);
+	bcn_frm->header_len = MIN_802_11_HDR_LEN;
+	bcn_frm->frame_info = cpu_to_le16(RSI_DATA_DESC_MAC_BBP_INFO |
+					  RSI_DATA_DESC_NO_ACK_IND |
+					  RSI_DATA_DESC_BEACON_FRAME |
+					  RSI_DATA_DESC_INSERT_TSF |
+					  RSI_DATA_DESC_INSERT_SEQ_NO |
+					  RATE_INFO_ENABLE);
+	bcn_frm->rate_info = cpu_to_le16(vap_id << 14);
+	bcn_frm->qid_tid = BEACON_HW_Q;
 
-	if (wh->addr1[0] & BIT(0))
-		msg[3] |= cpu_to_le16(RSI_BROADCAST_PKT);
+	if (conf_is_ht40_plus(conf)) {
+		bcn_frm->bbp_info = cpu_to_le16(LOWER_20_ENABLE);
+		bcn_frm->bbp_info |= cpu_to_le16(LOWER_20_ENABLE >> 12);
+	} else if (conf_is_ht40_minus(conf)) {
+		bcn_frm->bbp_info = cpu_to_le16(UPPER_20_ENABLE);
+		bcn_frm->bbp_info |= cpu_to_le16(UPPER_20_ENABLE >> 12);
+	}
 
 	if (common->band == NL80211_BAND_2GHZ)
-		msg[4] = cpu_to_le16(RSI_11B_MODE);
+		bcn_frm->bbp_info |= cpu_to_le16(RSI_RATE_1);
 	else
-		msg[4] = cpu_to_le16((RSI_RATE_6 & 0x0f) | RSI_11G_MODE);
+		bcn_frm->bbp_info |= cpu_to_le16(RSI_RATE_6);
 
-	if (conf_is_ht40(conf)) {
-		msg[4] = cpu_to_le16(0xB | RSI_11G_MODE);
-		msg[5] = cpu_to_le16(0x6);
-	}
+	if (mac_bcn->data[tim_offset + 2] == 0)
+		bcn_frm->frame_info |= cpu_to_le16(RSI_DATA_DESC_DTIM_BEACON);
 
-	/* Indicate to firmware to give cfm */
-	if ((skb->data[16] == IEEE80211_STYPE_PROBE_REQ) && (!bss->assoc)) {
-		msg[1] |= cpu_to_le16(BIT(10));
-		msg[7] = cpu_to_le16(PROBEREQ_CONFIRM);
-		common->mgmt_q_block = true;
-	}
+	memcpy(&skb->data[FRAME_DESC_SZ], mac_bcn->data, mac_bcn->len);
+	skb_put(skb, mac_bcn->len + FRAME_DESC_SZ);
 
-	msg[7] |= cpu_to_le16(vap_id << 8);
+	dev_kfree_skb(mac_bcn);
 
-	status = adapter->host_intf_ops->write_pkt(common->priv, (u8 *)msg,
-						   skb->len);
-	if (status)
-		rsi_dbg(ERR_ZONE, "%s: Failed to write the packet\n", __func__);
-
-err:
-	rsi_indicate_tx_status(common->priv, skb, status);
-	return status;
+	return 0;
 }
 
 static void bl_cmd_timeout(unsigned long priv)
diff --git a/drivers/net/wireless/rsi/rsi_91x_mac80211.c b/drivers/net/wireless/rsi/rsi_91x_mac80211.c
index 021e5ac5f107..fa12c05d9e23 100644
--- a/drivers/net/wireless/rsi/rsi_91x_mac80211.c
+++ b/drivers/net/wireless/rsi/rsi_91x_mac80211.c
@@ -18,6 +18,7 @@
 #include "rsi_debugfs.h"
 #include "rsi_mgmt.h"
 #include "rsi_common.h"
+#include "rsi_ps.h"
 
 static const struct ieee80211_channel rsi_2ghz_channels[] = {
 	{ .band = NL80211_BAND_2GHZ, .center_freq = 2412,
@@ -121,6 +122,23 @@ const u16 rsi_mcsrates[8] = {
 	RSI_RATE_MCS4, RSI_RATE_MCS5, RSI_RATE_MCS6, RSI_RATE_MCS7
 };
 
+static const u32 rsi_max_ap_stas[16] = {
+	32,	/* 1 - Wi-Fi alone */
+	0,	/* 2 */
+	0,	/* 3 */
+	0,	/* 4 - BT EDR alone */
+	4,	/* 5 - STA + BT EDR */
+	32,	/* 6 - AP + BT EDR */
+	0,	/* 7 */
+	0,	/* 8 - BT LE alone */
+	4,	/* 9 - STA + BE LE */
+	0,	/* 10 */
+	0,	/* 11 */
+	0,	/* 12 */
+	1,	/* 13 - STA + BT Dual */
+	4,	/* 14 - AP + BT Dual */
+};
+
 /**
  * rsi_is_cipher_wep() -  This function determines if the cipher is WEP or not.
  * @common: Pointer to the driver private structure.
@@ -229,12 +247,20 @@ void rsi_indicate_tx_status(struct rsi_hw *adapter,
 			    int status)
 {
 	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	struct skb_info *tx_params;
 
-	memset(info->driver_data, 0, IEEE80211_TX_INFO_DRIVER_DATA_SIZE);
+	if (!adapter->hw) {
+		rsi_dbg(ERR_ZONE, "##### No MAC #####\n");
+		return;
+	}
 
 	if (!status)
 		info->flags |= IEEE80211_TX_STAT_ACK;
 
+	tx_params = (struct skb_info *)info->driver_data;
+	skb_pull(skb, tx_params->internal_hdr_size);
+	memset(info->driver_data, 0, IEEE80211_TX_INFO_DRIVER_DATA_SIZE);
+
 	ieee80211_tx_status_irqsafe(adapter->hw, skb);
 }
 
@@ -271,11 +297,12 @@ static int rsi_mac80211_start(struct ieee80211_hw *hw)
 	struct rsi_hw *adapter = hw->priv;
 	struct rsi_common *common = adapter->priv;
 
+	rsi_dbg(ERR_ZONE, "===> Interface UP <===\n");
 	mutex_lock(&common->mutex);
 	common->iface_down = false;
-	mutex_unlock(&common->mutex);
-
+	wiphy_rfkill_start_polling(hw->wiphy);
 	rsi_send_rx_filter_frame(common, 0);
+	mutex_unlock(&common->mutex);
 
 	return 0;
 }
@@ -291,8 +318,14 @@ static void rsi_mac80211_stop(struct ieee80211_hw *hw)
 	struct rsi_hw *adapter = hw->priv;
 	struct rsi_common *common = adapter->priv;
 
+	rsi_dbg(ERR_ZONE, "===> Interface DOWN <===\n");
 	mutex_lock(&common->mutex);
 	common->iface_down = true;
+	wiphy_rfkill_stop_polling(hw->wiphy);
+
+	/* Block all rx frames */
+	rsi_send_rx_filter_frame(common, 0xffff);
+
 	mutex_unlock(&common->mutex);
 }
 
@@ -309,24 +342,51 @@ static int rsi_mac80211_add_interface(struct ieee80211_hw *hw,
 {
 	struct rsi_hw *adapter = hw->priv;
 	struct rsi_common *common = adapter->priv;
+	enum opmode intf_mode;
 	int ret = -EOPNOTSUPP;
 
+	vif->driver_flags |= IEEE80211_VIF_SUPPORTS_UAPSD;
 	mutex_lock(&common->mutex);
+
+	if (adapter->sc_nvifs > 1) {
+		mutex_unlock(&common->mutex);
+		return -EOPNOTSUPP;
+	}
+
 	switch (vif->type) {
 	case NL80211_IFTYPE_STATION:
-		if (!adapter->sc_nvifs) {
-			++adapter->sc_nvifs;
-			adapter->vifs[0] = vif;
-			ret = rsi_set_vap_capabilities(common,
-						       STA_OPMODE,
-						       VAP_ADD);
-		}
+		rsi_dbg(INFO_ZONE, "Station Mode");
+		intf_mode = STA_OPMODE;
+		break;
+	case NL80211_IFTYPE_AP:
+		rsi_dbg(INFO_ZONE, "AP Mode");
+		intf_mode = AP_OPMODE;
 		break;
 	default:
 		rsi_dbg(ERR_ZONE,
 			"%s: Interface type %d not supported\n", __func__,
 			vif->type);
+		goto out;
+	}
+
+	adapter->vifs[adapter->sc_nvifs++] = vif;
+	ret = rsi_set_vap_capabilities(common, intf_mode, common->mac_addr,
+				       0, VAP_ADD);
+	if (ret) {
+		rsi_dbg(ERR_ZONE, "Failed to set VAP capabilities\n");
+		goto out;
+	}
+
+	if (vif->type == NL80211_IFTYPE_AP) {
+		int i;
+
+		rsi_send_rx_filter_frame(common, DISALLOW_BEACONS);
+		common->min_rate = RSI_RATE_AUTO;
+		for (i = 0; i < common->max_stations; i++)
+			common->stations[i].sta = NULL;
 	}
+
+out:
 	mutex_unlock(&common->mutex);
 
 	return ret;
@@ -345,13 +405,32 @@ static void rsi_mac80211_remove_interface(struct ieee80211_hw *hw,
 {
 	struct rsi_hw *adapter = hw->priv;
 	struct rsi_common *common = adapter->priv;
+	enum opmode opmode;
+
+	rsi_dbg(INFO_ZONE, "Remove Interface Called\n");
 
 	mutex_lock(&common->mutex);
-	if (vif->type == NL80211_IFTYPE_STATION) {
-		adapter->sc_nvifs--;
-		rsi_set_vap_capabilities(common, STA_OPMODE, VAP_DELETE);
+
+	if (adapter->sc_nvifs <= 0) {
+		mutex_unlock(&common->mutex);
+		return;
 	}
 
+	switch (vif->type) {
+	case NL80211_IFTYPE_STATION:
+		opmode = STA_OPMODE;
+		break;
+	case NL80211_IFTYPE_AP:
+		opmode = AP_OPMODE;
+		break;
+	default:
+		mutex_unlock(&common->mutex);
+		return;
+	}
+	rsi_set_vap_capabilities(common, opmode, vif->addr,
+				 0, VAP_DELETE);
+	adapter->sc_nvifs--;
+
 	if (!memcmp(adapter->vifs[0], vif, sizeof(struct ieee80211_vif)))
 		adapter->vifs[0] = NULL;
 	mutex_unlock(&common->mutex);
@@ -452,6 +531,8 @@ static int rsi_mac80211_config(struct ieee80211_hw *hw,
 {
 	struct rsi_hw *adapter = hw->priv;
 	struct rsi_common *common = adapter->priv;
+	struct ieee80211_vif *vif = adapter->vifs[0];
+	struct ieee80211_conf *conf = &hw->conf;
 	int status = -EOPNOTSUPP;
 
 	mutex_lock(&common->mutex);
@@ -465,6 +546,28 @@ static int rsi_mac80211_config(struct ieee80211_hw *hw,
 		status = rsi_config_power(hw);
 	}
 
+	/* Power save parameters */
+	if ((changed & IEEE80211_CONF_CHANGE_PS) &&
+	    (vif->type == NL80211_IFTYPE_STATION)) {
+		unsigned long flags;
+
+		spin_lock_irqsave(&adapter->ps_lock, flags);
+		if (conf->flags & IEEE80211_CONF_PS)
+			rsi_enable_ps(adapter);
+		else
+			rsi_disable_ps(adapter);
+		spin_unlock_irqrestore(&adapter->ps_lock, flags);
+	}
+
+	/* RTS threshold */
+	if (changed & WIPHY_PARAM_RTS_THRESHOLD) {
+		rsi_dbg(INFO_ZONE, "RTS threshold\n");
+		if ((common->rts_threshold) <= IEEE80211_MAX_RTS_THRESHOLD) {
+			rsi_dbg(INFO_ZONE,
+				"%s: Sending vap updates....\n", __func__);
+			status = rsi_send_vap_dynamic_update(common);
+		}
+	}
 	mutex_unlock(&common->mutex);
 
 	return status;
@@ -507,6 +610,8 @@ static void rsi_mac80211_bss_info_changed(struct ieee80211_hw *hw,
 {
 	struct rsi_hw *adapter = hw->priv;
 	struct rsi_common *common = adapter->priv;
+	struct ieee80211_bss_conf *bss = &vif->bss_conf;
+	struct ieee80211_conf *conf = &hw->conf;
 	u16 rx_filter_word = 0;
 
 	mutex_lock(&common->mutex);
@@ -521,10 +626,24 @@ static void rsi_mac80211_bss_info_changed(struct ieee80211_hw *hw,
 			rsi_send_rx_filter_frame(common, rx_filter_word);
 		}
 		rsi_inform_bss_status(common,
+				      STA_OPMODE,
 				      bss_conf->assoc,
 				      bss_conf->bssid,
 				      bss_conf->qos,
-				      bss_conf->aid);
+				      bss_conf->aid,
+				      NULL, 0);
+		adapter->ps_info.dtim_interval_duration = bss->dtim_period;
+		adapter->ps_info.listen_interval = conf->listen_interval;
+
+	/* If U-APSD is updated, send ps parameters to firmware */
+	if (bss->assoc) {
+		if (common->uapsd_bitmap) {
+			rsi_dbg(INFO_ZONE, "Configuring UAPSD\n");
+			rsi_conf_uapsd(adapter);
+		}
+	} else {
+		common->uapsd_bitmap = 0;
+	}
 	}
 
 	if (changed & BSS_CHANGED_CQM) {
@@ -535,6 +654,18 @@ static void rsi_mac80211_bss_info_changed(struct ieee80211_hw *hw,
 			common->cqm_info.rssi_thold,
 			common->cqm_info.rssi_hyst);
 	}
+
+	if ((changed & BSS_CHANGED_BEACON_ENABLED) &&
+	    (vif->type == NL80211_IFTYPE_AP)) {
+		if (bss->enable_beacon) {
+			rsi_dbg(INFO_ZONE, "===> BEACON ENABLED <===\n");
+			common->beacon_enabled = 1;
+		} else {
+			rsi_dbg(INFO_ZONE, "===> BEACON DISABLED <===\n");
+			common->beacon_enabled = 0;
+		}
+	}
+
 	mutex_unlock(&common->mutex);
 }
 
@@ -606,6 +737,12 @@ static int rsi_mac80211_conf_tx(struct ieee80211_hw *hw,
 	memcpy(&common->edca_params[idx],
 	       params,
 	       sizeof(struct ieee80211_tx_queue_params));
+
+	if (params->uapsd)
+		common->uapsd_bitmap |= idx;
+	else
+		common->uapsd_bitmap &= (~idx);
+
 	mutex_unlock(&common->mutex);
 
 	return 0;
@@ -617,15 +754,18 @@ static int rsi_mac80211_conf_tx(struct ieee80211_hw *hw,
  * @vif: Pointer to the ieee80211_vif structure.
  * @key: Pointer to the ieee80211_key_conf structure.
  *
- * Return: status: 0 on success, -1 on failure.
+ * Return: status: 0 on success, negative error codes on failure.
  */
 static int rsi_hal_key_config(struct ieee80211_hw *hw,
 			      struct ieee80211_vif *vif,
-			      struct ieee80211_key_conf *key)
+			      struct ieee80211_key_conf *key,
+			      struct ieee80211_sta *sta)
 {
 	struct rsi_hw *adapter = hw->priv;
+	struct rsi_sta *rsta = NULL;
 	int status;
 	u8 key_type;
+	s16 sta_id = 0;
 
 	if (key->flags & IEEE80211_KEY_FLAG_PAIRWISE)
 		key_type = RSI_PAIRWISE_KEY;
@@ -635,23 +775,35 @@ static int rsi_hal_key_config(struct ieee80211_hw *hw,
 	rsi_dbg(ERR_ZONE, "%s: Cipher 0x%x key_type: %d key_len: %d\n",
 		__func__, key->cipher, key_type, key->keylen);
 
-	if ((key->cipher == WLAN_CIPHER_SUITE_WEP104) ||
-	    (key->cipher == WLAN_CIPHER_SUITE_WEP40)) {
-		status = rsi_hal_load_key(adapter->priv,
-					  key->key,
-					  key->keylen,
-					  RSI_PAIRWISE_KEY,
-					  key->keyidx,
-					  key->cipher);
-		if (status)
-			return status;
+	if (vif->type == NL80211_IFTYPE_AP) {
+		if (sta) {
+			rsta = rsi_find_sta(adapter->priv, sta->addr);
+			if (rsta)
+				sta_id = rsta->sta_id;
+		}
+		adapter->priv->key = key;
+	} else {
+		if ((key->cipher == WLAN_CIPHER_SUITE_WEP104) ||
+		    (key->cipher == WLAN_CIPHER_SUITE_WEP40)) {
+			status = rsi_hal_load_key(adapter->priv,
+						  key->key,
+						  key->keylen,
+						  RSI_PAIRWISE_KEY,
+						  key->keyidx,
+						  key->cipher,
+						  sta_id);
+			if (status)
+				return status;
+		}
 	}
+
 	return rsi_hal_load_key(adapter->priv,
 				key->key,
 				key->keylen,
 				key_type,
 				key->keyidx,
-				key->cipher);
+				key->cipher,
+				sta_id);
 }
 
 /**
@@ -679,7 +831,7 @@ static int rsi_mac80211_set_key(struct ieee80211_hw *hw,
 	switch (cmd) {
 	case SET_KEY:
 		secinfo->security_enable = true;
-		status = rsi_hal_key_config(hw, vif, key);
+		status = rsi_hal_key_config(hw, vif, key, sta);
 		if (status) {
 			mutex_unlock(&common->mutex);
 			return status;
@@ -697,10 +849,11 @@ static int rsi_mac80211_set_key(struct ieee80211_hw *hw,
 		break;
 
 	case DISABLE_KEY:
-		secinfo->security_enable = false;
+		if (vif->type == NL80211_IFTYPE_STATION)
+			secinfo->security_enable = false;
 		rsi_dbg(ERR_ZONE, "%s: RSI del key\n", __func__);
 		memset(key, 0, sizeof(struct ieee80211_key_conf));
-		status = rsi_hal_key_config(hw, vif, key);
+		status = rsi_hal_key_config(hw, vif, key, sta);
 		break;
 
 	default:
@@ -729,9 +882,11 @@ static int rsi_mac80211_ampdu_action(struct ieee80211_hw *hw,
 	int status = -EOPNOTSUPP;
 	struct rsi_hw *adapter = hw->priv;
 	struct rsi_common *common = adapter->priv;
-	u16 seq_no = 0;
+	struct rsi_sta *rsta = NULL;
+	u16 seq_no = 0, seq_start = 0;
 	u8 ii = 0;
 	struct ieee80211_sta *sta = params->sta;
+	u8 sta_id = 0;
 	enum ieee80211_ampdu_mlme_action action = params->action;
 	u16 tid = params->tid;
 	u16 *ssn = &params->ssn;
@@ -743,17 +898,32 @@ static int rsi_mac80211_ampdu_action(struct ieee80211_hw *hw,
 	}
 
 	mutex_lock(&common->mutex);
-	rsi_dbg(INFO_ZONE, "%s: AMPDU action %d called\n", __func__, action);
+
 	if (ssn != NULL)
 		seq_no = *ssn;
 
+	if (vif->type == NL80211_IFTYPE_AP) {
+		rsta = rsi_find_sta(common, sta->addr);
+		if (!rsta) {
+			rsi_dbg(ERR_ZONE, "No station mapped\n");
+			status = 0;
+			goto unlock;
+		}
+		sta_id = rsta->sta_id;
+	}
+
+	rsi_dbg(INFO_ZONE,
+		"%s: AMPDU action tid=%d ssn=0x%x, buf_size=%d sta_id=%d\n",
+		__func__, tid, seq_no, buf_size, sta_id);
+
 	switch (action) {
 	case IEEE80211_AMPDU_RX_START:
 		status = rsi_send_aggregation_params_frame(common,
 							   tid,
 							   seq_no,
 							   buf_size,
-							   STA_RX_ADDBA_DONE);
+							   STA_RX_ADDBA_DONE,
+							   sta_id);
 		break;
 
 	case IEEE80211_AMPDU_RX_STOP:
@@ -761,11 +931,15 @@ static int rsi_mac80211_ampdu_action(struct ieee80211_hw *hw,
 							   tid,
 							   0,
 							   buf_size,
-							   STA_RX_DELBA);
+							   STA_RX_DELBA,
+							   sta_id);
 		break;
 
 	case IEEE80211_AMPDU_TX_START:
-		common->vif_info[ii].seq_start = seq_no;
+		if (vif->type == NL80211_IFTYPE_STATION)
+			common->vif_info[ii].seq_start = seq_no;
+		else if (vif->type == NL80211_IFTYPE_AP)
+			rsta->seq_start[tid] = seq_no;
 		ieee80211_start_tx_ba_cb_irqsafe(vif, sta->addr, tid);
 		status = 0;
 		break;
@@ -777,18 +951,23 @@ static int rsi_mac80211_ampdu_action(struct ieee80211_hw *hw,
 							   tid,
 							   seq_no,
 							   buf_size,
-							   STA_TX_DELBA);
+							   STA_TX_DELBA,
+							   sta_id);
 		if (!status)
 			ieee80211_stop_tx_ba_cb_irqsafe(vif, sta->addr, tid);
 		break;
 
 	case IEEE80211_AMPDU_TX_OPERATIONAL:
+		if (vif->type == NL80211_IFTYPE_STATION)
+			seq_start = common->vif_info[ii].seq_start;
+		else if (vif->type == NL80211_IFTYPE_AP)
+			seq_start = rsta->seq_start[tid];
 		status = rsi_send_aggregation_params_frame(common,
 							   tid,
-							   common->vif_info[ii]
-								.seq_start,
+							   seq_start,
 							   buf_size,
-							   STA_TX_ADDBA_DONE);
+							   STA_TX_ADDBA_DONE,
+							   sta_id);
 		break;
 
 	default:
@@ -796,6 +975,7 @@ static int rsi_mac80211_ampdu_action(struct ieee80211_hw *hw,
 		break;
 	}
 
+unlock:
 	mutex_unlock(&common->mutex);
 	return status;
 }
@@ -1014,7 +1194,7 @@ static void rsi_set_min_rate(struct ieee80211_hw *hw,
  * @vif: Pointer to the ieee80211_vif structure.
  * @sta: Pointer to the ieee80211_sta structure.
  *
- * Return: 0 on success, -1 on failure.
+ * Return: 0 on success, negative error codes on failure.
  */
 static int rsi_mac80211_sta_add(struct ieee80211_hw *hw,
 				struct ieee80211_vif *vif,
@@ -1022,22 +1202,101 @@ static int rsi_mac80211_sta_add(struct ieee80211_hw *hw,
 {
 	struct rsi_hw *adapter = hw->priv;
 	struct rsi_common *common = adapter->priv;
+	bool sta_exist = false;
+	struct rsi_sta *rsta;
+	int status = 0;
+
+	rsi_dbg(INFO_ZONE, "Station Add: %pM\n", sta->addr);
 
 	mutex_lock(&common->mutex);
 
-	rsi_set_min_rate(hw, sta, common);
+	if (vif->type == NL80211_IFTYPE_AP) {
+		u8 cnt;
+		int sta_idx = -1;
+		int free_index = -1;
+
+		/* Check if max stations reached */
+		if (common->num_stations >= common->max_stations) {
+			rsi_dbg(ERR_ZONE, "Reject: Max Stations exists\n");
+			status = -EOPNOTSUPP;
+			goto unlock;
+		}
+		for (cnt = 0; cnt < common->max_stations; cnt++) {
+			rsta = &common->stations[cnt];
 
-	if ((sta->ht_cap.cap & IEEE80211_HT_CAP_SGI_20) ||
-	    (sta->ht_cap.cap & IEEE80211_HT_CAP_SGI_40)) {
-		common->vif_info[0].sgi = true;
+			if (!rsta->sta) {
+				if (free_index < 0)
+					free_index = cnt;
+				continue;
+			}
+			if (!memcmp(rsta->sta->addr, sta->addr, ETH_ALEN)) {
+				rsi_dbg(INFO_ZONE, "Station exists\n");
+				sta_idx = cnt;
+				sta_exist = true;
+				break;
+			}
+		}
+		if (!sta_exist) {
+			if (free_index >= 0)
+				sta_idx = free_index;
+		}
+		if (sta_idx < 0) {
+			rsi_dbg(ERR_ZONE,
+				"%s: Some problem reaching here...\n",
+				__func__);
+			status = -EINVAL;
+			goto unlock;
+		}
+		rsta = &common->stations[sta_idx];
+		rsta->sta = sta;
+		rsta->sta_id = sta_idx;
+		for (cnt = 0; cnt < IEEE80211_NUM_TIDS; cnt++)
+			rsta->start_tx_aggr[cnt] = false;
+		for (cnt = 0; cnt < IEEE80211_NUM_TIDS; cnt++)
+			rsta->seq_start[cnt] = 0;
+		if (!sta_exist) {
+			rsi_dbg(INFO_ZONE, "New Station\n");
+
+			/* Send peer notify to device */
+			rsi_dbg(INFO_ZONE, "Indicate bss status to device\n");
+			rsi_inform_bss_status(common, AP_OPMODE, 1, sta->addr,
+					      sta->wme, sta->aid, sta, sta_idx);
+
+			if (common->key) {
+				struct ieee80211_key_conf *key = common->key;
+
+				if ((key->cipher == WLAN_CIPHER_SUITE_WEP104) ||
+				    (key->cipher == WLAN_CIPHER_SUITE_WEP40))
+					rsi_hal_load_key(adapter->priv,
+							 key->key,
+							 key->keylen,
+							 RSI_PAIRWISE_KEY,
+							 key->keyidx,
+							 key->cipher,
+							 sta_idx);
+			}
+
+			common->num_stations++;
+		}
 	}
 
-	if (sta->ht_cap.ht_supported)
-		ieee80211_start_tx_ba_session(sta, 0, 0);
+	if (vif->type == NL80211_IFTYPE_STATION) {
+		rsi_set_min_rate(hw, sta, common);
+		if (sta->ht_cap.ht_supported) {
+			common->vif_info[0].is_ht = true;
+			common->bitrate_mask[NL80211_BAND_2GHZ] =
+					sta->supp_rates[NL80211_BAND_2GHZ];
+			if ((sta->ht_cap.cap & IEEE80211_HT_CAP_SGI_20) ||
+			    (sta->ht_cap.cap & IEEE80211_HT_CAP_SGI_40))
+				common->vif_info[0].sgi = true;
+			ieee80211_start_tx_ba_session(sta, 0, 0);
+		}
+	}
 
+unlock:
 	mutex_unlock(&common->mutex);
 
-	return 0;
+	return status;
 }
 
 /**
@@ -1047,7 +1306,7 @@ static int rsi_mac80211_sta_add(struct ieee80211_hw *hw,
  * @vif: Pointer to the ieee80211_vif structure.
  * @sta: Pointer to the ieee80211_sta structure.
  *
- * Return: 0 on success, -1 on failure.
+ * Return: 0 on success, negative error codes on failure.
  */
 static int rsi_mac80211_sta_remove(struct ieee80211_hw *hw,
 				   struct ieee80211_vif *vif,
@@ -1055,21 +1314,55 @@ static int rsi_mac80211_sta_remove(struct ieee80211_hw *hw,
 {
 	struct rsi_hw *adapter = hw->priv;
 	struct rsi_common *common = adapter->priv;
+	struct ieee80211_bss_conf *bss = &vif->bss_conf;
+	struct rsi_sta *rsta;
+
+	rsi_dbg(INFO_ZONE, "Station Remove: %pM\n", sta->addr);
 
 	mutex_lock(&common->mutex);
 
-	/* Resetting all the fields to default values */
-	common->bitrate_mask[NL80211_BAND_2GHZ] = 0;
-	common->bitrate_mask[NL80211_BAND_5GHZ] = 0;
-	common->min_rate = 0xffff;
-	common->vif_info[0].is_ht = false;
-	common->vif_info[0].sgi = false;
-	common->vif_info[0].seq_start = 0;
-	common->secinfo.ptk_cipher = 0;
-	common->secinfo.gtk_cipher = 0;
+	if (vif->type == NL80211_IFTYPE_AP) {
+		u8 sta_idx, cnt;
+
+		/* Send peer notify to device */
+		rsi_dbg(INFO_ZONE, "Indicate bss status to device\n");
+		for (sta_idx = 0; sta_idx < common->max_stations; sta_idx++) {
+			rsta = &common->stations[sta_idx];
+
+			if (!rsta->sta)
+				continue;
+			if (!memcmp(rsta->sta->addr, sta->addr, ETH_ALEN)) {
+				rsi_inform_bss_status(common, AP_OPMODE, 0,
+						      sta->addr, sta->wme,
+						      sta->aid, sta, sta_idx);
+				rsta->sta = NULL;
+				rsta->sta_id = -1;
+				for (cnt = 0; cnt < IEEE80211_NUM_TIDS; cnt++)
+					rsta->start_tx_aggr[cnt] = false;
+				if (common->num_stations > 0)
+					common->num_stations--;
+				break;
+			}
+		}
+		if (sta_idx >= common->max_stations)
+			rsi_dbg(ERR_ZONE, "%s: No station found\n", __func__);
+	}
 
-	rsi_send_rx_filter_frame(common, 0);
-	
+	if (vif->type == NL80211_IFTYPE_STATION) {
+		/* Resetting all the fields to default values */
+		memcpy((u8 *)bss->bssid, (u8 *)sta->addr, ETH_ALEN);
+		bss->qos = sta->wme;
+		common->bitrate_mask[NL80211_BAND_2GHZ] = 0;
+		common->bitrate_mask[NL80211_BAND_5GHZ] = 0;
+		common->min_rate = 0xffff;
+		common->vif_info[0].is_ht = false;
+		common->vif_info[0].sgi = false;
+		common->vif_info[0].seq_start = 0;
+		common->secinfo.ptk_cipher = 0;
+		common->secinfo.gtk_cipher = 0;
+		if (!common->iface_down)
+			rsi_send_rx_filter_frame(common, 0);
+	}
 	mutex_unlock(&common->mutex);
 	
 	return 0;
@@ -1133,7 +1426,7 @@ fail_set_antenna:
  * @tx_ant: Bitmap for tx antenna
  * @rx_ant: Bitmap for rx antenna
  * 
- * Return: 0 on success, -1 on failure.
+ * Return: 0 on success, negative error codes on failure.
  */
 static int rsi_mac80211_get_antenna(struct ieee80211_hw *hw,
 				    u32 *tx_ant, u32 *rx_ant)
@@ -1151,6 +1444,21 @@ static int rsi_mac80211_get_antenna(struct ieee80211_hw *hw,
 	return 0;	
 }
 
+static int rsi_map_region_code(enum nl80211_dfs_regions region_code)
+{
+	switch (region_code) {
+	case NL80211_DFS_FCC:
+		return RSI_REGION_FCC;
+	case NL80211_DFS_ETSI:
+		return RSI_REGION_ETSI;
+	case NL80211_DFS_JP:
+		return RSI_REGION_TELEC;
+	case NL80211_DFS_UNSET:
+		return RSI_REGION_WORLD;
+	}
+	return RSI_REGION_WORLD;
+}
+
 static void rsi_reg_notify(struct wiphy *wiphy,
 			   struct regulatory_request *request)
 {
@@ -1158,26 +1466,49 @@ static void rsi_reg_notify(struct wiphy *wiphy,
 	struct ieee80211_channel *ch;
 	struct ieee80211_hw *hw = wiphy_to_ieee80211_hw(wiphy);
 	struct rsi_hw * adapter = hw->priv; 
+	struct rsi_common *common = adapter->priv;
 	int i;
-
-	sband = wiphy->bands[NL80211_BAND_5GHZ];
 	
-	for (i = 0; i < sband->n_channels; i++) {
-		ch = &sband->channels[i];
-		if (ch->flags & IEEE80211_CHAN_DISABLED)
-			continue;
+	mutex_lock(&common->mutex);
+
+	rsi_dbg(INFO_ZONE, "country = %s dfs_region = %d\n",
+		request->alpha2, request->dfs_region);
+
+	if (common->num_supp_bands > 1) {
+		sband = wiphy->bands[NL80211_BAND_5GHZ];
 
-		if (ch->flags & IEEE80211_CHAN_RADAR)
-			ch->flags |= IEEE80211_CHAN_NO_IR;
+		for (i = 0; i < sband->n_channels; i++) {
+			ch = &sband->channels[i];
+			if (ch->flags & IEEE80211_CHAN_DISABLED)
+				continue;
+
+			if (ch->flags & IEEE80211_CHAN_RADAR)
+				ch->flags |= IEEE80211_CHAN_NO_IR;
+		}
 	}
+	adapter->dfs_region = rsi_map_region_code(request->dfs_region);
+	rsi_dbg(INFO_ZONE, "RSI region code = %d\n", adapter->dfs_region);
 	
-	rsi_dbg(INFO_ZONE,
-		"country = %s dfs_region = %d\n",
-		request->alpha2, request->dfs_region);
-	adapter->dfs_region = request->dfs_region;
+	adapter->country[0] = request->alpha2[0];
+	adapter->country[1] = request->alpha2[1];
+
+	mutex_unlock(&common->mutex);
+}
+
+static void rsi_mac80211_rfkill_poll(struct ieee80211_hw *hw)
+{
+	struct rsi_hw *adapter = hw->priv;
+	struct rsi_common *common = adapter->priv;
+
+	mutex_lock(&common->mutex);
+	if (common->fsm_state != FSM_MAC_INIT_DONE)
+		wiphy_rfkill_set_hw_state(hw->wiphy, true);
+	else
+		wiphy_rfkill_set_hw_state(hw->wiphy, false);
+	mutex_unlock(&common->mutex);
 }
 
-static struct ieee80211_ops mac80211_ops = {
+static const struct ieee80211_ops mac80211_ops = {
 	.tx = rsi_mac80211_tx,
 	.start = rsi_mac80211_start,
 	.stop = rsi_mac80211_stop,
@@ -1195,13 +1526,14 @@ static struct ieee80211_ops mac80211_ops = {
 	.sta_remove = rsi_mac80211_sta_remove,
 	.set_antenna = rsi_mac80211_set_antenna,
 	.get_antenna = rsi_mac80211_get_antenna,
+	.rfkill_poll = rsi_mac80211_rfkill_poll,
 };
 
 /**
  * rsi_mac80211_attach() - This function is used to initialize Mac80211 stack.
  * @common: Pointer to the driver private structure.
  *
- * Return: 0 on success, -1 on failure.
+ * Return: 0 on success, negative error codes on failure.
  */
 int rsi_mac80211_attach(struct rsi_common *common)
 {
@@ -1229,12 +1561,16 @@ int rsi_mac80211_attach(struct rsi_common *common)
 	ieee80211_hw_set(hw, SIGNAL_DBM);
 	ieee80211_hw_set(hw, HAS_RATE_CONTROL);
 	ieee80211_hw_set(hw, AMPDU_AGGREGATION);
+	ieee80211_hw_set(hw, SUPPORTS_PS);
+	ieee80211_hw_set(hw, SUPPORTS_DYNAMIC_PS);
 
 	hw->queues = MAX_HW_QUEUES;
 	hw->extra_tx_headroom = RSI_NEEDED_HEADROOM;
 
 	hw->max_rates = 1;
 	hw->max_rate_tries = MAX_RETRIES;
+	hw->uapsd_queues = RSI_IEEE80211_UAPSD_QUEUES;
+	hw->uapsd_max_sp_len = IEEE80211_WMM_IE_STA_QOSINFO_SP_ALL;
 
 	hw->max_tx_aggregation_subframes = 6;
 	rsi_register_rates_channels(adapter, NL80211_BAND_2GHZ);
@@ -1244,7 +1580,8 @@ int rsi_mac80211_attach(struct rsi_common *common)
 	SET_IEEE80211_PERM_ADDR(hw, common->mac_addr);
 	ether_addr_copy(hw->wiphy->addr_mask, addr_mask);
 
-	wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION);
+	wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION) |
+				 BIT(NL80211_IFTYPE_AP);
 	wiphy->signal_type = CFG80211_SIGNAL_TYPE_MBM;
 	wiphy->retry_short = RETRY_SHORT;
 	wiphy->retry_long  = RETRY_LONG;
@@ -1259,6 +1596,14 @@ int rsi_mac80211_attach(struct rsi_common *common)
 	wiphy->bands[NL80211_BAND_5GHZ] =
 		&adapter->sbands[NL80211_BAND_5GHZ];
 
+	/* AP Parameters */
+	wiphy->max_ap_assoc_sta = rsi_max_ap_stas[common->oper_mode - 1];
+	common->max_stations = wiphy->max_ap_assoc_sta;
+	rsi_dbg(ERR_ZONE, "Max Stations Allowed = %d\n", common->max_stations);
+	hw->sta_data_size = sizeof(struct rsi_sta);
+	wiphy->flags = WIPHY_FLAG_REPORTS_OBSS;
+	wiphy->flags |= WIPHY_FLAG_AP_UAPSD;
+	wiphy->features |= NL80211_FEATURE_INACTIVITY_TIMER;
 	wiphy->reg_notifier = rsi_reg_notify;
 
 	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_CQM_RSSI_LIST);
diff --git a/drivers/net/wireless/rsi/rsi_91x_main.c b/drivers/net/wireless/rsi/rsi_91x_main.c
index f1cde0ca81f9..3e1e80888d98 100644
--- a/drivers/net/wireless/rsi/rsi_91x_main.c
+++ b/drivers/net/wireless/rsi/rsi_91x_main.c
@@ -220,7 +220,8 @@ struct rsi_hw *rsi_91x_init(void)
 
 	rsi_init_event(&common->tx_thread.event);
 	mutex_init(&common->mutex);
-	mutex_init(&common->tx_rxlock);
+	mutex_init(&common->tx_lock);
+	mutex_init(&common->rx_lock);
 
 	if (rsi_create_kthread(common,
 			       &common->tx_thread,
@@ -230,6 +231,8 @@ struct rsi_hw *rsi_91x_init(void)
 		goto err;
 	}
 
+	rsi_default_ps_params(adapter);
+	spin_lock_init(&adapter->ps_lock);
 	common->init_done = true;
 	return adapter;
 
diff --git a/drivers/net/wireless/rsi/rsi_91x_mgmt.c b/drivers/net/wireless/rsi/rsi_91x_mgmt.c
index d4d365b5d2d6..f7b550f900c4 100644
--- a/drivers/net/wireless/rsi/rsi_91x_mgmt.c
+++ b/drivers/net/wireless/rsi/rsi_91x_mgmt.c
@@ -17,6 +17,8 @@
 #include <linux/etherdevice.h>
 #include "rsi_mgmt.h"
 #include "rsi_common.h"
+#include "rsi_ps.h"
+#include "rsi_hal.h"
 
 static struct bootup_params boot_params_20 = {
 	.magic_number = cpu_to_le16(0x5aa5),
@@ -230,6 +232,8 @@ static void rsi_set_default_parameters(struct rsi_common *common)
 	common->rf_power_val = 0; /* Default 1.9V */
 	common->wlan_rf_power_mode = 0;
 	common->obm_ant_sel_val = 2;
+	common->beacon_interval = RSI_BEACON_INTERVAL;
+	common->dtim_cnt = RSI_DTIM_COUNT;
 }
 
 /**
@@ -266,11 +270,15 @@ static int rsi_send_internal_mgmt_frame(struct rsi_common *common,
 					struct sk_buff *skb)
 {
 	struct skb_info *tx_params;
+	struct rsi_cmd_desc *desc;
 
 	if (skb == NULL) {
 		rsi_dbg(ERR_ZONE, "%s: Unable to allocate skb\n", __func__);
 		return -ENOMEM;
 	}
+	desc = (struct rsi_cmd_desc *)skb->data;
+	desc->desc_dword0.len_qno |= cpu_to_le16(DESC_IMMEDIATE_WAKEUP);
+	skb->priority = MGMT_SOFT_Q;
 	tx_params = (struct skb_info *)&IEEE80211_SKB_CB(skb)->driver_data;
 	tx_params->flags |= INTERNAL_MGMT_PKT;
 	skb_queue_tail(&common->tx_queue[MGMT_SOFT_Q], skb);
@@ -298,10 +306,11 @@ static int rsi_load_radio_caps(struct rsi_common *common)
 		      0xf0, 0xf0, 0xf0, 0xf0,
 		      0xf0, 0xf0, 0xf0, 0xf0};
 	struct sk_buff *skb;
+	u16 frame_len = sizeof(struct rsi_radio_caps);
 
 	rsi_dbg(INFO_ZONE, "%s: Sending rate symbol req frame\n", __func__);
 
-	skb = dev_alloc_skb(sizeof(struct rsi_radio_caps));
+	skb = dev_alloc_skb(frame_len);
 
 	if (!skb) {
 		rsi_dbg(ERR_ZONE, "%s: Failed in allocation of skb\n",
@@ -309,37 +318,40 @@ static int rsi_load_radio_caps(struct rsi_common *common)
 		return -ENOMEM;
 	}
 
-	memset(skb->data, 0, sizeof(struct rsi_radio_caps));
+	memset(skb->data, 0, frame_len);
 	radio_caps = (struct rsi_radio_caps *)skb->data;
 
-	radio_caps->desc_word[1] = cpu_to_le16(RADIO_CAPABILITIES);
-	radio_caps->desc_word[4] = cpu_to_le16(RSI_RF_TYPE << 8);
+	radio_caps->desc_dword0.frame_type = RADIO_CAPABILITIES;
+	radio_caps->channel_num = common->channel;
+	radio_caps->rf_model = RSI_RF_TYPE;
 
 	if (common->channel_width == BW_40MHZ) {
-		radio_caps->desc_word[7] |= cpu_to_le16(RSI_LMAC_CLOCK_80MHZ);
-		radio_caps->desc_word[7] |= cpu_to_le16(RSI_ENABLE_40MHZ);
+		radio_caps->radio_cfg_info = RSI_LMAC_CLOCK_80MHZ;
+		radio_caps->radio_cfg_info |= RSI_ENABLE_40MHZ;
 
 		if (common->fsm_state == FSM_MAC_INIT_DONE) {
 			struct ieee80211_hw *hw = adapter->hw;
 			struct ieee80211_conf *conf = &hw->conf;
+
 			if (conf_is_ht40_plus(conf)) {
-				radio_caps->desc_word[5] =
-					cpu_to_le16(LOWER_20_ENABLE);
-				radio_caps->desc_word[5] |=
-					cpu_to_le16(LOWER_20_ENABLE >> 12);
+				radio_caps->radio_cfg_info =
+					RSI_CMDDESC_LOWER_20_ENABLE;
+				radio_caps->radio_info =
+					RSI_CMDDESC_LOWER_20_ENABLE;
 			} else if (conf_is_ht40_minus(conf)) {
-				radio_caps->desc_word[5] =
-					cpu_to_le16(UPPER_20_ENABLE);
-				radio_caps->desc_word[5] |=
-					cpu_to_le16(UPPER_20_ENABLE >> 12);
+				radio_caps->radio_cfg_info =
+					RSI_CMDDESC_UPPER_20_ENABLE;
+				radio_caps->radio_info =
+					RSI_CMDDESC_UPPER_20_ENABLE;
 			} else {
-				radio_caps->desc_word[5] =
-					cpu_to_le16(BW_40MHZ << 12);
-				radio_caps->desc_word[5] |=
-					cpu_to_le16(FULL40M_ENABLE);
+				radio_caps->radio_cfg_info =
+					RSI_CMDDESC_40MHZ;
+				radio_caps->radio_info =
+					RSI_CMDDESC_FULL_40_ENABLE;
 			}
 		}
 	}
+	radio_caps->radio_info |= radio_id;
 
 	radio_caps->sifs_tx_11n = cpu_to_le16(SIFS_TX_11N_VALUE);
 	radio_caps->sifs_tx_11b = cpu_to_le16(SIFS_TX_11B_VALUE);
@@ -348,8 +360,6 @@ static int rsi_load_radio_caps(struct rsi_common *common)
 	radio_caps->cck_ack_tout = cpu_to_le16(CCK_ACK_TOUT_VALUE);
 	radio_caps->preamble_type = cpu_to_le16(LONG_PREAMBLE);
 
-	radio_caps->desc_word[7] |= cpu_to_le16(radio_id << 8);
-
 	for (ii = 0; ii < MAX_HW_QUEUES; ii++) {
 		radio_caps->qos_params[ii].cont_win_min_q = cpu_to_le16(3);
 		radio_caps->qos_params[ii].cont_win_max_q = cpu_to_le16(0x3f);
@@ -357,7 +367,7 @@ static int rsi_load_radio_caps(struct rsi_common *common)
 		radio_caps->qos_params[ii].txop_q = 0;
 	}
 
-	for (ii = 0; ii < MAX_HW_QUEUES - 4; ii++) {
+	for (ii = 0; ii < NUM_EDCA_QUEUES; ii++) {
 		radio_caps->qos_params[ii].cont_win_min_q =
 			cpu_to_le16(common->edca_params[ii].cw_min);
 		radio_caps->qos_params[ii].cont_win_max_q =
@@ -368,17 +378,19 @@ static int rsi_load_radio_caps(struct rsi_common *common)
 			cpu_to_le16(common->edca_params[ii].txop);
 	}
 
+	radio_caps->qos_params[BROADCAST_HW_Q].txop_q = cpu_to_le16(0xffff);
+	radio_caps->qos_params[MGMT_HW_Q].txop_q = 0;
+	radio_caps->qos_params[BEACON_HW_Q].txop_q = cpu_to_le16(0xffff);
+
 	memcpy(&common->rate_pwr[0], &gc[0], 40);
 	for (ii = 0; ii < 20; ii++)
 		radio_caps->gcpd_per_rate[inx++] =
 			cpu_to_le16(common->rate_pwr[ii]  & 0x00FF);
 
-	radio_caps->desc_word[0] = cpu_to_le16((sizeof(struct rsi_radio_caps) -
-						FRAME_DESC_SZ) |
-					       (RSI_WIFI_MGMT_Q << 12));
-
+	rsi_set_len_qno(&radio_caps->desc_dword0.len_qno,
+			(frame_len - FRAME_DESC_SZ), RSI_WIFI_MGMT_Q);
 
-	skb_put(skb, (sizeof(struct rsi_radio_caps)));
+	skb_put(skb, frame_len);
 
 	return rsi_send_internal_mgmt_frame(common, skb);
 }
@@ -394,8 +406,7 @@ static int rsi_load_radio_caps(struct rsi_common *common)
  */
 static int rsi_mgmt_pkt_to_core(struct rsi_common *common,
 				u8 *msg,
-				s32 msg_len,
-				u8 type)
+				s32 msg_len)
 {
 	struct rsi_hw *adapter = common->priv;
 	struct ieee80211_tx_info *info;
@@ -403,37 +414,30 @@ static int rsi_mgmt_pkt_to_core(struct rsi_common *common,
 	u8 pad_bytes = msg[4];
 	struct sk_buff *skb;
 
-	if (type == RX_DOT11_MGMT) {
-		if (!adapter->sc_nvifs)
-			return -ENOLINK;
+	if (!adapter->sc_nvifs)
+		return -ENOLINK;
 
-		msg_len -= pad_bytes;
-		if (msg_len <= 0) {
-			rsi_dbg(MGMT_RX_ZONE,
-				"%s: Invalid rx msg of len = %d\n",
-				__func__, msg_len);
-			return -EINVAL;
-		}
+	msg_len -= pad_bytes;
+	if (msg_len <= 0) {
+		rsi_dbg(MGMT_RX_ZONE,
+			"%s: Invalid rx msg of len = %d\n",
+			__func__, msg_len);
+		return -EINVAL;
+	}
 
-		skb = dev_alloc_skb(msg_len);
-		if (!skb) {
-			rsi_dbg(ERR_ZONE, "%s: Failed to allocate skb\n",
-				__func__);
-			return -ENOMEM;
-		}
+	skb = dev_alloc_skb(msg_len);
+	if (!skb)
+		return -ENOMEM;
 
-		skb_put_data(skb,
-			     (u8 *)(msg + FRAME_DESC_SZ + pad_bytes),
-			     msg_len);
+	skb_put_data(skb,
+		     (u8 *)(msg + FRAME_DESC_SZ + pad_bytes),
+		     msg_len);
 
-		info = IEEE80211_SKB_CB(skb);
-		rx_params = (struct skb_info *)info->driver_data;
-		rx_params->rssi = rsi_get_rssi(msg);
-		rx_params->channel = rsi_get_channel(msg);
-		rsi_indicate_pkt_to_os(common, skb);
-	} else {
-		rsi_dbg(MGMT_TX_ZONE, "%s: Internal Packet\n", __func__);
-	}
+	info = IEEE80211_SKB_CB(skb);
+	rx_params = (struct skb_info *)info->driver_data;
+	rx_params->rssi = rsi_get_rssi(msg);
+	rx_params->channel = rsi_get_channel(msg);
+	rsi_indicate_pkt_to_os(common, skb);
 
 	return 0;
 }
@@ -451,20 +455,23 @@ static int rsi_mgmt_pkt_to_core(struct rsi_common *common,
  * Return: status: 0 on success, corresponding negative error code on failure.
  */
 static int rsi_hal_send_sta_notify_frame(struct rsi_common *common,
-					 u8 opmode,
+					 enum opmode opmode,
 					 u8 notify_event,
 					 const unsigned char *bssid,
 					 u8 qos_enable,
-					 u16 aid)
+					 u16 aid,
+					 u16 sta_id)
 {
+	struct ieee80211_vif *vif = common->priv->vifs[0];
 	struct sk_buff *skb = NULL;
 	struct rsi_peer_notify *peer_notify;
 	u16 vap_id = 0;
 	int status;
+	u16 frame_len = sizeof(struct rsi_peer_notify);
 
 	rsi_dbg(MGMT_TX_ZONE, "%s: Sending sta notify frame\n", __func__);
 
-	skb = dev_alloc_skb(sizeof(struct rsi_peer_notify));
+	skb = dev_alloc_skb(frame_len);
 
 	if (!skb) {
 		rsi_dbg(ERR_ZONE, "%s: Failed in allocation of skb\n",
@@ -472,10 +479,13 @@ static int rsi_hal_send_sta_notify_frame(struct rsi_common *common,
 		return -ENOMEM;
 	}
 
-	memset(skb->data, 0, sizeof(struct rsi_peer_notify));
+	memset(skb->data, 0, frame_len);
 	peer_notify = (struct rsi_peer_notify *)skb->data;
 
-	peer_notify->command = cpu_to_le16(opmode << 1);
+	if (opmode == STA_OPMODE)
+		peer_notify->command = cpu_to_le16(PEER_TYPE_AP << 1);
+	else if (opmode == AP_OPMODE)
+		peer_notify->command = cpu_to_le16(PEER_TYPE_STA << 1);
 
 	switch (notify_event) {
 	case STA_CONNECTED:
@@ -490,20 +500,22 @@ static int rsi_hal_send_sta_notify_frame(struct rsi_common *common,
 
 	peer_notify->command |= cpu_to_le16((aid & 0xfff) << 4);
 	ether_addr_copy(peer_notify->mac_addr, bssid);
-
+	peer_notify->mpdu_density = cpu_to_le16(RSI_MPDU_DENSITY);
 	peer_notify->sta_flags = cpu_to_le32((qos_enable) ? 1 : 0);
 
-	peer_notify->desc_word[0] =
-		cpu_to_le16((sizeof(struct rsi_peer_notify) - FRAME_DESC_SZ) |
-			    (RSI_WIFI_MGMT_Q << 12));
-	peer_notify->desc_word[1] = cpu_to_le16(PEER_NOTIFY);
-	peer_notify->desc_word[7] |= cpu_to_le16(vap_id << 8);
+	rsi_set_len_qno(&peer_notify->desc.desc_dword0.len_qno,
+			(frame_len - FRAME_DESC_SZ),
+			RSI_WIFI_MGMT_Q);
+	peer_notify->desc.desc_dword0.frame_type = PEER_NOTIFY;
+	peer_notify->desc.desc_dword3.qid_tid = sta_id;
+	peer_notify->desc.desc_dword3.sta_id = vap_id;
 
-	skb_put(skb, sizeof(struct rsi_peer_notify));
+	skb_put(skb, frame_len);
 
 	status = rsi_send_internal_mgmt_frame(common, skb);
 
-	if (!status && qos_enable) {
+	if ((vif->type == NL80211_IFTYPE_STATION) &&
+	    (!status && qos_enable)) {
 		rsi_set_contention_vals(common);
 		status = rsi_load_radio_caps(common);
 	}
@@ -525,13 +537,14 @@ int rsi_send_aggregation_params_frame(struct rsi_common *common,
 				      u16 tid,
 				      u16 ssn,
 				      u8 buf_size,
-				      u8 event)
+				      u8 event,
+				      u8 sta_id)
 {
 	struct sk_buff *skb = NULL;
-	struct rsi_mac_frame *mgmt_frame;
-	u8 peer_id = 0;
+	struct rsi_aggr_params *aggr_params;
+	u16 frame_len = sizeof(struct rsi_aggr_params);
 
-	skb = dev_alloc_skb(FRAME_DESC_SZ);
+	skb = dev_alloc_skb(frame_len);
 
 	if (!skb) {
 		rsi_dbg(ERR_ZONE, "%s: Failed in allocation of skb\n",
@@ -539,37 +552,29 @@ int rsi_send_aggregation_params_frame(struct rsi_common *common,
 		return -ENOMEM;
 	}
 
-	memset(skb->data, 0, FRAME_DESC_SZ);
-	mgmt_frame = (struct rsi_mac_frame *)skb->data;
+	memset(skb->data, 0, frame_len);
+	aggr_params = (struct rsi_aggr_params *)skb->data;
 
 	rsi_dbg(MGMT_TX_ZONE, "%s: Sending AMPDU indication frame\n", __func__);
 
-	mgmt_frame->desc_word[0] = cpu_to_le16(RSI_WIFI_MGMT_Q << 12);
-	mgmt_frame->desc_word[1] = cpu_to_le16(AMPDU_IND);
+	rsi_set_len_qno(&aggr_params->desc_dword0.len_qno, 0, RSI_WIFI_MGMT_Q);
+	aggr_params->desc_dword0.frame_type = AMPDU_IND;
 
+	aggr_params->aggr_params = tid & RSI_AGGR_PARAMS_TID_MASK;
+	aggr_params->peer_id = sta_id;
 	if (event == STA_TX_ADDBA_DONE) {
-		mgmt_frame->desc_word[4] = cpu_to_le16(ssn);
-		mgmt_frame->desc_word[5] = cpu_to_le16(buf_size);
-		mgmt_frame->desc_word[7] =
-		cpu_to_le16((tid | (START_AMPDU_AGGR << 4) | (peer_id << 8)));
+		aggr_params->seq_start = cpu_to_le16(ssn);
+		aggr_params->baw_size = cpu_to_le16(buf_size);
+		aggr_params->aggr_params |= RSI_AGGR_PARAMS_START;
 	} else if (event == STA_RX_ADDBA_DONE) {
-		mgmt_frame->desc_word[4] = cpu_to_le16(ssn);
-		mgmt_frame->desc_word[7] = cpu_to_le16(tid |
-						       (START_AMPDU_AGGR << 4) |
-						       (RX_BA_INDICATION << 5) |
-						       (peer_id << 8));
-	} else if (event == STA_TX_DELBA) {
-		mgmt_frame->desc_word[7] = cpu_to_le16(tid |
-						       (STOP_AMPDU_AGGR << 4) |
-						       (peer_id << 8));
+		aggr_params->seq_start = cpu_to_le16(ssn);
+		aggr_params->aggr_params |= (RSI_AGGR_PARAMS_START |
+					     RSI_AGGR_PARAMS_RX_AGGR);
 	} else if (event == STA_RX_DELBA) {
-		mgmt_frame->desc_word[7] = cpu_to_le16(tid |
-						       (STOP_AMPDU_AGGR << 4) |
-						       (RX_BA_INDICATION << 5) |
-						       (peer_id << 8));
+		aggr_params->aggr_params |= RSI_AGGR_PARAMS_RX_AGGR;
 	}
 
-	skb_put(skb, FRAME_DESC_SZ);
+	skb_put(skb, frame_len);
 
 	return rsi_send_internal_mgmt_frame(common, skb);
 }
@@ -584,34 +589,36 @@ int rsi_send_aggregation_params_frame(struct rsi_common *common,
 static int rsi_program_bb_rf(struct rsi_common *common)
 {
 	struct sk_buff *skb;
-	struct rsi_mac_frame *mgmt_frame;
+	struct rsi_bb_rf_prog *bb_rf_prog;
+	u16 frame_len = sizeof(struct rsi_bb_rf_prog);
 
 	rsi_dbg(MGMT_TX_ZONE, "%s: Sending program BB/RF frame\n", __func__);
 
-	skb = dev_alloc_skb(FRAME_DESC_SZ);
+	skb = dev_alloc_skb(frame_len);
 	if (!skb) {
 		rsi_dbg(ERR_ZONE, "%s: Failed in allocation of skb\n",
 			__func__);
 		return -ENOMEM;
 	}
 
-	memset(skb->data, 0, FRAME_DESC_SZ);
-	mgmt_frame = (struct rsi_mac_frame *)skb->data;
+	memset(skb->data, 0, frame_len);
+	bb_rf_prog = (struct rsi_bb_rf_prog *)skb->data;
 
-	mgmt_frame->desc_word[0] = cpu_to_le16(RSI_WIFI_MGMT_Q << 12);
-	mgmt_frame->desc_word[1] = cpu_to_le16(BBP_PROG_IN_TA);
-	mgmt_frame->desc_word[4] = cpu_to_le16(common->endpoint);
+	rsi_set_len_qno(&bb_rf_prog->desc_dword0.len_qno, 0, RSI_WIFI_MGMT_Q);
+	bb_rf_prog->desc_dword0.frame_type = BBP_PROG_IN_TA;
+	bb_rf_prog->endpoint = common->endpoint;
+	bb_rf_prog->rf_power_mode = common->wlan_rf_power_mode;
 
 	if (common->rf_reset) {
-		mgmt_frame->desc_word[7] =  cpu_to_le16(RF_RESET_ENABLE);
+		bb_rf_prog->flags =  cpu_to_le16(RF_RESET_ENABLE);
 		rsi_dbg(MGMT_TX_ZONE, "%s: ===> RF RESET REQUEST SENT <===\n",
 			__func__);
 		common->rf_reset = 0;
 	}
 	common->bb_rf_prog_count = 1;
-	mgmt_frame->desc_word[7] |= cpu_to_le16(PUT_BBP_RESET |
-				     BBP_REG_WRITE | (RSI_RF_TYPE << 4));
-	skb_put(skb, FRAME_DESC_SZ);
+	bb_rf_prog->flags |= cpu_to_le16(PUT_BBP_RESET | BBP_REG_WRITE |
+					 (RSI_RF_TYPE << 4));
+	skb_put(skb, frame_len);
 
 	return rsi_send_internal_mgmt_frame(common, skb);
 }
@@ -625,6 +632,8 @@ static int rsi_program_bb_rf(struct rsi_common *common)
  */
 int rsi_set_vap_capabilities(struct rsi_common *common,
 			     enum opmode mode,
+			     u8 *mac_addr,
+			     u8 vap_id,
 			     u8 vap_status)
 {
 	struct sk_buff *skb = NULL;
@@ -632,59 +641,60 @@ int rsi_set_vap_capabilities(struct rsi_common *common,
 	struct rsi_hw *adapter = common->priv;
 	struct ieee80211_hw *hw = adapter->hw;
 	struct ieee80211_conf *conf = &hw->conf;
-	u16 vap_id = 0;
+	u16 frame_len = sizeof(struct rsi_vap_caps);
 
 	rsi_dbg(MGMT_TX_ZONE, "%s: Sending VAP capabilities frame\n", __func__);
 
-	skb = dev_alloc_skb(sizeof(struct rsi_vap_caps));
+	skb = dev_alloc_skb(frame_len);
 	if (!skb) {
 		rsi_dbg(ERR_ZONE, "%s: Failed in allocation of skb\n",
 			__func__);
 		return -ENOMEM;
 	}
 
-	memset(skb->data, 0, sizeof(struct rsi_vap_caps));
+	memset(skb->data, 0, frame_len);
 	vap_caps = (struct rsi_vap_caps *)skb->data;
 
-	vap_caps->desc_word[0] = cpu_to_le16((sizeof(struct rsi_vap_caps) -
-					     FRAME_DESC_SZ) |
-					     (RSI_WIFI_MGMT_Q << 12));
-	vap_caps->desc_word[1] = cpu_to_le16(VAP_CAPABILITIES);
-	vap_caps->desc_word[2] = cpu_to_le16(vap_status << 8);
-	vap_caps->desc_word[4] = cpu_to_le16(mode |
-					     (common->channel_width << 8));
-	vap_caps->desc_word[7] = cpu_to_le16((vap_id << 8) |
-					     (common->mac_id << 4) |
-					     common->radio_id);
-
-	memcpy(vap_caps->mac_addr, common->mac_addr, IEEE80211_ADDR_LEN);
+	rsi_set_len_qno(&vap_caps->desc_dword0.len_qno,
+			(frame_len - FRAME_DESC_SZ), RSI_WIFI_MGMT_Q);
+	vap_caps->desc_dword0.frame_type = VAP_CAPABILITIES;
+	vap_caps->status = vap_status;
+	vap_caps->vif_type = mode;
+	vap_caps->channel_bw = common->channel_width;
+	vap_caps->vap_id = vap_id;
+	vap_caps->radioid_macid = ((common->mac_id & 0xf) << 4) |
+				   (common->radio_id & 0xf);
+
+	memcpy(vap_caps->mac_addr, mac_addr, IEEE80211_ADDR_LEN);
 	vap_caps->keep_alive_period = cpu_to_le16(90);
 	vap_caps->frag_threshold = cpu_to_le16(IEEE80211_MAX_FRAG_THRESHOLD);
 
 	vap_caps->rts_threshold = cpu_to_le16(common->rts_threshold);
-	vap_caps->default_mgmt_rate = cpu_to_le32(RSI_RATE_6);
 
 	if (common->band == NL80211_BAND_5GHZ) {
-		vap_caps->default_ctrl_rate = cpu_to_le32(RSI_RATE_6);
-		if (conf_is_ht40(&common->priv->hw->conf)) {
-			vap_caps->default_ctrl_rate |=
-				cpu_to_le32(FULL40M_ENABLE << 16);
-		}
+		vap_caps->default_ctrl_rate = cpu_to_le16(RSI_RATE_6);
+		vap_caps->default_mgmt_rate = cpu_to_le32(RSI_RATE_6);
 	} else {
-		vap_caps->default_ctrl_rate = cpu_to_le32(RSI_RATE_1);
+		vap_caps->default_ctrl_rate = cpu_to_le16(RSI_RATE_1);
+		vap_caps->default_mgmt_rate = cpu_to_le32(RSI_RATE_1);
+	}
+	if (conf_is_ht40(conf)) {
 		if (conf_is_ht40_minus(conf))
-			vap_caps->default_ctrl_rate |=
-				cpu_to_le32(UPPER_20_ENABLE << 16);
+			vap_caps->ctrl_rate_flags =
+				cpu_to_le16(UPPER_20_ENABLE);
 		else if (conf_is_ht40_plus(conf))
-			vap_caps->default_ctrl_rate |=
-				cpu_to_le32(LOWER_20_ENABLE << 16);
+			vap_caps->ctrl_rate_flags =
+				cpu_to_le16(LOWER_20_ENABLE);
+		else
+			vap_caps->ctrl_rate_flags =
+				cpu_to_le16(FULL40M_ENABLE);
 	}
 
 	vap_caps->default_data_rate = 0;
-	vap_caps->beacon_interval = cpu_to_le16(200);
-	vap_caps->dtim_period = cpu_to_le16(4);
+	vap_caps->beacon_interval = cpu_to_le16(common->beacon_interval);
+	vap_caps->dtim_period = cpu_to_le16(common->dtim_cnt);
 
-	skb_put(skb, sizeof(*vap_caps));
+	skb_put(skb, frame_len);
 
 	return rsi_send_internal_mgmt_frame(common, skb);
 }
@@ -705,58 +715,66 @@ int rsi_hal_load_key(struct rsi_common *common,
 		     u16 key_len,
 		     u8 key_type,
 		     u8 key_id,
-		     u32 cipher)
+		     u32 cipher,
+		     s16 sta_id)
 {
+	struct ieee80211_vif *vif = common->priv->vifs[0];
 	struct sk_buff *skb = NULL;
 	struct rsi_set_key *set_key;
 	u16 key_descriptor = 0;
+	u16 frame_len = sizeof(struct rsi_set_key);
 
 	rsi_dbg(MGMT_TX_ZONE, "%s: Sending load key frame\n", __func__);
 
-	skb = dev_alloc_skb(sizeof(struct rsi_set_key));
+	skb = dev_alloc_skb(frame_len);
 	if (!skb) {
 		rsi_dbg(ERR_ZONE, "%s: Failed in allocation of skb\n",
 			__func__);
 		return -ENOMEM;
 	}
 
-	memset(skb->data, 0, sizeof(struct rsi_set_key));
+	memset(skb->data, 0, frame_len);
 	set_key = (struct rsi_set_key *)skb->data;
 
+	if (key_type == RSI_GROUP_KEY) {
+		key_descriptor = RSI_KEY_TYPE_BROADCAST;
+		if (vif->type == NL80211_IFTYPE_AP)
+			key_descriptor |= RSI_KEY_MODE_AP;
+	}
 	if ((cipher == WLAN_CIPHER_SUITE_WEP40) ||
 	    (cipher == WLAN_CIPHER_SUITE_WEP104)) {
-		key_len += 1;
-		key_descriptor |= BIT(2);
+		key_id = 0;
+		key_descriptor |= RSI_WEP_KEY;
 		if (key_len >= 13)
-			key_descriptor |= BIT(3);
+			key_descriptor |= RSI_WEP_KEY_104;
 	} else if (cipher != KEY_TYPE_CLEAR) {
-		key_descriptor |= BIT(4);
-		if (key_type == RSI_PAIRWISE_KEY)
-			key_id = 0;
+		key_descriptor |= RSI_CIPHER_WPA;
 		if (cipher == WLAN_CIPHER_SUITE_TKIP)
-			key_descriptor |= BIT(5);
+			key_descriptor |= RSI_CIPHER_TKIP;
 	}
-	key_descriptor |= (key_type | BIT(13) | (key_id << 14));
-
-	set_key->desc_word[0] = cpu_to_le16((sizeof(struct rsi_set_key) -
-					    FRAME_DESC_SZ) |
-					    (RSI_WIFI_MGMT_Q << 12));
-	set_key->desc_word[1] = cpu_to_le16(SET_KEY_REQ);
-	set_key->desc_word[4] = cpu_to_le16(key_descriptor);
-
-	if ((cipher == WLAN_CIPHER_SUITE_WEP40) ||
-	    (cipher == WLAN_CIPHER_SUITE_WEP104)) {
-		memcpy(&set_key->key[key_id][1],
-		       data,
-		       key_len * 2);
+	key_descriptor |= RSI_PROTECT_DATA_FRAMES;
+	key_descriptor |= ((key_id << RSI_KEY_ID_OFFSET) & RSI_KEY_ID_MASK);
+
+	rsi_set_len_qno(&set_key->desc_dword0.len_qno,
+			(frame_len - FRAME_DESC_SZ), RSI_WIFI_MGMT_Q);
+	set_key->desc_dword0.frame_type = SET_KEY_REQ;
+	set_key->key_desc = cpu_to_le16(key_descriptor);
+	set_key->sta_id = sta_id;
+
+	if (data) {
+		if ((cipher == WLAN_CIPHER_SUITE_WEP40) ||
+		    (cipher == WLAN_CIPHER_SUITE_WEP104)) {
+			memcpy(&set_key->key[key_id][1], data, key_len * 2);
+		} else {
+			memcpy(&set_key->key[0][0], data, key_len);
+		}
+		memcpy(set_key->tx_mic_key, &data[16], 8);
+		memcpy(set_key->rx_mic_key, &data[24], 8);
 	} else {
-		memcpy(&set_key->key[0][0], data, key_len);
+		memset(&set_key[FRAME_DESC_SZ], 0, frame_len - FRAME_DESC_SZ);
 	}
 
-	memcpy(set_key->tx_mic_key, &data[16], 8);
-	memcpy(set_key->rx_mic_key, &data[24], 8);
-
-	skb_put(skb, sizeof(struct rsi_set_key));
+	skb_put(skb, frame_len);
 
 	return rsi_send_internal_mgmt_frame(common, skb);
 }
@@ -970,12 +988,13 @@ int rsi_set_channel(struct rsi_common *common,
 		    struct ieee80211_channel *channel)
 {
 	struct sk_buff *skb = NULL;
-	struct rsi_mac_frame *mgmt_frame;
+	struct rsi_chan_config *chan_cfg;
+	u16 frame_len = sizeof(struct rsi_chan_config);
 
 	rsi_dbg(MGMT_TX_ZONE,
 		"%s: Sending scan req frame\n", __func__);
 
-	skb = dev_alloc_skb(FRAME_DESC_SZ);
+	skb = dev_alloc_skb(frame_len);
 	if (!skb) {
 		rsi_dbg(ERR_ZONE, "%s: Failed in allocation of skb\n",
 			__func__);
@@ -986,37 +1005,33 @@ int rsi_set_channel(struct rsi_common *common,
 		dev_kfree_skb(skb);
 		return 0;
 	}
-	memset(skb->data, 0, FRAME_DESC_SZ);
-	mgmt_frame = (struct rsi_mac_frame *)skb->data;
-
-	mgmt_frame->desc_word[0] = cpu_to_le16(RSI_WIFI_MGMT_Q << 12);
-	mgmt_frame->desc_word[1] = cpu_to_le16(SCAN_REQUEST);
-	mgmt_frame->desc_word[4] = cpu_to_le16(channel->hw_value);
-
-	mgmt_frame->desc_word[4] |=
-		cpu_to_le16(((char)(channel->max_antenna_gain)) << 8);
-	mgmt_frame->desc_word[5] =
-		cpu_to_le16((char)(channel->max_antenna_gain));
-
-	mgmt_frame->desc_word[7] = cpu_to_le16(PUT_BBP_RESET |
-					       BBP_REG_WRITE |
-					       (RSI_RF_TYPE << 4));
-
-	if (!(channel->flags & IEEE80211_CHAN_NO_IR) &&
-	       !(channel->flags & IEEE80211_CHAN_RADAR)) {
+	memset(skb->data, 0, frame_len);
+	chan_cfg = (struct rsi_chan_config *)skb->data;
+
+	rsi_set_len_qno(&chan_cfg->desc_dword0.len_qno, 0, RSI_WIFI_MGMT_Q);
+	chan_cfg->desc_dword0.frame_type = SCAN_REQUEST;
+	chan_cfg->channel_number = channel->hw_value;
+	chan_cfg->antenna_gain_offset_2g = channel->max_antenna_gain;
+	chan_cfg->antenna_gain_offset_5g = channel->max_antenna_gain;
+	chan_cfg->region_rftype = (RSI_RF_TYPE & 0xf) << 4;
+
+	if ((channel->flags & IEEE80211_CHAN_NO_IR) ||
+	    (channel->flags & IEEE80211_CHAN_RADAR)) {
+		chan_cfg->antenna_gain_offset_2g |= RSI_CHAN_RADAR;
+	} else {
 		if (common->tx_power < channel->max_power)
-			mgmt_frame->desc_word[6] = cpu_to_le16(common->tx_power);
+			chan_cfg->tx_power = cpu_to_le16(common->tx_power);
 		else
-			mgmt_frame->desc_word[6] = cpu_to_le16(channel->max_power);
+			chan_cfg->tx_power = cpu_to_le16(channel->max_power);
 	}
-	mgmt_frame->desc_word[7] = cpu_to_le16(common->priv->dfs_region);
+	chan_cfg->region_rftype |= (common->priv->dfs_region & 0xf);
 
 	if (common->channel_width == BW_40MHZ)
-		mgmt_frame->desc_word[5] |= cpu_to_le16(0x1 << 8);
+		chan_cfg->channel_width = 0x1;
 
 	common->channel = channel->hw_value;
 
-	skb_put(skb, FRAME_DESC_SZ);
+	skb_put(skb, frame_len);
 
 	return rsi_send_internal_mgmt_frame(common, skb);
 }
@@ -1058,6 +1073,37 @@ int rsi_send_radio_params_update(struct rsi_common *common)
 	return rsi_send_internal_mgmt_frame(common, skb);
 }
 
+/* This function programs the threshold. */
+int rsi_send_vap_dynamic_update(struct rsi_common *common)
+{
+	struct sk_buff *skb;
+	struct rsi_dynamic_s *dynamic_frame;
+
+	rsi_dbg(MGMT_TX_ZONE,
+		"%s: Sending vap update indication frame\n", __func__);
+
+	skb = dev_alloc_skb(sizeof(struct rsi_dynamic_s));
+	if (!skb)
+		return -ENOMEM;
+
+	memset(skb->data, 0, sizeof(struct rsi_dynamic_s));
+	dynamic_frame = (struct rsi_dynamic_s *)skb->data;
+	rsi_set_len_qno(&dynamic_frame->desc_dword0.len_qno,
+			sizeof(dynamic_frame->frame_body), RSI_WIFI_MGMT_Q);
+
+	dynamic_frame->desc_dword0.frame_type = VAP_DYNAMIC_UPDATE;
+	dynamic_frame->desc_dword2.pkt_info =
+					cpu_to_le32(common->rts_threshold);
+	/* Beacon miss threshold */
+	dynamic_frame->frame_body.keep_alive_period =
+					cpu_to_le16(RSI_DEF_KEEPALIVE);
+	dynamic_frame->desc_dword3.sta_id = 0; /* vap id */
+
+	skb_put(skb, sizeof(struct rsi_dynamic_s));
+
+	return rsi_send_internal_mgmt_frame(common, skb);
+}
+
 /**
  * rsi_compare() - This function is used to compare two integers
  * @a: pointer to the first integer
@@ -1112,8 +1158,11 @@ static bool rsi_map_rates(u16 rate, int *offset)
  *
  * Return: 0 on success, corresponding error code on failure.
  */
-static int rsi_send_auto_rate_request(struct rsi_common *common)
+static int rsi_send_auto_rate_request(struct rsi_common *common,
+				      struct ieee80211_sta *sta,
+				      u16 sta_id)
 {
+	struct ieee80211_vif *vif = common->priv->vifs[0];
 	struct sk_buff *skb;
 	struct rsi_auto_rate *auto_rate;
 	int ii = 0, jj = 0, kk = 0;
@@ -1121,11 +1170,15 @@ static int rsi_send_auto_rate_request(struct rsi_common *common)
 	u8 band = hw->conf.chandef.chan->band;
 	u8 num_supported_rates = 0;
 	u8 rate_table_offset, rate_offset = 0;
-	u32 rate_bitmap = common->bitrate_mask[band];
-
+	u32 rate_bitmap;
 	u16 *selected_rates, min_rate;
+	bool is_ht = false, is_sgi = false;
+	u16 frame_len = sizeof(struct rsi_auto_rate);
+
+	rsi_dbg(MGMT_TX_ZONE,
+		"%s: Sending auto rate request frame\n", __func__);
 
-	skb = dev_alloc_skb(sizeof(struct rsi_auto_rate));
+	skb = dev_alloc_skb(frame_len);
 	if (!skb) {
 		rsi_dbg(ERR_ZONE, "%s: Failed in allocation of skb\n",
 			__func__);
@@ -1140,8 +1193,6 @@ static int rsi_send_auto_rate_request(struct rsi_common *common)
 		return -ENOMEM;
 	}
 
-	memset(skb->data, 0, sizeof(struct rsi_auto_rate));
-
 	auto_rate = (struct rsi_auto_rate *)skb->data;
 
 	auto_rate->aarf_rssi = cpu_to_le16(((u16)3 << 6) | (u16)(18 & 0x3f));
@@ -1150,16 +1201,35 @@ static int rsi_send_auto_rate_request(struct rsi_common *common)
 	auto_rate->initial_boundary = cpu_to_le16(3);
 	auto_rate->max_threshold_limt = cpu_to_le16(27);
 
-	auto_rate->desc_word[1] = cpu_to_le16(AUTO_RATE_IND);
+	auto_rate->desc.desc_dword0.frame_type = AUTO_RATE_IND;
 
 	if (common->channel_width == BW_40MHZ)
-		auto_rate->desc_word[7] |= cpu_to_le16(1);
+		auto_rate->desc.desc_dword3.qid_tid = BW_40MHZ;
+	auto_rate->desc.desc_dword3.sta_id = sta_id;
+
+	if (vif->type == NL80211_IFTYPE_STATION) {
+		rate_bitmap = common->bitrate_mask[band];
+		is_ht = common->vif_info[0].is_ht;
+		is_sgi = common->vif_info[0].sgi;
+	} else {
+		rate_bitmap = sta->supp_rates[band];
+		is_ht = sta->ht_cap.ht_supported;
+		if ((sta->ht_cap.cap & IEEE80211_HT_CAP_SGI_20) ||
+		    (sta->ht_cap.cap & IEEE80211_HT_CAP_SGI_40))
+			is_sgi = true;
+	}
 
 	if (band == NL80211_BAND_2GHZ) {
-		min_rate = RSI_RATE_1;
+		if ((rate_bitmap == 0) && (is_ht))
+			min_rate = RSI_RATE_MCS0;
+		else
+			min_rate = RSI_RATE_1;
 		rate_table_offset = 0;
 	} else {
-		min_rate = RSI_RATE_6;
+		if ((rate_bitmap == 0) && (is_ht))
+			min_rate = RSI_RATE_MCS0;
+		else
+			min_rate = RSI_RATE_6;
 		rate_table_offset = 4;
 	}
 
@@ -1173,7 +1243,7 @@ static int rsi_send_auto_rate_request(struct rsi_common *common)
 	}
 	num_supported_rates = jj;
 
-	if (common->vif_info[0].is_ht) {
+	if (is_ht) {
 		for (ii = 0; ii < ARRAY_SIZE(mcs); ii++)
 			selected_rates[jj++] = mcs[ii];
 		num_supported_rates += ARRAY_SIZE(mcs);
@@ -1194,13 +1264,15 @@ static int rsi_send_auto_rate_request(struct rsi_common *common)
 	}
 
 	/* loading HT rates in the bottom half of the auto rate table */
-	if (common->vif_info[0].is_ht) {
+	if (is_ht) {
 		for (ii = rate_offset, kk = ARRAY_SIZE(rsi_mcsrates) - 1;
 		     ii < rate_offset + 2 * ARRAY_SIZE(rsi_mcsrates); ii++) {
-			if (common->vif_info[0].sgi ||
-			    conf_is_ht40(&common->priv->hw->conf))
+			if (is_sgi || conf_is_ht40(&common->priv->hw->conf))
 				auto_rate->supported_rates[ii++] =
 					cpu_to_le16(rsi_mcsrates[kk] | BIT(9));
+			else
+				auto_rate->supported_rates[ii++] =
+					cpu_to_le16(rsi_mcsrates[kk]);
 			auto_rate->supported_rates[ii] =
 				cpu_to_le16(rsi_mcsrates[kk--]);
 		}
@@ -1216,15 +1288,12 @@ static int rsi_send_auto_rate_request(struct rsi_common *common)
 
 	auto_rate->num_supported_rates = cpu_to_le16(num_supported_rates * 2);
 	auto_rate->moderate_rate_inx = cpu_to_le16(num_supported_rates / 2);
-	auto_rate->desc_word[7] |= cpu_to_le16(0 << 8);
 	num_supported_rates *= 2;
 
-	auto_rate->desc_word[0] = cpu_to_le16((sizeof(*auto_rate) -
-					       FRAME_DESC_SZ) |
-					       (RSI_WIFI_MGMT_Q << 12));
+	rsi_set_len_qno(&auto_rate->desc.desc_dword0.len_qno,
+			(frame_len - FRAME_DESC_SZ), RSI_WIFI_MGMT_Q);
 
-	skb_put(skb,
-		sizeof(struct rsi_auto_rate));
+	skb_put(skb, frame_len);
 	kfree(selected_rates);
 
 	return rsi_send_internal_mgmt_frame(common, skb);
@@ -1243,27 +1312,40 @@ static int rsi_send_auto_rate_request(struct rsi_common *common)
  * Return: None.
  */
 void rsi_inform_bss_status(struct rsi_common *common,
+			   enum opmode opmode,
 			   u8 status,
-			   const unsigned char *bssid,
+			   const u8 *addr,
 			   u8 qos_enable,
-			   u16 aid)
+			   u16 aid,
+			   struct ieee80211_sta *sta,
+			   u16 sta_id)
 {
 	if (status) {
+		if (opmode == STA_OPMODE)
+			common->hw_data_qs_blocked = true;
 		rsi_hal_send_sta_notify_frame(common,
-					      RSI_IFTYPE_STATION,
+					      opmode,
 					      STA_CONNECTED,
-					      bssid,
+					      addr,
 					      qos_enable,
-					      aid);
+					      aid, sta_id);
 		if (common->min_rate == 0xffff)
-			rsi_send_auto_rate_request(common);
+			rsi_send_auto_rate_request(common, sta, sta_id);
+		if (opmode == STA_OPMODE) {
+			if (!rsi_send_block_unblock_frame(common, false))
+				common->hw_data_qs_blocked = false;
+		}
 	} else {
+		if (opmode == STA_OPMODE)
+			common->hw_data_qs_blocked = true;
 		rsi_hal_send_sta_notify_frame(common,
-					      RSI_IFTYPE_STATION,
+					      opmode,
 					      STA_DISCONNECTED,
-					      bssid,
+					      addr,
 					      qos_enable,
-					      aid);
+					      aid, sta_id);
+		if (opmode == STA_OPMODE)
+			rsi_send_block_unblock_frame(common, true);
 	}
 }
 
@@ -1276,7 +1358,8 @@ void rsi_inform_bss_status(struct rsi_common *common,
  */
 static int rsi_eeprom_read(struct rsi_common *common)
 {
-	struct rsi_mac_frame *mgmt_frame;
+	struct rsi_eeprom_read_frame *mgmt_frame;
+	struct rsi_hw *adapter = common->priv;
 	struct sk_buff *skb;
 
 	rsi_dbg(MGMT_TX_ZONE, "%s: Sending EEPROM read req frame\n", __func__);
@@ -1289,18 +1372,21 @@ static int rsi_eeprom_read(struct rsi_common *common)
 	}
 
 	memset(skb->data, 0, FRAME_DESC_SZ);
-	mgmt_frame = (struct rsi_mac_frame *)skb->data;
+	mgmt_frame = (struct rsi_eeprom_read_frame *)skb->data;
 
 	/* FrameType */
-	mgmt_frame->desc_word[1] = cpu_to_le16(EEPROM_READ_TYPE);
-	mgmt_frame->desc_word[0] = cpu_to_le16(RSI_WIFI_MGMT_Q << 12);
+	rsi_set_len_qno(&mgmt_frame->len_qno, 0, RSI_WIFI_MGMT_Q);
+	mgmt_frame->pkt_type = EEPROM_READ;
+
 	/* Number of bytes to read */
-	mgmt_frame->desc_word[3] = cpu_to_le16(ETH_ALEN +
-					       WLAN_MAC_MAGIC_WORD_LEN +
-					       WLAN_HOST_MODE_LEN +
-					       WLAN_FW_VERSION_LEN);
+	mgmt_frame->pkt_info =
+		cpu_to_le32((adapter->eeprom.length << RSI_EEPROM_LEN_OFFSET) &
+			    RSI_EEPROM_LEN_MASK);
+	mgmt_frame->pkt_info |= cpu_to_le32((3 << RSI_EEPROM_HDR_SIZE_OFFSET) &
+					    RSI_EEPROM_HDR_SIZE_MASK);
+
 	/* Address to read */
-	mgmt_frame->desc_word[4] = cpu_to_le16(WLAN_MAC_EEPROM_ADDR);
+	mgmt_frame->eeprom_offset = cpu_to_le32(adapter->eeprom.offset);
 
 	skb_put(skb, FRAME_DESC_SZ);
 
@@ -1317,7 +1403,7 @@ static int rsi_eeprom_read(struct rsi_common *common)
  */
 int rsi_send_block_unblock_frame(struct rsi_common *common, bool block_event)
 {
-	struct rsi_mac_frame *mgmt_frame;
+	struct rsi_block_unblock_data *mgmt_frame;
 	struct sk_buff *skb;
 
 	rsi_dbg(MGMT_TX_ZONE, "%s: Sending block/unblock frame\n", __func__);
@@ -1330,23 +1416,25 @@ int rsi_send_block_unblock_frame(struct rsi_common *common, bool block_event)
 	}
 
 	memset(skb->data, 0, FRAME_DESC_SZ);
-	mgmt_frame = (struct rsi_mac_frame *)skb->data;
+	mgmt_frame = (struct rsi_block_unblock_data *)skb->data;
 
-	mgmt_frame->desc_word[0] = cpu_to_le16(RSI_WIFI_MGMT_Q << 12);
-	mgmt_frame->desc_word[1] = cpu_to_le16(BLOCK_HW_QUEUE);
+	rsi_set_len_qno(&mgmt_frame->desc_dword0.len_qno, 0, RSI_WIFI_MGMT_Q);
+	mgmt_frame->desc_dword0.frame_type = BLOCK_HW_QUEUE;
+	mgmt_frame->host_quiet_info = QUIET_INFO_VALID;
 
 	if (block_event) {
 		rsi_dbg(INFO_ZONE, "blocking the data qs\n");
-		mgmt_frame->desc_word[4] = cpu_to_le16(0xf);
+		mgmt_frame->block_q_bitmap = cpu_to_le16(0xf);
+		mgmt_frame->block_q_bitmap |= cpu_to_le16(0xf << 4);
 	} else {
 		rsi_dbg(INFO_ZONE, "unblocking the data qs\n");
-		mgmt_frame->desc_word[5] = cpu_to_le16(0xf);
+		mgmt_frame->unblock_q_bitmap = cpu_to_le16(0xf);
+		mgmt_frame->unblock_q_bitmap |= cpu_to_le16(0xf << 4);
 	}
 
 	skb_put(skb, FRAME_DESC_SZ);
 
 	return rsi_send_internal_mgmt_frame(common, skb);
-
 }
 
 /**
@@ -1383,6 +1471,61 @@ int rsi_send_rx_filter_frame(struct rsi_common *common, u16 rx_filter_word)
 	return rsi_send_internal_mgmt_frame(common, skb);
 }
 
+int rsi_send_ps_request(struct rsi_hw *adapter, bool enable)
+{
+	struct rsi_common *common = adapter->priv;
+	struct ieee80211_bss_conf *bss = &adapter->vifs[0]->bss_conf;
+	struct rsi_request_ps *ps;
+	struct rsi_ps_info *ps_info;
+	struct sk_buff *skb;
+	int frame_len = sizeof(*ps);
+
+	skb = dev_alloc_skb(frame_len);
+	if (!skb)
+		return -ENOMEM;
+	memset(skb->data, 0, frame_len);
+
+	ps = (struct rsi_request_ps *)skb->data;
+	ps_info = &adapter->ps_info;
+
+	rsi_set_len_qno(&ps->desc.desc_dword0.len_qno,
+			(frame_len - FRAME_DESC_SZ), RSI_WIFI_MGMT_Q);
+	ps->desc.desc_dword0.frame_type = WAKEUP_SLEEP_REQUEST;
+	if (enable) {
+		ps->ps_sleep.enable = RSI_PS_ENABLE;
+		ps->desc.desc_dword3.token = cpu_to_le16(RSI_SLEEP_REQUEST);
+	} else {
+		ps->ps_sleep.enable = RSI_PS_DISABLE;
+		ps->desc.desc_dword0.len_qno |= cpu_to_le16(RSI_PS_DISABLE_IND);
+		ps->desc.desc_dword3.token = cpu_to_le16(RSI_WAKEUP_REQUEST);
+	}
+
+	ps->ps_uapsd_acs = common->uapsd_bitmap;
+
+	ps->ps_sleep.sleep_type = ps_info->sleep_type;
+	ps->ps_sleep.num_bcns_per_lis_int =
+		cpu_to_le16(ps_info->num_bcns_per_lis_int);
+	ps->ps_sleep.sleep_duration =
+		cpu_to_le32(ps_info->deep_sleep_wakeup_period);
+
+	if (bss->assoc)
+		ps->ps_sleep.connected_sleep = RSI_CONNECTED_SLEEP;
+	else
+		ps->ps_sleep.connected_sleep = RSI_DEEP_SLEEP;
+
+	ps->ps_listen_interval = cpu_to_le32(ps_info->listen_interval);
+	ps->ps_dtim_interval_duration =
+		cpu_to_le32(ps_info->dtim_interval_duration);
+
+	if (ps_info->listen_interval > ps_info->dtim_interval_duration)
+		ps->ps_listen_interval = cpu_to_le32(RSI_PS_DISABLE);
+
+	ps->ps_num_dtim_intervals = cpu_to_le16(ps_info->num_dtims_per_sleep);
+	skb_put(skb, frame_len);
+
+	return rsi_send_internal_mgmt_frame(common, skb);
+}
+
 /**
  * rsi_set_antenna() - This fuction send antenna configuration request
  *		       to device
@@ -1394,7 +1537,7 @@ int rsi_send_rx_filter_frame(struct rsi_common *common, u16 rx_filter_word)
  */
 int rsi_set_antenna(struct rsi_common *common, u8 antenna)
 {
-	struct rsi_mac_frame *cmd_frame;
+	struct rsi_ant_sel_frame *ant_sel_frame;
 	struct sk_buff *skb;
 
 	skb = dev_alloc_skb(FRAME_DESC_SZ);
@@ -1405,17 +1548,43 @@ int rsi_set_antenna(struct rsi_common *common, u8 antenna)
 	}
 
 	memset(skb->data, 0, FRAME_DESC_SZ);
-	cmd_frame = (struct rsi_mac_frame *)skb->data;
-
-	cmd_frame->desc_word[1] = cpu_to_le16(ANT_SEL_FRAME);
-	cmd_frame->desc_word[3] = cpu_to_le16(antenna & 0x00ff);
-	cmd_frame->desc_word[0] = cpu_to_le16(RSI_WIFI_MGMT_Q << 12);
 
+	ant_sel_frame = (struct rsi_ant_sel_frame *)skb->data;
+	ant_sel_frame->desc_dword0.frame_type = ANT_SEL_FRAME;
+	ant_sel_frame->sub_frame_type = ANTENNA_SEL_TYPE;
+	ant_sel_frame->ant_value = cpu_to_le16(antenna & ANTENNA_MASK_VALUE);
+	rsi_set_len_qno(&ant_sel_frame->desc_dword0.len_qno,
+			0, RSI_WIFI_MGMT_Q);
 	skb_put(skb, FRAME_DESC_SZ);
 
 	return rsi_send_internal_mgmt_frame(common, skb);
 }
 
+static int rsi_send_beacon(struct rsi_common *common)
+{
+	struct sk_buff *skb = NULL;
+	u8 dword_align_bytes = 0;
+
+	skb = dev_alloc_skb(MAX_MGMT_PKT_SIZE);
+	if (!skb)
+		return -ENOMEM;
+
+	memset(skb->data, 0, MAX_MGMT_PKT_SIZE);
+
+	dword_align_bytes = ((unsigned long)skb->data & 0x3f);
+	if (dword_align_bytes)
+		skb_pull(skb, (64 - dword_align_bytes));
+	if (rsi_prepare_beacon(common, skb)) {
+		rsi_dbg(ERR_ZONE, "Failed to prepare beacon\n");
+		return -EINVAL;
+	}
+	skb_queue_tail(&common->tx_queue[MGMT_BEACON_Q], skb);
+	rsi_set_event(&common->tx_thread.event);
+	rsi_dbg(DATA_TX_ZONE, "%s: Added to beacon queue\n", __func__);
+
+	return 0;
+}
+
 /**
  * rsi_handle_ta_confirm_type() - This function handles the confirm frames.
  * @common: Pointer to the driver private structure.
@@ -1426,19 +1595,25 @@ int rsi_set_antenna(struct rsi_common *common, u8 antenna)
 static int rsi_handle_ta_confirm_type(struct rsi_common *common,
 				      u8 *msg)
 {
+	struct rsi_hw *adapter = common->priv;
 	u8 sub_type = (msg[15] & 0xff);
+	u16 msg_len = ((u16 *)msg)[0] & 0xfff;
+	u8 offset;
 
 	switch (sub_type) {
 	case BOOTUP_PARAMS_REQUEST:
 		rsi_dbg(FSM_ZONE, "%s: Boot up params confirm received\n",
 			__func__);
 		if (common->fsm_state == FSM_BOOT_PARAMS_SENT) {
+			adapter->eeprom.length = (IEEE80211_ADDR_LEN +
+						  WLAN_MAC_MAGIC_WORD_LEN +
+						  WLAN_HOST_MODE_LEN);
+			adapter->eeprom.offset = WLAN_MAC_EEPROM_ADDR;
 			if (rsi_eeprom_read(common)) {
 				common->fsm_state = FSM_CARD_NOT_READY;
 				goto out;
-			} else {
-				common->fsm_state = FSM_EEPROM_READ_MAC_ADDR;
 			}
+			common->fsm_state = FSM_EEPROM_READ_MAC_ADDR;
 		} else {
 			rsi_dbg(INFO_ZONE,
 				"%s: Received bootup params cfm in %d state\n",
@@ -1447,30 +1622,52 @@ static int rsi_handle_ta_confirm_type(struct rsi_common *common,
 		}
 		break;
 
-	case EEPROM_READ_TYPE:
+	case EEPROM_READ:
+		rsi_dbg(FSM_ZONE, "EEPROM READ confirm received\n");
+		if (msg_len <= 0) {
+			rsi_dbg(FSM_ZONE,
+				"%s: [EEPROM_READ] Invalid len %d\n",
+				__func__, msg_len);
+			goto out;
+		}
+		if (msg[16] != MAGIC_WORD) {
+			rsi_dbg(FSM_ZONE,
+				"%s: [EEPROM_READ] Invalid token\n", __func__);
+			common->fsm_state = FSM_CARD_NOT_READY;
+			goto out;
+		}
 		if (common->fsm_state == FSM_EEPROM_READ_MAC_ADDR) {
-			if (msg[16] == MAGIC_WORD) {
-				u8 offset = (FRAME_DESC_SZ + WLAN_HOST_MODE_LEN
-					     + WLAN_MAC_MAGIC_WORD_LEN);
-				memcpy(common->mac_addr,
-				       &msg[offset],
-				       ETH_ALEN);
-				memcpy(&common->fw_ver,
-				       &msg[offset + ETH_ALEN],
-				       sizeof(struct version_info));
-
-			} else {
+			offset = (FRAME_DESC_SZ + WLAN_HOST_MODE_LEN +
+				  WLAN_MAC_MAGIC_WORD_LEN);
+			memcpy(common->mac_addr, &msg[offset], ETH_ALEN);
+			adapter->eeprom.length =
+				((WLAN_MAC_MAGIC_WORD_LEN + 3) & (~3));
+			adapter->eeprom.offset = WLAN_EEPROM_RFTYPE_ADDR;
+			if (rsi_eeprom_read(common)) {
+				rsi_dbg(ERR_ZONE,
+					"%s: Failed reading RF band\n",
+					__func__);
 				common->fsm_state = FSM_CARD_NOT_READY;
-				break;
+				goto out;
+			}
+			common->fsm_state = FSM_EEPROM_READ_RF_TYPE;
+		} else if (common->fsm_state == FSM_EEPROM_READ_RF_TYPE) {
+			if ((msg[17] & 0x3) == 0x3) {
+				rsi_dbg(INIT_ZONE, "Dual band supported\n");
+				common->band = NL80211_BAND_5GHZ;
+				common->num_supp_bands = 2;
+			} else if ((msg[17] & 0x3) == 0x1) {
+				rsi_dbg(INIT_ZONE,
+					"Only 2.4Ghz band supported\n");
+				common->band = NL80211_BAND_2GHZ;
+				common->num_supp_bands = 1;
 			}
 			if (rsi_send_reset_mac(common))
 				goto out;
-			else
-				common->fsm_state = FSM_RESET_MAC_SENT;
+			common->fsm_state = FSM_RESET_MAC_SENT;
 		} else {
-			rsi_dbg(ERR_ZONE,
-				"%s: Received eeprom mac addr in %d state\n",
-				__func__, common->fsm_state);
+			rsi_dbg(ERR_ZONE, "%s: Invalid EEPROM read type\n",
+				__func__);
 			return 0;
 		}
 		break;
@@ -1527,7 +1724,9 @@ static int rsi_handle_ta_confirm_type(struct rsi_common *common,
 			return 0;
 		}
 		break;
-
+	case WAKEUP_SLEEP_REQUEST:
+		rsi_dbg(INFO_ZONE, "Wakeup/Sleep confirmation.\n");
+		return rsi_handle_ps_confirm(adapter, msg);
 	default:
 		rsi_dbg(INFO_ZONE, "%s: Invalid TA confirm pkt received\n",
 			__func__);
@@ -1590,20 +1789,34 @@ int rsi_mgmt_pkt_recv(struct rsi_common *common, u8 *msg)
 	rsi_dbg(FSM_ZONE, "%s: Msg Len: %d, Msg Type: %4x\n",
 		__func__, msg_len, msg_type);
 
-	if (msg_type == TA_CONFIRM_TYPE) {
+	switch (msg_type) {
+	case TA_CONFIRM_TYPE:
 		return rsi_handle_ta_confirm_type(common, msg);
-	} else if (msg_type == CARD_READY_IND) {
+	case CARD_READY_IND:
 		rsi_dbg(FSM_ZONE, "%s: Card ready indication received\n",
 			__func__);
 		return rsi_handle_card_ready(common, msg);
-	} else if (msg_type == TX_STATUS_IND) {
+	case TX_STATUS_IND:
 		if (msg[15] == PROBEREQ_CONFIRM) {
 			common->mgmt_q_block = false;
 			rsi_dbg(FSM_ZONE, "%s: Probe confirm received\n",
 				__func__);
 		}
-	} else {
-		return rsi_mgmt_pkt_to_core(common, msg, msg_len, msg_type);
+		break;
+	case BEACON_EVENT_IND:
+		rsi_dbg(INFO_ZONE, "Beacon event\n");
+		if (common->fsm_state != FSM_MAC_INIT_DONE)
+			return -1;
+		if (common->iface_down)
+			return -1;
+		if (!common->beacon_enabled)
+			return -1;
+		rsi_send_beacon(common);
+		break;
+	case RX_DOT11_MGMT:
+		return rsi_mgmt_pkt_to_core(common, msg, msg_len);
+	default:
+		rsi_dbg(INFO_ZONE, "Received packet type: 0x%x\n", msg_type);
 	}
 	return 0;
 }
diff --git a/drivers/net/wireless/rsi/rsi_91x_ps.c b/drivers/net/wireless/rsi/rsi_91x_ps.c
new file mode 100644
index 000000000000..48c79f035c59
--- /dev/null
+++ b/drivers/net/wireless/rsi/rsi_91x_ps.c
@@ -0,0 +1,146 @@
+/**
+ * Copyright (c) 2014 Redpine Signals Inc.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <linux/etherdevice.h>
+#include <linux/if.h>
+#include <linux/version.h>
+#include "rsi_debugfs.h"
+#include "rsi_mgmt.h"
+#include "rsi_common.h"
+#include "rsi_ps.h"
+
+char *str_psstate(enum ps_state state)
+{
+	switch (state) {
+	case PS_NONE:
+		return "PS_NONE";
+	case PS_DISABLE_REQ_SENT:
+		return "PS_DISABLE_REQ_SENT";
+	case PS_ENABLE_REQ_SENT:
+		return "PS_ENABLE_REQ_SENT";
+	case PS_ENABLED:
+		return "PS_ENABLED";
+	default:
+		return "INVALID_STATE";
+	}
+	return "INVALID_STATE";
+}
+
+static inline void rsi_modify_ps_state(struct rsi_hw *adapter,
+				       enum ps_state nstate)
+{
+	rsi_dbg(INFO_ZONE, "PS state changed %s => %s\n",
+		str_psstate(adapter->ps_state),
+		str_psstate(nstate));
+
+	adapter->ps_state = nstate;
+}
+
+void rsi_default_ps_params(struct rsi_hw *adapter)
+{
+	struct rsi_ps_info *ps_info = &adapter->ps_info;
+
+	ps_info->enabled = true;
+	ps_info->sleep_type = RSI_SLEEP_TYPE_LP;
+	ps_info->tx_threshold = 0;
+	ps_info->rx_threshold = 0;
+	ps_info->tx_hysterisis = 0;
+	ps_info->rx_hysterisis = 0;
+	ps_info->monitor_interval = 0;
+	ps_info->listen_interval = RSI_DEF_LISTEN_INTERVAL;
+	ps_info->num_bcns_per_lis_int = 0;
+	ps_info->dtim_interval_duration = 0;
+	ps_info->num_dtims_per_sleep = 0;
+	ps_info->deep_sleep_wakeup_period = RSI_DEF_DS_WAKEUP_PERIOD;
+}
+
+void rsi_enable_ps(struct rsi_hw *adapter)
+{
+	if (adapter->ps_state != PS_NONE) {
+		rsi_dbg(ERR_ZONE,
+			"%s: Cannot accept enable PS in %s state\n",
+			__func__, str_psstate(adapter->ps_state));
+		return;
+	}
+
+	if (rsi_send_ps_request(adapter, true)) {
+		rsi_dbg(ERR_ZONE,
+			"%s: Failed to send PS request to device\n",
+			__func__);
+		return;
+	}
+
+	rsi_modify_ps_state(adapter, PS_ENABLE_REQ_SENT);
+}
+
+void rsi_disable_ps(struct rsi_hw *adapter)
+{
+	if (adapter->ps_state != PS_ENABLED) {
+		rsi_dbg(ERR_ZONE,
+			"%s: Cannot accept disable PS in %s state\n",
+			__func__, str_psstate(adapter->ps_state));
+		return;
+	}
+
+	if (rsi_send_ps_request(adapter, false)) {
+		rsi_dbg(ERR_ZONE,
+			"%s: Failed to send PS request to device\n",
+			__func__);
+		return;
+	}
+
+	rsi_modify_ps_state(adapter, PS_DISABLE_REQ_SENT);
+}
+
+void rsi_conf_uapsd(struct rsi_hw *adapter)
+{
+	int ret;
+
+	if (adapter->ps_state != PS_ENABLED)
+		return;
+
+	ret = rsi_send_ps_request(adapter, false);
+	if (!ret)
+		ret = rsi_send_ps_request(adapter, true);
+	if (ret)
+		rsi_dbg(ERR_ZONE,
+			"%s: Failed to send PS request to device\n",
+			__func__);
+}
+
+int rsi_handle_ps_confirm(struct rsi_hw *adapter, u8 *msg)
+{
+	u16 cfm_type = get_unaligned_le16(msg + PS_CONFIRM_INDEX);
+
+	switch (cfm_type) {
+	case RSI_SLEEP_REQUEST:
+		if (adapter->ps_state == PS_ENABLE_REQ_SENT)
+			rsi_modify_ps_state(adapter, PS_ENABLED);
+		break;
+	case RSI_WAKEUP_REQUEST:
+		if (adapter->ps_state == PS_DISABLE_REQ_SENT)
+			rsi_modify_ps_state(adapter, PS_NONE);
+		break;
+	default:
+		rsi_dbg(ERR_ZONE,
+			"Invalid PS confirm type %x in state %s\n",
+			cfm_type, str_psstate(adapter->ps_state));
+		return -1;
+	}
+
+	return 0;
+}
+
diff --git a/drivers/net/wireless/rsi/rsi_91x_sdio.c b/drivers/net/wireless/rsi/rsi_91x_sdio.c
index e5ea99bb2dd8..8d3a4839b6ef 100644
--- a/drivers/net/wireless/rsi/rsi_91x_sdio.c
+++ b/drivers/net/wireless/rsi/rsi_91x_sdio.c
@@ -138,12 +138,15 @@ static int rsi_issue_sdiocommand(struct sdio_func *func,
 static void rsi_handle_interrupt(struct sdio_func *function)
 {
 	struct rsi_hw *adapter = sdio_get_drvdata(function);
+	struct rsi_91x_sdiodev *dev =
+		(struct rsi_91x_sdiodev *)adapter->rsi_dev;
 
 	if (adapter->priv->fsm_state == FSM_FW_NOT_LOADED)
 		return;
-	sdio_release_host(function);
+
+	dev->sdio_irq_task = current;
 	rsi_interrupt_handler(adapter);
-	sdio_claim_host(function);
+	dev->sdio_irq_task = NULL;
 }
 
 /**
@@ -219,26 +222,18 @@ static void rsi_reset_card(struct sdio_func *pfunction)
 	if (err)
 		rsi_dbg(ERR_ZONE, "%s: CMD0 failed : %d\n", __func__, err);
 
-	if (!host->ocr_avail) {
-		/* Issue CMD5, arg = 0 */
-		err = rsi_issue_sdiocommand(pfunction,
-					    SD_IO_SEND_OP_COND,
-					    0,
-					    (MMC_RSP_R4 | MMC_CMD_BCR),
-					    &resp);
-		if (err)
-			rsi_dbg(ERR_ZONE, "%s: CMD5 failed : %d\n",
-				__func__, err);
-		host->ocr_avail = resp;
-	}
+	/* Issue CMD5, arg = 0 */
+	err = rsi_issue_sdiocommand(pfunction,	SD_IO_SEND_OP_COND, 0,
+				    (MMC_RSP_R4 | MMC_CMD_BCR), &resp);
+	if (err)
+		rsi_dbg(ERR_ZONE, "%s: CMD5 failed : %d\n", __func__, err);
+	card->ocr = resp;
 
 	/* Issue CMD5, arg = ocr. Wait till card is ready  */
 	for (i = 0; i < 100; i++) {
-		err = rsi_issue_sdiocommand(pfunction,
-					    SD_IO_SEND_OP_COND,
-					    host->ocr_avail,
-					    (MMC_RSP_R4 | MMC_CMD_BCR),
-					    &resp);
+		err = rsi_issue_sdiocommand(pfunction, SD_IO_SEND_OP_COND,
+					    card->ocr,
+					    (MMC_RSP_R4 | MMC_CMD_BCR), &resp);
 		if (err) {
 			rsi_dbg(ERR_ZONE, "%s: CMD5 failed : %d\n",
 				__func__, err);
@@ -415,14 +410,16 @@ int rsi_sdio_read_register(struct rsi_hw *adapter,
 	u8 fun_num = 0;
 	int status;
 
-	sdio_claim_host(dev->pfunction);
+	if (likely(dev->sdio_irq_task != current))
+		sdio_claim_host(dev->pfunction);
 
 	if (fun_num == 0)
 		*data = sdio_f0_readb(dev->pfunction, addr, &status);
 	else
 		*data = sdio_readb(dev->pfunction, addr, &status);
 
-	sdio_release_host(dev->pfunction);
+	if (likely(dev->sdio_irq_task != current))
+		sdio_release_host(dev->pfunction);
 
 	return status;
 }
@@ -446,14 +443,16 @@ int rsi_sdio_write_register(struct rsi_hw *adapter,
 		(struct rsi_91x_sdiodev *)adapter->rsi_dev;
 	int status = 0;
 
-	sdio_claim_host(dev->pfunction);
+	if (likely(dev->sdio_irq_task != current))
+		sdio_claim_host(dev->pfunction);
 
 	if (function == 0)
 		sdio_f0_writeb(dev->pfunction, *data, addr, &status);
 	else
 		sdio_writeb(dev->pfunction, *data, addr, &status);
 
-	sdio_release_host(dev->pfunction);
+	if (likely(dev->sdio_irq_task != current))
+		sdio_release_host(dev->pfunction);
 
 	return status;
 }
@@ -498,11 +497,13 @@ static int rsi_sdio_read_register_multiple(struct rsi_hw *adapter,
 		(struct rsi_91x_sdiodev *)adapter->rsi_dev;
 	u32 status;
 
-	sdio_claim_host(dev->pfunction);
+	if (likely(dev->sdio_irq_task != current))
+		sdio_claim_host(dev->pfunction);
 
 	status =  sdio_readsb(dev->pfunction, data, addr, count);
 
-	sdio_release_host(dev->pfunction);
+	if (likely(dev->sdio_irq_task != current))
+		sdio_release_host(dev->pfunction);
 
 	if (status != 0)
 		rsi_dbg(ERR_ZONE, "%s: Synch Cmd53 read failed\n", __func__);
@@ -540,11 +541,13 @@ int rsi_sdio_write_register_multiple(struct rsi_hw *adapter,
 		dev->write_fail++;
 	}
 
-	sdio_claim_host(dev->pfunction);
+	if (likely(dev->sdio_irq_task != current))
+		sdio_claim_host(dev->pfunction);
 
 	status = sdio_writesb(dev->pfunction, addr, data, count);
 
-	sdio_release_host(dev->pfunction);
+	if (likely(dev->sdio_irq_task != current))
+		sdio_release_host(dev->pfunction);
 
 	if (status) {
 		rsi_dbg(ERR_ZONE, "%s: Synch Cmd53 write failed %d\n",
@@ -581,7 +584,6 @@ static int rsi_sdio_load_data_master_write(struct rsi_hw *adapter,
 	}
 
 	for (offset = 0, i = 0; i < num_blocks; i++, offset += block_size) {
-		memset(temp_buf, 0, block_size);
 		memcpy(temp_buf, ta_firmware + offset, block_size);
 		lsb_address = (u16)base_address;
 		status = rsi_sdio_write_register_multiple
@@ -857,7 +859,7 @@ static int rsi_init_sdio_interface(struct rsi_hw *adapter,
 	sdio_release_host(pfunction);
 
 	adapter->determine_event_timeout = rsi_sdio_determine_event_timeout;
-	adapter->check_hw_queue_status = rsi_sdio_read_buffer_status_register;
+	adapter->check_hw_queue_status = rsi_sdio_check_buffer_status;
 
 #ifdef CONFIG_RSI_DEBUGFS
 	adapter->num_debugfs_entries = MAX_DEBUGFS_ENTRIES;
@@ -941,6 +943,84 @@ fail:
 	return 1;
 }
 
+static void ulp_read_write(struct rsi_hw *adapter, u16 addr, u32 data,
+			   u16 len_in_bits)
+{
+	rsi_sdio_master_reg_write(adapter, RSI_GSPI_DATA_REG1,
+				  ((addr << 6) | ((data >> 16) & 0xffff)), 2);
+	rsi_sdio_master_reg_write(adapter, RSI_GSPI_DATA_REG0,
+				  (data & 0xffff), 2);
+	rsi_sdio_master_reg_write(adapter, RSI_GSPI_CTRL_REG0,
+				  RSI_GSPI_CTRL_REG0_VALUE, 2);
+	rsi_sdio_master_reg_write(adapter, RSI_GSPI_CTRL_REG1,
+				  ((len_in_bits - 1) | RSI_GSPI_TRIG), 2);
+	msleep(20);
+}
+
+/*This function resets and re-initializes the chip.*/
+static void rsi_reset_chip(struct rsi_hw *adapter)
+{
+	__le32 data;
+	u8 sdio_interrupt_status = 0;
+	u8 request = 1;
+	int ret;
+
+	rsi_dbg(INFO_ZONE, "Writing disable to wakeup register\n");
+	ret =  rsi_sdio_write_register(adapter, 0, SDIO_WAKEUP_REG, &request);
+	if (ret < 0) {
+		rsi_dbg(ERR_ZONE,
+			"%s: Failed to write SDIO wakeup register\n", __func__);
+		return;
+	}
+	msleep(20);
+	ret =  rsi_sdio_read_register(adapter, RSI_FN1_INT_REGISTER,
+				      &sdio_interrupt_status);
+	if (ret < 0) {
+		rsi_dbg(ERR_ZONE, "%s: Failed to Read Intr Status Register\n",
+			__func__);
+		return;
+	}
+	rsi_dbg(INFO_ZONE, "%s: Intr Status Register value = %d\n",
+		__func__, sdio_interrupt_status);
+
+	/* Put Thread-Arch processor on hold */
+	if (rsi_sdio_master_access_msword(adapter, TA_BASE_ADDR)) {
+		rsi_dbg(ERR_ZONE,
+			"%s: Unable to set ms word to common reg\n",
+			__func__);
+		return;
+	}
+
+	data = TA_HOLD_THREAD_VALUE;
+	if (rsi_sdio_write_register_multiple(adapter, TA_HOLD_THREAD_REG |
+					     RSI_SD_REQUEST_MASTER,
+					     (u8 *)&data, 4)) {
+		rsi_dbg(ERR_ZONE,
+			"%s: Unable to hold Thread-Arch processor threads\n",
+			__func__);
+		return;
+	}
+
+	/* This msleep will ensure Thread-Arch processor to go to hold
+	 * and any pending dma transfers to rf spi in device to finish.
+	 */
+	msleep(100);
+
+	ulp_read_write(adapter, RSI_ULP_RESET_REG, RSI_ULP_WRITE_0, 32);
+	ulp_read_write(adapter, RSI_WATCH_DOG_TIMER_1, RSI_ULP_WRITE_2, 32);
+	ulp_read_write(adapter, RSI_WATCH_DOG_TIMER_2, RSI_ULP_WRITE_0, 32);
+	ulp_read_write(adapter, RSI_WATCH_DOG_DELAY_TIMER_1, RSI_ULP_WRITE_50,
+		       32);
+	ulp_read_write(adapter, RSI_WATCH_DOG_DELAY_TIMER_2, RSI_ULP_WRITE_0,
+		       32);
+	ulp_read_write(adapter, RSI_WATCH_DOG_TIMER_ENABLE,
+		       RSI_ULP_TIMER_ENABLE, 32);
+	/* This msleep will be sufficient for the ulp
+	 * read write operations to complete for chip reset.
+	 */
+	msleep(500);
+}
+
 /**
  * rsi_disconnect() - This function performs the reverse of the probe function.
  * @pfunction: Pointer to the sdio_func structure.
@@ -956,17 +1036,26 @@ static void rsi_disconnect(struct sdio_func *pfunction)
 		return;
 
 	dev = (struct rsi_91x_sdiodev *)adapter->rsi_dev;
+	sdio_claim_host(pfunction);
+	sdio_release_irq(pfunction);
+	sdio_release_host(pfunction);
+	mdelay(10);
 
-	dev->write_fail = 2;
 	rsi_mac80211_detach(adapter);
+	mdelay(10);
+
+	/* Reset Chip */
+	rsi_reset_chip(adapter);
 
-	sdio_claim_host(pfunction);
-	sdio_release_irq(pfunction);
-	sdio_disable_func(pfunction);
-	rsi_91x_deinit(adapter);
 	/* Resetting to take care of the case, where-in driver is re-loaded */
+	sdio_claim_host(pfunction);
 	rsi_reset_card(pfunction);
+	sdio_disable_func(pfunction);
 	sdio_release_host(pfunction);
+	dev->write_fail = 2;
+	rsi_91x_deinit(adapter);
+	rsi_dbg(ERR_ZONE, "##### RSI SDIO device disconnected #####\n");
+
 }
 
 #ifdef CONFIG_PM
diff --git a/drivers/net/wireless/rsi/rsi_91x_sdio_ops.c b/drivers/net/wireless/rsi/rsi_91x_sdio_ops.c
index df2a63b1f15c..8e2a95c486b0 100644
--- a/drivers/net/wireless/rsi/rsi_91x_sdio_ops.c
+++ b/drivers/net/wireless/rsi/rsi_91x_sdio_ops.c
@@ -69,20 +69,37 @@ int rsi_sdio_master_access_msword(struct rsi_hw *adapter, u16 ms_word)
 static int rsi_process_pkt(struct rsi_common *common)
 {
 	struct rsi_hw *adapter = common->priv;
+	struct rsi_91x_sdiodev *dev =
+		(struct rsi_91x_sdiodev *)adapter->rsi_dev;
 	u8 num_blks = 0;
 	u32 rcv_pkt_len = 0;
 	int status = 0;
+	u8 value = 0;
 
-	status = rsi_sdio_read_register(adapter,
-					SDIO_RX_NUM_BLOCKS_REG,
-					&num_blks);
+	num_blks = ((adapter->interrupt_status & 1) |
+			((adapter->interrupt_status >> RECV_NUM_BLOCKS) << 1));
 
-	if (status) {
-		rsi_dbg(ERR_ZONE,
-			"%s: Failed to read pkt length from the card:\n",
-			__func__);
-		return status;
+	if (!num_blks) {
+		status = rsi_sdio_read_register(adapter,
+						SDIO_RX_NUM_BLOCKS_REG,
+						&value);
+		if (status) {
+			rsi_dbg(ERR_ZONE,
+				"%s: Failed to read pkt length from the card:\n",
+				__func__);
+			return status;
+		}
+		num_blks = value & 0x1f;
+	}
+
+	if (dev->write_fail == 2)
+		rsi_sdio_ack_intr(common->priv, (1 << MSDU_PKT_PENDING));
+
+	if (unlikely(!num_blks)) {
+		dev->write_fail = 2;
+		return -1;
 	}
+
 	rcv_pkt_len = (num_blks * 256);
 
 	common->rx_data_pkt = kmalloc(rcv_pkt_len, GFP_KERNEL);
@@ -213,7 +230,7 @@ void rsi_interrupt_handler(struct rsi_hw *adapter)
 	dev->rx_info.sdio_int_counter++;
 
 	do {
-		mutex_lock(&common->tx_rxlock);
+		mutex_lock(&common->rx_lock);
 		status = rsi_sdio_read_register(common->priv,
 						RSI_FN1_INT_REGISTER,
 						&isr_status);
@@ -221,14 +238,15 @@ void rsi_interrupt_handler(struct rsi_hw *adapter)
 			rsi_dbg(ERR_ZONE,
 				"%s: Failed to Read Intr Status Register\n",
 				__func__);
-			mutex_unlock(&common->tx_rxlock);
+			mutex_unlock(&common->rx_lock);
 			return;
 		}
+		adapter->interrupt_status = isr_status;
 
 		if (isr_status == 0) {
 			rsi_set_event(&common->tx_thread.event);
 			dev->rx_info.sdio_intr_status_zero++;
-			mutex_unlock(&common->tx_rxlock);
+			mutex_unlock(&common->rx_lock);
 			return;
 		}
 
@@ -241,10 +259,12 @@ void rsi_interrupt_handler(struct rsi_hw *adapter)
 
 			switch (isr_type) {
 			case BUFFER_AVAILABLE:
-				dev->rx_info.watch_bufferfull_count = 0;
-				dev->rx_info.buffer_full = false;
-				dev->rx_info.semi_buffer_full = false;
-				dev->rx_info.mgmt_buffer_full = false;
+				status = rsi_sdio_check_buffer_status(adapter,
+								      0);
+				if (status < 0)
+					rsi_dbg(ERR_ZONE,
+						"%s: Failed to check buffer status\n",
+						__func__);
 				rsi_sdio_ack_intr(common->priv,
 						  (1 << PKT_BUFF_AVAILABLE));
 				rsi_set_event(&common->tx_thread.event);
@@ -252,7 +272,7 @@ void rsi_interrupt_handler(struct rsi_hw *adapter)
 				rsi_dbg(ISR_ZONE,
 					"%s: ==> BUFFER_AVAILABLE <==\n",
 					__func__);
-				dev->rx_info.buf_available_counter++;
+				dev->buff_status_updated = true;
 				break;
 
 			case FIRMWARE_ASSERT_IND:
@@ -286,7 +306,7 @@ void rsi_interrupt_handler(struct rsi_hw *adapter)
 					rsi_dbg(ERR_ZONE,
 						"%s: Failed to read pkt\n",
 						__func__);
-					mutex_unlock(&common->tx_rxlock);
+					mutex_unlock(&common->rx_lock);
 					return;
 				}
 				break;
@@ -301,28 +321,28 @@ void rsi_interrupt_handler(struct rsi_hw *adapter)
 			}
 			isr_status ^= BIT(isr_type - 1);
 		} while (isr_status);
-		mutex_unlock(&common->tx_rxlock);
+		mutex_unlock(&common->rx_lock);
 	} while (1);
 }
 
-/**
- * rsi_sdio_read_buffer_status_register() - This function is used to the read
- *					    buffer status register and set
- *					    relevant fields in
- *					    rsi_91x_sdiodev struct.
- * @adapter: Pointer to the driver hw structure.
- * @q_num: The Q number whose status is to be found.
- *
- * Return: status: -1 on failure or else queue full/stop is indicated.
+/* This function is used to read buffer status register and
+ * set relevant fields in rsi_91x_sdiodev struct.
  */
-int rsi_sdio_read_buffer_status_register(struct rsi_hw *adapter, u8 q_num)
+int rsi_sdio_check_buffer_status(struct rsi_hw *adapter, u8 q_num)
 {
 	struct rsi_common *common = adapter->priv;
 	struct rsi_91x_sdiodev *dev =
 		(struct rsi_91x_sdiodev *)adapter->rsi_dev;
 	u8 buf_status = 0;
 	int status = 0;
+	static int counter = 4;
+
+	if (!dev->buff_status_updated && counter) {
+		counter--;
+		goto out;
+	}
 
+	dev->buff_status_updated = false;
 	status = rsi_sdio_read_register(common->priv,
 					RSI_DEVICE_BUFFER_STATUS_REGISTER,
 					&buf_status);
@@ -357,10 +377,16 @@ int rsi_sdio_read_buffer_status_register(struct rsi_hw *adapter, u8 q_num)
 		dev->rx_info.semi_buffer_full = false;
 	}
 
+	if (dev->rx_info.mgmt_buffer_full || dev->rx_info.buf_full_counter)
+		counter = 1;
+	else
+		counter = 4;
+
+out:
 	if ((q_num == MGMT_SOFT_Q) && (dev->rx_info.mgmt_buffer_full))
 		return QUEUE_FULL;
 
-	if (dev->rx_info.buffer_full)
+	if ((q_num < MGMT_SOFT_Q) && (dev->rx_info.buffer_full))
 		return QUEUE_FULL;
 
 	return QUEUE_NOT_FULL;
diff --git a/drivers/net/wireless/rsi/rsi_91x_usb.c b/drivers/net/wireless/rsi/rsi_91x_usb.c
index bcd7f454ef30..81df09dd2636 100644
--- a/drivers/net/wireless/rsi/rsi_91x_usb.c
+++ b/drivers/net/wireless/rsi/rsi_91x_usb.c
@@ -29,19 +29,24 @@
  * Return: status: 0 on success, a negative error code on failure.
  */
 static int rsi_usb_card_write(struct rsi_hw *adapter,
-			      void *buf,
+			      u8 *buf,
 			      u16 len,
 			      u8 endpoint)
 {
 	struct rsi_91x_usbdev *dev = (struct rsi_91x_usbdev *)adapter->rsi_dev;
 	int status;
-	s32 transfer;
+	u8 *seg = dev->tx_buffer;
+	int transfer;
+	int ep = dev->bulkout_endpoint_addr[endpoint - 1];
 
+	memset(seg, 0, len + RSI_USB_TX_HEAD_ROOM);
+	memcpy(seg + RSI_USB_TX_HEAD_ROOM, buf, len);
+	len += RSI_USB_TX_HEAD_ROOM;
+	transfer = len;
 	status = usb_bulk_msg(dev->usbdev,
-			      usb_sndbulkpipe(dev->usbdev,
-			      dev->bulkout_endpoint_addr[endpoint - 1]),
-			      buf,
-			      len,
+			      usb_sndbulkpipe(dev->usbdev, ep),
+			      (void *)seg,
+			      (int)len,
 			      &transfer,
 			      HZ * 5);
 
@@ -68,23 +73,19 @@ static int rsi_write_multiple(struct rsi_hw *adapter,
 			      u8 *data,
 			      u32 count)
 {
-	struct rsi_91x_usbdev *dev = (struct rsi_91x_usbdev *)adapter->rsi_dev;
-	u8 *seg = dev->tx_buffer;
+	struct rsi_91x_usbdev *dev =
+		(struct rsi_91x_usbdev *)adapter->rsi_dev;
 
-	if (dev->write_fail)
-		return 0;
+	if (!adapter)
+		return -ENODEV;
 
-	if (endpoint == MGMT_EP) {
-		memset(seg, 0, RSI_USB_TX_HEAD_ROOM);
-		memcpy(seg + RSI_USB_TX_HEAD_ROOM, data, count);
-	} else {
-		seg = ((u8 *)data - RSI_USB_TX_HEAD_ROOM);
-	}
+	if (endpoint == 0)
+		return -EINVAL;
 
-	return rsi_usb_card_write(adapter,
-				  seg,
-				  count + RSI_USB_TX_HEAD_ROOM,
-				  endpoint);
+	if (dev->write_fail)
+		return -ENETDOWN;
+
+	return rsi_usb_card_write(adapter, data, count, endpoint);
 }
 
 /**
@@ -161,10 +162,13 @@ static int rsi_usb_reg_read(struct usb_device *usbdev,
 	u8 *buf;
 	int status = -ENOMEM;
 
-	buf  = kmalloc(0x04, GFP_KERNEL);
+	buf  = kmalloc(RSI_USB_CTRL_BUF_SIZE, GFP_KERNEL);
 	if (!buf)
 		return status;
 
+	if (len > RSI_USB_CTRL_BUF_SIZE)
+		return -EINVAL;
+
 	status = usb_control_msg(usbdev,
 				 usb_rcvctrlpipe(usbdev, 0),
 				 USB_VENDOR_REGISTER_READ,
@@ -203,10 +207,13 @@ static int rsi_usb_reg_write(struct usb_device *usbdev,
 	u8 *usb_reg_buf;
 	int status = -ENOMEM;
 
-	usb_reg_buf  = kmalloc(0x04, GFP_KERNEL);
+	usb_reg_buf  = kmalloc(RSI_USB_CTRL_BUF_SIZE, GFP_KERNEL);
 	if (!usb_reg_buf)
 		return status;
 
+	if (len > RSI_USB_CTRL_BUF_SIZE)
+		return -EINVAL;
+
 	usb_reg_buf[0] = (value & 0x00ff);
 	usb_reg_buf[1] = (value & 0xff00) >> 8;
 	usb_reg_buf[2] = 0x0;
@@ -380,10 +387,11 @@ static int rsi_usb_host_intf_write_pkt(struct rsi_hw *adapter,
 				       u8 *pkt,
 				       u32 len)
 {
-	u32 queueno = ((pkt[1] >> 4) & 0xf);
+	u32 queueno = ((pkt[1] >> 4) & 0x7);
 	u8 endpoint;
 
-	endpoint = ((queueno == RSI_WIFI_MGMT_Q) ? MGMT_EP : DATA_EP);
+	endpoint = ((queueno == RSI_WIFI_MGMT_Q || queueno == RSI_WIFI_DATA_Q ||
+		     queueno == RSI_COEX_Q) ? WLAN_EP : BT_EP);
 
 	return rsi_write_multiple(adapter,
 				  endpoint,
@@ -396,8 +404,15 @@ static int rsi_usb_master_reg_read(struct rsi_hw *adapter, u32 reg,
 {
 	struct usb_device *usbdev =
 		((struct rsi_91x_usbdev *)adapter->rsi_dev)->usbdev;
+	u16 temp;
+	int ret;
+
+	ret = rsi_usb_reg_read(usbdev, reg, &temp, len);
+	if (ret < 0)
+		return ret;
+	*value = temp;
 
-	return rsi_usb_reg_read(usbdev, reg, (u16 *)value, len);
+	return 0;
 }
 
 static int rsi_usb_master_reg_write(struct rsi_hw *adapter,
@@ -424,7 +439,6 @@ static int rsi_usb_load_data_master_write(struct rsi_hw *adapter,
 	rsi_dbg(INFO_ZONE, "num_blocks: %d\n", num_blocks);
 
 	for (cur_indx = 0, i = 0; i < num_blocks; i++, cur_indx += block_size) {
-		memset(temp_buf, 0, block_size);
 		memcpy(temp_buf, ta_firmware + cur_indx, block_size);
 		status = rsi_usb_write_register_multiple(adapter, base_address,
 							 (u8 *)(temp_buf),
@@ -558,6 +572,77 @@ fail_tx:
 	return status;
 }
 
+static int usb_ulp_read_write(struct rsi_hw *adapter, u16 addr, u32 data,
+			      u16 len_in_bits)
+{
+	int ret;
+
+	ret = rsi_usb_master_reg_write
+			(adapter, RSI_GSPI_DATA_REG1,
+			 ((addr << 6) | ((data >> 16) & 0xffff)), 2);
+	if (ret < 0)
+		return ret;
+
+	ret = rsi_usb_master_reg_write(adapter, RSI_GSPI_DATA_REG0,
+				       (data & 0xffff), 2);
+	if (ret < 0)
+		return ret;
+
+	/* Initializing GSPI for ULP read/writes */
+	rsi_usb_master_reg_write(adapter, RSI_GSPI_CTRL_REG0,
+				 RSI_GSPI_CTRL_REG0_VALUE, 2);
+
+	ret = rsi_usb_master_reg_write(adapter, RSI_GSPI_CTRL_REG1,
+				       ((len_in_bits - 1) | RSI_GSPI_TRIG), 2);
+	if (ret < 0)
+		return ret;
+
+	msleep(20);
+
+	return 0;
+}
+
+static int rsi_reset_card(struct rsi_hw *adapter)
+{
+	int ret;
+
+	rsi_dbg(INFO_ZONE, "Resetting Card...\n");
+	rsi_usb_master_reg_write(adapter, RSI_TA_HOLD_REG, 0xE, 4);
+
+	/* This msleep will ensure Thread-Arch processor to go to hold
+	 * and any pending dma transfers to rf in device to finish.
+	 */
+	msleep(100);
+
+	ret = usb_ulp_read_write(adapter, RSI_WATCH_DOG_TIMER_1,
+				 RSI_ULP_WRITE_2, 32);
+	if (ret < 0)
+		goto fail;
+	ret = usb_ulp_read_write(adapter, RSI_WATCH_DOG_TIMER_2,
+				 RSI_ULP_WRITE_0, 32);
+	if (ret < 0)
+		goto fail;
+	ret = usb_ulp_read_write(adapter, RSI_WATCH_DOG_DELAY_TIMER_1,
+				 RSI_ULP_WRITE_50, 32);
+	if (ret < 0)
+		goto fail;
+	ret = usb_ulp_read_write(adapter, RSI_WATCH_DOG_DELAY_TIMER_2,
+				 RSI_ULP_WRITE_0, 32);
+	if (ret < 0)
+		goto fail;
+	ret = usb_ulp_read_write(adapter, RSI_WATCH_DOG_TIMER_ENABLE,
+				 RSI_ULP_TIMER_ENABLE, 32);
+	if (ret < 0)
+		goto fail;
+
+	rsi_dbg(INFO_ZONE, "Reset card done\n");
+	return ret;
+
+fail:
+	rsi_dbg(ERR_ZONE, "Reset card failed\n");
+	return ret;
+}
+
 /**
  * rsi_probe() - This function is called by kernel when the driver provided
  *		 Vendor and device IDs are matched. All the initialization
@@ -641,6 +726,7 @@ static void rsi_disconnect(struct usb_interface *pfunction)
 		return;
 
 	rsi_mac80211_detach(adapter);
+	rsi_reset_card(adapter);
 	rsi_deinit_usb_interface(adapter);
 	rsi_91x_deinit(adapter);
 
diff --git a/drivers/net/wireless/rsi/rsi_91x_usb_ops.c b/drivers/net/wireless/rsi/rsi_91x_usb_ops.c
index d3e0a07604a6..465692b3c351 100644
--- a/drivers/net/wireless/rsi/rsi_91x_usb_ops.c
+++ b/drivers/net/wireless/rsi/rsi_91x_usb_ops.c
@@ -37,14 +37,14 @@ void rsi_usb_rx_thread(struct rsi_common *common)
 		if (atomic_read(&dev->rx_thread.thread_done))
 			goto out;
 
-		mutex_lock(&common->tx_rxlock);
+		mutex_lock(&common->rx_lock);
 		status = rsi_read_pkt(common, 0);
 		if (status) {
 			rsi_dbg(ERR_ZONE, "%s: Failed To read data", __func__);
-			mutex_unlock(&common->tx_rxlock);
+			mutex_unlock(&common->rx_lock);
 			return;
 		}
-		mutex_unlock(&common->tx_rxlock);
+		mutex_unlock(&common->rx_lock);
 		rsi_reset_event(&dev->rx_thread.event);
 		if (adapter->rx_urb_submit(adapter)) {
 			rsi_dbg(ERR_ZONE,
diff --git a/drivers/net/wireless/rsi/rsi_common.h b/drivers/net/wireless/rsi/rsi_common.h
index 44349696f5de..e579d694d13c 100644
--- a/drivers/net/wireless/rsi/rsi_common.h
+++ b/drivers/net/wireless/rsi/rsi_common.h
@@ -83,4 +83,5 @@ u16 rsi_get_connected_channel(struct rsi_hw *adapter);
 struct rsi_hw *rsi_91x_init(void);
 void rsi_91x_deinit(struct rsi_hw *adapter);
 int rsi_read_pkt(struct rsi_common *common, s32 rcv_pkt_len);
+struct rsi_sta *rsi_find_sta(struct rsi_common *common, u8 *mac_addr);
 #endif
diff --git a/drivers/net/wireless/rsi/rsi_hal.h b/drivers/net/wireless/rsi/rsi_hal.h
index 902dc540849c..7c145053da6d 100644
--- a/drivers/net/wireless/rsi/rsi_hal.h
+++ b/drivers/net/wireless/rsi/rsi_hal.h
@@ -52,6 +52,39 @@
 #define FW_LOADING_SUCCESSFUL		'S'
 #define LOADING_INITIATED		'1'
 
+#define RSI_ULP_RESET_REG		0x161
+#define RSI_WATCH_DOG_TIMER_1		0x16c
+#define RSI_WATCH_DOG_TIMER_2		0x16d
+#define RSI_WATCH_DOG_DELAY_TIMER_1		0x16e
+#define RSI_WATCH_DOG_DELAY_TIMER_2		0x16f
+#define RSI_WATCH_DOG_TIMER_ENABLE		0x170
+
+#define RSI_ULP_WRITE_0			00
+#define RSI_ULP_WRITE_2			02
+#define RSI_ULP_WRITE_50		50
+
+#define RSI_RESTART_WDT			BIT(11)
+#define RSI_BYPASS_ULP_ON_WDT		BIT(1)
+
+#define RSI_ULP_TIMER_ENABLE		((0xaa000) | RSI_RESTART_WDT |	\
+					 RSI_BYPASS_ULP_ON_WDT)
+#define RSI_RF_SPI_PROG_REG_BASE_ADDR	0x40080000
+
+#define RSI_GSPI_CTRL_REG0		(RSI_RF_SPI_PROG_REG_BASE_ADDR)
+#define RSI_GSPI_CTRL_REG1		(RSI_RF_SPI_PROG_REG_BASE_ADDR + 0x2)
+#define RSI_GSPI_DATA_REG0		(RSI_RF_SPI_PROG_REG_BASE_ADDR + 0x4)
+#define RSI_GSPI_DATA_REG1		(RSI_RF_SPI_PROG_REG_BASE_ADDR + 0x6)
+#define RSI_GSPI_DATA_REG2		(RSI_RF_SPI_PROG_REG_BASE_ADDR + 0x8)
+
+#define RSI_GSPI_CTRL_REG0_VALUE		0x340
+
+#define RSI_GSPI_DMA_MODE			BIT(13)
+
+#define RSI_GSPI_2_ULP			BIT(12)
+#define RSI_GSPI_TRIG			BIT(7)
+#define RSI_GSPI_READ			BIT(6)
+#define RSI_GSPI_RF_SPI_ACTIVE		BIT(8)
+
 /* Boot loader commands */
 #define SEND_RPS_FILE			'2'
 
@@ -66,6 +99,8 @@
 #define RSI_DEV_OPMODE_WIFI_ALONE	1
 #define RSI_DEV_COEX_MODE_WIFI_ALONE	1
 
+#define BBP_INFO_40MHZ 0x6
+
 struct bl_header {
 	__le32 flags;
 	__le32 image_no;
@@ -79,6 +114,37 @@ struct ta_metadata {
 	unsigned int address;
 };
 
+struct rsi_mgmt_desc {
+	__le16 len_qno;
+	u8 frame_type;
+	u8 misc_flags;
+	u8 xtend_desc_size;
+	u8 header_len;
+	__le16 frame_info;
+	u8 rate_info;
+	u8 reserved1;
+	__le16 bbp_info;
+	__le16 seq_ctrl;
+	u8 reserved2;
+	u8 sta_id;
+} __packed;
+
+struct rsi_data_desc {
+	__le16 len_qno;
+	u8 cfm_frame_type;
+	u8 misc_flags;
+	u8 xtend_desc_size;
+	u8 header_len;
+	__le16 frame_info;
+	__le16 rate_info;
+	__le16 bbp_info;
+	__le16 mac_flags;
+	u8 qid_tid;
+	u8 sta_id;
+} __packed;
+
 int rsi_hal_device_init(struct rsi_hw *adapter);
+int rsi_prepare_beacon(struct rsi_common *common, struct sk_buff *skb);
+int rsi_send_pkt_to_bus(struct rsi_common *common, struct sk_buff *skb);
 
 #endif
diff --git a/drivers/net/wireless/rsi/rsi_main.h b/drivers/net/wireless/rsi/rsi_main.h
index f3985250b593..2c18dde633ea 100644
--- a/drivers/net/wireless/rsi/rsi_main.h
+++ b/drivers/net/wireless/rsi/rsi_main.h
@@ -21,6 +21,17 @@
 #include <linux/skbuff.h>
 #include <net/mac80211.h>
 
+struct rsi_sta {
+	struct ieee80211_sta *sta;
+	s16 sta_id;
+	u16 seq_start[IEEE80211_NUM_TIDS];
+	bool start_tx_aggr[IEEE80211_NUM_TIDS];
+};
+
+struct rsi_hw;
+
+#include "rsi_ps.h"
+
 #define ERR_ZONE                        BIT(0)  /* For Error Msgs             */
 #define INFO_ZONE                       BIT(1)  /* For General Status Msgs    */
 #define INIT_ZONE                       BIT(2)  /* For Driver Init Seq Msgs   */
@@ -37,10 +48,13 @@ enum RSI_FSM_STATES {
 	FSM_COMMON_DEV_PARAMS_SENT,
 	FSM_BOOT_PARAMS_SENT,
 	FSM_EEPROM_READ_MAC_ADDR,
+	FSM_EEPROM_READ_RF_TYPE,
 	FSM_RESET_MAC_SENT,
 	FSM_RADIO_CAPS_SENT,
 	FSM_BB_RF_PROG_SENT,
-	FSM_MAC_INIT_DONE
+	FSM_MAC_INIT_DONE,
+
+	NUM_FSM_STATES
 };
 
 extern u32 rsi_zone_enabled;
@@ -51,18 +65,24 @@ extern __printf(2, 3) void rsi_dbg(u32 zone, const char *fmt, ...);
 #define IEEE80211_ADDR_LEN              6
 #define FRAME_DESC_SZ                   16
 #define MIN_802_11_HDR_LEN              24
+#define RSI_DEF_KEEPALIVE               90
 
 #define DATA_QUEUE_WATER_MARK           400
 #define MIN_DATA_QUEUE_WATER_MARK       300
 #define MULTICAST_WATER_MARK            200
 #define MAC_80211_HDR_FRAME_CONTROL     0
 #define WME_NUM_AC                      4
-#define NUM_SOFT_QUEUES                 5
-#define MAX_HW_QUEUES                   8
+#define NUM_SOFT_QUEUES                 6
+#define MAX_HW_QUEUES                   12
 #define INVALID_QUEUE                   0xff
 #define MAX_CONTINUOUS_VO_PKTS          8
 #define MAX_CONTINUOUS_VI_PKTS          4
 
+/* Hardware queue info */
+#define BROADCAST_HW_Q			9
+#define MGMT_HW_Q			10
+#define BEACON_HW_Q			11
+
 /* Queue information */
 #define RSI_COEX_Q			0x0
 #define RSI_WIFI_MGMT_Q                 0x4
@@ -70,6 +90,7 @@ extern __printf(2, 3) void rsi_dbg(u32 zone, const char *fmt, ...);
 #define IEEE80211_MGMT_FRAME            0x00
 #define IEEE80211_CTL_FRAME             0x04
 
+#define RSI_MAX_ASSOC_STAS		32
 #define IEEE80211_QOS_TID               0x0f
 #define IEEE80211_NONQOS_TID            16
 
@@ -102,6 +123,7 @@ struct skb_info {
 	u16 channel;
 	s8 tid;
 	s8 sta_id;
+	u8 internal_hdr_size;
 };
 
 enum edca_queue {
@@ -109,7 +131,8 @@ enum edca_queue {
 	BE_Q,
 	VI_Q,
 	VO_Q,
-	MGMT_SOFT_Q
+	MGMT_SOFT_Q,
+	MGMT_BEACON_Q
 };
 
 struct security_info {
@@ -126,8 +149,8 @@ struct wmm_qinfo {
 };
 
 struct transmit_q_stats {
-	u32 total_tx_pkt_send[NUM_EDCA_QUEUES + 1];
-	u32 total_tx_pkt_freed[NUM_EDCA_QUEUES + 1];
+	u32 total_tx_pkt_send[NUM_EDCA_QUEUES + 2];
+	u32 total_tx_pkt_freed[NUM_EDCA_QUEUES + 2];
 };
 
 struct vif_priv {
@@ -155,7 +178,18 @@ struct cqm_info {
 	u32 rssi_hyst;
 };
 
-struct rsi_hw;
+struct xtended_desc {
+	u8 confirm_frame_type;
+	u8 retry_cnt;
+	u16 reserved;
+};
+
+enum rsi_dfs_regions {
+	RSI_REGION_FCC = 0,
+	RSI_REGION_ETSI,
+	RSI_REGION_TELEC,
+	RSI_REGION_WORLD
+};
 
 struct rsi_common {
 	struct rsi_hw *priv;
@@ -166,15 +200,18 @@ struct rsi_common {
 	struct version_info fw_ver;
 
 	struct rsi_thread tx_thread;
-	struct sk_buff_head tx_queue[NUM_EDCA_QUEUES + 1];
+	struct sk_buff_head tx_queue[NUM_EDCA_QUEUES + 2];
 	/* Mutex declaration */
 	struct mutex mutex;
-	/* Mutex used between tx/rx threads */
-	struct mutex tx_rxlock;
+	/* Mutex used for tx thread */
+	struct mutex tx_lock;
+	/* Mutex used for rx thread */
+	struct mutex rx_lock;
 	u8 endpoint;
 
 	/* Channel/band related */
 	u8 band;
+	u8 num_supp_bands;
 	u8 channel_width;
 
 	u16 rts_threshold;
@@ -216,11 +253,23 @@ struct rsi_common {
 	u16 oper_mode;
 	u8 lp_ps_handshake_mode;
 	u8 ulp_ps_handshake_mode;
+	u8 uapsd_bitmap;
 	u8 rf_power_val;
 	u8 wlan_rf_power_mode;
 	u8 obm_ant_sel_val;
 	int tx_power;
 	u8 ant_in_use;
+
+	u16 beacon_interval;
+	u8 dtim_cnt;
+
+	/* AP mode parameters */
+	u8 beacon_enabled;
+	u16 beacon_cnt;
+	struct rsi_sta stations[RSI_MAX_ASSOC_STAS + 1];
+	int num_stations;
+	int max_stations;
+	struct ieee80211_key_conf *key;
 };
 
 enum host_intf {
@@ -228,6 +277,19 @@ enum host_intf {
 	RSI_HOST_INTF_USB
 };
 
+struct eepromrw_info {
+	u32 offset;
+	u32 length;
+	u8  write;
+	u16 eeprom_erase;
+	u8 data[480];
+};
+
+struct eeprom_read {
+	u16 length;
+	u16 off_set;
+};
+
 struct rsi_hw {
 	struct rsi_common *priv;
 	u8 device_model;
@@ -241,6 +303,9 @@ struct rsi_hw {
 
 	enum host_intf rsi_host_intf;
 	u16 block_size;
+	enum ps_state ps_state;
+	struct rsi_ps_info ps_info;
+	spinlock_t ps_lock; /*To protect power save config*/
 	u32 usb_buffer_status_reg;
 #ifdef CONFIG_RSI_DEBUGFS
 	struct rsi_debugfs *dfsentry;
@@ -250,7 +315,10 @@ struct rsi_hw {
 	struct timer_list bl_cmd_timer;
 	bool blcmd_timer_expired;
 	u32 flash_capacity;
+	struct eepromrw_info eeprom;
+	u32 interrupt_status;
 	u8 dfs_region;
+	char country[2];
 	void *rsi_dev;
 	struct rsi_host_intf_ops *host_intf_ops;
 	int (*check_hw_queue_status)(struct rsi_hw *adapter, u8 q_num);
diff --git a/drivers/net/wireless/rsi/rsi_mgmt.h b/drivers/net/wireless/rsi/rsi_mgmt.h
index dcb6db728cbd..c6e1fa669a27 100644
--- a/drivers/net/wireless/rsi/rsi_mgmt.h
+++ b/drivers/net/wireless/rsi/rsi_mgmt.h
@@ -43,11 +43,13 @@
 #define WLAN_HOST_MODE_LEN              0x04
 #define WLAN_FW_VERSION_LEN             0x08
 #define MAGIC_WORD                      0x5A
+#define WLAN_EEPROM_RFTYPE_ADDR		424
 
 /* Receive Frame Types */
 #define TA_CONFIRM_TYPE                 0x01
 #define RX_DOT11_MGMT                   0x02
 #define TX_STATUS_IND                   0x04
+#define BEACON_EVENT_IND		0x08
 #define PROBEREQ_CONFIRM                2
 #define CARD_READY_IND                  0x00
 
@@ -61,8 +63,20 @@
 #define BBP_REG_WRITE                   0
 #define RF_RESET_ENABLE                 BIT(3)
 #define RATE_INFO_ENABLE                BIT(0)
+#define MORE_DATA_PRESENT		BIT(1)
 #define RSI_BROADCAST_PKT               BIT(9)
-
+#define RSI_DESC_REQUIRE_CFM_TO_HOST	BIT(2)
+#define RSI_ADD_DELTA_TSF_VAP_ID	BIT(3)
+#define RSI_FETCH_RETRY_CNT_FRM_HST	BIT(4)
+#define RSI_QOS_ENABLE			BIT(12)
+#define RSI_REKEY_PURPOSE		BIT(13)
+#define RSI_ENCRYPT_PKT			BIT(15)
+#define RSI_SET_PS_ENABLE		BIT(12)
+
+#define RSI_CMDDESC_40MHZ		BIT(4)
+#define RSI_CMDDESC_UPPER_20_ENABLE	BIT(5)
+#define RSI_CMDDESC_LOWER_20_ENABLE	BIT(6)
+#define RSI_CMDDESC_FULL_40_ENABLE	(BIT(5) | BIT(6))
 #define UPPER_20_ENABLE                 (0x2 << 12)
 #define LOWER_20_ENABLE                 (0x4 << 12)
 #define FULL40M_ENABLE                  0x6
@@ -120,6 +134,7 @@
 #define RSI_RATE_MCS6                   0x106
 #define RSI_RATE_MCS7                   0x107
 #define RSI_RATE_MCS7_SG                0x307
+#define RSI_RATE_AUTO			0xffff
 
 #define BW_20MHZ                        0
 #define BW_40MHZ                        1
@@ -143,6 +158,8 @@
 
 #define ANTENNA_SEL_INT			0x02 /* RF_OUT_2 / Integerated */
 #define ANTENNA_SEL_UFL			0x03 /* RF_OUT_1 / U.FL */
+#define ANTENNA_MASK_VALUE		0x00ff
+#define ANTENNA_SEL_TYPE		1
 
 /* Rx filter word definitions */
 #define PROMISCOUS_MODE			BIT(0)
@@ -153,9 +170,38 @@
 #define ALLOW_CONN_PEER_MGMT_WHILE_BUF_FULL BIT(5)
 #define DISALLOW_BROADCAST_DATA		BIT(6)
 
+#define RSI_MPDU_DENSITY		0x8
+#define RSI_CHAN_RADAR			BIT(7)
+#define RSI_BEACON_INTERVAL		200
+#define RSI_DTIM_COUNT			2
+
+#define RSI_PS_DISABLE_IND		BIT(15)
+#define RSI_PS_ENABLE			1
+#define RSI_PS_DISABLE			0
+#define RSI_DEEP_SLEEP			1
+#define RSI_CONNECTED_SLEEP		2
+#define RSI_SLEEP_REQUEST		1
+#define RSI_WAKEUP_REQUEST		2
+
+#define RSI_IEEE80211_UAPSD_QUEUES \
+	(IEEE80211_WMM_IE_STA_QOSINFO_AC_VO | \
+	 IEEE80211_WMM_IE_STA_QOSINFO_AC_VI | \
+	 IEEE80211_WMM_IE_STA_QOSINFO_AC_BE | \
+	 IEEE80211_WMM_IE_STA_QOSINFO_AC_BK)
+
+#define RSI_DATA_DESC_MAC_BBP_INFO	BIT(0)
+#define RSI_DATA_DESC_NO_ACK_IND	BIT(9)
+#define RSI_DATA_DESC_QOS_EN		BIT(12)
+#define RSI_DATA_DESC_NORMAL_FRAME	0x00
+#define RSI_DATA_DESC_DTIM_BEACON_GATED_FRAME	BIT(10)
+#define RSI_DATA_DESC_BEACON_FRAME	BIT(11)
+#define RSI_DATA_DESC_DTIM_BEACON	(BIT(10) | BIT(11))
+#define RSI_DATA_DESC_INSERT_TSF	BIT(15)
+#define RSI_DATA_DESC_INSERT_SEQ_NO	BIT(2)
+
 enum opmode {
-	STA_OPMODE = 1,
-	AP_OPMODE = 2
+	AP_OPMODE = 0,
+	STA_OPMODE,
 };
 
 enum vap_status {
@@ -164,6 +210,10 @@ enum vap_status {
 	VAP_UPDATE = 3
 };
 
+enum peer_type {
+	PEER_TYPE_AP,
+	PEER_TYPE_STA,
+};
 extern struct ieee80211_rate rsi_rates[12];
 extern const u16 rsi_mcsrates[8];
 
@@ -192,7 +242,7 @@ enum cmd_frame_type {
 	AUTO_RATE_IND,
 	BOOTUP_PARAMS_REQUEST,
 	VAP_CAPABILITIES,
-	EEPROM_READ_TYPE ,
+	EEPROM_READ,
 	EEPROM_WRITE,
 	GPIO_PIN_CONFIG ,
 	SET_RX_FILTER,
@@ -205,6 +255,7 @@ enum cmd_frame_type {
 	CW_MODE_REQ,
 	PER_CMD_PKT,
 	ANT_SEL_FRAME = 0x20,
+	VAP_DYNAMIC_UPDATE = 0x27,
 	COMMON_DEV_CONFIG = 0x28,
 	RADIO_PARAMS_UPDATE = 0x29
 };
@@ -213,13 +264,52 @@ struct rsi_mac_frame {
 	__le16 desc_word[8];
 } __packed;
 
+#define PWR_SAVE_WAKEUP_IND		BIT(0)
+#define TCP_CHECK_SUM_OFFLOAD		BIT(1)
+#define CONFIRM_REQUIRED_TO_HOST	BIT(2)
+#define ADD_DELTA_TSF			BIT(3)
+#define FETCH_RETRY_CNT_FROM_HOST_DESC	BIT(4)
+#define EOSP_INDICATION			BIT(5)
+#define REQUIRE_TSF_SYNC_CONFIRM	BIT(6)
+#define ENCAP_MGMT_PKT			BIT(7)
+#define DESC_IMMEDIATE_WAKEUP		BIT(15)
+
+struct rsi_cmd_desc_dword0 {
+	__le16 len_qno;
+	u8 frame_type;
+	u8 misc_flags;
+};
+
+struct rsi_cmd_desc_dword1 {
+	u8 xtend_desc_size;
+	u8 reserved1;
+	__le16 reserved2;
+};
+
+struct rsi_cmd_desc_dword2 {
+	__le32 pkt_info; /* Packet specific data */
+};
+
+struct rsi_cmd_desc_dword3 {
+	__le16 token;
+	u8 qid_tid;
+	u8 sta_id;
+};
+
+struct rsi_cmd_desc {
+	struct rsi_cmd_desc_dword0 desc_dword0;
+	struct rsi_cmd_desc_dword1 desc_dword1;
+	struct rsi_cmd_desc_dword2 desc_dword2;
+	struct rsi_cmd_desc_dword3 desc_dword3;
+};
+
 struct rsi_boot_params {
 	__le16 desc_word[8];
 	struct bootup_params bootup_params;
 } __packed;
 
 struct rsi_peer_notify {
-	__le16 desc_word[8];
+	struct rsi_cmd_desc desc;
 	u8 mac_addr[6];
 	__le16 command;
 	__le16 mpdu_density;
@@ -227,31 +317,116 @@ struct rsi_peer_notify {
 	__le32 sta_flags;
 } __packed;
 
+/* Aggregation params flags */
+#define RSI_AGGR_PARAMS_TID_MASK	0xf
+#define RSI_AGGR_PARAMS_START		BIT(4)
+#define RSI_AGGR_PARAMS_RX_AGGR		BIT(5)
+struct rsi_aggr_params {
+	struct rsi_cmd_desc_dword0 desc_dword0;
+	struct rsi_cmd_desc_dword0 desc_dword1;
+	__le16 seq_start;
+	__le16 baw_size;
+	__le16 token;
+	u8 aggr_params;
+	u8 peer_id;
+} __packed;
+
+struct rsi_bb_rf_prog {
+	struct rsi_cmd_desc_dword0 desc_dword0;
+	__le16 reserved1;
+	u8 rf_power_mode;
+	u8 reserved2;
+	u8 endpoint;
+	u8 reserved3;
+	__le16 reserved4;
+	__le16 reserved5;
+	__le16 flags;
+} __packed;
+
+struct rsi_chan_config {
+	struct rsi_cmd_desc_dword0 desc_dword0;
+	struct rsi_cmd_desc_dword1 desc_dword1;
+	u8 channel_number;
+	u8 antenna_gain_offset_2g;
+	u8 antenna_gain_offset_5g;
+	u8 channel_width;
+	__le16 tx_power;
+	u8 region_rftype;
+	u8 flags;
+} __packed;
+
 struct rsi_vap_caps {
-	__le16 desc_word[8];
+	struct rsi_cmd_desc_dword0 desc_dword0;
+	u8 reserved1;
+	u8 status;
+	__le16 reserved2;
+	u8 vif_type;
+	u8 channel_bw;
+	__le16 antenna_info;
+	u8 radioid_macid;
+	u8 vap_id;
+	__le16 reserved3;
 	u8 mac_addr[6];
 	__le16 keep_alive_period;
 	u8 bssid[6];
-	__le16 reserved;
+	__le16 reserved4;
 	__le32 flags;
 	__le16 frag_threshold;
 	__le16 rts_threshold;
 	__le32 default_mgmt_rate;
-	__le32 default_ctrl_rate;
+	__le16 default_ctrl_rate;
+	__le16 ctrl_rate_flags;
 	__le32 default_data_rate;
 	__le16 beacon_interval;
 	__le16 dtim_period;
+	__le16 beacon_miss_threshold;
 } __packed;
 
+struct rsi_ant_sel_frame {
+	struct rsi_cmd_desc_dword0 desc_dword0;
+	u8 reserved;
+	u8 sub_frame_type;
+	__le16 ant_value;
+	__le32 reserved1;
+	__le32 reserved2;
+} __packed;
+
+struct rsi_dynamic_s {
+	struct rsi_cmd_desc_dword0 desc_dword0;
+	struct rsi_cmd_desc_dword1 desc_dword1;
+	struct rsi_cmd_desc_dword2 desc_dword2;
+	struct rsi_cmd_desc_dword3 desc_dword3;
+	struct framebody {
+		__le16 data_rate;
+		__le16 mgmt_rate;
+		__le16 keep_alive_period;
+	} frame_body;
+} __packed;
+
+/* Key descriptor flags */
+#define RSI_KEY_TYPE_BROADCAST	BIT(1)
+#define RSI_WEP_KEY		BIT(2)
+#define RSI_WEP_KEY_104		BIT(3)
+#define RSI_CIPHER_WPA		BIT(4)
+#define RSI_CIPHER_TKIP		BIT(5)
+#define RSI_KEY_MODE_AP		BIT(7)
+#define RSI_PROTECT_DATA_FRAMES	BIT(13)
+#define RSI_KEY_ID_MASK		0xC0
+#define RSI_KEY_ID_OFFSET	14
 struct rsi_set_key {
-	__le16 desc_word[8];
+	struct rsi_cmd_desc_dword0 desc_dword0;
+	struct rsi_cmd_desc_dword1 desc_dword1;
+	__le16 key_desc;
+	__le32 bpn;
+	u8 sta_id;
+	u8 vap_id;
 	u8 key[4][32];
 	u8 tx_mic_key[8];
 	u8 rx_mic_key[8];
 } __packed;
 
 struct rsi_auto_rate {
-	__le16 desc_word[8];
+	struct rsi_cmd_desc desc;
 	__le16 failure_limit;
 	__le16 initial_boundary;
 	__le16 max_threshold_limt;
@@ -262,6 +437,19 @@ struct rsi_auto_rate {
 	__le16 supported_rates[40];
 } __packed;
 
+#define QUIET_INFO_VALID	BIT(0)
+#define QUIET_ENABLE		BIT(1)
+struct rsi_block_unblock_data {
+	struct rsi_cmd_desc_dword0 desc_dword0;
+	u8 xtend_desc_size;
+	u8 host_quiet_info;
+	__le16 reserved;
+	__le16 block_q_bitmap;
+	__le16 unblock_q_bitmap;
+	__le16 token;
+	__le16 flush_q_bitmap;
+} __packed;
+
 struct qos_params {
 	__le16 cont_win_min_q;
 	__le16 cont_win_max_q;
@@ -270,7 +458,14 @@ struct qos_params {
 } __packed;
 
 struct rsi_radio_caps {
-	__le16 desc_word[8];
+	struct rsi_cmd_desc_dword0 desc_dword0;
+	struct rsi_cmd_desc_dword0 desc_dword1;
+	u8 channel_num;
+	u8 rf_model;
+	__le16 ppe_ack_rate;
+	__le16 mode_11j;
+	u8 radio_cfg_info;
+	u8 radio_info;
 	struct qos_params qos_params[MAX_HW_QUEUES];
 	u8 num_11n_rates;
 	u8 num_11ac_rates;
@@ -353,6 +548,34 @@ struct rsi_config_vals {
 	u8 reserved2[16];
 } __packed;
 
+/* Packet info flags */
+#define RSI_EEPROM_HDR_SIZE_OFFSET		8
+#define RSI_EEPROM_HDR_SIZE_MASK		0x300
+#define RSI_EEPROM_LEN_OFFSET			20
+#define RSI_EEPROM_LEN_MASK			0xFFF00000
+
+struct rsi_eeprom_read_frame {
+	__le16 len_qno;
+	u8 pkt_type;
+	u8 misc_flags;
+	__le32 pkt_info;
+	__le32 eeprom_offset;
+	__le16 delay_ms;
+	__le16 reserved3;
+} __packed;
+
+struct rsi_request_ps {
+	struct rsi_cmd_desc desc;
+	struct ps_sleep_params ps_sleep;
+	u8 ps_mimic_support;
+	u8 ps_uapsd_acs;
+	u8 ps_uapsd_wakeup_period;
+	u8 reserved;
+	__le32 ps_listen_interval;
+	__le32 ps_dtim_interval_duration;
+	__le16 ps_num_dtim_intervals;
+} __packed;
+
 static inline u32 rsi_get_queueno(u8 *addr, u16 offset)
 {
 	return (le16_to_cpu(*(__le16 *)&addr[offset]) & 0x7000) >> 12;
@@ -385,16 +608,19 @@ static inline void rsi_set_len_qno(__le16 *addr, u16 len, u8 qno)
 
 int rsi_mgmt_pkt_recv(struct rsi_common *common, u8 *msg);
 int rsi_set_vap_capabilities(struct rsi_common *common, enum opmode mode,
-			     u8 vap_status);
+			     u8 *mac_addr, u8 vap_id, u8 vap_status);
 int rsi_send_aggregation_params_frame(struct rsi_common *common, u16 tid,
-				      u16 ssn, u8 buf_size, u8 event);
+				      u16 ssn, u8 buf_size, u8 event,
+				      u8 sta_id);
 int rsi_hal_load_key(struct rsi_common *common, u8 *data, u16 key_len,
-		     u8 key_type, u8 key_id, u32 cipher);
+		     u8 key_type, u8 key_id, u32 cipher, s16 sta_id);
 int rsi_set_channel(struct rsi_common *common,
 		    struct ieee80211_channel *channel);
+int rsi_send_vap_dynamic_update(struct rsi_common *common);
 int rsi_send_block_unblock_frame(struct rsi_common *common, bool event);
-void rsi_inform_bss_status(struct rsi_common *common, u8 status,
-			   const u8 *bssid, u8 qos_enable, u16 aid);
+void rsi_inform_bss_status(struct rsi_common *common, enum opmode opmode,
+			   u8 status, const u8 *addr, u8 qos_enable, u16 aid,
+			   struct ieee80211_sta *sta, u16 sta_id);
 void rsi_indicate_pkt_to_os(struct rsi_common *common, struct sk_buff *skb);
 int rsi_mac80211_attach(struct rsi_common *common);
 void rsi_indicate_tx_status(struct rsi_hw *common, struct sk_buff *skb,
diff --git a/drivers/net/wireless/rsi/rsi_ps.h b/drivers/net/wireless/rsi/rsi_ps.h
new file mode 100644
index 000000000000..d8475873df36
--- /dev/null
+++ b/drivers/net/wireless/rsi/rsi_ps.h
@@ -0,0 +1,64 @@
+/**
+ * Copyright (c) 2017 Redpine Signals Inc.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef __RSI_PS_H__
+#define __RSI_PS_H__
+
+#define PS_CONFIRM_INDEX	12
+#define RSI_DEF_DS_WAKEUP_PERIOD	200
+#define RSI_DEF_LISTEN_INTERVAL		200
+#define RSI_SLEEP_TYPE_LP		1
+
+enum ps_state {
+	PS_NONE = 0,
+	PS_ENABLE_REQ_SENT = 1,
+	PS_DISABLE_REQ_SENT = 2,
+	PS_ENABLED = 3
+};
+
+struct ps_sleep_params {
+	u8 enable;
+	u8 sleep_type;
+	u8 connected_sleep;
+	u8 reserved1;
+	__le16 num_bcns_per_lis_int;
+	__le16 wakeup_type;
+	__le32 sleep_duration;
+} __packed;
+
+struct rsi_ps_info {
+	u8 enabled;
+	u8 sleep_type;
+	u8 tx_threshold;
+	u8 rx_threshold;
+	u8 tx_hysterisis;
+	u8 rx_hysterisis;
+	u16 monitor_interval;
+	u32 listen_interval;
+	u16 num_bcns_per_lis_int;
+	u32 dtim_interval_duration;
+	u16 num_dtims_per_sleep;
+	u32 deep_sleep_wakeup_period;
+} __packed;
+
+char *str_psstate(enum ps_state state);
+void rsi_enable_ps(struct rsi_hw *adapter);
+void rsi_disable_ps(struct rsi_hw *adapter);
+int rsi_handle_ps_confirm(struct rsi_hw *adapter, u8 *msg);
+void rsi_default_ps_params(struct rsi_hw *hw);
+int rsi_send_ps_request(struct rsi_hw *adapter, bool enable);
+void rsi_conf_uapsd(struct rsi_hw *adapter);
+#endif
diff --git a/drivers/net/wireless/rsi/rsi_sdio.h b/drivers/net/wireless/rsi/rsi_sdio.h
index 9fb73f68282a..95e4bed57baf 100644
--- a/drivers/net/wireless/rsi/rsi_sdio.h
+++ b/drivers/net/wireless/rsi/rsi_sdio.h
@@ -41,6 +41,7 @@ enum sdio_interrupt_type {
 #define PKT_BUFF_FULL                           1
 #define PKT_MGMT_BUFF_FULL                      2
 #define MSDU_PKT_PENDING                        3
+#define RECV_NUM_BLOCKS                         4
 /* Interrupt Bit Related Macros */
 #define PKT_BUFF_AVAILABLE                      1
 #define FW_ASSERT_IND                           2
@@ -58,6 +59,7 @@ enum sdio_interrupt_type {
 #define SDIO_READ_START_LVL                     0x000FC
 #define SDIO_READ_FIFO_CTL                      0x000FD
 #define SDIO_WRITE_FIFO_CTL                     0x000FE
+#define SDIO_WAKEUP_REG				0x000FF
 #define SDIO_FUN1_INTR_CLR_REG                  0x0008
 #define SDIO_REG_HIGH_SPEED                     0x0013
 
@@ -103,7 +105,7 @@ struct receive_info {
 
 struct rsi_91x_sdiodev {
 	struct sdio_func *pfunction;
-	struct task_struct *in_sdio_litefi_irq;
+	struct task_struct *sdio_irq_task;
 	struct receive_info rx_info;
 	u32 next_read_delay;
 	u32 sdio_high_speed_enable;
@@ -112,6 +114,7 @@ struct rsi_91x_sdiodev {
 	u8 prev_desc[16];
 	u16 tx_blk_size;
 	u8 write_fail;
+	bool buff_status_updated;
 };
 
 void rsi_interrupt_handler(struct rsi_hw *adapter);
@@ -125,5 +128,5 @@ int rsi_sdio_write_register_multiple(struct rsi_hw *adapter, u32 addr,
 int rsi_sdio_master_access_msword(struct rsi_hw *adapter, u16 ms_word);
 void rsi_sdio_ack_intr(struct rsi_hw *adapter, u8 int_bit);
 int rsi_sdio_determine_event_timeout(struct rsi_hw *adapter);
-int rsi_sdio_read_buffer_status_register(struct rsi_hw *adapter, u8 q_num);
+int rsi_sdio_check_buffer_status(struct rsi_hw *adapter, u8 q_num);
 #endif
diff --git a/drivers/net/wireless/rsi/rsi_usb.h b/drivers/net/wireless/rsi/rsi_usb.h
index 59513ac61fb3..891daea2d932 100644
--- a/drivers/net/wireless/rsi/rsi_usb.h
+++ b/drivers/net/wireless/rsi/rsi_usb.h
@@ -25,6 +25,7 @@
 #define USB_INTERNAL_REG_1           0x25000
 #define RSI_USB_READY_MAGIC_NUM      0xab
 #define FW_STATUS_REG                0x41050012
+#define RSI_TA_HOLD_REG              0x22000844
 
 #define USB_VENDOR_REGISTER_READ     0x15
 #define USB_VENDOR_REGISTER_WRITE    0x16
@@ -32,10 +33,11 @@
 
 #define MAX_RX_URBS                  1
 #define MAX_BULK_EP                  8
-#define MGMT_EP                      1
-#define DATA_EP                      2
+#define WLAN_EP                      1
+#define BT_EP                        2
 
 #define RSI_USB_BUF_SIZE	     4096
+#define RSI_USB_CTRL_BUF_SIZE	     0x04
 
 struct rsi_91x_usbdev {
 	struct rsi_thread rx_thread;
diff --git a/drivers/net/wireless/ti/wl1251/main.c b/drivers/net/wireless/ti/wl1251/main.c
index 08f0477f78d9..9915d83a4a30 100644
--- a/drivers/net/wireless/ti/wl1251/main.c
+++ b/drivers/net/wireless/ti/wl1251/main.c
@@ -1571,6 +1571,7 @@ struct ieee80211_hw *wl1251_alloc_hw(void)
 
 	wl->state = WL1251_STATE_OFF;
 	mutex_init(&wl->mutex);
+	spin_lock_init(&wl->wl_lock);
 
 	wl->tx_mgmt_frm_rate = DEFAULT_HW_GEN_TX_RATE;
 	wl->tx_mgmt_frm_mod = DEFAULT_HW_GEN_MODULATION_TYPE;
diff --git a/drivers/net/wireless/ti/wlcore/main.c b/drivers/net/wireless/ti/wlcore/main.c
index 60aaa850fbd1..c346c021b999 100644
--- a/drivers/net/wireless/ti/wlcore/main.c
+++ b/drivers/net/wireless/ti/wlcore/main.c
@@ -6016,6 +6016,8 @@ static int wl1271_register_hw(struct wl1271 *wl)
 {
 	int ret;
 	u32 oui_addr = 0, nic_addr = 0;
+	struct platform_device *pdev = wl->pdev;
+	struct wlcore_platdev_data *pdev_data = dev_get_platdata(&pdev->dev);
 
 	if (wl->mac80211_registered)
 		return 0;
@@ -6040,6 +6042,27 @@ static int wl1271_register_hw(struct wl1271 *wl)
 		nic_addr = wl->fuse_nic_addr + 1;
 	}
 
+	if (oui_addr == 0xdeadbe && nic_addr == 0xef0000) {
+		wl1271_warning("Detected unconfigured mac address in nvs, derive from fuse instead.\n");
+		if (!strcmp(pdev_data->family->name, "wl18xx")) {
+			wl1271_warning("This default nvs file can be removed from the file system\n");
+		} else {
+			wl1271_warning("Your device performance is not optimized.\n");
+			wl1271_warning("Please use the calibrator tool to configure your device.\n");
+		}
+
+		if (wl->fuse_oui_addr == 0 && wl->fuse_nic_addr == 0) {
+			wl1271_warning("Fuse mac address is zero. using random mac\n");
+			/* Use TI oui and a random nic */
+			oui_addr = WLCORE_TI_OUI_ADDRESS;
+			nic_addr = get_random_int();
+		} else {
+			oui_addr = wl->fuse_oui_addr;
+			/* fuse has the BD_ADDR, the WLAN addresses are the next two */
+			nic_addr = wl->fuse_nic_addr + 1;
+		}
+	}
+
 	wl12xx_derive_mac_addresses(wl, oui_addr, nic_addr);
 
 	ret = ieee80211_register_hw(wl->hw);
diff --git a/drivers/net/wireless/ti/wlcore/sdio.c b/drivers/net/wireless/ti/wlcore/sdio.c
index 2fb38717346f..f8a1fea64e25 100644
--- a/drivers/net/wireless/ti/wlcore/sdio.c
+++ b/drivers/net/wireless/ti/wlcore/sdio.c
@@ -230,6 +230,7 @@ static const struct wilink_family_data wl128x_data = {
 static const struct wilink_family_data wl18xx_data = {
 	.name = "wl18xx",
 	.cfg_name = "ti-connectivity/wl18xx-conf.bin",
+	.nvs_name = "ti-connectivity/wl1271-nvs.bin",
 };
 
 static const struct of_device_id wlcore_sdio_of_match_table[] = {
diff --git a/drivers/net/wireless/ti/wlcore/spi.c b/drivers/net/wireless/ti/wlcore/spi.c
index fdabb9242cca..62ce54a949e9 100644
--- a/drivers/net/wireless/ti/wlcore/spi.c
+++ b/drivers/net/wireless/ti/wlcore/spi.c
@@ -92,6 +92,7 @@ static const struct wilink_family_data wl128x_data = {
 static const struct wilink_family_data wl18xx_data = {
 	.name = "wl18xx",
 	.cfg_name = "ti-connectivity/wl18xx-conf.bin",
+	.nvs_name = "ti-connectivity/wl1271-nvs.bin",
 };
 
 struct wl12xx_spi_glue {
diff --git a/drivers/net/wireless/ti/wlcore/sysfs.c b/drivers/net/wireless/ti/wlcore/sysfs.c
index a9218e5b0efc..b72e2101488b 100644
--- a/drivers/net/wireless/ti/wlcore/sysfs.c
+++ b/drivers/net/wireless/ti/wlcore/sysfs.c
@@ -138,7 +138,7 @@ static ssize_t wl1271_sysfs_read_fwlog(struct file *filp, struct kobject *kobj,
 	return len;
 }
 
-static struct bin_attribute fwlog_attr = {
+static const struct bin_attribute fwlog_attr = {
 	.attr = {.name = "fwlog", .mode = S_IRUSR},
 	.read = wl1271_sysfs_read_fwlog,
 };
diff --git a/drivers/net/wireless/ti/wlcore/wlcore.h b/drivers/net/wireless/ti/wlcore/wlcore.h
index 1827546ba807..95fbedc8ea34 100644
--- a/drivers/net/wireless/ti/wlcore/wlcore.h
+++ b/drivers/net/wireless/ti/wlcore/wlcore.h
@@ -40,6 +40,9 @@
 /* wl12xx/wl18xx maximum transmission power (in dBm) */
 #define WLCORE_MAX_TXPWR        25
 
+/* Texas Instruments pre assigned OUI */
+#define WLCORE_TI_OUI_ADDRESS 0x080028
+
 /* forward declaration */
 struct wl1271_tx_hw_descr;
 enum wl_rx_buf_align;
diff --git a/drivers/net/wireless/wl3501_cs.c b/drivers/net/wireless/wl3501_cs.c
index acec0d9ec422..da62220b9c01 100644
--- a/drivers/net/wireless/wl3501_cs.c
+++ b/drivers/net/wireless/wl3501_cs.c
@@ -965,7 +965,7 @@ static inline void wl3501_md_ind_interrupt(struct net_device *dev,
 			    &addr4, sizeof(addr4));
 	if (!(addr4[0] == 0xAA && addr4[1] == 0xAA &&
 	      addr4[2] == 0x03 && addr4[4] == 0x00)) {
-		printk(KERN_INFO "Insupported packet type!\n");
+		printk(KERN_INFO "Unsupported packet type!\n");
 		return;
 	}
 	pkt_len = sig.size + 12 - 24 - 4 - 6;
diff --git a/drivers/net/wireless/zydas/zd1201.c b/drivers/net/wireless/zydas/zd1201.c
index 7f586d76cf17..581e8577a221 100644
--- a/drivers/net/wireless/zydas/zd1201.c
+++ b/drivers/net/wireless/zydas/zd1201.c
@@ -25,7 +25,7 @@
 #include <linux/firmware.h>
 #include "zd1201.h"
 
-static struct usb_device_id zd1201_table[] = {
+static const struct usb_device_id zd1201_table[] = {
 	{USB_DEVICE(0x0586, 0x3400)}, /* Peabird Wireless USB Adapter */
 	{USB_DEVICE(0x0ace, 0x1201)}, /* ZyDAS ZD1201 Wireless USB Adapter */
 	{USB_DEVICE(0x050d, 0x6051)}, /* Belkin F5D6051 usb  adapter */
diff --git a/drivers/net/wireless/zydas/zd1211rw/zd_rf_rf2959.c b/drivers/net/wireless/zydas/zd1211rw/zd_rf_rf2959.c
index a93f657a41c7..d4e512f50945 100644
--- a/drivers/net/wireless/zydas/zd1211rw/zd_rf_rf2959.c
+++ b/drivers/net/wireless/zydas/zd1211rw/zd_rf_rf2959.c
@@ -61,7 +61,7 @@ static void dump_regwrite(u32 rw)
 
 	switch (reg) {
 	case 0:
-		PDEBUG("reg0 CFG1 ref_sel %d hybernate %d rf_vco_reg_en %d"
+		PDEBUG("reg0 CFG1 ref_sel %d hibernate %d rf_vco_reg_en %d"
 		       " if_vco_reg_en %d if_vga_en %d",
 		       bits(rw, 14, 15), bit(rw, 3), bit(rw, 2), bit(rw, 1),
 		       bit(rw, 0));
diff --git a/drivers/net/wireless/zydas/zd1211rw/zd_usb.c b/drivers/net/wireless/zydas/zd1211rw/zd_usb.c
index 01ca1d57b3d9..c30bf118c67d 100644
--- a/drivers/net/wireless/zydas/zd1211rw/zd_usb.c
+++ b/drivers/net/wireless/zydas/zd1211rw/zd_usb.c
@@ -35,7 +35,7 @@
 #include "zd_mac.h"
 #include "zd_usb.h"
 
-static struct usb_device_id usb_ids[] = {
+static const struct usb_device_id usb_ids[] = {
 	/* ZD1211 */
 	{ USB_DEVICE(0x0105, 0x145f), .driver_info = DEVICE_ZD1211 },
 	{ USB_DEVICE(0x0586, 0x3401), .driver_info = DEVICE_ZD1211 },
diff --git a/drivers/net/xen-netback/interface.c b/drivers/net/xen-netback/interface.c
index e322a862ddfe..ee8ed9da00ad 100644
--- a/drivers/net/xen-netback/interface.c
+++ b/drivers/net/xen-netback/interface.c
@@ -551,8 +551,8 @@ int xenvif_init_queue(struct xenvif_queue *queue)
 	for (i = 0; i < MAX_PENDING_REQS; i++) {
 		queue->pending_tx_info[i].callback_struct = (struct ubuf_info)
 			{ .callback = xenvif_zerocopy_callback,
-			  .ctx = NULL,
-			  .desc = i };
+			  { { .ctx = NULL,
+			      .desc = i } } };
 		queue->grant_tx_handle[i] = NETBACK_INVALID_HANDLE;
 	}
 
diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c
index 7b61adb6270c..523387e71a80 100644
--- a/drivers/net/xen-netfront.c
+++ b/drivers/net/xen-netfront.c
@@ -611,7 +611,7 @@ static int xennet_start_xmit(struct sk_buff *skb, struct net_device *dev)
 		nskb = skb_copy(skb, GFP_ATOMIC);
 		if (!nskb)
 			goto drop;
-		dev_kfree_skb_any(skb);
+		dev_consume_skb_any(skb);
 		skb = nskb;
 		page = virt_to_page(skb->data);
 		offset = offset_in_page(skb->data);
diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c
index 9a03c5871efe..f58d8e305323 100644
--- a/drivers/ntb/ntb_transport.c
+++ b/drivers/ntb/ntb_transport.c
@@ -924,10 +924,8 @@ out1:
 		ntb_free_mw(nt, i);
 
 	/* if there's an actual failure, we should just bail */
-	if (rc < 0) {
-		ntb_link_disable(ndev);
+	if (rc < 0)
 		return;
-	}
 
 out:
 	if (ntb_link_is_up(ndev, NULL, NULL) == 1)
@@ -1059,7 +1057,7 @@ static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
 	int node;
 	int rc, i;
 
-	mw_count = ntb_mw_count(ndev, PIDX);
+	mw_count = ntb_peer_mw_count(ndev);
 
 	if (!ndev->ops->mw_set_trans) {
 		dev_err(&ndev->dev, "Inbound MW based NTB API is required\n");
diff --git a/drivers/ntb/test/ntb_tool.c b/drivers/ntb/test/ntb_tool.c
index f002bf48a08d..a69815c45ce6 100644
--- a/drivers/ntb/test/ntb_tool.c
+++ b/drivers/ntb/test/ntb_tool.c
@@ -959,7 +959,7 @@ static int tool_probe(struct ntb_client *self, struct ntb_dev *ntb)
 	tc->ntb = ntb;
 	init_waitqueue_head(&tc->link_wq);
 
-	tc->mw_count = min(ntb_mw_count(tc->ntb, PIDX), MAX_MWS);
+	tc->mw_count = min(ntb_peer_mw_count(tc->ntb), MAX_MWS);
 	for (i = 0; i < tc->mw_count; i++) {
 		rc = tool_init_mw(tc, i);
 		if (rc)
diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c
index 14323faf8bd9..d5612bd1cc81 100644
--- a/drivers/nvdimm/btt.c
+++ b/drivers/nvdimm/btt.c
@@ -31,6 +31,16 @@ enum log_ent_request {
 	LOG_OLD_ENT
 };
 
+static struct device *to_dev(struct arena_info *arena)
+{
+	return &arena->nd_btt->dev;
+}
+
+static u64 adjust_initial_offset(struct nd_btt *nd_btt, u64 offset)
+{
+	return offset + nd_btt->initial_offset;
+}
+
 static int arena_read_bytes(struct arena_info *arena, resource_size_t offset,
 		void *buf, size_t n, unsigned long flags)
 {
@@ -38,7 +48,7 @@ static int arena_read_bytes(struct arena_info *arena, resource_size_t offset,
 	struct nd_namespace_common *ndns = nd_btt->ndns;
 
 	/* arena offsets may be shifted from the base of the device */
-	offset += arena->nd_btt->initial_offset;
+	offset = adjust_initial_offset(nd_btt, offset);
 	return nvdimm_read_bytes(ndns, offset, buf, n, flags);
 }
 
@@ -49,7 +59,7 @@ static int arena_write_bytes(struct arena_info *arena, resource_size_t offset,
 	struct nd_namespace_common *ndns = nd_btt->ndns;
 
 	/* arena offsets may be shifted from the base of the device */
-	offset += arena->nd_btt->initial_offset;
+	offset = adjust_initial_offset(nd_btt, offset);
 	return nvdimm_write_bytes(ndns, offset, buf, n, flags);
 }
 
@@ -62,8 +72,10 @@ static int btt_info_write(struct arena_info *arena, struct btt_sb *super)
 	 * We rely on that to make sure rw_bytes does error clearing
 	 * correctly, so make sure that is the case.
 	 */
-	WARN_ON_ONCE(!IS_ALIGNED(arena->infooff, 512));
-	WARN_ON_ONCE(!IS_ALIGNED(arena->info2off, 512));
+	dev_WARN_ONCE(to_dev(arena), !IS_ALIGNED(arena->infooff, 512),
+		"arena->infooff: %#llx is unaligned\n", arena->infooff);
+	dev_WARN_ONCE(to_dev(arena), !IS_ALIGNED(arena->info2off, 512),
+		"arena->info2off: %#llx is unaligned\n", arena->info2off);
 
 	ret = arena_write_bytes(arena, arena->info2off, super,
 			sizeof(struct btt_sb), 0);
@@ -76,7 +88,6 @@ static int btt_info_write(struct arena_info *arena, struct btt_sb *super)
 
 static int btt_info_read(struct arena_info *arena, struct btt_sb *super)
 {
-	WARN_ON(!super);
 	return arena_read_bytes(arena, arena->infooff, super,
 			sizeof(struct btt_sb), 0);
 }
@@ -92,7 +103,10 @@ static int __btt_map_write(struct arena_info *arena, u32 lba, __le32 mapping,
 {
 	u64 ns_off = arena->mapoff + (lba * MAP_ENT_SIZE);
 
-	WARN_ON(lba >= arena->external_nlba);
+	if (unlikely(lba >= arena->external_nlba))
+		dev_err_ratelimited(to_dev(arena),
+			"%s: lba %#x out of range (max: %#x)\n",
+			__func__, lba, arena->external_nlba);
 	return arena_write_bytes(arena, ns_off, &mapping, MAP_ENT_SIZE, flags);
 }
 
@@ -106,7 +120,7 @@ static int btt_map_write(struct arena_info *arena, u32 lba, u32 mapping,
 	 * This 'mapping' is supposed to be just the LBA mapping, without
 	 * any flags set, so strip the flag bits.
 	 */
-	mapping &= MAP_LBA_MASK;
+	mapping = ent_lba(mapping);
 
 	ze = (z_flag << 1) + e_flag;
 	switch (ze) {
@@ -131,7 +145,8 @@ static int btt_map_write(struct arena_info *arena, u32 lba, u32 mapping,
 		 * construed as a valid 'normal' case, but we decide not to,
 		 * to avoid confusion
 		 */
-		WARN_ONCE(1, "Invalid use of Z and E flags\n");
+		dev_err_ratelimited(to_dev(arena),
+			"Invalid use of Z and E flags\n");
 		return -EIO;
 	}
 
@@ -147,7 +162,10 @@ static int btt_map_read(struct arena_info *arena, u32 lba, u32 *mapping,
 	u32 raw_mapping, postmap, ze, z_flag, e_flag;
 	u64 ns_off = arena->mapoff + (lba * MAP_ENT_SIZE);
 
-	WARN_ON(lba >= arena->external_nlba);
+	if (unlikely(lba >= arena->external_nlba))
+		dev_err_ratelimited(to_dev(arena),
+			"%s: lba %#x out of range (max: %#x)\n",
+			__func__, lba, arena->external_nlba);
 
 	ret = arena_read_bytes(arena, ns_off, &in, MAP_ENT_SIZE, rwb_flags);
 	if (ret)
@@ -155,10 +173,10 @@ static int btt_map_read(struct arena_info *arena, u32 lba, u32 *mapping,
 
 	raw_mapping = le32_to_cpu(in);
 
-	z_flag = (raw_mapping & MAP_TRIM_MASK) >> MAP_TRIM_SHIFT;
-	e_flag = (raw_mapping & MAP_ERR_MASK) >> MAP_ERR_SHIFT;
+	z_flag = ent_z_flag(raw_mapping);
+	e_flag = ent_e_flag(raw_mapping);
 	ze = (z_flag << 1) + e_flag;
-	postmap = raw_mapping & MAP_LBA_MASK;
+	postmap = ent_lba(raw_mapping);
 
 	/* Reuse the {z,e}_flag variables for *trim and *error */
 	z_flag = 0;
@@ -195,7 +213,6 @@ static int btt_map_read(struct arena_info *arena, u32 lba, u32 *mapping,
 static int btt_log_read_pair(struct arena_info *arena, u32 lane,
 			struct log_entry *ent)
 {
-	WARN_ON(!ent);
 	return arena_read_bytes(arena,
 			arena->logoff + (2 * lane * LOG_ENT_SIZE), ent,
 			2 * LOG_ENT_SIZE, 0);
@@ -299,11 +316,6 @@ static int btt_log_get_old(struct log_entry *ent)
 	return old;
 }
 
-static struct device *to_dev(struct arena_info *arena)
-{
-	return &arena->nd_btt->dev;
-}
-
 /*
  * This function copies the desired (old/new) log entry into ent if
  * it is not NULL. It returns the sub-slot number (0 or 1)
@@ -381,7 +393,9 @@ static int btt_flog_write(struct arena_info *arena, u32 lane, u32 sub,
 	arena->freelist[lane].sub = 1 - arena->freelist[lane].sub;
 	if (++(arena->freelist[lane].seq) == 4)
 		arena->freelist[lane].seq = 1;
-	arena->freelist[lane].block = le32_to_cpu(ent->old_map);
+	if (ent_e_flag(ent->old_map))
+		arena->freelist[lane].has_err = 1;
+	arena->freelist[lane].block = le32_to_cpu(ent_lba(ent->old_map));
 
 	return ret;
 }
@@ -407,12 +421,14 @@ static int btt_map_init(struct arena_info *arena)
 	 * make sure rw_bytes does error clearing correctly, so make sure that
 	 * is the case.
 	 */
-	WARN_ON_ONCE(!IS_ALIGNED(arena->mapoff, 512));
+	dev_WARN_ONCE(to_dev(arena), !IS_ALIGNED(arena->mapoff, 512),
+		"arena->mapoff: %#llx is unaligned\n", arena->mapoff);
 
 	while (mapsize) {
 		size_t size = min(mapsize, chunk_size);
 
-		WARN_ON_ONCE(size < 512);
+		dev_WARN_ONCE(to_dev(arena), size < 512,
+			"chunk size: %#zx is unaligned\n", size);
 		ret = arena_write_bytes(arena, arena->mapoff + offset, zerobuf,
 				size, 0);
 		if (ret)
@@ -449,12 +465,14 @@ static int btt_log_init(struct arena_info *arena)
 	 * make sure rw_bytes does error clearing correctly, so make sure that
 	 * is the case.
 	 */
-	WARN_ON_ONCE(!IS_ALIGNED(arena->logoff, 512));
+	dev_WARN_ONCE(to_dev(arena), !IS_ALIGNED(arena->logoff, 512),
+		"arena->logoff: %#llx is unaligned\n", arena->logoff);
 
 	while (logsize) {
 		size_t size = min(logsize, chunk_size);
 
-		WARN_ON_ONCE(size < 512);
+		dev_WARN_ONCE(to_dev(arena), size < 512,
+			"chunk size: %#zx is unaligned\n", size);
 		ret = arena_write_bytes(arena, arena->logoff + offset, zerobuf,
 				size, 0);
 		if (ret)
@@ -480,6 +498,40 @@ static int btt_log_init(struct arena_info *arena)
 	return ret;
 }
 
+static u64 to_namespace_offset(struct arena_info *arena, u64 lba)
+{
+	return arena->dataoff + ((u64)lba * arena->internal_lbasize);
+}
+
+static int arena_clear_freelist_error(struct arena_info *arena, u32 lane)
+{
+	int ret = 0;
+
+	if (arena->freelist[lane].has_err) {
+		void *zero_page = page_address(ZERO_PAGE(0));
+		u32 lba = arena->freelist[lane].block;
+		u64 nsoff = to_namespace_offset(arena, lba);
+		unsigned long len = arena->sector_size;
+
+		mutex_lock(&arena->err_lock);
+
+		while (len) {
+			unsigned long chunk = min(len, PAGE_SIZE);
+
+			ret = arena_write_bytes(arena, nsoff, zero_page,
+				chunk, 0);
+			if (ret)
+				break;
+			len -= chunk;
+			nsoff += chunk;
+			if (len == 0)
+				arena->freelist[lane].has_err = 0;
+		}
+		mutex_unlock(&arena->err_lock);
+	}
+	return ret;
+}
+
 static int btt_freelist_init(struct arena_info *arena)
 {
 	int old, new, ret;
@@ -505,6 +557,17 @@ static int btt_freelist_init(struct arena_info *arena)
 		arena->freelist[i].seq = nd_inc_seq(le32_to_cpu(log_new.seq));
 		arena->freelist[i].block = le32_to_cpu(log_new.old_map);
 
+		/*
+		 * FIXME: if error clearing fails during init, we want to make
+		 * the BTT read-only
+		 */
+		if (ent_e_flag(log_new.old_map)) {
+			ret = arena_clear_freelist_error(arena, i);
+			if (ret)
+				dev_err_ratelimited(to_dev(arena),
+					"Unable to clear known errors\n");
+		}
+
 		/* This implies a newly created or untouched flog entry */
 		if (log_new.old_map == log_new.new_map)
 			continue;
@@ -525,7 +588,6 @@ static int btt_freelist_init(struct arena_info *arena)
 			if (ret)
 				return ret;
 		}
-
 	}
 
 	return 0;
@@ -566,6 +628,7 @@ static struct arena_info *alloc_arena(struct btt *btt, size_t size,
 	if (!arena)
 		return NULL;
 	arena->nd_btt = btt->nd_btt;
+	arena->sector_size = btt->sector_size;
 
 	if (!size)
 		return arena;
@@ -694,6 +757,7 @@ static int discover_arenas(struct btt *btt)
 		arena->external_lba_start = cur_nlba;
 		parse_arena_meta(arena, super, cur_off);
 
+		mutex_init(&arena->err_lock);
 		ret = btt_freelist_init(arena);
 		if (ret)
 			goto out;
@@ -904,11 +968,6 @@ static void unlock_map(struct arena_info *arena, u32 premap)
 	spin_unlock(&arena->map_locks[idx].lock);
 }
 
-static u64 to_namespace_offset(struct arena_info *arena, u64 lba)
-{
-	return arena->dataoff + ((u64)lba * arena->internal_lbasize);
-}
-
 static int btt_data_read(struct arena_info *arena, struct page *page,
 			unsigned int off, u32 lba, u32 len)
 {
@@ -1032,6 +1091,7 @@ static int btt_read_pg(struct btt *btt, struct bio_integrity_payload *bip,
 		 */
 		while (1) {
 			u32 new_map;
+			int new_t, new_e;
 
 			if (t_flag) {
 				zero_fill_data(page, off, cur_len);
@@ -1050,20 +1110,29 @@ static int btt_read_pg(struct btt *btt, struct bio_integrity_payload *bip,
 			 */
 			barrier();
 
-			ret = btt_map_read(arena, premap, &new_map, &t_flag,
-						&e_flag, NVDIMM_IO_ATOMIC);
+			ret = btt_map_read(arena, premap, &new_map, &new_t,
+						&new_e, NVDIMM_IO_ATOMIC);
 			if (ret)
 				goto out_rtt;
 
-			if (postmap == new_map)
+			if ((postmap == new_map) && (t_flag == new_t) &&
+					(e_flag == new_e))
 				break;
 
 			postmap = new_map;
+			t_flag = new_t;
+			e_flag = new_e;
 		}
 
 		ret = btt_data_read(arena, page, off, postmap, cur_len);
-		if (ret)
+		if (ret) {
+			int rc;
+
+			/* Media error - set the e_flag */
+			rc = btt_map_write(arena, premap, postmap, 0, 1,
+				NVDIMM_IO_ATOMIC);
 			goto out_rtt;
+		}
 
 		if (bip) {
 			ret = btt_rw_integrity(btt, bip, arena, postmap, READ);
@@ -1088,6 +1157,21 @@ static int btt_read_pg(struct btt *btt, struct bio_integrity_payload *bip,
 	return ret;
 }
 
+/*
+ * Normally, arena_{read,write}_bytes will take care of the initial offset
+ * adjustment, but in the case of btt_is_badblock, where we query is_bad_pmem,
+ * we need the final, raw namespace offset here
+ */
+static bool btt_is_badblock(struct btt *btt, struct arena_info *arena,
+		u32 postmap)
+{
+	u64 nsoff = adjust_initial_offset(arena->nd_btt,
+			to_namespace_offset(arena, postmap));
+	sector_t phys_sector = nsoff >> 9;
+
+	return is_bad_pmem(btt->phys_bb, phys_sector, arena->internal_lbasize);
+}
+
 static int btt_write_pg(struct btt *btt, struct bio_integrity_payload *bip,
 			sector_t sector, struct page *page, unsigned int off,
 			unsigned int len)
@@ -1100,7 +1184,9 @@ static int btt_write_pg(struct btt *btt, struct bio_integrity_payload *bip,
 
 	while (len) {
 		u32 cur_len;
+		int e_flag;
 
+ retry:
 		lane = nd_region_acquire_lane(btt->nd_region);
 
 		ret = lba_to_arena(btt, sector, &premap, &arena);
@@ -1113,6 +1199,21 @@ static int btt_write_pg(struct btt *btt, struct bio_integrity_payload *bip,
 			goto out_lane;
 		}
 
+		if (btt_is_badblock(btt, arena, arena->freelist[lane].block))
+			arena->freelist[lane].has_err = 1;
+
+		if (mutex_is_locked(&arena->err_lock)
+				|| arena->freelist[lane].has_err) {
+			nd_region_release_lane(btt->nd_region, lane);
+
+			ret = arena_clear_freelist_error(arena, lane);
+			if (ret)
+				return ret;
+
+			/* OK to acquire a different lane/free block */
+			goto retry;
+		}
+
 		new_postmap = arena->freelist[lane].block;
 
 		/* Wait if the new block is being read from */
@@ -1138,7 +1239,7 @@ static int btt_write_pg(struct btt *btt, struct bio_integrity_payload *bip,
 		}
 
 		lock_map(arena, premap);
-		ret = btt_map_read(arena, premap, &old_postmap, NULL, NULL,
+		ret = btt_map_read(arena, premap, &old_postmap, NULL, &e_flag,
 				NVDIMM_IO_ATOMIC);
 		if (ret)
 			goto out_map;
@@ -1146,6 +1247,8 @@ static int btt_write_pg(struct btt *btt, struct bio_integrity_payload *bip,
 			ret = -EIO;
 			goto out_map;
 		}
+		if (e_flag)
+			set_e_flag(old_postmap);
 
 		log.lba = cpu_to_le32(premap);
 		log.old_map = cpu_to_le32(old_postmap);
@@ -1156,13 +1259,20 @@ static int btt_write_pg(struct btt *btt, struct bio_integrity_payload *bip,
 		if (ret)
 			goto out_map;
 
-		ret = btt_map_write(arena, premap, new_postmap, 0, 0, 0);
+		ret = btt_map_write(arena, premap, new_postmap, 0, 0,
+			NVDIMM_IO_ATOMIC);
 		if (ret)
 			goto out_map;
 
 		unlock_map(arena, premap);
 		nd_region_release_lane(btt->nd_region, lane);
 
+		if (e_flag) {
+			ret = arena_clear_freelist_error(arena, lane);
+			if (ret)
+				return ret;
+		}
+
 		len -= cur_len;
 		off += cur_len;
 		sector += btt->sector_size >> SECTOR_SHIFT;
@@ -1211,11 +1321,13 @@ static blk_qc_t btt_make_request(struct request_queue *q, struct bio *bio)
 	bio_for_each_segment(bvec, bio, iter) {
 		unsigned int len = bvec.bv_len;
 
-		BUG_ON(len > PAGE_SIZE);
-		/* Make sure len is in multiples of sector size. */
-		/* XXX is this right? */
-		BUG_ON(len < btt->sector_size);
-		BUG_ON(len % btt->sector_size);
+		if (len > PAGE_SIZE || len < btt->sector_size ||
+				len % btt->sector_size) {
+			dev_err_ratelimited(&btt->nd_btt->dev,
+				"unaligned bio segment (len: %d)\n", len);
+			bio->bi_status = BLK_STS_IOERR;
+			break;
+		}
 
 		err = btt_do_bvec(btt, bip, bvec.bv_page, len, bvec.bv_offset,
 				  op_is_write(bio_op(bio)), iter.bi_sector);
@@ -1241,8 +1353,10 @@ static int btt_rw_page(struct block_device *bdev, sector_t sector,
 {
 	struct btt *btt = bdev->bd_disk->private_data;
 	int rc;
+	unsigned int len;
 
-	rc = btt_do_bvec(btt, NULL, page, PAGE_SIZE, 0, is_write, sector);
+	len = hpage_nr_pages(page) * PAGE_SIZE;
+	rc = btt_do_bvec(btt, NULL, page, len, 0, is_write, sector);
 	if (rc == 0)
 		page_endio(page, is_write, 0);
 
@@ -1343,6 +1457,7 @@ static struct btt *btt_init(struct nd_btt *nd_btt, unsigned long long rawsize,
 {
 	int ret;
 	struct btt *btt;
+	struct nd_namespace_io *nsio;
 	struct device *dev = &nd_btt->dev;
 
 	btt = devm_kzalloc(dev, sizeof(struct btt), GFP_KERNEL);
@@ -1356,6 +1471,8 @@ static struct btt *btt_init(struct nd_btt *nd_btt, unsigned long long rawsize,
 	INIT_LIST_HEAD(&btt->arena_list);
 	mutex_init(&btt->init_lock);
 	btt->nd_region = nd_region;
+	nsio = to_nd_namespace_io(&nd_btt->ndns->dev);
+	btt->phys_bb = &nsio->bb;
 
 	ret = discover_arenas(btt);
 	if (ret) {
@@ -1429,6 +1546,8 @@ int nvdimm_namespace_attach_btt(struct nd_namespace_common *ndns)
 	}
 
 	btt_sb = devm_kzalloc(&nd_btt->dev, sizeof(*btt_sb), GFP_KERNEL);
+	if (!btt_sb)
+		return -ENOMEM;
 
 	/*
 	 * If this returns < 0, that is ok as it just means there wasn't
diff --git a/drivers/nvdimm/btt.h b/drivers/nvdimm/btt.h
index 888e862907a0..578c2057524d 100644
--- a/drivers/nvdimm/btt.h
+++ b/drivers/nvdimm/btt.h
@@ -15,6 +15,7 @@
 #ifndef _LINUX_BTT_H
 #define _LINUX_BTT_H
 
+#include <linux/badblocks.h>
 #include <linux/types.h>
 
 #define BTT_SIG_LEN 16
@@ -38,6 +39,11 @@
 #define IB_FLAG_ERROR 0x00000001
 #define IB_FLAG_ERROR_MASK 0x00000001
 
+#define ent_lba(ent) (ent & MAP_LBA_MASK)
+#define ent_e_flag(ent) (!!(ent & MAP_ERR_MASK))
+#define ent_z_flag(ent) (!!(ent & MAP_TRIM_MASK))
+#define set_e_flag(ent) (ent |= MAP_ERR_MASK)
+
 enum btt_init_state {
 	INIT_UNCHECKED = 0,
 	INIT_NOTFOUND,
@@ -78,6 +84,7 @@ struct free_entry {
 	u32 block;
 	u8 sub;
 	u8 seq;
+	u8 has_err;
 };
 
 struct aligned_lock {
@@ -104,6 +111,7 @@ struct aligned_lock {
  *			handle incoming writes.
  * @version_major:	Metadata layout version major.
  * @version_minor:	Metadata layout version minor.
+ * @sector_size:	The Linux sector size - 512 or 4096
  * @nextoff:		Offset in bytes to the start of the next arena.
  * @infooff:		Offset in bytes to the info block of this arena.
  * @dataoff:		Offset in bytes to the data area of this arena.
@@ -131,6 +139,7 @@ struct arena_info {
 	u32 nfree;
 	u16 version_major;
 	u16 version_minor;
+	u32 sector_size;
 	/* Byte offsets to the different on-media structures */
 	u64 nextoff;
 	u64 infooff;
@@ -147,6 +156,7 @@ struct arena_info {
 	struct dentry *debugfs_dir;
 	/* Arena flags */
 	u32 flags;
+	struct mutex err_lock;
 };
 
 /**
@@ -181,6 +191,7 @@ struct btt {
 	struct mutex init_lock;
 	int init_state;
 	int num_arenas;
+	struct badblocks *phys_bb;
 };
 
 bool nd_btt_arena_is_valid(struct nd_btt *nd_btt, struct btt_sb *super);
diff --git a/drivers/nvdimm/btt_devs.c b/drivers/nvdimm/btt_devs.c
index 3e359d282f8e..d58925295aa7 100644
--- a/drivers/nvdimm/btt_devs.c
+++ b/drivers/nvdimm/btt_devs.c
@@ -61,7 +61,7 @@ static ssize_t sector_size_show(struct device *dev,
 {
 	struct nd_btt *nd_btt = to_nd_btt(dev);
 
-	return nd_sector_size_show(nd_btt->lbasize, btt_lbasize_supported, buf);
+	return nd_size_select_show(nd_btt->lbasize, btt_lbasize_supported, buf);
 }
 
 static ssize_t sector_size_store(struct device *dev,
@@ -72,7 +72,7 @@ static ssize_t sector_size_store(struct device *dev,
 
 	device_lock(dev);
 	nvdimm_bus_lock(dev);
-	rc = nd_sector_size_store(dev, buf, &nd_btt->lbasize,
+	rc = nd_size_select_store(dev, buf, &nd_btt->lbasize,
 			btt_lbasize_supported);
 	dev_dbg(dev, "%s: result: %zd wrote: %s%s", __func__,
 			rc, buf, buf[len - 1] == '\n' ? "" : "\n");
diff --git a/drivers/nvdimm/bus.c b/drivers/nvdimm/bus.c
index 937fafa1886a..baf283986a7e 100644
--- a/drivers/nvdimm/bus.c
+++ b/drivers/nvdimm/bus.c
@@ -11,6 +11,7 @@
  * General Public License for more details.
  */
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#include <linux/sched/mm.h>
 #include <linux/vmalloc.h>
 #include <linux/uaccess.h>
 #include <linux/module.h>
@@ -234,6 +235,7 @@ long nvdimm_clear_poison(struct device *dev, phys_addr_t phys,
 	struct nd_cmd_clear_error clear_err;
 	struct nd_cmd_ars_cap ars_cap;
 	u32 clear_err_unit, mask;
+	unsigned int noio_flag;
 	int cmd_rc, rc;
 
 	if (!nvdimm_bus)
@@ -250,8 +252,10 @@ long nvdimm_clear_poison(struct device *dev, phys_addr_t phys,
 	memset(&ars_cap, 0, sizeof(ars_cap));
 	ars_cap.address = phys;
 	ars_cap.length = len;
+	noio_flag = memalloc_noio_save();
 	rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, &ars_cap,
 			sizeof(ars_cap), &cmd_rc);
+	memalloc_noio_restore(noio_flag);
 	if (rc < 0)
 		return rc;
 	if (cmd_rc < 0)
@@ -266,8 +270,10 @@ long nvdimm_clear_poison(struct device *dev, phys_addr_t phys,
 	memset(&clear_err, 0, sizeof(clear_err));
 	clear_err.address = phys;
 	clear_err.length = len;
+	noio_flag = memalloc_noio_save();
 	rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_CLEAR_ERROR, &clear_err,
 			sizeof(clear_err), &cmd_rc);
+	memalloc_noio_restore(noio_flag);
 	if (rc < 0)
 		return rc;
 	if (cmd_rc < 0)
@@ -905,19 +911,20 @@ static int __nd_ioctl(struct nvdimm_bus *nvdimm_bus, struct nvdimm *nvdimm,
 		int read_only, unsigned int ioctl_cmd, unsigned long arg)
 {
 	struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
-	size_t buf_len = 0, in_len = 0, out_len = 0;
 	static char out_env[ND_CMD_MAX_ENVELOPE];
 	static char in_env[ND_CMD_MAX_ENVELOPE];
 	const struct nd_cmd_desc *desc = NULL;
 	unsigned int cmd = _IOC_NR(ioctl_cmd);
-	unsigned int func = cmd;
-	void __user *p = (void __user *) arg;
 	struct device *dev = &nvdimm_bus->dev;
-	struct nd_cmd_pkg pkg;
+	void __user *p = (void __user *) arg;
 	const char *cmd_name, *dimm_name;
+	u32 in_len = 0, out_len = 0;
+	unsigned int func = cmd;
 	unsigned long cmd_mask;
-	void *buf;
+	struct nd_cmd_pkg pkg;
 	int rc, i, cmd_rc;
+	u64 buf_len = 0;
+	void *buf;
 
 	if (nvdimm) {
 		desc = nd_cmd_dimm_desc(cmd);
@@ -977,13 +984,9 @@ static int __nd_ioctl(struct nvdimm_bus *nvdimm_bus, struct nvdimm *nvdimm,
 
 	if (cmd == ND_CMD_CALL) {
 		func = pkg.nd_command;
-		dev_dbg(dev, "%s:%s, idx: %llu, in: %zu, out: %zu, len %zu\n",
+		dev_dbg(dev, "%s:%s, idx: %llu, in: %u, out: %u, len %llu\n",
 				__func__, dimm_name, pkg.nd_command,
 				in_len, out_len, buf_len);
-
-		for (i = 0; i < ARRAY_SIZE(pkg.nd_reserved2); i++)
-			if (pkg.nd_reserved2[i])
-				return -EINVAL;
 	}
 
 	/* process an output envelope */
@@ -1007,9 +1010,9 @@ static int __nd_ioctl(struct nvdimm_bus *nvdimm_bus, struct nvdimm *nvdimm,
 		out_len += out_size;
 	}
 
-	buf_len = out_len + in_len;
+	buf_len = (u64) out_len + (u64) in_len;
 	if (buf_len > ND_IOCTL_MAX_BUFLEN) {
-		dev_dbg(dev, "%s:%s cmd: %s buf_len: %zu > %d\n", __func__,
+		dev_dbg(dev, "%s:%s cmd: %s buf_len: %llu > %d\n", __func__,
 				dimm_name, cmd_name, buf_len,
 				ND_IOCTL_MAX_BUFLEN);
 		return -EINVAL;
diff --git a/drivers/nvdimm/claim.c b/drivers/nvdimm/claim.c
index 47770460f3d3..b2fc29b8279b 100644
--- a/drivers/nvdimm/claim.c
+++ b/drivers/nvdimm/claim.c
@@ -280,18 +280,11 @@ static int nsio_rw_bytes(struct nd_namespace_common *ndns,
 	}
 
 	if (unlikely(is_bad_pmem(&nsio->bb, sector, sz_align))) {
-		/*
-		 * FIXME: nsio_rw_bytes() may be called from atomic
-		 * context in the btt case and the ACPI DSM path for
-		 * clearing the error takes sleeping locks and allocates
-		 * memory. An explicit error clearing path, and support
-		 * for tracking badblocks in BTT metadata is needed to
-		 * work around this collision.
-		 */
 		if (IS_ALIGNED(offset, 512) && IS_ALIGNED(size, 512)
 				&& !(flags & NVDIMM_IO_ATOMIC)) {
 			long cleared;
 
+			might_sleep();
 			cleared = nvdimm_clear_poison(&ndns->dev,
 					nsio->res.start + offset, size);
 			if (cleared < size)
diff --git a/drivers/nvdimm/core.c b/drivers/nvdimm/core.c
index 75bc08c6838c..bb71f0cf8f5d 100644
--- a/drivers/nvdimm/core.c
+++ b/drivers/nvdimm/core.c
@@ -277,14 +277,14 @@ int nd_uuid_store(struct device *dev, u8 **uuid_out, const char *buf,
 	return 0;
 }
 
-ssize_t nd_sector_size_show(unsigned long current_lbasize,
+ssize_t nd_size_select_show(unsigned long current_size,
 		const unsigned long *supported, char *buf)
 {
 	ssize_t len = 0;
 	int i;
 
 	for (i = 0; supported[i]; i++)
-		if (current_lbasize == supported[i])
+		if (current_size == supported[i])
 			len += sprintf(buf + len, "[%ld] ", supported[i]);
 		else
 			len += sprintf(buf + len, "%ld ", supported[i]);
@@ -292,8 +292,8 @@ ssize_t nd_sector_size_show(unsigned long current_lbasize,
 	return len;
 }
 
-ssize_t nd_sector_size_store(struct device *dev, const char *buf,
-		unsigned long *current_lbasize, const unsigned long *supported)
+ssize_t nd_size_select_store(struct device *dev, const char *buf,
+		unsigned long *current_size, const unsigned long *supported)
 {
 	unsigned long lbasize;
 	int rc, i;
@@ -310,7 +310,7 @@ ssize_t nd_sector_size_store(struct device *dev, const char *buf,
 			break;
 
 	if (supported[i]) {
-		*current_lbasize = lbasize;
+		*current_size = lbasize;
 		return 0;
 	} else {
 		return -EINVAL;
diff --git a/drivers/nvdimm/label.c b/drivers/nvdimm/label.c
index 87796f840777..9c5f108910e3 100644
--- a/drivers/nvdimm/label.c
+++ b/drivers/nvdimm/label.c
@@ -45,12 +45,14 @@ unsigned sizeof_namespace_label(struct nvdimm_drvdata *ndd)
 	return ndd->nslabel_size;
 }
 
-size_t sizeof_namespace_index(struct nvdimm_drvdata *ndd)
+int nvdimm_num_label_slots(struct nvdimm_drvdata *ndd)
 {
-	u32 index_span;
+	return ndd->nsarea.config_size / (sizeof_namespace_label(ndd) + 1);
+}
 
-	if (ndd->nsindex_size)
-		return ndd->nsindex_size;
+size_t sizeof_namespace_index(struct nvdimm_drvdata *ndd)
+{
+	u32 nslot, space, size;
 
 	/*
 	 * The minimum index space is 512 bytes, with that amount of
@@ -60,16 +62,16 @@ size_t sizeof_namespace_index(struct nvdimm_drvdata *ndd)
 	 * starts to waste space at larger config_sizes, but it's
 	 * unlikely we'll ever see anything but 128K.
 	 */
-	index_span = ndd->nsarea.config_size / (sizeof_namespace_label(ndd) + 1);
-	index_span /= NSINDEX_ALIGN * 2;
-	ndd->nsindex_size = index_span * NSINDEX_ALIGN;
-
-	return ndd->nsindex_size;
-}
-
-int nvdimm_num_label_slots(struct nvdimm_drvdata *ndd)
-{
-	return ndd->nsarea.config_size / (sizeof_namespace_label(ndd) + 1);
+	nslot = nvdimm_num_label_slots(ndd);
+	space = ndd->nsarea.config_size - nslot * sizeof_namespace_label(ndd);
+	size = ALIGN(sizeof(struct nd_namespace_index) + DIV_ROUND_UP(nslot, 8),
+			NSINDEX_ALIGN) * 2;
+	if (size <= space)
+		return size / 2;
+
+	dev_err(ndd->dev, "label area (%d) too small to host (%d byte) labels\n",
+			ndd->nsarea.config_size, sizeof_namespace_label(ndd));
+	return 0;
 }
 
 static int __nd_label_validate(struct nvdimm_drvdata *ndd)
diff --git a/drivers/nvdimm/namespace_devs.c b/drivers/nvdimm/namespace_devs.c
index 5f1c6756e57c..1427a386a033 100644
--- a/drivers/nvdimm/namespace_devs.c
+++ b/drivers/nvdimm/namespace_devs.c
@@ -1313,14 +1313,14 @@ static ssize_t sector_size_show(struct device *dev,
 	if (is_namespace_blk(dev)) {
 		struct nd_namespace_blk *nsblk = to_nd_namespace_blk(dev);
 
-		return nd_sector_size_show(nsblk->lbasize,
+		return nd_size_select_show(nsblk->lbasize,
 				blk_lbasize_supported, buf);
 	}
 
 	if (is_namespace_pmem(dev)) {
 		struct nd_namespace_pmem *nspm = to_nd_namespace_pmem(dev);
 
-		return nd_sector_size_show(nspm->lbasize,
+		return nd_size_select_show(nspm->lbasize,
 				pmem_lbasize_supported, buf);
 	}
 	return -ENXIO;
@@ -1352,7 +1352,7 @@ static ssize_t sector_size_store(struct device *dev,
 	if (to_ndns(dev)->claim)
 		rc = -EBUSY;
 	if (rc >= 0)
-		rc = nd_sector_size_store(dev, buf, lbasize, supported);
+		rc = nd_size_select_store(dev, buf, lbasize, supported);
 	if (rc >= 0)
 		rc = nd_namespace_label_update(nd_region, dev);
 	dev_dbg(dev, "%s: result: %zd %s: %s%s", __func__,
diff --git a/drivers/nvdimm/nd.h b/drivers/nvdimm/nd.h
index e1b5715bd91f..9c758a91372b 100644
--- a/drivers/nvdimm/nd.h
+++ b/drivers/nvdimm/nd.h
@@ -42,7 +42,7 @@ struct nd_poison {
 
 struct nvdimm_drvdata {
 	struct device *dev;
-	int nsindex_size, nslabel_size;
+	int nslabel_size;
 	struct nd_cmd_get_config_size nsarea;
 	void *data;
 	int ns_current, ns_next;
@@ -134,6 +134,7 @@ struct nd_mapping {
 	struct nvdimm *nvdimm;
 	u64 start;
 	u64 size;
+	int position;
 	struct list_head labels;
 	struct mutex lock;
 	/*
@@ -233,10 +234,10 @@ void nd_device_unregister(struct device *dev, enum nd_async_mode mode);
 void nd_device_notify(struct device *dev, enum nvdimm_event event);
 int nd_uuid_store(struct device *dev, u8 **uuid_out, const char *buf,
 		size_t len);
-ssize_t nd_sector_size_show(unsigned long current_lbasize,
+ssize_t nd_size_select_show(unsigned long current_size,
 		const unsigned long *supported, char *buf);
-ssize_t nd_sector_size_store(struct device *dev, const char *buf,
-		unsigned long *current_lbasize, const unsigned long *supported);
+ssize_t nd_size_select_store(struct device *dev, const char *buf,
+		unsigned long *current_size, const unsigned long *supported);
 int __init nvdimm_init(void);
 int __init nd_region_init(void);
 int __init nd_label_init(void);
@@ -285,6 +286,13 @@ static inline struct device *nd_btt_create(struct nd_region *nd_region)
 
 struct nd_pfn *to_nd_pfn(struct device *dev);
 #if IS_ENABLED(CONFIG_NVDIMM_PFN)
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+#define PFN_DEFAULT_ALIGNMENT HPAGE_PMD_SIZE
+#else
+#define PFN_DEFAULT_ALIGNMENT PAGE_SIZE
+#endif
+
 int nd_pfn_probe(struct device *dev, struct nd_namespace_common *ndns);
 bool is_nd_pfn(struct device *dev);
 struct device *nd_pfn_create(struct nd_region *nd_region);
@@ -390,21 +398,22 @@ int nd_region_activate(struct nd_region *nd_region);
 void __nd_iostat_start(struct bio *bio, unsigned long *start);
 static inline bool nd_iostat_start(struct bio *bio, unsigned long *start)
 {
-	struct gendisk *disk = bio->bi_bdev->bd_disk;
+	struct gendisk *disk = bio->bi_disk;
 
 	if (!blk_queue_io_stat(disk->queue))
 		return false;
 
 	*start = jiffies;
-	generic_start_io_acct(bio_data_dir(bio),
+	generic_start_io_acct(disk->queue, bio_data_dir(bio),
 			      bio_sectors(bio), &disk->part0);
 	return true;
 }
 static inline void nd_iostat_end(struct bio *bio, unsigned long start)
 {
-	struct gendisk *disk = bio->bi_bdev->bd_disk;
+	struct gendisk *disk = bio->bi_disk;
 
-	generic_end_io_acct(bio_data_dir(bio), &disk->part0, start);
+	generic_end_io_acct(disk->queue, bio_data_dir(bio), &disk->part0,
+				start);
 }
 static inline bool is_bad_pmem(struct badblocks *bb, sector_t sector,
 		unsigned int len)
diff --git a/drivers/nvdimm/pfn_devs.c b/drivers/nvdimm/pfn_devs.c
index 5fcb6f5b22a2..9576c444f0ab 100644
--- a/drivers/nvdimm/pfn_devs.c
+++ b/drivers/nvdimm/pfn_devs.c
@@ -111,24 +111,27 @@ static ssize_t align_show(struct device *dev,
 	return sprintf(buf, "%ld\n", nd_pfn->align);
 }
 
-static ssize_t __align_store(struct nd_pfn *nd_pfn, const char *buf)
+static const unsigned long *nd_pfn_supported_alignments(void)
 {
-	unsigned long val;
-	int rc;
-
-	rc = kstrtoul(buf, 0, &val);
-	if (rc)
-		return rc;
-
-	if (!is_power_of_2(val) || val < PAGE_SIZE || val > SZ_1G)
-		return -EINVAL;
+	/*
+	 * This needs to be a non-static variable because the *_SIZE
+	 * macros aren't always constants.
+	 */
+	const unsigned long supported_alignments[] = {
+		PAGE_SIZE,
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+		HPAGE_PMD_SIZE,
+#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
+		HPAGE_PUD_SIZE,
+#endif
+#endif
+		0,
+	};
+	static unsigned long data[ARRAY_SIZE(supported_alignments)];
 
-	if (nd_pfn->dev.driver)
-		return -EBUSY;
-	else
-		nd_pfn->align = val;
+	memcpy(data, supported_alignments, sizeof(data));
 
-	return 0;
+	return data;
 }
 
 static ssize_t align_store(struct device *dev,
@@ -139,7 +142,8 @@ static ssize_t align_store(struct device *dev,
 
 	device_lock(dev);
 	nvdimm_bus_lock(dev);
-	rc = __align_store(nd_pfn, buf);
+	rc = nd_size_select_store(dev, buf, &nd_pfn->align,
+			nd_pfn_supported_alignments());
 	dev_dbg(dev, "%s: result: %zd wrote: %s%s", __func__,
 			rc, buf, buf[len - 1] == '\n' ? "" : "\n");
 	nvdimm_bus_unlock(dev);
@@ -260,6 +264,13 @@ static ssize_t size_show(struct device *dev,
 }
 static DEVICE_ATTR_RO(size);
 
+static ssize_t supported_alignments_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	return nd_size_select_show(0, nd_pfn_supported_alignments(), buf);
+}
+static DEVICE_ATTR_RO(supported_alignments);
+
 static struct attribute *nd_pfn_attributes[] = {
 	&dev_attr_mode.attr,
 	&dev_attr_namespace.attr,
@@ -267,6 +278,7 @@ static struct attribute *nd_pfn_attributes[] = {
 	&dev_attr_align.attr,
 	&dev_attr_resource.attr,
 	&dev_attr_size.attr,
+	&dev_attr_supported_alignments.attr,
 	NULL,
 };
 
@@ -290,7 +302,7 @@ struct device *nd_pfn_devinit(struct nd_pfn *nd_pfn,
 		return NULL;
 
 	nd_pfn->mode = PFN_MODE_NONE;
-	nd_pfn->align = HPAGE_SIZE;
+	nd_pfn->align = PFN_DEFAULT_ALIGNMENT;
 	dev = &nd_pfn->dev;
 	device_initialize(&nd_pfn->dev);
 	if (ndns && !__nd_attach_ndns(&nd_pfn->dev, ndns, &nd_pfn->ndns)) {
@@ -638,11 +650,12 @@ static int nd_pfn_init(struct nd_pfn *nd_pfn)
 			/ PAGE_SIZE);
 	if (nd_pfn->mode == PFN_MODE_PMEM) {
 		/*
-		 * vmemmap_populate_hugepages() allocates the memmap array in
-		 * HPAGE_SIZE chunks.
+		 * The altmap should be padded out to the block size used
+		 * when populating the vmemmap. This *should* be equal to
+		 * PMD_SIZE for most architectures.
 		 */
 		offset = ALIGN(start + SZ_8K + 64 * npfns + dax_label_reserve,
-				max(nd_pfn->align, HPAGE_SIZE)) - start;
+				max(nd_pfn->align, PMD_SIZE)) - start;
 	} else if (nd_pfn->mode == PFN_MODE_RAM)
 		offset = ALIGN(start + SZ_8K + dax_label_reserve,
 				nd_pfn->align) - start;
diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c
index f7099adaabc0..39dfd7affa31 100644
--- a/drivers/nvdimm/pmem.c
+++ b/drivers/nvdimm/pmem.c
@@ -80,22 +80,40 @@ static blk_status_t pmem_clear_poison(struct pmem_device *pmem,
 static void write_pmem(void *pmem_addr, struct page *page,
 		unsigned int off, unsigned int len)
 {
-	void *mem = kmap_atomic(page);
-
-	memcpy_flushcache(pmem_addr, mem + off, len);
-	kunmap_atomic(mem);
+	unsigned int chunk;
+	void *mem;
+
+	while (len) {
+		mem = kmap_atomic(page);
+		chunk = min_t(unsigned int, len, PAGE_SIZE);
+		memcpy_flushcache(pmem_addr, mem + off, chunk);
+		kunmap_atomic(mem);
+		len -= chunk;
+		off = 0;
+		page++;
+		pmem_addr += PAGE_SIZE;
+	}
 }
 
 static blk_status_t read_pmem(struct page *page, unsigned int off,
 		void *pmem_addr, unsigned int len)
 {
+	unsigned int chunk;
 	int rc;
-	void *mem = kmap_atomic(page);
-
-	rc = memcpy_mcsafe(mem + off, pmem_addr, len);
-	kunmap_atomic(mem);
-	if (rc)
-		return BLK_STS_IOERR;
+	void *mem;
+
+	while (len) {
+		mem = kmap_atomic(page);
+		chunk = min_t(unsigned int, len, PAGE_SIZE);
+		rc = memcpy_mcsafe(mem + off, pmem_addr, chunk);
+		kunmap_atomic(mem);
+		if (rc)
+			return BLK_STS_IOERR;
+		len -= chunk;
+		off = 0;
+		page++;
+		pmem_addr += PAGE_SIZE;
+	}
 	return BLK_STS_OK;
 }
 
@@ -188,7 +206,8 @@ static int pmem_rw_page(struct block_device *bdev, sector_t sector,
 	struct pmem_device *pmem = bdev->bd_queue->queuedata;
 	blk_status_t rc;
 
-	rc = pmem_do_bvec(pmem, page, PAGE_SIZE, 0, is_write, sector);
+	rc = pmem_do_bvec(pmem, page, hpage_nr_pages(page) * PAGE_SIZE,
+			  0, is_write, sector);
 
 	/*
 	 * The ->rw_page interface is subtle and tricky.  The core
@@ -243,16 +262,9 @@ static size_t pmem_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff,
 	return copy_from_iter_flushcache(addr, bytes, i);
 }
 
-static void pmem_dax_flush(struct dax_device *dax_dev, pgoff_t pgoff,
-		void *addr, size_t size)
-{
-	arch_wb_cache_pmem(addr, size);
-}
-
 static const struct dax_operations pmem_dax_ops = {
 	.direct_access = pmem_dax_direct_access,
 	.copy_from_iter = pmem_copy_from_iter,
-	.flush = pmem_dax_flush,
 };
 
 static const struct attribute_group *pmem_attribute_groups[] = {
diff --git a/drivers/nvdimm/pmem.h b/drivers/nvdimm/pmem.h
index 5434321cad67..c5917f040fa7 100644
--- a/drivers/nvdimm/pmem.h
+++ b/drivers/nvdimm/pmem.h
@@ -5,20 +5,6 @@
 #include <linux/pfn_t.h>
 #include <linux/fs.h>
 
-#ifdef CONFIG_ARCH_HAS_PMEM_API
-#define ARCH_MEMREMAP_PMEM MEMREMAP_WB
-void arch_wb_cache_pmem(void *addr, size_t size);
-void arch_invalidate_pmem(void *addr, size_t size);
-#else
-#define ARCH_MEMREMAP_PMEM MEMREMAP_WT
-static inline void arch_wb_cache_pmem(void *addr, size_t size)
-{
-}
-static inline void arch_invalidate_pmem(void *addr, size_t size)
-{
-}
-#endif
-
 /* this definition is in it's own header for tools/testing/nvdimm to consume */
 struct pmem_device {
 	/* One contiguous memory region per device */
diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c
index 5954cfbea3fc..829d760f651c 100644
--- a/drivers/nvdimm/region_devs.c
+++ b/drivers/nvdimm/region_devs.c
@@ -723,8 +723,9 @@ static ssize_t mappingN(struct device *dev, char *buf, int n)
 	nd_mapping = &nd_region->mapping[n];
 	nvdimm = nd_mapping->nvdimm;
 
-	return sprintf(buf, "%s,%llu,%llu\n", dev_name(&nvdimm->dev),
-			nd_mapping->start, nd_mapping->size);
+	return sprintf(buf, "%s,%llu,%llu,%d\n", dev_name(&nvdimm->dev),
+			nd_mapping->start, nd_mapping->size,
+			nd_mapping->position);
 }
 
 #define REGION_MAPPING(idx) \
@@ -965,6 +966,7 @@ static struct nd_region *nd_region_create(struct nvdimm_bus *nvdimm_bus,
 		nd_region->mapping[i].nvdimm = nvdimm;
 		nd_region->mapping[i].start = mapping->start;
 		nd_region->mapping[i].size = mapping->size;
+		nd_region->mapping[i].position = mapping->position;
 		INIT_LIST_HEAD(&nd_region->mapping[i].labels);
 		mutex_init(&nd_region->mapping[i].lock);
 
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 37046ac2c441..acc816b67582 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -76,6 +76,11 @@ static DEFINE_SPINLOCK(dev_list_lock);
 
 static struct class *nvme_class;
 
+static __le32 nvme_get_log_dw10(u8 lid, size_t size)
+{
+	return cpu_to_le32((((size / 4) - 1) << 16) | lid);
+}
+
 int nvme_reset_ctrl(struct nvme_ctrl *ctrl)
 {
 	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
@@ -108,7 +113,16 @@ static blk_status_t nvme_error_status(struct request *req)
 	case NVME_SC_WRITE_FAULT:
 	case NVME_SC_READ_ERROR:
 	case NVME_SC_UNWRITTEN_BLOCK:
+	case NVME_SC_ACCESS_DENIED:
+	case NVME_SC_READ_ONLY:
 		return BLK_STS_MEDIUM;
+	case NVME_SC_GUARD_CHECK:
+	case NVME_SC_APPTAG_CHECK:
+	case NVME_SC_REFTAG_CHECK:
+	case NVME_SC_INVALID_PI:
+		return BLK_STS_PROTECTION;
+	case NVME_SC_RESERVATION_CONFLICT:
+		return BLK_STS_NEXUS;
 	default:
 		return BLK_STS_IOERR;
 	}
@@ -162,9 +176,10 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
 		enum nvme_ctrl_state new_state)
 {
 	enum nvme_ctrl_state old_state;
+	unsigned long flags;
 	bool changed = false;
 
-	spin_lock_irq(&ctrl->lock);
+	spin_lock_irqsave(&ctrl->lock, flags);
 
 	old_state = ctrl->state;
 	switch (new_state) {
@@ -225,7 +240,7 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
 	if (changed)
 		ctrl->state = new_state;
 
-	spin_unlock_irq(&ctrl->lock);
+	spin_unlock_irqrestore(&ctrl->lock, flags);
 
 	return changed;
 }
@@ -307,7 +322,7 @@ static int nvme_toggle_streams(struct nvme_ctrl *ctrl, bool enable)
 	memset(&c, 0, sizeof(c));
 
 	c.directive.opcode = nvme_admin_directive_send;
-	c.directive.nsid = cpu_to_le32(0xffffffff);
+	c.directive.nsid = cpu_to_le32(NVME_NSID_ALL);
 	c.directive.doper = NVME_DIR_SND_ID_OP_ENABLE;
 	c.directive.dtype = NVME_DIR_IDENTIFY;
 	c.directive.tdtype = NVME_DIR_STREAMS;
@@ -357,7 +372,7 @@ static int nvme_configure_directives(struct nvme_ctrl *ctrl)
 	if (ret)
 		return ret;
 
-	ret = nvme_get_stream_params(ctrl, &s, 0xffffffff);
+	ret = nvme_get_stream_params(ctrl, &s, NVME_NSID_ALL);
 	if (ret)
 		return ret;
 
@@ -585,10 +600,44 @@ int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
 }
 EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
 
-int __nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
-		void __user *ubuffer, unsigned bufflen,
-		void __user *meta_buffer, unsigned meta_len, u32 meta_seed,
-		u32 *result, unsigned timeout)
+static void *nvme_add_user_metadata(struct bio *bio, void __user *ubuf,
+		unsigned len, u32 seed, bool write)
+{
+	struct bio_integrity_payload *bip;
+	int ret = -ENOMEM;
+	void *buf;
+
+	buf = kmalloc(len, GFP_KERNEL);
+	if (!buf)
+		goto out;
+
+	ret = -EFAULT;
+	if (write && copy_from_user(buf, ubuf, len))
+		goto out_free_meta;
+
+	bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
+	if (IS_ERR(bip)) {
+		ret = PTR_ERR(bip);
+		goto out_free_meta;
+	}
+
+	bip->bip_iter.bi_size = len;
+	bip->bip_iter.bi_sector = seed;
+	ret = bio_integrity_add_page(bio, virt_to_page(buf), len,
+			offset_in_page(buf));
+	if (ret == len)
+		return buf;
+	ret = -ENOMEM;
+out_free_meta:
+	kfree(buf);
+out:
+	return ERR_PTR(ret);
+}
+
+static int nvme_submit_user_cmd(struct request_queue *q,
+		struct nvme_command *cmd, void __user *ubuffer,
+		unsigned bufflen, void __user *meta_buffer, unsigned meta_len,
+		u32 meta_seed, u32 *result, unsigned timeout)
 {
 	bool write = nvme_is_write(cmd);
 	struct nvme_ns *ns = q->queuedata;
@@ -610,50 +659,17 @@ int __nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
 		if (ret)
 			goto out;
 		bio = req->bio;
-
-		if (!disk)
-			goto submit;
-		bio->bi_bdev = bdget_disk(disk, 0);
-		if (!bio->bi_bdev) {
-			ret = -ENODEV;
-			goto out_unmap;
-		}
-
-		if (meta_buffer && meta_len) {
-			struct bio_integrity_payload *bip;
-
-			meta = kmalloc(meta_len, GFP_KERNEL);
-			if (!meta) {
-				ret = -ENOMEM;
+		bio->bi_disk = disk;
+		if (disk && meta_buffer && meta_len) {
+			meta = nvme_add_user_metadata(bio, meta_buffer, meta_len,
+					meta_seed, write);
+			if (IS_ERR(meta)) {
+				ret = PTR_ERR(meta);
 				goto out_unmap;
 			}
-
-			if (write) {
-				if (copy_from_user(meta, meta_buffer,
-						meta_len)) {
-					ret = -EFAULT;
-					goto out_free_meta;
-				}
-			}
-
-			bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
-			if (IS_ERR(bip)) {
-				ret = PTR_ERR(bip);
-				goto out_free_meta;
-			}
-
-			bip->bip_iter.bi_size = meta_len;
-			bip->bip_iter.bi_sector = meta_seed;
-
-			ret = bio_integrity_add_page(bio, virt_to_page(meta),
-					meta_len, offset_in_page(meta));
-			if (ret != meta_len) {
-				ret = -ENOMEM;
-				goto out_free_meta;
-			}
 		}
 	}
- submit:
+
 	blk_execute_rq(req->q, disk, req, 0);
 	if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
 		ret = -EINTR;
@@ -665,27 +681,15 @@ int __nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
 		if (copy_to_user(meta_buffer, meta, meta_len))
 			ret = -EFAULT;
 	}
- out_free_meta:
 	kfree(meta);
  out_unmap:
-	if (bio) {
-		if (disk && bio->bi_bdev)
-			bdput(bio->bi_bdev);
+	if (bio)
 		blk_rq_unmap_user(bio);
-	}
  out:
 	blk_mq_free_request(req);
 	return ret;
 }
 
-int nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
-		void __user *ubuffer, unsigned bufflen, u32 *result,
-		unsigned timeout)
-{
-	return __nvme_submit_user_cmd(q, cmd, ubuffer, bufflen, NULL, 0, 0,
-			result, timeout);
-}
-
 static void nvme_keep_alive_end_io(struct request *rq, blk_status_t status)
 {
 	struct nvme_ctrl *ctrl = rq->end_io_data;
@@ -775,7 +779,8 @@ static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
 	return error;
 }
 
-static int nvme_identify_ns_descs(struct nvme_ns *ns, unsigned nsid)
+static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl, unsigned nsid,
+		u8 *eui64, u8 *nguid, uuid_t *uuid)
 {
 	struct nvme_command c = { };
 	int status;
@@ -791,7 +796,7 @@ static int nvme_identify_ns_descs(struct nvme_ns *ns, unsigned nsid)
 	if (!data)
 		return -ENOMEM;
 
-	status = nvme_submit_sync_cmd(ns->ctrl->admin_q, &c, data,
+	status = nvme_submit_sync_cmd(ctrl->admin_q, &c, data,
 				      NVME_IDENTIFY_DATA_SIZE);
 	if (status)
 		goto free_data;
@@ -805,33 +810,33 @@ static int nvme_identify_ns_descs(struct nvme_ns *ns, unsigned nsid)
 		switch (cur->nidt) {
 		case NVME_NIDT_EUI64:
 			if (cur->nidl != NVME_NIDT_EUI64_LEN) {
-				dev_warn(ns->ctrl->device,
+				dev_warn(ctrl->device,
 					 "ctrl returned bogus length: %d for NVME_NIDT_EUI64\n",
 					 cur->nidl);
 				goto free_data;
 			}
 			len = NVME_NIDT_EUI64_LEN;
-			memcpy(ns->eui, data + pos + sizeof(*cur), len);
+			memcpy(eui64, data + pos + sizeof(*cur), len);
 			break;
 		case NVME_NIDT_NGUID:
 			if (cur->nidl != NVME_NIDT_NGUID_LEN) {
-				dev_warn(ns->ctrl->device,
+				dev_warn(ctrl->device,
 					 "ctrl returned bogus length: %d for NVME_NIDT_NGUID\n",
 					 cur->nidl);
 				goto free_data;
 			}
 			len = NVME_NIDT_NGUID_LEN;
-			memcpy(ns->nguid, data + pos + sizeof(*cur), len);
+			memcpy(nguid, data + pos + sizeof(*cur), len);
 			break;
 		case NVME_NIDT_UUID:
 			if (cur->nidl != NVME_NIDT_UUID_LEN) {
-				dev_warn(ns->ctrl->device,
+				dev_warn(ctrl->device,
 					 "ctrl returned bogus length: %d for NVME_NIDT_UUID\n",
 					 cur->nidl);
 				goto free_data;
 			}
 			len = NVME_NIDT_UUID_LEN;
-			uuid_copy(&ns->uuid, data + pos + sizeof(*cur));
+			uuid_copy(uuid, data + pos + sizeof(*cur));
 			break;
 		default:
 			/* Skip unnkown types */
@@ -856,9 +861,10 @@ static int nvme_identify_ns_list(struct nvme_ctrl *dev, unsigned nsid, __le32 *n
 	return nvme_submit_sync_cmd(dev->admin_q, &c, ns_list, 0x1000);
 }
 
-static int nvme_identify_ns(struct nvme_ctrl *dev, unsigned nsid,
-		struct nvme_id_ns **id)
+static struct nvme_id_ns *nvme_identify_ns(struct nvme_ctrl *ctrl,
+		unsigned nsid)
 {
+	struct nvme_id_ns *id;
 	struct nvme_command c = { };
 	int error;
 
@@ -867,15 +873,18 @@ static int nvme_identify_ns(struct nvme_ctrl *dev, unsigned nsid,
 	c.identify.nsid = cpu_to_le32(nsid);
 	c.identify.cns = NVME_ID_CNS_NS;
 
-	*id = kmalloc(sizeof(struct nvme_id_ns), GFP_KERNEL);
-	if (!*id)
-		return -ENOMEM;
+	id = kmalloc(sizeof(*id), GFP_KERNEL);
+	if (!id)
+		return NULL;
 
-	error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
-			sizeof(struct nvme_id_ns));
-	if (error)
-		kfree(*id);
-	return error;
+	error = nvme_submit_sync_cmd(ctrl->admin_q, &c, id, sizeof(*id));
+	if (error) {
+		dev_warn(ctrl->device, "Identify namespace failed\n");
+		kfree(id);
+		return NULL;
+	}
+
+	return id;
 }
 
 static int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
@@ -970,7 +979,7 @@ static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
 	c.rw.apptag = cpu_to_le16(io.apptag);
 	c.rw.appmask = cpu_to_le16(io.appmask);
 
-	return __nvme_submit_user_cmd(ns->queue, &c,
+	return nvme_submit_user_cmd(ns->queue, &c,
 			(void __user *)(uintptr_t)io.addr, length,
 			metadata, meta_len, io.slba, NULL, 0);
 }
@@ -1008,7 +1017,8 @@ static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 
 	status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
 			(void __user *)(uintptr_t)cmd.addr, cmd.data_len,
-			&cmd.result, timeout);
+			(void __user *)(uintptr_t)cmd.metadata, cmd.metadata,
+			0, &cmd.result, timeout);
 	if (status >= 0) {
 		if (put_user(cmd.result, &ucmd->result))
 			return -EFAULT;
@@ -1166,32 +1176,21 @@ static void nvme_config_discard(struct nvme_ns *ns)
 		blk_queue_max_write_zeroes_sectors(ns->queue, UINT_MAX);
 }
 
-static int nvme_revalidate_ns(struct nvme_ns *ns, struct nvme_id_ns **id)
+static void nvme_report_ns_ids(struct nvme_ctrl *ctrl, unsigned int nsid,
+		struct nvme_id_ns *id, u8 *eui64, u8 *nguid, uuid_t *uuid)
 {
-	if (nvme_identify_ns(ns->ctrl, ns->ns_id, id)) {
-		dev_warn(ns->ctrl->dev, "%s: Identify failure\n", __func__);
-		return -ENODEV;
-	}
-
-	if ((*id)->ncap == 0) {
-		kfree(*id);
-		return -ENODEV;
-	}
-
-	if (ns->ctrl->vs >= NVME_VS(1, 1, 0))
-		memcpy(ns->eui, (*id)->eui64, sizeof(ns->eui));
-	if (ns->ctrl->vs >= NVME_VS(1, 2, 0))
-		memcpy(ns->nguid, (*id)->nguid, sizeof(ns->nguid));
-	if (ns->ctrl->vs >= NVME_VS(1, 3, 0)) {
+	if (ctrl->vs >= NVME_VS(1, 1, 0))
+		memcpy(eui64, id->eui64, sizeof(id->eui64));
+	if (ctrl->vs >= NVME_VS(1, 2, 0))
+		memcpy(nguid, id->nguid, sizeof(id->nguid));
+	if (ctrl->vs >= NVME_VS(1, 3, 0)) {
 		 /* Don't treat error as fatal we potentially
 		  * already have a NGUID or EUI-64
 		  */
-		if (nvme_identify_ns_descs(ns, ns->ns_id))
-			dev_warn(ns->ctrl->device,
+		if (nvme_identify_ns_descs(ctrl, nsid, eui64, nguid, uuid))
+			dev_warn(ctrl->device,
 				 "%s: Identify Descriptors failed\n", __func__);
 	}
-
-	return 0;
 }
 
 static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id)
@@ -1232,22 +1231,38 @@ static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id)
 static int nvme_revalidate_disk(struct gendisk *disk)
 {
 	struct nvme_ns *ns = disk->private_data;
-	struct nvme_id_ns *id = NULL;
-	int ret;
+	struct nvme_ctrl *ctrl = ns->ctrl;
+	struct nvme_id_ns *id;
+	u8 eui64[8] = { 0 }, nguid[16] = { 0 };
+	uuid_t uuid = uuid_null;
+	int ret = 0;
 
 	if (test_bit(NVME_NS_DEAD, &ns->flags)) {
 		set_capacity(disk, 0);
 		return -ENODEV;
 	}
 
-	ret = nvme_revalidate_ns(ns, &id);
-	if (ret)
-		return ret;
+	id = nvme_identify_ns(ctrl, ns->ns_id);
+	if (!id)
+		return -ENODEV;
 
-	__nvme_revalidate_disk(disk, id);
-	kfree(id);
+	if (id->ncap == 0) {
+		ret = -ENODEV;
+		goto out;
+	}
 
-	return 0;
+	nvme_report_ns_ids(ctrl, ns->ns_id, id, eui64, nguid, &uuid);
+	if (!uuid_equal(&ns->uuid, &uuid) ||
+	    memcmp(&ns->nguid, &nguid, sizeof(ns->nguid)) ||
+	    memcmp(&ns->eui, &eui64, sizeof(ns->eui))) {
+		dev_err(ctrl->device,
+			"identifiers changed for nsid %d\n", ns->ns_id);
+		ret = -ENODEV;
+	}
+
+out:
+	kfree(id);
+	return ret;
 }
 
 static char nvme_pr_type(enum pr_type type)
@@ -1447,7 +1462,7 @@ int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap)
 
 	ctrl->ctrl_config = NVME_CC_CSS_NVM;
 	ctrl->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
-	ctrl->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
+	ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE;
 	ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
 	ctrl->ctrl_config |= NVME_CC_ENABLE;
 
@@ -1460,7 +1475,7 @@ EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
 
 int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
 {
-	unsigned long timeout = jiffies + (shutdown_timeout * HZ);
+	unsigned long timeout = jiffies + (ctrl->shutdown_timeout * HZ);
 	u32 csts;
 	int ret;
 
@@ -1509,6 +1524,23 @@ static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
 	blk_queue_write_cache(q, vwc, vwc);
 }
 
+static int nvme_configure_timestamp(struct nvme_ctrl *ctrl)
+{
+	__le64 ts;
+	int ret;
+
+	if (!(ctrl->oncs & NVME_CTRL_ONCS_TIMESTAMP))
+		return 0;
+
+	ts = cpu_to_le64(ktime_to_ms(ktime_get_real()));
+	ret = nvme_set_features(ctrl, NVME_FEAT_TIMESTAMP, 0, &ts, sizeof(ts),
+			NULL);
+	if (ret)
+		dev_warn_once(ctrl->device,
+			"could not set timestamp (%d)\n", ret);
+	return ret;
+}
+
 static int nvme_configure_apst(struct nvme_ctrl *ctrl)
 {
 	/*
@@ -1811,6 +1843,20 @@ int nvme_init_identify(struct nvme_ctrl *ctrl)
 	ctrl->sgls = le32_to_cpu(id->sgls);
 	ctrl->kas = le16_to_cpu(id->kas);
 
+	if (id->rtd3e) {
+		/* us -> s */
+		u32 transition_time = le32_to_cpu(id->rtd3e) / 1000000;
+
+		ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time,
+						 shutdown_timeout, 60);
+
+		if (ctrl->shutdown_timeout != shutdown_timeout)
+			dev_warn(ctrl->device,
+				 "Shutdown timeout set to %u seconds\n",
+				 ctrl->shutdown_timeout);
+	} else
+		ctrl->shutdown_timeout = shutdown_timeout;
+
 	ctrl->npss = id->npss;
 	ctrl->apsta = id->apsta;
 	prev_apst_enabled = ctrl->apst_enabled;
@@ -1851,6 +1897,8 @@ int nvme_init_identify(struct nvme_ctrl *ctrl)
 		ctrl->cntlid = le16_to_cpu(id->cntlid);
 		ctrl->hmpre = le32_to_cpu(id->hmpre);
 		ctrl->hmmin = le32_to_cpu(id->hmmin);
+		ctrl->hmminds = le32_to_cpu(id->hmminds);
+		ctrl->hmmaxd = le16_to_cpu(id->hmmaxd);
 	}
 
 	kfree(id);
@@ -1863,6 +1911,10 @@ int nvme_init_identify(struct nvme_ctrl *ctrl)
 	ret = nvme_configure_apst(ctrl);
 	if (ret < 0)
 		return ret;
+	
+	ret = nvme_configure_timestamp(ctrl);
+	if (ret < 0)
+		return ret;
 
 	ret = nvme_configure_directives(ctrl);
 	if (ret < 0)
@@ -2318,13 +2370,20 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
 
 	sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->instance);
 
-	if (nvme_revalidate_ns(ns, &id))
+	id = nvme_identify_ns(ctrl, nsid);
+	if (!id)
 		goto out_free_queue;
 
-	if (nvme_nvm_ns_supported(ns, id) &&
-				nvme_nvm_register(ns, disk_name, node)) {
-		dev_warn(ctrl->device, "%s: LightNVM init failure\n", __func__);
+	if (id->ncap == 0)
 		goto out_free_id;
+
+	nvme_report_ns_ids(ctrl, ns->ns_id, id, ns->eui, ns->nguid, &ns->uuid);
+
+	if ((ctrl->quirks & NVME_QUIRK_LIGHTNVM) && id->vs[0] == 0x1) {
+		if (nvme_nvm_register(ns, disk_name, node)) {
+			dev_warn(ctrl->device, "LightNVM init failure\n");
+			goto out_free_id;
+		}
 	}
 
 	disk = alloc_disk_node(0, node);
@@ -2541,6 +2600,71 @@ static void nvme_async_event_work(struct work_struct *work)
 	spin_unlock_irq(&ctrl->lock);
 }
 
+static bool nvme_ctrl_pp_status(struct nvme_ctrl *ctrl)
+{
+
+	u32 csts;
+
+	if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts))
+		return false;
+
+	if (csts == ~0)
+		return false;
+
+	return ((ctrl->ctrl_config & NVME_CC_ENABLE) && (csts & NVME_CSTS_PP));
+}
+
+static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl)
+{
+	struct nvme_command c = { };
+	struct nvme_fw_slot_info_log *log;
+
+	log = kmalloc(sizeof(*log), GFP_KERNEL);
+	if (!log)
+		return;
+
+	c.common.opcode = nvme_admin_get_log_page;
+	c.common.nsid = cpu_to_le32(NVME_NSID_ALL);
+	c.common.cdw10[0] = nvme_get_log_dw10(NVME_LOG_FW_SLOT, sizeof(*log));
+
+	if (!nvme_submit_sync_cmd(ctrl->admin_q, &c, log, sizeof(*log)))
+		dev_warn(ctrl->device,
+				"Get FW SLOT INFO log error\n");
+	kfree(log);
+}
+
+static void nvme_fw_act_work(struct work_struct *work)
+{
+	struct nvme_ctrl *ctrl = container_of(work,
+				struct nvme_ctrl, fw_act_work);
+	unsigned long fw_act_timeout;
+
+	if (ctrl->mtfa)
+		fw_act_timeout = jiffies +
+				msecs_to_jiffies(ctrl->mtfa * 100);
+	else
+		fw_act_timeout = jiffies +
+				msecs_to_jiffies(admin_timeout * 1000);
+
+	nvme_stop_queues(ctrl);
+	while (nvme_ctrl_pp_status(ctrl)) {
+		if (time_after(jiffies, fw_act_timeout)) {
+			dev_warn(ctrl->device,
+				"Fw activation timeout, reset controller\n");
+			nvme_reset_ctrl(ctrl);
+			break;
+		}
+		msleep(100);
+	}
+
+	if (ctrl->state != NVME_CTRL_LIVE)
+		return;
+
+	nvme_start_queues(ctrl);
+	/* read FW slot informationi to clear the AER*/
+	nvme_get_fw_slot_info(ctrl);
+}
+
 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
 		union nvme_result *res)
 {
@@ -2567,6 +2691,9 @@ void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
 		dev_info(ctrl->device, "rescanning\n");
 		nvme_queue_scan(ctrl);
 		break;
+	case NVME_AER_NOTICE_FW_ACT_STARTING:
+		schedule_work(&ctrl->fw_act_work);
+		break;
 	default:
 		dev_warn(ctrl->device, "async event result %08x\n", result);
 	}
@@ -2614,6 +2741,7 @@ void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
 	nvme_stop_keep_alive(ctrl);
 	flush_work(&ctrl->async_event_work);
 	flush_work(&ctrl->scan_work);
+	cancel_work_sync(&ctrl->fw_act_work);
 }
 EXPORT_SYMBOL_GPL(nvme_stop_ctrl);
 
@@ -2677,6 +2805,7 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
 	ctrl->quirks = quirks;
 	INIT_WORK(&ctrl->scan_work, nvme_scan_work);
 	INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
+	INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work);
 
 	ret = nvme_set_instance(ctrl);
 	if (ret)
diff --git a/drivers/nvme/host/fabrics.c b/drivers/nvme/host/fabrics.c
index 2e582a240943..47307752dc65 100644
--- a/drivers/nvme/host/fabrics.c
+++ b/drivers/nvme/host/fabrics.c
@@ -22,7 +22,7 @@
 #include "fabrics.h"
 
 static LIST_HEAD(nvmf_transports);
-static DEFINE_MUTEX(nvmf_transports_mutex);
+static DECLARE_RWSEM(nvmf_transports_rwsem);
 
 static LIST_HEAD(nvmf_hosts);
 static DEFINE_MUTEX(nvmf_hosts_mutex);
@@ -75,7 +75,7 @@ static struct nvmf_host *nvmf_host_default(void)
 
 	kref_init(&host->ref);
 	snprintf(host->nqn, NVMF_NQN_SIZE,
-		"nqn.2014-08.org.nvmexpress:NVMf:uuid:%pUb", &host->id);
+		"nqn.2014-08.org.nvmexpress:uuid:%pUb", &host->id);
 
 	mutex_lock(&nvmf_hosts_mutex);
 	list_add_tail(&host->list, &nvmf_hosts);
@@ -495,9 +495,9 @@ int nvmf_register_transport(struct nvmf_transport_ops *ops)
 	if (!ops->create_ctrl)
 		return -EINVAL;
 
-	mutex_lock(&nvmf_transports_mutex);
+	down_write(&nvmf_transports_rwsem);
 	list_add_tail(&ops->entry, &nvmf_transports);
-	mutex_unlock(&nvmf_transports_mutex);
+	up_write(&nvmf_transports_rwsem);
 
 	return 0;
 }
@@ -514,9 +514,9 @@ EXPORT_SYMBOL_GPL(nvmf_register_transport);
  */
 void nvmf_unregister_transport(struct nvmf_transport_ops *ops)
 {
-	mutex_lock(&nvmf_transports_mutex);
+	down_write(&nvmf_transports_rwsem);
 	list_del(&ops->entry);
-	mutex_unlock(&nvmf_transports_mutex);
+	up_write(&nvmf_transports_rwsem);
 }
 EXPORT_SYMBOL_GPL(nvmf_unregister_transport);
 
@@ -525,7 +525,7 @@ static struct nvmf_transport_ops *nvmf_lookup_transport(
 {
 	struct nvmf_transport_ops *ops;
 
-	lockdep_assert_held(&nvmf_transports_mutex);
+	lockdep_assert_held(&nvmf_transports_rwsem);
 
 	list_for_each_entry(ops, &nvmf_transports, entry) {
 		if (strcmp(ops->name, opts->transport) == 0)
@@ -735,6 +735,7 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts,
 				goto out;
 			}
 			if (uuid_parse(p, &hostid)) {
+				pr_err("Invalid hostid %s\n", p);
 				ret = -EINVAL;
 				goto out;
 			}
@@ -794,7 +795,8 @@ static int nvmf_check_allowed_opts(struct nvmf_ctrl_options *opts,
 		int i;
 
 		for (i = 0; i < ARRAY_SIZE(opt_tokens); i++) {
-			if (opt_tokens[i].token & ~allowed_opts) {
+			if ((opt_tokens[i].token & opts->mask) &&
+			    (opt_tokens[i].token & ~allowed_opts)) {
 				pr_warn("invalid parameter '%s'\n",
 					opt_tokens[i].pattern);
 			}
@@ -849,7 +851,7 @@ nvmf_create_ctrl(struct device *dev, const char *buf, size_t count)
 		goto out_free_opts;
 	opts->mask &= ~NVMF_REQUIRED_OPTS;
 
-	mutex_lock(&nvmf_transports_mutex);
+	down_read(&nvmf_transports_rwsem);
 	ops = nvmf_lookup_transport(opts);
 	if (!ops) {
 		pr_info("no handler found for transport %s.\n",
@@ -876,16 +878,16 @@ nvmf_create_ctrl(struct device *dev, const char *buf, size_t count)
 		dev_warn(ctrl->device,
 			"controller returned incorrect NQN: \"%s\".\n",
 			ctrl->subnqn);
-		mutex_unlock(&nvmf_transports_mutex);
+		up_read(&nvmf_transports_rwsem);
 		ctrl->ops->delete_ctrl(ctrl);
 		return ERR_PTR(-EINVAL);
 	}
 
-	mutex_unlock(&nvmf_transports_mutex);
+	up_read(&nvmf_transports_rwsem);
 	return ctrl;
 
 out_unlock:
-	mutex_unlock(&nvmf_transports_mutex);
+	up_read(&nvmf_transports_rwsem);
 out_free_opts:
 	nvmf_free_options(opts);
 	return ERR_PTR(ret);
diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c
index 5c2a08ef08ba..d2e882c0f496 100644
--- a/drivers/nvme/host/fc.c
+++ b/drivers/nvme/host/fc.c
@@ -220,6 +220,90 @@ static int __nvme_fc_del_ctrl(struct nvme_fc_ctrl *);
 static void __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *,
 			struct nvme_fc_queue *, unsigned int);
 
+static void
+nvme_fc_free_lport(struct kref *ref)
+{
+	struct nvme_fc_lport *lport =
+		container_of(ref, struct nvme_fc_lport, ref);
+	unsigned long flags;
+
+	WARN_ON(lport->localport.port_state != FC_OBJSTATE_DELETED);
+	WARN_ON(!list_empty(&lport->endp_list));
+
+	/* remove from transport list */
+	spin_lock_irqsave(&nvme_fc_lock, flags);
+	list_del(&lport->port_list);
+	spin_unlock_irqrestore(&nvme_fc_lock, flags);
+
+	/* let the LLDD know we've finished tearing it down */
+	lport->ops->localport_delete(&lport->localport);
+
+	ida_simple_remove(&nvme_fc_local_port_cnt, lport->localport.port_num);
+	ida_destroy(&lport->endp_cnt);
+
+	put_device(lport->dev);
+
+	kfree(lport);
+}
+
+static void
+nvme_fc_lport_put(struct nvme_fc_lport *lport)
+{
+	kref_put(&lport->ref, nvme_fc_free_lport);
+}
+
+static int
+nvme_fc_lport_get(struct nvme_fc_lport *lport)
+{
+	return kref_get_unless_zero(&lport->ref);
+}
+
+
+static struct nvme_fc_lport *
+nvme_fc_attach_to_unreg_lport(struct nvme_fc_port_info *pinfo)
+{
+	struct nvme_fc_lport *lport;
+	unsigned long flags;
+
+	spin_lock_irqsave(&nvme_fc_lock, flags);
+
+	list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
+		if (lport->localport.node_name != pinfo->node_name ||
+		    lport->localport.port_name != pinfo->port_name)
+			continue;
+
+		if (lport->localport.port_state != FC_OBJSTATE_DELETED) {
+			lport = ERR_PTR(-EEXIST);
+			goto out_done;
+		}
+
+		if (!nvme_fc_lport_get(lport)) {
+			/*
+			 * fails if ref cnt already 0. If so,
+			 * act as if lport already deleted
+			 */
+			lport = NULL;
+			goto out_done;
+		}
+
+		/* resume the lport */
+
+		lport->localport.port_role = pinfo->port_role;
+		lport->localport.port_id = pinfo->port_id;
+		lport->localport.port_state = FC_OBJSTATE_ONLINE;
+
+		spin_unlock_irqrestore(&nvme_fc_lock, flags);
+
+		return lport;
+	}
+
+	lport = NULL;
+
+out_done:
+	spin_unlock_irqrestore(&nvme_fc_lock, flags);
+
+	return lport;
+}
 
 /**
  * nvme_fc_register_localport - transport entry point called by an
@@ -257,6 +341,28 @@ nvme_fc_register_localport(struct nvme_fc_port_info *pinfo,
 		goto out_reghost_failed;
 	}
 
+	/*
+	 * look to see if there is already a localport that had been
+	 * deregistered and in the process of waiting for all the
+	 * references to fully be removed.  If the references haven't
+	 * expired, we can simply re-enable the localport. Remoteports
+	 * and controller reconnections should resume naturally.
+	 */
+	newrec = nvme_fc_attach_to_unreg_lport(pinfo);
+
+	/* found an lport, but something about its state is bad */
+	if (IS_ERR(newrec)) {
+		ret = PTR_ERR(newrec);
+		goto out_reghost_failed;
+
+	/* found existing lport, which was resumed */
+	} else if (newrec) {
+		*portptr = &newrec->localport;
+		return 0;
+	}
+
+	/* nothing found - allocate a new localport struct */
+
 	newrec = kmalloc((sizeof(*newrec) + template->local_priv_sz),
 			 GFP_KERNEL);
 	if (!newrec) {
@@ -310,44 +416,6 @@ out_reghost_failed:
 }
 EXPORT_SYMBOL_GPL(nvme_fc_register_localport);
 
-static void
-nvme_fc_free_lport(struct kref *ref)
-{
-	struct nvme_fc_lport *lport =
-		container_of(ref, struct nvme_fc_lport, ref);
-	unsigned long flags;
-
-	WARN_ON(lport->localport.port_state != FC_OBJSTATE_DELETED);
-	WARN_ON(!list_empty(&lport->endp_list));
-
-	/* remove from transport list */
-	spin_lock_irqsave(&nvme_fc_lock, flags);
-	list_del(&lport->port_list);
-	spin_unlock_irqrestore(&nvme_fc_lock, flags);
-
-	/* let the LLDD know we've finished tearing it down */
-	lport->ops->localport_delete(&lport->localport);
-
-	ida_simple_remove(&nvme_fc_local_port_cnt, lport->localport.port_num);
-	ida_destroy(&lport->endp_cnt);
-
-	put_device(lport->dev);
-
-	kfree(lport);
-}
-
-static void
-nvme_fc_lport_put(struct nvme_fc_lport *lport)
-{
-	kref_put(&lport->ref, nvme_fc_free_lport);
-}
-
-static int
-nvme_fc_lport_get(struct nvme_fc_lport *lport)
-{
-	return kref_get_unless_zero(&lport->ref);
-}
-
 /**
  * nvme_fc_unregister_localport - transport entry point called by an
  *                              LLDD to deregister/remove a previously
@@ -2168,7 +2236,6 @@ static const struct blk_mq_ops nvme_fc_mq_ops = {
 	.complete	= nvme_fc_complete_rq,
 	.init_request	= nvme_fc_init_request,
 	.exit_request	= nvme_fc_exit_request,
-	.reinit_request	= nvme_fc_reinit_request,
 	.init_hctx	= nvme_fc_init_hctx,
 	.poll		= nvme_fc_poll,
 	.timeout	= nvme_fc_timeout,
@@ -2269,7 +2336,7 @@ nvme_fc_reinit_io_queues(struct nvme_fc_ctrl *ctrl)
 
 	nvme_fc_init_io_queues(ctrl);
 
-	ret = blk_mq_reinit_tagset(&ctrl->tag_set);
+	ret = blk_mq_reinit_tagset(&ctrl->tag_set, nvme_fc_reinit_request);
 	if (ret)
 		goto out_free_io_queues;
 
@@ -2655,7 +2722,6 @@ static const struct blk_mq_ops nvme_fc_admin_mq_ops = {
 	.complete	= nvme_fc_complete_rq,
 	.init_request	= nvme_fc_init_request,
 	.exit_request	= nvme_fc_exit_request,
-	.reinit_request	= nvme_fc_reinit_request,
 	.init_hctx	= nvme_fc_init_admin_hctx,
 	.timeout	= nvme_fc_timeout,
 };
@@ -2733,6 +2799,7 @@ nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
 	ret = blk_mq_alloc_tag_set(&ctrl->admin_tag_set);
 	if (ret)
 		goto out_free_queues;
+	ctrl->ctrl.admin_tagset = &ctrl->admin_tag_set;
 
 	ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set);
 	if (IS_ERR(ctrl->ctrl.admin_q)) {
diff --git a/drivers/nvme/host/lightnvm.c b/drivers/nvme/host/lightnvm.c
index be8541335e31..1f79e3f141e6 100644
--- a/drivers/nvme/host/lightnvm.c
+++ b/drivers/nvme/host/lightnvm.c
@@ -643,17 +643,9 @@ static int nvme_nvm_submit_user_cmd(struct request_queue *q,
 			vcmd->ph_rw.metadata = cpu_to_le64(metadata_dma);
 		}
 
-		if (!disk)
-			goto submit;
-
-		bio->bi_bdev = bdget_disk(disk, 0);
-		if (!bio->bi_bdev) {
-			ret = -ENODEV;
-			goto err_meta;
-		}
+		bio->bi_disk = disk;
 	}
 
-submit:
 	blk_execute_rq(q, NULL, rq, 0);
 
 	if (nvme_req(rq)->flags & NVME_REQ_CANCELLED)
@@ -673,11 +665,8 @@ err_meta:
 	if (meta_buf && meta_len)
 		dma_pool_free(dev->dma_pool, metadata, metadata_dma);
 err_map:
-	if (bio) {
-		if (disk && bio->bi_bdev)
-			bdput(bio->bi_bdev);
+	if (bio)
 		blk_rq_unmap_user(bio);
-	}
 err_ppa:
 	if (ppa_buf && ppa_len)
 		dma_pool_free(dev->dma_pool, ppa_list, ppa_dma);
@@ -966,29 +955,3 @@ void nvme_nvm_unregister_sysfs(struct nvme_ns *ns)
 	sysfs_remove_group(&disk_to_dev(ns->disk)->kobj,
 					&nvm_dev_attr_group);
 }
-
-/* move to shared place when used in multiple places. */
-#define PCI_VENDOR_ID_CNEX 0x1d1d
-#define PCI_DEVICE_ID_CNEX_WL 0x2807
-#define PCI_DEVICE_ID_CNEX_QEMU 0x1f1f
-
-int nvme_nvm_ns_supported(struct nvme_ns *ns, struct nvme_id_ns *id)
-{
-	struct nvme_ctrl *ctrl = ns->ctrl;
-	/* XXX: this is poking into PCI structures from generic code! */
-	struct pci_dev *pdev = to_pci_dev(ctrl->dev);
-
-	/* QEMU NVMe simulator - PCI ID + Vendor specific bit */
-	if (pdev->vendor == PCI_VENDOR_ID_CNEX &&
-				pdev->device == PCI_DEVICE_ID_CNEX_QEMU &&
-							id->vs[0] == 0x1)
-		return 1;
-
-	/* CNEX Labs - PCI ID + Vendor specific bit */
-	if (pdev->vendor == PCI_VENDOR_ID_CNEX &&
-				pdev->device == PCI_DEVICE_ID_CNEX_WL &&
-							id->vs[0] == 0x1)
-		return 1;
-
-	return 0;
-}
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 8f2a168ddc01..d3f3c4447515 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -75,6 +75,11 @@ enum nvme_quirks {
 	 * The deepest sleep state should not be used.
 	 */
 	NVME_QUIRK_NO_DEEPEST_PS		= (1 << 5),
+
+	/*
+	 * Supports the LighNVM command set if indicated in vs[1].
+	 */
+	NVME_QUIRK_LIGHTNVM			= (1 << 6),
 };
 
 /*
@@ -125,6 +130,7 @@ struct nvme_ctrl {
 	struct kref kref;
 	int instance;
 	struct blk_mq_tag_set *tagset;
+	struct blk_mq_tag_set *admin_tagset;
 	struct list_head namespaces;
 	struct mutex namespaces_mutex;
 	struct device *device;	/* char device */
@@ -142,6 +148,7 @@ struct nvme_ctrl {
 	u16 cntlid;
 
 	u32 ctrl_config;
+	u16 mtfa;
 	u32 queue_count;
 
 	u64 cap;
@@ -160,6 +167,7 @@ struct nvme_ctrl {
 	u16 kas;
 	u8 npss;
 	u8 apsta;
+	unsigned int shutdown_timeout;
 	unsigned int kato;
 	bool subsystem;
 	unsigned long quirks;
@@ -167,13 +175,17 @@ struct nvme_ctrl {
 	struct work_struct scan_work;
 	struct work_struct async_event_work;
 	struct delayed_work ka_work;
+	struct work_struct fw_act_work;
 
 	/* Power saving configuration */
 	u64 ps_max_latency_us;
 	bool apst_enabled;
 
+	/* PCIe only: */
 	u32 hmpre;
 	u32 hmmin;
+	u32 hmminds;
+	u16 hmmaxd;
 
 	/* Fabrics only */
 	u16 sqsize;
@@ -207,13 +219,9 @@ struct nvme_ns {
 	bool ext;
 	u8 pi_type;
 	unsigned long flags;
-	u16 noiob;
-
 #define NVME_NS_REMOVING 0
 #define NVME_NS_DEAD     1
-
-	u64 mode_select_num_blocks;
-	u32 mode_select_block_len;
+	u16 noiob;
 };
 
 struct nvme_ctrl_ops {
@@ -314,20 +322,12 @@ int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
 		union nvme_result *result, void *buffer, unsigned bufflen,
 		unsigned timeout, int qid, int at_head, int flags);
-int nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
-		void __user *ubuffer, unsigned bufflen, u32 *result,
-		unsigned timeout);
-int __nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
-		void __user *ubuffer, unsigned bufflen,
-		void __user *meta_buffer, unsigned meta_len, u32 meta_seed,
-		u32 *result, unsigned timeout);
 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count);
 void nvme_start_keep_alive(struct nvme_ctrl *ctrl);
 void nvme_stop_keep_alive(struct nvme_ctrl *ctrl);
 int nvme_reset_ctrl(struct nvme_ctrl *ctrl);
 
 #ifdef CONFIG_NVM
-int nvme_nvm_ns_supported(struct nvme_ns *ns, struct nvme_id_ns *id);
 int nvme_nvm_register(struct nvme_ns *ns, char *disk_name, int node);
 void nvme_nvm_unregister(struct nvme_ns *ns);
 int nvme_nvm_register_sysfs(struct nvme_ns *ns);
@@ -346,10 +346,6 @@ static inline int nvme_nvm_register_sysfs(struct nvme_ns *ns)
 	return 0;
 }
 static inline void nvme_nvm_unregister_sysfs(struct nvme_ns *ns) {};
-static inline int nvme_nvm_ns_supported(struct nvme_ns *ns, struct nvme_id_ns *id)
-{
-	return 0;
-}
 static inline int nvme_nvm_ioctl(struct nvme_ns *ns, unsigned int cmd,
 							unsigned long arg)
 {
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 74a124a06264..4a2121335f48 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -109,6 +109,7 @@ struct nvme_dev {
 	/* host memory buffer support: */
 	u64 host_mem_size;
 	u32 nr_host_mem_descs;
+	dma_addr_t host_mem_descs_dma;
 	struct nvme_host_mem_buf_desc *host_mem_descs;
 	void **host_mem_desc_bufs;
 };
@@ -555,8 +556,10 @@ static blk_status_t nvme_setup_prps(struct nvme_dev *dev, struct request *req)
 	int nprps, i;
 
 	length -= (page_size - offset);
-	if (length <= 0)
+	if (length <= 0) {
+		iod->first_dma = 0;
 		return BLK_STS_OK;
+	}
 
 	dma_len -= (page_size - offset);
 	if (dma_len) {
@@ -666,7 +669,7 @@ static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req,
 		if (blk_rq_map_integrity_sg(q, req->bio, &iod->meta_sg) != 1)
 			goto out_unmap;
 
-		if (rq_data_dir(req))
+		if (req_op(req) == REQ_OP_WRITE)
 			nvme_dif_remap(req, nvme_dif_prep);
 
 		if (!dma_map_sg(dev->dev, &iod->meta_sg, 1, dma_dir))
@@ -694,7 +697,7 @@ static void nvme_unmap_data(struct nvme_dev *dev, struct request *req)
 	if (iod->nents) {
 		dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
 		if (blk_integrity_rq(req)) {
-			if (!rq_data_dir(req))
+			if (req_op(req) == REQ_OP_READ)
 				nvme_dif_remap(req, nvme_dif_complete);
 			dma_unmap_sg(dev->dev, &iod->meta_sg, 1, dma_dir);
 		}
@@ -801,6 +804,7 @@ static inline void nvme_handle_cqe(struct nvme_queue *nvmeq,
 		return;
 	}
 
+	nvmeq->cqe_seen = 1;
 	req = blk_mq_tag_to_rq(*nvmeq->tags, cqe->command_id);
 	nvme_end_request(req, cqe->status, cqe->result);
 }
@@ -830,10 +834,8 @@ static void nvme_process_cq(struct nvme_queue *nvmeq)
 		consumed++;
 	}
 
-	if (consumed) {
+	if (consumed)
 		nvme_ring_cq_doorbell(nvmeq);
-		nvmeq->cqe_seen = 1;
-	}
 }
 
 static irqreturn_t nvme_irq(int irq, void *data)
@@ -1377,6 +1379,7 @@ static int nvme_alloc_admin_tags(struct nvme_dev *dev)
 
 		if (blk_mq_alloc_tag_set(&dev->admin_tagset))
 			return -ENOMEM;
+		dev->ctrl.admin_tagset = &dev->admin_tagset;
 
 		dev->ctrl.admin_q = blk_mq_init_queue(&dev->admin_tagset);
 		if (IS_ERR(dev->ctrl.admin_q)) {
@@ -1566,16 +1569,10 @@ static inline void nvme_release_cmb(struct nvme_dev *dev)
 
 static int nvme_set_host_mem(struct nvme_dev *dev, u32 bits)
 {
-	size_t len = dev->nr_host_mem_descs * sizeof(*dev->host_mem_descs);
+	u64 dma_addr = dev->host_mem_descs_dma;
 	struct nvme_command c;
-	u64 dma_addr;
 	int ret;
 
-	dma_addr = dma_map_single(dev->dev, dev->host_mem_descs, len,
-			DMA_TO_DEVICE);
-	if (dma_mapping_error(dev->dev, dma_addr))
-		return -ENOMEM;
-
 	memset(&c, 0, sizeof(c));
 	c.features.opcode	= nvme_admin_set_features;
 	c.features.fid		= cpu_to_le32(NVME_FEAT_HOST_MEM_BUF);
@@ -1592,7 +1589,6 @@ static int nvme_set_host_mem(struct nvme_dev *dev, u32 bits)
 			 "failed to set host mem (err %d, flags %#x).\n",
 			 ret, bits);
 	}
-	dma_unmap_single(dev->dev, dma_addr, len, DMA_TO_DEVICE);
 	return ret;
 }
 
@@ -1610,25 +1606,31 @@ static void nvme_free_host_mem(struct nvme_dev *dev)
 
 	kfree(dev->host_mem_desc_bufs);
 	dev->host_mem_desc_bufs = NULL;
-	kfree(dev->host_mem_descs);
+	dma_free_coherent(dev->dev,
+			dev->nr_host_mem_descs * sizeof(*dev->host_mem_descs),
+			dev->host_mem_descs, dev->host_mem_descs_dma);
 	dev->host_mem_descs = NULL;
 }
 
-static int nvme_alloc_host_mem(struct nvme_dev *dev, u64 min, u64 preferred)
+static int __nvme_alloc_host_mem(struct nvme_dev *dev, u64 preferred,
+		u32 chunk_size)
 {
 	struct nvme_host_mem_buf_desc *descs;
-	u32 chunk_size, max_entries, len;
+	u32 max_entries, len;
+	dma_addr_t descs_dma;
 	int i = 0;
 	void **bufs;
 	u64 size = 0, tmp;
 
-	/* start big and work our way down */
-	chunk_size = min(preferred, (u64)PAGE_SIZE << MAX_ORDER);
-retry:
 	tmp = (preferred + chunk_size - 1);
 	do_div(tmp, chunk_size);
 	max_entries = tmp;
-	descs = kcalloc(max_entries, sizeof(*descs), GFP_KERNEL);
+
+	if (dev->ctrl.hmmaxd && dev->ctrl.hmmaxd < max_entries)
+		max_entries = dev->ctrl.hmmaxd;
+
+	descs = dma_zalloc_coherent(dev->dev, max_entries * sizeof(*descs),
+			&descs_dma, GFP_KERNEL);
 	if (!descs)
 		goto out;
 
@@ -1650,18 +1652,13 @@ retry:
 		i++;
 	}
 
-	if (!size || (min && size < min)) {
-		dev_warn(dev->ctrl.device,
-			"failed to allocate host memory buffer.\n");
+	if (!size)
 		goto out_free_bufs;
-	}
 
-	dev_info(dev->ctrl.device,
-		"allocated %lld MiB host memory buffer.\n",
-		size >> ilog2(SZ_1M));
 	dev->nr_host_mem_descs = i;
 	dev->host_mem_size = size;
 	dev->host_mem_descs = descs;
+	dev->host_mem_descs_dma = descs_dma;
 	dev->host_mem_desc_bufs = bufs;
 	return 0;
 
@@ -1675,23 +1672,38 @@ out_free_bufs:
 
 	kfree(bufs);
 out_free_descs:
-	kfree(descs);
+	dma_free_coherent(dev->dev, max_entries * sizeof(*descs), descs,
+			descs_dma);
 out:
-	/* try a smaller chunk size if we failed early */
-	if (chunk_size >= PAGE_SIZE * 2 && (i == 0 || size < min)) {
-		chunk_size /= 2;
-		goto retry;
-	}
 	dev->host_mem_descs = NULL;
 	return -ENOMEM;
 }
 
-static void nvme_setup_host_mem(struct nvme_dev *dev)
+static int nvme_alloc_host_mem(struct nvme_dev *dev, u64 min, u64 preferred)
+{
+	u32 chunk_size;
+
+	/* start big and work our way down */
+	for (chunk_size = min_t(u64, preferred, PAGE_SIZE * MAX_ORDER_NR_PAGES);
+	     chunk_size >= max_t(u32, dev->ctrl.hmminds * 4096, PAGE_SIZE * 2);
+	     chunk_size /= 2) {
+		if (!__nvme_alloc_host_mem(dev, preferred, chunk_size)) {
+			if (!min || dev->host_mem_size >= min)
+				return 0;
+			nvme_free_host_mem(dev);
+		}
+	}
+
+	return -ENOMEM;
+}
+
+static int nvme_setup_host_mem(struct nvme_dev *dev)
 {
 	u64 max = (u64)max_host_mem_size_mb * SZ_1M;
 	u64 preferred = (u64)dev->ctrl.hmpre * 4096;
 	u64 min = (u64)dev->ctrl.hmmin * 4096;
 	u32 enable_bits = NVME_HOST_MEM_ENABLE;
+	int ret = 0;
 
 	preferred = min(preferred, max);
 	if (min > max) {
@@ -1699,7 +1711,7 @@ static void nvme_setup_host_mem(struct nvme_dev *dev)
 			"min host memory (%lld MiB) above limit (%d MiB).\n",
 			min >> ilog2(SZ_1M), max_host_mem_size_mb);
 		nvme_free_host_mem(dev);
-		return;
+		return 0;
 	}
 
 	/*
@@ -1713,12 +1725,21 @@ static void nvme_setup_host_mem(struct nvme_dev *dev)
 	}
 
 	if (!dev->host_mem_descs) {
-		if (nvme_alloc_host_mem(dev, min, preferred))
-			return;
+		if (nvme_alloc_host_mem(dev, min, preferred)) {
+			dev_warn(dev->ctrl.device,
+				"failed to allocate host memory buffer.\n");
+			return 0; /* controller must work without HMB */
+		}
+
+		dev_info(dev->ctrl.device,
+			"allocated %lld MiB host memory buffer.\n",
+			dev->host_mem_size >> ilog2(SZ_1M));
 	}
 
-	if (nvme_set_host_mem(dev, enable_bits))
+	ret = nvme_set_host_mem(dev, enable_bits);
+	if (ret)
 		nvme_free_host_mem(dev);
+	return ret;
 }
 
 static int nvme_setup_io_queues(struct nvme_dev *dev)
@@ -2162,8 +2183,11 @@ static void nvme_reset_work(struct work_struct *work)
 				 "unable to allocate dma for dbbuf\n");
 	}
 
-	if (dev->ctrl.hmpre)
-		nvme_setup_host_mem(dev);
+	if (dev->ctrl.hmpre) {
+		result = nvme_setup_host_mem(dev);
+		if (result < 0)
+			goto out;
+	}
 
 	result = nvme_setup_io_queues(dev);
 	if (result)
@@ -2495,6 +2519,10 @@ static const struct pci_device_id nvme_id_table[] = {
 		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
 	{ PCI_DEVICE(0x144d, 0xa822),   /* Samsung PM1725a */
 		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
+	{ PCI_DEVICE(0x1d1d, 0x1f1f),	/* LighNVM qemu device */
+		.driver_data = NVME_QUIRK_LIGHTNVM, },
+	{ PCI_DEVICE(0x1d1d, 0x2807),	/* CNEX WL */
+		.driver_data = NVME_QUIRK_LIGHTNVM, },
 	{ PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2003) },
diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
index da04df1af231..58983000964b 100644
--- a/drivers/nvme/host/rdma.c
+++ b/drivers/nvme/host/rdma.c
@@ -19,6 +19,7 @@
 #include <linux/string.h>
 #include <linux/atomic.h>
 #include <linux/blk-mq.h>
+#include <linux/blk-mq-rdma.h>
 #include <linux/types.h>
 #include <linux/list.h>
 #include <linux/mutex.h>
@@ -36,8 +37,6 @@
 
 #define NVME_RDMA_CONNECT_TIMEOUT_MS	3000		/* 3 second */
 
-#define NVME_RDMA_MAX_SEGMENT_SIZE	0xffffff	/* 24-bit SGL field */
-
 #define NVME_RDMA_MAX_SEGMENTS		256
 
 #define NVME_RDMA_MAX_INLINE_SEGMENTS	1
@@ -151,6 +150,9 @@ static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,
 		struct rdma_cm_event *event);
 static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc);
 
+static const struct blk_mq_ops nvme_rdma_mq_ops;
+static const struct blk_mq_ops nvme_rdma_admin_mq_ops;
+
 /* XXX: really should move to a generic header sooner or later.. */
 static inline void put_unaligned_le24(u32 val, u8 *p)
 {
@@ -463,14 +465,10 @@ static int nvme_rdma_create_queue_ib(struct nvme_rdma_queue *queue)
 	ibdev = queue->device->dev;
 
 	/*
-	 * The admin queue is barely used once the controller is live, so don't
-	 * bother to spread it out.
+	 * Spread I/O queues completion vectors according their queue index.
+	 * Admin queues can always go on completion vector 0.
 	 */
-	if (idx == 0)
-		comp_vector = 0;
-	else
-		comp_vector = idx % ibdev->num_comp_vectors;
-
+	comp_vector = idx == 0 ? idx : idx - 1;
 
 	/* +1 for ib_stop_cq */
 	queue->ib_cq = ib_alloc_cq(ibdev, queue,
@@ -503,7 +501,7 @@ out_put_dev:
 	return ret;
 }
 
-static int nvme_rdma_init_queue(struct nvme_rdma_ctrl *ctrl,
+static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl,
 		int idx, size_t queue_size)
 {
 	struct nvme_rdma_queue *queue;
@@ -561,60 +559,90 @@ out_destroy_cm_id:
 
 static void nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)
 {
+	if (!test_and_clear_bit(NVME_RDMA_Q_LIVE, &queue->flags))
+		return;
+
 	rdma_disconnect(queue->cm_id);
 	ib_drain_qp(queue->qp);
 }
 
 static void nvme_rdma_free_queue(struct nvme_rdma_queue *queue)
 {
+	if (test_and_set_bit(NVME_RDMA_Q_DELETING, &queue->flags))
+		return;
+
 	nvme_rdma_destroy_queue_ib(queue);
 	rdma_destroy_id(queue->cm_id);
 }
 
-static void nvme_rdma_stop_and_free_queue(struct nvme_rdma_queue *queue)
+static void nvme_rdma_free_io_queues(struct nvme_rdma_ctrl *ctrl)
 {
-	if (test_and_set_bit(NVME_RDMA_Q_DELETING, &queue->flags))
-		return;
-	nvme_rdma_stop_queue(queue);
-	nvme_rdma_free_queue(queue);
+	int i;
+
+	for (i = 1; i < ctrl->ctrl.queue_count; i++)
+		nvme_rdma_free_queue(&ctrl->queues[i]);
 }
 
-static void nvme_rdma_free_io_queues(struct nvme_rdma_ctrl *ctrl)
+static void nvme_rdma_stop_io_queues(struct nvme_rdma_ctrl *ctrl)
 {
 	int i;
 
 	for (i = 1; i < ctrl->ctrl.queue_count; i++)
-		nvme_rdma_stop_and_free_queue(&ctrl->queues[i]);
+		nvme_rdma_stop_queue(&ctrl->queues[i]);
+}
+
+static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx)
+{
+	int ret;
+
+	if (idx)
+		ret = nvmf_connect_io_queue(&ctrl->ctrl, idx);
+	else
+		ret = nvmf_connect_admin_queue(&ctrl->ctrl);
+
+	if (!ret)
+		set_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[idx].flags);
+	else
+		dev_info(ctrl->ctrl.device,
+			"failed to connect queue: %d ret=%d\n", idx, ret);
+	return ret;
 }
 
-static int nvme_rdma_connect_io_queues(struct nvme_rdma_ctrl *ctrl)
+static int nvme_rdma_start_io_queues(struct nvme_rdma_ctrl *ctrl)
 {
 	int i, ret = 0;
 
 	for (i = 1; i < ctrl->ctrl.queue_count; i++) {
-		ret = nvmf_connect_io_queue(&ctrl->ctrl, i);
-		if (ret) {
-			dev_info(ctrl->ctrl.device,
-				"failed to connect i/o queue: %d\n", ret);
-			goto out_free_queues;
-		}
-		set_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[i].flags);
+		ret = nvme_rdma_start_queue(ctrl, i);
+		if (ret)
+			goto out_stop_queues;
 	}
 
 	return 0;
 
-out_free_queues:
-	nvme_rdma_free_io_queues(ctrl);
+out_stop_queues:
+	for (i--; i >= 1; i--)
+		nvme_rdma_stop_queue(&ctrl->queues[i]);
 	return ret;
 }
 
-static int nvme_rdma_init_io_queues(struct nvme_rdma_ctrl *ctrl)
+static int nvme_rdma_alloc_io_queues(struct nvme_rdma_ctrl *ctrl)
 {
 	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
+	struct ib_device *ibdev = ctrl->device->dev;
 	unsigned int nr_io_queues;
 	int i, ret;
 
 	nr_io_queues = min(opts->nr_io_queues, num_online_cpus());
+
+	/*
+	 * we map queues according to the device irq vectors for
+	 * optimal locality so we don't need more queues than
+	 * completion vectors.
+	 */
+	nr_io_queues = min_t(unsigned int, nr_io_queues,
+				ibdev->num_comp_vectors);
+
 	ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
 	if (ret)
 		return ret;
@@ -627,32 +655,230 @@ static int nvme_rdma_init_io_queues(struct nvme_rdma_ctrl *ctrl)
 		"creating %d I/O queues.\n", nr_io_queues);
 
 	for (i = 1; i < ctrl->ctrl.queue_count; i++) {
-		ret = nvme_rdma_init_queue(ctrl, i,
-					   ctrl->ctrl.opts->queue_size);
-		if (ret) {
-			dev_info(ctrl->ctrl.device,
-				"failed to initialize i/o queue: %d\n", ret);
+		ret = nvme_rdma_alloc_queue(ctrl, i,
+				ctrl->ctrl.sqsize + 1);
+		if (ret)
 			goto out_free_queues;
-		}
 	}
 
 	return 0;
 
 out_free_queues:
 	for (i--; i >= 1; i--)
-		nvme_rdma_stop_and_free_queue(&ctrl->queues[i]);
+		nvme_rdma_free_queue(&ctrl->queues[i]);
 
 	return ret;
 }
 
-static void nvme_rdma_destroy_admin_queue(struct nvme_rdma_ctrl *ctrl)
+static void nvme_rdma_free_tagset(struct nvme_ctrl *nctrl, bool admin)
+{
+	struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
+	struct blk_mq_tag_set *set = admin ?
+			&ctrl->admin_tag_set : &ctrl->tag_set;
+
+	blk_mq_free_tag_set(set);
+	nvme_rdma_dev_put(ctrl->device);
+}
+
+static struct blk_mq_tag_set *nvme_rdma_alloc_tagset(struct nvme_ctrl *nctrl,
+		bool admin)
+{
+	struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
+	struct blk_mq_tag_set *set;
+	int ret;
+
+	if (admin) {
+		set = &ctrl->admin_tag_set;
+		memset(set, 0, sizeof(*set));
+		set->ops = &nvme_rdma_admin_mq_ops;
+		set->queue_depth = NVME_RDMA_AQ_BLKMQ_DEPTH;
+		set->reserved_tags = 2; /* connect + keep-alive */
+		set->numa_node = NUMA_NO_NODE;
+		set->cmd_size = sizeof(struct nvme_rdma_request) +
+			SG_CHUNK_SIZE * sizeof(struct scatterlist);
+		set->driver_data = ctrl;
+		set->nr_hw_queues = 1;
+		set->timeout = ADMIN_TIMEOUT;
+	} else {
+		set = &ctrl->tag_set;
+		memset(set, 0, sizeof(*set));
+		set->ops = &nvme_rdma_mq_ops;
+		set->queue_depth = nctrl->opts->queue_size;
+		set->reserved_tags = 1; /* fabric connect */
+		set->numa_node = NUMA_NO_NODE;
+		set->flags = BLK_MQ_F_SHOULD_MERGE;
+		set->cmd_size = sizeof(struct nvme_rdma_request) +
+			SG_CHUNK_SIZE * sizeof(struct scatterlist);
+		set->driver_data = ctrl;
+		set->nr_hw_queues = nctrl->queue_count - 1;
+		set->timeout = NVME_IO_TIMEOUT;
+	}
+
+	ret = blk_mq_alloc_tag_set(set);
+	if (ret)
+		goto out;
+
+	/*
+	 * We need a reference on the device as long as the tag_set is alive,
+	 * as the MRs in the request structures need a valid ib_device.
+	 */
+	ret = nvme_rdma_dev_get(ctrl->device);
+	if (!ret) {
+		ret = -EINVAL;
+		goto out_free_tagset;
+	}
+
+	return set;
+
+out_free_tagset:
+	blk_mq_free_tag_set(set);
+out:
+	return ERR_PTR(ret);
+}
+
+static void nvme_rdma_destroy_admin_queue(struct nvme_rdma_ctrl *ctrl,
+		bool remove)
 {
 	nvme_rdma_free_qe(ctrl->queues[0].device->dev, &ctrl->async_event_sqe,
 			sizeof(struct nvme_command), DMA_TO_DEVICE);
-	nvme_rdma_stop_and_free_queue(&ctrl->queues[0]);
-	blk_cleanup_queue(ctrl->ctrl.admin_q);
-	blk_mq_free_tag_set(&ctrl->admin_tag_set);
-	nvme_rdma_dev_put(ctrl->device);
+	nvme_rdma_stop_queue(&ctrl->queues[0]);
+	if (remove) {
+		blk_cleanup_queue(ctrl->ctrl.admin_q);
+		nvme_rdma_free_tagset(&ctrl->ctrl, true);
+	}
+	nvme_rdma_free_queue(&ctrl->queues[0]);
+}
+
+static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
+		bool new)
+{
+	int error;
+
+	error = nvme_rdma_alloc_queue(ctrl, 0, NVME_AQ_DEPTH);
+	if (error)
+		return error;
+
+	ctrl->device = ctrl->queues[0].device;
+
+	ctrl->max_fr_pages = min_t(u32, NVME_RDMA_MAX_SEGMENTS,
+		ctrl->device->dev->attrs.max_fast_reg_page_list_len);
+
+	if (new) {
+		ctrl->ctrl.admin_tagset = nvme_rdma_alloc_tagset(&ctrl->ctrl, true);
+		if (IS_ERR(ctrl->ctrl.admin_tagset))
+			goto out_free_queue;
+
+		ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set);
+		if (IS_ERR(ctrl->ctrl.admin_q)) {
+			error = PTR_ERR(ctrl->ctrl.admin_q);
+			goto out_free_tagset;
+		}
+	} else {
+		error = blk_mq_reinit_tagset(&ctrl->admin_tag_set,
+					     nvme_rdma_reinit_request);
+		if (error)
+			goto out_free_queue;
+	}
+
+	error = nvme_rdma_start_queue(ctrl, 0);
+	if (error)
+		goto out_cleanup_queue;
+
+	error = ctrl->ctrl.ops->reg_read64(&ctrl->ctrl, NVME_REG_CAP,
+			&ctrl->ctrl.cap);
+	if (error) {
+		dev_err(ctrl->ctrl.device,
+			"prop_get NVME_REG_CAP failed\n");
+		goto out_cleanup_queue;
+	}
+
+	ctrl->ctrl.sqsize =
+		min_t(int, NVME_CAP_MQES(ctrl->ctrl.cap), ctrl->ctrl.sqsize);
+
+	error = nvme_enable_ctrl(&ctrl->ctrl, ctrl->ctrl.cap);
+	if (error)
+		goto out_cleanup_queue;
+
+	ctrl->ctrl.max_hw_sectors =
+		(ctrl->max_fr_pages - 1) << (ilog2(SZ_4K) - 9);
+
+	error = nvme_init_identify(&ctrl->ctrl);
+	if (error)
+		goto out_cleanup_queue;
+
+	error = nvme_rdma_alloc_qe(ctrl->queues[0].device->dev,
+			&ctrl->async_event_sqe, sizeof(struct nvme_command),
+			DMA_TO_DEVICE);
+	if (error)
+		goto out_cleanup_queue;
+
+	return 0;
+
+out_cleanup_queue:
+	if (new)
+		blk_cleanup_queue(ctrl->ctrl.admin_q);
+out_free_tagset:
+	if (new)
+		nvme_rdma_free_tagset(&ctrl->ctrl, true);
+out_free_queue:
+	nvme_rdma_free_queue(&ctrl->queues[0]);
+	return error;
+}
+
+static void nvme_rdma_destroy_io_queues(struct nvme_rdma_ctrl *ctrl,
+		bool remove)
+{
+	nvme_rdma_stop_io_queues(ctrl);
+	if (remove) {
+		blk_cleanup_queue(ctrl->ctrl.connect_q);
+		nvme_rdma_free_tagset(&ctrl->ctrl, false);
+	}
+	nvme_rdma_free_io_queues(ctrl);
+}
+
+static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new)
+{
+	int ret;
+
+	ret = nvme_rdma_alloc_io_queues(ctrl);
+	if (ret)
+		return ret;
+
+	if (new) {
+		ctrl->ctrl.tagset = nvme_rdma_alloc_tagset(&ctrl->ctrl, false);
+		if (IS_ERR(ctrl->ctrl.tagset))
+			goto out_free_io_queues;
+
+		ctrl->ctrl.connect_q = blk_mq_init_queue(&ctrl->tag_set);
+		if (IS_ERR(ctrl->ctrl.connect_q)) {
+			ret = PTR_ERR(ctrl->ctrl.connect_q);
+			goto out_free_tag_set;
+		}
+	} else {
+		ret = blk_mq_reinit_tagset(&ctrl->tag_set,
+					   nvme_rdma_reinit_request);
+		if (ret)
+			goto out_free_io_queues;
+
+		blk_mq_update_nr_hw_queues(&ctrl->tag_set,
+			ctrl->ctrl.queue_count - 1);
+	}
+
+	ret = nvme_rdma_start_io_queues(ctrl);
+	if (ret)
+		goto out_cleanup_connect_q;
+
+	return 0;
+
+out_cleanup_connect_q:
+	if (new)
+		blk_cleanup_queue(ctrl->ctrl.connect_q);
+out_free_tag_set:
+	if (new)
+		nvme_rdma_free_tagset(&ctrl->ctrl, false);
+out_free_io_queues:
+	nvme_rdma_free_io_queues(ctrl);
+	return ret;
 }
 
 static void nvme_rdma_free_ctrl(struct nvme_ctrl *nctrl)
@@ -701,45 +927,18 @@ static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work)
 
 	++ctrl->ctrl.nr_reconnects;
 
-	if (ctrl->ctrl.queue_count > 1) {
-		nvme_rdma_free_io_queues(ctrl);
-
-		ret = blk_mq_reinit_tagset(&ctrl->tag_set);
-		if (ret)
-			goto requeue;
-	}
-
-	nvme_rdma_stop_and_free_queue(&ctrl->queues[0]);
-
-	ret = blk_mq_reinit_tagset(&ctrl->admin_tag_set);
-	if (ret)
-		goto requeue;
-
-	ret = nvme_rdma_init_queue(ctrl, 0, NVME_AQ_DEPTH);
-	if (ret)
-		goto requeue;
-
-	ret = nvmf_connect_admin_queue(&ctrl->ctrl);
-	if (ret)
-		goto requeue;
-
-	set_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[0].flags);
+	if (ctrl->ctrl.queue_count > 1)
+		nvme_rdma_destroy_io_queues(ctrl, false);
 
-	ret = nvme_enable_ctrl(&ctrl->ctrl, ctrl->ctrl.cap);
+	nvme_rdma_destroy_admin_queue(ctrl, false);
+	ret = nvme_rdma_configure_admin_queue(ctrl, false);
 	if (ret)
 		goto requeue;
 
 	if (ctrl->ctrl.queue_count > 1) {
-		ret = nvme_rdma_init_io_queues(ctrl);
-		if (ret)
-			goto requeue;
-
-		ret = nvme_rdma_connect_io_queues(ctrl);
+		ret = nvme_rdma_configure_io_queues(ctrl, false);
 		if (ret)
 			goto requeue;
-
-		blk_mq_update_nr_hw_queues(&ctrl->tag_set,
-				ctrl->ctrl.queue_count - 1);
 	}
 
 	changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
@@ -762,16 +961,15 @@ static void nvme_rdma_error_recovery_work(struct work_struct *work)
 {
 	struct nvme_rdma_ctrl *ctrl = container_of(work,
 			struct nvme_rdma_ctrl, err_work);
-	int i;
 
 	nvme_stop_ctrl(&ctrl->ctrl);
 
-	for (i = 0; i < ctrl->ctrl.queue_count; i++)
-		clear_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[i].flags);
-
-	if (ctrl->ctrl.queue_count > 1)
+	if (ctrl->ctrl.queue_count > 1) {
 		nvme_stop_queues(&ctrl->ctrl);
+		nvme_rdma_stop_io_queues(ctrl);
+	}
 	blk_mq_quiesce_queue(ctrl->ctrl.admin_q);
+	nvme_rdma_stop_queue(&ctrl->queues[0]);
 
 	/* We must take care of fastfail/requeue all our inflight requests */
 	if (ctrl->ctrl.queue_count > 1)
@@ -856,7 +1054,7 @@ static void nvme_rdma_unmap_data(struct nvme_rdma_queue *queue,
 
 	if (req->mr->need_inval) {
 		res = nvme_rdma_inv_rkey(queue, req);
-		if (res < 0) {
+		if (unlikely(res < 0)) {
 			dev_err(ctrl->ctrl.device,
 				"Queueing INV WR for rkey %#x failed (%d)\n",
 				req->mr->rkey, res);
@@ -920,8 +1118,12 @@ static int nvme_rdma_map_sg_fr(struct nvme_rdma_queue *queue,
 	struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl;
 	int nr;
 
-	nr = ib_map_mr_sg(req->mr, req->sg_table.sgl, count, NULL, PAGE_SIZE);
-	if (nr < count) {
+	/*
+	 * Align the MR to a 4K page size to match the ctrl page size and
+	 * the block virtual boundary.
+	 */
+	nr = ib_map_mr_sg(req->mr, req->sg_table.sgl, count, NULL, SZ_4K);
+	if (unlikely(nr < count)) {
 		if (nr < 0)
 			return nr;
 		return -EINVAL;
@@ -1057,7 +1259,7 @@ static int nvme_rdma_post_send(struct nvme_rdma_queue *queue,
 		first = &wr;
 
 	ret = ib_post_send(queue->qp, first, &bad_wr);
-	if (ret) {
+	if (unlikely(ret)) {
 		dev_err(queue->ctrl->ctrl.device,
 			     "%s failed with error code %d\n", __func__, ret);
 	}
@@ -1083,7 +1285,7 @@ static int nvme_rdma_post_recv(struct nvme_rdma_queue *queue,
 	wr.num_sge  = 1;
 
 	ret = ib_post_recv(queue->qp, &wr, &bad_wr);
-	if (ret) {
+	if (unlikely(ret)) {
 		dev_err(queue->ctrl->ctrl.device,
 			"%s failed with error code %d\n", __func__, ret);
 	}
@@ -1443,7 +1645,7 @@ static blk_status_t nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx,
 	blk_mq_start_request(rq);
 
 	err = nvme_rdma_map_data(queue, rq, c);
-	if (err < 0) {
+	if (unlikely(err < 0)) {
 		dev_err(queue->ctrl->ctrl.device,
 			     "Failed to map data (%d)\n", err);
 		nvme_cleanup_cmd(rq);
@@ -1457,7 +1659,7 @@ static blk_status_t nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx,
 		flush = true;
 	err = nvme_rdma_post_send(queue, sqe, req->sge, req->num_sge,
 			req->mr->need_inval ? &req->reg_wr.wr : NULL, flush);
-	if (err) {
+	if (unlikely(err)) {
 		nvme_rdma_unmap_data(queue, rq);
 		goto err;
 	}
@@ -1498,15 +1700,22 @@ static void nvme_rdma_complete_rq(struct request *rq)
 	nvme_complete_rq(rq);
 }
 
+static int nvme_rdma_map_queues(struct blk_mq_tag_set *set)
+{
+	struct nvme_rdma_ctrl *ctrl = set->driver_data;
+
+	return blk_mq_rdma_map_queues(set, ctrl->device->dev, 0);
+}
+
 static const struct blk_mq_ops nvme_rdma_mq_ops = {
 	.queue_rq	= nvme_rdma_queue_rq,
 	.complete	= nvme_rdma_complete_rq,
 	.init_request	= nvme_rdma_init_request,
 	.exit_request	= nvme_rdma_exit_request,
-	.reinit_request	= nvme_rdma_reinit_request,
 	.init_hctx	= nvme_rdma_init_hctx,
 	.poll		= nvme_rdma_poll,
 	.timeout	= nvme_rdma_timeout,
+	.map_queues	= nvme_rdma_map_queues,
 };
 
 static const struct blk_mq_ops nvme_rdma_admin_mq_ops = {
@@ -1514,103 +1723,11 @@ static const struct blk_mq_ops nvme_rdma_admin_mq_ops = {
 	.complete	= nvme_rdma_complete_rq,
 	.init_request	= nvme_rdma_init_request,
 	.exit_request	= nvme_rdma_exit_request,
-	.reinit_request	= nvme_rdma_reinit_request,
 	.init_hctx	= nvme_rdma_init_admin_hctx,
 	.timeout	= nvme_rdma_timeout,
 };
 
-static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl)
-{
-	int error;
-
-	error = nvme_rdma_init_queue(ctrl, 0, NVME_AQ_DEPTH);
-	if (error)
-		return error;
-
-	ctrl->device = ctrl->queues[0].device;
-
-	/*
-	 * We need a reference on the device as long as the tag_set is alive,
-	 * as the MRs in the request structures need a valid ib_device.
-	 */
-	error = -EINVAL;
-	if (!nvme_rdma_dev_get(ctrl->device))
-		goto out_free_queue;
-
-	ctrl->max_fr_pages = min_t(u32, NVME_RDMA_MAX_SEGMENTS,
-		ctrl->device->dev->attrs.max_fast_reg_page_list_len);
-
-	memset(&ctrl->admin_tag_set, 0, sizeof(ctrl->admin_tag_set));
-	ctrl->admin_tag_set.ops = &nvme_rdma_admin_mq_ops;
-	ctrl->admin_tag_set.queue_depth = NVME_RDMA_AQ_BLKMQ_DEPTH;
-	ctrl->admin_tag_set.reserved_tags = 2; /* connect + keep-alive */
-	ctrl->admin_tag_set.numa_node = NUMA_NO_NODE;
-	ctrl->admin_tag_set.cmd_size = sizeof(struct nvme_rdma_request) +
-		SG_CHUNK_SIZE * sizeof(struct scatterlist);
-	ctrl->admin_tag_set.driver_data = ctrl;
-	ctrl->admin_tag_set.nr_hw_queues = 1;
-	ctrl->admin_tag_set.timeout = ADMIN_TIMEOUT;
-
-	error = blk_mq_alloc_tag_set(&ctrl->admin_tag_set);
-	if (error)
-		goto out_put_dev;
-
-	ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set);
-	if (IS_ERR(ctrl->ctrl.admin_q)) {
-		error = PTR_ERR(ctrl->ctrl.admin_q);
-		goto out_free_tagset;
-	}
-
-	error = nvmf_connect_admin_queue(&ctrl->ctrl);
-	if (error)
-		goto out_cleanup_queue;
-
-	set_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[0].flags);
-
-	error = nvmf_reg_read64(&ctrl->ctrl, NVME_REG_CAP,
-			&ctrl->ctrl.cap);
-	if (error) {
-		dev_err(ctrl->ctrl.device,
-			"prop_get NVME_REG_CAP failed\n");
-		goto out_cleanup_queue;
-	}
-
-	ctrl->ctrl.sqsize =
-		min_t(int, NVME_CAP_MQES(ctrl->ctrl.cap), ctrl->ctrl.sqsize);
-
-	error = nvme_enable_ctrl(&ctrl->ctrl, ctrl->ctrl.cap);
-	if (error)
-		goto out_cleanup_queue;
-
-	ctrl->ctrl.max_hw_sectors =
-		(ctrl->max_fr_pages - 1) << (PAGE_SHIFT - 9);
-
-	error = nvme_init_identify(&ctrl->ctrl);
-	if (error)
-		goto out_cleanup_queue;
-
-	error = nvme_rdma_alloc_qe(ctrl->queues[0].device->dev,
-			&ctrl->async_event_sqe, sizeof(struct nvme_command),
-			DMA_TO_DEVICE);
-	if (error)
-		goto out_cleanup_queue;
-
-	return 0;
-
-out_cleanup_queue:
-	blk_cleanup_queue(ctrl->ctrl.admin_q);
-out_free_tagset:
-	/* disconnect and drain the queue before freeing the tagset */
-	nvme_rdma_stop_queue(&ctrl->queues[0]);
-	blk_mq_free_tag_set(&ctrl->admin_tag_set);
-out_put_dev:
-	nvme_rdma_dev_put(ctrl->device);
-out_free_queue:
-	nvme_rdma_free_queue(&ctrl->queues[0]);
-	return error;
-}
-
-static void nvme_rdma_shutdown_ctrl(struct nvme_rdma_ctrl *ctrl)
+static void nvme_rdma_shutdown_ctrl(struct nvme_rdma_ctrl *ctrl, bool shutdown)
 {
 	cancel_work_sync(&ctrl->err_work);
 	cancel_delayed_work_sync(&ctrl->reconnect_work);
@@ -1619,33 +1736,26 @@ static void nvme_rdma_shutdown_ctrl(struct nvme_rdma_ctrl *ctrl)
 		nvme_stop_queues(&ctrl->ctrl);
 		blk_mq_tagset_busy_iter(&ctrl->tag_set,
 					nvme_cancel_request, &ctrl->ctrl);
-		nvme_rdma_free_io_queues(ctrl);
+		nvme_rdma_destroy_io_queues(ctrl, shutdown);
 	}
 
-	if (test_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[0].flags))
+	if (shutdown)
 		nvme_shutdown_ctrl(&ctrl->ctrl);
+	else
+		nvme_disable_ctrl(&ctrl->ctrl, ctrl->ctrl.cap);
 
 	blk_mq_quiesce_queue(ctrl->ctrl.admin_q);
 	blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
 				nvme_cancel_request, &ctrl->ctrl);
 	blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);
-	nvme_rdma_destroy_admin_queue(ctrl);
+	nvme_rdma_destroy_admin_queue(ctrl, shutdown);
 }
 
-static void __nvme_rdma_remove_ctrl(struct nvme_rdma_ctrl *ctrl, bool shutdown)
+static void nvme_rdma_remove_ctrl(struct nvme_rdma_ctrl *ctrl)
 {
-	nvme_stop_ctrl(&ctrl->ctrl);
 	nvme_remove_namespaces(&ctrl->ctrl);
-	if (shutdown)
-		nvme_rdma_shutdown_ctrl(ctrl);
-
+	nvme_rdma_shutdown_ctrl(ctrl, true);
 	nvme_uninit_ctrl(&ctrl->ctrl);
-	if (ctrl->ctrl.tagset) {
-		blk_cleanup_queue(ctrl->ctrl.connect_q);
-		blk_mq_free_tag_set(&ctrl->tag_set);
-		nvme_rdma_dev_put(ctrl->device);
-	}
-
 	nvme_put_ctrl(&ctrl->ctrl);
 }
 
@@ -1654,7 +1764,8 @@ static void nvme_rdma_del_ctrl_work(struct work_struct *work)
 	struct nvme_rdma_ctrl *ctrl = container_of(work,
 				struct nvme_rdma_ctrl, delete_work);
 
-	__nvme_rdma_remove_ctrl(ctrl, true);
+	nvme_stop_ctrl(&ctrl->ctrl);
+	nvme_rdma_remove_ctrl(ctrl);
 }
 
 static int __nvme_rdma_del_ctrl(struct nvme_rdma_ctrl *ctrl)
@@ -1686,14 +1797,6 @@ static int nvme_rdma_del_ctrl(struct nvme_ctrl *nctrl)
 	return ret;
 }
 
-static void nvme_rdma_remove_ctrl_work(struct work_struct *work)
-{
-	struct nvme_rdma_ctrl *ctrl = container_of(work,
-				struct nvme_rdma_ctrl, delete_work);
-
-	__nvme_rdma_remove_ctrl(ctrl, false);
-}
-
 static void nvme_rdma_reset_ctrl_work(struct work_struct *work)
 {
 	struct nvme_rdma_ctrl *ctrl =
@@ -1702,30 +1805,16 @@ static void nvme_rdma_reset_ctrl_work(struct work_struct *work)
 	bool changed;
 
 	nvme_stop_ctrl(&ctrl->ctrl);
-	nvme_rdma_shutdown_ctrl(ctrl);
+	nvme_rdma_shutdown_ctrl(ctrl, false);
 
-	ret = nvme_rdma_configure_admin_queue(ctrl);
-	if (ret) {
-		/* ctrl is already shutdown, just remove the ctrl */
-		INIT_WORK(&ctrl->delete_work, nvme_rdma_remove_ctrl_work);
-		goto del_dead_ctrl;
-	}
+	ret = nvme_rdma_configure_admin_queue(ctrl, false);
+	if (ret)
+		goto out_fail;
 
 	if (ctrl->ctrl.queue_count > 1) {
-		ret = blk_mq_reinit_tagset(&ctrl->tag_set);
-		if (ret)
-			goto del_dead_ctrl;
-
-		ret = nvme_rdma_init_io_queues(ctrl);
-		if (ret)
-			goto del_dead_ctrl;
-
-		ret = nvme_rdma_connect_io_queues(ctrl);
+		ret = nvme_rdma_configure_io_queues(ctrl, false);
 		if (ret)
-			goto del_dead_ctrl;
-
-		blk_mq_update_nr_hw_queues(&ctrl->tag_set,
-				ctrl->ctrl.queue_count - 1);
+			goto out_fail;
 	}
 
 	changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
@@ -1735,10 +1824,9 @@ static void nvme_rdma_reset_ctrl_work(struct work_struct *work)
 
 	return;
 
-del_dead_ctrl:
-	/* Deleting this dead controller... */
+out_fail:
 	dev_warn(ctrl->ctrl.device, "Removing after reset failure\n");
-	WARN_ON(!queue_work(nvme_wq, &ctrl->delete_work));
+	nvme_rdma_remove_ctrl(ctrl);
 }
 
 static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = {
@@ -1754,62 +1842,6 @@ static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = {
 	.get_address		= nvmf_get_address,
 };
 
-static int nvme_rdma_create_io_queues(struct nvme_rdma_ctrl *ctrl)
-{
-	int ret;
-
-	ret = nvme_rdma_init_io_queues(ctrl);
-	if (ret)
-		return ret;
-
-	/*
-	 * We need a reference on the device as long as the tag_set is alive,
-	 * as the MRs in the request structures need a valid ib_device.
-	 */
-	ret = -EINVAL;
-	if (!nvme_rdma_dev_get(ctrl->device))
-		goto out_free_io_queues;
-
-	memset(&ctrl->tag_set, 0, sizeof(ctrl->tag_set));
-	ctrl->tag_set.ops = &nvme_rdma_mq_ops;
-	ctrl->tag_set.queue_depth = ctrl->ctrl.opts->queue_size;
-	ctrl->tag_set.reserved_tags = 1; /* fabric connect */
-	ctrl->tag_set.numa_node = NUMA_NO_NODE;
-	ctrl->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
-	ctrl->tag_set.cmd_size = sizeof(struct nvme_rdma_request) +
-		SG_CHUNK_SIZE * sizeof(struct scatterlist);
-	ctrl->tag_set.driver_data = ctrl;
-	ctrl->tag_set.nr_hw_queues = ctrl->ctrl.queue_count - 1;
-	ctrl->tag_set.timeout = NVME_IO_TIMEOUT;
-
-	ret = blk_mq_alloc_tag_set(&ctrl->tag_set);
-	if (ret)
-		goto out_put_dev;
-	ctrl->ctrl.tagset = &ctrl->tag_set;
-
-	ctrl->ctrl.connect_q = blk_mq_init_queue(&ctrl->tag_set);
-	if (IS_ERR(ctrl->ctrl.connect_q)) {
-		ret = PTR_ERR(ctrl->ctrl.connect_q);
-		goto out_free_tag_set;
-	}
-
-	ret = nvme_rdma_connect_io_queues(ctrl);
-	if (ret)
-		goto out_cleanup_connect_q;
-
-	return 0;
-
-out_cleanup_connect_q:
-	blk_cleanup_queue(ctrl->ctrl.connect_q);
-out_free_tag_set:
-	blk_mq_free_tag_set(&ctrl->tag_set);
-out_put_dev:
-	nvme_rdma_dev_put(ctrl->device);
-out_free_io_queues:
-	nvme_rdma_free_io_queues(ctrl);
-	return ret;
-}
-
 static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev,
 		struct nvmf_ctrl_options *opts)
 {
@@ -1867,7 +1899,7 @@ static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev,
 	if (!ctrl->queues)
 		goto out_uninit_ctrl;
 
-	ret = nvme_rdma_configure_admin_queue(ctrl);
+	ret = nvme_rdma_configure_admin_queue(ctrl, true);
 	if (ret)
 		goto out_kfree_queues;
 
@@ -1902,7 +1934,7 @@ static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev,
 	}
 
 	if (opts->nr_io_queues) {
-		ret = nvme_rdma_create_io_queues(ctrl);
+		ret = nvme_rdma_configure_io_queues(ctrl, true);
 		if (ret)
 			goto out_remove_admin_queue;
 	}
@@ -1924,7 +1956,7 @@ static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev,
 	return &ctrl->ctrl;
 
 out_remove_admin_queue:
-	nvme_rdma_destroy_admin_queue(ctrl);
+	nvme_rdma_destroy_admin_queue(ctrl, true);
 out_kfree_queues:
 	kfree(ctrl->queues);
 out_uninit_ctrl:
@@ -1946,10 +1978,6 @@ static struct nvmf_transport_ops nvme_rdma_transport = {
 	.create_ctrl	= nvme_rdma_create_ctrl,
 };
 
-static void nvme_rdma_add_one(struct ib_device *ib_device)
-{
-}
-
 static void nvme_rdma_remove_one(struct ib_device *ib_device, void *client_data)
 {
 	struct nvme_rdma_ctrl *ctrl;
@@ -1971,7 +1999,6 @@ static void nvme_rdma_remove_one(struct ib_device *ib_device, void *client_data)
 
 static struct ib_client nvme_rdma_ib_client = {
 	.name   = "nvme_rdma",
-	.add = nvme_rdma_add_one,
 	.remove = nvme_rdma_remove_one
 };
 
diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c
index 2d7a98ab53fb..c4a0bf36e752 100644
--- a/drivers/nvme/target/admin-cmd.c
+++ b/drivers/nvme/target/admin-cmd.c
@@ -100,7 +100,7 @@ static u16 nvmet_get_smart_log(struct nvmet_req *req,
 	u16 status;
 
 	WARN_ON(req == NULL || slog == NULL);
-	if (req->cmd->get_log_page.nsid == cpu_to_le32(0xFFFFFFFF))
+	if (req->cmd->get_log_page.nsid == cpu_to_le32(NVME_NSID_ALL))
 		status = nvmet_get_smart_log_all(req, slog);
 	else
 		status = nvmet_get_smart_log_nsid(req, slog);
@@ -168,15 +168,6 @@ out:
 	nvmet_req_complete(req, status);
 }
 
-static void copy_and_pad(char *dst, int dst_len, const char *src, int src_len)
-{
-	int len = min(src_len, dst_len);
-
-	memcpy(dst, src, len);
-	if (dst_len > len)
-		memset(dst + len, ' ', dst_len - len);
-}
-
 static void nvmet_execute_identify_ctrl(struct nvmet_req *req)
 {
 	struct nvmet_ctrl *ctrl = req->sq->ctrl;
@@ -196,14 +187,9 @@ static void nvmet_execute_identify_ctrl(struct nvmet_req *req)
 
 	bin2hex(id->sn, &ctrl->subsys->serial,
 		min(sizeof(ctrl->subsys->serial), sizeof(id->sn) / 2));
-	copy_and_pad(id->mn, sizeof(id->mn), model, sizeof(model) - 1);
-	copy_and_pad(id->fr, sizeof(id->fr), UTS_RELEASE, strlen(UTS_RELEASE));
-
-	memset(id->mn, ' ', sizeof(id->mn));
-	strncpy((char *)id->mn, "Linux", sizeof(id->mn));
-
-	memset(id->fr, ' ', sizeof(id->fr));
-	strncpy((char *)id->fr, UTS_RELEASE, sizeof(id->fr));
+	memcpy_and_pad(id->mn, sizeof(id->mn), model, sizeof(model) - 1, ' ');
+	memcpy_and_pad(id->fr, sizeof(id->fr),
+		       UTS_RELEASE, strlen(UTS_RELEASE), ' ');
 
 	id->rab = 6;
 
@@ -457,7 +443,7 @@ static void nvmet_execute_set_features(struct nvmet_req *req)
 	u32 val32;
 	u16 status = 0;
 
-	switch (cdw10 & 0xf) {
+	switch (cdw10 & 0xff) {
 	case NVME_FEAT_NUM_QUEUES:
 		nvmet_set_result(req,
 			(subsys->max_qid - 1) | ((subsys->max_qid - 1) << 16));
@@ -467,6 +453,9 @@ static void nvmet_execute_set_features(struct nvmet_req *req)
 		req->sq->ctrl->kato = DIV_ROUND_UP(val32, 1000);
 		nvmet_set_result(req, req->sq->ctrl->kato);
 		break;
+	case NVME_FEAT_HOST_ID:
+		status = NVME_SC_CMD_SEQ_ERROR | NVME_SC_DNR;
+		break;
 	default:
 		status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 		break;
@@ -481,7 +470,7 @@ static void nvmet_execute_get_features(struct nvmet_req *req)
 	u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10[0]);
 	u16 status = 0;
 
-	switch (cdw10 & 0xf) {
+	switch (cdw10 & 0xff) {
 	/*
 	 * These features are mandatory in the spec, but we don't
 	 * have a useful way to implement them.  We'll eventually
@@ -515,6 +504,16 @@ static void nvmet_execute_get_features(struct nvmet_req *req)
 	case NVME_FEAT_KATO:
 		nvmet_set_result(req, req->sq->ctrl->kato * 1000);
 		break;
+	case NVME_FEAT_HOST_ID:
+		/* need 128-bit host identifier flag */
+		if (!(req->cmd->common.cdw10[1] & cpu_to_le32(1 << 0))) {
+			status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
+			break;
+		}
+
+		status = nvmet_copy_to_sgl(req, 0, &req->sq->ctrl->hostid,
+				sizeof(req->sq->ctrl->hostid));
+		break;
 	default:
 		status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
 		break;
diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c
index 0a0067e771f5..b6aeb1d70951 100644
--- a/drivers/nvme/target/configfs.c
+++ b/drivers/nvme/target/configfs.c
@@ -444,7 +444,7 @@ static struct config_group *nvmet_ns_make(struct config_group *group,
 		goto out;
 
 	ret = -EINVAL;
-	if (nsid == 0 || nsid == 0xffffffff)
+	if (nsid == 0 || nsid == NVME_NSID_ALL)
 		goto out;
 
 	ret = -ENOMEM;
diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c
index f4b02bb4a1a8..7c23eaf8e563 100644
--- a/drivers/nvme/target/core.c
+++ b/drivers/nvme/target/core.c
@@ -538,37 +538,37 @@ EXPORT_SYMBOL_GPL(nvmet_req_uninit);
 
 static inline bool nvmet_cc_en(u32 cc)
 {
-	return cc & 0x1;
+	return (cc >> NVME_CC_EN_SHIFT) & 0x1;
 }
 
 static inline u8 nvmet_cc_css(u32 cc)
 {
-	return (cc >> 4) & 0x7;
+	return (cc >> NVME_CC_CSS_SHIFT) & 0x7;
 }
 
 static inline u8 nvmet_cc_mps(u32 cc)
 {
-	return (cc >> 7) & 0xf;
+	return (cc >> NVME_CC_MPS_SHIFT) & 0xf;
 }
 
 static inline u8 nvmet_cc_ams(u32 cc)
 {
-	return (cc >> 11) & 0x7;
+	return (cc >> NVME_CC_AMS_SHIFT) & 0x7;
 }
 
 static inline u8 nvmet_cc_shn(u32 cc)
 {
-	return (cc >> 14) & 0x3;
+	return (cc >> NVME_CC_SHN_SHIFT) & 0x3;
 }
 
 static inline u8 nvmet_cc_iosqes(u32 cc)
 {
-	return (cc >> 16) & 0xf;
+	return (cc >> NVME_CC_IOSQES_SHIFT) & 0xf;
 }
 
 static inline u8 nvmet_cc_iocqes(u32 cc)
 {
-	return (cc >> 20) & 0xf;
+	return (cc >> NVME_CC_IOCQES_SHIFT) & 0xf;
 }
 
 static void nvmet_start_ctrl(struct nvmet_ctrl *ctrl)
@@ -749,6 +749,7 @@ u16 nvmet_alloc_ctrl(const char *subsysnqn, const char *hostnqn,
 			hostnqn, subsysnqn);
 		req->rsp->result.u32 = IPO_IATTR_CONNECT_DATA(hostnqn);
 		up_read(&nvmet_config_sem);
+		status = NVME_SC_CONNECT_INVALID_HOST | NVME_SC_DNR;
 		goto out_put_subsystem;
 	}
 	up_read(&nvmet_config_sem);
diff --git a/drivers/nvme/target/fabrics-cmd.c b/drivers/nvme/target/fabrics-cmd.c
index 3cc17269504b..859a66725291 100644
--- a/drivers/nvme/target/fabrics-cmd.c
+++ b/drivers/nvme/target/fabrics-cmd.c
@@ -154,6 +154,7 @@ static void nvmet_execute_admin_connect(struct nvmet_req *req)
 				  le32_to_cpu(c->kato), &ctrl);
 	if (status)
 		goto out;
+	uuid_copy(&ctrl->hostid, &d->hostid);
 
 	status = nvmet_install_queue(ctrl, req);
 	if (status) {
diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c
index 1b7f2520a20d..421e43bf1dd7 100644
--- a/drivers/nvme/target/fc.c
+++ b/drivers/nvme/target/fc.c
@@ -58,7 +58,8 @@ struct nvmet_fc_ls_iod {
 	struct work_struct		work;
 } __aligned(sizeof(unsigned long long));
 
-#define NVMET_FC_MAX_KB_PER_XFR		256
+#define NVMET_FC_MAX_SEQ_LENGTH		(256 * 1024)
+#define NVMET_FC_MAX_XFR_SGENTS		(NVMET_FC_MAX_SEQ_LENGTH / PAGE_SIZE)
 
 enum nvmet_fcp_datadir {
 	NVMET_FCP_NODATA,
@@ -74,9 +75,7 @@ struct nvmet_fc_fcp_iod {
 	struct nvme_fc_ersp_iu		rspiubuf;
 	dma_addr_t			rspdma;
 	struct scatterlist		*data_sg;
-	struct scatterlist		*next_sg;
 	int				data_sg_cnt;
-	u32				next_sg_offset;
 	u32				total_length;
 	u32				offset;
 	enum nvmet_fcp_datadir		io_dir;
@@ -112,6 +111,7 @@ struct nvmet_fc_tgtport {
 	struct ida			assoc_cnt;
 	struct nvmet_port		*port;
 	struct kref			ref;
+	u32				max_sg_cnt;
 };
 
 struct nvmet_fc_defer_fcp_req {
@@ -394,7 +394,7 @@ nvmet_fc_free_ls_iodlist(struct nvmet_fc_tgtport *tgtport)
 static struct nvmet_fc_ls_iod *
 nvmet_fc_alloc_ls_iod(struct nvmet_fc_tgtport *tgtport)
 {
-	static struct nvmet_fc_ls_iod *iod;
+	struct nvmet_fc_ls_iod *iod;
 	unsigned long flags;
 
 	spin_lock_irqsave(&tgtport->lock, flags);
@@ -471,7 +471,7 @@ nvmet_fc_destroy_fcp_iodlist(struct nvmet_fc_tgtport *tgtport,
 static struct nvmet_fc_fcp_iod *
 nvmet_fc_alloc_fcp_iod(struct nvmet_fc_tgt_queue *queue)
 {
-	static struct nvmet_fc_fcp_iod *fod;
+	struct nvmet_fc_fcp_iod *fod;
 
 	lockdep_assert_held(&queue->qlock);
 
@@ -704,7 +704,7 @@ nvmet_fc_delete_target_queue(struct nvmet_fc_tgt_queue *queue)
 {
 	struct nvmet_fc_tgtport *tgtport = queue->assoc->tgtport;
 	struct nvmet_fc_fcp_iod *fod = queue->fod;
-	struct nvmet_fc_defer_fcp_req *deferfcp;
+	struct nvmet_fc_defer_fcp_req *deferfcp, *tempptr;
 	unsigned long flags;
 	int i, writedataactive;
 	bool disconnect;
@@ -735,7 +735,8 @@ nvmet_fc_delete_target_queue(struct nvmet_fc_tgt_queue *queue)
 	}
 
 	/* Cleanup defer'ed IOs in queue */
-	list_for_each_entry(deferfcp, &queue->avail_defer_list, req_list) {
+	list_for_each_entry_safe(deferfcp, tempptr, &queue->avail_defer_list,
+				req_list) {
 		list_del(&deferfcp->req_list);
 		kfree(deferfcp);
 	}
@@ -993,6 +994,8 @@ nvmet_fc_register_targetport(struct nvmet_fc_port_info *pinfo,
 	INIT_LIST_HEAD(&newrec->assoc_list);
 	kref_init(&newrec->ref);
 	ida_init(&newrec->assoc_cnt);
+	newrec->max_sg_cnt = min_t(u32, NVMET_FC_MAX_XFR_SGENTS,
+					template->max_sgl_segments);
 
 	ret = nvmet_fc_alloc_ls_iodlist(newrec);
 	if (ret) {
@@ -1865,51 +1868,23 @@ nvmet_fc_transfer_fcp_data(struct nvmet_fc_tgtport *tgtport,
 				struct nvmet_fc_fcp_iod *fod, u8 op)
 {
 	struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
-	struct scatterlist *sg, *datasg;
 	unsigned long flags;
-	u32 tlen, sg_off;
+	u32 tlen;
 	int ret;
 
 	fcpreq->op = op;
 	fcpreq->offset = fod->offset;
 	fcpreq->timeout = NVME_FC_TGTOP_TIMEOUT_SEC;
-	tlen = min_t(u32, (NVMET_FC_MAX_KB_PER_XFR * 1024),
+
+	tlen = min_t(u32, tgtport->max_sg_cnt * PAGE_SIZE,
 			(fod->total_length - fod->offset));
-	tlen = min_t(u32, tlen, NVME_FC_MAX_SEGMENTS * PAGE_SIZE);
-	tlen = min_t(u32, tlen, fod->tgtport->ops->max_sgl_segments
-					* PAGE_SIZE);
 	fcpreq->transfer_length = tlen;
 	fcpreq->transferred_length = 0;
 	fcpreq->fcp_error = 0;
 	fcpreq->rsplen = 0;
 
-	fcpreq->sg_cnt = 0;
-
-	datasg = fod->next_sg;
-	sg_off = fod->next_sg_offset;
-
-	for (sg = fcpreq->sg ; tlen; sg++) {
-		*sg = *datasg;
-		if (sg_off) {
-			sg->offset += sg_off;
-			sg->length -= sg_off;
-			sg->dma_address += sg_off;
-			sg_off = 0;
-		}
-		if (tlen < sg->length) {
-			sg->length = tlen;
-			fod->next_sg = datasg;
-			fod->next_sg_offset += tlen;
-		} else if (tlen == sg->length) {
-			fod->next_sg_offset = 0;
-			fod->next_sg = sg_next(datasg);
-		} else {
-			fod->next_sg_offset = 0;
-			datasg = sg_next(datasg);
-		}
-		tlen -= sg->length;
-		fcpreq->sg_cnt++;
-	}
+	fcpreq->sg = &fod->data_sg[fod->offset / PAGE_SIZE];
+	fcpreq->sg_cnt = DIV_ROUND_UP(tlen, PAGE_SIZE);
 
 	/*
 	 * If the last READDATA request: check if LLDD supports
@@ -2224,8 +2199,6 @@ nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport,
 	fod->req.sg = fod->data_sg;
 	fod->req.sg_cnt = fod->data_sg_cnt;
 	fod->offset = 0;
-	fod->next_sg = fod->data_sg;
-	fod->next_sg_offset = 0;
 
 	if (fod->io_dir == NVMET_FCP_WRITE) {
 		/* pull the data over before invoking nvmet layer */
diff --git a/drivers/nvme/target/fcloop.c b/drivers/nvme/target/fcloop.c
index 1bb9d5b311b1..1cb9847ec261 100644
--- a/drivers/nvme/target/fcloop.c
+++ b/drivers/nvme/target/fcloop.c
@@ -193,9 +193,6 @@ out_free_options:
 
 #define TGTPORT_OPTS	(NVMF_OPT_WWNN | NVMF_OPT_WWPN)
 
-#define ALL_OPTS	(NVMF_OPT_WWNN | NVMF_OPT_WWPN | NVMF_OPT_ROLES | \
-			 NVMF_OPT_FCADDR | NVMF_OPT_LPWWNN | NVMF_OPT_LPWWPN)
-
 
 static DEFINE_SPINLOCK(fcloop_lock);
 static LIST_HEAD(fcloop_lports);
diff --git a/drivers/nvme/target/io-cmd.c b/drivers/nvme/target/io-cmd.c
index 3b4d47a6abdb..0d4c23dc4532 100644
--- a/drivers/nvme/target/io-cmd.c
+++ b/drivers/nvme/target/io-cmd.c
@@ -68,7 +68,7 @@ static void nvmet_execute_rw(struct nvmet_req *req)
 
 	nvmet_inline_bio_init(req);
 	bio = &req->inline_bio;
-	bio->bi_bdev = req->ns->bdev;
+	bio_set_dev(bio, req->ns->bdev);
 	bio->bi_iter.bi_sector = sector;
 	bio->bi_private = req;
 	bio->bi_end_io = nvmet_bio_done;
@@ -80,7 +80,7 @@ static void nvmet_execute_rw(struct nvmet_req *req)
 			struct bio *prev = bio;
 
 			bio = bio_alloc(GFP_KERNEL, min(sg_cnt, BIO_MAX_PAGES));
-			bio->bi_bdev = req->ns->bdev;
+			bio_set_dev(bio, req->ns->bdev);
 			bio->bi_iter.bi_sector = sector;
 			bio_set_op_attrs(bio, op, op_flags);
 
@@ -104,7 +104,7 @@ static void nvmet_execute_flush(struct nvmet_req *req)
 	nvmet_inline_bio_init(req);
 	bio = &req->inline_bio;
 
-	bio->bi_bdev = req->ns->bdev;
+	bio_set_dev(bio, req->ns->bdev);
 	bio->bi_private = req;
 	bio->bi_end_io = nvmet_bio_done;
 	bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
diff --git a/drivers/nvme/target/loop.c b/drivers/nvme/target/loop.c
index 717ed7ddb2f6..92628c432926 100644
--- a/drivers/nvme/target/loop.c
+++ b/drivers/nvme/target/loop.c
@@ -375,6 +375,7 @@ static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl)
 	error = blk_mq_alloc_tag_set(&ctrl->admin_tag_set);
 	if (error)
 		goto out_free_sq;
+	ctrl->ctrl.admin_tagset = &ctrl->admin_tag_set;
 
 	ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set);
 	if (IS_ERR(ctrl->ctrl.admin_q)) {
diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h
index e3b244c7e443..7d261ab894f4 100644
--- a/drivers/nvme/target/nvmet.h
+++ b/drivers/nvme/target/nvmet.h
@@ -115,6 +115,7 @@ struct nvmet_ctrl {
 	u32			cc;
 	u32			csts;
 
+	uuid_t			hostid;
 	u16			cntlid;
 	u32			kato;
 
diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c
index 56a4cba690b5..76d2bb793afe 100644
--- a/drivers/nvme/target/rdma.c
+++ b/drivers/nvme/target/rdma.c
@@ -1510,10 +1510,6 @@ static struct nvmet_fabrics_ops nvmet_rdma_ops = {
 	.delete_ctrl		= nvmet_rdma_delete_ctrl,
 };
 
-static void nvmet_rdma_add_one(struct ib_device *ib_device)
-{
-}
-
 static void nvmet_rdma_remove_one(struct ib_device *ib_device, void *client_data)
 {
 	struct nvmet_rdma_queue *queue;
@@ -1534,7 +1530,6 @@ static void nvmet_rdma_remove_one(struct ib_device *ib_device, void *client_data
 
 static struct ib_client nvmet_rdma_ib_client = {
 	.name   = "nvmet_rdma",
-	.add = nvmet_rdma_add_one,
 	.remove = nvmet_rdma_remove_one
 };
 
diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c
index 4c49285168fb..de54c7f5048a 100644
--- a/drivers/nvmem/core.c
+++ b/drivers/nvmem/core.c
@@ -290,7 +290,7 @@ static struct nvmem_cell *nvmem_find_cell(const char *cell_id)
 	mutex_lock(&nvmem_cells_mutex);
 
 	list_for_each_entry(p, &nvmem_cells, node)
-		if (p && !strcmp(p->name, cell_id)) {
+		if (!strcmp(p->name, cell_id)) {
 			mutex_unlock(&nvmem_cells_mutex);
 			return p;
 		}
@@ -794,8 +794,8 @@ struct nvmem_cell *of_nvmem_cell_get(struct device_node *np,
 
 	addr = of_get_property(cell_np, "reg", &len);
 	if (!addr || (len < 2 * sizeof(u32))) {
-		dev_err(&nvmem->dev, "nvmem: invalid reg on %s\n",
-			cell_np->full_name);
+		dev_err(&nvmem->dev, "nvmem: invalid reg on %pOF\n",
+			cell_np);
 		rval  = -EINVAL;
 		goto err_mem;
 	}
@@ -1111,6 +1111,43 @@ int nvmem_cell_write(struct nvmem_cell *cell, void *buf, size_t len)
 EXPORT_SYMBOL_GPL(nvmem_cell_write);
 
 /**
+ * nvmem_cell_read_u32() - Read a cell value as an u32
+ *
+ * @dev: Device that requests the nvmem cell.
+ * @cell_id: Name of nvmem cell to read.
+ * @val: pointer to output value.
+ *
+ * Return: 0 on success or negative errno.
+ */
+int nvmem_cell_read_u32(struct device *dev, const char *cell_id, u32 *val)
+{
+	struct nvmem_cell *cell;
+	void *buf;
+	size_t len;
+
+	cell = nvmem_cell_get(dev, cell_id);
+	if (IS_ERR(cell))
+		return PTR_ERR(cell);
+
+	buf = nvmem_cell_read(cell, &len);
+	if (IS_ERR(buf)) {
+		nvmem_cell_put(cell);
+		return PTR_ERR(buf);
+	}
+	if (len != sizeof(*val)) {
+		kfree(buf);
+		nvmem_cell_put(cell);
+		return -EINVAL;
+	}
+	memcpy(val, buf, sizeof(*val));
+
+	kfree(buf);
+	nvmem_cell_put(cell);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(nvmem_cell_read_u32);
+
+/**
  * nvmem_device_cell_read() - Read a given nvmem device and cell
  *
  * @nvmem: nvmem device to read from.
diff --git a/drivers/nvmem/lpc18xx_eeprom.c b/drivers/nvmem/lpc18xx_eeprom.c
index c81ae4c6da74..6c7e2c424a4e 100644
--- a/drivers/nvmem/lpc18xx_eeprom.c
+++ b/drivers/nvmem/lpc18xx_eeprom.c
@@ -197,7 +197,7 @@ static int lpc18xx_eeprom_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	rst = devm_reset_control_get(dev, NULL);
+	rst = devm_reset_control_get_exclusive(dev, NULL);
 	if (IS_ERR(rst)) {
 		dev_err(dev, "failed to get reset: %ld\n", PTR_ERR(rst));
 		ret = PTR_ERR(rst);
diff --git a/drivers/of/address.c b/drivers/of/address.c
index 580bbf6ca2b1..792722e7d458 100644
--- a/drivers/of/address.c
+++ b/drivers/of/address.c
@@ -274,10 +274,9 @@ struct of_pci_range *of_pci_range_parser_one(struct of_pci_range_parser *parser,
 
 	/* Now consume following elements while they are contiguous */
 	while (parser->range + parser->np <= parser->end) {
-		u32 flags, pci_space;
+		u32 flags;
 		u64 pci_addr, cpu_addr, size;
 
-		pci_space = be32_to_cpup(parser->range);
 		flags = of_bus_pci_get_flags(parser->range);
 		pci_addr = of_read_number(parser->range + 1, ns);
 		cpu_addr = of_translate_address(parser->node,
@@ -559,7 +558,7 @@ static u64 __of_translate_address(struct device_node *dev,
 	int na, ns, pna, pns;
 	u64 result = OF_BAD_ADDR;
 
-	pr_debug("** translation for device %s **\n", of_node_full_name(dev));
+	pr_debug("** translation for device %pOF **\n", dev);
 
 	/* Increase refcount at current level */
 	of_node_get(dev);
@@ -573,13 +572,13 @@ static u64 __of_translate_address(struct device_node *dev,
 	/* Count address cells & copy address locally */
 	bus->count_cells(dev, &na, &ns);
 	if (!OF_CHECK_COUNTS(na, ns)) {
-		pr_debug("Bad cell count for %s\n", of_node_full_name(dev));
+		pr_debug("Bad cell count for %pOF\n", dev);
 		goto bail;
 	}
 	memcpy(addr, in_addr, na * 4);
 
-	pr_debug("bus is %s (na=%d, ns=%d) on %s\n",
-	    bus->name, na, ns, of_node_full_name(parent));
+	pr_debug("bus is %s (na=%d, ns=%d) on %pOF\n",
+	    bus->name, na, ns, parent);
 	of_dump_addr("translating address:", addr, na);
 
 	/* Translate */
@@ -600,13 +599,12 @@ static u64 __of_translate_address(struct device_node *dev,
 		pbus = of_match_bus(parent);
 		pbus->count_cells(dev, &pna, &pns);
 		if (!OF_CHECK_COUNTS(pna, pns)) {
-			pr_err("Bad cell count for %s\n",
-			       of_node_full_name(dev));
+			pr_err("Bad cell count for %pOF\n", dev);
 			break;
 		}
 
-		pr_debug("parent bus is %s (na=%d, ns=%d) on %s\n",
-		    pbus->name, pna, pns, of_node_full_name(parent));
+		pr_debug("parent bus is %s (na=%d, ns=%d) on %pOF\n",
+		    pbus->name, pna, pns, parent);
 
 		/* Apply bus translation */
 		if (of_translate_one(dev, bus, pbus, addr, na, ns, pna, rprop))
@@ -855,7 +853,7 @@ int of_dma_get_range(struct device_node *np, u64 *dma_addr, u64 *paddr, u64 *siz
 	}
 
 	if (!ranges) {
-		pr_debug("no dma-ranges found for node(%s)\n", np->full_name);
+		pr_debug("no dma-ranges found for node(%pOF)\n", np);
 		ret = -ENODEV;
 		goto out;
 	}
@@ -872,8 +870,8 @@ int of_dma_get_range(struct device_node *np, u64 *dma_addr, u64 *paddr, u64 *siz
 	dmaaddr = of_read_number(ranges, naddr);
 	*paddr = of_translate_dma_address(np, ranges);
 	if (*paddr == OF_BAD_ADDR) {
-		pr_err("translation of DMA address(%pad) to CPU address failed node(%s)\n",
-		       dma_addr, np->full_name);
+		pr_err("translation of DMA address(%pad) to CPU address failed node(%pOF)\n",
+		       dma_addr, np);
 		ret = -EINVAL;
 		goto out;
 	}
diff --git a/drivers/of/base.c b/drivers/of/base.c
index 686628d1dfa6..260d33c0f26c 100644
--- a/drivers/of/base.c
+++ b/drivers/of/base.c
@@ -60,14 +60,13 @@ DEFINE_RAW_SPINLOCK(devtree_lock);
 
 int of_n_addr_cells(struct device_node *np)
 {
-	const __be32 *ip;
+	u32 cells;
 
 	do {
 		if (np->parent)
 			np = np->parent;
-		ip = of_get_property(np, "#address-cells", NULL);
-		if (ip)
-			return be32_to_cpup(ip);
+		if (!of_property_read_u32(np, "#address-cells", &cells))
+			return cells;
 	} while (np->parent);
 	/* No #address-cells property for the root node */
 	return OF_ROOT_NODE_ADDR_CELLS_DEFAULT;
@@ -76,14 +75,13 @@ EXPORT_SYMBOL(of_n_addr_cells);
 
 int of_n_size_cells(struct device_node *np)
 {
-	const __be32 *ip;
+	u32 cells;
 
 	do {
 		if (np->parent)
 			np = np->parent;
-		ip = of_get_property(np, "#size-cells", NULL);
-		if (ip)
-			return be32_to_cpup(ip);
+		if (!of_property_read_u32(np, "#size-cells", &cells))
+			return cells;
 	} while (np->parent);
 	/* No #size-cells property for the root node */
 	return OF_ROOT_NODE_SIZE_CELLS_DEFAULT;
@@ -160,7 +158,7 @@ int __of_add_property_sysfs(struct device_node *np, struct property *pp)
 	pp->attr.read = of_node_property_read;
 
 	rc = sysfs_create_bin_file(&np->kobj, &pp->attr);
-	WARN(rc, "error adding attribute %s to node %s\n", pp->name, np->full_name);
+	WARN(rc, "error adding attribute %s to node %pOF\n", pp->name, np);
 	return rc;
 }
 
@@ -1122,7 +1120,7 @@ EXPORT_SYMBOL(of_find_node_by_phandle);
 void of_print_phandle_args(const char *msg, const struct of_phandle_args *args)
 {
 	int i;
-	printk("%s %s", msg, of_node_full_name(args->np));
+	printk("%s %pOF", msg, args->np);
 	for (i = 0; i < args->args_count; i++) {
 		const char delim = i ? ',' : ':';
 
@@ -1184,17 +1182,17 @@ int of_phandle_iterator_next(struct of_phandle_iterator *it)
 
 		if (it->cells_name) {
 			if (!it->node) {
-				pr_err("%s: could not find phandle\n",
-				       it->parent->full_name);
+				pr_err("%pOF: could not find phandle\n",
+				       it->parent);
 				goto err;
 			}
 
 			if (of_property_read_u32(it->node, it->cells_name,
 						 &count)) {
-				pr_err("%s: could not get %s for %s\n",
-				       it->parent->full_name,
+				pr_err("%pOF: could not get %s for %pOF\n",
+				       it->parent,
 				       it->cells_name,
-				       it->node->full_name);
+				       it->node);
 				goto err;
 			}
 		} else {
@@ -1206,8 +1204,8 @@ int of_phandle_iterator_next(struct of_phandle_iterator *it)
 		 * property data length
 		 */
 		if (it->cur + count > it->list_end) {
-			pr_err("%s: arguments longer than property\n",
-			       it->parent->full_name);
+			pr_err("%pOF: arguments longer than property\n",
+			       it->parent);
 			goto err;
 		}
 	}
@@ -1639,8 +1637,8 @@ static void of_alias_add(struct alias_prop *ap, struct device_node *np,
 	strncpy(ap->stem, stem, stem_len);
 	ap->stem[stem_len] = 0;
 	list_add_tail(&ap->link, &aliases_lookup);
-	pr_debug("adding DT alias:%s: stem=%s id=%i node=%s\n",
-		 ap->alias, ap->stem, ap->id, of_node_full_name(np));
+	pr_debug("adding DT alias:%s: stem=%s id=%i node=%pOF\n",
+		 ap->alias, ap->stem, ap->id, np);
 }
 
 /**
@@ -1664,11 +1662,13 @@ void of_alias_scan(void * (*dt_alloc)(u64 size, u64 align))
 
 	if (of_chosen) {
 		/* linux,stdout-path and /aliases/stdout are for legacy compatibility */
-		const char *name = of_get_property(of_chosen, "stdout-path", NULL);
-		if (!name)
-			name = of_get_property(of_chosen, "linux,stdout-path", NULL);
+		const char *name = NULL;
+
+		if (of_property_read_string(of_chosen, "stdout-path", &name))
+			of_property_read_string(of_chosen, "linux,stdout-path",
+						&name);
 		if (IS_ENABLED(CONFIG_PPC) && !name)
-			name = of_get_property(of_aliases, "stdout", NULL);
+			of_property_read_string(of_aliases, "stdout", &name);
 		if (name)
 			of_stdout = of_find_node_opts_by_path(name, &of_stdout_options);
 	}
diff --git a/drivers/of/device.c b/drivers/of/device.c
index 28c38c756f92..64b710265d39 100644
--- a/drivers/of/device.c
+++ b/drivers/of/device.c
@@ -9,6 +9,9 @@
 #include <linux/module.h>
 #include <linux/mod_devicetable.h>
 #include <linux/slab.h>
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include <linux/amba/bus.h>
 
 #include <asm/errno.h>
 #include "of_private.h"
@@ -58,7 +61,7 @@ int of_device_add(struct platform_device *ofdev)
 	/* name and id have to be set so that the platform bus doesn't get
 	 * confused on matching */
 	ofdev->name = dev_name(&ofdev->dev);
-	ofdev->id = -1;
+	ofdev->id = PLATFORM_DEVID_NONE;
 
 	/*
 	 * If this device has not binding numa node in devicetree, that is
@@ -84,30 +87,28 @@ int of_device_add(struct platform_device *ofdev)
  */
 int of_dma_configure(struct device *dev, struct device_node *np)
 {
-	u64 dma_addr, paddr, size;
+	u64 dma_addr, paddr, size = 0;
 	int ret;
 	bool coherent;
 	unsigned long offset;
 	const struct iommu_ops *iommu;
-
-	/*
-	 * Set default coherent_dma_mask to 32 bit.  Drivers are expected to
-	 * setup the correct supported mask.
-	 */
-	if (!dev->coherent_dma_mask)
-		dev->coherent_dma_mask = DMA_BIT_MASK(32);
-
-	/*
-	 * Set it to coherent_dma_mask by default if the architecture
-	 * code has not set it.
-	 */
-	if (!dev->dma_mask)
-		dev->dma_mask = &dev->coherent_dma_mask;
+	u64 mask;
 
 	ret = of_dma_get_range(np, &dma_addr, &paddr, &size);
 	if (ret < 0) {
+		/*
+		 * For legacy reasons, we have to assume some devices need
+		 * DMA configuration regardless of whether "dma-ranges" is
+		 * correctly specified or not.
+		 */
+		if (!dev_is_pci(dev) &&
+#ifdef CONFIG_ARM_AMBA
+		    dev->bus != &amba_bustype &&
+#endif
+		    dev->bus != &platform_bus_type)
+			return ret == -ENODEV ? 0 : ret;
+
 		dma_addr = offset = 0;
-		size = max(dev->coherent_dma_mask, dev->coherent_dma_mask + 1);
 	} else {
 		offset = PFN_DOWN(paddr - dma_addr);
 
@@ -128,16 +129,31 @@ int of_dma_configure(struct device *dev, struct device_node *np)
 		dev_dbg(dev, "dma_pfn_offset(%#08lx)\n", offset);
 	}
 
+	/*
+	 * Set default coherent_dma_mask to 32 bit.  Drivers are expected to
+	 * setup the correct supported mask.
+	 */
+	if (!dev->coherent_dma_mask)
+		dev->coherent_dma_mask = DMA_BIT_MASK(32);
+	/*
+	 * Set it to coherent_dma_mask by default if the architecture
+	 * code has not set it.
+	 */
+	if (!dev->dma_mask)
+		dev->dma_mask = &dev->coherent_dma_mask;
+
+	if (!size)
+		size = max(dev->coherent_dma_mask, dev->coherent_dma_mask + 1);
+
 	dev->dma_pfn_offset = offset;
 
 	/*
 	 * Limit coherent and dma mask based on size and default mask
 	 * set by the driver.
 	 */
-	dev->coherent_dma_mask = min(dev->coherent_dma_mask,
-				     DMA_BIT_MASK(ilog2(dma_addr + size)));
-	*dev->dma_mask = min((*dev->dma_mask),
-			     DMA_BIT_MASK(ilog2(dma_addr + size)));
+	mask = DMA_BIT_MASK(ilog2(dma_addr + size - 1) + 1);
+	dev->coherent_dma_mask &= mask;
+	*dev->dma_mask &= mask;
 
 	coherent = of_dma_is_coherent(np);
 	dev_dbg(dev, "device is%sdma coherent\n",
@@ -196,8 +212,10 @@ EXPORT_SYMBOL(of_device_get_match_data);
 static ssize_t of_device_get_modalias(struct device *dev, char *str, ssize_t len)
 {
 	const char *compat;
-	int cplen, i;
-	ssize_t tsize, csize, repend;
+	char *c;
+	struct property *p;
+	ssize_t csize;
+	ssize_t tsize;
 
 	if ((!dev) || (!dev->of_node))
 		return -ENODEV;
@@ -205,42 +223,28 @@ static ssize_t of_device_get_modalias(struct device *dev, char *str, ssize_t len
 	/* Name & Type */
 	csize = snprintf(str, len, "of:N%sT%s", dev->of_node->name,
 			 dev->of_node->type);
-
-	/* Get compatible property if any */
-	compat = of_get_property(dev->of_node, "compatible", &cplen);
-	if (!compat)
-		return csize;
-
-	/* Find true end (we tolerate multiple \0 at the end */
-	for (i = (cplen - 1); i >= 0 && !compat[i]; i--)
-		cplen--;
-	if (!cplen)
-		return csize;
-	cplen++;
-
-	/* Check space (need cplen+1 chars including final \0) */
-	tsize = csize + cplen;
-	repend = tsize;
-
-	if (csize >= len)		/* @ the limit, all is already filled */
-		return tsize;
-
-	if (tsize >= len) {		/* limit compat list */
-		cplen = len - csize - 1;
-		repend = len;
-	}
-
-	/* Copy and do char replacement */
-	memcpy(&str[csize + 1], compat, cplen);
-	for (i = csize; i < repend; i++) {
-		char c = str[i];
-		if (c == '\0')
-			str[i] = 'C';
-		else if (c == ' ')
-			str[i] = '_';
+	tsize = csize;
+	len -= csize;
+	if (str)
+		str += csize;
+
+	of_property_for_each_string(dev->of_node, "compatible", p, compat) {
+		csize = strlen(compat) + 1;
+		tsize += csize;
+		if (csize > len)
+			continue;
+
+		csize = snprintf(str, len, "C%s", compat);
+		for (c = str; c; ) {
+			c = strchr(c, ' ');
+			if (c)
+				*c++ = '_';
+		}
+		len -= csize;
+		str += csize;
 	}
 
-	return repend;
+	return tsize;
 }
 
 int of_device_request_module(struct device *dev)
@@ -274,6 +278,8 @@ ssize_t of_device_modalias(struct device *dev, char *str, ssize_t len)
 	ssize_t sl = of_device_get_modalias(dev, str, len - 2);
 	if (sl < 0)
 		return sl;
+	if (sl > len - 2)
+		return -ENOMEM;
 
 	str[sl++] = '\n';
 	str[sl] = 0;
@@ -288,25 +294,22 @@ void of_device_uevent(struct device *dev, struct kobj_uevent_env *env)
 {
 	const char *compat;
 	struct alias_prop *app;
-	int seen = 0, cplen, sl;
+	struct property *p;
+	int seen = 0;
 
 	if ((!dev) || (!dev->of_node))
 		return;
 
 	add_uevent_var(env, "OF_NAME=%s", dev->of_node->name);
-	add_uevent_var(env, "OF_FULLNAME=%s", dev->of_node->full_name);
+	add_uevent_var(env, "OF_FULLNAME=%pOF", dev->of_node);
 	if (dev->of_node->type && strcmp("<NULL>", dev->of_node->type) != 0)
 		add_uevent_var(env, "OF_TYPE=%s", dev->of_node->type);
 
 	/* Since the compatible field can contain pretty much anything
 	 * it's not really legal to split it out with commas. We split it
 	 * up using a number of environment variables instead. */
-	compat = of_get_property(dev->of_node, "compatible", &cplen);
-	while (compat && *compat && cplen > 0) {
+	of_property_for_each_string(dev->of_node, "compatible", p, compat) {
 		add_uevent_var(env, "OF_COMPATIBLE_%d=%s", seen, compat);
-		sl = strlen(compat) + 1;
-		compat += sl;
-		cplen -= sl;
 		seen++;
 	}
 	add_uevent_var(env, "OF_COMPATIBLE_N=%d", seen);
diff --git a/drivers/of/dynamic.c b/drivers/of/dynamic.c
index 0542cf8b6e3d..301b6db2b48d 100644
--- a/drivers/of/dynamic.c
+++ b/drivers/of/dynamic.c
@@ -98,14 +98,14 @@ int of_reconfig_notify(unsigned long action, struct of_reconfig_data *p)
 	switch (action) {
 	case OF_RECONFIG_ATTACH_NODE:
 	case OF_RECONFIG_DETACH_NODE:
-		pr_debug("notify %-15s %s\n", action_names[action],
-			pr->dn->full_name);
+		pr_debug("notify %-15s %pOF\n", action_names[action],
+			pr->dn);
 		break;
 	case OF_RECONFIG_ADD_PROPERTY:
 	case OF_RECONFIG_REMOVE_PROPERTY:
 	case OF_RECONFIG_UPDATE_PROPERTY:
-		pr_debug("notify %-15s %s:%s\n", action_names[action],
-			pr->dn->full_name, pr->prop->name);
+		pr_debug("notify %-15s %pOF:%s\n", action_names[action],
+			pr->dn, pr->prop->name);
 		break;
 
 	}
@@ -328,11 +328,10 @@ void of_node_release(struct kobject *kobj)
 
 	/* We should never be releasing nodes that haven't been detached. */
 	if (!of_node_check_flag(node, OF_DETACHED)) {
-		pr_err("ERROR: Bad of_node_put() on %s\n", node->full_name);
+		pr_err("ERROR: Bad of_node_put() on %pOF\n", node);
 		dump_stack();
 		return;
 	}
-
 	if (!of_node_check_flag(node, OF_DYNAMIC))
 		return;
 
@@ -462,13 +461,13 @@ static void __of_changeset_entry_dump(struct of_changeset_entry *ce)
 	case OF_RECONFIG_ADD_PROPERTY:
 	case OF_RECONFIG_REMOVE_PROPERTY:
 	case OF_RECONFIG_UPDATE_PROPERTY:
-		pr_debug("cset<%p> %-15s %s/%s\n", ce, action_names[ce->action],
-			ce->np->full_name, ce->prop->name);
+		pr_debug("cset<%p> %-15s %pOF/%s\n", ce, action_names[ce->action],
+			ce->np, ce->prop->name);
 		break;
 	case OF_RECONFIG_ATTACH_NODE:
 	case OF_RECONFIG_DETACH_NODE:
-		pr_debug("cset<%p> %-15s %s\n", ce, action_names[ce->action],
-			ce->np->full_name);
+		pr_debug("cset<%p> %-15s %pOF\n", ce, action_names[ce->action],
+			ce->np);
 		break;
 	}
 }
@@ -539,7 +538,7 @@ static void __of_changeset_entry_notify(struct of_changeset_entry *ce, bool reve
 	}
 
 	if (ret)
-		pr_err("changeset notifier error @%s\n", ce->np->full_name);
+		pr_err("changeset notifier error @%pOF\n", ce->np);
 }
 
 static int __of_changeset_entry_apply(struct of_changeset_entry *ce)
@@ -570,8 +569,8 @@ static int __of_changeset_entry_apply(struct of_changeset_entry *ce)
 
 		ret = __of_add_property(ce->np, ce->prop);
 		if (ret) {
-			pr_err("changeset: add_property failed @%s/%s\n",
-				ce->np->full_name,
+			pr_err("changeset: add_property failed @%pOF/%s\n",
+				ce->np,
 				ce->prop->name);
 			break;
 		}
@@ -579,8 +578,8 @@ static int __of_changeset_entry_apply(struct of_changeset_entry *ce)
 	case OF_RECONFIG_REMOVE_PROPERTY:
 		ret = __of_remove_property(ce->np, ce->prop);
 		if (ret) {
-			pr_err("changeset: remove_property failed @%s/%s\n",
-				ce->np->full_name,
+			pr_err("changeset: remove_property failed @%pOF/%s\n",
+				ce->np,
 				ce->prop->name);
 			break;
 		}
@@ -598,8 +597,8 @@ static int __of_changeset_entry_apply(struct of_changeset_entry *ce)
 
 		ret = __of_update_property(ce->np, ce->prop, &old_prop);
 		if (ret) {
-			pr_err("changeset: update_property failed @%s/%s\n",
-				ce->np->full_name,
+			pr_err("changeset: update_property failed @%pOF/%s\n",
+				ce->np,
 				ce->prop->name);
 			break;
 		}
diff --git a/drivers/of/irq.c b/drivers/of/irq.c
index ab21c846eb27..ec00ae7384c2 100644
--- a/drivers/of/irq.c
+++ b/drivers/of/irq.c
@@ -59,20 +59,19 @@ EXPORT_SYMBOL_GPL(irq_of_parse_and_map);
 struct device_node *of_irq_find_parent(struct device_node *child)
 {
 	struct device_node *p;
-	const __be32 *parp;
+	phandle parent;
 
 	if (!of_node_get(child))
 		return NULL;
 
 	do {
-		parp = of_get_property(child, "interrupt-parent", NULL);
-		if (parp == NULL)
+		if (of_property_read_u32(child, "interrupt-parent", &parent)) {
 			p = of_get_parent(child);
-		else {
+		} else	{
 			if (of_irq_workarounds & OF_IMAP_NO_PHANDLE)
 				p = of_node_get(of_irq_dflt_pic);
 			else
-				p = of_find_node_by_phandle(be32_to_cpup(parp));
+				p = of_find_node_by_phandle(parent);
 		}
 		of_node_put(child);
 		child = p;
@@ -117,11 +116,8 @@ int of_irq_parse_raw(const __be32 *addr, struct of_phandle_args *out_irq)
 	 * is none, we are nice and just walk up the tree
 	 */
 	do {
-		tmp = of_get_property(ipar, "#interrupt-cells", NULL);
-		if (tmp != NULL) {
-			intsize = be32_to_cpu(*tmp);
+		if (!of_property_read_u32(ipar, "#interrupt-cells", &intsize))
 			break;
-		}
 		tnode = ipar;
 		ipar = of_irq_find_parent(ipar);
 		of_node_put(tnode);
@@ -131,7 +127,7 @@ int of_irq_parse_raw(const __be32 *addr, struct of_phandle_args *out_irq)
 		goto fail;
 	}
 
-	pr_debug("of_irq_parse_raw: ipar=%s, size=%d\n", of_node_full_name(ipar), intsize);
+	pr_debug("of_irq_parse_raw: ipar=%pOF, size=%d\n", ipar, intsize);
 
 	if (out_irq->args_count != intsize)
 		goto fail;
@@ -169,8 +165,7 @@ int of_irq_parse_raw(const __be32 *addr, struct of_phandle_args *out_irq)
 		/* Now check if cursor is an interrupt-controller and if it is
 		 * then we are done
 		 */
-		if (of_get_property(ipar, "interrupt-controller", NULL) !=
-				NULL) {
+		if (of_property_read_bool(ipar, "interrupt-controller")) {
 			pr_debug(" -> got it !\n");
 			return 0;
 		}
@@ -229,14 +224,14 @@ int of_irq_parse_raw(const __be32 *addr, struct of_phandle_args *out_irq)
 			/* Get #interrupt-cells and #address-cells of new
 			 * parent
 			 */
-			tmp = of_get_property(newpar, "#interrupt-cells", NULL);
-			if (tmp == NULL) {
+			if (of_property_read_u32(newpar, "#interrupt-cells",
+						 &newintsize)) {
 				pr_debug(" -> parent lacks #interrupt-cells!\n");
 				goto fail;
 			}
-			newintsize = be32_to_cpu(*tmp);
-			tmp = of_get_property(newpar, "#address-cells", NULL);
-			newaddrsize = (tmp == NULL) ? 0 : be32_to_cpu(*tmp);
+			if (of_property_read_u32(newpar, "#address-cells",
+						 &newaddrsize))
+				newaddrsize = 0;
 
 			pr_debug(" -> newintsize=%d, newaddrsize=%d\n",
 			    newintsize, newaddrsize);
@@ -269,7 +264,7 @@ int of_irq_parse_raw(const __be32 *addr, struct of_phandle_args *out_irq)
 	skiplevel:
 		/* Iterate again with new parent */
 		out_irq->np = newpar;
-		pr_debug(" -> new parent: %s\n", of_node_full_name(newpar));
+		pr_debug(" -> new parent: %pOF\n", newpar);
 		of_node_put(ipar);
 		ipar = newpar;
 		newpar = NULL;
@@ -297,11 +292,11 @@ EXPORT_SYMBOL_GPL(of_irq_parse_raw);
 int of_irq_parse_one(struct device_node *device, int index, struct of_phandle_args *out_irq)
 {
 	struct device_node *p;
-	const __be32 *intspec, *tmp, *addr;
-	u32 intsize, intlen;
+	const __be32 *addr;
+	u32 intsize;
 	int i, res;
 
-	pr_debug("of_irq_parse_one: dev=%s, index=%d\n", of_node_full_name(device), index);
+	pr_debug("of_irq_parse_one: dev=%pOF, index=%d\n", device, index);
 
 	/* OldWorld mac stuff is "special", handle out of line */
 	if (of_irq_workarounds & OF_IMAP_OLDWORLD_MAC)
@@ -316,42 +311,32 @@ int of_irq_parse_one(struct device_node *device, int index, struct of_phandle_ar
 	if (!res)
 		return of_irq_parse_raw(addr, out_irq);
 
-	/* Get the interrupts property */
-	intspec = of_get_property(device, "interrupts", &intlen);
-	if (intspec == NULL)
-		return -EINVAL;
-
-	intlen /= sizeof(*intspec);
-
-	pr_debug(" intspec=%d intlen=%d\n", be32_to_cpup(intspec), intlen);
-
 	/* Look for the interrupt parent. */
 	p = of_irq_find_parent(device);
 	if (p == NULL)
 		return -EINVAL;
 
 	/* Get size of interrupt specifier */
-	tmp = of_get_property(p, "#interrupt-cells", NULL);
-	if (tmp == NULL) {
+	if (of_property_read_u32(p, "#interrupt-cells", &intsize)) {
 		res = -EINVAL;
 		goto out;
 	}
-	intsize = be32_to_cpu(*tmp);
 
-	pr_debug(" intsize=%d intlen=%d\n", intsize, intlen);
-
-	/* Check index */
-	if ((index + 1) * intsize > intlen) {
-		res = -EINVAL;
-		goto out;
-	}
+	pr_debug(" parent=%pOF, intsize=%d\n", p, intsize);
 
 	/* Copy intspec into irq structure */
-	intspec += index * intsize;
 	out_irq->np = p;
 	out_irq->args_count = intsize;
-	for (i = 0; i < intsize; i++)
-		out_irq->args[i] = be32_to_cpup(intspec++);
+	for (i = 0; i < intsize; i++) {
+		res = of_property_read_u32_index(device, "interrupts",
+						 (index * intsize) + i,
+						 out_irq->args + i);
+		if (res)
+			goto out;
+	}
+
+	pr_debug(" intspec=%d\n", *out_irq->args);
+
 
 	/* Check if there are any interrupt-map translations to process */
 	res = of_irq_parse_raw(addr, out_irq);
@@ -508,7 +493,7 @@ void __init of_irq_init(const struct of_device_id *matches)
 	INIT_LIST_HEAD(&intc_parent_list);
 
 	for_each_matching_node_and_match(np, matches, &match) {
-		if (!of_find_property(np, "interrupt-controller", NULL) ||
+		if (!of_property_read_bool(np, "interrupt-controller") ||
 				!of_device_is_available(np))
 			continue;
 
@@ -555,8 +540,8 @@ void __init of_irq_init(const struct of_device_id *matches)
 
 			of_node_set_flag(desc->dev, OF_POPULATED);
 
-			pr_debug("of_irq_init: init %s (%p), parent %p\n",
-				 desc->dev->full_name,
+			pr_debug("of_irq_init: init %pOF (%p), parent %p\n",
+				 desc->dev,
 				 desc->dev, desc->interrupt_parent);
 			ret = desc->irq_init_cb(desc->dev,
 						desc->interrupt_parent);
diff --git a/drivers/of/of_mdio.c b/drivers/of/of_mdio.c
index e0dbd6e48a98..d94dd8b77abd 100644
--- a/drivers/of/of_mdio.c
+++ b/drivers/of/of_mdio.c
@@ -166,8 +166,8 @@ static bool of_mdiobus_child_is_phy(struct device_node *child)
 
 	if (of_match_node(whitelist_phys, child)) {
 		pr_warn(FW_WARN
-			"%s: Whitelisted compatible string. Please remove\n",
-			child->full_name);
+			"%pOF: Whitelisted compatible string. Please remove\n",
+			child);
 		return true;
 	}
 
@@ -421,19 +421,14 @@ int of_phy_register_fixed_link(struct device_node *np)
 {
 	struct fixed_phy_status status = {};
 	struct device_node *fixed_link_node;
-	const __be32 *fixed_link_prop;
-	int link_gpio;
-	int len, err;
-	struct phy_device *phy;
+	u32 fixed_link_prop[5];
 	const char *managed;
+	int link_gpio = -1;
 
-	err = of_property_read_string(np, "managed", &managed);
-	if (err == 0) {
-		if (strcmp(managed, "in-band-status") == 0) {
-			/* status is zeroed, namely its .link member */
-			phy = fixed_phy_register(PHY_POLL, &status, -1, np);
-			return PTR_ERR_OR_ZERO(phy);
-		}
+	if (of_property_read_string(np, "managed", &managed) == 0 &&
+	    strcmp(managed, "in-band-status") == 0) {
+		/* status is zeroed, namely its .link member */
+		goto register_phy;
 	}
 
 	/* New binding */
@@ -456,23 +451,25 @@ int of_phy_register_fixed_link(struct device_node *np)
 		if (link_gpio == -EPROBE_DEFER)
 			return -EPROBE_DEFER;
 
-		phy = fixed_phy_register(PHY_POLL, &status, link_gpio, np);
-		return PTR_ERR_OR_ZERO(phy);
+		goto register_phy;
 	}
 
 	/* Old binding */
-	fixed_link_prop = of_get_property(np, "fixed-link", &len);
-	if (fixed_link_prop && len == (5 * sizeof(__be32))) {
+	if (of_property_read_u32_array(np, "fixed-link", fixed_link_prop,
+				       ARRAY_SIZE(fixed_link_prop)) == 0) {
 		status.link = 1;
-		status.duplex = be32_to_cpu(fixed_link_prop[1]);
-		status.speed = be32_to_cpu(fixed_link_prop[2]);
-		status.pause = be32_to_cpu(fixed_link_prop[3]);
-		status.asym_pause = be32_to_cpu(fixed_link_prop[4]);
-		phy = fixed_phy_register(PHY_POLL, &status, -1, np);
-		return PTR_ERR_OR_ZERO(phy);
+		status.duplex = fixed_link_prop[1];
+		status.speed  = fixed_link_prop[2];
+		status.pause  = fixed_link_prop[3];
+		status.asym_pause = fixed_link_prop[4];
+		goto register_phy;
 	}
 
 	return -ENODEV;
+
+register_phy:
+	return PTR_ERR_OR_ZERO(fixed_phy_register(PHY_POLL, &status, link_gpio,
+						  np));
 }
 EXPORT_SYMBOL(of_phy_register_fixed_link);
 
diff --git a/drivers/of/of_pci.c b/drivers/of/of_pci.c
index c9d4d3a7b0fe..e9ec931f5b9a 100644
--- a/drivers/of/of_pci.c
+++ b/drivers/of/of_pci.c
@@ -57,15 +57,14 @@ EXPORT_SYMBOL_GPL(of_pci_find_child_device);
  */
 int of_pci_get_devfn(struct device_node *np)
 {
-	unsigned int size;
-	const __be32 *reg;
+	u32 reg[5];
+	int error;
 
-	reg = of_get_property(np, "reg", &size);
+	error = of_property_read_u32_array(np, "reg", reg, ARRAY_SIZE(reg));
+	if (error)
+		return error;
 
-	if (!reg || size < 5 * sizeof(__be32))
-		return -EINVAL;
-
-	return (be32_to_cpup(reg) >> 8) & 0xff;
+	return (reg[0] >> 8) & 0xff;
 }
 EXPORT_SYMBOL_GPL(of_pci_get_devfn);
 
@@ -78,16 +77,17 @@ EXPORT_SYMBOL_GPL(of_pci_get_devfn);
  */
 int of_pci_parse_bus_range(struct device_node *node, struct resource *res)
 {
-	const __be32 *values;
-	int len;
+	u32 bus_range[2];
+	int error;
 
-	values = of_get_property(node, "bus-range", &len);
-	if (!values || len < sizeof(*values) * 2)
-		return -EINVAL;
+	error = of_property_read_u32_array(node, "bus-range", bus_range,
+					   ARRAY_SIZE(bus_range));
+	if (error)
+		return error;
 
 	res->name = node->name;
-	res->start = be32_to_cpup(values++);
-	res->end = be32_to_cpup(values);
+	res->start = bus_range[0];
+	res->end = bus_range[1];
 	res->flags = IORESOURCE_BUS;
 
 	return 0;
@@ -105,17 +105,14 @@ EXPORT_SYMBOL_GPL(of_pci_parse_bus_range);
  */
 int of_get_pci_domain_nr(struct device_node *node)
 {
-	const __be32 *value;
-	int len;
-	u16 domain;
-
-	value = of_get_property(node, "linux,pci-domain", &len);
-	if (!value || len < sizeof(*value))
-		return -EINVAL;
+	u32 domain;
+	int error;
 
-	domain = (u16)be32_to_cpup(value);
+	error = of_property_read_u32(node, "linux,pci-domain", &domain);
+	if (error)
+		return error;
 
-	return domain;
+	return (u16)domain;
 }
 EXPORT_SYMBOL_GPL(of_get_pci_domain_nr);
 
@@ -204,15 +201,15 @@ int of_pci_get_host_bridge_resources(struct device_node *dev,
 	if (!bus_range)
 		return -ENOMEM;
 
-	pr_info("host bridge %s ranges:\n", dev->full_name);
+	pr_info("host bridge %pOF ranges:\n", dev);
 
 	err = of_pci_parse_bus_range(dev, bus_range);
 	if (err) {
 		bus_range->start = busno;
 		bus_range->end = bus_max;
 		bus_range->flags = IORESOURCE_BUS;
-		pr_info("  No bus range found for %s, using %pR\n",
-			dev->full_name, bus_range);
+		pr_info("  No bus range found for %pOF, using %pR\n",
+			dev, bus_range);
 	} else {
 		if (bus_range->end > bus_range->start + bus_max)
 			bus_range->end = bus_range->start + bus_max;
@@ -258,14 +255,14 @@ int of_pci_get_host_bridge_resources(struct device_node *dev,
 
 		if (resource_type(res) == IORESOURCE_IO) {
 			if (!io_base) {
-				pr_err("I/O range found for %s. Please provide an io_base pointer to save CPU base address\n",
-					dev->full_name);
+				pr_err("I/O range found for %pOF. Please provide an io_base pointer to save CPU base address\n",
+					dev);
 				err = -EINVAL;
 				goto conversion_failed;
 			}
 			if (*io_base != (resource_size_t)OF_BAD_ADDR)
-				pr_warn("More than one I/O resource converted for %s. CPU base address for old range lost!\n",
-					dev->full_name);
+				pr_warn("More than one I/O resource converted for %pOF. CPU base address for old range lost!\n",
+					dev);
 			*io_base = range.cpu_addr;
 		}
 
@@ -325,7 +322,7 @@ int of_pci_map_rid(struct device_node *np, u32 rid,
 	}
 
 	if (!map_len || map_len % (4 * sizeof(*map))) {
-		pr_err("%s: Error: Bad %s length: %d\n", np->full_name,
+		pr_err("%pOF: Error: Bad %s length: %d\n", np,
 			map_name, map_len);
 		return -EINVAL;
 	}
@@ -349,8 +346,8 @@ int of_pci_map_rid(struct device_node *np, u32 rid,
 		u32 rid_len = be32_to_cpup(map + 3);
 
 		if (rid_base & ~map_mask) {
-			pr_err("%s: Invalid %s translation - %s-mask (0x%x) ignores rid-base (0x%x)\n",
-				np->full_name, map_name, map_name,
+			pr_err("%pOF: Invalid %s translation - %s-mask (0x%x) ignores rid-base (0x%x)\n",
+				np, map_name, map_name,
 				map_mask, rid_base);
 			return -EFAULT;
 		}
@@ -375,14 +372,13 @@ int of_pci_map_rid(struct device_node *np, u32 rid,
 		if (id_out)
 			*id_out = masked_rid - rid_base + out_base;
 
-		pr_debug("%s: %s, using mask %08x, rid-base: %08x, out-base: %08x, length: %08x, rid: %08x -> %08x\n",
-			np->full_name, map_name, map_mask, rid_base, out_base,
+		pr_debug("%pOF: %s, using mask %08x, rid-base: %08x, out-base: %08x, length: %08x, rid: %08x -> %08x\n",
+			np, map_name, map_mask, rid_base, out_base,
 			rid_len, rid, *id_out);
 		return 0;
 	}
 
-	pr_err("%s: Invalid %s translation - no match for rid 0x%x on %s\n",
-		np->full_name, map_name, rid,
-		target && *target ? (*target)->full_name : "any target");
+	pr_err("%pOF: Invalid %s translation - no match for rid 0x%x on %pOF\n",
+		np, map_name, rid, target && *target ? *target : NULL);
 	return -EFAULT;
 }
diff --git a/drivers/of/overlay.c b/drivers/of/overlay.c
index c0e4ee1cd1ba..8ecfee31ab6d 100644
--- a/drivers/of/overlay.c
+++ b/drivers/of/overlay.c
@@ -35,6 +35,7 @@
 struct of_overlay_info {
 	struct device_node *target;
 	struct device_node *overlay;
+	bool is_symbols_node;
 };
 
 /**
@@ -55,7 +56,8 @@ struct of_overlay {
 };
 
 static int of_overlay_apply_one(struct of_overlay *ov,
-		struct device_node *target, const struct device_node *overlay);
+		struct device_node *target, const struct device_node *overlay,
+		bool is_symbols_node);
 
 static BLOCKING_NOTIFIER_HEAD(of_overlay_chain);
 
@@ -92,10 +94,74 @@ static int of_overlay_notify(struct of_overlay *ov,
 	return 0;
 }
 
+static struct property *dup_and_fixup_symbol_prop(struct of_overlay *ov,
+		const struct property *prop)
+{
+	struct of_overlay_info *ovinfo;
+	struct property *new;
+	const char *overlay_name;
+	char *label_path;
+	char *symbol_path;
+	const char *target_path;
+	int k;
+	int label_path_len;
+	int overlay_name_len;
+	int target_path_len;
+
+	if (!prop->value)
+		return NULL;
+	symbol_path = prop->value;
+
+	new = kzalloc(sizeof(*new), GFP_KERNEL);
+	if (!new)
+		return NULL;
+
+	for (k = 0; k < ov->count; k++) {
+		ovinfo = &ov->ovinfo_tab[k];
+		overlay_name = ovinfo->overlay->full_name;
+		overlay_name_len = strlen(overlay_name);
+		if (!strncasecmp(symbol_path, overlay_name, overlay_name_len))
+			break;
+	}
+
+	if (k >= ov->count)
+		goto err_free;
+
+	target_path = ovinfo->target->full_name;
+	target_path_len = strlen(target_path);
+
+	label_path = symbol_path + overlay_name_len;
+	label_path_len = strlen(label_path);
+
+	new->name = kstrdup(prop->name, GFP_KERNEL);
+	new->length = target_path_len + label_path_len + 1;
+	new->value = kzalloc(new->length, GFP_KERNEL);
+
+	if (!new->name || !new->value)
+		goto err_free;
+
+	strcpy(new->value, target_path);
+	strcpy(new->value + target_path_len, label_path);
+
+	/* mark the property as dynamic */
+	of_property_set_flag(new, OF_DYNAMIC);
+
+	return new;
+
+ err_free:
+	kfree(new->name);
+	kfree(new->value);
+	kfree(new);
+	return NULL;
+
+
+}
+
 static int of_overlay_apply_single_property(struct of_overlay *ov,
-		struct device_node *target, struct property *prop)
+		struct device_node *target, struct property *prop,
+		bool is_symbols_node)
 {
-	struct property *propn, *tprop;
+	struct property *propn = NULL, *tprop;
 
 	/* NOTE: Multiple changes of single properties not supported */
 	tprop = of_find_property(target, prop->name, NULL);
@@ -106,7 +172,15 @@ static int of_overlay_apply_single_property(struct of_overlay *ov,
 	    of_prop_cmp(prop->name, "linux,phandle") == 0)
 		return 0;
 
-	propn = __of_prop_dup(prop, GFP_KERNEL);
+	if (is_symbols_node) {
+		/* changing a property in __symbols__ node not allowed */
+		if (tprop)
+			return -EINVAL;
+		propn = dup_and_fixup_symbol_prop(ov, prop);
+	} else {
+		propn = __of_prop_dup(prop, GFP_KERNEL);
+	}
+
 	if (propn == NULL)
 		return -ENOMEM;
 
@@ -130,18 +204,21 @@ static int of_overlay_apply_single_device_node(struct of_overlay *ov,
 		return -ENOMEM;
 
 	/* NOTE: Multiple mods of created nodes not supported */
-	tchild = of_get_child_by_name(target, cname);
+	for_each_child_of_node(target, tchild)
+		if (!of_node_cmp(cname, kbasename(tchild->full_name)))
+			break;
+
 	if (tchild != NULL) {
 		/* new overlay phandle value conflicts with existing value */
 		if (child->phandle)
 			return -EINVAL;
 
 		/* apply overlay recursively */
-		ret = of_overlay_apply_one(ov, tchild, child);
+		ret = of_overlay_apply_one(ov, tchild, child, 0);
 		of_node_put(tchild);
 	} else {
 		/* create empty tree as a target */
-		tchild = __of_node_dup(child, "%s/%s", target->full_name, cname);
+		tchild = __of_node_dup(child, "%pOF/%s", target, cname);
 		if (!tchild)
 			return -ENOMEM;
 
@@ -152,7 +229,7 @@ static int of_overlay_apply_single_device_node(struct of_overlay *ov,
 		if (ret)
 			return ret;
 
-		ret = of_overlay_apply_one(ov, tchild, child);
+		ret = of_overlay_apply_one(ov, tchild, child, 0);
 		if (ret)
 			return ret;
 	}
@@ -168,26 +245,32 @@ static int of_overlay_apply_single_device_node(struct of_overlay *ov,
  * by using the changeset.
  */
 static int of_overlay_apply_one(struct of_overlay *ov,
-		struct device_node *target, const struct device_node *overlay)
+		struct device_node *target, const struct device_node *overlay,
+		bool is_symbols_node)
 {
 	struct device_node *child;
 	struct property *prop;
 	int ret;
 
 	for_each_property_of_node(overlay, prop) {
-		ret = of_overlay_apply_single_property(ov, target, prop);
+		ret = of_overlay_apply_single_property(ov, target, prop,
+						       is_symbols_node);
 		if (ret) {
-			pr_err("Failed to apply prop @%s/%s\n",
-			       target->full_name, prop->name);
+			pr_err("Failed to apply prop @%pOF/%s\n",
+			       target, prop->name);
 			return ret;
 		}
 	}
 
+	/* do not allow symbols node to have any children */
+	if (is_symbols_node)
+		return 0;
+
 	for_each_child_of_node(overlay, child) {
 		ret = of_overlay_apply_single_device_node(ov, target, child);
 		if (ret != 0) {
-			pr_err("Failed to apply single node @%s/%s\n",
-			       target->full_name, child->name);
+			pr_err("Failed to apply single node @%pOF/%s\n",
+			       target, child->name);
 			of_node_put(child);
 			return ret;
 		}
@@ -213,9 +296,10 @@ static int of_overlay_apply(struct of_overlay *ov)
 	for (i = 0; i < ov->count; i++) {
 		struct of_overlay_info *ovinfo = &ov->ovinfo_tab[i];
 
-		err = of_overlay_apply_one(ov, ovinfo->target, ovinfo->overlay);
+		err = of_overlay_apply_one(ov, ovinfo->target, ovinfo->overlay,
+					   ovinfo->is_symbols_node);
 		if (err != 0) {
-			pr_err("apply failed '%s'\n", ovinfo->target->full_name);
+			pr_err("apply failed '%pOF'\n", ovinfo->target);
 			return err;
 		}
 	}
@@ -311,6 +395,9 @@ static int of_build_overlay_info(struct of_overlay *ov,
 	for_each_child_of_node(tree, node)
 		cnt++;
 
+	if (of_get_child_by_name(tree, "__symbols__"))
+		cnt++;
+
 	ovinfo = kcalloc(cnt, sizeof(*ovinfo), GFP_KERNEL);
 	if (ovinfo == NULL)
 		return -ENOMEM;
@@ -322,6 +409,20 @@ static int of_build_overlay_info(struct of_overlay *ov,
 			cnt++;
 	}
 
+	node = of_get_child_by_name(tree, "__symbols__");
+	if (node) {
+		ovinfo[cnt].overlay = node;
+		ovinfo[cnt].target = of_find_node_by_path("/__symbols__");
+		ovinfo[cnt].is_symbols_node = 1;
+
+		if (!ovinfo[cnt].target) {
+			pr_err("no symbols in root of device tree.\n");
+			return -EINVAL;
+		}
+
+		cnt++;
+	}
+
 	/* if nothing filled, return error */
 	if (cnt == 0) {
 		kfree(ovinfo);
@@ -400,8 +501,8 @@ int of_overlay_create(struct device_node *tree)
 	/* build the overlay info structures */
 	err = of_build_overlay_info(ov, tree);
 	if (err) {
-		pr_err("of_build_overlay_info() failed for tree@%s\n",
-		       tree->full_name);
+		pr_err("of_build_overlay_info() failed for tree@%pOF\n",
+		       tree);
 		goto err_free_idr;
 	}
 
@@ -480,9 +581,8 @@ static int overlay_is_topmost(struct of_overlay *ov, struct device_node *dn)
 		/* check against each subtree affected by this overlay */
 		list_for_each_entry(ce, &ovt->cset.entries, node) {
 			if (overlay_subtree_check(ce->np, dn)) {
-				pr_err("%s: #%d clashes #%d @%s\n",
-					__func__, ov->id, ovt->id,
-					dn->full_name);
+				pr_err("%s: #%d clashes #%d @%pOF\n",
+					__func__, ov->id, ovt->id, dn);
 				return 0;
 			}
 		}
diff --git a/drivers/of/platform.c b/drivers/of/platform.c
index b19524623498..ac15d0e3d27d 100644
--- a/drivers/of/platform.c
+++ b/drivers/of/platform.c
@@ -118,7 +118,7 @@ struct platform_device *of_device_alloc(struct device_node *np,
 	int rc, i, num_reg = 0, num_irq;
 	struct resource *res, temp_res;
 
-	dev = platform_device_alloc("", -1);
+	dev = platform_device_alloc("", PLATFORM_DEVID_NONE);
 	if (!dev)
 		return NULL;
 
@@ -228,7 +228,7 @@ static struct amba_device *of_amba_device_create(struct device_node *node,
 	const void *prop;
 	int i, ret;
 
-	pr_debug("Creating amba device %s\n", node->full_name);
+	pr_debug("Creating amba device %pOF\n", node);
 
 	if (!of_device_is_available(node) ||
 	    of_node_test_and_set_flag(node, OF_POPULATED))
@@ -259,15 +259,15 @@ static struct amba_device *of_amba_device_create(struct device_node *node,
 
 	ret = of_address_to_resource(node, 0, &dev->res);
 	if (ret) {
-		pr_err("amba: of_address_to_resource() failed (%d) for %s\n",
-		       ret, node->full_name);
+		pr_err("amba: of_address_to_resource() failed (%d) for %pOF\n",
+		       ret, node);
 		goto err_free;
 	}
 
 	ret = amba_device_add(dev, &iomem_resource);
 	if (ret) {
-		pr_err("amba_device_add() failed (%d) for %s\n",
-		       ret, node->full_name);
+		pr_err("amba_device_add() failed (%d) for %pOF\n",
+		       ret, node);
 		goto err_free;
 	}
 
@@ -310,7 +310,7 @@ static const struct of_dev_auxdata *of_dev_lookup(const struct of_dev_auxdata *l
 		if (!of_address_to_resource(np, 0, &res))
 			if (res.start != auxdata->phys_addr)
 				continue;
-		pr_debug("%s: devname=%s\n", np->full_name, auxdata->name);
+		pr_debug("%pOF: devname=%s\n", np, auxdata->name);
 		return auxdata;
 	}
 
@@ -323,7 +323,7 @@ static const struct of_dev_auxdata *of_dev_lookup(const struct of_dev_auxdata *l
 		if (!of_device_is_compatible(np, auxdata->compatible))
 			continue;
 		if (!auxdata->phys_addr && !auxdata->name) {
-			pr_debug("%s: compatible match\n", np->full_name);
+			pr_debug("%pOF: compatible match\n", np);
 			return auxdata;
 		}
 	}
@@ -356,14 +356,14 @@ static int of_platform_bus_create(struct device_node *bus,
 
 	/* Make sure it has a compatible property */
 	if (strict && (!of_get_property(bus, "compatible", NULL))) {
-		pr_debug("%s() - skipping %s, no compatible prop\n",
-			 __func__, bus->full_name);
+		pr_debug("%s() - skipping %pOF, no compatible prop\n",
+			 __func__, bus);
 		return 0;
 	}
 
 	if (of_node_check_flag(bus, OF_POPULATED_BUS)) {
-		pr_debug("%s() - skipping %s, already populated\n",
-			__func__, bus->full_name);
+		pr_debug("%s() - skipping %pOF, already populated\n",
+			__func__, bus);
 		return 0;
 	}
 
@@ -387,7 +387,7 @@ static int of_platform_bus_create(struct device_node *bus,
 		return 0;
 
 	for_each_child_of_node(bus, child) {
-		pr_debug("   create child: %s\n", child->full_name);
+		pr_debug("   create child: %pOF\n", child);
 		rc = of_platform_bus_create(child, matches, lookup, &dev->dev, strict);
 		if (rc) {
 			of_node_put(child);
@@ -419,7 +419,7 @@ int of_platform_bus_probe(struct device_node *root,
 		return -EINVAL;
 
 	pr_debug("%s()\n", __func__);
-	pr_debug(" starting at: %s\n", root->full_name);
+	pr_debug(" starting at: %pOF\n", root);
 
 	/* Do a self check of bus type, if there's a match, create children */
 	if (of_match_node(matches, root)) {
@@ -471,7 +471,7 @@ int of_platform_populate(struct device_node *root,
 		return -EINVAL;
 
 	pr_debug("%s()\n", __func__);
-	pr_debug(" starting at: %s\n", root->full_name);
+	pr_debug(" starting at: %pOF\n", root);
 
 	for_each_child_of_node(root, child) {
 		rc = of_platform_bus_create(child, matches, lookup, parent, true);
@@ -660,8 +660,8 @@ static int of_platform_notify(struct notifier_block *nb,
 		of_dev_put(pdev_parent);
 
 		if (pdev == NULL) {
-			pr_err("%s: failed to create for '%s'\n",
-					__func__, rd->dn->full_name);
+			pr_err("%s: failed to create for '%pOF'\n",
+					__func__, rd->dn);
 			/* of_platform_device_create tosses the error code */
 			return notifier_from_errno(-EINVAL);
 		}
diff --git a/drivers/of/property.c b/drivers/of/property.c
index 067f9fab7b77..fbb72116e9d4 100644
--- a/drivers/of/property.c
+++ b/drivers/of/property.c
@@ -55,8 +55,8 @@ int of_property_count_elems_of_size(const struct device_node *np,
 		return -ENODATA;
 
 	if (prop->length % elem_size != 0) {
-		pr_err("size of %s in node %s is not a multiple of %d\n",
-		       propname, np->full_name, elem_size);
+		pr_err("size of %s in node %pOF is not a multiple of %d\n",
+		       propname, np, elem_size);
 		return -EINVAL;
 	}
 
@@ -537,8 +537,8 @@ int of_graph_parse_endpoint(const struct device_node *node,
 {
 	struct device_node *port_node = of_get_parent(node);
 
-	WARN_ONCE(!port_node, "%s(): endpoint %s has no parent node\n",
-		  __func__, node->full_name);
+	WARN_ONCE(!port_node, "%s(): endpoint %pOF has no parent node\n",
+		  __func__, node);
 
 	memset(endpoint, 0, sizeof(*endpoint));
 
@@ -621,14 +621,13 @@ struct device_node *of_graph_get_next_endpoint(const struct device_node *parent,
 		of_node_put(node);
 
 		if (!port) {
-			pr_err("graph: no port node found in %s\n",
-			       parent->full_name);
+			pr_err("graph: no port node found in %pOF\n", parent);
 			return NULL;
 		}
 	} else {
 		port = of_get_parent(prev);
-		if (WARN_ONCE(!port, "%s(): endpoint %s has no parent node\n",
-			      __func__, prev->full_name))
+		if (WARN_ONCE(!port, "%s(): endpoint %pOF has no parent node\n",
+			      __func__, prev))
 			return NULL;
 	}
 
@@ -797,8 +796,8 @@ struct device_node *of_graph_get_remote_node(const struct device_node *node,
 
 	endpoint_node = of_graph_get_endpoint_by_regs(node, port, endpoint);
 	if (!endpoint_node) {
-		pr_debug("no valid endpoint (%d, %d) for node %s\n",
-			 port, endpoint, node->full_name);
+		pr_debug("no valid endpoint (%d, %d) for node %pOF\n",
+			 port, endpoint, node);
 		return NULL;
 	}
 
@@ -828,23 +827,23 @@ static void of_fwnode_put(struct fwnode_handle *fwnode)
 	of_node_put(to_of_node(fwnode));
 }
 
-static bool of_fwnode_device_is_available(struct fwnode_handle *fwnode)
+static bool of_fwnode_device_is_available(const struct fwnode_handle *fwnode)
 {
 	return of_device_is_available(to_of_node(fwnode));
 }
 
-static bool of_fwnode_property_present(struct fwnode_handle *fwnode,
+static bool of_fwnode_property_present(const struct fwnode_handle *fwnode,
 				       const char *propname)
 {
 	return of_property_read_bool(to_of_node(fwnode), propname);
 }
 
-static int of_fwnode_property_read_int_array(struct fwnode_handle *fwnode,
+static int of_fwnode_property_read_int_array(const struct fwnode_handle *fwnode,
 					     const char *propname,
 					     unsigned int elem_size, void *val,
 					     size_t nval)
 {
-	struct device_node *node = to_of_node(fwnode);
+	const struct device_node *node = to_of_node(fwnode);
 
 	if (!val)
 		return of_property_count_elems_of_size(node, propname,
@@ -864,24 +863,26 @@ static int of_fwnode_property_read_int_array(struct fwnode_handle *fwnode,
 	return -ENXIO;
 }
 
-static int of_fwnode_property_read_string_array(struct fwnode_handle *fwnode,
-						const char *propname,
-						const char **val, size_t nval)
+static int
+of_fwnode_property_read_string_array(const struct fwnode_handle *fwnode,
+				     const char *propname, const char **val,
+				     size_t nval)
 {
-	struct device_node *node = to_of_node(fwnode);
+	const struct device_node *node = to_of_node(fwnode);
 
 	return val ?
 		of_property_read_string_array(node, propname, val, nval) :
 		of_property_count_strings(node, propname);
 }
 
-static struct fwnode_handle *of_fwnode_get_parent(struct fwnode_handle *fwnode)
+static struct fwnode_handle *
+of_fwnode_get_parent(const struct fwnode_handle *fwnode)
 {
 	return of_fwnode_handle(of_get_parent(to_of_node(fwnode)));
 }
 
 static struct fwnode_handle *
-of_fwnode_get_next_child_node(struct fwnode_handle *fwnode,
+of_fwnode_get_next_child_node(const struct fwnode_handle *fwnode,
 			      struct fwnode_handle *child)
 {
 	return of_fwnode_handle(of_get_next_available_child(to_of_node(fwnode),
@@ -889,10 +890,10 @@ of_fwnode_get_next_child_node(struct fwnode_handle *fwnode,
 }
 
 static struct fwnode_handle *
-of_fwnode_get_named_child_node(struct fwnode_handle *fwnode,
+of_fwnode_get_named_child_node(const struct fwnode_handle *fwnode,
 			       const char *childname)
 {
-	struct device_node *node = to_of_node(fwnode);
+	const struct device_node *node = to_of_node(fwnode);
 	struct device_node *child;
 
 	for_each_available_child_of_node(node, child)
@@ -902,8 +903,38 @@ of_fwnode_get_named_child_node(struct fwnode_handle *fwnode,
 	return NULL;
 }
 
+static int
+of_fwnode_get_reference_args(const struct fwnode_handle *fwnode,
+			     const char *prop, const char *nargs_prop,
+			     unsigned int nargs, unsigned int index,
+			     struct fwnode_reference_args *args)
+{
+	struct of_phandle_args of_args;
+	unsigned int i;
+	int ret;
+
+	if (nargs_prop)
+		ret = of_parse_phandle_with_args(to_of_node(fwnode), prop,
+						 nargs_prop, index, &of_args);
+	else
+		ret = of_parse_phandle_with_fixed_args(to_of_node(fwnode), prop,
+						       nargs, index, &of_args);
+	if (ret < 0)
+		return ret;
+	if (!args)
+		return 0;
+
+	args->nargs = of_args.args_count;
+	args->fwnode = of_fwnode_handle(of_args.np);
+
+	for (i = 0; i < NR_FWNODE_REFERENCE_ARGS; i++)
+		args->args[i] = i < of_args.args_count ? of_args.args[i] : 0;
+
+	return 0;
+}
+
 static struct fwnode_handle *
-of_fwnode_graph_get_next_endpoint(struct fwnode_handle *fwnode,
+of_fwnode_graph_get_next_endpoint(const struct fwnode_handle *fwnode,
 				  struct fwnode_handle *prev)
 {
 	return of_fwnode_handle(of_graph_get_next_endpoint(to_of_node(fwnode),
@@ -911,10 +942,10 @@ of_fwnode_graph_get_next_endpoint(struct fwnode_handle *fwnode,
 }
 
 static struct fwnode_handle *
-of_fwnode_graph_get_remote_endpoint(struct fwnode_handle *fwnode)
+of_fwnode_graph_get_remote_endpoint(const struct fwnode_handle *fwnode)
 {
-	return of_fwnode_handle(of_parse_phandle(to_of_node(fwnode),
-						 "remote-endpoint", 0));
+	return of_fwnode_handle(
+		of_graph_get_remote_endpoint(to_of_node(fwnode)));
 }
 
 static struct fwnode_handle *
@@ -934,10 +965,10 @@ of_fwnode_graph_get_port_parent(struct fwnode_handle *fwnode)
 	return of_fwnode_handle(of_get_next_parent(np));
 }
 
-static int of_fwnode_graph_parse_endpoint(struct fwnode_handle *fwnode,
+static int of_fwnode_graph_parse_endpoint(const struct fwnode_handle *fwnode,
 					  struct fwnode_endpoint *endpoint)
 {
-	struct device_node *node = to_of_node(fwnode);
+	const struct device_node *node = to_of_node(fwnode);
 	struct device_node *port_node = of_get_parent(node);
 
 	endpoint->local_fwnode = fwnode;
@@ -960,8 +991,10 @@ const struct fwnode_operations of_fwnode_ops = {
 	.get_parent = of_fwnode_get_parent,
 	.get_next_child_node = of_fwnode_get_next_child_node,
 	.get_named_child_node = of_fwnode_get_named_child_node,
+	.get_reference_args = of_fwnode_get_reference_args,
 	.graph_get_next_endpoint = of_fwnode_graph_get_next_endpoint,
 	.graph_get_remote_endpoint = of_fwnode_graph_get_remote_endpoint,
 	.graph_get_port_parent = of_fwnode_graph_get_port_parent,
 	.graph_parse_endpoint = of_fwnode_graph_parse_endpoint,
 };
+EXPORT_SYMBOL_GPL(of_fwnode_ops);
diff --git a/drivers/of/unittest-data/Makefile b/drivers/of/unittest-data/Makefile
index 6e00a9c69e58..2d135fba94c1 100644
--- a/drivers/of/unittest-data/Makefile
+++ b/drivers/of/unittest-data/Makefile
@@ -1,18 +1,27 @@
 obj-y += testcases.dtb.o
+
+targets += testcases.dtb testcases.dtb.S
+
+ifdef CONFIG_OF_OVERLAY
+
 obj-y += overlay.dtb.o
 obj-y += overlay_bad_phandle.dtb.o
+obj-y += overlay_bad_symbol.dtb.o
 obj-y += overlay_base.dtb.o
 
-targets += testcases.dtb testcases.dtb.S
 targets += overlay.dtb overlay.dtb.S
 targets += overlay_bad_phandle.dtb overlay_bad_phandle.dtb.S
+targets += overlay_bad_symbol.dtb overlay_bad_symbol.dtb.S
 targets += overlay_base.dtb overlay_base.dtb.S
 
-.PRECIOUS: \
-	$(obj)/%.dtb.S \
-	$(obj)/%.dtb
-
 # enable creation of __symbols__ node
 DTC_FLAGS_overlay := -@
 DTC_FLAGS_overlay_bad_phandle := -@
+DTC_FLAGS_overlay_bad_symbol := -@
 DTC_FLAGS_overlay_base := -@
+
+endif
+
+.PRECIOUS: \
+	$(obj)/%.dtb.S \
+	$(obj)/%.dtb
diff --git a/drivers/of/unittest-data/overlay.dts b/drivers/of/unittest-data/overlay.dts
index 6cd7e6a0c13e..9e791fcf05dd 100644
--- a/drivers/of/unittest-data/overlay.dts
+++ b/drivers/of/unittest-data/overlay.dts
@@ -7,7 +7,7 @@
 		target = <&electric_1>;
 
 		__overlay__ {
-			status = "ok";
+			status = "okay";
 
 			hvac_2: hvac-large-1 {
 				compatible = "ot,hvac-large";
@@ -23,9 +23,24 @@
 		__overlay__ {
 			#address-cells = <1>;
 			#size-cells = <1>;
-			status = "ok";
+			status = "okay";
 
-			ride@200 {
+			ride@100 {
+				#address-cells = <1>;
+				#size-cells = <1>;
+
+				track@30 {
+					incline-up = < 48 32 16 >;
+				};
+
+				track@40 {
+					incline-up = < 47 31 15 >;
+				};
+			};
+
+			ride_200: ride@200 {
+				#address-cells = <1>;
+				#size-cells = <1>;
 				compatible = "ot,ferris-wheel";
 				reg = < 0x00000200 0x100 >;
 				hvac-provider = < &hvac_2 >;
@@ -36,6 +51,14 @@
 				spin-rph = < 30 >;
 				gondolas = < 16 >;
 				gondola-capacity = < 6 >;
+
+				ride_200_left: track@10 {
+					reg = < 0x00000010 0x10 >;
+				};
+
+				ride_200_right: track@20 {
+					reg = < 0x00000020 0x10 >;
+				};
 			};
 		};
 	};
@@ -44,7 +67,7 @@
 		target = <&lights_2>;
 
 		__overlay__ {
-			status = "ok";
+			status = "okay";
 			color = "purple", "white", "red", "green";
 			rate = < 3 256 >;
 		};
diff --git a/drivers/of/unittest-data/overlay_bad_symbol.dts b/drivers/of/unittest-data/overlay_bad_symbol.dts
new file mode 100644
index 000000000000..09261cb9a67e
--- /dev/null
+++ b/drivers/of/unittest-data/overlay_bad_symbol.dts
@@ -0,0 +1,22 @@
+/dts-v1/;
+/plugin/;
+
+/ {
+
+	fragment@0 {
+		target = <&electric_1>;
+
+		__overlay__ {
+
+			// This label should cause an error when the overlay
+			// is applied.  There is already a symbol hvac_1
+			// in the base tree
+			hvac_1: hvac-medium-2 {
+				compatible = "ot,hvac-medium";
+				heat-range = < 50 75 >;
+				cool-range = < 60 80 >;
+			};
+
+		};
+	};
+};
diff --git a/drivers/of/unittest-data/overlay_base.dts b/drivers/of/unittest-data/overlay_base.dts
index 5566b27fb61a..453d0bd83320 100644
--- a/drivers/of/unittest-data/overlay_base.dts
+++ b/drivers/of/unittest-data/overlay_base.dts
@@ -44,6 +44,8 @@
 			orientation = < 127 >;
 
 			ride@100 {
+				#address-cells = <1>;
+				#size-cells = <1>;
 				compatible = "ot,roller-coaster";
 				reg = < 0x00000100 0x100 >;
 				hvac-provider = < &hvac_1 >;
@@ -53,6 +55,15 @@
 				spin-controller = < &spin_ctrl_2 5 &spin_ctrl_2 7 >;
 				spin-controller-names = "track_1", "track_2";
 				queues = < 2 >;
+
+				track@30 {
+					reg = < 0x00000030 0x10 >;
+				};
+
+				track@40 {
+					reg = < 0x00000040 0x10 >;
+				};
+
 			};
 		};
 
diff --git a/drivers/of/unittest.c b/drivers/of/unittest.c
index 0107fc680335..29a35cb1da64 100644
--- a/drivers/of/unittest.c
+++ b/drivers/of/unittest.c
@@ -46,46 +46,54 @@ static struct unittest_results {
 static void __init of_unittest_find_node_by_name(void)
 {
 	struct device_node *np;
-	const char *options;
+	const char *options, *name;
 
 	np = of_find_node_by_path("/testcase-data");
-	unittest(np && !strcmp("/testcase-data", np->full_name),
+	name = kasprintf(GFP_KERNEL, "%pOF", np);
+	unittest(np && !strcmp("/testcase-data", name),
 		"find /testcase-data failed\n");
 	of_node_put(np);
+	kfree(name);
 
 	/* Test if trailing '/' works */
 	np = of_find_node_by_path("/testcase-data/");
 	unittest(!np, "trailing '/' on /testcase-data/ should fail\n");
 
 	np = of_find_node_by_path("/testcase-data/phandle-tests/consumer-a");
-	unittest(np && !strcmp("/testcase-data/phandle-tests/consumer-a", np->full_name),
+	name = kasprintf(GFP_KERNEL, "%pOF", np);
+	unittest(np && !strcmp("/testcase-data/phandle-tests/consumer-a", name),
 		"find /testcase-data/phandle-tests/consumer-a failed\n");
 	of_node_put(np);
+	kfree(name);
 
 	np = of_find_node_by_path("testcase-alias");
-	unittest(np && !strcmp("/testcase-data", np->full_name),
+	name = kasprintf(GFP_KERNEL, "%pOF", np);
+	unittest(np && !strcmp("/testcase-data", name),
 		"find testcase-alias failed\n");
 	of_node_put(np);
+	kfree(name);
 
 	/* Test if trailing '/' works on aliases */
 	np = of_find_node_by_path("testcase-alias/");
 	unittest(!np, "trailing '/' on testcase-alias/ should fail\n");
 
 	np = of_find_node_by_path("testcase-alias/phandle-tests/consumer-a");
-	unittest(np && !strcmp("/testcase-data/phandle-tests/consumer-a", np->full_name),
+	name = kasprintf(GFP_KERNEL, "%pOF", np);
+	unittest(np && !strcmp("/testcase-data/phandle-tests/consumer-a", name),
 		"find testcase-alias/phandle-tests/consumer-a failed\n");
 	of_node_put(np);
+	kfree(name);
 
 	np = of_find_node_by_path("/testcase-data/missing-path");
-	unittest(!np, "non-existent path returned node %s\n", np->full_name);
+	unittest(!np, "non-existent path returned node %pOF\n", np);
 	of_node_put(np);
 
 	np = of_find_node_by_path("missing-alias");
-	unittest(!np, "non-existent alias returned node %s\n", np->full_name);
+	unittest(!np, "non-existent alias returned node %pOF\n", np);
 	of_node_put(np);
 
 	np = of_find_node_by_path("testcase-alias/missing-path");
-	unittest(!np, "non-existent alias with relative path returned node %s\n", np->full_name);
+	unittest(!np, "non-existent alias with relative path returned node %pOF\n", np);
 	of_node_put(np);
 
 	np = of_find_node_opts_by_path("/testcase-data:testoption", &options);
@@ -315,8 +323,8 @@ static void __init of_unittest_check_phandles(void)
 
 		hash_for_each_possible(phandle_ht, nh, node, np->phandle) {
 			if (nh->np->phandle == np->phandle) {
-				pr_info("Duplicate phandle! %i used by %s and %s\n",
-					np->phandle, nh->np->full_name, np->full_name);
+				pr_info("Duplicate phandle! %i used by %pOF and %pOF\n",
+					np->phandle, nh->np, np);
 				dup_count++;
 				break;
 			}
@@ -406,8 +414,8 @@ static void __init of_unittest_parse_phandle_with_args(void)
 			passed = false;
 		}
 
-		unittest(passed, "index %i - data error on node %s rc=%i\n",
-			 i, args.np->full_name, rc);
+		unittest(passed, "index %i - data error on node %pOF rc=%i\n",
+			 i, args.np, rc);
 	}
 
 	/* Check for missing list property */
@@ -590,7 +598,7 @@ static void __init of_unittest_changeset(void)
 
 	/* Make sure node names are constructed correctly */
 	unittest((np = of_find_node_by_path("/testcase-data/changeset/n2/n21")),
-		 "'%s' not added\n", n21->full_name);
+		 "'%pOF' not added\n", n21);
 	of_node_put(np);
 
 	unittest(!of_changeset_revert(&chgset), "revert failed\n");
@@ -621,8 +629,8 @@ static void __init of_unittest_parse_interrupts(void)
 		passed &= (args.args_count == 1);
 		passed &= (args.args[0] == (i + 1));
 
-		unittest(passed, "index %i - data error on node %s rc=%i\n",
-			 i, args.np->full_name, rc);
+		unittest(passed, "index %i - data error on node %pOF rc=%i\n",
+			 i, args.np, rc);
 	}
 	of_node_put(np);
 
@@ -667,8 +675,8 @@ static void __init of_unittest_parse_interrupts(void)
 		default:
 			passed = false;
 		}
-		unittest(passed, "index %i - data error on node %s rc=%i\n",
-			 i, args.np->full_name, rc);
+		unittest(passed, "index %i - data error on node %pOF rc=%i\n",
+			 i, args.np, rc);
 	}
 	of_node_put(np);
 }
@@ -737,8 +745,8 @@ static void __init of_unittest_parse_interrupts_extended(void)
 			passed = false;
 		}
 
-		unittest(passed, "index %i - data error on node %s rc=%i\n",
-			 i, args.np->full_name, rc);
+		unittest(passed, "index %i - data error on node %pOF rc=%i\n",
+			 i, args.np, rc);
 	}
 	of_node_put(np);
 }
@@ -917,8 +925,11 @@ static int attach_node_and_children(struct device_node *np)
 {
 	struct device_node *next, *dup, *child;
 	unsigned long flags;
+	const char *full_name;
 
-	dup = of_find_node_by_path(np->full_name);
+	full_name = kasprintf(GFP_KERNEL, "%pOF", np);
+	dup = of_find_node_by_path(full_name);
+	kfree(full_name);
 	if (dup) {
 		update_node_properties(np, dup);
 		return 0;
@@ -1023,7 +1034,7 @@ static int unittest_probe(struct platform_device *pdev)
 
 	}
 
-	dev_dbg(dev, "%s for node @%s\n", __func__, np->full_name);
+	dev_dbg(dev, "%s for node @%pOF\n", __func__, np);
 
 	of_platform_populate(np, NULL, NULL, &pdev->dev);
 
@@ -1035,7 +1046,7 @@ static int unittest_remove(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct device_node *np = dev->of_node;
 
-	dev_dbg(dev, "%s for node @%s\n", __func__, np->full_name);
+	dev_dbg(dev, "%s for node @%pOF\n", __func__, np);
 	return 0;
 }
 
@@ -1649,7 +1660,7 @@ static int unittest_i2c_bus_probe(struct platform_device *pdev)
 
 	}
 
-	dev_dbg(dev, "%s for node @%s\n", __func__, np->full_name);
+	dev_dbg(dev, "%s for node @%pOF\n", __func__, np);
 
 	std = devm_kzalloc(dev, sizeof(*std), GFP_KERNEL);
 	if (!std) {
@@ -1687,7 +1698,7 @@ static int unittest_i2c_bus_remove(struct platform_device *pdev)
 	struct device_node *np = dev->of_node;
 	struct unittest_i2c_bus_data *std = platform_get_drvdata(pdev);
 
-	dev_dbg(dev, "%s for node @%s\n", __func__, np->full_name);
+	dev_dbg(dev, "%s for node @%pOF\n", __func__, np);
 	i2c_del_adapter(&std->adap);
 
 	return 0;
@@ -1718,7 +1729,7 @@ static int unittest_i2c_dev_probe(struct i2c_client *client,
 		return -EINVAL;
 	}
 
-	dev_dbg(dev, "%s for node @%s\n", __func__, np->full_name);
+	dev_dbg(dev, "%s for node @%pOF\n", __func__, np);
 
 	return 0;
 };
@@ -1728,7 +1739,7 @@ static int unittest_i2c_dev_remove(struct i2c_client *client)
 	struct device *dev = &client->dev;
 	struct device_node *np = client->dev.of_node;
 
-	dev_dbg(dev, "%s for node @%s\n", __func__, np->full_name);
+	dev_dbg(dev, "%s for node @%pOF\n", __func__, np);
 	return 0;
 }
 
@@ -1763,7 +1774,7 @@ static int unittest_i2c_mux_probe(struct i2c_client *client,
 	struct i2c_mux_core *muxc;
 	u32 reg, max_reg;
 
-	dev_dbg(dev, "%s for node @%s\n", __func__, np->full_name);
+	dev_dbg(dev, "%s for node @%pOF\n", __func__, np);
 
 	if (!np) {
 		dev_err(dev, "No OF node\n");
@@ -1808,7 +1819,7 @@ static int unittest_i2c_mux_remove(struct i2c_client *client)
 	struct device_node *np = client->dev.of_node;
 	struct i2c_mux_core *muxc = i2c_get_clientdata(client);
 
-	dev_dbg(dev, "%s for node @%s\n", __func__, np->full_name);
+	dev_dbg(dev, "%s for node @%pOF\n", __func__, np);
 	i2c_mux_del_adapters(muxc);
 	return 0;
 }
@@ -1983,6 +1994,8 @@ out:
 static inline void __init of_unittest_overlay(void) { }
 #endif
 
+#ifdef CONFIG_OF_OVERLAY
+
 /*
  * __dtb_ot_begin[] and __dtb_ot_end[] are created by cmd_dt_S_dtb
  * in scripts/Makefile.lib
@@ -2010,14 +2023,14 @@ struct overlay_info {
 OVERLAY_INFO_EXTERN(overlay_base);
 OVERLAY_INFO_EXTERN(overlay);
 OVERLAY_INFO_EXTERN(overlay_bad_phandle);
-
-#ifdef CONFIG_OF_OVERLAY
+OVERLAY_INFO_EXTERN(overlay_bad_symbol);
 
 /* order of entries is hard-coded into users of overlays[] */
 static struct overlay_info overlays[] = {
 	OVERLAY_INFO(overlay_base, -9999),
 	OVERLAY_INFO(overlay, 0),
 	OVERLAY_INFO(overlay_bad_phandle, -EINVAL),
+	OVERLAY_INFO(overlay_bad_symbol, -EINVAL),
 	{}
 };
 
@@ -2289,6 +2302,10 @@ static __init void of_unittest_overlay_high_level(void)
 
 	unittest(overlay_data_add(2),
 		 "Adding overlay 'overlay_bad_phandle' failed\n");
+
+	unittest(overlay_data_add(3),
+		 "Adding overlay 'overlay_bad_symbol' failed\n");
+
 	return;
 
 err_unlock:
diff --git a/drivers/parisc/asp.c b/drivers/parisc/asp.c
index 6a1ab2512a53..3163b6752d3d 100644
--- a/drivers/parisc/asp.c
+++ b/drivers/parisc/asp.c
@@ -118,12 +118,12 @@ static int __init asp_init_chip(struct parisc_device *dev)
 	return ret;
 }
 
-static struct parisc_device_id asp_tbl[] = {
+static const struct parisc_device_id asp_tbl[] __initconst = {
 	{ HPHW_BA, HVERSION_REV_ANY_ID, HVERSION_ANY_ID, 0x00070 },
 	{ 0, }
 };
 
-struct parisc_driver asp_driver = {
+struct parisc_driver asp_driver __refdata = {
 	.name =		"asp",
 	.id_table =	asp_tbl,
 	.probe =	asp_init_chip,
diff --git a/drivers/parisc/ccio-dma.c b/drivers/parisc/ccio-dma.c
index 6aa1e7f6672f..acba1f56af3e 100644
--- a/drivers/parisc/ccio-dma.c
+++ b/drivers/parisc/ccio-dma.c
@@ -1241,7 +1241,7 @@ ccio_get_iotlb_size(struct parisc_device *dev)
 #endif /* 0 */
 
 /* We *can't* support JAVA (T600). Venture there at your own risk. */
-static const struct parisc_device_id ccio_tbl[] = {
+static const struct parisc_device_id ccio_tbl[] __initconst = {
 	{ HPHW_IOA, HVERSION_REV_ANY_ID, U2_IOA_RUNWAY, 0xb }, /* U2 */
 	{ HPHW_IOA, HVERSION_REV_ANY_ID, UTURN_IOA_RUNWAY, 0xb }, /* UTurn */
 	{ 0, }
@@ -1249,7 +1249,7 @@ static const struct parisc_device_id ccio_tbl[] = {
 
 static int ccio_probe(struct parisc_device *dev);
 
-static struct parisc_driver ccio_driver = {
+static struct parisc_driver ccio_driver __refdata = {
 	.name =		"ccio",
 	.id_table =	ccio_tbl,
 	.probe =	ccio_probe,
diff --git a/drivers/parisc/ccio-rm-dma.c b/drivers/parisc/ccio-rm-dma.c
index 1bf988010855..df7932af48b7 100644
--- a/drivers/parisc/ccio-rm-dma.c
+++ b/drivers/parisc/ccio-rm-dma.c
@@ -163,7 +163,7 @@ static struct pci_dma_ops ccio_ops = {
 ** If so, initialize the chip and tell other partners in crime they
 ** have work to do.
 */
-static int
+static int __init
 ccio_probe(struct parisc_device *dev)
 {
 	printk(KERN_INFO "%s found %s at 0x%lx\n", MODULE_NAME,
@@ -184,13 +184,13 @@ ccio_probe(struct parisc_device *dev)
 	return 0;
 }
 
-static struct parisc_device_id ccio_tbl[] = {
+static const struct parisc_device_id ccio_tbl[] __initconst = {
 	{ HPHW_BCPORT, HVERSION_REV_ANY_ID, U2_BC_GSC, 0xc },
 	{ HPHW_BCPORT, HVERSION_REV_ANY_ID, UTURN_BC_GSC, 0xc },
 	{ 0, }
 };
 
-static struct parisc_driver ccio_driver = {
+static struct parisc_driver ccio_driver __refdata = {
 	.name =		"U2/Uturn",
 	.id_table =	ccio_tbl,
 	.probe =	ccio_probe,
diff --git a/drivers/parisc/dino.c b/drivers/parisc/dino.c
index 5c63b920b471..0b3fb99d9b89 100644
--- a/drivers/parisc/dino.c
+++ b/drivers/parisc/dino.c
@@ -956,7 +956,7 @@ static int __init dino_probe(struct parisc_device *dev)
 
 	dino_dev->hba.dev = dev;
 	dino_dev->hba.base_addr = ioremap_nocache(hpa, 4096);
-	dino_dev->hba.lmmio_space_offset = 0;	/* CPU addrs == bus addrs */
+	dino_dev->hba.lmmio_space_offset = PCI_F_EXTEND;
 	spin_lock_init(&dino_dev->dinosaur_pen);
 	dino_dev->hba.iommu = ccio_get_iommu(dev);
 
@@ -1022,7 +1022,7 @@ static int __init dino_probe(struct parisc_device *dev)
  * and 725 firmware misreport it as 0x08080 for no adequately explained
  * reason.
  */
-static struct parisc_device_id dino_tbl[] = {
+static const struct parisc_device_id dino_tbl[] __initconst = {
 	{ HPHW_A_DMA, HVERSION_REV_ANY_ID, 0x004, 0x0009D },/* Card-mode Dino */
 	{ HPHW_A_DMA, HVERSION_REV_ANY_ID, HVERSION_ANY_ID, 0x08080 }, /* XXX */
 	{ HPHW_BRIDGE, HVERSION_REV_ANY_ID, 0x680, 0xa }, /* Bridge-mode Dino */
@@ -1031,7 +1031,7 @@ static struct parisc_device_id dino_tbl[] = {
 	{ 0, }
 };
 
-static struct parisc_driver dino_driver = {
+static struct parisc_driver dino_driver __refdata = {
 	.name =		"dino",
 	.id_table =	dino_tbl,
 	.probe =	dino_probe,
diff --git a/drivers/parisc/eisa.c b/drivers/parisc/eisa.c
index 7e2f6d5a6aaf..9ff434f354bd 100644
--- a/drivers/parisc/eisa.c
+++ b/drivers/parisc/eisa.c
@@ -393,7 +393,7 @@ error_release:
 	return result;
 }
 
-static const struct parisc_device_id eisa_tbl[] = {
+static const struct parisc_device_id eisa_tbl[] __initconst = {
 	{ HPHW_BA, HVERSION_REV_ANY_ID, HVERSION_ANY_ID, 0x00076 }, /* Mongoose */
 	{ HPHW_BA, HVERSION_REV_ANY_ID, HVERSION_ANY_ID, 0x00090 }, /* Wax EISA */
 	{ 0, }
@@ -401,7 +401,7 @@ static const struct parisc_device_id eisa_tbl[] = {
 
 MODULE_DEVICE_TABLE(parisc, eisa_tbl);
 
-static struct parisc_driver eisa_driver = {
+static struct parisc_driver eisa_driver __refdata = {
 	.name =		"eisa_ba",
 	.id_table =	eisa_tbl,
 	.probe =	eisa_probe,
diff --git a/drivers/parisc/hppb.c b/drivers/parisc/hppb.c
index 898208e4f302..ebc7b617e5d0 100644
--- a/drivers/parisc/hppb.c
+++ b/drivers/parisc/hppb.c
@@ -45,7 +45,7 @@ static struct hppb_card hppb_card_head = {
  * (return 1). If so, initialize the chip and tell other partners in crime 
  * they have work to do.
  */
-static int hppb_probe(struct parisc_device *dev)
+static int __init hppb_probe(struct parisc_device *dev)
 {
 	int status;
 	struct hppb_card *card = &hppb_card_head;
@@ -81,7 +81,7 @@ static int hppb_probe(struct parisc_device *dev)
         return 0;
 }
 
-static struct parisc_device_id hppb_tbl[] = {
+static const struct parisc_device_id hppb_tbl[] __initconst = {
         { HPHW_BCPORT, HVERSION_REV_ANY_ID, 0x500, 0xc }, /* E25 and K */
         { HPHW_BCPORT, 0x0, 0x501, 0xc }, /* E35 */
         { HPHW_BCPORT, 0x0, 0x502, 0xc }, /* E45 */
@@ -89,7 +89,7 @@ static struct parisc_device_id hppb_tbl[] = {
         { 0, }
 };
 
-static struct parisc_driver hppb_driver = {
+static struct parisc_driver hppb_driver __refdata = {
         .name =         "gecko_boa",
         .id_table =     hppb_tbl,
 	.probe =        hppb_probe,
diff --git a/drivers/parisc/lasi.c b/drivers/parisc/lasi.c
index e65727ca9fc0..4c9225431500 100644
--- a/drivers/parisc/lasi.c
+++ b/drivers/parisc/lasi.c
@@ -227,12 +227,12 @@ static int __init lasi_init_chip(struct parisc_device *dev)
 	return ret;
 }
 
-static struct parisc_device_id lasi_tbl[] = {
+static struct parisc_device_id lasi_tbl[] __initdata = {
 	{ HPHW_BA, HVERSION_REV_ANY_ID, HVERSION_ANY_ID, 0x00081 },
 	{ 0, }
 };
 
-struct parisc_driver lasi_driver = {
+struct parisc_driver lasi_driver __refdata = {
 	.name =		"lasi",
 	.id_table =	lasi_tbl,
 	.probe =	lasi_init_chip,
diff --git a/drivers/parisc/lba_pci.c b/drivers/parisc/lba_pci.c
index bc286cbbbc9b..a25fed52f7e9 100644
--- a/drivers/parisc/lba_pci.c
+++ b/drivers/parisc/lba_pci.c
@@ -667,6 +667,42 @@ extend_lmmio_len(unsigned long start, unsigned long end, unsigned long lba_len)
 #define truncate_pat_collision(r,n)  (0)
 #endif
 
+static void pcibios_allocate_bridge_resources(struct pci_dev *dev)
+{
+	int idx;
+	struct resource *r;
+
+	for (idx = PCI_BRIDGE_RESOURCES; idx < PCI_NUM_RESOURCES; idx++) {
+		r = &dev->resource[idx];
+		if (!r->flags)
+			continue;
+		if (r->parent)	/* Already allocated */
+			continue;
+		if (!r->start || pci_claim_bridge_resource(dev, idx) < 0) {
+			/*
+			 * Something is wrong with the region.
+			 * Invalidate the resource to prevent
+			 * child resource allocations in this
+			 * range.
+			 */
+			r->start = r->end = 0;
+			r->flags = 0;
+		}
+	}
+}
+
+static void pcibios_allocate_bus_resources(struct pci_bus *bus)
+{
+	struct pci_bus *child;
+
+	/* Depth-First Search on bus tree */
+	if (bus->self)
+		pcibios_allocate_bridge_resources(bus->self);
+	list_for_each_entry(child, &bus->children, node)
+		pcibios_allocate_bus_resources(child);
+}
+
+
 /*
 ** The algorithm is generic code.
 ** But it needs to access local data structures to get the IRQ base.
@@ -693,11 +729,11 @@ lba_fixup_bus(struct pci_bus *bus)
 	** pci_alloc_primary_bus() mangles this.
 	*/
 	if (bus->parent) {
-		int i;
 		/* PCI-PCI Bridge */
 		pci_read_bridge_bases(bus);
-		for (i = PCI_BRIDGE_RESOURCES; i < PCI_NUM_RESOURCES; i++)
-			pci_claim_bridge_resource(bus->self, i);
+
+		/* check and allocate bridge resources */
+		pcibios_allocate_bus_resources(bus);
 	} else {
 		/* Host-PCI Bridge */
 		int err;
@@ -1613,14 +1649,14 @@ lba_driver_probe(struct parisc_device *dev)
 	return 0;
 }
 
-static struct parisc_device_id lba_tbl[] = {
+static const struct parisc_device_id lba_tbl[] __initconst = {
 	{ HPHW_BRIDGE, HVERSION_REV_ANY_ID, ELROY_HVERS, 0xa },
 	{ HPHW_BRIDGE, HVERSION_REV_ANY_ID, MERCURY_HVERS, 0xa },
 	{ HPHW_BRIDGE, HVERSION_REV_ANY_ID, QUICKSILVER_HVERS, 0xa },
 	{ 0, }
 };
 
-static struct parisc_driver lba_driver = {
+static struct parisc_driver lba_driver __refdata = {
 	.name =		MODULE_NAME,
 	.id_table =	lba_tbl,
 	.probe =	lba_driver_probe,
diff --git a/drivers/parisc/sba_iommu.c b/drivers/parisc/sba_iommu.c
index 4086f79d58d5..0a9c762a70fa 100644
--- a/drivers/parisc/sba_iommu.c
+++ b/drivers/parisc/sba_iommu.c
@@ -1905,7 +1905,7 @@ static const struct file_operations sba_proc_bitmap_fops = {
 };
 #endif /* CONFIG_PROC_FS */
 
-static struct parisc_device_id sba_tbl[] = {
+static const struct parisc_device_id sba_tbl[] __initconst = {
 	{ HPHW_IOA, HVERSION_REV_ANY_ID, ASTRO_RUNWAY_PORT, 0xb },
 	{ HPHW_BCPORT, HVERSION_REV_ANY_ID, IKE_MERCED_PORT, 0xc },
 	{ HPHW_BCPORT, HVERSION_REV_ANY_ID, REO_MERCED_PORT, 0xc },
@@ -1916,7 +1916,7 @@ static struct parisc_device_id sba_tbl[] = {
 
 static int sba_driver_callback(struct parisc_device *);
 
-static struct parisc_driver sba_driver = {
+static struct parisc_driver sba_driver __refdata = {
 	.name =		MODULE_NAME,
 	.id_table =	sba_tbl,
 	.probe =	sba_driver_callback,
@@ -1927,7 +1927,7 @@ static struct parisc_driver sba_driver = {
 ** If so, initialize the chip and tell other partners in crime they
 ** have work to do.
 */
-static int sba_driver_callback(struct parisc_device *dev)
+static int __init sba_driver_callback(struct parisc_device *dev)
 {
 	struct sba_device *sba_dev;
 	u32 func_class;
diff --git a/drivers/parisc/superio.c b/drivers/parisc/superio.c
index deeaed544222..0441777fc777 100644
--- a/drivers/parisc/superio.c
+++ b/drivers/parisc/superio.c
@@ -482,14 +482,14 @@ superio_probe(struct pci_dev *dev, const struct pci_device_id *id)
 	return -ENODEV;
 }
 
-static const struct pci_device_id superio_tbl[] = {
+static const struct pci_device_id superio_tbl[] __initconst = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_NS, PCI_DEVICE_ID_NS_87560_LIO) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_NS, PCI_DEVICE_ID_NS_87560_USB) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_NS, PCI_DEVICE_ID_NS_87415) },
 	{ 0, }
 };
 
-static struct pci_driver superio_driver = {
+static struct pci_driver superio_driver __refdata = {
 	.name =         SUPERIO,
 	.id_table =     superio_tbl,
 	.probe =        superio_probe,
diff --git a/drivers/parisc/wax.c b/drivers/parisc/wax.c
index da9d5ad1353c..6a3e40702b3b 100644
--- a/drivers/parisc/wax.c
+++ b/drivers/parisc/wax.c
@@ -125,14 +125,14 @@ static int __init wax_init_chip(struct parisc_device *dev)
 	return ret;
 }
 
-static struct parisc_device_id wax_tbl[] = {
+static const struct parisc_device_id wax_tbl[] __initconst = {
   	{ HPHW_BA, HVERSION_REV_ANY_ID, HVERSION_ANY_ID, 0x0008e },
 	{ 0, }
 };
 
 MODULE_DEVICE_TABLE(parisc, wax_tbl);
 
-struct parisc_driver wax_driver = {
+struct parisc_driver wax_driver __refdata = {
 	.name =		"wax",
 	.id_table =	wax_tbl,
 	.probe =	wax_init_chip,
diff --git a/drivers/parport/daisy.c b/drivers/parport/daisy.c
index 46eb15fb57ff..5484a46dafda 100644
--- a/drivers/parport/daisy.c
+++ b/drivers/parport/daisy.c
@@ -44,7 +44,7 @@ static struct daisydev {
 } *topology = NULL;
 static DEFINE_SPINLOCK(topology_lock);
 
-static int numdevs = 0;
+static int numdevs;
 
 /* Forward-declaration of lower-level functions. */
 static int mux_present(struct parport *port);
diff --git a/drivers/parport/parport_atari.c b/drivers/parport/parport_atari.c
index a81cd2a2747f..9fbf6ccd54de 100644
--- a/drivers/parport/parport_atari.c
+++ b/drivers/parport/parport_atari.c
@@ -18,7 +18,7 @@
 #include <asm/irq.h>
 #include <asm/atariints.h>
 
-static struct parport *this_port = NULL;
+static struct parport *this_port;
 
 static unsigned char
 parport_atari_read_data(struct parport *p)
diff --git a/drivers/parport/parport_ax88796.c b/drivers/parport/parport_ax88796.c
index 8f8c9f3aa691..2fc91edb058d 100644
--- a/drivers/parport/parport_ax88796.c
+++ b/drivers/parport/parport_ax88796.c
@@ -358,8 +358,7 @@ static int parport_ax88796_probe(struct platform_device *pdev)
  exit_unmap:
 	iounmap(dd->base);
  exit_res:
-	release_resource(dd->io);
-	kfree(dd->io);
+	release_mem_region(dd->io->start, size);
  exit_mem:
 	kfree(dd);
 	return ret;
@@ -373,8 +372,7 @@ static int parport_ax88796_remove(struct platform_device *pdev)
 	free_irq(p->irq, p);
 	parport_remove_port(p);
 	iounmap(dd->base);
-	release_resource(dd->io);
-	kfree(dd->io);
+	release_mem_region(dd->io->start, resource_size(dd->io));
 	kfree(dd);
 
 	return 0;
diff --git a/drivers/parport/parport_gsc.c b/drivers/parport/parport_gsc.c
index 3858b87fd0bb..5f710aaaf3da 100644
--- a/drivers/parport/parport_gsc.c
+++ b/drivers/parport/parport_gsc.c
@@ -346,7 +346,7 @@ struct parport *parport_gsc_probe_port(unsigned long base,
 
 static int parport_count;
 
-static int parport_init_chip(struct parisc_device *dev)
+static int __init parport_init_chip(struct parisc_device *dev)
 {
 	struct parport *p;
 	unsigned long port;
@@ -381,7 +381,7 @@ static int parport_init_chip(struct parisc_device *dev)
 	return 0;
 }
 
-static int parport_remove_chip(struct parisc_device *dev)
+static int __exit parport_remove_chip(struct parisc_device *dev)
 {
 	struct parport *p = dev_get_drvdata(&dev->dev);
 	if (p) {
@@ -403,18 +403,18 @@ static int parport_remove_chip(struct parisc_device *dev)
 	return 0;
 }
 
-static struct parisc_device_id parport_tbl[] = {
+static const struct parisc_device_id parport_tbl[] __initconst = {
 	{ HPHW_FIO, HVERSION_REV_ANY_ID, HVERSION_ANY_ID, 0x74 },
 	{ 0, }
 };
 
 MODULE_DEVICE_TABLE(parisc, parport_tbl);
 
-static struct parisc_driver parport_driver = {
+static struct parisc_driver parport_driver __refdata = {
 	.name		= "Parallel",
 	.id_table	= parport_tbl,
 	.probe		= parport_init_chip,
-	.remove		= parport_remove_chip,
+	.remove		= __exit_p(parport_remove_chip),
 };
 
 int parport_gsc_init(void)
diff --git a/drivers/parport/parport_ip32.c b/drivers/parport/parport_ip32.c
index dcbeeb220dda..0186db7680d4 100644
--- a/drivers/parport/parport_ip32.c
+++ b/drivers/parport/parport_ip32.c
@@ -138,7 +138,7 @@ static unsigned int features =	~0U;
 static bool verbose_probing =	DEFAULT_VERBOSE_PROBING;
 
 /* We do not support more than one port. */
-static struct parport *this_port = NULL;
+static struct parport *this_port;
 
 /* Timing constants for FIFO modes.  */
 #define FIFO_NFAULT_TIMEOUT	100	/* milliseconds */
diff --git a/drivers/parport/parport_mfc3.c b/drivers/parport/parport_mfc3.c
index 2f650f68af14..7f4be0e484c7 100644
--- a/drivers/parport/parport_mfc3.c
+++ b/drivers/parport/parport_mfc3.c
@@ -174,7 +174,7 @@ DPRINTK(KERN_DEBUG "read_status %02x\n", status);
 	return status;
 }
 
-static int use_cnt = 0;
+static int use_cnt;
 
 static irqreturn_t mfc3_interrupt(int irq, void *dev_id)
 {
diff --git a/drivers/parport/parport_pc.c b/drivers/parport/parport_pc.c
index 5548193a28a6..489492b608cf 100644
--- a/drivers/parport/parport_pc.c
+++ b/drivers/parport/parport_pc.c
@@ -1083,9 +1083,9 @@ static void show_parconfig_winbond(int io, int key)
 		printk(KERN_INFO "Winbond LPT Config: active=%s, io=0x%02x%02x irq=%d, ",
 		       (cr30 & 0x01) ? "yes" : "no", cr60, cr61, cr70 & 0x0f);
 		if ((cr74 & 0x07) > 3)
-			printk("dma=none\n");
+			pr_cont("dma=none\n");
 		else
-			printk("dma=%d\n", cr74 & 0x07);
+			pr_cont("dma=%d\n", cr74 & 0x07);
 		printk(KERN_INFO
 		    "Winbond LPT Config: irqtype=%s, ECP fifo threshold=%d\n",
 					irqtypes[crf0>>7], (crf0>>3)&0x0f);
@@ -1685,14 +1685,14 @@ static int parport_ECP_supported(struct parport *pb)
 			pb->base, config, configb);
 		printk(KERN_DEBUG "0x%lx: ECP settings irq=", pb->base);
 		if ((configb >> 3) & 0x07)
-			printk("%d", intrline[(configb >> 3) & 0x07]);
+			pr_cont("%d", intrline[(configb >> 3) & 0x07]);
 		else
-			printk("<none or set by other means>");
-		printk(" dma=");
+			pr_cont("<none or set by other means>");
+		pr_cont(" dma=");
 		if ((configb & 0x03) == 0x00)
-			printk("<none or set by other means>\n");
+			pr_cont("<none or set by other means>\n");
 		else
-			printk("%d\n", configb & 0x07);
+			pr_cont("%d\n", configb & 0x07);
 	}
 
 	/* Go back to mode 000 */
@@ -2399,8 +2399,8 @@ static int sio_ite_8872_probe(struct pci_dev *pdev, int autoirq, int autodma,
 			"parport_pc: ITE 8872 parallel port: io=0x%X",
 								ite8872_lpt);
 		if (irq != PARPORT_IRQ_NONE)
-			printk(", irq=%d", irq);
-		printk("\n");
+			pr_cont(", irq=%d", irq);
+		pr_cont("\n");
 		return 1;
 	}
 
@@ -2581,10 +2581,10 @@ static int sio_via_probe(struct pci_dev *pdev, int autoirq, int autodma,
 		printk(KERN_INFO
 			"parport_pc: VIA parallel port: io=0x%X", port1);
 		if (irq != PARPORT_IRQ_NONE)
-			printk(", irq=%d", irq);
+			pr_cont(", irq=%d", irq);
 		if (dma != PARPORT_DMA_NONE)
-			printk(", dma=%d", dma);
-		printk("\n");
+			pr_cont(", dma=%d", dma);
+		pr_cont("\n");
 		return 1;
 	}
 
diff --git a/drivers/pci/dwc/Kconfig b/drivers/pci/dwc/Kconfig
index d275aadc47ee..22ec82fcdea2 100644
--- a/drivers/pci/dwc/Kconfig
+++ b/drivers/pci/dwc/Kconfig
@@ -25,7 +25,7 @@ config PCI_DRA7XX
 	 work either as EP or RC. In order to enable host-specific features
 	 PCI_DRA7XX_HOST must be selected and in order to enable device-
 	 specific features PCI_DRA7XX_EP must be selected. This uses
-	 the Designware core.
+	 the DesignWare core.
 
 if PCI_DRA7XX
 
@@ -97,8 +97,8 @@ config PCI_KEYSTONE
 	select PCIE_DW_HOST
 	help
 	  Say Y here if you want to enable PCI controller support on Keystone
-	  SoCs. The PCI controller on Keystone is based on Designware hardware
-	  and therefore the driver re-uses the Designware core functions to
+	  SoCs. The PCI controller on Keystone is based on DesignWare hardware
+	  and therefore the driver re-uses the DesignWare core functions to
 	  implement the driver.
 
 config PCI_LAYERSCAPE
@@ -132,7 +132,7 @@ config PCIE_QCOM
 	select PCIE_DW_HOST
 	help
 	  Say Y here to enable PCIe controller support on Qualcomm SoCs. The
-	  PCIe controller uses the Designware core plus Qualcomm-specific
+	  PCIe controller uses the DesignWare core plus Qualcomm-specific
 	  hardware wrappers.
 
 config PCIE_ARMADA_8K
@@ -145,8 +145,8 @@ config PCIE_ARMADA_8K
 	help
 	  Say Y here if you want to enable PCIe controller support on
 	  Armada-8K SoCs. The PCIe controller on Armada-8K is based on
-	  Designware hardware and therefore the driver re-uses the
-	  Designware core functions to implement the driver.
+	  DesignWare hardware and therefore the driver re-uses the
+	  DesignWare core functions to implement the driver.
 
 config PCIE_ARTPEC6
 	bool "Axis ARTPEC-6 PCIe controller"
diff --git a/drivers/pci/dwc/pci-dra7xx.c b/drivers/pci/dwc/pci-dra7xx.c
index f2fc5f47064e..34427a6a15af 100644
--- a/drivers/pci/dwc/pci-dra7xx.c
+++ b/drivers/pci/dwc/pci-dra7xx.c
@@ -195,7 +195,7 @@ static void dra7xx_pcie_enable_interrupts(struct dra7xx_pcie *dra7xx)
 	dra7xx_pcie_enable_msi_interrupts(dra7xx);
 }
 
-static void dra7xx_pcie_host_init(struct pcie_port *pp)
+static int dra7xx_pcie_host_init(struct pcie_port *pp)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 	struct dra7xx_pcie *dra7xx = to_dra7xx_pcie(pci);
@@ -206,6 +206,8 @@ static void dra7xx_pcie_host_init(struct pcie_port *pp)
 	dw_pcie_wait_for_link(pci);
 	dw_pcie_msi_init(pp);
 	dra7xx_pcie_enable_interrupts(dra7xx);
+
+	return 0;
 }
 
 static const struct dw_pcie_host_ops dra7xx_pcie_host_ops = {
@@ -238,7 +240,7 @@ static int dra7xx_pcie_init_irq_domain(struct pcie_port *pp)
 		return -ENODEV;
 	}
 
-	dra7xx->irq_domain = irq_domain_add_linear(pcie_intc_node, 4,
+	dra7xx->irq_domain = irq_domain_add_linear(pcie_intc_node, PCI_NUM_INTX,
 						   &intx_domain_ops, pp);
 	if (!dra7xx->irq_domain) {
 		dev_err(dev, "Failed to get a INTx IRQ domain\n");
@@ -275,7 +277,6 @@ static irqreturn_t dra7xx_pcie_msi_irq_handler(int irq, void *arg)
 	return IRQ_HANDLED;
 }
 
-
 static irqreturn_t dra7xx_pcie_irq_handler(int irq, void *arg)
 {
 	struct dra7xx_pcie *dra7xx = arg;
@@ -335,10 +336,23 @@ static irqreturn_t dra7xx_pcie_irq_handler(int irq, void *arg)
 	return IRQ_HANDLED;
 }
 
+static void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar)
+{
+	u32 reg;
+
+	reg = PCI_BASE_ADDRESS_0 + (4 * bar);
+	dw_pcie_writel_dbi2(pci, reg, 0x0);
+	dw_pcie_writel_dbi(pci, reg, 0x0);
+}
+
 static void dra7xx_pcie_ep_init(struct dw_pcie_ep *ep)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 	struct dra7xx_pcie *dra7xx = to_dra7xx_pcie(pci);
+	enum pci_barno bar;
+
+	for (bar = BAR_0; bar <= BAR_5; bar++)
+		dw_pcie_ep_reset_bar(pci, bar);
 
 	dra7xx_pcie_enable_wrapper_interrupts(dra7xx);
 }
@@ -435,7 +449,7 @@ static int __init dra7xx_add_pcie_port(struct dra7xx_pcie *dra7xx,
 	pp->irq = platform_get_irq(pdev, 1);
 	if (pp->irq < 0) {
 		dev_err(dev, "missing IRQ resource\n");
-		return -EINVAL;
+		return pp->irq;
 	}
 
 	ret = devm_request_irq(dev, pp->irq, dra7xx_pcie_msi_irq_handler,
@@ -616,8 +630,8 @@ static int __init dra7xx_pcie_probe(struct platform_device *pdev)
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
-		dev_err(dev, "missing IRQ resource\n");
-		return -EINVAL;
+		dev_err(dev, "missing IRQ resource: %d\n", irq);
+		return irq;
 	}
 
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ti_conf");
diff --git a/drivers/pci/dwc/pci-exynos.c b/drivers/pci/dwc/pci-exynos.c
index c78c06552590..5596fdedbb94 100644
--- a/drivers/pci/dwc/pci-exynos.c
+++ b/drivers/pci/dwc/pci-exynos.c
@@ -581,13 +581,15 @@ static int exynos_pcie_link_up(struct dw_pcie *pci)
 	return 0;
 }
 
-static void exynos_pcie_host_init(struct pcie_port *pp)
+static int exynos_pcie_host_init(struct pcie_port *pp)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 	struct exynos_pcie *ep = to_exynos_pcie(pci);
 
 	exynos_pcie_establish_link(ep);
 	exynos_pcie_enable_interrupts(ep);
+
+	return 0;
 }
 
 static const struct dw_pcie_host_ops exynos_pcie_host_ops = {
@@ -605,9 +607,9 @@ static int __init exynos_add_pcie_port(struct exynos_pcie *ep,
 	int ret;
 
 	pp->irq = platform_get_irq(pdev, 1);
-	if (!pp->irq) {
+	if (pp->irq < 0) {
 		dev_err(dev, "failed to get irq\n");
-		return -ENODEV;
+		return pp->irq;
 	}
 	ret = devm_request_irq(dev, pp->irq, exynos_pcie_irq_handler,
 				IRQF_SHARED, "exynos-pcie", ep);
@@ -618,9 +620,9 @@ static int __init exynos_add_pcie_port(struct exynos_pcie *ep,
 
 	if (IS_ENABLED(CONFIG_PCI_MSI)) {
 		pp->msi_irq = platform_get_irq(pdev, 0);
-		if (!pp->msi_irq) {
+		if (pp->msi_irq < 0) {
 			dev_err(dev, "failed to get msi irq\n");
-			return -ENODEV;
+			return pp->msi_irq;
 		}
 
 		ret = devm_request_irq(dev, pp->msi_irq,
diff --git a/drivers/pci/dwc/pci-imx6.c b/drivers/pci/dwc/pci-imx6.c
index bf5c3616e344..b73483534a5b 100644
--- a/drivers/pci/dwc/pci-imx6.c
+++ b/drivers/pci/dwc/pci-imx6.c
@@ -636,7 +636,7 @@ err_reset_phy:
 	return ret;
 }
 
-static void imx6_pcie_host_init(struct pcie_port *pp)
+static int imx6_pcie_host_init(struct pcie_port *pp)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 	struct imx6_pcie *imx6_pcie = to_imx6_pcie(pci);
@@ -649,6 +649,8 @@ static void imx6_pcie_host_init(struct pcie_port *pp)
 
 	if (IS_ENABLED(CONFIG_PCI_MSI))
 		dw_pcie_msi_init(pp);
+
+	return 0;
 }
 
 static int imx6_pcie_link_up(struct dw_pcie *pci)
@@ -778,14 +780,15 @@ static int imx6_pcie_probe(struct platform_device *pdev)
 		}
 		break;
 	case IMX7D:
-		imx6_pcie->pciephy_reset = devm_reset_control_get(dev,
-								  "pciephy");
+		imx6_pcie->pciephy_reset = devm_reset_control_get_exclusive(dev,
+									    "pciephy");
 		if (IS_ERR(imx6_pcie->pciephy_reset)) {
 			dev_err(dev, "Failed to get PCIEPHY reset control\n");
 			return PTR_ERR(imx6_pcie->pciephy_reset);
 		}
 
-		imx6_pcie->apps_reset = devm_reset_control_get(dev, "apps");
+		imx6_pcie->apps_reset = devm_reset_control_get_exclusive(dev,
+									 "apps");
 		if (IS_ERR(imx6_pcie->apps_reset)) {
 			dev_err(dev, "Failed to get PCIE APPS reset control\n");
 			return PTR_ERR(imx6_pcie->apps_reset);
diff --git a/drivers/pci/dwc/pci-keystone-dw.c b/drivers/pci/dwc/pci-keystone-dw.c
index 8bc626e640c8..2fb20b887d2a 100644
--- a/drivers/pci/dwc/pci-keystone-dw.c
+++ b/drivers/pci/dwc/pci-keystone-dw.c
@@ -1,5 +1,5 @@
 /*
- * Designware application register space functions for Keystone PCI controller
+ * DesignWare application register space functions for Keystone PCI controller
  *
  * Copyright (C) 2013-2014 Texas Instruments., Ltd.
  *		http://www.ti.com
@@ -168,16 +168,12 @@ void ks_dw_pcie_msi_clear_irq(struct pcie_port *pp, int irq)
 
 static void ks_dw_pcie_msi_irq_mask(struct irq_data *d)
 {
-	struct keystone_pcie *ks_pcie;
 	struct msi_desc *msi;
 	struct pcie_port *pp;
-	struct dw_pcie *pci;
 	u32 offset;
 
 	msi = irq_data_get_msi_desc(d);
 	pp = (struct pcie_port *) msi_desc_to_pci_sysdata(msi);
-	pci = to_dw_pcie_from_pp(pp);
-	ks_pcie = to_keystone_pcie(pci);
 	offset = d->irq - irq_linear_revmap(pp->irq_domain, 0);
 
 	/* Mask the end point if PVM implemented */
@@ -191,16 +187,12 @@ static void ks_dw_pcie_msi_irq_mask(struct irq_data *d)
 
 static void ks_dw_pcie_msi_irq_unmask(struct irq_data *d)
 {
-	struct keystone_pcie *ks_pcie;
 	struct msi_desc *msi;
 	struct pcie_port *pp;
-	struct dw_pcie *pci;
 	u32 offset;
 
 	msi = irq_data_get_msi_desc(d);
 	pp = (struct pcie_port *) msi_desc_to_pci_sysdata(msi);
-	pci = to_dw_pcie_from_pp(pp);
-	ks_pcie = to_keystone_pcie(pci);
 	offset = d->irq - irq_linear_revmap(pp->irq_domain, 0);
 
 	/* Mask the end point if PVM implemented */
@@ -259,7 +251,7 @@ void ks_dw_pcie_enable_legacy_irqs(struct keystone_pcie *ks_pcie)
 {
 	int i;
 
-	for (i = 0; i < MAX_LEGACY_IRQS; i++)
+	for (i = 0; i < PCI_NUM_INTX; i++)
 		ks_dw_app_writel(ks_pcie, IRQ_ENABLE_SET + (i << 4), 0x1);
 }
 
@@ -565,7 +557,7 @@ int __init ks_dw_pcie_host_init(struct keystone_pcie *ks_pcie,
 	/* Create legacy IRQ domain */
 	ks_pcie->legacy_irq_domain =
 			irq_domain_add_linear(ks_pcie->legacy_intc_np,
-					MAX_LEGACY_IRQS,
+					PCI_NUM_INTX,
 					&ks_dw_pcie_legacy_irq_domain_ops,
 					NULL);
 	if (!ks_pcie->legacy_irq_domain) {
diff --git a/drivers/pci/dwc/pci-keystone.c b/drivers/pci/dwc/pci-keystone.c
index 4783cec1f78d..5bee3af47588 100644
--- a/drivers/pci/dwc/pci-keystone.c
+++ b/drivers/pci/dwc/pci-keystone.c
@@ -32,10 +32,6 @@
 
 #define DRIVER_NAME	"keystone-pcie"
 
-/* driver specific constants */
-#define MAX_MSI_HOST_IRQS		8
-#define MAX_LEGACY_HOST_IRQS		4
-
 /* DEV_STAT_CTRL */
 #define PCIE_CAP_BASE		0x70
 
@@ -173,7 +169,7 @@ static int ks_pcie_get_irq_controller_info(struct keystone_pcie *ks_pcie,
 
 	if (legacy) {
 		np_temp = &ks_pcie->legacy_intc_np;
-		max_host_irqs = MAX_LEGACY_HOST_IRQS;
+		max_host_irqs = PCI_NUM_INTX;
 		host_irqs = &ks_pcie->legacy_host_irqs[0];
 	} else {
 		np_temp = &ks_pcie->msi_intc_np;
@@ -261,7 +257,7 @@ static int keystone_pcie_fault(unsigned long addr, unsigned int fsr,
 	return 0;
 }
 
-static void __init ks_pcie_host_init(struct pcie_port *pp)
+static int __init ks_pcie_host_init(struct pcie_port *pp)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 	struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
@@ -289,6 +285,8 @@ static void __init ks_pcie_host_init(struct pcie_port *pp)
 	 */
 	hook_fault_code(17, keystone_pcie_fault, SIGBUS, 0,
 			"Asynchronous external abort");
+
+	return 0;
 }
 
 static const struct dw_pcie_host_ops keystone_pcie_host_ops = {
diff --git a/drivers/pci/dwc/pci-keystone.h b/drivers/pci/dwc/pci-keystone.h
index 74c5825882df..30b7bc2ac380 100644
--- a/drivers/pci/dwc/pci-keystone.h
+++ b/drivers/pci/dwc/pci-keystone.h
@@ -12,9 +12,7 @@
  * published by the Free Software Foundation.
  */
 
-#define MAX_LEGACY_IRQS			4
 #define MAX_MSI_HOST_IRQS		8
-#define MAX_LEGACY_HOST_IRQS		4
 
 struct keystone_pcie {
 	struct dw_pcie		*pci;
@@ -22,7 +20,7 @@ struct keystone_pcie {
 	/* PCI Device ID */
 	u32			device_id;
 	int			num_legacy_host_irqs;
-	int			legacy_host_irqs[MAX_LEGACY_HOST_IRQS];
+	int			legacy_host_irqs[PCI_NUM_INTX];
 	struct			device_node *legacy_intc_np;
 
 	int			num_msi_host_irqs;
diff --git a/drivers/pci/dwc/pci-layerscape.c b/drivers/pci/dwc/pci-layerscape.c
index fd861289ad8b..87fa486bee2c 100644
--- a/drivers/pci/dwc/pci-layerscape.c
+++ b/drivers/pci/dwc/pci-layerscape.c
@@ -33,7 +33,8 @@
 
 /* PEX Internal Configuration Registers */
 #define PCIE_STRFMR1		0x71c /* Symbol Timer & Filter Mask Register1 */
-#define PCIE_DBI_RO_WR_EN	0x8bc /* DBI Read-Only Write Enable Register */
+
+#define PCIE_IATU_NUM		6
 
 struct ls_pcie_drvdata {
 	u32 lut_offset;
@@ -72,14 +73,6 @@ static void ls_pcie_clear_multifunction(struct ls_pcie *pcie)
 	iowrite8(PCI_HEADER_TYPE_BRIDGE, pci->dbi_base + PCI_HEADER_TYPE);
 }
 
-/* Fix class value */
-static void ls_pcie_fix_class(struct ls_pcie *pcie)
-{
-	struct dw_pcie *pci = pcie->pci;
-
-	iowrite16(PCI_CLASS_BRIDGE_PCI, pci->dbi_base + PCI_CLASS_DEVICE);
-}
-
 /* Drop MSG TLP except for Vendor MSG */
 static void ls_pcie_drop_msg_tlp(struct ls_pcie *pcie)
 {
@@ -91,6 +84,14 @@ static void ls_pcie_drop_msg_tlp(struct ls_pcie *pcie)
 	iowrite32(val, pci->dbi_base + PCIE_STRFMR1);
 }
 
+static void ls_pcie_disable_outbound_atus(struct ls_pcie *pcie)
+{
+	int i;
+
+	for (i = 0; i < PCIE_IATU_NUM; i++)
+		dw_pcie_disable_atu(pcie->pci, DW_PCIE_REGION_OUTBOUND, i);
+}
+
 static int ls1021_pcie_link_up(struct dw_pcie *pci)
 {
 	u32 state;
@@ -108,33 +109,6 @@ static int ls1021_pcie_link_up(struct dw_pcie *pci)
 	return 1;
 }
 
-static void ls1021_pcie_host_init(struct pcie_port *pp)
-{
-	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
-	struct ls_pcie *pcie = to_ls_pcie(pci);
-	struct device *dev = pci->dev;
-	u32 index[2];
-
-	pcie->scfg = syscon_regmap_lookup_by_phandle(dev->of_node,
-						     "fsl,pcie-scfg");
-	if (IS_ERR(pcie->scfg)) {
-		dev_err(dev, "No syscfg phandle specified\n");
-		pcie->scfg = NULL;
-		return;
-	}
-
-	if (of_property_read_u32_array(dev->of_node,
-				       "fsl,pcie-scfg", index, 2)) {
-		pcie->scfg = NULL;
-		return;
-	}
-	pcie->index = index[1];
-
-	dw_pcie_setup_rc(pp);
-
-	ls_pcie_drop_msg_tlp(pcie);
-}
-
 static int ls_pcie_link_up(struct dw_pcie *pci)
 {
 	struct ls_pcie *pcie = to_ls_pcie(pci);
@@ -150,16 +124,54 @@ static int ls_pcie_link_up(struct dw_pcie *pci)
 	return 1;
 }
 
-static void ls_pcie_host_init(struct pcie_port *pp)
+static int ls_pcie_host_init(struct pcie_port *pp)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 	struct ls_pcie *pcie = to_ls_pcie(pci);
 
-	iowrite32(1, pci->dbi_base + PCIE_DBI_RO_WR_EN);
-	ls_pcie_fix_class(pcie);
+	/*
+	 * Disable outbound windows configured by the bootloader to avoid
+	 * one transaction hitting multiple outbound windows.
+	 * dw_pcie_setup_rc() will reconfigure the outbound windows.
+	 */
+	ls_pcie_disable_outbound_atus(pcie);
+
+	dw_pcie_dbi_ro_wr_en(pci);
 	ls_pcie_clear_multifunction(pcie);
+	dw_pcie_dbi_ro_wr_dis(pci);
+
 	ls_pcie_drop_msg_tlp(pcie);
-	iowrite32(0, pci->dbi_base + PCIE_DBI_RO_WR_EN);
+
+	dw_pcie_setup_rc(pp);
+
+	return 0;
+}
+
+static int ls1021_pcie_host_init(struct pcie_port *pp)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+	struct ls_pcie *pcie = to_ls_pcie(pci);
+	struct device *dev = pci->dev;
+	u32 index[2];
+	int ret;
+
+	pcie->scfg = syscon_regmap_lookup_by_phandle(dev->of_node,
+						     "fsl,pcie-scfg");
+	if (IS_ERR(pcie->scfg)) {
+		ret = PTR_ERR(pcie->scfg);
+		dev_err(dev, "No syscfg phandle specified\n");
+		pcie->scfg = NULL;
+		return ret;
+	}
+
+	if (of_property_read_u32_array(dev->of_node,
+				       "fsl,pcie-scfg", index, 2)) {
+		pcie->scfg = NULL;
+		return -EINVAL;
+	}
+	pcie->index = index[1];
+
+	return ls_pcie_host_init(pp);
 }
 
 static int ls_pcie_msi_host_init(struct pcie_port *pp,
@@ -232,12 +244,22 @@ static struct ls_pcie_drvdata ls2080_drvdata = {
 	.dw_pcie_ops = &dw_ls_pcie_ops,
 };
 
+static struct ls_pcie_drvdata ls2088_drvdata = {
+	.lut_offset = 0x80000,
+	.ltssm_shift = 0,
+	.lut_dbg = 0x407fc,
+	.ops = &ls_pcie_host_ops,
+	.dw_pcie_ops = &dw_ls_pcie_ops,
+};
+
 static const struct of_device_id ls_pcie_of_match[] = {
 	{ .compatible = "fsl,ls1021a-pcie", .data = &ls1021_drvdata },
 	{ .compatible = "fsl,ls1043a-pcie", .data = &ls1043_drvdata },
 	{ .compatible = "fsl,ls1046a-pcie", .data = &ls1046_drvdata },
 	{ .compatible = "fsl,ls2080a-pcie", .data = &ls2080_drvdata },
 	{ .compatible = "fsl,ls2085a-pcie", .data = &ls2080_drvdata },
+	{ .compatible = "fsl,ls2088a-pcie", .data = &ls2088_drvdata },
+	{ .compatible = "fsl,ls1088a-pcie", .data = &ls2088_drvdata },
 	{ },
 };
 
diff --git a/drivers/pci/dwc/pcie-armada8k.c b/drivers/pci/dwc/pcie-armada8k.c
index ea8f34af6a85..370d057c0046 100644
--- a/drivers/pci/dwc/pcie-armada8k.c
+++ b/drivers/pci/dwc/pcie-armada8k.c
@@ -134,13 +134,15 @@ static void armada8k_pcie_establish_link(struct armada8k_pcie *pcie)
 		dev_err(pci->dev, "Link not up after reconfiguration\n");
 }
 
-static void armada8k_pcie_host_init(struct pcie_port *pp)
+static int armada8k_pcie_host_init(struct pcie_port *pp)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 	struct armada8k_pcie *pcie = to_armada8k_pcie(pci);
 
 	dw_pcie_setup_rc(pp);
 	armada8k_pcie_establish_link(pcie);
+
+	return 0;
 }
 
 static irqreturn_t armada8k_pcie_irq_handler(int irq, void *arg)
@@ -176,9 +178,9 @@ static int armada8k_add_pcie_port(struct armada8k_pcie *pcie,
 	pp->ops = &armada8k_pcie_host_ops;
 
 	pp->irq = platform_get_irq(pdev, 0);
-	if (!pp->irq) {
+	if (pp->irq < 0) {
 		dev_err(dev, "failed to get irq for port\n");
-		return -ENODEV;
+		return pp->irq;
 	}
 
 	ret = devm_request_irq(dev, pp->irq, armada8k_pcie_irq_handler,
@@ -226,7 +228,9 @@ static int armada8k_pcie_probe(struct platform_device *pdev)
 	if (IS_ERR(pcie->clk))
 		return PTR_ERR(pcie->clk);
 
-	clk_prepare_enable(pcie->clk);
+	ret = clk_prepare_enable(pcie->clk);
+	if (ret)
+		return ret;
 
 	/* Get the dw-pcie unit configuration/control registers base. */
 	base = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ctrl");
diff --git a/drivers/pci/dwc/pcie-artpec6.c b/drivers/pci/dwc/pcie-artpec6.c
index 01c6f7823672..6653619db6a1 100644
--- a/drivers/pci/dwc/pcie-artpec6.c
+++ b/drivers/pci/dwc/pcie-artpec6.c
@@ -141,12 +141,6 @@ static int artpec6_pcie_establish_link(struct artpec6_pcie *artpec6_pcie)
 	artpec6_pcie_writel(artpec6_pcie, PCIECFG, val);
 	usleep_range(100, 200);
 
-	/*
-	 * Enable writing to config regs. This is required as the Synopsys
-	 * driver changes the class code. That register needs DBI write enable.
-	 */
-	dw_pcie_writel_dbi(pci, MISC_CONTROL_1_OFF, DBI_RO_WR_EN);
-
 	/* setup root complex */
 	dw_pcie_setup_rc(pp);
 
@@ -175,13 +169,15 @@ static void artpec6_pcie_enable_interrupts(struct artpec6_pcie *artpec6_pcie)
 		dw_pcie_msi_init(pp);
 }
 
-static void artpec6_pcie_host_init(struct pcie_port *pp)
+static int artpec6_pcie_host_init(struct pcie_port *pp)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 	struct artpec6_pcie *artpec6_pcie = to_artpec6_pcie(pci);
 
 	artpec6_pcie_establish_link(artpec6_pcie);
 	artpec6_pcie_enable_interrupts(artpec6_pcie);
+
+	return 0;
 }
 
 static const struct dw_pcie_host_ops artpec6_pcie_host_ops = {
@@ -207,9 +203,9 @@ static int artpec6_add_pcie_port(struct artpec6_pcie *artpec6_pcie,
 
 	if (IS_ENABLED(CONFIG_PCI_MSI)) {
 		pp->msi_irq = platform_get_irq_byname(pdev, "msi");
-		if (pp->msi_irq <= 0) {
+		if (pp->msi_irq < 0) {
 			dev_err(dev, "failed to get MSI irq\n");
-			return -ENODEV;
+			return pp->msi_irq;
 		}
 
 		ret = devm_request_irq(dev, pp->msi_irq,
diff --git a/drivers/pci/dwc/pcie-designware-ep.c b/drivers/pci/dwc/pcie-designware-ep.c
index 398406393f37..d53d5f168363 100644
--- a/drivers/pci/dwc/pcie-designware-ep.c
+++ b/drivers/pci/dwc/pcie-designware-ep.c
@@ -1,5 +1,5 @@
 /**
- * Synopsys Designware PCIe Endpoint controller driver
+ * Synopsys DesignWare PCIe Endpoint controller driver
  *
  * Copyright (C) 2017 Texas Instruments
  * Author: Kishon Vijay Abraham I <kishon@ti.com>
@@ -283,7 +283,6 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep)
 {
 	int ret;
 	void *addr;
-	enum pci_barno bar;
 	struct pci_epc *epc;
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 	struct device *dev = pci->dev;
@@ -312,9 +311,6 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep)
 		return -ENOMEM;
 	ep->outbound_addr = addr;
 
-	for (bar = BAR_0; bar <= BAR_5; bar++)
-		dw_pcie_ep_reset_bar(pci, bar);
-
 	if (ep->ops->ep_init)
 		ep->ops->ep_init(ep);
 
@@ -328,7 +324,8 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep)
 	if (ret < 0)
 		epc->max_functions = 1;
 
-	ret = pci_epc_mem_init(epc, ep->phys_base, ep->addr_size);
+	ret = __pci_epc_mem_init(epc, ep->phys_base, ep->addr_size,
+				 ep->page_size);
 	if (ret < 0) {
 		dev_err(dev, "Failed to initialize address space\n");
 		return ret;
diff --git a/drivers/pci/dwc/pcie-designware-host.c b/drivers/pci/dwc/pcie-designware-host.c
index d29c020da082..81e2157a7cfb 100644
--- a/drivers/pci/dwc/pcie-designware-host.c
+++ b/drivers/pci/dwc/pcie-designware-host.c
@@ -1,5 +1,5 @@
 /*
- * Synopsys Designware PCIe host controller driver
+ * Synopsys DesignWare PCIe host controller driver
  *
  * Copyright (C) 2013 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
@@ -71,9 +71,9 @@ irqreturn_t dw_handle_msi_irq(struct pcie_port *pp)
 		while ((pos = find_next_bit((unsigned long *) &val, 32,
 					    pos)) != 32) {
 			irq = irq_find_mapping(pp->irq_domain, i * 32 + pos);
+			generic_handle_irq(irq);
 			dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_STATUS + i * 12,
 					    4, 1 << pos);
-			generic_handle_irq(irq);
 			pos++;
 		}
 	}
@@ -401,8 +401,11 @@ int dw_pcie_host_init(struct pcie_port *pp)
 		}
 	}
 
-	if (pp->ops->host_init)
-		pp->ops->host_init(pp);
+	if (pp->ops->host_init) {
+		ret = pp->ops->host_init(pp);
+		if (ret)
+			goto error;
+	}
 
 	pp->root_bus_nr = pp->busn->start;
 
@@ -594,10 +597,12 @@ void dw_pcie_setup_rc(struct pcie_port *pp)
 	dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_1, 0x00000000);
 
 	/* setup interrupt pins */
+	dw_pcie_dbi_ro_wr_en(pci);
 	val = dw_pcie_readl_dbi(pci, PCI_INTERRUPT_LINE);
 	val &= 0xffff00ff;
 	val |= 0x00000100;
 	dw_pcie_writel_dbi(pci, PCI_INTERRUPT_LINE, val);
+	dw_pcie_dbi_ro_wr_dis(pci);
 
 	/* setup bus numbers */
 	val = dw_pcie_readl_dbi(pci, PCI_PRIMARY_BUS);
@@ -634,8 +639,12 @@ void dw_pcie_setup_rc(struct pcie_port *pp)
 
 	dw_pcie_wr_own_conf(pp, PCI_BASE_ADDRESS_0, 4, 0);
 
+	/* Enable write permission for the DBI read-only register */
+	dw_pcie_dbi_ro_wr_en(pci);
 	/* program correct class for RC */
 	dw_pcie_wr_own_conf(pp, PCI_CLASS_DEVICE, 2, PCI_CLASS_BRIDGE_PCI);
+	/* Better disable write permission right after the update */
+	dw_pcie_dbi_ro_wr_dis(pci);
 
 	dw_pcie_rd_own_conf(pp, PCIE_LINK_WIDTH_SPEED_CONTROL, 4, &val);
 	val |= PORT_LOGIC_SPEED_CHANGE;
diff --git a/drivers/pci/dwc/pcie-designware-plat.c b/drivers/pci/dwc/pcie-designware-plat.c
index 091b4e7ad059..168e2380f493 100644
--- a/drivers/pci/dwc/pcie-designware-plat.c
+++ b/drivers/pci/dwc/pcie-designware-plat.c
@@ -35,7 +35,7 @@ static irqreturn_t dw_plat_pcie_msi_irq_handler(int irq, void *arg)
 	return dw_handle_msi_irq(pp);
 }
 
-static void dw_plat_pcie_host_init(struct pcie_port *pp)
+static int dw_plat_pcie_host_init(struct pcie_port *pp)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 
@@ -44,6 +44,8 @@ static void dw_plat_pcie_host_init(struct pcie_port *pp)
 
 	if (IS_ENABLED(CONFIG_PCI_MSI))
 		dw_pcie_msi_init(pp);
+
+	return 0;
 }
 
 static const struct dw_pcie_host_ops dw_plat_pcie_host_ops = {
diff --git a/drivers/pci/dwc/pcie-designware.c b/drivers/pci/dwc/pcie-designware.c
index 0e03af279259..88abdddee2ad 100644
--- a/drivers/pci/dwc/pcie-designware.c
+++ b/drivers/pci/dwc/pcie-designware.c
@@ -1,5 +1,5 @@
 /*
- * Synopsys Designware PCIe host controller driver
+ * Synopsys DesignWare PCIe host controller driver
  *
  * Copyright (C) 2013 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
@@ -107,8 +107,9 @@ static void dw_pcie_writel_ob_unroll(struct dw_pcie *pci, u32 index, u32 reg,
 	dw_pcie_writel_dbi(pci, offset + reg, val);
 }
 
-void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index, int type,
-				      u64 cpu_addr, u64 pci_addr, u32 size)
+static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index,
+					     int type, u64 cpu_addr,
+					     u64 pci_addr, u32 size)
 {
 	u32 retries, val;
 
@@ -177,7 +178,7 @@ void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type,
 	 */
 	for (retries = 0; retries < LINK_WAIT_MAX_IATU_RETRIES; retries++) {
 		val = dw_pcie_readl_dbi(pci, PCIE_ATU_CR2);
-		if (val == PCIE_ATU_ENABLE)
+		if (val & PCIE_ATU_ENABLE)
 			return;
 
 		usleep_range(LINK_WAIT_IATU_MIN, LINK_WAIT_IATU_MAX);
@@ -200,8 +201,9 @@ static void dw_pcie_writel_ib_unroll(struct dw_pcie *pci, u32 index, u32 reg,
 	dw_pcie_writel_dbi(pci, offset + reg, val);
 }
 
-int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, int index, int bar,
-				    u64 cpu_addr, enum dw_pcie_as_type as_type)
+static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, int index,
+					   int bar, u64 cpu_addr,
+					   enum dw_pcie_as_type as_type)
 {
 	int type;
 	u32 retries, val;
diff --git a/drivers/pci/dwc/pcie-designware.h b/drivers/pci/dwc/pcie-designware.h
index b4d2a89f8e58..e5d9d77b778e 100644
--- a/drivers/pci/dwc/pcie-designware.h
+++ b/drivers/pci/dwc/pcie-designware.h
@@ -1,5 +1,5 @@
 /*
- * Synopsys Designware PCIe host controller driver
+ * Synopsys DesignWare PCIe host controller driver
  *
  * Copyright (C) 2013 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
@@ -76,6 +76,9 @@
 #define PCIE_ATU_FUNC(x)		(((x) & 0x7) << 16)
 #define PCIE_ATU_UPPER_TARGET		0x91C
 
+#define PCIE_MISC_CONTROL_1_OFF		0x8BC
+#define PCIE_DBI_RO_WR_EN		(0x1 << 0)
+
 /*
  * iATU Unroll-specific register definitions
  * From 4.80 core version the address translation will be made by unroll
@@ -134,7 +137,7 @@ struct dw_pcie_host_ops {
 			     unsigned int devfn, int where, int size, u32 *val);
 	int (*wr_other_conf)(struct pcie_port *pp, struct pci_bus *bus,
 			     unsigned int devfn, int where, int size, u32 val);
-	void (*host_init)(struct pcie_port *pp);
+	int (*host_init)(struct pcie_port *pp);
 	void (*msi_set_irq)(struct pcie_port *pp, int irq);
 	void (*msi_clear_irq)(struct pcie_port *pp, int irq);
 	phys_addr_t (*get_msi_addr)(struct pcie_port *pp);
@@ -186,6 +189,7 @@ struct dw_pcie_ep {
 	struct dw_pcie_ep_ops	*ops;
 	phys_addr_t		phys_base;
 	size_t			addr_size;
+	size_t			page_size;
 	u8			bar_to_atu[6];
 	phys_addr_t		*outbound_addr;
 	unsigned long		ib_window_map;
@@ -279,6 +283,28 @@ static inline u32 dw_pcie_readl_dbi2(struct dw_pcie *pci, u32 reg)
 	return __dw_pcie_read_dbi(pci, pci->dbi_base2, reg, 0x4);
 }
 
+static inline void dw_pcie_dbi_ro_wr_en(struct dw_pcie *pci)
+{
+	u32 reg;
+	u32 val;
+
+	reg = PCIE_MISC_CONTROL_1_OFF;
+	val = dw_pcie_readl_dbi(pci, reg);
+	val |= PCIE_DBI_RO_WR_EN;
+	dw_pcie_writel_dbi(pci, reg, val);
+}
+
+static inline void dw_pcie_dbi_ro_wr_dis(struct dw_pcie *pci)
+{
+	u32 reg;
+	u32 val;
+
+	reg = PCIE_MISC_CONTROL_1_OFF;
+	val = dw_pcie_readl_dbi(pci, reg);
+	val &= ~PCIE_DBI_RO_WR_EN;
+	dw_pcie_writel_dbi(pci, reg, val);
+}
+
 #ifdef CONFIG_PCIE_DW_HOST
 irqreturn_t dw_handle_msi_irq(struct pcie_port *pp);
 void dw_pcie_msi_init(struct pcie_port *pp);
diff --git a/drivers/pci/dwc/pcie-hisi.c b/drivers/pci/dwc/pcie-hisi.c
index e51acee0ddf3..a20179169e06 100644
--- a/drivers/pci/dwc/pcie-hisi.c
+++ b/drivers/pci/dwc/pcie-hisi.c
@@ -223,7 +223,7 @@ static int hisi_pcie_link_up(struct dw_pcie *pci)
 	return hisi_pcie->soc_ops->hisi_pcie_link_up(hisi_pcie);
 }
 
-static struct dw_pcie_host_ops hisi_pcie_host_ops = {
+static const struct dw_pcie_host_ops hisi_pcie_host_ops = {
 	.rd_own_conf = hisi_pcie_cfg_read,
 	.wr_own_conf = hisi_pcie_cfg_write,
 };
@@ -268,7 +268,6 @@ static int hisi_pcie_probe(struct platform_device *pdev)
 	struct dw_pcie *pci;
 	struct hisi_pcie *hisi_pcie;
 	struct resource *reg;
-	struct device_driver *driver;
 	int ret;
 
 	hisi_pcie = devm_kzalloc(dev, sizeof(*hisi_pcie), GFP_KERNEL);
@@ -282,8 +281,6 @@ static int hisi_pcie_probe(struct platform_device *pdev)
 	pci->dev = dev;
 	pci->ops = &dw_pcie_ops;
 
-	driver = dev->driver;
-
 	hisi_pcie->pci = pci;
 
 	hisi_pcie->soc_ops = of_device_get_match_data(dev);
diff --git a/drivers/pci/dwc/pcie-kirin.c b/drivers/pci/dwc/pcie-kirin.c
index 33fddb9f6739..dc3033cf3c19 100644
--- a/drivers/pci/dwc/pcie-kirin.c
+++ b/drivers/pci/dwc/pcie-kirin.c
@@ -430,9 +430,11 @@ static int kirin_pcie_establish_link(struct pcie_port *pp)
 	return 0;
 }
 
-static void kirin_pcie_host_init(struct pcie_port *pp)
+static int kirin_pcie_host_init(struct pcie_port *pp)
 {
 	kirin_pcie_establish_link(pp);
+
+	return 0;
 }
 
 static struct dw_pcie_ops kirin_dw_pcie_ops = {
@@ -441,7 +443,7 @@ static struct dw_pcie_ops kirin_dw_pcie_ops = {
 	.link_up = kirin_pcie_link_up,
 };
 
-static struct dw_pcie_host_ops kirin_pcie_host_ops = {
+static const struct dw_pcie_host_ops kirin_pcie_host_ops = {
 	.rd_own_conf = kirin_pcie_rd_own_conf,
 	.wr_own_conf = kirin_pcie_wr_own_conf,
 	.host_init = kirin_pcie_host_init,
diff --git a/drivers/pci/dwc/pcie-qcom.c b/drivers/pci/dwc/pcie-qcom.c
index 68c5f2ab5bc8..ce7ba5b7552a 100644
--- a/drivers/pci/dwc/pcie-qcom.c
+++ b/drivers/pci/dwc/pcie-qcom.c
@@ -37,6 +37,20 @@
 #include "pcie-designware.h"
 
 #define PCIE20_PARF_SYS_CTRL			0x00
+#define MST_WAKEUP_EN				BIT(13)
+#define SLV_WAKEUP_EN				BIT(12)
+#define MSTR_ACLK_CGC_DIS			BIT(10)
+#define SLV_ACLK_CGC_DIS			BIT(9)
+#define CORE_CLK_CGC_DIS			BIT(6)
+#define AUX_PWR_DET				BIT(4)
+#define L23_CLK_RMV_DIS				BIT(2)
+#define L1_CLK_RMV_DIS				BIT(1)
+
+#define PCIE20_COMMAND_STATUS			0x04
+#define CMD_BME_VAL				0x4
+#define PCIE20_DEVICE_CONTROL2_STATUS2		0x98
+#define PCIE_CAP_CPL_TIMEOUT_DISABLE		0x10
+
 #define PCIE20_PARF_PHY_CTRL			0x40
 #define PCIE20_PARF_PHY_REFCLK			0x4C
 #define PCIE20_PARF_DBI_BASE_ADDR		0x168
@@ -58,10 +72,22 @@
 #define CFG_BRIDGE_SB_INIT			BIT(0)
 
 #define PCIE20_CAP				0x70
+#define PCIE20_CAP_LINK_CAPABILITIES		(PCIE20_CAP + 0xC)
+#define PCIE20_CAP_ACTIVE_STATE_LINK_PM_SUPPORT	(BIT(10) | BIT(11))
+#define PCIE20_CAP_LINK_1			(PCIE20_CAP + 0x14)
+#define PCIE_CAP_LINK1_VAL			0x2FD7F
+
+#define PCIE20_PARF_Q2A_FLUSH			0x1AC
+
+#define PCIE20_MISC_CONTROL_1_REG		0x8BC
+#define DBI_RO_WR_EN				1
 
 #define PERST_DELAY_US				1000
 
-struct qcom_pcie_resources_v0 {
+#define PCIE20_v3_PARF_SLV_ADDR_SPACE_SIZE	0x358
+#define SLV_ADDR_SPACE_SZ			0x10000000
+
+struct qcom_pcie_resources_2_1_0 {
 	struct clk *iface_clk;
 	struct clk *core_clk;
 	struct clk *phy_clk;
@@ -75,7 +101,7 @@ struct qcom_pcie_resources_v0 {
 	struct regulator *vdda_refclk;
 };
 
-struct qcom_pcie_resources_v1 {
+struct qcom_pcie_resources_1_0_0 {
 	struct clk *iface;
 	struct clk *aux;
 	struct clk *master_bus;
@@ -84,7 +110,7 @@ struct qcom_pcie_resources_v1 {
 	struct regulator *vdda;
 };
 
-struct qcom_pcie_resources_v2 {
+struct qcom_pcie_resources_2_3_2 {
 	struct clk *aux_clk;
 	struct clk *master_clk;
 	struct clk *slave_clk;
@@ -92,7 +118,7 @@ struct qcom_pcie_resources_v2 {
 	struct clk *pipe_clk;
 };
 
-struct qcom_pcie_resources_v3 {
+struct qcom_pcie_resources_2_4_0 {
 	struct clk *aux_clk;
 	struct clk *master_clk;
 	struct clk *slave_clk;
@@ -110,11 +136,21 @@ struct qcom_pcie_resources_v3 {
 	struct reset_control *phy_ahb_reset;
 };
 
+struct qcom_pcie_resources_2_3_3 {
+	struct clk *iface;
+	struct clk *axi_m_clk;
+	struct clk *axi_s_clk;
+	struct clk *ahb_clk;
+	struct clk *aux_clk;
+	struct reset_control *rst[7];
+};
+
 union qcom_pcie_resources {
-	struct qcom_pcie_resources_v0 v0;
-	struct qcom_pcie_resources_v1 v1;
-	struct qcom_pcie_resources_v2 v2;
-	struct qcom_pcie_resources_v3 v3;
+	struct qcom_pcie_resources_1_0_0 v1_0_0;
+	struct qcom_pcie_resources_2_1_0 v2_1_0;
+	struct qcom_pcie_resources_2_3_2 v2_3_2;
+	struct qcom_pcie_resources_2_3_3 v2_3_3;
+	struct qcom_pcie_resources_2_4_0 v2_4_0;
 };
 
 struct qcom_pcie;
@@ -124,6 +160,7 @@ struct qcom_pcie_ops {
 	int (*init)(struct qcom_pcie *pcie);
 	int (*post_init)(struct qcom_pcie *pcie);
 	void (*deinit)(struct qcom_pcie *pcie);
+	void (*post_deinit)(struct qcom_pcie *pcie);
 	void (*ltssm_enable)(struct qcom_pcie *pcie);
 };
 
@@ -141,13 +178,13 @@ struct qcom_pcie {
 
 static void qcom_ep_reset_assert(struct qcom_pcie *pcie)
 {
-	gpiod_set_value(pcie->reset, 1);
+	gpiod_set_value_cansleep(pcie->reset, 1);
 	usleep_range(PERST_DELAY_US, PERST_DELAY_US + 500);
 }
 
 static void qcom_ep_reset_deassert(struct qcom_pcie *pcie)
 {
-	gpiod_set_value(pcie->reset, 0);
+	gpiod_set_value_cansleep(pcie->reset, 0);
 	usleep_range(PERST_DELAY_US, PERST_DELAY_US + 500);
 }
 
@@ -172,7 +209,7 @@ static int qcom_pcie_establish_link(struct qcom_pcie *pcie)
 	return dw_pcie_wait_for_link(pci);
 }
 
-static void qcom_pcie_v0_v1_ltssm_enable(struct qcom_pcie *pcie)
+static void qcom_pcie_2_1_0_ltssm_enable(struct qcom_pcie *pcie)
 {
 	u32 val;
 
@@ -182,9 +219,9 @@ static void qcom_pcie_v0_v1_ltssm_enable(struct qcom_pcie *pcie)
 	writel(val, pcie->elbi + PCIE20_ELBI_SYS_CTRL);
 }
 
-static int qcom_pcie_get_resources_v0(struct qcom_pcie *pcie)
+static int qcom_pcie_get_resources_2_1_0(struct qcom_pcie *pcie)
 {
-	struct qcom_pcie_resources_v0 *res = &pcie->res.v0;
+	struct qcom_pcie_resources_2_1_0 *res = &pcie->res.v2_1_0;
 	struct dw_pcie *pci = pcie->pci;
 	struct device *dev = pci->dev;
 
@@ -212,29 +249,29 @@ static int qcom_pcie_get_resources_v0(struct qcom_pcie *pcie)
 	if (IS_ERR(res->phy_clk))
 		return PTR_ERR(res->phy_clk);
 
-	res->pci_reset = devm_reset_control_get(dev, "pci");
+	res->pci_reset = devm_reset_control_get_exclusive(dev, "pci");
 	if (IS_ERR(res->pci_reset))
 		return PTR_ERR(res->pci_reset);
 
-	res->axi_reset = devm_reset_control_get(dev, "axi");
+	res->axi_reset = devm_reset_control_get_exclusive(dev, "axi");
 	if (IS_ERR(res->axi_reset))
 		return PTR_ERR(res->axi_reset);
 
-	res->ahb_reset = devm_reset_control_get(dev, "ahb");
+	res->ahb_reset = devm_reset_control_get_exclusive(dev, "ahb");
 	if (IS_ERR(res->ahb_reset))
 		return PTR_ERR(res->ahb_reset);
 
-	res->por_reset = devm_reset_control_get(dev, "por");
+	res->por_reset = devm_reset_control_get_exclusive(dev, "por");
 	if (IS_ERR(res->por_reset))
 		return PTR_ERR(res->por_reset);
 
-	res->phy_reset = devm_reset_control_get(dev, "phy");
+	res->phy_reset = devm_reset_control_get_exclusive(dev, "phy");
 	return PTR_ERR_OR_ZERO(res->phy_reset);
 }
 
-static void qcom_pcie_deinit_v0(struct qcom_pcie *pcie)
+static void qcom_pcie_deinit_2_1_0(struct qcom_pcie *pcie)
 {
-	struct qcom_pcie_resources_v0 *res = &pcie->res.v0;
+	struct qcom_pcie_resources_2_1_0 *res = &pcie->res.v2_1_0;
 
 	reset_control_assert(res->pci_reset);
 	reset_control_assert(res->axi_reset);
@@ -249,9 +286,9 @@ static void qcom_pcie_deinit_v0(struct qcom_pcie *pcie)
 	regulator_disable(res->vdda_refclk);
 }
 
-static int qcom_pcie_init_v0(struct qcom_pcie *pcie)
+static int qcom_pcie_init_2_1_0(struct qcom_pcie *pcie)
 {
-	struct qcom_pcie_resources_v0 *res = &pcie->res.v0;
+	struct qcom_pcie_resources_2_1_0 *res = &pcie->res.v2_1_0;
 	struct dw_pcie *pci = pcie->pci;
 	struct device *dev = pci->dev;
 	u32 val;
@@ -367,9 +404,9 @@ err_refclk:
 	return ret;
 }
 
-static int qcom_pcie_get_resources_v1(struct qcom_pcie *pcie)
+static int qcom_pcie_get_resources_1_0_0(struct qcom_pcie *pcie)
 {
-	struct qcom_pcie_resources_v1 *res = &pcie->res.v1;
+	struct qcom_pcie_resources_1_0_0 *res = &pcie->res.v1_0_0;
 	struct dw_pcie *pci = pcie->pci;
 	struct device *dev = pci->dev;
 
@@ -393,13 +430,13 @@ static int qcom_pcie_get_resources_v1(struct qcom_pcie *pcie)
 	if (IS_ERR(res->slave_bus))
 		return PTR_ERR(res->slave_bus);
 
-	res->core = devm_reset_control_get(dev, "core");
+	res->core = devm_reset_control_get_exclusive(dev, "core");
 	return PTR_ERR_OR_ZERO(res->core);
 }
 
-static void qcom_pcie_deinit_v1(struct qcom_pcie *pcie)
+static void qcom_pcie_deinit_1_0_0(struct qcom_pcie *pcie)
 {
-	struct qcom_pcie_resources_v1 *res = &pcie->res.v1;
+	struct qcom_pcie_resources_1_0_0 *res = &pcie->res.v1_0_0;
 
 	reset_control_assert(res->core);
 	clk_disable_unprepare(res->slave_bus);
@@ -409,9 +446,9 @@ static void qcom_pcie_deinit_v1(struct qcom_pcie *pcie)
 	regulator_disable(res->vdda);
 }
 
-static int qcom_pcie_init_v1(struct qcom_pcie *pcie)
+static int qcom_pcie_init_1_0_0(struct qcom_pcie *pcie)
 {
-	struct qcom_pcie_resources_v1 *res = &pcie->res.v1;
+	struct qcom_pcie_resources_1_0_0 *res = &pcie->res.v1_0_0;
 	struct dw_pcie *pci = pcie->pci;
 	struct device *dev = pci->dev;
 	int ret;
@@ -477,7 +514,7 @@ err_res:
 	return ret;
 }
 
-static void qcom_pcie_v2_ltssm_enable(struct qcom_pcie *pcie)
+static void qcom_pcie_2_3_2_ltssm_enable(struct qcom_pcie *pcie)
 {
 	u32 val;
 
@@ -487,9 +524,9 @@ static void qcom_pcie_v2_ltssm_enable(struct qcom_pcie *pcie)
 	writel(val, pcie->parf + PCIE20_PARF_LTSSM);
 }
 
-static int qcom_pcie_get_resources_v2(struct qcom_pcie *pcie)
+static int qcom_pcie_get_resources_2_3_2(struct qcom_pcie *pcie)
 {
-	struct qcom_pcie_resources_v2 *res = &pcie->res.v2;
+	struct qcom_pcie_resources_2_3_2 *res = &pcie->res.v2_3_2;
 	struct dw_pcie *pci = pcie->pci;
 	struct device *dev = pci->dev;
 
@@ -513,20 +550,26 @@ static int qcom_pcie_get_resources_v2(struct qcom_pcie *pcie)
 	return PTR_ERR_OR_ZERO(res->pipe_clk);
 }
 
-static void qcom_pcie_deinit_v2(struct qcom_pcie *pcie)
+static void qcom_pcie_deinit_2_3_2(struct qcom_pcie *pcie)
 {
-	struct qcom_pcie_resources_v2 *res = &pcie->res.v2;
+	struct qcom_pcie_resources_2_3_2 *res = &pcie->res.v2_3_2;
 
-	clk_disable_unprepare(res->pipe_clk);
 	clk_disable_unprepare(res->slave_clk);
 	clk_disable_unprepare(res->master_clk);
 	clk_disable_unprepare(res->cfg_clk);
 	clk_disable_unprepare(res->aux_clk);
 }
 
-static int qcom_pcie_init_v2(struct qcom_pcie *pcie)
+static void qcom_pcie_post_deinit_2_3_2(struct qcom_pcie *pcie)
 {
-	struct qcom_pcie_resources_v2 *res = &pcie->res.v2;
+	struct qcom_pcie_resources_2_3_2 *res = &pcie->res.v2_3_2;
+
+	clk_disable_unprepare(res->pipe_clk);
+}
+
+static int qcom_pcie_init_2_3_2(struct qcom_pcie *pcie)
+{
+	struct qcom_pcie_resources_2_3_2 *res = &pcie->res.v2_3_2;
 	struct dw_pcie *pci = pcie->pci;
 	struct device *dev = pci->dev;
 	u32 val;
@@ -589,9 +632,9 @@ err_cfg_clk:
 	return ret;
 }
 
-static int qcom_pcie_post_init_v2(struct qcom_pcie *pcie)
+static int qcom_pcie_post_init_2_3_2(struct qcom_pcie *pcie)
 {
-	struct qcom_pcie_resources_v2 *res = &pcie->res.v2;
+	struct qcom_pcie_resources_2_3_2 *res = &pcie->res.v2_3_2;
 	struct dw_pcie *pci = pcie->pci;
 	struct device *dev = pci->dev;
 	int ret;
@@ -605,9 +648,9 @@ static int qcom_pcie_post_init_v2(struct qcom_pcie *pcie)
 	return 0;
 }
 
-static int qcom_pcie_get_resources_v3(struct qcom_pcie *pcie)
+static int qcom_pcie_get_resources_2_4_0(struct qcom_pcie *pcie)
 {
-	struct qcom_pcie_resources_v3 *res = &pcie->res.v3;
+	struct qcom_pcie_resources_2_4_0 *res = &pcie->res.v2_4_0;
 	struct dw_pcie *pci = pcie->pci;
 	struct device *dev = pci->dev;
 
@@ -623,60 +666,64 @@ static int qcom_pcie_get_resources_v3(struct qcom_pcie *pcie)
 	if (IS_ERR(res->slave_clk))
 		return PTR_ERR(res->slave_clk);
 
-	res->axi_m_reset = devm_reset_control_get(dev, "axi_m");
+	res->axi_m_reset = devm_reset_control_get_exclusive(dev, "axi_m");
 	if (IS_ERR(res->axi_m_reset))
 		return PTR_ERR(res->axi_m_reset);
 
-	res->axi_s_reset = devm_reset_control_get(dev, "axi_s");
+	res->axi_s_reset = devm_reset_control_get_exclusive(dev, "axi_s");
 	if (IS_ERR(res->axi_s_reset))
 		return PTR_ERR(res->axi_s_reset);
 
-	res->pipe_reset = devm_reset_control_get(dev, "pipe");
+	res->pipe_reset = devm_reset_control_get_exclusive(dev, "pipe");
 	if (IS_ERR(res->pipe_reset))
 		return PTR_ERR(res->pipe_reset);
 
-	res->axi_m_vmid_reset = devm_reset_control_get(dev, "axi_m_vmid");
+	res->axi_m_vmid_reset = devm_reset_control_get_exclusive(dev,
+								 "axi_m_vmid");
 	if (IS_ERR(res->axi_m_vmid_reset))
 		return PTR_ERR(res->axi_m_vmid_reset);
 
-	res->axi_s_xpu_reset = devm_reset_control_get(dev, "axi_s_xpu");
+	res->axi_s_xpu_reset = devm_reset_control_get_exclusive(dev,
+								"axi_s_xpu");
 	if (IS_ERR(res->axi_s_xpu_reset))
 		return PTR_ERR(res->axi_s_xpu_reset);
 
-	res->parf_reset = devm_reset_control_get(dev, "parf");
+	res->parf_reset = devm_reset_control_get_exclusive(dev, "parf");
 	if (IS_ERR(res->parf_reset))
 		return PTR_ERR(res->parf_reset);
 
-	res->phy_reset = devm_reset_control_get(dev, "phy");
+	res->phy_reset = devm_reset_control_get_exclusive(dev, "phy");
 	if (IS_ERR(res->phy_reset))
 		return PTR_ERR(res->phy_reset);
 
-	res->axi_m_sticky_reset = devm_reset_control_get(dev, "axi_m_sticky");
+	res->axi_m_sticky_reset = devm_reset_control_get_exclusive(dev,
+								   "axi_m_sticky");
 	if (IS_ERR(res->axi_m_sticky_reset))
 		return PTR_ERR(res->axi_m_sticky_reset);
 
-	res->pipe_sticky_reset = devm_reset_control_get(dev, "pipe_sticky");
+	res->pipe_sticky_reset = devm_reset_control_get_exclusive(dev,
+								  "pipe_sticky");
 	if (IS_ERR(res->pipe_sticky_reset))
 		return PTR_ERR(res->pipe_sticky_reset);
 
-	res->pwr_reset = devm_reset_control_get(dev, "pwr");
+	res->pwr_reset = devm_reset_control_get_exclusive(dev, "pwr");
 	if (IS_ERR(res->pwr_reset))
 		return PTR_ERR(res->pwr_reset);
 
-	res->ahb_reset = devm_reset_control_get(dev, "ahb");
+	res->ahb_reset = devm_reset_control_get_exclusive(dev, "ahb");
 	if (IS_ERR(res->ahb_reset))
 		return PTR_ERR(res->ahb_reset);
 
-	res->phy_ahb_reset = devm_reset_control_get(dev, "phy_ahb");
+	res->phy_ahb_reset = devm_reset_control_get_exclusive(dev, "phy_ahb");
 	if (IS_ERR(res->phy_ahb_reset))
 		return PTR_ERR(res->phy_ahb_reset);
 
 	return 0;
 }
 
-static void qcom_pcie_deinit_v3(struct qcom_pcie *pcie)
+static void qcom_pcie_deinit_2_4_0(struct qcom_pcie *pcie)
 {
-	struct qcom_pcie_resources_v3 *res = &pcie->res.v3;
+	struct qcom_pcie_resources_2_4_0 *res = &pcie->res.v2_4_0;
 
 	reset_control_assert(res->axi_m_reset);
 	reset_control_assert(res->axi_s_reset);
@@ -692,9 +739,9 @@ static void qcom_pcie_deinit_v3(struct qcom_pcie *pcie)
 	clk_disable_unprepare(res->slave_clk);
 }
 
-static int qcom_pcie_init_v3(struct qcom_pcie *pcie)
+static int qcom_pcie_init_2_4_0(struct qcom_pcie *pcie)
 {
-	struct qcom_pcie_resources_v3 *res = &pcie->res.v3;
+	struct qcom_pcie_resources_2_4_0 *res = &pcie->res.v2_4_0;
 	struct dw_pcie *pci = pcie->pci;
 	struct device *dev = pci->dev;
 	u32 val;
@@ -884,6 +931,166 @@ err_rst_phy:
 	return ret;
 }
 
+static int qcom_pcie_get_resources_2_3_3(struct qcom_pcie *pcie)
+{
+	struct qcom_pcie_resources_2_3_3 *res = &pcie->res.v2_3_3;
+	struct dw_pcie *pci = pcie->pci;
+	struct device *dev = pci->dev;
+	int i;
+	const char *rst_names[] = { "axi_m", "axi_s", "pipe",
+				    "axi_m_sticky", "sticky",
+				    "ahb", "sleep", };
+
+	res->iface = devm_clk_get(dev, "iface");
+	if (IS_ERR(res->iface))
+		return PTR_ERR(res->iface);
+
+	res->axi_m_clk = devm_clk_get(dev, "axi_m");
+	if (IS_ERR(res->axi_m_clk))
+		return PTR_ERR(res->axi_m_clk);
+
+	res->axi_s_clk = devm_clk_get(dev, "axi_s");
+	if (IS_ERR(res->axi_s_clk))
+		return PTR_ERR(res->axi_s_clk);
+
+	res->ahb_clk = devm_clk_get(dev, "ahb");
+	if (IS_ERR(res->ahb_clk))
+		return PTR_ERR(res->ahb_clk);
+
+	res->aux_clk = devm_clk_get(dev, "aux");
+	if (IS_ERR(res->aux_clk))
+		return PTR_ERR(res->aux_clk);
+
+	for (i = 0; i < ARRAY_SIZE(rst_names); i++) {
+		res->rst[i] = devm_reset_control_get(dev, rst_names[i]);
+		if (IS_ERR(res->rst[i]))
+			return PTR_ERR(res->rst[i]);
+	}
+
+	return 0;
+}
+
+static void qcom_pcie_deinit_2_3_3(struct qcom_pcie *pcie)
+{
+	struct qcom_pcie_resources_2_3_3 *res = &pcie->res.v2_3_3;
+
+	clk_disable_unprepare(res->iface);
+	clk_disable_unprepare(res->axi_m_clk);
+	clk_disable_unprepare(res->axi_s_clk);
+	clk_disable_unprepare(res->ahb_clk);
+	clk_disable_unprepare(res->aux_clk);
+}
+
+static int qcom_pcie_init_2_3_3(struct qcom_pcie *pcie)
+{
+	struct qcom_pcie_resources_2_3_3 *res = &pcie->res.v2_3_3;
+	struct dw_pcie *pci = pcie->pci;
+	struct device *dev = pci->dev;
+	int i, ret;
+	u32 val;
+
+	for (i = 0; i < ARRAY_SIZE(res->rst); i++) {
+		ret = reset_control_assert(res->rst[i]);
+		if (ret) {
+			dev_err(dev, "reset #%d assert failed (%d)\n", i, ret);
+			return ret;
+		}
+	}
+
+	usleep_range(2000, 2500);
+
+	for (i = 0; i < ARRAY_SIZE(res->rst); i++) {
+		ret = reset_control_deassert(res->rst[i]);
+		if (ret) {
+			dev_err(dev, "reset #%d deassert failed (%d)\n", i,
+				ret);
+			return ret;
+		}
+	}
+
+	/*
+	 * Don't have a way to see if the reset has completed.
+	 * Wait for some time.
+	 */
+	usleep_range(2000, 2500);
+
+	ret = clk_prepare_enable(res->iface);
+	if (ret) {
+		dev_err(dev, "cannot prepare/enable core clock\n");
+		goto err_clk_iface;
+	}
+
+	ret = clk_prepare_enable(res->axi_m_clk);
+	if (ret) {
+		dev_err(dev, "cannot prepare/enable core clock\n");
+		goto err_clk_axi_m;
+	}
+
+	ret = clk_prepare_enable(res->axi_s_clk);
+	if (ret) {
+		dev_err(dev, "cannot prepare/enable axi slave clock\n");
+		goto err_clk_axi_s;
+	}
+
+	ret = clk_prepare_enable(res->ahb_clk);
+	if (ret) {
+		dev_err(dev, "cannot prepare/enable ahb clock\n");
+		goto err_clk_ahb;
+	}
+
+	ret = clk_prepare_enable(res->aux_clk);
+	if (ret) {
+		dev_err(dev, "cannot prepare/enable aux clock\n");
+		goto err_clk_aux;
+	}
+
+	writel(SLV_ADDR_SPACE_SZ,
+		pcie->parf + PCIE20_v3_PARF_SLV_ADDR_SPACE_SIZE);
+
+	val = readl(pcie->parf + PCIE20_PARF_PHY_CTRL);
+	val &= ~BIT(0);
+	writel(val, pcie->parf + PCIE20_PARF_PHY_CTRL);
+
+	writel(0, pcie->parf + PCIE20_PARF_DBI_BASE_ADDR);
+
+	writel(MST_WAKEUP_EN | SLV_WAKEUP_EN | MSTR_ACLK_CGC_DIS
+		| SLV_ACLK_CGC_DIS | CORE_CLK_CGC_DIS |
+		AUX_PWR_DET | L23_CLK_RMV_DIS | L1_CLK_RMV_DIS,
+		pcie->parf + PCIE20_PARF_SYS_CTRL);
+	writel(0, pcie->parf + PCIE20_PARF_Q2A_FLUSH);
+
+	writel(CMD_BME_VAL, pci->dbi_base + PCIE20_COMMAND_STATUS);
+	writel(DBI_RO_WR_EN, pci->dbi_base + PCIE20_MISC_CONTROL_1_REG);
+	writel(PCIE_CAP_LINK1_VAL, pci->dbi_base + PCIE20_CAP_LINK_1);
+
+	val = readl(pci->dbi_base + PCIE20_CAP_LINK_CAPABILITIES);
+	val &= ~PCIE20_CAP_ACTIVE_STATE_LINK_PM_SUPPORT;
+	writel(val, pci->dbi_base + PCIE20_CAP_LINK_CAPABILITIES);
+
+	writel(PCIE_CAP_CPL_TIMEOUT_DISABLE, pci->dbi_base +
+		PCIE20_DEVICE_CONTROL2_STATUS2);
+
+	return 0;
+
+err_clk_aux:
+	clk_disable_unprepare(res->ahb_clk);
+err_clk_ahb:
+	clk_disable_unprepare(res->axi_s_clk);
+err_clk_axi_s:
+	clk_disable_unprepare(res->axi_m_clk);
+err_clk_axi_m:
+	clk_disable_unprepare(res->iface);
+err_clk_iface:
+	/*
+	 * Not checking for failure, will anyway return
+	 * the original failure in 'ret'.
+	 */
+	for (i = 0; i < ARRAY_SIZE(res->rst); i++)
+		reset_control_assert(res->rst[i]);
+
+	return ret;
+}
+
 static int qcom_pcie_link_up(struct dw_pcie *pci)
 {
 	u16 val = readw(pci->dbi_base + PCIE20_CAP + PCI_EXP_LNKSTA);
@@ -891,7 +1098,7 @@ static int qcom_pcie_link_up(struct dw_pcie *pci)
 	return !!(val & PCI_EXP_LNKSTA_DLLLA);
 }
 
-static void qcom_pcie_host_init(struct pcie_port *pp)
+static int qcom_pcie_host_init(struct pcie_port *pp)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 	struct qcom_pcie *pcie = to_qcom_pcie(pci);
@@ -901,14 +1108,17 @@ static void qcom_pcie_host_init(struct pcie_port *pp)
 
 	ret = pcie->ops->init(pcie);
 	if (ret)
-		goto err_deinit;
+		return ret;
 
 	ret = phy_power_on(pcie->phy);
 	if (ret)
 		goto err_deinit;
 
-	if (pcie->ops->post_init)
-		pcie->ops->post_init(pcie);
+	if (pcie->ops->post_init) {
+		ret = pcie->ops->post_init(pcie);
+		if (ret)
+			goto err_disable_phy;
+	}
 
 	dw_pcie_setup_rc(pp);
 
@@ -921,12 +1131,17 @@ static void qcom_pcie_host_init(struct pcie_port *pp)
 	if (ret)
 		goto err;
 
-	return;
+	return 0;
 err:
 	qcom_ep_reset_assert(pcie);
+	if (pcie->ops->post_deinit)
+		pcie->ops->post_deinit(pcie);
+err_disable_phy:
 	phy_power_off(pcie->phy);
 err_deinit:
 	pcie->ops->deinit(pcie);
+
+	return ret;
 }
 
 static int qcom_pcie_rd_own_conf(struct pcie_port *pp, int where, int size,
@@ -950,37 +1165,50 @@ static const struct dw_pcie_host_ops qcom_pcie_dw_ops = {
 	.rd_own_conf = qcom_pcie_rd_own_conf,
 };
 
-static const struct qcom_pcie_ops ops_v0 = {
-	.get_resources = qcom_pcie_get_resources_v0,
-	.init = qcom_pcie_init_v0,
-	.deinit = qcom_pcie_deinit_v0,
-	.ltssm_enable = qcom_pcie_v0_v1_ltssm_enable,
+/* Qcom IP rev.: 2.1.0	Synopsys IP rev.: 4.01a */
+static const struct qcom_pcie_ops ops_2_1_0 = {
+	.get_resources = qcom_pcie_get_resources_2_1_0,
+	.init = qcom_pcie_init_2_1_0,
+	.deinit = qcom_pcie_deinit_2_1_0,
+	.ltssm_enable = qcom_pcie_2_1_0_ltssm_enable,
 };
 
-static const struct qcom_pcie_ops ops_v1 = {
-	.get_resources = qcom_pcie_get_resources_v1,
-	.init = qcom_pcie_init_v1,
-	.deinit = qcom_pcie_deinit_v1,
-	.ltssm_enable = qcom_pcie_v0_v1_ltssm_enable,
+/* Qcom IP rev.: 1.0.0	Synopsys IP rev.: 4.11a */
+static const struct qcom_pcie_ops ops_1_0_0 = {
+	.get_resources = qcom_pcie_get_resources_1_0_0,
+	.init = qcom_pcie_init_1_0_0,
+	.deinit = qcom_pcie_deinit_1_0_0,
+	.ltssm_enable = qcom_pcie_2_1_0_ltssm_enable,
 };
 
-static const struct qcom_pcie_ops ops_v2 = {
-	.get_resources = qcom_pcie_get_resources_v2,
-	.init = qcom_pcie_init_v2,
-	.post_init = qcom_pcie_post_init_v2,
-	.deinit = qcom_pcie_deinit_v2,
-	.ltssm_enable = qcom_pcie_v2_ltssm_enable,
+/* Qcom IP rev.: 2.3.2	Synopsys IP rev.: 4.21a */
+static const struct qcom_pcie_ops ops_2_3_2 = {
+	.get_resources = qcom_pcie_get_resources_2_3_2,
+	.init = qcom_pcie_init_2_3_2,
+	.post_init = qcom_pcie_post_init_2_3_2,
+	.deinit = qcom_pcie_deinit_2_3_2,
+	.post_deinit = qcom_pcie_post_deinit_2_3_2,
+	.ltssm_enable = qcom_pcie_2_3_2_ltssm_enable,
 };
 
-static const struct dw_pcie_ops dw_pcie_ops = {
-	.link_up = qcom_pcie_link_up,
+/* Qcom IP rev.: 2.4.0	Synopsys IP rev.: 4.20a */
+static const struct qcom_pcie_ops ops_2_4_0 = {
+	.get_resources = qcom_pcie_get_resources_2_4_0,
+	.init = qcom_pcie_init_2_4_0,
+	.deinit = qcom_pcie_deinit_2_4_0,
+	.ltssm_enable = qcom_pcie_2_3_2_ltssm_enable,
 };
 
-static const struct qcom_pcie_ops ops_v3 = {
-	.get_resources = qcom_pcie_get_resources_v3,
-	.init = qcom_pcie_init_v3,
-	.deinit = qcom_pcie_deinit_v3,
-	.ltssm_enable = qcom_pcie_v2_ltssm_enable,
+/* Qcom IP rev.: 2.3.3	Synopsys IP rev.: 4.30a */
+static const struct qcom_pcie_ops ops_2_3_3 = {
+	.get_resources = qcom_pcie_get_resources_2_3_3,
+	.init = qcom_pcie_init_2_3_3,
+	.deinit = qcom_pcie_deinit_2_3_3,
+	.ltssm_enable = qcom_pcie_2_3_2_ltssm_enable,
+};
+
+static const struct dw_pcie_ops dw_pcie_ops = {
+	.link_up = qcom_pcie_link_up,
 };
 
 static int qcom_pcie_probe(struct platform_device *pdev)
@@ -1069,11 +1297,12 @@ static int qcom_pcie_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id qcom_pcie_match[] = {
-	{ .compatible = "qcom,pcie-ipq8064", .data = &ops_v0 },
-	{ .compatible = "qcom,pcie-apq8064", .data = &ops_v0 },
-	{ .compatible = "qcom,pcie-apq8084", .data = &ops_v1 },
-	{ .compatible = "qcom,pcie-msm8996", .data = &ops_v2 },
-	{ .compatible = "qcom,pcie-ipq4019", .data = &ops_v3 },
+	{ .compatible = "qcom,pcie-apq8084", .data = &ops_1_0_0 },
+	{ .compatible = "qcom,pcie-ipq8064", .data = &ops_2_1_0 },
+	{ .compatible = "qcom,pcie-apq8064", .data = &ops_2_1_0 },
+	{ .compatible = "qcom,pcie-msm8996", .data = &ops_2_3_2 },
+	{ .compatible = "qcom,pcie-ipq8074", .data = &ops_2_3_3 },
+	{ .compatible = "qcom,pcie-ipq4019", .data = &ops_2_4_0 },
 	{ }
 };
 
diff --git a/drivers/pci/dwc/pcie-spear13xx.c b/drivers/pci/dwc/pcie-spear13xx.c
index 80897291e0fb..709189d23b31 100644
--- a/drivers/pci/dwc/pcie-spear13xx.c
+++ b/drivers/pci/dwc/pcie-spear13xx.c
@@ -177,13 +177,15 @@ static int spear13xx_pcie_link_up(struct dw_pcie *pci)
 	return 0;
 }
 
-static void spear13xx_pcie_host_init(struct pcie_port *pp)
+static int spear13xx_pcie_host_init(struct pcie_port *pp)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 	struct spear13xx_pcie *spear13xx_pcie = to_spear13xx_pcie(pci);
 
 	spear13xx_pcie_establish_link(spear13xx_pcie);
 	spear13xx_pcie_enable_interrupts(spear13xx_pcie);
+
+	return 0;
 }
 
 static const struct dw_pcie_host_ops spear13xx_pcie_host_ops = {
@@ -199,9 +201,9 @@ static int spear13xx_add_pcie_port(struct spear13xx_pcie *spear13xx_pcie,
 	int ret;
 
 	pp->irq = platform_get_irq(pdev, 0);
-	if (!pp->irq) {
+	if (pp->irq < 0) {
 		dev_err(dev, "failed to get irq\n");
-		return -ENODEV;
+		return pp->irq;
 	}
 	ret = devm_request_irq(dev, pp->irq, spear13xx_pcie_irq_handler,
 			       IRQF_SHARED | IRQF_NO_THREAD,
diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c
index 53fff8030337..4ddc6e8f9fe7 100644
--- a/drivers/pci/endpoint/functions/pci-epf-test.c
+++ b/drivers/pci/endpoint/functions/pci-epf-test.c
@@ -54,6 +54,8 @@ static struct workqueue_struct *kpcitest_workqueue;
 struct pci_epf_test {
 	void			*reg[6];
 	struct pci_epf		*epf;
+	enum pci_barno		test_reg_bar;
+	bool			linkup_notifier;
 	struct delayed_work	cmd_handler;
 };
 
@@ -74,7 +76,12 @@ static struct pci_epf_header test_header = {
 	.interrupt_pin	= PCI_INTERRUPT_INTA,
 };
 
-static int bar_size[] = { 512, 1024, 16384, 131072, 1048576 };
+struct pci_epf_test_data {
+	enum pci_barno	test_reg_bar;
+	bool		linkup_notifier;
+};
+
+static int bar_size[] = { 512, 512, 1024, 16384, 131072, 1048576 };
 
 static int pci_epf_test_copy(struct pci_epf_test *epf_test)
 {
@@ -86,7 +93,8 @@ static int pci_epf_test_copy(struct pci_epf_test *epf_test)
 	struct pci_epf *epf = epf_test->epf;
 	struct device *dev = &epf->dev;
 	struct pci_epc *epc = epf->epc;
-	struct pci_epf_test_reg *reg = epf_test->reg[0];
+	enum pci_barno test_reg_bar = epf_test->test_reg_bar;
+	struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar];
 
 	src_addr = pci_epc_mem_alloc_addr(epc, &src_phys_addr, reg->size);
 	if (!src_addr) {
@@ -145,7 +153,8 @@ static int pci_epf_test_read(struct pci_epf_test *epf_test)
 	struct pci_epf *epf = epf_test->epf;
 	struct device *dev = &epf->dev;
 	struct pci_epc *epc = epf->epc;
-	struct pci_epf_test_reg *reg = epf_test->reg[0];
+	enum pci_barno test_reg_bar = epf_test->test_reg_bar;
+	struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar];
 
 	src_addr = pci_epc_mem_alloc_addr(epc, &phys_addr, reg->size);
 	if (!src_addr) {
@@ -195,7 +204,8 @@ static int pci_epf_test_write(struct pci_epf_test *epf_test)
 	struct pci_epf *epf = epf_test->epf;
 	struct device *dev = &epf->dev;
 	struct pci_epc *epc = epf->epc;
-	struct pci_epf_test_reg *reg = epf_test->reg[0];
+	enum pci_barno test_reg_bar = epf_test->test_reg_bar;
+	struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar];
 
 	dst_addr = pci_epc_mem_alloc_addr(epc, &phys_addr, reg->size);
 	if (!dst_addr) {
@@ -247,7 +257,8 @@ static void pci_epf_test_raise_irq(struct pci_epf_test *epf_test)
 	u8 msi_count;
 	struct pci_epf *epf = epf_test->epf;
 	struct pci_epc *epc = epf->epc;
-	struct pci_epf_test_reg *reg = epf_test->reg[0];
+	enum pci_barno test_reg_bar = epf_test->test_reg_bar;
+	struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar];
 
 	reg->status |= STATUS_IRQ_RAISED;
 	msi_count = pci_epc_get_msi(epc);
@@ -263,22 +274,28 @@ static void pci_epf_test_cmd_handler(struct work_struct *work)
 	int ret;
 	u8 irq;
 	u8 msi_count;
+	u32 command;
 	struct pci_epf_test *epf_test = container_of(work, struct pci_epf_test,
 						     cmd_handler.work);
 	struct pci_epf *epf = epf_test->epf;
 	struct pci_epc *epc = epf->epc;
-	struct pci_epf_test_reg *reg = epf_test->reg[0];
+	enum pci_barno test_reg_bar = epf_test->test_reg_bar;
+	struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar];
 
-	if (!reg->command)
+	command = reg->command;
+	if (!command)
 		goto reset_handler;
 
-	if (reg->command & COMMAND_RAISE_LEGACY_IRQ) {
+	reg->command = 0;
+	reg->status = 0;
+
+	if (command & COMMAND_RAISE_LEGACY_IRQ) {
 		reg->status = STATUS_IRQ_RAISED;
 		pci_epc_raise_irq(epc, PCI_EPC_IRQ_LEGACY, 0);
 		goto reset_handler;
 	}
 
-	if (reg->command & COMMAND_WRITE) {
+	if (command & COMMAND_WRITE) {
 		ret = pci_epf_test_write(epf_test);
 		if (ret)
 			reg->status |= STATUS_WRITE_FAIL;
@@ -288,7 +305,7 @@ static void pci_epf_test_cmd_handler(struct work_struct *work)
 		goto reset_handler;
 	}
 
-	if (reg->command & COMMAND_READ) {
+	if (command & COMMAND_READ) {
 		ret = pci_epf_test_read(epf_test);
 		if (!ret)
 			reg->status |= STATUS_READ_SUCCESS;
@@ -298,7 +315,7 @@ static void pci_epf_test_cmd_handler(struct work_struct *work)
 		goto reset_handler;
 	}
 
-	if (reg->command & COMMAND_COPY) {
+	if (command & COMMAND_COPY) {
 		ret = pci_epf_test_copy(epf_test);
 		if (!ret)
 			reg->status |= STATUS_COPY_SUCCESS;
@@ -308,9 +325,9 @@ static void pci_epf_test_cmd_handler(struct work_struct *work)
 		goto reset_handler;
 	}
 
-	if (reg->command & COMMAND_RAISE_MSI_IRQ) {
+	if (command & COMMAND_RAISE_MSI_IRQ) {
 		msi_count = pci_epc_get_msi(epc);
-		irq = (reg->command & MSI_NUMBER_MASK) >> MSI_NUMBER_SHIFT;
+		irq = (command & MSI_NUMBER_MASK) >> MSI_NUMBER_SHIFT;
 		if (irq > msi_count || msi_count <= 0)
 			goto reset_handler;
 		reg->status = STATUS_IRQ_RAISED;
@@ -319,8 +336,6 @@ static void pci_epf_test_cmd_handler(struct work_struct *work)
 	}
 
 reset_handler:
-	reg->command = 0;
-
 	queue_delayed_work(kpcitest_workqueue, &epf_test->cmd_handler,
 			   msecs_to_jiffies(1));
 }
@@ -358,6 +373,7 @@ static int pci_epf_test_set_bar(struct pci_epf *epf)
 	struct pci_epc *epc = epf->epc;
 	struct device *dev = &epf->dev;
 	struct pci_epf_test *epf_test = epf_get_drvdata(epf);
+	enum pci_barno test_reg_bar = epf_test->test_reg_bar;
 
 	flags = PCI_BASE_ADDRESS_SPACE_MEMORY | PCI_BASE_ADDRESS_MEM_TYPE_32;
 	if (sizeof(dma_addr_t) == 0x8)
@@ -370,7 +386,7 @@ static int pci_epf_test_set_bar(struct pci_epf *epf)
 		if (ret) {
 			pci_epf_free_space(epf, epf_test->reg[bar], bar);
 			dev_err(dev, "failed to set BAR%d\n", bar);
-			if (bar == BAR_0)
+			if (bar == test_reg_bar)
 				return ret;
 		}
 	}
@@ -384,17 +400,20 @@ static int pci_epf_test_alloc_space(struct pci_epf *epf)
 	struct device *dev = &epf->dev;
 	void *base;
 	int bar;
+	enum pci_barno test_reg_bar = epf_test->test_reg_bar;
 
 	base = pci_epf_alloc_space(epf, sizeof(struct pci_epf_test_reg),
-				   BAR_0);
+				   test_reg_bar);
 	if (!base) {
 		dev_err(dev, "failed to allocated register space\n");
 		return -ENOMEM;
 	}
-	epf_test->reg[0] = base;
+	epf_test->reg[test_reg_bar] = base;
 
-	for (bar = BAR_1; bar <= BAR_5; bar++) {
-		base = pci_epf_alloc_space(epf, bar_size[bar - 1], bar);
+	for (bar = BAR_0; bar <= BAR_5; bar++) {
+		if (bar == test_reg_bar)
+			continue;
+		base = pci_epf_alloc_space(epf, bar_size[bar], bar);
 		if (!base)
 			dev_err(dev, "failed to allocate space for BAR%d\n",
 				bar);
@@ -407,6 +426,7 @@ static int pci_epf_test_alloc_space(struct pci_epf *epf)
 static int pci_epf_test_bind(struct pci_epf *epf)
 {
 	int ret;
+	struct pci_epf_test *epf_test = epf_get_drvdata(epf);
 	struct pci_epf_header *header = epf->header;
 	struct pci_epc *epc = epf->epc;
 	struct device *dev = &epf->dev;
@@ -432,13 +452,34 @@ static int pci_epf_test_bind(struct pci_epf *epf)
 	if (ret)
 		return ret;
 
+	if (!epf_test->linkup_notifier)
+		queue_work(kpcitest_workqueue, &epf_test->cmd_handler.work);
+
 	return 0;
 }
 
+static const struct pci_epf_device_id pci_epf_test_ids[] = {
+	{
+		.name = "pci_epf_test",
+	},
+	{},
+};
+
 static int pci_epf_test_probe(struct pci_epf *epf)
 {
 	struct pci_epf_test *epf_test;
 	struct device *dev = &epf->dev;
+	const struct pci_epf_device_id *match;
+	struct pci_epf_test_data *data;
+	enum pci_barno test_reg_bar = BAR_0;
+	bool linkup_notifier = true;
+
+	match = pci_epf_match_device(pci_epf_test_ids, epf);
+	data = (struct pci_epf_test_data *)match->driver_data;
+	if (data) {
+		test_reg_bar = data->test_reg_bar;
+		linkup_notifier = data->linkup_notifier;
+	}
 
 	epf_test = devm_kzalloc(dev, sizeof(*epf_test), GFP_KERNEL);
 	if (!epf_test)
@@ -446,6 +487,8 @@ static int pci_epf_test_probe(struct pci_epf *epf)
 
 	epf->header = &test_header;
 	epf_test->epf = epf;
+	epf_test->test_reg_bar = test_reg_bar;
+	epf_test->linkup_notifier = linkup_notifier;
 
 	INIT_DELAYED_WORK(&epf_test->cmd_handler, pci_epf_test_cmd_handler);
 
@@ -453,31 +496,15 @@ static int pci_epf_test_probe(struct pci_epf *epf)
 	return 0;
 }
 
-static int pci_epf_test_remove(struct pci_epf *epf)
-{
-	struct pci_epf_test *epf_test = epf_get_drvdata(epf);
-
-	kfree(epf_test);
-	return 0;
-}
-
 static struct pci_epf_ops ops = {
 	.unbind	= pci_epf_test_unbind,
 	.bind	= pci_epf_test_bind,
 	.linkup = pci_epf_test_linkup,
 };
 
-static const struct pci_epf_device_id pci_epf_test_ids[] = {
-	{
-		.name = "pci_epf_test",
-	},
-	{},
-};
-
 static struct pci_epf_driver test_driver = {
 	.driver.name	= "pci_epf_test",
 	.probe		= pci_epf_test_probe,
-	.remove		= pci_epf_test_remove,
 	.id_table	= pci_epf_test_ids,
 	.ops		= &ops,
 	.owner		= THIS_MODULE,
diff --git a/drivers/pci/endpoint/pci-epc-core.c b/drivers/pci/endpoint/pci-epc-core.c
index caa7be10e473..42c2a1156325 100644
--- a/drivers/pci/endpoint/pci-epc-core.c
+++ b/drivers/pci/endpoint/pci-epc-core.c
@@ -21,6 +21,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/slab.h>
 #include <linux/module.h>
+#include <linux/of_device.h>
 
 #include <linux/pci-epc.h>
 #include <linux/pci-epf.h>
@@ -370,6 +371,7 @@ EXPORT_SYMBOL_GPL(pci_epc_write_header);
 int pci_epc_add_epf(struct pci_epc *epc, struct pci_epf *epf)
 {
 	unsigned long flags;
+	struct device *dev = epc->dev.parent;
 
 	if (epf->epc)
 		return -EBUSY;
@@ -381,8 +383,12 @@ int pci_epc_add_epf(struct pci_epc *epc, struct pci_epf *epf)
 		return -EINVAL;
 
 	epf->epc = epc;
-	dma_set_coherent_mask(&epf->dev, epc->dev.coherent_dma_mask);
-	epf->dev.dma_mask = epc->dev.dma_mask;
+	if (dev->of_node) {
+		of_dma_configure(&epf->dev, dev->of_node);
+	} else {
+		dma_set_coherent_mask(&epf->dev, epc->dev.coherent_dma_mask);
+		epf->dev.dma_mask = epc->dev.dma_mask;
+	}
 
 	spin_lock_irqsave(&epc->lock, flags);
 	list_add_tail(&epf->list, &epc->pci_epf);
@@ -500,6 +506,7 @@ __pci_epc_create(struct device *dev, const struct pci_epc_ops *ops,
 	dma_set_coherent_mask(&epc->dev, dev->coherent_dma_mask);
 	epc->dev.class = pci_epc_class;
 	epc->dev.dma_mask = dev->dma_mask;
+	epc->dev.parent = dev;
 	epc->ops = ops;
 
 	ret = dev_set_name(&epc->dev, "%s", dev_name(dev));
diff --git a/drivers/pci/endpoint/pci-epc-mem.c b/drivers/pci/endpoint/pci-epc-mem.c
index 3a94cc1caf22..83b7d5d3fc3e 100644
--- a/drivers/pci/endpoint/pci-epc-mem.c
+++ b/drivers/pci/endpoint/pci-epc-mem.c
@@ -24,21 +24,54 @@
 #include <linux/pci-epc.h>
 
 /**
- * pci_epc_mem_init() - initialize the pci_epc_mem structure
+ * pci_epc_mem_get_order() - determine the allocation order of a memory size
+ * @mem: address space of the endpoint controller
+ * @size: the size for which to get the order
+ *
+ * Reimplement get_order() for mem->page_size since the generic get_order
+ * always gets order with a constant PAGE_SIZE.
+ */
+static int pci_epc_mem_get_order(struct pci_epc_mem *mem, size_t size)
+{
+	int order;
+	unsigned int page_shift = ilog2(mem->page_size);
+
+	size--;
+	size >>= page_shift;
+#if BITS_PER_LONG == 32
+	order = fls(size);
+#else
+	order = fls64(size);
+#endif
+	return order;
+}
+
+/**
+ * __pci_epc_mem_init() - initialize the pci_epc_mem structure
  * @epc: the EPC device that invoked pci_epc_mem_init
  * @phys_base: the physical address of the base
  * @size: the size of the address space
+ * @page_size: size of each page
  *
  * Invoke to initialize the pci_epc_mem structure used by the
  * endpoint functions to allocate mapped PCI address.
  */
-int pci_epc_mem_init(struct pci_epc *epc, phys_addr_t phys_base, size_t size)
+int __pci_epc_mem_init(struct pci_epc *epc, phys_addr_t phys_base, size_t size,
+		       size_t page_size)
 {
 	int ret;
 	struct pci_epc_mem *mem;
 	unsigned long *bitmap;
-	int pages = size >> PAGE_SHIFT;
-	int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long);
+	unsigned int page_shift;
+	int pages;
+	int bitmap_size;
+
+	if (page_size < PAGE_SIZE)
+		page_size = PAGE_SIZE;
+
+	page_shift = ilog2(page_size);
+	pages = size >> page_shift;
+	bitmap_size = BITS_TO_LONGS(pages) * sizeof(long);
 
 	mem = kzalloc(sizeof(*mem), GFP_KERNEL);
 	if (!mem) {
@@ -54,6 +87,7 @@ int pci_epc_mem_init(struct pci_epc *epc, phys_addr_t phys_base, size_t size)
 
 	mem->bitmap = bitmap;
 	mem->phys_base = phys_base;
+	mem->page_size = page_size;
 	mem->pages = pages;
 	mem->size = size;
 
@@ -67,7 +101,7 @@ err_mem:
 err:
 return ret;
 }
-EXPORT_SYMBOL_GPL(pci_epc_mem_init);
+EXPORT_SYMBOL_GPL(__pci_epc_mem_init);
 
 /**
  * pci_epc_mem_exit() - cleanup the pci_epc_mem structure
@@ -101,13 +135,17 @@ void __iomem *pci_epc_mem_alloc_addr(struct pci_epc *epc,
 	int pageno;
 	void __iomem *virt_addr;
 	struct pci_epc_mem *mem = epc->mem;
-	int order = get_order(size);
+	unsigned int page_shift = ilog2(mem->page_size);
+	int order;
+
+	size = ALIGN(size, mem->page_size);
+	order = pci_epc_mem_get_order(mem, size);
 
 	pageno = bitmap_find_free_region(mem->bitmap, mem->pages, order);
 	if (pageno < 0)
 		return NULL;
 
-	*phys_addr = mem->phys_base + (pageno << PAGE_SHIFT);
+	*phys_addr = mem->phys_base + (pageno << page_shift);
 	virt_addr = ioremap(*phys_addr, size);
 	if (!virt_addr)
 		bitmap_release_region(mem->bitmap, pageno, order);
@@ -129,11 +167,14 @@ void pci_epc_mem_free_addr(struct pci_epc *epc, phys_addr_t phys_addr,
 			   void __iomem *virt_addr, size_t size)
 {
 	int pageno;
-	int order = get_order(size);
 	struct pci_epc_mem *mem = epc->mem;
+	unsigned int page_shift = ilog2(mem->page_size);
+	int order;
 
 	iounmap(virt_addr);
-	pageno = (phys_addr - mem->phys_base) >> PAGE_SHIFT;
+	pageno = (phys_addr - mem->phys_base) >> page_shift;
+	size = ALIGN(size, mem->page_size);
+	order = pci_epc_mem_get_order(mem, size);
 	bitmap_release_region(mem->bitmap, pageno, order);
 }
 EXPORT_SYMBOL_GPL(pci_epc_mem_free_addr);
diff --git a/drivers/pci/endpoint/pci-epf-core.c b/drivers/pci/endpoint/pci-epf-core.c
index 6877d6a5bcc9..ae1611a62808 100644
--- a/drivers/pci/endpoint/pci-epf-core.c
+++ b/drivers/pci/endpoint/pci-epf-core.c
@@ -27,7 +27,7 @@
 #include <linux/pci-ep-cfs.h>
 
 static struct bus_type pci_epf_bus_type;
-static struct device_type pci_epf_type;
+static const struct device_type pci_epf_type;
 
 /**
  * pci_epf_linkup() - Notify the function driver that EPC device has
@@ -267,6 +267,22 @@ err_ret:
 }
 EXPORT_SYMBOL_GPL(pci_epf_create);
 
+const struct pci_epf_device_id *
+pci_epf_match_device(const struct pci_epf_device_id *id, struct pci_epf *epf)
+{
+	if (!id || !epf)
+		return NULL;
+
+	while (*id->name) {
+		if (strcmp(epf->name, id->name) == 0)
+			return id;
+		id++;
+	}
+
+	return NULL;
+}
+EXPORT_SYMBOL_GPL(pci_epf_match_device);
+
 static void pci_epf_dev_release(struct device *dev)
 {
 	struct pci_epf *epf = to_pci_epf(dev);
@@ -275,7 +291,7 @@ static void pci_epf_dev_release(struct device *dev)
 	kfree(epf);
 }
 
-static struct device_type pci_epf_type = {
+static const struct device_type pci_epf_type = {
 	.release	= pci_epf_dev_release,
 };
 
@@ -317,11 +333,12 @@ static int pci_epf_device_probe(struct device *dev)
 
 static int pci_epf_device_remove(struct device *dev)
 {
-	int ret;
+	int ret = 0;
 	struct pci_epf *epf = to_pci_epf(dev);
 	struct pci_epf_driver *driver = to_pci_epf_driver(dev->driver);
 
-	ret = driver->remove(epf);
+	if (driver->remove)
+		ret = driver->remove(epf);
 	epf->driver = NULL;
 
 	return ret;
diff --git a/drivers/pci/host/Kconfig b/drivers/pci/host/Kconfig
index 89d61c2cbfaa..b868803792d8 100644
--- a/drivers/pci/host/Kconfig
+++ b/drivers/pci/host/Kconfig
@@ -71,7 +71,7 @@ config PCI_HOST_GENERIC
 
 config PCIE_XILINX
 	bool "Xilinx AXI PCIe host bridge support"
-	depends on ARCH_ZYNQ || MICROBLAZE
+	depends on ARCH_ZYNQ || MICROBLAZE || (MIPS && PCI_DRIVERS_GENERIC)
 	help
 	  Say 'Y' here if you want kernel to support the Xilinx AXI PCIe
 	  Host Bridge driver.
@@ -182,14 +182,13 @@ config PCIE_ROCKCHIP
 
 config PCIE_MEDIATEK
 	bool "MediaTek PCIe controller"
-	depends on ARM && (ARCH_MEDIATEK || COMPILE_TEST)
+	depends on (ARM || ARM64) && (ARCH_MEDIATEK || COMPILE_TEST)
 	depends on OF
 	depends on PCI
 	select PCIEPORTBUS
 	help
 	  Say Y here if you want to enable PCIe controller support on
-	  MT7623 series SoCs.  There is one single root complex with 3 root
-	  ports available.  Each port supports Gen2 lane x1.
+	  MediaTek SoCs.
 
 config PCIE_TANGO_SMP8759
 	bool "Tango SMP8759 PCIe controller (DANGEROUS)"
diff --git a/drivers/pci/host/pci-aardvark.c b/drivers/pci/host/pci-aardvark.c
index 5fb9b620ac78..89f4e3d072d7 100644
--- a/drivers/pci/host/pci-aardvark.c
+++ b/drivers/pci/host/pci-aardvark.c
@@ -191,7 +191,6 @@
 #define LINK_WAIT_USLEEP_MIN		90000
 #define LINK_WAIT_USLEEP_MAX		100000
 
-#define LEGACY_IRQ_NUM			4
 #define MSI_IRQ_NUM			32
 
 struct advk_pcie {
@@ -729,7 +728,7 @@ static int advk_pcie_init_irq_domain(struct advk_pcie *pcie)
 	irq_chip->irq_unmask = advk_pcie_irq_unmask;
 
 	pcie->irq_domain =
-		irq_domain_add_linear(pcie_intc_node, LEGACY_IRQ_NUM,
+		irq_domain_add_linear(pcie_intc_node, PCI_NUM_INTX,
 				      &advk_pcie_irq_domain_ops, pcie);
 	if (!pcie->irq_domain) {
 		dev_err(dev, "Failed to get a INTx IRQ domain\n");
@@ -786,7 +785,7 @@ static void advk_pcie_handle_int(struct advk_pcie *pcie)
 		advk_pcie_handle_msi(pcie);
 
 	/* Process legacy interrupts */
-	for (i = 0; i < LEGACY_IRQ_NUM; i++) {
+	for (i = 0; i < PCI_NUM_INTX; i++) {
 		if (!(status & PCIE_ISR0_INTX_ASSERT(i)))
 			continue;
 
diff --git a/drivers/pci/host/pci-ftpci100.c b/drivers/pci/host/pci-ftpci100.c
index 5162dffc102b..96028f01bc90 100644
--- a/drivers/pci/host/pci-ftpci100.c
+++ b/drivers/pci/host/pci-ftpci100.c
@@ -350,12 +350,12 @@ static int faraday_pci_setup_cascaded_irq(struct faraday_pci *p)
 
 	/* All PCI IRQs cascade off this one */
 	irq = of_irq_get(intc, 0);
-	if (!irq) {
+	if (irq <= 0) {
 		dev_err(p->dev, "failed to get parent IRQ\n");
-		return -EINVAL;
+		return irq ?: -EINVAL;
 	}
 
-	p->irqdomain = irq_domain_add_linear(intc, 4,
+	p->irqdomain = irq_domain_add_linear(intc, PCI_NUM_INTX,
 					     &faraday_pci_irqdomain_ops, p);
 	if (!p->irqdomain) {
 		dev_err(p->dev, "failed to create Gemini PCI IRQ domain\n");
diff --git a/drivers/pci/host/pci-hyperv.c b/drivers/pci/host/pci-hyperv.c
index 415dcc69a502..0fe3ea164ee5 100644
--- a/drivers/pci/host/pci-hyperv.c
+++ b/drivers/pci/host/pci-hyperv.c
@@ -50,6 +50,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/pci.h>
+#include <linux/delay.h>
 #include <linux/semaphore.h>
 #include <linux/irqdomain.h>
 #include <asm/irqdomain.h>
@@ -562,52 +563,6 @@ static void put_pcichild(struct hv_pci_dev *hv_pcidev,
 static void get_hvpcibus(struct hv_pcibus_device *hv_pcibus);
 static void put_hvpcibus(struct hv_pcibus_device *hv_pcibus);
 
-
-/*
- * Temporary CPU to vCPU mapping to address transitioning
- * vmbus_cpu_number_to_vp_number() being migrated to
- * hv_cpu_number_to_vp_number() in a separate patch. Once that patch
- * has been picked up in the main line, remove this code here and use
- * the official code.
- */
-static struct hv_tmpcpumap
-{
-	bool initialized;
-	u32 vp_index[NR_CPUS];
-} hv_tmpcpumap;
-
-static void hv_tmpcpumap_init_cpu(void *_unused)
-{
-	int cpu = smp_processor_id();
-	u64 vp_index;
-
-	hv_get_vp_index(vp_index);
-
-	hv_tmpcpumap.vp_index[cpu] = vp_index;
-}
-
-static void hv_tmpcpumap_init(void)
-{
-	if (hv_tmpcpumap.initialized)
-		return;
-
-	memset(hv_tmpcpumap.vp_index, -1, sizeof(hv_tmpcpumap.vp_index));
-	on_each_cpu(hv_tmpcpumap_init_cpu, NULL, true);
-	hv_tmpcpumap.initialized = true;
-}
-
-/**
- * hv_tmp_cpu_nr_to_vp_nr() - Convert Linux CPU nr to Hyper-V vCPU nr
- *
- * Remove once vmbus_cpu_number_to_vp_number() has been converted to
- * hv_cpu_number_to_vp_number() and replace callers appropriately.
- */
-static u32 hv_tmp_cpu_nr_to_vp_nr(int cpu)
-{
-	return hv_tmpcpumap.vp_index[cpu];
-}
-
-
 /**
  * devfn_to_wslot() - Convert from Linux PCI slot to Windows
  * @devfn:	The Linux representation of PCI slot
@@ -971,7 +926,7 @@ static void hv_irq_unmask(struct irq_data *data)
 		var_size = 1 + HV_VP_SET_BANK_COUNT_MAX;
 
 		for_each_cpu_and(cpu, dest, cpu_online_mask) {
-			cpu_vmbus = hv_tmp_cpu_nr_to_vp_nr(cpu);
+			cpu_vmbus = hv_cpu_number_to_vp_number(cpu);
 
 			if (cpu_vmbus >= HV_VP_SET_BANK_COUNT_MAX * 64) {
 				dev_err(&hbus->hdev->device,
@@ -986,7 +941,7 @@ static void hv_irq_unmask(struct irq_data *data)
 	} else {
 		for_each_cpu_and(cpu, dest, cpu_online_mask) {
 			params->int_target.vp_mask |=
-				(1ULL << hv_tmp_cpu_nr_to_vp_nr(cpu));
+				(1ULL << hv_cpu_number_to_vp_number(cpu));
 		}
 	}
 
@@ -1063,7 +1018,7 @@ static u32 hv_compose_msi_req_v2(
 	 */
 	cpu = cpumask_first_and(affinity, cpu_online_mask);
 	int_pkt->int_desc.processor_array[0] =
-		hv_tmp_cpu_nr_to_vp_nr(cpu);
+		hv_cpu_number_to_vp_number(cpu);
 	int_pkt->int_desc.processor_count = 1;
 
 	return sizeof(*int_pkt);
@@ -1159,7 +1114,12 @@ static void hv_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 		goto free_int_desc;
 	}
 
-	wait_for_completion(&comp.comp_pkt.host_event);
+	/*
+	 * Since this function is called with IRQ locks held, can't
+	 * do normal wait for completion; instead poll.
+	 */
+	while (!try_wait_for_completion(&comp.comp_pkt.host_event))
+		udelay(100);
 
 	if (comp.comp_pkt.completion_status < 0) {
 		dev_err(&hbus->hdev->device,
@@ -2490,8 +2450,6 @@ static int hv_pci_probe(struct hv_device *hdev,
 		return -ENOMEM;
 	hbus->state = hv_pcibus_init;
 
-	hv_tmpcpumap_init();
-
 	/*
 	 * The PCI bus "domain" is what is called "segment" in ACPI and
 	 * other specs.  Pull it from the instance ID, to get something
diff --git a/drivers/pci/host/pci-mvebu.c b/drivers/pci/host/pci-mvebu.c
index f353a6eb2f01..8d88f19dc171 100644
--- a/drivers/pci/host/pci-mvebu.c
+++ b/drivers/pci/host/pci-mvebu.c
@@ -1054,8 +1054,8 @@ static int mvebu_pcie_parse_port(struct mvebu_pcie *pcie,
 	port->pcie = pcie;
 
 	if (of_property_read_u32(child, "marvell,pcie-port", &port->port)) {
-		dev_warn(dev, "ignoring %s, missing pcie-port property\n",
-			 of_node_full_name(child));
+		dev_warn(dev, "ignoring %pOF, missing pcie-port property\n",
+			 child);
 		goto skip;
 	}
 
@@ -1106,8 +1106,8 @@ static int mvebu_pcie_parse_port(struct mvebu_pcie *pcie,
 		}
 
 		if (flags & OF_GPIO_ACTIVE_LOW) {
-			dev_info(dev, "%s: reset gpio is active low\n",
-				 of_node_full_name(child));
+			dev_info(dev, "%pOF: reset gpio is active low\n",
+				 child);
 			gpio_flags = GPIOF_ACTIVE_LOW |
 				     GPIOF_OUT_INIT_LOW;
 		} else {
@@ -1186,8 +1186,7 @@ static int mvebu_pcie_powerup(struct mvebu_pcie_port *port)
  */
 static void mvebu_pcie_powerdown(struct mvebu_pcie_port *port)
 {
-	if (port->reset_gpio)
-		gpiod_set_value_cansleep(port->reset_gpio, 1);
+	gpiod_set_value_cansleep(port->reset_gpio, 1);
 
 	clk_disable_unprepare(port->clk);
 }
diff --git a/drivers/pci/host/pci-tegra.c b/drivers/pci/host/pci-tegra.c
index b3722b7709df..9c40da54f88a 100644
--- a/drivers/pci/host/pci-tegra.c
+++ b/drivers/pci/host/pci-tegra.c
@@ -1147,15 +1147,15 @@ static int tegra_pcie_resets_get(struct tegra_pcie *pcie)
 {
 	struct device *dev = pcie->dev;
 
-	pcie->pex_rst = devm_reset_control_get(dev, "pex");
+	pcie->pex_rst = devm_reset_control_get_exclusive(dev, "pex");
 	if (IS_ERR(pcie->pex_rst))
 		return PTR_ERR(pcie->pex_rst);
 
-	pcie->afi_rst = devm_reset_control_get(dev, "afi");
+	pcie->afi_rst = devm_reset_control_get_exclusive(dev, "afi");
 	if (IS_ERR(pcie->afi_rst))
 		return PTR_ERR(pcie->afi_rst);
 
-	pcie->pcie_xrst = devm_reset_control_get(dev, "pcie_x");
+	pcie->pcie_xrst = devm_reset_control_get_exclusive(dev, "pcie_x");
 	if (IS_ERR(pcie->pcie_xrst))
 		return PTR_ERR(pcie->pcie_xrst);
 
@@ -1703,8 +1703,7 @@ static int tegra_pcie_get_legacy_regulators(struct tegra_pcie *pcie)
 		pcie->num_supplies = 2;
 
 	if (pcie->num_supplies == 0) {
-		dev_err(dev, "device %s not supported in legacy mode\n",
-			np->full_name);
+		dev_err(dev, "device %pOF not supported in legacy mode\n", np);
 		return -ENODEV;
 	}
 
diff --git a/drivers/pci/host/pci-xgene-msi.c b/drivers/pci/host/pci-xgene-msi.c
index f1b633bce525..1f42a202b021 100644
--- a/drivers/pci/host/pci-xgene-msi.c
+++ b/drivers/pci/host/pci-xgene-msi.c
@@ -489,7 +489,7 @@ static int xgene_msi_probe(struct platform_device *pdev)
 		if (virt_msir < 0) {
 			dev_err(&pdev->dev, "Cannot translate IRQ index %d\n",
 				irq_index);
-			rc = -EINVAL;
+			rc = virt_msir;
 			goto error;
 		}
 		xgene_msi->msi_groups[irq_index].gic_irq = virt_msir;
diff --git a/drivers/pci/host/pci-xgene.c b/drivers/pci/host/pci-xgene.c
index bd897479a215..087645116ecb 100644
--- a/drivers/pci/host/pci-xgene.c
+++ b/drivers/pci/host/pci-xgene.c
@@ -61,7 +61,7 @@
 #define SZ_1T				(SZ_1G*1024ULL)
 #define PIPE_PHY_RATE_RD(src)		((0xc000 & (u32)(src)) >> 0xe)
 
-#define ROOT_CAP_AND_CTRL		0x5C
+#define XGENE_V1_PCI_EXP_CAP		0x40
 
 /* PCIe IP version */
 #define XGENE_PCIE_IP_VER_UNKN		0
@@ -160,7 +160,7 @@ static bool xgene_pcie_hide_rc_bars(struct pci_bus *bus, int offset)
 }
 
 static void __iomem *xgene_pcie_map_bus(struct pci_bus *bus, unsigned int devfn,
-			      int offset)
+					int offset)
 {
 	if ((pci_is_root_bus(bus) && devfn != 0) ||
 	    xgene_pcie_hide_rc_bars(bus, offset))
@@ -189,7 +189,7 @@ static int xgene_pcie_config_read32(struct pci_bus *bus, unsigned int devfn,
 	 * Avoid this by not claiming to support CRS.
 	 */
 	if (pci_is_root_bus(bus) && (port->version == XGENE_PCIE_IP_VER_1) &&
-	    ((where & ~0x3) == ROOT_CAP_AND_CTRL))
+	    ((where & ~0x3) == XGENE_V1_PCI_EXP_CAP + PCI_EXP_RTCTL))
 		*val &= ~(PCI_EXP_RTCAP_CRSVIS << 16);
 
 	if (size <= 2)
@@ -265,12 +265,12 @@ static int xgene_v1_pcie_ecam_init(struct pci_config_window *cfg)
 }
 
 struct pci_ecam_ops xgene_v1_pcie_ecam_ops = {
-	.bus_shift      = 16,
-	.init           = xgene_v1_pcie_ecam_init,
-	.pci_ops        = {
-		.map_bus        = xgene_pcie_map_bus,
-		.read           = xgene_pcie_config_read32,
-		.write          = pci_generic_config_write,
+	.bus_shift	= 16,
+	.init		= xgene_v1_pcie_ecam_init,
+	.pci_ops	= {
+		.map_bus	= xgene_pcie_map_bus,
+		.read		= xgene_pcie_config_read32,
+		.write		= pci_generic_config_write,
 	}
 };
 
@@ -280,12 +280,12 @@ static int xgene_v2_pcie_ecam_init(struct pci_config_window *cfg)
 }
 
 struct pci_ecam_ops xgene_v2_pcie_ecam_ops = {
-	.bus_shift      = 16,
-	.init           = xgene_v2_pcie_ecam_init,
-	.pci_ops        = {
-		.map_bus        = xgene_pcie_map_bus,
-		.read           = xgene_pcie_config_read32,
-		.write          = pci_generic_config_write,
+	.bus_shift	= 16,
+	.init		= xgene_v2_pcie_ecam_init,
+	.pci_ops	= {
+		.map_bus	= xgene_pcie_map_bus,
+		.read		= xgene_pcie_config_read32,
+		.write		= pci_generic_config_write,
 	}
 };
 #endif
@@ -318,7 +318,7 @@ static u64 xgene_pcie_set_ib_mask(struct xgene_pcie_port *port, u32 addr,
 }
 
 static void xgene_pcie_linkup(struct xgene_pcie_port *port,
-				   u32 *lanes, u32 *speed)
+			      u32 *lanes, u32 *speed)
 {
 	u32 val32;
 
@@ -593,8 +593,7 @@ static void xgene_pcie_clear_config(struct xgene_pcie_port *port)
 		xgene_pcie_writel(port, i, 0);
 }
 
-static int xgene_pcie_setup(struct xgene_pcie_port *port,
-			    struct list_head *res,
+static int xgene_pcie_setup(struct xgene_pcie_port *port, struct list_head *res,
 			    resource_size_t io_base)
 {
 	struct device *dev = port->dev;
@@ -706,9 +705,9 @@ static const struct of_device_id xgene_pcie_match_table[] = {
 
 static struct platform_driver xgene_pcie_driver = {
 	.driver = {
-		   .name = "xgene-pcie",
-		   .of_match_table = of_match_ptr(xgene_pcie_match_table),
-		   .suppress_bind_attrs = true,
+		.name = "xgene-pcie",
+		.of_match_table = of_match_ptr(xgene_pcie_match_table),
+		.suppress_bind_attrs = true,
 	},
 	.probe = xgene_pcie_probe_bridge,
 };
diff --git a/drivers/pci/host/pcie-altera-msi.c b/drivers/pci/host/pcie-altera-msi.c
index 4e5d628e8cd4..d8141f4865de 100644
--- a/drivers/pci/host/pcie-altera-msi.c
+++ b/drivers/pci/host/pcie-altera-msi.c
@@ -64,13 +64,11 @@ static void altera_msi_isr(struct irq_desc *desc)
 	struct irq_chip *chip = irq_desc_get_chip(desc);
 	struct altera_msi *msi;
 	unsigned long status;
-	u32 num_of_vectors;
 	u32 bit;
 	u32 virq;
 
 	chained_irq_enter(chip, desc);
 	msi = irq_desc_get_handler_data(desc);
-	num_of_vectors = msi->num_of_vectors;
 
 	while ((status = msi_readl(msi, MSI_STATUS)) != 0) {
 		for_each_set_bit(bit, &status, msi->num_of_vectors) {
@@ -267,9 +265,9 @@ static int altera_msi_probe(struct platform_device *pdev)
 		return ret;
 
 	msi->irq = platform_get_irq(pdev, 0);
-	if (msi->irq <= 0) {
+	if (msi->irq < 0) {
 		dev_err(&pdev->dev, "failed to map IRQ: %d\n", msi->irq);
-		ret = -ENODEV;
+		ret = msi->irq;
 		goto err;
 	}
 
diff --git a/drivers/pci/host/pcie-altera.c b/drivers/pci/host/pcie-altera.c
index 4ea4f8f5dc77..b468b8cccf8d 100644
--- a/drivers/pci/host/pcie-altera.c
+++ b/drivers/pci/host/pcie-altera.c
@@ -76,8 +76,6 @@
 #define LINK_UP_TIMEOUT			HZ
 #define LINK_RETRAIN_TIMEOUT		HZ
 
-#define INTX_NUM			4
-
 #define DWORD_MASK			3
 
 struct altera_pcie {
@@ -464,6 +462,7 @@ static int altera_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
 
 static const struct irq_domain_ops intx_domain_ops = {
 	.map = altera_pcie_intx_map,
+	.xlate = pci_irqd_intx_xlate,
 };
 
 static void altera_pcie_isr(struct irq_desc *desc)
@@ -481,11 +480,11 @@ static void altera_pcie_isr(struct irq_desc *desc)
 
 	while ((status = cra_readl(pcie, P2A_INT_STATUS)
 		& P2A_INT_STS_ALL) != 0) {
-		for_each_set_bit(bit, &status, INTX_NUM) {
+		for_each_set_bit(bit, &status, PCI_NUM_INTX) {
 			/* clear interrupts */
 			cra_writel(pcie, 1 << bit, P2A_INT_STATUS);
 
-			virq = irq_find_mapping(pcie->irq_domain, bit + 1);
+			virq = irq_find_mapping(pcie->irq_domain, bit);
 			if (virq)
 				generic_handle_irq(virq);
 			else
@@ -536,7 +535,7 @@ static int altera_pcie_init_irq_domain(struct altera_pcie *pcie)
 	struct device_node *node = dev->of_node;
 
 	/* Setup INTx */
-	pcie->irq_domain = irq_domain_add_linear(node, INTX_NUM + 1,
+	pcie->irq_domain = irq_domain_add_linear(node, PCI_NUM_INTX,
 					&intx_domain_ops, pcie);
 	if (!pcie->irq_domain) {
 		dev_err(dev, "Failed to get a INTx IRQ domain\n");
@@ -559,9 +558,9 @@ static int altera_pcie_parse_dt(struct altera_pcie *pcie)
 
 	/* setup IRQ */
 	pcie->irq = platform_get_irq(pdev, 0);
-	if (pcie->irq <= 0) {
+	if (pcie->irq < 0) {
 		dev_err(dev, "failed to get IRQ: %d\n", pcie->irq);
-		return -EINVAL;
+		return pcie->irq;
 	}
 
 	irq_set_chained_handler_and_data(pcie->irq, altera_pcie_isr, pcie);
diff --git a/drivers/pci/host/pcie-iproc-msi.c b/drivers/pci/host/pcie-iproc-msi.c
index 9fad7915f82a..2d0f535a2f69 100644
--- a/drivers/pci/host/pcie-iproc-msi.c
+++ b/drivers/pci/host/pcie-iproc-msi.c
@@ -317,7 +317,6 @@ static void iproc_msi_handler(struct irq_desc *desc)
 	struct irq_chip *chip = irq_desc_get_chip(desc);
 	struct iproc_msi_grp *grp;
 	struct iproc_msi *msi;
-	struct iproc_pcie *pcie;
 	u32 eq, head, tail, nr_events;
 	unsigned long hwirq;
 	int virq;
@@ -326,7 +325,6 @@ static void iproc_msi_handler(struct irq_desc *desc)
 
 	grp = irq_desc_get_handler_data(desc);
 	msi = grp->msi;
-	pcie = msi->pcie;
 	eq = grp->eq;
 
 	/*
diff --git a/drivers/pci/host/pcie-iproc-platform.c b/drivers/pci/host/pcie-iproc-platform.c
index 22531190bc40..a5073a921a04 100644
--- a/drivers/pci/host/pcie-iproc-platform.c
+++ b/drivers/pci/host/pcie-iproc-platform.c
@@ -134,6 +134,13 @@ static int iproc_pcie_pltfm_remove(struct platform_device *pdev)
 	return iproc_pcie_remove(pcie);
 }
 
+static void iproc_pcie_pltfm_shutdown(struct platform_device *pdev)
+{
+	struct iproc_pcie *pcie = platform_get_drvdata(pdev);
+
+	iproc_pcie_shutdown(pcie);
+}
+
 static struct platform_driver iproc_pcie_pltfm_driver = {
 	.driver = {
 		.name = "iproc-pcie",
@@ -141,6 +148,7 @@ static struct platform_driver iproc_pcie_pltfm_driver = {
 	},
 	.probe = iproc_pcie_pltfm_probe,
 	.remove = iproc_pcie_pltfm_remove,
+	.shutdown = iproc_pcie_pltfm_shutdown,
 };
 module_platform_driver(iproc_pcie_pltfm_driver);
 
diff --git a/drivers/pci/host/pcie-iproc.c b/drivers/pci/host/pcie-iproc.c
index c57486348856..3a8b9d20ee57 100644
--- a/drivers/pci/host/pcie-iproc.c
+++ b/drivers/pci/host/pcie-iproc.c
@@ -31,68 +31,71 @@
 
 #include "pcie-iproc.h"
 
-#define EP_PERST_SOURCE_SELECT_SHIFT 2
-#define EP_PERST_SOURCE_SELECT       BIT(EP_PERST_SOURCE_SELECT_SHIFT)
-#define EP_MODE_SURVIVE_PERST_SHIFT  1
-#define EP_MODE_SURVIVE_PERST        BIT(EP_MODE_SURVIVE_PERST_SHIFT)
-#define RC_PCIE_RST_OUTPUT_SHIFT     0
-#define RC_PCIE_RST_OUTPUT           BIT(RC_PCIE_RST_OUTPUT_SHIFT)
-#define PAXC_RESET_MASK              0x7f
-
-#define GIC_V3_CFG_SHIFT             0
-#define GIC_V3_CFG                   BIT(GIC_V3_CFG_SHIFT)
-
-#define MSI_ENABLE_CFG_SHIFT         0
-#define MSI_ENABLE_CFG               BIT(MSI_ENABLE_CFG_SHIFT)
-
-#define CFG_IND_ADDR_MASK            0x00001ffc
-
-#define CFG_ADDR_BUS_NUM_SHIFT       20
-#define CFG_ADDR_BUS_NUM_MASK        0x0ff00000
-#define CFG_ADDR_DEV_NUM_SHIFT       15
-#define CFG_ADDR_DEV_NUM_MASK        0x000f8000
-#define CFG_ADDR_FUNC_NUM_SHIFT      12
-#define CFG_ADDR_FUNC_NUM_MASK       0x00007000
-#define CFG_ADDR_REG_NUM_SHIFT       2
-#define CFG_ADDR_REG_NUM_MASK        0x00000ffc
-#define CFG_ADDR_CFG_TYPE_SHIFT      0
-#define CFG_ADDR_CFG_TYPE_MASK       0x00000003
-
-#define SYS_RC_INTX_MASK             0xf
-
-#define PCIE_PHYLINKUP_SHIFT         3
-#define PCIE_PHYLINKUP               BIT(PCIE_PHYLINKUP_SHIFT)
-#define PCIE_DL_ACTIVE_SHIFT         2
-#define PCIE_DL_ACTIVE               BIT(PCIE_DL_ACTIVE_SHIFT)
-
-#define APB_ERR_EN_SHIFT             0
-#define APB_ERR_EN                   BIT(APB_ERR_EN_SHIFT)
+#define EP_PERST_SOURCE_SELECT_SHIFT	2
+#define EP_PERST_SOURCE_SELECT		BIT(EP_PERST_SOURCE_SELECT_SHIFT)
+#define EP_MODE_SURVIVE_PERST_SHIFT	1
+#define EP_MODE_SURVIVE_PERST		BIT(EP_MODE_SURVIVE_PERST_SHIFT)
+#define RC_PCIE_RST_OUTPUT_SHIFT	0
+#define RC_PCIE_RST_OUTPUT		BIT(RC_PCIE_RST_OUTPUT_SHIFT)
+#define PAXC_RESET_MASK			0x7f
+
+#define GIC_V3_CFG_SHIFT		0
+#define GIC_V3_CFG			BIT(GIC_V3_CFG_SHIFT)
+
+#define MSI_ENABLE_CFG_SHIFT		0
+#define MSI_ENABLE_CFG			BIT(MSI_ENABLE_CFG_SHIFT)
+
+#define CFG_IND_ADDR_MASK		0x00001ffc
+
+#define CFG_ADDR_BUS_NUM_SHIFT		20
+#define CFG_ADDR_BUS_NUM_MASK		0x0ff00000
+#define CFG_ADDR_DEV_NUM_SHIFT		15
+#define CFG_ADDR_DEV_NUM_MASK		0x000f8000
+#define CFG_ADDR_FUNC_NUM_SHIFT		12
+#define CFG_ADDR_FUNC_NUM_MASK		0x00007000
+#define CFG_ADDR_REG_NUM_SHIFT		2
+#define CFG_ADDR_REG_NUM_MASK		0x00000ffc
+#define CFG_ADDR_CFG_TYPE_SHIFT		0
+#define CFG_ADDR_CFG_TYPE_MASK		0x00000003
+
+#define SYS_RC_INTX_MASK		0xf
+
+#define PCIE_PHYLINKUP_SHIFT		3
+#define PCIE_PHYLINKUP			BIT(PCIE_PHYLINKUP_SHIFT)
+#define PCIE_DL_ACTIVE_SHIFT		2
+#define PCIE_DL_ACTIVE			BIT(PCIE_DL_ACTIVE_SHIFT)
+
+#define APB_ERR_EN_SHIFT		0
+#define APB_ERR_EN			BIT(APB_ERR_EN_SHIFT)
+
+#define CFG_RETRY_STATUS		0xffff0001
+#define CFG_RETRY_STATUS_TIMEOUT_US	500000 /* 500 milliseconds */
 
 /* derive the enum index of the outbound/inbound mapping registers */
-#define MAP_REG(base_reg, index)      ((base_reg) + (index) * 2)
+#define MAP_REG(base_reg, index)	((base_reg) + (index) * 2)
 
 /*
  * Maximum number of outbound mapping window sizes that can be supported by any
  * OARR/OMAP mapping pair
  */
-#define MAX_NUM_OB_WINDOW_SIZES      4
+#define MAX_NUM_OB_WINDOW_SIZES		4
 
-#define OARR_VALID_SHIFT             0
-#define OARR_VALID                   BIT(OARR_VALID_SHIFT)
-#define OARR_SIZE_CFG_SHIFT          1
+#define OARR_VALID_SHIFT		0
+#define OARR_VALID			BIT(OARR_VALID_SHIFT)
+#define OARR_SIZE_CFG_SHIFT		1
 
 /*
  * Maximum number of inbound mapping region sizes that can be supported by an
  * IARR
  */
-#define MAX_NUM_IB_REGION_SIZES      9
+#define MAX_NUM_IB_REGION_SIZES		9
 
-#define IMAP_VALID_SHIFT             0
-#define IMAP_VALID                   BIT(IMAP_VALID_SHIFT)
+#define IMAP_VALID_SHIFT		0
+#define IMAP_VALID			BIT(IMAP_VALID_SHIFT)
 
-#define PCI_EXP_CAP			0xac
+#define IPROC_PCI_EXP_CAP		0xac
 
-#define IPROC_PCIE_REG_INVALID 0xffff
+#define IPROC_PCIE_REG_INVALID		0xffff
 
 /**
  * iProc PCIe outbound mapping controller specific parameters
@@ -304,80 +307,80 @@ enum iproc_pcie_reg {
 
 /* iProc PCIe PAXB BCMA registers */
 static const u16 iproc_pcie_reg_paxb_bcma[] = {
-	[IPROC_PCIE_CLK_CTRL]         = 0x000,
-	[IPROC_PCIE_CFG_IND_ADDR]     = 0x120,
-	[IPROC_PCIE_CFG_IND_DATA]     = 0x124,
-	[IPROC_PCIE_CFG_ADDR]         = 0x1f8,
-	[IPROC_PCIE_CFG_DATA]         = 0x1fc,
-	[IPROC_PCIE_INTX_EN]          = 0x330,
-	[IPROC_PCIE_LINK_STATUS]      = 0xf0c,
+	[IPROC_PCIE_CLK_CTRL]		= 0x000,
+	[IPROC_PCIE_CFG_IND_ADDR]	= 0x120,
+	[IPROC_PCIE_CFG_IND_DATA]	= 0x124,
+	[IPROC_PCIE_CFG_ADDR]		= 0x1f8,
+	[IPROC_PCIE_CFG_DATA]		= 0x1fc,
+	[IPROC_PCIE_INTX_EN]		= 0x330,
+	[IPROC_PCIE_LINK_STATUS]	= 0xf0c,
 };
 
 /* iProc PCIe PAXB registers */
 static const u16 iproc_pcie_reg_paxb[] = {
-	[IPROC_PCIE_CLK_CTRL]         = 0x000,
-	[IPROC_PCIE_CFG_IND_ADDR]     = 0x120,
-	[IPROC_PCIE_CFG_IND_DATA]     = 0x124,
-	[IPROC_PCIE_CFG_ADDR]         = 0x1f8,
-	[IPROC_PCIE_CFG_DATA]         = 0x1fc,
-	[IPROC_PCIE_INTX_EN]          = 0x330,
-	[IPROC_PCIE_OARR0]            = 0xd20,
-	[IPROC_PCIE_OMAP0]            = 0xd40,
-	[IPROC_PCIE_OARR1]            = 0xd28,
-	[IPROC_PCIE_OMAP1]            = 0xd48,
-	[IPROC_PCIE_LINK_STATUS]      = 0xf0c,
-	[IPROC_PCIE_APB_ERR_EN]       = 0xf40,
+	[IPROC_PCIE_CLK_CTRL]		= 0x000,
+	[IPROC_PCIE_CFG_IND_ADDR]	= 0x120,
+	[IPROC_PCIE_CFG_IND_DATA]	= 0x124,
+	[IPROC_PCIE_CFG_ADDR]		= 0x1f8,
+	[IPROC_PCIE_CFG_DATA]		= 0x1fc,
+	[IPROC_PCIE_INTX_EN]		= 0x330,
+	[IPROC_PCIE_OARR0]		= 0xd20,
+	[IPROC_PCIE_OMAP0]		= 0xd40,
+	[IPROC_PCIE_OARR1]		= 0xd28,
+	[IPROC_PCIE_OMAP1]		= 0xd48,
+	[IPROC_PCIE_LINK_STATUS]	= 0xf0c,
+	[IPROC_PCIE_APB_ERR_EN]		= 0xf40,
 };
 
 /* iProc PCIe PAXB v2 registers */
 static const u16 iproc_pcie_reg_paxb_v2[] = {
-	[IPROC_PCIE_CLK_CTRL]         = 0x000,
-	[IPROC_PCIE_CFG_IND_ADDR]     = 0x120,
-	[IPROC_PCIE_CFG_IND_DATA]     = 0x124,
-	[IPROC_PCIE_CFG_ADDR]         = 0x1f8,
-	[IPROC_PCIE_CFG_DATA]         = 0x1fc,
-	[IPROC_PCIE_INTX_EN]          = 0x330,
-	[IPROC_PCIE_OARR0]            = 0xd20,
-	[IPROC_PCIE_OMAP0]            = 0xd40,
-	[IPROC_PCIE_OARR1]            = 0xd28,
-	[IPROC_PCIE_OMAP1]            = 0xd48,
-	[IPROC_PCIE_OARR2]            = 0xd60,
-	[IPROC_PCIE_OMAP2]            = 0xd68,
-	[IPROC_PCIE_OARR3]            = 0xdf0,
-	[IPROC_PCIE_OMAP3]            = 0xdf8,
-	[IPROC_PCIE_IARR0]            = 0xd00,
-	[IPROC_PCIE_IMAP0]            = 0xc00,
-	[IPROC_PCIE_IARR2]            = 0xd10,
-	[IPROC_PCIE_IMAP2]            = 0xcc0,
-	[IPROC_PCIE_IARR3]            = 0xe00,
-	[IPROC_PCIE_IMAP3]            = 0xe08,
-	[IPROC_PCIE_IARR4]            = 0xe68,
-	[IPROC_PCIE_IMAP4]            = 0xe70,
-	[IPROC_PCIE_LINK_STATUS]      = 0xf0c,
-	[IPROC_PCIE_APB_ERR_EN]       = 0xf40,
+	[IPROC_PCIE_CLK_CTRL]		= 0x000,
+	[IPROC_PCIE_CFG_IND_ADDR]	= 0x120,
+	[IPROC_PCIE_CFG_IND_DATA]	= 0x124,
+	[IPROC_PCIE_CFG_ADDR]		= 0x1f8,
+	[IPROC_PCIE_CFG_DATA]		= 0x1fc,
+	[IPROC_PCIE_INTX_EN]		= 0x330,
+	[IPROC_PCIE_OARR0]		= 0xd20,
+	[IPROC_PCIE_OMAP0]		= 0xd40,
+	[IPROC_PCIE_OARR1]		= 0xd28,
+	[IPROC_PCIE_OMAP1]		= 0xd48,
+	[IPROC_PCIE_OARR2]		= 0xd60,
+	[IPROC_PCIE_OMAP2]		= 0xd68,
+	[IPROC_PCIE_OARR3]		= 0xdf0,
+	[IPROC_PCIE_OMAP3]		= 0xdf8,
+	[IPROC_PCIE_IARR0]		= 0xd00,
+	[IPROC_PCIE_IMAP0]		= 0xc00,
+	[IPROC_PCIE_IARR2]		= 0xd10,
+	[IPROC_PCIE_IMAP2]		= 0xcc0,
+	[IPROC_PCIE_IARR3]		= 0xe00,
+	[IPROC_PCIE_IMAP3]		= 0xe08,
+	[IPROC_PCIE_IARR4]		= 0xe68,
+	[IPROC_PCIE_IMAP4]		= 0xe70,
+	[IPROC_PCIE_LINK_STATUS]	= 0xf0c,
+	[IPROC_PCIE_APB_ERR_EN]		= 0xf40,
 };
 
 /* iProc PCIe PAXC v1 registers */
 static const u16 iproc_pcie_reg_paxc[] = {
-	[IPROC_PCIE_CLK_CTRL]         = 0x000,
-	[IPROC_PCIE_CFG_IND_ADDR]     = 0x1f0,
-	[IPROC_PCIE_CFG_IND_DATA]     = 0x1f4,
-	[IPROC_PCIE_CFG_ADDR]         = 0x1f8,
-	[IPROC_PCIE_CFG_DATA]         = 0x1fc,
+	[IPROC_PCIE_CLK_CTRL]		= 0x000,
+	[IPROC_PCIE_CFG_IND_ADDR]	= 0x1f0,
+	[IPROC_PCIE_CFG_IND_DATA]	= 0x1f4,
+	[IPROC_PCIE_CFG_ADDR]		= 0x1f8,
+	[IPROC_PCIE_CFG_DATA]		= 0x1fc,
 };
 
 /* iProc PCIe PAXC v2 registers */
 static const u16 iproc_pcie_reg_paxc_v2[] = {
-	[IPROC_PCIE_MSI_GIC_MODE]     = 0x050,
-	[IPROC_PCIE_MSI_BASE_ADDR]    = 0x074,
-	[IPROC_PCIE_MSI_WINDOW_SIZE]  = 0x078,
-	[IPROC_PCIE_MSI_ADDR_LO]      = 0x07c,
-	[IPROC_PCIE_MSI_ADDR_HI]      = 0x080,
-	[IPROC_PCIE_MSI_EN_CFG]       = 0x09c,
-	[IPROC_PCIE_CFG_IND_ADDR]     = 0x1f0,
-	[IPROC_PCIE_CFG_IND_DATA]     = 0x1f4,
-	[IPROC_PCIE_CFG_ADDR]         = 0x1f8,
-	[IPROC_PCIE_CFG_DATA]         = 0x1fc,
+	[IPROC_PCIE_MSI_GIC_MODE]	= 0x050,
+	[IPROC_PCIE_MSI_BASE_ADDR]	= 0x074,
+	[IPROC_PCIE_MSI_WINDOW_SIZE]	= 0x078,
+	[IPROC_PCIE_MSI_ADDR_LO]	= 0x07c,
+	[IPROC_PCIE_MSI_ADDR_HI]	= 0x080,
+	[IPROC_PCIE_MSI_EN_CFG]		= 0x09c,
+	[IPROC_PCIE_CFG_IND_ADDR]	= 0x1f0,
+	[IPROC_PCIE_CFG_IND_DATA]	= 0x1f4,
+	[IPROC_PCIE_CFG_ADDR]		= 0x1f8,
+	[IPROC_PCIE_CFG_DATA]		= 0x1fc,
 };
 
 static inline struct iproc_pcie *iproc_data(struct pci_bus *bus)
@@ -448,18 +451,112 @@ static inline void iproc_pcie_apb_err_disable(struct pci_bus *bus,
 	}
 }
 
+static void __iomem *iproc_pcie_map_ep_cfg_reg(struct iproc_pcie *pcie,
+					       unsigned int busno,
+					       unsigned int slot,
+					       unsigned int fn,
+					       int where)
+{
+	u16 offset;
+	u32 val;
+
+	/* EP device access */
+	val = (busno << CFG_ADDR_BUS_NUM_SHIFT) |
+		(slot << CFG_ADDR_DEV_NUM_SHIFT) |
+		(fn << CFG_ADDR_FUNC_NUM_SHIFT) |
+		(where & CFG_ADDR_REG_NUM_MASK) |
+		(1 & CFG_ADDR_CFG_TYPE_MASK);
+
+	iproc_pcie_write_reg(pcie, IPROC_PCIE_CFG_ADDR, val);
+	offset = iproc_pcie_reg_offset(pcie, IPROC_PCIE_CFG_DATA);
+
+	if (iproc_pcie_reg_is_invalid(offset))
+		return NULL;
+
+	return (pcie->base + offset);
+}
+
+static unsigned int iproc_pcie_cfg_retry(void __iomem *cfg_data_p)
+{
+	int timeout = CFG_RETRY_STATUS_TIMEOUT_US;
+	unsigned int data;
+
+	/*
+	 * As per PCIe spec r3.1, sec 2.3.2, CRS Software Visibility only
+	 * affects config reads of the Vendor ID.  For config writes or any
+	 * other config reads, the Root may automatically reissue the
+	 * configuration request again as a new request.
+	 *
+	 * For config reads, this hardware returns CFG_RETRY_STATUS data
+	 * when it receives a CRS completion, regardless of the address of
+	 * the read or the CRS Software Visibility Enable bit.  As a
+	 * partial workaround for this, we retry in software any read that
+	 * returns CFG_RETRY_STATUS.
+	 *
+	 * Note that a non-Vendor ID config register may have a value of
+	 * CFG_RETRY_STATUS.  If we read that, we can't distinguish it from
+	 * a CRS completion, so we will incorrectly retry the read and
+	 * eventually return the wrong data (0xffffffff).
+	 */
+	data = readl(cfg_data_p);
+	while (data == CFG_RETRY_STATUS && timeout--) {
+		udelay(1);
+		data = readl(cfg_data_p);
+	}
+
+	if (data == CFG_RETRY_STATUS)
+		data = 0xffffffff;
+
+	return data;
+}
+
+static int iproc_pcie_config_read(struct pci_bus *bus, unsigned int devfn,
+				  int where, int size, u32 *val)
+{
+	struct iproc_pcie *pcie = iproc_data(bus);
+	unsigned int slot = PCI_SLOT(devfn);
+	unsigned int fn = PCI_FUNC(devfn);
+	unsigned int busno = bus->number;
+	void __iomem *cfg_data_p;
+	unsigned int data;
+	int ret;
+
+	/* root complex access */
+	if (busno == 0) {
+		ret = pci_generic_config_read32(bus, devfn, where, size, val);
+		if (ret != PCIBIOS_SUCCESSFUL)
+			return ret;
+
+		/* Don't advertise CRS SV support */
+		if ((where & ~0x3) == IPROC_PCI_EXP_CAP + PCI_EXP_RTCTL)
+			*val &= ~(PCI_EXP_RTCAP_CRSVIS << 16);
+		return PCIBIOS_SUCCESSFUL;
+	}
+
+	cfg_data_p = iproc_pcie_map_ep_cfg_reg(pcie, busno, slot, fn, where);
+
+	if (!cfg_data_p)
+		return PCIBIOS_DEVICE_NOT_FOUND;
+
+	data = iproc_pcie_cfg_retry(cfg_data_p);
+
+	*val = data;
+	if (size <= 2)
+		*val = (data >> (8 * (where & 3))) & ((1 << (size * 8)) - 1);
+
+	return PCIBIOS_SUCCESSFUL;
+}
+
 /**
  * Note access to the configuration registers are protected at the higher layer
  * by 'pci_lock' in drivers/pci/access.c
  */
 static void __iomem *iproc_pcie_map_cfg_bus(struct iproc_pcie *pcie,
-					    int busno,
-					    unsigned int devfn,
+					    int busno, unsigned int devfn,
 					    int where)
 {
 	unsigned slot = PCI_SLOT(devfn);
 	unsigned fn = PCI_FUNC(devfn);
-	u32 val;
 	u16 offset;
 
 	/* root complex access */
@@ -484,18 +581,7 @@ static void __iomem *iproc_pcie_map_cfg_bus(struct iproc_pcie *pcie,
 		if (slot > 0)
 			return NULL;
 
-	/* EP device access */
-	val = (busno << CFG_ADDR_BUS_NUM_SHIFT) |
-		(slot << CFG_ADDR_DEV_NUM_SHIFT) |
-		(fn << CFG_ADDR_FUNC_NUM_SHIFT) |
-		(where & CFG_ADDR_REG_NUM_MASK) |
-		(1 & CFG_ADDR_CFG_TYPE_MASK);
-	iproc_pcie_write_reg(pcie, IPROC_PCIE_CFG_ADDR, val);
-	offset = iproc_pcie_reg_offset(pcie, IPROC_PCIE_CFG_DATA);
-	if (iproc_pcie_reg_is_invalid(offset))
-		return NULL;
-	else
-		return (pcie->base + offset);
+	return iproc_pcie_map_ep_cfg_reg(pcie, busno, slot, fn, where);
 }
 
 static void __iomem *iproc_pcie_bus_map_cfg_bus(struct pci_bus *bus,
@@ -554,9 +640,13 @@ static int iproc_pcie_config_read32(struct pci_bus *bus, unsigned int devfn,
 				    int where, int size, u32 *val)
 {
 	int ret;
+	struct iproc_pcie *pcie = iproc_data(bus);
 
 	iproc_pcie_apb_err_disable(bus, true);
-	ret = pci_generic_config_read32(bus, devfn, where, size, val);
+	if (pcie->type == IPROC_PCIE_PAXB_V2)
+		ret = iproc_pcie_config_read(bus, devfn, where, size, val);
+	else
+		ret = pci_generic_config_read32(bus, devfn, where, size, val);
 	iproc_pcie_apb_err_disable(bus, false);
 
 	return ret;
@@ -580,7 +670,7 @@ static struct pci_ops iproc_pcie_ops = {
 	.write = iproc_pcie_config_write32,
 };
 
-static void iproc_pcie_reset(struct iproc_pcie *pcie)
+static void iproc_pcie_perst_ctrl(struct iproc_pcie *pcie, bool assert)
 {
 	u32 val;
 
@@ -592,26 +682,33 @@ static void iproc_pcie_reset(struct iproc_pcie *pcie)
 	if (pcie->ep_is_internal)
 		return;
 
-	/*
-	 * Select perst_b signal as reset source. Put the device into reset,
-	 * and then bring it out of reset
-	 */
-	val = iproc_pcie_read_reg(pcie, IPROC_PCIE_CLK_CTRL);
-	val &= ~EP_PERST_SOURCE_SELECT & ~EP_MODE_SURVIVE_PERST &
-		~RC_PCIE_RST_OUTPUT;
-	iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val);
-	udelay(250);
-
-	val |= RC_PCIE_RST_OUTPUT;
-	iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val);
-	msleep(100);
+	if (assert) {
+		val = iproc_pcie_read_reg(pcie, IPROC_PCIE_CLK_CTRL);
+		val &= ~EP_PERST_SOURCE_SELECT & ~EP_MODE_SURVIVE_PERST &
+			~RC_PCIE_RST_OUTPUT;
+		iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val);
+		udelay(250);
+	} else {
+		val = iproc_pcie_read_reg(pcie, IPROC_PCIE_CLK_CTRL);
+		val |= RC_PCIE_RST_OUTPUT;
+		iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val);
+		msleep(100);
+	}
+}
+
+int iproc_pcie_shutdown(struct iproc_pcie *pcie)
+{
+	iproc_pcie_perst_ctrl(pcie, true);
+	msleep(500);
+
+	return 0;
 }
+EXPORT_SYMBOL_GPL(iproc_pcie_shutdown);
 
 static int iproc_pcie_check_link(struct iproc_pcie *pcie)
 {
 	struct device *dev = pcie->dev;
 	u32 hdr_type, link_ctrl, link_status, class, val;
-	u16 pos = PCI_EXP_CAP;
 	bool link_is_active = false;
 
 	/*
@@ -628,16 +725,16 @@ static int iproc_pcie_check_link(struct iproc_pcie *pcie)
 	}
 
 	/* make sure we are not in EP mode */
-	iproc_pci_raw_config_read32(pcie,  0, PCI_HEADER_TYPE, 1, &hdr_type);
+	iproc_pci_raw_config_read32(pcie, 0, PCI_HEADER_TYPE, 1, &hdr_type);
 	if ((hdr_type & 0x7f) != PCI_HEADER_TYPE_BRIDGE) {
 		dev_err(dev, "in EP mode, hdr=%#02x\n", hdr_type);
 		return -EFAULT;
 	}
 
 	/* force class to PCI_CLASS_BRIDGE_PCI (0x0604) */
-#define PCI_BRIDGE_CTRL_REG_OFFSET 0x43c
-#define PCI_CLASS_BRIDGE_MASK      0xffff00
-#define PCI_CLASS_BRIDGE_SHIFT     8
+#define PCI_BRIDGE_CTRL_REG_OFFSET	0x43c
+#define PCI_CLASS_BRIDGE_MASK		0xffff00
+#define PCI_CLASS_BRIDGE_SHIFT		8
 	iproc_pci_raw_config_read32(pcie, 0, PCI_BRIDGE_CTRL_REG_OFFSET,
 				    4, &class);
 	class &= ~PCI_CLASS_BRIDGE_MASK;
@@ -646,31 +743,31 @@ static int iproc_pcie_check_link(struct iproc_pcie *pcie)
 				     4, class);
 
 	/* check link status to see if link is active */
-	iproc_pci_raw_config_read32(pcie, 0, pos + PCI_EXP_LNKSTA,
+	iproc_pci_raw_config_read32(pcie, 0, IPROC_PCI_EXP_CAP + PCI_EXP_LNKSTA,
 				    2, &link_status);
 	if (link_status & PCI_EXP_LNKSTA_NLW)
 		link_is_active = true;
 
 	if (!link_is_active) {
 		/* try GEN 1 link speed */
-#define PCI_TARGET_LINK_SPEED_MASK    0xf
-#define PCI_TARGET_LINK_SPEED_GEN2    0x2
-#define PCI_TARGET_LINK_SPEED_GEN1    0x1
+#define PCI_TARGET_LINK_SPEED_MASK	0xf
+#define PCI_TARGET_LINK_SPEED_GEN2	0x2
+#define PCI_TARGET_LINK_SPEED_GEN1	0x1
 		iproc_pci_raw_config_read32(pcie, 0,
-					  pos + PCI_EXP_LNKCTL2, 4,
-					  &link_ctrl);
+					    IPROC_PCI_EXP_CAP + PCI_EXP_LNKCTL2,
+					    4, &link_ctrl);
 		if ((link_ctrl & PCI_TARGET_LINK_SPEED_MASK) ==
 		    PCI_TARGET_LINK_SPEED_GEN2) {
 			link_ctrl &= ~PCI_TARGET_LINK_SPEED_MASK;
 			link_ctrl |= PCI_TARGET_LINK_SPEED_GEN1;
 			iproc_pci_raw_config_write32(pcie, 0,
-						pos + PCI_EXP_LNKCTL2,
-						4, link_ctrl);
+					IPROC_PCI_EXP_CAP + PCI_EXP_LNKCTL2,
+					4, link_ctrl);
 			msleep(100);
 
 			iproc_pci_raw_config_read32(pcie, 0,
-						pos + PCI_EXP_LNKSTA,
-						2, &link_status);
+					IPROC_PCI_EXP_CAP + PCI_EXP_LNKSTA,
+					2, &link_status);
 			if (link_status & PCI_EXP_LNKSTA_NLW)
 				link_is_active = true;
 		}
@@ -1223,6 +1320,8 @@ static int iproc_pcie_rev_init(struct iproc_pcie *pcie)
 		pcie->ib.nr_regions = ARRAY_SIZE(paxb_v2_ib_map);
 		pcie->ib_map = paxb_v2_ib_map;
 		pcie->need_msi_steer = true;
+		dev_warn(dev, "reads of config registers that contain %#x return incorrect data\n",
+			 CFG_RETRY_STATUS);
 		break;
 	case IPROC_PCIE_PAXC:
 		regs = iproc_pcie_reg_paxc;
@@ -1286,7 +1385,8 @@ int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res)
 		goto err_exit_phy;
 	}
 
-	iproc_pcie_reset(pcie);
+	iproc_pcie_perst_ctrl(pcie, true);
+	iproc_pcie_perst_ctrl(pcie, false);
 
 	if (pcie->need_ob_cfg) {
 		ret = iproc_pcie_map_ranges(pcie, res);
diff --git a/drivers/pci/host/pcie-iproc.h b/drivers/pci/host/pcie-iproc.h
index 0bbe2ea44f3e..a6b55cec9a66 100644
--- a/drivers/pci/host/pcie-iproc.h
+++ b/drivers/pci/host/pcie-iproc.h
@@ -110,6 +110,7 @@ struct iproc_pcie {
 
 int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res);
 int iproc_pcie_remove(struct iproc_pcie *pcie);
+int iproc_pcie_shutdown(struct iproc_pcie *pcie);
 
 #ifdef CONFIG_PCIE_IPROC_MSI
 int iproc_msi_init(struct iproc_pcie *pcie, struct device_node *node);
diff --git a/drivers/pci/host/pcie-mediatek.c b/drivers/pci/host/pcie-mediatek.c
index 5a9d8589ea0b..db93efdf1d63 100644
--- a/drivers/pci/host/pcie-mediatek.c
+++ b/drivers/pci/host/pcie-mediatek.c
@@ -3,6 +3,7 @@
  *
  * Copyright (c) 2017 MediaTek Inc.
  * Author: Ryder Lee <ryder.lee@mediatek.com>
+ *	   Honghui Zhang <honghui.zhang@mediatek.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -16,6 +17,9 @@
 
 #include <linux/clk.h>
 #include <linux/delay.h>
+#include <linux/iopoll.h>
+#include <linux/irq.h>
+#include <linux/irqdomain.h>
 #include <linux/kernel.h>
 #include <linux/of_address.h>
 #include <linux/of_pci.h>
@@ -63,16 +67,104 @@
 #define PCIE_FC_CREDIT_MASK	(GENMASK(31, 31) | GENMASK(28, 16))
 #define PCIE_FC_CREDIT_VAL(x)	((x) << 16)
 
+/* PCIe V2 share registers */
+#define PCIE_SYS_CFG_V2		0x0
+#define PCIE_CSR_LTSSM_EN(x)	BIT(0 + (x) * 8)
+#define PCIE_CSR_ASPM_L1_EN(x)	BIT(1 + (x) * 8)
+
+/* PCIe V2 per-port registers */
+#define PCIE_MSI_VECTOR		0x0c0
+#define PCIE_INT_MASK		0x420
+#define INTX_MASK		GENMASK(19, 16)
+#define INTX_SHIFT		16
+#define PCIE_INT_STATUS		0x424
+#define MSI_STATUS		BIT(23)
+#define PCIE_IMSI_STATUS	0x42c
+#define PCIE_IMSI_ADDR		0x430
+#define MSI_MASK		BIT(23)
+#define MTK_MSI_IRQS_NUM	32
+
+#define PCIE_AHB_TRANS_BASE0_L	0x438
+#define PCIE_AHB_TRANS_BASE0_H	0x43c
+#define AHB2PCIE_SIZE(x)	((x) & GENMASK(4, 0))
+#define PCIE_AXI_WINDOW0	0x448
+#define WIN_ENABLE		BIT(7)
+
+/* PCIe V2 configuration transaction header */
+#define PCIE_CFG_HEADER0	0x460
+#define PCIE_CFG_HEADER1	0x464
+#define PCIE_CFG_HEADER2	0x468
+#define PCIE_CFG_WDATA		0x470
+#define PCIE_APP_TLP_REQ	0x488
+#define PCIE_CFG_RDATA		0x48c
+#define APP_CFG_REQ		BIT(0)
+#define APP_CPL_STATUS		GENMASK(7, 5)
+
+#define CFG_WRRD_TYPE_0		4
+#define CFG_WR_FMT		2
+#define CFG_RD_FMT		0
+
+#define CFG_DW0_LENGTH(length)	((length) & GENMASK(9, 0))
+#define CFG_DW0_TYPE(type)	(((type) << 24) & GENMASK(28, 24))
+#define CFG_DW0_FMT(fmt)	(((fmt) << 29) & GENMASK(31, 29))
+#define CFG_DW2_REGN(regn)	((regn) & GENMASK(11, 2))
+#define CFG_DW2_FUN(fun)	(((fun) << 16) & GENMASK(18, 16))
+#define CFG_DW2_DEV(dev)	(((dev) << 19) & GENMASK(23, 19))
+#define CFG_DW2_BUS(bus)	(((bus) << 24) & GENMASK(31, 24))
+#define CFG_HEADER_DW0(type, fmt) \
+	(CFG_DW0_LENGTH(1) | CFG_DW0_TYPE(type) | CFG_DW0_FMT(fmt))
+#define CFG_HEADER_DW1(where, size) \
+	(GENMASK(((size) - 1), 0) << ((where) & 0x3))
+#define CFG_HEADER_DW2(regn, fun, dev, bus) \
+	(CFG_DW2_REGN(regn) | CFG_DW2_FUN(fun) | \
+	CFG_DW2_DEV(dev) | CFG_DW2_BUS(bus))
+
+#define PCIE_RST_CTRL		0x510
+#define PCIE_PHY_RSTB		BIT(0)
+#define PCIE_PIPE_SRSTB		BIT(1)
+#define PCIE_MAC_SRSTB		BIT(2)
+#define PCIE_CRSTB		BIT(3)
+#define PCIE_PERSTB		BIT(8)
+#define PCIE_LINKDOWN_RST_EN	GENMASK(15, 13)
+#define PCIE_LINK_STATUS_V2	0x804
+#define PCIE_PORT_LINKUP_V2	BIT(10)
+
+struct mtk_pcie_port;
+
+/**
+ * struct mtk_pcie_soc - differentiate between host generations
+ * @has_msi: whether this host supports MSI interrupts or not
+ * @ops: pointer to configuration access functions
+ * @startup: pointer to controller setting functions
+ * @setup_irq: pointer to initialize IRQ functions
+ */
+struct mtk_pcie_soc {
+	bool has_msi;
+	struct pci_ops *ops;
+	int (*startup)(struct mtk_pcie_port *port);
+	int (*setup_irq)(struct mtk_pcie_port *port, struct device_node *node);
+};
+
 /**
  * struct mtk_pcie_port - PCIe port information
  * @base: IO mapped register base
  * @list: port list
  * @pcie: pointer to PCIe host info
  * @reset: pointer to port reset control
- * @sys_ck: pointer to bus clock
- * @phy: pointer to phy control block
+ * @sys_ck: pointer to transaction/data link layer clock
+ * @ahb_ck: pointer to AHB slave interface operating clock for CSR access
+ *          and RC initiated MMIO access
+ * @axi_ck: pointer to application layer MMIO channel operating clock
+ * @aux_ck: pointer to pe2_mac_bridge and pe2_mac_core operating clock
+ *          when pcie_mac_ck/pcie_pipe_ck is turned off
+ * @obff_ck: pointer to OBFF functional block operating clock
+ * @pipe_ck: pointer to LTSSM and PHY/MAC layer operating clock
+ * @phy: pointer to PHY control block
  * @lane: lane count
- * @index: port index
+ * @slot: port slot
+ * @irq_domain: legacy INTx IRQ domain
+ * @msi_domain: MSI IRQ domain
+ * @msi_irq_in_use: bit map for assigned MSI IRQ
  */
 struct mtk_pcie_port {
 	void __iomem *base;
@@ -80,9 +172,17 @@ struct mtk_pcie_port {
 	struct mtk_pcie *pcie;
 	struct reset_control *reset;
 	struct clk *sys_ck;
+	struct clk *ahb_ck;
+	struct clk *axi_ck;
+	struct clk *aux_ck;
+	struct clk *obff_ck;
+	struct clk *pipe_ck;
 	struct phy *phy;
 	u32 lane;
-	u32 index;
+	u32 slot;
+	struct irq_domain *irq_domain;
+	struct irq_domain *msi_domain;
+	DECLARE_BITMAP(msi_irq_in_use, MTK_MSI_IRQS_NUM);
 };
 
 /**
@@ -96,6 +196,7 @@ struct mtk_pcie_port {
  * @busn: bus range
  * @offset: IO / Memory offset
  * @ports: pointer to PCIe port information
+ * @soc: pointer to SoC-dependent operations
  */
 struct mtk_pcie {
 	struct device *dev;
@@ -111,13 +212,9 @@ struct mtk_pcie {
 		resource_size_t io;
 	} offset;
 	struct list_head ports;
+	const struct mtk_pcie_soc *soc;
 };
 
-static inline bool mtk_pcie_link_up(struct mtk_pcie_port *port)
-{
-	return !!(readl(port->base + PCIE_LINK_STATUS) & PCIE_PORT_LINKUP);
-}
-
 static void mtk_pcie_subsys_powerdown(struct mtk_pcie *pcie)
 {
 	struct device *dev = pcie->dev;
@@ -146,6 +243,12 @@ static void mtk_pcie_put_resources(struct mtk_pcie *pcie)
 
 	list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
 		phy_power_off(port->phy);
+		phy_exit(port->phy);
+		clk_disable_unprepare(port->pipe_ck);
+		clk_disable_unprepare(port->obff_ck);
+		clk_disable_unprepare(port->axi_ck);
+		clk_disable_unprepare(port->aux_ck);
+		clk_disable_unprepare(port->ahb_ck);
 		clk_disable_unprepare(port->sys_ck);
 		mtk_pcie_port_free(port);
 	}
@@ -153,11 +256,412 @@ static void mtk_pcie_put_resources(struct mtk_pcie *pcie)
 	mtk_pcie_subsys_powerdown(pcie);
 }
 
+static int mtk_pcie_check_cfg_cpld(struct mtk_pcie_port *port)
+{
+	u32 val;
+	int err;
+
+	err = readl_poll_timeout_atomic(port->base + PCIE_APP_TLP_REQ, val,
+					!(val & APP_CFG_REQ), 10,
+					100 * USEC_PER_MSEC);
+	if (err)
+		return PCIBIOS_SET_FAILED;
+
+	if (readl(port->base + PCIE_APP_TLP_REQ) & APP_CPL_STATUS)
+		return PCIBIOS_SET_FAILED;
+
+	return PCIBIOS_SUCCESSFUL;
+}
+
+static int mtk_pcie_hw_rd_cfg(struct mtk_pcie_port *port, u32 bus, u32 devfn,
+			      int where, int size, u32 *val)
+{
+	u32 tmp;
+
+	/* Write PCIe configuration transaction header for Cfgrd */
+	writel(CFG_HEADER_DW0(CFG_WRRD_TYPE_0, CFG_RD_FMT),
+	       port->base + PCIE_CFG_HEADER0);
+	writel(CFG_HEADER_DW1(where, size), port->base + PCIE_CFG_HEADER1);
+	writel(CFG_HEADER_DW2(where, PCI_FUNC(devfn), PCI_SLOT(devfn), bus),
+	       port->base + PCIE_CFG_HEADER2);
+
+	/* Trigger h/w to transmit Cfgrd TLP */
+	tmp = readl(port->base + PCIE_APP_TLP_REQ);
+	tmp |= APP_CFG_REQ;
+	writel(tmp, port->base + PCIE_APP_TLP_REQ);
+
+	/* Check completion status */
+	if (mtk_pcie_check_cfg_cpld(port))
+		return PCIBIOS_SET_FAILED;
+
+	/* Read cpld payload of Cfgrd */
+	*val = readl(port->base + PCIE_CFG_RDATA);
+
+	if (size == 1)
+		*val = (*val >> (8 * (where & 3))) & 0xff;
+	else if (size == 2)
+		*val = (*val >> (8 * (where & 3))) & 0xffff;
+
+	return PCIBIOS_SUCCESSFUL;
+}
+
+static int mtk_pcie_hw_wr_cfg(struct mtk_pcie_port *port, u32 bus, u32 devfn,
+			      int where, int size, u32 val)
+{
+	/* Write PCIe configuration transaction header for Cfgwr */
+	writel(CFG_HEADER_DW0(CFG_WRRD_TYPE_0, CFG_WR_FMT),
+	       port->base + PCIE_CFG_HEADER0);
+	writel(CFG_HEADER_DW1(where, size), port->base + PCIE_CFG_HEADER1);
+	writel(CFG_HEADER_DW2(where, PCI_FUNC(devfn), PCI_SLOT(devfn), bus),
+	       port->base + PCIE_CFG_HEADER2);
+
+	/* Write Cfgwr data */
+	val = val << 8 * (where & 3);
+	writel(val, port->base + PCIE_CFG_WDATA);
+
+	/* Trigger h/w to transmit Cfgwr TLP */
+	val = readl(port->base + PCIE_APP_TLP_REQ);
+	val |= APP_CFG_REQ;
+	writel(val, port->base + PCIE_APP_TLP_REQ);
+
+	/* Check completion status */
+	return mtk_pcie_check_cfg_cpld(port);
+}
+
+static struct mtk_pcie_port *mtk_pcie_find_port(struct pci_bus *bus,
+						unsigned int devfn)
+{
+	struct mtk_pcie *pcie = bus->sysdata;
+	struct mtk_pcie_port *port;
+
+	list_for_each_entry(port, &pcie->ports, list)
+		if (port->slot == PCI_SLOT(devfn))
+			return port;
+
+	return NULL;
+}
+
+static int mtk_pcie_config_read(struct pci_bus *bus, unsigned int devfn,
+				int where, int size, u32 *val)
+{
+	struct mtk_pcie_port *port;
+	u32 bn = bus->number;
+	int ret;
+
+	port = mtk_pcie_find_port(bus, devfn);
+	if (!port) {
+		*val = ~0;
+		return PCIBIOS_DEVICE_NOT_FOUND;
+	}
+
+	ret = mtk_pcie_hw_rd_cfg(port, bn, devfn, where, size, val);
+	if (ret)
+		*val = ~0;
+
+	return ret;
+}
+
+static int mtk_pcie_config_write(struct pci_bus *bus, unsigned int devfn,
+				 int where, int size, u32 val)
+{
+	struct mtk_pcie_port *port;
+	u32 bn = bus->number;
+
+	port = mtk_pcie_find_port(bus, devfn);
+	if (!port)
+		return PCIBIOS_DEVICE_NOT_FOUND;
+
+	return mtk_pcie_hw_wr_cfg(port, bn, devfn, where, size, val);
+}
+
+static struct pci_ops mtk_pcie_ops_v2 = {
+	.read  = mtk_pcie_config_read,
+	.write = mtk_pcie_config_write,
+};
+
+static int mtk_pcie_startup_port_v2(struct mtk_pcie_port *port)
+{
+	struct mtk_pcie *pcie = port->pcie;
+	struct resource *mem = &pcie->mem;
+	u32 val;
+	size_t size;
+	int err;
+
+	/* MT7622 platforms need to enable LTSSM and ASPM from PCIe subsys */
+	if (pcie->base) {
+		val = readl(pcie->base + PCIE_SYS_CFG_V2);
+		val |= PCIE_CSR_LTSSM_EN(port->slot) |
+		       PCIE_CSR_ASPM_L1_EN(port->slot);
+		writel(val, pcie->base + PCIE_SYS_CFG_V2);
+	}
+
+	/* Assert all reset signals */
+	writel(0, port->base + PCIE_RST_CTRL);
+
+	/*
+	 * Enable PCIe link down reset, if link status changed from link up to
+	 * link down, this will reset MAC control registers and configuration
+	 * space.
+	 */
+	writel(PCIE_LINKDOWN_RST_EN, port->base + PCIE_RST_CTRL);
+
+	/* De-assert PHY, PE, PIPE, MAC and configuration reset	*/
+	val = readl(port->base + PCIE_RST_CTRL);
+	val |= PCIE_PHY_RSTB | PCIE_PERSTB | PCIE_PIPE_SRSTB |
+	       PCIE_MAC_SRSTB | PCIE_CRSTB;
+	writel(val, port->base + PCIE_RST_CTRL);
+
+	/* 100ms timeout value should be enough for Gen1/2 training */
+	err = readl_poll_timeout(port->base + PCIE_LINK_STATUS_V2, val,
+				 !!(val & PCIE_PORT_LINKUP_V2), 20,
+				 100 * USEC_PER_MSEC);
+	if (err)
+		return -ETIMEDOUT;
+
+	/* Set INTx mask */
+	val = readl(port->base + PCIE_INT_MASK);
+	val &= ~INTX_MASK;
+	writel(val, port->base + PCIE_INT_MASK);
+
+	/* Set AHB to PCIe translation windows */
+	size = mem->end - mem->start;
+	val = lower_32_bits(mem->start) | AHB2PCIE_SIZE(fls(size));
+	writel(val, port->base + PCIE_AHB_TRANS_BASE0_L);
+
+	val = upper_32_bits(mem->start);
+	writel(val, port->base + PCIE_AHB_TRANS_BASE0_H);
+
+	/* Set PCIe to AXI translation memory space.*/
+	val = fls(0xffffffff) | WIN_ENABLE;
+	writel(val, port->base + PCIE_AXI_WINDOW0);
+
+	return 0;
+}
+
+static int mtk_pcie_msi_alloc(struct mtk_pcie_port *port)
+{
+	int msi;
+
+	msi = find_first_zero_bit(port->msi_irq_in_use, MTK_MSI_IRQS_NUM);
+	if (msi < MTK_MSI_IRQS_NUM)
+		set_bit(msi, port->msi_irq_in_use);
+	else
+		return -ENOSPC;
+
+	return msi;
+}
+
+static void mtk_pcie_msi_free(struct mtk_pcie_port *port, unsigned long hwirq)
+{
+	clear_bit(hwirq, port->msi_irq_in_use);
+}
+
+static int mtk_pcie_msi_setup_irq(struct msi_controller *chip,
+				  struct pci_dev *pdev, struct msi_desc *desc)
+{
+	struct mtk_pcie_port *port;
+	struct msi_msg msg;
+	unsigned int irq;
+	int hwirq;
+	phys_addr_t msg_addr;
+
+	port = mtk_pcie_find_port(pdev->bus, pdev->devfn);
+	if (!port)
+		return -EINVAL;
+
+	hwirq = mtk_pcie_msi_alloc(port);
+	if (hwirq < 0)
+		return hwirq;
+
+	irq = irq_create_mapping(port->msi_domain, hwirq);
+	if (!irq) {
+		mtk_pcie_msi_free(port, hwirq);
+		return -EINVAL;
+	}
+
+	chip->dev = &pdev->dev;
+
+	irq_set_msi_desc(irq, desc);
+
+	/* MT2712/MT7622 only support 32-bit MSI addresses */
+	msg_addr = virt_to_phys(port->base + PCIE_MSI_VECTOR);
+	msg.address_hi = 0;
+	msg.address_lo = lower_32_bits(msg_addr);
+	msg.data = hwirq;
+
+	pci_write_msi_msg(irq, &msg);
+
+	return 0;
+}
+
+static void mtk_msi_teardown_irq(struct msi_controller *chip, unsigned int irq)
+{
+	struct pci_dev *pdev = to_pci_dev(chip->dev);
+	struct irq_data *d = irq_get_irq_data(irq);
+	irq_hw_number_t hwirq = irqd_to_hwirq(d);
+	struct mtk_pcie_port *port;
+
+	port = mtk_pcie_find_port(pdev->bus, pdev->devfn);
+	if (!port)
+		return;
+
+	irq_dispose_mapping(irq);
+	mtk_pcie_msi_free(port, hwirq);
+}
+
+static struct msi_controller mtk_pcie_msi_chip = {
+	.setup_irq = mtk_pcie_msi_setup_irq,
+	.teardown_irq = mtk_msi_teardown_irq,
+};
+
+static struct irq_chip mtk_msi_irq_chip = {
+	.name = "MTK PCIe MSI",
+	.irq_enable = pci_msi_unmask_irq,
+	.irq_disable = pci_msi_mask_irq,
+	.irq_mask = pci_msi_mask_irq,
+	.irq_unmask = pci_msi_unmask_irq,
+};
+
+static int mtk_pcie_msi_map(struct irq_domain *domain, unsigned int irq,
+			    irq_hw_number_t hwirq)
+{
+	irq_set_chip_and_handler(irq, &mtk_msi_irq_chip, handle_simple_irq);
+	irq_set_chip_data(irq, domain->host_data);
+
+	return 0;
+}
+
+static const struct irq_domain_ops msi_domain_ops = {
+	.map = mtk_pcie_msi_map,
+};
+
+static void mtk_pcie_enable_msi(struct mtk_pcie_port *port)
+{
+	u32 val;
+	phys_addr_t msg_addr;
+
+	msg_addr = virt_to_phys(port->base + PCIE_MSI_VECTOR);
+	val = lower_32_bits(msg_addr);
+	writel(val, port->base + PCIE_IMSI_ADDR);
+
+	val = readl(port->base + PCIE_INT_MASK);
+	val &= ~MSI_MASK;
+	writel(val, port->base + PCIE_INT_MASK);
+}
+
+static int mtk_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
+			     irq_hw_number_t hwirq)
+{
+	irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq);
+	irq_set_chip_data(irq, domain->host_data);
+
+	return 0;
+}
+
+static const struct irq_domain_ops intx_domain_ops = {
+	.map = mtk_pcie_intx_map,
+};
+
+static int mtk_pcie_init_irq_domain(struct mtk_pcie_port *port,
+				    struct device_node *node)
+{
+	struct device *dev = port->pcie->dev;
+	struct device_node *pcie_intc_node;
+
+	/* Setup INTx */
+	pcie_intc_node = of_get_next_child(node, NULL);
+	if (!pcie_intc_node) {
+		dev_err(dev, "no PCIe Intc node found\n");
+		return -ENODEV;
+	}
+
+	port->irq_domain = irq_domain_add_linear(pcie_intc_node, PCI_NUM_INTX,
+						 &intx_domain_ops, port);
+	if (!port->irq_domain) {
+		dev_err(dev, "failed to get INTx IRQ domain\n");
+		return -ENODEV;
+	}
+
+	if (IS_ENABLED(CONFIG_PCI_MSI)) {
+		port->msi_domain = irq_domain_add_linear(node, MTK_MSI_IRQS_NUM,
+							 &msi_domain_ops,
+							 &mtk_pcie_msi_chip);
+		if (!port->msi_domain) {
+			dev_err(dev, "failed to create MSI IRQ domain\n");
+			return -ENODEV;
+		}
+		mtk_pcie_enable_msi(port);
+	}
+
+	return 0;
+}
+
+static irqreturn_t mtk_pcie_intr_handler(int irq, void *data)
+{
+	struct mtk_pcie_port *port = (struct mtk_pcie_port *)data;
+	unsigned long status;
+	u32 virq;
+	u32 bit = INTX_SHIFT;
+
+	while ((status = readl(port->base + PCIE_INT_STATUS)) & INTX_MASK) {
+		for_each_set_bit_from(bit, &status, PCI_NUM_INTX + INTX_SHIFT) {
+			/* Clear the INTx */
+			writel(1 << bit, port->base + PCIE_INT_STATUS);
+			virq = irq_find_mapping(port->irq_domain,
+						bit - INTX_SHIFT);
+			generic_handle_irq(virq);
+		}
+	}
+
+	if (IS_ENABLED(CONFIG_PCI_MSI)) {
+		while ((status = readl(port->base + PCIE_INT_STATUS)) & MSI_STATUS) {
+			unsigned long imsi_status;
+
+			while ((imsi_status = readl(port->base + PCIE_IMSI_STATUS))) {
+				for_each_set_bit(bit, &imsi_status, MTK_MSI_IRQS_NUM) {
+					/* Clear the MSI */
+					writel(1 << bit, port->base + PCIE_IMSI_STATUS);
+					virq = irq_find_mapping(port->msi_domain, bit);
+					generic_handle_irq(virq);
+				}
+			}
+			/* Clear MSI interrupt status */
+			writel(MSI_STATUS, port->base + PCIE_INT_STATUS);
+		}
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int mtk_pcie_setup_irq(struct mtk_pcie_port *port,
+			      struct device_node *node)
+{
+	struct mtk_pcie *pcie = port->pcie;
+	struct device *dev = pcie->dev;
+	struct platform_device *pdev = to_platform_device(dev);
+	int err, irq;
+
+	irq = platform_get_irq(pdev, port->slot);
+	err = devm_request_irq(dev, irq, mtk_pcie_intr_handler,
+			       IRQF_SHARED, "mtk-pcie", port);
+	if (err) {
+		dev_err(dev, "unable to request IRQ %d\n", irq);
+		return err;
+	}
+
+	err = mtk_pcie_init_irq_domain(port, node);
+	if (err) {
+		dev_err(dev, "failed to init PCIe IRQ domain\n");
+		return err;
+	}
+
+	return 0;
+}
+
 static void __iomem *mtk_pcie_map_bus(struct pci_bus *bus,
 				      unsigned int devfn, int where)
 {
-	struct pci_host_bridge *host = pci_find_host_bridge(bus);
-	struct mtk_pcie *pcie = pci_host_bridge_priv(host);
+	struct mtk_pcie *pcie = bus->sysdata;
 
 	writel(PCIE_CONF_ADDR(where, PCI_FUNC(devfn), PCI_SLOT(devfn),
 			      bus->number), pcie->base + PCIE_CFG_ADDR);
@@ -171,16 +675,34 @@ static struct pci_ops mtk_pcie_ops = {
 	.write = pci_generic_config_write,
 };
 
-static void mtk_pcie_configure_rc(struct mtk_pcie_port *port)
+static int mtk_pcie_startup_port(struct mtk_pcie_port *port)
 {
 	struct mtk_pcie *pcie = port->pcie;
-	u32 func = PCI_FUNC(port->index << 3);
-	u32 slot = PCI_SLOT(port->index << 3);
+	u32 func = PCI_FUNC(port->slot << 3);
+	u32 slot = PCI_SLOT(port->slot << 3);
 	u32 val;
+	int err;
+
+	/* assert port PERST_N */
+	val = readl(pcie->base + PCIE_SYS_CFG);
+	val |= PCIE_PORT_PERST(port->slot);
+	writel(val, pcie->base + PCIE_SYS_CFG);
+
+	/* de-assert port PERST_N */
+	val = readl(pcie->base + PCIE_SYS_CFG);
+	val &= ~PCIE_PORT_PERST(port->slot);
+	writel(val, pcie->base + PCIE_SYS_CFG);
+
+	/* 100ms timeout value should be enough for Gen1/2 training */
+	err = readl_poll_timeout(port->base + PCIE_LINK_STATUS, val,
+				 !!(val & PCIE_PORT_LINKUP), 20,
+				 100 * USEC_PER_MSEC);
+	if (err)
+		return -ETIMEDOUT;
 
 	/* enable interrupt */
 	val = readl(pcie->base + PCIE_INT_ENABLE);
-	val |= PCIE_PORT_INT_EN(port->index);
+	val |= PCIE_PORT_INT_EN(port->slot);
 	writel(val, pcie->base + PCIE_INT_ENABLE);
 
 	/* map to all DDR region. We need to set it before cfg operation. */
@@ -209,67 +731,94 @@ static void mtk_pcie_configure_rc(struct mtk_pcie_port *port)
 	writel(PCIE_CONF_ADDR(PCIE_FTS_NUM, func, slot, 0),
 	       pcie->base + PCIE_CFG_ADDR);
 	writel(val, pcie->base + PCIE_CFG_DATA);
+
+	return 0;
 }
 
-static void mtk_pcie_assert_ports(struct mtk_pcie_port *port)
+static void mtk_pcie_enable_port(struct mtk_pcie_port *port)
 {
 	struct mtk_pcie *pcie = port->pcie;
-	u32 val;
+	struct device *dev = pcie->dev;
+	int err;
 
-	/* assert port PERST_N */
-	val = readl(pcie->base + PCIE_SYS_CFG);
-	val |= PCIE_PORT_PERST(port->index);
-	writel(val, pcie->base + PCIE_SYS_CFG);
+	err = clk_prepare_enable(port->sys_ck);
+	if (err) {
+		dev_err(dev, "failed to enable sys_ck%d clock\n", port->slot);
+		goto err_sys_clk;
+	}
 
-	/* de-assert port PERST_N */
-	val = readl(pcie->base + PCIE_SYS_CFG);
-	val &= ~PCIE_PORT_PERST(port->index);
-	writel(val, pcie->base + PCIE_SYS_CFG);
+	err = clk_prepare_enable(port->ahb_ck);
+	if (err) {
+		dev_err(dev, "failed to enable ahb_ck%d\n", port->slot);
+		goto err_ahb_clk;
+	}
 
-	/* PCIe v2.0 need at least 100ms delay to train from Gen1 to Gen2 */
-	msleep(100);
-}
+	err = clk_prepare_enable(port->aux_ck);
+	if (err) {
+		dev_err(dev, "failed to enable aux_ck%d\n", port->slot);
+		goto err_aux_clk;
+	}
 
-static void mtk_pcie_enable_ports(struct mtk_pcie_port *port)
-{
-	struct device *dev = port->pcie->dev;
-	int err;
+	err = clk_prepare_enable(port->axi_ck);
+	if (err) {
+		dev_err(dev, "failed to enable axi_ck%d\n", port->slot);
+		goto err_axi_clk;
+	}
 
-	err = clk_prepare_enable(port->sys_ck);
+	err = clk_prepare_enable(port->obff_ck);
 	if (err) {
-		dev_err(dev, "failed to enable port%d clock\n", port->index);
-		goto err_sys_clk;
+		dev_err(dev, "failed to enable obff_ck%d\n", port->slot);
+		goto err_obff_clk;
+	}
+
+	err = clk_prepare_enable(port->pipe_ck);
+	if (err) {
+		dev_err(dev, "failed to enable pipe_ck%d\n", port->slot);
+		goto err_pipe_clk;
 	}
 
 	reset_control_assert(port->reset);
 	reset_control_deassert(port->reset);
 
+	err = phy_init(port->phy);
+	if (err) {
+		dev_err(dev, "failed to initialize port%d phy\n", port->slot);
+		goto err_phy_init;
+	}
+
 	err = phy_power_on(port->phy);
 	if (err) {
-		dev_err(dev, "failed to power on port%d phy\n", port->index);
+		dev_err(dev, "failed to power on port%d phy\n", port->slot);
 		goto err_phy_on;
 	}
 
-	mtk_pcie_assert_ports(port);
-
-	/* if link up, then setup root port configuration space */
-	if (mtk_pcie_link_up(port)) {
-		mtk_pcie_configure_rc(port);
+	if (!pcie->soc->startup(port))
 		return;
-	}
 
-	dev_info(dev, "Port%d link down\n", port->index);
+	dev_info(dev, "Port%d link down\n", port->slot);
 
 	phy_power_off(port->phy);
 err_phy_on:
+	phy_exit(port->phy);
+err_phy_init:
+	clk_disable_unprepare(port->pipe_ck);
+err_pipe_clk:
+	clk_disable_unprepare(port->obff_ck);
+err_obff_clk:
+	clk_disable_unprepare(port->axi_ck);
+err_axi_clk:
+	clk_disable_unprepare(port->aux_ck);
+err_aux_clk:
+	clk_disable_unprepare(port->ahb_ck);
+err_ahb_clk:
 	clk_disable_unprepare(port->sys_ck);
 err_sys_clk:
 	mtk_pcie_port_free(port);
 }
 
-static int mtk_pcie_parse_ports(struct mtk_pcie *pcie,
-				struct device_node *node,
-				int index)
+static int mtk_pcie_parse_port(struct mtk_pcie *pcie,
+			       struct device_node *node,
+			       int slot)
 {
 	struct mtk_pcie_port *port;
 	struct resource *regs;
@@ -288,34 +837,87 @@ static int mtk_pcie_parse_ports(struct mtk_pcie *pcie,
 		return err;
 	}
 
-	regs = platform_get_resource(pdev, IORESOURCE_MEM, index + 1);
+	snprintf(name, sizeof(name), "port%d", slot);
+	regs = platform_get_resource_byname(pdev, IORESOURCE_MEM, name);
 	port->base = devm_ioremap_resource(dev, regs);
 	if (IS_ERR(port->base)) {
-		dev_err(dev, "failed to map port%d base\n", index);
+		dev_err(dev, "failed to map port%d base\n", slot);
 		return PTR_ERR(port->base);
 	}
 
-	snprintf(name, sizeof(name), "sys_ck%d", index);
+	snprintf(name, sizeof(name), "sys_ck%d", slot);
 	port->sys_ck = devm_clk_get(dev, name);
 	if (IS_ERR(port->sys_ck)) {
-		dev_err(dev, "failed to get port%d clock\n", index);
+		dev_err(dev, "failed to get sys_ck%d clock\n", slot);
 		return PTR_ERR(port->sys_ck);
 	}
 
-	snprintf(name, sizeof(name), "pcie-rst%d", index);
-	port->reset = devm_reset_control_get_optional(dev, name);
+	/* sys_ck might be divided into the following parts in some chips */
+	snprintf(name, sizeof(name), "ahb_ck%d", slot);
+	port->ahb_ck = devm_clk_get(dev, name);
+	if (IS_ERR(port->ahb_ck)) {
+		if (PTR_ERR(port->ahb_ck) == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+
+		port->ahb_ck = NULL;
+	}
+
+	snprintf(name, sizeof(name), "axi_ck%d", slot);
+	port->axi_ck = devm_clk_get(dev, name);
+	if (IS_ERR(port->axi_ck)) {
+		if (PTR_ERR(port->axi_ck) == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+
+		port->axi_ck = NULL;
+	}
+
+	snprintf(name, sizeof(name), "aux_ck%d", slot);
+	port->aux_ck = devm_clk_get(dev, name);
+	if (IS_ERR(port->aux_ck)) {
+		if (PTR_ERR(port->aux_ck) == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+
+		port->aux_ck = NULL;
+	}
+
+	snprintf(name, sizeof(name), "obff_ck%d", slot);
+	port->obff_ck = devm_clk_get(dev, name);
+	if (IS_ERR(port->obff_ck)) {
+		if (PTR_ERR(port->obff_ck) == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+
+		port->obff_ck = NULL;
+	}
+
+	snprintf(name, sizeof(name), "pipe_ck%d", slot);
+	port->pipe_ck = devm_clk_get(dev, name);
+	if (IS_ERR(port->pipe_ck)) {
+		if (PTR_ERR(port->pipe_ck) == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+
+		port->pipe_ck = NULL;
+	}
+
+	snprintf(name, sizeof(name), "pcie-rst%d", slot);
+	port->reset = devm_reset_control_get_optional_exclusive(dev, name);
 	if (PTR_ERR(port->reset) == -EPROBE_DEFER)
 		return PTR_ERR(port->reset);
 
 	/* some platforms may use default PHY setting */
-	snprintf(name, sizeof(name), "pcie-phy%d", index);
+	snprintf(name, sizeof(name), "pcie-phy%d", slot);
 	port->phy = devm_phy_optional_get(dev, name);
 	if (IS_ERR(port->phy))
 		return PTR_ERR(port->phy);
 
-	port->index = index;
+	port->slot = slot;
 	port->pcie = pcie;
 
+	if (pcie->soc->setup_irq) {
+		err = pcie->soc->setup_irq(port, node);
+		if (err)
+			return err;
+	}
+
 	INIT_LIST_HEAD(&port->list);
 	list_add_tail(&port->list, &pcie->ports);
 
@@ -329,12 +931,14 @@ static int mtk_pcie_subsys_powerup(struct mtk_pcie *pcie)
 	struct resource *regs;
 	int err;
 
-	/* get shared registers */
-	regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	pcie->base = devm_ioremap_resource(dev, regs);
-	if (IS_ERR(pcie->base)) {
-		dev_err(dev, "failed to map shared register\n");
-		return PTR_ERR(pcie->base);
+	/* get shared registers, which are optional */
+	regs = platform_get_resource_byname(pdev, IORESOURCE_MEM, "subsys");
+	if (regs) {
+		pcie->base = devm_ioremap_resource(dev, regs);
+		if (IS_ERR(pcie->base)) {
+			dev_err(dev, "failed to map shared register\n");
+			return PTR_ERR(pcie->base);
+		}
 	}
 
 	pcie->free_ck = devm_clk_get(dev, "free_ck");
@@ -422,7 +1026,7 @@ static int mtk_pcie_setup(struct mtk_pcie *pcie)
 	}
 
 	for_each_available_child_of_node(node, child) {
-		int index;
+		int slot;
 
 		err = of_pci_get_devfn(child);
 		if (err < 0) {
@@ -430,9 +1034,9 @@ static int mtk_pcie_setup(struct mtk_pcie *pcie)
 			return err;
 		}
 
-		index = PCI_SLOT(err);
+		slot = PCI_SLOT(err);
 
-		err = mtk_pcie_parse_ports(pcie, child, index);
+		err = mtk_pcie_parse_port(pcie, child, slot);
 		if (err)
 			return err;
 	}
@@ -443,7 +1047,7 @@ static int mtk_pcie_setup(struct mtk_pcie *pcie)
 
 	/* enable each port, and then check link status */
 	list_for_each_entry_safe(port, tmp, &pcie->ports, list)
-		mtk_pcie_enable_ports(port);
+		mtk_pcie_enable_port(port);
 
 	/* power down PCIe subsys if slots are all empty (link down) */
 	if (list_empty(&pcie->ports))
@@ -480,9 +1084,12 @@ static int mtk_pcie_register_host(struct pci_host_bridge *host)
 
 	host->busnr = pcie->busn.start;
 	host->dev.parent = pcie->dev;
-	host->ops = &mtk_pcie_ops;
+	host->ops = pcie->soc->ops;
 	host->map_irq = of_irq_parse_and_map_pci;
 	host->swizzle_irq = pci_common_swizzle;
+	host->sysdata = pcie;
+	if (IS_ENABLED(CONFIG_PCI_MSI) && pcie->soc->has_msi)
+		host->msi = &mtk_pcie_msi_chip;
 
 	err = pci_scan_root_bus_bridge(host);
 	if (err < 0)
@@ -513,6 +1120,7 @@ static int mtk_pcie_probe(struct platform_device *pdev)
 	pcie = pci_host_bridge_priv(host);
 
 	pcie->dev = dev;
+	pcie->soc = of_device_get_match_data(dev);
 	platform_set_drvdata(pdev, pcie);
 	INIT_LIST_HEAD(&pcie->ports);
 
@@ -537,9 +1145,23 @@ put_resources:
 	return err;
 }
 
+static const struct mtk_pcie_soc mtk_pcie_soc_v1 = {
+	.ops = &mtk_pcie_ops,
+	.startup = mtk_pcie_startup_port,
+};
+
+static const struct mtk_pcie_soc mtk_pcie_soc_v2 = {
+	.has_msi = true,
+	.ops = &mtk_pcie_ops_v2,
+	.startup = mtk_pcie_startup_port_v2,
+	.setup_irq = mtk_pcie_setup_irq,
+};
+
 static const struct of_device_id mtk_pcie_ids[] = {
-	{ .compatible = "mediatek,mt7623-pcie"},
-	{ .compatible = "mediatek,mt2701-pcie"},
+	{ .compatible = "mediatek,mt2701-pcie", .data = &mtk_pcie_soc_v1 },
+	{ .compatible = "mediatek,mt7623-pcie", .data = &mtk_pcie_soc_v1 },
+	{ .compatible = "mediatek,mt2712-pcie", .data = &mtk_pcie_soc_v2 },
+	{ .compatible = "mediatek,mt7622-pcie", .data = &mtk_pcie_soc_v2 },
 	{},
 };
 
diff --git a/drivers/pci/host/pcie-rcar.c b/drivers/pci/host/pcie-rcar.c
index 246d485b24c6..4e0b25d09b0c 100644
--- a/drivers/pci/host/pcie-rcar.c
+++ b/drivers/pci/host/pcie-rcar.c
@@ -471,10 +471,8 @@ static int rcar_pcie_enable(struct rcar_pcie *pcie)
 		bridge->msi = &pcie->msi.chip;
 
 	ret = pci_scan_root_bus_bridge(bridge);
-	if (ret < 0) {
-		kfree(bridge);
+	if (ret < 0)
 		return ret;
-	}
 
 	bus = bridge->bus;
 
@@ -1190,14 +1188,16 @@ static int rcar_pcie_probe(struct platform_device *pdev)
 
 	return 0;
 
-err_free_bridge:
-	pci_free_host_bridge(bridge);
-
 err_pm_put:
 	pm_runtime_put(dev);
 
 err_pm_disable:
 	pm_runtime_disable(dev);
+
+err_free_bridge:
+	pci_free_host_bridge(bridge);
+	pci_free_resource_list(&pcie->resources);
+
 	return err;
 }
 
diff --git a/drivers/pci/host/pcie-rockchip.c b/drivers/pci/host/pcie-rockchip.c
index 7bb9870f6d8c..9051c6c8fea4 100644
--- a/drivers/pci/host/pcie-rockchip.c
+++ b/drivers/pci/host/pcie-rockchip.c
@@ -6,7 +6,7 @@
  * Author: Shawn Lin <shawn.lin@rock-chips.com>
  *         Wenrui Li <wenrui.li@rock-chips.com>
  *
- * Bits taken from Synopsys Designware Host controller driver and
+ * Bits taken from Synopsys DesignWare Host controller driver and
  * ARM PCI Host generic driver.
  *
  * This program is free software: you can redistribute it and/or modify
@@ -15,6 +15,7 @@
  * (at your option) any later version.
  */
 
+#include <linux/bitrev.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
 #include <linux/gpio/consumer.h>
@@ -47,6 +48,7 @@
 #define HIWORD_UPDATE_BIT(val)		HIWORD_UPDATE(val, val)
 
 #define ENCODE_LANES(x)			((((x) >> 1) & 3) << 4)
+#define MAX_LANE_NUM			4
 
 #define PCIE_CLIENT_BASE		0x0
 #define PCIE_CLIENT_CONFIG		(PCIE_CLIENT_BASE + 0x00)
@@ -111,6 +113,9 @@
 #define   PCIE_CORE_TXCREDIT_CFG1_MUI_SHIFT	16
 #define   PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(x) \
 		(((x) >> 3) << PCIE_CORE_TXCREDIT_CFG1_MUI_SHIFT)
+#define PCIE_CORE_LANE_MAP             (PCIE_CORE_CTRL_MGMT_BASE + 0x200)
+#define   PCIE_CORE_LANE_MAP_MASK              0x0000000f
+#define   PCIE_CORE_LANE_MAP_REVERSE           BIT(16)
 #define PCIE_CORE_INT_STATUS		(PCIE_CORE_CTRL_MGMT_BASE + 0x20c)
 #define   PCIE_CORE_INT_PRFPE			BIT(0)
 #define   PCIE_CORE_INT_CRFPE			BIT(1)
@@ -210,7 +215,8 @@
 struct rockchip_pcie {
 	void	__iomem *reg_base;		/* DT axi-base */
 	void	__iomem *apb_base;		/* DT apb-base */
-	struct	phy *phy;
+	bool    legacy_phy;
+	struct  phy *phys[MAX_LANE_NUM];
 	struct	reset_control *core_rst;
 	struct	reset_control *mgmt_rst;
 	struct	reset_control *mgmt_sticky_rst;
@@ -222,11 +228,13 @@ struct rockchip_pcie {
 	struct	clk *aclk_perf_pcie;
 	struct	clk *hclk_pcie;
 	struct	clk *clk_pcie_pm;
+	struct	regulator *vpcie12v; /* 12V power supply */
 	struct	regulator *vpcie3v3; /* 3.3V power supply */
 	struct	regulator *vpcie1v8; /* 1.8V power supply */
 	struct	regulator *vpcie0v9; /* 0.9V power supply */
 	struct	gpio_desc *ep_gpio;
 	u32	lanes;
+	u8      lanes_map;
 	u8	root_bus_nr;
 	int	link_gen;
 	struct	device *dev;
@@ -299,6 +307,24 @@ static int rockchip_pcie_valid_device(struct rockchip_pcie *rockchip,
 	return 1;
 }
 
+static u8 rockchip_pcie_lane_map(struct rockchip_pcie *rockchip)
+{
+	u32 val;
+	u8 map;
+
+	if (rockchip->legacy_phy)
+		return GENMASK(MAX_LANE_NUM - 1, 0);
+
+	val = rockchip_pcie_read(rockchip, PCIE_CORE_LANE_MAP);
+	map = val & PCIE_CORE_LANE_MAP_MASK;
+
+	/* The link may be using a reverse-indexed mapping. */
+	if (val & PCIE_CORE_LANE_MAP_REVERSE)
+		map = bitrev8(map) >> 4;
+
+	return map;
+}
+
 static int rockchip_pcie_rd_own_conf(struct rockchip_pcie *rockchip,
 				     int where, int size, u32 *val)
 {
@@ -514,10 +540,10 @@ static void rockchip_pcie_set_power_limit(struct rockchip_pcie *rockchip)
 static int rockchip_pcie_init_port(struct rockchip_pcie *rockchip)
 {
 	struct device *dev = rockchip->dev;
-	int err;
+	int err, i;
 	u32 status;
 
-	gpiod_set_value(rockchip->ep_gpio, 0);
+	gpiod_set_value_cansleep(rockchip->ep_gpio, 0);
 
 	err = reset_control_assert(rockchip->aclk_rst);
 	if (err) {
@@ -537,34 +563,36 @@ static int rockchip_pcie_init_port(struct rockchip_pcie *rockchip)
 		return err;
 	}
 
-	err = phy_init(rockchip->phy);
-	if (err < 0) {
-		dev_err(dev, "fail to init phy, err %d\n", err);
-		return err;
+	for (i = 0; i < MAX_LANE_NUM; i++) {
+		err = phy_init(rockchip->phys[i]);
+		if (err) {
+			dev_err(dev, "init phy%d err %d\n", i, err);
+			goto err_exit_phy;
+		}
 	}
 
 	err = reset_control_assert(rockchip->core_rst);
 	if (err) {
 		dev_err(dev, "assert core_rst err %d\n", err);
-		return err;
+		goto err_exit_phy;
 	}
 
 	err = reset_control_assert(rockchip->mgmt_rst);
 	if (err) {
 		dev_err(dev, "assert mgmt_rst err %d\n", err);
-		return err;
+		goto err_exit_phy;
 	}
 
 	err = reset_control_assert(rockchip->mgmt_sticky_rst);
 	if (err) {
 		dev_err(dev, "assert mgmt_sticky_rst err %d\n", err);
-		return err;
+		goto err_exit_phy;
 	}
 
 	err = reset_control_assert(rockchip->pipe_rst);
 	if (err) {
 		dev_err(dev, "assert pipe_rst err %d\n", err);
-		return err;
+		goto err_exit_phy;
 	}
 
 	udelay(10);
@@ -572,19 +600,19 @@ static int rockchip_pcie_init_port(struct rockchip_pcie *rockchip)
 	err = reset_control_deassert(rockchip->pm_rst);
 	if (err) {
 		dev_err(dev, "deassert pm_rst err %d\n", err);
-		return err;
+		goto err_exit_phy;
 	}
 
 	err = reset_control_deassert(rockchip->aclk_rst);
 	if (err) {
 		dev_err(dev, "deassert aclk_rst err %d\n", err);
-		return err;
+		goto err_exit_phy;
 	}
 
 	err = reset_control_deassert(rockchip->pclk_rst);
 	if (err) {
 		dev_err(dev, "deassert pclk_rst err %d\n", err);
-		return err;
+		goto err_exit_phy;
 	}
 
 	if (rockchip->link_gen == 2)
@@ -602,10 +630,12 @@ static int rockchip_pcie_init_port(struct rockchip_pcie *rockchip)
 			    PCIE_CLIENT_MODE_RC,
 			    PCIE_CLIENT_CONFIG);
 
-	err = phy_power_on(rockchip->phy);
-	if (err) {
-		dev_err(dev, "fail to power on phy, err %d\n", err);
-		return err;
+	for (i = 0; i < MAX_LANE_NUM; i++) {
+		err = phy_power_on(rockchip->phys[i]);
+		if (err) {
+			dev_err(dev, "power on phy%d err %d\n", i, err);
+			goto err_power_off_phy;
+		}
 	}
 
 	/*
@@ -615,25 +645,25 @@ static int rockchip_pcie_init_port(struct rockchip_pcie *rockchip)
 	err = reset_control_deassert(rockchip->mgmt_sticky_rst);
 	if (err) {
 		dev_err(dev, "deassert mgmt_sticky_rst err %d\n", err);
-		return err;
+		goto err_power_off_phy;
 	}
 
 	err = reset_control_deassert(rockchip->core_rst);
 	if (err) {
 		dev_err(dev, "deassert core_rst err %d\n", err);
-		return err;
+		goto err_power_off_phy;
 	}
 
 	err = reset_control_deassert(rockchip->mgmt_rst);
 	if (err) {
 		dev_err(dev, "deassert mgmt_rst err %d\n", err);
-		return err;
+		goto err_power_off_phy;
 	}
 
 	err = reset_control_deassert(rockchip->pipe_rst);
 	if (err) {
 		dev_err(dev, "deassert pipe_rst err %d\n", err);
-		return err;
+		goto err_power_off_phy;
 	}
 
 	/* Fix the transmitted FTS count desired to exit from L0s. */
@@ -658,7 +688,7 @@ static int rockchip_pcie_init_port(struct rockchip_pcie *rockchip)
 	rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE,
 			    PCIE_CLIENT_CONFIG);
 
-	gpiod_set_value(rockchip->ep_gpio, 1);
+	gpiod_set_value_cansleep(rockchip->ep_gpio, 1);
 
 	/* 500ms timeout value should be enough for Gen1/2 training */
 	err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_BASIC_STATUS1,
@@ -666,7 +696,7 @@ static int rockchip_pcie_init_port(struct rockchip_pcie *rockchip)
 				 500 * USEC_PER_MSEC);
 	if (err) {
 		dev_err(dev, "PCIe link training gen1 timeout!\n");
-		return -ETIMEDOUT;
+		goto err_power_off_phy;
 	}
 
 	if (rockchip->link_gen == 2) {
@@ -691,6 +721,15 @@ static int rockchip_pcie_init_port(struct rockchip_pcie *rockchip)
 			  PCIE_CORE_PL_CONF_LANE_SHIFT);
 	dev_dbg(dev, "current link width is x%d\n", status);
 
+	/* Power off unused lane(s) */
+	rockchip->lanes_map = rockchip_pcie_lane_map(rockchip);
+	for (i = 0; i < MAX_LANE_NUM; i++) {
+		if (!(rockchip->lanes_map & BIT(i))) {
+			dev_dbg(dev, "idling lane %d\n", i);
+			phy_power_off(rockchip->phys[i]);
+		}
+	}
+
 	rockchip_pcie_write(rockchip, ROCKCHIP_VENDOR_ID,
 			    PCIE_CORE_CONFIG_VENDOR);
 	rockchip_pcie_write(rockchip,
@@ -715,6 +754,26 @@ static int rockchip_pcie_init_port(struct rockchip_pcie *rockchip)
 	rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCSR);
 
 	return 0;
+err_power_off_phy:
+	while (i--)
+		phy_power_off(rockchip->phys[i]);
+	i = MAX_LANE_NUM;
+err_exit_phy:
+	while (i--)
+		phy_exit(rockchip->phys[i]);
+	return err;
+}
+
+static void rockchip_pcie_deinit_phys(struct rockchip_pcie *rockchip)
+{
+	int i;
+
+	for (i = 0; i < MAX_LANE_NUM; i++) {
+		/* inactive lanes are already powered off */
+		if (rockchip->lanes_map & BIT(i))
+			phy_power_off(rockchip->phys[i]);
+		phy_exit(rockchip->phys[i]);
+	}
 }
 
 static irqreturn_t rockchip_pcie_subsys_irq_handler(int irq, void *arg)
@@ -853,6 +912,91 @@ static void rockchip_pcie_legacy_int_handler(struct irq_desc *desc)
 	chained_irq_exit(chip, desc);
 }
 
+static int rockchip_pcie_get_phys(struct rockchip_pcie *rockchip)
+{
+	struct device *dev = rockchip->dev;
+	struct phy *phy;
+	char *name;
+	u32 i;
+
+	phy = devm_phy_get(dev, "pcie-phy");
+	if (!IS_ERR(phy)) {
+		rockchip->legacy_phy = true;
+		rockchip->phys[0] = phy;
+		dev_warn(dev, "legacy phy model is deprecated!\n");
+		return 0;
+	}
+
+	if (PTR_ERR(phy) == -EPROBE_DEFER)
+		return PTR_ERR(phy);
+
+	dev_dbg(dev, "missing legacy phy; search for per-lane PHY\n");
+
+	for (i = 0; i < MAX_LANE_NUM; i++) {
+		name = kasprintf(GFP_KERNEL, "pcie-phy-%u", i);
+		if (!name)
+			return -ENOMEM;
+
+		phy = devm_of_phy_get(dev, dev->of_node, name);
+		kfree(name);
+
+		if (IS_ERR(phy)) {
+			if (PTR_ERR(phy) != -EPROBE_DEFER)
+				dev_err(dev, "missing phy for lane %d: %ld\n",
+					i, PTR_ERR(phy));
+			return PTR_ERR(phy);
+		}
+
+		rockchip->phys[i] = phy;
+	}
+
+	return 0;
+}
+
+static int rockchip_pcie_setup_irq(struct rockchip_pcie *rockchip)
+{
+	int irq, err;
+	struct device *dev = rockchip->dev;
+	struct platform_device *pdev = to_platform_device(dev);
+
+	irq = platform_get_irq_byname(pdev, "sys");
+	if (irq < 0) {
+		dev_err(dev, "missing sys IRQ resource\n");
+		return irq;
+	}
+
+	err = devm_request_irq(dev, irq, rockchip_pcie_subsys_irq_handler,
+			       IRQF_SHARED, "pcie-sys", rockchip);
+	if (err) {
+		dev_err(dev, "failed to request PCIe subsystem IRQ\n");
+		return err;
+	}
+
+	irq = platform_get_irq_byname(pdev, "legacy");
+	if (irq < 0) {
+		dev_err(dev, "missing legacy IRQ resource\n");
+		return irq;
+	}
+
+	irq_set_chained_handler_and_data(irq,
+					 rockchip_pcie_legacy_int_handler,
+					 rockchip);
+
+	irq = platform_get_irq_byname(pdev, "client");
+	if (irq < 0) {
+		dev_err(dev, "missing client IRQ resource\n");
+		return irq;
+	}
+
+	err = devm_request_irq(dev, irq, rockchip_pcie_client_irq_handler,
+			       IRQF_SHARED, "pcie-client", rockchip);
+	if (err) {
+		dev_err(dev, "failed to request PCIe client IRQ\n");
+		return err;
+	}
+
+	return 0;
+}
 
 /**
  * rockchip_pcie_parse_dt - Parse Device Tree
@@ -866,7 +1010,6 @@ static int rockchip_pcie_parse_dt(struct rockchip_pcie *rockchip)
 	struct platform_device *pdev = to_platform_device(dev);
 	struct device_node *node = dev->of_node;
 	struct resource *regs;
-	int irq;
 	int err;
 
 	regs = platform_get_resource_byname(pdev,
@@ -883,12 +1026,9 @@ static int rockchip_pcie_parse_dt(struct rockchip_pcie *rockchip)
 	if (IS_ERR(rockchip->apb_base))
 		return PTR_ERR(rockchip->apb_base);
 
-	rockchip->phy = devm_phy_get(dev, "pcie-phy");
-	if (IS_ERR(rockchip->phy)) {
-		if (PTR_ERR(rockchip->phy) != -EPROBE_DEFER)
-			dev_err(dev, "missing phy\n");
-		return PTR_ERR(rockchip->phy);
-	}
+	err = rockchip_pcie_get_phys(rockchip);
+	if (err)
+		return err;
 
 	rockchip->lanes = 1;
 	err = of_property_read_u32(node, "num-lanes", &rockchip->lanes);
@@ -903,49 +1043,50 @@ static int rockchip_pcie_parse_dt(struct rockchip_pcie *rockchip)
 	if (rockchip->link_gen < 0 || rockchip->link_gen > 2)
 		rockchip->link_gen = 2;
 
-	rockchip->core_rst = devm_reset_control_get(dev, "core");
+	rockchip->core_rst = devm_reset_control_get_exclusive(dev, "core");
 	if (IS_ERR(rockchip->core_rst)) {
 		if (PTR_ERR(rockchip->core_rst) != -EPROBE_DEFER)
 			dev_err(dev, "missing core reset property in node\n");
 		return PTR_ERR(rockchip->core_rst);
 	}
 
-	rockchip->mgmt_rst = devm_reset_control_get(dev, "mgmt");
+	rockchip->mgmt_rst = devm_reset_control_get_exclusive(dev, "mgmt");
 	if (IS_ERR(rockchip->mgmt_rst)) {
 		if (PTR_ERR(rockchip->mgmt_rst) != -EPROBE_DEFER)
 			dev_err(dev, "missing mgmt reset property in node\n");
 		return PTR_ERR(rockchip->mgmt_rst);
 	}
 
-	rockchip->mgmt_sticky_rst = devm_reset_control_get(dev, "mgmt-sticky");
+	rockchip->mgmt_sticky_rst = devm_reset_control_get_exclusive(dev,
+								     "mgmt-sticky");
 	if (IS_ERR(rockchip->mgmt_sticky_rst)) {
 		if (PTR_ERR(rockchip->mgmt_sticky_rst) != -EPROBE_DEFER)
 			dev_err(dev, "missing mgmt-sticky reset property in node\n");
 		return PTR_ERR(rockchip->mgmt_sticky_rst);
 	}
 
-	rockchip->pipe_rst = devm_reset_control_get(dev, "pipe");
+	rockchip->pipe_rst = devm_reset_control_get_exclusive(dev, "pipe");
 	if (IS_ERR(rockchip->pipe_rst)) {
 		if (PTR_ERR(rockchip->pipe_rst) != -EPROBE_DEFER)
 			dev_err(dev, "missing pipe reset property in node\n");
 		return PTR_ERR(rockchip->pipe_rst);
 	}
 
-	rockchip->pm_rst = devm_reset_control_get(dev, "pm");
+	rockchip->pm_rst = devm_reset_control_get_exclusive(dev, "pm");
 	if (IS_ERR(rockchip->pm_rst)) {
 		if (PTR_ERR(rockchip->pm_rst) != -EPROBE_DEFER)
 			dev_err(dev, "missing pm reset property in node\n");
 		return PTR_ERR(rockchip->pm_rst);
 	}
 
-	rockchip->pclk_rst = devm_reset_control_get(dev, "pclk");
+	rockchip->pclk_rst = devm_reset_control_get_exclusive(dev, "pclk");
 	if (IS_ERR(rockchip->pclk_rst)) {
 		if (PTR_ERR(rockchip->pclk_rst) != -EPROBE_DEFER)
 			dev_err(dev, "missing pclk reset property in node\n");
 		return PTR_ERR(rockchip->pclk_rst);
 	}
 
-	rockchip->aclk_rst = devm_reset_control_get(dev, "aclk");
+	rockchip->aclk_rst = devm_reset_control_get_exclusive(dev, "aclk");
 	if (IS_ERR(rockchip->aclk_rst)) {
 		if (PTR_ERR(rockchip->aclk_rst) != -EPROBE_DEFER)
 			dev_err(dev, "missing aclk reset property in node\n");
@@ -982,40 +1123,15 @@ static int rockchip_pcie_parse_dt(struct rockchip_pcie *rockchip)
 		return PTR_ERR(rockchip->clk_pcie_pm);
 	}
 
-	irq = platform_get_irq_byname(pdev, "sys");
-	if (irq < 0) {
-		dev_err(dev, "missing sys IRQ resource\n");
-		return -EINVAL;
-	}
-
-	err = devm_request_irq(dev, irq, rockchip_pcie_subsys_irq_handler,
-			       IRQF_SHARED, "pcie-sys", rockchip);
-	if (err) {
-		dev_err(dev, "failed to request PCIe subsystem IRQ\n");
+	err = rockchip_pcie_setup_irq(rockchip);
+	if (err)
 		return err;
-	}
-
-	irq = platform_get_irq_byname(pdev, "legacy");
-	if (irq < 0) {
-		dev_err(dev, "missing legacy IRQ resource\n");
-		return -EINVAL;
-	}
-
-	irq_set_chained_handler_and_data(irq,
-					 rockchip_pcie_legacy_int_handler,
-					 rockchip);
-
-	irq = platform_get_irq_byname(pdev, "client");
-	if (irq < 0) {
-		dev_err(dev, "missing client IRQ resource\n");
-		return -EINVAL;
-	}
 
-	err = devm_request_irq(dev, irq, rockchip_pcie_client_irq_handler,
-			       IRQF_SHARED, "pcie-client", rockchip);
-	if (err) {
-		dev_err(dev, "failed to request PCIe client IRQ\n");
-		return err;
+	rockchip->vpcie12v = devm_regulator_get_optional(dev, "vpcie12v");
+	if (IS_ERR(rockchip->vpcie12v)) {
+		if (PTR_ERR(rockchip->vpcie12v) == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+		dev_info(dev, "no vpcie12v regulator found\n");
 	}
 
 	rockchip->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3");
@@ -1047,11 +1163,19 @@ static int rockchip_pcie_set_vpcie(struct rockchip_pcie *rockchip)
 	struct device *dev = rockchip->dev;
 	int err;
 
+	if (!IS_ERR(rockchip->vpcie12v)) {
+		err = regulator_enable(rockchip->vpcie12v);
+		if (err) {
+			dev_err(dev, "fail to enable vpcie12v regulator\n");
+			goto err_out;
+		}
+	}
+
 	if (!IS_ERR(rockchip->vpcie3v3)) {
 		err = regulator_enable(rockchip->vpcie3v3);
 		if (err) {
 			dev_err(dev, "fail to enable vpcie3v3 regulator\n");
-			goto err_out;
+			goto err_disable_12v;
 		}
 	}
 
@@ -1079,6 +1203,9 @@ err_disable_1v8:
 err_disable_3v3:
 	if (!IS_ERR(rockchip->vpcie3v3))
 		regulator_disable(rockchip->vpcie3v3);
+err_disable_12v:
+	if (!IS_ERR(rockchip->vpcie12v))
+		regulator_disable(rockchip->vpcie12v);
 err_out:
 	return err;
 }
@@ -1116,7 +1243,7 @@ static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip)
 		return -EINVAL;
 	}
 
-	rockchip->irq_domain = irq_domain_add_linear(intc, 4,
+	rockchip->irq_domain = irq_domain_add_linear(intc, PCI_NUM_INTX,
 						    &intx_domain_ops, rockchip);
 	if (!rockchip->irq_domain) {
 		dev_err(dev, "failed to get a INTx IRQ domain\n");
@@ -1270,6 +1397,56 @@ static int rockchip_pcie_wait_l2(struct rockchip_pcie *rockchip)
 	return 0;
 }
 
+static int rockchip_pcie_enable_clocks(struct rockchip_pcie *rockchip)
+{
+	struct device *dev = rockchip->dev;
+	int err;
+
+	err = clk_prepare_enable(rockchip->aclk_pcie);
+	if (err) {
+		dev_err(dev, "unable to enable aclk_pcie clock\n");
+		return err;
+	}
+
+	err = clk_prepare_enable(rockchip->aclk_perf_pcie);
+	if (err) {
+		dev_err(dev, "unable to enable aclk_perf_pcie clock\n");
+		goto err_aclk_perf_pcie;
+	}
+
+	err = clk_prepare_enable(rockchip->hclk_pcie);
+	if (err) {
+		dev_err(dev, "unable to enable hclk_pcie clock\n");
+		goto err_hclk_pcie;
+	}
+
+	err = clk_prepare_enable(rockchip->clk_pcie_pm);
+	if (err) {
+		dev_err(dev, "unable to enable clk_pcie_pm clock\n");
+		goto err_clk_pcie_pm;
+	}
+
+	return 0;
+
+err_clk_pcie_pm:
+	clk_disable_unprepare(rockchip->hclk_pcie);
+err_hclk_pcie:
+	clk_disable_unprepare(rockchip->aclk_perf_pcie);
+err_aclk_perf_pcie:
+	clk_disable_unprepare(rockchip->aclk_pcie);
+	return err;
+}
+
+static void rockchip_pcie_disable_clocks(void *data)
+{
+	struct rockchip_pcie *rockchip = data;
+
+	clk_disable_unprepare(rockchip->clk_pcie_pm);
+	clk_disable_unprepare(rockchip->hclk_pcie);
+	clk_disable_unprepare(rockchip->aclk_perf_pcie);
+	clk_disable_unprepare(rockchip->aclk_pcie);
+}
+
 static int __maybe_unused rockchip_pcie_suspend_noirq(struct device *dev)
 {
 	struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
@@ -1286,13 +1463,9 @@ static int __maybe_unused rockchip_pcie_suspend_noirq(struct device *dev)
 		return ret;
 	}
 
-	phy_power_off(rockchip->phy);
-	phy_exit(rockchip->phy);
+	rockchip_pcie_deinit_phys(rockchip);
 
-	clk_disable_unprepare(rockchip->clk_pcie_pm);
-	clk_disable_unprepare(rockchip->hclk_pcie);
-	clk_disable_unprepare(rockchip->aclk_perf_pcie);
-	clk_disable_unprepare(rockchip->aclk_pcie);
+	rockchip_pcie_disable_clocks(rockchip);
 
 	if (!IS_ERR(rockchip->vpcie0v9))
 		regulator_disable(rockchip->vpcie0v9);
@@ -1313,21 +1486,9 @@ static int __maybe_unused rockchip_pcie_resume_noirq(struct device *dev)
 		}
 	}
 
-	err = clk_prepare_enable(rockchip->clk_pcie_pm);
+	err = rockchip_pcie_enable_clocks(rockchip);
 	if (err)
-		goto err_pcie_pm;
-
-	err = clk_prepare_enable(rockchip->hclk_pcie);
-	if (err)
-		goto err_hclk_pcie;
-
-	err = clk_prepare_enable(rockchip->aclk_perf_pcie);
-	if (err)
-		goto err_aclk_perf_pcie;
-
-	err = clk_prepare_enable(rockchip->aclk_pcie);
-	if (err)
-		goto err_aclk_pcie;
+		goto err_disable_0v9;
 
 	err = rockchip_pcie_init_port(rockchip);
 	if (err)
@@ -1335,7 +1496,7 @@ static int __maybe_unused rockchip_pcie_resume_noirq(struct device *dev)
 
 	err = rockchip_pcie_cfg_atu(rockchip);
 	if (err)
-		goto err_pcie_resume;
+		goto err_err_deinit_port;
 
 	/* Need this to enter L1 again */
 	rockchip_pcie_update_txcredit_mui(rockchip);
@@ -1343,15 +1504,13 @@ static int __maybe_unused rockchip_pcie_resume_noirq(struct device *dev)
 
 	return 0;
 
+err_err_deinit_port:
+	rockchip_pcie_deinit_phys(rockchip);
 err_pcie_resume:
-	clk_disable_unprepare(rockchip->aclk_pcie);
-err_aclk_pcie:
-	clk_disable_unprepare(rockchip->aclk_perf_pcie);
-err_aclk_perf_pcie:
-	clk_disable_unprepare(rockchip->hclk_pcie);
-err_hclk_pcie:
-	clk_disable_unprepare(rockchip->clk_pcie_pm);
-err_pcie_pm:
+	rockchip_pcie_disable_clocks(rockchip);
+err_disable_0v9:
+	if (!IS_ERR(rockchip->vpcie0v9))
+		regulator_disable(rockchip->vpcie0v9);
 	return err;
 }
 
@@ -1385,29 +1544,9 @@ static int rockchip_pcie_probe(struct platform_device *pdev)
 	if (err)
 		return err;
 
-	err = clk_prepare_enable(rockchip->aclk_pcie);
-	if (err) {
-		dev_err(dev, "unable to enable aclk_pcie clock\n");
-		goto err_aclk_pcie;
-	}
-
-	err = clk_prepare_enable(rockchip->aclk_perf_pcie);
-	if (err) {
-		dev_err(dev, "unable to enable aclk_perf_pcie clock\n");
-		goto err_aclk_perf_pcie;
-	}
-
-	err = clk_prepare_enable(rockchip->hclk_pcie);
-	if (err) {
-		dev_err(dev, "unable to enable hclk_pcie clock\n");
-		goto err_hclk_pcie;
-	}
-
-	err = clk_prepare_enable(rockchip->clk_pcie_pm);
-	if (err) {
-		dev_err(dev, "unable to enable hclk_pcie clock\n");
-		goto err_pcie_pm;
-	}
+	err = rockchip_pcie_enable_clocks(rockchip);
+	if (err)
+		return err;
 
 	err = rockchip_pcie_set_vpcie(rockchip);
 	if (err) {
@@ -1423,12 +1562,12 @@ static int rockchip_pcie_probe(struct platform_device *pdev)
 
 	err = rockchip_pcie_init_irq_domain(rockchip);
 	if (err < 0)
-		goto err_vpcie;
+		goto err_deinit_port;
 
 	err = of_pci_get_host_bridge_resources(dev->of_node, 0, 0xff,
 					       &res, &io_base);
 	if (err)
-		goto err_vpcie;
+		goto err_remove_irq_domain;
 
 	err = devm_request_pci_bus_resources(dev, &res);
 	if (err)
@@ -1466,12 +1605,12 @@ static int rockchip_pcie_probe(struct platform_device *pdev)
 
 	err = rockchip_pcie_cfg_atu(rockchip);
 	if (err)
-		goto err_free_res;
+		goto err_unmap_iospace;
 
 	rockchip->msg_region = devm_ioremap(dev, rockchip->msg_bus_addr, SZ_1M);
 	if (!rockchip->msg_region) {
 		err = -ENOMEM;
-		goto err_free_res;
+		goto err_unmap_iospace;
 	}
 
 	list_splice_init(&res, &bridge->windows);
@@ -1484,7 +1623,7 @@ static int rockchip_pcie_probe(struct platform_device *pdev)
 
 	err = pci_scan_root_bus_bridge(bridge);
 	if (err < 0)
-		goto err_free_res;
+		goto err_unmap_iospace;
 
 	bus = bridge->bus;
 
@@ -1498,9 +1637,17 @@ static int rockchip_pcie_probe(struct platform_device *pdev)
 	pci_bus_add_devices(bus);
 	return 0;
 
+err_unmap_iospace:
+	pci_unmap_iospace(rockchip->io);
 err_free_res:
 	pci_free_resource_list(&res);
+err_remove_irq_domain:
+	irq_domain_remove(rockchip->irq_domain);
+err_deinit_port:
+	rockchip_pcie_deinit_phys(rockchip);
 err_vpcie:
+	if (!IS_ERR(rockchip->vpcie12v))
+		regulator_disable(rockchip->vpcie12v);
 	if (!IS_ERR(rockchip->vpcie3v3))
 		regulator_disable(rockchip->vpcie3v3);
 	if (!IS_ERR(rockchip->vpcie1v8))
@@ -1508,14 +1655,7 @@ err_vpcie:
 	if (!IS_ERR(rockchip->vpcie0v9))
 		regulator_disable(rockchip->vpcie0v9);
 err_set_vpcie:
-	clk_disable_unprepare(rockchip->clk_pcie_pm);
-err_pcie_pm:
-	clk_disable_unprepare(rockchip->hclk_pcie);
-err_hclk_pcie:
-	clk_disable_unprepare(rockchip->aclk_perf_pcie);
-err_aclk_perf_pcie:
-	clk_disable_unprepare(rockchip->aclk_pcie);
-err_aclk_pcie:
+	rockchip_pcie_disable_clocks(rockchip);
 	return err;
 }
 
@@ -1529,14 +1669,12 @@ static int rockchip_pcie_remove(struct platform_device *pdev)
 	pci_unmap_iospace(rockchip->io);
 	irq_domain_remove(rockchip->irq_domain);
 
-	phy_power_off(rockchip->phy);
-	phy_exit(rockchip->phy);
+	rockchip_pcie_deinit_phys(rockchip);
 
-	clk_disable_unprepare(rockchip->clk_pcie_pm);
-	clk_disable_unprepare(rockchip->hclk_pcie);
-	clk_disable_unprepare(rockchip->aclk_perf_pcie);
-	clk_disable_unprepare(rockchip->aclk_pcie);
+	rockchip_pcie_disable_clocks(rockchip);
 
+	if (!IS_ERR(rockchip->vpcie12v))
+		regulator_disable(rockchip->vpcie12v);
 	if (!IS_ERR(rockchip->vpcie3v3))
 		regulator_disable(rockchip->vpcie3v3);
 	if (!IS_ERR(rockchip->vpcie1v8))
diff --git a/drivers/pci/host/pcie-xilinx-nwl.c b/drivers/pci/host/pcie-xilinx-nwl.c
index eec641a34fc5..65dea98b2643 100644
--- a/drivers/pci/host/pcie-xilinx-nwl.c
+++ b/drivers/pci/host/pcie-xilinx-nwl.c
@@ -133,7 +133,6 @@
 #define CFG_DMA_REG_BAR			GENMASK(2, 0)
 
 #define INT_PCI_MSI_NR			(2 * 32)
-#define INTX_NUM			4
 
 /* Readin the PS_LINKUP */
 #define PS_LINKUP_OFFSET		0x00000238
@@ -334,9 +333,8 @@ static void nwl_pcie_leg_handler(struct irq_desc *desc)
 
 	while ((status = nwl_bridge_readl(pcie, MSGF_LEG_STATUS) &
 				MSGF_LEG_SR_MASKALL) != 0) {
-		for_each_set_bit(bit, &status, INTX_NUM) {
-			virq = irq_find_mapping(pcie->legacy_irq_domain,
-						bit + 1);
+		for_each_set_bit(bit, &status, PCI_NUM_INTX) {
+			virq = irq_find_mapping(pcie->legacy_irq_domain, bit);
 			if (virq)
 				generic_handle_irq(virq);
 		}
@@ -436,6 +434,7 @@ static int nwl_legacy_map(struct irq_domain *domain, unsigned int irq,
 
 static const struct irq_domain_ops legacy_domain_ops = {
 	.map = nwl_legacy_map,
+	.xlate = pci_irqd_intx_xlate,
 };
 
 #ifdef CONFIG_PCI_MSI
@@ -559,7 +558,7 @@ static int nwl_pcie_init_irq_domain(struct nwl_pcie *pcie)
 	}
 
 	pcie->legacy_irq_domain = irq_domain_add_linear(legacy_intc_node,
-							INTX_NUM,
+							PCI_NUM_INTX,
 							&legacy_domain_ops,
 							pcie);
 
@@ -813,7 +812,7 @@ static int nwl_pcie_parse_dt(struct nwl_pcie *pcie,
 	pcie->irq_intx = platform_get_irq_byname(pdev, "intx");
 	if (pcie->irq_intx < 0) {
 		dev_err(dev, "failed to get intx IRQ %d\n", pcie->irq_intx);
-		return -EINVAL;
+		return pcie->irq_intx;
 	}
 
 	irq_set_chained_handler_and_data(pcie->irq_intx,
diff --git a/drivers/pci/host/pcie-xilinx.c b/drivers/pci/host/pcie-xilinx.c
index f63fa5e0278c..94e13cb8608f 100644
--- a/drivers/pci/host/pcie-xilinx.c
+++ b/drivers/pci/host/pcie-xilinx.c
@@ -5,7 +5,7 @@
  *
  * Based on the Tegra PCIe driver
  *
- * Bits taken from Synopsys Designware Host controller driver and
+ * Bits taken from Synopsys DesignWare Host controller driver and
  * ARM PCI Host generic driver.
  *
  * This program is free software: you can redistribute it and/or modify
@@ -60,6 +60,7 @@
 #define XILINX_PCIE_INTR_MST_SLVERR	BIT(27)
 #define XILINX_PCIE_INTR_MST_ERRP	BIT(28)
 #define XILINX_PCIE_IMR_ALL_MASK	0x1FF30FED
+#define XILINX_PCIE_IMR_ENABLE_MASK	0x1FF30F0D
 #define XILINX_PCIE_IDR_ALL_MASK	0xFFFFFFFF
 
 /* Root Port Error FIFO Read Register definitions */
@@ -369,6 +370,7 @@ static int xilinx_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
 /* INTx IRQ Domain operations */
 static const struct irq_domain_ops intx_domain_ops = {
 	.map = xilinx_pcie_intx_map,
+	.xlate = pci_irqd_intx_xlate,
 };
 
 /* PCIe HW Functions */
@@ -384,7 +386,7 @@ static irqreturn_t xilinx_pcie_intr_handler(int irq, void *data)
 {
 	struct xilinx_pcie_port *port = (struct xilinx_pcie_port *)data;
 	struct device *dev = port->dev;
-	u32 val, mask, status, msi_data;
+	u32 val, mask, status;
 
 	/* Read interrupt decode and mask registers */
 	val = pcie_read(port, XILINX_PCIE_REG_IDR);
@@ -424,8 +426,7 @@ static irqreturn_t xilinx_pcie_intr_handler(int irq, void *data)
 		xilinx_pcie_clear_err_interrupts(port);
 	}
 
-	if (status & XILINX_PCIE_INTR_INTX) {
-		/* INTx interrupt received */
+	if (status & (XILINX_PCIE_INTR_INTX | XILINX_PCIE_INTR_MSI)) {
 		val = pcie_read(port, XILINX_PCIE_REG_RPIFR1);
 
 		/* Check whether interrupt valid */
@@ -434,41 +435,24 @@ static irqreturn_t xilinx_pcie_intr_handler(int irq, void *data)
 			goto error;
 		}
 
-		if (!(val & XILINX_PCIE_RPIFR1_MSI_INTR)) {
-			/* Clear interrupt FIFO register 1 */
-			pcie_write(port, XILINX_PCIE_RPIFR1_ALL_MASK,
-				   XILINX_PCIE_REG_RPIFR1);
-
-			/* Handle INTx Interrupt */
-			val = ((val & XILINX_PCIE_RPIFR1_INTR_MASK) >>
-				XILINX_PCIE_RPIFR1_INTR_SHIFT) + 1;
-			generic_handle_irq(irq_find_mapping(port->leg_domain,
-							    val));
-		}
-	}
-
-	if (status & XILINX_PCIE_INTR_MSI) {
-		/* MSI Interrupt */
-		val = pcie_read(port, XILINX_PCIE_REG_RPIFR1);
-
-		if (!(val & XILINX_PCIE_RPIFR1_INTR_VALID)) {
-			dev_warn(dev, "RP Intr FIFO1 read error\n");
-			goto error;
-		}
-
+		/* Decode the IRQ number */
 		if (val & XILINX_PCIE_RPIFR1_MSI_INTR) {
-			msi_data = pcie_read(port, XILINX_PCIE_REG_RPIFR2) &
-				   XILINX_PCIE_RPIFR2_MSG_DATA;
+			val = pcie_read(port, XILINX_PCIE_REG_RPIFR2) &
+				XILINX_PCIE_RPIFR2_MSG_DATA;
+		} else {
+			val = (val & XILINX_PCIE_RPIFR1_INTR_MASK) >>
+				XILINX_PCIE_RPIFR1_INTR_SHIFT;
+			val = irq_find_mapping(port->leg_domain, val);
+		}
 
-			/* Clear interrupt FIFO register 1 */
-			pcie_write(port, XILINX_PCIE_RPIFR1_ALL_MASK,
-				   XILINX_PCIE_REG_RPIFR1);
+		/* Clear interrupt FIFO register 1 */
+		pcie_write(port, XILINX_PCIE_RPIFR1_ALL_MASK,
+			   XILINX_PCIE_REG_RPIFR1);
 
-			if (IS_ENABLED(CONFIG_PCI_MSI)) {
-				/* Handle MSI Interrupt */
-				generic_handle_irq(msi_data);
-			}
-		}
+		/* Handle the interrupt */
+		if (IS_ENABLED(CONFIG_PCI_MSI) ||
+		    !(val & XILINX_PCIE_RPIFR1_MSI_INTR))
+			generic_handle_irq(val);
 	}
 
 	if (status & XILINX_PCIE_INTR_SLV_UNSUPP)
@@ -524,7 +508,7 @@ static int xilinx_pcie_init_irq_domain(struct xilinx_pcie_port *port)
 		return -ENODEV;
 	}
 
-	port->leg_domain = irq_domain_add_linear(pcie_intc_node, 4,
+	port->leg_domain = irq_domain_add_linear(pcie_intc_node, PCI_NUM_INTX,
 						 &intx_domain_ops,
 						 port);
 	if (!port->leg_domain) {
@@ -571,8 +555,8 @@ static void xilinx_pcie_init_port(struct xilinx_pcie_port *port)
 			 XILINX_PCIE_IMR_ALL_MASK,
 		   XILINX_PCIE_REG_IDR);
 
-	/* Enable all interrupts */
-	pcie_write(port, XILINX_PCIE_IMR_ALL_MASK, XILINX_PCIE_REG_IMR);
+	/* Enable all interrupts we handle */
+	pcie_write(port, XILINX_PCIE_IMR_ENABLE_MASK, XILINX_PCIE_REG_IMR);
 
 	/* Enable the Bridge enable bit */
 	pcie_write(port, pcie_read(port, XILINX_PCIE_REG_RPSC) |
diff --git a/drivers/pci/host/vmd.c b/drivers/pci/host/vmd.c
index 6088c3083194..509893bc3e63 100644
--- a/drivers/pci/host/vmd.c
+++ b/drivers/pci/host/vmd.c
@@ -183,7 +183,7 @@ static struct vmd_irq_list *vmd_next_irq(struct vmd_dev *vmd, struct msi_desc *d
 	int i, best = 1;
 	unsigned long flags;
 
-	if (!desc->msi_attrib.is_msix || vmd->msix_count == 1)
+	if (pci_is_bridge(msi_desc_to_pci_dev(desc)) || vmd->msix_count == 1)
 		return &vmd->irqs[0];
 
 	raw_spin_lock_irqsave(&list_lock, flags);
@@ -697,7 +697,7 @@ static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id)
 		return -ENODEV;
 
 	vmd->msix_count = pci_alloc_irq_vectors(dev, 1, vmd->msix_count,
-					PCI_IRQ_MSIX | PCI_IRQ_AFFINITY);
+					PCI_IRQ_MSIX);
 	if (vmd->msix_count < 0)
 		return vmd->msix_count;
 
@@ -755,6 +755,11 @@ static void vmd_remove(struct pci_dev *dev)
 static int vmd_suspend(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
+	struct vmd_dev *vmd = pci_get_drvdata(pdev);
+	int i;
+
+	for (i = 0; i < vmd->msix_count; i++)
+                devm_free_irq(dev, pci_irq_vector(pdev, i), &vmd->irqs[i]);
 
 	pci_save_state(pdev);
 	return 0;
@@ -763,6 +768,16 @@ static int vmd_suspend(struct device *dev)
 static int vmd_resume(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
+	struct vmd_dev *vmd = pci_get_drvdata(pdev);
+	int err, i;
+
+	for (i = 0; i < vmd->msix_count; i++) {
+		err = devm_request_irq(dev, pci_irq_vector(pdev, i),
+				       vmd_irq, IRQF_NO_THREAD,
+				       "vmd", &vmd->irqs[i]);
+		if (err)
+			return err;
+	}
 
 	pci_restore_state(pdev);
 	return 0;
diff --git a/drivers/pci/hotplug/cpcihp_zt5550.c b/drivers/pci/hotplug/cpcihp_zt5550.c
index 5f49c3fd736a..2f8659a148f5 100644
--- a/drivers/pci/hotplug/cpcihp_zt5550.c
+++ b/drivers/pci/hotplug/cpcihp_zt5550.c
@@ -280,7 +280,7 @@ static void zt5550_hc_remove_one(struct pci_dev *pdev)
 }
 
 
-static struct pci_device_id zt5550_hc_pci_tbl[] = {
+static const struct pci_device_id zt5550_hc_pci_tbl[] = {
 	{ PCI_VENDOR_ID_ZIATECH, PCI_DEVICE_ID_ZIATECH_5550_HC, PCI_ANY_ID, PCI_ANY_ID, },
 	{ 0, }
 };
diff --git a/drivers/pci/hotplug/cpqphp_core.c b/drivers/pci/hotplug/cpqphp_core.c
index 33d300d12411..4d06b8461255 100644
--- a/drivers/pci/hotplug/cpqphp_core.c
+++ b/drivers/pci/hotplug/cpqphp_core.c
@@ -1417,7 +1417,7 @@ static void __exit unload_cpqphpd(void)
 		iounmap(smbios_start);
 }
 
-static struct pci_device_id hpcd_pci_tbl[] = {
+static const struct pci_device_id hpcd_pci_tbl[] = {
 	{
 	/* handle any PCI Hotplug controller */
 	.class =        ((PCI_CLASS_SYSTEM_PCI_HOTPLUG << 8) | 0x00),
diff --git a/drivers/pci/hotplug/ibmphp_core.c b/drivers/pci/hotplug/ibmphp_core.c
index 5efd01d84498..73cf84645c82 100644
--- a/drivers/pci/hotplug/ibmphp_core.c
+++ b/drivers/pci/hotplug/ibmphp_core.c
@@ -852,7 +852,7 @@ static int set_bus(struct slot *slot_cur)
 	u8 speed;
 	u8 cmd = 0x0;
 	int retval;
-	static struct pci_device_id ciobx[] = {
+	static const struct pci_device_id ciobx[] = {
 		{ PCI_DEVICE(PCI_VENDOR_ID_SERVERWORKS, 0x0101) },
 		{ },
 	};
diff --git a/drivers/pci/hotplug/ibmphp_ebda.c b/drivers/pci/hotplug/ibmphp_ebda.c
index 43e345ac296b..a6a4dac798e5 100644
--- a/drivers/pci/hotplug/ibmphp_ebda.c
+++ b/drivers/pci/hotplug/ibmphp_ebda.c
@@ -1153,7 +1153,7 @@ void ibmphp_free_ebda_pci_rsrc_queue(void)
 	}
 }
 
-static struct pci_device_id id_table[] = {
+static const struct pci_device_id id_table[] = {
 	{
 		.vendor		= PCI_VENDOR_ID_IBM,
 		.device		= HPC_DEVICE_ID,
diff --git a/drivers/pci/hotplug/pciehp_hpc.c b/drivers/pci/hotplug/pciehp_hpc.c
index 026830a138ae..e5d5ce9e3010 100644
--- a/drivers/pci/hotplug/pciehp_hpc.c
+++ b/drivers/pci/hotplug/pciehp_hpc.c
@@ -586,6 +586,14 @@ static irqreturn_t pciehp_isr(int irq, void *dev_id)
 	events = status & (PCI_EXP_SLTSTA_ABP | PCI_EXP_SLTSTA_PFD |
 			   PCI_EXP_SLTSTA_PDC | PCI_EXP_SLTSTA_CC |
 			   PCI_EXP_SLTSTA_DLLSC);
+
+	/*
+	 * If we've already reported a power fault, don't report it again
+	 * until we've done something to handle it.
+	 */
+	if (ctrl->power_fault_detected)
+		events &= ~PCI_EXP_SLTSTA_PFD;
+
 	if (!events)
 		return IRQ_NONE;
 
diff --git a/drivers/pci/hotplug/pnv_php.c b/drivers/pci/hotplug/pnv_php.c
index 7c203198b582..74f6a17e4614 100644
--- a/drivers/pci/hotplug/pnv_php.c
+++ b/drivers/pci/hotplug/pnv_php.c
@@ -163,8 +163,8 @@ static void pnv_php_detach_device_nodes(struct device_node *parent)
 		of_node_put(dn);
 		refcount = kref_read(&dn->kobj.kref);
 		if (refcount != 1)
-			pr_warn("Invalid refcount %d on <%s>\n",
-				refcount, of_node_full_name(dn));
+			pr_warn("Invalid refcount %d on <%pOF>\n",
+				refcount, dn);
 
 		of_detach_node(dn);
 	}
diff --git a/drivers/pci/hotplug/rpadlpar_core.c b/drivers/pci/hotplug/rpadlpar_core.c
index 3f93a4e79595..a3449d717a99 100644
--- a/drivers/pci/hotplug/rpadlpar_core.c
+++ b/drivers/pci/hotplug/rpadlpar_core.c
@@ -150,8 +150,8 @@ static void dlpar_pci_add_bus(struct device_node *dn)
 	/* Add EADS device to PHB bus, adding new entry to bus->devices */
 	dev = of_create_pci_dev(dn, phb->bus, pdn->devfn);
 	if (!dev) {
-		printk(KERN_ERR "%s: failed to create pci dev for %s\n",
-				__func__, dn->full_name);
+		printk(KERN_ERR "%s: failed to create pci dev for %pOF\n",
+				__func__, dn);
 		return;
 	}
 
diff --git a/drivers/pci/hotplug/rpadlpar_sysfs.c b/drivers/pci/hotplug/rpadlpar_sysfs.c
index a796301ea03f..edb5d8a53020 100644
--- a/drivers/pci/hotplug/rpadlpar_sysfs.c
+++ b/drivers/pci/hotplug/rpadlpar_sysfs.c
@@ -102,7 +102,7 @@ static struct attribute *default_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group dlpar_attr_group = {
+static const struct attribute_group dlpar_attr_group = {
 	.attrs = default_attrs,
 };
 
diff --git a/drivers/pci/hotplug/rpaphp_core.c b/drivers/pci/hotplug/rpaphp_core.c
index 8d132024f06e..1e29abaaea08 100644
--- a/drivers/pci/hotplug/rpaphp_core.c
+++ b/drivers/pci/hotplug/rpaphp_core.c
@@ -318,7 +318,7 @@ int rpaphp_add_slot(struct device_node *dn)
 	if (!is_php_dn(dn, &indexes, &names, &types, &power_domains))
 		return 0;
 
-	dbg("Entry %s: dn->full_name=%s\n", __func__, dn->full_name);
+	dbg("Entry %s: dn=%pOF\n", __func__, dn);
 
 	/* register PCI devices */
 	name = (char *) &names[1];
diff --git a/drivers/pci/hotplug/rpaphp_pci.c b/drivers/pci/hotplug/rpaphp_pci.c
index ea41ea1d3c00..32aabc533be8 100644
--- a/drivers/pci/hotplug/rpaphp_pci.c
+++ b/drivers/pci/hotplug/rpaphp_pci.c
@@ -95,7 +95,7 @@ int rpaphp_enable_slot(struct slot *slot)
 
 	bus = pci_find_bus_by_node(slot->dn);
 	if (!bus) {
-		err("%s: no pci_bus for dn %s\n", __func__, slot->dn->full_name);
+		err("%s: no pci_bus for dn %pOF\n", __func__, slot->dn);
 		return -EINVAL;
 	}
 
@@ -125,7 +125,7 @@ int rpaphp_enable_slot(struct slot *slot)
 
 		if (rpaphp_debug) {
 			struct pci_dev *dev;
-			dbg("%s: pci_devs of slot[%s]\n", __func__, slot->dn->full_name);
+			dbg("%s: pci_devs of slot[%pOF]\n", __func__, slot->dn);
 			list_for_each_entry(dev, &bus->devices, bus_list)
 				dbg("\t%s\n", pci_name(dev));
 		}
diff --git a/drivers/pci/hotplug/rpaphp_slot.c b/drivers/pci/hotplug/rpaphp_slot.c
index 388c4d8fcdd1..489862360f2c 100644
--- a/drivers/pci/hotplug/rpaphp_slot.c
+++ b/drivers/pci/hotplug/rpaphp_slot.c
@@ -122,8 +122,8 @@ int rpaphp_register_slot(struct slot *slot)
 	int retval;
 	int slotno = -1;
 
-	dbg("%s registering slot:path[%s] index[%x], name[%s] pdomain[%x] type[%d]\n",
-		__func__, slot->dn->full_name, slot->index, slot->name,
+	dbg("%s registering slot:path[%pOF] index[%x], name[%s] pdomain[%x] type[%d]\n",
+		__func__, slot->dn, slot->index, slot->name,
 		slot->power_domain, slot->type);
 
 	/* should not try to register the same slot twice */
diff --git a/drivers/pci/hotplug/shpchp_core.c b/drivers/pci/hotplug/shpchp_core.c
index 3454dc7385f1..7bfb87bd2b7e 100644
--- a/drivers/pci/hotplug/shpchp_core.c
+++ b/drivers/pci/hotplug/shpchp_core.c
@@ -351,7 +351,7 @@ static void shpc_remove(struct pci_dev *dev)
 	kfree(ctrl);
 }
 
-static struct pci_device_id shpcd_pci_tbl[] = {
+static const struct pci_device_id shpcd_pci_tbl[] = {
 	{PCI_DEVICE_CLASS(((PCI_CLASS_BRIDGE_PCI << 8) | 0x00), ~0)},
 	{ /* end: all zeroes */ }
 };
diff --git a/drivers/pci/hotplug/shpchp_hpc.c b/drivers/pci/hotplug/shpchp_hpc.c
index de0ea474fb73..e5824c7b7b6b 100644
--- a/drivers/pci/hotplug/shpchp_hpc.c
+++ b/drivers/pci/hotplug/shpchp_hpc.c
@@ -1062,6 +1062,8 @@ int shpc_init(struct controller *ctrl, struct pci_dev *pdev)
 		if (rc) {
 			ctrl_info(ctrl, "Can't get msi for the hotplug controller\n");
 			ctrl_info(ctrl, "Use INTx for the hotplug controller\n");
+		} else {
+			pci_set_master(pdev);
 		}
 
 		rc = request_irq(ctrl->pci_dev->irq, shpc_isr, IRQF_SHARED,
diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
index 120485d6f352..ac41c8be9200 100644
--- a/drivers/pci/iov.c
+++ b/drivers/pci/iov.c
@@ -331,7 +331,6 @@ failed:
 	while (i--)
 		pci_iov_remove_virtfn(dev, i, 0);
 
-	pcibios_sriov_disable(dev);
 err_pcibios:
 	iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE);
 	pci_cfg_access_lock(dev);
@@ -339,6 +338,8 @@ err_pcibios:
 	ssleep(1);
 	pci_cfg_access_unlock(dev);
 
+	pcibios_sriov_disable(dev);
+
 	if (iov->link != dev->devfn)
 		sysfs_remove_link(&dev->dev.kobj, "dep_link");
 
@@ -357,14 +358,14 @@ static void sriov_disable(struct pci_dev *dev)
 	for (i = 0; i < iov->num_VFs; i++)
 		pci_iov_remove_virtfn(dev, i, 0);
 
-	pcibios_sriov_disable(dev);
-
 	iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE);
 	pci_cfg_access_lock(dev);
 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
 	ssleep(1);
 	pci_cfg_access_unlock(dev);
 
+	pcibios_sriov_disable(dev);
+
 	if (iov->link != dev->devfn)
 		sysfs_remove_link(&dev->dev.kobj, "dep_link");
 
diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
index 253d92409bb3..496ed9130600 100644
--- a/drivers/pci/msi.c
+++ b/drivers/pci/msi.c
@@ -538,12 +538,9 @@ msi_setup_entry(struct pci_dev *dev, int nvec, const struct irq_affinity *affd)
 	struct msi_desc *entry;
 	u16 control;
 
-	if (affd) {
+	if (affd)
 		masks = irq_create_affinity_masks(nvec, affd);
-		if (!masks)
-			dev_err(&dev->dev, "can't allocate MSI affinity masks for %d vectors\n",
-				nvec);
-	}
+
 
 	/* MSI Entry Initialization */
 	entry = alloc_msi_entry(&dev->dev, nvec, masks);
@@ -679,12 +676,8 @@ static int msix_setup_entries(struct pci_dev *dev, void __iomem *base,
 	struct msi_desc *entry;
 	int ret, i;
 
-	if (affd) {
+	if (affd)
 		masks = irq_create_affinity_masks(nvec, affd);
-		if (!masks)
-			dev_err(&dev->dev, "can't allocate MSI-X affinity masks for %d vectors\n",
-				nvec);
-	}
 
 	for (i = 0, curmsk = masks; i < nvec; i++) {
 		entry = alloc_msi_entry(&dev->dev, 1, curmsk);
@@ -1458,13 +1451,30 @@ struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode,
 }
 EXPORT_SYMBOL_GPL(pci_msi_create_irq_domain);
 
+/*
+ * Users of the generic MSI infrastructure expect a device to have a single ID,
+ * so with DMA aliases we have to pick the least-worst compromise. Devices with
+ * DMA phantom functions tend to still emit MSIs from the real function number,
+ * so we ignore those and only consider topological aliases where either the
+ * alias device or RID appears on a different bus number. We also make the
+ * reasonable assumption that bridges are walked in an upstream direction (so
+ * the last one seen wins), and the much braver assumption that the most likely
+ * case is that of PCI->PCIe so we should always use the alias RID. This echoes
+ * the logic from intel_irq_remapping's set_msi_sid(), which presumably works
+ * well enough in practice; in the face of the horrible PCIe<->PCI-X conditions
+ * for taking ownership all we can really do is close our eyes and hope...
+ */
 static int get_msi_id_cb(struct pci_dev *pdev, u16 alias, void *data)
 {
 	u32 *pa = data;
+	u8 bus = PCI_BUS_NUM(*pa);
+
+	if (pdev->bus->number != bus || PCI_BUS_NUM(alias) != bus)
+		*pa = alias;
 
-	*pa = alias;
 	return 0;
 }
+
 /**
  * pci_msi_domain_get_msi_rid - Get the MSI requester id (RID)
  * @domain:	The interrupt domain
@@ -1478,7 +1488,7 @@ static int get_msi_id_cb(struct pci_dev *pdev, u16 alias, void *data)
 u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev)
 {
 	struct device_node *of_node;
-	u32 rid = 0;
+	u32 rid = PCI_DEVID(pdev->bus->number, pdev->devfn);
 
 	pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid);
 
@@ -1494,14 +1504,14 @@ u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev)
  * @pdev:	The PCI device
  *
  * Use the firmware data to find a device-specific MSI domain
- * (i.e. not one that is ste as a default).
+ * (i.e. not one that is set as a default).
  *
- * Returns: The coresponding MSI domain or NULL if none has been found.
+ * Returns: The corresponding MSI domain or NULL if none has been found.
  */
 struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev)
 {
 	struct irq_domain *dom;
-	u32 rid = 0;
+	u32 rid = PCI_DEVID(pdev->bus->number, pdev->devfn);
 
 	pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid);
 	dom = of_msi_map_get_device_domain(&pdev->dev, rid);
diff --git a/drivers/pci/pci-acpi.c b/drivers/pci/pci-acpi.c
index e70c1c7ba1bf..a8da543b3814 100644
--- a/drivers/pci/pci-acpi.c
+++ b/drivers/pci/pci-acpi.c
@@ -573,7 +573,7 @@ static int acpi_pci_propagate_wakeup(struct pci_bus *bus, bool enable)
 {
 	while (bus->parent) {
 		if (acpi_pm_device_can_wakeup(&bus->self->dev))
-			return acpi_pm_set_device_wakeup(&bus->self->dev, enable);
+			return acpi_pm_set_bridge_wakeup(&bus->self->dev, enable);
 
 		bus = bus->parent;
 	}
@@ -581,7 +581,7 @@ static int acpi_pci_propagate_wakeup(struct pci_bus *bus, bool enable)
 	/* We have reached the root bus. */
 	if (bus->bridge) {
 		if (acpi_pm_device_can_wakeup(bus->bridge))
-			return acpi_pm_set_device_wakeup(bus->bridge, enable);
+			return acpi_pm_set_bridge_wakeup(bus->bridge, enable);
 	}
 	return 0;
 }
diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c
index d51e8738f9c2..11bd267fc137 100644
--- a/drivers/pci/pci-driver.c
+++ b/drivers/pci/pci-driver.c
@@ -647,9 +647,7 @@ static int pci_legacy_resume(struct device *dev)
 static void pci_pm_default_resume(struct pci_dev *pci_dev)
 {
 	pci_fixup_device(pci_fixup_resume, pci_dev);
-
-	if (!pci_has_subordinate(pci_dev))
-		pci_enable_wake(pci_dev, PCI_D0, false);
+	pci_enable_wake(pci_dev, PCI_D0, false);
 }
 
 static void pci_pm_default_suspend(struct pci_dev *pci_dev)
@@ -1307,6 +1305,7 @@ int __pci_register_driver(struct pci_driver *drv, struct module *owner,
 	drv->driver.bus = &pci_bus_type;
 	drv->driver.owner = owner;
 	drv->driver.mod_name = mod_name;
+	drv->driver.groups = drv->groups;
 
 	spin_lock_init(&drv->dynids.lock);
 	INIT_LIST_HEAD(&drv->dynids.list);
diff --git a/drivers/pci/pci-label.c b/drivers/pci/pci-label.c
index a7a41d9c29df..7e9e79575d93 100644
--- a/drivers/pci/pci-label.c
+++ b/drivers/pci/pci-label.c
@@ -123,7 +123,7 @@ static struct attribute *smbios_attributes[] = {
 	NULL,
 };
 
-static struct attribute_group smbios_attr_group = {
+static const struct attribute_group smbios_attr_group = {
 	.attrs = smbios_attributes,
 	.is_visible = smbios_instance_string_exist,
 };
@@ -260,7 +260,7 @@ static struct attribute *acpi_attributes[] = {
 	NULL,
 };
 
-static struct attribute_group acpi_attr_group = {
+static const struct attribute_group acpi_attr_group = {
 	.attrs = acpi_attributes,
 	.is_visible = acpi_index_string_exist,
 };
diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c
index 2f3780b50723..1eecfa301f7f 100644
--- a/drivers/pci/pci-sysfs.c
+++ b/drivers/pci/pci-sysfs.c
@@ -556,9 +556,9 @@ static ssize_t devspec_show(struct device *dev,
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct device_node *np = pci_device_to_OF_node(pdev);
 
-	if (np == NULL || np->full_name == NULL)
+	if (np == NULL)
 		return 0;
-	return sprintf(buf, "%s", np->full_name);
+	return sprintf(buf, "%pOF", np);
 }
 static DEVICE_ATTR_RO(devspec);
 #endif
@@ -1211,11 +1211,8 @@ static ssize_t pci_resource_io(struct file *filp, struct kobject *kobj,
 {
 	struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj));
 	int bar = (unsigned long)attr->private;
-	struct resource *res;
 	unsigned long port = off;
 
-	res = &pdev->resource[bar];
-
 	port += pci_resource_start(pdev, bar);
 
 	if (port > pci_resource_end(pdev, bar))
@@ -1431,7 +1428,7 @@ static ssize_t pci_read_rom(struct file *filp, struct kobject *kobj,
 	return count;
 }
 
-static struct bin_attribute pci_config_attr = {
+static const struct bin_attribute pci_config_attr = {
 	.attr =	{
 		.name = "config",
 		.mode = S_IRUGO | S_IWUSR,
@@ -1441,7 +1438,7 @@ static struct bin_attribute pci_config_attr = {
 	.write = pci_write_config,
 };
 
-static struct bin_attribute pcie_config_attr = {
+static const struct bin_attribute pcie_config_attr = {
 	.attr =	{
 		.name = "config",
 		.mode = S_IRUGO | S_IWUSR,
@@ -1735,7 +1732,7 @@ const struct attribute_group *pcie_dev_groups[] = {
 	NULL,
 };
 
-static struct attribute_group pci_dev_hp_attr_group = {
+static const struct attribute_group pci_dev_hp_attr_group = {
 	.attrs = pci_dev_hp_attrs,
 	.is_visible = pci_dev_hp_attrs_are_visible,
 };
@@ -1759,23 +1756,23 @@ static umode_t sriov_attrs_are_visible(struct kobject *kobj,
 	return a->mode;
 }
 
-static struct attribute_group sriov_dev_attr_group = {
+static const struct attribute_group sriov_dev_attr_group = {
 	.attrs = sriov_dev_attrs,
 	.is_visible = sriov_attrs_are_visible,
 };
 #endif /* CONFIG_PCI_IOV */
 
-static struct attribute_group pci_dev_attr_group = {
+static const struct attribute_group pci_dev_attr_group = {
 	.attrs = pci_dev_dev_attrs,
 	.is_visible = pci_dev_attrs_are_visible,
 };
 
-static struct attribute_group pci_bridge_attr_group = {
+static const struct attribute_group pci_bridge_attr_group = {
 	.attrs = pci_bridge_attrs,
 	.is_visible = pci_bridge_attrs_are_visible,
 };
 
-static struct attribute_group pcie_dev_attr_group = {
+static const struct attribute_group pcie_dev_attr_group = {
 	.attrs = pcie_dev_attrs,
 	.is_visible = pcie_dev_attrs_are_visible,
 };
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index b4b7eab29400..6078dfc11b11 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -514,7 +514,7 @@ EXPORT_SYMBOL(pci_find_resource);
  */
 struct pci_dev *pci_find_pcie_root_port(struct pci_dev *dev)
 {
-	struct pci_dev *bridge, *highest_pcie_bridge = NULL;
+	struct pci_dev *bridge, *highest_pcie_bridge = dev;
 
 	bridge = pci_upstream_bridge(dev);
 	while (bridge && pci_is_pcie(bridge)) {
@@ -892,7 +892,9 @@ EXPORT_SYMBOL_GPL(__pci_complete_power_transition);
  * -EINVAL if the requested state is invalid.
  * -EIO if device does not support PCI PM or its PM capabilities register has a
  * wrong version, or device doesn't support the requested state.
+ * 0 if the transition is to D1 or D2 but D1 and D2 are not supported.
  * 0 if device already is in the requested state.
+ * 0 if the transition is to D3 but D3 is not supported.
  * 0 if device's power state has been successfully changed.
  */
 int pci_set_power_state(struct pci_dev *dev, pci_power_t state)
@@ -1912,6 +1914,13 @@ int pci_enable_wake(struct pci_dev *dev, pci_power_t state, bool enable)
 {
 	int ret = 0;
 
+	/*
+	 * Bridges can only signal wakeup on behalf of subordinate devices,
+	 * but that is set up elsewhere, so skip them.
+	 */
+	if (pci_has_subordinate(dev))
+		return 0;
+
 	/* Don't do the same thing twice in a row for one device. */
 	if (!!enable == !!dev->wakeup_prepared)
 		return 0;
@@ -3811,27 +3820,49 @@ int pci_wait_for_pending_transaction(struct pci_dev *dev)
 }
 EXPORT_SYMBOL(pci_wait_for_pending_transaction);
 
-/*
- * We should only need to wait 100ms after FLR, but some devices take longer.
- * Wait for up to 1000ms for config space to return something other than -1.
- * Intel IGD requires this when an LCD panel is attached.  We read the 2nd
- * dword because VFs don't implement the 1st dword.
- */
 static void pci_flr_wait(struct pci_dev *dev)
 {
-	int i = 0;
+	int delay = 1, timeout = 60000;
 	u32 id;
 
-	do {
-		msleep(100);
+	/*
+	 * Per PCIe r3.1, sec 6.6.2, a device must complete an FLR within
+	 * 100ms, but may silently discard requests while the FLR is in
+	 * progress.  Wait 100ms before trying to access the device.
+	 */
+	msleep(100);
+
+	/*
+	 * After 100ms, the device should not silently discard config
+	 * requests, but it may still indicate that it needs more time by
+	 * responding to them with CRS completions.  The Root Port will
+	 * generally synthesize ~0 data to complete the read (except when
+	 * CRS SV is enabled and the read was for the Vendor ID; in that
+	 * case it synthesizes 0x0001 data).
+	 *
+	 * Wait for the device to return a non-CRS completion.  Read the
+	 * Command register instead of Vendor ID so we don't have to
+	 * contend with the CRS SV value.
+	 */
+	pci_read_config_dword(dev, PCI_COMMAND, &id);
+	while (id == ~0) {
+		if (delay > timeout) {
+			dev_warn(&dev->dev, "not ready %dms after FLR; giving up\n",
+				 100 + delay - 1);
+			return;
+		}
+
+		if (delay > 1000)
+			dev_info(&dev->dev, "not ready %dms after FLR; waiting\n",
+				 100 + delay - 1);
+
+		msleep(delay);
+		delay *= 2;
 		pci_read_config_dword(dev, PCI_COMMAND, &id);
-	} while (i++ < 10 && id == ~0);
+	}
 
-	if (id == ~0)
-		dev_warn(&dev->dev, "Failed to return from FLR\n");
-	else if (i > 1)
-		dev_info(&dev->dev, "Required additional %dms to return from FLR\n",
-			 (i - 1) * 100);
+	if (delay > 1000)
+		dev_info(&dev->dev, "ready %dms after FLR\n", 100 + delay - 1);
 }
 
 /**
@@ -5398,8 +5429,8 @@ static int of_pci_bus_find_domain_nr(struct device *parent)
 		use_dt_domains = 0;
 		domain = pci_get_new_domain_nr();
 	} else {
-		dev_err(parent, "Node %s has inconsistent \"linux,pci-domain\" property in DT\n",
-			parent->of_node->full_name);
+		dev_err(parent, "Node %pOF has inconsistent \"linux,pci-domain\" property in DT\n",
+			parent->of_node);
 		domain = -1;
 	}
 
diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
index 22e061738c6f..a6560c9baa52 100644
--- a/drivers/pci/pci.h
+++ b/drivers/pci/pci.h
@@ -235,6 +235,7 @@ enum pci_bar_type {
 	pci_bar_mem64,		/* A 64-bit memory BAR */
 };
 
+int pci_configure_extended_tags(struct pci_dev *dev, void *ign);
 bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *pl,
 				int crs_timeout);
 int pci_setup_device(struct pci_dev *dev);
diff --git a/drivers/pci/pcie/aer/aerdrv.c b/drivers/pci/pcie/aer/aerdrv.c
index dea186a9d6b6..6ff5f5b4f5e6 100644
--- a/drivers/pci/pcie/aer/aerdrv.c
+++ b/drivers/pci/pcie/aer/aerdrv.c
@@ -32,16 +32,9 @@
 
 static int aer_probe(struct pcie_device *dev);
 static void aer_remove(struct pcie_device *dev);
-static pci_ers_result_t aer_error_detected(struct pci_dev *dev,
-	enum pci_channel_state error);
 static void aer_error_resume(struct pci_dev *dev);
 static pci_ers_result_t aer_root_reset(struct pci_dev *dev);
 
-static const struct pci_error_handlers aer_error_handlers = {
-	.error_detected = aer_error_detected,
-	.resume		= aer_error_resume,
-};
-
 static struct pcie_port_service_driver aerdriver = {
 	.name		= "aer",
 	.port_type	= PCI_EXP_TYPE_ROOT_PORT,
@@ -49,9 +42,7 @@ static struct pcie_port_service_driver aerdriver = {
 
 	.probe		= aer_probe,
 	.remove		= aer_remove,
-
-	.err_handler	= &aer_error_handlers,
-
+	.error_resume	= aer_error_resume,
 	.reset_link	= aer_root_reset,
 };
 
@@ -350,20 +341,6 @@ static pci_ers_result_t aer_root_reset(struct pci_dev *dev)
 }
 
 /**
- * aer_error_detected - update severity status
- * @dev: pointer to Root Port's pci_dev data structure
- * @error: error severity being notified by port bus
- *
- * Invoked by Port Bus driver during error recovery.
- */
-static pci_ers_result_t aer_error_detected(struct pci_dev *dev,
-			enum pci_channel_state error)
-{
-	/* Root Port has no impact. Always recovers. */
-	return PCI_ERS_RESULT_CAN_RECOVER;
-}
-
-/**
  * aer_error_resume - clean up corresponding error status bits
  * @dev: pointer to Root Port's pci_dev data structure
  *
diff --git a/drivers/pci/pcie/aer/aerdrv_core.c b/drivers/pci/pcie/aer/aerdrv_core.c
index b1303b32053f..890efcc574cb 100644
--- a/drivers/pci/pcie/aer/aerdrv_core.c
+++ b/drivers/pci/pcie/aer/aerdrv_core.c
@@ -5,10 +5,10 @@
  * License.  See the file "COPYING" in the main directory of this archive
  * for more details.
  *
- * This file implements the core part of PCI-Express AER. When an pci-express
+ * This file implements the core part of PCIe AER. When a PCIe
  * error is delivered, an error message will be collected and printed to
  * console, then, an error recovery procedure will be executed by following
- * the pci error recovery rules.
+ * the PCI error recovery rules.
  *
  * Copyright (C) 2006 Intel Corp.
  *	Tom Long Nguyen (tom.l.nguyen@intel.com)
diff --git a/drivers/pci/pcie/pcie-dpc.c b/drivers/pci/pcie/pcie-dpc.c
index c39f32e42b4d..2d976a623ddc 100644
--- a/drivers/pci/pcie/pcie-dpc.c
+++ b/drivers/pci/pcie/pcie-dpc.c
@@ -16,17 +16,62 @@
 #include <linux/pcieport_if.h>
 #include "../pci.h"
 
+struct rp_pio_header_log_regs {
+	u32 dw0;
+	u32 dw1;
+	u32 dw2;
+	u32 dw3;
+};
+
+struct dpc_rp_pio_regs {
+	u32 status;
+	u32 mask;
+	u32 severity;
+	u32 syserror;
+	u32 exception;
+
+	struct rp_pio_header_log_regs header_log;
+	u32 impspec_log;
+	u32 tlp_prefix_log[4];
+	u32 log_size;
+	u16 first_error;
+};
+
 struct dpc_dev {
 	struct pcie_device	*dev;
 	struct work_struct	work;
 	int			cap_pos;
 	bool			rp;
+	u32			rp_pio_status;
+};
+
+static const char * const rp_pio_error_string[] = {
+	"Configuration Request received UR Completion",	 /* Bit Position 0  */
+	"Configuration Request received CA Completion",	 /* Bit Position 1  */
+	"Configuration Request Completion Timeout",	 /* Bit Position 2  */
+	NULL,
+	NULL,
+	NULL,
+	NULL,
+	NULL,
+	"I/O Request received UR Completion",		 /* Bit Position 8  */
+	"I/O Request received CA Completion",		 /* Bit Position 9  */
+	"I/O Request Completion Timeout",		 /* Bit Position 10 */
+	NULL,
+	NULL,
+	NULL,
+	NULL,
+	NULL,
+	"Memory Request received UR Completion",	 /* Bit Position 16 */
+	"Memory Request received CA Completion",	 /* Bit Position 17 */
+	"Memory Request Completion Timeout",		 /* Bit Position 18 */
 };
 
 static int dpc_wait_rp_inactive(struct dpc_dev *dpc)
 {
 	unsigned long timeout = jiffies + HZ;
 	struct pci_dev *pdev = dpc->dev->port;
+	struct device *dev = &dpc->dev->device;
 	u16 status;
 
 	pci_read_config_word(pdev, dpc->cap_pos + PCI_EXP_DPC_STATUS, &status);
@@ -36,15 +81,17 @@ static int dpc_wait_rp_inactive(struct dpc_dev *dpc)
 		pci_read_config_word(pdev, dpc->cap_pos + PCI_EXP_DPC_STATUS, &status);
 	}
 	if (status & PCI_EXP_DPC_RP_BUSY) {
-		dev_warn(&pdev->dev, "DPC root port still busy\n");
+		dev_warn(dev, "DPC root port still busy\n");
 		return -EBUSY;
 	}
 	return 0;
 }
 
-static void dpc_wait_link_inactive(struct pci_dev *pdev)
+static void dpc_wait_link_inactive(struct dpc_dev *dpc)
 {
 	unsigned long timeout = jiffies + HZ;
+	struct pci_dev *pdev = dpc->dev->port;
+	struct device *dev = &dpc->dev->device;
 	u16 lnk_status;
 
 	pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnk_status);
@@ -54,7 +101,7 @@ static void dpc_wait_link_inactive(struct pci_dev *pdev)
 		pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnk_status);
 	}
 	if (lnk_status & PCI_EXP_LNKSTA_DLLLA)
-		dev_warn(&pdev->dev, "Link state not disabled for DPC event\n");
+		dev_warn(dev, "Link state not disabled for DPC event\n");
 }
 
 static void interrupt_event_handler(struct work_struct *work)
@@ -76,17 +123,132 @@ static void interrupt_event_handler(struct work_struct *work)
 	}
 	pci_unlock_rescan_remove();
 
-	dpc_wait_link_inactive(pdev);
+	dpc_wait_link_inactive(dpc);
 	if (dpc->rp && dpc_wait_rp_inactive(dpc))
 		return;
+	if (dpc->rp && dpc->rp_pio_status) {
+		pci_write_config_dword(pdev,
+				      dpc->cap_pos + PCI_EXP_DPC_RP_PIO_STATUS,
+				      dpc->rp_pio_status);
+		dpc->rp_pio_status = 0;
+	}
+
 	pci_write_config_word(pdev, dpc->cap_pos + PCI_EXP_DPC_STATUS,
 		PCI_EXP_DPC_STATUS_TRIGGER | PCI_EXP_DPC_STATUS_INTERRUPT);
 }
 
+static void dpc_rp_pio_print_tlp_header(struct device *dev,
+					struct rp_pio_header_log_regs *t)
+{
+	dev_err(dev, "TLP Header: %#010x %#010x %#010x %#010x\n",
+		t->dw0, t->dw1, t->dw2, t->dw3);
+}
+
+static void dpc_rp_pio_print_error(struct dpc_dev *dpc,
+				   struct dpc_rp_pio_regs *rp_pio)
+{
+	struct device *dev = &dpc->dev->device;
+	int i;
+	u32 status;
+
+	dev_err(dev, "rp_pio_status: %#010x, rp_pio_mask: %#010x\n",
+		rp_pio->status, rp_pio->mask);
+
+	dev_err(dev, "RP PIO severity=%#010x, syserror=%#010x, exception=%#010x\n",
+		rp_pio->severity, rp_pio->syserror, rp_pio->exception);
+
+	status = (rp_pio->status & ~rp_pio->mask);
+
+	for (i = 0; i < ARRAY_SIZE(rp_pio_error_string); i++) {
+		if (!(status & (1 << i)))
+			continue;
+
+		dev_err(dev, "[%2d] %s%s\n", i, rp_pio_error_string[i],
+			rp_pio->first_error == i ? " (First)" : "");
+	}
+
+	dpc_rp_pio_print_tlp_header(dev, &rp_pio->header_log);
+	if (rp_pio->log_size == 4)
+		return;
+	dev_err(dev, "RP PIO ImpSpec Log %#010x\n", rp_pio->impspec_log);
+
+	for (i = 0; i < rp_pio->log_size - 5; i++)
+		dev_err(dev, "TLP Prefix Header: dw%d, %#010x\n", i,
+			rp_pio->tlp_prefix_log[i]);
+}
+
+static void dpc_rp_pio_get_info(struct dpc_dev *dpc,
+				struct dpc_rp_pio_regs *rp_pio)
+{
+	struct pci_dev *pdev = dpc->dev->port;
+	struct device *dev = &dpc->dev->device;
+	int i;
+	u16 cap;
+	u16 status;
+
+	pci_read_config_dword(pdev, dpc->cap_pos + PCI_EXP_DPC_RP_PIO_STATUS,
+			      &rp_pio->status);
+	pci_read_config_dword(pdev, dpc->cap_pos + PCI_EXP_DPC_RP_PIO_MASK,
+			      &rp_pio->mask);
+
+	pci_read_config_dword(pdev, dpc->cap_pos + PCI_EXP_DPC_RP_PIO_SEVERITY,
+			      &rp_pio->severity);
+	pci_read_config_dword(pdev, dpc->cap_pos + PCI_EXP_DPC_RP_PIO_SYSERROR,
+			      &rp_pio->syserror);
+	pci_read_config_dword(pdev, dpc->cap_pos + PCI_EXP_DPC_RP_PIO_EXCEPTION,
+			      &rp_pio->exception);
+
+	/* Get First Error Pointer */
+	pci_read_config_word(pdev, dpc->cap_pos + PCI_EXP_DPC_STATUS, &status);
+	rp_pio->first_error = (status & 0x1f00) >> 8;
+
+	pci_read_config_word(pdev, dpc->cap_pos + PCI_EXP_DPC_CAP, &cap);
+	rp_pio->log_size = (cap & PCI_EXP_DPC_RP_PIO_LOG_SIZE) >> 8;
+	if (rp_pio->log_size < 4 || rp_pio->log_size > 9) {
+		dev_err(dev, "RP PIO log size %u is invalid\n",
+			rp_pio->log_size);
+		return;
+	}
+
+	pci_read_config_dword(pdev,
+			      dpc->cap_pos + PCI_EXP_DPC_RP_PIO_HEADER_LOG,
+			      &rp_pio->header_log.dw0);
+	pci_read_config_dword(pdev,
+			      dpc->cap_pos + PCI_EXP_DPC_RP_PIO_HEADER_LOG + 4,
+			      &rp_pio->header_log.dw1);
+	pci_read_config_dword(pdev,
+			      dpc->cap_pos + PCI_EXP_DPC_RP_PIO_HEADER_LOG + 8,
+			      &rp_pio->header_log.dw2);
+	pci_read_config_dword(pdev,
+			      dpc->cap_pos + PCI_EXP_DPC_RP_PIO_HEADER_LOG + 12,
+			      &rp_pio->header_log.dw3);
+	if (rp_pio->log_size == 4)
+		return;
+
+	pci_read_config_dword(pdev,
+			      dpc->cap_pos + PCI_EXP_DPC_RP_PIO_IMPSPEC_LOG,
+			      &rp_pio->impspec_log);
+	for (i = 0; i < rp_pio->log_size - 5; i++)
+		pci_read_config_dword(pdev,
+			dpc->cap_pos + PCI_EXP_DPC_RP_PIO_TLPPREFIX_LOG,
+			&rp_pio->tlp_prefix_log[i]);
+}
+
+static void dpc_process_rp_pio_error(struct dpc_dev *dpc)
+{
+	struct dpc_rp_pio_regs rp_pio_regs;
+
+	dpc_rp_pio_get_info(dpc, &rp_pio_regs);
+	dpc_rp_pio_print_error(dpc, &rp_pio_regs);
+
+	dpc->rp_pio_status = rp_pio_regs.status;
+}
+
 static irqreturn_t dpc_irq(int irq, void *context)
 {
 	struct dpc_dev *dpc = (struct dpc_dev *)context;
 	struct pci_dev *pdev = dpc->dev->port;
+	struct device *dev = &dpc->dev->device;
 	u16 status, source;
 
 	pci_read_config_word(pdev, dpc->cap_pos + PCI_EXP_DPC_STATUS, &status);
@@ -95,20 +257,24 @@ static irqreturn_t dpc_irq(int irq, void *context)
 	if (!status || status == (u16)(~0))
 		return IRQ_NONE;
 
-	dev_info(&dpc->dev->device, "DPC containment event, status:%#06x source:%#06x\n",
+	dev_info(dev, "DPC containment event, status:%#06x source:%#06x\n",
 		status, source);
 
 	if (status & PCI_EXP_DPC_STATUS_TRIGGER) {
 		u16 reason = (status >> 1) & 0x3;
 		u16 ext_reason = (status >> 5) & 0x3;
 
-		dev_warn(&dpc->dev->device, "DPC %s detected, remove downstream devices\n",
+		dev_warn(dev, "DPC %s detected, remove downstream devices\n",
 			 (reason == 0) ? "unmasked uncorrectable error" :
 			 (reason == 1) ? "ERR_NONFATAL" :
 			 (reason == 2) ? "ERR_FATAL" :
 			 (ext_reason == 0) ? "RP PIO error" :
 			 (ext_reason == 1) ? "software trigger" :
 					     "reserved error");
+		/* show RP PIO error detail information */
+		if (reason == 3 && ext_reason == 0)
+			dpc_process_rp_pio_error(dpc);
+
 		schedule_work(&dpc->work);
 	}
 	return IRQ_HANDLED;
@@ -119,10 +285,11 @@ static int dpc_probe(struct pcie_device *dev)
 {
 	struct dpc_dev *dpc;
 	struct pci_dev *pdev = dev->port;
+	struct device *device = &dev->device;
 	int status;
 	u16 ctl, cap;
 
-	dpc = devm_kzalloc(&dev->device, sizeof(*dpc), GFP_KERNEL);
+	dpc = devm_kzalloc(device, sizeof(*dpc), GFP_KERNEL);
 	if (!dpc)
 		return -ENOMEM;
 
@@ -131,10 +298,10 @@ static int dpc_probe(struct pcie_device *dev)
 	INIT_WORK(&dpc->work, interrupt_event_handler);
 	set_service_data(dev, dpc);
 
-	status = devm_request_irq(&dev->device, dev->irq, dpc_irq, IRQF_SHARED,
+	status = devm_request_irq(device, dev->irq, dpc_irq, IRQF_SHARED,
 				  "pcie-dpc", dpc);
 	if (status) {
-		dev_warn(&dev->device, "request IRQ%d failed: %d\n", dev->irq,
+		dev_warn(device, "request IRQ%d failed: %d\n", dev->irq,
 			 status);
 		return status;
 	}
@@ -147,7 +314,7 @@ static int dpc_probe(struct pcie_device *dev)
 	ctl = (ctl & 0xfff4) | PCI_EXP_DPC_CTL_EN_NONFATAL | PCI_EXP_DPC_CTL_INT_EN;
 	pci_write_config_word(pdev, dpc->cap_pos + PCI_EXP_DPC_CTL, ctl);
 
-	dev_info(&dev->device, "DPC error containment capabilities: Int Msg #%d, RPExt%c PoisonedTLP%c SwTrigger%c RP PIO Log %d, DL_ActiveErr%c\n",
+	dev_info(device, "DPC error containment capabilities: Int Msg #%d, RPExt%c PoisonedTLP%c SwTrigger%c RP PIO Log %d, DL_ActiveErr%c\n",
 		cap & 0xf, FLAG(cap, PCI_EXP_DPC_CAP_RP_EXT),
 		FLAG(cap, PCI_EXP_DPC_CAP_POISONED_TLP),
 		FLAG(cap, PCI_EXP_DPC_CAP_SW_TRIGGER), (cap >> 8) & 0xf,
diff --git a/drivers/pci/pcie/portdrv_pci.c b/drivers/pci/pcie/portdrv_pci.c
index 8aa3f14bc87d..083276e03c38 100644
--- a/drivers/pci/pcie/portdrv_pci.c
+++ b/drivers/pci/pcie/portdrv_pci.c
@@ -21,7 +21,6 @@
 
 #include "../pci.h"
 #include "portdrv.h"
-#include "aer/aerdrv.h"
 
 /* If this switch is set, PCIe port native services should not be enabled. */
 bool pcie_ports_disabled;
@@ -177,108 +176,20 @@ static void pcie_portdrv_remove(struct pci_dev *dev)
 	pcie_port_device_remove(dev);
 }
 
-static int error_detected_iter(struct device *device, void *data)
-{
-	struct pcie_device *pcie_device;
-	struct pcie_port_service_driver *driver;
-	struct aer_broadcast_data *result_data;
-	pci_ers_result_t status;
-
-	result_data = (struct aer_broadcast_data *) data;
-
-	if (device->bus == &pcie_port_bus_type && device->driver) {
-		driver = to_service_driver(device->driver);
-		if (!driver ||
-			!driver->err_handler ||
-			!driver->err_handler->error_detected)
-			return 0;
-
-		pcie_device = to_pcie_device(device);
-
-		/* Forward error detected message to service drivers */
-		status = driver->err_handler->error_detected(
-			pcie_device->port,
-			result_data->state);
-		result_data->result =
-			merge_result(result_data->result, status);
-	}
-
-	return 0;
-}
-
 static pci_ers_result_t pcie_portdrv_error_detected(struct pci_dev *dev,
 					enum pci_channel_state error)
 {
-	struct aer_broadcast_data data = {error, PCI_ERS_RESULT_CAN_RECOVER};
-
-	/* get true return value from &data */
-	device_for_each_child(&dev->dev, &data, error_detected_iter);
-	return data.result;
-}
-
-static int mmio_enabled_iter(struct device *device, void *data)
-{
-	struct pcie_device *pcie_device;
-	struct pcie_port_service_driver *driver;
-	pci_ers_result_t status, *result;
-
-	result = (pci_ers_result_t *) data;
-
-	if (device->bus == &pcie_port_bus_type && device->driver) {
-		driver = to_service_driver(device->driver);
-		if (driver &&
-			driver->err_handler &&
-			driver->err_handler->mmio_enabled) {
-			pcie_device = to_pcie_device(device);
-
-			/* Forward error message to service drivers */
-			status = driver->err_handler->mmio_enabled(
-					pcie_device->port);
-			*result = merge_result(*result, status);
-		}
-	}
-
-	return 0;
+	/* Root Port has no impact. Always recovers. */
+	return PCI_ERS_RESULT_CAN_RECOVER;
 }
 
 static pci_ers_result_t pcie_portdrv_mmio_enabled(struct pci_dev *dev)
 {
-	pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED;
-
-	/* get true return value from &status */
-	device_for_each_child(&dev->dev, &status, mmio_enabled_iter);
-	return status;
-}
-
-static int slot_reset_iter(struct device *device, void *data)
-{
-	struct pcie_device *pcie_device;
-	struct pcie_port_service_driver *driver;
-	pci_ers_result_t status, *result;
-
-	result = (pci_ers_result_t *) data;
-
-	if (device->bus == &pcie_port_bus_type && device->driver) {
-		driver = to_service_driver(device->driver);
-		if (driver &&
-			driver->err_handler &&
-			driver->err_handler->slot_reset) {
-			pcie_device = to_pcie_device(device);
-
-			/* Forward error message to service drivers */
-			status = driver->err_handler->slot_reset(
-					pcie_device->port);
-			*result = merge_result(*result, status);
-		}
-	}
-
-	return 0;
+	return PCI_ERS_RESULT_RECOVERED;
 }
 
 static pci_ers_result_t pcie_portdrv_slot_reset(struct pci_dev *dev)
 {
-	pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED;
-
 	/* If fatal, restore cfg space for possible link reset at upstream */
 	if (dev->error_state == pci_channel_io_frozen) {
 		dev->state_saved = true;
@@ -287,9 +198,7 @@ static pci_ers_result_t pcie_portdrv_slot_reset(struct pci_dev *dev)
 		pci_enable_pcie_error_reporting(dev);
 	}
 
-	/* get true return value from &status */
-	device_for_each_child(&dev->dev, &status, slot_reset_iter);
-	return status;
+	return PCI_ERS_RESULT_RECOVERED;
 }
 
 static int resume_iter(struct device *device, void *data)
@@ -299,13 +208,11 @@ static int resume_iter(struct device *device, void *data)
 
 	if (device->bus == &pcie_port_bus_type && device->driver) {
 		driver = to_service_driver(device->driver);
-		if (driver &&
-			driver->err_handler &&
-			driver->err_handler->resume) {
+		if (driver && driver->error_resume) {
 			pcie_device = to_pcie_device(device);
 
 			/* Forward error message to service drivers */
-			driver->err_handler->resume(pcie_device->port);
+			driver->error_resume(pcie_device->port);
 		}
 	}
 
@@ -353,7 +260,7 @@ static int __init dmi_pcie_pme_disable_msi(const struct dmi_system_id *d)
 	return 0;
 }
 
-static struct dmi_system_id __initdata pcie_portdrv_dmi_table[] = {
+static const struct dmi_system_id pcie_portdrv_dmi_table[] __initconst = {
 	/*
 	 * Boxes that should not use MSI for PCIe PME signaling.
 	 */
diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
index c31310db0404..ff94b69738a8 100644
--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -1745,21 +1745,92 @@ static void program_hpp_type2(struct pci_dev *dev, struct hpp_type2 *hpp)
 	 */
 }
 
-static void pci_configure_extended_tags(struct pci_dev *dev)
+int pci_configure_extended_tags(struct pci_dev *dev, void *ign)
 {
-	u32 dev_cap;
+	struct pci_host_bridge *host;
+	u32 cap;
+	u16 ctl;
 	int ret;
 
 	if (!pci_is_pcie(dev))
-		return;
+		return 0;
 
-	ret = pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, &dev_cap);
+	ret = pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, &cap);
 	if (ret)
-		return;
+		return 0;
+
+	if (!(cap & PCI_EXP_DEVCAP_EXT_TAG))
+		return 0;
 
-	if (dev_cap & PCI_EXP_DEVCAP_EXT_TAG)
+	ret = pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl);
+	if (ret)
+		return 0;
+
+	host = pci_find_host_bridge(dev->bus);
+	if (!host)
+		return 0;
+
+	/*
+	 * If some device in the hierarchy doesn't handle Extended Tags
+	 * correctly, make sure they're disabled.
+	 */
+	if (host->no_ext_tags) {
+		if (ctl & PCI_EXP_DEVCTL_EXT_TAG) {
+			dev_info(&dev->dev, "disabling Extended Tags\n");
+			pcie_capability_clear_word(dev, PCI_EXP_DEVCTL,
+						   PCI_EXP_DEVCTL_EXT_TAG);
+		}
+		return 0;
+	}
+
+	if (!(ctl & PCI_EXP_DEVCTL_EXT_TAG)) {
+		dev_info(&dev->dev, "enabling Extended Tags\n");
 		pcie_capability_set_word(dev, PCI_EXP_DEVCTL,
 					 PCI_EXP_DEVCTL_EXT_TAG);
+	}
+	return 0;
+}
+
+/**
+ * pcie_relaxed_ordering_enabled - Probe for PCIe relaxed ordering enable
+ * @dev: PCI device to query
+ *
+ * Returns true if the device has enabled relaxed ordering attribute.
+ */
+bool pcie_relaxed_ordering_enabled(struct pci_dev *dev)
+{
+	u16 v;
+
+	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &v);
+
+	return !!(v & PCI_EXP_DEVCTL_RELAX_EN);
+}
+EXPORT_SYMBOL(pcie_relaxed_ordering_enabled);
+
+static void pci_configure_relaxed_ordering(struct pci_dev *dev)
+{
+	struct pci_dev *root;
+
+	/* PCI_EXP_DEVICE_RELAX_EN is RsvdP in VFs */
+	if (dev->is_virtfn)
+		return;
+
+	if (!pcie_relaxed_ordering_enabled(dev))
+		return;
+
+	/*
+	 * For now, we only deal with Relaxed Ordering issues with Root
+	 * Ports. Peer-to-Peer DMA is another can of worms.
+	 */
+	root = pci_find_pcie_root_port(dev);
+	if (!root)
+		return;
+
+	if (root->dev_flags & PCI_DEV_FLAGS_NO_RELAXED_ORDERING) {
+		pcie_capability_clear_word(dev, PCI_EXP_DEVCTL,
+					   PCI_EXP_DEVCTL_RELAX_EN);
+		dev_info(&dev->dev, "Disable Relaxed Ordering because the Root Port didn't support it\n");
+	}
 }
 
 static void pci_configure_device(struct pci_dev *dev)
@@ -1768,7 +1839,8 @@ static void pci_configure_device(struct pci_dev *dev)
 	int ret;
 
 	pci_configure_mps(dev);
-	pci_configure_extended_tags(dev);
+	pci_configure_extended_tags(dev, NULL);
+	pci_configure_relaxed_ordering(dev);
 
 	memset(&hpp, 0, sizeof(hpp));
 	ret = pci_get_hp_params(dev, &hpp);
@@ -1824,42 +1896,69 @@ struct pci_dev *pci_alloc_dev(struct pci_bus *bus)
 }
 EXPORT_SYMBOL(pci_alloc_dev);
 
-bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
-				int crs_timeout)
+static bool pci_bus_crs_vendor_id(u32 l)
+{
+	return (l & 0xffff) == 0x0001;
+}
+
+static bool pci_bus_wait_crs(struct pci_bus *bus, int devfn, u32 *l,
+			     int timeout)
 {
 	int delay = 1;
 
-	if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, l))
-		return false;
+	if (!pci_bus_crs_vendor_id(*l))
+		return true;	/* not a CRS completion */
 
-	/* some broken boards return 0 or ~0 if a slot is empty: */
-	if (*l == 0xffffffff || *l == 0x00000000 ||
-	    *l == 0x0000ffff || *l == 0xffff0000)
-		return false;
+	if (!timeout)
+		return false;	/* CRS, but caller doesn't want to wait */
 
 	/*
-	 * Configuration Request Retry Status.  Some root ports return the
-	 * actual device ID instead of the synthetic ID (0xFFFF) required
-	 * by the PCIe spec.  Ignore the device ID and only check for
-	 * (vendor id == 1).
+	 * We got the reserved Vendor ID that indicates a completion with
+	 * Configuration Request Retry Status (CRS).  Retry until we get a
+	 * valid Vendor ID or we time out.
 	 */
-	while ((*l & 0xffff) == 0x0001) {
-		if (!crs_timeout)
+	while (pci_bus_crs_vendor_id(*l)) {
+		if (delay > timeout) {
+			pr_warn("pci %04x:%02x:%02x.%d: not ready after %dms; giving up\n",
+				pci_domain_nr(bus), bus->number,
+				PCI_SLOT(devfn), PCI_FUNC(devfn), delay - 1);
+
 			return false;
+		}
+		if (delay >= 1000)
+			pr_info("pci %04x:%02x:%02x.%d: not ready after %dms; waiting\n",
+				pci_domain_nr(bus), bus->number,
+				PCI_SLOT(devfn), PCI_FUNC(devfn), delay - 1);
 
 		msleep(delay);
 		delay *= 2;
+
 		if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, l))
 			return false;
-		/* Card hasn't responded in 60 seconds?  Must be stuck. */
-		if (delay > crs_timeout) {
-			printk(KERN_WARNING "pci %04x:%02x:%02x.%d: not responding\n",
-			       pci_domain_nr(bus), bus->number, PCI_SLOT(devfn),
-			       PCI_FUNC(devfn));
-			return false;
-		}
 	}
 
+	if (delay >= 1000)
+		pr_info("pci %04x:%02x:%02x.%d: ready after %dms\n",
+			pci_domain_nr(bus), bus->number,
+			PCI_SLOT(devfn), PCI_FUNC(devfn), delay - 1);
+
+	return true;
+}
+
+bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
+				int timeout)
+{
+	if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, l))
+		return false;
+
+	/* some broken boards return 0 or ~0 if a slot is empty: */
+	if (*l == 0xffffffff || *l == 0x00000000 ||
+	    *l == 0x0000ffff || *l == 0xffff0000)
+		return false;
+
+	if (pci_bus_crs_vendor_id(*l))
+		return pci_bus_wait_crs(bus, devfn, l, timeout);
+
 	return true;
 }
 EXPORT_SYMBOL(pci_bus_read_dev_vendor_id);
@@ -2288,6 +2387,15 @@ void pcie_bus_configure_settings(struct pci_bus *bus)
 }
 EXPORT_SYMBOL_GPL(pcie_bus_configure_settings);
 
+/*
+ * Called after each bus is probed, but before its children are examined.  This
+ * is marked as __weak because multiple architectures define it.
+ */
+void __weak pcibios_fixup_bus(struct pci_bus *bus)
+{
+       /* nothing to do, expected to be removed in the future */
+}
+
 unsigned int pci_scan_child_bus(struct pci_bus *bus)
 {
 	unsigned int devfn, pass, max = bus->busn_res.start;
diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
index 6967c6b4cf6b..a4d33619a7bb 100644
--- a/drivers/pci/quirks.c
+++ b/drivers/pci/quirks.c
@@ -25,6 +25,7 @@
 #include <linux/sched.h>
 #include <linux/ktime.h>
 #include <linux/mm.h>
+#include <linux/platform_data/x86/apple.h>
 #include <asm/dma.h>	/* isa_dma_bridge_buggy */
 #include "pci.h"
 
@@ -1706,7 +1707,7 @@ static int dmi_disable_ioapicreroute(const struct dmi_system_id *d)
 	return 0;
 }
 
-static struct dmi_system_id boot_interrupt_dmi_table[] = {
+static const struct dmi_system_id boot_interrupt_dmi_table[] = {
 	/*
 	 * Systems to exclude from boot interrupt reroute quirks
 	 */
@@ -2061,7 +2062,7 @@ DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_INTEL, PCI_ANY_ID,
 
 /*
  * The 82575 and 82598 may experience data corruption issues when transitioning
- * out of L0S.  To prevent this we need to disable L0S on the pci-e link
+ * out of L0S.  To prevent this we need to disable L0S on the PCIe link.
  */
 static void quirk_disable_aspm_l0s(struct pci_dev *dev)
 {
@@ -3447,7 +3448,7 @@ static void quirk_apple_poweroff_thunderbolt(struct pci_dev *dev)
 {
 	acpi_handle bridge, SXIO, SXFP, SXLV;
 
-	if (!dmi_match(DMI_BOARD_VENDOR, "Apple Inc."))
+	if (!x86_apple_machine)
 		return;
 	if (pci_pcie_type(dev) != PCI_EXP_TYPE_UPSTREAM)
 		return;
@@ -3492,7 +3493,7 @@ static void quirk_apple_wait_for_thunderbolt(struct pci_dev *dev)
 	struct pci_dev *sibling = NULL;
 	struct pci_dev *nhi = NULL;
 
-	if (!dmi_match(DMI_BOARD_VENDOR, "Apple Inc."))
+	if (!x86_apple_machine)
 		return;
 	if (pci_pcie_type(dev) != PCI_EXP_TYPE_DOWNSTREAM)
 		return;
@@ -4016,6 +4017,95 @@ DECLARE_PCI_FIXUP_CLASS_EARLY(0x1797, 0x6869, PCI_CLASS_NOT_DEFINED, 8,
 			      quirk_tw686x_class);
 
 /*
+ * Some devices have problems with Transaction Layer Packets with the Relaxed
+ * Ordering Attribute set.  Such devices should mark themselves and other
+ * Device Drivers should check before sending TLPs with RO set.
+ */
+static void quirk_relaxedordering_disable(struct pci_dev *dev)
+{
+	dev->dev_flags |= PCI_DEV_FLAGS_NO_RELAXED_ORDERING;
+	dev_info(&dev->dev, "Disable Relaxed Ordering Attributes to avoid PCIe Completion erratum\n");
+}
+
+/*
+ * Intel Xeon processors based on Broadwell/Haswell microarchitecture Root
+ * Complex has a Flow Control Credit issue which can cause performance
+ * problems with Upstream Transaction Layer Packets with Relaxed Ordering set.
+ */
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x6f01, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x6f02, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x6f03, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x6f04, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x6f05, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x6f06, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x6f07, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x6f08, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x6f09, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x6f0a, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x6f0b, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x6f0c, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x6f0d, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x6f0e, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x2f01, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x2f02, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x2f03, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x2f04, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x2f05, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x2f06, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x2f07, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x2f08, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x2f09, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x2f0a, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x2f0b, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x2f0c, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x2f0d, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, 0x2f0e, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+
+/*
+ * The AMD ARM A1100 (AKA "SEATTLE") SoC has a bug in its PCIe Root Complex
+ * where Upstream Transaction Layer Packets with the Relaxed Ordering
+ * Attribute clear are allowed to bypass earlier TLPs with Relaxed Ordering
+ * set.  This is a violation of the PCIe 3.0 Transaction Ordering Rules
+ * outlined in Section 2.4.1 (PCI Express(r) Base Specification Revision 3.0
+ * November 10, 2010).  As a result, on this platform we can't use Relaxed
+ * Ordering for Upstream TLPs.
+ */
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_AMD, 0x1a00, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_AMD, 0x1a01, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_AMD, 0x1a02, PCI_CLASS_NOT_DEFINED, 8,
+			      quirk_relaxedordering_disable);
+
+/*
  * Per PCIe r3.0, sec 2.2.9, "Completion headers must supply the same
  * values for the Attribute as were supplied in the header of the
  * corresponding Request, except as explicitly allowed when IDO is used."
@@ -4137,6 +4227,18 @@ static int pci_quirk_cavium_acs(struct pci_dev *dev, u16 acs_flags)
 	return acs_flags ? 0 : 1;
 }
 
+static int pci_quirk_xgene_acs(struct pci_dev *dev, u16 acs_flags)
+{
+	/*
+	 * X-Gene root matching this quirk do not allow peer-to-peer
+	 * transactions with others, allowing masking out these bits as if they
+	 * were unimplemented in the ACS capability.
+	 */
+	acs_flags &= ~(PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF);
+
+	return acs_flags ? 0 : 1;
+}
+
 /*
  * Many Intel PCH root ports do provide ACS-like features to disable peer
  * transactions and validate bus numbers in requests, but do not provide an
@@ -4385,6 +4487,8 @@ static const struct pci_dev_acs_enabled {
 	{ 0x10df, 0x720, pci_quirk_mf_endpoint_acs }, /* Emulex Skyhawk-R */
 	/* Cavium ThunderX */
 	{ PCI_VENDOR_ID_CAVIUM, PCI_ANY_ID, pci_quirk_cavium_acs },
+	/* APM X-Gene */
+	{ PCI_VENDOR_ID_AMCC, 0xE004, pci_quirk_xgene_acs },
 	{ 0 }
 };
 
@@ -4657,23 +4761,6 @@ static void quirk_intel_qat_vf_cap(struct pci_dev *pdev)
 }
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x443, quirk_intel_qat_vf_cap);
 
-/*
- * VMD-enabled root ports will change the source ID for all messages
- * to the VMD device. Rather than doing device matching with the source
- * ID, the AER driver should traverse the child device tree, reading
- * AER registers to find the faulting device.
- */
-static void quirk_no_aersid(struct pci_dev *pdev)
-{
-	/* VMD Domain */
-	if (pdev->bus->sysdata && pci_domain_nr(pdev->bus) >= 0x10000)
-		pdev->bus->bus_flags |= PCI_BUS_FLAGS_NO_AERSID;
-}
-DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2030, quirk_no_aersid);
-DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2031, quirk_no_aersid);
-DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2032, quirk_no_aersid);
-DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2033, quirk_no_aersid);
-
 /* FLR may cause some 82579 devices to hang. */
 static void quirk_intel_no_flr(struct pci_dev *dev)
 {
@@ -4681,3 +4768,34 @@ static void quirk_intel_no_flr(struct pci_dev *dev)
 }
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x1502, quirk_intel_no_flr);
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x1503, quirk_intel_no_flr);
+
+static void quirk_no_ext_tags(struct pci_dev *pdev)
+{
+	struct pci_host_bridge *bridge = pci_find_host_bridge(pdev->bus);
+
+	if (!bridge)
+		return;
+
+	bridge->no_ext_tags = 1;
+	dev_info(&pdev->dev, "disabling Extended Tags (this device can't handle them)\n");
+
+	pci_walk_bus(bridge->bus, pci_configure_extended_tags, NULL);
+}
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SERVERWORKS, 0x0140, quirk_no_ext_tags);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SERVERWORKS, 0x0142, quirk_no_ext_tags);
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_SERVERWORKS, 0x0144, quirk_no_ext_tags);
+
+#ifdef CONFIG_PCI_ATS
+/*
+ * Some devices have a broken ATS implementation causing IOMMU stalls.
+ * Don't use ATS for those devices.
+ */
+static void quirk_no_ats(struct pci_dev *pdev)
+{
+	dev_info(&pdev->dev, "disabling ATS (broken on this device)\n");
+	pdev->ats_cap = 0;
+}
+
+/* AMD Stoney platform GPU */
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ATI, 0x98e4, quirk_no_ats);
+#endif /* CONFIG_PCI_ATS */
diff --git a/drivers/pci/setup-irq.c b/drivers/pci/setup-irq.c
index 81eda3d93a5d..86106c44ce94 100644
--- a/drivers/pci/setup-irq.c
+++ b/drivers/pci/setup-irq.c
@@ -17,12 +17,6 @@
 #include <linux/cache.h>
 #include "pci.h"
 
-void __weak pcibios_update_irq(struct pci_dev *dev, int irq)
-{
-	dev_dbg(&dev->dev, "assigning IRQ %02d\n", irq);
-	pci_write_config_byte(dev, PCI_INTERRUPT_LINE, irq);
-}
-
 void pci_assign_irq(struct pci_dev *dev)
 {
 	u8 pin;
@@ -65,29 +59,5 @@ void pci_assign_irq(struct pci_dev *dev)
 
 	/* Always tell the device, so the driver knows what is
 	   the real IRQ to use; the device does not use it. */
-	pcibios_update_irq(dev, irq);
-}
-
-void pci_fixup_irqs(u8 (*swizzle)(struct pci_dev *, u8 *),
-		    int (*map_irq)(const struct pci_dev *, u8, u8))
-{
-	/*
-	 * Implement pci_fixup_irqs() through pci_assign_irq().
-	 * This code should be remove eventually, it is a wrapper
-	 * around pci_assign_irq() interface to keep current
-	 * pci_fixup_irqs() behaviour unchanged on architecture
-	 * code still relying on its interface.
-	 */
-	struct pci_dev *dev = NULL;
-	struct pci_host_bridge *hbrg = NULL;
-
-	for_each_pci_dev(dev) {
-		hbrg = pci_find_host_bridge(dev->bus);
-		hbrg->swizzle_irq = swizzle;
-		hbrg->map_irq = map_irq;
-		pci_assign_irq(dev);
-		hbrg->swizzle_irq = NULL;
-		hbrg->map_irq = NULL;
-	}
+	pci_write_config_byte(dev, PCI_INTERRUPT_LINE, irq);
 }
-EXPORT_SYMBOL_GPL(pci_fixup_irqs);
diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c
index 85774b7a316a..e576e1a8d978 100644
--- a/drivers/pci/setup-res.c
+++ b/drivers/pci/setup-res.c
@@ -234,6 +234,19 @@ static int pci_revert_fw_address(struct resource *res, struct pci_dev *dev,
 	return 0;
 }
 
+/*
+ * We don't have to worry about legacy ISA devices, so nothing to do here.
+ * This is marked as __weak because multiple architectures define it; it should
+ * eventually go away.
+ */
+resource_size_t __weak pcibios_align_resource(void *data,
+					      const struct resource *res,
+					      resource_size_t size,
+					      resource_size_t align)
+{
+       return res->start;
+}
+
 static int __pci_assign_resource(struct pci_bus *bus, struct pci_dev *dev,
 		int resno, resource_size_t size, resource_size_t align)
 {
diff --git a/drivers/pcmcia/db1xxx_ss.c b/drivers/pcmcia/db1xxx_ss.c
index 944674ee3464..19e17829f515 100644
--- a/drivers/pcmcia/db1xxx_ss.c
+++ b/drivers/pcmcia/db1xxx_ss.c
@@ -131,22 +131,27 @@ static irqreturn_t db1000_pcmcia_stschgirq(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
+/* Db/Pb1200 have separate per-socket insertion and ejection
+ * interrupts which stay asserted as long as the card is
+ * inserted/missing.  The one which caused us to be called
+ * needs to be disabled and the other one enabled.
+ */
 static irqreturn_t db1200_pcmcia_cdirq(int irq, void *data)
 {
+	disable_irq_nosync(irq);
+	return IRQ_WAKE_THREAD;
+}
+
+static irqreturn_t db1200_pcmcia_cdirq_fn(int irq, void *data)
+{
 	struct db1x_pcmcia_sock *sock = data;
 
-	/* Db/Pb1200 have separate per-socket insertion and ejection
-	 * interrupts which stay asserted as long as the card is
-	 * inserted/missing.  The one which caused us to be called
-	 * needs to be disabled and the other one enabled.
-	 */
-	if (irq == sock->insert_irq) {
-		disable_irq_nosync(sock->insert_irq);
+	/* Wait a bit for the signals to stop bouncing. */
+	msleep(100);
+	if (irq == sock->insert_irq)
 		enable_irq(sock->eject_irq);
-	} else {
-		disable_irq_nosync(sock->eject_irq);
+	else
 		enable_irq(sock->insert_irq);
-	}
 
 	pcmcia_parse_events(&sock->socket, SS_DETECT);
 
@@ -172,13 +177,13 @@ static int db1x_pcmcia_setup_irqs(struct db1x_pcmcia_sock *sock)
 	 */
 	if ((sock->board_type == BOARD_TYPE_DB1200) ||
 	    (sock->board_type == BOARD_TYPE_DB1300)) {
-		ret = request_irq(sock->insert_irq, db1200_pcmcia_cdirq,
-				  0, "pcmcia_insert", sock);
+		ret = request_threaded_irq(sock->insert_irq, db1200_pcmcia_cdirq,
+			db1200_pcmcia_cdirq_fn, 0, "pcmcia_insert", sock);
 		if (ret)
 			goto out1;
 
-		ret = request_irq(sock->eject_irq, db1200_pcmcia_cdirq,
-				  0, "pcmcia_eject", sock);
+		ret = request_threaded_irq(sock->eject_irq, db1200_pcmcia_cdirq,
+			db1200_pcmcia_cdirq_fn, 0, "pcmcia_eject", sock);
 		if (ret) {
 			free_irq(sock->insert_irq, sock);
 			goto out1;
diff --git a/drivers/perf/arm_pmu.c b/drivers/perf/arm_pmu.c
index 1c5e0f333779..d14fc2e67f93 100644
--- a/drivers/perf/arm_pmu.c
+++ b/drivers/perf/arm_pmu.c
@@ -47,6 +47,9 @@ armpmu_map_cache_event(const unsigned (*cache_map)
 	if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
 		return -EINVAL;
 
+	if (!cache_map)
+		return -ENOENT;
+
 	ret = (int)(*cache_map)[cache_type][cache_op][cache_result];
 
 	if (ret == CACHE_OP_UNSUPPORTED)
@@ -63,6 +66,9 @@ armpmu_map_hw_event(const unsigned (*event_map)[PERF_COUNT_HW_MAX], u64 config)
 	if (config >= PERF_COUNT_HW_MAX)
 		return -EINVAL;
 
+	if (!event_map)
+		return -ENOENT;
+
 	mapping = (*event_map)[config];
 	return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping;
 }
diff --git a/drivers/perf/xgene_pmu.c b/drivers/perf/xgene_pmu.c
index e841282d690c..eb23311bc70c 100644
--- a/drivers/perf/xgene_pmu.c
+++ b/drivers/perf/xgene_pmu.c
@@ -1147,7 +1147,6 @@ xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx)
 {
 	struct device *dev = xgene_pmu->dev;
 	struct xgene_pmu_dev *pmu;
-	int rc;
 
 	pmu = devm_kzalloc(dev, sizeof(*pmu), GFP_KERNEL);
 	if (!pmu)
@@ -1159,7 +1158,7 @@ xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx)
 	switch (pmu->inf->type) {
 	case PMU_TYPE_L3C:
 		if (!(xgene_pmu->l3c_active_mask & pmu->inf->enable_mask))
-			goto dev_err;
+			return -ENODEV;
 		if (xgene_pmu->version == PCP_PMU_V3)
 			pmu->attr_groups = l3c_pmu_v3_attr_groups;
 		else
@@ -1177,7 +1176,7 @@ xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx)
 		break;
 	case PMU_TYPE_MCB:
 		if (!(xgene_pmu->mcb_active_mask & pmu->inf->enable_mask))
-			goto dev_err;
+			return -ENODEV;
 		if (xgene_pmu->version == PCP_PMU_V3)
 			pmu->attr_groups = mcb_pmu_v3_attr_groups;
 		else
@@ -1185,7 +1184,7 @@ xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx)
 		break;
 	case PMU_TYPE_MC:
 		if (!(xgene_pmu->mc_active_mask & pmu->inf->enable_mask))
-			goto dev_err;
+			return -ENODEV;
 		if (xgene_pmu->version == PCP_PMU_V3)
 			pmu->attr_groups = mc_pmu_v3_attr_groups;
 		else
@@ -1195,19 +1194,14 @@ xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx)
 		return -EINVAL;
 	}
 
-	rc = xgene_init_perf(pmu, ctx->name);
-	if (rc) {
+	if (xgene_init_perf(pmu, ctx->name)) {
 		dev_err(dev, "%s PMU: Failed to init perf driver\n", ctx->name);
-		goto dev_err;
+		return -ENODEV;
 	}
 
 	dev_info(dev, "%s PMU registered\n", ctx->name);
 
-	return rc;
-
-dev_err:
-	devm_kfree(dev, pmu);
-	return -ENODEV;
+	return 0;
 }
 
 static void _xgene_pmu_isr(int irq, struct xgene_pmu_dev *pmu_dev)
@@ -1515,13 +1509,13 @@ xgene_pmu_dev_ctx *acpi_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu,
 	acpi_dev_free_resource_list(&resource_list);
 	if (rc < 0) {
 		dev_err(dev, "PMU type %d: No resource address found\n", type);
-		goto err;
+		return NULL;
 	}
 
 	dev_csr = devm_ioremap_resource(dev, &res);
 	if (IS_ERR(dev_csr)) {
 		dev_err(dev, "PMU type %d: Fail to map resource\n", type);
-		goto err;
+		return NULL;
 	}
 
 	/* A PMU device node without enable-bit-index is always enabled */
@@ -1535,7 +1529,7 @@ xgene_pmu_dev_ctx *acpi_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu,
 	ctx->name = xgene_pmu_dev_name(dev, type, enable_bit);
 	if (!ctx->name) {
 		dev_err(dev, "PMU type %d: Fail to get device name\n", type);
-		goto err;
+		return NULL;
 	}
 	inf = &ctx->inf;
 	inf->type = type;
@@ -1543,9 +1537,6 @@ xgene_pmu_dev_ctx *acpi_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu,
 	inf->enable_mask = 1 << enable_bit;
 
 	return ctx;
-err:
-	devm_kfree(dev, ctx);
-	return NULL;
 }
 
 static const struct acpi_device_id xgene_pmu_acpi_type_match[] = {
@@ -1663,20 +1654,20 @@ xgene_pmu_dev_ctx *fdt_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu,
 	void __iomem *dev_csr;
 	struct resource res;
 	int enable_bit;
-	int rc;
 
 	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
 	if (!ctx)
 		return NULL;
-	rc = of_address_to_resource(np, 0, &res);
-	if (rc < 0) {
+
+	if (of_address_to_resource(np, 0, &res) < 0) {
 		dev_err(dev, "PMU type %d: No resource address found\n", type);
-		goto err;
+		return NULL;
 	}
+
 	dev_csr = devm_ioremap_resource(dev, &res);
 	if (IS_ERR(dev_csr)) {
 		dev_err(dev, "PMU type %d: Fail to map resource\n", type);
-		goto err;
+		return NULL;
 	}
 
 	/* A PMU device node without enable-bit-index is always enabled */
@@ -1686,17 +1677,15 @@ xgene_pmu_dev_ctx *fdt_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu,
 	ctx->name = xgene_pmu_dev_name(dev, type, enable_bit);
 	if (!ctx->name) {
 		dev_err(dev, "PMU type %d: Fail to get device name\n", type);
-		goto err;
+		return NULL;
 	}
+
 	inf = &ctx->inf;
 	inf->type = type;
 	inf->csr = dev_csr;
 	inf->enable_mask = 1 << enable_bit;
 
 	return ctx;
-err:
-	devm_kfree(dev, ctx);
-	return NULL;
 }
 
 static int fdt_pmu_probe_pmu_dev(struct xgene_pmu *xgene_pmu,
@@ -1868,22 +1857,20 @@ static int xgene_pmu_probe(struct platform_device *pdev)
 	xgene_pmu->pcppmu_csr = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(xgene_pmu->pcppmu_csr)) {
 		dev_err(&pdev->dev, "ioremap failed for PCP PMU resource\n");
-		rc = PTR_ERR(xgene_pmu->pcppmu_csr);
-		goto err;
+		return PTR_ERR(xgene_pmu->pcppmu_csr);
 	}
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
 		dev_err(&pdev->dev, "No IRQ resource\n");
-		rc = -EINVAL;
-		goto err;
+		return -EINVAL;
 	}
 	rc = devm_request_irq(&pdev->dev, irq, xgene_pmu_isr,
 				IRQF_NOBALANCING | IRQF_NO_THREAD,
 				dev_name(&pdev->dev), xgene_pmu);
 	if (rc) {
 		dev_err(&pdev->dev, "Could not request IRQ %d\n", irq);
-		goto err;
+		return rc;
 	}
 
 	raw_spin_lock_init(&xgene_pmu->lock);
@@ -1903,42 +1890,29 @@ static int xgene_pmu_probe(struct platform_device *pdev)
 	rc = irq_set_affinity(irq, &xgene_pmu->cpu);
 	if (rc) {
 		dev_err(&pdev->dev, "Failed to set interrupt affinity!\n");
-		goto err;
+		return rc;
 	}
 
 	/* Walk through the tree for all PMU perf devices */
 	rc = xgene_pmu_probe_pmu_dev(xgene_pmu, pdev);
 	if (rc) {
 		dev_err(&pdev->dev, "No PMU perf devices found!\n");
-		goto err;
+		return rc;
 	}
 
 	/* Enable interrupt */
 	xgene_pmu->ops->unmask_int(xgene_pmu);
 
 	return 0;
-
-err:
-	if (xgene_pmu->pcppmu_csr)
-		devm_iounmap(&pdev->dev, xgene_pmu->pcppmu_csr);
-	devm_kfree(&pdev->dev, xgene_pmu);
-
-	return rc;
 }
 
 static void
 xgene_pmu_dev_cleanup(struct xgene_pmu *xgene_pmu, struct list_head *pmus)
 {
 	struct xgene_pmu_dev_ctx *ctx;
-	struct device *dev = xgene_pmu->dev;
-	struct xgene_pmu_dev *pmu_dev;
 
 	list_for_each_entry(ctx, pmus, next) {
-		pmu_dev = ctx->pmu_dev;
-		if (pmu_dev->inf->csr)
-			devm_iounmap(dev, pmu_dev->inf->csr);
-		devm_kfree(dev, ctx);
-		devm_kfree(dev, pmu_dev);
+		perf_pmu_unregister(&ctx->pmu_dev->pmu);
 	}
 }
 
@@ -1951,10 +1925,6 @@ static int xgene_pmu_remove(struct platform_device *pdev)
 	xgene_pmu_dev_cleanup(xgene_pmu, &xgene_pmu->mcbpmus);
 	xgene_pmu_dev_cleanup(xgene_pmu, &xgene_pmu->mcpmus);
 
-	if (xgene_pmu->pcppmu_csr)
-		devm_iounmap(&pdev->dev, xgene_pmu->pcppmu_csr);
-	devm_kfree(&pdev->dev, xgene_pmu);
-
 	return 0;
 }
 
diff --git a/drivers/phy/Kconfig b/drivers/phy/Kconfig
index c1807d4a0079..5c8d452e35e2 100644
--- a/drivers/phy/Kconfig
+++ b/drivers/phy/Kconfig
@@ -26,14 +26,6 @@ config PHY_LPC18XX_USB_OTG
 	  This driver is need for USB0 support on LPC18xx/43xx and takes
 	  care of enabling and clock setup.
 
-config PHY_MT65XX_USB3
-	tristate "Mediatek USB3.0 PHY Driver"
-	depends on ARCH_MEDIATEK && OF
-	select GENERIC_PHY
-	help
-	  Say 'Y' here to add support for Mediatek USB3.0 PHY driver,
-	  it supports multiple usb2.0 and usb3.0 ports.
-
 config PHY_PISTACHIO_USB
 	tristate "IMG Pistachio USB2.0 PHY driver"
 	depends on MACH_PISTACHIO
@@ -52,9 +44,12 @@ source "drivers/phy/allwinner/Kconfig"
 source "drivers/phy/amlogic/Kconfig"
 source "drivers/phy/broadcom/Kconfig"
 source "drivers/phy/hisilicon/Kconfig"
+source "drivers/phy/lantiq/Kconfig"
 source "drivers/phy/marvell/Kconfig"
+source "drivers/phy/mediatek/Kconfig"
 source "drivers/phy/motorola/Kconfig"
 source "drivers/phy/qualcomm/Kconfig"
+source "drivers/phy/ralink/Kconfig"
 source "drivers/phy/renesas/Kconfig"
 source "drivers/phy/rockchip/Kconfig"
 source "drivers/phy/samsung/Kconfig"
diff --git a/drivers/phy/Makefile b/drivers/phy/Makefile
index f252201e0ec9..3a52dcb09566 100644
--- a/drivers/phy/Makefile
+++ b/drivers/phy/Makefile
@@ -4,12 +4,12 @@
 
 obj-$(CONFIG_GENERIC_PHY)		+= phy-core.o
 obj-$(CONFIG_PHY_LPC18XX_USB_OTG)	+= phy-lpc18xx-usb-otg.o
-obj-$(CONFIG_PHY_MT65XX_USB3)		+= phy-mt65xx-usb3.o
 obj-$(CONFIG_PHY_XGENE)			+= phy-xgene.o
 obj-$(CONFIG_PHY_PISTACHIO_USB)		+= phy-pistachio-usb.o
-
 obj-$(CONFIG_ARCH_SUNXI)		+= allwinner/
 obj-$(CONFIG_ARCH_MESON)		+= amlogic/
+obj-$(CONFIG_LANTIQ)			+= lantiq/
+obj-$(CONFIG_ARCH_MEDIATEK)		+= mediatek/
 obj-$(CONFIG_ARCH_RENESAS)		+= renesas/
 obj-$(CONFIG_ARCH_ROCKCHIP)		+= rockchip/
 obj-$(CONFIG_ARCH_TEGRA)		+= tegra/
@@ -18,6 +18,7 @@ obj-y					+= broadcom/	\
 					   marvell/	\
 					   motorola/	\
 					   qualcomm/	\
+					   ralink/	\
 					   samsung/	\
 					   st/		\
 					   ti/
diff --git a/drivers/phy/allwinner/phy-sun4i-usb.c b/drivers/phy/allwinner/phy-sun4i-usb.c
index bbf06cfe5898..1161e11fb3cf 100644
--- a/drivers/phy/allwinner/phy-sun4i-usb.c
+++ b/drivers/phy/allwinner/phy-sun4i-usb.c
@@ -87,6 +87,16 @@
 #define PHY_DISCON_TH_SEL		0x2a
 #define PHY_SQUELCH_DETECT		0x3c
 
+/* A83T specific control bits for PHY0 */
+#define PHY_CTL_VBUSVLDEXT		BIT(5)
+#define PHY_CTL_SIDDQ			BIT(3)
+
+/* A83T specific control bits for PHY2 HSIC */
+#define SUNXI_EHCI_HS_FORCE		BIT(20)
+#define SUNXI_HSIC_CONNECT_DET		BIT(17)
+#define SUNXI_HSIC_CONNECT_INT		BIT(16)
+#define SUNXI_HSIC			BIT(1)
+
 #define MAX_PHYS			4
 
 /*
@@ -100,6 +110,7 @@ enum sun4i_usb_phy_type {
 	sun4i_a10_phy,
 	sun6i_a31_phy,
 	sun8i_a33_phy,
+	sun8i_a83t_phy,
 	sun8i_h3_phy,
 	sun8i_v3s_phy,
 	sun50i_a64_phy,
@@ -107,6 +118,7 @@ enum sun4i_usb_phy_type {
 
 struct sun4i_usb_phy_cfg {
 	int num_phys;
+	int hsic_index;
 	enum sun4i_usb_phy_type type;
 	u32 disc_thresh;
 	u8 phyctl_offset;
@@ -126,6 +138,7 @@ struct sun4i_usb_phy_data {
 		struct regulator *vbus;
 		struct reset_control *reset;
 		struct clk *clk;
+		struct clk *clk2;
 		bool regulator_on;
 		int index;
 	} phys[MAX_PHYS];
@@ -232,6 +245,7 @@ static void sun4i_usb_phy_write(struct sun4i_usb_phy *phy, u32 addr, u32 data,
 
 static void sun4i_usb_phy_passby(struct sun4i_usb_phy *phy, int enable)
 {
+	struct sun4i_usb_phy_data *phy_data = to_sun4i_usb_phy_data(phy);
 	u32 bits, reg_value;
 
 	if (!phy->pmu)
@@ -240,6 +254,11 @@ static void sun4i_usb_phy_passby(struct sun4i_usb_phy *phy, int enable)
 	bits = SUNXI_AHB_ICHR8_EN | SUNXI_AHB_INCR4_BURST_EN |
 		SUNXI_AHB_INCRX_ALIGN_EN | SUNXI_ULPI_BYPASS_EN;
 
+	/* A83T USB2 is HSIC */
+	if (phy_data->cfg->type == sun8i_a83t_phy && phy->index == 2)
+		bits |= SUNXI_EHCI_HS_FORCE | SUNXI_HSIC_CONNECT_INT |
+			SUNXI_HSIC;
+
 	reg_value = readl(phy->pmu);
 
 	if (enable)
@@ -261,27 +280,43 @@ static int sun4i_usb_phy_init(struct phy *_phy)
 	if (ret)
 		return ret;
 
-	ret = reset_control_deassert(phy->reset);
+	ret = clk_prepare_enable(phy->clk2);
 	if (ret) {
 		clk_disable_unprepare(phy->clk);
 		return ret;
 	}
 
-	if (phy->pmu && data->cfg->enable_pmu_unk1) {
-		val = readl(phy->pmu + REG_PMU_UNK1);
-		writel(val & ~2, phy->pmu + REG_PMU_UNK1);
+	ret = reset_control_deassert(phy->reset);
+	if (ret) {
+		clk_disable_unprepare(phy->clk2);
+		clk_disable_unprepare(phy->clk);
+		return ret;
 	}
 
-	/* Enable USB 45 Ohm resistor calibration */
-	if (phy->index == 0)
-		sun4i_usb_phy_write(phy, PHY_RES45_CAL_EN, 0x01, 1);
+	if (data->cfg->type == sun8i_a83t_phy) {
+		if (phy->index == 0) {
+			val = readl(data->base + data->cfg->phyctl_offset);
+			val |= PHY_CTL_VBUSVLDEXT;
+			val &= ~PHY_CTL_SIDDQ;
+			writel(val, data->base + data->cfg->phyctl_offset);
+		}
+	} else {
+		if (phy->pmu && data->cfg->enable_pmu_unk1) {
+			val = readl(phy->pmu + REG_PMU_UNK1);
+			writel(val & ~2, phy->pmu + REG_PMU_UNK1);
+		}
 
-	/* Adjust PHY's magnitude and rate */
-	sun4i_usb_phy_write(phy, PHY_TX_AMPLITUDE_TUNE, 0x14, 5);
+		/* Enable USB 45 Ohm resistor calibration */
+		if (phy->index == 0)
+			sun4i_usb_phy_write(phy, PHY_RES45_CAL_EN, 0x01, 1);
 
-	/* Disconnect threshold adjustment */
-	sun4i_usb_phy_write(phy, PHY_DISCON_TH_SEL,
-			    data->cfg->disc_thresh, 2);
+		/* Adjust PHY's magnitude and rate */
+		sun4i_usb_phy_write(phy, PHY_TX_AMPLITUDE_TUNE, 0x14, 5);
+
+		/* Disconnect threshold adjustment */
+		sun4i_usb_phy_write(phy, PHY_DISCON_TH_SEL,
+				    data->cfg->disc_thresh, 2);
+	}
 
 	sun4i_usb_phy_passby(phy, 1);
 
@@ -307,6 +342,13 @@ static int sun4i_usb_phy_exit(struct phy *_phy)
 	struct sun4i_usb_phy_data *data = to_sun4i_usb_phy_data(phy);
 
 	if (phy->index == 0) {
+		if (data->cfg->type == sun8i_a83t_phy) {
+			void __iomem *phyctl = data->base +
+				data->cfg->phyctl_offset;
+
+			writel(readl(phyctl) | PHY_CTL_SIDDQ, phyctl);
+		}
+
 		/* Disable pull-ups */
 		sun4i_usb_phy0_update_iscr(_phy, ISCR_DPDM_PULLUP_EN, 0);
 		sun4i_usb_phy0_update_iscr(_phy, ISCR_ID_PULLUP_EN, 0);
@@ -315,6 +357,7 @@ static int sun4i_usb_phy_exit(struct phy *_phy)
 
 	sun4i_usb_phy_passby(phy, 0);
 	reset_control_assert(phy->reset);
+	clk_disable_unprepare(phy->clk2);
 	clk_disable_unprepare(phy->clk);
 
 	return 0;
@@ -653,19 +696,25 @@ static int sun4i_usb_phy_probe(struct platform_device *pdev)
 
 	data->id_det_gpio = devm_gpiod_get_optional(dev, "usb0_id_det",
 						    GPIOD_IN);
-	if (IS_ERR(data->id_det_gpio))
+	if (IS_ERR(data->id_det_gpio)) {
+		dev_err(dev, "Couldn't request ID GPIO\n");
 		return PTR_ERR(data->id_det_gpio);
+	}
 
 	data->vbus_det_gpio = devm_gpiod_get_optional(dev, "usb0_vbus_det",
 						      GPIOD_IN);
-	if (IS_ERR(data->vbus_det_gpio))
+	if (IS_ERR(data->vbus_det_gpio)) {
+		dev_err(dev, "Couldn't request VBUS detect GPIO\n");
 		return PTR_ERR(data->vbus_det_gpio);
+	}
 
 	if (of_find_property(np, "usb0_vbus_power-supply", NULL)) {
 		data->vbus_power_supply = devm_power_supply_get_by_phandle(dev,
 						     "usb0_vbus_power-supply");
-		if (IS_ERR(data->vbus_power_supply))
+		if (IS_ERR(data->vbus_power_supply)) {
+			dev_err(dev, "Couldn't get the VBUS power supply\n");
 			return PTR_ERR(data->vbus_power_supply);
+		}
 
 		if (!data->vbus_power_supply)
 			return -EPROBE_DEFER;
@@ -674,8 +723,10 @@ static int sun4i_usb_phy_probe(struct platform_device *pdev)
 	data->dr_mode = of_usb_get_dr_mode_by_phy(np, 0);
 
 	data->extcon = devm_extcon_dev_allocate(dev, sun4i_usb_phy0_cable);
-	if (IS_ERR(data->extcon))
+	if (IS_ERR(data->extcon)) {
+		dev_err(dev, "Couldn't allocate our extcon device\n");
 		return PTR_ERR(data->extcon);
+	}
 
 	ret = devm_extcon_dev_register(dev, data->extcon);
 	if (ret) {
@@ -690,8 +741,13 @@ static int sun4i_usb_phy_probe(struct platform_device *pdev)
 		snprintf(name, sizeof(name), "usb%d_vbus", i);
 		phy->vbus = devm_regulator_get_optional(dev, name);
 		if (IS_ERR(phy->vbus)) {
-			if (PTR_ERR(phy->vbus) == -EPROBE_DEFER)
+			if (PTR_ERR(phy->vbus) == -EPROBE_DEFER) {
+				dev_err(dev,
+					"Couldn't get regulator %s... Deferring probe\n",
+					name);
 				return -EPROBE_DEFER;
+			}
+
 			phy->vbus = NULL;
 		}
 
@@ -706,6 +762,17 @@ static int sun4i_usb_phy_probe(struct platform_device *pdev)
 			return PTR_ERR(phy->clk);
 		}
 
+		/* The first PHY is always tied to OTG, and never HSIC */
+		if (data->cfg->hsic_index && i == data->cfg->hsic_index) {
+			/* HSIC needs secondary clock */
+			snprintf(name, sizeof(name), "usb%d_hsic_12M", i);
+			phy->clk2 = devm_clk_get(dev, name);
+			if (IS_ERR(phy->clk2)) {
+				dev_err(dev, "failed to get clock %s\n", name);
+				return PTR_ERR(phy->clk2);
+			}
+		}
+
 		snprintf(name, sizeof(name), "usb%d_reset", i);
 		phy->reset = devm_reset_control_get(dev, name);
 		if (IS_ERR(phy->reset)) {
@@ -775,6 +842,8 @@ static int sun4i_usb_phy_probe(struct platform_device *pdev)
 		return PTR_ERR(phy_provider);
 	}
 
+	dev_dbg(dev, "successfully loaded\n");
+
 	return 0;
 }
 
@@ -832,6 +901,14 @@ static const struct sun4i_usb_phy_cfg sun8i_a33_cfg = {
 	.enable_pmu_unk1 = false,
 };
 
+static const struct sun4i_usb_phy_cfg sun8i_a83t_cfg = {
+	.num_phys = 3,
+	.hsic_index = 2,
+	.type = sun8i_a83t_phy,
+	.phyctl_offset = REG_PHYCTL_A33,
+	.dedicated_clocks = true,
+};
+
 static const struct sun4i_usb_phy_cfg sun8i_h3_cfg = {
 	.num_phys = 4,
 	.type = sun8i_h3_phy,
@@ -868,6 +945,7 @@ static const struct of_device_id sun4i_usb_phy_of_match[] = {
 	{ .compatible = "allwinner,sun7i-a20-usb-phy", .data = &sun7i_a20_cfg },
 	{ .compatible = "allwinner,sun8i-a23-usb-phy", .data = &sun8i_a23_cfg },
 	{ .compatible = "allwinner,sun8i-a33-usb-phy", .data = &sun8i_a33_cfg },
+	{ .compatible = "allwinner,sun8i-a83t-usb-phy", .data = &sun8i_a83t_cfg },
 	{ .compatible = "allwinner,sun8i-h3-usb-phy", .data = &sun8i_h3_cfg },
 	{ .compatible = "allwinner,sun8i-v3s-usb-phy", .data = &sun8i_v3s_cfg },
 	{ .compatible = "allwinner,sun50i-a64-usb-phy",
diff --git a/drivers/phy/broadcom/phy-bcm-ns2-usbdrd.c b/drivers/phy/broadcom/phy-bcm-ns2-usbdrd.c
index 9ae59e223131..d099a0c8cee5 100644
--- a/drivers/phy/broadcom/phy-bcm-ns2-usbdrd.c
+++ b/drivers/phy/broadcom/phy-bcm-ns2-usbdrd.c
@@ -253,16 +253,16 @@ static void extcon_work(struct work_struct *work)
 	vbus = gpiod_get_value_cansleep(driver->vbus_gpiod);
 
 	if (!id && vbus) { /* Host connected */
-		extcon_set_cable_state_(driver->edev, EXTCON_USB_HOST, true);
+		extcon_set_state_sync(driver->edev, EXTCON_USB_HOST, true);
 		pr_debug("Host cable connected\n");
 		driver->data->new_state = EVT_HOST;
 		connect_change(driver);
 	} else if (id && !vbus) { /* Disconnected */
-		extcon_set_cable_state_(driver->edev, EXTCON_USB_HOST, false);
-		extcon_set_cable_state_(driver->edev, EXTCON_USB, false);
+		extcon_set_state_sync(driver->edev, EXTCON_USB_HOST, false);
+		extcon_set_state_sync(driver->edev, EXTCON_USB, false);
 		pr_debug("Cable disconnected\n");
 	} else if (id && vbus) { /* Device connected */
-		extcon_set_cable_state_(driver->edev, EXTCON_USB, true);
+		extcon_set_state_sync(driver->edev, EXTCON_USB, true);
 		pr_debug("Device cable connected\n");
 		driver->data->new_state = EVT_DEVICE;
 		connect_change(driver);
diff --git a/drivers/phy/broadcom/phy-brcm-sata.c b/drivers/phy/broadcom/phy-brcm-sata.c
index e6544c8b1ace..9d7f74fe3d7c 100644
--- a/drivers/phy/broadcom/phy-brcm-sata.c
+++ b/drivers/phy/broadcom/phy-brcm-sata.c
@@ -335,7 +335,7 @@ static int brcm_nsp_sata_init(struct brcm_sata_port *port)
 
 	/* Wait for pll_seq_done bit */
 	try = 50;
-	while (try--) {
+	while (--try) {
 		val = brcm_sata_phy_rd(base, BLOCK0_REG_BANK,
 					BLOCK0_XGXSSTATUS);
 		if (val & BLOCK0_XGXSSTATUS_PLL_LOCK)
diff --git a/drivers/phy/lantiq/Kconfig b/drivers/phy/lantiq/Kconfig
new file mode 100644
index 000000000000..326d88a6417d
--- /dev/null
+++ b/drivers/phy/lantiq/Kconfig
@@ -0,0 +1,9 @@
+#
+# Phy drivers for Lantiq / Intel platforms
+#
+config PHY_LANTIQ_RCU_USB2
+	tristate "Lantiq XWAY SoC RCU based USB PHY"
+	depends on OF && (SOC_TYPE_XWAY || COMPILE_TEST)
+	select GENERIC_PHY
+	help
+	  Support for the USB PHY(s) on the Lantiq / Intel XWAY family SoCs.
diff --git a/drivers/phy/lantiq/Makefile b/drivers/phy/lantiq/Makefile
new file mode 100644
index 000000000000..f73eb56a5416
--- /dev/null
+++ b/drivers/phy/lantiq/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_PHY_LANTIQ_RCU_USB2)	+= phy-lantiq-rcu-usb2.o
diff --git a/drivers/phy/lantiq/phy-lantiq-rcu-usb2.c b/drivers/phy/lantiq/phy-lantiq-rcu-usb2.c
new file mode 100644
index 000000000000..986224fca9e9
--- /dev/null
+++ b/drivers/phy/lantiq/phy-lantiq-rcu-usb2.c
@@ -0,0 +1,254 @@
+/*
+ * Lantiq XWAY SoC RCU module based USB 1.1/2.0 PHY driver
+ *
+ * Copyright (C) 2016 Martin Blumenstingl <martin.blumenstingl@googlemail.com>
+ * Copyright (C) 2017 Hauke Mehrtens <hauke@hauke-m.de>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+
+/* Transmitter HS Pre-Emphasis Enable */
+#define RCU_CFG1_TX_PEE		BIT(0)
+/* Disconnect Threshold */
+#define RCU_CFG1_DIS_THR_MASK	0x00038000
+#define RCU_CFG1_DIS_THR_SHIFT	15
+
+struct ltq_rcu_usb2_bits {
+	u8 hostmode;
+	u8 slave_endianness;
+	u8 host_endianness;
+	bool have_ana_cfg;
+};
+
+struct ltq_rcu_usb2_priv {
+	struct regmap			*regmap;
+	unsigned int			phy_reg_offset;
+	unsigned int			ana_cfg1_reg_offset;
+	const struct ltq_rcu_usb2_bits	*reg_bits;
+	struct device			*dev;
+	struct phy			*phy;
+	struct clk			*phy_gate_clk;
+	struct reset_control		*ctrl_reset;
+	struct reset_control		*phy_reset;
+};
+
+static const struct ltq_rcu_usb2_bits xway_rcu_usb2_reg_bits = {
+	.hostmode = 11,
+	.slave_endianness = 9,
+	.host_endianness = 10,
+	.have_ana_cfg = false,
+};
+
+static const struct ltq_rcu_usb2_bits xrx100_rcu_usb2_reg_bits = {
+	.hostmode = 11,
+	.slave_endianness = 17,
+	.host_endianness = 10,
+	.have_ana_cfg = false,
+};
+
+static const struct ltq_rcu_usb2_bits xrx200_rcu_usb2_reg_bits = {
+	.hostmode = 11,
+	.slave_endianness = 9,
+	.host_endianness = 10,
+	.have_ana_cfg = true,
+};
+
+static const struct of_device_id ltq_rcu_usb2_phy_of_match[] = {
+	{
+		.compatible = "lantiq,ase-usb2-phy",
+		.data = &xway_rcu_usb2_reg_bits,
+	},
+	{
+		.compatible = "lantiq,danube-usb2-phy",
+		.data = &xway_rcu_usb2_reg_bits,
+	},
+	{
+		.compatible = "lantiq,xrx100-usb2-phy",
+		.data = &xrx100_rcu_usb2_reg_bits,
+	},
+	{
+		.compatible = "lantiq,xrx200-usb2-phy",
+		.data = &xrx200_rcu_usb2_reg_bits,
+	},
+	{
+		.compatible = "lantiq,xrx300-usb2-phy",
+		.data = &xrx200_rcu_usb2_reg_bits,
+	},
+	{ },
+};
+MODULE_DEVICE_TABLE(of, ltq_rcu_usb2_phy_of_match);
+
+static int ltq_rcu_usb2_phy_init(struct phy *phy)
+{
+	struct ltq_rcu_usb2_priv *priv = phy_get_drvdata(phy);
+
+	if (priv->reg_bits->have_ana_cfg) {
+		regmap_update_bits(priv->regmap, priv->ana_cfg1_reg_offset,
+			RCU_CFG1_TX_PEE, RCU_CFG1_TX_PEE);
+		regmap_update_bits(priv->regmap, priv->ana_cfg1_reg_offset,
+			RCU_CFG1_DIS_THR_MASK, 7 << RCU_CFG1_DIS_THR_SHIFT);
+	}
+
+	/* Configure core to host mode */
+	regmap_update_bits(priv->regmap, priv->phy_reg_offset,
+			   BIT(priv->reg_bits->hostmode), 0);
+
+	/* Select DMA endianness (Host-endian: big-endian) */
+	regmap_update_bits(priv->regmap, priv->phy_reg_offset,
+		BIT(priv->reg_bits->slave_endianness), 0);
+	regmap_update_bits(priv->regmap, priv->phy_reg_offset,
+		BIT(priv->reg_bits->host_endianness),
+		BIT(priv->reg_bits->host_endianness));
+
+	return 0;
+}
+
+static int ltq_rcu_usb2_phy_power_on(struct phy *phy)
+{
+	struct ltq_rcu_usb2_priv *priv = phy_get_drvdata(phy);
+	struct device *dev = priv->dev;
+	int ret;
+
+	reset_control_deassert(priv->phy_reset);
+
+	ret = clk_prepare_enable(priv->phy_gate_clk);
+	if (ret)
+		dev_err(dev, "failed to enable PHY gate\n");
+
+	return ret;
+}
+
+static int ltq_rcu_usb2_phy_power_off(struct phy *phy)
+{
+	struct ltq_rcu_usb2_priv *priv = phy_get_drvdata(phy);
+
+	reset_control_assert(priv->phy_reset);
+
+	clk_disable_unprepare(priv->phy_gate_clk);
+
+	return 0;
+}
+
+static struct phy_ops ltq_rcu_usb2_phy_ops = {
+	.init		= ltq_rcu_usb2_phy_init,
+	.power_on	= ltq_rcu_usb2_phy_power_on,
+	.power_off	= ltq_rcu_usb2_phy_power_off,
+	.owner		= THIS_MODULE,
+};
+
+static int ltq_rcu_usb2_of_parse(struct ltq_rcu_usb2_priv *priv,
+				 struct platform_device *pdev)
+{
+	struct device *dev = priv->dev;
+	const __be32 *offset;
+	int ret;
+
+	priv->reg_bits = of_device_get_match_data(dev);
+
+	priv->regmap = syscon_node_to_regmap(dev->of_node->parent);
+	if (IS_ERR(priv->regmap)) {
+		dev_err(dev, "Failed to lookup RCU regmap\n");
+		return PTR_ERR(priv->regmap);
+	}
+
+	offset = of_get_address(dev->of_node, 0, NULL, NULL);
+	if (!offset) {
+		dev_err(dev, "Failed to get RCU PHY reg offset\n");
+		return -ENOENT;
+	}
+	priv->phy_reg_offset = __be32_to_cpu(*offset);
+
+	if (priv->reg_bits->have_ana_cfg) {
+		offset = of_get_address(dev->of_node, 1, NULL, NULL);
+		if (!offset) {
+			dev_err(dev, "Failed to get RCU ANA CFG1 reg offset\n");
+			return -ENOENT;
+		}
+		priv->ana_cfg1_reg_offset = __be32_to_cpu(*offset);
+	}
+
+	priv->phy_gate_clk = devm_clk_get(dev, "phy");
+	if (IS_ERR(priv->phy_gate_clk)) {
+		dev_err(dev, "Unable to get USB phy gate clk\n");
+		return PTR_ERR(priv->phy_gate_clk);
+	}
+
+	priv->ctrl_reset = devm_reset_control_get_shared(dev, "ctrl");
+	if (IS_ERR(priv->ctrl_reset)) {
+		if (PTR_ERR(priv->ctrl_reset) != -EPROBE_DEFER)
+			dev_err(dev, "failed to get 'ctrl' reset\n");
+		return PTR_ERR(priv->ctrl_reset);
+	}
+
+	priv->phy_reset = devm_reset_control_get_optional(dev, "phy");
+	if (IS_ERR(priv->phy_reset))
+		return PTR_ERR(priv->phy_reset);
+
+	return 0;
+}
+
+static int ltq_rcu_usb2_phy_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct ltq_rcu_usb2_priv *priv;
+	struct phy_provider *provider;
+	int ret;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->dev = dev;
+
+	ret = ltq_rcu_usb2_of_parse(priv, pdev);
+	if (ret)
+		return ret;
+
+	/* Reset USB core through reset controller */
+	reset_control_deassert(priv->ctrl_reset);
+
+	reset_control_assert(priv->phy_reset);
+
+	priv->phy = devm_phy_create(dev, dev->of_node, &ltq_rcu_usb2_phy_ops);
+	if (IS_ERR(priv->phy)) {
+		dev_err(dev, "failed to create PHY\n");
+		return PTR_ERR(priv->phy);
+	}
+
+	phy_set_drvdata(priv->phy, priv);
+
+	provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
+	if (IS_ERR(provider))
+		return PTR_ERR(provider);
+
+	dev_set_drvdata(priv->dev, priv);
+	return 0;
+}
+
+static struct platform_driver ltq_rcu_usb2_phy_driver = {
+	.probe	= ltq_rcu_usb2_phy_probe,
+	.driver = {
+		.name	= "lantiq-rcu-usb2-phy",
+		.of_match_table	= ltq_rcu_usb2_phy_of_match,
+	}
+};
+module_platform_driver(ltq_rcu_usb2_phy_driver);
+
+MODULE_AUTHOR("Martin Blumenstingl <martin.blumenstingl@googlemail.com>");
+MODULE_DESCRIPTION("Lantiq XWAY USB2 PHY driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/phy/marvell/Kconfig b/drivers/phy/marvell/Kconfig
index 048d8893bc2e..68e321225400 100644
--- a/drivers/phy/marvell/Kconfig
+++ b/drivers/phy/marvell/Kconfig
@@ -21,6 +21,17 @@ config PHY_BERLIN_USB
 	help
 	  Enable this to support the USB PHY on Marvell Berlin SoCs.
 
+config PHY_MVEBU_CP110_COMPHY
+	tristate "Marvell CP110 comphy driver"
+	depends on ARCH_MVEBU || COMPILE_TEST
+	depends on OF
+	select GENERIC_PHY
+	help
+	  This driver allows to control the comphy, an hardware block providing
+	  shared serdes PHYs on Marvell Armada 7k/8k (in the CP110). Its serdes
+	  lanes can be used by various controllers (Ethernet, sata, usb,
+	  PCIe...).
+
 config PHY_MVEBU_SATA
 	def_bool y
 	depends on ARCH_DOVE || MACH_DOVE || MACH_KIRKWOOD
diff --git a/drivers/phy/marvell/Makefile b/drivers/phy/marvell/Makefile
index 3fc188f59118..0cf6a7cbaf9f 100644
--- a/drivers/phy/marvell/Makefile
+++ b/drivers/phy/marvell/Makefile
@@ -1,6 +1,7 @@
 obj-$(CONFIG_ARMADA375_USBCLUSTER_PHY)	+= phy-armada375-usb2.o
 obj-$(CONFIG_PHY_BERLIN_SATA)		+= phy-berlin-sata.o
 obj-$(CONFIG_PHY_BERLIN_USB)		+= phy-berlin-usb.o
+obj-$(CONFIG_PHY_MVEBU_CP110_COMPHY)	+= phy-mvebu-cp110-comphy.o
 obj-$(CONFIG_PHY_MVEBU_SATA)		+= phy-mvebu-sata.o
 obj-$(CONFIG_PHY_PXA_28NM_HSIC)		+= phy-pxa-28nm-hsic.o
 obj-$(CONFIG_PHY_PXA_28NM_USB2)		+= phy-pxa-28nm-usb2.o
diff --git a/drivers/phy/marvell/phy-mvebu-cp110-comphy.c b/drivers/phy/marvell/phy-mvebu-cp110-comphy.c
new file mode 100644
index 000000000000..73ebad6634a7
--- /dev/null
+++ b/drivers/phy/marvell/phy-mvebu-cp110-comphy.c
@@ -0,0 +1,644 @@
+/*
+ * Copyright (C) 2017 Marvell
+ *
+ * Antoine Tenart <antoine.tenart@free-electrons.com>
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2. This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+/* Relative to priv->base */
+#define MVEBU_COMPHY_SERDES_CFG0(n)		(0x0 + (n) * 0x1000)
+#define     MVEBU_COMPHY_SERDES_CFG0_PU_PLL	BIT(1)
+#define     MVEBU_COMPHY_SERDES_CFG0_GEN_RX(n)	((n) << 3)
+#define     MVEBU_COMPHY_SERDES_CFG0_GEN_TX(n)	((n) << 7)
+#define     MVEBU_COMPHY_SERDES_CFG0_PU_RX	BIT(11)
+#define     MVEBU_COMPHY_SERDES_CFG0_PU_TX	BIT(12)
+#define     MVEBU_COMPHY_SERDES_CFG0_HALF_BUS	BIT(14)
+#define MVEBU_COMPHY_SERDES_CFG1(n)		(0x4 + (n) * 0x1000)
+#define     MVEBU_COMPHY_SERDES_CFG1_RESET	BIT(3)
+#define     MVEBU_COMPHY_SERDES_CFG1_RX_INIT	BIT(4)
+#define     MVEBU_COMPHY_SERDES_CFG1_CORE_RESET	BIT(5)
+#define     MVEBU_COMPHY_SERDES_CFG1_RF_RESET	BIT(6)
+#define MVEBU_COMPHY_SERDES_CFG2(n)		(0x8 + (n) * 0x1000)
+#define     MVEBU_COMPHY_SERDES_CFG2_DFE_EN	BIT(4)
+#define MVEBU_COMPHY_SERDES_STATUS0(n)		(0x18 + (n) * 0x1000)
+#define     MVEBU_COMPHY_SERDES_STATUS0_TX_PLL_RDY	BIT(2)
+#define     MVEBU_COMPHY_SERDES_STATUS0_RX_PLL_RDY	BIT(3)
+#define     MVEBU_COMPHY_SERDES_STATUS0_RX_INIT		BIT(4)
+#define MVEBU_COMPHY_PWRPLL_CTRL(n)		(0x804 + (n) * 0x1000)
+#define     MVEBU_COMPHY_PWRPLL_CTRL_RFREQ(n)	((n) << 0)
+#define     MVEBU_COMPHY_PWRPLL_PHY_MODE(n)	((n) << 5)
+#define MVEBU_COMPHY_IMP_CAL(n)			(0x80c + (n) * 0x1000)
+#define     MVEBU_COMPHY_IMP_CAL_TX_EXT(n)	((n) << 10)
+#define     MVEBU_COMPHY_IMP_CAL_TX_EXT_EN	BIT(15)
+#define MVEBU_COMPHY_DFE_RES(n)			(0x81c + (n) * 0x1000)
+#define     MVEBU_COMPHY_DFE_RES_FORCE_GEN_TBL	BIT(15)
+#define MVEBU_COMPHY_COEF(n)			(0x828 + (n) * 0x1000)
+#define     MVEBU_COMPHY_COEF_DFE_EN		BIT(14)
+#define     MVEBU_COMPHY_COEF_DFE_CTRL		BIT(15)
+#define MVEBU_COMPHY_GEN1_S0(n)			(0x834 + (n) * 0x1000)
+#define     MVEBU_COMPHY_GEN1_S0_TX_AMP(n)	((n) << 1)
+#define     MVEBU_COMPHY_GEN1_S0_TX_EMPH(n)	((n) << 7)
+#define MVEBU_COMPHY_GEN1_S1(n)			(0x838 + (n) * 0x1000)
+#define     MVEBU_COMPHY_GEN1_S1_RX_MUL_PI(n)	((n) << 0)
+#define     MVEBU_COMPHY_GEN1_S1_RX_MUL_PF(n)	((n) << 3)
+#define     MVEBU_COMPHY_GEN1_S1_RX_MUL_FI(n)	((n) << 6)
+#define     MVEBU_COMPHY_GEN1_S1_RX_MUL_FF(n)	((n) << 8)
+#define     MVEBU_COMPHY_GEN1_S1_RX_DFE_EN	BIT(10)
+#define     MVEBU_COMPHY_GEN1_S1_RX_DIV(n)	((n) << 11)
+#define MVEBU_COMPHY_GEN1_S2(n)			(0x8f4 + (n) * 0x1000)
+#define     MVEBU_COMPHY_GEN1_S2_TX_EMPH(n)	((n) << 0)
+#define     MVEBU_COMPHY_GEN1_S2_TX_EMPH_EN	BIT(4)
+#define MVEBU_COMPHY_LOOPBACK(n)		(0x88c + (n) * 0x1000)
+#define     MVEBU_COMPHY_LOOPBACK_DBUS_WIDTH(n)	((n) << 1)
+#define MVEBU_COMPHY_VDD_CAL0(n)		(0x908 + (n) * 0x1000)
+#define     MVEBU_COMPHY_VDD_CAL0_CONT_MODE	BIT(15)
+#define MVEBU_COMPHY_EXT_SELV(n)		(0x914 + (n) * 0x1000)
+#define     MVEBU_COMPHY_EXT_SELV_RX_SAMPL(n)	((n) << 5)
+#define MVEBU_COMPHY_MISC_CTRL0(n)		(0x93c + (n) * 0x1000)
+#define     MVEBU_COMPHY_MISC_CTRL0_ICP_FORCE	BIT(5)
+#define     MVEBU_COMPHY_MISC_CTRL0_REFCLK_SEL	BIT(10)
+#define MVEBU_COMPHY_RX_CTRL1(n)		(0x940 + (n) * 0x1000)
+#define     MVEBU_COMPHY_RX_CTRL1_RXCLK2X_SEL	BIT(11)
+#define     MVEBU_COMPHY_RX_CTRL1_CLK8T_EN	BIT(12)
+#define MVEBU_COMPHY_SPEED_DIV(n)		(0x954 + (n) * 0x1000)
+#define     MVEBU_COMPHY_SPEED_DIV_TX_FORCE	BIT(7)
+#define MVEBU_SP_CALIB(n)			(0x96c + (n) * 0x1000)
+#define     MVEBU_SP_CALIB_SAMPLER(n)		((n) << 8)
+#define     MVEBU_SP_CALIB_SAMPLER_EN		BIT(12)
+#define MVEBU_COMPHY_TX_SLEW_RATE(n)		(0x974 + (n) * 0x1000)
+#define     MVEBU_COMPHY_TX_SLEW_RATE_EMPH(n)	((n) << 5)
+#define     MVEBU_COMPHY_TX_SLEW_RATE_SLC(n)	((n) << 10)
+#define MVEBU_COMPHY_DLT_CTRL(n)		(0x984 + (n) * 0x1000)
+#define     MVEBU_COMPHY_DLT_CTRL_DTL_FLOOP_EN	BIT(2)
+#define MVEBU_COMPHY_FRAME_DETECT0(n)		(0xa14 + (n) * 0x1000)
+#define     MVEBU_COMPHY_FRAME_DETECT0_PATN(n)	((n) << 7)
+#define MVEBU_COMPHY_FRAME_DETECT3(n)		(0xa20 + (n) * 0x1000)
+#define     MVEBU_COMPHY_FRAME_DETECT3_LOST_TIMEOUT_EN	BIT(12)
+#define MVEBU_COMPHY_DME(n)			(0xa28 + (n) * 0x1000)
+#define     MVEBU_COMPHY_DME_ETH_MODE		BIT(7)
+#define MVEBU_COMPHY_TRAINING0(n)		(0xa68 + (n) * 0x1000)
+#define     MVEBU_COMPHY_TRAINING0_P2P_HOLD	BIT(15)
+#define MVEBU_COMPHY_TRAINING5(n)		(0xaa4 + (n) * 0x1000)
+#define	    MVEBU_COMPHY_TRAINING5_RX_TIMER(n)	((n) << 0)
+#define MVEBU_COMPHY_TX_TRAIN_PRESET(n)		(0xb1c + (n) * 0x1000)
+#define     MVEBU_COMPHY_TX_TRAIN_PRESET_16B_AUTO_EN	BIT(8)
+#define     MVEBU_COMPHY_TX_TRAIN_PRESET_PRBS11		BIT(9)
+#define MVEBU_COMPHY_GEN1_S3(n)			(0xc40 + (n) * 0x1000)
+#define     MVEBU_COMPHY_GEN1_S3_FBCK_SEL	BIT(9)
+#define MVEBU_COMPHY_GEN1_S4(n)			(0xc44 + (n) * 0x1000)
+#define	    MVEBU_COMPHY_GEN1_S4_DFE_RES(n)	((n) << 8)
+#define MVEBU_COMPHY_TX_PRESET(n)		(0xc68 + (n) * 0x1000)
+#define     MVEBU_COMPHY_TX_PRESET_INDEX(n)	((n) << 0)
+#define MVEBU_COMPHY_GEN1_S5(n)			(0xd38 + (n) * 0x1000)
+#define     MVEBU_COMPHY_GEN1_S5_ICP(n)		((n) << 0)
+
+/* Relative to priv->regmap */
+#define MVEBU_COMPHY_CONF1(n)			(0x1000 + (n) * 0x28)
+#define     MVEBU_COMPHY_CONF1_PWRUP		BIT(1)
+#define     MVEBU_COMPHY_CONF1_USB_PCIE		BIT(2)	/* 0: Ethernet/SATA */
+#define MVEBU_COMPHY_CONF6(n)			(0x1014 + (n) * 0x28)
+#define     MVEBU_COMPHY_CONF6_40B		BIT(18)
+#define MVEBU_COMPHY_SELECTOR			0x1140
+#define     MVEBU_COMPHY_SELECTOR_PHY(n)	((n) * 0x4)
+
+#define MVEBU_COMPHY_LANES	6
+#define MVEBU_COMPHY_PORTS	3
+
+struct mvebu_comhy_conf {
+	enum phy_mode mode;
+	unsigned lane;
+	unsigned port;
+	u32 mux;
+};
+
+#define MVEBU_COMPHY_CONF(_lane, _port, _mode, _mux)	\
+	{						\
+		.lane = _lane,				\
+		.port = _port,				\
+		.mode = _mode,				\
+		.mux = _mux,				\
+	}
+
+static const struct mvebu_comhy_conf mvebu_comphy_cp110_modes[] = {
+	/* lane 0 */
+	MVEBU_COMPHY_CONF(0, 1, PHY_MODE_SGMII, 0x1),
+	/* lane 1 */
+	MVEBU_COMPHY_CONF(1, 2, PHY_MODE_SGMII, 0x1),
+	/* lane 2 */
+	MVEBU_COMPHY_CONF(2, 0, PHY_MODE_SGMII, 0x1),
+	MVEBU_COMPHY_CONF(2, 0, PHY_MODE_10GKR, 0x1),
+	/* lane 3 */
+	MVEBU_COMPHY_CONF(3, 1, PHY_MODE_SGMII, 0x2),
+	/* lane 4 */
+	MVEBU_COMPHY_CONF(4, 0, PHY_MODE_SGMII, 0x2),
+	MVEBU_COMPHY_CONF(4, 0, PHY_MODE_10GKR, 0x2),
+	MVEBU_COMPHY_CONF(4, 1, PHY_MODE_SGMII, 0x1),
+	/* lane 5 */
+	MVEBU_COMPHY_CONF(5, 2, PHY_MODE_SGMII, 0x1),
+};
+
+struct mvebu_comphy_priv {
+	void __iomem *base;
+	struct regmap *regmap;
+	struct device *dev;
+	int modes[MVEBU_COMPHY_LANES];
+};
+
+struct mvebu_comphy_lane {
+	struct mvebu_comphy_priv *priv;
+	unsigned id;
+	enum phy_mode mode;
+	int port;
+};
+
+static int mvebu_comphy_get_mux(int lane, int port, enum phy_mode mode)
+{
+	int i, n = ARRAY_SIZE(mvebu_comphy_cp110_modes);
+
+	/* Unused PHY mux value is 0x0 */
+	if (mode == PHY_MODE_INVALID)
+		return 0;
+
+	for (i = 0; i < n; i++) {
+		if (mvebu_comphy_cp110_modes[i].lane == lane &&
+		    mvebu_comphy_cp110_modes[i].port == port &&
+		    mvebu_comphy_cp110_modes[i].mode == mode)
+			break;
+	}
+
+	if (i == n)
+		return -EINVAL;
+
+	return mvebu_comphy_cp110_modes[i].mux;
+}
+
+static void mvebu_comphy_ethernet_init_reset(struct mvebu_comphy_lane *lane,
+					     enum phy_mode mode)
+{
+	struct mvebu_comphy_priv *priv = lane->priv;
+	u32 val;
+
+	regmap_read(priv->regmap, MVEBU_COMPHY_CONF1(lane->id), &val);
+	val &= ~MVEBU_COMPHY_CONF1_USB_PCIE;
+	val |= MVEBU_COMPHY_CONF1_PWRUP;
+	regmap_write(priv->regmap, MVEBU_COMPHY_CONF1(lane->id), val);
+
+	/* Select baud rates and PLLs */
+	val = readl(priv->base + MVEBU_COMPHY_SERDES_CFG0(lane->id));
+	val &= ~(MVEBU_COMPHY_SERDES_CFG0_PU_PLL |
+		 MVEBU_COMPHY_SERDES_CFG0_PU_RX |
+		 MVEBU_COMPHY_SERDES_CFG0_PU_TX |
+		 MVEBU_COMPHY_SERDES_CFG0_HALF_BUS |
+		 MVEBU_COMPHY_SERDES_CFG0_GEN_RX(0xf) |
+		 MVEBU_COMPHY_SERDES_CFG0_GEN_TX(0xf));
+	if (mode == PHY_MODE_10GKR)
+		val |= MVEBU_COMPHY_SERDES_CFG0_GEN_RX(0xe) |
+		       MVEBU_COMPHY_SERDES_CFG0_GEN_TX(0xe);
+	else if (mode == PHY_MODE_SGMII)
+		val |= MVEBU_COMPHY_SERDES_CFG0_GEN_RX(0x6) |
+		       MVEBU_COMPHY_SERDES_CFG0_GEN_TX(0x6) |
+		       MVEBU_COMPHY_SERDES_CFG0_HALF_BUS;
+	writel(val, priv->base + MVEBU_COMPHY_SERDES_CFG0(lane->id));
+
+	/* reset */
+	val = readl(priv->base + MVEBU_COMPHY_SERDES_CFG1(lane->id));
+	val &= ~(MVEBU_COMPHY_SERDES_CFG1_RESET |
+		 MVEBU_COMPHY_SERDES_CFG1_CORE_RESET |
+		 MVEBU_COMPHY_SERDES_CFG1_RF_RESET);
+	writel(val, priv->base + MVEBU_COMPHY_SERDES_CFG1(lane->id));
+
+	/* de-assert reset */
+	val = readl(priv->base + MVEBU_COMPHY_SERDES_CFG1(lane->id));
+	val |= MVEBU_COMPHY_SERDES_CFG1_RESET |
+	       MVEBU_COMPHY_SERDES_CFG1_CORE_RESET;
+	writel(val, priv->base + MVEBU_COMPHY_SERDES_CFG1(lane->id));
+
+	/* wait until clocks are ready */
+	mdelay(1);
+
+	/* exlicitly disable 40B, the bits isn't clear on reset */
+	regmap_read(priv->regmap, MVEBU_COMPHY_CONF6(lane->id), &val);
+	val &= ~MVEBU_COMPHY_CONF6_40B;
+	regmap_write(priv->regmap, MVEBU_COMPHY_CONF6(lane->id), val);
+
+	/* refclk selection */
+	val = readl(priv->base + MVEBU_COMPHY_MISC_CTRL0(lane->id));
+	val &= ~MVEBU_COMPHY_MISC_CTRL0_REFCLK_SEL;
+	if (mode == PHY_MODE_10GKR)
+		val |= MVEBU_COMPHY_MISC_CTRL0_ICP_FORCE;
+	writel(val, priv->base + MVEBU_COMPHY_MISC_CTRL0(lane->id));
+
+	/* power and pll selection */
+	val = readl(priv->base + MVEBU_COMPHY_PWRPLL_CTRL(lane->id));
+	val &= ~(MVEBU_COMPHY_PWRPLL_CTRL_RFREQ(0x1f) |
+		 MVEBU_COMPHY_PWRPLL_PHY_MODE(0x7));
+	val |= MVEBU_COMPHY_PWRPLL_CTRL_RFREQ(0x1) |
+	       MVEBU_COMPHY_PWRPLL_PHY_MODE(0x4);
+	writel(val, priv->base + MVEBU_COMPHY_PWRPLL_CTRL(lane->id));
+
+	val = readl(priv->base + MVEBU_COMPHY_LOOPBACK(lane->id));
+	val &= ~MVEBU_COMPHY_LOOPBACK_DBUS_WIDTH(0x7);
+	val |= MVEBU_COMPHY_LOOPBACK_DBUS_WIDTH(0x1);
+	writel(val, priv->base + MVEBU_COMPHY_LOOPBACK(lane->id));
+}
+
+static int mvebu_comphy_init_plls(struct mvebu_comphy_lane *lane,
+				  enum phy_mode mode)
+{
+	struct mvebu_comphy_priv *priv = lane->priv;
+	u32 val;
+
+	/* SERDES external config */
+	val = readl(priv->base + MVEBU_COMPHY_SERDES_CFG0(lane->id));
+	val |= MVEBU_COMPHY_SERDES_CFG0_PU_PLL |
+	       MVEBU_COMPHY_SERDES_CFG0_PU_RX |
+	       MVEBU_COMPHY_SERDES_CFG0_PU_TX;
+	writel(val, priv->base + MVEBU_COMPHY_SERDES_CFG0(lane->id));
+
+	/* check rx/tx pll */
+	readl_poll_timeout(priv->base + MVEBU_COMPHY_SERDES_STATUS0(lane->id),
+			   val,
+			   val & (MVEBU_COMPHY_SERDES_STATUS0_RX_PLL_RDY |
+				  MVEBU_COMPHY_SERDES_STATUS0_TX_PLL_RDY),
+			   1000, 150000);
+	if (!(val & (MVEBU_COMPHY_SERDES_STATUS0_RX_PLL_RDY |
+		     MVEBU_COMPHY_SERDES_STATUS0_TX_PLL_RDY)))
+		return -ETIMEDOUT;
+
+	/* rx init */
+	val = readl(priv->base + MVEBU_COMPHY_SERDES_CFG1(lane->id));
+	val |= MVEBU_COMPHY_SERDES_CFG1_RX_INIT;
+	writel(val, priv->base + MVEBU_COMPHY_SERDES_CFG1(lane->id));
+
+	/* check rx */
+	readl_poll_timeout(priv->base + MVEBU_COMPHY_SERDES_STATUS0(lane->id),
+			   val, val & MVEBU_COMPHY_SERDES_STATUS0_RX_INIT,
+			   1000, 10000);
+	if (!(val & MVEBU_COMPHY_SERDES_STATUS0_RX_INIT))
+		return -ETIMEDOUT;
+
+	val = readl(priv->base + MVEBU_COMPHY_SERDES_CFG1(lane->id));
+	val &= ~MVEBU_COMPHY_SERDES_CFG1_RX_INIT;
+	writel(val, priv->base + MVEBU_COMPHY_SERDES_CFG1(lane->id));
+
+	return 0;
+}
+
+static int mvebu_comphy_set_mode_sgmii(struct phy *phy)
+{
+	struct mvebu_comphy_lane *lane = phy_get_drvdata(phy);
+	struct mvebu_comphy_priv *priv = lane->priv;
+	u32 val;
+
+	mvebu_comphy_ethernet_init_reset(lane, PHY_MODE_SGMII);
+
+	val = readl(priv->base + MVEBU_COMPHY_RX_CTRL1(lane->id));
+	val &= ~MVEBU_COMPHY_RX_CTRL1_CLK8T_EN;
+	val |= MVEBU_COMPHY_RX_CTRL1_RXCLK2X_SEL;
+	writel(val, priv->base + MVEBU_COMPHY_RX_CTRL1(lane->id));
+
+	val = readl(priv->base + MVEBU_COMPHY_DLT_CTRL(lane->id));
+	val &= ~MVEBU_COMPHY_DLT_CTRL_DTL_FLOOP_EN;
+	writel(val, priv->base + MVEBU_COMPHY_DLT_CTRL(lane->id));
+
+	regmap_read(priv->regmap, MVEBU_COMPHY_CONF1(lane->id), &val);
+	val &= ~MVEBU_COMPHY_CONF1_USB_PCIE;
+	val |= MVEBU_COMPHY_CONF1_PWRUP;
+	regmap_write(priv->regmap, MVEBU_COMPHY_CONF1(lane->id), val);
+
+	val = readl(priv->base + MVEBU_COMPHY_GEN1_S0(lane->id));
+	val &= ~MVEBU_COMPHY_GEN1_S0_TX_EMPH(0xf);
+	val |= MVEBU_COMPHY_GEN1_S0_TX_EMPH(0x1);
+	writel(val, priv->base + MVEBU_COMPHY_GEN1_S0(lane->id));
+
+	return mvebu_comphy_init_plls(lane, PHY_MODE_SGMII);
+}
+
+static int mvebu_comphy_set_mode_10gkr(struct phy *phy)
+{
+	struct mvebu_comphy_lane *lane = phy_get_drvdata(phy);
+	struct mvebu_comphy_priv *priv = lane->priv;
+	u32 val;
+
+	mvebu_comphy_ethernet_init_reset(lane, PHY_MODE_10GKR);
+
+	val = readl(priv->base + MVEBU_COMPHY_RX_CTRL1(lane->id));
+	val |= MVEBU_COMPHY_RX_CTRL1_RXCLK2X_SEL |
+	       MVEBU_COMPHY_RX_CTRL1_CLK8T_EN;
+	writel(val, priv->base + MVEBU_COMPHY_RX_CTRL1(lane->id));
+
+	val = readl(priv->base + MVEBU_COMPHY_DLT_CTRL(lane->id));
+	val |= MVEBU_COMPHY_DLT_CTRL_DTL_FLOOP_EN;
+	writel(val, priv->base + MVEBU_COMPHY_DLT_CTRL(lane->id));
+
+	/* Speed divider */
+	val = readl(priv->base + MVEBU_COMPHY_SPEED_DIV(lane->id));
+	val |= MVEBU_COMPHY_SPEED_DIV_TX_FORCE;
+	writel(val, priv->base + MVEBU_COMPHY_SPEED_DIV(lane->id));
+
+	val = readl(priv->base + MVEBU_COMPHY_SERDES_CFG2(lane->id));
+	val |= MVEBU_COMPHY_SERDES_CFG2_DFE_EN;
+	writel(val, priv->base + MVEBU_COMPHY_SERDES_CFG2(lane->id));
+
+	/* DFE resolution */
+	val = readl(priv->base + MVEBU_COMPHY_DFE_RES(lane->id));
+	val |= MVEBU_COMPHY_DFE_RES_FORCE_GEN_TBL;
+	writel(val, priv->base + MVEBU_COMPHY_DFE_RES(lane->id));
+
+	val = readl(priv->base + MVEBU_COMPHY_GEN1_S0(lane->id));
+	val &= ~(MVEBU_COMPHY_GEN1_S0_TX_AMP(0x1f) |
+		 MVEBU_COMPHY_GEN1_S0_TX_EMPH(0xf));
+	val |= MVEBU_COMPHY_GEN1_S0_TX_AMP(0x1c) |
+	       MVEBU_COMPHY_GEN1_S0_TX_EMPH(0xe);
+	writel(val, priv->base + MVEBU_COMPHY_GEN1_S0(lane->id));
+
+	val = readl(priv->base + MVEBU_COMPHY_GEN1_S2(lane->id));
+	val &= ~MVEBU_COMPHY_GEN1_S2_TX_EMPH(0xf);
+	val |= MVEBU_COMPHY_GEN1_S2_TX_EMPH_EN;
+	writel(val, priv->base + MVEBU_COMPHY_GEN1_S2(lane->id));
+
+	val = readl(priv->base + MVEBU_COMPHY_TX_SLEW_RATE(lane->id));
+	val |= MVEBU_COMPHY_TX_SLEW_RATE_EMPH(0x3) |
+	       MVEBU_COMPHY_TX_SLEW_RATE_SLC(0x3f);
+	writel(val, priv->base + MVEBU_COMPHY_TX_SLEW_RATE(lane->id));
+
+	/* Impedance calibration */
+	val = readl(priv->base + MVEBU_COMPHY_IMP_CAL(lane->id));
+	val &= ~MVEBU_COMPHY_IMP_CAL_TX_EXT(0x1f);
+	val |= MVEBU_COMPHY_IMP_CAL_TX_EXT(0xe) |
+	       MVEBU_COMPHY_IMP_CAL_TX_EXT_EN;
+	writel(val, priv->base + MVEBU_COMPHY_IMP_CAL(lane->id));
+
+	val = readl(priv->base + MVEBU_COMPHY_GEN1_S5(lane->id));
+	val &= ~MVEBU_COMPHY_GEN1_S5_ICP(0xf);
+	writel(val, priv->base + MVEBU_COMPHY_GEN1_S5(lane->id));
+
+	val = readl(priv->base + MVEBU_COMPHY_GEN1_S1(lane->id));
+	val &= ~(MVEBU_COMPHY_GEN1_S1_RX_MUL_PI(0x7) |
+		 MVEBU_COMPHY_GEN1_S1_RX_MUL_PF(0x7) |
+		 MVEBU_COMPHY_GEN1_S1_RX_MUL_FI(0x3) |
+		 MVEBU_COMPHY_GEN1_S1_RX_MUL_FF(0x3));
+	val |= MVEBU_COMPHY_GEN1_S1_RX_DFE_EN |
+	       MVEBU_COMPHY_GEN1_S1_RX_MUL_PI(0x2) |
+	       MVEBU_COMPHY_GEN1_S1_RX_MUL_PF(0x2) |
+	       MVEBU_COMPHY_GEN1_S1_RX_MUL_FF(0x1) |
+	       MVEBU_COMPHY_GEN1_S1_RX_DIV(0x3);
+	writel(val, priv->base + MVEBU_COMPHY_GEN1_S1(lane->id));
+
+	val = readl(priv->base + MVEBU_COMPHY_COEF(lane->id));
+	val &= ~(MVEBU_COMPHY_COEF_DFE_EN | MVEBU_COMPHY_COEF_DFE_CTRL);
+	writel(val, priv->base + MVEBU_COMPHY_COEF(lane->id));
+
+	val = readl(priv->base + MVEBU_COMPHY_GEN1_S4(lane->id));
+	val &= ~MVEBU_COMPHY_GEN1_S4_DFE_RES(0x3);
+	val |= MVEBU_COMPHY_GEN1_S4_DFE_RES(0x1);
+	writel(val, priv->base + MVEBU_COMPHY_GEN1_S4(lane->id));
+
+	val = readl(priv->base + MVEBU_COMPHY_GEN1_S3(lane->id));
+	val |= MVEBU_COMPHY_GEN1_S3_FBCK_SEL;
+	writel(val, priv->base + MVEBU_COMPHY_GEN1_S3(lane->id));
+
+	/* rx training timer */
+	val = readl(priv->base + MVEBU_COMPHY_TRAINING5(lane->id));
+	val &= ~MVEBU_COMPHY_TRAINING5_RX_TIMER(0x3ff);
+	val |= MVEBU_COMPHY_TRAINING5_RX_TIMER(0x13);
+	writel(val, priv->base + MVEBU_COMPHY_TRAINING5(lane->id));
+
+	/* tx train peak to peak hold */
+	val = readl(priv->base + MVEBU_COMPHY_TRAINING0(lane->id));
+	val |= MVEBU_COMPHY_TRAINING0_P2P_HOLD;
+	writel(val, priv->base + MVEBU_COMPHY_TRAINING0(lane->id));
+
+	val = readl(priv->base + MVEBU_COMPHY_TX_PRESET(lane->id));
+	val &= ~MVEBU_COMPHY_TX_PRESET_INDEX(0xf);
+	val |= MVEBU_COMPHY_TX_PRESET_INDEX(0x2);	/* preset coeff */
+	writel(val, priv->base + MVEBU_COMPHY_TX_PRESET(lane->id));
+
+	val = readl(priv->base + MVEBU_COMPHY_FRAME_DETECT3(lane->id));
+	val &= ~MVEBU_COMPHY_FRAME_DETECT3_LOST_TIMEOUT_EN;
+	writel(val, priv->base + MVEBU_COMPHY_FRAME_DETECT3(lane->id));
+
+	val = readl(priv->base + MVEBU_COMPHY_TX_TRAIN_PRESET(lane->id));
+	val |= MVEBU_COMPHY_TX_TRAIN_PRESET_16B_AUTO_EN |
+	       MVEBU_COMPHY_TX_TRAIN_PRESET_PRBS11;
+	writel(val, priv->base + MVEBU_COMPHY_TX_TRAIN_PRESET(lane->id));
+
+	val = readl(priv->base + MVEBU_COMPHY_FRAME_DETECT0(lane->id));
+	val &= ~MVEBU_COMPHY_FRAME_DETECT0_PATN(0x1ff);
+	val |= MVEBU_COMPHY_FRAME_DETECT0_PATN(0x88);
+	writel(val, priv->base + MVEBU_COMPHY_FRAME_DETECT0(lane->id));
+
+	val = readl(priv->base + MVEBU_COMPHY_DME(lane->id));
+	val |= MVEBU_COMPHY_DME_ETH_MODE;
+	writel(val, priv->base + MVEBU_COMPHY_DME(lane->id));
+
+	val = readl(priv->base + MVEBU_COMPHY_VDD_CAL0(lane->id));
+	val |= MVEBU_COMPHY_VDD_CAL0_CONT_MODE;
+	writel(val, priv->base + MVEBU_COMPHY_VDD_CAL0(lane->id));
+
+	val = readl(priv->base + MVEBU_SP_CALIB(lane->id));
+	val &= ~MVEBU_SP_CALIB_SAMPLER(0x3);
+	val |= MVEBU_SP_CALIB_SAMPLER(0x3) |
+	       MVEBU_SP_CALIB_SAMPLER_EN;
+	writel(val, priv->base + MVEBU_SP_CALIB(lane->id));
+	val &= ~MVEBU_SP_CALIB_SAMPLER_EN;
+	writel(val, priv->base + MVEBU_SP_CALIB(lane->id));
+
+	/* External rx regulator */
+	val = readl(priv->base + MVEBU_COMPHY_EXT_SELV(lane->id));
+	val &= ~MVEBU_COMPHY_EXT_SELV_RX_SAMPL(0x1f);
+	val |= MVEBU_COMPHY_EXT_SELV_RX_SAMPL(0x1a);
+	writel(val, priv->base + MVEBU_COMPHY_EXT_SELV(lane->id));
+
+	return mvebu_comphy_init_plls(lane, PHY_MODE_10GKR);
+}
+
+static int mvebu_comphy_power_on(struct phy *phy)
+{
+	struct mvebu_comphy_lane *lane = phy_get_drvdata(phy);
+	struct mvebu_comphy_priv *priv = lane->priv;
+	int ret;
+	u32 mux, val;
+
+	mux = mvebu_comphy_get_mux(lane->id, lane->port, lane->mode);
+	if (mux < 0)
+		return -ENOTSUPP;
+
+	regmap_read(priv->regmap, MVEBU_COMPHY_SELECTOR, &val);
+	val &= ~(0xf << MVEBU_COMPHY_SELECTOR_PHY(lane->id));
+	val |= mux << MVEBU_COMPHY_SELECTOR_PHY(lane->id);
+	regmap_write(priv->regmap, MVEBU_COMPHY_SELECTOR, val);
+
+	switch (lane->mode) {
+	case PHY_MODE_SGMII:
+		ret = mvebu_comphy_set_mode_sgmii(phy);
+		break;
+	case PHY_MODE_10GKR:
+		ret = mvebu_comphy_set_mode_10gkr(phy);
+		break;
+	default:
+		return -ENOTSUPP;
+	}
+
+	/* digital reset */
+	val = readl(priv->base + MVEBU_COMPHY_SERDES_CFG1(lane->id));
+	val |= MVEBU_COMPHY_SERDES_CFG1_RF_RESET;
+	writel(val, priv->base + MVEBU_COMPHY_SERDES_CFG1(lane->id));
+
+	return ret;
+}
+
+static int mvebu_comphy_set_mode(struct phy *phy, enum phy_mode mode)
+{
+	struct mvebu_comphy_lane *lane = phy_get_drvdata(phy);
+
+	if (mvebu_comphy_get_mux(lane->id, lane->port, mode) < 0)
+		return -EINVAL;
+
+	lane->mode = mode;
+	return 0;
+}
+
+static int mvebu_comphy_power_off(struct phy *phy)
+{
+	struct mvebu_comphy_lane *lane = phy_get_drvdata(phy);
+	struct mvebu_comphy_priv *priv = lane->priv;
+	u32 val;
+
+	val = readl(priv->base + MVEBU_COMPHY_SERDES_CFG1(lane->id));
+	val &= ~(MVEBU_COMPHY_SERDES_CFG1_RESET |
+		 MVEBU_COMPHY_SERDES_CFG1_CORE_RESET |
+		 MVEBU_COMPHY_SERDES_CFG1_RF_RESET);
+	writel(val, priv->base + MVEBU_COMPHY_SERDES_CFG1(lane->id));
+
+	regmap_read(priv->regmap, MVEBU_COMPHY_SELECTOR, &val);
+	val &= ~(0xf << MVEBU_COMPHY_SELECTOR_PHY(lane->id));
+	regmap_write(priv->regmap, MVEBU_COMPHY_SELECTOR, val);
+
+	return 0;
+}
+
+static const struct phy_ops mvebu_comphy_ops = {
+	.power_on	= mvebu_comphy_power_on,
+	.power_off	= mvebu_comphy_power_off,
+	.set_mode	= mvebu_comphy_set_mode,
+	.owner		= THIS_MODULE,
+};
+
+static struct phy *mvebu_comphy_xlate(struct device *dev,
+				      struct of_phandle_args *args)
+{
+	struct mvebu_comphy_lane *lane;
+	struct phy *phy;
+
+	if (WARN_ON(args->args[0] >= MVEBU_COMPHY_PORTS))
+		return ERR_PTR(-EINVAL);
+
+	phy = of_phy_simple_xlate(dev, args);
+	if (IS_ERR(phy))
+		return phy;
+
+	lane = phy_get_drvdata(phy);
+	if (lane->port >= 0)
+		return ERR_PTR(-EBUSY);
+	lane->port = args->args[0];
+
+	return phy;
+}
+
+static int mvebu_comphy_probe(struct platform_device *pdev)
+{
+	struct mvebu_comphy_priv *priv;
+	struct phy_provider *provider;
+	struct device_node *child;
+	struct resource *res;
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->dev = &pdev->dev;
+	priv->regmap =
+		syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
+						"marvell,system-controller");
+	if (IS_ERR(priv->regmap))
+		return PTR_ERR(priv->regmap);
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	priv->base = devm_ioremap_resource(&pdev->dev, res);
+	if (!priv->base)
+		return -ENOMEM;
+
+	for_each_available_child_of_node(pdev->dev.of_node, child) {
+		struct mvebu_comphy_lane *lane;
+		struct phy *phy;
+		int ret;
+		u32 val;
+
+		ret = of_property_read_u32(child, "reg", &val);
+		if (ret < 0) {
+			dev_err(&pdev->dev, "missing 'reg' property (%d)\n",
+				ret);
+			continue;
+		}
+
+		if (val >= MVEBU_COMPHY_LANES) {
+			dev_err(&pdev->dev, "invalid 'reg' property\n");
+			continue;
+		}
+
+		lane = devm_kzalloc(&pdev->dev, sizeof(*lane), GFP_KERNEL);
+		if (!lane)
+			return -ENOMEM;
+
+		phy = devm_phy_create(&pdev->dev, child, &mvebu_comphy_ops);
+		if (IS_ERR(phy))
+			return PTR_ERR(phy);
+
+		lane->priv = priv;
+		lane->mode = PHY_MODE_INVALID;
+		lane->id = val;
+		lane->port = -1;
+		phy_set_drvdata(phy, lane);
+
+		/*
+		 * Once all modes are supported in this driver we should call
+		 * mvebu_comphy_power_off(phy) here to avoid relying on the
+		 * bootloader/firmware configuration.
+		 */
+	}
+
+	dev_set_drvdata(&pdev->dev, priv);
+	provider = devm_of_phy_provider_register(&pdev->dev,
+						 mvebu_comphy_xlate);
+	return PTR_ERR_OR_ZERO(provider);
+}
+
+static const struct of_device_id mvebu_comphy_of_match_table[] = {
+	{ .compatible = "marvell,comphy-cp110" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, mvebu_comphy_of_match_table);
+
+static struct platform_driver mvebu_comphy_driver = {
+	.probe	= mvebu_comphy_probe,
+	.driver	= {
+		.name = "mvebu-comphy",
+		.of_match_table = mvebu_comphy_of_match_table,
+	},
+};
+module_platform_driver(mvebu_comphy_driver);
+
+MODULE_AUTHOR("Antoine Tenart <antoine.tenart@free-electrons.com>");
+MODULE_DESCRIPTION("Common PHY driver for mvebu SoCs");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/phy/mediatek/Kconfig b/drivers/phy/mediatek/Kconfig
new file mode 100644
index 000000000000..88ab4e25e34f
--- /dev/null
+++ b/drivers/phy/mediatek/Kconfig
@@ -0,0 +1,14 @@
+#
+# Phy drivers for Mediatek devices
+#
+config PHY_MTK_TPHY
+    tristate "MediaTek T-PHY Driver"
+    depends on ARCH_MEDIATEK && OF
+    select GENERIC_PHY
+    help
+      Say 'Y' here to add support for MediaTek T-PHY driver,
+      it supports multiple usb2.0, usb3.0 ports, PCIe and
+	  SATA, and meanwhile supports two version T-PHY which have
+	  different banks layout, the T-PHY with shared banks between
+	  multi-ports is first version, otherwise is second veriosn,
+	  so you can easily distinguish them by banks layout.
diff --git a/drivers/phy/mediatek/Makefile b/drivers/phy/mediatek/Makefile
new file mode 100644
index 000000000000..763a92eefa00
--- /dev/null
+++ b/drivers/phy/mediatek/Makefile
@@ -0,0 +1,5 @@
+#
+# Makefile for the phy drivers.
+#
+
+obj-$(CONFIG_PHY_MTK_TPHY)		+= phy-mtk-tphy.o
diff --git a/drivers/phy/phy-mt65xx-usb3.c b/drivers/phy/mediatek/phy-mtk-tphy.c
index 59b110f795c3..e3baad78521f 100644
--- a/drivers/phy/phy-mt65xx-usb3.c
+++ b/drivers/phy/mediatek/phy-mtk-tphy.c
@@ -29,7 +29,7 @@
 #define SSUSB_SIFSLV_V1_U2FREQ		0x100	/* shared by u2 phys */
 /* u2 phy bank */
 #define SSUSB_SIFSLV_V1_U2PHY_COM	0x000
-/* u3 phy banks */
+/* u3/pcie/sata phy banks */
 #define SSUSB_SIFSLV_V1_U3PHYD		0x000
 #define SSUSB_SIFSLV_V1_U3PHYA		0x200
 
@@ -38,7 +38,7 @@
 #define SSUSB_SIFSLV_V2_MISC		0x000
 #define SSUSB_SIFSLV_V2_U2FREQ		0x100
 #define SSUSB_SIFSLV_V2_U2PHY_COM	0x300
-/* u3 phy banks */
+/* u3/pcie/sata phy banks */
 #define SSUSB_SIFSLV_V2_SPLLC		0x000
 #define SSUSB_SIFSLV_V2_CHIP		0x100
 #define SSUSB_SIFSLV_V2_U3PHYD		0x200
@@ -99,6 +99,23 @@
 #define P2C_RG_SESSEND			BIT(4)
 #define P2C_RG_AVALID			BIT(2)
 
+#define U3P_U3_CHIP_GPIO_CTLD		0x0c
+#define P3C_REG_IP_SW_RST		BIT(31)
+#define P3C_MCU_BUS_CK_GATE_EN		BIT(30)
+#define P3C_FORCE_IP_SW_RST		BIT(29)
+
+#define U3P_U3_CHIP_GPIO_CTLE		0x10
+#define P3C_RG_SWRST_U3_PHYD		BIT(25)
+#define P3C_RG_SWRST_U3_PHYD_FORCE_EN	BIT(24)
+
+#define U3P_U3_PHYA_REG0	0x000
+#define P3A_RG_CLKDRV_OFF		GENMASK(3, 2)
+#define P3A_RG_CLKDRV_OFF_VAL(x)	((0x3 & (x)) << 2)
+
+#define U3P_U3_PHYA_REG1	0x004
+#define P3A_RG_CLKDRV_AMP		GENMASK(31, 29)
+#define P3A_RG_CLKDRV_AMP_VAL(x)	((0x7 & (x)) << 29)
+
 #define U3P_U3_PHYA_REG6	0x018
 #define P3A_RG_TX_EIDLE_CM		GENMASK(31, 28)
 #define P3A_RG_TX_EIDLE_CM_VAL(x)	((0xf & (x)) << 28)
@@ -108,9 +125,40 @@
 #define P3A_RG_RX_DAC_MUX_VAL(x)	((0x1f & (x)) << 1)
 
 #define U3P_U3_PHYA_DA_REG0	0x100
+#define P3A_RG_XTAL_EXT_PE2H		GENMASK(17, 16)
+#define P3A_RG_XTAL_EXT_PE2H_VAL(x)	((0x3 & (x)) << 16)
+#define P3A_RG_XTAL_EXT_PE1H		GENMASK(13, 12)
+#define P3A_RG_XTAL_EXT_PE1H_VAL(x)	((0x3 & (x)) << 12)
 #define P3A_RG_XTAL_EXT_EN_U3		GENMASK(11, 10)
 #define P3A_RG_XTAL_EXT_EN_U3_VAL(x)	((0x3 & (x)) << 10)
 
+#define U3P_U3_PHYA_DA_REG4	0x108
+#define P3A_RG_PLL_DIVEN_PE2H		GENMASK(21, 19)
+#define P3A_RG_PLL_BC_PE2H		GENMASK(7, 6)
+#define P3A_RG_PLL_BC_PE2H_VAL(x)	((0x3 & (x)) << 6)
+
+#define U3P_U3_PHYA_DA_REG5	0x10c
+#define P3A_RG_PLL_BR_PE2H		GENMASK(29, 28)
+#define P3A_RG_PLL_BR_PE2H_VAL(x)	((0x3 & (x)) << 28)
+#define P3A_RG_PLL_IC_PE2H		GENMASK(15, 12)
+#define P3A_RG_PLL_IC_PE2H_VAL(x)	((0xf & (x)) << 12)
+
+#define U3P_U3_PHYA_DA_REG6	0x110
+#define P3A_RG_PLL_IR_PE2H		GENMASK(19, 16)
+#define P3A_RG_PLL_IR_PE2H_VAL(x)	((0xf & (x)) << 16)
+
+#define U3P_U3_PHYA_DA_REG7	0x114
+#define P3A_RG_PLL_BP_PE2H		GENMASK(19, 16)
+#define P3A_RG_PLL_BP_PE2H_VAL(x)	((0xf & (x)) << 16)
+
+#define U3P_U3_PHYA_DA_REG20	0x13c
+#define P3A_RG_PLL_DELTA1_PE2H		GENMASK(31, 16)
+#define P3A_RG_PLL_DELTA1_PE2H_VAL(x)	((0xffff & (x)) << 16)
+
+#define U3P_U3_PHYA_DA_REG25	0x148
+#define P3A_RG_PLL_DELTA_PE2H		GENMASK(15, 0)
+#define P3A_RG_PLL_DELTA_PE2H_VAL(x)	(0xffff & (x))
+
 #define U3P_U3_PHYD_LFPS1		0x00c
 #define P3D_RG_FWAKE_TH		GENMASK(21, 16)
 #define P3D_RG_FWAKE_TH_VAL(x)	((0x3f & (x)) << 16)
@@ -151,15 +199,74 @@
 #define U3P_SR_COEF_DIVISOR	1000
 #define U3P_FM_DET_CYCLE_CNT	1024
 
-enum mt_phy_version {
-	MT_PHY_V1 = 1,
-	MT_PHY_V2,
+/* SATA register setting */
+#define PHYD_CTRL_SIGNAL_MODE4		0x1c
+/* CDR Charge Pump P-path current adjustment */
+#define RG_CDR_BICLTD1_GEN1_MSK		GENMASK(23, 20)
+#define RG_CDR_BICLTD1_GEN1_VAL(x)	((0xf & (x)) << 20)
+#define RG_CDR_BICLTD0_GEN1_MSK		GENMASK(11, 8)
+#define RG_CDR_BICLTD0_GEN1_VAL(x)	((0xf & (x)) << 8)
+
+#define PHYD_DESIGN_OPTION2		0x24
+/* Symbol lock count selection */
+#define RG_LOCK_CNT_SEL_MSK		GENMASK(5, 4)
+#define RG_LOCK_CNT_SEL_VAL(x)		((0x3 & (x)) << 4)
+
+#define PHYD_DESIGN_OPTION9	0x40
+/* COMWAK GAP width window */
+#define RG_TG_MAX_MSK		GENMASK(20, 16)
+#define RG_TG_MAX_VAL(x)	((0x1f & (x)) << 16)
+/* COMINIT GAP width window */
+#define RG_T2_MAX_MSK		GENMASK(13, 8)
+#define RG_T2_MAX_VAL(x)	((0x3f & (x)) << 8)
+/* COMWAK GAP width window */
+#define RG_TG_MIN_MSK		GENMASK(7, 5)
+#define RG_TG_MIN_VAL(x)	((0x7 & (x)) << 5)
+/* COMINIT GAP width window */
+#define RG_T2_MIN_MSK		GENMASK(4, 0)
+#define RG_T2_MIN_VAL(x)	(0x1f & (x))
+
+#define ANA_RG_CTRL_SIGNAL1		0x4c
+/* TX driver tail current control for 0dB de-empahsis mdoe for Gen1 speed */
+#define RG_IDRV_0DB_GEN1_MSK		GENMASK(13, 8)
+#define RG_IDRV_0DB_GEN1_VAL(x)		((0x3f & (x)) << 8)
+
+#define ANA_RG_CTRL_SIGNAL4		0x58
+#define RG_CDR_BICLTR_GEN1_MSK		GENMASK(23, 20)
+#define RG_CDR_BICLTR_GEN1_VAL(x)	((0xf & (x)) << 20)
+/* Loop filter R1 resistance adjustment for Gen1 speed */
+#define RG_CDR_BR_GEN2_MSK		GENMASK(10, 8)
+#define RG_CDR_BR_GEN2_VAL(x)		((0x7 & (x)) << 8)
+
+#define ANA_RG_CTRL_SIGNAL6		0x60
+/* I-path capacitance adjustment for Gen1 */
+#define RG_CDR_BC_GEN1_MSK		GENMASK(28, 24)
+#define RG_CDR_BC_GEN1_VAL(x)		((0x1f & (x)) << 24)
+#define RG_CDR_BIRLTR_GEN1_MSK		GENMASK(4, 0)
+#define RG_CDR_BIRLTR_GEN1_VAL(x)	(0x1f & (x))
+
+#define ANA_EQ_EYE_CTRL_SIGNAL1		0x6c
+/* RX Gen1 LEQ tuning step */
+#define RG_EQ_DLEQ_LFI_GEN1_MSK		GENMASK(11, 8)
+#define RG_EQ_DLEQ_LFI_GEN1_VAL(x)	((0xf & (x)) << 8)
+
+#define ANA_EQ_EYE_CTRL_SIGNAL4		0xd8
+#define RG_CDR_BIRLTD0_GEN1_MSK		GENMASK(20, 16)
+#define RG_CDR_BIRLTD0_GEN1_VAL(x)	((0x1f & (x)) << 16)
+
+#define ANA_EQ_EYE_CTRL_SIGNAL5		0xdc
+#define RG_CDR_BIRLTD0_GEN3_MSK		GENMASK(4, 0)
+#define RG_CDR_BIRLTD0_GEN3_VAL(x)	(0x1f & (x))
+
+enum mtk_phy_version {
+	MTK_PHY_V1 = 1,
+	MTK_PHY_V2,
 };
 
-struct mt65xx_phy_pdata {
+struct mtk_phy_pdata {
 	/* avoid RX sensitivity level degradation only for mt8173 */
 	bool avoid_rx_sen_degradation;
-	enum mt_phy_version version;
+	enum mtk_phy_version version;
 };
 
 struct u2phy_banks {
@@ -175,7 +282,7 @@ struct u3phy_banks {
 	void __iomem *phya; /* include u3phya_da */
 };
 
-struct mt65xx_phy_instance {
+struct mtk_phy_instance {
 	struct phy *phy;
 	void __iomem *port_base;
 	union {
@@ -187,18 +294,18 @@ struct mt65xx_phy_instance {
 	u8 type;
 };
 
-struct mt65xx_u3phy {
+struct mtk_tphy {
 	struct device *dev;
 	void __iomem *sif_base;	/* only shared sif */
 	/* deprecated, use @ref_clk instead in phy instance */
 	struct clk *u3phya_ref;	/* reference clock of usb3 anolog phy */
-	const struct mt65xx_phy_pdata *pdata;
-	struct mt65xx_phy_instance **phys;
+	const struct mtk_phy_pdata *pdata;
+	struct mtk_phy_instance **phys;
 	int nphys;
 };
 
-static void hs_slew_rate_calibrate(struct mt65xx_u3phy *u3phy,
-	struct mt65xx_phy_instance *instance)
+static void hs_slew_rate_calibrate(struct mtk_tphy *tphy,
+	struct mtk_phy_instance *instance)
 {
 	struct u2phy_banks *u2_banks = &instance->u2_banks;
 	void __iomem *fmreg = u2_banks->fmreg;
@@ -222,7 +329,7 @@ static void hs_slew_rate_calibrate(struct mt65xx_u3phy *u3phy,
 	tmp = readl(fmreg + U3P_U2FREQ_FMCR0);
 	tmp &= ~(P2F_RG_CYCLECNT | P2F_RG_MONCLK_SEL);
 	tmp |= P2F_RG_CYCLECNT_VAL(U3P_FM_DET_CYCLE_CNT);
-	if (u3phy->pdata->version == MT_PHY_V1)
+	if (tphy->pdata->version == MTK_PHY_V1)
 		tmp |= P2F_RG_MONCLK_SEL_VAL(instance->index >> 1);
 
 	writel(tmp, fmreg + U3P_U2FREQ_FMCR0);
@@ -257,7 +364,7 @@ static void hs_slew_rate_calibrate(struct mt65xx_u3phy *u3phy,
 		/* if FM detection fail, set default value */
 		calibration_val = 4;
 	}
-	dev_dbg(u3phy->dev, "phy:%d, fm_out:%d, calib:%d\n",
+	dev_dbg(tphy->dev, "phy:%d, fm_out:%d, calib:%d\n",
 		instance->index, fm_out, calibration_val);
 
 	/* set HS slew rate */
@@ -272,8 +379,8 @@ static void hs_slew_rate_calibrate(struct mt65xx_u3phy *u3phy,
 	writel(tmp, com + U3P_USBPHYACR5);
 }
 
-static void u3_phy_instance_init(struct mt65xx_u3phy *u3phy,
-	struct mt65xx_phy_instance *instance)
+static void u3_phy_instance_init(struct mtk_tphy *tphy,
+	struct mtk_phy_instance *instance)
 {
 	struct u3phy_banks *u3_banks = &instance->u3_banks;
 	u32 tmp;
@@ -319,11 +426,11 @@ static void u3_phy_instance_init(struct mt65xx_u3phy *u3phy,
 	tmp |= P3D_RG_RXDET_STB2_SET_P3_VAL(0x10);
 	writel(tmp, u3_banks->phyd + U3P_U3_PHYD_RXDET2);
 
-	dev_dbg(u3phy->dev, "%s(%d)\n", __func__, instance->index);
+	dev_dbg(tphy->dev, "%s(%d)\n", __func__, instance->index);
 }
 
-static void phy_instance_init(struct mt65xx_u3phy *u3phy,
-	struct mt65xx_phy_instance *instance)
+static void u2_phy_instance_init(struct mtk_tphy *tphy,
+	struct mtk_phy_instance *instance)
 {
 	struct u2phy_banks *u2_banks = &instance->u2_banks;
 	void __iomem *com = u2_banks->com;
@@ -355,7 +462,7 @@ static void phy_instance_init(struct mt65xx_u3phy *u3phy,
 		writel(tmp, com + U3P_U2PHYACR4);
 	}
 
-	if (u3phy->pdata->avoid_rx_sen_degradation) {
+	if (tphy->pdata->avoid_rx_sen_degradation) {
 		if (!index) {
 			tmp = readl(com + U3P_USBPHYACR2);
 			tmp |= PA2_RG_SIF_U2PLL_FORCE_EN;
@@ -381,11 +488,11 @@ static void phy_instance_init(struct mt65xx_u3phy *u3phy,
 	tmp |= PA6_RG_U2_SQTH_VAL(2);
 	writel(tmp, com + U3P_USBPHYACR6);
 
-	dev_dbg(u3phy->dev, "%s(%d)\n", __func__, index);
+	dev_dbg(tphy->dev, "%s(%d)\n", __func__, index);
 }
 
-static void phy_instance_power_on(struct mt65xx_u3phy *u3phy,
-	struct mt65xx_phy_instance *instance)
+static void u2_phy_instance_power_on(struct mtk_tphy *tphy,
+	struct mtk_phy_instance *instance)
 {
 	struct u2phy_banks *u2_banks = &instance->u2_banks;
 	void __iomem *com = u2_banks->com;
@@ -408,7 +515,7 @@ static void phy_instance_power_on(struct mt65xx_u3phy *u3phy,
 	tmp &= ~P2C_RG_SESSEND;
 	writel(tmp, com + U3P_U2PHYDTM1);
 
-	if (u3phy->pdata->avoid_rx_sen_degradation && index) {
+	if (tphy->pdata->avoid_rx_sen_degradation && index) {
 		tmp = readl(com + U3D_U2PHYDCR0);
 		tmp |= P2C_RG_SIF_U2PLL_FORCE_ON;
 		writel(tmp, com + U3D_U2PHYDCR0);
@@ -417,11 +524,11 @@ static void phy_instance_power_on(struct mt65xx_u3phy *u3phy,
 		tmp |= P2C_RG_SUSPENDM | P2C_FORCE_SUSPENDM;
 		writel(tmp, com + U3P_U2PHYDTM0);
 	}
-	dev_dbg(u3phy->dev, "%s(%d)\n", __func__, index);
+	dev_dbg(tphy->dev, "%s(%d)\n", __func__, index);
 }
 
-static void phy_instance_power_off(struct mt65xx_u3phy *u3phy,
-	struct mt65xx_phy_instance *instance)
+static void u2_phy_instance_power_off(struct mtk_tphy *tphy,
+	struct mtk_phy_instance *instance)
 {
 	struct u2phy_banks *u2_banks = &instance->u2_banks;
 	void __iomem *com = u2_banks->com;
@@ -449,24 +556,24 @@ static void phy_instance_power_off(struct mt65xx_u3phy *u3phy,
 	tmp |= P2C_RG_SESSEND;
 	writel(tmp, com + U3P_U2PHYDTM1);
 
-	if (u3phy->pdata->avoid_rx_sen_degradation && index) {
+	if (tphy->pdata->avoid_rx_sen_degradation && index) {
 		tmp = readl(com + U3D_U2PHYDCR0);
 		tmp &= ~P2C_RG_SIF_U2PLL_FORCE_ON;
 		writel(tmp, com + U3D_U2PHYDCR0);
 	}
 
-	dev_dbg(u3phy->dev, "%s(%d)\n", __func__, index);
+	dev_dbg(tphy->dev, "%s(%d)\n", __func__, index);
 }
 
-static void phy_instance_exit(struct mt65xx_u3phy *u3phy,
-	struct mt65xx_phy_instance *instance)
+static void u2_phy_instance_exit(struct mtk_tphy *tphy,
+	struct mtk_phy_instance *instance)
 {
 	struct u2phy_banks *u2_banks = &instance->u2_banks;
 	void __iomem *com = u2_banks->com;
 	u32 index = instance->index;
 	u32 tmp;
 
-	if (u3phy->pdata->avoid_rx_sen_degradation && index) {
+	if (tphy->pdata->avoid_rx_sen_degradation && index) {
 		tmp = readl(com + U3D_U2PHYDCR0);
 		tmp &= ~P2C_RG_SIF_U2PLL_FORCE_ON;
 		writel(tmp, com + U3D_U2PHYDCR0);
@@ -477,109 +584,307 @@ static void phy_instance_exit(struct mt65xx_u3phy *u3phy,
 	}
 }
 
-static void phy_v1_banks_init(struct mt65xx_u3phy *u3phy,
-			      struct mt65xx_phy_instance *instance)
+static void pcie_phy_instance_init(struct mtk_tphy *tphy,
+	struct mtk_phy_instance *instance)
+{
+	struct u3phy_banks *u3_banks = &instance->u3_banks;
+	u32 tmp;
+
+	if (tphy->pdata->version != MTK_PHY_V1)
+		return;
+
+	tmp = readl(u3_banks->phya + U3P_U3_PHYA_DA_REG0);
+	tmp &= ~(P3A_RG_XTAL_EXT_PE1H | P3A_RG_XTAL_EXT_PE2H);
+	tmp |= P3A_RG_XTAL_EXT_PE1H_VAL(0x2) | P3A_RG_XTAL_EXT_PE2H_VAL(0x2);
+	writel(tmp, u3_banks->phya + U3P_U3_PHYA_DA_REG0);
+
+	/* ref clk drive */
+	tmp = readl(u3_banks->phya + U3P_U3_PHYA_REG1);
+	tmp &= ~P3A_RG_CLKDRV_AMP;
+	tmp |= P3A_RG_CLKDRV_AMP_VAL(0x4);
+	writel(tmp, u3_banks->phya + U3P_U3_PHYA_REG1);
+
+	tmp = readl(u3_banks->phya + U3P_U3_PHYA_REG0);
+	tmp &= ~P3A_RG_CLKDRV_OFF;
+	tmp |= P3A_RG_CLKDRV_OFF_VAL(0x1);
+	writel(tmp, u3_banks->phya + U3P_U3_PHYA_REG0);
+
+	/* SSC delta -5000ppm */
+	tmp = readl(u3_banks->phya + U3P_U3_PHYA_DA_REG20);
+	tmp &= ~P3A_RG_PLL_DELTA1_PE2H;
+	tmp |= P3A_RG_PLL_DELTA1_PE2H_VAL(0x3c);
+	writel(tmp, u3_banks->phya + U3P_U3_PHYA_DA_REG20);
+
+	tmp = readl(u3_banks->phya + U3P_U3_PHYA_DA_REG25);
+	tmp &= ~P3A_RG_PLL_DELTA_PE2H;
+	tmp |= P3A_RG_PLL_DELTA_PE2H_VAL(0x36);
+	writel(tmp, u3_banks->phya + U3P_U3_PHYA_DA_REG25);
+
+	/* change pll BW 0.6M */
+	tmp = readl(u3_banks->phya + U3P_U3_PHYA_DA_REG5);
+	tmp &= ~(P3A_RG_PLL_BR_PE2H | P3A_RG_PLL_IC_PE2H);
+	tmp |= P3A_RG_PLL_BR_PE2H_VAL(0x1) | P3A_RG_PLL_IC_PE2H_VAL(0x1);
+	writel(tmp, u3_banks->phya + U3P_U3_PHYA_DA_REG5);
+
+	tmp = readl(u3_banks->phya + U3P_U3_PHYA_DA_REG4);
+	tmp &= ~(P3A_RG_PLL_DIVEN_PE2H | P3A_RG_PLL_BC_PE2H);
+	tmp |= P3A_RG_PLL_BC_PE2H_VAL(0x3);
+	writel(tmp, u3_banks->phya + U3P_U3_PHYA_DA_REG4);
+
+	tmp = readl(u3_banks->phya + U3P_U3_PHYA_DA_REG6);
+	tmp &= ~P3A_RG_PLL_IR_PE2H;
+	tmp |= P3A_RG_PLL_IR_PE2H_VAL(0x2);
+	writel(tmp, u3_banks->phya + U3P_U3_PHYA_DA_REG6);
+
+	tmp = readl(u3_banks->phya + U3P_U3_PHYA_DA_REG7);
+	tmp &= ~P3A_RG_PLL_BP_PE2H;
+	tmp |= P3A_RG_PLL_BP_PE2H_VAL(0xa);
+	writel(tmp, u3_banks->phya + U3P_U3_PHYA_DA_REG7);
+
+	/* Tx Detect Rx Timing: 10us -> 5us */
+	tmp = readl(u3_banks->phyd + U3P_U3_PHYD_RXDET1);
+	tmp &= ~P3D_RG_RXDET_STB2_SET;
+	tmp |= P3D_RG_RXDET_STB2_SET_VAL(0x10);
+	writel(tmp, u3_banks->phyd + U3P_U3_PHYD_RXDET1);
+
+	tmp = readl(u3_banks->phyd + U3P_U3_PHYD_RXDET2);
+	tmp &= ~P3D_RG_RXDET_STB2_SET_P3;
+	tmp |= P3D_RG_RXDET_STB2_SET_P3_VAL(0x10);
+	writel(tmp, u3_banks->phyd + U3P_U3_PHYD_RXDET2);
+
+	/* wait for PCIe subsys register to active */
+	usleep_range(2500, 3000);
+	dev_dbg(tphy->dev, "%s(%d)\n", __func__, instance->index);
+}
+
+static void pcie_phy_instance_power_on(struct mtk_tphy *tphy,
+	struct mtk_phy_instance *instance)
+{
+	struct u3phy_banks *bank = &instance->u3_banks;
+	u32 tmp;
+
+	tmp = readl(bank->chip + U3P_U3_CHIP_GPIO_CTLD);
+	tmp &= ~(P3C_FORCE_IP_SW_RST | P3C_MCU_BUS_CK_GATE_EN |
+		P3C_REG_IP_SW_RST);
+	writel(tmp, bank->chip + U3P_U3_CHIP_GPIO_CTLD);
+
+	tmp = readl(bank->chip + U3P_U3_CHIP_GPIO_CTLE);
+	tmp &= ~(P3C_RG_SWRST_U3_PHYD_FORCE_EN | P3C_RG_SWRST_U3_PHYD);
+	writel(tmp, bank->chip + U3P_U3_CHIP_GPIO_CTLE);
+}
+
+static void pcie_phy_instance_power_off(struct mtk_tphy *tphy,
+	struct mtk_phy_instance *instance)
+
+{
+	struct u3phy_banks *bank = &instance->u3_banks;
+	u32 tmp;
+
+	tmp = readl(bank->chip + U3P_U3_CHIP_GPIO_CTLD);
+	tmp |= P3C_FORCE_IP_SW_RST | P3C_REG_IP_SW_RST;
+	writel(tmp, bank->chip + U3P_U3_CHIP_GPIO_CTLD);
+
+	tmp = readl(bank->chip + U3P_U3_CHIP_GPIO_CTLE);
+	tmp |= P3C_RG_SWRST_U3_PHYD_FORCE_EN | P3C_RG_SWRST_U3_PHYD;
+	writel(tmp, bank->chip + U3P_U3_CHIP_GPIO_CTLE);
+}
+
+static void sata_phy_instance_init(struct mtk_tphy *tphy,
+	struct mtk_phy_instance *instance)
+{
+	struct u3phy_banks *u3_banks = &instance->u3_banks;
+	void __iomem *phyd = u3_banks->phyd;
+	u32 tmp;
+
+	/* charge current adjustment */
+	tmp = readl(phyd + ANA_RG_CTRL_SIGNAL6);
+	tmp &= ~(RG_CDR_BIRLTR_GEN1_MSK | RG_CDR_BC_GEN1_MSK);
+	tmp |= RG_CDR_BIRLTR_GEN1_VAL(0x6) | RG_CDR_BC_GEN1_VAL(0x1a);
+	writel(tmp, phyd + ANA_RG_CTRL_SIGNAL6);
+
+	tmp = readl(phyd + ANA_EQ_EYE_CTRL_SIGNAL4);
+	tmp &= ~RG_CDR_BIRLTD0_GEN1_MSK;
+	tmp |= RG_CDR_BIRLTD0_GEN1_VAL(0x18);
+	writel(tmp, phyd + ANA_EQ_EYE_CTRL_SIGNAL4);
+
+	tmp = readl(phyd + ANA_EQ_EYE_CTRL_SIGNAL5);
+	tmp &= ~RG_CDR_BIRLTD0_GEN3_MSK;
+	tmp |= RG_CDR_BIRLTD0_GEN3_VAL(0x06);
+	writel(tmp, phyd + ANA_EQ_EYE_CTRL_SIGNAL5);
+
+	tmp = readl(phyd + ANA_RG_CTRL_SIGNAL4);
+	tmp &= ~(RG_CDR_BICLTR_GEN1_MSK | RG_CDR_BR_GEN2_MSK);
+	tmp |= RG_CDR_BICLTR_GEN1_VAL(0x0c) | RG_CDR_BR_GEN2_VAL(0x07);
+	writel(tmp, phyd + ANA_RG_CTRL_SIGNAL4);
+
+	tmp = readl(phyd + PHYD_CTRL_SIGNAL_MODE4);
+	tmp &= ~(RG_CDR_BICLTD0_GEN1_MSK | RG_CDR_BICLTD1_GEN1_MSK);
+	tmp |= RG_CDR_BICLTD0_GEN1_VAL(0x08) | RG_CDR_BICLTD1_GEN1_VAL(0x02);
+	writel(tmp, phyd + PHYD_CTRL_SIGNAL_MODE4);
+
+	tmp = readl(phyd + PHYD_DESIGN_OPTION2);
+	tmp &= ~RG_LOCK_CNT_SEL_MSK;
+	tmp |= RG_LOCK_CNT_SEL_VAL(0x02);
+	writel(tmp, phyd + PHYD_DESIGN_OPTION2);
+
+	tmp = readl(phyd + PHYD_DESIGN_OPTION9);
+	tmp &= ~(RG_T2_MIN_MSK | RG_TG_MIN_MSK |
+		 RG_T2_MAX_MSK | RG_TG_MAX_MSK);
+	tmp |= RG_T2_MIN_VAL(0x12) | RG_TG_MIN_VAL(0x04) |
+	       RG_T2_MAX_VAL(0x31) | RG_TG_MAX_VAL(0x0e);
+	writel(tmp, phyd + PHYD_DESIGN_OPTION9);
+
+	tmp = readl(phyd + ANA_RG_CTRL_SIGNAL1);
+	tmp &= ~RG_IDRV_0DB_GEN1_MSK;
+	tmp |= RG_IDRV_0DB_GEN1_VAL(0x20);
+	writel(tmp, phyd + ANA_RG_CTRL_SIGNAL1);
+
+	tmp = readl(phyd + ANA_EQ_EYE_CTRL_SIGNAL1);
+	tmp &= ~RG_EQ_DLEQ_LFI_GEN1_MSK;
+	tmp |= RG_EQ_DLEQ_LFI_GEN1_VAL(0x03);
+	writel(tmp, phyd + ANA_EQ_EYE_CTRL_SIGNAL1);
+
+	dev_dbg(tphy->dev, "%s(%d)\n", __func__, instance->index);
+}
+
+static void phy_v1_banks_init(struct mtk_tphy *tphy,
+			      struct mtk_phy_instance *instance)
 {
 	struct u2phy_banks *u2_banks = &instance->u2_banks;
 	struct u3phy_banks *u3_banks = &instance->u3_banks;
 
-	if (instance->type == PHY_TYPE_USB2) {
+	switch (instance->type) {
+	case PHY_TYPE_USB2:
 		u2_banks->misc = NULL;
-		u2_banks->fmreg = u3phy->sif_base + SSUSB_SIFSLV_V1_U2FREQ;
+		u2_banks->fmreg = tphy->sif_base + SSUSB_SIFSLV_V1_U2FREQ;
 		u2_banks->com = instance->port_base + SSUSB_SIFSLV_V1_U2PHY_COM;
-	} else if (instance->type == PHY_TYPE_USB3) {
-		u3_banks->spllc = u3phy->sif_base + SSUSB_SIFSLV_V1_SPLLC;
+		break;
+	case PHY_TYPE_USB3:
+	case PHY_TYPE_PCIE:
+		u3_banks->spllc = tphy->sif_base + SSUSB_SIFSLV_V1_SPLLC;
 		u3_banks->chip = NULL;
 		u3_banks->phyd = instance->port_base + SSUSB_SIFSLV_V1_U3PHYD;
 		u3_banks->phya = instance->port_base + SSUSB_SIFSLV_V1_U3PHYA;
+		break;
+	case PHY_TYPE_SATA:
+		u3_banks->phyd = instance->port_base + SSUSB_SIFSLV_V1_U3PHYD;
+		break;
+	default:
+		dev_err(tphy->dev, "incompatible PHY type\n");
+		return;
 	}
 }
 
-static void phy_v2_banks_init(struct mt65xx_u3phy *u3phy,
-			      struct mt65xx_phy_instance *instance)
+static void phy_v2_banks_init(struct mtk_tphy *tphy,
+			      struct mtk_phy_instance *instance)
 {
 	struct u2phy_banks *u2_banks = &instance->u2_banks;
 	struct u3phy_banks *u3_banks = &instance->u3_banks;
 
-	if (instance->type == PHY_TYPE_USB2) {
+	switch (instance->type) {
+	case PHY_TYPE_USB2:
 		u2_banks->misc = instance->port_base + SSUSB_SIFSLV_V2_MISC;
 		u2_banks->fmreg = instance->port_base + SSUSB_SIFSLV_V2_U2FREQ;
 		u2_banks->com = instance->port_base + SSUSB_SIFSLV_V2_U2PHY_COM;
-	} else if (instance->type == PHY_TYPE_USB3) {
+		break;
+	case PHY_TYPE_USB3:
+	case PHY_TYPE_PCIE:
 		u3_banks->spllc = instance->port_base + SSUSB_SIFSLV_V2_SPLLC;
 		u3_banks->chip = instance->port_base + SSUSB_SIFSLV_V2_CHIP;
 		u3_banks->phyd = instance->port_base + SSUSB_SIFSLV_V2_U3PHYD;
 		u3_banks->phya = instance->port_base + SSUSB_SIFSLV_V2_U3PHYA;
+		break;
+	default:
+		dev_err(tphy->dev, "incompatible PHY type\n");
+		return;
 	}
 }
 
-static int mt65xx_phy_init(struct phy *phy)
+static int mtk_phy_init(struct phy *phy)
 {
-	struct mt65xx_phy_instance *instance = phy_get_drvdata(phy);
-	struct mt65xx_u3phy *u3phy = dev_get_drvdata(phy->dev.parent);
+	struct mtk_phy_instance *instance = phy_get_drvdata(phy);
+	struct mtk_tphy *tphy = dev_get_drvdata(phy->dev.parent);
 	int ret;
 
-	ret = clk_prepare_enable(u3phy->u3phya_ref);
+	ret = clk_prepare_enable(tphy->u3phya_ref);
 	if (ret) {
-		dev_err(u3phy->dev, "failed to enable u3phya_ref\n");
+		dev_err(tphy->dev, "failed to enable u3phya_ref\n");
 		return ret;
 	}
 
 	ret = clk_prepare_enable(instance->ref_clk);
 	if (ret) {
-		dev_err(u3phy->dev, "failed to enable ref_clk\n");
+		dev_err(tphy->dev, "failed to enable ref_clk\n");
 		return ret;
 	}
 
-	if (instance->type == PHY_TYPE_USB2)
-		phy_instance_init(u3phy, instance);
-	else
-		u3_phy_instance_init(u3phy, instance);
+	switch (instance->type) {
+	case PHY_TYPE_USB2:
+		u2_phy_instance_init(tphy, instance);
+		break;
+	case PHY_TYPE_USB3:
+		u3_phy_instance_init(tphy, instance);
+		break;
+	case PHY_TYPE_PCIE:
+		pcie_phy_instance_init(tphy, instance);
+		break;
+	case PHY_TYPE_SATA:
+		sata_phy_instance_init(tphy, instance);
+		break;
+	default:
+		dev_err(tphy->dev, "incompatible PHY type\n");
+		return -EINVAL;
+	}
 
 	return 0;
 }
 
-static int mt65xx_phy_power_on(struct phy *phy)
+static int mtk_phy_power_on(struct phy *phy)
 {
-	struct mt65xx_phy_instance *instance = phy_get_drvdata(phy);
-	struct mt65xx_u3phy *u3phy = dev_get_drvdata(phy->dev.parent);
+	struct mtk_phy_instance *instance = phy_get_drvdata(phy);
+	struct mtk_tphy *tphy = dev_get_drvdata(phy->dev.parent);
 
 	if (instance->type == PHY_TYPE_USB2) {
-		phy_instance_power_on(u3phy, instance);
-		hs_slew_rate_calibrate(u3phy, instance);
+		u2_phy_instance_power_on(tphy, instance);
+		hs_slew_rate_calibrate(tphy, instance);
+	} else if (instance->type == PHY_TYPE_PCIE) {
+		pcie_phy_instance_power_on(tphy, instance);
 	}
+
 	return 0;
 }
 
-static int mt65xx_phy_power_off(struct phy *phy)
+static int mtk_phy_power_off(struct phy *phy)
 {
-	struct mt65xx_phy_instance *instance = phy_get_drvdata(phy);
-	struct mt65xx_u3phy *u3phy = dev_get_drvdata(phy->dev.parent);
+	struct mtk_phy_instance *instance = phy_get_drvdata(phy);
+	struct mtk_tphy *tphy = dev_get_drvdata(phy->dev.parent);
 
 	if (instance->type == PHY_TYPE_USB2)
-		phy_instance_power_off(u3phy, instance);
+		u2_phy_instance_power_off(tphy, instance);
+	else if (instance->type == PHY_TYPE_PCIE)
+		pcie_phy_instance_power_off(tphy, instance);
 
 	return 0;
 }
 
-static int mt65xx_phy_exit(struct phy *phy)
+static int mtk_phy_exit(struct phy *phy)
 {
-	struct mt65xx_phy_instance *instance = phy_get_drvdata(phy);
-	struct mt65xx_u3phy *u3phy = dev_get_drvdata(phy->dev.parent);
+	struct mtk_phy_instance *instance = phy_get_drvdata(phy);
+	struct mtk_tphy *tphy = dev_get_drvdata(phy->dev.parent);
 
 	if (instance->type == PHY_TYPE_USB2)
-		phy_instance_exit(u3phy, instance);
+		u2_phy_instance_exit(tphy, instance);
 
 	clk_disable_unprepare(instance->ref_clk);
-	clk_disable_unprepare(u3phy->u3phya_ref);
+	clk_disable_unprepare(tphy->u3phya_ref);
 	return 0;
 }
 
-static struct phy *mt65xx_phy_xlate(struct device *dev,
+static struct phy *mtk_phy_xlate(struct device *dev,
 					struct of_phandle_args *args)
 {
-	struct mt65xx_u3phy *u3phy = dev_get_drvdata(dev);
-	struct mt65xx_phy_instance *instance = NULL;
+	struct mtk_tphy *tphy = dev_get_drvdata(dev);
+	struct mtk_phy_instance *instance = NULL;
 	struct device_node *phy_np = args->np;
 	int index;
 
@@ -588,9 +893,9 @@ static struct phy *mt65xx_phy_xlate(struct device *dev,
 		return ERR_PTR(-EINVAL);
 	}
 
-	for (index = 0; index < u3phy->nphys; index++)
-		if (phy_np == u3phy->phys[index]->phy->dev.of_node) {
-			instance = u3phy->phys[index];
+	for (index = 0; index < tphy->nphys; index++)
+		if (phy_np == tphy->phys[index]->phy->dev.of_node) {
+			instance = tphy->phys[index];
 			break;
 		}
 
@@ -601,15 +906,17 @@ static struct phy *mt65xx_phy_xlate(struct device *dev,
 
 	instance->type = args->args[0];
 	if (!(instance->type == PHY_TYPE_USB2 ||
-	      instance->type == PHY_TYPE_USB3)) {
+	      instance->type == PHY_TYPE_USB3 ||
+	      instance->type == PHY_TYPE_PCIE ||
+	      instance->type == PHY_TYPE_SATA)) {
 		dev_err(dev, "unsupported device type: %d\n", instance->type);
 		return ERR_PTR(-EINVAL);
 	}
 
-	if (u3phy->pdata->version == MT_PHY_V1) {
-		phy_v1_banks_init(u3phy, instance);
-	} else if (u3phy->pdata->version == MT_PHY_V2) {
-		phy_v2_banks_init(u3phy, instance);
+	if (tphy->pdata->version == MTK_PHY_V1) {
+		phy_v1_banks_init(tphy, instance);
+	} else if (tphy->pdata->version == MTK_PHY_V2) {
+		phy_v2_banks_init(tphy, instance);
 	} else {
 		dev_err(dev, "phy version is not supported\n");
 		return ERR_PTR(-EINVAL);
@@ -618,38 +925,40 @@ static struct phy *mt65xx_phy_xlate(struct device *dev,
 	return instance->phy;
 }
 
-static const struct phy_ops mt65xx_u3phy_ops = {
-	.init		= mt65xx_phy_init,
-	.exit		= mt65xx_phy_exit,
-	.power_on	= mt65xx_phy_power_on,
-	.power_off	= mt65xx_phy_power_off,
+static const struct phy_ops mtk_tphy_ops = {
+	.init		= mtk_phy_init,
+	.exit		= mtk_phy_exit,
+	.power_on	= mtk_phy_power_on,
+	.power_off	= mtk_phy_power_off,
 	.owner		= THIS_MODULE,
 };
 
-static const struct mt65xx_phy_pdata mt2701_pdata = {
+static const struct mtk_phy_pdata tphy_v1_pdata = {
 	.avoid_rx_sen_degradation = false,
-	.version = MT_PHY_V1,
+	.version = MTK_PHY_V1,
 };
 
-static const struct mt65xx_phy_pdata mt2712_pdata = {
+static const struct mtk_phy_pdata tphy_v2_pdata = {
 	.avoid_rx_sen_degradation = false,
-	.version = MT_PHY_V2,
+	.version = MTK_PHY_V2,
 };
 
-static const struct mt65xx_phy_pdata mt8173_pdata = {
+static const struct mtk_phy_pdata mt8173_pdata = {
 	.avoid_rx_sen_degradation = true,
-	.version = MT_PHY_V1,
+	.version = MTK_PHY_V1,
 };
 
-static const struct of_device_id mt65xx_u3phy_id_table[] = {
-	{ .compatible = "mediatek,mt2701-u3phy", .data = &mt2701_pdata },
-	{ .compatible = "mediatek,mt2712-u3phy", .data = &mt2712_pdata },
+static const struct of_device_id mtk_tphy_id_table[] = {
+	{ .compatible = "mediatek,mt2701-u3phy", .data = &tphy_v1_pdata },
+	{ .compatible = "mediatek,mt2712-u3phy", .data = &tphy_v2_pdata },
 	{ .compatible = "mediatek,mt8173-u3phy", .data = &mt8173_pdata },
+	{ .compatible = "mediatek,generic-tphy-v1", .data = &tphy_v1_pdata },
+	{ .compatible = "mediatek,generic-tphy-v2", .data = &tphy_v2_pdata },
 	{ },
 };
-MODULE_DEVICE_TABLE(of, mt65xx_u3phy_id_table);
+MODULE_DEVICE_TABLE(of, mtk_tphy_id_table);
 
-static int mt65xx_u3phy_probe(struct platform_device *pdev)
+static int mtk_tphy_probe(struct platform_device *pdev)
 {
 	const struct of_device_id *match;
 	struct device *dev = &pdev->dev;
@@ -657,50 +966,50 @@ static int mt65xx_u3phy_probe(struct platform_device *pdev)
 	struct device_node *child_np;
 	struct phy_provider *provider;
 	struct resource *sif_res;
-	struct mt65xx_u3phy *u3phy;
+	struct mtk_tphy *tphy;
 	struct resource res;
 	int port, retval;
 
-	match = of_match_node(mt65xx_u3phy_id_table, pdev->dev.of_node);
+	match = of_match_node(mtk_tphy_id_table, pdev->dev.of_node);
 	if (!match)
 		return -EINVAL;
 
-	u3phy = devm_kzalloc(dev, sizeof(*u3phy), GFP_KERNEL);
-	if (!u3phy)
+	tphy = devm_kzalloc(dev, sizeof(*tphy), GFP_KERNEL);
+	if (!tphy)
 		return -ENOMEM;
 
-	u3phy->pdata = match->data;
-	u3phy->nphys = of_get_child_count(np);
-	u3phy->phys = devm_kcalloc(dev, u3phy->nphys,
-				       sizeof(*u3phy->phys), GFP_KERNEL);
-	if (!u3phy->phys)
+	tphy->pdata = match->data;
+	tphy->nphys = of_get_child_count(np);
+	tphy->phys = devm_kcalloc(dev, tphy->nphys,
+				       sizeof(*tphy->phys), GFP_KERNEL);
+	if (!tphy->phys)
 		return -ENOMEM;
 
-	u3phy->dev = dev;
-	platform_set_drvdata(pdev, u3phy);
+	tphy->dev = dev;
+	platform_set_drvdata(pdev, tphy);
 
-	if (u3phy->pdata->version == MT_PHY_V1) {
+	if (tphy->pdata->version == MTK_PHY_V1) {
 		/* get banks shared by multiple phys */
 		sif_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-		u3phy->sif_base = devm_ioremap_resource(dev, sif_res);
-		if (IS_ERR(u3phy->sif_base)) {
+		tphy->sif_base = devm_ioremap_resource(dev, sif_res);
+		if (IS_ERR(tphy->sif_base)) {
 			dev_err(dev, "failed to remap sif regs\n");
-			return PTR_ERR(u3phy->sif_base);
+			return PTR_ERR(tphy->sif_base);
 		}
 	}
 
 	/* it's deprecated, make it optional for backward compatibility */
-	u3phy->u3phya_ref = devm_clk_get(dev, "u3phya_ref");
-	if (IS_ERR(u3phy->u3phya_ref)) {
-		if (PTR_ERR(u3phy->u3phya_ref) == -EPROBE_DEFER)
+	tphy->u3phya_ref = devm_clk_get(dev, "u3phya_ref");
+	if (IS_ERR(tphy->u3phya_ref)) {
+		if (PTR_ERR(tphy->u3phya_ref) == -EPROBE_DEFER)
 			return -EPROBE_DEFER;
 
-		u3phy->u3phya_ref = NULL;
+		tphy->u3phya_ref = NULL;
 	}
 
 	port = 0;
 	for_each_child_of_node(np, child_np) {
-		struct mt65xx_phy_instance *instance;
+		struct mtk_phy_instance *instance;
 		struct phy *phy;
 
 		instance = devm_kzalloc(dev, sizeof(*instance), GFP_KERNEL);
@@ -709,9 +1018,9 @@ static int mt65xx_u3phy_probe(struct platform_device *pdev)
 			goto put_child;
 		}
 
-		u3phy->phys[port] = instance;
+		tphy->phys[port] = instance;
 
-		phy = devm_phy_create(dev, child_np, &mt65xx_u3phy_ops);
+		phy = devm_phy_create(dev, child_np, &mtk_tphy_ops);
 		if (IS_ERR(phy)) {
 			dev_err(dev, "failed to create phy\n");
 			retval = PTR_ERR(phy);
@@ -738,7 +1047,7 @@ static int mt65xx_u3phy_probe(struct platform_device *pdev)
 		port++;
 
 		/* if deprecated clock is provided, ignore instance's one */
-		if (u3phy->u3phya_ref)
+		if (tphy->u3phya_ref)
 			continue;
 
 		instance->ref_clk = devm_clk_get(&phy->dev, "ref");
@@ -749,7 +1058,7 @@ static int mt65xx_u3phy_probe(struct platform_device *pdev)
 		}
 	}
 
-	provider = devm_of_phy_provider_register(dev, mt65xx_phy_xlate);
+	provider = devm_of_phy_provider_register(dev, mtk_phy_xlate);
 
 	return PTR_ERR_OR_ZERO(provider);
 put_child:
@@ -757,16 +1066,16 @@ put_child:
 	return retval;
 }
 
-static struct platform_driver mt65xx_u3phy_driver = {
-	.probe		= mt65xx_u3phy_probe,
+static struct platform_driver mtk_tphy_driver = {
+	.probe		= mtk_tphy_probe,
 	.driver		= {
-		.name	= "mt65xx-u3phy",
-		.of_match_table = mt65xx_u3phy_id_table,
+		.name	= "mtk-tphy",
+		.of_match_table = mtk_tphy_id_table,
 	},
 };
 
-module_platform_driver(mt65xx_u3phy_driver);
+module_platform_driver(mtk_tphy_driver);
 
 MODULE_AUTHOR("Chunfeng Yun <chunfeng.yun@mediatek.com>");
-MODULE_DESCRIPTION("mt65xx USB PHY driver");
+MODULE_DESCRIPTION("MediaTek T-PHY driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/phy/motorola/phy-cpcap-usb.c b/drivers/phy/motorola/phy-cpcap-usb.c
index 9b63efa5ae4d..accaaaccb662 100644
--- a/drivers/phy/motorola/phy-cpcap-usb.c
+++ b/drivers/phy/motorola/phy-cpcap-usb.c
@@ -506,7 +506,7 @@ static void cpcap_usb_init_optional_gpios(struct cpcap_phy_ddata *ddata)
 		if (IS_ERR(ddata->gpio[i])) {
 			dev_info(ddata->dev, "no mode change GPIO%i: %li\n",
 				 i, PTR_ERR(ddata->gpio[i]));
-				 ddata->gpio[i] = NULL;
+			ddata->gpio[i] = NULL;
 		}
 	}
 }
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp.c b/drivers/phy/qualcomm/phy-qcom-qmp.c
index 78ca62897784..e17f0351ccc2 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp.c
+++ b/drivers/phy/qualcomm/phy-qcom-qmp.c
@@ -59,6 +59,7 @@
 #define QSERDES_COM_PLL_RCTRL_MODE1			0x088
 #define QSERDES_COM_PLL_CCTRL_MODE0			0x090
 #define QSERDES_COM_PLL_CCTRL_MODE1			0x094
+#define QSERDES_COM_BIAS_EN_CTRL_BY_PSM			0x0a8
 #define QSERDES_COM_SYSCLK_EN_SEL			0x0ac
 #define QSERDES_COM_RESETSM_CNTRL			0x0b4
 #define QSERDES_COM_RESTRIM_CTRL			0x0bc
@@ -143,6 +144,11 @@
 #define QPHY_LOCK_DETECT_CONFIG3			0x88
 #define QPHY_PWRUP_RESET_DLY_TIME_AUXCLK		0xa0
 #define QPHY_LP_WAKEUP_DLY_TIME_AUXCLK			0xa4
+#define QPHY_PLL_LOCK_CHK_DLY_TIME_AUXCLK_LSB		0x1A8
+#define QPHY_OSC_DTCT_ACTIONS				0x1AC
+#define QPHY_RX_SIGDET_LVL				0x1D8
+#define QPHY_L1SS_WAKEUP_DLY_TIME_AUXCLK_LSB		0x1DC
+#define QPHY_L1SS_WAKEUP_DLY_TIME_AUXCLK_MSB		0x1E0
 
 /* QPHY_SW_RESET bit */
 #define SW_RESET				BIT(0)
@@ -382,6 +388,85 @@ static const struct qmp_phy_init_tbl msm8996_usb3_pcs_tbl[] = {
 	QMP_PHY_INIT_CFG(QPHY_POWER_STATE_CONFIG2, 0x08),
 };
 
+static const struct qmp_phy_init_tbl ipq8074_pcie_serdes_tbl[] = {
+	QMP_PHY_INIT_CFG(QSERDES_COM_BIAS_EN_CLKBUFLR_EN, 0x18),
+	QMP_PHY_INIT_CFG(QSERDES_COM_CLK_ENABLE1, 0x10),
+	QMP_PHY_INIT_CFG(QSERDES_COM_BG_TRIM, 0xf),
+	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP_EN, 0x1),
+	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_MAP, 0x0),
+	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_TIMER1, 0x1f),
+	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_TIMER2, 0x3f),
+	QMP_PHY_INIT_CFG(QSERDES_COM_CMN_CONFIG, 0x6),
+	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_IVCO, 0xf),
+	QMP_PHY_INIT_CFG(QSERDES_COM_HSCLK_SEL, 0x0),
+	QMP_PHY_INIT_CFG(QSERDES_COM_SVS_MODE_CLK_SEL, 0x1),
+	QMP_PHY_INIT_CFG(QSERDES_COM_CORE_CLK_EN, 0x20),
+	QMP_PHY_INIT_CFG(QSERDES_COM_CORECLK_DIV, 0xa),
+	QMP_PHY_INIT_CFG(QSERDES_COM_RESETSM_CNTRL, 0x20),
+	QMP_PHY_INIT_CFG(QSERDES_COM_BG_TIMER, 0xa),
+	QMP_PHY_INIT_CFG(QSERDES_COM_SYSCLK_EN_SEL, 0xa),
+	QMP_PHY_INIT_CFG(QSERDES_COM_DEC_START_MODE0, 0x82),
+	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START3_MODE0, 0x3),
+	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START2_MODE0, 0x55),
+	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START1_MODE0, 0x55),
+	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP3_MODE0, 0x0),
+	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP2_MODE0, 0xD),
+	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP1_MODE0, 0xD04),
+	QMP_PHY_INIT_CFG(QSERDES_COM_CLK_SELECT, 0x33),
+	QMP_PHY_INIT_CFG(QSERDES_COM_SYS_CLK_CTRL, 0x2),
+	QMP_PHY_INIT_CFG(QSERDES_COM_SYSCLK_BUF_ENABLE, 0x1f),
+	QMP_PHY_INIT_CFG(QSERDES_COM_CP_CTRL_MODE0, 0xb),
+	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_RCTRL_MODE0, 0x16),
+	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_CCTRL_MODE0, 0x28),
+	QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_GAIN1_MODE0, 0x0),
+	QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_GAIN0_MODE0, 0x80),
+	QMP_PHY_INIT_CFG(QSERDES_COM_BIAS_EN_CTRL_BY_PSM, 0x1),
+	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_CTRL, 0xa),
+	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_EN_CENTER, 0x1),
+	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_PER1, 0x31),
+	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_PER2, 0x1),
+	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_ADJ_PER1, 0x2),
+	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_ADJ_PER2, 0x0),
+	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_STEP_SIZE1, 0x2f),
+	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_STEP_SIZE2, 0x19),
+	QMP_PHY_INIT_CFG(QSERDES_COM_CLK_EP_DIV, 0x19),
+	QMP_PHY_INIT_CFG(QSERDES_RX_SIGDET_CNTRL, 0x7),
+};
+
+static const struct qmp_phy_init_tbl ipq8074_pcie_tx_tbl[] = {
+	QMP_PHY_INIT_CFG(QSERDES_TX_HIGHZ_TRANSCEIVEREN_BIAS_DRVR_EN, 0x45),
+	QMP_PHY_INIT_CFG(QSERDES_TX_LANE_MODE, 0x6),
+	QMP_PHY_INIT_CFG(QSERDES_TX_RES_CODE_LANE_OFFSET, 0x2),
+	QMP_PHY_INIT_CFG(QSERDES_TX_RCV_DETECT_LVL_2, 0x12),
+};
+
+static const struct qmp_phy_init_tbl ipq8074_pcie_rx_tbl[] = {
+	QMP_PHY_INIT_CFG(QSERDES_RX_SIGDET_ENABLES, 0x1c),
+	QMP_PHY_INIT_CFG(QSERDES_RX_SIGDET_DEGLITCH_CNTRL, 0x14),
+	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQU_ADAPTOR_CNTRL2, 0x1),
+	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQU_ADAPTOR_CNTRL3, 0x0),
+	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQU_ADAPTOR_CNTRL4, 0xdb),
+	QMP_PHY_INIT_CFG(QSERDES_RX_UCDR_SO_SATURATION_AND_ENABLE, 0x4b),
+	QMP_PHY_INIT_CFG(QSERDES_RX_UCDR_SO_GAIN, 0x4),
+	QMP_PHY_INIT_CFG(QSERDES_RX_UCDR_SO_GAIN_HALF, 0x4),
+};
+
+static const struct qmp_phy_init_tbl ipq8074_pcie_pcs_tbl[] = {
+	QMP_PHY_INIT_CFG(QPHY_ENDPOINT_REFCLK_DRIVE, 0x4),
+	QMP_PHY_INIT_CFG(QPHY_OSC_DTCT_ACTIONS, 0x0),
+	QMP_PHY_INIT_CFG(QPHY_PWRUP_RESET_DLY_TIME_AUXCLK, 0x40),
+	QMP_PHY_INIT_CFG(QPHY_L1SS_WAKEUP_DLY_TIME_AUXCLK_MSB, 0x0),
+	QMP_PHY_INIT_CFG(QPHY_L1SS_WAKEUP_DLY_TIME_AUXCLK_LSB, 0x40),
+	QMP_PHY_INIT_CFG(QPHY_PLL_LOCK_CHK_DLY_TIME_AUXCLK_LSB, 0x0),
+	QMP_PHY_INIT_CFG(QPHY_LP_WAKEUP_DLY_TIME_AUXCLK, 0x40),
+	QMP_PHY_INIT_CFG_L(QPHY_PLL_LOCK_CHK_DLY_TIME, 0x73),
+	QMP_PHY_INIT_CFG(QPHY_RX_SIGDET_LVL, 0x99),
+	QMP_PHY_INIT_CFG(QPHY_TXDEEMPH_M6DB_V0, 0x15),
+	QMP_PHY_INIT_CFG(QPHY_TXDEEMPH_M3P5DB_V0, 0xe),
+	QMP_PHY_INIT_CFG_L(QPHY_SW_RESET, 0x0),
+	QMP_PHY_INIT_CFG_L(QPHY_START_CTRL, 0x3),
+};
+
 /* struct qmp_phy_cfg - per-PHY initialization config */
 struct qmp_phy_cfg {
 	/* phy-type - PCIE/UFS/USB */
@@ -580,6 +665,42 @@ static const struct qmp_phy_cfg msm8996_usb3phy_cfg = {
 	.mask_pcs_ready		= PHYSTATUS,
 };
 
+/* list of resets */
+static const char * const ipq8074_pciephy_reset_l[] = {
+	"phy", "common",
+};
+
+static const struct qmp_phy_cfg ipq8074_pciephy_cfg = {
+	.type			= PHY_TYPE_PCIE,
+	.nlanes			= 1,
+
+	.serdes_tbl		= ipq8074_pcie_serdes_tbl,
+	.serdes_tbl_num		= ARRAY_SIZE(ipq8074_pcie_serdes_tbl),
+	.tx_tbl			= ipq8074_pcie_tx_tbl,
+	.tx_tbl_num		= ARRAY_SIZE(ipq8074_pcie_tx_tbl),
+	.rx_tbl			= ipq8074_pcie_rx_tbl,
+	.rx_tbl_num		= ARRAY_SIZE(ipq8074_pcie_rx_tbl),
+	.pcs_tbl		= ipq8074_pcie_pcs_tbl,
+	.pcs_tbl_num		= ARRAY_SIZE(ipq8074_pcie_pcs_tbl),
+	.clk_list		= NULL,
+	.num_clks		= 0,
+	.reset_list		= ipq8074_pciephy_reset_l,
+	.num_resets		= ARRAY_SIZE(ipq8074_pciephy_reset_l),
+	.vreg_list		= NULL,
+	.num_vregs		= 0,
+	.regs			= pciephy_regs_layout,
+
+	.start_ctrl		= SERDES_START | PCS_START,
+	.pwrdn_ctrl		= SW_PWRDN | REFCLK_DRV_DSBL,
+	.mask_pcs_ready		= PHYSTATUS,
+
+	.has_phy_com_ctrl	= false,
+	.has_lane_rst		= false,
+	.has_pwrdn_delay	= true,
+	.pwrdn_delay_min	= 995,		/* us */
+	.pwrdn_delay_max	= 1005,		/* us */
+};
+
 static void qcom_qmp_phy_configure(void __iomem *base,
 				   const unsigned int *regs,
 				   const struct qmp_phy_init_tbl tbl[],
@@ -654,8 +775,6 @@ static int qcom_qmp_phy_com_init(struct qcom_qmp *qmp)
 		if (ret) {
 			dev_err(qmp->dev, "%s reset deassert failed\n",
 				qmp->cfg->reset_list[i]);
-			while (--i >= 0)
-				reset_control_assert(qmp->resets[i]);
 			goto err_rst;
 		}
 	}
@@ -684,7 +803,7 @@ static int qcom_qmp_phy_com_init(struct qcom_qmp *qmp)
 		if (ret) {
 			dev_err(qmp->dev,
 				"phy common block init timed-out\n");
-			goto err_com_init;
+			goto err_rst;
 		}
 	}
 
@@ -692,11 +811,11 @@ static int qcom_qmp_phy_com_init(struct qcom_qmp *qmp)
 
 	return 0;
 
-err_com_init:
+err_rst:
 	while (--i >= 0)
 		reset_control_assert(qmp->resets[i]);
-err_rst:
 	mutex_unlock(&qmp->phy_mutex);
+
 	return ret;
 }
 
@@ -749,14 +868,13 @@ static int qcom_qmp_phy_init(struct phy *phy)
 		if (ret) {
 			dev_err(qmp->dev, "failed to enable %s clk, err=%d\n",
 				qmp->cfg->clk_list[i], ret);
-			while (--i >= 0)
-				clk_disable_unprepare(qmp->clks[i]);
+			goto err_clk;
 		}
 	}
 
 	ret = qcom_qmp_phy_com_init(qmp);
 	if (ret)
-		goto err_com_init;
+		goto err_clk;
 
 	if (cfg->has_lane_rst) {
 		ret = reset_control_deassert(qphy->lane_rst);
@@ -804,7 +922,7 @@ err_pcs_ready:
 		reset_control_assert(qphy->lane_rst);
 err_lane_rst:
 	qcom_qmp_phy_com_exit(qmp);
-err_com_init:
+err_clk:
 	while (--i >= 0)
 		clk_disable_unprepare(qmp->clks[i]);
 
@@ -925,29 +1043,28 @@ static int qcom_qmp_phy_clk_init(struct device *dev)
  *    clk  |   +-------+   |                   +-----+
  *         +---------------+
  */
-static int phy_pipe_clk_register(struct qcom_qmp *qmp, int id)
+static int phy_pipe_clk_register(struct qcom_qmp *qmp, struct device_node *np)
 {
-	char name[24];
 	struct clk_fixed_rate *fixed;
 	struct clk_init_data init = { };
+	int ret;
 
-	switch (qmp->cfg->type) {
-	case PHY_TYPE_USB3:
-		snprintf(name, sizeof(name), "usb3_phy_pipe_clk_src");
-		break;
-	case PHY_TYPE_PCIE:
-		snprintf(name, sizeof(name), "pcie_%d_pipe_clk_src", id);
-		break;
-	default:
+	if ((qmp->cfg->type != PHY_TYPE_USB3) &&
+	    (qmp->cfg->type != PHY_TYPE_PCIE)) {
 		/* not all phys register pipe clocks, so return success */
 		return 0;
 	}
 
+	ret = of_property_read_string(np, "clock-output-names", &init.name);
+	if (ret) {
+		dev_err(qmp->dev, "%s: No clock-output-names\n", np->name);
+		return ret;
+	}
+
 	fixed = devm_kzalloc(qmp->dev, sizeof(*fixed), GFP_KERNEL);
 	if (!fixed)
 		return -ENOMEM;
 
-	init.name = name;
 	init.ops = &clk_fixed_rate_ops;
 
 	/* controllers using QMP phys use 125MHz pipe clock interface */
@@ -1049,6 +1166,9 @@ static const struct of_device_id qcom_qmp_phy_of_match_table[] = {
 	}, {
 		.compatible = "qcom,msm8996-qmp-usb3-phy",
 		.data = &msm8996_usb3phy_cfg,
+	}, {
+		.compatible = "qcom,ipq8074-qmp-pcie-phy",
+		.data = &ipq8074_pciephy_cfg,
 	},
 	{ },
 };
@@ -1122,7 +1242,7 @@ static int qcom_qmp_phy_probe(struct platform_device *pdev)
 		 * Register the pipe clock provided by phy.
 		 * See function description to see details of this pipe clock.
 		 */
-		ret = phy_pipe_clk_register(qmp, id);
+		ret = phy_pipe_clk_register(qmp, child);
 		if (ret) {
 			dev_err(qmp->dev,
 				"failed to register pipe clock source\n");
diff --git a/drivers/phy/qualcomm/phy-qcom-usb-hs.c b/drivers/phy/qualcomm/phy-qcom-usb-hs.c
index 4b20abc3ae2f..2d0c70b5589f 100644
--- a/drivers/phy/qualcomm/phy-qcom-usb-hs.c
+++ b/drivers/phy/qualcomm/phy-qcom-usb-hs.c
@@ -155,12 +155,12 @@ static int qcom_usb_hs_phy_power_on(struct phy *phy)
 	}
 
 	if (uphy->vbus_edev) {
-		state = extcon_get_cable_state_(uphy->vbus_edev, EXTCON_USB);
+		state = extcon_get_state(uphy->vbus_edev, EXTCON_USB);
 		/* setup initial state */
 		qcom_usb_hs_phy_vbus_notifier(&uphy->vbus_notify, state,
 					      uphy->vbus_edev);
-		ret = extcon_register_notifier(uphy->vbus_edev, EXTCON_USB,
-				&uphy->vbus_notify);
+		ret = devm_extcon_register_notifier(&ulpi->dev, uphy->vbus_edev,
+				EXTCON_USB, &uphy->vbus_notify);
 		if (ret)
 			goto err_ulpi;
 	}
@@ -179,16 +179,8 @@ err_sleep:
 
 static int qcom_usb_hs_phy_power_off(struct phy *phy)
 {
-	int ret;
 	struct qcom_usb_hs_phy *uphy = phy_get_drvdata(phy);
 
-	if (uphy->vbus_edev) {
-		ret = extcon_unregister_notifier(uphy->vbus_edev, EXTCON_USB,
-						 &uphy->vbus_notify);
-		if (ret)
-			return ret;
-	}
-
 	regulator_disable(uphy->v3p3);
 	regulator_disable(uphy->v1p8);
 	clk_disable_unprepare(uphy->sleep_clk);
diff --git a/drivers/phy/ralink/Kconfig b/drivers/phy/ralink/Kconfig
new file mode 100644
index 000000000000..b17635b407bc
--- /dev/null
+++ b/drivers/phy/ralink/Kconfig
@@ -0,0 +1,11 @@
+#
+# PHY drivers for Ralink platforms.
+#
+config PHY_RALINK_USB
+	tristate "Ralink USB PHY driver"
+	depends on RALINK || COMPILE_TEST
+	select GENERIC_PHY
+	select MFD_SYSCON
+	help
+	  This option enables support for the Ralink USB PHY found inside
+	  RT3352, MT7620, MT7628 and MT7688.
diff --git a/drivers/phy/ralink/Makefile b/drivers/phy/ralink/Makefile
new file mode 100644
index 000000000000..5c9e326e8757
--- /dev/null
+++ b/drivers/phy/ralink/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_PHY_RALINK_USB)	+= phy-ralink-usb.o
diff --git a/drivers/phy/ralink/phy-ralink-usb.c b/drivers/phy/ralink/phy-ralink-usb.c
new file mode 100644
index 000000000000..4fea31f8ac1c
--- /dev/null
+++ b/drivers/phy/ralink/phy-ralink-usb.c
@@ -0,0 +1,249 @@
+/*
+ * Copyright (C) 2017 John Crispin <john@phrozen.org>
+ *
+ * Based on code from
+ * Allwinner Technology Co., Ltd. <www.allwinnertech.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of_platform.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+
+#define RT_SYSC_REG_SYSCFG1		0x014
+#define RT_SYSC_REG_CLKCFG1		0x030
+#define RT_SYSC_REG_USB_PHY_CFG		0x05c
+
+#define OFS_U2_PHY_AC0			0x800
+#define OFS_U2_PHY_AC1			0x804
+#define OFS_U2_PHY_AC2			0x808
+#define OFS_U2_PHY_ACR0			0x810
+#define OFS_U2_PHY_ACR1			0x814
+#define OFS_U2_PHY_ACR2			0x818
+#define OFS_U2_PHY_ACR3			0x81C
+#define OFS_U2_PHY_ACR4			0x820
+#define OFS_U2_PHY_AMON0		0x824
+#define OFS_U2_PHY_DCR0			0x860
+#define OFS_U2_PHY_DCR1			0x864
+#define OFS_U2_PHY_DTM0			0x868
+#define OFS_U2_PHY_DTM1			0x86C
+
+#define RT_RSTCTRL_UDEV			BIT(25)
+#define RT_RSTCTRL_UHST			BIT(22)
+#define RT_SYSCFG1_USB0_HOST_MODE	BIT(10)
+
+#define MT7620_CLKCFG1_UPHY0_CLK_EN	BIT(25)
+#define MT7620_CLKCFG1_UPHY1_CLK_EN	BIT(22)
+#define RT_CLKCFG1_UPHY1_CLK_EN		BIT(20)
+#define RT_CLKCFG1_UPHY0_CLK_EN		BIT(18)
+
+#define USB_PHY_UTMI_8B60M		BIT(1)
+#define UDEV_WAKEUP			BIT(0)
+
+struct ralink_usb_phy {
+	struct reset_control	*rstdev;
+	struct reset_control	*rsthost;
+	u32			clk;
+	struct phy		*phy;
+	void __iomem		*base;
+	struct regmap		*sysctl;
+};
+
+static void u2_phy_w32(struct ralink_usb_phy *phy, u32 val, u32 reg)
+{
+	writel(val, phy->base + reg);
+}
+
+static u32 u2_phy_r32(struct ralink_usb_phy *phy, u32 reg)
+{
+	return readl(phy->base + reg);
+}
+
+static void ralink_usb_phy_init(struct ralink_usb_phy *phy)
+{
+	u2_phy_r32(phy, OFS_U2_PHY_AC2);
+	u2_phy_r32(phy, OFS_U2_PHY_ACR0);
+	u2_phy_r32(phy, OFS_U2_PHY_DCR0);
+
+	u2_phy_w32(phy, 0x00ffff02, OFS_U2_PHY_DCR0);
+	u2_phy_r32(phy, OFS_U2_PHY_DCR0);
+	u2_phy_w32(phy, 0x00555502, OFS_U2_PHY_DCR0);
+	u2_phy_r32(phy, OFS_U2_PHY_DCR0);
+	u2_phy_w32(phy, 0x00aaaa02, OFS_U2_PHY_DCR0);
+	u2_phy_r32(phy, OFS_U2_PHY_DCR0);
+	u2_phy_w32(phy, 0x00000402, OFS_U2_PHY_DCR0);
+	u2_phy_r32(phy, OFS_U2_PHY_DCR0);
+	u2_phy_w32(phy, 0x0048086a, OFS_U2_PHY_AC0);
+	u2_phy_w32(phy, 0x4400001c, OFS_U2_PHY_AC1);
+	u2_phy_w32(phy, 0xc0200000, OFS_U2_PHY_ACR3);
+	u2_phy_w32(phy, 0x02000000, OFS_U2_PHY_DTM0);
+}
+
+static int ralink_usb_phy_power_on(struct phy *_phy)
+{
+	struct ralink_usb_phy *phy = phy_get_drvdata(_phy);
+	u32 t;
+
+	/* enable the phy */
+	regmap_update_bits(phy->sysctl, RT_SYSC_REG_CLKCFG1,
+			   phy->clk, phy->clk);
+
+	/* setup host mode */
+	regmap_update_bits(phy->sysctl, RT_SYSC_REG_SYSCFG1,
+			   RT_SYSCFG1_USB0_HOST_MODE,
+			   RT_SYSCFG1_USB0_HOST_MODE);
+
+	/* deassert the reset lines */
+	reset_control_deassert(phy->rsthost);
+	reset_control_deassert(phy->rstdev);
+
+	/*
+	 * The SDK kernel had a delay of 100ms. however on device
+	 * testing showed that 10ms is enough
+	 */
+	mdelay(10);
+
+	if (phy->base)
+		ralink_usb_phy_init(phy);
+
+	/* print some status info */
+	regmap_read(phy->sysctl, RT_SYSC_REG_USB_PHY_CFG, &t);
+	dev_info(&phy->phy->dev, "remote usb device wakeup %s\n",
+		(t & UDEV_WAKEUP) ? ("enabled") : ("disabled"));
+	if (t & USB_PHY_UTMI_8B60M)
+		dev_info(&phy->phy->dev, "UTMI 8bit 60MHz\n");
+	else
+		dev_info(&phy->phy->dev, "UTMI 16bit 30MHz\n");
+
+	return 0;
+}
+
+static int ralink_usb_phy_power_off(struct phy *_phy)
+{
+	struct ralink_usb_phy *phy = phy_get_drvdata(_phy);
+
+	/* disable the phy */
+	regmap_update_bits(phy->sysctl, RT_SYSC_REG_CLKCFG1,
+			   phy->clk, 0);
+
+	/* assert the reset lines */
+	reset_control_assert(phy->rstdev);
+	reset_control_assert(phy->rsthost);
+
+	return 0;
+}
+
+static struct phy_ops ralink_usb_phy_ops = {
+	.power_on	= ralink_usb_phy_power_on,
+	.power_off	= ralink_usb_phy_power_off,
+	.owner		= THIS_MODULE,
+};
+
+static const struct of_device_id ralink_usb_phy_of_match[] = {
+	{
+		.compatible = "ralink,rt3352-usbphy",
+		.data = (void *)(uintptr_t)(RT_CLKCFG1_UPHY1_CLK_EN |
+					    RT_CLKCFG1_UPHY0_CLK_EN)
+	},
+	{
+		.compatible = "mediatek,mt7620-usbphy",
+		.data = (void *)(uintptr_t)(MT7620_CLKCFG1_UPHY1_CLK_EN |
+					    MT7620_CLKCFG1_UPHY0_CLK_EN)
+	},
+	{
+		.compatible = "mediatek,mt7628-usbphy",
+		.data = (void *)(uintptr_t)(MT7620_CLKCFG1_UPHY1_CLK_EN |
+					    MT7620_CLKCFG1_UPHY0_CLK_EN) },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, ralink_usb_phy_of_match);
+
+static int ralink_usb_phy_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct resource *res;
+	struct phy_provider *phy_provider;
+	const struct of_device_id *match;
+	struct ralink_usb_phy *phy;
+
+	match = of_match_device(ralink_usb_phy_of_match, &pdev->dev);
+	if (!match)
+		return -ENODEV;
+
+	phy = devm_kzalloc(dev, sizeof(*phy), GFP_KERNEL);
+	if (!phy)
+		return -ENOMEM;
+
+	phy->clk = (uintptr_t)match->data;
+	phy->base = NULL;
+
+	phy->sysctl = syscon_regmap_lookup_by_phandle(dev->of_node, "ralink,sysctl");
+	if (IS_ERR(phy->sysctl)) {
+		dev_err(dev, "failed to get sysctl registers\n");
+		return PTR_ERR(phy->sysctl);
+	}
+
+	/* The MT7628 and MT7688 require extra setup of PHY registers. */
+	if (of_device_is_compatible(dev->of_node, "mediatek,mt7628-usbphy")) {
+		res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+		phy->base = devm_ioremap_resource(&pdev->dev, res);
+		if (IS_ERR(phy->base)) {
+			dev_err(dev, "failed to remap register memory\n");
+			return PTR_ERR(phy->base);
+		}
+	}
+
+	phy->rsthost = devm_reset_control_get(&pdev->dev, "host");
+	if (IS_ERR(phy->rsthost)) {
+		dev_err(dev, "host reset is missing\n");
+		return PTR_ERR(phy->rsthost);
+	}
+
+	phy->rstdev = devm_reset_control_get(&pdev->dev, "device");
+	if (IS_ERR(phy->rstdev)) {
+		dev_err(dev, "device reset is missing\n");
+		return PTR_ERR(phy->rstdev);
+	}
+
+	phy->phy = devm_phy_create(dev, NULL, &ralink_usb_phy_ops);
+	if (IS_ERR(phy->phy)) {
+		dev_err(dev, "failed to create PHY\n");
+		return PTR_ERR(phy->phy);
+	}
+	phy_set_drvdata(phy->phy, phy);
+
+	phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
+
+	return PTR_ERR_OR_ZERO(phy_provider);
+}
+
+static struct platform_driver ralink_usb_phy_driver = {
+	.probe	= ralink_usb_phy_probe,
+	.driver = {
+		.of_match_table	= ralink_usb_phy_of_match,
+		.name  = "ralink-usb-phy",
+	}
+};
+module_platform_driver(ralink_usb_phy_driver);
+
+MODULE_DESCRIPTION("Ralink USB phy driver");
+MODULE_AUTHOR("John Crispin <john@phrozen.org>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/phy/rockchip/phy-rockchip-inno-usb2.c b/drivers/phy/rockchip/phy-rockchip-inno-usb2.c
index 626883d9d176..ee7ce5ee53f9 100644
--- a/drivers/phy/rockchip/phy-rockchip-inno-usb2.c
+++ b/drivers/phy/rockchip/phy-rockchip-inno-usb2.c
@@ -172,6 +172,8 @@ struct rockchip_usb2phy_cfg {
  * @vbus_attached: otg device vbus status.
  * @bvalid_irq: IRQ number assigned for vbus valid rise detection.
  * @ls_irq: IRQ number assigned for linestate detection.
+ * @otg_mux_irq: IRQ number which multiplex otg-id/otg-bvalid/linestate
+ *		 irqs to one irq in otg-port.
  * @mutex: for register updating in sm_work.
  * @chg_work: charge detect work.
  * @otg_sm_work: OTG state machine work.
@@ -189,6 +191,7 @@ struct rockchip_usb2phy_port {
 	bool		vbus_attached;
 	int		bvalid_irq;
 	int		ls_irq;
+	int		otg_mux_irq;
 	struct mutex	mutex;
 	struct		delayed_work chg_work;
 	struct		delayed_work otg_sm_work;
@@ -202,6 +205,7 @@ struct rockchip_usb2phy_port {
 /**
  * struct rockchip_usb2phy: usb2.0 phy driver data.
  * @grf: General Register Files regmap.
+ * @usbgrf: USB General Register Files regmap.
  * @clk: clock struct of phy input clk.
  * @clk480m: clock struct of phy output clk.
  * @clk_hw: clock struct of phy output clk management.
@@ -216,6 +220,7 @@ struct rockchip_usb2phy_port {
 struct rockchip_usb2phy {
 	struct device	*dev;
 	struct regmap	*grf;
+	struct regmap	*usbgrf;
 	struct clk	*clk;
 	struct clk	*clk480m;
 	struct clk_hw	clk480m_hw;
@@ -227,7 +232,12 @@ struct rockchip_usb2phy {
 	struct rockchip_usb2phy_port	ports[USB2PHY_NUM_PORTS];
 };
 
-static inline int property_enable(struct rockchip_usb2phy *rphy,
+static inline struct regmap *get_reg_base(struct rockchip_usb2phy *rphy)
+{
+	return rphy->usbgrf == NULL ? rphy->grf : rphy->usbgrf;
+}
+
+static inline int property_enable(struct regmap *base,
 				  const struct usb2phy_reg *reg, bool en)
 {
 	unsigned int val, mask, tmp;
@@ -236,17 +246,17 @@ static inline int property_enable(struct rockchip_usb2phy *rphy,
 	mask = GENMASK(reg->bitend, reg->bitstart);
 	val = (tmp << reg->bitstart) | (mask << BIT_WRITEABLE_SHIFT);
 
-	return regmap_write(rphy->grf, reg->offset, val);
+	return regmap_write(base, reg->offset, val);
 }
 
-static inline bool property_enabled(struct rockchip_usb2phy *rphy,
+static inline bool property_enabled(struct regmap *base,
 				    const struct usb2phy_reg *reg)
 {
 	int ret;
 	unsigned int tmp, orig;
 	unsigned int mask = GENMASK(reg->bitend, reg->bitstart);
 
-	ret = regmap_read(rphy->grf, reg->offset, &orig);
+	ret = regmap_read(base, reg->offset, &orig);
 	if (ret)
 		return false;
 
@@ -258,11 +268,12 @@ static int rockchip_usb2phy_clk480m_prepare(struct clk_hw *hw)
 {
 	struct rockchip_usb2phy *rphy =
 		container_of(hw, struct rockchip_usb2phy, clk480m_hw);
+	struct regmap *base = get_reg_base(rphy);
 	int ret;
 
 	/* turn on 480m clk output if it is off */
-	if (!property_enabled(rphy, &rphy->phy_cfg->clkout_ctl)) {
-		ret = property_enable(rphy, &rphy->phy_cfg->clkout_ctl, true);
+	if (!property_enabled(base, &rphy->phy_cfg->clkout_ctl)) {
+		ret = property_enable(base, &rphy->phy_cfg->clkout_ctl, true);
 		if (ret)
 			return ret;
 
@@ -277,17 +288,19 @@ static void rockchip_usb2phy_clk480m_unprepare(struct clk_hw *hw)
 {
 	struct rockchip_usb2phy *rphy =
 		container_of(hw, struct rockchip_usb2phy, clk480m_hw);
+	struct regmap *base = get_reg_base(rphy);
 
 	/* turn off 480m clk output */
-	property_enable(rphy, &rphy->phy_cfg->clkout_ctl, false);
+	property_enable(base, &rphy->phy_cfg->clkout_ctl, false);
 }
 
 static int rockchip_usb2phy_clk480m_prepared(struct clk_hw *hw)
 {
 	struct rockchip_usb2phy *rphy =
 		container_of(hw, struct rockchip_usb2phy, clk480m_hw);
+	struct regmap *base = get_reg_base(rphy);
 
-	return property_enabled(rphy, &rphy->phy_cfg->clkout_ctl);
+	return property_enabled(base, &rphy->phy_cfg->clkout_ctl);
 }
 
 static unsigned long
@@ -409,13 +422,13 @@ static int rockchip_usb2phy_init(struct phy *phy)
 		if (rport->mode != USB_DR_MODE_HOST &&
 		    rport->mode != USB_DR_MODE_UNKNOWN) {
 			/* clear bvalid status and enable bvalid detect irq */
-			ret = property_enable(rphy,
+			ret = property_enable(rphy->grf,
 					      &rport->port_cfg->bvalid_det_clr,
 					      true);
 			if (ret)
 				goto out;
 
-			ret = property_enable(rphy,
+			ret = property_enable(rphy->grf,
 					      &rport->port_cfg->bvalid_det_en,
 					      true);
 			if (ret)
@@ -429,11 +442,13 @@ static int rockchip_usb2phy_init(struct phy *phy)
 		}
 	} else if (rport->port_id == USB2PHY_PORT_HOST) {
 		/* clear linestate and enable linestate detect irq */
-		ret = property_enable(rphy, &rport->port_cfg->ls_det_clr, true);
+		ret = property_enable(rphy->grf,
+				      &rport->port_cfg->ls_det_clr, true);
 		if (ret)
 			goto out;
 
-		ret = property_enable(rphy, &rport->port_cfg->ls_det_en, true);
+		ret = property_enable(rphy->grf,
+				      &rport->port_cfg->ls_det_en, true);
 		if (ret)
 			goto out;
 
@@ -449,6 +464,7 @@ static int rockchip_usb2phy_power_on(struct phy *phy)
 {
 	struct rockchip_usb2phy_port *rport = phy_get_drvdata(phy);
 	struct rockchip_usb2phy *rphy = dev_get_drvdata(phy->dev.parent);
+	struct regmap *base = get_reg_base(rphy);
 	int ret;
 
 	dev_dbg(&rport->phy->dev, "port power on\n");
@@ -460,7 +476,7 @@ static int rockchip_usb2phy_power_on(struct phy *phy)
 	if (ret)
 		return ret;
 
-	ret = property_enable(rphy, &rport->port_cfg->phy_sus, false);
+	ret = property_enable(base, &rport->port_cfg->phy_sus, false);
 	if (ret)
 		return ret;
 
@@ -475,6 +491,7 @@ static int rockchip_usb2phy_power_off(struct phy *phy)
 {
 	struct rockchip_usb2phy_port *rport = phy_get_drvdata(phy);
 	struct rockchip_usb2phy *rphy = dev_get_drvdata(phy->dev.parent);
+	struct regmap *base = get_reg_base(rphy);
 	int ret;
 
 	dev_dbg(&rport->phy->dev, "port power off\n");
@@ -482,7 +499,7 @@ static int rockchip_usb2phy_power_off(struct phy *phy)
 	if (rport->suspended)
 		return 0;
 
-	ret = property_enable(rphy, &rport->port_cfg->phy_sus, true);
+	ret = property_enable(base, &rport->port_cfg->phy_sus, true);
 	if (ret)
 		return ret;
 
@@ -526,11 +543,11 @@ static void rockchip_usb2phy_otg_sm_work(struct work_struct *work)
 	bool vbus_attach, sch_work, notify_charger;
 
 	if (rport->utmi_avalid)
-		vbus_attach =
-			property_enabled(rphy, &rport->port_cfg->utmi_avalid);
+		vbus_attach = property_enabled(rphy->grf,
+					       &rport->port_cfg->utmi_avalid);
 	else
-		vbus_attach =
-			property_enabled(rphy, &rport->port_cfg->utmi_bvalid);
+		vbus_attach = property_enabled(rphy->grf,
+					       &rport->port_cfg->utmi_bvalid);
 
 	sch_work = false;
 	notify_charger = false;
@@ -545,7 +562,7 @@ static void rockchip_usb2phy_otg_sm_work(struct work_struct *work)
 			rockchip_usb2phy_power_off(rport->phy);
 		/* fall through */
 	case OTG_STATE_B_IDLE:
-		if (extcon_get_cable_state_(rphy->edev, EXTCON_USB_HOST) > 0) {
+		if (extcon_get_state(rphy->edev, EXTCON_USB_HOST) > 0) {
 			dev_dbg(&rport->phy->dev, "usb otg host connect\n");
 			rport->state = OTG_STATE_A_HOST;
 			rockchip_usb2phy_power_on(rport->phy);
@@ -598,7 +615,7 @@ static void rockchip_usb2phy_otg_sm_work(struct work_struct *work)
 			rport->vbus_attached = vbus_attach;
 
 			if (notify_charger && rphy->edev) {
-				extcon_set_cable_state_(rphy->edev,
+				extcon_set_state_sync(rphy->edev,
 							cable, vbus_attach);
 				if (cable == EXTCON_CHG_USB_SDP)
 					extcon_set_state_sync(rphy->edev,
@@ -619,7 +636,7 @@ static void rockchip_usb2phy_otg_sm_work(struct work_struct *work)
 		sch_work = true;
 		break;
 	case OTG_STATE_A_HOST:
-		if (extcon_get_cable_state_(rphy->edev, EXTCON_USB_HOST) == 0) {
+		if (extcon_get_state(rphy->edev, EXTCON_USB_HOST) == 0) {
 			dev_dbg(&rport->phy->dev, "usb otg host disconnect\n");
 			rport->state = OTG_STATE_B_IDLE;
 			rockchip_usb2phy_power_off(rport->phy);
@@ -650,22 +667,28 @@ static const char *chg_to_string(enum power_supply_type chg_type)
 static void rockchip_chg_enable_dcd(struct rockchip_usb2phy *rphy,
 				    bool en)
 {
-	property_enable(rphy, &rphy->phy_cfg->chg_det.rdm_pdwn_en, en);
-	property_enable(rphy, &rphy->phy_cfg->chg_det.idp_src_en, en);
+	struct regmap *base = get_reg_base(rphy);
+
+	property_enable(base, &rphy->phy_cfg->chg_det.rdm_pdwn_en, en);
+	property_enable(base, &rphy->phy_cfg->chg_det.idp_src_en, en);
 }
 
 static void rockchip_chg_enable_primary_det(struct rockchip_usb2phy *rphy,
 					    bool en)
 {
-	property_enable(rphy, &rphy->phy_cfg->chg_det.vdp_src_en, en);
-	property_enable(rphy, &rphy->phy_cfg->chg_det.idm_sink_en, en);
+	struct regmap *base = get_reg_base(rphy);
+
+	property_enable(base, &rphy->phy_cfg->chg_det.vdp_src_en, en);
+	property_enable(base, &rphy->phy_cfg->chg_det.idm_sink_en, en);
 }
 
 static void rockchip_chg_enable_secondary_det(struct rockchip_usb2phy *rphy,
 					      bool en)
 {
-	property_enable(rphy, &rphy->phy_cfg->chg_det.vdm_src_en, en);
-	property_enable(rphy, &rphy->phy_cfg->chg_det.idp_sink_en, en);
+	struct regmap *base = get_reg_base(rphy);
+
+	property_enable(base, &rphy->phy_cfg->chg_det.vdm_src_en, en);
+	property_enable(base, &rphy->phy_cfg->chg_det.idp_sink_en, en);
 }
 
 #define CHG_DCD_POLL_TIME	(100 * HZ / 1000)
@@ -677,6 +700,7 @@ static void rockchip_chg_detect_work(struct work_struct *work)
 	struct rockchip_usb2phy_port *rport =
 		container_of(work, struct rockchip_usb2phy_port, chg_work.work);
 	struct rockchip_usb2phy *rphy = dev_get_drvdata(rport->phy->dev.parent);
+	struct regmap *base = get_reg_base(rphy);
 	bool is_dcd, tmout, vout;
 	unsigned long delay;
 
@@ -687,7 +711,7 @@ static void rockchip_chg_detect_work(struct work_struct *work)
 		if (!rport->suspended)
 			rockchip_usb2phy_power_off(rport->phy);
 		/* put the controller in non-driving mode */
-		property_enable(rphy, &rphy->phy_cfg->chg_det.opmode, false);
+		property_enable(base, &rphy->phy_cfg->chg_det.opmode, false);
 		/* Start DCD processing stage 1 */
 		rockchip_chg_enable_dcd(rphy, true);
 		rphy->chg_state = USB_CHG_STATE_WAIT_FOR_DCD;
@@ -696,7 +720,8 @@ static void rockchip_chg_detect_work(struct work_struct *work)
 		break;
 	case USB_CHG_STATE_WAIT_FOR_DCD:
 		/* get data contact detection status */
-		is_dcd = property_enabled(rphy, &rphy->phy_cfg->chg_det.dp_det);
+		is_dcd = property_enabled(rphy->grf,
+					  &rphy->phy_cfg->chg_det.dp_det);
 		tmout = ++rphy->dcd_retries == CHG_DCD_MAX_RETRIES;
 		/* stage 2 */
 		if (is_dcd || tmout) {
@@ -713,7 +738,8 @@ static void rockchip_chg_detect_work(struct work_struct *work)
 		}
 		break;
 	case USB_CHG_STATE_DCD_DONE:
-		vout = property_enabled(rphy, &rphy->phy_cfg->chg_det.cp_det);
+		vout = property_enabled(rphy->grf,
+					&rphy->phy_cfg->chg_det.cp_det);
 		rockchip_chg_enable_primary_det(rphy, false);
 		if (vout) {
 			/* Voltage Source on DM, Probe on DP  */
@@ -734,7 +760,8 @@ static void rockchip_chg_detect_work(struct work_struct *work)
 		}
 		break;
 	case USB_CHG_STATE_PRIMARY_DONE:
-		vout = property_enabled(rphy, &rphy->phy_cfg->chg_det.dcp_det);
+		vout = property_enabled(rphy->grf,
+					&rphy->phy_cfg->chg_det.dcp_det);
 		/* Turn off voltage source */
 		rockchip_chg_enable_secondary_det(rphy, false);
 		if (vout)
@@ -748,7 +775,7 @@ static void rockchip_chg_detect_work(struct work_struct *work)
 		/* fall through */
 	case USB_CHG_STATE_DETECTED:
 		/* put the controller in normal mode */
-		property_enable(rphy, &rphy->phy_cfg->chg_det.opmode, true);
+		property_enable(base, &rphy->phy_cfg->chg_det.opmode, true);
 		rockchip_usb2phy_otg_sm_work(&rport->otg_sm_work.work);
 		dev_info(&rport->phy->dev, "charger = %s\n",
 			 chg_to_string(rphy->chg_type));
@@ -790,8 +817,7 @@ static void rockchip_usb2phy_sm_work(struct work_struct *work)
 	if (ret < 0)
 		goto next_schedule;
 
-	ret = regmap_read(rphy->grf, rport->port_cfg->utmi_hstdet.offset,
-			  &uhd);
+	ret = regmap_read(rphy->grf, rport->port_cfg->utmi_hstdet.offset, &uhd);
 	if (ret < 0)
 		goto next_schedule;
 
@@ -845,8 +871,8 @@ static void rockchip_usb2phy_sm_work(struct work_struct *work)
 		 * activate the linestate detection to get the next device
 		 * plug-in irq.
 		 */
-		property_enable(rphy, &rport->port_cfg->ls_det_clr, true);
-		property_enable(rphy, &rport->port_cfg->ls_det_en, true);
+		property_enable(rphy->grf, &rport->port_cfg->ls_det_clr, true);
+		property_enable(rphy->grf, &rport->port_cfg->ls_det_en, true);
 
 		/*
 		 * we don't need to rearm the delayed work when the phy port
@@ -869,14 +895,14 @@ static irqreturn_t rockchip_usb2phy_linestate_irq(int irq, void *data)
 	struct rockchip_usb2phy_port *rport = data;
 	struct rockchip_usb2phy *rphy = dev_get_drvdata(rport->phy->dev.parent);
 
-	if (!property_enabled(rphy, &rport->port_cfg->ls_det_st))
+	if (!property_enabled(rphy->grf, &rport->port_cfg->ls_det_st))
 		return IRQ_NONE;
 
 	mutex_lock(&rport->mutex);
 
 	/* disable linestate detect irq and clear its status */
-	property_enable(rphy, &rport->port_cfg->ls_det_en, false);
-	property_enable(rphy, &rport->port_cfg->ls_det_clr, true);
+	property_enable(rphy->grf, &rport->port_cfg->ls_det_en, false);
+	property_enable(rphy->grf, &rport->port_cfg->ls_det_clr, true);
 
 	mutex_unlock(&rport->mutex);
 
@@ -896,13 +922,13 @@ static irqreturn_t rockchip_usb2phy_bvalid_irq(int irq, void *data)
 	struct rockchip_usb2phy_port *rport = data;
 	struct rockchip_usb2phy *rphy = dev_get_drvdata(rport->phy->dev.parent);
 
-	if (!property_enabled(rphy, &rport->port_cfg->bvalid_det_st))
+	if (!property_enabled(rphy->grf, &rport->port_cfg->bvalid_det_st))
 		return IRQ_NONE;
 
 	mutex_lock(&rport->mutex);
 
 	/* clear bvalid detect irq pending status */
-	property_enable(rphy, &rport->port_cfg->bvalid_det_clr, true);
+	property_enable(rphy->grf, &rport->port_cfg->bvalid_det_clr, true);
 
 	mutex_unlock(&rport->mutex);
 
@@ -911,6 +937,17 @@ static irqreturn_t rockchip_usb2phy_bvalid_irq(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
+static irqreturn_t rockchip_usb2phy_otg_mux_irq(int irq, void *data)
+{
+	struct rockchip_usb2phy_port *rport = data;
+	struct rockchip_usb2phy *rphy = dev_get_drvdata(rport->phy->dev.parent);
+
+	if (property_enabled(rphy->grf, &rport->port_cfg->bvalid_det_st))
+		return rockchip_usb2phy_bvalid_irq(irq, data);
+	else
+		return IRQ_NONE;
+}
+
 static int rockchip_usb2phy_host_port_init(struct rockchip_usb2phy *rphy,
 					   struct rockchip_usb2phy_port *rport,
 					   struct device_node *child_np)
@@ -987,27 +1024,50 @@ static int rockchip_usb2phy_otg_port_init(struct rockchip_usb2phy *rphy,
 	rport->utmi_avalid =
 		of_property_read_bool(child_np, "rockchip,utmi-avalid");
 
-	rport->bvalid_irq = of_irq_get_byname(child_np, "otg-bvalid");
-	if (rport->bvalid_irq < 0) {
-		dev_err(rphy->dev, "no vbus valid irq provided\n");
-		ret = rport->bvalid_irq;
-		goto out;
-	}
+	/*
+	 * Some SoCs use one interrupt with otg-id/otg-bvalid/linestate
+	 * interrupts muxed together, so probe the otg-mux interrupt first,
+	 * if not found, then look for the regular interrupts one by one.
+	 */
+	rport->otg_mux_irq = of_irq_get_byname(child_np, "otg-mux");
+	if (rport->otg_mux_irq > 0) {
+		ret = devm_request_threaded_irq(rphy->dev, rport->otg_mux_irq,
+						NULL,
+						rockchip_usb2phy_otg_mux_irq,
+						IRQF_ONESHOT,
+						"rockchip_usb2phy_otg",
+						rport);
+		if (ret) {
+			dev_err(rphy->dev,
+				"failed to request otg-mux irq handle\n");
+			goto out;
+		}
+	} else {
+		rport->bvalid_irq = of_irq_get_byname(child_np, "otg-bvalid");
+		if (rport->bvalid_irq < 0) {
+			dev_err(rphy->dev, "no vbus valid irq provided\n");
+			ret = rport->bvalid_irq;
+			goto out;
+		}
 
-	ret = devm_request_threaded_irq(rphy->dev, rport->bvalid_irq, NULL,
-					rockchip_usb2phy_bvalid_irq,
-					IRQF_ONESHOT,
-					"rockchip_usb2phy_bvalid", rport);
-	if (ret) {
-		dev_err(rphy->dev, "failed to request otg-bvalid irq handle\n");
-		goto out;
+		ret = devm_request_threaded_irq(rphy->dev, rport->bvalid_irq,
+						NULL,
+						rockchip_usb2phy_bvalid_irq,
+						IRQF_ONESHOT,
+						"rockchip_usb2phy_bvalid",
+						rport);
+		if (ret) {
+			dev_err(rphy->dev,
+				"failed to request otg-bvalid irq handle\n");
+			goto out;
+		}
 	}
 
 	if (!IS_ERR(rphy->edev)) {
 		rport->event_nb.notifier_call = rockchip_otg_event;
 
-		ret = extcon_register_notifier(rphy->edev, EXTCON_USB_HOST,
-					       &rport->event_nb);
+		ret = devm_extcon_register_notifier(rphy->dev, rphy->edev,
+					EXTCON_USB_HOST, &rport->event_nb);
 		if (ret)
 			dev_err(rphy->dev, "register USB HOST notifier failed\n");
 	}
@@ -1045,6 +1105,16 @@ static int rockchip_usb2phy_probe(struct platform_device *pdev)
 	if (IS_ERR(rphy->grf))
 		return PTR_ERR(rphy->grf);
 
+	if (of_device_is_compatible(np, "rockchip,rv1108-usb2phy")) {
+		rphy->usbgrf =
+			syscon_regmap_lookup_by_phandle(dev->of_node,
+							"rockchip,usbgrf");
+		if (IS_ERR(rphy->usbgrf))
+			return PTR_ERR(rphy->usbgrf);
+	} else {
+		rphy->usbgrf = NULL;
+	}
+
 	if (of_property_read_u32(np, "reg", &reg)) {
 		dev_err(dev, "the reg property is not assigned in %s node\n",
 			np->name);
@@ -1327,11 +1397,54 @@ static const struct rockchip_usb2phy_cfg rk3399_phy_cfgs[] = {
 	{ /* sentinel */ }
 };
 
+static const struct rockchip_usb2phy_cfg rv1108_phy_cfgs[] = {
+	{
+		.reg = 0x100,
+		.num_ports	= 2,
+		.clkout_ctl	= { 0x108, 4, 4, 1, 0 },
+		.port_cfgs	= {
+			[USB2PHY_PORT_OTG] = {
+				.phy_sus	= { 0x0100, 15, 0, 0, 0x1d1 },
+				.bvalid_det_en	= { 0x0680, 3, 3, 0, 1 },
+				.bvalid_det_st	= { 0x0690, 3, 3, 0, 1 },
+				.bvalid_det_clr = { 0x06a0, 3, 3, 0, 1 },
+				.ls_det_en	= { 0x0680, 2, 2, 0, 1 },
+				.ls_det_st	= { 0x0690, 2, 2, 0, 1 },
+				.ls_det_clr	= { 0x06a0, 2, 2, 0, 1 },
+				.utmi_bvalid	= { 0x0804, 10, 10, 0, 1 },
+				.utmi_ls	= { 0x0804, 13, 12, 0, 1 },
+			},
+			[USB2PHY_PORT_HOST] = {
+				.phy_sus	= { 0x0104, 15, 0, 0, 0x1d1 },
+				.ls_det_en	= { 0x0680, 4, 4, 0, 1 },
+				.ls_det_st	= { 0x0690, 4, 4, 0, 1 },
+				.ls_det_clr	= { 0x06a0, 4, 4, 0, 1 },
+				.utmi_ls	= { 0x0804, 9, 8, 0, 1 },
+				.utmi_hstdet	= { 0x0804, 7, 7, 0, 1 }
+			}
+		},
+		.chg_det = {
+			.opmode		= { 0x0100, 3, 0, 5, 1 },
+			.cp_det		= { 0x0804, 1, 1, 0, 1 },
+			.dcp_det	= { 0x0804, 0, 0, 0, 1 },
+			.dp_det		= { 0x0804, 2, 2, 0, 1 },
+			.idm_sink_en	= { 0x0108, 8, 8, 0, 1 },
+			.idp_sink_en	= { 0x0108, 7, 7, 0, 1 },
+			.idp_src_en	= { 0x0108, 9, 9, 0, 1 },
+			.rdm_pdwn_en	= { 0x0108, 10, 10, 0, 1 },
+			.vdm_src_en	= { 0x0108, 12, 12, 0, 1 },
+			.vdp_src_en	= { 0x0108, 11, 11, 0, 1 },
+		},
+	},
+	{ /* sentinel */ }
+};
+
 static const struct of_device_id rockchip_usb2phy_dt_match[] = {
 	{ .compatible = "rockchip,rk3228-usb2phy", .data = &rk3228_phy_cfgs },
 	{ .compatible = "rockchip,rk3328-usb2phy", .data = &rk3328_phy_cfgs },
 	{ .compatible = "rockchip,rk3366-usb2phy", .data = &rk3366_phy_cfgs },
 	{ .compatible = "rockchip,rk3399-usb2phy", .data = &rk3399_phy_cfgs },
+	{ .compatible = "rockchip,rv1108-usb2phy", .data = &rv1108_phy_cfgs },
 	{}
 };
 MODULE_DEVICE_TABLE(of, rockchip_usb2phy_dt_match);
diff --git a/drivers/phy/rockchip/phy-rockchip-pcie.c b/drivers/phy/rockchip/phy-rockchip-pcie.c
index 6904633cad68..7cbdde029c0a 100644
--- a/drivers/phy/rockchip/phy-rockchip-pcie.c
+++ b/drivers/phy/rockchip/phy-rockchip-pcie.c
@@ -73,10 +73,38 @@ struct rockchip_pcie_data {
 struct rockchip_pcie_phy {
 	struct rockchip_pcie_data *phy_data;
 	struct regmap *reg_base;
+	struct phy_pcie_instance {
+		struct phy *phy;
+		u32 index;
+	} phys[PHY_MAX_LANE_NUM];
+	struct mutex pcie_mutex;
 	struct reset_control *phy_rst;
 	struct clk *clk_pciephy_ref;
+	int pwr_cnt;
+	int init_cnt;
 };
 
+static struct rockchip_pcie_phy *to_pcie_phy(struct phy_pcie_instance *inst)
+{
+	return container_of(inst, struct rockchip_pcie_phy,
+					phys[inst->index]);
+}
+
+static struct phy *rockchip_pcie_phy_of_xlate(struct device *dev,
+					      struct of_phandle_args *args)
+{
+	struct rockchip_pcie_phy *rk_phy = dev_get_drvdata(dev);
+
+	if (args->args_count == 0)
+		return rk_phy->phys[0].phy;
+
+	if (WARN_ON(args->args[0] >= PHY_MAX_LANE_NUM))
+		return ERR_PTR(-ENODEV);
+
+	return rk_phy->phys[args->args[0]].phy;
+}
+
+
 static inline void phy_wr_cfg(struct rockchip_pcie_phy *rk_phy,
 			      u32 addr, u32 data)
 {
@@ -116,29 +144,59 @@ static inline u32 phy_rd_cfg(struct rockchip_pcie_phy *rk_phy,
 
 static int rockchip_pcie_phy_power_off(struct phy *phy)
 {
-	struct rockchip_pcie_phy *rk_phy = phy_get_drvdata(phy);
+	struct phy_pcie_instance *inst = phy_get_drvdata(phy);
+	struct rockchip_pcie_phy *rk_phy = to_pcie_phy(inst);
 	int err = 0;
 
+	mutex_lock(&rk_phy->pcie_mutex);
+
+	regmap_write(rk_phy->reg_base,
+		     rk_phy->phy_data->pcie_laneoff,
+		     HIWORD_UPDATE(PHY_LANE_IDLE_OFF,
+				   PHY_LANE_IDLE_MASK,
+				   PHY_LANE_IDLE_A_SHIFT + inst->index));
+
+	if (--rk_phy->pwr_cnt)
+		goto err_out;
+
 	err = reset_control_assert(rk_phy->phy_rst);
 	if (err) {
 		dev_err(&phy->dev, "assert phy_rst err %d\n", err);
-		return err;
+		goto err_restore;
 	}
 
+err_out:
+	mutex_unlock(&rk_phy->pcie_mutex);
 	return 0;
+
+err_restore:
+	rk_phy->pwr_cnt++;
+	regmap_write(rk_phy->reg_base,
+		     rk_phy->phy_data->pcie_laneoff,
+		     HIWORD_UPDATE(!PHY_LANE_IDLE_OFF,
+				   PHY_LANE_IDLE_MASK,
+				   PHY_LANE_IDLE_A_SHIFT + inst->index));
+	mutex_unlock(&rk_phy->pcie_mutex);
+	return err;
 }
 
 static int rockchip_pcie_phy_power_on(struct phy *phy)
 {
-	struct rockchip_pcie_phy *rk_phy = phy_get_drvdata(phy);
+	struct phy_pcie_instance *inst = phy_get_drvdata(phy);
+	struct rockchip_pcie_phy *rk_phy = to_pcie_phy(inst);
 	int err = 0;
 	u32 status;
 	unsigned long timeout;
 
+	mutex_lock(&rk_phy->pcie_mutex);
+
+	if (rk_phy->pwr_cnt++)
+		goto err_out;
+
 	err = reset_control_deassert(rk_phy->phy_rst);
 	if (err) {
 		dev_err(&phy->dev, "deassert phy_rst err %d\n", err);
-		return err;
+		goto err_pwr_cnt;
 	}
 
 	regmap_write(rk_phy->reg_base, rk_phy->phy_data->pcie_conf,
@@ -146,6 +204,12 @@ static int rockchip_pcie_phy_power_on(struct phy *phy)
 				   PHY_CFG_ADDR_MASK,
 				   PHY_CFG_ADDR_SHIFT));
 
+	regmap_write(rk_phy->reg_base,
+		     rk_phy->phy_data->pcie_laneoff,
+		     HIWORD_UPDATE(!PHY_LANE_IDLE_OFF,
+				   PHY_LANE_IDLE_MASK,
+				   PHY_LANE_IDLE_A_SHIFT + inst->index));
+
 	/*
 	 * No documented timeout value for phy operation below,
 	 * so we make it large enough here. And we use loop-break
@@ -214,18 +278,29 @@ static int rockchip_pcie_phy_power_on(struct phy *phy)
 		goto err_pll_lock;
 	}
 
+err_out:
+	mutex_unlock(&rk_phy->pcie_mutex);
 	return 0;
 
 err_pll_lock:
 	reset_control_assert(rk_phy->phy_rst);
+err_pwr_cnt:
+	rk_phy->pwr_cnt--;
+	mutex_unlock(&rk_phy->pcie_mutex);
 	return err;
 }
 
 static int rockchip_pcie_phy_init(struct phy *phy)
 {
-	struct rockchip_pcie_phy *rk_phy = phy_get_drvdata(phy);
+	struct phy_pcie_instance *inst = phy_get_drvdata(phy);
+	struct rockchip_pcie_phy *rk_phy = to_pcie_phy(inst);
 	int err = 0;
 
+	mutex_lock(&rk_phy->pcie_mutex);
+
+	if (rk_phy->init_cnt++)
+		goto err_out;
+
 	err = clk_prepare_enable(rk_phy->clk_pciephy_ref);
 	if (err) {
 		dev_err(&phy->dev, "Fail to enable pcie ref clock.\n");
@@ -238,20 +313,33 @@ static int rockchip_pcie_phy_init(struct phy *phy)
 		goto err_reset;
 	}
 
-	return err;
+err_out:
+	mutex_unlock(&rk_phy->pcie_mutex);
+	return 0;
 
 err_reset:
+
 	clk_disable_unprepare(rk_phy->clk_pciephy_ref);
 err_refclk:
+	rk_phy->init_cnt--;
+	mutex_unlock(&rk_phy->pcie_mutex);
 	return err;
 }
 
 static int rockchip_pcie_phy_exit(struct phy *phy)
 {
-	struct rockchip_pcie_phy *rk_phy = phy_get_drvdata(phy);
+	struct phy_pcie_instance *inst = phy_get_drvdata(phy);
+	struct rockchip_pcie_phy *rk_phy = to_pcie_phy(inst);
+
+	mutex_lock(&rk_phy->pcie_mutex);
+
+	if (--rk_phy->init_cnt)
+		goto err_init_cnt;
 
 	clk_disable_unprepare(rk_phy->clk_pciephy_ref);
 
+err_init_cnt:
+	mutex_unlock(&rk_phy->pcie_mutex);
 	return 0;
 }
 
@@ -283,10 +371,11 @@ static int rockchip_pcie_phy_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct rockchip_pcie_phy *rk_phy;
-	struct phy *generic_phy;
 	struct phy_provider *phy_provider;
 	struct regmap *grf;
 	const struct of_device_id *of_id;
+	int i;
+	u32 phy_num;
 
 	grf = syscon_node_to_regmap(dev->parent->of_node);
 	if (IS_ERR(grf)) {
@@ -305,6 +394,8 @@ static int rockchip_pcie_phy_probe(struct platform_device *pdev)
 	rk_phy->phy_data = (struct rockchip_pcie_data *)of_id->data;
 	rk_phy->reg_base = grf;
 
+	mutex_init(&rk_phy->pcie_mutex);
+
 	rk_phy->phy_rst = devm_reset_control_get(dev, "phy");
 	if (IS_ERR(rk_phy->phy_rst)) {
 		if (PTR_ERR(rk_phy->phy_rst) != -EPROBE_DEFER)
@@ -319,14 +410,26 @@ static int rockchip_pcie_phy_probe(struct platform_device *pdev)
 		return PTR_ERR(rk_phy->clk_pciephy_ref);
 	}
 
-	generic_phy = devm_phy_create(dev, dev->of_node, &ops);
-	if (IS_ERR(generic_phy)) {
-		dev_err(dev, "failed to create PHY\n");
-		return PTR_ERR(generic_phy);
+	/* parse #phy-cells to see if it's legacy PHY model */
+	if (of_property_read_u32(dev->of_node, "#phy-cells", &phy_num))
+		return -ENOENT;
+
+	phy_num = (phy_num == 0) ? 1 : PHY_MAX_LANE_NUM;
+	dev_dbg(dev, "phy number is %d\n", phy_num);
+
+	for (i = 0; i < phy_num; i++) {
+		rk_phy->phys[i].phy = devm_phy_create(dev, dev->of_node, &ops);
+		if (IS_ERR(rk_phy->phys[i].phy)) {
+			dev_err(dev, "failed to create PHY%d\n", i);
+			return PTR_ERR(rk_phy->phys[i].phy);
+		}
+		rk_phy->phys[i].index = i;
+		phy_set_drvdata(rk_phy->phys[i].phy, &rk_phy->phys[i]);
 	}
 
-	phy_set_drvdata(generic_phy, rk_phy);
-	phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
+	platform_set_drvdata(pdev, rk_phy);
+	phy_provider = devm_of_phy_provider_register(dev,
+					rockchip_pcie_phy_of_xlate);
 
 	return PTR_ERR_OR_ZERO(phy_provider);
 }
diff --git a/drivers/phy/rockchip/phy-rockchip-typec.c b/drivers/phy/rockchip/phy-rockchip-typec.c
index 7cfb0f8995de..4d2c57f21d76 100644
--- a/drivers/phy/rockchip/phy-rockchip-typec.c
+++ b/drivers/phy/rockchip/phy-rockchip-typec.c
@@ -622,12 +622,11 @@ static int tcphy_get_mode(struct rockchip_typec_phy *tcphy)
 	struct extcon_dev *edev = tcphy->extcon;
 	union extcon_property_value property;
 	unsigned int id;
-	bool dfp, ufp, dp;
+	bool ufp, dp;
 	u8 mode;
 	int ret;
 
 	ufp = extcon_get_state(edev, EXTCON_USB);
-	dfp = extcon_get_state(edev, EXTCON_USB_HOST);
 	dp = extcon_get_state(edev, EXTCON_DISP_DP);
 
 	mode = MODE_DFP_USB;
diff --git a/drivers/phy/samsung/phy-exynos-dp-video.c b/drivers/phy/samsung/phy-exynos-dp-video.c
index bb3279dbf88c..2dd6dd1f37a8 100644
--- a/drivers/phy/samsung/phy-exynos-dp-video.c
+++ b/drivers/phy/samsung/phy-exynos-dp-video.c
@@ -16,6 +16,7 @@
 #include <linux/mfd/syscon.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
+#include <linux/of_device.h>
 #include <linux/phy/phy.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
@@ -78,7 +79,6 @@ static int exynos_dp_video_phy_probe(struct platform_device *pdev)
 {
 	struct exynos_dp_video_phy *state;
 	struct device *dev = &pdev->dev;
-	const struct of_device_id *match;
 	struct phy_provider *phy_provider;
 	struct phy *phy;
 
@@ -93,8 +93,7 @@ static int exynos_dp_video_phy_probe(struct platform_device *pdev)
 		return PTR_ERR(state->regs);
 	}
 
-	match = of_match_node(exynos_dp_video_phy_of_match, dev->of_node);
-	state->drvdata = match->data;
+	state->drvdata = of_device_get_match_data(dev);
 
 	phy = devm_phy_create(dev, NULL, &exynos_dp_video_phy_ops);
 	if (IS_ERR(phy)) {
diff --git a/drivers/phy/samsung/phy-exynos5-usbdrd.c b/drivers/phy/samsung/phy-exynos5-usbdrd.c
index 7c41daa2c625..22c68f58b181 100644
--- a/drivers/phy/samsung/phy-exynos5-usbdrd.c
+++ b/drivers/phy/samsung/phy-exynos5-usbdrd.c
@@ -18,6 +18,7 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
+#include <linux/of_device.h>
 #include <linux/phy/phy.h>
 #include <linux/platform_device.h>
 #include <linux/mutex.h>
@@ -662,7 +663,6 @@ static int exynos5_usbdrd_phy_probe(struct platform_device *pdev)
 	struct exynos5_usbdrd_phy *phy_drd;
 	struct phy_provider *phy_provider;
 	struct resource *res;
-	const struct of_device_id *match;
 	const struct exynos5_usbdrd_phy_drvdata *drv_data;
 	struct regmap *reg_pmu;
 	u32 pmu_offset;
@@ -681,9 +681,10 @@ static int exynos5_usbdrd_phy_probe(struct platform_device *pdev)
 	if (IS_ERR(phy_drd->reg_phy))
 		return PTR_ERR(phy_drd->reg_phy);
 
-	match = of_match_node(exynos5_usbdrd_phy_of_match, pdev->dev.of_node);
+	drv_data = of_device_get_match_data(dev);
+	if (!drv_data)
+		return -EINVAL;
 
-	drv_data = match->data;
 	phy_drd->drv_data = drv_data;
 
 	ret = exynos5_usbdrd_phy_clk_handle(phy_drd);
diff --git a/drivers/phy/samsung/phy-samsung-usb2.c b/drivers/phy/samsung/phy-samsung-usb2.c
index 1d22d93b552d..ea818866985a 100644
--- a/drivers/phy/samsung/phy-samsung-usb2.c
+++ b/drivers/phy/samsung/phy-samsung-usb2.c
@@ -14,6 +14,7 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
+#include <linux/of_device.h>
 #include <linux/phy/phy.h>
 #include <linux/platform_device.h>
 #include <linux/spinlock.h>
@@ -142,7 +143,6 @@ MODULE_DEVICE_TABLE(of, samsung_usb2_phy_of_match);
 
 static int samsung_usb2_phy_probe(struct platform_device *pdev)
 {
-	const struct of_device_id *match;
 	const struct samsung_usb2_phy_config *cfg;
 	struct device *dev = &pdev->dev;
 	struct phy_provider *phy_provider;
@@ -155,12 +155,9 @@ static int samsung_usb2_phy_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
-	match = of_match_node(samsung_usb2_phy_of_match, pdev->dev.of_node);
-	if (!match) {
-		dev_err(dev, "of_match_node() failed\n");
+	cfg = of_device_get_match_data(dev);
+	if (!cfg)
 		return -EINVAL;
-	}
-	cfg = match->data;
 
 	drv = devm_kzalloc(dev, sizeof(struct samsung_usb2_phy_driver) +
 		cfg->num_phys * sizeof(struct samsung_usb2_phy_instance),
diff --git a/drivers/phy/ti/phy-ti-pipe3.c b/drivers/phy/ti/phy-ti-pipe3.c
index 9c84d32c6f60..0e564f32749f 100644
--- a/drivers/phy/ti/phy-ti-pipe3.c
+++ b/drivers/phy/ti/phy-ti-pipe3.c
@@ -118,12 +118,12 @@ static struct pipe3_dpll_map dpll_map_usb[] = {
 };
 
 static struct pipe3_dpll_map dpll_map_sata[] = {
-	{12000000, {1000, 7, 4, 6, 0} },	/* 12 MHz */
-	{16800000, {714, 7, 4, 6, 0} },		/* 16.8 MHz */
+	{12000000, {625, 4, 4, 6, 0} },	/* 12 MHz */
+	{16800000, {625, 6, 4, 7, 0} },		/* 16.8 MHz */
 	{19200000, {625, 7, 4, 6, 0} },		/* 19.2 MHz */
-	{20000000, {600, 7, 4, 6, 0} },		/* 20 MHz */
-	{26000000, {461, 7, 4, 6, 0} },		/* 26 MHz */
-	{38400000, {312, 7, 4, 6, 0} },		/* 38.4 MHz */
+	{20000000, {750, 9, 4, 6, 0} },		/* 20 MHz */
+	{26000000, {750, 12, 4, 6, 0} },	/* 26 MHz */
+	{38400000, {625, 15, 4, 6, 0} },	/* 38.4 MHz */
 	{ },					/* Terminator */
 };
 
diff --git a/drivers/phy/ti/phy-twl4030-usb.c b/drivers/phy/ti/phy-twl4030-usb.c
index 2990b3965460..a44680d64f9b 100644
--- a/drivers/phy/ti/phy-twl4030-usb.c
+++ b/drivers/phy/ti/phy-twl4030-usb.c
@@ -36,7 +36,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/usb/musb.h>
 #include <linux/usb/ulpi.h>
-#include <linux/i2c/twl.h>
+#include <linux/mfd/twl.h>
 #include <linux/regulator/consumer.h>
 #include <linux/err.h>
 #include <linux/slab.h>
@@ -185,7 +185,7 @@ struct twl4030_usb {
 static int twl4030_i2c_write_u8_verify(struct twl4030_usb *twl,
 		u8 module, u8 data, u8 address)
 {
-	u8 check;
+	u8 check = 0xFF;
 
 	if ((twl_i2c_write_u8(module, data, address) >= 0) &&
 	    (twl_i2c_read_u8(module, &check, address) >= 0) &&
diff --git a/drivers/pinctrl/Kconfig b/drivers/pinctrl/Kconfig
index e14b46c7b37f..1778cf4f81c7 100644
--- a/drivers/pinctrl/Kconfig
+++ b/drivers/pinctrl/Kconfig
@@ -146,6 +146,13 @@ config PINCTRL_FALCON
 	depends on SOC_FALCON
 	depends on PINCTRL_LANTIQ
 
+config PINCTRL_GEMINI
+	bool
+	depends on ARCH_GEMINI
+	default ARCH_GEMINI
+	select PINMUX
+	select MFD_SYSCON
+
 config PINCTRL_MCP23S08
 	tristate "Microchip MCP23xxx I/O expander"
 	depends on SPI_MASTER || I2C
@@ -331,6 +338,15 @@ config PINCTRL_INGENIC
 	select GENERIC_PINMUX_FUNCTIONS
 	select REGMAP_MMIO
 
+config PINCTRL_RK805
+	tristate "Pinctrl and GPIO driver for RK805 PMIC"
+	depends on MFD_RK808
+	select GPIOLIB
+	select PINMUX
+	select GENERIC_PINCONF
+	help
+	  This selects the pinctrl driver for RK805.
+
 source "drivers/pinctrl/aspeed/Kconfig"
 source "drivers/pinctrl/bcm/Kconfig"
 source "drivers/pinctrl/berlin/Kconfig"
@@ -343,6 +359,7 @@ source "drivers/pinctrl/qcom/Kconfig"
 source "drivers/pinctrl/samsung/Kconfig"
 source "drivers/pinctrl/sh-pfc/Kconfig"
 source "drivers/pinctrl/spear/Kconfig"
+source "drivers/pinctrl/sprd/Kconfig"
 source "drivers/pinctrl/stm32/Kconfig"
 source "drivers/pinctrl/sunxi/Kconfig"
 source "drivers/pinctrl/tegra/Kconfig"
diff --git a/drivers/pinctrl/Makefile b/drivers/pinctrl/Makefile
index 2bc641d62400..c16e27900dbb 100644
--- a/drivers/pinctrl/Makefile
+++ b/drivers/pinctrl/Makefile
@@ -18,6 +18,7 @@ obj-$(CONFIG_PINCTRL_AMD)	+= pinctrl-amd.o
 obj-$(CONFIG_PINCTRL_DA850_PUPD) += pinctrl-da850-pupd.o
 obj-$(CONFIG_PINCTRL_DIGICOLOR)	+= pinctrl-digicolor.o
 obj-$(CONFIG_PINCTRL_FALCON)	+= pinctrl-falcon.o
+obj-$(CONFIG_PINCTRL_GEMINI)	+= pinctrl-gemini.o
 obj-$(CONFIG_PINCTRL_MAX77620)	+= pinctrl-max77620.o
 obj-$(CONFIG_PINCTRL_MCP23S08)	+= pinctrl-mcp23s08.o
 obj-$(CONFIG_PINCTRL_MESON)	+= meson/
@@ -42,6 +43,7 @@ obj-$(CONFIG_PINCTRL_TB10X)	+= pinctrl-tb10x.o
 obj-$(CONFIG_PINCTRL_ST) 	+= pinctrl-st.o
 obj-$(CONFIG_PINCTRL_ZYNQ)	+= pinctrl-zynq.o
 obj-$(CONFIG_PINCTRL_INGENIC)	+= pinctrl-ingenic.o
+obj-$(CONFIG_PINCTRL_RK805)	+= pinctrl-rk805.o
 
 obj-$(CONFIG_ARCH_ASPEED)	+= aspeed/
 obj-y				+= bcm/
@@ -55,6 +57,7 @@ obj-$(CONFIG_ARCH_QCOM)		+= qcom/
 obj-$(CONFIG_PINCTRL_SAMSUNG)	+= samsung/
 obj-$(CONFIG_PINCTRL_SH_PFC)	+= sh-pfc/
 obj-$(CONFIG_PINCTRL_SPEAR)	+= spear/
+obj-y				+= sprd/
 obj-$(CONFIG_PINCTRL_STM32)	+= stm32/
 obj-$(CONFIG_PINCTRL_SUNXI)	+= sunxi/
 obj-y				+= ti/
diff --git a/drivers/pinctrl/aspeed/pinctrl-aspeed-g4.c b/drivers/pinctrl/aspeed/pinctrl-aspeed-g4.c
index cf3106cec048..05b153034517 100644
--- a/drivers/pinctrl/aspeed/pinctrl-aspeed-g4.c
+++ b/drivers/pinctrl/aspeed/pinctrl-aspeed-g4.c
@@ -1006,15 +1006,23 @@ SS_PIN_DECL(H3, GPIOQ5, SDA14);
 
 FUNC_GROUP_DECL(I2C14, H4, H3);
 
-#define DASH9028_DESC	SIG_DESC_SET(SCU90, 28)
+/*
+ * There are several opportunities to document USB port 4 in the datasheet, but
+ * it is only mentioned in one location. Particularly, the Multi-function Pins
+ * Mapping and Control table in the datasheet elides the signal names,
+ * suggesting that port 4 may not actually be functional. As such we define the
+ * signal names and control bit, but don't export the capability's function or
+ * group.
+ */
+#define USB11H3_DESC	SIG_DESC_SET(SCU90, 28)
 
 #define H2 134
-SIG_EXPR_LIST_DECL_SINGLE(DASHH2, DASHH2, DASH9028_DESC);
-SS_PIN_DECL(H2, GPIOQ6, DASHH2);
+SIG_EXPR_LIST_DECL_SINGLE(USB11HDP3, USB11H3, USB11H3_DESC);
+SS_PIN_DECL(H2, GPIOQ6, USB11HDP3);
 
 #define H1 135
-SIG_EXPR_LIST_DECL_SINGLE(DASHH1, DASHH1, DASH9028_DESC);
-SS_PIN_DECL(H1, GPIOQ7, DASHH1);
+SIG_EXPR_LIST_DECL_SINGLE(USB11HDN3, USB11H3, USB11H3_DESC);
+SS_PIN_DECL(H1, GPIOQ7, USB11HDN3);
 
 #define V20 136
 SSSF_PIN_DECL(V20, GPIOR0, ROMCS1, SIG_DESC_SET(SCU88, 24));
@@ -1706,10 +1714,42 @@ FUNC_GROUP_DECL(VPO12, U21, T19, V22, U20, R22, P18, P19, P20, P21, P22, M19,
 FUNC_GROUP_DECL(VPO24, U21, T19, V22, U20, L22, K18, V21, W22, R22, P18, P19,
 		P20, P21, P22, M19, M20, M21, M22, L18, L19);
 
+#define USB11H2_DESC	SIG_DESC_SET(SCU90, 3)
+#define USB11D1_DESC	SIG_DESC_BIT(SCU90, 3, 0)
+
+#define K4 220
+SIG_EXPR_LIST_DECL_SINGLE(USB11HDP2, USB11H2, USB11H2_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(USB11DP1, USB11D1, USB11D1_DESC);
+MS_PIN_DECL_(K4, SIG_EXPR_LIST_PTR(USB11HDP2), SIG_EXPR_LIST_PTR(USB11DP1));
+
+#define K3 221
+SIG_EXPR_LIST_DECL_SINGLE(USB11HDN1, USB11H2, USB11H2_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(USB11DDN1, USB11D1, USB11D1_DESC);
+MS_PIN_DECL_(K3, SIG_EXPR_LIST_PTR(USB11HDN1), SIG_EXPR_LIST_PTR(USB11DDN1));
+
+FUNC_GROUP_DECL(USB11H2, K4, K3);
+FUNC_GROUP_DECL(USB11D1, K4, K3);
+
+#define USB2H1_DESC	SIG_DESC_SET(SCU90, 29)
+#define USB2D1_DESC	SIG_DESC_BIT(SCU90, 29, 0)
+
+#define AB21 222
+SIG_EXPR_LIST_DECL_SINGLE(USB2HDP1, USB2H1, USB2H1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(USB2DDP1, USB2D1, USB2D1_DESC);
+MS_PIN_DECL_(AB21, SIG_EXPR_LIST_PTR(USB2HDP1), SIG_EXPR_LIST_PTR(USB2DDP1));
+
+#define AB20 223
+SIG_EXPR_LIST_DECL_SINGLE(USB2HDN1, USB2H1, USB2H1_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(USB2DDN1, USB2D1, USB2D1_DESC);
+MS_PIN_DECL_(AB20, SIG_EXPR_LIST_PTR(USB2HDN1), SIG_EXPR_LIST_PTR(USB2DDN1));
+
+FUNC_GROUP_DECL(USB2H1, AB21, AB20);
+FUNC_GROUP_DECL(USB2D1, AB21, AB20);
+
 /* Note we account for GPIOY4-GPIOY7 even though they're not valid, thus 216
- * pins becomes 220.
+ * pins becomes 220. Four additional non-GPIO-capable pins are present for USB.
  */
-#define ASPEED_G4_NR_PINS 220
+#define ASPEED_G4_NR_PINS 224
 
 /* Pins, groups and functions are sort(1):ed alphabetically for sanity */
 
@@ -1749,6 +1789,8 @@ static struct pinctrl_pin_desc aspeed_g4_pins[ASPEED_G4_NR_PINS] = {
 	ASPEED_PINCTRL_PIN(AB5),
 	ASPEED_PINCTRL_PIN(AB6),
 	ASPEED_PINCTRL_PIN(AB7),
+	ASPEED_PINCTRL_PIN(AB20),
+	ASPEED_PINCTRL_PIN(AB21),
 	ASPEED_PINCTRL_PIN(B1),
 	ASPEED_PINCTRL_PIN(B10),
 	ASPEED_PINCTRL_PIN(B11),
@@ -1848,6 +1890,8 @@ static struct pinctrl_pin_desc aspeed_g4_pins[ASPEED_G4_NR_PINS] = {
 	ASPEED_PINCTRL_PIN(J5),
 	ASPEED_PINCTRL_PIN(K18),
 	ASPEED_PINCTRL_PIN(K20),
+	ASPEED_PINCTRL_PIN(K3),
+	ASPEED_PINCTRL_PIN(K4),
 	ASPEED_PINCTRL_PIN(K5),
 	ASPEED_PINCTRL_PIN(L1),
 	ASPEED_PINCTRL_PIN(L18),
@@ -2070,6 +2114,10 @@ static const struct aspeed_pin_group aspeed_g4_groups[] = {
 	ASPEED_PINCTRL_GROUP(TXD3),
 	ASPEED_PINCTRL_GROUP(TXD4),
 	ASPEED_PINCTRL_GROUP(UART6),
+	ASPEED_PINCTRL_GROUP(USB11D1),
+	ASPEED_PINCTRL_GROUP(USB11H2),
+	ASPEED_PINCTRL_GROUP(USB2D1),
+	ASPEED_PINCTRL_GROUP(USB2H1),
 	ASPEED_PINCTRL_GROUP(USBCKI),
 	ASPEED_PINCTRL_GROUP(VGABIOS_ROM),
 	ASPEED_PINCTRL_GROUP(VGAHS),
@@ -2221,6 +2269,10 @@ static const struct aspeed_pin_function aspeed_g4_functions[] = {
 	ASPEED_PINCTRL_FUNC(TXD3),
 	ASPEED_PINCTRL_FUNC(TXD4),
 	ASPEED_PINCTRL_FUNC(UART6),
+	ASPEED_PINCTRL_FUNC(USB11D1),
+	ASPEED_PINCTRL_FUNC(USB11H2),
+	ASPEED_PINCTRL_FUNC(USB2D1),
+	ASPEED_PINCTRL_FUNC(USB2H1),
 	ASPEED_PINCTRL_FUNC(USBCKI),
 	ASPEED_PINCTRL_FUNC(VGABIOS_ROM),
 	ASPEED_PINCTRL_FUNC(VGAHS),
@@ -2349,7 +2401,7 @@ static struct aspeed_pinctrl_data aspeed_g4_pinctrl_data = {
 	.nconfigs = ARRAY_SIZE(aspeed_g4_configs),
 };
 
-static struct pinmux_ops aspeed_g4_pinmux_ops = {
+static const struct pinmux_ops aspeed_g4_pinmux_ops = {
 	.get_functions_count = aspeed_pinmux_get_fn_count,
 	.get_function_name = aspeed_pinmux_get_fn_name,
 	.get_function_groups = aspeed_pinmux_get_fn_groups,
@@ -2358,7 +2410,7 @@ static struct pinmux_ops aspeed_g4_pinmux_ops = {
 	.strict = true,
 };
 
-static struct pinctrl_ops aspeed_g4_pinctrl_ops = {
+static const struct pinctrl_ops aspeed_g4_pinctrl_ops = {
 	.get_groups_count = aspeed_pinctrl_get_groups_count,
 	.get_group_name = aspeed_pinctrl_get_group_name,
 	.get_group_pins = aspeed_pinctrl_get_group_pins,
diff --git a/drivers/pinctrl/aspeed/pinctrl-aspeed-g5.c b/drivers/pinctrl/aspeed/pinctrl-aspeed-g5.c
index 68aa04664a62..187abd7693cf 100644
--- a/drivers/pinctrl/aspeed/pinctrl-aspeed-g5.c
+++ b/drivers/pinctrl/aspeed/pinctrl-aspeed-g5.c
@@ -25,7 +25,7 @@
 #include "../pinctrl-utils.h"
 #include "pinctrl-aspeed.h"
 
-#define ASPEED_G5_NR_PINS 232
+#define ASPEED_G5_NR_PINS 236
 
 #define COND1		{ ASPEED_IP_SCU, SCU90, BIT(6), 0, 0 }
 #define COND2		{ ASPEED_IP_SCU, SCU94, GENMASK(1, 0), 0, 0 }
@@ -1724,6 +1724,48 @@ FUNC_GROUP_DECL(LPCRST, G22);
 
 FUNC_GROUP_DECL(ESPI, G21, G20, D22, E22, C22, F21, F22, G22);
 
+#define A7 232
+SIG_EXPR_LIST_DECL_SINGLE(USB2AHDP, USB2AH, SIG_DESC_SET(SCU90, 29));
+SIG_EXPR_LIST_DECL_SINGLE(USB2ADDP, USB2AD, SIG_DESC_BIT(SCU90, 29, 0));
+MS_PIN_DECL_(A7, SIG_EXPR_LIST_PTR(USB2AHDP), SIG_EXPR_LIST_PTR(USB2ADDP));
+
+#define A8 233
+SIG_EXPR_LIST_DECL_SINGLE(USB2AHDN, USB2AH, SIG_DESC_SET(SCU90, 29));
+SIG_EXPR_LIST_DECL_SINGLE(USB2ADDN, USB2AD, SIG_DESC_BIT(SCU90, 29, 0));
+MS_PIN_DECL_(A8, SIG_EXPR_LIST_PTR(USB2AHDN), SIG_EXPR_LIST_PTR(USB2ADDN));
+
+FUNC_GROUP_DECL(USB2AH, A7, A8);
+FUNC_GROUP_DECL(USB2AD, A7, A8);
+
+#define USB11BHID_DESC  { ASPEED_IP_SCU, SCU94, GENMASK(14, 13), 0, 0 }
+#define USB2BD_DESC   { ASPEED_IP_SCU, SCU94, GENMASK(14, 13), 1, 0 }
+#define USB2BH1_DESC { ASPEED_IP_SCU, SCU94, GENMASK(14, 13), 2, 0 }
+#define USB2BH2_DESC { ASPEED_IP_SCU, SCU94, GENMASK(14, 13), 3, 0 }
+
+#define B6 234
+SIG_EXPR_LIST_DECL_SINGLE(USB11BDP, USB11BHID, USB11BHID_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(USB2BDDP, USB2BD, USB2BD_DESC);
+SIG_EXPR_DECL(USB2BHDP1, USB2BH, USB2BH1_DESC);
+SIG_EXPR_DECL(USB2BHDP2, USB2BH, USB2BH2_DESC);
+SIG_EXPR_LIST_DECL(USB2BHDP, SIG_EXPR_PTR(USB2BHDP1, USB2BH),
+		SIG_EXPR_PTR(USB2BHDP2, USB2BH));
+MS_PIN_DECL_(B6, SIG_EXPR_LIST_PTR(USB11BDP), SIG_EXPR_LIST_PTR(USB2BDDP),
+		SIG_EXPR_LIST_PTR(USB2BHDP));
+
+#define A6 235
+SIG_EXPR_LIST_DECL_SINGLE(USB11BDN, USB11BHID, USB11BHID_DESC);
+SIG_EXPR_LIST_DECL_SINGLE(USB2BDN, USB2BD, USB2BD_DESC);
+SIG_EXPR_DECL(USB2BHDN1, USB2BH, USB2BH1_DESC);
+SIG_EXPR_DECL(USB2BHDN2, USB2BH, USB2BH2_DESC);
+SIG_EXPR_LIST_DECL(USB2BHDN, SIG_EXPR_PTR(USB2BHDN1, USB2BH),
+		SIG_EXPR_PTR(USB2BHDN2, USB2BH));
+MS_PIN_DECL_(A6, SIG_EXPR_LIST_PTR(USB11BDN), SIG_EXPR_LIST_PTR(USB2BDN),
+		SIG_EXPR_LIST_PTR(USB2BHDN));
+
+FUNC_GROUP_DECL(USB11BHID, B6, A6);
+FUNC_GROUP_DECL(USB2BD, B6, A6);
+FUNC_GROUP_DECL(USB2BH, B6, A6);
+
 /* Pins, groups and functions are sort(1):ed alphabetically for sanity */
 
 static struct pinctrl_pin_desc aspeed_g5_pins[ASPEED_G5_NR_PINS] = {
@@ -1743,6 +1785,9 @@ static struct pinctrl_pin_desc aspeed_g5_pins[ASPEED_G5_NR_PINS] = {
 	ASPEED_PINCTRL_PIN(A3),
 	ASPEED_PINCTRL_PIN(A4),
 	ASPEED_PINCTRL_PIN(A5),
+	ASPEED_PINCTRL_PIN(A6),
+	ASPEED_PINCTRL_PIN(A7),
+	ASPEED_PINCTRL_PIN(A8),
 	ASPEED_PINCTRL_PIN(A9),
 	ASPEED_PINCTRL_PIN(AA1),
 	ASPEED_PINCTRL_PIN(AA19),
@@ -1777,6 +1822,7 @@ static struct pinctrl_pin_desc aspeed_g5_pins[ASPEED_G5_NR_PINS] = {
 	ASPEED_PINCTRL_PIN(B3),
 	ASPEED_PINCTRL_PIN(B4),
 	ASPEED_PINCTRL_PIN(B5),
+	ASPEED_PINCTRL_PIN(B6),
 	ASPEED_PINCTRL_PIN(B9),
 	ASPEED_PINCTRL_PIN(C1),
 	ASPEED_PINCTRL_PIN(C11),
@@ -2111,6 +2157,11 @@ static const struct aspeed_pin_group aspeed_g5_groups[] = {
 	ASPEED_PINCTRL_GROUP(TXD3),
 	ASPEED_PINCTRL_GROUP(TXD4),
 	ASPEED_PINCTRL_GROUP(UART6),
+	ASPEED_PINCTRL_GROUP(USB11BHID),
+	ASPEED_PINCTRL_GROUP(USB2AD),
+	ASPEED_PINCTRL_GROUP(USB2AH),
+	ASPEED_PINCTRL_GROUP(USB2BD),
+	ASPEED_PINCTRL_GROUP(USB2BH),
 	ASPEED_PINCTRL_GROUP(USBCKI),
 	ASPEED_PINCTRL_GROUP(VGABIOSROM),
 	ASPEED_PINCTRL_GROUP(VGAHS),
@@ -2275,6 +2326,11 @@ static const struct aspeed_pin_function aspeed_g5_functions[] = {
 	ASPEED_PINCTRL_FUNC(TXD3),
 	ASPEED_PINCTRL_FUNC(TXD4),
 	ASPEED_PINCTRL_FUNC(UART6),
+	ASPEED_PINCTRL_FUNC(USB11BHID),
+	ASPEED_PINCTRL_FUNC(USB2AD),
+	ASPEED_PINCTRL_FUNC(USB2AH),
+	ASPEED_PINCTRL_FUNC(USB2BD),
+	ASPEED_PINCTRL_FUNC(USB2BH),
 	ASPEED_PINCTRL_FUNC(USBCKI),
 	ASPEED_PINCTRL_FUNC(VGABIOSROM),
 	ASPEED_PINCTRL_FUNC(VGAHS),
@@ -2436,7 +2492,7 @@ static struct aspeed_pinctrl_data aspeed_g5_pinctrl_data = {
 	.nconfigs = ARRAY_SIZE(aspeed_g5_configs),
 };
 
-static struct pinmux_ops aspeed_g5_pinmux_ops = {
+static const struct pinmux_ops aspeed_g5_pinmux_ops = {
 	.get_functions_count = aspeed_pinmux_get_fn_count,
 	.get_function_name = aspeed_pinmux_get_fn_name,
 	.get_function_groups = aspeed_pinmux_get_fn_groups,
@@ -2445,7 +2501,7 @@ static struct pinmux_ops aspeed_g5_pinmux_ops = {
 	.strict = true,
 };
 
-static struct pinctrl_ops aspeed_g5_pinctrl_ops = {
+static const struct pinctrl_ops aspeed_g5_pinctrl_ops = {
 	.get_groups_count = aspeed_pinctrl_get_groups_count,
 	.get_group_name = aspeed_pinctrl_get_group_name,
 	.get_group_pins = aspeed_pinctrl_get_group_pins,
@@ -2454,7 +2510,7 @@ static struct pinctrl_ops aspeed_g5_pinctrl_ops = {
 	.dt_free_map = pinctrl_utils_free_map,
 };
 
-static struct pinconf_ops aspeed_g5_conf_ops = {
+static const struct pinconf_ops aspeed_g5_conf_ops = {
 	.is_generic = true,
 	.pin_config_get = aspeed_pin_config_get,
 	.pin_config_set = aspeed_pin_config_set,
diff --git a/drivers/pinctrl/aspeed/pinctrl-aspeed.c b/drivers/pinctrl/aspeed/pinctrl-aspeed.c
index a86a4d66099c..7f13ce8450a3 100644
--- a/drivers/pinctrl/aspeed/pinctrl-aspeed.c
+++ b/drivers/pinctrl/aspeed/pinctrl-aspeed.c
@@ -213,6 +213,27 @@ static int aspeed_sig_expr_set(const struct aspeed_sig_expr *expr,
 		if (desc->ip == ASPEED_IP_SCU && desc->reg == HW_STRAP2)
 			continue;
 
+		/* On AST2500, Set bits in SCU7C are cleared from SCU70 */
+		if (desc->ip == ASPEED_IP_SCU && desc->reg == HW_STRAP1) {
+			unsigned int rev_id;
+
+			ret = regmap_read(maps[ASPEED_IP_SCU],
+				HW_REVISION_ID, &rev_id);
+			if (ret < 0)
+				return ret;
+
+			if (0x04 == (rev_id >> 24)) {
+				u32 value = ~val & desc->mask;
+
+				if (value) {
+					ret = regmap_write(maps[desc->ip],
+						HW_REVISION_ID, value);
+					if (ret < 0)
+						return ret;
+				}
+			}
+		}
+
 		ret = regmap_update_bits(maps[desc->ip], desc->reg,
 					 desc->mask, val);
 
diff --git a/drivers/pinctrl/aspeed/pinctrl-aspeed.h b/drivers/pinctrl/aspeed/pinctrl-aspeed.h
index fa125db828f5..d4d7f032c1da 100644
--- a/drivers/pinctrl/aspeed/pinctrl-aspeed.h
+++ b/drivers/pinctrl/aspeed/pinctrl-aspeed.h
@@ -251,6 +251,7 @@
 #define SCU3C           0x3C /* System Reset Control/Status Register */
 #define SCU48           0x48 /* MAC Interface Clock Delay Setting */
 #define HW_STRAP1       0x70 /* AST2400 strapping is 33 bits, is split */
+#define HW_REVISION_ID  0x7C /* Silicon revision ID register */
 #define SCU80           0x80 /* Multi-function Pin Control #1 */
 #define SCU84           0x84 /* Multi-function Pin Control #2 */
 #define SCU88           0x88 /* Multi-function Pin Control #3 */
diff --git a/drivers/pinctrl/bcm/pinctrl-bcm281xx.c b/drivers/pinctrl/bcm/pinctrl-bcm281xx.c
index a7cceffcedfa..bc3b232a727a 100644
--- a/drivers/pinctrl/bcm/pinctrl-bcm281xx.c
+++ b/drivers/pinctrl/bcm/pinctrl-bcm281xx.c
@@ -1384,7 +1384,7 @@ static int bcm281xx_pinctrl_pin_config_set(struct pinctrl_dev *pctldev,
 	return 0;
 }
 
-static struct pinconf_ops bcm281xx_pinctrl_pinconf_ops = {
+static const struct pinconf_ops bcm281xx_pinctrl_pinconf_ops = {
 	.pin_config_get = bcm281xx_pinctrl_pin_config_get,
 	.pin_config_set = bcm281xx_pinctrl_pin_config_set,
 };
diff --git a/drivers/pinctrl/bcm/pinctrl-bcm2835.c b/drivers/pinctrl/bcm/pinctrl-bcm2835.c
index 230883168e99..0944310225db 100644
--- a/drivers/pinctrl/bcm/pinctrl-bcm2835.c
+++ b/drivers/pinctrl/bcm/pinctrl-bcm2835.c
@@ -92,7 +92,6 @@ struct bcm2835_pinctrl {
 	struct gpio_chip gpio_chip;
 	struct pinctrl_gpio_range gpio_range;
 
-	int irq_group[BCM2835_NUM_IRQS];
 	spinlock_t irq_lock[BCM2835_NUM_BANKS];
 };
 
@@ -353,7 +352,7 @@ static int bcm2835_gpio_direction_output(struct gpio_chip *chip,
 	return pinctrl_gpio_direction_output(chip->base + offset);
 }
 
-static struct gpio_chip bcm2835_gpio_chip = {
+static const struct gpio_chip bcm2835_gpio_chip = {
 	.label = MODULE_NAME,
 	.owner = THIS_MODULE,
 	.request = gpiochip_generic_request,
@@ -400,7 +399,7 @@ static void bcm2835_gpio_irq_handler(struct irq_desc *desc)
 
 	for (i = 0; i < ARRAY_SIZE(pc->irq); i++) {
 		if (pc->irq[i] == irq) {
-			group = pc->irq_group[i];
+			group = i;
 			break;
 		}
 	}
@@ -692,8 +691,7 @@ static int bcm2835_pctl_dt_node_to_map_func(struct bcm2835_pinctrl *pc,
 	struct pinctrl_map *map = *maps;
 
 	if (fnum >= ARRAY_SIZE(bcm2835_functions)) {
-		dev_err(pc->dev, "%s: invalid brcm,function %d\n",
-			of_node_full_name(np), fnum);
+		dev_err(pc->dev, "%pOF: invalid brcm,function %d\n", np, fnum);
 		return -EINVAL;
 	}
 
@@ -713,8 +711,7 @@ static int bcm2835_pctl_dt_node_to_map_pull(struct bcm2835_pinctrl *pc,
 	unsigned long *configs;
 
 	if (pull > 2) {
-		dev_err(pc->dev, "%s: invalid brcm,pull %d\n",
-			of_node_full_name(np), pull);
+		dev_err(pc->dev, "%pOF: invalid brcm,pull %d\n", np, pull);
 		return -EINVAL;
 	}
 
@@ -745,8 +742,7 @@ static int bcm2835_pctl_dt_node_to_map(struct pinctrl_dev *pctldev,
 
 	pins = of_find_property(np, "brcm,pins", NULL);
 	if (!pins) {
-		dev_err(pc->dev, "%s: missing brcm,pins property\n",
-				of_node_full_name(np));
+		dev_err(pc->dev, "%pOF: missing brcm,pins property\n", np);
 		return -EINVAL;
 	}
 
@@ -755,8 +751,8 @@ static int bcm2835_pctl_dt_node_to_map(struct pinctrl_dev *pctldev,
 
 	if (!funcs && !pulls) {
 		dev_err(pc->dev,
-			"%s: neither brcm,function nor brcm,pull specified\n",
-			of_node_full_name(np));
+			"%pOF: neither brcm,function nor brcm,pull specified\n",
+			np);
 		return -EINVAL;
 	}
 
@@ -766,15 +762,15 @@ static int bcm2835_pctl_dt_node_to_map(struct pinctrl_dev *pctldev,
 
 	if (num_funcs > 1 && num_funcs != num_pins) {
 		dev_err(pc->dev,
-			"%s: brcm,function must have 1 or %d entries\n",
-			of_node_full_name(np), num_pins);
+			"%pOF: brcm,function must have 1 or %d entries\n",
+			np, num_pins);
 		return -EINVAL;
 	}
 
 	if (num_pulls > 1 && num_pulls != num_pins) {
 		dev_err(pc->dev,
-			"%s: brcm,pull must have 1 or %d entries\n",
-			of_node_full_name(np), num_pins);
+			"%pOF: brcm,pull must have 1 or %d entries\n",
+			np, num_pins);
 		return -EINVAL;
 	}
 
@@ -793,8 +789,8 @@ static int bcm2835_pctl_dt_node_to_map(struct pinctrl_dev *pctldev,
 		if (err)
 			goto out;
 		if (pin >= ARRAY_SIZE(bcm2835_gpio_pins)) {
-			dev_err(pc->dev, "%s: invalid brcm,pins value %d\n",
-				of_node_full_name(np), pin);
+			dev_err(pc->dev, "%pOF: invalid brcm,pins value %d\n",
+				np, pin);
 			err = -EINVAL;
 			goto out;
 		}
@@ -1047,7 +1043,6 @@ static int bcm2835_pinctrl_probe(struct platform_device *pdev)
 
 	for (i = 0; i < BCM2835_NUM_IRQS; i++) {
 		pc->irq[i] = irq_of_parse_and_map(np, i);
-		pc->irq_group[i] = i;
 
 		if (pc->irq[i] == 0)
 			continue;
diff --git a/drivers/pinctrl/berlin/berlin.c b/drivers/pinctrl/berlin/berlin.c
index 8f0dc02f7624..cc3bd2efafe3 100644
--- a/drivers/pinctrl/berlin/berlin.c
+++ b/drivers/pinctrl/berlin/berlin.c
@@ -206,8 +206,8 @@ static int berlin_pinctrl_add_function(struct berlin_pinctrl *pctrl,
 static int berlin_pinctrl_build_state(struct platform_device *pdev)
 {
 	struct berlin_pinctrl *pctrl = platform_get_drvdata(pdev);
-	struct berlin_desc_group const *desc_group;
-	struct berlin_desc_function const *desc_function;
+	const struct berlin_desc_group *desc_group;
+	const struct berlin_desc_function *desc_function;
 	int i, max_functions = 0;
 
 	pctrl->nfunctions = 0;
diff --git a/drivers/pinctrl/core.c b/drivers/pinctrl/core.c
index c5e2c5705058..56fbe4c3e800 100644
--- a/drivers/pinctrl/core.c
+++ b/drivers/pinctrl/core.c
@@ -264,7 +264,7 @@ static int pinctrl_register_one_pin(struct pinctrl_dev *pctldev,
 }
 
 static int pinctrl_register_pins(struct pinctrl_dev *pctldev,
-				 struct pinctrl_pin_desc const *pins,
+				 const struct pinctrl_pin_desc *pins,
 				 unsigned num_descs)
 {
 	unsigned i;
@@ -686,7 +686,7 @@ EXPORT_SYMBOL_GPL(pinctrl_generic_remove_group);
 static void pinctrl_generic_free_groups(struct pinctrl_dev *pctldev)
 {
 	struct radix_tree_iter iter;
-	void **slot;
+	void __rcu **slot;
 
 	radix_tree_for_each_slot(slot, &pctldev->pin_group_tree, &iter, 0)
 		radix_tree_delete(&pctldev->pin_group_tree, iter.index);
@@ -907,7 +907,7 @@ static struct pinctrl_state *create_state(struct pinctrl *p,
 }
 
 static int add_setting(struct pinctrl *p, struct pinctrl_dev *pctldev,
-		       struct pinctrl_map const *map)
+		       const struct pinctrl_map *map)
 {
 	struct pinctrl_state *state;
 	struct pinctrl_setting *setting;
@@ -995,7 +995,7 @@ static struct pinctrl *create_pinctrl(struct device *dev,
 	const char *devname;
 	struct pinctrl_maps *maps_node;
 	int i;
-	struct pinctrl_map const *map;
+	const struct pinctrl_map *map;
 	int ret;
 
 	/*
@@ -1321,7 +1321,7 @@ void devm_pinctrl_put(struct pinctrl *p)
 }
 EXPORT_SYMBOL_GPL(devm_pinctrl_put);
 
-int pinctrl_register_map(struct pinctrl_map const *maps, unsigned num_maps,
+int pinctrl_register_map(const struct pinctrl_map *maps, unsigned num_maps,
 			 bool dup)
 {
 	int i, ret;
@@ -1380,7 +1380,6 @@ int pinctrl_register_map(struct pinctrl_map const *maps, unsigned num_maps,
 		maps_node->maps = kmemdup(maps, sizeof(*maps) * num_maps,
 					  GFP_KERNEL);
 		if (!maps_node->maps) {
-			pr_err("failed to duplicate mapping table\n");
 			kfree(maps_node);
 			return -ENOMEM;
 		}
@@ -1402,13 +1401,13 @@ int pinctrl_register_map(struct pinctrl_map const *maps, unsigned num_maps,
  *	function will perform a shallow copy for the mapping entries.
  * @num_maps: the number of maps in the mapping table
  */
-int pinctrl_register_mappings(struct pinctrl_map const *maps,
+int pinctrl_register_mappings(const struct pinctrl_map *maps,
 			      unsigned num_maps)
 {
 	return pinctrl_register_map(maps, num_maps, true);
 }
 
-void pinctrl_unregister_map(struct pinctrl_map const *map)
+void pinctrl_unregister_map(const struct pinctrl_map *map)
 {
 	struct pinctrl_maps *maps_node;
 
@@ -1702,7 +1701,7 @@ static int pinctrl_maps_show(struct seq_file *s, void *what)
 {
 	struct pinctrl_maps *maps_node;
 	int i;
-	struct pinctrl_map const *map;
+	const struct pinctrl_map *map;
 
 	seq_puts(s, "Pinctrl maps:\n");
 
diff --git a/drivers/pinctrl/core.h b/drivers/pinctrl/core.h
index 1c35de59a658..7880c3adc450 100644
--- a/drivers/pinctrl/core.h
+++ b/drivers/pinctrl/core.h
@@ -179,7 +179,7 @@ struct pin_desc {
  */
 struct pinctrl_maps {
 	struct list_head node;
-	struct pinctrl_map const *maps;
+	const struct pinctrl_map *maps;
 	unsigned num_maps;
 };
 
@@ -243,9 +243,9 @@ extern struct pinctrl_gpio_range *
 pinctrl_find_gpio_range_from_pin_nolock(struct pinctrl_dev *pctldev,
 					unsigned int pin);
 
-int pinctrl_register_map(struct pinctrl_map const *maps, unsigned num_maps,
+int pinctrl_register_map(const struct pinctrl_map *maps, unsigned num_maps,
 			 bool dup);
-void pinctrl_unregister_map(struct pinctrl_map const *map);
+void pinctrl_unregister_map(const struct pinctrl_map *map);
 
 extern int pinctrl_force_sleep(struct pinctrl_dev *pctldev);
 extern int pinctrl_force_default(struct pinctrl_dev *pctldev);
diff --git a/drivers/pinctrl/devicetree.c b/drivers/pinctrl/devicetree.c
index 0e5c9f14a706..1ff6c3573493 100644
--- a/drivers/pinctrl/devicetree.c
+++ b/drivers/pinctrl/devicetree.c
@@ -83,7 +83,6 @@ static int dt_remember_or_free_map(struct pinctrl *p, const char *statename,
 	/* Remember the converted mapping table entries */
 	dt_map = kzalloc(sizeof(*dt_map), GFP_KERNEL);
 	if (!dt_map) {
-		dev_err(p->dev, "failed to alloc struct pinctrl_dt_map\n");
 		dt_free_map(pctldev, map, num_maps);
 		return -ENOMEM;
 	}
@@ -117,8 +116,8 @@ static int dt_to_map_one_config(struct pinctrl *p,
 	for (;;) {
 		np_pctldev = of_get_next_parent(np_pctldev);
 		if (!np_pctldev || of_node_is_root(np_pctldev)) {
-			dev_info(p->dev, "could not find pctldev for node %s, deferring probe\n",
-				np_config->full_name);
+			dev_info(p->dev, "could not find pctldev for node %pOF, deferring probe\n",
+				np_config);
 			of_node_put(np_pctldev);
 			/* OK let's just assume this will appear later then */
 			return -EPROBE_DEFER;
@@ -158,10 +157,8 @@ static int dt_remember_dummy_state(struct pinctrl *p, const char *statename)
 	struct pinctrl_map *map;
 
 	map = kzalloc(sizeof(*map), GFP_KERNEL);
-	if (!map) {
-		dev_err(p->dev, "failed to alloc struct pinctrl_map\n");
+	if (!map)
 		return -ENOMEM;
-	}
 
 	/* There is no pctldev for PIN_MAP_TYPE_DUMMY_STATE */
 	map->type = PIN_MAP_TYPE_DUMMY_STATE;
diff --git a/drivers/pinctrl/freescale/Kconfig b/drivers/pinctrl/freescale/Kconfig
index 0b266b2aecd4..4dbc576ae27c 100644
--- a/drivers/pinctrl/freescale/Kconfig
+++ b/drivers/pinctrl/freescale/Kconfig
@@ -103,6 +103,13 @@ config PINCTRL_IMX7D
 	help
 	  Say Y here to enable the imx7d pinctrl driver
 
+config PINCTRL_IMX7ULP
+	bool "IMX7ULP pinctrl driver"
+	depends on SOC_IMX7ULP
+	select PINCTRL_IMX
+	help
+	  Say Y here to enable the imx7ulp pinctrl driver
+
 config PINCTRL_VF610
 	bool "Freescale Vybrid VF610 pinctrl driver"
 	depends on SOC_VF610
diff --git a/drivers/pinctrl/freescale/Makefile b/drivers/pinctrl/freescale/Makefile
index d44c9e253f21..525a5ff5dcb4 100644
--- a/drivers/pinctrl/freescale/Makefile
+++ b/drivers/pinctrl/freescale/Makefile
@@ -14,6 +14,7 @@ obj-$(CONFIG_PINCTRL_IMX6SL)	+= pinctrl-imx6sl.o
 obj-$(CONFIG_PINCTRL_IMX6SX)	+= pinctrl-imx6sx.o
 obj-$(CONFIG_PINCTRL_IMX6UL)	+= pinctrl-imx6ul.o
 obj-$(CONFIG_PINCTRL_IMX7D)	+= pinctrl-imx7d.o
+obj-$(CONFIG_PINCTRL_IMX7ULP)	+= pinctrl-imx7ulp.o
 obj-$(CONFIG_PINCTRL_VF610)	+= pinctrl-vf610.o
 obj-$(CONFIG_PINCTRL_MXS)	+= pinctrl-mxs.o
 obj-$(CONFIG_PINCTRL_IMX23)	+= pinctrl-imx23.o
diff --git a/drivers/pinctrl/freescale/pinctrl-imx.c b/drivers/pinctrl/freescale/pinctrl-imx.c
index 72aca758f4c6..6e472691d8ee 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx.c
@@ -35,18 +35,6 @@
 #define IMX_NO_PAD_CTL	0x80000000	/* no pin config need */
 #define IMX_PAD_SION 0x40000000		/* set SION */
 
-/**
- * @dev: a pointer back to containing device
- * @base: the offset to the controller in virtual memory
- */
-struct imx_pinctrl {
-	struct device *dev;
-	struct pinctrl_dev *pctl;
-	void __iomem *base;
-	void __iomem *input_sel_base;
-	struct imx_pinctrl_soc_info *info;
-};
-
 static inline const struct group_desc *imx_pinctrl_find_group_by_name(
 				struct pinctrl_dev *pctldev,
 				const char *name)
@@ -255,111 +243,11 @@ static int imx_pmx_set(struct pinctrl_dev *pctldev, unsigned selector,
 	return 0;
 }
 
-static int imx_pmx_gpio_request_enable(struct pinctrl_dev *pctldev,
-			struct pinctrl_gpio_range *range, unsigned offset)
-{
-	struct imx_pinctrl *ipctl = pinctrl_dev_get_drvdata(pctldev);
-	struct imx_pinctrl_soc_info *info = ipctl->info;
-	const struct imx_pin_reg *pin_reg;
-	struct group_desc *grp;
-	struct imx_pin *imx_pin;
-	unsigned int pin, group;
-	u32 reg;
-
-	/* Currently implementation only for shared mux/conf register */
-	if (!(info->flags & SHARE_MUX_CONF_REG))
-		return 0;
-
-	pin_reg = &info->pin_regs[offset];
-	if (pin_reg->mux_reg == -1)
-		return -EINVAL;
-
-	/* Find the pinctrl config with GPIO mux mode for the requested pin */
-	for (group = 0; group < pctldev->num_groups; group++) {
-		grp = pinctrl_generic_get_group(pctldev, group);
-		if (!grp)
-			continue;
-		for (pin = 0; pin < grp->num_pins; pin++) {
-			imx_pin = &((struct imx_pin *)(grp->data))[pin];
-			if (imx_pin->pin == offset && !imx_pin->mux_mode)
-				goto mux_pin;
-		}
-	}
-
-	return -EINVAL;
-
-mux_pin:
-	reg = readl(ipctl->base + pin_reg->mux_reg);
-	reg &= ~info->mux_mask;
-	reg |= imx_pin->config;
-	writel(reg, ipctl->base + pin_reg->mux_reg);
-
-	return 0;
-}
-
-static void imx_pmx_gpio_disable_free(struct pinctrl_dev *pctldev,
-			struct pinctrl_gpio_range *range, unsigned offset)
-{
-	struct imx_pinctrl *ipctl = pinctrl_dev_get_drvdata(pctldev);
-	struct imx_pinctrl_soc_info *info = ipctl->info;
-	const struct imx_pin_reg *pin_reg;
-	u32 reg;
-
-	/*
-	 * Only Vybrid has the input/output buffer enable flags (IBE/OBE)
-	 * They are part of the shared mux/conf register.
-	 */
-	if (!(info->flags & SHARE_MUX_CONF_REG))
-		return;
-
-	pin_reg = &info->pin_regs[offset];
-	if (pin_reg->mux_reg == -1)
-		return;
-
-	/* Clear IBE/OBE/PUE to disable the pin (Hi-Z) */
-	reg = readl(ipctl->base + pin_reg->mux_reg);
-	reg &= ~0x7;
-	writel(reg, ipctl->base + pin_reg->mux_reg);
-}
-
-static int imx_pmx_gpio_set_direction(struct pinctrl_dev *pctldev,
-	   struct pinctrl_gpio_range *range, unsigned offset, bool input)
-{
-	struct imx_pinctrl *ipctl = pinctrl_dev_get_drvdata(pctldev);
-	struct imx_pinctrl_soc_info *info = ipctl->info;
-	const struct imx_pin_reg *pin_reg;
-	u32 reg;
-
-	/*
-	 * Only Vybrid has the input/output buffer enable flags (IBE/OBE)
-	 * They are part of the shared mux/conf register.
-	 */
-	if (!(info->flags & SHARE_MUX_CONF_REG))
-		return 0;
-
-	pin_reg = &info->pin_regs[offset];
-	if (pin_reg->mux_reg == -1)
-		return -EINVAL;
-
-	/* IBE always enabled allows us to read the value "on the wire" */
-	reg = readl(ipctl->base + pin_reg->mux_reg);
-	if (input)
-		reg &= ~0x2;
-	else
-		reg |= 0x2;
-	writel(reg, ipctl->base + pin_reg->mux_reg);
-
-	return 0;
-}
-
-static const struct pinmux_ops imx_pmx_ops = {
+struct pinmux_ops imx_pmx_ops = {
 	.get_functions_count = pinmux_generic_get_function_count,
 	.get_function_name = pinmux_generic_get_function_name,
 	.get_function_groups = pinmux_generic_get_function_groups,
 	.set_mux = imx_pmx_set,
-	.gpio_request_enable = imx_pmx_gpio_request_enable,
-	.gpio_disable_free = imx_pmx_gpio_disable_free,
-	.gpio_set_direction = imx_pmx_gpio_set_direction,
 };
 
 /* decode generic config into raw register values */
@@ -563,26 +451,24 @@ static int imx_pinctrl_parse_groups(struct device_node *np,
 	 * do sanity check and calculate pins number
 	 *
 	 * First try legacy 'fsl,pins' property, then fall back to the
-	 * generic 'pins'.
+	 * generic 'pinmux'.
 	 *
-	 * Note: for generic 'pins' case, there's no CONFIG part in
+	 * Note: for generic 'pinmux' case, there's no CONFIG part in
 	 * the binding format.
 	 */
 	list = of_get_property(np, "fsl,pins", &size);
 	if (!list) {
-		list = of_get_property(np, "pins", &size);
+		list = of_get_property(np, "pinmux", &size);
 		if (!list) {
 			dev_err(info->dev,
-				"no fsl,pins and pins property in node %s\n",
-				np->full_name);
+				"no fsl,pins and pins property in node %pOF\n", np);
 			return -EINVAL;
 		}
 	}
 
 	/* we do not check return since it's safe node passed down */
 	if (!size || size % pin_size) {
-		dev_err(info->dev, "Invalid fsl,pins or pins property in node %s\n",
-			np->full_name);
+		dev_err(info->dev, "Invalid fsl,pins or pins property in node %pOF\n", np);
 		return -EINVAL;
 	}
 
@@ -666,7 +552,7 @@ static int imx_pinctrl_parse_functions(struct device_node *np,
 	func->name = np->name;
 	func->num_group_names = of_get_child_count(np);
 	if (func->num_group_names == 0) {
-		dev_err(info->dev, "no groups defined in %s\n", np->full_name);
+		dev_err(info->dev, "no groups defined in %pOF\n", np);
 		return -EINVAL;
 	}
 	func->group_names = devm_kcalloc(info->dev, func->num_group_names,
@@ -862,6 +748,9 @@ int imx_pinctrl_probe(struct platform_device *pdev,
 	imx_pinctrl_desc->custom_params = info->custom_params;
 	imx_pinctrl_desc->num_custom_params = info->num_custom_params;
 
+	/* platform specific callback */
+	imx_pmx_ops.gpio_set_direction = info->gpio_set_direction;
+
 	mutex_init(&info->mutex);
 
 	ipctl->info = info;
diff --git a/drivers/pinctrl/freescale/pinctrl-imx.h b/drivers/pinctrl/freescale/pinctrl-imx.h
index 880bba7fd1ab..5aa22b52c1d4 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx.h
+++ b/drivers/pinctrl/freescale/pinctrl-imx.h
@@ -16,9 +16,12 @@
 #define __DRIVERS_PINCTRL_IMX_H
 
 #include <linux/pinctrl/pinconf-generic.h>
+#include <linux/pinctrl/pinmux.h>
 
 struct platform_device;
 
+extern struct pinmux_ops imx_pmx_ops;
+
 /**
  * struct imx_pin - describes a single i.MX pin
  * @pin: the pin_id of this pin
@@ -76,6 +79,23 @@ struct imx_pinctrl_soc_info {
 	unsigned int num_decodes;
 	void (*fixup)(unsigned long *configs, unsigned int num_configs,
 		      u32 *raw_config);
+
+	int (*gpio_set_direction)(struct pinctrl_dev *pctldev,
+				  struct pinctrl_gpio_range *range,
+				  unsigned offset,
+				  bool input);
+};
+
+/**
+ * @dev: a pointer back to containing device
+ * @base: the offset to the controller in virtual memory
+ */
+struct imx_pinctrl {
+	struct device *dev;
+	struct pinctrl_dev *pctl;
+	void __iomem *base;
+	void __iomem *input_sel_base;
+	struct imx_pinctrl_soc_info *info;
 };
 
 #define IMX_CFG_PARAMS_DECODE(p, m, o) \
diff --git a/drivers/pinctrl/freescale/pinctrl-imx23.c b/drivers/pinctrl/freescale/pinctrl-imx23.c
index 89b4f160138f..c9405685971b 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx23.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx23.c
@@ -257,7 +257,7 @@ static const struct pinctrl_pin_desc imx23_pins[] = {
 	MXS_PINCTRL_PIN(EMI_CLKN),
 };
 
-static struct mxs_regs imx23_regs = {
+static const struct mxs_regs imx23_regs = {
 	.muxsel = 0x100,
 	.drive = 0x200,
 	.pull = 0x400,
diff --git a/drivers/pinctrl/freescale/pinctrl-imx28.c b/drivers/pinctrl/freescale/pinctrl-imx28.c
index 295236dfb0bc..87deb9ec938a 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx28.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx28.c
@@ -373,7 +373,7 @@ static const struct pinctrl_pin_desc imx28_pins[] = {
 	MXS_PINCTRL_PIN(EMI_CKE),
 };
 
-static struct mxs_regs imx28_regs = {
+static const struct mxs_regs imx28_regs = {
 	.muxsel = 0x100,
 	.drive = 0x300,
 	.pull = 0x600,
diff --git a/drivers/pinctrl/freescale/pinctrl-imx7ulp.c b/drivers/pinctrl/freescale/pinctrl-imx7ulp.c
new file mode 100644
index 000000000000..b7bebb292f37
--- /dev/null
+++ b/drivers/pinctrl/freescale/pinctrl-imx7ulp.c
@@ -0,0 +1,364 @@
+/*
+ * Copyright (C) 2016 Freescale Semiconductor, Inc.
+ * Copyright (C) 2017 NXP
+ *
+ * Author: Dong Aisheng <aisheng.dong@nxp.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/pinctrl/pinctrl.h>
+
+#include "pinctrl-imx.h"
+
+enum imx7ulp_pads {
+	IMX7ULP_PAD_PTC0 = 0,
+	IMX7ULP_PAD_PTC1,
+	IMX7ULP_PAD_PTC2,
+	IMX7ULP_PAD_PTC3,
+	IMX7ULP_PAD_PTC4,
+	IMX7ULP_PAD_PTC5,
+	IMX7ULP_PAD_PTC6,
+	IMX7ULP_PAD_PTC7,
+	IMX7ULP_PAD_PTC8,
+	IMX7ULP_PAD_PTC9,
+	IMX7ULP_PAD_PTC10,
+	IMX7ULP_PAD_PTC11,
+	IMX7ULP_PAD_PTC12,
+	IMX7ULP_PAD_PTC13,
+	IMX7ULP_PAD_PTC14,
+	IMX7ULP_PAD_PTC15,
+	IMX7ULP_PAD_PTC16,
+	IMX7ULP_PAD_PTC17,
+	IMX7ULP_PAD_PTC18,
+	IMX7ULP_PAD_PTC19,
+	IMX7ULP_PAD_RESERVE0,
+	IMX7ULP_PAD_RESERVE1,
+	IMX7ULP_PAD_RESERVE2,
+	IMX7ULP_PAD_RESERVE3,
+	IMX7ULP_PAD_RESERVE4,
+	IMX7ULP_PAD_RESERVE5,
+	IMX7ULP_PAD_RESERVE6,
+	IMX7ULP_PAD_RESERVE7,
+	IMX7ULP_PAD_RESERVE8,
+	IMX7ULP_PAD_RESERVE9,
+	IMX7ULP_PAD_RESERVE10,
+	IMX7ULP_PAD_RESERVE11,
+	IMX7ULP_PAD_PTD0,
+	IMX7ULP_PAD_PTD1,
+	IMX7ULP_PAD_PTD2,
+	IMX7ULP_PAD_PTD3,
+	IMX7ULP_PAD_PTD4,
+	IMX7ULP_PAD_PTD5,
+	IMX7ULP_PAD_PTD6,
+	IMX7ULP_PAD_PTD7,
+	IMX7ULP_PAD_PTD8,
+	IMX7ULP_PAD_PTD9,
+	IMX7ULP_PAD_PTD10,
+	IMX7ULP_PAD_PTD11,
+	IMX7ULP_PAD_RESERVE12,
+	IMX7ULP_PAD_RESERVE13,
+	IMX7ULP_PAD_RESERVE14,
+	IMX7ULP_PAD_RESERVE15,
+	IMX7ULP_PAD_RESERVE16,
+	IMX7ULP_PAD_RESERVE17,
+	IMX7ULP_PAD_RESERVE18,
+	IMX7ULP_PAD_RESERVE19,
+	IMX7ULP_PAD_RESERVE20,
+	IMX7ULP_PAD_RESERVE21,
+	IMX7ULP_PAD_RESERVE22,
+	IMX7ULP_PAD_RESERVE23,
+	IMX7ULP_PAD_RESERVE24,
+	IMX7ULP_PAD_RESERVE25,
+	IMX7ULP_PAD_RESERVE26,
+	IMX7ULP_PAD_RESERVE27,
+	IMX7ULP_PAD_RESERVE28,
+	IMX7ULP_PAD_RESERVE29,
+	IMX7ULP_PAD_RESERVE30,
+	IMX7ULP_PAD_RESERVE31,
+	IMX7ULP_PAD_PTE0,
+	IMX7ULP_PAD_PTE1,
+	IMX7ULP_PAD_PTE2,
+	IMX7ULP_PAD_PTE3,
+	IMX7ULP_PAD_PTE4,
+	IMX7ULP_PAD_PTE5,
+	IMX7ULP_PAD_PTE6,
+	IMX7ULP_PAD_PTE7,
+	IMX7ULP_PAD_PTE8,
+	IMX7ULP_PAD_PTE9,
+	IMX7ULP_PAD_PTE10,
+	IMX7ULP_PAD_PTE11,
+	IMX7ULP_PAD_PTE12,
+	IMX7ULP_PAD_PTE13,
+	IMX7ULP_PAD_PTE14,
+	IMX7ULP_PAD_PTE15,
+	IMX7ULP_PAD_RESERVE32,
+	IMX7ULP_PAD_RESERVE33,
+	IMX7ULP_PAD_RESERVE34,
+	IMX7ULP_PAD_RESERVE35,
+	IMX7ULP_PAD_RESERVE36,
+	IMX7ULP_PAD_RESERVE37,
+	IMX7ULP_PAD_RESERVE38,
+	IMX7ULP_PAD_RESERVE39,
+	IMX7ULP_PAD_RESERVE40,
+	IMX7ULP_PAD_RESERVE41,
+	IMX7ULP_PAD_RESERVE42,
+	IMX7ULP_PAD_RESERVE43,
+	IMX7ULP_PAD_RESERVE44,
+	IMX7ULP_PAD_RESERVE45,
+	IMX7ULP_PAD_RESERVE46,
+	IMX7ULP_PAD_RESERVE47,
+	IMX7ULP_PAD_PTF0,
+	IMX7ULP_PAD_PTF1,
+	IMX7ULP_PAD_PTF2,
+	IMX7ULP_PAD_PTF3,
+	IMX7ULP_PAD_PTF4,
+	IMX7ULP_PAD_PTF5,
+	IMX7ULP_PAD_PTF6,
+	IMX7ULP_PAD_PTF7,
+	IMX7ULP_PAD_PTF8,
+	IMX7ULP_PAD_PTF9,
+	IMX7ULP_PAD_PTF10,
+	IMX7ULP_PAD_PTF11,
+	IMX7ULP_PAD_PTF12,
+	IMX7ULP_PAD_PTF13,
+	IMX7ULP_PAD_PTF14,
+	IMX7ULP_PAD_PTF15,
+	IMX7ULP_PAD_PTF16,
+	IMX7ULP_PAD_PTF17,
+	IMX7ULP_PAD_PTF18,
+	IMX7ULP_PAD_PTF19,
+};
+
+/* Pad names for the pinmux subsystem */
+static const struct pinctrl_pin_desc imx7ulp_pinctrl_pads[] = {
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTC0),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTC1),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTC2),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTC3),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTC4),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTC5),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTC6),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTC7),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTC8),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTC9),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTC10),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTC11),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTC12),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTC13),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTC14),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTC15),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTC16),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTC17),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTC18),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTC19),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE0),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE1),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE2),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE3),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE4),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE5),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE6),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE7),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE8),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE9),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE10),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE11),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTD0),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTD1),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTD2),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTD3),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTD4),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTD5),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTD6),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTD7),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTD8),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTD9),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTD10),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTD11),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE12),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE13),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE14),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE15),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE16),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE17),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE18),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE19),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE20),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE21),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE22),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE23),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE24),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE25),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE26),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE27),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE28),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE29),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE30),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE31),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTE0),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTE1),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTE2),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTE3),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTE4),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTE5),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTE6),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTE7),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTE8),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTE9),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTE10),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTE11),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTE12),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTE13),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTE14),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTE15),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE32),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE33),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE34),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE35),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE36),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE37),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE38),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE39),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE40),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE41),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE42),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE43),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE44),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE45),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE46),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_RESERVE47),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTF0),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTF1),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTF2),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTF3),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTF4),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTF5),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTF6),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTF7),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTF8),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTF9),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTF10),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTF11),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTF12),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTF13),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTF14),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTF15),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTF16),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTF17),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTF18),
+	IMX_PINCTRL_PIN(IMX7ULP_PAD_PTF19),
+};
+
+#define BM_OBE_ENABLED		BIT(17)
+#define BM_IBE_ENABLED		BIT(16)
+#define BM_LK_ENABLED		BIT(15)
+#define BM_MUX_MODE		0xf00
+#define BP_MUX_MODE		8
+#define BM_PULL_ENABLED		BIT(1)
+
+struct imx_cfg_params_decode imx7ulp_cfg_decodes[] = {
+	IMX_CFG_PARAMS_DECODE(PIN_CONFIG_DRIVE_STRENGTH, 		BIT(6), 6),
+	IMX_CFG_PARAMS_DECODE(PIN_CONFIG_DRIVE_PUSH_PULL,		BIT(5), 5),
+	IMX_CFG_PARAMS_DECODE(PIN_CONFIG_SLEW_RATE,			BIT(2), 2),
+	IMX_CFG_PARAMS_DECODE(PIN_CONFIG_BIAS_DISABLE,			BIT(1), 1),
+	IMX_CFG_PARAMS_DECODE(PIN_CONFIG_BIAS_PULL_UP,			BIT(0), 0),
+
+	IMX_CFG_PARAMS_DECODE_INVERT(PIN_CONFIG_DRIVE_OPEN_DRAIN,	BIT(5), 5),
+	IMX_CFG_PARAMS_DECODE_INVERT(PIN_CONFIG_BIAS_PULL_DOWN,		BIT(0), 0),
+};
+
+static void imx7ulp_cfg_params_fixup(unsigned long *configs,
+				    unsigned int num_configs,
+				    u32 *raw_config)
+{
+	enum pin_config_param param;
+	u32 param_val;
+	int i;
+
+	/* lock field disabled */
+	*raw_config &= ~BM_LK_ENABLED;
+
+	for (i = 0; i < num_configs; i++) {
+		param = pinconf_to_config_param(configs[i]);
+		param_val = pinconf_to_config_argument(configs[i]);
+
+		if ((param == PIN_CONFIG_BIAS_PULL_UP) ||
+		    (param == PIN_CONFIG_BIAS_PULL_DOWN)) {
+			/* pull enabled */
+			*raw_config |= BM_PULL_ENABLED;
+
+			return;
+		}
+	}
+}
+
+static int imx7ulp_pmx_gpio_set_direction(struct pinctrl_dev *pctldev,
+					  struct pinctrl_gpio_range *range,
+					  unsigned offset, bool input)
+{
+	struct imx_pinctrl *ipctl = pinctrl_dev_get_drvdata(pctldev);
+	struct imx_pinctrl_soc_info *info = ipctl->info;
+	const struct imx_pin_reg *pin_reg;
+	u32 reg;
+
+	pin_reg = &info->pin_regs[offset];
+	if (pin_reg->mux_reg == -1)
+		return -EINVAL;
+
+	reg = readl(ipctl->base + pin_reg->mux_reg);
+	if (input)
+		reg = (reg & ~BM_OBE_ENABLED) | BM_IBE_ENABLED;
+	else
+		reg = (reg & ~BM_IBE_ENABLED) | BM_OBE_ENABLED;
+	writel(reg, ipctl->base + pin_reg->mux_reg);
+
+	return 0;
+}
+
+static struct imx_pinctrl_soc_info imx7ulp_pinctrl_info = {
+	.pins = imx7ulp_pinctrl_pads,
+	.npins = ARRAY_SIZE(imx7ulp_pinctrl_pads),
+	.flags = ZERO_OFFSET_VALID | SHARE_MUX_CONF_REG,
+	.gpio_set_direction = imx7ulp_pmx_gpio_set_direction,
+	.mux_mask = BM_MUX_MODE,
+	.mux_shift = BP_MUX_MODE,
+	.generic_pinconf = true,
+	.decodes = imx7ulp_cfg_decodes,
+	.num_decodes = ARRAY_SIZE(imx7ulp_cfg_decodes),
+	.fixup = imx7ulp_cfg_params_fixup,
+};
+
+static const struct of_device_id imx7ulp_pinctrl_of_match[] = {
+	{ .compatible = "fsl,imx7ulp-iomuxc1", },
+	{ /* sentinel */ }
+};
+
+static int imx7ulp_pinctrl_probe(struct platform_device *pdev)
+{
+	return imx_pinctrl_probe(pdev, &imx7ulp_pinctrl_info);
+}
+
+static struct platform_driver imx7ulp_pinctrl_driver = {
+	.driver = {
+		.name = "imx7ulp-pinctrl",
+		.of_match_table = of_match_ptr(imx7ulp_pinctrl_of_match),
+		.suppress_bind_attrs = true,
+	},
+	.probe = imx7ulp_pinctrl_probe,
+};
+
+static int __init imx7ulp_pinctrl_init(void)
+{
+	return platform_driver_register(&imx7ulp_pinctrl_driver);
+}
+arch_initcall(imx7ulp_pinctrl_init);
diff --git a/drivers/pinctrl/freescale/pinctrl-vf610.c b/drivers/pinctrl/freescale/pinctrl-vf610.c
index 3bd85564d1e4..ac18bb6d6d5e 100644
--- a/drivers/pinctrl/freescale/pinctrl-vf610.c
+++ b/drivers/pinctrl/freescale/pinctrl-vf610.c
@@ -295,10 +295,35 @@ static const struct pinctrl_pin_desc vf610_pinctrl_pads[] = {
 	IMX_PINCTRL_PIN(VF610_PAD_PTA7),
 };
 
+static int vf610_pmx_gpio_set_direction(struct pinctrl_dev *pctldev,
+					struct pinctrl_gpio_range *range,
+					unsigned offset, bool input)
+{
+	struct imx_pinctrl *ipctl = pinctrl_dev_get_drvdata(pctldev);
+	struct imx_pinctrl_soc_info *info = ipctl->info;
+	const struct imx_pin_reg *pin_reg;
+	u32 reg;
+
+	pin_reg = &info->pin_regs[offset];
+	if (pin_reg->mux_reg == -1)
+		return -EINVAL;
+
+	/* IBE always enabled allows us to read the value "on the wire" */
+	reg = readl(ipctl->base + pin_reg->mux_reg);
+	if (input)
+		reg &= ~0x2;
+	else
+		reg |= 0x2;
+	writel(reg, ipctl->base + pin_reg->mux_reg);
+
+	return 0;
+}
+
 static struct imx_pinctrl_soc_info vf610_pinctrl_info = {
 	.pins = vf610_pinctrl_pads,
 	.npins = ARRAY_SIZE(vf610_pinctrl_pads),
 	.flags = SHARE_MUX_CONF_REG | ZERO_OFFSET_VALID,
+	.gpio_set_direction = vf610_pmx_gpio_set_direction,
 	.mux_mask = 0x700000,
 	.mux_shift = 20,
 };
diff --git a/drivers/pinctrl/intel/Kconfig b/drivers/pinctrl/intel/Kconfig
index b82d6ff3116f..f30720a752f3 100644
--- a/drivers/pinctrl/intel/Kconfig
+++ b/drivers/pinctrl/intel/Kconfig
@@ -1,6 +1,7 @@
 #
 # Intel pin control drivers
 #
+if (X86 || COMPILE_TEST)
 
 config PINCTRL_BAYTRAIL
 	bool "Intel Baytrail GPIO pin control"
@@ -64,6 +65,14 @@ config PINCTRL_CANNONLAKE
 	  This pinctrl driver provides an interface that allows configuring
 	  of Intel Cannon Lake PCH pins and using them as GPIOs.
 
+config PINCTRL_DENVERTON
+	tristate "Intel Denverton pinctrl and GPIO driver"
+	depends on ACPI
+	select PINCTRL_INTEL
+	help
+	  This pinctrl driver provides an interface that allows configuring
+	  of Intel Denverton SoC pins and using them as GPIOs.
+
 config PINCTRL_GEMINILAKE
 	tristate "Intel Gemini Lake SoC pinctrl and GPIO driver"
 	depends on ACPI
@@ -72,6 +81,14 @@ config PINCTRL_GEMINILAKE
 	  This pinctrl driver provides an interface that allows configuring
 	  of Intel Gemini Lake SoC pins and using them as GPIOs.
 
+config PINCTRL_LEWISBURG
+	tristate "Intel Lewisburg pinctrl and GPIO driver"
+	depends on ACPI
+	select PINCTRL_INTEL
+	help
+	  This pinctrl driver provides an interface that allows configuring
+	  of Intel Lewisburg pins and using them as GPIOs.
+
 config PINCTRL_SUNRISEPOINT
 	tristate "Intel Sunrisepoint pinctrl and GPIO driver"
 	depends on ACPI
@@ -80,3 +97,5 @@ config PINCTRL_SUNRISEPOINT
 	  Sunrisepoint is the PCH of Intel Skylake. This pinctrl driver
 	  provides an interface that allows configuring of PCH pins and
 	  using them as GPIOs.
+
+endif
diff --git a/drivers/pinctrl/intel/Makefile b/drivers/pinctrl/intel/Makefile
index 81df3cf408e3..c12874da5992 100644
--- a/drivers/pinctrl/intel/Makefile
+++ b/drivers/pinctrl/intel/Makefile
@@ -6,5 +6,7 @@ obj-$(CONFIG_PINCTRL_MERRIFIELD)	+= pinctrl-merrifield.o
 obj-$(CONFIG_PINCTRL_INTEL)		+= pinctrl-intel.o
 obj-$(CONFIG_PINCTRL_BROXTON)		+= pinctrl-broxton.o
 obj-$(CONFIG_PINCTRL_CANNONLAKE)	+= pinctrl-cannonlake.o
+obj-$(CONFIG_PINCTRL_DENVERTON)		+= pinctrl-denverton.o
 obj-$(CONFIG_PINCTRL_GEMINILAKE)	+= pinctrl-geminilake.o
+obj-$(CONFIG_PINCTRL_LEWISBURG)		+= pinctrl-lewisburg.o
 obj-$(CONFIG_PINCTRL_SUNRISEPOINT)	+= pinctrl-sunrisepoint.o
diff --git a/drivers/pinctrl/intel/pinctrl-baytrail.c b/drivers/pinctrl/intel/pinctrl-baytrail.c
index fa3c5758ac67..0f3a02495aeb 100644
--- a/drivers/pinctrl/intel/pinctrl-baytrail.c
+++ b/drivers/pinctrl/intel/pinctrl-baytrail.c
@@ -981,12 +981,12 @@ static int byt_gpio_request_enable(struct pinctrl_dev *pctl_dev,
 	 */
 	value = readl(reg) & BYT_PIN_MUX;
 	gpio_mux = byt_get_gpio_mux(vg, offset);
-	if (WARN_ON(gpio_mux != value)) {
+	if (gpio_mux != value) {
 		value = readl(reg) & ~BYT_PIN_MUX;
 		value |= gpio_mux;
 		writel(value, reg);
 
-		dev_warn(&vg->pdev->dev,
+		dev_warn(&vg->pdev->dev, FW_BUG
 			 "pin %u forcibly re-configured as GPIO\n", offset);
 	}
 
diff --git a/drivers/pinctrl/intel/pinctrl-cannonlake.c b/drivers/pinctrl/intel/pinctrl-cannonlake.c
index 3bc609b67dc2..e130599be571 100644
--- a/drivers/pinctrl/intel/pinctrl-cannonlake.c
+++ b/drivers/pinctrl/intel/pinctrl-cannonlake.c
@@ -2,7 +2,8 @@
  * Intel Cannon Lake PCH pinctrl/GPIO driver
  *
  * Copyright (C) 2017, Intel Corporation
- * Author: Mika Westerberg <mika.westerberg@linux.intel.com>
+ * Authors: Andy Shevchenko <andriy.shevchenko@linux.intel.com>
+ *          Mika Westerberg <mika.westerberg@linux.intel.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -42,6 +43,426 @@
 		.ngpps = ARRAY_SIZE(g),			\
 	}
 
+/* Cannon Lake-H */
+static const struct pinctrl_pin_desc cnlh_pins[] = {
+	/* GPP_A */
+	PINCTRL_PIN(0, "RCINB"),
+	PINCTRL_PIN(1, "LAD_0"),
+	PINCTRL_PIN(2, "LAD_1"),
+	PINCTRL_PIN(3, "LAD_2"),
+	PINCTRL_PIN(4, "LAD_3"),
+	PINCTRL_PIN(5, "LFRAMEB"),
+	PINCTRL_PIN(6, "SERIRQ"),
+	PINCTRL_PIN(7, "PIRQAB"),
+	PINCTRL_PIN(8, "CLKRUNB"),
+	PINCTRL_PIN(9, "CLKOUT_LPC_0"),
+	PINCTRL_PIN(10, "CLKOUT_LPC_1"),
+	PINCTRL_PIN(11, "PMEB"),
+	PINCTRL_PIN(12, "BM_BUSYB"),
+	PINCTRL_PIN(13, "SUSWARNB_SUSPWRDNACK"),
+	PINCTRL_PIN(14, "SUS_STATB"),
+	PINCTRL_PIN(15, "SUSACKB"),
+	PINCTRL_PIN(16, "CLKOUT_48"),
+	PINCTRL_PIN(17, "SD_VDD1_PWR_EN_B"),
+	PINCTRL_PIN(18, "ISH_GP_0"),
+	PINCTRL_PIN(19, "ISH_GP_1"),
+	PINCTRL_PIN(20, "ISH_GP_2"),
+	PINCTRL_PIN(21, "ISH_GP_3"),
+	PINCTRL_PIN(22, "ISH_GP_4"),
+	PINCTRL_PIN(23, "ISH_GP_5"),
+	PINCTRL_PIN(24, "ESPI_CLK_LOOPBK"),
+	/* GPP_B */
+	PINCTRL_PIN(25, "GSPI0_CS1B"),
+	PINCTRL_PIN(26, "GSPI1_CS1B"),
+	PINCTRL_PIN(27, "VRALERTB"),
+	PINCTRL_PIN(28, "CPU_GP_2"),
+	PINCTRL_PIN(29, "CPU_GP_3"),
+	PINCTRL_PIN(30, "SRCCLKREQB_0"),
+	PINCTRL_PIN(31, "SRCCLKREQB_1"),
+	PINCTRL_PIN(32, "SRCCLKREQB_2"),
+	PINCTRL_PIN(33, "SRCCLKREQB_3"),
+	PINCTRL_PIN(34, "SRCCLKREQB_4"),
+	PINCTRL_PIN(35, "SRCCLKREQB_5"),
+	PINCTRL_PIN(36, "SSP_MCLK"),
+	PINCTRL_PIN(37, "SLP_S0B"),
+	PINCTRL_PIN(38, "PLTRSTB"),
+	PINCTRL_PIN(39, "SPKR"),
+	PINCTRL_PIN(40, "GSPI0_CS0B"),
+	PINCTRL_PIN(41, "GSPI0_CLK"),
+	PINCTRL_PIN(42, "GSPI0_MISO"),
+	PINCTRL_PIN(43, "GSPI0_MOSI"),
+	PINCTRL_PIN(44, "GSPI1_CS0B"),
+	PINCTRL_PIN(45, "GSPI1_CLK"),
+	PINCTRL_PIN(46, "GSPI1_MISO"),
+	PINCTRL_PIN(47, "GSPI1_MOSI"),
+	PINCTRL_PIN(48, "SML1ALERTB"),
+	PINCTRL_PIN(49, "GSPI0_CLK_LOOPBK"),
+	PINCTRL_PIN(50, "GSPI1_CLK_LOOPBK"),
+	/* GPP_C */
+	PINCTRL_PIN(51, "SMBCLK"),
+	PINCTRL_PIN(52, "SMBDATA"),
+	PINCTRL_PIN(53, "SMBALERTB"),
+	PINCTRL_PIN(54, "SML0CLK"),
+	PINCTRL_PIN(55, "SML0DATA"),
+	PINCTRL_PIN(56, "SML0ALERTB"),
+	PINCTRL_PIN(57, "SML1CLK"),
+	PINCTRL_PIN(58, "SML1DATA"),
+	PINCTRL_PIN(59, "UART0_RXD"),
+	PINCTRL_PIN(60, "UART0_TXD"),
+	PINCTRL_PIN(61, "UART0_RTSB"),
+	PINCTRL_PIN(62, "UART0_CTSB"),
+	PINCTRL_PIN(63, "UART1_RXD"),
+	PINCTRL_PIN(64, "UART1_TXD"),
+	PINCTRL_PIN(65, "UART1_RTSB"),
+	PINCTRL_PIN(66, "UART1_CTSB"),
+	PINCTRL_PIN(67, "I2C0_SDA"),
+	PINCTRL_PIN(68, "I2C0_SCL"),
+	PINCTRL_PIN(69, "I2C1_SDA"),
+	PINCTRL_PIN(70, "I2C1_SCL"),
+	PINCTRL_PIN(71, "UART2_RXD"),
+	PINCTRL_PIN(72, "UART2_TXD"),
+	PINCTRL_PIN(73, "UART2_RTSB"),
+	PINCTRL_PIN(74, "UART2_CTSB"),
+	/* GPP_D */
+	PINCTRL_PIN(75, "SPI1_CSB"),
+	PINCTRL_PIN(76, "SPI1_CLK"),
+	PINCTRL_PIN(77, "SPI1_MISO_IO_1"),
+	PINCTRL_PIN(78, "SPI1_MOSI_IO_0"),
+	PINCTRL_PIN(79, "ISH_I2C2_SDA"),
+	PINCTRL_PIN(80, "SSP2_SFRM"),
+	PINCTRL_PIN(81, "SSP2_TXD"),
+	PINCTRL_PIN(82, "SSP2_RXD"),
+	PINCTRL_PIN(83, "SSP2_SCLK"),
+	PINCTRL_PIN(84, "ISH_SPI_CSB"),
+	PINCTRL_PIN(85, "ISH_SPI_CLK"),
+	PINCTRL_PIN(86, "ISH_SPI_MISO"),
+	PINCTRL_PIN(87, "ISH_SPI_MOSI"),
+	PINCTRL_PIN(88, "ISH_UART0_RXD"),
+	PINCTRL_PIN(89, "ISH_UART0_TXD"),
+	PINCTRL_PIN(90, "ISH_UART0_RTSB"),
+	PINCTRL_PIN(91, "ISH_UART0_CTSB"),
+	PINCTRL_PIN(92, "DMIC_CLK_1"),
+	PINCTRL_PIN(93, "DMIC_DATA_1"),
+	PINCTRL_PIN(94, "DMIC_CLK_0"),
+	PINCTRL_PIN(95, "DMIC_DATA_0"),
+	PINCTRL_PIN(96, "SPI1_IO_2"),
+	PINCTRL_PIN(97, "SPI1_IO_3"),
+	PINCTRL_PIN(98, "ISH_I2C2_SCL"),
+	/* GPP_G */
+	PINCTRL_PIN(99, "SD3_CMD"),
+	PINCTRL_PIN(100, "SD3_D0"),
+	PINCTRL_PIN(101, "SD3_D1"),
+	PINCTRL_PIN(102, "SD3_D2"),
+	PINCTRL_PIN(103, "SD3_D3"),
+	PINCTRL_PIN(104, "SD3_CDB"),
+	PINCTRL_PIN(105, "SD3_CLK"),
+	PINCTRL_PIN(106, "SD3_WP"),
+	/* AZA */
+	PINCTRL_PIN(107, "HDA_BCLK"),
+	PINCTRL_PIN(108, "HDA_RSTB"),
+	PINCTRL_PIN(109, "HDA_SYNC"),
+	PINCTRL_PIN(110, "HDA_SDO"),
+	PINCTRL_PIN(111, "HDA_SDI_0"),
+	PINCTRL_PIN(112, "HDA_SDI_1"),
+	PINCTRL_PIN(113, "SSP1_SFRM"),
+	PINCTRL_PIN(114, "SSP1_TXD"),
+	/* vGPIO */
+	PINCTRL_PIN(115, "CNV_BTEN"),
+	PINCTRL_PIN(116, "CNV_GNEN"),
+	PINCTRL_PIN(117, "CNV_WFEN"),
+	PINCTRL_PIN(118, "CNV_WCEN"),
+	PINCTRL_PIN(119, "CNV_BT_HOST_WAKEB"),
+	PINCTRL_PIN(120, "vCNV_GNSS_HOST_WAKEB"),
+	PINCTRL_PIN(121, "vSD3_CD_B"),
+	PINCTRL_PIN(122, "CNV_BT_IF_SELECT"),
+	PINCTRL_PIN(123, "vCNV_BT_UART_TXD"),
+	PINCTRL_PIN(124, "vCNV_BT_UART_RXD"),
+	PINCTRL_PIN(125, "vCNV_BT_UART_CTS_B"),
+	PINCTRL_PIN(126, "vCNV_BT_UART_RTS_B"),
+	PINCTRL_PIN(127, "vCNV_MFUART1_TXD"),
+	PINCTRL_PIN(128, "vCNV_MFUART1_RXD"),
+	PINCTRL_PIN(129, "vCNV_MFUART1_CTS_B"),
+	PINCTRL_PIN(130, "vCNV_MFUART1_RTS_B"),
+	PINCTRL_PIN(131, "vCNV_GNSS_UART_TXD"),
+	PINCTRL_PIN(132, "vCNV_GNSS_UART_RXD"),
+	PINCTRL_PIN(133, "vCNV_GNSS_UART_CTS_B"),
+	PINCTRL_PIN(134, "vCNV_GNSS_UART_RTS_B"),
+	PINCTRL_PIN(135, "vUART0_TXD"),
+	PINCTRL_PIN(136, "vUART0_RXD"),
+	PINCTRL_PIN(137, "vUART0_CTS_B"),
+	PINCTRL_PIN(138, "vUART0_RTSB"),
+	PINCTRL_PIN(139, "vISH_UART0_TXD"),
+	PINCTRL_PIN(140, "vISH_UART0_RXD"),
+	PINCTRL_PIN(141, "vISH_UART0_CTS_B"),
+	PINCTRL_PIN(142, "vISH_UART0_RTSB"),
+	PINCTRL_PIN(143, "vISH_UART1_TXD"),
+	PINCTRL_PIN(144, "vISH_UART1_RXD"),
+	PINCTRL_PIN(145, "vISH_UART1_CTS_B"),
+	PINCTRL_PIN(146, "vISH_UART1_RTS_B"),
+	PINCTRL_PIN(147, "vCNV_BT_I2S_BCLK"),
+	PINCTRL_PIN(148, "vCNV_BT_I2S_WS_SYNC"),
+	PINCTRL_PIN(149, "vCNV_BT_I2S_SDO"),
+	PINCTRL_PIN(150, "vCNV_BT_I2S_SDI"),
+	PINCTRL_PIN(151, "vSSP2_SCLK"),
+	PINCTRL_PIN(152, "vSSP2_SFRM"),
+	PINCTRL_PIN(153, "vSSP2_TXD"),
+	PINCTRL_PIN(154, "vSSP2_RXD"),
+	/* GPP_K */
+	PINCTRL_PIN(155, "FAN_TACH_0"),
+	PINCTRL_PIN(156, "FAN_TACH_1"),
+	PINCTRL_PIN(157, "FAN_TACH_2"),
+	PINCTRL_PIN(158, "FAN_TACH_3"),
+	PINCTRL_PIN(159, "FAN_TACH_4"),
+	PINCTRL_PIN(160, "FAN_TACH_5"),
+	PINCTRL_PIN(161, "FAN_TACH_6"),
+	PINCTRL_PIN(162, "FAN_TACH_7"),
+	PINCTRL_PIN(163, "FAN_PWM_0"),
+	PINCTRL_PIN(164, "FAN_PWM_1"),
+	PINCTRL_PIN(165, "FAN_PWM_2"),
+	PINCTRL_PIN(166, "FAN_PWM_3"),
+	PINCTRL_PIN(167, "GSXDOUT"),
+	PINCTRL_PIN(168, "GSXSLOAD"),
+	PINCTRL_PIN(169, "GSXDIN"),
+	PINCTRL_PIN(170, "GSXSRESETB"),
+	PINCTRL_PIN(171, "GSXCLK"),
+	PINCTRL_PIN(172, "ADR_COMPLETE"),
+	PINCTRL_PIN(173, "NMIB"),
+	PINCTRL_PIN(174, "SMIB"),
+	PINCTRL_PIN(175, "CORE_VID_0"),
+	PINCTRL_PIN(176, "CORE_VID_1"),
+	PINCTRL_PIN(177, "IMGCLKOUT_0"),
+	PINCTRL_PIN(178, "IMGCLKOUT_1"),
+	/* GPP_H */
+	PINCTRL_PIN(179, "SRCCLKREQB_6"),
+	PINCTRL_PIN(180, "SRCCLKREQB_7"),
+	PINCTRL_PIN(181, "SRCCLKREQB_8"),
+	PINCTRL_PIN(182, "SRCCLKREQB_9"),
+	PINCTRL_PIN(183, "SRCCLKREQB_10"),
+	PINCTRL_PIN(184, "SRCCLKREQB_11"),
+	PINCTRL_PIN(185, "SRCCLKREQB_12"),
+	PINCTRL_PIN(186, "SRCCLKREQB_13"),
+	PINCTRL_PIN(187, "SRCCLKREQB_14"),
+	PINCTRL_PIN(188, "SRCCLKREQB_15"),
+	PINCTRL_PIN(189, "SML2CLK"),
+	PINCTRL_PIN(190, "SML2DATA"),
+	PINCTRL_PIN(191, "SML2ALERTB"),
+	PINCTRL_PIN(192, "SML3CLK"),
+	PINCTRL_PIN(193, "SML3DATA"),
+	PINCTRL_PIN(194, "SML3ALERTB"),
+	PINCTRL_PIN(195, "SML4CLK"),
+	PINCTRL_PIN(196, "SML4DATA"),
+	PINCTRL_PIN(197, "SML4ALERTB"),
+	PINCTRL_PIN(198, "ISH_I2C0_SDA"),
+	PINCTRL_PIN(199, "ISH_I2C0_SCL"),
+	PINCTRL_PIN(200, "ISH_I2C1_SDA"),
+	PINCTRL_PIN(201, "ISH_I2C1_SCL"),
+	PINCTRL_PIN(202, "TIME_SYNC_0"),
+	/* GPP_E */
+	PINCTRL_PIN(203, "SATAXPCIE_0"),
+	PINCTRL_PIN(204, "SATAXPCIE_1"),
+	PINCTRL_PIN(205, "SATAXPCIE_2"),
+	PINCTRL_PIN(206, "CPU_GP_0"),
+	PINCTRL_PIN(207, "SATA_DEVSLP_0"),
+	PINCTRL_PIN(208, "SATA_DEVSLP_1"),
+	PINCTRL_PIN(209, "SATA_DEVSLP_2"),
+	PINCTRL_PIN(210, "CPU_GP_1"),
+	PINCTRL_PIN(211, "SATA_LEDB"),
+	PINCTRL_PIN(212, "USB2_OCB_0"),
+	PINCTRL_PIN(213, "USB2_OCB_1"),
+	PINCTRL_PIN(214, "USB2_OCB_2"),
+	PINCTRL_PIN(215, "USB2_OCB_3"),
+	/* GPP_F */
+	PINCTRL_PIN(216, "SATAXPCIE_3"),
+	PINCTRL_PIN(217, "SATAXPCIE_4"),
+	PINCTRL_PIN(218, "SATAXPCIE_5"),
+	PINCTRL_PIN(219, "SATAXPCIE_6"),
+	PINCTRL_PIN(220, "SATAXPCIE_7"),
+	PINCTRL_PIN(221, "SATA_DEVSLP_3"),
+	PINCTRL_PIN(222, "SATA_DEVSLP_4"),
+	PINCTRL_PIN(223, "SATA_DEVSLP_5"),
+	PINCTRL_PIN(224, "SATA_DEVSLP_6"),
+	PINCTRL_PIN(225, "SATA_DEVSLP_7"),
+	PINCTRL_PIN(226, "SATA_SCLOCK"),
+	PINCTRL_PIN(227, "SATA_SLOAD"),
+	PINCTRL_PIN(228, "SATA_SDATAOUT1"),
+	PINCTRL_PIN(229, "SATA_SDATAOUT0"),
+	PINCTRL_PIN(230, "EXT_PWR_GATEB"),
+	PINCTRL_PIN(231, "USB2_OCB_4"),
+	PINCTRL_PIN(232, "USB2_OCB_5"),
+	PINCTRL_PIN(233, "USB2_OCB_6"),
+	PINCTRL_PIN(234, "USB2_OCB_7"),
+	PINCTRL_PIN(235, "L_VDDEN"),
+	PINCTRL_PIN(236, "L_BKLTEN"),
+	PINCTRL_PIN(237, "L_BKLTCTL"),
+	PINCTRL_PIN(238, "DDPF_CTRLCLK"),
+	PINCTRL_PIN(239, "DDPF_CTRLDATA"),
+	/* SPI */
+	PINCTRL_PIN(240, "SPI0_IO_2"),
+	PINCTRL_PIN(241, "SPI0_IO_3"),
+	PINCTRL_PIN(242, "SPI0_MOSI_IO_0"),
+	PINCTRL_PIN(243, "SPI0_MISO_IO_1"),
+	PINCTRL_PIN(244, "SPI0_TPM_CSB"),
+	PINCTRL_PIN(245, "SPI0_FLASH_0_CSB"),
+	PINCTRL_PIN(246, "SPI0_FLASH_1_CSB"),
+	PINCTRL_PIN(247, "SPI0_CLK"),
+	PINCTRL_PIN(248, "SPI0_CLK_LOOPBK"),
+	/* CPU */
+	PINCTRL_PIN(249, "HDACPU_SDI"),
+	PINCTRL_PIN(250, "HDACPU_SDO"),
+	PINCTRL_PIN(251, "HDACPU_SCLK"),
+	PINCTRL_PIN(252, "PM_SYNC"),
+	PINCTRL_PIN(253, "PECI"),
+	PINCTRL_PIN(254, "CPUPWRGD"),
+	PINCTRL_PIN(255, "THRMTRIPB"),
+	PINCTRL_PIN(256, "PLTRST_CPUB"),
+	PINCTRL_PIN(257, "PM_DOWN"),
+	PINCTRL_PIN(258, "TRIGGER_IN"),
+	PINCTRL_PIN(259, "TRIGGER_OUT"),
+	/* JTAG */
+	PINCTRL_PIN(260, "JTAG_TDO"),
+	PINCTRL_PIN(261, "JTAGX"),
+	PINCTRL_PIN(262, "PRDYB"),
+	PINCTRL_PIN(263, "PREQB"),
+	PINCTRL_PIN(264, "CPU_TRSTB"),
+	PINCTRL_PIN(265, "JTAG_TDI"),
+	PINCTRL_PIN(266, "JTAG_TMS"),
+	PINCTRL_PIN(267, "JTAG_TCK"),
+	PINCTRL_PIN(268, "ITP_PMODE"),
+	/* GPP_I */
+	PINCTRL_PIN(269, "DDSP_HPD_0"),
+	PINCTRL_PIN(270, "DDSP_HPD_1"),
+	PINCTRL_PIN(271, "DDSP_HPD_2"),
+	PINCTRL_PIN(272, "DDSP_HPD_3"),
+	PINCTRL_PIN(273, "EDP_HPD"),
+	PINCTRL_PIN(274, "DDPB_CTRLCLK"),
+	PINCTRL_PIN(275, "DDPB_CTRLDATA"),
+	PINCTRL_PIN(276, "DDPC_CTRLCLK"),
+	PINCTRL_PIN(277, "DDPC_CTRLDATA"),
+	PINCTRL_PIN(278, "DDPD_CTRLCLK"),
+	PINCTRL_PIN(279, "DDPD_CTRLDATA"),
+	PINCTRL_PIN(280, "M2_SKT2_CFG_0"),
+	PINCTRL_PIN(281, "M2_SKT2_CFG_1"),
+	PINCTRL_PIN(282, "M2_SKT2_CFG_2"),
+	PINCTRL_PIN(283, "M2_SKT2_CFG_3"),
+	PINCTRL_PIN(284, "SYS_PWROK"),
+	PINCTRL_PIN(285, "SYS_RESETB"),
+	PINCTRL_PIN(286, "MLK_RSTB"),
+	/* GPP_J */
+	PINCTRL_PIN(287, "CNV_PA_BLANKING"),
+	PINCTRL_PIN(288, "CNV_GNSS_FTA"),
+	PINCTRL_PIN(289, "CNV_GNSS_SYSCK"),
+	PINCTRL_PIN(290, "CNV_RF_RESET_B"),
+	PINCTRL_PIN(291, "CNV_BRI_DT"),
+	PINCTRL_PIN(292, "CNV_BRI_RSP"),
+	PINCTRL_PIN(293, "CNV_RGI_DT"),
+	PINCTRL_PIN(294, "CNV_RGI_RSP"),
+	PINCTRL_PIN(295, "CNV_MFUART2_RXD"),
+	PINCTRL_PIN(296, "CNV_MFUART2_TXD"),
+	PINCTRL_PIN(297, "CNV_MODEM_CLKREQ"),
+	PINCTRL_PIN(298, "A4WP_PRESENT"),
+};
+
+static const struct intel_padgroup cnlh_community0_gpps[] = {
+	CNL_GPP(0, 0, 24),	/* GPP_A */
+	CNL_GPP(1, 25, 50),	/* GPP_B */
+};
+
+static const struct intel_padgroup cnlh_community1_gpps[] = {
+	CNL_GPP(0, 51, 74),	/* GPP_C */
+	CNL_GPP(1, 75, 98),	/* GPP_D */
+	CNL_GPP(2, 99, 106),	/* GPP_G */
+	CNL_GPP(3, 107, 114),	/* AZA */
+	CNL_GPP(4, 115, 146),	/* vGPIO_0 */
+	CNL_GPP(5, 147, 154),	/* vGPIO_1 */
+};
+
+static const struct intel_padgroup cnlh_community3_gpps[] = {
+	CNL_GPP(0, 155, 178),	/* GPP_K */
+	CNL_GPP(1, 179, 202),	/* GPP_H */
+	CNL_GPP(2, 203, 215),	/* GPP_E */
+	CNL_GPP(3, 216, 239),	/* GPP_F */
+	CNL_GPP(4, 240, 248),	/* SPI */
+};
+
+static const struct intel_padgroup cnlh_community4_gpps[] = {
+	CNL_GPP(0, 249, 259),	/* CPU */
+	CNL_GPP(1, 260, 268),	/* JTAG */
+	CNL_GPP(2, 269, 286),	/* GPP_I */
+	CNL_GPP(3, 287, 298),	/* GPP_J */
+};
+
+static const unsigned int cnlh_spi0_pins[] = { 40, 41, 42, 43 };
+static const unsigned int cnlh_spi1_pins[] = { 44, 45, 46, 47 };
+static const unsigned int cnlh_spi2_pins[] = { 84, 85, 86, 87 };
+
+static const unsigned int cnlh_uart0_pins[] = { 59, 60, 61, 62 };
+static const unsigned int cnlh_uart1_pins[] = { 63, 64, 65, 66 };
+static const unsigned int cnlh_uart2_pins[] = { 71, 72, 73, 74 };
+
+static const unsigned int cnlh_i2c0_pins[] = { 67, 68 };
+static const unsigned int cnlh_i2c1_pins[] = { 69, 70 };
+static const unsigned int cnlh_i2c2_pins[] = { 88, 89 };
+static const unsigned int cnlh_i2c3_pins[] = { 79, 98 };
+
+static const struct intel_pingroup cnlh_groups[] = {
+	PIN_GROUP("spi0_grp", cnlh_spi0_pins, 1),
+	PIN_GROUP("spi1_grp", cnlh_spi1_pins, 1),
+	PIN_GROUP("spi2_grp", cnlh_spi2_pins, 3),
+	PIN_GROUP("uart0_grp", cnlh_uart0_pins, 1),
+	PIN_GROUP("uart1_grp", cnlh_uart1_pins, 1),
+	PIN_GROUP("uart2_grp", cnlh_uart2_pins, 1),
+	PIN_GROUP("i2c0_grp", cnlh_i2c0_pins, 1),
+	PIN_GROUP("i2c1_grp", cnlh_i2c1_pins, 1),
+	PIN_GROUP("i2c2_grp", cnlh_i2c2_pins, 3),
+	PIN_GROUP("i2c3_grp", cnlh_i2c3_pins, 2),
+};
+
+static const char * const cnlh_spi0_groups[] = { "spi0_grp" };
+static const char * const cnlh_spi1_groups[] = { "spi1_grp" };
+static const char * const cnlh_spi2_groups[] = { "spi2_grp" };
+static const char * const cnlh_uart0_groups[] = { "uart0_grp" };
+static const char * const cnlh_uart1_groups[] = { "uart1_grp" };
+static const char * const cnlh_uart2_groups[] = { "uart2_grp" };
+static const char * const cnlh_i2c0_groups[] = { "i2c0_grp" };
+static const char * const cnlh_i2c1_groups[] = { "i2c1_grp" };
+static const char * const cnlh_i2c2_groups[] = { "i2c2_grp" };
+static const char * const cnlh_i2c3_groups[] = { "i2c3_grp" };
+
+static const struct intel_function cnlh_functions[] = {
+	FUNCTION("spi0", cnlh_spi0_groups),
+	FUNCTION("spi1", cnlh_spi1_groups),
+	FUNCTION("spi2", cnlh_spi2_groups),
+	FUNCTION("uart0", cnlh_uart0_groups),
+	FUNCTION("uart1", cnlh_uart1_groups),
+	FUNCTION("uart2", cnlh_uart2_groups),
+	FUNCTION("i2c0", cnlh_i2c0_groups),
+	FUNCTION("i2c1", cnlh_i2c1_groups),
+	FUNCTION("i2c2", cnlh_i2c2_groups),
+	FUNCTION("i2c3", cnlh_i2c3_groups),
+};
+
+static const struct intel_community cnlh_communities[] = {
+	CNL_COMMUNITY(0, 0, 50, cnlh_community0_gpps),
+	CNL_COMMUNITY(1, 51, 154, cnlh_community1_gpps),
+	/*
+	 * ACPI MMIO resources are returned in reverse order for
+	 * communities 3 and 4.
+	 */
+	CNL_COMMUNITY(3, 155, 248, cnlh_community3_gpps),
+	CNL_COMMUNITY(2, 249, 298, cnlh_community4_gpps),
+};
+
+static const struct intel_pinctrl_soc_data cnlh_soc_data = {
+	.pins = cnlh_pins,
+	.npins = ARRAY_SIZE(cnlh_pins),
+	.groups = cnlh_groups,
+	.ngroups = ARRAY_SIZE(cnlh_groups),
+	.functions = cnlh_functions,
+	.nfunctions = ARRAY_SIZE(cnlh_functions),
+	.communities = cnlh_communities,
+	.ncommunities = ARRAY_SIZE(cnlh_communities),
+};
+
 /* Cannon Lake-LP */
 static const struct pinctrl_pin_desc cnllp_pins[] = {
 	/* GPP_A */
@@ -403,6 +824,7 @@ static const struct intel_pinctrl_soc_data cnllp_soc_data = {
 };
 
 static const struct acpi_device_id cnl_pinctrl_acpi_match[] = {
+	{ "INT3450", (kernel_ulong_t)&cnlh_soc_data },
 	{ "INT34BB", (kernel_ulong_t)&cnllp_soc_data },
 	{ },
 };
diff --git a/drivers/pinctrl/intel/pinctrl-denverton.c b/drivers/pinctrl/intel/pinctrl-denverton.c
new file mode 100644
index 000000000000..4500880240f2
--- /dev/null
+++ b/drivers/pinctrl/intel/pinctrl-denverton.c
@@ -0,0 +1,302 @@
+/*
+ * Intel Denverton SoC pinctrl/GPIO driver
+ *
+ * Copyright (C) 2017, Intel Corporation
+ * Author: Mika Westerberg <mika.westerberg@linux.intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/acpi.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm.h>
+#include <linux/pinctrl/pinctrl.h>
+
+#include "pinctrl-intel.h"
+
+#define DNV_PAD_OWN	0x020
+#define DNV_HOSTSW_OWN	0x0C0
+#define DNV_PADCFGLOCK	0x090
+#define DNV_GPI_IE	0x120
+
+#define DNV_GPP(n, s, e)				\
+	{						\
+		.reg_num = (n),				\
+		.base = (s),				\
+		.size = ((e) - (s) + 1),		\
+	}
+
+#define DNV_COMMUNITY(b, s, e, g)			\
+	{						\
+		.barno = (b),				\
+		.padown_offset = DNV_PAD_OWN,		\
+		.padcfglock_offset = DNV_PADCFGLOCK,	\
+		.hostown_offset = DNV_HOSTSW_OWN,	\
+		.ie_offset = DNV_GPI_IE,		\
+		.pin_base = (s),			\
+		.npins = ((e) - (s) + 1),		\
+		.gpps = (g),				\
+		.ngpps = ARRAY_SIZE(g),			\
+	}
+
+static const struct pinctrl_pin_desc dnv_pins[] = {
+	/* North ALL */
+	PINCTRL_PIN(0, "GBE0_SDP0"),
+	PINCTRL_PIN(1, "GBE1_SDP0"),
+	PINCTRL_PIN(2, "GBE0_SDP1"),
+	PINCTRL_PIN(3, "GBE1_SDP1"),
+	PINCTRL_PIN(4, "GBE0_SDP2"),
+	PINCTRL_PIN(5, "GBE1_SDP2"),
+	PINCTRL_PIN(6, "GBE0_SDP3"),
+	PINCTRL_PIN(7, "GBE1_SDP3"),
+	PINCTRL_PIN(8, "GBE2_LED0"),
+	PINCTRL_PIN(9, "GBE2_LED1"),
+	PINCTRL_PIN(10, "GBE0_I2C_CLK"),
+	PINCTRL_PIN(11, "GBE0_I2C_DATA"),
+	PINCTRL_PIN(12, "GBE1_I2C_CLK"),
+	PINCTRL_PIN(13, "GBE1_I2C_DATA"),
+	PINCTRL_PIN(14, "NCSI_RXD0"),
+	PINCTRL_PIN(15, "NCSI_CLK_IN"),
+	PINCTRL_PIN(16, "NCSI_RXD1"),
+	PINCTRL_PIN(17, "NCSI_CRS_DV"),
+	PINCTRL_PIN(18, "NCSI_ARB_IN"),
+	PINCTRL_PIN(19, "NCSI_TX_EN"),
+	PINCTRL_PIN(20, "NCSI_TXD0"),
+	PINCTRL_PIN(21, "NCSI_TXD1"),
+	PINCTRL_PIN(22, "NCSI_ARB_OUT"),
+	PINCTRL_PIN(23, "GBE0_LED0"),
+	PINCTRL_PIN(24, "GBE0_LED1"),
+	PINCTRL_PIN(25, "GBE1_LED0"),
+	PINCTRL_PIN(26, "GBE1_LED1"),
+	PINCTRL_PIN(27, "GPIO_0"),
+	PINCTRL_PIN(28, "PCIE_CLKREQ0_N"),
+	PINCTRL_PIN(29, "PCIE_CLKREQ1_N"),
+	PINCTRL_PIN(30, "PCIE_CLKREQ2_N"),
+	PINCTRL_PIN(31, "PCIE_CLKREQ3_N"),
+	PINCTRL_PIN(32, "PCIE_CLKREQ4_N"),
+	PINCTRL_PIN(33, "GPIO_1"),
+	PINCTRL_PIN(34, "GPIO_2"),
+	PINCTRL_PIN(35, "SVID_ALERT_N"),
+	PINCTRL_PIN(36, "SVID_DATA"),
+	PINCTRL_PIN(37, "SVID_CLK"),
+	PINCTRL_PIN(38, "THERMTRIP_N"),
+	PINCTRL_PIN(39, "PROCHOT_N"),
+	PINCTRL_PIN(40, "MEMHOT_N"),
+	/* South DFX */
+	PINCTRL_PIN(41, "DFX_PORT_CLK0"),
+	PINCTRL_PIN(42, "DFX_PORT_CLK1"),
+	PINCTRL_PIN(43, "DFX_PORT0"),
+	PINCTRL_PIN(44, "DFX_PORT1"),
+	PINCTRL_PIN(45, "DFX_PORT2"),
+	PINCTRL_PIN(46, "DFX_PORT3"),
+	PINCTRL_PIN(47, "DFX_PORT4"),
+	PINCTRL_PIN(48, "DFX_PORT5"),
+	PINCTRL_PIN(49, "DFX_PORT6"),
+	PINCTRL_PIN(50, "DFX_PORT7"),
+	PINCTRL_PIN(51, "DFX_PORT8"),
+	PINCTRL_PIN(52, "DFX_PORT9"),
+	PINCTRL_PIN(53, "DFX_PORT10"),
+	PINCTRL_PIN(54, "DFX_PORT11"),
+	PINCTRL_PIN(55, "DFX_PORT12"),
+	PINCTRL_PIN(56, "DFX_PORT13"),
+	PINCTRL_PIN(57, "DFX_PORT14"),
+	PINCTRL_PIN(58, "DFX_PORT15"),
+	/* South GPP0 */
+	PINCTRL_PIN(59, "GPIO_12"),
+	PINCTRL_PIN(60, "SMB5_GBE_ALRT_N"),
+	PINCTRL_PIN(61, "PCIE_CLKREQ5_N"),
+	PINCTRL_PIN(62, "PCIE_CLKREQ6_N"),
+	PINCTRL_PIN(63, "PCIE_CLKREQ7_N"),
+	PINCTRL_PIN(64, "UART0_RXD"),
+	PINCTRL_PIN(65, "UART0_TXD"),
+	PINCTRL_PIN(66, "SMB5_GBE_CLK"),
+	PINCTRL_PIN(67, "SMB5_GBE_DATA"),
+	PINCTRL_PIN(68, "ERROR2_N"),
+	PINCTRL_PIN(69, "ERROR1_N"),
+	PINCTRL_PIN(70, "ERROR0_N"),
+	PINCTRL_PIN(71, "IERR_N"),
+	PINCTRL_PIN(72, "MCERR_N"),
+	PINCTRL_PIN(73, "SMB0_LEG_CLK"),
+	PINCTRL_PIN(74, "SMB0_LEG_DATA"),
+	PINCTRL_PIN(75, "SMB0_LEG_ALRT_N"),
+	PINCTRL_PIN(76, "SMB1_HOST_DATA"),
+	PINCTRL_PIN(77, "SMB1_HOST_CLK"),
+	PINCTRL_PIN(78, "SMB2_PECI_DATA"),
+	PINCTRL_PIN(79, "SMB2_PECI_CLK"),
+	PINCTRL_PIN(80, "SMB4_CSME0_DATA"),
+	PINCTRL_PIN(81, "SMB4_CSME0_CLK"),
+	PINCTRL_PIN(82, "SMB4_CSME0_ALRT_N"),
+	PINCTRL_PIN(83, "USB_OC0_N"),
+	PINCTRL_PIN(84, "FLEX_CLK_SE0"),
+	PINCTRL_PIN(85, "FLEX_CLK_SE1"),
+	PINCTRL_PIN(86, "GPIO_4"),
+	PINCTRL_PIN(87, "GPIO_5"),
+	PINCTRL_PIN(88, "GPIO_6"),
+	PINCTRL_PIN(89, "GPIO_7"),
+	PINCTRL_PIN(90, "SATA0_LED_N"),
+	PINCTRL_PIN(91, "SATA1_LED_N"),
+	PINCTRL_PIN(92, "SATA_PDETECT0"),
+	PINCTRL_PIN(93, "SATA_PDETECT1"),
+	PINCTRL_PIN(94, "SATA0_SDOUT"),
+	PINCTRL_PIN(95, "SATA1_SDOUT"),
+	PINCTRL_PIN(96, "UART1_RXD"),
+	PINCTRL_PIN(97, "UART1_TXD"),
+	PINCTRL_PIN(98, "GPIO_8"),
+	PINCTRL_PIN(99, "GPIO_9"),
+	PINCTRL_PIN(100, "TCK"),
+	PINCTRL_PIN(101, "TRST_N"),
+	PINCTRL_PIN(102, "TMS"),
+	PINCTRL_PIN(103, "TDI"),
+	PINCTRL_PIN(104, "TDO"),
+	PINCTRL_PIN(105, "CX_PRDY_N"),
+	PINCTRL_PIN(106, "CX_PREQ_N"),
+	PINCTRL_PIN(107, "CTBTRIGINOUT"),
+	PINCTRL_PIN(108, "CTBTRIGOUT"),
+	PINCTRL_PIN(109, "DFX_SPARE2"),
+	PINCTRL_PIN(110, "DFX_SPARE3"),
+	PINCTRL_PIN(111, "DFX_SPARE4"),
+	/* South GPP1 */
+	PINCTRL_PIN(112, "SUSPWRDNACK"),
+	PINCTRL_PIN(113, "PMU_SUSCLK"),
+	PINCTRL_PIN(114, "ADR_TRIGGER"),
+	PINCTRL_PIN(115, "PMU_SLP_S45_N"),
+	PINCTRL_PIN(116, "PMU_SLP_S3_N"),
+	PINCTRL_PIN(117, "PMU_WAKE_N"),
+	PINCTRL_PIN(118, "PMU_PWRBTN_N"),
+	PINCTRL_PIN(119, "PMU_RESETBUTTON_N"),
+	PINCTRL_PIN(120, "PMU_PLTRST_N"),
+	PINCTRL_PIN(121, "SUS_STAT_N"),
+	PINCTRL_PIN(122, "SLP_S0IX_N"),
+	PINCTRL_PIN(123, "SPI_CS0_N"),
+	PINCTRL_PIN(124, "SPI_CS1_N"),
+	PINCTRL_PIN(125, "SPI_MOSI_IO0"),
+	PINCTRL_PIN(126, "SPI_MISO_IO1"),
+	PINCTRL_PIN(127, "SPI_IO2"),
+	PINCTRL_PIN(128, "SPI_IO3"),
+	PINCTRL_PIN(129, "SPI_CLK"),
+	PINCTRL_PIN(130, "SPI_CLK_LOOPBK"),
+	PINCTRL_PIN(131, "ESPI_IO0"),
+	PINCTRL_PIN(132, "ESPI_IO1"),
+	PINCTRL_PIN(133, "ESPI_IO2"),
+	PINCTRL_PIN(134, "ESPI_IO3"),
+	PINCTRL_PIN(135, "ESPI_CS0_N"),
+	PINCTRL_PIN(136, "ESPI_CLK"),
+	PINCTRL_PIN(137, "ESPI_RST_N"),
+	PINCTRL_PIN(138, "ESPI_ALRT0_N"),
+	PINCTRL_PIN(139, "GPIO_10"),
+	PINCTRL_PIN(140, "GPIO_11"),
+	PINCTRL_PIN(141, "ESPI_CLK_LOOPBK"),
+	PINCTRL_PIN(142, "EMMC_CMD"),
+	PINCTRL_PIN(143, "EMMC_STROBE"),
+	PINCTRL_PIN(144, "EMMC_CLK"),
+	PINCTRL_PIN(145, "EMMC_D0"),
+	PINCTRL_PIN(146, "EMMC_D1"),
+	PINCTRL_PIN(147, "EMMC_D2"),
+	PINCTRL_PIN(148, "EMMC_D3"),
+	PINCTRL_PIN(149, "EMMC_D4"),
+	PINCTRL_PIN(150, "EMMC_D5"),
+	PINCTRL_PIN(151, "EMMC_D6"),
+	PINCTRL_PIN(152, "EMMC_D7"),
+	PINCTRL_PIN(153, "GPIO_3"),
+};
+
+static const unsigned int dnv_uart0_pins[] = { 60, 61, 64, 65 };
+static const unsigned int dnv_uart0_modes[] = { 2, 3, 1, 1 };
+static const unsigned int dnv_uart1_pins[] = { 94, 95, 96, 97 };
+static const unsigned int dnv_uart2_pins[] = { 60, 61, 62, 63 };
+static const unsigned int dnv_uart2_modes[] = { 1, 1, 2, 2 };
+static const unsigned int dnv_emmc_pins[] = {
+	142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152,
+};
+
+static const struct intel_pingroup dnv_groups[] = {
+	PIN_GROUP("uart0_grp", dnv_uart0_pins, dnv_uart0_modes),
+	PIN_GROUP("uart1_grp", dnv_uart1_pins, 1),
+	PIN_GROUP("uart2_grp", dnv_uart2_pins, dnv_uart2_modes),
+	PIN_GROUP("emmc_grp", dnv_emmc_pins, 1),
+};
+
+static const char * const dnv_uart0_groups[] = { "uart0_grp" };
+static const char * const dnv_uart1_groups[] = { "uart1_grp" };
+static const char * const dnv_uart2_groups[] = { "uart2_grp" };
+static const char * const dnv_emmc_groups[] = { "emmc_grp" };
+
+static const struct intel_function dnv_functions[] = {
+	FUNCTION("uart0", dnv_uart0_groups),
+	FUNCTION("uart1", dnv_uart1_groups),
+	FUNCTION("uart2", dnv_uart2_groups),
+	FUNCTION("emmc", dnv_emmc_groups),
+};
+
+static const struct intel_padgroup dnv_north_gpps[] = {
+	DNV_GPP(0, 0, 31),	/* North ALL_0 */
+	DNV_GPP(1, 32, 40),	/* North ALL_1 */
+};
+
+static const struct intel_padgroup dnv_south_gpps[] = {
+	DNV_GPP(0, 41, 58),	/* South DFX */
+	DNV_GPP(1, 59, 90),	/* South GPP0_0 */
+	DNV_GPP(2, 91, 111),	/* South GPP0_1 */
+	DNV_GPP(3, 112, 143),	/* South GPP1_0 */
+	DNV_GPP(4, 144, 153),	/* South GPP1_1 */
+};
+
+static const struct intel_community dnv_communities[] = {
+	DNV_COMMUNITY(0, 0, 40, dnv_north_gpps),
+	DNV_COMMUNITY(1, 41, 153, dnv_south_gpps),
+};
+
+static const struct intel_pinctrl_soc_data dnv_soc_data = {
+	.pins = dnv_pins,
+	.npins = ARRAY_SIZE(dnv_pins),
+	.groups = dnv_groups,
+	.ngroups = ARRAY_SIZE(dnv_groups),
+	.functions = dnv_functions,
+	.nfunctions = ARRAY_SIZE(dnv_functions),
+	.communities = dnv_communities,
+	.ncommunities = ARRAY_SIZE(dnv_communities),
+};
+
+static int dnv_pinctrl_probe(struct platform_device *pdev)
+{
+	return intel_pinctrl_probe(pdev, &dnv_soc_data);
+}
+
+static const struct dev_pm_ops dnv_pinctrl_pm_ops = {
+	SET_LATE_SYSTEM_SLEEP_PM_OPS(intel_pinctrl_suspend,
+				     intel_pinctrl_resume)
+};
+
+static const struct acpi_device_id dnv_pinctrl_acpi_match[] = {
+	{ "INTC3000" },
+	{ }
+};
+MODULE_DEVICE_TABLE(acpi, dnv_pinctrl_acpi_match);
+
+static struct platform_driver dnv_pinctrl_driver = {
+	.probe = dnv_pinctrl_probe,
+	.driver = {
+		.name = "denverton-pinctrl",
+		.acpi_match_table = dnv_pinctrl_acpi_match,
+		.pm = &dnv_pinctrl_pm_ops,
+	},
+};
+
+static int __init dnv_pinctrl_init(void)
+{
+	return platform_driver_register(&dnv_pinctrl_driver);
+}
+subsys_initcall(dnv_pinctrl_init);
+
+static void __exit dnv_pinctrl_exit(void)
+{
+	platform_driver_unregister(&dnv_pinctrl_driver);
+}
+module_exit(dnv_pinctrl_exit);
+
+MODULE_AUTHOR("Mika Westerberg <mika.westerberg@linux.intel.com>");
+MODULE_DESCRIPTION("Intel Denverton SoC pinctrl/GPIO driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/pinctrl/intel/pinctrl-intel.c b/drivers/pinctrl/intel/pinctrl-intel.c
index 6dc1096d3d34..71df0f70b61f 100644
--- a/drivers/pinctrl/intel/pinctrl-intel.c
+++ b/drivers/pinctrl/intel/pinctrl-intel.c
@@ -751,33 +751,38 @@ static int intel_gpio_get(struct gpio_chip *chip, unsigned offset)
 {
 	struct intel_pinctrl *pctrl = gpiochip_get_data(chip);
 	void __iomem *reg;
+	u32 padcfg0;
 
 	reg = intel_get_padcfg(pctrl, offset, PADCFG0);
 	if (!reg)
 		return -EINVAL;
 
-	return !!(readl(reg) & PADCFG0_GPIORXSTATE);
+	padcfg0 = readl(reg);
+	if (!(padcfg0 & PADCFG0_GPIOTXDIS))
+		return !!(padcfg0 & PADCFG0_GPIOTXSTATE);
+
+	return !!(padcfg0 & PADCFG0_GPIORXSTATE);
 }
 
 static void intel_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 {
 	struct intel_pinctrl *pctrl = gpiochip_get_data(chip);
+	unsigned long flags;
 	void __iomem *reg;
+	u32 padcfg0;
 
 	reg = intel_get_padcfg(pctrl, offset, PADCFG0);
-	if (reg) {
-		unsigned long flags;
-		u32 padcfg0;
+	if (!reg)
+		return;
 
-		raw_spin_lock_irqsave(&pctrl->lock, flags);
-		padcfg0 = readl(reg);
-		if (value)
-			padcfg0 |= PADCFG0_GPIOTXSTATE;
-		else
-			padcfg0 &= ~PADCFG0_GPIOTXSTATE;
-		writel(padcfg0, reg);
-		raw_spin_unlock_irqrestore(&pctrl->lock, flags);
-	}
+	raw_spin_lock_irqsave(&pctrl->lock, flags);
+	padcfg0 = readl(reg);
+	if (value)
+		padcfg0 |= PADCFG0_GPIOTXSTATE;
+	else
+		padcfg0 &= ~PADCFG0_GPIOTXSTATE;
+	writel(padcfg0, reg);
+	raw_spin_unlock_irqrestore(&pctrl->lock, flags);
 }
 
 static int intel_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
@@ -1035,6 +1040,7 @@ static struct irq_chip intel_gpio_irqchip = {
 	.irq_unmask = intel_gpio_irq_unmask,
 	.irq_set_type = intel_gpio_irq_type,
 	.irq_set_wake = intel_gpio_irq_wake,
+	.flags = IRQCHIP_MASK_ON_SUSPEND,
 };
 
 static int intel_gpio_probe(struct intel_pinctrl *pctrl, int irq)
diff --git a/drivers/pinctrl/intel/pinctrl-lewisburg.c b/drivers/pinctrl/intel/pinctrl-lewisburg.c
new file mode 100644
index 000000000000..14d56ea6cfdc
--- /dev/null
+++ b/drivers/pinctrl/intel/pinctrl-lewisburg.c
@@ -0,0 +1,343 @@
+/*
+ * Intel Lewisburg pinctrl/GPIO driver
+ *
+ * Copyright (C) 2017, Intel Corporation
+ * Author: Mika Westerberg <mika.westerberg@linux.intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/acpi.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm.h>
+#include <linux/pinctrl/pinctrl.h>
+
+#include "pinctrl-intel.h"
+
+#define LBG_PAD_OWN	0x020
+#define LBG_PADCFGLOCK	0x060
+#define LBG_HOSTSW_OWN	0x080
+#define LBG_GPI_IE	0x110
+
+#define LBG_COMMUNITY(b, s, e)				\
+	{						\
+		.barno = (b),				\
+		.padown_offset = LBG_PAD_OWN,		\
+		.padcfglock_offset = LBG_PADCFGLOCK,	\
+		.hostown_offset = LBG_HOSTSW_OWN,	\
+		.ie_offset = LBG_GPI_IE,		\
+		.gpp_size = 24,				\
+		.pin_base = (s),			\
+		.npins = ((e) - (s) + 1),		\
+	}
+
+static const struct pinctrl_pin_desc lbg_pins[] = {
+	/* GPP_A */
+	PINCTRL_PIN(0, "RCINB"),
+	PINCTRL_PIN(1, "LAD_0"),
+	PINCTRL_PIN(2, "LAD_1"),
+	PINCTRL_PIN(3, "LAD_2"),
+	PINCTRL_PIN(4, "LAD_3"),
+	PINCTRL_PIN(5, "LFRAMEB"),
+	PINCTRL_PIN(6, "SERIRQ"),
+	PINCTRL_PIN(7, "PIRQAB"),
+	PINCTRL_PIN(8, "CLKRUNB"),
+	PINCTRL_PIN(9, "CLKOUT_LPC_0"),
+	PINCTRL_PIN(10, "CLKOUT_LPC_1"),
+	PINCTRL_PIN(11, "PMEB"),
+	PINCTRL_PIN(12, "BM_BUSYB"),
+	PINCTRL_PIN(13, "SUSWARNB_SUSPWRDNACK"),
+	PINCTRL_PIN(14, "ESPI_RESETB"),
+	PINCTRL_PIN(15, "SUSACKB"),
+	PINCTRL_PIN(16, "CLKOUT_LPC_2"),
+	PINCTRL_PIN(17, "GPP_A_17"),
+	PINCTRL_PIN(18, "GPP_A_18"),
+	PINCTRL_PIN(19, "GPP_A_19"),
+	PINCTRL_PIN(20, "GPP_A_20"),
+	PINCTRL_PIN(21, "GPP_A_21"),
+	PINCTRL_PIN(22, "GPP_A_22"),
+	PINCTRL_PIN(23, "GPP_A_23"),
+	/* GPP_B */
+	PINCTRL_PIN(24, "CORE_VID_0"),
+	PINCTRL_PIN(25, "CORE_VID_1"),
+	PINCTRL_PIN(26, "VRALERTB"),
+	PINCTRL_PIN(27, "CPU_GP_2"),
+	PINCTRL_PIN(28, "CPU_GP_3"),
+	PINCTRL_PIN(29, "SRCCLKREQB_0"),
+	PINCTRL_PIN(30, "SRCCLKREQB_1"),
+	PINCTRL_PIN(31, "SRCCLKREQB_2"),
+	PINCTRL_PIN(32, "SRCCLKREQB_3"),
+	PINCTRL_PIN(33, "SRCCLKREQB_4"),
+	PINCTRL_PIN(34, "SRCCLKREQB_5"),
+	PINCTRL_PIN(35, "GPP_B_11"),
+	PINCTRL_PIN(36, "GLB_RST_WARN_N"),
+	PINCTRL_PIN(37, "PLTRSTB"),
+	PINCTRL_PIN(38, "SPKR"),
+	PINCTRL_PIN(39, "GPP_B_15"),
+	PINCTRL_PIN(40, "GPP_B_16"),
+	PINCTRL_PIN(41, "GPP_B_17"),
+	PINCTRL_PIN(42, "GPP_B_18"),
+	PINCTRL_PIN(43, "GPP_B_19"),
+	PINCTRL_PIN(44, "GPP_B_20"),
+	PINCTRL_PIN(45, "GPP_B_21"),
+	PINCTRL_PIN(46, "GPP_B_22"),
+	PINCTRL_PIN(47, "SML1ALERTB"),
+	/* GPP_F */
+	PINCTRL_PIN(48, "SATAXPCIE_3"),
+	PINCTRL_PIN(49, "SATAXPCIE_4"),
+	PINCTRL_PIN(50, "SATAXPCIE_5"),
+	PINCTRL_PIN(51, "SATAXPCIE_6"),
+	PINCTRL_PIN(52, "SATAXPCIE_7"),
+	PINCTRL_PIN(53, "SATA_DEVSLP_3"),
+	PINCTRL_PIN(54, "SATA_DEVSLP_4"),
+	PINCTRL_PIN(55, "SATA_DEVSLP_5"),
+	PINCTRL_PIN(56, "SATA_DEVSLP_6"),
+	PINCTRL_PIN(57, "SATA_DEVSLP_7"),
+	PINCTRL_PIN(58, "SATA_SCLOCK"),
+	PINCTRL_PIN(59, "SATA_SLOAD"),
+	PINCTRL_PIN(60, "SATA_SDATAOUT1"),
+	PINCTRL_PIN(61, "SATA_SDATAOUT0"),
+	PINCTRL_PIN(62, "SSATA_LEDB"),
+	PINCTRL_PIN(63, "USB2_OCB_4"),
+	PINCTRL_PIN(64, "USB2_OCB_5"),
+	PINCTRL_PIN(65, "USB2_OCB_6"),
+	PINCTRL_PIN(66, "USB2_OCB_7"),
+	PINCTRL_PIN(67, "GBE_SMBUS_CLK"),
+	PINCTRL_PIN(68, "GBE_SMBDATA"),
+	PINCTRL_PIN(69, "GBE_SMBALRTN"),
+	PINCTRL_PIN(70, "SSATA_SCLOCK"),
+	PINCTRL_PIN(71, "SSATA_SLOAD"),
+	/* GPP_C */
+	PINCTRL_PIN(72, "SMBCLK"),
+	PINCTRL_PIN(73, "SMBDATA"),
+	PINCTRL_PIN(74, "SMBALERTB"),
+	PINCTRL_PIN(75, "SML0CLK"),
+	PINCTRL_PIN(76, "SML0DATA"),
+	PINCTRL_PIN(77, "SML0ALERTB"),
+	PINCTRL_PIN(78, "SML1CLK"),
+	PINCTRL_PIN(79, "SML1DATA"),
+	PINCTRL_PIN(80, "GPP_C_8"),
+	PINCTRL_PIN(81, "GPP_C_9"),
+	PINCTRL_PIN(82, "GPP_C_10"),
+	PINCTRL_PIN(83, "GPP_C_11"),
+	PINCTRL_PIN(84, "GPP_C_12"),
+	PINCTRL_PIN(85, "GPP_C_13"),
+	PINCTRL_PIN(86, "GPP_C_14"),
+	PINCTRL_PIN(87, "GPP_C_15"),
+	PINCTRL_PIN(88, "GPP_C_16"),
+	PINCTRL_PIN(89, "GPP_C_17"),
+	PINCTRL_PIN(90, "GPP_C_18"),
+	PINCTRL_PIN(91, "GPP_C_19"),
+	PINCTRL_PIN(92, "GPP_C_20"),
+	PINCTRL_PIN(93, "GPP_C_21"),
+	PINCTRL_PIN(94, "GPP_C_22"),
+	PINCTRL_PIN(95, "GPP_C_23"),
+	/* GPP_D */
+	PINCTRL_PIN(96, "GPP_D_0"),
+	PINCTRL_PIN(97, "GPP_D_1"),
+	PINCTRL_PIN(98, "GPP_D_2"),
+	PINCTRL_PIN(99, "GPP_D_3"),
+	PINCTRL_PIN(100, "GPP_D_4"),
+	PINCTRL_PIN(101, "SSP0_SFRM"),
+	PINCTRL_PIN(102, "SSP0_TXD"),
+	PINCTRL_PIN(103, "SSP0_RXD"),
+	PINCTRL_PIN(104, "SSP0_SCLK"),
+	PINCTRL_PIN(105, "SSATA_DEVSLP_3"),
+	PINCTRL_PIN(106, "SSATA_DEVSLP_4"),
+	PINCTRL_PIN(107, "SSATA_DEVSLP_5"),
+	PINCTRL_PIN(108, "SSATA_SDATAOUT1"),
+	PINCTRL_PIN(109, "SML0BCLK_SML0BCLKIE"),
+	PINCTRL_PIN(110, "SML0BDATA_SML0BDATAIE"),
+	PINCTRL_PIN(111, "SSATA_SDATAOUT0"),
+	PINCTRL_PIN(112, "SML0BALERTB_SML0BALERTBIE"),
+	PINCTRL_PIN(113, "DMIC_CLK_1"),
+	PINCTRL_PIN(114, "DMIC_DATA_1"),
+	PINCTRL_PIN(115, "DMIC_CLK_0"),
+	PINCTRL_PIN(116, "DMIC_DATA_0"),
+	PINCTRL_PIN(117, "IE_UART_RXD"),
+	PINCTRL_PIN(118, "IE_UART_TXD"),
+	PINCTRL_PIN(119, "GPP_D_23"),
+	/* GPP_E */
+	PINCTRL_PIN(120, "SATAXPCIE_0"),
+	PINCTRL_PIN(121, "SATAXPCIE_1"),
+	PINCTRL_PIN(122, "SATAXPCIE_2"),
+	PINCTRL_PIN(123, "CPU_GP_0"),
+	PINCTRL_PIN(124, "SATA_DEVSLP_0"),
+	PINCTRL_PIN(125, "SATA_DEVSLP_1"),
+	PINCTRL_PIN(126, "SATA_DEVSLP_2"),
+	PINCTRL_PIN(127, "CPU_GP_1"),
+	PINCTRL_PIN(128, "SATA_LEDB"),
+	PINCTRL_PIN(129, "USB2_OCB_0"),
+	PINCTRL_PIN(130, "USB2_OCB_1"),
+	PINCTRL_PIN(131, "USB2_OCB_2"),
+	PINCTRL_PIN(132, "USB2_OCB_3"),
+	/* GPP_I */
+	PINCTRL_PIN(133, "GBE_TDO"),
+	PINCTRL_PIN(134, "GBE_TCK"),
+	PINCTRL_PIN(135, "GBE_TMS"),
+	PINCTRL_PIN(136, "GBE_TDI"),
+	PINCTRL_PIN(137, "DO_RESET_INB"),
+	PINCTRL_PIN(138, "DO_RESET_OUTB"),
+	PINCTRL_PIN(139, "RESET_DONE"),
+	PINCTRL_PIN(140, "GBE_TRST_N"),
+	PINCTRL_PIN(141, "GBE_PCI_DIS"),
+	PINCTRL_PIN(142, "GBE_LAN_DIS"),
+	PINCTRL_PIN(143, "GPP_I_10"),
+	PINCTRL_PIN(144, "GPIO_RCOMP_3P3"),
+	/* GPP_J */
+	PINCTRL_PIN(145, "GBE_LED_0_0"),
+	PINCTRL_PIN(146, "GBE_LED_0_1"),
+	PINCTRL_PIN(147, "GBE_LED_1_0"),
+	PINCTRL_PIN(148, "GBE_LED_1_1"),
+	PINCTRL_PIN(149, "GBE_LED_2_0"),
+	PINCTRL_PIN(150, "GBE_LED_2_1"),
+	PINCTRL_PIN(151, "GBE_LED_3_0"),
+	PINCTRL_PIN(152, "GBE_LED_3_1"),
+	PINCTRL_PIN(153, "GBE_SCL_0"),
+	PINCTRL_PIN(154, "GBE_SDA_0"),
+	PINCTRL_PIN(155, "GBE_SCL_1"),
+	PINCTRL_PIN(156, "GBE_SDA_1"),
+	PINCTRL_PIN(157, "GBE_SCL_2"),
+	PINCTRL_PIN(158, "GBE_SDA_2"),
+	PINCTRL_PIN(159, "GBE_SCL_3"),
+	PINCTRL_PIN(160, "GBE_SDA_3"),
+	PINCTRL_PIN(161, "GBE_SDP_0_0"),
+	PINCTRL_PIN(162, "GBE_SDP_0_1"),
+	PINCTRL_PIN(163, "GBE_SDP_1_0"),
+	PINCTRL_PIN(164, "GBE_SDP_1_1"),
+	PINCTRL_PIN(165, "GBE_SDP_2_0"),
+	PINCTRL_PIN(166, "GBE_SDP_2_1"),
+	PINCTRL_PIN(167, "GBE_SDP_3_0"),
+	PINCTRL_PIN(168, "GBE_SDP_3_1"),
+	/* GPP_K */
+	PINCTRL_PIN(169, "GBE_RMIICLK"),
+	PINCTRL_PIN(170, "GBE_RMII_TXD_0"),
+	PINCTRL_PIN(171, "GBE_RMII_TXD_1"),
+	PINCTRL_PIN(172, "GBE_RMII_TX_EN"),
+	PINCTRL_PIN(173, "GBE_RMII_CRS_DV"),
+	PINCTRL_PIN(174, "GBE_RMII_RXD_0"),
+	PINCTRL_PIN(175, "GBE_RMII_RXD_1"),
+	PINCTRL_PIN(176, "GBE_RMII_RX_ER"),
+	PINCTRL_PIN(177, "GBE_RMII_ARBIN"),
+	PINCTRL_PIN(178, "GBE_RMII_ARB_OUT"),
+	PINCTRL_PIN(179, "PE_RST_N"),
+	PINCTRL_PIN(180, "GPIO_RCOMP_1P8_3P3"),
+	/* GPP_G */
+	PINCTRL_PIN(181, "FAN_TACH_0"),
+	PINCTRL_PIN(182, "FAN_TACH_1"),
+	PINCTRL_PIN(183, "FAN_TACH_2"),
+	PINCTRL_PIN(184, "FAN_TACH_3"),
+	PINCTRL_PIN(185, "FAN_TACH_4"),
+	PINCTRL_PIN(186, "FAN_TACH_5"),
+	PINCTRL_PIN(187, "FAN_TACH_6"),
+	PINCTRL_PIN(188, "FAN_TACH_7"),
+	PINCTRL_PIN(189, "FAN_PWM_0"),
+	PINCTRL_PIN(190, "FAN_PWM_1"),
+	PINCTRL_PIN(191, "FAN_PWM_2"),
+	PINCTRL_PIN(192, "FAN_PWM_3"),
+	PINCTRL_PIN(193, "GSXDOUT"),
+	PINCTRL_PIN(194, "GSXSLOAD"),
+	PINCTRL_PIN(195, "GSXDIN"),
+	PINCTRL_PIN(196, "GSXSRESETB"),
+	PINCTRL_PIN(197, "GSXCLK"),
+	PINCTRL_PIN(198, "ADR_COMPLETE"),
+	PINCTRL_PIN(199, "NMIB"),
+	PINCTRL_PIN(200, "SMIB"),
+	PINCTRL_PIN(201, "SSATA_DEVSLP_0"),
+	PINCTRL_PIN(202, "SSATA_DEVSLP_1"),
+	PINCTRL_PIN(203, "SSATA_DEVSLP_2"),
+	PINCTRL_PIN(204, "SSATAXPCIE0_SSATAGP0"),
+	/* GPP_H */
+	PINCTRL_PIN(205, "SRCCLKREQB_6"),
+	PINCTRL_PIN(206, "SRCCLKREQB_7"),
+	PINCTRL_PIN(207, "SRCCLKREQB_8"),
+	PINCTRL_PIN(208, "SRCCLKREQB_9"),
+	PINCTRL_PIN(209, "SRCCLKREQB_10"),
+	PINCTRL_PIN(210, "SRCCLKREQB_11"),
+	PINCTRL_PIN(211, "SRCCLKREQB_12"),
+	PINCTRL_PIN(212, "SRCCLKREQB_13"),
+	PINCTRL_PIN(213, "SRCCLKREQB_14"),
+	PINCTRL_PIN(214, "SRCCLKREQB_15"),
+	PINCTRL_PIN(215, "SML2CLK"),
+	PINCTRL_PIN(216, "SML2DATA"),
+	PINCTRL_PIN(217, "SML2ALERTB"),
+	PINCTRL_PIN(218, "SML3CLK"),
+	PINCTRL_PIN(219, "SML3DATA"),
+	PINCTRL_PIN(220, "SML3ALERTB"),
+	PINCTRL_PIN(221, "SML4CLK"),
+	PINCTRL_PIN(222, "SML4DATA"),
+	PINCTRL_PIN(223, "SML4ALERTB"),
+	PINCTRL_PIN(224, "SSATAXPCIE1_SSATAGP1"),
+	PINCTRL_PIN(225, "SSATAXPCIE2_SSATAGP2"),
+	PINCTRL_PIN(226, "SSATAXPCIE3_SSATAGP3"),
+	PINCTRL_PIN(227, "SSATAXPCIE4_SSATAGP4"),
+	PINCTRL_PIN(228, "SSATAXPCIE5_SSATAGP5"),
+	/* GPP_L */
+	PINCTRL_PIN(229, "VISA2CH0_D0"),
+	PINCTRL_PIN(230, "VISA2CH0_D1"),
+	PINCTRL_PIN(231, "VISA2CH0_D2"),
+	PINCTRL_PIN(232, "VISA2CH0_D3"),
+	PINCTRL_PIN(233, "VISA2CH0_D4"),
+	PINCTRL_PIN(234, "VISA2CH0_D5"),
+	PINCTRL_PIN(235, "VISA2CH0_D6"),
+	PINCTRL_PIN(236, "VISA2CH0_D7"),
+	PINCTRL_PIN(237, "VISA2CH0_CLK"),
+	PINCTRL_PIN(238, "VISA2CH1_D0"),
+	PINCTRL_PIN(239, "VISA2CH1_D1"),
+	PINCTRL_PIN(240, "VISA2CH1_D2"),
+	PINCTRL_PIN(241, "VISA2CH1_D3"),
+	PINCTRL_PIN(242, "VISA2CH1_D4"),
+	PINCTRL_PIN(243, "VISA2CH1_D5"),
+	PINCTRL_PIN(244, "VISA2CH1_D6"),
+	PINCTRL_PIN(245, "VISA2CH1_D7"),
+	PINCTRL_PIN(246, "VISA2CH1_CLK"),
+};
+
+static const struct intel_community lbg_communities[] = {
+	LBG_COMMUNITY(0, 0, 71),
+	LBG_COMMUNITY(1, 72, 132),
+	LBG_COMMUNITY(3, 133, 144),
+	LBG_COMMUNITY(4, 145, 180),
+	LBG_COMMUNITY(5, 181, 246),
+};
+
+static const struct intel_pinctrl_soc_data lbg_soc_data = {
+	.pins = lbg_pins,
+	.npins = ARRAY_SIZE(lbg_pins),
+	.communities = lbg_communities,
+	.ncommunities = ARRAY_SIZE(lbg_communities),
+};
+
+static int lbg_pinctrl_probe(struct platform_device *pdev)
+{
+	return intel_pinctrl_probe(pdev, &lbg_soc_data);
+}
+
+static const struct dev_pm_ops lbg_pinctrl_pm_ops = {
+	SET_LATE_SYSTEM_SLEEP_PM_OPS(intel_pinctrl_suspend,
+				     intel_pinctrl_resume)
+};
+
+static const struct acpi_device_id lbg_pinctrl_acpi_match[] = {
+	{ "INT3536" },
+	{ }
+};
+MODULE_DEVICE_TABLE(acpi, lbg_pinctrl_acpi_match);
+
+static struct platform_driver lbg_pinctrl_driver = {
+	.probe = lbg_pinctrl_probe,
+	.driver = {
+		.name = "lewisburg-pinctrl",
+		.acpi_match_table = lbg_pinctrl_acpi_match,
+		.pm = &lbg_pinctrl_pm_ops,
+	},
+};
+
+module_platform_driver(lbg_pinctrl_driver);
+
+MODULE_AUTHOR("Mika Westerberg <mika.westerberg@linux.intel.com>");
+MODULE_DESCRIPTION("Intel Lewisburg pinctrl/GPIO driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-mt2701.h b/drivers/pinctrl/mediatek/pinctrl-mtk-mt2701.h
index f90642078c31..1035df49301f 100644
--- a/drivers/pinctrl/mediatek/pinctrl-mtk-mt2701.h
+++ b/drivers/pinctrl/mediatek/pinctrl-mtk-mt2701.h
@@ -223,6 +223,8 @@ static const struct mtk_desc_pin mtk_pins_mt2701[] = {
 		MTK_EINT_FUNCTION(0, 0),
 		MTK_FUNCTION(0, "GPIO22"),
 		MTK_FUNCTION(1, "UCTS0"),
+		/* MT7623 take function 2 as PCIE0_PERST_N */
+		MTK_FUNCTION(2, "PCIE0_PERST_N"),
 		MTK_FUNCTION(3, "KCOL3"),
 		MTK_FUNCTION(4, "CONN_DSP_JDO"),
 		MTK_FUNCTION(5, "EXT_FRAME_SYNC"),
@@ -235,6 +237,8 @@ static const struct mtk_desc_pin mtk_pins_mt2701[] = {
 		MTK_EINT_FUNCTION(0, 1),
 		MTK_FUNCTION(0, "GPIO23"),
 		MTK_FUNCTION(1, "URTS0"),
+		/* MT7623 take function 2 as PCIE1_PERST_N */
+		MTK_FUNCTION(2, "PCIE1_PERST_N"),
 		MTK_FUNCTION(3, "KCOL2"),
 		MTK_FUNCTION(4, "CONN_MCU_TDO"),
 		MTK_FUNCTION(5, "EXT_FRAME_SYNC"),
@@ -247,6 +251,8 @@ static const struct mtk_desc_pin mtk_pins_mt2701[] = {
 		MTK_EINT_FUNCTION(0, 2),
 		MTK_FUNCTION(0, "GPIO24"),
 		MTK_FUNCTION(1, "UCTS1"),
+		/* MT7623 take function 2 as PCIE2_PERST_N */
+		MTK_FUNCTION(2, "PCIE2_PERST_N"),
 		MTK_FUNCTION(3, "KCOL1"),
 		MTK_FUNCTION(4, "CONN_MCU_DBGACK_N"),
 		MTK_FUNCTION(7, "DBG_MON_A[28]"),
@@ -308,6 +314,8 @@ static const struct mtk_desc_pin mtk_pins_mt2701[] = {
 		MTK_FUNCTION(3, "KROW0"),
 		MTK_FUNCTION(4, "CONN_MCU_TMS"),
 		MTK_FUNCTION(5, "CONN_MCU_AICE_JMSC"),
+		/* MT7623 take function 6 as PCIE2_PERST_N */
+		MTK_FUNCTION(6, "PCIE2_PERST_N"),
 		MTK_FUNCTION(7, "DBG_MON_A[23]"),
 		MTK_FUNCTION(14, "PCIE2_PERST_N")
 	),
@@ -1787,6 +1795,8 @@ static const struct mtk_desc_pin mtk_pins_mt2701[] = {
 		MTK_FUNCTION(0, "GPIO208"),
 		MTK_FUNCTION(1, "AUD_EXT_CK1"),
 		MTK_FUNCTION(2, "PWM0"),
+		/* MT7623 take function 3 as PCIE0_PERST_N */
+		MTK_FUNCTION(3, "PCIE0_PERST_N"),
 		MTK_FUNCTION(4, "ANT_SEL5"),
 		MTK_FUNCTION(5, "DISP_PWM"),
 		MTK_FUNCTION(7, "DBG_MON_A[31]"),
@@ -1799,6 +1809,8 @@ static const struct mtk_desc_pin mtk_pins_mt2701[] = {
 		MTK_FUNCTION(0, "GPIO209"),
 		MTK_FUNCTION(1, "AUD_EXT_CK2"),
 		MTK_FUNCTION(2, "MSDC1_WP"),
+		/* MT7623 take function 3 as PCIE1_PERST_N */
+		MTK_FUNCTION(3, "PCIE1_PERST_N"),
 		MTK_FUNCTION(5, "PWM1"),
 		MTK_FUNCTION(7, "DBG_MON_A[32]"),
 		MTK_FUNCTION(11, "PCIE1_PERST_N")
diff --git a/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c b/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c
index 0c6d7812d6fd..71b944748304 100644
--- a/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c
+++ b/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c
@@ -190,14 +190,14 @@ static struct armada_37xx_pin_group armada_37xx_sb_groups[] = {
 		       "mii", "mii_err"),
 };
 
-const struct armada_37xx_pin_data armada_37xx_pin_nb = {
+static const struct armada_37xx_pin_data armada_37xx_pin_nb = {
 	.nr_pins = 36,
 	.name = "GPIO1",
 	.groups = armada_37xx_nb_groups,
 	.ngroups = ARRAY_SIZE(armada_37xx_nb_groups),
 };
 
-const struct armada_37xx_pin_data armada_37xx_pin_sb = {
+static const struct armada_37xx_pin_data armada_37xx_pin_sb = {
 	.nr_pins = 30,
 	.name = "GPIO2",
 	.groups = armada_37xx_sb_groups,
@@ -254,7 +254,7 @@ static int armada_37xx_pin_config_group_set(struct pinctrl_dev *pctldev,
 	return -ENOTSUPP;
 }
 
-static struct pinconf_ops armada_37xx_pinconf_ops = {
+static const struct pinconf_ops armada_37xx_pinconf_ops = {
 	.is_generic = true,
 	.pin_config_group_get = armada_37xx_pin_config_group_get,
 	.pin_config_group_set = armada_37xx_pin_config_group_set,
@@ -550,9 +550,9 @@ static int armada_37xx_irq_set_wake(struct irq_data *d, unsigned int on)
 	spin_lock_irqsave(&info->irq_lock, flags);
 	val = readl(info->base + reg);
 	if (on)
-		val |= d->mask;
+		val |= (BIT(d->hwirq % GPIO_PER_REG));
 	else
-		val &= ~d->mask;
+		val &= ~(BIT(d->hwirq % GPIO_PER_REG));
 	writel(val, info->base + reg);
 	spin_unlock_irqrestore(&info->irq_lock, flags);
 
@@ -571,10 +571,10 @@ static int armada_37xx_irq_set_type(struct irq_data *d, unsigned int type)
 	val = readl(info->base + reg);
 	switch (type) {
 	case IRQ_TYPE_EDGE_RISING:
-		val &= ~d->mask;
+		val &= ~(BIT(d->hwirq % GPIO_PER_REG));
 		break;
 	case IRQ_TYPE_EDGE_FALLING:
-		val |= d->mask;
+		val |= (BIT(d->hwirq % GPIO_PER_REG));
 		break;
 	default:
 		spin_unlock_irqrestore(&info->irq_lock, flags);
@@ -624,11 +624,27 @@ static void armada_37xx_irq_handler(struct irq_desc *desc)
 	chained_irq_exit(chip, desc);
 }
 
+static unsigned int armada_37xx_irq_startup(struct irq_data *d)
+{
+	struct gpio_chip *chip = irq_data_get_irq_chip_data(d);
+	int irq = d->hwirq - chip->irq_base;
+	/*
+	 * The mask field is a "precomputed bitmask for accessing the
+	 * chip registers" which was introduced for the generic
+	 * irqchip framework. As we don't use this framework, we can
+	 * reuse this field for our own usage.
+	 */
+	d->mask = BIT(irq % GPIO_PER_REG);
+
+	armada_37xx_irq_unmask(d);
+
+	return 0;
+}
+
 static int armada_37xx_irqchip_register(struct platform_device *pdev,
 					struct armada_37xx_pinctrl *info)
 {
 	struct device_node *np = info->dev->of_node;
-	int nrirqs = info->data->nr_pins;
 	struct gpio_chip *gc = &info->gpio_chip;
 	struct irq_chip *irqchip = &info->irq_chip;
 	struct resource res;
@@ -666,8 +682,8 @@ static int armada_37xx_irqchip_register(struct platform_device *pdev,
 	irqchip->irq_unmask = armada_37xx_irq_unmask;
 	irqchip->irq_set_wake = armada_37xx_irq_set_wake;
 	irqchip->irq_set_type = armada_37xx_irq_set_type;
+	irqchip->irq_startup = armada_37xx_irq_startup;
 	irqchip->name = info->data->name;
-
 	ret = gpiochip_irqchip_add(gc, irqchip, 0,
 				   handle_edge_irq, IRQ_TYPE_NONE);
 	if (ret) {
@@ -680,19 +696,6 @@ static int armada_37xx_irqchip_register(struct platform_device *pdev,
 	 * controller. But we do not take advantage of this and use
 	 * the chained irq with all of them.
 	 */
-	for (i = 0; i < nrirqs; i++) {
-		struct irq_data *d = irq_get_irq_data(gc->irq_base + i);
-
-		/*
-		 * The mask field is a "precomputed bitmask for
-		 * accessing the chip registers" which was introduced
-		 * for the generic irqchip framework. As we don't use
-		 * this framework, we can reuse this field for our own
-		 * usage.
-		 */
-		d->mask = BIT(i % GPIO_PER_REG);
-	}
-
 	for (i = 0; i < nr_irq_parent; i++) {
 		int irq = irq_of_parse_and_map(np, i);
 
diff --git a/drivers/pinctrl/nomadik/pinctrl-abx500.c b/drivers/pinctrl/nomadik/pinctrl-abx500.c
index f95001bc1d58..b32c0d602024 100644
--- a/drivers/pinctrl/nomadik/pinctrl-abx500.c
+++ b/drivers/pinctrl/nomadik/pinctrl-abx500.c
@@ -647,7 +647,7 @@ static inline void abx500_gpio_dbg_show_one(struct seq_file *s,
 #define abx500_gpio_dbg_show	NULL
 #endif
 
-static struct gpio_chip abx500gpio_chip = {
+static const struct gpio_chip abx500gpio_chip = {
 	.label			= "abx500-gpio",
 	.owner			= THIS_MODULE,
 	.request		= gpiochip_generic_request,
diff --git a/drivers/pinctrl/nomadik/pinctrl-nomadik.c b/drivers/pinctrl/nomadik/pinctrl-nomadik.c
index d318ca055489..a53f1a9b1ed2 100644
--- a/drivers/pinctrl/nomadik/pinctrl-nomadik.c
+++ b/drivers/pinctrl/nomadik/pinctrl-nomadik.c
@@ -1078,7 +1078,7 @@ static struct nmk_gpio_chip *nmk_gpio_populate_chip(struct device_node *np,
 	res = platform_get_resource(gpio_pdev, IORESOURCE_MEM, 0);
 	base = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(base))
-		return base;
+		return ERR_CAST(base);
 	nmk_chip->addr = base;
 
 	clk = clk_get(&gpio_pdev->dev, NULL);
diff --git a/drivers/pinctrl/pinconf-generic.c b/drivers/pinctrl/pinconf-generic.c
index fc0c230aa11f..8eaa25c3384f 100644
--- a/drivers/pinctrl/pinconf-generic.c
+++ b/drivers/pinctrl/pinconf-generic.c
@@ -47,6 +47,7 @@ static const struct pin_config_item conf_items[] = {
 	PCONFDUMP(PIN_CONFIG_OUTPUT_ENABLE, "output enabled", NULL, false),
 	PCONFDUMP(PIN_CONFIG_OUTPUT, "pin output", "level", true),
 	PCONFDUMP(PIN_CONFIG_POWER_SOURCE, "pin power source", "selector", true),
+	PCONFDUMP(PIN_CONFIG_SLEEP_HARDWARE_STATE, "sleep hardware state", NULL, false),
 	PCONFDUMP(PIN_CONFIG_SLEW_RATE, "slew rate", NULL, true),
 };
 
@@ -178,6 +179,7 @@ static const struct pinconf_generic_params dt_params[] = {
 	{ "output-high", PIN_CONFIG_OUTPUT, 1, },
 	{ "output-low", PIN_CONFIG_OUTPUT, 0, },
 	{ "power-source", PIN_CONFIG_POWER_SOURCE, 0 },
+	{ "sleep-hardware-state", PIN_CONFIG_SLEEP_HARDWARE_STATE, 0 },
 	{ "slew-rate", PIN_CONFIG_SLEW_RATE, 0 },
 };
 
@@ -316,16 +318,15 @@ int pinconf_generic_dt_subnode_to_map(struct pinctrl_dev *pctldev,
 	if (ret < 0) {
 		/* EINVAL=missing, which is fine since it's optional */
 		if (ret != -EINVAL)
-			dev_err(dev, "%s: could not parse property function\n",
-				of_node_full_name(np));
+			dev_err(dev, "%pOF: could not parse property function\n",
+				np);
 		function = NULL;
 	}
 
 	ret = pinconf_generic_parse_dt_config(np, pctldev, &configs,
 					      &num_configs);
 	if (ret < 0) {
-		dev_err(dev, "%s: could not parse node property\n",
-			of_node_full_name(np));
+		dev_err(dev, "%pOF: could not parse node property\n", np);
 		return ret;
 	}
 
diff --git a/drivers/pinctrl/pinconf.c b/drivers/pinctrl/pinconf.c
index 7fc417e4ae96..d3fe14394b73 100644
--- a/drivers/pinctrl/pinconf.c
+++ b/drivers/pinctrl/pinconf.c
@@ -37,7 +37,7 @@ int pinconf_check_ops(struct pinctrl_dev *pctldev)
 	return 0;
 }
 
-int pinconf_validate_map(struct pinctrl_map const *map, int i)
+int pinconf_validate_map(const struct pinctrl_map *map, int i)
 {
 	if (!map->data.configs.group_or_pin) {
 		pr_err("failed to register map %s (%d): no group/pin given\n",
@@ -106,7 +106,7 @@ unlock:
 	return ret;
 }
 
-int pinconf_map_to_setting(struct pinctrl_map const *map,
+int pinconf_map_to_setting(const struct pinctrl_map *map,
 			  struct pinctrl_setting *setting)
 {
 	struct pinctrl_dev *pctldev = setting->pctldev;
@@ -143,11 +143,11 @@ int pinconf_map_to_setting(struct pinctrl_map const *map,
 	return 0;
 }
 
-void pinconf_free_setting(struct pinctrl_setting const *setting)
+void pinconf_free_setting(const struct pinctrl_setting *setting)
 {
 }
 
-int pinconf_apply_setting(struct pinctrl_setting const *setting)
+int pinconf_apply_setting(const struct pinctrl_setting *setting)
 {
 	struct pinctrl_dev *pctldev = setting->pctldev;
 	const struct pinconf_ops *ops = pctldev->desc->confops;
@@ -205,7 +205,7 @@ int pinconf_set_config(struct pinctrl_dev *pctldev, unsigned pin,
 	const struct pinconf_ops *ops;
 
 	ops = pctldev->desc->confops;
-	if (!ops)
+	if (!ops || !ops->pin_config_set)
 		return -ENOTSUPP;
 
 	return ops->pin_config_set(pctldev, pin, configs, nconfigs);
@@ -235,7 +235,7 @@ static void pinconf_show_config(struct seq_file *s, struct pinctrl_dev *pctldev,
 	}
 }
 
-void pinconf_show_map(struct seq_file *s, struct pinctrl_map const *map)
+void pinconf_show_map(struct seq_file *s, const struct pinctrl_map *map)
 {
 	struct pinctrl_dev *pctldev;
 
@@ -259,7 +259,7 @@ void pinconf_show_map(struct seq_file *s, struct pinctrl_map const *map)
 }
 
 void pinconf_show_setting(struct seq_file *s,
-			  struct pinctrl_setting const *setting)
+			  const struct pinctrl_setting *setting)
 {
 	struct pinctrl_dev *pctldev = setting->pctldev;
 	const struct pinctrl_ops *pctlops = pctldev->desc->pctlops;
diff --git a/drivers/pinctrl/pinconf.h b/drivers/pinctrl/pinconf.h
index bf8aff9abf32..6c722505f893 100644
--- a/drivers/pinctrl/pinconf.h
+++ b/drivers/pinctrl/pinconf.h
@@ -14,11 +14,11 @@
 #ifdef CONFIG_PINCONF
 
 int pinconf_check_ops(struct pinctrl_dev *pctldev);
-int pinconf_validate_map(struct pinctrl_map const *map, int i);
-int pinconf_map_to_setting(struct pinctrl_map const *map,
+int pinconf_validate_map(const struct pinctrl_map *map, int i);
+int pinconf_map_to_setting(const struct pinctrl_map *map,
 			  struct pinctrl_setting *setting);
-void pinconf_free_setting(struct pinctrl_setting const *setting);
-int pinconf_apply_setting(struct pinctrl_setting const *setting);
+void pinconf_free_setting(const struct pinctrl_setting *setting);
+int pinconf_apply_setting(const struct pinctrl_setting *setting);
 
 int pinconf_set_config(struct pinctrl_dev *pctldev, unsigned pin,
 		       unsigned long *configs, size_t nconfigs);
@@ -39,22 +39,22 @@ static inline int pinconf_check_ops(struct pinctrl_dev *pctldev)
 	return 0;
 }
 
-static inline int pinconf_validate_map(struct pinctrl_map const *map, int i)
+static inline int pinconf_validate_map(const struct pinctrl_map *map, int i)
 {
 	return 0;
 }
 
-static inline int pinconf_map_to_setting(struct pinctrl_map const *map,
+static inline int pinconf_map_to_setting(const struct pinctrl_map *map,
 			  struct pinctrl_setting *setting)
 {
 	return 0;
 }
 
-static inline void pinconf_free_setting(struct pinctrl_setting const *setting)
+static inline void pinconf_free_setting(const struct pinctrl_setting *setting)
 {
 }
 
-static inline int pinconf_apply_setting(struct pinctrl_setting const *setting)
+static inline int pinconf_apply_setting(const struct pinctrl_setting *setting)
 {
 	return 0;
 }
@@ -69,21 +69,21 @@ static inline int pinconf_set_config(struct pinctrl_dev *pctldev, unsigned pin,
 
 #if defined(CONFIG_PINCONF) && defined(CONFIG_DEBUG_FS)
 
-void pinconf_show_map(struct seq_file *s, struct pinctrl_map const *map);
+void pinconf_show_map(struct seq_file *s, const struct pinctrl_map *map);
 void pinconf_show_setting(struct seq_file *s,
-			  struct pinctrl_setting const *setting);
+			  const struct pinctrl_setting *setting);
 void pinconf_init_device_debugfs(struct dentry *devroot,
 				 struct pinctrl_dev *pctldev);
 
 #else
 
 static inline void pinconf_show_map(struct seq_file *s,
-				    struct pinctrl_map const *map)
+				    const struct pinctrl_map *map)
 {
 }
 
 static inline void pinconf_show_setting(struct seq_file *s,
-			  struct pinctrl_setting const *setting)
+					const struct pinctrl_setting *setting)
 {
 }
 
diff --git a/drivers/pinctrl/pinctrl-adi2.c b/drivers/pinctrl/pinctrl-adi2.c
index 54569a7eac59..56aa181084ac 100644
--- a/drivers/pinctrl/pinctrl-adi2.c
+++ b/drivers/pinctrl/pinctrl-adi2.c
@@ -612,7 +612,7 @@ static int adi_get_group_pins(struct pinctrl_dev *pctldev, unsigned selector,
 	return 0;
 }
 
-static struct pinctrl_ops adi_pctrl_ops = {
+static const struct pinctrl_ops adi_pctrl_ops = {
 	.get_groups_count = adi_get_groups_count,
 	.get_group_name = adi_get_group_name,
 	.get_group_pins = adi_get_group_pins,
@@ -696,7 +696,7 @@ static int adi_pinmux_request_gpio(struct pinctrl_dev *pctldev,
 	return 0;
 }
 
-static struct pinmux_ops adi_pinmux_ops = {
+static const struct pinmux_ops adi_pinmux_ops = {
 	.set_mux = adi_pinmux_set,
 	.get_functions_count = adi_pinmux_get_funcs_count,
 	.get_function_name = adi_pinmux_get_func_name,
diff --git a/drivers/pinctrl/pinctrl-amd.c b/drivers/pinctrl/pinctrl-amd.c
index e6779d4352a2..3f6b34febbf1 100644
--- a/drivers/pinctrl/pinctrl-amd.c
+++ b/drivers/pinctrl/pinctrl-amd.c
@@ -36,6 +36,7 @@
 #include <linux/pinctrl/pinconf.h>
 #include <linux/pinctrl/pinconf-generic.h>
 
+#include "core.h"
 #include "pinctrl-utils.h"
 #include "pinctrl-amd.h"
 
@@ -725,6 +726,69 @@ static const struct pinconf_ops amd_pinconf_ops = {
 	.pin_config_group_set = amd_pinconf_group_set,
 };
 
+#ifdef CONFIG_PM_SLEEP
+static bool amd_gpio_should_save(struct amd_gpio *gpio_dev, unsigned int pin)
+{
+	const struct pin_desc *pd = pin_desc_get(gpio_dev->pctrl, pin);
+
+	if (!pd)
+		return false;
+
+	/*
+	 * Only restore the pin if it is actually in use by the kernel (or
+	 * by userspace).
+	 */
+	if (pd->mux_owner || pd->gpio_owner ||
+	    gpiochip_line_is_irq(&gpio_dev->gc, pin))
+		return true;
+
+	return false;
+}
+
+int amd_gpio_suspend(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct amd_gpio *gpio_dev = platform_get_drvdata(pdev);
+	struct pinctrl_desc *desc = gpio_dev->pctrl->desc;
+	int i;
+
+	for (i = 0; i < desc->npins; i++) {
+		int pin = desc->pins[i].number;
+
+		if (!amd_gpio_should_save(gpio_dev, pin))
+			continue;
+
+		gpio_dev->saved_regs[i] = readl(gpio_dev->base + pin*4);
+	}
+
+	return 0;
+}
+
+int amd_gpio_resume(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct amd_gpio *gpio_dev = platform_get_drvdata(pdev);
+	struct pinctrl_desc *desc = gpio_dev->pctrl->desc;
+	int i;
+
+	for (i = 0; i < desc->npins; i++) {
+		int pin = desc->pins[i].number;
+
+		if (!amd_gpio_should_save(gpio_dev, pin))
+			continue;
+
+		writel(gpio_dev->saved_regs[i], gpio_dev->base + pin*4);
+	}
+
+	return 0;
+}
+
+static const struct dev_pm_ops amd_gpio_pm_ops = {
+	SET_LATE_SYSTEM_SLEEP_PM_OPS(amd_gpio_suspend,
+				     amd_gpio_resume)
+};
+#endif
+
 static struct pinctrl_desc amd_pinctrl_desc = {
 	.pins	= kerncz_pins,
 	.npins = ARRAY_SIZE(kerncz_pins),
@@ -760,10 +824,18 @@ static int amd_gpio_probe(struct platform_device *pdev)
 
 	irq_base = platform_get_irq(pdev, 0);
 	if (irq_base < 0) {
-		dev_err(&pdev->dev, "Failed to get gpio IRQ.\n");
-		return -EINVAL;
+		dev_err(&pdev->dev, "Failed to get gpio IRQ: %d\n", irq_base);
+		return irq_base;
 	}
 
+#ifdef CONFIG_PM_SLEEP
+	gpio_dev->saved_regs = devm_kcalloc(&pdev->dev, amd_pinctrl_desc.npins,
+					    sizeof(*gpio_dev->saved_regs),
+					    GFP_KERNEL);
+	if (!gpio_dev->saved_regs)
+		return -ENOMEM;
+#endif
+
 	gpio_dev->pdev = pdev;
 	gpio_dev->gc.direction_input	= amd_gpio_direction_input;
 	gpio_dev->gc.direction_output	= amd_gpio_direction_output;
@@ -853,6 +925,9 @@ static struct platform_driver amd_gpio_driver = {
 	.driver		= {
 		.name	= "amd_gpio",
 		.acpi_match_table = ACPI_PTR(amd_gpio_acpi_match),
+#ifdef CONFIG_PM_SLEEP
+		.pm	= &amd_gpio_pm_ops,
+#endif
 	},
 	.probe		= amd_gpio_probe,
 	.remove		= amd_gpio_remove,
diff --git a/drivers/pinctrl/pinctrl-amd.h b/drivers/pinctrl/pinctrl-amd.h
index 5b1cb965c767..8fa453a59da5 100644
--- a/drivers/pinctrl/pinctrl-amd.h
+++ b/drivers/pinctrl/pinctrl-amd.h
@@ -97,6 +97,7 @@ struct amd_gpio {
 	unsigned int            hwbank_num;
 	struct resource         *res;
 	struct platform_device  *pdev;
+	u32			*saved_regs;
 };
 
 /*  KERNCZ configuration*/
diff --git a/drivers/pinctrl/pinctrl-artpec6.c b/drivers/pinctrl/pinctrl-artpec6.c
index 357516d524bd..e33781cd0a05 100644
--- a/drivers/pinctrl/pinctrl-artpec6.c
+++ b/drivers/pinctrl/pinctrl-artpec6.c
@@ -445,7 +445,7 @@ static unsigned int artpec6_pconf_drive_field_to_mA(int field)
 	}
 }
 
-static struct pinctrl_ops artpec6_pctrl_ops = {
+static const struct pinctrl_ops artpec6_pctrl_ops = {
 	.get_group_pins		= artpec6_get_group_pins,
 	.get_groups_count	= artpec6_get_groups_count,
 	.get_group_name		= artpec6_get_group_name,
diff --git a/drivers/pinctrl/pinctrl-at91-pio4.c b/drivers/pinctrl/pinctrl-at91-pio4.c
index dc8591543dee..b1ca838dd80a 100644
--- a/drivers/pinctrl/pinctrl-at91-pio4.c
+++ b/drivers/pinctrl/pinctrl-at91-pio4.c
@@ -494,8 +494,8 @@ static int atmel_pctl_dt_subnode_to_map(struct pinctrl_dev *pctldev,
 	ret = pinconf_generic_parse_dt_config(np, pctldev, &configs,
 					      &num_configs);
 	if (ret < 0) {
-		dev_err(pctldev->dev, "%s: could not parse node property\n",
-			of_node_full_name(np));
+		dev_err(pctldev->dev, "%pOF: could not parse node property\n",
+			np);
 		return ret;
 	}
 
@@ -504,8 +504,7 @@ static int atmel_pctl_dt_subnode_to_map(struct pinctrl_dev *pctldev,
 
 	num_pins = pins->length / sizeof(u32);
 	if (!num_pins) {
-		dev_err(pctldev->dev, "no pins found in node %s\n",
-			of_node_full_name(np));
+		dev_err(pctldev->dev, "no pins found in node %pOF\n", np);
 		ret = -EINVAL;
 		goto exit;
 	}
@@ -584,8 +583,8 @@ static int atmel_pctl_dt_node_to_map(struct pinctrl_dev *pctldev,
 
 	if (ret < 0) {
 		pinctrl_utils_free_map(pctldev, *map, *num_maps);
-		dev_err(pctldev->dev, "can't create maps for node %s\n",
-			np_config->full_name);
+		dev_err(pctldev->dev, "can't create maps for node %pOF\n",
+			np_config);
 	}
 
 	return ret;
diff --git a/drivers/pinctrl/pinctrl-coh901.c b/drivers/pinctrl/pinctrl-coh901.c
index 741b39eaeb8b..ac155e7d3412 100644
--- a/drivers/pinctrl/pinctrl-coh901.c
+++ b/drivers/pinctrl/pinctrl-coh901.c
@@ -387,7 +387,7 @@ int u300_gpio_config_set(struct gpio_chip *chip, unsigned offset,
 	return 0;
 }
 
-static struct gpio_chip u300_gpio_chip = {
+static const struct gpio_chip u300_gpio_chip = {
 	.label			= "u300-gpio-chip",
 	.owner			= THIS_MODULE,
 	.request		= gpiochip_generic_request,
diff --git a/drivers/pinctrl/pinctrl-digicolor.c b/drivers/pinctrl/pinctrl-digicolor.c
index 639a57ecc7c2..ce269ced4d49 100644
--- a/drivers/pinctrl/pinctrl-digicolor.c
+++ b/drivers/pinctrl/pinctrl-digicolor.c
@@ -79,7 +79,7 @@ static int dc_get_group_pins(struct pinctrl_dev *pctldev, unsigned selector,
 	return 0;
 }
 
-static struct pinctrl_ops dc_pinctrl_ops = {
+static const struct pinctrl_ops dc_pinctrl_ops = {
 	.get_groups_count	= dc_get_groups_count,
 	.get_group_name		= dc_get_group_name,
 	.get_group_pins		= dc_get_group_pins,
@@ -161,7 +161,7 @@ static int dc_pmx_request_gpio(struct pinctrl_dev *pcdev,
 	return 0;
 }
 
-static struct pinmux_ops dc_pmxops = {
+static const struct pinmux_ops dc_pmxops = {
 	.get_functions_count	= dc_get_functions_count,
 	.get_function_name	= dc_get_fname,
 	.get_function_groups	= dc_get_groups,
diff --git a/drivers/pinctrl/pinctrl-gemini.c b/drivers/pinctrl/pinctrl-gemini.c
new file mode 100644
index 000000000000..39e6221e7100
--- /dev/null
+++ b/drivers/pinctrl/pinctrl-gemini.c
@@ -0,0 +1,2359 @@
+/*
+ * Driver for the Gemini pin controller
+ *
+ * Copyright (C) 2017 Linus Walleij <linus.walleij@linaro.org>
+ *
+ * This is a group-only pin controller.
+ */
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/mfd/syscon.h>
+#include <linux/of.h>
+#include <linux/pinctrl/machine.h>
+#include <linux/pinctrl/pinctrl.h>
+#include <linux/pinctrl/pinmux.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/regmap.h>
+
+#include "pinctrl-utils.h"
+
+#define DRIVER_NAME "pinctrl-gemini"
+
+/**
+ * @dev: a pointer back to containing device
+ * @virtbase: the offset to the controller in virtual memory
+ * @map: regmap to access registers
+ * @is_3512: whether the SoC/package is the 3512 variant
+ * @is_3516: whether the SoC/package is the 3516 variant
+ * @flash_pin: whether the flash pin (extended pins for parallel
+ * flash) is set
+ */
+struct gemini_pmx {
+	struct device *dev;
+	struct pinctrl_dev *pctl;
+	struct regmap *map;
+	bool is_3512;
+	bool is_3516;
+	bool flash_pin;
+};
+
+/**
+ * struct gemini_pin_group - describes a Gemini pin group
+ * @name: the name of this specific pin group
+ * @pins: an array of discrete physical pins used in this group, taken
+ *	from the driver-local pin enumeration space
+ * @num_pins: the number of pins in this group array, i.e. the number of
+ *	elements in .pins so we can iterate over that array
+ * @mask: bits to clear to enable this when doing pin muxing
+ * @value: bits to set to enable this when doing pin muxing
+ */
+struct gemini_pin_group {
+	const char *name;
+	const unsigned int *pins;
+	const unsigned int num_pins;
+	u32 mask;
+	u32 value;
+};
+
+/*
+ * Global Miscellaneous Control Register
+ * This register controls all Gemini pad/pin multiplexing
+ *
+ * It is a tricky register though:
+ * - For the bits named *_ENABLE, once you DISABLE something, it simply cannot
+ *   be brought back online, so it means permanent disablement of the
+ *   corresponding pads.
+ * - For the bits named *_DISABLE, once you enable something, it cannot be
+ *   DISABLED again. So you select a flash configuration once, and then
+ *   you are stuck with it.
+ */
+#define GLOBAL_WORD_ID		0x00
+#define GLOBAL_STATUS		0x04
+#define GLOBAL_STATUS_FLPIN	BIT(20)
+#define GLOBAL_MISC_CTRL	0x30
+#define TVC_CLK_PAD_ENABLE	BIT(20)
+#define PCI_CLK_PAD_ENABLE	BIT(17)
+#define LPC_CLK_PAD_ENABLE	BIT(16)
+#define TVC_PADS_ENABLE		BIT(9)
+#define SSP_PADS_ENABLE		BIT(8)
+#define LCD_PADS_ENABLE		BIT(7)
+#define LPC_PADS_ENABLE		BIT(6)
+#define PCI_PADS_ENABLE		BIT(5)
+#define IDE_PADS_ENABLE		BIT(4)
+#define DRAM_PADS_POWERDOWN	BIT(3)
+#define NAND_PADS_DISABLE	BIT(2)
+#define PFLASH_PADS_DISABLE	BIT(1)
+#define SFLASH_PADS_DISABLE	BIT(0)
+#define PADS_MASK		(GENMASK(9, 0) | BIT(16) | BIT(17) | BIT(20))
+#define PADS_MAXBIT		20
+
+/* Ordered by bit index */
+static const char * const gemini_padgroups[] = {
+	"serial flash",
+	"parallel flash",
+	"NAND flash",
+	"DRAM",
+	"IDE",
+	"PCI",
+	"LPC",
+	"LCD",
+	"SSP",
+	"TVC",
+	NULL, NULL, NULL, NULL, NULL, NULL,
+	"LPC CLK",
+	"PCI CLK",
+	NULL, NULL,
+	"TVC CLK",
+};
+
+static const struct pinctrl_pin_desc gemini_3512_pins[] = {
+	/* Row A */
+	PINCTRL_PIN(0, "A1 VREF CTRL"),
+	PINCTRL_PIN(1, "A2 VCC2IO CTRL"),
+	PINCTRL_PIN(2, "A3 DRAM CK"),
+	PINCTRL_PIN(3, "A4 DRAM CK N"),
+	PINCTRL_PIN(4, "A5 DRAM A5"),
+	PINCTRL_PIN(5, "A6 DRAM CKE"),
+	PINCTRL_PIN(6, "A7 DRAM DQ11"),
+	PINCTRL_PIN(7, "A8 DRAM DQ0"),
+	PINCTRL_PIN(8, "A9 DRAM DQ5"),
+	PINCTRL_PIN(9, "A10 DRAM DQ6"),
+	PINCTRL_PIN(10, "A11 DRAM DRAM VREF"),
+	PINCTRL_PIN(11, "A12 DRAM BA1"),
+	PINCTRL_PIN(12, "A13 DRAM A2"),
+	PINCTRL_PIN(13, "A14 PCI GNT1 N"),
+	PINCTRL_PIN(14, "A15 PCI REQ9 N"),
+	PINCTRL_PIN(15, "A16 PCI REQ2 N"),
+	PINCTRL_PIN(16, "A17 PCI REQ3 N"),
+	PINCTRL_PIN(17, "A18 PCI AD31"),
+	/* Row B */
+	PINCTRL_PIN(18, "B1 VCCK CTRL"),
+	PINCTRL_PIN(19, "B2 PWR EN"),
+	PINCTRL_PIN(20, "B3 RTC CLKI"),
+	PINCTRL_PIN(21, "B4 DRAM A4"),
+	PINCTRL_PIN(22, "B5 DRAM A6"),
+	PINCTRL_PIN(23, "B6 DRAM A12"),
+	PINCTRL_PIN(24, "B7 DRAM DQS1"),
+	PINCTRL_PIN(25, "B8 DRAM DQ15"),
+	PINCTRL_PIN(26, "B9 DRAM DQ4"),
+	PINCTRL_PIN(27, "B10 DRAM DQS0"),
+	PINCTRL_PIN(28, "B11 DRAM WE N"),
+	PINCTRL_PIN(29, "B12 DRAM A10"),
+	PINCTRL_PIN(30, "B13 DRAM A3"),
+	PINCTRL_PIN(31, "B14 PCI GNT0 N"),
+	PINCTRL_PIN(32, "B15 PCI GNT3 N"),
+	PINCTRL_PIN(33, "B16 PCI REQ1 N"),
+	PINCTRL_PIN(34, "B17 PCI AD30"),
+	PINCTRL_PIN(35, "B18 PCI AD29"),
+	/* Row C */
+	PINCTRL_PIN(36, "C1 CIR RST N"), /* REALLY? CIR is not in 3512... */
+	PINCTRL_PIN(37, "C2 XTALI"),
+	PINCTRL_PIN(38, "C3 PWR BTN"),
+	PINCTRL_PIN(39, "C4 RTC CLKO"),
+	PINCTRL_PIN(40, "C5 DRAM A7"),
+	PINCTRL_PIN(41, "C6 DRAM A11"),
+	PINCTRL_PIN(42, "C7 DRAM DQ10"),
+	PINCTRL_PIN(43, "C8 DRAM DQ14"),
+	PINCTRL_PIN(44, "C9 DRAM DQ3"),
+	PINCTRL_PIN(45, "C10 DRAM DQ7"),
+	PINCTRL_PIN(46, "C11 DRAM CAS N"),
+	PINCTRL_PIN(47, "C12 DRAM A0"),
+	PINCTRL_PIN(48, "C13 PCI INT0 N"),
+	PINCTRL_PIN(49, "C14 EXT RESET N"),
+	PINCTRL_PIN(50, "C15 PCI GNT2 N"),
+	PINCTRL_PIN(51, "C16 PCI AD28"),
+	PINCTRL_PIN(52, "C17 PCI AD27"),
+	PINCTRL_PIN(53, "C18 PCI AD26"),
+	/* Row D */
+	PINCTRL_PIN(54, "D1 AVCCKHA"),
+	PINCTRL_PIN(55, "D2 AGNDIOHA"),
+	PINCTRL_PIN(56, "D3 XTALO"),
+	PINCTRL_PIN(57, "D4 AVCC3IOHA"),
+	PINCTRL_PIN(58, "D5 DRAM A8"),
+	PINCTRL_PIN(59, "D6 DRAM A9"),
+	PINCTRL_PIN(60, "D7 DRAM DQ9"),
+	PINCTRL_PIN(61, "D8 DRAM DQ13"),
+	PINCTRL_PIN(62, "D9 DRAM DQ2"),
+	PINCTRL_PIN(63, "D10 DRAM A13"),
+	PINCTRL_PIN(64, "D11 DRAM RAS N"),
+	PINCTRL_PIN(65, "D12 DRAM A1"),
+	PINCTRL_PIN(66, "D13 PCI INTC N"),
+	PINCTRL_PIN(67, "D14 PCI CLK"),
+	PINCTRL_PIN(68, "D15 PCI AD25"),
+	PINCTRL_PIN(69, "D16 PCI AD24"),
+	PINCTRL_PIN(70, "D17 PCI CBE3 N"),
+	PINCTRL_PIN(71, "D18 PCI AD23"),
+	/* Row E */
+	PINCTRL_PIN(72, "E1 AVCC3IOHA"),
+	PINCTRL_PIN(73, "E2 EBG"),
+	PINCTRL_PIN(74, "E3 AVCC3IOHB"),
+	PINCTRL_PIN(75, "E4 REXT"),
+	PINCTRL_PIN(76, "E5 GND"),
+	PINCTRL_PIN(77, "E6 DRAM DQM1"),
+	PINCTRL_PIN(78, "E7 DRAM DQ8"),
+	PINCTRL_PIN(79, "E8 DRAM DQ12"),
+	PINCTRL_PIN(80, "E9 DRAM DQ1"),
+	PINCTRL_PIN(81, "E10 DRAM DQM0"),
+	PINCTRL_PIN(82, "E11 DRAM BA0"),
+	PINCTRL_PIN(83, "E12 PCI INTA N"),
+	PINCTRL_PIN(84, "E13 PCI INTB N"),
+	PINCTRL_PIN(85, "E14 GND"),
+	PINCTRL_PIN(86, "E15 PCI AD22"),
+	PINCTRL_PIN(87, "E16 PCI AD21"),
+	PINCTRL_PIN(88, "E17 PCI AD20"),
+	PINCTRL_PIN(89, "E18 PCI AD19"),
+	/* Row F */
+	PINCTRL_PIN(90, "F1 SATA0 RXDP"),
+	PINCTRL_PIN(91, "F2 SATA0 RXDN"),
+	PINCTRL_PIN(92, "F3 AGNDK 0"),
+	PINCTRL_PIN(93, "F4 AVCC3 S"),
+	PINCTRL_PIN(94, "F5 AVCCK P"),
+	PINCTRL_PIN(95, "F6 GND"),
+	PINCTRL_PIN(96, "F7 VCC2IOHA 2"),
+	PINCTRL_PIN(97, "F8 VCC2IOHA 2"),
+	PINCTRL_PIN(98, "F9 V1"),
+	PINCTRL_PIN(99, "F10 V1"),
+	PINCTRL_PIN(100, "F11 VCC2IOHA 2"),
+	PINCTRL_PIN(101, "F12 VCC2IOHA 2"),
+	PINCTRL_PIN(102, "F13 GND"),
+	PINCTRL_PIN(103, "F14 PCI AD18"),
+	PINCTRL_PIN(104, "F15 PCI AD17"),
+	PINCTRL_PIN(105, "F16 PCI AD16"),
+	PINCTRL_PIN(106, "F17 PCI CBE2 N"),
+	PINCTRL_PIN(107, "F18 PCI FRAME N"),
+	/* Row G */
+	PINCTRL_PIN(108, "G1 SATA0 TXDP"),
+	PINCTRL_PIN(109, "G2 SATA0 TXDN"),
+	PINCTRL_PIN(110, "G3 AGNDK 1"),
+	PINCTRL_PIN(111, "G4 AVCCK 0"),
+	PINCTRL_PIN(112, "G5 TEST CLKOUT"),
+	PINCTRL_PIN(113, "G6 AGND"),
+	PINCTRL_PIN(114, "G7 GND"),
+	PINCTRL_PIN(115, "G8 VCC2IOHA 2"),
+	PINCTRL_PIN(116, "G9 V1"),
+	PINCTRL_PIN(117, "G10 V1"),
+	PINCTRL_PIN(118, "G11 VCC2IOHA 2"),
+	PINCTRL_PIN(119, "G12 GND"),
+	PINCTRL_PIN(120, "G13 VCC3IOHA"),
+	PINCTRL_PIN(121, "G14 PCI IRDY N"),
+	PINCTRL_PIN(122, "G15 PCI TRDY N"),
+	PINCTRL_PIN(123, "G16 PCI DEVSEL N"),
+	PINCTRL_PIN(124, "G17 PCI STOP N"),
+	PINCTRL_PIN(125, "G18 PCI PAR"),
+	/* Row H */
+	PINCTRL_PIN(126, "H1 SATA1 TXDP"),
+	PINCTRL_PIN(127, "H2 SATA1 TXDN"),
+	PINCTRL_PIN(128, "H3 AGNDK 2"),
+	PINCTRL_PIN(129, "H4 AVCCK 1"),
+	PINCTRL_PIN(130, "H5 AVCCK S"),
+	PINCTRL_PIN(131, "H6 AVCCKHB"),
+	PINCTRL_PIN(132, "H7 AGND"),
+	PINCTRL_PIN(133, "H8 GND"),
+	PINCTRL_PIN(134, "H9 GND"),
+	PINCTRL_PIN(135, "H10 GND"),
+	PINCTRL_PIN(136, "H11 GND"),
+	PINCTRL_PIN(137, "H12 VCC3IOHA"),
+	PINCTRL_PIN(138, "H13 VCC3IOHA"),
+	PINCTRL_PIN(139, "H14 PCI CBE1 N"),
+	PINCTRL_PIN(140, "H15 PCI AD15"),
+	PINCTRL_PIN(141, "H16 PCI AD14"),
+	PINCTRL_PIN(142, "H17 PCI AD13"),
+	PINCTRL_PIN(143, "H18 PCI AD12"),
+	/* Row J (for some reason I is skipped) */
+	PINCTRL_PIN(144, "J1 SATA1 RXDP"),
+	PINCTRL_PIN(145, "J2 SATA1 RXDN"),
+	PINCTRL_PIN(146, "J3 AGNDK 3"),
+	PINCTRL_PIN(147, "J4 AVCCK 2"),
+	PINCTRL_PIN(148, "J5 IDE DA1"),
+	PINCTRL_PIN(149, "J6 V1"),
+	PINCTRL_PIN(150, "J7 V1"),
+	PINCTRL_PIN(151, "J8 GND"),
+	PINCTRL_PIN(152, "J9 GND"),
+	PINCTRL_PIN(153, "J10 GND"),
+	PINCTRL_PIN(154, "J11 GND"),
+	PINCTRL_PIN(155, "J12 V1"),
+	PINCTRL_PIN(156, "J13 V1"),
+	PINCTRL_PIN(157, "J14 PCI AD11"),
+	PINCTRL_PIN(158, "J15 PCI AD10"),
+	PINCTRL_PIN(159, "J16 PCI AD9"),
+	PINCTRL_PIN(160, "J17 PCI AD8"),
+	PINCTRL_PIN(161, "J18 PCI CBE0 N"),
+	/* Row K */
+	PINCTRL_PIN(162, "K1 IDE CS1 N"),
+	PINCTRL_PIN(163, "K2 IDE CS0 N"),
+	PINCTRL_PIN(164, "K3 AVCCK 3"),
+	PINCTRL_PIN(165, "K4 IDE DA2"),
+	PINCTRL_PIN(166, "K5 IDE DA0"),
+	PINCTRL_PIN(167, "K6 V1"),
+	PINCTRL_PIN(168, "K7 V1"),
+	PINCTRL_PIN(169, "K8 GND"),
+	PINCTRL_PIN(170, "K9 GND"),
+	PINCTRL_PIN(171, "K10 GND"),
+	PINCTRL_PIN(172, "K11 GND"),
+	PINCTRL_PIN(173, "K12 V1"),
+	PINCTRL_PIN(174, "K13 V1"),
+	PINCTRL_PIN(175, "K14 PCI AD3"),
+	PINCTRL_PIN(176, "K15 PCI AD4"),
+	PINCTRL_PIN(177, "K16 PCI AD5"),
+	PINCTRL_PIN(178, "K17 PCI AD6"),
+	PINCTRL_PIN(179, "K18 PCI AD7"),
+	/* Row L */
+	PINCTRL_PIN(180, "L1 IDE INTRQ"),
+	PINCTRL_PIN(181, "L2 IDE DMACK N"),
+	PINCTRL_PIN(182, "L3 IDE IORDY"),
+	PINCTRL_PIN(183, "L4 IDE DIOR N"),
+	PINCTRL_PIN(184, "L5 IDE DIOW N"),
+	PINCTRL_PIN(185, "L6 VCC3IOHA"),
+	PINCTRL_PIN(186, "L7 VCC3IOHA"),
+	PINCTRL_PIN(187, "L8 GND"),
+	PINCTRL_PIN(188, "L9 GND"),
+	PINCTRL_PIN(189, "L10 GND"),
+	PINCTRL_PIN(190, "L11 GND"),
+	PINCTRL_PIN(191, "L12 VCC3IOHA"),
+	PINCTRL_PIN(192, "L13 VCC3IOHA"),
+	PINCTRL_PIN(193, "L14 GPIO0 30"),
+	PINCTRL_PIN(194, "L15 GPIO0 31"),
+	PINCTRL_PIN(195, "L16 PCI AD0"),
+	PINCTRL_PIN(196, "L17 PCI AD1"),
+	PINCTRL_PIN(197, "L18 PCI AD2"),
+	/* Row M */
+	PINCTRL_PIN(198, "M1 IDE DMARQ"),
+	PINCTRL_PIN(199, "M2 IDE DD15"),
+	PINCTRL_PIN(200, "M3 IDE DD0"),
+	PINCTRL_PIN(201, "M4 IDE DD14"),
+	PINCTRL_PIN(202, "M5 IDE DD1"),
+	PINCTRL_PIN(203, "M6 VCC3IOHA"),
+	PINCTRL_PIN(204, "M7 GND"),
+	PINCTRL_PIN(205, "M8 VCC2IOHA 1"),
+	PINCTRL_PIN(206, "M9 V1"),
+	PINCTRL_PIN(207, "M10 V1"),
+	PINCTRL_PIN(208, "M11 VCC3IOHA"),
+	PINCTRL_PIN(209, "M12 GND"),
+	PINCTRL_PIN(210, "M13 VCC3IOHA"),
+	PINCTRL_PIN(211, "M14 GPIO0 25"),
+	PINCTRL_PIN(212, "M15 GPIO0 26"),
+	PINCTRL_PIN(213, "M16 GPIO0 27"),
+	PINCTRL_PIN(214, "M17 GPIO0 28"),
+	PINCTRL_PIN(215, "M18 GPIO0 29"),
+	/* Row N */
+	PINCTRL_PIN(216, "N1 IDE DD13"),
+	PINCTRL_PIN(217, "N2 IDE DD2"),
+	PINCTRL_PIN(218, "N3 IDE DD12"),
+	PINCTRL_PIN(219, "N4 IDE DD3"),
+	PINCTRL_PIN(220, "N5 IDE DD11"),
+	PINCTRL_PIN(221, "N6 GND"),
+	PINCTRL_PIN(222, "N7 VCC2IOHA 1"),
+	PINCTRL_PIN(223, "N8 VCC2IOHA 1"),
+	PINCTRL_PIN(224, "N9 V1"),
+	PINCTRL_PIN(225, "N10 V1"),
+	PINCTRL_PIN(226, "N11 VCC3IOHA"),
+	PINCTRL_PIN(227, "N12 VCC3IOHA"),
+	PINCTRL_PIN(228, "N13 GND"),
+	PINCTRL_PIN(229, "N14 GPIO0 20"),
+	PINCTRL_PIN(230, "N15 GPIO0 21"),
+	PINCTRL_PIN(231, "N16 GPIO0 22"),
+	PINCTRL_PIN(232, "N17 GPIO0 23"),
+	PINCTRL_PIN(233, "N18 GPIO0 24"),
+	/* Row P (for some reason O is skipped) */
+	PINCTRL_PIN(234, "P1 IDE DD4"),
+	PINCTRL_PIN(235, "P2 IDE DD10"),
+	PINCTRL_PIN(236, "P3 IDE DD5"),
+	PINCTRL_PIN(237, "P4 IDE DD9"),
+	PINCTRL_PIN(238, "P5 GND"),
+	PINCTRL_PIN(239, "P6 USB XSCO"),
+	PINCTRL_PIN(240, "P7 GMAC0 TXD3"),
+	PINCTRL_PIN(241, "P8 GMAC0 TXEN"),
+	PINCTRL_PIN(242, "P9 GMAC0 RXD2"),
+	PINCTRL_PIN(243, "P10 GMAC1 TXC"),
+	PINCTRL_PIN(244, "P11 GMAC1 RXD1"),
+	PINCTRL_PIN(245, "P12 MODE SEL 1"),
+	PINCTRL_PIN(246, "P13 GPIO1 28"),
+	PINCTRL_PIN(247, "P14 GND"),
+	PINCTRL_PIN(248, "P15 GPIO0 5"),
+	PINCTRL_PIN(249, "P16 GPIO0 17"),
+	PINCTRL_PIN(250, "P17 GPIO0 18"),
+	PINCTRL_PIN(251, "P18 GPIO0 19"),
+	/* Row R (for some reason Q us skipped) */
+	PINCTRL_PIN(252, "R1 IDE DD6"),
+	PINCTRL_PIN(253, "R2 IDE DD8"),
+	PINCTRL_PIN(254, "R3 IDE DD7"),
+	PINCTRL_PIN(255, "R4 IDE RESET N"),
+	PINCTRL_PIN(256, "R5 ICE0 DBGACK"),
+	PINCTRL_PIN(257, "R6 USB XSCI"),
+	PINCTRL_PIN(258, "R7 GMAC0 TXD2"),
+	PINCTRL_PIN(259, "R8 GMAC0 RXDV"),
+	PINCTRL_PIN(260, "R9 GMAC0 RXD3"),
+	PINCTRL_PIN(261, "R10 GMAC1 TXD0"),
+	PINCTRL_PIN(262, "R11 GMAC1 RXD0"),
+	PINCTRL_PIN(263, "R12 MODE SEL 0"),
+	PINCTRL_PIN(264, "R13 MODE SEL 3"),
+	PINCTRL_PIN(265, "R14 GPIO0 0"),
+	PINCTRL_PIN(266, "R15 GPIO0 4"),
+	PINCTRL_PIN(267, "R16 GPIO0 9"),
+	PINCTRL_PIN(268, "R17 GPIO0 15"),
+	PINCTRL_PIN(269, "R18 GPIO0 16"),
+	/* Row T (for some reason S is skipped) */
+	PINCTRL_PIN(270, "T1 ICE0 DBGRQ"),
+	PINCTRL_PIN(271, "T2 ICE0 IDO"),
+	PINCTRL_PIN(272, "T3 ICE0 ICK"),
+	PINCTRL_PIN(273, "T4 ICE0 IMS"),
+	PINCTRL_PIN(274, "T5 ICE0 IDI"),
+	PINCTRL_PIN(275, "T6 USB RREF"),
+	PINCTRL_PIN(276, "T7 GMAC0 TXD1"),
+	PINCTRL_PIN(277, "T8 GMAC0 RXC"),
+	PINCTRL_PIN(278, "T9 GMAC0 CRS"),
+	PINCTRL_PIN(279, "T10 GMAC1 TXD1"),
+	PINCTRL_PIN(280, "T11 GMAC1 RXC"),
+	PINCTRL_PIN(281, "T12 GMAC1 CRS"),
+	PINCTRL_PIN(282, "T13 EXT CLK"),
+	PINCTRL_PIN(283, "T14 GPIO1 31"),
+	PINCTRL_PIN(284, "T15 GPIO0 3"),
+	PINCTRL_PIN(285, "T16 GPIO0 8"),
+	PINCTRL_PIN(286, "T17 GPIO0 12"),
+	PINCTRL_PIN(287, "T18 GPIO0 14"),
+	/* Row U */
+	PINCTRL_PIN(288, "U1 ICE0 IRST N"),
+	PINCTRL_PIN(289, "U2 USB0 VCCHSRT"),
+	PINCTRL_PIN(290, "U3 USB0 DP"),
+	PINCTRL_PIN(291, "U4 USB VCCA U20"),
+	PINCTRL_PIN(292, "U5 USB1 DP"),
+	PINCTRL_PIN(293, "U6 USB1 GNDHSRT 1"),
+	PINCTRL_PIN(294, "U7 GMAC0 TXD0"),
+	PINCTRL_PIN(295, "U8 GMAC0 RXD0"),
+	PINCTRL_PIN(296, "U9 GMAC1 COL"),
+	PINCTRL_PIN(297, "U10 GMAC1 TXD2"),
+	PINCTRL_PIN(298, "U11 GMAC1 RXDV"),
+	PINCTRL_PIN(299, "U12 GMAC1 RXD3"),
+	PINCTRL_PIN(300, "U13 MODE SEL 2"),
+	PINCTRL_PIN(301, "U14 GPIO1 30"),
+	PINCTRL_PIN(302, "U15 GPIO0 2"),
+	PINCTRL_PIN(303, "U16 GPIO0 7"),
+	PINCTRL_PIN(304, "U17 GPIO0 11"),
+	PINCTRL_PIN(305, "U18 GPIO0 13"),
+	/* Row V */
+	PINCTRL_PIN(306, "V1 USB0 GNDHSRT"),
+	PINCTRL_PIN(307, "V2 USB0 DM"),
+	PINCTRL_PIN(308, "V3 USB GNDA U20"),
+	PINCTRL_PIN(309, "V4 USB1 DM"),
+	PINCTRL_PIN(310, "V5 USB1 VCCHSRT1"),
+	PINCTRL_PIN(311, "V6 GMAC0 COL"),
+	PINCTRL_PIN(312, "V7 GMAC0 TXC"),
+	PINCTRL_PIN(313, "V8 GMAC0 RXD1"),
+	PINCTRL_PIN(314, "V9 REF CLK"),
+	PINCTRL_PIN(315, "V10 GMAC1 TXD3"),
+	PINCTRL_PIN(316, "V11 GMAC1 TXEN"),
+	PINCTRL_PIN(317, "V12 GMAC1 RXD2"),
+	PINCTRL_PIN(318, "V13 M30 CLK"),
+	PINCTRL_PIN(319, "V14 GPIO1 29"),
+	PINCTRL_PIN(320, "V15 GPIO0 1"),
+	PINCTRL_PIN(321, "V16 GPIO0 6"),
+	PINCTRL_PIN(322, "V17 GPIO0 10"),
+	PINCTRL_PIN(323, "V18 SYS RESET N"),
+};
+
+
+/* Digital ground */
+static const unsigned int gnd_3512_pins[] = {
+	76, 85, 95, 102, 114, 119, 133, 134, 135, 136, 151, 152, 153, 154, 169,
+	170, 171, 172, 187, 188, 189, 190, 204, 209, 221, 228, 238, 247
+};
+
+static const unsigned int dram_3512_pins[] = {
+	2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 21, 22, 23, 24, 25, 26, 27, 28, 29,
+	30, 40, 41, 42, 43, 44, 45, 46, 47, 58, 59, 60, 61, 62, 63, 64, 65, 77,
+	78, 79, 80, 81, 82
+};
+
+static const unsigned int rtc_3512_pins[] = { 57, 20, 39 };
+
+static const unsigned int power_3512_pins[] = { 19, 38, 36, 55, 37, 56, 54, 72 };
+
+static const unsigned int system_3512_pins[] = {
+	318, 264, 300, 245, 263, 282, 314, 323, 49,
+};
+
+static const unsigned int vcontrol_3512_pins[] = { 18, 0, 1 };
+
+static const unsigned int ice_3512_pins[] = { 256, 270, 271, 272, 273, 274, 288 };
+
+static const unsigned int ide_3512_pins[] = {
+	162, 163, 165, 166, 148, 180, 181, 182, 183, 184, 198, 199, 200, 201, 202,
+	216, 217, 218, 219, 220, 234, 235, 236, 237, 252, 253, 254, 255
+};
+
+static const unsigned int sata_3512_pins[] = {
+	75, 74, 73, 93, 94, 131, 112, 130, 92, 91, 90, 111, 110, 109, 108, 129,
+	128, 127, 126, 147, 146, 145, 144, 164
+};
+
+static const unsigned int usb_3512_pins[] = {
+	306, 289, 307, 290, 239, 257, 275, 308, 291, 309, 292, 310, 293
+};
+
+/* GMII, ethernet pins */
+static const unsigned int gmii_3512_pins[] = {
+	311, 240, 258, 276, 294, 312, 241, 259, 277, 295, 313, 242, 260, 278, 296,
+	315, 297, 279, 261, 243, 316, 298, 280, 262, 244, 317, 299, 281
+};
+
+static const unsigned int pci_3512_pins[] = {
+	13, 14, 15, 16, 17, 31, 32, 33, 34, 35, 48, 50, 51, 52, 53, 66, 67, 68, 69,
+	70, 71, 83, 84, 86, 87, 88, 89, 103, 104, 105, 106, 107, 121, 122, 123,
+	124, 125, 139, 140, 141, 142, 143, 157, 158, 159, 160, 161, 175, 176, 177,
+	178, 179, 195, 196, 197
+};
+
+/*
+ * Apparently the LPC interface is using the PCICLK for the clocking so
+ * PCI needs to be active at the same time.
+ */
+static const unsigned int lpc_3512_pins[] = {
+	285, /* LPC_LAD[0] */
+	304, /* LPC_SERIRQ */
+	286, /* LPC_LAD[2] */
+	305, /* LPC_LFRAME# */
+	287, /* LPC_LAD[3] */
+	268, /* LPC_LAD[1] */
+};
+
+/* Character LCD */
+static const unsigned int lcd_3512_pins[] = {
+	262, 244, 317, 299, 246, 319, 301, 283, 269, 233, 211
+};
+
+static const unsigned int ssp_3512_pins[] = {
+	285, /* SSP_97RST# SSP AC97 Reset, active low */
+	304, /* SSP_FSC */
+	286, /* SSP_ECLK */
+	305, /* SSP_TXD */
+	287, /* SSP_RXD */
+	268, /* SSP_SCLK */
+};
+
+static const unsigned int uart_rxtx_3512_pins[] = {
+	267, /* UART_SIN serial input, RX */
+	322, /* UART_SOUT serial output, TX */
+};
+
+static const unsigned int uart_modem_3512_pins[] = {
+	285, /* UART_NDCD DCD carrier detect */
+	304, /* UART_NDTR DTR data terminal ready */
+	286, /* UART_NDSR DSR data set ready */
+	305, /* UART_NRTS RTS request to send */
+	287, /* UART_NCTS CTS clear to send */
+	268, /* UART_NRI RI ring indicator */
+};
+
+static const unsigned int tvc_3512_pins[] = {
+	246, /* TVC_DATA[0] */
+	319, /* TVC_DATA[1] */
+	301, /* TVC_DATA[2] */
+	283, /* TVC_DATA[3] */
+	265, /* TVC_CLK */
+	320, /* TVC_DATA[4] */
+	302, /* TVC_DATA[5] */
+	284, /* TVC_DATA[6] */
+	266, /* TVC_DATA[7] */
+};
+
+/* NAND flash pins */
+static const unsigned int nflash_3512_pins[] = {
+	199, 200, 201, 202, 216, 217, 218, 219, 220, 234, 235, 236, 237, 252,
+	253, 254, 249, 250, 232, 233, 211, 193, 194
+};
+
+/* Parallel (NOR) flash pins, D[0-15], A[16-25], CE0, CE1, RB, WE, OE, ALE */
+static const unsigned int pflash_3512_pins[] = {
+	162, 163, 165, 166, 148, 199, 200, 201, 202, 216, 217, 218, 219, 220,
+	234, 235, 236, 237, 252, 253, 254, 251, 229, 232, 233, 211, 212, 213,
+	214, 215, 193, 194
+};
+
+/*
+ * The parallel flash can be set up in a 26-bit address bus mode exposing
+ * A[0-15] (A[15] takes the place of ALE), but it has the
+ * side effect of stealing pins from GMAC1 and TVC so these blocks cannot be
+ * used at the same time.
+ */
+static const unsigned int pflash_3512_pins_extended[] = {
+	162, 163, 165, 166, 148, 199, 200, 201, 202, 216, 217, 218, 219, 220,
+	234, 235, 236, 237, 252, 253, 254, 251, 229, 232, 233, 211, 212, 213,
+	214, 215, 193, 194,
+	/* The extra pins */
+	296, 315, 297, 279, 261, 243, 316, 298, 280, 262, 244, 317, 299, 281,
+	265,
+};
+
+/* Serial flash pins CE0, CE1, DI, DO, CK */
+static const unsigned int sflash_3512_pins[] = { 230, 231, 232, 233, 211 };
+
+/* The GPIO0A (0) pin overlap with TVC and extended parallel flash */
+static const unsigned int gpio0a_3512_pins[] = { 265 };
+
+/* The GPIO0B (1-4) pins overlap with TVC and ICE */
+static const unsigned int gpio0b_3512_pins[] = { 320, 302, 284, 266 };
+
+/* The GPIO0C (5-7) pins overlap with ICE */
+static const unsigned int gpio0c_3512_pins[] = { 248, 321, 303 };
+
+/* The GPIO0D (9,10) pins overlap with UART RX/TX */
+static const unsigned int gpio0d_3512_pins[] = { 267, 322 };
+
+/* The GPIO0E (8,11-15) pins overlap with LPC, UART modem pins, SSP */
+static const unsigned int gpio0e_3512_pins[] = { 285, 304, 286, 305, 287, 268 };
+
+/* The GPIO0F (16) pins overlap with LCD */
+static const unsigned int gpio0f_3512_pins[] = { 269 };
+
+/* The GPIO0G (17,18) pins overlap with NAND flash CE0, CE1 */
+static const unsigned int gpio0g_3512_pins[] = { 249, 250 };
+
+/* The GPIO0H (19,20) pins overlap with parallel flash CE0, CE1 */
+static const unsigned int gpio0h_3512_pins[] = { 251, 229 };
+
+/* The GPIO0I (21,22) pins overlap with serial flash CE0, CE1 */
+static const unsigned int gpio0i_3512_pins[] = { 230, 231 };
+
+/* The GPIO0J (23) pins overlap with all flash */
+static const unsigned int gpio0j_3512_pins[] = { 232 };
+
+/* The GPIO0K (24,25) pins overlap with all flash and LCD */
+static const unsigned int gpio0k_3512_pins[] = { 233, 211 };
+
+/* The GPIO0L (26-29) pins overlap with parallel flash */
+static const unsigned int gpio0l_3512_pins[] = { 212, 213, 214, 215 };
+
+/* The GPIO0M (30,31) pins overlap with parallel flash and NAND flash */
+static const unsigned int gpio0m_3512_pins[] = { 193, 194 };
+
+/* The GPIO1A (0-4) pins that overlap with IDE and parallel flash */
+static const unsigned int gpio1a_3512_pins[] = { 162, 163, 165, 166, 148 };
+
+/* The GPIO1B (5-10, 27) pins overlap with just IDE */
+static const unsigned int gpio1b_3512_pins[] = {
+	180, 181, 182, 183, 184, 198, 255
+};
+
+/* The GPIO1C (11-26) pins overlap with IDE, parallel flash and NAND flash */
+static const unsigned int gpio1c_3512_pins[] = {
+	199, 200, 201, 202, 216, 217, 218, 219, 220, 234, 235, 236, 237,
+	252, 253, 254
+};
+
+/* The GPIO1D (28-31) pins overlap with LCD and TVC */
+static const unsigned int gpio1d_3512_pins[] = { 246, 319, 301, 283 };
+
+/* The GPIO2A (0-3) pins overlap with GMII and extended parallel flash */
+static const unsigned int gpio2a_3512_pins[] = { 315, 297, 279, 261 };
+
+/* The GPIO2B (4-7) pins overlap with GMII, extended parallel flash and LCD */
+static const unsigned int gpio2b_3512_pins[] = { 262, 244, 317, 299 };
+
+/* The GPIO2C (8-31) pins overlap with PCI */
+static const unsigned int gpio2c_3512_pins[] = {
+	17, 34, 35, 51, 52, 53, 68, 69, 71, 86, 87, 88, 89, 103, 104, 105,
+	140, 141, 142, 143, 157, 158, 159, 160
+};
+
+/* Groups for the 3512 SoC/package */
+static const struct gemini_pin_group gemini_3512_pin_groups[] = {
+	{
+		.name = "gndgrp",
+		.pins = gnd_3512_pins,
+		.num_pins = ARRAY_SIZE(gnd_3512_pins),
+	},
+	{
+		.name = "dramgrp",
+		.pins = dram_3512_pins,
+		.num_pins = ARRAY_SIZE(dram_3512_pins),
+		.mask = DRAM_PADS_POWERDOWN,
+	},
+	{
+		.name = "rtcgrp",
+		.pins = rtc_3512_pins,
+		.num_pins = ARRAY_SIZE(rtc_3512_pins),
+	},
+	{
+		.name = "powergrp",
+		.pins = power_3512_pins,
+		.num_pins = ARRAY_SIZE(power_3512_pins),
+	},
+	{
+		.name = "systemgrp",
+		.pins = system_3512_pins,
+		.num_pins = ARRAY_SIZE(system_3512_pins),
+	},
+	{
+		.name = "vcontrolgrp",
+		.pins = vcontrol_3512_pins,
+		.num_pins = ARRAY_SIZE(vcontrol_3512_pins),
+	},
+	{
+		.name = "icegrp",
+		.pins = ice_3512_pins,
+		.num_pins = ARRAY_SIZE(ice_3512_pins),
+		/* Conflict with some GPIO groups */
+	},
+	{
+		.name = "idegrp",
+		.pins = ide_3512_pins,
+		.num_pins = ARRAY_SIZE(ide_3512_pins),
+		/* Conflict with all flash usage */
+		.value = IDE_PADS_ENABLE | NAND_PADS_DISABLE |
+			PFLASH_PADS_DISABLE | SFLASH_PADS_DISABLE,
+	},
+	{
+		.name = "satagrp",
+		.pins = sata_3512_pins,
+		.num_pins = ARRAY_SIZE(sata_3512_pins),
+	},
+	{
+		.name = "usbgrp",
+		.pins = usb_3512_pins,
+		.num_pins = ARRAY_SIZE(usb_3512_pins),
+	},
+	{
+		.name = "gmiigrp",
+		.pins = gmii_3512_pins,
+		.num_pins = ARRAY_SIZE(gmii_3512_pins),
+	},
+	{
+		.name = "pcigrp",
+		.pins = pci_3512_pins,
+		.num_pins = ARRAY_SIZE(pci_3512_pins),
+		/* Conflict only with GPIO2 */
+		.value = PCI_PADS_ENABLE | PCI_CLK_PAD_ENABLE,
+	},
+	{
+		.name = "lpcgrp",
+		.pins = lpc_3512_pins,
+		.num_pins = ARRAY_SIZE(lpc_3512_pins),
+		/* Conflict with SSP and UART modem pins */
+		.mask = SSP_PADS_ENABLE,
+		.value = LPC_PADS_ENABLE | LPC_CLK_PAD_ENABLE,
+	},
+	{
+		.name = "lcdgrp",
+		.pins = lcd_3512_pins,
+		.num_pins = ARRAY_SIZE(lcd_3512_pins),
+		/* Conflict with TVC and ICE */
+		.mask = TVC_PADS_ENABLE,
+		.value = LCD_PADS_ENABLE,
+	},
+	{
+		.name = "sspgrp",
+		.pins = ssp_3512_pins,
+		.num_pins = ARRAY_SIZE(ssp_3512_pins),
+		/* Conflict with LPC and UART modem pins */
+		.mask = LPC_PADS_ENABLE,
+		.value = SSP_PADS_ENABLE,
+	},
+	{
+		.name = "uartrxtxgrp",
+		.pins = uart_rxtx_3512_pins,
+		.num_pins = ARRAY_SIZE(uart_rxtx_3512_pins),
+		/* No conflicts except GPIO */
+	},
+	{
+		.name = "uartmodemgrp",
+		.pins = uart_modem_3512_pins,
+		.num_pins = ARRAY_SIZE(uart_modem_3512_pins),
+		/*
+		 * Conflict with LPC and SSP,
+		 * so when those are both disabled, modem UART can thrive.
+		 */
+		.mask = LPC_PADS_ENABLE | SSP_PADS_ENABLE,
+	},
+	{
+		.name = "tvcgrp",
+		.pins = tvc_3512_pins,
+		.num_pins = ARRAY_SIZE(tvc_3512_pins),
+		/* Conflict with character LCD and ICE */
+		.mask = LCD_PADS_ENABLE,
+		.value = TVC_PADS_ENABLE | TVC_CLK_PAD_ENABLE,
+	},
+	/*
+	 * The construction is done such that it is possible to use a serial
+	 * flash together with a NAND or parallel (NOR) flash, but it is not
+	 * possible to use NAND and parallel flash together. To use serial
+	 * flash with one of the two others, the muxbits need to be flipped
+	 * around before any access.
+	 */
+	{
+		.name = "nflashgrp",
+		.pins = nflash_3512_pins,
+		.num_pins = ARRAY_SIZE(nflash_3512_pins),
+		/* Conflict with IDE, parallel and serial flash */
+		.mask = NAND_PADS_DISABLE | IDE_PADS_ENABLE,
+		.value = PFLASH_PADS_DISABLE | SFLASH_PADS_DISABLE,
+	},
+	{
+		.name = "pflashgrp",
+		.pins = pflash_3512_pins,
+		.num_pins = ARRAY_SIZE(pflash_3512_pins),
+		/* Conflict with IDE, NAND and serial flash */
+		.mask = PFLASH_PADS_DISABLE | IDE_PADS_ENABLE,
+		.value = NAND_PADS_DISABLE | SFLASH_PADS_DISABLE,
+	},
+	{
+		.name = "sflashgrp",
+		.pins = sflash_3512_pins,
+		.num_pins = ARRAY_SIZE(sflash_3512_pins),
+		/* Conflict with IDE, NAND and parallel flash */
+		.mask = SFLASH_PADS_DISABLE | IDE_PADS_ENABLE,
+		.value = NAND_PADS_DISABLE | PFLASH_PADS_DISABLE,
+	},
+	{
+		.name = "gpio0agrp",
+		.pins = gpio0a_3512_pins,
+		.num_pins = ARRAY_SIZE(gpio0a_3512_pins),
+		/* Conflict with TVC */
+		.mask = TVC_PADS_ENABLE,
+	},
+	{
+		.name = "gpio0bgrp",
+		.pins = gpio0b_3512_pins,
+		.num_pins = ARRAY_SIZE(gpio0b_3512_pins),
+		/* Conflict with TVC and ICE */
+		.mask = TVC_PADS_ENABLE,
+	},
+	{
+		.name = "gpio0cgrp",
+		.pins = gpio0c_3512_pins,
+		.num_pins = ARRAY_SIZE(gpio0c_3512_pins),
+		/* Conflict with ICE */
+	},
+	{
+		.name = "gpio0dgrp",
+		.pins = gpio0d_3512_pins,
+		.num_pins = ARRAY_SIZE(gpio0d_3512_pins),
+		/* Conflict with UART RX/TX */
+	},
+	{
+		.name = "gpio0egrp",
+		.pins = gpio0e_3512_pins,
+		.num_pins = ARRAY_SIZE(gpio0e_3512_pins),
+		/* Conflict with LPC, UART modem pins, SSP */
+		.mask = LPC_PADS_ENABLE | SSP_PADS_ENABLE,
+	},
+	{
+		.name = "gpio0fgrp",
+		.pins = gpio0f_3512_pins,
+		.num_pins = ARRAY_SIZE(gpio0f_3512_pins),
+		/* Conflict with LCD */
+		.mask = LCD_PADS_ENABLE,
+	},
+	{
+		.name = "gpio0ggrp",
+		.pins = gpio0g_3512_pins,
+		.num_pins = ARRAY_SIZE(gpio0g_3512_pins),
+		/* Conflict with NAND flash */
+		.value = NAND_PADS_DISABLE,
+	},
+	{
+		.name = "gpio0hgrp",
+		.pins = gpio0h_3512_pins,
+		.num_pins = ARRAY_SIZE(gpio0h_3512_pins),
+		/* Conflict with parallel flash */
+		.value = PFLASH_PADS_DISABLE,
+	},
+	{
+		.name = "gpio0igrp",
+		.pins = gpio0i_3512_pins,
+		.num_pins = ARRAY_SIZE(gpio0i_3512_pins),
+		/* Conflict with serial flash */
+		.value = SFLASH_PADS_DISABLE,
+	},
+	{
+		.name = "gpio0jgrp",
+		.pins = gpio0j_3512_pins,
+		.num_pins = ARRAY_SIZE(gpio0j_3512_pins),
+		/* Conflict with all flash */
+		.value = PFLASH_PADS_DISABLE | NAND_PADS_DISABLE |
+			SFLASH_PADS_DISABLE,
+	},
+	{
+		.name = "gpio0kgrp",
+		.pins = gpio0k_3512_pins,
+		.num_pins = ARRAY_SIZE(gpio0k_3512_pins),
+		/* Conflict with all flash and LCD */
+		.mask = LCD_PADS_ENABLE,
+		.value = PFLASH_PADS_DISABLE | NAND_PADS_DISABLE |
+			SFLASH_PADS_DISABLE,
+	},
+	{
+		.name = "gpio0lgrp",
+		.pins = gpio0l_3512_pins,
+		.num_pins = ARRAY_SIZE(gpio0l_3512_pins),
+		/* Conflict with parallel flash */
+		.value = PFLASH_PADS_DISABLE,
+	},
+	{
+		.name = "gpio0mgrp",
+		.pins = gpio0m_3512_pins,
+		.num_pins = ARRAY_SIZE(gpio0m_3512_pins),
+		/* Conflict with parallel and NAND flash */
+		.value = PFLASH_PADS_DISABLE | NAND_PADS_DISABLE,
+	},
+	{
+		.name = "gpio1agrp",
+		.pins = gpio1a_3512_pins,
+		.num_pins = ARRAY_SIZE(gpio1a_3512_pins),
+		/* Conflict with IDE and parallel flash */
+		.mask = IDE_PADS_ENABLE,
+		.value = PFLASH_PADS_DISABLE,
+	},
+	{
+		.name = "gpio1bgrp",
+		.pins = gpio1b_3512_pins,
+		.num_pins = ARRAY_SIZE(gpio1b_3512_pins),
+		/* Conflict with IDE only */
+		.mask = IDE_PADS_ENABLE,
+	},
+	{
+		.name = "gpio1cgrp",
+		.pins = gpio1c_3512_pins,
+		.num_pins = ARRAY_SIZE(gpio1c_3512_pins),
+		/* Conflict with IDE, parallel and NAND flash */
+		.mask = IDE_PADS_ENABLE,
+		.value = NAND_PADS_DISABLE | PFLASH_PADS_DISABLE,
+	},
+	{
+		.name = "gpio1dgrp",
+		.pins = gpio1d_3512_pins,
+		.num_pins = ARRAY_SIZE(gpio1d_3512_pins),
+		/* Conflict with LCD and TVC */
+		.mask = LCD_PADS_ENABLE | TVC_PADS_ENABLE,
+	},
+	{
+		.name = "gpio2agrp",
+		.pins = gpio2a_3512_pins,
+		.num_pins = ARRAY_SIZE(gpio2a_3512_pins),
+		/* Conflict with GMII and extended parallel flash */
+	},
+	{
+		.name = "gpio2bgrp",
+		.pins = gpio2b_3512_pins,
+		.num_pins = ARRAY_SIZE(gpio2b_3512_pins),
+		/* Conflict with GMII, extended parallel flash and LCD */
+		.mask = LCD_PADS_ENABLE,
+	},
+	{
+		.name = "gpio2cgrp",
+		.pins = gpio2c_3512_pins,
+		.num_pins = ARRAY_SIZE(gpio2c_3512_pins),
+		/* Conflict with PCI */
+		.mask = PCI_PADS_ENABLE,
+	},
+};
+
+/* Pin names for the pinmux subsystem, 3516 variant */
+static const struct pinctrl_pin_desc gemini_3516_pins[] = {
+	/* Row A */
+	PINCTRL_PIN(0, "A1 AVCC3IOHA"),
+	PINCTRL_PIN(1, "A2 DRAM CK N"),
+	PINCTRL_PIN(2, "A3 DRAM CK"),
+	PINCTRL_PIN(3, "A4 DRAM DQM1"),
+	PINCTRL_PIN(4, "A5 DRAM DQ9"),
+	PINCTRL_PIN(5, "A6 DRAM DQ13"),
+	PINCTRL_PIN(6, "A7 DRAM DQ1"),
+	PINCTRL_PIN(7, "A8 DRAM DQ2"),
+	PINCTRL_PIN(8, "A9 DRAM DQ4"),
+	PINCTRL_PIN(9, "A10 DRAM VREF"),
+	PINCTRL_PIN(10, "A11 DRAM DQ24"),
+	PINCTRL_PIN(11, "A12 DRAM DQ28"),
+	PINCTRL_PIN(12, "A13 DRAM DQ30"),
+	PINCTRL_PIN(13, "A14 DRAM DQ18"),
+	PINCTRL_PIN(14, "A15 DRAM DQ21"),
+	PINCTRL_PIN(15, "A16 DRAM CAS_N"),
+	PINCTRL_PIN(16, "A17 DRAM BA1"),
+	PINCTRL_PIN(17, "A18 PCI INTA N"),
+	PINCTRL_PIN(18, "A19 PCI INTB N"),
+	PINCTRL_PIN(19, "A20 PCI INTC N"),
+	/* Row B */
+	PINCTRL_PIN(20, "B1 PWR EN"),
+	PINCTRL_PIN(21, "B2 GND"),
+	PINCTRL_PIN(22, "B3 RTC CLKO"),
+	PINCTRL_PIN(23, "B4 DRAM A5"),
+	PINCTRL_PIN(24, "B5 DRAM A6"),
+	PINCTRL_PIN(25, "B6 DRAM DQS1"),
+	PINCTRL_PIN(26, "B7 DRAM DQ11"),
+	PINCTRL_PIN(27, "B8 DRAM DQ0"),
+	PINCTRL_PIN(28, "B9 DRAM DQS0"),
+	PINCTRL_PIN(29, "B10 DRAM DQ7"),
+	PINCTRL_PIN(30, "B11 DRAM DQS3"),
+	PINCTRL_PIN(31, "B12 DRAM DQ27"),
+	PINCTRL_PIN(32, "B13 DRAM DQ31"),
+	PINCTRL_PIN(33, "B14 DRAM DQ20"),
+	PINCTRL_PIN(34, "B15 DRAM DQS2"),
+	PINCTRL_PIN(35, "B16 DRAM WE N"),
+	PINCTRL_PIN(36, "B17 DRAM A10"),
+	PINCTRL_PIN(37, "B18 DRAM A2"),
+	PINCTRL_PIN(38, "B19 GND"),
+	PINCTRL_PIN(39, "B20 PCI GNT0 N"),
+	/* Row C */
+	PINCTRL_PIN(40, "C1 AGNDIOHA"),
+	PINCTRL_PIN(41, "C2 XTALI"),
+	PINCTRL_PIN(42, "C3 GND"),
+	PINCTRL_PIN(43, "C4 RTC CLKI"),
+	PINCTRL_PIN(44, "C5 DRAM A12"),
+	PINCTRL_PIN(45, "C6 DRAM A11"),
+	PINCTRL_PIN(46, "C7 DRAM DQ8"),
+	PINCTRL_PIN(47, "C8 DRAM DQ10"),
+	PINCTRL_PIN(48, "C9 DRAM DQ3"),
+	PINCTRL_PIN(49, "C10 DRAM DQ6"),
+	PINCTRL_PIN(50, "C11 DRAM DQM0"),
+	PINCTRL_PIN(51, "C12 DRAM DQ26"),
+	PINCTRL_PIN(52, "C13 DRAM DQ16"),
+	PINCTRL_PIN(53, "C14 DRAM DQ22"),
+	PINCTRL_PIN(54, "C15 DRAM DQM2"),
+	PINCTRL_PIN(55, "C16 DRAM BA0"),
+	PINCTRL_PIN(56, "C17 DRAM A3"),
+	PINCTRL_PIN(57, "C18 GND"),
+	PINCTRL_PIN(58, "C19 PCI GNT1 N"),
+	PINCTRL_PIN(59, "C20 PCI REQ2 N"),
+	/* Row D */
+	PINCTRL_PIN(60, "D1 AVCC3IOAHA"),
+	PINCTRL_PIN(61, "D2 AVCCKHA"),
+	PINCTRL_PIN(62, "D3 XTALO"),
+	PINCTRL_PIN(63, "D4 GND"),
+	PINCTRL_PIN(64, "D5 CIR RXD"),
+	PINCTRL_PIN(65, "D6 DRAM A7"),
+	PINCTRL_PIN(66, "D7 DRAM A4"),
+	PINCTRL_PIN(67, "D8 DRAM A8"),
+	PINCTRL_PIN(68, "D9 DRAM CKE"),
+	PINCTRL_PIN(69, "D10 DRAM DQ14"),
+	PINCTRL_PIN(70, "D11 DRAM DQ5"),
+	PINCTRL_PIN(71, "D12 DRAM DQ25"),
+	PINCTRL_PIN(72, "D13 DRAM DQ17"),
+	PINCTRL_PIN(73, "D14 DRAM DQ23"),
+	PINCTRL_PIN(74, "D15 DRAM RAS N"),
+	PINCTRL_PIN(75, "D16 DRAM A1"),
+	PINCTRL_PIN(76, "D17 GND"),
+	PINCTRL_PIN(77, "D18 EXT RESET N"),
+	PINCTRL_PIN(78, "D19 PCI REQ1 N"),
+	PINCTRL_PIN(79, "D20 PCI REQ3 N"),
+	/* Row E */
+	PINCTRL_PIN(80, "E1 VCC2IO CTRL"),
+	PINCTRL_PIN(81, "E2 VREF CTRL"),
+	PINCTRL_PIN(82, "E3 CIR RST N"),
+	PINCTRL_PIN(83, "E4 PWR BTN"),
+	PINCTRL_PIN(84, "E5 GND"),
+	PINCTRL_PIN(85, "E6 CIR TXD"),
+	PINCTRL_PIN(86, "E7 VCCK CTRL"),
+	PINCTRL_PIN(87, "E8 DRAM A9"),
+	PINCTRL_PIN(88, "E9 DRAM DQ12"),
+	PINCTRL_PIN(89, "E10 DRAM DQ15"),
+	PINCTRL_PIN(90, "E11 DRAM DQM3"),
+	PINCTRL_PIN(91, "E12 DRAM DQ29"),
+	PINCTRL_PIN(92, "E13 DRAM DQ19"),
+	PINCTRL_PIN(93, "E14 DRAM A13"),
+	PINCTRL_PIN(94, "E15 DRAM A0"),
+	PINCTRL_PIN(95, "E16 GND"),
+	PINCTRL_PIN(96, "E17 PCI INTD N"),
+	PINCTRL_PIN(97, "E18 PCI GNT3 N"),
+	PINCTRL_PIN(98, "E19 PCI AD29"),
+	PINCTRL_PIN(99, "E20 PCI AD28"),
+	/* Row F */
+	PINCTRL_PIN(100, "F1 AVCCKHB"),
+	PINCTRL_PIN(101, "F2 AVCCK P"),
+	PINCTRL_PIN(102, "F3 EBG"),
+	PINCTRL_PIN(103, "F4 REXT"),
+	PINCTRL_PIN(104, "F5 AVCC3IOHB"),
+	PINCTRL_PIN(105, "F6 GND"),
+	PINCTRL_PIN(106, "F7 VCC2IOHA 2"),
+	PINCTRL_PIN(107, "F8 VCC2IOHA 2"),
+	PINCTRL_PIN(108, "F9 VCC2IOHA 2"),
+	PINCTRL_PIN(109, "F10 V1"),
+	PINCTRL_PIN(110, "F11 V1"),
+	PINCTRL_PIN(111, "F12 VCC2IOHA 2"),
+	PINCTRL_PIN(112, "F13 VCC2IOHA 2"),
+	PINCTRL_PIN(113, "F14 VCC2IOHA 2"),
+	PINCTRL_PIN(114, "F15 GND"),
+	PINCTRL_PIN(115, "F16 PCI CLK"),
+	PINCTRL_PIN(116, "F17 PCI GNT2 N"),
+	PINCTRL_PIN(117, "F18 PCI AD31"),
+	PINCTRL_PIN(118, "F19 PCI AD26"),
+	PINCTRL_PIN(119, "F20 PCI CBE3 N"),
+	/* Row G */
+	PINCTRL_PIN(120, "G1 SATA0 RXDP"),
+	PINCTRL_PIN(121, "G2 SATA0 RXDN"),
+	PINCTRL_PIN(122, "G3 AGNDK 0"),
+	PINCTRL_PIN(123, "G4 AVCCK S"),
+	PINCTRL_PIN(124, "G5 AVCC3 S"),
+	PINCTRL_PIN(125, "G6 VCC2IOHA 2"),
+	PINCTRL_PIN(126, "G7 GND"),
+	PINCTRL_PIN(127, "G8 VCC2IOHA 2"),
+	PINCTRL_PIN(128, "G9 V1"),
+	PINCTRL_PIN(129, "G10 V1"),
+	PINCTRL_PIN(130, "G11 V1"),
+	PINCTRL_PIN(131, "G12 V1"),
+	PINCTRL_PIN(132, "G13 VCC2IOHA 2"),
+	PINCTRL_PIN(133, "G14 GND"),
+	PINCTRL_PIN(134, "G15 VCC3IOHA"),
+	PINCTRL_PIN(135, "G16 PCI REQ0 N"),
+	PINCTRL_PIN(136, "G17 PCI AD30"),
+	PINCTRL_PIN(137, "G18 PCI AD24"),
+	PINCTRL_PIN(138, "G19 PCI AD23"),
+	PINCTRL_PIN(139, "G20 PCI AD21"),
+	/* Row H */
+	PINCTRL_PIN(140, "H1 SATA0 TXDP"),
+	PINCTRL_PIN(141, "H2 SATA0 TXDN"),
+	PINCTRL_PIN(142, "H3 AGNDK 1"),
+	PINCTRL_PIN(143, "H4 AVCCK 0"),
+	PINCTRL_PIN(144, "H5 TEST CLKOUT"),
+	PINCTRL_PIN(145, "H6 AGND"),
+	PINCTRL_PIN(146, "H7 VCC2IOHA 2"),
+	PINCTRL_PIN(147, "H8 GND"),
+	PINCTRL_PIN(148, "H9 GND"),
+	PINCTRL_PIN(149, "H10 GDN"),
+	PINCTRL_PIN(150, "H11 GND"),
+	PINCTRL_PIN(151, "H12 GND"),
+	PINCTRL_PIN(152, "H13 GND"),
+	PINCTRL_PIN(153, "H14 VCC3IOHA"),
+	PINCTRL_PIN(154, "H15 VCC3IOHA"),
+	PINCTRL_PIN(155, "H16 PCI AD27"),
+	PINCTRL_PIN(156, "H17 PCI AD25"),
+	PINCTRL_PIN(157, "H18 PCI AD22"),
+	PINCTRL_PIN(158, "H19 PCI AD18"),
+	PINCTRL_PIN(159, "H20 PCI AD17"),
+	/* Row J (for some reason I is skipped) */
+	PINCTRL_PIN(160, "J1 SATA1 TXDP"),
+	PINCTRL_PIN(161, "J2 SATA1 TXDN"),
+	PINCTRL_PIN(162, "J3 AGNDK 2"),
+	PINCTRL_PIN(163, "J4 AVCCK 1"),
+	PINCTRL_PIN(164, "J5 AGND"),
+	PINCTRL_PIN(165, "J6 AGND"),
+	PINCTRL_PIN(166, "J7 V1"),
+	PINCTRL_PIN(167, "J8 GND"),
+	PINCTRL_PIN(168, "J9 GND"),
+	PINCTRL_PIN(169, "J10 GND"),
+	PINCTRL_PIN(170, "J11 GND"),
+	PINCTRL_PIN(171, "J12 GND"),
+	PINCTRL_PIN(172, "J13 GND"),
+	PINCTRL_PIN(173, "J14 V1"),
+	PINCTRL_PIN(174, "J15 VCC3IOHA"),
+	PINCTRL_PIN(175, "J16 PCI AD19"),
+	PINCTRL_PIN(176, "J17 PCI AD20"),
+	PINCTRL_PIN(177, "J18 PCI AD16"),
+	PINCTRL_PIN(178, "J19 PCI CBE2 N"),
+	PINCTRL_PIN(179, "J20 PCI FRAME N"),
+	/* Row K */
+	PINCTRL_PIN(180, "K1 SATA1 RXDP"),
+	PINCTRL_PIN(181, "K2 SATA1 RXDN"),
+	PINCTRL_PIN(182, "K3 AGNDK 3"),
+	PINCTRL_PIN(183, "K4 AVCCK 2"),
+	PINCTRL_PIN(184, "K5 AGND"),
+	PINCTRL_PIN(185, "K6 V1"),
+	PINCTRL_PIN(186, "K7 V1"),
+	PINCTRL_PIN(187, "K8 GND"),
+	PINCTRL_PIN(188, "K9 GND"),
+	PINCTRL_PIN(189, "K10 GND"),
+	PINCTRL_PIN(190, "K11 GND"),
+	PINCTRL_PIN(191, "K12 GND"),
+	PINCTRL_PIN(192, "K13 GND"),
+	PINCTRL_PIN(193, "K14 V1"),
+	PINCTRL_PIN(194, "K15 V1"),
+	PINCTRL_PIN(195, "K16 PCI TRDY N"),
+	PINCTRL_PIN(196, "K17 PCI IRDY N"),
+	PINCTRL_PIN(197, "K18 PCI DEVSEL N"),
+	PINCTRL_PIN(198, "K19 PCI STOP N"),
+	PINCTRL_PIN(199, "K20 PCI PAR"),
+	/* Row L */
+	PINCTRL_PIN(200, "L1 IDE CS0 N"),
+	PINCTRL_PIN(201, "L2 IDE DA0"),
+	PINCTRL_PIN(202, "L3 AVCCK 3"),
+	PINCTRL_PIN(203, "L4 AGND"),
+	PINCTRL_PIN(204, "L5 IDE DIOR N"),
+	PINCTRL_PIN(205, "L6 V1"),
+	PINCTRL_PIN(206, "L7 V1"),
+	PINCTRL_PIN(207, "L8 GND"),
+	PINCTRL_PIN(208, "L9 GND"),
+	PINCTRL_PIN(209, "L10 GND"),
+	PINCTRL_PIN(210, "L11 GND"),
+	PINCTRL_PIN(211, "L12 GND"),
+	PINCTRL_PIN(212, "L13 GND"),
+	PINCTRL_PIN(213, "L14 V1"),
+	PINCTRL_PIN(214, "L15 V1"),
+	PINCTRL_PIN(215, "L16 PCI AD12"),
+	PINCTRL_PIN(216, "L17 PCI AD13"),
+	PINCTRL_PIN(217, "L18 PCI AD14"),
+	PINCTRL_PIN(218, "L19 PCI AD15"),
+	PINCTRL_PIN(219, "L20 PCI CBE1 N"),
+	/* Row M */
+	PINCTRL_PIN(220, "M1 IDE DA1"),
+	PINCTRL_PIN(221, "M2 IDE CS1 N"),
+	PINCTRL_PIN(222, "M3 IDE DA2"),
+	PINCTRL_PIN(223, "M4 IDE DMACK N"),
+	PINCTRL_PIN(224, "M5 IDE DD1"),
+	PINCTRL_PIN(225, "M6 VCC3IOHA"),
+	PINCTRL_PIN(226, "M7 V1"),
+	PINCTRL_PIN(227, "M8 GND"),
+	PINCTRL_PIN(228, "M9 GND"),
+	PINCTRL_PIN(229, "M10 GND"),
+	PINCTRL_PIN(230, "M11 GND"),
+	PINCTRL_PIN(231, "M12 GND"),
+	PINCTRL_PIN(232, "M13 GND"),
+	PINCTRL_PIN(233, "M14 V1"),
+	PINCTRL_PIN(234, "M15 VCC3IOHA"),
+	PINCTRL_PIN(235, "M16 PCI AD7"),
+	PINCTRL_PIN(236, "M17 PCI AD6"),
+	PINCTRL_PIN(237, "M18 PCI AD9"),
+	PINCTRL_PIN(238, "M19 PCI AD10"),
+	PINCTRL_PIN(239, "M20 PCI AD11"),
+	/* Row N */
+	PINCTRL_PIN(240, "N1 IDE IORDY"),
+	PINCTRL_PIN(241, "N2 IDE INTRQ"),
+	PINCTRL_PIN(242, "N3 IDE DIOW N"),
+	PINCTRL_PIN(243, "N4 IDE DD15"),
+	PINCTRL_PIN(244, "N5 IDE DMARQ"),
+	PINCTRL_PIN(245, "N6 VCC3IOHA"),
+	PINCTRL_PIN(246, "N7 VCC3IOHA"),
+	PINCTRL_PIN(247, "N8 GND"),
+	PINCTRL_PIN(248, "N9 GND"),
+	PINCTRL_PIN(249, "N10 GND"),
+	PINCTRL_PIN(250, "N11 GND"),
+	PINCTRL_PIN(251, "N12 GND"),
+	PINCTRL_PIN(252, "N13 GND"),
+	PINCTRL_PIN(253, "N14 VCC3IOHA"),
+	PINCTRL_PIN(254, "N15 VCC3IOHA"),
+	PINCTRL_PIN(255, "N16 PCI CLKRUN N"),
+	PINCTRL_PIN(256, "N17 PCI AD0"),
+	PINCTRL_PIN(257, "N18 PCI AD4"),
+	PINCTRL_PIN(258, "N19 PCI CBE0 N"),
+	PINCTRL_PIN(259, "N20 PCI AD8"),
+	/* Row P (for some reason O is skipped) */
+	PINCTRL_PIN(260, "P1 IDE DD0"),
+	PINCTRL_PIN(261, "P2 IDE DD14"),
+	PINCTRL_PIN(262, "P3 IDE DD2"),
+	PINCTRL_PIN(263, "P4 IDE DD4"),
+	PINCTRL_PIN(264, "P5 IDE DD3"),
+	PINCTRL_PIN(265, "P6 VCC3IOHA"),
+	PINCTRL_PIN(266, "P7 GND"),
+	PINCTRL_PIN(267, "P8 VCC2IOHA 1"),
+	PINCTRL_PIN(268, "P9 V1"),
+	PINCTRL_PIN(269, "P10 V1"),
+	PINCTRL_PIN(270, "P11 V1"),
+	PINCTRL_PIN(271, "P12 V1"),
+	PINCTRL_PIN(272, "P13 VCC3IOHA"),
+	PINCTRL_PIN(273, "P14 GND"),
+	PINCTRL_PIN(274, "P15 VCC3IOHA"),
+	PINCTRL_PIN(275, "P16 GPIO0 30"),
+	PINCTRL_PIN(276, "P17 GPIO0 28"),
+	PINCTRL_PIN(277, "P18 PCI AD1"),
+	PINCTRL_PIN(278, "P19 PCI AD3"),
+	PINCTRL_PIN(279, "P20 PCI AD5"),
+	/* Row R (for some reason Q us skipped) */
+	PINCTRL_PIN(280, "R1 IDE DD13"),
+	PINCTRL_PIN(281, "R2 IDE DD12"),
+	PINCTRL_PIN(282, "R3 IDE DD10"),
+	PINCTRL_PIN(283, "R4 IDE DD6"),
+	PINCTRL_PIN(284, "R5 ICE0 IDI"),
+	PINCTRL_PIN(285, "R6 GND"),
+	PINCTRL_PIN(286, "R7 VCC2IOHA 1"),
+	PINCTRL_PIN(287, "R8 VCC2IOHA 1"),
+	PINCTRL_PIN(288, "R9 VCC2IOHA 1"),
+	PINCTRL_PIN(289, "R10 V1"),
+	PINCTRL_PIN(290, "R11 V1"),
+	PINCTRL_PIN(291, "R12 VCC3IOHA"),
+	PINCTRL_PIN(292, "R13 VCC3IOHA"),
+	PINCTRL_PIN(293, "R14 VCC3IOHA"),
+	PINCTRL_PIN(294, "R15 GND"),
+	PINCTRL_PIN(295, "R16 GPIO0 23"),
+	PINCTRL_PIN(296, "R17 GPIO0 21"),
+	PINCTRL_PIN(297, "R18 GPIO0 26"),
+	PINCTRL_PIN(298, "R19 GPIO0 31"),
+	PINCTRL_PIN(299, "R20 PCI AD2"),
+	/* Row T (for some reason S is skipped) */
+	PINCTRL_PIN(300, "T1 IDE DD11"),
+	PINCTRL_PIN(301, "T2 IDE DD5"),
+	PINCTRL_PIN(302, "T3 IDE DD8"),
+	PINCTRL_PIN(303, "T4 ICE0 IDO"),
+	PINCTRL_PIN(304, "T5 GND"),
+	PINCTRL_PIN(305, "T6 USB GNDA U20"),
+	PINCTRL_PIN(306, "T7 GMAC0 TXD0"),
+	PINCTRL_PIN(307, "T8 GMAC0 TXEN"),
+	PINCTRL_PIN(308, "T9 GMAC1 TXD3"),
+	PINCTRL_PIN(309, "T10 GMAC1 RXDV"),
+	PINCTRL_PIN(310, "T11 GMAC1 RXD2"),
+	PINCTRL_PIN(311, "T12 GPIO1 29"),
+	PINCTRL_PIN(312, "T13 GPIO0 3"),
+	PINCTRL_PIN(313, "T14 GPIO0 9"),
+	PINCTRL_PIN(314, "T15 GPIO0 16"),
+	PINCTRL_PIN(315, "T16 GND"),
+	PINCTRL_PIN(316, "T17 GPIO0 14"),
+	PINCTRL_PIN(317, "T18 GPIO0 19"),
+	PINCTRL_PIN(318, "T19 GPIO0 27"),
+	PINCTRL_PIN(319, "T20 GPIO0 29"),
+	/* Row U */
+	PINCTRL_PIN(320, "U1 IDE DD9"),
+	PINCTRL_PIN(321, "U2 IDE DD7"),
+	PINCTRL_PIN(322, "U3 ICE0 ICK"),
+	PINCTRL_PIN(323, "U4 GND"),
+	PINCTRL_PIN(324, "U5 USB XSCO"),
+	PINCTRL_PIN(325, "U6 GMAC0 TXD1"),
+	PINCTRL_PIN(326, "U7 GMAC0 TXD3"),
+	PINCTRL_PIN(327, "U8 GMAC0 TXC"),
+	PINCTRL_PIN(328, "U9 GMAC0 RXD3"),
+	PINCTRL_PIN(329, "U10 GMAC1 TXD0"),
+	PINCTRL_PIN(330, "U11 GMAC1 CRS"),
+	PINCTRL_PIN(331, "U12 EXT CLK"),
+	PINCTRL_PIN(332, "U13 DEV DEF"),
+	PINCTRL_PIN(333, "U14 GPIO0 0"),
+	PINCTRL_PIN(334, "U15 GPIO0 4"),
+	PINCTRL_PIN(335, "U16 GPIO0 10"),
+	PINCTRL_PIN(336, "U17 GND"),
+	PINCTRL_PIN(337, "U18 GPIO0 17"),
+	PINCTRL_PIN(338, "U19 GPIO0 22"),
+	PINCTRL_PIN(339, "U20 GPIO0 25"),
+	/* Row V */
+	PINCTRL_PIN(340, "V1 ICE0 DBGACK"),
+	PINCTRL_PIN(341, "V2 ICE0 DBGRQ"),
+	PINCTRL_PIN(342, "V3 GND"),
+	PINCTRL_PIN(343, "V4 ICE0 IRST N"),
+	PINCTRL_PIN(344, "V5 USB XSCI"),
+	PINCTRL_PIN(345, "V6 GMAC0 COL"),
+	PINCTRL_PIN(346, "V7 GMAC0 TXD2"),
+	PINCTRL_PIN(347, "V8 GMAC0 RXDV"),
+	PINCTRL_PIN(348, "V9 GMAC0 RXD1"),
+	PINCTRL_PIN(349, "V10 GMAC1 COL"),
+	PINCTRL_PIN(350, "V11 GMAC1 TXC"),
+	PINCTRL_PIN(351, "V12 GMAC1 RXD1"),
+	PINCTRL_PIN(352, "V13 MODE SEL1"),
+	PINCTRL_PIN(353, "V14 GPIO1 28"),
+	PINCTRL_PIN(354, "V15 GPIO0 1"),
+	PINCTRL_PIN(355, "V16 GPIO0 8"),
+	PINCTRL_PIN(356, "V17 GPIO0 11"),
+	PINCTRL_PIN(357, "V18 GND"),
+	PINCTRL_PIN(358, "V19 GPIO0 18"),
+	PINCTRL_PIN(359, "V20 GPIO0 24"),
+	/* Row W */
+	PINCTRL_PIN(360, "W1 IDE RESET N"),
+	PINCTRL_PIN(361, "W2 GND"),
+	PINCTRL_PIN(362, "W3 USB0 VCCHSRT"),
+	PINCTRL_PIN(363, "W4 USB0 DP"),
+	PINCTRL_PIN(364, "W5 USB VCCA U20"),
+	PINCTRL_PIN(365, "W6 USB1 DP"),
+	PINCTRL_PIN(366, "W7 USB1 GNDHSRT"),
+	PINCTRL_PIN(367, "W8 GMAC0 RXD0"),
+	PINCTRL_PIN(368, "W9 GMAC0 CRS"),
+	PINCTRL_PIN(369, "W10 GMAC1 TXD2"),
+	PINCTRL_PIN(370, "W11 GMAC1 TXEN"),
+	PINCTRL_PIN(371, "W12 GMAC1 RXD3"),
+	PINCTRL_PIN(372, "W13 MODE SEL0"),
+	PINCTRL_PIN(373, "W14 MODE SEL3"),
+	PINCTRL_PIN(374, "W15 GPIO1 31"),
+	PINCTRL_PIN(375, "W16 GPIO0 5"),
+	PINCTRL_PIN(376, "W17 GPIO0 7"),
+	PINCTRL_PIN(377, "W18 GPIO0 12"),
+	PINCTRL_PIN(378, "W19 GND"),
+	PINCTRL_PIN(379, "W20 GPIO0 20"),
+	/* Row Y */
+	PINCTRL_PIN(380, "Y1 ICE0 IMS"),
+	PINCTRL_PIN(381, "Y2 USB0 GNDHSRT"),
+	PINCTRL_PIN(382, "Y3 USB0 DM"),
+	PINCTRL_PIN(383, "Y4 USB RREF"),
+	PINCTRL_PIN(384, "Y5 USB1 DM"),
+	PINCTRL_PIN(385, "Y6 USB1 VCCHSRT"),
+	PINCTRL_PIN(386, "Y7 GMAC0 RXC"),
+	PINCTRL_PIN(387, "Y8 GMAC0 RXD2"),
+	PINCTRL_PIN(388, "Y9 REF CLK"),
+	PINCTRL_PIN(389, "Y10 GMAC1 TXD1"),
+	PINCTRL_PIN(390, "Y11 GMAC1 RXC"),
+	PINCTRL_PIN(391, "Y12 GMAC1 RXD0"),
+	PINCTRL_PIN(392, "Y13 M30 CLK"),
+	PINCTRL_PIN(393, "Y14 MODE SEL2"),
+	PINCTRL_PIN(394, "Y15 GPIO1 30"),
+	PINCTRL_PIN(395, "Y16 GPIO0 2"),
+	PINCTRL_PIN(396, "Y17 GPIO0 6"),
+	PINCTRL_PIN(397, "Y18 SYS RESET N"),
+	PINCTRL_PIN(398, "Y19 GPIO0 13"),
+	PINCTRL_PIN(399, "Y20 GPIO0 15"),
+};
+
+/* Digital ground */
+static const unsigned int gnd_3516_pins[] = {
+	21, 38, 42, 57, 63, 76, 84, 95, 105, 114, 126, 133, 147, 148, 149, 150,
+	151, 152, 167, 168, 169, 170, 171, 172, 187, 188, 189, 190, 191, 192,
+	207, 208, 209, 210, 211, 212, 227, 228, 229, 230, 231, 232, 247, 248,
+	249, 250, 251, 252, 266, 273, 285, 294, 304, 315, 323, 336, 342, 357,
+	361, 378
+};
+
+static const unsigned int dram_3516_pins[] = {
+	1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 23, 24, 25, 26,
+	27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 44, 45, 46, 47, 48, 49, 50,
+	51, 52, 53, 54, 55, 56, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75,
+	87, 88, 89, 90, 91, 92, 93, 94
+};
+
+static const unsigned int rtc_3516_pins[] = { 0, 43, 22 };
+
+static const unsigned int power_3516_pins[] = { 20, 83, 40, 41, 60, 61, 62 };
+
+static const unsigned int cir_3516_pins[] = { 85, 64, 82 };
+
+static const unsigned int system_3516_pins[] = {
+	332, 392, 372, 373, 393, 352, 331, 388, 397, 77
+};
+
+static const unsigned int vcontrol_3516_pins[] = { 86, 81, 80 };
+
+static const unsigned int ice_3516_pins[] = { 340, 341, 303, 322, 380, 284, 343 };
+
+static const unsigned int ide_3516_pins[] = {
+	200, 201, 204, 220, 221, 222, 223, 224, 240, 241, 242, 243, 244, 260,
+	261, 262, 263, 264, 280, 281, 282, 283, 300, 301, 302, 320, 321, 360
+};
+
+static const unsigned int sata_3516_pins[] = {
+	100, 101, 102, 103, 104, 120, 121, 122, 123, 124, 140, 141, 142, 143,
+	144, 160, 161, 162, 163, 180, 181, 182, 183, 202
+};
+
+static const unsigned int usb_3516_pins[] = {
+	305, 324, 344, 362, 363, 364, 365, 366, 381, 382, 383, 384, 385
+};
+
+/* GMII, ethernet pins */
+static const unsigned int gmii_3516_pins[] = {
+	306, 307, 308, 309, 310, 325, 326, 327, 328, 329, 330, 345, 346, 347,
+	348, 349, 350, 351, 367, 368, 369, 370, 371, 386, 387, 389, 390, 391
+};
+
+static const unsigned int pci_3516_pins[] = {
+	17, 18, 19, 39, 58, 59, 78, 79, 96, 97, 98, 99, 115, 116, 117, 118,
+	119, 135, 136, 137, 138, 139, 155, 156, 157, 158, 159, 175, 176, 177,
+	178, 179, 195, 196, 197, 198, 199, 215, 216, 217, 218, 219, 235, 236,
+	237, 238, 239, 255, 256, 257, 258, 259, 277, 278, 279, 299
+};
+
+/*
+ * Apparently the LPC interface is using the PCICLK for the clocking so
+ * PCI needs to be active at the same time.
+ */
+static const unsigned int lpc_3516_pins[] = {
+	355, /* LPC_LAD[0] */
+	356, /* LPC_SERIRQ */
+	377, /* LPC_LAD[2] */
+	398, /* LPC_LFRAME# */
+	316, /* LPC_LAD[3] */
+	399, /* LPC_LAD[1] */
+};
+
+/* Character LCD */
+static const unsigned int lcd_3516_pins[] = {
+	391, 351, 310, 371, 353, 311, 394, 374, 314, 359, 339
+};
+
+static const unsigned int ssp_3516_pins[] = {
+	355, /* SSP_97RST# SSP AC97 Reset, active low */
+	356, /* SSP_FSC */
+	377, /* SSP_ECLK */
+	398, /* SSP_TXD */
+	316, /* SSP_RXD */
+	399, /* SSP_SCLK */
+};
+
+static const unsigned int uart_rxtx_3516_pins[] = {
+	313, /* UART_SIN serial input, RX */
+	335, /* UART_SOUT serial output, TX */
+};
+
+static const unsigned int uart_modem_3516_pins[] = {
+	355, /* UART_NDCD DCD carrier detect */
+	356, /* UART_NDTR DTR data terminal ready */
+	377, /* UART_NDSR DSR data set ready */
+	398, /* UART_NRTS RTS request to send */
+	316, /* UART_NCTS CTS clear to send */
+	399, /* UART_NRI RI ring indicator */
+};
+
+static const unsigned int tvc_3516_pins[] = {
+	353, /* TVC_DATA[0] */
+	311, /* TVC_DATA[1] */
+	394, /* TVC_DATA[2] */
+	374, /* TVC_DATA[3] */
+	333, /* TVC_CLK */
+	354, /* TVC_DATA[4] */
+	395, /* TVC_DATA[5] */
+	312, /* TVC_DATA[6] */
+	334, /* TVC_DATA[7] */
+};
+
+/* NAND flash pins */
+static const unsigned int nflash_3516_pins[] = {
+	243, 260, 261, 224, 280, 262, 281, 264, 300, 263, 282, 301, 320, 283,
+	302, 321, 337, 358, 295, 359, 339, 275, 298
+};
+
+/* Parallel (NOR) flash pins, D[0-15], A[16-25], CE0, CE1, RB, WE, OE, ALE */
+static const unsigned int pflash_3516_pins[] = {
+	221, 200, 222, 201, 220, 243, 260, 261, 224, 280, 262, 281, 264, 300,
+	263, 282, 301, 320, 283, 302, 321, 317, 379, 295, 359, 339, 297, 318,
+	276, 319, 275, 298
+};
+
+/*
+ * The parallel flash can be set up in a 26-bit address bus mode exposing
+ * A[0-15] (A[15] takes the place of ALE), but it has the
+ * side effect of stealing pins from GMAC1 and TVC so these blocks cannot be
+ * used at the same time.
+ */
+static const unsigned int pflash_3516_pins_extended[] = {
+	221, 200, 222, 201, 220, 243, 260, 261, 224, 280, 262, 281, 264, 300,
+	263, 282, 301, 320, 283, 302, 321, 317, 379, 295, 359, 339, 297, 318,
+	276, 319, 275, 298,
+	/* The extra pins */
+	349, 308, 369, 389, 329, 350, 370, 309, 390, 391, 351, 310, 371, 330,
+	333
+};
+
+/* Serial flash pins CE0, CE1, DI, DO, CK */
+static const unsigned int sflash_3516_pins[] = { 296, 338, 295, 359, 339 };
+
+/* The GPIO0A (0-4) pins overlap with TVC and extended parallel flash */
+static const unsigned int gpio0a_3516_pins[] = { 333, 354, 395, 312, 334 };
+
+/* The GPIO0B (5-7) pins overlap with ICE */
+static const unsigned int gpio0b_3516_pins[] = { 375, 396, 376 };
+
+/* The GPIO0C (8,11-15) pins overlap with LPC, UART and SSP */
+static const unsigned int gpio0c_3516_pins[] = { 355, 356, 377, 398, 316, 399 };
+
+/* The GPIO0D (9,10) pins overlap with UART RX/TX */
+static const unsigned int gpio0d_3516_pins[] = { 313, 335 };
+
+/* The GPIO0E (16) pins overlap with LCD */
+static const unsigned int gpio0e_3516_pins[] = { 314 };
+
+/* The GPIO0F (17,18) pins overlap with NAND flash CE0, CE1 */
+static const unsigned int gpio0f_3516_pins[] = { 337, 358 };
+
+/* The GPIO0G (19,20,26-29) pins overlap with parallel flash */
+static const unsigned int gpio0g_3516_pins[] = { 317, 379, 297, 318, 276, 319 };
+
+/* The GPIO0H (21,22) pins overlap with serial flash CE0, CE1 */
+static const unsigned int gpio0h_3516_pins[] = { 296, 338 };
+
+/* The GPIO0I (23) pins overlap with all flash */
+static const unsigned int gpio0i_3516_pins[] = { 295 };
+
+/* The GPIO0J (24,25) pins overlap with all flash and LCD */
+static const unsigned int gpio0j_3516_pins[] = { 359, 339 };
+
+/* The GPIO0K (30,31) pins overlap with NAND flash */
+static const unsigned int gpio0k_3516_pins[] = { 275, 298 };
+
+/* The GPIO1A (0-4) pins that overlap with IDE and parallel flash */
+static const unsigned int gpio1a_3516_pins[] = { 221, 200, 222, 201, 220 };
+
+/* The GPIO1B (5-10,27) pins overlap with just IDE */
+static const unsigned int gpio1b_3516_pins[] = { 241, 223, 240, 204, 242, 244, 360 };
+
+/* The GPIO1C (11-26) pins overlap with IDE, parallel flash and NAND flash */
+static const unsigned int gpio1c_3516_pins[] = {
+	243, 260, 261, 224, 280, 262, 281, 264, 300, 263, 282, 301, 320, 283,
+	302, 321
+};
+
+/* The GPIO1D (28-31) pins overlap with TVC */
+static const unsigned int gpio1d_3516_pins[] = { 353, 311, 394, 374 };
+
+/* The GPIO2A (0-3) pins overlap with GMII and extended parallel flash */
+static const unsigned int gpio2a_3516_pins[] = { 308, 369, 389, 329 };
+
+/* The GPIO2B (4-7) pins overlap with GMII, extended parallel flash and LCD */
+static const unsigned int gpio2b_3516_pins[] = { 391, 351, 310, 371 };
+
+/* The GPIO2C (8-31) pins overlap with PCI */
+static const unsigned int gpio2c_3516_pins[] = {
+	259, 237, 238, 239, 215, 216, 217, 218, 177, 159, 158, 175, 176, 139,
+	157, 138, 137, 156, 118, 155, 99, 98, 136, 117
+};
+
+/* Groups for the 3516 SoC/package */
+static const struct gemini_pin_group gemini_3516_pin_groups[] = {
+	{
+		.name = "gndgrp",
+		.pins = gnd_3516_pins,
+		.num_pins = ARRAY_SIZE(gnd_3516_pins),
+	},
+	{
+		.name = "dramgrp",
+		.pins = dram_3516_pins,
+		.num_pins = ARRAY_SIZE(dram_3516_pins),
+		.mask = DRAM_PADS_POWERDOWN,
+	},
+	{
+		.name = "rtcgrp",
+		.pins = rtc_3516_pins,
+		.num_pins = ARRAY_SIZE(rtc_3516_pins),
+	},
+	{
+		.name = "powergrp",
+		.pins = power_3516_pins,
+		.num_pins = ARRAY_SIZE(power_3516_pins),
+	},
+	{
+		.name = "cirgrp",
+		.pins = cir_3516_pins,
+		.num_pins = ARRAY_SIZE(cir_3516_pins),
+	},
+	{
+		.name = "systemgrp",
+		.pins = system_3516_pins,
+		.num_pins = ARRAY_SIZE(system_3516_pins),
+	},
+	{
+		.name = "vcontrolgrp",
+		.pins = vcontrol_3516_pins,
+		.num_pins = ARRAY_SIZE(vcontrol_3516_pins),
+	},
+	{
+		.name = "icegrp",
+		.pins = ice_3516_pins,
+		.num_pins = ARRAY_SIZE(ice_3516_pins),
+		/* Conflict with some GPIO groups */
+	},
+	{
+		.name = "idegrp",
+		.pins = ide_3516_pins,
+		.num_pins = ARRAY_SIZE(ide_3516_pins),
+		/* Conflict with all flash usage */
+		.value = IDE_PADS_ENABLE | NAND_PADS_DISABLE |
+			PFLASH_PADS_DISABLE | SFLASH_PADS_DISABLE,
+	},
+	{
+		.name = "satagrp",
+		.pins = sata_3516_pins,
+		.num_pins = ARRAY_SIZE(sata_3516_pins),
+	},
+	{
+		.name = "usbgrp",
+		.pins = usb_3516_pins,
+		.num_pins = ARRAY_SIZE(usb_3516_pins),
+	},
+	{
+		.name = "gmiigrp",
+		.pins = gmii_3516_pins,
+		.num_pins = ARRAY_SIZE(gmii_3516_pins),
+	},
+	{
+		.name = "pcigrp",
+		.pins = pci_3516_pins,
+		.num_pins = ARRAY_SIZE(pci_3516_pins),
+		/* Conflict only with GPIO2 */
+		.value = PCI_PADS_ENABLE | PCI_CLK_PAD_ENABLE,
+	},
+	{
+		.name = "lpcgrp",
+		.pins = lpc_3516_pins,
+		.num_pins = ARRAY_SIZE(lpc_3516_pins),
+		/* Conflict with SSP */
+		.mask = SSP_PADS_ENABLE,
+		.value = LPC_PADS_ENABLE | LPC_CLK_PAD_ENABLE,
+	},
+	{
+		.name = "lcdgrp",
+		.pins = lcd_3516_pins,
+		.num_pins = ARRAY_SIZE(lcd_3516_pins),
+		.mask = TVC_PADS_ENABLE,
+		.value = LCD_PADS_ENABLE,
+	},
+	{
+		.name = "sspgrp",
+		.pins = ssp_3516_pins,
+		.num_pins = ARRAY_SIZE(ssp_3516_pins),
+		/* Conflict with LPC */
+		.mask = LPC_PADS_ENABLE,
+		.value = SSP_PADS_ENABLE,
+	},
+	{
+		.name = "uartrxtxgrp",
+		.pins = uart_rxtx_3516_pins,
+		.num_pins = ARRAY_SIZE(uart_rxtx_3516_pins),
+		/* No conflicts except GPIO */
+	},
+	{
+		.name = "uartmodemgrp",
+		.pins = uart_modem_3516_pins,
+		.num_pins = ARRAY_SIZE(uart_modem_3516_pins),
+		/*
+		 * Conflict with LPC and SSP,
+		 * so when those are both disabled, modem UART can thrive.
+		 */
+		.mask = LPC_PADS_ENABLE | SSP_PADS_ENABLE,
+	},
+	{
+		.name = "tvcgrp",
+		.pins = tvc_3516_pins,
+		.num_pins = ARRAY_SIZE(tvc_3516_pins),
+		/* Conflict with character LCD */
+		.mask = LCD_PADS_ENABLE,
+		.value = TVC_PADS_ENABLE | TVC_CLK_PAD_ENABLE,
+	},
+	/*
+	 * The construction is done such that it is possible to use a serial
+	 * flash together with a NAND or parallel (NOR) flash, but it is not
+	 * possible to use NAND and parallel flash together. To use serial
+	 * flash with one of the two others, the muxbits need to be flipped
+	 * around before any access.
+	 */
+	{
+		.name = "nflashgrp",
+		.pins = nflash_3516_pins,
+		.num_pins = ARRAY_SIZE(nflash_3516_pins),
+		/* Conflict with IDE, parallel and serial flash */
+		.mask = NAND_PADS_DISABLE | IDE_PADS_ENABLE,
+		.value = PFLASH_PADS_DISABLE | SFLASH_PADS_DISABLE,
+	},
+	{
+		.name = "pflashgrp",
+		.pins = pflash_3516_pins,
+		.num_pins = ARRAY_SIZE(pflash_3516_pins),
+		/* Conflict with IDE, NAND and serial flash */
+		.mask = PFLASH_PADS_DISABLE | IDE_PADS_ENABLE,
+		.value = NAND_PADS_DISABLE | SFLASH_PADS_DISABLE,
+	},
+	{
+		.name = "sflashgrp",
+		.pins = sflash_3516_pins,
+		.num_pins = ARRAY_SIZE(sflash_3516_pins),
+		/* Conflict with IDE, NAND and parallel flash */
+		.mask = SFLASH_PADS_DISABLE | IDE_PADS_ENABLE,
+		.value = NAND_PADS_DISABLE | PFLASH_PADS_DISABLE,
+	},
+	{
+		.name = "gpio0agrp",
+		.pins = gpio0a_3516_pins,
+		.num_pins = ARRAY_SIZE(gpio0a_3516_pins),
+		/* Conflict with TVC and ICE */
+		.mask = TVC_PADS_ENABLE,
+	},
+	{
+		.name = "gpio0bgrp",
+		.pins = gpio0b_3516_pins,
+		.num_pins = ARRAY_SIZE(gpio0b_3516_pins),
+		/* Conflict with ICE */
+	},
+	{
+		.name = "gpio0cgrp",
+		.pins = gpio0c_3516_pins,
+		.num_pins = ARRAY_SIZE(gpio0c_3516_pins),
+		/* Conflict with LPC, UART and SSP */
+		.mask = LPC_PADS_ENABLE | SSP_PADS_ENABLE,
+	},
+	{
+		.name = "gpio0dgrp",
+		.pins = gpio0d_3516_pins,
+		.num_pins = ARRAY_SIZE(gpio0d_3516_pins),
+		/* Conflict with UART */
+	},
+	{
+		.name = "gpio0egrp",
+		.pins = gpio0e_3516_pins,
+		.num_pins = ARRAY_SIZE(gpio0e_3516_pins),
+		/* Conflict with LCD */
+		.mask = LCD_PADS_ENABLE,
+	},
+	{
+		.name = "gpio0fgrp",
+		.pins = gpio0f_3516_pins,
+		.num_pins = ARRAY_SIZE(gpio0f_3516_pins),
+		/* Conflict with NAND flash */
+		.value = NAND_PADS_DISABLE,
+	},
+	{
+		.name = "gpio0ggrp",
+		.pins = gpio0g_3516_pins,
+		.num_pins = ARRAY_SIZE(gpio0g_3516_pins),
+		/* Conflict with parallel flash */
+		.value = PFLASH_PADS_DISABLE,
+	},
+	{
+		.name = "gpio0hgrp",
+		.pins = gpio0h_3516_pins,
+		.num_pins = ARRAY_SIZE(gpio0h_3516_pins),
+		/* Conflict with serial flash */
+		.value = SFLASH_PADS_DISABLE,
+	},
+	{
+		.name = "gpio0igrp",
+		.pins = gpio0i_3516_pins,
+		.num_pins = ARRAY_SIZE(gpio0i_3516_pins),
+		/* Conflict with all flash */
+		.value = PFLASH_PADS_DISABLE | NAND_PADS_DISABLE |
+			SFLASH_PADS_DISABLE,
+	},
+	{
+		.name = "gpio0jgrp",
+		.pins = gpio0j_3516_pins,
+		.num_pins = ARRAY_SIZE(gpio0j_3516_pins),
+		/* Conflict with all flash and LCD */
+		.mask = LCD_PADS_ENABLE,
+		.value = PFLASH_PADS_DISABLE | NAND_PADS_DISABLE |
+			SFLASH_PADS_DISABLE,
+	},
+	{
+		.name = "gpio0kgrp",
+		.pins = gpio0k_3516_pins,
+		.num_pins = ARRAY_SIZE(gpio0k_3516_pins),
+		/* Conflict with parallel and NAND flash */
+		.value = PFLASH_PADS_DISABLE | NAND_PADS_DISABLE,
+	},
+	{
+		.name = "gpio1agrp",
+		.pins = gpio1a_3516_pins,
+		.num_pins = ARRAY_SIZE(gpio1a_3516_pins),
+		/* Conflict with IDE and parallel flash */
+		.mask = IDE_PADS_ENABLE,
+		.value = PFLASH_PADS_DISABLE,
+	},
+	{
+		.name = "gpio1bgrp",
+		.pins = gpio1b_3516_pins,
+		.num_pins = ARRAY_SIZE(gpio1b_3516_pins),
+		/* Conflict with IDE only */
+		.mask = IDE_PADS_ENABLE,
+	},
+	{
+		.name = "gpio1cgrp",
+		.pins = gpio1c_3516_pins,
+		.num_pins = ARRAY_SIZE(gpio1c_3516_pins),
+		/* Conflict with IDE, parallel and NAND flash */
+		.mask = IDE_PADS_ENABLE,
+		.value = NAND_PADS_DISABLE | PFLASH_PADS_DISABLE,
+	},
+	{
+		.name = "gpio1dgrp",
+		.pins = gpio1d_3516_pins,
+		.num_pins = ARRAY_SIZE(gpio1d_3516_pins),
+		/* Conflict with TVC */
+		.mask = TVC_PADS_ENABLE,
+	},
+	{
+		.name = "gpio2agrp",
+		.pins = gpio2a_3516_pins,
+		.num_pins = ARRAY_SIZE(gpio2a_3516_pins),
+		/* Conflict with GMII and extended parallel flash */
+	},
+	{
+		.name = "gpio2bgrp",
+		.pins = gpio2b_3516_pins,
+		.num_pins = ARRAY_SIZE(gpio2b_3516_pins),
+		/* Conflict with GMII, extended parallel flash and LCD */
+		.mask = LCD_PADS_ENABLE,
+	},
+	{
+		.name = "gpio2cgrp",
+		.pins = gpio2c_3516_pins,
+		.num_pins = ARRAY_SIZE(gpio2c_3516_pins),
+		/* Conflict with PCI */
+		.mask = PCI_PADS_ENABLE,
+	},
+};
+
+static int gemini_get_groups_count(struct pinctrl_dev *pctldev)
+{
+	struct gemini_pmx *pmx = pinctrl_dev_get_drvdata(pctldev);
+
+	if (pmx->is_3512)
+		return ARRAY_SIZE(gemini_3512_pin_groups);
+	if (pmx->is_3516)
+		return ARRAY_SIZE(gemini_3516_pin_groups);
+	return 0;
+}
+
+static const char *gemini_get_group_name(struct pinctrl_dev *pctldev,
+					 unsigned int selector)
+{
+	struct gemini_pmx *pmx = pinctrl_dev_get_drvdata(pctldev);
+
+	if (pmx->is_3512)
+		return gemini_3512_pin_groups[selector].name;
+	if (pmx->is_3516)
+		return gemini_3516_pin_groups[selector].name;
+	return NULL;
+}
+
+static int gemini_get_group_pins(struct pinctrl_dev *pctldev,
+				 unsigned int selector,
+				 const unsigned int **pins,
+				 unsigned int *num_pins)
+{
+	struct gemini_pmx *pmx = pinctrl_dev_get_drvdata(pctldev);
+
+	/* The special case with the 3516 flash pin */
+	if (pmx->flash_pin &&
+	    pmx->is_3512 &&
+	    !strcmp(gemini_3512_pin_groups[selector].name, "pflashgrp")) {
+		*pins = pflash_3512_pins_extended;
+		*num_pins = ARRAY_SIZE(pflash_3512_pins_extended);
+		return 0;
+	}
+	if (pmx->flash_pin &&
+	    pmx->is_3516 &&
+	    !strcmp(gemini_3516_pin_groups[selector].name, "pflashgrp")) {
+		*pins = pflash_3516_pins_extended;
+		*num_pins = ARRAY_SIZE(pflash_3516_pins_extended);
+		return 0;
+	}
+	if (pmx->is_3512) {
+		*pins = gemini_3512_pin_groups[selector].pins;
+		*num_pins = gemini_3512_pin_groups[selector].num_pins;
+	}
+	if (pmx->is_3516) {
+		*pins = gemini_3516_pin_groups[selector].pins;
+		*num_pins = gemini_3516_pin_groups[selector].num_pins;
+	}
+	return 0;
+}
+
+static void gemini_pin_dbg_show(struct pinctrl_dev *pctldev, struct seq_file *s,
+				unsigned int offset)
+{
+	seq_printf(s, " " DRIVER_NAME);
+}
+
+static int gemini_pinctrl_dt_subnode_to_map(struct pinctrl_dev *pctldev,
+					    struct device_node *np,
+					    struct pinctrl_map **map,
+					    unsigned int *reserved_maps,
+					    unsigned int *num_maps)
+{
+	int ret;
+	const char *function = NULL;
+	const char *group;
+	struct property *prop;
+
+	ret = of_property_read_string(np, "function", &function);
+	if (ret < 0)
+		return ret;
+
+	ret = of_property_count_strings(np, "groups");
+	if (ret < 0)
+		return ret;
+
+	ret = pinctrl_utils_reserve_map(pctldev, map, reserved_maps,
+					num_maps, ret);
+	if (ret < 0)
+		return ret;
+
+	of_property_for_each_string(np, "groups", prop, group) {
+		ret = pinctrl_utils_add_map_mux(pctldev, map, reserved_maps,
+						num_maps, group, function);
+		if (ret < 0)
+			return ret;
+		pr_debug("ADDED FUNCTION %s <-> GROUP %s\n",
+			 function, group);
+	}
+
+	return 0;
+}
+
+static int gemini_pinctrl_dt_node_to_map(struct pinctrl_dev *pctldev,
+				struct device_node *np_config,
+				struct pinctrl_map **map,
+				unsigned int *num_maps)
+{
+	unsigned int reserved_maps = 0;
+	struct device_node *np;
+	int ret;
+
+	*map = NULL;
+	*num_maps = 0;
+
+	for_each_child_of_node(np_config, np) {
+		ret = gemini_pinctrl_dt_subnode_to_map(pctldev, np, map,
+					&reserved_maps, num_maps);
+		if (ret < 0) {
+			pinctrl_utils_free_map(pctldev, *map, *num_maps);
+			return ret;
+		}
+	}
+
+	return 0;
+};
+
+static const struct pinctrl_ops gemini_pctrl_ops = {
+	.get_groups_count = gemini_get_groups_count,
+	.get_group_name = gemini_get_group_name,
+	.get_group_pins = gemini_get_group_pins,
+	.pin_dbg_show = gemini_pin_dbg_show,
+	.dt_node_to_map = gemini_pinctrl_dt_node_to_map,
+	.dt_free_map = pinctrl_utils_free_map,
+};
+
+/**
+ * struct gemini_pmx_func - describes Gemini pinmux functions
+ * @name: the name of this specific function
+ * @groups: corresponding pin groups
+ */
+struct gemini_pmx_func {
+	const char *name;
+	const char * const *groups;
+	const unsigned int num_groups;
+};
+
+static const char * const dramgrps[] = { "dramgrp" };
+static const char * const rtcgrps[] = { "rtcgrp" };
+static const char * const powergrps[] = { "powergrp" };
+static const char * const cirgrps[] = { "cirgrp" };
+static const char * const systemgrps[] = { "systemgrp" };
+static const char * const vcontrolgrps[] = { "vcontrolgrp" };
+static const char * const icegrps[] = { "icegrp" };
+static const char * const idegrps[] = { "idegrp" };
+static const char * const satagrps[] = { "satagrp" };
+static const char * const usbgrps[] = { "usbgrp" };
+static const char * const gmiigrps[] = { "gmiigrp" };
+static const char * const pcigrps[] = { "pcigrp" };
+static const char * const lpcgrps[] = { "lpcgrp" };
+static const char * const lcdgrps[] = { "lcdgrp" };
+static const char * const sspgrps[] = { "sspgrp" };
+static const char * const uartgrps[] = { "uartrxtxgrp", "uartmodemgrp" };
+static const char * const tvcgrps[] = { "tvcgrp" };
+static const char * const nflashgrps[] = { "nflashgrp" };
+static const char * const pflashgrps[] = { "pflashgrp", "pflashextgrp" };
+static const char * const sflashgrps[] = { "sflashgrp" };
+static const char * const gpio0grps[] = { "gpio0agrp", "gpio0bgrp", "gpio0cgrp",
+					  "gpio0dgrp", "gpio0egrp", "gpio0fgrp",
+					  "gpio0ggrp", "gpio0hgrp", "gpio0igrp",
+					  "gpio0jgrp", "gpio0kgrp" };
+static const char * const gpio1grps[] = { "gpio1agrp", "gpio1bgrp", "gpio1cgrp",
+					  "gpio1dgrp" };
+static const char * const gpio2grps[] = { "gpio2agrp", "gpio2bgrp", "gpio2cgrp" };
+
+static const struct gemini_pmx_func gemini_pmx_functions[] = {
+	{
+		.name = "dram",
+		.groups = dramgrps,
+		.num_groups = ARRAY_SIZE(idegrps),
+	},
+	{
+		.name = "rtc",
+		.groups = rtcgrps,
+		.num_groups = ARRAY_SIZE(rtcgrps),
+	},
+	{
+		.name = "power",
+		.groups = powergrps,
+		.num_groups = ARRAY_SIZE(powergrps),
+	},
+	{
+		/* This function is strictly unavailable on 3512 */
+		.name = "cir",
+		.groups = cirgrps,
+		.num_groups = ARRAY_SIZE(cirgrps),
+	},
+	{
+		.name = "system",
+		.groups = systemgrps,
+		.num_groups = ARRAY_SIZE(systemgrps),
+	},
+	{
+		.name = "vcontrol",
+		.groups = vcontrolgrps,
+		.num_groups = ARRAY_SIZE(vcontrolgrps),
+	},
+	{
+		.name = "ice",
+		.groups = icegrps,
+		.num_groups = ARRAY_SIZE(icegrps),
+	},
+	{
+		.name = "ide",
+		.groups = idegrps,
+		.num_groups = ARRAY_SIZE(idegrps),
+	},
+	{
+		.name = "sata",
+		.groups = satagrps,
+		.num_groups = ARRAY_SIZE(satagrps),
+	},
+	{
+		.name = "pci",
+		.groups = pcigrps,
+		.num_groups = ARRAY_SIZE(pcigrps),
+	},
+	{
+		.name = "lpc",
+		.groups = lpcgrps,
+		.num_groups = ARRAY_SIZE(lpcgrps),
+	},
+	{
+		.name = "lcd",
+		.groups = lcdgrps,
+		.num_groups = ARRAY_SIZE(lcdgrps),
+	},
+	{
+		.name = "ssp",
+		.groups = sspgrps,
+		.num_groups = ARRAY_SIZE(sspgrps),
+	},
+	{
+		.name = "uart",
+		.groups = uartgrps,
+		.num_groups = ARRAY_SIZE(uartgrps),
+	},
+	{
+		.name = "tvc",
+		.groups = tvcgrps,
+		.num_groups = ARRAY_SIZE(tvcgrps),
+	},
+	{
+		.name = "nflash",
+		.groups = nflashgrps,
+		.num_groups = ARRAY_SIZE(nflashgrps),
+	},
+	{
+		.name = "pflash",
+		.groups = pflashgrps,
+		.num_groups = ARRAY_SIZE(pflashgrps),
+	},
+	{
+		.name = "sflash",
+		.groups = sflashgrps,
+		.num_groups = ARRAY_SIZE(sflashgrps),
+	},
+	{
+		.name = "gpio0",
+		.groups = gpio0grps,
+		.num_groups = ARRAY_SIZE(gpio0grps),
+	},
+	{
+		.name = "gpio1",
+		.groups = gpio1grps,
+		.num_groups = ARRAY_SIZE(gpio1grps),
+	},
+	{
+		.name = "gpio2",
+		.groups = gpio2grps,
+		.num_groups = ARRAY_SIZE(gpio2grps),
+	},
+};
+
+
+static int gemini_pmx_set_mux(struct pinctrl_dev *pctldev,
+			      unsigned int selector,
+			      unsigned int group)
+{
+	struct gemini_pmx *pmx;
+	const struct gemini_pmx_func *func;
+	const struct gemini_pin_group *grp;
+	u32 before, after, expected;
+	unsigned long tmp;
+	int i;
+
+	pmx = pinctrl_dev_get_drvdata(pctldev);
+
+	func = &gemini_pmx_functions[selector];
+	if (pmx->is_3512)
+		grp = &gemini_3512_pin_groups[group];
+	else if (pmx->is_3516)
+		grp = &gemini_3516_pin_groups[group];
+	else {
+		dev_err(pmx->dev, "invalid SoC type\n");
+		return -ENODEV;
+	}
+
+	dev_info(pmx->dev,
+		 "ACTIVATE function \"%s\" with group \"%s\"\n",
+		 func->name, grp->name);
+
+	regmap_read(pmx->map, GLOBAL_MISC_CTRL, &before);
+	regmap_update_bits(pmx->map, GLOBAL_MISC_CTRL, grp->mask,
+			   grp->value);
+	regmap_read(pmx->map, GLOBAL_MISC_CTRL, &after);
+
+	/* Which bits changed */
+	before &= PADS_MASK;
+	after &= PADS_MASK;
+	expected = before &= ~grp->mask;
+	expected |= grp->value;
+	expected &= PADS_MASK;
+
+	/* Print changed states */
+	tmp = grp->mask;
+	for_each_set_bit(i, &tmp, PADS_MAXBIT) {
+		bool enabled = !(i > 3);
+
+		/* Did not go low though it should */
+		if (after & BIT(i)) {
+			dev_err(pmx->dev,
+				"pin group %s could not be %s: "
+				"probably a hardware limitation\n",
+				gemini_padgroups[i],
+				enabled ? "enabled" : "disabled");
+			dev_err(pmx->dev,
+				"GLOBAL MISC CTRL before: %08x, after %08x, expected %08x\n",
+				before, after, expected);
+		} else {
+			dev_info(pmx->dev,
+				 "padgroup %s %s\n",
+				 gemini_padgroups[i],
+				 enabled ? "enabled" : "disabled");
+		}
+	}
+
+	tmp = grp->value;
+	for_each_set_bit(i, &tmp, PADS_MAXBIT) {
+		bool enabled = (i > 3);
+
+		/* Did not go high though it should */
+		if (!(after & BIT(i))) {
+			dev_err(pmx->dev,
+				"pin group %s could not be %s: "
+				"probably a hardware limitation\n",
+				gemini_padgroups[i],
+				enabled ? "enabled" : "disabled");
+			dev_err(pmx->dev,
+				"GLOBAL MISC CTRL before: %08x, after %08x, expected %08x\n",
+				before, after, expected);
+		} else {
+			dev_info(pmx->dev,
+				 "padgroup %s %s\n",
+				 gemini_padgroups[i],
+				 enabled ? "enabled" : "disabled");
+		}
+	}
+
+	return 0;
+}
+
+static int gemini_pmx_get_funcs_count(struct pinctrl_dev *pctldev)
+{
+	return ARRAY_SIZE(gemini_pmx_functions);
+}
+
+static const char *gemini_pmx_get_func_name(struct pinctrl_dev *pctldev,
+					    unsigned int selector)
+{
+	return gemini_pmx_functions[selector].name;
+}
+
+static int gemini_pmx_get_groups(struct pinctrl_dev *pctldev,
+				 unsigned int selector,
+				 const char * const **groups,
+				 unsigned int * const num_groups)
+{
+	*groups = gemini_pmx_functions[selector].groups;
+	*num_groups = gemini_pmx_functions[selector].num_groups;
+	return 0;
+}
+
+static const struct pinmux_ops gemini_pmx_ops = {
+	.get_functions_count = gemini_pmx_get_funcs_count,
+	.get_function_name = gemini_pmx_get_func_name,
+	.get_function_groups = gemini_pmx_get_groups,
+	.set_mux = gemini_pmx_set_mux,
+};
+
+static struct pinctrl_desc gemini_pmx_desc = {
+	.name = DRIVER_NAME,
+	.pctlops = &gemini_pctrl_ops,
+	.pmxops = &gemini_pmx_ops,
+	.owner = THIS_MODULE,
+};
+
+static int gemini_pmx_probe(struct platform_device *pdev)
+{
+	struct gemini_pmx *pmx;
+	struct regmap *map;
+	struct device *dev = &pdev->dev;
+	struct device *parent;
+	unsigned long tmp;
+	u32 val;
+	int ret;
+	int i;
+
+	/* Create state holders etc for this driver */
+	pmx = devm_kzalloc(&pdev->dev, sizeof(*pmx), GFP_KERNEL);
+	if (!pmx)
+		return -ENOMEM;
+
+	pmx->dev = &pdev->dev;
+	parent = dev->parent;
+	if (!parent) {
+		dev_err(dev, "no parent to pin controller\n");
+		return -ENODEV;
+	}
+	map = syscon_node_to_regmap(parent->of_node);
+	if (IS_ERR(map)) {
+		dev_err(dev, "no syscon regmap\n");
+		return PTR_ERR(map);
+	}
+	pmx->map = map;
+
+	/* Check that regmap works at first call, then no more */
+	ret = regmap_read(map, GLOBAL_WORD_ID, &val);
+	if (ret) {
+		dev_err(dev, "cannot access regmap\n");
+		return ret;
+	}
+	val >>= 8;
+	val &= 0xffff;
+	if (val == 0x3512) {
+		pmx->is_3512 = true;
+		gemini_pmx_desc.pins = gemini_3512_pins;
+		gemini_pmx_desc.npins = ARRAY_SIZE(gemini_3512_pins);
+		dev_info(dev, "detected 3512 chip variant\n");
+	} else if (val == 0x3516) {
+		pmx->is_3516 = true;
+		gemini_pmx_desc.pins = gemini_3516_pins;
+		gemini_pmx_desc.npins = ARRAY_SIZE(gemini_3516_pins);
+		dev_info(dev, "detected 3516 chip variant\n");
+	} else {
+		dev_err(dev, "unknown chip ID: %04x\n", val);
+		return -ENODEV;
+	}
+
+	ret = regmap_read(map, GLOBAL_MISC_CTRL, &val);
+	dev_info(dev, "GLOBAL MISC CTRL at boot: 0x%08x\n", val);
+	/* Mask off relevant pads */
+	val &= PADS_MASK;
+	/* Invert the meaning of the DRAM+flash pads */
+	val ^= 0x0f;
+	/* Print initial state */
+	tmp = val;
+	for_each_set_bit(i, &tmp, PADS_MAXBIT) {
+		dev_info(dev, "pad group %s %s\n", gemini_padgroups[i],
+			 (val & BIT(i)) ? "enabled" : "disabled");
+	}
+
+	/* Check if flash pin is set */
+	regmap_read(map, GLOBAL_STATUS, &val);
+	pmx->flash_pin = !!(val & GLOBAL_STATUS_FLPIN);
+	dev_info(dev, "flash pin is %s\n", pmx->flash_pin ? "set" : "not set");
+
+	pmx->pctl = devm_pinctrl_register(dev, &gemini_pmx_desc, pmx);
+	if (IS_ERR(pmx->pctl)) {
+		dev_err(dev, "could not register pinmux driver\n");
+		return PTR_ERR(pmx->pctl);
+	}
+
+	dev_info(dev, "initialized Gemini pin control driver\n");
+
+	return 0;
+}
+
+static const struct of_device_id gemini_pinctrl_match[] = {
+	{ .compatible = "cortina,gemini-pinctrl" },
+	{},
+};
+
+static struct platform_driver gemini_pmx_driver = {
+	.driver = {
+		.name = DRIVER_NAME,
+		.of_match_table = gemini_pinctrl_match,
+	},
+	.probe = gemini_pmx_probe,
+};
+
+static int __init gemini_pmx_init(void)
+{
+	return platform_driver_register(&gemini_pmx_driver);
+}
+arch_initcall(gemini_pmx_init);
diff --git a/drivers/pinctrl/pinctrl-ingenic.c b/drivers/pinctrl/pinctrl-ingenic.c
index d8e8842967d6..d84761822243 100644
--- a/drivers/pinctrl/pinctrl-ingenic.c
+++ b/drivers/pinctrl/pinctrl-ingenic.c
@@ -460,7 +460,7 @@ static inline bool ingenic_get_pin_config(struct ingenic_pinctrl *jzpc,
 	return val & BIT(idx);
 }
 
-static struct pinctrl_ops ingenic_pctlops = {
+static const struct pinctrl_ops ingenic_pctlops = {
 	.get_groups_count = pinctrl_generic_get_group_count,
 	.get_group_name = pinctrl_generic_get_group_name,
 	.get_group_pins = pinctrl_generic_get_group_pins,
@@ -543,7 +543,7 @@ static int ingenic_pinmux_gpio_set_direction(struct pinctrl_dev *pctldev,
 	return 0;
 }
 
-static struct pinmux_ops ingenic_pmxops = {
+static const struct pinmux_ops ingenic_pmxops = {
 	.get_functions_count = pinmux_generic_get_function_count,
 	.get_function_name = pinmux_generic_get_function_name,
 	.get_function_groups = pinmux_generic_get_function_groups,
@@ -696,7 +696,7 @@ static int ingenic_pinconf_group_set(struct pinctrl_dev *pctldev,
 	return 0;
 }
 
-static struct pinconf_ops ingenic_confops = {
+static const struct pinconf_ops ingenic_confops = {
 	.is_generic = true,
 	.pin_config_get = ingenic_pinconf_get,
 	.pin_config_set = ingenic_pinconf_set,
diff --git a/drivers/pinctrl/pinctrl-rk805.c b/drivers/pinctrl/pinctrl-rk805.c
new file mode 100644
index 000000000000..b0bfd3082a1b
--- /dev/null
+++ b/drivers/pinctrl/pinctrl-rk805.c
@@ -0,0 +1,493 @@
+/*
+ * Pinctrl driver for Rockchip RK805 PMIC
+ *
+ * Copyright (c) 2017, Fuzhou Rockchip Electronics Co., Ltd
+ *
+ * Author: Joseph Chen <chenjh@rock-chips.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under  the terms of the GNU General  Public License as published by the
+ * Free Software Foundation;  either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * Based on the pinctrl-as3722 driver
+ */
+
+#include <linux/gpio/driver.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mfd/rk808.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/pinctrl/consumer.h>
+#include <linux/pinctrl/machine.h>
+#include <linux/pinctrl/pinctrl.h>
+#include <linux/pinctrl/pinconf-generic.h>
+#include <linux/pinctrl/pinconf.h>
+#include <linux/pinctrl/pinmux.h>
+#include <linux/pm.h>
+#include <linux/slab.h>
+
+#include "core.h"
+#include "pinconf.h"
+#include "pinctrl-utils.h"
+
+struct rk805_pin_function {
+	const char *name;
+	const char *const *groups;
+	unsigned int ngroups;
+	int mux_option;
+};
+
+struct rk805_pin_group {
+	const char *name;
+	const unsigned int pins[1];
+	unsigned int npins;
+};
+
+/*
+ * @reg: gpio setting register;
+ * @fun_mask: functions select mask value, when set is gpio;
+ * @dir_mask: input or output mask value, when set is output, otherwise input;
+ * @val_mask: gpio set value, when set is level high, otherwise low;
+ *
+ * Different PMIC has different pin features, belowing 3 mask members are not
+ * all necessary for every PMIC. For example, RK805 has 2 pins that can be used
+ * as output only GPIOs, so func_mask and dir_mask are not needed. RK816 has 1
+ * pin that can be used as TS/GPIO, so fun_mask, dir_mask and val_mask are all
+ * necessary.
+ */
+struct rk805_pin_config {
+	u8 reg;
+	u8 fun_msk;
+	u8 dir_msk;
+	u8 val_msk;
+};
+
+struct rk805_pctrl_info {
+	struct rk808 *rk808;
+	struct device *dev;
+	struct pinctrl_dev *pctl;
+	struct gpio_chip gpio_chip;
+	struct pinctrl_desc pinctrl_desc;
+	const struct rk805_pin_function *functions;
+	unsigned int num_functions;
+	const struct rk805_pin_group *groups;
+	int num_pin_groups;
+	const struct pinctrl_pin_desc *pins;
+	unsigned int num_pins;
+	struct rk805_pin_config *pin_cfg;
+};
+
+enum rk805_pinmux_option {
+	RK805_PINMUX_GPIO,
+};
+
+enum {
+	RK805_GPIO0,
+	RK805_GPIO1,
+};
+
+static const char *const rk805_gpio_groups[] = {
+	"gpio0",
+	"gpio1",
+};
+
+/* RK805: 2 output only GPIOs */
+static const struct pinctrl_pin_desc rk805_pins_desc[] = {
+	PINCTRL_PIN(RK805_GPIO0, "gpio0"),
+	PINCTRL_PIN(RK805_GPIO1, "gpio1"),
+};
+
+static const struct rk805_pin_function rk805_pin_functions[] = {
+	{
+		.name = "gpio",
+		.groups = rk805_gpio_groups,
+		.ngroups = ARRAY_SIZE(rk805_gpio_groups),
+		.mux_option = RK805_PINMUX_GPIO,
+	},
+};
+
+static const struct rk805_pin_group rk805_pin_groups[] = {
+	{
+		.name = "gpio0",
+		.pins = { RK805_GPIO0 },
+		.npins = 1,
+	},
+	{
+		.name = "gpio1",
+		.pins = { RK805_GPIO1 },
+		.npins = 1,
+	},
+};
+
+#define RK805_GPIO0_VAL_MSK	BIT(0)
+#define RK805_GPIO1_VAL_MSK	BIT(1)
+
+static struct rk805_pin_config rk805_gpio_cfgs[] = {
+	{
+		.reg = RK805_OUT_REG,
+		.val_msk = RK805_GPIO0_VAL_MSK,
+	},
+	{
+		.reg = RK805_OUT_REG,
+		.val_msk = RK805_GPIO1_VAL_MSK,
+	},
+};
+
+/* generic gpio chip */
+static int rk805_gpio_get(struct gpio_chip *chip, unsigned int offset)
+{
+	struct rk805_pctrl_info *pci = gpiochip_get_data(chip);
+	int ret, val;
+
+	ret = regmap_read(pci->rk808->regmap, pci->pin_cfg[offset].reg, &val);
+	if (ret) {
+		dev_err(pci->dev, "get gpio%d value failed\n", offset);
+		return ret;
+	}
+
+	return !!(val & pci->pin_cfg[offset].val_msk);
+}
+
+static void rk805_gpio_set(struct gpio_chip *chip,
+			   unsigned int offset,
+			   int value)
+{
+	struct rk805_pctrl_info *pci = gpiochip_get_data(chip);
+	int ret;
+
+	ret = regmap_update_bits(pci->rk808->regmap,
+				 pci->pin_cfg[offset].reg,
+				 pci->pin_cfg[offset].val_msk,
+				 value ? pci->pin_cfg[offset].val_msk : 0);
+	if (ret)
+		dev_err(pci->dev, "set gpio%d value %d failed\n",
+			offset, value);
+}
+
+static int rk805_gpio_direction_input(struct gpio_chip *chip,
+				      unsigned int offset)
+{
+	return pinctrl_gpio_direction_input(chip->base + offset);
+}
+
+static int rk805_gpio_direction_output(struct gpio_chip *chip,
+				       unsigned int offset, int value)
+{
+	rk805_gpio_set(chip, offset, value);
+	return pinctrl_gpio_direction_output(chip->base + offset);
+}
+
+static int rk805_gpio_get_direction(struct gpio_chip *chip, unsigned int offset)
+{
+	struct rk805_pctrl_info *pci = gpiochip_get_data(chip);
+	unsigned int val;
+	int ret;
+
+	/* default output*/
+	if (!pci->pin_cfg[offset].dir_msk)
+		return 0;
+
+	ret = regmap_read(pci->rk808->regmap,
+			  pci->pin_cfg[offset].reg,
+			  &val);
+	if (ret) {
+		dev_err(pci->dev, "get gpio%d direction failed\n", offset);
+		return ret;
+	}
+
+	return !(val & pci->pin_cfg[offset].dir_msk);
+}
+
+static struct gpio_chip rk805_gpio_chip = {
+	.label			= "rk805-gpio",
+	.request		= gpiochip_generic_request,
+	.free			= gpiochip_generic_free,
+	.get_direction		= rk805_gpio_get_direction,
+	.get			= rk805_gpio_get,
+	.set			= rk805_gpio_set,
+	.direction_input	= rk805_gpio_direction_input,
+	.direction_output	= rk805_gpio_direction_output,
+	.can_sleep		= true,
+	.base			= -1,
+	.owner			= THIS_MODULE,
+};
+
+/* generic pinctrl */
+static int rk805_pinctrl_get_groups_count(struct pinctrl_dev *pctldev)
+{
+	struct rk805_pctrl_info *pci = pinctrl_dev_get_drvdata(pctldev);
+
+	return pci->num_pin_groups;
+}
+
+static const char *rk805_pinctrl_get_group_name(struct pinctrl_dev *pctldev,
+						unsigned int group)
+{
+	struct rk805_pctrl_info *pci = pinctrl_dev_get_drvdata(pctldev);
+
+	return pci->groups[group].name;
+}
+
+static int rk805_pinctrl_get_group_pins(struct pinctrl_dev *pctldev,
+					unsigned int group,
+					const unsigned int **pins,
+					unsigned int *num_pins)
+{
+	struct rk805_pctrl_info *pci = pinctrl_dev_get_drvdata(pctldev);
+
+	*pins = pci->groups[group].pins;
+	*num_pins = pci->groups[group].npins;
+
+	return 0;
+}
+
+static const struct pinctrl_ops rk805_pinctrl_ops = {
+	.get_groups_count = rk805_pinctrl_get_groups_count,
+	.get_group_name = rk805_pinctrl_get_group_name,
+	.get_group_pins = rk805_pinctrl_get_group_pins,
+	.dt_node_to_map = pinconf_generic_dt_node_to_map_pin,
+	.dt_free_map = pinctrl_utils_free_map,
+};
+
+static int rk805_pinctrl_get_funcs_count(struct pinctrl_dev *pctldev)
+{
+	struct rk805_pctrl_info *pci = pinctrl_dev_get_drvdata(pctldev);
+
+	return pci->num_functions;
+}
+
+static const char *rk805_pinctrl_get_func_name(struct pinctrl_dev *pctldev,
+					       unsigned int function)
+{
+	struct rk805_pctrl_info *pci = pinctrl_dev_get_drvdata(pctldev);
+
+	return pci->functions[function].name;
+}
+
+static int rk805_pinctrl_get_func_groups(struct pinctrl_dev *pctldev,
+					 unsigned int function,
+					 const char *const **groups,
+					 unsigned int *const num_groups)
+{
+	struct rk805_pctrl_info *pci = pinctrl_dev_get_drvdata(pctldev);
+
+	*groups = pci->functions[function].groups;
+	*num_groups = pci->functions[function].ngroups;
+
+	return 0;
+}
+
+static int _rk805_pinctrl_set_mux(struct pinctrl_dev *pctldev,
+				  unsigned int offset,
+				  int mux)
+{
+	struct rk805_pctrl_info *pci = pinctrl_dev_get_drvdata(pctldev);
+	int ret;
+
+	if (!pci->pin_cfg[offset].fun_msk)
+		return 0;
+
+	if (mux == RK805_PINMUX_GPIO) {
+		ret = regmap_update_bits(pci->rk808->regmap,
+					 pci->pin_cfg[offset].reg,
+					 pci->pin_cfg[offset].fun_msk,
+					 pci->pin_cfg[offset].fun_msk);
+		if (ret) {
+			dev_err(pci->dev, "set gpio%d GPIO failed\n", offset);
+			return ret;
+		}
+	} else {
+		dev_err(pci->dev, "Couldn't find function mux %d\n", mux);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int rk805_pinctrl_set_mux(struct pinctrl_dev *pctldev,
+				 unsigned int function,
+				 unsigned int group)
+{
+	struct rk805_pctrl_info *pci = pinctrl_dev_get_drvdata(pctldev);
+	int mux = pci->functions[function].mux_option;
+	int offset = group;
+
+	return _rk805_pinctrl_set_mux(pctldev, offset, mux);
+}
+
+static int rk805_pmx_gpio_set_direction(struct pinctrl_dev *pctldev,
+					struct pinctrl_gpio_range *range,
+					unsigned int offset, bool input)
+{
+	struct rk805_pctrl_info *pci = pinctrl_dev_get_drvdata(pctldev);
+	int ret;
+
+	/* switch to gpio function */
+	ret = _rk805_pinctrl_set_mux(pctldev, offset, RK805_PINMUX_GPIO);
+	if (ret) {
+		dev_err(pci->dev, "set gpio%d mux failed\n", offset);
+		return ret;
+	}
+
+	/* set direction */
+	if (!pci->pin_cfg[offset].dir_msk)
+		return 0;
+
+	ret = regmap_update_bits(pci->rk808->regmap,
+				 pci->pin_cfg[offset].reg,
+				 pci->pin_cfg[offset].dir_msk,
+				 input ? 0 : pci->pin_cfg[offset].dir_msk);
+	if (ret) {
+		dev_err(pci->dev, "set gpio%d direction failed\n", offset);
+		return ret;
+	}
+
+	return ret;
+}
+
+static const struct pinmux_ops rk805_pinmux_ops = {
+	.get_functions_count	= rk805_pinctrl_get_funcs_count,
+	.get_function_name	= rk805_pinctrl_get_func_name,
+	.get_function_groups	= rk805_pinctrl_get_func_groups,
+	.set_mux		= rk805_pinctrl_set_mux,
+	.gpio_set_direction	= rk805_pmx_gpio_set_direction,
+};
+
+static int rk805_pinconf_get(struct pinctrl_dev *pctldev,
+			     unsigned int pin, unsigned long *config)
+{
+	struct rk805_pctrl_info *pci = pinctrl_dev_get_drvdata(pctldev);
+	enum pin_config_param param = pinconf_to_config_param(*config);
+	u32 arg = 0;
+
+	switch (param) {
+	case PIN_CONFIG_OUTPUT:
+		arg = rk805_gpio_get(&pci->gpio_chip, pin);
+		break;
+	default:
+		dev_err(pci->dev, "Properties not supported\n");
+		return -ENOTSUPP;
+	}
+
+	*config = pinconf_to_config_packed(param, (u16)arg);
+
+	return 0;
+}
+
+static int rk805_pinconf_set(struct pinctrl_dev *pctldev,
+			     unsigned int pin, unsigned long *configs,
+			     unsigned int num_configs)
+{
+	struct rk805_pctrl_info *pci = pinctrl_dev_get_drvdata(pctldev);
+	enum pin_config_param param;
+	u32 i, arg = 0;
+
+	for (i = 0; i < num_configs; i++) {
+		param = pinconf_to_config_param(configs[i]);
+		arg = pinconf_to_config_argument(configs[i]);
+
+		switch (param) {
+		case PIN_CONFIG_OUTPUT:
+			rk805_gpio_set(&pci->gpio_chip, pin, arg);
+			rk805_pmx_gpio_set_direction(pctldev, NULL, pin, false);
+			break;
+		default:
+			dev_err(pci->dev, "Properties not supported\n");
+			return -ENOTSUPP;
+		}
+	}
+
+	return 0;
+}
+
+static const struct pinconf_ops rk805_pinconf_ops = {
+	.pin_config_get = rk805_pinconf_get,
+	.pin_config_set = rk805_pinconf_set,
+};
+
+static struct pinctrl_desc rk805_pinctrl_desc = {
+	.name = "rk805-pinctrl",
+	.pctlops = &rk805_pinctrl_ops,
+	.pmxops = &rk805_pinmux_ops,
+	.confops = &rk805_pinconf_ops,
+	.owner = THIS_MODULE,
+};
+
+static int rk805_pinctrl_probe(struct platform_device *pdev)
+{
+	struct rk805_pctrl_info *pci;
+	int ret;
+
+	pci = devm_kzalloc(&pdev->dev, sizeof(*pci), GFP_KERNEL);
+	if (!pci)
+		return -ENOMEM;
+
+	pci->dev = &pdev->dev;
+	pci->dev->of_node = pdev->dev.parent->of_node;
+	pci->rk808 = dev_get_drvdata(pdev->dev.parent);
+
+	pci->pinctrl_desc = rk805_pinctrl_desc;
+	pci->gpio_chip = rk805_gpio_chip;
+	pci->gpio_chip.parent = &pdev->dev;
+	pci->gpio_chip.of_node = pdev->dev.parent->of_node;
+
+	platform_set_drvdata(pdev, pci);
+
+	switch (pci->rk808->variant) {
+	case RK805_ID:
+		pci->pins = rk805_pins_desc;
+		pci->num_pins = ARRAY_SIZE(rk805_pins_desc);
+		pci->functions = rk805_pin_functions;
+		pci->num_functions = ARRAY_SIZE(rk805_pin_functions);
+		pci->groups = rk805_pin_groups;
+		pci->num_pin_groups = ARRAY_SIZE(rk805_pin_groups);
+		pci->pinctrl_desc.pins = rk805_pins_desc;
+		pci->pinctrl_desc.npins = ARRAY_SIZE(rk805_pins_desc);
+		pci->pin_cfg = rk805_gpio_cfgs;
+		pci->gpio_chip.ngpio = ARRAY_SIZE(rk805_gpio_cfgs);
+		break;
+	default:
+		dev_err(&pdev->dev, "unsupported RK805 ID %lu\n",
+			pci->rk808->variant);
+		return -EINVAL;
+	}
+
+	/* Add gpio chip */
+	ret = devm_gpiochip_add_data(&pdev->dev, &pci->gpio_chip, pci);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "Couldn't add gpiochip\n");
+		return ret;
+	}
+
+	/* Add pinctrl */
+	pci->pctl = devm_pinctrl_register(&pdev->dev, &pci->pinctrl_desc, pci);
+	if (IS_ERR(pci->pctl)) {
+		dev_err(&pdev->dev, "Couldn't add pinctrl\n");
+		return PTR_ERR(pci->pctl);
+	}
+
+	/* Add pin range */
+	ret = gpiochip_add_pin_range(&pci->gpio_chip, dev_name(&pdev->dev),
+				     0, 0, pci->gpio_chip.ngpio);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "Couldn't add gpiochip pin range\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static struct platform_driver rk805_pinctrl_driver = {
+	.probe = rk805_pinctrl_probe,
+	.driver = {
+		.name = "rk805-pinctrl",
+	},
+};
+module_platform_driver(rk805_pinctrl_driver);
+
+MODULE_DESCRIPTION("RK805 pin control and GPIO driver");
+MODULE_AUTHOR("Joseph Chen <chenjh@rock-chips.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/pinctrl/pinctrl-rockchip.c b/drivers/pinctrl/pinctrl-rockchip.c
index e831647c56a6..b5cb7858ffdc 100644
--- a/drivers/pinctrl/pinctrl-rockchip.c
+++ b/drivers/pinctrl/pinctrl-rockchip.c
@@ -62,6 +62,7 @@ enum rockchip_pinctrl_type {
 	RV1108,
 	RK2928,
 	RK3066B,
+	RK3128,
 	RK3188,
 	RK3288,
 	RK3368,
@@ -76,7 +77,6 @@ enum rockchip_pinctrl_type {
 #define IOMUX_SOURCE_PMU	BIT(2)
 #define IOMUX_UNROUTED		BIT(3)
 #define IOMUX_WIDTH_3BIT	BIT(4)
-#define IOMUX_RECALCED		BIT(5)
 
 /**
  * @type: iomux variant using IOMUX_* constants
@@ -166,6 +166,7 @@ struct rockchip_pin_bank {
 	struct pinctrl_gpio_range	grange;
 	raw_spinlock_t			slock;
 	u32				toggle_edge_mode;
+	u32				recalced_mask;
 	u32				route_mask;
 };
 
@@ -291,6 +292,22 @@ struct rockchip_pin_bank {
 
 /**
  * struct rockchip_mux_recalced_data: represent a pin iomux data.
+ * @num: bank number.
+ * @pin: pin number.
+ * @bit: index at register.
+ * @reg: register offset.
+ * @mask: mask bit
+ */
+struct rockchip_mux_recalced_data {
+	u8 num;
+	u8 pin;
+	u32 reg;
+	u8 bit;
+	u8 mask;
+};
+
+/**
+ * struct rockchip_mux_recalced_data: represent a pin iomux data.
  * @bank_num: bank number.
  * @pin: index at register or used to calc index.
  * @func: the min pin.
@@ -317,6 +334,8 @@ struct rockchip_pin_ctrl {
 	int				pmu_mux_offset;
 	int				grf_drv_offset;
 	int				pmu_drv_offset;
+	struct rockchip_mux_recalced_data *iomux_recalced;
+	u32				niomux_recalced;
 	struct rockchip_mux_route_data *iomux_routes;
 	u32				niomux_routes;
 
@@ -326,8 +345,6 @@ struct rockchip_pin_ctrl {
 	void	(*drv_calc_reg)(struct rockchip_pin_bank *bank,
 				    int pin_num, struct regmap **regmap,
 				    int *reg, u8 *bit);
-	void	(*iomux_recalc)(u8 bank_num, int pin, int *reg,
-				u8 *bit, int *mask);
 	int	(*schmitt_calc_reg)(struct rockchip_pin_bank *bank,
 				    int pin_num, struct regmap **regmap,
 				    int *reg, u8 *bit);
@@ -382,22 +399,6 @@ struct rockchip_pinctrl {
 	unsigned int			nfunctions;
 };
 
-/**
- * struct rockchip_mux_recalced_data: represent a pin iomux data.
- * @num: bank number.
- * @pin: pin number.
- * @bit: index at register.
- * @reg: register offset.
- * @mask: mask bit
- */
-struct rockchip_mux_recalced_data {
-	u8 num;
-	u8 pin;
-	u8 reg;
-	u8 bit;
-	u8 mask;
-};
-
 static struct regmap_config rockchip_regmap_config = {
 	.reg_bits = 32,
 	.val_bits = 32,
@@ -557,7 +558,105 @@ static const struct pinctrl_ops rockchip_pctrl_ops = {
  * Hardware access
  */
 
-static const struct rockchip_mux_recalced_data rk3328_mux_recalced_data[] = {
+static struct rockchip_mux_recalced_data rv1108_mux_recalced_data[] = {
+	{
+		.num = 1,
+		.pin = 0,
+		.reg = 0x418,
+		.bit = 0,
+		.mask = 0x3
+	}, {
+		.num = 1,
+		.pin = 1,
+		.reg = 0x418,
+		.bit = 2,
+		.mask = 0x3
+	}, {
+		.num = 1,
+		.pin = 2,
+		.reg = 0x418,
+		.bit = 4,
+		.mask = 0x3
+	}, {
+		.num = 1,
+		.pin = 3,
+		.reg = 0x418,
+		.bit = 6,
+		.mask = 0x3
+	}, {
+		.num = 1,
+		.pin = 4,
+		.reg = 0x418,
+		.bit = 8,
+		.mask = 0x3
+	}, {
+		.num = 1,
+		.pin = 5,
+		.reg = 0x418,
+		.bit = 10,
+		.mask = 0x3
+	}, {
+		.num = 1,
+		.pin = 6,
+		.reg = 0x418,
+		.bit = 12,
+		.mask = 0x3
+	}, {
+		.num = 1,
+		.pin = 7,
+		.reg = 0x418,
+		.bit = 14,
+		.mask = 0x3
+	}, {
+		.num = 1,
+		.pin = 8,
+		.reg = 0x41c,
+		.bit = 0,
+		.mask = 0x3
+	}, {
+		.num = 1,
+		.pin = 9,
+		.reg = 0x41c,
+		.bit = 2,
+		.mask = 0x3
+	},
+};
+
+static  struct rockchip_mux_recalced_data rk3128_mux_recalced_data[] = {
+	{
+		.num = 2,
+		.pin = 20,
+		.reg = 0xe8,
+		.bit = 0,
+		.mask = 0x7
+	}, {
+		.num = 2,
+		.pin = 21,
+		.reg = 0xe8,
+		.bit = 4,
+		.mask = 0x7
+	}, {
+		.num = 2,
+		.pin = 22,
+		.reg = 0xe8,
+		.bit = 8,
+		.mask = 0x7
+	}, {
+		.num = 2,
+		.pin = 23,
+		.reg = 0xe8,
+		.bit = 12,
+		.mask = 0x7
+	}, {
+		.num = 2,
+		.pin = 24,
+		.reg = 0xd4,
+		.bit = 12,
+		.mask = 0x7
+	},
+};
+
+static struct rockchip_mux_recalced_data rk3328_mux_recalced_data[] = {
 	{
 		.num = 2,
 		.pin = 12,
@@ -579,20 +678,22 @@ static const struct rockchip_mux_recalced_data rk3328_mux_recalced_data[] = {
 	},
 };
 
-static void rk3328_recalc_mux(u8 bank_num, int pin, int *reg,
-			      u8 *bit, int *mask)
+static void rockchip_get_recalced_mux(struct rockchip_pin_bank *bank, int pin,
+				      int *reg, u8 *bit, int *mask)
 {
-	const struct rockchip_mux_recalced_data *data = NULL;
+	struct rockchip_pinctrl *info = bank->drvdata;
+	struct rockchip_pin_ctrl *ctrl = info->ctrl;
+	struct rockchip_mux_recalced_data *data;
 	int i;
 
-	for (i = 0; i < ARRAY_SIZE(rk3328_mux_recalced_data); i++)
-		if (rk3328_mux_recalced_data[i].num == bank_num &&
-		    rk3328_mux_recalced_data[i].pin == pin) {
-			data = &rk3328_mux_recalced_data[i];
+	for (i = 0; i < ctrl->niomux_recalced; i++) {
+		data = &ctrl->iomux_recalced[i];
+		if (data->num == bank->bank_num &&
+		    data->pin == pin)
 			break;
-		}
+	}
 
-	if (!data)
+	if (i >= ctrl->niomux_recalced)
 		return;
 
 	*reg = data->reg;
@@ -600,6 +701,59 @@ static void rk3328_recalc_mux(u8 bank_num, int pin, int *reg,
 	*bit = data->bit;
 }
 
+static struct rockchip_mux_route_data rk3128_mux_route_data[] = {
+	{
+		/* spi-0 */
+		.bank_num = 1,
+		.pin = 10,
+		.func = 1,
+		.route_offset = 0x144,
+		.route_val = BIT(16 + 3) | BIT(16 + 4),
+	}, {
+		/* spi-1 */
+		.bank_num = 1,
+		.pin = 27,
+		.func = 3,
+		.route_offset = 0x144,
+		.route_val = BIT(16 + 3) | BIT(16 + 4) | BIT(3),
+	}, {
+		/* spi-2 */
+		.bank_num = 0,
+		.pin = 13,
+		.func = 2,
+		.route_offset = 0x144,
+		.route_val = BIT(16 + 3) | BIT(16 + 4) | BIT(4),
+	}, {
+		/* i2s-0 */
+		.bank_num = 1,
+		.pin = 5,
+		.func = 1,
+		.route_offset = 0x144,
+		.route_val = BIT(16 + 5),
+	}, {
+		/* i2s-1 */
+		.bank_num = 0,
+		.pin = 14,
+		.func = 1,
+		.route_offset = 0x144,
+		.route_val = BIT(16 + 5) | BIT(5),
+	}, {
+		/* emmc-0 */
+		.bank_num = 1,
+		.pin = 22,
+		.func = 2,
+		.route_offset = 0x144,
+		.route_val = BIT(16 + 6),
+	}, {
+		/* emmc-1 */
+		.bank_num = 2,
+		.pin = 4,
+		.func = 2,
+		.route_offset = 0x144,
+		.route_val = BIT(16 + 6) | BIT(6),
+	},
+};
+
 static struct rockchip_mux_route_data rk3228_mux_route_data[] = {
 	{
 		/* pwm0-0 */
@@ -877,7 +1031,6 @@ static bool rockchip_get_mux_route(struct rockchip_pin_bank *bank, int pin,
 static int rockchip_get_mux(struct rockchip_pin_bank *bank, int pin)
 {
 	struct rockchip_pinctrl *info = bank->drvdata;
-	struct rockchip_pin_ctrl *ctrl = info->ctrl;
 	int iomux_num = (pin / 8);
 	struct regmap *regmap;
 	unsigned int val;
@@ -916,8 +1069,8 @@ static int rockchip_get_mux(struct rockchip_pin_bank *bank, int pin)
 		mask = 0x3;
 	}
 
-	if (ctrl->iomux_recalc && (mux_type & IOMUX_RECALCED))
-		ctrl->iomux_recalc(bank->bank_num, pin, &reg, &bit, &mask);
+	if (bank->recalced_mask & BIT(pin))
+		rockchip_get_recalced_mux(bank, pin, &reg, &bit, &mask);
 
 	ret = regmap_read(regmap, reg, &val);
 	if (ret)
@@ -967,7 +1120,6 @@ static int rockchip_verify_mux(struct rockchip_pin_bank *bank,
 static int rockchip_set_mux(struct rockchip_pin_bank *bank, int pin, int mux)
 {
 	struct rockchip_pinctrl *info = bank->drvdata;
-	struct rockchip_pin_ctrl *ctrl = info->ctrl;
 	int iomux_num = (pin / 8);
 	struct regmap *regmap;
 	int reg, ret, mask, mux_type;
@@ -1005,8 +1157,8 @@ static int rockchip_set_mux(struct rockchip_pin_bank *bank, int pin, int mux)
 		mask = 0x3;
 	}
 
-	if (ctrl->iomux_recalc && (mux_type & IOMUX_RECALCED))
-		ctrl->iomux_recalc(bank->bank_num, pin, &reg, &bit, &mask);
+	if (bank->recalced_mask & BIT(pin))
+		rockchip_get_recalced_mux(bank, pin, &reg, &bit, &mask);
 
 	if (bank->route_mask & BIT(pin)) {
 		if (rockchip_get_mux_route(bank, pin, mux, &route_reg,
@@ -1084,6 +1236,36 @@ static void rv1108_calc_drv_reg_and_bit(struct rockchip_pin_bank *bank,
 	*bit *= RV1108_DRV_BITS_PER_PIN;
 }
 
+#define RV1108_SCHMITT_PMU_OFFSET		0x30
+#define RV1108_SCHMITT_GRF_OFFSET		0x388
+#define RV1108_SCHMITT_BANK_STRIDE		8
+#define RV1108_SCHMITT_PINS_PER_GRF_REG		16
+#define RV1108_SCHMITT_PINS_PER_PMU_REG		8
+
+static int rv1108_calc_schmitt_reg_and_bit(struct rockchip_pin_bank *bank,
+					   int pin_num,
+					   struct regmap **regmap,
+					   int *reg, u8 *bit)
+{
+	struct rockchip_pinctrl *info = bank->drvdata;
+	int pins_per_reg;
+
+	if (bank->bank_num == 0) {
+		*regmap = info->regmap_pmu;
+		*reg = RV1108_SCHMITT_PMU_OFFSET;
+		pins_per_reg = RV1108_SCHMITT_PINS_PER_PMU_REG;
+	} else {
+		*regmap = info->regmap_base;
+		*reg = RV1108_SCHMITT_GRF_OFFSET;
+		pins_per_reg = RV1108_SCHMITT_PINS_PER_GRF_REG;
+		*reg += (bank->bank_num  - 1) * RV1108_SCHMITT_BANK_STRIDE;
+	}
+	*reg += ((pin_num / pins_per_reg) * 4);
+	*bit = pin_num % pins_per_reg;
+
+	return 0;
+}
+
 #define RK2928_PULL_OFFSET		0x118
 #define RK2928_PULL_PINS_PER_REG	16
 #define RK2928_PULL_BANK_STRIDE		8
@@ -1102,6 +1284,22 @@ static void rk2928_calc_pull_reg_and_bit(struct rockchip_pin_bank *bank,
 	*bit = pin_num % RK2928_PULL_PINS_PER_REG;
 };
 
+#define RK3128_PULL_OFFSET	0x118
+
+static void rk3128_calc_pull_reg_and_bit(struct rockchip_pin_bank *bank,
+					 int pin_num, struct regmap **regmap,
+					 int *reg, u8 *bit)
+{
+	struct rockchip_pinctrl *info = bank->drvdata;
+
+	*regmap = info->regmap_base;
+	*reg = RK3128_PULL_OFFSET;
+	*reg += bank->bank_num * RK2928_PULL_BANK_STRIDE;
+	*reg += ((pin_num / RK2928_PULL_PINS_PER_REG) * 4);
+
+	*bit = pin_num % RK2928_PULL_PINS_PER_REG;
+}
+
 #define RK3188_PULL_OFFSET		0x164
 #define RK3188_PULL_BITS_PER_PIN	2
 #define RK3188_PULL_PINS_PER_REG	8
@@ -1571,6 +1769,7 @@ static int rockchip_get_pull(struct rockchip_pin_bank *bank, int pin_num)
 
 	switch (ctrl->type) {
 	case RK2928:
+	case RK3128:
 		return !(data & BIT(bit))
 				? PIN_CONFIG_BIAS_PULL_PIN_DEFAULT
 				: PIN_CONFIG_BIAS_DISABLE;
@@ -1611,6 +1810,7 @@ static int rockchip_set_pull(struct rockchip_pin_bank *bank,
 
 	switch (ctrl->type) {
 	case RK2928:
+	case RK3128:
 		data = BIT(bit + 16);
 		if (pull == PIN_CONFIG_BIAS_DISABLE)
 			data |= BIT(bit);
@@ -1865,6 +2065,7 @@ static bool rockchip_pinconf_pull_valid(struct rockchip_pin_ctrl *ctrl,
 {
 	switch (ctrl->type) {
 	case RK2928:
+	case RK3128:
 		return (pull == PIN_CONFIG_BIAS_PULL_PIN_DEFAULT ||
 					pull == PIN_CONFIG_BIAS_DISABLE);
 	case RK3066B:
@@ -2853,6 +3054,16 @@ static struct rockchip_pin_ctrl *rockchip_pinctrl_get_soc_data(
 			bank_pins += 8;
 		}
 
+		/* calculate the per-bank recalced_mask */
+		for (j = 0; j < ctrl->niomux_recalced; j++) {
+			int pin = 0;
+
+			if (ctrl->iomux_recalced[j].num == bank->bank_num) {
+				pin = ctrl->iomux_recalced[j].pin;
+				bank->recalced_mask |= BIT(pin);
+			}
+		}
+
 		/* calculate the per-bank route_mask */
 		for (j = 0; j < ctrl->niomux_routes; j++) {
 			int pin = 0;
@@ -3015,8 +3226,11 @@ static struct rockchip_pin_ctrl rv1108_pin_ctrl = {
 	.type			= RV1108,
 	.grf_mux_offset		= 0x10,
 	.pmu_mux_offset		= 0x0,
+	.iomux_recalced		= rv1108_mux_recalced_data,
+	.niomux_recalced	= ARRAY_SIZE(rv1108_mux_recalced_data),
 	.pull_calc_reg		= rv1108_calc_pull_reg_and_bit,
 	.drv_calc_reg		= rv1108_calc_drv_reg_and_bit,
+	.schmitt_calc_reg	= rv1108_calc_schmitt_reg_and_bit,
 };
 
 static struct rockchip_pin_bank rk2928_pin_banks[] = {
@@ -3083,6 +3297,26 @@ static struct rockchip_pin_ctrl rk3066b_pin_ctrl = {
 		.grf_mux_offset	= 0x60,
 };
 
+static struct rockchip_pin_bank rk3128_pin_banks[] = {
+	PIN_BANK(0, 32, "gpio0"),
+	PIN_BANK(1, 32, "gpio1"),
+	PIN_BANK(2, 32, "gpio2"),
+	PIN_BANK(3, 32, "gpio3"),
+};
+
+static struct rockchip_pin_ctrl rk3128_pin_ctrl = {
+		.pin_banks		= rk3128_pin_banks,
+		.nr_banks		= ARRAY_SIZE(rk3128_pin_banks),
+		.label			= "RK3128-GPIO",
+		.type			= RK3128,
+		.grf_mux_offset		= 0xa8,
+		.iomux_recalced		= rk3128_mux_recalced_data,
+		.niomux_recalced	= ARRAY_SIZE(rk3128_mux_recalced_data),
+		.iomux_routes		= rk3128_mux_route_data,
+		.niomux_routes		= ARRAY_SIZE(rk3128_mux_route_data),
+		.pull_calc_reg		= rk3128_calc_pull_reg_and_bit,
+};
+
 static struct rockchip_pin_bank rk3188_pin_banks[] = {
 	PIN_BANK_IOMUX_FLAGS(0, 32, "gpio0", IOMUX_GPIO_ONLY, 0, 0, 0),
 	PIN_BANK(1, 32, "gpio1"),
@@ -3165,12 +3399,12 @@ static struct rockchip_pin_bank rk3328_pin_banks[] = {
 	PIN_BANK_IOMUX_FLAGS(0, 32, "gpio0", 0, 0, 0, 0),
 	PIN_BANK_IOMUX_FLAGS(1, 32, "gpio1", 0, 0, 0, 0),
 	PIN_BANK_IOMUX_FLAGS(2, 32, "gpio2", 0,
-			     IOMUX_WIDTH_3BIT | IOMUX_RECALCED,
-			     IOMUX_WIDTH_3BIT | IOMUX_RECALCED,
+			     IOMUX_WIDTH_3BIT,
+			     IOMUX_WIDTH_3BIT,
 			     0),
 	PIN_BANK_IOMUX_FLAGS(3, 32, "gpio3",
 			     IOMUX_WIDTH_3BIT,
-			     IOMUX_WIDTH_3BIT | IOMUX_RECALCED,
+			     IOMUX_WIDTH_3BIT,
 			     0,
 			     0),
 };
@@ -3181,11 +3415,12 @@ static struct rockchip_pin_ctrl rk3328_pin_ctrl = {
 		.label			= "RK3328-GPIO",
 		.type			= RK3288,
 		.grf_mux_offset		= 0x0,
+		.iomux_recalced		= rk3328_mux_recalced_data,
+		.niomux_recalced	= ARRAY_SIZE(rk3328_mux_recalced_data),
 		.iomux_routes		= rk3328_mux_route_data,
 		.niomux_routes		= ARRAY_SIZE(rk3328_mux_route_data),
 		.pull_calc_reg		= rk3228_calc_pull_reg_and_bit,
 		.drv_calc_reg		= rk3228_calc_drv_reg_and_bit,
-		.iomux_recalc		= rk3328_recalc_mux,
 		.schmitt_calc_reg	= rk3328_calc_schmitt_reg_and_bit,
 };
 
@@ -3290,6 +3525,8 @@ static const struct of_device_id rockchip_pinctrl_dt_match[] = {
 		.data = &rk3066a_pin_ctrl },
 	{ .compatible = "rockchip,rk3066b-pinctrl",
 		.data = &rk3066b_pin_ctrl },
+	{ .compatible = "rockchip,rk3128-pinctrl",
+		.data = (void *)&rk3128_pin_ctrl },
 	{ .compatible = "rockchip,rk3188-pinctrl",
 		.data = &rk3188_pin_ctrl },
 	{ .compatible = "rockchip,rk3228-pinctrl",
diff --git a/drivers/pinctrl/pinctrl-rza1.c b/drivers/pinctrl/pinctrl-rza1.c
index dc164da10446..04d058706b80 100644
--- a/drivers/pinctrl/pinctrl-rza1.c
+++ b/drivers/pinctrl/pinctrl-rza1.c
@@ -723,7 +723,7 @@ static void rza1_gpio_set(struct gpio_chip *chip, unsigned int gpio,
 	rza1_pin_set(port, gpio, value);
 }
 
-static struct gpio_chip rza1_gpiochip_template = {
+static const struct gpio_chip rza1_gpiochip_template = {
 	.request		= rza1_gpio_request,
 	.free			= rza1_gpio_free,
 	.get_direction		= rza1_gpio_get_direction,
@@ -1026,7 +1026,7 @@ static int rza1_set_mux(struct pinctrl_dev *pctldev, unsigned int selector,
 	return 0;
 }
 
-static struct pinmux_ops rza1_pinmux_ops = {
+static const struct pinmux_ops rza1_pinmux_ops = {
 	.get_functions_count	= pinmux_generic_get_function_count,
 	.get_function_name	= pinmux_generic_get_function_name,
 	.get_function_groups	= pinmux_generic_get_function_groups,
@@ -1088,7 +1088,7 @@ static int rza1_parse_gpiochip(struct rza1_pinctrl *rza1_pctl,
 	 */
 	pinctrl_base = of_args.args[1];
 	gpioport = RZA1_PIN_ID_TO_PORT(pinctrl_base);
-	if (gpioport > RZA1_NPORTS) {
+	if (gpioport >= RZA1_NPORTS) {
 		dev_err(rza1_pctl->dev,
 			"Invalid values in property %s\n", list_name);
 		return -EINVAL;
@@ -1096,8 +1096,8 @@ static int rza1_parse_gpiochip(struct rza1_pinctrl *rza1_pctl,
 
 	*chip		= rza1_gpiochip_template;
 	chip->base	= -1;
-	chip->label	= devm_kasprintf(rza1_pctl->dev, GFP_KERNEL, "%s-%u",
-					 np->name, gpioport);
+	chip->label	= devm_kasprintf(rza1_pctl->dev, GFP_KERNEL, "%s",
+					 np->name);
 	chip->ngpio	= of_args.args[2];
 	chip->of_node	= np;
 	chip->parent	= rza1_pctl->dev;
diff --git a/drivers/pinctrl/pinctrl-st.c b/drivers/pinctrl/pinctrl-st.c
index 3ae8066bc127..a5205b94b2e6 100644
--- a/drivers/pinctrl/pinctrl-st.c
+++ b/drivers/pinctrl/pinctrl-st.c
@@ -861,7 +861,7 @@ static void st_pctl_dt_free_map(struct pinctrl_dev *pctldev,
 {
 }
 
-static struct pinctrl_ops st_pctlops = {
+static const struct pinctrl_ops st_pctlops = {
 	.get_groups_count	= st_pctl_get_groups_count,
 	.get_group_pins		= st_pctl_get_group_pins,
 	.get_group_name		= st_pctl_get_group_name,
@@ -928,7 +928,7 @@ static int st_pmx_set_gpio_direction(struct pinctrl_dev *pctldev,
 	return 0;
 }
 
-static struct pinmux_ops st_pmxops = {
+static const struct pinmux_ops st_pmxops = {
 	.get_functions_count	= st_pmx_get_funcs_count,
 	.get_function_name	= st_pmx_get_fname,
 	.get_function_groups	= st_pmx_get_groups,
@@ -1025,7 +1025,7 @@ static void st_pinconf_dbg_show(struct pinctrl_dev *pctldev,
 		ST_PINCONF_UNPACK_RT_DELAY(config));
 }
 
-static struct pinconf_ops st_confops = {
+static const struct pinconf_ops st_confops = {
 	.pin_config_get		= st_pinconf_get,
 	.pin_config_set		= st_pinconf_set,
 	.pin_config_dbg_show	= st_pinconf_dbg_show,
@@ -1442,7 +1442,7 @@ static void st_gpio_irqmux_handler(struct irq_desc *desc)
 	chained_irq_exit(chip, desc);
 }
 
-static struct gpio_chip st_gpio_template = {
+static const struct gpio_chip st_gpio_template = {
 	.request		= gpiochip_generic_request,
 	.free			= gpiochip_generic_free,
 	.get			= st_gpio_get,
@@ -1521,7 +1521,7 @@ static int st_gpiolib_register_bank(struct st_pinctrl *info,
 	 *	[irqN]----> [gpio-bank (n)]
 	 */
 
-	if (of_irq_to_resource(np, 0, &irq_res)) {
+	if (of_irq_to_resource(np, 0, &irq_res) > 0) {
 		gpio_irq = irq_res.start;
 		gpiochip_set_chained_irqchip(&bank->gpio_chip, &st_gpio_irqchip,
 					     gpio_irq, st_gpio_irq_handler);
@@ -1537,7 +1537,7 @@ static int st_gpiolib_register_bank(struct st_pinctrl *info,
 			return err;
 		}
 	} else {
-		dev_info(dev, "No IRQ support for %s bank\n", np->full_name);
+		dev_info(dev, "No IRQ support for %pOF bank\n", np);
 	}
 
 	return 0;
diff --git a/drivers/pinctrl/pinctrl-tb10x.c b/drivers/pinctrl/pinctrl-tb10x.c
index edfba506e958..2e90a6d8fb3b 100644
--- a/drivers/pinctrl/pinctrl-tb10x.c
+++ b/drivers/pinctrl/pinctrl-tb10x.c
@@ -557,8 +557,8 @@ static int tb10x_dt_node_to_map(struct pinctrl_dev *pctl,
 	int ret = 0;
 
 	if (of_property_read_string(np_config, "abilis,function", &string)) {
-		pr_err("%s: No abilis,function property in device tree.\n",
-			np_config->full_name);
+		pr_err("%pOF: No abilis,function property in device tree.\n",
+			np_config);
 		return -EINVAL;
 	}
 
@@ -577,7 +577,7 @@ out:
 	return ret;
 }
 
-static struct pinctrl_ops tb10x_pinctrl_ops = {
+static const struct pinctrl_ops tb10x_pinctrl_ops = {
 	.get_groups_count = tb10x_get_groups_count,
 	.get_group_name   = tb10x_get_group_name,
 	.get_group_pins   = tb10x_get_group_pins,
@@ -738,7 +738,7 @@ static int tb10x_pctl_set_mux(struct pinctrl_dev *pctl,
 	return 0;
 }
 
-static struct pinmux_ops tb10x_pinmux_ops = {
+static const struct pinmux_ops tb10x_pinmux_ops = {
 	.get_functions_count = tb10x_get_functions_count,
 	.get_function_name = tb10x_get_function_name,
 	.get_function_groups = tb10x_get_function_groups,
diff --git a/drivers/pinctrl/pinctrl-tz1090-pdc.c b/drivers/pinctrl/pinctrl-tz1090-pdc.c
index e70e36283b3b..5cfa93cecf73 100644
--- a/drivers/pinctrl/pinctrl-tz1090-pdc.c
+++ b/drivers/pinctrl/pinctrl-tz1090-pdc.c
@@ -486,7 +486,7 @@ static int tz1090_pdc_pinctrl_dt_node_to_map(struct pinctrl_dev *pctldev,
 	return 0;
 }
 
-static struct pinctrl_ops tz1090_pdc_pinctrl_ops = {
+static const struct pinctrl_ops tz1090_pdc_pinctrl_ops = {
 	.get_groups_count	= tz1090_pdc_pinctrl_get_groups_count,
 	.get_group_name		= tz1090_pdc_pinctrl_get_group_name,
 	.get_group_pins		= tz1090_pdc_pinctrl_get_group_pins,
@@ -631,7 +631,7 @@ static void tz1090_pdc_pinctrl_gpio_disable_free(
 	}
 }
 
-static struct pinmux_ops tz1090_pdc_pinmux_ops = {
+static const struct pinmux_ops tz1090_pdc_pinmux_ops = {
 	.get_functions_count	= tz1090_pdc_pinctrl_get_funcs_count,
 	.get_function_name	= tz1090_pdc_pinctrl_get_func_name,
 	.get_function_groups	= tz1090_pdc_pinctrl_get_func_groups,
@@ -905,7 +905,7 @@ next_config:
 	return 0;
 }
 
-static struct pinconf_ops tz1090_pdc_pinconf_ops = {
+static const struct pinconf_ops tz1090_pdc_pinconf_ops = {
 	.is_generic			= true,
 	.pin_config_get			= tz1090_pdc_pinconf_get,
 	.pin_config_set			= tz1090_pdc_pinconf_set,
diff --git a/drivers/pinctrl/pinctrl-tz1090.c b/drivers/pinctrl/pinctrl-tz1090.c
index 04cbe530bf29..74d1ffcc2199 100644
--- a/drivers/pinctrl/pinctrl-tz1090.c
+++ b/drivers/pinctrl/pinctrl-tz1090.c
@@ -1201,7 +1201,7 @@ static int tz1090_pinctrl_dt_node_to_map(struct pinctrl_dev *pctldev,
 	return 0;
 }
 
-static struct pinctrl_ops tz1090_pinctrl_ops = {
+static const struct pinctrl_ops tz1090_pinctrl_ops = {
 	.get_groups_count	= tz1090_pinctrl_get_groups_count,
 	.get_group_name		= tz1090_pinctrl_get_group_name,
 	.get_group_pins		= tz1090_pinctrl_get_group_pins,
@@ -1513,7 +1513,7 @@ static void tz1090_pinctrl_gpio_disable_free(struct pinctrl_dev *pctldev,
 	tz1090_pinctrl_gpio_select(pmx, pin, false);
 }
 
-static struct pinmux_ops tz1090_pinmux_ops = {
+static const struct pinmux_ops tz1090_pinmux_ops = {
 	.get_functions_count	= tz1090_pinctrl_get_funcs_count,
 	.get_function_name	= tz1090_pinctrl_get_func_name,
 	.get_function_groups	= tz1090_pinctrl_get_func_groups,
@@ -1920,7 +1920,7 @@ next_config:
 	return 0;
 }
 
-static struct pinconf_ops tz1090_pinconf_ops = {
+static const struct pinconf_ops tz1090_pinconf_ops = {
 	.is_generic			= true,
 	.pin_config_get			= tz1090_pinconf_get,
 	.pin_config_set			= tz1090_pinconf_set,
diff --git a/drivers/pinctrl/pinctrl-zynq.c b/drivers/pinctrl/pinctrl-zynq.c
index b51a46dfdcc3..a0daf27042bd 100644
--- a/drivers/pinctrl/pinctrl-zynq.c
+++ b/drivers/pinctrl/pinctrl-zynq.c
@@ -45,7 +45,7 @@
  * @syscon:		Syscon regmap
  * @pctrl_offset:	Offset for pinctrl into the @syscon space
  * @groups:		Pingroups
- * @ngroupos:		Number of @groups
+ * @ngroups:		Number of @groups
  * @funcs:		Pinmux functions
  * @nfuncs:		Number of @funcs
  */
@@ -62,7 +62,7 @@ struct zynq_pinctrl {
 struct zynq_pctrl_group {
 	const char *name;
 	const unsigned int *pins;
-	const unsigned npins;
+	const unsigned int npins;
 };
 
 /**
@@ -841,7 +841,7 @@ static int zynq_pctrl_get_groups_count(struct pinctrl_dev *pctldev)
 }
 
 static const char *zynq_pctrl_get_group_name(struct pinctrl_dev *pctldev,
-					     unsigned selector)
+					     unsigned int selector)
 {
 	struct zynq_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
 
@@ -849,9 +849,9 @@ static const char *zynq_pctrl_get_group_name(struct pinctrl_dev *pctldev,
 }
 
 static int zynq_pctrl_get_group_pins(struct pinctrl_dev *pctldev,
-				     unsigned selector,
-				     const unsigned **pins,
-				     unsigned *num_pins)
+				     unsigned int selector,
+				     const unsigned int **pins,
+				     unsigned int *num_pins)
 {
 	struct zynq_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
 
@@ -878,7 +878,7 @@ static int zynq_pmux_get_functions_count(struct pinctrl_dev *pctldev)
 }
 
 static const char *zynq_pmux_get_function_name(struct pinctrl_dev *pctldev,
-					       unsigned selector)
+					       unsigned int selector)
 {
 	struct zynq_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
 
@@ -886,7 +886,7 @@ static const char *zynq_pmux_get_function_name(struct pinctrl_dev *pctldev,
 }
 
 static int zynq_pmux_get_function_groups(struct pinctrl_dev *pctldev,
-					 unsigned selector,
+					 unsigned int selector,
 					 const char * const **groups,
 					 unsigned * const num_groups)
 {
@@ -898,8 +898,8 @@ static int zynq_pmux_get_function_groups(struct pinctrl_dev *pctldev,
 }
 
 static int zynq_pinmux_set_mux(struct pinctrl_dev *pctldev,
-			       unsigned function,
-			       unsigned group)
+			       unsigned int function,
+			       unsigned int  group)
 {
 	int i, ret;
 	struct zynq_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
@@ -986,8 +986,8 @@ static const struct pinconf_generic_params zynq_dt_params[] = {
 };
 
 #ifdef CONFIG_DEBUG_FS
-static const struct pin_config_item zynq_conf_items[ARRAY_SIZE(zynq_dt_params)] = {
-	PCONFDUMP(PIN_CONFIG_IOSTANDARD, "IO-standard", NULL, true),
+static const struct pin_config_item zynq_conf_items[ARRAY_SIZE(zynq_dt_params)]
+	= { PCONFDUMP(PIN_CONFIG_IOSTANDARD, "IO-standard", NULL, true),
 };
 #endif
 
@@ -997,7 +997,7 @@ static unsigned int zynq_pinconf_iostd_get(u32 reg)
 }
 
 static int zynq_pinconf_cfg_get(struct pinctrl_dev *pctldev,
-				unsigned pin,
+				unsigned int pin,
 				unsigned long *config)
 {
 	u32 reg;
@@ -1054,9 +1054,9 @@ static int zynq_pinconf_cfg_get(struct pinctrl_dev *pctldev,
 }
 
 static int zynq_pinconf_cfg_set(struct pinctrl_dev *pctldev,
-				unsigned pin,
+				unsigned int pin,
 				unsigned long *configs,
-				unsigned num_configs)
+				unsigned int num_configs)
 {
 	int i, ret;
 	u32 reg;
@@ -1130,9 +1130,9 @@ static int zynq_pinconf_cfg_set(struct pinctrl_dev *pctldev,
 }
 
 static int zynq_pinconf_group_set(struct pinctrl_dev *pctldev,
-				  unsigned selector,
+				  unsigned int selector,
 				  unsigned long *configs,
-				  unsigned num_configs)
+				  unsigned int  num_configs)
 {
 	int i, ret;
 	struct zynq_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
diff --git a/drivers/pinctrl/pinmux.c b/drivers/pinctrl/pinmux.c
index 16b3ae5e4f44..55502fc4479c 100644
--- a/drivers/pinctrl/pinmux.c
+++ b/drivers/pinctrl/pinmux.c
@@ -61,7 +61,7 @@ int pinmux_check_ops(struct pinctrl_dev *pctldev)
 	return 0;
 }
 
-int pinmux_validate_map(struct pinctrl_map const *map, int i)
+int pinmux_validate_map(const struct pinctrl_map *map, int i)
 {
 	if (!map->data.mux.function) {
 		pr_err("failed to register map %s (%d): no function given\n",
@@ -312,7 +312,7 @@ static int pinmux_func_name_to_selector(struct pinctrl_dev *pctldev,
 	return -EINVAL;
 }
 
-int pinmux_map_to_setting(struct pinctrl_map const *map,
+int pinmux_map_to_setting(const struct pinctrl_map *map,
 			  struct pinctrl_setting *setting)
 {
 	struct pinctrl_dev *pctldev = setting->pctldev;
@@ -372,12 +372,12 @@ int pinmux_map_to_setting(struct pinctrl_map const *map,
 	return 0;
 }
 
-void pinmux_free_setting(struct pinctrl_setting const *setting)
+void pinmux_free_setting(const struct pinctrl_setting *setting)
 {
 	/* This function is currently unused */
 }
 
-int pinmux_enable_setting(struct pinctrl_setting const *setting)
+int pinmux_enable_setting(const struct pinctrl_setting *setting)
 {
 	struct pinctrl_dev *pctldev = setting->pctldev;
 	const struct pinctrl_ops *pctlops = pctldev->desc->pctlops;
@@ -458,7 +458,7 @@ err_pin_request:
 	return ret;
 }
 
-void pinmux_disable_setting(struct pinctrl_setting const *setting)
+void pinmux_disable_setting(const struct pinctrl_setting *setting)
 {
 	struct pinctrl_dev *pctldev = setting->pctldev;
 	const struct pinctrl_ops *pctlops = pctldev->desc->pctlops;
@@ -627,7 +627,7 @@ static int pinmux_pins_show(struct seq_file *s, void *what)
 	return 0;
 }
 
-void pinmux_show_map(struct seq_file *s, struct pinctrl_map const *map)
+void pinmux_show_map(struct seq_file *s, const struct pinctrl_map *map)
 {
 	seq_printf(s, "group %s\nfunction %s\n",
 		map->data.mux.group ? map->data.mux.group : "(default)",
@@ -635,7 +635,7 @@ void pinmux_show_map(struct seq_file *s, struct pinctrl_map const *map)
 }
 
 void pinmux_show_setting(struct seq_file *s,
-			 struct pinctrl_setting const *setting)
+			 const struct pinctrl_setting *setting)
 {
 	struct pinctrl_dev *pctldev = setting->pctldev;
 	const struct pinmux_ops *pmxops = pctldev->desc->pmxops;
@@ -833,7 +833,7 @@ EXPORT_SYMBOL_GPL(pinmux_generic_remove_function);
 void pinmux_generic_free_functions(struct pinctrl_dev *pctldev)
 {
 	struct radix_tree_iter iter;
-	void **slot;
+	void __rcu **slot;
 
 	radix_tree_for_each_slot(slot, &pctldev->pin_function_tree, &iter, 0)
 		radix_tree_delete(&pctldev->pin_function_tree, iter.index);
diff --git a/drivers/pinctrl/pinmux.h b/drivers/pinctrl/pinmux.h
index 248d8ea30e26..a331fcdbedd9 100644
--- a/drivers/pinctrl/pinmux.h
+++ b/drivers/pinctrl/pinmux.h
@@ -14,7 +14,7 @@
 
 int pinmux_check_ops(struct pinctrl_dev *pctldev);
 
-int pinmux_validate_map(struct pinctrl_map const *map, int i);
+int pinmux_validate_map(const struct pinctrl_map *map, int i);
 
 int pinmux_request_gpio(struct pinctrl_dev *pctldev,
 			struct pinctrl_gpio_range *range,
@@ -25,11 +25,11 @@ int pinmux_gpio_direction(struct pinctrl_dev *pctldev,
 			  struct pinctrl_gpio_range *range,
 			  unsigned pin, bool input);
 
-int pinmux_map_to_setting(struct pinctrl_map const *map,
+int pinmux_map_to_setting(const struct pinctrl_map *map,
 			  struct pinctrl_setting *setting);
-void pinmux_free_setting(struct pinctrl_setting const *setting);
-int pinmux_enable_setting(struct pinctrl_setting const *setting);
-void pinmux_disable_setting(struct pinctrl_setting const *setting);
+void pinmux_free_setting(const struct pinctrl_setting *setting);
+int pinmux_enable_setting(const struct pinctrl_setting *setting);
+void pinmux_disable_setting(const struct pinctrl_setting *setting);
 
 #else
 
@@ -38,7 +38,7 @@ static inline int pinmux_check_ops(struct pinctrl_dev *pctldev)
 	return 0;
 }
 
-static inline int pinmux_validate_map(struct pinctrl_map const *map, int i)
+static inline int pinmux_validate_map(const struct pinctrl_map *map, int i)
 {
 	return 0;
 }
@@ -63,23 +63,22 @@ static inline int pinmux_gpio_direction(struct pinctrl_dev *pctldev,
 	return 0;
 }
 
-static inline int pinmux_map_to_setting(struct pinctrl_map const *map,
+static inline int pinmux_map_to_setting(const struct pinctrl_map *map,
 			  struct pinctrl_setting *setting)
 {
 	return 0;
 }
 
-static inline void pinmux_free_setting(struct pinctrl_setting const *setting)
+static inline void pinmux_free_setting(const struct pinctrl_setting *setting)
 {
 }
 
-static inline int pinmux_enable_setting(struct pinctrl_setting const *setting)
+static inline int pinmux_enable_setting(const struct pinctrl_setting *setting)
 {
 	return 0;
 }
 
-static inline void pinmux_disable_setting(
-			struct pinctrl_setting const *setting)
+static inline void pinmux_disable_setting(const struct pinctrl_setting *setting)
 {
 }
 
@@ -87,21 +86,21 @@ static inline void pinmux_disable_setting(
 
 #if defined(CONFIG_PINMUX) && defined(CONFIG_DEBUG_FS)
 
-void pinmux_show_map(struct seq_file *s, struct pinctrl_map const *map);
+void pinmux_show_map(struct seq_file *s, const struct pinctrl_map *map);
 void pinmux_show_setting(struct seq_file *s,
-			 struct pinctrl_setting const *setting);
+			 const struct pinctrl_setting *setting);
 void pinmux_init_device_debugfs(struct dentry *devroot,
 				struct pinctrl_dev *pctldev);
 
 #else
 
 static inline void pinmux_show_map(struct seq_file *s,
-				   struct pinctrl_map const *map)
+				   const struct pinctrl_map *map)
 {
 }
 
 static inline void pinmux_show_setting(struct seq_file *s,
-				       struct pinctrl_setting const *setting)
+				       const struct pinctrl_setting *setting)
 {
 }
 
diff --git a/drivers/pinctrl/qcom/pinctrl-apq8064.c b/drivers/pinctrl/qcom/pinctrl-apq8064.c
index cd96699b1929..bcf9e615ff61 100644
--- a/drivers/pinctrl/qcom/pinctrl-apq8064.c
+++ b/drivers/pinctrl/qcom/pinctrl-apq8064.c
@@ -295,6 +295,12 @@ enum apq8064_functions {
 	APQ_MUX_cam_mclk,
 	APQ_MUX_codec_mic_i2s,
 	APQ_MUX_codec_spkr_i2s,
+	APQ_MUX_gp_clk_0a,
+	APQ_MUX_gp_clk_0b,
+	APQ_MUX_gp_clk_1a,
+	APQ_MUX_gp_clk_1b,
+	APQ_MUX_gp_clk_2a,
+	APQ_MUX_gp_clk_2b,
 	APQ_MUX_gpio,
 	APQ_MUX_gsbi1,
 	APQ_MUX_gsbi2,
@@ -354,6 +360,24 @@ static const char * const gpio_groups[] = {
 	"gpio78", "gpio79", "gpio80", "gpio81", "gpio82", "gpio83", "gpio84",
 	"gpio85", "gpio86", "gpio87", "gpio88", "gpio89"
 };
+static const char * const gp_clk_0a_groups[] = {
+	"gpio3"
+};
+static const char * const gp_clk_0b_groups[] = {
+	"gpio34"
+};
+static const char * const gp_clk_1a_groups[] = {
+	"gpio4"
+};
+static const char * const gp_clk_1b_groups[] = {
+	"gpio50"
+};
+static const char * const gp_clk_2a_groups[] = {
+	"gpio32"
+};
+static const char * const gp_clk_2b_groups[] = {
+	"gpio25"
+};
 static const char * const ps_hold_groups[] = {
 	"gpio78"
 };
@@ -452,6 +476,12 @@ static const struct msm_function apq8064_functions[] = {
 	FUNCTION(cam_mclk),
 	FUNCTION(codec_mic_i2s),
 	FUNCTION(codec_spkr_i2s),
+	FUNCTION(gp_clk_0a),
+	FUNCTION(gp_clk_0b),
+	FUNCTION(gp_clk_1a),
+	FUNCTION(gp_clk_1b),
+	FUNCTION(gp_clk_2a),
+	FUNCTION(gp_clk_2b),
 	FUNCTION(gpio),
 	FUNCTION(gsbi1),
 	FUNCTION(gsbi2),
@@ -490,8 +520,8 @@ static const struct msm_pingroup apq8064_groups[] = {
 	PINGROUP(0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
 	PINGROUP(1, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
 	PINGROUP(2, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(3, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(4, NA, NA, cam_mclk, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(3, NA, gp_clk_0a, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(4, NA, NA, cam_mclk, gp_clk_1a, NA, NA, NA, NA, NA, NA),
 	PINGROUP(5, NA, cam_mclk, NA, NA, NA, NA, NA, NA, NA, NA),
 	PINGROUP(6, gsbi3, NA, NA, NA, NA, NA, NA, NA, NA, NA),
 	PINGROUP(7, gsbi3, NA, NA, NA, NA, NA, NA, NA, NA, NA),
@@ -512,16 +542,16 @@ static const struct msm_pingroup apq8064_groups[] = {
 	PINGROUP(22, gsbi2, NA, NA, NA, NA, NA, NA, NA, NA, NA),
 	PINGROUP(23, gsbi2, NA, NA, NA, NA, NA, NA, NA, NA, NA),
 	PINGROUP(24, gsbi2, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(25, gsbi2, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(25, gsbi2, gp_clk_2b, NA, NA, NA, NA, NA, NA, NA, NA),
 	PINGROUP(26, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
 	PINGROUP(27, mi2s, NA, NA, NA, NA, NA, NA, NA, NA, NA),
 	PINGROUP(28, mi2s, NA, NA, NA, NA, NA, NA, NA, NA, NA),
 	PINGROUP(29, mi2s, NA, NA, NA, NA, NA, NA, NA, NA, NA),
 	PINGROUP(30, mi2s, NA, NA, NA, NA, NA, NA, NA, NA, NA),
 	PINGROUP(31, mi2s, NA, gsbi5_spi_cs2, gsbi6_spi_cs2, gsbi7_spi_cs2, NA, NA, NA, NA, NA),
-	PINGROUP(32, mi2s, NA, NA, NA, NA, gsbi5_spi_cs3, gsbi6_spi_cs3, gsbi7_spi_cs3, NA, NA),
+	PINGROUP(32, mi2s, gp_clk_2a, NA, NA, NA, gsbi5_spi_cs3, gsbi6_spi_cs3, gsbi7_spi_cs3, NA, NA),
 	PINGROUP(33, mi2s, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(34, codec_mic_i2s, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(34, codec_mic_i2s, gp_clk_0b, NA, NA, NA, NA, NA, NA, NA, NA),
 	PINGROUP(35, codec_mic_i2s, NA, NA, NA, NA, NA, NA, NA, NA, NA),
 	PINGROUP(36, codec_mic_i2s, NA, NA, NA, NA, NA, NA, NA, NA, NA),
 	PINGROUP(37, codec_mic_i2s, NA, NA, NA, NA, NA, NA, NA, NA, NA),
@@ -537,7 +567,7 @@ static const struct msm_pingroup apq8064_groups[] = {
 	PINGROUP(47, spkr_i2s, gsbi5_spi_cs1, gsbi6_spi_cs1, gsbi7_spi_cs1, NA, NA, NA, NA, NA, NA),
 	PINGROUP(48, spkr_i2s, NA, NA, NA, NA, NA, NA, NA, NA, NA),
 	PINGROUP(49, spkr_i2s, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(50, spkr_i2s, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(50, spkr_i2s, gp_clk_1b, NA, NA, NA, NA, NA, NA, NA, NA),
 	PINGROUP(51, NA, gsbi5, NA, NA, NA, NA, NA, NA, NA, NA),
 	PINGROUP(52, NA, gsbi5, NA, NA, NA, NA, NA, NA, NA, NA),
 	PINGROUP(53, NA, gsbi5, NA, NA, NA, NA, NA, NA, NA, NA),
diff --git a/drivers/pinctrl/qcom/pinctrl-ipq4019.c b/drivers/pinctrl/qcom/pinctrl-ipq4019.c
index 743d1f458205..1979b14b6fc3 100644
--- a/drivers/pinctrl/qcom/pinctrl-ipq4019.c
+++ b/drivers/pinctrl/qcom/pinctrl-ipq4019.c
@@ -277,12 +277,49 @@ DECLARE_QCA_GPIO_PINS(99);
 
 enum ipq4019_functions {
 	qca_mux_gpio,
-	qca_mux_blsp_uart1,
+	qca_mux_aud_pin,
+	qca_mux_audio_pwm,
 	qca_mux_blsp_i2c0,
 	qca_mux_blsp_i2c1,
-	qca_mux_blsp_uart0,
-	qca_mux_blsp_spi1,
 	qca_mux_blsp_spi0,
+	qca_mux_blsp_spi1,
+	qca_mux_blsp_uart0,
+	qca_mux_blsp_uart1,
+	qca_mux_chip_rst,
+	qca_mux_i2s_rx,
+	qca_mux_i2s_spdif_in,
+	qca_mux_i2s_spdif_out,
+	qca_mux_i2s_td,
+	qca_mux_i2s_tx,
+	qca_mux_jtag,
+	qca_mux_led0,
+	qca_mux_led1,
+	qca_mux_led2,
+	qca_mux_led3,
+	qca_mux_led4,
+	qca_mux_led5,
+	qca_mux_led6,
+	qca_mux_led7,
+	qca_mux_led8,
+	qca_mux_led9,
+	qca_mux_led10,
+	qca_mux_led11,
+	qca_mux_mdc,
+	qca_mux_mdio,
+	qca_mux_pcie,
+	qca_mux_pmu,
+	qca_mux_prng_rosc,
+	qca_mux_qpic,
+	qca_mux_rgmii,
+	qca_mux_rmii,
+	qca_mux_sdio,
+	qca_mux_smart0,
+	qca_mux_smart1,
+	qca_mux_smart2,
+	qca_mux_smart3,
+	qca_mux_tm,
+	qca_mux_wifi0,
+	qca_mux_wifi1,
 	qca_mux_NA,
 };
 
@@ -303,108 +340,331 @@ static const char * const gpio_groups[] = {
 	"gpio92", "gpio93", "gpio94", "gpio95", "gpio96", "gpio97", "gpio98",
 	"gpio99",
 };
-
-static const char * const blsp_uart1_groups[] = {
-	"gpio8", "gpio9", "gpio10", "gpio11",
+static const char * const aud_pin_groups[] = {
+	"gpio48", "gpio49", "gpio50", "gpio51",
+};
+static const char * const audio_pwm_groups[] = {
+	"gpio30", "gpio31", "gpio32", "gpio33", "gpio64", "gpio65", "gpio66",
+	"gpio67",
 };
 static const char * const blsp_i2c0_groups[] = {
 	"gpio10", "gpio11", "gpio20", "gpio21", "gpio58", "gpio59",
 };
-static const char * const blsp_spi0_groups[] = {
-	"gpio12", "gpio13", "gpio14", "gpio15", "gpio45",
-	"gpio54", "gpio55", "gpio56", "gpio57",
-};
 static const char * const blsp_i2c1_groups[] = {
 	"gpio12", "gpio13", "gpio34", "gpio35",
 };
-static const char * const blsp_uart0_groups[] = {
-	"gpio16", "gpio17", "gpio60", "gpio61",
+static const char * const blsp_spi0_groups[] = {
+	"gpio12", "gpio13", "gpio14", "gpio15", "gpio45", "gpio54", "gpio55",
+	"gpio56", "gpio57",
 };
 static const char * const blsp_spi1_groups[] = {
 	"gpio44", "gpio45", "gpio46", "gpio47",
 };
+static const char * const blsp_uart0_groups[] = {
+	"gpio16", "gpio17", "gpio60", "gpio61",
+};
+static const char * const blsp_uart1_groups[] = {
+	"gpio8", "gpio9", "gpio10", "gpio11",
+};
+static const char * const chip_rst_groups[] = {
+	"gpio62",
+};
+static const char * const i2s_rx_groups[] = {
+	"gpio0", "gpio1", "gpio2", "gpio20", "gpio21", "gpio22", "gpio23",
+	"gpio58", "gpio60", "gpio61", "gpio63",
+};
+static const char * const i2s_spdif_in_groups[] = {
+	"gpio34", "gpio59", "gpio63",
+};
+static const char * const i2s_spdif_out_groups[] = {
+	"gpio35", "gpio62", "gpio63",
+};
+static const char * const i2s_td_groups[] = {
+	"gpio27", "gpio28", "gpio29", "gpio54", "gpio55", "gpio56", "gpio63",
+};
+static const char * const i2s_tx_groups[] = {
+	"gpio24", "gpio25", "gpio26", "gpio52", "gpio53", "gpio57", "gpio60",
+	"gpio61",
+};
+static const char * const jtag_groups[] = {
+	"gpio0", "gpio1", "gpio2", "gpio3", "gpio4", "gpio5",
+};
+static const char * const led0_groups[] = {
+	"gpio16", "gpio36", "gpio60",
+};
+static const char * const led1_groups[] = {
+	"gpio17", "gpio37", "gpio61",
+};
+static const char * const led2_groups[] = {
+	"gpio36", "gpio38", "gpio58",
+};
+static const char * const led3_groups[] = {
+	"gpio39",
+};
+static const char * const led4_groups[] = {
+	"gpio40",
+};
+static const char * const led5_groups[] = {
+	"gpio44",
+};
+static const char * const led6_groups[] = {
+	"gpio45",
+};
+static const char * const led7_groups[] = {
+	"gpio46",
+};
+static const char * const led8_groups[] = {
+	"gpio47",
+};
+static const char * const led9_groups[] = {
+	"gpio48",
+};
+static const char * const led10_groups[] = {
+	"gpio49",
+};
+static const char * const led11_groups[] = {
+	"gpio50",
+};
+static const char * const mdc_groups[] = {
+	"gpio7", "gpio52",
+};
+static const char * const mdio_groups[] = {
+	"gpio6", "gpio53",
+};
+static const char * const pcie_groups[] = {
+	"gpio39", "gpio52",
+};
+static const char * const pmu_groups[] = {
+	"gpio54", "gpio55",
+};
+static const char * const prng_rosc_groups[] = {
+	"gpio53",
+};
+static const char * const qpic_groups[] = {
+	"gpio52", "gpio53", "gpio54", "gpio55", "gpio56", "gpio57", "gpio58",
+	"gpio59", "gpio60", "gpio61", "gpio62", "gpio63", "gpio64", "gpio65",
+	"gpio66", "gpio67", "gpio68", "gpio69",
+};
+static const char * const rgmii_groups[] = {
+	"gpio22", "gpio23", "gpio24", "gpio25", "gpio26", "gpio27", "gpio28",
+	"gpio29", "gpio30", "gpio31", "gpio32", "gpio33",
+};
+static const char * const rmii_groups[] = {
+	"gpio36", "gpio37", "gpio38", "gpio39", "gpio40", "gpio41", "gpio42",
+	"gpio43", "gpio44", "gpio45", "gpio46", "gpio47", "gpio48", "gpio49",
+	"gpio50", "gpio51",
+};
+static const char * const sdio_groups[] = {
+	"gpio23", "gpio24", "gpio25", "gpio26", "gpio27", "gpio28", "gpio29",
+	"gpio30", "gpio31", "gpio32",
+};
+static const char * const smart0_groups[] = {
+	"gpio0", "gpio1", "gpio2", "gpio5", "gpio44", "gpio45", "gpio46",
+	"gpio47",
+};
+static const char * const smart1_groups[] = {
+	"gpio8", "gpio9", "gpio16", "gpio17", "gpio58", "gpio59", "gpio60",
+	"gpio61",
+};
+static const char * const smart2_groups[] = {
+	"gpio40", "gpio41", "gpio48", "gpio49",
+};
+static const char * const smart3_groups[] = {
+	"gpio58", "gpio59", "gpio60", "gpio61",
+};
+static const char * const tm_groups[] = {
+	"gpio52", "gpio53", "gpio54", "gpio55", "gpio56", "gpio57", "gpio58",
+	"gpio59", "gpio60", "gpio61", "gpio62", "gpio63",
+};
+static const char * const wifi0_groups[] = {
+	"gpio37", "gpio40", "gpio41", "gpio42", "gpio50", "gpio51", "gpio52",
+	"gpio53", "gpio56", "gpio57", "gpio58", "gpio98",
+};
+static const char * const wifi1_groups[] = {
+	"gpio37", "gpio40", "gpio41", "gpio43", "gpio50", "gpio51", "gpio52",
+	"gpio53", "gpio56", "gpio57", "gpio58", "gpio98",
+};
 
 static const struct msm_function ipq4019_functions[] = {
-	FUNCTION(gpio),
-	FUNCTION(blsp_uart1),
+	FUNCTION(aud_pin),
+	FUNCTION(audio_pwm),
 	FUNCTION(blsp_i2c0),
 	FUNCTION(blsp_i2c1),
-	FUNCTION(blsp_uart0),
-	FUNCTION(blsp_spi1),
 	FUNCTION(blsp_spi0),
+	FUNCTION(blsp_spi1),
+	FUNCTION(blsp_uart0),
+	FUNCTION(blsp_uart1),
+	FUNCTION(chip_rst),
+	FUNCTION(gpio),
+	FUNCTION(i2s_rx),
+	FUNCTION(i2s_spdif_in),
+	FUNCTION(i2s_spdif_out),
+	FUNCTION(i2s_td),
+	FUNCTION(i2s_tx),
+	FUNCTION(jtag),
+	FUNCTION(led0),
+	FUNCTION(led1),
+	FUNCTION(led2),
+	FUNCTION(led3),
+	FUNCTION(led4),
+	FUNCTION(led5),
+	FUNCTION(led6),
+	FUNCTION(led7),
+	FUNCTION(led8),
+	FUNCTION(led9),
+	FUNCTION(led10),
+	FUNCTION(led11),
+	FUNCTION(mdc),
+	FUNCTION(mdio),
+	FUNCTION(pcie),
+	FUNCTION(pmu),
+	FUNCTION(prng_rosc),
+	FUNCTION(qpic),
+	FUNCTION(rgmii),
+	FUNCTION(rmii),
+	FUNCTION(sdio),
+	FUNCTION(smart0),
+	FUNCTION(smart1),
+	FUNCTION(smart2),
+	FUNCTION(smart3),
+	FUNCTION(tm),
+	FUNCTION(wifi0),
+	FUNCTION(wifi1),
 };
 
 static const struct msm_pingroup ipq4019_groups[] = {
-	PINGROUP(0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(1, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(2, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(3, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(4, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(5, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(6, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(7, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(8, blsp_uart1, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(9, blsp_uart1, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(10, blsp_uart1, NA, NA, blsp_i2c0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(11, blsp_uart1, NA, NA, blsp_i2c0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(12, blsp_spi0, blsp_i2c1, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(13, blsp_spi0, blsp_i2c1, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(14, blsp_spi0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(15, blsp_spi0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(16, blsp_uart0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(17, blsp_uart0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(0, jtag, smart0, i2s_rx, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(1, jtag, smart0, i2s_rx, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(2, jtag, smart0, i2s_rx, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(3, jtag, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(4, jtag, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(5, jtag, smart0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(6, mdio, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(7, mdc, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(8, blsp_uart1, NA, NA, smart1, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(9, blsp_uart1, NA, NA, smart1, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(10, blsp_uart1, NA, NA, blsp_i2c0, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(11, blsp_uart1, NA, NA, blsp_i2c0, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(12, blsp_spi0, blsp_i2c1, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(13, blsp_spi0, blsp_i2c1, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(14, blsp_spi0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(15, blsp_spi0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(16, blsp_uart0, led0, smart1, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(17, blsp_uart0, led1, smart1, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
 	PINGROUP(18, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
 	PINGROUP(19, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(20, blsp_i2c0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(21, blsp_i2c0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(22, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(23, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(24, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(25, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(26, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(27, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(28, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(29, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(30, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(31, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(32, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(33, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(34, blsp_i2c1, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(35, blsp_i2c1, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(36, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(37, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(38, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(39, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(40, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(41, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(42, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(43, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(44, NA, blsp_spi1, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(45, NA, blsp_spi1, blsp_spi0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(46, NA, blsp_spi1, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(47, NA, blsp_spi1, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(48, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(49, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(50, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(51, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(52, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(53, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(54, NA, blsp_spi0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(55, NA, blsp_spi0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(56, NA, blsp_spi0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(57, NA, blsp_spi0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(58, NA, NA, blsp_i2c0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(59, NA, blsp_i2c0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(60, NA, blsp_uart0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(61, NA, blsp_uart0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(62, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(63, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(64, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(65, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(66, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(67, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(68, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(69, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(20, blsp_i2c0, i2s_rx, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(21, blsp_i2c0, i2s_rx, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(22, rgmii, i2s_rx, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(23, sdio, rgmii, i2s_rx, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(24, sdio, rgmii, i2s_tx, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(25, sdio, rgmii, i2s_tx, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(26, sdio, rgmii, i2s_tx, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(27, sdio, rgmii, i2s_td, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(28, sdio, rgmii, i2s_td, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(29, sdio, rgmii, i2s_td, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(30, sdio, rgmii, audio_pwm, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(31, sdio, rgmii, audio_pwm, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(32, sdio, rgmii, audio_pwm, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(33, rgmii, audio_pwm, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(34, blsp_i2c1, i2s_spdif_in, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA, NA),
+	PINGROUP(35, blsp_i2c1, i2s_spdif_out, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA, NA),
+	PINGROUP(36, rmii, led2, led0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(37, rmii, wifi0, wifi1, led1, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(38, rmii, led2, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(39, rmii, pcie, led3, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(40, rmii, wifi0, wifi1, smart2, led4, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(41, rmii, wifi0, wifi1, smart2, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(42, rmii, wifi0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(43, rmii, wifi1, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(44, rmii, blsp_spi1, smart0, led5, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(45, rmii, blsp_spi1, blsp_spi0, smart0, led6, NA, NA, NA, NA,
+		 NA, NA, NA, NA, NA),
+	PINGROUP(46, rmii, blsp_spi1, smart0, led7, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(47, rmii, blsp_spi1, smart0, led8, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(48, rmii, aud_pin, smart2, led9, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(49, rmii, aud_pin, smart2, led10, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(50, rmii, aud_pin, wifi0, wifi1, led11, NA, NA, NA, NA, NA,
+		 NA, NA, NA, NA),
+	PINGROUP(51, rmii, aud_pin, wifi0, wifi1, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(52, qpic, mdc, pcie, i2s_tx, NA, NA, NA, tm, wifi0, wifi1, NA,
+		 NA, NA, NA),
+	PINGROUP(53, qpic, mdio, i2s_tx, prng_rosc, NA, tm, wifi0, wifi1, NA,
+		 NA, NA, NA, NA, NA),
+	PINGROUP(54, qpic, blsp_spi0, i2s_td, NA, pmu, NA, NA, NA, tm, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(55, qpic, blsp_spi0, i2s_td, NA, pmu, NA, NA, NA, tm, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(56, qpic, blsp_spi0, i2s_td, NA, NA, tm, wifi0, wifi1, NA, NA,
+		 NA, NA, NA, NA),
+	PINGROUP(57, qpic, blsp_spi0, i2s_tx, NA, NA, tm, wifi0, wifi1, NA, NA,
+		 NA, NA, NA, NA),
+	PINGROUP(58, qpic, led2, blsp_i2c0, smart3, smart1, i2s_rx, NA, NA, tm,
+		 wifi0, wifi1, NA, NA, NA),
+	PINGROUP(59, qpic, blsp_i2c0, smart3, smart1, i2s_spdif_in, NA, NA, NA,
+		 NA, NA, tm, NA, NA, NA),
+	PINGROUP(60, qpic, blsp_uart0, smart1, smart3, led0, i2s_tx, i2s_rx,
+		 NA, NA, NA, NA, NA, tm, NA),
+	PINGROUP(61, qpic, blsp_uart0, smart1, smart3, led1, i2s_tx, i2s_rx,
+		 NA, NA, NA, NA, NA, tm, NA),
+	PINGROUP(62, qpic, chip_rst, NA, NA, i2s_spdif_out, NA, NA, NA, NA, NA,
+		 tm, NA, NA, NA),
+	PINGROUP(63, qpic, NA, NA, NA, i2s_td, i2s_rx, i2s_spdif_out,
+		 i2s_spdif_in, NA, NA, NA, NA, tm, NA),
+	PINGROUP(64, qpic, audio_pwm, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(65, qpic, audio_pwm, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(66, qpic, audio_pwm, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(67, qpic, audio_pwm, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(68, qpic, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(69, qpic, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
 	PINGROUP(70, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
 	PINGROUP(71, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
 	PINGROUP(72, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
@@ -433,7 +693,8 @@ static const struct msm_pingroup ipq4019_groups[] = {
 	PINGROUP(95, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
 	PINGROUP(96, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
 	PINGROUP(97, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
-	PINGROUP(98, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(98, wifi0, wifi1, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
 	PINGROUP(99, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
 };
 
@@ -445,6 +706,7 @@ static const struct msm_pinctrl_soc_data ipq4019_pinctrl = {
 	.groups = ipq4019_groups,
 	.ngroups = ARRAY_SIZE(ipq4019_groups),
 	.ngpios = 100,
+	.pull_no_keeper = true,
 };
 
 static int ipq4019_pinctrl_probe(struct platform_device *pdev)
diff --git a/drivers/pinctrl/qcom/pinctrl-msm.c b/drivers/pinctrl/qcom/pinctrl-msm.c
index 273badd92561..ff491da64dab 100644
--- a/drivers/pinctrl/qcom/pinctrl-msm.c
+++ b/drivers/pinctrl/qcom/pinctrl-msm.c
@@ -202,10 +202,11 @@ static int msm_config_reg(struct msm_pinctrl *pctrl,
 	return 0;
 }
 
-#define MSM_NO_PULL	0
-#define MSM_PULL_DOWN	1
-#define MSM_KEEPER	2
-#define MSM_PULL_UP	3
+#define MSM_NO_PULL		0
+#define MSM_PULL_DOWN		1
+#define MSM_KEEPER		2
+#define MSM_PULL_UP_NO_KEEPER	2
+#define MSM_PULL_UP		3
 
 static unsigned msm_regval_to_drive(u32 val)
 {
@@ -243,10 +244,16 @@ static int msm_config_group_get(struct pinctrl_dev *pctldev,
 		arg = arg == MSM_PULL_DOWN;
 		break;
 	case PIN_CONFIG_BIAS_BUS_HOLD:
+		if (pctrl->soc->pull_no_keeper)
+			return -ENOTSUPP;
+
 		arg = arg == MSM_KEEPER;
 		break;
 	case PIN_CONFIG_BIAS_PULL_UP:
-		arg = arg == MSM_PULL_UP;
+		if (pctrl->soc->pull_no_keeper)
+			arg = arg == MSM_PULL_UP_NO_KEEPER;
+		else
+			arg = arg == MSM_PULL_UP;
 		break;
 	case PIN_CONFIG_DRIVE_STRENGTH:
 		arg = msm_regval_to_drive(arg);
@@ -309,10 +316,16 @@ static int msm_config_group_set(struct pinctrl_dev *pctldev,
 			arg = MSM_PULL_DOWN;
 			break;
 		case PIN_CONFIG_BIAS_BUS_HOLD:
+			if (pctrl->soc->pull_no_keeper)
+				return -ENOTSUPP;
+
 			arg = MSM_KEEPER;
 			break;
 		case PIN_CONFIG_BIAS_PULL_UP:
-			arg = MSM_PULL_UP;
+			if (pctrl->soc->pull_no_keeper)
+				arg = MSM_PULL_UP_NO_KEEPER;
+			else
+				arg = MSM_PULL_UP;
 			break;
 		case PIN_CONFIG_DRIVE_STRENGTH:
 			/* Check for invalid values */
@@ -521,7 +534,7 @@ static void msm_gpio_dbg_show(struct seq_file *s, struct gpio_chip *chip)
 #define msm_gpio_dbg_show NULL
 #endif
 
-static struct gpio_chip msm_gpio_template = {
+static const struct gpio_chip msm_gpio_template = {
 	.direction_input  = msm_gpio_direction_input,
 	.direction_output = msm_gpio_direction_output,
 	.get_direction    = msm_gpio_get_direction,
diff --git a/drivers/pinctrl/qcom/pinctrl-msm.h b/drivers/pinctrl/qcom/pinctrl-msm.h
index 54fdd04ce9d5..9b9feea540ff 100644
--- a/drivers/pinctrl/qcom/pinctrl-msm.h
+++ b/drivers/pinctrl/qcom/pinctrl-msm.h
@@ -99,13 +99,14 @@ struct msm_pingroup {
 
 /**
  * struct msm_pinctrl_soc_data - Qualcomm pin controller driver configuration
- * @pins:       An array describing all pins the pin controller affects.
- * @npins:      The number of entries in @pins.
- * @functions:  An array describing all mux functions the SoC supports.
- * @nfunctions: The number of entries in @functions.
- * @groups:     An array describing all pin groups the pin SoC supports.
- * @ngroups:    The numbmer of entries in @groups.
- * @ngpio:      The number of pingroups the driver should expose as GPIOs.
+ * @pins:	    An array describing all pins the pin controller affects.
+ * @npins:	    The number of entries in @pins.
+ * @functions:	    An array describing all mux functions the SoC supports.
+ * @nfunctions:	    The number of entries in @functions.
+ * @groups:	    An array describing all pin groups the pin SoC supports.
+ * @ngroups:	    The numbmer of entries in @groups.
+ * @ngpio:	    The number of pingroups the driver should expose as GPIOs.
+ * @pull_no_keeper: The SoC does not support keeper bias.
  */
 struct msm_pinctrl_soc_data {
 	const struct pinctrl_pin_desc *pins;
@@ -115,6 +116,7 @@ struct msm_pinctrl_soc_data {
 	const struct msm_pingroup *groups;
 	unsigned ngroups;
 	unsigned ngpios;
+	bool pull_no_keeper;
 };
 
 int msm_pinctrl_probe(struct platform_device *pdev,
diff --git a/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c b/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c
index 664b641fd776..c2c0bab04257 100644
--- a/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c
+++ b/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c
@@ -40,6 +40,8 @@
 #define PMIC_GPIO_SUBTYPE_GPIOC_4CH		0x5
 #define PMIC_GPIO_SUBTYPE_GPIO_8CH		0x9
 #define PMIC_GPIO_SUBTYPE_GPIOC_8CH		0xd
+#define PMIC_GPIO_SUBTYPE_GPIO_LV		0x10
+#define PMIC_GPIO_SUBTYPE_GPIO_MV		0x11
 
 #define PMIC_MPP_REG_RT_STS			0x10
 #define PMIC_MPP_REG_RT_STS_VAL_MASK		0x1
@@ -48,8 +50,11 @@
 #define PMIC_GPIO_REG_MODE_CTL			0x40
 #define PMIC_GPIO_REG_DIG_VIN_CTL		0x41
 #define PMIC_GPIO_REG_DIG_PULL_CTL		0x42
+#define PMIC_GPIO_REG_LV_MV_DIG_OUT_SOURCE_CTL	0x44
+#define PMIC_GPIO_REG_DIG_IN_CTL		0x43
 #define PMIC_GPIO_REG_DIG_OUT_CTL		0x45
 #define PMIC_GPIO_REG_EN_CTL			0x46
+#define PMIC_GPIO_REG_LV_MV_ANA_PASS_THRU_SEL	0x4A
 
 /* PMIC_GPIO_REG_MODE_CTL */
 #define PMIC_GPIO_REG_MODE_VALUE_SHIFT		0x1
@@ -58,6 +63,12 @@
 #define PMIC_GPIO_REG_MODE_DIR_SHIFT		4
 #define PMIC_GPIO_REG_MODE_DIR_MASK		0x7
 
+#define PMIC_GPIO_MODE_DIGITAL_INPUT		0
+#define PMIC_GPIO_MODE_DIGITAL_OUTPUT		1
+#define PMIC_GPIO_MODE_DIGITAL_INPUT_OUTPUT	2
+#define PMIC_GPIO_MODE_ANALOG_PASS_THRU		3
+#define PMIC_GPIO_REG_LV_MV_MODE_DIR_MASK	0x3
+
 /* PMIC_GPIO_REG_DIG_VIN_CTL */
 #define PMIC_GPIO_REG_VIN_SHIFT			0
 #define PMIC_GPIO_REG_VIN_MASK			0x7
@@ -69,6 +80,16 @@
 #define PMIC_GPIO_PULL_DOWN			4
 #define PMIC_GPIO_PULL_DISABLE			5
 
+/* PMIC_GPIO_REG_LV_MV_DIG_OUT_SOURCE_CTL for LV/MV */
+#define PMIC_GPIO_LV_MV_OUTPUT_INVERT		0x80
+#define PMIC_GPIO_LV_MV_OUTPUT_INVERT_SHIFT	7
+#define PMIC_GPIO_LV_MV_OUTPUT_SOURCE_SEL_MASK	0xF
+
+/* PMIC_GPIO_REG_DIG_IN_CTL */
+#define PMIC_GPIO_LV_MV_DIG_IN_DTEST_EN		0x80
+#define PMIC_GPIO_LV_MV_DIG_IN_DTEST_SEL_MASK	0x7
+#define PMIC_GPIO_DIG_IN_DTEST_SEL_MASK		0xf
+
 /* PMIC_GPIO_REG_DIG_OUT_CTL */
 #define PMIC_GPIO_REG_OUT_STRENGTH_SHIFT	0
 #define PMIC_GPIO_REG_OUT_STRENGTH_MASK		0x3
@@ -88,9 +109,29 @@
 
 #define PMIC_GPIO_PHYSICAL_OFFSET		1
 
+/* PMIC_GPIO_REG_LV_MV_ANA_PASS_THRU_SEL */
+#define PMIC_GPIO_LV_MV_ANA_MUX_SEL_MASK		0x3
+
 /* Qualcomm specific pin configurations */
 #define PMIC_GPIO_CONF_PULL_UP			(PIN_CONFIG_END + 1)
 #define PMIC_GPIO_CONF_STRENGTH			(PIN_CONFIG_END + 2)
+#define PMIC_GPIO_CONF_ATEST			(PIN_CONFIG_END + 3)
+#define PMIC_GPIO_CONF_ANALOG_PASS		(PIN_CONFIG_END + 4)
+#define PMIC_GPIO_CONF_DTEST_BUFFER		(PIN_CONFIG_END + 5)
+
+/* The index of each function in pmic_gpio_functions[] array */
+enum pmic_gpio_func_index {
+	PMIC_GPIO_FUNC_INDEX_NORMAL,
+	PMIC_GPIO_FUNC_INDEX_PAIRED,
+	PMIC_GPIO_FUNC_INDEX_FUNC1,
+	PMIC_GPIO_FUNC_INDEX_FUNC2,
+	PMIC_GPIO_FUNC_INDEX_FUNC3,
+	PMIC_GPIO_FUNC_INDEX_FUNC4,
+	PMIC_GPIO_FUNC_INDEX_DTEST1,
+	PMIC_GPIO_FUNC_INDEX_DTEST2,
+	PMIC_GPIO_FUNC_INDEX_DTEST3,
+	PMIC_GPIO_FUNC_INDEX_DTEST4,
+};
 
 /**
  * struct pmic_gpio_pad - keep current GPIO settings
@@ -102,12 +143,16 @@
  *	open-drain or open-source mode.
  * @output_enabled: Set to true if GPIO output logic is enabled.
  * @input_enabled: Set to true if GPIO input buffer logic is enabled.
+ * @analog_pass: Set to true if GPIO is in analog-pass-through mode.
+ * @lv_mv_type: Set to true if GPIO subtype is GPIO_LV(0x10) or GPIO_MV(0x11).
  * @num_sources: Number of power-sources supported by this GPIO.
  * @power_source: Current power-source used.
  * @buffer_type: Push-pull, open-drain or open-source.
  * @pullup: Constant current which flow trough GPIO output buffer.
  * @strength: No, Low, Medium, High
  * @function: See pmic_gpio_functions[]
+ * @atest: the ATEST selection for GPIO analog-pass-through mode
+ * @dtest_buffer: the DTEST buffer selection for digital input mode.
  */
 struct pmic_gpio_pad {
 	u16		base;
@@ -117,12 +162,16 @@ struct pmic_gpio_pad {
 	bool		have_buffer;
 	bool		output_enabled;
 	bool		input_enabled;
+	bool		analog_pass;
+	bool		lv_mv_type;
 	unsigned int	num_sources;
 	unsigned int	power_source;
 	unsigned int	buffer_type;
 	unsigned int	pullup;
 	unsigned int	strength;
 	unsigned int	function;
+	unsigned int	atest;
+	unsigned int	dtest_buffer;
 };
 
 struct pmic_gpio_state {
@@ -135,12 +184,18 @@ struct pmic_gpio_state {
 static const struct pinconf_generic_params pmic_gpio_bindings[] = {
 	{"qcom,pull-up-strength",	PMIC_GPIO_CONF_PULL_UP,		0},
 	{"qcom,drive-strength",		PMIC_GPIO_CONF_STRENGTH,	0},
+	{"qcom,atest",			PMIC_GPIO_CONF_ATEST,		0},
+	{"qcom,analog-pass",		PMIC_GPIO_CONF_ANALOG_PASS,	0},
+	{"qcom,dtest-buffer",           PMIC_GPIO_CONF_DTEST_BUFFER,    0},
 };
 
 #ifdef CONFIG_DEBUG_FS
 static const struct pin_config_item pmic_conf_items[ARRAY_SIZE(pmic_gpio_bindings)] = {
 	PCONFDUMP(PMIC_GPIO_CONF_PULL_UP,  "pull up strength", NULL, true),
 	PCONFDUMP(PMIC_GPIO_CONF_STRENGTH, "drive-strength", NULL, true),
+	PCONFDUMP(PMIC_GPIO_CONF_ATEST, "atest", NULL, true),
+	PCONFDUMP(PMIC_GPIO_CONF_ANALOG_PASS, "analog-pass", NULL, true),
+	PCONFDUMP(PMIC_GPIO_CONF_DTEST_BUFFER, "dtest-buffer", NULL, true),
 };
 #endif
 
@@ -153,10 +208,16 @@ static const char *const pmic_gpio_groups[] = {
 };
 
 static const char *const pmic_gpio_functions[] = {
-	PMIC_GPIO_FUNC_NORMAL, PMIC_GPIO_FUNC_PAIRED,
-	PMIC_GPIO_FUNC_FUNC1, PMIC_GPIO_FUNC_FUNC2,
-	PMIC_GPIO_FUNC_DTEST1, PMIC_GPIO_FUNC_DTEST2,
-	PMIC_GPIO_FUNC_DTEST3, PMIC_GPIO_FUNC_DTEST4,
+	[PMIC_GPIO_FUNC_INDEX_NORMAL]	= PMIC_GPIO_FUNC_NORMAL,
+	[PMIC_GPIO_FUNC_INDEX_PAIRED]	= PMIC_GPIO_FUNC_PAIRED,
+	[PMIC_GPIO_FUNC_INDEX_FUNC1]	= PMIC_GPIO_FUNC_FUNC1,
+	[PMIC_GPIO_FUNC_INDEX_FUNC2]	= PMIC_GPIO_FUNC_FUNC2,
+	[PMIC_GPIO_FUNC_INDEX_FUNC3]	= PMIC_GPIO_FUNC_FUNC3,
+	[PMIC_GPIO_FUNC_INDEX_FUNC4]	= PMIC_GPIO_FUNC_FUNC4,
+	[PMIC_GPIO_FUNC_INDEX_DTEST1]	= PMIC_GPIO_FUNC_DTEST1,
+	[PMIC_GPIO_FUNC_INDEX_DTEST2]	= PMIC_GPIO_FUNC_DTEST2,
+	[PMIC_GPIO_FUNC_INDEX_DTEST3]	= PMIC_GPIO_FUNC_DTEST3,
+	[PMIC_GPIO_FUNC_INDEX_DTEST4]	= PMIC_GPIO_FUNC_DTEST4,
 };
 
 static int pmic_gpio_read(struct pmic_gpio_state *state,
@@ -244,25 +305,67 @@ static int pmic_gpio_set_mux(struct pinctrl_dev *pctldev, unsigned function,
 	unsigned int val;
 	int ret;
 
+	if (function > PMIC_GPIO_FUNC_INDEX_DTEST4) {
+		pr_err("function: %d is not defined\n", function);
+		return -EINVAL;
+	}
+
 	pad = pctldev->desc->pins[pin].drv_data;
+	/*
+	 * Non-LV/MV subtypes only support 2 special functions,
+	 * offsetting the dtestx function values by 2
+	 */
+	if (!pad->lv_mv_type) {
+		if (function == PMIC_GPIO_FUNC_INDEX_FUNC3 ||
+				function == PMIC_GPIO_FUNC_INDEX_FUNC4) {
+			pr_err("LV/MV subtype doesn't have func3/func4\n");
+			return -EINVAL;
+		}
+		if (function >= PMIC_GPIO_FUNC_INDEX_DTEST1)
+			function -= (PMIC_GPIO_FUNC_INDEX_DTEST1 -
+					PMIC_GPIO_FUNC_INDEX_FUNC3);
+	}
 
 	pad->function = function;
 
-	val = 0;
-	if (pad->output_enabled) {
-		if (pad->input_enabled)
-			val = 2;
-		else
-			val = 1;
-	}
+	if (pad->analog_pass)
+		val = PMIC_GPIO_MODE_ANALOG_PASS_THRU;
+	else if (pad->output_enabled && pad->input_enabled)
+		val = PMIC_GPIO_MODE_DIGITAL_INPUT_OUTPUT;
+	else if (pad->output_enabled)
+		val = PMIC_GPIO_MODE_DIGITAL_OUTPUT;
+	else
+		val = PMIC_GPIO_MODE_DIGITAL_INPUT;
 
-	val = val << PMIC_GPIO_REG_MODE_DIR_SHIFT;
-	val |= pad->function << PMIC_GPIO_REG_MODE_FUNCTION_SHIFT;
-	val |= pad->out_value & PMIC_GPIO_REG_MODE_VALUE_SHIFT;
+	if (pad->lv_mv_type) {
+		ret = pmic_gpio_write(state, pad,
+				PMIC_GPIO_REG_MODE_CTL, val);
+		if (ret < 0)
+			return ret;
 
-	ret = pmic_gpio_write(state, pad, PMIC_GPIO_REG_MODE_CTL, val);
-	if (ret < 0)
-		return ret;
+		val = pad->atest - 1;
+		ret = pmic_gpio_write(state, pad,
+				PMIC_GPIO_REG_LV_MV_ANA_PASS_THRU_SEL, val);
+		if (ret < 0)
+			return ret;
+
+		val = pad->out_value
+			<< PMIC_GPIO_LV_MV_OUTPUT_INVERT_SHIFT;
+		val |= pad->function
+			& PMIC_GPIO_LV_MV_OUTPUT_SOURCE_SEL_MASK;
+		ret = pmic_gpio_write(state, pad,
+			PMIC_GPIO_REG_LV_MV_DIG_OUT_SOURCE_CTL, val);
+		if (ret < 0)
+			return ret;
+	} else {
+		val = val << PMIC_GPIO_REG_MODE_DIR_SHIFT;
+		val |= pad->function << PMIC_GPIO_REG_MODE_FUNCTION_SHIFT;
+		val |= pad->out_value & PMIC_GPIO_REG_MODE_VALUE_SHIFT;
+
+		ret = pmic_gpio_write(state, pad, PMIC_GPIO_REG_MODE_CTL, val);
+		if (ret < 0)
+			return ret;
+	}
 
 	val = pad->is_enabled << PMIC_GPIO_REG_MASTER_EN_SHIFT;
 
@@ -322,6 +425,15 @@ static int pmic_gpio_config_get(struct pinctrl_dev *pctldev,
 	case PMIC_GPIO_CONF_STRENGTH:
 		arg = pad->strength;
 		break;
+	case PMIC_GPIO_CONF_ATEST:
+		arg = pad->atest;
+		break;
+	case PMIC_GPIO_CONF_ANALOG_PASS:
+		arg = pad->analog_pass;
+		break;
+	case PMIC_GPIO_CONF_DTEST_BUFFER:
+		arg = pad->dtest_buffer;
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -375,7 +487,7 @@ static int pmic_gpio_config_set(struct pinctrl_dev *pctldev, unsigned int pin,
 			pad->is_enabled = false;
 			break;
 		case PIN_CONFIG_POWER_SOURCE:
-			if (arg > pad->num_sources)
+			if (arg >= pad->num_sources)
 				return -EINVAL;
 			pad->power_source = arg;
 			break;
@@ -396,6 +508,21 @@ static int pmic_gpio_config_set(struct pinctrl_dev *pctldev, unsigned int pin,
 				return -EINVAL;
 			pad->strength = arg;
 			break;
+		case PMIC_GPIO_CONF_ATEST:
+			if (!pad->lv_mv_type || arg > 4)
+				return -EINVAL;
+			pad->atest = arg;
+			break;
+		case PMIC_GPIO_CONF_ANALOG_PASS:
+			if (!pad->lv_mv_type)
+				return -EINVAL;
+			pad->analog_pass = true;
+			break;
+		case PMIC_GPIO_CONF_DTEST_BUFFER:
+			if (arg > 4)
+				return -EINVAL;
+			pad->dtest_buffer = arg;
+			break;
 		default:
 			return -EINVAL;
 		}
@@ -420,19 +547,60 @@ static int pmic_gpio_config_set(struct pinctrl_dev *pctldev, unsigned int pin,
 	if (ret < 0)
 		return ret;
 
-	val = 0;
-	if (pad->output_enabled) {
-		if (pad->input_enabled)
-			val = 2;
-		else
-			val = 1;
+	if (pad->dtest_buffer == 0) {
+		val = 0;
+	} else {
+		if (pad->lv_mv_type) {
+			val = pad->dtest_buffer - 1;
+			val |= PMIC_GPIO_LV_MV_DIG_IN_DTEST_EN;
+		} else {
+			val = BIT(pad->dtest_buffer - 1);
+		}
 	}
+	ret = pmic_gpio_write(state, pad, PMIC_GPIO_REG_DIG_IN_CTL, val);
+	if (ret < 0)
+		return ret;
+
+	if (pad->analog_pass)
+		val = PMIC_GPIO_MODE_ANALOG_PASS_THRU;
+	else if (pad->output_enabled && pad->input_enabled)
+		val = PMIC_GPIO_MODE_DIGITAL_INPUT_OUTPUT;
+	else if (pad->output_enabled)
+		val = PMIC_GPIO_MODE_DIGITAL_OUTPUT;
+	else
+		val = PMIC_GPIO_MODE_DIGITAL_INPUT;
 
-	val = val << PMIC_GPIO_REG_MODE_DIR_SHIFT;
-	val |= pad->function << PMIC_GPIO_REG_MODE_FUNCTION_SHIFT;
-	val |= pad->out_value & PMIC_GPIO_REG_MODE_VALUE_SHIFT;
+	if (pad->lv_mv_type) {
+		ret = pmic_gpio_write(state, pad,
+				PMIC_GPIO_REG_MODE_CTL, val);
+		if (ret < 0)
+			return ret;
 
-	return pmic_gpio_write(state, pad, PMIC_GPIO_REG_MODE_CTL, val);
+		val = pad->atest - 1;
+		ret = pmic_gpio_write(state, pad,
+				PMIC_GPIO_REG_LV_MV_ANA_PASS_THRU_SEL, val);
+		if (ret < 0)
+			return ret;
+
+		val = pad->out_value
+			<< PMIC_GPIO_LV_MV_OUTPUT_INVERT_SHIFT;
+		val |= pad->function
+			& PMIC_GPIO_LV_MV_OUTPUT_SOURCE_SEL_MASK;
+		ret = pmic_gpio_write(state, pad,
+			PMIC_GPIO_REG_LV_MV_DIG_OUT_SOURCE_CTL, val);
+		if (ret < 0)
+			return ret;
+	} else {
+		val = val << PMIC_GPIO_REG_MODE_DIR_SHIFT;
+		val |= pad->function << PMIC_GPIO_REG_MODE_FUNCTION_SHIFT;
+		val |= pad->out_value & PMIC_GPIO_REG_MODE_VALUE_SHIFT;
+
+		ret = pmic_gpio_write(state, pad, PMIC_GPIO_REG_MODE_CTL, val);
+		if (ret < 0)
+			return ret;
+	}
+
+	return ret;
 }
 
 static void pmic_gpio_config_dbg_show(struct pinctrl_dev *pctldev,
@@ -440,7 +608,7 @@ static void pmic_gpio_config_dbg_show(struct pinctrl_dev *pctldev,
 {
 	struct pmic_gpio_state *state = pinctrl_dev_get_drvdata(pctldev);
 	struct pmic_gpio_pad *pad;
-	int ret, val;
+	int ret, val, function;
 
 	static const char *const biases[] = {
 		"pull-up 30uA", "pull-up 1.5uA", "pull-up 31.5uA",
@@ -462,7 +630,6 @@ static void pmic_gpio_config_dbg_show(struct pinctrl_dev *pctldev,
 	if (val < 0 || !(val >> PMIC_GPIO_REG_MASTER_EN_SHIFT)) {
 		seq_puts(s, " ---");
 	} else {
-
 		if (pad->input_enabled) {
 			ret = pmic_gpio_read(state, pad, PMIC_MPP_REG_RT_STS);
 			if (ret < 0)
@@ -471,14 +638,29 @@ static void pmic_gpio_config_dbg_show(struct pinctrl_dev *pctldev,
 			ret &= PMIC_MPP_REG_RT_STS_VAL_MASK;
 			pad->out_value = ret;
 		}
-
-		seq_printf(s, " %-4s", pad->output_enabled ? "out" : "in");
-		seq_printf(s, " %-7s", pmic_gpio_functions[pad->function]);
+		/*
+		 * For the non-LV/MV subtypes only 2 special functions are
+		 * available, offsetting the dtest function values by 2.
+		 */
+		function = pad->function;
+		if (!pad->lv_mv_type &&
+				pad->function >= PMIC_GPIO_FUNC_INDEX_FUNC3)
+			function += PMIC_GPIO_FUNC_INDEX_DTEST1 -
+				PMIC_GPIO_FUNC_INDEX_FUNC3;
+
+		if (pad->analog_pass)
+			seq_puts(s, " analog-pass");
+		else
+			seq_printf(s, " %-4s",
+					pad->output_enabled ? "out" : "in");
+		seq_printf(s, " %-7s", pmic_gpio_functions[function]);
 		seq_printf(s, " vin-%d", pad->power_source);
 		seq_printf(s, " %-27s", biases[pad->pullup]);
 		seq_printf(s, " %-10s", buffer_types[pad->buffer_type]);
 		seq_printf(s, " %-4s", pad->out_value ? "high" : "low");
 		seq_printf(s, " %-7s", strengths[pad->strength]);
+		seq_printf(s, " atest-%d", pad->atest);
+		seq_printf(s, " dtest-%d", pad->dtest_buffer);
 	}
 }
 
@@ -618,40 +800,71 @@ static int pmic_gpio_populate(struct pmic_gpio_state *state,
 	case PMIC_GPIO_SUBTYPE_GPIOC_8CH:
 		pad->num_sources = 8;
 		break;
+	case PMIC_GPIO_SUBTYPE_GPIO_LV:
+		pad->num_sources = 1;
+		pad->have_buffer = true;
+		pad->lv_mv_type = true;
+		break;
+	case PMIC_GPIO_SUBTYPE_GPIO_MV:
+		pad->num_sources = 2;
+		pad->have_buffer = true;
+		pad->lv_mv_type = true;
+		break;
 	default:
 		dev_err(state->dev, "unknown GPIO type 0x%x\n", subtype);
 		return -ENODEV;
 	}
 
-	val = pmic_gpio_read(state, pad, PMIC_GPIO_REG_MODE_CTL);
-	if (val < 0)
-		return val;
+	if (pad->lv_mv_type) {
+		val = pmic_gpio_read(state, pad,
+				PMIC_GPIO_REG_LV_MV_DIG_OUT_SOURCE_CTL);
+		if (val < 0)
+			return val;
+
+		pad->out_value = !!(val & PMIC_GPIO_LV_MV_OUTPUT_INVERT);
+		pad->function = val & PMIC_GPIO_LV_MV_OUTPUT_SOURCE_SEL_MASK;
+
+		val = pmic_gpio_read(state, pad, PMIC_GPIO_REG_MODE_CTL);
+		if (val < 0)
+			return val;
+
+		dir = val & PMIC_GPIO_REG_LV_MV_MODE_DIR_MASK;
+	} else {
+		val = pmic_gpio_read(state, pad, PMIC_GPIO_REG_MODE_CTL);
+		if (val < 0)
+			return val;
+
+		pad->out_value = val & PMIC_GPIO_REG_MODE_VALUE_SHIFT;
 
-	pad->out_value = val & PMIC_GPIO_REG_MODE_VALUE_SHIFT;
+		dir = val >> PMIC_GPIO_REG_MODE_DIR_SHIFT;
+		dir &= PMIC_GPIO_REG_MODE_DIR_MASK;
+		pad->function = val >> PMIC_GPIO_REG_MODE_FUNCTION_SHIFT;
+		pad->function &= PMIC_GPIO_REG_MODE_FUNCTION_MASK;
+	}
 
-	dir = val >> PMIC_GPIO_REG_MODE_DIR_SHIFT;
-	dir &= PMIC_GPIO_REG_MODE_DIR_MASK;
 	switch (dir) {
-	case 0:
+	case PMIC_GPIO_MODE_DIGITAL_INPUT:
 		pad->input_enabled = true;
 		pad->output_enabled = false;
 		break;
-	case 1:
+	case PMIC_GPIO_MODE_DIGITAL_OUTPUT:
 		pad->input_enabled = false;
 		pad->output_enabled = true;
 		break;
-	case 2:
+	case PMIC_GPIO_MODE_DIGITAL_INPUT_OUTPUT:
 		pad->input_enabled = true;
 		pad->output_enabled = true;
 		break;
+	case PMIC_GPIO_MODE_ANALOG_PASS_THRU:
+		if (!pad->lv_mv_type)
+			return -ENODEV;
+		pad->analog_pass = true;
+		break;
 	default:
 		dev_err(state->dev, "unknown GPIO direction\n");
 		return -ENODEV;
 	}
 
-	pad->function = val >> PMIC_GPIO_REG_MODE_FUNCTION_SHIFT;
-	pad->function &= PMIC_GPIO_REG_MODE_FUNCTION_MASK;
-
 	val = pmic_gpio_read(state, pad, PMIC_GPIO_REG_DIG_VIN_CTL);
 	if (val < 0)
 		return val;
@@ -666,6 +879,18 @@ static int pmic_gpio_populate(struct pmic_gpio_state *state,
 	pad->pullup = val >> PMIC_GPIO_REG_PULL_SHIFT;
 	pad->pullup &= PMIC_GPIO_REG_PULL_MASK;
 
+	val = pmic_gpio_read(state, pad, PMIC_GPIO_REG_DIG_IN_CTL);
+	if (val < 0)
+		return val;
+
+	if (pad->lv_mv_type && (val & PMIC_GPIO_LV_MV_DIG_IN_DTEST_EN))
+		pad->dtest_buffer =
+			(val & PMIC_GPIO_LV_MV_DIG_IN_DTEST_SEL_MASK) + 1;
+	else if (!pad->lv_mv_type)
+		pad->dtest_buffer = ffs(val);
+	else
+		pad->dtest_buffer = 0;
+
 	val = pmic_gpio_read(state, pad, PMIC_GPIO_REG_DIG_OUT_CTL);
 	if (val < 0)
 		return val;
@@ -676,6 +901,14 @@ static int pmic_gpio_populate(struct pmic_gpio_state *state,
 	pad->buffer_type = val >> PMIC_GPIO_REG_OUT_TYPE_SHIFT;
 	pad->buffer_type &= PMIC_GPIO_REG_OUT_TYPE_MASK;
 
+	if (pad->lv_mv_type) {
+		val = pmic_gpio_read(state, pad,
+				PMIC_GPIO_REG_LV_MV_ANA_PASS_THRU_SEL);
+		if (val < 0)
+			return val;
+		pad->atest = (val & PMIC_GPIO_LV_MV_ANA_MUX_SEL_MASK) + 1;
+	}
+
 	/* Pin could be disabled with PIN_CONFIG_BIAS_HIGH_IMPEDANCE */
 	pad->is_enabled = true;
 	return 0;
diff --git a/drivers/pinctrl/qcom/pinctrl-ssbi-gpio.c b/drivers/pinctrl/qcom/pinctrl-ssbi-gpio.c
index d3f5501d17ee..f53e32a9d8fc 100644
--- a/drivers/pinctrl/qcom/pinctrl-ssbi-gpio.c
+++ b/drivers/pinctrl/qcom/pinctrl-ssbi-gpio.c
@@ -588,7 +588,7 @@ static void pm8xxx_gpio_dbg_show(struct seq_file *s, struct gpio_chip *chip)
 #define pm8xxx_gpio_dbg_show NULL
 #endif
 
-static struct gpio_chip pm8xxx_gpio_template = {
+static const struct gpio_chip pm8xxx_gpio_template = {
 	.direction_input = pm8xxx_gpio_direction_input,
 	.direction_output = pm8xxx_gpio_direction_output,
 	.get = pm8xxx_gpio_get,
diff --git a/drivers/pinctrl/qcom/pinctrl-ssbi-mpp.c b/drivers/pinctrl/qcom/pinctrl-ssbi-mpp.c
index 0d1392fc32dd..1e513bd6d0a9 100644
--- a/drivers/pinctrl/qcom/pinctrl-ssbi-mpp.c
+++ b/drivers/pinctrl/qcom/pinctrl-ssbi-mpp.c
@@ -643,7 +643,7 @@ static void pm8xxx_mpp_dbg_show(struct seq_file *s, struct gpio_chip *chip)
 #define pm8xxx_mpp_dbg_show NULL
 #endif
 
-static struct gpio_chip pm8xxx_mpp_template = {
+static const struct gpio_chip pm8xxx_mpp_template = {
 	.direction_input = pm8xxx_mpp_direction_input,
 	.direction_output = pm8xxx_mpp_direction_output,
 	.get = pm8xxx_mpp_get,
diff --git a/drivers/pinctrl/samsung/pinctrl-exynos.c b/drivers/pinctrl/samsung/pinctrl-exynos.c
index 731530a9ce38..c8d0de7ea160 100644
--- a/drivers/pinctrl/samsung/pinctrl-exynos.c
+++ b/drivers/pinctrl/samsung/pinctrl-exynos.c
@@ -31,6 +31,8 @@
 #include <linux/err.h>
 #include <linux/soc/samsung/exynos-pmu.h>
 
+#include <dt-bindings/pinctrl/samsung.h>
+
 #include "pinctrl-samsung.h"
 #include "pinctrl-exynos.h"
 
@@ -149,15 +151,10 @@ static int exynos_irq_set_type(struct irq_data *irqd, unsigned int type)
 
 static int exynos_irq_request_resources(struct irq_data *irqd)
 {
-	struct irq_chip *chip = irq_data_get_irq_chip(irqd);
-	struct exynos_irq_chip *our_chip = to_exynos_irq_chip(chip);
 	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(irqd);
 	const struct samsung_pin_bank_type *bank_type = bank->type;
-	unsigned int shift = EXYNOS_EINT_CON_LEN * irqd->hwirq;
-	unsigned long reg_con = our_chip->eint_con + bank->eint_offset;
-	unsigned long flags;
-	unsigned int mask;
-	unsigned int con;
+	unsigned long reg_con, flags;
+	unsigned int shift, mask, con;
 	int ret;
 
 	ret = gpiochip_lock_as_irq(&bank->gpio_chip, irqd->hwirq);
@@ -174,10 +171,10 @@ static int exynos_irq_request_resources(struct irq_data *irqd)
 
 	spin_lock_irqsave(&bank->slock, flags);
 
-	con = readl(bank->eint_base + reg_con);
+	con = readl(bank->pctl_base + reg_con);
 	con &= ~(mask << shift);
-	con |= EXYNOS_EINT_FUNC << shift;
-	writel(con, bank->eint_base + reg_con);
+	con |= EXYNOS_PIN_FUNC_EINT << shift;
+	writel(con, bank->pctl_base + reg_con);
 
 	spin_unlock_irqrestore(&bank->slock, flags);
 
@@ -186,15 +183,10 @@ static int exynos_irq_request_resources(struct irq_data *irqd)
 
 static void exynos_irq_release_resources(struct irq_data *irqd)
 {
-	struct irq_chip *chip = irq_data_get_irq_chip(irqd);
-	struct exynos_irq_chip *our_chip = to_exynos_irq_chip(chip);
 	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(irqd);
 	const struct samsung_pin_bank_type *bank_type = bank->type;
-	unsigned int shift = EXYNOS_EINT_CON_LEN * irqd->hwirq;
-	unsigned long reg_con = our_chip->eint_con + bank->eint_offset;
-	unsigned long flags;
-	unsigned int mask;
-	unsigned int con;
+	unsigned long reg_con, flags;
+	unsigned int shift, mask, con;
 
 	reg_con = bank->pctl_offset + bank_type->reg_offset[PINCFG_TYPE_FUNC];
 	shift = irqd->hwirq * bank_type->fld_width[PINCFG_TYPE_FUNC];
@@ -202,10 +194,10 @@ static void exynos_irq_release_resources(struct irq_data *irqd)
 
 	spin_lock_irqsave(&bank->slock, flags);
 
-	con = readl(bank->eint_base + reg_con);
+	con = readl(bank->pctl_base + reg_con);
 	con &= ~(mask << shift);
-	con |= FUNC_INPUT << shift;
-	writel(con, bank->eint_base + reg_con);
+	con |= EXYNOS_PIN_FUNC_INPUT << shift;
+	writel(con, bank->pctl_base + reg_con);
 
 	spin_unlock_irqrestore(&bank->slock, flags);
 
diff --git a/drivers/pinctrl/samsung/pinctrl-exynos.h b/drivers/pinctrl/samsung/pinctrl-exynos.h
index b90139715c8f..7639b926c5c1 100644
--- a/drivers/pinctrl/samsung/pinctrl-exynos.h
+++ b/drivers/pinctrl/samsung/pinctrl-exynos.h
@@ -32,7 +32,6 @@
 #define EXYNOS7_WKUP_EMASK_OFFSET	0x900
 #define EXYNOS7_WKUP_EPEND_OFFSET	0xA00
 #define EXYNOS_SVC_OFFSET		0xB08
-#define EXYNOS_EINT_FUNC		0xF
 
 /* helpers to access interrupt service register */
 #define EXYNOS_SVC_GROUP_SHIFT		3
diff --git a/drivers/pinctrl/samsung/pinctrl-s3c24xx.c b/drivers/pinctrl/samsung/pinctrl-s3c24xx.c
index 49774851e84a..edf27264b603 100644
--- a/drivers/pinctrl/samsung/pinctrl-s3c24xx.c
+++ b/drivers/pinctrl/samsung/pinctrl-s3c24xx.c
@@ -151,7 +151,7 @@ static void s3c24xx_eint_set_function(struct samsung_pinctrl_drv_data *d,
 	u32 val;
 
 	/* Make sure that pin is configured as interrupt */
-	reg = bank->pctl_base + bank->pctl_offset;
+	reg = d->virt_base + bank->pctl_offset;
 	shift = pin * bank_type->fld_width[PINCFG_TYPE_FUNC];
 	mask = (1 << bank_type->fld_width[PINCFG_TYPE_FUNC]) - 1;
 
@@ -184,7 +184,7 @@ static int s3c24xx_eint_type(struct irq_data *data, unsigned int type)
 	s3c24xx_eint_set_handler(data, type);
 
 	/* Set up interrupt trigger */
-	reg = bank->eint_base + EINT_REG(index);
+	reg = d->virt_base + EINT_REG(index);
 	shift = EINT_OFFS(index);
 
 	val = readl(reg);
@@ -259,29 +259,32 @@ static void s3c2410_demux_eint0_3(struct irq_desc *desc)
 static void s3c2412_eint0_3_ack(struct irq_data *data)
 {
 	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(data);
+	struct samsung_pinctrl_drv_data *d = bank->drvdata;
 
 	unsigned long bitval = 1UL << data->hwirq;
-	writel(bitval, bank->eint_base + EINTPEND_REG);
+	writel(bitval, d->virt_base + EINTPEND_REG);
 }
 
 static void s3c2412_eint0_3_mask(struct irq_data *data)
 {
 	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(data);
+	struct samsung_pinctrl_drv_data *d = bank->drvdata;
 	unsigned long mask;
 
-	mask = readl(bank->eint_base + EINTMASK_REG);
+	mask = readl(d->virt_base + EINTMASK_REG);
 	mask |= (1UL << data->hwirq);
-	writel(mask, bank->eint_base + EINTMASK_REG);
+	writel(mask, d->virt_base + EINTMASK_REG);
 }
 
 static void s3c2412_eint0_3_unmask(struct irq_data *data)
 {
 	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(data);
+	struct samsung_pinctrl_drv_data *d = bank->drvdata;
 	unsigned long mask;
 
-	mask = readl(bank->eint_base + EINTMASK_REG);
+	mask = readl(d->virt_base + EINTMASK_REG);
 	mask &= ~(1UL << data->hwirq);
-	writel(mask, bank->eint_base + EINTMASK_REG);
+	writel(mask, d->virt_base + EINTMASK_REG);
 }
 
 static struct irq_chip s3c2412_eint0_3_chip = {
@@ -316,31 +319,34 @@ static void s3c2412_demux_eint0_3(struct irq_desc *desc)
 static void s3c24xx_eint_ack(struct irq_data *data)
 {
 	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(data);
+	struct samsung_pinctrl_drv_data *d = bank->drvdata;
 	unsigned char index = bank->eint_offset + data->hwirq;
 
-	writel(1UL << index, bank->eint_base + EINTPEND_REG);
+	writel(1UL << index, d->virt_base + EINTPEND_REG);
 }
 
 static void s3c24xx_eint_mask(struct irq_data *data)
 {
 	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(data);
+	struct samsung_pinctrl_drv_data *d = bank->drvdata;
 	unsigned char index = bank->eint_offset + data->hwirq;
 	unsigned long mask;
 
-	mask = readl(bank->eint_base + EINTMASK_REG);
+	mask = readl(d->virt_base + EINTMASK_REG);
 	mask |= (1UL << index);
-	writel(mask, bank->eint_base + EINTMASK_REG);
+	writel(mask, d->virt_base + EINTMASK_REG);
 }
 
 static void s3c24xx_eint_unmask(struct irq_data *data)
 {
 	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(data);
+	struct samsung_pinctrl_drv_data *d = bank->drvdata;
 	unsigned char index = bank->eint_offset + data->hwirq;
 	unsigned long mask;
 
-	mask = readl(bank->eint_base + EINTMASK_REG);
+	mask = readl(d->virt_base + EINTMASK_REG);
 	mask &= ~(1UL << index);
-	writel(mask, bank->eint_base + EINTMASK_REG);
+	writel(mask, d->virt_base + EINTMASK_REG);
 }
 
 static struct irq_chip s3c24xx_eint_chip = {
@@ -356,14 +362,13 @@ static inline void s3c24xx_demux_eint(struct irq_desc *desc,
 {
 	struct s3c24xx_eint_data *data = irq_desc_get_handler_data(desc);
 	struct irq_chip *chip = irq_desc_get_chip(desc);
-	struct irq_data *irqd = irq_desc_get_irq_data(desc);
-	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(irqd);
+	struct samsung_pinctrl_drv_data *d = data->drvdata;
 	unsigned int pend, mask;
 
 	chained_irq_enter(chip, desc);
 
-	pend = readl(bank->eint_base + EINTPEND_REG);
-	mask = readl(bank->eint_base + EINTMASK_REG);
+	pend = readl(d->virt_base + EINTPEND_REG);
+	mask = readl(d->virt_base + EINTMASK_REG);
 
 	pend &= ~mask;
 	pend &= range;
diff --git a/drivers/pinctrl/samsung/pinctrl-s3c64xx.c b/drivers/pinctrl/samsung/pinctrl-s3c64xx.c
index 4a88d7446e87..e63663b32907 100644
--- a/drivers/pinctrl/samsung/pinctrl-s3c64xx.c
+++ b/drivers/pinctrl/samsung/pinctrl-s3c64xx.c
@@ -280,7 +280,7 @@ static void s3c64xx_irq_set_function(struct samsung_pinctrl_drv_data *d,
 	u32 val;
 
 	/* Make sure that pin is configured as interrupt */
-	reg = bank->pctl_base + bank->pctl_offset;
+	reg = d->virt_base + bank->pctl_offset;
 	shift = pin;
 	if (bank_type->fld_width[PINCFG_TYPE_FUNC] * shift >= 32) {
 		/* 4-bit bank type with 2 con regs */
@@ -308,8 +308,9 @@ static void s3c64xx_irq_set_function(struct samsung_pinctrl_drv_data *d,
 static inline void s3c64xx_gpio_irq_set_mask(struct irq_data *irqd, bool mask)
 {
 	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(irqd);
+	struct samsung_pinctrl_drv_data *d = bank->drvdata;
 	unsigned char index = EINT_OFFS(bank->eint_offset) + irqd->hwirq;
-	void __iomem *reg = bank->eint_base + EINTMASK_REG(bank->eint_offset);
+	void __iomem *reg = d->virt_base + EINTMASK_REG(bank->eint_offset);
 	u32 val;
 
 	val = readl(reg);
@@ -333,8 +334,9 @@ static void s3c64xx_gpio_irq_mask(struct irq_data *irqd)
 static void s3c64xx_gpio_irq_ack(struct irq_data *irqd)
 {
 	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(irqd);
+	struct samsung_pinctrl_drv_data *d = bank->drvdata;
 	unsigned char index = EINT_OFFS(bank->eint_offset) + irqd->hwirq;
-	void __iomem *reg = bank->eint_base + EINTPEND_REG(bank->eint_offset);
+	void __iomem *reg = d->virt_base + EINTPEND_REG(bank->eint_offset);
 
 	writel(1 << index, reg);
 }
@@ -357,7 +359,7 @@ static int s3c64xx_gpio_irq_set_type(struct irq_data *irqd, unsigned int type)
 	s3c64xx_irq_set_handler(irqd, type);
 
 	/* Set up interrupt trigger */
-	reg = bank->eint_base + EINTCON_REG(bank->eint_offset);
+	reg = d->virt_base + EINTCON_REG(bank->eint_offset);
 	shift = EINT_OFFS(bank->eint_offset) + irqd->hwirq;
 	shift = 4 * (shift / 4); /* 4 EINTs per trigger selector */
 
@@ -409,8 +411,7 @@ static void s3c64xx_eint_gpio_irq(struct irq_desc *desc)
 {
 	struct irq_chip *chip = irq_desc_get_chip(desc);
 	struct s3c64xx_eint_gpio_data *data = irq_desc_get_handler_data(desc);
-	struct irq_data *irqd = irq_desc_get_irq_data(desc);
-	struct samsung_pin_bank *bank = irq_data_get_irq_chip_data(irqd);
+	struct samsung_pinctrl_drv_data *drvdata = data->drvdata;
 
 	chained_irq_enter(chip, desc);
 
@@ -420,7 +421,7 @@ static void s3c64xx_eint_gpio_irq(struct irq_desc *desc)
 		unsigned int pin;
 		unsigned int virq;
 
-		svc = readl(bank->eint_base + SERVICE_REG);
+		svc = readl(drvdata->virt_base + SERVICE_REG);
 		group = SVC_GROUP(svc);
 		pin = svc & SVC_NUM_MASK;
 
@@ -515,15 +516,15 @@ static inline void s3c64xx_eint0_irq_set_mask(struct irq_data *irqd, bool mask)
 {
 	struct s3c64xx_eint0_domain_data *ddata =
 					irq_data_get_irq_chip_data(irqd);
-	struct samsung_pin_bank *bank = ddata->bank;
+	struct samsung_pinctrl_drv_data *d = ddata->bank->drvdata;
 	u32 val;
 
-	val = readl(bank->eint_base + EINT0MASK_REG);
+	val = readl(d->virt_base + EINT0MASK_REG);
 	if (mask)
 		val |= 1 << ddata->eints[irqd->hwirq];
 	else
 		val &= ~(1 << ddata->eints[irqd->hwirq]);
-	writel(val, bank->eint_base + EINT0MASK_REG);
+	writel(val, d->virt_base + EINT0MASK_REG);
 }
 
 static void s3c64xx_eint0_irq_unmask(struct irq_data *irqd)
@@ -540,10 +541,10 @@ static void s3c64xx_eint0_irq_ack(struct irq_data *irqd)
 {
 	struct s3c64xx_eint0_domain_data *ddata =
 					irq_data_get_irq_chip_data(irqd);
-	struct samsung_pin_bank *bank = ddata->bank;
+	struct samsung_pinctrl_drv_data *d = ddata->bank->drvdata;
 
 	writel(1 << ddata->eints[irqd->hwirq],
-					bank->eint_base + EINT0PEND_REG);
+					d->virt_base + EINT0PEND_REG);
 }
 
 static int s3c64xx_eint0_irq_set_type(struct irq_data *irqd, unsigned int type)
@@ -551,7 +552,7 @@ static int s3c64xx_eint0_irq_set_type(struct irq_data *irqd, unsigned int type)
 	struct s3c64xx_eint0_domain_data *ddata =
 					irq_data_get_irq_chip_data(irqd);
 	struct samsung_pin_bank *bank = ddata->bank;
-	struct samsung_pinctrl_drv_data *d = ddata->bank->drvdata;
+	struct samsung_pinctrl_drv_data *d = bank->drvdata;
 	void __iomem *reg;
 	int trigger;
 	u8 shift;
@@ -566,7 +567,7 @@ static int s3c64xx_eint0_irq_set_type(struct irq_data *irqd, unsigned int type)
 	s3c64xx_irq_set_handler(irqd, type);
 
 	/* Set up interrupt trigger */
-	reg = bank->eint_base + EINT0CON0_REG;
+	reg = d->virt_base + EINT0CON0_REG;
 	shift = ddata->eints[irqd->hwirq];
 	if (shift >= EINT_MAX_PER_REG) {
 		reg += 4;
@@ -598,19 +599,14 @@ static struct irq_chip s3c64xx_eint0_irq_chip = {
 static inline void s3c64xx_irq_demux_eint(struct irq_desc *desc, u32 range)
 {
 	struct irq_chip *chip = irq_desc_get_chip(desc);
-	struct irq_data *irqd = irq_desc_get_irq_data(desc);
-	struct s3c64xx_eint0_domain_data *ddata =
-					irq_data_get_irq_chip_data(irqd);
-	struct samsung_pin_bank *bank = ddata->bank;
-
 	struct s3c64xx_eint0_data *data = irq_desc_get_handler_data(desc);
-
+	struct samsung_pinctrl_drv_data *drvdata = data->drvdata;
 	unsigned int pend, mask;
 
 	chained_irq_enter(chip, desc);
 
-	pend = readl(bank->eint_base + EINT0PEND_REG);
-	mask = readl(bank->eint_base + EINT0MASK_REG);
+	pend = readl(drvdata->virt_base + EINT0PEND_REG);
+	mask = readl(drvdata->virt_base + EINT0MASK_REG);
 
 	pend = pend & range & ~mask;
 	pend &= range;
diff --git a/drivers/pinctrl/samsung/pinctrl-samsung.c b/drivers/pinctrl/samsung/pinctrl-samsung.c
index f542642eed8d..e04f7fe0a65d 100644
--- a/drivers/pinctrl/samsung/pinctrl-samsung.c
+++ b/drivers/pinctrl/samsung/pinctrl-samsung.c
@@ -30,6 +30,8 @@
 #include <linux/of_device.h>
 #include <linux/spinlock.h>
 
+#include <dt-bindings/pinctrl/samsung.h>
+
 #include "../core.h"
 #include "pinctrl-samsung.h"
 
@@ -586,7 +588,7 @@ static int samsung_gpio_set_direction(struct gpio_chip *gc,
 	data = readl(reg);
 	data &= ~(mask << shift);
 	if (!input)
-		data |= FUNC_OUTPUT << shift;
+		data |= EXYNOS_PIN_FUNC_OUTPUT << shift;
 	writel(data, reg);
 
 	return 0;
@@ -679,7 +681,7 @@ static int samsung_pinctrl_create_function(struct device *dev,
 
 	npins = of_property_count_strings(func_np, "samsung,pins");
 	if (npins < 1) {
-		dev_err(dev, "invalid pin list in %s node", func_np->name);
+		dev_err(dev, "invalid pin list in %pOFn node", func_np);
 		return -EINVAL;
 	}
 
@@ -696,8 +698,8 @@ static int samsung_pinctrl_create_function(struct device *dev,
 							i, &gname);
 		if (ret) {
 			dev_err(dev,
-				"failed to read pin name %d from %s node\n",
-				i, func_np->name);
+				"failed to read pin name %d from %pOFn node\n",
+				i, func_np);
 			return ret;
 		}
 
@@ -958,7 +960,7 @@ samsung_pinctrl_get_soc_data(struct samsung_pinctrl_drv_data *d,
 	struct samsung_pin_bank *bank;
 	struct resource *res;
 	void __iomem *virt_base[SAMSUNG_PINCTRL_NUM_RESOURCES];
-	int i;
+	unsigned int i;
 
 	id = of_alias_get_id(node, "pinctrl");
 	if (id < 0) {
@@ -1013,6 +1015,12 @@ samsung_pinctrl_get_soc_data(struct samsung_pinctrl_drv_data *d,
 		bank->eint_base = virt_base[0];
 		bank->pctl_base = virt_base[bdata->pctl_res_idx];
 	}
+	/*
+	 * Legacy platforms should provide only one resource with IO memory.
+	 * Store it as virt_base because legacy driver needs to access it
+	 * through samsung_pinctrl_drv_data.
+	 */
+	d->virt_base = virt_base[0];
 
 	for_each_child_of_node(node, np) {
 		if (!of_find_property(np, "gpio-controller", NULL))
diff --git a/drivers/pinctrl/samsung/pinctrl-samsung.h b/drivers/pinctrl/samsung/pinctrl-samsung.h
index 515a61035e54..9af07af6cad6 100644
--- a/drivers/pinctrl/samsung/pinctrl-samsung.h
+++ b/drivers/pinctrl/samsung/pinctrl-samsung.h
@@ -25,10 +25,6 @@
 
 #include <linux/gpio.h>
 
-/* pinmux function number for pin as gpio output line */
-#define FUNC_INPUT	0x0
-#define FUNC_OUTPUT	0x1
-
 /**
  * enum pincfg_type - possible pin configuration types supported.
  * @PINCFG_TYPE_FUNC: Function configuration.
@@ -234,8 +230,8 @@ struct samsung_retention_data {
  */
 struct samsung_pin_ctrl {
 	const struct samsung_pin_bank_data *pin_banks;
-	u32		nr_banks;
-	int		nr_ext_resources;
+	unsigned int	nr_banks;
+	unsigned int	nr_ext_resources;
 	const struct samsung_retention_data *retention_data;
 
 	int		(*eint_gpio_init)(struct samsung_pinctrl_drv_data *);
@@ -247,6 +243,10 @@ struct samsung_pin_ctrl {
 /**
  * struct samsung_pinctrl_drv_data: wrapper for holding driver data together.
  * @node: global list node
+ * @virt_base: register base address of the controller; this will be equal
+ *             to each bank samsung_pin_bank->pctl_base and used on legacy
+ *             platforms (like S3C24XX or S3C64XX) which has to access the base
+ *             through samsung_pinctrl_drv_data, not samsung_pin_bank).
  * @dev: device instance representing the controller.
  * @irq: interrpt number used by the controller to notify gpio interrupts.
  * @ctrl: pin controller instance managed by the driver.
@@ -262,6 +262,7 @@ struct samsung_pin_ctrl {
  */
 struct samsung_pinctrl_drv_data {
 	struct list_head		node;
+	void __iomem			*virt_base;
 	struct device			*dev;
 	int				irq;
 
@@ -274,7 +275,7 @@ struct samsung_pinctrl_drv_data {
 	unsigned int			nr_functions;
 
 	struct samsung_pin_bank		*pin_banks;
-	u32				nr_banks;
+	unsigned int			nr_banks;
 	unsigned int			pin_base;
 	unsigned int			nr_pins;
 
diff --git a/drivers/pinctrl/sh-pfc/Kconfig b/drivers/pinctrl/sh-pfc/Kconfig
index 24f76a05a5a9..5d5312eb7102 100644
--- a/drivers/pinctrl/sh-pfc/Kconfig
+++ b/drivers/pinctrl/sh-pfc/Kconfig
@@ -89,6 +89,11 @@ config PINCTRL_PFC_R8A7796
         depends on ARCH_R8A7796
         select PINCTRL_SH_PFC
 
+config PINCTRL_PFC_R8A77995
+        def_bool y
+        depends on ARCH_R8A77995
+        select PINCTRL_SH_PFC
+
 config PINCTRL_PFC_SH7203
 	def_bool y
 	depends on CPU_SUBTYPE_SH7203
diff --git a/drivers/pinctrl/sh-pfc/Makefile b/drivers/pinctrl/sh-pfc/Makefile
index 33d28eed9ba3..1d4f05a96bd4 100644
--- a/drivers/pinctrl/sh-pfc/Makefile
+++ b/drivers/pinctrl/sh-pfc/Makefile
@@ -15,6 +15,7 @@ obj-$(CONFIG_PINCTRL_PFC_R8A7794)	+= pfc-r8a7794.o
 obj-$(CONFIG_PINCTRL_PFC_R8A7795)	+= pfc-r8a7795.o
 obj-$(CONFIG_PINCTRL_PFC_R8A7795)	+= pfc-r8a7795-es1.o
 obj-$(CONFIG_PINCTRL_PFC_R8A7796)	+= pfc-r8a7796.o
+obj-$(CONFIG_PINCTRL_PFC_R8A77995)	+= pfc-r8a77995.o
 obj-$(CONFIG_PINCTRL_PFC_SH7203)	+= pfc-sh7203.o
 obj-$(CONFIG_PINCTRL_PFC_SH7264)	+= pfc-sh7264.o
 obj-$(CONFIG_PINCTRL_PFC_SH7269)	+= pfc-sh7269.o
diff --git a/drivers/pinctrl/sh-pfc/core.c b/drivers/pinctrl/sh-pfc/core.c
index e72391d5e57d..0c5e952461fd 100644
--- a/drivers/pinctrl/sh-pfc/core.c
+++ b/drivers/pinctrl/sh-pfc/core.c
@@ -551,6 +551,12 @@ static const struct of_device_id sh_pfc_of_table[] = {
 		.data = &r8a7796_pinmux_info,
 	},
 #endif
+#ifdef CONFIG_PINCTRL_PFC_R8A77995
+	{
+		.compatible = "renesas,pfc-r8a77995",
+		.data = &r8a77995_pinmux_info,
+	},
+#endif
 #ifdef CONFIG_PINCTRL_PFC_SH73A0
 	{
 		.compatible = "renesas,pfc-sh73a0",
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7791.c b/drivers/pinctrl/sh-pfc/pfc-r8a7791.c
index 4c5ffbd75be7..10bd35f8c894 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a7791.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a7791.c
@@ -2589,6 +2589,17 @@ static const unsigned int mmc_data8_mux[] = {
 	MMC_D0_MARK, MMC_D1_MARK, MMC_D2_MARK, MMC_D3_MARK,
 	MMC_D4_MARK, MMC_D5_MARK, MMC_D6_MARK, MMC_D7_MARK,
 };
+static const unsigned int mmc_data8_b_pins[] = {
+	/* D[0:7] */
+	RCAR_GP_PIN(6, 18), RCAR_GP_PIN(6, 19),
+	RCAR_GP_PIN(6, 20), RCAR_GP_PIN(6, 21),
+	RCAR_GP_PIN(6, 22), RCAR_GP_PIN(6, 23),
+	RCAR_GP_PIN(6, 6), RCAR_GP_PIN(6, 7),
+};
+static const unsigned int mmc_data8_b_mux[] = {
+	MMC_D0_MARK, MMC_D1_MARK, MMC_D2_MARK, MMC_D3_MARK,
+	MMC_D4_MARK, MMC_D5_MARK, MMC_D6_B_MARK, MMC_D7_B_MARK,
+};
 static const unsigned int mmc_ctrl_pins[] = {
 	/* CLK, CMD */
 	RCAR_GP_PIN(6, 16), RCAR_GP_PIN(6, 17),
@@ -4420,7 +4431,7 @@ static const unsigned int vin2_clk_mux[] = {
 };
 
 static const struct {
-	struct sh_pfc_pin_group common[341];
+	struct sh_pfc_pin_group common[342];
 	struct sh_pfc_pin_group r8a779x[9];
 } pinmux_groups = {
 	.common = {
@@ -4523,6 +4534,7 @@ static const struct {
 		SH_PFC_PIN_GROUP(mmc_data1),
 		SH_PFC_PIN_GROUP(mmc_data4),
 		SH_PFC_PIN_GROUP(mmc_data8),
+		SH_PFC_PIN_GROUP(mmc_data8_b),
 		SH_PFC_PIN_GROUP(mmc_ctrl),
 		SH_PFC_PIN_GROUP(msiof0_clk),
 		SH_PFC_PIN_GROUP(msiof0_sync),
@@ -4955,6 +4967,7 @@ static const char * const mmc_groups[] = {
 	"mmc_data1",
 	"mmc_data4",
 	"mmc_data8",
+	"mmc_data8_b",
 	"mmc_ctrl",
 };
 
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7795.c b/drivers/pinctrl/sh-pfc/pfc-r8a7795.c
index 1656295af2b0..8b35772cda98 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a7795.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a7795.c
@@ -61,7 +61,7 @@
 #define GPSR1_24	F_(RD_WR_N,		IP4_31_28)
 #define GPSR1_23	F_(RD_N,		IP4_27_24)
 #define GPSR1_22	F_(BS_N,		IP4_23_20)
-#define GPSR1_21	F_(CS1_N_A26,		IP4_19_16)
+#define GPSR1_21	F_(CS1_N,		IP4_19_16)
 #define GPSR1_20	F_(CS0_N,		IP4_15_12)
 #define GPSR1_19	F_(A19,			IP4_11_8)
 #define GPSR1_18	F_(A18,			IP4_7_4)
@@ -168,8 +168,8 @@
 #define GPSR5_0		F_(SCK0,		IP11_27_24)
 
 /* GPSR6 */
-#define GPSR6_31	F_(USB3_OVC,		IP18_7_4)
-#define GPSR6_30	F_(USB3_PWEN,		IP18_3_0)
+#define GPSR6_31	F_(USB2_CH3_OVC,	IP18_7_4)
+#define GPSR6_30	F_(USB2_CH3_PWEN,	IP18_3_0)
 #define GPSR6_29	F_(USB30_OVC,		IP17_31_28)
 #define GPSR6_28	F_(USB30_PWEN,		IP17_27_24)
 #define GPSR6_27	F_(USB1_OVC,		IP17_23_20)
@@ -215,8 +215,8 @@
 #define IP0_15_12	FM(AVB_LINK)		F_(0, 0)	FM(MSIOF2_SCK_C)	FM(TX4_A)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP0_19_16	FM(AVB_AVTP_MATCH_A)	F_(0, 0)	FM(MSIOF2_RXD_C)	FM(CTS4_N_A)			F_(0, 0)	FM(FSCLKST2_N_A) F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP0_23_20	FM(AVB_AVTP_CAPTURE_A)	F_(0, 0)	FM(MSIOF2_TXD_C)	FM(RTS4_N_TANS_A)		F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP0_27_24	FM(IRQ0)		FM(QPOLB)	F_(0, 0)		FM(DU_CDE)			FM(VI4_DATA0_B) FM(CAN0_TX_B)	FM(CANFD0_TX_B)		FM(MSIOF3_SS1_E) F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP0_31_28	FM(IRQ1)		FM(QPOLA)	F_(0, 0)		FM(DU_DISP)			FM(VI4_DATA1_B) FM(CAN0_RX_B)	FM(CANFD0_RX_B)		FM(MSIOF3_SS2_E) F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_27_24	FM(IRQ0)		FM(QPOLB)	F_(0, 0)		FM(DU_CDE)			FM(VI4_DATA0_B) FM(CAN0_TX_B)	FM(CANFD0_TX_B)		FM(MSIOF3_SS2_E) F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_31_28	FM(IRQ1)		FM(QPOLA)	F_(0, 0)		FM(DU_DISP)			FM(VI4_DATA1_B) FM(CAN0_RX_B)	FM(CANFD0_RX_B)		FM(MSIOF3_SS1_E) F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP1_3_0		FM(IRQ2)		FM(QCPV_QDE)	F_(0, 0)		FM(DU_EXODDF_DU_ODDF_DISP_CDE)	FM(VI4_DATA2_B) F_(0, 0)	F_(0, 0)		FM(MSIOF3_SYNC_E) F_(0, 0)		FM(PWM3_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP1_7_4		FM(IRQ3)		FM(QSTVB_QVE)	FM(A25)			FM(DU_DOTCLKOUT1)		FM(VI4_DATA3_B) F_(0, 0)	F_(0, 0)		FM(MSIOF3_SCK_E) F_(0, 0)		FM(PWM4_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP1_11_8	FM(IRQ4)		FM(QSTH_QHS)	FM(A24)			FM(DU_EXHSYNC_DU_HSYNC)		FM(VI4_DATA4_B) F_(0, 0)	F_(0, 0)		FM(MSIOF3_RXD_E) F_(0, 0)		FM(PWM5_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
@@ -247,7 +247,7 @@
 #define IP4_7_4		FM(A18)			FM(LCDOUT10)	F_(0, 0)		F_(0, 0)			FM(VI4_HSYNC_N)	F_(0, 0)	FM(DU_DG2)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP4_11_8	FM(A19)			FM(LCDOUT11)	F_(0, 0)		F_(0, 0)			FM(VI4_CLKENB)	F_(0, 0)	FM(DU_DG3)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP4_15_12	FM(CS0_N)		F_(0, 0)	F_(0, 0)		F_(0, 0)			F_(0, 0)	FM(VI5_CLKENB)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP4_19_16	FM(CS1_N_A26)		F_(0, 0)	F_(0, 0)		F_(0, 0)			F_(0, 0)	FM(VI5_CLK)	F_(0, 0)		FM(EX_WAIT0_B)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_19_16	FM(CS1_N)		F_(0, 0)	F_(0, 0)		F_(0, 0)			F_(0, 0)	FM(VI5_CLK)	F_(0, 0)		FM(EX_WAIT0_B)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP4_23_20	FM(BS_N)		FM(QSTVA_QVS)	FM(MSIOF3_SCK_D)	FM(SCK3)			FM(HSCK3)	F_(0, 0)	F_(0, 0)		F_(0, 0)	FM(CAN1_TX)		FM(CANFD1_TX)	FM(IETX_A)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP4_27_24	FM(RD_N)		F_(0, 0)	FM(MSIOF3_SYNC_D)	FM(RX3_A)			FM(HRX3_A)	F_(0, 0)	F_(0, 0)		F_(0, 0)	FM(CAN0_TX_A)		FM(CANFD0_TX_A)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP4_31_28	FM(RD_WR_N)		F_(0, 0)	FM(MSIOF3_RXD_D)	FM(TX3_A)			FM(HTX3_A)	F_(0, 0)	F_(0, 0)		F_(0, 0)	FM(CAN0_RX_A)		FM(CANFD0_RX_A)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
@@ -270,7 +270,6 @@
 #define IP7_3_0		FM(D13)			FM(LCDOUT5)	FM(MSIOF2_SS2_D)	FM(TX4_C)			FM(VI4_DATA5_A)	F_(0, 0)	FM(DU_DR5)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP7_7_4		FM(D14)			FM(LCDOUT6)	FM(MSIOF3_SS1_A)	FM(HRX3_C)			FM(VI4_DATA6_A)	F_(0, 0)	FM(DU_DR6)		FM(SCL6_C)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP7_11_8	FM(D15)			FM(LCDOUT7)	FM(MSIOF3_SS2_A)	FM(HTX3_C)			FM(VI4_DATA7_A)	F_(0, 0)	FM(DU_DR7)		FM(SDA6_C)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP7_15_12	FM(FSCLKST)		F_(0, 0)	F_(0, 0)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP7_19_16	FM(SD0_CLK)		F_(0, 0)	FM(MSIOF1_SCK_E)	F_(0, 0)			F_(0, 0)	F_(0, 0)	FM(STP_OPWM_0_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 
 /* IPSRx */		/* 0 */			/* 1 */		/* 2 */			/* 3 */				/* 4 */		/* 5 */		/* 6 */			/* 7 */		/* 8 */			/* 9 */		/* A */		/* B */		/* C - F */
@@ -285,24 +284,24 @@
 #define IP8_23_20	FM(SD1_DAT1)		FM(SD2_DAT5)	FM(MSIOF1_TXD_G)	FM(NFDATA14_B)			F_(0, 0)	FM(TS_SPSYNC1_B)FM(STP_ISSYNC_1_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP8_27_24	FM(SD1_DAT2)		FM(SD2_DAT6)	FM(MSIOF1_SS1_G)	FM(NFDATA15_B)			F_(0, 0)	FM(TS_SDAT1_B)	FM(STP_ISD_1_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP8_31_28	FM(SD1_DAT3)		FM(SD2_DAT7)	FM(MSIOF1_SS2_G)	FM(NFRB_N_B)			F_(0, 0)	FM(TS_SDEN1_B)	FM(STP_ISEN_1_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP9_3_0		FM(SD2_CLK)		F_(0, 0)	F_(0, 0)		FM(NFDATA8)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP9_7_4		FM(SD2_CMD)		F_(0, 0)	F_(0, 0)		FM(NFDATA9)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP9_11_8	FM(SD2_DAT0)		F_(0, 0)	F_(0, 0)		FM(NFDATA10)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP9_15_12	FM(SD2_DAT1)		F_(0, 0)	F_(0, 0)		FM(NFDATA11)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP9_19_16	FM(SD2_DAT2)		F_(0, 0)	F_(0, 0)		FM(NFDATA12)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP9_23_20	FM(SD2_DAT3)		F_(0, 0)	F_(0, 0)		FM(NFDATA13)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP9_27_24	FM(SD2_DS)		F_(0, 0)	F_(0, 0)		FM(NFALE)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	FM(SATA_DEVSLP_B)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP9_31_28	FM(SD3_CLK)		F_(0, 0)	F_(0, 0)		FM(NFWE_N)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP10_3_0	FM(SD3_CMD)		F_(0, 0)	F_(0, 0)		FM(NFRE_N)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP10_7_4	FM(SD3_DAT0)		F_(0, 0)	F_(0, 0)		FM(NFDATA0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP10_11_8	FM(SD3_DAT1)		F_(0, 0)	F_(0, 0)		FM(NFDATA1)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP10_15_12	FM(SD3_DAT2)		F_(0, 0)	F_(0, 0)		FM(NFDATA2)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP10_19_16	FM(SD3_DAT3)		F_(0, 0)	F_(0, 0)		FM(NFDATA3)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP10_23_20	FM(SD3_DAT4)		FM(SD2_CD_A)	F_(0, 0)		FM(NFDATA4)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP10_27_24	FM(SD3_DAT5)		FM(SD2_WP_A)	F_(0, 0)		FM(NFDATA5)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP10_31_28	FM(SD3_DAT6)		FM(SD3_CD)	F_(0, 0)		FM(NFDATA6)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP11_3_0	FM(SD3_DAT7)		FM(SD3_WP)	F_(0, 0)		FM(NFDATA7)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP11_7_4	FM(SD3_DS)		F_(0, 0)	F_(0, 0)		FM(NFCLE)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_3_0		FM(SD2_CLK)		F_(0, 0)	FM(NFDATA8)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_7_4		FM(SD2_CMD)		F_(0, 0)	FM(NFDATA9)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_11_8	FM(SD2_DAT0)		F_(0, 0)	FM(NFDATA10)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_15_12	FM(SD2_DAT1)		F_(0, 0)	FM(NFDATA11)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_19_16	FM(SD2_DAT2)		F_(0, 0)	FM(NFDATA12)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_23_20	FM(SD2_DAT3)		F_(0, 0)	FM(NFDATA13)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_27_24	FM(SD2_DS)		F_(0, 0)	FM(NFALE)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	FM(SATA_DEVSLP_B)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_31_28	FM(SD3_CLK)		F_(0, 0)	FM(NFWE_N)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_3_0	FM(SD3_CMD)		F_(0, 0)	FM(NFRE_N)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_7_4	FM(SD3_DAT0)		F_(0, 0)	FM(NFDATA0)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_11_8	FM(SD3_DAT1)		F_(0, 0)	FM(NFDATA1)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_15_12	FM(SD3_DAT2)		F_(0, 0)	FM(NFDATA2)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_19_16	FM(SD3_DAT3)		F_(0, 0)	FM(NFDATA3)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_23_20	FM(SD3_DAT4)		FM(SD2_CD_A)	FM(NFDATA4)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_27_24	FM(SD3_DAT5)		FM(SD2_WP_A)	FM(NFDATA5)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_31_28	FM(SD3_DAT6)		FM(SD3_CD)	FM(NFDATA6)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_3_0	FM(SD3_DAT7)		FM(SD3_WP)	FM(NFDATA7)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_7_4	FM(SD3_DS)		F_(0, 0)	FM(NFCLE)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP11_11_8	FM(SD0_CD)		F_(0, 0)	FM(NFDATA14_A)		F_(0, 0)			FM(SCL2_B)	FM(SIM0_RST_A)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 
 /* IPSRx */		/* 0 */			/* 1 */		/* 2 */			/* 3 */				/* 4 */		/* 5 */		/* 6 */			/* 7 */		/* 8 */			/* 9 */		/* A */		/* B */		/* C - F */
@@ -361,8 +360,8 @@
 #define IP17_23_20	FM(USB1_OVC)		F_(0, 0)	FM(MSIOF1_SS2_C)	F_(0, 0)			FM(SSI_WS1_A)	FM(TS_SDAT0_E)	FM(STP_ISD_0_E)		FM(FMIN_B)	FM(RIF2_SYNC_B)		F_(0, 0)	FM(REMOCON_B)	F_(0, 0)	F_(0, 0) FM(HCTS2_N_C) F_(0, 0) F_(0, 0)
 #define IP17_27_24	FM(USB30_PWEN)		F_(0, 0)	F_(0, 0)		FM(AUDIO_CLKOUT_B)		FM(SSI_SCK2_B)	FM(TS_SDEN1_D)	FM(STP_ISEN_1_D)	FM(STP_OPWM_0_E)FM(RIF3_D0_B)		F_(0, 0)	FM(TCLK2_B)	FM(TPU0TO0)	FM(BPFCLK_C) FM(HRTS2_N_C) F_(0, 0) F_(0, 0)
 #define IP17_31_28	FM(USB30_OVC)		F_(0, 0)	F_(0, 0)		FM(AUDIO_CLKOUT1_B)		FM(SSI_WS2_B)	FM(TS_SPSYNC1_D)FM(STP_ISSYNC_1_D)	FM(STP_IVCXO27_0_E)FM(RIF3_D1_B)	F_(0, 0)	FM(FSO_TOE_N)	FM(TPU0TO1)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP18_3_0	FM(USB3_PWEN)		F_(0, 0)	F_(0, 0)		FM(AUDIO_CLKOUT2_B)		FM(SSI_SCK9_B)	FM(TS_SDEN0_E)	FM(STP_ISEN_0_E)	F_(0, 0)	FM(RIF2_D0_B)		F_(0, 0)	F_(0, 0)	FM(TPU0TO2)	F_(0, 0) FM(FMCLK_C) FM(FMCLK_D) F_(0, 0)
-#define IP18_7_4	FM(USB3_OVC)		F_(0, 0)	F_(0, 0)		FM(AUDIO_CLKOUT3_B)		FM(SSI_WS9_B)	FM(TS_SPSYNC0_E)FM(STP_ISSYNC_0_E)	F_(0, 0)	FM(RIF2_D1_B)		F_(0, 0)	F_(0, 0)	FM(TPU0TO3)	F_(0, 0) FM(FMIN_C) FM(FMIN_D) F_(0, 0)
+#define IP18_3_0	FM(USB2_CH3_PWEN)	F_(0, 0)	F_(0, 0)		FM(AUDIO_CLKOUT2_B)		FM(SSI_SCK9_B)	FM(TS_SDEN0_E)	FM(STP_ISEN_0_E)	F_(0, 0)	FM(RIF2_D0_B)		F_(0, 0)	F_(0, 0)	FM(TPU0TO2)	FM(FMCLK_C) FM(FMCLK_D) F_(0, 0) F_(0, 0)
+#define IP18_7_4	FM(USB2_CH3_OVC)	F_(0, 0)	F_(0, 0)		FM(AUDIO_CLKOUT3_B)		FM(SSI_WS9_B)	FM(TS_SPSYNC0_E)FM(STP_ISSYNC_0_E)	F_(0, 0)	FM(RIF2_D1_B)		F_(0, 0)	F_(0, 0)	FM(TPU0TO3)	FM(FMIN_C) FM(FMIN_D) F_(0, 0) F_(0, 0)
 
 #define PINMUX_GPSR	\
 \
@@ -413,7 +412,7 @@ FM(IP0_31_28)	IP0_31_28	FM(IP1_31_28)	IP1_31_28	FM(IP2_31_28)	IP2_31_28	FM(IP3_3
 FM(IP4_3_0)	IP4_3_0		FM(IP5_3_0)	IP5_3_0		FM(IP6_3_0)	IP6_3_0		FM(IP7_3_0)	IP7_3_0 \
 FM(IP4_7_4)	IP4_7_4		FM(IP5_7_4)	IP5_7_4		FM(IP6_7_4)	IP6_7_4		FM(IP7_7_4)	IP7_7_4 \
 FM(IP4_11_8)	IP4_11_8	FM(IP5_11_8)	IP5_11_8	FM(IP6_11_8)	IP6_11_8	FM(IP7_11_8)	IP7_11_8 \
-FM(IP4_15_12)	IP4_15_12	FM(IP5_15_12)	IP5_15_12	FM(IP6_15_12)	IP6_15_12	FM(IP7_15_12)	IP7_15_12 \
+FM(IP4_15_12)	IP4_15_12	FM(IP5_15_12)	IP5_15_12	FM(IP6_15_12)	IP6_15_12 \
 FM(IP4_19_16)	IP4_19_16	FM(IP5_19_16)	IP5_19_16	FM(IP6_19_16)	IP6_19_16	FM(IP7_19_16)	IP7_19_16 \
 FM(IP4_23_20)	IP4_23_20	FM(IP5_23_20)	IP5_23_20	FM(IP6_23_20)	IP6_23_20	FM(IP7_23_20)	IP7_23_20 \
 FM(IP4_27_24)	IP4_27_24	FM(IP5_27_24)	IP5_27_24	FM(IP6_27_24)	IP6_27_24	FM(IP7_27_24)	IP7_27_24 \
@@ -469,7 +468,7 @@ FM(IP16_31_28)	IP16_31_28	FM(IP17_31_28)	IP17_31_28
 /* MOD_SEL1 */			/* 0 */			/* 1 */			/* 2 */			/* 3 */			/* 4 */			/* 5 */			/* 6 */			/* 7 */
 #define MOD_SEL1_31_30		FM(SEL_TSIF1_0)		FM(SEL_TSIF1_1)		FM(SEL_TSIF1_2)		FM(SEL_TSIF1_3)
 #define MOD_SEL1_29_28_27	FM(SEL_TSIF0_0)		FM(SEL_TSIF0_1)		FM(SEL_TSIF0_2)		FM(SEL_TSIF0_3)		FM(SEL_TSIF0_4)		F_(0, 0)		F_(0, 0)		F_(0, 0)
-#define MOD_SEL1_26		FM(SEL_TIMER_TMU_0)	FM(SEL_TIMER_TMU_1)
+#define MOD_SEL1_26		FM(SEL_TIMER_TMU1_0)	FM(SEL_TIMER_TMU1_1)
 #define MOD_SEL1_25_24		FM(SEL_SSP1_1_0)	FM(SEL_SSP1_1_1)	FM(SEL_SSP1_1_2)	FM(SEL_SSP1_1_3)
 #define MOD_SEL1_23_22_21	FM(SEL_SSP1_0_0)	FM(SEL_SSP1_0_1)	FM(SEL_SSP1_0_2)	FM(SEL_SSP1_0_3)	FM(SEL_SSP1_0_4)	F_(0, 0)		F_(0, 0)		F_(0, 0)
 #define MOD_SEL1_20		FM(SEL_SSI_0)		FM(SEL_SSI_1)
@@ -480,7 +479,7 @@ FM(IP16_31_28)	IP16_31_28	FM(IP17_31_28)	IP17_31_28
 #define MOD_SEL1_13		FM(SEL_SCIF3_0)		FM(SEL_SCIF3_1)
 #define MOD_SEL1_12		FM(SEL_SCIF2_0)		FM(SEL_SCIF2_1)
 #define MOD_SEL1_11		FM(SEL_SCIF1_0)		FM(SEL_SCIF1_1)
-#define MOD_SEL1_10		FM(SEL_SATA_0)		FM(SEL_SATA_1)
+#define MOD_SEL1_10		FM(SEL_SCIF_0)		FM(SEL_SCIF_1)
 #define MOD_SEL1_9		FM(SEL_REMOCON_0)	FM(SEL_REMOCON_1)
 #define MOD_SEL1_6		FM(SEL_RCAN0_0)		FM(SEL_RCAN0_1)
 #define MOD_SEL1_5		FM(SEL_PWM6_0)		FM(SEL_PWM6_1)
@@ -497,7 +496,6 @@ FM(IP16_31_28)	IP16_31_28	FM(IP17_31_28)	IP17_31_28
 #define MOD_SEL2_28_27		FM(SEL_FM_0)		FM(SEL_FM_1)		FM(SEL_FM_2)		FM(SEL_FM_3)
 #define MOD_SEL2_26		FM(SEL_SCIF5_0)		FM(SEL_SCIF5_1)
 #define MOD_SEL2_25_24_23	FM(SEL_I2C6_0)		FM(SEL_I2C6_1)		FM(SEL_I2C6_2)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)
-#define MOD_SEL2_22		FM(SEL_NDF_0)		FM(SEL_NDF_1)
 #define MOD_SEL2_21		FM(SEL_SSI2_0)		FM(SEL_SSI2_1)
 #define MOD_SEL2_20		FM(SEL_SSI9_0)		FM(SEL_SSI9_1)
 #define MOD_SEL2_19		FM(SEL_TIMER_TMU2_0)	FM(SEL_TIMER_TMU2_1)
@@ -514,7 +512,7 @@ MOD_SEL0_28_27					MOD_SEL2_28_27 \
 MOD_SEL0_26_25_24	MOD_SEL1_26		MOD_SEL2_26 \
 			MOD_SEL1_25_24		MOD_SEL2_25_24_23 \
 MOD_SEL0_23		MOD_SEL1_23_22_21 \
-MOD_SEL0_22					MOD_SEL2_22 \
+MOD_SEL0_22 \
 MOD_SEL0_21					MOD_SEL2_21 \
 MOD_SEL0_20		MOD_SEL1_20		MOD_SEL2_20 \
 MOD_SEL0_19		MOD_SEL1_19		MOD_SEL2_19 \
@@ -833,7 +831,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP4_15_12,	CS0_N),
 	PINMUX_IPSR_GPSR(IP4_15_12,	VI5_CLKENB),
 
-	PINMUX_IPSR_GPSR(IP4_19_16,	CS1_N_A26),
+	PINMUX_IPSR_GPSR(IP4_19_16,	CS1_N),
 	PINMUX_IPSR_GPSR(IP4_19_16,	VI5_CLK),
 	PINMUX_IPSR_MSEL(IP4_19_16,	EX_WAIT0_B,		SEL_LBSC_1),
 
@@ -986,8 +984,6 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP7_11_8,	DU_DR7),
 	PINMUX_IPSR_MSEL(IP7_11_8,	SDA6_C,			SEL_I2C6_2),
 
-	PINMUX_IPSR_GPSR(IP7_15_12,	FSCLKST),
-
 	PINMUX_IPSR_GPSR(IP7_19_16,	SD0_CLK),
 	PINMUX_IPSR_MSEL(IP7_19_16,	MSIOF1_SCK_E,		SEL_MSIOF1_4),
 	PINMUX_IPSR_MSEL(IP7_19_16,	STP_OPWM_0_B,		SEL_SSP1_0_1),
@@ -1023,35 +1019,35 @@ static const u16 pinmux_data[] = {
 
 	PINMUX_IPSR_GPSR(IP8_15_12,	SD1_CMD),
 	PINMUX_IPSR_MSEL(IP8_15_12,	MSIOF1_SYNC_G,		SEL_MSIOF1_6),
-	PINMUX_IPSR_MSEL(IP8_15_12,	NFCE_N_B,		SEL_NDF_1),
+	PINMUX_IPSR_GPSR(IP8_15_12,	NFCE_N_B),
 	PINMUX_IPSR_MSEL(IP8_15_12,	SIM0_D_A,		SEL_SIMCARD_0),
 	PINMUX_IPSR_MSEL(IP8_15_12,	STP_IVCXO27_1_B,	SEL_SSP1_1_1),
 
 	PINMUX_IPSR_GPSR(IP8_19_16,	SD1_DAT0),
 	PINMUX_IPSR_GPSR(IP8_19_16,	SD2_DAT4),
 	PINMUX_IPSR_MSEL(IP8_19_16,	MSIOF1_RXD_G,		SEL_MSIOF1_6),
-	PINMUX_IPSR_MSEL(IP8_19_16,	NFWP_N_B,		SEL_NDF_1),
+	PINMUX_IPSR_GPSR(IP8_19_16,	NFWP_N_B),
 	PINMUX_IPSR_MSEL(IP8_19_16,	TS_SCK1_B,		SEL_TSIF1_1),
 	PINMUX_IPSR_MSEL(IP8_19_16,	STP_ISCLK_1_B,		SEL_SSP1_1_1),
 
 	PINMUX_IPSR_GPSR(IP8_23_20,	SD1_DAT1),
 	PINMUX_IPSR_GPSR(IP8_23_20,	SD2_DAT5),
 	PINMUX_IPSR_MSEL(IP8_23_20,	MSIOF1_TXD_G,		SEL_MSIOF1_6),
-	PINMUX_IPSR_MSEL(IP8_23_20,	NFDATA14_B,		SEL_NDF_1),
+	PINMUX_IPSR_GPSR(IP8_23_20,	NFDATA14_B),
 	PINMUX_IPSR_MSEL(IP8_23_20,	TS_SPSYNC1_B,		SEL_TSIF1_1),
 	PINMUX_IPSR_MSEL(IP8_23_20,	STP_ISSYNC_1_B,		SEL_SSP1_1_1),
 
 	PINMUX_IPSR_GPSR(IP8_27_24,	SD1_DAT2),
 	PINMUX_IPSR_GPSR(IP8_27_24,	SD2_DAT6),
 	PINMUX_IPSR_MSEL(IP8_27_24,	MSIOF1_SS1_G,		SEL_MSIOF1_6),
-	PINMUX_IPSR_MSEL(IP8_27_24,	NFDATA15_B,		SEL_NDF_1),
+	PINMUX_IPSR_GPSR(IP8_27_24,	NFDATA15_B),
 	PINMUX_IPSR_MSEL(IP8_27_24,	TS_SDAT1_B,		SEL_TSIF1_1),
 	PINMUX_IPSR_MSEL(IP8_27_24,	STP_ISD_1_B,		SEL_SSP1_1_1),
 
 	PINMUX_IPSR_GPSR(IP8_31_28,	SD1_DAT3),
 	PINMUX_IPSR_GPSR(IP8_31_28,	SD2_DAT7),
 	PINMUX_IPSR_MSEL(IP8_31_28,	MSIOF1_SS2_G,		SEL_MSIOF1_6),
-	PINMUX_IPSR_MSEL(IP8_31_28,	NFRB_N_B,		SEL_NDF_1),
+	PINMUX_IPSR_GPSR(IP8_31_28,	NFRB_N_B),
 	PINMUX_IPSR_MSEL(IP8_31_28,	TS_SDEN1_B,		SEL_TSIF1_1),
 	PINMUX_IPSR_MSEL(IP8_31_28,	STP_ISEN_1_B,		SEL_SSP1_1_1),
 
@@ -1201,7 +1197,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP12_27_24,	ADICHS0),
 
 	PINMUX_IPSR_GPSR(IP12_31_28,	SCK2),
-	PINMUX_IPSR_MSEL(IP12_31_28,	SCIF_CLK_B,		SEL_SCIF1_1),
+	PINMUX_IPSR_MSEL(IP12_31_28,	SCIF_CLK_B,		SEL_SCIF_1),
 	PINMUX_IPSR_MSEL(IP12_31_28,	MSIOF1_SCK_B,		SEL_MSIOF1_1),
 	PINMUX_IPSR_MSEL(IP12_31_28,	TS_SCK1_C,		SEL_TSIF1_2),
 	PINMUX_IPSR_MSEL(IP12_31_28,	STP_ISCLK_1_C,		SEL_SSP1_1_2),
@@ -1271,12 +1267,12 @@ static const u16 pinmux_data[] = {
 	/* IPSR14 */
 	PINMUX_IPSR_GPSR(IP14_3_0,	MSIOF0_SS1),
 	PINMUX_IPSR_MSEL(IP14_3_0,	RX5_A,			SEL_SCIF5_0),
-	PINMUX_IPSR_MSEL(IP14_3_0,	NFWP_N_A,		SEL_NDF_0),
+	PINMUX_IPSR_GPSR(IP14_3_0,	NFWP_N_A),
 	PINMUX_IPSR_MSEL(IP14_3_0,	AUDIO_CLKA_C,		SEL_ADG_A_2),
 	PINMUX_IPSR_MSEL(IP14_3_0,	SSI_SCK2_A,		SEL_SSI_0),
 	PINMUX_IPSR_MSEL(IP14_3_0,	STP_IVCXO27_0_C,	SEL_SSP1_0_2),
 	PINMUX_IPSR_GPSR(IP14_3_0,	AUDIO_CLKOUT3_A),
-	PINMUX_IPSR_MSEL(IP14_3_0,	TCLK1_B,		SEL_TIMER_TMU_1),
+	PINMUX_IPSR_MSEL(IP14_3_0,	TCLK1_B,		SEL_TIMER_TMU1_1),
 
 	PINMUX_IPSR_GPSR(IP14_7_4,	MSIOF0_SS2),
 	PINMUX_IPSR_MSEL(IP14_7_4,	TX5_A,			SEL_SCIF5_0),
@@ -1392,7 +1388,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_MSEL(IP16_23_20,	STP_ISEN_1_A,		SEL_SSP1_1_0),
 	PINMUX_IPSR_MSEL(IP16_23_20,	RIF1_D0_A,		SEL_DRIF1_0),
 	PINMUX_IPSR_MSEL(IP16_23_20,	RIF3_D0_A,		SEL_DRIF3_0),
-	PINMUX_IPSR_MSEL(IP16_23_20,	TCLK2_A,		SEL_TIMER_TMU_0),
+	PINMUX_IPSR_MSEL(IP16_23_20,	TCLK2_A,		SEL_TIMER_TMU2_0),
 
 	PINMUX_IPSR_GPSR(IP16_27_24,	SSI_SDATA8),
 	PINMUX_IPSR_MSEL(IP16_27_24,	HRTS2_N_B,		SEL_HSCIF2_1),
@@ -1409,17 +1405,17 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_MSEL(IP16_31_28,	SSI_WS1_B,		SEL_SSI_1),
 	PINMUX_IPSR_GPSR(IP16_31_28,	SCK1),
 	PINMUX_IPSR_MSEL(IP16_31_28,	STP_IVCXO27_1_A,	SEL_SSP1_1_0),
-	PINMUX_IPSR_GPSR(IP16_31_28,	SCK5_A),
+	PINMUX_IPSR_MSEL(IP16_31_28,	SCK5_A,			SEL_SCIF5_0),
 
 	/* IPSR17 */
 	PINMUX_IPSR_MSEL(IP17_3_0,	AUDIO_CLKA_A,		SEL_ADG_A_0),
 	PINMUX_IPSR_GPSR(IP17_3_0,	CC5_OSCOUT),
 
 	PINMUX_IPSR_MSEL(IP17_7_4,	AUDIO_CLKB_B,		SEL_ADG_B_1),
-	PINMUX_IPSR_MSEL(IP17_7_4,	SCIF_CLK_A,		SEL_SCIF1_0),
+	PINMUX_IPSR_MSEL(IP17_7_4,	SCIF_CLK_A,		SEL_SCIF_0),
 	PINMUX_IPSR_MSEL(IP17_7_4,	STP_IVCXO27_1_D,	SEL_SSP1_1_3),
 	PINMUX_IPSR_MSEL(IP17_7_4,	REMOCON_A,		SEL_REMOCON_0),
-	PINMUX_IPSR_MSEL(IP17_7_4,	TCLK1_A,		SEL_TIMER_TMU_0),
+	PINMUX_IPSR_MSEL(IP17_7_4,	TCLK1_A,		SEL_TIMER_TMU1_0),
 
 	PINMUX_IPSR_GPSR(IP17_11_8,	USB0_PWEN),
 	PINMUX_IPSR_MSEL(IP17_11_8,	SIM0_RST_C,		SEL_SIMCARD_2),
@@ -1460,10 +1456,10 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP17_27_24,	AUDIO_CLKOUT_B),
 	PINMUX_IPSR_MSEL(IP17_27_24,	SSI_SCK2_B,		SEL_SSI_1),
 	PINMUX_IPSR_MSEL(IP17_27_24,	TS_SDEN1_D,		SEL_TSIF1_3),
-	PINMUX_IPSR_MSEL(IP17_27_24,	STP_ISEN_1_D,		SEL_SSP1_1_2),
+	PINMUX_IPSR_MSEL(IP17_27_24,	STP_ISEN_1_D,		SEL_SSP1_1_3),
 	PINMUX_IPSR_MSEL(IP17_27_24,	STP_OPWM_0_E,		SEL_SSP1_0_4),
 	PINMUX_IPSR_MSEL(IP17_27_24,	RIF3_D0_B,		SEL_DRIF3_1),
-	PINMUX_IPSR_MSEL(IP17_27_24,	TCLK2_B,		SEL_TIMER_TMU_1),
+	PINMUX_IPSR_MSEL(IP17_27_24,	TCLK2_B,		SEL_TIMER_TMU2_1),
 	PINMUX_IPSR_GPSR(IP17_27_24,	TPU0TO0),
 	PINMUX_IPSR_MSEL(IP17_27_24,	BPFCLK_C,		SEL_FM_2),
 	PINMUX_IPSR_MSEL(IP17_27_24,	HRTS2_N_C,		SEL_HSCIF2_2),
@@ -1479,7 +1475,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP17_31_28,	TPU0TO1),
 
 	/* IPSR18 */
-	PINMUX_IPSR_GPSR(IP18_3_0,	USB3_PWEN),
+	PINMUX_IPSR_GPSR(IP18_3_0,	USB2_CH3_PWEN),
 	PINMUX_IPSR_GPSR(IP18_3_0,	AUDIO_CLKOUT2_B),
 	PINMUX_IPSR_MSEL(IP18_3_0,	SSI_SCK9_B,		SEL_SSI_1),
 	PINMUX_IPSR_MSEL(IP18_3_0,	TS_SDEN0_E,		SEL_TSIF0_4),
@@ -1489,7 +1485,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_MSEL(IP18_3_0,	FMCLK_C,		SEL_FM_2),
 	PINMUX_IPSR_MSEL(IP18_3_0,	FMCLK_D,		SEL_FM_3),
 
-	PINMUX_IPSR_GPSR(IP18_7_4,	USB3_OVC),
+	PINMUX_IPSR_GPSR(IP18_7_4,	USB2_CH3_OVC),
 	PINMUX_IPSR_GPSR(IP18_7_4,	AUDIO_CLKOUT3_B),
 	PINMUX_IPSR_MSEL(IP18_7_4,	SSI_WS9_B,		SEL_SSI_1),
 	PINMUX_IPSR_MSEL(IP18_7_4,	TS_SPSYNC0_E,		SEL_TSIF0_4),
@@ -1744,6 +1740,704 @@ static const unsigned int du_disp_mux[] = {
 	DU_DISP_MARK,
 };
 
+/* - MSIOF0 ----------------------------------------------------------------- */
+static const unsigned int msiof0_clk_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(5, 17),
+};
+static const unsigned int msiof0_clk_mux[] = {
+	MSIOF0_SCK_MARK,
+};
+static const unsigned int msiof0_sync_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(5, 18),
+};
+static const unsigned int msiof0_sync_mux[] = {
+	MSIOF0_SYNC_MARK,
+};
+static const unsigned int msiof0_ss1_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(5, 19),
+};
+static const unsigned int msiof0_ss1_mux[] = {
+	MSIOF0_SS1_MARK,
+};
+static const unsigned int msiof0_ss2_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(5, 21),
+};
+static const unsigned int msiof0_ss2_mux[] = {
+	MSIOF0_SS2_MARK,
+};
+static const unsigned int msiof0_txd_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(5, 20),
+};
+static const unsigned int msiof0_txd_mux[] = {
+	MSIOF0_TXD_MARK,
+};
+static const unsigned int msiof0_rxd_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(5, 22),
+};
+static const unsigned int msiof0_rxd_mux[] = {
+	MSIOF0_RXD_MARK,
+};
+/* - MSIOF1 ----------------------------------------------------------------- */
+static const unsigned int msiof1_clk_a_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(6, 8),
+};
+static const unsigned int msiof1_clk_a_mux[] = {
+	MSIOF1_SCK_A_MARK,
+};
+static const unsigned int msiof1_sync_a_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(6, 9),
+};
+static const unsigned int msiof1_sync_a_mux[] = {
+	MSIOF1_SYNC_A_MARK,
+};
+static const unsigned int msiof1_ss1_a_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(6, 5),
+};
+static const unsigned int msiof1_ss1_a_mux[] = {
+	MSIOF1_SS1_A_MARK,
+};
+static const unsigned int msiof1_ss2_a_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(6, 6),
+};
+static const unsigned int msiof1_ss2_a_mux[] = {
+	MSIOF1_SS2_A_MARK,
+};
+static const unsigned int msiof1_txd_a_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(6, 7),
+};
+static const unsigned int msiof1_txd_a_mux[] = {
+	MSIOF1_TXD_A_MARK,
+};
+static const unsigned int msiof1_rxd_a_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(6, 10),
+};
+static const unsigned int msiof1_rxd_a_mux[] = {
+	MSIOF1_RXD_A_MARK,
+};
+static const unsigned int msiof1_clk_b_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(5, 9),
+};
+static const unsigned int msiof1_clk_b_mux[] = {
+	MSIOF1_SCK_B_MARK,
+};
+static const unsigned int msiof1_sync_b_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(5, 3),
+};
+static const unsigned int msiof1_sync_b_mux[] = {
+	MSIOF1_SYNC_B_MARK,
+};
+static const unsigned int msiof1_ss1_b_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(5, 4),
+};
+static const unsigned int msiof1_ss1_b_mux[] = {
+	MSIOF1_SS1_B_MARK,
+};
+static const unsigned int msiof1_ss2_b_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(5, 0),
+};
+static const unsigned int msiof1_ss2_b_mux[] = {
+	MSIOF1_SS2_B_MARK,
+};
+static const unsigned int msiof1_txd_b_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(5, 8),
+};
+static const unsigned int msiof1_txd_b_mux[] = {
+	MSIOF1_TXD_B_MARK,
+};
+static const unsigned int msiof1_rxd_b_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(5, 7),
+};
+static const unsigned int msiof1_rxd_b_mux[] = {
+	MSIOF1_RXD_B_MARK,
+};
+static const unsigned int msiof1_clk_c_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(6, 17),
+};
+static const unsigned int msiof1_clk_c_mux[] = {
+	MSIOF1_SCK_C_MARK,
+};
+static const unsigned int msiof1_sync_c_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(6, 18),
+};
+static const unsigned int msiof1_sync_c_mux[] = {
+	MSIOF1_SYNC_C_MARK,
+};
+static const unsigned int msiof1_ss1_c_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(6, 21),
+};
+static const unsigned int msiof1_ss1_c_mux[] = {
+	MSIOF1_SS1_C_MARK,
+};
+static const unsigned int msiof1_ss2_c_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(6, 27),
+};
+static const unsigned int msiof1_ss2_c_mux[] = {
+	MSIOF1_SS2_C_MARK,
+};
+static const unsigned int msiof1_txd_c_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(6, 20),
+};
+static const unsigned int msiof1_txd_c_mux[] = {
+	MSIOF1_TXD_C_MARK,
+};
+static const unsigned int msiof1_rxd_c_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(6, 19),
+};
+static const unsigned int msiof1_rxd_c_mux[] = {
+	MSIOF1_RXD_C_MARK,
+};
+static const unsigned int msiof1_clk_d_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(5, 12),
+};
+static const unsigned int msiof1_clk_d_mux[] = {
+	MSIOF1_SCK_D_MARK,
+};
+static const unsigned int msiof1_sync_d_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(5, 15),
+};
+static const unsigned int msiof1_sync_d_mux[] = {
+	MSIOF1_SYNC_D_MARK,
+};
+static const unsigned int msiof1_ss1_d_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(5, 16),
+};
+static const unsigned int msiof1_ss1_d_mux[] = {
+	MSIOF1_SS1_D_MARK,
+};
+static const unsigned int msiof1_ss2_d_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(5, 21),
+};
+static const unsigned int msiof1_ss2_d_mux[] = {
+	MSIOF1_SS2_D_MARK,
+};
+static const unsigned int msiof1_txd_d_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(5, 14),
+};
+static const unsigned int msiof1_txd_d_mux[] = {
+	MSIOF1_TXD_D_MARK,
+};
+static const unsigned int msiof1_rxd_d_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(5, 13),
+};
+static const unsigned int msiof1_rxd_d_mux[] = {
+	MSIOF1_RXD_D_MARK,
+};
+static const unsigned int msiof1_clk_e_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(3, 0),
+};
+static const unsigned int msiof1_clk_e_mux[] = {
+	MSIOF1_SCK_E_MARK,
+};
+static const unsigned int msiof1_sync_e_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(3, 1),
+};
+static const unsigned int msiof1_sync_e_mux[] = {
+	MSIOF1_SYNC_E_MARK,
+};
+static const unsigned int msiof1_ss1_e_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(3, 4),
+};
+static const unsigned int msiof1_ss1_e_mux[] = {
+	MSIOF1_SS1_E_MARK,
+};
+static const unsigned int msiof1_ss2_e_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(3, 5),
+};
+static const unsigned int msiof1_ss2_e_mux[] = {
+	MSIOF1_SS2_E_MARK,
+};
+static const unsigned int msiof1_txd_e_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(3, 3),
+};
+static const unsigned int msiof1_txd_e_mux[] = {
+	MSIOF1_TXD_E_MARK,
+};
+static const unsigned int msiof1_rxd_e_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(3, 2),
+};
+static const unsigned int msiof1_rxd_e_mux[] = {
+	MSIOF1_RXD_E_MARK,
+};
+static const unsigned int msiof1_clk_f_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(5, 23),
+};
+static const unsigned int msiof1_clk_f_mux[] = {
+	MSIOF1_SCK_F_MARK,
+};
+static const unsigned int msiof1_sync_f_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(5, 24),
+};
+static const unsigned int msiof1_sync_f_mux[] = {
+	MSIOF1_SYNC_F_MARK,
+};
+static const unsigned int msiof1_ss1_f_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(6, 1),
+};
+static const unsigned int msiof1_ss1_f_mux[] = {
+	MSIOF1_SS1_F_MARK,
+};
+static const unsigned int msiof1_ss2_f_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(6, 2),
+};
+static const unsigned int msiof1_ss2_f_mux[] = {
+	MSIOF1_SS2_F_MARK,
+};
+static const unsigned int msiof1_txd_f_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(6, 0),
+};
+static const unsigned int msiof1_txd_f_mux[] = {
+	MSIOF1_TXD_F_MARK,
+};
+static const unsigned int msiof1_rxd_f_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(5, 25),
+};
+static const unsigned int msiof1_rxd_f_mux[] = {
+	MSIOF1_RXD_F_MARK,
+};
+static const unsigned int msiof1_clk_g_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(3, 6),
+};
+static const unsigned int msiof1_clk_g_mux[] = {
+	MSIOF1_SCK_G_MARK,
+};
+static const unsigned int msiof1_sync_g_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(3, 7),
+};
+static const unsigned int msiof1_sync_g_mux[] = {
+	MSIOF1_SYNC_G_MARK,
+};
+static const unsigned int msiof1_ss1_g_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(3, 10),
+};
+static const unsigned int msiof1_ss1_g_mux[] = {
+	MSIOF1_SS1_G_MARK,
+};
+static const unsigned int msiof1_ss2_g_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(3, 11),
+};
+static const unsigned int msiof1_ss2_g_mux[] = {
+	MSIOF1_SS2_G_MARK,
+};
+static const unsigned int msiof1_txd_g_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(3, 9),
+};
+static const unsigned int msiof1_txd_g_mux[] = {
+	MSIOF1_TXD_G_MARK,
+};
+static const unsigned int msiof1_rxd_g_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(3, 8),
+};
+static const unsigned int msiof1_rxd_g_mux[] = {
+	MSIOF1_RXD_G_MARK,
+};
+/* - MSIOF2 ----------------------------------------------------------------- */
+static const unsigned int msiof2_clk_a_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(1, 9),
+};
+static const unsigned int msiof2_clk_a_mux[] = {
+	MSIOF2_SCK_A_MARK,
+};
+static const unsigned int msiof2_sync_a_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(1, 8),
+};
+static const unsigned int msiof2_sync_a_mux[] = {
+	MSIOF2_SYNC_A_MARK,
+};
+static const unsigned int msiof2_ss1_a_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(1, 6),
+};
+static const unsigned int msiof2_ss1_a_mux[] = {
+	MSIOF2_SS1_A_MARK,
+};
+static const unsigned int msiof2_ss2_a_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(1, 7),
+};
+static const unsigned int msiof2_ss2_a_mux[] = {
+	MSIOF2_SS2_A_MARK,
+};
+static const unsigned int msiof2_txd_a_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(1, 11),
+};
+static const unsigned int msiof2_txd_a_mux[] = {
+	MSIOF2_TXD_A_MARK,
+};
+static const unsigned int msiof2_rxd_a_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(1, 10),
+};
+static const unsigned int msiof2_rxd_a_mux[] = {
+	MSIOF2_RXD_A_MARK,
+};
+static const unsigned int msiof2_clk_b_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(0, 4),
+};
+static const unsigned int msiof2_clk_b_mux[] = {
+	MSIOF2_SCK_B_MARK,
+};
+static const unsigned int msiof2_sync_b_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(0, 5),
+};
+static const unsigned int msiof2_sync_b_mux[] = {
+	MSIOF2_SYNC_B_MARK,
+};
+static const unsigned int msiof2_ss1_b_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(0, 0),
+};
+static const unsigned int msiof2_ss1_b_mux[] = {
+	MSIOF2_SS1_B_MARK,
+};
+static const unsigned int msiof2_ss2_b_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(0, 1),
+};
+static const unsigned int msiof2_ss2_b_mux[] = {
+	MSIOF2_SS2_B_MARK,
+};
+static const unsigned int msiof2_txd_b_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(0, 7),
+};
+static const unsigned int msiof2_txd_b_mux[] = {
+	MSIOF2_TXD_B_MARK,
+};
+static const unsigned int msiof2_rxd_b_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(0, 6),
+};
+static const unsigned int msiof2_rxd_b_mux[] = {
+	MSIOF2_RXD_B_MARK,
+};
+static const unsigned int msiof2_clk_c_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(2, 12),
+};
+static const unsigned int msiof2_clk_c_mux[] = {
+	MSIOF2_SCK_C_MARK,
+};
+static const unsigned int msiof2_sync_c_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(2, 11),
+};
+static const unsigned int msiof2_sync_c_mux[] = {
+	MSIOF2_SYNC_C_MARK,
+};
+static const unsigned int msiof2_ss1_c_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(2, 10),
+};
+static const unsigned int msiof2_ss1_c_mux[] = {
+	MSIOF2_SS1_C_MARK,
+};
+static const unsigned int msiof2_ss2_c_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(2, 9),
+};
+static const unsigned int msiof2_ss2_c_mux[] = {
+	MSIOF2_SS2_C_MARK,
+};
+static const unsigned int msiof2_txd_c_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(2, 14),
+};
+static const unsigned int msiof2_txd_c_mux[] = {
+	MSIOF2_TXD_C_MARK,
+};
+static const unsigned int msiof2_rxd_c_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(2, 13),
+};
+static const unsigned int msiof2_rxd_c_mux[] = {
+	MSIOF2_RXD_C_MARK,
+};
+static const unsigned int msiof2_clk_d_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(0, 8),
+};
+static const unsigned int msiof2_clk_d_mux[] = {
+	MSIOF2_SCK_D_MARK,
+};
+static const unsigned int msiof2_sync_d_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(0, 9),
+};
+static const unsigned int msiof2_sync_d_mux[] = {
+	MSIOF2_SYNC_D_MARK,
+};
+static const unsigned int msiof2_ss1_d_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(0, 12),
+};
+static const unsigned int msiof2_ss1_d_mux[] = {
+	MSIOF2_SS1_D_MARK,
+};
+static const unsigned int msiof2_ss2_d_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(0, 13),
+};
+static const unsigned int msiof2_ss2_d_mux[] = {
+	MSIOF2_SS2_D_MARK,
+};
+static const unsigned int msiof2_txd_d_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(0, 11),
+};
+static const unsigned int msiof2_txd_d_mux[] = {
+	MSIOF2_TXD_D_MARK,
+};
+static const unsigned int msiof2_rxd_d_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(0, 10),
+};
+static const unsigned int msiof2_rxd_d_mux[] = {
+	MSIOF2_RXD_D_MARK,
+};
+/* - MSIOF3 ----------------------------------------------------------------- */
+static const unsigned int msiof3_clk_a_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(0, 0),
+};
+static const unsigned int msiof3_clk_a_mux[] = {
+	MSIOF3_SCK_A_MARK,
+};
+static const unsigned int msiof3_sync_a_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(0, 1),
+};
+static const unsigned int msiof3_sync_a_mux[] = {
+	MSIOF3_SYNC_A_MARK,
+};
+static const unsigned int msiof3_ss1_a_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(0, 14),
+};
+static const unsigned int msiof3_ss1_a_mux[] = {
+	MSIOF3_SS1_A_MARK,
+};
+static const unsigned int msiof3_ss2_a_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(0, 15),
+};
+static const unsigned int msiof3_ss2_a_mux[] = {
+	MSIOF3_SS2_A_MARK,
+};
+static const unsigned int msiof3_txd_a_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(0, 3),
+};
+static const unsigned int msiof3_txd_a_mux[] = {
+	MSIOF3_TXD_A_MARK,
+};
+static const unsigned int msiof3_rxd_a_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(0, 2),
+};
+static const unsigned int msiof3_rxd_a_mux[] = {
+	MSIOF3_RXD_A_MARK,
+};
+static const unsigned int msiof3_clk_b_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(1, 2),
+};
+static const unsigned int msiof3_clk_b_mux[] = {
+	MSIOF3_SCK_B_MARK,
+};
+static const unsigned int msiof3_sync_b_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(1, 0),
+};
+static const unsigned int msiof3_sync_b_mux[] = {
+	MSIOF3_SYNC_B_MARK,
+};
+static const unsigned int msiof3_ss1_b_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(1, 4),
+};
+static const unsigned int msiof3_ss1_b_mux[] = {
+	MSIOF3_SS1_B_MARK,
+};
+static const unsigned int msiof3_ss2_b_pins[] = {
+	/* SS2 */
+	RCAR_GP_PIN(1, 5),
+};
+static const unsigned int msiof3_ss2_b_mux[] = {
+	MSIOF3_SS2_B_MARK,
+};
+static const unsigned int msiof3_txd_b_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(1, 1),
+};
+static const unsigned int msiof3_txd_b_mux[] = {
+	MSIOF3_TXD_B_MARK,
+};
+static const unsigned int msiof3_rxd_b_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(1, 3),
+};
+static const unsigned int msiof3_rxd_b_mux[] = {
+	MSIOF3_RXD_B_MARK,
+};
+static const unsigned int msiof3_clk_c_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(1, 12),
+};
+static const unsigned int msiof3_clk_c_mux[] = {
+	MSIOF3_SCK_C_MARK,
+};
+static const unsigned int msiof3_sync_c_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(1, 13),
+};
+static const unsigned int msiof3_sync_c_mux[] = {
+	MSIOF3_SYNC_C_MARK,
+};
+static const unsigned int msiof3_txd_c_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(1, 15),
+};
+static const unsigned int msiof3_txd_c_mux[] = {
+	MSIOF3_TXD_C_MARK,
+};
+static const unsigned int msiof3_rxd_c_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(1, 14),
+};
+static const unsigned int msiof3_rxd_c_mux[] = {
+	MSIOF3_RXD_C_MARK,
+};
+static const unsigned int msiof3_clk_d_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(1, 22),
+};
+static const unsigned int msiof3_clk_d_mux[] = {
+	MSIOF3_SCK_D_MARK,
+};
+static const unsigned int msiof3_sync_d_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(1, 23),
+};
+static const unsigned int msiof3_sync_d_mux[] = {
+	MSIOF3_SYNC_D_MARK,
+};
+static const unsigned int msiof3_ss1_d_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(1, 26),
+};
+static const unsigned int msiof3_ss1_d_mux[] = {
+	MSIOF3_SS1_D_MARK,
+};
+static const unsigned int msiof3_txd_d_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(1, 25),
+};
+static const unsigned int msiof3_txd_d_mux[] = {
+	MSIOF3_TXD_D_MARK,
+};
+static const unsigned int msiof3_rxd_d_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(1, 24),
+};
+static const unsigned int msiof3_rxd_d_mux[] = {
+	MSIOF3_RXD_D_MARK,
+};
+static const unsigned int msiof3_clk_e_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(2, 3),
+};
+static const unsigned int msiof3_clk_e_mux[] = {
+	MSIOF3_SCK_E_MARK,
+};
+static const unsigned int msiof3_sync_e_pins[] = {
+	/* SYNC */
+	RCAR_GP_PIN(2, 2),
+};
+static const unsigned int msiof3_sync_e_mux[] = {
+	MSIOF3_SYNC_E_MARK,
+};
+static const unsigned int msiof3_ss1_e_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(2, 1),
+};
+static const unsigned int msiof3_ss1_e_mux[] = {
+	MSIOF3_SS1_E_MARK,
+};
+static const unsigned int msiof3_ss2_e_pins[] = {
+	/* SS1 */
+	RCAR_GP_PIN(2, 0),
+};
+static const unsigned int msiof3_ss2_e_mux[] = {
+	MSIOF3_SS2_E_MARK,
+};
+static const unsigned int msiof3_txd_e_pins[] = {
+	/* TXD */
+	RCAR_GP_PIN(2, 5),
+};
+static const unsigned int msiof3_txd_e_mux[] = {
+	MSIOF3_TXD_E_MARK,
+};
+static const unsigned int msiof3_rxd_e_pins[] = {
+	/* RXD */
+	RCAR_GP_PIN(2, 4),
+};
+static const unsigned int msiof3_rxd_e_mux[] = {
+	MSIOF3_RXD_E_MARK,
+};
+
 /* - PWM0 --------------------------------------------------------------------*/
 static const unsigned int pwm0_pins[] = {
 	/* PWM */
@@ -2056,6 +2750,39 @@ static const unsigned int scif_clk_b_mux[] = {
 	SCIF_CLK_B_MARK,
 };
 
+/* - USB0 ------------------------------------------------------------------- */
+static const unsigned int usb0_pins[] = {
+	/* PWEN, OVC */
+	RCAR_GP_PIN(6, 24), RCAR_GP_PIN(6, 25),
+};
+static const unsigned int usb0_mux[] = {
+	USB0_PWEN_MARK, USB0_OVC_MARK,
+};
+/* - USB1 ------------------------------------------------------------------- */
+static const unsigned int usb1_pins[] = {
+	/* PWEN, OVC */
+	RCAR_GP_PIN(6, 26), RCAR_GP_PIN(6, 27),
+};
+static const unsigned int usb1_mux[] = {
+	USB1_PWEN_MARK, USB1_OVC_MARK,
+};
+/* - USB2 ------------------------------------------------------------------- */
+static const unsigned int usb2_pins[] = {
+	/* PWEN, OVC */
+	RCAR_GP_PIN(6, 14), RCAR_GP_PIN(6, 15),
+};
+static const unsigned int usb2_mux[] = {
+	USB2_PWEN_MARK, USB2_OVC_MARK,
+};
+/* - USB2_CH3 --------------------------------------------------------------- */
+static const unsigned int usb2_ch3_pins[] = {
+	/* PWEN, OVC */
+	RCAR_GP_PIN(6, 30), RCAR_GP_PIN(6, 31),
+};
+static const unsigned int usb2_ch3_mux[] = {
+	USB2_CH3_PWEN_MARK, USB2_CH3_OVC_MARK,
+};
+
 static const struct sh_pfc_pin_group pinmux_groups[] = {
 	SH_PFC_PIN_GROUP(avb_link),
 	SH_PFC_PIN_GROUP(avb_magic),
@@ -2075,6 +2802,105 @@ static const struct sh_pfc_pin_group pinmux_groups[] = {
 	SH_PFC_PIN_GROUP(du_oddf),
 	SH_PFC_PIN_GROUP(du_cde),
 	SH_PFC_PIN_GROUP(du_disp),
+	SH_PFC_PIN_GROUP(msiof0_clk),
+	SH_PFC_PIN_GROUP(msiof0_sync),
+	SH_PFC_PIN_GROUP(msiof0_ss1),
+	SH_PFC_PIN_GROUP(msiof0_ss2),
+	SH_PFC_PIN_GROUP(msiof0_txd),
+	SH_PFC_PIN_GROUP(msiof0_rxd),
+	SH_PFC_PIN_GROUP(msiof1_clk_a),
+	SH_PFC_PIN_GROUP(msiof1_sync_a),
+	SH_PFC_PIN_GROUP(msiof1_ss1_a),
+	SH_PFC_PIN_GROUP(msiof1_ss2_a),
+	SH_PFC_PIN_GROUP(msiof1_txd_a),
+	SH_PFC_PIN_GROUP(msiof1_rxd_a),
+	SH_PFC_PIN_GROUP(msiof1_clk_b),
+	SH_PFC_PIN_GROUP(msiof1_sync_b),
+	SH_PFC_PIN_GROUP(msiof1_ss1_b),
+	SH_PFC_PIN_GROUP(msiof1_ss2_b),
+	SH_PFC_PIN_GROUP(msiof1_txd_b),
+	SH_PFC_PIN_GROUP(msiof1_rxd_b),
+	SH_PFC_PIN_GROUP(msiof1_clk_c),
+	SH_PFC_PIN_GROUP(msiof1_sync_c),
+	SH_PFC_PIN_GROUP(msiof1_ss1_c),
+	SH_PFC_PIN_GROUP(msiof1_ss2_c),
+	SH_PFC_PIN_GROUP(msiof1_txd_c),
+	SH_PFC_PIN_GROUP(msiof1_rxd_c),
+	SH_PFC_PIN_GROUP(msiof1_clk_d),
+	SH_PFC_PIN_GROUP(msiof1_sync_d),
+	SH_PFC_PIN_GROUP(msiof1_ss1_d),
+	SH_PFC_PIN_GROUP(msiof1_ss2_d),
+	SH_PFC_PIN_GROUP(msiof1_txd_d),
+	SH_PFC_PIN_GROUP(msiof1_rxd_d),
+	SH_PFC_PIN_GROUP(msiof1_clk_e),
+	SH_PFC_PIN_GROUP(msiof1_sync_e),
+	SH_PFC_PIN_GROUP(msiof1_ss1_e),
+	SH_PFC_PIN_GROUP(msiof1_ss2_e),
+	SH_PFC_PIN_GROUP(msiof1_txd_e),
+	SH_PFC_PIN_GROUP(msiof1_rxd_e),
+	SH_PFC_PIN_GROUP(msiof1_clk_f),
+	SH_PFC_PIN_GROUP(msiof1_sync_f),
+	SH_PFC_PIN_GROUP(msiof1_ss1_f),
+	SH_PFC_PIN_GROUP(msiof1_ss2_f),
+	SH_PFC_PIN_GROUP(msiof1_txd_f),
+	SH_PFC_PIN_GROUP(msiof1_rxd_f),
+	SH_PFC_PIN_GROUP(msiof1_clk_g),
+	SH_PFC_PIN_GROUP(msiof1_sync_g),
+	SH_PFC_PIN_GROUP(msiof1_ss1_g),
+	SH_PFC_PIN_GROUP(msiof1_ss2_g),
+	SH_PFC_PIN_GROUP(msiof1_txd_g),
+	SH_PFC_PIN_GROUP(msiof1_rxd_g),
+	SH_PFC_PIN_GROUP(msiof2_clk_a),
+	SH_PFC_PIN_GROUP(msiof2_sync_a),
+	SH_PFC_PIN_GROUP(msiof2_ss1_a),
+	SH_PFC_PIN_GROUP(msiof2_ss2_a),
+	SH_PFC_PIN_GROUP(msiof2_txd_a),
+	SH_PFC_PIN_GROUP(msiof2_rxd_a),
+	SH_PFC_PIN_GROUP(msiof2_clk_b),
+	SH_PFC_PIN_GROUP(msiof2_sync_b),
+	SH_PFC_PIN_GROUP(msiof2_ss1_b),
+	SH_PFC_PIN_GROUP(msiof2_ss2_b),
+	SH_PFC_PIN_GROUP(msiof2_txd_b),
+	SH_PFC_PIN_GROUP(msiof2_rxd_b),
+	SH_PFC_PIN_GROUP(msiof2_clk_c),
+	SH_PFC_PIN_GROUP(msiof2_sync_c),
+	SH_PFC_PIN_GROUP(msiof2_ss1_c),
+	SH_PFC_PIN_GROUP(msiof2_ss2_c),
+	SH_PFC_PIN_GROUP(msiof2_txd_c),
+	SH_PFC_PIN_GROUP(msiof2_rxd_c),
+	SH_PFC_PIN_GROUP(msiof2_clk_d),
+	SH_PFC_PIN_GROUP(msiof2_sync_d),
+	SH_PFC_PIN_GROUP(msiof2_ss1_d),
+	SH_PFC_PIN_GROUP(msiof2_ss2_d),
+	SH_PFC_PIN_GROUP(msiof2_txd_d),
+	SH_PFC_PIN_GROUP(msiof2_rxd_d),
+	SH_PFC_PIN_GROUP(msiof3_clk_a),
+	SH_PFC_PIN_GROUP(msiof3_sync_a),
+	SH_PFC_PIN_GROUP(msiof3_ss1_a),
+	SH_PFC_PIN_GROUP(msiof3_ss2_a),
+	SH_PFC_PIN_GROUP(msiof3_txd_a),
+	SH_PFC_PIN_GROUP(msiof3_rxd_a),
+	SH_PFC_PIN_GROUP(msiof3_clk_b),
+	SH_PFC_PIN_GROUP(msiof3_sync_b),
+	SH_PFC_PIN_GROUP(msiof3_ss1_b),
+	SH_PFC_PIN_GROUP(msiof3_ss2_b),
+	SH_PFC_PIN_GROUP(msiof3_txd_b),
+	SH_PFC_PIN_GROUP(msiof3_rxd_b),
+	SH_PFC_PIN_GROUP(msiof3_clk_c),
+	SH_PFC_PIN_GROUP(msiof3_sync_c),
+	SH_PFC_PIN_GROUP(msiof3_txd_c),
+	SH_PFC_PIN_GROUP(msiof3_rxd_c),
+	SH_PFC_PIN_GROUP(msiof3_clk_d),
+	SH_PFC_PIN_GROUP(msiof3_sync_d),
+	SH_PFC_PIN_GROUP(msiof3_ss1_d),
+	SH_PFC_PIN_GROUP(msiof3_txd_d),
+	SH_PFC_PIN_GROUP(msiof3_rxd_d),
+	SH_PFC_PIN_GROUP(msiof3_clk_e),
+	SH_PFC_PIN_GROUP(msiof3_sync_e),
+	SH_PFC_PIN_GROUP(msiof3_ss1_e),
+	SH_PFC_PIN_GROUP(msiof3_ss2_e),
+	SH_PFC_PIN_GROUP(msiof3_txd_e),
+	SH_PFC_PIN_GROUP(msiof3_rxd_e),
 	SH_PFC_PIN_GROUP(pwm0),
 	SH_PFC_PIN_GROUP(pwm1_a),
 	SH_PFC_PIN_GROUP(pwm1_b),
@@ -2117,6 +2943,10 @@ static const struct sh_pfc_pin_group pinmux_groups[] = {
 	SH_PFC_PIN_GROUP(scif5_clk_b),
 	SH_PFC_PIN_GROUP(scif_clk_a),
 	SH_PFC_PIN_GROUP(scif_clk_b),
+	SH_PFC_PIN_GROUP(usb0),
+	SH_PFC_PIN_GROUP(usb1),
+	SH_PFC_PIN_GROUP(usb2),
+	SH_PFC_PIN_GROUP(usb2_ch3),
 };
 
 static const char * const avb_groups[] = {
@@ -2143,6 +2973,117 @@ static const char * const du_groups[] = {
 	"du_disp",
 };
 
+static const char * const msiof0_groups[] = {
+	"msiof0_clk",
+	"msiof0_sync",
+	"msiof0_ss1",
+	"msiof0_ss2",
+	"msiof0_txd",
+	"msiof0_rxd",
+};
+
+static const char * const msiof1_groups[] = {
+	"msiof1_clk_a",
+	"msiof1_sync_a",
+	"msiof1_ss1_a",
+	"msiof1_ss2_a",
+	"msiof1_txd_a",
+	"msiof1_rxd_a",
+	"msiof1_clk_b",
+	"msiof1_sync_b",
+	"msiof1_ss1_b",
+	"msiof1_ss2_b",
+	"msiof1_txd_b",
+	"msiof1_rxd_b",
+	"msiof1_clk_c",
+	"msiof1_sync_c",
+	"msiof1_ss1_c",
+	"msiof1_ss2_c",
+	"msiof1_txd_c",
+	"msiof1_rxd_c",
+	"msiof1_clk_d",
+	"msiof1_sync_d",
+	"msiof1_ss1_d",
+	"msiof1_ss2_d",
+	"msiof1_txd_d",
+	"msiof1_rxd_d",
+	"msiof1_clk_e",
+	"msiof1_sync_e",
+	"msiof1_ss1_e",
+	"msiof1_ss2_e",
+	"msiof1_txd_e",
+	"msiof1_rxd_e",
+	"msiof1_clk_f",
+	"msiof1_sync_f",
+	"msiof1_ss1_f",
+	"msiof1_ss2_f",
+	"msiof1_txd_f",
+	"msiof1_rxd_f",
+	"msiof1_clk_g",
+	"msiof1_sync_g",
+	"msiof1_ss1_g",
+	"msiof1_ss2_g",
+	"msiof1_txd_g",
+	"msiof1_rxd_g",
+};
+
+static const char * const msiof2_groups[] = {
+	"msiof2_clk_a",
+	"msiof2_sync_a",
+	"msiof2_ss1_a",
+	"msiof2_ss2_a",
+	"msiof2_txd_a",
+	"msiof2_rxd_a",
+	"msiof2_clk_b",
+	"msiof2_sync_b",
+	"msiof2_ss1_b",
+	"msiof2_ss2_b",
+	"msiof2_txd_b",
+	"msiof2_rxd_b",
+	"msiof2_clk_c",
+	"msiof2_sync_c",
+	"msiof2_ss1_c",
+	"msiof2_ss2_c",
+	"msiof2_txd_c",
+	"msiof2_rxd_c",
+	"msiof2_clk_d",
+	"msiof2_sync_d",
+	"msiof2_ss1_d",
+	"msiof2_ss2_d",
+	"msiof2_txd_d",
+	"msiof2_rxd_d",
+};
+
+static const char * const msiof3_groups[] = {
+	"msiof3_clk_a",
+	"msiof3_sync_a",
+	"msiof3_ss1_a",
+	"msiof3_ss2_a",
+	"msiof3_txd_a",
+	"msiof3_rxd_a",
+	"msiof3_clk_b",
+	"msiof3_sync_b",
+	"msiof3_ss1_b",
+	"msiof3_ss2_b",
+	"msiof3_txd_b",
+	"msiof3_rxd_b",
+	"msiof3_clk_c",
+	"msiof3_sync_c",
+	"msiof3_txd_c",
+	"msiof3_rxd_c",
+	"msiof3_clk_d",
+	"msiof3_sync_d",
+	"msiof3_ss1_d",
+	"msiof3_txd_d",
+	"msiof3_rxd_d",
+	"msiof3_clk_e",
+	"msiof3_sync_e",
+	"msiof3_ss1_e",
+	"msiof3_ss2_e",
+	"msiof3_txd_e",
+	"msiof3_rxd_e",
+};
+
 static const char * const pwm0_groups[] = {
 	"pwm0",
 };
@@ -2227,9 +3168,29 @@ static const char * const scif_clk_groups[] = {
 	"scif_clk_b",
 };
 
+static const char * const usb0_groups[] = {
+	"usb0",
+};
+
+static const char * const usb1_groups[] = {
+	"usb1",
+};
+
+static const char * const usb2_groups[] = {
+	"usb2",
+};
+
+static const char * const usb2_ch3_groups[] = {
+	"usb2_ch3",
+};
+
 static const struct sh_pfc_function pinmux_functions[] = {
 	SH_PFC_FUNCTION(avb),
 	SH_PFC_FUNCTION(du),
+	SH_PFC_FUNCTION(msiof0),
+	SH_PFC_FUNCTION(msiof1),
+	SH_PFC_FUNCTION(msiof2),
+	SH_PFC_FUNCTION(msiof3),
 	SH_PFC_FUNCTION(pwm0),
 	SH_PFC_FUNCTION(pwm1),
 	SH_PFC_FUNCTION(pwm2),
@@ -2244,6 +3205,10 @@ static const struct sh_pfc_function pinmux_functions[] = {
 	SH_PFC_FUNCTION(scif4),
 	SH_PFC_FUNCTION(scif5),
 	SH_PFC_FUNCTION(scif_clk),
+	SH_PFC_FUNCTION(usb0),
+	SH_PFC_FUNCTION(usb1),
+	SH_PFC_FUNCTION(usb2),
+	SH_PFC_FUNCTION(usb2_ch3),
 };
 
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
@@ -2601,7 +3566,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP7_27_24
 		IP7_23_20
 		IP7_19_16
-		IP7_15_12
+		/* IP7_15_12 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		IP7_11_8
 		IP7_7_4
 		IP7_3_0 }
@@ -2782,7 +3747,8 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		MOD_SEL2_28_27
 		MOD_SEL2_26
 		MOD_SEL2_25_24_23
-		MOD_SEL2_22
+		/* RESERVED 22 */
+		0, 0,
 		MOD_SEL2_21
 		MOD_SEL2_20
 		MOD_SEL2_19
@@ -3049,8 +4015,8 @@ static const struct pinmux_drive_reg pinmux_drive_regs[] = {
 		{ RCAR_GP_PIN(6, 27), 20, 3 },	/* USB1_OVC */
 		{ RCAR_GP_PIN(6, 28), 16, 3 },	/* USB30_PWEN */
 		{ RCAR_GP_PIN(6, 29), 12, 3 },	/* USB30_OVC */
-		{ RCAR_GP_PIN(6, 30),  8, 3 },	/* USB3_PWEN */
-		{ RCAR_GP_PIN(6, 31),  4, 3 },	/* USB3_OVC */
+		{ RCAR_GP_PIN(6, 30),  8, 3 },	/* USB2_CH3_PWEN */
+		{ RCAR_GP_PIN(6, 31),  4, 3 },	/* USB2_CH3_OVC */
 	} },
 	{ },
 };
@@ -3177,7 +4143,7 @@ static const struct sh_pfc_bias_info bias_info[] = {
 	{ RCAR_GP_PIN(1, 24),    PU2,  5 },	/* RD_WR_N */
 	{ RCAR_GP_PIN(1, 23),    PU2,  4 },	/* RD_N */
 	{ RCAR_GP_PIN(1, 22),    PU2,  3 },	/* BS_N */
-	{ RCAR_GP_PIN(1, 21),    PU2,  2 },	/* CS1_N_A26 */
+	{ RCAR_GP_PIN(1, 21),    PU2,  2 },	/* CS1_N */
 	{ RCAR_GP_PIN(1, 20),    PU2,  1 },	/* CS0_N */
 	{ PIN_NUMBER('F', 1),    PU2,  0 },	/* CLKOUT */
 
@@ -3280,8 +4246,8 @@ static const struct sh_pfc_bias_info bias_info[] = {
 	{ RCAR_GP_PIN(5, 21),    PU5,  1 },	/* MSIOF0_SS2 */
 	{ RCAR_GP_PIN(5, 20),    PU5,  0 },	/* MSIOF0_TXD */
 
-	{ RCAR_GP_PIN(6, 31),    PU6,  6 },	/* USB3_OVC */
-	{ RCAR_GP_PIN(6, 30),    PU6,  5 },	/* USB3_PWEN */
+	{ RCAR_GP_PIN(6, 31),    PU6,  6 },	/* USB2_CH3_OVC */
+	{ RCAR_GP_PIN(6, 30),    PU6,  5 },	/* USB2_CH3_PWEN */
 	{ RCAR_GP_PIN(6, 29),    PU6,  4 },	/* USB30_OVC */
 	{ RCAR_GP_PIN(6, 28),    PU6,  3 },	/* USB30_PWEN */
 	{ RCAR_GP_PIN(6, 27),    PU6,  2 },	/* USB1_OVC */
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7796.c b/drivers/pinctrl/sh-pfc/pfc-r8a7796.c
index 98bf5d0e078e..200e1f4f6db9 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a7796.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a7796.c
@@ -67,7 +67,7 @@
 #define GPSR1_24	F_(RD_WR_N,		IP4_31_28)
 #define GPSR1_23	F_(RD_N,		IP4_27_24)
 #define GPSR1_22	F_(BS_N,		IP4_23_20)
-#define GPSR1_21	F_(CS1_N_A26,		IP4_19_16)
+#define GPSR1_21	F_(CS1_N,		IP4_19_16)
 #define GPSR1_20	F_(CS0_N,		IP4_15_12)
 #define GPSR1_19	F_(A19,			IP4_11_8)
 #define GPSR1_18	F_(A18,			IP4_7_4)
@@ -221,8 +221,8 @@
 #define IP0_15_12	FM(AVB_LINK)		F_(0, 0)	FM(MSIOF2_SCK_C)	FM(TX4_A)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP0_19_16	FM(AVB_AVTP_MATCH_A)	F_(0, 0)	FM(MSIOF2_RXD_C)	FM(CTS4_N_A)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP0_23_20	FM(AVB_AVTP_CAPTURE_A)	F_(0, 0)	FM(MSIOF2_TXD_C)	FM(RTS4_N_TANS_A)		F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP0_27_24	FM(IRQ0)		FM(QPOLB)	F_(0, 0)		FM(DU_CDE)			FM(VI4_DATA0_B) FM(CAN0_TX_B)	FM(CANFD0_TX_B)		FM(MSIOF3_SS1_E) F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP0_31_28	FM(IRQ1)		FM(QPOLA)	F_(0, 0)		FM(DU_DISP)			FM(VI4_DATA1_B) FM(CAN0_RX_B)	FM(CANFD0_RX_B)		FM(MSIOF3_SS2_E) F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_27_24	FM(IRQ0)		FM(QPOLB)	F_(0, 0)		FM(DU_CDE)			FM(VI4_DATA0_B) FM(CAN0_TX_B)	FM(CANFD0_TX_B)		FM(MSIOF3_SS2_E) F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_31_28	FM(IRQ1)		FM(QPOLA)	F_(0, 0)		FM(DU_DISP)			FM(VI4_DATA1_B) FM(CAN0_RX_B)	FM(CANFD0_RX_B)		FM(MSIOF3_SS1_E) F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP1_3_0		FM(IRQ2)		FM(QCPV_QDE)	F_(0, 0)		FM(DU_EXODDF_DU_ODDF_DISP_CDE)	FM(VI4_DATA2_B) F_(0, 0)	F_(0, 0)		FM(MSIOF3_SYNC_E) F_(0, 0)		FM(PWM3_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP1_7_4		FM(IRQ3)		FM(QSTVB_QVE)	FM(A25)			FM(DU_DOTCLKOUT1)		FM(VI4_DATA3_B) F_(0, 0)	F_(0, 0)		FM(MSIOF3_SCK_E) F_(0, 0)		FM(PWM4_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP1_11_8	FM(IRQ4)		FM(QSTH_QHS)	FM(A24)			FM(DU_EXHSYNC_DU_HSYNC)		FM(VI4_DATA4_B) F_(0, 0)	F_(0, 0)		FM(MSIOF3_RXD_E) F_(0, 0)		FM(PWM5_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
@@ -253,7 +253,7 @@
 #define IP4_7_4		FM(A18)			FM(LCDOUT10)	F_(0, 0)		F_(0, 0)			FM(VI4_HSYNC_N)	F_(0, 0)	FM(DU_DG2)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP4_11_8	FM(A19)			FM(LCDOUT11)	F_(0, 0)		F_(0, 0)			FM(VI4_CLKENB)	F_(0, 0)	FM(DU_DG3)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP4_15_12	FM(CS0_N)		F_(0, 0)	F_(0, 0)		F_(0, 0)			F_(0, 0)	FM(VI5_CLKENB)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP4_19_16	FM(CS1_N_A26)		F_(0, 0)	F_(0, 0)		F_(0, 0)			F_(0, 0)	FM(VI5_CLK)	F_(0, 0)		FM(EX_WAIT0_B)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_19_16	FM(CS1_N)		F_(0, 0)	F_(0, 0)		F_(0, 0)			F_(0, 0)	FM(VI5_CLK)	F_(0, 0)		FM(EX_WAIT0_B)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP4_23_20	FM(BS_N)		FM(QSTVA_QVS)	FM(MSIOF3_SCK_D)	FM(SCK3)			FM(HSCK3)	F_(0, 0)	F_(0, 0)		F_(0, 0)	FM(CAN1_TX)		FM(CANFD1_TX)	FM(IETX_A)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP4_27_24	FM(RD_N)		F_(0, 0)	FM(MSIOF3_SYNC_D)	FM(RX3_A)			FM(HRX3_A)	F_(0, 0)	F_(0, 0)		F_(0, 0)	FM(CAN0_TX_A)		FM(CANFD0_TX_A)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP4_31_28	FM(RD_WR_N)		F_(0, 0)	FM(MSIOF3_RXD_D)	FM(TX3_A)			FM(HTX3_A)	F_(0, 0)	F_(0, 0)		F_(0, 0)	FM(CAN0_RX_A)		FM(CANFD0_RX_A)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
@@ -278,7 +278,6 @@
 #define IP7_3_0		FM(D13)			FM(LCDOUT5)	FM(MSIOF2_SS2_D)	FM(TX4_C)			FM(VI4_DATA5_A)	F_(0, 0)	FM(DU_DR5)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP7_7_4		FM(D14)			FM(LCDOUT6)	FM(MSIOF3_SS1_A)	FM(HRX3_C)			FM(VI4_DATA6_A)	F_(0, 0)	FM(DU_DR6)		FM(SCL6_C)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP7_11_8	FM(D15)			FM(LCDOUT7)	FM(MSIOF3_SS2_A)	FM(HTX3_C)			FM(VI4_DATA7_A)	F_(0, 0)	FM(DU_DR7)		FM(SDA6_C)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP7_15_12	FM(FSCLKST)		F_(0, 0)	F_(0, 0)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP7_19_16	FM(SD0_CLK)		F_(0, 0)	FM(MSIOF1_SCK_E)	F_(0, 0)			F_(0, 0)	F_(0, 0)	FM(STP_OPWM_0_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP7_23_20	FM(SD0_CMD)		F_(0, 0)	FM(MSIOF1_SYNC_E)	F_(0, 0)			F_(0, 0)	F_(0, 0)	FM(STP_IVCXO27_0_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP7_27_24	FM(SD0_DAT0)		F_(0, 0)	FM(MSIOF1_RXD_E)	F_(0, 0)			F_(0, 0)	FM(TS_SCK0_B)	FM(STP_ISCLK_0_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
@@ -291,24 +290,24 @@
 #define IP8_23_20	FM(SD1_DAT1)		FM(SD2_DAT5)	FM(MSIOF1_TXD_G)	FM(NFDATA14_B)			F_(0, 0)	FM(TS_SPSYNC1_B)FM(STP_ISSYNC_1_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP8_27_24	FM(SD1_DAT2)		FM(SD2_DAT6)	FM(MSIOF1_SS1_G)	FM(NFDATA15_B)			F_(0, 0)	FM(TS_SDAT1_B)	FM(STP_ISD_1_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP8_31_28	FM(SD1_DAT3)		FM(SD2_DAT7)	FM(MSIOF1_SS2_G)	FM(NFRB_N_B)			F_(0, 0)	FM(TS_SDEN1_B)	FM(STP_ISEN_1_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP9_3_0		FM(SD2_CLK)		F_(0, 0)	F_(0, 0)		FM(NFDATA8)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP9_7_4		FM(SD2_CMD)		F_(0, 0)	F_(0, 0)		FM(NFDATA9)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP9_11_8	FM(SD2_DAT0)		F_(0, 0)	F_(0, 0)		FM(NFDATA10)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP9_15_12	FM(SD2_DAT1)		F_(0, 0)	F_(0, 0)		FM(NFDATA11)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP9_19_16	FM(SD2_DAT2)		F_(0, 0)	F_(0, 0)		FM(NFDATA12)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP9_23_20	FM(SD2_DAT3)		F_(0, 0)	F_(0, 0)		FM(NFDATA13)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP9_27_24	FM(SD2_DS)		F_(0, 0)	F_(0, 0)		FM(NFALE)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	FM(SATA_DEVSLP_B)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP9_31_28	FM(SD3_CLK)		F_(0, 0)	F_(0, 0)		FM(NFWE_N)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP10_3_0	FM(SD3_CMD)		F_(0, 0)	F_(0, 0)		FM(NFRE_N)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP10_7_4	FM(SD3_DAT0)		F_(0, 0)	F_(0, 0)		FM(NFDATA0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP10_11_8	FM(SD3_DAT1)		F_(0, 0)	F_(0, 0)		FM(NFDATA1)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP10_15_12	FM(SD3_DAT2)		F_(0, 0)	F_(0, 0)		FM(NFDATA2)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP10_19_16	FM(SD3_DAT3)		F_(0, 0)	F_(0, 0)		FM(NFDATA3)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP10_23_20	FM(SD3_DAT4)		FM(SD2_CD_A)	F_(0, 0)		FM(NFDATA4)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP10_27_24	FM(SD3_DAT5)		FM(SD2_WP_A)	F_(0, 0)		FM(NFDATA5)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP10_31_28	FM(SD3_DAT6)		FM(SD3_CD)	F_(0, 0)		FM(NFDATA6)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP11_3_0	FM(SD3_DAT7)		FM(SD3_WP)	F_(0, 0)		FM(NFDATA7)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP11_7_4	FM(SD3_DS)		F_(0, 0)	F_(0, 0)		FM(NFCLE)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_3_0		FM(SD2_CLK)		F_(0, 0)	FM(NFDATA8)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_7_4		FM(SD2_CMD)		F_(0, 0)	FM(NFDATA9)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_11_8	FM(SD2_DAT0)		F_(0, 0)	FM(NFDATA10)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_15_12	FM(SD2_DAT1)		F_(0, 0)	FM(NFDATA11)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_19_16	FM(SD2_DAT2)		F_(0, 0)	FM(NFDATA12)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_23_20	FM(SD2_DAT3)		F_(0, 0)	FM(NFDATA13)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_27_24	FM(SD2_DS)		F_(0, 0)	FM(NFALE)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_31_28	FM(SD3_CLK)		F_(0, 0)	FM(NFWE_N)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_3_0	FM(SD3_CMD)		F_(0, 0)	FM(NFRE_N)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_7_4	FM(SD3_DAT0)		F_(0, 0)	FM(NFDATA0)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_11_8	FM(SD3_DAT1)		F_(0, 0)	FM(NFDATA1)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_15_12	FM(SD3_DAT2)		F_(0, 0)	FM(NFDATA2)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_19_16	FM(SD3_DAT3)		F_(0, 0)	FM(NFDATA3)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_23_20	FM(SD3_DAT4)		FM(SD2_CD_A)	FM(NFDATA4)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_27_24	FM(SD3_DAT5)		FM(SD2_WP_A)	FM(NFDATA5)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_31_28	FM(SD3_DAT6)		FM(SD3_CD)	FM(NFDATA6)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_3_0	FM(SD3_DAT7)		FM(SD3_WP)	FM(NFDATA7)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_7_4	FM(SD3_DS)		F_(0, 0)	FM(NFCLE)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP11_11_8	FM(SD0_CD)		F_(0, 0)	FM(NFDATA14_A)		F_(0, 0)			FM(SCL2_B)	FM(SIM0_RST_A)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 
 /* IPSRx */		/* 0 */			/* 1 */		/* 2 */			/* 3 */				/* 4 */		/* 5 */		/* 6 */			/* 7 */		/* 8 */			/* 9 */		/* A */		/* B */		/* C - F */
@@ -319,14 +318,14 @@
 #define IP11_31_28	FM(RX0)			FM(HRX1_B)	F_(0, 0)		F_(0, 0)			F_(0, 0)	FM(TS_SCK0_C)	FM(STP_ISCLK_0_C)	FM(RIF0_D0_B)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP12_3_0	FM(TX0)			FM(HTX1_B)	F_(0, 0)		F_(0, 0)			F_(0, 0)	FM(TS_SPSYNC0_C)FM(STP_ISSYNC_0_C)	FM(RIF0_D1_B)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP12_7_4	FM(CTS0_N)		FM(HCTS1_N_B)	FM(MSIOF1_SYNC_B)	F_(0, 0)			F_(0, 0)	FM(TS_SPSYNC1_C)FM(STP_ISSYNC_1_C)	FM(RIF1_SYNC_B)	FM(AUDIO_CLKOUT_C)	FM(ADICS_SAMP)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP12_11_8	FM(RTS0_N_TANS)		FM(HRTS1_N_B)	FM(MSIOF1_SS1_B)	FM(AUDIO_CLKA_B)		FM(SCL2_A)	F_(0, 0)	FM(STP_IVCXO27_1_C)	FM(RIF0_SYNC_B)	FM(FSO_TOE_A)		FM(ADICHS1)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP12_11_8	FM(RTS0_N_TANS)		FM(HRTS1_N_B)	FM(MSIOF1_SS1_B)	FM(AUDIO_CLKA_B)		FM(SCL2_A)	F_(0, 0)	FM(STP_IVCXO27_1_C)	FM(RIF0_SYNC_B)	F_(0, 0)		FM(ADICHS1)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP12_15_12	FM(RX1_A)		FM(HRX1_A)	F_(0, 0)		F_(0, 0)			F_(0, 0)	FM(TS_SDAT0_C)	FM(STP_ISD_0_C)		FM(RIF1_CLK_C)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP12_19_16	FM(TX1_A)		FM(HTX1_A)	F_(0, 0)		F_(0, 0)			F_(0, 0)	FM(TS_SDEN0_C)	FM(STP_ISEN_0_C)	FM(RIF1_D0_C)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP12_23_20	FM(CTS1_N)		FM(HCTS1_N_A)	FM(MSIOF1_RXD_B)	F_(0, 0)			F_(0, 0)	FM(TS_SDEN1_C)	FM(STP_ISEN_1_C)	FM(RIF1_D0_B)	F_(0, 0)		FM(ADIDATA)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP12_27_24	FM(RTS1_N_TANS)		FM(HRTS1_N_A)	FM(MSIOF1_TXD_B)	F_(0, 0)			F_(0, 0)	FM(TS_SDAT1_C)	FM(STP_ISD_1_C)		FM(RIF1_D1_B)	F_(0, 0)		FM(ADICHS0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP12_31_28	FM(SCK2)		FM(SCIF_CLK_B)	FM(MSIOF1_SCK_B)	F_(0, 0)			F_(0, 0)	FM(TS_SCK1_C)	FM(STP_ISCLK_1_C)	FM(RIF1_CLK_B)	F_(0, 0)		FM(ADICLK)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP13_3_0	FM(TX2_A)		F_(0, 0)	F_(0, 0)		FM(SD2_CD_B)			FM(SCL1_A)	F_(0, 0)	FM(FMCLK_A)		FM(RIF1_D1_C)	F_(0, 0)		FM(FSO_CFE_0_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP13_7_4	FM(RX2_A)		F_(0, 0)	F_(0, 0)		FM(SD2_WP_B)			FM(SDA1_A)	F_(0, 0)	FM(FMIN_A)		FM(RIF1_SYNC_C)	F_(0, 0)		FM(FSO_CFE_1_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP13_3_0	FM(TX2_A)		F_(0, 0)	F_(0, 0)		FM(SD2_CD_B)			FM(SCL1_A)	F_(0, 0)	FM(FMCLK_A)		FM(RIF1_D1_C)	F_(0, 0)		FM(FSO_CFE_0_N)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP13_7_4	FM(RX2_A)		F_(0, 0)	F_(0, 0)		FM(SD2_WP_B)			FM(SDA1_A)	F_(0, 0)	FM(FMIN_A)		FM(RIF1_SYNC_C)	F_(0, 0)		FM(FSO_CFE_1_N)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP13_11_8	FM(HSCK0)		F_(0, 0)	FM(MSIOF1_SCK_D)	FM(AUDIO_CLKB_A)		FM(SSI_SDATA1_B)FM(TS_SCK0_D)	FM(STP_ISCLK_0_D)	FM(RIF0_CLK_C)	F_(0, 0)		F_(0, 0)	FM(RX5_B)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP13_15_12	FM(HRX0)		F_(0, 0)	FM(MSIOF1_RXD_D)	F_(0, 0)			FM(SSI_SDATA2_B)FM(TS_SDEN0_D)	FM(STP_ISEN_0_D)	FM(RIF0_D0_C)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP13_19_16	FM(HTX0)		F_(0, 0)	FM(MSIOF1_TXD_D)	F_(0, 0)			FM(SSI_SDATA9_B)FM(TS_SDAT0_D)	FM(STP_ISD_0_D)		FM(RIF0_D1_C)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
@@ -366,9 +365,9 @@
 #define IP17_19_16	FM(USB1_PWEN)		F_(0, 0)	F_(0, 0)		FM(SIM0_CLK_C)			FM(SSI_SCK1_A)	FM(TS_SCK0_E)	FM(STP_ISCLK_0_E)	FM(FMCLK_B)	FM(RIF2_CLK_B)		F_(0, 0)	FM(SPEEDIN_A)	F_(0, 0)	F_(0, 0) FM(HTX2_C) F_(0, 0) F_(0, 0)
 #define IP17_23_20	FM(USB1_OVC)		F_(0, 0)	FM(MSIOF1_SS2_C)	F_(0, 0)			FM(SSI_WS1_A)	FM(TS_SDAT0_E)	FM(STP_ISD_0_E)		FM(FMIN_B)	FM(RIF2_SYNC_B)		F_(0, 0)	FM(REMOCON_B)	F_(0, 0)	F_(0, 0) FM(HCTS2_N_C) F_(0, 0) F_(0, 0)
 #define IP17_27_24	FM(USB30_PWEN)		F_(0, 0)	F_(0, 0)		FM(AUDIO_CLKOUT_B)		FM(SSI_SCK2_B)	FM(TS_SDEN1_D)	FM(STP_ISEN_1_D)	FM(STP_OPWM_0_E)FM(RIF3_D0_B)		F_(0, 0)	FM(TCLK2_B)	FM(TPU0TO0)	FM(BPFCLK_C) FM(HRTS2_N_C) F_(0, 0) F_(0, 0)
-#define IP17_31_28	FM(USB30_OVC)		F_(0, 0)	F_(0, 0)		FM(AUDIO_CLKOUT1_B)		FM(SSI_WS2_B)	FM(TS_SPSYNC1_D)FM(STP_ISSYNC_1_D)	FM(STP_IVCXO27_0_E)FM(RIF3_D1_B)	F_(0, 0)	FM(FSO_TOE_B)	FM(TPU0TO1)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP18_3_0	FM(GP6_30)		F_(0, 0)	F_(0, 0)		FM(AUDIO_CLKOUT2_B)		FM(SSI_SCK9_B)	FM(TS_SDEN0_E)	FM(STP_ISEN_0_E)	F_(0, 0)	FM(RIF2_D0_B)		F_(0, 0)	FM(FSO_CFE_0_A)	FM(TPU0TO2)	F_(0, 0) FM(FMCLK_C) FM(FMCLK_D) F_(0, 0)
-#define IP18_7_4	FM(GP6_31)		F_(0, 0)	F_(0, 0)		FM(AUDIO_CLKOUT3_B)		FM(SSI_WS9_B)	FM(TS_SPSYNC0_E)FM(STP_ISSYNC_0_E)	F_(0, 0)	FM(RIF2_D1_B)		F_(0, 0)	FM(FSO_CFE_1_A)	FM(TPU0TO3)	F_(0, 0) FM(FMIN_C) FM(FMIN_D) F_(0, 0)
+#define IP17_31_28	FM(USB30_OVC)		F_(0, 0)	F_(0, 0)		FM(AUDIO_CLKOUT1_B)		FM(SSI_WS2_B)	FM(TS_SPSYNC1_D)FM(STP_ISSYNC_1_D)	FM(STP_IVCXO27_0_E)FM(RIF3_D1_B)	F_(0, 0)	FM(FSO_TOE_N)	FM(TPU0TO1)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP18_3_0	FM(GP6_30)		F_(0, 0)	F_(0, 0)		FM(AUDIO_CLKOUT2_B)		FM(SSI_SCK9_B)	FM(TS_SDEN0_E)	FM(STP_ISEN_0_E)	F_(0, 0)	FM(RIF2_D0_B)		F_(0, 0)	F_(0, 0)	FM(TPU0TO2)	FM(FMCLK_C) FM(FMCLK_D) F_(0, 0) F_(0, 0)
+#define IP18_7_4	FM(GP6_31)		F_(0, 0)	F_(0, 0)		FM(AUDIO_CLKOUT3_B)		FM(SSI_WS9_B)	FM(TS_SPSYNC0_E)FM(STP_ISSYNC_0_E)	F_(0, 0)	FM(RIF2_D1_B)		F_(0, 0)	F_(0, 0)	FM(TPU0TO3)	FM(FMIN_C) FM(FMIN_D) F_(0, 0) F_(0, 0)
 
 #define PINMUX_GPSR	\
 \
@@ -419,7 +418,7 @@ FM(IP0_31_28)	IP0_31_28	FM(IP1_31_28)	IP1_31_28	FM(IP2_31_28)	IP2_31_28	FM(IP3_3
 FM(IP4_3_0)	IP4_3_0		FM(IP5_3_0)	IP5_3_0		FM(IP6_3_0)	IP6_3_0		FM(IP7_3_0)	IP7_3_0 \
 FM(IP4_7_4)	IP4_7_4		FM(IP5_7_4)	IP5_7_4		FM(IP6_7_4)	IP6_7_4		FM(IP7_7_4)	IP7_7_4 \
 FM(IP4_11_8)	IP4_11_8	FM(IP5_11_8)	IP5_11_8	FM(IP6_11_8)	IP6_11_8	FM(IP7_11_8)	IP7_11_8 \
-FM(IP4_15_12)	IP4_15_12	FM(IP5_15_12)	IP5_15_12	FM(IP6_15_12)	IP6_15_12	FM(IP7_15_12)	IP7_15_12 \
+FM(IP4_15_12)	IP4_15_12	FM(IP5_15_12)	IP5_15_12	FM(IP6_15_12)	IP6_15_12 \
 FM(IP4_19_16)	IP4_19_16	FM(IP5_19_16)	IP5_19_16	FM(IP6_19_16)	IP6_19_16	FM(IP7_19_16)	IP7_19_16 \
 FM(IP4_23_20)	IP4_23_20	FM(IP5_23_20)	IP5_23_20	FM(IP6_23_20)	IP6_23_20	FM(IP7_23_20)	IP7_23_20 \
 FM(IP4_27_24)	IP4_27_24	FM(IP5_27_24)	IP5_27_24	FM(IP6_27_24)	IP6_27_24	FM(IP7_27_24)	IP7_27_24 \
@@ -463,7 +462,6 @@ FM(IP16_31_28)	IP16_31_28	FM(IP17_31_28)	IP17_31_28
 #define MOD_SEL0_19		FM(SEL_HSCIF4_0)	FM(SEL_HSCIF4_1)
 #define MOD_SEL0_18_17		FM(SEL_HSCIF3_0)	FM(SEL_HSCIF3_1)	FM(SEL_HSCIF3_2)	FM(SEL_HSCIF3_3)
 #define MOD_SEL0_16		FM(SEL_HSCIF1_0)	FM(SEL_HSCIF1_1)
-#define MOD_SEL0_15		FM(SEL_FSO_0)		FM(SEL_FSO_1)
 #define MOD_SEL0_14_13		FM(SEL_HSCIF2_0)	FM(SEL_HSCIF2_1)	FM(SEL_HSCIF2_2)	F_(0, 0)
 #define MOD_SEL0_12		FM(SEL_ETHERAVB_0)	FM(SEL_ETHERAVB_1)
 #define MOD_SEL0_11		FM(SEL_DRIF3_0)		FM(SEL_DRIF3_1)
@@ -472,7 +470,6 @@ FM(IP16_31_28)	IP16_31_28	FM(IP17_31_28)	IP17_31_28
 #define MOD_SEL0_7_6		FM(SEL_DRIF0_0)		FM(SEL_DRIF0_1)		FM(SEL_DRIF0_2)		F_(0, 0)
 #define MOD_SEL0_5		FM(SEL_CANFD0_0)	FM(SEL_CANFD0_1)
 #define MOD_SEL0_4_3		FM(SEL_ADG_A_0)		FM(SEL_ADG_A_1)		FM(SEL_ADG_A_2)		FM(SEL_ADG_A_3)
-#define MOD_SEL0_2		FM(SEL_5LINE_0)		FM(SEL_5LINE_1)
 
 /* MOD_SEL1 */			/* 0 */			/* 1 */			/* 2 */			/* 3 */			/* 4 */			/* 5 */			/* 6 */			/* 7 */
 #define MOD_SEL1_31_30		FM(SEL_TSIF1_0)		FM(SEL_TSIF1_1)		FM(SEL_TSIF1_2)		FM(SEL_TSIF1_3)
@@ -488,7 +485,7 @@ FM(IP16_31_28)	IP16_31_28	FM(IP17_31_28)	IP17_31_28
 #define MOD_SEL1_13		FM(SEL_SCIF3_0)		FM(SEL_SCIF3_1)
 #define MOD_SEL1_12		FM(SEL_SCIF2_0)		FM(SEL_SCIF2_1)
 #define MOD_SEL1_11		FM(SEL_SCIF1_0)		FM(SEL_SCIF1_1)
-#define MOD_SEL1_10		FM(SEL_SATA_0)		FM(SEL_SATA_1)
+#define MOD_SEL1_10		FM(SEL_SCIF_0)		FM(SEL_SCIF_1)
 #define MOD_SEL1_9		FM(SEL_REMOCON_0)	FM(SEL_REMOCON_1)
 #define MOD_SEL1_6		FM(SEL_RCAN0_0)		FM(SEL_RCAN0_1)
 #define MOD_SEL1_5		FM(SEL_PWM6_0)		FM(SEL_PWM6_1)
@@ -529,7 +526,7 @@ MOD_SEL0_19		MOD_SEL1_19		MOD_SEL2_19 \
 MOD_SEL0_18_17		MOD_SEL1_18_17		MOD_SEL2_18 \
 						MOD_SEL2_17 \
 MOD_SEL0_16		MOD_SEL1_16 \
-MOD_SEL0_15		MOD_SEL1_15_14 \
+			MOD_SEL1_15_14 \
 MOD_SEL0_14_13 \
 			MOD_SEL1_13 \
 MOD_SEL0_12		MOD_SEL1_12 \
@@ -541,7 +538,7 @@ MOD_SEL0_7_6 \
 MOD_SEL0_5		MOD_SEL1_5 \
 MOD_SEL0_4_3		MOD_SEL1_4 \
 			MOD_SEL1_3 \
-MOD_SEL0_2		MOD_SEL1_2 \
+			MOD_SEL1_2 \
 			MOD_SEL1_1 \
 			MOD_SEL1_0		MOD_SEL2_0
 
@@ -645,7 +642,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_MSEL(IP0_31_28,	VI4_DATA1_B,		SEL_VIN4_1),
 	PINMUX_IPSR_MSEL(IP0_31_28,	CAN0_RX_B,		SEL_RCAN0_1),
 	PINMUX_IPSR_MSEL(IP0_31_28,	CANFD0_RX_B,		SEL_CANFD0_1),
-	PINMUX_IPSR_MSEL(IP0_27_24,	MSIOF3_SS1_E,		SEL_MSIOF3_4),
+	PINMUX_IPSR_MSEL(IP0_31_28,	MSIOF3_SS1_E,		SEL_MSIOF3_4),
 
 	/* IPSR1 */
 	PINMUX_IPSR_GPSR(IP1_3_0,	IRQ2),
@@ -837,7 +834,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP4_15_12,	CS0_N),
 	PINMUX_IPSR_GPSR(IP4_15_12,	VI5_CLKENB),
 
-	PINMUX_IPSR_GPSR(IP4_19_16,	CS1_N_A26),
+	PINMUX_IPSR_GPSR(IP4_19_16,	CS1_N),
 	PINMUX_IPSR_GPSR(IP4_19_16,	VI5_CLK),
 	PINMUX_IPSR_MSEL(IP4_19_16,	EX_WAIT0_B,		SEL_LBSC_1),
 
@@ -990,8 +987,6 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP7_11_8,	DU_DR7),
 	PINMUX_IPSR_MSEL(IP7_11_8,	SDA6_C,			SEL_I2C6_2),
 
-	PINMUX_IPSR_GPSR(IP7_15_12,	FSCLKST),
-
 	PINMUX_IPSR_GPSR(IP7_19_16,	SD0_CLK),
 	PINMUX_IPSR_MSEL(IP7_19_16,	MSIOF1_SCK_E,		SEL_MSIOF1_4),
 	PINMUX_IPSR_MSEL(IP7_19_16,	STP_OPWM_0_B,		SEL_SSP1_0_1),
@@ -1173,7 +1168,6 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_MSEL(IP12_11_8,	SCL2_A,			SEL_I2C2_0),
 	PINMUX_IPSR_MSEL(IP12_11_8,	STP_IVCXO27_1_C,	SEL_SSP1_1_2),
 	PINMUX_IPSR_MSEL(IP12_11_8,	RIF0_SYNC_B,		SEL_DRIF0_1),
-	PINMUX_IPSR_MSEL(IP12_11_8,	FSO_TOE_A,		SEL_FSO_0),
 	PINMUX_IPSR_GPSR(IP12_11_8,	ADICHS1),
 
 	PINMUX_IPSR_MSEL(IP12_15_12,	RX1_A,			SEL_SCIF1_0),
@@ -1205,7 +1199,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP12_27_24,	ADICHS0),
 
 	PINMUX_IPSR_GPSR(IP12_31_28,	SCK2),
-	PINMUX_IPSR_MSEL(IP12_31_28,	SCIF_CLK_B,		SEL_SCIF1_1),
+	PINMUX_IPSR_MSEL(IP12_31_28,	SCIF_CLK_B,		SEL_SCIF_1),
 	PINMUX_IPSR_MSEL(IP12_31_28,	MSIOF1_SCK_B,		SEL_MSIOF1_1),
 	PINMUX_IPSR_MSEL(IP12_31_28,	TS_SCK1_C,		SEL_TSIF1_2),
 	PINMUX_IPSR_MSEL(IP12_31_28,	STP_ISCLK_1_C,		SEL_SSP1_1_2),
@@ -1218,14 +1212,14 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_MSEL(IP13_3_0,	SCL1_A,			SEL_I2C1_0),
 	PINMUX_IPSR_MSEL(IP13_3_0,	FMCLK_A,		SEL_FM_0),
 	PINMUX_IPSR_MSEL(IP13_3_0,	RIF1_D1_C,		SEL_DRIF1_2),
-	PINMUX_IPSR_MSEL(IP13_3_0,	FSO_CFE_0_B,		SEL_FSO_1),
+	PINMUX_IPSR_GPSR(IP13_3_0,	FSO_CFE_0_N),
 
 	PINMUX_IPSR_MSEL(IP13_7_4,	RX2_A,			SEL_SCIF2_0),
 	PINMUX_IPSR_MSEL(IP13_7_4,	SD2_WP_B,		SEL_SDHI2_1),
 	PINMUX_IPSR_MSEL(IP13_7_4,	SDA1_A,			SEL_I2C1_0),
 	PINMUX_IPSR_MSEL(IP13_7_4,	FMIN_A,			SEL_FM_0),
 	PINMUX_IPSR_MSEL(IP13_7_4,	RIF1_SYNC_C,		SEL_DRIF1_2),
-	PINMUX_IPSR_MSEL(IP13_7_4,	FSO_CFE_1_B,		SEL_FSO_1),
+	PINMUX_IPSR_GPSR(IP13_7_4,	FSO_CFE_1_N),
 
 	PINMUX_IPSR_GPSR(IP13_11_8,	HSCK0),
 	PINMUX_IPSR_MSEL(IP13_11_8,	MSIOF1_SCK_D,		SEL_MSIOF1_3),
@@ -1393,7 +1387,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_MSEL(IP16_23_20,	STP_ISEN_1_A,		SEL_SSP1_1_0),
 	PINMUX_IPSR_MSEL(IP16_23_20,	RIF1_D0_A,		SEL_DRIF1_0),
 	PINMUX_IPSR_MSEL(IP16_23_20,	RIF3_D0_A,		SEL_DRIF3_0),
-	PINMUX_IPSR_MSEL(IP16_23_20,	TCLK2_A,		SEL_TIMER_TMU_0),
+	PINMUX_IPSR_MSEL(IP16_23_20,	TCLK2_A,		SEL_TIMER_TMU2_0),
 
 	PINMUX_IPSR_GPSR(IP16_27_24,	SSI_SDATA8),
 	PINMUX_IPSR_MSEL(IP16_27_24,	HRTS2_N_B,		SEL_HSCIF2_1),
@@ -1410,14 +1404,14 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_MSEL(IP16_31_28,	SSI_WS1_B,		SEL_SSI_1),
 	PINMUX_IPSR_GPSR(IP16_31_28,	SCK1),
 	PINMUX_IPSR_MSEL(IP16_31_28,	STP_IVCXO27_1_A,	SEL_SSP1_1_0),
-	PINMUX_IPSR_GPSR(IP16_31_28,	SCK5_A),
+	PINMUX_IPSR_MSEL(IP16_31_28,	SCK5_A,			SEL_SCIF5_0),
 
 	/* IPSR17 */
 	PINMUX_IPSR_MSEL(IP17_3_0,	AUDIO_CLKA_A,		SEL_ADG_A_0),
 	PINMUX_IPSR_GPSR(IP17_3_0,	CC5_OSCOUT),
 
 	PINMUX_IPSR_MSEL(IP17_7_4,	AUDIO_CLKB_B,		SEL_ADG_B_1),
-	PINMUX_IPSR_MSEL(IP17_7_4,	SCIF_CLK_A,		SEL_SCIF1_0),
+	PINMUX_IPSR_MSEL(IP17_7_4,	SCIF_CLK_A,		SEL_SCIF_0),
 	PINMUX_IPSR_MSEL(IP17_7_4,	STP_IVCXO27_1_D,	SEL_SSP1_1_3),
 	PINMUX_IPSR_MSEL(IP17_7_4,	REMOCON_A,		SEL_REMOCON_0),
 	PINMUX_IPSR_MSEL(IP17_7_4,	TCLK1_A,		SEL_TIMER_TMU_0),
@@ -1461,10 +1455,10 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP17_27_24,	AUDIO_CLKOUT_B),
 	PINMUX_IPSR_MSEL(IP17_27_24,	SSI_SCK2_B,		SEL_SSI_1),
 	PINMUX_IPSR_MSEL(IP17_27_24,	TS_SDEN1_D,		SEL_TSIF1_3),
-	PINMUX_IPSR_MSEL(IP17_27_24,	STP_ISEN_1_D,		SEL_SSP1_1_2),
+	PINMUX_IPSR_MSEL(IP17_27_24,	STP_ISEN_1_D,		SEL_SSP1_1_3),
 	PINMUX_IPSR_MSEL(IP17_27_24,	STP_OPWM_0_E,		SEL_SSP1_0_4),
 	PINMUX_IPSR_MSEL(IP17_27_24,	RIF3_D0_B,		SEL_DRIF3_1),
-	PINMUX_IPSR_MSEL(IP17_27_24,	TCLK2_B,		SEL_TIMER_TMU_1),
+	PINMUX_IPSR_MSEL(IP17_27_24,	TCLK2_B,		SEL_TIMER_TMU2_1),
 	PINMUX_IPSR_GPSR(IP17_27_24,	TPU0TO0),
 	PINMUX_IPSR_MSEL(IP17_27_24,	BPFCLK_C,		SEL_FM_2),
 	PINMUX_IPSR_MSEL(IP17_27_24,	HRTS2_N_C,		SEL_HSCIF2_2),
@@ -1476,7 +1470,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_MSEL(IP17_31_28,	STP_ISSYNC_1_D,		SEL_SSP1_1_3),
 	PINMUX_IPSR_MSEL(IP17_31_28,	STP_IVCXO27_0_E,	SEL_SSP1_0_4),
 	PINMUX_IPSR_MSEL(IP17_31_28,	RIF3_D1_B,		SEL_DRIF3_1),
-	PINMUX_IPSR_MSEL(IP17_31_28,	FSO_TOE_B,		SEL_FSO_1),
+	PINMUX_IPSR_GPSR(IP17_31_28,	FSO_TOE_N),
 	PINMUX_IPSR_GPSR(IP17_31_28,	TPU0TO1),
 
 	/* IPSR18 */
@@ -1487,7 +1481,6 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_MSEL(IP18_3_0,	STP_ISEN_0_E,		SEL_SSP1_0_4),
 	PINMUX_IPSR_MSEL(IP18_3_0,	RIF2_D0_B,		SEL_DRIF2_1),
 	PINMUX_IPSR_GPSR(IP18_3_0,	TPU0TO2),
-	PINMUX_IPSR_MSEL(IP18_3_0,	FSO_CFE_0_A,		SEL_FSO_0),
 	PINMUX_IPSR_MSEL(IP18_3_0,	FMCLK_C,		SEL_FM_2),
 	PINMUX_IPSR_MSEL(IP18_3_0,	FMCLK_D,		SEL_FM_3),
 
@@ -1498,7 +1491,6 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_MSEL(IP18_7_4,	STP_ISSYNC_0_E,		SEL_SSP1_0_4),
 	PINMUX_IPSR_MSEL(IP18_7_4,	RIF2_D1_B,		SEL_DRIF2_1),
 	PINMUX_IPSR_GPSR(IP18_7_4,	TPU0TO3),
-	PINMUX_IPSR_MSEL(IP18_7_4,	FSO_CFE_1_A,		SEL_FSO_0),
 	PINMUX_IPSR_MSEL(IP18_7_4,	FMIN_C,			SEL_FM_2),
 	PINMUX_IPSR_MSEL(IP18_7_4,	FMIN_D,			SEL_FM_3),
 
@@ -3082,7 +3074,7 @@ static const unsigned int msiof3_ss2_e_pins[] = {
 	RCAR_GP_PIN(2, 0),
 };
 static const unsigned int msiof3_ss2_e_mux[] = {
-	MSIOF3_SS1_E_MARK,
+	MSIOF3_SS2_E_MARK,
 };
 static const unsigned int msiof3_txd_e_pins[] = {
 	/* TXD */
@@ -3796,6 +3788,32 @@ static const unsigned int ssi9_ctrl_b_mux[] = {
 	SSI_SCK9_B_MARK, SSI_WS9_B_MARK,
 };
 
+/* - USB0 ------------------------------------------------------------------- */
+static const unsigned int usb0_pins[] = {
+	/* PWEN, OVC */
+	RCAR_GP_PIN(6, 24), RCAR_GP_PIN(6, 25),
+};
+static const unsigned int usb0_mux[] = {
+	USB0_PWEN_MARK, USB0_OVC_MARK,
+};
+/* - USB1 ------------------------------------------------------------------- */
+static const unsigned int usb1_pins[] = {
+	/* PWEN, OVC */
+	RCAR_GP_PIN(6, 26), RCAR_GP_PIN(6, 27),
+};
+static const unsigned int usb1_mux[] = {
+	USB1_PWEN_MARK, USB1_OVC_MARK,
+};
+
+/* - USB30 ------------------------------------------------------------------ */
+static const unsigned int usb30_pins[] = {
+	/* PWEN, OVC */
+	RCAR_GP_PIN(6, 28), RCAR_GP_PIN(6, 29),
+};
+static const unsigned int usb30_mux[] = {
+	USB30_PWEN_MARK, USB30_OVC_MARK,
+};
+
 static const struct sh_pfc_pin_group pinmux_groups[] = {
 	SH_PFC_PIN_GROUP(audio_clk_a_a),
 	SH_PFC_PIN_GROUP(audio_clk_a_b),
@@ -4096,6 +4114,9 @@ static const struct sh_pfc_pin_group pinmux_groups[] = {
 	SH_PFC_PIN_GROUP(ssi9_data_b),
 	SH_PFC_PIN_GROUP(ssi9_ctrl_a),
 	SH_PFC_PIN_GROUP(ssi9_ctrl_b),
+	SH_PFC_PIN_GROUP(usb0),
+	SH_PFC_PIN_GROUP(usb1),
+	SH_PFC_PIN_GROUP(usb30),
 };
 
 static const char * const audio_clk_groups[] = {
@@ -4526,6 +4547,18 @@ static const char * const ssi_groups[] = {
 	"ssi9_ctrl_b",
 };
 
+static const char * const usb0_groups[] = {
+	"usb0",
+};
+
+static const char * const usb1_groups[] = {
+	"usb1",
+};
+
+static const char * const usb30_groups[] = {
+	"usb30",
+};
+
 static const struct sh_pfc_function pinmux_functions[] = {
 	SH_PFC_FUNCTION(audio_clk),
 	SH_PFC_FUNCTION(avb),
@@ -4570,6 +4603,9 @@ static const struct sh_pfc_function pinmux_functions[] = {
 	SH_PFC_FUNCTION(sdhi2),
 	SH_PFC_FUNCTION(sdhi3),
 	SH_PFC_FUNCTION(ssi),
+	SH_PFC_FUNCTION(usb0),
+	SH_PFC_FUNCTION(usb1),
+	SH_PFC_FUNCTION(usb30),
 };
 
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
@@ -4927,7 +4963,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP7_27_24
 		IP7_23_20
 		IP7_19_16
-		IP7_15_12
+		/* IP7_15_12 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		IP7_11_8
 		IP7_7_4
 		IP7_3_0 }
@@ -5060,7 +5096,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		MOD_SEL0_19
 		MOD_SEL0_18_17
 		MOD_SEL0_16
-		MOD_SEL0_15
+		0, 0, /* RESERVED 15 */
 		MOD_SEL0_14_13
 		MOD_SEL0_12
 		MOD_SEL0_11
@@ -5502,7 +5538,7 @@ static const struct sh_pfc_bias_info bias_info[] = {
 	{ RCAR_GP_PIN(1, 24),    PU2,  5 },	/* RD_WR_N */
 	{ RCAR_GP_PIN(1, 23),    PU2,  4 },	/* RD_N */
 	{ RCAR_GP_PIN(1, 22),    PU2,  3 },	/* BS_N */
-	{ RCAR_GP_PIN(1, 21),    PU2,  2 },	/* CS1_N_A26 */
+	{ RCAR_GP_PIN(1, 21),    PU2,  2 },	/* CS1_N */
 	{ RCAR_GP_PIN(1, 20),    PU2,  1 },	/* CS0_N */
 	{ RCAR_GP_PIN(1, 28),    PU2,  0 },	/* CLKOUT */
 
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a77995.c b/drivers/pinctrl/sh-pfc/pfc-r8a77995.c
new file mode 100644
index 000000000000..4f5ee1d7317d
--- /dev/null
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a77995.c
@@ -0,0 +1,1812 @@
+/*
+ * R8A77995 processor support - PFC hardware block.
+ *
+ * Copyright (C) 2017 Renesas Electronics Corp.
+ *
+ * This file is based on the drivers/pinctrl/sh-pfc/pfc-r8a7796.c
+ *
+ * R-Car Gen3 processor support - PFC hardware block.
+ *
+ * Copyright (C) 2015  Renesas Electronics Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ */
+
+#include <linux/kernel.h>
+
+#include "core.h"
+#include "sh_pfc.h"
+
+#define CPU_ALL_PORT(fn, sfx)			\
+		PORT_GP_9(0,  fn, sfx),		\
+		PORT_GP_32(1, fn, sfx),		\
+		PORT_GP_32(2, fn, sfx),		\
+		PORT_GP_CFG_10(3,  fn, sfx, SH_PFC_PIN_CFG_IO_VOLTAGE),	\
+		PORT_GP_32(4, fn, sfx),		\
+		PORT_GP_21(5, fn, sfx),		\
+		PORT_GP_14(6, fn, sfx)
+
+/*
+ * F_() : just information
+ * FM() : macro for FN_xxx / xxx_MARK
+ */
+
+/* GPSR0 */
+#define GPSR0_8		F_(MLB_SIG,		IP0_27_24)
+#define GPSR0_7		F_(MLB_DAT,		IP0_23_20)
+#define GPSR0_6		F_(MLB_CLK,		IP0_19_16)
+#define GPSR0_5		F_(MSIOF2_RXD,		IP0_15_12)
+#define GPSR0_4		F_(MSIOF2_TXD,		IP0_11_8)
+#define GPSR0_3		F_(MSIOF2_SCK,		IP0_7_4)
+#define GPSR0_2		F_(IRQ0_A,		IP0_3_0)
+#define GPSR0_1		FM(USB0_OVC)
+#define GPSR0_0		FM(USB0_PWEN)
+
+/* GPSR1 */
+#define GPSR1_31	F_(QPOLB,		IP4_27_24)
+#define GPSR1_30	F_(QPOLA,		IP4_23_20)
+#define GPSR1_29	F_(DU_CDE,		IP4_19_16)
+#define GPSR1_28	F_(DU_DISP_CDE,		IP4_15_12)
+#define GPSR1_27	F_(DU_DISP,		IP4_11_8)
+#define GPSR1_26	F_(DU_VSYNC,		IP4_7_4)
+#define GPSR1_25	F_(DU_HSYNC,		IP4_3_0)
+#define GPSR1_24	F_(DU_DOTCLKOUT0,	IP3_31_28)
+#define GPSR1_23	F_(DU_DR7,		IP3_27_24)
+#define GPSR1_22	F_(DU_DR6,		IP3_23_20)
+#define GPSR1_21	F_(DU_DR5,		IP3_19_16)
+#define GPSR1_20	F_(DU_DR4,		IP3_15_12)
+#define GPSR1_19	F_(DU_DR3,		IP3_11_8)
+#define GPSR1_18	F_(DU_DR2,		IP3_7_4)
+#define GPSR1_17	F_(DU_DR1,		IP3_3_0)
+#define GPSR1_16	F_(DU_DR0,		IP2_31_28)
+#define GPSR1_15	F_(DU_DG7,		IP2_27_24)
+#define GPSR1_14	F_(DU_DG6,		IP2_23_20)
+#define GPSR1_13	F_(DU_DG5,		IP2_19_16)
+#define GPSR1_12	F_(DU_DG4,		IP2_15_12)
+#define GPSR1_11	F_(DU_DG3,		IP2_11_8)
+#define GPSR1_10	F_(DU_DG2,		IP2_7_4)
+#define GPSR1_9		F_(DU_DG1,		IP2_3_0)
+#define GPSR1_8		F_(DU_DG0,		IP1_31_28)
+#define GPSR1_7		F_(DU_DB7,		IP1_27_24)
+#define GPSR1_6		F_(DU_DB6,		IP1_23_20)
+#define GPSR1_5		F_(DU_DB5,		IP1_19_16)
+#define GPSR1_4		F_(DU_DB4,		IP1_15_12)
+#define GPSR1_3		F_(DU_DB3,		IP1_11_8)
+#define GPSR1_2		F_(DU_DB2,		IP1_7_4)
+#define GPSR1_1		F_(DU_DB1,		IP1_3_0)
+#define GPSR1_0		F_(DU_DB0,		IP0_31_28)
+
+/* GPSR2 */
+#define GPSR2_31	F_(NFCE_N,		IP8_19_16)
+#define GPSR2_30	F_(NFCLE,		IP8_15_12)
+#define GPSR2_29	F_(NFALE,		IP8_11_8)
+#define GPSR2_28	F_(VI4_CLKENB,		IP8_7_4)
+#define GPSR2_27	F_(VI4_FIELD,		IP8_3_0)
+#define GPSR2_26	F_(VI4_HSYNC_N,		IP7_31_28)
+#define GPSR2_25	F_(VI4_VSYNC_N,		IP7_27_24)
+#define GPSR2_24	F_(VI4_DATA23,		IP7_23_20)
+#define GPSR2_23	F_(VI4_DATA22,		IP7_19_16)
+#define GPSR2_22	F_(VI4_DATA21,		IP7_15_12)
+#define GPSR2_21	F_(VI4_DATA20,		IP7_11_8)
+#define GPSR2_20	F_(VI4_DATA19,		IP7_7_4)
+#define GPSR2_19	F_(VI4_DATA18,		IP7_3_0)
+#define GPSR2_18	F_(VI4_DATA17,		IP6_31_28)
+#define GPSR2_17	F_(VI4_DATA16,		IP6_27_24)
+#define GPSR2_16	F_(VI4_DATA15,		IP6_23_20)
+#define GPSR2_15	F_(VI4_DATA14,		IP6_19_16)
+#define GPSR2_14	F_(VI4_DATA13,		IP6_15_12)
+#define GPSR2_13	F_(VI4_DATA12,		IP6_11_8)
+#define GPSR2_12	F_(VI4_DATA11,		IP6_7_4)
+#define GPSR2_11	F_(VI4_DATA10,		IP6_3_0)
+#define GPSR2_10	F_(VI4_DATA9,		IP5_31_28)
+#define GPSR2_9		F_(VI4_DATA8,		IP5_27_24)
+#define GPSR2_8		F_(VI4_DATA7,		IP5_23_20)
+#define GPSR2_7		F_(VI4_DATA6,		IP5_19_16)
+#define GPSR2_6		F_(VI4_DATA5,		IP5_15_12)
+#define GPSR2_5		FM(VI4_DATA4)
+#define GPSR2_4		F_(VI4_DATA3,		IP5_11_8)
+#define GPSR2_3		F_(VI4_DATA2,		IP5_7_4)
+#define GPSR2_2		F_(VI4_DATA1,		IP5_3_0)
+#define GPSR2_1		F_(VI4_DATA0,		IP4_31_28)
+#define GPSR2_0		FM(VI4_CLK)
+
+/* GPSR3 */
+#define GPSR3_9		F_(NFDATA7,		IP9_31_28)
+#define GPSR3_8		F_(NFDATA6,		IP9_27_24)
+#define GPSR3_7		F_(NFDATA5,		IP9_23_20)
+#define GPSR3_6		F_(NFDATA4,		IP9_19_16)
+#define GPSR3_5		F_(NFDATA3,		IP9_15_12)
+#define GPSR3_4		F_(NFDATA2,		IP9_11_8)
+#define GPSR3_3		F_(NFDATA1,		IP9_7_4)
+#define GPSR3_2		F_(NFDATA0,		IP9_3_0)
+#define GPSR3_1		F_(NFWE_N,		IP8_31_28)
+#define GPSR3_0		F_(NFRE_N,		IP8_27_24)
+
+/* GPSR4 */
+#define GPSR4_31	F_(CAN0_RX_A,		IP12_27_24)
+#define GPSR4_30	F_(CAN1_TX_A,		IP13_7_4)
+#define GPSR4_29	F_(CAN1_RX_A,		IP13_3_0)
+#define GPSR4_28	F_(CAN0_TX_A,		IP12_31_28)
+#define GPSR4_27	FM(TX2)
+#define GPSR4_26	FM(RX2)
+#define GPSR4_25	F_(SCK2,		IP12_11_8)
+#define GPSR4_24	F_(TX1_A,		IP12_7_4)
+#define GPSR4_23	F_(RX1_A,		IP12_3_0)
+#define GPSR4_22	F_(SCK1_A,		IP11_31_28)
+#define GPSR4_21	F_(TX0_A,		IP11_27_24)
+#define GPSR4_20	F_(RX0_A,		IP11_23_20)
+#define GPSR4_19	F_(SCK0_A,		IP11_19_16)
+#define GPSR4_18	F_(MSIOF1_RXD,		IP11_15_12)
+#define GPSR4_17	F_(MSIOF1_TXD,		IP11_11_8)
+#define GPSR4_16	F_(MSIOF1_SCK,		IP11_7_4)
+#define GPSR4_15	FM(MSIOF0_RXD)
+#define GPSR4_14	FM(MSIOF0_TXD)
+#define GPSR4_13	FM(MSIOF0_SYNC)
+#define GPSR4_12	FM(MSIOF0_SCK)
+#define GPSR4_11	F_(SDA1,		IP11_3_0)
+#define GPSR4_10	F_(SCL1,		IP10_31_28)
+#define GPSR4_9		FM(SDA0)
+#define GPSR4_8		FM(SCL0)
+#define GPSR4_7		F_(SSI_WS4_A,		IP10_27_24)
+#define GPSR4_6		F_(SSI_SDATA4_A,	IP10_23_20)
+#define GPSR4_5		F_(SSI_SCK4_A,		IP10_19_16)
+#define GPSR4_4		F_(SSI_WS34,		IP10_15_12)
+#define GPSR4_3		F_(SSI_SDATA3,		IP10_11_8)
+#define GPSR4_2		F_(SSI_SCK34,		IP10_7_4)
+#define GPSR4_1		F_(AUDIO_CLKA,		IP10_3_0)
+#define GPSR4_0		F_(NFRB_N,		IP8_23_20)
+
+/* GPSR5 */
+#define GPSR5_20	FM(AVB0_LINK)
+#define GPSR5_19	FM(AVB0_PHY_INT)
+#define GPSR5_18	FM(AVB0_MAGIC)
+#define GPSR5_17	FM(AVB0_MDC)
+#define GPSR5_16	FM(AVB0_MDIO)
+#define GPSR5_15	FM(AVB0_TXCREFCLK)
+#define GPSR5_14	FM(AVB0_TD3)
+#define GPSR5_13	FM(AVB0_TD2)
+#define GPSR5_12	FM(AVB0_TD1)
+#define GPSR5_11	FM(AVB0_TD0)
+#define GPSR5_10	FM(AVB0_TXC)
+#define GPSR5_9		FM(AVB0_TX_CTL)
+#define GPSR5_8		FM(AVB0_RD3)
+#define GPSR5_7		FM(AVB0_RD2)
+#define GPSR5_6		FM(AVB0_RD1)
+#define GPSR5_5		FM(AVB0_RD0)
+#define GPSR5_4		FM(AVB0_RXC)
+#define GPSR5_3		FM(AVB0_RX_CTL)
+#define GPSR5_2		F_(CAN_CLK,		IP12_23_20)
+#define GPSR5_1		F_(TPU0TO1_A,		IP12_19_16)
+#define GPSR5_0		F_(TPU0TO0_A,		IP12_15_12)
+
+/* GPSR6 */
+#define GPSR6_13	FM(RPC_INT_N)
+#define GPSR6_12	FM(RPC_RESET_N)
+#define GPSR6_11	FM(QSPI1_SSL)
+#define GPSR6_10	FM(QSPI1_IO3)
+#define GPSR6_9		FM(QSPI1_IO2)
+#define GPSR6_8		FM(QSPI1_MISO_IO1)
+#define GPSR6_7		FM(QSPI1_MOSI_IO0)
+#define GPSR6_6		FM(QSPI1_SPCLK)
+#define GPSR6_5		FM(QSPI0_SSL)
+#define GPSR6_4		FM(QSPI0_IO3)
+#define GPSR6_3		FM(QSPI0_IO2)
+#define GPSR6_2		FM(QSPI0_MISO_IO1)
+#define GPSR6_1		FM(QSPI0_MOSI_IO0)
+#define GPSR6_0		FM(QSPI0_SPCLK)
+
+/* IPSRx */		/* 0 */			/* 1 */			/* 2 */			/* 3 */		/* 4 */			/* 5 */		/* 6  - F */
+#define IP0_3_0		FM(IRQ0_A)		FM(MSIOF2_SYNC_B)	FM(USB0_IDIN)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_7_4		FM(MSIOF2_SCK)		F_(0, 0)		FM(USB0_IDPU)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_11_8	FM(MSIOF2_TXD)		FM(SCL3_A)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_15_12	FM(MSIOF2_RXD)		FM(SDA3_A)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_19_16	FM(MLB_CLK)		FM(MSIOF2_SYNC_A)	FM(SCK5_A)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_23_20	FM(MLB_DAT)		FM(MSIOF2_SS1)		FM(RX5_A)		FM(SCL3_B)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_27_24	FM(MLB_SIG)		FM(MSIOF2_SS2)		FM(TX5_A)		FM(SDA3_B)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_31_28	FM(DU_DB0)		FM(LCDOUT0)		FM(MSIOF3_TXD_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP1_3_0		FM(DU_DB1)		FM(LCDOUT1)		FM(MSIOF3_RXD_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP1_7_4		FM(DU_DB2)		FM(LCDOUT2)		FM(IRQ0_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP1_11_8	FM(DU_DB3)		FM(LCDOUT3)		FM(SCK5_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP1_15_12	FM(DU_DB4)		FM(LCDOUT4)		FM(RX5_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP1_19_16	FM(DU_DB5)		FM(LCDOUT5)		FM(TX5_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP1_23_20	FM(DU_DB6)		FM(LCDOUT6)		FM(MSIOF3_SS1_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP1_27_24	FM(DU_DB7)		FM(LCDOUT7)		FM(MSIOF3_SS2_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP1_31_28	FM(DU_DG0)		FM(LCDOUT8)		FM(MSIOF3_SCK_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_3_0		FM(DU_DG1)		FM(LCDOUT9)		FM(MSIOF3_SYNC_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_7_4		FM(DU_DG2)		FM(LCDOUT10)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_11_8	FM(DU_DG3)		FM(LCDOUT11)		FM(IRQ1_A)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_15_12	FM(DU_DG4)		FM(LCDOUT12)		FM(HSCK3_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_19_16	FM(DU_DG5)		FM(LCDOUT13)		FM(HTX3_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_23_20	FM(DU_DG6)		FM(LCDOUT14)		FM(HRX3_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_27_24	FM(DU_DG7)		FM(LCDOUT15)		FM(SCK4_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_31_28	FM(DU_DR0)		FM(LCDOUT16)		FM(RX4_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_3_0		FM(DU_DR1)		FM(LCDOUT17)		FM(TX4_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_7_4		FM(DU_DR2)		FM(LCDOUT18)		FM(PWM0_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_11_8	FM(DU_DR3)		FM(LCDOUT19)		FM(PWM1_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_15_12	FM(DU_DR4)		FM(LCDOUT20)		FM(TCLK2_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_19_16	FM(DU_DR5)		FM(LCDOUT21)		FM(NMI)			F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_23_20	FM(DU_DR6)		FM(LCDOUT22)		FM(PWM2_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_27_24	FM(DU_DR7)		FM(LCDOUT23)		FM(TCLK1_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_31_28	FM(DU_DOTCLKOUT0)	FM(QCLK)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+
+/* IPSRx */		/* 0 */			/* 1 */			/* 2 */			/* 3 */		/* 4 */			/* 5 */		/* 6  - F */
+#define IP4_3_0		FM(DU_HSYNC)		FM(QSTH_QHS)		FM(IRQ3_A)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_7_4		FM(DU_VSYNC)		FM(QSTVA_QVS)		FM(IRQ4_A)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_11_8	FM(DU_DISP)		FM(QSTVB_QVE)		FM(PWM3_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_15_12	FM(DU_DISP_CDE)		FM(QCPV_QDE)		FM(IRQ2_B)		FM(DU_DOTCLKIN1)F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_19_16	FM(DU_CDE)		FM(QSTB_QHE)		FM(SCK3_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_23_20	FM(QPOLA)		F_(0, 0)		FM(RX3_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_27_24	FM(QPOLB)		F_(0, 0)		FM(TX3_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_31_28	FM(VI4_DATA0)		FM(PWM0_A)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_3_0		FM(VI4_DATA1)		FM(PWM1_A)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_7_4		FM(VI4_DATA2)		FM(PWM2_A)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_11_8	FM(VI4_DATA3)		FM(PWM3_A)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_15_12	FM(VI4_DATA5)		FM(SCK4_A)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_19_16	FM(VI4_DATA6)		FM(IRQ2_A)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_23_20	FM(VI4_DATA7)		FM(TCLK2_A)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_27_24	FM(VI4_DATA8)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_31_28	FM(VI4_DATA9)		FM(MSIOF3_SS2_A)	FM(IRQ1_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_3_0		FM(VI4_DATA10)		FM(RX4_A)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_7_4		FM(VI4_DATA11)		FM(TX4_A)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_11_8	FM(VI4_DATA12)		FM(TCLK1_A)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_15_12	FM(VI4_DATA13)		FM(MSIOF3_SS1_A)	FM(HCTS3_N)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_19_16	FM(VI4_DATA14)		FM(SSI_SCK4_B)		FM(HRTS3_N)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_23_20	FM(VI4_DATA15)		FM(SSI_SDATA4_B)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_27_24	FM(VI4_DATA16)		FM(HRX3_A)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_31_28	FM(VI4_DATA17)		FM(HTX3_A)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP7_3_0		FM(VI4_DATA18)		FM(HSCK3_A)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP7_7_4		FM(VI4_DATA19)		FM(SSI_WS4_B)		F_(0, 0)		F_(0, 0)	FM(NFDATA15)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP7_11_8	FM(VI4_DATA20)		FM(MSIOF3_SYNC_A)	F_(0, 0)		F_(0, 0)	FM(NFDATA14)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP7_15_12	FM(VI4_DATA21)		FM(MSIOF3_TXD_A)	F_(0, 0)		F_(0, 0)	FM(NFDATA13)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP7_19_16	FM(VI4_DATA22)		FM(MSIOF3_RXD_A)	F_(0, 0)		F_(0, 0)	FM(NFDATA12)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP7_23_20	FM(VI4_DATA23)		FM(MSIOF3_SCK_A)	F_(0, 0)		F_(0, 0)	FM(NFDATA11)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP7_27_24	FM(VI4_VSYNC_N)		FM(SCK1_B)		F_(0, 0)		F_(0, 0)	FM(NFDATA10)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP7_31_28	FM(VI4_HSYNC_N)		FM(RX1_B)		F_(0, 0)		F_(0, 0)	FM(NFDATA9)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+
+/* IPSRx */		/* 0 */			/* 1 */			/* 2 */			/* 3 */		/* 4 */			/* 5 */		/* 6  - F */
+#define IP8_3_0		FM(VI4_FIELD)		FM(AUDIO_CLKB)		FM(IRQ5_A)		FM(SCIF_CLK)	FM(NFDATA8)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP8_7_4		FM(VI4_CLKENB)		FM(TX1_B)		F_(0, 0)		F_(0, 0)	FM(NFWP_N)		FM(DVC_MUTE_A)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP8_11_8	FM(NFALE)		FM(SCL2_B)		FM(IRQ3_B)		FM(PWM0_C)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP8_15_12	FM(NFCLE)		FM(SDA2_B)		FM(SCK3_A)		FM(PWM1_C)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP8_19_16	FM(NFCE_N)		F_(0, 0)		FM(RX3_A)		FM(PWM2_C)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP8_23_20	FM(NFRB_N)		F_(0, 0)		FM(TX3_A)		FM(PWM3_C)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP8_27_24	FM(NFRE_N)		FM(MMC_CMD)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP8_31_28	FM(NFWE_N)		FM(MMC_CLK)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_3_0		FM(NFDATA0)		FM(MMC_D0)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_7_4		FM(NFDATA1)		FM(MMC_D1)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_11_8	FM(NFDATA2)		FM(MMC_D2)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_15_12	FM(NFDATA3)		FM(MMC_D3)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_19_16	FM(NFDATA4)		FM(MMC_D4)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_23_20	FM(NFDATA5)		FM(MMC_D5)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_27_24	FM(NFDATA6)		FM(MMC_D6)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP9_31_28	FM(NFDATA7)		FM(MMC_D7)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_3_0	FM(AUDIO_CLKA)		F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)		FM(DVC_MUTE_B)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_7_4	FM(SSI_SCK34)		FM(FSO_CFE_0_N_A)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_11_8	FM(SSI_SDATA3)		FM(FSO_CFE_1_N_A)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_15_12	FM(SSI_WS34)		FM(FSO_TOE_N_A)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_19_16	FM(SSI_SCK4_A)		FM(HSCK0)		FM(AUDIO_CLKOUT)	FM(CAN0_RX_B)	FM(IRQ4_B)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_23_20	FM(SSI_SDATA4_A)	FM(HTX0)		FM(SCL2_A)		FM(CAN1_RX_B)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_27_24	FM(SSI_WS4_A)		FM(HRX0)		FM(SDA2_A)		FM(CAN1_TX_B)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP10_31_28	FM(SCL1)		FM(CTS1_N)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_3_0	FM(SDA1)		FM(RTS1_N_TANS)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_7_4	FM(MSIOF1_SCK)		FM(AVB0_AVTP_PPS_B)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_11_8	FM(MSIOF1_TXD)		FM(AVB0_AVTP_CAPTURE_B)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_15_12	FM(MSIOF1_RXD)		FM(AVB0_AVTP_MATCH_B)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_19_16	FM(SCK0_A)		FM(MSIOF1_SYNC)		FM(FSO_CFE_0_N_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_23_20	FM(RX0_A)		FM(MSIOF0_SS1)		FM(FSO_CFE_1_N_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_27_24	FM(TX0_A)		FM(MSIOF0_SS2)		FM(FSO_TOE_N_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_31_28	FM(SCK1_A)		FM(MSIOF1_SS2)		FM(TPU0TO2_B)		FM(CAN0_TX_B)	FM(AUDIO_CLKOUT1)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+
+/* IPSRx */		/* 0 */			/* 1 */			/* 2 */			/* 3 */		/* 4 */			/* 5 */		/* 6  - F */
+#define IP12_3_0	FM(RX1_A)		FM(CTS0_N)		FM(TPU0TO0_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP12_7_4	FM(TX1_A)		FM(RTS0_N_TANS)		FM(TPU0TO1_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP12_11_8	FM(SCK2)		FM(MSIOF1_SS1)		FM(TPU0TO3_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP12_15_12	FM(TPU0TO0_A)		FM(AVB0_AVTP_CAPTURE_A)	FM(HCTS0_N)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP12_19_16	FM(TPU0TO1_A)		FM(AVB0_AVTP_MATCH_A)	FM(HRTS0_N)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP12_23_20	FM(CAN_CLK)		FM(AVB0_AVTP_PPS_A)	FM(SCK0_B)		FM(IRQ5_B)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP12_27_24	FM(CAN0_RX_A)		FM(CANFD0_RX)		FM(RX0_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP12_31_28	FM(CAN0_TX_A)		FM(CANFD0_TX)		FM(TX0_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP13_3_0	FM(CAN1_RX_A)		FM(CANFD1_RX)		FM(TPU0TO2_A)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP13_7_4	FM(CAN1_TX_A)		FM(CANFD1_TX)		FM(TPU0TO3_A)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+
+#define PINMUX_GPSR	\
+\
+		GPSR1_31	GPSR2_31			GPSR4_31		 \
+		GPSR1_30	GPSR2_30			GPSR4_30		 \
+		GPSR1_29	GPSR2_29			GPSR4_29		 \
+		GPSR1_28	GPSR2_28			GPSR4_28		 \
+		GPSR1_27	GPSR2_27			GPSR4_27		 \
+		GPSR1_26	GPSR2_26			GPSR4_26		 \
+		GPSR1_25	GPSR2_25			GPSR4_25		 \
+		GPSR1_24	GPSR2_24			GPSR4_24		 \
+		GPSR1_23	GPSR2_23			GPSR4_23		 \
+		GPSR1_22	GPSR2_22			GPSR4_22		 \
+		GPSR1_21	GPSR2_21			GPSR4_21		 \
+		GPSR1_20	GPSR2_20			GPSR4_20	GPSR5_20 \
+		GPSR1_19	GPSR2_19			GPSR4_19	GPSR5_19 \
+		GPSR1_18	GPSR2_18			GPSR4_18	GPSR5_18 \
+		GPSR1_17	GPSR2_17			GPSR4_17	GPSR5_17 \
+		GPSR1_16	GPSR2_16			GPSR4_16	GPSR5_16 \
+		GPSR1_15	GPSR2_15			GPSR4_15	GPSR5_15 \
+		GPSR1_14	GPSR2_14			GPSR4_14	GPSR5_14 \
+		GPSR1_13	GPSR2_13			GPSR4_13	GPSR5_13	GPSR6_13 \
+		GPSR1_12	GPSR2_12			GPSR4_12	GPSR5_12	GPSR6_12 \
+		GPSR1_11	GPSR2_11			GPSR4_11	GPSR5_11	GPSR6_11 \
+		GPSR1_10	GPSR2_10			GPSR4_10	GPSR5_10	GPSR6_10 \
+		GPSR1_9		GPSR2_9		GPSR3_9		GPSR4_9		GPSR5_9		GPSR6_9 \
+GPSR0_8		GPSR1_8		GPSR2_8		GPSR3_8		GPSR4_8		GPSR5_8		GPSR6_8 \
+GPSR0_7		GPSR1_7		GPSR2_7		GPSR3_7		GPSR4_7		GPSR5_7		GPSR6_7 \
+GPSR0_6		GPSR1_6		GPSR2_6		GPSR3_6		GPSR4_6		GPSR5_6		GPSR6_6 \
+GPSR0_5		GPSR1_5		GPSR2_5		GPSR3_5		GPSR4_5		GPSR5_5		GPSR6_5 \
+GPSR0_4		GPSR1_4		GPSR2_4		GPSR3_4		GPSR4_4		GPSR5_4		GPSR6_4 \
+GPSR0_3		GPSR1_3		GPSR2_3		GPSR3_3		GPSR4_3		GPSR5_3		GPSR6_3 \
+GPSR0_2		GPSR1_2		GPSR2_2		GPSR3_2		GPSR4_2		GPSR5_2		GPSR6_2 \
+GPSR0_1		GPSR1_1		GPSR2_1		GPSR3_1		GPSR4_1		GPSR5_1		GPSR6_1 \
+GPSR0_0		GPSR1_0		GPSR2_0		GPSR3_0		GPSR4_0		GPSR5_0		GPSR6_0
+
+#define PINMUX_IPSR				\
+\
+FM(IP0_3_0)	IP0_3_0		FM(IP1_3_0)	IP1_3_0		FM(IP2_3_0)	IP2_3_0		FM(IP3_3_0)	IP3_3_0 \
+FM(IP0_7_4)	IP0_7_4		FM(IP1_7_4)	IP1_7_4		FM(IP2_7_4)	IP2_7_4		FM(IP3_7_4)	IP3_7_4 \
+FM(IP0_11_8)	IP0_11_8	FM(IP1_11_8)	IP1_11_8	FM(IP2_11_8)	IP2_11_8	FM(IP3_11_8)	IP3_11_8 \
+FM(IP0_15_12)	IP0_15_12	FM(IP1_15_12)	IP1_15_12	FM(IP2_15_12)	IP2_15_12	FM(IP3_15_12)	IP3_15_12 \
+FM(IP0_19_16)	IP0_19_16	FM(IP1_19_16)	IP1_19_16	FM(IP2_19_16)	IP2_19_16	FM(IP3_19_16)	IP3_19_16 \
+FM(IP0_23_20)	IP0_23_20	FM(IP1_23_20)	IP1_23_20	FM(IP2_23_20)	IP2_23_20	FM(IP3_23_20)	IP3_23_20 \
+FM(IP0_27_24)	IP0_27_24	FM(IP1_27_24)	IP1_27_24	FM(IP2_27_24)	IP2_27_24	FM(IP3_27_24)	IP3_27_24 \
+FM(IP0_31_28)	IP0_31_28	FM(IP1_31_28)	IP1_31_28	FM(IP2_31_28)	IP2_31_28	FM(IP3_31_28)	IP3_31_28 \
+\
+FM(IP4_3_0)	IP4_3_0		FM(IP5_3_0)	IP5_3_0		FM(IP6_3_0)	IP6_3_0		FM(IP7_3_0)	IP7_3_0 \
+FM(IP4_7_4)	IP4_7_4		FM(IP5_7_4)	IP5_7_4		FM(IP6_7_4)	IP6_7_4		FM(IP7_7_4)	IP7_7_4 \
+FM(IP4_11_8)	IP4_11_8	FM(IP5_11_8)	IP5_11_8	FM(IP6_11_8)	IP6_11_8	FM(IP7_11_8)	IP7_11_8 \
+FM(IP4_15_12)	IP4_15_12	FM(IP5_15_12)	IP5_15_12	FM(IP6_15_12)	IP6_15_12	FM(IP7_15_12)	IP7_15_12 \
+FM(IP4_19_16)	IP4_19_16	FM(IP5_19_16)	IP5_19_16	FM(IP6_19_16)	IP6_19_16	FM(IP7_19_16)	IP7_19_16 \
+FM(IP4_23_20)	IP4_23_20	FM(IP5_23_20)	IP5_23_20	FM(IP6_23_20)	IP6_23_20	FM(IP7_23_20)	IP7_23_20 \
+FM(IP4_27_24)	IP4_27_24	FM(IP5_27_24)	IP5_27_24	FM(IP6_27_24)	IP6_27_24	FM(IP7_27_24)	IP7_27_24 \
+FM(IP4_31_28)	IP4_31_28	FM(IP5_31_28)	IP5_31_28	FM(IP6_31_28)	IP6_31_28	FM(IP7_31_28)	IP7_31_28 \
+\
+FM(IP8_3_0)	IP8_3_0		FM(IP9_3_0)	IP9_3_0		FM(IP10_3_0)	IP10_3_0	FM(IP11_3_0)	IP11_3_0 \
+FM(IP8_7_4)	IP8_7_4		FM(IP9_7_4)	IP9_7_4		FM(IP10_7_4)	IP10_7_4	FM(IP11_7_4)	IP11_7_4 \
+FM(IP8_11_8)	IP8_11_8	FM(IP9_11_8)	IP9_11_8	FM(IP10_11_8)	IP10_11_8	FM(IP11_11_8)	IP11_11_8 \
+FM(IP8_15_12)	IP8_15_12	FM(IP9_15_12)	IP9_15_12	FM(IP10_15_12)	IP10_15_12	FM(IP11_15_12)	IP11_15_12 \
+FM(IP8_19_16)	IP8_19_16	FM(IP9_19_16)	IP9_19_16	FM(IP10_19_16)	IP10_19_16	FM(IP11_19_16)	IP11_19_16 \
+FM(IP8_23_20)	IP8_23_20	FM(IP9_23_20)	IP9_23_20	FM(IP10_23_20)	IP10_23_20	FM(IP11_23_20)	IP11_23_20 \
+FM(IP8_27_24)	IP8_27_24	FM(IP9_27_24)	IP9_27_24	FM(IP10_27_24)	IP10_27_24	FM(IP11_27_24)	IP11_27_24 \
+FM(IP8_31_28)	IP8_31_28	FM(IP9_31_28)	IP9_31_28	FM(IP10_31_28)	IP10_31_28	FM(IP11_31_28)	IP11_31_28 \
+\
+FM(IP12_3_0)	IP12_3_0	FM(IP13_3_0)	IP13_3_0 \
+FM(IP12_7_4)	IP12_7_4	FM(IP13_7_4)	IP13_7_4 \
+FM(IP12_11_8)	IP12_11_8 \
+FM(IP12_15_12)	IP12_15_12 \
+FM(IP12_19_16)	IP12_19_16 \
+FM(IP12_23_20)	IP12_23_20 \
+FM(IP12_27_24)	IP12_27_24 \
+FM(IP12_31_28)	IP12_31_28 \
+
+/* MOD_SEL0 */			/* 0 */			/* 1 */			/* 2 */			/* 3 */
+#define MOD_SEL0_30		FM(SEL_MSIOF2_0)	FM(SEL_MSIOF2_1)
+#define MOD_SEL0_29		FM(SEL_I2C3_0)		FM(SEL_I2C3_1)
+#define MOD_SEL0_28		FM(SEL_SCIF5_0)		FM(SEL_SCIF5_1)
+#define MOD_SEL0_27		FM(SEL_MSIOF3_0)	FM(SEL_MSIOF3_1)
+#define MOD_SEL0_26		FM(SEL_HSCIF3_0)	FM(SEL_HSCIF3_1)
+#define MOD_SEL0_25		FM(SEL_SCIF4_0)		FM(SEL_SCIF4_1)
+#define MOD_SEL0_24_23		FM(SEL_PWM0_0)		FM(SEL_PWM0_1)		FM(SEL_PWM0_2)		FM(SEL_PWM0_3)
+#define MOD_SEL0_22_21		FM(SEL_PWM1_0)		FM(SEL_PWM1_1)		FM(SEL_PWM1_2)		FM(SEL_PWM1_3)
+#define MOD_SEL0_20_19		FM(SEL_PWM2_0)		FM(SEL_PWM2_1)		FM(SEL_PWM2_2)		FM(SEL_PWM2_3)
+#define MOD_SEL0_18_17		FM(SEL_PWM3_0)		FM(SEL_PWM3_1)		FM(SEL_PWM3_2)		FM(SEL_PWM3_3)
+#define MOD_SEL0_15		FM(SEL_IRQ_0_0)		FM(SEL_IRQ_0_1)
+#define MOD_SEL0_14		FM(SEL_IRQ_1_0)		FM(SEL_IRQ_1_1)
+#define MOD_SEL0_13		FM(SEL_IRQ_2_0)		FM(SEL_IRQ_2_1)
+#define MOD_SEL0_12		FM(SEL_IRQ_3_0)		FM(SEL_IRQ_3_1)
+#define MOD_SEL0_11		FM(SEL_IRQ_4_0)		FM(SEL_IRQ_4_1)
+#define MOD_SEL0_10		FM(SEL_IRQ_5_0)		FM(SEL_IRQ_5_1)
+#define MOD_SEL0_5		FM(SEL_TMU_0_0)		FM(SEL_TMU_0_1)
+#define MOD_SEL0_4		FM(SEL_TMU_1_0)		FM(SEL_TMU_1_1)
+#define MOD_SEL0_3		FM(SEL_SCIF3_0)		FM(SEL_SCIF3_1)
+#define MOD_SEL0_2		FM(SEL_SCIF1_0)		FM(SEL_SCIF1_1)
+#define MOD_SEL0_1		FM(SEL_SCU_0)		FM(SEL_SCU_1)
+#define MOD_SEL0_0		FM(SEL_RFSO_0)		FM(SEL_RFSO_1)
+
+#define MOD_SEL1_31		FM(SEL_CAN0_0)		FM(SEL_CAN0_1)
+#define MOD_SEL1_30		FM(SEL_CAN1_0)		FM(SEL_CAN1_1)
+#define MOD_SEL1_29		FM(SEL_I2C2_0)		FM(SEL_I2C2_1)
+#define MOD_SEL1_28		FM(SEL_ETHERAVB_0)	FM(SEL_ETHERAVB_1)
+#define MOD_SEL1_27		FM(SEL_SCIF0_0)		FM(SEL_SCIF0_1)
+#define MOD_SEL1_26		FM(SEL_SSIF4_0)		FM(SEL_SSIF4_1)
+
+
+#define PINMUX_MOD_SELS	\
+\
+		MOD_SEL1_31 \
+MOD_SEL0_30	MOD_SEL1_30 \
+MOD_SEL0_29	MOD_SEL1_29 \
+MOD_SEL0_28	MOD_SEL1_28 \
+MOD_SEL0_27	MOD_SEL1_27 \
+MOD_SEL0_26	MOD_SEL1_26 \
+MOD_SEL0_25 \
+MOD_SEL0_24_23 \
+MOD_SEL0_22_21 \
+MOD_SEL0_20_19 \
+MOD_SEL0_18_17 \
+MOD_SEL0_15 \
+MOD_SEL0_14 \
+MOD_SEL0_13 \
+MOD_SEL0_12 \
+MOD_SEL0_11 \
+MOD_SEL0_10 \
+MOD_SEL0_5 \
+MOD_SEL0_4 \
+MOD_SEL0_3 \
+MOD_SEL0_2 \
+MOD_SEL0_1 \
+MOD_SEL0_0
+
+enum {
+	PINMUX_RESERVED = 0,
+
+	PINMUX_DATA_BEGIN,
+	GP_ALL(DATA),
+	PINMUX_DATA_END,
+
+#define F_(x, y)
+#define FM(x)	FN_##x,
+	PINMUX_FUNCTION_BEGIN,
+	GP_ALL(FN),
+	PINMUX_GPSR
+	PINMUX_IPSR
+	PINMUX_MOD_SELS
+	PINMUX_FUNCTION_END,
+#undef F_
+#undef FM
+
+#define F_(x, y)
+#define FM(x)	x##_MARK,
+	PINMUX_MARK_BEGIN,
+	PINMUX_GPSR
+	PINMUX_IPSR
+	PINMUX_MOD_SELS
+	PINMUX_MARK_END,
+#undef F_
+#undef FM
+};
+
+#define PINMUX_IPSR_MSEL2(ipsr, fn, msel1, msel2) \
+	PINMUX_DATA(fn##_MARK, FN_##msel1, FN_##msel2, FN_##fn, FN_##ipsr)
+
+#define PINMUX_IPSR_PHYS(ipsr, fn, msel) \
+	PINMUX_DATA(fn##_MARK, FN_##msel)
+
+static const u16 pinmux_data[] = {
+	PINMUX_DATA_GP_ALL(),
+
+	PINMUX_SINGLE(USB0_OVC),
+	PINMUX_SINGLE(USB0_PWEN),
+	PINMUX_SINGLE(VI4_DATA4),
+	PINMUX_SINGLE(VI4_CLK),
+	PINMUX_SINGLE(TX2),
+	PINMUX_SINGLE(RX2),
+	PINMUX_SINGLE(AVB0_LINK),
+	PINMUX_SINGLE(AVB0_PHY_INT),
+	PINMUX_SINGLE(AVB0_MAGIC),
+	PINMUX_SINGLE(AVB0_MDC),
+	PINMUX_SINGLE(AVB0_MDIO),
+	PINMUX_SINGLE(AVB0_TXCREFCLK),
+	PINMUX_SINGLE(AVB0_TD3),
+	PINMUX_SINGLE(AVB0_TD2),
+	PINMUX_SINGLE(AVB0_TD1),
+	PINMUX_SINGLE(AVB0_TD0),
+	PINMUX_SINGLE(AVB0_TXC),
+	PINMUX_SINGLE(AVB0_TX_CTL),
+	PINMUX_SINGLE(AVB0_RD3),
+	PINMUX_SINGLE(AVB0_RD2),
+	PINMUX_SINGLE(AVB0_RD1),
+	PINMUX_SINGLE(AVB0_RD0),
+	PINMUX_SINGLE(AVB0_RXC),
+	PINMUX_SINGLE(AVB0_RX_CTL),
+	PINMUX_SINGLE(RPC_INT_N),
+	PINMUX_SINGLE(RPC_RESET_N),
+	PINMUX_SINGLE(QSPI1_SSL),
+	PINMUX_SINGLE(QSPI1_IO3),
+	PINMUX_SINGLE(QSPI1_IO2),
+	PINMUX_SINGLE(QSPI1_MISO_IO1),
+	PINMUX_SINGLE(QSPI1_MOSI_IO0),
+	PINMUX_SINGLE(QSPI1_SPCLK),
+	PINMUX_SINGLE(QSPI0_SSL),
+	PINMUX_SINGLE(QSPI0_IO3),
+	PINMUX_SINGLE(QSPI0_IO2),
+	PINMUX_SINGLE(QSPI0_MISO_IO1),
+	PINMUX_SINGLE(QSPI0_MOSI_IO0),
+	PINMUX_SINGLE(QSPI0_SPCLK),
+
+	/* IPSR0 */
+	PINMUX_IPSR_MSEL(IP0_3_0,	IRQ0_A, SEL_IRQ_0_0),
+	PINMUX_IPSR_MSEL(IP0_3_0,	MSIOF2_SYNC_B, SEL_MSIOF2_1),
+	PINMUX_IPSR_GPSR(IP0_3_0,	USB0_IDIN),
+
+	PINMUX_IPSR_GPSR(IP0_7_4,	MSIOF2_SCK),
+	PINMUX_IPSR_GPSR(IP0_7_4,	USB0_IDPU),
+
+	PINMUX_IPSR_GPSR(IP0_11_8,	MSIOF2_TXD),
+	PINMUX_IPSR_MSEL(IP0_11_8,	SCL3_A, SEL_I2C3_0),
+
+	PINMUX_IPSR_GPSR(IP0_15_12,	MSIOF2_RXD),
+	PINMUX_IPSR_MSEL(IP0_15_12,	SDA3_A, SEL_I2C3_0),
+
+	PINMUX_IPSR_GPSR(IP0_19_16,	MLB_CLK),
+	PINMUX_IPSR_MSEL(IP0_19_16,	MSIOF2_SYNC_A, SEL_MSIOF2_0),
+	PINMUX_IPSR_MSEL(IP0_19_16,	SCK5_A, SEL_SCIF5_0),
+
+	PINMUX_IPSR_GPSR(IP0_23_20,	MLB_DAT),
+	PINMUX_IPSR_GPSR(IP0_23_20,	MSIOF2_SS1),
+	PINMUX_IPSR_MSEL(IP0_23_20,	RX5_A, SEL_SCIF5_0),
+	PINMUX_IPSR_MSEL(IP0_23_20,	SCL3_B, SEL_I2C3_1),
+
+	PINMUX_IPSR_GPSR(IP0_27_24,	MLB_SIG),
+	PINMUX_IPSR_GPSR(IP0_27_24,	MSIOF2_SS2),
+	PINMUX_IPSR_MSEL(IP0_27_24,	TX5_A, SEL_SCIF5_0),
+	PINMUX_IPSR_MSEL(IP0_27_24,	SDA3_B, SEL_I2C3_1),
+
+	PINMUX_IPSR_GPSR(IP0_31_28,	DU_DB0),
+	PINMUX_IPSR_GPSR(IP0_31_28,	LCDOUT0),
+	PINMUX_IPSR_MSEL(IP0_31_28,	MSIOF3_TXD_B, SEL_MSIOF3_1),
+
+	/* IPSR1 */
+	PINMUX_IPSR_GPSR(IP1_3_0,	DU_DB1),
+	PINMUX_IPSR_GPSR(IP1_3_0,	LCDOUT1),
+	PINMUX_IPSR_MSEL(IP1_3_0,	MSIOF3_RXD_B, SEL_MSIOF3_1),
+
+	PINMUX_IPSR_GPSR(IP1_7_4,	DU_DB2),
+	PINMUX_IPSR_GPSR(IP1_7_4,	LCDOUT2),
+	PINMUX_IPSR_MSEL(IP1_7_4,	IRQ0_B, SEL_IRQ_0_1),
+
+	PINMUX_IPSR_GPSR(IP1_11_8,	DU_DB3),
+	PINMUX_IPSR_GPSR(IP1_11_8,	LCDOUT3),
+	PINMUX_IPSR_MSEL(IP1_11_8,	SCK5_B, SEL_SCIF5_1),
+
+	PINMUX_IPSR_GPSR(IP1_15_12,	DU_DB4),
+	PINMUX_IPSR_GPSR(IP1_15_12,	LCDOUT4),
+	PINMUX_IPSR_MSEL(IP1_15_12,	RX5_B, SEL_SCIF5_1),
+
+	PINMUX_IPSR_GPSR(IP1_19_16,	DU_DB5),
+	PINMUX_IPSR_GPSR(IP1_19_16,	LCDOUT5),
+	PINMUX_IPSR_MSEL(IP1_19_16,	TX5_B, SEL_SCIF5_1),
+
+	PINMUX_IPSR_GPSR(IP1_23_20,	DU_DB6),
+	PINMUX_IPSR_GPSR(IP1_23_20,	LCDOUT6),
+	PINMUX_IPSR_MSEL(IP1_23_20,	MSIOF3_SS1_B, SEL_MSIOF3_1),
+
+	PINMUX_IPSR_GPSR(IP1_27_24,	DU_DB7),
+	PINMUX_IPSR_GPSR(IP1_27_24,	LCDOUT7),
+	PINMUX_IPSR_MSEL(IP1_27_24,	MSIOF3_SS2_B, SEL_MSIOF3_1),
+
+	PINMUX_IPSR_GPSR(IP1_31_28,	DU_DG0),
+	PINMUX_IPSR_GPSR(IP1_31_28,	LCDOUT8),
+	PINMUX_IPSR_MSEL(IP1_31_28,	MSIOF3_SCK_B, SEL_MSIOF3_1),
+
+	/* IPSR2 */
+	PINMUX_IPSR_GPSR(IP2_3_0,	DU_DG1),
+	PINMUX_IPSR_GPSR(IP2_3_0,	LCDOUT9),
+	PINMUX_IPSR_MSEL(IP2_3_0,	MSIOF3_SYNC_B, SEL_MSIOF3_1),
+
+	PINMUX_IPSR_GPSR(IP2_7_4,	DU_DG2),
+	PINMUX_IPSR_GPSR(IP2_7_4,	LCDOUT10),
+
+	PINMUX_IPSR_GPSR(IP2_11_8,	DU_DG3),
+	PINMUX_IPSR_GPSR(IP2_11_8,	LCDOUT11),
+	PINMUX_IPSR_MSEL(IP2_11_8,	IRQ1_A, SEL_IRQ_1_0),
+
+	PINMUX_IPSR_GPSR(IP2_15_12,	DU_DG4),
+	PINMUX_IPSR_GPSR(IP2_15_12,	LCDOUT12),
+	PINMUX_IPSR_MSEL(IP2_15_12,	HSCK3_B, SEL_HSCIF3_1),
+
+	PINMUX_IPSR_GPSR(IP2_19_16,	DU_DG5),
+	PINMUX_IPSR_GPSR(IP2_19_16,	LCDOUT13),
+	PINMUX_IPSR_MSEL(IP2_19_16,	HTX3_B, SEL_HSCIF3_1),
+
+	PINMUX_IPSR_GPSR(IP2_23_20,	DU_DG6),
+	PINMUX_IPSR_GPSR(IP2_23_20,	LCDOUT14),
+	PINMUX_IPSR_MSEL(IP2_23_20,	HRX3_B, SEL_HSCIF3_1),
+
+	PINMUX_IPSR_GPSR(IP2_27_24,	DU_DG7),
+	PINMUX_IPSR_GPSR(IP2_27_24,	LCDOUT15),
+	PINMUX_IPSR_MSEL(IP2_27_24,	SCK4_B, SEL_SCIF4_1),
+
+	PINMUX_IPSR_GPSR(IP2_31_28,	DU_DR0),
+	PINMUX_IPSR_GPSR(IP2_31_28,	LCDOUT16),
+	PINMUX_IPSR_MSEL(IP2_31_28,	RX4_B, SEL_SCIF4_1),
+
+	/* IPSR3 */
+	PINMUX_IPSR_GPSR(IP3_3_0,	DU_DR1),
+	PINMUX_IPSR_GPSR(IP3_3_0,	LCDOUT17),
+	PINMUX_IPSR_MSEL(IP3_3_0,	TX4_B, SEL_SCIF4_1),
+
+	PINMUX_IPSR_GPSR(IP3_7_4,	DU_DR2),
+	PINMUX_IPSR_GPSR(IP3_7_4,	LCDOUT18),
+	PINMUX_IPSR_MSEL(IP3_7_4,	PWM0_B, SEL_PWM0_2),
+
+	PINMUX_IPSR_GPSR(IP3_11_8,	DU_DR3),
+	PINMUX_IPSR_GPSR(IP3_11_8,	LCDOUT19),
+	PINMUX_IPSR_MSEL(IP3_11_8,	PWM1_B, SEL_PWM1_2),
+
+	PINMUX_IPSR_GPSR(IP3_15_12,	DU_DR4),
+	PINMUX_IPSR_GPSR(IP3_15_12,	LCDOUT20),
+	PINMUX_IPSR_MSEL(IP3_15_12,	TCLK2_B, SEL_TMU_0_1),
+
+	PINMUX_IPSR_GPSR(IP3_19_16,	DU_DR5),
+	PINMUX_IPSR_GPSR(IP3_19_16,	LCDOUT21),
+	PINMUX_IPSR_GPSR(IP3_19_16,	NMI),
+
+	PINMUX_IPSR_GPSR(IP3_23_20,	DU_DR6),
+	PINMUX_IPSR_GPSR(IP3_23_20,	LCDOUT22),
+	PINMUX_IPSR_MSEL(IP3_23_20,	PWM2_B, SEL_PWM2_2),
+
+	PINMUX_IPSR_GPSR(IP3_27_24,	DU_DR7),
+	PINMUX_IPSR_GPSR(IP3_27_24,	LCDOUT23),
+	PINMUX_IPSR_MSEL(IP3_27_24,	TCLK1_B, SEL_TMU_1_1),
+
+	PINMUX_IPSR_GPSR(IP3_31_28,	DU_DOTCLKOUT0),
+	PINMUX_IPSR_GPSR(IP3_31_28,	QCLK),
+
+	/* IPSR4 */
+	PINMUX_IPSR_GPSR(IP4_3_0,	DU_HSYNC),
+	PINMUX_IPSR_GPSR(IP4_3_0,	QSTH_QHS),
+	PINMUX_IPSR_MSEL(IP4_3_0,	IRQ3_A, SEL_IRQ_3_0),
+
+	PINMUX_IPSR_GPSR(IP4_7_4,	DU_VSYNC),
+	PINMUX_IPSR_GPSR(IP4_7_4,	QSTVA_QVS),
+	PINMUX_IPSR_MSEL(IP4_7_4,	IRQ4_A, SEL_IRQ_4_0),
+
+	PINMUX_IPSR_GPSR(IP4_11_8,	DU_DISP),
+	PINMUX_IPSR_GPSR(IP4_11_8,	QSTVB_QVE),
+	PINMUX_IPSR_MSEL(IP4_11_8,	PWM3_B, SEL_PWM3_2),
+
+	PINMUX_IPSR_GPSR(IP4_15_12,	DU_DISP_CDE),
+	PINMUX_IPSR_GPSR(IP4_15_12,	QCPV_QDE),
+	PINMUX_IPSR_MSEL(IP4_15_12,	IRQ2_B, SEL_IRQ_2_1),
+	PINMUX_IPSR_GPSR(IP4_15_12,	DU_DOTCLKIN1),
+
+	PINMUX_IPSR_GPSR(IP4_19_16,	DU_CDE),
+	PINMUX_IPSR_GPSR(IP4_19_16,	QSTB_QHE),
+	PINMUX_IPSR_MSEL(IP4_19_16,	SCK3_B, SEL_SCIF3_1),
+
+	PINMUX_IPSR_GPSR(IP4_23_20,	QPOLA),
+	PINMUX_IPSR_MSEL(IP4_23_20,	RX3_B, SEL_SCIF3_1),
+
+	PINMUX_IPSR_GPSR(IP4_27_24,	QPOLB),
+	PINMUX_IPSR_MSEL(IP4_27_24,	TX3_B, SEL_SCIF3_1),
+
+	PINMUX_IPSR_GPSR(IP4_31_28,	VI4_DATA0),
+	PINMUX_IPSR_MSEL(IP4_31_28,	PWM0_A, SEL_PWM0_0),
+
+	/* IPSR5 */
+	PINMUX_IPSR_GPSR(IP5_3_0,	VI4_DATA1),
+	PINMUX_IPSR_MSEL(IP5_3_0,	PWM1_A, SEL_PWM1_0),
+
+	PINMUX_IPSR_GPSR(IP5_7_4,	VI4_DATA2),
+	PINMUX_IPSR_MSEL(IP5_7_4,	PWM2_A, SEL_PWM2_0),
+
+	PINMUX_IPSR_GPSR(IP5_11_8,	VI4_DATA3),
+	PINMUX_IPSR_MSEL(IP5_11_8,	PWM3_A, SEL_PWM3_0),
+
+	PINMUX_IPSR_GPSR(IP5_15_12,	VI4_DATA5),
+	PINMUX_IPSR_MSEL(IP5_15_12,	SCK4_A, SEL_SCIF4_0),
+
+	PINMUX_IPSR_GPSR(IP5_19_16,	VI4_DATA6),
+	PINMUX_IPSR_MSEL(IP5_19_16,	IRQ2_A, SEL_IRQ_2_0),
+
+	PINMUX_IPSR_GPSR(IP5_23_20,	VI4_DATA7),
+	PINMUX_IPSR_MSEL(IP5_23_20,	TCLK2_A, SEL_TMU_0_0),
+
+	PINMUX_IPSR_GPSR(IP5_27_24,	VI4_DATA8),
+
+	PINMUX_IPSR_GPSR(IP5_31_28,	VI4_DATA9),
+	PINMUX_IPSR_MSEL(IP5_31_28,	MSIOF3_SS2_A, SEL_MSIOF3_0),
+	PINMUX_IPSR_MSEL(IP5_31_28,	IRQ1_B, SEL_IRQ_1_1),
+
+	/* IPSR6 */
+	PINMUX_IPSR_GPSR(IP6_3_0,	VI4_DATA10),
+	PINMUX_IPSR_MSEL(IP6_3_0,	RX4_A, SEL_SCIF4_0),
+
+	PINMUX_IPSR_GPSR(IP6_7_4,	VI4_DATA11),
+	PINMUX_IPSR_MSEL(IP6_7_4,	TX4_A, SEL_SCIF4_0),
+
+	PINMUX_IPSR_GPSR(IP6_11_8,	VI4_DATA12),
+	PINMUX_IPSR_MSEL(IP6_11_8,	TCLK1_A, SEL_TMU_1_0),
+
+	PINMUX_IPSR_GPSR(IP6_15_12,	VI4_DATA13),
+	PINMUX_IPSR_MSEL(IP6_15_12,	MSIOF3_SS1_A, SEL_MSIOF3_0),
+	PINMUX_IPSR_GPSR(IP6_15_12,	HCTS3_N),
+
+	PINMUX_IPSR_GPSR(IP6_19_16,	VI4_DATA14),
+	PINMUX_IPSR_MSEL(IP6_19_16,	SSI_SCK4_B, SEL_SSIF4_1),
+	PINMUX_IPSR_GPSR(IP6_19_16,	HRTS3_N),
+
+	PINMUX_IPSR_GPSR(IP6_23_20,	VI4_DATA15),
+	PINMUX_IPSR_MSEL(IP6_23_20,	SSI_SDATA4_B, SEL_SSIF4_1),
+
+	PINMUX_IPSR_GPSR(IP6_27_24,	VI4_DATA16),
+	PINMUX_IPSR_MSEL(IP6_27_24,	HRX3_A, SEL_HSCIF3_0),
+
+	PINMUX_IPSR_GPSR(IP6_31_28,	VI4_DATA17),
+	PINMUX_IPSR_MSEL(IP6_31_28,	HTX3_A, SEL_HSCIF3_0),
+
+	/* IPSR7 */
+	PINMUX_IPSR_GPSR(IP7_3_0,	VI4_DATA18),
+	PINMUX_IPSR_MSEL(IP7_3_0,	HSCK3_A, SEL_HSCIF3_0),
+
+	PINMUX_IPSR_GPSR(IP7_7_4,	VI4_DATA19),
+	PINMUX_IPSR_MSEL(IP7_7_4,	SSI_WS4_B, SEL_SSIF4_1),
+	PINMUX_IPSR_GPSR(IP7_7_4,	NFDATA15),
+
+	PINMUX_IPSR_GPSR(IP7_11_8,	VI4_DATA20),
+	PINMUX_IPSR_MSEL(IP7_11_8,	MSIOF3_SYNC_A, SEL_MSIOF3_0),
+	PINMUX_IPSR_GPSR(IP7_11_8,	NFDATA14),
+
+	PINMUX_IPSR_GPSR(IP7_15_12,	VI4_DATA21),
+	PINMUX_IPSR_MSEL(IP7_15_12,	MSIOF3_TXD_A, SEL_MSIOF3_0),
+
+	PINMUX_IPSR_GPSR(IP7_15_12,	NFDATA13),
+	PINMUX_IPSR_GPSR(IP7_19_16,	VI4_DATA22),
+	PINMUX_IPSR_MSEL(IP7_19_16,	MSIOF3_RXD_A, SEL_MSIOF3_0),
+
+	PINMUX_IPSR_GPSR(IP7_19_16,	NFDATA12),
+	PINMUX_IPSR_GPSR(IP7_23_20,	VI4_DATA23),
+	PINMUX_IPSR_MSEL(IP7_23_20,	MSIOF3_SCK_A, SEL_MSIOF3_0),
+
+	PINMUX_IPSR_GPSR(IP7_23_20,	NFDATA11),
+
+	PINMUX_IPSR_GPSR(IP7_27_24,	VI4_VSYNC_N),
+	PINMUX_IPSR_MSEL(IP7_27_24,	SCK1_B, SEL_SCIF1_1),
+	PINMUX_IPSR_GPSR(IP7_27_24,	NFDATA10),
+
+	PINMUX_IPSR_GPSR(IP7_31_28,	VI4_HSYNC_N),
+	PINMUX_IPSR_MSEL(IP7_31_28,	RX1_B, SEL_SCIF1_1),
+	PINMUX_IPSR_GPSR(IP7_31_28,	NFDATA9),
+
+	/* IPSR8 */
+	PINMUX_IPSR_GPSR(IP8_3_0,	VI4_FIELD),
+	PINMUX_IPSR_GPSR(IP8_3_0,	AUDIO_CLKB),
+	PINMUX_IPSR_MSEL(IP8_3_0,	IRQ5_A, SEL_IRQ_5_0),
+	PINMUX_IPSR_GPSR(IP8_3_0,	SCIF_CLK),
+	PINMUX_IPSR_GPSR(IP8_3_0,	NFDATA8),
+
+	PINMUX_IPSR_GPSR(IP8_7_4,	VI4_CLKENB),
+	PINMUX_IPSR_MSEL(IP8_7_4,	TX1_B, SEL_SCIF1_1),
+	PINMUX_IPSR_GPSR(IP8_7_4,	NFWP_N),
+	PINMUX_IPSR_MSEL(IP8_7_4,	DVC_MUTE_A, SEL_SCU_0),
+
+	PINMUX_IPSR_GPSR(IP8_11_8,	NFALE),
+	PINMUX_IPSR_MSEL(IP8_11_8,	SCL2_B, SEL_I2C2_1),
+	PINMUX_IPSR_MSEL(IP8_11_8,	IRQ3_B, SEL_IRQ_3_1),
+	PINMUX_IPSR_MSEL(IP8_11_8,	PWM0_C, SEL_PWM0_1),
+
+	PINMUX_IPSR_GPSR(IP8_15_12,	NFCLE),
+	PINMUX_IPSR_MSEL(IP8_15_12,	SDA2_B, SEL_I2C2_1),
+	PINMUX_IPSR_MSEL(IP8_15_12,	SCK3_A, SEL_SCIF3_0),
+	PINMUX_IPSR_MSEL(IP8_15_12,	PWM1_C, SEL_PWM1_1),
+
+	PINMUX_IPSR_GPSR(IP8_19_16,	NFCE_N),
+	PINMUX_IPSR_MSEL(IP8_19_16,	RX3_A, SEL_SCIF3_0),
+	PINMUX_IPSR_MSEL(IP8_19_16,	PWM2_C, SEL_PWM2_1),
+
+	PINMUX_IPSR_GPSR(IP8_23_20,	NFRB_N),
+	PINMUX_IPSR_MSEL(IP8_23_20,	TX3_A, SEL_SCIF3_0),
+	PINMUX_IPSR_MSEL(IP8_23_20,	PWM3_C, SEL_PWM3_1),
+
+	PINMUX_IPSR_GPSR(IP8_27_24,	NFRE_N),
+	PINMUX_IPSR_GPSR(IP8_27_24,	MMC_CMD),
+
+	PINMUX_IPSR_GPSR(IP8_31_28,	NFWE_N),
+	PINMUX_IPSR_GPSR(IP8_31_28,	MMC_CLK),
+
+	/* IPSR9 */
+	PINMUX_IPSR_GPSR(IP9_3_0,	NFDATA0),
+	PINMUX_IPSR_GPSR(IP9_3_0,	MMC_D0),
+
+	PINMUX_IPSR_GPSR(IP9_7_4,	NFDATA1),
+	PINMUX_IPSR_GPSR(IP9_7_4,	MMC_D1),
+
+	PINMUX_IPSR_GPSR(IP9_11_8,	NFDATA2),
+	PINMUX_IPSR_GPSR(IP9_11_8,	MMC_D2),
+
+	PINMUX_IPSR_GPSR(IP9_15_12,	NFDATA3),
+	PINMUX_IPSR_GPSR(IP9_15_12,	MMC_D3),
+
+	PINMUX_IPSR_GPSR(IP9_19_16,	NFDATA4),
+	PINMUX_IPSR_GPSR(IP9_19_16,	MMC_D4),
+
+	PINMUX_IPSR_GPSR(IP9_23_20,	NFDATA5),
+	PINMUX_IPSR_GPSR(IP9_23_20,	MMC_D5),
+
+	PINMUX_IPSR_GPSR(IP9_27_24,	NFDATA6),
+	PINMUX_IPSR_GPSR(IP9_27_24,	MMC_D6),
+
+	PINMUX_IPSR_GPSR(IP9_31_28,	NFDATA7),
+	PINMUX_IPSR_GPSR(IP9_31_28,	MMC_D7),
+
+	/* IPSR10 */
+	PINMUX_IPSR_GPSR(IP10_3_0,	AUDIO_CLKA),
+	PINMUX_IPSR_MSEL(IP10_3_0,	DVC_MUTE_B, SEL_SCU_1),
+
+	PINMUX_IPSR_GPSR(IP10_7_4,	SSI_SCK34),
+	PINMUX_IPSR_MSEL(IP10_7_4,	FSO_CFE_0_N_A, SEL_RFSO_0),
+
+	PINMUX_IPSR_GPSR(IP10_11_8,	SSI_SDATA3),
+	PINMUX_IPSR_MSEL(IP10_11_8,	FSO_CFE_1_N_A, SEL_RFSO_0),
+
+	PINMUX_IPSR_GPSR(IP10_15_12,	SSI_WS34),
+	PINMUX_IPSR_MSEL(IP10_15_12,	FSO_TOE_N_A, SEL_RFSO_0),
+
+	PINMUX_IPSR_MSEL(IP10_19_16,	SSI_SCK4_A, SEL_SSIF4_0),
+	PINMUX_IPSR_GPSR(IP10_19_16,	HSCK0),
+	PINMUX_IPSR_GPSR(IP10_19_16,	AUDIO_CLKOUT),
+	PINMUX_IPSR_MSEL(IP10_19_16,	CAN0_RX_B, SEL_CAN0_1),
+	PINMUX_IPSR_MSEL(IP10_19_16,	IRQ4_B, SEL_IRQ_4_1),
+
+	PINMUX_IPSR_MSEL(IP10_23_20,	SSI_SDATA4_A, SEL_SSIF4_0),
+	PINMUX_IPSR_GPSR(IP10_23_20,	HTX0),
+	PINMUX_IPSR_MSEL(IP10_23_20,	SCL2_A, SEL_I2C2_0),
+	PINMUX_IPSR_MSEL(IP10_23_20,	CAN1_RX_B, SEL_CAN1_1),
+
+	PINMUX_IPSR_MSEL(IP10_27_24,	SSI_WS4_A, SEL_SSIF4_0),
+	PINMUX_IPSR_GPSR(IP10_27_24,	HRX0),
+	PINMUX_IPSR_MSEL(IP10_27_24,	SDA2_A, SEL_I2C2_0),
+	PINMUX_IPSR_MSEL(IP10_27_24,	CAN1_TX_B, SEL_CAN1_1),
+
+	PINMUX_IPSR_GPSR(IP10_31_28,	SCL1),
+	PINMUX_IPSR_GPSR(IP10_31_28,	CTS1_N),
+
+	/* IPSR11 */
+	PINMUX_IPSR_GPSR(IP11_3_0,	SDA1),
+	PINMUX_IPSR_GPSR(IP11_3_0,	RTS1_N_TANS),
+
+	PINMUX_IPSR_GPSR(IP11_7_4,	MSIOF1_SCK),
+	PINMUX_IPSR_MSEL(IP11_7_4,	AVB0_AVTP_PPS_B, SEL_ETHERAVB_1),
+
+	PINMUX_IPSR_GPSR(IP11_11_8,	MSIOF1_TXD),
+	PINMUX_IPSR_MSEL(IP11_11_8,	AVB0_AVTP_CAPTURE_B, SEL_ETHERAVB_1),
+
+	PINMUX_IPSR_GPSR(IP11_15_12,	MSIOF1_RXD),
+	PINMUX_IPSR_MSEL(IP11_15_12,	AVB0_AVTP_MATCH_B, SEL_ETHERAVB_1),
+
+	PINMUX_IPSR_MSEL(IP11_19_16,	SCK0_A, SEL_SCIF0_0),
+	PINMUX_IPSR_GPSR(IP11_19_16,	MSIOF1_SYNC),
+	PINMUX_IPSR_MSEL(IP11_19_16,	FSO_CFE_0_N_B, SEL_RFSO_1),
+
+	PINMUX_IPSR_MSEL(IP11_23_20,	RX0_A, SEL_SCIF0_0),
+	PINMUX_IPSR_GPSR(IP11_23_20,	MSIOF0_SS1),
+	PINMUX_IPSR_MSEL(IP11_23_20,	FSO_CFE_1_N_B, SEL_RFSO_1),
+
+	PINMUX_IPSR_MSEL(IP11_27_24,	TX0_A, SEL_SCIF0_0),
+	PINMUX_IPSR_GPSR(IP11_27_24,	MSIOF0_SS2),
+	PINMUX_IPSR_MSEL(IP11_27_24,	FSO_TOE_N_B, SEL_RFSO_1),
+
+	PINMUX_IPSR_MSEL(IP11_31_28,	SCK1_A, SEL_SCIF1_0),
+	PINMUX_IPSR_GPSR(IP11_31_28,	MSIOF1_SS2),
+	PINMUX_IPSR_GPSR(IP11_31_28,	TPU0TO2_B),
+	PINMUX_IPSR_MSEL(IP11_31_28,	CAN0_TX_B, SEL_CAN0_1),
+	PINMUX_IPSR_GPSR(IP11_31_28,	AUDIO_CLKOUT1),
+
+	/* IPSR12 */
+	PINMUX_IPSR_MSEL(IP12_3_0,	RX1_A, SEL_SCIF1_0),
+	PINMUX_IPSR_GPSR(IP12_3_0,	CTS0_N),
+	PINMUX_IPSR_GPSR(IP12_3_0,	TPU0TO0_B),
+
+	PINMUX_IPSR_MSEL(IP12_7_4,	TX1_A, SEL_SCIF1_0),
+	PINMUX_IPSR_GPSR(IP12_7_4,	RTS0_N_TANS),
+	PINMUX_IPSR_GPSR(IP12_7_4,	TPU0TO1_B),
+
+	PINMUX_IPSR_GPSR(IP12_11_8,	SCK2),
+	PINMUX_IPSR_GPSR(IP12_11_8,	MSIOF1_SS1),
+	PINMUX_IPSR_GPSR(IP12_11_8,	TPU0TO3_B),
+
+	PINMUX_IPSR_GPSR(IP12_15_12,	TPU0TO0_A),
+	PINMUX_IPSR_MSEL(IP12_15_12,	AVB0_AVTP_CAPTURE_A, SEL_ETHERAVB_0),
+	PINMUX_IPSR_GPSR(IP12_15_12,	HCTS0_N),
+
+	PINMUX_IPSR_GPSR(IP12_19_16,	TPU0TO1_A),
+	PINMUX_IPSR_MSEL(IP12_19_16,	AVB0_AVTP_MATCH_A, SEL_ETHERAVB_0),
+	PINMUX_IPSR_GPSR(IP12_19_16,	HRTS0_N),
+
+	PINMUX_IPSR_GPSR(IP12_23_20,	CAN_CLK),
+	PINMUX_IPSR_MSEL(IP12_23_20,	AVB0_AVTP_PPS_A, SEL_ETHERAVB_0),
+	PINMUX_IPSR_MSEL(IP12_23_20,	SCK0_B, SEL_SCIF0_1),
+	PINMUX_IPSR_MSEL(IP12_23_20,	IRQ5_B, SEL_IRQ_5_1),
+
+	PINMUX_IPSR_MSEL(IP12_27_24,	CAN0_RX_A, SEL_CAN0_0),
+	PINMUX_IPSR_GPSR(IP12_27_24,	CANFD0_RX),
+	PINMUX_IPSR_MSEL(IP12_27_24,	RX0_B, SEL_SCIF0_1),
+
+	PINMUX_IPSR_MSEL(IP12_31_28,	CAN0_TX_A, SEL_CAN0_0),
+	PINMUX_IPSR_GPSR(IP12_31_28,	CANFD0_TX),
+	PINMUX_IPSR_MSEL(IP12_31_28,	TX0_B, SEL_SCIF0_1),
+
+	/* IPSR13 */
+	PINMUX_IPSR_MSEL(IP13_3_0,	CAN1_RX_A, SEL_CAN1_0),
+	PINMUX_IPSR_GPSR(IP13_3_0,	CANFD1_RX),
+	PINMUX_IPSR_GPSR(IP13_3_0,	TPU0TO2_A),
+
+	PINMUX_IPSR_MSEL(IP13_7_4,	CAN1_TX_A, SEL_CAN1_0),
+	PINMUX_IPSR_GPSR(IP13_7_4,	CANFD1_TX),
+	PINMUX_IPSR_GPSR(IP13_7_4,	TPU0TO3_A),
+};
+
+static const struct sh_pfc_pin pinmux_pins[] = {
+	PINMUX_GPIO_GP_ALL(),
+};
+
+/* - I2C -------------------------------------------------------------------- */
+static const unsigned int i2c0_pins[] = {
+	/* SCL, SDA */
+	RCAR_GP_PIN(4, 8), RCAR_GP_PIN(4, 9),
+};
+static const unsigned int i2c0_mux[] = {
+	SCL0_MARK, SDA0_MARK,
+};
+static const unsigned int i2c1_pins[] = {
+	/* SCL, SDA */
+	RCAR_GP_PIN(4, 10), RCAR_GP_PIN(4, 11),
+};
+static const unsigned int i2c1_mux[] = {
+	SCL1_MARK, SDA1_MARK,
+};
+static const unsigned int i2c2_a_pins[] = {
+	/* SCL, SDA */
+	RCAR_GP_PIN(4, 6), RCAR_GP_PIN(4, 7),
+};
+static const unsigned int i2c2_a_mux[] = {
+	SCL2_A_MARK, SDA2_A_MARK,
+};
+static const unsigned int i2c2_b_pins[] = {
+	/* SCL, SDA */
+	RCAR_GP_PIN(2, 29), RCAR_GP_PIN(2, 30),
+};
+static const unsigned int i2c2_b_mux[] = {
+	SCL2_B_MARK, SDA2_B_MARK,
+};
+static const unsigned int i2c3_a_pins[] = {
+	/* SCL, SDA */
+	RCAR_GP_PIN(0, 4), RCAR_GP_PIN(0, 5),
+};
+static const unsigned int i2c3_a_mux[] = {
+	SCL3_A_MARK, SDA3_A_MARK,
+};
+static const unsigned int i2c3_b_pins[] = {
+	/* SCL, SDA */
+	RCAR_GP_PIN(0, 7), RCAR_GP_PIN(0, 8),
+};
+static const unsigned int i2c3_b_mux[] = {
+	SCL3_B_MARK, SDA3_B_MARK,
+};
+
+/* - MMC ------------------------------------------------------------------- */
+static const unsigned int mmc_data1_pins[] = {
+	/* D0 */
+	RCAR_GP_PIN(3, 2),
+};
+static const unsigned int mmc_data1_mux[] = {
+	MMC_D0_MARK,
+};
+static const unsigned int mmc_data4_pins[] = {
+	/* D[0:3] */
+	RCAR_GP_PIN(3, 2), RCAR_GP_PIN(3, 3),
+	RCAR_GP_PIN(3, 4), RCAR_GP_PIN(3, 5),
+};
+static const unsigned int mmc_data4_mux[] = {
+	MMC_D0_MARK, MMC_D1_MARK,
+	MMC_D2_MARK, MMC_D3_MARK,
+};
+static const unsigned int mmc_data8_pins[] = {
+	/* D[0:7] */
+	RCAR_GP_PIN(3, 2), RCAR_GP_PIN(3, 3),
+	RCAR_GP_PIN(3, 4), RCAR_GP_PIN(3, 5),
+	RCAR_GP_PIN(3, 6), RCAR_GP_PIN(3, 7),
+	RCAR_GP_PIN(3, 8), RCAR_GP_PIN(3, 9),
+};
+static const unsigned int mmc_data8_mux[] = {
+	MMC_D0_MARK, MMC_D1_MARK,
+	MMC_D2_MARK, MMC_D3_MARK,
+	MMC_D4_MARK, MMC_D5_MARK,
+	MMC_D6_MARK, MMC_D7_MARK,
+};
+static const unsigned int mmc_ctrl_pins[] = {
+	/* CLK, CMD */
+	RCAR_GP_PIN(3, 1), RCAR_GP_PIN(3, 0),
+};
+static const unsigned int mmc_ctrl_mux[] = {
+	MMC_CLK_MARK, MMC_CMD_MARK,
+};
+
+/* - SCIF0 ------------------------------------------------------------------ */
+static const unsigned int scif0_data_a_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(4, 20), RCAR_GP_PIN(4, 21),
+};
+static const unsigned int scif0_data_a_mux[] = {
+	RX0_A_MARK, TX0_A_MARK,
+};
+static const unsigned int scif0_clk_a_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(4, 19),
+};
+static const unsigned int scif0_clk_a_mux[] = {
+	SCK0_A_MARK,
+};
+static const unsigned int scif0_data_b_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(4, 31), RCAR_GP_PIN(4, 28),
+};
+static const unsigned int scif0_data_b_mux[] = {
+	RX0_B_MARK, TX0_B_MARK,
+};
+static const unsigned int scif0_clk_b_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(5, 2),
+};
+static const unsigned int scif0_clk_b_mux[] = {
+	SCK0_B_MARK,
+};
+static const unsigned int scif0_ctrl_pins[] = {
+	/* RTS, CTS */
+	RCAR_GP_PIN(4, 24), RCAR_GP_PIN(4, 23),
+};
+static const unsigned int scif0_ctrl_mux[] = {
+	RTS0_N_TANS_MARK, CTS0_N_MARK,
+};
+/* - SCIF1 ------------------------------------------------------------------ */
+static const unsigned int scif1_data_a_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(4, 23), RCAR_GP_PIN(4, 24),
+};
+static const unsigned int scif1_data_a_mux[] = {
+	RX1_A_MARK, TX1_A_MARK,
+};
+static const unsigned int scif1_clk_a_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(4, 22),
+};
+static const unsigned int scif1_clk_a_mux[] = {
+	SCK1_A_MARK,
+};
+static const unsigned int scif1_data_b_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(2, 26), RCAR_GP_PIN(2, 28),
+};
+static const unsigned int scif1_data_b_mux[] = {
+	RX1_B_MARK, TX1_B_MARK,
+};
+static const unsigned int scif1_clk_b_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(2, 25),
+};
+static const unsigned int scif1_clk_b_mux[] = {
+	SCK1_B_MARK,
+};
+static const unsigned int scif1_ctrl_pins[] = {
+	/* RTS, CTS */
+	RCAR_GP_PIN(4, 11), RCAR_GP_PIN(4, 10),
+};
+static const unsigned int scif1_ctrl_mux[] = {
+	RTS1_N_TANS_MARK, CTS1_N_MARK,
+};
+
+/* - SCIF2 ------------------------------------------------------------------ */
+static const unsigned int scif2_data_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(4, 26), RCAR_GP_PIN(4, 27),
+};
+static const unsigned int scif2_data_mux[] = {
+	RX2_MARK, TX2_MARK,
+};
+static const unsigned int scif2_clk_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(4, 25),
+};
+static const unsigned int scif2_clk_mux[] = {
+	SCK2_MARK,
+};
+/* - SCIF3 ------------------------------------------------------------------ */
+static const unsigned int scif3_data_a_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(2, 31), RCAR_GP_PIN(4, 00),
+};
+static const unsigned int scif3_data_a_mux[] = {
+	RX3_A_MARK, TX3_A_MARK,
+};
+static const unsigned int scif3_clk_a_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(2, 30),
+};
+static const unsigned int scif3_clk_a_mux[] = {
+	SCK3_A_MARK,
+};
+static const unsigned int scif3_data_b_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(1, 30), RCAR_GP_PIN(1, 31),
+};
+static const unsigned int scif3_data_b_mux[] = {
+	RX3_B_MARK, TX3_B_MARK,
+};
+static const unsigned int scif3_clk_b_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(1, 29),
+};
+static const unsigned int scif3_clk_b_mux[] = {
+	SCK3_B_MARK,
+};
+/* - SCIF4 ------------------------------------------------------------------ */
+static const unsigned int scif4_data_a_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(2, 11), RCAR_GP_PIN(2, 12),
+};
+static const unsigned int scif4_data_a_mux[] = {
+	RX4_A_MARK, TX4_A_MARK,
+};
+static const unsigned int scif4_clk_a_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(2, 6),
+};
+static const unsigned int scif4_clk_a_mux[] = {
+	SCK4_A_MARK,
+};
+static const unsigned int scif4_data_b_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(1, 16), RCAR_GP_PIN(1, 17),
+};
+static const unsigned int scif4_data_b_mux[] = {
+	RX4_B_MARK, TX4_B_MARK,
+};
+static const unsigned int scif4_clk_b_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(1, 15),
+};
+static const unsigned int scif4_clk_b_mux[] = {
+	SCK4_B_MARK,
+};
+/* - SCIF5 ------------------------------------------------------------------ */
+static const unsigned int scif5_data_a_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(0, 7), RCAR_GP_PIN(0, 8),
+};
+static const unsigned int scif5_data_a_mux[] = {
+	RX5_A_MARK, TX5_A_MARK,
+};
+static const unsigned int scif5_clk_a_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(0, 6),
+};
+static const unsigned int scif5_clk_a_mux[] = {
+	SCK5_A_MARK,
+};
+static const unsigned int scif5_data_b_pins[] = {
+	/* RX, TX */
+	RCAR_GP_PIN(1, 4), RCAR_GP_PIN(1, 5),
+};
+static const unsigned int scif5_data_b_mux[] = {
+	RX5_B_MARK, TX5_B_MARK,
+};
+static const unsigned int scif5_clk_b_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(1, 3),
+};
+static const unsigned int scif5_clk_b_mux[] = {
+	SCK5_B_MARK,
+};
+/* - SCIF Clock ------------------------------------------------------------- */
+static const unsigned int scif_clk_pins[] = {
+	/* SCIF_CLK */
+	RCAR_GP_PIN(2, 27),
+};
+static const unsigned int scif_clk_mux[] = {
+	SCIF_CLK_MARK,
+};
+
+static const struct sh_pfc_pin_group pinmux_groups[] = {
+	SH_PFC_PIN_GROUP(i2c0),
+	SH_PFC_PIN_GROUP(i2c1),
+	SH_PFC_PIN_GROUP(i2c2_a),
+	SH_PFC_PIN_GROUP(i2c2_b),
+	SH_PFC_PIN_GROUP(i2c3_a),
+	SH_PFC_PIN_GROUP(i2c3_b),
+	SH_PFC_PIN_GROUP(mmc_data1),
+	SH_PFC_PIN_GROUP(mmc_data4),
+	SH_PFC_PIN_GROUP(mmc_data8),
+	SH_PFC_PIN_GROUP(mmc_ctrl),
+	SH_PFC_PIN_GROUP(scif0_data_a),
+	SH_PFC_PIN_GROUP(scif0_clk_a),
+	SH_PFC_PIN_GROUP(scif0_data_b),
+	SH_PFC_PIN_GROUP(scif0_clk_b),
+	SH_PFC_PIN_GROUP(scif0_ctrl),
+	SH_PFC_PIN_GROUP(scif1_data_a),
+	SH_PFC_PIN_GROUP(scif1_clk_a),
+	SH_PFC_PIN_GROUP(scif1_data_b),
+	SH_PFC_PIN_GROUP(scif1_clk_b),
+	SH_PFC_PIN_GROUP(scif1_ctrl),
+	SH_PFC_PIN_GROUP(scif2_data),
+	SH_PFC_PIN_GROUP(scif2_clk),
+	SH_PFC_PIN_GROUP(scif3_data_a),
+	SH_PFC_PIN_GROUP(scif3_clk_a),
+	SH_PFC_PIN_GROUP(scif3_data_b),
+	SH_PFC_PIN_GROUP(scif3_clk_b),
+	SH_PFC_PIN_GROUP(scif4_data_a),
+	SH_PFC_PIN_GROUP(scif4_clk_a),
+	SH_PFC_PIN_GROUP(scif4_data_b),
+	SH_PFC_PIN_GROUP(scif4_clk_b),
+	SH_PFC_PIN_GROUP(scif5_data_a),
+	SH_PFC_PIN_GROUP(scif5_clk_a),
+	SH_PFC_PIN_GROUP(scif5_data_b),
+	SH_PFC_PIN_GROUP(scif5_clk_b),
+	SH_PFC_PIN_GROUP(scif_clk),
+};
+
+static const char * const i2c0_groups[] = {
+	"i2c0",
+};
+static const char * const i2c1_groups[] = {
+	"i2c1",
+};
+
+static const char * const i2c2_groups[] = {
+	"i2c2_a",
+	"i2c2_b",
+};
+
+static const char * const i2c3_groups[] = {
+	"i2c3_a",
+	"i2c3_b",
+};
+
+static const char * const mmc_groups[] = {
+	"mmc_data1",
+	"mmc_data4",
+	"mmc_data8",
+	"mmc_ctrl",
+};
+
+static const char * const scif0_groups[] = {
+	"scif0_data_a",
+	"scif0_clk_a",
+	"scif0_data_b",
+	"scif0_clk_b",
+	"scif0_ctrl",
+};
+
+static const char * const scif1_groups[] = {
+	"scif1_data_a",
+	"scif1_clk_a",
+	"scif1_data_b",
+	"scif1_clk_b",
+	"scif1_ctrl",
+};
+
+static const char * const scif2_groups[] = {
+	"scif2_data",
+	"scif2_clk",
+};
+
+static const char * const scif3_groups[] = {
+	"scif3_data_a",
+	"scif3_clk_a",
+	"scif3_data_b",
+	"scif3_clk_b",
+};
+
+static const char * const scif4_groups[] = {
+	"scif4_data_a",
+	"scif4_clk_a",
+	"scif4_data_b",
+	"scif4_clk_b",
+};
+
+static const char * const scif5_groups[] = {
+	"scif5_data_a",
+	"scif5_clk_a",
+	"scif5_data_b",
+	"scif5_clk_b",
+};
+
+static const char * const scif_clk_groups[] = {
+	"scif_clk",
+};
+
+static const struct sh_pfc_function pinmux_functions[] = {
+	SH_PFC_FUNCTION(i2c0),
+	SH_PFC_FUNCTION(i2c1),
+	SH_PFC_FUNCTION(i2c2),
+	SH_PFC_FUNCTION(i2c3),
+	SH_PFC_FUNCTION(mmc),
+	SH_PFC_FUNCTION(scif0),
+	SH_PFC_FUNCTION(scif1),
+	SH_PFC_FUNCTION(scif2),
+	SH_PFC_FUNCTION(scif3),
+	SH_PFC_FUNCTION(scif4),
+	SH_PFC_FUNCTION(scif5),
+	SH_PFC_FUNCTION(scif_clk),
+};
+
+static const struct pinmux_cfg_reg pinmux_config_regs[] = {
+#define F_(x, y)	FN_##y
+#define FM(x)		FN_##x
+	{ PINMUX_CFG_REG("GPSR0", 0xe6060100, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_0_8_FN,	GPSR0_8,
+		GP_0_7_FN,	GPSR0_7,
+		GP_0_6_FN,	GPSR0_6,
+		GP_0_5_FN,	GPSR0_5,
+		GP_0_4_FN,	GPSR0_4,
+		GP_0_3_FN,	GPSR0_3,
+		GP_0_2_FN,	GPSR0_2,
+		GP_0_1_FN,	GPSR0_1,
+		GP_0_0_FN,	GPSR0_0, }
+	},
+	{ PINMUX_CFG_REG("GPSR1", 0xe6060104, 32, 1) {
+		GP_1_31_FN,	GPSR1_31,
+		GP_1_30_FN,	GPSR1_30,
+		GP_1_29_FN,	GPSR1_29,
+		GP_1_28_FN,	GPSR1_28,
+		GP_1_27_FN,	GPSR1_27,
+		GP_1_26_FN,	GPSR1_26,
+		GP_1_25_FN,	GPSR1_25,
+		GP_1_24_FN,	GPSR1_24,
+		GP_1_23_FN,	GPSR1_23,
+		GP_1_22_FN,	GPSR1_22,
+		GP_1_21_FN,	GPSR1_21,
+		GP_1_20_FN,	GPSR1_20,
+		GP_1_19_FN,	GPSR1_19,
+		GP_1_18_FN,	GPSR1_18,
+		GP_1_17_FN,	GPSR1_17,
+		GP_1_16_FN,	GPSR1_16,
+		GP_1_15_FN,	GPSR1_15,
+		GP_1_14_FN,	GPSR1_14,
+		GP_1_13_FN,	GPSR1_13,
+		GP_1_12_FN,	GPSR1_12,
+		GP_1_11_FN,	GPSR1_11,
+		GP_1_10_FN,	GPSR1_10,
+		GP_1_9_FN,	GPSR1_9,
+		GP_1_8_FN,	GPSR1_8,
+		GP_1_7_FN,	GPSR1_7,
+		GP_1_6_FN,	GPSR1_6,
+		GP_1_5_FN,	GPSR1_5,
+		GP_1_4_FN,	GPSR1_4,
+		GP_1_3_FN,	GPSR1_3,
+		GP_1_2_FN,	GPSR1_2,
+		GP_1_1_FN,	GPSR1_1,
+		GP_1_0_FN,	GPSR1_0, }
+	},
+	{ PINMUX_CFG_REG("GPSR2", 0xe6060108, 32, 1) {
+		GP_2_31_FN,	GPSR2_31,
+		GP_2_30_FN,	GPSR2_30,
+		GP_2_29_FN,	GPSR2_29,
+		GP_2_28_FN,	GPSR2_28,
+		GP_2_27_FN,	GPSR2_27,
+		GP_2_26_FN,	GPSR2_26,
+		GP_2_25_FN,	GPSR2_25,
+		GP_2_24_FN,	GPSR2_24,
+		GP_2_23_FN,	GPSR2_23,
+		GP_2_22_FN,	GPSR2_22,
+		GP_2_21_FN,	GPSR2_21,
+		GP_2_20_FN,	GPSR2_20,
+		GP_2_19_FN,	GPSR2_19,
+		GP_2_18_FN,	GPSR2_18,
+		GP_2_17_FN,	GPSR2_17,
+		GP_2_16_FN,	GPSR2_16,
+		GP_2_15_FN,	GPSR2_15,
+		GP_2_14_FN,	GPSR2_14,
+		GP_2_13_FN,	GPSR2_13,
+		GP_2_12_FN,	GPSR2_12,
+		GP_2_11_FN,	GPSR2_11,
+		GP_2_10_FN,	GPSR2_10,
+		GP_2_9_FN,	GPSR2_9,
+		GP_2_8_FN,	GPSR2_8,
+		GP_2_7_FN,	GPSR2_7,
+		GP_2_6_FN,	GPSR2_6,
+		GP_2_5_FN,	GPSR2_5,
+		GP_2_4_FN,	GPSR2_4,
+		GP_2_3_FN,	GPSR2_3,
+		GP_2_2_FN,	GPSR2_2,
+		GP_2_1_FN,	GPSR2_1,
+		GP_2_0_FN,	GPSR2_0, }
+	},
+	{ PINMUX_CFG_REG("GPSR3", 0xe606010c, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_3_9_FN,	GPSR3_9,
+		GP_3_8_FN,	GPSR3_8,
+		GP_3_7_FN,	GPSR3_7,
+		GP_3_6_FN,	GPSR3_6,
+		GP_3_5_FN,	GPSR3_5,
+		GP_3_4_FN,	GPSR3_4,
+		GP_3_3_FN,	GPSR3_3,
+		GP_3_2_FN,	GPSR3_2,
+		GP_3_1_FN,	GPSR3_1,
+		GP_3_0_FN,	GPSR3_0, }
+	},
+	{ PINMUX_CFG_REG("GPSR4", 0xe6060110, 32, 1) {
+		GP_4_31_FN,	GPSR4_31,
+		GP_4_30_FN,	GPSR4_30,
+		GP_4_29_FN,	GPSR4_29,
+		GP_4_28_FN,	GPSR4_28,
+		GP_4_27_FN,	GPSR4_27,
+		GP_4_26_FN,	GPSR4_26,
+		GP_4_25_FN,	GPSR4_25,
+		GP_4_24_FN,	GPSR4_24,
+		GP_4_23_FN,	GPSR4_23,
+		GP_4_22_FN,	GPSR4_22,
+		GP_4_21_FN,	GPSR4_21,
+		GP_4_20_FN,	GPSR4_20,
+		GP_4_19_FN,	GPSR4_19,
+		GP_4_18_FN,	GPSR4_18,
+		GP_4_17_FN,	GPSR4_17,
+		GP_4_16_FN,	GPSR4_16,
+		GP_4_15_FN,	GPSR4_15,
+		GP_4_14_FN,	GPSR4_14,
+		GP_4_13_FN,	GPSR4_13,
+		GP_4_12_FN,	GPSR4_12,
+		GP_4_11_FN,	GPSR4_11,
+		GP_4_10_FN,	GPSR4_10,
+		GP_4_9_FN,	GPSR4_9,
+		GP_4_8_FN,	GPSR4_8,
+		GP_4_7_FN,	GPSR4_7,
+		GP_4_6_FN,	GPSR4_6,
+		GP_4_5_FN,	GPSR4_5,
+		GP_4_4_FN,	GPSR4_4,
+		GP_4_3_FN,	GPSR4_3,
+		GP_4_2_FN,	GPSR4_2,
+		GP_4_1_FN,	GPSR4_1,
+		GP_4_0_FN,	GPSR4_0, }
+	},
+	{ PINMUX_CFG_REG("GPSR5", 0xe6060114, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_5_20_FN,	GPSR5_20,
+		GP_5_19_FN,	GPSR5_19,
+		GP_5_18_FN,	GPSR5_18,
+		GP_5_17_FN,	GPSR5_17,
+		GP_5_16_FN,	GPSR5_16,
+		GP_5_15_FN,	GPSR5_15,
+		GP_5_14_FN,	GPSR5_14,
+		GP_5_13_FN,	GPSR5_13,
+		GP_5_12_FN,	GPSR5_12,
+		GP_5_11_FN,	GPSR5_11,
+		GP_5_10_FN,	GPSR5_10,
+		GP_5_9_FN,	GPSR5_9,
+		GP_5_8_FN,	GPSR5_8,
+		GP_5_7_FN,	GPSR5_7,
+		GP_5_6_FN,	GPSR5_6,
+		GP_5_5_FN,	GPSR5_5,
+		GP_5_4_FN,	GPSR5_4,
+		GP_5_3_FN,	GPSR5_3,
+		GP_5_2_FN,	GPSR5_2,
+		GP_5_1_FN,	GPSR5_1,
+		GP_5_0_FN,	GPSR5_0, }
+	},
+	{ PINMUX_CFG_REG("GPSR6", 0xe6060118, 32, 1) {
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		0, 0,
+		GP_6_13_FN,	GPSR6_13,
+		GP_6_12_FN,	GPSR6_12,
+		GP_6_11_FN,	GPSR6_11,
+		GP_6_10_FN,	GPSR6_10,
+		GP_6_9_FN,	GPSR6_9,
+		GP_6_8_FN,	GPSR6_8,
+		GP_6_7_FN,	GPSR6_7,
+		GP_6_6_FN,	GPSR6_6,
+		GP_6_5_FN,	GPSR6_5,
+		GP_6_4_FN,	GPSR6_4,
+		GP_6_3_FN,	GPSR6_3,
+		GP_6_2_FN,	GPSR6_2,
+		GP_6_1_FN,	GPSR6_1,
+		GP_6_0_FN,	GPSR6_0, }
+	},
+#undef F_
+#undef FM
+
+#define F_(x, y)	x,
+#define FM(x)		FN_##x,
+	{ PINMUX_CFG_REG("IPSR0", 0xe6060200, 32, 4) {
+		IP0_31_28
+		IP0_27_24
+		IP0_23_20
+		IP0_19_16
+		IP0_15_12
+		IP0_11_8
+		IP0_7_4
+		IP0_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR1", 0xe6060204, 32, 4) {
+		IP1_31_28
+		IP1_27_24
+		IP1_23_20
+		IP1_19_16
+		IP1_15_12
+		IP1_11_8
+		IP1_7_4
+		IP1_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR2", 0xe6060208, 32, 4) {
+		IP2_31_28
+		IP2_27_24
+		IP2_23_20
+		IP2_19_16
+		IP2_15_12
+		IP2_11_8
+		IP2_7_4
+		IP2_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR3", 0xe606020c, 32, 4) {
+		IP3_31_28
+		IP3_27_24
+		IP3_23_20
+		IP3_19_16
+		IP3_15_12
+		IP3_11_8
+		IP3_7_4
+		IP3_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR4", 0xe6060210, 32, 4) {
+		IP4_31_28
+		IP4_27_24
+		IP4_23_20
+		IP4_19_16
+		IP4_15_12
+		IP4_11_8
+		IP4_7_4
+		IP4_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR5", 0xe6060214, 32, 4) {
+		IP5_31_28
+		IP5_27_24
+		IP5_23_20
+		IP5_19_16
+		IP5_15_12
+		IP5_11_8
+		IP5_7_4
+		IP5_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR6", 0xe6060218, 32, 4) {
+		IP6_31_28
+		IP6_27_24
+		IP6_23_20
+		IP6_19_16
+		IP6_15_12
+		IP6_11_8
+		IP6_7_4
+		IP6_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR7", 0xe606021c, 32, 4) {
+		IP7_31_28
+		IP7_27_24
+		IP7_23_20
+		IP7_19_16
+		IP7_15_12
+		IP7_11_8
+		IP7_7_4
+		IP7_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR8", 0xe6060220, 32, 4) {
+		IP8_31_28
+		IP8_27_24
+		IP8_23_20
+		IP8_19_16
+		IP8_15_12
+		IP8_11_8
+		IP8_7_4
+		IP8_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR9", 0xe6060224, 32, 4) {
+		IP9_31_28
+		IP9_27_24
+		IP9_23_20
+		IP9_19_16
+		IP9_15_12
+		IP9_11_8
+		IP9_7_4
+		IP9_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR10", 0xe6060228, 32, 4) {
+		IP10_31_28
+		IP10_27_24
+		IP10_23_20
+		IP10_19_16
+		IP10_15_12
+		IP10_11_8
+		IP10_7_4
+		IP10_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR11", 0xe606022c, 32, 4) {
+		IP11_31_28
+		IP11_27_24
+		IP11_23_20
+		IP11_19_16
+		IP11_15_12
+		IP11_11_8
+		IP11_7_4
+		IP11_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR12", 0xe6060230, 32, 4) {
+		IP12_31_28
+		IP12_27_24
+		IP12_23_20
+		IP12_19_16
+		IP12_15_12
+		IP12_11_8
+		IP12_7_4
+		IP12_3_0 }
+	},
+	{ PINMUX_CFG_REG("IPSR13", 0xe6060234, 32, 4) {
+		/* IP13_31_28 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP13_27_24 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP13_23_20 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP13_19_16 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP13_15_12 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* IP13_11_8  */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		IP13_7_4
+		IP13_3_0 }
+	},
+#undef F_
+#undef FM
+
+#define F_(x, y)	x,
+#define FM(x)		FN_##x,
+	{ PINMUX_CFG_REG_VAR("MOD_SEL0", 0xe6060500, 32,
+			     1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 1,
+			     1, 1, 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1) {
+		/* RESERVED 31 */
+		0, 0,
+		MOD_SEL0_30
+		MOD_SEL0_29
+		MOD_SEL0_28
+		MOD_SEL0_27
+		MOD_SEL0_26
+		MOD_SEL0_25
+		MOD_SEL0_24_23
+		MOD_SEL0_22_21
+		MOD_SEL0_20_19
+		MOD_SEL0_18_17
+		/* RESERVED 16 */
+		0, 0,
+		MOD_SEL0_15
+		MOD_SEL0_14
+		MOD_SEL0_13
+		MOD_SEL0_12
+		MOD_SEL0_11
+		MOD_SEL0_10
+		/* RESERVED 9, 8, 7, 6 */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		MOD_SEL0_5
+		MOD_SEL0_4
+		MOD_SEL0_3
+		MOD_SEL0_2
+		MOD_SEL0_1
+		MOD_SEL0_0 }
+	},
+	{ PINMUX_CFG_REG_VAR("MOD_SEL1", 0xe6060504, 32,
+			     1, 1, 1, 1, 1, 1, 2, 4, 4,
+			     4, 4, 4, 4) {
+		MOD_SEL1_31
+		MOD_SEL1_30
+		MOD_SEL1_29
+		MOD_SEL1_28
+		MOD_SEL1_27
+		MOD_SEL1_26
+		/* RESERVED 25, 24 */
+		0, 0, 0, 0,
+		/* RESERVED 23, 22, 21, 20 */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* RESERVED 19, 18, 17, 16 */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* RESERVED 15, 14, 13, 12 */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* RESERVED 11, 10, 9, 8  */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* RESERVED 7, 6, 5, 4  */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		/* RESERVED 3, 2, 1, 0  */
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+	},
+	{ },
+};
+
+static int r8a77995_pin_to_pocctrl(struct sh_pfc *pfc, unsigned int pin, u32 *pocctrl)
+{
+	int bit = -EINVAL;
+
+	*pocctrl = 0xe6060380;
+
+	if (pin >= RCAR_GP_PIN(3, 0) && pin <= RCAR_GP_PIN(3, 9))
+		bit = 29 - (pin - RCAR_GP_PIN(3, 0));
+
+	return bit;
+}
+
+static const struct sh_pfc_soc_operations r8a77995_pinmux_ops = {
+	.pin_to_pocctrl = r8a77995_pin_to_pocctrl,
+};
+
+const struct sh_pfc_soc_info r8a77995_pinmux_info = {
+	.name = "r8a77995_pfc",
+	.ops = &r8a77995_pinmux_ops,
+	.unlock_reg = 0xe6060000, /* PMMR */
+
+	.function = { PINMUX_FUNCTION_BEGIN, PINMUX_FUNCTION_END },
+
+	.pins = pinmux_pins,
+	.nr_pins = ARRAY_SIZE(pinmux_pins),
+	.groups = pinmux_groups,
+	.nr_groups = ARRAY_SIZE(pinmux_groups),
+	.functions = pinmux_functions,
+	.nr_functions = ARRAY_SIZE(pinmux_functions),
+
+	.cfg_regs = pinmux_config_regs,
+
+	.pinmux_data = pinmux_data,
+	.pinmux_data_size = ARRAY_SIZE(pinmux_data),
+};
diff --git a/drivers/pinctrl/sh-pfc/pinctrl.c b/drivers/pinctrl/sh-pfc/pinctrl.c
index a70157f0acf4..5c9d79981e6d 100644
--- a/drivers/pinctrl/sh-pfc/pinctrl.c
+++ b/drivers/pinctrl/sh-pfc/pinctrl.c
@@ -290,7 +290,7 @@ static int sh_pfc_dt_node_to_map(struct pinctrl_dev *pctldev,
 	if (*num_maps)
 		return 0;
 
-	dev_err(dev, "no mapping found in node %s\n", np->full_name);
+	dev_err(dev, "no mapping found in node %pOF\n", np);
 	ret = -EINVAL;
 
 done:
@@ -742,13 +742,16 @@ static int sh_pfc_pinconf_group_set(struct pinctrl_dev *pctldev, unsigned group,
 	struct sh_pfc_pinctrl *pmx = pinctrl_dev_get_drvdata(pctldev);
 	const unsigned int *pins;
 	unsigned int num_pins;
-	unsigned int i;
+	unsigned int i, ret;
 
 	pins = pmx->pfc->info->groups[group].pins;
 	num_pins = pmx->pfc->info->groups[group].nr_pins;
 
-	for (i = 0; i < num_pins; ++i)
-		sh_pfc_pinconf_set(pctldev, pins[i], configs, num_configs);
+	for (i = 0; i < num_pins; ++i) {
+		ret = sh_pfc_pinconf_set(pctldev, pins[i], configs, num_configs);
+		if (ret)
+			return ret;
+	}
 
 	return 0;
 }
diff --git a/drivers/pinctrl/sh-pfc/sh_pfc.h b/drivers/pinctrl/sh-pfc/sh_pfc.h
index 4376397123de..8688b405e081 100644
--- a/drivers/pinctrl/sh-pfc/sh_pfc.h
+++ b/drivers/pinctrl/sh-pfc/sh_pfc.h
@@ -271,6 +271,7 @@ extern const struct sh_pfc_soc_info r8a7794_pinmux_info;
 extern const struct sh_pfc_soc_info r8a7795_pinmux_info;
 extern const struct sh_pfc_soc_info r8a7795es1_pinmux_info;
 extern const struct sh_pfc_soc_info r8a7796_pinmux_info;
+extern const struct sh_pfc_soc_info r8a77995_pinmux_info;
 extern const struct sh_pfc_soc_info sh7203_pinmux_info;
 extern const struct sh_pfc_soc_info sh7264_pinmux_info;
 extern const struct sh_pfc_soc_info sh7269_pinmux_info;
@@ -389,9 +390,13 @@ extern const struct sh_pfc_soc_info shx3_pinmux_info;
 	PORT_GP_CFG_1(bank, 8,  fn, sfx, cfg)
 #define PORT_GP_9(bank, fn, sfx)	PORT_GP_CFG_9(bank, fn, sfx, 0)
 
-#define PORT_GP_CFG_12(bank, fn, sfx, cfg)				\
+#define PORT_GP_CFG_10(bank, fn, sfx, cfg)				\
 	PORT_GP_CFG_9(bank, fn, sfx, cfg),				\
-	PORT_GP_CFG_1(bank, 9,  fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 9,  fn, sfx, cfg)
+#define PORT_GP_10(bank, fn, sfx)	PORT_GP_CFG_10(bank, fn, sfx, 0)
+
+#define PORT_GP_CFG_12(bank, fn, sfx, cfg)				\
+	PORT_GP_CFG_10(bank, fn, sfx, cfg),				\
 	PORT_GP_CFG_1(bank, 10, fn, sfx, cfg),				\
 	PORT_GP_CFG_1(bank, 11, fn, sfx, cfg)
 #define PORT_GP_12(bank, fn, sfx)	PORT_GP_CFG_12(bank, fn, sfx, 0)
@@ -422,11 +427,19 @@ extern const struct sh_pfc_soc_info shx3_pinmux_info;
 	PORT_GP_CFG_1(bank, 17, fn, sfx, cfg)
 #define PORT_GP_18(bank, fn, sfx)	PORT_GP_CFG_18(bank, fn, sfx, 0)
 
-#define PORT_GP_CFG_23(bank, fn, sfx, cfg)				\
+#define PORT_GP_CFG_20(bank, fn, sfx, cfg)				\
 	PORT_GP_CFG_18(bank, fn, sfx, cfg),				\
 	PORT_GP_CFG_1(bank, 18, fn, sfx, cfg),				\
-	PORT_GP_CFG_1(bank, 19, fn, sfx, cfg),				\
-	PORT_GP_CFG_1(bank, 20, fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 19, fn, sfx, cfg)
+#define PORT_GP_20(bank, fn, sfx)	PORT_GP_CFG_20(bank, fn, sfx, 0)
+
+#define PORT_GP_CFG_21(bank, fn, sfx, cfg)				\
+	PORT_GP_CFG_20(bank, fn, sfx, cfg),				\
+	PORT_GP_CFG_1(bank, 20, fn, sfx, cfg)
+#define PORT_GP_21(bank, fn, sfx)	PORT_GP_CFG_21(bank, fn, sfx, 0)
+
+#define PORT_GP_CFG_23(bank, fn, sfx, cfg)				\
+	PORT_GP_CFG_21(bank, fn, sfx, cfg),				\
 	PORT_GP_CFG_1(bank, 21, fn, sfx, cfg),				\
 	PORT_GP_CFG_1(bank, 22, fn, sfx, cfg)
 #define PORT_GP_23(bank, fn, sfx)	PORT_GP_CFG_23(bank, fn, sfx, 0)
diff --git a/drivers/pinctrl/sirf/pinctrl-atlas7.c b/drivers/pinctrl/sirf/pinctrl-atlas7.c
index 1efa315a7dbe..4db9323251e3 100644
--- a/drivers/pinctrl/sirf/pinctrl-atlas7.c
+++ b/drivers/pinctrl/sirf/pinctrl-atlas7.c
@@ -549,7 +549,7 @@ static const struct pinctrl_pin_desc atlas7_ioc_pads[] = {
 	PINCTRL_PIN(163, "jtag_trstn"),
 };
 
-struct atlas7_pad_config atlas7_ioc_pad_confs[] = {
+static struct atlas7_pad_config atlas7_ioc_pad_confs[] = {
 	/* The Configuration of IOC_RTC Pads */
 	PADCONF(0, 3, 0x0, 0x100, 0x200, -1, 0, 0, 0, 0),
 	PADCONF(1, 3, 0x0, 0x100, 0x200, -1, 4, 2, 2, 0),
@@ -1002,7 +1002,7 @@ static const unsigned int vi_vip1_high8bit_pins[] = { 82, 83, 84, 103, 104,
 		105, 106, 107, 102, 97, 98, };
 
 /* definition of pin group table */
-struct atlas7_pin_group altas7_pin_groups[] = {
+static struct atlas7_pin_group altas7_pin_groups[] = {
 	GROUP("gnss_gpio_grp", gnss_gpio_pins),
 	GROUP("lcd_vip_gpio_grp", lcd_vip_gpio_pins),
 	GROUP("sdio_i2s_gpio_grp", sdio_i2s_gpio_pins),
@@ -4764,7 +4764,7 @@ static struct atlas7_pmx_func atlas7_pmx_functions[] = {
 			&vi_vip1_high8bit_grp_mux),
 };
 
-struct atlas7_pinctrl_data atlas7_ioc_data = {
+static struct atlas7_pinctrl_data atlas7_ioc_data = {
 	.pads = (struct pinctrl_pin_desc *)atlas7_ioc_pads,
 	.pads_cnt = ARRAY_SIZE(atlas7_ioc_pads),
 	.grps = (struct atlas7_pin_group *)altas7_pin_groups,
@@ -5261,7 +5261,7 @@ static int atlas7_pmx_gpio_request_enable(struct pinctrl_dev *pctldev,
 	return 0;
 }
 
-static struct pinmux_ops atlas7_pinmux_ops = {
+static const struct pinmux_ops atlas7_pinmux_ops = {
 	.get_functions_count = atlas7_pmx_get_funcs_count,
 	.get_function_name = atlas7_pmx_get_func_name,
 	.get_function_groups = atlas7_pmx_get_func_groups,
@@ -6078,12 +6078,15 @@ static int atlas7_gpio_probe(struct platform_device *pdev)
 		bank = &a7gc->banks[idx];
 		/* Set ctrl registers' base of this bank */
 		bank->base = ATLAS7_GPIO_BASE(a7gc, idx);
+		bank->gpio_offset = idx * NGPIO_OF_BANK;
 
 		/* Get interrupt number from DTS */
 		ret = of_irq_get(np, idx);
-		if (ret == -EPROBE_DEFER) {
+		if (ret <= 0) {
 			dev_err(&pdev->dev,
 				"Unable to find IRQ number. ret=%d\n", ret);
+			if (!ret)
+				ret = -ENXIO;
 			goto failed;
 		}
 		bank->irq = ret;
diff --git a/drivers/pinctrl/sirf/pinctrl-sirf.c b/drivers/pinctrl/sirf/pinctrl-sirf.c
index 0df72be60704..d3ef05973901 100644
--- a/drivers/pinctrl/sirf/pinctrl-sirf.c
+++ b/drivers/pinctrl/sirf/pinctrl-sirf.c
@@ -133,7 +133,7 @@ static void sirfsoc_dt_free_map(struct pinctrl_dev *pctldev,
 	kfree(map);
 }
 
-static struct pinctrl_ops sirfsoc_pctrl_ops = {
+static const struct pinctrl_ops sirfsoc_pctrl_ops = {
 	.get_groups_count = sirfsoc_get_groups_count,
 	.get_group_name = sirfsoc_get_group_name,
 	.get_group_pins = sirfsoc_get_group_pins,
@@ -229,7 +229,7 @@ static int sirfsoc_pinmux_request_gpio(struct pinctrl_dev *pmxdev,
 	return 0;
 }
 
-static struct pinmux_ops sirfsoc_pinmux_ops = {
+static const struct pinmux_ops sirfsoc_pinmux_ops = {
 	.set_mux = sirfsoc_pinmux_set_mux,
 	.get_functions_count = sirfsoc_pinmux_get_funcs_count,
 	.get_function_name = sirfsoc_pinmux_get_func_name,
@@ -810,7 +810,7 @@ static int sirfsoc_gpio_probe(struct device_node *np)
 	sgpio->chip.gc.set = sirfsoc_gpio_set_value;
 	sgpio->chip.gc.base = 0;
 	sgpio->chip.gc.ngpio = SIRFSOC_GPIO_BANK_SIZE * SIRFSOC_GPIO_NO_OF_BANKS;
-	sgpio->chip.gc.label = kstrdup(np->full_name, GFP_KERNEL);
+	sgpio->chip.gc.label = kasprintf(GFP_KERNEL, "%pOF", np);
 	sgpio->chip.gc.of_node = np;
 	sgpio->chip.gc.of_xlate = sirfsoc_gpio_of_xlate;
 	sgpio->chip.gc.of_gpio_n_cells = 2;
@@ -819,8 +819,8 @@ static int sirfsoc_gpio_probe(struct device_node *np)
 
 	err = gpiochip_add_data(&sgpio->chip.gc, sgpio);
 	if (err) {
-		dev_err(&pdev->dev, "%s: error in probe function with status %d\n",
-			np->full_name, err);
+		dev_err(&pdev->dev, "%pOF: error in probe function with status %d\n",
+			np, err);
 		goto out;
 	}
 
diff --git a/drivers/pinctrl/sprd/Kconfig b/drivers/pinctrl/sprd/Kconfig
new file mode 100644
index 000000000000..bc7f3fab22f1
--- /dev/null
+++ b/drivers/pinctrl/sprd/Kconfig
@@ -0,0 +1,20 @@
+#
+# Spreadtrum pin control drivers
+#
+
+config PINCTRL_SPRD
+	bool "Spreadtrum pinctrl driver"
+	depends on OF
+	depends on ARCH_SPRD || COMPILE_TEST
+	select PINMUX
+	select PINCONF
+	select GENERIC_PINCONF
+	select GENERIC_PINMUX_FUNCTIONS
+	help
+	  Say Y here to enable Spreadtrum pinctrl driver
+
+config PINCTRL_SPRD_SC9860
+	bool "Spreadtrum SC9860 pinctrl driver"
+	depends on PINCTRL_SPRD
+	help
+	  Say Y here to enable Spreadtrum SC9860 pinctrl driver
diff --git a/drivers/pinctrl/sprd/Makefile b/drivers/pinctrl/sprd/Makefile
new file mode 100644
index 000000000000..b6caa8cbc6dd
--- /dev/null
+++ b/drivers/pinctrl/sprd/Makefile
@@ -0,0 +1,2 @@
+obj-$(CONFIG_PINCTRL_SPRD)		+= pinctrl-sprd.o
+obj-$(CONFIG_PINCTRL_SPRD_SC9860)	+= pinctrl-sprd-sc9860.o
diff --git a/drivers/pinctrl/sprd/pinctrl-sprd-sc9860.c b/drivers/pinctrl/sprd/pinctrl-sprd-sc9860.c
new file mode 100644
index 000000000000..3cdad8bc8f93
--- /dev/null
+++ b/drivers/pinctrl/sprd/pinctrl-sprd-sc9860.c
@@ -0,0 +1,972 @@
+/*
+ * Spreadtrum pin controller driver
+ * Copyright (C) 2017 Spreadtrum  - http://www.spreadtrum.com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+#include "pinctrl-sprd.h"
+
+enum sprd_sc9860_pins {
+	/* pin global control register 0 */
+	SC9860_VIO28_0_IRTE = SPRD_PIN_INFO(0, GLOBAL_CTRL_PIN, 11, 1, 0),
+	SC9860_VIO_SD2_IRTE = SPRD_PIN_INFO(1, GLOBAL_CTRL_PIN, 10, 1, 0),
+	SC9860_VIO_SD0_IRTE = SPRD_PIN_INFO(2, GLOBAL_CTRL_PIN, 9, 1, 0),
+	SC9860_VIO_SIM2_IRTE = SPRD_PIN_INFO(3, GLOBAL_CTRL_PIN, 8, 1, 0),
+	SC9860_VIO_SIM1_IRTE = SPRD_PIN_INFO(4, GLOBAL_CTRL_PIN, 7, 1, 0),
+	SC9860_VIO_SIM0_IRTE = SPRD_PIN_INFO(5, GLOBAL_CTRL_PIN, 6, 1, 0),
+	SC9860_VIO28_0_MS = SPRD_PIN_INFO(6, GLOBAL_CTRL_PIN, 5, 1, 0),
+	SC9860_VIO_SD2_MS = SPRD_PIN_INFO(7, GLOBAL_CTRL_PIN, 4, 1, 0),
+	SC9860_VIO_SD0_MS = SPRD_PIN_INFO(8, GLOBAL_CTRL_PIN, 3, 1, 0),
+	SC9860_VIO_SIM2_MS = SPRD_PIN_INFO(9, GLOBAL_CTRL_PIN, 2, 1, 0),
+	SC9860_VIO_SIM1_MS = SPRD_PIN_INFO(10, GLOBAL_CTRL_PIN, 1, 1, 0),
+	SC9860_VIO_SIM0_MS = SPRD_PIN_INFO(11, GLOBAL_CTRL_PIN, 0, 1, 0),
+
+	/* pin global control register 2 */
+	SC9860_SPSPI_PIN_IN_SEL = SPRD_PIN_INFO(12, GLOBAL_CTRL_PIN, 31, 1, 2),
+	SC9860_UART1_USB30_PHY_SEL = SPRD_PIN_INFO(13, GLOBAL_CTRL_PIN, 30, 1, 2),
+	SC9860_USB30_PHY_DM_OE = SPRD_PIN_INFO(14, GLOBAL_CTRL_PIN, 29, 1, 2),
+	SC9860_USB30_PHY_DP_OE = SPRD_PIN_INFO(15, GLOBAL_CTRL_PIN, 28, 1, 2),
+	SC9860_UART5_SYS_SEL = SPRD_PIN_INFO(16, GLOBAL_CTRL_PIN, 25, 3, 2),
+	SC9860_ORP_URXD_PIN_IN_SEL = SPRD_PIN_INFO(17, GLOBAL_CTRL_PIN, 24, 1, 2),
+	SC9860_SIM2_SYS_SEL = SPRD_PIN_INFO(18, GLOBAL_CTRL_PIN, 23, 1, 2),
+	SC9860_SIM1_SYS_SEL = SPRD_PIN_INFO(19, GLOBAL_CTRL_PIN, 22, 1, 2),
+	SC9860_SIM0_SYS_SEL = SPRD_PIN_INFO(20, GLOBAL_CTRL_PIN, 21, 1, 2),
+	SC9860_CLK26MHZ_BUF_OUT_SEL = SPRD_PIN_INFO(21, GLOBAL_CTRL_PIN, 20, 1, 2),
+	SC9860_UART4_SYS_SEL = SPRD_PIN_INFO(22, GLOBAL_CTRL_PIN, 16, 3, 2),
+	SC9860_UART3_SYS_SEL = SPRD_PIN_INFO(23, GLOBAL_CTRL_PIN, 13, 3, 2),
+	SC9860_UART2_SYS_SEL = SPRD_PIN_INFO(24, GLOBAL_CTRL_PIN, 10, 3, 2),
+	SC9860_UART1_SYS_SEL = SPRD_PIN_INFO(25, GLOBAL_CTRL_PIN, 7, 3, 2),
+	SC9860_UART0_SYS_SEL = SPRD_PIN_INFO(26, GLOBAL_CTRL_PIN, 4, 3, 2),
+	SC9860_UART24_LOOP_SEL = SPRD_PIN_INFO(27, GLOBAL_CTRL_PIN, 3, 1, 2),
+	SC9860_UART23_LOOP_SEL = SPRD_PIN_INFO(28, GLOBAL_CTRL_PIN, 2, 1, 2),
+	SC9860_UART14_LOOP_SEL = SPRD_PIN_INFO(29, GLOBAL_CTRL_PIN, 1, 1, 2),
+	SC9860_UART13_LOOP_SEL = SPRD_PIN_INFO(30, GLOBAL_CTRL_PIN, 0, 1, 2),
+
+	/* pin global control register 3 */
+	SC9860_IIS3_SYS_SEL = SPRD_PIN_INFO(31, GLOBAL_CTRL_PIN, 18, 4, 3),
+	SC9860_IIS2_SYS_SEL = SPRD_PIN_INFO(32, GLOBAL_CTRL_PIN, 14, 4, 3),
+	SC9860_IIS1_SYS_SEL = SPRD_PIN_INFO(33, GLOBAL_CTRL_PIN, 10, 4, 3),
+	SC9860_IIS0_SYS_SEL = SPRD_PIN_INFO(34, GLOBAL_CTRL_PIN, 6, 4, 3),
+	SC9860_IIS23_LOOP_SEL = SPRD_PIN_INFO(35, GLOBAL_CTRL_PIN, 5, 1, 3),
+	SC9860_IIS13_LOOP_SEL = SPRD_PIN_INFO(36, GLOBAL_CTRL_PIN, 4, 1, 3),
+	SC9860_IIS12_LOOP_SEL = SPRD_PIN_INFO(37, GLOBAL_CTRL_PIN, 3, 1, 3),
+	SC9860_IIS03_LOOP_SEL = SPRD_PIN_INFO(38, GLOBAL_CTRL_PIN, 2, 1, 3),
+	SC9860_IIS02_LOOP_SEL = SPRD_PIN_INFO(39, GLOBAL_CTRL_PIN, 1, 1, 3),
+	SC9860_IIS01_LOOP_SEL = SPRD_PIN_INFO(40, GLOBAL_CTRL_PIN, 0, 1, 3),
+
+	/* pin global control register 4 */
+	SC9860_IIS6_SYS_SEL = SPRD_PIN_INFO(41, GLOBAL_CTRL_PIN, 27, 4, 4),
+	SC9860_IIS5_SYS_SEL = SPRD_PIN_INFO(42, GLOBAL_CTRL_PIN, 23, 4, 4),
+	SC9860_IIS4_SYS_SEL = SPRD_PIN_INFO(43, GLOBAL_CTRL_PIN, 19, 4, 4),
+	SC9860_I2C_INF6_SYS_SEL = SPRD_PIN_INFO(44, GLOBAL_CTRL_PIN, 8, 2, 4),
+	SC9860_I2C_INF4_SYS_SEL = SPRD_PIN_INFO(45, GLOBAL_CTRL_PIN, 6, 2, 4),
+	SC9860_I2C_INF2_SYS_SEL = SPRD_PIN_INFO(46, GLOBAL_CTRL_PIN, 4, 2, 4),
+	SC9860_I2C_INF1_SYS_SEL = SPRD_PIN_INFO(47, GLOBAL_CTRL_PIN, 2, 2, 4),
+	SC9860_I2C_INF0_SYS_SEL = SPRD_PIN_INFO(48, GLOBAL_CTRL_PIN, 0, 2, 4),
+
+	/* pin global control register 5 */
+	SC9860_GPIO_INF7_SYS_SEL = SPRD_PIN_INFO(49, GLOBAL_CTRL_PIN, 27, 1, 5),
+	SC9860_GPIO_INF6_SYS_SEL = SPRD_PIN_INFO(50, GLOBAL_CTRL_PIN, 26, 1, 5),
+	SC9860_GPIO_INF5_SYS_SEL = SPRD_PIN_INFO(51, GLOBAL_CTRL_PIN, 25, 1, 5),
+	SC9860_GPIO_INF4_SYS_SEL = SPRD_PIN_INFO(52, GLOBAL_CTRL_PIN, 24, 1, 5),
+	SC9860_GPIO_INF3_SYS_SEL = SPRD_PIN_INFO(53, GLOBAL_CTRL_PIN, 23, 1, 5),
+	SC9860_GPIO_INF2_SYS_SEL = SPRD_PIN_INFO(54, GLOBAL_CTRL_PIN, 22, 1, 5),
+	SC9860_GPIO_INF1_SYS_SEL = SPRD_PIN_INFO(55, GLOBAL_CTRL_PIN, 21, 1, 5),
+	SC9860_GPIO_INF0_SYS_SEL = SPRD_PIN_INFO(56, GLOBAL_CTRL_PIN, 20, 1, 5),
+	SC9860_WDRST_OUT_SEL = SPRD_PIN_INFO(57, GLOBAL_CTRL_PIN, 16, 3, 5),
+	SC9860_ADI_SYNC_PIN_OUT_SEL = SPRD_PIN_INFO(58, GLOBAL_CTRL_PIN, 14, 1, 5),
+	SC9860_CMRST_SEL = SPRD_PIN_INFO(59, GLOBAL_CTRL_PIN, 13, 1, 5),
+	SC9860_CMPD_SEL = SPRD_PIN_INFO(60, GLOBAL_CTRL_PIN, 12, 1, 5),
+	SC9860_TEST_DBG_MODE11 = SPRD_PIN_INFO(61, GLOBAL_CTRL_PIN, 11, 1, 5),
+	SC9860_TEST_DBG_MODE10 = SPRD_PIN_INFO(62, GLOBAL_CTRL_PIN, 10, 1, 5),
+	SC9860_TEST_DBG_MODE9 = SPRD_PIN_INFO(63, GLOBAL_CTRL_PIN, 9, 1, 5),
+	SC9860_TEST_DBG_MODE8 = SPRD_PIN_INFO(64, GLOBAL_CTRL_PIN, 8, 1, 5),
+	SC9860_TEST_DBG_MODE7 = SPRD_PIN_INFO(65, GLOBAL_CTRL_PIN, 7, 1, 5),
+	SC9860_TEST_DBG_MODE6 = SPRD_PIN_INFO(66, GLOBAL_CTRL_PIN, 6, 1, 5),
+	SC9860_TEST_DBG_MODE5 = SPRD_PIN_INFO(67, GLOBAL_CTRL_PIN, 5, 1, 5),
+	SC9860_TEST_DBG_MODE4 = SPRD_PIN_INFO(68, GLOBAL_CTRL_PIN, 4, 1, 5),
+	SC9860_TEST_DBG_MODE3 = SPRD_PIN_INFO(69, GLOBAL_CTRL_PIN, 3, 1, 5),
+	SC9860_TEST_DBG_MODE2 = SPRD_PIN_INFO(70, GLOBAL_CTRL_PIN, 2, 1, 5),
+	SC9860_TEST_DBG_MODE1 = SPRD_PIN_INFO(71, GLOBAL_CTRL_PIN, 1, 1, 5),
+	SC9860_TEST_DBG_MODE0 = SPRD_PIN_INFO(72, GLOBAL_CTRL_PIN, 0, 1, 5),
+
+	/* pin global control register 6 */
+	SC9860_SP_EIC_DPAD3_SEL = SPRD_PIN_INFO(73, GLOBAL_CTRL_PIN, 24, 8, 6),
+	SC9860_SP_EIC_DPAD2_SEL = SPRD_PIN_INFO(74, GLOBAL_CTRL_PIN, 16, 8, 6),
+	SC9860_SP_EIC_DPAD1_SEL = SPRD_PIN_INFO(75, GLOBAL_CTRL_PIN, 8, 8, 6),
+	SC9860_SP_EIC_DPAD0_SEL = SPRD_PIN_INFO(76, GLOBAL_CTRL_PIN, 0, 8, 6),
+
+	/* pin global control register 7 */
+	SC9860_SP_EIC_DPAD7_SEL = SPRD_PIN_INFO(77, GLOBAL_CTRL_PIN, 24, 8, 7),
+	SC9860_SP_EIC_DPAD6_SEL = SPRD_PIN_INFO(78, GLOBAL_CTRL_PIN, 16, 8, 7),
+	SC9860_SP_EIC_DPAD5_SEL = SPRD_PIN_INFO(79, GLOBAL_CTRL_PIN, 8, 8, 7),
+	SC9860_SP_EIC_DPAD4_SEL = SPRD_PIN_INFO(80, GLOBAL_CTRL_PIN, 0, 8, 7),
+
+	/* common pin registers definitions */
+	SC9860_RFCTL20 = SPRD_PIN_INFO(81, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL21 = SPRD_PIN_INFO(83, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL30 = SPRD_PIN_INFO(85, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL31 = SPRD_PIN_INFO(87, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL32 = SPRD_PIN_INFO(89, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL33 = SPRD_PIN_INFO(91, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL34 = SPRD_PIN_INFO(93, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL35 = SPRD_PIN_INFO(95, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL36 = SPRD_PIN_INFO(97, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL37 = SPRD_PIN_INFO(99, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL22 = SPRD_PIN_INFO(101, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL23 = SPRD_PIN_INFO(103, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL24 = SPRD_PIN_INFO(105, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL25 = SPRD_PIN_INFO(107, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL26 = SPRD_PIN_INFO(109, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL27 = SPRD_PIN_INFO(111, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL28 = SPRD_PIN_INFO(113, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL29 = SPRD_PIN_INFO(115, COMMON_PIN, 0, 0, 0),
+	SC9860_SCL2 = SPRD_PIN_INFO(117, COMMON_PIN, 0, 0, 0),
+	SC9860_SDA2 = SPRD_PIN_INFO(119, COMMON_PIN, 0, 0, 0),
+	SC9860_MTCK_ARM = SPRD_PIN_INFO(121, COMMON_PIN, 0, 0, 0),
+	SC9860_MTMS_ARM = SPRD_PIN_INFO(123, COMMON_PIN, 0, 0, 0),
+	SC9860_XTL_EN0 = SPRD_PIN_INFO(125, COMMON_PIN, 0, 0, 0),
+	SC9860_PTEST = SPRD_PIN_INFO(127, COMMON_PIN, 0, 0, 0),
+	SC9860_AUD_DAD1 = SPRD_PIN_INFO(129, COMMON_PIN, 0, 0, 0),
+	SC9860_AUD_ADD0 = SPRD_PIN_INFO(131, COMMON_PIN, 0, 0, 0),
+	SC9860_AUD_ADSYNC = SPRD_PIN_INFO(133, COMMON_PIN, 0, 0, 0),
+	SC9860_AUD_SCLK = SPRD_PIN_INFO(135, COMMON_PIN, 0, 0, 0),
+	SC9860_CHIP_SLEEP = SPRD_PIN_INFO(137, COMMON_PIN, 0, 0, 0),
+	SC9860_CLK_32K = SPRD_PIN_INFO(139, COMMON_PIN, 0, 0, 0),
+	SC9860_DCDC_ARM_EN = SPRD_PIN_INFO(141, COMMON_PIN, 0, 0, 0),
+	SC9860_EXT_RST_B = SPRD_PIN_INFO(143, COMMON_PIN, 0, 0, 0),
+	SC9860_ADI_D = SPRD_PIN_INFO(145, COMMON_PIN, 0, 0, 0),
+	SC9860_ADI_SCLK = SPRD_PIN_INFO(147, COMMON_PIN, 0, 0, 0),
+	SC9860_XTL_EN1 = SPRD_PIN_INFO(149, COMMON_PIN, 0, 0, 0),
+	SC9860_ANA_INT = SPRD_PIN_INFO(151, COMMON_PIN, 0, 0, 0),
+	SC9860_AUD_DAD0 = SPRD_PIN_INFO(153, COMMON_PIN, 0, 0, 0),
+	SC9860_AUD_DASYNC = SPRD_PIN_INFO(155, COMMON_PIN, 0, 0, 0),
+	SC9860_LCM_RSTN = SPRD_PIN_INFO(157, COMMON_PIN, 0, 0, 0),
+	SC9860_DSI_TE = SPRD_PIN_INFO(159, COMMON_PIN, 0, 0, 0),
+	SC9860_PWMA = SPRD_PIN_INFO(161, COMMON_PIN, 0, 0, 0),
+	SC9860_EXTINT0 = SPRD_PIN_INFO(163, COMMON_PIN, 0, 0, 0),
+	SC9860_EXTINT1 = SPRD_PIN_INFO(165, COMMON_PIN, 0, 0, 0),
+	SC9860_SDA1 = SPRD_PIN_INFO(167, COMMON_PIN, 0, 0, 0),
+	SC9860_SCL1 = SPRD_PIN_INFO(169, COMMON_PIN, 0, 0, 0),
+	SC9860_SIMCLK2 = SPRD_PIN_INFO(171, COMMON_PIN, 0, 0, 0),
+	SC9860_SIMDA2 = SPRD_PIN_INFO(173, COMMON_PIN, 0, 0, 0),
+	SC9860_SIMRST2 = SPRD_PIN_INFO(175, COMMON_PIN, 0, 0, 0),
+	SC9860_SIMCLK1 = SPRD_PIN_INFO(177, COMMON_PIN, 0, 0, 0),
+	SC9860_SIMDA1 = SPRD_PIN_INFO(179, COMMON_PIN, 0, 0, 0),
+	SC9860_SIMRST1 = SPRD_PIN_INFO(181, COMMON_PIN, 0, 0, 0),
+	SC9860_SIMCLK0 = SPRD_PIN_INFO(183, COMMON_PIN, 0, 0, 0),
+	SC9860_SIMDA0 = SPRD_PIN_INFO(185, COMMON_PIN, 0, 0, 0),
+	SC9860_SIMRST0 = SPRD_PIN_INFO(187, COMMON_PIN, 0, 0, 0),
+	SC9860_SD2_CMD = SPRD_PIN_INFO(189, COMMON_PIN, 0, 0, 0),
+	SC9860_SD2_D0 = SPRD_PIN_INFO(191, COMMON_PIN, 0, 0, 0),
+	SC9860_SD2_D1 = SPRD_PIN_INFO(193, COMMON_PIN, 0, 0, 0),
+	SC9860_SD2_CLK = SPRD_PIN_INFO(195, COMMON_PIN, 0, 0, 0),
+	SC9860_SD2_D2 = SPRD_PIN_INFO(197, COMMON_PIN, 0, 0, 0),
+	SC9860_SD2_D3 = SPRD_PIN_INFO(199, COMMON_PIN, 0, 0, 0),
+	SC9860_SD0_D3 = SPRD_PIN_INFO(201, COMMON_PIN, 0, 0, 0),
+	SC9860_SD0_D2 = SPRD_PIN_INFO(203, COMMON_PIN, 0, 0, 0),
+	SC9860_SD0_CMD = SPRD_PIN_INFO(205, COMMON_PIN, 0, 0, 0),
+	SC9860_SD0_D0 = SPRD_PIN_INFO(207, COMMON_PIN, 0, 0, 0),
+	SC9860_SD0_D1 = SPRD_PIN_INFO(209, COMMON_PIN, 0, 0, 0),
+	SC9860_SD0_CLK = SPRD_PIN_INFO(211, COMMON_PIN, 0, 0, 0),
+	SC9860_EMMC_CMD_reserved = SPRD_PIN_INFO(213, COMMON_PIN, 0, 0, 0),
+	SC9860_EMMC_CMD = SPRD_PIN_INFO(215, COMMON_PIN, 0, 0, 0),
+	SC9860_EMMC_D6 = SPRD_PIN_INFO(217, COMMON_PIN, 0, 0, 0),
+	SC9860_EMMC_D7 = SPRD_PIN_INFO(219, COMMON_PIN, 0, 0, 0),
+	SC9860_EMMC_CLK = SPRD_PIN_INFO(221, COMMON_PIN, 0, 0, 0),
+	SC9860_EMMC_D5 = SPRD_PIN_INFO(223, COMMON_PIN, 0, 0, 0),
+	SC9860_EMMC_D4 = SPRD_PIN_INFO(225, COMMON_PIN, 0, 0, 0),
+	SC9860_EMMC_DS = SPRD_PIN_INFO(227, COMMON_PIN, 0, 0, 0),
+	SC9860_EMMC_D3_reserved = SPRD_PIN_INFO(229, COMMON_PIN, 0, 0, 0),
+	SC9860_EMMC_D3 = SPRD_PIN_INFO(231, COMMON_PIN, 0, 0, 0),
+	SC9860_EMMC_RST = SPRD_PIN_INFO(233, COMMON_PIN, 0, 0, 0),
+	SC9860_EMMC_D1 = SPRD_PIN_INFO(235, COMMON_PIN, 0, 0, 0),
+	SC9860_EMMC_D2 = SPRD_PIN_INFO(237, COMMON_PIN, 0, 0, 0),
+	SC9860_EMMC_D0 = SPRD_PIN_INFO(239, COMMON_PIN, 0, 0, 0),
+	SC9860_IIS0DI = SPRD_PIN_INFO(241, COMMON_PIN, 0, 0, 0),
+	SC9860_IIS0DO = SPRD_PIN_INFO(243, COMMON_PIN, 0, 0, 0),
+	SC9860_IIS0CLK = SPRD_PIN_INFO(245, COMMON_PIN, 0, 0, 0),
+	SC9860_IIS0LRCK = SPRD_PIN_INFO(247, COMMON_PIN, 0, 0, 0),
+	SC9860_SD1_CLK = SPRD_PIN_INFO(249, COMMON_PIN, 0, 0, 0),
+	SC9860_SD1_CMD = SPRD_PIN_INFO(251, COMMON_PIN, 0, 0, 0),
+	SC9860_SD1_D0 = SPRD_PIN_INFO(253, COMMON_PIN, 0, 0, 0),
+	SC9860_SD1_D1 = SPRD_PIN_INFO(255, COMMON_PIN, 0, 0, 0),
+	SC9860_SD1_D2 = SPRD_PIN_INFO(257, COMMON_PIN, 0, 0, 0),
+	SC9860_SD1_D3 = SPRD_PIN_INFO(259, COMMON_PIN, 0, 0, 0),
+	SC9860_CLK_AUX0 = SPRD_PIN_INFO(261, COMMON_PIN, 0, 0, 0),
+	SC9860_WIFI_COEXIST = SPRD_PIN_INFO(263, COMMON_PIN, 0, 0, 0),
+	SC9860_BEIDOU_COEXIST = SPRD_PIN_INFO(265, COMMON_PIN, 0, 0, 0),
+	SC9860_U3TXD = SPRD_PIN_INFO(267, COMMON_PIN, 0, 0, 0),
+	SC9860_U3RXD = SPRD_PIN_INFO(269, COMMON_PIN, 0, 0, 0),
+	SC9860_U3CTS = SPRD_PIN_INFO(271, COMMON_PIN, 0, 0, 0),
+	SC9860_U3RTS = SPRD_PIN_INFO(273, COMMON_PIN, 0, 0, 0),
+	SC9860_U0TXD = SPRD_PIN_INFO(275, COMMON_PIN, 0, 0, 0),
+	SC9860_U0RXD = SPRD_PIN_INFO(277, COMMON_PIN, 0, 0, 0),
+	SC9860_U0CTS = SPRD_PIN_INFO(279, COMMON_PIN, 0, 0, 0),
+	SC9860_U0RTS = SPRD_PIN_INFO(281, COMMON_PIN, 0, 0, 0),
+	SC9860_IIS1DI = SPRD_PIN_INFO(283, COMMON_PIN, 0, 0, 0),
+	SC9860_IIS1DO = SPRD_PIN_INFO(285, COMMON_PIN, 0, 0, 0),
+	SC9860_IIS1CLK = SPRD_PIN_INFO(287, COMMON_PIN, 0, 0, 0),
+	SC9860_IIS1LRCK = SPRD_PIN_INFO(289, COMMON_PIN, 0, 0, 0),
+	SC9860_SPI0_CSN = SPRD_PIN_INFO(291, COMMON_PIN, 0, 0, 0),
+	SC9860_SPI0_DO = SPRD_PIN_INFO(293, COMMON_PIN, 0, 0, 0),
+	SC9860_SPI0_DI = SPRD_PIN_INFO(295, COMMON_PIN, 0, 0, 0),
+	SC9860_SPI0_CLK = SPRD_PIN_INFO(297, COMMON_PIN, 0, 0, 0),
+	SC9860_U2TXD = SPRD_PIN_INFO(299, COMMON_PIN, 0, 0, 0),
+	SC9860_U2RXD = SPRD_PIN_INFO(301, COMMON_PIN, 0, 0, 0),
+	SC9860_U4TXD = SPRD_PIN_INFO(303, COMMON_PIN, 0, 0, 0),
+	SC9860_U4RXD = SPRD_PIN_INFO(305, COMMON_PIN, 0, 0, 0),
+	SC9860_CMMCLK1 = SPRD_PIN_INFO(307, COMMON_PIN, 0, 0, 0),
+	SC9860_CMRST1 = SPRD_PIN_INFO(309, COMMON_PIN, 0, 0, 0),
+	SC9860_CMMCLK0 = SPRD_PIN_INFO(311, COMMON_PIN, 0, 0, 0),
+	SC9860_CMRST0 = SPRD_PIN_INFO(313, COMMON_PIN, 0, 0, 0),
+	SC9860_CMPD0 = SPRD_PIN_INFO(315, COMMON_PIN, 0, 0, 0),
+	SC9860_CMPD1 = SPRD_PIN_INFO(317, COMMON_PIN, 0, 0, 0),
+	SC9860_SCL0 = SPRD_PIN_INFO(319, COMMON_PIN, 0, 0, 0),
+	SC9860_SDA0 = SPRD_PIN_INFO(321, COMMON_PIN, 0, 0, 0),
+	SC9860_SDA6 = SPRD_PIN_INFO(323, COMMON_PIN, 0, 0, 0),
+	SC9860_SCL6 = SPRD_PIN_INFO(325, COMMON_PIN, 0, 0, 0),
+	SC9860_U1TXD = SPRD_PIN_INFO(327, COMMON_PIN, 0, 0, 0),
+	SC9860_U1RXD = SPRD_PIN_INFO(329, COMMON_PIN, 0, 0, 0),
+	SC9860_KEYOUT0 = SPRD_PIN_INFO(331, COMMON_PIN, 0, 0, 0),
+	SC9860_KEYOUT1 = SPRD_PIN_INFO(333, COMMON_PIN, 0, 0, 0),
+	SC9860_KEYOUT2 = SPRD_PIN_INFO(335, COMMON_PIN, 0, 0, 0),
+	SC9860_KEYIN0 = SPRD_PIN_INFO(337, COMMON_PIN, 0, 0, 0),
+	SC9860_KEYIN1 = SPRD_PIN_INFO(339, COMMON_PIN, 0, 0, 0),
+	SC9860_KEYIN2 = SPRD_PIN_INFO(341, COMMON_PIN, 0, 0, 0),
+	SC9860_IIS3DI = SPRD_PIN_INFO(343, COMMON_PIN, 0, 0, 0),
+	SC9860_IIS3DO = SPRD_PIN_INFO(345, COMMON_PIN, 0, 0, 0),
+	SC9860_IIS3CLK = SPRD_PIN_INFO(347, COMMON_PIN, 0, 0, 0),
+	SC9860_IIS3LRCK = SPRD_PIN_INFO(349, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL0 = SPRD_PIN_INFO(351, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL1 = SPRD_PIN_INFO(353, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL10 = SPRD_PIN_INFO(355, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL11 = SPRD_PIN_INFO(357, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL12 = SPRD_PIN_INFO(359, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL13 = SPRD_PIN_INFO(361, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL14 = SPRD_PIN_INFO(363, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL15 = SPRD_PIN_INFO(365, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL16 = SPRD_PIN_INFO(367, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL17 = SPRD_PIN_INFO(369, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL18 = SPRD_PIN_INFO(371, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL19 = SPRD_PIN_INFO(373, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL2 = SPRD_PIN_INFO(375, COMMON_PIN, 0, 0, 0),
+	SC9860_EXTINT5 = SPRD_PIN_INFO(377, COMMON_PIN, 0, 0, 0),
+	SC9860_EXTINT6 = SPRD_PIN_INFO(379, COMMON_PIN, 0, 0, 0),
+	SC9860_EXTINT7 = SPRD_PIN_INFO(381, COMMON_PIN, 0, 0, 0),
+	SC9860_GPIO30 = SPRD_PIN_INFO(383, COMMON_PIN, 0, 0, 0),
+	SC9860_GPIO31 = SPRD_PIN_INFO(385, COMMON_PIN, 0, 0, 0),
+	SC9860_GPIO32 = SPRD_PIN_INFO(387, COMMON_PIN, 0, 0, 0),
+	SC9860_GPIO33 = SPRD_PIN_INFO(389, COMMON_PIN, 0, 0, 0),
+	SC9860_GPIO34 = SPRD_PIN_INFO(391, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL3 = SPRD_PIN_INFO(393, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL4 = SPRD_PIN_INFO(395, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL5 = SPRD_PIN_INFO(397, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL6 = SPRD_PIN_INFO(399, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL7 = SPRD_PIN_INFO(401, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL8 = SPRD_PIN_INFO(403, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL9 = SPRD_PIN_INFO(405, COMMON_PIN, 0, 0, 0),
+	SC9860_RFFE0_SCK0 = SPRD_PIN_INFO(407, COMMON_PIN, 0, 0, 0),
+	SC9860_GPIO38 = SPRD_PIN_INFO(409, COMMON_PIN, 0, 0, 0),
+	SC9860_RFFE0_SDA0 = SPRD_PIN_INFO(411, COMMON_PIN, 0, 0, 0),
+	SC9860_GPIO39 = SPRD_PIN_INFO(413, COMMON_PIN, 0, 0, 0),
+	SC9860_RFFE1_SCK0 = SPRD_PIN_INFO(415, COMMON_PIN, 0, 0, 0),
+	SC9860_GPIO181 = SPRD_PIN_INFO(417, COMMON_PIN, 0, 0, 0),
+	SC9860_RFFE1_SDA0 = SPRD_PIN_INFO(419, COMMON_PIN, 0, 0, 0),
+	SC9860_GPIO182 = SPRD_PIN_INFO(421, COMMON_PIN, 0, 0, 0),
+	SC9860_RF_LVDS0_ADC_ON = SPRD_PIN_INFO(423, COMMON_PIN, 0, 0, 0),
+	SC9860_RF_LVDS0_DAC_ON = SPRD_PIN_INFO(425, COMMON_PIN, 0, 0, 0),
+	SC9860_RFSCK0 = SPRD_PIN_INFO(427, COMMON_PIN, 0, 0, 0),
+	SC9860_RFSDA0 = SPRD_PIN_INFO(429, COMMON_PIN, 0, 0, 0),
+	SC9860_RFSEN0 = SPRD_PIN_INFO(431, COMMON_PIN, 0, 0, 0),
+	SC9860_RF_LVDS1_ADC_ON = SPRD_PIN_INFO(433, COMMON_PIN, 0, 0, 0),
+	SC9860_RF_LVDS1_DAC_ON = SPRD_PIN_INFO(435, COMMON_PIN, 0, 0, 0),
+	SC9860_RFSCK1 = SPRD_PIN_INFO(437, COMMON_PIN, 0, 0, 0),
+	SC9860_RFSDA1 = SPRD_PIN_INFO(439, COMMON_PIN, 0, 0, 0),
+	SC9860_RFSEN1 = SPRD_PIN_INFO(441, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL38 = SPRD_PIN_INFO(443, COMMON_PIN, 0, 0, 0),
+	SC9860_RFCTL39 = SPRD_PIN_INFO(445, COMMON_PIN, 0, 0, 0),
+
+	/* MSIC pin registers definitions */
+	SC9860_RFCTL20_MISC = SPRD_PIN_INFO(82, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL21_MISC = SPRD_PIN_INFO(84, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL30_MISC = SPRD_PIN_INFO(86, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL31_MISC = SPRD_PIN_INFO(88, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL32_MISC = SPRD_PIN_INFO(90, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL33_MISC = SPRD_PIN_INFO(92, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL34_MISC = SPRD_PIN_INFO(94, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL35_MISC = SPRD_PIN_INFO(96, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL36_MISC = SPRD_PIN_INFO(98, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL37_MISC = SPRD_PIN_INFO(100, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL22_MISC = SPRD_PIN_INFO(102, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL23_MISC = SPRD_PIN_INFO(104, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL24_MISC = SPRD_PIN_INFO(106, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL25_MISC = SPRD_PIN_INFO(108, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL26_MISC = SPRD_PIN_INFO(110, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL27_MISC = SPRD_PIN_INFO(112, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL28_MISC = SPRD_PIN_INFO(114, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL29_MISC = SPRD_PIN_INFO(116, MISC_PIN, 0, 0, 0),
+	SC9860_SCL2_MISC = SPRD_PIN_INFO(118, MISC_PIN, 0, 0, 0),
+	SC9860_SDA2_MISC = SPRD_PIN_INFO(120, MISC_PIN, 0, 0, 0),
+	SC9860_MTCK_ARM_MISC = SPRD_PIN_INFO(122, MISC_PIN, 0, 0, 0),
+	SC9860_MTMS_ARM_MISC = SPRD_PIN_INFO(124, MISC_PIN, 0, 0, 0),
+	SC9860_XTL_EN0_MISC = SPRD_PIN_INFO(126, MISC_PIN, 0, 0, 0),
+	SC9860_PTEST_MISC = SPRD_PIN_INFO(128, MISC_PIN, 0, 0, 0),
+	SC9860_AUD_DAD1_MISC = SPRD_PIN_INFO(130, MISC_PIN, 0, 0, 0),
+	SC9860_AUD_ADD0_MISC = SPRD_PIN_INFO(132, MISC_PIN, 0, 0, 0),
+	SC9860_AUD_ADSYNC_MISC = SPRD_PIN_INFO(134, MISC_PIN, 0, 0, 0),
+	SC9860_AUD_SCLK_MISC = SPRD_PIN_INFO(136, MISC_PIN, 0, 0, 0),
+	SC9860_CHIP_SLEEP_MISC = SPRD_PIN_INFO(138, MISC_PIN, 0, 0, 0),
+	SC9860_CLK_32K_MISC = SPRD_PIN_INFO(140, MISC_PIN, 0, 0, 0),
+	SC9860_DCDC_ARM_EN_MISC = SPRD_PIN_INFO(142, MISC_PIN, 0, 0, 0),
+	SC9860_EXT_RST_B_MISC = SPRD_PIN_INFO(144, MISC_PIN, 0, 0, 0),
+	SC9860_ADI_D_MISC = SPRD_PIN_INFO(146, MISC_PIN, 0, 0, 0),
+	SC9860_ADI_SCLK_MISC = SPRD_PIN_INFO(148, MISC_PIN, 0, 0, 0),
+	SC9860_XTL_EN1_MISC = SPRD_PIN_INFO(150, MISC_PIN, 0, 0, 0),
+	SC9860_ANA_INT_MISC = SPRD_PIN_INFO(152, MISC_PIN, 0, 0, 0),
+	SC9860_AUD_DAD0_MISC = SPRD_PIN_INFO(154, MISC_PIN, 0, 0, 0),
+	SC9860_AUD_DASYNC_MISC = SPRD_PIN_INFO(156, MISC_PIN, 0, 0, 0),
+	SC9860_LCM_RSTN_MISC = SPRD_PIN_INFO(158, MISC_PIN, 0, 0, 0),
+	SC9860_DSI_TE_MISC = SPRD_PIN_INFO(160, MISC_PIN, 0, 0, 0),
+	SC9860_PWMA_MISC = SPRD_PIN_INFO(162, MISC_PIN, 0, 0, 0),
+	SC9860_EXTINT0_MISC = SPRD_PIN_INFO(164, MISC_PIN, 0, 0, 0),
+	SC9860_EXTINT1_MISC = SPRD_PIN_INFO(166, MISC_PIN, 0, 0, 0),
+	SC9860_SDA1_MISC = SPRD_PIN_INFO(168, MISC_PIN, 0, 0, 0),
+	SC9860_SCL1_MISC = SPRD_PIN_INFO(170, MISC_PIN, 0, 0, 0),
+	SC9860_SIMCLK2_MISC = SPRD_PIN_INFO(172, MISC_PIN, 0, 0, 0),
+	SC9860_SIMDA2_MISC = SPRD_PIN_INFO(174, MISC_PIN, 0, 0, 0),
+	SC9860_SIMRST2_MISC = SPRD_PIN_INFO(176, MISC_PIN, 0, 0, 0),
+	SC9860_SIMCLK1_MISC = SPRD_PIN_INFO(178, MISC_PIN, 0, 0, 0),
+	SC9860_SIMDA1_MISC = SPRD_PIN_INFO(180, MISC_PIN, 0, 0, 0),
+	SC9860_SIMRST1_MISC = SPRD_PIN_INFO(182, MISC_PIN, 0, 0, 0),
+	SC9860_SIMCLK0_MISC = SPRD_PIN_INFO(184, MISC_PIN, 0, 0, 0),
+	SC9860_SIMDA0_MISC = SPRD_PIN_INFO(186, MISC_PIN, 0, 0, 0),
+	SC9860_SIMRST0_MISC = SPRD_PIN_INFO(188, MISC_PIN, 0, 0, 0),
+	SC9860_SD2_CMD_MISC = SPRD_PIN_INFO(190, MISC_PIN, 0, 0, 0),
+	SC9860_SD2_D0_MISC = SPRD_PIN_INFO(192, MISC_PIN, 0, 0, 0),
+	SC9860_SD2_D1_MISC = SPRD_PIN_INFO(194, MISC_PIN, 0, 0, 0),
+	SC9860_SD2_CLK_MISC = SPRD_PIN_INFO(196, MISC_PIN, 0, 0, 0),
+	SC9860_SD2_D2_MISC = SPRD_PIN_INFO(198, MISC_PIN, 0, 0, 0),
+	SC9860_SD2_D3_MISC = SPRD_PIN_INFO(200, MISC_PIN, 0, 0, 0),
+	SC9860_SD0_D3_MISC = SPRD_PIN_INFO(202, MISC_PIN, 0, 0, 0),
+	SC9860_SD0_D2_MISC = SPRD_PIN_INFO(204, MISC_PIN, 0, 0, 0),
+	SC9860_SD0_CMD_MISC = SPRD_PIN_INFO(206, MISC_PIN, 0, 0, 0),
+	SC9860_SD0_D0_MISC = SPRD_PIN_INFO(208, MISC_PIN, 0, 0, 0),
+	SC9860_SD0_D1_MISC = SPRD_PIN_INFO(210, MISC_PIN, 0, 0, 0),
+	SC9860_SD0_CLK_MISC = SPRD_PIN_INFO(212, MISC_PIN, 0, 0, 0),
+	SC9860_EMMC_CMD_reserved_MISC = SPRD_PIN_INFO(214, MISC_PIN, 0, 0, 0),
+	SC9860_EMMC_CMD_MISC = SPRD_PIN_INFO(216, MISC_PIN, 0, 0, 0),
+	SC9860_EMMC_D6_MISC = SPRD_PIN_INFO(218, MISC_PIN, 0, 0, 0),
+	SC9860_EMMC_D7_MISC = SPRD_PIN_INFO(220, MISC_PIN, 0, 0, 0),
+	SC9860_EMMC_CLK_MISC = SPRD_PIN_INFO(222, MISC_PIN, 0, 0, 0),
+	SC9860_EMMC_D5_MISC = SPRD_PIN_INFO(224, MISC_PIN, 0, 0, 0),
+	SC9860_EMMC_D4_MISC = SPRD_PIN_INFO(226, MISC_PIN, 0, 0, 0),
+	SC9860_EMMC_DS_MISC = SPRD_PIN_INFO(228, MISC_PIN, 0, 0, 0),
+	SC9860_EMMC_D3_reserved_MISC = SPRD_PIN_INFO(230, MISC_PIN, 0, 0, 0),
+	SC9860_EMMC_D3_MISC = SPRD_PIN_INFO(232, MISC_PIN, 0, 0, 0),
+	SC9860_EMMC_RST_MISC = SPRD_PIN_INFO(234, MISC_PIN, 0, 0, 0),
+	SC9860_EMMC_D1_MISC = SPRD_PIN_INFO(236, MISC_PIN, 0, 0, 0),
+	SC9860_EMMC_D2_MISC = SPRD_PIN_INFO(238, MISC_PIN, 0, 0, 0),
+	SC9860_EMMC_D0_MISC = SPRD_PIN_INFO(240, MISC_PIN, 0, 0, 0),
+	SC9860_IIS0DI_MISC = SPRD_PIN_INFO(242, MISC_PIN, 0, 0, 0),
+	SC9860_IIS0DO_MISC = SPRD_PIN_INFO(244, MISC_PIN, 0, 0, 0),
+	SC9860_IIS0CLK_MISC = SPRD_PIN_INFO(246, MISC_PIN, 0, 0, 0),
+	SC9860_IIS0LRCK_MISC = SPRD_PIN_INFO(248, MISC_PIN, 0, 0, 0),
+	SC9860_SD1_CLK_MISC = SPRD_PIN_INFO(250, MISC_PIN, 0, 0, 0),
+	SC9860_SD1_CMD_MISC = SPRD_PIN_INFO(252, MISC_PIN, 0, 0, 0),
+	SC9860_SD1_D0_MISC = SPRD_PIN_INFO(254, MISC_PIN, 0, 0, 0),
+	SC9860_SD1_D1_MISC = SPRD_PIN_INFO(256, MISC_PIN, 0, 0, 0),
+	SC9860_SD1_D2_MISC = SPRD_PIN_INFO(258, MISC_PIN, 0, 0, 0),
+	SC9860_SD1_D3_MISC = SPRD_PIN_INFO(260, MISC_PIN, 0, 0, 0),
+	SC9860_CLK_AUX0_MISC = SPRD_PIN_INFO(262, MISC_PIN, 0, 0, 0),
+	SC9860_WIFI_COEXIST_MISC = SPRD_PIN_INFO(264, MISC_PIN, 0, 0, 0),
+	SC9860_BEIDOU_COEXIST_MISC = SPRD_PIN_INFO(266, MISC_PIN, 0, 0, 0),
+	SC9860_U3TXD_MISC = SPRD_PIN_INFO(268, MISC_PIN, 0, 0, 0),
+	SC9860_U3RXD_MISC = SPRD_PIN_INFO(270, MISC_PIN, 0, 0, 0),
+	SC9860_U3CTS_MISC = SPRD_PIN_INFO(272, MISC_PIN, 0, 0, 0),
+	SC9860_U3RTS_MISC = SPRD_PIN_INFO(274, MISC_PIN, 0, 0, 0),
+	SC9860_U0TXD_MISC = SPRD_PIN_INFO(276, MISC_PIN, 0, 0, 0),
+	SC9860_U0RXD_MISC = SPRD_PIN_INFO(278, MISC_PIN, 0, 0, 0),
+	SC9860_U0CTS_MISC = SPRD_PIN_INFO(280, MISC_PIN, 0, 0, 0),
+	SC9860_U0RTS_MISC = SPRD_PIN_INFO(282, MISC_PIN, 0, 0, 0),
+	SC9860_IIS1DI_MISC = SPRD_PIN_INFO(284, MISC_PIN, 0, 0, 0),
+	SC9860_IIS1DO_MISC = SPRD_PIN_INFO(286, MISC_PIN, 0, 0, 0),
+	SC9860_IIS1CLK_MISC = SPRD_PIN_INFO(288, MISC_PIN, 0, 0, 0),
+	SC9860_IIS1LRCK_MISC = SPRD_PIN_INFO(290, MISC_PIN, 0, 0, 0),
+	SC9860_SPI0_CSN_MISC = SPRD_PIN_INFO(292, MISC_PIN, 0, 0, 0),
+	SC9860_SPI0_DO_MISC = SPRD_PIN_INFO(294, MISC_PIN, 0, 0, 0),
+	SC9860_SPI0_DI_MISC = SPRD_PIN_INFO(296, MISC_PIN, 0, 0, 0),
+	SC9860_SPI0_CLK_MISC = SPRD_PIN_INFO(298, MISC_PIN, 0, 0, 0),
+	SC9860_U2TXD_MISC = SPRD_PIN_INFO(300, MISC_PIN, 0, 0, 0),
+	SC9860_U2RXD_MISC = SPRD_PIN_INFO(302, MISC_PIN, 0, 0, 0),
+	SC9860_U4TXD_MISC = SPRD_PIN_INFO(304, MISC_PIN, 0, 0, 0),
+	SC9860_U4RXD_MISC = SPRD_PIN_INFO(306, MISC_PIN, 0, 0, 0),
+	SC9860_CMMCLK1_MISC = SPRD_PIN_INFO(308, MISC_PIN, 0, 0, 0),
+	SC9860_CMRST1_MISC = SPRD_PIN_INFO(310, MISC_PIN, 0, 0, 0),
+	SC9860_CMMCLK0_MISC = SPRD_PIN_INFO(312, MISC_PIN, 0, 0, 0),
+	SC9860_CMRST0_MISC = SPRD_PIN_INFO(314, MISC_PIN, 0, 0, 0),
+	SC9860_CMPD0_MISC = SPRD_PIN_INFO(316, MISC_PIN, 0, 0, 0),
+	SC9860_CMPD1_MISC = SPRD_PIN_INFO(318, MISC_PIN, 0, 0, 0),
+	SC9860_SCL0_MISC = SPRD_PIN_INFO(320, MISC_PIN, 0, 0, 0),
+	SC9860_SDA0_MISC = SPRD_PIN_INFO(322, MISC_PIN, 0, 0, 0),
+	SC9860_SDA6_MISC = SPRD_PIN_INFO(324, MISC_PIN, 0, 0, 0),
+	SC9860_SCL6_MISC = SPRD_PIN_INFO(326, MISC_PIN, 0, 0, 0),
+	SC9860_U1TXD_MISC = SPRD_PIN_INFO(328, MISC_PIN, 0, 0, 0),
+	SC9860_U1RXD_MISC = SPRD_PIN_INFO(330, MISC_PIN, 0, 0, 0),
+	SC9860_KEYOUT0_MISC = SPRD_PIN_INFO(332, MISC_PIN, 0, 0, 0),
+	SC9860_KEYOUT1_MISC = SPRD_PIN_INFO(334, MISC_PIN, 0, 0, 0),
+	SC9860_KEYOUT2_MISC = SPRD_PIN_INFO(336, MISC_PIN, 0, 0, 0),
+	SC9860_KEYIN0_MISC = SPRD_PIN_INFO(338, MISC_PIN, 0, 0, 0),
+	SC9860_KEYIN1_MISC = SPRD_PIN_INFO(340, MISC_PIN, 0, 0, 0),
+	SC9860_KEYIN2_MISC = SPRD_PIN_INFO(342, MISC_PIN, 0, 0, 0),
+	SC9860_IIS3DI_MISC = SPRD_PIN_INFO(344, MISC_PIN, 0, 0, 0),
+	SC9860_IIS3DO_MISC = SPRD_PIN_INFO(346, MISC_PIN, 0, 0, 0),
+	SC9860_IIS3CLK_MISC = SPRD_PIN_INFO(348, MISC_PIN, 0, 0, 0),
+	SC9860_IIS3LRCK_MISC = SPRD_PIN_INFO(350, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL0_MISC = SPRD_PIN_INFO(352, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL1_MISC = SPRD_PIN_INFO(354, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL10_MISC = SPRD_PIN_INFO(356, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL11_MISC = SPRD_PIN_INFO(358, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL12_MISC = SPRD_PIN_INFO(360, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL13_MISC = SPRD_PIN_INFO(362, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL14_MISC = SPRD_PIN_INFO(364, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL15_MISC = SPRD_PIN_INFO(366, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL16_MISC = SPRD_PIN_INFO(368, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL17_MISC = SPRD_PIN_INFO(370, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL18_MISC = SPRD_PIN_INFO(372, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL19_MISC = SPRD_PIN_INFO(374, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL2_MISC = SPRD_PIN_INFO(376, MISC_PIN, 0, 0, 0),
+	SC9860_EXTINT5_MISC = SPRD_PIN_INFO(378, MISC_PIN, 0, 0, 0),
+	SC9860_EXTINT6_MISC = SPRD_PIN_INFO(380, MISC_PIN, 0, 0, 0),
+	SC9860_EXTINT7_MISC = SPRD_PIN_INFO(382, MISC_PIN, 0, 0, 0),
+	SC9860_GPIO30_MISC = SPRD_PIN_INFO(384, MISC_PIN, 0, 0, 0),
+	SC9860_GPIO31_MISC = SPRD_PIN_INFO(386, MISC_PIN, 0, 0, 0),
+	SC9860_GPIO32_MISC = SPRD_PIN_INFO(388, MISC_PIN, 0, 0, 0),
+	SC9860_GPIO33_MISC = SPRD_PIN_INFO(390, MISC_PIN, 0, 0, 0),
+	SC9860_GPIO34_MISC = SPRD_PIN_INFO(392, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL3_MISC = SPRD_PIN_INFO(394, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL4_MISC = SPRD_PIN_INFO(396, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL5_MISC = SPRD_PIN_INFO(398, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL6_MISC = SPRD_PIN_INFO(400, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL7_MISC = SPRD_PIN_INFO(402, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL8_MISC = SPRD_PIN_INFO(404, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL9_MISC = SPRD_PIN_INFO(406, MISC_PIN, 0, 0, 0),
+	SC9860_RFFE0_SCK0_MISC = SPRD_PIN_INFO(408, MISC_PIN, 0, 0, 0),
+	SC9860_GPIO38_MISC = SPRD_PIN_INFO(410, MISC_PIN, 0, 0, 0),
+	SC9860_RFFE0_SDA0_MISC = SPRD_PIN_INFO(412, MISC_PIN, 0, 0, 0),
+	SC9860_GPIO39_MISC = SPRD_PIN_INFO(414, MISC_PIN, 0, 0, 0),
+	SC9860_RFFE1_SCK0_MISC = SPRD_PIN_INFO(416, MISC_PIN, 0, 0, 0),
+	SC9860_GPIO181_MISC = SPRD_PIN_INFO(418, MISC_PIN, 0, 0, 0),
+	SC9860_RFFE1_SDA0_MISC = SPRD_PIN_INFO(420, MISC_PIN, 0, 0, 0),
+	SC9860_GPIO182_MISC = SPRD_PIN_INFO(422, MISC_PIN, 0, 0, 0),
+	SC9860_RF_LVDS0_ADC_ON_MISC = SPRD_PIN_INFO(424, MISC_PIN, 0, 0, 0),
+	SC9860_RF_LVDS0_DAC_ON_MISC = SPRD_PIN_INFO(426, MISC_PIN, 0, 0, 0),
+	SC9860_RFSCK0_MISC = SPRD_PIN_INFO(428, MISC_PIN, 0, 0, 0),
+	SC9860_RFSDA0_MISC = SPRD_PIN_INFO(430, MISC_PIN, 0, 0, 0),
+	SC9860_RFSEN0_MISC = SPRD_PIN_INFO(432, MISC_PIN, 0, 0, 0),
+	SC9860_RF_LVDS1_ADC_ON_MISC = SPRD_PIN_INFO(434, MISC_PIN, 0, 0, 0),
+	SC9860_RF_LVDS1_DAC_ON_MISC = SPRD_PIN_INFO(436, MISC_PIN, 0, 0, 0),
+	SC9860_RFSCK1_MISC = SPRD_PIN_INFO(438, MISC_PIN, 0, 0, 0),
+	SC9860_RFSDA1_MISC = SPRD_PIN_INFO(440, MISC_PIN, 0, 0, 0),
+	SC9860_RFSEN1_MISC = SPRD_PIN_INFO(442, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL38_MISC = SPRD_PIN_INFO(444, MISC_PIN, 0, 0, 0),
+	SC9860_RFCTL39_MISC = SPRD_PIN_INFO(446, MISC_PIN, 0, 0, 0),
+};
+
+static struct sprd_pins_info sprd_sc9860_pins_info[] = {
+	SPRD_PINCTRL_PIN(SC9860_VIO28_0_IRTE),
+	SPRD_PINCTRL_PIN(SC9860_VIO_SD2_IRTE),
+	SPRD_PINCTRL_PIN(SC9860_VIO_SD0_IRTE),
+	SPRD_PINCTRL_PIN(SC9860_VIO_SIM2_IRTE),
+	SPRD_PINCTRL_PIN(SC9860_VIO_SIM1_IRTE),
+	SPRD_PINCTRL_PIN(SC9860_VIO_SIM0_IRTE),
+	SPRD_PINCTRL_PIN(SC9860_VIO28_0_MS),
+	SPRD_PINCTRL_PIN(SC9860_VIO_SD2_MS),
+	SPRD_PINCTRL_PIN(SC9860_VIO_SD0_MS),
+	SPRD_PINCTRL_PIN(SC9860_VIO_SIM2_MS),
+	SPRD_PINCTRL_PIN(SC9860_VIO_SIM1_MS),
+	SPRD_PINCTRL_PIN(SC9860_VIO_SIM0_MS),
+	SPRD_PINCTRL_PIN(SC9860_SPSPI_PIN_IN_SEL),
+	SPRD_PINCTRL_PIN(SC9860_UART1_USB30_PHY_SEL),
+	SPRD_PINCTRL_PIN(SC9860_USB30_PHY_DM_OE),
+	SPRD_PINCTRL_PIN(SC9860_USB30_PHY_DP_OE),
+	SPRD_PINCTRL_PIN(SC9860_UART5_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_ORP_URXD_PIN_IN_SEL),
+	SPRD_PINCTRL_PIN(SC9860_SIM2_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_SIM1_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_SIM0_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_CLK26MHZ_BUF_OUT_SEL),
+	SPRD_PINCTRL_PIN(SC9860_UART4_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_UART3_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_UART2_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_UART1_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_UART0_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_UART24_LOOP_SEL),
+	SPRD_PINCTRL_PIN(SC9860_UART23_LOOP_SEL),
+	SPRD_PINCTRL_PIN(SC9860_UART14_LOOP_SEL),
+	SPRD_PINCTRL_PIN(SC9860_UART13_LOOP_SEL),
+	SPRD_PINCTRL_PIN(SC9860_IIS3_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_IIS2_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_IIS1_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_IIS0_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_IIS23_LOOP_SEL),
+	SPRD_PINCTRL_PIN(SC9860_IIS13_LOOP_SEL),
+	SPRD_PINCTRL_PIN(SC9860_IIS12_LOOP_SEL),
+	SPRD_PINCTRL_PIN(SC9860_IIS03_LOOP_SEL),
+	SPRD_PINCTRL_PIN(SC9860_IIS02_LOOP_SEL),
+	SPRD_PINCTRL_PIN(SC9860_IIS01_LOOP_SEL),
+	SPRD_PINCTRL_PIN(SC9860_IIS6_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_IIS5_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_IIS4_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_I2C_INF6_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_I2C_INF4_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_I2C_INF2_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_I2C_INF1_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_I2C_INF0_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_GPIO_INF7_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_GPIO_INF6_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_GPIO_INF5_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_GPIO_INF4_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_GPIO_INF3_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_GPIO_INF2_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_GPIO_INF1_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_GPIO_INF0_SYS_SEL),
+	SPRD_PINCTRL_PIN(SC9860_WDRST_OUT_SEL),
+	SPRD_PINCTRL_PIN(SC9860_ADI_SYNC_PIN_OUT_SEL),
+	SPRD_PINCTRL_PIN(SC9860_CMRST_SEL),
+	SPRD_PINCTRL_PIN(SC9860_CMPD_SEL),
+	SPRD_PINCTRL_PIN(SC9860_TEST_DBG_MODE11),
+	SPRD_PINCTRL_PIN(SC9860_TEST_DBG_MODE10),
+	SPRD_PINCTRL_PIN(SC9860_TEST_DBG_MODE9),
+	SPRD_PINCTRL_PIN(SC9860_TEST_DBG_MODE8),
+	SPRD_PINCTRL_PIN(SC9860_TEST_DBG_MODE7),
+	SPRD_PINCTRL_PIN(SC9860_TEST_DBG_MODE6),
+	SPRD_PINCTRL_PIN(SC9860_TEST_DBG_MODE5),
+	SPRD_PINCTRL_PIN(SC9860_TEST_DBG_MODE4),
+	SPRD_PINCTRL_PIN(SC9860_TEST_DBG_MODE3),
+	SPRD_PINCTRL_PIN(SC9860_TEST_DBG_MODE2),
+	SPRD_PINCTRL_PIN(SC9860_TEST_DBG_MODE1),
+	SPRD_PINCTRL_PIN(SC9860_TEST_DBG_MODE0),
+	SPRD_PINCTRL_PIN(SC9860_SP_EIC_DPAD3_SEL),
+	SPRD_PINCTRL_PIN(SC9860_SP_EIC_DPAD2_SEL),
+	SPRD_PINCTRL_PIN(SC9860_SP_EIC_DPAD1_SEL),
+	SPRD_PINCTRL_PIN(SC9860_SP_EIC_DPAD0_SEL),
+	SPRD_PINCTRL_PIN(SC9860_SP_EIC_DPAD7_SEL),
+	SPRD_PINCTRL_PIN(SC9860_SP_EIC_DPAD6_SEL),
+	SPRD_PINCTRL_PIN(SC9860_SP_EIC_DPAD5_SEL),
+	SPRD_PINCTRL_PIN(SC9860_SP_EIC_DPAD4_SEL),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL20),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL21),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL30),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL31),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL32),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL33),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL34),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL35),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL36),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL37),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL22),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL23),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL24),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL25),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL26),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL27),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL28),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL29),
+	SPRD_PINCTRL_PIN(SC9860_SCL2),
+	SPRD_PINCTRL_PIN(SC9860_SDA2),
+	SPRD_PINCTRL_PIN(SC9860_MTCK_ARM),
+	SPRD_PINCTRL_PIN(SC9860_MTMS_ARM),
+	SPRD_PINCTRL_PIN(SC9860_XTL_EN0),
+	SPRD_PINCTRL_PIN(SC9860_PTEST),
+	SPRD_PINCTRL_PIN(SC9860_AUD_DAD1),
+	SPRD_PINCTRL_PIN(SC9860_AUD_ADD0),
+	SPRD_PINCTRL_PIN(SC9860_AUD_ADSYNC),
+	SPRD_PINCTRL_PIN(SC9860_AUD_SCLK),
+	SPRD_PINCTRL_PIN(SC9860_CHIP_SLEEP),
+	SPRD_PINCTRL_PIN(SC9860_CLK_32K),
+	SPRD_PINCTRL_PIN(SC9860_DCDC_ARM_EN),
+	SPRD_PINCTRL_PIN(SC9860_EXT_RST_B),
+	SPRD_PINCTRL_PIN(SC9860_ADI_D),
+	SPRD_PINCTRL_PIN(SC9860_ADI_SCLK),
+	SPRD_PINCTRL_PIN(SC9860_XTL_EN1),
+	SPRD_PINCTRL_PIN(SC9860_ANA_INT),
+	SPRD_PINCTRL_PIN(SC9860_AUD_DAD0),
+	SPRD_PINCTRL_PIN(SC9860_AUD_DASYNC),
+	SPRD_PINCTRL_PIN(SC9860_LCM_RSTN),
+	SPRD_PINCTRL_PIN(SC9860_DSI_TE),
+	SPRD_PINCTRL_PIN(SC9860_PWMA),
+	SPRD_PINCTRL_PIN(SC9860_EXTINT0),
+	SPRD_PINCTRL_PIN(SC9860_EXTINT1),
+	SPRD_PINCTRL_PIN(SC9860_SDA1),
+	SPRD_PINCTRL_PIN(SC9860_SCL1),
+	SPRD_PINCTRL_PIN(SC9860_SIMCLK2),
+	SPRD_PINCTRL_PIN(SC9860_SIMDA2),
+	SPRD_PINCTRL_PIN(SC9860_SIMRST2),
+	SPRD_PINCTRL_PIN(SC9860_SIMCLK1),
+	SPRD_PINCTRL_PIN(SC9860_SIMDA1),
+	SPRD_PINCTRL_PIN(SC9860_SIMRST1),
+	SPRD_PINCTRL_PIN(SC9860_SIMCLK0),
+	SPRD_PINCTRL_PIN(SC9860_SIMDA0),
+	SPRD_PINCTRL_PIN(SC9860_SIMRST0),
+	SPRD_PINCTRL_PIN(SC9860_SD2_CMD),
+	SPRD_PINCTRL_PIN(SC9860_SD2_D0),
+	SPRD_PINCTRL_PIN(SC9860_SD2_D1),
+	SPRD_PINCTRL_PIN(SC9860_SD2_CLK),
+	SPRD_PINCTRL_PIN(SC9860_SD2_D2),
+	SPRD_PINCTRL_PIN(SC9860_SD2_D3),
+	SPRD_PINCTRL_PIN(SC9860_SD0_D3),
+	SPRD_PINCTRL_PIN(SC9860_SD0_D2),
+	SPRD_PINCTRL_PIN(SC9860_SD0_CMD),
+	SPRD_PINCTRL_PIN(SC9860_SD0_D0),
+	SPRD_PINCTRL_PIN(SC9860_SD0_D1),
+	SPRD_PINCTRL_PIN(SC9860_SD0_CLK),
+	SPRD_PINCTRL_PIN(SC9860_EMMC_CMD),
+	SPRD_PINCTRL_PIN(SC9860_EMMC_D6),
+	SPRD_PINCTRL_PIN(SC9860_EMMC_D7),
+	SPRD_PINCTRL_PIN(SC9860_EMMC_CLK),
+	SPRD_PINCTRL_PIN(SC9860_EMMC_D5),
+	SPRD_PINCTRL_PIN(SC9860_EMMC_D4),
+	SPRD_PINCTRL_PIN(SC9860_EMMC_DS),
+	SPRD_PINCTRL_PIN(SC9860_EMMC_D3),
+	SPRD_PINCTRL_PIN(SC9860_EMMC_RST),
+	SPRD_PINCTRL_PIN(SC9860_EMMC_D1),
+	SPRD_PINCTRL_PIN(SC9860_EMMC_D2),
+	SPRD_PINCTRL_PIN(SC9860_EMMC_D0),
+	SPRD_PINCTRL_PIN(SC9860_IIS0DI),
+	SPRD_PINCTRL_PIN(SC9860_IIS0DO),
+	SPRD_PINCTRL_PIN(SC9860_IIS0CLK),
+	SPRD_PINCTRL_PIN(SC9860_IIS0LRCK),
+	SPRD_PINCTRL_PIN(SC9860_SD1_CLK),
+	SPRD_PINCTRL_PIN(SC9860_SD1_CMD),
+	SPRD_PINCTRL_PIN(SC9860_SD1_D0),
+	SPRD_PINCTRL_PIN(SC9860_SD1_D1),
+	SPRD_PINCTRL_PIN(SC9860_SD1_D2),
+	SPRD_PINCTRL_PIN(SC9860_SD1_D3),
+	SPRD_PINCTRL_PIN(SC9860_CLK_AUX0),
+	SPRD_PINCTRL_PIN(SC9860_WIFI_COEXIST),
+	SPRD_PINCTRL_PIN(SC9860_BEIDOU_COEXIST),
+	SPRD_PINCTRL_PIN(SC9860_U3TXD),
+	SPRD_PINCTRL_PIN(SC9860_U3RXD),
+	SPRD_PINCTRL_PIN(SC9860_U3CTS),
+	SPRD_PINCTRL_PIN(SC9860_U3RTS),
+	SPRD_PINCTRL_PIN(SC9860_U0TXD),
+	SPRD_PINCTRL_PIN(SC9860_U0RXD),
+	SPRD_PINCTRL_PIN(SC9860_U0CTS),
+	SPRD_PINCTRL_PIN(SC9860_U0RTS),
+	SPRD_PINCTRL_PIN(SC9860_IIS1DI),
+	SPRD_PINCTRL_PIN(SC9860_IIS1DO),
+	SPRD_PINCTRL_PIN(SC9860_IIS1CLK),
+	SPRD_PINCTRL_PIN(SC9860_IIS1LRCK),
+	SPRD_PINCTRL_PIN(SC9860_SPI0_CSN),
+	SPRD_PINCTRL_PIN(SC9860_SPI0_DO),
+	SPRD_PINCTRL_PIN(SC9860_SPI0_DI),
+	SPRD_PINCTRL_PIN(SC9860_SPI0_CLK),
+	SPRD_PINCTRL_PIN(SC9860_U2TXD),
+	SPRD_PINCTRL_PIN(SC9860_U2RXD),
+	SPRD_PINCTRL_PIN(SC9860_U4TXD),
+	SPRD_PINCTRL_PIN(SC9860_U4RXD),
+	SPRD_PINCTRL_PIN(SC9860_CMMCLK1),
+	SPRD_PINCTRL_PIN(SC9860_CMRST1),
+	SPRD_PINCTRL_PIN(SC9860_CMMCLK0),
+	SPRD_PINCTRL_PIN(SC9860_CMRST0),
+	SPRD_PINCTRL_PIN(SC9860_CMPD0),
+	SPRD_PINCTRL_PIN(SC9860_CMPD1),
+	SPRD_PINCTRL_PIN(SC9860_SCL0),
+	SPRD_PINCTRL_PIN(SC9860_SDA0),
+	SPRD_PINCTRL_PIN(SC9860_SDA6),
+	SPRD_PINCTRL_PIN(SC9860_SCL6),
+	SPRD_PINCTRL_PIN(SC9860_U1TXD),
+	SPRD_PINCTRL_PIN(SC9860_U1RXD),
+	SPRD_PINCTRL_PIN(SC9860_KEYOUT0),
+	SPRD_PINCTRL_PIN(SC9860_KEYOUT1),
+	SPRD_PINCTRL_PIN(SC9860_KEYOUT2),
+	SPRD_PINCTRL_PIN(SC9860_KEYIN0),
+	SPRD_PINCTRL_PIN(SC9860_KEYIN1),
+	SPRD_PINCTRL_PIN(SC9860_KEYIN2),
+	SPRD_PINCTRL_PIN(SC9860_IIS3DI),
+	SPRD_PINCTRL_PIN(SC9860_IIS3DO),
+	SPRD_PINCTRL_PIN(SC9860_IIS3CLK),
+	SPRD_PINCTRL_PIN(SC9860_IIS3LRCK),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL0),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL1),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL10),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL11),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL12),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL13),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL14),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL15),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL16),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL17),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL18),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL19),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL2),
+	SPRD_PINCTRL_PIN(SC9860_EXTINT5),
+	SPRD_PINCTRL_PIN(SC9860_EXTINT6),
+	SPRD_PINCTRL_PIN(SC9860_EXTINT7),
+	SPRD_PINCTRL_PIN(SC9860_GPIO30),
+	SPRD_PINCTRL_PIN(SC9860_GPIO31),
+	SPRD_PINCTRL_PIN(SC9860_GPIO32),
+	SPRD_PINCTRL_PIN(SC9860_GPIO33),
+	SPRD_PINCTRL_PIN(SC9860_GPIO34),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL3),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL4),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL5),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL6),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL7),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL8),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL9),
+	SPRD_PINCTRL_PIN(SC9860_RFFE0_SCK0),
+	SPRD_PINCTRL_PIN(SC9860_GPIO38),
+	SPRD_PINCTRL_PIN(SC9860_RFFE0_SDA0),
+	SPRD_PINCTRL_PIN(SC9860_GPIO39),
+	SPRD_PINCTRL_PIN(SC9860_RFFE1_SCK0),
+	SPRD_PINCTRL_PIN(SC9860_GPIO181),
+	SPRD_PINCTRL_PIN(SC9860_RFFE1_SDA0),
+	SPRD_PINCTRL_PIN(SC9860_GPIO182),
+	SPRD_PINCTRL_PIN(SC9860_RF_LVDS0_ADC_ON),
+	SPRD_PINCTRL_PIN(SC9860_RF_LVDS0_DAC_ON),
+	SPRD_PINCTRL_PIN(SC9860_RFSCK0),
+	SPRD_PINCTRL_PIN(SC9860_RFSDA0),
+	SPRD_PINCTRL_PIN(SC9860_RFSEN0),
+	SPRD_PINCTRL_PIN(SC9860_RF_LVDS1_ADC_ON),
+	SPRD_PINCTRL_PIN(SC9860_RF_LVDS1_DAC_ON),
+	SPRD_PINCTRL_PIN(SC9860_RFSCK1),
+	SPRD_PINCTRL_PIN(SC9860_RFSDA1),
+	SPRD_PINCTRL_PIN(SC9860_RFSEN1),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL38),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL39),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL20_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL21_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL30_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL31_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL32_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL33_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL34_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL35_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL36_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL37_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL22_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL23_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL24_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL25_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL26_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL27_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL28_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL29_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SCL2_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SDA2_MISC),
+	SPRD_PINCTRL_PIN(SC9860_MTCK_ARM_MISC),
+	SPRD_PINCTRL_PIN(SC9860_MTMS_ARM_MISC),
+	SPRD_PINCTRL_PIN(SC9860_XTL_EN0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_PTEST_MISC),
+	SPRD_PINCTRL_PIN(SC9860_AUD_DAD1_MISC),
+	SPRD_PINCTRL_PIN(SC9860_AUD_ADD0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_AUD_ADSYNC_MISC),
+	SPRD_PINCTRL_PIN(SC9860_AUD_SCLK_MISC),
+	SPRD_PINCTRL_PIN(SC9860_CHIP_SLEEP_MISC),
+	SPRD_PINCTRL_PIN(SC9860_CLK_32K_MISC),
+	SPRD_PINCTRL_PIN(SC9860_DCDC_ARM_EN_MISC),
+	SPRD_PINCTRL_PIN(SC9860_EXT_RST_B_MISC),
+	SPRD_PINCTRL_PIN(SC9860_ADI_D_MISC),
+	SPRD_PINCTRL_PIN(SC9860_ADI_SCLK_MISC),
+	SPRD_PINCTRL_PIN(SC9860_XTL_EN1_MISC),
+	SPRD_PINCTRL_PIN(SC9860_ANA_INT_MISC),
+	SPRD_PINCTRL_PIN(SC9860_AUD_DAD0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_AUD_DASYNC_MISC),
+	SPRD_PINCTRL_PIN(SC9860_LCM_RSTN_MISC),
+	SPRD_PINCTRL_PIN(SC9860_DSI_TE_MISC),
+	SPRD_PINCTRL_PIN(SC9860_PWMA_MISC),
+	SPRD_PINCTRL_PIN(SC9860_EXTINT0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_EXTINT1_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SDA1_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SCL1_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SIMCLK2_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SIMDA2_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SIMRST2_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SIMCLK1_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SIMDA1_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SIMRST1_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SIMCLK0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SIMDA0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SIMRST0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SD2_CMD_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SD2_D0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SD2_D1_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SD2_CLK_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SD2_D2_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SD2_D3_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SD0_D3_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SD0_D2_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SD0_CMD_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SD0_D0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SD0_D1_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SD0_CLK_MISC),
+	SPRD_PINCTRL_PIN(SC9860_EMMC_CMD_MISC),
+	SPRD_PINCTRL_PIN(SC9860_EMMC_D6_MISC),
+	SPRD_PINCTRL_PIN(SC9860_EMMC_D7_MISC),
+	SPRD_PINCTRL_PIN(SC9860_EMMC_CLK_MISC),
+	SPRD_PINCTRL_PIN(SC9860_EMMC_D5_MISC),
+	SPRD_PINCTRL_PIN(SC9860_EMMC_D4_MISC),
+	SPRD_PINCTRL_PIN(SC9860_EMMC_DS_MISC),
+	SPRD_PINCTRL_PIN(SC9860_EMMC_D3_MISC),
+	SPRD_PINCTRL_PIN(SC9860_EMMC_RST_MISC),
+	SPRD_PINCTRL_PIN(SC9860_EMMC_D1_MISC),
+	SPRD_PINCTRL_PIN(SC9860_EMMC_D2_MISC),
+	SPRD_PINCTRL_PIN(SC9860_EMMC_D0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_IIS0DI_MISC),
+	SPRD_PINCTRL_PIN(SC9860_IIS0DO_MISC),
+	SPRD_PINCTRL_PIN(SC9860_IIS0CLK_MISC),
+	SPRD_PINCTRL_PIN(SC9860_IIS0LRCK_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SD1_CLK_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SD1_CMD_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SD1_D0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SD1_D1_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SD1_D2_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SD1_D3_MISC),
+	SPRD_PINCTRL_PIN(SC9860_CLK_AUX0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_WIFI_COEXIST_MISC),
+	SPRD_PINCTRL_PIN(SC9860_BEIDOU_COEXIST_MISC),
+	SPRD_PINCTRL_PIN(SC9860_U3TXD_MISC),
+	SPRD_PINCTRL_PIN(SC9860_U3RXD_MISC),
+	SPRD_PINCTRL_PIN(SC9860_U3CTS_MISC),
+	SPRD_PINCTRL_PIN(SC9860_U3RTS_MISC),
+	SPRD_PINCTRL_PIN(SC9860_U0TXD_MISC),
+	SPRD_PINCTRL_PIN(SC9860_U0RXD_MISC),
+	SPRD_PINCTRL_PIN(SC9860_U0CTS_MISC),
+	SPRD_PINCTRL_PIN(SC9860_U0RTS_MISC),
+	SPRD_PINCTRL_PIN(SC9860_IIS1DI_MISC),
+	SPRD_PINCTRL_PIN(SC9860_IIS1DO_MISC),
+	SPRD_PINCTRL_PIN(SC9860_IIS1CLK_MISC),
+	SPRD_PINCTRL_PIN(SC9860_IIS1LRCK_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SPI0_CSN_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SPI0_DO_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SPI0_DI_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SPI0_CLK_MISC),
+	SPRD_PINCTRL_PIN(SC9860_U2TXD_MISC),
+	SPRD_PINCTRL_PIN(SC9860_U2RXD_MISC),
+	SPRD_PINCTRL_PIN(SC9860_U4TXD_MISC),
+	SPRD_PINCTRL_PIN(SC9860_U4RXD_MISC),
+	SPRD_PINCTRL_PIN(SC9860_CMMCLK1_MISC),
+	SPRD_PINCTRL_PIN(SC9860_CMRST1_MISC),
+	SPRD_PINCTRL_PIN(SC9860_CMMCLK0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_CMRST0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_CMPD0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_CMPD1_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SCL0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SDA0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SDA6_MISC),
+	SPRD_PINCTRL_PIN(SC9860_SCL6_MISC),
+	SPRD_PINCTRL_PIN(SC9860_U1TXD_MISC),
+	SPRD_PINCTRL_PIN(SC9860_U1RXD_MISC),
+	SPRD_PINCTRL_PIN(SC9860_KEYOUT0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_KEYOUT1_MISC),
+	SPRD_PINCTRL_PIN(SC9860_KEYOUT2_MISC),
+	SPRD_PINCTRL_PIN(SC9860_KEYIN0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_KEYIN1_MISC),
+	SPRD_PINCTRL_PIN(SC9860_KEYIN2_MISC),
+	SPRD_PINCTRL_PIN(SC9860_IIS3DI_MISC),
+	SPRD_PINCTRL_PIN(SC9860_IIS3DO_MISC),
+	SPRD_PINCTRL_PIN(SC9860_IIS3CLK_MISC),
+	SPRD_PINCTRL_PIN(SC9860_IIS3LRCK_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL1_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL10_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL11_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL12_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL13_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL14_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL15_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL16_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL17_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL18_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL19_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL2_MISC),
+	SPRD_PINCTRL_PIN(SC9860_EXTINT5_MISC),
+	SPRD_PINCTRL_PIN(SC9860_EXTINT6_MISC),
+	SPRD_PINCTRL_PIN(SC9860_EXTINT7_MISC),
+	SPRD_PINCTRL_PIN(SC9860_GPIO30_MISC),
+	SPRD_PINCTRL_PIN(SC9860_GPIO31_MISC),
+	SPRD_PINCTRL_PIN(SC9860_GPIO32_MISC),
+	SPRD_PINCTRL_PIN(SC9860_GPIO33_MISC),
+	SPRD_PINCTRL_PIN(SC9860_GPIO34_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL3_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL4_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL5_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL6_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL7_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL8_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL9_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFFE0_SCK0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_GPIO38_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFFE0_SDA0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_GPIO39_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFFE1_SCK0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_GPIO181_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFFE1_SDA0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_GPIO182_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RF_LVDS0_ADC_ON_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RF_LVDS0_DAC_ON_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFSCK0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFSDA0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFSEN0_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RF_LVDS1_ADC_ON_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RF_LVDS1_DAC_ON_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFSCK1_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFSDA1_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFSEN1_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL38_MISC),
+	SPRD_PINCTRL_PIN(SC9860_RFCTL39_MISC),
+};
+
+static int sprd_pinctrl_probe(struct platform_device *pdev)
+{
+	return sprd_pinctrl_core_probe(pdev, sprd_sc9860_pins_info,
+				       ARRAY_SIZE(sprd_sc9860_pins_info));
+}
+
+static const struct of_device_id sprd_pinctrl_of_match[] = {
+	{
+		.compatible = "sprd,sc9860-pinctrl",
+	},
+	{ },
+};
+MODULE_DEVICE_TABLE(of, sprd_pinctrl_of_match);
+
+static struct platform_driver sprd_pinctrl_driver = {
+	.driver = {
+		.name = "sprd-pinctrl",
+		.owner = THIS_MODULE,
+		.of_match_table = sprd_pinctrl_of_match,
+	},
+	.probe = sprd_pinctrl_probe,
+	.remove = sprd_pinctrl_remove,
+	.shutdown = sprd_pinctrl_shutdown,
+};
+
+static int sprd_pinctrl_init(void)
+{
+	return platform_driver_register(&sprd_pinctrl_driver);
+}
+module_init(sprd_pinctrl_init);
+
+static void sprd_pinctrl_exit(void)
+{
+	platform_driver_unregister(&sprd_pinctrl_driver);
+}
+module_exit(sprd_pinctrl_exit);
+
+MODULE_DESCRIPTION("SPREADTRUM Pin Controller Driver");
+MODULE_AUTHOR("Baolin Wang <baolin.wang@spreadtrum.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/pinctrl/sprd/pinctrl-sprd.c b/drivers/pinctrl/sprd/pinctrl-sprd.c
new file mode 100644
index 000000000000..63529911445c
--- /dev/null
+++ b/drivers/pinctrl/sprd/pinctrl-sprd.c
@@ -0,0 +1,1117 @@
+/*
+ * Spreadtrum pin controller driver
+ * Copyright (C) 2017 Spreadtrum  - http://www.spreadtrum.com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+
+#include <linux/debugfs.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/pinctrl/machine.h>
+#include <linux/pinctrl/pinconf.h>
+#include <linux/pinctrl/pinconf-generic.h>
+#include <linux/pinctrl/pinctrl.h>
+#include <linux/pinctrl/pinmux.h>
+#include <linux/slab.h>
+
+#include "../core.h"
+#include "../pinmux.h"
+#include "../pinconf.h"
+#include "../pinctrl-utils.h"
+#include "pinctrl-sprd.h"
+
+#define PINCTRL_BIT_MASK(width)		(~(~0UL << (width)))
+#define PINCTRL_REG_OFFSET		0x20
+#define PINCTRL_REG_MISC_OFFSET		0x4020
+#define PINCTRL_REG_LEN			0x4
+
+#define PIN_FUNC_MASK			(BIT(4) | BIT(5))
+#define PIN_FUNC_SEL_1			~PIN_FUNC_MASK
+#define PIN_FUNC_SEL_2			BIT(4)
+#define PIN_FUNC_SEL_3			BIT(5)
+#define PIN_FUNC_SEL_4			PIN_FUNC_MASK
+
+#define AP_SLEEP_MODE			BIT(13)
+#define PUBCP_SLEEP_MODE		BIT(14)
+#define TGLDSP_SLEEP_MODE		BIT(15)
+#define AGDSP_SLEEP_MODE		BIT(16)
+#define SLEEP_MODE_MASK			GENMASK(3, 0)
+#define SLEEP_MODE_SHIFT		13
+
+#define SLEEP_INPUT			BIT(1)
+#define SLEEP_INPUT_MASK		0x1
+#define SLEEP_INPUT_SHIFT		1
+
+#define SLEEP_OUTPUT			BIT(0)
+#define SLEEP_OUTPUT_MASK		0x1
+#define SLEEP_OUTPUT_SHIFT		0
+
+#define DRIVE_STRENGTH_MASK		GENMASK(3, 0)
+#define DRIVE_STRENGTH_SHIFT		19
+
+#define SLEEP_PULL_DOWN			BIT(2)
+#define SLEEP_PULL_DOWN_MASK		0x1
+#define SLEEP_PULL_DOWN_SHIFT		2
+
+#define PULL_DOWN			BIT(6)
+#define PULL_DOWN_MASK			0x1
+#define PULL_DOWN_SHIFT			6
+
+#define SLEEP_PULL_UP			BIT(3)
+#define SLEEP_PULL_UP_MASK		0x1
+#define SLEEP_PULL_UP_SHIFT		3
+
+#define PULL_UP_20K			(BIT(12) | BIT(7))
+#define PULL_UP_4_7K			BIT(12)
+#define PULL_UP_MASK			0x21
+#define PULL_UP_SHIFT			7
+
+#define INPUT_SCHMITT			BIT(11)
+#define INPUT_SCHMITT_MASK		0x1
+#define INPUT_SCHMITT_SHIFT		11
+
+enum pin_sleep_mode {
+	AP_SLEEP = BIT(0),
+	PUBCP_SLEEP = BIT(1),
+	TGLDSP_SLEEP = BIT(2),
+	AGDSP_SLEEP = BIT(3),
+};
+
+enum pin_func_sel {
+	PIN_FUNC_1,
+	PIN_FUNC_2,
+	PIN_FUNC_3,
+	PIN_FUNC_4,
+	PIN_FUNC_MAX,
+};
+
+/**
+ * struct sprd_pin: represent one pin's description
+ * @name: pin name
+ * @number: pin number
+ * @type: pin type, can be GLOBAL_CTRL_PIN/COMMON_PIN/MISC_PIN
+ * @reg: pin register address
+ * @bit_offset: bit offset in pin register
+ * @bit_width: bit width in pin register
+ */
+struct sprd_pin {
+	const char *name;
+	unsigned int number;
+	enum pin_type type;
+	unsigned long reg;
+	unsigned long bit_offset;
+	unsigned long bit_width;
+};
+
+/**
+ * struct sprd_pin_group: represent one group's description
+ * @name: group name
+ * @npins: pin numbers of this group
+ * @pins: pointer to pins array
+ */
+struct sprd_pin_group {
+	const char *name;
+	unsigned int npins;
+	unsigned int *pins;
+};
+
+/**
+ * struct sprd_pinctrl_soc_info: represent the SoC's pins description
+ * @groups: pointer to groups of pins
+ * @ngroups: group numbers of the whole SoC
+ * @pins: pointer to pins description
+ * @npins: pin numbers of the whole SoC
+ * @grp_names: pointer to group names array
+ */
+struct sprd_pinctrl_soc_info {
+	struct sprd_pin_group *groups;
+	unsigned int ngroups;
+	struct sprd_pin *pins;
+	unsigned int npins;
+	const char **grp_names;
+};
+
+/**
+ * struct sprd_pinctrl: represent the pin controller device
+ * @dev: pointer to the device structure
+ * @pctl: pointer to the pinctrl handle
+ * @base: base address of the controller
+ * @info: pointer to SoC's pins description information
+ */
+struct sprd_pinctrl {
+	struct device *dev;
+	struct pinctrl_dev *pctl;
+	void __iomem *base;
+	struct sprd_pinctrl_soc_info *info;
+};
+
+enum sprd_pinconf_params {
+	SPRD_PIN_CONFIG_CONTROL = PIN_CONFIG_END + 1,
+	SPRD_PIN_CONFIG_SLEEP_MODE = PIN_CONFIG_END + 2,
+};
+
+static int sprd_pinctrl_get_id_by_name(struct sprd_pinctrl *sprd_pctl,
+				       const char *name)
+{
+	struct sprd_pinctrl_soc_info *info = sprd_pctl->info;
+	int i;
+
+	for (i = 0; i < info->npins; i++) {
+		if (!strcmp(info->pins[i].name, name))
+			return info->pins[i].number;
+	}
+
+	return -ENODEV;
+}
+
+static struct sprd_pin *
+sprd_pinctrl_get_pin_by_id(struct sprd_pinctrl *sprd_pctl, unsigned int id)
+{
+	struct sprd_pinctrl_soc_info *info = sprd_pctl->info;
+	struct sprd_pin *pin = NULL;
+	int i;
+
+	for (i = 0; i < info->npins; i++) {
+		if (info->pins[i].number == id) {
+			pin = &info->pins[i];
+			break;
+		}
+	}
+
+	return pin;
+}
+
+static const struct sprd_pin_group *
+sprd_pinctrl_find_group_by_name(struct sprd_pinctrl *sprd_pctl,
+				const char *name)
+{
+	struct sprd_pinctrl_soc_info *info = sprd_pctl->info;
+	const struct sprd_pin_group *grp = NULL;
+	int i;
+
+	for (i = 0; i < info->ngroups; i++) {
+		if (!strcmp(info->groups[i].name, name)) {
+			grp = &info->groups[i];
+			break;
+		}
+	}
+
+	return grp;
+}
+
+static int sprd_pctrl_group_count(struct pinctrl_dev *pctldev)
+{
+	struct sprd_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
+	struct sprd_pinctrl_soc_info *info = pctl->info;
+
+	return info->ngroups;
+}
+
+static const char *sprd_pctrl_group_name(struct pinctrl_dev *pctldev,
+					 unsigned int selector)
+{
+	struct sprd_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
+	struct sprd_pinctrl_soc_info *info = pctl->info;
+
+	return info->groups[selector].name;
+}
+
+static int sprd_pctrl_group_pins(struct pinctrl_dev *pctldev,
+				 unsigned int selector,
+				 const unsigned int **pins,
+				 unsigned int *npins)
+{
+	struct sprd_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
+	struct sprd_pinctrl_soc_info *info = pctl->info;
+
+	if (selector >= info->ngroups)
+		return -EINVAL;
+
+	*pins = info->groups[selector].pins;
+	*npins = info->groups[selector].npins;
+
+	return 0;
+}
+
+static int sprd_dt_node_to_map(struct pinctrl_dev *pctldev,
+			       struct device_node *np,
+			       struct pinctrl_map **map,
+			       unsigned int *num_maps)
+{
+	struct sprd_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
+	const struct sprd_pin_group *grp;
+	unsigned long *configs = NULL;
+	unsigned int num_configs = 0;
+	unsigned int reserved_maps = 0;
+	unsigned int reserve = 0;
+	const char *function;
+	enum pinctrl_map_type type;
+	int ret;
+
+	grp = sprd_pinctrl_find_group_by_name(pctl, np->name);
+	if (!grp) {
+		dev_err(pctl->dev, "unable to find group for node %s\n",
+			of_node_full_name(np));
+		return -EINVAL;
+	}
+
+	ret = of_property_count_strings(np, "pins");
+	if (ret < 0)
+		return ret;
+
+	if (ret == 1)
+		type = PIN_MAP_TYPE_CONFIGS_PIN;
+	else
+		type = PIN_MAP_TYPE_CONFIGS_GROUP;
+
+	ret = of_property_read_string(np, "function", &function);
+	if (ret < 0) {
+		if (ret != -EINVAL)
+			dev_err(pctl->dev,
+				"%s: could not parse property function\n",
+				of_node_full_name(np));
+		function = NULL;
+	}
+
+	ret = pinconf_generic_parse_dt_config(np, pctldev, &configs,
+					      &num_configs);
+	if (ret < 0) {
+		dev_err(pctl->dev, "%s: could not parse node property\n",
+			of_node_full_name(np));
+		return ret;
+	}
+
+	*map = NULL;
+	*num_maps = 0;
+
+	if (function != NULL)
+		reserve++;
+	if (num_configs)
+		reserve++;
+
+	ret = pinctrl_utils_reserve_map(pctldev, map, &reserved_maps,
+					num_maps, reserve);
+	if (ret < 0)
+		goto out;
+
+	if (function) {
+		ret = pinctrl_utils_add_map_mux(pctldev, map,
+						&reserved_maps, num_maps,
+						grp->name, function);
+		if (ret < 0)
+			goto out;
+	}
+
+	if (num_configs) {
+		const char *group_or_pin;
+		unsigned int pin_id;
+
+		if (type == PIN_MAP_TYPE_CONFIGS_PIN) {
+			pin_id = grp->pins[0];
+			group_or_pin = pin_get_name(pctldev, pin_id);
+		} else {
+			group_or_pin = grp->name;
+		}
+
+		ret = pinctrl_utils_add_map_configs(pctldev, map,
+						    &reserved_maps, num_maps,
+						    group_or_pin, configs,
+						    num_configs, type);
+	}
+
+out:
+	kfree(configs);
+	return ret;
+}
+
+static void sprd_pctrl_dbg_show(struct pinctrl_dev *pctldev, struct seq_file *s,
+				unsigned int offset)
+{
+	seq_printf(s, "%s", dev_name(pctldev->dev));
+}
+
+static const struct pinctrl_ops sprd_pctrl_ops = {
+	.get_groups_count = sprd_pctrl_group_count,
+	.get_group_name = sprd_pctrl_group_name,
+	.get_group_pins = sprd_pctrl_group_pins,
+	.pin_dbg_show = sprd_pctrl_dbg_show,
+	.dt_node_to_map = sprd_dt_node_to_map,
+	.dt_free_map = pinctrl_utils_free_map,
+};
+
+static int sprd_pmx_get_function_count(struct pinctrl_dev *pctldev)
+{
+	return PIN_FUNC_MAX;
+}
+
+static const char *sprd_pmx_get_function_name(struct pinctrl_dev *pctldev,
+					      unsigned int selector)
+{
+	switch (selector) {
+	case PIN_FUNC_1:
+		return "func1";
+	case PIN_FUNC_2:
+		return "func2";
+	case PIN_FUNC_3:
+		return "func3";
+	case PIN_FUNC_4:
+		return "func4";
+	default:
+		return "null";
+	}
+}
+
+static int sprd_pmx_get_function_groups(struct pinctrl_dev *pctldev,
+					unsigned int selector,
+					const char * const **groups,
+					unsigned int * const num_groups)
+{
+	struct sprd_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
+	struct sprd_pinctrl_soc_info *info = pctl->info;
+
+	*groups = info->grp_names;
+	*num_groups = info->ngroups;
+
+	return 0;
+}
+
+static int sprd_pmx_set_mux(struct pinctrl_dev *pctldev,
+			    unsigned int func_selector,
+			    unsigned int group_selector)
+{
+	struct sprd_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
+	struct sprd_pinctrl_soc_info *info = pctl->info;
+	struct sprd_pin_group *grp = &info->groups[group_selector];
+	unsigned int i, grp_pins = grp->npins;
+	unsigned long reg;
+	unsigned int val = 0;
+
+	if (group_selector >= info->ngroups)
+		return -EINVAL;
+
+	switch (func_selector) {
+	case PIN_FUNC_1:
+		val &= PIN_FUNC_SEL_1;
+		break;
+	case PIN_FUNC_2:
+		val |= PIN_FUNC_SEL_2;
+		break;
+	case PIN_FUNC_3:
+		val |= PIN_FUNC_SEL_3;
+		break;
+	case PIN_FUNC_4:
+		val |= PIN_FUNC_SEL_4;
+		break;
+	default:
+		break;
+	}
+
+	for (i = 0; i < grp_pins; i++) {
+		unsigned int pin_id = grp->pins[i];
+		struct sprd_pin *pin = sprd_pinctrl_get_pin_by_id(pctl, pin_id);
+
+		if (!pin || pin->type != COMMON_PIN)
+			continue;
+
+		reg = readl((void __iomem *)pin->reg);
+		reg &= ~PIN_FUNC_MASK;
+		reg |= val;
+		writel(reg, (void __iomem *)pin->reg);
+	}
+
+	return 0;
+}
+
+static const struct pinmux_ops sprd_pmx_ops = {
+	.get_functions_count = sprd_pmx_get_function_count,
+	.get_function_name = sprd_pmx_get_function_name,
+	.get_function_groups = sprd_pmx_get_function_groups,
+	.set_mux = sprd_pmx_set_mux,
+};
+
+static int sprd_pinconf_get(struct pinctrl_dev *pctldev, unsigned int pin_id,
+			    unsigned long *config)
+{
+	struct sprd_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
+	struct sprd_pin *pin = sprd_pinctrl_get_pin_by_id(pctl, pin_id);
+	unsigned int param = pinconf_to_config_param(*config);
+	unsigned int reg, arg;
+
+	if (!pin)
+		return -EINVAL;
+
+	if (pin->type == GLOBAL_CTRL_PIN) {
+		reg = (readl((void __iomem *)pin->reg) >>
+			   pin->bit_offset) & PINCTRL_BIT_MASK(pin->bit_width);
+	} else {
+		reg = readl((void __iomem *)pin->reg);
+	}
+
+	if (pin->type == GLOBAL_CTRL_PIN &&
+	    param == SPRD_PIN_CONFIG_CONTROL) {
+		arg = reg;
+	} else if (pin->type == COMMON_PIN) {
+		switch (param) {
+		case SPRD_PIN_CONFIG_SLEEP_MODE:
+			arg = (reg >> SLEEP_MODE_SHIFT) & SLEEP_MODE_MASK;
+			break;
+		case PIN_CONFIG_INPUT_ENABLE:
+			arg = (reg >> SLEEP_INPUT_SHIFT) & SLEEP_INPUT_MASK;
+			break;
+		case PIN_CONFIG_OUTPUT:
+			arg = reg & SLEEP_OUTPUT_MASK;
+			break;
+		case PIN_CONFIG_SLEEP_HARDWARE_STATE:
+			arg = 0;
+			break;
+		default:
+			return -ENOTSUPP;
+		}
+	} else if (pin->type == MISC_PIN) {
+		switch (param) {
+		case PIN_CONFIG_DRIVE_STRENGTH:
+			arg = (reg >> DRIVE_STRENGTH_SHIFT) &
+				DRIVE_STRENGTH_MASK;
+			break;
+		case PIN_CONFIG_BIAS_PULL_DOWN:
+			/* combine sleep pull down and pull down config */
+			arg = ((reg >> SLEEP_PULL_DOWN_SHIFT) &
+			       SLEEP_PULL_DOWN_MASK) << 16;
+			arg |= (reg >> PULL_DOWN_SHIFT) & PULL_DOWN_MASK;
+			break;
+		case PIN_CONFIG_INPUT_SCHMITT_ENABLE:
+			arg = (reg >> INPUT_SCHMITT_SHIFT) & INPUT_SCHMITT_MASK;
+			break;
+		case PIN_CONFIG_BIAS_PULL_UP:
+			/* combine sleep pull up and pull up config */
+			arg = ((reg >> SLEEP_PULL_UP_SHIFT) &
+			       SLEEP_PULL_UP_MASK) << 16;
+			arg |= (reg >> PULL_UP_SHIFT) & PULL_UP_MASK;
+			break;
+		case PIN_CONFIG_SLEEP_HARDWARE_STATE:
+			arg = 0;
+			break;
+		default:
+			return -ENOTSUPP;
+		}
+	} else {
+		return -ENOTSUPP;
+	}
+
+	*config = pinconf_to_config_packed(param, arg);
+	return 0;
+}
+
+static unsigned int sprd_pinconf_drive(unsigned int mA)
+{
+	unsigned int val = 0;
+
+	switch (mA) {
+	case 2:
+		break;
+	case 4:
+		val |= BIT(19);
+		break;
+	case 6:
+		val |= BIT(20);
+		break;
+	case 8:
+		val |= BIT(19) | BIT(20);
+		break;
+	case 10:
+		val |= BIT(21);
+		break;
+	case 12:
+		val |= BIT(21) | BIT(19);
+		break;
+	case 14:
+		val |= BIT(21) | BIT(20);
+		break;
+	case 16:
+		val |= BIT(19) | BIT(20) | BIT(21);
+		break;
+	case 20:
+		val |= BIT(22);
+		break;
+	case 21:
+		val |= BIT(22) | BIT(19);
+		break;
+	case 24:
+		val |= BIT(22) | BIT(20);
+		break;
+	case 25:
+		val |= BIT(22) | BIT(20) | BIT(19);
+		break;
+	case 27:
+		val |= BIT(22) | BIT(21);
+		break;
+	case 29:
+		val |= BIT(22) | BIT(21) | BIT(19);
+		break;
+	case 31:
+		val |= BIT(22) | BIT(21) | BIT(20);
+		break;
+	case 33:
+		val |= BIT(22) | BIT(21) | BIT(20) | BIT(19);
+		break;
+	default:
+		break;
+	}
+
+	return val;
+}
+
+static bool sprd_pinctrl_check_sleep_config(unsigned long *configs,
+					    unsigned int num_configs)
+{
+	unsigned int param;
+	int i;
+
+	for (i = 0; i < num_configs; i++) {
+		param = pinconf_to_config_param(configs[i]);
+		if (param == PIN_CONFIG_SLEEP_HARDWARE_STATE)
+			return true;
+	}
+
+	return false;
+}
+
+static int sprd_pinconf_set(struct pinctrl_dev *pctldev, unsigned int pin_id,
+			    unsigned long *configs, unsigned int num_configs)
+{
+	struct sprd_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
+	struct sprd_pin *pin = sprd_pinctrl_get_pin_by_id(pctl, pin_id);
+	bool is_sleep_config;
+	unsigned long reg;
+	int i;
+
+	if (!pin)
+		return -EINVAL;
+
+	is_sleep_config = sprd_pinctrl_check_sleep_config(configs, num_configs);
+
+	for (i = 0; i < num_configs; i++) {
+		unsigned int param, arg, shift, mask, val;
+
+		param = pinconf_to_config_param(configs[i]);
+		arg = pinconf_to_config_argument(configs[i]);
+
+		val = 0;
+		shift = 0;
+		mask = 0;
+		if (pin->type == GLOBAL_CTRL_PIN &&
+		    param == SPRD_PIN_CONFIG_CONTROL) {
+			val = arg;
+		} else if (pin->type == COMMON_PIN) {
+			switch (param) {
+			case SPRD_PIN_CONFIG_SLEEP_MODE:
+				if (arg & AP_SLEEP)
+					val |= AP_SLEEP_MODE;
+				if (arg & PUBCP_SLEEP)
+					val |= PUBCP_SLEEP_MODE;
+				if (arg & TGLDSP_SLEEP)
+					val |= TGLDSP_SLEEP_MODE;
+				if (arg & AGDSP_SLEEP)
+					val |= AGDSP_SLEEP_MODE;
+
+				mask = SLEEP_MODE_MASK;
+				shift = SLEEP_MODE_SHIFT;
+				break;
+			case PIN_CONFIG_INPUT_ENABLE:
+				if (is_sleep_config == true) {
+					if (arg > 0)
+						val |= SLEEP_INPUT;
+					else
+						val &= ~SLEEP_INPUT;
+
+					mask = SLEEP_INPUT_MASK;
+					shift = SLEEP_INPUT_SHIFT;
+				}
+				break;
+			case PIN_CONFIG_OUTPUT:
+				if (is_sleep_config == true) {
+					val |= SLEEP_OUTPUT;
+					mask = SLEEP_OUTPUT_MASK;
+					shift = SLEEP_OUTPUT_SHIFT;
+				}
+				break;
+			case PIN_CONFIG_SLEEP_HARDWARE_STATE:
+				continue;
+			default:
+				return -ENOTSUPP;
+			}
+		} else if (pin->type == MISC_PIN) {
+			switch (param) {
+			case PIN_CONFIG_DRIVE_STRENGTH:
+				if (arg < 2 || arg > 60)
+					return -EINVAL;
+
+				val = sprd_pinconf_drive(arg);
+				mask = DRIVE_STRENGTH_MASK;
+				shift = DRIVE_STRENGTH_SHIFT;
+				break;
+			case PIN_CONFIG_BIAS_PULL_DOWN:
+				if (is_sleep_config == true) {
+					val |= SLEEP_PULL_DOWN;
+					mask = SLEEP_PULL_DOWN_MASK;
+					shift = SLEEP_PULL_DOWN_SHIFT;
+				} else {
+					val |= PULL_DOWN;
+					mask = PULL_DOWN_MASK;
+					shift = PULL_DOWN_SHIFT;
+				}
+				break;
+			case PIN_CONFIG_INPUT_SCHMITT_ENABLE:
+				if (arg > 0)
+					val |= INPUT_SCHMITT;
+				else
+					val &= ~INPUT_SCHMITT;
+
+				mask = INPUT_SCHMITT_MASK;
+				shift = INPUT_SCHMITT_SHIFT;
+				break;
+			case PIN_CONFIG_BIAS_PULL_UP:
+				if (is_sleep_config == true) {
+					val |= SLEEP_PULL_UP;
+					mask = SLEEP_PULL_UP_MASK;
+					shift = SLEEP_PULL_UP_SHIFT;
+				} else {
+					if (arg == 20000)
+						val |= PULL_UP_20K;
+					else if (arg == 4700)
+						val |= PULL_UP_4_7K;
+
+					mask = PULL_UP_MASK;
+					shift = PULL_UP_SHIFT;
+				}
+				break;
+			case PIN_CONFIG_SLEEP_HARDWARE_STATE:
+				continue;
+			default:
+				return -ENOTSUPP;
+			}
+		} else {
+			return -ENOTSUPP;
+		}
+
+		if (pin->type == GLOBAL_CTRL_PIN) {
+			reg = readl((void __iomem *)pin->reg);
+			reg &= ~(PINCTRL_BIT_MASK(pin->bit_width)
+				<< pin->bit_offset);
+			reg |= (val & PINCTRL_BIT_MASK(pin->bit_width))
+				<< pin->bit_offset;
+			writel(reg, (void __iomem *)pin->reg);
+		} else {
+			reg = readl((void __iomem *)pin->reg);
+			reg &= ~(mask << shift);
+			reg |= val;
+			writel(reg, (void __iomem *)pin->reg);
+		}
+	}
+
+	return 0;
+}
+
+static int sprd_pinconf_group_get(struct pinctrl_dev *pctldev,
+				  unsigned int selector, unsigned long *config)
+{
+	struct sprd_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
+	struct sprd_pinctrl_soc_info *info = pctl->info;
+	struct sprd_pin_group *grp;
+	unsigned int pin_id;
+
+	if (selector >= info->ngroups)
+		return -EINVAL;
+
+	grp = &info->groups[selector];
+	pin_id = grp->pins[0];
+
+	return sprd_pinconf_get(pctldev, pin_id, config);
+}
+
+static int sprd_pinconf_group_set(struct pinctrl_dev *pctldev,
+				  unsigned int selector,
+				  unsigned long *configs,
+				  unsigned int num_configs)
+{
+	struct sprd_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
+	struct sprd_pinctrl_soc_info *info = pctl->info;
+	struct sprd_pin_group *grp;
+	int ret, i;
+
+	if (selector >= info->ngroups)
+		return -EINVAL;
+
+	grp = &info->groups[selector];
+
+	for (i = 0; i < grp->npins; i++) {
+		unsigned int pin_id = grp->pins[i];
+
+		ret = sprd_pinconf_set(pctldev, pin_id, configs, num_configs);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int sprd_pinconf_get_config(struct pinctrl_dev *pctldev,
+				   unsigned int pin_id,
+				   unsigned long *config)
+{
+	struct sprd_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
+	struct sprd_pin *pin = sprd_pinctrl_get_pin_by_id(pctl, pin_id);
+
+	if (!pin)
+		return -EINVAL;
+
+	if (pin->type == GLOBAL_CTRL_PIN) {
+		*config = (readl((void __iomem *)pin->reg) >>
+			   pin->bit_offset) & PINCTRL_BIT_MASK(pin->bit_width);
+	} else {
+		*config = readl((void __iomem *)pin->reg);
+	}
+
+	return 0;
+}
+
+static void sprd_pinconf_dbg_show(struct pinctrl_dev *pctldev,
+				  struct seq_file *s, unsigned int pin_id)
+{
+	unsigned long config;
+	int ret;
+
+	ret = sprd_pinconf_get_config(pctldev, pin_id, &config);
+	if (ret)
+		return;
+
+	seq_printf(s, "0x%lx", config);
+}
+
+static void sprd_pinconf_group_dbg_show(struct pinctrl_dev *pctldev,
+					struct seq_file *s,
+					unsigned int selector)
+{
+	struct sprd_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
+	struct sprd_pinctrl_soc_info *info = pctl->info;
+	struct sprd_pin_group *grp;
+	unsigned long config;
+	const char *name;
+	int i, ret;
+
+	if (selector >= info->ngroups)
+		return;
+
+	grp = &info->groups[selector];
+
+	seq_printf(s, "\n");
+	for (i = 0; i < grp->npins; i++, config++) {
+		unsigned int pin_id = grp->pins[i];
+
+		name = pin_get_name(pctldev, pin_id);
+		ret = sprd_pinconf_get_config(pctldev, pin_id, &config);
+		if (ret)
+			return;
+
+		seq_printf(s, "%s: 0x%lx ", name, config);
+	}
+}
+
+static const struct pinconf_ops sprd_pinconf_ops = {
+	.is_generic = true,
+	.pin_config_get = sprd_pinconf_get,
+	.pin_config_set = sprd_pinconf_set,
+	.pin_config_group_get = sprd_pinconf_group_get,
+	.pin_config_group_set = sprd_pinconf_group_set,
+	.pin_config_dbg_show = sprd_pinconf_dbg_show,
+	.pin_config_group_dbg_show = sprd_pinconf_group_dbg_show,
+};
+
+static const struct pinconf_generic_params sprd_dt_params[] = {
+	{"sprd,control", SPRD_PIN_CONFIG_CONTROL, 0},
+	{"sprd,sleep-mode", SPRD_PIN_CONFIG_SLEEP_MODE, 0},
+};
+
+#ifdef CONFIG_DEBUG_FS
+static const struct pin_config_item sprd_conf_items[] = {
+	PCONFDUMP(SPRD_PIN_CONFIG_CONTROL, "global control", NULL, true),
+	PCONFDUMP(SPRD_PIN_CONFIG_SLEEP_MODE, "sleep mode", NULL, true),
+};
+#endif
+
+static struct pinctrl_desc sprd_pinctrl_desc = {
+	.pctlops = &sprd_pctrl_ops,
+	.pmxops = &sprd_pmx_ops,
+	.confops = &sprd_pinconf_ops,
+	.num_custom_params = ARRAY_SIZE(sprd_dt_params),
+	.custom_params = sprd_dt_params,
+#ifdef CONFIG_DEBUG_FS
+	.custom_conf_items = sprd_conf_items,
+#endif
+	.owner = THIS_MODULE,
+};
+
+static int sprd_pinctrl_parse_groups(struct device_node *np,
+				     struct sprd_pinctrl *sprd_pctl,
+				     struct sprd_pin_group *grp)
+{
+	struct property *prop;
+	const char *pin_name;
+	int ret, i = 0;
+
+	ret = of_property_count_strings(np, "pins");
+	if (ret < 0)
+		return ret;
+
+	grp->name = np->name;
+	grp->npins = ret;
+	grp->pins = devm_kzalloc(sprd_pctl->dev, grp->npins *
+				 sizeof(unsigned int), GFP_KERNEL);
+	if (!grp->pins)
+		return -ENOMEM;
+
+	of_property_for_each_string(np, "pins", prop, pin_name) {
+		ret = sprd_pinctrl_get_id_by_name(sprd_pctl, pin_name);
+		if (ret >= 0)
+			grp->pins[i++] = ret;
+	}
+
+	for (i = 0; i < grp->npins; i++) {
+		dev_dbg(sprd_pctl->dev,
+			"Group[%s] contains [%d] pins: id = %d\n",
+			grp->name, grp->npins, grp->pins[i]);
+	}
+
+	return 0;
+}
+
+static unsigned int sprd_pinctrl_get_groups(struct device_node *np)
+{
+	struct device_node *child;
+	unsigned int group_cnt, cnt;
+
+	group_cnt = of_get_child_count(np);
+
+	for_each_child_of_node(np, child) {
+		cnt = of_get_child_count(child);
+		if (cnt > 0)
+			group_cnt += cnt;
+	}
+
+	return group_cnt;
+}
+
+static int sprd_pinctrl_parse_dt(struct sprd_pinctrl *sprd_pctl)
+{
+	struct sprd_pinctrl_soc_info *info = sprd_pctl->info;
+	struct device_node *np = sprd_pctl->dev->of_node;
+	struct device_node *child, *sub_child;
+	struct sprd_pin_group *grp;
+	const char **temp;
+	int ret;
+
+	if (!np)
+		return -ENODEV;
+
+	info->ngroups = sprd_pinctrl_get_groups(np);
+	if (!info->ngroups)
+		return 0;
+
+	info->groups = devm_kzalloc(sprd_pctl->dev, info->ngroups *
+				    sizeof(struct sprd_pin_group),
+				    GFP_KERNEL);
+	if (!info->groups)
+		return -ENOMEM;
+
+	info->grp_names = devm_kzalloc(sprd_pctl->dev,
+				       info->ngroups * sizeof(char *),
+				       GFP_KERNEL);
+	if (!info->grp_names)
+		return -ENOMEM;
+
+	temp = info->grp_names;
+	grp = info->groups;
+
+	for_each_child_of_node(np, child) {
+		ret = sprd_pinctrl_parse_groups(child, sprd_pctl, grp);
+		if (ret)
+			return ret;
+
+		*temp++ = grp->name;
+		grp++;
+
+		if (of_get_child_count(child) > 0) {
+			for_each_child_of_node(child, sub_child) {
+				ret = sprd_pinctrl_parse_groups(sub_child,
+								sprd_pctl, grp);
+				if (ret)
+					return ret;
+
+				*temp++ = grp->name;
+				grp++;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int sprd_pinctrl_add_pins(struct sprd_pinctrl *sprd_pctl,
+				 struct sprd_pins_info *sprd_soc_pin_info,
+				 int pins_cnt)
+{
+	struct sprd_pinctrl_soc_info *info = sprd_pctl->info;
+	unsigned int ctrl_pin = 0, com_pin = 0;
+	struct sprd_pin *pin;
+	int i;
+
+	info->npins = pins_cnt;
+	info->pins = devm_kzalloc(sprd_pctl->dev,
+				  info->npins * sizeof(struct sprd_pin),
+				  GFP_KERNEL);
+	if (!info->pins)
+		return -ENOMEM;
+
+	for (i = 0, pin = info->pins; i < info->npins; i++, pin++) {
+		unsigned int reg;
+
+		pin->name = sprd_soc_pin_info[i].name;
+		pin->type = sprd_soc_pin_info[i].type;
+		pin->number = sprd_soc_pin_info[i].num;
+		reg = sprd_soc_pin_info[i].reg;
+		if (pin->type == GLOBAL_CTRL_PIN) {
+			pin->reg = (unsigned long)sprd_pctl->base +
+				PINCTRL_REG_LEN * reg;
+			pin->bit_offset = sprd_soc_pin_info[i].bit_offset;
+			pin->bit_width = sprd_soc_pin_info[i].bit_width;
+			ctrl_pin++;
+		} else if (pin->type == COMMON_PIN) {
+			pin->reg = (unsigned long)sprd_pctl->base +
+				PINCTRL_REG_OFFSET + PINCTRL_REG_LEN *
+				(i - ctrl_pin);
+			com_pin++;
+		} else if (pin->type == MISC_PIN) {
+			pin->reg = (unsigned long)sprd_pctl->base +
+				PINCTRL_REG_MISC_OFFSET + PINCTRL_REG_LEN *
+				(i - ctrl_pin - com_pin);
+		}
+	}
+
+	for (i = 0, pin = info->pins; i < info->npins; pin++, i++) {
+		dev_dbg(sprd_pctl->dev, "pin name[%s-%d], type = %d, "
+			"bit offset = %ld, bit width = %ld, reg = 0x%lx\n",
+			pin->name, pin->number, pin->type,
+			pin->bit_offset, pin->bit_width, pin->reg);
+	}
+
+	return 0;
+}
+
+int sprd_pinctrl_core_probe(struct platform_device *pdev,
+			    struct sprd_pins_info *sprd_soc_pin_info,
+			    int pins_cnt)
+{
+	struct sprd_pinctrl *sprd_pctl;
+	struct sprd_pinctrl_soc_info *pinctrl_info;
+	struct pinctrl_pin_desc *pin_desc;
+	struct resource *res;
+	int ret, i;
+
+	sprd_pctl = devm_kzalloc(&pdev->dev, sizeof(struct sprd_pinctrl),
+				 GFP_KERNEL);
+	if (!sprd_pctl)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	sprd_pctl->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(sprd_pctl->base))
+		return PTR_ERR(sprd_pctl->base);
+
+	pinctrl_info = devm_kzalloc(&pdev->dev,
+				    sizeof(struct sprd_pinctrl_soc_info),
+				    GFP_KERNEL);
+	if (!pinctrl_info)
+		return -ENOMEM;
+
+	sprd_pctl->info = pinctrl_info;
+	sprd_pctl->dev = &pdev->dev;
+	platform_set_drvdata(pdev, sprd_pctl);
+
+	ret = sprd_pinctrl_add_pins(sprd_pctl, sprd_soc_pin_info, pins_cnt);
+	if (ret) {
+		dev_err(&pdev->dev, "fail to add pins information\n");
+		return ret;
+	}
+
+	pin_desc = devm_kzalloc(&pdev->dev, pinctrl_info->npins *
+				sizeof(struct pinctrl_pin_desc),
+				GFP_KERNEL);
+	if (!pin_desc)
+		return -ENOMEM;
+
+	for (i = 0; i < pinctrl_info->npins; i++) {
+		pin_desc[i].number = pinctrl_info->pins[i].number;
+		pin_desc[i].name = pinctrl_info->pins[i].name;
+		pin_desc[i].drv_data = pinctrl_info;
+	}
+
+	sprd_pinctrl_desc.pins = pin_desc;
+	sprd_pinctrl_desc.name = dev_name(&pdev->dev);
+	sprd_pinctrl_desc.npins = pinctrl_info->npins;
+
+	sprd_pctl->pctl = pinctrl_register(&sprd_pinctrl_desc,
+					   &pdev->dev, (void *)sprd_pctl);
+	if (IS_ERR(sprd_pctl->pctl)) {
+		dev_err(&pdev->dev, "could not register pinctrl driver\n");
+		return PTR_ERR(sprd_pctl->pctl);
+	}
+
+	ret = sprd_pinctrl_parse_dt(sprd_pctl);
+	if (ret) {
+		dev_err(&pdev->dev, "fail to parse dt properties\n");
+		pinctrl_unregister(sprd_pctl->pctl);
+		return ret;
+	}
+
+	return 0;
+}
+
+int sprd_pinctrl_remove(struct platform_device *pdev)
+{
+	struct sprd_pinctrl *sprd_pctl = platform_get_drvdata(pdev);
+
+	pinctrl_unregister(sprd_pctl->pctl);
+	return 0;
+}
+
+void sprd_pinctrl_shutdown(struct platform_device *pdev)
+{
+	struct pinctrl *pinctl;
+	struct pinctrl_state *state;
+
+	pinctl = devm_pinctrl_get(&pdev->dev);
+	if (IS_ERR(pinctl))
+		return;
+	state = pinctrl_lookup_state(pinctl, "shutdown");
+	if (IS_ERR(state))
+		return;
+	pinctrl_select_state(pinctl, state);
+}
+
+MODULE_DESCRIPTION("SPREADTRUM Pin Controller Driver");
+MODULE_AUTHOR("Baolin Wang <baolin.wang@spreadtrum.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/pinctrl/sprd/pinctrl-sprd.h b/drivers/pinctrl/sprd/pinctrl-sprd.h
new file mode 100644
index 000000000000..31a43fec38c4
--- /dev/null
+++ b/drivers/pinctrl/sprd/pinctrl-sprd.h
@@ -0,0 +1,67 @@
+/*
+ * Driver header file for pin controller driver
+ * Copyright (C) 2017 Spreadtrum  - http://www.spreadtrum.com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+
+#ifndef __PINCTRL_SPRD_H__
+#define __PINCTRL_SPRD_H__
+
+struct platform_device;
+
+#define NUM_OFFSET	(20)
+#define TYPE_OFFSET	(16)
+#define BIT_OFFSET	(8)
+#define WIDTH_OFFSET	(4)
+
+#define SPRD_PIN_INFO(num, type, offset, width, reg)	\
+		(((num) & 0xFFF) << NUM_OFFSET |	\
+		 ((type) & 0xF) << TYPE_OFFSET |	\
+		 ((offset) & 0xFF) << BIT_OFFSET |	\
+		 ((width) & 0xF) << WIDTH_OFFSET |	\
+		 ((reg) & 0xF))
+
+#define SPRD_PINCTRL_PIN(pin)	SPRD_PINCTRL_PIN_DATA(pin, #pin)
+
+#define SPRD_PINCTRL_PIN_DATA(a, b)				\
+	{							\
+		.name = b,					\
+		.num = (((a) >> NUM_OFFSET) & 0xfff),		\
+		.type = (((a) >> TYPE_OFFSET) & 0xf),		\
+		.bit_offset = (((a) >> BIT_OFFSET) & 0xff),	\
+		.bit_width = ((a) >> WIDTH_OFFSET & 0xf),	\
+		.reg = ((a) & 0xf)				\
+	}
+
+enum pin_type {
+	GLOBAL_CTRL_PIN,
+	COMMON_PIN,
+	MISC_PIN,
+};
+
+struct sprd_pins_info {
+	const char *name;
+	unsigned int num;
+	enum pin_type type;
+
+	/* for global control pins configuration */
+	unsigned long bit_offset;
+	unsigned long bit_width;
+	unsigned int reg;
+};
+
+int sprd_pinctrl_core_probe(struct platform_device *pdev,
+			    struct sprd_pins_info *sprd_soc_pin_info,
+			    int pins_cnt);
+int sprd_pinctrl_remove(struct platform_device *pdev);
+void sprd_pinctrl_shutdown(struct platform_device *pdev);
+
+#endif /* __PINCTRL_SPRD_H__ */
diff --git a/drivers/pinctrl/stm32/pinctrl-stm32.c b/drivers/pinctrl/stm32/pinctrl-stm32.c
index 06431ff49ffb..50299ad96659 100644
--- a/drivers/pinctrl/stm32/pinctrl-stm32.c
+++ b/drivers/pinctrl/stm32/pinctrl-stm32.c
@@ -952,7 +952,7 @@ static int stm32_gpiolib_register_bank(struct stm32_pinctrl *pctl,
 	int npins = STM32_GPIO_PINS_PER_BANK;
 	int bank_nr, err;
 
-	rstc = of_reset_control_get(np, NULL);
+	rstc = of_reset_control_get_exclusive(np, NULL);
 	if (!IS_ERR(rstc))
 		reset_control_deassert(rstc);
 
diff --git a/drivers/pinctrl/sunxi/Kconfig b/drivers/pinctrl/sunxi/Kconfig
index 31f85ca92669..bfce99d86dfc 100644
--- a/drivers/pinctrl/sunxi/Kconfig
+++ b/drivers/pinctrl/sunxi/Kconfig
@@ -7,7 +7,7 @@ config PINCTRL_SUNXI
 	select GPIOLIB
 
 config PINCTRL_SUN4I_A10
-	def_bool MACH_SUN4I || MACH_SUN7I
+	def_bool MACH_SUN4I || MACH_SUN7I || MACH_SUN8I
 	select PINCTRL_SUNXI
 
 config PINCTRL_SUN5I
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun4i-a10.c b/drivers/pinctrl/sunxi/pinctrl-sun4i-a10.c
index 47a392bc73c8..f763d8d62d6e 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun4i-a10.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun4i-a10.c
@@ -26,7 +26,8 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION(0x3, "spi1"),		/* CS0 */
 		  SUNXI_FUNCTION(0x4, "uart2"),		/* RTS */
 		  SUNXI_FUNCTION_VARIANT(0x5, "gmac",	/* GRXD3 */
-					 PINCTRL_SUN7I_A20)),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(A, 1),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
@@ -34,7 +35,8 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION(0x3, "spi1"),		/* CLK */
 		  SUNXI_FUNCTION(0x4, "uart2"),		/* CTS */
 		  SUNXI_FUNCTION_VARIANT(0x5, "gmac",	/* GRXD2 */
-					 PINCTRL_SUN7I_A20)),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(A, 2),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
@@ -42,7 +44,8 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION(0x3, "spi1"),		/* MOSI */
 		  SUNXI_FUNCTION(0x4, "uart2"),		/* TX */
 		  SUNXI_FUNCTION_VARIANT(0x5, "gmac",	/* GRXD1 */
-					 PINCTRL_SUN7I_A20)),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(A, 3),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
@@ -50,65 +53,75 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION(0x3, "spi1"),		/* MISO */
 		  SUNXI_FUNCTION(0x4, "uart2"),		/* RX */
 		  SUNXI_FUNCTION_VARIANT(0x5, "gmac",	/* GRXD0 */
-					 PINCTRL_SUN7I_A20)),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(A, 4),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
 		  SUNXI_FUNCTION(0x2, "emac"),		/* ETXD3 */
 		  SUNXI_FUNCTION(0x3, "spi1"),		/* CS1 */
 		  SUNXI_FUNCTION_VARIANT(0x5, "gmac",	/* GTXD3 */
-					 PINCTRL_SUN7I_A20)),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(A, 5),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
 		  SUNXI_FUNCTION(0x2, "emac"),		/* ETXD2 */
 		  SUNXI_FUNCTION(0x3, "spi3"),		/* CS0 */
 		  SUNXI_FUNCTION_VARIANT(0x5, "gmac",	/* GTXD2 */
-					 PINCTRL_SUN7I_A20)),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(A, 6),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
 		  SUNXI_FUNCTION(0x2, "emac"),		/* ETXD1 */
 		  SUNXI_FUNCTION(0x3, "spi3"),		/* CLK */
 		  SUNXI_FUNCTION_VARIANT(0x5, "gmac",	/* GTXD1 */
-					 PINCTRL_SUN7I_A20)),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(A, 7),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
 		  SUNXI_FUNCTION(0x2, "emac"),		/* ETXD0 */
 		  SUNXI_FUNCTION(0x3, "spi3"),		/* MOSI */
 		  SUNXI_FUNCTION_VARIANT(0x5, "gmac",	/* GTXD0 */
-					 PINCTRL_SUN7I_A20)),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(A, 8),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
 		  SUNXI_FUNCTION(0x2, "emac"),		/* ERXCK */
 		  SUNXI_FUNCTION(0x3, "spi3"),		/* MISO */
 		  SUNXI_FUNCTION_VARIANT(0x5, "gmac",	/* GRXCK */
-					 PINCTRL_SUN7I_A20)),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(A, 9),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
 		  SUNXI_FUNCTION(0x2, "emac"),		/* ERXERR */
 		  SUNXI_FUNCTION(0x3, "spi3"),		/* CS1 */
 		  SUNXI_FUNCTION_VARIANT(0x5, "gmac",	/* GNULL / ERXERR */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION_VARIANT(0x6, "i2s1",	/* MCLK */
-					 PINCTRL_SUN7I_A20)),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(A, 10),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
 		  SUNXI_FUNCTION(0x2, "emac"),		/* ERXDV */
 		  SUNXI_FUNCTION(0x4, "uart1"),		/* TX */
 		  SUNXI_FUNCTION_VARIANT(0x5, "gmac",	/* GRXDV */
-					 PINCTRL_SUN7I_A20)),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(A, 11),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
 		  SUNXI_FUNCTION(0x2, "emac"),		/* EMDC */
 		  SUNXI_FUNCTION(0x4, "uart1"),		/* RX */
 		  SUNXI_FUNCTION_VARIANT(0x5, "gmac",	/* EMDC */
-					 PINCTRL_SUN7I_A20)),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(A, 12),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
@@ -116,7 +129,8 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION(0x3, "uart6"),		/* TX */
 		  SUNXI_FUNCTION(0x4, "uart1"),		/* RTS */
 		  SUNXI_FUNCTION_VARIANT(0x5, "gmac",	/* EMDIO */
-					 PINCTRL_SUN7I_A20)),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(A, 13),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
@@ -124,7 +138,8 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION(0x3, "uart6"),		/* RX */
 		  SUNXI_FUNCTION(0x4, "uart1"),		/* CTS */
 		  SUNXI_FUNCTION_VARIANT(0x5, "gmac",	/* GTXCTL / ETXEN */
-					 PINCTRL_SUN7I_A20)),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(A, 14),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
@@ -132,9 +147,11 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION(0x3, "uart7"),		/* TX */
 		  SUNXI_FUNCTION(0x4, "uart1"),		/* DTR */
 		  SUNXI_FUNCTION_VARIANT(0x5, "gmac",	/* GNULL / ETXCK */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION_VARIANT(0x6, "i2s1",	/* BCLK */
-					 PINCTRL_SUN7I_A20)),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(A, 15),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
@@ -142,9 +159,11 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION(0x3, "uart7"),		/* RX */
 		  SUNXI_FUNCTION(0x4, "uart1"),		/* DSR */
 		  SUNXI_FUNCTION_VARIANT(0x5, "gmac",	/* GTXCK / ECRS */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION_VARIANT(0x6, "i2s1",	/* LRCK */
-					 PINCTRL_SUN7I_A20)),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(A, 16),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
@@ -152,9 +171,11 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION(0x3, "can"),		/* TX */
 		  SUNXI_FUNCTION(0x4, "uart1"),		/* DCD */
 		  SUNXI_FUNCTION_VARIANT(0x5, "gmac",	/* GCLKIN / ECOL */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION_VARIANT(0x6, "i2s1",	/* DO */
-					 PINCTRL_SUN7I_A20)),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(A, 17),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
@@ -162,14 +183,18 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION(0x3, "can"),		/* RX */
 		  SUNXI_FUNCTION(0x4, "uart1"),		/* RING */
 		  SUNXI_FUNCTION_VARIANT(0x5, "gmac",	/* GNULL / ETXERR */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION_VARIANT(0x6, "i2s1",	/* DI */
-					 PINCTRL_SUN7I_A20)),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40)),
 	/* Hole */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 0),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
-		  SUNXI_FUNCTION(0x2, "i2c0")),		/* SCK */
+		  SUNXI_FUNCTION(0x2, "i2c0"),		/* SCK */
+		  SUNXI_FUNCTION_VARIANT(0x3, "pll_lock_dbg",
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 1),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
@@ -177,11 +202,19 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 2),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
-		  SUNXI_FUNCTION(0x2, "pwm")),		/* PWM0 */
+		  SUNXI_FUNCTION_VARIANT(0x2, "pwm",	/* PWM0 */
+					 PINCTRL_SUN4I_A10 |
+					 PINCTRL_SUN7I_A20),
+		  SUNXI_FUNCTION_VARIANT(0x3, "pwm",	/* PWM0 */
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 3),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
-		  SUNXI_FUNCTION(0x2, "ir0"),		/* TX */
+		  SUNXI_FUNCTION_VARIANT(0x2, "ir0",	/* TX */
+					 PINCTRL_SUN4I_A10 |
+					 PINCTRL_SUN7I_A20),
+		  SUNXI_FUNCTION_VARIANT(0x3, "pwm",	/* PWM1 */
+					 PINCTRL_SUN8I_R40),
 		/*
 		 * The SPDIF block is not referenced at all in the A10 user
 		 * manual. However it is described in the code leaked and the
@@ -205,7 +238,8 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION_VARIANT(0x2, "i2s",	/* MCLK */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x2, "i2s0",	/* MCLK */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION(0x3, "ac97")),		/* MCLK */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 6),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
@@ -213,7 +247,8 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION_VARIANT(0x2, "i2s",	/* BCLK */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x2, "i2s0",	/* BCLK */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION(0x3, "ac97")),		/* BCLK */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 7),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
@@ -221,7 +256,8 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION_VARIANT(0x2, "i2s",	/* LRCK */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x2, "i2s0",	/* LRCK */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION(0x3, "ac97")),		/* SYNC */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 8),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
@@ -229,7 +265,8 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION_VARIANT(0x2, "i2s",	/* DO0 */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x2, "i2s0",	/* DO0 */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION(0x3, "ac97")),		/* DO */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 9),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
@@ -237,31 +274,41 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION_VARIANT(0x2, "i2s",	/* DO1 */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x2, "i2s0",	/* DO1 */
-					 PINCTRL_SUN7I_A20)),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
+		  SUNXI_FUNCTION_VARIANT(0x4, "pwm",	/* PWM6 */
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 10),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
 		  SUNXI_FUNCTION_VARIANT(0x2, "i2s",	/* DO2 */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x2, "i2s0",	/* DO2 */
-					 PINCTRL_SUN7I_A20)),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
+		  SUNXI_FUNCTION_VARIANT(0x4, "pwm",	/* PWM7 */
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 11),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
 		  SUNXI_FUNCTION_VARIANT(0x2, "i2s",	/* DO3 */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x2, "i2s0",	/* DO3 */
-					 PINCTRL_SUN7I_A20)),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 12),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
 		  SUNXI_FUNCTION_VARIANT(0x2, "i2s",	/* DI */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x2, "i2s0",	/* DI */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION(0x3, "ac97"),		/* DI */
 		/* Undocumented mux function on A10 - See SPDIF MCLK above */
-		  SUNXI_FUNCTION(0x4, "spdif")),        /* SPDIF IN */
+		  SUNXI_FUNCTION_VARIANT(0x4, "spdif",	/* SPDIF IN */
+					 PINCTRL_SUN4I_A10 |
+					 PINCTRL_SUN7I_A20)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 13),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
@@ -299,16 +346,22 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 20),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
-		  SUNXI_FUNCTION(0x2, "i2c2")),		/* SCK */
+		  SUNXI_FUNCTION(0x2, "i2c2"),		/* SCK */
+		  SUNXI_FUNCTION_VARIANT(0x4, "pwm",	/* PWM4 */
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 21),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
-		  SUNXI_FUNCTION(0x2, "i2c2")),		/* SDA */
+		  SUNXI_FUNCTION(0x2, "i2c2"),		/* SDA */
+		  SUNXI_FUNCTION_VARIANT(0x4, "pwm",	/* PWM5 */
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 22),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
 		  SUNXI_FUNCTION(0x2, "uart0"),		/* TX */
-		  SUNXI_FUNCTION(0x3, "ir1")),		/* TX */
+		  SUNXI_FUNCTION_VARIANT(0x3, "ir1",	/* TX */
+					 PINCTRL_SUN4I_A10 |
+					 PINCTRL_SUN7I_A20)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(B, 23),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
@@ -341,7 +394,9 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(C, 5),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
-		  SUNXI_FUNCTION(0x2, "nand0")),	/* NRE# */
+		  SUNXI_FUNCTION(0x2, "nand0"),		/* NRE# */
+		  SUNXI_FUNCTION_VARIANT(0x3, "mmc2",	/* DS */
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(C, 6),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
@@ -375,19 +430,27 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(C, 12),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
-		  SUNXI_FUNCTION(0x2, "nand0")),	/* NDQ4 */
+		  SUNXI_FUNCTION(0x2, "nand0"),		/* NDQ4 */
+		  SUNXI_FUNCTION_VARIANT(0x3, "mmc2",	/* D4 */
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(C, 13),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
-		  SUNXI_FUNCTION(0x2, "nand0")),	/* NDQ5 */
+		  SUNXI_FUNCTION(0x2, "nand0"),		/* NDQ5 */
+		  SUNXI_FUNCTION_VARIANT(0x3, "mmc2",	/* D5 */
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(C, 14),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
-		  SUNXI_FUNCTION(0x2, "nand0")),	/* NDQ6 */
+		  SUNXI_FUNCTION(0x2, "nand0"),		/* NDQ6 */
+		  SUNXI_FUNCTION_VARIANT(0x3, "mmc2",	/* D6 */
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(C, 15),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
-		  SUNXI_FUNCTION(0x2, "nand0")),	/* NDQ7 */
+		  SUNXI_FUNCTION(0x2, "nand0"),		/* NDQ7 */
+		  SUNXI_FUNCTION_VARIANT(0x3, "mmc2",	/* D7 */
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(C, 16),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
@@ -427,7 +490,9 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(C, 24),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
-		  SUNXI_FUNCTION(0x2, "nand0")),	/* NDQS */
+		  SUNXI_FUNCTION(0x2, "nand0"),		/* NDQS */
+		  SUNXI_FUNCTION_VARIANT(0x3, "mmc2",	/* RST */
+					 PINCTRL_SUN8I_R40)),
 	/* Hole */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(D, 0),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
@@ -728,14 +793,18 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION(0x2, "ts1"),		/* D5 */
 		  SUNXI_FUNCTION(0x3, "csi1"),		/* D5 */
 		  SUNXI_FUNCTION(0x4, "uart3"),		/* CTS */
-		  SUNXI_FUNCTION(0x5, "csi0")),		/* D13 */
+		  SUNXI_FUNCTION(0x5, "csi0"),		/* D13 */
+		  SUNXI_FUNCTION_VARIANT(0x6, "bist",	/* RESULT0 */
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(G, 10),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
 		  SUNXI_FUNCTION(0x2, "ts1"),		/* D6 */
 		  SUNXI_FUNCTION(0x3, "csi1"),		/* D6 */
 		  SUNXI_FUNCTION(0x4, "uart4"),		/* TX */
-		  SUNXI_FUNCTION(0x5, "csi0")),		/* D14 */
+		  SUNXI_FUNCTION(0x5, "csi0"),		/* D14 */
+		  SUNXI_FUNCTION_VARIANT(0x6, "bist",	/* RESULT1 */
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(G, 11),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
@@ -805,7 +874,9 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION_VARIANT(0x3, "pata",	/* ATAD2 */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION(0x4, "uart5"),		/* TX */
-		  SUNXI_FUNCTION(0x5, "ms"),		/* BS */
+		  SUNXI_FUNCTION_VARIANT(0x5, "ms",	/* BS */
+					 PINCTRL_SUN4I_A10 |
+					 PINCTRL_SUN7I_A20),
 		  SUNXI_FUNCTION_IRQ(0x6, 6),		/* EINT6 */
 		  SUNXI_FUNCTION(0x7, "csi1")),		/* D6 */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(H, 7),
@@ -815,7 +886,9 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION_VARIANT(0x3, "pata",	/* ATAD3 */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION(0x4, "uart5"),		/* RX */
-		  SUNXI_FUNCTION(0x5, "ms"),		/* CLK */
+		  SUNXI_FUNCTION_VARIANT(0x5, "ms",	/* CLK */
+					 PINCTRL_SUN4I_A10 |
+					 PINCTRL_SUN7I_A20),
 		  SUNXI_FUNCTION_IRQ(0x6, 7),		/* EINT7 */
 		  SUNXI_FUNCTION(0x7, "csi1")),		/* D7 */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(H, 8),
@@ -825,9 +898,12 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION_VARIANT(0x3, "pata",	/* ATAD4 */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x3, "emac",	/* ERXD3 */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION(0x4, "keypad"),	/* IN0 */
-		  SUNXI_FUNCTION(0x5, "ms"),		/* D0 */
+		  SUNXI_FUNCTION_VARIANT(0x5, "ms",	/* D0 */
+					 PINCTRL_SUN4I_A10 |
+					 PINCTRL_SUN7I_A20),
 		  SUNXI_FUNCTION_IRQ(0x6, 8),		/* EINT8 */
 		  SUNXI_FUNCTION(0x7, "csi1")),		/* D8 */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(H, 9),
@@ -837,9 +913,12 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION_VARIANT(0x3, "pata",	/* ATAD5 */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x3, "emac",	/* ERXD2 */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION(0x4, "keypad"),	/* IN1 */
-		  SUNXI_FUNCTION(0x5, "ms"),		/* D1 */
+		  SUNXI_FUNCTION_VARIANT(0x5, "ms",	/* D1 */
+					 PINCTRL_SUN4I_A10 |
+					 PINCTRL_SUN7I_A20),
 		  SUNXI_FUNCTION_IRQ(0x6, 9),		/* EINT9 */
 		  SUNXI_FUNCTION(0x7, "csi1")),		/* D9 */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(H, 10),
@@ -849,9 +928,12 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION_VARIANT(0x3, "pata",	/* ATAD6 */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x3, "emac",	/* ERXD1 */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION(0x4, "keypad"),	/* IN2 */
-		  SUNXI_FUNCTION(0x5, "ms"),		/* D2 */
+		  SUNXI_FUNCTION_VARIANT(0x5, "ms",	/* D2 */
+					 PINCTRL_SUN4I_A10 |
+					 PINCTRL_SUN7I_A20),
 		  SUNXI_FUNCTION_IRQ(0x6, 10),		/* EINT10 */
 		  SUNXI_FUNCTION(0x7, "csi1")),		/* D10 */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(H, 11),
@@ -861,9 +943,12 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION_VARIANT(0x3, "pata",	/* ATAD7 */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x3, "emac",	/* ERXD0 */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION(0x4, "keypad"),	/* IN3 */
-		  SUNXI_FUNCTION(0x5, "ms"),		/* D3 */
+		  SUNXI_FUNCTION_VARIANT(0x5, "ms",	/* D3 */
+					 PINCTRL_SUN4I_A10 |
+					 PINCTRL_SUN7I_A20),
 		  SUNXI_FUNCTION_IRQ(0x6, 11),		/* EINT11 */
 		  SUNXI_FUNCTION(0x7, "csi1")),		/* D11 */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(H, 12),
@@ -892,7 +977,8 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION_VARIANT(0x3, "pata",	/* ATAD10 */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x3, "emac",	/* ETXD3 */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION(0x4, "keypad"),	/* IN4 */
 		  SUNXI_FUNCTION(0x5, "sim"),		/* VPPEN */
 		  SUNXI_FUNCTION_IRQ(0x6, 14),		/* EINT14 */
@@ -904,7 +990,8 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION_VARIANT(0x3, "pata",	/* ATAD11 */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x3, "emac",	/* ETXD2 */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION(0x4, "keypad"),	/* IN5 */
 		  SUNXI_FUNCTION(0x5, "sim"),		/* VPPPP */
 		  SUNXI_FUNCTION_IRQ(0x6, 15),		/* EINT15 */
@@ -916,7 +1003,8 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION_VARIANT(0x3, "pata",	/* ATAD12 */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x3, "emac",	/* ETXD1 */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION(0x4, "keypad"),	/* IN6 */
 		  SUNXI_FUNCTION(0x5, "sim"),		/* DET */
 		  SUNXI_FUNCTION_IRQ(0x6, 16),		/* EINT16 */
@@ -928,7 +1016,8 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION_VARIANT(0x3, "pata",	/* ATAD13 */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x3, "emac",	/* ETXD0 */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION(0x4, "keypad"),	/* IN7 */
 		  SUNXI_FUNCTION(0x5, "sim"),		/* VCCEN */
 		  SUNXI_FUNCTION_IRQ(0x6, 17),		/* EINT17 */
@@ -940,7 +1029,8 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION_VARIANT(0x3, "pata",	/* ATAD14 */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x3, "emac",	/* ERXCK */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION(0x4, "keypad"),	/* OUT0 */
 		  SUNXI_FUNCTION(0x5, "sim"),		/* SCK */
 		  SUNXI_FUNCTION_IRQ(0x6, 18),		/* EINT18 */
@@ -952,7 +1042,8 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION_VARIANT(0x3, "pata",	/* ATAD15 */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x3, "emac",	/* ERXERR */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION(0x4, "keypad"),	/* OUT1 */
 		  SUNXI_FUNCTION(0x5, "sim"),		/* SDA */
 		  SUNXI_FUNCTION_IRQ(0x6, 19),		/* EINT19 */
@@ -964,7 +1055,8 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION_VARIANT(0x3, "pata",	/* ATAOE */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x3, "emac",	/* ERXDV */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION(0x4, "can"),		/* TX */
 		  SUNXI_FUNCTION_IRQ(0x6, 20),		/* EINT20 */
 		  SUNXI_FUNCTION(0x7, "csi1")),		/* D20 */
@@ -975,7 +1067,8 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION_VARIANT(0x3, "pata",	/* ATADREQ */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x3, "emac",	/* EMDC */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION(0x4, "can"),		/* RX */
 		  SUNXI_FUNCTION_IRQ(0x6, 21),		/* EINT21 */
 		  SUNXI_FUNCTION(0x7, "csi1")),		/* D21 */
@@ -986,7 +1079,8 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION_VARIANT(0x3, "pata",	/* ATADACK */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x3, "emac",	/* EMDIO */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION(0x4, "keypad"),	/* OUT2 */
 		  SUNXI_FUNCTION(0x5, "mmc1"),		/* CMD */
 		  SUNXI_FUNCTION(0x7, "csi1")),		/* D22 */
@@ -997,7 +1091,8 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION_VARIANT(0x3, "pata",	/* ATACS0 */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x3, "emac",	/* ETXEN */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION(0x4, "keypad"),	/* OUT3 */
 		  SUNXI_FUNCTION(0x5, "mmc1"),		/* CLK */
 		  SUNXI_FUNCTION(0x7, "csi1")),		/* D23 */
@@ -1008,7 +1103,8 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION_VARIANT(0x3, "pata",	/* ATACS1 */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x3, "emac",	/* ETXCK */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION(0x4, "keypad"),	/* OUT4 */
 		  SUNXI_FUNCTION(0x5, "mmc1"),		/* D0 */
 		  SUNXI_FUNCTION(0x7, "csi1")),		/* PCLK */
@@ -1019,7 +1115,8 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION_VARIANT(0x3, "pata",	/* ATAIORDY */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x3, "emac",	/* ECRS */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION(0x4, "keypad"),	/* OUT5 */
 		  SUNXI_FUNCTION(0x5, "mmc1"),		/* D1 */
 		  SUNXI_FUNCTION(0x7, "csi1")),		/* FIELD */
@@ -1030,7 +1127,8 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION_VARIANT(0x3, "pata",	/* ATAIOR */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x3, "emac",	/* ECOL */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION(0x4, "keypad"),	/* OUT6 */
 		  SUNXI_FUNCTION(0x5, "mmc1"),		/* D2 */
 		  SUNXI_FUNCTION(0x7, "csi1")),		/* HSYNC */
@@ -1041,7 +1139,8 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION_VARIANT(0x3, "pata",	/* ATAIOW */
 					 PINCTRL_SUN4I_A10),
 		  SUNXI_FUNCTION_VARIANT(0x3, "emac",	/* ETXERR */
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION(0x4, "keypad"),	/* OUT7 */
 		  SUNXI_FUNCTION(0x5, "mmc1"),		/* D3 */
 		  SUNXI_FUNCTION(0x7, "csi1")),		/* VSYNC */
@@ -1050,23 +1149,27 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
 		  SUNXI_FUNCTION_VARIANT(0x3, "i2c3",	/* SCK */
-					 PINCTRL_SUN7I_A20)),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(I, 1),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
 		  SUNXI_FUNCTION_VARIANT(0x3, "i2c3",	/* SDA */
-					 PINCTRL_SUN7I_A20)),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(I, 2),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
 		  SUNXI_FUNCTION_VARIANT(0x3, "i2c4",	/* SCK */
-					 PINCTRL_SUN7I_A20)),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(I, 3),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
 		  SUNXI_FUNCTION(0x2, "pwm"),		/* PWM1 */
 		  SUNXI_FUNCTION_VARIANT(0x3, "i2c3",	/* SDA */
-					 PINCTRL_SUN7I_A20)),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(I, 4),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
@@ -1109,7 +1212,8 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION(0x2, "spi0"),		/* MOSI */
 		  SUNXI_FUNCTION(0x3, "uart6"),		/* TX */
 		  SUNXI_FUNCTION_VARIANT(0x4, "clk_out_a",
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION_IRQ(0x6, 24)),		/* EINT24 */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(I, 13),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
@@ -1117,7 +1221,8 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION(0x2, "spi0"),		/* MISO */
 		  SUNXI_FUNCTION(0x3, "uart6"),		/* RX */
 		  SUNXI_FUNCTION_VARIANT(0x4, "clk_out_b",
-					 PINCTRL_SUN7I_A20),
+					 PINCTRL_SUN7I_A20 |
+					 PINCTRL_SUN8I_R40),
 		  SUNXI_FUNCTION_IRQ(0x6, 25)),		/* EINT25 */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(I, 14),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
@@ -1162,13 +1267,21 @@ static const struct sunxi_desc_pin sun4i_a10_pins[] = {
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
 		  SUNXI_FUNCTION(0x2, "ps2"),		/* SCK0 */
 		  SUNXI_FUNCTION(0x3, "uart7"),		/* TX */
-		  SUNXI_FUNCTION(0x4, "hdmi")),		/* HSCL */
+		  SUNXI_FUNCTION_VARIANT(0x4, "hdmi",	/* HSCL */
+					 PINCTRL_SUN4I_A10 |
+					 PINCTRL_SUN7I_A20),
+		  SUNXI_FUNCTION_VARIANT(0x6, "pwm",	/* PWM2 */
+					 PINCTRL_SUN8I_R40)),
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(I, 21),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
 		  SUNXI_FUNCTION(0x2, "ps2"),		/* SDA0 */
 		  SUNXI_FUNCTION(0x3, "uart7"),		/* RX */
-		  SUNXI_FUNCTION(0x4, "hdmi")),		/* HSDA */
+		  SUNXI_FUNCTION_VARIANT(0x4, "hdmi",	/* HSDA */
+					 PINCTRL_SUN4I_A10 |
+					 PINCTRL_SUN7I_A20),
+		  SUNXI_FUNCTION_VARIANT(0x6, "pwm",	/* PWM3 */
+					 PINCTRL_SUN8I_R40)),
 };
 
 static const struct sunxi_pinctrl_desc sun4i_a10_pinctrl_data = {
@@ -1195,6 +1308,10 @@ static const struct of_device_id sun4i_a10_pinctrl_match[] = {
 		.compatible = "allwinner,sun7i-a20-pinctrl",
 		.data = (void *)PINCTRL_SUN7I_A20
 	},
+	{
+		.compatible = "allwinner,sun8i-r40-pinctrl",
+		.data = (void *)PINCTRL_SUN8I_R40
+	},
 	{}
 };
 
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun50i-h5.c b/drivers/pinctrl/sunxi/pinctrl-sun50i-h5.c
index ccf9419e9418..97b48336f84a 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun50i-h5.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun50i-h5.c
@@ -19,6 +19,7 @@
 #include <linux/platform_device.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
+#include <linux/of_irq.h>
 #include <linux/pinctrl/pinctrl.h>
 
 #include "pinctrl-sunxi.h"
@@ -530,17 +531,36 @@ static const struct sunxi_desc_pin sun50i_h5_pins[] = {
 		  SUNXI_FUNCTION_IRQ_BANK(0x6, 2, 13)),	/* PG_EINT13 */
 };
 
-static const struct sunxi_pinctrl_desc sun50i_h5_pinctrl_data = {
+static const struct sunxi_pinctrl_desc sun50i_h5_pinctrl_data_broken = {
 	.pins = sun50i_h5_pins,
 	.npins = ARRAY_SIZE(sun50i_h5_pins),
 	.irq_banks = 2,
 	.irq_read_needs_mux = true
 };
 
+static const struct sunxi_pinctrl_desc sun50i_h5_pinctrl_data = {
+	.pins = sun50i_h5_pins,
+	.npins = ARRAY_SIZE(sun50i_h5_pins),
+	.irq_banks = 3,
+	.irq_read_needs_mux = true
+};
+
 static int sun50i_h5_pinctrl_probe(struct platform_device *pdev)
 {
-	return sunxi_pinctrl_init(pdev,
-				  &sun50i_h5_pinctrl_data);
+	switch (of_irq_count(pdev->dev.of_node)) {
+	case 2:
+		dev_warn(&pdev->dev,
+			 "Your device tree's pinctrl node is broken, which has no IRQ of PG bank routed.\n");
+		dev_warn(&pdev->dev,
+			 "Please update the device tree, otherwise PG bank IRQ won't work.\n");
+		return sunxi_pinctrl_init(pdev,
+					  &sun50i_h5_pinctrl_data_broken);
+	case 3:
+		return sunxi_pinctrl_init(pdev,
+					  &sun50i_h5_pinctrl_data);
+	default:
+		return -EINVAL;
+	}
 }
 
 static const struct of_device_id sun50i_h5_pinctrl_match[] = {
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun6i-a31-r.c b/drivers/pinctrl/sunxi/pinctrl-sun6i-a31-r.c
index a22bd88a1f03..49a1deb97bb7 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun6i-a31-r.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun6i-a31-r.c
@@ -25,12 +25,12 @@ static const struct sunxi_desc_pin sun6i_a31_r_pins[] = {
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(L, 0),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
-		  SUNXI_FUNCTION(0x2, "s_twi"),		/* SCK */
+		  SUNXI_FUNCTION(0x2, "s_i2c"),		/* SCK */
 		  SUNXI_FUNCTION(0x3, "s_p2wi")),	/* SCK */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(L, 1),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
-		  SUNXI_FUNCTION(0x2, "s_twi"),		/* SDA */
+		  SUNXI_FUNCTION(0x2, "s_i2c"),		/* SDA */
 		  SUNXI_FUNCTION(0x3, "s_p2wi")),	/* SDA */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(L, 2),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
@@ -113,7 +113,7 @@ static int sun6i_a31_r_pinctrl_probe(struct platform_device *pdev)
 	struct reset_control *rstc;
 	int ret;
 
-	rstc = devm_reset_control_get(&pdev->dev, NULL);
+	rstc = devm_reset_control_get_exclusive(&pdev->dev, NULL);
 	if (IS_ERR(rstc)) {
 		dev_err(&pdev->dev, "Reset controller missing\n");
 		return PTR_ERR(rstc);
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun8i-a23-r.c b/drivers/pinctrl/sunxi/pinctrl-sun8i-a23-r.c
index 2292e05a397b..67ee6f9b3b68 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun8i-a23-r.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun8i-a23-r.c
@@ -29,13 +29,13 @@ static const struct sunxi_desc_pin sun8i_a23_r_pins[] = {
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
 		  SUNXI_FUNCTION(0x2, "s_rsb"),		/* SCK */
-		  SUNXI_FUNCTION(0x3, "s_twi"),		/* SCK */
+		  SUNXI_FUNCTION(0x3, "s_i2c"),		/* SCK */
 		  SUNXI_FUNCTION_IRQ_BANK(0x4, 0, 0)),	/* PL_EINT0 */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(L, 1),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
 		  SUNXI_FUNCTION(0x2, "s_rsb"),		/* SDA */
-		  SUNXI_FUNCTION(0x3, "s_twi"),		/* SDA */
+		  SUNXI_FUNCTION(0x3, "s_i2c"),		/* SDA */
 		  SUNXI_FUNCTION_IRQ_BANK(0x4, 0, 1)),	/* PL_EINT1 */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(L, 2),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
@@ -100,7 +100,7 @@ static int sun8i_a23_r_pinctrl_probe(struct platform_device *pdev)
 	struct reset_control *rstc;
 	int ret;
 
-	rstc = devm_reset_control_get(&pdev->dev, NULL);
+	rstc = devm_reset_control_get_exclusive(&pdev->dev, NULL);
 	if (IS_ERR(rstc)) {
 		dev_err(&pdev->dev, "Reset controller missing\n");
 		return PTR_ERR(rstc);
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun8i-h3-r.c b/drivers/pinctrl/sunxi/pinctrl-sun8i-h3-r.c
index 686ec212120b..ebfd9a26628c 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun8i-h3-r.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun8i-h3-r.c
@@ -20,12 +20,12 @@ static const struct sunxi_desc_pin sun8i_h3_r_pins[] = {
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(L, 0),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
-		  SUNXI_FUNCTION(0x2, "s_twi"),         /* SCK */
+		  SUNXI_FUNCTION(0x2, "s_i2c"),         /* SCK */
 		  SUNXI_FUNCTION_IRQ_BANK(0x6, 0, 0)),	/* PL_EINT0 */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(L, 1),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
 		  SUNXI_FUNCTION(0x1, "gpio_out"),
-		  SUNXI_FUNCTION(0x2, "s_twi"),         /* SDA */
+		  SUNXI_FUNCTION(0x2, "s_i2c"),         /* SDA */
 		  SUNXI_FUNCTION_IRQ_BANK(0x6, 0, 1)),	/* PL_EINT1 */
 	SUNXI_PIN(SUNXI_PINCTRL_PIN(L, 2),
 		  SUNXI_FUNCTION(0x0, "gpio_in"),
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun8i-v3s.c b/drivers/pinctrl/sunxi/pinctrl-sun8i-v3s.c
index c86d3c42a905..496ba34e1f5f 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun8i-v3s.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun8i-v3s.c
@@ -297,6 +297,7 @@ static const struct sunxi_pinctrl_desc sun8i_v3s_pinctrl_data = {
 	.pins = sun8i_v3s_pins,
 	.npins = ARRAY_SIZE(sun8i_v3s_pins),
 	.irq_banks = 2,
+	.irq_bank_base = 1,
 	.irq_read_needs_mux = true
 };
 
diff --git a/drivers/pinctrl/sunxi/pinctrl-sunxi.c b/drivers/pinctrl/sunxi/pinctrl-sunxi.c
index 0dfd7fa66c48..52edf3b5988d 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sunxi.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sunxi.c
@@ -564,7 +564,8 @@ static int sunxi_pconf_group_set(struct pinctrl_dev *pctldev,
 			val = arg / 10 - 1;
 			break;
 		case PIN_CONFIG_BIAS_DISABLE:
-			continue;
+			val = 0;
+			break;
 		case PIN_CONFIG_BIAS_PULL_UP:
 			if (arg == 0)
 				return -EINVAL;
diff --git a/drivers/pinctrl/tegra/pinctrl-tegra-xusb.c b/drivers/pinctrl/tegra/pinctrl-tegra-xusb.c
index ebedc2d32411..9d653c24219c 100644
--- a/drivers/pinctrl/tegra/pinctrl-tegra-xusb.c
+++ b/drivers/pinctrl/tegra/pinctrl-tegra-xusb.c
@@ -901,7 +901,7 @@ int tegra_xusb_padctl_legacy_probe(struct platform_device *pdev)
 	if (IS_ERR(padctl->regs))
 		return PTR_ERR(padctl->regs);
 
-	padctl->rst = devm_reset_control_get(&pdev->dev, NULL);
+	padctl->rst = devm_reset_control_get_exclusive(&pdev->dev, NULL);
 	if (IS_ERR(padctl->rst))
 		return PTR_ERR(padctl->rst);
 
diff --git a/drivers/pinctrl/ti/pinctrl-ti-iodelay.c b/drivers/pinctrl/ti/pinctrl-ti-iodelay.c
index 362c50918c13..5c1b6325d80d 100644
--- a/drivers/pinctrl/ti/pinctrl-ti-iodelay.c
+++ b/drivers/pinctrl/ti/pinctrl-ti-iodelay.c
@@ -716,7 +716,7 @@ static void ti_iodelay_pinconf_group_dbg_show(struct pinctrl_dev *pctldev,
 }
 #endif
 
-static struct pinctrl_ops ti_iodelay_pinctrl_ops = {
+static const struct pinctrl_ops ti_iodelay_pinctrl_ops = {
 	.get_groups_count = pinctrl_generic_get_group_count,
 	.get_group_name = pinctrl_generic_get_group_name,
 	.get_group_pins = pinctrl_generic_get_group_pins,
@@ -726,7 +726,7 @@ static struct pinctrl_ops ti_iodelay_pinctrl_ops = {
 	.dt_node_to_map = ti_iodelay_dt_node_to_map,
 };
 
-static struct pinconf_ops ti_iodelay_pinctrl_pinconf_ops = {
+static const struct pinconf_ops ti_iodelay_pinctrl_pinconf_ops = {
 	.pin_config_group_get = ti_iodelay_pinconf_group_get,
 	.pin_config_group_set = ti_iodelay_pinconf_group_set,
 #ifdef CONFIG_DEBUG_FS
diff --git a/drivers/pinctrl/uniphier/Kconfig b/drivers/pinctrl/uniphier/Kconfig
index e5826eaa7170..9f2a1c666def 100644
--- a/drivers/pinctrl/uniphier/Kconfig
+++ b/drivers/pinctrl/uniphier/Kconfig
@@ -40,4 +40,8 @@ config PINCTRL_UNIPHIER_LD20
 	bool "UniPhier LD20 SoC pinctrl driver"
 	default ARM64
 
+config PINCTRL_UNIPHIER_PXS3
+	bool "UniPhier PXs3 SoC pinctrl driver"
+	default ARM64
+
 endif
diff --git a/drivers/pinctrl/uniphier/Makefile b/drivers/pinctrl/uniphier/Makefile
index 9f4bc8aa6f68..d592ff77d60f 100644
--- a/drivers/pinctrl/uniphier/Makefile
+++ b/drivers/pinctrl/uniphier/Makefile
@@ -8,3 +8,4 @@ obj-$(CONFIG_PINCTRL_UNIPHIER_PXS2)	+= pinctrl-uniphier-pxs2.o
 obj-$(CONFIG_PINCTRL_UNIPHIER_LD6B)	+= pinctrl-uniphier-ld6b.o
 obj-$(CONFIG_PINCTRL_UNIPHIER_LD11)	+= pinctrl-uniphier-ld11.o
 obj-$(CONFIG_PINCTRL_UNIPHIER_LD20)	+= pinctrl-uniphier-ld20.o
+obj-$(CONFIG_PINCTRL_UNIPHIER_PXS3)	+= pinctrl-uniphier-pxs3.o
diff --git a/drivers/pinctrl/uniphier/pinctrl-uniphier-core.c b/drivers/pinctrl/uniphier/pinctrl-uniphier-core.c
index 30dec0ee7f35..f9267fabe6b0 100644
--- a/drivers/pinctrl/uniphier/pinctrl-uniphier-core.c
+++ b/drivers/pinctrl/uniphier/pinctrl-uniphier-core.c
@@ -13,7 +13,7 @@
  * GNU General Public License for more details.
  */
 
-#include <linux/export.h>
+#include <linux/list.h>
 #include <linux/mfd/syscon.h>
 #include <linux/of.h>
 #include <linux/pinctrl/pinconf.h>
@@ -33,13 +33,21 @@
 #define UNIPHIER_PINCTRL_DRV2CTRL_BASE	0x1900
 #define UNIPHIER_PINCTRL_DRV3CTRL_BASE	0x1980
 #define UNIPHIER_PINCTRL_PUPDCTRL_BASE	0x1a00
-#define UNIPHIER_PINCTRL_IECTRL		0x1d00
+#define UNIPHIER_PINCTRL_IECTRL_BASE	0x1d00
+
+struct uniphier_pinctrl_reg_region {
+	struct list_head node;
+	unsigned int base;
+	unsigned int nregs;
+	u32 vals[0];
+};
 
 struct uniphier_pinctrl_priv {
 	struct pinctrl_desc pctldesc;
 	struct pinctrl_dev *pctldev;
 	struct regmap *regmap;
 	struct uniphier_pinctrl_socdata *socdata;
+	struct list_head reg_regions;
 };
 
 static int uniphier_pctl_get_groups_count(struct pinctrl_dev *pctldev)
@@ -139,10 +147,11 @@ static const struct pinctrl_ops uniphier_pctlops = {
 };
 
 static int uniphier_conf_pin_bias_get(struct pinctrl_dev *pctldev,
-				      const struct pin_desc *desc,
+				      unsigned int pin,
 				      enum pin_config_param param)
 {
 	struct uniphier_pinctrl_priv *priv = pinctrl_dev_get_drvdata(pctldev);
+	const struct pin_desc *desc = pin_desc_get(pctldev, pin);
 	enum uniphier_pin_pull_dir pull_dir =
 				uniphier_pin_get_pull_dir(desc->drv_data);
 	unsigned int pupdctrl, reg, shift, val;
@@ -189,10 +198,10 @@ static int uniphier_conf_pin_bias_get(struct pinctrl_dev *pctldev,
 }
 
 static int uniphier_conf_pin_drive_get(struct pinctrl_dev *pctldev,
-				       const struct pin_desc *desc,
-				       u16 *strength)
+				       unsigned int pin, u32 *strength)
 {
 	struct uniphier_pinctrl_priv *priv = pinctrl_dev_get_drvdata(pctldev);
+	const struct pin_desc *desc = pin_desc_get(pctldev, pin);
 	enum uniphier_pin_drv_type type =
 				uniphier_pin_get_drv_type(desc->drv_data);
 	const unsigned int strength_1bit[] = {4, 8};
@@ -249,46 +258,52 @@ static int uniphier_conf_pin_drive_get(struct pinctrl_dev *pctldev,
 }
 
 static int uniphier_conf_pin_input_enable_get(struct pinctrl_dev *pctldev,
-					      const struct pin_desc *desc)
+					      unsigned int pin)
 {
 	struct uniphier_pinctrl_priv *priv = pinctrl_dev_get_drvdata(pctldev);
+	const struct pin_desc *desc = pin_desc_get(pctldev, pin);
 	unsigned int iectrl = uniphier_pin_get_iectrl(desc->drv_data);
-	unsigned int val;
+	unsigned int reg, mask, val;
 	int ret;
 
 	if (iectrl == UNIPHIER_PIN_IECTRL_NONE)
 		/* This pin is always input-enabled. */
 		return 0;
 
-	ret = regmap_read(priv->regmap, UNIPHIER_PINCTRL_IECTRL, &val);
+	if (priv->socdata->caps & UNIPHIER_PINCTRL_CAPS_PERPIN_IECTRL)
+		iectrl = pin;
+
+	reg = UNIPHIER_PINCTRL_IECTRL_BASE + iectrl / 32 * 4;
+	mask = BIT(iectrl % 32);
+
+	ret = regmap_read(priv->regmap, reg, &val);
 	if (ret)
 		return ret;
 
-	return val & BIT(iectrl) ? 0 : -EINVAL;
+	return val & mask ? 0 : -EINVAL;
 }
 
 static int uniphier_conf_pin_config_get(struct pinctrl_dev *pctldev,
 					unsigned pin,
 					unsigned long *configs)
 {
-	const struct pin_desc *desc = pin_desc_get(pctldev, pin);
 	enum pin_config_param param = pinconf_to_config_param(*configs);
 	bool has_arg = false;
-	u16 arg;
+	u32 arg;
 	int ret;
 
 	switch (param) {
 	case PIN_CONFIG_BIAS_DISABLE:
 	case PIN_CONFIG_BIAS_PULL_UP:
 	case PIN_CONFIG_BIAS_PULL_DOWN:
-		ret = uniphier_conf_pin_bias_get(pctldev, desc, param);
+		ret = uniphier_conf_pin_bias_get(pctldev, pin, param);
 		break;
 	case PIN_CONFIG_DRIVE_STRENGTH:
-		ret = uniphier_conf_pin_drive_get(pctldev, desc, &arg);
+		ret = uniphier_conf_pin_drive_get(pctldev, pin, &arg);
 		has_arg = true;
 		break;
 	case PIN_CONFIG_INPUT_ENABLE:
-		ret = uniphier_conf_pin_input_enable_get(pctldev, desc);
+		ret = uniphier_conf_pin_input_enable_get(pctldev, pin);
 		break;
 	default:
 		/* unsupported parameter */
@@ -303,10 +318,11 @@ static int uniphier_conf_pin_config_get(struct pinctrl_dev *pctldev,
 }
 
 static int uniphier_conf_pin_bias_set(struct pinctrl_dev *pctldev,
-				      const struct pin_desc *desc,
+				      unsigned int pin,
 				      enum pin_config_param param, u32 arg)
 {
 	struct uniphier_pinctrl_priv *priv = pinctrl_dev_get_drvdata(pctldev);
+	const struct pin_desc *desc = pin_desc_get(pctldev, pin);
 	enum uniphier_pin_pull_dir pull_dir =
 				uniphier_pin_get_pull_dir(desc->drv_data);
 	unsigned int pupdctrl, reg, shift;
@@ -377,10 +393,10 @@ static int uniphier_conf_pin_bias_set(struct pinctrl_dev *pctldev,
 }
 
 static int uniphier_conf_pin_drive_set(struct pinctrl_dev *pctldev,
-				       const struct pin_desc *desc,
-				       u16 strength)
+				       unsigned int pin, u32 strength)
 {
 	struct uniphier_pinctrl_priv *priv = pinctrl_dev_get_drvdata(pctldev);
+	const struct pin_desc *desc = pin_desc_get(pctldev, pin);
 	enum uniphier_pin_drv_type type =
 				uniphier_pin_get_drv_type(desc->drv_data);
 	const unsigned int strength_1bit[] = {4, 8, -1};
@@ -438,10 +454,10 @@ static int uniphier_conf_pin_drive_set(struct pinctrl_dev *pctldev,
 }
 
 static int uniphier_conf_pin_input_enable(struct pinctrl_dev *pctldev,
-					  const struct pin_desc *desc,
-					  u16 enable)
+					  unsigned int pin, u32 enable)
 {
 	struct uniphier_pinctrl_priv *priv = pinctrl_dev_get_drvdata(pctldev);
+	const struct pin_desc *desc = pin_desc_get(pctldev, pin);
 	unsigned int iectrl = uniphier_pin_get_iectrl(desc->drv_data);
 	unsigned int reg, mask;
 
@@ -457,7 +473,10 @@ static int uniphier_conf_pin_input_enable(struct pinctrl_dev *pctldev,
 	if (iectrl == UNIPHIER_PIN_IECTRL_NONE)
 		return enable ? 0 : -EINVAL;
 
-	reg = UNIPHIER_PINCTRL_IECTRL + iectrl / 32 * 4;
+	if (priv->socdata->caps & UNIPHIER_PINCTRL_CAPS_PERPIN_IECTRL)
+		iectrl = pin;
+
+	reg = UNIPHIER_PINCTRL_IECTRL_BASE + iectrl / 32 * 4;
 	mask = BIT(iectrl % 32);
 
 	return regmap_update_bits(priv->regmap, reg, mask, enable ? mask : 0);
@@ -468,7 +487,6 @@ static int uniphier_conf_pin_config_set(struct pinctrl_dev *pctldev,
 					unsigned long *configs,
 					unsigned num_configs)
 {
-	const struct pin_desc *desc = pin_desc_get(pctldev, pin);
 	int i, ret;
 
 	for (i = 0; i < num_configs; i++) {
@@ -481,15 +499,14 @@ static int uniphier_conf_pin_config_set(struct pinctrl_dev *pctldev,
 		case PIN_CONFIG_BIAS_PULL_UP:
 		case PIN_CONFIG_BIAS_PULL_DOWN:
 		case PIN_CONFIG_BIAS_PULL_PIN_DEFAULT:
-			ret = uniphier_conf_pin_bias_set(pctldev, desc,
+			ret = uniphier_conf_pin_bias_set(pctldev, pin,
 							 param, arg);
 			break;
 		case PIN_CONFIG_DRIVE_STRENGTH:
-			ret = uniphier_conf_pin_drive_set(pctldev, desc, arg);
+			ret = uniphier_conf_pin_drive_set(pctldev, pin, arg);
 			break;
 		case PIN_CONFIG_INPUT_ENABLE:
-			ret = uniphier_conf_pin_input_enable(pctldev, desc,
-							     arg);
+			ret = uniphier_conf_pin_input_enable(pctldev, pin, arg);
 			break;
 		default:
 			dev_err(pctldev->dev,
@@ -569,8 +586,7 @@ static int uniphier_pmx_set_one_mux(struct pinctrl_dev *pctldev, unsigned pin,
 	int ret;
 
 	/* some pins need input-enabling */
-	ret = uniphier_conf_pin_input_enable(pctldev,
-					     pin_desc_get(pctldev, pin), 1);
+	ret = uniphier_conf_pin_input_enable(pctldev, pin, 1);
 	if (ret)
 		return ret;
 
@@ -649,30 +665,27 @@ static int uniphier_pmx_gpio_request_enable(struct pinctrl_dev *pctldev,
 					    unsigned offset)
 {
 	struct uniphier_pinctrl_priv *priv = pinctrl_dev_get_drvdata(pctldev);
-	const struct uniphier_pinctrl_group *groups = priv->socdata->groups;
-	int groups_count = priv->socdata->groups_count;
-	enum uniphier_pinmux_gpio_range_type range_type;
-	int i, j;
-
-	if (strstr(range->name, "irq"))
-		range_type = UNIPHIER_PINMUX_GPIO_RANGE_IRQ;
-	else
-		range_type = UNIPHIER_PINMUX_GPIO_RANGE_PORT;
-
-	for (i = 0; i < groups_count; i++) {
-		if (groups[i].range_type != range_type)
-			continue;
-
-		for (j = 0; j < groups[i].num_pins; j++)
-			if (groups[i].pins[j] == offset)
-				goto found;
+	unsigned int gpio_offset;
+	int muxval, i;
+
+	if (range->pins) {
+		for (i = 0; i < range->npins; i++)
+			if (range->pins[i] == offset)
+				break;
+
+		if (WARN_ON(i == range->npins))
+			return -EINVAL;
+
+		gpio_offset = i;
+	} else {
+		gpio_offset = offset - range->pin_base;
 	}
 
-	dev_err(pctldev->dev, "pin %u does not support GPIO\n", offset);
-	return -EINVAL;
+	gpio_offset += range->id;
+
+	muxval = priv->socdata->get_gpio_muxval(offset, gpio_offset);
 
-found:
-	return uniphier_pmx_set_one_mux(pctldev, offset, groups[i].muxvals[j]);
+	return uniphier_pmx_set_one_mux(pctldev, offset, muxval);
 }
 
 static const struct pinmux_ops uniphier_pmxops = {
@@ -684,12 +697,177 @@ static const struct pinmux_ops uniphier_pmxops = {
 	.strict = true,
 };
 
+#ifdef CONFIG_PM_SLEEP
+static int uniphier_pinctrl_suspend(struct device *dev)
+{
+	struct uniphier_pinctrl_priv *priv = dev_get_drvdata(dev);
+	struct uniphier_pinctrl_reg_region *r;
+	int ret;
+
+	list_for_each_entry(r, &priv->reg_regions, node) {
+		ret = regmap_bulk_read(priv->regmap, r->base, r->vals,
+				       r->nregs);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int uniphier_pinctrl_resume(struct device *dev)
+{
+	struct uniphier_pinctrl_priv *priv = dev_get_drvdata(dev);
+	struct uniphier_pinctrl_reg_region *r;
+	int ret;
+
+	list_for_each_entry(r, &priv->reg_regions, node) {
+		ret = regmap_bulk_write(priv->regmap, r->base, r->vals,
+					r->nregs);
+		if (ret)
+			return ret;
+	}
+
+	if (priv->socdata->caps & UNIPHIER_PINCTRL_CAPS_DBGMUX_SEPARATE) {
+		ret = regmap_write(priv->regmap,
+				   UNIPHIER_PINCTRL_LOAD_PINMUX, 1);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int uniphier_pinctrl_add_reg_region(struct device *dev,
+					   struct uniphier_pinctrl_priv *priv,
+					   unsigned int base,
+					   unsigned int count,
+					   unsigned int width)
+{
+	struct uniphier_pinctrl_reg_region *region;
+	unsigned int nregs;
+
+	if (!count)
+		return 0;
+
+	nregs = DIV_ROUND_UP(count * width, 32);
+
+	region = devm_kzalloc(dev,
+			      sizeof(*region) + sizeof(region->vals[0]) * nregs,
+			      GFP_KERNEL);
+	if (!region)
+		return -ENOMEM;
+
+	region->base = base;
+	region->nregs = nregs;
+
+	list_add_tail(&region->node, &priv->reg_regions);
+
+	return 0;
+}
+#endif
+
+static int uniphier_pinctrl_pm_init(struct device *dev,
+				    struct uniphier_pinctrl_priv *priv)
+{
+#ifdef CONFIG_PM_SLEEP
+	const struct uniphier_pinctrl_socdata *socdata = priv->socdata;
+	unsigned int num_drvctrl = 0;
+	unsigned int num_drv2ctrl = 0;
+	unsigned int num_drv3ctrl = 0;
+	unsigned int num_pupdctrl = 0;
+	unsigned int num_iectrl = 0;
+	unsigned int iectrl, drvctrl, pupdctrl;
+	enum uniphier_pin_drv_type drv_type;
+	enum uniphier_pin_pull_dir pull_dir;
+	int i, ret;
+
+	for (i = 0; i < socdata->npins; i++) {
+		void *drv_data = socdata->pins[i].drv_data;
+
+		drvctrl = uniphier_pin_get_drvctrl(drv_data);
+		drv_type = uniphier_pin_get_drv_type(drv_data);
+		pupdctrl = uniphier_pin_get_pupdctrl(drv_data);
+		pull_dir = uniphier_pin_get_pull_dir(drv_data);
+		iectrl = uniphier_pin_get_iectrl(drv_data);
+
+		switch (drv_type) {
+		case UNIPHIER_PIN_DRV_1BIT:
+			num_drvctrl = max(num_drvctrl, drvctrl + 1);
+			break;
+		case UNIPHIER_PIN_DRV_2BIT:
+			num_drv2ctrl = max(num_drv2ctrl, drvctrl + 1);
+			break;
+		case UNIPHIER_PIN_DRV_3BIT:
+			num_drv3ctrl = max(num_drv3ctrl, drvctrl + 1);
+			break;
+		default:
+			break;
+		}
+
+		if (pull_dir == UNIPHIER_PIN_PULL_UP ||
+		    pull_dir == UNIPHIER_PIN_PULL_DOWN)
+			num_pupdctrl = max(num_pupdctrl, pupdctrl + 1);
+
+		if (iectrl != UNIPHIER_PIN_IECTRL_NONE) {
+			if (socdata->caps & UNIPHIER_PINCTRL_CAPS_PERPIN_IECTRL)
+				iectrl = i;
+			num_iectrl = max(num_iectrl, iectrl + 1);
+		}
+	}
+
+	INIT_LIST_HEAD(&priv->reg_regions);
+
+	ret = uniphier_pinctrl_add_reg_region(dev, priv,
+					      UNIPHIER_PINCTRL_PINMUX_BASE,
+					      socdata->npins, 8);
+	if (ret)
+		return ret;
+
+	ret = uniphier_pinctrl_add_reg_region(dev, priv,
+					      UNIPHIER_PINCTRL_DRVCTRL_BASE,
+					      num_drvctrl, 1);
+	if (ret)
+		return ret;
+
+	ret = uniphier_pinctrl_add_reg_region(dev, priv,
+					      UNIPHIER_PINCTRL_DRV2CTRL_BASE,
+					      num_drv2ctrl, 2);
+	if (ret)
+		return ret;
+
+	ret = uniphier_pinctrl_add_reg_region(dev, priv,
+					      UNIPHIER_PINCTRL_DRV3CTRL_BASE,
+					      num_drv3ctrl, 3);
+	if (ret)
+		return ret;
+
+	ret = uniphier_pinctrl_add_reg_region(dev, priv,
+					      UNIPHIER_PINCTRL_PUPDCTRL_BASE,
+					      num_pupdctrl, 1);
+	if (ret)
+		return ret;
+
+	ret = uniphier_pinctrl_add_reg_region(dev, priv,
+					      UNIPHIER_PINCTRL_IECTRL_BASE,
+					      num_iectrl, 1);
+	if (ret)
+		return ret;
+#endif
+	return 0;
+}
+
+const struct dev_pm_ops uniphier_pinctrl_pm_ops = {
+	SET_LATE_SYSTEM_SLEEP_PM_OPS(uniphier_pinctrl_suspend,
+				     uniphier_pinctrl_resume)
+};
+
 int uniphier_pinctrl_probe(struct platform_device *pdev,
 			   struct uniphier_pinctrl_socdata *socdata)
 {
 	struct device *dev = &pdev->dev;
 	struct uniphier_pinctrl_priv *priv;
 	struct device_node *parent;
+	int ret;
 
 	if (!socdata ||
 	    !socdata->pins || !socdata->npins ||
@@ -721,6 +899,10 @@ int uniphier_pinctrl_probe(struct platform_device *pdev,
 	priv->pctldesc.confops = &uniphier_confops;
 	priv->pctldesc.owner = dev->driver->owner;
 
+	ret = uniphier_pinctrl_pm_init(dev, priv);
+	if (ret)
+		return ret;
+
 	priv->pctldev = devm_pinctrl_register(dev, &priv->pctldesc, priv);
 	if (IS_ERR(priv->pctldev)) {
 		dev_err(dev, "failed to register UniPhier pinctrl driver\n");
@@ -731,4 +913,3 @@ int uniphier_pinctrl_probe(struct platform_device *pdev,
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(uniphier_pinctrl_probe);
diff --git a/drivers/pinctrl/uniphier/pinctrl-uniphier-ld11.c b/drivers/pinctrl/uniphier/pinctrl-uniphier-ld11.c
index ad73db8d067b..9c5e359a63de 100644
--- a/drivers/pinctrl/uniphier/pinctrl-uniphier-ld11.c
+++ b/drivers/pinctrl/uniphier/pinctrl-uniphier-ld11.c
@@ -21,7 +21,7 @@
 #include "pinctrl-uniphier.h"
 
 static const struct pinctrl_pin_desc uniphier_ld11_pins[] = {
-	UNIPHIER_PINCTRL_PIN(0, "XECS1", 0,
+	UNIPHIER_PINCTRL_PIN(0, "XECS1", UNIPHIER_PIN_IECTRL_EXIST,
 			     0, UNIPHIER_PIN_DRV_1BIT,
 			     0, UNIPHIER_PIN_PULL_UP),
 	UNIPHIER_PINCTRL_PIN(1, "ERXW", UNIPHIER_PIN_IECTRL_NONE,
@@ -30,13 +30,13 @@ static const struct pinctrl_pin_desc uniphier_ld11_pins[] = {
 	UNIPHIER_PINCTRL_PIN(2, "XERWE1", UNIPHIER_PIN_IECTRL_NONE,
 			     2, UNIPHIER_PIN_DRV_1BIT,
 			     2, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(3, "XNFWP", 3,
+	UNIPHIER_PINCTRL_PIN(3, "XNFWP", UNIPHIER_PIN_IECTRL_EXIST,
 			     3, UNIPHIER_PIN_DRV_1BIT,
 			     3, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(4, "XNFCE0", 4,
+	UNIPHIER_PINCTRL_PIN(4, "XNFCE0", UNIPHIER_PIN_IECTRL_EXIST,
 			     4, UNIPHIER_PIN_DRV_1BIT,
 			     4, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(5, "NFRYBY0", 5,
+	UNIPHIER_PINCTRL_PIN(5, "NFRYBY0", UNIPHIER_PIN_IECTRL_EXIST,
 			     5, UNIPHIER_PIN_DRV_1BIT,
 			     5, UNIPHIER_PIN_PULL_UP),
 	UNIPHIER_PINCTRL_PIN(6, "XNFRE", UNIPHIER_PIN_IECTRL_NONE,
@@ -51,425 +51,427 @@ static const struct pinctrl_pin_desc uniphier_ld11_pins[] = {
 	UNIPHIER_PINCTRL_PIN(9, "NFCLE", UNIPHIER_PIN_IECTRL_NONE,
 			     9, UNIPHIER_PIN_DRV_1BIT,
 			     9, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(10, "NFD0", 10,
+	UNIPHIER_PINCTRL_PIN(10, "NFD0", UNIPHIER_PIN_IECTRL_EXIST,
 			     10, UNIPHIER_PIN_DRV_1BIT,
 			     10, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(11, "NFD1", 11,
+	UNIPHIER_PINCTRL_PIN(11, "NFD1", UNIPHIER_PIN_IECTRL_EXIST,
 			     11, UNIPHIER_PIN_DRV_1BIT,
 			     11, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(12, "NFD2", 12,
+	UNIPHIER_PINCTRL_PIN(12, "NFD2", UNIPHIER_PIN_IECTRL_EXIST,
 			     12, UNIPHIER_PIN_DRV_1BIT,
 			     12, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(13, "NFD3", 13,
+	UNIPHIER_PINCTRL_PIN(13, "NFD3", UNIPHIER_PIN_IECTRL_EXIST,
 			     13, UNIPHIER_PIN_DRV_1BIT,
 			     13, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(14, "NFD4", 14,
+	UNIPHIER_PINCTRL_PIN(14, "NFD4", UNIPHIER_PIN_IECTRL_EXIST,
 			     14, UNIPHIER_PIN_DRV_1BIT,
 			     14, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(15, "NFD5", 15,
+	UNIPHIER_PINCTRL_PIN(15, "NFD5", UNIPHIER_PIN_IECTRL_EXIST,
 			     15, UNIPHIER_PIN_DRV_1BIT,
 			     15, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(16, "NFD6", 16,
+	UNIPHIER_PINCTRL_PIN(16, "NFD6", UNIPHIER_PIN_IECTRL_EXIST,
 			     16, UNIPHIER_PIN_DRV_1BIT,
 			     16, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(17, "NFD7", 17,
+	UNIPHIER_PINCTRL_PIN(17, "NFD7", UNIPHIER_PIN_IECTRL_EXIST,
 			     17, UNIPHIER_PIN_DRV_1BIT,
 			     17, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(18, "XERST", 18,
+	UNIPHIER_PINCTRL_PIN(18, "XERST", UNIPHIER_PIN_IECTRL_EXIST,
 			     0, UNIPHIER_PIN_DRV_2BIT,
 			     18, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(19, "MMCCLK", 19,
+	UNIPHIER_PINCTRL_PIN(19, "MMCCLK", UNIPHIER_PIN_IECTRL_EXIST,
 			     1, UNIPHIER_PIN_DRV_2BIT,
 			     19, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(20, "MMCCMD", 20,
+	UNIPHIER_PINCTRL_PIN(20, "MMCCMD", UNIPHIER_PIN_IECTRL_EXIST,
 			     2, UNIPHIER_PIN_DRV_2BIT,
 			     20, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(21, "MMCDS", 21,
+	UNIPHIER_PINCTRL_PIN(21, "MMCDS", UNIPHIER_PIN_IECTRL_EXIST,
 			     3, UNIPHIER_PIN_DRV_2BIT,
 			     21, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(22, "MMCDAT0", 22,
+	UNIPHIER_PINCTRL_PIN(22, "MMCDAT0", UNIPHIER_PIN_IECTRL_EXIST,
 			     4, UNIPHIER_PIN_DRV_2BIT,
 			     22, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(23, "MMCDAT1", 23,
+	UNIPHIER_PINCTRL_PIN(23, "MMCDAT1", UNIPHIER_PIN_IECTRL_EXIST,
 			     5, UNIPHIER_PIN_DRV_2BIT,
 			     23, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(24, "MMCDAT2", 24,
+	UNIPHIER_PINCTRL_PIN(24, "MMCDAT2", UNIPHIER_PIN_IECTRL_EXIST,
 			     6, UNIPHIER_PIN_DRV_2BIT,
 			     24, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(25, "MMCDAT3", 25,
+	UNIPHIER_PINCTRL_PIN(25, "MMCDAT3", UNIPHIER_PIN_IECTRL_EXIST,
 			     7, UNIPHIER_PIN_DRV_2BIT,
 			     25, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(26, "MMCDAT4", 26,
+	UNIPHIER_PINCTRL_PIN(26, "MMCDAT4", UNIPHIER_PIN_IECTRL_EXIST,
 			     8, UNIPHIER_PIN_DRV_2BIT,
 			     26, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(27, "MMCDAT5", 27,
+	UNIPHIER_PINCTRL_PIN(27, "MMCDAT5", UNIPHIER_PIN_IECTRL_EXIST,
 			     9, UNIPHIER_PIN_DRV_2BIT,
 			     27, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(28, "MMCDAT6", 28,
+	UNIPHIER_PINCTRL_PIN(28, "MMCDAT6", UNIPHIER_PIN_IECTRL_EXIST,
 			     10, UNIPHIER_PIN_DRV_2BIT,
 			     28, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(29, "MMCDAT7", 29,
+	UNIPHIER_PINCTRL_PIN(29, "MMCDAT7", UNIPHIER_PIN_IECTRL_EXIST,
 			     11, UNIPHIER_PIN_DRV_2BIT,
 			     29, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(46, "USB0VBUS", 46,
+	UNIPHIER_PINCTRL_PIN(46, "USB0VBUS", UNIPHIER_PIN_IECTRL_EXIST,
 			     46, UNIPHIER_PIN_DRV_1BIT,
 			     46, UNIPHIER_PIN_PULL_DOWN),
 	UNIPHIER_PINCTRL_PIN(47, "USB0OD", UNIPHIER_PIN_IECTRL_NONE,
 			     47, UNIPHIER_PIN_DRV_1BIT,
 			     47, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(48, "USB1VBUS", 48,
+	UNIPHIER_PINCTRL_PIN(48, "USB1VBUS", UNIPHIER_PIN_IECTRL_EXIST,
 			     48, UNIPHIER_PIN_DRV_1BIT,
 			     48, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(49, "USB1OD", 49,
+	UNIPHIER_PINCTRL_PIN(49, "USB1OD", UNIPHIER_PIN_IECTRL_EXIST,
 			     49, UNIPHIER_PIN_DRV_1BIT,
 			     49, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(50, "USB2VBUS", 50,
+	UNIPHIER_PINCTRL_PIN(50, "USB2VBUS", UNIPHIER_PIN_IECTRL_EXIST,
 			     50, UNIPHIER_PIN_DRV_1BIT,
 			     50, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(51, "USB2OD", 51,
+	UNIPHIER_PINCTRL_PIN(51, "USB2OD", UNIPHIER_PIN_IECTRL_EXIST,
 			     51, UNIPHIER_PIN_DRV_1BIT,
 			     51, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(54, "TXD0", 54,
+	UNIPHIER_PINCTRL_PIN(54, "TXD0", UNIPHIER_PIN_IECTRL_EXIST,
 			     54, UNIPHIER_PIN_DRV_1BIT,
 			     54, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(55, "RXD0", 55,
+	UNIPHIER_PINCTRL_PIN(55, "RXD0", UNIPHIER_PIN_IECTRL_EXIST,
 			     55, UNIPHIER_PIN_DRV_1BIT,
 			     55, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(56, "SPISYNC0", 56,
+	UNIPHIER_PINCTRL_PIN(56, "SPISYNC0", UNIPHIER_PIN_IECTRL_EXIST,
 			     56, UNIPHIER_PIN_DRV_1BIT,
 			     56, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(57, "SPISCLK0", 57,
+	UNIPHIER_PINCTRL_PIN(57, "SPISCLK0", UNIPHIER_PIN_IECTRL_EXIST,
 			     57, UNIPHIER_PIN_DRV_1BIT,
 			     57, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(58, "SPITXD0", 58,
+	UNIPHIER_PINCTRL_PIN(58, "SPITXD0", UNIPHIER_PIN_IECTRL_EXIST,
 			     58, UNIPHIER_PIN_DRV_1BIT,
 			     58, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(59, "SPIRXD0", 59,
+	UNIPHIER_PINCTRL_PIN(59, "SPIRXD0", UNIPHIER_PIN_IECTRL_EXIST,
 			     59, UNIPHIER_PIN_DRV_1BIT,
 			     59, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(60, "AGCI", 60,
+	UNIPHIER_PINCTRL_PIN(60, "AGCI", UNIPHIER_PIN_IECTRL_EXIST,
 			     60, UNIPHIER_PIN_DRV_1BIT,
 			     60, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(61, "DMDSDA0", 61,
+	UNIPHIER_PINCTRL_PIN(61, "DMDSDA0", UNIPHIER_PIN_IECTRL_EXIST,
 			     -1, UNIPHIER_PIN_DRV_FIXED4,
 			     -1, UNIPHIER_PIN_PULL_NONE),
-	UNIPHIER_PINCTRL_PIN(62, "DMDSCL0", 62,
+	UNIPHIER_PINCTRL_PIN(62, "DMDSCL0", UNIPHIER_PIN_IECTRL_EXIST,
 			     -1, UNIPHIER_PIN_DRV_FIXED4,
 			     -1, UNIPHIER_PIN_PULL_NONE),
-	UNIPHIER_PINCTRL_PIN(63, "SDA0", 63,
+	UNIPHIER_PINCTRL_PIN(63, "SDA0", UNIPHIER_PIN_IECTRL_EXIST,
 			     -1, UNIPHIER_PIN_DRV_FIXED4,
 			     -1, UNIPHIER_PIN_PULL_NONE),
-	UNIPHIER_PINCTRL_PIN(64, "SCL0", 64,
+	UNIPHIER_PINCTRL_PIN(64, "SCL0", UNIPHIER_PIN_IECTRL_EXIST,
 			     -1, UNIPHIER_PIN_DRV_FIXED4,
 			     -1, UNIPHIER_PIN_PULL_NONE),
-	UNIPHIER_PINCTRL_PIN(65, "SDA1", 65,
+	UNIPHIER_PINCTRL_PIN(65, "SDA1", UNIPHIER_PIN_IECTRL_EXIST,
 			     -1, UNIPHIER_PIN_DRV_FIXED4,
 			     -1, UNIPHIER_PIN_PULL_NONE),
-	UNIPHIER_PINCTRL_PIN(66, "SCL1", 66,
+	UNIPHIER_PINCTRL_PIN(66, "SCL1", UNIPHIER_PIN_IECTRL_EXIST,
 			     -1, UNIPHIER_PIN_DRV_FIXED4,
 			     -1, UNIPHIER_PIN_PULL_NONE),
-	UNIPHIER_PINCTRL_PIN(67, "HIN", 67,
+	UNIPHIER_PINCTRL_PIN(67, "HIN", UNIPHIER_PIN_IECTRL_EXIST,
 			     -1, UNIPHIER_PIN_DRV_FIXED5,
 			     -1, UNIPHIER_PIN_PULL_NONE),
-	UNIPHIER_PINCTRL_PIN(68, "VIN", 68,
+	UNIPHIER_PINCTRL_PIN(68, "VIN", UNIPHIER_PIN_IECTRL_EXIST,
 			     -1, UNIPHIER_PIN_DRV_FIXED5,
 			     -1, UNIPHIER_PIN_PULL_NONE),
-	UNIPHIER_PINCTRL_PIN(69, "PCA00", 69,
+	UNIPHIER_PINCTRL_PIN(69, "PCA00", UNIPHIER_PIN_IECTRL_EXIST,
 			     69, UNIPHIER_PIN_DRV_1BIT,
 			     69, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(70, "PCA01", 70,
+	UNIPHIER_PINCTRL_PIN(70, "PCA01", UNIPHIER_PIN_IECTRL_EXIST,
 			     70, UNIPHIER_PIN_DRV_1BIT,
 			     70, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(71, "PCA02", 71,
+	UNIPHIER_PINCTRL_PIN(71, "PCA02", UNIPHIER_PIN_IECTRL_EXIST,
 			     71, UNIPHIER_PIN_DRV_1BIT,
 			     71, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(72, "PCA03", 72,
+	UNIPHIER_PINCTRL_PIN(72, "PCA03", UNIPHIER_PIN_IECTRL_EXIST,
 			     72, UNIPHIER_PIN_DRV_1BIT,
 			     72, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(73, "PCA04", 73,
+	UNIPHIER_PINCTRL_PIN(73, "PCA04", UNIPHIER_PIN_IECTRL_EXIST,
 			     73, UNIPHIER_PIN_DRV_1BIT,
 			     73, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(74, "PCA05", 74,
+	UNIPHIER_PINCTRL_PIN(74, "PCA05", UNIPHIER_PIN_IECTRL_EXIST,
 			     74, UNIPHIER_PIN_DRV_1BIT,
 			     74, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(75, "PCA06", 75,
+	UNIPHIER_PINCTRL_PIN(75, "PCA06", UNIPHIER_PIN_IECTRL_EXIST,
 			     75, UNIPHIER_PIN_DRV_1BIT,
 			     75, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(76, "PCA07", 76,
+	UNIPHIER_PINCTRL_PIN(76, "PCA07", UNIPHIER_PIN_IECTRL_EXIST,
 			     76, UNIPHIER_PIN_DRV_1BIT,
 			     76, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(77, "PCA08", 77,
+	UNIPHIER_PINCTRL_PIN(77, "PCA08", UNIPHIER_PIN_IECTRL_EXIST,
 			     77, UNIPHIER_PIN_DRV_1BIT,
 			     77, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(78, "PCA09", 78,
+	UNIPHIER_PINCTRL_PIN(78, "PCA09", UNIPHIER_PIN_IECTRL_EXIST,
 			     78, UNIPHIER_PIN_DRV_1BIT,
 			     78, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(79, "PCA10", 79,
+	UNIPHIER_PINCTRL_PIN(79, "PCA10", UNIPHIER_PIN_IECTRL_EXIST,
 			     79, UNIPHIER_PIN_DRV_1BIT,
 			     79, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(80, "PCA11", 80,
+	UNIPHIER_PINCTRL_PIN(80, "PCA11", UNIPHIER_PIN_IECTRL_EXIST,
 			     80, UNIPHIER_PIN_DRV_1BIT,
 			     80, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(81, "PCA12", 81,
+	UNIPHIER_PINCTRL_PIN(81, "PCA12", UNIPHIER_PIN_IECTRL_EXIST,
 			     81, UNIPHIER_PIN_DRV_1BIT,
 			     81, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(82, "PCA13", 82,
+	UNIPHIER_PINCTRL_PIN(82, "PCA13", UNIPHIER_PIN_IECTRL_EXIST,
 			     82, UNIPHIER_PIN_DRV_1BIT,
 			     82, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(83, "PCA14", 83,
+	UNIPHIER_PINCTRL_PIN(83, "PCA14", UNIPHIER_PIN_IECTRL_EXIST,
 			     83, UNIPHIER_PIN_DRV_1BIT,
 			     83, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(84, "PC0READY", 84,
+	UNIPHIER_PINCTRL_PIN(84, "PC0READY", UNIPHIER_PIN_IECTRL_EXIST,
 			     84, UNIPHIER_PIN_DRV_1BIT,
 			     84, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(85, "PC0CD1", 85,
+	UNIPHIER_PINCTRL_PIN(85, "PC0CD1", UNIPHIER_PIN_IECTRL_EXIST,
 			     85, UNIPHIER_PIN_DRV_1BIT,
 			     85, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(86, "PC0CD2", 86,
+	UNIPHIER_PINCTRL_PIN(86, "PC0CD2", UNIPHIER_PIN_IECTRL_EXIST,
 			     86, UNIPHIER_PIN_DRV_1BIT,
 			     86, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(87, "PC0WAIT", 87,
+	UNIPHIER_PINCTRL_PIN(87, "PC0WAIT", UNIPHIER_PIN_IECTRL_EXIST,
 			     87, UNIPHIER_PIN_DRV_1BIT,
 			     87, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(88, "PC0RESET", 88,
+	UNIPHIER_PINCTRL_PIN(88, "PC0RESET", UNIPHIER_PIN_IECTRL_EXIST,
 			     88, UNIPHIER_PIN_DRV_1BIT,
 			     88, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(89, "PC0CE1", 89,
+	UNIPHIER_PINCTRL_PIN(89, "PC0CE1", UNIPHIER_PIN_IECTRL_EXIST,
 			     89, UNIPHIER_PIN_DRV_1BIT,
 			     89, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(90, "PC0WE", 90,
+	UNIPHIER_PINCTRL_PIN(90, "PC0WE", UNIPHIER_PIN_IECTRL_EXIST,
 			     90, UNIPHIER_PIN_DRV_1BIT,
 			     90, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(91, "PC0OE", 91,
+	UNIPHIER_PINCTRL_PIN(91, "PC0OE", UNIPHIER_PIN_IECTRL_EXIST,
 			     91, UNIPHIER_PIN_DRV_1BIT,
 			     91, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(92, "PC0IOWR", 92,
+	UNIPHIER_PINCTRL_PIN(92, "PC0IOWR", UNIPHIER_PIN_IECTRL_EXIST,
 			     92, UNIPHIER_PIN_DRV_1BIT,
 			     92, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(93, "PC0IORD", 93,
+	UNIPHIER_PINCTRL_PIN(93, "PC0IORD", UNIPHIER_PIN_IECTRL_EXIST,
 			     93, UNIPHIER_PIN_DRV_1BIT,
 			     93, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(94, "PCD00", 94,
+	UNIPHIER_PINCTRL_PIN(94, "PCD00", UNIPHIER_PIN_IECTRL_EXIST,
 			     94, UNIPHIER_PIN_DRV_1BIT,
 			     94, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(95, "PCD01", 95,
+	UNIPHIER_PINCTRL_PIN(95, "PCD01", UNIPHIER_PIN_IECTRL_EXIST,
 			     95, UNIPHIER_PIN_DRV_1BIT,
 			     95, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(96, "PCD02", 96,
+	UNIPHIER_PINCTRL_PIN(96, "PCD02", UNIPHIER_PIN_IECTRL_EXIST,
 			     96, UNIPHIER_PIN_DRV_1BIT,
 			     96, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(97, "PCD03", 97,
+	UNIPHIER_PINCTRL_PIN(97, "PCD03", UNIPHIER_PIN_IECTRL_EXIST,
 			     97, UNIPHIER_PIN_DRV_1BIT,
 			     97, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(98, "PCD04", 98,
+	UNIPHIER_PINCTRL_PIN(98, "PCD04", UNIPHIER_PIN_IECTRL_EXIST,
 			     98, UNIPHIER_PIN_DRV_1BIT,
 			     98, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(99, "PCD05", 99,
+	UNIPHIER_PINCTRL_PIN(99, "PCD05", UNIPHIER_PIN_IECTRL_EXIST,
 			     99, UNIPHIER_PIN_DRV_1BIT,
 			     99, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(100, "PCD06", 100,
+	UNIPHIER_PINCTRL_PIN(100, "PCD06", UNIPHIER_PIN_IECTRL_EXIST,
 			     100, UNIPHIER_PIN_DRV_1BIT,
 			     100, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(101, "PCD07", 101,
+	UNIPHIER_PINCTRL_PIN(101, "PCD07", UNIPHIER_PIN_IECTRL_EXIST,
 			     101, UNIPHIER_PIN_DRV_1BIT,
 			     101, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(102, "HS0BCLKIN", 102,
+	UNIPHIER_PINCTRL_PIN(102, "HS0BCLKIN", UNIPHIER_PIN_IECTRL_EXIST,
 			     102, UNIPHIER_PIN_DRV_1BIT,
 			     102, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(103, "HS0SYNCIN", 103,
+	UNIPHIER_PINCTRL_PIN(103, "HS0SYNCIN", UNIPHIER_PIN_IECTRL_EXIST,
 			     103, UNIPHIER_PIN_DRV_1BIT,
 			     103, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(104, "HS0VALIN", 104,
+	UNIPHIER_PINCTRL_PIN(104, "HS0VALIN", UNIPHIER_PIN_IECTRL_EXIST,
 			     104, UNIPHIER_PIN_DRV_1BIT,
 			     104, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(105, "HS0DIN0", 105,
+	UNIPHIER_PINCTRL_PIN(105, "HS0DIN0", UNIPHIER_PIN_IECTRL_EXIST,
 			     105, UNIPHIER_PIN_DRV_1BIT,
 			     105, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(106, "HS0DIN1", 106,
+	UNIPHIER_PINCTRL_PIN(106, "HS0DIN1", UNIPHIER_PIN_IECTRL_EXIST,
 			     106, UNIPHIER_PIN_DRV_1BIT,
 			     106, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(107, "HS0DIN2", 107,
+	UNIPHIER_PINCTRL_PIN(107, "HS0DIN2", UNIPHIER_PIN_IECTRL_EXIST,
 			     107, UNIPHIER_PIN_DRV_1BIT,
 			     107, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(108, "HS0DIN3", 108,
+	UNIPHIER_PINCTRL_PIN(108, "HS0DIN3", UNIPHIER_PIN_IECTRL_EXIST,
 			     108, UNIPHIER_PIN_DRV_1BIT,
 			     108, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(109, "HS0DIN4", 109,
+	UNIPHIER_PINCTRL_PIN(109, "HS0DIN4", UNIPHIER_PIN_IECTRL_EXIST,
 			     109, UNIPHIER_PIN_DRV_1BIT,
 			     109, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(110, "HS0DIN5", 110,
+	UNIPHIER_PINCTRL_PIN(110, "HS0DIN5", UNIPHIER_PIN_IECTRL_EXIST,
 			     110, UNIPHIER_PIN_DRV_1BIT,
 			     110, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(111, "HS0DIN6", 111,
+	UNIPHIER_PINCTRL_PIN(111, "HS0DIN6", UNIPHIER_PIN_IECTRL_EXIST,
 			     111, UNIPHIER_PIN_DRV_1BIT,
 			     111, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(112, "HS0DIN7", 112,
+	UNIPHIER_PINCTRL_PIN(112, "HS0DIN7", UNIPHIER_PIN_IECTRL_EXIST,
 			     112, UNIPHIER_PIN_DRV_1BIT,
 			     112, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(113, "HS0BCLKOUT", 113,
+	UNIPHIER_PINCTRL_PIN(113, "HS0BCLKOUT", UNIPHIER_PIN_IECTRL_EXIST,
 			     113, UNIPHIER_PIN_DRV_1BIT,
 			     113, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(114, "HS0SYNCOUT", 114,
+	UNIPHIER_PINCTRL_PIN(114, "HS0SYNCOUT", UNIPHIER_PIN_IECTRL_EXIST,
 			     114, UNIPHIER_PIN_DRV_1BIT,
 			     114, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(115, "HS0VALOUT", 115,
+	UNIPHIER_PINCTRL_PIN(115, "HS0VALOUT", UNIPHIER_PIN_IECTRL_EXIST,
 			     115, UNIPHIER_PIN_DRV_1BIT,
 			     115, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(116, "HS0DOUT0", 116,
+	UNIPHIER_PINCTRL_PIN(116, "HS0DOUT0", UNIPHIER_PIN_IECTRL_EXIST,
 			     116, UNIPHIER_PIN_DRV_1BIT,
 			     116, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(117, "HS0DOUT1", 117,
+	UNIPHIER_PINCTRL_PIN(117, "HS0DOUT1", UNIPHIER_PIN_IECTRL_EXIST,
 			     117, UNIPHIER_PIN_DRV_1BIT,
 			     117, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(118, "HS0DOUT2", 118,
+	UNIPHIER_PINCTRL_PIN(118, "HS0DOUT2", UNIPHIER_PIN_IECTRL_EXIST,
 			     118, UNIPHIER_PIN_DRV_1BIT,
 			     118, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(119, "HS0DOUT3", 119,
+	UNIPHIER_PINCTRL_PIN(119, "HS0DOUT3", UNIPHIER_PIN_IECTRL_EXIST,
 			     119, UNIPHIER_PIN_DRV_1BIT,
 			     119, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(120, "HS0DOUT4", 120,
+	UNIPHIER_PINCTRL_PIN(120, "HS0DOUT4", UNIPHIER_PIN_IECTRL_EXIST,
 			     120, UNIPHIER_PIN_DRV_1BIT,
 			     120, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(121, "HS0DOUT5", 121,
+	UNIPHIER_PINCTRL_PIN(121, "HS0DOUT5", UNIPHIER_PIN_IECTRL_EXIST,
 			     121, UNIPHIER_PIN_DRV_1BIT,
 			     121, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(122, "HS0DOUT6", 122,
+	UNIPHIER_PINCTRL_PIN(122, "HS0DOUT6", UNIPHIER_PIN_IECTRL_EXIST,
 			     122, UNIPHIER_PIN_DRV_1BIT,
 			     122, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(123, "HS0DOUT7", 123,
+	UNIPHIER_PINCTRL_PIN(123, "HS0DOUT7", UNIPHIER_PIN_IECTRL_EXIST,
 			     123, UNIPHIER_PIN_DRV_1BIT,
 			     123, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(124, "HS1BCLKIN", 124,
+	UNIPHIER_PINCTRL_PIN(124, "HS1BCLKIN", UNIPHIER_PIN_IECTRL_EXIST,
 			     124, UNIPHIER_PIN_DRV_1BIT,
 			     124, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(125, "HS1SYNCIN", 125,
+	UNIPHIER_PINCTRL_PIN(125, "HS1SYNCIN", UNIPHIER_PIN_IECTRL_EXIST,
 			     125, UNIPHIER_PIN_DRV_1BIT,
 			     125, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(126, "HS1VALIN", 126,
+	UNIPHIER_PINCTRL_PIN(126, "HS1VALIN", UNIPHIER_PIN_IECTRL_EXIST,
 			     126, UNIPHIER_PIN_DRV_1BIT,
 			     126, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(127, "HS1DIN0", 127,
+	UNIPHIER_PINCTRL_PIN(127, "HS1DIN0", UNIPHIER_PIN_IECTRL_EXIST,
 			     127, UNIPHIER_PIN_DRV_1BIT,
 			     127, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(128, "HS1DIN1", 128,
+	UNIPHIER_PINCTRL_PIN(128, "HS1DIN1", UNIPHIER_PIN_IECTRL_EXIST,
 			     128, UNIPHIER_PIN_DRV_1BIT,
 			     128, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(129, "HS1DIN2", 129,
+	UNIPHIER_PINCTRL_PIN(129, "HS1DIN2", UNIPHIER_PIN_IECTRL_EXIST,
 			     129, UNIPHIER_PIN_DRV_1BIT,
 			     129, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(130, "HS1DIN3", 130,
+	UNIPHIER_PINCTRL_PIN(130, "HS1DIN3", UNIPHIER_PIN_IECTRL_EXIST,
 			     130, UNIPHIER_PIN_DRV_1BIT,
 			     130, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(131, "HS1DIN4", 131,
+	UNIPHIER_PINCTRL_PIN(131, "HS1DIN4", UNIPHIER_PIN_IECTRL_EXIST,
 			     131, UNIPHIER_PIN_DRV_1BIT,
 			     131, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(132, "HS1DIN5", 132,
+	UNIPHIER_PINCTRL_PIN(132, "HS1DIN5", UNIPHIER_PIN_IECTRL_EXIST,
 			     132, UNIPHIER_PIN_DRV_1BIT,
 			     132, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(133, "HS1DIN6", 133,
+	UNIPHIER_PINCTRL_PIN(133, "HS1DIN6", UNIPHIER_PIN_IECTRL_EXIST,
 			     133, UNIPHIER_PIN_DRV_1BIT,
 			     133, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(134, "HS1DIN7", 134,
+	UNIPHIER_PINCTRL_PIN(134, "HS1DIN7", UNIPHIER_PIN_IECTRL_EXIST,
 			     134, UNIPHIER_PIN_DRV_1BIT,
 			     134, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(135, "AO1IEC", 135,
+	UNIPHIER_PINCTRL_PIN(135, "AO1IEC", UNIPHIER_PIN_IECTRL_EXIST,
 			     135, UNIPHIER_PIN_DRV_1BIT,
 			     135, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(136, "AO1ARC", 136,
+	UNIPHIER_PINCTRL_PIN(136, "AO1ARC", UNIPHIER_PIN_IECTRL_EXIST,
 			     136, UNIPHIER_PIN_DRV_1BIT,
 			     136, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(137, "AO1DACCK", 137,
+	UNIPHIER_PINCTRL_PIN(137, "AO1DACCK", UNIPHIER_PIN_IECTRL_EXIST,
 			     137, UNIPHIER_PIN_DRV_1BIT,
 			     137, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(138, "AO1BCK", 138,
+	UNIPHIER_PINCTRL_PIN(138, "AO1BCK", UNIPHIER_PIN_IECTRL_EXIST,
 			     138, UNIPHIER_PIN_DRV_1BIT,
 			     138, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(139, "AO1LRCK", 139,
+	UNIPHIER_PINCTRL_PIN(139, "AO1LRCK", UNIPHIER_PIN_IECTRL_EXIST,
 			     139, UNIPHIER_PIN_DRV_1BIT,
 			     139, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(140, "AO1D0", 140,
+	UNIPHIER_PINCTRL_PIN(140, "AO1D0", UNIPHIER_PIN_IECTRL_EXIST,
 			     140, UNIPHIER_PIN_DRV_1BIT,
 			     140, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(141, "AO1D1", 141,
+	UNIPHIER_PINCTRL_PIN(141, "AO1D1", UNIPHIER_PIN_IECTRL_EXIST,
 			     141, UNIPHIER_PIN_DRV_1BIT,
 			     141, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(142, "AO1D2", 142,
+	UNIPHIER_PINCTRL_PIN(142, "AO1D2", UNIPHIER_PIN_IECTRL_EXIST,
 			     142, UNIPHIER_PIN_DRV_1BIT,
 			     142, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(143, "XIRQ9", 143,
+	UNIPHIER_PINCTRL_PIN(143, "XIRQ9", UNIPHIER_PIN_IECTRL_EXIST,
 			     143, UNIPHIER_PIN_DRV_1BIT,
 			     143, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(144, "XIRQ10", 144,
+	UNIPHIER_PINCTRL_PIN(144, "XIRQ10", UNIPHIER_PIN_IECTRL_EXIST,
 			     144, UNIPHIER_PIN_DRV_1BIT,
 			     144, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(145, "XIRQ11", 145,
+	UNIPHIER_PINCTRL_PIN(145, "XIRQ11", UNIPHIER_PIN_IECTRL_EXIST,
 			     145, UNIPHIER_PIN_DRV_1BIT,
 			     145, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(146, "XIRQ13", 146,
+	UNIPHIER_PINCTRL_PIN(146, "XIRQ13", UNIPHIER_PIN_IECTRL_EXIST,
 			     146, UNIPHIER_PIN_DRV_1BIT,
 			     146, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(147, "PWMA", 147,
+	UNIPHIER_PINCTRL_PIN(147, "PWMA", UNIPHIER_PIN_IECTRL_EXIST,
 			     147, UNIPHIER_PIN_DRV_1BIT,
 			     147, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(148, "LR_GOUT", 148,
+	UNIPHIER_PINCTRL_PIN(148, "LR_GOUT", UNIPHIER_PIN_IECTRL_EXIST,
 			     148, UNIPHIER_PIN_DRV_1BIT,
 			     148, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(149, "XIRQ0", 149,
+	UNIPHIER_PINCTRL_PIN(149, "XIRQ0", UNIPHIER_PIN_IECTRL_EXIST,
 			     149, UNIPHIER_PIN_DRV_1BIT,
 			     149, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(150, "XIRQ1", 150,
+	UNIPHIER_PINCTRL_PIN(150, "XIRQ1", UNIPHIER_PIN_IECTRL_EXIST,
 			     150, UNIPHIER_PIN_DRV_1BIT,
 			     150, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(151, "XIRQ2", 151,
+	UNIPHIER_PINCTRL_PIN(151, "XIRQ2", UNIPHIER_PIN_IECTRL_EXIST,
 			     151, UNIPHIER_PIN_DRV_1BIT,
 			     151, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(152, "XIRQ3", 152,
+	UNIPHIER_PINCTRL_PIN(152, "XIRQ3", UNIPHIER_PIN_IECTRL_EXIST,
 			     152, UNIPHIER_PIN_DRV_1BIT,
 			     152, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(153, "XIRQ4", 153,
+	UNIPHIER_PINCTRL_PIN(153, "XIRQ4", UNIPHIER_PIN_IECTRL_EXIST,
 			     153, UNIPHIER_PIN_DRV_1BIT,
 			     153, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(154, "XIRQ5", 154,
+	UNIPHIER_PINCTRL_PIN(154, "XIRQ5", UNIPHIER_PIN_IECTRL_EXIST,
 			     154, UNIPHIER_PIN_DRV_1BIT,
 			     154, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(155, "XIRQ6", 155,
+	UNIPHIER_PINCTRL_PIN(155, "XIRQ6", UNIPHIER_PIN_IECTRL_EXIST,
 			     155, UNIPHIER_PIN_DRV_1BIT,
 			     155, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(156, "XIRQ7", 156,
+	UNIPHIER_PINCTRL_PIN(156, "XIRQ7", UNIPHIER_PIN_IECTRL_EXIST,
 			     156, UNIPHIER_PIN_DRV_1BIT,
 			     156, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(157, "XIRQ8", 157,
+	UNIPHIER_PINCTRL_PIN(157, "XIRQ8", UNIPHIER_PIN_IECTRL_EXIST,
 			     157, UNIPHIER_PIN_DRV_1BIT,
 			     157, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(158, "AGCBS", 158,
+	UNIPHIER_PINCTRL_PIN(158, "AGCBS", UNIPHIER_PIN_IECTRL_EXIST,
 			     158, UNIPHIER_PIN_DRV_1BIT,
 			     158, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(159, "XIRQ21", 159,
+	UNIPHIER_PINCTRL_PIN(159, "XIRQ21", UNIPHIER_PIN_IECTRL_EXIST,
 			     159, UNIPHIER_PIN_DRV_1BIT,
 			     159, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(160, "XIRQ22", 160,
+	UNIPHIER_PINCTRL_PIN(160, "XIRQ22", UNIPHIER_PIN_IECTRL_EXIST,
 			     160, UNIPHIER_PIN_DRV_1BIT,
 			     160, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(161, "XIRQ23", 161,
+	UNIPHIER_PINCTRL_PIN(161, "XIRQ23", UNIPHIER_PIN_IECTRL_EXIST,
 			     161, UNIPHIER_PIN_DRV_1BIT,
 			     161, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(162, "CH2CLK", 162,
+	UNIPHIER_PINCTRL_PIN(162, "CH2CLK", UNIPHIER_PIN_IECTRL_EXIST,
 			     162, UNIPHIER_PIN_DRV_1BIT,
 			     162, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(163, "CH2PSYNC", 163,
+	UNIPHIER_PINCTRL_PIN(163, "CH2PSYNC", UNIPHIER_PIN_IECTRL_EXIST,
 			     163, UNIPHIER_PIN_DRV_1BIT,
 			     163, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(164, "CH2VAL", 164,
+	UNIPHIER_PINCTRL_PIN(164, "CH2VAL", UNIPHIER_PIN_IECTRL_EXIST,
 			     164, UNIPHIER_PIN_DRV_1BIT,
 			     164, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(165, "CH2DATA", 165,
+	UNIPHIER_PINCTRL_PIN(165, "CH2DATA", UNIPHIER_PIN_IECTRL_EXIST,
 			     165, UNIPHIER_PIN_DRV_1BIT,
 			     165, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(166, "CK25O", 166,
+	UNIPHIER_PINCTRL_PIN(166, "CK25O", UNIPHIER_PIN_IECTRL_EXIST,
 			     166, UNIPHIER_PIN_DRV_1BIT,
 			     166, UNIPHIER_PIN_PULL_DOWN),
 };
 
+static const unsigned aout_pins[] = {135, 136, 137, 138, 139, 140, 141, 142};
+static const int aout_muxvals[] = {0, 0, 0, 0, 0, 0, 0, 0};
 static const unsigned emmc_pins[] = {18, 19, 20, 21, 22, 23, 24, 25};
 static const int emmc_muxvals[] = {0, 0, 0, 0, 0, 0, 0, 0};
 static const unsigned emmc_dat8_pins[] = {26, 27, 28, 29};
@@ -508,102 +510,44 @@ static const unsigned usb1_pins[] = {48, 49};
 static const int usb1_muxvals[] = {0, 0};
 static const unsigned usb2_pins[] = {50, 51};
 static const int usb2_muxvals[] = {0, 0};
-static const unsigned port_range0_pins[] = {
+static const unsigned int gpio_range0_pins[] = {
 	159, 160, 161, 162, 163, 164, 165, 166,		/* PORT0x */
 	0, 1, 2, 3, 4, 5, 6, 7,				/* PORT1x */
 	8, 9, 10, 11, 12, 13, 14, 15,			/* PORT2x */
 	16, 17, 18,					/* PORT30-32 */
 };
-static const int port_range0_muxvals[] = {
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT0x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT1x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT2x */
-	15, 15, 15,					/* PORT30-32 */
-};
-static const unsigned port_range1_pins[] = {
+static const unsigned int gpio_range1_pins[] = {
 	46, 47, 48, 49, 50,				/* PORT53-57 */
 	51,						/* PORT60 */
 };
-static const int port_range1_muxvals[] = {
-	15, 15, 15, 15, 15,				/* PORT53-57 */
-	15,						/* PORT60 */
-};
-static const unsigned port_range2_pins[] = {
+static const unsigned int gpio_range2_pins[] = {
 	54, 55, 56, 57, 58,				/* PORT63-67 */
 	59, 60, 69, 70, 71, 72, 73, 74,			/* PORT7x */
 	75, 76, 77, 78, 79, 80, 81, 82,			/* PORT8x */
 	83, 84, 85, 86, 87, 88, 89, 90,			/* PORT9x */
 	91, 92, 93, 94, 95, 96, 97, 98,			/* PORT10x */
 };
-static const int port_range2_muxvals[] = {
-	15, 15, 15, 15, 15,				/* PORT63-67 */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT7x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT8x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT9x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT10x */
-};
-static const unsigned port_range3_pins[] = {
+static const unsigned int gpio_range3_pins[] = {
 	99, 100, 101, 102, 103, 104, 105, 106,		/* PORT12x */
 	107, 108, 109, 110, 111, 112, 113, 114,		/* PORT13x */
 	115, 116, 117, 118, 119, 120, 121, 122,		/* PORT14x */
-};
-static const int port_range3_muxvals[] = {
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT12x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT13x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT14x */
-};
-static const unsigned port_range4_pins[] = {
+	149, 150, 151, 152, 153, 154, 155, 156,		/* XIRQ0-7 */
+	157, 143, 144, 145, 85, 146, 158, 84,		/* XIRQ8-15 */
+	141, 142, 148, 50, 51, 159, 160, 161,		/* XIRQ16-23 */
 	61, 62, 63, 64, 65, 66, 67, 68,			/* PORT18x */
 };
-static const int port_range4_muxvals[] = {
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT18x */
-};
-static const unsigned port_range5_pins[] = {
+static const unsigned int gpio_range4_pins[] = {
 	123, 124, 125, 126, 127, 128, 129, 130,		/* PORT20x */
 	131, 132, 133, 134, 135, 136, 137, 138,		/* PORT21x */
 	139, 140, 141, 142,				/* PORT220-223 */
 };
-static const int port_range5_muxvals[] = {
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT20x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT21x */
-	15, 15, 15, 15,					/* PORT220-223 */
-};
-static const unsigned port_range6_pins[] = {
+static const unsigned int gpio_range5_pins[] = {
 	147, 148, 149, 150, 151, 152, 153, 154,		/* PORT23x */
 	155, 156, 157, 143, 144, 145, 146, 158,		/* PORT24x */
 };
-static const int port_range6_muxvals[] = {
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT23x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT24x */
-};
-static const unsigned xirq_pins[] = {
-	149, 150, 151, 152, 153, 154, 155, 156,		/* XIRQ0-7 */
-	157, 143, 144, 145, 85, 146, 158, 84,		/* XIRQ8-15 */
-	141, 142, 148, 50, 51, 159, 160, 161,		/* XIRQ16-23 */
-};
-static const int xirq_muxvals[] = {
-	14, 14, 14, 14, 14, 14, 14, 14,			/* XIRQ0-7 */
-	14, 14, 14, 14, 13, 14, 14, 13,			/* XIRQ8-15 */
-	14, 14, 14, 14, 14, 14, 14, 14,			/* XIRQ16-23 */
-};
-static const unsigned xirq_alternatives_pins[] = {
-	94, 95, 96, 97, 98, 99, 100, 101,		/* XIRQ0-7 */
-	102, 103, 104, 105, 106, 107,			/* XIRQ8-11,13,14 */
-	108, 109, 110, 111, 112, 113, 114, 115,		/* XIRQ16-23 */
-	9, 10, 11, 12, 13, 14, 15, 16,			/* XIRQ4-11 */
-	17, 0, 1, 2, 3, 4, 5, 6, 7, 8,			/* XIRQ13,14,16-23 */
-	139, 140, 135, 147,				/* XIRQ17,18,21,22 */
-};
-static const int xirq_alternatives_muxvals[] = {
-	14, 14, 14, 14, 14, 14, 14, 14,			/* XIRQ0-7 */
-	14, 14, 14, 14, 14, 14,				/* XIRQ8-11,13,14 */
-	14, 14, 14, 14, 14, 14, 14, 14,			/* XIRQ16-23 */
-	14, 14, 14, 14, 14, 14, 14, 14,			/* XIRQ4-11 */
-	14, 14, 14, 14, 14, 14, 14, 14, 14, 14,		/* XIRQ13,14,16-23 */
-	14, 14, 14, 14,					/* XIRQ17,18,21,22 */
-};
 
 static const struct uniphier_pinctrl_group uniphier_ld11_groups[] = {
+	UNIPHIER_PINCTRL_GROUP(aout),
 	UNIPHIER_PINCTRL_GROUP(emmc),
 	UNIPHIER_PINCTRL_GROUP(emmc_dat8),
 	UNIPHIER_PINCTRL_GROUP(ether_rmii),
@@ -621,223 +565,15 @@ static const struct uniphier_pinctrl_group uniphier_ld11_groups[] = {
 	UNIPHIER_PINCTRL_GROUP(usb0),
 	UNIPHIER_PINCTRL_GROUP(usb1),
 	UNIPHIER_PINCTRL_GROUP(usb2),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(port_range0),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(port_range1),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(port_range2),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(port_range3),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(port_range4),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(port_range5),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(port_range6),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_IRQ(xirq),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_IRQ(xirq_alternatives),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port00, port_range0, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port01, port_range0, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port02, port_range0, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port03, port_range0, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port04, port_range0, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port05, port_range0, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port06, port_range0, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port07, port_range0, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port10, port_range0, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port11, port_range0, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port12, port_range0, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port13, port_range0, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port14, port_range0, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port15, port_range0, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port16, port_range0, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port17, port_range0, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port20, port_range0, 16),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port21, port_range0, 17),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port22, port_range0, 18),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port23, port_range0, 19),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port24, port_range0, 20),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port25, port_range0, 21),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port26, port_range0, 22),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port27, port_range0, 23),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port30, port_range0, 24),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port31, port_range0, 25),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port32, port_range0, 26),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port53, port_range1, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port54, port_range1, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port55, port_range1, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port56, port_range1, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port57, port_range1, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port60, port_range1, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port63, port_range2, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port64, port_range2, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port65, port_range2, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port66, port_range2, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port67, port_range2, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port70, port_range2, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port71, port_range2, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port72, port_range2, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port73, port_range2, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port74, port_range2, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port75, port_range2, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port76, port_range2, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port77, port_range2, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port80, port_range2, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port81, port_range2, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port82, port_range2, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port83, port_range2, 16),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port84, port_range2, 17),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port85, port_range2, 18),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port86, port_range2, 19),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port87, port_range2, 20),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port90, port_range2, 21),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port91, port_range2, 22),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port92, port_range2, 23),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port93, port_range2, 24),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port94, port_range2, 25),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port95, port_range2, 26),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port96, port_range2, 27),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port97, port_range2, 28),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port100, port_range2, 29),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port101, port_range2, 30),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port102, port_range2, 31),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port103, port_range2, 32),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port104, port_range2, 33),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port105, port_range2, 34),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port106, port_range2, 35),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port107, port_range2, 36),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port120, port_range3, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port121, port_range3, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port122, port_range3, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port123, port_range3, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port124, port_range3, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port125, port_range3, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port126, port_range3, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port127, port_range3, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port130, port_range3, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port131, port_range3, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port132, port_range3, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port133, port_range3, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port134, port_range3, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port135, port_range3, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port136, port_range3, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port137, port_range3, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port140, port_range3, 16),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port141, port_range3, 17),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port142, port_range3, 18),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port143, port_range3, 19),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port144, port_range3, 20),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port145, port_range3, 21),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port146, port_range3, 22),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port147, port_range3, 23),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port180, port_range4, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port181, port_range4, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port182, port_range4, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port183, port_range4, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port184, port_range4, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port185, port_range4, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port186, port_range4, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port187, port_range4, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port200, port_range5, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port201, port_range5, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port202, port_range5, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port203, port_range5, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port204, port_range5, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port205, port_range5, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port206, port_range5, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port207, port_range5, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port210, port_range5, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port211, port_range5, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port212, port_range5, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port213, port_range5, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port214, port_range5, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port215, port_range5, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port216, port_range5, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port217, port_range5, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port220, port_range5, 16),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port221, port_range5, 17),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port222, port_range5, 18),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port223, port_range5, 19),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port230, port_range6, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port231, port_range6, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port232, port_range6, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port233, port_range6, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port234, port_range6, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port235, port_range6, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port236, port_range6, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port237, port_range6, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port240, port_range6, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port241, port_range6, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port242, port_range6, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port243, port_range6, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port244, port_range6, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port245, port_range6, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port246, port_range6, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port247, port_range6, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq0, xirq, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq1, xirq, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq2, xirq, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq3, xirq, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq4, xirq, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq5, xirq, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq6, xirq, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq7, xirq, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq8, xirq, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq9, xirq, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq10, xirq, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq11, xirq, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq12, xirq, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq13, xirq, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq14, xirq, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq15, xirq, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq16, xirq, 16),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq17, xirq, 17),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq18, xirq, 18),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq19, xirq, 19),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq20, xirq, 20),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq21, xirq, 21),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq22, xirq, 22),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq23, xirq, 23),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq0b, xirq_alternatives, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq1b, xirq_alternatives, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq2b, xirq_alternatives, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq3b, xirq_alternatives, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq4b, xirq_alternatives, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq5b, xirq_alternatives, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq6b, xirq_alternatives, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq7b, xirq_alternatives, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq8b, xirq_alternatives, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq9b, xirq_alternatives, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq10b, xirq_alternatives, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq11b, xirq_alternatives, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq13b, xirq_alternatives, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq14b, xirq_alternatives, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq16b, xirq_alternatives, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq17b, xirq_alternatives, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq18b, xirq_alternatives, 16),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq19b, xirq_alternatives, 17),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq20b, xirq_alternatives, 18),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq21b, xirq_alternatives, 19),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq22b, xirq_alternatives, 20),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq23b, xirq_alternatives, 21),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq4c, xirq_alternatives, 22),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq5c, xirq_alternatives, 23),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq6c, xirq_alternatives, 24),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq7c, xirq_alternatives, 25),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq8c, xirq_alternatives, 26),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq9c, xirq_alternatives, 27),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq10c, xirq_alternatives, 28),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq11c, xirq_alternatives, 29),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq13c, xirq_alternatives, 30),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq14c, xirq_alternatives, 31),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq16c, xirq_alternatives, 32),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq17c, xirq_alternatives, 33),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq18c, xirq_alternatives, 34),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq19c, xirq_alternatives, 35),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq20c, xirq_alternatives, 36),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq21c, xirq_alternatives, 37),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq22c, xirq_alternatives, 38),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq23c, xirq_alternatives, 39),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq17d, xirq_alternatives, 40),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq18d, xirq_alternatives, 41),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq21d, xirq_alternatives, 42),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq22d, xirq_alternatives, 43),
+	UNIPHIER_PINCTRL_GROUP_GPIO(gpio_range0),
+	UNIPHIER_PINCTRL_GROUP_GPIO(gpio_range1),
+	UNIPHIER_PINCTRL_GROUP_GPIO(gpio_range2),
+	UNIPHIER_PINCTRL_GROUP_GPIO(gpio_range3),
+	UNIPHIER_PINCTRL_GROUP_GPIO(gpio_range4),
+	UNIPHIER_PINCTRL_GROUP_GPIO(gpio_range5),
 };
 
+static const char * const aout_groups[] = {"aout"};
 static const char * const emmc_groups[] = {"emmc", "emmc_dat8"};
 static const char * const ether_rmii_groups[] = {"ether_rmii"};
 static const char * const i2c0_groups[] = {"i2c0"};
@@ -854,70 +590,9 @@ static const char * const uart3_groups[] = {"uart3"};
 static const char * const usb0_groups[] = {"usb0"};
 static const char * const usb1_groups[] = {"usb1"};
 static const char * const usb2_groups[] = {"usb2"};
-static const char * const port_groups[] = {
-	"port00",  "port01",  "port02",  "port03",
-	"port04",  "port05",  "port06",  "port07",
-	"port10",  "port11",  "port12",  "port13",
-	"port14",  "port15",  "port16",  "port17",
-	"port20",  "port21",  "port22",  "port23",
-	"port24",  "port25",  "port26",  "port27",
-	"port30",  "port31",  "port32",
-	/* port33-52 missing */          "port53",
-	"port54",  "port55",  "port56",  "port57",
-	"port60", /* port61-62 missing*/ "port63",
-	"port64",  "port65",  "port66",  "port67",
-	"port70",  "port71",  "port72",  "port73",
-	"port74",  "port75",  "port76",  "port77",
-	"port80",  "port81",  "port82",  "port83",
-	"port84",  "port85",  "port86",  "port87",
-	"port90",  "port91",  "port92",  "port93",
-	"port94",  "port95",  "port96",  "port97",
-	"port100", "port101", "port102", "port103",
-	"port104", "port105", "port106", "port107",
-	/* port110-117 missing */
-	"port120", "port121", "port122", "port123",
-	"port124", "port125", "port126", "port127",
-	"port130", "port131", "port132", "port133",
-	"port134", "port135", "port136", "port137",
-	"port140", "port141", "port142", "port143",
-	"port144", "port145", "port146", "port147",
-	/* port150-177 missing */
-	"port180", "port181", "port182", "port183",
-	"port184", "port185", "port186", "port187",
-	/* port190-197 missing */
-	"port200", "port201", "port202", "port203",
-	"port204", "port205", "port206", "port207",
-	"port210", "port211", "port212", "port213",
-	"port214", "port215", "port216", "port217",
-	"port220", "port221", "port222", "port223",
-	/* port224-227 missing */
-	"port230", "port231", "port232", "port233",
-	"port234", "port235", "port236", "port237",
-	"port240", "port241", "port242", "port243",
-	"port244", "port245", "port246", "port247",
-};
-static const char * const xirq_groups[] = {
-	"xirq0",  "xirq1",  "xirq2",  "xirq3",
-	"xirq4",  "xirq5",  "xirq6",  "xirq7",
-	"xirq8",  "xirq9",  "xirq10", "xirq11",
-	"xirq12", "xirq13", "xirq14", "xirq15",
-	"xirq16", "xirq17", "xirq18", "xirq19",
-	"xirq20", "xirq21", "xirq22", "xirq23",
-	"xirq0b",  "xirq1b",  "xirq2b",  "xirq3b",
-	"xirq4b",  "xirq5b",  "xirq6b",  "xirq7b",
-	"xirq8b",  "xirq9b",  "xirq10b", "xirq11b",
-	/* none */ "xirq13b", "xirq14b", /* none */
-	"xirq16b", "xirq17b", "xirq18b", "xirq19b",
-	"xirq20b", "xirq21b", "xirq22b", "xirq23b",
-	"xirq4c",  "xirq5c",  "xirq6c",  "xirq7c",
-	"xirq8c",  "xirq9c",  "xirq10c", "xirq11c",
-	/* none */ "xirq13c", "xirq14c", /* none */
-	"xirq16c", "xirq17c", "xirq18c", "xirq19c",
-	"xirq20c", "xirq21c", "xirq22c", "xirq23c",
-	"xirq17d", "xirq18d", "xirq21d", "xirq22d",
-};
 
 static const struct uniphier_pinmux_function uniphier_ld11_functions[] = {
+	UNIPHIER_PINMUX_FUNCTION(aout),
 	UNIPHIER_PINMUX_FUNCTION(emmc),
 	UNIPHIER_PINMUX_FUNCTION(ether_rmii),
 	UNIPHIER_PINMUX_FUNCTION(i2c0),
@@ -933,10 +608,20 @@ static const struct uniphier_pinmux_function uniphier_ld11_functions[] = {
 	UNIPHIER_PINMUX_FUNCTION(usb0),
 	UNIPHIER_PINMUX_FUNCTION(usb1),
 	UNIPHIER_PINMUX_FUNCTION(usb2),
-	UNIPHIER_PINMUX_FUNCTION(port),
-	UNIPHIER_PINMUX_FUNCTION(xirq),
 };
 
+static int uniphier_ld11_get_gpio_muxval(unsigned int pin,
+					 unsigned int gpio_offset)
+{
+	if (gpio_offset == 132 || gpio_offset == 135)	/* XIRQ12, 15 */
+		return 13;
+
+	if (gpio_offset >= 120 && gpio_offset <= 143)	/* XIRQx */
+		return 14;
+
+	return 15;
+}
+
 static struct uniphier_pinctrl_socdata uniphier_ld11_pindata = {
 	.pins = uniphier_ld11_pins,
 	.npins = ARRAY_SIZE(uniphier_ld11_pins),
@@ -944,6 +629,7 @@ static struct uniphier_pinctrl_socdata uniphier_ld11_pindata = {
 	.groups_count = ARRAY_SIZE(uniphier_ld11_groups),
 	.functions = uniphier_ld11_functions,
 	.functions_count = ARRAY_SIZE(uniphier_ld11_functions),
+	.get_gpio_muxval = uniphier_ld11_get_gpio_muxval,
 	.caps = UNIPHIER_PINCTRL_CAPS_PERPIN_IECTRL,
 };
 
@@ -962,6 +648,7 @@ static struct platform_driver uniphier_ld11_pinctrl_driver = {
 	.driver = {
 		.name = "uniphier-ld11-pinctrl",
 		.of_match_table = uniphier_ld11_pinctrl_match,
+		.pm = &uniphier_pinctrl_pm_ops,
 	},
 };
 builtin_platform_driver(uniphier_ld11_pinctrl_driver);
diff --git a/drivers/pinctrl/uniphier/pinctrl-uniphier-ld20.c b/drivers/pinctrl/uniphier/pinctrl-uniphier-ld20.c
index 93006626028d..83341284dc44 100644
--- a/drivers/pinctrl/uniphier/pinctrl-uniphier-ld20.c
+++ b/drivers/pinctrl/uniphier/pinctrl-uniphier-ld20.c
@@ -21,536 +21,538 @@
 #include "pinctrl-uniphier.h"
 
 static const struct pinctrl_pin_desc uniphier_ld20_pins[] = {
-	UNIPHIER_PINCTRL_PIN(0, "XECS1", 0,
+	UNIPHIER_PINCTRL_PIN(0, "XECS1", UNIPHIER_PIN_IECTRL_EXIST,
 			     0, UNIPHIER_PIN_DRV_3BIT,
 			     0, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(1, "ERXW", 1,
+	UNIPHIER_PINCTRL_PIN(1, "ERXW", UNIPHIER_PIN_IECTRL_EXIST,
 			     1, UNIPHIER_PIN_DRV_3BIT,
 			     1, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(2, "XERWE1", 2,
+	UNIPHIER_PINCTRL_PIN(2, "XERWE1", UNIPHIER_PIN_IECTRL_EXIST,
 			     2, UNIPHIER_PIN_DRV_3BIT,
 			     2, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(3, "XNFWP", 3,
+	UNIPHIER_PINCTRL_PIN(3, "XNFWP", UNIPHIER_PIN_IECTRL_EXIST,
 			     3, UNIPHIER_PIN_DRV_3BIT,
 			     3, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(4, "XNFCE0", 4,
+	UNIPHIER_PINCTRL_PIN(4, "XNFCE0", UNIPHIER_PIN_IECTRL_EXIST,
 			     4, UNIPHIER_PIN_DRV_3BIT,
 			     4, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(5, "NFRYBY0", 5,
+	UNIPHIER_PINCTRL_PIN(5, "NFRYBY0", UNIPHIER_PIN_IECTRL_EXIST,
 			     5, UNIPHIER_PIN_DRV_3BIT,
 			     5, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(6, "XNFRE", 6,
+	UNIPHIER_PINCTRL_PIN(6, "XNFRE", UNIPHIER_PIN_IECTRL_EXIST,
 			     6, UNIPHIER_PIN_DRV_3BIT,
 			     6, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(7, "XNFWE", 7,
+	UNIPHIER_PINCTRL_PIN(7, "XNFWE", UNIPHIER_PIN_IECTRL_EXIST,
 			     7, UNIPHIER_PIN_DRV_3BIT,
 			     7, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(8, "NFALE", 8,
+	UNIPHIER_PINCTRL_PIN(8, "NFALE", UNIPHIER_PIN_IECTRL_EXIST,
 			     8, UNIPHIER_PIN_DRV_3BIT,
 			     8, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(9, "NFCLE", 9,
+	UNIPHIER_PINCTRL_PIN(9, "NFCLE", UNIPHIER_PIN_IECTRL_EXIST,
 			     9, UNIPHIER_PIN_DRV_3BIT,
 			     9, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(10, "NFD0", 10,
+	UNIPHIER_PINCTRL_PIN(10, "NFD0", UNIPHIER_PIN_IECTRL_EXIST,
 			     10, UNIPHIER_PIN_DRV_3BIT,
 			     10, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(11, "NFD1", 11,
+	UNIPHIER_PINCTRL_PIN(11, "NFD1", UNIPHIER_PIN_IECTRL_EXIST,
 			     11, UNIPHIER_PIN_DRV_3BIT,
 			     11, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(12, "NFD2", 12,
+	UNIPHIER_PINCTRL_PIN(12, "NFD2", UNIPHIER_PIN_IECTRL_EXIST,
 			     12, UNIPHIER_PIN_DRV_3BIT,
 			     12, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(13, "NFD3", 13,
+	UNIPHIER_PINCTRL_PIN(13, "NFD3", UNIPHIER_PIN_IECTRL_EXIST,
 			     13, UNIPHIER_PIN_DRV_3BIT,
 			     13, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(14, "NFD4", 14,
+	UNIPHIER_PINCTRL_PIN(14, "NFD4", UNIPHIER_PIN_IECTRL_EXIST,
 			     14, UNIPHIER_PIN_DRV_3BIT,
 			     14, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(15, "NFD5", 15,
+	UNIPHIER_PINCTRL_PIN(15, "NFD5", UNIPHIER_PIN_IECTRL_EXIST,
 			     15, UNIPHIER_PIN_DRV_3BIT,
 			     15, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(16, "NFD6", 16,
+	UNIPHIER_PINCTRL_PIN(16, "NFD6", UNIPHIER_PIN_IECTRL_EXIST,
 			     16, UNIPHIER_PIN_DRV_3BIT,
 			     16, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(17, "NFD7", 17,
+	UNIPHIER_PINCTRL_PIN(17, "NFD7", UNIPHIER_PIN_IECTRL_EXIST,
 			     17, UNIPHIER_PIN_DRV_3BIT,
 			     17, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(18, "XERST", 18,
+	UNIPHIER_PINCTRL_PIN(18, "XERST", UNIPHIER_PIN_IECTRL_EXIST,
 			     0, UNIPHIER_PIN_DRV_2BIT,
 			     18, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(19, "MMCCLK", 19,
+	UNIPHIER_PINCTRL_PIN(19, "MMCCLK", UNIPHIER_PIN_IECTRL_EXIST,
 			     1, UNIPHIER_PIN_DRV_2BIT,
 			     19, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(20, "MMCCMD", 20,
+	UNIPHIER_PINCTRL_PIN(20, "MMCCMD", UNIPHIER_PIN_IECTRL_EXIST,
 			     2, UNIPHIER_PIN_DRV_2BIT,
 			     20, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(21, "MMCDS", 21,
+	UNIPHIER_PINCTRL_PIN(21, "MMCDS", UNIPHIER_PIN_IECTRL_EXIST,
 			     3, UNIPHIER_PIN_DRV_2BIT,
 			     21, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(22, "MMCDAT0", 22,
+	UNIPHIER_PINCTRL_PIN(22, "MMCDAT0", UNIPHIER_PIN_IECTRL_EXIST,
 			     4, UNIPHIER_PIN_DRV_2BIT,
 			     22, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(23, "MMCDAT1", 23,
+	UNIPHIER_PINCTRL_PIN(23, "MMCDAT1", UNIPHIER_PIN_IECTRL_EXIST,
 			     5, UNIPHIER_PIN_DRV_2BIT,
 			     23, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(24, "MMCDAT2", 24,
+	UNIPHIER_PINCTRL_PIN(24, "MMCDAT2", UNIPHIER_PIN_IECTRL_EXIST,
 			     6, UNIPHIER_PIN_DRV_2BIT,
 			     24, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(25, "MMCDAT3", 25,
+	UNIPHIER_PINCTRL_PIN(25, "MMCDAT3", UNIPHIER_PIN_IECTRL_EXIST,
 			     7, UNIPHIER_PIN_DRV_2BIT,
 			     25, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(26, "MMCDAT4", 26,
+	UNIPHIER_PINCTRL_PIN(26, "MMCDAT4", UNIPHIER_PIN_IECTRL_EXIST,
 			     8, UNIPHIER_PIN_DRV_2BIT,
 			     26, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(27, "MMCDAT5", 27,
+	UNIPHIER_PINCTRL_PIN(27, "MMCDAT5", UNIPHIER_PIN_IECTRL_EXIST,
 			     9, UNIPHIER_PIN_DRV_2BIT,
 			     27, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(28, "MMCDAT6", 28,
+	UNIPHIER_PINCTRL_PIN(28, "MMCDAT6", UNIPHIER_PIN_IECTRL_EXIST,
 			     10, UNIPHIER_PIN_DRV_2BIT,
 			     28, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(29, "MMCDAT7", 29,
+	UNIPHIER_PINCTRL_PIN(29, "MMCDAT7", UNIPHIER_PIN_IECTRL_EXIST,
 			     11, UNIPHIER_PIN_DRV_2BIT,
 			     29, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(30, "MDC", 30,
+	UNIPHIER_PINCTRL_PIN(30, "MDC", UNIPHIER_PIN_IECTRL_EXIST,
 			     18, UNIPHIER_PIN_DRV_3BIT,
 			     30, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(31, "MDIO", 31,
+	UNIPHIER_PINCTRL_PIN(31, "MDIO", UNIPHIER_PIN_IECTRL_EXIST,
 			     19, UNIPHIER_PIN_DRV_3BIT,
 			     31, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(32, "MDIO_INTL", 32,
+	UNIPHIER_PINCTRL_PIN(32, "MDIO_INTL", UNIPHIER_PIN_IECTRL_EXIST,
 			     20, UNIPHIER_PIN_DRV_3BIT,
 			     32, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(33, "PHYRSTL", 33,
+	UNIPHIER_PINCTRL_PIN(33, "PHYRSTL", UNIPHIER_PIN_IECTRL_EXIST,
 			     21, UNIPHIER_PIN_DRV_3BIT,
 			     33, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(34, "RGMII_RXCLK", 34,
+	UNIPHIER_PINCTRL_PIN(34, "RGMII_RXCLK", UNIPHIER_PIN_IECTRL_EXIST,
 			     22, UNIPHIER_PIN_DRV_3BIT,
 			     34, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(35, "RGMII_RXD0", 35,
+	UNIPHIER_PINCTRL_PIN(35, "RGMII_RXD0", UNIPHIER_PIN_IECTRL_EXIST,
 			     23, UNIPHIER_PIN_DRV_3BIT,
 			     35, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(36, "RGMII_RXD1", 36,
+	UNIPHIER_PINCTRL_PIN(36, "RGMII_RXD1", UNIPHIER_PIN_IECTRL_EXIST,
 			     24, UNIPHIER_PIN_DRV_3BIT,
 			     36, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(37, "RGMII_RXD2", 37,
+	UNIPHIER_PINCTRL_PIN(37, "RGMII_RXD2", UNIPHIER_PIN_IECTRL_EXIST,
 			     25, UNIPHIER_PIN_DRV_3BIT,
 			     37, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(38, "RGMII_RXD3", 38,
+	UNIPHIER_PINCTRL_PIN(38, "RGMII_RXD3", UNIPHIER_PIN_IECTRL_EXIST,
 			     26, UNIPHIER_PIN_DRV_3BIT,
 			     38, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(39, "RGMII_RXCTL", 39,
+	UNIPHIER_PINCTRL_PIN(39, "RGMII_RXCTL", UNIPHIER_PIN_IECTRL_EXIST,
 			     27, UNIPHIER_PIN_DRV_3BIT,
 			     39, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(40, "RGMII_TXCLK", 40,
+	UNIPHIER_PINCTRL_PIN(40, "RGMII_TXCLK", UNIPHIER_PIN_IECTRL_EXIST,
 			     28, UNIPHIER_PIN_DRV_3BIT,
 			     40, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(41, "RGMII_TXD0", 41,
+	UNIPHIER_PINCTRL_PIN(41, "RGMII_TXD0", UNIPHIER_PIN_IECTRL_EXIST,
 			     29, UNIPHIER_PIN_DRV_3BIT,
 			     41, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(42, "RGMII_TXD1", 42,
+	UNIPHIER_PINCTRL_PIN(42, "RGMII_TXD1", UNIPHIER_PIN_IECTRL_EXIST,
 			     30, UNIPHIER_PIN_DRV_3BIT,
 			     42, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(43, "RGMII_TXD2", 43,
+	UNIPHIER_PINCTRL_PIN(43, "RGMII_TXD2", UNIPHIER_PIN_IECTRL_EXIST,
 			     31, UNIPHIER_PIN_DRV_3BIT,
 			     43, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(44, "RGMII_TXD3", 44,
+	UNIPHIER_PINCTRL_PIN(44, "RGMII_TXD3", UNIPHIER_PIN_IECTRL_EXIST,
 			     32, UNIPHIER_PIN_DRV_3BIT,
 			     44, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(45, "RGMII_TXCTL", 45,
+	UNIPHIER_PINCTRL_PIN(45, "RGMII_TXCTL", UNIPHIER_PIN_IECTRL_EXIST,
 			     33, UNIPHIER_PIN_DRV_3BIT,
 			     45, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(46, "USB0VBUS", 46,
+	UNIPHIER_PINCTRL_PIN(46, "USB0VBUS", UNIPHIER_PIN_IECTRL_EXIST,
 			     34, UNIPHIER_PIN_DRV_3BIT,
 			     46, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(47, "USB0OD", 47,
+	UNIPHIER_PINCTRL_PIN(47, "USB0OD", UNIPHIER_PIN_IECTRL_EXIST,
 			     35, UNIPHIER_PIN_DRV_3BIT,
 			     47, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(48, "USB1VBUS", 48,
+	UNIPHIER_PINCTRL_PIN(48, "USB1VBUS", UNIPHIER_PIN_IECTRL_EXIST,
 			     36, UNIPHIER_PIN_DRV_3BIT,
 			     48, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(49, "USB1OD", 49,
+	UNIPHIER_PINCTRL_PIN(49, "USB1OD", UNIPHIER_PIN_IECTRL_EXIST,
 			     37, UNIPHIER_PIN_DRV_3BIT,
 			     49, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(50, "USB2VBUS", 50,
+	UNIPHIER_PINCTRL_PIN(50, "USB2VBUS", UNIPHIER_PIN_IECTRL_EXIST,
 			     38, UNIPHIER_PIN_DRV_3BIT,
 			     50, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(51, "USB2OD", 51,
+	UNIPHIER_PINCTRL_PIN(51, "USB2OD", UNIPHIER_PIN_IECTRL_EXIST,
 			     39, UNIPHIER_PIN_DRV_3BIT,
 			     51, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(52, "USB3VBUS", 52,
+	UNIPHIER_PINCTRL_PIN(52, "USB3VBUS", UNIPHIER_PIN_IECTRL_EXIST,
 			     40, UNIPHIER_PIN_DRV_3BIT,
 			     52, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(53, "USB3OD", 53,
+	UNIPHIER_PINCTRL_PIN(53, "USB3OD", UNIPHIER_PIN_IECTRL_EXIST,
 			     41, UNIPHIER_PIN_DRV_3BIT,
 			     53, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(54, "TXD0", 54,
+	UNIPHIER_PINCTRL_PIN(54, "TXD0", UNIPHIER_PIN_IECTRL_EXIST,
 			     42, UNIPHIER_PIN_DRV_3BIT,
 			     54, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(55, "RXD0", 55,
+	UNIPHIER_PINCTRL_PIN(55, "RXD0", UNIPHIER_PIN_IECTRL_EXIST,
 			     43, UNIPHIER_PIN_DRV_3BIT,
 			     55, UNIPHIER_PIN_PULL_UP),
-	UNIPHIER_PINCTRL_PIN(56, "SPISYNC0", 56,
+	UNIPHIER_PINCTRL_PIN(56, "SPISYNC0", UNIPHIER_PIN_IECTRL_EXIST,
 			     44, UNIPHIER_PIN_DRV_3BIT,
 			     56, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(57, "SPISCLK0", 57,
+	UNIPHIER_PINCTRL_PIN(57, "SPISCLK0", UNIPHIER_PIN_IECTRL_EXIST,
 			     45, UNIPHIER_PIN_DRV_3BIT,
 			     57, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(58, "SPITXD0", 58,
+	UNIPHIER_PINCTRL_PIN(58, "SPITXD0", UNIPHIER_PIN_IECTRL_EXIST,
 			     46, UNIPHIER_PIN_DRV_3BIT,
 			     58, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(59, "SPIRXD0", 59,
+	UNIPHIER_PINCTRL_PIN(59, "SPIRXD0", UNIPHIER_PIN_IECTRL_EXIST,
 			     47, UNIPHIER_PIN_DRV_3BIT,
 			     59, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(60, "AGCI", 60,
+	UNIPHIER_PINCTRL_PIN(60, "AGCI", UNIPHIER_PIN_IECTRL_EXIST,
 			     48, UNIPHIER_PIN_DRV_3BIT,
 			     60, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(61, "DMDSDA0", 61,
+	UNIPHIER_PINCTRL_PIN(61, "DMDSDA0", UNIPHIER_PIN_IECTRL_EXIST,
 			     -1, UNIPHIER_PIN_DRV_FIXED4,
 			     -1, UNIPHIER_PIN_PULL_NONE),
-	UNIPHIER_PINCTRL_PIN(62, "DMDSCL0", 62,
+	UNIPHIER_PINCTRL_PIN(62, "DMDSCL0", UNIPHIER_PIN_IECTRL_EXIST,
 			     -1, UNIPHIER_PIN_DRV_FIXED4,
 			     -1, UNIPHIER_PIN_PULL_NONE),
-	UNIPHIER_PINCTRL_PIN(63, "SDA0", 63,
+	UNIPHIER_PINCTRL_PIN(63, "SDA0", UNIPHIER_PIN_IECTRL_EXIST,
 			     -1, UNIPHIER_PIN_DRV_FIXED4,
 			     -1, UNIPHIER_PIN_PULL_NONE),
-	UNIPHIER_PINCTRL_PIN(64, "SCL0", 64,
+	UNIPHIER_PINCTRL_PIN(64, "SCL0", UNIPHIER_PIN_IECTRL_EXIST,
 			     -1, UNIPHIER_PIN_DRV_FIXED4,
 			     -1, UNIPHIER_PIN_PULL_NONE),
-	UNIPHIER_PINCTRL_PIN(65, "SDA1", 65,
+	UNIPHIER_PINCTRL_PIN(65, "SDA1", UNIPHIER_PIN_IECTRL_EXIST,
 			     -1, UNIPHIER_PIN_DRV_FIXED4,
 			     -1, UNIPHIER_PIN_PULL_NONE),
-	UNIPHIER_PINCTRL_PIN(66, "SCL1", 66,
+	UNIPHIER_PINCTRL_PIN(66, "SCL1", UNIPHIER_PIN_IECTRL_EXIST,
 			     -1, UNIPHIER_PIN_DRV_FIXED4,
 			     -1, UNIPHIER_PIN_PULL_NONE),
-	UNIPHIER_PINCTRL_PIN(67, "HIN", 67,
+	UNIPHIER_PINCTRL_PIN(67, "HIN", UNIPHIER_PIN_IECTRL_EXIST,
 			     -1, UNIPHIER_PIN_DRV_FIXED4,
 			     -1, UNIPHIER_PIN_PULL_NONE),
-	UNIPHIER_PINCTRL_PIN(68, "VIN", 68,
+	UNIPHIER_PINCTRL_PIN(68, "VIN", UNIPHIER_PIN_IECTRL_EXIST,
 			     -1, UNIPHIER_PIN_DRV_FIXED4,
 			     -1, UNIPHIER_PIN_PULL_NONE),
-	UNIPHIER_PINCTRL_PIN(69, "PCA00", 69,
+	UNIPHIER_PINCTRL_PIN(69, "PCA00", UNIPHIER_PIN_IECTRL_EXIST,
 			     49, UNIPHIER_PIN_DRV_3BIT,
 			     69, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(70, "PCA01", 70,
+	UNIPHIER_PINCTRL_PIN(70, "PCA01", UNIPHIER_PIN_IECTRL_EXIST,
 			     50, UNIPHIER_PIN_DRV_3BIT,
 			     70, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(71, "PCA02", 71,
+	UNIPHIER_PINCTRL_PIN(71, "PCA02", UNIPHIER_PIN_IECTRL_EXIST,
 			     51, UNIPHIER_PIN_DRV_3BIT,
 			     71, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(72, "PCA03", 72,
+	UNIPHIER_PINCTRL_PIN(72, "PCA03", UNIPHIER_PIN_IECTRL_EXIST,
 			     52, UNIPHIER_PIN_DRV_3BIT,
 			     72, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(73, "PCA04", 73,
+	UNIPHIER_PINCTRL_PIN(73, "PCA04", UNIPHIER_PIN_IECTRL_EXIST,
 			     53, UNIPHIER_PIN_DRV_3BIT,
 			     73, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(74, "PCA05", 74,
+	UNIPHIER_PINCTRL_PIN(74, "PCA05", UNIPHIER_PIN_IECTRL_EXIST,
 			     54, UNIPHIER_PIN_DRV_3BIT,
 			     74, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(75, "PCA06", 75,
+	UNIPHIER_PINCTRL_PIN(75, "PCA06", UNIPHIER_PIN_IECTRL_EXIST,
 			     55, UNIPHIER_PIN_DRV_3BIT,
 			     75, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(76, "PCA07", 76,
+	UNIPHIER_PINCTRL_PIN(76, "PCA07", UNIPHIER_PIN_IECTRL_EXIST,
 			     56, UNIPHIER_PIN_DRV_3BIT,
 			     76, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(77, "PCA08", 77,
+	UNIPHIER_PINCTRL_PIN(77, "PCA08", UNIPHIER_PIN_IECTRL_EXIST,
 			     57, UNIPHIER_PIN_DRV_3BIT,
 			     77, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(78, "PCA09", 78,
+	UNIPHIER_PINCTRL_PIN(78, "PCA09", UNIPHIER_PIN_IECTRL_EXIST,
 			     58, UNIPHIER_PIN_DRV_3BIT,
 			     78, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(79, "PCA10", 79,
+	UNIPHIER_PINCTRL_PIN(79, "PCA10", UNIPHIER_PIN_IECTRL_EXIST,
 			     59, UNIPHIER_PIN_DRV_3BIT,
 			     79, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(80, "PCA11", 80,
+	UNIPHIER_PINCTRL_PIN(80, "PCA11", UNIPHIER_PIN_IECTRL_EXIST,
 			     60, UNIPHIER_PIN_DRV_3BIT,
 			     80, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(81, "PCA12", 81,
+	UNIPHIER_PINCTRL_PIN(81, "PCA12", UNIPHIER_PIN_IECTRL_EXIST,
 			     61, UNIPHIER_PIN_DRV_3BIT,
 			     81, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(82, "PCA13", 82,
+	UNIPHIER_PINCTRL_PIN(82, "PCA13", UNIPHIER_PIN_IECTRL_EXIST,
 			     62, UNIPHIER_PIN_DRV_3BIT,
 			     82, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(83, "PCA14", 83,
+	UNIPHIER_PINCTRL_PIN(83, "PCA14", UNIPHIER_PIN_IECTRL_EXIST,
 			     63, UNIPHIER_PIN_DRV_3BIT,
 			     83, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(84, "PC0READY", 84,
+	UNIPHIER_PINCTRL_PIN(84, "PC0READY", UNIPHIER_PIN_IECTRL_EXIST,
 			     0, UNIPHIER_PIN_DRV_1BIT,
 			     84, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(85, "PC0CD1", 85,
+	UNIPHIER_PINCTRL_PIN(85, "PC0CD1", UNIPHIER_PIN_IECTRL_EXIST,
 			     1, UNIPHIER_PIN_DRV_1BIT,
 			     85, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(86, "PC0CD2", 86,
+	UNIPHIER_PINCTRL_PIN(86, "PC0CD2", UNIPHIER_PIN_IECTRL_EXIST,
 			     2, UNIPHIER_PIN_DRV_1BIT,
 			     86, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(87, "PC0WAIT", 87,
+	UNIPHIER_PINCTRL_PIN(87, "PC0WAIT", UNIPHIER_PIN_IECTRL_EXIST,
 			     3, UNIPHIER_PIN_DRV_1BIT,
 			     87, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(88, "PC0RESET", 88,
+	UNIPHIER_PINCTRL_PIN(88, "PC0RESET", UNIPHIER_PIN_IECTRL_EXIST,
 			     4, UNIPHIER_PIN_DRV_1BIT,
 			     88, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(89, "PC0CE1", 89,
+	UNIPHIER_PINCTRL_PIN(89, "PC0CE1", UNIPHIER_PIN_IECTRL_EXIST,
 			     5, UNIPHIER_PIN_DRV_1BIT,
 			     89, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(90, "PC0WE", 90,
+	UNIPHIER_PINCTRL_PIN(90, "PC0WE", UNIPHIER_PIN_IECTRL_EXIST,
 			     6, UNIPHIER_PIN_DRV_1BIT,
 			     90, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(91, "PC0OE", 91,
+	UNIPHIER_PINCTRL_PIN(91, "PC0OE", UNIPHIER_PIN_IECTRL_EXIST,
 			     7, UNIPHIER_PIN_DRV_1BIT,
 			     91, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(92, "PC0IOWR", 92,
+	UNIPHIER_PINCTRL_PIN(92, "PC0IOWR", UNIPHIER_PIN_IECTRL_EXIST,
 			     8, UNIPHIER_PIN_DRV_1BIT,
 			     92, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(93, "PC0IORD", 93,
+	UNIPHIER_PINCTRL_PIN(93, "PC0IORD", UNIPHIER_PIN_IECTRL_EXIST,
 			     9, UNIPHIER_PIN_DRV_1BIT,
 			     93, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(94, "PCD00", 94,
+	UNIPHIER_PINCTRL_PIN(94, "PCD00", UNIPHIER_PIN_IECTRL_EXIST,
 			     10, UNIPHIER_PIN_DRV_1BIT,
 			     94, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(95, "PCD01", 95,
+	UNIPHIER_PINCTRL_PIN(95, "PCD01", UNIPHIER_PIN_IECTRL_EXIST,
 			     11, UNIPHIER_PIN_DRV_1BIT,
 			     95, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(96, "PCD02", 96,
+	UNIPHIER_PINCTRL_PIN(96, "PCD02", UNIPHIER_PIN_IECTRL_EXIST,
 			     12, UNIPHIER_PIN_DRV_1BIT,
 			     96, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(97, "PCD03", 97,
+	UNIPHIER_PINCTRL_PIN(97, "PCD03", UNIPHIER_PIN_IECTRL_EXIST,
 			     13, UNIPHIER_PIN_DRV_1BIT,
 			     97, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(98, "PCD04", 98,
+	UNIPHIER_PINCTRL_PIN(98, "PCD04", UNIPHIER_PIN_IECTRL_EXIST,
 			     14, UNIPHIER_PIN_DRV_1BIT,
 			     98, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(99, "PCD05", 99,
+	UNIPHIER_PINCTRL_PIN(99, "PCD05", UNIPHIER_PIN_IECTRL_EXIST,
 			     15, UNIPHIER_PIN_DRV_1BIT,
 			     99, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(100, "PCD06", 100,
+	UNIPHIER_PINCTRL_PIN(100, "PCD06", UNIPHIER_PIN_IECTRL_EXIST,
 			     16, UNIPHIER_PIN_DRV_1BIT,
 			     100, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(101, "PCD07", 101,
+	UNIPHIER_PINCTRL_PIN(101, "PCD07", UNIPHIER_PIN_IECTRL_EXIST,
 			     17, UNIPHIER_PIN_DRV_1BIT,
 			     101, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(102, "HS0BCLKIN", 102,
+	UNIPHIER_PINCTRL_PIN(102, "HS0BCLKIN", UNIPHIER_PIN_IECTRL_EXIST,
 			     18, UNIPHIER_PIN_DRV_1BIT,
 			     102, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(103, "HS0SYNCIN", 103,
+	UNIPHIER_PINCTRL_PIN(103, "HS0SYNCIN", UNIPHIER_PIN_IECTRL_EXIST,
 			     19, UNIPHIER_PIN_DRV_1BIT,
 			     103, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(104, "HS0VALIN", 104,
+	UNIPHIER_PINCTRL_PIN(104, "HS0VALIN", UNIPHIER_PIN_IECTRL_EXIST,
 			     20, UNIPHIER_PIN_DRV_1BIT,
 			     104, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(105, "HS0DIN0", 105,
+	UNIPHIER_PINCTRL_PIN(105, "HS0DIN0", UNIPHIER_PIN_IECTRL_EXIST,
 			     21, UNIPHIER_PIN_DRV_1BIT,
 			     105, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(106, "HS0DIN1", 106,
+	UNIPHIER_PINCTRL_PIN(106, "HS0DIN1", UNIPHIER_PIN_IECTRL_EXIST,
 			     22, UNIPHIER_PIN_DRV_1BIT,
 			     106, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(107, "HS0DIN2", 107,
+	UNIPHIER_PINCTRL_PIN(107, "HS0DIN2", UNIPHIER_PIN_IECTRL_EXIST,
 			     23, UNIPHIER_PIN_DRV_1BIT,
 			     107, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(108, "HS0DIN3", 108,
+	UNIPHIER_PINCTRL_PIN(108, "HS0DIN3", UNIPHIER_PIN_IECTRL_EXIST,
 			     24, UNIPHIER_PIN_DRV_1BIT,
 			     108, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(109, "HS0DIN4", 109,
+	UNIPHIER_PINCTRL_PIN(109, "HS0DIN4", UNIPHIER_PIN_IECTRL_EXIST,
 			     25, UNIPHIER_PIN_DRV_1BIT,
 			     109, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(110, "HS0DIN5", 110,
+	UNIPHIER_PINCTRL_PIN(110, "HS0DIN5", UNIPHIER_PIN_IECTRL_EXIST,
 			     26, UNIPHIER_PIN_DRV_1BIT,
 			     110, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(111, "HS0DIN6", 111,
+	UNIPHIER_PINCTRL_PIN(111, "HS0DIN6", UNIPHIER_PIN_IECTRL_EXIST,
 			     27, UNIPHIER_PIN_DRV_1BIT,
 			     111, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(112, "HS0DIN7", 112,
+	UNIPHIER_PINCTRL_PIN(112, "HS0DIN7", UNIPHIER_PIN_IECTRL_EXIST,
 			     28, UNIPHIER_PIN_DRV_1BIT,
 			     112, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(113, "HS0BCLKOUT", 113,
+	UNIPHIER_PINCTRL_PIN(113, "HS0BCLKOUT", UNIPHIER_PIN_IECTRL_EXIST,
 			     64, UNIPHIER_PIN_DRV_3BIT,
 			     113, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(114, "HS0SYNCOUT", 114,
+	UNIPHIER_PINCTRL_PIN(114, "HS0SYNCOUT", UNIPHIER_PIN_IECTRL_EXIST,
 			     65, UNIPHIER_PIN_DRV_3BIT,
 			     114, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(115, "HS0VALOUT", 115,
+	UNIPHIER_PINCTRL_PIN(115, "HS0VALOUT", UNIPHIER_PIN_IECTRL_EXIST,
 			     66, UNIPHIER_PIN_DRV_3BIT,
 			     115, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(116, "HS0DOUT0", 116,
+	UNIPHIER_PINCTRL_PIN(116, "HS0DOUT0", UNIPHIER_PIN_IECTRL_EXIST,
 			     67, UNIPHIER_PIN_DRV_3BIT,
 			     116, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(117, "HS0DOUT1", 117,
+	UNIPHIER_PINCTRL_PIN(117, "HS0DOUT1", UNIPHIER_PIN_IECTRL_EXIST,
 			     68, UNIPHIER_PIN_DRV_3BIT,
 			     117, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(118, "HS0DOUT2", 118,
+	UNIPHIER_PINCTRL_PIN(118, "HS0DOUT2", UNIPHIER_PIN_IECTRL_EXIST,
 			     69, UNIPHIER_PIN_DRV_3BIT,
 			     118, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(119, "HS0DOUT3", 119,
+	UNIPHIER_PINCTRL_PIN(119, "HS0DOUT3", UNIPHIER_PIN_IECTRL_EXIST,
 			     70, UNIPHIER_PIN_DRV_3BIT,
 			     119, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(120, "HS0DOUT4", 120,
+	UNIPHIER_PINCTRL_PIN(120, "HS0DOUT4", UNIPHIER_PIN_IECTRL_EXIST,
 			     71, UNIPHIER_PIN_DRV_3BIT,
 			     120, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(121, "HS0DOUT5", 121,
+	UNIPHIER_PINCTRL_PIN(121, "HS0DOUT5", UNIPHIER_PIN_IECTRL_EXIST,
 			     72, UNIPHIER_PIN_DRV_3BIT,
 			     121, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(122, "HS0DOUT6", 122,
+	UNIPHIER_PINCTRL_PIN(122, "HS0DOUT6", UNIPHIER_PIN_IECTRL_EXIST,
 			     73, UNIPHIER_PIN_DRV_3BIT,
 			     122, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(123, "HS0DOUT7", 123,
+	UNIPHIER_PINCTRL_PIN(123, "HS0DOUT7", UNIPHIER_PIN_IECTRL_EXIST,
 			     74, UNIPHIER_PIN_DRV_3BIT,
 			     123, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(124, "HS1BCLKIN", 124,
+	UNIPHIER_PINCTRL_PIN(124, "HS1BCLKIN", UNIPHIER_PIN_IECTRL_EXIST,
 			     75, UNIPHIER_PIN_DRV_3BIT,
 			     124, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(125, "HS1SYNCIN", 125,
+	UNIPHIER_PINCTRL_PIN(125, "HS1SYNCIN", UNIPHIER_PIN_IECTRL_EXIST,
 			     76, UNIPHIER_PIN_DRV_3BIT,
 			     125, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(126, "HS1VALIN", 126,
+	UNIPHIER_PINCTRL_PIN(126, "HS1VALIN", UNIPHIER_PIN_IECTRL_EXIST,
 			     77, UNIPHIER_PIN_DRV_3BIT,
 			     126, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(127, "HS1DIN0", 127,
+	UNIPHIER_PINCTRL_PIN(127, "HS1DIN0", UNIPHIER_PIN_IECTRL_EXIST,
 			     78, UNIPHIER_PIN_DRV_3BIT,
 			     127, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(128, "HS1DIN1", 128,
+	UNIPHIER_PINCTRL_PIN(128, "HS1DIN1", UNIPHIER_PIN_IECTRL_EXIST,
 			     79, UNIPHIER_PIN_DRV_3BIT,
 			     128, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(129, "HS1DIN2", 129,
+	UNIPHIER_PINCTRL_PIN(129, "HS1DIN2", UNIPHIER_PIN_IECTRL_EXIST,
 			     80, UNIPHIER_PIN_DRV_3BIT,
 			     129, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(130, "HS1DIN3", 130,
+	UNIPHIER_PINCTRL_PIN(130, "HS1DIN3", UNIPHIER_PIN_IECTRL_EXIST,
 			     81, UNIPHIER_PIN_DRV_3BIT,
 			     130, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(131, "HS1DIN4", 131,
+	UNIPHIER_PINCTRL_PIN(131, "HS1DIN4", UNIPHIER_PIN_IECTRL_EXIST,
 			     82, UNIPHIER_PIN_DRV_3BIT,
 			     131, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(132, "HS1DIN5", 132,
+	UNIPHIER_PINCTRL_PIN(132, "HS1DIN5", UNIPHIER_PIN_IECTRL_EXIST,
 			     83, UNIPHIER_PIN_DRV_3BIT,
 			     132, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(133, "HS1DIN6", 133,
+	UNIPHIER_PINCTRL_PIN(133, "HS1DIN6", UNIPHIER_PIN_IECTRL_EXIST,
 			     84, UNIPHIER_PIN_DRV_3BIT,
 			     133, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(134, "HS1DIN7", 134,
+	UNIPHIER_PINCTRL_PIN(134, "HS1DIN7", UNIPHIER_PIN_IECTRL_EXIST,
 			     85, UNIPHIER_PIN_DRV_3BIT,
 			     134, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(135, "AO1IEC", 135,
+	UNIPHIER_PINCTRL_PIN(135, "AO1IEC", UNIPHIER_PIN_IECTRL_EXIST,
 			     86, UNIPHIER_PIN_DRV_3BIT,
 			     135, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(136, "AO1ARC", 136,
+	UNIPHIER_PINCTRL_PIN(136, "AO1ARC", UNIPHIER_PIN_IECTRL_EXIST,
 			     87, UNIPHIER_PIN_DRV_3BIT,
 			     136, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(137, "AO1DACCK", 137,
+	UNIPHIER_PINCTRL_PIN(137, "AO1DACCK", UNIPHIER_PIN_IECTRL_EXIST,
 			     88, UNIPHIER_PIN_DRV_3BIT,
 			     137, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(138, "AO1BCK", 138,
+	UNIPHIER_PINCTRL_PIN(138, "AO1BCK", UNIPHIER_PIN_IECTRL_EXIST,
 			     89, UNIPHIER_PIN_DRV_3BIT,
 			     138, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(139, "AO1LRCK", 139,
+	UNIPHIER_PINCTRL_PIN(139, "AO1LRCK", UNIPHIER_PIN_IECTRL_EXIST,
 			     90, UNIPHIER_PIN_DRV_3BIT,
 			     139, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(140, "AO1D0", 140,
+	UNIPHIER_PINCTRL_PIN(140, "AO1D0", UNIPHIER_PIN_IECTRL_EXIST,
 			     91, UNIPHIER_PIN_DRV_3BIT,
 			     140, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(141, "AO1D1", 141,
+	UNIPHIER_PINCTRL_PIN(141, "AO1D1", UNIPHIER_PIN_IECTRL_EXIST,
 			     92, UNIPHIER_PIN_DRV_3BIT,
 			     141, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(142, "AO1D2", 142,
+	UNIPHIER_PINCTRL_PIN(142, "AO1D2", UNIPHIER_PIN_IECTRL_EXIST,
 			     93, UNIPHIER_PIN_DRV_3BIT,
 			     142, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(143, "HTPDN0", 143,
+	UNIPHIER_PINCTRL_PIN(143, "HTPDN0", UNIPHIER_PIN_IECTRL_EXIST,
 			     94, UNIPHIER_PIN_DRV_3BIT,
 			     143, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(144, "LOCKN0", 144,
+	UNIPHIER_PINCTRL_PIN(144, "LOCKN0", UNIPHIER_PIN_IECTRL_EXIST,
 			     95, UNIPHIER_PIN_DRV_3BIT,
 			     144, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(145, "HTPDN1", 145,
+	UNIPHIER_PINCTRL_PIN(145, "HTPDN1", UNIPHIER_PIN_IECTRL_EXIST,
 			     96, UNIPHIER_PIN_DRV_3BIT,
 			     145, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(146, "LOCKN1", 146,
+	UNIPHIER_PINCTRL_PIN(146, "LOCKN1", UNIPHIER_PIN_IECTRL_EXIST,
 			     97, UNIPHIER_PIN_DRV_3BIT,
 			     146, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(147, "PWMA", 147,
+	UNIPHIER_PINCTRL_PIN(147, "PWMA", UNIPHIER_PIN_IECTRL_EXIST,
 			     98, UNIPHIER_PIN_DRV_3BIT,
 			     147, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(148, "LR_GOUT", 148,
+	UNIPHIER_PINCTRL_PIN(148, "LR_GOUT", UNIPHIER_PIN_IECTRL_EXIST,
 			     99, UNIPHIER_PIN_DRV_3BIT,
 			     148, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(149, "XIRQ0", 149,
+	UNIPHIER_PINCTRL_PIN(149, "XIRQ0", UNIPHIER_PIN_IECTRL_EXIST,
 			     100, UNIPHIER_PIN_DRV_3BIT,
 			     149, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(150, "XIRQ1", 150,
+	UNIPHIER_PINCTRL_PIN(150, "XIRQ1", UNIPHIER_PIN_IECTRL_EXIST,
 			     101, UNIPHIER_PIN_DRV_3BIT,
 			     150, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(151, "XIRQ2", 151,
+	UNIPHIER_PINCTRL_PIN(151, "XIRQ2", UNIPHIER_PIN_IECTRL_EXIST,
 			     102, UNIPHIER_PIN_DRV_3BIT,
 			     151, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(152, "XIRQ3", 152,
+	UNIPHIER_PINCTRL_PIN(152, "XIRQ3", UNIPHIER_PIN_IECTRL_EXIST,
 			     103, UNIPHIER_PIN_DRV_3BIT,
 			     152, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(153, "XIRQ4", 153,
+	UNIPHIER_PINCTRL_PIN(153, "XIRQ4", UNIPHIER_PIN_IECTRL_EXIST,
 			     104, UNIPHIER_PIN_DRV_3BIT,
 			     153, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(154, "XIRQ5", 154,
+	UNIPHIER_PINCTRL_PIN(154, "XIRQ5", UNIPHIER_PIN_IECTRL_EXIST,
 			     105, UNIPHIER_PIN_DRV_3BIT,
 			     154, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(155, "XIRQ6", 155,
+	UNIPHIER_PINCTRL_PIN(155, "XIRQ6", UNIPHIER_PIN_IECTRL_EXIST,
 			     106, UNIPHIER_PIN_DRV_3BIT,
 			     155, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(156, "XIRQ7", 156,
+	UNIPHIER_PINCTRL_PIN(156, "XIRQ7", UNIPHIER_PIN_IECTRL_EXIST,
 			     107, UNIPHIER_PIN_DRV_3BIT,
 			     156, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(157, "XIRQ8", 157,
+	UNIPHIER_PINCTRL_PIN(157, "XIRQ8", UNIPHIER_PIN_IECTRL_EXIST,
 			     108, UNIPHIER_PIN_DRV_3BIT,
 			     157, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(158, "XIRQ9", 158,
+	UNIPHIER_PINCTRL_PIN(158, "XIRQ9", UNIPHIER_PIN_IECTRL_EXIST,
 			     109, UNIPHIER_PIN_DRV_3BIT,
 			     158, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(159, "XIRQ10", 159,
+	UNIPHIER_PINCTRL_PIN(159, "XIRQ10", UNIPHIER_PIN_IECTRL_EXIST,
 			     110, UNIPHIER_PIN_DRV_3BIT,
 			     159, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(160, "XIRQ11", 160,
+	UNIPHIER_PINCTRL_PIN(160, "XIRQ11", UNIPHIER_PIN_IECTRL_EXIST,
 			     111, UNIPHIER_PIN_DRV_3BIT,
 			     160, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(161, "XIRQ13", 161,
+	UNIPHIER_PINCTRL_PIN(161, "XIRQ13", UNIPHIER_PIN_IECTRL_EXIST,
 			     112, UNIPHIER_PIN_DRV_3BIT,
 			     161, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(162, "XIRQ14", 162,
+	UNIPHIER_PINCTRL_PIN(162, "XIRQ14", UNIPHIER_PIN_IECTRL_EXIST,
 			     113, UNIPHIER_PIN_DRV_3BIT,
 			     162, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(163, "XIRQ16", 163,
+	UNIPHIER_PINCTRL_PIN(163, "XIRQ16", UNIPHIER_PIN_IECTRL_EXIST,
 			     114, UNIPHIER_PIN_DRV_3BIT,
 			     163, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(164, "XIRQ17", 164,
+	UNIPHIER_PINCTRL_PIN(164, "XIRQ17", UNIPHIER_PIN_IECTRL_EXIST,
 			     115, UNIPHIER_PIN_DRV_3BIT,
 			     164, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(165, "XIRQ18", 165,
+	UNIPHIER_PINCTRL_PIN(165, "XIRQ18", UNIPHIER_PIN_IECTRL_EXIST,
 			     116, UNIPHIER_PIN_DRV_3BIT,
 			     165, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(166, "XIRQ19", 166,
+	UNIPHIER_PINCTRL_PIN(166, "XIRQ19", UNIPHIER_PIN_IECTRL_EXIST,
 			     117, UNIPHIER_PIN_DRV_3BIT,
 			     166, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(167, "XIRQ20", 167,
+	UNIPHIER_PINCTRL_PIN(167, "XIRQ20", UNIPHIER_PIN_IECTRL_EXIST,
 			     118, UNIPHIER_PIN_DRV_3BIT,
 			     167, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(168, "PORT00", 168,
+	UNIPHIER_PINCTRL_PIN(168, "PORT00", UNIPHIER_PIN_IECTRL_EXIST,
 			     119, UNIPHIER_PIN_DRV_3BIT,
 			     168, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(169, "PORT01", 169,
+	UNIPHIER_PINCTRL_PIN(169, "PORT01", UNIPHIER_PIN_IECTRL_EXIST,
 			     120, UNIPHIER_PIN_DRV_3BIT,
 			     169, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(170, "PORT02", 170,
+	UNIPHIER_PINCTRL_PIN(170, "PORT02", UNIPHIER_PIN_IECTRL_EXIST,
 			     121, UNIPHIER_PIN_DRV_3BIT,
 			     170, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(171, "PORT03", 171,
+	UNIPHIER_PINCTRL_PIN(171, "PORT03", UNIPHIER_PIN_IECTRL_EXIST,
 			     122, UNIPHIER_PIN_DRV_3BIT,
 			     171, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(172, "PORT04", 172,
+	UNIPHIER_PINCTRL_PIN(172, "PORT04", UNIPHIER_PIN_IECTRL_EXIST,
 			     123, UNIPHIER_PIN_DRV_3BIT,
 			     172, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(173, "CK27FO", 173,
+	UNIPHIER_PINCTRL_PIN(173, "CK27FO", UNIPHIER_PIN_IECTRL_EXIST,
 			     124, UNIPHIER_PIN_DRV_3BIT,
 			     173, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(174, "PHSYNCO", 174,
+	UNIPHIER_PINCTRL_PIN(174, "PHSYNCO", UNIPHIER_PIN_IECTRL_EXIST,
 			     125, UNIPHIER_PIN_DRV_3BIT,
 			     174, UNIPHIER_PIN_PULL_DOWN),
-	UNIPHIER_PINCTRL_PIN(175, "PVSYNCO", 175,
+	UNIPHIER_PINCTRL_PIN(175, "PVSYNCO", UNIPHIER_PIN_IECTRL_EXIST,
 			     126, UNIPHIER_PIN_DRV_3BIT,
 			     175, UNIPHIER_PIN_PULL_DOWN),
 };
 
+static const unsigned aout_pins[] = {135, 136, 137, 138, 139, 140, 141, 142};
+static const int aout_muxvals[] = {0, 0, 0, 0, 0, 0, 0, 0};
 static const unsigned emmc_pins[] = {18, 19, 20, 21, 22, 23, 24, 25};
 static const int emmc_muxvals[] = {0, 0, 0, 0, 0, 0, 0, 0};
 static const unsigned emmc_dat8_pins[] = {26, 27, 28, 29};
@@ -597,7 +599,7 @@ static const unsigned usb2_pins[] = {50, 51};
 static const int usb2_muxvals[] = {0, 0};
 static const unsigned usb3_pins[] = {52, 53};
 static const int usb3_muxvals[] = {0, 0};
-static const unsigned port_range0_pins[] = {
+static const unsigned int gpio_range0_pins[] = {
 	168, 169, 170, 171, 172, 173, 174, 175,		/* PORT0x */
 	0, 1, 2, 3, 4, 5, 6, 7,				/* PORT1x */
 	8, 9, 10, 11, 12, 13, 14, 15,			/* PORT2x */
@@ -610,36 +612,16 @@ static const unsigned port_range0_pins[] = {
 	83, 84, 85, 86, 87, 88, 89, 90,			/* PORT9x */
 	91, 92, 93, 94, 95, 96, 97, 98,			/* PORT10x */
 };
-static const int port_range0_muxvals[] = {
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT0x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT1x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT2x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT3x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT4x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT5x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT6x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT7x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT8x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT9x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT10x */
-};
-static const unsigned port_range1_pins[] = {
+static const unsigned int gpio_range1_pins[] = {
 	99, 100, 101, 102, 103, 104, 105, 106,		/* PORT12x */
 	107, 108, 109, 110, 111, 112, 113, 114,		/* PORT13x */
 	115, 116, 117, 118, 119, 120, 121, 122,		/* PORT14x */
-};
-static const int port_range1_muxvals[] = {
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT12x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT13x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT14x */
-};
-static const unsigned port_range2_pins[] = {
+	149, 150, 151, 152, 153, 154, 155, 156,		/* XIRQ0-7 */
+	157, 158, 159, 160, 85, 161, 162, 84,		/* XIRQ8-15 */
+	163, 164, 165, 166, 167, 146, 52, 53,		/* XIRQ16-23 */
 	61, 62, 63, 64, 65, 66, 67, 68,			/* PORT18x */
 };
-static const int port_range2_muxvals[] = {
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT18x */
-};
-static const unsigned port_range3_pins[] = {
+static const unsigned int gpio_range2_pins[] = {
 	123, 124, 125, 126, 127, 128, 129, 130,		/* PORT20x */
 	131, 132, 133, 134, 135, 136, 137, 138,		/* PORT21x */
 	139, 140, 141, 142, 143, 144, 145, 146,		/* PORT22x */
@@ -647,36 +629,9 @@ static const unsigned port_range3_pins[] = {
 	155, 156, 157, 158, 159, 160, 161, 162,		/* PORT24x */
 	163, 164, 165, 166, 167,			/* PORT250-254 */
 };
-static const int port_range3_muxvals[] = {
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT20x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT21x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT22x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT23x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT24x */
-	15, 15, 15, 15, 15,				/* PORT250-254 */
-};
-static const unsigned xirq_pins[] = {
-	149, 150, 151, 152, 153, 154, 155, 156,		/* XIRQ0-7 */
-	157, 158, 159, 160, 85, 161, 162, 84,		/* XIRQ8-15 */
-	163, 164, 165, 166, 167, 146, 52, 53,		/* XIRQ16-23 */
-};
-static const int xirq_muxvals[] = {
-	14, 14, 14, 14, 14, 14, 14, 14,			/* XIRQ0-7 */
-	14, 14, 14, 14, 13, 14, 14, 13,			/* XIRQ8-15 */
-	14, 14, 14, 14, 14, 14, 14, 14,			/* XIRQ16-23 */
-};
-static const unsigned xirq_alternatives_pins[] = {
-	94, 95, 96, 97, 98, 99, 100, 101,		/* XIRQ0-7 */
-	102, 103, 104, 105, 106, 107,			/* XIRQ8-11,13,14 */
-	108, 109, 110, 111, 112, 147, 141, 142,		/* XIRQ16-23 */
-};
-static const int xirq_alternatives_muxvals[] = {
-	14, 14, 14, 14, 14, 14, 14, 14,			/* XIRQ0-7 */
-	14, 14, 14, 14, 14, 14,				/* XIRQ8-11,13,14 */
-	14, 14, 14, 14, 14, 14, 14, 14,			/* XIRQ16-23 */
-};
 
 static const struct uniphier_pinctrl_group uniphier_ld20_groups[] = {
+	UNIPHIER_PINCTRL_GROUP(aout),
 	UNIPHIER_PINCTRL_GROUP(emmc),
 	UNIPHIER_PINCTRL_GROUP(emmc_dat8),
 	UNIPHIER_PINCTRL_GROUP(ether_rgmii),
@@ -697,225 +652,12 @@ static const struct uniphier_pinctrl_group uniphier_ld20_groups[] = {
 	UNIPHIER_PINCTRL_GROUP(usb1),
 	UNIPHIER_PINCTRL_GROUP(usb2),
 	UNIPHIER_PINCTRL_GROUP(usb3),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(port_range0),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(port_range1),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(port_range2),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(port_range3),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_IRQ(xirq),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_IRQ(xirq_alternatives),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port00, port_range0, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port01, port_range0, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port02, port_range0, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port03, port_range0, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port04, port_range0, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port05, port_range0, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port06, port_range0, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port07, port_range0, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port10, port_range0, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port11, port_range0, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port12, port_range0, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port13, port_range0, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port14, port_range0, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port15, port_range0, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port16, port_range0, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port17, port_range0, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port20, port_range0, 16),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port21, port_range0, 17),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port22, port_range0, 18),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port23, port_range0, 19),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port24, port_range0, 20),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port25, port_range0, 21),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port26, port_range0, 22),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port27, port_range0, 23),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port30, port_range0, 24),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port31, port_range0, 25),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port32, port_range0, 26),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port33, port_range0, 27),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port34, port_range0, 28),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port35, port_range0, 29),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port36, port_range0, 30),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port37, port_range0, 31),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port40, port_range0, 32),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port41, port_range0, 33),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port42, port_range0, 34),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port43, port_range0, 35),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port44, port_range0, 36),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port45, port_range0, 37),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port46, port_range0, 38),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port47, port_range0, 39),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port50, port_range0, 40),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port51, port_range0, 41),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port52, port_range0, 42),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port53, port_range0, 43),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port54, port_range0, 44),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port55, port_range0, 45),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port56, port_range0, 46),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port57, port_range0, 47),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port60, port_range0, 48),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port61, port_range0, 49),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port62, port_range0, 50),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port63, port_range0, 51),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port64, port_range0, 52),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port65, port_range0, 53),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port66, port_range0, 54),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port67, port_range0, 55),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port70, port_range0, 56),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port71, port_range0, 57),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port72, port_range0, 58),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port73, port_range0, 59),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port74, port_range0, 60),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port75, port_range0, 61),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port76, port_range0, 62),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port77, port_range0, 63),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port80, port_range0, 64),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port81, port_range0, 65),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port82, port_range0, 66),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port83, port_range0, 67),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port84, port_range0, 68),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port85, port_range0, 69),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port86, port_range0, 70),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port87, port_range0, 71),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port90, port_range0, 72),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port91, port_range0, 73),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port92, port_range0, 74),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port93, port_range0, 75),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port94, port_range0, 76),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port95, port_range0, 77),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port96, port_range0, 78),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port97, port_range0, 79),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port100, port_range0, 80),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port101, port_range0, 81),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port102, port_range0, 82),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port103, port_range0, 83),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port104, port_range0, 84),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port105, port_range0, 85),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port106, port_range0, 86),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port107, port_range0, 87),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port120, port_range1, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port121, port_range1, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port122, port_range1, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port123, port_range1, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port124, port_range1, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port125, port_range1, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port126, port_range1, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port127, port_range1, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port130, port_range1, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port131, port_range1, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port132, port_range1, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port133, port_range1, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port134, port_range1, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port135, port_range1, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port136, port_range1, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port137, port_range1, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port140, port_range1, 16),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port141, port_range1, 17),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port142, port_range1, 18),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port143, port_range1, 19),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port144, port_range1, 20),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port145, port_range1, 21),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port146, port_range1, 22),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port147, port_range1, 23),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port180, port_range2, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port181, port_range2, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port182, port_range2, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port183, port_range2, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port184, port_range2, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port185, port_range2, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port186, port_range2, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port187, port_range2, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port200, port_range3, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port201, port_range3, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port202, port_range3, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port203, port_range3, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port204, port_range3, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port205, port_range3, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port206, port_range3, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port207, port_range3, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port210, port_range3, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port211, port_range3, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port212, port_range3, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port213, port_range3, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port214, port_range3, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port215, port_range3, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port216, port_range3, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port217, port_range3, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port220, port_range3, 16),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port221, port_range3, 17),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port222, port_range3, 18),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port223, port_range3, 19),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port224, port_range3, 20),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port225, port_range3, 21),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port226, port_range3, 22),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port227, port_range3, 23),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port230, port_range3, 24),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port231, port_range3, 25),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port232, port_range3, 26),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port233, port_range3, 27),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port234, port_range3, 28),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port235, port_range3, 29),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port236, port_range3, 30),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port237, port_range3, 31),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port240, port_range3, 32),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port241, port_range3, 33),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port242, port_range3, 34),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port243, port_range3, 35),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port244, port_range3, 36),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port245, port_range3, 37),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port246, port_range3, 38),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port247, port_range3, 39),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port250, port_range3, 40),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port251, port_range3, 41),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port252, port_range3, 42),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port253, port_range3, 43),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port254, port_range3, 44),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq0, xirq, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq1, xirq, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq2, xirq, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq3, xirq, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq4, xirq, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq5, xirq, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq6, xirq, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq7, xirq, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq8, xirq, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq9, xirq, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq10, xirq, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq11, xirq, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq12, xirq, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq13, xirq, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq14, xirq, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq15, xirq, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq16, xirq, 16),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq17, xirq, 17),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq18, xirq, 18),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq19, xirq, 19),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq20, xirq, 20),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq21, xirq, 21),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq22, xirq, 22),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq23, xirq, 23),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq0b, xirq_alternatives, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq1b, xirq_alternatives, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq2b, xirq_alternatives, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq3b, xirq_alternatives, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq4b, xirq_alternatives, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq5b, xirq_alternatives, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq6b, xirq_alternatives, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq7b, xirq_alternatives, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq8b, xirq_alternatives, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq9b, xirq_alternatives, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq10b, xirq_alternatives, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq11b, xirq_alternatives, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq13b, xirq_alternatives, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq14b, xirq_alternatives, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq16b, xirq_alternatives, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq17b, xirq_alternatives, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq18b, xirq_alternatives, 16),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq19b, xirq_alternatives, 17),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq20b, xirq_alternatives, 18),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq21b, xirq_alternatives, 19),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq22b, xirq_alternatives, 20),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq23b, xirq_alternatives, 21),
+	UNIPHIER_PINCTRL_GROUP_GPIO(gpio_range0),
+	UNIPHIER_PINCTRL_GROUP_GPIO(gpio_range1),
+	UNIPHIER_PINCTRL_GROUP_GPIO(gpio_range2),
 };
 
+static const char * const aout_groups[] = {"aout"};
 static const char * const emmc_groups[] = {"emmc", "emmc_dat8"};
 static const char * const ether_rgmii_groups[] = {"ether_rgmii"};
 static const char * const ether_rmii_groups[] = {"ether_rmii"};
@@ -935,69 +677,9 @@ static const char * const usb0_groups[] = {"usb0"};
 static const char * const usb1_groups[] = {"usb1"};
 static const char * const usb2_groups[] = {"usb2"};
 static const char * const usb3_groups[] = {"usb3"};
-static const char * const port_groups[] = {
-	"port00", "port01", "port02", "port03",
-	"port04", "port05", "port06", "port07",
-	"port10", "port11", "port12", "port13",
-	"port14", "port15", "port16", "port17",
-	"port20", "port21", "port22", "port23",
-	"port24", "port25", "port26", "port27",
-	"port30", "port31", "port32", "port33",
-	"port34", "port35", "port36", "port37",
-	"port40", "port41", "port42", "port43",
-	"port44", "port45", "port46", "port47",
-	"port50", "port51", "port52", "port53",
-	"port54", "port55", "port56", "port57",
-	"port60", "port61", "port62", "port63",
-	"port64", "port65", "port66", "port67",
-	"port70", "port71", "port72", "port73",
-	"port74", "port75", "port76", "port77",
-	"port80", "port81", "port82", "port83",
-	"port84", "port85", "port86", "port87",
-	"port90", "port91", "port92", "port93",
-	"port94", "port95", "port96", "port97",
-	"port100", "port101", "port102", "port103",
-	"port104", "port105", "port106", "port107",
-	/* port110-117 missing */
-	"port120", "port121", "port122", "port123",
-	"port124", "port125", "port126", "port127",
-	"port130", "port131", "port132", "port133",
-	"port134", "port135", "port136", "port137",
-	"port140", "port141", "port142", "port143",
-	"port144", "port145", "port146", "port147",
-	/* port150-177 missing */
-	"port180", "port181", "port182", "port183",
-	"port184", "port185", "port186", "port187",
-	/* port190-197 missing */
-	"port200", "port201", "port202", "port203",
-	"port204", "port205", "port206", "port207",
-	"port210", "port211", "port212", "port213",
-	"port214", "port215", "port216", "port217",
-	"port220", "port221", "port222", "port223",
-	"port224", "port225", "port226", "port227",
-	"port230", "port231", "port232", "port233",
-	"port234", "port235", "port236", "port237",
-	"port240", "port241", "port242", "port243",
-	"port244", "port245", "port246", "port247",
-	"port250", "port251", "port252", "port253",
-	"port254",
-};
-static const char * const xirq_groups[] = {
-	"xirq0",  "xirq1",  "xirq2",  "xirq3",
-	"xirq4",  "xirq5",  "xirq6",  "xirq7",
-	"xirq8",  "xirq9",  "xirq10", "xirq11",
-	"xirq12", "xirq13", "xirq14", "xirq15",
-	"xirq16", "xirq17", "xirq18", "xirq19",
-	"xirq20", "xirq21", "xirq22", "xirq23",
-	"xirq0b",  "xirq1b",  "xirq2b",  "xirq3b",
-	"xirq4b",  "xirq5b",  "xirq6b",  "xirq7b",
-	"xirq8b",  "xirq9b",  "xirq10b", "xirq11b",
-	/* none */ "xirq13b", "xirq14b", /* none */
-	"xirq16b", "xirq17b", "xirq18b", "xirq19b",
-	"xirq20b", "xirq21b", "xirq22b", "xirq23b",
-};
 
 static const struct uniphier_pinmux_function uniphier_ld20_functions[] = {
+	UNIPHIER_PINMUX_FUNCTION(aout),
 	UNIPHIER_PINMUX_FUNCTION(emmc),
 	UNIPHIER_PINMUX_FUNCTION(ether_rgmii),
 	UNIPHIER_PINMUX_FUNCTION(ether_rmii),
@@ -1016,10 +698,20 @@ static const struct uniphier_pinmux_function uniphier_ld20_functions[] = {
 	UNIPHIER_PINMUX_FUNCTION(usb1),
 	UNIPHIER_PINMUX_FUNCTION(usb2),
 	UNIPHIER_PINMUX_FUNCTION(usb3),
-	UNIPHIER_PINMUX_FUNCTION(port),
-	UNIPHIER_PINMUX_FUNCTION(xirq),
 };
 
+static int uniphier_ld20_get_gpio_muxval(unsigned int pin,
+					 unsigned int gpio_offset)
+{
+	if (gpio_offset == 132 || gpio_offset == 135)	/* XIRQ12, 15 */
+		return 13;
+
+	if (gpio_offset >= 120 && gpio_offset <= 143)	/* XIRQx */
+		return 14;
+
+	return 15;
+}
+
 static struct uniphier_pinctrl_socdata uniphier_ld20_pindata = {
 	.pins = uniphier_ld20_pins,
 	.npins = ARRAY_SIZE(uniphier_ld20_pins),
@@ -1027,6 +719,7 @@ static struct uniphier_pinctrl_socdata uniphier_ld20_pindata = {
 	.groups_count = ARRAY_SIZE(uniphier_ld20_groups),
 	.functions = uniphier_ld20_functions,
 	.functions_count = ARRAY_SIZE(uniphier_ld20_functions),
+	.get_gpio_muxval = uniphier_ld20_get_gpio_muxval,
 	.caps = UNIPHIER_PINCTRL_CAPS_PERPIN_IECTRL,
 };
 
@@ -1045,6 +738,7 @@ static struct platform_driver uniphier_ld20_pinctrl_driver = {
 	.driver = {
 		.name = "uniphier-ld20-pinctrl",
 		.of_match_table = uniphier_ld20_pinctrl_match,
+		.pm = &uniphier_pinctrl_pm_ops,
 	},
 };
 builtin_platform_driver(uniphier_ld20_pinctrl_driver);
diff --git a/drivers/pinctrl/uniphier/pinctrl-uniphier-ld4.c b/drivers/pinctrl/uniphier/pinctrl-uniphier-ld4.c
index 8f2ad1c4c6f4..840382847212 100644
--- a/drivers/pinctrl/uniphier/pinctrl-uniphier-ld4.c
+++ b/drivers/pinctrl/uniphier/pinctrl-uniphier-ld4.c
@@ -606,59 +606,24 @@ static const unsigned usb2_pins[] = {155, 156};
 static const int usb2_muxvals[] = {4, 4};
 static const unsigned usb2b_pins[] = {67, 68};
 static const int usb2b_muxvals[] = {23, 23};
-static const unsigned port_range0_pins[] = {
-	135, 136, 137, 138, 139, 140, 141, 142,		/* PORT0x */
-	143, 144, 145, 146, 147, 148, 149, 150,		/* PORT1x */
-	151, 152, 153, 154, 155, 156, 157, 0,		/* PORT2x */
-	1, 2, 3, 4, 5, 120, 121, 122,			/* PORT3x */
-	24, 25, 26, 27, 28, 29, 30, 31,			/* PORT4x */
-	40, 41, 42, 43, 44, 45, 46, 47,			/* PORT5x */
-	48, 49, 50, 51, 52, 53, 54, 55,			/* PORT6x */
-	56, 85, 84, 59, 82, 61, 64, 65,			/* PORT7x */
-	8, 9, 10, 11, 12, 13, 14, 15,			/* PORT8x */
-	66, 67, 68, 69, 70, 71, 72, 73,			/* PORT9x */
-	74, 75, 89, 86, 78, 79, 80, 81,			/* PORT10x */
-	60, 83, 58, 57, 88, 87, 77, 76,			/* PORT11x */
-	90, 91, 92, 93, 94, 95, 96, 97,			/* PORT12x */
-	98, 99, 100, 6, 101, 114, 115, 116,		/* PORT13x */
-	103, 108, 21, 22, 23, 117, 118, 119,		/* PORT14x */
-};
-static const int port_range0_muxvals[] = {
-	0, 0, 0, 0, 0, 0, 0, 0,				/* PORT0x */
-	0, 0, 0, 0, 0, 0, 0, 0,				/* PORT1x */
-	0, 0, 0, 0, 0, 0, 0, 15,			/* PORT2x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT3x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT4x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT5x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT6x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT7x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT8x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT9x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT10x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT11x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT12x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT13x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT14x */
-};
-static const unsigned port_range1_pins[] = {
-	7,						/* PORT166 */
-};
-static const int port_range1_muxvals[] = {
-	15,						/* PORT166 */
-};
-static const unsigned xirq_range0_pins[] = {
-	151, 123, 124, 125, 126, 127, 128, 129,		/* XIRQ0-7 */
-	130, 131, 132, 133, 62,				/* XIRQ8-12 */
-};
-static const int xirq_range0_muxvals[] = {
-	14, 0, 0, 0, 0, 0, 0, 0,			/* XIRQ0-7 */
-	0, 0, 0, 0, 14,					/* XIRQ8-12 */
-};
-static const unsigned xirq_range1_pins[] = {
-	134, 63,					/* XIRQ14-15 */
-};
-static const int xirq_range1_muxvals[] = {
-	0, 14,						/* XIRQ14-15 */
+static const unsigned int gpio_range_pins[] = {
+	135, 136, 137, 138, 139, 140, 141, 142,	/* PORT0x */
+	143, 144, 145, 146, 147, 148, 149, 150,	/* PORT1x */
+	151, 152, 153, 154, 155, 156, 157, 0,	/* PORT2x */
+	1, 2, 3, 4, 5, 120, 121, 122,		/* PORT3x */
+	24, 25, 26, 27, 28, 29, 30, 31,		/* PORT4x */
+	40, 41, 42, 43, 44, 45, 46, 47,		/* PORT5x */
+	48, 49, 50, 51, 52, 53, 54, 55,		/* PORT6x */
+	56, 85, 84, 59, 82, 61, 64, 65,		/* PORT7x */
+	8, 9, 10, 11, 12, 13, 14, 15,		/* PORT8x */
+	66, 67, 68, 69, 70, 71, 72, 73,		/* PORT9x */
+	74, 75, 89, 86, 78, 79, 80, 81,		/* PORT10x */
+	60, 83, 58, 57, 88, 87, 77, 76,		/* PORT11x */
+	90, 91, 92, 93, 94, 95, 96, 97,		/* PORT12x */
+	98, 99, 100, 6, 101, 114, 115, 116,	/* PORT13x */
+	103, 108, 21, 22, 23, 117, 118, 119,	/* PORT14x */
+	151, 123, 124, 125, 126, 127, 128, 129,	/* XIRQ0-7 */
+	130, 131, 132, 133, 62, 7, 134, 63,   /* XIRQ8-12, PORT165, XIRQ14-15 */
 };
 
 static const struct uniphier_pinctrl_group uniphier_ld4_groups[] = {
@@ -687,146 +652,7 @@ static const struct uniphier_pinctrl_group uniphier_ld4_groups[] = {
 	UNIPHIER_PINCTRL_GROUP(usb1),
 	UNIPHIER_PINCTRL_GROUP(usb2),
 	UNIPHIER_PINCTRL_GROUP(usb2b),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(port_range0),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(port_range1),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_IRQ(xirq_range0),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_IRQ(xirq_range1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port00, port_range0, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port01, port_range0, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port02, port_range0, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port03, port_range0, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port04, port_range0, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port05, port_range0, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port06, port_range0, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port07, port_range0, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port10, port_range0, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port11, port_range0, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port12, port_range0, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port13, port_range0, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port14, port_range0, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port15, port_range0, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port16, port_range0, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port17, port_range0, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port20, port_range0, 16),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port21, port_range0, 17),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port22, port_range0, 18),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port23, port_range0, 19),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port24, port_range0, 20),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port25, port_range0, 21),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port26, port_range0, 22),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port27, port_range0, 23),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port30, port_range0, 24),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port31, port_range0, 25),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port32, port_range0, 26),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port33, port_range0, 27),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port34, port_range0, 28),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port35, port_range0, 29),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port36, port_range0, 30),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port37, port_range0, 31),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port40, port_range0, 32),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port41, port_range0, 33),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port42, port_range0, 34),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port43, port_range0, 35),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port44, port_range0, 36),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port45, port_range0, 37),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port46, port_range0, 38),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port47, port_range0, 39),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port50, port_range0, 40),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port51, port_range0, 41),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port52, port_range0, 42),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port53, port_range0, 43),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port54, port_range0, 44),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port55, port_range0, 45),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port56, port_range0, 46),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port57, port_range0, 47),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port60, port_range0, 48),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port61, port_range0, 49),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port62, port_range0, 50),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port63, port_range0, 51),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port64, port_range0, 52),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port65, port_range0, 53),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port66, port_range0, 54),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port67, port_range0, 55),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port70, port_range0, 56),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port71, port_range0, 57),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port72, port_range0, 58),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port73, port_range0, 59),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port74, port_range0, 60),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port75, port_range0, 61),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port76, port_range0, 62),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port77, port_range0, 63),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port80, port_range0, 64),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port81, port_range0, 65),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port82, port_range0, 66),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port83, port_range0, 67),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port84, port_range0, 68),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port85, port_range0, 69),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port86, port_range0, 70),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port87, port_range0, 71),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port90, port_range0, 72),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port91, port_range0, 73),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port92, port_range0, 74),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port93, port_range0, 75),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port94, port_range0, 76),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port95, port_range0, 77),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port96, port_range0, 78),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port97, port_range0, 79),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port100, port_range0, 80),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port101, port_range0, 81),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port102, port_range0, 82),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port103, port_range0, 83),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port104, port_range0, 84),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port105, port_range0, 85),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port106, port_range0, 86),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port107, port_range0, 87),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port110, port_range0, 88),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port111, port_range0, 89),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port112, port_range0, 90),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port113, port_range0, 91),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port114, port_range0, 92),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port115, port_range0, 93),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port116, port_range0, 94),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port117, port_range0, 95),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port120, port_range0, 96),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port121, port_range0, 97),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port122, port_range0, 98),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port123, port_range0, 99),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port124, port_range0, 100),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port125, port_range0, 101),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port126, port_range0, 102),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port127, port_range0, 103),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port130, port_range0, 104),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port131, port_range0, 105),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port132, port_range0, 106),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port133, port_range0, 107),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port134, port_range0, 108),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port135, port_range0, 109),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port136, port_range0, 110),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port137, port_range0, 111),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port140, port_range0, 112),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port141, port_range0, 113),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port142, port_range0, 114),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port143, port_range0, 115),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port144, port_range0, 116),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port145, port_range0, 117),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port146, port_range0, 118),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port147, port_range0, 119),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port165, port_range1, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq0, xirq_range0, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq1, xirq_range0, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq2, xirq_range0, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq3, xirq_range0, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq4, xirq_range0, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq5, xirq_range0, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq6, xirq_range0, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq7, xirq_range0, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq8, xirq_range0, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq9, xirq_range0, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq10, xirq_range0, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq11, xirq_range0, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq12, xirq_range0, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq14, xirq_range1, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq15, xirq_range1, 1),
+	UNIPHIER_PINCTRL_GROUP_GPIO(gpio_range),
 };
 
 static const char * const emmc_groups[] = {"emmc", "emmc_dat8"};
@@ -850,46 +676,6 @@ static const char * const uart3_groups[] = {"uart3"};
 static const char * const usb0_groups[] = {"usb0"};
 static const char * const usb1_groups[] = {"usb1"};
 static const char * const usb2_groups[] = {"usb2", "usb2b"};
-static const char * const port_groups[] = {
-	"port00",  "port01",  "port02",  "port03",
-	"port04",  "port05",  "port06",  "port07",
-	"port10",  "port11",  "port12",  "port13",
-	"port14",  "port15",  "port16",  "port17",
-	"port20",  "port21",  "port22",  "port23",
-	"port24",  "port25",  "port26",  "port27",
-	"port30",  "port31",  "port32",  "port33",
-	"port34",  "port35",  "port36",  "port37",
-	"port40",  "port41",  "port42",  "port43",
-	"port44",  "port45",  "port46",  "port47",
-	"port50",  "port51",  "port52",  "port53",
-	"port54",  "port55",  "port56",  "port57",
-	"port60",  "port61",  "port62",  "port63",
-	"port64",  "port65",  "port66",  "port67",
-	"port70",  "port71",  "port72",  "port73",
-	"port74",  "port75",  "port76",  "port77",
-	"port80",  "port81",  "port82",  "port83",
-	"port84",  "port85",  "port86",  "port87",
-	"port90",  "port91",  "port92",  "port93",
-	"port94",  "port95",  "port96",  "port97",
-	"port100", "port101", "port102", "port103",
-	"port104", "port105", "port106", "port107",
-	"port110", "port111", "port112", "port113",
-	"port114", "port115", "port116", "port117",
-	"port120", "port121", "port122", "port123",
-	"port124", "port125", "port126", "port127",
-	"port130", "port131", "port132", "port133",
-	"port134", "port135", "port136", "port137",
-	"port140", "port141", "port142", "port143",
-	"port144", "port145", "port146", "port147",
-	/* port150-164 missing */
-	/* none */ "port165",
-};
-static const char * const xirq_groups[] = {
-	"xirq0",  "xirq1",  "xirq2",  "xirq3",
-	"xirq4",  "xirq5",  "xirq6",  "xirq7",
-	"xirq8",  "xirq9",  "xirq10", "xirq11",
-	"xirq12", /* none*/ "xirq14", "xirq15",
-};
 
 static const struct uniphier_pinmux_function uniphier_ld4_functions[] = {
 	UNIPHIER_PINMUX_FUNCTION(emmc),
@@ -909,10 +695,25 @@ static const struct uniphier_pinmux_function uniphier_ld4_functions[] = {
 	UNIPHIER_PINMUX_FUNCTION(usb0),
 	UNIPHIER_PINMUX_FUNCTION(usb1),
 	UNIPHIER_PINMUX_FUNCTION(usb2),
-	UNIPHIER_PINMUX_FUNCTION(port),
-	UNIPHIER_PINMUX_FUNCTION(xirq),
 };
 
+static int uniphier_ld4_get_gpio_muxval(unsigned int pin,
+					unsigned int gpio_offset)
+{
+	switch (gpio_offset) {
+	case 0 ... 22:		/* PORT00-PORT26 */
+	case 121 ... 131:	/* XIRQ1-XIRQ11 */
+	case 134:		/* XIRQ14 */
+		return 0;
+	case 120:		/* XIRQ0 */
+	case 132:		/* XIRQ12 */
+	case 135:		/* XIRQ15 */
+		return 14;
+	default:
+		return 15;
+	}
+}
+
 static struct uniphier_pinctrl_socdata uniphier_ld4_pindata = {
 	.pins = uniphier_ld4_pins,
 	.npins = ARRAY_SIZE(uniphier_ld4_pins),
@@ -920,6 +721,7 @@ static struct uniphier_pinctrl_socdata uniphier_ld4_pindata = {
 	.groups_count = ARRAY_SIZE(uniphier_ld4_groups),
 	.functions = uniphier_ld4_functions,
 	.functions_count = ARRAY_SIZE(uniphier_ld4_functions),
+	.get_gpio_muxval = uniphier_ld4_get_gpio_muxval,
 	.caps = 0,
 };
 
@@ -938,6 +740,7 @@ static struct platform_driver uniphier_ld4_pinctrl_driver = {
 	.driver = {
 		.name = "uniphier-ld4-pinctrl",
 		.of_match_table = uniphier_ld4_pinctrl_match,
+		.pm = &uniphier_pinctrl_pm_ops,
 	},
 };
 builtin_platform_driver(uniphier_ld4_pinctrl_driver);
diff --git a/drivers/pinctrl/uniphier/pinctrl-uniphier-ld6b.c b/drivers/pinctrl/uniphier/pinctrl-uniphier-ld6b.c
index 8a0da937b670..493a90c6d733 100644
--- a/drivers/pinctrl/uniphier/pinctrl-uniphier-ld6b.c
+++ b/drivers/pinctrl/uniphier/pinctrl-uniphier-ld6b.c
@@ -803,7 +803,7 @@ static const unsigned usb2_pins[] = {60, 61};
 static const int usb2_muxvals[] = {0, 0};
 static const unsigned usb3_pins[] = {62, 63};
 static const int usb3_muxvals[] = {0, 0};
-static const unsigned port_range0_pins[] = {
+static const unsigned int gpio_range0_pins[] = {
 	127, 128, 129, 130, 131, 132, 133, 134,		/* PORT0x */
 	135, 136, 137, 138, 139, 140, 141, 142,		/* PORT1x */
 	0, 1, 2, 3, 4, 5, 6, 7,				/* PORT2x */
@@ -816,26 +816,13 @@ static const unsigned port_range0_pins[] = {
 	61, 62, 63, 64, 65, 66, 67, 68,			/* PORT9x */
 	69, 70, 71, 76, 77, 78, 79, 80,			/* PORT10x */
 };
-static const int port_range0_muxvals[] = {
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT0x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT1x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT2x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT3x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT4x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT5x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT6x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT7x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT8x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT9x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT10x */
-};
-static const unsigned port_range1_pins[] = {
+static const unsigned int gpio_range1_pins[] = {
 	81, 82, 83, 84, 85, 86, 87, 88,			/* PORT12x */
 	89, 90, 95, 96, 97, 98, 99, 100,		/* PORT13x */
 	101, 102, 103, 104, 105, 106, 107, 108,		/* PORT14x */
-	118, 119, 120, 121, 122, 123, 124, 125,		/* PORT15x */
-	126, 72, 73, 92, 177, 93, 94, 176,		/* PORT16x */
-	74, 91, 27, 28, 29, 75, 20, 26,			/* PORT17x */
+	118, 119, 120, 121, 122, 123, 124, 125,		/* XIRQ0-7 */
+	126, 72, 73, 92, 177, 93, 94, 176,		/* XIRQ8-15 */
+	74, 91, 27, 28, 29, 75, 20, 26,			/* XIRQ16-23 */
 	109, 110, 111, 112, 113, 114, 115, 116,		/* PORT18x */
 	117, 143, 144, 145, 146, 147, 148, 149,		/* PORT19x */
 	150, 151, 152, 153, 154, 155, 156, 157,		/* PORT20x */
@@ -848,35 +835,6 @@ static const unsigned port_range1_pins[] = {
 	218, 219, 220, 221, 223, 224, 225, 226,		/* PORT27x */
 	227, 228, 229, 230, 231, 232, 233, 234,		/* PORT28x */
 };
-static const int port_range1_muxvals[] = {
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT12x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT13x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT14x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT15x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT16x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT17x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT18x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT19x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT20x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT21x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT22x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT23x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT24x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT25x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT26x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT27x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT28x */
-};
-static const unsigned xirq_pins[] = {
-	118, 119, 120, 121, 122, 123, 124, 125,		/* XIRQ0-7 */
-	126, 72, 73, 92, 177, 93, 94, 176,		/* XIRQ8-15 */
-	74, 91, 27, 28, 29, 75, 20, 26,			/* XIRQ16-23 */
-};
-static const int xirq_muxvals[] = {
-	14, 14, 14, 14, 14, 14, 14, 14,			/* XIRQ0-7 */
-	14, 14, 14, 14, 14, 14, 14, 14,			/* XIRQ8-15 */
-	14, 14, 14, 14, 14, 14, 14, 14,			/* XIRQ16-23 */
-};
 
 static const struct uniphier_pinctrl_group uniphier_ld6b_groups[] = {
 	UNIPHIER_PINCTRL_GROUP(adinter),
@@ -907,257 +865,8 @@ static const struct uniphier_pinctrl_group uniphier_ld6b_groups[] = {
 	UNIPHIER_PINCTRL_GROUP(usb1),
 	UNIPHIER_PINCTRL_GROUP(usb2),
 	UNIPHIER_PINCTRL_GROUP(usb3),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(port_range0),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(port_range1),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_IRQ(xirq),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port00, port_range0, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port01, port_range0, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port02, port_range0, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port03, port_range0, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port04, port_range0, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port05, port_range0, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port06, port_range0, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port07, port_range0, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port10, port_range0, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port11, port_range0, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port12, port_range0, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port13, port_range0, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port14, port_range0, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port15, port_range0, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port16, port_range0, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port17, port_range0, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port20, port_range0, 16),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port21, port_range0, 17),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port22, port_range0, 18),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port23, port_range0, 19),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port24, port_range0, 20),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port25, port_range0, 21),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port26, port_range0, 22),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port27, port_range0, 23),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port30, port_range0, 24),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port31, port_range0, 25),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port32, port_range0, 26),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port33, port_range0, 27),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port34, port_range0, 28),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port35, port_range0, 29),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port36, port_range0, 30),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port37, port_range0, 31),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port40, port_range0, 32),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port41, port_range0, 33),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port42, port_range0, 34),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port43, port_range0, 35),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port44, port_range0, 36),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port45, port_range0, 37),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port46, port_range0, 38),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port47, port_range0, 39),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port50, port_range0, 40),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port51, port_range0, 41),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port52, port_range0, 42),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port53, port_range0, 43),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port54, port_range0, 44),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port55, port_range0, 45),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port56, port_range0, 46),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port57, port_range0, 47),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port60, port_range0, 48),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port61, port_range0, 49),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port62, port_range0, 50),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port63, port_range0, 51),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port64, port_range0, 52),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port65, port_range0, 53),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port66, port_range0, 54),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port67, port_range0, 55),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port70, port_range0, 56),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port71, port_range0, 57),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port72, port_range0, 58),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port73, port_range0, 59),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port74, port_range0, 60),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port75, port_range0, 61),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port76, port_range0, 62),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port77, port_range0, 63),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port80, port_range0, 64),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port81, port_range0, 65),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port82, port_range0, 66),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port83, port_range0, 67),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port84, port_range0, 68),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port85, port_range0, 69),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port86, port_range0, 70),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port87, port_range0, 71),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port90, port_range0, 72),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port91, port_range0, 73),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port92, port_range0, 74),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port93, port_range0, 75),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port94, port_range0, 76),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port95, port_range0, 77),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port96, port_range0, 78),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port97, port_range0, 79),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port100, port_range0, 80),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port101, port_range0, 81),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port102, port_range0, 82),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port103, port_range0, 83),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port104, port_range0, 84),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port105, port_range0, 85),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port106, port_range0, 86),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port107, port_range0, 87),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port120, port_range1, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port121, port_range1, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port122, port_range1, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port123, port_range1, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port124, port_range1, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port125, port_range1, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port126, port_range1, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port127, port_range1, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port130, port_range1, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port131, port_range1, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port132, port_range1, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port133, port_range1, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port134, port_range1, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port135, port_range1, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port136, port_range1, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port137, port_range1, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port140, port_range1, 16),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port141, port_range1, 17),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port142, port_range1, 18),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port143, port_range1, 19),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port144, port_range1, 20),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port145, port_range1, 21),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port146, port_range1, 22),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port147, port_range1, 23),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port150, port_range1, 24),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port151, port_range1, 25),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port152, port_range1, 26),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port153, port_range1, 27),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port154, port_range1, 28),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port155, port_range1, 29),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port156, port_range1, 30),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port157, port_range1, 31),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port160, port_range1, 32),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port161, port_range1, 33),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port162, port_range1, 34),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port163, port_range1, 35),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port164, port_range1, 36),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port165, port_range1, 37),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port166, port_range1, 38),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port167, port_range1, 39),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port170, port_range1, 40),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port171, port_range1, 41),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port172, port_range1, 42),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port173, port_range1, 43),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port174, port_range1, 44),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port175, port_range1, 45),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port176, port_range1, 46),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port177, port_range1, 47),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port180, port_range1, 48),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port181, port_range1, 49),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port182, port_range1, 50),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port183, port_range1, 51),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port184, port_range1, 52),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port185, port_range1, 53),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port186, port_range1, 54),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port187, port_range1, 55),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port190, port_range1, 56),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port191, port_range1, 57),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port192, port_range1, 58),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port193, port_range1, 59),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port194, port_range1, 60),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port195, port_range1, 61),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port196, port_range1, 62),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port197, port_range1, 63),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port200, port_range1, 64),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port201, port_range1, 65),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port202, port_range1, 66),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port203, port_range1, 67),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port204, port_range1, 68),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port205, port_range1, 69),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port206, port_range1, 70),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port207, port_range1, 71),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port210, port_range1, 72),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port211, port_range1, 73),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port212, port_range1, 74),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port213, port_range1, 75),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port214, port_range1, 76),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port215, port_range1, 77),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port216, port_range1, 78),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port217, port_range1, 79),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port220, port_range1, 80),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port221, port_range1, 81),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port222, port_range1, 82),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port223, port_range1, 83),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port224, port_range1, 84),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port225, port_range1, 85),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port226, port_range1, 86),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port227, port_range1, 87),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port230, port_range1, 88),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port231, port_range1, 89),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port232, port_range1, 90),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port233, port_range1, 91),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port234, port_range1, 92),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port235, port_range1, 93),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port236, port_range1, 94),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port237, port_range1, 95),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port240, port_range1, 96),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port241, port_range1, 97),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port242, port_range1, 98),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port243, port_range1, 99),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port244, port_range1, 100),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port245, port_range1, 101),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port246, port_range1, 102),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port247, port_range1, 103),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port250, port_range1, 104),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port251, port_range1, 105),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port252, port_range1, 106),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port253, port_range1, 107),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port254, port_range1, 108),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port255, port_range1, 109),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port256, port_range1, 110),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port257, port_range1, 111),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port260, port_range1, 112),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port261, port_range1, 113),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port262, port_range1, 114),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port263, port_range1, 115),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port264, port_range1, 116),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port265, port_range1, 117),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port266, port_range1, 118),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port267, port_range1, 119),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port270, port_range1, 120),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port271, port_range1, 121),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port272, port_range1, 122),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port273, port_range1, 123),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port274, port_range1, 124),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port275, port_range1, 125),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port276, port_range1, 126),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port277, port_range1, 127),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port280, port_range1, 128),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port281, port_range1, 129),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port282, port_range1, 130),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port283, port_range1, 131),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port284, port_range1, 132),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port285, port_range1, 133),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port286, port_range1, 134),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port287, port_range1, 135),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq0, xirq, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq1, xirq, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq2, xirq, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq3, xirq, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq4, xirq, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq5, xirq, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq6, xirq, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq7, xirq, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq8, xirq, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq9, xirq, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq10, xirq, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq11, xirq, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq12, xirq, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq13, xirq, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq14, xirq, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq15, xirq, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq16, xirq, 16),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq17, xirq, 17),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq18, xirq, 18),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq19, xirq, 19),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq20, xirq, 20),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq21, xirq, 21),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq22, xirq, 22),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq23, xirq, 23),
+	UNIPHIER_PINCTRL_GROUP_GPIO(gpio_range0),
+	UNIPHIER_PINCTRL_GROUP_GPIO(gpio_range1),
 };
 
 static const char * const adinter_groups[] = {"adinter"};
@@ -1183,73 +892,6 @@ static const char * const usb0_groups[] = {"usb0"};
 static const char * const usb1_groups[] = {"usb1"};
 static const char * const usb2_groups[] = {"usb2"};
 static const char * const usb3_groups[] = {"usb3"};
-static const char * const port_groups[] = {
-	"port00",  "port01",  "port02",  "port03",
-	"port04",  "port05",  "port06",  "port07",
-	"port10",  "port11",  "port12",  "port13",
-	"port14",  "port15",  "port16",  "port17",
-	"port20",  "port21",  "port22",  "port23",
-	"port24",  "port25",  "port26",  "port27",
-	"port30",  "port31",  "port32",  "port33",
-	"port34",  "port35",  "port36",  "port37",
-	"port40",  "port41",  "port42",  "port43",
-	"port44",  "port45",  "port46",  "port47",
-	"port50",  "port51",  "port52",  "port53",
-	"port54",  "port55",  "port56",  "port57",
-	"port60",  "port61",  "port62",  "port63",
-	"port64",  "port65",  "port66",  "port67",
-	"port70",  "port71",  "port72",  "port73",
-	"port74",  "port75",  "port76",  "port77",
-	"port80",  "port81",  "port82",  "port83",
-	"port84",  "port85",  "port86",  "port87",
-	"port90",  "port91",  "port92",  "port93",
-	"port94",  "port95",  "port96",  "port97",
-	"port100", "port101", "port102", "port103",
-	"port104", "port105", "port106", "port107",
-	/* port110-117 missing */
-	"port120", "port121", "port122", "port123",
-	"port124", "port125", "port126", "port127",
-	"port130", "port131", "port132", "port133",
-	"port134", "port135", "port136", "port137",
-	"port140", "port141", "port142", "port143",
-	"port144", "port145", "port146", "port147",
-	"port150", "port151", "port152", "port153",
-	"port154", "port155", "port156", "port157",
-	"port160", "port161", "port162", "port163",
-	"port164", "port165", "port166", "port167",
-	"port170", "port171", "port172", "port173",
-	"port174", "port175", "port176", "port177",
-	"port180", "port181", "port182", "port183",
-	"port184", "port185", "port186", "port187",
-	"port190", "port191", "port192", "port193",
-	"port194", "port195", "port196", "port197",
-	"port200", "port201", "port202", "port203",
-	"port204", "port205", "port206", "port207",
-	"port210", "port211", "port212", "port213",
-	"port214", "port215", "port216", "port217",
-	"port220", "port221", "port222", "port223",
-	"port224", "port225", "port226", "port227",
-	"port230", "port231", "port232", "port233",
-	"port234", "port235", "port236", "port237",
-	"port240", "port241", "port242", "port243",
-	"port244", "port245", "port246", "port247",
-	"port250", "port251", "port252", "port253",
-	"port254", "port255", "port256", "port257",
-	"port260", "port261", "port262", "port263",
-	"port264", "port265", "port266", "port267",
-	"port270", "port271", "port272", "port273",
-	"port274", "port275", "port276", "port277",
-	"port280", "port281", "port282", "port283",
-	"port284", "port285", "port286", "port287",
-};
-static const char * const xirq_groups[] = {
-	"xirq0",  "xirq1",  "xirq2",  "xirq3",
-	"xirq4",  "xirq5",  "xirq6",  "xirq7",
-	"xirq8",  "xirq9",  "xirq10", "xirq11",
-	"xirq12", "xirq13", "xirq14", "xirq15",
-	"xirq16", "xirq17", "xirq18", "xirq19",
-	"xirq20", "xirq21", "xirq22", "xirq23",
-};
 
 static const struct uniphier_pinmux_function uniphier_ld6b_functions[] = {
 	UNIPHIER_PINMUX_FUNCTION(adinter), /* Achip-Dchip interconnect */
@@ -1270,10 +912,18 @@ static const struct uniphier_pinmux_function uniphier_ld6b_functions[] = {
 	UNIPHIER_PINMUX_FUNCTION(usb1),
 	UNIPHIER_PINMUX_FUNCTION(usb2),
 	UNIPHIER_PINMUX_FUNCTION(usb3),
-	UNIPHIER_PINMUX_FUNCTION(port),
-	UNIPHIER_PINMUX_FUNCTION(xirq),
 };
 
+static int uniphier_ld6b_get_gpio_muxval(unsigned int pin,
+					 unsigned int gpio_offset)
+{
+	if (gpio_offset >= 120 && gpio_offset <= 143)	/* XIRQx */
+		/* 15 will do because XIRQ0-23 are aliases of PORT150-177. */
+		return 14;
+
+	return 15;
+}
+
 static struct uniphier_pinctrl_socdata uniphier_ld6b_pindata = {
 	.pins = uniphier_ld6b_pins,
 	.npins = ARRAY_SIZE(uniphier_ld6b_pins),
@@ -1281,6 +931,7 @@ static struct uniphier_pinctrl_socdata uniphier_ld6b_pindata = {
 	.groups_count = ARRAY_SIZE(uniphier_ld6b_groups),
 	.functions = uniphier_ld6b_functions,
 	.functions_count = ARRAY_SIZE(uniphier_ld6b_functions),
+	.get_gpio_muxval = uniphier_ld6b_get_gpio_muxval,
 	.caps = 0,
 };
 
@@ -1299,6 +950,7 @@ static struct platform_driver uniphier_ld6b_pinctrl_driver = {
 	.driver = {
 		.name = "uniphier-ld6b-pinctrl",
 		.of_match_table = uniphier_ld6b_pinctrl_match,
+		.pm = &uniphier_pinctrl_pm_ops,
 	},
 };
 builtin_platform_driver(uniphier_ld6b_pinctrl_driver);
diff --git a/drivers/pinctrl/uniphier/pinctrl-uniphier-pro4.c b/drivers/pinctrl/uniphier/pinctrl-uniphier-pro4.c
index c75e094b2d90..603204a00213 100644
--- a/drivers/pinctrl/uniphier/pinctrl-uniphier-pro4.c
+++ b/drivers/pinctrl/uniphier/pinctrl-uniphier-pro4.c
@@ -1086,87 +1086,38 @@ static const unsigned usb2_pins[] = {184, 185};
 static const int usb2_muxvals[] = {0, 0};
 static const unsigned usb3_pins[] = {187, 188};
 static const int usb3_muxvals[] = {0, 0};
-static const unsigned port_range0_pins[] = {
-	300, 301, 302, 303, 304, 305, 306, 307,		/* PORT0x */
-	308, 309, 310, 311, 312, 313, 314, 315,		/* PORT1x */
-	316, 317, 318, 16, 17, 18, 19, 20,		/* PORT2x */
-	21, 22, 23, 4, 93, 94, 95, 63,			/* PORT3x */
-	123, 122, 124, 125, 126, 141, 202, 203,		/* PORT4x */
-	204, 226, 227, 290, 291, 233, 280, 281,		/* PORT5x */
-	8, 7, 10, 29, 30, 48, 49, 50,			/* PORT6x */
-	40, 41, 42, 43, 44, 45, 46, 47,			/* PORT7x */
-	54, 51, 52, 53, 127, 128, 129, 130,		/* PORT8x */
-	131, 132, 57, 60, 134, 133, 135, 136,		/* PORT9x */
-	138, 137, 140, 139, 64, 65, 66, 67,		/* PORT10x */
-	107, 106, 105, 104, 113, 112, 111, 110,		/* PORT11x */
-	68, 69, 70, 71, 72, 73, 74, 75,			/* PORT12x */
-	76, 77, 78, 79, 80, 81, 82, 83,			/* PORT13x */
-	84, 85, 86, 87, 88, 89, 90, 91,			/* PORT14x */
-};
-static const int port_range0_muxvals[] = {
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT0x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT1x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT2x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT3x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT4x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT5x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT6x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT7x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT8x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT9x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT10x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT11x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT12x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT13x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT14x */
-};
-static const unsigned port_range1_pins[] = {
-	13, 14, 15,					/* PORT175-177 */
-	157, 158, 156, 154, 150, 151, 152, 153,		/* PORT18x */
-	326, 327, 325, 323, 319, 320, 321, 322,		/* PORT19x */
-	160, 161, 162, 163, 164, 165, 166, 167,		/* PORT20x */
-	168, 169, 170, 171, 172, 173, 174, 175,		/* PORT21x */
-	180, 181, 182, 183, 184, 185, 187, 188,		/* PORT22x */
-	193, 194, 195, 196, 197, 198, 199, 200,		/* PORT23x */
-	191, 192, 215, 216, 217, 218, 219, 220,		/* PORT24x */
-	222, 223, 224, 225, 228, 229, 230, 231,		/* PORT25x */
-	282, 283, 284, 285, 286, 287, 288, 289,		/* PORT26x */
-	292, 293, 294, 295, 296, 236, 237, 238,		/* PORT27x */
-	275, 276, 277, 278, 239, 240, 249, 250,		/* PORT28x */
-	251, 252, 261, 262, 263, 264, 273, 274,		/* PORT29x */
-	31, 32, 33, 34, 35, 36, 37, 38,			/* PORT30x */
-};
-static const int port_range1_muxvals[] = {
-	7, 7, 7,					/* PORT175-177 */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT18x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT19x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT20x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT21x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT22x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT23x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT24x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT25x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT26x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT27x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT28x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT29x */
-	7, 7, 7, 7, 7, 7, 7, 7,				/* PORT30x */
-};
-static const unsigned xirq_pins[] = {
-	11, 9, 12, 96, 97, 98, 108, 114,		/* XIRQ0-7 */
-	234, 186, 99, 100, 101, 102, 184, 301,		/* XIRQ8-15 */
-	302, 303, 304, 305, 306,			/* XIRQ16-20 */
-};
-static const int xirq_muxvals[] = {
-	7, 7, 7, 7, 7, 7, 7, 7,				/* XIRQ0-7 */
-	7, 7, 7, 7, 7, 7, 2, 2,				/* XIRQ8-15 */
-	2, 2, 2, 2, 2,					/* XIRQ16-20 */
-};
-static const unsigned xirq_alternatives_pins[] = {
-	184, 310, 316,
-};
-static const int xirq_alternatives_muxvals[] = {
-	2, 2, 2,
+static const unsigned int gpio_range_pins[] = {
+	300, 301, 302, 303, 304, 305, 306, 307,	/* PORT0x */
+	308, 309, 310, 311, 312, 313, 314, 315,	/* PORT1x */
+	316, 317, 318, 16, 17, 18, 19, 20,	/* PORT2x */
+	21, 22, 23, 4, 93, 94, 95, 63,		/* PORT3x */
+	123, 122, 124, 125, 126, 141, 202, 203,	/* PORT4x */
+	204, 226, 227, 290, 291, 233, 280, 281,	/* PORT5x */
+	8, 7, 10, 29, 30, 48, 49, 50,		/* PORT6x */
+	40, 41, 42, 43, 44, 45, 46, 47,		/* PORT7x */
+	54, 51, 52, 53, 127, 128, 129, 130,	/* PORT8x */
+	131, 132, 57, 60, 134, 133, 135, 136,	/* PORT9x */
+	138, 137, 140, 139, 64, 65, 66, 67,	/* PORT10x */
+	107, 106, 105, 104, 113, 112, 111, 110,	/* PORT11x */
+	68, 69, 70, 71, 72, 73, 74, 75,		/* PORT12x */
+	76, 77, 78, 79, 80, 81, 82, 83,		/* PORT13x */
+	84, 85, 86, 87, 88, 89, 90, 91,		/* PORT14x */
+	11, 9, 12, 96, 97, 98, 108, 114,	/* XIRQ0-7 */
+	234, 186, 99, 100, 101, 102, 300, 301,	/* XIRQ8-15 */
+	302, 303, 304, 305, 306, 13, 14, 15,	/* XIRQ16-20, PORT175-177 */
+	157, 158, 156, 154, 150, 151, 152, 153,	/* PORT18x */
+	326, 327, 325, 323, 319, 320, 321, 322,	/* PORT19x */
+	160, 161, 162, 163, 164, 165, 166, 167,	/* PORT20x */
+	168, 169, 170, 171, 172, 173, 174, 175,	/* PORT21x */
+	180, 181, 182, 183, 184, 185, 187, 188,	/* PORT22x */
+	193, 194, 195, 196, 197, 198, 199, 200,	/* PORT23x */
+	191, 192, 215, 216, 217, 218, 219, 220,	/* PORT24x */
+	222, 223, 224, 225, 228, 229, 230, 231,	/* PORT25x */
+	282, 283, 284, 285, 286, 287, 288, 289,	/* PORT26x */
+	292, 293, 294, 295, 296, 236, 237, 238,	/* PORT27x */
+	275, 276, 277, 278, 239, 240, 249, 250,	/* PORT28x */
+	251, 252, 261, 262, 263, 264, 273, 274,	/* PORT29x */
+	31, 32, 33, 34, 35, 36, 37, 38,		/* PORT30x */
 };
 
 static const struct uniphier_pinctrl_group uniphier_pro4_groups[] = {
@@ -1202,267 +1153,13 @@ static const struct uniphier_pinctrl_group uniphier_pro4_groups[] = {
 	UNIPHIER_PINCTRL_GROUP(usb1),
 	UNIPHIER_PINCTRL_GROUP(usb2),
 	UNIPHIER_PINCTRL_GROUP(usb3),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(port_range0),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(port_range1),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_IRQ(xirq),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_IRQ(xirq_alternatives),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port00, port_range0, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port01, port_range0, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port02, port_range0, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port03, port_range0, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port04, port_range0, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port05, port_range0, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port06, port_range0, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port07, port_range0, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port10, port_range0, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port11, port_range0, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port12, port_range0, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port13, port_range0, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port14, port_range0, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port15, port_range0, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port16, port_range0, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port17, port_range0, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port20, port_range0, 16),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port21, port_range0, 17),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port22, port_range0, 18),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port23, port_range0, 19),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port24, port_range0, 20),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port25, port_range0, 21),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port26, port_range0, 22),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port27, port_range0, 23),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port30, port_range0, 24),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port31, port_range0, 25),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port32, port_range0, 26),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port33, port_range0, 27),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port34, port_range0, 28),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port35, port_range0, 29),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port36, port_range0, 30),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port37, port_range0, 31),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port40, port_range0, 32),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port41, port_range0, 33),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port42, port_range0, 34),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port43, port_range0, 35),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port44, port_range0, 36),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port45, port_range0, 37),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port46, port_range0, 38),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port47, port_range0, 39),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port50, port_range0, 40),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port51, port_range0, 41),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port52, port_range0, 42),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port53, port_range0, 43),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port54, port_range0, 44),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port55, port_range0, 45),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port56, port_range0, 46),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port57, port_range0, 47),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port60, port_range0, 48),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port61, port_range0, 49),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port62, port_range0, 50),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port63, port_range0, 51),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port64, port_range0, 52),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port65, port_range0, 53),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port66, port_range0, 54),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port67, port_range0, 55),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port70, port_range0, 56),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port71, port_range0, 57),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port72, port_range0, 58),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port73, port_range0, 59),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port74, port_range0, 60),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port75, port_range0, 61),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port76, port_range0, 62),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port77, port_range0, 63),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port80, port_range0, 64),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port81, port_range0, 65),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port82, port_range0, 66),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port83, port_range0, 67),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port84, port_range0, 68),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port85, port_range0, 69),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port86, port_range0, 70),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port87, port_range0, 71),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port90, port_range0, 72),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port91, port_range0, 73),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port92, port_range0, 74),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port93, port_range0, 75),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port94, port_range0, 76),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port95, port_range0, 77),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port96, port_range0, 78),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port97, port_range0, 79),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port100, port_range0, 80),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port101, port_range0, 81),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port102, port_range0, 82),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port103, port_range0, 83),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port104, port_range0, 84),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port105, port_range0, 85),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port106, port_range0, 86),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port107, port_range0, 87),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port110, port_range0, 88),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port111, port_range0, 89),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port112, port_range0, 90),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port113, port_range0, 91),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port114, port_range0, 92),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port115, port_range0, 93),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port116, port_range0, 94),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port117, port_range0, 95),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port120, port_range0, 96),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port121, port_range0, 97),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port122, port_range0, 98),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port123, port_range0, 99),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port124, port_range0, 100),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port125, port_range0, 101),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port126, port_range0, 102),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port127, port_range0, 103),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port130, port_range0, 104),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port131, port_range0, 105),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port132, port_range0, 106),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port133, port_range0, 107),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port134, port_range0, 108),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port135, port_range0, 109),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port136, port_range0, 110),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port137, port_range0, 111),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port140, port_range0, 112),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port141, port_range0, 113),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port142, port_range0, 114),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port143, port_range0, 115),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port144, port_range0, 116),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port145, port_range0, 117),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port146, port_range0, 118),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port147, port_range0, 119),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port175, port_range1, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port176, port_range1, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port177, port_range1, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port180, port_range1, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port181, port_range1, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port182, port_range1, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port183, port_range1, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port184, port_range1, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port185, port_range1, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port186, port_range1, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port187, port_range1, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port190, port_range1, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port191, port_range1, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port192, port_range1, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port193, port_range1, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port194, port_range1, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port195, port_range1, 16),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port196, port_range1, 17),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port197, port_range1, 18),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port200, port_range1, 19),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port201, port_range1, 20),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port202, port_range1, 21),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port203, port_range1, 22),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port204, port_range1, 23),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port205, port_range1, 24),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port206, port_range1, 25),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port207, port_range1, 26),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port210, port_range1, 27),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port211, port_range1, 28),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port212, port_range1, 29),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port213, port_range1, 30),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port214, port_range1, 31),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port215, port_range1, 32),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port216, port_range1, 33),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port217, port_range1, 34),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port220, port_range1, 35),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port221, port_range1, 36),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port222, port_range1, 37),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port223, port_range1, 38),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port224, port_range1, 39),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port225, port_range1, 40),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port226, port_range1, 41),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port227, port_range1, 42),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port230, port_range1, 43),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port231, port_range1, 44),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port232, port_range1, 45),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port233, port_range1, 46),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port234, port_range1, 47),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port235, port_range1, 48),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port236, port_range1, 49),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port237, port_range1, 50),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port240, port_range1, 51),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port241, port_range1, 52),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port242, port_range1, 53),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port243, port_range1, 54),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port244, port_range1, 55),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port245, port_range1, 56),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port246, port_range1, 57),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port247, port_range1, 58),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port250, port_range1, 59),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port251, port_range1, 60),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port252, port_range1, 61),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port253, port_range1, 62),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port254, port_range1, 63),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port255, port_range1, 64),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port256, port_range1, 65),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port257, port_range1, 66),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port260, port_range1, 67),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port261, port_range1, 68),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port262, port_range1, 69),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port263, port_range1, 70),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port264, port_range1, 71),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port265, port_range1, 72),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port266, port_range1, 73),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port267, port_range1, 74),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port270, port_range1, 75),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port271, port_range1, 76),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port272, port_range1, 77),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port273, port_range1, 78),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port274, port_range1, 79),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port275, port_range1, 80),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port276, port_range1, 81),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port277, port_range1, 82),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port280, port_range1, 83),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port281, port_range1, 84),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port282, port_range1, 85),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port283, port_range1, 86),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port284, port_range1, 87),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port285, port_range1, 88),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port286, port_range1, 89),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port287, port_range1, 90),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port290, port_range1, 91),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port291, port_range1, 92),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port292, port_range1, 93),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port293, port_range1, 94),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port294, port_range1, 95),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port295, port_range1, 96),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port296, port_range1, 97),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port297, port_range1, 98),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port300, port_range1, 99),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port301, port_range1, 100),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port302, port_range1, 101),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port303, port_range1, 102),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port304, port_range1, 103),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port305, port_range1, 104),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port306, port_range1, 105),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port307, port_range1, 106),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq0, xirq, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq1, xirq, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq2, xirq, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq3, xirq, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq4, xirq, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq5, xirq, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq6, xirq, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq7, xirq, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq8, xirq, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq9, xirq, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq10, xirq, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq11, xirq, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq12, xirq, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq13, xirq, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq14, xirq, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq15, xirq, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq16, xirq, 16),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq17, xirq, 17),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq18, xirq, 18),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq19, xirq, 19),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq20, xirq, 20),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq14b, xirq_alternatives, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq17b, xirq_alternatives, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq18b, xirq_alternatives, 2),
+	UNIPHIER_PINCTRL_GROUP_GPIO(gpio_range),
 };
 
 static const char * const emmc_groups[] = {"emmc", "emmc_dat8"};
 static const char * const ether_mii_groups[] = {"ether_mii"};
 static const char * const ether_rgmii_groups[] = {"ether_rgmii"};
-static const char * const ether_rmii_groups[] = {"ether_rgmii", "ether_rgmiib"};
+static const char * const ether_rmii_groups[] = {"ether_rmii", "ether_rmiib"};
 static const char * const i2c0_groups[] = {"i2c0"};
 static const char * const i2c1_groups[] = {"i2c1"};
 static const char * const i2c2_groups[] = {"i2c2"};
@@ -1488,75 +1185,6 @@ static const char * const usb0_groups[] = {"usb0"};
 static const char * const usb1_groups[] = {"usb1"};
 static const char * const usb2_groups[] = {"usb2"};
 static const char * const usb3_groups[] = {"usb3"};
-static const char * const port_groups[] = {
-	"port00",  "port01",  "port02",  "port03",
-	"port04",  "port05",  "port06",  "port07",
-	"port10",  "port11",  "port12",  "port13",
-	"port14",  "port15",  "port16",  "port17",
-	"port20",  "port21",  "port22",  "port23",
-	"port24",  "port25",  "port26",  "port27",
-	"port30",  "port31",  "port32",  "port33",
-	"port34",  "port35",  "port36",  "port37",
-	"port40",  "port41",  "port42",  "port43",
-	"port44",  "port45",  "port46",  "port47",
-	"port50",  "port51",  "port52",  "port53",
-	"port54",  "port55",  "port56",  "port57",
-	"port60",  "port61",  "port62",  "port63",
-	"port64",  "port65",  "port66",  "port67",
-	"port70",  "port71",  "port72",  "port73",
-	"port74",  "port75",  "port76",  "port77",
-	"port80",  "port81",  "port82",  "port83",
-	"port84",  "port85",  "port86",  "port87",
-	"port90",  "port91",  "port92",  "port93",
-	"port94",  "port95",  "port96",  "port97",
-	"port100", "port101", "port102", "port103",
-	"port104", "port105", "port106", "port107",
-	"port110", "port111", "port112", "port113",
-	"port114", "port115", "port116", "port117",
-	"port120", "port121", "port122", "port123",
-	"port124", "port125", "port126", "port127",
-	"port130", "port131", "port132", "port133",
-	"port134", "port135", "port136", "port137",
-	"port140", "port141", "port142", "port143",
-	"port144", "port145", "port146", "port147",
-	/* port150-174 missing */
-	/* none */ "port175", "port176", "port177",
-	"port180", "port181", "port182", "port183",
-	"port184", "port185", "port186", "port187",
-	"port190", "port191", "port192", "port193",
-	"port194", "port195", "port196", "port197",
-	"port200", "port201", "port202", "port203",
-	"port204", "port205", "port206", "port207",
-	"port210", "port211", "port212", "port213",
-	"port214", "port215", "port216", "port217",
-	"port220", "port221", "port222", "port223",
-	"port224", "port225", "port226", "port227",
-	"port230", "port231", "port232", "port233",
-	"port234", "port235", "port236", "port237",
-	"port240", "port241", "port242", "port243",
-	"port244", "port245", "port246", "port247",
-	"port250", "port251", "port252", "port253",
-	"port254", "port255", "port256", "port257",
-	"port260", "port261", "port262", "port263",
-	"port264", "port265", "port266", "port267",
-	"port270", "port271", "port272", "port273",
-	"port274", "port275", "port276", "port277",
-	"port280", "port281", "port282", "port283",
-	"port284", "port285", "port286", "port287",
-	"port290", "port291", "port292", "port293",
-	"port294", "port295", "port296", "port297",
-	"port300", "port301", "port302", "port303",
-	"port304", "port305", "port306", "port307",
-};
-static const char * const xirq_groups[] = {
-	"xirq0",  "xirq1",  "xirq2",  "xirq3",
-	"xirq4",  "xirq5",  "xirq6",  "xirq7",
-	"xirq8",  "xirq9",  "xirq10", "xirq11",
-	"xirq12", "xirq13", "xirq14", "xirq15",
-	"xirq16", "xirq17", "xirq18", "xirq19",
-	"xirq20",
-	"xirq14b", "xirq17b", "xirq18b",
-};
 
 static const struct uniphier_pinmux_function uniphier_pro4_functions[] = {
 	UNIPHIER_PINMUX_FUNCTION(emmc),
@@ -1580,10 +1208,17 @@ static const struct uniphier_pinmux_function uniphier_pro4_functions[] = {
 	UNIPHIER_PINMUX_FUNCTION(usb1),
 	UNIPHIER_PINMUX_FUNCTION(usb2),
 	UNIPHIER_PINMUX_FUNCTION(usb3),
-	UNIPHIER_PINMUX_FUNCTION(port),
-	UNIPHIER_PINMUX_FUNCTION(xirq),
 };
 
+static int uniphier_pro4_get_gpio_muxval(unsigned int pin,
+					 unsigned int gpio_offset)
+{
+	if (gpio_offset >= 134 && gpio_offset <= 140)	/* XIRQ14-20 */
+		return 2;
+
+	return 7;
+}
+
 static struct uniphier_pinctrl_socdata uniphier_pro4_pindata = {
 	.pins = uniphier_pro4_pins,
 	.npins = ARRAY_SIZE(uniphier_pro4_pins),
@@ -1591,6 +1226,7 @@ static struct uniphier_pinctrl_socdata uniphier_pro4_pindata = {
 	.groups_count = ARRAY_SIZE(uniphier_pro4_groups),
 	.functions = uniphier_pro4_functions,
 	.functions_count = ARRAY_SIZE(uniphier_pro4_functions),
+	.get_gpio_muxval = uniphier_pro4_get_gpio_muxval,
 	.caps = UNIPHIER_PINCTRL_CAPS_DBGMUX_SEPARATE,
 };
 
@@ -1609,6 +1245,7 @@ static struct platform_driver uniphier_pro4_pinctrl_driver = {
 	.driver = {
 		.name = "uniphier-pro4-pinctrl",
 		.of_match_table = uniphier_pro4_pinctrl_match,
+		.pm = &uniphier_pinctrl_pm_ops,
 	},
 };
 builtin_platform_driver(uniphier_pro4_pinctrl_driver);
diff --git a/drivers/pinctrl/uniphier/pinctrl-uniphier-pro5.c b/drivers/pinctrl/uniphier/pinctrl-uniphier-pro5.c
index 04d00c398eaf..9381a4ff4389 100644
--- a/drivers/pinctrl/uniphier/pinctrl-uniphier-pro5.c
+++ b/drivers/pinctrl/uniphier/pinctrl-uniphier-pro5.c
@@ -854,87 +854,38 @@ static const unsigned usb1_pins[] = {126, 127};
 static const int usb1_muxvals[] = {0, 0};
 static const unsigned usb2_pins[] = {128, 129};
 static const int usb2_muxvals[] = {0, 0};
-static const unsigned port_range0_pins[] = {
-	89, 90, 91, 92, 93, 94, 95, 96,			/* PORT0x */
-	97, 98, 99, 100, 101, 102, 103, 104,		/* PORT1x */
-	251, 252, 253, 254, 255, 247, 248, 249,		/* PORT2x */
-	39, 40, 41, 42, 43, 44, 45, 46,			/* PORT3x */
-	156, 157, 158, 159, 160, 161, 162, 163,		/* PORT4x */
-	164, 165, 166, 167, 168, 169, 170, 171,		/* PORT5x */
-	190, 191, 192, 193, 194, 195, 196, 197,		/* PORT6x */
-	198, 199, 200, 201, 202, 203, 204, 205,		/* PORT7x */
-	120, 121, 122, 123, 55, 56, 57, 58,		/* PORT8x */
-	124, 125, 126, 127, 49, 50, 53, 54,		/* PORT9x */
-	148, 149, 150, 151, 152, 153, 154, 155,		/* PORT10x */
-	133, 134, 131, 130, 138, 139, 136, 135,		/* PORT11x */
-	28, 29, 30, 31, 32, 33, 34, 35,			/* PORT12x */
-	179, 180, 181, 182, 186, 187, 188, 189,		/* PORT13x */
-	4, 5, 6, 7, 8, 9, 10, 11,			/* PORT14x */
-};
-static const int port_range0_muxvals[] = {
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT0x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT1x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT2x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT3x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT4x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT5x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT6x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT7x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT8x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT9x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT10x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT11x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT12x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT13x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT14x */
-};
-static const unsigned port_range1_pins[] = {
-	109, 110, 111,					/* PORT175-177 */
-	206, 207, 208, 209, 210, 211, 212, 213,		/* PORT18x */
-	12, 13, 14, 15, 16, 17, 107, 108,		/* PORT19x */
-	140, 141, 142, 143, 144, 145, 146, 147,		/* PORT20x */
-	59, 60, 61, 62, 63, 64, 65, 66,			/* PORT21x */
-	214, 215, 216, 217, 218, 219, 220, 221,		/* PORT22x */
-	222, 223, 224, 225, 226, 227, 228, 229,		/* PORT23x */
-	19, 20, 21, 22, 23, 24, 25, 26,			/* PORT24x */
-	230, 231, 232, 233, 234, 235, 236, 237,		/* PORT25x */
-	239, 240, 241, 242, 243, 244, 245, 246,		/* PORT26x */
-	172, 173, 174, 175, 176, 177, 178, 129,		/* PORT27x */
-	0, 1, 2, 67, 85, 86, 87, 88,			/* PORT28x */
-	105, 106, 18, 27, 36, 128, 132, 137,		/* PORT29x */
-	183, 184, 185, 84, 47, 48, 51, 52,		/* PORT30x */
-};
-static const int port_range1_muxvals[] = {
-	15, 15, 15,					/* PORT175-177 */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT18x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT19x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT20x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT21x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT22x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT23x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT24x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT25x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT26x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT27x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT28x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT29x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT30x */
-};
-static const unsigned xirq_pins[] = {
-	68, 69, 70, 71, 72, 73, 74, 75,			/* XIRQ0-7 */
-	76, 77, 78, 79, 80, 81, 82, 83,			/* XIRQ8-15 */
-	84, 85, 86, 87, 88,				/* XIRQ16-20 */
-};
-static const int xirq_muxvals[] = {
-	14, 14, 14, 14, 14, 14, 14, 14,			/* XIRQ0-7 */
-	14, 14, 14, 14, 14, 14, 14, 14,			/* XIRQ8-15 */
-	14, 14, 14, 14, 14,				/* XIRQ16-20 */
-};
-static const unsigned xirq_alternatives_pins[] = {
-	91, 92, 239, 144, 240, 156, 241, 106, 128,
-};
-static const int xirq_alternatives_muxvals[] = {
-	14, 14, 14, 14, 14, 14, 14, 14, 14,
+static const unsigned int gpio_range_pins[] = {
+	89, 90, 91, 92, 93, 94, 95, 96,		/* PORT0x */
+	97, 98, 99, 100, 101, 102, 103, 104,	/* PORT1x */
+	251, 252, 253, 254, 255, 247, 248, 249,	/* PORT2x */
+	39, 40, 41, 42, 43, 44, 45, 46,		/* PORT3x */
+	156, 157, 158, 159, 160, 161, 162, 163,	/* PORT4x */
+	164, 165, 166, 167, 168, 169, 170, 171,	/* PORT5x */
+	190, 191, 192, 193, 194, 195, 196, 197,	/* PORT6x */
+	198, 199, 200, 201, 202, 203, 204, 205,	/* PORT7x */
+	120, 121, 122, 123, 55, 56, 57, 58,	/* PORT8x */
+	124, 125, 126, 127, 49, 50, 53, 54,	/* PORT9x */
+	148, 149, 150, 151, 152, 153, 154, 155,	/* PORT10x */
+	133, 134, 131, 130, 138, 139, 136, 135,	/* PORT11x */
+	28, 29, 30, 31, 32, 33, 34, 35,		/* PORT12x */
+	179, 180, 181, 182, 186, 187, 188, 189,	/* PORT13x */
+	4, 5, 6, 7, 8, 9, 10, 11,		/* PORT14x */
+	68, 69, 70, 71, 72, 73, 74, 75,		/* XIRQ0-7 */
+	76, 77, 78, 79, 80, 81, 82, 83,		/* XIRQ8-15 */
+	84, 85, 86, 87, 88, 109, 110, 111,	/* XIRQ16-20, PORT175-177 */
+	206, 207, 208, 209, 210, 211, 212, 213,	/* PORT18x */
+	12, 13, 14, 15, 16, 17, 107, 108,	/* PORT19x */
+	140, 141, 142, 143, 144, 145, 146, 147,	/* PORT20x */
+	59, 60, 61, 62, 63, 64, 65, 66,		/* PORT21x */
+	214, 215, 216, 217, 218, 219, 220, 221,	/* PORT22x */
+	222, 223, 224, 225, 226, 227, 228, 229,	/* PORT23x */
+	19, 20, 21, 22, 23, 24, 25, 26,		/* PORT24x */
+	230, 231, 232, 233, 234, 235, 236, 237,	/* PORT25x */
+	239, 240, 241, 242, 243, 244, 245, 246,	/* PORT26x */
+	172, 173, 174, 175, 176, 177, 178, 129,	/* PORT27x */
+	0, 1, 2, 67, 85, 86, 87, 88,		/* PORT28x */
+	105, 106, 18, 27, 36, 128, 132, 137,	/* PORT29x */
+	183, 184, 185, 84, 47, 48, 51, 52,	/* PORT30x */
 };
 
 static const struct uniphier_pinctrl_group uniphier_pro5_groups[] = {
@@ -968,267 +919,7 @@ static const struct uniphier_pinctrl_group uniphier_pro5_groups[] = {
 	UNIPHIER_PINCTRL_GROUP(usb0),
 	UNIPHIER_PINCTRL_GROUP(usb1),
 	UNIPHIER_PINCTRL_GROUP(usb2),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(port_range0),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(port_range1),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_IRQ(xirq),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_IRQ(xirq_alternatives),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port00, port_range0, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port01, port_range0, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port02, port_range0, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port03, port_range0, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port04, port_range0, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port05, port_range0, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port06, port_range0, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port07, port_range0, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port10, port_range0, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port11, port_range0, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port12, port_range0, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port13, port_range0, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port14, port_range0, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port15, port_range0, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port16, port_range0, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port17, port_range0, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port20, port_range0, 16),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port21, port_range0, 17),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port22, port_range0, 18),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port23, port_range0, 19),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port24, port_range0, 20),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port25, port_range0, 21),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port26, port_range0, 22),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port27, port_range0, 23),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port30, port_range0, 24),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port31, port_range0, 25),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port32, port_range0, 26),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port33, port_range0, 27),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port34, port_range0, 28),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port35, port_range0, 29),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port36, port_range0, 30),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port37, port_range0, 31),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port40, port_range0, 32),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port41, port_range0, 33),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port42, port_range0, 34),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port43, port_range0, 35),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port44, port_range0, 36),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port45, port_range0, 37),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port46, port_range0, 38),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port47, port_range0, 39),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port50, port_range0, 40),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port51, port_range0, 41),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port52, port_range0, 42),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port53, port_range0, 43),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port54, port_range0, 44),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port55, port_range0, 45),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port56, port_range0, 46),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port57, port_range0, 47),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port60, port_range0, 48),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port61, port_range0, 49),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port62, port_range0, 50),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port63, port_range0, 51),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port64, port_range0, 52),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port65, port_range0, 53),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port66, port_range0, 54),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port67, port_range0, 55),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port70, port_range0, 56),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port71, port_range0, 57),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port72, port_range0, 58),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port73, port_range0, 59),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port74, port_range0, 60),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port75, port_range0, 61),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port76, port_range0, 62),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port77, port_range0, 63),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port80, port_range0, 64),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port81, port_range0, 65),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port82, port_range0, 66),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port83, port_range0, 67),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port84, port_range0, 68),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port85, port_range0, 69),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port86, port_range0, 70),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port87, port_range0, 71),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port90, port_range0, 72),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port91, port_range0, 73),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port92, port_range0, 74),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port93, port_range0, 75),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port94, port_range0, 76),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port95, port_range0, 77),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port96, port_range0, 78),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port97, port_range0, 79),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port100, port_range0, 80),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port101, port_range0, 81),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port102, port_range0, 82),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port103, port_range0, 83),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port104, port_range0, 84),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port105, port_range0, 85),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port106, port_range0, 86),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port107, port_range0, 87),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port110, port_range0, 88),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port111, port_range0, 89),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port112, port_range0, 90),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port113, port_range0, 91),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port114, port_range0, 92),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port115, port_range0, 93),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port116, port_range0, 94),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port117, port_range0, 95),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port120, port_range0, 96),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port121, port_range0, 97),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port122, port_range0, 98),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port123, port_range0, 99),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port124, port_range0, 100),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port125, port_range0, 101),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port126, port_range0, 102),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port127, port_range0, 103),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port130, port_range0, 104),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port131, port_range0, 105),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port132, port_range0, 106),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port133, port_range0, 107),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port134, port_range0, 108),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port135, port_range0, 109),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port136, port_range0, 110),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port137, port_range0, 111),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port140, port_range0, 112),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port141, port_range0, 113),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port142, port_range0, 114),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port143, port_range0, 115),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port144, port_range0, 116),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port145, port_range0, 117),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port146, port_range0, 118),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port147, port_range0, 119),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port175, port_range1, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port176, port_range1, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port177, port_range1, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port180, port_range1, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port181, port_range1, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port182, port_range1, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port183, port_range1, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port184, port_range1, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port185, port_range1, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port186, port_range1, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port187, port_range1, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port190, port_range1, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port191, port_range1, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port192, port_range1, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port193, port_range1, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port194, port_range1, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port195, port_range1, 16),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port196, port_range1, 17),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port197, port_range1, 18),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port200, port_range1, 19),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port201, port_range1, 20),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port202, port_range1, 21),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port203, port_range1, 22),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port204, port_range1, 23),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port205, port_range1, 24),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port206, port_range1, 25),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port207, port_range1, 26),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port210, port_range1, 27),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port211, port_range1, 28),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port212, port_range1, 29),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port213, port_range1, 30),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port214, port_range1, 31),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port215, port_range1, 32),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port216, port_range1, 33),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port217, port_range1, 34),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port220, port_range1, 35),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port221, port_range1, 36),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port222, port_range1, 37),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port223, port_range1, 38),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port224, port_range1, 39),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port225, port_range1, 40),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port226, port_range1, 41),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port227, port_range1, 42),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port230, port_range1, 43),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port231, port_range1, 44),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port232, port_range1, 45),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port233, port_range1, 46),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port234, port_range1, 47),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port235, port_range1, 48),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port236, port_range1, 49),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port237, port_range1, 50),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port240, port_range1, 51),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port241, port_range1, 52),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port242, port_range1, 53),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port243, port_range1, 54),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port244, port_range1, 55),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port245, port_range1, 56),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port246, port_range1, 57),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port247, port_range1, 58),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port250, port_range1, 59),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port251, port_range1, 60),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port252, port_range1, 61),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port253, port_range1, 62),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port254, port_range1, 63),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port255, port_range1, 64),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port256, port_range1, 65),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port257, port_range1, 66),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port260, port_range1, 67),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port261, port_range1, 68),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port262, port_range1, 69),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port263, port_range1, 70),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port264, port_range1, 71),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port265, port_range1, 72),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port266, port_range1, 73),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port267, port_range1, 74),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port270, port_range1, 75),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port271, port_range1, 76),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port272, port_range1, 77),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port273, port_range1, 78),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port274, port_range1, 79),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port275, port_range1, 80),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port276, port_range1, 81),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port277, port_range1, 82),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port280, port_range1, 83),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port281, port_range1, 84),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port282, port_range1, 85),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port283, port_range1, 86),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port284, port_range1, 87),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port285, port_range1, 88),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port286, port_range1, 89),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port287, port_range1, 90),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port290, port_range1, 91),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port291, port_range1, 92),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port292, port_range1, 93),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port293, port_range1, 94),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port294, port_range1, 95),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port295, port_range1, 96),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port296, port_range1, 97),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port297, port_range1, 98),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port300, port_range1, 99),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port301, port_range1, 100),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port302, port_range1, 101),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port303, port_range1, 102),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port304, port_range1, 103),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port305, port_range1, 104),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port306, port_range1, 105),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port307, port_range1, 106),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq0, xirq, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq1, xirq, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq2, xirq, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq3, xirq, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq4, xirq, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq5, xirq, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq6, xirq, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq7, xirq, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq8, xirq, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq9, xirq, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq10, xirq, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq11, xirq, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq12, xirq, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq13, xirq, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq14, xirq, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq15, xirq, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq16, xirq, 16),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq17, xirq, 17),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq18, xirq, 18),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq19, xirq, 19),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq20, xirq, 20),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq3b, xirq_alternatives, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq4b, xirq_alternatives, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq16b, xirq_alternatives, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq17b, xirq_alternatives, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq17c, xirq_alternatives, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq18b, xirq_alternatives, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq18c, xirq_alternatives, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq19b, xirq_alternatives, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq20b, xirq_alternatives, 8),
+	UNIPHIER_PINCTRL_GROUP_GPIO(gpio_range),
 };
 
 static const char * const emmc_groups[] = {"emmc", "emmc_dat8"};
@@ -1256,76 +947,6 @@ static const char * const uart3_groups[] = {"uart3"};
 static const char * const usb0_groups[] = {"usb0"};
 static const char * const usb1_groups[] = {"usb1"};
 static const char * const usb2_groups[] = {"usb2"};
-static const char * const port_groups[] = {
-	"port00",  "port01",  "port02",  "port03",
-	"port04",  "port05",  "port06",  "port07",
-	"port10",  "port11",  "port12",  "port13",
-	"port14",  "port15",  "port16",  "port17",
-	"port20",  "port21",  "port22",  "port23",
-	"port24",  "port25",  "port26",  "port27",
-	"port30",  "port31",  "port32",  "port33",
-	"port34",  "port35",  "port36",  "port37",
-	"port40",  "port41",  "port42",  "port43",
-	"port44",  "port45",  "port46",  "port47",
-	"port50",  "port51",  "port52",  "port53",
-	"port54",  "port55",  "port56",  "port57",
-	"port60",  "port61",  "port62",  "port63",
-	"port64",  "port65",  "port66",  "port67",
-	"port70",  "port71",  "port72",  "port73",
-	"port74",  "port75",  "port76",  "port77",
-	"port80",  "port81",  "port82",  "port83",
-	"port84",  "port85",  "port86",  "port87",
-	"port90",  "port91",  "port92",  "port93",
-	"port94",  "port95",  "port96",  "port97",
-	"port100", "port101", "port102", "port103",
-	"port104", "port105", "port106", "port107",
-	"port110", "port111", "port112", "port113",
-	"port114", "port115", "port116", "port117",
-	"port120", "port121", "port122", "port123",
-	"port124", "port125", "port126", "port127",
-	"port130", "port131", "port132", "port133",
-	"port134", "port135", "port136", "port137",
-	"port140", "port141", "port142", "port143",
-	"port144", "port145", "port146", "port147",
-	/* port150-174 missing */
-	/* none */ "port175", "port176", "port177",
-	"port180", "port181", "port182", "port183",
-	"port184", "port185", "port186", "port187",
-	"port190", "port191", "port192", "port193",
-	"port194", "port195", "port196", "port197",
-	"port200", "port201", "port202", "port203",
-	"port204", "port205", "port206", "port207",
-	"port210", "port211", "port212", "port213",
-	"port214", "port215", "port216", "port217",
-	"port220", "port221", "port222", "port223",
-	"port224", "port225", "port226", "port227",
-	"port230", "port231", "port232", "port233",
-	"port234", "port235", "port236", "port237",
-	"port240", "port241", "port242", "port243",
-	"port244", "port245", "port246", "port247",
-	"port250", "port251", "port252", "port253",
-	"port254", "port255", "port256", "port257",
-	"port260", "port261", "port262", "port263",
-	"port264", "port265", "port266", "port267",
-	"port270", "port271", "port272", "port273",
-	"port274", "port275", "port276", "port277",
-	"port280", "port281", "port282", "port283",
-	"port284", "port285", "port286", "port287",
-	"port290", "port291", "port292", "port293",
-	"port294", "port295", "port296", "port297",
-	"port300", "port301", "port302", "port303",
-	"port304", "port305", "port306", "port307",
-};
-static const char * const xirq_groups[] = {
-	"xirq0",  "xirq1",  "xirq2",  "xirq3",
-	"xirq4",  "xirq5",  "xirq6",  "xirq7",
-	"xirq8",  "xirq9",  "xirq10", "xirq11",
-	"xirq12", "xirq13", "xirq14", "xirq15",
-	"xirq16", "xirq17", "xirq18", "xirq19",
-	"xirq20",
-	"xirq3b", "xirq4b", "xirq16b", "xirq17b", "xirq17c",
-	"xirq18b", "xirq18c", "xirq19b", "xirq20b",
-};
 
 static const struct uniphier_pinmux_function uniphier_pro5_functions[] = {
 	UNIPHIER_PINMUX_FUNCTION(emmc),
@@ -1345,10 +966,17 @@ static const struct uniphier_pinmux_function uniphier_pro5_functions[] = {
 	UNIPHIER_PINMUX_FUNCTION(usb0),
 	UNIPHIER_PINMUX_FUNCTION(usb1),
 	UNIPHIER_PINMUX_FUNCTION(usb2),
-	UNIPHIER_PINMUX_FUNCTION(port),
-	UNIPHIER_PINMUX_FUNCTION(xirq),
 };
 
+static int uniphier_pro5_get_gpio_muxval(unsigned int pin,
+					 unsigned int gpio_offset)
+{
+	if (gpio_offset >= 120 && gpio_offset <= 141)	/* XIRQ0-20 */
+		return 14;
+
+	return 15;
+}
+
 static struct uniphier_pinctrl_socdata uniphier_pro5_pindata = {
 	.pins = uniphier_pro5_pins,
 	.npins = ARRAY_SIZE(uniphier_pro5_pins),
@@ -1356,6 +984,7 @@ static struct uniphier_pinctrl_socdata uniphier_pro5_pindata = {
 	.groups_count = ARRAY_SIZE(uniphier_pro5_groups),
 	.functions = uniphier_pro5_functions,
 	.functions_count = ARRAY_SIZE(uniphier_pro5_functions),
+	.get_gpio_muxval = uniphier_pro5_get_gpio_muxval,
 	.caps = UNIPHIER_PINCTRL_CAPS_DBGMUX_SEPARATE,
 };
 
@@ -1374,6 +1003,7 @@ static struct platform_driver uniphier_pro5_pinctrl_driver = {
 	.driver = {
 		.name = "uniphier-pro5-pinctrl",
 		.of_match_table = uniphier_pro5_pinctrl_match,
+		.pm = &uniphier_pinctrl_pm_ops,
 	},
 };
 builtin_platform_driver(uniphier_pro5_pinctrl_driver);
diff --git a/drivers/pinctrl/uniphier/pinctrl-uniphier-pxs2.c b/drivers/pinctrl/uniphier/pinctrl-uniphier-pxs2.c
index 53b6b774654e..c0ef40ae99a7 100644
--- a/drivers/pinctrl/uniphier/pinctrl-uniphier-pxs2.c
+++ b/drivers/pinctrl/uniphier/pinctrl-uniphier-pxs2.c
@@ -790,7 +790,7 @@ static const unsigned usb2_pins[] = {60, 61};
 static const int usb2_muxvals[] = {8, 8};
 static const unsigned usb3_pins[] = {62, 63};
 static const int usb3_muxvals[] = {8, 8};
-static const unsigned port_range0_pins[] = {
+static const unsigned int gpio_range0_pins[] = {
 	127, 128, 129, 130, 131, 132, 133, 134,		/* PORT0x */
 	135, 136, 137, 138, 139, 140, 141, 142,		/* PORT1x */
 	0, 1, 2, 3, 4, 5, 6, 7,				/* PORT2x */
@@ -803,26 +803,13 @@ static const unsigned port_range0_pins[] = {
 	61, 62, 63, 64, 65, 66, 67, 68,			/* PORT9x */
 	69, 70, 71, 76, 77, 78, 79, 80,			/* PORT10x */
 };
-static const int port_range0_muxvals[] = {
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT0x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT1x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT2x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT3x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT4x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT5x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT6x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT7x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT8x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT9x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT10x */
-};
-static const unsigned port_range1_pins[] = {
+static const unsigned int gpio_range1_pins[] = {
 	81, 82, 83, 84, 85, 86, 87, 88,			/* PORT12x */
 	89, 90, 95, 96, 97, 98, 99, 100,		/* PORT13x */
 	101, 102, 103, 104, 105, 106, 107, 108,		/* PORT14x */
-	118, 119, 120, 121, 122, 123, 124, 125,		/* PORT15x */
-	126, 72, 73, 92, 177, 93, 94, 176,		/* PORT16x */
-	74, 91, 27, 28, 29, 75, 20, 26,			/* PORT17x */
+	118, 119, 120, 121, 122, 123, 124, 125,		/* XIRQ0-7 */
+	126, 72, 73, 92, 177, 93, 94, 176,		/* XIRQ8-15 */
+	74, 91, 27, 28, 29, 75, 20, 26,			/* XIRQ16-23 */
 	109, 110, 111, 112, 113, 114, 115, 116,		/* PORT18x */
 	117, 143, 144, 145, 146, 147, 148, 149,		/* PORT19x */
 	150, 151, 152, 153, 154, 155, 156, 157,		/* PORT20x */
@@ -835,35 +822,6 @@ static const unsigned port_range1_pins[] = {
 	218, 219, 220, 221, 223, 224, 225, 226,		/* PORT27x */
 	227, 228, 229, 230, 231, 232, 233, 234,		/* PORT28x */
 };
-static const int port_range1_muxvals[] = {
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT12x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT13x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT14x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT15x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT16x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT17x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT18x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT19x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT20x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT21x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT22x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT23x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT24x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT25x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT26x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT27x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT28x */
-};
-static const unsigned xirq_pins[] = {
-	118, 119, 120, 121, 122, 123, 124, 125,		/* XIRQ0-7 */
-	126, 72, 73, 92, 177, 93, 94, 176,		/* XIRQ8-15 */
-	74, 91, 27, 28, 29, 75, 20, 26,			/* XIRQ16-23 */
-};
-static const int xirq_muxvals[] = {
-	14, 14, 14, 14, 14, 14, 14, 14,			/* XIRQ0-7 */
-	14, 14, 14, 14, 14, 14, 14, 14,			/* XIRQ8-15 */
-	14, 14, 14, 14, 14, 14, 14, 14,			/* XIRQ16-23 */
-};
 
 static const struct uniphier_pinctrl_group uniphier_pxs2_groups[] = {
 	UNIPHIER_PINCTRL_GROUP(emmc),
@@ -892,257 +850,8 @@ static const struct uniphier_pinctrl_group uniphier_pxs2_groups[] = {
 	UNIPHIER_PINCTRL_GROUP(usb1),
 	UNIPHIER_PINCTRL_GROUP(usb2),
 	UNIPHIER_PINCTRL_GROUP(usb3),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(port_range0),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(port_range1),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_IRQ(xirq),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port00, port_range0, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port01, port_range0, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port02, port_range0, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port03, port_range0, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port04, port_range0, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port05, port_range0, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port06, port_range0, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port07, port_range0, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port10, port_range0, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port11, port_range0, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port12, port_range0, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port13, port_range0, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port14, port_range0, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port15, port_range0, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port16, port_range0, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port17, port_range0, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port20, port_range0, 16),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port21, port_range0, 17),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port22, port_range0, 18),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port23, port_range0, 19),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port24, port_range0, 20),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port25, port_range0, 21),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port26, port_range0, 22),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port27, port_range0, 23),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port30, port_range0, 24),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port31, port_range0, 25),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port32, port_range0, 26),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port33, port_range0, 27),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port34, port_range0, 28),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port35, port_range0, 29),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port36, port_range0, 30),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port37, port_range0, 31),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port40, port_range0, 32),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port41, port_range0, 33),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port42, port_range0, 34),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port43, port_range0, 35),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port44, port_range0, 36),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port45, port_range0, 37),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port46, port_range0, 38),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port47, port_range0, 39),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port50, port_range0, 40),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port51, port_range0, 41),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port52, port_range0, 42),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port53, port_range0, 43),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port54, port_range0, 44),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port55, port_range0, 45),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port56, port_range0, 46),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port57, port_range0, 47),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port60, port_range0, 48),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port61, port_range0, 49),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port62, port_range0, 50),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port63, port_range0, 51),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port64, port_range0, 52),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port65, port_range0, 53),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port66, port_range0, 54),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port67, port_range0, 55),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port70, port_range0, 56),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port71, port_range0, 57),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port72, port_range0, 58),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port73, port_range0, 59),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port74, port_range0, 60),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port75, port_range0, 61),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port76, port_range0, 62),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port77, port_range0, 63),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port80, port_range0, 64),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port81, port_range0, 65),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port82, port_range0, 66),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port83, port_range0, 67),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port84, port_range0, 68),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port85, port_range0, 69),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port86, port_range0, 70),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port87, port_range0, 71),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port90, port_range0, 72),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port91, port_range0, 73),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port92, port_range0, 74),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port93, port_range0, 75),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port94, port_range0, 76),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port95, port_range0, 77),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port96, port_range0, 78),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port97, port_range0, 79),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port100, port_range0, 80),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port101, port_range0, 81),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port102, port_range0, 82),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port103, port_range0, 83),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port104, port_range0, 84),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port105, port_range0, 85),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port106, port_range0, 86),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port107, port_range0, 87),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port120, port_range1, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port121, port_range1, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port122, port_range1, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port123, port_range1, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port124, port_range1, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port125, port_range1, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port126, port_range1, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port127, port_range1, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port130, port_range1, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port131, port_range1, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port132, port_range1, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port133, port_range1, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port134, port_range1, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port135, port_range1, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port136, port_range1, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port137, port_range1, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port140, port_range1, 16),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port141, port_range1, 17),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port142, port_range1, 18),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port143, port_range1, 19),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port144, port_range1, 20),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port145, port_range1, 21),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port146, port_range1, 22),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port147, port_range1, 23),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port150, port_range1, 24),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port151, port_range1, 25),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port152, port_range1, 26),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port153, port_range1, 27),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port154, port_range1, 28),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port155, port_range1, 29),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port156, port_range1, 30),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port157, port_range1, 31),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port160, port_range1, 32),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port161, port_range1, 33),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port162, port_range1, 34),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port163, port_range1, 35),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port164, port_range1, 36),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port165, port_range1, 37),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port166, port_range1, 38),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port167, port_range1, 39),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port170, port_range1, 40),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port171, port_range1, 41),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port172, port_range1, 42),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port173, port_range1, 43),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port174, port_range1, 44),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port175, port_range1, 45),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port176, port_range1, 46),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port177, port_range1, 47),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port180, port_range1, 48),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port181, port_range1, 49),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port182, port_range1, 50),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port183, port_range1, 51),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port184, port_range1, 52),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port185, port_range1, 53),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port186, port_range1, 54),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port187, port_range1, 55),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port190, port_range1, 56),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port191, port_range1, 57),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port192, port_range1, 58),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port193, port_range1, 59),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port194, port_range1, 60),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port195, port_range1, 61),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port196, port_range1, 62),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port197, port_range1, 63),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port200, port_range1, 64),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port201, port_range1, 65),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port202, port_range1, 66),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port203, port_range1, 67),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port204, port_range1, 68),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port205, port_range1, 69),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port206, port_range1, 70),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port207, port_range1, 71),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port210, port_range1, 72),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port211, port_range1, 73),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port212, port_range1, 74),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port213, port_range1, 75),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port214, port_range1, 76),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port215, port_range1, 77),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port216, port_range1, 78),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port217, port_range1, 79),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port220, port_range1, 80),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port221, port_range1, 81),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port222, port_range1, 82),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port223, port_range1, 83),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port224, port_range1, 84),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port225, port_range1, 85),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port226, port_range1, 86),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port227, port_range1, 87),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port230, port_range1, 88),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port231, port_range1, 89),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port232, port_range1, 90),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port233, port_range1, 91),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port234, port_range1, 92),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port235, port_range1, 93),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port236, port_range1, 94),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port237, port_range1, 95),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port240, port_range1, 96),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port241, port_range1, 97),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port242, port_range1, 98),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port243, port_range1, 99),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port244, port_range1, 100),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port245, port_range1, 101),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port246, port_range1, 102),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port247, port_range1, 103),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port250, port_range1, 104),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port251, port_range1, 105),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port252, port_range1, 106),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port253, port_range1, 107),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port254, port_range1, 108),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port255, port_range1, 109),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port256, port_range1, 110),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port257, port_range1, 111),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port260, port_range1, 112),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port261, port_range1, 113),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port262, port_range1, 114),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port263, port_range1, 115),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port264, port_range1, 116),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port265, port_range1, 117),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port266, port_range1, 118),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port267, port_range1, 119),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port270, port_range1, 120),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port271, port_range1, 121),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port272, port_range1, 122),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port273, port_range1, 123),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port274, port_range1, 124),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port275, port_range1, 125),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port276, port_range1, 126),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port277, port_range1, 127),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port280, port_range1, 128),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port281, port_range1, 129),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port282, port_range1, 130),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port283, port_range1, 131),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port284, port_range1, 132),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port285, port_range1, 133),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port286, port_range1, 134),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port287, port_range1, 135),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq0, xirq, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq1, xirq, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq2, xirq, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq3, xirq, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq4, xirq, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq5, xirq, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq6, xirq, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq7, xirq, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq8, xirq, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq9, xirq, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq10, xirq, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq11, xirq, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq12, xirq, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq13, xirq, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq14, xirq, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq15, xirq, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq16, xirq, 16),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq17, xirq, 17),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq18, xirq, 18),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq19, xirq, 19),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq20, xirq, 20),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq21, xirq, 21),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq22, xirq, 22),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq23, xirq, 23),
+	UNIPHIER_PINCTRL_GROUP_GPIO(gpio_range0),
+	UNIPHIER_PINCTRL_GROUP_GPIO(gpio_range1),
 };
 
 static const char * const emmc_groups[] = {"emmc", "emmc_dat8"};
@@ -1167,73 +876,6 @@ static const char * const usb0_groups[] = {"usb0"};
 static const char * const usb1_groups[] = {"usb1"};
 static const char * const usb2_groups[] = {"usb2"};
 static const char * const usb3_groups[] = {"usb3"};
-static const char * const port_groups[] = {
-	"port00",  "port01",  "port02",  "port03",
-	"port04",  "port05",  "port06",  "port07",
-	"port10",  "port11",  "port12",  "port13",
-	"port14",  "port15",  "port16",  "port17",
-	"port20",  "port21",  "port22",  "port23",
-	"port24",  "port25",  "port26",  "port27",
-	"port30",  "port31",  "port32",  "port33",
-	"port34",  "port35",  "port36",  "port37",
-	"port40",  "port41",  "port42",  "port43",
-	"port44",  "port45",  "port46",  "port47",
-	"port50",  "port51",  "port52",  "port53",
-	"port54",  "port55",  "port56",  "port57",
-	"port60",  "port61",  "port62",  "port63",
-	"port64",  "port65",  "port66",  "port67",
-	"port70",  "port71",  "port72",  "port73",
-	"port74",  "port75",  "port76",  "port77",
-	"port80",  "port81",  "port82",  "port83",
-	"port84",  "port85",  "port86",  "port87",
-	"port90",  "port91",  "port92",  "port93",
-	"port94",  "port95",  "port96",  "port97",
-	"port100", "port101", "port102", "port103",
-	"port104", "port105", "port106", "port107",
-	/* port110-117 missing */
-	"port120", "port121", "port122", "port123",
-	"port124", "port125", "port126", "port127",
-	"port130", "port131", "port132", "port133",
-	"port134", "port135", "port136", "port137",
-	"port140", "port141", "port142", "port143",
-	"port144", "port145", "port146", "port147",
-	"port150", "port151", "port152", "port153",
-	"port154", "port155", "port156", "port157",
-	"port160", "port161", "port162", "port163",
-	"port164", "port165", "port166", "port167",
-	"port170", "port171", "port172", "port173",
-	"port174", "port175", "port176", "port177",
-	"port180", "port181", "port182", "port183",
-	"port184", "port185", "port186", "port187",
-	"port190", "port191", "port192", "port193",
-	"port194", "port195", "port196", "port197",
-	"port200", "port201", "port202", "port203",
-	"port204", "port205", "port206", "port207",
-	"port210", "port211", "port212", "port213",
-	"port214", "port215", "port216", "port217",
-	"port220", "port221", "port222", "port223",
-	"port224", "port225", "port226", "port227",
-	"port230", "port231", "port232", "port233",
-	"port234", "port235", "port236", "port237",
-	"port240", "port241", "port242", "port243",
-	"port244", "port245", "port246", "port247",
-	"port250", "port251", "port252", "port253",
-	"port254", "port255", "port256", "port257",
-	"port260", "port261", "port262", "port263",
-	"port264", "port265", "port266", "port267",
-	"port270", "port271", "port272", "port273",
-	"port274", "port275", "port276", "port277",
-	"port280", "port281", "port282", "port283",
-	"port284", "port285", "port286", "port287",
-};
-static const char * const xirq_groups[] = {
-	"xirq0",  "xirq1",  "xirq2",  "xirq3",
-	"xirq4",  "xirq5",  "xirq6",  "xirq7",
-	"xirq8",  "xirq9",  "xirq10", "xirq11",
-	"xirq12", "xirq13", "xirq14", "xirq15",
-	"xirq16", "xirq17", "xirq18", "xirq19",
-	"xirq20", "xirq21", "xirq22", "xirq23",
-};
 
 static const struct uniphier_pinmux_function uniphier_pxs2_functions[] = {
 	UNIPHIER_PINMUX_FUNCTION(emmc),
@@ -1257,10 +899,18 @@ static const struct uniphier_pinmux_function uniphier_pxs2_functions[] = {
 	UNIPHIER_PINMUX_FUNCTION(usb1),
 	UNIPHIER_PINMUX_FUNCTION(usb2),
 	UNIPHIER_PINMUX_FUNCTION(usb3),
-	UNIPHIER_PINMUX_FUNCTION(port),
-	UNIPHIER_PINMUX_FUNCTION(xirq),
 };
 
+static int uniphier_pxs2_get_gpio_muxval(unsigned int pin,
+					 unsigned int gpio_offset)
+{
+	if (gpio_offset >= 120 && gpio_offset <= 143)	/* XIRQx */
+		/* 15 will do because XIRQ0-23 are aliases of PORT150-177. */
+		return 14;
+
+	return 15;
+}
+
 static struct uniphier_pinctrl_socdata uniphier_pxs2_pindata = {
 	.pins = uniphier_pxs2_pins,
 	.npins = ARRAY_SIZE(uniphier_pxs2_pins),
@@ -1268,6 +918,7 @@ static struct uniphier_pinctrl_socdata uniphier_pxs2_pindata = {
 	.groups_count = ARRAY_SIZE(uniphier_pxs2_groups),
 	.functions = uniphier_pxs2_functions,
 	.functions_count = ARRAY_SIZE(uniphier_pxs2_functions),
+	.get_gpio_muxval = uniphier_pxs2_get_gpio_muxval,
 	.caps = 0,
 };
 
@@ -1286,6 +937,7 @@ static struct platform_driver uniphier_pxs2_pinctrl_driver = {
 	.driver = {
 		.name = "uniphier-pxs2-pinctrl",
 		.of_match_table = uniphier_pxs2_pinctrl_match,
+		.pm = &uniphier_pinctrl_pm_ops,
 	},
 };
 builtin_platform_driver(uniphier_pxs2_pinctrl_driver);
diff --git a/drivers/pinctrl/uniphier/pinctrl-uniphier-pxs3.c b/drivers/pinctrl/uniphier/pinctrl-uniphier-pxs3.c
new file mode 100644
index 000000000000..d9f166f0cc86
--- /dev/null
+++ b/drivers/pinctrl/uniphier/pinctrl-uniphier-pxs3.c
@@ -0,0 +1,989 @@
+/*
+ * Copyright (C) 2017 Socionext Inc.
+ *   Author: Masahiro Yamada <yamada.masahiro@socionext.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/pinctrl/pinctrl.h>
+#include <linux/platform_device.h>
+
+#include "pinctrl-uniphier.h"
+
+static const struct pinctrl_pin_desc uniphier_pxs3_pins[] = {
+	UNIPHIER_PINCTRL_PIN(0, "LPST", UNIPHIER_PIN_IECTRL_EXIST,
+			     0, UNIPHIER_PIN_DRV_3BIT,
+			     0, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(1, "ED0", UNIPHIER_PIN_IECTRL_EXIST,
+			     1, UNIPHIER_PIN_DRV_3BIT,
+			     1, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(2, "ED1", UNIPHIER_PIN_IECTRL_EXIST,
+			     2, UNIPHIER_PIN_DRV_3BIT,
+			     2, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(3, "ED2", UNIPHIER_PIN_IECTRL_EXIST,
+			     3, UNIPHIER_PIN_DRV_3BIT,
+			     3, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(4, "ED3", UNIPHIER_PIN_IECTRL_EXIST,
+			     4, UNIPHIER_PIN_DRV_3BIT,
+			     4, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(5, "ED4", UNIPHIER_PIN_IECTRL_EXIST,
+			     5, UNIPHIER_PIN_DRV_3BIT,
+			     5, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(6, "ED5", UNIPHIER_PIN_IECTRL_EXIST,
+			     6, UNIPHIER_PIN_DRV_3BIT,
+			     6, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(7, "ED6", UNIPHIER_PIN_IECTRL_EXIST,
+			     7, UNIPHIER_PIN_DRV_3BIT,
+			     7, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(8, "ED7", UNIPHIER_PIN_IECTRL_EXIST,
+			     8, UNIPHIER_PIN_DRV_3BIT,
+			     8, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(9, "XERWE0", UNIPHIER_PIN_IECTRL_EXIST,
+			     9, UNIPHIER_PIN_DRV_3BIT,
+			     9, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(10, "XERWE1", UNIPHIER_PIN_IECTRL_EXIST,
+			     10, UNIPHIER_PIN_DRV_3BIT,
+			     10, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(11, "ERXW", UNIPHIER_PIN_IECTRL_EXIST,
+			     11, UNIPHIER_PIN_DRV_3BIT,
+			     11, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(12, "ES0", UNIPHIER_PIN_IECTRL_EXIST,
+			     12, UNIPHIER_PIN_DRV_3BIT,
+			     12, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(13, "ES1", UNIPHIER_PIN_IECTRL_EXIST,
+			     13, UNIPHIER_PIN_DRV_3BIT,
+			     13, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(14, "ES2", UNIPHIER_PIN_IECTRL_EXIST,
+			     14, UNIPHIER_PIN_DRV_3BIT,
+			     14, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(15, "XECS1", UNIPHIER_PIN_IECTRL_EXIST,
+			     15, UNIPHIER_PIN_DRV_3BIT,
+			     15, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(16, "XNFWP", UNIPHIER_PIN_IECTRL_EXIST,
+			     16, UNIPHIER_PIN_DRV_3BIT,
+			     16, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(17, "XNFCE0", UNIPHIER_PIN_IECTRL_EXIST,
+			     17, UNIPHIER_PIN_DRV_3BIT,
+			     17, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(18, "NFRYBY0", UNIPHIER_PIN_IECTRL_EXIST,
+			     18, UNIPHIER_PIN_DRV_3BIT,
+			     18, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(19, "XNFRE", UNIPHIER_PIN_IECTRL_EXIST,
+			     19, UNIPHIER_PIN_DRV_3BIT,
+			     19, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(20, "XNFWE", UNIPHIER_PIN_IECTRL_EXIST,
+			     20, UNIPHIER_PIN_DRV_3BIT,
+			     20, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(21, "NFALE", UNIPHIER_PIN_IECTRL_EXIST,
+			     21, UNIPHIER_PIN_DRV_3BIT,
+			     21, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(22, "NFCLE", UNIPHIER_PIN_IECTRL_EXIST,
+			     22, UNIPHIER_PIN_DRV_3BIT,
+			     22, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(23, "NFD0", UNIPHIER_PIN_IECTRL_EXIST,
+			     23, UNIPHIER_PIN_DRV_3BIT,
+			     23, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(24, "NFD1", UNIPHIER_PIN_IECTRL_EXIST,
+			     24, UNIPHIER_PIN_DRV_3BIT,
+			     24, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(25, "NFD2", UNIPHIER_PIN_IECTRL_EXIST,
+			     25, UNIPHIER_PIN_DRV_3BIT,
+			     25, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(26, "NFD3", UNIPHIER_PIN_IECTRL_EXIST,
+			     26, UNIPHIER_PIN_DRV_3BIT,
+			     26, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(27, "NFD4", UNIPHIER_PIN_IECTRL_EXIST,
+			     27, UNIPHIER_PIN_DRV_3BIT,
+			     27, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(28, "NFD5", UNIPHIER_PIN_IECTRL_EXIST,
+			     28, UNIPHIER_PIN_DRV_3BIT,
+			     28, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(29, "NFD6", UNIPHIER_PIN_IECTRL_EXIST,
+			     29, UNIPHIER_PIN_DRV_3BIT,
+			     29, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(30, "NFD7", UNIPHIER_PIN_IECTRL_EXIST,
+			     30, UNIPHIER_PIN_DRV_3BIT,
+			     30, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(31, "XERST", UNIPHIER_PIN_IECTRL_EXIST,
+			     0, UNIPHIER_PIN_DRV_2BIT,
+			     31, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(32, "MMCCLK", UNIPHIER_PIN_IECTRL_EXIST,
+			     1, UNIPHIER_PIN_DRV_2BIT,
+			     32, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(33, "MMCCMD", UNIPHIER_PIN_IECTRL_EXIST,
+			     2, UNIPHIER_PIN_DRV_2BIT,
+			     33, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(34, "MMCDS", UNIPHIER_PIN_IECTRL_EXIST,
+			     3, UNIPHIER_PIN_DRV_2BIT,
+			     34, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(35, "MMCDAT0", UNIPHIER_PIN_IECTRL_EXIST,
+			     4, UNIPHIER_PIN_DRV_2BIT,
+			     35, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(36, "MMCDAT1", UNIPHIER_PIN_IECTRL_EXIST,
+			     5, UNIPHIER_PIN_DRV_2BIT,
+			     36, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(37, "MMCDAT2", UNIPHIER_PIN_IECTRL_EXIST,
+			     6, UNIPHIER_PIN_DRV_2BIT,
+			     37, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(38, "MMCDAT3", UNIPHIER_PIN_IECTRL_EXIST,
+			     7, UNIPHIER_PIN_DRV_2BIT,
+			     38, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(39, "MMCDAT4", UNIPHIER_PIN_IECTRL_EXIST,
+			     8, UNIPHIER_PIN_DRV_2BIT,
+			     39, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(40, "MMCDAT5", UNIPHIER_PIN_IECTRL_EXIST,
+			     9, UNIPHIER_PIN_DRV_2BIT,
+			     40, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(41, "MMCDAT6", UNIPHIER_PIN_IECTRL_EXIST,
+			     10, UNIPHIER_PIN_DRV_2BIT,
+			     41, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(42, "MMCDAT7", UNIPHIER_PIN_IECTRL_EXIST,
+			     11, UNIPHIER_PIN_DRV_2BIT,
+			     42, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(43, "SDCLK", UNIPHIER_PIN_IECTRL_EXIST,
+			     12, UNIPHIER_PIN_DRV_2BIT,
+			     43, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(44, "SDCMD", UNIPHIER_PIN_IECTRL_EXIST,
+			     13, UNIPHIER_PIN_DRV_2BIT,
+			     44, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(45, "SDDAT0", UNIPHIER_PIN_IECTRL_EXIST,
+			     14, UNIPHIER_PIN_DRV_2BIT,
+			     45, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(46, "SDDAT1", UNIPHIER_PIN_IECTRL_EXIST,
+			     15, UNIPHIER_PIN_DRV_2BIT,
+			     46, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(47, "SDDAT2", UNIPHIER_PIN_IECTRL_EXIST,
+			     16, UNIPHIER_PIN_DRV_2BIT,
+			     47, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(48, "SDDAT3", UNIPHIER_PIN_IECTRL_EXIST,
+			     17, UNIPHIER_PIN_DRV_2BIT,
+			     48, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(49, "SDCD", UNIPHIER_PIN_IECTRL_EXIST,
+			     31, UNIPHIER_PIN_DRV_3BIT,
+			     49, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(50, "SDWP", UNIPHIER_PIN_IECTRL_EXIST,
+			     32, UNIPHIER_PIN_DRV_3BIT,
+			     50, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(51, "SDVOLC", UNIPHIER_PIN_IECTRL_EXIST,
+			     33, UNIPHIER_PIN_DRV_3BIT,
+			     51, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(52, "MDC0", UNIPHIER_PIN_IECTRL_EXIST,
+			     18, UNIPHIER_PIN_DRV_2BIT,
+			     52, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(53, "MDIO0", UNIPHIER_PIN_IECTRL_EXIST,
+			     19, UNIPHIER_PIN_DRV_2BIT,
+			     53, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(54, "MDIO0_INTL", UNIPHIER_PIN_IECTRL_EXIST,
+			     20, UNIPHIER_PIN_DRV_2BIT,
+			     54, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(55, "PHYRSTL0", UNIPHIER_PIN_IECTRL_EXIST,
+			     21, UNIPHIER_PIN_DRV_2BIT,
+			     55, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(56, "RGMII0_RXCLK", UNIPHIER_PIN_IECTRL_EXIST,
+			     22, UNIPHIER_PIN_DRV_2BIT,
+			     56, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(57, "RGMII0_RXD0", UNIPHIER_PIN_IECTRL_EXIST,
+			     23, UNIPHIER_PIN_DRV_2BIT,
+			     57, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(58, "RGMII0_RXD1", UNIPHIER_PIN_IECTRL_EXIST,
+			     24, UNIPHIER_PIN_DRV_2BIT,
+			     58, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(59, "RGMII0_RXD2", UNIPHIER_PIN_IECTRL_EXIST,
+			     25, UNIPHIER_PIN_DRV_2BIT,
+			     59, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(60, "RGMII0_RXD3", UNIPHIER_PIN_IECTRL_EXIST,
+			     26, UNIPHIER_PIN_DRV_2BIT,
+			     60, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(61, "RGMII0_RXCTL", UNIPHIER_PIN_IECTRL_EXIST,
+			     27, UNIPHIER_PIN_DRV_2BIT,
+			     61, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(62, "RGMII0_TXCLK", UNIPHIER_PIN_IECTRL_EXIST,
+			     28, UNIPHIER_PIN_DRV_2BIT,
+			     62, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(63, "RGMII0_TXD0", UNIPHIER_PIN_IECTRL_EXIST,
+			     29, UNIPHIER_PIN_DRV_2BIT,
+			     63, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(64, "RGMII0_TXD1", UNIPHIER_PIN_IECTRL_EXIST,
+			     30, UNIPHIER_PIN_DRV_2BIT,
+			     64, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(65, "RGMII0_TXD2", UNIPHIER_PIN_IECTRL_EXIST,
+			     31, UNIPHIER_PIN_DRV_2BIT,
+			     65, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(66, "RGMII0_TXD3", UNIPHIER_PIN_IECTRL_EXIST,
+			     32, UNIPHIER_PIN_DRV_2BIT,
+			     66, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(67, "RGMII0_TXCTL", UNIPHIER_PIN_IECTRL_EXIST,
+			     33, UNIPHIER_PIN_DRV_2BIT,
+			     67, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(68, "MDC1", UNIPHIER_PIN_IECTRL_EXIST,
+			     34, UNIPHIER_PIN_DRV_2BIT,
+			     68, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(69, "MDIO1", UNIPHIER_PIN_IECTRL_EXIST,
+			     35, UNIPHIER_PIN_DRV_2BIT,
+			     69, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(70, "MDIO1_INTL", UNIPHIER_PIN_IECTRL_EXIST,
+			     36, UNIPHIER_PIN_DRV_2BIT,
+			     70, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(71, "PHYRSTL1", UNIPHIER_PIN_IECTRL_EXIST,
+			     37, UNIPHIER_PIN_DRV_2BIT,
+			     71, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(72, "RGMII1_RXCLK", UNIPHIER_PIN_IECTRL_EXIST,
+			     38, UNIPHIER_PIN_DRV_2BIT,
+			     72, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(73, "RGMII1_RXD0", UNIPHIER_PIN_IECTRL_EXIST,
+			     39, UNIPHIER_PIN_DRV_2BIT,
+			     73, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(74, "RGMII1_RXD1", UNIPHIER_PIN_IECTRL_EXIST,
+			     40, UNIPHIER_PIN_DRV_2BIT,
+			     74, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(75, "RGMII1_RXD2", UNIPHIER_PIN_IECTRL_EXIST,
+			     41, UNIPHIER_PIN_DRV_2BIT,
+			     75, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(76, "RGMII1_RXD3", UNIPHIER_PIN_IECTRL_EXIST,
+			     42, UNIPHIER_PIN_DRV_2BIT,
+			     76, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(77, "RGMII1_RXCTL", UNIPHIER_PIN_IECTRL_EXIST,
+			     43, UNIPHIER_PIN_DRV_2BIT,
+			     77, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(78, "RGMII1_TXCLK", UNIPHIER_PIN_IECTRL_EXIST,
+			     44, UNIPHIER_PIN_DRV_2BIT,
+			     78, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(79, "RGMII1_TXD0", UNIPHIER_PIN_IECTRL_EXIST,
+			     45, UNIPHIER_PIN_DRV_2BIT,
+			     79, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(80, "RGMII1_TXD1", UNIPHIER_PIN_IECTRL_EXIST,
+			     46, UNIPHIER_PIN_DRV_2BIT,
+			     80, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(81, "RGMII1_TXD2", UNIPHIER_PIN_IECTRL_EXIST,
+			     47, UNIPHIER_PIN_DRV_2BIT,
+			     81, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(82, "RGMII1_TXD3", UNIPHIER_PIN_IECTRL_EXIST,
+			     48, UNIPHIER_PIN_DRV_2BIT,
+			     82, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(83, "RGMII1_TXCTL", UNIPHIER_PIN_IECTRL_EXIST,
+			     49, UNIPHIER_PIN_DRV_2BIT,
+			     83, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(84, "USB0VBUS", UNIPHIER_PIN_IECTRL_EXIST,
+			     34, UNIPHIER_PIN_DRV_3BIT,
+			     84, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(85, "USB0OD", UNIPHIER_PIN_IECTRL_EXIST,
+			     35, UNIPHIER_PIN_DRV_3BIT,
+			     85, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(86, "USB1VBUS", UNIPHIER_PIN_IECTRL_EXIST,
+			     36, UNIPHIER_PIN_DRV_3BIT,
+			     86, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(87, "USB1OD", UNIPHIER_PIN_IECTRL_EXIST,
+			     37, UNIPHIER_PIN_DRV_3BIT,
+			     87, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(88, "USB2VBUS", UNIPHIER_PIN_IECTRL_EXIST,
+			     38, UNIPHIER_PIN_DRV_3BIT,
+			     88, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(89, "USB2OD", UNIPHIER_PIN_IECTRL_EXIST,
+			     39, UNIPHIER_PIN_DRV_3BIT,
+			     89, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(90, "USB3VBUS", UNIPHIER_PIN_IECTRL_EXIST,
+			     40, UNIPHIER_PIN_DRV_3BIT,
+			     90, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(91, "USB3OD", UNIPHIER_PIN_IECTRL_EXIST,
+			     41, UNIPHIER_PIN_DRV_3BIT,
+			     91, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(92, "TXD0", UNIPHIER_PIN_IECTRL_EXIST,
+			     42, UNIPHIER_PIN_DRV_3BIT,
+			     92, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(93, "RXD0", UNIPHIER_PIN_IECTRL_EXIST,
+			     43, UNIPHIER_PIN_DRV_3BIT,
+			     93, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(94, "TXD1", UNIPHIER_PIN_IECTRL_EXIST,
+			     44, UNIPHIER_PIN_DRV_3BIT,
+			     94, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(95, "RXD1", UNIPHIER_PIN_IECTRL_EXIST,
+			     45, UNIPHIER_PIN_DRV_3BIT,
+			     95, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(96, "TXD2", UNIPHIER_PIN_IECTRL_EXIST,
+			     46, UNIPHIER_PIN_DRV_3BIT,
+			     96, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(97, "RXD2", UNIPHIER_PIN_IECTRL_EXIST,
+			     47, UNIPHIER_PIN_DRV_3BIT,
+			     97, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(98, "TXD3", UNIPHIER_PIN_IECTRL_EXIST,
+			     48, UNIPHIER_PIN_DRV_3BIT,
+			     98, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(99, "RXD3", UNIPHIER_PIN_IECTRL_EXIST,
+			     49, UNIPHIER_PIN_DRV_3BIT,
+			     99, UNIPHIER_PIN_PULL_UP),
+	UNIPHIER_PINCTRL_PIN(100, "SPISYNC0", UNIPHIER_PIN_IECTRL_EXIST,
+			     50, UNIPHIER_PIN_DRV_3BIT,
+			     100, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(101, "SPISCLK0", UNIPHIER_PIN_IECTRL_EXIST,
+			     51, UNIPHIER_PIN_DRV_3BIT,
+			     101, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(102, "SPITXD0", UNIPHIER_PIN_IECTRL_EXIST,
+			     52, UNIPHIER_PIN_DRV_3BIT,
+			     102, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(103, "SPIRXD0", UNIPHIER_PIN_IECTRL_EXIST,
+			     53, UNIPHIER_PIN_DRV_3BIT,
+			     103, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(104, "SDA0", UNIPHIER_PIN_IECTRL_EXIST,
+			     -1, UNIPHIER_PIN_DRV_FIXED4,
+			     -1, UNIPHIER_PIN_PULL_NONE),
+	UNIPHIER_PINCTRL_PIN(105, "SCL0", UNIPHIER_PIN_IECTRL_EXIST,
+			     -1, UNIPHIER_PIN_DRV_FIXED4,
+			     -1, UNIPHIER_PIN_PULL_NONE),
+	UNIPHIER_PINCTRL_PIN(106, "SDA1", UNIPHIER_PIN_IECTRL_EXIST,
+			     -1, UNIPHIER_PIN_DRV_FIXED4,
+			     -1, UNIPHIER_PIN_PULL_NONE),
+	UNIPHIER_PINCTRL_PIN(107, "SCL1", UNIPHIER_PIN_IECTRL_EXIST,
+			     -1, UNIPHIER_PIN_DRV_FIXED4,
+			     -1, UNIPHIER_PIN_PULL_NONE),
+	UNIPHIER_PINCTRL_PIN(108, "SDA2", UNIPHIER_PIN_IECTRL_EXIST,
+			     -1, UNIPHIER_PIN_DRV_FIXED4,
+			     -1, UNIPHIER_PIN_PULL_NONE),
+	UNIPHIER_PINCTRL_PIN(109, "SCL2", UNIPHIER_PIN_IECTRL_EXIST,
+			     -1, UNIPHIER_PIN_DRV_FIXED4,
+			     -1, UNIPHIER_PIN_PULL_NONE),
+	UNIPHIER_PINCTRL_PIN(110, "SDA3", UNIPHIER_PIN_IECTRL_EXIST,
+			     -1, UNIPHIER_PIN_DRV_FIXED4,
+			     -1, UNIPHIER_PIN_PULL_NONE),
+	UNIPHIER_PINCTRL_PIN(111, "SCL3", UNIPHIER_PIN_IECTRL_EXIST,
+			     -1, UNIPHIER_PIN_DRV_FIXED4,
+			     -1, UNIPHIER_PIN_PULL_NONE),
+	UNIPHIER_PINCTRL_PIN(112, "SMTRST0", UNIPHIER_PIN_IECTRL_EXIST,
+			     54, UNIPHIER_PIN_DRV_3BIT,
+			     112, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(113, "SMTCMD0", UNIPHIER_PIN_IECTRL_EXIST,
+			     55, UNIPHIER_PIN_DRV_3BIT,
+			     113, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(114, "SMTD0", UNIPHIER_PIN_IECTRL_EXIST,
+			     56, UNIPHIER_PIN_DRV_3BIT,
+			     114, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(115, "SMTSEL0", UNIPHIER_PIN_IECTRL_EXIST,
+			     57, UNIPHIER_PIN_DRV_3BIT,
+			     115, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(116, "SMTCLK0CG", UNIPHIER_PIN_IECTRL_EXIST,
+			     58, UNIPHIER_PIN_DRV_3BIT,
+			     116, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(117, "SMTDET0", UNIPHIER_PIN_IECTRL_EXIST,
+			     59, UNIPHIER_PIN_DRV_3BIT,
+			     117, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(118, "SMTRST1", UNIPHIER_PIN_IECTRL_EXIST,
+			     60, UNIPHIER_PIN_DRV_3BIT,
+			     118, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(119, "SMTCMD1", UNIPHIER_PIN_IECTRL_EXIST,
+			     61, UNIPHIER_PIN_DRV_3BIT,
+			     119, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(120, "SMTD1", UNIPHIER_PIN_IECTRL_EXIST,
+			     62, UNIPHIER_PIN_DRV_3BIT,
+			     120, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(121, "SMTSEL1", UNIPHIER_PIN_IECTRL_EXIST,
+			     63, UNIPHIER_PIN_DRV_3BIT,
+			     121, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(122, "SMTCLK1CG", UNIPHIER_PIN_IECTRL_EXIST,
+			     64, UNIPHIER_PIN_DRV_3BIT,
+			     122, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(123, "SMTDET1", UNIPHIER_PIN_IECTRL_EXIST,
+			     65, UNIPHIER_PIN_DRV_3BIT,
+			     123, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(124, "SMTRST2", UNIPHIER_PIN_IECTRL_EXIST,
+			     66, UNIPHIER_PIN_DRV_3BIT,
+			     124, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(125, "SMTCMD2", UNIPHIER_PIN_IECTRL_EXIST,
+			     67, UNIPHIER_PIN_DRV_3BIT,
+			     125, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(126, "SMTD2", UNIPHIER_PIN_IECTRL_EXIST,
+			     68, UNIPHIER_PIN_DRV_3BIT,
+			     126, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(127, "SMTSEL2", UNIPHIER_PIN_IECTRL_EXIST,
+			     69, UNIPHIER_PIN_DRV_3BIT,
+			     127, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(128, "SMTCLK2CG", UNIPHIER_PIN_IECTRL_EXIST,
+			     70, UNIPHIER_PIN_DRV_3BIT,
+			     128, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(129, "SMTDET2", UNIPHIER_PIN_IECTRL_EXIST,
+			     71, UNIPHIER_PIN_DRV_3BIT,
+			     129, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(130, "CH0CLK", UNIPHIER_PIN_IECTRL_EXIST,
+			     72, UNIPHIER_PIN_DRV_3BIT,
+			     130, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(131, "CH0PSYNC", UNIPHIER_PIN_IECTRL_EXIST,
+			     73, UNIPHIER_PIN_DRV_3BIT,
+			     131, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(132, "CH0VAL", UNIPHIER_PIN_IECTRL_EXIST,
+			     74, UNIPHIER_PIN_DRV_3BIT,
+			     132, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(133, "CH0DATA", UNIPHIER_PIN_IECTRL_EXIST,
+			     75, UNIPHIER_PIN_DRV_3BIT,
+			     133, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(134, "CH1CLK", UNIPHIER_PIN_IECTRL_EXIST,
+			     76, UNIPHIER_PIN_DRV_3BIT,
+			     134, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(135, "CH1PSYNC", UNIPHIER_PIN_IECTRL_EXIST,
+			     77, UNIPHIER_PIN_DRV_3BIT,
+			     135, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(136, "CH1VAL", UNIPHIER_PIN_IECTRL_EXIST,
+			     78, UNIPHIER_PIN_DRV_3BIT,
+			     136, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(137, "CH1DATA", UNIPHIER_PIN_IECTRL_EXIST,
+			     79, UNIPHIER_PIN_DRV_3BIT,
+			     137, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(138, "CH2CLK", UNIPHIER_PIN_IECTRL_EXIST,
+			     80, UNIPHIER_PIN_DRV_3BIT,
+			     138, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(139, "CH2PSYNC", UNIPHIER_PIN_IECTRL_EXIST,
+			     81, UNIPHIER_PIN_DRV_3BIT,
+			     139, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(140, "CH2VAL", UNIPHIER_PIN_IECTRL_EXIST,
+			     82, UNIPHIER_PIN_DRV_3BIT,
+			     140, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(141, "CH2DATA", UNIPHIER_PIN_IECTRL_EXIST,
+			     83, UNIPHIER_PIN_DRV_3BIT,
+			     141, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(142, "HS0BCLKIN", UNIPHIER_PIN_IECTRL_EXIST,
+			     84, UNIPHIER_PIN_DRV_3BIT,
+			     142, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(143, "HS0SYNCIN", UNIPHIER_PIN_IECTRL_EXIST,
+			     85, UNIPHIER_PIN_DRV_3BIT,
+			     143, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(144, "HS0VALIN", UNIPHIER_PIN_IECTRL_EXIST,
+			     86, UNIPHIER_PIN_DRV_3BIT,
+			     144, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(145, "HS0DIN0", UNIPHIER_PIN_IECTRL_EXIST,
+			     87, UNIPHIER_PIN_DRV_3BIT,
+			     145, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(146, "HS0DIN1", UNIPHIER_PIN_IECTRL_EXIST,
+			     88, UNIPHIER_PIN_DRV_3BIT,
+			     146, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(147, "HS0DIN2", UNIPHIER_PIN_IECTRL_EXIST,
+			     89, UNIPHIER_PIN_DRV_3BIT,
+			     147, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(148, "HS0DIN3", UNIPHIER_PIN_IECTRL_EXIST,
+			     90, UNIPHIER_PIN_DRV_3BIT,
+			     148, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(149, "HS0DIN4", UNIPHIER_PIN_IECTRL_EXIST,
+			     91, UNIPHIER_PIN_DRV_3BIT,
+			     149, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(150, "HS0DIN5", UNIPHIER_PIN_IECTRL_EXIST,
+			     92, UNIPHIER_PIN_DRV_3BIT,
+			     150, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(151, "HS0DIN6", UNIPHIER_PIN_IECTRL_EXIST,
+			     93, UNIPHIER_PIN_DRV_3BIT,
+			     151, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(152, "HS0DIN7", UNIPHIER_PIN_IECTRL_EXIST,
+			     94, UNIPHIER_PIN_DRV_3BIT,
+			     152, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(153, "HS1BCLKIN", UNIPHIER_PIN_IECTRL_EXIST,
+			     95, UNIPHIER_PIN_DRV_3BIT,
+			     153, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(154, "HS1SYNCIN", UNIPHIER_PIN_IECTRL_EXIST,
+			     96, UNIPHIER_PIN_DRV_3BIT,
+			     154, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(155, "HS1VALIN", UNIPHIER_PIN_IECTRL_EXIST,
+			     97, UNIPHIER_PIN_DRV_3BIT,
+			     155, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(156, "HS1DIN0", UNIPHIER_PIN_IECTRL_EXIST,
+			     98, UNIPHIER_PIN_DRV_3BIT,
+			     156, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(157, "HS1DIN1", UNIPHIER_PIN_IECTRL_EXIST,
+			     99, UNIPHIER_PIN_DRV_3BIT,
+			     157, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(158, "HS1DIN2", UNIPHIER_PIN_IECTRL_EXIST,
+			     100, UNIPHIER_PIN_DRV_3BIT,
+			     158, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(159, "HS1DIN3", UNIPHIER_PIN_IECTRL_EXIST,
+			     101, UNIPHIER_PIN_DRV_3BIT,
+			     159, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(160, "HS1DIN4", UNIPHIER_PIN_IECTRL_EXIST,
+			     102, UNIPHIER_PIN_DRV_3BIT,
+			     160, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(161, "HS1DIN5", UNIPHIER_PIN_IECTRL_EXIST,
+			     103, UNIPHIER_PIN_DRV_3BIT,
+			     161, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(162, "HS1DIN6", UNIPHIER_PIN_IECTRL_EXIST,
+			     104, UNIPHIER_PIN_DRV_3BIT,
+			     162, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(163, "HS1DIN7", UNIPHIER_PIN_IECTRL_EXIST,
+			     105, UNIPHIER_PIN_DRV_3BIT,
+			     163, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(164, "LINKCLK", UNIPHIER_PIN_IECTRL_EXIST,
+			     106, UNIPHIER_PIN_DRV_3BIT,
+			     164, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(165, "LINKREQ", UNIPHIER_PIN_IECTRL_EXIST,
+			     107, UNIPHIER_PIN_DRV_3BIT,
+			     165, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(166, "LINKCTL0", UNIPHIER_PIN_IECTRL_EXIST,
+			     108, UNIPHIER_PIN_DRV_3BIT,
+			     166, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(167, "LINKCTL1", UNIPHIER_PIN_IECTRL_EXIST,
+			     109, UNIPHIER_PIN_DRV_3BIT,
+			     167, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(168, "LINKDT0", UNIPHIER_PIN_IECTRL_EXIST,
+			     110, UNIPHIER_PIN_DRV_3BIT,
+			     168, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(169, "LINKDT1", UNIPHIER_PIN_IECTRL_EXIST,
+			     111, UNIPHIER_PIN_DRV_3BIT,
+			     169, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(170, "LINKDT2", UNIPHIER_PIN_IECTRL_EXIST,
+			     112, UNIPHIER_PIN_DRV_3BIT,
+			     170, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(171, "LINKDT3", UNIPHIER_PIN_IECTRL_EXIST,
+			     113, UNIPHIER_PIN_DRV_3BIT,
+			     171, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(172, "LINKDT4", UNIPHIER_PIN_IECTRL_EXIST,
+			     114, UNIPHIER_PIN_DRV_3BIT,
+			     172, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(173, "LINKDT5", UNIPHIER_PIN_IECTRL_EXIST,
+			     115, UNIPHIER_PIN_DRV_3BIT,
+			     173, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(174, "LINKDT6", UNIPHIER_PIN_IECTRL_EXIST,
+			     116, UNIPHIER_PIN_DRV_3BIT,
+			     174, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(175, "LINKDT7", UNIPHIER_PIN_IECTRL_EXIST,
+			     117, UNIPHIER_PIN_DRV_3BIT,
+			     175, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(176, "H0RXDDCSDA", UNIPHIER_PIN_IECTRL_EXIST,
+			     -1, UNIPHIER_PIN_DRV_FIXED4,
+			     -1, UNIPHIER_PIN_PULL_NONE),
+	UNIPHIER_PINCTRL_PIN(177, "H0RXDDCSCL", UNIPHIER_PIN_IECTRL_EXIST,
+			     -1, UNIPHIER_PIN_DRV_FIXED4,
+			     -1, UNIPHIER_PIN_PULL_NONE),
+	UNIPHIER_PINCTRL_PIN(178, "H0RXHPDO", UNIPHIER_PIN_IECTRL_EXIST,
+			     -1, UNIPHIER_PIN_DRV_FIXED4,
+			     -1, UNIPHIER_PIN_PULL_NONE),
+	UNIPHIER_PINCTRL_PIN(179, "H0RX5VDETI", UNIPHIER_PIN_IECTRL_EXIST,
+			     -1, UNIPHIER_PIN_DRV_FIXED4,
+			     -1, UNIPHIER_PIN_PULL_NONE),
+	UNIPHIER_PINCTRL_PIN(180, "H0TXDDCSDA", UNIPHIER_PIN_IECTRL_EXIST,
+			     -1, UNIPHIER_PIN_DRV_FIXED4,
+			     -1, UNIPHIER_PIN_PULL_NONE),
+	UNIPHIER_PINCTRL_PIN(181, "H0TXDDCSCL", UNIPHIER_PIN_IECTRL_EXIST,
+			     -1, UNIPHIER_PIN_DRV_FIXED4,
+			     -1, UNIPHIER_PIN_PULL_NONE),
+	UNIPHIER_PINCTRL_PIN(182, "H0TXHPDI", UNIPHIER_PIN_IECTRL_EXIST,
+			     -1, UNIPHIER_PIN_DRV_FIXED4,
+			     -1, UNIPHIER_PIN_PULL_NONE),
+	UNIPHIER_PINCTRL_PIN(183, "H1TXDDCSDA", UNIPHIER_PIN_IECTRL_EXIST,
+			     -1, UNIPHIER_PIN_DRV_FIXED4,
+			     -1, UNIPHIER_PIN_PULL_NONE),
+	UNIPHIER_PINCTRL_PIN(184, "H1TXDDCSCL", UNIPHIER_PIN_IECTRL_EXIST,
+			     -1, UNIPHIER_PIN_DRV_FIXED4,
+			     -1, UNIPHIER_PIN_PULL_NONE),
+	UNIPHIER_PINCTRL_PIN(185, "H1TXHPDI", UNIPHIER_PIN_IECTRL_EXIST,
+			     -1, UNIPHIER_PIN_DRV_FIXED4,
+			     -1, UNIPHIER_PIN_PULL_NONE),
+	UNIPHIER_PINCTRL_PIN(186, "AI1ADCCK", UNIPHIER_PIN_IECTRL_EXIST,
+			     118, UNIPHIER_PIN_DRV_3BIT,
+			     186, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(187, "AI1BCK", UNIPHIER_PIN_IECTRL_EXIST,
+			     119, UNIPHIER_PIN_DRV_3BIT,
+			     187, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(188, "AI1LRCK", UNIPHIER_PIN_IECTRL_EXIST,
+			     120, UNIPHIER_PIN_DRV_3BIT,
+			     188, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(189, "AI1D0", UNIPHIER_PIN_IECTRL_EXIST,
+			     121, UNIPHIER_PIN_DRV_3BIT,
+			     189, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(190, "AO1IEC", UNIPHIER_PIN_IECTRL_EXIST,
+			     122, UNIPHIER_PIN_DRV_3BIT,
+			     190, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(191, "AO2IEC", UNIPHIER_PIN_IECTRL_EXIST,
+			     123, UNIPHIER_PIN_DRV_3BIT,
+			     191, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(192, "AO2DACCK", UNIPHIER_PIN_IECTRL_EXIST,
+			     124, UNIPHIER_PIN_DRV_3BIT,
+			     192, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(193, "AO2BCK", UNIPHIER_PIN_IECTRL_EXIST,
+			     125, UNIPHIER_PIN_DRV_3BIT,
+			     193, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(194, "AO2LRCK", UNIPHIER_PIN_IECTRL_EXIST,
+			     126, UNIPHIER_PIN_DRV_3BIT,
+			     194, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(195, "AO2D0", UNIPHIER_PIN_IECTRL_EXIST,
+			     127, UNIPHIER_PIN_DRV_3BIT,
+			     195, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(196, "AO2D1", UNIPHIER_PIN_IECTRL_EXIST,
+			     128, UNIPHIER_PIN_DRV_3BIT,
+			     196, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(197, "AO2D2", UNIPHIER_PIN_IECTRL_EXIST,
+			     129, UNIPHIER_PIN_DRV_3BIT,
+			     197, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(198, "AO2D3", UNIPHIER_PIN_IECTRL_EXIST,
+			     130, UNIPHIER_PIN_DRV_3BIT,
+			     198, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(199, "AO3DACCK", UNIPHIER_PIN_IECTRL_EXIST,
+			     131, UNIPHIER_PIN_DRV_3BIT,
+			     199, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(200, "AO3BCK", UNIPHIER_PIN_IECTRL_EXIST,
+			     132, UNIPHIER_PIN_DRV_3BIT,
+			     200, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(201, "AO3LRCK", UNIPHIER_PIN_IECTRL_EXIST,
+			     133, UNIPHIER_PIN_DRV_3BIT,
+			     201, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(202, "AO3D0", UNIPHIER_PIN_IECTRL_EXIST,
+			     134, UNIPHIER_PIN_DRV_3BIT,
+			     202, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(203, "VI1CLK", UNIPHIER_PIN_IECTRL_EXIST,
+			     135, UNIPHIER_PIN_DRV_3BIT,
+			     203, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(204, "VI1G2", UNIPHIER_PIN_IECTRL_EXIST,
+			     136, UNIPHIER_PIN_DRV_3BIT,
+			     204, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(205, "VI1G3", UNIPHIER_PIN_IECTRL_EXIST,
+			     137, UNIPHIER_PIN_DRV_3BIT,
+			     205, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(206, "VI1G4", UNIPHIER_PIN_IECTRL_EXIST,
+			     138, UNIPHIER_PIN_DRV_3BIT,
+			     206, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(207, "VI1G5", UNIPHIER_PIN_IECTRL_EXIST,
+			     139, UNIPHIER_PIN_DRV_3BIT,
+			     207, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(208, "VI1G6", UNIPHIER_PIN_IECTRL_EXIST,
+			     140, UNIPHIER_PIN_DRV_3BIT,
+			     208, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(209, "VI1G7", UNIPHIER_PIN_IECTRL_EXIST,
+			     141, UNIPHIER_PIN_DRV_3BIT,
+			     209, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(210, "VI1G8", UNIPHIER_PIN_IECTRL_EXIST,
+			     142, UNIPHIER_PIN_DRV_3BIT,
+			     210, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(211, "VI1G9", UNIPHIER_PIN_IECTRL_EXIST,
+			     143, UNIPHIER_PIN_DRV_3BIT,
+			     211, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(212, "FANPWM", UNIPHIER_PIN_IECTRL_EXIST,
+			     144, UNIPHIER_PIN_DRV_3BIT,
+			     212, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(213, "CK27EXO", UNIPHIER_PIN_IECTRL_EXIST,
+			     145, UNIPHIER_PIN_DRV_3BIT,
+			     213, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(214, "CK27AO", UNIPHIER_PIN_IECTRL_EXIST,
+			     146, UNIPHIER_PIN_DRV_3BIT,
+			     214, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(215, "CK27EXI", UNIPHIER_PIN_IECTRL_EXIST,
+			     147, UNIPHIER_PIN_DRV_3BIT,
+			     215, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(216, "VEXCKA", UNIPHIER_PIN_IECTRL_EXIST,
+			     148, UNIPHIER_PIN_DRV_3BIT,
+			     216, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(217, "AEXCKA", UNIPHIER_PIN_IECTRL_EXIST,
+			     149, UNIPHIER_PIN_DRV_3BIT,
+			     217, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(218, "ASEL", UNIPHIER_PIN_IECTRL_EXIST,
+			     150, UNIPHIER_PIN_DRV_3BIT,
+			     218, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(219, "XIRQ0", UNIPHIER_PIN_IECTRL_EXIST,
+			     151, UNIPHIER_PIN_DRV_3BIT,
+			     219, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(220, "XIRQ1", UNIPHIER_PIN_IECTRL_EXIST,
+			     152, UNIPHIER_PIN_DRV_3BIT,
+			     220, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(221, "XIRQ2", UNIPHIER_PIN_IECTRL_EXIST,
+			     153, UNIPHIER_PIN_DRV_3BIT,
+			     221, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(222, "XIRQ3", UNIPHIER_PIN_IECTRL_EXIST,
+			     154, UNIPHIER_PIN_DRV_3BIT,
+			     222, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(223, "XIRQ4", UNIPHIER_PIN_IECTRL_EXIST,
+			     155, UNIPHIER_PIN_DRV_3BIT,
+			     223, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(224, "XIRQ5", UNIPHIER_PIN_IECTRL_EXIST,
+			     156, UNIPHIER_PIN_DRV_3BIT,
+			     224, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(225, "XIRQ6", UNIPHIER_PIN_IECTRL_EXIST,
+			     157, UNIPHIER_PIN_DRV_3BIT,
+			     225, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(226, "XIRQ7", UNIPHIER_PIN_IECTRL_EXIST,
+			     158, UNIPHIER_PIN_DRV_3BIT,
+			     226, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(227, "XIRQ8", UNIPHIER_PIN_IECTRL_EXIST,
+			     159, UNIPHIER_PIN_DRV_3BIT,
+			     227, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(228, "XIRQ9", UNIPHIER_PIN_IECTRL_EXIST,
+			     160, UNIPHIER_PIN_DRV_3BIT,
+			     228, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(229, "XIRQ10", UNIPHIER_PIN_IECTRL_EXIST,
+			     161, UNIPHIER_PIN_DRV_3BIT,
+			     229, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(230, "XIRQ11", UNIPHIER_PIN_IECTRL_EXIST,
+			     162, UNIPHIER_PIN_DRV_3BIT,
+			     230, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(231, "XIRQ12", UNIPHIER_PIN_IECTRL_EXIST,
+			     163, UNIPHIER_PIN_DRV_3BIT,
+			     231, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(232, "XIRQ13", UNIPHIER_PIN_IECTRL_EXIST,
+			     164, UNIPHIER_PIN_DRV_3BIT,
+			     232, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(233, "XIRQ14", UNIPHIER_PIN_IECTRL_EXIST,
+			     165, UNIPHIER_PIN_DRV_3BIT,
+			     233, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(234, "XIRQ15", UNIPHIER_PIN_IECTRL_EXIST,
+			     166, UNIPHIER_PIN_DRV_3BIT,
+			     234, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(235, "PORT00", UNIPHIER_PIN_IECTRL_EXIST,
+			     167, UNIPHIER_PIN_DRV_3BIT,
+			     235, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(236, "PORT01", UNIPHIER_PIN_IECTRL_EXIST,
+			     168, UNIPHIER_PIN_DRV_3BIT,
+			     236, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(237, "PORT02", UNIPHIER_PIN_IECTRL_EXIST,
+			     169, UNIPHIER_PIN_DRV_3BIT,
+			     237, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(238, "PORT03", UNIPHIER_PIN_IECTRL_EXIST,
+			     170, UNIPHIER_PIN_DRV_3BIT,
+			     238, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(239, "PORT04", UNIPHIER_PIN_IECTRL_EXIST,
+			     171, UNIPHIER_PIN_DRV_3BIT,
+			     239, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(240, "PORT05", UNIPHIER_PIN_IECTRL_EXIST,
+			     172, UNIPHIER_PIN_DRV_3BIT,
+			     240, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(241, "PORT06", UNIPHIER_PIN_IECTRL_EXIST,
+			     173, UNIPHIER_PIN_DRV_3BIT,
+			     241, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(242, "PORT07", UNIPHIER_PIN_IECTRL_EXIST,
+			     174, UNIPHIER_PIN_DRV_3BIT,
+			     242, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(243, "PORT10", UNIPHIER_PIN_IECTRL_EXIST,
+			     175, UNIPHIER_PIN_DRV_3BIT,
+			     243, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(244, "PORT11", UNIPHIER_PIN_IECTRL_EXIST,
+			     176, UNIPHIER_PIN_DRV_3BIT,
+			     244, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(245, "PORT12", UNIPHIER_PIN_IECTRL_EXIST,
+			     177, UNIPHIER_PIN_DRV_3BIT,
+			     245, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(246, "PORT13", UNIPHIER_PIN_IECTRL_EXIST,
+			     178, UNIPHIER_PIN_DRV_3BIT,
+			     246, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(247, "PORT14", UNIPHIER_PIN_IECTRL_EXIST,
+			     179, UNIPHIER_PIN_DRV_3BIT,
+			     247, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(248, "PORT15", UNIPHIER_PIN_IECTRL_EXIST,
+			     180, UNIPHIER_PIN_DRV_3BIT,
+			     248, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(249, "PORT16", UNIPHIER_PIN_IECTRL_EXIST,
+			     181, UNIPHIER_PIN_DRV_3BIT,
+			     249, UNIPHIER_PIN_PULL_DOWN),
+	UNIPHIER_PINCTRL_PIN(250, "PORT17", UNIPHIER_PIN_IECTRL_EXIST,
+			     182, UNIPHIER_PIN_DRV_3BIT,
+			     250, UNIPHIER_PIN_PULL_DOWN),
+};
+
+static const unsigned int emmc_pins[] = {31, 32, 33, 34, 35, 36, 37, 38};
+static const int emmc_muxvals[] = {0, 0, 0, 0, 0, 0, 0, 0};
+static const unsigned int emmc_dat8_pins[] = {39, 40, 41, 42};
+static const int emmc_dat8_muxvals[] = {0, 0, 0, 0};
+static const unsigned int ether_rgmii_pins[] = {52, 53, 54, 55, 56, 57, 58, 59,
+						60, 61, 62, 63, 64, 65, 66, 67};
+static const int ether_rgmii_muxvals[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+					  0, 0, 0};
+static const unsigned int ether_rmii_pins[] = {52, 53, 54, 55, 56, 57, 58, 59,
+					       61, 63, 64, 67};
+static const int ether_rmii_muxvals[] = {0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1};
+static const unsigned int ether1_rgmii_pins[] = {68, 69, 70, 71, 72, 73, 74,
+						 75, 76, 77, 78, 79, 80, 81,
+						 82, 83};
+static const int ether1_rgmii_muxvals[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+					   0, 0, 0, 0};
+static const unsigned int ether1_rmii_pins[] = {68, 69, 70, 71, 72, 73, 74, 75,
+						77, 79, 80, 83};
+static const int ether1_rmii_muxvals[] = {0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1};
+static const unsigned int i2c0_pins[] = {104, 105};
+static const int i2c0_muxvals[] = {0, 0};
+static const unsigned int i2c1_pins[] = {106, 107};
+static const int i2c1_muxvals[] = {0, 0};
+static const unsigned int i2c2_pins[] = {108, 109};
+static const int i2c2_muxvals[] = {0, 0};
+static const unsigned int i2c3_pins[] = {110, 111};
+static const int i2c3_muxvals[] = {0, 0};
+static const unsigned int nand_pins[] = {16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
+					 26, 27, 28, 29, 30};
+static const int nand_muxvals[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+static const unsigned int sd_pins[] = {43, 44, 45, 46, 47, 48, 49, 50, 51};
+static const int sd_muxvals[] = {0, 0, 0, 0, 0, 0, 0, 0, 0};
+static const unsigned int system_bus_pins[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
+					       11, 12, 13, 14};
+static const int system_bus_muxvals[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+					 0};
+static const unsigned int system_bus_cs1_pins[] = {15};
+static const int system_bus_cs1_muxvals[] = {0};
+static const unsigned int uart0_pins[] = {92, 93};
+static const int uart0_muxvals[] = {0, 0};
+static const unsigned int uart1_pins[] = {94, 95};
+static const int uart1_muxvals[] = {0, 0};
+static const unsigned int uart2_pins[] = {96, 97};
+static const int uart2_muxvals[] = {0, 0};
+static const unsigned int uart3_pins[] = {98, 99};
+static const int uart3_muxvals[] = {0, 0};
+static const unsigned int usb0_pins[] = {84, 85};
+static const int usb0_muxvals[] = {0, 0};
+static const unsigned int usb1_pins[] = {86, 87};
+static const int usb1_muxvals[] = {0, 0};
+static const unsigned int usb2_pins[] = {88, 89};
+static const int usb2_muxvals[] = {0, 0};
+static const unsigned int usb3_pins[] = {90, 91};
+static const int usb3_muxvals[] = {0, 0};
+static const unsigned int gpio_range0_pins[] = {
+	235, 236, 237, 238, 239, 240, 241, 242,		/* PORT0x */
+	243, 244, 245, 246, 247, 248, 249, 250,		/* PORT1x */
+	0, 1, 2, 3, 4, 5, 6, 7,				/* PORT2x */
+	8, 9, 10, 11, 12, 13, 14, 15,			/* PORT3x */
+	16, 17, 18, 19, 20, 21, 22, 23,			/* PORT4x */
+	24, 25, 26, 27, 28, 29, 30, 31,			/* PORT5x */
+	43, 44, 45, 46, 47, 48, 49, 50,			/* PORT6x */
+	51, 52, 53, 54, 55, 56, 57, 58,			/* PORT7x */
+	59, 60, 61, 62, 63, 64, 65, 66,			/* PORT8x */
+	67, 68, 69, 70, 71, 72, 73, 74,			/* PORT9x */
+	75, 76, 77, 78, 79, 80, 81, 82,			/* PORT10x */
+};
+static const unsigned int gpio_range1_pins[] = {
+	83, 84, 85, 86, 87, 88, 89, 90,			/* PORT13x */
+	91, 92, 93, 94, 95, 96, 97, 98,			/* PORT14x */
+	219, 220, 221, 222, 223, 224, 225, 226,		/* XIRQ0-7 */
+	227, 228, 229, 230, 231, 232, 233, 234,		/* XIRQ8-15 */
+	215, 216, 217, 218, 164, 165, 166, 167,		/* XIRQ16-23 */
+	104, 105, 106, 107, 108, 109, 110, 111,		/* PORT18x */
+	176, 177, 178, 179, 180, 181, 182, 183,		/* PORT19x */
+	184, 185,					/* PORT200-201 */
+};
+static const unsigned int gpio_range2_pins[] = {
+	99, 100, 101, 102, 103, 112, 113, 114,		/* PORT21x */
+	115, 116, 117, 118, 119, 120, 121, 122,		/* PORT22x */
+	123, 124, 125, 126, 127, 128, 129, 130,		/* PORT23x */
+	131, 132, 133, 134, 135, 136, 137, 138,		/* PORT24x */
+	139, 140, 141, 142, 143, 144, 145, 146,		/* PORT25x */
+	147, 148, 149, 150, 151, 152, 153, 154,		/* PORT26x */
+	155, 156, 157, 158, 159, 160, 161, 162,		/* PORT27x */
+	163, 164, 165, 166, 167, 168, 169, 170,		/* PORT28x */
+	171, 172, 173, 174, 175, 186, 187, 188,		/* PORT29x */
+	189, 190, 191, 192, 193, 194, 195, 196,		/* PORT30x */
+	197, 198, 199, 200, 201, 202, 203, 204,		/* PORT31x */
+	205, 206, 207, 208, 209, 210, 211, 212,		/* PORT32x */
+	213, 214, 215, 216, 217, 218, 219, 220,		/* PORT33x */
+	221, 222, 223, 224, 225, 226, 227, 228,		/* PORT34x */
+	229, 230, 231, 232, 233, 234,			/* PORT350-355 */
+};
+
+static const struct uniphier_pinctrl_group uniphier_pxs3_groups[] = {
+	UNIPHIER_PINCTRL_GROUP(emmc),
+	UNIPHIER_PINCTRL_GROUP(emmc_dat8),
+	UNIPHIER_PINCTRL_GROUP(ether_rgmii),
+	UNIPHIER_PINCTRL_GROUP(ether_rmii),
+	UNIPHIER_PINCTRL_GROUP(ether1_rgmii),
+	UNIPHIER_PINCTRL_GROUP(ether1_rmii),
+	UNIPHIER_PINCTRL_GROUP(i2c0),
+	UNIPHIER_PINCTRL_GROUP(i2c1),
+	UNIPHIER_PINCTRL_GROUP(i2c2),
+	UNIPHIER_PINCTRL_GROUP(i2c3),
+	UNIPHIER_PINCTRL_GROUP(nand),
+	UNIPHIER_PINCTRL_GROUP(sd),
+	UNIPHIER_PINCTRL_GROUP(system_bus),
+	UNIPHIER_PINCTRL_GROUP(system_bus_cs1),
+	UNIPHIER_PINCTRL_GROUP(uart0),
+	UNIPHIER_PINCTRL_GROUP(uart1),
+	UNIPHIER_PINCTRL_GROUP(uart2),
+	UNIPHIER_PINCTRL_GROUP(uart3),
+	UNIPHIER_PINCTRL_GROUP(usb0),
+	UNIPHIER_PINCTRL_GROUP(usb1),
+	UNIPHIER_PINCTRL_GROUP(usb2),
+	UNIPHIER_PINCTRL_GROUP(usb3),
+	UNIPHIER_PINCTRL_GROUP_GPIO(gpio_range0),
+	UNIPHIER_PINCTRL_GROUP_GPIO(gpio_range1),
+	UNIPHIER_PINCTRL_GROUP_GPIO(gpio_range2),
+};
+
+static const char * const emmc_groups[] = {"emmc", "emmc_dat8"};
+static const char * const ether_rgmii_groups[] = {"ether_rgmii"};
+static const char * const ether_rmii_groups[] = {"ether_rmii"};
+static const char * const ether1_rgmii_groups[] = {"ether1_rgmii"};
+static const char * const ether1_rmii_groups[] = {"ether1_rmii"};
+static const char * const i2c0_groups[] = {"i2c0"};
+static const char * const i2c1_groups[] = {"i2c1"};
+static const char * const i2c2_groups[] = {"i2c2"};
+static const char * const i2c3_groups[] = {"i2c3"};
+static const char * const nand_groups[] = {"nand"};
+static const char * const sd_groups[] = {"sd"};
+static const char * const system_bus_groups[] = {"system_bus",
+						 "system_bus_cs1"};
+static const char * const uart0_groups[] = {"uart0"};
+static const char * const uart1_groups[] = {"uart1"};
+static const char * const uart2_groups[] = {"uart2"};
+static const char * const uart3_groups[] = {"uart3"};
+static const char * const usb0_groups[] = {"usb0"};
+static const char * const usb1_groups[] = {"usb1"};
+static const char * const usb2_groups[] = {"usb2"};
+static const char * const usb3_groups[] = {"usb3"};
+
+static const struct uniphier_pinmux_function uniphier_pxs3_functions[] = {
+	UNIPHIER_PINMUX_FUNCTION(emmc),
+	UNIPHIER_PINMUX_FUNCTION(ether_rgmii),
+	UNIPHIER_PINMUX_FUNCTION(ether_rmii),
+	UNIPHIER_PINMUX_FUNCTION(ether1_rgmii),
+	UNIPHIER_PINMUX_FUNCTION(ether1_rmii),
+	UNIPHIER_PINMUX_FUNCTION(i2c0),
+	UNIPHIER_PINMUX_FUNCTION(i2c1),
+	UNIPHIER_PINMUX_FUNCTION(i2c2),
+	UNIPHIER_PINMUX_FUNCTION(i2c3),
+	UNIPHIER_PINMUX_FUNCTION(nand),
+	UNIPHIER_PINMUX_FUNCTION(sd),
+	UNIPHIER_PINMUX_FUNCTION(system_bus),
+	UNIPHIER_PINMUX_FUNCTION(uart0),
+	UNIPHIER_PINMUX_FUNCTION(uart1),
+	UNIPHIER_PINMUX_FUNCTION(uart2),
+	UNIPHIER_PINMUX_FUNCTION(uart3),
+	UNIPHIER_PINMUX_FUNCTION(usb0),
+	UNIPHIER_PINMUX_FUNCTION(usb1),
+	UNIPHIER_PINMUX_FUNCTION(usb2),
+	UNIPHIER_PINMUX_FUNCTION(usb3),
+};
+
+static int uniphier_pxs3_get_gpio_muxval(unsigned int pin,
+					 unsigned int gpio_offset)
+{
+	if (gpio_offset >= 120 && gpio_offset <= 143) {	/* XIRQx */
+		if (pin >= 219 && pin <= 234)
+			return 0;
+
+		return 14;
+	}
+
+	return 15;
+}
+
+static struct uniphier_pinctrl_socdata uniphier_pxs3_pindata = {
+	.pins = uniphier_pxs3_pins,
+	.npins = ARRAY_SIZE(uniphier_pxs3_pins),
+	.groups = uniphier_pxs3_groups,
+	.groups_count = ARRAY_SIZE(uniphier_pxs3_groups),
+	.functions = uniphier_pxs3_functions,
+	.functions_count = ARRAY_SIZE(uniphier_pxs3_functions),
+	.get_gpio_muxval = uniphier_pxs3_get_gpio_muxval,
+	.caps = UNIPHIER_PINCTRL_CAPS_PERPIN_IECTRL,
+};
+
+static int uniphier_pxs3_pinctrl_probe(struct platform_device *pdev)
+{
+	return uniphier_pinctrl_probe(pdev, &uniphier_pxs3_pindata);
+}
+
+static const struct of_device_id uniphier_pxs3_pinctrl_match[] = {
+	{ .compatible = "socionext,uniphier-pxs3-pinctrl" },
+	{ /* sentinel */ }
+};
+
+static struct platform_driver uniphier_pxs3_pinctrl_driver = {
+	.probe = uniphier_pxs3_pinctrl_probe,
+	.driver = {
+		.name = "uniphier-pxs3-pinctrl",
+		.of_match_table = uniphier_pxs3_pinctrl_match,
+		.pm = &uniphier_pinctrl_pm_ops,
+	},
+};
+builtin_platform_driver(uniphier_pxs3_pinctrl_driver);
diff --git a/drivers/pinctrl/uniphier/pinctrl-uniphier-sld8.c b/drivers/pinctrl/uniphier/pinctrl-uniphier-sld8.c
index 37deaf615dcf..1af430d701be 100644
--- a/drivers/pinctrl/uniphier/pinctrl-uniphier-sld8.c
+++ b/drivers/pinctrl/uniphier/pinctrl-uniphier-sld8.c
@@ -532,67 +532,28 @@ static const unsigned usb1_pins[] = {43, 44};
 static const int usb1_muxvals[] = {0, 0};
 static const unsigned usb2_pins[] = {114, 115};
 static const int usb2_muxvals[] = {1, 1};
-static const unsigned port_range0_pins[] = {
-	0, 1, 2, 3, 4, 5, 6, 7,				/* PORT0x */
-	8, 9, 10, 11, 12, 13, 14, 15,			/* PORT1x */
-	32, 33, 34, 35, 36, 37, 38, 39,			/* PORT2x */
-	59, 60, 61, 62, 63, 64, 65, 66,			/* PORT3x */
-	95, 96, 97, 98, 99, 100, 101, 57,		/* PORT4x */
-	70, 71, 72, 73, 74, 75, 76, 77,			/* PORT5x */
-	81, 83, 84, 85, 86, 89, 90, 91,			/* PORT6x */
-	118, 119, 120, 121, 122, 53, 54, 55,		/* PORT7x */
-	41, 42, 43, 44, 79, 80, 18, 19,			/* PORT8x */
-	110, 111, 112, 113, 114, 115, 16, 17,		/* PORT9x */
-	40, 67, 68, 69, 78, 92, 93, 94,			/* PORT10x */
-	48, 49, 46, 45, 123, 124, 125, 126,		/* PORT11x */
-	47, 127, 20, 56, 22,				/* PORT120-124 */
+static const unsigned int gpio_range0_pins[] = {
+	0, 1, 2, 3, 4, 5, 6, 7,			/* PORT0x */
+	8, 9, 10, 11, 12, 13, 14, 15,		/* PORT1x */
+	32, 33, 34, 35, 36, 37, 38, 39,		/* PORT2x */
+	59, 60, 61, 62, 63, 64, 65, 66,		/* PORT3x */
+	95, 96, 97, 98, 99, 100, 101, 57,	/* PORT4x */
+	70, 71, 72, 73, 74, 75, 76, 77,		/* PORT5x */
+	81, 83, 84, 85, 86, 89, 90, 91,		/* PORT6x */
+	118, 119, 120, 121, 122, 53, 54, 55,	/* PORT7x */
+	41, 42, 43, 44, 79, 80, 18, 19,		/* PORT8x */
+	110, 111, 112, 113, 114, 115, 16, 17,	/* PORT9x */
+	40, 67, 68, 69, 78, 92, 93, 94,		/* PORT10x */
+	48, 49, 46, 45, 123, 124, 125, 126,	/* PORT11x */
+	47, 127, 20, 56, 22,			/* PORT120-124 */
 };
-static const int port_range0_muxvals[] = {
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT0x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT1x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT2x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT3x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT4x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT5x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT6x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT7x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT8x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT9x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT10x */
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT11x */
-	15, 15, 15, 15, 15,				/* PORT120-124 */
+static const unsigned int gpio_range1_pins[] = {
+	116, 117,				/* PORT130-131 */
 };
-static const unsigned port_range1_pins[] = {
-	116, 117,					/* PORT130-131 */
-};
-static const int port_range1_muxvals[] = {
-	15, 15,						/* PORT130-131 */
-};
-static const unsigned port_range2_pins[] = {
-	102, 103, 104, 105, 106, 107, 108, 109,		/* PORT14x */
-};
-static const int port_range2_muxvals[] = {
-	15, 15, 15, 15, 15, 15, 15, 15,			/* PORT14x */
-};
-static const unsigned port_range3_pins[] = {
-	23,						/* PORT166 */
-};
-static const int port_range3_muxvals[] = {
-	15,						/* PORT166 */
-};
-static const unsigned xirq_range0_pins[] = {
-	128, 129, 130, 131, 132, 133, 134, 135,		/* XIRQ0-7 */
-	82, 87, 88, 50, 51,				/* XIRQ8-12 */
-};
-static const int xirq_range0_muxvals[] = {
-	0, 0, 0, 0, 0, 0, 0, 0,				/* XIRQ0-7 */
-	14, 14, 14, 14, 14,				/* XIRQ8-12 */
-};
-static const unsigned xirq_range1_pins[] = {
-	52, 58,						/* XIRQ14-15 */
-};
-static const int xirq_range1_muxvals[] = {
-	14, 14,						/* XIRQ14-15 */
+static const unsigned int gpio_range2_pins[] = {
+	102, 103, 104, 105, 106, 107, 108, 109,	/* PORT14x */
+	128, 129, 130, 131, 132, 133, 134, 135,	/* XIRQ0-7 */
+	82, 87, 88, 50, 51, 23, 52, 58,	      /* XIRQ8-12, PORT165, XIRQ14-15 */
 };
 
 static const struct uniphier_pinctrl_group uniphier_sld8_groups[] = {
@@ -620,139 +581,9 @@ static const struct uniphier_pinctrl_group uniphier_sld8_groups[] = {
 	UNIPHIER_PINCTRL_GROUP(usb0),
 	UNIPHIER_PINCTRL_GROUP(usb1),
 	UNIPHIER_PINCTRL_GROUP(usb2),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(port_range0),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(port_range1),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(port_range2),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(port_range3),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_IRQ(xirq_range0),
-	UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_IRQ(xirq_range1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port00, port_range0, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port01, port_range0, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port02, port_range0, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port03, port_range0, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port04, port_range0, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port05, port_range0, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port06, port_range0, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port07, port_range0, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port10, port_range0, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port11, port_range0, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port12, port_range0, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port13, port_range0, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port14, port_range0, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port15, port_range0, 13),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port16, port_range0, 14),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port17, port_range0, 15),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port20, port_range0, 16),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port21, port_range0, 17),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port22, port_range0, 18),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port23, port_range0, 19),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port24, port_range0, 20),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port25, port_range0, 21),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port26, port_range0, 22),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port27, port_range0, 23),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port30, port_range0, 24),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port31, port_range0, 25),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port32, port_range0, 26),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port33, port_range0, 27),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port34, port_range0, 28),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port35, port_range0, 29),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port36, port_range0, 30),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port37, port_range0, 31),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port40, port_range0, 32),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port41, port_range0, 33),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port42, port_range0, 34),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port43, port_range0, 35),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port44, port_range0, 36),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port45, port_range0, 37),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port46, port_range0, 38),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port47, port_range0, 39),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port50, port_range0, 40),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port51, port_range0, 41),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port52, port_range0, 42),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port53, port_range0, 43),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port54, port_range0, 44),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port55, port_range0, 45),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port56, port_range0, 46),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port57, port_range0, 47),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port60, port_range0, 48),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port61, port_range0, 49),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port62, port_range0, 50),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port63, port_range0, 51),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port64, port_range0, 52),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port65, port_range0, 53),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port66, port_range0, 54),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port67, port_range0, 55),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port70, port_range0, 56),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port71, port_range0, 57),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port72, port_range0, 58),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port73, port_range0, 59),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port74, port_range0, 60),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port75, port_range0, 61),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port76, port_range0, 62),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port77, port_range0, 63),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port80, port_range0, 64),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port81, port_range0, 65),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port82, port_range0, 66),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port83, port_range0, 67),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port84, port_range0, 68),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port85, port_range0, 69),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port86, port_range0, 70),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port87, port_range0, 71),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port90, port_range0, 72),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port91, port_range0, 73),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port92, port_range0, 74),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port93, port_range0, 75),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port94, port_range0, 76),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port95, port_range0, 77),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port96, port_range0, 78),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port97, port_range0, 79),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port100, port_range0, 80),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port101, port_range0, 81),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port102, port_range0, 82),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port103, port_range0, 83),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port104, port_range0, 84),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port105, port_range0, 85),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port106, port_range0, 86),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port107, port_range0, 87),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port110, port_range0, 88),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port111, port_range0, 89),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port112, port_range0, 90),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port113, port_range0, 91),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port114, port_range0, 92),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port115, port_range0, 93),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port116, port_range0, 94),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port117, port_range0, 95),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port120, port_range0, 96),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port121, port_range0, 97),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port122, port_range0, 98),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port123, port_range0, 99),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port124, port_range0, 100),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port130, port_range1, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port131, port_range1, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port140, port_range2, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port141, port_range2, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port142, port_range2, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port143, port_range2, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port144, port_range2, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port145, port_range2, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port146, port_range2, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port147, port_range2, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(port166, port_range3, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq0, xirq_range0, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq1, xirq_range0, 1),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq2, xirq_range0, 2),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq3, xirq_range0, 3),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq4, xirq_range0, 4),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq5, xirq_range0, 5),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq6, xirq_range0, 6),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq7, xirq_range0, 7),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq8, xirq_range0, 8),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq9, xirq_range0, 9),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq10, xirq_range0, 10),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq11, xirq_range0, 11),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq12, xirq_range0, 12),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq14, xirq_range1, 0),
-	UNIPHIER_PINCTRL_GROUP_SINGLE(xirq15, xirq_range1, 1),
+	UNIPHIER_PINCTRL_GROUP_GPIO(gpio_range0),
+	UNIPHIER_PINCTRL_GROUP_GPIO(gpio_range1),
+	UNIPHIER_PINCTRL_GROUP_GPIO(gpio_range2),
 };
 
 static const char * const emmc_groups[] = {"emmc", "emmc_dat8"};
@@ -777,46 +608,6 @@ static const char * const uart3_groups[] = {"uart3"};
 static const char * const usb0_groups[] = {"usb0"};
 static const char * const usb1_groups[] = {"usb1"};
 static const char * const usb2_groups[] = {"usb2"};
-static const char * const port_groups[] = {
-	"port00",  "port01",  "port02",  "port03",
-	"port04",  "port05",  "port06",  "port07",
-	"port10",  "port11",  "port12",  "port13",
-	"port14",  "port15",  "port16",  "port17",
-	"port20",  "port21",  "port22",  "port23",
-	"port24",  "port25",  "port26",  "port27",
-	"port30",  "port31",  "port32",  "port33",
-	"port34",  "port35",  "port36",  "port37",
-	"port40",  "port41",  "port42",  "port43",
-	"port44",  "port45",  "port46",  "port47",
-	"port50",  "port51",  "port52",  "port53",
-	"port54",  "port55",  "port56",  "port57",
-	"port60",  "port61",  "port62",  "port63",
-	"port64",  "port65",  "port66",  "port67",
-	"port70",  "port71",  "port72",  "port73",
-	"port74",  "port75",  "port76",  "port77",
-	"port80",  "port81",  "port82",  "port83",
-	"port84",  "port85",  "port86",  "port87",
-	"port90",  "port91",  "port92",  "port93",
-	"port94",  "port95",  "port96",  "port97",
-	"port100", "port101", "port102", "port103",
-	"port104", "port105", "port106", "port107",
-	"port110", "port111", "port112", "port113",
-	"port114", "port115", "port116", "port117",
-	"port120", "port121", "port122", "port123",
-	"port124", "port125", "port126", "port127",
-	"port130", "port131", "port132", "port133",
-	"port134", "port135", "port136", "port137",
-	"port140", "port141", "port142", "port143",
-	"port144", "port145", "port146", "port147",
-	/* port150-164 missing */
-	/* none */ "port165",
-};
-static const char * const xirq_groups[] = {
-	"xirq0",  "xirq1",  "xirq2",  "xirq3",
-	"xirq4",  "xirq5",  "xirq6",  "xirq7",
-	"xirq8",  "xirq9",  "xirq10", "xirq11",
-	"xirq12", /* none*/ "xirq14", "xirq15",
-};
 
 static const struct uniphier_pinmux_function uniphier_sld8_functions[] = {
 	UNIPHIER_PINMUX_FUNCTION(emmc),
@@ -836,10 +627,22 @@ static const struct uniphier_pinmux_function uniphier_sld8_functions[] = {
 	UNIPHIER_PINMUX_FUNCTION(usb0),
 	UNIPHIER_PINMUX_FUNCTION(usb1),
 	UNIPHIER_PINMUX_FUNCTION(usb2),
-	UNIPHIER_PINMUX_FUNCTION(port),
-	UNIPHIER_PINMUX_FUNCTION(xirq),
 };
 
+static int uniphier_sld8_get_gpio_muxval(unsigned int pin,
+					 unsigned int gpio_offset)
+{
+	switch (gpio_offset) {
+	case 120 ... 127:	/* XIRQ0-XIRQ7 */
+		return 0;
+	case 128 ... 132:	/* XIRQ8-12 */
+	case 134 ... 135:	/* XIRQ14-15 */
+		return 14;
+	default:
+		return 15;
+	}
+}
+
 static struct uniphier_pinctrl_socdata uniphier_sld8_pindata = {
 	.pins = uniphier_sld8_pins,
 	.npins = ARRAY_SIZE(uniphier_sld8_pins),
@@ -847,6 +650,7 @@ static struct uniphier_pinctrl_socdata uniphier_sld8_pindata = {
 	.groups_count = ARRAY_SIZE(uniphier_sld8_groups),
 	.functions = uniphier_sld8_functions,
 	.functions_count = ARRAY_SIZE(uniphier_sld8_functions),
+	.get_gpio_muxval = uniphier_sld8_get_gpio_muxval,
 	.caps = 0,
 };
 
@@ -865,6 +669,7 @@ static struct platform_driver uniphier_sld8_pinctrl_driver = {
 	.driver = {
 		.name = "uniphier-sld8-pinctrl",
 		.of_match_table = uniphier_sld8_pinctrl_match,
+		.pm = &uniphier_pinctrl_pm_ops,
 	},
 };
 builtin_platform_driver(uniphier_sld8_pinctrl_driver);
diff --git a/drivers/pinctrl/uniphier/pinctrl-uniphier.h b/drivers/pinctrl/uniphier/pinctrl-uniphier.h
index 6f2f33bf788f..0a3d2ac27503 100644
--- a/drivers/pinctrl/uniphier/pinctrl-uniphier.h
+++ b/drivers/pinctrl/uniphier/pinctrl-uniphier.h
@@ -17,7 +17,7 @@
 #define __PINCTRL_UNIPHIER_H__
 
 #include <linux/bitops.h>
-#include <linux/bug.h>
+#include <linux/build_bug.h>
 #include <linux/kernel.h>
 #include <linux/types.h>
 
@@ -25,7 +25,7 @@ struct platform_device;
 
 /* input enable control register bit */
 #define UNIPHIER_PIN_IECTRL_SHIFT	0
-#define UNIPHIER_PIN_IECTRL_BITS	8
+#define UNIPHIER_PIN_IECTRL_BITS	3
 #define UNIPHIER_PIN_IECTRL_MASK	((1UL << (UNIPHIER_PIN_IECTRL_BITS)) \
 					 - 1)
 
@@ -62,6 +62,7 @@ struct platform_device;
 #endif
 
 #define UNIPHIER_PIN_IECTRL_NONE	(UNIPHIER_PIN_IECTRL_MASK)
+#define UNIPHIER_PIN_IECTRL_EXIST	0
 
 /* drive control type */
 enum uniphier_pin_drv_type {
@@ -131,18 +132,11 @@ static inline unsigned int uniphier_pin_get_pull_dir(void *drv_data)
 						UNIPHIER_PIN_PULL_DIR_MASK;
 }
 
-enum uniphier_pinmux_gpio_range_type {
-	UNIPHIER_PINMUX_GPIO_RANGE_PORT,
-	UNIPHIER_PINMUX_GPIO_RANGE_IRQ,
-	UNIPHIER_PINMUX_GPIO_RANGE_NONE,
-};
-
 struct uniphier_pinctrl_group {
 	const char *name;
 	const unsigned *pins;
 	unsigned num_pins;
 	const int *muxvals;
-	enum uniphier_pinmux_gpio_range_type range_type;
 };
 
 struct uniphier_pinmux_function {
@@ -158,6 +152,7 @@ struct uniphier_pinctrl_socdata {
 	int groups_count;
 	const struct uniphier_pinmux_function *functions;
 	int functions_count;
+	int (*get_gpio_muxval)(unsigned int pin, unsigned int gpio_offset);
 	unsigned int caps;
 #define UNIPHIER_PINCTRL_CAPS_PERPIN_IECTRL	BIT(1)
 #define UNIPHIER_PINCTRL_CAPS_DBGMUX_SEPARATE	BIT(0)
@@ -170,33 +165,22 @@ struct uniphier_pinctrl_socdata {
 	.drv_data = (void *)UNIPHIER_PIN_ATTR_PACKED(c, d, e, f, g),	\
 }
 
-#define __UNIPHIER_PINCTRL_GROUP(grp, type)				\
+#define __UNIPHIER_PINCTRL_GROUP(grp, mux)				\
 	{								\
 		.name = #grp,						\
 		.pins = grp##_pins,					\
 		.num_pins = ARRAY_SIZE(grp##_pins),			\
-		.muxvals = grp##_muxvals +				\
-			BUILD_BUG_ON_ZERO(ARRAY_SIZE(grp##_pins) !=	\
-					  ARRAY_SIZE(grp##_muxvals)),	\
-		.range_type = type,					\
+		.muxvals = mux,						\
 	}
 
 #define UNIPHIER_PINCTRL_GROUP(grp)					\
-	__UNIPHIER_PINCTRL_GROUP(grp, UNIPHIER_PINMUX_GPIO_RANGE_NONE)
-
-#define UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_PORT(grp)			\
-	__UNIPHIER_PINCTRL_GROUP(grp, UNIPHIER_PINMUX_GPIO_RANGE_PORT)
-
-#define UNIPHIER_PINCTRL_GROUP_GPIO_RANGE_IRQ(grp)			\
-	__UNIPHIER_PINCTRL_GROUP(grp, UNIPHIER_PINMUX_GPIO_RANGE_IRQ)
+	__UNIPHIER_PINCTRL_GROUP(grp,					\
+			grp##_muxvals +					\
+			BUILD_BUG_ON_ZERO(ARRAY_SIZE(grp##_pins) !=	\
+					  ARRAY_SIZE(grp##_muxvals)))
 
-#define UNIPHIER_PINCTRL_GROUP_SINGLE(grp, array, ofst)			\
-	{								\
-		.name = #grp,						\
-		.pins = array##_pins + ofst,				\
-		.num_pins = 1,						\
-		.muxvals = array##_muxvals + ofst,			\
-	}
+#define UNIPHIER_PINCTRL_GROUP_GPIO(grp)				\
+	__UNIPHIER_PINCTRL_GROUP(grp, NULL)
 
 #define UNIPHIER_PINMUX_FUNCTION(func)					\
 	{								\
@@ -208,4 +192,6 @@ struct uniphier_pinctrl_socdata {
 int uniphier_pinctrl_probe(struct platform_device *pdev,
 			   struct uniphier_pinctrl_socdata *socdata);
 
+extern const struct dev_pm_ops uniphier_pinctrl_pm_ops;
+
 #endif /* __PINCTRL_UNIPHIER_H__ */
diff --git a/drivers/pinctrl/vt8500/pinctrl-wmt.c b/drivers/pinctrl/vt8500/pinctrl-wmt.c
index c207e60b734f..d73956bdc211 100644
--- a/drivers/pinctrl/vt8500/pinctrl-wmt.c
+++ b/drivers/pinctrl/vt8500/pinctrl-wmt.c
@@ -163,7 +163,7 @@ static int wmt_pmx_gpio_set_direction(struct pinctrl_dev *pctldev,
 	return 0;
 }
 
-static struct pinmux_ops wmt_pinmux_ops = {
+static const struct pinmux_ops wmt_pinmux_ops = {
 	.get_functions_count = wmt_pmx_get_functions_count,
 	.get_function_name = wmt_pmx_get_function_name,
 	.get_function_groups = wmt_pmx_get_function_groups,
@@ -409,7 +409,7 @@ fail:
 	return err;
 }
 
-static struct pinctrl_ops wmt_pctl_ops = {
+static const struct pinctrl_ops wmt_pctl_ops = {
 	.get_groups_count = wmt_get_groups_count,
 	.get_group_name	= wmt_get_group_name,
 	.get_group_pins	= wmt_get_group_pins,
@@ -472,7 +472,7 @@ static int wmt_pinconf_set(struct pinctrl_dev *pctldev, unsigned pin,
 	return 0;
 }
 
-static struct pinconf_ops wmt_pinconf_ops = {
+static const struct pinconf_ops wmt_pinconf_ops = {
 	.pin_config_get = wmt_pinconf_get,
 	.pin_config_set = wmt_pinconf_set,
 };
@@ -546,7 +546,7 @@ static int wmt_gpio_direction_output(struct gpio_chip *chip, unsigned offset,
 	return pinctrl_gpio_direction_output(chip->base + offset);
 }
 
-static struct gpio_chip wmt_gpio_chip = {
+static const struct gpio_chip wmt_gpio_chip = {
 	.label = "gpio-wmt",
 	.owner = THIS_MODULE,
 	.request = gpiochip_generic_request,
diff --git a/drivers/pinctrl/zte/pinctrl-zx.c b/drivers/pinctrl/zte/pinctrl-zx.c
index f828ee340a98..ded366bb6564 100644
--- a/drivers/pinctrl/zte/pinctrl-zx.c
+++ b/drivers/pinctrl/zte/pinctrl-zx.c
@@ -295,8 +295,7 @@ static int zx_pinctrl_build_state(struct platform_device *pdev)
 	pctldev->num_groups = ngroups;
 
 	/* Build function list from pin mux functions */
-	functions = devm_kzalloc(&pdev->dev, info->npins * sizeof(*functions),
-				 GFP_KERNEL);
+	functions = kcalloc(info->npins, sizeof(*functions), GFP_KERNEL);
 	if (!functions)
 		return -ENOMEM;
 
@@ -367,8 +366,10 @@ static int zx_pinctrl_build_state(struct platform_device *pdev)
 						func->num_group_names *
 						sizeof(*func->group_names),
 						GFP_KERNEL);
-				if (!func->group_names)
+				if (!func->group_names) {
+					kfree(functions);
 					return -ENOMEM;
+				}
 			}
 
 			group = func->group_names;
diff --git a/drivers/platform/chrome/chromeos_laptop.c b/drivers/platform/chrome/chromeos_laptop.c
index e8a44a9bc916..d8599736a41a 100644
--- a/drivers/platform/chrome/chromeos_laptop.c
+++ b/drivers/platform/chrome/chromeos_laptop.c
@@ -518,7 +518,7 @@ static struct chromeos_laptop cr48 = {
 	.callback = chromeos_laptop_dmi_matched, \
 	.driver_data = (void *)&board_
 
-static struct dmi_system_id chromeos_laptop_dmi_table[] __initdata = {
+static const struct dmi_system_id chromeos_laptop_dmi_table[] __initconst = {
 	{
 		.ident = "Samsung Series 5 550",
 		.matches = {
diff --git a/drivers/platform/chrome/chromeos_pstore.c b/drivers/platform/chrome/chromeos_pstore.c
index 308a853ac4f1..b0693fdec8c6 100644
--- a/drivers/platform/chrome/chromeos_pstore.c
+++ b/drivers/platform/chrome/chromeos_pstore.c
@@ -14,7 +14,7 @@
 #include <linux/platform_device.h>
 #include <linux/pstore_ram.h>
 
-static struct dmi_system_id chromeos_pstore_dmi_table[] __initdata = {
+static const struct dmi_system_id chromeos_pstore_dmi_table[] __initconst = {
 	{
 		/*
 		 * Today all Chromebooks/boxes ship with Google_* as version and
diff --git a/drivers/platform/chrome/cros_ec_lpc.c b/drivers/platform/chrome/cros_ec_lpc.c
index 2b6436d1b6a4..1baf720faf69 100644
--- a/drivers/platform/chrome/cros_ec_lpc.c
+++ b/drivers/platform/chrome/cros_ec_lpc.c
@@ -329,7 +329,7 @@ static const struct acpi_device_id cros_ec_lpc_acpi_device_ids[] = {
 };
 MODULE_DEVICE_TABLE(acpi, cros_ec_lpc_acpi_device_ids);
 
-static struct dmi_system_id cros_ec_lpc_dmi_table[] __initdata = {
+static const struct dmi_system_id cros_ec_lpc_dmi_table[] __initconst = {
 	{
 		/*
 		 * Today all Chromebooks/boxes ship with Google_* as version and
diff --git a/drivers/platform/x86/alienware-wmi.c b/drivers/platform/x86/alienware-wmi.c
index 0831b428c217..4eb8e1a472b2 100644
--- a/drivers/platform/x86/alienware-wmi.c
+++ b/drivers/platform/x86/alienware-wmi.c
@@ -255,12 +255,13 @@ static int parse_rgb(const char *buf, struct platform_zone *zone)
 
 static struct platform_zone *match_zone(struct device_attribute *attr)
 {
-	int i;
-	for (i = 0; i < quirks->num_zones; i++) {
-		if ((struct device_attribute *)zone_data[i].attr == attr) {
+	u8 zone;
+
+	for (zone = 0; zone < quirks->num_zones; zone++) {
+		if ((struct device_attribute *)zone_data[zone].attr == attr) {
 			pr_debug("alienware-wmi: matched zone location: %d\n",
-				 zone_data[i].location);
-			return &zone_data[i];
+				 zone_data[zone].location);
+			return &zone_data[zone];
 		}
 	}
 	return NULL;
@@ -420,7 +421,7 @@ static DEVICE_ATTR(lighting_control_state, 0644, show_control_state,
 
 static int alienware_zone_init(struct platform_device *dev)
 {
-	int i;
+	u8 zone;
 	char buffer[10];
 	char *name;
 
@@ -457,19 +458,19 @@ static int alienware_zone_init(struct platform_device *dev)
 	if (!zone_data)
 		return -ENOMEM;
 
-	for (i = 0; i < quirks->num_zones; i++) {
-		sprintf(buffer, "zone%02X", i);
+	for (zone = 0; zone < quirks->num_zones; zone++) {
+		sprintf(buffer, "zone%02hhX", zone);
 		name = kstrdup(buffer, GFP_KERNEL);
 		if (name == NULL)
 			return 1;
-		sysfs_attr_init(&zone_dev_attrs[i].attr);
-		zone_dev_attrs[i].attr.name = name;
-		zone_dev_attrs[i].attr.mode = 0644;
-		zone_dev_attrs[i].show = zone_show;
-		zone_dev_attrs[i].store = zone_set;
-		zone_data[i].location = i;
-		zone_attrs[i] = &zone_dev_attrs[i].attr;
-		zone_data[i].attr = &zone_dev_attrs[i];
+		sysfs_attr_init(&zone_dev_attrs[zone].attr);
+		zone_dev_attrs[zone].attr.name = name;
+		zone_dev_attrs[zone].attr.mode = 0644;
+		zone_dev_attrs[zone].show = zone_show;
+		zone_dev_attrs[zone].store = zone_set;
+		zone_data[zone].location = zone;
+		zone_attrs[zone] = &zone_dev_attrs[zone].attr;
+		zone_data[zone].attr = &zone_dev_attrs[zone];
 	}
 	zone_attrs[quirks->num_zones] = &dev_attr_lighting_control_state.attr;
 	zone_attribute_group.attrs = zone_attrs;
@@ -481,12 +482,13 @@ static int alienware_zone_init(struct platform_device *dev)
 
 static void alienware_zone_exit(struct platform_device *dev)
 {
+	u8 zone;
+
 	sysfs_remove_group(&dev->dev.kobj, &zone_attribute_group);
 	led_classdev_unregister(&global_led);
 	if (zone_dev_attrs) {
-		int i;
-		for (i = 0; i < quirks->num_zones; i++)
-			kfree(zone_dev_attrs[i].attr.name);
+		for (zone = 0; zone < quirks->num_zones; zone++)
+			kfree(zone_dev_attrs[zone].attr.name);
 	}
 	kfree(zone_dev_attrs);
 	kfree(zone_data);
diff --git a/drivers/platform/x86/asus-wmi.c b/drivers/platform/x86/asus-wmi.c
index 709e3a67391a..48e1541dc8d4 100644
--- a/drivers/platform/x86/asus-wmi.c
+++ b/drivers/platform/x86/asus-wmi.c
@@ -299,7 +299,7 @@ static int asus_wmi_evaluate_method(u32 method_id, u32 arg0, u32 arg1,
 	union acpi_object *obj;
 	u32 tmp = 0;
 
-	status = wmi_evaluate_method(ASUS_WMI_MGMT_GUID, 1, method_id,
+	status = wmi_evaluate_method(ASUS_WMI_MGMT_GUID, 0, method_id,
 				     &input, &output);
 
 	if (ACPI_FAILURE(status))
@@ -1946,7 +1946,7 @@ static int show_call(struct seq_file *m, void *data)
 	acpi_status status;
 
 	status = wmi_evaluate_method(ASUS_WMI_MGMT_GUID,
-				     1, asus->debug.method_id,
+				     0, asus->debug.method_id,
 				     &input, &output);
 
 	if (ACPI_FAILURE(status))
diff --git a/drivers/platform/x86/compal-laptop.c b/drivers/platform/x86/compal-laptop.c
index a8e4a539e704..6bcb750e1865 100644
--- a/drivers/platform/x86/compal-laptop.c
+++ b/drivers/platform/x86/compal-laptop.c
@@ -805,7 +805,7 @@ static int dmi_check_cb_extra(const struct dmi_system_id *id)
 	return 1;
 }
 
-static struct dmi_system_id __initdata compal_dmi_table[] = {
+static const struct dmi_system_id compal_dmi_table[] __initconst = {
 	{
 		.ident = "FL90/IFL90",
 		.matches = {
diff --git a/drivers/platform/x86/dell-wmi.c b/drivers/platform/x86/dell-wmi.c
index dad8f4afa17c..28d9f8696081 100644
--- a/drivers/platform/x86/dell-wmi.c
+++ b/drivers/platform/x86/dell-wmi.c
@@ -48,7 +48,6 @@ MODULE_LICENSE("GPL");
 #define DELL_EVENT_GUID "9DBB5994-A997-11DA-B012-B622A1EF5492"
 #define DELL_DESCRIPTOR_GUID "8D9DDCBC-A997-11DA-B012-B622A1EF5492"
 
-static u32 dell_wmi_interface_version;
 static bool wmi_requires_smbios_request;
 
 MODULE_ALIAS("wmi:"DELL_EVENT_GUID);
@@ -56,6 +55,7 @@ MODULE_ALIAS("wmi:"DELL_DESCRIPTOR_GUID);
 
 struct dell_wmi_priv {
 	struct input_dev *input_dev;
+	u32 interface_version;
 };
 
 static int __init dmi_matched(const struct dmi_system_id *dmi)
@@ -348,6 +348,7 @@ static void dell_wmi_process_key(struct wmi_device *wdev, int type, int code)
 static void dell_wmi_notify(struct wmi_device *wdev,
 			    union acpi_object *obj)
 {
+	struct dell_wmi_priv *priv = dev_get_drvdata(&wdev->dev);
 	u16 *buffer_entry, *buffer_end;
 	acpi_size buffer_size;
 	int len, i;
@@ -376,7 +377,7 @@ static void dell_wmi_notify(struct wmi_device *wdev,
 	 * So to prevent reading garbage from buffer we will process only first
 	 * one event on devices with WMI interface version 0.
 	 */
-	if (dell_wmi_interface_version == 0 && buffer_entry < buffer_end)
+	if (priv->interface_version == 0 && buffer_entry < buffer_end)
 		if (buffer_end > buffer_entry + buffer_entry[0] + 1)
 			buffer_end = buffer_entry + buffer_entry[0] + 1;
 
@@ -626,61 +627,67 @@ static void dell_wmi_input_destroy(struct wmi_device *wdev)
  * WMI Interface Version     8       4    <version>
  * WMI buffer length        12       4    4096
  */
-static int dell_wmi_check_descriptor_buffer(void)
+static int dell_wmi_check_descriptor_buffer(struct wmi_device *wdev)
 {
-	struct acpi_buffer out = { ACPI_ALLOCATE_BUFFER, NULL };
-	union acpi_object *obj;
-	acpi_status status;
+	struct dell_wmi_priv *priv = dev_get_drvdata(&wdev->dev);
+	union acpi_object *obj = NULL;
+	struct wmi_device *desc_dev;
 	u32 *buffer;
+	int ret;
 
-	status = wmi_query_block(DELL_DESCRIPTOR_GUID, 0, &out);
-	if (ACPI_FAILURE(status)) {
-		pr_err("Cannot read Dell descriptor buffer - %d\n", status);
-		return status;
+	desc_dev = wmidev_get_other_guid(wdev, DELL_DESCRIPTOR_GUID);
+	if (!desc_dev) {
+		dev_err(&wdev->dev, "Dell WMI descriptor does not exist\n");
+		return -ENODEV;
 	}
 
-	obj = (union acpi_object *)out.pointer;
+	obj = wmidev_block_query(desc_dev, 0);
 	if (!obj) {
-		pr_err("Dell descriptor buffer is empty\n");
-		return -EINVAL;
+		dev_err(&wdev->dev, "failed to read Dell WMI descriptor\n");
+		ret = -EIO;
+		goto out;
 	}
 
 	if (obj->type != ACPI_TYPE_BUFFER) {
-		pr_err("Cannot read Dell descriptor buffer\n");
-		kfree(obj);
-		return -EINVAL;
+		dev_err(&wdev->dev, "Dell descriptor has wrong type\n");
+		ret = -EINVAL;
+		goto out;
 	}
 
 	if (obj->buffer.length != 128) {
-		pr_err("Dell descriptor buffer has invalid length (%d)\n",
+		dev_err(&wdev->dev,
+			"Dell descriptor buffer has invalid length (%d)\n",
 			obj->buffer.length);
 		if (obj->buffer.length < 16) {
-			kfree(obj);
-			return -EINVAL;
+			ret = -EINVAL;
+			goto out;
 		}
 	}
 
 	buffer = (u32 *)obj->buffer.pointer;
 
 	if (buffer[0] != 0x4C4C4544 && buffer[1] != 0x494D5720)
-		pr_warn("Dell descriptor buffer has invalid signature (%*ph)\n",
+		dev_warn(&wdev->dev, "Dell descriptor buffer has invalid signature (%*ph)\n",
 			8, buffer);
 
 	if (buffer[2] != 0 && buffer[2] != 1)
-		pr_warn("Dell descriptor buffer has unknown version (%d)\n",
+		dev_warn(&wdev->dev, "Dell descriptor buffer has unknown version (%d)\n",
 			buffer[2]);
 
 	if (buffer[3] != 4096)
-		pr_warn("Dell descriptor buffer has invalid buffer length (%d)\n",
+		dev_warn(&wdev->dev, "Dell descriptor buffer has invalid buffer length (%d)\n",
 			buffer[3]);
 
-	dell_wmi_interface_version = buffer[2];
+	priv->interface_version = buffer[2];
+	ret = 0;
 
-	pr_info("Detected Dell WMI interface version %u\n",
-		dell_wmi_interface_version);
+	dev_info(&wdev->dev, "Detected Dell WMI interface version %u\n",
+		priv->interface_version);
 
+out:
 	kfree(obj);
-	return 0;
+	put_device(&desc_dev->dev);
+	return ret;
 }
 
 /*
@@ -717,17 +724,19 @@ static int dell_wmi_events_set_enabled(bool enable)
 
 static int dell_wmi_probe(struct wmi_device *wdev)
 {
+	struct dell_wmi_priv *priv;
 	int err;
 
-	struct dell_wmi_priv *priv = devm_kzalloc(
+	priv = devm_kzalloc(
 		&wdev->dev, sizeof(struct dell_wmi_priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+	dev_set_drvdata(&wdev->dev, priv);
 
-	err = dell_wmi_check_descriptor_buffer();
+	err = dell_wmi_check_descriptor_buffer(wdev);
 	if (err)
 		return err;
 
-	dev_set_drvdata(&wdev->dev, priv);
-
 	return dell_wmi_input_setup(wdev);
 }
 
diff --git a/drivers/platform/x86/hdaps.c b/drivers/platform/x86/hdaps.c
index 458e6c948c11..c26baf77938e 100644
--- a/drivers/platform/x86/hdaps.c
+++ b/drivers/platform/x86/hdaps.c
@@ -514,7 +514,7 @@ static int __init hdaps_dmi_match_invert(const struct dmi_system_id *id)
    "ThinkPad T42p", so the order of the entries matters.
    If your ThinkPad is not recognized, please update to latest
    BIOS. This is especially the case for some R52 ThinkPads. */
-static struct dmi_system_id __initdata hdaps_whitelist[] = {
+static const struct dmi_system_id hdaps_whitelist[] __initconst = {
 	HDAPS_DMI_MATCH_INVERT("IBM", "ThinkPad R50p", HDAPS_BOTH_AXES),
 	HDAPS_DMI_MATCH_NORMAL("IBM", "ThinkPad R50"),
 	HDAPS_DMI_MATCH_NORMAL("IBM", "ThinkPad R51"),
diff --git a/drivers/platform/x86/hp-wmi.c b/drivers/platform/x86/hp-wmi.c
index 0df4209648d1..b4ed3dc983d5 100644
--- a/drivers/platform/x86/hp-wmi.c
+++ b/drivers/platform/x86/hp-wmi.c
@@ -107,13 +107,6 @@ enum hp_wmi_hardware_mask {
 	HPWMI_TABLET_MASK	= 0x04,
 };
 
-#define BIOS_ARGS_INIT(write, ctype, size)				\
-	(struct bios_args)	{	.signature = 0x55434553,	\
-					.command = (write) ? 0x2 : 0x1,	\
-					.commandtype = (ctype),		\
-					.datasize = (size),		\
-					.data = 0 }
-
 struct bios_return {
 	u32 sigpass;
 	u32 return_code;
@@ -188,6 +181,22 @@ struct rfkill2_device {
 static int rfkill2_count;
 static struct rfkill2_device rfkill2[HPWMI_MAX_RFKILL2_DEVICES];
 
+/* map output size to the corresponding WMI method id */
+static inline int encode_outsize_for_pvsz(int outsize)
+{
+	if (outsize > 4096)
+		return -EINVAL;
+	if (outsize > 1024)
+		return 5;
+	if (outsize > 128)
+		return 4;
+	if (outsize > 4)
+		return 3;
+	if (outsize > 0)
+		return 2;
+	return 1;
+}
+
 /*
  * hp_wmi_perform_query
  *
@@ -211,6 +220,7 @@ static struct rfkill2_device rfkill2[HPWMI_MAX_RFKILL2_DEVICES];
 static int hp_wmi_perform_query(int query, enum hp_wmi_command command,
 				void *buffer, int insize, int outsize)
 {
+	int mid;
 	struct bios_return *bios_return;
 	int actual_outsize;
 	union acpi_object *obj;
@@ -225,11 +235,15 @@ static int hp_wmi_perform_query(int query, enum hp_wmi_command command,
 	struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
 	int ret = 0;
 
+	mid = encode_outsize_for_pvsz(outsize);
+	if (WARN_ON(mid < 0))
+		return mid;
+
 	if (WARN_ON(insize > sizeof(args.data)))
 		return -EINVAL;
 	memcpy(&args.data, buffer, insize);
 
-	wmi_evaluate_method(HPWMI_BIOS_GUID, 0, 0x3, &input, &output);
+	wmi_evaluate_method(HPWMI_BIOS_GUID, 0, mid, &input, &output);
 
 	obj = output.pointer;
 
diff --git a/drivers/platform/x86/ibm_rtl.c b/drivers/platform/x86/ibm_rtl.c
index c62e5e11ca4b..18d55cee5bcd 100644
--- a/drivers/platform/x86/ibm_rtl.c
+++ b/drivers/platform/x86/ibm_rtl.c
@@ -103,7 +103,7 @@ static void rtl_port_unmap(void __iomem *addr)
 static int ibm_rtl_write(u8 value)
 {
 	int ret = 0, count = 0;
-	static u32 cmd_port_val;
+	u32 cmd_port_val;
 
 	RTL_DEBUG("%s(%d)\n", __func__, value);
 
@@ -227,7 +227,7 @@ static void rtl_teardown_sysfs(void) {
 }
 
 
-static struct dmi_system_id __initdata ibm_rtl_dmi_table[] = {
+static const struct dmi_system_id ibm_rtl_dmi_table[] __initconst = {
 	{                                                  \
 		.matches = {                               \
 			DMI_MATCH(DMI_SYS_VENDOR, "IBM"),  \
diff --git a/drivers/platform/x86/ideapad-laptop.c b/drivers/platform/x86/ideapad-laptop.c
index 603fc6050971..fe98d4ac0df3 100644
--- a/drivers/platform/x86/ideapad-laptop.c
+++ b/drivers/platform/x86/ideapad-laptop.c
@@ -42,6 +42,8 @@
 
 #define IDEAPAD_RFKILL_DEV_NUM	(3)
 
+#define BM_CONSERVATION_BIT (5)
+
 #define CFG_BT_BIT	(16)
 #define CFG_3G_BIT	(17)
 #define CFG_WIFI_BIT	(18)
@@ -55,6 +57,11 @@ static const char *const ideapad_wmi_fnesc_events[] = {
 #endif
 
 enum {
+	BMCMD_CONSERVATION_ON = 3,
+	BMCMD_CONSERVATION_OFF = 5,
+};
+
+enum {
 	VPCCMD_R_VPC1 = 0x10,
 	VPCCMD_R_BL_MAX,
 	VPCCMD_R_BL,
@@ -123,6 +130,23 @@ static int read_method_int(acpi_handle handle, const char *method, int *val)
 	}
 }
 
+static int method_gbmd(acpi_handle handle, unsigned long *ret)
+{
+	int result, val;
+
+	result = read_method_int(handle, "GBMD", &val);
+	*ret = val;
+	return result;
+}
+
+static int method_sbmc(acpi_handle handle, int cmd)
+{
+	acpi_status status;
+
+	status = acpi_execute_simple_method(handle, "SBMC", cmd);
+	return ACPI_FAILURE(status) ? -1 : 0;
+}
+
 static int method_vpcr(acpi_handle handle, int cmd, int *ret)
 {
 	acpi_status status;
@@ -250,6 +274,13 @@ static int debugfs_status_show(struct seq_file *s, void *data)
 	if (!read_ec_data(priv->adev->handle, VPCCMD_R_CAMERA, &value))
 		seq_printf(s, "Camera status:\t%s(%lu)\n",
 			   value ? "On" : "Off", value);
+	seq_puts(s, "=====================\n");
+
+	if (!method_gbmd(priv->adev->handle, &value)) {
+		seq_printf(s, "Conservation mode:\t%s(%lu)\n",
+			   test_bit(BM_CONSERVATION_BIT, &value) ? "On" : "Off",
+			   value);
+	}
 
 	return 0;
 }
@@ -456,10 +487,45 @@ static ssize_t __maybe_unused touchpad_store(struct device *dev,
 
 static DEVICE_ATTR_RO(touchpad);
 
+static ssize_t conservation_mode_show(struct device *dev,
+				struct device_attribute *attr,
+				char *buf)
+{
+	struct ideapad_private *priv = dev_get_drvdata(dev);
+	unsigned long result;
+
+	if (method_gbmd(priv->adev->handle, &result))
+		return sprintf(buf, "-1\n");
+	return sprintf(buf, "%u\n", test_bit(BM_CONSERVATION_BIT, &result));
+}
+
+static ssize_t conservation_mode_store(struct device *dev,
+				 struct device_attribute *attr,
+				 const char *buf, size_t count)
+{
+	struct ideapad_private *priv = dev_get_drvdata(dev);
+	bool state;
+	int ret;
+
+	ret = kstrtobool(buf, &state);
+	if (ret)
+		return ret;
+
+	ret = method_sbmc(priv->adev->handle, state ?
+					      BMCMD_CONSERVATION_ON :
+					      BMCMD_CONSERVATION_OFF);
+	if (ret < 0)
+		return -EIO;
+	return count;
+}
+
+static DEVICE_ATTR_RW(conservation_mode);
+
 static struct attribute *ideapad_attributes[] = {
 	&dev_attr_camera_power.attr,
 	&dev_attr_fan_mode.attr,
 	&dev_attr_touchpad.attr,
+	&dev_attr_conservation_mode.attr,
 	NULL
 };
 
@@ -477,6 +543,9 @@ static umode_t ideapad_is_visible(struct kobject *kobj,
 		unsigned long value;
 		supported = !read_ec_data(priv->adev->handle, VPCCMD_R_FAN,
 					  &value);
+	} else if (attr == &dev_attr_conservation_mode.attr) {
+		supported = acpi_has_method(priv->adev->handle, "GBMD") &&
+			    acpi_has_method(priv->adev->handle, "SBMC");
 	} else
 		supported = true;
 
diff --git a/drivers/platform/x86/intel-hid.c b/drivers/platform/x86/intel-hid.c
index 8519e0f97bdd..e34fd70b67af 100644
--- a/drivers/platform/x86/intel-hid.c
+++ b/drivers/platform/x86/intel-hid.c
@@ -203,15 +203,26 @@ static void notify_handler(acpi_handle handle, u32 event, void *context)
 	acpi_status status;
 
 	if (priv->wakeup_mode) {
+		/*
+		 * Needed for wakeup from suspend-to-idle to work on some
+		 * platforms that don't expose the 5-button array, but still
+		 * send notifies with the power button event code to this
+		 * device object on power button actions while suspended.
+		 */
+		if (event == 0xce)
+			goto wakeup;
+
 		/* Wake up on 5-button array events only. */
 		if (event == 0xc0 || !priv->array)
 			return;
 
-		if (sparse_keymap_entry_from_scancode(priv->array, event))
-			pm_wakeup_hard_event(&device->dev);
-		else
+		if (!sparse_keymap_entry_from_scancode(priv->array, event)) {
 			dev_info(&device->dev, "unknown event 0x%x\n", event);
+			return;
+		}
 
+wakeup:
+		pm_wakeup_hard_event(&device->dev);
 		return;
 	}
 
@@ -219,7 +230,7 @@ static void notify_handler(acpi_handle handle, u32 event, void *context)
 	if (event != 0xc0) {
 		if (!priv->array ||
 		    !sparse_keymap_report_event(priv->array, event, 1, true))
-			dev_info(&device->dev, "unknown event 0x%x\n", event);
+			dev_dbg(&device->dev, "unknown event 0x%x\n", event);
 		return;
 	}
 
@@ -230,7 +241,7 @@ static void notify_handler(acpi_handle handle, u32 event, void *context)
 	}
 
 	if (!sparse_keymap_report_event(priv->input_dev, ev_index, 1, true))
-		dev_info(&device->dev, "unknown event index 0x%llx\n",
+		dev_dbg(&device->dev, "unknown event index 0x%llx\n",
 			 ev_index);
 }
 
diff --git a/drivers/platform/x86/intel-vbtn.c b/drivers/platform/x86/intel-vbtn.c
index 480926786cb8..58c5ff36523a 100644
--- a/drivers/platform/x86/intel-vbtn.c
+++ b/drivers/platform/x86/intel-vbtn.c
@@ -83,7 +83,7 @@ static void notify_handler(acpi_handle handle, u32 event, void *context)
 	} else if (sparse_keymap_report_event(priv->input_dev, event, 1, true)) {
 		return;
 	}
-	dev_info(&device->dev, "unknown event index 0x%x\n", event);
+	dev_dbg(&device->dev, "unknown event index 0x%x\n", event);
 }
 
 static int intel_vbtn_probe(struct platform_device *device)
diff --git a/drivers/platform/x86/intel_mid_powerbtn.c b/drivers/platform/x86/intel_mid_powerbtn.c
index 871cfa682519..d79fbf924b13 100644
--- a/drivers/platform/x86/intel_mid_powerbtn.c
+++ b/drivers/platform/x86/intel_mid_powerbtn.c
@@ -108,13 +108,13 @@ static irqreturn_t mid_pb_isr(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-static struct mid_pb_ddata mfld_ddata = {
+static const struct mid_pb_ddata mfld_ddata = {
 	.mirqlvl1_addr	= INTEL_MSIC_IRQLVL1MSK,
 	.pbstat_addr	= INTEL_MSIC_PBSTATUS,
 	.pbstat_mask	= MSIC_PB_LEVEL,
 };
 
-static struct mid_pb_ddata mrfld_ddata = {
+static const struct mid_pb_ddata mrfld_ddata = {
 	.mirqlvl1_addr	= BCOVE_IRQLVL1MSK,
 	.pbstat_addr	= BCOVE_PBSTATUS,
 	.pbstat_mask	= BCOVE_PB_LEVEL,
@@ -142,8 +142,10 @@ static int mid_pb_probe(struct platform_device *pdev)
 	if (!id)
 		return -ENODEV;
 
-	if (irq < 0)
-		return -EINVAL;
+	if (irq < 0) {
+		dev_err(&pdev->dev, "Failed to get IRQ: %d\n", irq);
+		return irq;
+	}
 
 	input = devm_input_allocate_device(&pdev->dev);
 	if (!input)
diff --git a/drivers/platform/x86/intel_oaktrail.c b/drivers/platform/x86/intel_oaktrail.c
index 6aa33c4a809f..5747f63c8d9f 100644
--- a/drivers/platform/x86/intel_oaktrail.c
+++ b/drivers/platform/x86/intel_oaktrail.c
@@ -299,7 +299,7 @@ static int dmi_check_cb(const struct dmi_system_id *id)
 	return 0;
 }
 
-static struct dmi_system_id __initdata oaktrail_dmi_table[] = {
+static const struct dmi_system_id oaktrail_dmi_table[] __initconst = {
 	{
 		.ident = "OakTrail platform",
 		.matches = {
diff --git a/drivers/platform/x86/intel_pmc_core.c b/drivers/platform/x86/intel_pmc_core.c
index 914bcd2edbde..17e08b42b0a9 100644
--- a/drivers/platform/x86/intel_pmc_core.c
+++ b/drivers/platform/x86/intel_pmc_core.c
@@ -110,6 +110,13 @@ static const struct pmc_reg_map spt_reg_map = {
 	.pfear_sts = spt_pfear_map,
 	.mphy_sts = spt_mphy_map,
 	.pll_sts = spt_pll_map,
+	.slp_s0_offset = SPT_PMC_SLP_S0_RES_COUNTER_OFFSET,
+	.ltr_ignore_offset = SPT_PMC_LTR_IGNORE_OFFSET,
+	.regmap_length = SPT_PMC_MMIO_REG_LEN,
+	.ppfear0_offset = SPT_PMC_XRAM_PPFEAR0A,
+	.ppfear_buckets = SPT_PPFEAR_NUM_ENTRIES,
+	.pm_cfg_offset = SPT_PMC_PM_CFG_OFFSET,
+	.pm_read_disable_bit = SPT_PMC_READ_DISABLE_BIT,
 };
 
 static const struct pci_device_id pmc_pci_ids[] = {
@@ -157,12 +164,13 @@ static inline u32 pmc_core_adjust_slp_s0_step(u32 value)
 int intel_pmc_slp_s0_counter_read(u32 *data)
 {
 	struct pmc_dev *pmcdev = &pmc;
+	const struct pmc_reg_map *map = pmcdev->map;
 	u32 value;
 
 	if (!pmcdev->has_slp_s0_res)
 		return -EACCES;
 
-	value = pmc_core_reg_read(pmcdev, SPT_PMC_SLP_S0_RES_COUNTER_OFFSET);
+	value = pmc_core_reg_read(pmcdev, map->slp_s0_offset);
 	*data = pmc_core_adjust_slp_s0_step(value);
 
 	return 0;
@@ -172,9 +180,10 @@ EXPORT_SYMBOL_GPL(intel_pmc_slp_s0_counter_read);
 static int pmc_core_dev_state_get(void *data, u64 *val)
 {
 	struct pmc_dev *pmcdev = data;
+	const struct pmc_reg_map *map = pmcdev->map;
 	u32 value;
 
-	value = pmc_core_reg_read(pmcdev, SPT_PMC_SLP_S0_RES_COUNTER_OFFSET);
+	value = pmc_core_reg_read(pmcdev, map->slp_s0_offset);
 	*val = pmc_core_adjust_slp_s0_step(value);
 
 	return 0;
@@ -187,8 +196,8 @@ static int pmc_core_check_read_lock_bit(void)
 	struct pmc_dev *pmcdev = &pmc;
 	u32 value;
 
-	value = pmc_core_reg_read(pmcdev, SPT_PMC_PM_CFG_OFFSET);
-	return value & BIT(SPT_PMC_READ_DISABLE_BIT);
+	value = pmc_core_reg_read(pmcdev, pmcdev->map->pm_cfg_offset);
+	return value & BIT(pmcdev->map->pm_read_disable_bit);
 }
 
 #if IS_ENABLED(CONFIG_DEBUG_FS)
@@ -204,12 +213,13 @@ static int pmc_core_ppfear_sts_show(struct seq_file *s, void *unused)
 {
 	struct pmc_dev *pmcdev = s->private;
 	const struct pmc_bit_map *map = pmcdev->map->pfear_sts;
-	u8 pf_regs[NUM_ENTRIES];
+	u8 pf_regs[PPFEAR_MAX_NUM_ENTRIES];
 	int index, iter;
 
-	iter = SPT_PMC_XRAM_PPFEAR0A;
+	iter = pmcdev->map->ppfear0_offset;
 
-	for (index = 0; index < NUM_ENTRIES; index++, iter++)
+	for (index = 0; index < pmcdev->map->ppfear_buckets &&
+	     index < PPFEAR_MAX_NUM_ENTRIES; index++, iter++)
 		pf_regs[index] = pmc_core_reg_read_byte(pmcdev, iter);
 
 	for (index = 0; map[index].name; index++)
@@ -376,6 +386,7 @@ static ssize_t pmc_core_ltr_ignore_write(struct file *file, const char __user
 *userbuf, size_t count, loff_t *ppos)
 {
 	struct pmc_dev *pmcdev = &pmc;
+	const struct pmc_reg_map *map = pmcdev->map;
 	u32 val, buf_size, fd;
 	int err = 0;
 
@@ -392,9 +403,9 @@ static ssize_t pmc_core_ltr_ignore_write(struct file *file, const char __user
 		goto out_unlock;
 	}
 
-	fd = pmc_core_reg_read(pmcdev, SPT_PMC_LTR_IGNORE_OFFSET);
+	fd = pmc_core_reg_read(pmcdev, map->ltr_ignore_offset);
 	fd |= (1U << val);
-	pmc_core_reg_write(pmcdev, SPT_PMC_LTR_IGNORE_OFFSET, fd);
+	pmc_core_reg_write(pmcdev, map->ltr_ignore_offset, fd);
 
 out_unlock:
 	mutex_unlock(&pmcdev->lock);
@@ -530,8 +541,8 @@ static int pmc_core_probe(struct pci_dev *dev, const struct pci_device_id *id)
 	}
 
 	mutex_init(&pmcdev->lock);
-	pmcdev->pmc_xram_read_bit = pmc_core_check_read_lock_bit();
 	pmcdev->map = map;
+	pmcdev->pmc_xram_read_bit = pmc_core_check_read_lock_bit();
 
 	err = pmc_core_dbgfs_register(pmcdev);
 	if (err < 0)
diff --git a/drivers/platform/x86/intel_pmc_core.h b/drivers/platform/x86/intel_pmc_core.h
index 5a48e7728479..3d225a9cc09f 100644
--- a/drivers/platform/x86/intel_pmc_core.h
+++ b/drivers/platform/x86/intel_pmc_core.h
@@ -38,7 +38,8 @@
 #define SPT_PMC_SLP_S0_RES_COUNTER_STEP		0x64
 #define PMC_BASE_ADDR_MASK			~(SPT_PMC_MMIO_REG_LEN - 1)
 #define MTPMC_MASK				0xffff0000
-#define NUM_ENTRIES				5
+#define PPFEAR_MAX_NUM_ENTRIES			5
+#define SPT_PPFEAR_NUM_ENTRIES			5
 #define SPT_PMC_READ_DISABLE_BIT		0x16
 #define SPT_PMC_MSG_FULL_STS_BIT		0x18
 #define NUM_RETRIES				100
@@ -126,10 +127,37 @@ struct pmc_bit_map {
 	u32 bit_mask;
 };
 
+/**
+ * struct pmc_reg_map - Structure used to define parameter unique to a
+			PCH family
+ * @pfear_sts:		Maps name of IP block to PPFEAR* bit
+ * @mphy_sts:		Maps name of MPHY lane to MPHY status lane status bit
+ * @pll_sts:		Maps name of PLL to corresponding bit status
+ * @slp_s0_offset:	PWRMBASE offset to read SLP_S0 residency
+ * @ltr_ignore_offset:	PWRMBASE offset to read/write LTR ignore bit
+ * @base_address:	Base address of PWRMBASE defined in BIOS writer guide
+ * @regmap_length:	Length of memory to map from PWRMBASE address to access
+ * @ppfear0_offset:	PWRMBASE offset to to read PPFEAR*
+ * @ppfear_buckets:	Number of 8 bits blocks to read all IP blocks from
+ *			PPFEAR
+ * @pm_cfg_offset:	PWRMBASE offset to PM_CFG register
+ * @pm_read_disable_bit: Bit index to read PMC_READ_DISABLE
+ *
+ * Each PCH has unique set of register offsets and bit indexes. This structure
+ * captures them to have a common implementation.
+ */
 struct pmc_reg_map {
 	const struct pmc_bit_map *pfear_sts;
 	const struct pmc_bit_map *mphy_sts;
 	const struct pmc_bit_map *pll_sts;
+	const u32 slp_s0_offset;
+	const u32 ltr_ignore_offset;
+	const u32 base_address;
+	const int regmap_length;
+	const u32 ppfear0_offset;
+	const int ppfear_buckets;
+	const u32 pm_cfg_offset;
+	const int pm_read_disable_bit;
 };
 
 /**
diff --git a/drivers/platform/x86/intel_scu_ipc.c b/drivers/platform/x86/intel_scu_ipc.c
index f7cf981502cd..2c85f75e32b0 100644
--- a/drivers/platform/x86/intel_scu_ipc.c
+++ b/drivers/platform/x86/intel_scu_ipc.c
@@ -72,20 +72,20 @@ struct intel_scu_ipc_pdata_t {
 	u8 irq_mode;
 };
 
-static struct intel_scu_ipc_pdata_t intel_scu_ipc_lincroft_pdata = {
+static const struct intel_scu_ipc_pdata_t intel_scu_ipc_lincroft_pdata = {
 	.i2c_base = 0xff12b000,
 	.i2c_len = 0x10,
 	.irq_mode = 0,
 };
 
 /* Penwell and Cloverview */
-static struct intel_scu_ipc_pdata_t intel_scu_ipc_penwell_pdata = {
+static const struct intel_scu_ipc_pdata_t intel_scu_ipc_penwell_pdata = {
 	.i2c_base = 0xff12b000,
 	.i2c_len = 0x10,
 	.irq_mode = 1,
 };
 
-static struct intel_scu_ipc_pdata_t intel_scu_ipc_tangier_pdata = {
+static const struct intel_scu_ipc_pdata_t intel_scu_ipc_tangier_pdata = {
 	.i2c_base  = 0xff00d000,
 	.i2c_len = 0x10,
 	.irq_mode = 0,
diff --git a/drivers/platform/x86/intel_telemetry_debugfs.c b/drivers/platform/x86/intel_telemetry_debugfs.c
index cd21df982abd..d4fc42b4cbeb 100644
--- a/drivers/platform/x86/intel_telemetry_debugfs.c
+++ b/drivers/platform/x86/intel_telemetry_debugfs.c
@@ -331,6 +331,7 @@ static struct telemetry_debugfs_conf telem_apl_debugfs_conf = {
 
 static const struct x86_cpu_id telemetry_debugfs_cpu_ids[] = {
 	TELEM_DEBUGFS_CPU(INTEL_FAM6_ATOM_GOLDMONT, telem_apl_debugfs_conf),
+	TELEM_DEBUGFS_CPU(INTEL_FAM6_ATOM_GEMINI_LAKE, telem_apl_debugfs_conf),
 	{}
 };
 
diff --git a/drivers/platform/x86/intel_telemetry_pltdrv.c b/drivers/platform/x86/intel_telemetry_pltdrv.c
index 6ebdbd2b04fc..e0424d5a795a 100644
--- a/drivers/platform/x86/intel_telemetry_pltdrv.c
+++ b/drivers/platform/x86/intel_telemetry_pltdrv.c
@@ -46,7 +46,6 @@
 #define TELEM_SAMPLING_DEFAULT_PERIOD	0xD
 
 #define TELEM_MAX_EVENTS_SRAM		28
-#define TELEM_MAX_OS_ALLOCATED_EVENTS	20
 #define TELEM_SSRAM_STARTTIME_OFFSET	8
 #define TELEM_SSRAM_EVTLOG_OFFSET	16
 
@@ -153,6 +152,30 @@ static struct telemetry_evtmap
 	{"PC2_AND_MEM_SHALLOW_IDLE_RES",	0x1D40},
 };
 
+static struct telemetry_evtmap
+	telemetry_glk_pss_default_events[TELEM_MAX_OS_ALLOCATED_EVENTS] = {
+	{"IA_CORE0_C6_RES",			0x0400},
+	{"IA_CORE0_C6_CTR",			0x0000},
+	{"IA_MODULE0_C7_RES",			0x0410},
+	{"IA_MODULE0_C7_CTR",			0x000C},
+	{"IA_C0_RES",				0x0805},
+	{"PCS_LTR",				0x2801},
+	{"PSTATES",				0x2802},
+	{"SOC_S0I3_RES",			0x0407},
+	{"SOC_S0I3_CTR",			0x0008},
+	{"PCS_S0I3_CTR",			0x0007},
+	{"PCS_C1E_RES",				0x0414},
+	{"PCS_IDLE_STATUS",			0x2806},
+	{"IA_PERF_LIMITS",			0x280B},
+	{"GT_PERF_LIMITS",			0x280C},
+	{"PCS_WAKEUP_S0IX_CTR",			0x0025},
+	{"PCS_IDLE_BLOCKED",			0x2C00},
+	{"PCS_S0IX_BLOCKED",			0x2C01},
+	{"PCS_S0IX_WAKE_REASONS",		0x2C02},
+	{"PCS_LTR_BLOCKING",			0x2C03},
+	{"PC2_AND_MEM_SHALLOW_IDLE_RES",	0x1D40},
+};
+
 /* APL specific Data */
 static struct telemetry_plt_config telem_apl_config = {
 	.pss_config = {
@@ -163,8 +186,19 @@ static struct telemetry_plt_config telem_apl_config = {
 	},
 };
 
+/* GLK specific Data */
+static struct telemetry_plt_config telem_glk_config = {
+	.pss_config = {
+		.telem_evts = telemetry_glk_pss_default_events,
+	},
+	.ioss_config = {
+		.telem_evts = telemetry_apl_ioss_default_events,
+	},
+};
+
 static const struct x86_cpu_id telemetry_cpu_ids[] = {
 	TELEM_CPU(INTEL_FAM6_ATOM_GOLDMONT, telem_apl_config),
+	TELEM_CPU(INTEL_FAM6_ATOM_GEMINI_LAKE, telem_glk_config),
 	{}
 };
 
diff --git a/drivers/platform/x86/mlx-platform.c b/drivers/platform/x86/mlx-platform.c
index 8f98c211b440..4f3de2a8c4df 100644
--- a/drivers/platform/x86/mlx-platform.c
+++ b/drivers/platform/x86/mlx-platform.c
@@ -247,7 +247,7 @@ static int __init mlxplat_dmi_msn21xx_matched(const struct dmi_system_id *dmi)
 	return 1;
 };
 
-static struct dmi_system_id mlxplat_dmi_table[] __initdata = {
+static const struct dmi_system_id mlxplat_dmi_table[] __initconst = {
 	{
 		.callback = mlxplat_dmi_default_matched,
 		.matches = {
diff --git a/drivers/platform/x86/msi-laptop.c b/drivers/platform/x86/msi-laptop.c
index 61b9014d2610..d5bfcc602090 100644
--- a/drivers/platform/x86/msi-laptop.c
+++ b/drivers/platform/x86/msi-laptop.c
@@ -605,7 +605,7 @@ static int dmi_check_cb(const struct dmi_system_id *dmi)
 	return 1;
 }
 
-static struct dmi_system_id __initdata msi_dmi_table[] = {
+static const struct dmi_system_id msi_dmi_table[] __initconst = {
 	{
 		.ident = "MSI S270",
 		.matches = {
diff --git a/drivers/platform/x86/msi-wmi.c b/drivers/platform/x86/msi-wmi.c
index f6209b739ec0..620138236c89 100644
--- a/drivers/platform/x86/msi-wmi.c
+++ b/drivers/platform/x86/msi-wmi.c
@@ -184,7 +184,7 @@ static const struct backlight_ops msi_backlight_ops = {
 static void msi_wmi_notify(u32 value, void *context)
 {
 	struct acpi_buffer response = { ACPI_ALLOCATE_BUFFER, NULL };
-	static struct key_entry *key;
+	struct key_entry *key;
 	union acpi_object *obj;
 	acpi_status status;
 
diff --git a/drivers/platform/x86/mxm-wmi.c b/drivers/platform/x86/mxm-wmi.c
index f4bad83053a9..35d8b9a939f9 100644
--- a/drivers/platform/x86/mxm-wmi.c
+++ b/drivers/platform/x86/mxm-wmi.c
@@ -53,7 +53,7 @@ int mxm_wmi_call_mxds(int adapter)
 
 	printk("calling mux switch %d\n", adapter);
 
-	status = wmi_evaluate_method(MXM_WMMX_GUID, 0x1, adapter, &input,
+	status = wmi_evaluate_method(MXM_WMMX_GUID, 0x0, adapter, &input,
 				     &output);
 
 	if (ACPI_FAILURE(status))
@@ -78,7 +78,7 @@ int mxm_wmi_call_mxmx(int adapter)
 
 	printk("calling mux switch %d\n", adapter);
 
-	status = wmi_evaluate_method(MXM_WMMX_GUID, 0x1, adapter, &input,
+	status = wmi_evaluate_method(MXM_WMMX_GUID, 0x0, adapter, &input,
 				     &output);
 
 	if (ACPI_FAILURE(status))
diff --git a/drivers/platform/x86/peaq-wmi.c b/drivers/platform/x86/peaq-wmi.c
index 77d1f90b0794..bc98ef95514a 100644
--- a/drivers/platform/x86/peaq-wmi.c
+++ b/drivers/platform/x86/peaq-wmi.c
@@ -39,7 +39,7 @@ static void peaq_wmi_poll(struct input_polled_dev *dev)
 	struct acpi_buffer input = { sizeof(dummy), &dummy };
 	struct acpi_buffer output = { sizeof(obj), &obj };
 
-	status = wmi_evaluate_method(PEAQ_DOLBY_BUTTON_GUID, 1,
+	status = wmi_evaluate_method(PEAQ_DOLBY_BUTTON_GUID, 0,
 				     PEAQ_DOLBY_BUTTON_METHOD_ID,
 				     &input, &output);
 	if (ACPI_FAILURE(status))
@@ -51,7 +51,7 @@ static void peaq_wmi_poll(struct input_polled_dev *dev)
 		return;
 	}
 
-	if (peaq_ignore_events_counter && --peaq_ignore_events_counter >= 0)
+	if (peaq_ignore_events_counter && peaq_ignore_events_counter--)
 		return;
 
 	if (obj.integer.value) {
diff --git a/drivers/platform/x86/samsung-laptop.c b/drivers/platform/x86/samsung-laptop.c
index 0c703feaeb88..d3cb26f6df73 100644
--- a/drivers/platform/x86/samsung-laptop.c
+++ b/drivers/platform/x86/samsung-laptop.c
@@ -1567,7 +1567,7 @@ static int __init samsung_dmi_matched(const struct dmi_system_id *d)
 	return 0;
 }
 
-static struct dmi_system_id __initdata samsung_dmi_table[] = {
+static const struct dmi_system_id samsung_dmi_table[] __initconst = {
 	{
 		.matches = {
 			DMI_MATCH(DMI_SYS_VENDOR,
diff --git a/drivers/platform/x86/samsung-q10.c b/drivers/platform/x86/samsung-q10.c
index e6aac725a0af..a2fb7fbc3273 100644
--- a/drivers/platform/x86/samsung-q10.c
+++ b/drivers/platform/x86/samsung-q10.c
@@ -95,7 +95,7 @@ static int __init dmi_check_callback(const struct dmi_system_id *id)
 	return 1;
 }
 
-static struct dmi_system_id __initdata samsungq10_dmi_table[] = {
+static const struct dmi_system_id samsungq10_dmi_table[] __initconst = {
 	{
 		.ident = "Samsung Q10",
 		.matches = {
diff --git a/drivers/platform/x86/sony-laptop.c b/drivers/platform/x86/sony-laptop.c
index bfae79534f44..a16cea2be9c3 100644
--- a/drivers/platform/x86/sony-laptop.c
+++ b/drivers/platform/x86/sony-laptop.c
@@ -4880,7 +4880,7 @@ static struct acpi_driver sony_pic_driver = {
 	.drv.pm = &sony_pic_pm,
 };
 
-static struct dmi_system_id __initdata sonypi_dmi_table[] = {
+static const struct dmi_system_id sonypi_dmi_table[] __initconst = {
 	{
 		.ident = "Sony Vaio",
 		.matches = {
diff --git a/drivers/platform/x86/thinkpad_acpi.c b/drivers/platform/x86/thinkpad_acpi.c
index b22573131e53..2242d6035d9e 100644
--- a/drivers/platform/x86/thinkpad_acpi.c
+++ b/drivers/platform/x86/thinkpad_acpi.c
@@ -24,7 +24,7 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #define TPACPI_VERSION "0.25"
-#define TPACPI_SYSFS_VERSION 0x020700
+#define TPACPI_SYSFS_VERSION 0x030000
 
 /*
  *  Changelog:
@@ -6342,7 +6342,7 @@ static int __init thermal_init(struct ibm_init_struct *iibm)
 
 	switch (thermal_read_mode) {
 	case TPACPI_THERMAL_TPEC_16:
-		res = sysfs_create_group(&tpacpi_sensors_pdev->dev.kobj,
+		res = sysfs_create_group(&tpacpi_hwmon->kobj,
 				&thermal_temp_input16_group);
 		if (res)
 			return res;
@@ -6350,7 +6350,7 @@ static int __init thermal_init(struct ibm_init_struct *iibm)
 	case TPACPI_THERMAL_TPEC_8:
 	case TPACPI_THERMAL_ACPI_TMP07:
 	case TPACPI_THERMAL_ACPI_UPDT:
-		res = sysfs_create_group(&tpacpi_sensors_pdev->dev.kobj,
+		res = sysfs_create_group(&tpacpi_hwmon->kobj,
 				&thermal_temp_input8_group);
 		if (res)
 			return res;
@@ -6367,13 +6367,13 @@ static void thermal_exit(void)
 {
 	switch (thermal_read_mode) {
 	case TPACPI_THERMAL_TPEC_16:
-		sysfs_remove_group(&tpacpi_sensors_pdev->dev.kobj,
+		sysfs_remove_group(&tpacpi_hwmon->kobj,
 				   &thermal_temp_input16_group);
 		break;
 	case TPACPI_THERMAL_TPEC_8:
 	case TPACPI_THERMAL_ACPI_TMP07:
 	case TPACPI_THERMAL_ACPI_UPDT:
-		sysfs_remove_group(&tpacpi_sensors_pdev->dev.kobj,
+		sysfs_remove_group(&tpacpi_hwmon->kobj,
 				   &thermal_temp_input8_group);
 		break;
 	case TPACPI_THERMAL_NONE:
@@ -8696,7 +8696,7 @@ static int __init fan_init(struct ibm_init_struct *iibm)
 			fan_attributes[ARRAY_SIZE(fan_attributes)-2] =
 					&dev_attr_fan2_input.attr;
 		}
-		rc = sysfs_create_group(&tpacpi_sensors_pdev->dev.kobj,
+		rc = sysfs_create_group(&tpacpi_hwmon->kobj,
 					 &fan_attr_group);
 		if (rc < 0)
 			return rc;
@@ -8704,7 +8704,7 @@ static int __init fan_init(struct ibm_init_struct *iibm)
 		rc = driver_create_file(&tpacpi_hwmon_pdriver.driver,
 					&driver_attr_fan_watchdog);
 		if (rc < 0) {
-			sysfs_remove_group(&tpacpi_sensors_pdev->dev.kobj,
+			sysfs_remove_group(&tpacpi_hwmon->kobj,
 					&fan_attr_group);
 			return rc;
 		}
@@ -8719,7 +8719,7 @@ static void fan_exit(void)
 		    "cancelling any pending fan watchdog tasks\n");
 
 	/* FIXME: can we really do this unconditionally? */
-	sysfs_remove_group(&tpacpi_sensors_pdev->dev.kobj, &fan_attr_group);
+	sysfs_remove_group(&tpacpi_hwmon->kobj, &fan_attr_group);
 	driver_remove_file(&tpacpi_hwmon_pdriver.driver,
 			   &driver_attr_fan_watchdog);
 
@@ -9149,16 +9149,6 @@ static void hotkey_driver_event(const unsigned int scancode)
 	tpacpi_driver_event(TP_HKEY_EV_HOTKEY_BASE + scancode);
 }
 
-/* sysfs name ---------------------------------------------------------- */
-static ssize_t thinkpad_acpi_pdev_name_show(struct device *dev,
-			   struct device_attribute *attr,
-			   char *buf)
-{
-	return snprintf(buf, PAGE_SIZE, "%s\n", TPACPI_NAME);
-}
-
-static DEVICE_ATTR(name, S_IRUGO, thinkpad_acpi_pdev_name_show, NULL);
-
 /* --------------------------------------------------------------------- */
 
 /* /proc support */
@@ -9696,8 +9686,6 @@ static void thinkpad_acpi_module_exit(void)
 	if (tpacpi_hwmon)
 		hwmon_device_unregister(tpacpi_hwmon);
 
-	if (tp_features.sensors_pdev_attrs_registered)
-		device_remove_file(&tpacpi_sensors_pdev->dev, &dev_attr_name);
 	if (tpacpi_sensors_pdev)
 		platform_device_unregister(tpacpi_sensors_pdev);
 	if (tpacpi_pdev)
@@ -9818,14 +9806,10 @@ static int __init thinkpad_acpi_module_init(void)
 		thinkpad_acpi_module_exit();
 		return ret;
 	}
-	ret = device_create_file(&tpacpi_sensors_pdev->dev, &dev_attr_name);
-	if (ret) {
-		pr_err("unable to create sysfs hwmon device attributes\n");
-		thinkpad_acpi_module_exit();
-		return ret;
-	}
 	tp_features.sensors_pdev_attrs_registered = 1;
-	tpacpi_hwmon = hwmon_device_register(&tpacpi_sensors_pdev->dev);
+	tpacpi_hwmon = hwmon_device_register_with_groups(
+		&tpacpi_sensors_pdev->dev, TPACPI_NAME, NULL, NULL);
+
 	if (IS_ERR(tpacpi_hwmon)) {
 		ret = PTR_ERR(tpacpi_hwmon);
 		tpacpi_hwmon = NULL;
diff --git a/drivers/platform/x86/toshiba-wmi.c b/drivers/platform/x86/toshiba-wmi.c
index 440528676170..03d7620cd6d7 100644
--- a/drivers/platform/x86/toshiba-wmi.c
+++ b/drivers/platform/x86/toshiba-wmi.c
@@ -64,7 +64,7 @@ static void toshiba_wmi_notify(u32 value, void *context)
 	kfree(response.pointer);
 }
 
-static struct dmi_system_id toshiba_wmi_dmi_table[] __initdata = {
+static const struct dmi_system_id toshiba_wmi_dmi_table[] __initconst = {
 	{
 		.ident = "Toshiba laptop",
 		.matches = {
diff --git a/drivers/platform/x86/wmi.c b/drivers/platform/x86/wmi.c
index e32ba575e8d9..0765b1797d4c 100644
--- a/drivers/platform/x86/wmi.c
+++ b/drivers/platform/x86/wmi.c
@@ -218,7 +218,7 @@ u32 method_id, const struct acpi_buffer *in, struct acpi_buffer *out)
 	if (!(block->flags & ACPI_WMI_METHOD))
 		return AE_BAD_DATA;
 
-	if (block->instance_count < instance)
+	if (block->instance_count <= instance)
 		return AE_BAD_PARAMETER;
 
 	input.count = 2;
@@ -265,7 +265,7 @@ static acpi_status __query_block(struct wmi_block *wblock, u8 instance,
 	block = &wblock->gblock;
 	handle = wblock->acpi_device->handle;
 
-	if (block->instance_count < instance)
+	if (block->instance_count <= instance)
 		return AE_BAD_PARAMETER;
 
 	/* Check GUID is a data block */
@@ -392,7 +392,7 @@ acpi_status wmi_set_block(const char *guid_string, u8 instance,
 	block = &wblock->gblock;
 	handle = wblock->acpi_device->handle;
 
-	if (block->instance_count < instance)
+	if (block->instance_count <= instance)
 		return AE_BAD_PARAMETER;
 
 	/* Check GUID is a data block */
diff --git a/drivers/pnp/pnpbios/core.c b/drivers/pnp/pnpbios/core.c
index 0ced908e7aa8..e681140b85d8 100644
--- a/drivers/pnp/pnpbios/core.c
+++ b/drivers/pnp/pnpbios/core.c
@@ -495,7 +495,7 @@ static int __init exploding_pnp_bios(const struct dmi_system_id *d)
 	return 0;
 }
 
-static struct dmi_system_id pnpbios_dmi_table[] __initdata = {
+static const struct dmi_system_id pnpbios_dmi_table[] __initconst = {
 	{			/* PnPBIOS GPF on boot */
 	 .callback = exploding_pnp_bios,
 	 .ident = "Higraded P14H",
diff --git a/drivers/power/avs/rockchip-io-domain.c b/drivers/power/avs/rockchip-io-domain.c
index 031a34372191..75f63e38a8d1 100644
--- a/drivers/power/avs/rockchip-io-domain.c
+++ b/drivers/power/avs/rockchip-io-domain.c
@@ -349,6 +349,36 @@ static const struct rockchip_iodomain_soc_data soc_data_rk3399_pmu = {
 	.init = rk3399_pmu_iodomain_init,
 };
 
+static const struct rockchip_iodomain_soc_data soc_data_rv1108 = {
+	.grf_offset = 0x404,
+	.supply_names = {
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+		NULL,
+		"vccio1",
+		"vccio2",
+		"vccio3",
+		"vccio5",
+		"vccio6",
+	},
+
+};
+
+static const struct rockchip_iodomain_soc_data soc_data_rv1108_pmu = {
+	.grf_offset = 0x104,
+	.supply_names = {
+		"pmu",
+	},
+};
+
 static const struct of_device_id rockchip_iodomain_match[] = {
 	{
 		.compatible = "rockchip,rk3188-io-voltage-domain",
@@ -382,6 +412,14 @@ static const struct of_device_id rockchip_iodomain_match[] = {
 		.compatible = "rockchip,rk3399-pmu-io-voltage-domain",
 		.data = (void *)&soc_data_rk3399_pmu
 	},
+	{
+		.compatible = "rockchip,rv1108-io-voltage-domain",
+		.data = (void *)&soc_data_rv1108
+	},
+	{
+		.compatible = "rockchip,rv1108-pmu-io-voltage-domain",
+		.data = (void *)&soc_data_rv1108_pmu
+	},
 	{ /* sentinel */ },
 };
 MODULE_DEVICE_TABLE(of, rockchip_iodomain_match);
diff --git a/drivers/power/reset/at91-sama5d2_shdwc.c b/drivers/power/reset/at91-sama5d2_shdwc.c
index 55fce8b75245..31080c254124 100644
--- a/drivers/power/reset/at91-sama5d2_shdwc.c
+++ b/drivers/power/reset/at91-sama5d2_shdwc.c
@@ -171,8 +171,8 @@ static u32 at91_shdwc_get_wakeup_input(struct platform_device *pdev,
 
 	for_each_child_of_node(np, cnp) {
 		if (of_property_read_u32(cnp, "reg", &wk_input)) {
-			dev_warn(&pdev->dev, "reg property is missing for %s\n",
-				 cnp->full_name);
+			dev_warn(&pdev->dev, "reg property is missing for %pOF\n",
+				 cnp);
 			continue;
 		}
 
diff --git a/drivers/power/supply/Kconfig b/drivers/power/supply/Kconfig
index 969f5005669c..5ab90c1f3f7c 100644
--- a/drivers/power/supply/Kconfig
+++ b/drivers/power/supply/Kconfig
@@ -198,6 +198,15 @@ config BATTERY_BQ27XXX_I2C
 	  Say Y here to enable support for batteries with BQ27xxx chips
 	  connected over an I2C bus.
 
+config BATTERY_BQ27XXX_HDQ
+	tristate "BQ27xxx HDQ support"
+	depends on BATTERY_BQ27XXX
+	depends on W1
+	default y
+	help
+	  Say Y here to enable support for batteries with BQ27xxx chips
+	  connected over an HDQ bus.
+
 config BATTERY_BQ27XXX_DT_UPDATES_NVM
 	bool "BQ27xxx support for update of NVM/flash data memory"
 	depends on BATTERY_BQ27XXX_I2C
@@ -313,6 +322,19 @@ config BATTERY_MAX17042
 	  with MAX17042. This driver also supports max17047/50 chips which are
 	  improved version of max17042.
 
+config BATTERY_MAX1721X
+	tristate "MAX17211/MAX17215 standalone gas-gauge"
+	depends on W1
+	select REGMAP_W1
+	help
+	  MAX1721x is fuel-gauge systems for lithium-ion (Li+) batteries
+	  in handheld and portable equipment. MAX17211 used with single cell
+	  battery. MAX17215 designed for muticell battery. Both them have
+	  OneWire (W1) host interface.
+
+	  Say Y here to enable support for the MAX17211/MAX17215 standalone
+	  battery gas-gauge.
+
 config BATTERY_Z2
 	tristate "Z2 battery driver"
 	depends on I2C && MACH_ZIPIT2
@@ -365,6 +387,7 @@ config BATTERY_RX51
 config CHARGER_CPCAP
 	tristate "CPCAP PMIC Charger Driver"
 	depends on MFD_CPCAP && IIO
+	depends on OMAP_USB2 || (!OMAP_USB2 && COMPILE_TEST)
 	default MFD_CPCAP
 	help
 	  Say Y to enable support for CPCAP PMIC charger driver for Motorola
diff --git a/drivers/power/supply/Makefile b/drivers/power/supply/Makefile
index a41f40957847..621a19058fec 100644
--- a/drivers/power/supply/Makefile
+++ b/drivers/power/supply/Makefile
@@ -38,12 +38,14 @@ obj-$(CONFIG_BATTERY_SBS)	+= sbs-battery.o
 obj-$(CONFIG_CHARGER_SBS)	+= sbs-charger.o
 obj-$(CONFIG_BATTERY_BQ27XXX)	+= bq27xxx_battery.o
 obj-$(CONFIG_BATTERY_BQ27XXX_I2C) += bq27xxx_battery_i2c.o
+obj-$(CONFIG_BATTERY_BQ27XXX_HDQ) += bq27xxx_battery_hdq.o
 obj-$(CONFIG_BATTERY_DA9030)	+= da9030_battery.o
 obj-$(CONFIG_BATTERY_DA9052)	+= da9052-battery.o
 obj-$(CONFIG_CHARGER_DA9150)	+= da9150-charger.o
 obj-$(CONFIG_BATTERY_DA9150)	+= da9150-fg.o
 obj-$(CONFIG_BATTERY_MAX17040)	+= max17040_battery.o
 obj-$(CONFIG_BATTERY_MAX17042)	+= max17042_battery.o
+obj-$(CONFIG_BATTERY_MAX1721X)	+= max1721x_battery.o
 obj-$(CONFIG_BATTERY_Z2)	+= z2_battery.o
 obj-$(CONFIG_BATTERY_RT5033)	+= rt5033_battery.o
 obj-$(CONFIG_CHARGER_RT9455)	+= rt9455_charger.o
diff --git a/drivers/power/supply/act8945a_charger.c b/drivers/power/supply/act8945a_charger.c
index d1eb2e359532..8e117b31ba79 100644
--- a/drivers/power/supply/act8945a_charger.c
+++ b/drivers/power/supply/act8945a_charger.c
@@ -596,9 +596,9 @@ static int act8945a_charger_probe(struct platform_device *pdev)
 		return ret;
 
 	irq = of_irq_get(pdev->dev.of_node, 0);
-	if (irq == -EPROBE_DEFER) {
+	if (irq <= 0) {
 		dev_err(&pdev->dev, "failed to find IRQ number\n");
-		return -EPROBE_DEFER;
+		return irq ?: -ENXIO;
 	}
 
 	ret = devm_request_irq(&pdev->dev, irq, act8945a_status_changed,
diff --git a/drivers/power/supply/bq24190_charger.c b/drivers/power/supply/bq24190_charger.c
index d5a707e14526..35ff406aca48 100644
--- a/drivers/power/supply/bq24190_charger.c
+++ b/drivers/power/supply/bq24190_charger.c
@@ -16,6 +16,9 @@
 #include <linux/of_device.h>
 #include <linux/pm_runtime.h>
 #include <linux/power_supply.h>
+#include <linux/power/bq24190_charger.h>
+#include <linux/regulator/driver.h>
+#include <linux/regulator/machine.h>
 #include <linux/workqueue.h>
 #include <linux/gpio.h>
 #include <linux/i2c.h>
@@ -43,6 +46,8 @@
 #define BQ24190_REG_POC_CHG_CONFIG_OTG			0x2
 #define BQ24190_REG_POC_SYS_MIN_MASK		(BIT(3) | BIT(2) | BIT(1))
 #define BQ24190_REG_POC_SYS_MIN_SHIFT		1
+#define BQ24190_REG_POC_SYS_MIN_MIN			3000
+#define BQ24190_REG_POC_SYS_MIN_MAX			3700
 #define BQ24190_REG_POC_BOOST_LIM_MASK		BIT(0)
 #define BQ24190_REG_POC_BOOST_LIM_SHIFT		0
 
@@ -57,9 +62,13 @@
 #define BQ24190_REG_PCTCC_IPRECHG_MASK		(BIT(7) | BIT(6) | BIT(5) | \
 						 BIT(4))
 #define BQ24190_REG_PCTCC_IPRECHG_SHIFT		4
+#define BQ24190_REG_PCTCC_IPRECHG_MIN			128
+#define BQ24190_REG_PCTCC_IPRECHG_MAX			2048
 #define BQ24190_REG_PCTCC_ITERM_MASK		(BIT(3) | BIT(2) | BIT(1) | \
 						 BIT(0))
 #define BQ24190_REG_PCTCC_ITERM_SHIFT		0
+#define BQ24190_REG_PCTCC_ITERM_MIN			128
+#define BQ24190_REG_PCTCC_ITERM_MAX			2048
 
 #define BQ24190_REG_CVC		0x04 /* Charge Voltage Control */
 #define BQ24190_REG_CVC_VREG_MASK		(BIT(7) | BIT(6) | BIT(5) | \
@@ -156,9 +165,13 @@ struct bq24190_dev_info {
 	struct extcon_dev		*extcon;
 	struct notifier_block		extcon_nb;
 	struct delayed_work		extcon_work;
+	struct delayed_work		input_current_limit_work;
 	char				model_name[I2C_NAME_SIZE];
 	bool				initialized;
 	bool				irq_event;
+	u16				sys_min;
+	u16				iprechg;
+	u16				iterm;
 	struct mutex			f_reg_lock;
 	u8				f_reg;
 	u8				ss_reg;
@@ -504,15 +517,112 @@ static int bq24190_sysfs_create_group(struct bq24190_dev_info *bdi)
 static inline void bq24190_sysfs_remove_group(struct bq24190_dev_info *bdi) {}
 #endif
 
-/*
- * According to the "Host Mode and default Mode" section of the
- * manual, a write to any register causes the bq24190 to switch
- * from default mode to host mode.  It will switch back to default
- * mode after a WDT timeout unless the WDT is turned off as well.
- * So, by simply turning off the WDT, we accomplish both with the
- * same write.
- */
-static int bq24190_set_mode_host(struct bq24190_dev_info *bdi)
+#ifdef CONFIG_REGULATOR
+static int bq24190_set_charge_mode(struct regulator_dev *dev, u8 val)
+{
+	struct bq24190_dev_info *bdi = rdev_get_drvdata(dev);
+	int ret;
+
+	ret = pm_runtime_get_sync(bdi->dev);
+	if (ret < 0) {
+		dev_warn(bdi->dev, "pm_runtime_get failed: %i\n", ret);
+		pm_runtime_put_noidle(bdi->dev);
+		return ret;
+	}
+
+	ret = bq24190_write_mask(bdi, BQ24190_REG_POC,
+				 BQ24190_REG_POC_CHG_CONFIG_MASK,
+				 BQ24190_REG_POC_CHG_CONFIG_SHIFT, val);
+
+	pm_runtime_mark_last_busy(bdi->dev);
+	pm_runtime_put_autosuspend(bdi->dev);
+
+	return ret;
+}
+
+static int bq24190_vbus_enable(struct regulator_dev *dev)
+{
+	return bq24190_set_charge_mode(dev, BQ24190_REG_POC_CHG_CONFIG_OTG);
+}
+
+static int bq24190_vbus_disable(struct regulator_dev *dev)
+{
+	return bq24190_set_charge_mode(dev, BQ24190_REG_POC_CHG_CONFIG_CHARGE);
+}
+
+static int bq24190_vbus_is_enabled(struct regulator_dev *dev)
+{
+	struct bq24190_dev_info *bdi = rdev_get_drvdata(dev);
+	int ret;
+	u8 val;
+
+	ret = pm_runtime_get_sync(bdi->dev);
+	if (ret < 0) {
+		dev_warn(bdi->dev, "pm_runtime_get failed: %i\n", ret);
+		pm_runtime_put_noidle(bdi->dev);
+		return ret;
+	}
+
+	ret = bq24190_read_mask(bdi, BQ24190_REG_POC,
+				BQ24190_REG_POC_CHG_CONFIG_MASK,
+				BQ24190_REG_POC_CHG_CONFIG_SHIFT, &val);
+
+	pm_runtime_mark_last_busy(bdi->dev);
+	pm_runtime_put_autosuspend(bdi->dev);
+
+	return ret ? ret : val == BQ24190_REG_POC_CHG_CONFIG_OTG;
+}
+
+static const struct regulator_ops bq24190_vbus_ops = {
+	.enable = bq24190_vbus_enable,
+	.disable = bq24190_vbus_disable,
+	.is_enabled = bq24190_vbus_is_enabled,
+};
+
+static const struct regulator_desc bq24190_vbus_desc = {
+	.name = "usb_otg_vbus",
+	.type = REGULATOR_VOLTAGE,
+	.owner = THIS_MODULE,
+	.ops = &bq24190_vbus_ops,
+	.fixed_uV = 5000000,
+	.n_voltages = 1,
+};
+
+static const struct regulator_init_data bq24190_vbus_init_data = {
+	.constraints = {
+		.valid_ops_mask = REGULATOR_CHANGE_STATUS,
+	},
+};
+
+static int bq24190_register_vbus_regulator(struct bq24190_dev_info *bdi)
+{
+	struct bq24190_platform_data *pdata = bdi->dev->platform_data;
+	struct regulator_config cfg = { };
+	struct regulator_dev *reg;
+	int ret = 0;
+
+	cfg.dev = bdi->dev;
+	if (pdata && pdata->regulator_init_data)
+		cfg.init_data = pdata->regulator_init_data;
+	else
+		cfg.init_data = &bq24190_vbus_init_data;
+	cfg.driver_data = bdi;
+	reg = devm_regulator_register(bdi->dev, &bq24190_vbus_desc, &cfg);
+	if (IS_ERR(reg)) {
+		ret = PTR_ERR(reg);
+		dev_err(bdi->dev, "Can't register regulator: %d\n", ret);
+	}
+
+	return ret;
+}
+#else
+static int bq24190_register_vbus_regulator(struct bq24190_dev_info *bdi)
+{
+	return 0;
+}
+#endif
+
+static int bq24190_set_config(struct bq24190_dev_info *bdi)
 {
 	int ret;
 	u8 v;
@@ -523,9 +633,52 @@ static int bq24190_set_mode_host(struct bq24190_dev_info *bdi)
 
 	bdi->watchdog = ((v & BQ24190_REG_CTTC_WATCHDOG_MASK) >>
 					BQ24190_REG_CTTC_WATCHDOG_SHIFT);
+
+	/*
+	 * According to the "Host Mode and default Mode" section of the
+	 * manual, a write to any register causes the bq24190 to switch
+	 * from default mode to host mode.  It will switch back to default
+	 * mode after a WDT timeout unless the WDT is turned off as well.
+	 * So, by simply turning off the WDT, we accomplish both with the
+	 * same write.
+	 */
 	v &= ~BQ24190_REG_CTTC_WATCHDOG_MASK;
 
-	return bq24190_write(bdi, BQ24190_REG_CTTC, v);
+	ret = bq24190_write(bdi, BQ24190_REG_CTTC, v);
+	if (ret < 0)
+		return ret;
+
+	if (bdi->sys_min) {
+		v = bdi->sys_min / 100 - 30; // manual section 9.5.1.2, table 9
+		ret = bq24190_write_mask(bdi, BQ24190_REG_POC,
+					 BQ24190_REG_POC_SYS_MIN_MASK,
+					 BQ24190_REG_POC_SYS_MIN_SHIFT,
+					 v);
+		if (ret < 0)
+			return ret;
+	}
+
+	if (bdi->iprechg) {
+		v = bdi->iprechg / 128 - 1; // manual section 9.5.1.4, table 11
+		ret = bq24190_write_mask(bdi, BQ24190_REG_PCTCC,
+					 BQ24190_REG_PCTCC_IPRECHG_MASK,
+					 BQ24190_REG_PCTCC_IPRECHG_SHIFT,
+					 v);
+		if (ret < 0)
+			return ret;
+	}
+
+	if (bdi->iterm) {
+		v = bdi->iterm / 128 - 1; // manual section 9.5.1.4, table 11
+		ret = bq24190_write_mask(bdi, BQ24190_REG_PCTCC,
+					 BQ24190_REG_PCTCC_ITERM_MASK,
+					 BQ24190_REG_PCTCC_ITERM_SHIFT,
+					 v);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
 }
 
 static int bq24190_register_reset(struct bq24190_dev_info *bdi)
@@ -773,6 +926,38 @@ static int bq24190_charger_set_temp_alert_max(struct bq24190_dev_info *bdi,
 	return bq24190_battery_set_temp_alert_max(bdi, val);
 }
 
+static int bq24190_charger_get_precharge(struct bq24190_dev_info *bdi,
+		union power_supply_propval *val)
+{
+	u8 v;
+	int ret;
+
+	ret = bq24190_read_mask(bdi, BQ24190_REG_PCTCC,
+			BQ24190_REG_PCTCC_IPRECHG_MASK,
+			BQ24190_REG_PCTCC_IPRECHG_SHIFT, &v);
+	if (ret < 0)
+		return ret;
+
+	val->intval = ++v * 128 * 1000;
+	return 0;
+}
+
+static int bq24190_charger_get_charge_term(struct bq24190_dev_info *bdi,
+		union power_supply_propval *val)
+{
+	u8 v;
+	int ret;
+
+	ret = bq24190_read_mask(bdi, BQ24190_REG_PCTCC,
+			BQ24190_REG_PCTCC_ITERM_MASK,
+			BQ24190_REG_PCTCC_ITERM_SHIFT, &v);
+	if (ret < 0)
+		return ret;
+
+	val->intval = ++v * 128 * 1000;
+	return 0;
+}
+
 static int bq24190_charger_get_current(struct bq24190_dev_info *bdi,
 		union power_supply_propval *val)
 {
@@ -865,6 +1050,33 @@ static int bq24190_charger_set_voltage(struct bq24190_dev_info *bdi,
 			ARRAY_SIZE(bq24190_cvc_vreg_values), val->intval);
 }
 
+static int bq24190_charger_get_iinlimit(struct bq24190_dev_info *bdi,
+		union power_supply_propval *val)
+{
+	int iinlimit, ret;
+
+	ret = bq24190_get_field_val(bdi, BQ24190_REG_ISC,
+			BQ24190_REG_ISC_IINLIM_MASK,
+			BQ24190_REG_ISC_IINLIM_SHIFT,
+			bq24190_isc_iinlim_values,
+			ARRAY_SIZE(bq24190_isc_iinlim_values), &iinlimit);
+	if (ret < 0)
+		return ret;
+
+	val->intval = iinlimit;
+	return 0;
+}
+
+static int bq24190_charger_set_iinlimit(struct bq24190_dev_info *bdi,
+		const union power_supply_propval *val)
+{
+	return bq24190_set_field_val(bdi, BQ24190_REG_ISC,
+			BQ24190_REG_ISC_IINLIM_MASK,
+			BQ24190_REG_ISC_IINLIM_SHIFT,
+			bq24190_isc_iinlim_values,
+			ARRAY_SIZE(bq24190_isc_iinlim_values), val->intval);
+}
+
 static int bq24190_charger_get_property(struct power_supply *psy,
 		enum power_supply_property psp, union power_supply_propval *val)
 {
@@ -893,6 +1105,12 @@ static int bq24190_charger_get_property(struct power_supply *psy,
 	case POWER_SUPPLY_PROP_TEMP_ALERT_MAX:
 		ret =  bq24190_charger_get_temp_alert_max(bdi, val);
 		break;
+	case POWER_SUPPLY_PROP_PRECHARGE_CURRENT:
+		ret = bq24190_charger_get_precharge(bdi, val);
+		break;
+	case POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT:
+		ret = bq24190_charger_get_charge_term(bdi, val);
+		break;
 	case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT:
 		ret = bq24190_charger_get_current(bdi, val);
 		break;
@@ -905,6 +1123,9 @@ static int bq24190_charger_get_property(struct power_supply *psy,
 	case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE_MAX:
 		ret = bq24190_charger_get_voltage_max(bdi, val);
 		break;
+	case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT:
+		ret = bq24190_charger_get_iinlimit(bdi, val);
+		break;
 	case POWER_SUPPLY_PROP_SCOPE:
 		val->intval = POWER_SUPPLY_SCOPE_SYSTEM;
 		ret = 0;
@@ -956,6 +1177,9 @@ static int bq24190_charger_set_property(struct power_supply *psy,
 	case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE:
 		ret = bq24190_charger_set_voltage(bdi, val);
 		break;
+	case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT:
+		ret = bq24190_charger_set_iinlimit(bdi, val);
+		break;
 	default:
 		ret = -EINVAL;
 	}
@@ -977,6 +1201,7 @@ static int bq24190_charger_property_is_writeable(struct power_supply *psy,
 	case POWER_SUPPLY_PROP_CHARGE_TYPE:
 	case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT:
 	case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE:
+	case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT:
 		ret = 1;
 		break;
 	default:
@@ -986,16 +1211,45 @@ static int bq24190_charger_property_is_writeable(struct power_supply *psy,
 	return ret;
 }
 
+static void bq24190_input_current_limit_work(struct work_struct *work)
+{
+	struct bq24190_dev_info *bdi =
+		container_of(work, struct bq24190_dev_info,
+			     input_current_limit_work.work);
+
+	power_supply_set_input_current_limit_from_supplier(bdi->charger);
+}
+
+/* Sync the input-current-limit with our parent supply (if we have one) */
+static void bq24190_charger_external_power_changed(struct power_supply *psy)
+{
+	struct bq24190_dev_info *bdi = power_supply_get_drvdata(psy);
+
+	/*
+	 * The Power-Good detection may take up to 220ms, sometimes
+	 * the external charger detection is quicker, and the bq24190 will
+	 * reset to iinlim based on its own charger detection (which is not
+	 * hooked up when using external charger detection) resulting in a
+	 * too low default 500mA iinlim. Delay setting the input-current-limit
+	 * for 300ms to avoid this.
+	 */
+	queue_delayed_work(system_wq, &bdi->input_current_limit_work,
+			   msecs_to_jiffies(300));
+}
+
 static enum power_supply_property bq24190_charger_properties[] = {
 	POWER_SUPPLY_PROP_CHARGE_TYPE,
 	POWER_SUPPLY_PROP_HEALTH,
 	POWER_SUPPLY_PROP_ONLINE,
 	POWER_SUPPLY_PROP_STATUS,
 	POWER_SUPPLY_PROP_TEMP_ALERT_MAX,
+	POWER_SUPPLY_PROP_PRECHARGE_CURRENT,
+	POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT,
 	POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT,
 	POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX,
 	POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE,
 	POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE_MAX,
+	POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT,
 	POWER_SUPPLY_PROP_SCOPE,
 	POWER_SUPPLY_PROP_MODEL_NAME,
 	POWER_SUPPLY_PROP_MANUFACTURER,
@@ -1013,6 +1267,7 @@ static const struct power_supply_desc bq24190_charger_desc = {
 	.get_property		= bq24190_charger_get_property,
 	.set_property		= bq24190_charger_set_property,
 	.property_is_writeable	= bq24190_charger_property_is_writeable,
+	.external_power_changed	= bq24190_charger_external_power_changed,
 };
 
 /* Battery power supply property routines */
@@ -1460,13 +1715,50 @@ static int bq24190_hw_init(struct bq24190_dev_info *bdi)
 	if (ret < 0)
 		return ret;
 
-	ret = bq24190_set_mode_host(bdi);
+	ret = bq24190_set_config(bdi);
 	if (ret < 0)
 		return ret;
 
 	return bq24190_read(bdi, BQ24190_REG_SS, &bdi->ss_reg);
 }
 
+static int bq24190_get_config(struct bq24190_dev_info *bdi)
+{
+	const char * const s = "ti,system-minimum-microvolt";
+	struct power_supply_battery_info info = {};
+	int v;
+
+	if (device_property_read_u32(bdi->dev, s, &v) == 0) {
+		v /= 1000;
+		if (v >= BQ24190_REG_POC_SYS_MIN_MIN
+		 && v <= BQ24190_REG_POC_SYS_MIN_MAX)
+			bdi->sys_min = v;
+		else
+			dev_warn(bdi->dev, "invalid value for %s: %u\n", s, v);
+	}
+
+	if (bdi->dev->of_node &&
+	    !power_supply_get_battery_info(bdi->charger, &info)) {
+		v = info.precharge_current_ua / 1000;
+		if (v >= BQ24190_REG_PCTCC_IPRECHG_MIN
+		 && v <= BQ24190_REG_PCTCC_IPRECHG_MAX)
+			bdi->iprechg = v;
+		else
+			dev_warn(bdi->dev, "invalid value for battery:precharge-current-microamp: %d\n",
+				 v);
+
+		v = info.charge_term_current_ua / 1000;
+		if (v >= BQ24190_REG_PCTCC_ITERM_MIN
+		 && v <= BQ24190_REG_PCTCC_ITERM_MAX)
+			bdi->iterm = v;
+		else
+			dev_warn(bdi->dev, "invalid value for battery:charge-term-current-microamp: %d\n",
+				 v);
+	}
+
+	return 0;
+}
+
 static int bq24190_probe(struct i2c_client *client,
 		const struct i2c_device_id *id)
 {
@@ -1494,10 +1786,12 @@ static int bq24190_probe(struct i2c_client *client,
 	mutex_init(&bdi->f_reg_lock);
 	bdi->f_reg = 0;
 	bdi->ss_reg = BQ24190_REG_SS_VBUS_STAT_MASK; /* impossible state */
+	INIT_DELAYED_WORK(&bdi->input_current_limit_work,
+			  bq24190_input_current_limit_work);
 
 	i2c_set_clientdata(client, bdi);
 
-	if (!client->irq) {
+	if (client->irq <= 0) {
 		dev_err(dev, "Can't get irq info\n");
 		return -EINVAL;
 	}
@@ -1530,13 +1824,8 @@ static int bq24190_probe(struct i2c_client *client,
 		goto out_pmrt;
 	}
 
-	ret = bq24190_hw_init(bdi);
-	if (ret < 0) {
-		dev_err(dev, "Hardware init failed\n");
-		goto out_pmrt;
-	}
-
 	charger_cfg.drv_data = bdi;
+	charger_cfg.of_node = dev->of_node;
 	charger_cfg.supplied_to = bq24190_charger_supplied_to;
 	charger_cfg.num_supplicants = ARRAY_SIZE(bq24190_charger_supplied_to),
 	bdi->charger = power_supply_register(dev, &bq24190_charger_desc,
@@ -1560,8 +1849,20 @@ static int bq24190_probe(struct i2c_client *client,
 		}
 	}
 
+	ret = bq24190_get_config(bdi);
+	if (ret < 0) {
+		dev_err(dev, "Can't get devicetree config\n");
+		goto out_charger;
+	}
+
+	ret = bq24190_hw_init(bdi);
+	if (ret < 0) {
+		dev_err(dev, "Hardware init failed\n");
+		goto out_charger;
+	}
+
 	ret = bq24190_sysfs_create_group(bdi);
-	if (ret) {
+	if (ret < 0) {
 		dev_err(dev, "Can't create sysfs entries\n");
 		goto out_charger;
 	}
@@ -1577,6 +1878,10 @@ static int bq24190_probe(struct i2c_client *client,
 		goto out_sysfs;
 	}
 
+	ret = bq24190_register_vbus_regulator(bdi);
+	if (ret < 0)
+		goto out_sysfs;
+
 	if (bdi->extcon) {
 		INIT_DELAYED_WORK(&bdi->extcon_work, bq24190_extcon_work);
 		bdi->extcon_nb.notifier_call = bq24190_extcon_event;
@@ -1704,7 +2009,7 @@ static __maybe_unused int bq24190_pm_resume(struct device *dev)
 	}
 
 	bq24190_register_reset(bdi);
-	bq24190_set_mode_host(bdi);
+	bq24190_set_config(bdi);
 	bq24190_read(bdi, BQ24190_REG_SS, &bdi->ss_reg);
 
 	if (error >= 0) {
@@ -1736,6 +2041,7 @@ MODULE_DEVICE_TABLE(i2c, bq24190_i2c_ids);
 #ifdef CONFIG_OF
 static const struct of_device_id bq24190_of_match[] = {
 	{ .compatible = "ti,bq24190", },
+	{ .compatible = "ti,bq24192i", },
 	{ },
 };
 MODULE_DEVICE_TABLE(of, bq24190_of_match);
diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c
index ed44439d0112..51f0961ecf3e 100644
--- a/drivers/power/supply/bq27xxx_battery.c
+++ b/drivers/power/supply/bq27xxx_battery.c
@@ -58,8 +58,6 @@
 
 #include <linux/power/bq27xxx_battery.h>
 
-#define DRIVER_VERSION		"1.2.0"
-
 #define BQ27XXX_MANUFACTURER	"Texas Instruments"
 
 /* BQ27XXX Flags */
@@ -132,8 +130,8 @@ enum bq27xxx_reg_index {
 	[BQ27XXX_DM_CKSUM] = 0x60
 
 /* Register mappings */
-static u8 bq27xxx_regs[][BQ27XXX_REG_MAX] = {
-	[BQ27000] = {
+static u8
+	bq27000_regs[BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_CTRL] = 0x00,
 		[BQ27XXX_REG_TEMP] = 0x06,
 		[BQ27XXX_REG_INT_TEMP] = INVALID_REG_ADDR,
@@ -157,7 +155,7 @@ static u8 bq27xxx_regs[][BQ27XXX_REG_MAX] = {
 		[BQ27XXX_DM_DATA] = INVALID_REG_ADDR,
 		[BQ27XXX_DM_CKSUM] = INVALID_REG_ADDR,
 	},
-	[BQ27010] = {
+	bq27010_regs[BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_CTRL] = 0x00,
 		[BQ27XXX_REG_TEMP] = 0x06,
 		[BQ27XXX_REG_INT_TEMP] = INVALID_REG_ADDR,
@@ -181,7 +179,7 @@ static u8 bq27xxx_regs[][BQ27XXX_REG_MAX] = {
 		[BQ27XXX_DM_DATA] = INVALID_REG_ADDR,
 		[BQ27XXX_DM_CKSUM] = INVALID_REG_ADDR,
 	},
-	[BQ2750X] = {
+	bq2750x_regs[BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_CTRL] = 0x00,
 		[BQ27XXX_REG_TEMP] = 0x06,
 		[BQ27XXX_REG_INT_TEMP] = 0x28,
@@ -201,47 +199,9 @@ static u8 bq27xxx_regs[][BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_AP] = INVALID_REG_ADDR,
 		BQ27XXX_DM_REG_ROWS,
 	},
-	[BQ2751X] = {
-		[BQ27XXX_REG_CTRL] = 0x00,
-		[BQ27XXX_REG_TEMP] = 0x06,
-		[BQ27XXX_REG_INT_TEMP] = 0x28,
-		[BQ27XXX_REG_VOLT] = 0x08,
-		[BQ27XXX_REG_AI] = 0x14,
-		[BQ27XXX_REG_FLAGS] = 0x0a,
-		[BQ27XXX_REG_TTE] = 0x16,
-		[BQ27XXX_REG_TTF] = INVALID_REG_ADDR,
-		[BQ27XXX_REG_TTES] = 0x1a,
-		[BQ27XXX_REG_TTECP] = INVALID_REG_ADDR,
-		[BQ27XXX_REG_NAC] = 0x0c,
-		[BQ27XXX_REG_FCC] = 0x12,
-		[BQ27XXX_REG_CYCT] = 0x1e,
-		[BQ27XXX_REG_AE] = INVALID_REG_ADDR,
-		[BQ27XXX_REG_SOC] = 0x20,
-		[BQ27XXX_REG_DCAP] = 0x2e,
-		[BQ27XXX_REG_AP] = INVALID_REG_ADDR,
-		BQ27XXX_DM_REG_ROWS,
-	},
-	[BQ27500] = {
-		[BQ27XXX_REG_CTRL] = 0x00,
-		[BQ27XXX_REG_TEMP] = 0x06,
-		[BQ27XXX_REG_INT_TEMP] = INVALID_REG_ADDR,
-		[BQ27XXX_REG_VOLT] = 0x08,
-		[BQ27XXX_REG_AI] = 0x14,
-		[BQ27XXX_REG_FLAGS] = 0x0a,
-		[BQ27XXX_REG_TTE] = 0x16,
-		[BQ27XXX_REG_TTF] = 0x18,
-		[BQ27XXX_REG_TTES] = 0x1c,
-		[BQ27XXX_REG_TTECP] = 0x26,
-		[BQ27XXX_REG_NAC] = 0x0c,
-		[BQ27XXX_REG_FCC] = 0x12,
-		[BQ27XXX_REG_CYCT] = 0x2a,
-		[BQ27XXX_REG_AE] = 0x22,
-		[BQ27XXX_REG_SOC] = 0x2c,
-		[BQ27XXX_REG_DCAP] = 0x3c,
-		[BQ27XXX_REG_AP] = 0x24,
-		BQ27XXX_DM_REG_ROWS,
-	},
-	[BQ27510G1] = {
+#define bq2751x_regs bq27510g3_regs
+#define bq2752x_regs bq27510g3_regs
+	bq27500_regs[BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_CTRL] = 0x00,
 		[BQ27XXX_REG_TEMP] = 0x06,
 		[BQ27XXX_REG_INT_TEMP] = INVALID_REG_ADDR,
@@ -261,27 +221,9 @@ static u8 bq27xxx_regs[][BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_AP] = 0x24,
 		BQ27XXX_DM_REG_ROWS,
 	},
-	[BQ27510G2] = {
-		[BQ27XXX_REG_CTRL] = 0x00,
-		[BQ27XXX_REG_TEMP] = 0x06,
-		[BQ27XXX_REG_INT_TEMP] = INVALID_REG_ADDR,
-		[BQ27XXX_REG_VOLT] = 0x08,
-		[BQ27XXX_REG_AI] = 0x14,
-		[BQ27XXX_REG_FLAGS] = 0x0a,
-		[BQ27XXX_REG_TTE] = 0x16,
-		[BQ27XXX_REG_TTF] = 0x18,
-		[BQ27XXX_REG_TTES] = 0x1c,
-		[BQ27XXX_REG_TTECP] = 0x26,
-		[BQ27XXX_REG_NAC] = 0x0c,
-		[BQ27XXX_REG_FCC] = 0x12,
-		[BQ27XXX_REG_CYCT] = 0x2a,
-		[BQ27XXX_REG_AE] = 0x22,
-		[BQ27XXX_REG_SOC] = 0x2c,
-		[BQ27XXX_REG_DCAP] = 0x3c,
-		[BQ27XXX_REG_AP] = 0x24,
-		BQ27XXX_DM_REG_ROWS,
-	},
-	[BQ27510G3] = {
+#define bq27510g1_regs bq27500_regs
+#define bq27510g2_regs bq27500_regs
+	bq27510g3_regs[BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_CTRL] = 0x00,
 		[BQ27XXX_REG_TEMP] = 0x06,
 		[BQ27XXX_REG_INT_TEMP] = 0x28,
@@ -301,7 +243,7 @@ static u8 bq27xxx_regs[][BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_AP] = INVALID_REG_ADDR,
 		BQ27XXX_DM_REG_ROWS,
 	},
-	[BQ27520G1] = {
+	bq27520g1_regs[BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_CTRL] = 0x00,
 		[BQ27XXX_REG_TEMP] = 0x06,
 		[BQ27XXX_REG_INT_TEMP] = INVALID_REG_ADDR,
@@ -321,7 +263,7 @@ static u8 bq27xxx_regs[][BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_AP] = 0x24,
 		BQ27XXX_DM_REG_ROWS,
 	},
-	[BQ27520G2] = {
+	bq27520g2_regs[BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_CTRL] = 0x00,
 		[BQ27XXX_REG_TEMP] = 0x06,
 		[BQ27XXX_REG_INT_TEMP] = 0x36,
@@ -341,7 +283,7 @@ static u8 bq27xxx_regs[][BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_AP] = 0x24,
 		BQ27XXX_DM_REG_ROWS,
 	},
-	[BQ27520G3] = {
+	bq27520g3_regs[BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_CTRL] = 0x00,
 		[BQ27XXX_REG_TEMP] = 0x06,
 		[BQ27XXX_REG_INT_TEMP] = 0x36,
@@ -361,7 +303,7 @@ static u8 bq27xxx_regs[][BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_AP] = 0x24,
 		BQ27XXX_DM_REG_ROWS,
 	},
-	[BQ27520G4] = {
+	bq27520g4_regs[BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_CTRL] = 0x00,
 		[BQ27XXX_REG_TEMP] = 0x06,
 		[BQ27XXX_REG_INT_TEMP] = 0x28,
@@ -381,7 +323,7 @@ static u8 bq27xxx_regs[][BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_AP] = INVALID_REG_ADDR,
 		BQ27XXX_DM_REG_ROWS,
 	},
-	[BQ27530] = {
+	bq27530_regs[BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_CTRL] = 0x00,
 		[BQ27XXX_REG_TEMP] = 0x06,
 		[BQ27XXX_REG_INT_TEMP] = 0x32,
@@ -401,7 +343,8 @@ static u8 bq27xxx_regs[][BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_AP] = 0x24,
 		BQ27XXX_DM_REG_ROWS,
 	},
-	[BQ27541] = {
+#define bq27531_regs bq27530_regs
+	bq27541_regs[BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_CTRL] = 0x00,
 		[BQ27XXX_REG_TEMP] = 0x06,
 		[BQ27XXX_REG_INT_TEMP] = 0x28,
@@ -421,7 +364,10 @@ static u8 bq27xxx_regs[][BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_AP] = 0x24,
 		BQ27XXX_DM_REG_ROWS,
 	},
-	[BQ27545] = {
+#define bq27542_regs bq27541_regs
+#define bq27546_regs bq27541_regs
+#define bq27742_regs bq27541_regs
+	bq27545_regs[BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_CTRL] = 0x00,
 		[BQ27XXX_REG_TEMP] = 0x06,
 		[BQ27XXX_REG_INT_TEMP] = 0x28,
@@ -441,7 +387,7 @@ static u8 bq27xxx_regs[][BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_AP] = 0x24,
 		BQ27XXX_DM_REG_ROWS,
 	},
-	[BQ27421] = {
+	bq27421_regs[BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_CTRL] = 0x00,
 		[BQ27XXX_REG_TEMP] = 0x02,
 		[BQ27XXX_REG_INT_TEMP] = 0x1e,
@@ -460,10 +406,12 @@ static u8 bq27xxx_regs[][BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_DCAP] = 0x3c,
 		[BQ27XXX_REG_AP] = 0x18,
 		BQ27XXX_DM_REG_ROWS,
-	},
-};
+	};
+#define bq27425_regs bq27421_regs
+#define bq27441_regs bq27421_regs
+#define bq27621_regs bq27421_regs
 
-static enum power_supply_property bq27000_battery_props[] = {
+static enum power_supply_property bq27000_props[] = {
 	POWER_SUPPLY_PROP_STATUS,
 	POWER_SUPPLY_PROP_PRESENT,
 	POWER_SUPPLY_PROP_VOLTAGE_NOW,
@@ -485,7 +433,7 @@ static enum power_supply_property bq27000_battery_props[] = {
 	POWER_SUPPLY_PROP_MANUFACTURER,
 };
 
-static enum power_supply_property bq27010_battery_props[] = {
+static enum power_supply_property bq27010_props[] = {
 	POWER_SUPPLY_PROP_STATUS,
 	POWER_SUPPLY_PROP_PRESENT,
 	POWER_SUPPLY_PROP_VOLTAGE_NOW,
@@ -505,25 +453,11 @@ static enum power_supply_property bq27010_battery_props[] = {
 	POWER_SUPPLY_PROP_MANUFACTURER,
 };
 
-static enum power_supply_property bq2750x_battery_props[] = {
-	POWER_SUPPLY_PROP_STATUS,
-	POWER_SUPPLY_PROP_PRESENT,
-	POWER_SUPPLY_PROP_VOLTAGE_NOW,
-	POWER_SUPPLY_PROP_CURRENT_NOW,
-	POWER_SUPPLY_PROP_CAPACITY,
-	POWER_SUPPLY_PROP_CAPACITY_LEVEL,
-	POWER_SUPPLY_PROP_TEMP,
-	POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW,
-	POWER_SUPPLY_PROP_TECHNOLOGY,
-	POWER_SUPPLY_PROP_CHARGE_FULL,
-	POWER_SUPPLY_PROP_CHARGE_NOW,
-	POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN,
-	POWER_SUPPLY_PROP_CYCLE_COUNT,
-	POWER_SUPPLY_PROP_HEALTH,
-	POWER_SUPPLY_PROP_MANUFACTURER,
-};
+#define bq2750x_props bq27510g3_props
+#define bq2751x_props bq27510g3_props
+#define bq2752x_props bq27510g3_props
 
-static enum power_supply_property bq2751x_battery_props[] = {
+static enum power_supply_property bq27500_props[] = {
 	POWER_SUPPLY_PROP_STATUS,
 	POWER_SUPPLY_PROP_PRESENT,
 	POWER_SUPPLY_PROP_VOLTAGE_NOW,
@@ -532,16 +466,21 @@ static enum power_supply_property bq2751x_battery_props[] = {
 	POWER_SUPPLY_PROP_CAPACITY_LEVEL,
 	POWER_SUPPLY_PROP_TEMP,
 	POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW,
+	POWER_SUPPLY_PROP_TIME_TO_FULL_NOW,
 	POWER_SUPPLY_PROP_TECHNOLOGY,
 	POWER_SUPPLY_PROP_CHARGE_FULL,
 	POWER_SUPPLY_PROP_CHARGE_NOW,
 	POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN,
 	POWER_SUPPLY_PROP_CYCLE_COUNT,
+	POWER_SUPPLY_PROP_ENERGY_NOW,
+	POWER_SUPPLY_PROP_POWER_AVG,
 	POWER_SUPPLY_PROP_HEALTH,
 	POWER_SUPPLY_PROP_MANUFACTURER,
 };
+#define bq27510g1_props bq27500_props
+#define bq27510g2_props bq27500_props
 
-static enum power_supply_property bq27500_battery_props[] = {
+static enum power_supply_property bq27510g3_props[] = {
 	POWER_SUPPLY_PROP_STATUS,
 	POWER_SUPPLY_PROP_PRESENT,
 	POWER_SUPPLY_PROP_VOLTAGE_NOW,
@@ -550,19 +489,16 @@ static enum power_supply_property bq27500_battery_props[] = {
 	POWER_SUPPLY_PROP_CAPACITY_LEVEL,
 	POWER_SUPPLY_PROP_TEMP,
 	POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW,
-	POWER_SUPPLY_PROP_TIME_TO_FULL_NOW,
 	POWER_SUPPLY_PROP_TECHNOLOGY,
 	POWER_SUPPLY_PROP_CHARGE_FULL,
 	POWER_SUPPLY_PROP_CHARGE_NOW,
 	POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN,
 	POWER_SUPPLY_PROP_CYCLE_COUNT,
-	POWER_SUPPLY_PROP_ENERGY_NOW,
-	POWER_SUPPLY_PROP_POWER_AVG,
 	POWER_SUPPLY_PROP_HEALTH,
 	POWER_SUPPLY_PROP_MANUFACTURER,
 };
 
-static enum power_supply_property bq27510g1_battery_props[] = {
+static enum power_supply_property bq27520g1_props[] = {
 	POWER_SUPPLY_PROP_STATUS,
 	POWER_SUPPLY_PROP_PRESENT,
 	POWER_SUPPLY_PROP_VOLTAGE_NOW,
@@ -576,14 +512,15 @@ static enum power_supply_property bq27510g1_battery_props[] = {
 	POWER_SUPPLY_PROP_CHARGE_FULL,
 	POWER_SUPPLY_PROP_CHARGE_NOW,
 	POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN,
-	POWER_SUPPLY_PROP_CYCLE_COUNT,
 	POWER_SUPPLY_PROP_ENERGY_NOW,
 	POWER_SUPPLY_PROP_POWER_AVG,
 	POWER_SUPPLY_PROP_HEALTH,
 	POWER_SUPPLY_PROP_MANUFACTURER,
 };
 
-static enum power_supply_property bq27510g2_battery_props[] = {
+#define bq27520g2_props bq27500_props
+
+static enum power_supply_property bq27520g3_props[] = {
 	POWER_SUPPLY_PROP_STATUS,
 	POWER_SUPPLY_PROP_PRESENT,
 	POWER_SUPPLY_PROP_VOLTAGE_NOW,
@@ -592,7 +529,6 @@ static enum power_supply_property bq27510g2_battery_props[] = {
 	POWER_SUPPLY_PROP_CAPACITY_LEVEL,
 	POWER_SUPPLY_PROP_TEMP,
 	POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW,
-	POWER_SUPPLY_PROP_TIME_TO_FULL_NOW,
 	POWER_SUPPLY_PROP_TECHNOLOGY,
 	POWER_SUPPLY_PROP_CHARGE_FULL,
 	POWER_SUPPLY_PROP_CHARGE_NOW,
@@ -604,7 +540,7 @@ static enum power_supply_property bq27510g2_battery_props[] = {
 	POWER_SUPPLY_PROP_MANUFACTURER,
 };
 
-static enum power_supply_property bq27510g3_battery_props[] = {
+static enum power_supply_property bq27520g4_props[] = {
 	POWER_SUPPLY_PROP_STATUS,
 	POWER_SUPPLY_PROP_PRESENT,
 	POWER_SUPPLY_PROP_VOLTAGE_NOW,
@@ -616,13 +552,12 @@ static enum power_supply_property bq27510g3_battery_props[] = {
 	POWER_SUPPLY_PROP_TECHNOLOGY,
 	POWER_SUPPLY_PROP_CHARGE_FULL,
 	POWER_SUPPLY_PROP_CHARGE_NOW,
-	POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN,
 	POWER_SUPPLY_PROP_CYCLE_COUNT,
 	POWER_SUPPLY_PROP_HEALTH,
 	POWER_SUPPLY_PROP_MANUFACTURER,
 };
 
-static enum power_supply_property bq27520g1_battery_props[] = {
+static enum power_supply_property bq27530_props[] = {
 	POWER_SUPPLY_PROP_STATUS,
 	POWER_SUPPLY_PROP_PRESENT,
 	POWER_SUPPLY_PROP_VOLTAGE_NOW,
@@ -631,18 +566,17 @@ static enum power_supply_property bq27520g1_battery_props[] = {
 	POWER_SUPPLY_PROP_CAPACITY_LEVEL,
 	POWER_SUPPLY_PROP_TEMP,
 	POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW,
-	POWER_SUPPLY_PROP_TIME_TO_FULL_NOW,
 	POWER_SUPPLY_PROP_TECHNOLOGY,
 	POWER_SUPPLY_PROP_CHARGE_FULL,
 	POWER_SUPPLY_PROP_CHARGE_NOW,
-	POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN,
-	POWER_SUPPLY_PROP_ENERGY_NOW,
 	POWER_SUPPLY_PROP_POWER_AVG,
 	POWER_SUPPLY_PROP_HEALTH,
+	POWER_SUPPLY_PROP_CYCLE_COUNT,
 	POWER_SUPPLY_PROP_MANUFACTURER,
 };
+#define bq27531_props bq27530_props
 
-static enum power_supply_property bq27520g2_battery_props[] = {
+static enum power_supply_property bq27541_props[] = {
 	POWER_SUPPLY_PROP_STATUS,
 	POWER_SUPPLY_PROP_PRESENT,
 	POWER_SUPPLY_PROP_VOLTAGE_NOW,
@@ -651,19 +585,20 @@ static enum power_supply_property bq27520g2_battery_props[] = {
 	POWER_SUPPLY_PROP_CAPACITY_LEVEL,
 	POWER_SUPPLY_PROP_TEMP,
 	POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW,
-	POWER_SUPPLY_PROP_TIME_TO_FULL_NOW,
 	POWER_SUPPLY_PROP_TECHNOLOGY,
 	POWER_SUPPLY_PROP_CHARGE_FULL,
 	POWER_SUPPLY_PROP_CHARGE_NOW,
 	POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN,
 	POWER_SUPPLY_PROP_CYCLE_COUNT,
-	POWER_SUPPLY_PROP_ENERGY_NOW,
 	POWER_SUPPLY_PROP_POWER_AVG,
 	POWER_SUPPLY_PROP_HEALTH,
 	POWER_SUPPLY_PROP_MANUFACTURER,
 };
+#define bq27542_props bq27541_props
+#define bq27546_props bq27541_props
+#define bq27742_props bq27541_props
 
-static enum power_supply_property bq27520g3_battery_props[] = {
+static enum power_supply_property bq27545_props[] = {
 	POWER_SUPPLY_PROP_STATUS,
 	POWER_SUPPLY_PROP_PRESENT,
 	POWER_SUPPLY_PROP_VOLTAGE_NOW,
@@ -675,15 +610,13 @@ static enum power_supply_property bq27520g3_battery_props[] = {
 	POWER_SUPPLY_PROP_TECHNOLOGY,
 	POWER_SUPPLY_PROP_CHARGE_FULL,
 	POWER_SUPPLY_PROP_CHARGE_NOW,
-	POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN,
+	POWER_SUPPLY_PROP_HEALTH,
 	POWER_SUPPLY_PROP_CYCLE_COUNT,
-	POWER_SUPPLY_PROP_ENERGY_NOW,
 	POWER_SUPPLY_PROP_POWER_AVG,
-	POWER_SUPPLY_PROP_HEALTH,
 	POWER_SUPPLY_PROP_MANUFACTURER,
 };
 
-static enum power_supply_property bq27520g4_battery_props[] = {
+static enum power_supply_property bq27421_props[] = {
 	POWER_SUPPLY_PROP_STATUS,
 	POWER_SUPPLY_PROP_PRESENT,
 	POWER_SUPPLY_PROP_VOLTAGE_NOW,
@@ -691,111 +624,144 @@ static enum power_supply_property bq27520g4_battery_props[] = {
 	POWER_SUPPLY_PROP_CAPACITY,
 	POWER_SUPPLY_PROP_CAPACITY_LEVEL,
 	POWER_SUPPLY_PROP_TEMP,
-	POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW,
 	POWER_SUPPLY_PROP_TECHNOLOGY,
 	POWER_SUPPLY_PROP_CHARGE_FULL,
 	POWER_SUPPLY_PROP_CHARGE_NOW,
-	POWER_SUPPLY_PROP_CYCLE_COUNT,
-	POWER_SUPPLY_PROP_HEALTH,
+	POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN,
 	POWER_SUPPLY_PROP_MANUFACTURER,
 };
+#define bq27425_props bq27421_props
+#define bq27441_props bq27421_props
+#define bq27621_props bq27421_props
 
-static enum power_supply_property bq27530_battery_props[] = {
-	POWER_SUPPLY_PROP_STATUS,
-	POWER_SUPPLY_PROP_PRESENT,
-	POWER_SUPPLY_PROP_VOLTAGE_NOW,
-	POWER_SUPPLY_PROP_CURRENT_NOW,
-	POWER_SUPPLY_PROP_CAPACITY,
-	POWER_SUPPLY_PROP_CAPACITY_LEVEL,
-	POWER_SUPPLY_PROP_TEMP,
-	POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW,
-	POWER_SUPPLY_PROP_TECHNOLOGY,
-	POWER_SUPPLY_PROP_CHARGE_FULL,
-	POWER_SUPPLY_PROP_CHARGE_NOW,
-	POWER_SUPPLY_PROP_POWER_AVG,
-	POWER_SUPPLY_PROP_HEALTH,
-	POWER_SUPPLY_PROP_CYCLE_COUNT,
-	POWER_SUPPLY_PROP_MANUFACTURER,
+struct bq27xxx_dm_reg {
+	u8 subclass_id;
+	u8 offset;
+	u8 bytes;
+	u16 min, max;
 };
 
-static enum power_supply_property bq27541_battery_props[] = {
-	POWER_SUPPLY_PROP_STATUS,
-	POWER_SUPPLY_PROP_PRESENT,
-	POWER_SUPPLY_PROP_VOLTAGE_NOW,
-	POWER_SUPPLY_PROP_CURRENT_NOW,
-	POWER_SUPPLY_PROP_CAPACITY,
-	POWER_SUPPLY_PROP_CAPACITY_LEVEL,
-	POWER_SUPPLY_PROP_TEMP,
-	POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW,
-	POWER_SUPPLY_PROP_TECHNOLOGY,
-	POWER_SUPPLY_PROP_CHARGE_FULL,
-	POWER_SUPPLY_PROP_CHARGE_NOW,
-	POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN,
-	POWER_SUPPLY_PROP_CYCLE_COUNT,
-	POWER_SUPPLY_PROP_POWER_AVG,
-	POWER_SUPPLY_PROP_HEALTH,
-	POWER_SUPPLY_PROP_MANUFACTURER,
+enum bq27xxx_dm_reg_id {
+	BQ27XXX_DM_DESIGN_CAPACITY = 0,
+	BQ27XXX_DM_DESIGN_ENERGY,
+	BQ27XXX_DM_TERMINATE_VOLTAGE,
 };
 
-static enum power_supply_property bq27545_battery_props[] = {
-	POWER_SUPPLY_PROP_STATUS,
-	POWER_SUPPLY_PROP_PRESENT,
-	POWER_SUPPLY_PROP_VOLTAGE_NOW,
-	POWER_SUPPLY_PROP_CURRENT_NOW,
-	POWER_SUPPLY_PROP_CAPACITY,
-	POWER_SUPPLY_PROP_CAPACITY_LEVEL,
-	POWER_SUPPLY_PROP_TEMP,
-	POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW,
-	POWER_SUPPLY_PROP_TECHNOLOGY,
-	POWER_SUPPLY_PROP_CHARGE_FULL,
-	POWER_SUPPLY_PROP_CHARGE_NOW,
-	POWER_SUPPLY_PROP_HEALTH,
-	POWER_SUPPLY_PROP_CYCLE_COUNT,
-	POWER_SUPPLY_PROP_POWER_AVG,
-	POWER_SUPPLY_PROP_MANUFACTURER,
+#define bq27000_dm_regs 0
+#define bq27010_dm_regs 0
+#define bq2750x_dm_regs 0
+#define bq2751x_dm_regs 0
+#define bq2752x_dm_regs 0
+
+#if 0 /* not yet tested */
+static struct bq27xxx_dm_reg bq27500_dm_regs[] = {
+	[BQ27XXX_DM_DESIGN_CAPACITY]   = { 48, 10, 2,    0, 65535 },
+	[BQ27XXX_DM_DESIGN_ENERGY]     = { }, /* missing on chip */
+	[BQ27XXX_DM_TERMINATE_VOLTAGE] = { 80, 48, 2, 1000, 32767 },
 };
+#else
+#define bq27500_dm_regs 0
+#endif
 
-static enum power_supply_property bq27421_battery_props[] = {
-	POWER_SUPPLY_PROP_STATUS,
-	POWER_SUPPLY_PROP_PRESENT,
-	POWER_SUPPLY_PROP_VOLTAGE_NOW,
-	POWER_SUPPLY_PROP_CURRENT_NOW,
-	POWER_SUPPLY_PROP_CAPACITY,
-	POWER_SUPPLY_PROP_CAPACITY_LEVEL,
-	POWER_SUPPLY_PROP_TEMP,
-	POWER_SUPPLY_PROP_TECHNOLOGY,
-	POWER_SUPPLY_PROP_CHARGE_FULL,
-	POWER_SUPPLY_PROP_CHARGE_NOW,
-	POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN,
-	POWER_SUPPLY_PROP_MANUFACTURER,
+/* todo create data memory definitions from datasheets and test on chips */
+#define bq27510g1_dm_regs 0
+#define bq27510g2_dm_regs 0
+#define bq27510g3_dm_regs 0
+#define bq27520g1_dm_regs 0
+#define bq27520g2_dm_regs 0
+#define bq27520g3_dm_regs 0
+#define bq27520g4_dm_regs 0
+#define bq27530_dm_regs 0
+#define bq27531_dm_regs 0
+#define bq27541_dm_regs 0
+#define bq27542_dm_regs 0
+#define bq27546_dm_regs 0
+#define bq27742_dm_regs 0
+
+#if 0 /* not yet tested */
+static struct bq27xxx_dm_reg bq27545_dm_regs[] = {
+	[BQ27XXX_DM_DESIGN_CAPACITY]   = { 48, 23, 2,    0, 32767 },
+	[BQ27XXX_DM_DESIGN_ENERGY]     = { 48, 25, 2,    0, 32767 },
+	[BQ27XXX_DM_TERMINATE_VOLTAGE] = { 80, 67, 2, 2800,  3700 },
 };
+#else
+#define bq27545_dm_regs 0
+#endif
 
-#define BQ27XXX_PROP(_id, _prop)		\
-	[_id] = {				\
-		.props = _prop,			\
-		.size = ARRAY_SIZE(_prop),	\
-	}
+static struct bq27xxx_dm_reg bq27421_dm_regs[] = {
+	[BQ27XXX_DM_DESIGN_CAPACITY]   = { 82, 10, 2,    0,  8000 },
+	[BQ27XXX_DM_DESIGN_ENERGY]     = { 82, 12, 2,    0, 32767 },
+	[BQ27XXX_DM_TERMINATE_VOLTAGE] = { 82, 16, 2, 2500,  3700 },
+};
+
+static struct bq27xxx_dm_reg bq27425_dm_regs[] = {
+	[BQ27XXX_DM_DESIGN_CAPACITY]   = { 82, 12, 2,    0, 32767 },
+	[BQ27XXX_DM_DESIGN_ENERGY]     = { 82, 14, 2,    0, 32767 },
+	[BQ27XXX_DM_TERMINATE_VOLTAGE] = { 82, 18, 2, 2800,  3700 },
+};
+
+#if 0 /* not yet tested */
+#define bq27441_dm_regs bq27421_dm_regs
+#else
+#define bq27441_dm_regs 0
+#endif
+
+#if 0 /* not yet tested */
+static struct bq27xxx_dm_reg bq27621_dm_regs[] = {
+	[BQ27XXX_DM_DESIGN_CAPACITY]   = { 82, 3, 2,    0,  8000 },
+	[BQ27XXX_DM_DESIGN_ENERGY]     = { 82, 5, 2,    0, 32767 },
+	[BQ27XXX_DM_TERMINATE_VOLTAGE] = { 82, 9, 2, 2500,  3700 },
+};
+#else
+#define bq27621_dm_regs 0
+#endif
+
+#define BQ27XXX_O_ZERO	0x00000001
+#define BQ27XXX_O_OTDC	0x00000002
+#define BQ27XXX_O_UTOT  0x00000004
+#define BQ27XXX_O_CFGUP	0x00000008
+#define BQ27XXX_O_RAM	0x00000010
+
+#define BQ27XXX_DATA(ref, key, opt) {		\
+	.opts = (opt),				\
+	.unseal_key = key,			\
+	.regs  = ref##_regs,			\
+	.dm_regs = ref##_dm_regs,		\
+	.props = ref##_props,			\
+	.props_size = ARRAY_SIZE(ref##_props) }
 
 static struct {
+	u32 opts;
+	u32 unseal_key;
+	u8 *regs;
+	struct bq27xxx_dm_reg *dm_regs;
 	enum power_supply_property *props;
-	size_t size;
-} bq27xxx_battery_props[] = {
-	BQ27XXX_PROP(BQ27000, bq27000_battery_props),
-	BQ27XXX_PROP(BQ27010, bq27010_battery_props),
-	BQ27XXX_PROP(BQ2750X, bq2750x_battery_props),
-	BQ27XXX_PROP(BQ2751X, bq2751x_battery_props),
-	BQ27XXX_PROP(BQ27500, bq27500_battery_props),
-	BQ27XXX_PROP(BQ27510G1, bq27510g1_battery_props),
-	BQ27XXX_PROP(BQ27510G2, bq27510g2_battery_props),
-	BQ27XXX_PROP(BQ27510G3, bq27510g3_battery_props),
-	BQ27XXX_PROP(BQ27520G1, bq27520g1_battery_props),
-	BQ27XXX_PROP(BQ27520G2, bq27520g2_battery_props),
-	BQ27XXX_PROP(BQ27520G3, bq27520g3_battery_props),
-	BQ27XXX_PROP(BQ27520G4, bq27520g4_battery_props),
-	BQ27XXX_PROP(BQ27530, bq27530_battery_props),
-	BQ27XXX_PROP(BQ27541, bq27541_battery_props),
-	BQ27XXX_PROP(BQ27545, bq27545_battery_props),
-	BQ27XXX_PROP(BQ27421, bq27421_battery_props),
+	size_t props_size;
+} bq27xxx_chip_data[] = {
+	[BQ27000]   = BQ27XXX_DATA(bq27000,   0         , BQ27XXX_O_ZERO),
+	[BQ27010]   = BQ27XXX_DATA(bq27010,   0         , BQ27XXX_O_ZERO),
+	[BQ2750X]   = BQ27XXX_DATA(bq2750x,   0         , BQ27XXX_O_OTDC),
+	[BQ2751X]   = BQ27XXX_DATA(bq2751x,   0         , BQ27XXX_O_OTDC),
+	[BQ2752X]   = BQ27XXX_DATA(bq2752x,   0         , BQ27XXX_O_OTDC),
+	[BQ27500]   = BQ27XXX_DATA(bq27500,   0x04143672, BQ27XXX_O_OTDC),
+	[BQ27510G1] = BQ27XXX_DATA(bq27510g1, 0         , BQ27XXX_O_OTDC),
+	[BQ27510G2] = BQ27XXX_DATA(bq27510g2, 0         , BQ27XXX_O_OTDC),
+	[BQ27510G3] = BQ27XXX_DATA(bq27510g3, 0         , BQ27XXX_O_OTDC),
+	[BQ27520G1] = BQ27XXX_DATA(bq27520g1, 0         , BQ27XXX_O_OTDC),
+	[BQ27520G2] = BQ27XXX_DATA(bq27520g2, 0         , BQ27XXX_O_OTDC),
+	[BQ27520G3] = BQ27XXX_DATA(bq27520g3, 0         , BQ27XXX_O_OTDC),
+	[BQ27520G4] = BQ27XXX_DATA(bq27520g4, 0         , BQ27XXX_O_OTDC),
+	[BQ27530]   = BQ27XXX_DATA(bq27530,   0         , BQ27XXX_O_UTOT),
+	[BQ27531]   = BQ27XXX_DATA(bq27531,   0         , BQ27XXX_O_UTOT),
+	[BQ27541]   = BQ27XXX_DATA(bq27541,   0         , BQ27XXX_O_OTDC),
+	[BQ27542]   = BQ27XXX_DATA(bq27542,   0         , BQ27XXX_O_OTDC),
+	[BQ27546]   = BQ27XXX_DATA(bq27546,   0         , BQ27XXX_O_OTDC),
+	[BQ27742]   = BQ27XXX_DATA(bq27742,   0         , BQ27XXX_O_OTDC),
+	[BQ27545]   = BQ27XXX_DATA(bq27545,   0x04143672, BQ27XXX_O_OTDC),
+	[BQ27421]   = BQ27XXX_DATA(bq27421,   0x80008000, BQ27XXX_O_UTOT | BQ27XXX_O_CFGUP | BQ27XXX_O_RAM),
+	[BQ27425]   = BQ27XXX_DATA(bq27425,   0x04143672, BQ27XXX_O_UTOT | BQ27XXX_O_CFGUP),
+	[BQ27441]   = BQ27XXX_DATA(bq27441,   0x80008000, BQ27XXX_O_UTOT | BQ27XXX_O_CFGUP | BQ27XXX_O_RAM),
+	[BQ27621]   = BQ27XXX_DATA(bq27621,   0x80008000, BQ27XXX_O_UTOT | BQ27XXX_O_CFGUP | BQ27XXX_O_RAM),
 };
 
 static DEFINE_MUTEX(bq27xxx_list_lock);
@@ -805,13 +771,6 @@ static LIST_HEAD(bq27xxx_battery_devices);
 
 #define BQ27XXX_DM_SZ	32
 
-struct bq27xxx_dm_reg {
-	u8 subclass_id;
-	u8 offset;
-	u8 bytes;
-	u16 min, max;
-};
-
 /**
  * struct bq27xxx_dm_buf - chip data memory buffer
  * @class: data memory subclass_id
@@ -844,12 +803,6 @@ static inline u16 *bq27xxx_dm_reg_ptr(struct bq27xxx_dm_buf *buf,
 	return NULL;
 }
 
-enum bq27xxx_dm_reg_id {
-	BQ27XXX_DM_DESIGN_CAPACITY = 0,
-	BQ27XXX_DM_DESIGN_ENERGY,
-	BQ27XXX_DM_TERMINATE_VOLTAGE,
-};
-
 static const char * const bq27xxx_dm_reg_name[] = {
 	[BQ27XXX_DM_DESIGN_CAPACITY] = "design-capacity",
 	[BQ27XXX_DM_DESIGN_ENERGY] = "design-energy",
@@ -1092,9 +1045,9 @@ static void bq27xxx_battery_update_dm_block(struct bq27xxx_device_info *di,
 	}
 
 #ifdef CONFIG_BATTERY_BQ27XXX_DT_UPDATES_NVM
-	if (!di->ram_chip && !bq27xxx_dt_to_nvm) {
+	if (!(di->opts & BQ27XXX_O_RAM) && !bq27xxx_dt_to_nvm) {
 #else
-	if (!di->ram_chip) {
+	if (!(di->opts & BQ27XXX_O_RAM)) {
 #endif
 		/* devicetree and NVM differ; defer to NVM */
 		dev_warn(di->dev, "%s has %u; update to %u disallowed "
@@ -1130,7 +1083,7 @@ static int bq27xxx_battery_cfgupdate_priv(struct bq27xxx_device_info *di, bool a
 			return ret;
 	} while (!!(ret & BQ27XXX_FLAG_CFGUP) != active && --try);
 
-	if (!try) {
+	if (!try && di->chip != BQ27425) { // 425 has a bug
 		dev_err(di->dev, "timed out waiting for cfgupdate flag %d\n", active);
 		return -EINVAL;
 	}
@@ -1162,7 +1115,7 @@ static inline int bq27xxx_battery_soft_reset(struct bq27xxx_device_info *di)
 static int bq27xxx_battery_write_dm_block(struct bq27xxx_device_info *di,
 					  struct bq27xxx_dm_buf *buf)
 {
-	bool cfgup = di->chip == BQ27421; /* assume related chips need cfgupdate */
+	bool cfgup = di->opts & BQ27XXX_O_CFGUP;
 	int ret;
 
 	if (!buf->dirty)
@@ -1261,7 +1214,7 @@ static void bq27xxx_battery_set_config(struct bq27xxx_device_info *di,
 
 	bq27xxx_battery_seal(di);
 
-	if (updated && di->chip != BQ27421) { /* not a cfgupdate chip, so reset */
+	if (updated && !(di->opts & BQ27XXX_O_CFGUP)) {
 		bq27xxx_write(di, BQ27XXX_REG_CTRL, BQ27XXX_RESET, false);
 		BQ27XXX_MSLEEP(300); /* reset time is not documented */
 	}
@@ -1328,7 +1281,7 @@ static int bq27xxx_battery_read_soc(struct bq27xxx_device_info *di)
 {
 	int soc;
 
-	if (di->chip == BQ27000 || di->chip == BQ27010)
+	if (di->opts & BQ27XXX_O_ZERO)
 		soc = bq27xxx_read(di, BQ27XXX_REG_SOC, true);
 	else
 		soc = bq27xxx_read(di, BQ27XXX_REG_SOC, false);
@@ -1354,7 +1307,7 @@ static int bq27xxx_battery_read_charge(struct bq27xxx_device_info *di, u8 reg)
 		return charge;
 	}
 
-	if (di->chip == BQ27000 || di->chip == BQ27010)
+	if (di->opts & BQ27XXX_O_ZERO)
 		charge *= BQ27XXX_CURRENT_CONSTANT / BQ27XXX_RS;
 	else
 		charge *= 1000;
@@ -1370,7 +1323,7 @@ static inline int bq27xxx_battery_read_nac(struct bq27xxx_device_info *di)
 {
 	int flags;
 
-	if (di->chip == BQ27000 || di->chip == BQ27010) {
+	if (di->opts & BQ27XXX_O_ZERO) {
 		flags = bq27xxx_read(di, BQ27XXX_REG_FLAGS, true);
 		if (flags >= 0 && (flags & BQ27000_FLAG_CI))
 			return -ENODATA;
@@ -1396,7 +1349,7 @@ static int bq27xxx_battery_read_dcap(struct bq27xxx_device_info *di)
 {
 	int dcap;
 
-	if (di->chip == BQ27000 || di->chip == BQ27010)
+	if (di->opts & BQ27XXX_O_ZERO)
 		dcap = bq27xxx_read(di, BQ27XXX_REG_DCAP, true);
 	else
 		dcap = bq27xxx_read(di, BQ27XXX_REG_DCAP, false);
@@ -1406,7 +1359,7 @@ static int bq27xxx_battery_read_dcap(struct bq27xxx_device_info *di)
 		return dcap;
 	}
 
-	if (di->chip == BQ27000 || di->chip == BQ27010)
+	if (di->opts & BQ27XXX_O_ZERO)
 		dcap = (dcap << 8) * BQ27XXX_CURRENT_CONSTANT / BQ27XXX_RS;
 	else
 		dcap *= 1000;
@@ -1428,7 +1381,7 @@ static int bq27xxx_battery_read_energy(struct bq27xxx_device_info *di)
 		return ae;
 	}
 
-	if (di->chip == BQ27000 || di->chip == BQ27010)
+	if (di->opts & BQ27XXX_O_ZERO)
 		ae *= BQ27XXX_POWER_CONSTANT / BQ27XXX_RS;
 	else
 		ae *= 1000;
@@ -1450,7 +1403,7 @@ static int bq27xxx_battery_read_temperature(struct bq27xxx_device_info *di)
 		return temp;
 	}
 
-	if (di->chip == BQ27000 || di->chip == BQ27010)
+	if (di->opts & BQ27XXX_O_ZERO)
 		temp = 5 * temp / 2;
 
 	return temp;
@@ -1507,7 +1460,7 @@ static int bq27xxx_battery_read_pwr_avg(struct bq27xxx_device_info *di)
 		return tval;
 	}
 
-	if (di->chip == BQ27000 || di->chip == BQ27010)
+	if (di->opts & BQ27XXX_O_ZERO)
 		return (tval * BQ27XXX_POWER_CONSTANT) / BQ27XXX_RS;
 	else
 		return tval;
@@ -1518,26 +1471,12 @@ static int bq27xxx_battery_read_pwr_avg(struct bq27xxx_device_info *di)
  */
 static bool bq27xxx_battery_overtemp(struct bq27xxx_device_info *di, u16 flags)
 {
-	switch (di->chip) {
-	case BQ2750X:
-	case BQ2751X:
-	case BQ27500:
-	case BQ27510G1:
-	case BQ27510G2:
-	case BQ27510G3:
-	case BQ27520G1:
-	case BQ27520G2:
-	case BQ27520G3:
-	case BQ27520G4:
-	case BQ27541:
-	case BQ27545:
+	if (di->opts & BQ27XXX_O_OTDC)
 		return flags & (BQ27XXX_FLAG_OTC | BQ27XXX_FLAG_OTD);
-	case BQ27530:
-	case BQ27421:
+        if (di->opts & BQ27XXX_O_UTOT)
 		return flags & BQ27XXX_FLAG_OT;
-	default:
-		return false;
-	}
+
+	return false;
 }
 
 /*
@@ -1545,7 +1484,7 @@ static bool bq27xxx_battery_overtemp(struct bq27xxx_device_info *di, u16 flags)
  */
 static bool bq27xxx_battery_undertemp(struct bq27xxx_device_info *di, u16 flags)
 {
-	if (di->chip == BQ27530 || di->chip == BQ27421)
+	if (di->opts & BQ27XXX_O_UTOT)
 		return flags & BQ27XXX_FLAG_UT;
 
 	return false;
@@ -1556,7 +1495,7 @@ static bool bq27xxx_battery_undertemp(struct bq27xxx_device_info *di, u16 flags)
  */
 static bool bq27xxx_battery_dead(struct bq27xxx_device_info *di, u16 flags)
 {
-	if (di->chip == BQ27000 || di->chip == BQ27010)
+	if (di->opts & BQ27XXX_O_ZERO)
 		return flags & (BQ27000_FLAG_EDV1 | BQ27000_FLAG_EDVF);
 	else
 		return flags & (BQ27XXX_FLAG_SOC1 | BQ27XXX_FLAG_SOCF);
@@ -1569,7 +1508,7 @@ static bool bq27xxx_battery_dead(struct bq27xxx_device_info *di, u16 flags)
 static int bq27xxx_battery_read_health(struct bq27xxx_device_info *di)
 {
 	int flags;
-	bool has_singe_flag = di->chip == BQ27000 || di->chip == BQ27010;
+	bool has_singe_flag = di->opts & BQ27XXX_O_ZERO;
 
 	flags = bq27xxx_read(di, BQ27XXX_REG_FLAGS, has_singe_flag);
 	if (flags < 0) {
@@ -1591,8 +1530,8 @@ static int bq27xxx_battery_read_health(struct bq27xxx_device_info *di)
 void bq27xxx_battery_update(struct bq27xxx_device_info *di)
 {
 	struct bq27xxx_reg_cache cache = {0, };
-	bool has_ci_flag = di->chip == BQ27000 || di->chip == BQ27010;
-	bool has_singe_flag = di->chip == BQ27000 || di->chip == BQ27010;
+	bool has_ci_flag = di->opts & BQ27XXX_O_ZERO;
+	bool has_singe_flag = di->opts & BQ27XXX_O_ZERO;
 
 	cache.flags = bq27xxx_read(di, BQ27XXX_REG_FLAGS, has_singe_flag);
 	if ((cache.flags & 0xff) == 0xff)
@@ -1670,7 +1609,7 @@ static int bq27xxx_battery_current(struct bq27xxx_device_info *di,
 		return curr;
 	}
 
-	if (di->chip == BQ27000 || di->chip == BQ27010) {
+	if (di->opts & BQ27XXX_O_ZERO) {
 		flags = bq27xxx_read(di, BQ27XXX_REG_FLAGS, true);
 		if (flags & BQ27000_FLAG_CHGS) {
 			dev_dbg(di->dev, "negative current!\n");
@@ -1691,7 +1630,7 @@ static int bq27xxx_battery_status(struct bq27xxx_device_info *di,
 {
 	int status;
 
-	if (di->chip == BQ27000 || di->chip == BQ27010) {
+	if (di->opts & BQ27XXX_O_ZERO) {
 		if (di->cache.flags & BQ27000_FLAG_FC)
 			status = POWER_SUPPLY_STATUS_FULL;
 		else if (di->cache.flags & BQ27000_FLAG_CHGS)
@@ -1719,7 +1658,7 @@ static int bq27xxx_battery_capacity_level(struct bq27xxx_device_info *di,
 {
 	int level;
 
-	if (di->chip == BQ27000 || di->chip == BQ27010) {
+	if (di->opts & BQ27XXX_O_ZERO) {
 		if (di->cache.flags & BQ27000_FLAG_FC)
 			level = POWER_SUPPLY_CAPACITY_LEVEL_FULL;
 		else if (di->cache.flags & BQ27000_FLAG_EDV1)
@@ -1884,7 +1823,11 @@ int bq27xxx_battery_setup(struct bq27xxx_device_info *di)
 
 	INIT_DELAYED_WORK(&di->work, bq27xxx_battery_poll);
 	mutex_init(&di->lock);
-	di->regs = bq27xxx_regs[di->chip];
+
+	di->regs       = bq27xxx_chip_data[di->chip].regs;
+	di->unseal_key = bq27xxx_chip_data[di->chip].unseal_key;
+	di->dm_regs    = bq27xxx_chip_data[di->chip].dm_regs;
+	di->opts       = bq27xxx_chip_data[di->chip].opts;
 
 	psy_desc = devm_kzalloc(di->dev, sizeof(*psy_desc), GFP_KERNEL);
 	if (!psy_desc)
@@ -1892,8 +1835,8 @@ int bq27xxx_battery_setup(struct bq27xxx_device_info *di)
 
 	psy_desc->name = di->name;
 	psy_desc->type = POWER_SUPPLY_TYPE_BATTERY;
-	psy_desc->properties = bq27xxx_battery_props[di->chip].props;
-	psy_desc->num_properties = bq27xxx_battery_props[di->chip].size;
+	psy_desc->properties = bq27xxx_chip_data[di->chip].props;
+	psy_desc->num_properties = bq27xxx_chip_data[di->chip].props_size;
 	psy_desc->get_property = bq27xxx_battery_get_property;
 	psy_desc->external_power_changed = bq27xxx_external_power_changed;
 
@@ -1903,8 +1846,6 @@ int bq27xxx_battery_setup(struct bq27xxx_device_info *di)
 		return PTR_ERR(di->bat);
 	}
 
-	dev_info(di->dev, "support ver. %s enabled\n", DRIVER_VERSION);
-
 	bq27xxx_battery_settings(di);
 	bq27xxx_battery_update(di);
 
@@ -1938,110 +1879,6 @@ void bq27xxx_battery_teardown(struct bq27xxx_device_info *di)
 }
 EXPORT_SYMBOL_GPL(bq27xxx_battery_teardown);
 
-static int bq27xxx_battery_platform_read(struct bq27xxx_device_info *di, u8 reg,
-					 bool single)
-{
-	struct device *dev = di->dev;
-	struct bq27xxx_platform_data *pdata = dev->platform_data;
-	unsigned int timeout = 3;
-	int upper, lower;
-	int temp;
-
-	if (!single) {
-		/* Make sure the value has not changed in between reading the
-		 * lower and the upper part */
-		upper = pdata->read(dev, reg + 1);
-		do {
-			temp = upper;
-			if (upper < 0)
-				return upper;
-
-			lower = pdata->read(dev, reg);
-			if (lower < 0)
-				return lower;
-
-			upper = pdata->read(dev, reg + 1);
-		} while (temp != upper && --timeout);
-
-		if (timeout == 0)
-			return -EIO;
-
-		return (upper << 8) | lower;
-	}
-
-	return pdata->read(dev, reg);
-}
-
-static int bq27xxx_battery_platform_probe(struct platform_device *pdev)
-{
-	struct bq27xxx_device_info *di;
-	struct bq27xxx_platform_data *pdata = pdev->dev.platform_data;
-
-	if (!pdata) {
-		dev_err(&pdev->dev, "no platform_data supplied\n");
-		return -EINVAL;
-	}
-
-	if (!pdata->read) {
-		dev_err(&pdev->dev, "no hdq read callback supplied\n");
-		return -EINVAL;
-	}
-
-	if (!pdata->chip) {
-		dev_err(&pdev->dev, "no device supplied\n");
-		return -EINVAL;
-	}
-
-	di = devm_kzalloc(&pdev->dev, sizeof(*di), GFP_KERNEL);
-	if (!di)
-		return -ENOMEM;
-
-	platform_set_drvdata(pdev, di);
-
-	di->dev = &pdev->dev;
-	di->chip = pdata->chip;
-	di->name = pdata->name ?: dev_name(&pdev->dev);
-	di->bus.read = bq27xxx_battery_platform_read;
-
-	return bq27xxx_battery_setup(di);
-}
-
-static int bq27xxx_battery_platform_remove(struct platform_device *pdev)
-{
-	struct bq27xxx_device_info *di = platform_get_drvdata(pdev);
-
-	bq27xxx_battery_teardown(di);
-
-	return 0;
-}
-
-static const struct platform_device_id bq27xxx_battery_platform_id_table[] = {
-	{ "bq27000-battery", },
-	{ /* sentinel */ }
-};
-MODULE_DEVICE_TABLE(platform, bq27xxx_battery_platform_id_table);
-
-#ifdef CONFIG_OF
-static const struct of_device_id bq27xxx_battery_platform_of_match_table[] = {
-	{ .compatible = "ti,bq27000" },
-	{},
-};
-MODULE_DEVICE_TABLE(of, bq27xxx_battery_platform_of_match_table);
-#endif
-
-static struct platform_driver bq27xxx_battery_platform_driver = {
-	.probe	= bq27xxx_battery_platform_probe,
-	.remove = bq27xxx_battery_platform_remove,
-	.driver = {
-		.name = "bq27000-battery",
-		.of_match_table = of_match_ptr(bq27xxx_battery_platform_of_match_table),
-	},
-	.id_table = bq27xxx_battery_platform_id_table,
-};
-module_platform_driver(bq27xxx_battery_platform_driver);
-
-MODULE_ALIAS("platform:bq27000-battery");
-
 MODULE_AUTHOR("Rodolfo Giometti <giometti@linux.it>");
 MODULE_DESCRIPTION("BQ27xxx battery monitor driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/power/supply/bq27xxx_battery_hdq.c b/drivers/power/supply/bq27xxx_battery_hdq.c
new file mode 100644
index 000000000000..9aff896c9802
--- /dev/null
+++ b/drivers/power/supply/bq27xxx_battery_hdq.c
@@ -0,0 +1,135 @@
+/*
+ * BQ27xxx battery monitor HDQ/1-wire driver
+ *
+ * Copyright (C) 2007-2017 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/types.h>
+#include <linux/platform_device.h>
+#include <linux/mutex.h>
+#include <linux/power/bq27xxx_battery.h>
+
+#include <linux/w1.h>
+
+#define W1_FAMILY_BQ27000	0x01
+
+#define HDQ_CMD_READ	(0 << 7)
+#define HDQ_CMD_WRITE	(1 << 7)
+
+static int F_ID;
+module_param(F_ID, int, S_IRUSR);
+MODULE_PARM_DESC(F_ID, "1-wire slave FID for BQ27xxx device");
+
+static int w1_bq27000_read(struct w1_slave *sl, unsigned int reg)
+{
+	u8 val;
+
+	mutex_lock(&sl->master->bus_mutex);
+	w1_write_8(sl->master, HDQ_CMD_READ | reg);
+	val = w1_read_8(sl->master);
+	mutex_unlock(&sl->master->bus_mutex);
+
+	return val;
+}
+
+static int bq27xxx_battery_hdq_read(struct bq27xxx_device_info *di, u8 reg,
+				    bool single)
+{
+	struct w1_slave *sl = dev_to_w1_slave(di->dev);
+	unsigned int timeout = 3;
+	int upper, lower;
+	int temp;
+
+	if (!single) {
+		/*
+		 * Make sure the value has not changed in between reading the
+		 * lower and the upper part
+		 */
+		upper = w1_bq27000_read(sl, reg + 1);
+		do {
+			temp = upper;
+			if (upper < 0)
+				return upper;
+
+			lower = w1_bq27000_read(sl, reg);
+			if (lower < 0)
+				return lower;
+
+			upper = w1_bq27000_read(sl, reg + 1);
+		} while (temp != upper && --timeout);
+
+		if (timeout == 0)
+			return -EIO;
+
+		return (upper << 8) | lower;
+	}
+
+	return w1_bq27000_read(sl, reg);
+}
+
+static int bq27xxx_battery_hdq_add_slave(struct w1_slave *sl)
+{
+	struct bq27xxx_device_info *di;
+
+	di = devm_kzalloc(&sl->dev, sizeof(*di), GFP_KERNEL);
+	if (!di)
+		return -ENOMEM;
+
+	dev_set_drvdata(&sl->dev, di);
+
+	di->dev = &sl->dev;
+	di->chip = BQ27000;
+	di->name = "bq27000-battery";
+	di->bus.read = bq27xxx_battery_hdq_read;
+
+	return bq27xxx_battery_setup(di);
+}
+
+static void bq27xxx_battery_hdq_remove_slave(struct w1_slave *sl)
+{
+	struct bq27xxx_device_info *di = dev_get_drvdata(&sl->dev);
+
+	bq27xxx_battery_teardown(di);
+}
+
+static struct w1_family_ops bq27xxx_battery_hdq_fops = {
+	.add_slave	= bq27xxx_battery_hdq_add_slave,
+	.remove_slave	= bq27xxx_battery_hdq_remove_slave,
+};
+
+static struct w1_family bq27xxx_battery_hdq_family = {
+	.fid = W1_FAMILY_BQ27000,
+	.fops = &bq27xxx_battery_hdq_fops,
+};
+
+static int __init bq27xxx_battery_hdq_init(void)
+{
+	if (F_ID)
+		bq27xxx_battery_hdq_family.fid = F_ID;
+
+	return w1_register_family(&bq27xxx_battery_hdq_family);
+}
+module_init(bq27xxx_battery_hdq_init);
+
+static void __exit bq27xxx_battery_hdq_exit(void)
+{
+	w1_unregister_family(&bq27xxx_battery_hdq_family);
+}
+module_exit(bq27xxx_battery_hdq_exit);
+
+MODULE_AUTHOR("Texas Instruments Ltd");
+MODULE_DESCRIPTION("BQ27xxx battery monitor HDQ/1-wire driver");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("w1-family-" __stringify(W1_FAMILY_BQ27000));
diff --git a/drivers/power/supply/bq27xxx_battery_i2c.c b/drivers/power/supply/bq27xxx_battery_i2c.c
index a5972214f074..0b11ed472f33 100644
--- a/drivers/power/supply/bq27xxx_battery_i2c.c
+++ b/drivers/power/supply/bq27xxx_battery_i2c.c
@@ -230,7 +230,7 @@ static const struct i2c_device_id bq27xxx_i2c_id_table[] = {
 	{ "bq27210", BQ27010 },
 	{ "bq27500", BQ2750X },
 	{ "bq27510", BQ2751X },
-	{ "bq27520", BQ2751X },
+	{ "bq27520", BQ2752X },
 	{ "bq27500-1", BQ27500 },
 	{ "bq27510g1", BQ27510G1 },
 	{ "bq27510g2", BQ27510G2 },
@@ -240,16 +240,16 @@ static const struct i2c_device_id bq27xxx_i2c_id_table[] = {
 	{ "bq27520g3", BQ27520G3 },
 	{ "bq27520g4", BQ27520G4 },
 	{ "bq27530", BQ27530 },
-	{ "bq27531", BQ27530 },
+	{ "bq27531", BQ27531 },
 	{ "bq27541", BQ27541 },
-	{ "bq27542", BQ27541 },
-	{ "bq27546", BQ27541 },
-	{ "bq27742", BQ27541 },
+	{ "bq27542", BQ27542 },
+	{ "bq27546", BQ27546 },
+	{ "bq27742", BQ27742 },
 	{ "bq27545", BQ27545 },
 	{ "bq27421", BQ27421 },
-	{ "bq27425", BQ27421 },
-	{ "bq27441", BQ27421 },
-	{ "bq27621", BQ27421 },
+	{ "bq27425", BQ27425 },
+	{ "bq27441", BQ27441 },
+	{ "bq27621", BQ27621 },
 	{},
 };
 MODULE_DEVICE_TABLE(i2c, bq27xxx_i2c_id_table);
diff --git a/drivers/power/supply/charger-manager.c b/drivers/power/supply/charger-manager.c
index adc3761831e1..6502fa7c2106 100644
--- a/drivers/power/supply/charger-manager.c
+++ b/drivers/power/supply/charger-manager.c
@@ -1632,8 +1632,7 @@ static int charger_manager_probe(struct platform_device *pdev)
 		return -ENODEV;
 	}
 
-	cm = devm_kzalloc(&pdev->dev,
-			sizeof(struct charger_manager),	GFP_KERNEL);
+	cm = devm_kzalloc(&pdev->dev, sizeof(*cm), GFP_KERNEL);
 	if (!cm)
 		return -ENOMEM;
 
@@ -1645,12 +1644,14 @@ static int charger_manager_probe(struct platform_device *pdev)
 	/* Initialize alarm timer */
 	if (alarmtimer_get_rtcdev()) {
 		cm_timer = devm_kzalloc(cm->dev, sizeof(*cm_timer), GFP_KERNEL);
+		if (!cm_timer)
+			return -ENOMEM;
 		alarm_init(cm_timer, ALARM_BOOTTIME, cm_timer_func);
 	}
 
 	/*
-	 * The following two do not need to be errors.
-	 * Users may intentionally ignore those two features.
+	 * Some of the following do not need to be errors.
+	 * Users may intentionally ignore those features.
 	 */
 	if (desc->fullbatt_uV == 0) {
 		dev_info(&pdev->dev, "Ignoring full-battery voltage threshold as it is not supplied\n");
diff --git a/drivers/power/supply/ds2780_battery.c b/drivers/power/supply/ds2780_battery.c
index 8edd4aa5f475..e5d81b493c45 100644
--- a/drivers/power/supply/ds2780_battery.c
+++ b/drivers/power/supply/ds2780_battery.c
@@ -663,7 +663,7 @@ static ssize_t ds2780_write_param_eeprom_bin(struct file *filp,
 	return count;
 }
 
-static struct bin_attribute ds2780_param_eeprom_bin_attr = {
+static const struct bin_attribute ds2780_param_eeprom_bin_attr = {
 	.attr = {
 		.name = "param_eeprom",
 		.mode = S_IRUGO | S_IWUSR,
@@ -708,7 +708,7 @@ static ssize_t ds2780_write_user_eeprom_bin(struct file *filp,
 	return count;
 }
 
-static struct bin_attribute ds2780_user_eeprom_bin_attr = {
+static const struct bin_attribute ds2780_user_eeprom_bin_attr = {
 	.attr = {
 		.name = "user_eeprom",
 		.mode = S_IRUGO | S_IWUSR,
diff --git a/drivers/power/supply/ds2781_battery.c b/drivers/power/supply/ds2781_battery.c
index 4400402f9ec5..efe83ef8670c 100644
--- a/drivers/power/supply/ds2781_battery.c
+++ b/drivers/power/supply/ds2781_battery.c
@@ -665,7 +665,7 @@ static ssize_t ds2781_write_param_eeprom_bin(struct file *filp,
 	return count;
 }
 
-static struct bin_attribute ds2781_param_eeprom_bin_attr = {
+static const struct bin_attribute ds2781_param_eeprom_bin_attr = {
 	.attr = {
 		.name = "param_eeprom",
 		.mode = S_IRUGO | S_IWUSR,
@@ -711,7 +711,7 @@ static ssize_t ds2781_write_user_eeprom_bin(struct file *filp,
 	return count;
 }
 
-static struct bin_attribute ds2781_user_eeprom_bin_attr = {
+static const struct bin_attribute ds2781_user_eeprom_bin_attr = {
 	.attr = {
 		.name = "user_eeprom",
 		.mode = S_IRUGO | S_IWUSR,
diff --git a/drivers/power/supply/lp8788-charger.c b/drivers/power/supply/lp8788-charger.c
index 677f7c40b25a..0f3432795f3c 100644
--- a/drivers/power/supply/lp8788-charger.c
+++ b/drivers/power/supply/lp8788-charger.c
@@ -626,7 +626,7 @@ static ssize_t lp8788_show_charger_status(struct device *dev,
 {
 	struct lp8788_charger *pchg = dev_get_drvdata(dev);
 	enum lp8788_charging_state state;
-	char *desc[LP8788_MAX_CHG_STATE] = {
+	static const char * const desc[LP8788_MAX_CHG_STATE] = {
 		[LP8788_OFF] = "CHARGER OFF",
 		[LP8788_WARM_UP] = "WARM UP",
 		[LP8788_LOW_INPUT] = "LOW INPUT STATE",
@@ -650,8 +650,10 @@ static ssize_t lp8788_show_eoc_time(struct device *dev,
 				struct device_attribute *attr, char *buf)
 {
 	struct lp8788_charger *pchg = dev_get_drvdata(dev);
-	char *stime[] = { "400ms", "5min", "10min", "15min",
-			"20min", "25min", "30min", "No timeout" };
+	static const char * const stime[] = {
+		"400ms", "5min", "10min", "15min",
+		"20min", "25min", "30min", "No timeout"
+	};
 	u8 val;
 
 	lp8788_read_byte(pchg->lp, LP8788_CHG_EOC, &val);
@@ -665,9 +667,13 @@ static ssize_t lp8788_show_eoc_level(struct device *dev,
 				struct device_attribute *attr, char *buf)
 {
 	struct lp8788_charger *pchg = dev_get_drvdata(dev);
-	char *abs_level[] = { "25mA", "49mA", "75mA", "98mA" };
-	char *relative_level[] = { "5%", "10%", "15%", "20%" };
-	char *level;
+	static const char * const abs_level[] = {
+			"25mA", "49mA", "75mA", "98mA"
+	};
+	static const char * const relative_level[] = {
+			"5%", "10%", "15%", "20%"
+	};
+	const char *level;
 	u8 val;
 	u8 mode;
 
diff --git a/drivers/power/supply/ltc2941-battery-gauge.c b/drivers/power/supply/ltc2941-battery-gauge.c
index 7efb908f4451..08e4fd9ee607 100644
--- a/drivers/power/supply/ltc2941-battery-gauge.c
+++ b/drivers/power/supply/ltc2941-battery-gauge.c
@@ -1,6 +1,6 @@
 /*
- * I2C client/driver for the Linear Technology LTC2941 and LTC2943
- * Battery Gas Gauge IC
+ * I2C client/driver for the Linear Technology LTC2941, LTC2942, LTC2943
+ * and LTC2944 Battery Gas Gauge IC
  *
  * Copyright (C) 2014 Topic Embedded Systems
  *
@@ -34,35 +34,39 @@ enum ltc294x_reg {
 	LTC294X_REG_CONTROL		= 0x01,
 	LTC294X_REG_ACC_CHARGE_MSB	= 0x02,
 	LTC294X_REG_ACC_CHARGE_LSB	= 0x03,
-	LTC294X_REG_THRESH_HIGH_MSB	= 0x04,
-	LTC294X_REG_THRESH_HIGH_LSB	= 0x05,
-	LTC294X_REG_THRESH_LOW_MSB	= 0x06,
-	LTC294X_REG_THRESH_LOW_LSB	= 0x07,
-	LTC294X_REG_VOLTAGE_MSB	= 0x08,
-	LTC294X_REG_VOLTAGE_LSB	= 0x09,
-	LTC294X_REG_CURRENT_MSB	= 0x0E,
-	LTC294X_REG_CURRENT_LSB	= 0x0F,
-	LTC294X_REG_TEMPERATURE_MSB	= 0x14,
-	LTC294X_REG_TEMPERATURE_LSB	= 0x15,
+	LTC294X_REG_VOLTAGE_MSB		= 0x08,
+	LTC294X_REG_VOLTAGE_LSB		= 0x09,
+	LTC2942_REG_TEMPERATURE_MSB	= 0x0C,
+	LTC2942_REG_TEMPERATURE_LSB	= 0x0D,
+	LTC2943_REG_CURRENT_MSB		= 0x0E,
+	LTC2943_REG_CURRENT_LSB		= 0x0F,
+	LTC2943_REG_TEMPERATURE_MSB	= 0x14,
+	LTC2943_REG_TEMPERATURE_LSB	= 0x15,
 };
 
-#define LTC2943_REG_CONTROL_MODE_MASK (BIT(7) | BIT(6))
-#define LTC2943_REG_CONTROL_MODE_SCAN BIT(7)
+enum ltc294x_id {
+	LTC2941_ID,
+	LTC2942_ID,
+	LTC2943_ID,
+	LTC2944_ID,
+};
+
+#define LTC2941_REG_STATUS_CHIP_ID	BIT(7)
+
+#define LTC2942_REG_CONTROL_MODE_SCAN	(BIT(7) | BIT(6))
+#define LTC2943_REG_CONTROL_MODE_SCAN	BIT(7)
 #define LTC294X_REG_CONTROL_PRESCALER_MASK	(BIT(5) | BIT(4) | BIT(3))
 #define LTC294X_REG_CONTROL_SHUTDOWN_MASK	(BIT(0))
 #define LTC294X_REG_CONTROL_PRESCALER_SET(x) \
 	((x << 3) & LTC294X_REG_CONTROL_PRESCALER_MASK)
 #define LTC294X_REG_CONTROL_ALCC_CONFIG_DISABLED	0
 
-#define LTC2941_NUM_REGS	0x08
-#define LTC2943_NUM_REGS	0x18
-
 struct ltc294x_info {
 	struct i2c_client *client;	/* I2C Client pointer */
 	struct power_supply *supply;	/* Supply pointer */
 	struct power_supply_desc supply_desc;	/* Supply description */
 	struct delayed_work work;	/* Work scheduler */
-	unsigned long num_regs;	/* Number of registers (chip type) */
+	enum ltc294x_id id;		/* Chip type */
 	int charge;	/* Last charge register content */
 	int r_sense;	/* mOhm */
 	int Qlsb;	/* nAh */
@@ -145,9 +149,18 @@ static int ltc294x_reset(const struct ltc294x_info *info, int prescaler_exp)
 
 	control = LTC294X_REG_CONTROL_PRESCALER_SET(prescaler_exp) |
 				LTC294X_REG_CONTROL_ALCC_CONFIG_DISABLED;
-	/* Put the 2943 into "monitor" mode, so it measures every 10 sec */
-	if (info->num_regs == LTC2943_NUM_REGS)
+	/* Put device into "monitor" mode */
+	switch (info->id) {
+	case LTC2942_ID:	/* 2942 measures every 2 sec */
+		control |= LTC2942_REG_CONTROL_MODE_SCAN;
+		break;
+	case LTC2943_ID:
+	case LTC2944_ID:	/* 2943 and 2944 measure every 10 sec */
 		control |= LTC2943_REG_CONTROL_MODE_SCAN;
+		break;
+	default:
+		break;
+	}
 
 	if (value != control) {
 		ret = ltc294x_write_regs(info->client,
@@ -252,7 +265,24 @@ static int ltc294x_get_voltage(const struct ltc294x_info *info, int *val)
 	ret = ltc294x_read_regs(info->client,
 		LTC294X_REG_VOLTAGE_MSB, &datar[0], 2);
 	value = (datar[0] << 8) | datar[1];
-	*val = ((value * 23600) / 0xFFFF) * 1000; /* in uV */
+	switch (info->id) {
+	case LTC2943_ID:
+		value *= 23600 * 2;
+		value /= 0xFFFF;
+		value *= 1000 / 2;
+		break;
+	case LTC2944_ID:
+		value *= 70800 / 5*4;
+		value /= 0xFFFF;
+		value *= 1000 * 5/4;
+		break;
+	default:
+		value *= 6000 * 10;
+		value /= 0xFFFF;
+		value *= 1000 / 10;
+		break;
+	}
+	*val = value;
 	return ret;
 }
 
@@ -263,27 +293,38 @@ static int ltc294x_get_current(const struct ltc294x_info *info, int *val)
 	s32 value;
 
 	ret = ltc294x_read_regs(info->client,
-		LTC294X_REG_CURRENT_MSB, &datar[0], 2);
+		LTC2943_REG_CURRENT_MSB, &datar[0], 2);
 	value = (datar[0] << 8) | datar[1];
 	value -= 0x7FFF;
+	if (info->id == LTC2944_ID)
+		value *= 64000;
+	else
+		value *= 60000;
 	/* Value is in range -32k..+32k, r_sense is usually 10..50 mOhm,
 	 * the formula below keeps everything in s32 range while preserving
 	 * enough digits */
-	*val = 1000 * ((60000 * value) / (info->r_sense * 0x7FFF)); /* in uA */
+	*val = 1000 * (value / (info->r_sense * 0x7FFF)); /* in uA */
 	return ret;
 }
 
 static int ltc294x_get_temperature(const struct ltc294x_info *info, int *val)
 {
+	enum ltc294x_reg reg;
 	int ret;
 	u8 datar[2];
 	u32 value;
 
-	ret = ltc294x_read_regs(info->client,
-		LTC294X_REG_TEMPERATURE_MSB, &datar[0], 2);
-	value = (datar[0] << 8) | datar[1];
-	/* Full-scale is 510 Kelvin, convert to centidegrees  */
-	*val = (((51000 * value) / 0xFFFF) - 27215);
+	if (info->id == LTC2942_ID) {
+		reg = LTC2942_REG_TEMPERATURE_MSB;
+		value = 60000;	/* Full-scale is 600 Kelvin */
+	} else {
+		reg = LTC2943_REG_TEMPERATURE_MSB;
+		value = 51000;	/* Full-scale is 510 Kelvin */
+	}
+	ret = ltc294x_read_regs(info->client, reg, &datar[0], 2);
+	value *= (datar[0] << 8) | datar[1];
+	/* Convert to centidegrees  */
+	*val = value / 0xFFFF - 27215;
 	return ret;
 }
 
@@ -357,8 +398,8 @@ static enum power_supply_property ltc294x_properties[] = {
 	POWER_SUPPLY_PROP_CHARGE_COUNTER,
 	POWER_SUPPLY_PROP_CHARGE_NOW,
 	POWER_SUPPLY_PROP_VOLTAGE_NOW,
-	POWER_SUPPLY_PROP_CURRENT_NOW,
 	POWER_SUPPLY_PROP_TEMP,
+	POWER_SUPPLY_PROP_CURRENT_NOW,
 };
 
 static int ltc294x_i2c_remove(struct i2c_client *client)
@@ -375,10 +416,11 @@ static int ltc294x_i2c_probe(struct i2c_client *client,
 {
 	struct power_supply_config psy_cfg = {};
 	struct ltc294x_info *info;
+	struct device_node *np;
 	int ret;
 	u32 prescaler_exp;
 	s32 r_sense;
-	struct device_node *np;
+	u8 status;
 
 	info = devm_kzalloc(&client->dev, sizeof(*info), GFP_KERNEL);
 	if (info == NULL)
@@ -388,7 +430,7 @@ static int ltc294x_i2c_probe(struct i2c_client *client,
 
 	np = of_node_get(client->dev.of_node);
 
-	info->num_regs = (unsigned long)of_device_get_match_data(&client->dev);
+	info->id = (enum ltc294x_id)of_device_get_match_data(&client->dev);
 	info->supply_desc.name = np->name;
 
 	/* r_sense can be negative, when sense+ is connected to the battery
@@ -409,7 +451,7 @@ static int ltc294x_i2c_probe(struct i2c_client *client,
 		prescaler_exp = LTC2941_MAX_PRESCALER_EXP;
 	}
 
-	if (info->num_regs == LTC2943_NUM_REGS) {
+	if (info->id == LTC2943_ID) {
 		if (prescaler_exp > LTC2943_MAX_PRESCALER_EXP)
 			prescaler_exp = LTC2943_MAX_PRESCALER_EXP;
 		info->Qlsb = ((340 * 50000) / r_sense) /
@@ -421,21 +463,39 @@ static int ltc294x_i2c_probe(struct i2c_client *client,
 				(128 / (1 << prescaler_exp));
 	}
 
+	/* Read status register to check for LTC2942 */
+	if (info->id == LTC2941_ID || info->id == LTC2942_ID) {
+		ret = ltc294x_read_regs(client, LTC294X_REG_STATUS, &status, 1);
+		if (ret < 0) {
+			dev_err(&client->dev,
+				"Could not read status register\n");
+			return ret;
+		}
+		if (status & LTC2941_REG_STATUS_CHIP_ID)
+			info->id = LTC2941_ID;
+		else
+			info->id = LTC2942_ID;
+	}
+
 	info->client = client;
 	info->supply_desc.type = POWER_SUPPLY_TYPE_BATTERY;
 	info->supply_desc.properties = ltc294x_properties;
-	if (info->num_regs >= LTC294X_REG_TEMPERATURE_LSB)
+	switch (info->id) {
+	case LTC2944_ID:
+	case LTC2943_ID:
 		info->supply_desc.num_properties =
 			ARRAY_SIZE(ltc294x_properties);
-	else if (info->num_regs >= LTC294X_REG_CURRENT_LSB)
+		break;
+	case LTC2942_ID:
 		info->supply_desc.num_properties =
 			ARRAY_SIZE(ltc294x_properties) - 1;
-	else if (info->num_regs >= LTC294X_REG_VOLTAGE_LSB)
-		info->supply_desc.num_properties =
-			ARRAY_SIZE(ltc294x_properties) - 2;
-	else
+		break;
+	case LTC2941_ID:
+	default:
 		info->supply_desc.num_properties =
 			ARRAY_SIZE(ltc294x_properties) - 3;
+		break;
+	}
 	info->supply_desc.get_property = ltc294x_get_property;
 	info->supply_desc.set_property = ltc294x_set_property;
 	info->supply_desc.property_is_writeable = ltc294x_property_is_writeable;
@@ -492,8 +552,10 @@ static SIMPLE_DEV_PM_OPS(ltc294x_pm_ops, ltc294x_suspend, ltc294x_resume);
 
 
 static const struct i2c_device_id ltc294x_i2c_id[] = {
-	{"ltc2941", LTC2941_NUM_REGS},
-	{"ltc2943", LTC2943_NUM_REGS},
+	{ "ltc2941", LTC2941_ID, },
+	{ "ltc2942", LTC2942_ID, },
+	{ "ltc2943", LTC2943_ID, },
+	{ "ltc2944", LTC2944_ID, },
 	{ },
 };
 MODULE_DEVICE_TABLE(i2c, ltc294x_i2c_id);
@@ -501,11 +563,19 @@ MODULE_DEVICE_TABLE(i2c, ltc294x_i2c_id);
 static const struct of_device_id ltc294x_i2c_of_match[] = {
 	{
 		.compatible = "lltc,ltc2941",
-		.data = (void *)LTC2941_NUM_REGS
+		.data = (void *)LTC2941_ID,
+	},
+	{
+		.compatible = "lltc,ltc2942",
+		.data = (void *)LTC2942_ID,
 	},
 	{
 		.compatible = "lltc,ltc2943",
-		.data = (void *)LTC2943_NUM_REGS
+		.data = (void *)LTC2943_ID,
+	},
+	{
+		.compatible = "lltc,ltc2944",
+		.data = (void *)LTC2944_ID,
 	},
 	{ },
 };
@@ -525,5 +595,5 @@ module_i2c_driver(ltc294x_driver);
 
 MODULE_AUTHOR("Auryn Verwegen, Topic Embedded Systems");
 MODULE_AUTHOR("Mike Looijmans, Topic Embedded Products");
-MODULE_DESCRIPTION("LTC2941/LTC2943 Battery Gas Gauge IC driver");
+MODULE_DESCRIPTION("LTC2941/LTC2942/LTC2943/LTC2944 Battery Gas Gauge IC driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/power/supply/max17042_battery.c b/drivers/power/supply/max17042_battery.c
index aecaaa2b0586..5b556a13f517 100644
--- a/drivers/power/supply/max17042_battery.c
+++ b/drivers/power/supply/max17042_battery.c
@@ -22,6 +22,7 @@
  * This driver is based on max17040_battery.c
  */
 
+#include <linux/acpi.h>
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/slab.h>
@@ -982,6 +983,8 @@ static int max17042_probe(struct i2c_client *client,
 	struct i2c_adapter *adapter = to_i2c_adapter(client->dev.parent);
 	const struct power_supply_desc *max17042_desc = &max17042_psy_desc;
 	struct power_supply_config psy_cfg = {};
+	const struct acpi_device_id *acpi_id = NULL;
+	struct device *dev = &client->dev;
 	struct max17042_chip *chip;
 	int ret;
 	int i;
@@ -995,7 +998,15 @@ static int max17042_probe(struct i2c_client *client,
 		return -ENOMEM;
 
 	chip->client = client;
-	chip->chip_type = id->driver_data;
+	if (id) {
+		chip->chip_type = id->driver_data;
+	} else {
+		acpi_id = acpi_match_device(dev->driver->acpi_match_table, dev);
+		if (!acpi_id)
+			return -ENODEV;
+
+		chip->chip_type = acpi_id->driver_data;
+	}
 	chip->regmap = devm_regmap_init_i2c(client, &max17042_regmap_config);
 	if (IS_ERR(chip->regmap)) {
 		dev_err(&client->dev, "Failed to initialize regmap\n");
@@ -1039,11 +1050,18 @@ static int max17042_probe(struct i2c_client *client,
 	}
 
 	if (client->irq) {
+		unsigned int flags = IRQF_TRIGGER_FALLING | IRQF_ONESHOT;
+
+		/*
+		 * On ACPI systems the IRQ may be handled by ACPI-event code,
+		 * so we need to share (if the ACPI code is willing to share).
+		 */
+		if (acpi_id)
+			flags |= IRQF_SHARED | IRQF_PROBE_SHARED;
+
 		ret = devm_request_threaded_irq(&client->dev, client->irq,
 						NULL,
-						max17042_thread_handler,
-						IRQF_TRIGGER_FALLING |
-						IRQF_ONESHOT,
+						max17042_thread_handler, flags,
 						chip->battery->desc->name,
 						chip);
 		if (!ret) {
@@ -1053,10 +1071,13 @@ static int max17042_probe(struct i2c_client *client,
 			max17042_set_soc_threshold(chip, 1);
 		} else {
 			client->irq = 0;
-			dev_err(&client->dev, "%s(): cannot get IRQ\n",
-				__func__);
+			if (ret != -EBUSY)
+				dev_err(&client->dev, "Failed to get IRQ\n");
 		}
 	}
+	/* Not able to update the charge threshold when exceeded? -> disable */
+	if (!client->irq)
+		regmap_write(chip->regmap, MAX17042_SALRT_Th, 0xff00);
 
 	regmap_read(chip->regmap, MAX17042_STATUS, &val);
 	if (val & STATUS_POR_BIT) {
@@ -1104,6 +1125,14 @@ static int max17042_resume(struct device *dev)
 static SIMPLE_DEV_PM_OPS(max17042_pm_ops, max17042_suspend,
 			max17042_resume);
 
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id max17042_acpi_match[] = {
+	{ "MAX17047", MAXIM_DEVICE_TYPE_MAX17047 },
+	{ }
+};
+MODULE_DEVICE_TABLE(acpi, max17042_acpi_match);
+#endif
+
 #ifdef CONFIG_OF
 static const struct of_device_id max17042_dt_match[] = {
 	{ .compatible = "maxim,max17042" },
@@ -1125,6 +1154,7 @@ MODULE_DEVICE_TABLE(i2c, max17042_id);
 static struct i2c_driver max17042_i2c_driver = {
 	.driver	= {
 		.name	= "max17042",
+		.acpi_match_table = ACPI_PTR(max17042_acpi_match),
 		.of_match_table = of_match_ptr(max17042_dt_match),
 		.pm	= &max17042_pm_ops,
 	},
diff --git a/drivers/power/supply/max1721x_battery.c b/drivers/power/supply/max1721x_battery.c
new file mode 100644
index 000000000000..9ee601a03d9b
--- /dev/null
+++ b/drivers/power/supply/max1721x_battery.c
@@ -0,0 +1,448 @@
+/*
+ * 1-Wire implementation for Maxim Semiconductor
+ * MAX7211/MAX17215 stanalone fuel gauge chip
+ *
+ * Copyright (C) 2017 Radioavionica Corporation
+ * Author: Alex A. Mihaylov <minimumlaw@rambler.ru>
+ *
+ * Use consistent with the GNU GPL is permitted,
+ * provided that this copyright notice is
+ * preserved in its entirety in all copies and derived works.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/w1.h>
+#include <linux/regmap.h>
+#include <linux/power_supply.h>
+
+#define W1_MAX1721X_FAMILY_ID		0x26
+#define DEF_DEV_NAME_MAX17211		"MAX17211"
+#define DEF_DEV_NAME_MAX17215		"MAX17215"
+#define DEF_DEV_NAME_UNKNOWN		"UNKNOWN"
+#define DEF_MFG_NAME			"MAXIM"
+
+#define PSY_MAX_NAME_LEN	32
+
+/* Number of valid register addresses in W1 mode */
+#define MAX1721X_MAX_REG_NR	0x1EF
+
+/* Factory settings (nonvilatile registers) (W1 specific) */
+#define MAX1721X_REG_NRSENSE	0x1CF	/* RSense in 10^-5 Ohm */
+/* Strings */
+#define MAX1721X_REG_MFG_STR	0x1CC
+#define MAX1721X_REG_MFG_NUMB	3
+#define MAX1721X_REG_DEV_STR	0x1DB
+#define MAX1721X_REG_DEV_NUMB	5
+/* HEX Strings */
+#define MAX1721X_REG_SER_HEX	0x1D8
+
+/* MAX172XX Output Registers for W1 chips */
+#define MAX172XX_REG_STATUS	0x000	/* status reg */
+#define MAX172XX_BAT_PRESENT	(1<<4)	/* battery connected bit */
+#define MAX172XX_REG_DEVNAME	0x021	/* chip config */
+#define MAX172XX_DEV_MASK	0x000F	/* chip type mask */
+#define MAX172X1_DEV		0x0001
+#define MAX172X5_DEV		0x0005
+#define MAX172XX_REG_TEMP	0x008	/* Temperature */
+#define MAX172XX_REG_BATT	0x0DA	/* Battery voltage */
+#define MAX172XX_REG_CURRENT	0x00A	/* Actual current */
+#define MAX172XX_REG_AVGCURRENT	0x00B	/* Average current */
+#define MAX172XX_REG_REPSOC	0x006	/* Percentage of charge */
+#define MAX172XX_REG_DESIGNCAP	0x018	/* Design capacity */
+#define MAX172XX_REG_REPCAP	0x005	/* Average capacity */
+#define MAX172XX_REG_TTE	0x011	/* Time to empty */
+#define MAX172XX_REG_TTF	0x020	/* Time to full */
+
+struct max17211_device_info {
+	char name[PSY_MAX_NAME_LEN];
+	struct power_supply *bat;
+	struct power_supply_desc bat_desc;
+	struct device *w1_dev;
+	struct regmap *regmap;
+	/* battery design format */
+	unsigned int rsense; /* in tenths uOhm */
+	char DeviceName[2 * MAX1721X_REG_DEV_NUMB + 1];
+	char ManufacturerName[2 * MAX1721X_REG_MFG_NUMB + 1];
+	char SerialNumber[13]; /* see get_sn_str() later for comment */
+};
+
+/* Convert regs value to power_supply units */
+
+static inline int max172xx_time_to_ps(unsigned int reg)
+{
+	return reg * 5625 / 1000;	/* in sec. */
+}
+
+static inline int max172xx_percent_to_ps(unsigned int reg)
+{
+	return reg / 256;	/* in percent from 0 to 100 */
+}
+
+static inline int max172xx_voltage_to_ps(unsigned int reg)
+{
+	return reg * 1250;	/* in uV */
+}
+
+static inline int max172xx_capacity_to_ps(unsigned int reg)
+{
+	return reg * 500;	/* in uAh */
+}
+
+/*
+ * Current and temperature is signed values, so unsigned regs
+ * value must be converted to signed type
+ */
+
+static inline int max172xx_temperature_to_ps(unsigned int reg)
+{
+	int val = (int16_t)(reg);
+
+	return val * 10 / 256; /* in tenths of deg. C */
+}
+
+/*
+ * Calculating current registers resolution:
+ *
+ * RSense stored in 10^-5 Ohm, so mesaurment voltage must be
+ * in 10^-11 Volts for get current in uA.
+ * 16 bit current reg fullscale +/-51.2mV is 102400 uV.
+ * So: 102400 / 65535 * 10^5 = 156252
+ */
+static inline int max172xx_current_to_voltage(unsigned int reg)
+{
+	int val = (int16_t)(reg);
+
+	return val * 156252;
+}
+
+
+static inline struct max17211_device_info *
+to_device_info(struct power_supply *psy)
+{
+	return power_supply_get_drvdata(psy);
+}
+
+static int max1721x_battery_get_property(struct power_supply *psy,
+	enum power_supply_property psp,
+	union power_supply_propval *val)
+{
+	struct max17211_device_info *info = to_device_info(psy);
+	unsigned int reg = 0;
+	int ret = 0;
+
+	switch (psp) {
+	case POWER_SUPPLY_PROP_PRESENT:
+		/*
+		 * POWER_SUPPLY_PROP_PRESENT will always readable via
+		 * sysfs interface. Value return 0 if battery not
+		 * present or unaccesable via W1.
+		 */
+		val->intval =
+			regmap_read(info->regmap, MAX172XX_REG_STATUS,
+			&reg) ? 0 : !(reg & MAX172XX_BAT_PRESENT);
+		break;
+	case POWER_SUPPLY_PROP_CAPACITY:
+		ret = regmap_read(info->regmap, MAX172XX_REG_REPSOC, &reg);
+		val->intval = max172xx_percent_to_ps(reg);
+		break;
+	case POWER_SUPPLY_PROP_VOLTAGE_NOW:
+		ret = regmap_read(info->regmap, MAX172XX_REG_BATT, &reg);
+		val->intval = max172xx_voltage_to_ps(reg);
+		break;
+	case POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN:
+		ret = regmap_read(info->regmap, MAX172XX_REG_DESIGNCAP, &reg);
+		val->intval = max172xx_capacity_to_ps(reg);
+		break;
+	case POWER_SUPPLY_PROP_CHARGE_AVG:
+		ret = regmap_read(info->regmap, MAX172XX_REG_REPCAP, &reg);
+		val->intval = max172xx_capacity_to_ps(reg);
+		break;
+	case POWER_SUPPLY_PROP_TIME_TO_EMPTY_AVG:
+		ret = regmap_read(info->regmap, MAX172XX_REG_TTE, &reg);
+		val->intval = max172xx_time_to_ps(reg);
+		break;
+	case POWER_SUPPLY_PROP_TIME_TO_FULL_AVG:
+		ret = regmap_read(info->regmap, MAX172XX_REG_TTF, &reg);
+		val->intval = max172xx_time_to_ps(reg);
+		break;
+	case POWER_SUPPLY_PROP_TEMP:
+		ret = regmap_read(info->regmap, MAX172XX_REG_TEMP, &reg);
+		val->intval = max172xx_temperature_to_ps(reg);
+		break;
+	/* We need signed current, so must cast info->rsense to signed type */
+	case POWER_SUPPLY_PROP_CURRENT_NOW:
+		ret = regmap_read(info->regmap, MAX172XX_REG_CURRENT, &reg);
+		val->intval =
+			max172xx_current_to_voltage(reg) / (int)info->rsense;
+		break;
+	case POWER_SUPPLY_PROP_CURRENT_AVG:
+		ret = regmap_read(info->regmap, MAX172XX_REG_AVGCURRENT, &reg);
+		val->intval =
+			max172xx_current_to_voltage(reg) / (int)info->rsense;
+		break;
+	/*
+	 * Strings already received and inited by probe.
+	 * We do dummy read for check battery still available.
+	 */
+	case POWER_SUPPLY_PROP_MODEL_NAME:
+		ret = regmap_read(info->regmap, MAX1721X_REG_DEV_STR, &reg);
+		val->strval = info->DeviceName;
+		break;
+	case POWER_SUPPLY_PROP_MANUFACTURER:
+		ret = regmap_read(info->regmap, MAX1721X_REG_MFG_STR, &reg);
+		val->strval = info->ManufacturerName;
+		break;
+	case POWER_SUPPLY_PROP_SERIAL_NUMBER:
+		ret = regmap_read(info->regmap, MAX1721X_REG_SER_HEX, &reg);
+		val->strval = info->SerialNumber;
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+static enum power_supply_property max1721x_battery_props[] = {
+	/* int */
+	POWER_SUPPLY_PROP_PRESENT,
+	POWER_SUPPLY_PROP_CAPACITY,
+	POWER_SUPPLY_PROP_VOLTAGE_NOW,
+	POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN,
+	POWER_SUPPLY_PROP_CHARGE_AVG,
+	POWER_SUPPLY_PROP_TIME_TO_EMPTY_AVG,
+	POWER_SUPPLY_PROP_TIME_TO_FULL_AVG,
+	POWER_SUPPLY_PROP_TEMP,
+	POWER_SUPPLY_PROP_CURRENT_NOW,
+	POWER_SUPPLY_PROP_CURRENT_AVG,
+	/* strings */
+	POWER_SUPPLY_PROP_MODEL_NAME,
+	POWER_SUPPLY_PROP_MANUFACTURER,
+	POWER_SUPPLY_PROP_SERIAL_NUMBER,
+};
+
+static int get_string(struct max17211_device_info *info,
+			uint16_t reg, uint8_t nr, char *str)
+{
+	unsigned int val;
+
+	if (!str || !(reg == MAX1721X_REG_MFG_STR ||
+			reg == MAX1721X_REG_DEV_STR))
+		return -EFAULT;
+
+	while (nr--) {
+		if (regmap_read(info->regmap, reg++, &val))
+			return -EFAULT;
+		*str++ = val>>8 & 0x00FF;
+		*str++ = val & 0x00FF;
+	}
+	return 0;
+}
+
+/* Maxim say: Serial number is a hex string up to 12 hex characters */
+static int get_sn_string(struct max17211_device_info *info, char *str)
+{
+	unsigned int val[3];
+
+	if (!str)
+		return -EFAULT;
+
+	if (regmap_read(info->regmap, MAX1721X_REG_SER_HEX, &val[0]))
+		return -EFAULT;
+	if (regmap_read(info->regmap, MAX1721X_REG_SER_HEX + 1, &val[1]))
+		return -EFAULT;
+	if (regmap_read(info->regmap, MAX1721X_REG_SER_HEX + 2, &val[2]))
+		return -EFAULT;
+
+	snprintf(str, 13, "%04X%04X%04X", val[0], val[1], val[2]);
+	return 0;
+}
+
+/*
+ * MAX1721x registers description for w1-regmap
+ */
+static const struct regmap_range max1721x_allow_range[] = {
+	regmap_reg_range(0, 0xDF),	/* volatile data */
+	regmap_reg_range(0x180, 0x1DF),	/* non-volatile memory */
+	regmap_reg_range(0x1E0, 0x1EF),	/* non-volatile history (unused) */
+};
+
+static const struct regmap_range max1721x_deny_range[] = {
+	/* volatile data unused registers */
+	regmap_reg_range(0x24, 0x26),
+	regmap_reg_range(0x30, 0x31),
+	regmap_reg_range(0x33, 0x34),
+	regmap_reg_range(0x37, 0x37),
+	regmap_reg_range(0x3B, 0x3C),
+	regmap_reg_range(0x40, 0x41),
+	regmap_reg_range(0x43, 0x44),
+	regmap_reg_range(0x47, 0x49),
+	regmap_reg_range(0x4B, 0x4C),
+	regmap_reg_range(0x4E, 0xAF),
+	regmap_reg_range(0xB1, 0xB3),
+	regmap_reg_range(0xB5, 0xB7),
+	regmap_reg_range(0xBF, 0xD0),
+	regmap_reg_range(0xDB, 0xDB),
+	/* hole between volatile and non-volatile registers */
+	regmap_reg_range(0xE0, 0x17F),
+};
+
+static const struct regmap_access_table max1721x_regs = {
+	.yes_ranges	= max1721x_allow_range,
+	.n_yes_ranges	= ARRAY_SIZE(max1721x_allow_range),
+	.no_ranges	= max1721x_deny_range,
+	.n_no_ranges	= ARRAY_SIZE(max1721x_deny_range),
+};
+
+/*
+ * Model Gauge M5 Algorithm output register
+ * Volatile data (must not be cached)
+ */
+static const struct regmap_range max1721x_volatile_allow[] = {
+	regmap_reg_range(0, 0xDF),
+};
+
+static const struct regmap_access_table max1721x_volatile_regs = {
+	.yes_ranges	= max1721x_volatile_allow,
+	.n_yes_ranges	= ARRAY_SIZE(max1721x_volatile_allow),
+};
+
+/*
+ * W1-regmap config
+ */
+static const struct regmap_config max1721x_regmap_w1_config = {
+	.reg_bits = 16,
+	.val_bits = 16,
+	.rd_table = &max1721x_regs,
+	.volatile_table = &max1721x_volatile_regs,
+	.max_register = MAX1721X_MAX_REG_NR,
+};
+
+static int devm_w1_max1721x_add_device(struct w1_slave *sl)
+{
+	struct power_supply_config psy_cfg = {};
+	struct max17211_device_info *info;
+
+	info = devm_kzalloc(&sl->dev, sizeof(*info), GFP_KERNEL);
+	if (!info)
+		return -ENOMEM;
+
+	sl->family_data = (void *)info;
+	info->w1_dev = &sl->dev;
+
+	/*
+	 * power_supply class battery name translated from W1 slave device
+	 * unical ID (look like 26-0123456789AB) to "max1721x-0123456789AB\0"
+	 * so, 26 (device family) correcpondent to max1721x devices.
+	 * Device name still unical for any numbers connected devices.
+	 */
+	snprintf(info->name, sizeof(info->name),
+		"max1721x-%012X", (unsigned int)sl->reg_num.id);
+	info->bat_desc.name = info->name;
+
+	/*
+	 * FixMe: battery device name exceed max len for thermal_zone device
+	 * name and translation to thermal_zone must be disabled.
+	 */
+	info->bat_desc.no_thermal = true;
+	info->bat_desc.type = POWER_SUPPLY_TYPE_BATTERY;
+	info->bat_desc.properties = max1721x_battery_props;
+	info->bat_desc.num_properties = ARRAY_SIZE(max1721x_battery_props);
+	info->bat_desc.get_property = max1721x_battery_get_property;
+	psy_cfg.drv_data = info;
+
+	/* regmap init */
+	info->regmap = devm_regmap_init_w1(info->w1_dev,
+					&max1721x_regmap_w1_config);
+	if (IS_ERR(info->regmap)) {
+		int err = PTR_ERR(info->regmap);
+
+		dev_err(info->w1_dev, "Failed to allocate register map: %d\n",
+			err);
+		return err;
+	}
+
+	/* rsense init */
+	info->rsense = 0;
+	if (regmap_read(info->regmap, MAX1721X_REG_NRSENSE, &info->rsense)) {
+		dev_err(info->w1_dev, "Can't read RSense. Hardware error.\n");
+		return -ENODEV;
+	}
+
+	if (!info->rsense) {
+		dev_warn(info->w1_dev, "RSenese not calibrated, set 10 mOhms!\n");
+		info->rsense = 1000; /* in regs in 10^-5 */
+	}
+	dev_info(info->w1_dev, "RSense: %d mOhms.\n", info->rsense / 100);
+
+	if (get_string(info, MAX1721X_REG_MFG_STR,
+			MAX1721X_REG_MFG_NUMB, info->ManufacturerName)) {
+		dev_err(info->w1_dev, "Can't read manufacturer. Hardware error.\n");
+		return -ENODEV;
+	}
+
+	if (!info->ManufacturerName[0])
+		strncpy(info->ManufacturerName, DEF_MFG_NAME,
+			2 * MAX1721X_REG_MFG_NUMB);
+
+	if (get_string(info, MAX1721X_REG_DEV_STR,
+			MAX1721X_REG_DEV_NUMB, info->DeviceName)) {
+		dev_err(info->w1_dev, "Can't read device. Hardware error.\n");
+		return -ENODEV;
+	}
+	if (!info->DeviceName[0]) {
+		unsigned int dev_name;
+
+		if (regmap_read(info->regmap,
+				MAX172XX_REG_DEVNAME, &dev_name)) {
+			dev_err(info->w1_dev, "Can't read device name reg.\n");
+			return -ENODEV;
+		}
+
+		switch (dev_name & MAX172XX_DEV_MASK) {
+		case MAX172X1_DEV:
+			strncpy(info->DeviceName, DEF_DEV_NAME_MAX17211,
+				2 * MAX1721X_REG_DEV_NUMB);
+			break;
+		case MAX172X5_DEV:
+			strncpy(info->DeviceName, DEF_DEV_NAME_MAX17215,
+				2 * MAX1721X_REG_DEV_NUMB);
+			break;
+		default:
+			strncpy(info->DeviceName, DEF_DEV_NAME_UNKNOWN,
+				2 * MAX1721X_REG_DEV_NUMB);
+		}
+	}
+
+	if (get_sn_string(info, info->SerialNumber)) {
+		dev_err(info->w1_dev, "Can't read serial. Hardware error.\n");
+		return -ENODEV;
+	}
+
+	info->bat = devm_power_supply_register(&sl->dev, &info->bat_desc,
+						&psy_cfg);
+	if (IS_ERR(info->bat)) {
+		dev_err(info->w1_dev, "failed to register battery\n");
+		return PTR_ERR(info->bat);
+	}
+
+	return 0;
+}
+
+static struct w1_family_ops w1_max1721x_fops = {
+	.add_slave = devm_w1_max1721x_add_device,
+};
+
+static struct w1_family w1_max1721x_family = {
+	.fid = W1_MAX1721X_FAMILY_ID,
+	.fops = &w1_max1721x_fops,
+};
+
+module_w1_family(w1_max1721x_family);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Alex A. Mihaylov <minimumlaw@rambler.ru>");
+MODULE_DESCRIPTION("Maxim MAX17211/MAX17215 Fuel Gauage IC driver");
+MODULE_ALIAS("w1-family-" __stringify(W1_MAX1721X_FAMILY_ID));
diff --git a/drivers/power/supply/olpc_battery.c b/drivers/power/supply/olpc_battery.c
index 9e29b1321648..3bc2eea7b3b7 100644
--- a/drivers/power/supply/olpc_battery.c
+++ b/drivers/power/supply/olpc_battery.c
@@ -535,7 +535,7 @@ static ssize_t olpc_bat_eeprom_read(struct file *filp, struct kobject *kobj,
 	return count;
 }
 
-static struct bin_attribute olpc_bat_eeprom = {
+static const struct bin_attribute olpc_bat_eeprom = {
 	.attr = {
 		.name = "eeprom",
 		.mode = S_IRUGO,
@@ -559,7 +559,7 @@ static ssize_t olpc_bat_error_read(struct device *dev,
 	return sprintf(buf, "%d\n", ec_byte);
 }
 
-static struct device_attribute olpc_bat_error = {
+static const struct device_attribute olpc_bat_error = {
 	.attr = {
 		.name = "error",
 		.mode = S_IRUGO,
diff --git a/drivers/power/supply/pcf50633-charger.c b/drivers/power/supply/pcf50633-charger.c
index b3c1873ad84d..1ad7ccce6075 100644
--- a/drivers/power/supply/pcf50633-charger.c
+++ b/drivers/power/supply/pcf50633-charger.c
@@ -254,7 +254,7 @@ static struct attribute *pcf50633_mbc_sysfs_entries[] = {
 	NULL,
 };
 
-static struct attribute_group mbc_attr_group = {
+static const struct attribute_group mbc_attr_group = {
 	.name	= NULL,			/* put in device directory */
 	.attrs	= pcf50633_mbc_sysfs_entries,
 };
diff --git a/drivers/power/supply/power_supply_core.c b/drivers/power/supply/power_supply_core.c
index 540d3e0aa011..02c6340ae36f 100644
--- a/drivers/power/supply/power_supply_core.c
+++ b/drivers/power/supply/power_supply_core.c
@@ -259,18 +259,14 @@ static int power_supply_check_supplies(struct power_supply *psy)
 	/* All supplies found, allocate char ** array for filling */
 	psy->supplied_from = devm_kzalloc(&psy->dev, sizeof(psy->supplied_from),
 					  GFP_KERNEL);
-	if (!psy->supplied_from) {
-		dev_err(&psy->dev, "Couldn't allocate memory for supply list\n");
+	if (!psy->supplied_from)
 		return -ENOMEM;
-	}
 
 	*psy->supplied_from = devm_kzalloc(&psy->dev,
 					   sizeof(char *) * (cnt - 1),
 					   GFP_KERNEL);
-	if (!*psy->supplied_from) {
-		dev_err(&psy->dev, "Couldn't allocate memory for supply list\n");
+	if (!*psy->supplied_from)
 		return -ENOMEM;
-	}
 
 	return power_supply_populate_supplied_from(psy);
 }
@@ -314,11 +310,12 @@ static int __power_supply_am_i_supplied(struct device *dev, void *_data)
 	struct power_supply *epsy = dev_get_drvdata(dev);
 	struct psy_am_i_supplied_data *data = _data;
 
-	data->count++;
-	if (__power_supply_is_supplied_by(epsy, data->psy))
+	if (__power_supply_is_supplied_by(epsy, data->psy)) {
+		data->count++;
 		if (!epsy->desc->get_property(epsy, POWER_SUPPLY_PROP_ONLINE,
 					&ret))
 			return ret.intval;
+	}
 
 	return 0;
 }
@@ -374,6 +371,47 @@ int power_supply_is_system_supplied(void)
 }
 EXPORT_SYMBOL_GPL(power_supply_is_system_supplied);
 
+static int __power_supply_get_supplier_max_current(struct device *dev,
+						   void *data)
+{
+	union power_supply_propval ret = {0,};
+	struct power_supply *epsy = dev_get_drvdata(dev);
+	struct power_supply *psy = data;
+
+	if (__power_supply_is_supplied_by(epsy, psy))
+		if (!epsy->desc->get_property(epsy,
+					      POWER_SUPPLY_PROP_CURRENT_MAX,
+					      &ret))
+			return ret.intval;
+
+	return 0;
+}
+
+int power_supply_set_input_current_limit_from_supplier(struct power_supply *psy)
+{
+	union power_supply_propval val = {0,};
+	int curr;
+
+	if (!psy->desc->set_property)
+		return -EINVAL;
+
+	/*
+	 * This function is not intended for use with a supply with multiple
+	 * suppliers, we simply pick the first supply to report a non 0
+	 * max-current.
+	 */
+	curr = class_for_each_device(power_supply_class, NULL, psy,
+				      __power_supply_get_supplier_max_current);
+	if (curr <= 0)
+		return (curr == 0) ? -ENODEV : curr;
+
+	val.intval = curr;
+
+	return psy->desc->set_property(psy,
+				POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, &val);
+}
+EXPORT_SYMBOL_GPL(power_supply_set_input_current_limit_from_supplier);
+
 int power_supply_set_battery_charged(struct power_supply *psy)
 {
 	if (atomic_read(&psy->use_cnt) >= 0 &&
diff --git a/drivers/power/supply/sbs-battery.c b/drivers/power/supply/sbs-battery.c
index f7059459f0fb..b19a73176910 100644
--- a/drivers/power/supply/sbs-battery.c
+++ b/drivers/power/supply/sbs-battery.c
@@ -12,25 +12,21 @@
  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/kernel.h>
+#include <linux/delay.h>
 #include <linux/err.h>
-#include <linux/power_supply.h>
+#include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
-#include <linux/slab.h>
+#include <linux/init.h>
 #include <linux/interrupt.h>
-#include <linux/gpio/consumer.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
 #include <linux/of.h>
-#include <linux/stat.h>
-
 #include <linux/power/sbs-battery.h>
+#include <linux/power_supply.h>
+#include <linux/slab.h>
+#include <linux/stat.h>
 
 enum {
 	REG_MANUFACTURER_DATA,
@@ -60,8 +56,8 @@ enum {
 #define BATTERY_MODE_OFFSET		0x03
 #define BATTERY_MODE_MASK		0x8000
 enum sbs_battery_mode {
-	BATTERY_MODE_AMPS,
-	BATTERY_MODE_WATTS
+	BATTERY_MODE_AMPS = 0,
+	BATTERY_MODE_WATTS = 0x8000
 };
 
 /* manufacturer access defines */
@@ -532,6 +528,8 @@ static enum sbs_battery_mode sbs_set_battery_mode(struct i2c_client *client,
 	if (ret < 0)
 		return ret;
 
+	usleep_range(1000, 2000);
+
 	return original_val & BATTERY_MODE_MASK;
 }
 
diff --git a/drivers/power/supply/twl4030_charger.c b/drivers/power/supply/twl4030_charger.c
index 9dff1b4b85fc..a5915f498eea 100644
--- a/drivers/power/supply/twl4030_charger.c
+++ b/drivers/power/supply/twl4030_charger.c
@@ -18,7 +18,7 @@
 #include <linux/err.h>
 #include <linux/platform_device.h>
 #include <linux/interrupt.h>
-#include <linux/i2c/twl.h>
+#include <linux/mfd/twl.h>
 #include <linux/power_supply.h>
 #include <linux/notifier.h>
 #include <linux/usb/otg.h>
diff --git a/drivers/power/supply/wm831x_power.c b/drivers/power/supply/wm831x_power.c
index 7082301da945..927050d4444d 100644
--- a/drivers/power/supply/wm831x_power.c
+++ b/drivers/power/supply/wm831x_power.c
@@ -13,6 +13,7 @@
 #include <linux/platform_device.h>
 #include <linux/power_supply.h>
 #include <linux/slab.h>
+#include <linux/usb/phy.h>
 
 #include <linux/mfd/wm831x/core.h>
 #include <linux/mfd/wm831x/auxadc.h>
@@ -31,6 +32,8 @@ struct wm831x_power {
 	char usb_name[20];
 	char battery_name[20];
 	bool have_battery;
+	struct usb_phy *usb_phy;
+	struct notifier_block usb_notify;
 };
 
 static int wm831x_power_check_online(struct wm831x *wm831x, int supply,
@@ -125,6 +128,43 @@ static enum power_supply_property wm831x_usb_props[] = {
 	POWER_SUPPLY_PROP_VOLTAGE_NOW,
 };
 
+/* In milliamps */
+static const unsigned int wm831x_usb_limits[] = {
+	0,
+	2,
+	100,
+	500,
+	900,
+	1500,
+	1800,
+	550,
+};
+
+static int wm831x_usb_limit_change(struct notifier_block *nb,
+				   unsigned long limit, void *data)
+{
+	struct wm831x_power *wm831x_power = container_of(nb,
+							 struct wm831x_power,
+							 usb_notify);
+	unsigned int i, best;
+
+	/* Find the highest supported limit */
+	best = 0;
+	for (i = 0; i < ARRAY_SIZE(wm831x_usb_limits); i++) {
+		if (limit >= wm831x_usb_limits[i] &&
+		    wm831x_usb_limits[best] < wm831x_usb_limits[i])
+			best = i;
+	}
+
+	dev_dbg(wm831x_power->wm831x->dev,
+		"Limiting USB current to %umA", wm831x_usb_limits[best]);
+
+	wm831x_set_bits(wm831x_power->wm831x, WM831X_POWER_STATE,
+		        WM831X_USB_ILIM_MASK, best);
+
+	return 0;
+}
+
 /*********************************************************************
  *		Battery properties
  *********************************************************************/
@@ -607,6 +647,33 @@ static int wm831x_power_probe(struct platform_device *pdev)
 		}
 	}
 
+	power->usb_phy = devm_usb_get_phy_by_phandle(&pdev->dev, "phys", 0);
+	ret = PTR_ERR_OR_ZERO(power->usb_phy);
+
+	switch (ret) {
+	case 0:
+		power->usb_notify.notifier_call = wm831x_usb_limit_change;
+		ret = usb_register_notifier(power->usb_phy, &power->usb_notify);
+		if (ret) {
+			dev_err(&pdev->dev, "Failed to register notifier: %d\n",
+				ret);
+			goto err_bat_irq;
+		}
+		break;
+	case -EINVAL:
+	case -ENODEV:
+		/* ignore missing usb-phy, it's optional */
+		power->usb_phy = NULL;
+		ret = 0;
+		break;
+	default:
+		dev_err(&pdev->dev, "Failed to find USB phy: %d\n", ret);
+		/* fall-through */
+	case -EPROBE_DEFER:
+		goto err_bat_irq;
+		break;
+	}
+
 	return ret;
 
 err_bat_irq:
@@ -637,6 +704,11 @@ static int wm831x_power_remove(struct platform_device *pdev)
 	struct wm831x *wm831x = wm831x_power->wm831x;
 	int irq, i;
 
+	if (wm831x_power->usb_phy) {
+		usb_unregister_notifier(wm831x_power->usb_phy,
+					&wm831x_power->usb_notify);
+	}
+
 	for (i = 0; i < ARRAY_SIZE(wm831x_bat_irqs); i++) {
 		irq = wm831x_irq(wm831x, 
 				 platform_get_irq_byname(pdev,
diff --git a/drivers/pps/Kconfig b/drivers/pps/Kconfig
index 4b29a7182d7b..c6008f296605 100644
--- a/drivers/pps/Kconfig
+++ b/drivers/pps/Kconfig
@@ -19,9 +19,10 @@ menuconfig PPS
 	  To compile this driver as a module, choose M here: the module
 	  will be called pps_core.ko.
 
+if PPS
+
 config PPS_DEBUG
 	bool "PPS debugging messages"
-	depends on PPS
 	help
 	  Say Y here if you want the PPS support to produce a bunch of debug
 	  messages to the system log.  Select this if you are having a
@@ -29,7 +30,7 @@ config PPS_DEBUG
 
 config NTP_PPS
 	bool "PPS kernel consumer support"
-	depends on PPS && !NO_HZ_COMMON
+	depends on !NO_HZ_COMMON
 	help
 	  This option adds support for direct in-kernel time
 	  synchronization using an external PPS signal.
@@ -39,3 +40,5 @@ config NTP_PPS
 source drivers/pps/clients/Kconfig
 
 source drivers/pps/generators/Kconfig
+
+endif # PPS
diff --git a/drivers/pps/clients/Kconfig b/drivers/pps/clients/Kconfig
index efec021ce662..7f02a9b1a1fd 100644
--- a/drivers/pps/clients/Kconfig
+++ b/drivers/pps/clients/Kconfig
@@ -3,11 +3,9 @@
 #
 
 comment "PPS clients support"
-	depends on PPS
 
 config PPS_CLIENT_KTIMER
 	tristate "Kernel timer client (Testing client, use for debug)"
-	depends on PPS
 	help
 	  If you say yes here you get support for a PPS debugging client
 	  which uses a kernel timer to generate the PPS signal.
@@ -17,21 +15,20 @@ config PPS_CLIENT_KTIMER
 
 config PPS_CLIENT_LDISC
 	tristate "PPS line discipline"
-	depends on PPS && TTY
+	depends on TTY
 	help
 	  If you say yes here you get support for a PPS source connected
 	  with the CD (Carrier Detect) pin of your serial port.
 
 config PPS_CLIENT_PARPORT
 	tristate "Parallel port PPS client"
-	depends on PPS && PARPORT
+	depends on PARPORT
 	help
 	  If you say yes here you get support for a PPS source connected
 	  with the interrupt pin of your parallel port.
 
 config PPS_CLIENT_GPIO
 	tristate "PPS client using GPIO"
-	depends on PPS
 	help
 	  If you say yes here you get support for a PPS source using
 	  GPIO. To be useful you must also register a platform device
diff --git a/drivers/pps/generators/Kconfig b/drivers/pps/generators/Kconfig
index 86b59378e71f..e4c4f3dc0728 100644
--- a/drivers/pps/generators/Kconfig
+++ b/drivers/pps/generators/Kconfig
@@ -3,11 +3,10 @@
 #
 
 comment "PPS generators support"
-	depends on PPS
 
 config PPS_GENERATOR_PARPORT
 	tristate "Parallel port PPS signal generator"
-	depends on PPS && PARPORT && BROKEN
+	depends on PARPORT && BROKEN
 	help
 	  If you say yes here you get support for a PPS signal generator which
 	  utilizes STROBE pin of a parallel port to send PPS signals. It uses
diff --git a/drivers/ptp/ptp_dte.c b/drivers/ptp/ptp_dte.c
index faf6f7a83713..6edd3b9c7f01 100644
--- a/drivers/ptp/ptp_dte.c
+++ b/drivers/ptp/ptp_dte.c
@@ -221,7 +221,7 @@ static int ptp_dte_enable(struct ptp_clock_info *ptp,
 	return -EOPNOTSUPP;
 }
 
-static struct ptp_clock_info ptp_dte_caps = {
+static const struct ptp_clock_info ptp_dte_caps = {
 	.owner		= THIS_MODULE,
 	.name		= "DTE PTP timer",
 	.max_adj	= 50000000,
diff --git a/drivers/ptp/ptp_ixp46x.c b/drivers/ptp/ptp_ixp46x.c
index 344a3bac210b..1171ffd210b3 100644
--- a/drivers/ptp/ptp_ixp46x.c
+++ b/drivers/ptp/ptp_ixp46x.c
@@ -236,7 +236,7 @@ static int ptp_ixp_enable(struct ptp_clock_info *ptp,
 	return -EOPNOTSUPP;
 }
 
-static struct ptp_clock_info ptp_ixp_caps = {
+static const struct ptp_clock_info ptp_ixp_caps = {
 	.owner		= THIS_MODULE,
 	.name		= "IXP46X timer",
 	.max_adj	= 66666655,
diff --git a/drivers/ptp/ptp_kvm.c b/drivers/ptp/ptp_kvm.c
index bb865695d7a6..2b1b212c219e 100644
--- a/drivers/ptp/ptp_kvm.c
+++ b/drivers/ptp/ptp_kvm.c
@@ -150,7 +150,7 @@ static int ptp_kvm_enable(struct ptp_clock_info *ptp,
 	return -EOPNOTSUPP;
 }
 
-static struct ptp_clock_info ptp_kvm_caps = {
+static const struct ptp_clock_info ptp_kvm_caps = {
 	.owner		= THIS_MODULE,
 	.name		= "KVM virtual PTP",
 	.max_adj	= 0,
diff --git a/drivers/ptp/ptp_pch.c b/drivers/ptp/ptp_pch.c
index 3aa22ae4d94c..b3285175f20f 100644
--- a/drivers/ptp/ptp_pch.c
+++ b/drivers/ptp/ptp_pch.c
@@ -509,7 +509,7 @@ static int ptp_pch_enable(struct ptp_clock_info *ptp,
 	return -EOPNOTSUPP;
 }
 
-static struct ptp_clock_info ptp_pch_caps = {
+static const struct ptp_clock_info ptp_pch_caps = {
 	.owner		= THIS_MODULE,
 	.name		= "PCH timer",
 	.max_adj	= 50000000,
diff --git a/drivers/pwm/Kconfig b/drivers/pwm/Kconfig
index 313c10789ca2..763ee50ea57d 100644
--- a/drivers/pwm/Kconfig
+++ b/drivers/pwm/Kconfig
@@ -300,7 +300,7 @@ config PWM_MEDIATEK
 	  Generic PWM framework driver for Mediatek ARM SoC.
 
 	  To compile this driver as a module, choose M here: the module
-	  will be called pwm-mxs.
+	  will be called pwm-mediatek.
 
 config PWM_MXS
 	tristate "Freescale MXS PWM support"
@@ -417,6 +417,16 @@ config PWM_STM32
 	  To compile this driver as a module, choose M here: the module
 	  will be called pwm-stm32.
 
+config PWM_STM32_LP
+	tristate "STMicroelectronics STM32 PWM LP"
+	depends on MFD_STM32_LPTIMER || COMPILE_TEST
+	help
+	  Generic PWM framework driver for STMicroelectronics STM32 SoCs
+	  with Low-Power Timer (LPTIM).
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called pwm-stm32-lp.
+
 config PWM_STMPE
 	bool "STMPE expander PWM export"
 	depends on MFD_STMPE
@@ -446,7 +456,7 @@ config PWM_TEGRA
 
 config  PWM_TIECAP
 	tristate "ECAP PWM support"
-	depends on ARCH_OMAP2PLUS || ARCH_DAVINCI_DA8XX
+	depends on ARCH_OMAP2PLUS || ARCH_DAVINCI_DA8XX || ARCH_KEYSTONE
 	help
 	  PWM driver support for the ECAP APWM controller found on AM33XX
 	  TI SOC
@@ -500,4 +510,13 @@ config PWM_VT8500
 	  To compile this driver as a module, choose M here: the module
 	  will be called pwm-vt8500.
 
+config PWM_ZX
+	tristate "ZTE ZX PWM support"
+	depends on ARCH_ZX
+	help
+	  Generic PWM framework driver for ZTE ZX family SoCs.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called pwm-zx.
+
 endif
diff --git a/drivers/pwm/Makefile b/drivers/pwm/Makefile
index 93da1f79a3b8..ebefba5f528b 100644
--- a/drivers/pwm/Makefile
+++ b/drivers/pwm/Makefile
@@ -40,6 +40,7 @@ obj-$(CONFIG_PWM_SAMSUNG)	+= pwm-samsung.o
 obj-$(CONFIG_PWM_SPEAR)		+= pwm-spear.o
 obj-$(CONFIG_PWM_STI)		+= pwm-sti.o
 obj-$(CONFIG_PWM_STM32)		+= pwm-stm32.o
+obj-$(CONFIG_PWM_STM32_LP)	+= pwm-stm32-lp.o
 obj-$(CONFIG_PWM_STMPE)		+= pwm-stmpe.o
 obj-$(CONFIG_PWM_SUN4I)		+= pwm-sun4i.o
 obj-$(CONFIG_PWM_TEGRA)		+= pwm-tegra.o
@@ -49,3 +50,4 @@ obj-$(CONFIG_PWM_TIPWMSS)	+= pwm-tipwmss.o
 obj-$(CONFIG_PWM_TWL)		+= pwm-twl.o
 obj-$(CONFIG_PWM_TWL_LED)	+= pwm-twl-led.o
 obj-$(CONFIG_PWM_VT8500)	+= pwm-vt8500.o
+obj-$(CONFIG_PWM_ZX)		+= pwm-zx.o
diff --git a/drivers/pwm/pwm-bcm2835.c b/drivers/pwm/pwm-bcm2835.c
index c5dbf16d810b..db001cba937f 100644
--- a/drivers/pwm/pwm-bcm2835.c
+++ b/drivers/pwm/pwm-bcm2835.c
@@ -167,6 +167,8 @@ static int bcm2835_pwm_probe(struct platform_device *pdev)
 	pc->chip.dev = &pdev->dev;
 	pc->chip.ops = &bcm2835_pwm_ops;
 	pc->chip.npwm = 2;
+	pc->chip.of_xlate = of_pwm_xlate_with_flags;
+	pc->chip.of_pwm_n_cells = 3;
 
 	platform_set_drvdata(pdev, pc);
 
diff --git a/drivers/pwm/pwm-hibvt.c b/drivers/pwm/pwm-hibvt.c
index 8dadc58d6cdf..27c107e78d59 100644
--- a/drivers/pwm/pwm-hibvt.c
+++ b/drivers/pwm/pwm-hibvt.c
@@ -208,7 +208,7 @@ static int hibvt_pwm_probe(struct platform_device *pdev)
 	if (ret < 0)
 		return ret;
 
-	pwm_chip->rstc = devm_reset_control_get(&pdev->dev, NULL);
+	pwm_chip->rstc = devm_reset_control_get_exclusive(&pdev->dev, NULL);
 	if (IS_ERR(pwm_chip->rstc)) {
 		clk_disable_unprepare(pwm_chip->clk);
 		return PTR_ERR(pwm_chip->rstc);
diff --git a/drivers/pwm/pwm-mediatek.c b/drivers/pwm/pwm-mediatek.c
index 5c11bc708a3c..b52f3afb2ba1 100644
--- a/drivers/pwm/pwm-mediatek.c
+++ b/drivers/pwm/pwm-mediatek.c
@@ -2,6 +2,7 @@
  * Mediatek Pulse Width Modulator driver
  *
  * Copyright (C) 2015 John Crispin <blogic@openwrt.org>
+ * Copyright (C) 2017 Zhi Mao <zhi.mao@mediatek.com>
  *
  * This file is licensed under the terms of the GNU General Public
  * License version 2. This program is licensed "as is" without any
@@ -29,6 +30,8 @@
 #define PWMDWIDTH		0x2c
 #define PWMTHRES		0x30
 
+#define PWM_CLK_DIV_MAX		7
+
 enum {
 	MTK_CLK_MAIN = 0,
 	MTK_CLK_TOP,
@@ -61,6 +64,42 @@ static inline struct mtk_pwm_chip *to_mtk_pwm_chip(struct pwm_chip *chip)
 	return container_of(chip, struct mtk_pwm_chip, chip);
 }
 
+static int mtk_pwm_clk_enable(struct pwm_chip *chip, struct pwm_device *pwm)
+{
+	struct mtk_pwm_chip *pc = to_mtk_pwm_chip(chip);
+	int ret;
+
+	ret = clk_prepare_enable(pc->clks[MTK_CLK_TOP]);
+	if (ret < 0)
+		return ret;
+
+	ret = clk_prepare_enable(pc->clks[MTK_CLK_MAIN]);
+	if (ret < 0)
+		goto disable_clk_top;
+
+	ret = clk_prepare_enable(pc->clks[MTK_CLK_PWM1 + pwm->hwpwm]);
+	if (ret < 0)
+		goto disable_clk_main;
+
+	return 0;
+
+disable_clk_main:
+	clk_disable_unprepare(pc->clks[MTK_CLK_MAIN]);
+disable_clk_top:
+	clk_disable_unprepare(pc->clks[MTK_CLK_TOP]);
+
+	return ret;
+}
+
+static void mtk_pwm_clk_disable(struct pwm_chip *chip, struct pwm_device *pwm)
+{
+	struct mtk_pwm_chip *pc = to_mtk_pwm_chip(chip);
+
+	clk_disable_unprepare(pc->clks[MTK_CLK_PWM1 + pwm->hwpwm]);
+	clk_disable_unprepare(pc->clks[MTK_CLK_MAIN]);
+	clk_disable_unprepare(pc->clks[MTK_CLK_TOP]);
+}
+
 static inline u32 mtk_pwm_readl(struct mtk_pwm_chip *chip, unsigned int num,
 				unsigned int offset)
 {
@@ -80,6 +119,11 @@ static int mtk_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 	struct mtk_pwm_chip *pc = to_mtk_pwm_chip(chip);
 	struct clk *clk = pc->clks[MTK_CLK_PWM1 + pwm->hwpwm];
 	u32 resolution, clkdiv = 0;
+	int ret;
+
+	ret = mtk_pwm_clk_enable(chip, pwm);
+	if (ret < 0)
+		return ret;
 
 	resolution = NSEC_PER_SEC / clk_get_rate(clk);
 
@@ -88,13 +132,18 @@ static int mtk_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 		clkdiv++;
 	}
 
-	if (clkdiv > 7)
+	if (clkdiv > PWM_CLK_DIV_MAX) {
+		mtk_pwm_clk_disable(chip, pwm);
+		dev_err(chip->dev, "period %d not supported\n", period_ns);
 		return -EINVAL;
+	}
 
-	mtk_pwm_writel(pc, pwm->hwpwm, PWMCON, BIT(15) | BIT(3) | clkdiv);
+	mtk_pwm_writel(pc, pwm->hwpwm, PWMCON, BIT(15) | clkdiv);
 	mtk_pwm_writel(pc, pwm->hwpwm, PWMDWIDTH, period_ns / resolution);
 	mtk_pwm_writel(pc, pwm->hwpwm, PWMTHRES, duty_ns / resolution);
 
+	mtk_pwm_clk_disable(chip, pwm);
+
 	return 0;
 }
 
@@ -104,7 +153,7 @@ static int mtk_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm)
 	u32 value;
 	int ret;
 
-	ret = clk_prepare(pc->clks[MTK_CLK_PWM1 + pwm->hwpwm]);
+	ret = mtk_pwm_clk_enable(chip, pwm);
 	if (ret < 0)
 		return ret;
 
@@ -124,7 +173,7 @@ static void mtk_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm)
 	value &= ~BIT(pwm->hwpwm);
 	writel(value, pc->regs);
 
-	clk_unprepare(pc->clks[MTK_CLK_PWM1 + pwm->hwpwm]);
+	mtk_pwm_clk_disable(chip, pwm);
 }
 
 static const struct pwm_ops mtk_pwm_ops = {
@@ -156,14 +205,6 @@ static int mtk_pwm_probe(struct platform_device *pdev)
 			return PTR_ERR(pc->clks[i]);
 	}
 
-	ret = clk_prepare(pc->clks[MTK_CLK_TOP]);
-	if (ret < 0)
-		return ret;
-
-	ret = clk_prepare(pc->clks[MTK_CLK_MAIN]);
-	if (ret < 0)
-		goto disable_clk_top;
-
 	platform_set_drvdata(pdev, pc);
 
 	pc->chip.dev = &pdev->dev;
@@ -174,26 +215,15 @@ static int mtk_pwm_probe(struct platform_device *pdev)
 	ret = pwmchip_add(&pc->chip);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "pwmchip_add() failed: %d\n", ret);
-		goto disable_clk_main;
+		return ret;
 	}
 
 	return 0;
-
-disable_clk_main:
-	clk_unprepare(pc->clks[MTK_CLK_MAIN]);
-disable_clk_top:
-	clk_unprepare(pc->clks[MTK_CLK_TOP]);
-
-	return ret;
 }
 
 static int mtk_pwm_remove(struct platform_device *pdev)
 {
 	struct mtk_pwm_chip *pc = platform_get_drvdata(pdev);
-	unsigned int i;
-
-	for (i = 0; i < pc->chip.npwm; i++)
-		pwm_disable(&pc->chip.pwms[i]);
 
 	return pwmchip_remove(&pc->chip);
 }
diff --git a/drivers/pwm/pwm-meson.c b/drivers/pwm/pwm-meson.c
index cb845edfe2b4..d589331d1884 100644
--- a/drivers/pwm/pwm-meson.c
+++ b/drivers/pwm/pwm-meson.c
@@ -441,7 +441,7 @@ static int meson_pwm_init_channels(struct meson_pwm *meson,
 	for (i = 0; i < meson->chip.npwm; i++) {
 		struct meson_pwm_channel *channel = &channels[i];
 
-		snprintf(name, sizeof(name), "%s#mux%u", np->full_name, i);
+		snprintf(name, sizeof(name), "%pOF#mux%u", np, i);
 
 		init.name = name;
 		init.ops = &clk_mux_ops;
diff --git a/drivers/pwm/pwm-pca9685.c b/drivers/pwm/pwm-pca9685.c
index 5f55cfab9b1c..a7eaf962a95b 100644
--- a/drivers/pwm/pwm-pca9685.c
+++ b/drivers/pwm/pwm-pca9685.c
@@ -241,11 +241,11 @@ static inline int pca9685_pwm_gpio_probe(struct pca9685 *pca)
 }
 #endif
 
-static void pca9685_set_sleep_mode(struct pca9685 *pca, int sleep)
+static void pca9685_set_sleep_mode(struct pca9685 *pca, bool enable)
 {
 	regmap_update_bits(pca->regmap, PCA9685_MODE1,
-			   MODE1_SLEEP, sleep ? MODE1_SLEEP : 0);
-	if (!sleep) {
+			   MODE1_SLEEP, enable ? MODE1_SLEEP : 0);
+	if (!enable) {
 		/* Wait 500us for the oscillator to be back up */
 		udelay(500);
 	}
@@ -272,13 +272,13 @@ static int pca9685_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 			 * state is guaranteed active here.
 			 */
 			/* Put chip into sleep mode */
-			pca9685_set_sleep_mode(pca, 1);
+			pca9685_set_sleep_mode(pca, true);
 
 			/* Change the chip-wide output frequency */
 			regmap_write(pca->regmap, PCA9685_PRESCALE, prescale);
 
 			/* Wake the chip up */
-			pca9685_set_sleep_mode(pca, 0);
+			pca9685_set_sleep_mode(pca, false);
 
 			pca->period_ns = period_ns;
 		} else {
@@ -534,7 +534,7 @@ static int pca9685_pwm_runtime_suspend(struct device *dev)
 	struct i2c_client *client = to_i2c_client(dev);
 	struct pca9685 *pca = i2c_get_clientdata(client);
 
-	pca9685_set_sleep_mode(pca, 1);
+	pca9685_set_sleep_mode(pca, true);
 	return 0;
 }
 
@@ -543,7 +543,7 @@ static int pca9685_pwm_runtime_resume(struct device *dev)
 	struct i2c_client *client = to_i2c_client(dev);
 	struct pca9685 *pca = i2c_get_clientdata(client);
 
-	pca9685_set_sleep_mode(pca, 0);
+	pca9685_set_sleep_mode(pca, false);
 	return 0;
 }
 #endif
diff --git a/drivers/pwm/pwm-renesas-tpu.c b/drivers/pwm/pwm-renesas-tpu.c
index 075c1a764ba2..29267d12fb4c 100644
--- a/drivers/pwm/pwm-renesas-tpu.c
+++ b/drivers/pwm/pwm-renesas-tpu.c
@@ -455,7 +455,6 @@ static const struct of_device_id tpu_of_table[] = {
 	{ .compatible = "renesas,tpu-r8a73a4", },
 	{ .compatible = "renesas,tpu-r8a7740", },
 	{ .compatible = "renesas,tpu-r8a7790", },
-	{ .compatible = "renesas,tpu-sh7372", },
 	{ .compatible = "renesas,tpu", },
 	{ },
 };
diff --git a/drivers/pwm/pwm-rockchip.c b/drivers/pwm/pwm-rockchip.c
index 744d56197286..4d99d468df09 100644
--- a/drivers/pwm/pwm-rockchip.c
+++ b/drivers/pwm/pwm-rockchip.c
@@ -27,12 +27,15 @@
 #define PWM_DUTY_NEGATIVE	(0 << 3)
 #define PWM_INACTIVE_NEGATIVE	(0 << 4)
 #define PWM_INACTIVE_POSITIVE	(1 << 4)
+#define PWM_POLARITY_MASK	(PWM_DUTY_POSITIVE | PWM_INACTIVE_POSITIVE)
 #define PWM_OUTPUT_LEFT		(0 << 5)
+#define PWM_LOCK_EN		(1 << 6)
 #define PWM_LP_DISABLE		(0 << 8)
 
 struct rockchip_pwm_chip {
 	struct pwm_chip chip;
 	struct clk *clk;
+	struct clk *pclk;
 	const struct rockchip_pwm_data *data;
 	void __iomem *base;
 };
@@ -48,13 +51,8 @@ struct rockchip_pwm_data {
 	struct rockchip_pwm_regs regs;
 	unsigned int prescaler;
 	bool supports_polarity;
-	const struct pwm_ops *ops;
-
-	void (*set_enable)(struct pwm_chip *chip,
-			   struct pwm_device *pwm, bool enable,
-			   enum pwm_polarity polarity);
-	void (*get_state)(struct pwm_chip *chip, struct pwm_device *pwm,
-			  struct pwm_state *state);
+	bool supports_lock;
+	u32 enable_conf;
 };
 
 static inline struct rockchip_pwm_chip *to_rockchip_pwm_chip(struct pwm_chip *c)
@@ -62,90 +60,18 @@ static inline struct rockchip_pwm_chip *to_rockchip_pwm_chip(struct pwm_chip *c)
 	return container_of(c, struct rockchip_pwm_chip, chip);
 }
 
-static void rockchip_pwm_set_enable_v1(struct pwm_chip *chip,
-				       struct pwm_device *pwm, bool enable,
-				       enum pwm_polarity polarity)
-{
-	struct rockchip_pwm_chip *pc = to_rockchip_pwm_chip(chip);
-	u32 enable_conf = PWM_CTRL_OUTPUT_EN | PWM_CTRL_TIMER_EN;
-	u32 val;
-
-	val = readl_relaxed(pc->base + pc->data->regs.ctrl);
-
-	if (enable)
-		val |= enable_conf;
-	else
-		val &= ~enable_conf;
-
-	writel_relaxed(val, pc->base + pc->data->regs.ctrl);
-}
-
-static void rockchip_pwm_get_state_v1(struct pwm_chip *chip,
-				      struct pwm_device *pwm,
-				      struct pwm_state *state)
-{
-	struct rockchip_pwm_chip *pc = to_rockchip_pwm_chip(chip);
-	u32 enable_conf = PWM_CTRL_OUTPUT_EN | PWM_CTRL_TIMER_EN;
-	u32 val;
-
-	val = readl_relaxed(pc->base + pc->data->regs.ctrl);
-	if ((val & enable_conf) == enable_conf)
-		state->enabled = true;
-}
-
-static void rockchip_pwm_set_enable_v2(struct pwm_chip *chip,
-				       struct pwm_device *pwm, bool enable,
-				       enum pwm_polarity polarity)
-{
-	struct rockchip_pwm_chip *pc = to_rockchip_pwm_chip(chip);
-	u32 enable_conf = PWM_OUTPUT_LEFT | PWM_LP_DISABLE | PWM_ENABLE |
-			  PWM_CONTINUOUS;
-	u32 val;
-
-	if (polarity == PWM_POLARITY_INVERSED)
-		enable_conf |= PWM_DUTY_NEGATIVE | PWM_INACTIVE_POSITIVE;
-	else
-		enable_conf |= PWM_DUTY_POSITIVE | PWM_INACTIVE_NEGATIVE;
-
-	val = readl_relaxed(pc->base + pc->data->regs.ctrl);
-
-	if (enable)
-		val |= enable_conf;
-	else
-		val &= ~enable_conf;
-
-	writel_relaxed(val, pc->base + pc->data->regs.ctrl);
-}
-
-static void rockchip_pwm_get_state_v2(struct pwm_chip *chip,
-				      struct pwm_device *pwm,
-				      struct pwm_state *state)
-{
-	struct rockchip_pwm_chip *pc = to_rockchip_pwm_chip(chip);
-	u32 enable_conf = PWM_OUTPUT_LEFT | PWM_LP_DISABLE | PWM_ENABLE |
-			  PWM_CONTINUOUS;
-	u32 val;
-
-	val = readl_relaxed(pc->base + pc->data->regs.ctrl);
-	if ((val & enable_conf) != enable_conf)
-		return;
-
-	state->enabled = true;
-
-	if (!(val & PWM_DUTY_POSITIVE))
-		state->polarity = PWM_POLARITY_INVERSED;
-}
-
 static void rockchip_pwm_get_state(struct pwm_chip *chip,
 				   struct pwm_device *pwm,
 				   struct pwm_state *state)
 {
 	struct rockchip_pwm_chip *pc = to_rockchip_pwm_chip(chip);
+	u32 enable_conf = pc->data->enable_conf;
 	unsigned long clk_rate;
 	u64 tmp;
+	u32 val;
 	int ret;
 
-	ret = clk_enable(pc->clk);
+	ret = clk_enable(pc->pclk);
 	if (ret)
 		return;
 
@@ -157,19 +83,31 @@ static void rockchip_pwm_get_state(struct pwm_chip *chip,
 
 	tmp = readl_relaxed(pc->base + pc->data->regs.duty);
 	tmp *= pc->data->prescaler * NSEC_PER_SEC;
-	state->duty_cycle = DIV_ROUND_CLOSEST_ULL(tmp, clk_rate);
+	state->duty_cycle =  DIV_ROUND_CLOSEST_ULL(tmp, clk_rate);
+
+	val = readl_relaxed(pc->base + pc->data->regs.ctrl);
+	if (pc->data->supports_polarity)
+		state->enabled = ((val & enable_conf) != enable_conf) ?
+				 false : true;
+	else
+		state->enabled = ((val & enable_conf) == enable_conf) ?
+				 true : false;
 
-	pc->data->get_state(chip, pwm, state);
+	if (pc->data->supports_polarity) {
+		if (!(val & PWM_DUTY_POSITIVE))
+			state->polarity = PWM_POLARITY_INVERSED;
+	}
 
-	clk_disable(pc->clk);
+	clk_disable(pc->pclk);
 }
 
-static int rockchip_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
-			       int duty_ns, int period_ns)
+static void rockchip_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
+			       struct pwm_state *state)
 {
 	struct rockchip_pwm_chip *pc = to_rockchip_pwm_chip(chip);
 	unsigned long period, duty;
 	u64 clk_rate, div;
+	u32 ctrl;
 
 	clk_rate = clk_get_rate(pc->clk);
 
@@ -178,26 +116,53 @@ static int rockchip_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 	 * bits, every possible input period can be obtained using the
 	 * default prescaler value for all practical clock rate values.
 	 */
-	div = clk_rate * period_ns;
+	div = clk_rate * state->period;
 	period = DIV_ROUND_CLOSEST_ULL(div,
 				       pc->data->prescaler * NSEC_PER_SEC);
 
-	div = clk_rate * duty_ns;
+	div = clk_rate * state->duty_cycle;
 	duty = DIV_ROUND_CLOSEST_ULL(div, pc->data->prescaler * NSEC_PER_SEC);
 
+	/*
+	 * Lock the period and duty of previous configuration, then
+	 * change the duty and period, that would not be effective.
+	 */
+	ctrl = readl_relaxed(pc->base + pc->data->regs.ctrl);
+	if (pc->data->supports_lock) {
+		ctrl |= PWM_LOCK_EN;
+		writel_relaxed(ctrl, pc->base + pc->data->regs.ctrl);
+	}
+
 	writel(period, pc->base + pc->data->regs.period);
 	writel(duty, pc->base + pc->data->regs.duty);
 
-	return 0;
+	if (pc->data->supports_polarity) {
+		ctrl &= ~PWM_POLARITY_MASK;
+		if (state->polarity == PWM_POLARITY_INVERSED)
+			ctrl |= PWM_DUTY_NEGATIVE | PWM_INACTIVE_POSITIVE;
+		else
+			ctrl |= PWM_DUTY_POSITIVE | PWM_INACTIVE_NEGATIVE;
+	}
+
+	/*
+	 * Unlock and set polarity at the same time,
+	 * the configuration of duty, period and polarity
+	 * would be effective together at next period.
+	 */
+	if (pc->data->supports_lock)
+		ctrl &= ~PWM_LOCK_EN;
+
+	writel(ctrl, pc->base + pc->data->regs.ctrl);
 }
 
 static int rockchip_pwm_enable(struct pwm_chip *chip,
-			 struct pwm_device *pwm,
-			 bool enable,
-			 enum pwm_polarity polarity)
+			       struct pwm_device *pwm,
+			       bool enable)
 {
 	struct rockchip_pwm_chip *pc = to_rockchip_pwm_chip(chip);
+	u32 enable_conf = pc->data->enable_conf;
 	int ret;
+	u32 val;
 
 	if (enable) {
 		ret = clk_enable(pc->clk);
@@ -205,7 +170,14 @@ static int rockchip_pwm_enable(struct pwm_chip *chip,
 			return ret;
 	}
 
-	pc->data->set_enable(chip, pwm, enable, polarity);
+	val = readl_relaxed(pc->base + pc->data->regs.ctrl);
+
+	if (enable)
+		val |= enable_conf;
+	else
+		val &= ~enable_conf;
+
+	writel_relaxed(val, pc->base + pc->data->regs.ctrl);
 
 	if (!enable)
 		clk_disable(pc->clk);
@@ -219,33 +191,26 @@ static int rockchip_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm,
 	struct rockchip_pwm_chip *pc = to_rockchip_pwm_chip(chip);
 	struct pwm_state curstate;
 	bool enabled;
-	int ret;
+	int ret = 0;
 
-	pwm_get_state(pwm, &curstate);
-	enabled = curstate.enabled;
-
-	ret = clk_enable(pc->clk);
+	ret = clk_enable(pc->pclk);
 	if (ret)
 		return ret;
 
-	if (state->polarity != curstate.polarity && enabled) {
-		ret = rockchip_pwm_enable(chip, pwm, false, state->polarity);
+	pwm_get_state(pwm, &curstate);
+	enabled = curstate.enabled;
+
+	if (state->polarity != curstate.polarity && enabled &&
+	    !pc->data->supports_lock) {
+		ret = rockchip_pwm_enable(chip, pwm, false);
 		if (ret)
 			goto out;
 		enabled = false;
 	}
 
-	ret = rockchip_pwm_config(chip, pwm, state->duty_cycle, state->period);
-	if (ret) {
-		if (enabled != curstate.enabled)
-			rockchip_pwm_enable(chip, pwm, !enabled,
-				      state->polarity);
-		goto out;
-	}
-
+	rockchip_pwm_config(chip, pwm, state);
 	if (state->enabled != enabled) {
-		ret = rockchip_pwm_enable(chip, pwm, state->enabled,
-				    state->polarity);
+		ret = rockchip_pwm_enable(chip, pwm, state->enabled);
 		if (ret)
 			goto out;
 	}
@@ -257,18 +222,12 @@ static int rockchip_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm,
 	rockchip_pwm_get_state(chip, pwm, state);
 
 out:
-	clk_disable(pc->clk);
+	clk_disable(pc->pclk);
 
 	return ret;
 }
 
-static const struct pwm_ops rockchip_pwm_ops_v1 = {
-	.get_state = rockchip_pwm_get_state,
-	.apply = rockchip_pwm_apply,
-	.owner = THIS_MODULE,
-};
-
-static const struct pwm_ops rockchip_pwm_ops_v2 = {
+static const struct pwm_ops rockchip_pwm_ops = {
 	.get_state = rockchip_pwm_get_state,
 	.apply = rockchip_pwm_apply,
 	.owner = THIS_MODULE,
@@ -282,9 +241,9 @@ static const struct rockchip_pwm_data pwm_data_v1 = {
 		.ctrl = 0x0c,
 	},
 	.prescaler = 2,
-	.ops = &rockchip_pwm_ops_v1,
-	.set_enable = rockchip_pwm_set_enable_v1,
-	.get_state = rockchip_pwm_get_state_v1,
+	.supports_polarity = false,
+	.supports_lock = false,
+	.enable_conf = PWM_CTRL_OUTPUT_EN | PWM_CTRL_TIMER_EN,
 };
 
 static const struct rockchip_pwm_data pwm_data_v2 = {
@@ -296,9 +255,9 @@ static const struct rockchip_pwm_data pwm_data_v2 = {
 	},
 	.prescaler = 1,
 	.supports_polarity = true,
-	.ops = &rockchip_pwm_ops_v2,
-	.set_enable = rockchip_pwm_set_enable_v2,
-	.get_state = rockchip_pwm_get_state_v2,
+	.supports_lock = false,
+	.enable_conf = PWM_OUTPUT_LEFT | PWM_LP_DISABLE | PWM_ENABLE |
+		       PWM_CONTINUOUS,
 };
 
 static const struct rockchip_pwm_data pwm_data_vop = {
@@ -310,15 +269,30 @@ static const struct rockchip_pwm_data pwm_data_vop = {
 	},
 	.prescaler = 1,
 	.supports_polarity = true,
-	.ops = &rockchip_pwm_ops_v2,
-	.set_enable = rockchip_pwm_set_enable_v2,
-	.get_state = rockchip_pwm_get_state_v2,
+	.supports_lock = false,
+	.enable_conf = PWM_OUTPUT_LEFT | PWM_LP_DISABLE | PWM_ENABLE |
+		       PWM_CONTINUOUS,
+};
+
+static const struct rockchip_pwm_data pwm_data_v3 = {
+	.regs = {
+		.duty = 0x08,
+		.period = 0x04,
+		.cntr = 0x00,
+		.ctrl = 0x0c,
+	},
+	.prescaler = 1,
+	.supports_polarity = true,
+	.supports_lock = true,
+	.enable_conf = PWM_OUTPUT_LEFT | PWM_LP_DISABLE | PWM_ENABLE |
+		       PWM_CONTINUOUS,
 };
 
 static const struct of_device_id rockchip_pwm_dt_ids[] = {
 	{ .compatible = "rockchip,rk2928-pwm", .data = &pwm_data_v1},
 	{ .compatible = "rockchip,rk3288-pwm", .data = &pwm_data_v2},
 	{ .compatible = "rockchip,vop-pwm", .data = &pwm_data_vop},
+	{ .compatible = "rockchip,rk3328-pwm", .data = &pwm_data_v3},
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, rockchip_pwm_dt_ids);
@@ -328,7 +302,7 @@ static int rockchip_pwm_probe(struct platform_device *pdev)
 	const struct of_device_id *id;
 	struct rockchip_pwm_chip *pc;
 	struct resource *r;
-	int ret;
+	int ret, count;
 
 	id = of_match_device(rockchip_pwm_dt_ids, &pdev->dev);
 	if (!id)
@@ -343,19 +317,49 @@ static int rockchip_pwm_probe(struct platform_device *pdev)
 	if (IS_ERR(pc->base))
 		return PTR_ERR(pc->base);
 
-	pc->clk = devm_clk_get(&pdev->dev, NULL);
-	if (IS_ERR(pc->clk))
-		return PTR_ERR(pc->clk);
+	pc->clk = devm_clk_get(&pdev->dev, "pwm");
+	if (IS_ERR(pc->clk)) {
+		pc->clk = devm_clk_get(&pdev->dev, NULL);
+		if (IS_ERR(pc->clk)) {
+			ret = PTR_ERR(pc->clk);
+			if (ret != -EPROBE_DEFER)
+				dev_err(&pdev->dev, "Can't get bus clk: %d\n",
+					ret);
+			return ret;
+		}
+	}
+
+	count = of_count_phandle_with_args(pdev->dev.of_node,
+					   "clocks", "#clock-cells");
+	if (count == 2)
+		pc->pclk = devm_clk_get(&pdev->dev, "pclk");
+	else
+		pc->pclk = pc->clk;
+
+	if (IS_ERR(pc->pclk)) {
+		ret = PTR_ERR(pc->pclk);
+		if (ret != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "Can't get APB clk: %d\n", ret);
+		return ret;
+	}
 
 	ret = clk_prepare_enable(pc->clk);
-	if (ret)
+	if (ret) {
+		dev_err(&pdev->dev, "Can't prepare enable bus clk: %d\n", ret);
 		return ret;
+	}
+
+	ret = clk_prepare(pc->pclk);
+	if (ret) {
+		dev_err(&pdev->dev, "Can't prepare APB clk: %d\n", ret);
+		goto err_clk;
+	}
 
 	platform_set_drvdata(pdev, pc);
 
 	pc->data = id->data;
 	pc->chip.dev = &pdev->dev;
-	pc->chip.ops = pc->data->ops;
+	pc->chip.ops = &rockchip_pwm_ops;
 	pc->chip.base = -1;
 	pc->chip.npwm = 1;
 
@@ -368,12 +372,20 @@ static int rockchip_pwm_probe(struct platform_device *pdev)
 	if (ret < 0) {
 		clk_unprepare(pc->clk);
 		dev_err(&pdev->dev, "pwmchip_add() failed: %d\n", ret);
+		goto err_pclk;
 	}
 
 	/* Keep the PWM clk enabled if the PWM appears to be up and running. */
 	if (!pwm_is_enabled(pc->chip.pwms))
 		clk_disable(pc->clk);
 
+	return 0;
+
+err_pclk:
+	clk_unprepare(pc->pclk);
+err_clk:
+	clk_disable_unprepare(pc->clk);
+
 	return ret;
 }
 
@@ -395,6 +407,7 @@ static int rockchip_pwm_remove(struct platform_device *pdev)
 	if (pwm_is_enabled(pc->chip.pwms))
 		clk_disable(pc->clk);
 
+	clk_unprepare(pc->pclk);
 	clk_unprepare(pc->clk);
 
 	return pwmchip_remove(&pc->chip);
diff --git a/drivers/pwm/pwm-samsung.c b/drivers/pwm/pwm-samsung.c
index f113cda47032..062f2cfc45ec 100644
--- a/drivers/pwm/pwm-samsung.c
+++ b/drivers/pwm/pwm-samsung.c
@@ -3,6 +3,7 @@
  * Copyright (c) 2008 Simtec Electronics
  *     Ben Dooks <ben@simtec.co.uk>, <ben-linux@fluff.org>
  * Copyright (c) 2013 Tomasz Figa <tomasz.figa@gmail.com>
+ * Copyright (c) 2017 Samsung Electronics Co., Ltd.
  *
  * PWM driver for Samsung SoCs
  *
@@ -74,6 +75,7 @@ struct samsung_pwm_channel {
  * @chip:		generic PWM chip
  * @variant:		local copy of hardware variant data
  * @inverter_mask:	inverter status for all channels - one bit per channel
+ * @disabled_mask:	disabled status for all channels - one bit per channel
  * @base:		base address of mapped PWM registers
  * @base_clk:		base clock used to drive the timers
  * @tclk0:		external clock 0 (can be ERR_PTR if not present)
@@ -83,6 +85,7 @@ struct samsung_pwm_chip {
 	struct pwm_chip chip;
 	struct samsung_pwm_variant variant;
 	u8 inverter_mask;
+	u8 disabled_mask;
 
 	void __iomem *base;
 	struct clk *base_clk;
@@ -257,6 +260,8 @@ static int pwm_samsung_enable(struct pwm_chip *chip, struct pwm_device *pwm)
 	tcon |= TCON_START(tcon_chan) | TCON_AUTORELOAD(tcon_chan);
 	writel(tcon, our_chip->base + REG_TCON);
 
+	our_chip->disabled_mask &= ~BIT(pwm->hwpwm);
+
 	spin_unlock_irqrestore(&samsung_pwm_lock, flags);
 
 	return 0;
@@ -275,6 +280,8 @@ static void pwm_samsung_disable(struct pwm_chip *chip, struct pwm_device *pwm)
 	tcon &= ~TCON_AUTORELOAD(tcon_chan);
 	writel(tcon, our_chip->base + REG_TCON);
 
+	our_chip->disabled_mask |= BIT(pwm->hwpwm);
+
 	spin_unlock_irqrestore(&samsung_pwm_lock, flags);
 }
 
@@ -297,8 +304,8 @@ static void pwm_samsung_manual_update(struct samsung_pwm_chip *chip,
 	spin_unlock_irqrestore(&samsung_pwm_lock, flags);
 }
 
-static int pwm_samsung_config(struct pwm_chip *chip, struct pwm_device *pwm,
-			      int duty_ns, int period_ns)
+static int __pwm_samsung_config(struct pwm_chip *chip, struct pwm_device *pwm,
+				int duty_ns, int period_ns, bool force_period)
 {
 	struct samsung_pwm_chip *our_chip = to_samsung_pwm_chip(chip);
 	struct samsung_pwm_channel *chan = pwm_get_chip_data(pwm);
@@ -312,9 +319,6 @@ static int pwm_samsung_config(struct pwm_chip *chip, struct pwm_device *pwm,
 	if (period_ns > NSEC_PER_SEC)
 		return -ERANGE;
 
-	if (period_ns == chan->period_ns && duty_ns == chan->duty_ns)
-		return 0;
-
 	tcnt = readl(our_chip->base + REG_TCNTB(pwm->hwpwm));
 	oldtcmp = readl(our_chip->base + REG_TCMPB(pwm->hwpwm));
 
@@ -322,7 +326,7 @@ static int pwm_samsung_config(struct pwm_chip *chip, struct pwm_device *pwm,
 	++tcnt;
 
 	/* Check to see if we are changing the clock rate of the PWM. */
-	if (chan->period_ns != period_ns) {
+	if (chan->period_ns != period_ns || force_period) {
 		unsigned long tin_rate;
 		u32 period;
 
@@ -381,6 +385,12 @@ static int pwm_samsung_config(struct pwm_chip *chip, struct pwm_device *pwm,
 	return 0;
 }
 
+static int pwm_samsung_config(struct pwm_chip *chip, struct pwm_device *pwm,
+			      int duty_ns, int period_ns)
+{
+	return __pwm_samsung_config(chip, pwm, duty_ns, period_ns, false);
+}
+
 static void pwm_samsung_set_invert(struct samsung_pwm_chip *chip,
 				   unsigned int channel, bool invert)
 {
@@ -592,51 +602,41 @@ static int pwm_samsung_remove(struct platform_device *pdev)
 }
 
 #ifdef CONFIG_PM_SLEEP
-static int pwm_samsung_suspend(struct device *dev)
+static int pwm_samsung_resume(struct device *dev)
 {
-	struct samsung_pwm_chip *chip = dev_get_drvdata(dev);
+	struct samsung_pwm_chip *our_chip = dev_get_drvdata(dev);
+	struct pwm_chip *chip = &our_chip->chip;
 	unsigned int i;
 
-	/*
-	 * No one preserves these values during suspend so reset them.
-	 * Otherwise driver leaves PWM unconfigured if same values are
-	 * passed to pwm_config() next time.
-	 */
-	for (i = 0; i < SAMSUNG_PWM_NUM; ++i) {
-		struct pwm_device *pwm = &chip->chip.pwms[i];
+	for (i = 0; i < SAMSUNG_PWM_NUM; i++) {
+		struct pwm_device *pwm = &chip->pwms[i];
 		struct samsung_pwm_channel *chan = pwm_get_chip_data(pwm);
 
 		if (!chan)
 			continue;
 
-		chan->period_ns = 0;
-		chan->duty_ns = 0;
-	}
-
-	return 0;
-}
+		if (our_chip->variant.output_mask & BIT(i))
+			pwm_samsung_set_invert(our_chip, i,
+					our_chip->inverter_mask & BIT(i));
 
-static int pwm_samsung_resume(struct device *dev)
-{
-	struct samsung_pwm_chip *chip = dev_get_drvdata(dev);
-	unsigned int chan;
+		if (chan->period_ns) {
+			__pwm_samsung_config(chip, pwm, chan->duty_ns,
+					     chan->period_ns, true);
+			/* needed to make PWM disable work on Odroid-XU3 */
+			pwm_samsung_manual_update(our_chip, pwm);
+		}
 
-	/*
-	 * Inverter setting must be preserved across suspend/resume
-	 * as nobody really seems to configure it more than once.
-	 */
-	for (chan = 0; chan < SAMSUNG_PWM_NUM; ++chan) {
-		if (chip->variant.output_mask & BIT(chan))
-			pwm_samsung_set_invert(chip, chan,
-					chip->inverter_mask & BIT(chan));
+		if (our_chip->disabled_mask & BIT(i))
+			pwm_samsung_disable(chip, pwm);
+		else
+			pwm_samsung_enable(chip, pwm);
 	}
 
 	return 0;
 }
 #endif
 
-static SIMPLE_DEV_PM_OPS(pwm_samsung_pm_ops, pwm_samsung_suspend,
-			 pwm_samsung_resume);
+static SIMPLE_DEV_PM_OPS(pwm_samsung_pm_ops, NULL, pwm_samsung_resume);
 
 static struct platform_driver pwm_samsung_driver = {
 	.driver		= {
diff --git a/drivers/pwm/pwm-stm32-lp.c b/drivers/pwm/pwm-stm32-lp.c
new file mode 100644
index 000000000000..9793b296108f
--- /dev/null
+++ b/drivers/pwm/pwm-stm32-lp.c
@@ -0,0 +1,246 @@
+/*
+ * STM32 Low-Power Timer PWM driver
+ *
+ * Copyright (C) STMicroelectronics 2017
+ *
+ * Author: Gerald Baeza <gerald.baeza@st.com>
+ *
+ * License terms: GNU General Public License (GPL), version 2
+ *
+ * Inspired by Gerald Baeza's pwm-stm32 driver
+ */
+
+#include <linux/bitfield.h>
+#include <linux/mfd/stm32-lptimer.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pwm.h>
+
+struct stm32_pwm_lp {
+	struct pwm_chip chip;
+	struct clk *clk;
+	struct regmap *regmap;
+};
+
+static inline struct stm32_pwm_lp *to_stm32_pwm_lp(struct pwm_chip *chip)
+{
+	return container_of(chip, struct stm32_pwm_lp, chip);
+}
+
+/* STM32 Low-Power Timer is preceded by a configurable power-of-2 prescaler */
+#define STM32_LPTIM_MAX_PRESCALER	128
+
+static int stm32_pwm_lp_apply(struct pwm_chip *chip, struct pwm_device *pwm,
+			      struct pwm_state *state)
+{
+	struct stm32_pwm_lp *priv = to_stm32_pwm_lp(chip);
+	unsigned long long prd, div, dty;
+	struct pwm_state cstate;
+	u32 val, mask, cfgr, presc = 0;
+	bool reenable;
+	int ret;
+
+	pwm_get_state(pwm, &cstate);
+	reenable = !cstate.enabled;
+
+	if (!state->enabled) {
+		if (cstate.enabled) {
+			/* Disable LP timer */
+			ret = regmap_write(priv->regmap, STM32_LPTIM_CR, 0);
+			if (ret)
+				return ret;
+			/* disable clock to PWM counter */
+			clk_disable(priv->clk);
+		}
+		return 0;
+	}
+
+	/* Calculate the period and prescaler value */
+	div = (unsigned long long)clk_get_rate(priv->clk) * state->period;
+	do_div(div, NSEC_PER_SEC);
+	prd = div;
+	while (div > STM32_LPTIM_MAX_ARR) {
+		presc++;
+		if ((1 << presc) > STM32_LPTIM_MAX_PRESCALER) {
+			dev_err(priv->chip.dev, "max prescaler exceeded\n");
+			return -EINVAL;
+		}
+		div = prd >> presc;
+	}
+	prd = div;
+
+	/* Calculate the duty cycle */
+	dty = prd * state->duty_cycle;
+	do_div(dty, state->period);
+
+	if (!cstate.enabled) {
+		/* enable clock to drive PWM counter */
+		ret = clk_enable(priv->clk);
+		if (ret)
+			return ret;
+	}
+
+	ret = regmap_read(priv->regmap, STM32_LPTIM_CFGR, &cfgr);
+	if (ret)
+		goto err;
+
+	if ((FIELD_GET(STM32_LPTIM_PRESC, cfgr) != presc) ||
+	    (FIELD_GET(STM32_LPTIM_WAVPOL, cfgr) != state->polarity)) {
+		val = FIELD_PREP(STM32_LPTIM_PRESC, presc);
+		val |= FIELD_PREP(STM32_LPTIM_WAVPOL, state->polarity);
+		mask = STM32_LPTIM_PRESC | STM32_LPTIM_WAVPOL;
+
+		/* Must disable LP timer to modify CFGR */
+		reenable = true;
+		ret = regmap_write(priv->regmap, STM32_LPTIM_CR, 0);
+		if (ret)
+			goto err;
+
+		ret = regmap_update_bits(priv->regmap, STM32_LPTIM_CFGR, mask,
+					 val);
+		if (ret)
+			goto err;
+	}
+
+	if (reenable) {
+		/* Must (re)enable LP timer to modify CMP & ARR */
+		ret = regmap_write(priv->regmap, STM32_LPTIM_CR,
+				   STM32_LPTIM_ENABLE);
+		if (ret)
+			goto err;
+	}
+
+	ret = regmap_write(priv->regmap, STM32_LPTIM_ARR, prd - 1);
+	if (ret)
+		goto err;
+
+	ret = regmap_write(priv->regmap, STM32_LPTIM_CMP, prd - (1 + dty));
+	if (ret)
+		goto err;
+
+	/* ensure CMP & ARR registers are properly written */
+	ret = regmap_read_poll_timeout(priv->regmap, STM32_LPTIM_ISR, val,
+				       (val & STM32_LPTIM_CMPOK_ARROK),
+				       100, 1000);
+	if (ret) {
+		dev_err(priv->chip.dev, "ARR/CMP registers write issue\n");
+		goto err;
+	}
+	ret = regmap_write(priv->regmap, STM32_LPTIM_ICR,
+			   STM32_LPTIM_CMPOKCF_ARROKCF);
+	if (ret)
+		goto err;
+
+	if (reenable) {
+		/* Start LP timer in continuous mode */
+		ret = regmap_update_bits(priv->regmap, STM32_LPTIM_CR,
+					 STM32_LPTIM_CNTSTRT,
+					 STM32_LPTIM_CNTSTRT);
+		if (ret) {
+			regmap_write(priv->regmap, STM32_LPTIM_CR, 0);
+			goto err;
+		}
+	}
+
+	return 0;
+err:
+	if (!cstate.enabled)
+		clk_disable(priv->clk);
+
+	return ret;
+}
+
+static void stm32_pwm_lp_get_state(struct pwm_chip *chip,
+				   struct pwm_device *pwm,
+				   struct pwm_state *state)
+{
+	struct stm32_pwm_lp *priv = to_stm32_pwm_lp(chip);
+	unsigned long rate = clk_get_rate(priv->clk);
+	u32 val, presc, prd;
+	u64 tmp;
+
+	regmap_read(priv->regmap, STM32_LPTIM_CR, &val);
+	state->enabled = !!FIELD_GET(STM32_LPTIM_ENABLE, val);
+	/* Keep PWM counter clock refcount in sync with PWM initial state */
+	if (state->enabled)
+		clk_enable(priv->clk);
+
+	regmap_read(priv->regmap, STM32_LPTIM_CFGR, &val);
+	presc = FIELD_GET(STM32_LPTIM_PRESC, val);
+	state->polarity = FIELD_GET(STM32_LPTIM_WAVPOL, val);
+
+	regmap_read(priv->regmap, STM32_LPTIM_ARR, &prd);
+	tmp = prd + 1;
+	tmp = (tmp << presc) * NSEC_PER_SEC;
+	state->period = DIV_ROUND_CLOSEST_ULL(tmp, rate);
+
+	regmap_read(priv->regmap, STM32_LPTIM_CMP, &val);
+	tmp = prd - val;
+	tmp = (tmp << presc) * NSEC_PER_SEC;
+	state->duty_cycle = DIV_ROUND_CLOSEST_ULL(tmp, rate);
+}
+
+static const struct pwm_ops stm32_pwm_lp_ops = {
+	.owner = THIS_MODULE,
+	.apply = stm32_pwm_lp_apply,
+	.get_state = stm32_pwm_lp_get_state,
+};
+
+static int stm32_pwm_lp_probe(struct platform_device *pdev)
+{
+	struct stm32_lptimer *ddata = dev_get_drvdata(pdev->dev.parent);
+	struct stm32_pwm_lp *priv;
+	int ret;
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->regmap = ddata->regmap;
+	priv->clk = ddata->clk;
+	priv->chip.base = -1;
+	priv->chip.dev = &pdev->dev;
+	priv->chip.ops = &stm32_pwm_lp_ops;
+	priv->chip.npwm = 1;
+
+	ret = pwmchip_add(&priv->chip);
+	if (ret < 0)
+		return ret;
+
+	platform_set_drvdata(pdev, priv);
+
+	return 0;
+}
+
+static int stm32_pwm_lp_remove(struct platform_device *pdev)
+{
+	struct stm32_pwm_lp *priv = platform_get_drvdata(pdev);
+	unsigned int i;
+
+	for (i = 0; i < priv->chip.npwm; i++)
+		if (pwm_is_enabled(&priv->chip.pwms[i]))
+			pwm_disable(&priv->chip.pwms[i]);
+
+	return pwmchip_remove(&priv->chip);
+}
+
+static const struct of_device_id stm32_pwm_lp_of_match[] = {
+	{ .compatible = "st,stm32-pwm-lp", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, stm32_pwm_lp_of_match);
+
+static struct platform_driver stm32_pwm_lp_driver = {
+	.probe	= stm32_pwm_lp_probe,
+	.remove	= stm32_pwm_lp_remove,
+	.driver	= {
+		.name = "stm32-pwm-lp",
+		.of_match_table = of_match_ptr(stm32_pwm_lp_of_match),
+	},
+};
+module_platform_driver(stm32_pwm_lp_driver);
+
+MODULE_ALIAS("platform:stm32-pwm-lp");
+MODULE_DESCRIPTION("STMicroelectronics STM32 PWM LP driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/pwm/pwm-tegra.c b/drivers/pwm/pwm-tegra.c
index e9b33f09ff09..f8ebbece57b7 100644
--- a/drivers/pwm/pwm-tegra.c
+++ b/drivers/pwm/pwm-tegra.c
@@ -218,7 +218,7 @@ static int tegra_pwm_probe(struct platform_device *pdev)
 	 */
 	pwm->clk_rate = clk_get_rate(pwm->clk);
 
-	pwm->rst = devm_reset_control_get(&pdev->dev, "pwm");
+	pwm->rst = devm_reset_control_get_exclusive(&pdev->dev, "pwm");
 	if (IS_ERR(pwm->rst)) {
 		ret = PTR_ERR(pwm->rst);
 		dev_err(&pdev->dev, "Reset control is not found: %d\n", ret);
diff --git a/drivers/pwm/pwm-tiecap.c b/drivers/pwm/pwm-tiecap.c
index 6ec342dd3eea..34b228626bd5 100644
--- a/drivers/pwm/pwm-tiecap.c
+++ b/drivers/pwm/pwm-tiecap.c
@@ -39,15 +39,15 @@
 #define ECCTL2_TSCTR_FREERUN	BIT(4)
 
 struct ecap_context {
-	u32	cap3;
-	u32	cap4;
-	u16	ecctl2;
+	u32 cap3;
+	u32 cap4;
+	u16 ecctl2;
 };
 
 struct ecap_pwm_chip {
-	struct pwm_chip	chip;
-	unsigned int	clk_rate;
-	void __iomem	*mmio_base;
+	struct pwm_chip chip;
+	unsigned int clk_rate;
+	void __iomem *mmio_base;
 	struct ecap_context ctx;
 };
 
@@ -64,9 +64,9 @@ static int ecap_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 		int duty_ns, int period_ns)
 {
 	struct ecap_pwm_chip *pc = to_ecap_pwm_chip(chip);
+	u32 period_cycles, duty_cycles;
 	unsigned long long c;
-	unsigned long period_cycles, duty_cycles;
-	unsigned int reg_val;
+	u16 value;
 
 	if (period_ns > NSEC_PER_SEC)
 		return -ERANGE;
@@ -74,7 +74,7 @@ static int ecap_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 	c = pc->clk_rate;
 	c = c * period_ns;
 	do_div(c, NSEC_PER_SEC);
-	period_cycles = (unsigned long)c;
+	period_cycles = (u32)c;
 
 	if (period_cycles < 1) {
 		period_cycles = 1;
@@ -83,17 +83,17 @@ static int ecap_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 		c = pc->clk_rate;
 		c = c * duty_ns;
 		do_div(c, NSEC_PER_SEC);
-		duty_cycles = (unsigned long)c;
+		duty_cycles = (u32)c;
 	}
 
 	pm_runtime_get_sync(pc->chip.dev);
 
-	reg_val = readw(pc->mmio_base + ECCTL2);
+	value = readw(pc->mmio_base + ECCTL2);
 
 	/* Configure APWM mode & disable sync option */
-	reg_val |= ECCTL2_APWM_MODE | ECCTL2_SYNC_SEL_DISA;
+	value |= ECCTL2_APWM_MODE | ECCTL2_SYNC_SEL_DISA;
 
-	writew(reg_val, pc->mmio_base + ECCTL2);
+	writew(value, pc->mmio_base + ECCTL2);
 
 	if (!pwm_is_enabled(pwm)) {
 		/* Update active registers if not running */
@@ -110,40 +110,45 @@ static int ecap_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 	}
 
 	if (!pwm_is_enabled(pwm)) {
-		reg_val = readw(pc->mmio_base + ECCTL2);
+		value = readw(pc->mmio_base + ECCTL2);
 		/* Disable APWM mode to put APWM output Low */
-		reg_val &= ~ECCTL2_APWM_MODE;
-		writew(reg_val, pc->mmio_base + ECCTL2);
+		value &= ~ECCTL2_APWM_MODE;
+		writew(value, pc->mmio_base + ECCTL2);
 	}
 
 	pm_runtime_put_sync(pc->chip.dev);
+
 	return 0;
 }
 
 static int ecap_pwm_set_polarity(struct pwm_chip *chip, struct pwm_device *pwm,
-		enum pwm_polarity polarity)
+				 enum pwm_polarity polarity)
 {
 	struct ecap_pwm_chip *pc = to_ecap_pwm_chip(chip);
-	unsigned short reg_val;
+	u16 value;
 
 	pm_runtime_get_sync(pc->chip.dev);
-	reg_val = readw(pc->mmio_base + ECCTL2);
+
+	value = readw(pc->mmio_base + ECCTL2);
+
 	if (polarity == PWM_POLARITY_INVERSED)
 		/* Duty cycle defines LOW period of PWM */
-		reg_val |= ECCTL2_APWM_POL_LOW;
+		value |= ECCTL2_APWM_POL_LOW;
 	else
 		/* Duty cycle defines HIGH period of PWM */
-		reg_val &= ~ECCTL2_APWM_POL_LOW;
+		value &= ~ECCTL2_APWM_POL_LOW;
+
+	writew(value, pc->mmio_base + ECCTL2);
 
-	writew(reg_val, pc->mmio_base + ECCTL2);
 	pm_runtime_put_sync(pc->chip.dev);
+
 	return 0;
 }
 
 static int ecap_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm)
 {
 	struct ecap_pwm_chip *pc = to_ecap_pwm_chip(chip);
-	unsigned int reg_val;
+	u16 value;
 
 	/* Leave clock enabled on enabling PWM */
 	pm_runtime_get_sync(pc->chip.dev);
@@ -152,24 +157,25 @@ static int ecap_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm)
 	 * Enable 'Free run Time stamp counter mode' to start counter
 	 * and  'APWM mode' to enable APWM output
 	 */
-	reg_val = readw(pc->mmio_base + ECCTL2);
-	reg_val |= ECCTL2_TSCTR_FREERUN | ECCTL2_APWM_MODE;
-	writew(reg_val, pc->mmio_base + ECCTL2);
+	value = readw(pc->mmio_base + ECCTL2);
+	value |= ECCTL2_TSCTR_FREERUN | ECCTL2_APWM_MODE;
+	writew(value, pc->mmio_base + ECCTL2);
+
 	return 0;
 }
 
 static void ecap_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm)
 {
 	struct ecap_pwm_chip *pc = to_ecap_pwm_chip(chip);
-	unsigned int reg_val;
+	u16 value;
 
 	/*
 	 * Disable 'Free run Time stamp counter mode' to stop counter
 	 * and 'APWM mode' to put APWM output to low
 	 */
-	reg_val = readw(pc->mmio_base + ECCTL2);
-	reg_val &= ~(ECCTL2_TSCTR_FREERUN | ECCTL2_APWM_MODE);
-	writew(reg_val, pc->mmio_base + ECCTL2);
+	value = readw(pc->mmio_base + ECCTL2);
+	value &= ~(ECCTL2_TSCTR_FREERUN | ECCTL2_APWM_MODE);
+	writew(value, pc->mmio_base + ECCTL2);
 
 	/* Disable clock on PWM disable */
 	pm_runtime_put_sync(pc->chip.dev);
@@ -184,12 +190,12 @@ static void ecap_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm)
 }
 
 static const struct pwm_ops ecap_pwm_ops = {
-	.free		= ecap_pwm_free,
-	.config		= ecap_pwm_config,
-	.set_polarity	= ecap_pwm_set_polarity,
-	.enable		= ecap_pwm_enable,
-	.disable	= ecap_pwm_disable,
-	.owner		= THIS_MODULE,
+	.free = ecap_pwm_free,
+	.config = ecap_pwm_config,
+	.set_polarity = ecap_pwm_set_polarity,
+	.enable = ecap_pwm_enable,
+	.disable = ecap_pwm_disable,
+	.owner = THIS_MODULE,
 };
 
 static const struct of_device_id ecap_of_match[] = {
@@ -202,10 +208,10 @@ MODULE_DEVICE_TABLE(of, ecap_of_match);
 static int ecap_pwm_probe(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node;
-	int ret;
+	struct ecap_pwm_chip *pc;
 	struct resource *r;
 	struct clk *clk;
-	struct ecap_pwm_chip *pc;
+	int ret;
 
 	pc = devm_kzalloc(&pdev->dev, sizeof(*pc), GFP_KERNEL);
 	if (!pc)
@@ -248,9 +254,9 @@ static int ecap_pwm_probe(struct platform_device *pdev)
 		return ret;
 	}
 
+	platform_set_drvdata(pdev, pc);
 	pm_runtime_enable(&pdev->dev);
 
-	platform_set_drvdata(pdev, pc);
 	return 0;
 }
 
@@ -259,6 +265,7 @@ static int ecap_pwm_remove(struct platform_device *pdev)
 	struct ecap_pwm_chip *pc = platform_get_drvdata(pdev);
 
 	pm_runtime_disable(&pdev->dev);
+
 	return pwmchip_remove(&pc->chip);
 }
 
@@ -311,14 +318,13 @@ static SIMPLE_DEV_PM_OPS(ecap_pwm_pm_ops, ecap_pwm_suspend, ecap_pwm_resume);
 
 static struct platform_driver ecap_pwm_driver = {
 	.driver = {
-		.name	= "ecap",
+		.name = "ecap",
 		.of_match_table = ecap_of_match,
-		.pm	= &ecap_pwm_pm_ops,
+		.pm = &ecap_pwm_pm_ops,
 	},
 	.probe = ecap_pwm_probe,
 	.remove = ecap_pwm_remove,
 };
-
 module_platform_driver(ecap_pwm_driver);
 
 MODULE_DESCRIPTION("ECAP PWM driver");
diff --git a/drivers/pwm/pwm-tiehrpwm.c b/drivers/pwm/pwm-tiehrpwm.c
index b5c6b0636893..4c22cb395040 100644
--- a/drivers/pwm/pwm-tiehrpwm.c
+++ b/drivers/pwm/pwm-tiehrpwm.c
@@ -122,12 +122,12 @@ struct ehrpwm_context {
 };
 
 struct ehrpwm_pwm_chip {
-	struct pwm_chip	chip;
-	unsigned int	clk_rate;
-	void __iomem	*mmio_base;
+	struct pwm_chip chip;
+	unsigned long clk_rate;
+	void __iomem *mmio_base;
 	unsigned long period_cycles[NUM_PWM_CHANNEL];
 	enum pwm_polarity polarity[NUM_PWM_CHANNEL];
-	struct	clk	*tbclk;
+	struct clk *tbclk;
 	struct ehrpwm_context ctx;
 };
 
@@ -136,25 +136,26 @@ static inline struct ehrpwm_pwm_chip *to_ehrpwm_pwm_chip(struct pwm_chip *chip)
 	return container_of(chip, struct ehrpwm_pwm_chip, chip);
 }
 
-static inline u16 ehrpwm_read(void __iomem *base, int offset)
+static inline u16 ehrpwm_read(void __iomem *base, unsigned int offset)
 {
 	return readw(base + offset);
 }
 
-static inline void ehrpwm_write(void __iomem *base, int offset, unsigned int val)
+static inline void ehrpwm_write(void __iomem *base, unsigned int offset,
+				u16 value)
 {
-	writew(val & 0xFFFF, base + offset);
+	writew(value, base + offset);
 }
 
-static void ehrpwm_modify(void __iomem *base, int offset,
-		unsigned short mask, unsigned short val)
+static void ehrpwm_modify(void __iomem *base, unsigned int offset, u16 mask,
+			  u16 value)
 {
-	unsigned short regval;
+	unsigned short val;
 
-	regval = readw(base + offset);
-	regval &= ~mask;
-	regval |= val & mask;
-	writew(regval, base + offset);
+	val = readw(base + offset);
+	val &= ~mask;
+	val |= value & mask;
+	writew(val, base + offset);
 }
 
 /**
@@ -163,14 +164,13 @@ static void ehrpwm_modify(void __iomem *base, int offset,
  * @prescale_div:	prescaler value set
  * @tb_clk_div:		Time Base Control prescaler bits
  */
-static int set_prescale_div(unsigned long rqst_prescaler,
-		unsigned short *prescale_div, unsigned short *tb_clk_div)
+static int set_prescale_div(unsigned long rqst_prescaler, u16 *prescale_div,
+			    u16 *tb_clk_div)
 {
 	unsigned int clkdiv, hspclkdiv;
 
 	for (clkdiv = 0; clkdiv <= CLKDIV_MAX; clkdiv++) {
 		for (hspclkdiv = 0; hspclkdiv <= HSPCLKDIV_MAX; hspclkdiv++) {
-
 			/*
 			 * calculations for prescaler value :
 			 * prescale_div = HSPCLKDIVIDER * CLKDIVIDER.
@@ -191,13 +191,14 @@ static int set_prescale_div(unsigned long rqst_prescaler,
 			}
 		}
 	}
+
 	return 1;
 }
 
 static void configure_polarity(struct ehrpwm_pwm_chip *pc, int chan)
 {
-	int aqctl_reg;
-	unsigned short aqctl_val, aqctl_mask;
+	u16 aqctl_val, aqctl_mask;
+	unsigned int aqctl_reg;
 
 	/*
 	 * Configure PWM output to HIGH/LOW level on counter
@@ -232,13 +233,13 @@ static void configure_polarity(struct ehrpwm_pwm_chip *pc, int chan)
  * duty_ns   = 10^9 * (ps_divval * duty_cycles) / PWM_CLK_RATE
  */
 static int ehrpwm_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
-		int duty_ns, int period_ns)
+			     int duty_ns, int period_ns)
 {
 	struct ehrpwm_pwm_chip *pc = to_ehrpwm_pwm_chip(chip);
+	u32 period_cycles, duty_cycles;
+	u16 ps_divval, tb_divval;
+	unsigned int i, cmp_reg;
 	unsigned long long c;
-	unsigned long period_cycles, duty_cycles;
-	unsigned short ps_divval, tb_divval;
-	int i, cmp_reg;
 
 	if (period_ns > NSEC_PER_SEC)
 		return -ERANGE;
@@ -272,8 +273,9 @@ static int ehrpwm_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 			if (i == pwm->hwpwm)
 				continue;
 
-			dev_err(chip->dev, "Period value conflicts with channel %d\n",
-					i);
+			dev_err(chip->dev,
+				"period value conflicts with channel %u\n",
+				i);
 			return -EINVAL;
 		}
 	}
@@ -282,7 +284,7 @@ static int ehrpwm_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 
 	/* Configure clock prescaler to support Low frequency PWM wave */
 	if (set_prescale_div(period_cycles/PERIOD_MAX, &ps_divval,
-				&tb_divval)) {
+			     &tb_divval)) {
 		dev_err(chip->dev, "Unsupported values\n");
 		return -EINVAL;
 	}
@@ -303,7 +305,7 @@ static int ehrpwm_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 
 	/* Configure ehrpwm counter for up-count mode */
 	ehrpwm_modify(pc->mmio_base, TBCTL, TBCTL_CTRMODE_MASK,
-			TBCTL_CTRMODE_UP);
+		      TBCTL_CTRMODE_UP);
 
 	if (pwm->hwpwm == 1)
 		/* Channel 1 configured with compare B register */
@@ -315,23 +317,26 @@ static int ehrpwm_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 	ehrpwm_write(pc->mmio_base, cmp_reg, duty_cycles);
 
 	pm_runtime_put_sync(chip->dev);
+
 	return 0;
 }
 
 static int ehrpwm_pwm_set_polarity(struct pwm_chip *chip,
-		struct pwm_device *pwm,	enum pwm_polarity polarity)
+				   struct pwm_device *pwm,
+				   enum pwm_polarity polarity)
 {
 	struct ehrpwm_pwm_chip *pc = to_ehrpwm_pwm_chip(chip);
 
 	/* Configuration of polarity in hardware delayed, do at enable */
 	pc->polarity[pwm->hwpwm] = polarity;
+
 	return 0;
 }
 
 static int ehrpwm_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm)
 {
 	struct ehrpwm_pwm_chip *pc = to_ehrpwm_pwm_chip(chip);
-	unsigned short aqcsfrc_val, aqcsfrc_mask;
+	u16 aqcsfrc_val, aqcsfrc_mask;
 	int ret;
 
 	/* Leave clock enabled on enabling PWM */
@@ -348,7 +353,7 @@ static int ehrpwm_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm)
 
 	/* Changes to shadow mode */
 	ehrpwm_modify(pc->mmio_base, AQSFRC, AQSFRC_RLDCSF_MASK,
-			AQSFRC_RLDCSF_ZRO);
+		      AQSFRC_RLDCSF_ZRO);
 
 	ehrpwm_modify(pc->mmio_base, AQCSFRC, aqcsfrc_mask, aqcsfrc_val);
 
@@ -358,20 +363,21 @@ static int ehrpwm_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm)
 	/* Enable TBCLK before enabling PWM device */
 	ret = clk_enable(pc->tbclk);
 	if (ret) {
-		dev_err(chip->dev, "Failed to enable TBCLK for %s\n",
-			dev_name(pc->chip.dev));
+		dev_err(chip->dev, "Failed to enable TBCLK for %s: %d\n",
+			dev_name(pc->chip.dev), ret);
 		return ret;
 	}
 
 	/* Enable time counter for free_run */
 	ehrpwm_modify(pc->mmio_base, TBCTL, TBCTL_RUN_MASK, TBCTL_FREE_RUN);
+
 	return 0;
 }
 
 static void ehrpwm_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm)
 {
 	struct ehrpwm_pwm_chip *pc = to_ehrpwm_pwm_chip(chip);
-	unsigned short aqcsfrc_val, aqcsfrc_mask;
+	u16 aqcsfrc_val, aqcsfrc_mask;
 
 	/* Action Qualifier puts PWM output low forcefully */
 	if (pwm->hwpwm) {
@@ -387,7 +393,7 @@ static void ehrpwm_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm)
 	 * Action Qualifier control on PWM output from next TBCLK
 	 */
 	ehrpwm_modify(pc->mmio_base, AQSFRC, AQSFRC_RLDCSF_MASK,
-			AQSFRC_RLDCSF_IMDT);
+		      AQSFRC_RLDCSF_IMDT);
 
 	ehrpwm_modify(pc->mmio_base, AQCSFRC, aqcsfrc_mask, aqcsfrc_val);
 
@@ -415,17 +421,17 @@ static void ehrpwm_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm)
 }
 
 static const struct pwm_ops ehrpwm_pwm_ops = {
-	.free		= ehrpwm_pwm_free,
-	.config		= ehrpwm_pwm_config,
-	.set_polarity	= ehrpwm_pwm_set_polarity,
-	.enable		= ehrpwm_pwm_enable,
-	.disable	= ehrpwm_pwm_disable,
-	.owner		= THIS_MODULE,
+	.free = ehrpwm_pwm_free,
+	.config = ehrpwm_pwm_config,
+	.set_polarity = ehrpwm_pwm_set_polarity,
+	.enable = ehrpwm_pwm_enable,
+	.disable = ehrpwm_pwm_disable,
+	.owner = THIS_MODULE,
 };
 
 static const struct of_device_id ehrpwm_of_match[] = {
-	{ .compatible	= "ti,am3352-ehrpwm" },
-	{ .compatible	= "ti,am33xx-ehrpwm" },
+	{ .compatible = "ti,am3352-ehrpwm" },
+	{ .compatible = "ti,am33xx-ehrpwm" },
 	{},
 };
 MODULE_DEVICE_TABLE(of, ehrpwm_of_match);
@@ -433,10 +439,10 @@ MODULE_DEVICE_TABLE(of, ehrpwm_of_match);
 static int ehrpwm_pwm_probe(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node;
-	int ret;
+	struct ehrpwm_pwm_chip *pc;
 	struct resource *r;
 	struct clk *clk;
-	struct ehrpwm_pwm_chip *pc;
+	int ret;
 
 	pc = devm_kzalloc(&pdev->dev, sizeof(*pc), GFP_KERNEL);
 	if (!pc)
@@ -489,13 +495,18 @@ static int ehrpwm_pwm_probe(struct platform_device *pdev)
 	ret = pwmchip_add(&pc->chip);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "pwmchip_add() failed: %d\n", ret);
-		return ret;
+		goto err_clk_unprepare;
 	}
 
+	platform_set_drvdata(pdev, pc);
 	pm_runtime_enable(&pdev->dev);
 
-	platform_set_drvdata(pdev, pc);
 	return 0;
+
+err_clk_unprepare:
+	clk_unprepare(pc->tbclk);
+
+	return ret;
 }
 
 static int ehrpwm_pwm_remove(struct platform_device *pdev)
@@ -504,8 +515,8 @@ static int ehrpwm_pwm_remove(struct platform_device *pdev)
 
 	clk_unprepare(pc->tbclk);
 
-	pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
+
 	return pwmchip_remove(&pc->chip);
 }
 
@@ -513,6 +524,7 @@ static int ehrpwm_pwm_remove(struct platform_device *pdev)
 static void ehrpwm_pwm_save_context(struct ehrpwm_pwm_chip *pc)
 {
 	pm_runtime_get_sync(pc->chip.dev);
+
 	pc->ctx.tbctl = ehrpwm_read(pc->mmio_base, TBCTL);
 	pc->ctx.tbprd = ehrpwm_read(pc->mmio_base, TBPRD);
 	pc->ctx.cmpa = ehrpwm_read(pc->mmio_base, CMPA);
@@ -521,6 +533,7 @@ static void ehrpwm_pwm_save_context(struct ehrpwm_pwm_chip *pc)
 	pc->ctx.aqctlb = ehrpwm_read(pc->mmio_base, AQCTLB);
 	pc->ctx.aqsfrc = ehrpwm_read(pc->mmio_base, AQSFRC);
 	pc->ctx.aqcsfrc = ehrpwm_read(pc->mmio_base, AQCSFRC);
+
 	pm_runtime_put_sync(pc->chip.dev);
 }
 
@@ -539,9 +552,10 @@ static void ehrpwm_pwm_restore_context(struct ehrpwm_pwm_chip *pc)
 static int ehrpwm_pwm_suspend(struct device *dev)
 {
 	struct ehrpwm_pwm_chip *pc = dev_get_drvdata(dev);
-	int i;
+	unsigned int i;
 
 	ehrpwm_pwm_save_context(pc);
+
 	for (i = 0; i < pc->chip.npwm; i++) {
 		struct pwm_device *pwm = &pc->chip.pwms[i];
 
@@ -551,13 +565,14 @@ static int ehrpwm_pwm_suspend(struct device *dev)
 		/* Disable explicitly if PWM is running */
 		pm_runtime_put_sync(dev);
 	}
+
 	return 0;
 }
 
 static int ehrpwm_pwm_resume(struct device *dev)
 {
 	struct ehrpwm_pwm_chip *pc = dev_get_drvdata(dev);
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < pc->chip.npwm; i++) {
 		struct pwm_device *pwm = &pc->chip.pwms[i];
@@ -568,24 +583,25 @@ static int ehrpwm_pwm_resume(struct device *dev)
 		/* Enable explicitly if PWM was running */
 		pm_runtime_get_sync(dev);
 	}
+
 	ehrpwm_pwm_restore_context(pc);
+
 	return 0;
 }
 #endif
 
 static SIMPLE_DEV_PM_OPS(ehrpwm_pwm_pm_ops, ehrpwm_pwm_suspend,
-		ehrpwm_pwm_resume);
+			 ehrpwm_pwm_resume);
 
 static struct platform_driver ehrpwm_pwm_driver = {
 	.driver = {
-		.name	= "ehrpwm",
+		.name = "ehrpwm",
 		.of_match_table = ehrpwm_of_match,
-		.pm	= &ehrpwm_pwm_pm_ops,
+		.pm = &ehrpwm_pwm_pm_ops,
 	},
 	.probe = ehrpwm_pwm_probe,
 	.remove = ehrpwm_pwm_remove,
 };
-
 module_platform_driver(ehrpwm_pwm_driver);
 
 MODULE_DESCRIPTION("EHRPWM PWM driver");
diff --git a/drivers/pwm/pwm-twl-led.c b/drivers/pwm/pwm-twl-led.c
index 21eff991d0e3..01153622778b 100644
--- a/drivers/pwm/pwm-twl-led.c
+++ b/drivers/pwm/pwm-twl-led.c
@@ -24,7 +24,7 @@
 #include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/pwm.h>
-#include <linux/i2c/twl.h>
+#include <linux/mfd/twl.h>
 #include <linux/slab.h>
 
 /*
diff --git a/drivers/pwm/pwm-twl.c b/drivers/pwm/pwm-twl.c
index 9de617b76680..b7a45be99815 100644
--- a/drivers/pwm/pwm-twl.c
+++ b/drivers/pwm/pwm-twl.c
@@ -21,7 +21,7 @@
 #include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/pwm.h>
-#include <linux/i2c/twl.h>
+#include <linux/mfd/twl.h>
 #include <linux/slab.h>
 
 /*
diff --git a/drivers/pwm/pwm-vt8500.c b/drivers/pwm/pwm-vt8500.c
index 8141a4984126..3a78dd09ac81 100644
--- a/drivers/pwm/pwm-vt8500.c
+++ b/drivers/pwm/pwm-vt8500.c
@@ -241,6 +241,7 @@ static int vt8500_pwm_probe(struct platform_device *pdev)
 	ret = pwmchip_add(&chip->chip);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "failed to add PWM chip\n");
+		clk_unprepare(chip->clk);
 		return ret;
 	}
 
diff --git a/drivers/pwm/pwm-zx.c b/drivers/pwm/pwm-zx.c
new file mode 100644
index 000000000000..5d27c16edfb1
--- /dev/null
+++ b/drivers/pwm/pwm-zx.c
@@ -0,0 +1,282 @@
+/*
+ * Copyright (C) 2017 Sanechips Technology Co., Ltd.
+ * Copyright 2017 Linaro Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pwm.h>
+#include <linux/slab.h>
+
+#define ZX_PWM_MODE		0x0
+#define ZX_PWM_CLKDIV_SHIFT	2
+#define ZX_PWM_CLKDIV_MASK	GENMASK(11, 2)
+#define ZX_PWM_CLKDIV(x)	(((x) << ZX_PWM_CLKDIV_SHIFT) & \
+					 ZX_PWM_CLKDIV_MASK)
+#define ZX_PWM_POLAR		BIT(1)
+#define ZX_PWM_EN		BIT(0)
+#define ZX_PWM_PERIOD		0x4
+#define ZX_PWM_DUTY		0x8
+
+#define ZX_PWM_CLKDIV_MAX	1023
+#define ZX_PWM_PERIOD_MAX	65535
+
+struct zx_pwm_chip {
+	struct pwm_chip chip;
+	struct clk *pclk;
+	struct clk *wclk;
+	void __iomem *base;
+};
+
+static inline struct zx_pwm_chip *to_zx_pwm_chip(struct pwm_chip *chip)
+{
+	return container_of(chip, struct zx_pwm_chip, chip);
+}
+
+static inline u32 zx_pwm_readl(struct zx_pwm_chip *zpc, unsigned int hwpwm,
+			       unsigned int offset)
+{
+	return readl(zpc->base + (hwpwm + 1) * 0x10 + offset);
+}
+
+static inline void zx_pwm_writel(struct zx_pwm_chip *zpc, unsigned int hwpwm,
+				 unsigned int offset, u32 value)
+{
+	writel(value, zpc->base + (hwpwm + 1) * 0x10 + offset);
+}
+
+static void zx_pwm_set_mask(struct zx_pwm_chip *zpc, unsigned int hwpwm,
+			    unsigned int offset, u32 mask, u32 value)
+{
+	u32 data;
+
+	data = zx_pwm_readl(zpc, hwpwm, offset);
+	data &= ~mask;
+	data |= value & mask;
+	zx_pwm_writel(zpc, hwpwm, offset, data);
+}
+
+static void zx_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm,
+			     struct pwm_state *state)
+{
+	struct zx_pwm_chip *zpc = to_zx_pwm_chip(chip);
+	unsigned long rate;
+	unsigned int div;
+	u32 value;
+	u64 tmp;
+
+	value = zx_pwm_readl(zpc, pwm->hwpwm, ZX_PWM_MODE);
+
+	if (value & ZX_PWM_POLAR)
+		state->polarity = PWM_POLARITY_NORMAL;
+	else
+		state->polarity = PWM_POLARITY_INVERSED;
+
+	if (value & ZX_PWM_EN)
+		state->enabled = true;
+	else
+		state->enabled = false;
+
+	div = (value & ZX_PWM_CLKDIV_MASK) >> ZX_PWM_CLKDIV_SHIFT;
+	rate = clk_get_rate(zpc->wclk);
+
+	tmp = zx_pwm_readl(zpc, pwm->hwpwm, ZX_PWM_PERIOD);
+	tmp *= div * NSEC_PER_SEC;
+	state->period = DIV_ROUND_CLOSEST_ULL(tmp, rate);
+
+	tmp = zx_pwm_readl(zpc, pwm->hwpwm, ZX_PWM_DUTY);
+	tmp *= div * NSEC_PER_SEC;
+	state->duty_cycle = DIV_ROUND_CLOSEST_ULL(tmp, rate);
+}
+
+static int zx_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
+			 unsigned int duty_ns, unsigned int period_ns)
+{
+	struct zx_pwm_chip *zpc = to_zx_pwm_chip(chip);
+	unsigned int period_cycles, duty_cycles;
+	unsigned long long c;
+	unsigned int div = 1;
+	unsigned long rate;
+
+	/* Find out the best divider */
+	rate = clk_get_rate(zpc->wclk);
+
+	while (1) {
+		c = rate / div;
+		c = c * period_ns;
+		do_div(c, NSEC_PER_SEC);
+
+		if (c < ZX_PWM_PERIOD_MAX)
+			break;
+
+		div++;
+
+		if (div > ZX_PWM_CLKDIV_MAX)
+			return -ERANGE;
+	}
+
+	/* Calculate duty cycles */
+	period_cycles = c;
+	c *= duty_ns;
+	do_div(c, period_ns);
+	duty_cycles = c;
+
+	/*
+	 * If the PWM is being enabled, we have to temporarily disable it
+	 * before configuring the registers.
+	 */
+	if (pwm_is_enabled(pwm))
+		zx_pwm_set_mask(zpc, pwm->hwpwm, ZX_PWM_MODE, ZX_PWM_EN, 0);
+
+	/* Set up registers */
+	zx_pwm_set_mask(zpc, pwm->hwpwm, ZX_PWM_MODE, ZX_PWM_CLKDIV_MASK,
+			ZX_PWM_CLKDIV(div));
+	zx_pwm_writel(zpc, pwm->hwpwm, ZX_PWM_PERIOD, period_cycles);
+	zx_pwm_writel(zpc, pwm->hwpwm, ZX_PWM_DUTY, duty_cycles);
+
+	/* Re-enable the PWM if needed */
+	if (pwm_is_enabled(pwm))
+		zx_pwm_set_mask(zpc, pwm->hwpwm, ZX_PWM_MODE,
+				ZX_PWM_EN, ZX_PWM_EN);
+
+	return 0;
+}
+
+static int zx_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm,
+			struct pwm_state *state)
+{
+	struct zx_pwm_chip *zpc = to_zx_pwm_chip(chip);
+	struct pwm_state cstate;
+	int ret;
+
+	pwm_get_state(pwm, &cstate);
+
+	if (state->polarity != cstate.polarity)
+		zx_pwm_set_mask(zpc, pwm->hwpwm, ZX_PWM_MODE, ZX_PWM_POLAR,
+				(state->polarity == PWM_POLARITY_INVERSED) ?
+				 0 : ZX_PWM_POLAR);
+
+	if (state->period != cstate.period ||
+	    state->duty_cycle != cstate.duty_cycle) {
+		ret = zx_pwm_config(chip, pwm, state->duty_cycle,
+				    state->period);
+		if (ret)
+			return ret;
+	}
+
+	if (state->enabled != cstate.enabled) {
+		if (state->enabled) {
+			ret = clk_prepare_enable(zpc->wclk);
+			if (ret)
+				return ret;
+
+			zx_pwm_set_mask(zpc, pwm->hwpwm, ZX_PWM_MODE,
+					ZX_PWM_EN, ZX_PWM_EN);
+		} else {
+			zx_pwm_set_mask(zpc, pwm->hwpwm, ZX_PWM_MODE,
+					ZX_PWM_EN, 0);
+			clk_disable_unprepare(zpc->wclk);
+		}
+	}
+
+	return 0;
+}
+
+static const struct pwm_ops zx_pwm_ops = {
+	.apply = zx_pwm_apply,
+	.get_state = zx_pwm_get_state,
+	.owner = THIS_MODULE,
+};
+
+static int zx_pwm_probe(struct platform_device *pdev)
+{
+	struct zx_pwm_chip *zpc;
+	struct resource *res;
+	unsigned int i;
+	int ret;
+
+	zpc = devm_kzalloc(&pdev->dev, sizeof(*zpc), GFP_KERNEL);
+	if (!zpc)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	zpc->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(zpc->base))
+		return PTR_ERR(zpc->base);
+
+	zpc->pclk = devm_clk_get(&pdev->dev, "pclk");
+	if (IS_ERR(zpc->pclk))
+		return PTR_ERR(zpc->pclk);
+
+	zpc->wclk = devm_clk_get(&pdev->dev, "wclk");
+	if (IS_ERR(zpc->wclk))
+		return PTR_ERR(zpc->wclk);
+
+	ret = clk_prepare_enable(zpc->pclk);
+	if (ret)
+		return ret;
+
+	zpc->chip.dev = &pdev->dev;
+	zpc->chip.ops = &zx_pwm_ops;
+	zpc->chip.base = -1;
+	zpc->chip.npwm = 4;
+	zpc->chip.of_xlate = of_pwm_xlate_with_flags;
+	zpc->chip.of_pwm_n_cells = 3;
+
+	/*
+	 * PWM devices may be enabled by firmware, and let's disable all of
+	 * them initially to save power.
+	 */
+	for (i = 0; i < zpc->chip.npwm; i++)
+		zx_pwm_set_mask(zpc, i, ZX_PWM_MODE, ZX_PWM_EN, 0);
+
+	ret = pwmchip_add(&zpc->chip);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "failed to add PWM chip: %d\n", ret);
+		return ret;
+	}
+
+	platform_set_drvdata(pdev, zpc);
+
+	return 0;
+}
+
+static int zx_pwm_remove(struct platform_device *pdev)
+{
+	struct zx_pwm_chip *zpc = platform_get_drvdata(pdev);
+	int ret;
+
+	ret = pwmchip_remove(&zpc->chip);
+	clk_disable_unprepare(zpc->pclk);
+
+	return ret;
+}
+
+static const struct of_device_id zx_pwm_dt_ids[] = {
+	{ .compatible = "zte,zx296718-pwm", },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, zx_pwm_dt_ids);
+
+static struct platform_driver zx_pwm_driver = {
+	.driver = {
+		.name = "zx-pwm",
+		.of_match_table = zx_pwm_dt_ids,
+	},
+	.probe = zx_pwm_probe,
+	.remove = zx_pwm_remove,
+};
+module_platform_driver(zx_pwm_driver);
+
+MODULE_ALIAS("platform:zx-pwm");
+MODULE_AUTHOR("Shawn Guo <shawn.guo@linaro.org>");
+MODULE_DESCRIPTION("ZTE ZX PWM Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/regulator/Kconfig b/drivers/regulator/Kconfig
index 99b9362331b5..0fd6195601ba 100644
--- a/drivers/regulator/Kconfig
+++ b/drivers/regulator/Kconfig
@@ -559,6 +559,15 @@ config REGULATOR_MT6323
 	  This driver supports the control of different power rails of device
 	  through regulator interface.
 
+config REGULATOR_MT6380
+	tristate "MediaTek MT6380 PMIC"
+	depends on MTK_PMIC_WRAP
+	help
+	  Say y here to select this option to enable the power regulator of
+	  MediaTek MT6380 PMIC.
+	  This driver supports the control of different power rails of device
+	  through regulator interface.
+
 config REGULATOR_MT6397
 	tristate "MediaTek MT6397 PMIC"
 	depends on MFD_MT6397
@@ -687,11 +696,11 @@ config REGULATOR_RC5T583
 	  outputs which can be controlled by i2c communication.
 
 config REGULATOR_RK808
-	tristate "Rockchip RK808/RK818 Power regulators"
+	tristate "Rockchip RK805/RK808/RK818 Power regulators"
 	depends on MFD_RK808
 	help
 	  Select this option to enable the power regulator of ROCKCHIP
-	  PMIC RK808 and RK818.
+	  PMIC RK805,RK808 and RK818.
 	  This driver supports the control of different power rails of device
 	  through regulator interface. The device supports multiple DCDC/LDO
 	  outputs which can be controlled by i2c communication.
@@ -700,8 +709,8 @@ config REGULATOR_RN5T618
 	tristate "Ricoh RN5T567/618 voltage regulators"
 	depends on MFD_RN5T618
 	help
-	  Say y here to support the regulators found on Ricoh RN5T567 or
-	  RN5T618 PMIC.
+	  Say y here to support the regulators found on Ricoh RN5T567,
+	  RN5T618 or RC5T619 PMIC.
 
 config REGULATOR_RT5033
 	tristate "Richtek RT5033 Regulators"
@@ -746,6 +755,18 @@ config REGULATOR_SKY81452
 	  This driver can also be built as a module. If so, the module
 	  will be called sky81452-regulator.
 
+config REGULATOR_STM32_VREFBUF
+	tristate "STMicroelectronics STM32 VREFBUF"
+	depends on ARCH_STM32 || COMPILE_TEST
+	help
+	  This driver supports STMicroelectronics STM32 VREFBUF (voltage
+	  reference buffer) which can be used as voltage reference for
+	  internal ADCs, DACs and also for external components through
+	  dedicated Vref+ pin.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called stm32-vrefbuf.
+
 config REGULATOR_TI_ABB
 	tristate "TI Adaptive Body Bias on-chip LDO"
 	depends on ARCH_OMAP
diff --git a/drivers/regulator/Makefile b/drivers/regulator/Makefile
index 95b1e86ae692..cbb6e45c77b2 100644
--- a/drivers/regulator/Makefile
+++ b/drivers/regulator/Makefile
@@ -72,6 +72,7 @@ obj-$(CONFIG_REGULATOR_MC13892) += mc13892-regulator.o
 obj-$(CONFIG_REGULATOR_MC13XXX_CORE) +=  mc13xxx-regulator-core.o
 obj-$(CONFIG_REGULATOR_MT6311) += mt6311-regulator.o
 obj-$(CONFIG_REGULATOR_MT6323)	+= mt6323-regulator.o
+obj-$(CONFIG_REGULATOR_MT6380)	+= mt6380-regulator.o
 obj-$(CONFIG_REGULATOR_MT6397)	+= mt6397-regulator.o
 obj-$(CONFIG_REGULATOR_QCOM_RPM) += qcom_rpm-regulator.o
 obj-$(CONFIG_REGULATOR_QCOM_SMD_RPM) += qcom_smd-regulator.o
@@ -94,6 +95,7 @@ obj-$(CONFIG_REGULATOR_S2MPA01) += s2mpa01.o
 obj-$(CONFIG_REGULATOR_S2MPS11) += s2mps11.o
 obj-$(CONFIG_REGULATOR_S5M8767) += s5m8767.o
 obj-$(CONFIG_REGULATOR_SKY81452) += sky81452-regulator.o
+obj-$(CONFIG_REGULATOR_STM32_VREFBUF) += stm32-vrefbuf.o
 obj-$(CONFIG_REGULATOR_STW481X_VMMC) += stw481x-vmmc.o
 obj-$(CONFIG_REGULATOR_TI_ABB) += ti-abb-regulator.o
 obj-$(CONFIG_REGULATOR_TPS6105X) += tps6105x-regulator.o
diff --git a/drivers/regulator/axp20x-regulator.c b/drivers/regulator/axp20x-regulator.c
index e2608fe770b9..f18b36dd57dd 100644
--- a/drivers/regulator/axp20x-regulator.c
+++ b/drivers/regulator/axp20x-regulator.c
@@ -691,6 +691,9 @@ static int axp20x_regulator_probe(struct platform_device *pdev)
 		    (regulators == axp809_regulators && i == AXP809_DC1SW)) {
 			new_desc = devm_kzalloc(&pdev->dev, sizeof(*desc),
 						GFP_KERNEL);
+			if (!new_desc)
+				return -ENOMEM;
+
 			*new_desc = regulators[i];
 			new_desc->supply_name = dcdc1_name;
 			desc = new_desc;
@@ -700,6 +703,9 @@ static int axp20x_regulator_probe(struct platform_device *pdev)
 		    (regulators == axp809_regulators && i == AXP809_DC5LDO)) {
 			new_desc = devm_kzalloc(&pdev->dev, sizeof(*desc),
 						GFP_KERNEL);
+			if (!new_desc)
+				return -ENOMEM;
+
 			*new_desc = regulators[i];
 			new_desc->supply_name = dcdc5_name;
 			desc = new_desc;
diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c
index e567fa54980b..b64b7916507f 100644
--- a/drivers/regulator/core.c
+++ b/drivers/regulator/core.c
@@ -204,8 +204,8 @@ static struct device_node *of_get_regulator(struct device *dev, const char *supp
 	regnode = of_parse_phandle(dev->of_node, prop_name, 0);
 
 	if (!regnode) {
-		dev_dbg(dev, "Looking up %s property in node %s failed\n",
-				prop_name, dev->of_node->full_name);
+		dev_dbg(dev, "Looking up %s property in node %pOF failed\n",
+				prop_name, dev->of_node);
 		return NULL;
 	}
 	return regnode;
@@ -2396,6 +2396,14 @@ static void regulator_disable_work(struct work_struct *work)
 	count = rdev->deferred_disables;
 	rdev->deferred_disables = 0;
 
+	/*
+	 * Workqueue functions queue the new work instance while the previous
+	 * work instance is being processed. Cancel the queued work instance
+	 * as the work instance under processing does the job of the queued
+	 * work instance.
+	 */
+	cancel_delayed_work(&rdev->disable_work);
+
 	for (i = 0; i < count; i++) {
 		ret = _regulator_disable(rdev);
 		if (ret != 0)
@@ -2439,10 +2447,10 @@ int regulator_disable_deferred(struct regulator *regulator, int ms)
 
 	mutex_lock(&rdev->mutex);
 	rdev->deferred_disables++;
+	mod_delayed_work(system_power_efficient_wq, &rdev->disable_work,
+			 msecs_to_jiffies(ms));
 	mutex_unlock(&rdev->mutex);
 
-	queue_delayed_work(system_power_efficient_wq, &rdev->disable_work,
-			   msecs_to_jiffies(ms));
 	return 0;
 }
 EXPORT_SYMBOL_GPL(regulator_disable_deferred);
diff --git a/drivers/regulator/cpcap-regulator.c b/drivers/regulator/cpcap-regulator.c
index cc98aceed1c1..f541b80f1b54 100644
--- a/drivers/regulator/cpcap-regulator.c
+++ b/drivers/regulator/cpcap-regulator.c
@@ -77,6 +77,8 @@
 #define CPCAP_BIT_VAUDIO_MODE0		BIT(1)
 #define CPCAP_BIT_V_AUDIO_EN		BIT(0)
 
+#define CPCAP_BIT_AUDIO_NORMAL_MODE	0x00
+
 /*
  * Off mode configuration bit. Used currently only by SW5 on omap4. There's
  * the following comment in Motorola Linux kernel tree for it:
@@ -121,6 +123,7 @@ struct cpcap_regulator {
 		.enable_val = (mode_val),				\
 		.disable_val = (off_val),				\
 		.ramp_delay = (volt_trans_time),			\
+		.of_map_mode = cpcap_map_mode,				\
 	},								\
 	.assign_reg = (assignment_reg),					\
 	.assign_mask = (assignment_mask),				\
@@ -211,13 +214,25 @@ static int cpcap_regulator_disable(struct regulator_dev *rdev)
 	return error;
 }
 
+static unsigned int cpcap_map_mode(unsigned int mode)
+{
+	switch (mode) {
+	case CPCAP_BIT_AUDIO_NORMAL_MODE:
+		return REGULATOR_MODE_NORMAL;
+	case CPCAP_BIT_AUDIO_LOW_PWR:
+		return REGULATOR_MODE_STANDBY;
+	default:
+		return -EINVAL;
+	}
+}
+
 static unsigned int cpcap_regulator_get_mode(struct regulator_dev *rdev)
 {
 	int value;
 
 	regmap_read(rdev->regmap, rdev->desc->enable_reg, &value);
 
-	if (!(value & CPCAP_BIT_AUDIO_LOW_PWR))
+	if (value & CPCAP_BIT_AUDIO_LOW_PWR)
 		return REGULATOR_MODE_STANDBY;
 
 	return REGULATOR_MODE_NORMAL;
@@ -230,10 +245,10 @@ static int cpcap_regulator_set_mode(struct regulator_dev *rdev,
 
 	switch (mode) {
 	case REGULATOR_MODE_NORMAL:
-		value = CPCAP_BIT_AUDIO_LOW_PWR;
+		value = CPCAP_BIT_AUDIO_NORMAL_MODE;
 		break;
 	case REGULATOR_MODE_STANDBY:
-		value = 0;
+		value = CPCAP_BIT_AUDIO_LOW_PWR;
 		break;
 	default:
 		return -EINVAL;
diff --git a/drivers/regulator/da9063-regulator.c b/drivers/regulator/da9063-regulator.c
index c6af343f54ea..6a8f9cd69f52 100644
--- a/drivers/regulator/da9063-regulator.c
+++ b/drivers/regulator/da9063-regulator.c
@@ -736,7 +736,7 @@ static int da9063_regulator_probe(struct platform_device *pdev)
 	if (IS_ERR(regl_pdata) || regl_pdata->n_regulators == 0) {
 		dev_err(&pdev->dev,
 			"No regulators defined for the platform\n");
-		return PTR_ERR(regl_pdata);
+		return -ENODEV;
 	}
 
 	/* Find regulators set for particular device model */
diff --git a/drivers/regulator/fan53555.c b/drivers/regulator/fan53555.c
index 60f431831582..a3bc8037153e 100644
--- a/drivers/regulator/fan53555.c
+++ b/drivers/regulator/fan53555.c
@@ -407,14 +407,8 @@ static int fan53555_regulator_probe(struct i2c_client *client,
 
 	di->regulator = pdata->regulator;
 	if (client->dev.of_node) {
-		const struct of_device_id *match;
-
-		match = of_match_device(of_match_ptr(fan53555_dt_ids),
-					&client->dev);
-		if (!match)
-			return -ENODEV;
-
-		di->vendor = (unsigned long) match->data;
+		di->vendor =
+			(unsigned long)of_device_get_match_data(&client->dev);
 	} else {
 		/* if no ramp constraint set, get the pdata ramp_delay */
 		if (!di->regulator->constraints.ramp_delay) {
@@ -476,7 +470,10 @@ static const struct i2c_device_id fan53555_id[] = {
 		.name = "fan53555",
 		.driver_data = FAN53555_VENDOR_FAIRCHILD
 	}, {
-		.name = "syr82x",
+		.name = "syr827",
+		.driver_data = FAN53555_VENDOR_SILERGY
+	}, {
+		.name = "syr828",
 		.driver_data = FAN53555_VENDOR_SILERGY
 	},
 	{ },
diff --git a/drivers/regulator/ltc3589.c b/drivers/regulator/ltc3589.c
index 853a06ad86d6..18d5b01ddcb2 100644
--- a/drivers/regulator/ltc3589.c
+++ b/drivers/regulator/ltc3589.c
@@ -539,7 +539,7 @@ static int ltc3589_probe(struct i2c_client *client,
 	return 0;
 }
 
-static struct i2c_device_id ltc3589_i2c_id[] = {
+static const struct i2c_device_id ltc3589_i2c_id[] = {
 	{ "ltc3589",   LTC3589   },
 	{ "ltc3589-1", LTC3589_1 },
 	{ "ltc3589-2", LTC3589_2 },
diff --git a/drivers/regulator/max1586.c b/drivers/regulator/max1586.c
index 6779c2b53674..66bbaa999433 100644
--- a/drivers/regulator/max1586.c
+++ b/drivers/regulator/max1586.c
@@ -169,7 +169,7 @@ static int of_get_max1586_platform_data(struct device *dev,
 
 	if (of_property_read_u32(np, "v3-gain",
 				 &pdata->v3_gain) < 0) {
-		dev_err(dev, "%s has no 'v3-gain' property\n", np->full_name);
+		dev_err(dev, "%pOF has no 'v3-gain' property\n", np);
 		return -EINVAL;
 	}
 
diff --git a/drivers/regulator/mt6380-regulator.c b/drivers/regulator/mt6380-regulator.c
new file mode 100644
index 000000000000..127dd720cbcc
--- /dev/null
+++ b/drivers/regulator/mt6380-regulator.c
@@ -0,0 +1,352 @@
+/*
+ * Copyright (c) 2017 MediaTek Inc.
+ * Author: Chenglin Xu <chenglin.xu@mediatek.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/regulator/driver.h>
+#include <linux/regulator/machine.h>
+#include <linux/regulator/mt6380-regulator.h>
+#include <linux/regulator/of_regulator.h>
+
+/* PMIC Registers */
+#define MT6380_ALDO_CON_0                         0x0000
+#define MT6380_BTLDO_CON_0                        0x0004
+#define MT6380_COMP_CON_0                         0x0008
+#define MT6380_CPUBUCK_CON_0                      0x000C
+#define MT6380_CPUBUCK_CON_1                      0x0010
+#define MT6380_CPUBUCK_CON_2                      0x0014
+#define MT6380_DDRLDO_CON_0                       0x0018
+#define MT6380_MLDO_CON_0                         0x001C
+#define MT6380_PALDO_CON_0                        0x0020
+#define MT6380_PHYLDO_CON_0                       0x0024
+#define MT6380_SIDO_CON_0                         0x0028
+#define MT6380_SIDO_CON_1                         0x002C
+#define MT6380_SIDO_CON_2                         0x0030
+#define MT6380_SLDO_CON_0                         0x0034
+#define MT6380_TLDO_CON_0                         0x0038
+#define MT6380_STARTUP_CON_0                      0x003C
+#define MT6380_STARTUP_CON_1                      0x0040
+#define MT6380_SMPS_TOP_CON_0                     0x0044
+#define MT6380_SMPS_TOP_CON_1                     0x0048
+#define MT6380_ANA_CTRL_0                         0x0050
+#define MT6380_ANA_CTRL_1                         0x0054
+#define MT6380_ANA_CTRL_2                         0x0058
+#define MT6380_ANA_CTRL_3                         0x005C
+#define MT6380_ANA_CTRL_4                         0x0060
+#define MT6380_SPK_CON9                           0x0064
+#define MT6380_SPK_CON11                          0x0068
+#define MT6380_SPK_CON12                          0x006A
+#define MT6380_CLK_CTRL                           0x0070
+#define MT6380_PINMUX_CTRL                        0x0074
+#define MT6380_IO_CTRL                            0x0078
+#define MT6380_SLP_MODE_CTRL_0                    0x007C
+#define MT6380_SLP_MODE_CTRL_1                    0x0080
+#define MT6380_SLP_MODE_CTRL_2                    0x0084
+#define MT6380_SLP_MODE_CTRL_3                    0x0088
+#define MT6380_SLP_MODE_CTRL_4                    0x008C
+#define MT6380_SLP_MODE_CTRL_5                    0x0090
+#define MT6380_SLP_MODE_CTRL_6                    0x0094
+#define MT6380_SLP_MODE_CTRL_7                    0x0098
+#define MT6380_SLP_MODE_CTRL_8                    0x009C
+#define MT6380_FCAL_CTRL_0                        0x00A0
+#define MT6380_FCAL_CTRL_1                        0x00A4
+#define MT6380_LDO_CTRL_0                         0x00A8
+#define MT6380_LDO_CTRL_1                         0x00AC
+#define MT6380_LDO_CTRL_2                         0x00B0
+#define MT6380_LDO_CTRL_3                         0x00B4
+#define MT6380_LDO_CTRL_4                         0x00B8
+#define MT6380_DEBUG_CTRL_0                       0x00BC
+#define MT6380_EFU_CTRL_0                         0x0200
+#define MT6380_EFU_CTRL_1                         0x0201
+#define MT6380_EFU_CTRL_2                         0x0202
+#define MT6380_EFU_CTRL_3                         0x0203
+#define MT6380_EFU_CTRL_4                         0x0204
+#define MT6380_EFU_CTRL_5                         0x0205
+#define MT6380_EFU_CTRL_6                         0x0206
+#define MT6380_EFU_CTRL_7                         0x0207
+#define MT6380_EFU_CTRL_8                         0x0208
+
+#define MT6380_REGULATOR_MODE_AUTO	0
+#define MT6380_REGULATOR_MODE_FORCE_PWM	1
+
+/*
+ * mt6380 regulators' information
+ *
+ * @desc: standard fields of regulator description
+ * @vselon_reg: Register sections for hardware control mode of bucks
+ * @modeset_reg: Register for controlling the buck/LDO control mode
+ * @modeset_mask: Mask for controlling the buck/LDO control mode
+ */
+struct mt6380_regulator_info {
+	struct regulator_desc desc;
+	u32 vselon_reg;
+	u32 modeset_reg;
+	u32 modeset_mask;
+};
+
+#define MT6380_BUCK(match, vreg, min, max, step, volt_ranges, enreg,	\
+		    vosel, vosel_mask, enbit, voselon, _modeset_reg,	\
+		    _modeset_mask)					\
+[MT6380_ID_##vreg] = {							\
+	.desc = {							\
+		.name = #vreg,						\
+		.of_match = of_match_ptr(match),			\
+		.ops = &mt6380_volt_range_ops,				\
+		.type = REGULATOR_VOLTAGE,				\
+		.id = MT6380_ID_##vreg,					\
+		.owner = THIS_MODULE,					\
+		.n_voltages = ((max) - (min)) / (step) + 1,		\
+		.linear_ranges = volt_ranges,				\
+		.n_linear_ranges = ARRAY_SIZE(volt_ranges),		\
+		.vsel_reg = vosel,					\
+		.vsel_mask = vosel_mask,				\
+		.enable_reg = enreg,					\
+		.enable_mask = BIT(enbit),				\
+	},								\
+	.vselon_reg = voselon,						\
+	.modeset_reg = _modeset_reg,					\
+	.modeset_mask = _modeset_mask,					\
+}
+
+#define MT6380_LDO(match, vreg, ldo_volt_table, enreg, enbit, vosel,	\
+		   vosel_mask, _modeset_reg, _modeset_mask)		\
+[MT6380_ID_##vreg] = {							\
+	.desc = {							\
+		.name = #vreg,						\
+		.of_match = of_match_ptr(match),			\
+		.ops = &mt6380_volt_table_ops,				\
+		.type = REGULATOR_VOLTAGE,				\
+		.id = MT6380_ID_##vreg,					\
+		.owner = THIS_MODULE,					\
+		.n_voltages = ARRAY_SIZE(ldo_volt_table),		\
+		.volt_table = ldo_volt_table,				\
+		.vsel_reg = vosel,					\
+		.vsel_mask = vosel_mask,				\
+		.enable_reg = enreg,					\
+		.enable_mask = BIT(enbit),				\
+	},								\
+	.modeset_reg = _modeset_reg,					\
+	.modeset_mask = _modeset_mask,					\
+}
+
+#define MT6380_REG_FIXED(match, vreg, enreg, enbit, volt,		\
+			 _modeset_reg, _modeset_mask)			\
+[MT6380_ID_##vreg] = {							\
+	.desc = {							\
+		.name = #vreg,						\
+		.of_match = of_match_ptr(match),			\
+		.ops = &mt6380_volt_fixed_ops,				\
+		.type = REGULATOR_VOLTAGE,				\
+		.id = MT6380_ID_##vreg,					\
+		.owner = THIS_MODULE,					\
+		.n_voltages = 1,					\
+		.enable_reg = enreg,					\
+		.enable_mask = BIT(enbit),				\
+		.min_uV = volt,						\
+	},								\
+	.modeset_reg = _modeset_reg,					\
+	.modeset_mask = _modeset_mask,					\
+}
+
+static const struct regulator_linear_range buck_volt_range1[] = {
+	REGULATOR_LINEAR_RANGE(600000, 0, 0xfe, 6250),
+};
+
+static const struct regulator_linear_range buck_volt_range2[] = {
+	REGULATOR_LINEAR_RANGE(600000, 0, 0xfe, 6250),
+};
+
+static const struct regulator_linear_range buck_volt_range3[] = {
+	REGULATOR_LINEAR_RANGE(1200000, 0, 0x3c, 25000),
+};
+
+static const u32 ldo_volt_table1[] = {
+	1400000, 1350000, 1300000, 1250000, 1200000, 1150000, 1100000, 1050000,
+};
+
+static const u32 ldo_volt_table2[] = {
+	2200000, 3300000,
+};
+
+static const u32 ldo_volt_table3[] = {
+	1240000, 1390000, 1540000, 1840000,
+};
+
+static const u32 ldo_volt_table4[] = {
+	2200000, 3300000,
+};
+
+static int mt6380_regulator_set_mode(struct regulator_dev *rdev,
+				     unsigned int mode)
+{
+	int ret, val = 0;
+	struct mt6380_regulator_info *info = rdev_get_drvdata(rdev);
+
+	switch (mode) {
+	case REGULATOR_MODE_NORMAL:
+		val = MT6380_REGULATOR_MODE_AUTO;
+		break;
+	case REGULATOR_MODE_FAST:
+		val = MT6380_REGULATOR_MODE_FORCE_PWM;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	val <<= ffs(info->modeset_mask) - 1;
+
+	ret = regmap_update_bits(rdev->regmap, info->modeset_reg,
+				 info->modeset_mask, val);
+
+	return ret;
+}
+
+static unsigned int mt6380_regulator_get_mode(struct regulator_dev *rdev)
+{
+	unsigned int val;
+	unsigned int mode;
+	int ret;
+	struct mt6380_regulator_info *info = rdev_get_drvdata(rdev);
+
+	ret = regmap_read(rdev->regmap, info->modeset_reg, &val);
+	if (ret < 0)
+		return ret;
+
+	val &= info->modeset_mask;
+	val >>= ffs(info->modeset_mask) - 1;
+
+	switch (val) {
+	case MT6380_REGULATOR_MODE_AUTO:
+		mode = REGULATOR_MODE_NORMAL;
+		break;
+	case MT6380_REGULATOR_MODE_FORCE_PWM:
+		mode = REGULATOR_MODE_FAST;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return mode;
+}
+
+static const struct regulator_ops mt6380_volt_range_ops = {
+	.list_voltage = regulator_list_voltage_linear_range,
+	.map_voltage = regulator_map_voltage_linear_range,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.set_voltage_time_sel = regulator_set_voltage_time_sel,
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.set_mode = mt6380_regulator_set_mode,
+	.get_mode = mt6380_regulator_get_mode,
+};
+
+static const struct regulator_ops mt6380_volt_table_ops = {
+	.list_voltage = regulator_list_voltage_table,
+	.map_voltage = regulator_map_voltage_iterate,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.set_voltage_time_sel = regulator_set_voltage_time_sel,
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.set_mode = mt6380_regulator_set_mode,
+	.get_mode = mt6380_regulator_get_mode,
+};
+
+static const struct regulator_ops mt6380_volt_fixed_ops = {
+	.list_voltage = regulator_list_voltage_linear,
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.set_mode = mt6380_regulator_set_mode,
+	.get_mode = mt6380_regulator_get_mode,
+};
+
+/* The array is indexed by id(MT6380_ID_XXX) */
+static struct mt6380_regulator_info mt6380_regulators[] = {
+	MT6380_BUCK("buck-vcore1", VCPU, 600000, 1393750, 6250,
+		    buck_volt_range1, MT6380_ANA_CTRL_3, MT6380_ANA_CTRL_1,
+		    0xfe, 3, MT6380_ANA_CTRL_1,
+		    MT6380_CPUBUCK_CON_0, 0x8000000),
+	MT6380_BUCK("buck-vcore", VCORE, 600000, 1393750, 6250,
+		    buck_volt_range2, MT6380_ANA_CTRL_3, MT6380_ANA_CTRL_2,
+		    0xfe, 2, MT6380_ANA_CTRL_2, MT6380_SIDO_CON_0, 0x1000000),
+	MT6380_BUCK("buck-vrf", VRF, 1200000, 1575000, 25000,
+		    buck_volt_range3, MT6380_ANA_CTRL_3, MT6380_SIDO_CON_0,
+		    0x78, 1, MT6380_SIDO_CON_0, MT6380_SIDO_CON_0, 0x8000),
+	MT6380_LDO("ldo-vm", VMLDO, ldo_volt_table1, MT6380_LDO_CTRL_0,
+		   1, MT6380_MLDO_CON_0, 0xE000, MT6380_ANA_CTRL_1, 0x4000000),
+	MT6380_LDO("ldo-va", VALDO, ldo_volt_table2, MT6380_LDO_CTRL_0,
+		   2, MT6380_ALDO_CON_0, 0x400, MT6380_ALDO_CON_0, 0x20),
+	MT6380_REG_FIXED("ldo-vphy", VPHYLDO, MT6380_LDO_CTRL_0, 7, 1800000,
+			 MT6380_PHYLDO_CON_0, 0x80),
+	MT6380_LDO("ldo-vddr", VDDRLDO, ldo_volt_table3, MT6380_LDO_CTRL_0,
+		   8, MT6380_DDRLDO_CON_0, 0x3000, MT6380_DDRLDO_CON_0, 0x80),
+	MT6380_LDO("ldo-vt", VTLDO, ldo_volt_table4, MT6380_LDO_CTRL_0, 3,
+		   MT6380_TLDO_CON_0, 0x400, MT6380_TLDO_CON_0, 0x20),
+};
+
+static int mt6380_regulator_probe(struct platform_device *pdev)
+{
+	struct regmap *regmap = dev_get_regmap(pdev->dev.parent, NULL);
+	struct regulator_config config = {};
+	struct regulator_dev *rdev;
+	int i;
+
+	for (i = 0; i < MT6380_MAX_REGULATOR; i++) {
+		config.dev = &pdev->dev;
+		config.driver_data = &mt6380_regulators[i];
+		config.regmap = regmap;
+		rdev = devm_regulator_register(&pdev->dev,
+					       &mt6380_regulators[i].desc,
+				&config);
+		if (IS_ERR(rdev)) {
+			dev_err(&pdev->dev, "failed to register %s\n",
+				mt6380_regulators[i].desc.name);
+			return PTR_ERR(rdev);
+		}
+	}
+	return 0;
+}
+
+static const struct platform_device_id mt6380_platform_ids[] = {
+	{"mt6380-regulator", 0},
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(platform, mt6380_platform_ids);
+
+static const struct of_device_id mt6380_of_match[] = {
+	{ .compatible = "mediatek,mt6380-regulator", },
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, mt6380_of_match);
+
+static struct platform_driver mt6380_regulator_driver = {
+	.driver = {
+		.name = "mt6380-regulator",
+		.of_match_table = of_match_ptr(mt6380_of_match),
+	},
+	.probe = mt6380_regulator_probe,
+	.id_table = mt6380_platform_ids,
+};
+
+module_platform_driver(mt6380_regulator_driver);
+
+MODULE_AUTHOR("Chenglin Xu <chenglin.xu@mediatek.com>");
+MODULE_DESCRIPTION("Regulator Driver for MediaTek MT6380 PMIC");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/regulator/of_regulator.c b/drivers/regulator/of_regulator.c
index 96bf75458da5..14637a01ba2d 100644
--- a/drivers/regulator/of_regulator.c
+++ b/drivers/regulator/of_regulator.c
@@ -150,7 +150,7 @@ static void of_get_regulation_constraints(struct device_node *np,
 			suspend_state = &constraints->state_disk;
 			break;
 		case PM_SUSPEND_ON:
-		case PM_SUSPEND_FREEZE:
+		case PM_SUSPEND_TO_IDLE:
 		case PM_SUSPEND_STANDBY:
 		default:
 			continue;
@@ -333,7 +333,7 @@ struct regulator_init_data *regulator_of_get_init_data(struct device *dev,
 		search = of_get_child_by_name(dev->of_node,
 					      desc->regulators_node);
 	else
-		search = dev->of_node;
+		search = of_node_get(dev->of_node);
 
 	if (!search) {
 		dev_dbg(dev, "Failed to find regulator container node '%s'\n",
diff --git a/drivers/regulator/pv88090-regulator.c b/drivers/regulator/pv88090-regulator.c
index ab51e254d13a..7a0c15957bd0 100644
--- a/drivers/regulator/pv88090-regulator.c
+++ b/drivers/regulator/pv88090-regulator.c
@@ -43,7 +43,7 @@ enum {
 struct pv88090_regulator {
 	struct regulator_desc desc;
 	/* Current limiting */
-	unsigned	n_current_limits;
+	unsigned int n_current_limits;
 	const int	*current_limits;
 	unsigned int limit_mask;
 	unsigned int conf;
@@ -398,9 +398,14 @@ static int pv88090_i2c_probe(struct i2c_client *i2c,
 				return ret;
 
 			range = (range >>
-				 (PV88080_BUCK_VRANGE_GAIN_SHIFT + i - 1)) &
-				PV88080_BUCK_VRANGE_GAIN_MASK;
+				 (PV88090_BUCK_VRANGE_GAIN_SHIFT + i - 1)) &
+				PV88090_BUCK_VRANGE_GAIN_MASK;
 			index = ((range << 1) | conf2);
+			if (index > PV88090_ID_BUCK3) {
+				dev_err(chip->dev,
+					"Invalid index(%d)\n", index);
+				return -EINVAL;
+			}
 
 			pv88090_regulator_info[i].desc.min_uV
 				= pv88090_buck_vol[index].min_uV;
diff --git a/drivers/regulator/pv88090-regulator.h b/drivers/regulator/pv88090-regulator.h
index d7aca8d8266d..62d9029277f4 100644
--- a/drivers/regulator/pv88090-regulator.h
+++ b/drivers/regulator/pv88090-regulator.h
@@ -89,10 +89,10 @@
 #define PV88090_BUCK_VDAC_RANGE_2			0x01
 
 /* PV88090_REG_BUCK_FOLD_RANGE (addr=0x61) */
-#define PV88080_BUCK_VRANGE_GAIN_SHIFT			3
-#define PV88080_BUCK_VRANGE_GAIN_MASK			0x01
+#define PV88090_BUCK_VRANGE_GAIN_SHIFT			3
+#define PV88090_BUCK_VRANGE_GAIN_MASK			0x01
 
-#define PV88080_BUCK_VRANGE_GAIN_1			0x00
-#define PV88080_BUCK_VRANGE_GAIN_2			0x01
+#define PV88090_BUCK_VRANGE_GAIN_1			0x00
+#define PV88090_BUCK_VRANGE_GAIN_2			0x01
 
 #endif	/* __PV88090_REGISTERS_H__ */
diff --git a/drivers/regulator/pwm-regulator.c b/drivers/regulator/pwm-regulator.c
index 1b88e0e15a70..a2fd140eff81 100644
--- a/drivers/regulator/pwm-regulator.c
+++ b/drivers/regulator/pwm-regulator.c
@@ -122,8 +122,7 @@ static int pwm_regulator_enable(struct regulator_dev *dev)
 {
 	struct pwm_regulator_data *drvdata = rdev_get_drvdata(dev);
 
-	if (drvdata->enb_gpio)
-		gpiod_set_value_cansleep(drvdata->enb_gpio, 1);
+	gpiod_set_value_cansleep(drvdata->enb_gpio, 1);
 
 	return pwm_enable(drvdata->pwm);
 }
@@ -134,8 +133,7 @@ static int pwm_regulator_disable(struct regulator_dev *dev)
 
 	pwm_disable(drvdata->pwm);
 
-	if (drvdata->enb_gpio)
-		gpiod_set_value_cansleep(drvdata->enb_gpio, 0);
+	gpiod_set_value_cansleep(drvdata->enb_gpio, 0);
 
 	return 0;
 }
diff --git a/drivers/regulator/qcom_rpm-regulator.c b/drivers/regulator/qcom_rpm-regulator.c
index 1b2acc43fea1..88dc0b0f003c 100644
--- a/drivers/regulator/qcom_rpm-regulator.c
+++ b/drivers/regulator/qcom_rpm-regulator.c
@@ -959,6 +959,11 @@ static int rpm_reg_probe(struct platform_device *pdev)
 	}
 
 	match = of_match_device(rpm_of_match, &pdev->dev);
+	if (!match) {
+		dev_err(&pdev->dev, "failed to match device\n");
+		return -ENODEV;
+	}
+
 	for (reg = match->data; reg->name; reg++) {
 		vreg = devm_kmalloc(&pdev->dev, sizeof(*vreg), GFP_KERNEL);
 		if (!vreg)
diff --git a/drivers/regulator/qcom_smd-regulator.c b/drivers/regulator/qcom_smd-regulator.c
index f35994a2a5be..940fe1b78411 100644
--- a/drivers/regulator/qcom_smd-regulator.c
+++ b/drivers/regulator/qcom_smd-regulator.c
@@ -570,6 +570,11 @@ static int rpm_reg_probe(struct platform_device *pdev)
 	}
 
 	match = of_match_device(rpm_of_match, &pdev->dev);
+	if (!match) {
+		dev_err(&pdev->dev, "failed to match device\n");
+		return -ENODEV;
+	}
+
 	for (reg = match->data; reg->name; reg++) {
 		vreg = devm_kzalloc(&pdev->dev, sizeof(*vreg), GFP_KERNEL);
 		if (!vreg)
diff --git a/drivers/regulator/rk808-regulator.c b/drivers/regulator/rk808-regulator.c
index a16d81420612..213b68743cc8 100644
--- a/drivers/regulator/rk808-regulator.c
+++ b/drivers/regulator/rk808-regulator.c
@@ -65,6 +65,27 @@
 /* max steps for increase voltage of Buck1/2, equal 100mv*/
 #define MAX_STEPS_ONE_TIME 8
 
+#define RK805_DESC(_id, _match, _supply, _min, _max, _step, _vreg,      \
+	_vmask, _ereg, _emask, _etime)                                  \
+	[_id] = {                                                       \
+		.name           = (_match),                             \
+		.supply_name    = (_supply),                            \
+		.of_match       = of_match_ptr(_match),                 \
+		.regulators_node = of_match_ptr("regulators"),          \
+		.type           = REGULATOR_VOLTAGE,                    \
+		.id             = (_id),                                \
+		.n_voltages     = (((_max) - (_min)) / (_step) + 1),    \
+		.owner          = THIS_MODULE,                          \
+		.min_uV         = (_min) * 1000,                        \
+		.uV_step        = (_step) * 1000,                       \
+		.vsel_reg       = (_vreg),                              \
+		.vsel_mask      = (_vmask),                             \
+		.enable_reg     = (_ereg),                              \
+		.enable_mask    = (_emask),                             \
+		.enable_time    = (_etime),                             \
+		.ops            = &rk805_reg_ops,                       \
+	}
+
 #define RK8XX_DESC(_id, _match, _supply, _min, _max, _step, _vreg,	\
 	_vmask, _ereg, _emask, _etime)					\
 	[_id] = {							\
@@ -298,6 +319,28 @@ static int rk808_set_suspend_voltage_range(struct regulator_dev *rdev, int uv)
 				  sel);
 }
 
+static int rk805_set_suspend_enable(struct regulator_dev *rdev)
+{
+	unsigned int reg;
+
+	reg = rdev->desc->enable_reg + RK808_SLP_SET_OFF_REG_OFFSET;
+
+	return regmap_update_bits(rdev->regmap, reg,
+				  rdev->desc->enable_mask,
+				  rdev->desc->enable_mask);
+}
+
+static int rk805_set_suspend_disable(struct regulator_dev *rdev)
+{
+	unsigned int reg;
+
+	reg = rdev->desc->enable_reg + RK808_SLP_SET_OFF_REG_OFFSET;
+
+	return regmap_update_bits(rdev->regmap, reg,
+				  rdev->desc->enable_mask,
+				  0);
+}
+
 static int rk808_set_suspend_enable(struct regulator_dev *rdev)
 {
 	unsigned int reg;
@@ -320,6 +363,27 @@ static int rk808_set_suspend_disable(struct regulator_dev *rdev)
 				  rdev->desc->enable_mask);
 }
 
+static struct regulator_ops rk805_reg_ops = {
+		.list_voltage           = regulator_list_voltage_linear,
+		.map_voltage            = regulator_map_voltage_linear,
+		.get_voltage_sel        = regulator_get_voltage_sel_regmap,
+		.set_voltage_sel        = regulator_set_voltage_sel_regmap,
+		.enable                 = regulator_enable_regmap,
+		.disable                = regulator_disable_regmap,
+		.is_enabled             = regulator_is_enabled_regmap,
+		.set_suspend_voltage    = rk808_set_suspend_voltage,
+		.set_suspend_enable     = rk805_set_suspend_enable,
+		.set_suspend_disable    = rk805_set_suspend_disable,
+};
+
+static struct regulator_ops rk805_switch_ops = {
+		.enable                 = regulator_enable_regmap,
+		.disable                = regulator_disable_regmap,
+		.is_enabled             = regulator_is_enabled_regmap,
+		.set_suspend_enable     = rk805_set_suspend_enable,
+		.set_suspend_disable    = rk805_set_suspend_disable,
+};
+
 static struct regulator_ops rk808_buck1_2_ops = {
 	.list_voltage		= regulator_list_voltage_linear,
 	.map_voltage		= regulator_map_voltage_linear,
@@ -369,6 +433,68 @@ static struct regulator_ops rk808_switch_ops = {
 	.set_suspend_disable	= rk808_set_suspend_disable,
 };
 
+static const struct regulator_desc rk805_reg[] = {
+	{
+		.name = "DCDC_REG1",
+		.supply_name = "vcc1",
+		.of_match = of_match_ptr("DCDC_REG1"),
+		.regulators_node = of_match_ptr("regulators"),
+		.id = RK805_ID_DCDC1,
+		.ops = &rk805_reg_ops,
+		.type = REGULATOR_VOLTAGE,
+		.min_uV = 712500,
+		.uV_step = 12500,
+		.n_voltages = 64,
+		.vsel_reg = RK805_BUCK1_ON_VSEL_REG,
+		.vsel_mask = RK818_BUCK_VSEL_MASK,
+		.enable_reg = RK805_DCDC_EN_REG,
+		.enable_mask = BIT(0),
+		.owner = THIS_MODULE,
+	}, {
+		.name = "DCDC_REG2",
+		.supply_name = "vcc2",
+		.of_match = of_match_ptr("DCDC_REG2"),
+		.regulators_node = of_match_ptr("regulators"),
+		.id = RK805_ID_DCDC2,
+		.ops = &rk805_reg_ops,
+		.type = REGULATOR_VOLTAGE,
+		.min_uV = 712500,
+		.uV_step = 12500,
+		.n_voltages = 64,
+		.vsel_reg = RK805_BUCK2_ON_VSEL_REG,
+		.vsel_mask = RK818_BUCK_VSEL_MASK,
+		.enable_reg = RK805_DCDC_EN_REG,
+		.enable_mask = BIT(1),
+		.owner = THIS_MODULE,
+	}, {
+		.name = "DCDC_REG3",
+		.supply_name = "vcc3",
+		.of_match = of_match_ptr("DCDC_REG3"),
+		.regulators_node = of_match_ptr("regulators"),
+		.id = RK805_ID_DCDC3,
+		.ops = &rk805_switch_ops,
+		.type = REGULATOR_VOLTAGE,
+		.n_voltages = 1,
+		.enable_reg = RK805_DCDC_EN_REG,
+		.enable_mask = BIT(2),
+		.owner = THIS_MODULE,
+	},
+
+	RK805_DESC(RK805_ID_DCDC4, "DCDC_REG4", "vcc4", 800, 3400, 100,
+		RK805_BUCK4_ON_VSEL_REG, RK818_BUCK4_VSEL_MASK,
+		RK805_DCDC_EN_REG, BIT(3), 0),
+
+	RK805_DESC(RK805_ID_LDO1, "LDO_REG1", "vcc5", 800, 3400, 100,
+		RK805_LDO1_ON_VSEL_REG, RK818_LDO_VSEL_MASK, RK805_LDO_EN_REG,
+		BIT(0), 400),
+	RK805_DESC(RK805_ID_LDO2, "LDO_REG2", "vcc5", 800, 3400, 100,
+		RK805_LDO2_ON_VSEL_REG, RK818_LDO_VSEL_MASK, RK805_LDO_EN_REG,
+		BIT(1), 400),
+	RK805_DESC(RK805_ID_LDO3, "LDO_REG3", "vcc6", 800, 3400, 100,
+		RK805_LDO3_ON_VSEL_REG, RK818_LDO_VSEL_MASK, RK805_LDO_EN_REG,
+		BIT(2), 400),
+};
+
 static const struct regulator_desc rk808_reg[] = {
 	{
 		.name = "DCDC_REG1",
@@ -625,6 +751,10 @@ static int rk808_regulator_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, pdata);
 
 	switch (rk808->variant) {
+	case RK805_ID:
+		regulators = rk805_reg;
+		nregulators = RK805_NUM_REGULATORS;
+		break;
 	case RK808_ID:
 		regulators = rk808_reg;
 		nregulators = RK808_NUM_REGULATORS;
diff --git a/drivers/regulator/rn5t618-regulator.c b/drivers/regulator/rn5t618-regulator.c
index 8d2819e36654..ef2be56460fe 100644
--- a/drivers/regulator/rn5t618-regulator.c
+++ b/drivers/regulator/rn5t618-regulator.c
@@ -79,6 +79,29 @@ static struct regulator_desc rn5t618_regulators[] = {
 	REG(LDORTC2, LDOEN2, BIT(5), LDORTC2DAC, 0x7f, 900000, 3500000, 25000),
 };
 
+static struct regulator_desc rc5t619_regulators[] = {
+	/* DCDC */
+	REG(DCDC1, DC1CTL, BIT(0), DC1DAC, 0xff, 600000, 3500000, 12500),
+	REG(DCDC2, DC2CTL, BIT(0), DC2DAC, 0xff, 600000, 3500000, 12500),
+	REG(DCDC3, DC3CTL, BIT(0), DC3DAC, 0xff, 600000, 3500000, 12500),
+	REG(DCDC4, DC4CTL, BIT(0), DC4DAC, 0xff, 600000, 3500000, 12500),
+	REG(DCDC5, DC5CTL, BIT(0), DC5DAC, 0xff, 600000, 3500000, 12500),
+	/* LDO */
+	REG(LDO1, LDOEN1, BIT(0), LDO1DAC, 0x7f, 900000, 3500000, 25000),
+	REG(LDO2, LDOEN1, BIT(1), LDO2DAC, 0x7f, 900000, 3500000, 25000),
+	REG(LDO3, LDOEN1, BIT(2), LDO3DAC, 0x7f, 900000, 3500000, 25000),
+	REG(LDO4, LDOEN1, BIT(3), LDO4DAC, 0x7f, 900000, 3500000, 25000),
+	REG(LDO5, LDOEN1, BIT(4), LDO5DAC, 0x7f, 600000, 3500000, 25000),
+	REG(LDO6, LDOEN1, BIT(5), LDO6DAC, 0x7f, 600000, 3500000, 25000),
+	REG(LDO7, LDOEN1, BIT(6), LDO7DAC, 0x7f, 900000, 3500000, 25000),
+	REG(LDO8, LDOEN1, BIT(7), LDO8DAC, 0x7f, 900000, 3500000, 25000),
+	REG(LDO9, LDOEN2, BIT(0), LDO9DAC, 0x7f, 900000, 3500000, 25000),
+	REG(LDO10, LDOEN2, BIT(0), LDO10DAC, 0x7f, 900000, 3500000, 25000),
+	/* LDO RTC */
+	REG(LDORTC1, LDOEN2, BIT(4), LDORTCDAC, 0x7f, 1700000, 3500000, 25000),
+	REG(LDORTC2, LDOEN2, BIT(5), LDORTC2DAC, 0x7f, 900000, 3500000, 25000),
+};
+
 static int rn5t618_regulator_probe(struct platform_device *pdev)
 {
 	struct rn5t618 *rn5t618 = dev_get_drvdata(pdev->dev.parent);
@@ -86,13 +109,20 @@ static int rn5t618_regulator_probe(struct platform_device *pdev)
 	struct regulator_dev *rdev;
 	struct regulator_desc *regulators;
 	int i;
+	int num_regulators = 0;
 
 	switch (rn5t618->variant) {
 	case RN5T567:
 		regulators = rn5t567_regulators;
+		num_regulators = ARRAY_SIZE(rn5t567_regulators);
 		break;
 	case RN5T618:
 		regulators = rn5t618_regulators;
+		num_regulators = ARRAY_SIZE(rn5t618_regulators);
+		break;
+	case RC5T619:
+		regulators = rc5t619_regulators;
+		num_regulators = ARRAY_SIZE(rc5t619_regulators);
 		break;
 	default:
 		return -EINVAL;
@@ -101,10 +131,7 @@ static int rn5t618_regulator_probe(struct platform_device *pdev)
 	config.dev = pdev->dev.parent;
 	config.regmap = rn5t618->regmap;
 
-	for (i = 0; i < RN5T618_REG_NUM; i++) {
-		if (!regulators[i].name)
-			continue;
-
+	for (i = 0; i < num_regulators; i++) {
 		rdev = devm_regulator_register(&pdev->dev,
 					       &regulators[i],
 					       &config);
diff --git a/drivers/regulator/s5m8767.c b/drivers/regulator/s5m8767.c
index 383cd7533721..4836947e1521 100644
--- a/drivers/regulator/s5m8767.c
+++ b/drivers/regulator/s5m8767.c
@@ -590,8 +590,8 @@ static int s5m8767_pmic_dt_parse_pdata(struct platform_device *pdev,
 		if (of_property_read_u32(reg_np, "op_mode",
 				&rmode->mode)) {
 			dev_warn(iodev->dev,
-				"no op_mode property property at %s\n",
-				reg_np->full_name);
+				"no op_mode property property at %pOF\n",
+				reg_np);
 
 			rmode->mode = S5M8767_OPMODE_NORMAL_MODE;
 		}
diff --git a/drivers/regulator/stm32-vrefbuf.c b/drivers/regulator/stm32-vrefbuf.c
new file mode 100644
index 000000000000..72c8b3e1022b
--- /dev/null
+++ b/drivers/regulator/stm32-vrefbuf.c
@@ -0,0 +1,202 @@
+/*
+ * Copyright (C) STMicroelectronics 2017
+ *
+ * Author: Fabrice Gasnier <fabrice.gasnier@st.com>
+ *
+ * License terms:  GNU General Public License (GPL), version 2
+ */
+
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/regulator/driver.h>
+#include <linux/regulator/of_regulator.h>
+
+/* STM32 VREFBUF registers */
+#define STM32_VREFBUF_CSR		0x00
+
+/* STM32 VREFBUF CSR bitfields */
+#define STM32_VRS			GENMASK(6, 4)
+#define STM32_VRR			BIT(3)
+#define STM32_HIZ			BIT(1)
+#define STM32_ENVR			BIT(0)
+
+struct stm32_vrefbuf {
+	void __iomem *base;
+	struct clk *clk;
+};
+
+static const unsigned int stm32_vrefbuf_voltages[] = {
+	/* Matches resp. VRS = 000b, 001b, 010b, 011b */
+	2500000, 2048000, 1800000, 1500000,
+};
+
+static int stm32_vrefbuf_enable(struct regulator_dev *rdev)
+{
+	struct stm32_vrefbuf *priv = rdev_get_drvdata(rdev);
+	u32 val = readl_relaxed(priv->base + STM32_VREFBUF_CSR);
+	int ret;
+
+	val = (val & ~STM32_HIZ) | STM32_ENVR;
+	writel_relaxed(val, priv->base + STM32_VREFBUF_CSR);
+
+	/*
+	 * Vrefbuf startup time depends on external capacitor: wait here for
+	 * VRR to be set. That means output has reached expected value.
+	 * ~650us sleep should be enough for caps up to 1.5uF. Use 10ms as
+	 * arbitrary timeout.
+	 */
+	ret = readl_poll_timeout(priv->base + STM32_VREFBUF_CSR, val,
+				 !(val & STM32_VRR), 650, 10000);
+	if (ret) {
+		dev_err(&rdev->dev, "stm32 vrefbuf timed out!\n");
+		val = readl_relaxed(priv->base + STM32_VREFBUF_CSR);
+		val = (val & ~STM32_ENVR) | STM32_HIZ;
+		writel_relaxed(val, priv->base + STM32_VREFBUF_CSR);
+	}
+
+	return ret;
+}
+
+static int stm32_vrefbuf_disable(struct regulator_dev *rdev)
+{
+	struct stm32_vrefbuf *priv = rdev_get_drvdata(rdev);
+	u32 val = readl_relaxed(priv->base + STM32_VREFBUF_CSR);
+
+	val = (val & ~STM32_ENVR) | STM32_HIZ;
+	writel_relaxed(val, priv->base + STM32_VREFBUF_CSR);
+
+	return 0;
+}
+
+static int stm32_vrefbuf_is_enabled(struct regulator_dev *rdev)
+{
+	struct stm32_vrefbuf *priv = rdev_get_drvdata(rdev);
+
+	return readl_relaxed(priv->base + STM32_VREFBUF_CSR) & STM32_ENVR;
+}
+
+static int stm32_vrefbuf_set_voltage_sel(struct regulator_dev *rdev,
+					 unsigned sel)
+{
+	struct stm32_vrefbuf *priv = rdev_get_drvdata(rdev);
+	u32 val = readl_relaxed(priv->base + STM32_VREFBUF_CSR);
+
+	val = (val & ~STM32_VRS) | FIELD_PREP(STM32_VRS, sel);
+	writel_relaxed(val, priv->base + STM32_VREFBUF_CSR);
+
+	return 0;
+}
+
+static int stm32_vrefbuf_get_voltage_sel(struct regulator_dev *rdev)
+{
+	struct stm32_vrefbuf *priv = rdev_get_drvdata(rdev);
+	u32 val = readl_relaxed(priv->base + STM32_VREFBUF_CSR);
+
+	return FIELD_GET(STM32_VRS, val);
+}
+
+static const struct regulator_ops stm32_vrefbuf_volt_ops = {
+	.enable		= stm32_vrefbuf_enable,
+	.disable	= stm32_vrefbuf_disable,
+	.is_enabled	= stm32_vrefbuf_is_enabled,
+	.get_voltage_sel = stm32_vrefbuf_get_voltage_sel,
+	.set_voltage_sel = stm32_vrefbuf_set_voltage_sel,
+	.list_voltage	= regulator_list_voltage_table,
+};
+
+static const struct regulator_desc stm32_vrefbuf_regu = {
+	.name = "vref",
+	.supply_name = "vdda",
+	.volt_table = stm32_vrefbuf_voltages,
+	.n_voltages = ARRAY_SIZE(stm32_vrefbuf_voltages),
+	.ops = &stm32_vrefbuf_volt_ops,
+	.type = REGULATOR_VOLTAGE,
+	.owner = THIS_MODULE,
+};
+
+static int stm32_vrefbuf_probe(struct platform_device *pdev)
+{
+	struct resource *res;
+	struct stm32_vrefbuf *priv;
+	struct regulator_config config = { };
+	struct regulator_dev *rdev;
+	int ret;
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	priv->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(priv->base))
+		return PTR_ERR(priv->base);
+
+	priv->clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(priv->clk))
+		return PTR_ERR(priv->clk);
+
+	ret = clk_prepare_enable(priv->clk);
+	if (ret) {
+		dev_err(&pdev->dev, "clk prepare failed with error %d\n", ret);
+		return ret;
+	}
+
+	config.dev = &pdev->dev;
+	config.driver_data = priv;
+	config.of_node = pdev->dev.of_node;
+	config.init_data = of_get_regulator_init_data(&pdev->dev,
+						      pdev->dev.of_node,
+						      &stm32_vrefbuf_regu);
+
+	rdev = regulator_register(&stm32_vrefbuf_regu, &config);
+	if (IS_ERR(rdev)) {
+		ret = PTR_ERR(rdev);
+		dev_err(&pdev->dev, "register failed with error %d\n", ret);
+		goto err_clk_dis;
+	}
+	platform_set_drvdata(pdev, rdev);
+
+	return 0;
+
+err_clk_dis:
+	clk_disable_unprepare(priv->clk);
+
+	return ret;
+}
+
+static int stm32_vrefbuf_remove(struct platform_device *pdev)
+{
+	struct regulator_dev *rdev = platform_get_drvdata(pdev);
+	struct stm32_vrefbuf *priv = rdev_get_drvdata(rdev);
+
+	regulator_unregister(rdev);
+	clk_disable_unprepare(priv->clk);
+
+	return 0;
+};
+
+static const struct of_device_id stm32_vrefbuf_of_match[] = {
+	{ .compatible = "st,stm32-vrefbuf", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, stm32_vrefbuf_of_match);
+
+static struct platform_driver stm32_vrefbuf_driver = {
+	.probe = stm32_vrefbuf_probe,
+	.remove = stm32_vrefbuf_remove,
+	.driver = {
+		.name  = "stm32-vrefbuf",
+		.of_match_table = of_match_ptr(stm32_vrefbuf_of_match),
+	},
+};
+module_platform_driver(stm32_vrefbuf_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Fabrice Gasnier <fabrice.gasnier@st.com>");
+MODULE_DESCRIPTION("STMicroelectronics STM32 VREFBUF driver");
+MODULE_ALIAS("platform:stm32-vrefbuf");
diff --git a/drivers/regulator/twl-regulator.c b/drivers/regulator/twl-regulator.c
index 6c9ec84121bd..a4456db5849d 100644
--- a/drivers/regulator/twl-regulator.c
+++ b/drivers/regulator/twl-regulator.c
@@ -20,7 +20,7 @@
 #include <linux/regulator/driver.h>
 #include <linux/regulator/machine.h>
 #include <linux/regulator/of_regulator.h>
-#include <linux/i2c/twl.h>
+#include <linux/mfd/twl.h>
 #include <linux/delay.h>
 
 /*
diff --git a/drivers/regulator/twl6030-regulator.c b/drivers/regulator/twl6030-regulator.c
index 56aada387887..219cbd910dbf 100644
--- a/drivers/regulator/twl6030-regulator.c
+++ b/drivers/regulator/twl6030-regulator.c
@@ -21,7 +21,7 @@
 #include <linux/regulator/driver.h>
 #include <linux/regulator/machine.h>
 #include <linux/regulator/of_regulator.h>
-#include <linux/i2c/twl.h>
+#include <linux/mfd/twl.h>
 #include <linux/delay.h>
 
 struct twlreg_info {
diff --git a/drivers/remoteproc/Kconfig b/drivers/remoteproc/Kconfig
index 8891a8e50f12..df63e44526ac 100644
--- a/drivers/remoteproc/Kconfig
+++ b/drivers/remoteproc/Kconfig
@@ -12,6 +12,15 @@ config REMOTEPROC
 
 if REMOTEPROC
 
+config IMX_REMOTEPROC
+	tristate "IMX6/7 remoteproc support"
+	depends on SOC_IMX6SX || SOC_IMX7D
+	help
+	  Say y here to support iMX's remote processors (Cortex M4
+	  on iMX7D) via the remote processor framework.
+
+	  It's safe to say N here.
+
 config OMAP_REMOTEPROC
 	tristate "OMAP remoteproc support"
 	depends on HAS_DMA
@@ -83,6 +92,7 @@ config QCOM_ADSP_PIL
 	depends on OF && ARCH_QCOM
 	depends on QCOM_SMEM
 	depends on RPMSG_QCOM_SMD || (COMPILE_TEST && RPMSG_QCOM_SMD=n)
+	depends on RPMSG_QCOM_GLINK_SMEM || RPMSG_QCOM_GLINK_SMEM=n
 	select MFD_SYSCON
 	select QCOM_MDT_LOADER
 	select QCOM_RPROC_COMMON
diff --git a/drivers/remoteproc/Makefile b/drivers/remoteproc/Makefile
index f1ce5fc8a2f3..1a0b3dd44b8c 100644
--- a/drivers/remoteproc/Makefile
+++ b/drivers/remoteproc/Makefile
@@ -8,6 +8,7 @@ remoteproc-y				+= remoteproc_debugfs.o
 remoteproc-y				+= remoteproc_sysfs.o
 remoteproc-y				+= remoteproc_virtio.o
 remoteproc-y				+= remoteproc_elf_loader.o
+obj-$(CONFIG_IMX_REMOTEPROC)		+= imx_rproc.o
 obj-$(CONFIG_OMAP_REMOTEPROC)		+= omap_remoteproc.o
 obj-$(CONFIG_WKUP_M3_RPROC)		+= wkup_m3_rproc.o
 obj-$(CONFIG_DA8XX_REMOTEPROC)		+= da8xx_remoteproc.o
diff --git a/drivers/remoteproc/da8xx_remoteproc.c b/drivers/remoteproc/da8xx_remoteproc.c
index 99539cec1329..bf3b9034c319 100644
--- a/drivers/remoteproc/da8xx_remoteproc.c
+++ b/drivers/remoteproc/da8xx_remoteproc.c
@@ -16,6 +16,7 @@
 #include <linux/irq.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/of_reserved_mem.h>
 #include <linux/platform_device.h>
 #include <linux/remoteproc.h>
 
@@ -38,9 +39,27 @@ MODULE_PARM_DESC(da8xx_fw_name,
 #define SYSCFG_CHIPSIG3 BIT(3)
 #define SYSCFG_CHIPSIG4 BIT(4)
 
+#define DA8XX_RPROC_LOCAL_ADDRESS_MASK	(SZ_16M - 1)
+
+/**
+ * struct da8xx_rproc_mem - internal memory structure
+ * @cpu_addr: MPU virtual address of the memory region
+ * @bus_addr: Bus address used to access the memory region
+ * @dev_addr: Device address of the memory region from DSP view
+ * @size: Size of the memory region
+ */
+struct da8xx_rproc_mem {
+	void __iomem *cpu_addr;
+	phys_addr_t bus_addr;
+	u32 dev_addr;
+	size_t size;
+};
+
 /**
  * struct da8xx_rproc - da8xx remote processor instance state
  * @rproc: rproc handle
+ * @mem: internal memory regions data
+ * @num_mems: number of internal memory regions
  * @dsp_clk: placeholder for platform's DSP clk
  * @ack_fxn: chip-specific ack function for ack'ing irq
  * @irq_data: ack_fxn function parameter
@@ -50,6 +69,8 @@ MODULE_PARM_DESC(da8xx_fw_name,
  */
 struct da8xx_rproc {
 	struct rproc *rproc;
+	struct da8xx_rproc_mem *mem;
+	int num_mems;
 	struct clk *dsp_clk;
 	void (*ack_fxn)(struct irq_data *data);
 	struct irq_data *irq_data;
@@ -158,6 +179,44 @@ static const struct rproc_ops da8xx_rproc_ops = {
 	.kick = da8xx_rproc_kick,
 };
 
+static int da8xx_rproc_get_internal_memories(struct platform_device *pdev,
+					     struct da8xx_rproc *drproc)
+{
+	static const char * const mem_names[] = {"l2sram", "l1pram", "l1dram"};
+	int num_mems = ARRAY_SIZE(mem_names);
+	struct device *dev = &pdev->dev;
+	struct resource *res;
+	int i;
+
+	drproc->mem = devm_kcalloc(dev, num_mems, sizeof(*drproc->mem),
+				   GFP_KERNEL);
+	if (!drproc->mem)
+		return -ENOMEM;
+
+	for (i = 0; i < num_mems; i++) {
+		res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+						   mem_names[i]);
+		drproc->mem[i].cpu_addr = devm_ioremap_resource(dev, res);
+		if (IS_ERR(drproc->mem[i].cpu_addr)) {
+			dev_err(dev, "failed to parse and map %s memory\n",
+				mem_names[i]);
+			return PTR_ERR(drproc->mem[i].cpu_addr);
+		}
+		drproc->mem[i].bus_addr = res->start;
+		drproc->mem[i].dev_addr =
+				res->start & DA8XX_RPROC_LOCAL_ADDRESS_MASK;
+		drproc->mem[i].size = resource_size(res);
+
+		dev_dbg(dev, "memory %8s: bus addr %pa size 0x%x va %p da 0x%x\n",
+			mem_names[i], &drproc->mem[i].bus_addr,
+			drproc->mem[i].size, drproc->mem[i].cpu_addr,
+			drproc->mem[i].dev_addr);
+	}
+	drproc->num_mems = num_mems;
+
+	return 0;
+}
+
 static int da8xx_rproc_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -184,12 +243,14 @@ static int da8xx_rproc_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
-	bootreg_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	bootreg_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+						   "host1cfg");
 	bootreg = devm_ioremap_resource(dev, bootreg_res);
 	if (IS_ERR(bootreg))
 		return PTR_ERR(bootreg);
 
-	chipsig_res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+	chipsig_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+						   "chipsig");
 	chipsig = devm_ioremap_resource(dev, chipsig_res);
 	if (IS_ERR(chipsig))
 		return PTR_ERR(chipsig);
@@ -201,16 +262,31 @@ static int da8xx_rproc_probe(struct platform_device *pdev)
 		return PTR_ERR(dsp_clk);
 	}
 
+	if (dev->of_node) {
+		ret = of_reserved_mem_device_init(dev);
+		if (ret) {
+			dev_err(dev, "device does not have specific CMA pool: %d\n",
+				ret);
+			return ret;
+		}
+	}
+
 	rproc = rproc_alloc(dev, "dsp", &da8xx_rproc_ops, da8xx_fw_name,
 		sizeof(*drproc));
-	if (!rproc)
-		return -ENOMEM;
+	if (!rproc) {
+		ret = -ENOMEM;
+		goto free_mem;
+	}
 
 	drproc = rproc->priv;
 	drproc->rproc = rproc;
 	drproc->dsp_clk = dsp_clk;
 	rproc->has_iommu = false;
 
+	ret = da8xx_rproc_get_internal_memories(pdev, drproc);
+	if (ret)
+		goto free_rproc;
+
 	platform_set_drvdata(pdev, rproc);
 
 	/* everything the ISR needs is now setup, so hook it up */
@@ -247,7 +323,9 @@ static int da8xx_rproc_probe(struct platform_device *pdev)
 
 free_rproc:
 	rproc_free(rproc);
-
+free_mem:
+	if (dev->of_node)
+		of_reserved_mem_device_release(dev);
 	return ret;
 }
 
@@ -255,6 +333,7 @@ static int da8xx_rproc_remove(struct platform_device *pdev)
 {
 	struct rproc *rproc = platform_get_drvdata(pdev);
 	struct da8xx_rproc *drproc = (struct da8xx_rproc *)rproc->priv;
+	struct device *dev = &pdev->dev;
 
 	/*
 	 * The devm subsystem might end up releasing things before
@@ -265,15 +344,24 @@ static int da8xx_rproc_remove(struct platform_device *pdev)
 
 	rproc_del(rproc);
 	rproc_free(rproc);
+	if (dev->of_node)
+		of_reserved_mem_device_release(dev);
 
 	return 0;
 }
 
+static const struct of_device_id davinci_rproc_of_match[] __maybe_unused = {
+	{ .compatible = "ti,da850-dsp", },
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, davinci_rproc_of_match);
+
 static struct platform_driver da8xx_rproc_driver = {
 	.probe = da8xx_rproc_probe,
 	.remove = da8xx_rproc_remove,
 	.driver = {
 		.name = "davinci-rproc",
+		.of_match_table = of_match_ptr(davinci_rproc_of_match),
 	},
 };
 
diff --git a/drivers/remoteproc/imx_rproc.c b/drivers/remoteproc/imx_rproc.c
new file mode 100644
index 000000000000..612d91403341
--- /dev/null
+++ b/drivers/remoteproc/imx_rproc.c
@@ -0,0 +1,426 @@
+/*
+ * Copyright (c) 2017 Pengutronix, Oleksij Rempel <kernel@pengutronix.de>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation.
+ */
+
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/remoteproc.h>
+
+#define IMX7D_SRC_SCR			0x0C
+#define IMX7D_ENABLE_M4			BIT(3)
+#define IMX7D_SW_M4P_RST		BIT(2)
+#define IMX7D_SW_M4C_RST		BIT(1)
+#define IMX7D_SW_M4C_NON_SCLR_RST	BIT(0)
+
+#define IMX7D_M4_RST_MASK		(IMX7D_ENABLE_M4 | IMX7D_SW_M4P_RST \
+					 | IMX7D_SW_M4C_RST \
+					 | IMX7D_SW_M4C_NON_SCLR_RST)
+
+#define IMX7D_M4_START			(IMX7D_ENABLE_M4 | IMX7D_SW_M4P_RST \
+					 | IMX7D_SW_M4C_RST)
+#define IMX7D_M4_STOP			IMX7D_SW_M4C_NON_SCLR_RST
+
+/* Address: 0x020D8000 */
+#define IMX6SX_SRC_SCR			0x00
+#define IMX6SX_ENABLE_M4		BIT(22)
+#define IMX6SX_SW_M4P_RST		BIT(12)
+#define IMX6SX_SW_M4C_NON_SCLR_RST	BIT(4)
+#define IMX6SX_SW_M4C_RST		BIT(3)
+
+#define IMX6SX_M4_START			(IMX6SX_ENABLE_M4 | IMX6SX_SW_M4P_RST \
+					 | IMX6SX_SW_M4C_RST)
+#define IMX6SX_M4_STOP			IMX6SX_SW_M4C_NON_SCLR_RST
+#define IMX6SX_M4_RST_MASK		(IMX6SX_ENABLE_M4 | IMX6SX_SW_M4P_RST \
+					 | IMX6SX_SW_M4C_NON_SCLR_RST \
+					 | IMX6SX_SW_M4C_RST)
+
+#define IMX7D_RPROC_MEM_MAX		8
+
+/**
+ * struct imx_rproc_mem - slim internal memory structure
+ * @cpu_addr: MPU virtual address of the memory region
+ * @sys_addr: Bus address used to access the memory region
+ * @size: Size of the memory region
+ */
+struct imx_rproc_mem {
+	void __iomem *cpu_addr;
+	phys_addr_t sys_addr;
+	size_t size;
+};
+
+/* att flags */
+/* M4 own area. Can be mapped at probe */
+#define ATT_OWN		BIT(1)
+
+/* address translation table */
+struct imx_rproc_att {
+	u32 da;	/* device address (From Cortex M4 view)*/
+	u32 sa;	/* system bus address */
+	u32 size; /* size of reg range */
+	int flags;
+};
+
+struct imx_rproc_dcfg {
+	u32				src_reg;
+	u32				src_mask;
+	u32				src_start;
+	u32				src_stop;
+	const struct imx_rproc_att	*att;
+	size_t				att_size;
+};
+
+struct imx_rproc {
+	struct device			*dev;
+	struct regmap			*regmap;
+	struct rproc			*rproc;
+	const struct imx_rproc_dcfg	*dcfg;
+	struct imx_rproc_mem		mem[IMX7D_RPROC_MEM_MAX];
+	struct clk			*clk;
+};
+
+static const struct imx_rproc_att imx_rproc_att_imx7d[] = {
+	/* dev addr , sys addr  , size	    , flags */
+	/* OCRAM_S (M4 Boot code) - alias */
+	{ 0x00000000, 0x00180000, 0x00008000, 0 },
+	/* OCRAM_S (Code) */
+	{ 0x00180000, 0x00180000, 0x00008000, ATT_OWN },
+	/* OCRAM (Code) - alias */
+	{ 0x00900000, 0x00900000, 0x00020000, 0 },
+	/* OCRAM_EPDC (Code) - alias */
+	{ 0x00920000, 0x00920000, 0x00020000, 0 },
+	/* OCRAM_PXP (Code) - alias */
+	{ 0x00940000, 0x00940000, 0x00008000, 0 },
+	/* TCML (Code) */
+	{ 0x1FFF8000, 0x007F8000, 0x00008000, ATT_OWN },
+	/* DDR (Code) - alias, first part of DDR (Data) */
+	{ 0x10000000, 0x80000000, 0x0FFF0000, 0 },
+
+	/* TCMU (Data) */
+	{ 0x20000000, 0x00800000, 0x00008000, ATT_OWN },
+	/* OCRAM (Data) */
+	{ 0x20200000, 0x00900000, 0x00020000, 0 },
+	/* OCRAM_EPDC (Data) */
+	{ 0x20220000, 0x00920000, 0x00020000, 0 },
+	/* OCRAM_PXP (Data) */
+	{ 0x20240000, 0x00940000, 0x00008000, 0 },
+	/* DDR (Data) */
+	{ 0x80000000, 0x80000000, 0x60000000, 0 },
+};
+
+static const struct imx_rproc_att imx_rproc_att_imx6sx[] = {
+	/* dev addr , sys addr  , size	    , flags */
+	/* TCML (M4 Boot Code) - alias */
+	{ 0x00000000, 0x007F8000, 0x00008000, 0 },
+	/* OCRAM_S (Code) */
+	{ 0x00180000, 0x008F8000, 0x00004000, 0 },
+	/* OCRAM_S (Code) - alias */
+	{ 0x00180000, 0x008FC000, 0x00004000, 0 },
+	/* TCML (Code) */
+	{ 0x1FFF8000, 0x007F8000, 0x00008000, ATT_OWN },
+	/* DDR (Code) - alias, first part of DDR (Data) */
+	{ 0x10000000, 0x80000000, 0x0FFF8000, 0 },
+
+	/* TCMU (Data) */
+	{ 0x20000000, 0x00800000, 0x00008000, ATT_OWN },
+	/* OCRAM_S (Data) - alias? */
+	{ 0x208F8000, 0x008F8000, 0x00004000, 0 },
+	/* DDR (Data) */
+	{ 0x80000000, 0x80000000, 0x60000000, 0 },
+};
+
+static const struct imx_rproc_dcfg imx_rproc_cfg_imx7d = {
+	.src_reg	= IMX7D_SRC_SCR,
+	.src_mask	= IMX7D_M4_RST_MASK,
+	.src_start	= IMX7D_M4_START,
+	.src_stop	= IMX7D_M4_STOP,
+	.att		= imx_rproc_att_imx7d,
+	.att_size	= ARRAY_SIZE(imx_rproc_att_imx7d),
+};
+
+static const struct imx_rproc_dcfg imx_rproc_cfg_imx6sx = {
+	.src_reg	= IMX6SX_SRC_SCR,
+	.src_mask	= IMX6SX_M4_RST_MASK,
+	.src_start	= IMX6SX_M4_START,
+	.src_stop	= IMX6SX_M4_STOP,
+	.att		= imx_rproc_att_imx6sx,
+	.att_size	= ARRAY_SIZE(imx_rproc_att_imx6sx),
+};
+
+static int imx_rproc_start(struct rproc *rproc)
+{
+	struct imx_rproc *priv = rproc->priv;
+	const struct imx_rproc_dcfg *dcfg = priv->dcfg;
+	struct device *dev = priv->dev;
+	int ret;
+
+	ret = regmap_update_bits(priv->regmap, dcfg->src_reg,
+				 dcfg->src_mask, dcfg->src_start);
+	if (ret)
+		dev_err(dev, "Filed to enable M4!\n");
+
+	return ret;
+}
+
+static int imx_rproc_stop(struct rproc *rproc)
+{
+	struct imx_rproc *priv = rproc->priv;
+	const struct imx_rproc_dcfg *dcfg = priv->dcfg;
+	struct device *dev = priv->dev;
+	int ret;
+
+	ret = regmap_update_bits(priv->regmap, dcfg->src_reg,
+				 dcfg->src_mask, dcfg->src_stop);
+	if (ret)
+		dev_err(dev, "Filed to stop M4!\n");
+
+	return ret;
+}
+
+static int imx_rproc_da_to_sys(struct imx_rproc *priv, u64 da,
+			       int len, u64 *sys)
+{
+	const struct imx_rproc_dcfg *dcfg = priv->dcfg;
+	int i;
+
+	/* parse address translation table */
+	for (i = 0; i < dcfg->att_size; i++) {
+		const struct imx_rproc_att *att = &dcfg->att[i];
+
+		if (da >= att->da && da + len < att->da + att->size) {
+			unsigned int offset = da - att->da;
+
+			*sys = att->sa + offset;
+			return 0;
+		}
+	}
+
+	dev_warn(priv->dev, "Translation filed: da = 0x%llx len = 0x%x\n",
+		 da, len);
+	return -ENOENT;
+}
+
+static void *imx_rproc_da_to_va(struct rproc *rproc, u64 da, int len)
+{
+	struct imx_rproc *priv = rproc->priv;
+	void *va = NULL;
+	u64 sys;
+	int i;
+
+	if (len <= 0)
+		return NULL;
+
+	/*
+	 * On device side we have many aliases, so we need to convert device
+	 * address (M4) to system bus address first.
+	 */
+	if (imx_rproc_da_to_sys(priv, da, len, &sys))
+		return NULL;
+
+	for (i = 0; i < IMX7D_RPROC_MEM_MAX; i++) {
+		if (sys >= priv->mem[i].sys_addr && sys + len <
+		    priv->mem[i].sys_addr +  priv->mem[i].size) {
+			unsigned int offset = sys - priv->mem[i].sys_addr;
+			/* __force to make sparse happy with type conversion */
+			va = (__force void *)(priv->mem[i].cpu_addr + offset);
+			break;
+		}
+	}
+
+	dev_dbg(&rproc->dev, "da = 0x%llx len = 0x%x va = 0x%p\n", da, len, va);
+
+	return va;
+}
+
+static const struct rproc_ops imx_rproc_ops = {
+	.start		= imx_rproc_start,
+	.stop		= imx_rproc_stop,
+	.da_to_va       = imx_rproc_da_to_va,
+};
+
+static int imx_rproc_addr_init(struct imx_rproc *priv,
+			       struct platform_device *pdev)
+{
+	const struct imx_rproc_dcfg *dcfg = priv->dcfg;
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	int a, b = 0, err, nph;
+
+	/* remap required addresses */
+	for (a = 0; a < dcfg->att_size; a++) {
+		const struct imx_rproc_att *att = &dcfg->att[a];
+
+		if (!(att->flags & ATT_OWN))
+			continue;
+
+		if (b > IMX7D_RPROC_MEM_MAX)
+			break;
+
+		priv->mem[b].cpu_addr = devm_ioremap(&pdev->dev,
+						     att->sa, att->size);
+		if (IS_ERR(priv->mem[b].cpu_addr)) {
+			dev_err(dev, "devm_ioremap_resource failed\n");
+			err = PTR_ERR(priv->mem[b].cpu_addr);
+			return err;
+		}
+		priv->mem[b].sys_addr = att->sa;
+		priv->mem[b].size = att->size;
+		b++;
+	}
+
+	/* memory-region is optional property */
+	nph = of_count_phandle_with_args(np, "memory-region", NULL);
+	if (nph <= 0)
+		return 0;
+
+	/* remap optional addresses */
+	for (a = 0; a < nph; a++) {
+		struct device_node *node;
+		struct resource res;
+
+		node = of_parse_phandle(np, "memory-region", a);
+		err = of_address_to_resource(node, 0, &res);
+		if (err) {
+			dev_err(dev, "unable to resolve memory region\n");
+			return err;
+		}
+
+		if (b > IMX7D_RPROC_MEM_MAX)
+			break;
+
+		priv->mem[b].cpu_addr = devm_ioremap_resource(&pdev->dev, &res);
+		if (IS_ERR(priv->mem[b].cpu_addr)) {
+			dev_err(dev, "devm_ioremap_resource failed\n");
+			err = PTR_ERR(priv->mem[b].cpu_addr);
+			return err;
+		}
+		priv->mem[b].sys_addr = res.start;
+		priv->mem[b].size = resource_size(&res);
+		b++;
+	}
+
+	return 0;
+}
+
+static int imx_rproc_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct imx_rproc *priv;
+	struct rproc *rproc;
+	struct regmap_config config = { .name = "imx-rproc" };
+	const struct imx_rproc_dcfg *dcfg;
+	struct regmap *regmap;
+	int ret;
+
+	regmap = syscon_regmap_lookup_by_phandle(np, "syscon");
+	if (IS_ERR(regmap)) {
+		dev_err(dev, "failed to find syscon\n");
+		return PTR_ERR(regmap);
+	}
+	regmap_attach_dev(dev, regmap, &config);
+
+	/* set some other name then imx */
+	rproc = rproc_alloc(dev, "imx-rproc", &imx_rproc_ops,
+			    NULL, sizeof(*priv));
+	if (!rproc) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	dcfg = of_device_get_match_data(dev);
+	if (!dcfg)
+		return -EINVAL;
+
+	priv = rproc->priv;
+	priv->rproc = rproc;
+	priv->regmap = regmap;
+	priv->dcfg = dcfg;
+	priv->dev = dev;
+
+	dev_set_drvdata(dev, rproc);
+
+	ret = imx_rproc_addr_init(priv, pdev);
+	if (ret) {
+		dev_err(dev, "filed on imx_rproc_addr_init\n");
+		goto err_put_rproc;
+	}
+
+	priv->clk = devm_clk_get(dev, NULL);
+	if (IS_ERR(priv->clk)) {
+		dev_err(dev, "Failed to get clock\n");
+		rproc_free(rproc);
+		return PTR_ERR(priv->clk);
+	}
+
+	/*
+	 * clk for M4 block including memory. Should be
+	 * enabled before .start for FW transfer.
+	 */
+	ret = clk_prepare_enable(priv->clk);
+	if (ret) {
+		dev_err(&rproc->dev, "Failed to enable clock\n");
+		rproc_free(rproc);
+		return ret;
+	}
+
+	ret = rproc_add(rproc);
+	if (ret) {
+		dev_err(dev, "rproc_add failed\n");
+		goto err_put_clk;
+	}
+
+	return ret;
+
+err_put_clk:
+	clk_disable_unprepare(priv->clk);
+err_put_rproc:
+	rproc_free(rproc);
+err:
+	return ret;
+}
+
+static int imx_rproc_remove(struct platform_device *pdev)
+{
+	struct rproc *rproc = platform_get_drvdata(pdev);
+	struct imx_rproc *priv = rproc->priv;
+
+	clk_disable_unprepare(priv->clk);
+	rproc_del(rproc);
+	rproc_free(rproc);
+
+	return 0;
+}
+
+static const struct of_device_id imx_rproc_of_match[] = {
+	{ .compatible = "fsl,imx7d-cm4", .data = &imx_rproc_cfg_imx7d },
+	{ .compatible = "fsl,imx6sx-cm4", .data = &imx_rproc_cfg_imx6sx },
+	{},
+};
+MODULE_DEVICE_TABLE(of, imx_rproc_of_match);
+
+static struct platform_driver imx_rproc_driver = {
+	.probe = imx_rproc_probe,
+	.remove = imx_rproc_remove,
+	.driver = {
+		.name = "imx-rproc",
+		.of_match_table = imx_rproc_of_match,
+	},
+};
+
+module_platform_driver(imx_rproc_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("IMX6SX/7D remote processor control driver");
+MODULE_AUTHOR("Oleksij Rempel <o.rempel@pengutronix.de>");
diff --git a/drivers/remoteproc/keystone_remoteproc.c b/drivers/remoteproc/keystone_remoteproc.c
index 5f776bfd674a..aaac31134e39 100644
--- a/drivers/remoteproc/keystone_remoteproc.c
+++ b/drivers/remoteproc/keystone_remoteproc.c
@@ -410,7 +410,7 @@ static int keystone_rproc_probe(struct platform_device *pdev)
 	if (ret)
 		goto free_rproc;
 
-	ksproc->reset = devm_reset_control_get(dev, NULL);
+	ksproc->reset = devm_reset_control_get_exclusive(dev, NULL);
 	if (IS_ERR(ksproc->reset)) {
 		ret = PTR_ERR(ksproc->reset);
 		goto free_rproc;
@@ -505,6 +505,7 @@ static const struct of_device_id keystone_rproc_of_match[] = {
 	{ .compatible = "ti,k2hk-dsp", },
 	{ .compatible = "ti,k2l-dsp", },
 	{ .compatible = "ti,k2e-dsp", },
+	{ .compatible = "ti,k2g-dsp", },
 	{ /* sentinel */ },
 };
 MODULE_DEVICE_TABLE(of, keystone_rproc_of_match);
diff --git a/drivers/remoteproc/qcom_adsp_pil.c b/drivers/remoteproc/qcom_adsp_pil.c
index 49fe2f807e1d..3f6af54dbc96 100644
--- a/drivers/remoteproc/qcom_adsp_pil.c
+++ b/drivers/remoteproc/qcom_adsp_pil.c
@@ -38,6 +38,7 @@ struct adsp_data {
 	const char *firmware_name;
 	int pas_id;
 	bool has_aggre2_clk;
+	const char *ssr_name;
 };
 
 struct qcom_adsp {
@@ -71,7 +72,9 @@ struct qcom_adsp {
 	void *mem_region;
 	size_t mem_size;
 
+	struct qcom_rproc_glink glink_subdev;
 	struct qcom_rproc_subdev smd_subdev;
+	struct qcom_rproc_ssr ssr_subdev;
 };
 
 static int adsp_load(struct rproc *rproc, const struct firmware *fw)
@@ -266,10 +269,7 @@ static int adsp_init_regulator(struct qcom_adsp *adsp)
 	regulator_set_load(adsp->cx_supply, 100000);
 
 	adsp->px_supply = devm_regulator_get(adsp->dev, "px");
-	if (IS_ERR(adsp->px_supply))
-		return PTR_ERR(adsp->px_supply);
-
-	return 0;
+	return PTR_ERR_OR_ZERO(adsp->px_supply);
 }
 
 static int adsp_request_irq(struct qcom_adsp *adsp,
@@ -401,7 +401,9 @@ static int adsp_probe(struct platform_device *pdev)
 		goto free_rproc;
 	}
 
+	qcom_add_glink_subdev(rproc, &adsp->glink_subdev);
 	qcom_add_smd_subdev(rproc, &adsp->smd_subdev);
+	qcom_add_ssr_subdev(rproc, &adsp->ssr_subdev, desc->ssr_name);
 
 	ret = rproc_add(rproc);
 	if (ret)
@@ -422,7 +424,9 @@ static int adsp_remove(struct platform_device *pdev)
 	qcom_smem_state_put(adsp->state);
 	rproc_del(adsp->rproc);
 
+	qcom_remove_glink_subdev(adsp->rproc, &adsp->glink_subdev);
 	qcom_remove_smd_subdev(adsp->rproc, &adsp->smd_subdev);
+	qcom_remove_ssr_subdev(adsp->rproc, &adsp->ssr_subdev);
 	rproc_free(adsp->rproc);
 
 	return 0;
@@ -433,6 +437,7 @@ static const struct adsp_data adsp_resource_init = {
 		.firmware_name = "adsp.mdt",
 		.pas_id = 1,
 		.has_aggre2_clk = false,
+		.ssr_name = "lpass",
 };
 
 static const struct adsp_data slpi_resource_init = {
@@ -440,6 +445,7 @@ static const struct adsp_data slpi_resource_init = {
 		.firmware_name = "slpi.mdt",
 		.pas_id = 12,
 		.has_aggre2_clk = true,
+		.ssr_name = "dsps",
 };
 
 static const struct of_device_id adsp_of_match[] = {
diff --git a/drivers/remoteproc/qcom_common.c b/drivers/remoteproc/qcom_common.c
index bb90481215c6..d487040b528b 100644
--- a/drivers/remoteproc/qcom_common.c
+++ b/drivers/remoteproc/qcom_common.c
@@ -18,13 +18,19 @@
 #include <linux/firmware.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/notifier.h>
 #include <linux/remoteproc.h>
+#include <linux/rpmsg/qcom_glink.h>
 #include <linux/rpmsg/qcom_smd.h>
 
 #include "remoteproc_internal.h"
 #include "qcom_common.h"
 
+#define to_glink_subdev(d) container_of(d, struct qcom_rproc_glink, subdev)
 #define to_smd_subdev(d) container_of(d, struct qcom_rproc_subdev, subdev)
+#define to_ssr_subdev(d) container_of(d, struct qcom_rproc_ssr, subdev)
+
+static BLOCKING_NOTIFIER_HEAD(ssr_notifiers);
 
 /**
  * qcom_mdt_find_rsc_table() - provide dummy resource table for remoteproc
@@ -45,13 +51,60 @@ struct resource_table *qcom_mdt_find_rsc_table(struct rproc *rproc,
 }
 EXPORT_SYMBOL_GPL(qcom_mdt_find_rsc_table);
 
+static int glink_subdev_probe(struct rproc_subdev *subdev)
+{
+	struct qcom_rproc_glink *glink = to_glink_subdev(subdev);
+
+	glink->edge = qcom_glink_smem_register(glink->dev, glink->node);
+
+	return IS_ERR(glink->edge) ? PTR_ERR(glink->edge) : 0;
+}
+
+static void glink_subdev_remove(struct rproc_subdev *subdev)
+{
+	struct qcom_rproc_glink *glink = to_glink_subdev(subdev);
+
+	qcom_glink_smem_unregister(glink->edge);
+	glink->edge = NULL;
+}
+
+/**
+ * qcom_add_glink_subdev() - try to add a GLINK subdevice to rproc
+ * @rproc:	rproc handle to parent the subdevice
+ * @glink:	reference to a GLINK subdev context
+ */
+void qcom_add_glink_subdev(struct rproc *rproc, struct qcom_rproc_glink *glink)
+{
+	struct device *dev = &rproc->dev;
+
+	glink->node = of_get_child_by_name(dev->parent->of_node, "glink-edge");
+	if (!glink->node)
+		return;
+
+	glink->dev = dev;
+	rproc_add_subdev(rproc, &glink->subdev, glink_subdev_probe, glink_subdev_remove);
+}
+EXPORT_SYMBOL_GPL(qcom_add_glink_subdev);
+
+/**
+ * qcom_remove_glink_subdev() - remove a GLINK subdevice from rproc
+ * @rproc:	rproc handle
+ * @glink:	reference to a GLINK subdev context
+ */
+void qcom_remove_glink_subdev(struct rproc *rproc, struct qcom_rproc_glink *glink)
+{
+	rproc_remove_subdev(rproc, &glink->subdev);
+	of_node_put(glink->node);
+}
+EXPORT_SYMBOL_GPL(qcom_remove_glink_subdev);
+
 static int smd_subdev_probe(struct rproc_subdev *subdev)
 {
 	struct qcom_rproc_subdev *smd = to_smd_subdev(subdev);
 
 	smd->edge = qcom_smd_register_edge(smd->dev, smd->node);
 
-	return IS_ERR(smd->edge) ? PTR_ERR(smd->edge) : 0;
+	return PTR_ERR_OR_ZERO(smd->edge);
 }
 
 static void smd_subdev_remove(struct rproc_subdev *subdev)
@@ -92,5 +145,72 @@ void qcom_remove_smd_subdev(struct rproc *rproc, struct qcom_rproc_subdev *smd)
 }
 EXPORT_SYMBOL_GPL(qcom_remove_smd_subdev);
 
+/**
+ * qcom_register_ssr_notifier() - register SSR notification handler
+ * @nb:		notifier_block to notify for restart notifications
+ *
+ * Returns 0 on success, negative errno on failure.
+ *
+ * This register the @notify function as handler for restart notifications. As
+ * remote processors are stopped this function will be called, with the SSR
+ * name passed as a parameter.
+ */
+int qcom_register_ssr_notifier(struct notifier_block *nb)
+{
+	return blocking_notifier_chain_register(&ssr_notifiers, nb);
+}
+EXPORT_SYMBOL_GPL(qcom_register_ssr_notifier);
+
+/**
+ * qcom_unregister_ssr_notifier() - unregister SSR notification handler
+ * @nb:		notifier_block to unregister
+ */
+void qcom_unregister_ssr_notifier(struct notifier_block *nb)
+{
+	blocking_notifier_chain_unregister(&ssr_notifiers, nb);
+}
+EXPORT_SYMBOL_GPL(qcom_unregister_ssr_notifier);
+
+static int ssr_notify_start(struct rproc_subdev *subdev)
+{
+	return  0;
+}
+
+static void ssr_notify_stop(struct rproc_subdev *subdev)
+{
+	struct qcom_rproc_ssr *ssr = to_ssr_subdev(subdev);
+
+	blocking_notifier_call_chain(&ssr_notifiers, 0, (void *)ssr->name);
+}
+
+/**
+ * qcom_add_ssr_subdev() - register subdevice as restart notification source
+ * @rproc:	rproc handle
+ * @ssr:	SSR subdevice handle
+ * @ssr_name:	identifier to use for notifications originating from @rproc
+ *
+ * As the @ssr is registered with the @rproc SSR events will be sent to all
+ * registered listeners in the system as the remoteproc is shut down.
+ */
+void qcom_add_ssr_subdev(struct rproc *rproc, struct qcom_rproc_ssr *ssr,
+			 const char *ssr_name)
+{
+	ssr->name = ssr_name;
+
+	rproc_add_subdev(rproc, &ssr->subdev, ssr_notify_start, ssr_notify_stop);
+}
+EXPORT_SYMBOL_GPL(qcom_add_ssr_subdev);
+
+/**
+ * qcom_remove_ssr_subdev() - remove subdevice as restart notification source
+ * @rproc:	rproc handle
+ * @ssr:	SSR subdevice handle
+ */
+void qcom_remove_ssr_subdev(struct rproc *rproc, struct qcom_rproc_ssr *ssr)
+{
+	rproc_remove_subdev(rproc, &ssr->subdev);
+}
+EXPORT_SYMBOL_GPL(qcom_remove_ssr_subdev);
+
 MODULE_DESCRIPTION("Qualcomm Remoteproc helper driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/remoteproc/qcom_common.h b/drivers/remoteproc/qcom_common.h
index db5c826d5cd4..4f8bc168473c 100644
--- a/drivers/remoteproc/qcom_common.h
+++ b/drivers/remoteproc/qcom_common.h
@@ -4,6 +4,14 @@
 #include <linux/remoteproc.h>
 #include "remoteproc_internal.h"
 
+struct qcom_rproc_glink {
+	struct rproc_subdev subdev;
+
+	struct device *dev;
+	struct device_node *node;
+	struct qcom_glink *edge;
+};
+
 struct qcom_rproc_subdev {
 	struct rproc_subdev subdev;
 
@@ -12,11 +20,24 @@ struct qcom_rproc_subdev {
 	struct qcom_smd_edge *edge;
 };
 
+struct qcom_rproc_ssr {
+	struct rproc_subdev subdev;
+
+	const char *name;
+};
+
 struct resource_table *qcom_mdt_find_rsc_table(struct rproc *rproc,
 					       const struct firmware *fw,
 					       int *tablesz);
 
+void qcom_add_glink_subdev(struct rproc *rproc, struct qcom_rproc_glink *glink);
+void qcom_remove_glink_subdev(struct rproc *rproc, struct qcom_rproc_glink *glink);
+
 void qcom_add_smd_subdev(struct rproc *rproc, struct qcom_rproc_subdev *smd);
 void qcom_remove_smd_subdev(struct rproc *rproc, struct qcom_rproc_subdev *smd);
 
+void qcom_add_ssr_subdev(struct rproc *rproc, struct qcom_rproc_ssr *ssr,
+			 const char *ssr_name);
+void qcom_remove_ssr_subdev(struct rproc *rproc, struct qcom_rproc_ssr *ssr);
+
 #endif
diff --git a/drivers/remoteproc/qcom_q6v5_pil.c b/drivers/remoteproc/qcom_q6v5_pil.c
index 8fd697a3cf8f..2d3d5ac92c06 100644
--- a/drivers/remoteproc/qcom_q6v5_pil.c
+++ b/drivers/remoteproc/qcom_q6v5_pil.c
@@ -153,6 +153,7 @@ struct q6v5 {
 	size_t mpss_size;
 
 	struct qcom_rproc_subdev smd_subdev;
+	struct qcom_rproc_ssr ssr_subdev;
 };
 
 static int q6v5_regulator_init(struct device *dev, struct reg_info *regs,
@@ -867,7 +868,8 @@ static int q6v5_init_clocks(struct device *dev, struct clk **clks,
 
 static int q6v5_init_reset(struct q6v5 *qproc)
 {
-	qproc->mss_restart = devm_reset_control_get(qproc->dev, NULL);
+	qproc->mss_restart = devm_reset_control_get_exclusive(qproc->dev,
+							      NULL);
 	if (IS_ERR(qproc->mss_restart)) {
 		dev_err(qproc->dev, "failed to acquire mss restart\n");
 		return PTR_ERR(qproc->mss_restart);
@@ -1038,6 +1040,7 @@ static int q6v5_probe(struct platform_device *pdev)
 	}
 
 	qcom_add_smd_subdev(rproc, &qproc->smd_subdev);
+	qcom_add_ssr_subdev(rproc, &qproc->ssr_subdev, "mpss");
 
 	ret = rproc_add(rproc);
 	if (ret)
@@ -1058,6 +1061,7 @@ static int q6v5_remove(struct platform_device *pdev)
 	rproc_del(qproc->rproc);
 
 	qcom_remove_smd_subdev(qproc->rproc, &qproc->smd_subdev);
+	qcom_remove_ssr_subdev(qproc->rproc, &qproc->ssr_subdev);
 	rproc_free(qproc->rproc);
 
 	return 0;
diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c
index 564061dcc019..eab14b414bf0 100644
--- a/drivers/remoteproc/remoteproc_core.c
+++ b/drivers/remoteproc/remoteproc_core.c
@@ -794,7 +794,7 @@ static void rproc_remove_subdevices(struct rproc *rproc)
 {
 	struct rproc_subdev *subdev;
 
-	list_for_each_entry(subdev, &rproc->subdevs, node)
+	list_for_each_entry_reverse(subdev, &rproc->subdevs, node)
 		subdev->remove(subdev);
 }
 
@@ -1119,7 +1119,7 @@ static void rproc_crash_handler_work(struct work_struct *work)
 }
 
 /**
- * __rproc_boot() - boot a remote processor
+ * rproc_boot() - boot a remote processor
  * @rproc: handle of a remote processor
  *
  * Boot a remote processor (i.e. load its firmware, power it on, ...).
@@ -1129,7 +1129,7 @@ static void rproc_crash_handler_work(struct work_struct *work)
  *
  * Returns 0 on success, and an appropriate error value otherwise.
  */
-static int __rproc_boot(struct rproc *rproc)
+int rproc_boot(struct rproc *rproc)
 {
 	const struct firmware *firmware_p;
 	struct device *dev;
@@ -1180,15 +1180,6 @@ unlock_mutex:
 	mutex_unlock(&rproc->lock);
 	return ret;
 }
-
-/**
- * rproc_boot() - boot a remote processor
- * @rproc: handle of a remote processor
- */
-int rproc_boot(struct rproc *rproc)
-{
-	return __rproc_boot(rproc);
-}
 EXPORT_SYMBOL(rproc_boot);
 
 /**
@@ -1369,7 +1360,7 @@ static void rproc_type_release(struct device *dev)
 	kfree(rproc);
 }
 
-static struct device_type rproc_type = {
+static const struct device_type rproc_type = {
 	.name		= "remoteproc",
 	.release	= rproc_type_release,
 };
@@ -1440,6 +1431,7 @@ struct rproc *rproc_alloc(struct device *dev, const char *name,
 	rproc->dev.parent = dev;
 	rproc->dev.type = &rproc_type;
 	rproc->dev.class = &rproc_class;
+	rproc->dev.driver_data = rproc;
 
 	/* Assign a unique device index and name */
 	rproc->index = ida_simple_get(&rproc_dev_index, 0, 0, GFP_KERNEL);
@@ -1579,6 +1571,23 @@ void rproc_remove_subdev(struct rproc *rproc, struct rproc_subdev *subdev)
 EXPORT_SYMBOL(rproc_remove_subdev);
 
 /**
+ * rproc_get_by_child() - acquire rproc handle of @dev's ancestor
+ * @dev:	child device to find ancestor of
+ *
+ * Returns the ancestor rproc instance, or NULL if not found.
+ */
+struct rproc *rproc_get_by_child(struct device *dev)
+{
+	for (dev = dev->parent; dev; dev = dev->parent) {
+		if (dev->type == &rproc_type)
+			return dev->driver_data;
+	}
+
+	return NULL;
+}
+EXPORT_SYMBOL(rproc_get_by_child);
+
+/**
  * rproc_report_crash() - rproc crash reporter function
  * @rproc: remote processor
  * @type: crash type
diff --git a/drivers/remoteproc/remoteproc_internal.h b/drivers/remoteproc/remoteproc_internal.h
index 1e9e5b3f021c..c1077bec5d0b 100644
--- a/drivers/remoteproc/remoteproc_internal.h
+++ b/drivers/remoteproc/remoteproc_internal.h
@@ -48,7 +48,6 @@ struct rproc_fw_ops {
 /* from remoteproc_core.c */
 void rproc_release(struct kref *kref);
 irqreturn_t rproc_vq_interrupt(struct rproc *rproc, int vq_id);
-int rproc_boot_nowait(struct rproc *rproc);
 void rproc_vdev_release(struct kref *ref);
 
 /* from remoteproc_virtio.c */
diff --git a/drivers/remoteproc/st_remoteproc.c b/drivers/remoteproc/st_remoteproc.c
index d534bf23dc56..aacef0ea3b90 100644
--- a/drivers/remoteproc/st_remoteproc.c
+++ b/drivers/remoteproc/st_remoteproc.c
@@ -212,7 +212,8 @@ static int st_rproc_parse_dt(struct platform_device *pdev)
 	int err;
 
 	if (ddata->config->sw_reset) {
-		ddata->sw_reset = devm_reset_control_get(dev, "sw_reset");
+		ddata->sw_reset = devm_reset_control_get_exclusive(dev,
+								   "sw_reset");
 		if (IS_ERR(ddata->sw_reset)) {
 			dev_err(dev, "Failed to get S/W Reset\n");
 			return PTR_ERR(ddata->sw_reset);
@@ -220,7 +221,8 @@ static int st_rproc_parse_dt(struct platform_device *pdev)
 	}
 
 	if (ddata->config->pwr_reset) {
-		ddata->pwr_reset = devm_reset_control_get(dev, "pwr_reset");
+		ddata->pwr_reset = devm_reset_control_get_exclusive(dev,
+								    "pwr_reset");
 		if (IS_ERR(ddata->pwr_reset)) {
 			dev_err(dev, "Failed to get Power Reset\n");
 			return PTR_ERR(ddata->pwr_reset);
diff --git a/drivers/reset/Kconfig b/drivers/reset/Kconfig
index 608c071e4bbf..e0c393214264 100644
--- a/drivers/reset/Kconfig
+++ b/drivers/reset/Kconfig
@@ -34,12 +34,11 @@ config RESET_BERLIN
 	help
 	  This enables the reset controller driver for Marvell Berlin SoCs.
 
-config RESET_GEMINI
-	bool "Gemini Reset Driver" if COMPILE_TEST
-	default ARCH_GEMINI
-	select MFD_SYSCON
+config RESET_HSDK_V1
+	bool "HSDK v1 Reset Driver"
+	default n
 	help
-	  This enables the reset controller driver for Cortina Systems Gemini.
+	  This enables the reset controller driver for HSDK v1.
 
 config RESET_IMX7
 	bool "i.MX7 Reset Driver" if COMPILE_TEST
@@ -48,6 +47,12 @@ config RESET_IMX7
 	help
 	  This enables the reset controller driver for i.MX7 SoCs.
 
+config RESET_LANTIQ
+	bool "Lantiq XWAY Reset Driver" if COMPILE_TEST
+	default SOC_TYPE_XWAY
+	help
+	  This enables the reset controller driver for Lantiq / Intel XWAY SoCs.
+
 config RESET_LPC18XX
 	bool "LPC18xx/43xx Reset Driver" if COMPILE_TEST
 	default ARCH_LPC18XX
diff --git a/drivers/reset/Makefile b/drivers/reset/Makefile
index 7081f9da2599..d368367110e5 100644
--- a/drivers/reset/Makefile
+++ b/drivers/reset/Makefile
@@ -5,8 +5,9 @@ obj-$(CONFIG_ARCH_TEGRA) += tegra/
 obj-$(CONFIG_RESET_A10SR) += reset-a10sr.o
 obj-$(CONFIG_RESET_ATH79) += reset-ath79.o
 obj-$(CONFIG_RESET_BERLIN) += reset-berlin.o
-obj-$(CONFIG_RESET_GEMINI) += reset-gemini.o
+obj-$(CONFIG_RESET_HSDK_V1) += reset-hsdk-v1.o
 obj-$(CONFIG_RESET_IMX7) += reset-imx7.o
+obj-$(CONFIG_RESET_LANTIQ) += reset-lantiq.o
 obj-$(CONFIG_RESET_LPC18XX) += reset-lpc18xx.o
 obj-$(CONFIG_RESET_MESON) += reset-meson.o
 obj-$(CONFIG_RESET_OXNAS) += reset-oxnas.o
diff --git a/drivers/reset/core.c b/drivers/reset/core.c
index 0090784ff410..1d21c6f7d56c 100644
--- a/drivers/reset/core.c
+++ b/drivers/reset/core.c
@@ -43,11 +43,24 @@ struct reset_control {
 	unsigned int id;
 	struct kref refcnt;
 	bool shared;
+	bool array;
 	atomic_t deassert_count;
 	atomic_t triggered_count;
 };
 
 /**
+ * struct reset_control_array - an array of reset controls
+ * @base: reset control for compatibility with reset control API functions
+ * @num_rstcs: number of reset controls
+ * @rstc: array of reset controls
+ */
+struct reset_control_array {
+	struct reset_control base;
+	unsigned int num_rstcs;
+	struct reset_control *rstc[];
+};
+
+/**
  * of_reset_simple_xlate - translate reset_spec to the reset line number
  * @rcdev: a pointer to the reset controller device
  * @reset_spec: reset line specifier as found in the device tree
@@ -135,6 +148,65 @@ int devm_reset_controller_register(struct device *dev,
 }
 EXPORT_SYMBOL_GPL(devm_reset_controller_register);
 
+static inline struct reset_control_array *
+rstc_to_array(struct reset_control *rstc) {
+	return container_of(rstc, struct reset_control_array, base);
+}
+
+static int reset_control_array_reset(struct reset_control_array *resets)
+{
+	int ret, i;
+
+	for (i = 0; i < resets->num_rstcs; i++) {
+		ret = reset_control_reset(resets->rstc[i]);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int reset_control_array_assert(struct reset_control_array *resets)
+{
+	int ret, i;
+
+	for (i = 0; i < resets->num_rstcs; i++) {
+		ret = reset_control_assert(resets->rstc[i]);
+		if (ret)
+			goto err;
+	}
+
+	return 0;
+
+err:
+	while (i--)
+		reset_control_deassert(resets->rstc[i]);
+	return ret;
+}
+
+static int reset_control_array_deassert(struct reset_control_array *resets)
+{
+	int ret, i;
+
+	for (i = 0; i < resets->num_rstcs; i++) {
+		ret = reset_control_deassert(resets->rstc[i]);
+		if (ret)
+			goto err;
+	}
+
+	return 0;
+
+err:
+	while (i--)
+		reset_control_assert(resets->rstc[i]);
+	return ret;
+}
+
+static inline bool reset_control_is_array(struct reset_control *rstc)
+{
+	return rstc->array;
+}
+
 /**
  * reset_control_reset - reset the controlled device
  * @rstc: reset controller
@@ -158,6 +230,9 @@ int reset_control_reset(struct reset_control *rstc)
 	if (WARN_ON(IS_ERR(rstc)))
 		return -EINVAL;
 
+	if (reset_control_is_array(rstc))
+		return reset_control_array_reset(rstc_to_array(rstc));
+
 	if (!rstc->rcdev->ops->reset)
 		return -ENOTSUPP;
 
@@ -202,8 +277,8 @@ int reset_control_assert(struct reset_control *rstc)
 	if (WARN_ON(IS_ERR(rstc)))
 		return -EINVAL;
 
-	if (!rstc->rcdev->ops->assert)
-		return -ENOTSUPP;
+	if (reset_control_is_array(rstc))
+		return reset_control_array_assert(rstc_to_array(rstc));
 
 	if (rstc->shared) {
 		if (WARN_ON(atomic_read(&rstc->triggered_count) != 0))
@@ -214,6 +289,21 @@ int reset_control_assert(struct reset_control *rstc)
 
 		if (atomic_dec_return(&rstc->deassert_count) != 0)
 			return 0;
+
+		/*
+		 * Shared reset controls allow the reset line to be in any state
+		 * after this call, so doing nothing is a valid option.
+		 */
+		if (!rstc->rcdev->ops->assert)
+			return 0;
+	} else {
+		/*
+		 * If the reset controller does not implement .assert(), there
+		 * is no way to guarantee that the reset line is asserted after
+		 * this call.
+		 */
+		if (!rstc->rcdev->ops->assert)
+			return -ENOTSUPP;
 	}
 
 	return rstc->rcdev->ops->assert(rstc->rcdev, rstc->id);
@@ -240,8 +330,8 @@ int reset_control_deassert(struct reset_control *rstc)
 	if (WARN_ON(IS_ERR(rstc)))
 		return -EINVAL;
 
-	if (!rstc->rcdev->ops->deassert)
-		return -ENOTSUPP;
+	if (reset_control_is_array(rstc))
+		return reset_control_array_deassert(rstc_to_array(rstc));
 
 	if (rstc->shared) {
 		if (WARN_ON(atomic_read(&rstc->triggered_count) != 0))
@@ -251,6 +341,16 @@ int reset_control_deassert(struct reset_control *rstc)
 			return 0;
 	}
 
+	/*
+	 * If the reset controller does not implement .deassert(), we assume
+	 * that it handles self-deasserting reset lines via .reset(). In that
+	 * case, the reset lines are deasserted by default. If that is not the
+	 * case, the reset controller driver should implement .deassert() and
+	 * return -ENOTSUPP.
+	 */
+	if (!rstc->rcdev->ops->deassert)
+		return 0;
+
 	return rstc->rcdev->ops->deassert(rstc->rcdev, rstc->id);
 }
 EXPORT_SYMBOL_GPL(reset_control_deassert);
@@ -266,7 +366,7 @@ int reset_control_status(struct reset_control *rstc)
 	if (!rstc)
 		return 0;
 
-	if (WARN_ON(IS_ERR(rstc)))
+	if (WARN_ON(IS_ERR(rstc)) || reset_control_is_array(rstc))
 		return -EINVAL;
 
 	if (rstc->rcdev->ops->status)
@@ -404,6 +504,16 @@ struct reset_control *__reset_control_get(struct device *dev, const char *id,
 }
 EXPORT_SYMBOL_GPL(__reset_control_get);
 
+static void reset_control_array_put(struct reset_control_array *resets)
+{
+	int i;
+
+	mutex_lock(&reset_list_mutex);
+	for (i = 0; i < resets->num_rstcs; i++)
+		__reset_control_put_internal(resets->rstc[i]);
+	mutex_unlock(&reset_list_mutex);
+}
+
 /**
  * reset_control_put - free the reset controller
  * @rstc: reset controller
@@ -413,6 +523,11 @@ void reset_control_put(struct reset_control *rstc)
 	if (IS_ERR_OR_NULL(rstc))
 		return;
 
+	if (reset_control_is_array(rstc)) {
+		reset_control_array_put(rstc_to_array(rstc));
+		return;
+	}
+
 	mutex_lock(&reset_list_mutex);
 	__reset_control_put_internal(rstc);
 	mutex_unlock(&reset_list_mutex);
@@ -472,3 +587,116 @@ int device_reset(struct device *dev)
 	return ret;
 }
 EXPORT_SYMBOL_GPL(device_reset);
+
+/**
+ * APIs to manage an array of reset controls.
+ */
+/**
+ * of_reset_control_get_count - Count number of resets available with a device
+ *
+ * @node: device node that contains 'resets'.
+ *
+ * Returns positive reset count on success, or error number on failure and
+ * on count being zero.
+ */
+static int of_reset_control_get_count(struct device_node *node)
+{
+	int count;
+
+	if (!node)
+		return -EINVAL;
+
+	count = of_count_phandle_with_args(node, "resets", "#reset-cells");
+	if (count == 0)
+		count = -ENOENT;
+
+	return count;
+}
+
+/**
+ * of_reset_control_array_get - Get a list of reset controls using
+ *				device node.
+ *
+ * @np: device node for the device that requests the reset controls array
+ * @shared: whether reset controls are shared or not
+ * @optional: whether it is optional to get the reset controls
+ *
+ * Returns pointer to allocated reset_control_array on success or
+ * error on failure
+ */
+struct reset_control *
+of_reset_control_array_get(struct device_node *np, bool shared, bool optional)
+{
+	struct reset_control_array *resets;
+	struct reset_control *rstc;
+	int num, i;
+
+	num = of_reset_control_get_count(np);
+	if (num < 0)
+		return optional ? NULL : ERR_PTR(num);
+
+	resets = kzalloc(sizeof(*resets) + sizeof(resets->rstc[0]) * num,
+			 GFP_KERNEL);
+	if (!resets)
+		return ERR_PTR(-ENOMEM);
+
+	for (i = 0; i < num; i++) {
+		rstc = __of_reset_control_get(np, NULL, i, shared, optional);
+		if (IS_ERR(rstc))
+			goto err_rst;
+		resets->rstc[i] = rstc;
+	}
+	resets->num_rstcs = num;
+	resets->base.array = true;
+
+	return &resets->base;
+
+err_rst:
+	mutex_lock(&reset_list_mutex);
+	while (--i >= 0)
+		__reset_control_put_internal(resets->rstc[i]);
+	mutex_unlock(&reset_list_mutex);
+
+	kfree(resets);
+
+	return rstc;
+}
+EXPORT_SYMBOL_GPL(of_reset_control_array_get);
+
+/**
+ * devm_reset_control_array_get - Resource managed reset control array get
+ *
+ * @dev: device that requests the list of reset controls
+ * @shared: whether reset controls are shared or not
+ * @optional: whether it is optional to get the reset controls
+ *
+ * The reset control array APIs are intended for a list of resets
+ * that just have to be asserted or deasserted, without any
+ * requirements on the order.
+ *
+ * Returns pointer to allocated reset_control_array on success or
+ * error on failure
+ */
+struct reset_control *
+devm_reset_control_array_get(struct device *dev, bool shared, bool optional)
+{
+	struct reset_control **devres;
+	struct reset_control *rstc;
+
+	devres = devres_alloc(devm_reset_control_release, sizeof(*devres),
+			      GFP_KERNEL);
+	if (!devres)
+		return ERR_PTR(-ENOMEM);
+
+	rstc = of_reset_control_array_get(dev->of_node, shared, optional);
+	if (IS_ERR(rstc)) {
+		devres_free(devres);
+		return rstc;
+	}
+
+	*devres = rstc;
+	devres_add(dev, devres);
+
+	return rstc;
+}
+EXPORT_SYMBOL_GPL(devm_reset_control_array_get);
diff --git a/drivers/reset/reset-gemini.c b/drivers/reset/reset-gemini.c
deleted file mode 100644
index a2478997c75b..000000000000
--- a/drivers/reset/reset-gemini.c
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Cortina Gemini Reset controller driver
- * Copyright (C) 2017 Linus Walleij <linus.walleij@linaro.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#include <linux/err.h>
-#include <linux/init.h>
-#include <linux/mfd/syscon.h>
-#include <linux/regmap.h>
-#include <linux/of.h>
-#include <linux/platform_device.h>
-#include <linux/reset-controller.h>
-#include <dt-bindings/reset/cortina,gemini-reset.h>
-
-/**
- * struct gemini_reset - gemini reset controller
- * @map: regmap to access the containing system controller
- * @rcdev: reset controller device
- */
-struct gemini_reset {
-	struct regmap *map;
-	struct reset_controller_dev rcdev;
-};
-
-#define GEMINI_GLOBAL_SOFT_RESET 0x0c
-
-#define to_gemini_reset(p) \
-	container_of((p), struct gemini_reset, rcdev)
-
-/*
- * This is a self-deasserting reset controller.
- */
-static int gemini_reset(struct reset_controller_dev *rcdev,
-			unsigned long id)
-{
-	struct gemini_reset *gr = to_gemini_reset(rcdev);
-
-	/* Manual says to always set BIT 30 (CPU1) to 1 */
-	return regmap_write(gr->map,
-			    GEMINI_GLOBAL_SOFT_RESET,
-			    BIT(GEMINI_RESET_CPU1) | BIT(id));
-}
-
-static int gemini_reset_status(struct reset_controller_dev *rcdev,
-			     unsigned long id)
-{
-	struct gemini_reset *gr = to_gemini_reset(rcdev);
-	u32 val;
-	int ret;
-
-	ret = regmap_read(gr->map, GEMINI_GLOBAL_SOFT_RESET, &val);
-	if (ret)
-		return ret;
-
-	return !!(val & BIT(id));
-}
-
-static const struct reset_control_ops gemini_reset_ops = {
-	.reset = gemini_reset,
-	.status = gemini_reset_status,
-};
-
-static int gemini_reset_probe(struct platform_device *pdev)
-{
-	struct gemini_reset *gr;
-	struct device *dev = &pdev->dev;
-	struct device_node *np = dev->of_node;
-	int ret;
-
-	gr = devm_kzalloc(dev, sizeof(*gr), GFP_KERNEL);
-	if (!gr)
-		return -ENOMEM;
-
-	gr->map = syscon_node_to_regmap(np);
-	if (IS_ERR(gr->map)) {
-		ret = PTR_ERR(gr->map);
-		dev_err(dev, "unable to get regmap (%d)", ret);
-		return ret;
-	}
-	gr->rcdev.owner = THIS_MODULE;
-	gr->rcdev.nr_resets = 32;
-	gr->rcdev.ops = &gemini_reset_ops;
-	gr->rcdev.of_node = pdev->dev.of_node;
-
-	ret = devm_reset_controller_register(&pdev->dev, &gr->rcdev);
-	if (ret)
-		return ret;
-
-	dev_info(dev, "registered Gemini reset controller\n");
-	return 0;
-}
-
-static const struct of_device_id gemini_reset_dt_ids[] = {
-	{ .compatible = "cortina,gemini-syscon", },
-	{ /* sentinel */ },
-};
-
-static struct platform_driver gemini_reset_driver = {
-	.probe	= gemini_reset_probe,
-	.driver = {
-		.name		= "gemini-reset",
-		.of_match_table	= gemini_reset_dt_ids,
-		.suppress_bind_attrs = true,
-	},
-};
-builtin_platform_driver(gemini_reset_driver);
diff --git a/drivers/reset/reset-hsdk-v1.c b/drivers/reset/reset-hsdk-v1.c
new file mode 100644
index 000000000000..bca13e4bf622
--- /dev/null
+++ b/drivers/reset/reset-hsdk-v1.c
@@ -0,0 +1,137 @@
+/*
+ * Copyright (C) 2017 Synopsys.
+ *
+ * Synopsys HSDKv1 SDP reset driver.
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2. This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/reset-controller.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+
+#define to_hsdkv1_rst(p)	container_of((p), struct hsdkv1_rst, rcdev)
+
+struct hsdkv1_rst {
+	void __iomem			*regs_ctl;
+	void __iomem			*regs_rst;
+	spinlock_t			lock;
+	struct reset_controller_dev	rcdev;
+};
+
+static const u32 rst_map[] = {
+	BIT(16), /* APB_RST  */
+	BIT(17), /* AXI_RST  */
+	BIT(18), /* ETH_RST  */
+	BIT(19), /* USB_RST  */
+	BIT(20), /* SDIO_RST */
+	BIT(21), /* HDMI_RST */
+	BIT(22), /* GFX_RST  */
+	BIT(25), /* DMAC_RST */
+	BIT(31), /* EBI_RST  */
+};
+
+#define HSDK_MAX_RESETS			ARRAY_SIZE(rst_map)
+
+#define CGU_SYS_RST_CTRL		0x0
+#define CGU_IP_SW_RESET			0x0
+#define CGU_IP_SW_RESET_DELAY_SHIFT	16
+#define CGU_IP_SW_RESET_DELAY_MASK	GENMASK(31, CGU_IP_SW_RESET_DELAY_SHIFT)
+#define CGU_IP_SW_RESET_DELAY		0
+#define CGU_IP_SW_RESET_RESET		BIT(0)
+#define SW_RESET_TIMEOUT		10000
+
+static void hsdkv1_reset_config(struct hsdkv1_rst *rst, unsigned long id)
+{
+	writel(rst_map[id], rst->regs_ctl + CGU_SYS_RST_CTRL);
+}
+
+static int hsdkv1_reset_do(struct hsdkv1_rst *rst)
+{
+	u32 reg;
+
+	reg = readl(rst->regs_rst + CGU_IP_SW_RESET);
+	reg &= ~CGU_IP_SW_RESET_DELAY_MASK;
+	reg |= CGU_IP_SW_RESET_DELAY << CGU_IP_SW_RESET_DELAY_SHIFT;
+	reg |= CGU_IP_SW_RESET_RESET;
+	writel(reg, rst->regs_rst + CGU_IP_SW_RESET);
+
+	/* wait till reset bit is back to 0 */
+	return readl_poll_timeout_atomic(rst->regs_rst + CGU_IP_SW_RESET, reg,
+		!(reg & CGU_IP_SW_RESET_RESET), 5, SW_RESET_TIMEOUT);
+}
+
+static int hsdkv1_reset_reset(struct reset_controller_dev *rcdev,
+			      unsigned long id)
+{
+	struct hsdkv1_rst *rst = to_hsdkv1_rst(rcdev);
+	unsigned long flags;
+	int ret;
+
+	spin_lock_irqsave(&rst->lock, flags);
+	hsdkv1_reset_config(rst, id);
+	ret = hsdkv1_reset_do(rst);
+	spin_unlock_irqrestore(&rst->lock, flags);
+
+	return ret;
+}
+
+static const struct reset_control_ops hsdkv1_reset_ops = {
+	.reset	= hsdkv1_reset_reset,
+};
+
+static int hsdkv1_reset_probe(struct platform_device *pdev)
+{
+	struct hsdkv1_rst *rst;
+	struct resource *mem;
+
+	rst = devm_kzalloc(&pdev->dev, sizeof(*rst), GFP_KERNEL);
+	if (!rst)
+		return -ENOMEM;
+
+	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	rst->regs_ctl = devm_ioremap_resource(&pdev->dev, mem);
+	if (IS_ERR(rst->regs_ctl))
+		return PTR_ERR(rst->regs_ctl);
+
+	mem = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+	rst->regs_rst = devm_ioremap_resource(&pdev->dev, mem);
+	if (IS_ERR(rst->regs_rst))
+		return PTR_ERR(rst->regs_rst);
+
+	spin_lock_init(&rst->lock);
+
+	rst->rcdev.owner = THIS_MODULE;
+	rst->rcdev.ops = &hsdkv1_reset_ops;
+	rst->rcdev.of_node = pdev->dev.of_node;
+	rst->rcdev.nr_resets = HSDK_MAX_RESETS;
+	rst->rcdev.of_reset_n_cells = 1;
+
+	return reset_controller_register(&rst->rcdev);
+}
+
+static const struct of_device_id hsdkv1_reset_dt_match[] = {
+	{ .compatible = "snps,hsdk-v1.0-reset" },
+	{ },
+};
+
+static struct platform_driver hsdkv1_reset_driver = {
+	.probe	= hsdkv1_reset_probe,
+	.driver	= {
+		.name = "hsdk-v1.0-reset",
+		.of_match_table = hsdkv1_reset_dt_match,
+	},
+};
+builtin_platform_driver(hsdkv1_reset_driver);
+
+MODULE_AUTHOR("Eugeniy Paltsev <Eugeniy.Paltsev@synopsys.com>");
+MODULE_DESCRIPTION("Synopsys HSDKv1 SDP reset driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/reset/reset-lantiq.c b/drivers/reset/reset-lantiq.c
new file mode 100644
index 000000000000..11a582e50d30
--- /dev/null
+++ b/drivers/reset/reset-lantiq.c
@@ -0,0 +1,212 @@
+/*
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License version 2 as published
+ *  by the Free Software Foundation.
+ *
+ *  Copyright (C) 2010 John Crispin <blogic@phrozen.org>
+ *  Copyright (C) 2013-2015 Lantiq Beteiligungs-GmbH & Co.KG
+ *  Copyright (C) 2016 Martin Blumenstingl <martin.blumenstingl@googlemail.com>
+ *  Copyright (C) 2017 Hauke Mehrtens <hauke@hauke-m.de>
+ */
+
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/reset-controller.h>
+#include <linux/of_address.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
+
+#define LANTIQ_RCU_RESET_TIMEOUT	10000
+
+struct lantiq_rcu_reset_priv {
+	struct reset_controller_dev rcdev;
+	struct device *dev;
+	struct regmap *regmap;
+	u32 reset_offset;
+	u32 status_offset;
+};
+
+static struct lantiq_rcu_reset_priv *to_lantiq_rcu_reset_priv(
+	struct reset_controller_dev *rcdev)
+{
+	return container_of(rcdev, struct lantiq_rcu_reset_priv, rcdev);
+}
+
+static int lantiq_rcu_reset_status(struct reset_controller_dev *rcdev,
+				   unsigned long id)
+{
+	struct lantiq_rcu_reset_priv *priv = to_lantiq_rcu_reset_priv(rcdev);
+	unsigned int status = (id >> 8) & 0x1f;
+	u32 val;
+	int ret;
+
+	ret = regmap_read(priv->regmap, priv->status_offset, &val);
+	if (ret)
+		return ret;
+
+	return !!(val & BIT(status));
+}
+
+static int lantiq_rcu_reset_status_timeout(struct reset_controller_dev *rcdev,
+					   unsigned long id, bool assert)
+{
+	int ret;
+	int retry = LANTIQ_RCU_RESET_TIMEOUT;
+
+	do {
+		ret = lantiq_rcu_reset_status(rcdev, id);
+		if (ret < 0)
+			return ret;
+		if (ret == assert)
+			return 0;
+		usleep_range(20, 40);
+	} while (--retry);
+
+	return -ETIMEDOUT;
+}
+
+static int lantiq_rcu_reset_update(struct reset_controller_dev *rcdev,
+				   unsigned long id, bool assert)
+{
+	struct lantiq_rcu_reset_priv *priv = to_lantiq_rcu_reset_priv(rcdev);
+	unsigned int set = id & 0x1f;
+	u32 val = assert ? BIT(set) : 0;
+	int ret;
+
+	ret = regmap_update_bits(priv->regmap, priv->reset_offset, BIT(set),
+				 val);
+	if (ret) {
+		dev_err(priv->dev, "Failed to set reset bit %u\n", set);
+		return ret;
+	}
+
+
+	ret = lantiq_rcu_reset_status_timeout(rcdev, id, assert);
+	if (ret)
+		dev_err(priv->dev, "Failed to %s bit %u\n",
+			assert ? "assert" : "deassert", set);
+
+	return ret;
+}
+
+static int lantiq_rcu_reset_assert(struct reset_controller_dev *rcdev,
+			     unsigned long id)
+{
+	return lantiq_rcu_reset_update(rcdev, id, true);
+}
+
+static int lantiq_rcu_reset_deassert(struct reset_controller_dev *rcdev,
+			       unsigned long id)
+{
+	return lantiq_rcu_reset_update(rcdev, id, false);
+}
+
+static int lantiq_rcu_reset_reset(struct reset_controller_dev *rcdev,
+			    unsigned long id)
+{
+	int ret;
+
+	ret = lantiq_rcu_reset_assert(rcdev, id);
+	if (ret)
+		return ret;
+
+	return lantiq_rcu_reset_deassert(rcdev, id);
+}
+
+static const struct reset_control_ops lantiq_rcu_reset_ops = {
+	.assert = lantiq_rcu_reset_assert,
+	.deassert = lantiq_rcu_reset_deassert,
+	.status = lantiq_rcu_reset_status,
+	.reset	= lantiq_rcu_reset_reset,
+};
+
+static int lantiq_rcu_reset_of_parse(struct platform_device *pdev,
+			       struct lantiq_rcu_reset_priv *priv)
+{
+	struct device *dev = &pdev->dev;
+	const __be32 *offset;
+
+	priv->regmap = syscon_node_to_regmap(dev->of_node->parent);
+	if (IS_ERR(priv->regmap)) {
+		dev_err(&pdev->dev, "Failed to lookup RCU regmap\n");
+		return PTR_ERR(priv->regmap);
+	}
+
+	offset = of_get_address(dev->of_node, 0, NULL, NULL);
+	if (!offset) {
+		dev_err(&pdev->dev, "Failed to get RCU reset offset\n");
+		return -ENOENT;
+	}
+	priv->reset_offset = __be32_to_cpu(*offset);
+
+	offset = of_get_address(dev->of_node, 1, NULL, NULL);
+	if (!offset) {
+		dev_err(&pdev->dev, "Failed to get RCU status offset\n");
+		return -ENOENT;
+	}
+	priv->status_offset = __be32_to_cpu(*offset);
+
+	return 0;
+}
+
+static int lantiq_rcu_reset_xlate(struct reset_controller_dev *rcdev,
+				  const struct of_phandle_args *reset_spec)
+{
+	unsigned int status, set;
+
+	set = reset_spec->args[0];
+	status = reset_spec->args[1];
+
+	if (set >= rcdev->nr_resets || status >= rcdev->nr_resets)
+		return -EINVAL;
+
+	return (status << 8) | set;
+}
+
+static int lantiq_rcu_reset_probe(struct platform_device *pdev)
+{
+	struct lantiq_rcu_reset_priv *priv;
+	int err;
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->dev = &pdev->dev;
+	platform_set_drvdata(pdev, priv);
+
+	err = lantiq_rcu_reset_of_parse(pdev, priv);
+	if (err)
+		return err;
+
+	priv->rcdev.ops = &lantiq_rcu_reset_ops;
+	priv->rcdev.owner = THIS_MODULE;
+	priv->rcdev.of_node = pdev->dev.of_node;
+	priv->rcdev.nr_resets = 32;
+	priv->rcdev.of_xlate = lantiq_rcu_reset_xlate;
+	priv->rcdev.of_reset_n_cells = 2;
+
+	return reset_controller_register(&priv->rcdev);
+}
+
+static const struct of_device_id lantiq_rcu_reset_dt_ids[] = {
+	{ .compatible = "lantiq,danube-reset", },
+	{ .compatible = "lantiq,xrx200-reset", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, lantiq_rcu_reset_dt_ids);
+
+static struct platform_driver lantiq_rcu_reset_driver = {
+	.probe	= lantiq_rcu_reset_probe,
+	.driver = {
+		.name		= "lantiq-reset",
+		.of_match_table	= lantiq_rcu_reset_dt_ids,
+	},
+};
+module_platform_driver(lantiq_rcu_reset_driver);
+
+MODULE_AUTHOR("Martin Blumenstingl <martin.blumenstingl@googlemail.com>");
+MODULE_DESCRIPTION("Lantiq XWAY RCU Reset Controller Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/reset/reset-socfpga.c b/drivers/reset/reset-socfpga.c
index 07224c019892..c60904ff40b8 100644
--- a/drivers/reset/reset-socfpga.c
+++ b/drivers/reset/reset-socfpga.c
@@ -109,8 +109,8 @@ static int socfpga_reset_probe(struct platform_device *pdev)
 	 * Do not continue, when we encounter an old DT.
 	 */
 	if (!of_find_property(pdev->dev.of_node, "#reset-cells", NULL)) {
-		dev_err(&pdev->dev, "%s missing #reset-cells property\n",
-			pdev->dev.of_node->full_name);
+		dev_err(&pdev->dev, "%pOF missing #reset-cells property\n",
+			pdev->dev.of_node);
 		return -EINVAL;
 	}
 
diff --git a/drivers/reset/reset-sunxi.c b/drivers/reset/reset-sunxi.c
index cd585cd2f04d..2c7dd1fd08df 100644
--- a/drivers/reset/reset-sunxi.c
+++ b/drivers/reset/reset-sunxi.c
@@ -107,7 +107,7 @@ static int sunxi_reset_init(struct device_node *np)
 	spin_lock_init(&data->lock);
 
 	data->rcdev.owner = THIS_MODULE;
-	data->rcdev.nr_resets = size * 32;
+	data->rcdev.nr_resets = size * 8;
 	data->rcdev.ops = &sunxi_reset_ops;
 	data->rcdev.of_node = np;
 
@@ -162,7 +162,7 @@ static int sunxi_reset_probe(struct platform_device *pdev)
 	spin_lock_init(&data->lock);
 
 	data->rcdev.owner = THIS_MODULE;
-	data->rcdev.nr_resets = resource_size(res) * 32;
+	data->rcdev.nr_resets = resource_size(res) * 8;
 	data->rcdev.ops = &sunxi_reset_ops;
 	data->rcdev.of_node = pdev->dev.of_node;
 
diff --git a/drivers/reset/reset-uniphier.c b/drivers/reset/reset-uniphier.c
index c4ba89832796..bda2dd196ae5 100644
--- a/drivers/reset/reset-uniphier.c
+++ b/drivers/reset/reset-uniphier.c
@@ -50,59 +50,35 @@ struct uniphier_reset_data {
 	}
 
 /* System reset data */
-#define UNIPHIER_SLD3_SYS_RESET_NAND(id)		\
-	UNIPHIER_RESETX((id), 0x2004, 2)
-
-#define UNIPHIER_LD11_SYS_RESET_NAND(id)		\
-	UNIPHIER_RESETX((id), 0x200c, 0)
-
-#define UNIPHIER_LD11_SYS_RESET_EMMC(id)		\
-	UNIPHIER_RESETX((id), 0x200c, 2)
-
-#define UNIPHIER_SLD3_SYS_RESET_STDMAC(id)		\
-	UNIPHIER_RESETX((id), 0x2000, 10)
-
-#define UNIPHIER_LD11_SYS_RESET_STDMAC(id)		\
-	UNIPHIER_RESETX((id), 0x200c, 8)
-
-#define UNIPHIER_PRO4_SYS_RESET_GIO(id)			\
-	UNIPHIER_RESETX((id), 0x2000, 6)
-
-#define UNIPHIER_LD20_SYS_RESET_GIO(id)			\
-	UNIPHIER_RESETX((id), 0x200c, 5)
-
-#define UNIPHIER_PRO4_SYS_RESET_USB3(id, ch)		\
-	UNIPHIER_RESETX((id), 0x2000 + 0x4 * (ch), 17)
-
-static const struct uniphier_reset_data uniphier_sld3_sys_reset_data[] = {
-	UNIPHIER_SLD3_SYS_RESET_NAND(2),
-	UNIPHIER_SLD3_SYS_RESET_STDMAC(8),	/* Ether, HSC, MIO */
+static const struct uniphier_reset_data uniphier_ld4_sys_reset_data[] = {
+	UNIPHIER_RESETX(2, 0x2000, 2),		/* NAND */
+	UNIPHIER_RESETX(8, 0x2000, 10),		/* STDMAC (Ether, HSC, MIO) */
 	UNIPHIER_RESET_END,
 };
 
 static const struct uniphier_reset_data uniphier_pro4_sys_reset_data[] = {
-	UNIPHIER_SLD3_SYS_RESET_NAND(2),
-	UNIPHIER_SLD3_SYS_RESET_STDMAC(8),	/* HSC, MIO, RLE */
-	UNIPHIER_PRO4_SYS_RESET_GIO(12),	/* Ether, SATA, USB3 */
-	UNIPHIER_PRO4_SYS_RESET_USB3(14, 0),
-	UNIPHIER_PRO4_SYS_RESET_USB3(15, 1),
+	UNIPHIER_RESETX(2, 0x2000, 2),		/* NAND */
+	UNIPHIER_RESETX(8, 0x2000, 10),		/* STDMAC (HSC, MIO, RLE) */
+	UNIPHIER_RESETX(12, 0x2000, 6),		/* GIO (Ether, SATA, USB3) */
+	UNIPHIER_RESETX(14, 0x2000, 17),	/* USB30 */
+	UNIPHIER_RESETX(15, 0x2004, 17),	/* USB31 */
 	UNIPHIER_RESET_END,
 };
 
 static const struct uniphier_reset_data uniphier_pro5_sys_reset_data[] = {
-	UNIPHIER_SLD3_SYS_RESET_NAND(2),
-	UNIPHIER_SLD3_SYS_RESET_STDMAC(8),	/* HSC */
-	UNIPHIER_PRO4_SYS_RESET_GIO(12),	/* PCIe, USB3 */
-	UNIPHIER_PRO4_SYS_RESET_USB3(14, 0),
-	UNIPHIER_PRO4_SYS_RESET_USB3(15, 1),
+	UNIPHIER_RESETX(2, 0x2000, 2),		/* NAND */
+	UNIPHIER_RESETX(8, 0x2000, 10),		/* STDMAC (HSC) */
+	UNIPHIER_RESETX(12, 0x2000, 6),		/* GIO (PCIe, USB3) */
+	UNIPHIER_RESETX(14, 0x2000, 17),	/* USB30 */
+	UNIPHIER_RESETX(15, 0x2004, 17),	/* USB31 */
 	UNIPHIER_RESET_END,
 };
 
 static const struct uniphier_reset_data uniphier_pxs2_sys_reset_data[] = {
-	UNIPHIER_SLD3_SYS_RESET_NAND(2),
-	UNIPHIER_SLD3_SYS_RESET_STDMAC(8),	/* HSC, RLE */
-	UNIPHIER_PRO4_SYS_RESET_USB3(14, 0),
-	UNIPHIER_PRO4_SYS_RESET_USB3(15, 1),
+	UNIPHIER_RESETX(2, 0x2000, 2),		/* NAND */
+	UNIPHIER_RESETX(8, 0x2000, 10),		/* STDMAC (HSC, RLE) */
+	UNIPHIER_RESETX(14, 0x2000, 17),	/* USB30 */
+	UNIPHIER_RESETX(15, 0x2004, 17),	/* USB31 */
 	UNIPHIER_RESETX(16, 0x2014, 4),		/* USB30-PHY0 */
 	UNIPHIER_RESETX(17, 0x2014, 0),		/* USB30-PHY1 */
 	UNIPHIER_RESETX(18, 0x2014, 2),		/* USB30-PHY2 */
@@ -114,21 +90,27 @@ static const struct uniphier_reset_data uniphier_pxs2_sys_reset_data[] = {
 };
 
 static const struct uniphier_reset_data uniphier_ld11_sys_reset_data[] = {
-	UNIPHIER_LD11_SYS_RESET_NAND(2),
-	UNIPHIER_LD11_SYS_RESET_EMMC(4),
-	UNIPHIER_LD11_SYS_RESET_STDMAC(8),	/* HSC, MIO */
+	UNIPHIER_RESETX(2, 0x200c, 0),		/* NAND */
+	UNIPHIER_RESETX(4, 0x200c, 2),		/* eMMC */
+	UNIPHIER_RESETX(8, 0x200c, 8),		/* STDMAC (HSC, MIO) */
+	UNIPHIER_RESETX(40, 0x2008, 0),		/* AIO */
+	UNIPHIER_RESETX(41, 0x2008, 1),		/* EVEA */
+	UNIPHIER_RESETX(42, 0x2010, 2),		/* EXIV */
 	UNIPHIER_RESET_END,
 };
 
 static const struct uniphier_reset_data uniphier_ld20_sys_reset_data[] = {
-	UNIPHIER_LD11_SYS_RESET_NAND(2),
-	UNIPHIER_LD11_SYS_RESET_EMMC(4),
-	UNIPHIER_LD11_SYS_RESET_STDMAC(8),	/* HSC */
-	UNIPHIER_LD20_SYS_RESET_GIO(12),	/* PCIe, USB3 */
+	UNIPHIER_RESETX(2, 0x200c, 0),		/* NAND */
+	UNIPHIER_RESETX(4, 0x200c, 2),		/* eMMC */
+	UNIPHIER_RESETX(8, 0x200c, 8),		/* STDMAC (HSC) */
+	UNIPHIER_RESETX(12, 0x200c, 5),		/* GIO (PCIe, USB3) */
 	UNIPHIER_RESETX(16, 0x200c, 12),	/* USB30-PHY0 */
 	UNIPHIER_RESETX(17, 0x200c, 13),	/* USB30-PHY1 */
 	UNIPHIER_RESETX(18, 0x200c, 14),	/* USB30-PHY2 */
 	UNIPHIER_RESETX(19, 0x200c, 15),	/* USB30-PHY3 */
+	UNIPHIER_RESETX(40, 0x2008, 0),		/* AIO */
+	UNIPHIER_RESETX(41, 0x2008, 1),		/* EVEA */
+	UNIPHIER_RESETX(42, 0x2010, 2),		/* EXIV */
 	UNIPHIER_RESET_END,
 };
 
@@ -151,7 +133,7 @@ static const struct uniphier_reset_data uniphier_ld20_sys_reset_data[] = {
 #define UNIPHIER_MIO_RESET_DMAC(id)			\
 	UNIPHIER_RESETX((id), 0x110, 17)
 
-static const struct uniphier_reset_data uniphier_sld3_mio_reset_data[] = {
+static const struct uniphier_reset_data uniphier_ld4_mio_reset_data[] = {
 	UNIPHIER_MIO_RESET_SD(0, 0),
 	UNIPHIER_MIO_RESET_SD(1, 1),
 	UNIPHIER_MIO_RESET_SD(2, 2),
@@ -163,11 +145,9 @@ static const struct uniphier_reset_data uniphier_sld3_mio_reset_data[] = {
 	UNIPHIER_MIO_RESET_USB2(8, 0),
 	UNIPHIER_MIO_RESET_USB2(9, 1),
 	UNIPHIER_MIO_RESET_USB2(10, 2),
-	UNIPHIER_MIO_RESET_USB2(11, 3),
 	UNIPHIER_MIO_RESET_USB2_BRIDGE(12, 0),
 	UNIPHIER_MIO_RESET_USB2_BRIDGE(13, 1),
 	UNIPHIER_MIO_RESET_USB2_BRIDGE(14, 2),
-	UNIPHIER_MIO_RESET_USB2_BRIDGE(15, 3),
 	UNIPHIER_RESET_END,
 };
 
@@ -216,6 +196,12 @@ static const struct uniphier_reset_data uniphier_pro4_peri_reset_data[] = {
 	UNIPHIER_RESET_END,
 };
 
+/* Analog signal amplifiers reset data */
+static const struct uniphier_reset_data uniphier_ld11_adamv_reset_data[] = {
+	UNIPHIER_RESETX(0, 0x10, 6), /* EVEA */
+	UNIPHIER_RESET_END,
+};
+
 /* core implementaton */
 struct uniphier_reset_priv {
 	struct reset_controller_dev rcdev;
@@ -346,12 +332,8 @@ static int uniphier_reset_probe(struct platform_device *pdev)
 static const struct of_device_id uniphier_reset_match[] = {
 	/* System reset */
 	{
-		.compatible = "socionext,uniphier-sld3-reset",
-		.data = uniphier_sld3_sys_reset_data,
-	},
-	{
 		.compatible = "socionext,uniphier-ld4-reset",
-		.data = uniphier_sld3_sys_reset_data,
+		.data = uniphier_ld4_sys_reset_data,
 	},
 	{
 		.compatible = "socionext,uniphier-pro4-reset",
@@ -359,7 +341,7 @@ static const struct of_device_id uniphier_reset_match[] = {
 	},
 	{
 		.compatible = "socionext,uniphier-sld8-reset",
-		.data = uniphier_sld3_sys_reset_data,
+		.data = uniphier_ld4_sys_reset_data,
 	},
 	{
 		.compatible = "socionext,uniphier-pro5-reset",
@@ -379,20 +361,16 @@ static const struct of_device_id uniphier_reset_match[] = {
 	},
 	/* Media I/O reset, SD reset */
 	{
-		.compatible = "socionext,uniphier-sld3-mio-reset",
-		.data = uniphier_sld3_mio_reset_data,
-	},
-	{
 		.compatible = "socionext,uniphier-ld4-mio-reset",
-		.data = uniphier_sld3_mio_reset_data,
+		.data = uniphier_ld4_mio_reset_data,
 	},
 	{
 		.compatible = "socionext,uniphier-pro4-mio-reset",
-		.data = uniphier_sld3_mio_reset_data,
+		.data = uniphier_ld4_mio_reset_data,
 	},
 	{
 		.compatible = "socionext,uniphier-sld8-mio-reset",
-		.data = uniphier_sld3_mio_reset_data,
+		.data = uniphier_ld4_mio_reset_data,
 	},
 	{
 		.compatible = "socionext,uniphier-pro5-sd-reset",
@@ -404,7 +382,7 @@ static const struct of_device_id uniphier_reset_match[] = {
 	},
 	{
 		.compatible = "socionext,uniphier-ld11-mio-reset",
-		.data = uniphier_sld3_mio_reset_data,
+		.data = uniphier_ld4_mio_reset_data,
 	},
 	{
 		.compatible = "socionext,uniphier-ld11-sd-reset",
@@ -443,6 +421,15 @@ static const struct of_device_id uniphier_reset_match[] = {
 		.compatible = "socionext,uniphier-ld20-peri-reset",
 		.data = uniphier_pro4_peri_reset_data,
 	},
+	/* Analog signal amplifiers reset */
+	{
+		.compatible = "socionext,uniphier-ld11-adamv-reset",
+		.data = uniphier_ld11_adamv_reset_data,
+	},
+	{
+		.compatible = "socionext,uniphier-ld20-adamv-reset",
+		.data = uniphier_ld11_adamv_reset_data,
+	},
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, uniphier_reset_match);
diff --git a/drivers/reset/reset-zx2967.c b/drivers/reset/reset-zx2967.c
index 4dabb9ec4841..4f319f7753d4 100644
--- a/drivers/reset/reset-zx2967.c
+++ b/drivers/reset/reset-zx2967.c
@@ -55,7 +55,7 @@ static int zx2967_reset_deassert(struct reset_controller_dev *rcdev,
 	return zx2967_reset_act(rcdev, id, false);
 }
 
-static struct reset_control_ops zx2967_reset_ops = {
+static const struct reset_control_ops zx2967_reset_ops = {
 	.assert		= zx2967_reset_assert,
 	.deassert	= zx2967_reset_deassert,
 };
diff --git a/drivers/rpmsg/Kconfig b/drivers/rpmsg/Kconfig
index 1323a245763b..0fe6eac46512 100644
--- a/drivers/rpmsg/Kconfig
+++ b/drivers/rpmsg/Kconfig
@@ -13,9 +13,13 @@ config RPMSG_CHAR
 	  in /dev. They make it possible for user-space programs to send and
 	  receive rpmsg packets.
 
+config RPMSG_QCOM_GLINK_NATIVE
+	tristate
+	select RPMSG
+
 config RPMSG_QCOM_GLINK_RPM
 	tristate "Qualcomm RPM Glink driver"
-	select RPMSG
+        select RPMSG_QCOM_GLINK_NATIVE
 	depends on HAS_IOMEM
 	depends on MAILBOX
 	help
@@ -23,6 +27,16 @@ config RPMSG_QCOM_GLINK_RPM
 	  which serves as a channel for communication with the RPM in GLINK
 	  enabled systems.
 
+config RPMSG_QCOM_GLINK_SMEM
+	tristate "Qualcomm SMEM Glink driver"
+	select RPMSG_QCOM_GLINK_NATIVE
+	depends on MAILBOX
+	depends on QCOM_SMEM
+	help
+	  Say y here to enable support for the GLINK SMEM communication driver,
+	  which provides support for using the GLINK communication protocol
+	  over SMEM.
+
 config RPMSG_QCOM_SMD
 	tristate "Qualcomm Shared Memory Driver (SMD)"
 	depends on QCOM_SMEM
diff --git a/drivers/rpmsg/Makefile b/drivers/rpmsg/Makefile
index 28cc19088cc0..c71f4ab1ae17 100644
--- a/drivers/rpmsg/Makefile
+++ b/drivers/rpmsg/Makefile
@@ -1,5 +1,7 @@
 obj-$(CONFIG_RPMSG)		+= rpmsg_core.o
 obj-$(CONFIG_RPMSG_CHAR)	+= rpmsg_char.o
 obj-$(CONFIG_RPMSG_QCOM_GLINK_RPM) += qcom_glink_rpm.o
+obj-$(CONFIG_RPMSG_QCOM_GLINK_NATIVE) += qcom_glink_native.o
+obj-$(CONFIG_RPMSG_QCOM_GLINK_SMEM) += qcom_glink_smem.o
 obj-$(CONFIG_RPMSG_QCOM_SMD)	+= qcom_smd.o
 obj-$(CONFIG_RPMSG_VIRTIO)	+= virtio_rpmsg_bus.o
diff --git a/drivers/rpmsg/qcom_glink_native.c b/drivers/rpmsg/qcom_glink_native.c
new file mode 100644
index 000000000000..5a5e927ea50f
--- /dev/null
+++ b/drivers/rpmsg/qcom_glink_native.c
@@ -0,0 +1,1612 @@
+/*
+ * Copyright (c) 2016-2017, Linaro Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/idr.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/list.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/rpmsg.h>
+#include <linux/sizes.h>
+#include <linux/slab.h>
+#include <linux/workqueue.h>
+#include <linux/mailbox_client.h>
+
+#include "rpmsg_internal.h"
+#include "qcom_glink_native.h"
+
+#define GLINK_NAME_SIZE		32
+#define GLINK_VERSION_1		1
+
+#define RPM_GLINK_CID_MIN	1
+#define RPM_GLINK_CID_MAX	65536
+
+struct glink_msg {
+	__le16 cmd;
+	__le16 param1;
+	__le32 param2;
+	u8 data[];
+} __packed;
+
+/**
+ * struct glink_defer_cmd - deferred incoming control message
+ * @node:	list node
+ * @msg:	message header
+ * data:	payload of the message
+ *
+ * Copy of a received control message, to be added to @rx_queue and processed
+ * by @rx_work of @qcom_glink.
+ */
+struct glink_defer_cmd {
+	struct list_head node;
+
+	struct glink_msg msg;
+	u8 data[];
+};
+
+/**
+ * struct glink_core_rx_intent - RX intent
+ * RX intent
+ *
+ * data: pointer to the data (may be NULL for zero-copy)
+ * id: remote or local intent ID
+ * size: size of the original intent (do not modify)
+ * reuse: To mark if the intent can be reused after first use
+ * in_use: To mark if intent is already in use for the channel
+ * offset: next write offset (initially 0)
+ */
+struct glink_core_rx_intent {
+	void *data;
+	u32 id;
+	size_t size;
+	bool reuse;
+	bool in_use;
+	u32 offset;
+
+	struct list_head node;
+};
+
+/**
+ * struct qcom_glink - driver context, relates to one remote subsystem
+ * @dev:	reference to the associated struct device
+ * @mbox_client: mailbox client
+ * @mbox_chan:  mailbox channel
+ * @rx_pipe:	pipe object for receive FIFO
+ * @tx_pipe:	pipe object for transmit FIFO
+ * @irq:	IRQ for signaling incoming events
+ * @rx_work:	worker for handling received control messages
+ * @rx_lock:	protects the @rx_queue
+ * @rx_queue:	queue of received control messages to be processed in @rx_work
+ * @tx_lock:	synchronizes operations on the tx fifo
+ * @idr_lock:	synchronizes @lcids and @rcids modifications
+ * @lcids:	idr of all channels with a known local channel id
+ * @rcids:	idr of all channels with a known remote channel id
+ */
+struct qcom_glink {
+	struct device *dev;
+
+	struct mbox_client mbox_client;
+	struct mbox_chan *mbox_chan;
+
+	struct qcom_glink_pipe *rx_pipe;
+	struct qcom_glink_pipe *tx_pipe;
+
+	int irq;
+
+	struct work_struct rx_work;
+	spinlock_t rx_lock;
+	struct list_head rx_queue;
+
+	struct mutex tx_lock;
+
+	spinlock_t idr_lock;
+	struct idr lcids;
+	struct idr rcids;
+	unsigned long features;
+
+	bool intentless;
+};
+
+enum {
+	GLINK_STATE_CLOSED,
+	GLINK_STATE_OPENING,
+	GLINK_STATE_OPEN,
+	GLINK_STATE_CLOSING,
+};
+
+/**
+ * struct glink_channel - internal representation of a channel
+ * @rpdev:	rpdev reference, only used for primary endpoints
+ * @ept:	rpmsg endpoint this channel is associated with
+ * @glink:	qcom_glink context handle
+ * @refcount:	refcount for the channel object
+ * @recv_lock:	guard for @ept.cb
+ * @name:	unique channel name/identifier
+ * @lcid:	channel id, in local space
+ * @rcid:	channel id, in remote space
+ * @intent_lock: lock for protection of @liids, @riids
+ * @liids:	idr of all local intents
+ * @riids:	idr of all remote intents
+ * @intent_work: worker responsible for transmitting rx_done packets
+ * @done_intents: list of intents that needs to be announced rx_done
+ * @buf:	receive buffer, for gathering fragments
+ * @buf_offset:	write offset in @buf
+ * @buf_size:	size of current @buf
+ * @open_ack:	completed once remote has acked the open-request
+ * @open_req:	completed once open-request has been received
+ * @intent_req_lock: Synchronises multiple intent requests
+ * @intent_req_result: Result of intent request
+ * @intent_req_comp: Completion for intent_req signalling
+ */
+struct glink_channel {
+	struct rpmsg_endpoint ept;
+
+	struct rpmsg_device *rpdev;
+	struct qcom_glink *glink;
+
+	struct kref refcount;
+
+	spinlock_t recv_lock;
+
+	char *name;
+	unsigned int lcid;
+	unsigned int rcid;
+
+	spinlock_t intent_lock;
+	struct idr liids;
+	struct idr riids;
+	struct work_struct intent_work;
+	struct list_head done_intents;
+
+	struct glink_core_rx_intent *buf;
+	int buf_offset;
+	int buf_size;
+
+	struct completion open_ack;
+	struct completion open_req;
+
+	struct mutex intent_req_lock;
+	bool intent_req_result;
+	struct completion intent_req_comp;
+};
+
+#define to_glink_channel(_ept) container_of(_ept, struct glink_channel, ept)
+
+static const struct rpmsg_endpoint_ops glink_endpoint_ops;
+
+#define RPM_CMD_VERSION			0
+#define RPM_CMD_VERSION_ACK		1
+#define RPM_CMD_OPEN			2
+#define RPM_CMD_CLOSE			3
+#define RPM_CMD_OPEN_ACK		4
+#define RPM_CMD_INTENT			5
+#define RPM_CMD_RX_DONE			6
+#define RPM_CMD_RX_INTENT_REQ		7
+#define RPM_CMD_RX_INTENT_REQ_ACK	8
+#define RPM_CMD_TX_DATA			9
+#define RPM_CMD_CLOSE_ACK		11
+#define RPM_CMD_TX_DATA_CONT		12
+#define RPM_CMD_READ_NOTIF		13
+#define RPM_CMD_RX_DONE_W_REUSE		14
+
+#define GLINK_FEATURE_INTENTLESS	BIT(1)
+
+static void qcom_glink_rx_done_work(struct work_struct *work);
+
+static struct glink_channel *qcom_glink_alloc_channel(struct qcom_glink *glink,
+						      const char *name)
+{
+	struct glink_channel *channel;
+
+	channel = kzalloc(sizeof(*channel), GFP_KERNEL);
+	if (!channel)
+		return ERR_PTR(-ENOMEM);
+
+	/* Setup glink internal glink_channel data */
+	spin_lock_init(&channel->recv_lock);
+	spin_lock_init(&channel->intent_lock);
+
+	channel->glink = glink;
+	channel->name = kstrdup(name, GFP_KERNEL);
+
+	init_completion(&channel->open_req);
+	init_completion(&channel->open_ack);
+
+	INIT_LIST_HEAD(&channel->done_intents);
+	INIT_WORK(&channel->intent_work, qcom_glink_rx_done_work);
+
+	idr_init(&channel->liids);
+	idr_init(&channel->riids);
+	kref_init(&channel->refcount);
+
+	return channel;
+}
+
+static void qcom_glink_channel_release(struct kref *ref)
+{
+	struct glink_channel *channel = container_of(ref, struct glink_channel,
+						     refcount);
+	unsigned long flags;
+
+	spin_lock_irqsave(&channel->intent_lock, flags);
+	idr_destroy(&channel->liids);
+	idr_destroy(&channel->riids);
+	spin_unlock_irqrestore(&channel->intent_lock, flags);
+
+	kfree(channel->name);
+	kfree(channel);
+}
+
+static size_t qcom_glink_rx_avail(struct qcom_glink *glink)
+{
+	return glink->rx_pipe->avail(glink->rx_pipe);
+}
+
+static void qcom_glink_rx_peak(struct qcom_glink *glink,
+			       void *data, unsigned int offset, size_t count)
+{
+	glink->rx_pipe->peak(glink->rx_pipe, data, offset, count);
+}
+
+static void qcom_glink_rx_advance(struct qcom_glink *glink, size_t count)
+{
+	glink->rx_pipe->advance(glink->rx_pipe, count);
+}
+
+static size_t qcom_glink_tx_avail(struct qcom_glink *glink)
+{
+	return glink->tx_pipe->avail(glink->tx_pipe);
+}
+
+static void qcom_glink_tx_write(struct qcom_glink *glink,
+				const void *hdr, size_t hlen,
+				const void *data, size_t dlen)
+{
+	glink->tx_pipe->write(glink->tx_pipe, hdr, hlen, data, dlen);
+}
+
+static int qcom_glink_tx(struct qcom_glink *glink,
+			 const void *hdr, size_t hlen,
+			 const void *data, size_t dlen, bool wait)
+{
+	unsigned int tlen = hlen + dlen;
+	int ret;
+
+	/* Reject packets that are too big */
+	if (tlen >= glink->tx_pipe->length)
+		return -EINVAL;
+
+	ret = mutex_lock_interruptible(&glink->tx_lock);
+	if (ret)
+		return ret;
+
+	while (qcom_glink_tx_avail(glink) < tlen) {
+		if (!wait) {
+			ret = -EAGAIN;
+			goto out;
+		}
+
+		usleep_range(10000, 15000);
+	}
+
+	qcom_glink_tx_write(glink, hdr, hlen, data, dlen);
+
+	mbox_send_message(glink->mbox_chan, NULL);
+	mbox_client_txdone(glink->mbox_chan, 0);
+
+out:
+	mutex_unlock(&glink->tx_lock);
+
+	return ret;
+}
+
+static int qcom_glink_send_version(struct qcom_glink *glink)
+{
+	struct glink_msg msg;
+
+	msg.cmd = cpu_to_le16(RPM_CMD_VERSION);
+	msg.param1 = cpu_to_le16(GLINK_VERSION_1);
+	msg.param2 = cpu_to_le32(glink->features);
+
+	return qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
+}
+
+static void qcom_glink_send_version_ack(struct qcom_glink *glink)
+{
+	struct glink_msg msg;
+
+	msg.cmd = cpu_to_le16(RPM_CMD_VERSION_ACK);
+	msg.param1 = cpu_to_le16(GLINK_VERSION_1);
+	msg.param2 = cpu_to_le32(glink->features);
+
+	qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
+}
+
+static void qcom_glink_send_open_ack(struct qcom_glink *glink,
+				     struct glink_channel *channel)
+{
+	struct glink_msg msg;
+
+	msg.cmd = cpu_to_le16(RPM_CMD_OPEN_ACK);
+	msg.param1 = cpu_to_le16(channel->rcid);
+	msg.param2 = cpu_to_le32(0);
+
+	qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
+}
+
+static void qcom_glink_handle_intent_req_ack(struct qcom_glink *glink,
+					     unsigned int cid, bool granted)
+{
+	struct glink_channel *channel;
+	unsigned long flags;
+
+	spin_lock_irqsave(&glink->idr_lock, flags);
+	channel = idr_find(&glink->rcids, cid);
+	spin_unlock_irqrestore(&glink->idr_lock, flags);
+	if (!channel) {
+		dev_err(glink->dev, "unable to find channel\n");
+		return;
+	}
+
+	channel->intent_req_result = granted;
+	complete(&channel->intent_req_comp);
+}
+
+/**
+ * qcom_glink_send_open_req() - send a RPM_CMD_OPEN request to the remote
+ * @glink: Ptr to the glink edge
+ * @channel: Ptr to the channel that the open req is sent
+ *
+ * Allocates a local channel id and sends a RPM_CMD_OPEN message to the remote.
+ * Will return with refcount held, regardless of outcome.
+ *
+ * Returns 0 on success, negative errno otherwise.
+ */
+static int qcom_glink_send_open_req(struct qcom_glink *glink,
+				    struct glink_channel *channel)
+{
+	struct {
+		struct glink_msg msg;
+		u8 name[GLINK_NAME_SIZE];
+	} __packed req;
+	int name_len = strlen(channel->name) + 1;
+	int req_len = ALIGN(sizeof(req.msg) + name_len, 8);
+	int ret;
+	unsigned long flags;
+
+	kref_get(&channel->refcount);
+
+	spin_lock_irqsave(&glink->idr_lock, flags);
+	ret = idr_alloc_cyclic(&glink->lcids, channel,
+			       RPM_GLINK_CID_MIN, RPM_GLINK_CID_MAX,
+			       GFP_ATOMIC);
+	spin_unlock_irqrestore(&glink->idr_lock, flags);
+	if (ret < 0)
+		return ret;
+
+	channel->lcid = ret;
+
+	req.msg.cmd = cpu_to_le16(RPM_CMD_OPEN);
+	req.msg.param1 = cpu_to_le16(channel->lcid);
+	req.msg.param2 = cpu_to_le32(name_len);
+	strcpy(req.name, channel->name);
+
+	ret = qcom_glink_tx(glink, &req, req_len, NULL, 0, true);
+	if (ret)
+		goto remove_idr;
+
+	return 0;
+
+remove_idr:
+	spin_lock_irqsave(&glink->idr_lock, flags);
+	idr_remove(&glink->lcids, channel->lcid);
+	channel->lcid = 0;
+	spin_unlock_irqrestore(&glink->idr_lock, flags);
+
+	return ret;
+}
+
+static void qcom_glink_send_close_req(struct qcom_glink *glink,
+				      struct glink_channel *channel)
+{
+	struct glink_msg req;
+
+	req.cmd = cpu_to_le16(RPM_CMD_CLOSE);
+	req.param1 = cpu_to_le16(channel->lcid);
+	req.param2 = 0;
+
+	qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true);
+}
+
+static void qcom_glink_send_close_ack(struct qcom_glink *glink,
+				      unsigned int rcid)
+{
+	struct glink_msg req;
+
+	req.cmd = cpu_to_le16(RPM_CMD_CLOSE_ACK);
+	req.param1 = cpu_to_le16(rcid);
+	req.param2 = 0;
+
+	qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true);
+}
+
+static void qcom_glink_rx_done_work(struct work_struct *work)
+{
+	struct glink_channel *channel = container_of(work, struct glink_channel,
+						     intent_work);
+	struct qcom_glink *glink = channel->glink;
+	struct glink_core_rx_intent *intent, *tmp;
+	struct {
+		u16 id;
+		u16 lcid;
+		u32 liid;
+	} __packed cmd;
+
+	unsigned int cid = channel->lcid;
+	unsigned int iid;
+	bool reuse;
+	unsigned long flags;
+
+	spin_lock_irqsave(&channel->intent_lock, flags);
+	list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) {
+		list_del(&intent->node);
+		spin_unlock_irqrestore(&channel->intent_lock, flags);
+		iid = intent->id;
+		reuse = intent->reuse;
+
+		cmd.id = reuse ? RPM_CMD_RX_DONE_W_REUSE : RPM_CMD_RX_DONE;
+		cmd.lcid = cid;
+		cmd.liid = iid;
+
+		qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
+		if (!reuse) {
+			kfree(intent->data);
+			kfree(intent);
+		}
+		spin_lock_irqsave(&channel->intent_lock, flags);
+	}
+	spin_unlock_irqrestore(&channel->intent_lock, flags);
+}
+
+static void qcom_glink_rx_done(struct qcom_glink *glink,
+			       struct glink_channel *channel,
+			       struct glink_core_rx_intent *intent)
+{
+	/* We don't send RX_DONE to intentless systems */
+	if (glink->intentless) {
+		kfree(intent->data);
+		kfree(intent);
+		return;
+	}
+
+	/* Take it off the tree of receive intents */
+	if (!intent->reuse) {
+		spin_lock(&channel->intent_lock);
+		idr_remove(&channel->liids, intent->id);
+		spin_unlock(&channel->intent_lock);
+	}
+
+	/* Schedule the sending of a rx_done indication */
+	spin_lock(&channel->intent_lock);
+	list_add_tail(&intent->node, &channel->done_intents);
+	spin_unlock(&channel->intent_lock);
+
+	schedule_work(&channel->intent_work);
+}
+
+/**
+ * qcom_glink_receive_version() - receive version/features from remote system
+ *
+ * @glink:	pointer to transport interface
+ * @r_version:	remote version
+ * @r_features:	remote features
+ *
+ * This function is called in response to a remote-initiated version/feature
+ * negotiation sequence.
+ */
+static void qcom_glink_receive_version(struct qcom_glink *glink,
+				       u32 version,
+				       u32 features)
+{
+	switch (version) {
+	case 0:
+		break;
+	case GLINK_VERSION_1:
+		glink->features &= features;
+		/* FALLTHROUGH */
+	default:
+		qcom_glink_send_version_ack(glink);
+		break;
+	}
+}
+
+/**
+ * qcom_glink_receive_version_ack() - receive negotiation ack from remote system
+ *
+ * @glink:	pointer to transport interface
+ * @r_version:	remote version response
+ * @r_features:	remote features response
+ *
+ * This function is called in response to a local-initiated version/feature
+ * negotiation sequence and is the counter-offer from the remote side based
+ * upon the initial version and feature set requested.
+ */
+static void qcom_glink_receive_version_ack(struct qcom_glink *glink,
+					   u32 version,
+					   u32 features)
+{
+	switch (version) {
+	case 0:
+		/* Version negotiation failed */
+		break;
+	case GLINK_VERSION_1:
+		if (features == glink->features)
+			break;
+
+		glink->features &= features;
+		/* FALLTHROUGH */
+	default:
+		qcom_glink_send_version(glink);
+		break;
+	}
+}
+
+/**
+ * qcom_glink_send_intent_req_ack() - convert an rx intent request ack cmd to
+				      wire format and transmit
+ * @glink:	The transport to transmit on.
+ * @channel:	The glink channel
+ * @granted:	The request response to encode.
+ *
+ * Return: 0 on success or standard Linux error code.
+ */
+static int qcom_glink_send_intent_req_ack(struct qcom_glink *glink,
+					  struct glink_channel *channel,
+					  bool granted)
+{
+	struct glink_msg msg;
+
+	msg.cmd = cpu_to_le16(RPM_CMD_RX_INTENT_REQ_ACK);
+	msg.param1 = cpu_to_le16(channel->lcid);
+	msg.param2 = cpu_to_le32(granted);
+
+	qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
+
+	return 0;
+}
+
+/**
+ * qcom_glink_advertise_intent - convert an rx intent cmd to wire format and
+ *			   transmit
+ * @glink:	The transport to transmit on.
+ * @channel:	The local channel
+ * @size:	The intent to pass on to remote.
+ *
+ * Return: 0 on success or standard Linux error code.
+ */
+static int qcom_glink_advertise_intent(struct qcom_glink *glink,
+				       struct glink_channel *channel,
+				       struct glink_core_rx_intent *intent)
+{
+	struct command {
+		u16 id;
+		u16 lcid;
+		u32 count;
+		u32 size;
+		u32 liid;
+	} __packed;
+	struct command cmd;
+
+	cmd.id = cpu_to_le16(RPM_CMD_INTENT);
+	cmd.lcid = cpu_to_le16(channel->lcid);
+	cmd.count = cpu_to_le32(1);
+	cmd.size = cpu_to_le32(intent->size);
+	cmd.liid = cpu_to_le32(intent->id);
+
+	qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
+
+	return 0;
+}
+
+static struct glink_core_rx_intent *
+qcom_glink_alloc_intent(struct qcom_glink *glink,
+			struct glink_channel *channel,
+			size_t size,
+			bool reuseable)
+{
+	struct glink_core_rx_intent *intent;
+	int ret;
+	unsigned long flags;
+
+	intent = kzalloc(sizeof(*intent), GFP_KERNEL);
+
+	if (!intent)
+		return NULL;
+
+	intent->data = kzalloc(size, GFP_KERNEL);
+	if (!intent->data)
+		return NULL;
+
+	spin_lock_irqsave(&channel->intent_lock, flags);
+	ret = idr_alloc_cyclic(&channel->liids, intent, 1, -1, GFP_ATOMIC);
+	if (ret < 0) {
+		spin_unlock_irqrestore(&channel->intent_lock, flags);
+		return NULL;
+	}
+	spin_unlock_irqrestore(&channel->intent_lock, flags);
+
+	intent->id = ret;
+	intent->size = size;
+	intent->reuse = reuseable;
+
+	return intent;
+}
+
+static void qcom_glink_handle_rx_done(struct qcom_glink *glink,
+				      u32 cid, uint32_t iid,
+				      bool reuse)
+{
+	struct glink_core_rx_intent *intent;
+	struct glink_channel *channel;
+	unsigned long flags;
+
+	spin_lock_irqsave(&glink->idr_lock, flags);
+	channel = idr_find(&glink->rcids, cid);
+	spin_unlock_irqrestore(&glink->idr_lock, flags);
+	if (!channel) {
+		dev_err(glink->dev, "invalid channel id received\n");
+		return;
+	}
+
+	spin_lock_irqsave(&channel->intent_lock, flags);
+	intent = idr_find(&channel->riids, iid);
+
+	if (!intent) {
+		spin_unlock_irqrestore(&channel->intent_lock, flags);
+		dev_err(glink->dev, "invalid intent id received\n");
+		return;
+	}
+
+	intent->in_use = false;
+
+	if (!reuse) {
+		idr_remove(&channel->riids, intent->id);
+		kfree(intent);
+	}
+	spin_unlock_irqrestore(&channel->intent_lock, flags);
+}
+
+/**
+ * qcom_glink_handle_intent_req() - Receive a request for rx_intent
+ *					    from remote side
+ * if_ptr:      Pointer to the transport interface
+ * rcid:	Remote channel ID
+ * size:	size of the intent
+ *
+ * The function searches for the local channel to which the request for
+ * rx_intent has arrived and allocates and notifies the remote back
+ */
+static void qcom_glink_handle_intent_req(struct qcom_glink *glink,
+					 u32 cid, size_t size)
+{
+	struct glink_core_rx_intent *intent;
+	struct glink_channel *channel;
+	unsigned long flags;
+
+	spin_lock_irqsave(&glink->idr_lock, flags);
+	channel = idr_find(&glink->rcids, cid);
+	spin_unlock_irqrestore(&glink->idr_lock, flags);
+
+	if (!channel) {
+		pr_err("%s channel not found for cid %d\n", __func__, cid);
+		return;
+	}
+
+	intent = qcom_glink_alloc_intent(glink, channel, size, false);
+	if (intent)
+		qcom_glink_advertise_intent(glink, channel, intent);
+
+	qcom_glink_send_intent_req_ack(glink, channel, !!intent);
+}
+
+static int qcom_glink_rx_defer(struct qcom_glink *glink, size_t extra)
+{
+	struct glink_defer_cmd *dcmd;
+
+	extra = ALIGN(extra, 8);
+
+	if (qcom_glink_rx_avail(glink) < sizeof(struct glink_msg) + extra) {
+		dev_dbg(glink->dev, "Insufficient data in rx fifo");
+		return -ENXIO;
+	}
+
+	dcmd = kzalloc(sizeof(*dcmd) + extra, GFP_ATOMIC);
+	if (!dcmd)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&dcmd->node);
+
+	qcom_glink_rx_peak(glink, &dcmd->msg, 0, sizeof(dcmd->msg) + extra);
+
+	spin_lock(&glink->rx_lock);
+	list_add_tail(&dcmd->node, &glink->rx_queue);
+	spin_unlock(&glink->rx_lock);
+
+	schedule_work(&glink->rx_work);
+	qcom_glink_rx_advance(glink, sizeof(dcmd->msg) + extra);
+
+	return 0;
+}
+
+static int qcom_glink_rx_data(struct qcom_glink *glink, size_t avail)
+{
+	struct glink_core_rx_intent *intent;
+	struct glink_channel *channel;
+	struct {
+		struct glink_msg msg;
+		__le32 chunk_size;
+		__le32 left_size;
+	} __packed hdr;
+	unsigned int chunk_size;
+	unsigned int left_size;
+	unsigned int rcid;
+	unsigned int liid;
+	int ret = 0;
+	unsigned long flags;
+
+	if (avail < sizeof(hdr)) {
+		dev_dbg(glink->dev, "Not enough data in fifo\n");
+		return -EAGAIN;
+	}
+
+	qcom_glink_rx_peak(glink, &hdr, 0, sizeof(hdr));
+	chunk_size = le32_to_cpu(hdr.chunk_size);
+	left_size = le32_to_cpu(hdr.left_size);
+
+	if (avail < sizeof(hdr) + chunk_size) {
+		dev_dbg(glink->dev, "Payload not yet in fifo\n");
+		return -EAGAIN;
+	}
+
+	if (WARN(chunk_size % 4, "Incoming data must be word aligned\n"))
+		return -EINVAL;
+
+	rcid = le16_to_cpu(hdr.msg.param1);
+	spin_lock_irqsave(&glink->idr_lock, flags);
+	channel = idr_find(&glink->rcids, rcid);
+	spin_unlock_irqrestore(&glink->idr_lock, flags);
+	if (!channel) {
+		dev_dbg(glink->dev, "Data on non-existing channel\n");
+
+		/* Drop the message */
+		goto advance_rx;
+	}
+
+	if (glink->intentless) {
+		/* Might have an ongoing, fragmented, message to append */
+		if (!channel->buf) {
+			intent = kzalloc(sizeof(*intent), GFP_ATOMIC);
+			if (!intent)
+				return -ENOMEM;
+
+			intent->data = kmalloc(chunk_size + left_size,
+					       GFP_ATOMIC);
+			if (!intent->data) {
+				kfree(intent);
+				return -ENOMEM;
+			}
+
+			intent->id = 0xbabababa;
+			intent->size = chunk_size + left_size;
+			intent->offset = 0;
+
+			channel->buf = intent;
+		} else {
+			intent = channel->buf;
+		}
+	} else {
+		liid = le32_to_cpu(hdr.msg.param2);
+
+		spin_lock_irqsave(&channel->intent_lock, flags);
+		intent = idr_find(&channel->liids, liid);
+		spin_unlock_irqrestore(&channel->intent_lock, flags);
+
+		if (!intent) {
+			dev_err(glink->dev,
+				"no intent found for channel %s intent %d",
+				channel->name, liid);
+			goto advance_rx;
+		}
+	}
+
+	if (intent->size - intent->offset < chunk_size) {
+		dev_err(glink->dev, "Insufficient space in intent\n");
+
+		/* The packet header lied, drop payload */
+		goto advance_rx;
+	}
+
+	qcom_glink_rx_peak(glink, intent->data + intent->offset,
+			   sizeof(hdr), chunk_size);
+	intent->offset += chunk_size;
+
+	/* Handle message when no fragments remain to be received */
+	if (!left_size) {
+		spin_lock(&channel->recv_lock);
+		if (channel->ept.cb) {
+			channel->ept.cb(channel->ept.rpdev,
+					intent->data,
+					intent->offset,
+					channel->ept.priv,
+					RPMSG_ADDR_ANY);
+		}
+		spin_unlock(&channel->recv_lock);
+
+		intent->offset = 0;
+		channel->buf = NULL;
+
+		qcom_glink_rx_done(glink, channel, intent);
+	}
+
+advance_rx:
+	qcom_glink_rx_advance(glink, ALIGN(sizeof(hdr) + chunk_size, 8));
+
+	return ret;
+}
+
+static void qcom_glink_handle_intent(struct qcom_glink *glink,
+				     unsigned int cid,
+				     unsigned int count,
+				     size_t avail)
+{
+	struct glink_core_rx_intent *intent;
+	struct glink_channel *channel;
+	struct intent_pair {
+		__le32 size;
+		__le32 iid;
+	};
+
+	struct {
+		struct glink_msg msg;
+		struct intent_pair intents[];
+	} __packed * msg;
+
+	const size_t msglen = sizeof(*msg) + sizeof(struct intent_pair) * count;
+	int ret;
+	int i;
+	unsigned long flags;
+
+	if (avail < msglen) {
+		dev_dbg(glink->dev, "Not enough data in fifo\n");
+		return;
+	}
+
+	spin_lock_irqsave(&glink->idr_lock, flags);
+	channel = idr_find(&glink->rcids, cid);
+	spin_unlock_irqrestore(&glink->idr_lock, flags);
+	if (!channel) {
+		dev_err(glink->dev, "intents for non-existing channel\n");
+		return;
+	}
+
+	msg = kmalloc(msglen, GFP_ATOMIC);
+	if (!msg)
+		return;
+
+	qcom_glink_rx_peak(glink, msg, 0, msglen);
+
+	for (i = 0; i < count; ++i) {
+		intent = kzalloc(sizeof(*intent), GFP_ATOMIC);
+		if (!intent)
+			break;
+
+		intent->id = le32_to_cpu(msg->intents[i].iid);
+		intent->size = le32_to_cpu(msg->intents[i].size);
+
+		spin_lock_irqsave(&channel->intent_lock, flags);
+		ret = idr_alloc(&channel->riids, intent,
+				intent->id, intent->id + 1, GFP_ATOMIC);
+		spin_unlock_irqrestore(&channel->intent_lock, flags);
+
+		if (ret < 0)
+			dev_err(glink->dev, "failed to store remote intent\n");
+	}
+
+	kfree(msg);
+	qcom_glink_rx_advance(glink, ALIGN(msglen, 8));
+}
+
+static int qcom_glink_rx_open_ack(struct qcom_glink *glink, unsigned int lcid)
+{
+	struct glink_channel *channel;
+
+	spin_lock(&glink->idr_lock);
+	channel = idr_find(&glink->lcids, lcid);
+	spin_unlock(&glink->idr_lock);
+	if (!channel) {
+		dev_err(glink->dev, "Invalid open ack packet\n");
+		return -EINVAL;
+	}
+
+	complete(&channel->open_ack);
+
+	return 0;
+}
+
+static irqreturn_t qcom_glink_native_intr(int irq, void *data)
+{
+	struct qcom_glink *glink = data;
+	struct glink_msg msg;
+	unsigned int param1;
+	unsigned int param2;
+	unsigned int avail;
+	unsigned int cmd;
+	int ret = 0;
+
+	for (;;) {
+		avail = qcom_glink_rx_avail(glink);
+		if (avail < sizeof(msg))
+			break;
+
+		qcom_glink_rx_peak(glink, &msg, 0, sizeof(msg));
+
+		cmd = le16_to_cpu(msg.cmd);
+		param1 = le16_to_cpu(msg.param1);
+		param2 = le32_to_cpu(msg.param2);
+
+		switch (cmd) {
+		case RPM_CMD_VERSION:
+		case RPM_CMD_VERSION_ACK:
+		case RPM_CMD_CLOSE:
+		case RPM_CMD_CLOSE_ACK:
+		case RPM_CMD_RX_INTENT_REQ:
+			ret = qcom_glink_rx_defer(glink, 0);
+			break;
+		case RPM_CMD_OPEN_ACK:
+			ret = qcom_glink_rx_open_ack(glink, param1);
+			qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
+			break;
+		case RPM_CMD_OPEN:
+			ret = qcom_glink_rx_defer(glink, param2);
+			break;
+		case RPM_CMD_TX_DATA:
+		case RPM_CMD_TX_DATA_CONT:
+			ret = qcom_glink_rx_data(glink, avail);
+			break;
+		case RPM_CMD_READ_NOTIF:
+			qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
+
+			mbox_send_message(glink->mbox_chan, NULL);
+			mbox_client_txdone(glink->mbox_chan, 0);
+			break;
+		case RPM_CMD_INTENT:
+			qcom_glink_handle_intent(glink, param1, param2, avail);
+			break;
+		case RPM_CMD_RX_DONE:
+			qcom_glink_handle_rx_done(glink, param1, param2, false);
+			qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
+			break;
+		case RPM_CMD_RX_DONE_W_REUSE:
+			qcom_glink_handle_rx_done(glink, param1, param2, true);
+			qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
+			break;
+		case RPM_CMD_RX_INTENT_REQ_ACK:
+			qcom_glink_handle_intent_req_ack(glink, param1, param2);
+			qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
+			break;
+		default:
+			dev_err(glink->dev, "unhandled rx cmd: %d\n", cmd);
+			ret = -EINVAL;
+			break;
+		}
+
+		if (ret)
+			break;
+	}
+
+	return IRQ_HANDLED;
+}
+
+/* Locally initiated rpmsg_create_ept */
+static struct glink_channel *qcom_glink_create_local(struct qcom_glink *glink,
+						     const char *name)
+{
+	struct glink_channel *channel;
+	int ret;
+	unsigned long flags;
+
+	channel = qcom_glink_alloc_channel(glink, name);
+	if (IS_ERR(channel))
+		return ERR_CAST(channel);
+
+	ret = qcom_glink_send_open_req(glink, channel);
+	if (ret)
+		goto release_channel;
+
+	ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ);
+	if (!ret)
+		goto err_timeout;
+
+	ret = wait_for_completion_timeout(&channel->open_req, 5 * HZ);
+	if (!ret)
+		goto err_timeout;
+
+	qcom_glink_send_open_ack(glink, channel);
+
+	return channel;
+
+err_timeout:
+	/* qcom_glink_send_open_req() did register the channel in lcids*/
+	spin_lock_irqsave(&glink->idr_lock, flags);
+	idr_remove(&glink->lcids, channel->lcid);
+	spin_unlock_irqrestore(&glink->idr_lock, flags);
+
+release_channel:
+	/* Release qcom_glink_send_open_req() reference */
+	kref_put(&channel->refcount, qcom_glink_channel_release);
+	/* Release qcom_glink_alloc_channel() reference */
+	kref_put(&channel->refcount, qcom_glink_channel_release);
+
+	return ERR_PTR(-ETIMEDOUT);
+}
+
+/* Remote initiated rpmsg_create_ept */
+static int qcom_glink_create_remote(struct qcom_glink *glink,
+				    struct glink_channel *channel)
+{
+	int ret;
+
+	qcom_glink_send_open_ack(glink, channel);
+
+	ret = qcom_glink_send_open_req(glink, channel);
+	if (ret)
+		goto close_link;
+
+	ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ);
+	if (!ret) {
+		ret = -ETIMEDOUT;
+		goto close_link;
+	}
+
+	return 0;
+
+close_link:
+	/*
+	 * Send a close request to "undo" our open-ack. The close-ack will
+	 * release the last reference.
+	 */
+	qcom_glink_send_close_req(glink, channel);
+
+	/* Release qcom_glink_send_open_req() reference */
+	kref_put(&channel->refcount, qcom_glink_channel_release);
+
+	return ret;
+}
+
+static struct rpmsg_endpoint *qcom_glink_create_ept(struct rpmsg_device *rpdev,
+						    rpmsg_rx_cb_t cb,
+						    void *priv,
+						    struct rpmsg_channel_info
+									chinfo)
+{
+	struct glink_channel *parent = to_glink_channel(rpdev->ept);
+	struct glink_channel *channel;
+	struct qcom_glink *glink = parent->glink;
+	struct rpmsg_endpoint *ept;
+	const char *name = chinfo.name;
+	int cid;
+	int ret;
+	unsigned long flags;
+
+	spin_lock_irqsave(&glink->idr_lock, flags);
+	idr_for_each_entry(&glink->rcids, channel, cid) {
+		if (!strcmp(channel->name, name))
+			break;
+	}
+	spin_unlock_irqrestore(&glink->idr_lock, flags);
+
+	if (!channel) {
+		channel = qcom_glink_create_local(glink, name);
+		if (IS_ERR(channel))
+			return NULL;
+	} else {
+		ret = qcom_glink_create_remote(glink, channel);
+		if (ret)
+			return NULL;
+	}
+
+	ept = &channel->ept;
+	ept->rpdev = rpdev;
+	ept->cb = cb;
+	ept->priv = priv;
+	ept->ops = &glink_endpoint_ops;
+
+	return ept;
+}
+
+static int qcom_glink_announce_create(struct rpmsg_device *rpdev)
+{
+	struct glink_channel *channel = to_glink_channel(rpdev->ept);
+	struct glink_core_rx_intent *intent;
+	struct qcom_glink *glink = channel->glink;
+	int num_intents = glink->intentless ? 0 : 5;
+
+	/* Channel is now open, advertise base set of intents */
+	while (num_intents--) {
+		intent = qcom_glink_alloc_intent(glink, channel, SZ_1K, true);
+		if (!intent)
+			break;
+
+		qcom_glink_advertise_intent(glink, channel, intent);
+	}
+
+	return 0;
+}
+
+static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept)
+{
+	struct glink_channel *channel = to_glink_channel(ept);
+	struct qcom_glink *glink = channel->glink;
+	unsigned long flags;
+
+	spin_lock_irqsave(&channel->recv_lock, flags);
+	channel->ept.cb = NULL;
+	spin_unlock_irqrestore(&channel->recv_lock, flags);
+
+	/* Decouple the potential rpdev from the channel */
+	channel->rpdev = NULL;
+
+	qcom_glink_send_close_req(glink, channel);
+}
+
+static int qcom_glink_request_intent(struct qcom_glink *glink,
+				     struct glink_channel *channel,
+				     size_t size)
+{
+	struct {
+		u16 id;
+		u16 cid;
+		u32 size;
+	} __packed cmd;
+
+	int ret;
+
+	mutex_lock(&channel->intent_req_lock);
+
+	reinit_completion(&channel->intent_req_comp);
+
+	cmd.id = RPM_CMD_RX_INTENT_REQ;
+	cmd.cid = channel->lcid;
+	cmd.size = size;
+
+	ret = qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
+	if (ret)
+		return ret;
+
+	ret = wait_for_completion_timeout(&channel->intent_req_comp, 10 * HZ);
+	if (!ret) {
+		dev_err(glink->dev, "intent request timed out\n");
+		ret = -ETIMEDOUT;
+	} else {
+		ret = channel->intent_req_result ? 0 : -ECANCELED;
+	}
+
+	mutex_unlock(&channel->intent_req_lock);
+	return ret;
+}
+
+static int __qcom_glink_send(struct glink_channel *channel,
+			     void *data, int len, bool wait)
+{
+	struct qcom_glink *glink = channel->glink;
+	struct glink_core_rx_intent *intent = NULL;
+	struct glink_core_rx_intent *tmp;
+	int iid = 0;
+	struct {
+		struct glink_msg msg;
+		__le32 chunk_size;
+		__le32 left_size;
+	} __packed req;
+	int ret;
+	unsigned long flags;
+
+	if (!glink->intentless) {
+		while (!intent) {
+			spin_lock_irqsave(&channel->intent_lock, flags);
+			idr_for_each_entry(&channel->riids, tmp, iid) {
+				if (tmp->size >= len && !tmp->in_use) {
+					tmp->in_use = true;
+					intent = tmp;
+					break;
+				}
+			}
+			spin_unlock_irqrestore(&channel->intent_lock, flags);
+
+			/* We found an available intent */
+			if (intent)
+				break;
+
+			if (!wait)
+				return -EBUSY;
+
+			ret = qcom_glink_request_intent(glink, channel, len);
+			if (ret < 0)
+				return ret;
+		}
+
+		iid = intent->id;
+	}
+
+	req.msg.cmd = cpu_to_le16(RPM_CMD_TX_DATA);
+	req.msg.param1 = cpu_to_le16(channel->lcid);
+	req.msg.param2 = cpu_to_le32(iid);
+	req.chunk_size = cpu_to_le32(len);
+	req.left_size = cpu_to_le32(0);
+
+	ret = qcom_glink_tx(glink, &req, sizeof(req), data, len, wait);
+
+	/* Mark intent available if we failed */
+	if (ret && intent)
+		intent->in_use = false;
+
+	return ret;
+}
+
+static int qcom_glink_send(struct rpmsg_endpoint *ept, void *data, int len)
+{
+	struct glink_channel *channel = to_glink_channel(ept);
+
+	return __qcom_glink_send(channel, data, len, true);
+}
+
+static int qcom_glink_trysend(struct rpmsg_endpoint *ept, void *data, int len)
+{
+	struct glink_channel *channel = to_glink_channel(ept);
+
+	return __qcom_glink_send(channel, data, len, false);
+}
+
+/*
+ * Finds the device_node for the glink child interested in this channel.
+ */
+static struct device_node *qcom_glink_match_channel(struct device_node *node,
+						    const char *channel)
+{
+	struct device_node *child;
+	const char *name;
+	const char *key;
+	int ret;
+
+	for_each_available_child_of_node(node, child) {
+		key = "qcom,glink-channels";
+		ret = of_property_read_string(child, key, &name);
+		if (ret)
+			continue;
+
+		if (strcmp(name, channel) == 0)
+			return child;
+	}
+
+	return NULL;
+}
+
+static const struct rpmsg_device_ops glink_device_ops = {
+	.create_ept = qcom_glink_create_ept,
+	.announce_create = qcom_glink_announce_create,
+};
+
+static const struct rpmsg_endpoint_ops glink_endpoint_ops = {
+	.destroy_ept = qcom_glink_destroy_ept,
+	.send = qcom_glink_send,
+	.trysend = qcom_glink_trysend,
+};
+
+static void qcom_glink_rpdev_release(struct device *dev)
+{
+	struct rpmsg_device *rpdev = to_rpmsg_device(dev);
+	struct glink_channel *channel = to_glink_channel(rpdev->ept);
+
+	channel->rpdev = NULL;
+	kfree(rpdev);
+}
+
+static int qcom_glink_rx_open(struct qcom_glink *glink, unsigned int rcid,
+			      char *name)
+{
+	struct glink_channel *channel;
+	struct rpmsg_device *rpdev;
+	bool create_device = false;
+	struct device_node *node;
+	int lcid;
+	int ret;
+	unsigned long flags;
+
+	spin_lock_irqsave(&glink->idr_lock, flags);
+	idr_for_each_entry(&glink->lcids, channel, lcid) {
+		if (!strcmp(channel->name, name))
+			break;
+	}
+	spin_unlock_irqrestore(&glink->idr_lock, flags);
+
+	if (!channel) {
+		channel = qcom_glink_alloc_channel(glink, name);
+		if (IS_ERR(channel))
+			return PTR_ERR(channel);
+
+		/* The opening dance was initiated by the remote */
+		create_device = true;
+	}
+
+	spin_lock_irqsave(&glink->idr_lock, flags);
+	ret = idr_alloc(&glink->rcids, channel, rcid, rcid + 1, GFP_ATOMIC);
+	if (ret < 0) {
+		dev_err(glink->dev, "Unable to insert channel into rcid list\n");
+		spin_unlock_irqrestore(&glink->idr_lock, flags);
+		goto free_channel;
+	}
+	channel->rcid = ret;
+	spin_unlock_irqrestore(&glink->idr_lock, flags);
+
+	complete(&channel->open_req);
+
+	if (create_device) {
+		rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL);
+		if (!rpdev) {
+			ret = -ENOMEM;
+			goto rcid_remove;
+		}
+
+		rpdev->ept = &channel->ept;
+		strncpy(rpdev->id.name, name, RPMSG_NAME_SIZE);
+		rpdev->src = RPMSG_ADDR_ANY;
+		rpdev->dst = RPMSG_ADDR_ANY;
+		rpdev->ops = &glink_device_ops;
+
+		node = qcom_glink_match_channel(glink->dev->of_node, name);
+		rpdev->dev.of_node = node;
+		rpdev->dev.parent = glink->dev;
+		rpdev->dev.release = qcom_glink_rpdev_release;
+
+		ret = rpmsg_register_device(rpdev);
+		if (ret)
+			goto free_rpdev;
+
+		channel->rpdev = rpdev;
+	}
+
+	return 0;
+
+free_rpdev:
+	kfree(rpdev);
+rcid_remove:
+	spin_lock_irqsave(&glink->idr_lock, flags);
+	idr_remove(&glink->rcids, channel->rcid);
+	channel->rcid = 0;
+	spin_unlock_irqrestore(&glink->idr_lock, flags);
+free_channel:
+	/* Release the reference, iff we took it */
+	if (create_device)
+		kref_put(&channel->refcount, qcom_glink_channel_release);
+
+	return ret;
+}
+
+static void qcom_glink_rx_close(struct qcom_glink *glink, unsigned int rcid)
+{
+	struct rpmsg_channel_info chinfo;
+	struct glink_channel *channel;
+	unsigned long flags;
+
+	spin_lock_irqsave(&glink->idr_lock, flags);
+	channel = idr_find(&glink->rcids, rcid);
+	spin_unlock_irqrestore(&glink->idr_lock, flags);
+	if (WARN(!channel, "close request on unknown channel\n"))
+		return;
+
+	/* cancel pending rx_done work */
+	cancel_work_sync(&channel->intent_work);
+
+	if (channel->rpdev) {
+		strncpy(chinfo.name, channel->name, sizeof(chinfo.name));
+		chinfo.src = RPMSG_ADDR_ANY;
+		chinfo.dst = RPMSG_ADDR_ANY;
+
+		rpmsg_unregister_device(glink->dev, &chinfo);
+	}
+
+	qcom_glink_send_close_ack(glink, channel->rcid);
+
+	spin_lock_irqsave(&glink->idr_lock, flags);
+	idr_remove(&glink->rcids, channel->rcid);
+	channel->rcid = 0;
+	spin_unlock_irqrestore(&glink->idr_lock, flags);
+
+	kref_put(&channel->refcount, qcom_glink_channel_release);
+}
+
+static void qcom_glink_rx_close_ack(struct qcom_glink *glink, unsigned int lcid)
+{
+	struct glink_channel *channel;
+	unsigned long flags;
+
+	spin_lock_irqsave(&glink->idr_lock, flags);
+	channel = idr_find(&glink->lcids, lcid);
+	if (WARN(!channel, "close ack on unknown channel\n")) {
+		spin_unlock_irqrestore(&glink->idr_lock, flags);
+		return;
+	}
+
+	idr_remove(&glink->lcids, channel->lcid);
+	channel->lcid = 0;
+	spin_unlock_irqrestore(&glink->idr_lock, flags);
+
+	kref_put(&channel->refcount, qcom_glink_channel_release);
+}
+
+static void qcom_glink_work(struct work_struct *work)
+{
+	struct qcom_glink *glink = container_of(work, struct qcom_glink,
+						rx_work);
+	struct glink_defer_cmd *dcmd;
+	struct glink_msg *msg;
+	unsigned long flags;
+	unsigned int param1;
+	unsigned int param2;
+	unsigned int cmd;
+
+	for (;;) {
+		spin_lock_irqsave(&glink->rx_lock, flags);
+		if (list_empty(&glink->rx_queue)) {
+			spin_unlock_irqrestore(&glink->rx_lock, flags);
+			break;
+		}
+		dcmd = list_first_entry(&glink->rx_queue,
+					struct glink_defer_cmd, node);
+		list_del(&dcmd->node);
+		spin_unlock_irqrestore(&glink->rx_lock, flags);
+
+		msg = &dcmd->msg;
+		cmd = le16_to_cpu(msg->cmd);
+		param1 = le16_to_cpu(msg->param1);
+		param2 = le32_to_cpu(msg->param2);
+
+		switch (cmd) {
+		case RPM_CMD_VERSION:
+			qcom_glink_receive_version(glink, param1, param2);
+			break;
+		case RPM_CMD_VERSION_ACK:
+			qcom_glink_receive_version_ack(glink, param1, param2);
+			break;
+		case RPM_CMD_OPEN:
+			qcom_glink_rx_open(glink, param1, msg->data);
+			break;
+		case RPM_CMD_CLOSE:
+			qcom_glink_rx_close(glink, param1);
+			break;
+		case RPM_CMD_CLOSE_ACK:
+			qcom_glink_rx_close_ack(glink, param1);
+			break;
+		case RPM_CMD_RX_INTENT_REQ:
+			qcom_glink_handle_intent_req(glink, param1, param2);
+			break;
+		default:
+			WARN(1, "Unknown defer object %d\n", cmd);
+			break;
+		}
+
+		kfree(dcmd);
+	}
+}
+
+struct qcom_glink *qcom_glink_native_probe(struct device *dev,
+					   unsigned long features,
+					   struct qcom_glink_pipe *rx,
+					   struct qcom_glink_pipe *tx,
+					   bool intentless)
+{
+	int irq;
+	int ret;
+	struct qcom_glink *glink;
+
+	glink = devm_kzalloc(dev, sizeof(*glink), GFP_KERNEL);
+	if (!glink)
+		return ERR_PTR(-ENOMEM);
+
+	glink->dev = dev;
+	glink->tx_pipe = tx;
+	glink->rx_pipe = rx;
+
+	glink->features = features;
+	glink->intentless = intentless;
+
+	mutex_init(&glink->tx_lock);
+	spin_lock_init(&glink->rx_lock);
+	INIT_LIST_HEAD(&glink->rx_queue);
+	INIT_WORK(&glink->rx_work, qcom_glink_work);
+
+	spin_lock_init(&glink->idr_lock);
+	idr_init(&glink->lcids);
+	idr_init(&glink->rcids);
+
+	glink->mbox_client.dev = dev;
+	glink->mbox_chan = mbox_request_channel(&glink->mbox_client, 0);
+	if (IS_ERR(glink->mbox_chan)) {
+		if (PTR_ERR(glink->mbox_chan) != -EPROBE_DEFER)
+			dev_err(dev, "failed to acquire IPC channel\n");
+		return ERR_CAST(glink->mbox_chan);
+	}
+
+	irq = of_irq_get(dev->of_node, 0);
+	ret = devm_request_irq(dev, irq,
+			       qcom_glink_native_intr,
+			       IRQF_NO_SUSPEND | IRQF_SHARED,
+			       "glink-native", glink);
+	if (ret) {
+		dev_err(dev, "failed to request IRQ\n");
+		return ERR_PTR(ret);
+	}
+
+	glink->irq = irq;
+
+	ret = qcom_glink_send_version(glink);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return glink;
+}
+EXPORT_SYMBOL_GPL(qcom_glink_native_probe);
+
+static int qcom_glink_remove_device(struct device *dev, void *data)
+{
+	device_unregister(dev);
+
+	return 0;
+}
+
+void qcom_glink_native_remove(struct qcom_glink *glink)
+{
+	struct glink_channel *channel;
+	int cid;
+	int ret;
+	unsigned long flags;
+
+	disable_irq(glink->irq);
+	cancel_work_sync(&glink->rx_work);
+
+	ret = device_for_each_child(glink->dev, NULL, qcom_glink_remove_device);
+	if (ret)
+		dev_warn(glink->dev, "Can't remove GLINK devices: %d\n", ret);
+
+	spin_lock_irqsave(&glink->idr_lock, flags);
+	/* Release any defunct local channels, waiting for close-ack */
+	idr_for_each_entry(&glink->lcids, channel, cid)
+		kref_put(&channel->refcount, qcom_glink_channel_release);
+
+	idr_destroy(&glink->lcids);
+	idr_destroy(&glink->rcids);
+	spin_unlock_irqrestore(&glink->idr_lock, flags);
+	mbox_free_channel(glink->mbox_chan);
+}
+EXPORT_SYMBOL_GPL(qcom_glink_native_remove);
+
+void qcom_glink_native_unregister(struct qcom_glink *glink)
+{
+	device_unregister(glink->dev);
+}
+EXPORT_SYMBOL_GPL(qcom_glink_native_unregister);
diff --git a/drivers/rpmsg/qcom_glink_native.h b/drivers/rpmsg/qcom_glink_native.h
new file mode 100644
index 000000000000..0cae8a8199f8
--- /dev/null
+++ b/drivers/rpmsg/qcom_glink_native.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2016-2017, Linaro Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __QCOM_GLINK_NATIVE_H__
+#define __QCOM_GLINK_NATIVE_H__
+
+#define GLINK_FEATURE_INTENT_REUSE	BIT(0)
+#define GLINK_FEATURE_MIGRATION		BIT(1)
+#define GLINK_FEATURE_TRACER_PKT	BIT(2)
+
+struct qcom_glink_pipe {
+	size_t length;
+
+	size_t (*avail)(struct qcom_glink_pipe *glink_pipe);
+
+	void (*peak)(struct qcom_glink_pipe *glink_pipe, void *data,
+		     unsigned int offset, size_t count);
+	void (*advance)(struct qcom_glink_pipe *glink_pipe, size_t count);
+
+	void (*write)(struct qcom_glink_pipe *glink_pipe,
+		      const void *hdr, size_t hlen,
+		      const void *data, size_t dlen);
+};
+
+struct qcom_glink;
+
+struct qcom_glink *qcom_glink_native_probe(struct device *dev,
+					   unsigned long features,
+					   struct qcom_glink_pipe *rx,
+					   struct qcom_glink_pipe *tx,
+					   bool intentless);
+void qcom_glink_native_remove(struct qcom_glink *glink);
+
+void qcom_glink_native_unregister(struct qcom_glink *glink);
+#endif
diff --git a/drivers/rpmsg/qcom_glink_rpm.c b/drivers/rpmsg/qcom_glink_rpm.c
index 3559a3e84c1e..69b25d157d0f 100644
--- a/drivers/rpmsg/qcom_glink_rpm.c
+++ b/drivers/rpmsg/qcom_glink_rpm.c
@@ -27,6 +27,7 @@
 #include <linux/mailbox_client.h>
 
 #include "rpmsg_internal.h"
+#include "qcom_glink_native.h"
 
 #define RPM_TOC_SIZE		256
 #define RPM_TOC_MAGIC		0x67727430 /* grt0 */
@@ -36,10 +37,7 @@
 #define RPM_TX_FIFO_ID		0x61703272 /* ap2r */
 #define RPM_RX_FIFO_ID		0x72326170 /* r2ap */
 
-#define GLINK_NAME_SIZE		32
-
-#define RPM_GLINK_CID_MIN	1
-#define RPM_GLINK_CID_MAX	65536
+#define to_rpm_pipe(p) container_of(p, struct glink_rpm_pipe, native)
 
 struct rpm_toc_entry {
 	__le32 id;
@@ -54,170 +52,18 @@ struct rpm_toc {
 	struct rpm_toc_entry entries[];
 } __packed;
 
-struct glink_msg {
-	__le16 cmd;
-	__le16 param1;
-	__le32 param2;
-	u8 data[];
-} __packed;
-
 struct glink_rpm_pipe {
+	struct qcom_glink_pipe native;
+
 	void __iomem *tail;
 	void __iomem *head;
 
 	void __iomem *fifo;
-
-	size_t length;
-};
-
-/**
- * struct glink_defer_cmd - deferred incoming control message
- * @node:	list node
- * @msg:	message header
- * data:	payload of the message
- *
- * Copy of a received control message, to be added to @rx_queue and processed
- * by @rx_work of @glink_rpm.
- */
-struct glink_defer_cmd {
-	struct list_head node;
-
-	struct glink_msg msg;
-	u8 data[];
-};
-
-/**
- * struct glink_rpm - driver context, relates to one remote subsystem
- * @dev:	reference to the associated struct device
- * @doorbell:	"rpm_hlos" ipc doorbell
- * @rx_pipe:	pipe object for receive FIFO
- * @tx_pipe:	pipe object for transmit FIFO
- * @irq:	IRQ for signaling incoming events
- * @rx_work:	worker for handling received control messages
- * @rx_lock:	protects the @rx_queue
- * @rx_queue:	queue of received control messages to be processed in @rx_work
- * @tx_lock:	synchronizes operations on the tx fifo
- * @idr_lock:	synchronizes @lcids and @rcids modifications
- * @lcids:	idr of all channels with a known local channel id
- * @rcids:	idr of all channels with a known remote channel id
- */
-struct glink_rpm {
-	struct device *dev;
-
-	struct mbox_client mbox_client;
-	struct mbox_chan *mbox_chan;
-
-	struct glink_rpm_pipe rx_pipe;
-	struct glink_rpm_pipe tx_pipe;
-
-	int irq;
-
-	struct work_struct rx_work;
-	spinlock_t rx_lock;
-	struct list_head rx_queue;
-
-	struct mutex tx_lock;
-
-	struct mutex idr_lock;
-	struct idr lcids;
-	struct idr rcids;
-};
-
-enum {
-	GLINK_STATE_CLOSED,
-	GLINK_STATE_OPENING,
-	GLINK_STATE_OPEN,
-	GLINK_STATE_CLOSING,
 };
 
-/**
- * struct glink_channel - internal representation of a channel
- * @rpdev:	rpdev reference, only used for primary endpoints
- * @ept:	rpmsg endpoint this channel is associated with
- * @glink:	glink_rpm context handle
- * @refcount:	refcount for the channel object
- * @recv_lock:	guard for @ept.cb
- * @name:	unique channel name/identifier
- * @lcid:	channel id, in local space
- * @rcid:	channel id, in remote space
- * @buf:	receive buffer, for gathering fragments
- * @buf_offset:	write offset in @buf
- * @buf_size:	size of current @buf
- * @open_ack:	completed once remote has acked the open-request
- * @open_req:	completed once open-request has been received
- */
-struct glink_channel {
-	struct rpmsg_endpoint ept;
-
-	struct rpmsg_device *rpdev;
-	struct glink_rpm *glink;
-
-	struct kref refcount;
-
-	spinlock_t recv_lock;
-
-	char *name;
-	unsigned int lcid;
-	unsigned int rcid;
-
-	void *buf;
-	int buf_offset;
-	int buf_size;
-
-	struct completion open_ack;
-	struct completion open_req;
-};
-
-#define to_glink_channel(_ept) container_of(_ept, struct glink_channel, ept)
-
-static const struct rpmsg_endpoint_ops glink_endpoint_ops;
-
-#define RPM_CMD_VERSION			0
-#define RPM_CMD_VERSION_ACK		1
-#define RPM_CMD_OPEN			2
-#define RPM_CMD_CLOSE			3
-#define RPM_CMD_OPEN_ACK		4
-#define RPM_CMD_TX_DATA			9
-#define RPM_CMD_CLOSE_ACK		11
-#define RPM_CMD_TX_DATA_CONT		12
-#define RPM_CMD_READ_NOTIF		13
-
-#define GLINK_FEATURE_INTENTLESS	BIT(1)
-
-static struct glink_channel *glink_rpm_alloc_channel(struct glink_rpm *glink,
-						     const char *name)
-{
-	struct glink_channel *channel;
-
-	channel = kzalloc(sizeof(*channel), GFP_KERNEL);
-	if (!channel)
-		return ERR_PTR(-ENOMEM);
-
-	/* Setup glink internal glink_channel data */
-	spin_lock_init(&channel->recv_lock);
-	channel->glink = glink;
-	channel->name = kstrdup(name, GFP_KERNEL);
-
-	init_completion(&channel->open_req);
-	init_completion(&channel->open_ack);
-
-	kref_init(&channel->refcount);
-
-	return channel;
-}
-
-static void glink_rpm_channel_release(struct kref *ref)
-{
-	struct glink_channel *channel = container_of(ref, struct glink_channel,
-						     refcount);
-
-	kfree(channel->name);
-	kfree(channel);
-}
-
-static size_t glink_rpm_rx_avail(struct glink_rpm *glink)
+static size_t glink_rpm_rx_avail(struct qcom_glink_pipe *glink_pipe)
 {
-	struct glink_rpm_pipe *pipe = &glink->rx_pipe;
+	struct glink_rpm_pipe *pipe = to_rpm_pipe(glink_pipe);
 	unsigned int head;
 	unsigned int tail;
 
@@ -225,21 +71,24 @@ static size_t glink_rpm_rx_avail(struct glink_rpm *glink)
 	tail = readl(pipe->tail);
 
 	if (head < tail)
-		return pipe->length - tail + head;
+		return pipe->native.length - tail + head;
 	else
 		return head - tail;
 }
 
-static void glink_rpm_rx_peak(struct glink_rpm *glink,
-			      void *data, size_t count)
+static void glink_rpm_rx_peak(struct qcom_glink_pipe *glink_pipe,
+			      void *data, unsigned int offset, size_t count)
 {
-	struct glink_rpm_pipe *pipe = &glink->rx_pipe;
+	struct glink_rpm_pipe *pipe = to_rpm_pipe(glink_pipe);
 	unsigned int tail;
 	size_t len;
 
 	tail = readl(pipe->tail);
+	tail += offset;
+	if (tail >= pipe->native.length)
+		tail -= pipe->native.length;
 
-	len = min_t(size_t, count, pipe->length - tail);
+	len = min_t(size_t, count, pipe->native.length - tail);
 	if (len) {
 		__ioread32_copy(data, pipe->fifo + tail,
 				len / sizeof(u32));
@@ -251,24 +100,24 @@ static void glink_rpm_rx_peak(struct glink_rpm *glink,
 	}
 }
 
-static void glink_rpm_rx_advance(struct glink_rpm *glink,
+static void glink_rpm_rx_advance(struct qcom_glink_pipe *glink_pipe,
 				 size_t count)
 {
-	struct glink_rpm_pipe *pipe = &glink->rx_pipe;
+	struct glink_rpm_pipe *pipe = to_rpm_pipe(glink_pipe);
 	unsigned int tail;
 
 	tail = readl(pipe->tail);
 
 	tail += count;
-	if (tail >= pipe->length)
-		tail -= pipe->length;
+	if (tail >= pipe->native.length)
+		tail -= pipe->native.length;
 
 	writel(tail, pipe->tail);
 }
 
-static size_t glink_rpm_tx_avail(struct glink_rpm *glink)
+static size_t glink_rpm_tx_avail(struct qcom_glink_pipe *glink_pipe)
 {
-	struct glink_rpm_pipe *pipe = &glink->tx_pipe;
+	struct glink_rpm_pipe *pipe = to_rpm_pipe(glink_pipe);
 	unsigned int head;
 	unsigned int tail;
 
@@ -276,19 +125,18 @@ static size_t glink_rpm_tx_avail(struct glink_rpm *glink)
 	tail = readl(pipe->tail);
 
 	if (tail <= head)
-		return pipe->length - head + tail;
+		return pipe->native.length - head + tail;
 	else
 		return tail - head;
 }
 
-static unsigned int glink_rpm_tx_write(struct glink_rpm *glink,
-				       unsigned int head,
-				       const void *data, size_t count)
+static unsigned int glink_rpm_tx_write_one(struct glink_rpm_pipe *pipe,
+					   unsigned int head,
+					   const void *data, size_t count)
 {
-	struct glink_rpm_pipe *pipe = &glink->tx_pipe;
 	size_t len;
 
-	len = min_t(size_t, count, pipe->length - head);
+	len = min_t(size_t, count, pipe->native.length - head);
 	if (len) {
 		__iowrite32_copy(pipe->fifo + head, data,
 				 len / sizeof(u32));
@@ -300,725 +148,43 @@ static unsigned int glink_rpm_tx_write(struct glink_rpm *glink,
 	}
 
 	head += count;
-	if (head >= pipe->length)
-		head -= pipe->length;
+	if (head >= pipe->native.length)
+		head -= pipe->native.length;
 
 	return head;
 }
 
-static int glink_rpm_tx(struct glink_rpm *glink,
-			const void *hdr, size_t hlen,
-			const void *data, size_t dlen, bool wait)
+static void glink_rpm_tx_write(struct qcom_glink_pipe *glink_pipe,
+			       const void *hdr, size_t hlen,
+			       const void *data, size_t dlen)
 {
-	struct glink_rpm_pipe *pipe = &glink->tx_pipe;
+	struct glink_rpm_pipe *pipe = to_rpm_pipe(glink_pipe);
+	size_t tlen = hlen + dlen;
+	size_t aligned_dlen;
 	unsigned int head;
-	unsigned int tlen = hlen + dlen;
-	int ret;
-
-	/* Reject packets that are too big */
-	if (tlen >= glink->tx_pipe.length)
-		return -EINVAL;
-
-	if (WARN(tlen % 8, "Unaligned TX request"))
-		return -EINVAL;
-
-	ret = mutex_lock_interruptible(&glink->tx_lock);
-	if (ret)
-		return ret;
-
-	while (glink_rpm_tx_avail(glink) < tlen) {
-		if (!wait) {
-			ret = -ENOMEM;
-			goto out;
-		}
-
-		msleep(10);
-	}
-
-	head = readl(pipe->head);
-	head = glink_rpm_tx_write(glink, head, hdr, hlen);
-	head = glink_rpm_tx_write(glink, head, data, dlen);
-	writel(head, pipe->head);
-
-	mbox_send_message(glink->mbox_chan, NULL);
-	mbox_client_txdone(glink->mbox_chan, 0);
-
-out:
-	mutex_unlock(&glink->tx_lock);
+	char padding[8] = {0};
+	size_t pad;
 
-	return ret;
-}
-
-static int glink_rpm_send_version(struct glink_rpm *glink)
-{
-	struct glink_msg msg;
-
-	msg.cmd = cpu_to_le16(RPM_CMD_VERSION);
-	msg.param1 = cpu_to_le16(1);
-	msg.param2 = cpu_to_le32(GLINK_FEATURE_INTENTLESS);
-
-	return glink_rpm_tx(glink, &msg, sizeof(msg), NULL, 0, true);
-}
-
-static void glink_rpm_send_version_ack(struct glink_rpm *glink)
-{
-	struct glink_msg msg;
-
-	msg.cmd = cpu_to_le16(RPM_CMD_VERSION_ACK);
-	msg.param1 = cpu_to_le16(1);
-	msg.param2 = cpu_to_le32(0);
-
-	glink_rpm_tx(glink, &msg, sizeof(msg), NULL, 0, true);
-}
-
-static void glink_rpm_send_open_ack(struct glink_rpm *glink,
-					 struct glink_channel *channel)
-{
-	struct glink_msg msg;
-
-	msg.cmd = cpu_to_le16(RPM_CMD_OPEN_ACK);
-	msg.param1 = cpu_to_le16(channel->rcid);
-	msg.param2 = cpu_to_le32(0);
-
-	glink_rpm_tx(glink, &msg, sizeof(msg), NULL, 0, true);
-}
-
-/**
- * glink_rpm_send_open_req() - send a RPM_CMD_OPEN request to the remote
- * @glink:
- * @channel:
- *
- * Allocates a local channel id and sends a RPM_CMD_OPEN message to the remote.
- * Will return with refcount held, regardless of outcome.
- *
- * Returns 0 on success, negative errno otherwise.
- */
-static int glink_rpm_send_open_req(struct glink_rpm *glink,
-					 struct glink_channel *channel)
-{
-	struct {
-		struct glink_msg msg;
-		u8 name[GLINK_NAME_SIZE];
-	} __packed req;
-	int name_len = strlen(channel->name) + 1;
-	int req_len = ALIGN(sizeof(req.msg) + name_len, 8);
-	int ret;
-
-	kref_get(&channel->refcount);
-
-	mutex_lock(&glink->idr_lock);
-	ret = idr_alloc_cyclic(&glink->lcids, channel,
-			       RPM_GLINK_CID_MIN, RPM_GLINK_CID_MAX, GFP_KERNEL);
-	mutex_unlock(&glink->idr_lock);
-	if (ret < 0)
-		return ret;
-
-	channel->lcid = ret;
-
-	req.msg.cmd = cpu_to_le16(RPM_CMD_OPEN);
-	req.msg.param1 = cpu_to_le16(channel->lcid);
-	req.msg.param2 = cpu_to_le32(name_len);
-	strcpy(req.name, channel->name);
-
-	ret = glink_rpm_tx(glink, &req, req_len, NULL, 0, true);
-	if (ret)
-		goto remove_idr;
-
-	return 0;
-
-remove_idr:
-	mutex_lock(&glink->idr_lock);
-	idr_remove(&glink->lcids, channel->lcid);
-	channel->lcid = 0;
-	mutex_unlock(&glink->idr_lock);
-
-	return ret;
-}
-
-static void glink_rpm_send_close_req(struct glink_rpm *glink,
-					  struct glink_channel *channel)
-{
-	struct glink_msg req;
-
-	req.cmd = cpu_to_le16(RPM_CMD_CLOSE);
-	req.param1 = cpu_to_le16(channel->lcid);
-	req.param2 = 0;
-
-	glink_rpm_tx(glink, &req, sizeof(req), NULL, 0, true);
-}
-
-static void glink_rpm_send_close_ack(struct glink_rpm *glink, unsigned int rcid)
-{
-	struct glink_msg req;
-
-	req.cmd = cpu_to_le16(RPM_CMD_CLOSE_ACK);
-	req.param1 = cpu_to_le16(rcid);
-	req.param2 = 0;
-
-	glink_rpm_tx(glink, &req, sizeof(req), NULL, 0, true);
-}
-
-static int glink_rpm_rx_defer(struct glink_rpm *glink, size_t extra)
-{
-	struct glink_defer_cmd *dcmd;
-
-	extra = ALIGN(extra, 8);
-
-	if (glink_rpm_rx_avail(glink) < sizeof(struct glink_msg) + extra) {
-		dev_dbg(glink->dev, "Insufficient data in rx fifo");
-		return -ENXIO;
-	}
-
-	dcmd = kzalloc(sizeof(*dcmd) + extra, GFP_ATOMIC);
-	if (!dcmd)
-		return -ENOMEM;
-
-	INIT_LIST_HEAD(&dcmd->node);
-
-	glink_rpm_rx_peak(glink, &dcmd->msg, sizeof(dcmd->msg) + extra);
-
-	spin_lock(&glink->rx_lock);
-	list_add_tail(&dcmd->node, &glink->rx_queue);
-	spin_unlock(&glink->rx_lock);
-
-	schedule_work(&glink->rx_work);
-	glink_rpm_rx_advance(glink, sizeof(dcmd->msg) + extra);
-
-	return 0;
-}
-
-static int glink_rpm_rx_data(struct glink_rpm *glink, size_t avail)
-{
-	struct glink_channel *channel;
-	struct {
-		struct glink_msg msg;
-		__le32 chunk_size;
-		__le32 left_size;
-	} __packed hdr;
-	unsigned int chunk_size;
-	unsigned int left_size;
-	unsigned int rcid;
-
-	if (avail < sizeof(hdr)) {
-		dev_dbg(glink->dev, "Not enough data in fifo\n");
-		return -EAGAIN;
-	}
-
-	glink_rpm_rx_peak(glink, &hdr, sizeof(hdr));
-	chunk_size = le32_to_cpu(hdr.chunk_size);
-	left_size = le32_to_cpu(hdr.left_size);
-
-	if (avail < sizeof(hdr) + chunk_size) {
-		dev_dbg(glink->dev, "Payload not yet in fifo\n");
-		return -EAGAIN;
-	}
-
-	if (WARN(chunk_size % 4, "Incoming data must be word aligned\n"))
-		return -EINVAL;
-
-	rcid = le16_to_cpu(hdr.msg.param1);
-	channel = idr_find(&glink->rcids, rcid);
-	if (!channel) {
-		dev_dbg(glink->dev, "Data on non-existing channel\n");
-
-		/* Drop the message */
-		glink_rpm_rx_advance(glink, ALIGN(sizeof(hdr) + chunk_size, 8));
-		return 0;
-	}
-
-	/* Might have an ongoing, fragmented, message to append */
-	if (!channel->buf) {
-		channel->buf = kmalloc(chunk_size + left_size, GFP_ATOMIC);
-		if (!channel->buf)
-			return -ENOMEM;
-
-		channel->buf_size = chunk_size + left_size;
-		channel->buf_offset = 0;
-	}
-
-	glink_rpm_rx_advance(glink, sizeof(hdr));
-
-	if (channel->buf_size - channel->buf_offset < chunk_size) {
-		dev_err(glink->dev, "Insufficient space in input buffer\n");
-
-		/* The packet header lied, drop payload */
-		glink_rpm_rx_advance(glink, chunk_size);
-		return -ENOMEM;
-	}
-
-	glink_rpm_rx_peak(glink, channel->buf + channel->buf_offset, chunk_size);
-	channel->buf_offset += chunk_size;
-
-	/* Handle message when no fragments remain to be received */
-	if (!left_size) {
-		spin_lock(&channel->recv_lock);
-		if (channel->ept.cb) {
-			channel->ept.cb(channel->ept.rpdev,
-					channel->buf,
-					channel->buf_offset,
-					channel->ept.priv,
-					RPMSG_ADDR_ANY);
-		}
-		spin_unlock(&channel->recv_lock);
-
-		kfree(channel->buf);
-		channel->buf = NULL;
-		channel->buf_size = 0;
-	}
-
-	/* Each message starts at 8 byte aligned address */
-	glink_rpm_rx_advance(glink, ALIGN(chunk_size, 8));
-
-	return 0;
-}
-
-static int glink_rpm_rx_open_ack(struct glink_rpm *glink, unsigned int lcid)
-{
-	struct glink_channel *channel;
-
-	channel = idr_find(&glink->lcids, lcid);
-	if (!channel) {
-		dev_err(glink->dev, "Invalid open ack packet\n");
-		return -EINVAL;
-	}
-
-	complete(&channel->open_ack);
-
-	return 0;
-}
-
-static irqreturn_t glink_rpm_intr(int irq, void *data)
-{
-	struct glink_rpm *glink = data;
-	struct glink_msg msg;
-	unsigned int param1;
-	unsigned int param2;
-	unsigned int avail;
-	unsigned int cmd;
-	int ret;
-
-	for (;;) {
-		avail = glink_rpm_rx_avail(glink);
-		if (avail < sizeof(msg))
-			break;
-
-		glink_rpm_rx_peak(glink, &msg, sizeof(msg));
-
-		cmd = le16_to_cpu(msg.cmd);
-		param1 = le16_to_cpu(msg.param1);
-		param2 = le32_to_cpu(msg.param2);
-
-		switch (cmd) {
-		case RPM_CMD_VERSION:
-		case RPM_CMD_VERSION_ACK:
-		case RPM_CMD_CLOSE:
-		case RPM_CMD_CLOSE_ACK:
-			ret = glink_rpm_rx_defer(glink, 0);
-			break;
-		case RPM_CMD_OPEN_ACK:
-			ret = glink_rpm_rx_open_ack(glink, param1);
-			glink_rpm_rx_advance(glink, ALIGN(sizeof(msg), 8));
-			break;
-		case RPM_CMD_OPEN:
-			ret = glink_rpm_rx_defer(glink, param2);
-			break;
-		case RPM_CMD_TX_DATA:
-		case RPM_CMD_TX_DATA_CONT:
-			ret = glink_rpm_rx_data(glink, avail);
-			break;
-		case RPM_CMD_READ_NOTIF:
-			glink_rpm_rx_advance(glink, ALIGN(sizeof(msg), 8));
-
-			mbox_send_message(glink->mbox_chan, NULL);
-			mbox_client_txdone(glink->mbox_chan, 0);
-
-			ret = 0;
-			break;
-		default:
-			dev_err(glink->dev, "unhandled rx cmd: %d\n", cmd);
-			ret = -EINVAL;
-			break;
-		}
-
-		if (ret)
-			break;
-	}
-
-	return IRQ_HANDLED;
-}
-
-/* Locally initiated rpmsg_create_ept */
-static struct glink_channel *glink_rpm_create_local(struct glink_rpm *glink,
-						    const char *name)
-{
-	struct glink_channel *channel;
-	int ret;
-
-	channel = glink_rpm_alloc_channel(glink, name);
-	if (IS_ERR(channel))
-		return ERR_CAST(channel);
-
-	ret = glink_rpm_send_open_req(glink, channel);
-	if (ret)
-		goto release_channel;
-
-	ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ);
-	if (!ret)
-		goto err_timeout;
-
-	ret = wait_for_completion_timeout(&channel->open_req, 5 * HZ);
-	if (!ret)
-		goto err_timeout;
-
-	glink_rpm_send_open_ack(glink, channel);
-
-	return channel;
-
-err_timeout:
-	/* glink_rpm_send_open_req() did register the channel in lcids*/
-	mutex_lock(&glink->idr_lock);
-	idr_remove(&glink->lcids, channel->lcid);
-	mutex_unlock(&glink->idr_lock);
-
-release_channel:
-	/* Release glink_rpm_send_open_req() reference */
-	kref_put(&channel->refcount, glink_rpm_channel_release);
-	/* Release glink_rpm_alloc_channel() reference */
-	kref_put(&channel->refcount, glink_rpm_channel_release);
-
-	return ERR_PTR(-ETIMEDOUT);
-}
-
-/* Remote initiated rpmsg_create_ept */
-static int glink_rpm_create_remote(struct glink_rpm *glink,
-				   struct glink_channel *channel)
-{
-	int ret;
-
-	glink_rpm_send_open_ack(glink, channel);
-
-	ret = glink_rpm_send_open_req(glink, channel);
-	if (ret)
-		goto close_link;
-
-	ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ);
-	if (!ret) {
-		ret = -ETIMEDOUT;
-		goto close_link;
-	}
-
-	return 0;
+	/* Header length comes from glink native and is always 4 byte aligned */
+	if (WARN(hlen % 4, "Glink Header length must be 4 bytes aligned\n"))
+		return;
 
-close_link:
 	/*
-	 * Send a close request to "undo" our open-ack. The close-ack will
-	 * release the last reference.
+	 * Move the unaligned tail of the message to the padding chunk, to
+	 * ensure word aligned accesses
 	 */
-	glink_rpm_send_close_req(glink, channel);
-
-	/* Release glink_rpm_send_open_req() reference */
-	kref_put(&channel->refcount, glink_rpm_channel_release);
-
-	return ret;
-}
-
-static struct rpmsg_endpoint *glink_rpm_create_ept(struct rpmsg_device *rpdev,
-						  rpmsg_rx_cb_t cb, void *priv,
-						  struct rpmsg_channel_info chinfo)
-{
-	struct glink_channel *parent = to_glink_channel(rpdev->ept);
-	struct glink_channel *channel;
-	struct glink_rpm *glink = parent->glink;
-	struct rpmsg_endpoint *ept;
-	const char *name = chinfo.name;
-	int cid;
-	int ret;
-
-	idr_for_each_entry(&glink->rcids, channel, cid) {
-		if (!strcmp(channel->name, name))
-			break;
-	}
-
-	if (!channel) {
-		channel = glink_rpm_create_local(glink, name);
-		if (IS_ERR(channel))
-			return NULL;
-	} else {
-		ret = glink_rpm_create_remote(glink, channel);
-		if (ret)
-			return NULL;
-	}
-
-	ept = &channel->ept;
-	ept->rpdev = rpdev;
-	ept->cb = cb;
-	ept->priv = priv;
-	ept->ops = &glink_endpoint_ops;
-
-	return ept;
-}
-
-static void glink_rpm_destroy_ept(struct rpmsg_endpoint *ept)
-{
-	struct glink_channel *channel = to_glink_channel(ept);
-	struct glink_rpm *glink = channel->glink;
-	unsigned long flags;
-
-	spin_lock_irqsave(&channel->recv_lock, flags);
-	channel->ept.cb = NULL;
-	spin_unlock_irqrestore(&channel->recv_lock, flags);
-
-	/* Decouple the potential rpdev from the channel */
-	channel->rpdev = NULL;
-
-	glink_rpm_send_close_req(glink, channel);
-}
-
-static int __glink_rpm_send(struct glink_channel *channel,
-			     void *data, int len, bool wait)
-{
-	struct glink_rpm *glink = channel->glink;
-	struct {
-		struct glink_msg msg;
-		__le32 chunk_size;
-		__le32 left_size;
-	} __packed req;
-
-	if (WARN(len % 8, "RPM GLINK expects 8 byte aligned messages\n"))
-		return -EINVAL;
-
-	req.msg.cmd = cpu_to_le16(RPM_CMD_TX_DATA);
-	req.msg.param1 = cpu_to_le16(channel->lcid);
-	req.msg.param2 = cpu_to_le32(channel->rcid);
-	req.chunk_size = cpu_to_le32(len);
-	req.left_size = cpu_to_le32(0);
-
-	return glink_rpm_tx(glink, &req, sizeof(req), data, len, wait);
-}
-
-static int glink_rpm_send(struct rpmsg_endpoint *ept, void *data, int len)
-{
-	struct glink_channel *channel = to_glink_channel(ept);
-
-	return __glink_rpm_send(channel, data, len, true);
-}
-
-static int glink_rpm_trysend(struct rpmsg_endpoint *ept, void *data, int len)
-{
-	struct glink_channel *channel = to_glink_channel(ept);
-
-	return __glink_rpm_send(channel, data, len, false);
-}
-
-/*
- * Finds the device_node for the glink child interested in this channel.
- */
-static struct device_node *glink_rpm_match_channel(struct device_node *node,
-						    const char *channel)
-{
-	struct device_node *child;
-	const char *name;
-	const char *key;
-	int ret;
-
-	for_each_available_child_of_node(node, child) {
-		key = "qcom,glink-channels";
-		ret = of_property_read_string(child, key, &name);
-		if (ret)
-			continue;
-
-		if (strcmp(name, channel) == 0)
-			return child;
-	}
-
-	return NULL;
-}
-
-static const struct rpmsg_device_ops glink_device_ops = {
-	.create_ept = glink_rpm_create_ept,
-};
-
-static const struct rpmsg_endpoint_ops glink_endpoint_ops = {
-	.destroy_ept = glink_rpm_destroy_ept,
-	.send = glink_rpm_send,
-	.trysend = glink_rpm_trysend,
-};
-
-static void glink_rpm_rpdev_release(struct device *dev)
-{
-	struct rpmsg_device *rpdev = to_rpmsg_device(dev);
-	struct glink_channel *channel = to_glink_channel(rpdev->ept);
-
-	channel->rpdev = NULL;
-	kfree(rpdev);
-}
-
-static int glink_rpm_rx_open(struct glink_rpm *glink, unsigned int rcid,
-			     char *name)
-{
-	struct glink_channel *channel;
-	struct rpmsg_device *rpdev;
-	bool create_device = false;
-	int lcid;
-	int ret;
-
-	idr_for_each_entry(&glink->lcids, channel, lcid) {
-		if (!strcmp(channel->name, name))
-			break;
-	}
-
-	if (!channel) {
-		channel = glink_rpm_alloc_channel(glink, name);
-		if (IS_ERR(channel))
-			return PTR_ERR(channel);
-
-		/* The opening dance was initiated by the remote */
-		create_device = true;
-	}
-
-	mutex_lock(&glink->idr_lock);
-	ret = idr_alloc(&glink->rcids, channel, rcid, rcid + 1, GFP_KERNEL);
-	if (ret < 0) {
-		dev_err(glink->dev, "Unable to insert channel into rcid list\n");
-		mutex_unlock(&glink->idr_lock);
-		goto free_channel;
-	}
-	channel->rcid = ret;
-	mutex_unlock(&glink->idr_lock);
-
-	complete(&channel->open_req);
-
-	if (create_device) {
-		rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL);
-		if (!rpdev) {
-			ret = -ENOMEM;
-			goto rcid_remove;
-		}
-
-		rpdev->ept = &channel->ept;
-		strncpy(rpdev->id.name, name, RPMSG_NAME_SIZE);
-		rpdev->src = RPMSG_ADDR_ANY;
-		rpdev->dst = RPMSG_ADDR_ANY;
-		rpdev->ops = &glink_device_ops;
-
-		rpdev->dev.of_node = glink_rpm_match_channel(glink->dev->of_node, name);
-		rpdev->dev.parent = glink->dev;
-		rpdev->dev.release = glink_rpm_rpdev_release;
-
-		ret = rpmsg_register_device(rpdev);
-		if (ret)
-			goto free_rpdev;
-
-		channel->rpdev = rpdev;
-	}
-
-	return 0;
-
-free_rpdev:
-	kfree(rpdev);
-rcid_remove:
-	mutex_lock(&glink->idr_lock);
-	idr_remove(&glink->rcids, channel->rcid);
-	channel->rcid = 0;
-	mutex_unlock(&glink->idr_lock);
-free_channel:
-	/* Release the reference, iff we took it */
-	if (create_device)
-		kref_put(&channel->refcount, glink_rpm_channel_release);
-
-	return ret;
-}
-
-static void glink_rpm_rx_close(struct glink_rpm *glink, unsigned int rcid)
-{
-	struct rpmsg_channel_info chinfo;
-	struct glink_channel *channel;
-
-	channel = idr_find(&glink->rcids, rcid);
-	if (WARN(!channel, "close request on unknown channel\n"))
-		return;
-
-	if (channel->rpdev) {
-		strncpy(chinfo.name, channel->name, sizeof(chinfo.name));
-		chinfo.src = RPMSG_ADDR_ANY;
-		chinfo.dst = RPMSG_ADDR_ANY;
-
-		rpmsg_unregister_device(glink->dev, &chinfo);
-	}
-
-	glink_rpm_send_close_ack(glink, channel->rcid);
-
-	mutex_lock(&glink->idr_lock);
-	idr_remove(&glink->rcids, channel->rcid);
-	channel->rcid = 0;
-	mutex_unlock(&glink->idr_lock);
-
-	kref_put(&channel->refcount, glink_rpm_channel_release);
-}
+	aligned_dlen = ALIGN_DOWN(dlen, 4);
+	if (aligned_dlen != dlen)
+		memcpy(padding, data + aligned_dlen, dlen - aligned_dlen);
 
-static void glink_rpm_rx_close_ack(struct glink_rpm *glink, unsigned int lcid)
-{
-	struct glink_channel *channel;
-
-	channel = idr_find(&glink->lcids, lcid);
-	if (WARN(!channel, "close ack on unknown channel\n"))
-		return;
-
-	mutex_lock(&glink->idr_lock);
-	idr_remove(&glink->lcids, channel->lcid);
-	channel->lcid = 0;
-	mutex_unlock(&glink->idr_lock);
-
-	kref_put(&channel->refcount, glink_rpm_channel_release);
-}
-
-static void glink_rpm_work(struct work_struct *work)
-{
-	struct glink_rpm *glink = container_of(work, struct glink_rpm, rx_work);
-	struct glink_defer_cmd *dcmd;
-	struct glink_msg *msg;
-	unsigned long flags;
-	unsigned int param1;
-	unsigned int param2;
-	unsigned int cmd;
-
-	for (;;) {
-		spin_lock_irqsave(&glink->rx_lock, flags);
-		if (list_empty(&glink->rx_queue)) {
-			spin_unlock_irqrestore(&glink->rx_lock, flags);
-			break;
-		}
-		dcmd = list_first_entry(&glink->rx_queue, struct glink_defer_cmd, node);
-		list_del(&dcmd->node);
-		spin_unlock_irqrestore(&glink->rx_lock, flags);
-
-		msg = &dcmd->msg;
-		cmd = le16_to_cpu(msg->cmd);
-		param1 = le16_to_cpu(msg->param1);
-		param2 = le32_to_cpu(msg->param2);
-
-		switch (cmd) {
-		case RPM_CMD_VERSION:
-			glink_rpm_send_version_ack(glink);
-			break;
-		case RPM_CMD_VERSION_ACK:
-			break;
-		case RPM_CMD_OPEN:
-			glink_rpm_rx_open(glink, param1, msg->data);
-			break;
-		case RPM_CMD_CLOSE:
-			glink_rpm_rx_close(glink, param1);
-			break;
-		case RPM_CMD_CLOSE_ACK:
-			glink_rpm_rx_close_ack(glink, param1);
-			break;
-		default:
-			WARN(1, "Unknown defer object %d\n", cmd);
-			break;
-		}
+	head = readl(pipe->head);
+	head = glink_rpm_tx_write_one(pipe, head, hdr, hlen);
+	head = glink_rpm_tx_write_one(pipe, head, data, aligned_dlen);
 
-		kfree(dcmd);
-	}
+	pad = ALIGN(tlen, 8) - ALIGN_DOWN(tlen, 4);
+	if (pad)
+		head = glink_rpm_tx_write_one(pipe, head, padding, pad);
+	writel(head, pipe->head);
 }
 
 static int glink_rpm_parse_toc(struct device *dev,
@@ -1067,14 +233,14 @@ static int glink_rpm_parse_toc(struct device *dev,
 
 		switch (id) {
 		case RPM_RX_FIFO_ID:
-			rx->length = size;
+			rx->native.length = size;
 
 			rx->tail = msg_ram + offset;
 			rx->head = msg_ram + offset + sizeof(u32);
 			rx->fifo = msg_ram + offset + 2 * sizeof(u32);
 			break;
 		case RPM_TX_FIFO_ID:
-			tx->length = size;
+			tx->native.length = size;
 
 			tx->tail = msg_ram + offset;
 			tx->head = msg_ram + offset + sizeof(u32);
@@ -1098,38 +264,21 @@ err_inval:
 
 static int glink_rpm_probe(struct platform_device *pdev)
 {
-	struct glink_rpm *glink;
+	struct qcom_glink *glink;
+	struct glink_rpm_pipe *rx_pipe;
+	struct glink_rpm_pipe *tx_pipe;
 	struct device_node *np;
 	void __iomem *msg_ram;
 	size_t msg_ram_size;
 	struct device *dev = &pdev->dev;
 	struct resource r;
-	int irq;
 	int ret;
 
-	glink = devm_kzalloc(dev, sizeof(*glink), GFP_KERNEL);
-	if (!glink)
+	rx_pipe = devm_kzalloc(&pdev->dev, sizeof(*rx_pipe), GFP_KERNEL);
+	tx_pipe = devm_kzalloc(&pdev->dev, sizeof(*tx_pipe), GFP_KERNEL);
+	if (!rx_pipe || !tx_pipe)
 		return -ENOMEM;
 
-	glink->dev = dev;
-
-	mutex_init(&glink->tx_lock);
-	spin_lock_init(&glink->rx_lock);
-	INIT_LIST_HEAD(&glink->rx_queue);
-	INIT_WORK(&glink->rx_work, glink_rpm_work);
-
-	mutex_init(&glink->idr_lock);
-	idr_init(&glink->lcids);
-	idr_init(&glink->rcids);
-
-	glink->mbox_client.dev = &pdev->dev;
-	glink->mbox_chan = mbox_request_channel(&glink->mbox_client, 0);
-	if (IS_ERR(glink->mbox_chan)) {
-		if (PTR_ERR(glink->mbox_chan) != -EPROBE_DEFER)
-			dev_err(&pdev->dev, "failed to acquire IPC channel\n");
-		return PTR_ERR(glink->mbox_chan);
-	}
-
 	np = of_parse_phandle(dev->of_node, "qcom,rpm-msg-ram", 0);
 	ret = of_address_to_resource(np, 0, &r);
 	of_node_put(np);
@@ -1142,61 +291,38 @@ static int glink_rpm_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	ret = glink_rpm_parse_toc(dev, msg_ram, msg_ram_size,
-				  &glink->rx_pipe, &glink->tx_pipe);
+				  rx_pipe, tx_pipe);
 	if (ret)
 		return ret;
 
-	writel(0, glink->tx_pipe.head);
-	writel(0, glink->rx_pipe.tail);
+	/* Pipe specific accessors */
+	rx_pipe->native.avail = glink_rpm_rx_avail;
+	rx_pipe->native.peak = glink_rpm_rx_peak;
+	rx_pipe->native.advance = glink_rpm_rx_advance;
+	tx_pipe->native.avail = glink_rpm_tx_avail;
+	tx_pipe->native.write = glink_rpm_tx_write;
 
-	irq = platform_get_irq(pdev, 0);
-	ret = devm_request_irq(dev, irq,
-			       glink_rpm_intr,
-			       IRQF_NO_SUSPEND | IRQF_SHARED,
-			       "glink-rpm", glink);
-	if (ret) {
-		dev_err(dev, "Failed to request IRQ\n");
-		return ret;
-	}
-
-	glink->irq = irq;
+	writel(0, tx_pipe->head);
+	writel(0, rx_pipe->tail);
 
-	ret = glink_rpm_send_version(glink);
-	if (ret)
-		return ret;
+	glink = qcom_glink_native_probe(&pdev->dev,
+					0,
+					&rx_pipe->native,
+					&tx_pipe->native,
+					true);
+	if (IS_ERR(glink))
+		return PTR_ERR(glink);
 
 	platform_set_drvdata(pdev, glink);
 
 	return 0;
 }
 
-static int glink_rpm_remove_device(struct device *dev, void *data)
-{
-	device_unregister(dev);
-
-	return 0;
-}
-
 static int glink_rpm_remove(struct platform_device *pdev)
 {
-	struct glink_rpm *glink = platform_get_drvdata(pdev);
-	struct glink_channel *channel;
-	int cid;
-	int ret;
-
-	disable_irq(glink->irq);
-	cancel_work_sync(&glink->rx_work);
-
-	ret = device_for_each_child(glink->dev, NULL, glink_rpm_remove_device);
-	if (ret)
-		dev_warn(glink->dev, "Can't remove GLINK devices: %d\n", ret);
-
-	/* Release any defunct local channels, waiting for close-ack */
-	idr_for_each_entry(&glink->lcids, channel, cid)
-		kref_put(&channel->refcount, glink_rpm_channel_release);
+	struct qcom_glink *glink = platform_get_drvdata(pdev);
 
-	idr_destroy(&glink->lcids);
-	idr_destroy(&glink->rcids);
+	qcom_glink_native_remove(glink);
 
 	return 0;
 }
diff --git a/drivers/rpmsg/qcom_glink_smem.c b/drivers/rpmsg/qcom_glink_smem.c
new file mode 100644
index 000000000000..5cdaa5f8fb61
--- /dev/null
+++ b/drivers/rpmsg/qcom_glink_smem.c
@@ -0,0 +1,316 @@
+/*
+ * Copyright (c) 2016, Linaro Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/interrupt.h>
+#include <linux/platform_device.h>
+#include <linux/mfd/syscon.h>
+#include <linux/slab.h>
+#include <linux/rpmsg.h>
+#include <linux/idr.h>
+#include <linux/circ_buf.h>
+#include <linux/soc/qcom/smem.h>
+#include <linux/sizes.h>
+#include <linux/delay.h>
+#include <linux/regmap.h>
+#include <linux/workqueue.h>
+#include <linux/list.h>
+
+#include <linux/delay.h>
+#include <linux/rpmsg.h>
+#include <linux/rpmsg/qcom_glink.h>
+
+#include "qcom_glink_native.h"
+
+#define FIFO_FULL_RESERVE 8
+#define FIFO_ALIGNMENT 8
+#define TX_BLOCKED_CMD_RESERVE 8 /* size of struct read_notif_request */
+
+#define SMEM_GLINK_NATIVE_XPRT_DESCRIPTOR	478
+#define SMEM_GLINK_NATIVE_XPRT_FIFO_0		479
+#define SMEM_GLINK_NATIVE_XPRT_FIFO_1		480
+
+struct glink_smem_pipe {
+	struct qcom_glink_pipe native;
+
+	__le32 *tail;
+	__le32 *head;
+
+	void *fifo;
+
+	int remote_pid;
+};
+
+#define to_smem_pipe(p) container_of(p, struct glink_smem_pipe, native)
+
+static size_t glink_smem_rx_avail(struct qcom_glink_pipe *np)
+{
+	struct glink_smem_pipe *pipe = to_smem_pipe(np);
+	size_t len;
+	void *fifo;
+	u32 head;
+	u32 tail;
+
+	if (!pipe->fifo) {
+		fifo = qcom_smem_get(pipe->remote_pid,
+				     SMEM_GLINK_NATIVE_XPRT_FIFO_1, &len);
+		if (IS_ERR(fifo)) {
+			pr_err("failed to acquire RX fifo handle: %ld\n",
+			       PTR_ERR(fifo));
+			return 0;
+		}
+
+		pipe->fifo = fifo;
+		pipe->native.length = len;
+	}
+
+	head = le32_to_cpu(*pipe->head);
+	tail = le32_to_cpu(*pipe->tail);
+
+	if (head < tail)
+		return pipe->native.length - tail + head;
+	else
+		return head - tail;
+}
+
+static void glink_smem_rx_peak(struct qcom_glink_pipe *np,
+			       void *data, unsigned int offset, size_t count)
+{
+	struct glink_smem_pipe *pipe = to_smem_pipe(np);
+	size_t len;
+	u32 tail;
+
+	tail = le32_to_cpu(*pipe->tail);
+	tail += offset;
+	if (tail >= pipe->native.length)
+		tail -= pipe->native.length;
+
+	len = min_t(size_t, count, pipe->native.length - tail);
+	if (len) {
+		__ioread32_copy(data, pipe->fifo + tail,
+				len / sizeof(u32));
+	}
+
+	if (len != count) {
+		__ioread32_copy(data + len, pipe->fifo,
+				(count - len) / sizeof(u32));
+	}
+}
+
+static void glink_smem_rx_advance(struct qcom_glink_pipe *np,
+				  size_t count)
+{
+	struct glink_smem_pipe *pipe = to_smem_pipe(np);
+	u32 tail;
+
+	tail = le32_to_cpu(*pipe->tail);
+
+	tail += count;
+	if (tail > pipe->native.length)
+		tail -= pipe->native.length;
+
+	*pipe->tail = cpu_to_le32(tail);
+}
+
+static size_t glink_smem_tx_avail(struct qcom_glink_pipe *np)
+{
+	struct glink_smem_pipe *pipe = to_smem_pipe(np);
+	u32 head;
+	u32 tail;
+	u32 avail;
+
+	head = le32_to_cpu(*pipe->head);
+	tail = le32_to_cpu(*pipe->tail);
+
+	if (tail <= head)
+		avail = pipe->native.length - head + tail;
+	else
+		avail = tail - head;
+
+	if (avail < (FIFO_FULL_RESERVE + TX_BLOCKED_CMD_RESERVE))
+		avail = 0;
+	else
+		avail -= FIFO_FULL_RESERVE + TX_BLOCKED_CMD_RESERVE;
+
+	return avail;
+}
+
+static unsigned int glink_smem_tx_write_one(struct glink_smem_pipe *pipe,
+					    unsigned int head,
+					    const void *data, size_t count)
+{
+	size_t len;
+
+	len = min_t(size_t, count, pipe->native.length - head);
+	if (len)
+		memcpy(pipe->fifo + head, data, len);
+
+	if (len != count)
+		memcpy(pipe->fifo, data + len, count - len);
+
+	head += count;
+	if (head >= pipe->native.length)
+		head -= pipe->native.length;
+
+	return head;
+}
+
+static void glink_smem_tx_write(struct qcom_glink_pipe *glink_pipe,
+				const void *hdr, size_t hlen,
+				const void *data, size_t dlen)
+{
+	struct glink_smem_pipe *pipe = to_smem_pipe(glink_pipe);
+	unsigned int head;
+
+	head = le32_to_cpu(*pipe->head);
+
+	head = glink_smem_tx_write_one(pipe, head, hdr, hlen);
+	head = glink_smem_tx_write_one(pipe, head, data, dlen);
+
+	/* Ensure head is always aligned to 8 bytes */
+	head = ALIGN(head, 8);
+	if (head >= pipe->native.length)
+		head -= pipe->native.length;
+
+	*pipe->head = cpu_to_le32(head);
+}
+
+static void qcom_glink_smem_release(struct device *dev)
+{
+	kfree(dev);
+}
+
+struct qcom_glink *qcom_glink_smem_register(struct device *parent,
+					    struct device_node *node)
+{
+	struct glink_smem_pipe *rx_pipe;
+	struct glink_smem_pipe *tx_pipe;
+	struct qcom_glink *glink;
+	struct device *dev;
+	u32 remote_pid;
+	__le32 *descs;
+	size_t size;
+	int ret;
+
+	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+	if (!dev)
+		return ERR_PTR(-ENOMEM);
+
+	dev->parent = parent;
+	dev->of_node = node;
+	dev->release = qcom_glink_smem_release;
+	dev_set_name(dev, "%s:%s", node->parent->name, node->name);
+	ret = device_register(dev);
+	if (ret) {
+		pr_err("failed to register glink edge\n");
+		return ERR_PTR(ret);
+	}
+
+	ret = of_property_read_u32(dev->of_node, "qcom,remote-pid",
+				   &remote_pid);
+	if (ret) {
+		dev_err(dev, "failed to parse qcom,remote-pid\n");
+		goto err_put_dev;
+	}
+
+	rx_pipe = devm_kzalloc(dev, sizeof(*rx_pipe), GFP_KERNEL);
+	tx_pipe = devm_kzalloc(dev, sizeof(*tx_pipe), GFP_KERNEL);
+	if (!rx_pipe || !tx_pipe) {
+		ret = -ENOMEM;
+		goto err_put_dev;
+	}
+
+	ret = qcom_smem_alloc(remote_pid,
+			      SMEM_GLINK_NATIVE_XPRT_DESCRIPTOR, 32);
+	if (ret && ret != -EEXIST) {
+		dev_err(dev, "failed to allocate glink descriptors\n");
+		goto err_put_dev;
+	}
+
+	descs = qcom_smem_get(remote_pid,
+			      SMEM_GLINK_NATIVE_XPRT_DESCRIPTOR, &size);
+	if (IS_ERR(descs)) {
+		dev_err(dev, "failed to acquire xprt descriptor\n");
+		ret = PTR_ERR(descs);
+		goto err_put_dev;
+	}
+
+	if (size != 32) {
+		dev_err(dev, "glink descriptor of invalid size\n");
+		ret = -EINVAL;
+		goto err_put_dev;
+	}
+
+	tx_pipe->tail = &descs[0];
+	tx_pipe->head = &descs[1];
+	rx_pipe->tail = &descs[2];
+	rx_pipe->head = &descs[3];
+
+	ret = qcom_smem_alloc(remote_pid, SMEM_GLINK_NATIVE_XPRT_FIFO_0,
+			      SZ_16K);
+	if (ret && ret != -EEXIST) {
+		dev_err(dev, "failed to allocate TX fifo\n");
+		goto err_put_dev;
+	}
+
+	tx_pipe->fifo = qcom_smem_get(remote_pid, SMEM_GLINK_NATIVE_XPRT_FIFO_0,
+				      &tx_pipe->native.length);
+	if (IS_ERR(tx_pipe->fifo)) {
+		dev_err(dev, "failed to acquire TX fifo\n");
+		ret = PTR_ERR(tx_pipe->fifo);
+		goto err_put_dev;
+	}
+
+	rx_pipe->native.avail = glink_smem_rx_avail;
+	rx_pipe->native.peak = glink_smem_rx_peak;
+	rx_pipe->native.advance = glink_smem_rx_advance;
+	rx_pipe->remote_pid = remote_pid;
+
+	tx_pipe->native.avail = glink_smem_tx_avail;
+	tx_pipe->native.write = glink_smem_tx_write;
+	tx_pipe->remote_pid = remote_pid;
+
+	*rx_pipe->tail = 0;
+	*tx_pipe->head = 0;
+
+	glink = qcom_glink_native_probe(dev,
+					GLINK_FEATURE_INTENT_REUSE,
+					&rx_pipe->native, &tx_pipe->native,
+					false);
+	if (IS_ERR(glink)) {
+		ret = PTR_ERR(glink);
+		goto err_put_dev;
+	}
+
+	return glink;
+
+err_put_dev:
+	put_device(dev);
+
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(qcom_glink_smem_register);
+
+void qcom_glink_smem_unregister(struct qcom_glink *glink)
+{
+	qcom_glink_native_remove(glink);
+	qcom_glink_native_unregister(glink);
+}
+EXPORT_SYMBOL_GPL(qcom_glink_smem_unregister);
+
+MODULE_AUTHOR("Bjorn Andersson <bjorn.andersson@linaro.org>");
+MODULE_DESCRIPTION("Qualcomm GLINK SMEM driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/rpmsg/qcom_smd.c b/drivers/rpmsg/qcom_smd.c
index a0a39a8821a3..b01774e9fac0 100644
--- a/drivers/rpmsg/qcom_smd.c
+++ b/drivers/rpmsg/qcom_smd.c
@@ -1368,6 +1368,7 @@ struct qcom_smd_edge *qcom_smd_register_edge(struct device *parent,
 
 	edge->dev.parent = parent;
 	edge->dev.release = qcom_smd_edge_release;
+	edge->dev.of_node = node;
 	edge->dev.groups = qcom_smd_edge_groups;
 	dev_set_name(&edge->dev, "%s:%s", dev_name(parent), node->name);
 	ret = device_register(&edge->dev);
diff --git a/drivers/rpmsg/virtio_rpmsg_bus.c b/drivers/rpmsg/virtio_rpmsg_bus.c
index eee2a9f77d37..82b83002fcba 100644
--- a/drivers/rpmsg/virtio_rpmsg_bus.c
+++ b/drivers/rpmsg/virtio_rpmsg_bus.c
@@ -45,6 +45,7 @@
  * @rbufs:	kernel address of rx buffers
  * @sbufs:	kernel address of tx buffers
  * @num_bufs:	total number of buffers for rx and tx
+ * @buf_size:   size of one rx or tx buffer
  * @last_sbuf:	index of last tx buffer used
  * @bufs_dma:	dma base addr of the buffers
  * @tx_lock:	protects svq, sbufs and sleepers, to allow concurrent senders.
@@ -65,6 +66,7 @@ struct virtproc_info {
 	struct virtqueue *rvq, *svq;
 	void *rbufs, *sbufs;
 	unsigned int num_bufs;
+	unsigned int buf_size;
 	int last_sbuf;
 	dma_addr_t bufs_dma;
 	struct mutex tx_lock;
@@ -158,7 +160,7 @@ struct virtio_rpmsg_channel {
  * processor.
  */
 #define MAX_RPMSG_NUM_BUFS	(512)
-#define RPMSG_BUF_SIZE		(512)
+#define MAX_RPMSG_BUF_SIZE	(512)
 
 /*
  * Local addresses are dynamically allocated on-demand.
@@ -193,6 +195,28 @@ static const struct rpmsg_endpoint_ops virtio_endpoint_ops = {
 };
 
 /**
+ * rpmsg_sg_init - initialize scatterlist according to cpu address location
+ * @sg: scatterlist to fill
+ * @cpu_addr: virtual address of the buffer
+ * @len: buffer length
+ *
+ * An internal function filling scatterlist according to virtual address
+ * location (in vmalloc or in kernel).
+ */
+static void
+rpmsg_sg_init(struct scatterlist *sg, void *cpu_addr, unsigned int len)
+{
+	if (is_vmalloc_addr(cpu_addr)) {
+		sg_init_table(sg, 1);
+		sg_set_page(sg, vmalloc_to_page(cpu_addr), len,
+			    offset_in_page(cpu_addr));
+	} else {
+		WARN_ON(!virt_addr_valid(cpu_addr));
+		sg_init_one(sg, cpu_addr, len);
+	}
+}
+
+/**
  * __ept_release() - deallocate an rpmsg endpoint
  * @kref: the ept's reference count
  *
@@ -435,7 +459,7 @@ static void *get_a_tx_buf(struct virtproc_info *vrp)
 	 * (half of our buffers are used for sending messages)
 	 */
 	if (vrp->last_sbuf < vrp->num_bufs / 2)
-		ret = vrp->sbufs + RPMSG_BUF_SIZE * vrp->last_sbuf++;
+		ret = vrp->sbufs + vrp->buf_size * vrp->last_sbuf++;
 	/* or recycle a used one */
 	else
 		ret = virtqueue_get_buf(vrp->svq, &len);
@@ -561,7 +585,7 @@ static int rpmsg_send_offchannel_raw(struct rpmsg_device *rpdev,
 	 * messaging), or to improve the buffer allocator, to support
 	 * variable-length buffer sizes.
 	 */
-	if (len > RPMSG_BUF_SIZE - sizeof(struct rpmsg_hdr)) {
+	if (len > vrp->buf_size - sizeof(struct rpmsg_hdr)) {
 		dev_err(dev, "message is too big (%d)\n", len);
 		return -EMSGSIZE;
 	}
@@ -610,7 +634,7 @@ static int rpmsg_send_offchannel_raw(struct rpmsg_device *rpdev,
 			 msg, sizeof(*msg) + msg->len, true);
 #endif
 
-	sg_init_one(&sg, msg, sizeof(*msg) + len);
+	rpmsg_sg_init(&sg, msg, sizeof(*msg) + len);
 
 	mutex_lock(&vrp->tx_lock);
 
@@ -632,7 +656,6 @@ out:
 	mutex_unlock(&vrp->tx_lock);
 	return err;
 }
-EXPORT_SYMBOL(rpmsg_send_offchannel_raw);
 
 static int virtio_rpmsg_send(struct rpmsg_endpoint *ept, void *data, int len)
 {
@@ -702,7 +725,7 @@ static int rpmsg_recv_single(struct virtproc_info *vrp, struct device *dev,
 	 * We currently use fixed-sized buffers, so trivially sanitize
 	 * the reported payload length.
 	 */
-	if (len > RPMSG_BUF_SIZE ||
+	if (len > vrp->buf_size ||
 	    msg->len > (len - sizeof(struct rpmsg_hdr))) {
 		dev_warn(dev, "inbound msg too big: (%d, %d)\n", len, msg->len);
 		return -EINVAL;
@@ -735,7 +758,7 @@ static int rpmsg_recv_single(struct virtproc_info *vrp, struct device *dev,
 		dev_warn(dev, "msg received with no recipient\n");
 
 	/* publish the real size of the buffer */
-	sg_init_one(&sg, msg, RPMSG_BUF_SIZE);
+	rpmsg_sg_init(&sg, msg, vrp->buf_size);
 
 	/* add the buffer back to the remote processor's virtqueue */
 	err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, msg, GFP_KERNEL);
@@ -892,7 +915,9 @@ static int rpmsg_probe(struct virtio_device *vdev)
 	else
 		vrp->num_bufs = MAX_RPMSG_NUM_BUFS;
 
-	total_buf_space = vrp->num_bufs * RPMSG_BUF_SIZE;
+	vrp->buf_size = MAX_RPMSG_BUF_SIZE;
+
+	total_buf_space = vrp->num_bufs * vrp->buf_size;
 
 	/* allocate coherent memory for the buffers */
 	bufs_va = dma_alloc_coherent(vdev->dev.parent->parent,
@@ -915,9 +940,9 @@ static int rpmsg_probe(struct virtio_device *vdev)
 	/* set up the receive buffers */
 	for (i = 0; i < vrp->num_bufs / 2; i++) {
 		struct scatterlist sg;
-		void *cpu_addr = vrp->rbufs + i * RPMSG_BUF_SIZE;
+		void *cpu_addr = vrp->rbufs + i * vrp->buf_size;
 
-		sg_init_one(&sg, cpu_addr, RPMSG_BUF_SIZE);
+		rpmsg_sg_init(&sg, cpu_addr, vrp->buf_size);
 
 		err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr,
 					  GFP_KERNEL);
@@ -982,7 +1007,7 @@ static int rpmsg_remove_device(struct device *dev, void *data)
 static void rpmsg_remove(struct virtio_device *vdev)
 {
 	struct virtproc_info *vrp = vdev->priv;
-	size_t total_buf_space = vrp->num_bufs * RPMSG_BUF_SIZE;
+	size_t total_buf_space = vrp->num_bufs * vrp->buf_size;
 	int ret;
 
 	vdev->config->reset(vdev);
diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig
index 72419ac2c52a..e0e58f3b1420 100644
--- a/drivers/rtc/Kconfig
+++ b/drivers/rtc/Kconfig
@@ -227,14 +227,14 @@ config RTC_DRV_AS3722
 	  will be called rtc-as3722.
 
 config RTC_DRV_DS1307
-	tristate "Dallas/Maxim DS1307/37/38/39/40, ST M41T00, EPSON RX-8025, ISL12057"
+	tristate "Dallas/Maxim DS1307/37/38/39/40/41, ST M41T00, EPSON RX-8025, ISL12057"
 	help
 	  If you say yes here you get support for various compatible RTC
 	  chips (often with battery backup) connected with I2C. This driver
-	  should handle DS1307, DS1337, DS1338, DS1339, DS1340, ST M41T00,
-	  EPSON RX-8025, Intersil ISL12057 and probably other chips. In some
-	  cases the RTC must already have been initialized (by manufacturing or
-	  a bootloader).
+	  should handle DS1307, DS1337, DS1338, DS1339, DS1340, DS1341,
+	  ST M41T00, EPSON RX-8025, Intersil ISL12057 and probably other chips.
+	  In some cases the RTC must already have been initialized (by
+	  manufacturing or a bootloader).
 
 	  The first seven registers on these chips hold an RTC, and other
 	  registers may add features such as NVRAM, a trickle charger for
@@ -371,11 +371,11 @@ config RTC_DRV_MAX77686
 	  will be called rtc-max77686.
 
 config RTC_DRV_RK808
-	tristate "Rockchip RK808/RK818 RTC"
+	tristate "Rockchip RK805/RK808/RK818 RTC"
 	depends on MFD_RK808
 	help
 	  If you say yes here you will get support for the
-	  RTC of RK808 and RK818 PMIC.
+	  RTC of RK805, RK808 and RK818 PMIC.
 
 	  This driver can also be built as a module. If so, the module
 	  will be called rk808-rtc.
@@ -1765,6 +1765,14 @@ config RTC_DRV_CPCAP
 	   Say y here for CPCAP rtc found on some Motorola phones
 	   and tablets such as Droid 4.
 
+config RTC_DRV_RTD119X
+	bool "Realtek RTD129x RTC"
+	depends on ARCH_REALTEK || COMPILE_TEST
+	default ARCH_REALTEK
+	help
+	  If you say yes here, you get support for the RTD1295 SoC
+	  Real Time Clock.
+
 comment "HID Sensor RTC drivers"
 
 config RTC_DRV_HID_SENSOR_TIME
@@ -1780,5 +1788,13 @@ config RTC_DRV_HID_SENSOR_TIME
 	  If this driver is compiled as a module, it will be named
 	  rtc-hid-sensor-time.
 
+config RTC_DRV_GOLDFISH
+	tristate "Goldfish Real Time Clock"
+	depends on MIPS && (GOLDFISH || COMPILE_TEST)
+	help
+	  Say yes to enable RTC driver for the Goldfish based virtual platform.
+
+	  Goldfish is a code name for the virtual platform developed by Google
+	  for Android emulation.
 
 endif # RTC_CLASS
diff --git a/drivers/rtc/Makefile b/drivers/rtc/Makefile
index acd366b41c85..7230014c92af 100644
--- a/drivers/rtc/Makefile
+++ b/drivers/rtc/Makefile
@@ -131,6 +131,7 @@ obj-$(CONFIG_RTC_DRV_RP5C01)	+= rtc-rp5c01.o
 obj-$(CONFIG_RTC_DRV_RS5C313)	+= rtc-rs5c313.o
 obj-$(CONFIG_RTC_DRV_RS5C348)	+= rtc-rs5c348.o
 obj-$(CONFIG_RTC_DRV_RS5C372)	+= rtc-rs5c372.o
+obj-$(CONFIG_RTC_DRV_RTD119X)	+= rtc-rtd119x.o
 obj-$(CONFIG_RTC_DRV_RV3029C2)	+= rtc-rv3029c2.o
 obj-$(CONFIG_RTC_DRV_RV8803)	+= rtc-rv8803.o
 obj-$(CONFIG_RTC_DRV_RX4581)	+= rtc-rx4581.o
@@ -170,3 +171,4 @@ obj-$(CONFIG_RTC_DRV_WM8350)	+= rtc-wm8350.o
 obj-$(CONFIG_RTC_DRV_X1205)	+= rtc-x1205.o
 obj-$(CONFIG_RTC_DRV_XGENE)	+= rtc-xgene.o
 obj-$(CONFIG_RTC_DRV_ZYNQMP)	+= rtc-zynqmp.o
+obj-$(CONFIG_RTC_DRV_GOLDFISH)	+= rtc-goldfish.o
diff --git a/drivers/rtc/rtc-dev.c b/drivers/rtc/rtc-dev.c
index 794bc4fa4937..00efe24a6063 100644
--- a/drivers/rtc/rtc-dev.c
+++ b/drivers/rtc/rtc-dev.c
@@ -24,28 +24,19 @@ static dev_t rtc_devt;
 
 static int rtc_dev_open(struct inode *inode, struct file *file)
 {
-	int err;
 	struct rtc_device *rtc = container_of(inode->i_cdev,
 					struct rtc_device, char_dev);
-	const struct rtc_class_ops *ops = rtc->ops;
 
 	if (test_and_set_bit_lock(RTC_DEV_BUSY, &rtc->flags))
 		return -EBUSY;
 
 	file->private_data = rtc;
 
-	err = ops->open ? ops->open(rtc->dev.parent) : 0;
-	if (err == 0) {
-		spin_lock_irq(&rtc->irq_lock);
-		rtc->irq_data = 0;
-		spin_unlock_irq(&rtc->irq_lock);
-
-		return 0;
-	}
+	spin_lock_irq(&rtc->irq_lock);
+	rtc->irq_data = 0;
+	spin_unlock_irq(&rtc->irq_lock);
 
-	/* something has gone wrong */
-	clear_bit_unlock(RTC_DEV_BUSY, &rtc->flags);
-	return err;
+	return 0;
 }
 
 #ifdef CONFIG_RTC_INTF_DEV_UIE_EMUL
@@ -438,9 +429,6 @@ static int rtc_dev_release(struct inode *inode, struct file *file)
 	rtc_update_irq_enable(rtc, 0);
 	rtc_irq_set_state(rtc, NULL, 0);
 
-	if (rtc->ops->release)
-		rtc->ops->release(rtc->dev.parent);
-
 	clear_bit_unlock(RTC_DEV_BUSY, &rtc->flags);
 	return 0;
 }
diff --git a/drivers/rtc/rtc-dm355evm.c b/drivers/rtc/rtc-dm355evm.c
index f225cd873ff6..97d8259b9494 100644
--- a/drivers/rtc/rtc-dm355evm.c
+++ b/drivers/rtc/rtc-dm355evm.c
@@ -13,7 +13,7 @@
 #include <linux/rtc.h>
 #include <linux/platform_device.h>
 
-#include <linux/i2c/dm355evm_msp.h>
+#include <linux/mfd/dm355evm_msp.h>
 #include <linux/module.h>
 
 
diff --git a/drivers/rtc/rtc-ds1307.c b/drivers/rtc/rtc-ds1307.c
index 4fac49e55d47..e7d9215c9201 100644
--- a/drivers/rtc/rtc-ds1307.c
+++ b/drivers/rtc/rtc-ds1307.c
@@ -39,6 +39,7 @@ enum ds_type {
 	ds_1338,
 	ds_1339,
 	ds_1340,
+	ds_1341,
 	ds_1388,
 	ds_3231,
 	m41t0,
@@ -50,7 +51,6 @@ enum ds_type {
 	/* rs5c372 too?  different address... */
 };
 
-
 /* RTC registers don't differ much, except for the century flag */
 #define DS1307_REG_SECS		0x00	/* 00-59 */
 #	define DS1307_BIT_CH		0x80
@@ -113,11 +113,7 @@ enum ds_type {
 #	define RX8025_BIT_VDET		0x40
 #	define RX8025_BIT_XST		0x20
 
-
 struct ds1307 {
-	u8			offset; /* register's offset */
-	u8			regs[11];
-	u16			nvram_offset;
 	struct nvmem_config	nvmem_cfg;
 	enum ds_type		type;
 	unsigned long		flags;
@@ -126,7 +122,6 @@ struct ds1307 {
 	struct device		*dev;
 	struct regmap		*regmap;
 	const char		*name;
-	int			irq;
 	struct rtc_device	*rtc;
 #ifdef CONFIG_COMMON_CLK
 	struct clk_hw		clks[2];
@@ -137,18 +132,47 @@ struct chip_desc {
 	unsigned		alarm:1;
 	u16			nvram_offset;
 	u16			nvram_size;
+	u8			offset; /* register's offset */
 	u8			century_reg;
 	u8			century_enable_bit;
 	u8			century_bit;
+	u8			bbsqi_bit;
+	irq_handler_t		irq_handler;
+	const struct rtc_class_ops *rtc_ops;
 	u16			trickle_charger_reg;
-	u8			trickle_charger_setup;
-	u8			(*do_trickle_setup)(struct ds1307 *, uint32_t,
+	u8			(*do_trickle_setup)(struct ds1307 *, u32,
 						    bool);
 };
 
-static u8 do_trickle_setup_ds1339(struct ds1307 *, uint32_t ohms, bool diode);
+static int ds1307_get_time(struct device *dev, struct rtc_time *t);
+static int ds1307_set_time(struct device *dev, struct rtc_time *t);
+static u8 do_trickle_setup_ds1339(struct ds1307 *, u32 ohms, bool diode);
+static irqreturn_t rx8130_irq(int irq, void *dev_id);
+static int rx8130_read_alarm(struct device *dev, struct rtc_wkalrm *t);
+static int rx8130_set_alarm(struct device *dev, struct rtc_wkalrm *t);
+static int rx8130_alarm_irq_enable(struct device *dev, unsigned int enabled);
+static irqreturn_t mcp794xx_irq(int irq, void *dev_id);
+static int mcp794xx_read_alarm(struct device *dev, struct rtc_wkalrm *t);
+static int mcp794xx_set_alarm(struct device *dev, struct rtc_wkalrm *t);
+static int mcp794xx_alarm_irq_enable(struct device *dev, unsigned int enabled);
 
-static struct chip_desc chips[last_ds_type] = {
+static const struct rtc_class_ops rx8130_rtc_ops = {
+	.read_time      = ds1307_get_time,
+	.set_time       = ds1307_set_time,
+	.read_alarm     = rx8130_read_alarm,
+	.set_alarm      = rx8130_set_alarm,
+	.alarm_irq_enable = rx8130_alarm_irq_enable,
+};
+
+static const struct rtc_class_ops mcp794xx_rtc_ops = {
+	.read_time      = ds1307_get_time,
+	.set_time       = ds1307_set_time,
+	.read_alarm     = mcp794xx_read_alarm,
+	.set_alarm      = mcp794xx_set_alarm,
+	.alarm_irq_enable = mcp794xx_alarm_irq_enable,
+};
+
+static const struct chip_desc chips[last_ds_type] = {
 	[ds_1307] = {
 		.nvram_offset	= 8,
 		.nvram_size	= 56,
@@ -170,6 +194,7 @@ static struct chip_desc chips[last_ds_type] = {
 		.alarm		= 1,
 		.century_reg	= DS1307_REG_MONTH,
 		.century_bit	= DS1337_BIT_CENTURY,
+		.bbsqi_bit	= DS1339_BIT_BBSQI,
 		.trickle_charger_reg = 0x10,
 		.do_trickle_setup = &do_trickle_setup_ds1339,
 	},
@@ -179,25 +204,36 @@ static struct chip_desc chips[last_ds_type] = {
 		.century_bit	= DS1340_BIT_CENTURY,
 		.trickle_charger_reg = 0x08,
 	},
+	[ds_1341] = {
+		.century_reg	= DS1307_REG_MONTH,
+		.century_bit	= DS1337_BIT_CENTURY,
+	},
 	[ds_1388] = {
+		.offset		= 1,
 		.trickle_charger_reg = 0x0a,
 	},
 	[ds_3231] = {
 		.alarm		= 1,
 		.century_reg	= DS1307_REG_MONTH,
 		.century_bit	= DS1337_BIT_CENTURY,
+		.bbsqi_bit	= DS3231_BIT_BBSQW,
 	},
 	[rx_8130] = {
 		.alarm		= 1,
 		/* this is battery backed SRAM */
 		.nvram_offset	= 0x20,
 		.nvram_size	= 4,	/* 32bit (4 word x 8 bit) */
+		.offset		= 0x10,
+		.irq_handler = rx8130_irq,
+		.rtc_ops = &rx8130_rtc_ops,
 	},
 	[mcp794xx] = {
 		.alarm		= 1,
 		/* this is battery backed SRAM */
 		.nvram_offset	= 0x20,
 		.nvram_size	= 0x40,
+		.irq_handler = mcp794xx_irq,
+		.rtc_ops = &mcp794xx_rtc_ops,
 	},
 };
 
@@ -209,6 +245,7 @@ static const struct i2c_device_id ds1307_id[] = {
 	{ "ds1339", ds_1339 },
 	{ "ds1388", ds_1388 },
 	{ "ds1340", ds_1340 },
+	{ "ds1341", ds_1341 },
 	{ "ds3231", ds_3231 },
 	{ "m41t0", m41t0 },
 	{ "m41t00", m41t00 },
@@ -253,6 +290,10 @@ static const struct of_device_id ds1307_of_match[] = {
 		.data = (void *)ds_1340
 	},
 	{
+		.compatible = "dallas,ds1341",
+		.data = (void *)ds_1341
+	},
+	{
 		.compatible = "maxim,ds3231",
 		.data = (void *)ds_3231
 	},
@@ -298,6 +339,7 @@ static const struct acpi_device_id ds1307_acpi_ids[] = {
 	{ .id = "DS1339", .driver_data = ds_1339 },
 	{ .id = "DS1388", .driver_data = ds_1388 },
 	{ .id = "DS1340", .driver_data = ds_1340 },
+	{ .id = "DS1341", .driver_data = ds_1341 },
 	{ .id = "DS3231", .driver_data = ds_3231 },
 	{ .id = "M41T0", .driver_data = m41t0 },
 	{ .id = "M41T00", .driver_data = m41t00 },
@@ -352,34 +394,36 @@ static int ds1307_get_time(struct device *dev, struct rtc_time *t)
 	struct ds1307	*ds1307 = dev_get_drvdata(dev);
 	int		tmp, ret;
 	const struct chip_desc *chip = &chips[ds1307->type];
+	u8 regs[7];
 
 	/* read the RTC date and time registers all at once */
-	ret = regmap_bulk_read(ds1307->regmap, ds1307->offset, ds1307->regs, 7);
+	ret = regmap_bulk_read(ds1307->regmap, chip->offset, regs,
+			       sizeof(regs));
 	if (ret) {
 		dev_err(dev, "%s error %d\n", "read", ret);
 		return ret;
 	}
 
-	dev_dbg(dev, "%s: %7ph\n", "read", ds1307->regs);
+	dev_dbg(dev, "%s: %7ph\n", "read", regs);
 
 	/* if oscillator fail bit is set, no data can be trusted */
 	if (ds1307->type == m41t0 &&
-	    ds1307->regs[DS1307_REG_MIN] & M41T0_BIT_OF) {
+	    regs[DS1307_REG_MIN] & M41T0_BIT_OF) {
 		dev_warn_once(dev, "oscillator failed, set time!\n");
 		return -EINVAL;
 	}
 
-	t->tm_sec = bcd2bin(ds1307->regs[DS1307_REG_SECS] & 0x7f);
-	t->tm_min = bcd2bin(ds1307->regs[DS1307_REG_MIN] & 0x7f);
-	tmp = ds1307->regs[DS1307_REG_HOUR] & 0x3f;
+	t->tm_sec = bcd2bin(regs[DS1307_REG_SECS] & 0x7f);
+	t->tm_min = bcd2bin(regs[DS1307_REG_MIN] & 0x7f);
+	tmp = regs[DS1307_REG_HOUR] & 0x3f;
 	t->tm_hour = bcd2bin(tmp);
-	t->tm_wday = bcd2bin(ds1307->regs[DS1307_REG_WDAY] & 0x07) - 1;
-	t->tm_mday = bcd2bin(ds1307->regs[DS1307_REG_MDAY] & 0x3f);
-	tmp = ds1307->regs[DS1307_REG_MONTH] & 0x1f;
+	t->tm_wday = bcd2bin(regs[DS1307_REG_WDAY] & 0x07) - 1;
+	t->tm_mday = bcd2bin(regs[DS1307_REG_MDAY] & 0x3f);
+	tmp = regs[DS1307_REG_MONTH] & 0x1f;
 	t->tm_mon = bcd2bin(tmp) - 1;
-	t->tm_year = bcd2bin(ds1307->regs[DS1307_REG_YEAR]) + 100;
+	t->tm_year = bcd2bin(regs[DS1307_REG_YEAR]) + 100;
 
-	if (ds1307->regs[chip->century_reg] & chip->century_bit &&
+	if (regs[chip->century_reg] & chip->century_bit &&
 	    IS_ENABLED(CONFIG_RTC_DRV_DS1307_CENTURY))
 		t->tm_year += 100;
 
@@ -399,7 +443,7 @@ static int ds1307_set_time(struct device *dev, struct rtc_time *t)
 	const struct chip_desc *chip = &chips[ds1307->type];
 	int		result;
 	int		tmp;
-	u8		*buf = ds1307->regs;
+	u8		regs[7];
 
 	dev_dbg(dev, "%s secs=%d, mins=%d, "
 		"hours=%d, mday=%d, mon=%d, year=%d, wday=%d\n",
@@ -418,35 +462,36 @@ static int ds1307_set_time(struct device *dev, struct rtc_time *t)
 		return -EINVAL;
 #endif
 
-	buf[DS1307_REG_SECS] = bin2bcd(t->tm_sec);
-	buf[DS1307_REG_MIN] = bin2bcd(t->tm_min);
-	buf[DS1307_REG_HOUR] = bin2bcd(t->tm_hour);
-	buf[DS1307_REG_WDAY] = bin2bcd(t->tm_wday + 1);
-	buf[DS1307_REG_MDAY] = bin2bcd(t->tm_mday);
-	buf[DS1307_REG_MONTH] = bin2bcd(t->tm_mon + 1);
+	regs[DS1307_REG_SECS] = bin2bcd(t->tm_sec);
+	regs[DS1307_REG_MIN] = bin2bcd(t->tm_min);
+	regs[DS1307_REG_HOUR] = bin2bcd(t->tm_hour);
+	regs[DS1307_REG_WDAY] = bin2bcd(t->tm_wday + 1);
+	regs[DS1307_REG_MDAY] = bin2bcd(t->tm_mday);
+	regs[DS1307_REG_MONTH] = bin2bcd(t->tm_mon + 1);
 
 	/* assume 20YY not 19YY */
 	tmp = t->tm_year - 100;
-	buf[DS1307_REG_YEAR] = bin2bcd(tmp);
+	regs[DS1307_REG_YEAR] = bin2bcd(tmp);
 
 	if (chip->century_enable_bit)
-		buf[chip->century_reg] |= chip->century_enable_bit;
+		regs[chip->century_reg] |= chip->century_enable_bit;
 	if (t->tm_year > 199 && chip->century_bit)
-		buf[chip->century_reg] |= chip->century_bit;
+		regs[chip->century_reg] |= chip->century_bit;
 
 	if (ds1307->type == mcp794xx) {
 		/*
 		 * these bits were cleared when preparing the date/time
 		 * values and need to be set again before writing the
-		 * buffer out to the device.
+		 * regsfer out to the device.
 		 */
-		buf[DS1307_REG_SECS] |= MCP794XX_BIT_ST;
-		buf[DS1307_REG_WDAY] |= MCP794XX_BIT_VBATEN;
+		regs[DS1307_REG_SECS] |= MCP794XX_BIT_ST;
+		regs[DS1307_REG_WDAY] |= MCP794XX_BIT_VBATEN;
 	}
 
-	dev_dbg(dev, "%s: %7ph\n", "write", buf);
+	dev_dbg(dev, "%s: %7ph\n", "write", regs);
 
-	result = regmap_bulk_write(ds1307->regmap, ds1307->offset, buf, 7);
+	result = regmap_bulk_write(ds1307->regmap, chip->offset, regs,
+				   sizeof(regs));
 	if (result) {
 		dev_err(dev, "%s error %d\n", "write", result);
 		return result;
@@ -458,33 +503,34 @@ static int ds1337_read_alarm(struct device *dev, struct rtc_wkalrm *t)
 {
 	struct ds1307		*ds1307 = dev_get_drvdata(dev);
 	int			ret;
+	u8			regs[9];
 
 	if (!test_bit(HAS_ALARM, &ds1307->flags))
 		return -EINVAL;
 
 	/* read all ALARM1, ALARM2, and status registers at once */
 	ret = regmap_bulk_read(ds1307->regmap, DS1339_REG_ALARM1_SECS,
-			       ds1307->regs, 9);
+			       regs, sizeof(regs));
 	if (ret) {
 		dev_err(dev, "%s error %d\n", "alarm read", ret);
 		return ret;
 	}
 
 	dev_dbg(dev, "%s: %4ph, %3ph, %2ph\n", "alarm read",
-		&ds1307->regs[0], &ds1307->regs[4], &ds1307->regs[7]);
+		&regs[0], &regs[4], &regs[7]);
 
 	/*
 	 * report alarm time (ALARM1); assume 24 hour and day-of-month modes,
 	 * and that all four fields are checked matches
 	 */
-	t->time.tm_sec = bcd2bin(ds1307->regs[0] & 0x7f);
-	t->time.tm_min = bcd2bin(ds1307->regs[1] & 0x7f);
-	t->time.tm_hour = bcd2bin(ds1307->regs[2] & 0x3f);
-	t->time.tm_mday = bcd2bin(ds1307->regs[3] & 0x3f);
+	t->time.tm_sec = bcd2bin(regs[0] & 0x7f);
+	t->time.tm_min = bcd2bin(regs[1] & 0x7f);
+	t->time.tm_hour = bcd2bin(regs[2] & 0x3f);
+	t->time.tm_mday = bcd2bin(regs[3] & 0x3f);
 
 	/* ... and status */
-	t->enabled = !!(ds1307->regs[7] & DS1337_BIT_A1IE);
-	t->pending = !!(ds1307->regs[8] & DS1337_BIT_A1I);
+	t->enabled = !!(regs[7] & DS1337_BIT_A1IE);
+	t->pending = !!(regs[8] & DS1337_BIT_A1I);
 
 	dev_dbg(dev, "%s secs=%d, mins=%d, "
 		"hours=%d, mday=%d, enabled=%d, pending=%d\n",
@@ -498,7 +544,7 @@ static int ds1337_read_alarm(struct device *dev, struct rtc_wkalrm *t)
 static int ds1337_set_alarm(struct device *dev, struct rtc_wkalrm *t)
 {
 	struct ds1307		*ds1307 = dev_get_drvdata(dev);
-	unsigned char		*buf = ds1307->regs;
+	unsigned char		regs[9];
 	u8			control, status;
 	int			ret;
 
@@ -512,33 +558,35 @@ static int ds1337_set_alarm(struct device *dev, struct rtc_wkalrm *t)
 		t->enabled, t->pending);
 
 	/* read current status of both alarms and the chip */
-	ret = regmap_bulk_read(ds1307->regmap, DS1339_REG_ALARM1_SECS, buf, 9);
+	ret = regmap_bulk_read(ds1307->regmap, DS1339_REG_ALARM1_SECS, regs,
+			       sizeof(regs));
 	if (ret) {
 		dev_err(dev, "%s error %d\n", "alarm write", ret);
 		return ret;
 	}
-	control = ds1307->regs[7];
-	status = ds1307->regs[8];
+	control = regs[7];
+	status = regs[8];
 
 	dev_dbg(dev, "%s: %4ph, %3ph, %02x %02x\n", "alarm set (old status)",
-		&ds1307->regs[0], &ds1307->regs[4], control, status);
+		&regs[0], &regs[4], control, status);
 
 	/* set ALARM1, using 24 hour and day-of-month modes */
-	buf[0] = bin2bcd(t->time.tm_sec);
-	buf[1] = bin2bcd(t->time.tm_min);
-	buf[2] = bin2bcd(t->time.tm_hour);
-	buf[3] = bin2bcd(t->time.tm_mday);
+	regs[0] = bin2bcd(t->time.tm_sec);
+	regs[1] = bin2bcd(t->time.tm_min);
+	regs[2] = bin2bcd(t->time.tm_hour);
+	regs[3] = bin2bcd(t->time.tm_mday);
 
 	/* set ALARM2 to non-garbage */
-	buf[4] = 0;
-	buf[5] = 0;
-	buf[6] = 0;
+	regs[4] = 0;
+	regs[5] = 0;
+	regs[6] = 0;
 
 	/* disable alarms */
-	buf[7] = control & ~(DS1337_BIT_A1IE | DS1337_BIT_A2IE);
-	buf[8] = status & ~(DS1337_BIT_A1I | DS1337_BIT_A2I);
+	regs[7] = control & ~(DS1337_BIT_A1IE | DS1337_BIT_A2IE);
+	regs[8] = status & ~(DS1337_BIT_A1I | DS1337_BIT_A2I);
 
-	ret = regmap_bulk_write(ds1307->regmap, DS1339_REG_ALARM1_SECS, buf, 9);
+	ret = regmap_bulk_write(ds1307->regmap, DS1339_REG_ALARM1_SECS, regs,
+				sizeof(regs));
 	if (ret) {
 		dev_err(dev, "can't set alarm time\n");
 		return ret;
@@ -547,8 +595,8 @@ static int ds1337_set_alarm(struct device *dev, struct rtc_wkalrm *t)
 	/* optionally enable ALARM1 */
 	if (t->enabled) {
 		dev_dbg(dev, "alarm IRQ armed\n");
-		buf[7] |= DS1337_BIT_A1IE;	/* only ALARM1 is used */
-		regmap_write(ds1307->regmap, DS1337_REG_CONTROL, buf[7]);
+		regs[7] |= DS1337_BIT_A1IE;	/* only ALARM1 is used */
+		regmap_write(ds1307->regmap, DS1337_REG_CONTROL, regs[7]);
 	}
 
 	return 0;
@@ -584,11 +632,11 @@ static const struct rtc_class_ops ds13xx_rtc_ops = {
 #define RX8130_REG_ALARM_HOUR		0x08
 #define RX8130_REG_ALARM_WEEK_OR_DAY	0x09
 #define RX8130_REG_EXTENSION		0x0c
-#define RX8130_REG_EXTENSION_WADA	(1 << 3)
+#define RX8130_REG_EXTENSION_WADA	BIT(3)
 #define RX8130_REG_FLAG			0x0d
-#define RX8130_REG_FLAG_AF		(1 << 3)
+#define RX8130_REG_FLAG_AF		BIT(3)
 #define RX8130_REG_CONTROL0		0x0e
-#define RX8130_REG_CONTROL0_AIE		(1 << 3)
+#define RX8130_REG_CONTROL0_AIE		BIT(3)
 
 static irqreturn_t rx8130_irq(int irq, void *dev_id)
 {
@@ -600,7 +648,8 @@ static irqreturn_t rx8130_irq(int irq, void *dev_id)
 	mutex_lock(lock);
 
 	/* Read control registers. */
-	ret = regmap_bulk_read(ds1307->regmap, RX8130_REG_EXTENSION, ctl, 3);
+	ret = regmap_bulk_read(ds1307->regmap, RX8130_REG_EXTENSION, ctl,
+			       sizeof(ctl));
 	if (ret < 0)
 		goto out;
 	if (!(ctl[1] & RX8130_REG_FLAG_AF))
@@ -608,7 +657,8 @@ static irqreturn_t rx8130_irq(int irq, void *dev_id)
 	ctl[1] &= ~RX8130_REG_FLAG_AF;
 	ctl[2] &= ~RX8130_REG_CONTROL0_AIE;
 
-	ret = regmap_bulk_write(ds1307->regmap, RX8130_REG_EXTENSION, ctl, 3);
+	ret = regmap_bulk_write(ds1307->regmap, RX8130_REG_EXTENSION, ctl,
+				sizeof(ctl));
 	if (ret < 0)
 		goto out;
 
@@ -630,12 +680,14 @@ static int rx8130_read_alarm(struct device *dev, struct rtc_wkalrm *t)
 		return -EINVAL;
 
 	/* Read alarm registers. */
-	ret = regmap_bulk_read(ds1307->regmap, RX8130_REG_ALARM_MIN, ald, 3);
+	ret = regmap_bulk_read(ds1307->regmap, RX8130_REG_ALARM_MIN, ald,
+			       sizeof(ald));
 	if (ret < 0)
 		return ret;
 
 	/* Read control registers. */
-	ret = regmap_bulk_read(ds1307->regmap, RX8130_REG_EXTENSION, ctl, 3);
+	ret = regmap_bulk_read(ds1307->regmap, RX8130_REG_EXTENSION, ctl,
+			       sizeof(ctl));
 	if (ret < 0)
 		return ret;
 
@@ -676,7 +728,8 @@ static int rx8130_set_alarm(struct device *dev, struct rtc_wkalrm *t)
 		t->enabled, t->pending);
 
 	/* Read control registers. */
-	ret = regmap_bulk_read(ds1307->regmap, RX8130_REG_EXTENSION, ctl, 3);
+	ret = regmap_bulk_read(ds1307->regmap, RX8130_REG_EXTENSION, ctl,
+			       sizeof(ctl));
 	if (ret < 0)
 		return ret;
 
@@ -684,7 +737,8 @@ static int rx8130_set_alarm(struct device *dev, struct rtc_wkalrm *t)
 	ctl[1] |= RX8130_REG_FLAG_AF;
 	ctl[2] &= ~RX8130_REG_CONTROL0_AIE;
 
-	ret = regmap_bulk_write(ds1307->regmap, RX8130_REG_EXTENSION, ctl, 3);
+	ret = regmap_bulk_write(ds1307->regmap, RX8130_REG_EXTENSION, ctl,
+				sizeof(ctl));
 	if (ret < 0)
 		return ret;
 
@@ -693,7 +747,8 @@ static int rx8130_set_alarm(struct device *dev, struct rtc_wkalrm *t)
 	ald[1] = bin2bcd(t->time.tm_hour);
 	ald[2] = bin2bcd(t->time.tm_mday);
 
-	ret = regmap_bulk_write(ds1307->regmap, RX8130_REG_ALARM_MIN, ald, 3);
+	ret = regmap_bulk_write(ds1307->regmap, RX8130_REG_ALARM_MIN, ald,
+				sizeof(ald));
 	if (ret < 0)
 		return ret;
 
@@ -702,7 +757,8 @@ static int rx8130_set_alarm(struct device *dev, struct rtc_wkalrm *t)
 
 	ctl[2] |= RX8130_REG_CONTROL0_AIE;
 
-	return regmap_bulk_write(ds1307->regmap, RX8130_REG_EXTENSION, ctl, 3);
+	return regmap_bulk_write(ds1307->regmap, RX8130_REG_EXTENSION, ctl,
+				 sizeof(ctl));
 }
 
 static int rx8130_alarm_irq_enable(struct device *dev, unsigned int enabled)
@@ -725,14 +781,6 @@ static int rx8130_alarm_irq_enable(struct device *dev, unsigned int enabled)
 	return regmap_write(ds1307->regmap, RX8130_REG_CONTROL0, reg);
 }
 
-static const struct rtc_class_ops rx8130_rtc_ops = {
-	.read_time	= ds1307_get_time,
-	.set_time	= ds1307_set_time,
-	.read_alarm	= rx8130_read_alarm,
-	.set_alarm	= rx8130_set_alarm,
-	.alarm_irq_enable = rx8130_alarm_irq_enable,
-};
-
 /*----------------------------------------------------------------------*/
 
 /*
@@ -748,11 +796,11 @@ static const struct rtc_class_ops rx8130_rtc_ops = {
 #define MCP794XX_REG_ALARM0_CTRL	0x0d
 #define MCP794XX_REG_ALARM1_BASE	0x11
 #define MCP794XX_REG_ALARM1_CTRL	0x14
-#	define MCP794XX_BIT_ALMX_IF	(1 << 3)
-#	define MCP794XX_BIT_ALMX_C0	(1 << 4)
-#	define MCP794XX_BIT_ALMX_C1	(1 << 5)
-#	define MCP794XX_BIT_ALMX_C2	(1 << 6)
-#	define MCP794XX_BIT_ALMX_POL	(1 << 7)
+#	define MCP794XX_BIT_ALMX_IF	BIT(3)
+#	define MCP794XX_BIT_ALMX_C0	BIT(4)
+#	define MCP794XX_BIT_ALMX_C1	BIT(5)
+#	define MCP794XX_BIT_ALMX_C2	BIT(6)
+#	define MCP794XX_BIT_ALMX_POL	BIT(7)
 #	define MCP794XX_MSK_ALMX_MATCH	(MCP794XX_BIT_ALMX_C0 | \
 					 MCP794XX_BIT_ALMX_C1 | \
 					 MCP794XX_BIT_ALMX_C2)
@@ -793,37 +841,38 @@ out:
 static int mcp794xx_read_alarm(struct device *dev, struct rtc_wkalrm *t)
 {
 	struct ds1307 *ds1307 = dev_get_drvdata(dev);
-	u8 *regs = ds1307->regs;
+	u8 regs[10];
 	int ret;
 
 	if (!test_bit(HAS_ALARM, &ds1307->flags))
 		return -EINVAL;
 
 	/* Read control and alarm 0 registers. */
-	ret = regmap_bulk_read(ds1307->regmap, MCP794XX_REG_CONTROL, regs, 10);
+	ret = regmap_bulk_read(ds1307->regmap, MCP794XX_REG_CONTROL, regs,
+			       sizeof(regs));
 	if (ret)
 		return ret;
 
 	t->enabled = !!(regs[0] & MCP794XX_BIT_ALM0_EN);
 
 	/* Report alarm 0 time assuming 24-hour and day-of-month modes. */
-	t->time.tm_sec = bcd2bin(ds1307->regs[3] & 0x7f);
-	t->time.tm_min = bcd2bin(ds1307->regs[4] & 0x7f);
-	t->time.tm_hour = bcd2bin(ds1307->regs[5] & 0x3f);
-	t->time.tm_wday = bcd2bin(ds1307->regs[6] & 0x7) - 1;
-	t->time.tm_mday = bcd2bin(ds1307->regs[7] & 0x3f);
-	t->time.tm_mon = bcd2bin(ds1307->regs[8] & 0x1f) - 1;
+	t->time.tm_sec = bcd2bin(regs[3] & 0x7f);
+	t->time.tm_min = bcd2bin(regs[4] & 0x7f);
+	t->time.tm_hour = bcd2bin(regs[5] & 0x3f);
+	t->time.tm_wday = bcd2bin(regs[6] & 0x7) - 1;
+	t->time.tm_mday = bcd2bin(regs[7] & 0x3f);
+	t->time.tm_mon = bcd2bin(regs[8] & 0x1f) - 1;
 	t->time.tm_year = -1;
 	t->time.tm_yday = -1;
 	t->time.tm_isdst = -1;
 
 	dev_dbg(dev, "%s, sec=%d min=%d hour=%d wday=%d mday=%d mon=%d "
-		"enabled=%d polarity=%d irq=%d match=%d\n", __func__,
+		"enabled=%d polarity=%d irq=%d match=%lu\n", __func__,
 		t->time.tm_sec, t->time.tm_min, t->time.tm_hour,
 		t->time.tm_wday, t->time.tm_mday, t->time.tm_mon, t->enabled,
-		!!(ds1307->regs[6] & MCP794XX_BIT_ALMX_POL),
-		!!(ds1307->regs[6] & MCP794XX_BIT_ALMX_IF),
-		(ds1307->regs[6] & MCP794XX_MSK_ALMX_MATCH) >> 4);
+		!!(regs[6] & MCP794XX_BIT_ALMX_POL),
+		!!(regs[6] & MCP794XX_BIT_ALMX_IF),
+		(regs[6] & MCP794XX_MSK_ALMX_MATCH) >> 4);
 
 	return 0;
 }
@@ -831,7 +880,7 @@ static int mcp794xx_read_alarm(struct device *dev, struct rtc_wkalrm *t)
 static int mcp794xx_set_alarm(struct device *dev, struct rtc_wkalrm *t)
 {
 	struct ds1307 *ds1307 = dev_get_drvdata(dev);
-	unsigned char *regs = ds1307->regs;
+	unsigned char regs[10];
 	int ret;
 
 	if (!test_bit(HAS_ALARM, &ds1307->flags))
@@ -844,7 +893,8 @@ static int mcp794xx_set_alarm(struct device *dev, struct rtc_wkalrm *t)
 		t->enabled, t->pending);
 
 	/* Read control and alarm 0 registers. */
-	ret = regmap_bulk_read(ds1307->regmap, MCP794XX_REG_CONTROL, regs, 10);
+	ret = regmap_bulk_read(ds1307->regmap, MCP794XX_REG_CONTROL, regs,
+			       sizeof(regs));
 	if (ret)
 		return ret;
 
@@ -863,7 +913,8 @@ static int mcp794xx_set_alarm(struct device *dev, struct rtc_wkalrm *t)
 	/* Disable interrupt. We will not enable until completely programmed */
 	regs[0] &= ~MCP794XX_BIT_ALM0_EN;
 
-	ret = regmap_bulk_write(ds1307->regmap, MCP794XX_REG_CONTROL, regs, 10);
+	ret = regmap_bulk_write(ds1307->regmap, MCP794XX_REG_CONTROL, regs,
+				sizeof(regs));
 	if (ret)
 		return ret;
 
@@ -885,22 +936,15 @@ static int mcp794xx_alarm_irq_enable(struct device *dev, unsigned int enabled)
 				  enabled ? MCP794XX_BIT_ALM0_EN : 0);
 }
 
-static const struct rtc_class_ops mcp794xx_rtc_ops = {
-	.read_time	= ds1307_get_time,
-	.set_time	= ds1307_set_time,
-	.read_alarm	= mcp794xx_read_alarm,
-	.set_alarm	= mcp794xx_set_alarm,
-	.alarm_irq_enable = mcp794xx_alarm_irq_enable,
-};
-
 /*----------------------------------------------------------------------*/
 
 static int ds1307_nvram_read(void *priv, unsigned int offset, void *val,
 			     size_t bytes)
 {
 	struct ds1307 *ds1307 = priv;
+	const struct chip_desc *chip = &chips[ds1307->type];
 
-	return regmap_bulk_read(ds1307->regmap, ds1307->nvram_offset + offset,
+	return regmap_bulk_read(ds1307->regmap, chip->nvram_offset + offset,
 				val, bytes);
 }
 
@@ -908,15 +952,16 @@ static int ds1307_nvram_write(void *priv, unsigned int offset, void *val,
 			      size_t bytes)
 {
 	struct ds1307 *ds1307 = priv;
+	const struct chip_desc *chip = &chips[ds1307->type];
 
-	return regmap_bulk_write(ds1307->regmap, ds1307->nvram_offset + offset,
+	return regmap_bulk_write(ds1307->regmap, chip->nvram_offset + offset,
 				 val, bytes);
 }
 
 /*----------------------------------------------------------------------*/
 
 static u8 do_trickle_setup_ds1339(struct ds1307 *ds1307,
-				  uint32_t ohms, bool diode)
+				  u32 ohms, bool diode)
 {
 	u8 setup = (diode) ? DS1307_TRICKLE_CHARGER_DIODE :
 		DS1307_TRICKLE_CHARGER_NO_DIODE;
@@ -939,23 +984,23 @@ static u8 do_trickle_setup_ds1339(struct ds1307 *ds1307,
 	return setup;
 }
 
-static void ds1307_trickle_init(struct ds1307 *ds1307,
-				struct chip_desc *chip)
+static u8 ds1307_trickle_init(struct ds1307 *ds1307,
+			      const struct chip_desc *chip)
 {
-	uint32_t ohms = 0;
+	u32 ohms;
 	bool diode = true;
 
 	if (!chip->do_trickle_setup)
-		goto out;
+		return 0;
+
 	if (device_property_read_u32(ds1307->dev, "trickle-resistor-ohms",
 				     &ohms))
-		goto out;
+		return 0;
+
 	if (device_property_read_bool(ds1307->dev, "trickle-diode-disable"))
 		diode = false;
-	chip->trickle_charger_setup = chip->do_trickle_setup(ds1307,
-							     ohms, diode);
-out:
-	return;
+
+	return chip->do_trickle_setup(ds1307, ohms, diode);
 }
 
 /*----------------------------------------------------------------------*/
@@ -995,7 +1040,7 @@ static int ds3231_hwmon_read_temp(struct device *dev, s32 *mC)
 }
 
 static ssize_t ds3231_hwmon_show_temp(struct device *dev,
-				struct device_attribute *attr, char *buf)
+				      struct device_attribute *attr, char *buf)
 {
 	int ret;
 	s32 temp;
@@ -1006,8 +1051,8 @@ static ssize_t ds3231_hwmon_show_temp(struct device *dev,
 
 	return sprintf(buf, "%d\n", temp);
 }
-static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, ds3231_hwmon_show_temp,
-			NULL, 0);
+static SENSOR_DEVICE_ATTR(temp1_input, 0444, ds3231_hwmon_show_temp,
+			  NULL, 0);
 
 static struct attribute *ds3231_hwmon_attrs[] = {
 	&sensor_dev_attr_temp1_input.dev_attr.attr,
@@ -1023,7 +1068,8 @@ static void ds1307_hwmon_register(struct ds1307 *ds1307)
 		return;
 
 	dev = devm_hwmon_device_register_with_groups(ds1307->dev, ds1307->name,
-						ds1307, ds3231_hwmon_groups);
+						     ds1307,
+						     ds3231_hwmon_groups);
 	if (IS_ERR(dev)) {
 		dev_warn(ds1307->dev, "unable to register hwmon device %ld\n",
 			 PTR_ERR(dev));
@@ -1095,7 +1141,7 @@ static unsigned long ds3231_clk_sqw_recalc_rate(struct clk_hw *hw,
 }
 
 static long ds3231_clk_sqw_round_rate(struct clk_hw *hw, unsigned long rate,
-					unsigned long *prate)
+				      unsigned long *prate)
 {
 	int i;
 
@@ -1108,7 +1154,7 @@ static long ds3231_clk_sqw_round_rate(struct clk_hw *hw, unsigned long rate,
 }
 
 static int ds3231_clk_sqw_set_rate(struct clk_hw *hw, unsigned long rate,
-					unsigned long parent_rate)
+				   unsigned long parent_rate)
 {
 	struct ds1307 *ds1307 = clk_sqw_to_ds1307(hw);
 	int control = 0;
@@ -1168,7 +1214,7 @@ static const struct clk_ops ds3231_clk_sqw_ops = {
 };
 
 static unsigned long ds3231_clk_32khz_recalc_rate(struct clk_hw *hw,
-						unsigned long parent_rate)
+						  unsigned long parent_rate)
 {
 	return 32768;
 }
@@ -1259,7 +1305,7 @@ static int ds3231_clks_register(struct ds1307 *ds1307)
 
 		/* optional override of the clockname */
 		of_property_read_string_index(node, "clock-output-names", i,
-						&init.name);
+					      &init.name);
 		ds1307->clks[i].init = &init;
 
 		onecell->clks[i] = devm_clk_register(ds1307->dev,
@@ -1301,7 +1347,6 @@ static void ds1307_clks_register(struct ds1307 *ds1307)
 static const struct regmap_config regmap_config = {
 	.reg_bits = 8,
 	.val_bits = 8,
-	.max_register = 0x12,
 };
 
 static int ds1307_probe(struct i2c_client *client,
@@ -1310,22 +1355,14 @@ static int ds1307_probe(struct i2c_client *client,
 	struct ds1307		*ds1307;
 	int			err = -ENODEV;
 	int			tmp, wday;
-	struct chip_desc	*chip;
-	bool			want_irq = false;
+	const struct chip_desc	*chip;
+	bool			want_irq;
 	bool			ds1307_can_wakeup_device = false;
-	unsigned char		*buf;
+	unsigned char		regs[8];
 	struct ds1307_platform_data *pdata = dev_get_platdata(&client->dev);
 	struct rtc_time		tm;
 	unsigned long		timestamp;
-
-	irq_handler_t	irq_handler = ds1307_irq;
-
-	static const int	bbsqi_bitpos[] = {
-		[ds_1337] = 0,
-		[ds_1339] = DS1339_BIT_BBSQI,
-		[ds_3231] = DS3231_BIT_BBSQW,
-	};
-	const struct rtc_class_ops *rtc_ops = &ds13xx_rtc_ops;
+	u8			trickle_charger_setup = 0;
 
 	ds1307 = devm_kzalloc(&client->dev, sizeof(struct ds1307), GFP_KERNEL);
 	if (!ds1307)
@@ -1334,7 +1371,6 @@ static int ds1307_probe(struct i2c_client *client,
 	dev_set_drvdata(&client->dev, ds1307);
 	ds1307->dev = &client->dev;
 	ds1307->name = client->name;
-	ds1307->irq = client->irq;
 
 	ds1307->regmap = devm_regmap_init_i2c(client, &regmap_config);
 	if (IS_ERR(ds1307->regmap)) {
@@ -1362,23 +1398,22 @@ static int ds1307_probe(struct i2c_client *client,
 		ds1307->type = acpi_id->driver_data;
 	}
 
+	want_irq = client->irq > 0 && chip->alarm;
+
 	if (!pdata)
-		ds1307_trickle_init(ds1307, chip);
+		trickle_charger_setup = ds1307_trickle_init(ds1307, chip);
 	else if (pdata->trickle_charger_setup)
-		chip->trickle_charger_setup = pdata->trickle_charger_setup;
+		trickle_charger_setup = pdata->trickle_charger_setup;
 
-	if (chip->trickle_charger_setup && chip->trickle_charger_reg) {
+	if (trickle_charger_setup && chip->trickle_charger_reg) {
+		trickle_charger_setup |= DS13XX_TRICKLE_CHARGER_MAGIC;
 		dev_dbg(ds1307->dev,
 			"writing trickle charger info 0x%x to 0x%x\n",
-		    DS13XX_TRICKLE_CHARGER_MAGIC | chip->trickle_charger_setup,
-		    chip->trickle_charger_reg);
+			trickle_charger_setup, chip->trickle_charger_reg);
 		regmap_write(ds1307->regmap, chip->trickle_charger_reg,
-		    DS13XX_TRICKLE_CHARGER_MAGIC |
-		    chip->trickle_charger_setup);
+			     trickle_charger_setup);
 	}
 
-	buf = ds1307->regs;
-
 #ifdef CONFIG_OF
 /*
  * For devices with no IRQ directly connected to the SoC, the RTC chip
@@ -1388,31 +1423,27 @@ static int ds1307_probe(struct i2c_client *client,
  * This will guarantee the 'wakealarm' sysfs entry is available on the device,
  * if supported by the RTC.
  */
-	if (of_property_read_bool(client->dev.of_node, "wakeup-source")) {
-		ds1307_can_wakeup_device = true;
-	}
-	/* Intersil ISL12057 DT backward compatibility */
-	if (of_property_read_bool(client->dev.of_node,
-				  "isil,irq2-can-wakeup-machine")) {
+	if (chip->alarm && of_property_read_bool(client->dev.of_node,
+						 "wakeup-source"))
 		ds1307_can_wakeup_device = true;
-	}
 #endif
 
 	switch (ds1307->type) {
 	case ds_1337:
 	case ds_1339:
+	case ds_1341:
 	case ds_3231:
 		/* get registers that the "rtc" read below won't read... */
 		err = regmap_bulk_read(ds1307->regmap, DS1337_REG_CONTROL,
-				       buf, 2);
+				       regs, 2);
 		if (err) {
 			dev_dbg(ds1307->dev, "read error %d\n", err);
 			goto exit;
 		}
 
 		/* oscillator off?  turn it on, so clock can tick. */
-		if (ds1307->regs[0] & DS1337_BIT_nEOSC)
-			ds1307->regs[0] &= ~DS1337_BIT_nEOSC;
+		if (regs[0] & DS1337_BIT_nEOSC)
+			regs[0] &= ~DS1337_BIT_nEOSC;
 
 		/*
 		 * Using IRQ or defined as wakeup-source?
@@ -1420,114 +1451,92 @@ static int ds1307_probe(struct i2c_client *client,
 		 * For some variants, be sure alarms can trigger when we're
 		 * running on Vbackup (BBSQI/BBSQW)
 		 */
-		if (chip->alarm && (ds1307->irq > 0 ||
-				    ds1307_can_wakeup_device)) {
-			ds1307->regs[0] |= DS1337_BIT_INTCN
-					| bbsqi_bitpos[ds1307->type];
-			ds1307->regs[0] &= ~(DS1337_BIT_A2IE | DS1337_BIT_A1IE);
-
-			want_irq = true;
+		if (want_irq || ds1307_can_wakeup_device) {
+			regs[0] |= DS1337_BIT_INTCN | chip->bbsqi_bit;
+			regs[0] &= ~(DS1337_BIT_A2IE | DS1337_BIT_A1IE);
 		}
 
 		regmap_write(ds1307->regmap, DS1337_REG_CONTROL,
-			     ds1307->regs[0]);
+			     regs[0]);
 
 		/* oscillator fault?  clear flag, and warn */
-		if (ds1307->regs[1] & DS1337_BIT_OSF) {
+		if (regs[1] & DS1337_BIT_OSF) {
 			regmap_write(ds1307->regmap, DS1337_REG_STATUS,
-				     ds1307->regs[1] & ~DS1337_BIT_OSF);
+				     regs[1] & ~DS1337_BIT_OSF);
 			dev_warn(ds1307->dev, "SET TIME!\n");
 		}
 		break;
 
 	case rx_8025:
 		err = regmap_bulk_read(ds1307->regmap,
-				       RX8025_REG_CTRL1 << 4 | 0x08, buf, 2);
+				       RX8025_REG_CTRL1 << 4 | 0x08, regs, 2);
 		if (err) {
 			dev_dbg(ds1307->dev, "read error %d\n", err);
 			goto exit;
 		}
 
 		/* oscillator off?  turn it on, so clock can tick. */
-		if (!(ds1307->regs[1] & RX8025_BIT_XST)) {
-			ds1307->regs[1] |= RX8025_BIT_XST;
+		if (!(regs[1] & RX8025_BIT_XST)) {
+			regs[1] |= RX8025_BIT_XST;
 			regmap_write(ds1307->regmap,
 				     RX8025_REG_CTRL2 << 4 | 0x08,
-				     ds1307->regs[1]);
+				     regs[1]);
 			dev_warn(ds1307->dev,
 				 "oscillator stop detected - SET TIME!\n");
 		}
 
-		if (ds1307->regs[1] & RX8025_BIT_PON) {
-			ds1307->regs[1] &= ~RX8025_BIT_PON;
+		if (regs[1] & RX8025_BIT_PON) {
+			regs[1] &= ~RX8025_BIT_PON;
 			regmap_write(ds1307->regmap,
 				     RX8025_REG_CTRL2 << 4 | 0x08,
-				     ds1307->regs[1]);
+				     regs[1]);
 			dev_warn(ds1307->dev, "power-on detected\n");
 		}
 
-		if (ds1307->regs[1] & RX8025_BIT_VDET) {
-			ds1307->regs[1] &= ~RX8025_BIT_VDET;
+		if (regs[1] & RX8025_BIT_VDET) {
+			regs[1] &= ~RX8025_BIT_VDET;
 			regmap_write(ds1307->regmap,
 				     RX8025_REG_CTRL2 << 4 | 0x08,
-				     ds1307->regs[1]);
+				     regs[1]);
 			dev_warn(ds1307->dev, "voltage drop detected\n");
 		}
 
 		/* make sure we are running in 24hour mode */
-		if (!(ds1307->regs[0] & RX8025_BIT_2412)) {
+		if (!(regs[0] & RX8025_BIT_2412)) {
 			u8 hour;
 
 			/* switch to 24 hour mode */
 			regmap_write(ds1307->regmap,
 				     RX8025_REG_CTRL1 << 4 | 0x08,
-				     ds1307->regs[0] | RX8025_BIT_2412);
+				     regs[0] | RX8025_BIT_2412);
 
 			err = regmap_bulk_read(ds1307->regmap,
 					       RX8025_REG_CTRL1 << 4 | 0x08,
-					       buf, 2);
+					       regs, 2);
 			if (err) {
 				dev_dbg(ds1307->dev, "read error %d\n", err);
 				goto exit;
 			}
 
 			/* correct hour */
-			hour = bcd2bin(ds1307->regs[DS1307_REG_HOUR]);
+			hour = bcd2bin(regs[DS1307_REG_HOUR]);
 			if (hour == 12)
 				hour = 0;
-			if (ds1307->regs[DS1307_REG_HOUR] & DS1307_BIT_PM)
+			if (regs[DS1307_REG_HOUR] & DS1307_BIT_PM)
 				hour += 12;
 
 			regmap_write(ds1307->regmap,
 				     DS1307_REG_HOUR << 4 | 0x08, hour);
 		}
 		break;
-	case rx_8130:
-		ds1307->offset = 0x10; /* Seconds starts at 0x10 */
-		rtc_ops = &rx8130_rtc_ops;
-		if (chip->alarm && ds1307->irq > 0) {
-			irq_handler = rx8130_irq;
-			want_irq = true;
-		}
-		break;
-	case ds_1388:
-		ds1307->offset = 1; /* Seconds starts at 1 */
-		break;
-	case mcp794xx:
-		rtc_ops = &mcp794xx_rtc_ops;
-		if (chip->alarm && (ds1307->irq > 0 ||
-				    ds1307_can_wakeup_device)) {
-			irq_handler = mcp794xx_irq;
-			want_irq = true;
-		}
-		break;
 	default:
 		break;
 	}
 
 read_rtc:
 	/* read RTC registers */
-	err = regmap_bulk_read(ds1307->regmap, ds1307->offset, buf, 8);
+	err = regmap_bulk_read(ds1307->regmap, chip->offset, regs,
+			       sizeof(regs));
 	if (err) {
 		dev_dbg(ds1307->dev, "read error %d\n", err);
 		goto exit;
@@ -1538,7 +1547,7 @@ read_rtc:
 	 * specify the extra bits as must-be-zero, but there are
 	 * still a few values that are clearly out-of-range.
 	 */
-	tmp = ds1307->regs[DS1307_REG_SECS];
+	tmp = regs[DS1307_REG_SECS];
 	switch (ds1307->type) {
 	case ds_1307:
 	case m41t0:
@@ -1557,10 +1566,10 @@ read_rtc:
 			regmap_write(ds1307->regmap, DS1307_REG_SECS, 0);
 
 		/* oscillator fault?  clear flag, and warn */
-		if (ds1307->regs[DS1307_REG_CONTROL] & DS1338_BIT_OSF) {
+		if (regs[DS1307_REG_CONTROL] & DS1338_BIT_OSF) {
 			regmap_write(ds1307->regmap, DS1307_REG_CONTROL,
-					ds1307->regs[DS1307_REG_CONTROL] &
-					~DS1338_BIT_OSF);
+				     regs[DS1307_REG_CONTROL] &
+				     ~DS1338_BIT_OSF);
 			dev_warn(ds1307->dev, "SET TIME!\n");
 			goto read_rtc;
 		}
@@ -1584,9 +1593,9 @@ read_rtc:
 		break;
 	case mcp794xx:
 		/* make sure that the backup battery is enabled */
-		if (!(ds1307->regs[DS1307_REG_WDAY] & MCP794XX_BIT_VBATEN)) {
+		if (!(regs[DS1307_REG_WDAY] & MCP794XX_BIT_VBATEN)) {
 			regmap_write(ds1307->regmap, DS1307_REG_WDAY,
-				     ds1307->regs[DS1307_REG_WDAY] |
+				     regs[DS1307_REG_WDAY] |
 				     MCP794XX_BIT_VBATEN);
 		}
 
@@ -1603,7 +1612,7 @@ read_rtc:
 		break;
 	}
 
-	tmp = ds1307->regs[DS1307_REG_HOUR];
+	tmp = regs[DS1307_REG_HOUR];
 	switch (ds1307->type) {
 	case ds_1340:
 	case m41t0:
@@ -1626,9 +1635,9 @@ read_rtc:
 		tmp = bcd2bin(tmp & 0x1f);
 		if (tmp == 12)
 			tmp = 0;
-		if (ds1307->regs[DS1307_REG_HOUR] & DS1307_BIT_PM)
+		if (regs[DS1307_REG_HOUR] & DS1307_BIT_PM)
 			tmp += 12;
-		regmap_write(ds1307->regmap, ds1307->offset + DS1307_REG_HOUR,
+		regmap_write(ds1307->regmap, chip->offset + DS1307_REG_HOUR,
 			     bin2bcd(tmp));
 	}
 
@@ -1651,19 +1660,16 @@ read_rtc:
 				   MCP794XX_REG_WEEKDAY_WDAY_MASK,
 				   tm.tm_wday + 1);
 
-	if (want_irq) {
+	if (want_irq || ds1307_can_wakeup_device) {
 		device_set_wakeup_capable(ds1307->dev, true);
 		set_bit(HAS_ALARM, &ds1307->flags);
 	}
 
 	ds1307->rtc = devm_rtc_allocate_device(ds1307->dev);
-	if (IS_ERR(ds1307->rtc)) {
+	if (IS_ERR(ds1307->rtc))
 		return PTR_ERR(ds1307->rtc);
-	}
 
-	if (ds1307_can_wakeup_device && ds1307->irq <= 0) {
-		/* Disable request for an IRQ */
-		want_irq = false;
+	if (ds1307_can_wakeup_device && !want_irq) {
 		dev_info(ds1307->dev,
 			 "'wakeup-source' is set, request for an IRQ is disabled!\n");
 		/* We cannot support UIE mode if we do not have an IRQ line */
@@ -1671,8 +1677,8 @@ read_rtc:
 	}
 
 	if (want_irq) {
-		err = devm_request_threaded_irq(ds1307->dev,
-						ds1307->irq, NULL, irq_handler,
+		err = devm_request_threaded_irq(ds1307->dev, client->irq, NULL,
+						chip->irq_handler ?: ds1307_irq,
 						IRQF_SHARED | IRQF_ONESHOT,
 						ds1307->name, ds1307);
 		if (err) {
@@ -1680,8 +1686,9 @@ read_rtc:
 			device_set_wakeup_capable(ds1307->dev, false);
 			clear_bit(HAS_ALARM, &ds1307->flags);
 			dev_err(ds1307->dev, "unable to request IRQ!\n");
-		} else
+		} else {
 			dev_dbg(ds1307->dev, "got IRQ %d\n", client->irq);
+		}
 	}
 
 	if (chip->nvram_size) {
@@ -1692,13 +1699,12 @@ read_rtc:
 		ds1307->nvmem_cfg.reg_read = ds1307_nvram_read;
 		ds1307->nvmem_cfg.reg_write = ds1307_nvram_write;
 		ds1307->nvmem_cfg.priv = ds1307;
-		ds1307->nvram_offset = chip->nvram_offset;
 
 		ds1307->rtc->nvmem_config = &ds1307->nvmem_cfg;
 		ds1307->rtc->nvram_old_abi = true;
 	}
 
-	ds1307->rtc->ops = rtc_ops;
+	ds1307->rtc->ops = chip->rtc_ops ?: &ds13xx_rtc_ops;
 	err = rtc_register_device(ds1307->rtc);
 	if (err)
 		return err;
diff --git a/drivers/rtc/rtc-ds1672.c b/drivers/rtc/rtc-ds1672.c
index 7bf46bfe11a4..9caaccccaa57 100644
--- a/drivers/rtc/rtc-ds1672.c
+++ b/drivers/rtc/rtc-ds1672.c
@@ -190,7 +190,7 @@ static int ds1672_probe(struct i2c_client *client,
 	return 0;
 }
 
-static struct i2c_device_id ds1672_id[] = {
+static const struct i2c_device_id ds1672_id[] = {
 	{ "ds1672", 0 },
 	{ }
 };
diff --git a/drivers/rtc/rtc-em3027.c b/drivers/rtc/rtc-em3027.c
index 4f4930a2004c..b0ef8cfe742d 100644
--- a/drivers/rtc/rtc-em3027.c
+++ b/drivers/rtc/rtc-em3027.c
@@ -132,7 +132,7 @@ static int em3027_probe(struct i2c_client *client,
 	return 0;
 }
 
-static struct i2c_device_id em3027_id[] = {
+static const struct i2c_device_id em3027_id[] = {
 	{ "em3027", 0 },
 	{ }
 };
diff --git a/drivers/rtc/rtc-goldfish.c b/drivers/rtc/rtc-goldfish.c
new file mode 100644
index 000000000000..d67769265185
--- /dev/null
+++ b/drivers/rtc/rtc-goldfish.c
@@ -0,0 +1,237 @@
+/* drivers/rtc/rtc-goldfish.c
+ *
+ * Copyright (C) 2007 Google, Inc.
+ * Copyright (C) 2017 Imagination Technologies Ltd.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/rtc.h>
+#include <linux/io.h>
+
+#define TIMER_TIME_LOW		0x00	/* get low bits of current time  */
+					/*   and update TIMER_TIME_HIGH  */
+#define TIMER_TIME_HIGH	0x04	/* get high bits of time at last */
+					/*   TIMER_TIME_LOW read         */
+#define TIMER_ALARM_LOW	0x08	/* set low bits of alarm and     */
+					/*   activate it                 */
+#define TIMER_ALARM_HIGH	0x0c	/* set high bits of next alarm   */
+#define TIMER_IRQ_ENABLED	0x10
+#define TIMER_CLEAR_ALARM	0x14
+#define TIMER_ALARM_STATUS	0x18
+#define TIMER_CLEAR_INTERRUPT	0x1c
+
+struct goldfish_rtc {
+	void __iomem *base;
+	int irq;
+	struct rtc_device *rtc;
+};
+
+static int goldfish_rtc_read_alarm(struct device *dev,
+				   struct rtc_wkalrm *alrm)
+{
+	u64 rtc_alarm;
+	u64 rtc_alarm_low;
+	u64 rtc_alarm_high;
+	void __iomem *base;
+	struct goldfish_rtc *rtcdrv;
+
+	rtcdrv = dev_get_drvdata(dev);
+	base = rtcdrv->base;
+
+	rtc_alarm_low = readl(base + TIMER_ALARM_LOW);
+	rtc_alarm_high = readl(base + TIMER_ALARM_HIGH);
+	rtc_alarm = (rtc_alarm_high << 32) | rtc_alarm_low;
+
+	do_div(rtc_alarm, NSEC_PER_SEC);
+	memset(alrm, 0, sizeof(struct rtc_wkalrm));
+
+	rtc_time_to_tm(rtc_alarm, &alrm->time);
+
+	if (readl(base + TIMER_ALARM_STATUS))
+		alrm->enabled = 1;
+	else
+		alrm->enabled = 0;
+
+	return 0;
+}
+
+static int goldfish_rtc_set_alarm(struct device *dev,
+				  struct rtc_wkalrm *alrm)
+{
+	struct goldfish_rtc *rtcdrv;
+	unsigned long rtc_alarm;
+	u64 rtc_alarm64;
+	u64 rtc_status_reg;
+	void __iomem *base;
+	int ret = 0;
+
+	rtcdrv = dev_get_drvdata(dev);
+	base = rtcdrv->base;
+
+	if (alrm->enabled) {
+		ret = rtc_tm_to_time(&alrm->time, &rtc_alarm);
+		if (ret != 0)
+			return ret;
+
+		rtc_alarm64 = rtc_alarm * NSEC_PER_SEC;
+		writel((rtc_alarm64 >> 32), base + TIMER_ALARM_HIGH);
+		writel(rtc_alarm64, base + TIMER_ALARM_LOW);
+	} else {
+		/*
+		 * if this function was called with enabled=0
+		 * then it could mean that the application is
+		 * trying to cancel an ongoing alarm
+		 */
+		rtc_status_reg = readl(base + TIMER_ALARM_STATUS);
+		if (rtc_status_reg)
+			writel(1, base + TIMER_CLEAR_ALARM);
+	}
+
+	return ret;
+}
+
+static int goldfish_rtc_alarm_irq_enable(struct device *dev,
+					 unsigned int enabled)
+{
+	void __iomem *base;
+	struct goldfish_rtc *rtcdrv;
+
+	rtcdrv = dev_get_drvdata(dev);
+	base = rtcdrv->base;
+
+	if (enabled)
+		writel(1, base + TIMER_IRQ_ENABLED);
+	else
+		writel(0, base + TIMER_IRQ_ENABLED);
+
+	return 0;
+}
+
+static irqreturn_t goldfish_rtc_interrupt(int irq, void *dev_id)
+{
+	struct goldfish_rtc *rtcdrv = dev_id;
+	void __iomem *base = rtcdrv->base;
+
+	writel(1, base + TIMER_CLEAR_INTERRUPT);
+
+	rtc_update_irq(rtcdrv->rtc, 1, RTC_IRQF | RTC_AF);
+
+	return IRQ_HANDLED;
+}
+
+static int goldfish_rtc_read_time(struct device *dev, struct rtc_time *tm)
+{
+	struct goldfish_rtc *rtcdrv;
+	void __iomem *base;
+	u64 time_high;
+	u64 time_low;
+	u64 time;
+
+	rtcdrv = dev_get_drvdata(dev);
+	base = rtcdrv->base;
+
+	time_low = readl(base + TIMER_TIME_LOW);
+	time_high = readl(base + TIMER_TIME_HIGH);
+	time = (time_high << 32) | time_low;
+
+	do_div(time, NSEC_PER_SEC);
+
+	rtc_time_to_tm(time, tm);
+
+	return 0;
+}
+
+static int goldfish_rtc_set_time(struct device *dev, struct rtc_time *tm)
+{
+	struct goldfish_rtc *rtcdrv;
+	void __iomem *base;
+	unsigned long now;
+	u64 now64;
+	int ret;
+
+	rtcdrv = dev_get_drvdata(dev);
+	base = rtcdrv->base;
+
+	ret = rtc_tm_to_time(tm, &now);
+	if (ret == 0) {
+		now64 = now * NSEC_PER_SEC;
+		writel((now64 >> 32), base + TIMER_TIME_HIGH);
+		writel(now64, base + TIMER_TIME_LOW);
+	}
+
+	return ret;
+}
+
+static const struct rtc_class_ops goldfish_rtc_ops = {
+	.read_time	= goldfish_rtc_read_time,
+	.set_time	= goldfish_rtc_set_time,
+	.read_alarm	= goldfish_rtc_read_alarm,
+	.set_alarm	= goldfish_rtc_set_alarm,
+	.alarm_irq_enable = goldfish_rtc_alarm_irq_enable
+};
+
+static int goldfish_rtc_probe(struct platform_device *pdev)
+{
+	struct goldfish_rtc *rtcdrv;
+	struct resource *r;
+	int err;
+
+	rtcdrv = devm_kzalloc(&pdev->dev, sizeof(*rtcdrv), GFP_KERNEL);
+	if (!rtcdrv)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, rtcdrv);
+
+	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!r)
+		return -ENODEV;
+
+	rtcdrv->base = devm_ioremap_resource(&pdev->dev, r);
+	if (IS_ERR(rtcdrv->base))
+		return -ENODEV;
+
+	rtcdrv->irq = platform_get_irq(pdev, 0);
+	if (rtcdrv->irq < 0)
+		return -ENODEV;
+
+	rtcdrv->rtc = devm_rtc_device_register(&pdev->dev, pdev->name,
+					       &goldfish_rtc_ops,
+					       THIS_MODULE);
+	if (IS_ERR(rtcdrv->rtc))
+		return PTR_ERR(rtcdrv->rtc);
+
+	err = devm_request_irq(&pdev->dev, rtcdrv->irq,
+			       goldfish_rtc_interrupt,
+			       0, pdev->name, rtcdrv);
+	if (err)
+		return err;
+
+	return 0;
+}
+
+static const struct of_device_id goldfish_rtc_of_match[] = {
+	{ .compatible = "google,goldfish-rtc", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, goldfish_rtc_of_match);
+
+static struct platform_driver goldfish_rtc = {
+	.probe = goldfish_rtc_probe,
+	.driver = {
+		.name = "goldfish_rtc",
+		.of_match_table = goldfish_rtc_of_match,
+	}
+};
+
+module_platform_driver(goldfish_rtc);
diff --git a/drivers/rtc/rtc-m41t80.c b/drivers/rtc/rtc-m41t80.c
index 8940e9e43ea0..f4c070ea8384 100644
--- a/drivers/rtc/rtc-m41t80.c
+++ b/drivers/rtc/rtc-m41t80.c
@@ -440,28 +440,6 @@ static int m41t80_resume(struct device *dev)
 
 static SIMPLE_DEV_PM_OPS(m41t80_pm, m41t80_suspend, m41t80_resume);
 
-static ssize_t flags_show(struct device *dev,
-			  struct device_attribute *attr, char *buf)
-{
-	struct i2c_client *client = to_i2c_client(dev);
-	int val;
-
-	val = i2c_smbus_read_byte_data(client, M41T80_REG_FLAGS);
-	if (val < 0)
-		return val;
-	return sprintf(buf, "%#x\n", val);
-}
-static DEVICE_ATTR_RO(flags);
-
-static struct attribute *attrs[] = {
-	&dev_attr_flags.attr,
-	NULL,
-};
-
-static struct attribute_group attr_group = {
-	.attrs = attrs,
-};
-
 #ifdef CONFIG_COMMON_CLK
 #define sqw_to_m41t80_data(_hw) container_of(_hw, struct m41t80_data, sqw)
 
@@ -912,13 +890,6 @@ static struct notifier_block wdt_notifier = {
  *****************************************************************************
  */
 
-static void m41t80_remove_sysfs_group(void *_dev)
-{
-	struct device *dev = _dev;
-
-	sysfs_remove_group(&dev->kobj, &attr_group);
-}
-
 static int m41t80_probe(struct i2c_client *client,
 			const struct i2c_device_id *id)
 {
@@ -927,6 +898,7 @@ static int m41t80_probe(struct i2c_client *client,
 	struct rtc_device *rtc = NULL;
 	struct rtc_time tm;
 	struct m41t80_data *m41t80_data = NULL;
+	bool wakeup_source = false;
 
 	if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_I2C_BLOCK |
 				     I2C_FUNC_SMBUS_BYTE_DATA)) {
@@ -947,6 +919,10 @@ static int m41t80_probe(struct i2c_client *client,
 		m41t80_data->features = id->driver_data;
 	i2c_set_clientdata(client, m41t80_data);
 
+#ifdef CONFIG_OF
+	wakeup_source = of_property_read_bool(client->dev.of_node,
+					      "wakeup-source");
+#endif
 	if (client->irq > 0) {
 		rc = devm_request_threaded_irq(&client->dev, client->irq,
 					       NULL, m41t80_handle_irq,
@@ -955,14 +931,16 @@ static int m41t80_probe(struct i2c_client *client,
 		if (rc) {
 			dev_warn(&client->dev, "unable to request IRQ, alarms disabled\n");
 			client->irq = 0;
-		} else {
-			m41t80_rtc_ops.read_alarm = m41t80_read_alarm;
-			m41t80_rtc_ops.set_alarm = m41t80_set_alarm;
-			m41t80_rtc_ops.alarm_irq_enable = m41t80_alarm_irq_enable;
-			/* Enable the wakealarm */
-			device_init_wakeup(&client->dev, true);
+			wakeup_source = false;
 		}
 	}
+	if (client->irq > 0 || wakeup_source) {
+		m41t80_rtc_ops.read_alarm = m41t80_read_alarm;
+		m41t80_rtc_ops.set_alarm = m41t80_set_alarm;
+		m41t80_rtc_ops.alarm_irq_enable = m41t80_alarm_irq_enable;
+		/* Enable the wakealarm */
+		device_init_wakeup(&client->dev, true);
+	}
 
 	rtc = devm_rtc_device_register(&client->dev, client->name,
 				       &m41t80_rtc_ops, THIS_MODULE);
@@ -970,6 +948,10 @@ static int m41t80_probe(struct i2c_client *client,
 		return PTR_ERR(rtc);
 
 	m41t80_data->rtc = rtc;
+	if (client->irq <= 0) {
+		/* We cannot support UIE mode if we do not have an IRQ line */
+		rtc->uie_unsupported = 1;
+	}
 
 	/* Make sure HT (Halt Update) bit is cleared */
 	rc = i2c_smbus_read_byte_data(client, M41T80_REG_ALARM_HOUR);
@@ -1004,21 +986,6 @@ static int m41t80_probe(struct i2c_client *client,
 		return rc;
 	}
 
-	/* Export sysfs entries */
-	rc = sysfs_create_group(&(&client->dev)->kobj, &attr_group);
-	if (rc) {
-		dev_err(&client->dev, "Failed to create sysfs group: %d\n", rc);
-		return rc;
-	}
-
-	rc = devm_add_action_or_reset(&client->dev, m41t80_remove_sysfs_group,
-				      &client->dev);
-	if (rc) {
-		dev_err(&client->dev,
-			"Failed to add sysfs cleanup action: %d\n", rc);
-		return rc;
-	}
-
 #ifdef CONFIG_RTC_DRV_M41T80_WDT
 	if (m41t80_data->features & M41T80_FEATURE_HT) {
 		save_client = client;
diff --git a/drivers/rtc/rtc-max6900.c b/drivers/rtc/rtc-max6900.c
index 48b6b411f8b2..cbdc86a560ba 100644
--- a/drivers/rtc/rtc-max6900.c
+++ b/drivers/rtc/rtc-max6900.c
@@ -226,7 +226,7 @@ max6900_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	return 0;
 }
 
-static struct i2c_device_id max6900_id[] = {
+static const struct i2c_device_id max6900_id[] = {
 	{ "max6900", 0 },
 	{ }
 };
diff --git a/drivers/rtc/rtc-max8925.c b/drivers/rtc/rtc-max8925.c
index 16d129a0bb3b..67d6fc2d23e6 100644
--- a/drivers/rtc/rtc-max8925.c
+++ b/drivers/rtc/rtc-max8925.c
@@ -234,8 +234,6 @@ static int max8925_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 		ret = max8925_reg_write(info->rtc, MAX8925_ALARM0_CNTL, 0x77);
 	else
 		ret = max8925_reg_write(info->rtc, MAX8925_ALARM0_CNTL, 0x0);
-	if (ret < 0)
-		goto out;
 out:
 	return ret;
 }
diff --git a/drivers/rtc/rtc-mxc.c b/drivers/rtc/rtc-mxc.c
index 401f46d8f21b..bce427d202ee 100644
--- a/drivers/rtc/rtc-mxc.c
+++ b/drivers/rtc/rtc-mxc.c
@@ -238,26 +238,6 @@ static irqreturn_t mxc_rtc_interrupt(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-/*
- * Clear all interrupts and release the IRQ
- */
-static void mxc_rtc_release(struct device *dev)
-{
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_plat_data *pdata = platform_get_drvdata(pdev);
-	void __iomem *ioaddr = pdata->ioaddr;
-
-	spin_lock_irq(&pdata->rtc->irq_lock);
-
-	/* Disable all rtc interrupts */
-	writew(0, ioaddr + RTC_RTCIENR);
-
-	/* Clear all interrupt status */
-	writew(0xffffffff, ioaddr + RTC_RTCISR);
-
-	spin_unlock_irq(&pdata->rtc->irq_lock);
-}
-
 static int mxc_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled)
 {
 	mxc_rtc_irq_enable(dev, RTC_ALM_BIT, enabled);
@@ -343,7 +323,6 @@ static int mxc_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 
 /* RTC layer */
 static const struct rtc_class_ops mxc_rtc_ops = {
-	.release		= mxc_rtc_release,
 	.read_time		= mxc_rtc_read_time,
 	.set_mmss64		= mxc_rtc_set_mmss,
 	.read_alarm		= mxc_rtc_read_alarm,
diff --git a/drivers/rtc/rtc-puv3.c b/drivers/rtc/rtc-puv3.c
index c0a6e638c672..9e83be32ff43 100644
--- a/drivers/rtc/rtc-puv3.c
+++ b/drivers/rtc/rtc-puv3.c
@@ -157,49 +157,7 @@ static int puv3_rtc_proc(struct device *dev, struct seq_file *seq)
 	return 0;
 }
 
-static int puv3_rtc_open(struct device *dev)
-{
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_device *rtc_dev = platform_get_drvdata(pdev);
-	int ret;
-
-	ret = request_irq(puv3_rtc_alarmno, puv3_rtc_alarmirq,
-			0, "pkunity-rtc alarm", rtc_dev);
-
-	if (ret) {
-		dev_err(dev, "IRQ%d error %d\n", puv3_rtc_alarmno, ret);
-		return ret;
-	}
-
-	ret = request_irq(puv3_rtc_tickno, puv3_rtc_tickirq,
-			0, "pkunity-rtc tick", rtc_dev);
-
-	if (ret) {
-		dev_err(dev, "IRQ%d error %d\n", puv3_rtc_tickno, ret);
-		goto tick_err;
-	}
-
-	return ret;
-
- tick_err:
-	free_irq(puv3_rtc_alarmno, rtc_dev);
-	return ret;
-}
-
-static void puv3_rtc_release(struct device *dev)
-{
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rtc_device *rtc_dev = platform_get_drvdata(pdev);
-
-	/* do not clear AIE here, it may be needed for wake */
-	puv3_rtc_setpie(dev, 0);
-	free_irq(puv3_rtc_alarmno, rtc_dev);
-	free_irq(puv3_rtc_tickno, rtc_dev);
-}
-
 static const struct rtc_class_ops puv3_rtcops = {
-	.open		= puv3_rtc_open,
-	.release	= puv3_rtc_release,
 	.read_time	= puv3_rtc_gettime,
 	.set_time	= puv3_rtc_settime,
 	.read_alarm	= puv3_rtc_getalarm,
@@ -222,10 +180,6 @@ static void puv3_rtc_enable(struct device *dev, int en)
 
 static int puv3_rtc_remove(struct platform_device *dev)
 {
-	struct rtc_device *rtc = platform_get_drvdata(dev);
-
-	rtc_device_unregister(rtc);
-
 	puv3_rtc_setpie(&dev->dev, 0);
 	puv3_rtc_setaie(&dev->dev, 0);
 
@@ -259,6 +213,24 @@ static int puv3_rtc_probe(struct platform_device *pdev)
 	dev_dbg(&pdev->dev, "PKUnity_rtc: tick irq %d, alarm irq %d\n",
 		 puv3_rtc_tickno, puv3_rtc_alarmno);
 
+	rtc = devm_rtc_allocate_device(&pdev->dev);
+	if (IS_ERR(rtc))
+		return PTR_ERR(rtc);
+
+	ret = devm_request_irq(&pdev->dev, puv3_rtc_alarmno, puv3_rtc_alarmirq,
+			       0, "pkunity-rtc alarm", rtc);
+	if (ret) {
+		dev_err(&pdev->dev, "IRQ%d error %d\n", puv3_rtc_alarmno, ret);
+		return ret;
+	}
+
+	ret = devm_request_irq(&pdev->dev, puv3_rtc_tickno, puv3_rtc_tickirq,
+			       0, "pkunity-rtc tick", rtc);
+	if (ret) {
+		dev_err(&pdev->dev, "IRQ%d error %d\n", puv3_rtc_tickno, ret);
+		return ret;
+	}
+
 	/* get the memory region */
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	if (res == NULL) {
@@ -278,12 +250,10 @@ static int puv3_rtc_probe(struct platform_device *pdev)
 	puv3_rtc_enable(&pdev->dev, 1);
 
 	/* register RTC and exit */
-	rtc = rtc_device_register("pkunity", &pdev->dev, &puv3_rtcops,
-				  THIS_MODULE);
-
-	if (IS_ERR(rtc)) {
+	rtc->ops = &puv3_rtcops;
+	ret = rtc_register_device(rtc);
+	if (ret) {
 		dev_err(&pdev->dev, "cannot attach rtc\n");
-		ret = PTR_ERR(rtc);
 		goto err_nortc;
 	}
 
diff --git a/drivers/rtc/rtc-pxa.c b/drivers/rtc/rtc-pxa.c
index fe4985b54608..47304f5664d8 100644
--- a/drivers/rtc/rtc-pxa.c
+++ b/drivers/rtc/rtc-pxa.c
@@ -348,7 +348,7 @@ static int __init pxa_rtc_probe(struct platform_device *pdev)
 		dev_err(dev, "No alarm IRQ resource defined\n");
 		return -ENXIO;
 	}
-	pxa_rtc_open(dev);
+
 	pxa_rtc->base = devm_ioremap(dev, pxa_rtc->ress->start,
 				resource_size(pxa_rtc->ress));
 	if (!pxa_rtc->base) {
@@ -356,6 +356,8 @@ static int __init pxa_rtc_probe(struct platform_device *pdev)
 		return -ENOMEM;
 	}
 
+	pxa_rtc_open(dev);
+
 	sa1100_rtc->rcnr = pxa_rtc->base + 0x0;
 	sa1100_rtc->rtsr = pxa_rtc->base + 0x8;
 	sa1100_rtc->rtar = pxa_rtc->base + 0x4;
diff --git a/drivers/rtc/rtc-rtd119x.c b/drivers/rtc/rtc-rtd119x.c
new file mode 100644
index 000000000000..b233559d950b
--- /dev/null
+++ b/drivers/rtc/rtc-rtd119x.c
@@ -0,0 +1,242 @@
+/*
+ * Realtek RTD129x RTC
+ *
+ * Copyright (c) 2017 Andreas Färber
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ */
+
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include <linux/rtc.h>
+#include <linux/spinlock.h>
+
+#define RTD_RTCSEC		0x00
+#define RTD_RTCMIN		0x04
+#define RTD_RTCHR		0x08
+#define RTD_RTCDATE1		0x0c
+#define RTD_RTCDATE2		0x10
+#define RTD_RTCACR		0x28
+#define RTD_RTCEN		0x2c
+#define RTD_RTCCR		0x30
+
+#define RTD_RTCSEC_RTCSEC_MASK		0x7f
+
+#define RTD_RTCMIN_RTCMIN_MASK		0x3f
+
+#define RTD_RTCHR_RTCHR_MASK		0x1f
+
+#define RTD_RTCDATE1_RTCDATE1_MASK	0xff
+
+#define RTD_RTCDATE2_RTCDATE2_MASK	0x7f
+
+#define RTD_RTCACR_RTCPWR		BIT(7)
+
+#define RTD_RTCEN_RTCEN_MASK		0xff
+
+#define RTD_RTCCR_RTCRST		BIT(6)
+
+struct rtd119x_rtc {
+	void __iomem *base;
+	struct clk *clk;
+	struct rtc_device *rtcdev;
+	unsigned int base_year;
+};
+
+static inline int rtd119x_rtc_days_in_year(int year)
+{
+	return 365 + (is_leap_year(year) ? 1 : 0);
+}
+
+static void rtd119x_rtc_reset(struct device *dev)
+{
+	struct rtd119x_rtc *data = dev_get_drvdata(dev);
+	u32 val;
+
+	val = readl_relaxed(data->base + RTD_RTCCR);
+	val |= RTD_RTCCR_RTCRST;
+	writel_relaxed(val, data->base + RTD_RTCCR);
+
+	val &= ~RTD_RTCCR_RTCRST;
+	writel(val, data->base + RTD_RTCCR);
+}
+
+static void rtd119x_rtc_set_enabled(struct device *dev, bool enable)
+{
+	struct rtd119x_rtc *data = dev_get_drvdata(dev);
+	u32 val;
+
+	val = readl_relaxed(data->base + RTD_RTCEN);
+	if (enable) {
+		if ((val & RTD_RTCEN_RTCEN_MASK) == 0x5a)
+			return;
+		writel_relaxed(0x5a, data->base + RTD_RTCEN);
+	} else {
+		writel_relaxed(0, data->base + RTD_RTCEN);
+	}
+}
+
+static int rtd119x_rtc_read_time(struct device *dev, struct rtc_time *tm)
+{
+	struct rtd119x_rtc *data = dev_get_drvdata(dev);
+	s32 day;
+	u32 sec;
+	unsigned int year;
+	int tries = 0;
+
+	while (true) {
+		tm->tm_sec = (readl_relaxed(data->base + RTD_RTCSEC) & RTD_RTCSEC_RTCSEC_MASK) >> 1;
+		tm->tm_min  = readl_relaxed(data->base + RTD_RTCMIN) & RTD_RTCMIN_RTCMIN_MASK;
+		tm->tm_hour = readl_relaxed(data->base + RTD_RTCHR) & RTD_RTCHR_RTCHR_MASK;
+		day  =  readl_relaxed(data->base + RTD_RTCDATE1) & RTD_RTCDATE1_RTCDATE1_MASK;
+		day |= (readl_relaxed(data->base + RTD_RTCDATE2) & RTD_RTCDATE2_RTCDATE2_MASK) << 8;
+		sec  = (readl_relaxed(data->base + RTD_RTCSEC) & RTD_RTCSEC_RTCSEC_MASK) >> 1;
+		tries++;
+
+		if (sec == tm->tm_sec)
+			break;
+
+		if (tries >= 3)
+			return -EINVAL;
+	}
+	if (tries > 1)
+		dev_dbg(dev, "%s: needed %i tries\n", __func__, tries);
+
+	year = data->base_year;
+	while (day >= rtd119x_rtc_days_in_year(year)) {
+		day -= rtd119x_rtc_days_in_year(year);
+		year++;
+	}
+	tm->tm_year = year - 1900;
+	tm->tm_yday = day;
+
+	tm->tm_mon = 0;
+	while (day >= rtc_month_days(tm->tm_mon, year)) {
+		day -= rtc_month_days(tm->tm_mon, year);
+		tm->tm_mon++;
+	}
+	tm->tm_mday = day + 1;
+
+	return 0;
+}
+
+static int rtd119x_rtc_set_time(struct device *dev, struct rtc_time *tm)
+{
+	struct rtd119x_rtc *data = dev_get_drvdata(dev);
+	unsigned int day;
+	int i;
+
+	if (1900 + tm->tm_year < data->base_year)
+		return -EINVAL;
+
+	day = 0;
+	for (i = data->base_year; i < 1900 + tm->tm_year; i++)
+		day += rtd119x_rtc_days_in_year(i);
+
+	day += tm->tm_yday;
+	if (day > 0x7fff)
+		return -EINVAL;
+
+	rtd119x_rtc_set_enabled(dev, false);
+
+	writel_relaxed((tm->tm_sec << 1) & RTD_RTCSEC_RTCSEC_MASK, data->base + RTD_RTCSEC);
+	writel_relaxed(tm->tm_min & RTD_RTCMIN_RTCMIN_MASK, data->base + RTD_RTCMIN);
+	writel_relaxed(tm->tm_hour & RTD_RTCHR_RTCHR_MASK, data->base + RTD_RTCHR);
+	writel_relaxed(day & RTD_RTCDATE1_RTCDATE1_MASK, data->base + RTD_RTCDATE1);
+	writel_relaxed((day >> 8) & RTD_RTCDATE2_RTCDATE2_MASK, data->base + RTD_RTCDATE2);
+
+	rtd119x_rtc_set_enabled(dev, true);
+
+	return 0;
+}
+
+static const struct rtc_class_ops rtd119x_rtc_ops = {
+	.read_time	= rtd119x_rtc_read_time,
+	.set_time	= rtd119x_rtc_set_time,
+};
+
+static const struct of_device_id rtd119x_rtc_dt_ids[] = {
+	 { .compatible = "realtek,rtd1295-rtc" },
+	 { }
+};
+
+static int rtd119x_rtc_probe(struct platform_device *pdev)
+{
+	struct rtd119x_rtc *data;
+	struct resource *res;
+	u32 val;
+	int ret;
+
+	data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, data);
+	data->base_year = 2014;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	data->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(data->base))
+		return PTR_ERR(data->base);
+
+	data->clk = of_clk_get(pdev->dev.of_node, 0);
+	if (IS_ERR(data->clk))
+		return PTR_ERR(data->clk);
+
+	ret = clk_prepare_enable(data->clk);
+	if (ret) {
+		clk_put(data->clk);
+		return ret;
+	}
+
+	val = readl_relaxed(data->base + RTD_RTCACR);
+	if (!(val & RTD_RTCACR_RTCPWR)) {
+		writel_relaxed(RTD_RTCACR_RTCPWR, data->base + RTD_RTCACR);
+
+		rtd119x_rtc_reset(&pdev->dev);
+
+		writel_relaxed(0, data->base + RTD_RTCMIN);
+		writel_relaxed(0, data->base + RTD_RTCHR);
+		writel_relaxed(0, data->base + RTD_RTCDATE1);
+		writel_relaxed(0, data->base + RTD_RTCDATE2);
+	}
+
+	rtd119x_rtc_set_enabled(&pdev->dev, true);
+
+	data->rtcdev = devm_rtc_device_register(&pdev->dev, "rtc",
+						&rtd119x_rtc_ops, THIS_MODULE);
+	if (IS_ERR(data->rtcdev)) {
+		dev_err(&pdev->dev, "failed to register rtc device");
+		clk_disable_unprepare(data->clk);
+		clk_put(data->clk);
+		return PTR_ERR(data->rtcdev);
+	}
+
+	return 0;
+}
+
+static int rtd119x_rtc_remove(struct platform_device *pdev)
+{
+	struct rtd119x_rtc *data = platform_get_drvdata(pdev);
+
+	rtd119x_rtc_set_enabled(&pdev->dev, false);
+
+	clk_disable_unprepare(data->clk);
+	clk_put(data->clk);
+
+	return 0;
+}
+
+static struct platform_driver rtd119x_rtc_driver = {
+	.probe = rtd119x_rtc_probe,
+	.remove = rtd119x_rtc_remove,
+	.driver = {
+		.name = "rtd1295-rtc",
+		.of_match_table	= rtd119x_rtc_dt_ids,
+	},
+};
+builtin_platform_driver(rtd119x_rtc_driver);
diff --git a/drivers/rtc/rtc-rv3029c2.c b/drivers/rtc/rtc-rv3029c2.c
index 85fa1da03762..aa09771de04f 100644
--- a/drivers/rtc/rtc-rv3029c2.c
+++ b/drivers/rtc/rtc-rv3029c2.c
@@ -868,7 +868,7 @@ static int rv3029_i2c_probe(struct i2c_client *client,
 	return rv3029_probe(&client->dev, regmap, client->irq, client->name);
 }
 
-static struct i2c_device_id rv3029_id[] = {
+static const struct i2c_device_id rv3029_id[] = {
 	{ "rv3029", 0 },
 	{ "rv3029c2", 0 },
 	{ }
diff --git a/drivers/rtc/rtc-s35390a.c b/drivers/rtc/rtc-s35390a.c
index 449820eeefe8..7067bca5c20d 100644
--- a/drivers/rtc/rtc-s35390a.c
+++ b/drivers/rtc/rtc-s35390a.c
@@ -106,33 +106,12 @@ static int s35390a_get_reg(struct s35390a *s35390a, int reg, char *buf, int len)
 	return 0;
 }
 
-/*
- * Returns <0 on error, 0 if rtc is setup fine and 1 if the chip was reset.
- * To keep the information if an irq is pending, pass the value read from
- * STATUS1 to the caller.
- */
-static int s35390a_reset(struct s35390a *s35390a, char *status1)
+static int s35390a_init(struct s35390a *s35390a)
 {
 	char buf;
 	int ret;
 	unsigned initcount = 0;
 
-	ret = s35390a_get_reg(s35390a, S35390A_CMD_STATUS1, status1, 1);
-	if (ret < 0)
-		return ret;
-
-	if (*status1 & S35390A_FLAG_POC)
-		/*
-		 * Do not communicate for 0.5 seconds since the power-on
-		 * detection circuit is in operation.
-		 */
-		msleep(500);
-	else if (!(*status1 & S35390A_FLAG_BLD))
-		/*
-		 * If both POC and BLD are unset everything is fine.
-		 */
-		return 0;
-
 	/*
 	 * At least one of POC and BLD are set, so reinitialise chip. Keeping
 	 * this information in the hardware to know later that the time isn't
@@ -142,7 +121,6 @@ static int s35390a_reset(struct s35390a *s35390a, char *status1)
 	 * The 24H bit is kept over reset, so set it already here.
 	 */
 initialize:
-	*status1 = S35390A_FLAG_24H;
 	buf = S35390A_FLAG_RESET | S35390A_FLAG_24H;
 	ret = s35390a_set_reg(s35390a, S35390A_CMD_STATUS1, &buf, 1);
 
@@ -165,6 +143,34 @@ initialize:
 	return 1;
 }
 
+/*
+ * Returns <0 on error, 0 if rtc is setup fine and 1 if the chip was reset.
+ * To keep the information if an irq is pending, pass the value read from
+ * STATUS1 to the caller.
+ */
+static int s35390a_read_status(struct s35390a *s35390a, char *status1)
+{
+	int ret;
+
+	ret = s35390a_get_reg(s35390a, S35390A_CMD_STATUS1, status1, 1);
+	if (ret < 0)
+		return ret;
+
+	if (*status1 & S35390A_FLAG_POC) {
+		/*
+		 * Do not communicate for 0.5 seconds since the power-on
+		 * detection circuit is in operation.
+		 */
+		msleep(500);
+		return 1;
+	} else if (*status1 & S35390A_FLAG_BLD)
+		return 1;
+	/*
+	 * If both POC and BLD are unset everything is fine.
+	 */
+	return 0;
+}
+
 static int s35390a_disable_test_mode(struct s35390a *s35390a)
 {
 	char buf[1];
@@ -208,13 +214,16 @@ static int s35390a_set_datetime(struct i2c_client *client, struct rtc_time *tm)
 {
 	struct s35390a	*s35390a = i2c_get_clientdata(client);
 	int i, err;
-	char buf[7];
+	char buf[7], status;
 
 	dev_dbg(&client->dev, "%s: tm is secs=%d, mins=%d, hours=%d mday=%d, "
 		"mon=%d, year=%d, wday=%d\n", __func__, tm->tm_sec,
 		tm->tm_min, tm->tm_hour, tm->tm_mday, tm->tm_mon, tm->tm_year,
 		tm->tm_wday);
 
+	if (s35390a_read_status(s35390a, &status) == 1)
+		s35390a_init(s35390a);
+
 	buf[S35390A_BYTE_YEAR] = bin2bcd(tm->tm_year - 100);
 	buf[S35390A_BYTE_MONTH] = bin2bcd(tm->tm_mon + 1);
 	buf[S35390A_BYTE_DAY] = bin2bcd(tm->tm_mday);
@@ -235,9 +244,12 @@ static int s35390a_set_datetime(struct i2c_client *client, struct rtc_time *tm)
 static int s35390a_get_datetime(struct i2c_client *client, struct rtc_time *tm)
 {
 	struct s35390a *s35390a = i2c_get_clientdata(client);
-	char buf[7];
+	char buf[7], status;
 	int i, err;
 
+	if (s35390a_read_status(s35390a, &status) == 1)
+		return -EINVAL;
+
 	err = s35390a_get_reg(s35390a, S35390A_CMD_TIME1, buf, sizeof(buf));
 	if (err < 0)
 		return err;
@@ -392,12 +404,42 @@ static int s35390a_rtc_set_time(struct device *dev, struct rtc_time *tm)
 	return s35390a_set_datetime(to_i2c_client(dev), tm);
 }
 
+static int s35390a_rtc_ioctl(struct device *dev, unsigned int cmd,
+			     unsigned long arg)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct s35390a *s35390a = i2c_get_clientdata(client);
+	char sts;
+	int err;
+
+	switch (cmd) {
+	case RTC_VL_READ:
+		/* s35390a_reset set lowvoltage flag and init RTC if needed */
+		err = s35390a_read_status(s35390a, &sts);
+		if (err < 0)
+			return err;
+		if (copy_to_user((void __user *)arg, &err, sizeof(int)))
+			return -EFAULT;
+		break;
+	case RTC_VL_CLR:
+		/* update flag and clear register */
+		err = s35390a_init(s35390a);
+		if (err < 0)
+			return err;
+		break;
+	default:
+		return -ENOIOCTLCMD;
+	}
+
+	return 0;
+}
+
 static const struct rtc_class_ops s35390a_rtc_ops = {
 	.read_time	= s35390a_rtc_read_time,
 	.set_time	= s35390a_rtc_set_time,
 	.set_alarm	= s35390a_rtc_set_alarm,
 	.read_alarm	= s35390a_rtc_read_alarm,
-
+	.ioctl          = s35390a_rtc_ioctl,
 };
 
 static struct i2c_driver s35390a_driver;
@@ -405,7 +447,7 @@ static struct i2c_driver s35390a_driver;
 static int s35390a_probe(struct i2c_client *client,
 			 const struct i2c_device_id *id)
 {
-	int err, err_reset;
+	int err, err_read;
 	unsigned int i;
 	struct s35390a *s35390a;
 	struct rtc_time tm;
@@ -438,9 +480,9 @@ static int s35390a_probe(struct i2c_client *client,
 		}
 	}
 
-	err_reset = s35390a_reset(s35390a, &status1);
-	if (err_reset < 0) {
-		err = err_reset;
+	err_read = s35390a_read_status(s35390a, &status1);
+	if (err_read < 0) {
+		err = err_read;
 		dev_err(&client->dev, "error resetting chip\n");
 		goto exit_dummy;
 	}
@@ -466,7 +508,7 @@ static int s35390a_probe(struct i2c_client *client,
 		}
 	}
 
-	if (err_reset > 0 || s35390a_get_datetime(client, &tm) < 0)
+	if (err_read > 0 || s35390a_get_datetime(client, &tm) < 0)
 		dev_warn(&client->dev, "clock needs to be set\n");
 
 	device_set_wakeup_capable(&client->dev, 1);
diff --git a/drivers/rtc/rtc-sa1100.c b/drivers/rtc/rtc-sa1100.c
index c2187bf6c7e4..ed71d1113627 100644
--- a/drivers/rtc/rtc-sa1100.c
+++ b/drivers/rtc/rtc-sa1100.c
@@ -95,46 +95,6 @@ static irqreturn_t sa1100_rtc_interrupt(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-static int sa1100_rtc_open(struct device *dev)
-{
-	struct sa1100_rtc *info = dev_get_drvdata(dev);
-	struct rtc_device *rtc = info->rtc;
-	int ret;
-
-	ret = request_irq(info->irq_1hz, sa1100_rtc_interrupt, 0, "rtc 1Hz", dev);
-	if (ret) {
-		dev_err(dev, "IRQ %d already in use.\n", info->irq_1hz);
-		goto fail_ui;
-	}
-	ret = request_irq(info->irq_alarm, sa1100_rtc_interrupt, 0, "rtc Alrm", dev);
-	if (ret) {
-		dev_err(dev, "IRQ %d already in use.\n", info->irq_alarm);
-		goto fail_ai;
-	}
-	rtc->max_user_freq = RTC_FREQ;
-	rtc_irq_set_freq(rtc, NULL, RTC_FREQ);
-
-	return 0;
-
- fail_ai:
-	free_irq(info->irq_1hz, dev);
- fail_ui:
-	clk_disable_unprepare(info->clk);
-	return ret;
-}
-
-static void sa1100_rtc_release(struct device *dev)
-{
-	struct sa1100_rtc *info = dev_get_drvdata(dev);
-
-	spin_lock_irq(&info->lock);
-	writel_relaxed(0, info->rtsr);
-	spin_unlock_irq(&info->lock);
-
-	free_irq(info->irq_alarm, dev);
-	free_irq(info->irq_1hz, dev);
-}
-
 static int sa1100_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled)
 {
 	u32 rtsr;
@@ -216,8 +176,6 @@ static int sa1100_rtc_proc(struct device *dev, struct seq_file *seq)
 }
 
 static const struct rtc_class_ops sa1100_rtc_ops = {
-	.open = sa1100_rtc_open,
-	.release = sa1100_rtc_release,
 	.read_time = sa1100_rtc_read_time,
 	.set_time = sa1100_rtc_set_time,
 	.read_alarm = sa1100_rtc_read_alarm,
@@ -265,6 +223,9 @@ int sa1100_rtc_init(struct platform_device *pdev, struct sa1100_rtc *info)
 	}
 	info->rtc = rtc;
 
+	rtc->max_user_freq = RTC_FREQ;
+	rtc_irq_set_freq(rtc, NULL, RTC_FREQ);
+
 	/* Fix for a nasty initialization problem the in SA11xx RTSR register.
 	 * See also the comments in sa1100_rtc_interrupt().
 	 *
@@ -299,6 +260,7 @@ static int sa1100_rtc_probe(struct platform_device *pdev)
 	struct resource *iores;
 	void __iomem *base;
 	int irq_1hz, irq_alarm;
+	int ret;
 
 	irq_1hz = platform_get_irq_byname(pdev, "rtc 1Hz");
 	irq_alarm = platform_get_irq_byname(pdev, "rtc alarm");
@@ -311,6 +273,19 @@ static int sa1100_rtc_probe(struct platform_device *pdev)
 	info->irq_1hz = irq_1hz;
 	info->irq_alarm = irq_alarm;
 
+	ret = devm_request_irq(&pdev->dev, irq_1hz, sa1100_rtc_interrupt, 0,
+			       "rtc 1Hz", &pdev->dev);
+	if (ret) {
+		dev_err(&pdev->dev, "IRQ %d already in use.\n", irq_1hz);
+		return ret;
+	}
+	ret = devm_request_irq(&pdev->dev, irq_alarm, sa1100_rtc_interrupt, 0,
+			       "rtc Alrm", &pdev->dev);
+	if (ret) {
+		dev_err(&pdev->dev, "IRQ %d already in use.\n", irq_alarm);
+		return ret;
+	}
+
 	iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	base = devm_ioremap_resource(&pdev->dev, iores);
 	if (IS_ERR(base))
@@ -339,8 +314,12 @@ static int sa1100_rtc_remove(struct platform_device *pdev)
 {
 	struct sa1100_rtc *info = platform_get_drvdata(pdev);
 
-	if (info)
+	if (info) {
+		spin_lock_irq(&info->lock);
+		writel_relaxed(0, info->rtsr);
+		spin_unlock_irq(&info->lock);
 		clk_disable_unprepare(info->clk);
+	}
 
 	return 0;
 }
diff --git a/drivers/rtc/rtc-sun6i.c b/drivers/rtc/rtc-sun6i.c
index 39cbc1238b92..3d2216ccd860 100644
--- a/drivers/rtc/rtc-sun6i.c
+++ b/drivers/rtc/rtc-sun6i.c
@@ -73,6 +73,9 @@
 #define SUN6I_ALARM_CONFIG			0x0050
 #define SUN6I_ALARM_CONFIG_WAKEUP		BIT(0)
 
+#define SUN6I_LOSC_OUT_GATING			0x0060
+#define SUN6I_LOSC_OUT_GATING_EN		BIT(0)
+
 /*
  * Get date values
  */
@@ -125,6 +128,7 @@ struct sun6i_rtc_dev {
 	struct clk_hw hw;
 	struct clk_hw *int_osc;
 	struct clk *losc;
+	struct clk *ext_losc;
 
 	spinlock_t lock;
 };
@@ -188,23 +192,24 @@ static void __init sun6i_rtc_clk_init(struct device_node *node)
 	struct clk_init_data init = {
 		.ops		= &sun6i_rtc_osc_ops,
 	};
+	const char *clkout_name = "osc32k-out";
 	const char *parents[2];
 
 	rtc = kzalloc(sizeof(*rtc), GFP_KERNEL);
 	if (!rtc)
 		return;
-	spin_lock_init(&rtc->lock);
 
-	clk_data = kzalloc(sizeof(*clk_data) + sizeof(*clk_data->hws),
+	clk_data = kzalloc(sizeof(*clk_data) + (sizeof(*clk_data->hws) * 2),
 			   GFP_KERNEL);
 	if (!clk_data)
 		return;
+
 	spin_lock_init(&rtc->lock);
 
 	rtc->base = of_io_request_and_map(node, 0, of_node_full_name(node));
 	if (IS_ERR(rtc->base)) {
 		pr_crit("Can't map RTC registers");
-		return;
+		goto err;
 	}
 
 	/* Switch to the external, more precise, oscillator */
@@ -216,7 +221,7 @@ static void __init sun6i_rtc_clk_init(struct device_node *node)
 
 	/* Deal with old DTs */
 	if (!of_get_property(node, "clocks", NULL))
-		return;
+		goto err;
 
 	rtc->int_osc = clk_hw_register_fixed_rate_with_accuracy(NULL,
 								"rtc-int-osc",
@@ -235,7 +240,8 @@ static void __init sun6i_rtc_clk_init(struct device_node *node)
 
 	init.parent_names = parents;
 	init.num_parents = of_clk_get_parent_count(node) + 1;
-	of_property_read_string(node, "clock-output-names", &init.name);
+	of_property_read_string_index(node, "clock-output-names", 0,
+				      &init.name);
 
 	rtc->losc = clk_register(NULL, &rtc->hw);
 	if (IS_ERR(rtc->losc)) {
@@ -243,9 +249,25 @@ static void __init sun6i_rtc_clk_init(struct device_node *node)
 		return;
 	}
 
-	clk_data->num = 1;
+	of_property_read_string_index(node, "clock-output-names", 1,
+				      &clkout_name);
+	rtc->ext_losc = clk_register_gate(NULL, clkout_name, rtc->hw.init->name,
+					  0, rtc->base + SUN6I_LOSC_OUT_GATING,
+					  SUN6I_LOSC_OUT_GATING_EN, 0,
+					  &rtc->lock);
+	if (IS_ERR(rtc->ext_losc)) {
+		pr_crit("Couldn't register the LOSC external gate\n");
+		return;
+	}
+
+	clk_data->num = 2;
 	clk_data->hws[0] = &rtc->hw;
+	clk_data->hws[1] = __clk_get_hw(rtc->ext_losc);
 	of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data);
+	return;
+
+err:
+	kfree(clk_data);
 }
 CLK_OF_DECLARE_DRIVER(sun6i_rtc_clk, "allwinner,sun6i-a31-rtc",
 		      sun6i_rtc_clk_init);
diff --git a/drivers/rtc/rtc-twl.c b/drivers/rtc/rtc-twl.c
index c18c39212ce6..3472e79f2b17 100644
--- a/drivers/rtc/rtc-twl.c
+++ b/drivers/rtc/rtc-twl.c
@@ -31,7 +31,7 @@
 #include <linux/interrupt.h>
 #include <linux/of.h>
 
-#include <linux/i2c/twl.h>
+#include <linux/mfd/twl.h>
 
 enum twl_class {
 	TWL_4030 = 0,
diff --git a/drivers/rtc/rtc-vr41xx.c b/drivers/rtc/rtc-vr41xx.c
index e1b86bb01062..7ce22967fd16 100644
--- a/drivers/rtc/rtc-vr41xx.c
+++ b/drivers/rtc/rtc-vr41xx.c
@@ -119,23 +119,6 @@ static inline void write_elapsed_second(unsigned long sec)
 	spin_unlock_irq(&rtc_lock);
 }
 
-static void vr41xx_rtc_release(struct device *dev)
-{
-
-	spin_lock_irq(&rtc_lock);
-
-	rtc1_write(ECMPLREG, 0);
-	rtc1_write(ECMPMREG, 0);
-	rtc1_write(ECMPHREG, 0);
-	rtc1_write(RTCL1LREG, 0);
-	rtc1_write(RTCL1HREG, 0);
-
-	spin_unlock_irq(&rtc_lock);
-
-	disable_irq(aie_irq);
-	disable_irq(pie_irq);
-}
-
 static int vr41xx_rtc_read_time(struct device *dev, struct rtc_time *time)
 {
 	unsigned long epoch_sec, elapsed_sec;
@@ -272,7 +255,6 @@ static irqreturn_t rtclong1_interrupt(int irq, void *dev_id)
 }
 
 static const struct rtc_class_ops vr41xx_rtc_ops = {
-	.release		= vr41xx_rtc_release,
 	.ioctl			= vr41xx_rtc_ioctl,
 	.read_time		= vr41xx_rtc_read_time,
 	.set_time		= vr41xx_rtc_set_time,
diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c
index 670ac0a4ef49..ea19b4ff87a2 100644
--- a/drivers/s390/block/dasd.c
+++ b/drivers/s390/block/dasd.c
@@ -62,7 +62,6 @@ MODULE_LICENSE("GPL");
 static int  dasd_alloc_queue(struct dasd_block *);
 static void dasd_setup_queue(struct dasd_block *);
 static void dasd_free_queue(struct dasd_block *);
-static void dasd_flush_request_queue(struct dasd_block *);
 static int dasd_flush_block_queue(struct dasd_block *);
 static void dasd_device_tasklet(struct dasd_device *);
 static void dasd_block_tasklet(struct dasd_block *);
@@ -158,7 +157,6 @@ struct dasd_block *dasd_alloc_block(void)
 	/* open_count = 0 means device online but not in use */
 	atomic_set(&block->open_count, -1);
 
-	spin_lock_init(&block->request_queue_lock);
 	atomic_set(&block->tasklet_scheduled, 0);
 	tasklet_init(&block->tasklet,
 		     (void (*)(unsigned long)) dasd_block_tasklet,
@@ -391,7 +389,6 @@ static int dasd_state_ready_to_basic(struct dasd_device *device)
 			device->state = DASD_STATE_READY;
 			return rc;
 		}
-		dasd_flush_request_queue(block);
 		dasd_destroy_partitions(block);
 		block->blocks = 0;
 		block->bp_block = 0;
@@ -801,11 +798,12 @@ static void dasd_profile_end(struct dasd_block *block,
 			     struct dasd_ccw_req *cqr,
 			     struct request *req)
 {
-	long strtime, irqtime, endtime, tottime;	/* in microseconds */
-	long tottimeps, sectors;
+	unsigned long strtime, irqtime, endtime, tottime;
+	unsigned long tottimeps, sectors;
 	struct dasd_device *device;
 	int sectors_ind, tottime_ind, tottimeps_ind, strtime_ind;
 	int irqtime_ind, irqtimeps_ind, endtime_ind;
+	struct dasd_profile_info *data;
 
 	device = cqr->startdev;
 	if (!(dasd_global_profile_level ||
@@ -835,6 +833,11 @@ static void dasd_profile_end(struct dasd_block *block,
 
 	spin_lock(&dasd_global_profile.lock);
 	if (dasd_global_profile.data) {
+		data = dasd_global_profile.data;
+		data->dasd_sum_times += tottime;
+		data->dasd_sum_time_str += strtime;
+		data->dasd_sum_time_irq += irqtime;
+		data->dasd_sum_time_end += endtime;
 		dasd_profile_end_add_data(dasd_global_profile.data,
 					  cqr->startdev != block->base,
 					  cqr->cpmode == 1,
@@ -847,7 +850,12 @@ static void dasd_profile_end(struct dasd_block *block,
 	spin_unlock(&dasd_global_profile.lock);
 
 	spin_lock(&block->profile.lock);
-	if (block->profile.data)
+	if (block->profile.data) {
+		data = block->profile.data;
+		data->dasd_sum_times += tottime;
+		data->dasd_sum_time_str += strtime;
+		data->dasd_sum_time_irq += irqtime;
+		data->dasd_sum_time_end += endtime;
 		dasd_profile_end_add_data(block->profile.data,
 					  cqr->startdev != block->base,
 					  cqr->cpmode == 1,
@@ -856,10 +864,16 @@ static void dasd_profile_end(struct dasd_block *block,
 					  tottimeps_ind, strtime_ind,
 					  irqtime_ind, irqtimeps_ind,
 					  endtime_ind);
+	}
 	spin_unlock(&block->profile.lock);
 
 	spin_lock(&device->profile.lock);
-	if (device->profile.data)
+	if (device->profile.data) {
+		data = device->profile.data;
+		data->dasd_sum_times += tottime;
+		data->dasd_sum_time_str += strtime;
+		data->dasd_sum_time_irq += irqtime;
+		data->dasd_sum_time_end += endtime;
 		dasd_profile_end_add_data(device->profile.data,
 					  cqr->startdev != block->base,
 					  cqr->cpmode == 1,
@@ -868,6 +882,7 @@ static void dasd_profile_end(struct dasd_block *block,
 					  tottimeps_ind, strtime_ind,
 					  irqtime_ind, irqtimeps_ind,
 					  endtime_ind);
+	}
 	spin_unlock(&device->profile.lock);
 }
 
@@ -989,6 +1004,14 @@ static void dasd_stats_seq_print(struct seq_file *m,
 	seq_printf(m, "total_sectors %u\n", data->dasd_io_sects);
 	seq_printf(m, "total_pav %u\n", data->dasd_io_alias);
 	seq_printf(m, "total_hpf %u\n", data->dasd_io_tpm);
+	seq_printf(m, "avg_total %lu\n", data->dasd_io_reqs ?
+		   data->dasd_sum_times / data->dasd_io_reqs : 0UL);
+	seq_printf(m, "avg_build_to_ssch %lu\n", data->dasd_io_reqs ?
+		   data->dasd_sum_time_str / data->dasd_io_reqs : 0UL);
+	seq_printf(m, "avg_ssch_to_irq %lu\n", data->dasd_io_reqs ?
+		   data->dasd_sum_time_irq / data->dasd_io_reqs : 0UL);
+	seq_printf(m, "avg_irq_to_end %lu\n", data->dasd_io_reqs ?
+		   data->dasd_sum_time_end / data->dasd_io_reqs : 0UL);
 	seq_puts(m, "histogram_sectors ");
 	dasd_stats_array(m, data->dasd_io_secs);
 	seq_puts(m, "histogram_io_times ");
@@ -1619,8 +1642,10 @@ void dasd_generic_handle_state_change(struct dasd_device *device)
 
 	dasd_device_remove_stop_bits(device, DASD_STOPPED_PENDING);
 	dasd_schedule_device_bh(device);
-	if (device->block)
+	if (device->block) {
 		dasd_schedule_block_bh(device->block);
+		blk_mq_run_hw_queues(device->block->request_queue, true);
+	}
 }
 EXPORT_SYMBOL_GPL(dasd_generic_handle_state_change);
 
@@ -1639,7 +1664,7 @@ void dasd_int_handler(struct ccw_device *cdev, unsigned long intparm,
 {
 	struct dasd_ccw_req *cqr, *next;
 	struct dasd_device *device;
-	unsigned long long now;
+	unsigned long now;
 	int nrf_suppressed = 0;
 	int fp_suppressed = 0;
 	u8 *sense = NULL;
@@ -2612,6 +2637,7 @@ static void dasd_block_timeout(unsigned long ptr)
 	dasd_device_remove_stop_bits(block->base, DASD_STOPPED_PENDING);
 	spin_unlock_irqrestore(get_ccwdev_lock(block->base->cdev), flags);
 	dasd_schedule_block_bh(block);
+	blk_mq_run_hw_queues(block->request_queue, true);
 }
 
 /*
@@ -2651,115 +2677,11 @@ static void __dasd_process_erp(struct dasd_device *device,
 	erp_fn(cqr);
 }
 
-/*
- * Fetch requests from the block device queue.
- */
-static void __dasd_process_request_queue(struct dasd_block *block)
-{
-	struct request_queue *queue;
-	struct request *req;
-	struct dasd_ccw_req *cqr;
-	struct dasd_device *basedev;
-	unsigned long flags;
-	queue = block->request_queue;
-	basedev = block->base;
-	/* No queue ? Then there is nothing to do. */
-	if (queue == NULL)
-		return;
-
-	/*
-	 * We requeue request from the block device queue to the ccw
-	 * queue only in two states. In state DASD_STATE_READY the
-	 * partition detection is done and we need to requeue requests
-	 * for that. State DASD_STATE_ONLINE is normal block device
-	 * operation.
-	 */
-	if (basedev->state < DASD_STATE_READY) {
-		while ((req = blk_fetch_request(block->request_queue)))
-			__blk_end_request_all(req, BLK_STS_IOERR);
-		return;
-	}
-
-	/*
-	 * if device is stopped do not fetch new requests
-	 * except failfast is active which will let requests fail
-	 * immediately in __dasd_block_start_head()
-	 */
-	if (basedev->stopped && !(basedev->features & DASD_FEATURE_FAILFAST))
-		return;
-
-	/* Now we try to fetch requests from the request queue */
-	while ((req = blk_peek_request(queue))) {
-		if (basedev->features & DASD_FEATURE_READONLY &&
-		    rq_data_dir(req) == WRITE) {
-			DBF_DEV_EVENT(DBF_ERR, basedev,
-				      "Rejecting write request %p",
-				      req);
-			blk_start_request(req);
-			__blk_end_request_all(req, BLK_STS_IOERR);
-			continue;
-		}
-		if (test_bit(DASD_FLAG_ABORTALL, &basedev->flags) &&
-		    (basedev->features & DASD_FEATURE_FAILFAST ||
-		     blk_noretry_request(req))) {
-			DBF_DEV_EVENT(DBF_ERR, basedev,
-				      "Rejecting failfast request %p",
-				      req);
-			blk_start_request(req);
-			__blk_end_request_all(req, BLK_STS_TIMEOUT);
-			continue;
-		}
-		cqr = basedev->discipline->build_cp(basedev, block, req);
-		if (IS_ERR(cqr)) {
-			if (PTR_ERR(cqr) == -EBUSY)
-				break;	/* normal end condition */
-			if (PTR_ERR(cqr) == -ENOMEM)
-				break;	/* terminate request queue loop */
-			if (PTR_ERR(cqr) == -EAGAIN) {
-				/*
-				 * The current request cannot be build right
-				 * now, we have to try later. If this request
-				 * is the head-of-queue we stop the device
-				 * for 1/2 second.
-				 */
-				if (!list_empty(&block->ccw_queue))
-					break;
-				spin_lock_irqsave(
-					get_ccwdev_lock(basedev->cdev), flags);
-				dasd_device_set_stop_bits(basedev,
-							  DASD_STOPPED_PENDING);
-				spin_unlock_irqrestore(
-					get_ccwdev_lock(basedev->cdev), flags);
-				dasd_block_set_timer(block, HZ/2);
-				break;
-			}
-			DBF_DEV_EVENT(DBF_ERR, basedev,
-				      "CCW creation failed (rc=%ld) "
-				      "on request %p",
-				      PTR_ERR(cqr), req);
-			blk_start_request(req);
-			__blk_end_request_all(req, BLK_STS_IOERR);
-			continue;
-		}
-		/*
-		 *  Note: callback is set to dasd_return_cqr_cb in
-		 * __dasd_block_start_head to cover erp requests as well
-		 */
-		cqr->callback_data = (void *) req;
-		cqr->status = DASD_CQR_FILLED;
-		req->completion_data = cqr;
-		blk_start_request(req);
-		list_add_tail(&cqr->blocklist, &block->ccw_queue);
-		INIT_LIST_HEAD(&cqr->devlist);
-		dasd_profile_start(block, cqr, req);
-	}
-}
-
 static void __dasd_cleanup_cqr(struct dasd_ccw_req *cqr)
 {
 	struct request *req;
-	int status;
 	blk_status_t error = BLK_STS_OK;
+	int status;
 
 	req = (struct request *) cqr->callback_data;
 	dasd_profile_end(cqr->block, cqr, req);
@@ -2783,7 +2705,19 @@ static void __dasd_cleanup_cqr(struct dasd_ccw_req *cqr)
 			break;
 		}
 	}
-	__blk_end_request_all(req, error);
+
+	/*
+	 * We need to take care for ETIMEDOUT errors here since the
+	 * complete callback does not get called in this case.
+	 * Take care of all errors here and avoid additional code to
+	 * transfer the error value to the complete callback.
+	 */
+	if (error) {
+		blk_mq_end_request(req, error);
+		blk_mq_run_hw_queues(req->q, true);
+	} else {
+		blk_mq_complete_request(req);
+	}
 }
 
 /*
@@ -2912,27 +2846,30 @@ static void dasd_block_tasklet(struct dasd_block *block)
 	struct list_head final_queue;
 	struct list_head *l, *n;
 	struct dasd_ccw_req *cqr;
+	struct dasd_queue *dq;
 
 	atomic_set(&block->tasklet_scheduled, 0);
 	INIT_LIST_HEAD(&final_queue);
-	spin_lock(&block->queue_lock);
+	spin_lock_irq(&block->queue_lock);
 	/* Finish off requests on ccw queue */
 	__dasd_process_block_ccw_queue(block, &final_queue);
-	spin_unlock(&block->queue_lock);
+	spin_unlock_irq(&block->queue_lock);
+
 	/* Now call the callback function of requests with final status */
-	spin_lock_irq(&block->request_queue_lock);
 	list_for_each_safe(l, n, &final_queue) {
 		cqr = list_entry(l, struct dasd_ccw_req, blocklist);
+		dq = cqr->dq;
+		spin_lock_irq(&dq->lock);
 		list_del_init(&cqr->blocklist);
 		__dasd_cleanup_cqr(cqr);
+		spin_unlock_irq(&dq->lock);
 	}
-	spin_lock(&block->queue_lock);
-	/* Get new request from the block device request queue */
-	__dasd_process_request_queue(block);
+
+	spin_lock_irq(&block->queue_lock);
 	/* Now check if the head of the ccw queue needs to be started. */
 	__dasd_block_start_head(block);
-	spin_unlock(&block->queue_lock);
-	spin_unlock_irq(&block->request_queue_lock);
+	spin_unlock_irq(&block->queue_lock);
+
 	if (waitqueue_active(&shutdown_waitq))
 		wake_up(&shutdown_waitq);
 	dasd_put_device(block->base);
@@ -2951,14 +2888,13 @@ static int _dasd_requeue_request(struct dasd_ccw_req *cqr)
 {
 	struct dasd_block *block = cqr->block;
 	struct request *req;
-	unsigned long flags;
 
 	if (!block)
 		return -EINVAL;
-	spin_lock_irqsave(&block->request_queue_lock, flags);
+	spin_lock_irq(&cqr->dq->lock);
 	req = (struct request *) cqr->callback_data;
-	blk_requeue_request(block->request_queue, req);
-	spin_unlock_irqrestore(&block->request_queue_lock, flags);
+	blk_mq_requeue_request(req, false);
+	spin_unlock_irq(&cqr->dq->lock);
 
 	return 0;
 }
@@ -2973,6 +2909,7 @@ static int dasd_flush_block_queue(struct dasd_block *block)
 	struct dasd_ccw_req *cqr, *n;
 	int rc, i;
 	struct list_head flush_queue;
+	unsigned long flags;
 
 	INIT_LIST_HEAD(&flush_queue);
 	spin_lock_bh(&block->queue_lock);
@@ -3011,11 +2948,11 @@ restart_cb:
 			goto restart_cb;
 		}
 		/* call the callback function */
-		spin_lock_irq(&block->request_queue_lock);
+		spin_lock_irqsave(&cqr->dq->lock, flags);
 		cqr->endclk = get_tod_clock();
 		list_del_init(&cqr->blocklist);
 		__dasd_cleanup_cqr(cqr);
-		spin_unlock_irq(&block->request_queue_lock);
+		spin_unlock_irqrestore(&cqr->dq->lock, flags);
 	}
 	return rc;
 }
@@ -3043,42 +2980,114 @@ EXPORT_SYMBOL(dasd_schedule_block_bh);
 /*
  * Dasd request queue function. Called from ll_rw_blk.c
  */
-static void do_dasd_request(struct request_queue *queue)
+static blk_status_t do_dasd_request(struct blk_mq_hw_ctx *hctx,
+				    const struct blk_mq_queue_data *qd)
 {
-	struct dasd_block *block;
+	struct dasd_block *block = hctx->queue->queuedata;
+	struct dasd_queue *dq = hctx->driver_data;
+	struct request *req = qd->rq;
+	struct dasd_device *basedev;
+	struct dasd_ccw_req *cqr;
+	blk_status_t rc = BLK_STS_OK;
 
-	block = queue->queuedata;
+	basedev = block->base;
+	spin_lock_irq(&dq->lock);
+	if (basedev->state < DASD_STATE_READY) {
+		DBF_DEV_EVENT(DBF_ERR, basedev,
+			      "device not ready for request %p", req);
+		rc = BLK_STS_IOERR;
+		goto out;
+	}
+
+	/*
+	 * if device is stopped do not fetch new requests
+	 * except failfast is active which will let requests fail
+	 * immediately in __dasd_block_start_head()
+	 */
+	if (basedev->stopped && !(basedev->features & DASD_FEATURE_FAILFAST)) {
+		DBF_DEV_EVENT(DBF_ERR, basedev,
+			      "device stopped request %p", req);
+		rc = BLK_STS_RESOURCE;
+		goto out;
+	}
+
+	if (basedev->features & DASD_FEATURE_READONLY &&
+	    rq_data_dir(req) == WRITE) {
+		DBF_DEV_EVENT(DBF_ERR, basedev,
+			      "Rejecting write request %p", req);
+		rc = BLK_STS_IOERR;
+		goto out;
+	}
+
+	if (test_bit(DASD_FLAG_ABORTALL, &basedev->flags) &&
+	    (basedev->features & DASD_FEATURE_FAILFAST ||
+	     blk_noretry_request(req))) {
+		DBF_DEV_EVENT(DBF_ERR, basedev,
+			      "Rejecting failfast request %p", req);
+		rc = BLK_STS_IOERR;
+		goto out;
+	}
+
+	cqr = basedev->discipline->build_cp(basedev, block, req);
+	if (IS_ERR(cqr)) {
+		if (PTR_ERR(cqr) == -EBUSY ||
+		    PTR_ERR(cqr) == -ENOMEM ||
+		    PTR_ERR(cqr) == -EAGAIN) {
+			rc = BLK_STS_RESOURCE;
+			goto out;
+		}
+		DBF_DEV_EVENT(DBF_ERR, basedev,
+			      "CCW creation failed (rc=%ld) on request %p",
+			      PTR_ERR(cqr), req);
+		rc = BLK_STS_IOERR;
+		goto out;
+	}
+	/*
+	 *  Note: callback is set to dasd_return_cqr_cb in
+	 * __dasd_block_start_head to cover erp requests as well
+	 */
+	cqr->callback_data = req;
+	cqr->status = DASD_CQR_FILLED;
+	cqr->dq = dq;
+	req->completion_data = cqr;
+	blk_mq_start_request(req);
 	spin_lock(&block->queue_lock);
-	/* Get new request from the block device request queue */
-	__dasd_process_request_queue(block);
-	/* Now check if the head of the ccw queue needs to be started. */
-	__dasd_block_start_head(block);
+	list_add_tail(&cqr->blocklist, &block->ccw_queue);
+	INIT_LIST_HEAD(&cqr->devlist);
+	dasd_profile_start(block, cqr, req);
+	dasd_schedule_block_bh(block);
 	spin_unlock(&block->queue_lock);
+
+out:
+	spin_unlock_irq(&dq->lock);
+	return rc;
 }
 
 /*
  * Block timeout callback, called from the block layer
  *
- * request_queue lock is held on entry.
- *
  * Return values:
  * BLK_EH_RESET_TIMER if the request should be left running
  * BLK_EH_NOT_HANDLED if the request is handled or terminated
  *		      by the driver.
  */
-enum blk_eh_timer_return dasd_times_out(struct request *req)
+enum blk_eh_timer_return dasd_times_out(struct request *req, bool reserved)
 {
 	struct dasd_ccw_req *cqr = req->completion_data;
 	struct dasd_block *block = req->q->queuedata;
 	struct dasd_device *device;
+	unsigned long flags;
 	int rc = 0;
 
 	if (!cqr)
 		return BLK_EH_NOT_HANDLED;
 
+	spin_lock_irqsave(&cqr->dq->lock, flags);
 	device = cqr->startdev ? cqr->startdev : block->base;
-	if (!device->blk_timeout)
+	if (!device->blk_timeout) {
+		spin_unlock_irqrestore(&cqr->dq->lock, flags);
 		return BLK_EH_RESET_TIMER;
+	}
 	DBF_DEV_EVENT(DBF_WARNING, device,
 		      " dasd_times_out cqr %p status %x",
 		      cqr, cqr->status);
@@ -3128,19 +3137,64 @@ enum blk_eh_timer_return dasd_times_out(struct request *req)
 	}
 	dasd_schedule_block_bh(block);
 	spin_unlock(&block->queue_lock);
+	spin_unlock_irqrestore(&cqr->dq->lock, flags);
 
 	return rc ? BLK_EH_RESET_TIMER : BLK_EH_NOT_HANDLED;
 }
 
+static int dasd_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
+			  unsigned int idx)
+{
+	struct dasd_queue *dq = kzalloc(sizeof(*dq), GFP_KERNEL);
+
+	if (!dq)
+		return -ENOMEM;
+
+	spin_lock_init(&dq->lock);
+	hctx->driver_data = dq;
+
+	return 0;
+}
+
+static void dasd_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int idx)
+{
+	kfree(hctx->driver_data);
+	hctx->driver_data = NULL;
+}
+
+static void dasd_request_done(struct request *req)
+{
+	blk_mq_end_request(req, 0);
+	blk_mq_run_hw_queues(req->q, true);
+}
+
+static struct blk_mq_ops dasd_mq_ops = {
+	.queue_rq = do_dasd_request,
+	.complete = dasd_request_done,
+	.timeout = dasd_times_out,
+	.init_hctx = dasd_init_hctx,
+	.exit_hctx = dasd_exit_hctx,
+};
+
 /*
  * Allocate and initialize request queue and default I/O scheduler.
  */
 static int dasd_alloc_queue(struct dasd_block *block)
 {
-	block->request_queue = blk_init_queue(do_dasd_request,
-					       &block->request_queue_lock);
-	if (block->request_queue == NULL)
-		return -ENOMEM;
+	int rc;
+
+	block->tag_set.ops = &dasd_mq_ops;
+	block->tag_set.nr_hw_queues = DASD_NR_HW_QUEUES;
+	block->tag_set.queue_depth = DASD_MAX_LCU_DEV * DASD_REQ_PER_DEV;
+	block->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
+
+	rc = blk_mq_alloc_tag_set(&block->tag_set);
+	if (rc)
+		return rc;
+
+	block->request_queue = blk_mq_init_queue(&block->tag_set);
+	if (IS_ERR(block->request_queue))
+		return PTR_ERR(block->request_queue);
 
 	block->request_queue->queuedata = block;
 
@@ -3152,7 +3206,9 @@ static int dasd_alloc_queue(struct dasd_block *block)
  */
 static void dasd_setup_queue(struct dasd_block *block)
 {
+	unsigned int logical_block_size = block->bp_block;
 	struct request_queue *q = block->request_queue;
+	unsigned int max_bytes, max_discard_sectors;
 	int max;
 
 	if (block->base->features & DASD_FEATURE_USERAW) {
@@ -3169,7 +3225,7 @@ static void dasd_setup_queue(struct dasd_block *block)
 	}
 	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q);
 	q->limits.max_dev_sectors = max;
-	blk_queue_logical_block_size(q, block->bp_block);
+	blk_queue_logical_block_size(q, logical_block_size);
 	blk_queue_max_hw_sectors(q, max);
 	blk_queue_max_segments(q, USHRT_MAX);
 	/* with page sized segments we can translate each segement into
@@ -3177,6 +3233,21 @@ static void dasd_setup_queue(struct dasd_block *block)
 	 */
 	blk_queue_max_segment_size(q, PAGE_SIZE);
 	blk_queue_segment_boundary(q, PAGE_SIZE - 1);
+
+	/* Only activate blocklayer discard support for devices that support it */
+	if (block->base->features & DASD_FEATURE_DISCARD) {
+		q->limits.discard_granularity = logical_block_size;
+		q->limits.discard_alignment = PAGE_SIZE;
+
+		/* Calculate max_discard_sectors and make it PAGE aligned */
+		max_bytes = USHRT_MAX * logical_block_size;
+		max_bytes = ALIGN(max_bytes, PAGE_SIZE) - PAGE_SIZE;
+		max_discard_sectors = max_bytes / logical_block_size;
+
+		blk_queue_max_discard_sectors(q, max_discard_sectors);
+		blk_queue_max_write_zeroes_sectors(q, max_discard_sectors);
+		queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q);
+	}
 }
 
 /*
@@ -3186,26 +3257,11 @@ static void dasd_free_queue(struct dasd_block *block)
 {
 	if (block->request_queue) {
 		blk_cleanup_queue(block->request_queue);
+		blk_mq_free_tag_set(&block->tag_set);
 		block->request_queue = NULL;
 	}
 }
 
-/*
- * Flush request on the request queue.
- */
-static void dasd_flush_request_queue(struct dasd_block *block)
-{
-	struct request *req;
-
-	if (!block->request_queue)
-		return;
-
-	spin_lock_irq(&block->request_queue_lock);
-	while ((req = blk_fetch_request(block->request_queue)))
-		__blk_end_request_all(req, BLK_STS_IOERR);
-	spin_unlock_irq(&block->request_queue_lock);
-}
-
 static int dasd_open(struct block_device *bdev, fmode_t mode)
 {
 	struct dasd_device *base;
@@ -3701,8 +3757,10 @@ int dasd_generic_path_operational(struct dasd_device *device)
 		return 1;
 	}
 	dasd_schedule_device_bh(device);
-	if (device->block)
+	if (device->block) {
 		dasd_schedule_block_bh(device->block);
+		blk_mq_run_hw_queues(device->block->request_queue, true);
+		}
 
 	if (!device->stopped)
 		wake_up(&generic_waitq);
@@ -3965,8 +4023,10 @@ int dasd_generic_restore_device(struct ccw_device *cdev)
 		 */
 		device->stopped |= DASD_UNRESUMED_PM;
 
-	if (device->block)
+	if (device->block) {
 		dasd_schedule_block_bh(device->block);
+		blk_mq_run_hw_queues(device->block->request_queue, true);
+	}
 
 	clear_bit(DASD_FLAG_SUSPENDED, &device->flags);
 	dasd_put_device(device);
diff --git a/drivers/s390/block/dasd_3990_erp.c b/drivers/s390/block/dasd_3990_erp.c
index 107cd3361e29..e448a0fc0c09 100644
--- a/drivers/s390/block/dasd_3990_erp.c
+++ b/drivers/s390/block/dasd_3990_erp.c
@@ -2231,7 +2231,7 @@ static void dasd_3990_erp_account_error(struct dasd_ccw_req *erp)
 	struct dasd_device *device = erp->startdev;
 	__u8 lpum = erp->refers->irb.esw.esw1.lpum;
 	int pos = pathmask_to_pos(lpum);
-	unsigned long long clk;
+	unsigned long clk;
 
 	if (!device->path_thrhld)
 		return;
diff --git a/drivers/s390/block/dasd_devmap.c b/drivers/s390/block/dasd_devmap.c
index 779dce069cc5..c95a4784c191 100644
--- a/drivers/s390/block/dasd_devmap.c
+++ b/drivers/s390/block/dasd_devmap.c
@@ -1326,7 +1326,7 @@ dasd_timeout_store(struct device *dev, struct device_attribute *attr,
 {
 	struct dasd_device *device;
 	struct request_queue *q;
-	unsigned long val, flags;
+	unsigned long val;
 
 	device = dasd_device_from_cdev(to_ccwdev(dev));
 	if (IS_ERR(device) || !device->block)
@@ -1342,16 +1342,10 @@ dasd_timeout_store(struct device *dev, struct device_attribute *attr,
 		dasd_put_device(device);
 		return -ENODEV;
 	}
-	spin_lock_irqsave(&device->block->request_queue_lock, flags);
-	if (!val)
-		blk_queue_rq_timed_out(q, NULL);
-	else
-		blk_queue_rq_timed_out(q, dasd_times_out);
 
 	device->blk_timeout = val;
 
 	blk_queue_rq_timeout(q, device->blk_timeout * HZ);
-	spin_unlock_irqrestore(&device->block->request_queue_lock, flags);
 
 	dasd_put_device(device);
 	return count;
@@ -1634,7 +1628,7 @@ static struct attribute * dasd_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group dasd_attr_group = {
+static const struct attribute_group dasd_attr_group = {
 	.attrs = dasd_attrs,
 };
 
@@ -1676,6 +1670,7 @@ dasd_set_feature(struct ccw_device *cdev, int feature, int flag)
 	spin_unlock(&dasd_devmap_lock);
 	return 0;
 }
+EXPORT_SYMBOL(dasd_set_feature);
 
 
 int
diff --git a/drivers/s390/block/dasd_diag.c b/drivers/s390/block/dasd_diag.c
index 5667146c6a0a..98fb28e49d2c 100644
--- a/drivers/s390/block/dasd_diag.c
+++ b/drivers/s390/block/dasd_diag.c
@@ -235,7 +235,7 @@ static void dasd_ext_handler(struct ext_code ext_code,
 {
 	struct dasd_ccw_req *cqr, *next;
 	struct dasd_device *device;
-	unsigned long long expires;
+	unsigned long expires;
 	unsigned long flags;
 	addr_t ip;
 	int rc;
diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c
index c3e5ad641b0b..8eafcd5fa004 100644
--- a/drivers/s390/block/dasd_eckd.c
+++ b/drivers/s390/block/dasd_eckd.c
@@ -3254,11 +3254,7 @@ static struct dasd_ccw_req *dasd_eckd_build_cp_cmd_track(
 	/* 1x prefix + one read/write ccw per track */
 	cplength = 1 + trkcount;
 
-	/* on 31-bit we need space for two 32 bit addresses per page
-	 * on 64-bit one 64 bit address
-	 */
-	datasize = sizeof(struct PFX_eckd_data) +
-		cidaw * sizeof(unsigned long long);
+	datasize = sizeof(struct PFX_eckd_data) + cidaw * sizeof(unsigned long);
 
 	/* Allocate the ccw request. */
 	cqr = dasd_smalloc_request(DASD_ECKD_MAGIC, cplength, datasize,
@@ -3856,7 +3852,7 @@ static struct dasd_ccw_req *dasd_eckd_build_cp_raw(struct dasd_device *startdev,
 	}
 	size = ALIGN(size, 8);
 
-	datasize = size + cidaw * sizeof(unsigned long long);
+	datasize = size + cidaw * sizeof(unsigned long);
 
 	/* Allocate the ccw request. */
 	cqr = dasd_smalloc_request(DASD_ECKD_MAGIC, cplength,
diff --git a/drivers/s390/block/dasd_eckd.h b/drivers/s390/block/dasd_eckd.h
index fb1f537d986a..34e153a6b19c 100644
--- a/drivers/s390/block/dasd_eckd.h
+++ b/drivers/s390/block/dasd_eckd.h
@@ -165,7 +165,7 @@ struct DE_eckd_data {
 	__u8 ga_extended;	/* Global Attributes Extended	*/
 	struct ch_t beg_ext;
 	struct ch_t end_ext;
-	unsigned long long ep_sys_time; /* Ext Parameter - System Time Stamp */
+	unsigned long ep_sys_time; /* Ext Parameter - System Time Stamp */
 	__u8 ep_format;        /* Extended Parameter format byte       */
 	__u8 ep_prio;          /* Extended Parameter priority I/O byte */
 	__u8 ep_reserved1;     /* Extended Parameter Reserved	       */
diff --git a/drivers/s390/block/dasd_erp.c b/drivers/s390/block/dasd_erp.c
index 9e3419124264..6389feb2fb7a 100644
--- a/drivers/s390/block/dasd_erp.c
+++ b/drivers/s390/block/dasd_erp.c
@@ -124,7 +124,7 @@ dasd_default_erp_action(struct dasd_ccw_req *cqr)
 struct dasd_ccw_req *dasd_default_erp_postaction(struct dasd_ccw_req *cqr)
 {
 	int success;
-	unsigned long long startclk, stopclk;
+	unsigned long startclk, stopclk;
 	struct dasd_device *startdev;
 
 	BUG_ON(cqr->refers == NULL || cqr->function == NULL);
diff --git a/drivers/s390/block/dasd_fba.c b/drivers/s390/block/dasd_fba.c
index 462cab5d4302..6168ccdb389c 100644
--- a/drivers/s390/block/dasd_fba.c
+++ b/drivers/s390/block/dasd_fba.c
@@ -174,6 +174,9 @@ dasd_fba_check_characteristics(struct dasd_device *device)
 	if (readonly)
 		set_bit(DASD_FLAG_DEVICE_RO, &device->flags);
 
+	/* FBA supports discard, set the according feature bit */
+	dasd_set_feature(cdev, DASD_FEATURE_DISCARD, 1);
+
 	dev_info(&device->cdev->dev,
 		 "New FBA DASD %04X/%02X (CU %04X/%02X) with %d MB "
 		 "and %d B/blk%s\n",
@@ -247,9 +250,192 @@ static void dasd_fba_check_for_device_change(struct dasd_device *device,
 		dasd_generic_handle_state_change(device);
 };
 
-static struct dasd_ccw_req *dasd_fba_build_cp(struct dasd_device * memdev,
-					      struct dasd_block *block,
-					      struct request *req)
+
+/*
+ * Builds a CCW with no data payload
+ */
+static void ccw_write_no_data(struct ccw1 *ccw)
+{
+	ccw->cmd_code = DASD_FBA_CCW_WRITE;
+	ccw->flags |= CCW_FLAG_SLI;
+	ccw->count = 0;
+}
+
+/*
+ * Builds a CCW that writes only zeroes.
+ */
+static void ccw_write_zero(struct ccw1 *ccw, int count)
+{
+	ccw->cmd_code = DASD_FBA_CCW_WRITE;
+	ccw->flags |= CCW_FLAG_SLI;
+	ccw->count = count;
+	ccw->cda = (__u32) (addr_t) page_to_phys(ZERO_PAGE(0));
+}
+
+/*
+ * Helper function to count the amount of necessary CCWs within a given range
+ * with 4k alignment and command chaining in mind.
+ */
+static int count_ccws(sector_t first_rec, sector_t last_rec,
+		      unsigned int blocks_per_page)
+{
+	sector_t wz_stop = 0, d_stop = 0;
+	int cur_pos = 0;
+	int count = 0;
+
+	if (first_rec % blocks_per_page != 0) {
+		wz_stop = first_rec + blocks_per_page -
+			(first_rec % blocks_per_page) - 1;
+		if (wz_stop > last_rec)
+			wz_stop = last_rec;
+		cur_pos = wz_stop - first_rec + 1;
+		count++;
+	}
+
+	if (last_rec - (first_rec + cur_pos) + 1 >= blocks_per_page) {
+		if ((last_rec - blocks_per_page + 1) % blocks_per_page != 0)
+			d_stop = last_rec - ((last_rec - blocks_per_page + 1) %
+					     blocks_per_page);
+		else
+			d_stop = last_rec;
+
+		cur_pos += d_stop - (first_rec + cur_pos) + 1;
+		count++;
+	}
+
+	if (cur_pos == 0 || first_rec + cur_pos - 1 < last_rec)
+		count++;
+
+	return count;
+}
+
+/*
+ * This function builds a CCW request for block layer discard requests.
+ * Each page in the z/VM hypervisor that represents certain records of an FBA
+ * device will be padded with zeros. This is a special behaviour of the WRITE
+ * command which is triggered when no data payload is added to the CCW.
+ *
+ * Note: Due to issues in some z/VM versions, we can't fully utilise this
+ * special behaviour. We have to keep a 4k (or 8 block) alignment in mind to
+ * work around those issues and write actual zeroes to the unaligned parts in
+ * the request. This workaround might be removed in the future.
+ */
+static struct dasd_ccw_req *dasd_fba_build_cp_discard(
+						struct dasd_device *memdev,
+						struct dasd_block *block,
+						struct request *req)
+{
+	struct LO_fba_data *LO_data;
+	struct dasd_ccw_req *cqr;
+	struct ccw1 *ccw;
+
+	sector_t wz_stop = 0, d_stop = 0;
+	sector_t first_rec, last_rec;
+
+	unsigned int blksize = block->bp_block;
+	unsigned int blocks_per_page;
+	int wz_count = 0;
+	int d_count = 0;
+	int cur_pos = 0; /* Current position within the extent */
+	int count = 0;
+	int cplength;
+	int datasize;
+	int nr_ccws;
+
+	first_rec = blk_rq_pos(req) >> block->s2b_shift;
+	last_rec =
+		(blk_rq_pos(req) + blk_rq_sectors(req) - 1) >> block->s2b_shift;
+	count = last_rec - first_rec + 1;
+
+	blocks_per_page = BLOCKS_PER_PAGE(blksize);
+	nr_ccws = count_ccws(first_rec, last_rec, blocks_per_page);
+
+	/* define extent + nr_ccws * locate record + nr_ccws * single CCW */
+	cplength = 1 + 2 * nr_ccws;
+	datasize = sizeof(struct DE_fba_data) +
+		nr_ccws * (sizeof(struct LO_fba_data) + sizeof(struct ccw1));
+
+	cqr = dasd_smalloc_request(DASD_FBA_MAGIC, cplength, datasize, memdev);
+	if (IS_ERR(cqr))
+		return cqr;
+
+	ccw = cqr->cpaddr;
+
+	define_extent(ccw++, cqr->data, WRITE, blksize, first_rec, count);
+	LO_data = cqr->data + sizeof(struct DE_fba_data);
+
+	/* First part is not aligned. Calculate range to write zeroes. */
+	if (first_rec % blocks_per_page != 0) {
+		wz_stop = first_rec + blocks_per_page -
+			(first_rec % blocks_per_page) - 1;
+		if (wz_stop > last_rec)
+			wz_stop = last_rec;
+		wz_count = wz_stop - first_rec + 1;
+
+		ccw[-1].flags |= CCW_FLAG_CC;
+		locate_record(ccw++, LO_data++, WRITE, cur_pos, wz_count);
+
+		ccw[-1].flags |= CCW_FLAG_CC;
+		ccw_write_zero(ccw++, wz_count * blksize);
+
+		cur_pos = wz_count;
+	}
+
+	/* We can do proper discard when we've got at least blocks_per_page blocks. */
+	if (last_rec - (first_rec + cur_pos) + 1 >= blocks_per_page) {
+		/* is last record at page boundary? */
+		if ((last_rec - blocks_per_page + 1) % blocks_per_page != 0)
+			d_stop = last_rec - ((last_rec - blocks_per_page + 1) %
+					     blocks_per_page);
+		else
+			d_stop = last_rec;
+
+		d_count = d_stop - (first_rec + cur_pos) + 1;
+
+		ccw[-1].flags |= CCW_FLAG_CC;
+		locate_record(ccw++, LO_data++, WRITE, cur_pos, d_count);
+
+		ccw[-1].flags |= CCW_FLAG_CC;
+		ccw_write_no_data(ccw++);
+
+		cur_pos += d_count;
+	}
+
+	/* We might still have some bits left which need to be zeroed. */
+	if (cur_pos == 0 || first_rec + cur_pos - 1 < last_rec) {
+		if (d_stop != 0)
+			wz_count = last_rec - d_stop;
+		else if (wz_stop != 0)
+			wz_count = last_rec - wz_stop;
+		else
+			wz_count = count;
+
+		ccw[-1].flags |= CCW_FLAG_CC;
+		locate_record(ccw++, LO_data++, WRITE, cur_pos, wz_count);
+
+		ccw[-1].flags |= CCW_FLAG_CC;
+		ccw_write_zero(ccw++, wz_count * blksize);
+	}
+
+	if (blk_noretry_request(req) ||
+	    block->base->features & DASD_FEATURE_FAILFAST)
+		set_bit(DASD_CQR_FLAGS_FAILFAST, &cqr->flags);
+
+	cqr->startdev = memdev;
+	cqr->memdev = memdev;
+	cqr->block = block;
+	cqr->expires = memdev->default_expires * HZ;	/* default 5 minutes */
+	cqr->retries = memdev->default_retries;
+	cqr->buildclk = get_tod_clock();
+	cqr->status = DASD_CQR_FILLED;
+
+	return cqr;
+}
+
+static struct dasd_ccw_req *dasd_fba_build_cp_regular(
+						struct dasd_device *memdev,
+						struct dasd_block *block,
+						struct request *req)
 {
 	struct dasd_fba_private *private = block->base->private;
 	unsigned long *idaws;
@@ -372,6 +558,16 @@ static struct dasd_ccw_req *dasd_fba_build_cp(struct dasd_device * memdev,
 	return cqr;
 }
 
+static struct dasd_ccw_req *dasd_fba_build_cp(struct dasd_device *memdev,
+					      struct dasd_block *block,
+					      struct request *req)
+{
+	if (req_op(req) == REQ_OP_DISCARD || req_op(req) == REQ_OP_WRITE_ZEROES)
+		return dasd_fba_build_cp_discard(memdev, block, req);
+	else
+		return dasd_fba_build_cp_regular(memdev, block, req);
+}
+
 static int
 dasd_fba_free_cp(struct dasd_ccw_req *cqr, struct request *req)
 {
diff --git a/drivers/s390/block/dasd_int.h b/drivers/s390/block/dasd_int.h
index dca7cb1e6f65..db470bd10175 100644
--- a/drivers/s390/block/dasd_int.h
+++ b/drivers/s390/block/dasd_int.h
@@ -56,6 +56,7 @@
 #include <asm/dasd.h>
 #include <asm/idals.h>
 #include <linux/bitops.h>
+#include <linux/blk-mq.h>
 
 /* DASD discipline magic */
 #define DASD_ECKD_MAGIC 0xC5C3D2C4
@@ -167,6 +168,9 @@ do { \
 	printk(d_loglevel PRINTK_HEADER " " d_string "\n", d_args); \
 } while(0)
 
+/* Macro to calculate number of blocks per page */
+#define BLOCKS_PER_PAGE(blksize) (PAGE_SIZE / blksize)
+
 struct dasd_ccw_req {
 	unsigned int magic;		/* Eye catcher */
 	struct list_head devlist;	/* for dasd_device request queue */
@@ -182,6 +186,7 @@ struct dasd_ccw_req {
 	char status;			/* status of this request */
 	short retries;			/* A retry counter */
 	unsigned long flags;        	/* flags of this request */
+	struct dasd_queue *dq;
 
 	/* ... and how */
 	unsigned long starttime;	/* jiffies time of request start */
@@ -196,10 +201,10 @@ struct dasd_ccw_req {
 	void *function; 		/* originating ERP action */
 
 	/* these are for statistics only */
-	unsigned long long buildclk;	/* TOD-clock of request generation */
-	unsigned long long startclk;	/* TOD-clock of request start */
-	unsigned long long stopclk;	/* TOD-clock of request interrupt */
-	unsigned long long endclk;	/* TOD-clock of request termination */
+	unsigned long buildclk;		/* TOD-clock of request generation */
+	unsigned long startclk;		/* TOD-clock of request start */
+	unsigned long stopclk;		/* TOD-clock of request interrupt */
+	unsigned long endclk;		/* TOD-clock of request termination */
 
         /* Callback that is called after reaching final status. */
 	void (*callback)(struct dasd_ccw_req *, void *data);
@@ -245,6 +250,16 @@ struct dasd_ccw_req {
 #define DASD_CQR_SUPPRESS_IL	6	/* Suppress 'Incorrect Length' error */
 #define DASD_CQR_SUPPRESS_CR	7	/* Suppress 'Command Reject' error */
 
+/*
+ * There is no reliable way to determine the number of available CPUs on
+ * LPAR but there is no big performance difference between 1 and the
+ * maximum CPU number.
+ * 64 is a good trade off performance wise.
+ */
+#define DASD_NR_HW_QUEUES 64
+#define DASD_MAX_LCU_DEV 256
+#define DASD_REQ_PER_DEV 4
+
 /* Signature for error recovery functions. */
 typedef struct dasd_ccw_req *(*dasd_erp_fn_t) (struct dasd_ccw_req *);
 
@@ -423,7 +438,7 @@ struct dasd_path {
 	u8 chpid;
 	struct dasd_conf_data *conf_data;
 	atomic_t error_count;
-	unsigned long long errorclk;
+	unsigned long errorclk;
 };
 
 
@@ -454,6 +469,10 @@ struct dasd_profile_info {
 	unsigned int dasd_read_time2[32];  /* hist. of time from start to irq */
 	unsigned int dasd_read_time3[32];  /* hist. of time from irq to end */
 	unsigned int dasd_read_nr_req[32]; /* hist. of # of requests in chanq */
+	unsigned long dasd_sum_times;	   /* sum of request times */
+	unsigned long dasd_sum_time_str;   /* sum of time from build to start */
+	unsigned long dasd_sum_time_irq;   /* sum of time from start to irq */
+	unsigned long dasd_sum_time_end;   /* sum of time from irq to end */
 };
 
 struct dasd_profile {
@@ -532,10 +551,11 @@ struct dasd_block {
 	struct gendisk *gdp;
 	struct request_queue *request_queue;
 	spinlock_t request_queue_lock;
+	struct blk_mq_tag_set tag_set;
 	struct block_device *bdev;
 	atomic_t open_count;
 
-	unsigned long long blocks; /* size of volume in blocks */
+	unsigned long blocks;	   /* size of volume in blocks */
 	unsigned int bp_block;	   /* bytes per block */
 	unsigned int s2b_shift;	   /* log2 (bp_block/512) */
 
@@ -556,6 +576,10 @@ struct dasd_attention_data {
 	__u8 lpum;
 };
 
+struct dasd_queue {
+	spinlock_t lock;
+};
+
 /* reasons why device (ccw_device_start) was stopped */
 #define DASD_STOPPED_NOT_ACC 1         /* not accessible */
 #define DASD_STOPPED_QUIESCE 2         /* Quiesced */
@@ -724,7 +748,7 @@ void dasd_free_device(struct dasd_device *);
 struct dasd_block *dasd_alloc_block(void);
 void dasd_free_block(struct dasd_block *);
 
-enum blk_eh_timer_return dasd_times_out(struct request *req);
+enum blk_eh_timer_return dasd_times_out(struct request *req, bool reserved);
 
 void dasd_enable_device(struct dasd_device *);
 void dasd_set_target_state(struct dasd_device *, int);
diff --git a/drivers/s390/block/dasd_proc.c b/drivers/s390/block/dasd_proc.c
index 70dc2c4cd3f7..7104d6765773 100644
--- a/drivers/s390/block/dasd_proc.c
+++ b/drivers/s390/block/dasd_proc.c
@@ -90,7 +90,7 @@ dasd_devices_show(struct seq_file *m, void *v)
 			seq_printf(m, "n/f	 ");
 		else
 			seq_printf(m,
-				   "at blocksize: %d, %lld blocks, %lld MB",
+				   "at blocksize: %u, %lu blocks, %lu MB",
 				   block->bp_block, block->blocks,
 				   ((block->bp_block >> 9) *
 				    block->blocks) >> 11);
diff --git a/drivers/s390/block/dcssblk.c b/drivers/s390/block/dcssblk.c
index 68bae4f6bd88..7abb240847c0 100644
--- a/drivers/s390/block/dcssblk.c
+++ b/drivers/s390/block/dcssblk.c
@@ -856,14 +856,14 @@ dcssblk_make_request(struct request_queue *q, struct bio *bio)
 	blk_queue_split(q, &bio);
 
 	bytes_done = 0;
-	dev_info = bio->bi_bdev->bd_disk->private_data;
+	dev_info = bio->bi_disk->private_data;
 	if (dev_info == NULL)
 		goto fail;
 	if ((bio->bi_iter.bi_sector & 7) != 0 ||
 	    (bio->bi_iter.bi_size & 4095) != 0)
 		/* Request is not page-aligned. */
 		goto fail;
-	if (bio_end_sector(bio) > get_capacity(bio->bi_bdev->bd_disk)) {
+	if (bio_end_sector(bio) > get_capacity(bio->bi_disk)) {
 		/* Request beyond end of DCSS segment. */
 		goto fail;
 	}
diff --git a/drivers/s390/block/scm_blk.c b/drivers/s390/block/scm_blk.c
index 0071febac9e6..2e7fd966c515 100644
--- a/drivers/s390/block/scm_blk.c
+++ b/drivers/s390/block/scm_blk.c
@@ -249,13 +249,13 @@ static void scm_request_requeue(struct scm_request *scmrq)
 static void scm_request_finish(struct scm_request *scmrq)
 {
 	struct scm_blk_dev *bdev = scmrq->bdev;
+	int *error;
 	int i;
 
 	for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++) {
-		if (scmrq->error)
-			blk_mq_end_request(scmrq->request[i], scmrq->error);
-		else
-			blk_mq_complete_request(scmrq->request[i]);
+		error = blk_mq_rq_to_pdu(scmrq->request[i]);
+		*error = scmrq->error;
+		blk_mq_complete_request(scmrq->request[i]);
 	}
 
 	atomic_dec(&bdev->queued_reqs);
@@ -415,7 +415,9 @@ void scm_blk_irq(struct scm_device *scmdev, void *data, blk_status_t error)
 
 static void scm_blk_request_done(struct request *req)
 {
-	blk_mq_end_request(req, 0);
+	int *error = blk_mq_rq_to_pdu(req);
+
+	blk_mq_end_request(req, *error);
 }
 
 static const struct block_device_operations scm_blk_devops = {
@@ -448,6 +450,7 @@ int scm_blk_dev_setup(struct scm_blk_dev *bdev, struct scm_device *scmdev)
 	atomic_set(&bdev->queued_reqs, 0);
 
 	bdev->tag_set.ops = &scm_mq_ops;
+	bdev->tag_set.cmd_size = sizeof(int);
 	bdev->tag_set.nr_hw_queues = nr_requests;
 	bdev->tag_set.queue_depth = nr_requests_per_io * nr_requests;
 	bdev->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
diff --git a/drivers/s390/block/xpram.c b/drivers/s390/block/xpram.c
index a48f0d40c1d2..571a0709e1e5 100644
--- a/drivers/s390/block/xpram.c
+++ b/drivers/s390/block/xpram.c
@@ -183,7 +183,7 @@ static unsigned long xpram_highest_page_index(void)
  */
 static blk_qc_t xpram_make_request(struct request_queue *q, struct bio *bio)
 {
-	xpram_device_t *xdev = bio->bi_bdev->bd_disk->private_data;
+	xpram_device_t *xdev = bio->bi_disk->private_data;
 	struct bio_vec bvec;
 	struct bvec_iter iter;
 	unsigned int index;
diff --git a/drivers/s390/char/Kconfig b/drivers/s390/char/Kconfig
index b3f1c458905f..97c4c9fdd53d 100644
--- a/drivers/s390/char/Kconfig
+++ b/drivers/s390/char/Kconfig
@@ -169,10 +169,21 @@ config VMCP
 	def_bool y
 	prompt "Support for the z/VM CP interface"
 	depends on S390
+	select CMA
 	help
 	  Select this option if you want to be able to interact with the control
 	  program on z/VM
 
+config VMCP_CMA_SIZE
+	int "Memory in MiB reserved for z/VM CP interface"
+	default "4"
+	depends on VMCP
+	help
+	  Specify the default amount of memory in MiB reserved for the z/VM CP
+	  interface. If needed this memory is used for large contiguous memory
+	  allocations. The default can be changed with the kernel command line
+	  parameter "vmcp_cma".
+
 config MONREADER
 	def_tristate m
 	prompt "API for reading z/VM monitor service records"
diff --git a/drivers/s390/char/raw3270.c b/drivers/s390/char/raw3270.c
index 710f2292911d..5d4f053d7c38 100644
--- a/drivers/s390/char/raw3270.c
+++ b/drivers/s390/char/raw3270.c
@@ -1082,7 +1082,7 @@ static struct attribute * raw3270_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group raw3270_attr_group = {
+static const struct attribute_group raw3270_attr_group = {
 	.attrs = raw3270_attrs,
 };
 
diff --git a/drivers/s390/char/sclp_cmd.c b/drivers/s390/char/sclp_cmd.c
index b9c5522b8a68..dff8b94871f0 100644
--- a/drivers/s390/char/sclp_cmd.c
+++ b/drivers/s390/char/sclp_cmd.c
@@ -252,6 +252,7 @@ static int sclp_attach_storage(u8 id)
 	if (!sccb)
 		return -ENOMEM;
 	sccb->header.length = PAGE_SIZE;
+	sccb->header.function_code = 0x40;
 	rc = sclp_sync_request_timeout(0x00080001 | id << 8, sccb,
 				       SCLP_QUEUE_INTERVAL);
 	if (rc)
diff --git a/drivers/s390/char/sclp_config.c b/drivers/s390/char/sclp_config.c
index 1406fb688a26..7003d52c2191 100644
--- a/drivers/s390/char/sclp_config.c
+++ b/drivers/s390/char/sclp_config.c
@@ -135,7 +135,7 @@ static ssize_t sysfs_ofb_data_write(struct file *filp, struct kobject *kobj,
 	return rc ?: count;
 }
 
-static struct bin_attribute ofb_bin_attr = {
+static const struct bin_attribute ofb_bin_attr = {
 	.attr = {
 		.name = "event_data",
 		.mode = S_IWUSR,
diff --git a/drivers/s390/char/sclp_early.c b/drivers/s390/char/sclp_early.c
index efd84d1d178b..bc1fc00910b0 100644
--- a/drivers/s390/char/sclp_early.c
+++ b/drivers/s390/char/sclp_early.c
@@ -39,7 +39,7 @@ struct read_info_sccb {
 	u8	fac84;			/* 84 */
 	u8	fac85;			/* 85 */
 	u8	_pad_86[91 - 86];	/* 86-90 */
-	u8	flags;			/* 91 */
+	u8	fac91;			/* 91 */
 	u8	_pad_92[98 - 92];	/* 92-97 */
 	u8	fac98;			/* 98 */
 	u8	hamaxpow;		/* 99 */
@@ -103,6 +103,8 @@ static void __init sclp_early_facilities_detect(struct read_info_sccb *sccb)
 	sclp.has_kss = !!(sccb->fac98 & 0x01);
 	if (sccb->fac85 & 0x02)
 		S390_lowcore.machine_flags |= MACHINE_FLAG_ESOP;
+	if (sccb->fac91 & 0x40)
+		S390_lowcore.machine_flags |= MACHINE_FLAG_TLB_GUEST;
 	sclp.rnmax = sccb->rnmax ? sccb->rnmax : sccb->rnmax2;
 	sclp.rzm = sccb->rnsize ? sccb->rnsize : sccb->rnsize2;
 	sclp.rzm <<= 20;
@@ -139,7 +141,7 @@ static void __init sclp_early_facilities_detect(struct read_info_sccb *sccb)
 
 	/* Save IPL information */
 	sclp_ipl_info.is_valid = 1;
-	if (sccb->flags & 0x2)
+	if (sccb->fac91 & 0x2)
 		sclp_ipl_info.has_dump = 1;
 	memcpy(&sclp_ipl_info.loadparm, &sccb->loadparm, LOADPARM_LEN);
 
diff --git a/drivers/s390/char/sclp_ocf.c b/drivers/s390/char/sclp_ocf.c
index f59b71776bbd..f9cbb1ab047b 100644
--- a/drivers/s390/char/sclp_ocf.c
+++ b/drivers/s390/char/sclp_ocf.c
@@ -126,7 +126,7 @@ static struct attribute *ocf_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group ocf_attr_group = {
+static const struct attribute_group ocf_attr_group = {
 	.attrs = ocf_attrs,
 };
 
diff --git a/drivers/s390/char/tape_core.c b/drivers/s390/char/tape_core.c
index 3c379da2eef8..9dd4534823b3 100644
--- a/drivers/s390/char/tape_core.c
+++ b/drivers/s390/char/tape_core.c
@@ -175,7 +175,7 @@ static struct attribute *tape_attrs[] = {
 	NULL
 };
 
-static struct attribute_group tape_attr_group = {
+static const struct attribute_group tape_attr_group = {
 	.attrs = tape_attrs,
 };
 
diff --git a/drivers/s390/char/vmcp.c b/drivers/s390/char/vmcp.c
index 98749fa817da..7898bbcc28fc 100644
--- a/drivers/s390/char/vmcp.c
+++ b/drivers/s390/char/vmcp.c
@@ -17,15 +17,85 @@
 #include <linux/kernel.h>
 #include <linux/miscdevice.h>
 #include <linux/slab.h>
+#include <linux/uaccess.h>
 #include <linux/export.h>
+#include <linux/mutex.h>
+#include <linux/cma.h>
+#include <linux/mm.h>
 #include <asm/compat.h>
 #include <asm/cpcmd.h>
 #include <asm/debug.h>
-#include <linux/uaccess.h>
-#include "vmcp.h"
+#include <asm/vmcp.h>
+
+struct vmcp_session {
+	char *response;
+	unsigned int bufsize;
+	unsigned int cma_alloc : 1;
+	int resp_size;
+	int resp_code;
+	struct mutex mutex;
+};
 
 static debug_info_t *vmcp_debug;
 
+static unsigned long vmcp_cma_size __initdata = CONFIG_VMCP_CMA_SIZE * 1024 * 1024;
+static struct cma *vmcp_cma;
+
+static int __init early_parse_vmcp_cma(char *p)
+{
+	vmcp_cma_size = ALIGN(memparse(p, NULL), PAGE_SIZE);
+	return 0;
+}
+early_param("vmcp_cma", early_parse_vmcp_cma);
+
+void __init vmcp_cma_reserve(void)
+{
+	if (!MACHINE_IS_VM)
+		return;
+	cma_declare_contiguous(0, vmcp_cma_size, 0, 0, 0, false, "vmcp", &vmcp_cma);
+}
+
+static void vmcp_response_alloc(struct vmcp_session *session)
+{
+	struct page *page = NULL;
+	int nr_pages, order;
+
+	order = get_order(session->bufsize);
+	nr_pages = ALIGN(session->bufsize, PAGE_SIZE) >> PAGE_SHIFT;
+	/*
+	 * For anything below order 3 allocations rely on the buddy
+	 * allocator. If such low-order allocations can't be handled
+	 * anymore the system won't work anyway.
+	 */
+	if (order > 2)
+		page = cma_alloc(vmcp_cma, nr_pages, 0, GFP_KERNEL);
+	if (page) {
+		session->response = (char *)page_to_phys(page);
+		session->cma_alloc = 1;
+		return;
+	}
+	session->response = (char *)__get_free_pages(GFP_KERNEL | __GFP_RETRY_MAYFAIL, order);
+}
+
+static void vmcp_response_free(struct vmcp_session *session)
+{
+	int nr_pages, order;
+	struct page *page;
+
+	if (!session->response)
+		return;
+	order = get_order(session->bufsize);
+	nr_pages = ALIGN(session->bufsize, PAGE_SIZE) >> PAGE_SHIFT;
+	if (session->cma_alloc) {
+		page = phys_to_page((unsigned long)session->response);
+		cma_release(vmcp_cma, page, nr_pages);
+		session->cma_alloc = 0;
+	} else {
+		free_pages((unsigned long)session->response, order);
+	}
+	session->response = NULL;
+}
+
 static int vmcp_open(struct inode *inode, struct file *file)
 {
 	struct vmcp_session *session;
@@ -51,7 +121,7 @@ static int vmcp_release(struct inode *inode, struct file *file)
 
 	session = file->private_data;
 	file->private_data = NULL;
-	free_pages((unsigned long)session->response, get_order(session->bufsize));
+	vmcp_response_free(session);
 	kfree(session);
 	return 0;
 }
@@ -97,9 +167,7 @@ vmcp_write(struct file *file, const char __user *buff, size_t count,
 		return -ERESTARTSYS;
 	}
 	if (!session->response)
-		session->response = (char *)__get_free_pages(GFP_KERNEL
-						| __GFP_RETRY_MAYFAIL | GFP_DMA,
-						get_order(session->bufsize));
+		vmcp_response_alloc(session);
 	if (!session->response) {
 		mutex_unlock(&session->mutex);
 		kfree(cmd);
@@ -130,8 +198,8 @@ vmcp_write(struct file *file, const char __user *buff, size_t count,
 static long vmcp_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 {
 	struct vmcp_session *session;
+	int ret = -ENOTTY;
 	int __user *argp;
-	int temp;
 
 	session = file->private_data;
 	if (is_compat_task())
@@ -142,28 +210,26 @@ static long vmcp_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 		return -ERESTARTSYS;
 	switch (cmd) {
 	case VMCP_GETCODE:
-		temp = session->resp_code;
-		mutex_unlock(&session->mutex);
-		return put_user(temp, argp);
+		ret = put_user(session->resp_code, argp);
+		break;
 	case VMCP_SETBUF:
-		free_pages((unsigned long)session->response,
-				get_order(session->bufsize));
-		session->response=NULL;
-		temp = get_user(session->bufsize, argp);
-		if (get_order(session->bufsize) > 8) {
+		vmcp_response_free(session);
+		ret = get_user(session->bufsize, argp);
+		if (ret)
 			session->bufsize = PAGE_SIZE;
-			temp = -EINVAL;
+		if (!session->bufsize || get_order(session->bufsize) > 8) {
+			session->bufsize = PAGE_SIZE;
+			ret = -EINVAL;
 		}
-		mutex_unlock(&session->mutex);
-		return temp;
+		break;
 	case VMCP_GETSIZE:
-		temp = session->resp_size;
-		mutex_unlock(&session->mutex);
-		return put_user(temp, argp);
+		ret = put_user(session->resp_size, argp);
+		break;
 	default:
-		mutex_unlock(&session->mutex);
-		return -ENOIOCTLCMD;
+		break;
 	}
+	mutex_unlock(&session->mutex);
+	return ret;
 }
 
 static const struct file_operations vmcp_fops = {
diff --git a/drivers/s390/char/vmcp.h b/drivers/s390/char/vmcp.h
deleted file mode 100644
index 1e29b0418382..000000000000
--- a/drivers/s390/char/vmcp.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright IBM Corp. 2004, 2005
- * Interface implementation for communication with the z/VM control program
- * Version 1.0
- * Author(s): Christian Borntraeger <cborntra@de.ibm.com>
- *
- *
- * z/VMs CP offers the possibility to issue commands via the diagnose code 8
- * this driver implements a character device that issues these commands and
- * returns the answer of CP.
- *
- * The idea of this driver is based on cpint from Neale Ferguson
- */
-
-#include <linux/ioctl.h>
-#include <linux/mutex.h>
-
-#define VMCP_GETCODE _IOR(0x10, 1, int)
-#define VMCP_SETBUF _IOW(0x10, 2, int)
-#define VMCP_GETSIZE _IOR(0x10, 3, int)
-
-struct vmcp_session {
-	unsigned int bufsize;
-	char *response;
-	int resp_size;
-	int resp_code;
-	/* As we use copy_from/to_user, which might     *
-	 * sleep and cannot use a spinlock              */
-	struct mutex mutex;
-};
diff --git a/drivers/s390/cio/chp.c b/drivers/s390/cio/chp.c
index 432fc40990bd..f4166f80c4d4 100644
--- a/drivers/s390/cio/chp.c
+++ b/drivers/s390/cio/chp.c
@@ -143,7 +143,7 @@ static ssize_t chp_measurement_chars_read(struct file *filp,
 				       sizeof(chp->cmg_chars));
 }
 
-static struct bin_attribute chp_measurement_chars_attr = {
+static const struct bin_attribute chp_measurement_chars_attr = {
 	.attr = {
 		.name = "measurement_chars",
 		.mode = S_IRUSR,
@@ -197,7 +197,7 @@ static ssize_t chp_measurement_read(struct file *filp, struct kobject *kobj,
 	return count;
 }
 
-static struct bin_attribute chp_measurement_attr = {
+static const struct bin_attribute chp_measurement_attr = {
 	.attr = {
 		.name = "measurement",
 		.mode = S_IRUSR,
diff --git a/drivers/s390/cio/device.c b/drivers/s390/cio/device.c
index 7be01a58b44f..489b583f263d 100644
--- a/drivers/s390/cio/device.c
+++ b/drivers/s390/cio/device.c
@@ -612,7 +612,7 @@ static struct attribute *io_subchannel_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group io_subchannel_attr_group = {
+static const struct attribute_group io_subchannel_attr_group = {
 	.attrs = io_subchannel_attrs,
 };
 
@@ -626,7 +626,7 @@ static struct attribute * ccwdev_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group ccwdev_attr_group = {
+static const struct attribute_group ccwdev_attr_group = {
 	.attrs = ccwdev_attrs,
 };
 
diff --git a/drivers/s390/cio/vfio_ccw_cp.c b/drivers/s390/cio/vfio_ccw_cp.c
index ba6ac83a6c25..5ccfdc80d0ec 100644
--- a/drivers/s390/cio/vfio_ccw_cp.c
+++ b/drivers/s390/cio/vfio_ccw_cp.c
@@ -481,7 +481,7 @@ static int ccwchain_fetch_tic(struct ccwchain *chain,
 		ccw_tail = ccw_head + (iter->ch_len - 1) * sizeof(struct ccw1);
 
 		if ((ccw_head <= ccw->cda) && (ccw->cda <= ccw_tail)) {
-			ccw->cda = (__u32) (addr_t) (iter->ch_ccw +
+			ccw->cda = (__u32) (addr_t) (((char *)iter->ch_ccw) +
 						     (ccw->cda - ccw_head));
 			return 0;
 		}
diff --git a/drivers/s390/crypto/ap_asm.h b/drivers/s390/crypto/ap_asm.h
index 287b4ad0999e..cd350345b3d2 100644
--- a/drivers/s390/crypto/ap_asm.h
+++ b/drivers/s390/crypto/ap_asm.h
@@ -69,16 +69,19 @@ static inline struct ap_queue_status ap_rapq(ap_qid_t qid)
 }
 
 /**
- * ap_aqic(): Enable interruption for a specific AP.
+ * ap_aqic(): Control interruption for a specific AP.
  * @qid: The AP queue number
+ * @qirqctrl: struct ap_qirq_ctrl (64 bit value)
  * @ind: The notification indicator byte
  *
  * Returns AP queue status.
  */
-static inline struct ap_queue_status ap_aqic(ap_qid_t qid, void *ind)
+static inline struct ap_queue_status ap_aqic(ap_qid_t qid,
+					     struct ap_qirq_ctrl qirqctrl,
+					     void *ind)
 {
 	register unsigned long reg0 asm ("0") = qid | (3UL << 24);
-	register unsigned long reg1_in asm ("1") = (8UL << 44) | AP_ISC;
+	register struct ap_qirq_ctrl reg1_in asm ("1") = qirqctrl;
 	register struct ap_queue_status reg1_out asm ("1");
 	register void *reg2 asm ("2") = ind;
 
diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c
index 6dee598979e7..5f0be2040272 100644
--- a/drivers/s390/crypto/ap_bus.c
+++ b/drivers/s390/crypto/ap_bus.c
@@ -166,26 +166,51 @@ static int ap_configuration_available(void)
 }
 
 /**
+ * ap_apft_available(): Test if AP facilities test (APFT)
+ * facility is available.
+ *
+ * Returns 1 if APFT is is available.
+ */
+static int ap_apft_available(void)
+{
+	return test_facility(15);
+}
+
+/**
  * ap_test_queue(): Test adjunct processor queue.
  * @qid: The AP queue number
+ * @tbit: Test facilities bit
  * @info: Pointer to queue descriptor
  *
  * Returns AP queue status structure.
  */
-static inline struct ap_queue_status
-ap_test_queue(ap_qid_t qid, unsigned long *info)
+struct ap_queue_status ap_test_queue(ap_qid_t qid,
+				     int tbit,
+				     unsigned long *info)
 {
-	if (test_facility(15))
-		qid |= 1UL << 23;		/* set APFT T bit*/
+	if (tbit)
+		qid |= 1UL << 23; /* set T bit*/
 	return ap_tapq(qid, info);
 }
+EXPORT_SYMBOL(ap_test_queue);
 
-static inline int ap_query_configuration(void)
+/*
+ * ap_query_configuration(): Fetch cryptographic config info
+ *
+ * Returns the ap configuration info fetched via PQAP(QCI).
+ * On success 0 is returned, on failure a negative errno
+ * is returned, e.g. if the PQAP(QCI) instruction is not
+ * available, the return value will be -EOPNOTSUPP.
+ */
+int ap_query_configuration(struct ap_config_info *info)
 {
-	if (!ap_configuration)
+	if (!ap_configuration_available())
 		return -EOPNOTSUPP;
-	return ap_qci(ap_configuration);
+	if (!info)
+		return -EINVAL;
+	return ap_qci(info);
 }
+EXPORT_SYMBOL(ap_query_configuration);
 
 /**
  * ap_init_configuration(): Allocate and query configuration array.
@@ -198,7 +223,7 @@ static void ap_init_configuration(void)
 	ap_configuration = kzalloc(sizeof(*ap_configuration), GFP_KERNEL);
 	if (!ap_configuration)
 		return;
-	if (ap_query_configuration() != 0) {
+	if (ap_query_configuration(ap_configuration) != 0) {
 		kfree(ap_configuration);
 		ap_configuration = NULL;
 		return;
@@ -261,7 +286,7 @@ static int ap_query_queue(ap_qid_t qid, int *queue_depth, int *device_type,
 	if (!ap_test_config_card_id(AP_QID_CARD(qid)))
 		return -ENODEV;
 
-	status = ap_test_queue(qid, &info);
+	status = ap_test_queue(qid, ap_apft_available(), &info);
 	switch (status.response_code) {
 	case AP_RESPONSE_NORMAL:
 		*queue_depth = (int)(info & 0xff);
@@ -940,7 +965,9 @@ static int ap_select_domain(void)
 		for (j = 0; j < AP_DEVICES; j++) {
 			if (!ap_test_config_card_id(j))
 				continue;
-			status = ap_test_queue(AP_MKQID(j, i), NULL);
+			status = ap_test_queue(AP_MKQID(j, i),
+					       ap_apft_available(),
+					       NULL);
 			if (status.response_code != AP_RESPONSE_NORMAL)
 				continue;
 			count++;
@@ -993,7 +1020,7 @@ static void ap_scan_bus(struct work_struct *unused)
 
 	AP_DBF(DBF_DEBUG, "ap_scan_bus running\n");
 
-	ap_query_configuration();
+	ap_query_configuration(ap_configuration);
 	if (ap_select_domain() != 0)
 		goto out;
 
diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h
index 4dc7c88fb054..754cf2223cfb 100644
--- a/drivers/s390/crypto/ap_bus.h
+++ b/drivers/s390/crypto/ap_bus.h
@@ -28,6 +28,7 @@
 
 #include <linux/device.h>
 #include <linux/types.h>
+#include <asm/ap.h>
 
 #define AP_DEVICES 64		/* Number of AP devices. */
 #define AP_DOMAINS 256		/* Number of AP domains. */
@@ -40,41 +41,6 @@ extern int ap_domain_index;
 extern spinlock_t ap_list_lock;
 extern struct list_head ap_card_list;
 
-/**
- * The ap_qid_t identifier of an ap queue. It contains a
- * 6 bit card index and a 4 bit queue index (domain).
- */
-typedef unsigned int ap_qid_t;
-
-#define AP_MKQID(_card, _queue) (((_card) & 63) << 8 | ((_queue) & 255))
-#define AP_QID_CARD(_qid) (((_qid) >> 8) & 63)
-#define AP_QID_QUEUE(_qid) ((_qid) & 255)
-
-/**
- * structy ap_queue_status - Holds the AP queue status.
- * @queue_empty: Shows if queue is empty
- * @replies_waiting: Waiting replies
- * @queue_full: Is 1 if the queue is full
- * @pad: A 4 bit pad
- * @int_enabled: Shows if interrupts are enabled for the AP
- * @response_code: Holds the 8 bit response code
- * @pad2: A 16 bit pad
- *
- * The ap queue status word is returned by all three AP functions
- * (PQAP, NQAP and DQAP).  There's a set of flags in the first
- * byte, followed by a 1 byte response code.
- */
-struct ap_queue_status {
-	unsigned int queue_empty	: 1;
-	unsigned int replies_waiting	: 1;
-	unsigned int queue_full		: 1;
-	unsigned int pad1		: 4;
-	unsigned int int_enabled	: 1;
-	unsigned int response_code	: 8;
-	unsigned int pad2		: 16;
-} __packed;
-
-
 static inline int ap_test_bit(unsigned int *ptr, unsigned int nr)
 {
 	return (*ptr & (0x80000000u >> nr)) != 0;
@@ -238,17 +204,6 @@ struct ap_message {
 			struct ap_message *);
 };
 
-struct ap_config_info {
-	unsigned int special_command:1;
-	unsigned int ap_extended:1;
-	unsigned char reserved1:6;
-	unsigned char reserved2[15];
-	unsigned int apm[8];		/* AP ID mask */
-	unsigned int aqm[8];		/* AP queue mask */
-	unsigned int adm[8];		/* AP domain mask */
-	unsigned char reserved4[16];
-} __packed;
-
 /**
  * ap_init_message() - Initialize ap_message.
  * Initialize a message before using. Otherwise this might result in
diff --git a/drivers/s390/crypto/ap_queue.c b/drivers/s390/crypto/ap_queue.c
index 0f1a5d02acb0..56b96edffd5b 100644
--- a/drivers/s390/crypto/ap_queue.c
+++ b/drivers/s390/crypto/ap_queue.c
@@ -16,6 +16,25 @@
 #include "ap_asm.h"
 
 /**
+ * ap_queue_irq_ctrl(): Control interruption on a AP queue.
+ * @qirqctrl: struct ap_qirq_ctrl (64 bit value)
+ * @ind: The notification indicator byte
+ *
+ * Returns AP queue status.
+ *
+ * Control interruption on the given AP queue.
+ * Just a simple wrapper function for the low level PQAP(AQIC)
+ * instruction available for other kernel modules.
+ */
+struct ap_queue_status ap_queue_irq_ctrl(ap_qid_t qid,
+					 struct ap_qirq_ctrl qirqctrl,
+					 void *ind)
+{
+	return ap_aqic(qid, qirqctrl, ind);
+}
+EXPORT_SYMBOL(ap_queue_irq_ctrl);
+
+/**
  * ap_queue_enable_interruption(): Enable interruption on an AP queue.
  * @qid: The AP queue number
  * @ind: the notification indicator byte
@@ -27,8 +46,11 @@
 static int ap_queue_enable_interruption(struct ap_queue *aq, void *ind)
 {
 	struct ap_queue_status status;
+	struct ap_qirq_ctrl qirqctrl = { 0 };
 
-	status = ap_aqic(aq->qid, ind);
+	qirqctrl.ir = 1;
+	qirqctrl.isc = AP_ISC;
+	status = ap_aqic(aq->qid, qirqctrl, ind);
 	switch (status.response_code) {
 	case AP_RESPONSE_NORMAL:
 	case AP_RESPONSE_OTHERWISE_CHANGED:
@@ -362,7 +384,7 @@ static enum ap_wait ap_sm_setirq_wait(struct ap_queue *aq)
 		/* Get the status with TAPQ */
 		status = ap_tapq(aq->qid, NULL);
 
-	if (status.int_enabled == 1) {
+	if (status.irq_enabled == 1) {
 		/* Irqs are now enabled */
 		aq->interrupt = AP_INTR_ENABLED;
 		aq->state = (aq->queue_count > 0) ?
diff --git a/drivers/s390/crypto/zcrypt_card.c b/drivers/s390/crypto/zcrypt_card.c
index 53436ea52230..f85dacf1c284 100644
--- a/drivers/s390/crypto/zcrypt_card.c
+++ b/drivers/s390/crypto/zcrypt_card.c
@@ -98,7 +98,7 @@ static struct attribute *zcrypt_card_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group zcrypt_card_attr_group = {
+static const struct attribute_group zcrypt_card_attr_group = {
 	.attrs = zcrypt_card_attrs,
 };
 
diff --git a/drivers/s390/crypto/zcrypt_msgtype6.c b/drivers/s390/crypto/zcrypt_msgtype6.c
index 4fddb4319481..afd20cee7ea0 100644
--- a/drivers/s390/crypto/zcrypt_msgtype6.c
+++ b/drivers/s390/crypto/zcrypt_msgtype6.c
@@ -140,7 +140,7 @@ struct function_and_rules_block {
  *   + 0x000A 'MRP     ' (MCL3 'PK' or CEX2C 'PK')
  * - VUD block
  */
-static struct CPRBX static_cprbx = {
+static const struct CPRBX static_cprbx = {
 	.cprb_len	=  0x00DC,
 	.cprb_ver_id	=  0x02,
 	.func_id	= {0x54, 0x32},
diff --git a/drivers/s390/crypto/zcrypt_queue.c b/drivers/s390/crypto/zcrypt_queue.c
index a303f3b2c328..4742be0eec24 100644
--- a/drivers/s390/crypto/zcrypt_queue.c
+++ b/drivers/s390/crypto/zcrypt_queue.c
@@ -89,7 +89,7 @@ static struct attribute *zcrypt_queue_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group zcrypt_queue_attr_group = {
+static const struct attribute_group zcrypt_queue_attr_group = {
 	.attrs = zcrypt_queue_attrs,
 };
 
diff --git a/drivers/s390/net/ctcm_main.c b/drivers/s390/net/ctcm_main.c
index 2ade6131a89f..26363e0816fe 100644
--- a/drivers/s390/net/ctcm_main.c
+++ b/drivers/s390/net/ctcm_main.c
@@ -305,7 +305,7 @@ static long ctcm_check_irb_error(struct ccw_device *cdev, struct irb *irb)
  *  ch		The channel, the sense code belongs to.
  *  sense	The sense code to inspect.
  */
-static inline void ccw_unit_check(struct channel *ch, __u8 sense)
+static void ccw_unit_check(struct channel *ch, __u8 sense)
 {
 	CTCM_DBF_TEXT_(TRACE, CTC_DBF_DEBUG,
 			"%s(%s): %02x",
diff --git a/drivers/s390/net/lcs.c b/drivers/s390/net/lcs.c
index 619da81dca70..d01b5c2a7760 100644
--- a/drivers/s390/net/lcs.c
+++ b/drivers/s390/net/lcs.c
@@ -327,8 +327,7 @@ lcs_set_allowed_threads(struct lcs_card *card, unsigned long threads)
 	spin_unlock_irqrestore(&card->mask_lock, flags);
 	wake_up(&card->wait_q);
 }
-static inline int
-lcs_threads_running(struct lcs_card *card, unsigned long threads)
+static int lcs_threads_running(struct lcs_card *card, unsigned long threads)
 {
         unsigned long flags;
         int rc = 0;
@@ -346,8 +345,7 @@ lcs_wait_for_threads(struct lcs_card *card, unsigned long threads)
                         lcs_threads_running(card, threads) == 0);
 }
 
-static inline int
-lcs_set_thread_start_bit(struct lcs_card *card, unsigned long thread)
+static int lcs_set_thread_start_bit(struct lcs_card *card, unsigned long thread)
 {
         unsigned long flags;
 
@@ -373,8 +371,7 @@ lcs_clear_thread_running_bit(struct lcs_card *card, unsigned long thread)
         wake_up(&card->wait_q);
 }
 
-static inline int
-__lcs_do_run_thread(struct lcs_card *card, unsigned long thread)
+static int __lcs_do_run_thread(struct lcs_card *card, unsigned long thread)
 {
         unsigned long flags;
         int rc = 0;
@@ -444,8 +441,7 @@ lcs_setup_card(struct lcs_card *card)
 	INIT_LIST_HEAD(&card->lancmd_waiters);
 }
 
-static inline void
-lcs_clear_multicast_list(struct lcs_card *card)
+static void lcs_clear_multicast_list(struct lcs_card *card)
 {
 #ifdef	CONFIG_IP_MULTICAST
 	struct lcs_ipm_list *ipm;
@@ -656,8 +652,7 @@ __lcs_resume_channel(struct lcs_channel *channel)
 /**
  * Make a buffer ready for processing.
  */
-static inline void
-__lcs_ready_buffer_bits(struct lcs_channel *channel, int index)
+static void __lcs_ready_buffer_bits(struct lcs_channel *channel, int index)
 {
 	int prev, next;
 
@@ -1169,8 +1164,8 @@ lcs_get_mac_for_ipm(__be32 ipm, char *mac, struct net_device *dev)
 /**
  * function called by net device to handle multicast address relevant things
  */
-static inline void
-lcs_remove_mc_addresses(struct lcs_card *card, struct in_device *in4_dev)
+static void lcs_remove_mc_addresses(struct lcs_card *card,
+				    struct in_device *in4_dev)
 {
 	struct ip_mc_list *im4;
 	struct list_head *l;
@@ -1196,8 +1191,9 @@ lcs_remove_mc_addresses(struct lcs_card *card, struct in_device *in4_dev)
 	spin_unlock_irqrestore(&card->ipm_lock, flags);
 }
 
-static inline struct lcs_ipm_list *
-lcs_check_addr_entry(struct lcs_card *card, struct ip_mc_list *im4, char *buf)
+static struct lcs_ipm_list *lcs_check_addr_entry(struct lcs_card *card,
+						 struct ip_mc_list *im4,
+						 char *buf)
 {
 	struct lcs_ipm_list *tmp, *ipm = NULL;
 	struct list_head *l;
@@ -1218,8 +1214,8 @@ lcs_check_addr_entry(struct lcs_card *card, struct ip_mc_list *im4, char *buf)
 	return ipm;
 }
 
-static inline void
-lcs_set_mc_addresses(struct lcs_card *card, struct in_device *in4_dev)
+static void lcs_set_mc_addresses(struct lcs_card *card,
+				 struct in_device *in4_dev)
 {
 
 	struct ip_mc_list *im4;
diff --git a/drivers/s390/net/netiucv.c b/drivers/s390/net/netiucv.c
index 7e0e6a4019f3..b9c7c1e61da2 100644
--- a/drivers/s390/net/netiucv.c
+++ b/drivers/s390/net/netiucv.c
@@ -249,14 +249,14 @@ struct ll_header {
  * Compatibility macros for busy handling
  * of network devices.
  */
-static inline void netiucv_clear_busy(struct net_device *dev)
+static void netiucv_clear_busy(struct net_device *dev)
 {
 	struct netiucv_priv *priv = netdev_priv(dev);
 	clear_bit(0, &priv->tbusy);
 	netif_wake_queue(dev);
 }
 
-static inline int netiucv_test_and_set_busy(struct net_device *dev)
+static int netiucv_test_and_set_busy(struct net_device *dev)
 {
 	struct netiucv_priv *priv = netdev_priv(dev);
 	netif_stop_queue(dev);
diff --git a/drivers/s390/net/qeth_core.h b/drivers/s390/net/qeth_core.h
index 7a0ffc71b25d..59e09854c4f7 100644
--- a/drivers/s390/net/qeth_core.h
+++ b/drivers/s390/net/qeth_core.h
@@ -857,11 +857,6 @@ static inline int qeth_get_ip_version(struct sk_buff *skb)
 	}
 }
 
-static inline int qeth_get_ip_protocol(struct sk_buff *skb)
-{
-	return ip_hdr(skb)->protocol;
-}
-
 static inline void qeth_put_buffer_pool_entry(struct qeth_card *card,
 		struct qeth_buffer_pool_entry *entry)
 {
@@ -951,10 +946,13 @@ int qeth_get_priority_queue(struct qeth_card *, struct sk_buff *, int, int);
 int qeth_get_elements_no(struct qeth_card *card, struct sk_buff *skb,
 			 int extra_elems, int data_offset);
 int qeth_get_elements_for_frags(struct sk_buff *);
-int qeth_do_send_packet_fast(struct qeth_card *, struct qeth_qdio_out_q *,
-			struct sk_buff *, struct qeth_hdr *, int, int);
-int qeth_do_send_packet(struct qeth_card *, struct qeth_qdio_out_q *,
-		    struct sk_buff *, struct qeth_hdr *, int);
+int qeth_do_send_packet_fast(struct qeth_card *card,
+			     struct qeth_qdio_out_q *queue, struct sk_buff *skb,
+			     struct qeth_hdr *hdr, unsigned int offset,
+			     unsigned int hd_len);
+int qeth_do_send_packet(struct qeth_card *card, struct qeth_qdio_out_q *queue,
+			struct sk_buff *skb, struct qeth_hdr *hdr,
+			unsigned int hd_len, unsigned int offset, int elements);
 int qeth_do_ioctl(struct net_device *dev, struct ifreq *rq, int cmd);
 int qeth_core_get_sset_count(struct net_device *, int);
 void qeth_core_get_ethtool_stats(struct net_device *,
@@ -987,6 +985,7 @@ int qeth_set_features(struct net_device *, netdev_features_t);
 int qeth_recover_features(struct net_device *);
 netdev_features_t qeth_fix_features(struct net_device *, netdev_features_t);
 int qeth_vm_request_mac(struct qeth_card *card);
+int qeth_push_hdr(struct sk_buff *skb, struct qeth_hdr **hdr, unsigned int len);
 
 /* exports for OSN */
 int qeth_osn_assist(struct net_device *, void *, int);
diff --git a/drivers/s390/net/qeth_core_main.c b/drivers/s390/net/qeth_core_main.c
index 4792cabb862e..bae7440abc01 100644
--- a/drivers/s390/net/qeth_core_main.c
+++ b/drivers/s390/net/qeth_core_main.c
@@ -101,7 +101,7 @@ void qeth_close_dev(struct qeth_card *card)
 }
 EXPORT_SYMBOL_GPL(qeth_close_dev);
 
-static inline const char *qeth_get_cardname(struct qeth_card *card)
+static const char *qeth_get_cardname(struct qeth_card *card)
 {
 	if (card->info.guestlan) {
 		switch (card->info.type) {
@@ -330,7 +330,7 @@ static struct qeth_qdio_q *qeth_alloc_qdio_queue(void)
 	return q;
 }
 
-static inline int qeth_cq_init(struct qeth_card *card)
+static int qeth_cq_init(struct qeth_card *card)
 {
 	int rc;
 
@@ -352,7 +352,7 @@ out:
 	return rc;
 }
 
-static inline int qeth_alloc_cq(struct qeth_card *card)
+static int qeth_alloc_cq(struct qeth_card *card)
 {
 	int rc;
 
@@ -397,7 +397,7 @@ kmsg_out:
 	goto out;
 }
 
-static inline void qeth_free_cq(struct qeth_card *card)
+static void qeth_free_cq(struct qeth_card *card)
 {
 	if (card->qdio.c_q) {
 		--card->qdio.no_in_queues;
@@ -408,8 +408,9 @@ static inline void qeth_free_cq(struct qeth_card *card)
 	card->qdio.out_bufstates = NULL;
 }
 
-static inline enum iucv_tx_notify qeth_compute_cq_notification(int sbalf15,
-	int delayed) {
+static enum iucv_tx_notify qeth_compute_cq_notification(int sbalf15,
+							int delayed)
+{
 	enum iucv_tx_notify n;
 
 	switch (sbalf15) {
@@ -432,8 +433,8 @@ static inline enum iucv_tx_notify qeth_compute_cq_notification(int sbalf15,
 	return n;
 }
 
-static inline void qeth_cleanup_handled_pending(struct qeth_qdio_out_q *q,
-	int bidx, int forced_cleanup)
+static void qeth_cleanup_handled_pending(struct qeth_qdio_out_q *q, int bidx,
+					 int forced_cleanup)
 {
 	if (q->card->options.cq != QETH_CQ_ENABLED)
 		return;
@@ -475,8 +476,9 @@ static inline void qeth_cleanup_handled_pending(struct qeth_qdio_out_q *q,
 }
 
 
-static inline void qeth_qdio_handle_aob(struct qeth_card *card,
-		unsigned long phys_aob_addr) {
+static void qeth_qdio_handle_aob(struct qeth_card *card,
+				 unsigned long phys_aob_addr)
+{
 	struct qaob *aob;
 	struct qeth_qdio_out_buffer *buffer;
 	enum iucv_tx_notify notification;
@@ -2228,7 +2230,7 @@ static int qeth_cm_setup(struct qeth_card *card)
 
 }
 
-static inline int qeth_get_initial_mtu_for_card(struct qeth_card *card)
+static int qeth_get_initial_mtu_for_card(struct qeth_card *card)
 {
 	switch (card->info.type) {
 	case QETH_CARD_TYPE_UNKNOWN:
@@ -2251,7 +2253,7 @@ static inline int qeth_get_initial_mtu_for_card(struct qeth_card *card)
 	}
 }
 
-static inline int qeth_get_mtu_outof_framesize(int framesize)
+static int qeth_get_mtu_outof_framesize(int framesize)
 {
 	switch (framesize) {
 	case 0x4000:
@@ -2267,7 +2269,7 @@ static inline int qeth_get_mtu_outof_framesize(int framesize)
 	}
 }
 
-static inline int qeth_mtu_is_valid(struct qeth_card *card, int mtu)
+static int qeth_mtu_is_valid(struct qeth_card *card, int mtu)
 {
 	switch (card->info.type) {
 	case QETH_CARD_TYPE_OSD:
@@ -2738,8 +2740,8 @@ static void qeth_initialize_working_pool_list(struct qeth_card *card)
 	}
 }
 
-static inline struct qeth_buffer_pool_entry *qeth_find_free_buffer_pool_entry(
-		struct qeth_card *card)
+static struct qeth_buffer_pool_entry *qeth_find_free_buffer_pool_entry(
+					struct qeth_card *card)
 {
 	struct list_head *plh;
 	struct qeth_buffer_pool_entry *entry;
@@ -2870,7 +2872,7 @@ int qeth_init_qdio_queues(struct qeth_card *card)
 }
 EXPORT_SYMBOL_GPL(qeth_init_qdio_queues);
 
-static inline __u8 qeth_get_ipa_adp_type(enum qeth_link_types link_type)
+static __u8 qeth_get_ipa_adp_type(enum qeth_link_types link_type)
 {
 	switch (link_type) {
 	case QETH_LINK_TYPE_HSTR:
@@ -3888,27 +3890,45 @@ int qeth_hdr_chk_and_bounce(struct sk_buff *skb, struct qeth_hdr **hdr, int len)
 }
 EXPORT_SYMBOL_GPL(qeth_hdr_chk_and_bounce);
 
-static inline void __qeth_fill_buffer(struct sk_buff *skb,
-	struct qdio_buffer *buffer, int is_tso, int *next_element_to_fill,
-	int offset)
+/**
+ * qeth_push_hdr() - push a qeth_hdr onto an skb.
+ * @skb: skb that the qeth_hdr should be pushed onto.
+ * @hdr: double pointer to a qeth_hdr. When returning with >= 0,
+ *	 it contains a valid pointer to a qeth_hdr.
+ * @len: length of the hdr that needs to be pushed on.
+ *
+ * Returns the pushed length. If the header can't be pushed on
+ * (eg. because it would cross a page boundary), it is allocated from
+ * the cache instead and 0 is returned.
+ * Error to create the hdr is indicated by returning with < 0.
+ */
+int qeth_push_hdr(struct sk_buff *skb, struct qeth_hdr **hdr, unsigned int len)
 {
-	int length = skb_headlen(skb);
-	int length_here;
-	int element;
-	char *data;
-	int first_lap, cnt;
-	struct skb_frag_struct *frag;
-
-	element = *next_element_to_fill;
-	data = skb->data;
-	first_lap = (is_tso == 0 ? 1 : 0);
-
-	if (offset >= 0) {
-		data = skb->data + offset;
-		length -= offset;
-		first_lap = 0;
+	if (skb_headroom(skb) >= len &&
+	    qeth_get_elements_for_range((addr_t)skb->data - len,
+					(addr_t)skb->data) == 1) {
+		*hdr = skb_push(skb, len);
+		return len;
 	}
+	/* fall back */
+	*hdr = kmem_cache_alloc(qeth_core_header_cache, GFP_ATOMIC);
+	if (!*hdr)
+		return -ENOMEM;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(qeth_push_hdr);
+
+static void __qeth_fill_buffer(struct sk_buff *skb,
+			       struct qeth_qdio_out_buffer *buf,
+			       bool is_first_elem, unsigned int offset)
+{
+	struct qdio_buffer *buffer = buf->buffer;
+	int element = buf->next_element_to_fill;
+	int length = skb_headlen(skb) - offset;
+	char *data = skb->data + offset;
+	int length_here, cnt;
 
+	/* map linear part into buffer element(s) */
 	while (length > 0) {
 		/* length_here is the remaining amount of data in this page */
 		length_here = PAGE_SIZE - ((unsigned long) data % PAGE_SIZE);
@@ -3918,34 +3938,28 @@ static inline void __qeth_fill_buffer(struct sk_buff *skb,
 		buffer->element[element].addr = data;
 		buffer->element[element].length = length_here;
 		length -= length_here;
-		if (!length) {
-			if (first_lap)
-				if (skb_shinfo(skb)->nr_frags)
-					buffer->element[element].eflags =
-						SBAL_EFLAGS_FIRST_FRAG;
-				else
-					buffer->element[element].eflags = 0;
-			else
+		if (is_first_elem) {
+			is_first_elem = false;
+			if (length || skb_is_nonlinear(skb))
+				/* skb needs additional elements */
 				buffer->element[element].eflags =
-				    SBAL_EFLAGS_MIDDLE_FRAG;
-		} else {
-			if (first_lap)
-				buffer->element[element].eflags =
-				    SBAL_EFLAGS_FIRST_FRAG;
+					SBAL_EFLAGS_FIRST_FRAG;
 			else
-				buffer->element[element].eflags =
-				    SBAL_EFLAGS_MIDDLE_FRAG;
+				buffer->element[element].eflags = 0;
+		} else {
+			buffer->element[element].eflags =
+				SBAL_EFLAGS_MIDDLE_FRAG;
 		}
 		data += length_here;
 		element++;
-		first_lap = 0;
 	}
 
+	/* map page frags into buffer element(s) */
 	for (cnt = 0; cnt < skb_shinfo(skb)->nr_frags; cnt++) {
-		frag = &skb_shinfo(skb)->frags[cnt];
-		data = (char *)page_to_phys(skb_frag_page(frag)) +
-			frag->page_offset;
-		length = frag->size;
+		skb_frag_t *frag = &skb_shinfo(skb)->frags[cnt];
+
+		data = skb_frag_address(frag);
+		length = skb_frag_size(frag);
 		while (length > 0) {
 			length_here = PAGE_SIZE -
 				((unsigned long) data % PAGE_SIZE);
@@ -3964,48 +3978,45 @@ static inline void __qeth_fill_buffer(struct sk_buff *skb,
 
 	if (buffer->element[element - 1].eflags)
 		buffer->element[element - 1].eflags = SBAL_EFLAGS_LAST_FRAG;
-	*next_element_to_fill = element;
+	buf->next_element_to_fill = element;
 }
 
-static inline int qeth_fill_buffer(struct qeth_qdio_out_q *queue,
-		struct qeth_qdio_out_buffer *buf, struct sk_buff *skb,
-		struct qeth_hdr *hdr, int offset, int hd_len)
+/**
+ * qeth_fill_buffer() - map skb into an output buffer
+ * @queue:	QDIO queue to submit the buffer on
+ * @buf:	buffer to transport the skb
+ * @skb:	skb to map into the buffer
+ * @hdr:	qeth_hdr for this skb. Either at skb->data, or allocated
+ *		from qeth_core_header_cache.
+ * @offset:	when mapping the skb, start at skb->data + offset
+ * @hd_len:	if > 0, build a dedicated header element of this size
+ */
+static int qeth_fill_buffer(struct qeth_qdio_out_q *queue,
+			    struct qeth_qdio_out_buffer *buf,
+			    struct sk_buff *skb, struct qeth_hdr *hdr,
+			    unsigned int offset, unsigned int hd_len)
 {
-	struct qdio_buffer *buffer;
-	int flush_cnt = 0, hdr_len, large_send = 0;
+	struct qdio_buffer *buffer = buf->buffer;
+	bool is_first_elem = true;
+	int flush_cnt = 0;
 
-	buffer = buf->buffer;
 	refcount_inc(&skb->users);
 	skb_queue_tail(&buf->skb_list, skb);
 
-	/*check first on TSO ....*/
-	if (hdr->hdr.l3.id == QETH_HEADER_TYPE_TSO) {
+	/* build dedicated header element */
+	if (hd_len) {
 		int element = buf->next_element_to_fill;
+		is_first_elem = false;
 
-		hdr_len = sizeof(struct qeth_hdr_tso) +
-			((struct qeth_hdr_tso *)hdr)->ext.dg_hdr_len;
-		/*fill first buffer entry only with header information */
-		buffer->element[element].addr = skb->data;
-		buffer->element[element].length = hdr_len;
-		buffer->element[element].eflags = SBAL_EFLAGS_FIRST_FRAG;
-		buf->next_element_to_fill++;
-		skb->data += hdr_len;
-		skb->len  -= hdr_len;
-		large_send = 1;
-	}
-
-	if (offset >= 0) {
-		int element = buf->next_element_to_fill;
 		buffer->element[element].addr = hdr;
-		buffer->element[element].length = sizeof(struct qeth_hdr) +
-							hd_len;
+		buffer->element[element].length = hd_len;
 		buffer->element[element].eflags = SBAL_EFLAGS_FIRST_FRAG;
-		buf->is_header[element] = 1;
+		/* remember to free cache-allocated qeth_hdr: */
+		buf->is_header[element] = ((void *)hdr != skb->data);
 		buf->next_element_to_fill++;
 	}
 
-	__qeth_fill_buffer(skb, buffer, large_send,
-		(int *)&buf->next_element_to_fill, offset);
+	__qeth_fill_buffer(skb, buf, is_first_elem, offset);
 
 	if (!queue->do_pack) {
 		QETH_CARD_TEXT(queue->card, 6, "fillbfnp");
@@ -4030,8 +4041,9 @@ static inline int qeth_fill_buffer(struct qeth_qdio_out_q *queue,
 }
 
 int qeth_do_send_packet_fast(struct qeth_card *card,
-		struct qeth_qdio_out_q *queue, struct sk_buff *skb,
-		struct qeth_hdr *hdr, int offset, int hd_len)
+			     struct qeth_qdio_out_q *queue, struct sk_buff *skb,
+			     struct qeth_hdr *hdr, unsigned int offset,
+			     unsigned int hd_len)
 {
 	struct qeth_qdio_out_buffer *buffer;
 	int index;
@@ -4061,8 +4073,9 @@ out:
 EXPORT_SYMBOL_GPL(qeth_do_send_packet_fast);
 
 int qeth_do_send_packet(struct qeth_card *card, struct qeth_qdio_out_q *queue,
-		struct sk_buff *skb, struct qeth_hdr *hdr,
-		int elements_needed)
+			struct sk_buff *skb, struct qeth_hdr *hdr,
+			unsigned int offset, unsigned int hd_len,
+			int elements_needed)
 {
 	struct qeth_qdio_out_buffer *buffer;
 	int start_index;
@@ -4111,7 +4124,7 @@ int qeth_do_send_packet(struct qeth_card *card, struct qeth_qdio_out_q *queue,
 			}
 		}
 	}
-	tmp = qeth_fill_buffer(queue, buffer, skb, hdr, -1, 0);
+	tmp = qeth_fill_buffer(queue, buffer, skb, hdr, offset, hd_len);
 	queue->next_buf_to_fill = (queue->next_buf_to_fill + tmp) %
 				  QDIO_MAX_BUFFERS_PER_Q;
 	flush_count += tmp;
@@ -4834,7 +4847,7 @@ out:
 }
 EXPORT_SYMBOL_GPL(qeth_vm_request_mac);
 
-static inline int qeth_get_qdio_q_format(struct qeth_card *card)
+static int qeth_get_qdio_q_format(struct qeth_card *card)
 {
 	if (card->info.type == QETH_CARD_TYPE_IQD)
 		return QDIO_IQDIO_QFMT;
@@ -4899,9 +4912,12 @@ out:
 	return;
 }
 
-static inline void qeth_qdio_establish_cq(struct qeth_card *card,
-	struct qdio_buffer **in_sbal_ptrs,
-	void (**queue_start_poll) (struct ccw_device *, int, unsigned long)) {
+static void qeth_qdio_establish_cq(struct qeth_card *card,
+				   struct qdio_buffer **in_sbal_ptrs,
+				   void (**queue_start_poll)
+					(struct ccw_device *, int,
+					 unsigned long))
+{
 	int i;
 
 	if (card->options.cq == QETH_CQ_ENABLED) {
@@ -5193,9 +5209,10 @@ out:
 }
 EXPORT_SYMBOL_GPL(qeth_core_hardsetup_card);
 
-static inline int qeth_create_skb_frag(struct qeth_qdio_buffer *qethbuffer,
-		struct qdio_buffer_element *element,
-		struct sk_buff **pskb, int offset, int *pfrag, int data_len)
+static int qeth_create_skb_frag(struct qeth_qdio_buffer *qethbuffer,
+				struct qdio_buffer_element *element,
+				struct sk_buff **pskb, int offset, int *pfrag,
+				int data_len)
 {
 	struct page *page = virt_to_page(element->addr);
 	if (*pskb == NULL) {
diff --git a/drivers/s390/net/qeth_core_sys.c b/drivers/s390/net/qeth_core_sys.c
index 6d255c22656d..d1ee9e30c68b 100644
--- a/drivers/s390/net/qeth_core_sys.c
+++ b/drivers/s390/net/qeth_core_sys.c
@@ -78,7 +78,7 @@ static ssize_t qeth_dev_card_type_show(struct device *dev,
 
 static DEVICE_ATTR(card_type, 0444, qeth_dev_card_type_show, NULL);
 
-static inline const char *qeth_get_bufsize_str(struct qeth_card *card)
+static const char *qeth_get_bufsize_str(struct qeth_card *card)
 {
 	if (card->qdio.in_buf_size == 16384)
 		return "16k";
diff --git a/drivers/s390/net/qeth_l2_main.c b/drivers/s390/net/qeth_l2_main.c
index ad110abfdd47..760b023eae95 100644
--- a/drivers/s390/net/qeth_l2_main.c
+++ b/drivers/s390/net/qeth_l2_main.c
@@ -231,13 +231,7 @@ static void qeth_l2_del_all_macs(struct qeth_card *card)
 	spin_unlock_bh(&card->mclock);
 }
 
-static inline u32 qeth_l2_mac_hash(const u8 *addr)
-{
-	return get_unaligned((u32 *)(&addr[2]));
-}
-
-static inline int qeth_l2_get_cast_type(struct qeth_card *card,
-			struct sk_buff *skb)
+static int qeth_l2_get_cast_type(struct qeth_card *card, struct sk_buff *skb)
 {
 	if (card->info.type == QETH_CARD_TYPE_OSN)
 		return RTN_UNSPEC;
@@ -248,8 +242,8 @@ static inline int qeth_l2_get_cast_type(struct qeth_card *card,
 	return RTN_UNSPEC;
 }
 
-static inline void qeth_l2_hdr_csum(struct qeth_card *card,
-				    struct qeth_hdr *hdr, struct sk_buff *skb)
+static void qeth_l2_hdr_csum(struct qeth_card *card, struct qeth_hdr *hdr,
+			     struct sk_buff *skb)
 {
 	struct iphdr *iph = ip_hdr(skb);
 
@@ -265,13 +259,14 @@ static inline void qeth_l2_hdr_csum(struct qeth_card *card,
 		card->perf_stats.tx_csum++;
 }
 
-static void qeth_l2_fill_header(struct qeth_card *card, struct qeth_hdr *hdr,
-			struct sk_buff *skb, int cast_type)
+static void qeth_l2_fill_header(struct qeth_hdr *hdr, struct sk_buff *skb,
+				int cast_type, unsigned int data_len)
 {
 	struct vlan_ethhdr *veth = (struct vlan_ethhdr *)skb_mac_header(skb);
 
 	memset(hdr, 0, sizeof(struct qeth_hdr));
 	hdr->hdr.l2.id = QETH_HEADER_TYPE_LAYER2;
+	hdr->hdr.l2.pkt_length = data_len;
 
 	/* set byte byte 3 to casting flags */
 	if (cast_type == RTN_MULTICAST)
@@ -281,7 +276,6 @@ static void qeth_l2_fill_header(struct qeth_card *card, struct qeth_hdr *hdr,
 	else
 		hdr->hdr.l2.flags[2] |= QETH_LAYER2_FLAG_UNICAST;
 
-	hdr->hdr.l2.pkt_length = skb->len - sizeof(struct qeth_hdr);
 	/* VSWITCH relies on the VLAN
 	 * information to be present in
 	 * the QDIO header */
@@ -519,15 +513,6 @@ static int qeth_l2_request_initial_mac(struct qeth_card *card)
 		/* fall back to alternative mechanism: */
 	}
 
-	if (qeth_is_supported(card, IPA_SETADAPTERPARMS)) {
-		rc = qeth_query_setadapterparms(card);
-		if (rc) {
-			QETH_DBF_MESSAGE(2, "could not query adapter "
-				"parameters on device %s: x%x\n",
-				CARD_BUS_ID(card), rc);
-		}
-	}
-
 	if (card->info.type == QETH_CARD_TYPE_IQD ||
 	    card->info.type == QETH_CARD_TYPE_OSM ||
 	    card->info.type == QETH_CARD_TYPE_OSX ||
@@ -615,13 +600,13 @@ static void qeth_promisc_to_bridge(struct qeth_card *card)
  * only if there is not in the hash table storage already
  *
 */
-static	void
-qeth_l2_add_mac(struct qeth_card *card, struct netdev_hw_addr *ha, u8 is_uc)
+static void qeth_l2_add_mac(struct qeth_card *card, struct netdev_hw_addr *ha,
+			    u8 is_uc)
 {
+	u32 mac_hash = get_unaligned((u32 *)(&ha->addr[2]));
 	struct qeth_mac *mac;
 
-	hash_for_each_possible(card->mac_htable, mac, hnode,
-			qeth_l2_mac_hash(ha->addr)) {
+	hash_for_each_possible(card->mac_htable, mac, hnode, mac_hash) {
 		if (is_uc == mac->is_uc &&
 		    !memcmp(ha->addr, mac->mac_addr, OSA_ADDR_LEN)) {
 			mac->disp_flag = QETH_DISP_ADDR_DO_NOTHING;
@@ -638,9 +623,7 @@ qeth_l2_add_mac(struct qeth_card *card, struct netdev_hw_addr *ha, u8 is_uc)
 	mac->is_uc = is_uc;
 	mac->disp_flag = QETH_DISP_ADDR_ADD;
 
-	hash_add(card->mac_htable, &mac->hnode,
-			qeth_l2_mac_hash(mac->mac_addr));
-
+	hash_add(card->mac_htable, &mac->hnode, mac_hash);
 }
 
 static void qeth_l2_set_rx_mode(struct net_device *dev)
@@ -693,21 +676,127 @@ static void qeth_l2_set_rx_mode(struct net_device *dev)
 		qeth_promisc_to_bridge(card);
 }
 
-static netdev_tx_t qeth_l2_hard_start_xmit(struct sk_buff *skb,
-					   struct net_device *dev)
+static int qeth_l2_xmit_iqd(struct qeth_card *card, struct sk_buff *skb,
+			    struct qeth_qdio_out_q *queue, int cast_type)
 {
+	unsigned int data_offset = ETH_HLEN;
+	struct qeth_hdr *hdr;
 	int rc;
+
+	hdr = kmem_cache_alloc(qeth_core_header_cache, GFP_ATOMIC);
+	if (!hdr)
+		return -ENOMEM;
+	qeth_l2_fill_header(hdr, skb, cast_type, skb->len);
+	skb_copy_from_linear_data(skb, ((char *)hdr) + sizeof(*hdr),
+				  data_offset);
+
+	if (!qeth_get_elements_no(card, skb, 1, data_offset)) {
+		rc = -E2BIG;
+		goto out;
+	}
+	rc = qeth_do_send_packet_fast(card, queue, skb, hdr, data_offset,
+				      sizeof(*hdr) + data_offset);
+out:
+	if (rc)
+		kmem_cache_free(qeth_core_header_cache, hdr);
+	return rc;
+}
+
+static int qeth_l2_xmit_osa(struct qeth_card *card, struct sk_buff *skb,
+			    struct qeth_qdio_out_q *queue, int cast_type)
+{
+	int push_len = sizeof(struct qeth_hdr);
+	unsigned int elements, nr_frags;
+	unsigned int hdr_elements = 0;
 	struct qeth_hdr *hdr = NULL;
-	int elements = 0;
+	unsigned int hd_len = 0;
+	int rc;
+
+	/* fix hardware limitation: as long as we do not have sbal
+	 * chaining we can not send long frag lists
+	 */
+	if (!qeth_get_elements_no(card, skb, 0, 0)) {
+		rc = skb_linearize(skb);
+
+		if (card->options.performance_stats) {
+			if (rc)
+				card->perf_stats.tx_linfail++;
+			else
+				card->perf_stats.tx_lin++;
+		}
+		if (rc)
+			return rc;
+	}
+	nr_frags = skb_shinfo(skb)->nr_frags;
+
+	rc = skb_cow_head(skb, push_len);
+	if (rc)
+		return rc;
+	push_len = qeth_push_hdr(skb, &hdr, push_len);
+	if (push_len < 0)
+		return push_len;
+	if (!push_len) {
+		/* hdr was allocated from cache */
+		hd_len = sizeof(*hdr);
+		hdr_elements = 1;
+	}
+	qeth_l2_fill_header(hdr, skb, cast_type, skb->len - push_len);
+	if (skb->ip_summed == CHECKSUM_PARTIAL)
+		qeth_l2_hdr_csum(card, hdr, skb);
+
+	elements = qeth_get_elements_no(card, skb, hdr_elements, 0);
+	if (!elements) {
+		rc = -E2BIG;
+		goto out;
+	}
+	elements += hdr_elements;
+
+	/* TODO: remove the skb_orphan() once TX completion is fast enough */
+	skb_orphan(skb);
+	rc = qeth_do_send_packet(card, queue, skb, hdr, 0, hd_len, elements);
+out:
+	if (!rc) {
+		if (card->options.performance_stats && nr_frags) {
+			card->perf_stats.sg_skbs_sent++;
+			/* nr_frags + skb->data */
+			card->perf_stats.sg_frags_sent += nr_frags + 1;
+		}
+	} else {
+		if (hd_len)
+			kmem_cache_free(qeth_core_header_cache, hdr);
+		if (rc == -EBUSY)
+			/* roll back to ETH header */
+			skb_pull(skb, push_len);
+	}
+	return rc;
+}
+
+static int qeth_l2_xmit_osn(struct qeth_card *card, struct sk_buff *skb,
+			    struct qeth_qdio_out_q *queue)
+{
+	unsigned int elements;
+	struct qeth_hdr *hdr;
+
+	if (skb->protocol == htons(ETH_P_IPV6))
+		return -EPROTONOSUPPORT;
+
+	hdr = (struct qeth_hdr *)skb->data;
+	elements = qeth_get_elements_no(card, skb, 0, 0);
+	if (!elements)
+		return -E2BIG;
+	if (qeth_hdr_chk_and_bounce(skb, &hdr, sizeof(*hdr)))
+		return -EINVAL;
+	return qeth_do_send_packet(card, queue, skb, hdr, 0, 0, elements);
+}
+
+static netdev_tx_t qeth_l2_hard_start_xmit(struct sk_buff *skb,
+					   struct net_device *dev)
+{
 	struct qeth_card *card = dev->ml_priv;
-	struct sk_buff *new_skb = skb;
 	int cast_type = qeth_l2_get_cast_type(card, skb);
 	struct qeth_qdio_out_q *queue;
 	int tx_bytes = skb->len;
-	int data_offset = -1;
-	int elements_needed = 0;
-	int hd_len = 0;
-	int nr_frags;
+	int rc;
 
 	if (card->qdio.do_prio_queueing || (cast_type &&
 					card->info.is_multicast_different))
@@ -721,118 +810,38 @@ static netdev_tx_t qeth_l2_hard_start_xmit(struct sk_buff *skb,
 		goto tx_drop;
 	}
 
-	if ((card->info.type == QETH_CARD_TYPE_OSN) &&
-	    (skb->protocol == htons(ETH_P_IPV6)))
-		goto tx_drop;
-
 	if (card->options.performance_stats) {
 		card->perf_stats.outbound_cnt++;
 		card->perf_stats.outbound_start_time = qeth_get_micros();
 	}
 	netif_stop_queue(dev);
 
-	/* fix hardware limitation: as long as we do not have sbal
-	 * chaining we can not send long frag lists
-	 */
-	if ((card->info.type != QETH_CARD_TYPE_IQD) &&
-	    !qeth_get_elements_no(card, new_skb, 0, 0)) {
-		int lin_rc = skb_linearize(new_skb);
-
-		if (card->options.performance_stats) {
-			if (lin_rc)
-				card->perf_stats.tx_linfail++;
-			else
-				card->perf_stats.tx_lin++;
-		}
-		if (lin_rc)
-			goto tx_drop;
-	}
-
-	if (card->info.type == QETH_CARD_TYPE_OSN)
-		hdr = (struct qeth_hdr *)skb->data;
-	else {
-		if (card->info.type == QETH_CARD_TYPE_IQD) {
-			new_skb = skb;
-			data_offset = ETH_HLEN;
-			hd_len = ETH_HLEN;
-			hdr = kmem_cache_alloc(qeth_core_header_cache,
-						GFP_ATOMIC);
-			if (!hdr)
-				goto tx_drop;
-			elements_needed++;
-			skb_reset_mac_header(new_skb);
-			qeth_l2_fill_header(card, hdr, new_skb, cast_type);
-			hdr->hdr.l2.pkt_length = new_skb->len;
-			memcpy(((char *)hdr) + sizeof(struct qeth_hdr),
-				skb_mac_header(new_skb), ETH_HLEN);
-		} else {
-			/* create a clone with writeable headroom */
-			new_skb = skb_realloc_headroom(skb,
-						sizeof(struct qeth_hdr));
-			if (!new_skb)
-				goto tx_drop;
-			hdr = skb_push(new_skb, sizeof(struct qeth_hdr));
-			skb_set_mac_header(new_skb, sizeof(struct qeth_hdr));
-			qeth_l2_fill_header(card, hdr, new_skb, cast_type);
-			if (new_skb->ip_summed == CHECKSUM_PARTIAL)
-				qeth_l2_hdr_csum(card, hdr, new_skb);
-		}
-	}
-
-	elements = qeth_get_elements_no(card, new_skb, elements_needed,
-					(data_offset > 0) ? data_offset : 0);
-	if (!elements) {
-		if (data_offset >= 0)
-			kmem_cache_free(qeth_core_header_cache, hdr);
-		goto tx_drop;
+	switch (card->info.type) {
+	case QETH_CARD_TYPE_OSN:
+		rc = qeth_l2_xmit_osn(card, skb, queue);
+		break;
+	case QETH_CARD_TYPE_IQD:
+		rc = qeth_l2_xmit_iqd(card, skb, queue, cast_type);
+		break;
+	default:
+		rc = qeth_l2_xmit_osa(card, skb, queue, cast_type);
 	}
 
-	if (card->info.type != QETH_CARD_TYPE_IQD) {
-		if (qeth_hdr_chk_and_bounce(new_skb, &hdr,
-		    sizeof(struct qeth_hdr_layer2)))
-			goto tx_drop;
-		rc = qeth_do_send_packet(card, queue, new_skb, hdr,
-					 elements);
-	} else
-		rc = qeth_do_send_packet_fast(card, queue, new_skb, hdr,
-					      data_offset, hd_len);
 	if (!rc) {
 		card->stats.tx_packets++;
 		card->stats.tx_bytes += tx_bytes;
-		if (card->options.performance_stats) {
-			nr_frags = skb_shinfo(new_skb)->nr_frags;
-			if (nr_frags) {
-				card->perf_stats.sg_skbs_sent++;
-				/* nr_frags + skb->data */
-				card->perf_stats.sg_frags_sent += nr_frags + 1;
-			}
-		}
-		if (new_skb != skb)
-			dev_kfree_skb_any(skb);
-		rc = NETDEV_TX_OK;
-	} else {
-		if (data_offset >= 0)
-			kmem_cache_free(qeth_core_header_cache, hdr);
-
-		if (rc == -EBUSY) {
-			if (new_skb != skb)
-				dev_kfree_skb_any(new_skb);
-			return NETDEV_TX_BUSY;
-		} else
-			goto tx_drop;
-	}
-
-	netif_wake_queue(dev);
-	if (card->options.performance_stats)
-		card->perf_stats.outbound_time += qeth_get_micros() -
-			card->perf_stats.outbound_start_time;
-	return rc;
+		if (card->options.performance_stats)
+			card->perf_stats.outbound_time += qeth_get_micros() -
+				card->perf_stats.outbound_start_time;
+		netif_wake_queue(dev);
+		return NETDEV_TX_OK;
+	} else if (rc == -EBUSY) {
+		return NETDEV_TX_BUSY;
+	} /* else fall through */
 
 tx_drop:
 	card->stats.tx_dropped++;
 	card->stats.tx_errors++;
-	if ((new_skb != skb) && new_skb)
-		dev_kfree_skb_any(new_skb);
 	dev_kfree_skb_any(skb);
 	netif_wake_queue(dev);
 	return NETDEV_TX_OK;
@@ -1010,6 +1019,12 @@ static int qeth_l2_setup_netdev(struct qeth_card *card)
 			card->dev->vlan_features |= NETIF_F_RXCSUM;
 		}
 	}
+	if (card->info.type != QETH_CARD_TYPE_OSN &&
+	    card->info.type != QETH_CARD_TYPE_IQD) {
+		card->dev->priv_flags &= ~IFF_TX_SKB_SHARING;
+		card->dev->needed_headroom = sizeof(struct qeth_hdr);
+	}
+
 	card->info.broadcast_capable = 1;
 	qeth_l2_request_initial_mac(card);
 	card->dev->gso_max_size = (QETH_MAX_BUFFER_ELEMENTS(card) - 1) *
@@ -1744,11 +1759,26 @@ static int qeth_bridgeport_makerc(struct qeth_card *card,
 	return rc;
 }
 
-static inline int ipa_cmd_sbp(struct qeth_card *card)
+static struct qeth_cmd_buffer *qeth_sbp_build_cmd(struct qeth_card *card,
+						  enum qeth_ipa_sbp_cmd sbp_cmd,
+						  unsigned int cmd_length)
 {
-	return (card->info.type == QETH_CARD_TYPE_IQD) ?
-		IPA_CMD_SETBRIDGEPORT_IQD :
-		IPA_CMD_SETBRIDGEPORT_OSA;
+	enum qeth_ipa_cmds ipa_cmd = (card->info.type == QETH_CARD_TYPE_IQD) ?
+					IPA_CMD_SETBRIDGEPORT_IQD :
+					IPA_CMD_SETBRIDGEPORT_OSA;
+	struct qeth_cmd_buffer *iob;
+	struct qeth_ipa_cmd *cmd;
+
+	iob = qeth_get_ipacmd_buffer(card, ipa_cmd, 0);
+	if (!iob)
+		return iob;
+	cmd = (struct qeth_ipa_cmd *)(iob->data+IPA_PDU_HEADER_SIZE);
+	cmd->data.sbp.hdr.cmdlength = sizeof(struct qeth_ipacmd_sbp_hdr) +
+				      cmd_length;
+	cmd->data.sbp.hdr.command_code = sbp_cmd;
+	cmd->data.sbp.hdr.used_total = 1;
+	cmd->data.sbp.hdr.seq_no = 1;
+	return iob;
 }
 
 static int qeth_bridgeport_query_support_cb(struct qeth_card *card,
@@ -1778,21 +1808,13 @@ static int qeth_bridgeport_query_support_cb(struct qeth_card *card,
 static void qeth_bridgeport_query_support(struct qeth_card *card)
 {
 	struct qeth_cmd_buffer *iob;
-	struct qeth_ipa_cmd *cmd;
 	struct _qeth_sbp_cbctl cbctl;
 
 	QETH_CARD_TEXT(card, 2, "brqsuppo");
-	iob = qeth_get_ipacmd_buffer(card, ipa_cmd_sbp(card), 0);
+	iob = qeth_sbp_build_cmd(card, IPA_SBP_QUERY_COMMANDS_SUPPORTED,
+				 sizeof(struct qeth_sbp_query_cmds_supp));
 	if (!iob)
 		return;
-	cmd = (struct qeth_ipa_cmd *)(iob->data+IPA_PDU_HEADER_SIZE);
-	cmd->data.sbp.hdr.cmdlength =
-		sizeof(struct qeth_ipacmd_sbp_hdr) +
-		sizeof(struct qeth_sbp_query_cmds_supp);
-	cmd->data.sbp.hdr.command_code =
-		IPA_SBP_QUERY_COMMANDS_SUPPORTED;
-	cmd->data.sbp.hdr.used_total = 1;
-	cmd->data.sbp.hdr.seq_no = 1;
 	if (qeth_send_ipa_cmd(card, iob, qeth_bridgeport_query_support_cb,
 							(void *)&cbctl) ||
 	    qeth_bridgeport_makerc(card, &cbctl,
@@ -1846,7 +1868,6 @@ int qeth_bridgeport_query_ports(struct qeth_card *card,
 {
 	int rc = 0;
 	struct qeth_cmd_buffer *iob;
-	struct qeth_ipa_cmd *cmd;
 	struct _qeth_sbp_cbctl cbctl = {
 		.data = {
 			.qports = {
@@ -1859,16 +1880,9 @@ int qeth_bridgeport_query_ports(struct qeth_card *card,
 	QETH_CARD_TEXT(card, 2, "brqports");
 	if (!(card->options.sbp.supported_funcs & IPA_SBP_QUERY_BRIDGE_PORTS))
 		return -EOPNOTSUPP;
-	iob = qeth_get_ipacmd_buffer(card, ipa_cmd_sbp(card), 0);
+	iob = qeth_sbp_build_cmd(card, IPA_SBP_QUERY_BRIDGE_PORTS, 0);
 	if (!iob)
 		return -ENOMEM;
-	cmd = (struct qeth_ipa_cmd *)(iob->data+IPA_PDU_HEADER_SIZE);
-	cmd->data.sbp.hdr.cmdlength =
-		sizeof(struct qeth_ipacmd_sbp_hdr);
-	cmd->data.sbp.hdr.command_code =
-		IPA_SBP_QUERY_BRIDGE_PORTS;
-	cmd->data.sbp.hdr.used_total = 1;
-	cmd->data.sbp.hdr.seq_no = 1;
 	rc = qeth_send_ipa_cmd(card, iob, qeth_bridgeport_query_ports_cb,
 				(void *)&cbctl);
 	if (rc < 0)
@@ -1900,7 +1914,6 @@ int qeth_bridgeport_setrole(struct qeth_card *card, enum qeth_sbp_roles role)
 	int rc = 0;
 	int cmdlength;
 	struct qeth_cmd_buffer *iob;
-	struct qeth_ipa_cmd *cmd;
 	struct _qeth_sbp_cbctl cbctl;
 	enum qeth_ipa_sbp_cmd setcmd;
 
@@ -1908,32 +1921,24 @@ int qeth_bridgeport_setrole(struct qeth_card *card, enum qeth_sbp_roles role)
 	switch (role) {
 	case QETH_SBP_ROLE_NONE:
 		setcmd = IPA_SBP_RESET_BRIDGE_PORT_ROLE;
-		cmdlength =  sizeof(struct qeth_ipacmd_sbp_hdr) +
-			sizeof(struct qeth_sbp_reset_role);
+		cmdlength = sizeof(struct qeth_sbp_reset_role);
 		break;
 	case QETH_SBP_ROLE_PRIMARY:
 		setcmd = IPA_SBP_SET_PRIMARY_BRIDGE_PORT;
-		cmdlength =  sizeof(struct qeth_ipacmd_sbp_hdr) +
-			sizeof(struct qeth_sbp_set_primary);
+		cmdlength = sizeof(struct qeth_sbp_set_primary);
 		break;
 	case QETH_SBP_ROLE_SECONDARY:
 		setcmd = IPA_SBP_SET_SECONDARY_BRIDGE_PORT;
-		cmdlength =  sizeof(struct qeth_ipacmd_sbp_hdr) +
-			sizeof(struct qeth_sbp_set_secondary);
+		cmdlength = sizeof(struct qeth_sbp_set_secondary);
 		break;
 	default:
 		return -EINVAL;
 	}
 	if (!(card->options.sbp.supported_funcs & setcmd))
 		return -EOPNOTSUPP;
-	iob = qeth_get_ipacmd_buffer(card, ipa_cmd_sbp(card), 0);
+	iob = qeth_sbp_build_cmd(card, setcmd, cmdlength);
 	if (!iob)
 		return -ENOMEM;
-	cmd = (struct qeth_ipa_cmd *)(iob->data+IPA_PDU_HEADER_SIZE);
-	cmd->data.sbp.hdr.cmdlength = cmdlength;
-	cmd->data.sbp.hdr.command_code = setcmd;
-	cmd->data.sbp.hdr.used_total = 1;
-	cmd->data.sbp.hdr.seq_no = 1;
 	rc = qeth_send_ipa_cmd(card, iob, qeth_bridgeport_set_cb,
 				(void *)&cbctl);
 	if (rc < 0)
diff --git a/drivers/s390/net/qeth_l3_main.c b/drivers/s390/net/qeth_l3_main.c
index d42e758518ed..ab661a431f7c 100644
--- a/drivers/s390/net/qeth_l3_main.c
+++ b/drivers/s390/net/qeth_l3_main.c
@@ -247,7 +247,8 @@ int qeth_l3_delete_ip(struct qeth_card *card, struct qeth_ipaddr *tmp_addr)
 		return -ENOENT;
 
 	addr->ref_counter--;
-	if (addr->type == QETH_IP_TYPE_NORMAL && addr->ref_counter > 0)
+	if (addr->ref_counter > 0 && (addr->type == QETH_IP_TYPE_NORMAL ||
+				      addr->type == QETH_IP_TYPE_RXIP))
 		return rc;
 	if (addr->in_progress)
 		return -EINPROGRESS;
@@ -329,8 +330,9 @@ int qeth_l3_add_ip(struct qeth_card *card, struct qeth_ipaddr *tmp_addr)
 			kfree(addr);
 		}
 	} else {
-			if (addr->type == QETH_IP_TYPE_NORMAL)
-				addr->ref_counter++;
+		if (addr->type == QETH_IP_TYPE_NORMAL ||
+		    addr->type == QETH_IP_TYPE_RXIP)
+			addr->ref_counter++;
 	}
 
 	return rc;
@@ -784,11 +786,11 @@ void qeth_l3_del_rxip(struct qeth_card *card, enum qeth_prot_versions proto,
 	ipaddr = qeth_l3_get_addr_buffer(proto);
 	if (ipaddr) {
 		if (proto == QETH_PROT_IPV4) {
-			QETH_CARD_TEXT(card, 2, "addrxip4");
+			QETH_CARD_TEXT(card, 2, "delrxip4");
 			memcpy(&ipaddr->u.a4.addr, addr, 4);
 			ipaddr->u.a4.mask = 0;
 		} else if (proto == QETH_PROT_IPV6) {
-			QETH_CARD_TEXT(card, 2, "addrxip6");
+			QETH_CARD_TEXT(card, 2, "delrxip6");
 			memcpy(&ipaddr->u.a6.addr, addr, 16);
 			ipaddr->u.a6.pfxlen = 0;
 		}
@@ -867,7 +869,7 @@ static int qeth_l3_deregister_addr_entry(struct qeth_card *card,
 	return rc;
 }
 
-static inline u8 qeth_l3_get_qeth_hdr_flags4(int cast_type)
+static u8 qeth_l3_get_qeth_hdr_flags4(int cast_type)
 {
 	if (cast_type == RTN_MULTICAST)
 		return QETH_CAST_MULTICAST;
@@ -876,7 +878,7 @@ static inline u8 qeth_l3_get_qeth_hdr_flags4(int cast_type)
 	return QETH_CAST_UNICAST;
 }
 
-static inline u8 qeth_l3_get_qeth_hdr_flags6(int cast_type)
+static u8 qeth_l3_get_qeth_hdr_flags6(int cast_type)
 {
 	u8 ct = QETH_HDR_PASSTHRU | QETH_HDR_IPV6;
 	if (cast_type == RTN_MULTICAST)
@@ -890,22 +892,10 @@ static inline u8 qeth_l3_get_qeth_hdr_flags6(int cast_type)
 
 static int qeth_l3_setadapter_parms(struct qeth_card *card)
 {
-	int rc;
+	int rc = 0;
 
 	QETH_DBF_TEXT(SETUP, 2, "setadprm");
 
-	if (!qeth_is_supported(card, IPA_SETADAPTERPARMS)) {
-		dev_info(&card->gdev->dev,
-			"set adapter parameters not supported.\n");
-		QETH_DBF_TEXT(SETUP, 2, " notsupp");
-		return 0;
-	}
-	rc = qeth_query_setadapterparms(card);
-	if (rc) {
-		QETH_DBF_MESSAGE(2, "%s couldn't set adapter parameters: "
-			"0x%x\n", dev_name(&card->gdev->dev), rc);
-		return rc;
-	}
 	if (qeth_adp_supported(card, IPA_SETADP_ALTER_MAC_ADDRESS)) {
 		rc = qeth_setadpparms_change_macaddr(card);
 		if (rc)
@@ -1656,9 +1646,8 @@ static int qeth_l3_vlan_rx_kill_vid(struct net_device *dev,
 	return 0;
 }
 
-static inline int qeth_l3_rebuild_skb(struct qeth_card *card,
-			struct sk_buff *skb, struct qeth_hdr *hdr,
-			unsigned short *vlan_id)
+static int qeth_l3_rebuild_skb(struct qeth_card *card, struct sk_buff *skb,
+			       struct qeth_hdr *hdr, unsigned short *vlan_id)
 {
 	__u16 prot;
 	struct iphdr *ip_hdr;
@@ -2408,7 +2397,7 @@ static int qeth_l3_do_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
 	return rc;
 }
 
-inline int qeth_l3_get_cast_type(struct qeth_card *card, struct sk_buff *skb)
+static int qeth_l3_get_cast_type(struct qeth_card *card, struct sk_buff *skb)
 {
 	int cast_type = RTN_UNSPEC;
 	struct neighbour *n = NULL;
@@ -2546,8 +2535,8 @@ static void qeth_l3_fill_header(struct qeth_card *card, struct qeth_hdr *hdr,
 	rcu_read_unlock();
 }
 
-static inline void qeth_l3_hdr_csum(struct qeth_card *card,
-		struct qeth_hdr *hdr, struct sk_buff *skb)
+static void qeth_l3_hdr_csum(struct qeth_card *card, struct qeth_hdr *hdr,
+			     struct sk_buff *skb)
 {
 	struct iphdr *iph = ip_hdr(skb);
 
@@ -2582,7 +2571,7 @@ static void qeth_tso_fill_header(struct qeth_card *card,
 	hdr->ext.hdr_len     = 28;
 	/*insert non-fix values */
 	hdr->ext.mss = skb_shinfo(skb)->gso_size;
-	hdr->ext.dg_hdr_len = (__u16)(iph->ihl*4 + tcph->doff*4);
+	hdr->ext.dg_hdr_len = (__u16)(ip_hdrlen(skb) + tcp_hdrlen(skb));
 	hdr->ext.payload_len = (__u16)(skb->len - hdr->ext.dg_hdr_len -
 				       sizeof(struct qeth_hdr_tso));
 	tcph->check = 0;
@@ -2648,9 +2637,10 @@ static netdev_tx_t qeth_l3_hard_start_xmit(struct sk_buff *skb,
 			qeth_get_priority_queue(card, skb, ipv, cast_type) :
 			card->qdio.default_out_queue];
 	int tx_bytes = skb->len;
+	unsigned int hd_len = 0;
 	bool use_tso;
 	int data_offset = -1;
-	int nr_frags;
+	unsigned int nr_frags;
 
 	if (((card->info.type == QETH_CARD_TYPE_IQD) &&
 	     (((card->options.cq != QETH_CQ_ENABLED) && !ipv) ||
@@ -2675,11 +2665,12 @@ static netdev_tx_t qeth_l3_hard_start_xmit(struct sk_buff *skb,
 
 	/* Ignore segment size from skb_is_gso(), 1 page is always used. */
 	use_tso = skb_is_gso(skb) &&
-		  (qeth_get_ip_protocol(skb) == IPPROTO_TCP) && (ipv == 4);
+		  (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4);
 
 	if (card->info.type == QETH_CARD_TYPE_IQD) {
 		new_skb = skb;
 		data_offset = ETH_HLEN;
+		hd_len = sizeof(*hdr);
 		hdr = kmem_cache_alloc(qeth_core_header_cache, GFP_ATOMIC);
 		if (!hdr)
 			goto tx_drop;
@@ -2727,6 +2718,7 @@ static netdev_tx_t qeth_l3_hard_start_xmit(struct sk_buff *skb,
 		if (lin_rc)
 			goto tx_drop;
 	}
+	nr_frags = skb_shinfo(new_skb)->nr_frags;
 
 	if (use_tso) {
 		hdr = skb_push(new_skb, sizeof(struct qeth_hdr_tso));
@@ -2766,19 +2758,21 @@ static netdev_tx_t qeth_l3_hard_start_xmit(struct sk_buff *skb,
 
 	if (card->info.type != QETH_CARD_TYPE_IQD) {
 		int len;
-		if (use_tso)
-			len = ((unsigned long)tcp_hdr(new_skb) +
-				tcp_hdrlen(new_skb)) -
-				(unsigned long)new_skb->data;
-		else
+		if (use_tso) {
+			hd_len = sizeof(struct qeth_hdr_tso) +
+				 ip_hdrlen(new_skb) + tcp_hdrlen(new_skb);
+			len = hd_len;
+		} else {
 			len = sizeof(struct qeth_hdr_layer3);
+		}
 
 		if (qeth_hdr_chk_and_bounce(new_skb, &hdr, len))
 			goto tx_drop;
-		rc = qeth_do_send_packet(card, queue, new_skb, hdr, elements);
+		rc = qeth_do_send_packet(card, queue, new_skb, hdr, hd_len,
+					 hd_len, elements);
 	} else
 		rc = qeth_do_send_packet_fast(card, queue, new_skb, hdr,
-					      data_offset, 0);
+					      data_offset, hd_len);
 
 	if (!rc) {
 		card->stats.tx_packets++;
@@ -2786,7 +2780,6 @@ static netdev_tx_t qeth_l3_hard_start_xmit(struct sk_buff *skb,
 		if (new_skb != skb)
 			dev_kfree_skb_any(skb);
 		if (card->options.performance_stats) {
-			nr_frags = skb_shinfo(new_skb)->nr_frags;
 			if (use_tso) {
 				card->perf_stats.large_send_bytes += tx_bytes;
 				card->perf_stats.large_send_cnt++;
diff --git a/drivers/s390/net/qeth_l3_sys.c b/drivers/s390/net/qeth_l3_sys.c
index f2f94f59e0fa..e8bcc314cc5f 100644
--- a/drivers/s390/net/qeth_l3_sys.c
+++ b/drivers/s390/net/qeth_l3_sys.c
@@ -350,7 +350,7 @@ static struct attribute *qeth_l3_device_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group qeth_l3_device_attr_group = {
+static const struct attribute_group qeth_l3_device_attr_group = {
 	.attrs = qeth_l3_device_attrs,
 };
 
@@ -680,7 +680,7 @@ static struct attribute *qeth_ipato_device_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group qeth_device_ipato_group = {
+static const struct attribute_group qeth_device_ipato_group = {
 	.name = "ipa_takeover",
 	.attrs = qeth_ipato_device_attrs,
 };
@@ -843,7 +843,7 @@ static struct attribute *qeth_vipa_device_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group qeth_device_vipa_group = {
+static const struct attribute_group qeth_device_vipa_group = {
 	.name = "vipa",
 	.attrs = qeth_vipa_device_attrs,
 };
@@ -895,9 +895,26 @@ static ssize_t qeth_l3_dev_rxip_add4_show(struct device *dev,
 static int qeth_l3_parse_rxipe(const char *buf, enum qeth_prot_versions proto,
 		 u8 *addr)
 {
+	__be32 ipv4_addr;
+	struct in6_addr ipv6_addr;
+
 	if (qeth_l3_string_to_ipaddr(buf, proto, addr)) {
 		return -EINVAL;
 	}
+	if (proto == QETH_PROT_IPV4) {
+		memcpy(&ipv4_addr, addr, sizeof(ipv4_addr));
+		if (ipv4_is_multicast(ipv4_addr)) {
+			QETH_DBF_MESSAGE(2, "multicast rxip not supported.\n");
+			return -EINVAL;
+		}
+	} else if (proto == QETH_PROT_IPV6) {
+		memcpy(&ipv6_addr, addr, sizeof(ipv6_addr));
+		if (ipv6_addr_is_multicast(&ipv6_addr)) {
+			QETH_DBF_MESSAGE(2, "multicast rxip not supported.\n");
+			return -EINVAL;
+		}
+	}
+
 	return 0;
 }
 
@@ -1006,7 +1023,7 @@ static struct attribute *qeth_rxip_device_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group qeth_device_rxip_group = {
+static const struct attribute_group qeth_device_rxip_group = {
 	.name = "rxip",
 	.attrs = qeth_rxip_device_attrs,
 };
diff --git a/drivers/s390/scsi/zfcp_aux.c b/drivers/s390/scsi/zfcp_aux.c
index bcc8f3dfd4c4..82ac331d9125 100644
--- a/drivers/s390/scsi/zfcp_aux.c
+++ b/drivers/s390/scsi/zfcp_aux.c
@@ -29,7 +29,6 @@
 #define KMSG_COMPONENT "zfcp"
 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
 
-#include <linux/miscdevice.h>
 #include <linux/seq_file.h>
 #include <linux/slab.h>
 #include <linux/module.h>
diff --git a/drivers/s390/scsi/zfcp_dbf.c b/drivers/s390/scsi/zfcp_dbf.c
index d5bf36ec8a75..8227076c9cbb 100644
--- a/drivers/s390/scsi/zfcp_dbf.c
+++ b/drivers/s390/scsi/zfcp_dbf.c
@@ -3,7 +3,7 @@
  *
  * Debug traces for zfcp.
  *
- * Copyright IBM Corp. 2002, 2016
+ * Copyright IBM Corp. 2002, 2017
  */
 
 #define KMSG_COMPONENT "zfcp"
@@ -113,8 +113,12 @@ void zfcp_dbf_hba_fsf_uss(char *tag, struct zfcp_fsf_req *req)
 	struct zfcp_dbf *dbf = req->adapter->dbf;
 	struct fsf_status_read_buffer *srb = req->data;
 	struct zfcp_dbf_hba *rec = &dbf->hba_buf;
+	static int const level = 2;
 	unsigned long flags;
 
+	if (unlikely(!debug_level_enabled(dbf->hba, level)))
+		return;
+
 	spin_lock_irqsave(&dbf->hba_lock, flags);
 	memset(rec, 0, sizeof(*rec));
 
@@ -142,7 +146,7 @@ void zfcp_dbf_hba_fsf_uss(char *tag, struct zfcp_fsf_req *req)
 		zfcp_dbf_pl_write(dbf, srb->payload.data, rec->pl_len,
 				  "fsf_uss", req->req_id);
 log:
-	debug_event(dbf->hba, 2, rec, sizeof(*rec));
+	debug_event(dbf->hba, level, rec, sizeof(*rec));
 	spin_unlock_irqrestore(&dbf->hba_lock, flags);
 }
 
@@ -156,8 +160,12 @@ void zfcp_dbf_hba_bit_err(char *tag, struct zfcp_fsf_req *req)
 	struct zfcp_dbf *dbf = req->adapter->dbf;
 	struct zfcp_dbf_hba *rec = &dbf->hba_buf;
 	struct fsf_status_read_buffer *sr_buf = req->data;
+	static int const level = 1;
 	unsigned long flags;
 
+	if (unlikely(!debug_level_enabled(dbf->hba, level)))
+		return;
+
 	spin_lock_irqsave(&dbf->hba_lock, flags);
 	memset(rec, 0, sizeof(*rec));
 
@@ -169,7 +177,7 @@ void zfcp_dbf_hba_bit_err(char *tag, struct zfcp_fsf_req *req)
 	memcpy(&rec->u.be, &sr_buf->payload.bit_error,
 	       sizeof(struct fsf_bit_error_payload));
 
-	debug_event(dbf->hba, 1, rec, sizeof(*rec));
+	debug_event(dbf->hba, level, rec, sizeof(*rec));
 	spin_unlock_irqrestore(&dbf->hba_lock, flags);
 }
 
@@ -186,8 +194,12 @@ void zfcp_dbf_hba_def_err(struct zfcp_adapter *adapter, u64 req_id, u16 scount,
 	struct zfcp_dbf *dbf = adapter->dbf;
 	struct zfcp_dbf_pay *payload = &dbf->pay_buf;
 	unsigned long flags;
+	static int const level = 1;
 	u16 length;
 
+	if (unlikely(!debug_level_enabled(dbf->pay, level)))
+		return;
+
 	if (!pl)
 		return;
 
@@ -202,7 +214,7 @@ void zfcp_dbf_hba_def_err(struct zfcp_adapter *adapter, u64 req_id, u16 scount,
 
 	while (payload->counter < scount && (char *)pl[payload->counter]) {
 		memcpy(payload->data, (char *)pl[payload->counter], length);
-		debug_event(dbf->pay, 1, payload, zfcp_dbf_plen(length));
+		debug_event(dbf->pay, level, payload, zfcp_dbf_plen(length));
 		payload->counter++;
 	}
 
@@ -217,15 +229,19 @@ void zfcp_dbf_hba_basic(char *tag, struct zfcp_adapter *adapter)
 {
 	struct zfcp_dbf *dbf = adapter->dbf;
 	struct zfcp_dbf_hba *rec = &dbf->hba_buf;
+	static int const level = 1;
 	unsigned long flags;
 
+	if (unlikely(!debug_level_enabled(dbf->hba, level)))
+		return;
+
 	spin_lock_irqsave(&dbf->hba_lock, flags);
 	memset(rec, 0, sizeof(*rec));
 
 	memcpy(rec->tag, tag, ZFCP_DBF_TAG_LEN);
 	rec->id = ZFCP_DBF_HBA_BASIC;
 
-	debug_event(dbf->hba, 1, rec, sizeof(*rec));
+	debug_event(dbf->hba, level, rec, sizeof(*rec));
 	spin_unlock_irqrestore(&dbf->hba_lock, flags);
 }
 
@@ -264,9 +280,13 @@ void zfcp_dbf_rec_trig(char *tag, struct zfcp_adapter *adapter,
 {
 	struct zfcp_dbf *dbf = adapter->dbf;
 	struct zfcp_dbf_rec *rec = &dbf->rec_buf;
+	static int const level = 1;
 	struct list_head *entry;
 	unsigned long flags;
 
+	if (unlikely(!debug_level_enabled(dbf->rec, level)))
+		return;
+
 	spin_lock_irqsave(&dbf->rec_lock, flags);
 	memset(rec, 0, sizeof(*rec));
 
@@ -283,7 +303,7 @@ void zfcp_dbf_rec_trig(char *tag, struct zfcp_adapter *adapter,
 	rec->u.trig.want = want;
 	rec->u.trig.need = need;
 
-	debug_event(dbf->rec, 1, rec, sizeof(*rec));
+	debug_event(dbf->rec, level, rec, sizeof(*rec));
 	spin_unlock_irqrestore(&dbf->rec_lock, flags);
 }
 
@@ -300,6 +320,9 @@ void zfcp_dbf_rec_run_lvl(int level, char *tag, struct zfcp_erp_action *erp)
 	struct zfcp_dbf_rec *rec = &dbf->rec_buf;
 	unsigned long flags;
 
+	if (!debug_level_enabled(dbf->rec, level))
+		return;
+
 	spin_lock_irqsave(&dbf->rec_lock, flags);
 	memset(rec, 0, sizeof(*rec));
 
@@ -345,8 +368,12 @@ void zfcp_dbf_rec_run_wka(char *tag, struct zfcp_fc_wka_port *wka_port,
 {
 	struct zfcp_dbf *dbf = wka_port->adapter->dbf;
 	struct zfcp_dbf_rec *rec = &dbf->rec_buf;
+	static int const level = 1;
 	unsigned long flags;
 
+	if (unlikely(!debug_level_enabled(dbf->rec, level)))
+		return;
+
 	spin_lock_irqsave(&dbf->rec_lock, flags);
 	memset(rec, 0, sizeof(*rec));
 
@@ -362,10 +389,12 @@ void zfcp_dbf_rec_run_wka(char *tag, struct zfcp_fc_wka_port *wka_port,
 	rec->u.run.rec_action = ~0;
 	rec->u.run.rec_count = ~0;
 
-	debug_event(dbf->rec, 1, rec, sizeof(*rec));
+	debug_event(dbf->rec, level, rec, sizeof(*rec));
 	spin_unlock_irqrestore(&dbf->rec_lock, flags);
 }
 
+#define ZFCP_DBF_SAN_LEVEL 1
+
 static inline
 void zfcp_dbf_san(char *tag, struct zfcp_dbf *dbf,
 		  char *paytag, struct scatterlist *sg, u8 id, u16 len,
@@ -408,7 +437,7 @@ void zfcp_dbf_san(char *tag, struct zfcp_dbf *dbf,
 				      (u16)(sg->length - offset));
 			/* cap_len <= pay_sum < cap_len+ZFCP_DBF_PAY_MAX_REC */
 			memcpy(payload->data, sg_virt(sg) + offset, pay_len);
-			debug_event(dbf->pay, 1, payload,
+			debug_event(dbf->pay, ZFCP_DBF_SAN_LEVEL, payload,
 				    zfcp_dbf_plen(pay_len));
 			payload->counter++;
 			offset += pay_len;
@@ -418,7 +447,7 @@ void zfcp_dbf_san(char *tag, struct zfcp_dbf *dbf,
 	spin_unlock(&dbf->pay_lock);
 
 out:
-	debug_event(dbf->san, 1, rec, sizeof(*rec));
+	debug_event(dbf->san, ZFCP_DBF_SAN_LEVEL, rec, sizeof(*rec));
 	spin_unlock_irqrestore(&dbf->san_lock, flags);
 }
 
@@ -434,6 +463,9 @@ void zfcp_dbf_san_req(char *tag, struct zfcp_fsf_req *fsf, u32 d_id)
 	struct zfcp_fsf_ct_els *ct_els = fsf->data;
 	u16 length;
 
+	if (unlikely(!debug_level_enabled(dbf->san, ZFCP_DBF_SAN_LEVEL)))
+		return;
+
 	length = (u16)zfcp_qdio_real_bytes(ct_els->req);
 	zfcp_dbf_san(tag, dbf, "san_req", ct_els->req, ZFCP_DBF_SAN_REQ,
 		     length, fsf->req_id, d_id, length);
@@ -447,6 +479,7 @@ static u16 zfcp_dbf_san_res_cap_len_if_gpn_ft(char *tag,
 	struct fc_ct_hdr *reqh = sg_virt(ct_els->req);
 	struct fc_ns_gid_ft *reqn = (struct fc_ns_gid_ft *)(reqh + 1);
 	struct scatterlist *resp_entry = ct_els->resp;
+	struct fc_ct_hdr *resph;
 	struct fc_gpn_ft_resp *acc;
 	int max_entries, x, last = 0;
 
@@ -460,7 +493,7 @@ static u16 zfcp_dbf_san_res_cap_len_if_gpn_ft(char *tag,
 	      && reqh->ct_fs_subtype == FC_NS_SUBTYPE
 	      && reqh->ct_options == 0
 	      && reqh->_ct_resvd1 == 0
-	      && reqh->ct_cmd == FC_NS_GPN_FT
+	      && reqh->ct_cmd == cpu_to_be16(FC_NS_GPN_FT)
 	      /* reqh->ct_mr_size can vary so do not match but read below */
 	      && reqh->_ct_resvd2 == 0
 	      && reqh->ct_reason == 0
@@ -473,7 +506,15 @@ static u16 zfcp_dbf_san_res_cap_len_if_gpn_ft(char *tag,
 		return len; /* not GPN_FT response so do not cap */
 
 	acc = sg_virt(resp_entry);
-	max_entries = (reqh->ct_mr_size * 4 / sizeof(struct fc_gpn_ft_resp))
+
+	/* cap all but accept CT responses to at least the CT header */
+	resph = (struct fc_ct_hdr *)acc;
+	if ((ct_els->status) ||
+	    (resph->ct_cmd != cpu_to_be16(FC_FS_ACC)))
+		return max(FC_CT_HDR_LEN, ZFCP_DBF_SAN_MAX_PAYLOAD);
+
+	max_entries = (be16_to_cpu(reqh->ct_mr_size) * 4 /
+		       sizeof(struct fc_gpn_ft_resp))
 		+ 1 /* zfcp_fc_scan_ports: bytes correct, entries off-by-one
 		     * to account for header as 1st pseudo "entry" */;
 
@@ -503,6 +544,9 @@ void zfcp_dbf_san_res(char *tag, struct zfcp_fsf_req *fsf)
 	struct zfcp_fsf_ct_els *ct_els = fsf->data;
 	u16 length;
 
+	if (unlikely(!debug_level_enabled(dbf->san, ZFCP_DBF_SAN_LEVEL)))
+		return;
+
 	length = (u16)zfcp_qdio_real_bytes(ct_els->resp);
 	zfcp_dbf_san(tag, dbf, "san_res", ct_els->resp, ZFCP_DBF_SAN_RES,
 		     length, fsf->req_id, ct_els->d_id,
@@ -522,6 +566,9 @@ void zfcp_dbf_san_in_els(char *tag, struct zfcp_fsf_req *fsf)
 	u16 length;
 	struct scatterlist sg;
 
+	if (unlikely(!debug_level_enabled(dbf->san, ZFCP_DBF_SAN_LEVEL)))
+		return;
+
 	length = (u16)(srb->length -
 			offsetof(struct fsf_status_read_buffer, payload));
 	sg_init_one(&sg, srb->payload.data, length);
@@ -555,8 +602,8 @@ void zfcp_dbf_scsi(char *tag, int level, struct scsi_cmnd *sc,
 	rec->scsi_retries = sc->retries;
 	rec->scsi_allowed = sc->allowed;
 	rec->scsi_id = sc->device->id;
-	/* struct zfcp_dbf_scsi needs to be updated to handle 64bit LUNs */
 	rec->scsi_lun = (u32)sc->device->lun;
+	rec->scsi_lun_64_hi = (u32)(sc->device->lun >> 32);
 	rec->host_scribble = (unsigned long)sc->host_scribble;
 
 	memcpy(rec->scsi_opcode, sc->cmnd,
@@ -564,19 +611,31 @@ void zfcp_dbf_scsi(char *tag, int level, struct scsi_cmnd *sc,
 
 	if (fsf) {
 		rec->fsf_req_id = fsf->req_id;
-		fcp_rsp = (struct fcp_resp_with_ext *)
-				&(fsf->qtcb->bottom.io.fcp_rsp);
+		rec->pl_len = FCP_RESP_WITH_EXT;
+		fcp_rsp = &(fsf->qtcb->bottom.io.fcp_rsp.iu);
+		/* mandatory parts of FCP_RSP IU in this SCSI record */
 		memcpy(&rec->fcp_rsp, fcp_rsp, FCP_RESP_WITH_EXT);
 		if (fcp_rsp->resp.fr_flags & FCP_RSP_LEN_VAL) {
 			fcp_rsp_info = (struct fcp_resp_rsp_info *) &fcp_rsp[1];
 			rec->fcp_rsp_info = fcp_rsp_info->rsp_code;
+			rec->pl_len += be32_to_cpu(fcp_rsp->ext.fr_rsp_len);
 		}
 		if (fcp_rsp->resp.fr_flags & FCP_SNS_LEN_VAL) {
-			rec->pl_len = min((u16)SCSI_SENSE_BUFFERSIZE,
-					  (u16)ZFCP_DBF_PAY_MAX_REC);
-			zfcp_dbf_pl_write(dbf, sc->sense_buffer, rec->pl_len,
-					  "fcp_sns", fsf->req_id);
+			rec->pl_len += be32_to_cpu(fcp_rsp->ext.fr_sns_len);
 		}
+		/* complete FCP_RSP IU in associated PAYload record
+		 * but only if there are optional parts
+		 */
+		if (fcp_rsp->resp.fr_flags != 0)
+			zfcp_dbf_pl_write(
+				dbf, fcp_rsp,
+				/* at least one full PAY record
+				 * but not beyond hardware response field
+				 */
+				min_t(u16, max_t(u16, rec->pl_len,
+						 ZFCP_DBF_PAY_MAX_REC),
+				      FSF_FCP_RSP_SIZE),
+				"fcp_riu", fsf->req_id);
 	}
 
 	debug_event(dbf->scsi, level, rec, sizeof(*rec));
diff --git a/drivers/s390/scsi/zfcp_dbf.h b/drivers/s390/scsi/zfcp_dbf.h
index db186d44cfaf..3508c00458f4 100644
--- a/drivers/s390/scsi/zfcp_dbf.h
+++ b/drivers/s390/scsi/zfcp_dbf.h
@@ -2,7 +2,7 @@
  * zfcp device driver
  * debug feature declarations
  *
- * Copyright IBM Corp. 2008, 2016
+ * Copyright IBM Corp. 2008, 2017
  */
 
 #ifndef ZFCP_DBF_H
@@ -204,16 +204,17 @@ enum zfcp_dbf_scsi_id {
  * @id: unique number of recovery record type
  * @tag: identifier string specifying the location of initiation
  * @scsi_id: scsi device id
- * @scsi_lun: scsi device logical unit number
+ * @scsi_lun: scsi device logical unit number, low part of 64 bit, old 32 bit
  * @scsi_result: scsi result
  * @scsi_retries: current retry number of scsi request
  * @scsi_allowed: allowed retries
- * @fcp_rsp_info: FCP response info
+ * @fcp_rsp_info: FCP response info code
  * @scsi_opcode: scsi opcode
  * @fsf_req_id: request id of fsf request
  * @host_scribble: LLD specific data attached to SCSI request
- * @pl_len: length of paload stored as zfcp_dbf_pay
- * @fsf_rsp: response for fsf request
+ * @pl_len: length of payload stored as zfcp_dbf_pay
+ * @fcp_rsp: response for FCP request
+ * @scsi_lun_64_hi: scsi device logical unit number, high part of 64 bit
  */
 struct zfcp_dbf_scsi {
 	u8 id;
@@ -230,6 +231,7 @@ struct zfcp_dbf_scsi {
 	u64 host_scribble;
 	u16 pl_len;
 	struct fcp_resp_with_ext fcp_rsp;
+	u32 scsi_lun_64_hi;
 } __packed;
 
 /**
@@ -299,7 +301,7 @@ bool zfcp_dbf_hba_fsf_resp_suppress(struct zfcp_fsf_req *req)
 
 	if (qtcb->prefix.qtcb_type != FSF_IO_COMMAND)
 		return false; /* not an FCP response */
-	fcp_rsp = (struct fcp_resp *)&qtcb->bottom.io.fcp_rsp;
+	fcp_rsp = &qtcb->bottom.io.fcp_rsp.iu.resp;
 	rsp_flags = fcp_rsp->fr_flags;
 	fr_status = fcp_rsp->fr_status;
 	return (fsf_stat == FSF_FCP_RSP_AVAILABLE) &&
@@ -323,7 +325,11 @@ void zfcp_dbf_hba_fsf_response(struct zfcp_fsf_req *req)
 {
 	struct fsf_qtcb *qtcb = req->qtcb;
 
-	if ((qtcb->prefix.prot_status != FSF_PROT_GOOD) &&
+	if (unlikely(req->status & (ZFCP_STATUS_FSFREQ_DISMISSED |
+				    ZFCP_STATUS_FSFREQ_ERROR))) {
+		zfcp_dbf_hba_fsf_resp("fs_rerr", 3, req);
+
+	} else if ((qtcb->prefix.prot_status != FSF_PROT_GOOD) &&
 	    (qtcb->prefix.prot_status != FSF_PROT_FSF_STATUS_PRESENTED)) {
 		zfcp_dbf_hba_fsf_resp("fs_perr", 1, req);
 
@@ -401,7 +407,8 @@ void zfcp_dbf_scsi_abort(char *tag, struct scsi_cmnd *scmd,
  * @flag: indicates type of reset (Target Reset, Logical Unit Reset)
  */
 static inline
-void zfcp_dbf_scsi_devreset(char *tag, struct scsi_cmnd *scmnd, u8 flag)
+void zfcp_dbf_scsi_devreset(char *tag, struct scsi_cmnd *scmnd, u8 flag,
+			    struct zfcp_fsf_req *fsf_req)
 {
 	char tmp_tag[ZFCP_DBF_TAG_LEN];
 
@@ -411,7 +418,7 @@ void zfcp_dbf_scsi_devreset(char *tag, struct scsi_cmnd *scmnd, u8 flag)
 		memcpy(tmp_tag, "lr_", 3);
 
 	memcpy(&tmp_tag[3], tag, 4);
-	_zfcp_dbf_scsi(tmp_tag, 1, scmnd, NULL);
+	_zfcp_dbf_scsi(tmp_tag, 1, scmnd, fsf_req);
 }
 
 /**
diff --git a/drivers/s390/scsi/zfcp_erp.c b/drivers/s390/scsi/zfcp_erp.c
index 7ccfce559034..37408f5f81ce 100644
--- a/drivers/s390/scsi/zfcp_erp.c
+++ b/drivers/s390/scsi/zfcp_erp.c
@@ -572,9 +572,8 @@ static void zfcp_erp_memwait_handler(unsigned long data)
 
 static void zfcp_erp_strategy_memwait(struct zfcp_erp_action *erp_action)
 {
-	init_timer(&erp_action->timer);
-	erp_action->timer.function = zfcp_erp_memwait_handler;
-	erp_action->timer.data = (unsigned long) erp_action;
+	setup_timer(&erp_action->timer, zfcp_erp_memwait_handler,
+		    (unsigned long) erp_action);
 	erp_action->timer.expires = jiffies + HZ;
 	add_timer(&erp_action->timer);
 }
diff --git a/drivers/s390/scsi/zfcp_ext.h b/drivers/s390/scsi/zfcp_ext.h
index 9afdbc32b23f..a9e968717dd9 100644
--- a/drivers/s390/scsi/zfcp_ext.h
+++ b/drivers/s390/scsi/zfcp_ext.h
@@ -41,7 +41,6 @@ extern void zfcp_dbf_rec_run_wka(char *, struct zfcp_fc_wka_port *, u64);
 extern void zfcp_dbf_hba_fsf_uss(char *, struct zfcp_fsf_req *);
 extern void zfcp_dbf_hba_fsf_res(char *, int, struct zfcp_fsf_req *);
 extern void zfcp_dbf_hba_bit_err(char *, struct zfcp_fsf_req *);
-extern void zfcp_dbf_hba_berr(struct zfcp_dbf *, struct zfcp_fsf_req *);
 extern void zfcp_dbf_hba_def_err(struct zfcp_adapter *, u64, u16, void **);
 extern void zfcp_dbf_hba_basic(char *, struct zfcp_adapter *);
 extern void zfcp_dbf_san_req(char *, struct zfcp_fsf_req *, u32);
diff --git a/drivers/s390/scsi/zfcp_fc.c b/drivers/s390/scsi/zfcp_fc.c
index 7331eea67435..8210645c2111 100644
--- a/drivers/s390/scsi/zfcp_fc.c
+++ b/drivers/s390/scsi/zfcp_fc.c
@@ -3,7 +3,7 @@
  *
  * Fibre Channel related functions for the zfcp device driver.
  *
- * Copyright IBM Corp. 2008, 2010
+ * Copyright IBM Corp. 2008, 2017
  */
 
 #define KMSG_COMPONENT "zfcp"
@@ -29,7 +29,7 @@ static u32 zfcp_fc_rscn_range_mask[] = {
 };
 
 static bool no_auto_port_rescan;
-module_param_named(no_auto_port_rescan, no_auto_port_rescan, bool, 0600);
+module_param(no_auto_port_rescan, bool, 0600);
 MODULE_PARM_DESC(no_auto_port_rescan,
 		 "no automatic port_rescan (default off)");
 
@@ -260,7 +260,8 @@ static void zfcp_fc_incoming_rscn(struct zfcp_fsf_req *fsf_req)
 	page = (struct fc_els_rscn_page *) head;
 
 	/* see FC-FS */
-	no_entries = head->rscn_plen / sizeof(struct fc_els_rscn_page);
+	no_entries = be16_to_cpu(head->rscn_plen) /
+		sizeof(struct fc_els_rscn_page);
 
 	for (i = 1; i < no_entries; i++) {
 		/* skip head and start with 1st element */
@@ -296,7 +297,7 @@ static void zfcp_fc_incoming_plogi(struct zfcp_fsf_req *req)
 
 	status_buffer = (struct fsf_status_read_buffer *) req->data;
 	plogi = (struct fc_els_flogi *) status_buffer->payload.data;
-	zfcp_fc_incoming_wwpn(req, plogi->fl_wwpn);
+	zfcp_fc_incoming_wwpn(req, be64_to_cpu(plogi->fl_wwpn));
 }
 
 static void zfcp_fc_incoming_logo(struct zfcp_fsf_req *req)
@@ -306,7 +307,7 @@ static void zfcp_fc_incoming_logo(struct zfcp_fsf_req *req)
 	struct fc_els_logo *logo =
 		(struct fc_els_logo *) status_buffer->payload.data;
 
-	zfcp_fc_incoming_wwpn(req, logo->fl_n_port_wwn);
+	zfcp_fc_incoming_wwpn(req, be64_to_cpu(logo->fl_n_port_wwn));
 }
 
 /**
@@ -335,7 +336,7 @@ static void zfcp_fc_ns_gid_pn_eval(struct zfcp_fc_req *fc_req)
 
 	if (ct_els->status)
 		return;
-	if (gid_pn_rsp->ct_hdr.ct_cmd != FC_FS_ACC)
+	if (gid_pn_rsp->ct_hdr.ct_cmd != cpu_to_be16(FC_FS_ACC))
 		return;
 
 	/* looks like a valid d_id */
@@ -352,8 +353,8 @@ static void zfcp_fc_ct_ns_init(struct fc_ct_hdr *ct_hdr, u16 cmd, u16 mr_size)
 	ct_hdr->ct_rev = FC_CT_REV;
 	ct_hdr->ct_fs_type = FC_FST_DIR;
 	ct_hdr->ct_fs_subtype = FC_NS_SUBTYPE;
-	ct_hdr->ct_cmd = cmd;
-	ct_hdr->ct_mr_size = mr_size / 4;
+	ct_hdr->ct_cmd = cpu_to_be16(cmd);
+	ct_hdr->ct_mr_size = cpu_to_be16(mr_size / 4);
 }
 
 static int zfcp_fc_ns_gid_pn_request(struct zfcp_port *port,
@@ -376,7 +377,7 @@ static int zfcp_fc_ns_gid_pn_request(struct zfcp_port *port,
 
 	zfcp_fc_ct_ns_init(&gid_pn_req->ct_hdr,
 			   FC_NS_GID_PN, ZFCP_FC_CT_SIZE_PAGE);
-	gid_pn_req->gid_pn.fn_wwpn = port->wwpn;
+	gid_pn_req->gid_pn.fn_wwpn = cpu_to_be64(port->wwpn);
 
 	ret = zfcp_fsf_send_ct(&adapter->gs->ds, &fc_req->ct_els,
 			       adapter->pool.gid_pn_req,
@@ -460,26 +461,26 @@ void zfcp_fc_trigger_did_lookup(struct zfcp_port *port)
  */
 void zfcp_fc_plogi_evaluate(struct zfcp_port *port, struct fc_els_flogi *plogi)
 {
-	if (plogi->fl_wwpn != port->wwpn) {
+	if (be64_to_cpu(plogi->fl_wwpn) != port->wwpn) {
 		port->d_id = 0;
 		dev_warn(&port->adapter->ccw_device->dev,
 			 "A port opened with WWPN 0x%016Lx returned data that "
 			 "identifies it as WWPN 0x%016Lx\n",
 			 (unsigned long long) port->wwpn,
-			 (unsigned long long) plogi->fl_wwpn);
+			 (unsigned long long) be64_to_cpu(plogi->fl_wwpn));
 		return;
 	}
 
-	port->wwnn = plogi->fl_wwnn;
-	port->maxframe_size = plogi->fl_csp.sp_bb_data;
+	port->wwnn = be64_to_cpu(plogi->fl_wwnn);
+	port->maxframe_size = be16_to_cpu(plogi->fl_csp.sp_bb_data);
 
-	if (plogi->fl_cssp[0].cp_class & FC_CPC_VALID)
+	if (plogi->fl_cssp[0].cp_class & cpu_to_be16(FC_CPC_VALID))
 		port->supported_classes |= FC_COS_CLASS1;
-	if (plogi->fl_cssp[1].cp_class & FC_CPC_VALID)
+	if (plogi->fl_cssp[1].cp_class & cpu_to_be16(FC_CPC_VALID))
 		port->supported_classes |= FC_COS_CLASS2;
-	if (plogi->fl_cssp[2].cp_class & FC_CPC_VALID)
+	if (plogi->fl_cssp[2].cp_class & cpu_to_be16(FC_CPC_VALID))
 		port->supported_classes |= FC_COS_CLASS3;
-	if (plogi->fl_cssp[3].cp_class & FC_CPC_VALID)
+	if (plogi->fl_cssp[3].cp_class & cpu_to_be16(FC_CPC_VALID))
 		port->supported_classes |= FC_COS_CLASS4;
 }
 
@@ -497,9 +498,9 @@ static void zfcp_fc_adisc_handler(void *data)
 	}
 
 	if (!port->wwnn)
-		port->wwnn = adisc_resp->adisc_wwnn;
+		port->wwnn = be64_to_cpu(adisc_resp->adisc_wwnn);
 
-	if ((port->wwpn != adisc_resp->adisc_wwpn) ||
+	if ((port->wwpn != be64_to_cpu(adisc_resp->adisc_wwpn)) ||
 	    !(atomic_read(&port->status) & ZFCP_STATUS_COMMON_OPEN)) {
 		zfcp_erp_port_reopen(port, ZFCP_STATUS_COMMON_ERP_FAILED,
 				     "fcadh_2");
@@ -538,8 +539,8 @@ static int zfcp_fc_adisc(struct zfcp_port *port)
 
 	/* acc. to FC-FS, hard_nport_id in ADISC should not be set for ports
 	   without FC-AL-2 capability, so we don't set it */
-	fc_req->u.adisc.req.adisc_wwpn = fc_host_port_name(shost);
-	fc_req->u.adisc.req.adisc_wwnn = fc_host_node_name(shost);
+	fc_req->u.adisc.req.adisc_wwpn = cpu_to_be64(fc_host_port_name(shost));
+	fc_req->u.adisc.req.adisc_wwnn = cpu_to_be64(fc_host_node_name(shost));
 	fc_req->u.adisc.req.adisc_cmd = ELS_ADISC;
 	hton24(fc_req->u.adisc.req.adisc_port_id, fc_host_port_id(shost));
 
@@ -666,8 +667,8 @@ static int zfcp_fc_eval_gpn_ft(struct zfcp_fc_req *fc_req,
 	if (ct_els->status)
 		return -EIO;
 
-	if (hdr->ct_cmd != FC_FS_ACC) {
-		if (hdr->ct_reason == FC_BA_RJT_UNABLE)
+	if (hdr->ct_cmd != cpu_to_be16(FC_FS_ACC)) {
+		if (hdr->ct_reason == FC_FS_RJT_UNABL)
 			return -EAGAIN; /* might be a temporary condition */
 		return -EIO;
 	}
@@ -693,10 +694,11 @@ static int zfcp_fc_eval_gpn_ft(struct zfcp_fc_req *fc_req,
 		if (d_id >= FC_FID_WELL_KNOWN_BASE)
 			continue;
 		/* skip the adapter's port and known remote ports */
-		if (acc->fp_wwpn == fc_host_port_name(adapter->scsi_host))
+		if (be64_to_cpu(acc->fp_wwpn) ==
+		    fc_host_port_name(adapter->scsi_host))
 			continue;
 
-		port = zfcp_port_enqueue(adapter, acc->fp_wwpn,
+		port = zfcp_port_enqueue(adapter, be64_to_cpu(acc->fp_wwpn),
 					 ZFCP_STATUS_COMMON_NOESC, d_id);
 		if (!IS_ERR(port))
 			zfcp_erp_port_reopen(port, 0, "fcegpf1");
diff --git a/drivers/s390/scsi/zfcp_fc.h b/drivers/s390/scsi/zfcp_fc.h
index df2b541c8287..41f22d3dc6d1 100644
--- a/drivers/s390/scsi/zfcp_fc.h
+++ b/drivers/s390/scsi/zfcp_fc.h
@@ -4,7 +4,7 @@
  * Fibre Channel related definitions and inline functions for the zfcp
  * device driver
  *
- * Copyright IBM Corp. 2009
+ * Copyright IBM Corp. 2009, 2017
  */
 
 #ifndef ZFCP_FC_H
@@ -212,6 +212,8 @@ static inline
 void zfcp_fc_scsi_to_fcp(struct fcp_cmnd *fcp, struct scsi_cmnd *scsi,
 			 u8 tm_flags)
 {
+	u32 datalen;
+
 	int_to_scsilun(scsi->device->lun, (struct scsi_lun *) &fcp->fc_lun);
 
 	if (unlikely(tm_flags)) {
@@ -228,10 +230,13 @@ void zfcp_fc_scsi_to_fcp(struct fcp_cmnd *fcp, struct scsi_cmnd *scsi,
 
 	memcpy(fcp->fc_cdb, scsi->cmnd, scsi->cmd_len);
 
-	fcp->fc_dl = scsi_bufflen(scsi);
+	datalen = scsi_bufflen(scsi);
+	fcp->fc_dl = cpu_to_be32(datalen);
 
-	if (scsi_get_prot_type(scsi) == SCSI_PROT_DIF_TYPE1)
-		fcp->fc_dl += fcp->fc_dl / scsi->device->sector_size * 8;
+	if (scsi_get_prot_type(scsi) == SCSI_PROT_DIF_TYPE1) {
+		datalen += datalen / scsi->device->sector_size * 8;
+		fcp->fc_dl = cpu_to_be32(datalen);
+	}
 }
 
 /**
@@ -266,19 +271,23 @@ void zfcp_fc_eval_fcp_rsp(struct fcp_resp_with_ext *fcp_rsp,
 	if (unlikely(rsp_flags & FCP_SNS_LEN_VAL)) {
 		sense = (char *) &fcp_rsp[1];
 		if (rsp_flags & FCP_RSP_LEN_VAL)
-			sense += fcp_rsp->ext.fr_rsp_len;
-		sense_len = min(fcp_rsp->ext.fr_sns_len,
-				(u32) SCSI_SENSE_BUFFERSIZE);
+			sense += be32_to_cpu(fcp_rsp->ext.fr_rsp_len);
+		sense_len = min_t(u32, be32_to_cpu(fcp_rsp->ext.fr_sns_len),
+				  SCSI_SENSE_BUFFERSIZE);
 		memcpy(scsi->sense_buffer, sense, sense_len);
 	}
 
 	if (unlikely(rsp_flags & FCP_RESID_UNDER)) {
-		resid = fcp_rsp->ext.fr_resid;
+		resid = be32_to_cpu(fcp_rsp->ext.fr_resid);
 		scsi_set_resid(scsi, resid);
 		if (scsi_bufflen(scsi) - resid < scsi->underflow &&
 		     !(rsp_flags & FCP_SNS_LEN_VAL) &&
 		     fcp_rsp->resp.fr_status == SAM_STAT_GOOD)
 			set_host_byte(scsi, DID_ERROR);
+	} else if (unlikely(rsp_flags & FCP_RESID_OVER)) {
+		/* FCP_DL was not sufficient for SCSI data length */
+		if (fcp_rsp->resp.fr_status == SAM_STAT_GOOD)
+			set_host_byte(scsi, DID_ERROR);
 	}
 }
 
diff --git a/drivers/s390/scsi/zfcp_fsf.c b/drivers/s390/scsi/zfcp_fsf.c
index 27ff38f839fc..69d1dc3ec79d 100644
--- a/drivers/s390/scsi/zfcp_fsf.c
+++ b/drivers/s390/scsi/zfcp_fsf.c
@@ -3,7 +3,7 @@
  *
  * Implementation of FSF commands.
  *
- * Copyright IBM Corp. 2002, 2015
+ * Copyright IBM Corp. 2002, 2017
  */
 
 #define KMSG_COMPONENT "zfcp"
@@ -197,8 +197,6 @@ static void zfcp_fsf_status_read_link_down(struct zfcp_fsf_req *req)
 
 	switch (sr_buf->status_subtype) {
 	case FSF_STATUS_READ_SUB_NO_PHYSICAL_LINK:
-		zfcp_fsf_link_down_info_eval(req, ldi);
-		break;
 	case FSF_STATUS_READ_SUB_FDISC_FAILED:
 		zfcp_fsf_link_down_info_eval(req, ldi);
 		break;
@@ -476,8 +474,8 @@ static int zfcp_fsf_exchange_config_evaluate(struct zfcp_fsf_req *req)
 	if (req->data)
 		memcpy(req->data, bottom, sizeof(*bottom));
 
-	fc_host_port_name(shost) = nsp->fl_wwpn;
-	fc_host_node_name(shost) = nsp->fl_wwnn;
+	fc_host_port_name(shost) = be64_to_cpu(nsp->fl_wwpn);
+	fc_host_node_name(shost) = be64_to_cpu(nsp->fl_wwnn);
 	fc_host_supported_classes(shost) = FC_COS_CLASS2 | FC_COS_CLASS3;
 
 	adapter->timer_ticks = bottom->timer_interval & ZFCP_FSF_TIMER_INT_MASK;
@@ -503,8 +501,8 @@ static int zfcp_fsf_exchange_config_evaluate(struct zfcp_fsf_req *req)
 	switch (bottom->fc_topology) {
 	case FSF_TOPO_P2P:
 		adapter->peer_d_id = ntoh24(bottom->peer_d_id);
-		adapter->peer_wwpn = plogi->fl_wwpn;
-		adapter->peer_wwnn = plogi->fl_wwnn;
+		adapter->peer_wwpn = be64_to_cpu(plogi->fl_wwpn);
+		adapter->peer_wwnn = be64_to_cpu(plogi->fl_wwnn);
 		fc_host_port_type(shost) = FC_PORTTYPE_PTP;
 		break;
 	case FSF_TOPO_FABRIC:
@@ -928,8 +926,8 @@ static void zfcp_fsf_send_ct_handler(struct zfcp_fsf_req *req)
 
 	switch (header->fsf_status) {
         case FSF_GOOD:
-		zfcp_dbf_san_res("fsscth2", req);
 		ct->status = 0;
+		zfcp_dbf_san_res("fsscth2", req);
 		break;
         case FSF_SERVICE_CLASS_NOT_SUPPORTED:
 		zfcp_fsf_class_not_supp(req);
@@ -991,8 +989,7 @@ static int zfcp_fsf_setup_ct_els_sbals(struct zfcp_fsf_req *req,
 		qtcb->bottom.support.resp_buf_length =
 			zfcp_qdio_real_bytes(sg_resp);
 
-		zfcp_qdio_set_data_div(qdio, &req->qdio_req,
-					zfcp_qdio_sbale_count(sg_req));
+		zfcp_qdio_set_data_div(qdio, &req->qdio_req, sg_nents(sg_req));
 		zfcp_qdio_set_sbale_last(qdio, &req->qdio_req);
 		zfcp_qdio_set_scount(qdio, &req->qdio_req);
 		return 0;
@@ -1109,8 +1106,8 @@ static void zfcp_fsf_send_els_handler(struct zfcp_fsf_req *req)
 
 	switch (header->fsf_status) {
 	case FSF_GOOD:
-		zfcp_dbf_san_res("fsselh1", req);
 		send_els->status = 0;
+		zfcp_dbf_san_res("fsselh1", req);
 		break;
 	case FSF_SERVICE_CLASS_NOT_SUPPORTED:
 		zfcp_fsf_class_not_supp(req);
@@ -1394,6 +1391,8 @@ static void zfcp_fsf_open_port_handler(struct zfcp_fsf_req *req)
 	case FSF_ADAPTER_STATUS_AVAILABLE:
 		switch (header->fsf_status_qual.word[0]) {
 		case FSF_SQ_INVOKE_LINK_TEST_PROCEDURE:
+			/* no zfcp_fc_test_link() with failed open port */
+			/* fall through */
 		case FSF_SQ_ULP_DEPENDENT_ERP_REQUIRED:
 		case FSF_SQ_NO_RETRY_POSSIBLE:
 			req->status |= ZFCP_STATUS_FSFREQ_ERROR;
@@ -2142,7 +2141,8 @@ static void zfcp_fsf_fcp_cmnd_handler(struct zfcp_fsf_req *req)
 		zfcp_scsi_dif_sense_error(scpnt, 0x3);
 		goto skip_fsfstatus;
 	}
-	fcp_rsp = (struct fcp_resp_with_ext *) &req->qtcb->bottom.io.fcp_rsp;
+	BUILD_BUG_ON(sizeof(struct fcp_resp_with_ext) > FSF_FCP_RSP_SIZE);
+	fcp_rsp = &req->qtcb->bottom.io.fcp_rsp.iu;
 	zfcp_fc_eval_fcp_rsp(fcp_rsp, scpnt);
 
 skip_fsfstatus:
@@ -2255,10 +2255,12 @@ int zfcp_fsf_fcp_cmnd(struct scsi_cmnd *scsi_cmnd)
 	if (zfcp_fsf_set_data_dir(scsi_cmnd, &io->data_direction))
 		goto failed_scsi_cmnd;
 
-	fcp_cmnd = (struct fcp_cmnd *) &req->qtcb->bottom.io.fcp_cmnd;
+	BUILD_BUG_ON(sizeof(struct fcp_cmnd) > FSF_FCP_CMND_SIZE);
+	fcp_cmnd = &req->qtcb->bottom.io.fcp_cmnd.iu;
 	zfcp_fc_scsi_to_fcp(fcp_cmnd, scsi_cmnd, 0);
 
-	if (scsi_prot_sg_count(scsi_cmnd)) {
+	if ((scsi_get_prot_op(scsi_cmnd) != SCSI_PROT_NORMAL) &&
+	    scsi_prot_sg_count(scsi_cmnd)) {
 		zfcp_qdio_set_data_div(qdio, &req->qdio_req,
 				       scsi_prot_sg_count(scsi_cmnd));
 		retval = zfcp_qdio_sbals_from_sg(qdio, &req->qdio_req,
@@ -2299,7 +2301,7 @@ static void zfcp_fsf_fcp_task_mgmt_handler(struct zfcp_fsf_req *req)
 
 	zfcp_fsf_fcp_handler_common(req);
 
-	fcp_rsp = (struct fcp_resp_with_ext *) &req->qtcb->bottom.io.fcp_rsp;
+	fcp_rsp = &req->qtcb->bottom.io.fcp_rsp.iu;
 	rsp_info = (struct fcp_resp_rsp_info *) &fcp_rsp[1];
 
 	if ((rsp_info->rsp_code != FCP_TMF_CMPL) ||
@@ -2348,7 +2350,7 @@ struct zfcp_fsf_req *zfcp_fsf_fcp_task_mgmt(struct scsi_cmnd *scmnd,
 
 	zfcp_qdio_set_sbale_last(qdio, &req->qdio_req);
 
-	fcp_cmnd = (struct fcp_cmnd *) &req->qtcb->bottom.io.fcp_cmnd;
+	fcp_cmnd = &req->qtcb->bottom.io.fcp_cmnd.iu;
 	zfcp_fc_scsi_to_fcp(fcp_cmnd, scmnd, tm_flags);
 
 	zfcp_fsf_start_timer(req, ZFCP_SCSI_ER_TIMEOUT);
@@ -2392,7 +2394,6 @@ void zfcp_fsf_reqid_check(struct zfcp_qdio *qdio, int sbal_idx)
 			      req_id, dev_name(&adapter->ccw_device->dev));
 		}
 
-		fsf_req->qdio_req.sbal_response = sbal_idx;
 		zfcp_fsf_req_complete(fsf_req);
 
 		if (likely(sbale->eflags & SBAL_EFLAGS_LAST_ENTRY))
diff --git a/drivers/s390/scsi/zfcp_fsf.h b/drivers/s390/scsi/zfcp_fsf.h
index ea3c76ac0de1..88feba5bfda4 100644
--- a/drivers/s390/scsi/zfcp_fsf.h
+++ b/drivers/s390/scsi/zfcp_fsf.h
@@ -3,7 +3,7 @@
  *
  * Interface to the FSF support functions.
  *
- * Copyright IBM Corp. 2002, 2016
+ * Copyright IBM Corp. 2002, 2017
  */
 
 #ifndef FSF_H
@@ -312,8 +312,14 @@ struct fsf_qtcb_bottom_io {
 	u32 data_block_length;
 	u32 prot_data_length;
 	u8  res2[4];
-	u8  fcp_cmnd[FSF_FCP_CMND_SIZE];
-	u8  fcp_rsp[FSF_FCP_RSP_SIZE];
+	union {
+		u8		byte[FSF_FCP_CMND_SIZE];
+		struct fcp_cmnd iu;
+	}   fcp_cmnd;
+	union {
+		u8			 byte[FSF_FCP_RSP_SIZE];
+		struct fcp_resp_with_ext iu;
+	}   fcp_rsp;
 	u8  res3[64];
 } __attribute__ ((packed));
 
diff --git a/drivers/s390/scsi/zfcp_qdio.c b/drivers/s390/scsi/zfcp_qdio.c
index dbf2b54703f7..9e358fc04b78 100644
--- a/drivers/s390/scsi/zfcp_qdio.c
+++ b/drivers/s390/scsi/zfcp_qdio.c
@@ -14,7 +14,7 @@
 #include "zfcp_ext.h"
 #include "zfcp_qdio.h"
 
-static bool enable_multibuffer = 1;
+static bool enable_multibuffer = true;
 module_param_named(datarouter, enable_multibuffer, bool, 0400);
 MODULE_PARM_DESC(datarouter, "Enable hardware data router support (default on)");
 
diff --git a/drivers/s390/scsi/zfcp_qdio.h b/drivers/s390/scsi/zfcp_qdio.h
index 497cd379b0d1..7f647a90c750 100644
--- a/drivers/s390/scsi/zfcp_qdio.h
+++ b/drivers/s390/scsi/zfcp_qdio.h
@@ -54,7 +54,6 @@ struct zfcp_qdio {
  * @sbal_last: last sbal for this request
  * @sbal_limit: last possible sbal for this request
  * @sbale_curr: current sbale at creation of this request
- * @sbal_response: sbal used in interrupt
  * @qdio_outb_usage: usage of outbound queue
  */
 struct zfcp_qdio_req {
@@ -64,7 +63,6 @@ struct zfcp_qdio_req {
 	u8	sbal_last;
 	u8	sbal_limit;
 	u8	sbale_curr;
-	u8	sbal_response;
 	u16	qdio_outb_usage;
 };
 
@@ -225,21 +223,6 @@ void zfcp_qdio_set_data_div(struct zfcp_qdio *qdio,
 }
 
 /**
- * zfcp_qdio_sbale_count - count sbale used
- * @sg: pointer to struct scatterlist
- */
-static inline
-unsigned int zfcp_qdio_sbale_count(struct scatterlist *sg)
-{
-	unsigned int count = 0;
-
-	for (; sg; sg = sg_next(sg))
-		count++;
-
-	return count;
-}
-
-/**
  * zfcp_qdio_real_bytes - count bytes used
  * @sg: pointer to struct scatterlist
  */
diff --git a/drivers/s390/scsi/zfcp_scsi.c b/drivers/s390/scsi/zfcp_scsi.c
index 0678cf714c0e..ec3ddd1d31d5 100644
--- a/drivers/s390/scsi/zfcp_scsi.c
+++ b/drivers/s390/scsi/zfcp_scsi.c
@@ -3,7 +3,7 @@
  *
  * Interface to Linux SCSI midlayer.
  *
- * Copyright IBM Corp. 2002, 2016
+ * Copyright IBM Corp. 2002, 2017
  */
 
 #define KMSG_COMPONENT "zfcp"
@@ -28,7 +28,7 @@ static bool enable_dif;
 module_param_named(dif, enable_dif, bool, 0400);
 MODULE_PARM_DESC(dif, "Enable DIF/DIX data integrity support");
 
-static bool allow_lun_scan = 1;
+static bool allow_lun_scan = true;
 module_param(allow_lun_scan, bool, 0600);
 MODULE_PARM_DESC(allow_lun_scan, "For NPIV, scan and attach all storage LUNs");
 
@@ -273,25 +273,29 @@ static int zfcp_task_mgmt_function(struct scsi_cmnd *scpnt, u8 tm_flags)
 
 		zfcp_erp_wait(adapter);
 		ret = fc_block_scsi_eh(scpnt);
-		if (ret)
+		if (ret) {
+			zfcp_dbf_scsi_devreset("fiof", scpnt, tm_flags, NULL);
 			return ret;
+		}
 
 		if (!(atomic_read(&adapter->status) &
 		      ZFCP_STATUS_COMMON_RUNNING)) {
-			zfcp_dbf_scsi_devreset("nres", scpnt, tm_flags);
+			zfcp_dbf_scsi_devreset("nres", scpnt, tm_flags, NULL);
 			return SUCCESS;
 		}
 	}
-	if (!fsf_req)
+	if (!fsf_req) {
+		zfcp_dbf_scsi_devreset("reqf", scpnt, tm_flags, NULL);
 		return FAILED;
+	}
 
 	wait_for_completion(&fsf_req->completion);
 
 	if (fsf_req->status & ZFCP_STATUS_FSFREQ_TMFUNCFAILED) {
-		zfcp_dbf_scsi_devreset("fail", scpnt, tm_flags);
+		zfcp_dbf_scsi_devreset("fail", scpnt, tm_flags, fsf_req);
 		retval = FAILED;
 	} else {
-		zfcp_dbf_scsi_devreset("okay", scpnt, tm_flags);
+		zfcp_dbf_scsi_devreset("okay", scpnt, tm_flags, fsf_req);
 		zfcp_scsi_forget_cmnds(zfcp_sdev, tm_flags);
 	}
 
diff --git a/drivers/scsi/53c700.c b/drivers/scsi/53c700.c
index 4b3b08025ef6..6be77b3aa8a5 100644
--- a/drivers/scsi/53c700.c
+++ b/drivers/scsi/53c700.c
@@ -168,7 +168,6 @@ MODULE_LICENSE("GPL");
 
 STATIC int NCR_700_queuecommand(struct Scsi_Host *h, struct scsi_cmnd *);
 STATIC int NCR_700_abort(struct scsi_cmnd * SCpnt);
-STATIC int NCR_700_bus_reset(struct scsi_cmnd * SCpnt);
 STATIC int NCR_700_host_reset(struct scsi_cmnd * SCpnt);
 STATIC void NCR_700_chip_setup(struct Scsi_Host *host);
 STATIC void NCR_700_chip_reset(struct Scsi_Host *host);
@@ -315,7 +314,6 @@ NCR_700_detect(struct scsi_host_template *tpnt,
 	/* Fill in the missing routines from the host template */
 	tpnt->queuecommand = NCR_700_queuecommand;
 	tpnt->eh_abort_handler = NCR_700_abort;
-	tpnt->eh_bus_reset_handler = NCR_700_bus_reset;
 	tpnt->eh_host_reset_handler = NCR_700_host_reset;
 	tpnt->can_queue = NCR_700_COMMAND_SLOTS_PER_HOST;
 	tpnt->sg_tablesize = NCR_700_SG_SEGMENTS;
@@ -1938,14 +1936,14 @@ NCR_700_abort(struct scsi_cmnd * SCp)
 }
 
 STATIC int
-NCR_700_bus_reset(struct scsi_cmnd * SCp)
+NCR_700_host_reset(struct scsi_cmnd * SCp)
 {
 	DECLARE_COMPLETION_ONSTACK(complete);
 	struct NCR_700_Host_Parameters *hostdata = 
 		(struct NCR_700_Host_Parameters *)SCp->device->host->hostdata[0];
 
 	scmd_printk(KERN_INFO, SCp,
-		"New error handler wants BUS reset, cmd %p\n\t", SCp);
+		"New error handler wants HOST reset, cmd %p\n\t", SCp);
 	scsi_print_command(SCp);
 
 	/* In theory, eh_complete should always be null because the
@@ -1960,6 +1958,7 @@ NCR_700_bus_reset(struct scsi_cmnd * SCp)
 
 	hostdata->eh_complete = &complete;
 	NCR_700_internal_bus_reset(SCp->device->host);
+	NCR_700_chip_reset(SCp->device->host);
 
 	spin_unlock_irq(SCp->device->host->host_lock);
 	wait_for_completion(&complete);
@@ -1974,22 +1973,6 @@ NCR_700_bus_reset(struct scsi_cmnd * SCp)
 	return SUCCESS;
 }
 
-STATIC int
-NCR_700_host_reset(struct scsi_cmnd * SCp)
-{
-	scmd_printk(KERN_INFO, SCp, "New error handler wants HOST reset\n\t");
-	scsi_print_command(SCp);
-
-	spin_lock_irq(SCp->device->host->host_lock);
-
-	NCR_700_internal_bus_reset(SCp->device->host);
-	NCR_700_chip_reset(SCp->device->host);
-
-	spin_unlock_irq(SCp->device->host->host_lock);
-
-	return SUCCESS;
-}
-
 STATIC void
 NCR_700_set_period(struct scsi_target *STp, int period)
 {
diff --git a/drivers/scsi/Kconfig b/drivers/scsi/Kconfig
index f4538d7a3016..41366339b950 100644
--- a/drivers/scsi/Kconfig
+++ b/drivers/scsi/Kconfig
@@ -47,6 +47,17 @@ config SCSI_NETLINK
 	default	n
 	depends on NET
 
+config SCSI_MQ_DEFAULT
+	bool "SCSI: use blk-mq I/O path by default"
+	depends on SCSI
+	---help---
+	  This option enables the new blk-mq based I/O path for SCSI
+	  devices by default.  With the option the scsi_mod.use_blk_mq
+	  module/boot option defaults to Y, without it to N, but it can
+	  still be overridden either way.
+
+	  If unsure say N.
+
 config SCSI_PROC_FS
 	bool "legacy /proc/scsi/ support"
 	depends on SCSI && PROC_FS
@@ -272,7 +283,7 @@ config SCSI_ISCSI_ATTRS
 config SCSI_SAS_ATTRS
 	tristate "SAS Transport Attributes"
 	depends on SCSI
-	select BLK_DEV_BSG
+	select BLK_DEV_BSGLIB
 	help
 	  If you wish to export transport-specific information about
 	  each attached SAS device to sysfs, say Y.
diff --git a/drivers/scsi/NCR5380.c b/drivers/scsi/NCR5380.c
index acc33440bca0..8a0812221d72 100644
--- a/drivers/scsi/NCR5380.c
+++ b/drivers/scsi/NCR5380.c
@@ -2296,13 +2296,13 @@ out:
 
 
 /**
- * NCR5380_bus_reset - reset the SCSI bus
+ * NCR5380_host_reset - reset the SCSI host
  * @cmd: SCSI command undergoing EH
  *
  * Returns SUCCESS
  */
 
-static int NCR5380_bus_reset(struct scsi_cmnd *cmd)
+static int NCR5380_host_reset(struct scsi_cmnd *cmd)
 {
 	struct Scsi_Host *instance = cmd->device->host;
 	struct NCR5380_hostdata *hostdata = shost_priv(instance);
diff --git a/drivers/scsi/NCR_Q720.c b/drivers/scsi/NCR_Q720.c
index 05835bf1bf9c..54e7d26908ee 100644
--- a/drivers/scsi/NCR_Q720.c
+++ b/drivers/scsi/NCR_Q720.c
@@ -217,8 +217,7 @@ NCR_Q720_probe(struct device *dev)
 	}
 	
 	if (dma_declare_coherent_memory(dev, base_addr, base_addr,
-					mem_size, DMA_MEMORY_MAP)
-	    != DMA_MEMORY_MAP) {
+					mem_size, 0)) {
 		printk(KERN_ERR "NCR_Q720: DMA declare memory failed\n");
 		goto out_release_region;
 	}
diff --git a/drivers/scsi/a2091.c b/drivers/scsi/a2091.c
index 9176bfbd5745..61aadc7acb49 100644
--- a/drivers/scsi/a2091.c
+++ b/drivers/scsi/a2091.c
@@ -147,22 +147,6 @@ static void dma_stop(struct Scsi_Host *instance, struct scsi_cmnd *SCpnt,
 	}
 }
 
-static int a2091_bus_reset(struct scsi_cmnd *cmd)
-{
-	struct Scsi_Host *instance = cmd->device->host;
-
-	/* FIXME perform bus-specific reset */
-
-	/* FIXME 2: kill this function, and let midlayer fall back
-	   to the same action, calling wd33c93_host_reset() */
-
-	spin_lock_irq(instance->host_lock);
-	wd33c93_host_reset(cmd);
-	spin_unlock_irq(instance->host_lock);
-
-	return SUCCESS;
-}
-
 static struct scsi_host_template a2091_scsi_template = {
 	.module			= THIS_MODULE,
 	.name			= "Commodore A2091/A590 SCSI",
@@ -171,7 +155,6 @@ static struct scsi_host_template a2091_scsi_template = {
 	.proc_name		= "A2901",
 	.queuecommand		= wd33c93_queuecommand,
 	.eh_abort_handler	= wd33c93_abort,
-	.eh_bus_reset_handler	= a2091_bus_reset,
 	.eh_host_reset_handler	= wd33c93_host_reset,
 	.can_queue		= CAN_QUEUE,
 	.this_id		= 7,
diff --git a/drivers/scsi/a3000.c b/drivers/scsi/a3000.c
index e6375b4de79e..2427a8541247 100644
--- a/drivers/scsi/a3000.c
+++ b/drivers/scsi/a3000.c
@@ -162,22 +162,6 @@ static void dma_stop(struct Scsi_Host *instance, struct scsi_cmnd *SCpnt,
 	}
 }
 
-static int a3000_bus_reset(struct scsi_cmnd *cmd)
-{
-	struct Scsi_Host *instance = cmd->device->host;
-
-	/* FIXME perform bus-specific reset */
-
-	/* FIXME 2: kill this entire function, which should
-	   cause mid-layer to call wd33c93_host_reset anyway? */
-
-	spin_lock_irq(instance->host_lock);
-	wd33c93_host_reset(cmd);
-	spin_unlock_irq(instance->host_lock);
-
-	return SUCCESS;
-}
-
 static struct scsi_host_template amiga_a3000_scsi_template = {
 	.module			= THIS_MODULE,
 	.name			= "Amiga 3000 built-in SCSI",
@@ -186,7 +170,6 @@ static struct scsi_host_template amiga_a3000_scsi_template = {
 	.proc_name		= "A3000",
 	.queuecommand		= wd33c93_queuecommand,
 	.eh_abort_handler	= wd33c93_abort,
-	.eh_bus_reset_handler	= a3000_bus_reset,
 	.eh_host_reset_handler	= wd33c93_host_reset,
 	.can_queue		= CAN_QUEUE,
 	.this_id		= 7,
diff --git a/drivers/scsi/aacraid/aachba.c b/drivers/scsi/aacraid/aachba.c
index 4591113c49de..a64285ab0728 100644
--- a/drivers/scsi/aacraid/aachba.c
+++ b/drivers/scsi/aacraid/aachba.c
@@ -549,7 +549,9 @@ static void get_container_name_callback(void *context, struct fib * fibptr)
 	if ((le32_to_cpu(get_name_reply->status) == CT_OK)
 	 && (get_name_reply->data[0] != '\0')) {
 		char *sp = get_name_reply->data;
-		sp[sizeof(((struct aac_get_name_resp *)NULL)->data)] = '\0';
+		int data_size = FIELD_SIZEOF(struct aac_get_name_resp, data);
+
+		sp[data_size - 1] = '\0';
 		while (*sp == ' ')
 			++sp;
 		if (*sp) {
@@ -579,21 +581,25 @@ static void get_container_name_callback(void *context, struct fib * fibptr)
 static int aac_get_container_name(struct scsi_cmnd * scsicmd)
 {
 	int status;
+	int data_size;
 	struct aac_get_name *dinfo;
 	struct fib * cmd_fibcontext;
 	struct aac_dev * dev;
 
 	dev = (struct aac_dev *)scsicmd->device->host->hostdata;
 
+	data_size = FIELD_SIZEOF(struct aac_get_name_resp, data);
+
 	cmd_fibcontext = aac_fib_alloc_tag(dev, scsicmd);
 
 	aac_fib_init(cmd_fibcontext);
 	dinfo = (struct aac_get_name *) fib_data(cmd_fibcontext);
+	scsicmd->SCp.phase = AAC_OWNER_FIRMWARE;
 
 	dinfo->command = cpu_to_le32(VM_ContainerConfig);
 	dinfo->type = cpu_to_le32(CT_READ_NAME);
 	dinfo->cid = cpu_to_le32(scmd_id(scsicmd));
-	dinfo->count = cpu_to_le32(sizeof(((struct aac_get_name_resp *)NULL)->data));
+	dinfo->count = cpu_to_le32(data_size - 1);
 
 	status = aac_fib_send(ContainerCommand,
 		  cmd_fibcontext,
@@ -606,10 +612,8 @@ static int aac_get_container_name(struct scsi_cmnd * scsicmd)
 	/*
 	 *	Check that the command queued to the controller
 	 */
-	if (status == -EINPROGRESS) {
-		scsicmd->SCp.phase = AAC_OWNER_FIRMWARE;
+	if (status == -EINPROGRESS)
 		return 0;
-	}
 
 	printk(KERN_WARNING "aac_get_container_name: aac_fib_send failed with status: %d.\n", status);
 	aac_fib_complete(cmd_fibcontext);
@@ -720,6 +724,7 @@ static void _aac_probe_container1(void * context, struct fib * fibptr)
 
 	dinfo->count = cpu_to_le32(scmd_id(scsicmd));
 	dinfo->type = cpu_to_le32(FT_FILESYS);
+	scsicmd->SCp.phase = AAC_OWNER_FIRMWARE;
 
 	status = aac_fib_send(ContainerCommand,
 			  fibptr,
@@ -731,9 +736,7 @@ static void _aac_probe_container1(void * context, struct fib * fibptr)
 	/*
 	 *	Check that the command queued to the controller
 	 */
-	if (status == -EINPROGRESS)
-		scsicmd->SCp.phase = AAC_OWNER_FIRMWARE;
-	else if (status < 0) {
+	if (status < 0 && status != -EINPROGRESS) {
 		/* Inherit results from VM_NameServe, if any */
 		dresp->status = cpu_to_le32(ST_OK);
 		_aac_probe_container2(context, fibptr);
@@ -761,6 +764,7 @@ static int _aac_probe_container(struct scsi_cmnd * scsicmd, int (*callback)(stru
 		dinfo->count = cpu_to_le32(scmd_id(scsicmd));
 		dinfo->type = cpu_to_le32(FT_FILESYS);
 		scsicmd->SCp.ptr = (char *)callback;
+		scsicmd->SCp.phase = AAC_OWNER_FIRMWARE;
 
 		status = aac_fib_send(ContainerCommand,
 			  fibptr,
@@ -772,10 +776,9 @@ static int _aac_probe_container(struct scsi_cmnd * scsicmd, int (*callback)(stru
 		/*
 		 *	Check that the command queued to the controller
 		 */
-		if (status == -EINPROGRESS) {
-			scsicmd->SCp.phase = AAC_OWNER_FIRMWARE;
+		if (status == -EINPROGRESS)
 			return 0;
-		}
+
 		if (status < 0) {
 			scsicmd->SCp.ptr = NULL;
 			aac_fib_complete(fibptr);
@@ -1121,6 +1124,7 @@ static int aac_get_container_serial(struct scsi_cmnd * scsicmd)
 	dinfo->command = cpu_to_le32(VM_ContainerConfig);
 	dinfo->type = cpu_to_le32(CT_CID_TO_32BITS_UID);
 	dinfo->cid = cpu_to_le32(scmd_id(scsicmd));
+	scsicmd->SCp.phase = AAC_OWNER_FIRMWARE;
 
 	status = aac_fib_send(ContainerCommand,
 		  cmd_fibcontext,
@@ -1133,10 +1137,8 @@ static int aac_get_container_serial(struct scsi_cmnd * scsicmd)
 	/*
 	 *	Check that the command queued to the controller
 	 */
-	if (status == -EINPROGRESS) {
-		scsicmd->SCp.phase = AAC_OWNER_FIRMWARE;
+	if (status == -EINPROGRESS)
 		return 0;
-	}
 
 	printk(KERN_WARNING "aac_get_container_serial: aac_fib_send failed with status: %d.\n", status);
 	aac_fib_complete(cmd_fibcontext);
@@ -2330,16 +2332,14 @@ static int aac_read(struct scsi_cmnd * scsicmd)
 	 *	Alocate and initialize a Fib
 	 */
 	cmd_fibcontext = aac_fib_alloc_tag(dev, scsicmd);
-
+	scsicmd->SCp.phase = AAC_OWNER_FIRMWARE;
 	status = aac_adapter_read(cmd_fibcontext, scsicmd, lba, count);
 
 	/*
 	 *	Check that the command queued to the controller
 	 */
-	if (status == -EINPROGRESS) {
-		scsicmd->SCp.phase = AAC_OWNER_FIRMWARE;
+	if (status == -EINPROGRESS)
 		return 0;
-	}
 
 	printk(KERN_WARNING "aac_read: aac_fib_send failed with status: %d.\n", status);
 	/*
@@ -2424,16 +2424,14 @@ static int aac_write(struct scsi_cmnd * scsicmd)
 	 *	Allocate and initialize a Fib then setup a BlockWrite command
 	 */
 	cmd_fibcontext = aac_fib_alloc_tag(dev, scsicmd);
-
+	scsicmd->SCp.phase = AAC_OWNER_FIRMWARE;
 	status = aac_adapter_write(cmd_fibcontext, scsicmd, lba, count, fua);
 
 	/*
 	 *	Check that the command queued to the controller
 	 */
-	if (status == -EINPROGRESS) {
-		scsicmd->SCp.phase = AAC_OWNER_FIRMWARE;
+	if (status == -EINPROGRESS)
 		return 0;
-	}
 
 	printk(KERN_WARNING "aac_write: aac_fib_send failed with status: %d\n", status);
 	/*
@@ -2583,6 +2581,7 @@ static int aac_synchronize(struct scsi_cmnd *scsicmd)
 	synchronizecmd->cid = cpu_to_le32(scmd_id(scsicmd));
 	synchronizecmd->count =
 	     cpu_to_le32(sizeof(((struct aac_synchronize_reply *)NULL)->data));
+	scsicmd->SCp.phase = AAC_OWNER_FIRMWARE;
 
 	/*
 	 *	Now send the Fib to the adapter
@@ -2598,10 +2597,8 @@ static int aac_synchronize(struct scsi_cmnd *scsicmd)
 	/*
 	 *	Check that the command queued to the controller
 	 */
-	if (status == -EINPROGRESS) {
-		scsicmd->SCp.phase = AAC_OWNER_FIRMWARE;
+	if (status == -EINPROGRESS)
 		return 0;
-	}
 
 	printk(KERN_WARNING
 		"aac_synchronize: aac_fib_send failed with status: %d.\n", status);
@@ -2661,6 +2658,7 @@ static int aac_start_stop(struct scsi_cmnd *scsicmd)
 	pmcmd->cid = cpu_to_le32(sdev_id(sdev));
 	pmcmd->parm = (scsicmd->cmnd[1] & 1) ?
 		cpu_to_le32(CT_PM_UNIT_IMMEDIATE) : 0;
+	scsicmd->SCp.phase = AAC_OWNER_FIRMWARE;
 
 	/*
 	 *	Now send the Fib to the adapter
@@ -2676,10 +2674,8 @@ static int aac_start_stop(struct scsi_cmnd *scsicmd)
 	/*
 	 *	Check that the command queued to the controller
 	 */
-	if (status == -EINPROGRESS) {
-		scsicmd->SCp.phase = AAC_OWNER_FIRMWARE;
+	if (status == -EINPROGRESS)
 		return 0;
-	}
 
 	aac_fib_complete(cmd_fibcontext);
 	aac_fib_free(cmd_fibcontext);
@@ -3687,16 +3683,14 @@ static int aac_send_srb_fib(struct scsi_cmnd* scsicmd)
 	 *	Allocate and initialize a Fib then setup a BlockWrite command
 	 */
 	cmd_fibcontext = aac_fib_alloc_tag(dev, scsicmd);
-
+	scsicmd->SCp.phase = AAC_OWNER_FIRMWARE;
 	status = aac_adapter_scsi(cmd_fibcontext, scsicmd);
 
 	/*
 	 *	Check that the command queued to the controller
 	 */
-	if (status == -EINPROGRESS) {
-		scsicmd->SCp.phase = AAC_OWNER_FIRMWARE;
+	if (status == -EINPROGRESS)
 		return 0;
-	}
 
 	printk(KERN_WARNING "aac_srb: aac_fib_send failed with status: %d\n", status);
 	aac_fib_complete(cmd_fibcontext);
@@ -3734,15 +3728,14 @@ static int aac_send_hba_fib(struct scsi_cmnd *scsicmd)
 	if (!cmd_fibcontext)
 		return -1;
 
+	scsicmd->SCp.phase = AAC_OWNER_FIRMWARE;
 	status = aac_adapter_hba(cmd_fibcontext, scsicmd);
 
 	/*
 	 *	Check that the command queued to the controller
 	 */
-	if (status == -EINPROGRESS) {
-		scsicmd->SCp.phase = AAC_OWNER_FIRMWARE;
+	if (status == -EINPROGRESS)
 		return 0;
-	}
 
 	pr_warn("aac_hba_cmd_req: aac_fib_send failed with status: %d\n",
 		status);
@@ -3758,6 +3751,8 @@ static long aac_build_sg(struct scsi_cmnd *scsicmd, struct sgmap *psg)
 	struct aac_dev *dev;
 	unsigned long byte_count = 0;
 	int nseg;
+	struct scatterlist *sg;
+	int i;
 
 	dev = (struct aac_dev *)scsicmd->device->host->hostdata;
 	// Get rid of old data
@@ -3766,32 +3761,29 @@ static long aac_build_sg(struct scsi_cmnd *scsicmd, struct sgmap *psg)
 	psg->sg[0].count = 0;
 
 	nseg = scsi_dma_map(scsicmd);
-	if (nseg < 0)
+	if (nseg <= 0)
 		return nseg;
-	if (nseg) {
-		struct scatterlist *sg;
-		int i;
 
-		psg->count = cpu_to_le32(nseg);
+	psg->count = cpu_to_le32(nseg);
 
-		scsi_for_each_sg(scsicmd, sg, nseg, i) {
-			psg->sg[i].addr = cpu_to_le32(sg_dma_address(sg));
-			psg->sg[i].count = cpu_to_le32(sg_dma_len(sg));
-			byte_count += sg_dma_len(sg);
-		}
-		/* hba wants the size to be exact */
-		if (byte_count > scsi_bufflen(scsicmd)) {
-			u32 temp = le32_to_cpu(psg->sg[i-1].count) -
-				(byte_count - scsi_bufflen(scsicmd));
-			psg->sg[i-1].count = cpu_to_le32(temp);
-			byte_count = scsi_bufflen(scsicmd);
-		}
-		/* Check for command underflow */
-		if(scsicmd->underflow && (byte_count < scsicmd->underflow)){
-			printk(KERN_WARNING"aacraid: cmd len %08lX cmd underflow %08X\n",
-					byte_count, scsicmd->underflow);
-		}
+	scsi_for_each_sg(scsicmd, sg, nseg, i) {
+		psg->sg[i].addr = cpu_to_le32(sg_dma_address(sg));
+		psg->sg[i].count = cpu_to_le32(sg_dma_len(sg));
+		byte_count += sg_dma_len(sg);
 	}
+	/* hba wants the size to be exact */
+	if (byte_count > scsi_bufflen(scsicmd)) {
+		u32 temp = le32_to_cpu(psg->sg[i-1].count) -
+			(byte_count - scsi_bufflen(scsicmd));
+		psg->sg[i-1].count = cpu_to_le32(temp);
+		byte_count = scsi_bufflen(scsicmd);
+	}
+	/* Check for command underflow */
+	if (scsicmd->underflow && (byte_count < scsicmd->underflow)) {
+		printk(KERN_WARNING"aacraid: cmd len %08lX cmd underflow %08X\n",
+		       byte_count, scsicmd->underflow);
+	}
+
 	return byte_count;
 }
 
@@ -3802,6 +3794,8 @@ static long aac_build_sg64(struct scsi_cmnd *scsicmd, struct sgmap64 *psg)
 	unsigned long byte_count = 0;
 	u64 addr;
 	int nseg;
+	struct scatterlist *sg;
+	int i;
 
 	dev = (struct aac_dev *)scsicmd->device->host->hostdata;
 	// Get rid of old data
@@ -3811,34 +3805,31 @@ static long aac_build_sg64(struct scsi_cmnd *scsicmd, struct sgmap64 *psg)
 	psg->sg[0].count = 0;
 
 	nseg = scsi_dma_map(scsicmd);
-	if (nseg < 0)
+	if (nseg <= 0)
 		return nseg;
-	if (nseg) {
-		struct scatterlist *sg;
-		int i;
-
-		scsi_for_each_sg(scsicmd, sg, nseg, i) {
-			int count = sg_dma_len(sg);
-			addr = sg_dma_address(sg);
-			psg->sg[i].addr[0] = cpu_to_le32(addr & 0xffffffff);
-			psg->sg[i].addr[1] = cpu_to_le32(addr>>32);
-			psg->sg[i].count = cpu_to_le32(count);
-			byte_count += count;
-		}
-		psg->count = cpu_to_le32(nseg);
-		/* hba wants the size to be exact */
-		if (byte_count > scsi_bufflen(scsicmd)) {
-			u32 temp = le32_to_cpu(psg->sg[i-1].count) -
-				(byte_count - scsi_bufflen(scsicmd));
-			psg->sg[i-1].count = cpu_to_le32(temp);
-			byte_count = scsi_bufflen(scsicmd);
-		}
-		/* Check for command underflow */
-		if(scsicmd->underflow && (byte_count < scsicmd->underflow)){
-			printk(KERN_WARNING"aacraid: cmd len %08lX cmd underflow %08X\n",
-					byte_count, scsicmd->underflow);
-		}
+
+	scsi_for_each_sg(scsicmd, sg, nseg, i) {
+		int count = sg_dma_len(sg);
+		addr = sg_dma_address(sg);
+		psg->sg[i].addr[0] = cpu_to_le32(addr & 0xffffffff);
+		psg->sg[i].addr[1] = cpu_to_le32(addr>>32);
+		psg->sg[i].count = cpu_to_le32(count);
+		byte_count += count;
+	}
+	psg->count = cpu_to_le32(nseg);
+	/* hba wants the size to be exact */
+	if (byte_count > scsi_bufflen(scsicmd)) {
+		u32 temp = le32_to_cpu(psg->sg[i-1].count) -
+			(byte_count - scsi_bufflen(scsicmd));
+		psg->sg[i-1].count = cpu_to_le32(temp);
+		byte_count = scsi_bufflen(scsicmd);
+	}
+	/* Check for command underflow */
+	if (scsicmd->underflow && (byte_count < scsicmd->underflow)) {
+		printk(KERN_WARNING"aacraid: cmd len %08lX cmd underflow %08X\n",
+		       byte_count, scsicmd->underflow);
 	}
+
 	return byte_count;
 }
 
@@ -3846,6 +3837,8 @@ static long aac_build_sgraw(struct scsi_cmnd *scsicmd, struct sgmapraw *psg)
 {
 	unsigned long byte_count = 0;
 	int nseg;
+	struct scatterlist *sg;
+	int i;
 
 	// Get rid of old data
 	psg->count = 0;
@@ -3857,37 +3850,34 @@ static long aac_build_sgraw(struct scsi_cmnd *scsicmd, struct sgmapraw *psg)
 	psg->sg[0].flags = 0;
 
 	nseg = scsi_dma_map(scsicmd);
-	if (nseg < 0)
+	if (nseg <= 0)
 		return nseg;
-	if (nseg) {
-		struct scatterlist *sg;
-		int i;
-
-		scsi_for_each_sg(scsicmd, sg, nseg, i) {
-			int count = sg_dma_len(sg);
-			u64 addr = sg_dma_address(sg);
-			psg->sg[i].next = 0;
-			psg->sg[i].prev = 0;
-			psg->sg[i].addr[1] = cpu_to_le32((u32)(addr>>32));
-			psg->sg[i].addr[0] = cpu_to_le32((u32)(addr & 0xffffffff));
-			psg->sg[i].count = cpu_to_le32(count);
-			psg->sg[i].flags = 0;
-			byte_count += count;
-		}
-		psg->count = cpu_to_le32(nseg);
-		/* hba wants the size to be exact */
-		if (byte_count > scsi_bufflen(scsicmd)) {
-			u32 temp = le32_to_cpu(psg->sg[i-1].count) -
-				(byte_count - scsi_bufflen(scsicmd));
-			psg->sg[i-1].count = cpu_to_le32(temp);
-			byte_count = scsi_bufflen(scsicmd);
-		}
-		/* Check for command underflow */
-		if(scsicmd->underflow && (byte_count < scsicmd->underflow)){
-			printk(KERN_WARNING"aacraid: cmd len %08lX cmd underflow %08X\n",
-					byte_count, scsicmd->underflow);
-		}
+
+	scsi_for_each_sg(scsicmd, sg, nseg, i) {
+		int count = sg_dma_len(sg);
+		u64 addr = sg_dma_address(sg);
+		psg->sg[i].next = 0;
+		psg->sg[i].prev = 0;
+		psg->sg[i].addr[1] = cpu_to_le32((u32)(addr>>32));
+		psg->sg[i].addr[0] = cpu_to_le32((u32)(addr & 0xffffffff));
+		psg->sg[i].count = cpu_to_le32(count);
+		psg->sg[i].flags = 0;
+		byte_count += count;
+	}
+	psg->count = cpu_to_le32(nseg);
+	/* hba wants the size to be exact */
+	if (byte_count > scsi_bufflen(scsicmd)) {
+		u32 temp = le32_to_cpu(psg->sg[i-1].count) -
+			(byte_count - scsi_bufflen(scsicmd));
+		psg->sg[i-1].count = cpu_to_le32(temp);
+		byte_count = scsi_bufflen(scsicmd);
+	}
+	/* Check for command underflow */
+	if (scsicmd->underflow && (byte_count < scsicmd->underflow)) {
+		printk(KERN_WARNING"aacraid: cmd len %08lX cmd underflow %08X\n",
+		       byte_count, scsicmd->underflow);
 	}
+
 	return byte_count;
 }
 
@@ -3896,75 +3886,77 @@ static long aac_build_sgraw2(struct scsi_cmnd *scsicmd,
 {
 	unsigned long byte_count = 0;
 	int nseg;
+	struct scatterlist *sg;
+	int i, conformable = 0;
+	u32 min_size = PAGE_SIZE, cur_size;
 
 	nseg = scsi_dma_map(scsicmd);
-	if (nseg < 0)
+	if (nseg <= 0)
 		return nseg;
-	if (nseg) {
-		struct scatterlist *sg;
-		int i, conformable = 0;
-		u32 min_size = PAGE_SIZE, cur_size;
-
-		scsi_for_each_sg(scsicmd, sg, nseg, i) {
-			int count = sg_dma_len(sg);
-			u64 addr = sg_dma_address(sg);
-
-			BUG_ON(i >= sg_max);
-			rio2->sge[i].addrHigh = cpu_to_le32((u32)(addr>>32));
-			rio2->sge[i].addrLow = cpu_to_le32((u32)(addr & 0xffffffff));
-			cur_size = cpu_to_le32(count);
-			rio2->sge[i].length = cur_size;
-			rio2->sge[i].flags = 0;
-			if (i == 0) {
-				conformable = 1;
-				rio2->sgeFirstSize = cur_size;
-			} else if (i == 1) {
-				rio2->sgeNominalSize = cur_size;
+
+	scsi_for_each_sg(scsicmd, sg, nseg, i) {
+		int count = sg_dma_len(sg);
+		u64 addr = sg_dma_address(sg);
+
+		BUG_ON(i >= sg_max);
+		rio2->sge[i].addrHigh = cpu_to_le32((u32)(addr>>32));
+		rio2->sge[i].addrLow = cpu_to_le32((u32)(addr & 0xffffffff));
+		cur_size = cpu_to_le32(count);
+		rio2->sge[i].length = cur_size;
+		rio2->sge[i].flags = 0;
+		if (i == 0) {
+			conformable = 1;
+			rio2->sgeFirstSize = cur_size;
+		} else if (i == 1) {
+			rio2->sgeNominalSize = cur_size;
+			min_size = cur_size;
+		} else if ((i+1) < nseg && cur_size != rio2->sgeNominalSize) {
+			conformable = 0;
+			if (cur_size < min_size)
 				min_size = cur_size;
-			} else if ((i+1) < nseg && cur_size != rio2->sgeNominalSize) {
-				conformable = 0;
-				if (cur_size < min_size)
-					min_size = cur_size;
-			}
-			byte_count += count;
 		}
+		byte_count += count;
+	}
 
-		/* hba wants the size to be exact */
-		if (byte_count > scsi_bufflen(scsicmd)) {
-			u32 temp = le32_to_cpu(rio2->sge[i-1].length) -
-				(byte_count - scsi_bufflen(scsicmd));
-			rio2->sge[i-1].length = cpu_to_le32(temp);
-			byte_count = scsi_bufflen(scsicmd);
-		}
+	/* hba wants the size to be exact */
+	if (byte_count > scsi_bufflen(scsicmd)) {
+		u32 temp = le32_to_cpu(rio2->sge[i-1].length) -
+			(byte_count - scsi_bufflen(scsicmd));
+		rio2->sge[i-1].length = cpu_to_le32(temp);
+		byte_count = scsi_bufflen(scsicmd);
+	}
 
-		rio2->sgeCnt = cpu_to_le32(nseg);
-		rio2->flags |= cpu_to_le16(RIO2_SG_FORMAT_IEEE1212);
-		/* not conformable: evaluate required sg elements */
-		if (!conformable) {
-			int j, nseg_new = nseg, err_found;
-			for (i = min_size / PAGE_SIZE; i >= 1; --i) {
-				err_found = 0;
-				nseg_new = 2;
-				for (j = 1; j < nseg - 1; ++j) {
-					if (rio2->sge[j].length % (i*PAGE_SIZE)) {
-						err_found = 1;
-						break;
-					}
-					nseg_new += (rio2->sge[j].length / (i*PAGE_SIZE));
-				}
-				if (!err_found)
+	rio2->sgeCnt = cpu_to_le32(nseg);
+	rio2->flags |= cpu_to_le16(RIO2_SG_FORMAT_IEEE1212);
+	/* not conformable: evaluate required sg elements */
+	if (!conformable) {
+		int j, nseg_new = nseg, err_found;
+		for (i = min_size / PAGE_SIZE; i >= 1; --i) {
+			err_found = 0;
+			nseg_new = 2;
+			for (j = 1; j < nseg - 1; ++j) {
+				if (rio2->sge[j].length % (i*PAGE_SIZE)) {
+					err_found = 1;
 					break;
+				}
+				nseg_new += (rio2->sge[j].length / (i*PAGE_SIZE));
 			}
-			if (i > 0 && nseg_new <= sg_max)
-				aac_convert_sgraw2(rio2, i, nseg, nseg_new);
-		} else
-			rio2->flags |= cpu_to_le16(RIO2_SGL_CONFORMANT);
+			if (!err_found)
+				break;
+		}
+		if (i > 0 && nseg_new <= sg_max) {
+			int ret = aac_convert_sgraw2(rio2, i, nseg, nseg_new);
 
-		/* Check for command underflow */
-		if (scsicmd->underflow && (byte_count < scsicmd->underflow)) {
-			printk(KERN_WARNING"aacraid: cmd len %08lX cmd underflow %08X\n",
-					byte_count, scsicmd->underflow);
+			if (ret < 0)
+				return ret;
 		}
+	} else
+		rio2->flags |= cpu_to_le16(RIO2_SGL_CONFORMANT);
+
+	/* Check for command underflow */
+	if (scsicmd->underflow && (byte_count < scsicmd->underflow)) {
+		printk(KERN_WARNING"aacraid: cmd len %08lX cmd underflow %08X\n",
+		       byte_count, scsicmd->underflow);
 	}
 
 	return byte_count;
@@ -3981,7 +3973,7 @@ static int aac_convert_sgraw2(struct aac_raw_io2 *rio2, int pages, int nseg, int
 
 	sge = kmalloc(nseg_new * sizeof(struct sge_ieee1212), GFP_ATOMIC);
 	if (sge == NULL)
-		return -1;
+		return -ENOMEM;
 
 	for (i = 1, pos = 1; i < nseg-1; ++i) {
 		for (j = 0; j < rio2->sge[i].length / (pages * PAGE_SIZE); ++j) {
diff --git a/drivers/scsi/aacraid/aacraid.h b/drivers/scsi/aacraid/aacraid.h
index d31a9bc2ba69..92fabf2b0c24 100644
--- a/drivers/scsi/aacraid/aacraid.h
+++ b/drivers/scsi/aacraid/aacraid.h
@@ -1723,6 +1723,7 @@ struct aac_dev
 #define FIB_CONTEXT_FLAG_FASTRESP		(0x00000008)
 #define FIB_CONTEXT_FLAG_NATIVE_HBA		(0x00000010)
 #define FIB_CONTEXT_FLAG_NATIVE_HBA_TMF	(0x00000020)
+#define FIB_CONTEXT_FLAG_SCSI_CMD	(0x00000040)
 
 /*
  *	Define the command values
@@ -2274,7 +2275,7 @@ struct aac_get_name_resp {
 	__le32		parm3;
 	__le32		parm4;
 	__le32		parm5;
-	u8		data[16];
+	u8		data[17];
 };
 
 #define CT_CID_TO_32BITS_UID 165
diff --git a/drivers/scsi/aacraid/comminit.c b/drivers/scsi/aacraid/comminit.c
index 9ee025b1d0e0..97d269f16888 100644
--- a/drivers/scsi/aacraid/comminit.c
+++ b/drivers/scsi/aacraid/comminit.c
@@ -520,9 +520,9 @@ struct aac_dev *aac_init_adapter(struct aac_dev *dev)
 			dev->raw_io_64 = 1;
 		dev->sync_mode = aac_sync_mode;
 		if (dev->a_ops.adapter_comm &&
-			(status[1] & AAC_OPT_NEW_COMM)) {
-				dev->comm_interface = AAC_COMM_MESSAGE;
-				dev->raw_io_interface = 1;
+		    (status[1] & AAC_OPT_NEW_COMM)) {
+			dev->comm_interface = AAC_COMM_MESSAGE;
+			dev->raw_io_interface = 1;
 			if ((status[1] & AAC_OPT_NEW_COMM_TYPE1)) {
 				/* driver supports TYPE1 (Tupelo) */
 				dev->comm_interface = AAC_COMM_MESSAGE_TYPE1;
diff --git a/drivers/scsi/aacraid/commsup.c b/drivers/scsi/aacraid/commsup.c
index 1c617ccfaf12..dfe8e70f8d99 100644
--- a/drivers/scsi/aacraid/commsup.c
+++ b/drivers/scsi/aacraid/commsup.c
@@ -770,7 +770,8 @@ int aac_hba_send(u8 command, struct fib *fibptr, fib_callback callback,
 		/* bit1 of request_id must be 0 */
 		hbacmd->request_id =
 			cpu_to_le32((((u32)(fibptr - dev->fibs)) << 2) + 1);
-	} else
+		fibptr->flags |= FIB_CONTEXT_FLAG_SCSI_CMD;
+	} else if (command != HBA_IU_TYPE_SCSI_TM_REQ)
 		return -EINVAL;
 
 
diff --git a/drivers/scsi/aacraid/linit.c b/drivers/scsi/aacraid/linit.c
index 0f277df73af0..87cc4a93e637 100644
--- a/drivers/scsi/aacraid/linit.c
+++ b/drivers/scsi/aacraid/linit.c
@@ -814,111 +814,225 @@ static int aac_eh_abort(struct scsi_cmnd* cmd)
 	return ret;
 }
 
+static u8 aac_eh_tmf_lun_reset_fib(struct aac_hba_map_info *info,
+				   struct fib *fib, u64 tmf_lun)
+{
+	struct aac_hba_tm_req *tmf;
+	u64 address;
+
+	/* start a HBA_TMF_LUN_RESET TMF request */
+	tmf = (struct aac_hba_tm_req *)fib->hw_fib_va;
+	memset(tmf, 0, sizeof(*tmf));
+	tmf->tmf = HBA_TMF_LUN_RESET;
+	tmf->it_nexus = info->rmw_nexus;
+	int_to_scsilun(tmf_lun, (struct scsi_lun *)tmf->lun);
+
+	address = (u64)fib->hw_error_pa;
+	tmf->error_ptr_hi = cpu_to_le32
+		((u32)(address >> 32));
+	tmf->error_ptr_lo = cpu_to_le32
+		((u32)(address & 0xffffffff));
+	tmf->error_length = cpu_to_le32(FW_ERROR_BUFFER_SIZE);
+	fib->hbacmd_size = sizeof(*tmf);
+
+	return HBA_IU_TYPE_SCSI_TM_REQ;
+}
+
+static u8 aac_eh_tmf_hard_reset_fib(struct aac_hba_map_info *info,
+				    struct fib *fib)
+{
+	struct aac_hba_reset_req *rst;
+	u64 address;
+
+	/* already tried, start a hard reset now */
+	rst = (struct aac_hba_reset_req *)fib->hw_fib_va;
+	memset(rst, 0, sizeof(*rst));
+	rst->it_nexus = info->rmw_nexus;
+
+	address = (u64)fib->hw_error_pa;
+	rst->error_ptr_hi = cpu_to_le32((u32)(address >> 32));
+	rst->error_ptr_lo = cpu_to_le32
+		((u32)(address & 0xffffffff));
+	rst->error_length = cpu_to_le32(FW_ERROR_BUFFER_SIZE);
+	fib->hbacmd_size = sizeof(*rst);
+
+       return HBA_IU_TYPE_SATA_REQ;
+}
+
+void aac_tmf_callback(void *context, struct fib *fibptr)
+{
+	struct aac_hba_resp *err =
+		&((struct aac_native_hba *)fibptr->hw_fib_va)->resp.err;
+	struct aac_hba_map_info *info = context;
+	int res;
+
+	switch (err->service_response) {
+	case HBA_RESP_SVCRES_TMF_REJECTED:
+		res = -1;
+		break;
+	case HBA_RESP_SVCRES_TMF_LUN_INVALID:
+		res = 0;
+		break;
+	case HBA_RESP_SVCRES_TMF_COMPLETE:
+	case HBA_RESP_SVCRES_TMF_SUCCEEDED:
+		res = 0;
+		break;
+	default:
+		res = -2;
+		break;
+	}
+	aac_fib_complete(fibptr);
+
+	info->reset_state = res;
+}
+
 /*
- *	aac_eh_reset	- Reset command handling
+ *	aac_eh_dev_reset	- Device reset command handling
  *	@scsi_cmd:	SCSI command block causing the reset
  *
  */
-static int aac_eh_reset(struct scsi_cmnd* cmd)
+static int aac_eh_dev_reset(struct scsi_cmnd *cmd)
 {
 	struct scsi_device * dev = cmd->device;
 	struct Scsi_Host * host = dev->host;
 	struct aac_dev * aac = (struct aac_dev *)host->hostdata;
+	struct aac_hba_map_info *info;
 	int count;
 	u32 bus, cid;
+	struct fib *fib;
 	int ret = FAILED;
-	int status = 0;
-	__le32 supported_options2 = 0;
-	bool is_mu_reset;
-	bool is_ignore_reset;
-	bool is_doorbell_reset;
-
+	int status;
+	u8 command;
 
 	bus = aac_logical_to_phys(scmd_channel(cmd));
 	cid = scmd_id(cmd);
-	if (bus < AAC_MAX_BUSES && cid < AAC_MAX_TARGETS &&
-		aac->hba_map[bus][cid].devtype == AAC_DEVTYPE_NATIVE_RAW) {
-		struct fib *fib;
-		int status;
-		u64 address;
-		u8 command;
+	info = &aac->hba_map[bus][cid];
+	if (bus >= AAC_MAX_BUSES || cid >= AAC_MAX_TARGETS ||
+	    info->devtype != AAC_DEVTYPE_NATIVE_RAW)
+		return FAILED;
 
-		pr_err("%s: Host adapter reset request. SCSI hang ?\n",
-			AAC_DRIVERNAME);
+	if (info->reset_state > 0)
+		return FAILED;
 
-		fib = aac_fib_alloc(aac);
-		if (!fib)
-			return ret;
+	pr_err("%s: Host adapter reset request. SCSI hang ?\n",
+	       AAC_DRIVERNAME);
+
+	fib = aac_fib_alloc(aac);
+	if (!fib)
+		return ret;
+
+	/* start a HBA_TMF_LUN_RESET TMF request */
+	command = aac_eh_tmf_lun_reset_fib(info, fib, dev->lun);
 
+	info->reset_state = 1;
 
-		if (aac->hba_map[bus][cid].reset_state == 0) {
-			struct aac_hba_tm_req *tmf;
-
-			/* start a HBA_TMF_LUN_RESET TMF request */
-			tmf = (struct aac_hba_tm_req *)fib->hw_fib_va;
-			memset(tmf, 0, sizeof(*tmf));
-			tmf->tmf = HBA_TMF_LUN_RESET;
-			tmf->it_nexus = aac->hba_map[bus][cid].rmw_nexus;
-			tmf->lun[1] = cmd->device->lun;
-
-			address = (u64)fib->hw_error_pa;
-			tmf->error_ptr_hi = cpu_to_le32
-					((u32)(address >> 32));
-			tmf->error_ptr_lo = cpu_to_le32
-					((u32)(address & 0xffffffff));
-			tmf->error_length = cpu_to_le32(FW_ERROR_BUFFER_SIZE);
-			fib->hbacmd_size = sizeof(*tmf);
-
-			command = HBA_IU_TYPE_SCSI_TM_REQ;
-			aac->hba_map[bus][cid].reset_state++;
-		} else if (aac->hba_map[bus][cid].reset_state >= 1) {
-			struct aac_hba_reset_req *rst;
-
-			/* already tried, start a hard reset now */
-			rst = (struct aac_hba_reset_req *)fib->hw_fib_va;
-			memset(rst, 0, sizeof(*rst));
-			/* reset_type is already zero... */
-			rst->it_nexus = aac->hba_map[bus][cid].rmw_nexus;
-
-			address = (u64)fib->hw_error_pa;
-			rst->error_ptr_hi = cpu_to_le32((u32)(address >> 32));
-			rst->error_ptr_lo = cpu_to_le32
-				((u32)(address & 0xffffffff));
-			rst->error_length = cpu_to_le32(FW_ERROR_BUFFER_SIZE);
-			fib->hbacmd_size = sizeof(*rst);
-
-			command = HBA_IU_TYPE_SATA_REQ;
-			aac->hba_map[bus][cid].reset_state = 0;
+	status = aac_hba_send(command, fib,
+			      (fib_callback) aac_tmf_callback,
+			      (void *) info);
+
+	/* Wait up to 15 seconds for completion */
+	for (count = 0; count < 15; ++count) {
+		if (info->reset_state == 0) {
+			ret = info->reset_state == 0 ? SUCCESS : FAILED;
+			break;
 		}
-		cmd->SCp.sent_command = 0;
+		msleep(1000);
+	}
 
-		status = aac_hba_send(command, fib,
-				  (fib_callback) aac_hba_callback,
-				  (void *) cmd);
+	return ret;
+}
 
-		/* Wait up to 15 seconds for completion */
-		for (count = 0; count < 15; ++count) {
-			if (cmd->SCp.sent_command) {
-				ret = SUCCESS;
-				break;
-			}
-			msleep(1000);
+/*
+ *	aac_eh_target_reset	- Target reset command handling
+ *	@scsi_cmd:	SCSI command block causing the reset
+ *
+ */
+static int aac_eh_target_reset(struct scsi_cmnd *cmd)
+{
+	struct scsi_device * dev = cmd->device;
+	struct Scsi_Host * host = dev->host;
+	struct aac_dev * aac = (struct aac_dev *)host->hostdata;
+	struct aac_hba_map_info *info;
+	int count;
+	u32 bus, cid;
+	int ret = FAILED;
+	struct fib *fib;
+	int status;
+	u8 command;
+
+	bus = aac_logical_to_phys(scmd_channel(cmd));
+	cid = scmd_id(cmd);
+	info = &aac->hba_map[bus][cid];
+	if (bus >= AAC_MAX_BUSES || cid >= AAC_MAX_TARGETS ||
+	    info->devtype != AAC_DEVTYPE_NATIVE_RAW)
+		return FAILED;
+
+	if (info->reset_state > 0)
+		return FAILED;
+
+	pr_err("%s: Host adapter reset request. SCSI hang ?\n",
+	       AAC_DRIVERNAME);
+
+	fib = aac_fib_alloc(aac);
+	if (!fib)
+		return ret;
+
+
+	/* already tried, start a hard reset now */
+	command = aac_eh_tmf_hard_reset_fib(info, fib);
+
+	info->reset_state = 2;
+
+	status = aac_hba_send(command, fib,
+			      (fib_callback) aac_tmf_callback,
+			      (void *) info);
+
+	/* Wait up to 15 seconds for completion */
+	for (count = 0; count < 15; ++count) {
+		if (info->reset_state <= 0) {
+			ret = info->reset_state == 0 ? SUCCESS : FAILED;
+			break;
 		}
+		msleep(1000);
+	}
 
-		if (ret == SUCCESS)
-			goto out;
+	return ret;
+}
 
-	} else {
+/*
+ *	aac_eh_bus_reset	- Bus reset command handling
+ *	@scsi_cmd:	SCSI command block causing the reset
+ *
+ */
+static int aac_eh_bus_reset(struct scsi_cmnd* cmd)
+{
+	struct scsi_device * dev = cmd->device;
+	struct Scsi_Host * host = dev->host;
+	struct aac_dev * aac = (struct aac_dev *)host->hostdata;
+	int count;
+	u32 cmd_bus;
+	int status = 0;
 
-		/* Mark the assoc. FIB to not complete, eh handler does this */
-		for (count = 0;
-			count < (host->can_queue + AAC_NUM_MGT_FIB);
-			++count) {
-			struct fib *fib = &aac->fibs[count];
 
-			if (fib->hw_fib_va->header.XferState &&
-				(fib->flags & FIB_CONTEXT_FLAG) &&
-				(fib->callback_data == cmd)) {
+	cmd_bus = aac_logical_to_phys(scmd_channel(cmd));
+	/* Mark the assoc. FIB to not complete, eh handler does this */
+	for (count = 0; count < (host->can_queue + AAC_NUM_MGT_FIB); ++count) {
+		struct fib *fib = &aac->fibs[count];
+
+		if (fib->hw_fib_va->header.XferState &&
+		    (fib->flags & FIB_CONTEXT_FLAG) &&
+		    (fib->flags & FIB_CONTEXT_FLAG_SCSI_CMD)) {
+			struct aac_hba_map_info *info;
+			u32 bus, cid;
+
+			cmd = (struct scsi_cmnd *)fib->callback_data;
+			bus = aac_logical_to_phys(scmd_channel(cmd));
+			if (bus != cmd_bus)
+				continue;
+			cid = scmd_id(cmd);
+			info = &aac->hba_map[bus][cid];
+			if (bus >= AAC_MAX_BUSES || cid >= AAC_MAX_TARGETS ||
+			    info->devtype != AAC_DEVTYPE_NATIVE_RAW) {
 				fib->flags |= FIB_CONTEXT_FLAG_TIMED_OUT;
 				cmd->SCp.phase = AAC_OWNER_ERROR_HANDLER;
 			}
@@ -935,8 +1049,24 @@ static int aac_eh_reset(struct scsi_cmnd* cmd)
 		dev_err(&aac->pdev->dev, "Adapter health - %d\n", status);
 
 	count = get_num_of_incomplete_fibs(aac);
-	if (count == 0)
-		return SUCCESS;
+	return (count == 0) ? SUCCESS : FAILED;
+}
+
+/*
+ *	aac_eh_host_reset	- Host reset command handling
+ *	@scsi_cmd:	SCSI command block causing the reset
+ *
+ */
+int aac_eh_host_reset(struct scsi_cmnd *cmd)
+{
+	struct scsi_device * dev = cmd->device;
+	struct Scsi_Host * host = dev->host;
+	struct aac_dev * aac = (struct aac_dev *)host->hostdata;
+	int ret = FAILED;
+	__le32 supported_options2 = 0;
+	bool is_mu_reset;
+	bool is_ignore_reset;
+	bool is_doorbell_reset;
 
 	/*
 	 * Check if reset is supported by the firmware
@@ -954,11 +1084,24 @@ static int aac_eh_reset(struct scsi_cmnd* cmd)
 	 && aac_check_reset
 	 && (aac_check_reset != -1 || !is_ignore_reset)) {
 		/* Bypass wait for command quiesce */
-		aac_reset_adapter(aac, 2, IOP_HWSOFT_RESET);
+		if (aac_reset_adapter(aac, 2, IOP_HWSOFT_RESET) == 0)
+			ret = SUCCESS;
+	}
+	/*
+	 * Reset EH state
+	 */
+	if (ret == SUCCESS) {
+		int bus, cid;
+		struct aac_hba_map_info *info;
+
+		for (bus = 0; bus < AAC_MAX_BUSES; bus++) {
+			for (cid = 0; cid < AAC_MAX_TARGETS; cid++) {
+				info = &aac->hba_map[bus][cid];
+				if (info->devtype == AAC_DEVTYPE_NATIVE_RAW)
+					info->reset_state = 0;
+			}
+		}
 	}
-	ret = SUCCESS;
-
-out:
 	return ret;
 }
 
@@ -1382,7 +1525,10 @@ static struct scsi_host_template aac_driver_template = {
 	.change_queue_depth		= aac_change_queue_depth,
 	.sdev_attrs			= aac_dev_attrs,
 	.eh_abort_handler		= aac_eh_abort,
-	.eh_host_reset_handler		= aac_eh_reset,
+	.eh_device_reset_handler	= aac_eh_dev_reset,
+	.eh_target_reset_handler	= aac_eh_target_reset,
+	.eh_bus_reset_handler		= aac_eh_bus_reset,
+	.eh_host_reset_handler		= aac_eh_host_reset,
 	.can_queue			= AAC_NUM_IO_FIB,
 	.this_id			= MAXIMUM_NUM_CONTAINERS,
 	.sg_tablesize			= 16,
@@ -1457,7 +1603,7 @@ static int aac_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	/*
 	 * Only series 7 needs freset.
 	 */
-	 if (pdev->device == PMC_DEVICE_S7)
+	if (pdev->device == PMC_DEVICE_S7)
 		pdev->needs_freset = 1;
 
 	list_for_each_entry(aac, &aac_devices, entry) {
diff --git a/drivers/scsi/aha152x.c b/drivers/scsi/aha152x.c
index ce5dc73d85bb..bc0058df31c6 100644
--- a/drivers/scsi/aha152x.c
+++ b/drivers/scsi/aha152x.c
@@ -1140,6 +1140,9 @@ static void free_hard_reset_SCs(struct Scsi_Host *shpnt, Scsi_Cmnd **SCs)
 /*
  * Reset the bus
  *
+ * AIC-6260 has a hard reset (MRST signal), but apparently
+ * one cannot trigger it via software. So live with
+ * a soft reset; no-one seemed to have cared.
  */
 static int aha152x_bus_reset_host(struct Scsi_Host *shpnt)
 {
@@ -1223,15 +1226,6 @@ int aha152x_host_reset_host(struct Scsi_Host *shpnt)
 }
 
 /*
- * Reset the host (bus and controller)
- *
- */
-static int aha152x_host_reset(Scsi_Cmnd *SCpnt)
-{
-	return aha152x_host_reset_host(SCpnt->device->host);
-}
-
-/*
  * Return the "logical geometry"
  *
  */
@@ -2917,7 +2911,6 @@ static struct scsi_host_template aha152x_driver_template = {
 	.eh_abort_handler		= aha152x_abort,
 	.eh_device_reset_handler	= aha152x_device_reset,
 	.eh_bus_reset_handler		= aha152x_bus_reset,
-	.eh_host_reset_handler		= aha152x_host_reset,
 	.bios_param			= aha152x_biosparam,
 	.can_queue			= 1,
 	.this_id			= 7,
diff --git a/drivers/scsi/aha1542.c b/drivers/scsi/aha1542.c
index a23cc9ac5acd..124217927c4a 100644
--- a/drivers/scsi/aha1542.c
+++ b/drivers/scsi/aha1542.c
@@ -986,7 +986,7 @@ static struct isa_driver aha1542_isa_driver = {
 static int isa_registered;
 
 #ifdef CONFIG_PNP
-static struct pnp_device_id aha1542_pnp_ids[] = {
+static const struct pnp_device_id aha1542_pnp_ids[] = {
 	{ .id = "ADP1542" },
 	{ .id = "" }
 };
diff --git a/drivers/scsi/aic7xxx/Makefile b/drivers/scsi/aic7xxx/Makefile
index 07b60a780c06..b03ba0df7a83 100644
--- a/drivers/scsi/aic7xxx/Makefile
+++ b/drivers/scsi/aic7xxx/Makefile
@@ -59,7 +59,8 @@ $(obj)/aic7xxx_seq.h: $(src)/aic7xxx.seq $(src)/aic7xxx.reg $(obj)/aicasm/aicasm
 			      $(aicasm-7xxx-opts-y) -o $(obj)/aic7xxx_seq.h \
 			      $(srctree)/$(src)/aic7xxx.seq
 
-$(aic7xxx-gen-y): $(obj)/aic7xxx_seq.h
+$(aic7xxx-gen-y): $(objtree)/$(obj)/aic7xxx_seq.h
+	@true
 else
 $(obj)/aic7xxx_reg_print.c: $(src)/aic7xxx_reg_print.c_shipped
 endif
@@ -76,7 +77,8 @@ $(obj)/aic79xx_seq.h: $(src)/aic79xx.seq $(src)/aic79xx.reg $(obj)/aicasm/aicasm
 			      $(aicasm-79xx-opts-y) -o $(obj)/aic79xx_seq.h \
 			      $(srctree)/$(src)/aic79xx.seq
 
-$(aic79xx-gen-y): $(obj)/aic79xx_seq.h
+$(aic79xx-gen-y): $(objtree)/$(obj)/aic79xx_seq.h
+	@true
 else
 $(obj)/aic79xx_reg_print.c: $(src)/aic79xx_reg_print.c_shipped
 endif
diff --git a/drivers/scsi/aic7xxx/aic79xx_reg.h_shipped b/drivers/scsi/aic7xxx/aic79xx_reg.h_shipped
index cdcead071ef6..ddcd5a7701ac 100644
--- a/drivers/scsi/aic7xxx/aic79xx_reg.h_shipped
+++ b/drivers/scsi/aic7xxx/aic79xx_reg.h_shipped
@@ -13,13 +13,6 @@ typedef struct ahd_reg_parse_entry {
 } ahd_reg_parse_entry_t;
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_mode_ptr_print;
-#else
-#define ahd_mode_ptr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "MODE_PTR", 0x00, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_intstat_print;
 #else
 #define ahd_intstat_print(regvalue, cur_col, wrap) \
@@ -27,27 +20,6 @@ ahd_reg_print_t ahd_intstat_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_seqintcode_print;
-#else
-#define ahd_seqintcode_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SEQINTCODE", 0x02, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_error_print;
-#else
-#define ahd_error_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "ERROR", 0x04, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_hescb_qoff_print;
-#else
-#define ahd_hescb_qoff_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "HESCB_QOFF", 0x08, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_hs_mailbox_print;
 #else
 #define ahd_hs_mailbox_print(regvalue, cur_col, wrap) \
@@ -62,27 +34,6 @@ ahd_reg_print_t ahd_seqintstat_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_clrseqintstat_print;
-#else
-#define ahd_clrseqintstat_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "CLRSEQINTSTAT", 0x0c, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_swtimer_print;
-#else
-#define ahd_swtimer_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SWTIMER", 0x0e, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_sescb_qoff_print;
-#else
-#define ahd_sescb_qoff_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SESCB_QOFF", 0x12, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_intctl_print;
 #else
 #define ahd_intctl_print(regvalue, cur_col, wrap) \
@@ -111,111 +62,6 @@ ahd_reg_print_t ahd_sg_cache_shadow_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_lqin_print;
-#else
-#define ahd_lqin_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "LQIN", 0x20, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_lunptr_print;
-#else
-#define ahd_lunptr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "LUNPTR", 0x22, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_cmdlenptr_print;
-#else
-#define ahd_cmdlenptr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "CMDLENPTR", 0x25, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_attrptr_print;
-#else
-#define ahd_attrptr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "ATTRPTR", 0x26, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_flagptr_print;
-#else
-#define ahd_flagptr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "FLAGPTR", 0x27, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_cmdptr_print;
-#else
-#define ahd_cmdptr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "CMDPTR", 0x28, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_qnextptr_print;
-#else
-#define ahd_qnextptr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "QNEXTPTR", 0x29, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_abrtbyteptr_print;
-#else
-#define ahd_abrtbyteptr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "ABRTBYTEPTR", 0x2b, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_abrtbitptr_print;
-#else
-#define ahd_abrtbitptr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "ABRTBITPTR", 0x2c, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_lunlen_print;
-#else
-#define ahd_lunlen_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "LUNLEN", 0x30, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_cdblimit_print;
-#else
-#define ahd_cdblimit_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "CDBLIMIT", 0x31, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_maxcmd_print;
-#else
-#define ahd_maxcmd_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "MAXCMD", 0x32, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_maxcmdcnt_print;
-#else
-#define ahd_maxcmdcnt_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "MAXCMDCNT", 0x33, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_lqctl1_print;
-#else
-#define ahd_lqctl1_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "LQCTL1", 0x38, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_lqctl2_print;
-#else
-#define ahd_lqctl2_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "LQCTL2", 0x39, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_scsiseq0_print;
 #else
 #define ahd_scsiseq0_print(regvalue, cur_col, wrap) \
@@ -230,13 +76,6 @@ ahd_reg_print_t ahd_scsiseq1_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_sxfrctl0_print;
-#else
-#define ahd_sxfrctl0_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SXFRCTL0", 0x3c, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_dffstat_print;
 #else
 #define ahd_dffstat_print(regvalue, cur_col, wrap) \
@@ -244,13 +83,6 @@ ahd_reg_print_t ahd_dffstat_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_multargid_print;
-#else
-#define ahd_multargid_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "MULTARGID", 0x40, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_scsisigi_print;
 #else
 #define ahd_scsisigi_print(regvalue, cur_col, wrap) \
@@ -265,13 +97,6 @@ ahd_reg_print_t ahd_scsiphase_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_scsidat_print;
-#else
-#define ahd_scsidat_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SCSIDAT", 0x44, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_scsibus_print;
 #else
 #define ahd_scsibus_print(regvalue, cur_col, wrap) \
@@ -279,13 +104,6 @@ ahd_reg_print_t ahd_scsibus_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_targidin_print;
-#else
-#define ahd_targidin_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "TARGIDIN", 0x48, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_selid_print;
 #else
 #define ahd_selid_print(regvalue, cur_col, wrap) \
@@ -293,10 +111,10 @@ ahd_reg_print_t ahd_selid_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_sblkctl_print;
+ahd_reg_print_t ahd_simode0_print;
 #else
-#define ahd_sblkctl_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SBLKCTL", 0x4a, regvalue, cur_col, wrap)
+#define ahd_simode0_print(regvalue, cur_col, wrap) \
+    ahd_print_register(NULL, 0, "SIMODE0", 0x4b, regvalue, cur_col, wrap)
 #endif
 
 #if AIC_DEBUG_REGISTERS
@@ -307,13 +125,6 @@ ahd_reg_print_t ahd_sstat0_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_simode0_print;
-#else
-#define ahd_simode0_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SIMODE0", 0x4b, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_sstat1_print;
 #else
 #define ahd_sstat1_print(regvalue, cur_col, wrap) \
@@ -328,13 +139,6 @@ ahd_reg_print_t ahd_sstat2_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_clrsint2_print;
-#else
-#define ahd_clrsint2_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "CLRSINT2", 0x4d, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_perrdiag_print;
 #else
 #define ahd_perrdiag_print(regvalue, cur_col, wrap) \
@@ -342,13 +146,6 @@ ahd_reg_print_t ahd_perrdiag_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_lqistate_print;
-#else
-#define ahd_lqistate_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "LQISTATE", 0x4e, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_soffcnt_print;
 #else
 #define ahd_soffcnt_print(regvalue, cur_col, wrap) \
@@ -356,13 +153,6 @@ ahd_reg_print_t ahd_soffcnt_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_lqostate_print;
-#else
-#define ahd_lqostate_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "LQOSTATE", 0x4f, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_lqistat0_print;
 #else
 #define ahd_lqistat0_print(regvalue, cur_col, wrap) \
@@ -370,27 +160,6 @@ ahd_reg_print_t ahd_lqistat0_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_clrlqiint0_print;
-#else
-#define ahd_clrlqiint0_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "CLRLQIINT0", 0x50, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_lqimode0_print;
-#else
-#define ahd_lqimode0_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "LQIMODE0", 0x50, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_lqimode1_print;
-#else
-#define ahd_lqimode1_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "LQIMODE1", 0x51, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_lqistat1_print;
 #else
 #define ahd_lqistat1_print(regvalue, cur_col, wrap) \
@@ -398,13 +167,6 @@ ahd_reg_print_t ahd_lqistat1_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_clrlqiint1_print;
-#else
-#define ahd_clrlqiint1_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "CLRLQIINT1", 0x51, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_lqistat2_print;
 #else
 #define ahd_lqistat2_print(regvalue, cur_col, wrap) \
@@ -419,20 +181,6 @@ ahd_reg_print_t ahd_sstat3_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_simode3_print;
-#else
-#define ahd_simode3_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SIMODE3", 0x53, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_clrsint3_print;
-#else
-#define ahd_clrsint3_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "CLRSINT3", 0x53, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_lqostat0_print;
 #else
 #define ahd_lqostat0_print(regvalue, cur_col, wrap) \
@@ -440,27 +188,6 @@ ahd_reg_print_t ahd_lqostat0_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_clrlqoint0_print;
-#else
-#define ahd_clrlqoint0_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "CLRLQOINT0", 0x54, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_lqomode0_print;
-#else
-#define ahd_lqomode0_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "LQOMODE0", 0x54, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_lqomode1_print;
-#else
-#define ahd_lqomode1_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "LQOMODE1", 0x55, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_lqostat1_print;
 #else
 #define ahd_lqostat1_print(regvalue, cur_col, wrap) \
@@ -468,13 +195,6 @@ ahd_reg_print_t ahd_lqostat1_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_clrlqoint1_print;
-#else
-#define ahd_clrlqoint1_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "CLRLQOINT1", 0x55, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_lqostat2_print;
 #else
 #define ahd_lqostat2_print(regvalue, cur_col, wrap) \
@@ -482,13 +202,6 @@ ahd_reg_print_t ahd_lqostat2_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_os_space_cnt_print;
-#else
-#define ahd_os_space_cnt_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "OS_SPACE_CNT", 0x56, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_simode1_print;
 #else
 #define ahd_simode1_print(regvalue, cur_col, wrap) \
@@ -496,13 +209,6 @@ ahd_reg_print_t ahd_simode1_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_gsfifo_print;
-#else
-#define ahd_gsfifo_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "GSFIFO", 0x58, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_dffsxfrctl_print;
 #else
 #define ahd_dffsxfrctl_print(regvalue, cur_col, wrap) \
@@ -510,27 +216,6 @@ ahd_reg_print_t ahd_dffsxfrctl_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_lqoscsctl_print;
-#else
-#define ahd_lqoscsctl_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "LQOSCSCTL", 0x5a, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_nextscb_print;
-#else
-#define ahd_nextscb_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "NEXTSCB", 0x5a, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_clrseqintsrc_print;
-#else
-#define ahd_clrseqintsrc_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "CLRSEQINTSRC", 0x5b, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_seqintsrc_print;
 #else
 #define ahd_seqintsrc_print(regvalue, cur_col, wrap) \
@@ -538,13 +223,6 @@ ahd_reg_print_t ahd_seqintsrc_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_currscb_print;
-#else
-#define ahd_currscb_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "CURRSCB", 0x5c, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_seqimode_print;
 #else
 #define ahd_seqimode_print(regvalue, cur_col, wrap) \
@@ -559,90 +237,6 @@ ahd_reg_print_t ahd_mdffstat_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_lastscb_print;
-#else
-#define ahd_lastscb_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "LASTSCB", 0x5e, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_negoaddr_print;
-#else
-#define ahd_negoaddr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "NEGOADDR", 0x60, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_negperiod_print;
-#else
-#define ahd_negperiod_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "NEGPERIOD", 0x61, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_negoffset_print;
-#else
-#define ahd_negoffset_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "NEGOFFSET", 0x62, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_negppropts_print;
-#else
-#define ahd_negppropts_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "NEGPPROPTS", 0x63, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_negconopts_print;
-#else
-#define ahd_negconopts_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "NEGCONOPTS", 0x64, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_annexcol_print;
-#else
-#define ahd_annexcol_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "ANNEXCOL", 0x65, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_annexdat_print;
-#else
-#define ahd_annexdat_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "ANNEXDAT", 0x66, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_scschkn_print;
-#else
-#define ahd_scschkn_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SCSCHKN", 0x66, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_iownid_print;
-#else
-#define ahd_iownid_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "IOWNID", 0x67, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_shcnt_print;
-#else
-#define ahd_shcnt_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SHCNT", 0x68, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_townid_print;
-#else
-#define ahd_townid_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "TOWNID", 0x69, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_seloid_print;
 #else
 #define ahd_seloid_print(regvalue, cur_col, wrap) \
@@ -650,90 +244,6 @@ ahd_reg_print_t ahd_seloid_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_scbhaddr_print;
-#else
-#define ahd_scbhaddr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SCBHADDR", 0x7c, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_sghaddr_print;
-#else
-#define ahd_sghaddr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SGHADDR", 0x7c, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_scbhcnt_print;
-#else
-#define ahd_scbhcnt_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SCBHCNT", 0x84, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_sghcnt_print;
-#else
-#define ahd_sghcnt_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SGHCNT", 0x84, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_pcixctl_print;
-#else
-#define ahd_pcixctl_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "PCIXCTL", 0x93, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_dchspltstat0_print;
-#else
-#define ahd_dchspltstat0_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "DCHSPLTSTAT0", 0x96, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_dchspltstat1_print;
-#else
-#define ahd_dchspltstat1_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "DCHSPLTSTAT1", 0x97, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_sgspltstat0_print;
-#else
-#define ahd_sgspltstat0_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SGSPLTSTAT0", 0x9e, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_sgspltstat1_print;
-#else
-#define ahd_sgspltstat1_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SGSPLTSTAT1", 0x9f, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_df0pcistat_print;
-#else
-#define ahd_df0pcistat_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "DF0PCISTAT", 0xa0, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_reg0_print;
-#else
-#define ahd_reg0_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "REG0", 0xa0, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_reg_isr_print;
-#else
-#define ahd_reg_isr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "REG_ISR", 0xa4, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_sg_state_print;
 #else
 #define ahd_sg_state_print(regvalue, cur_col, wrap) \
@@ -741,27 +251,6 @@ ahd_reg_print_t ahd_sg_state_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_targpcistat_print;
-#else
-#define ahd_targpcistat_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "TARGPCISTAT", 0xa7, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_scbautoptr_print;
-#else
-#define ahd_scbautoptr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SCBAUTOPTR", 0xab, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_ccscbaddr_print;
-#else
-#define ahd_ccscbaddr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "CCSCBADDR", 0xac, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_ccscbctl_print;
 #else
 #define ahd_ccscbctl_print(regvalue, cur_col, wrap) \
@@ -776,69 +265,6 @@ ahd_reg_print_t ahd_ccsgctl_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_ccscbram_print;
-#else
-#define ahd_ccscbram_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "CCSCBRAM", 0xb0, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_brddat_print;
-#else
-#define ahd_brddat_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "BRDDAT", 0xb8, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_seeadr_print;
-#else
-#define ahd_seeadr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SEEADR", 0xba, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_seedat_print;
-#else
-#define ahd_seedat_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SEEDAT", 0xbc, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_seectl_print;
-#else
-#define ahd_seectl_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SEECTL", 0xbe, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_seestat_print;
-#else
-#define ahd_seestat_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SEESTAT", 0xbe, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_dspdatactl_print;
-#else
-#define ahd_dspdatactl_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "DSPDATACTL", 0xc1, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_dspselect_print;
-#else
-#define ahd_dspselect_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "DSPSELECT", 0xc4, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_wrtbiasctl_print;
-#else
-#define ahd_wrtbiasctl_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "WRTBIASCTL", 0xc5, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_seqctl0_print;
 #else
 #define ahd_seqctl0_print(regvalue, cur_col, wrap) \
@@ -853,62 +279,6 @@ ahd_reg_print_t ahd_seqintctl_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_prgmcnt_print;
-#else
-#define ahd_prgmcnt_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "PRGMCNT", 0xde, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_none_print;
-#else
-#define ahd_none_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "NONE", 0xea, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_intvec1_addr_print;
-#else
-#define ahd_intvec1_addr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "INTVEC1_ADDR", 0xf4, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_curaddr_print;
-#else
-#define ahd_curaddr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "CURADDR", 0xf4, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_intvec2_addr_print;
-#else
-#define ahd_intvec2_addr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "INTVEC2_ADDR", 0xf6, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_longjmp_addr_print;
-#else
-#define ahd_longjmp_addr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "LONGJMP_ADDR", 0xf8, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_accum_save_print;
-#else
-#define ahd_accum_save_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "ACCUM_SAVE", 0xfa, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_waiting_scb_tails_print;
-#else
-#define ahd_waiting_scb_tails_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "WAITING_SCB_TAILS", 0x100, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_sram_base_print;
 #else
 #define ahd_sram_base_print(regvalue, cur_col, wrap) \
@@ -916,62 +286,6 @@ ahd_reg_print_t ahd_sram_base_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_waiting_tid_head_print;
-#else
-#define ahd_waiting_tid_head_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "WAITING_TID_HEAD", 0x120, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_waiting_tid_tail_print;
-#else
-#define ahd_waiting_tid_tail_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "WAITING_TID_TAIL", 0x122, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_next_queued_scb_addr_print;
-#else
-#define ahd_next_queued_scb_addr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "NEXT_QUEUED_SCB_ADDR", 0x124, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_complete_scb_head_print;
-#else
-#define ahd_complete_scb_head_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "COMPLETE_SCB_HEAD", 0x128, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_complete_scb_dmainprog_head_print;
-#else
-#define ahd_complete_scb_dmainprog_head_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "COMPLETE_SCB_DMAINPROG_HEAD", 0x12a, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_complete_dma_scb_head_print;
-#else
-#define ahd_complete_dma_scb_head_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "COMPLETE_DMA_SCB_HEAD", 0x12c, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_complete_dma_scb_tail_print;
-#else
-#define ahd_complete_dma_scb_tail_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "COMPLETE_DMA_SCB_TAIL", 0x12e, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_complete_on_qfreeze_head_print;
-#else
-#define ahd_complete_on_qfreeze_head_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "COMPLETE_ON_QFREEZE_HEAD", 0x130, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_qfreeze_count_print;
 #else
 #define ahd_qfreeze_count_print(regvalue, cur_col, wrap) \
@@ -993,13 +307,6 @@ ahd_reg_print_t ahd_saved_mode_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_msg_out_print;
-#else
-#define ahd_msg_out_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "MSG_OUT", 0x137, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_seq_flags_print;
 #else
 #define ahd_seq_flags_print(regvalue, cur_col, wrap) \
@@ -1014,48 +321,6 @@ ahd_reg_print_t ahd_lastphase_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_qoutfifo_entry_valid_tag_print;
-#else
-#define ahd_qoutfifo_entry_valid_tag_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "QOUTFIFO_ENTRY_VALID_TAG", 0x13d, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_kernel_tqinpos_print;
-#else
-#define ahd_kernel_tqinpos_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "KERNEL_TQINPOS", 0x13e, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_qoutfifo_next_addr_print;
-#else
-#define ahd_qoutfifo_next_addr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "QOUTFIFO_NEXT_ADDR", 0x144, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_last_msg_print;
-#else
-#define ahd_last_msg_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "LAST_MSG", 0x14a, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_scsiseq_template_print;
-#else
-#define ahd_scsiseq_template_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SCSISEQ_TEMPLATE", 0x14b, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_initiator_tag_print;
-#else
-#define ahd_initiator_tag_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "INITIATOR_TAG", 0x14c, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_seq_flags2_print;
 #else
 #define ahd_seq_flags2_print(regvalue, cur_col, wrap) \
@@ -1063,62 +328,6 @@ ahd_reg_print_t ahd_seq_flags2_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_allocfifo_scbptr_print;
-#else
-#define ahd_allocfifo_scbptr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "ALLOCFIFO_SCBPTR", 0x14e, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_int_coalescing_timer_print;
-#else
-#define ahd_int_coalescing_timer_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "INT_COALESCING_TIMER", 0x150, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_int_coalescing_maxcmds_print;
-#else
-#define ahd_int_coalescing_maxcmds_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "INT_COALESCING_MAXCMDS", 0x152, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_int_coalescing_mincmds_print;
-#else
-#define ahd_int_coalescing_mincmds_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "INT_COALESCING_MINCMDS", 0x153, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_cmds_pending_print;
-#else
-#define ahd_cmds_pending_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "CMDS_PENDING", 0x154, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_int_coalescing_cmdcount_print;
-#else
-#define ahd_int_coalescing_cmdcount_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "INT_COALESCING_CMDCOUNT", 0x156, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_local_hs_mailbox_print;
-#else
-#define ahd_local_hs_mailbox_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "LOCAL_HS_MAILBOX", 0x157, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_cmdsize_table_print;
-#else
-#define ahd_cmdsize_table_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "CMDSIZE_TABLE", 0x158, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_mk_message_scb_print;
 #else
 #define ahd_mk_message_scb_print(regvalue, cur_col, wrap) \
@@ -1140,27 +349,6 @@ ahd_reg_print_t ahd_scb_base_print;
 #endif
 
 #if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_scb_residual_datacnt_print;
-#else
-#define ahd_scb_residual_datacnt_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SCB_RESIDUAL_DATACNT", 0x180, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_scb_sense_busaddr_print;
-#else
-#define ahd_scb_sense_busaddr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SCB_SENSE_BUSADDR", 0x18c, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_scb_tag_print;
-#else
-#define ahd_scb_tag_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SCB_TAG", 0x190, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
 ahd_reg_print_t ahd_scb_control_print;
 #else
 #define ahd_scb_control_print(regvalue, cur_col, wrap) \
@@ -1174,69 +362,6 @@ ahd_reg_print_t ahd_scb_scsiid_print;
     ahd_print_register(NULL, 0, "SCB_SCSIID", 0x193, regvalue, cur_col, wrap)
 #endif
 
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_scb_lun_print;
-#else
-#define ahd_scb_lun_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SCB_LUN", 0x194, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_scb_task_attribute_print;
-#else
-#define ahd_scb_task_attribute_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SCB_TASK_ATTRIBUTE", 0x195, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_scb_task_management_print;
-#else
-#define ahd_scb_task_management_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SCB_TASK_MANAGEMENT", 0x197, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_scb_dataptr_print;
-#else
-#define ahd_scb_dataptr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SCB_DATAPTR", 0x198, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_scb_datacnt_print;
-#else
-#define ahd_scb_datacnt_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SCB_DATACNT", 0x1a0, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_scb_sgptr_print;
-#else
-#define ahd_scb_sgptr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SCB_SGPTR", 0x1a4, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_scb_busaddr_print;
-#else
-#define ahd_scb_busaddr_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SCB_BUSADDR", 0x1a8, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_scb_next2_print;
-#else
-#define ahd_scb_next2_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SCB_NEXT2", 0x1ae, regvalue, cur_col, wrap)
-#endif
-
-#if AIC_DEBUG_REGISTERS
-ahd_reg_print_t ahd_scb_disconnected_lists_print;
-#else
-#define ahd_scb_disconnected_lists_print(regvalue, cur_col, wrap) \
-    ahd_print_register(NULL, 0, "SCB_DISCONNECTED_LISTS", 0x1b8, regvalue, cur_col, wrap)
-#endif
-
 
 #define	MODE_PTR        		0x00
 #define		DST_MODE        	0x70
@@ -1292,15 +417,6 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		CLRCMDINT       	0x02
 #define		CLRSPLTINT      	0x01
 
-#define	ERROR           		0x04
-#define		CIOPARERR       	0x80
-#define		CIOACCESFAIL    	0x40
-#define		MPARERR         	0x20
-#define		DPARERR         	0x10
-#define		SQPARERR        	0x08
-#define		ILLOPCODE       	0x04
-#define		DSCTMOUT        	0x02
-
 #define	CLRERR          		0x04
 #define		CLRCIOPARERR    	0x80
 #define		CLRCIOACCESFAIL 	0x40
@@ -1310,6 +426,15 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		CLRILLOPCODE    	0x04
 #define		CLRDSCTMOUT     	0x02
 
+#define	ERROR           		0x04
+#define		CIOPARERR       	0x80
+#define		CIOACCESFAIL    	0x40
+#define		MPARERR         	0x20
+#define		DPARERR         	0x10
+#define		SQPARERR        	0x08
+#define		ILLOPCODE       	0x04
+#define		DSCTMOUT        	0x02
+
 #define	HCNTRL          		0x05
 #define		SEQ_RESET       	0x80
 #define		POWRDN          	0x40
@@ -1404,22 +529,22 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		FIFOFULL        	0x02
 #define		FIFOEMP         	0x01
 
-#define	SG_CACHE_SHADOW 		0x1b
-#define		ODD_SEG         	0x04
-#define		LAST_SEG        	0x02
-#define		LAST_SEG_DONE   	0x01
-
 #define	ARBCTL          		0x1b
 #define		RESET_HARB      	0x80
 #define		RETRY_SWEN      	0x08
 #define		USE_TIME        	0x07
 
-#define	SG_CACHE_PRE    		0x1b
+#define	SG_CACHE_SHADOW 		0x1b
+#define		ODD_SEG         	0x04
+#define		LAST_SEG        	0x02
+#define		LAST_SEG_DONE   	0x01
 
-#define	LQIN            		0x20
+#define	SG_CACHE_PRE    		0x1b
 
 #define	TYPEPTR         		0x20
 
+#define	LQIN            		0x20
+
 #define	TAGPTR          		0x21
 
 #define	LUNPTR          		0x22
@@ -1479,14 +604,6 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		SINGLECMD       	0x02
 #define		ABORTPENDING    	0x01
 
-#define	SCSBIST0        		0x39
-#define		GSBISTERR       	0x40
-#define		GSBISTDONE      	0x20
-#define		GSBISTRUN       	0x10
-#define		OSBISTERR       	0x04
-#define		OSBISTDONE      	0x02
-#define		OSBISTRUN       	0x01
-
 #define	LQCTL2          		0x39
 #define		LQIRETRY        	0x80
 #define		LQICONTINUE     	0x40
@@ -1497,10 +614,13 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		LQOTOIDLE       	0x02
 #define		LQOPAUSE        	0x01
 
-#define	SCSBIST1        		0x3a
-#define		NTBISTERR       	0x04
-#define		NTBISTDONE      	0x02
-#define		NTBISTRUN       	0x01
+#define	SCSBIST0        		0x39
+#define		GSBISTERR       	0x40
+#define		GSBISTDONE      	0x20
+#define		GSBISTRUN       	0x10
+#define		OSBISTERR       	0x04
+#define		OSBISTDONE      	0x02
+#define		OSBISTRUN       	0x01
 
 #define	SCSISEQ0        		0x3a
 #define		TEMODEO         	0x80
@@ -1509,8 +629,15 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		FORCEBUSFREE    	0x10
 #define		SCSIRSTO        	0x01
 
+#define	SCSBIST1        		0x3a
+#define		NTBISTERR       	0x04
+#define		NTBISTDONE      	0x02
+#define		NTBISTRUN       	0x01
+
 #define	SCSISEQ1        		0x3b
 
+#define	BUSINITID       		0x3c
+
 #define	SXFRCTL0        		0x3c
 #define		DFON            	0x80
 #define		DFPEXP          	0x40
@@ -1519,8 +646,6 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 
 #define	DLCOUNT         		0x3c
 
-#define	BUSINITID       		0x3c
-
 #define	SXFRCTL1        		0x3d
 #define		BITBUCKET       	0x80
 #define		ENSACHK         	0x40
@@ -1545,6 +670,8 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		CURRFIFO_1      	0x01
 #define		CURRFIFO_0      	0x00
 
+#define	MULTARGID       		0x40
+
 #define	SCSISIGO        		0x40
 #define		CDO             	0x80
 #define		IOO             	0x40
@@ -1555,8 +682,6 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		REQO            	0x02
 #define		ACKO            	0x01
 
-#define	MULTARGID       		0x40
-
 #define	SCSISIGI        		0x41
 #define		ATNI            	0x10
 #define		SELI            	0x08
@@ -1603,14 +728,14 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		ENAB20          	0x04
 #define		SELWIDE         	0x02
 
-#define	CLRSINT0        		0x4b
-#define		CLRSELDO        	0x40
-#define		CLRSELDI        	0x20
-#define		CLRSELINGO      	0x10
-#define		CLRIOERR        	0x08
-#define		CLROVERRUN      	0x04
-#define		CLRSPIORDY      	0x02
-#define		CLRARBDO        	0x01
+#define	SIMODE0         		0x4b
+#define		ENSELDO         	0x40
+#define		ENSELDI         	0x20
+#define		ENSELINGO       	0x10
+#define		ENIOERR         	0x08
+#define		ENOVERRUN       	0x04
+#define		ENSPIORDY       	0x02
+#define		ENARBDO         	0x01
 
 #define	SSTAT0          		0x4b
 #define		TARGET          	0x80
@@ -1622,23 +747,14 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		SPIORDY         	0x02
 #define		ARBDO           	0x01
 
-#define	SIMODE0         		0x4b
-#define		ENSELDO         	0x40
-#define		ENSELDI         	0x20
-#define		ENSELINGO       	0x10
-#define		ENIOERR         	0x08
-#define		ENOVERRUN       	0x04
-#define		ENSPIORDY       	0x02
-#define		ENARBDO         	0x01
-
-#define	CLRSINT1        		0x4c
-#define		CLRSELTIMEO     	0x80
-#define		CLRATNO         	0x40
-#define		CLRSCSIRSTI     	0x20
-#define		CLRBUSFREE      	0x08
-#define		CLRSCSIPERR     	0x04
-#define		CLRSTRB2FAST    	0x02
-#define		CLRREQINIT      	0x01
+#define	CLRSINT0        		0x4b
+#define		CLRSELDO        	0x40
+#define		CLRSELDI        	0x20
+#define		CLRSELINGO      	0x10
+#define		CLRIOERR        	0x08
+#define		CLROVERRUN      	0x04
+#define		CLRSPIORDY      	0x02
+#define		CLRARBDO        	0x01
 
 #define	SSTAT1          		0x4c
 #define		SELTO           	0x80
@@ -1650,6 +766,20 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		STRB2FAST       	0x02
 #define		REQINIT         	0x01
 
+#define	CLRSINT1        		0x4c
+#define		CLRSELTIMEO     	0x80
+#define		CLRATNO         	0x40
+#define		CLRSCSIRSTI     	0x20
+#define		CLRBUSFREE      	0x08
+#define		CLRSCSIPERR     	0x04
+#define		CLRSTRB2FAST    	0x02
+#define		CLRREQINIT      	0x01
+
+#define	SIMODE2         		0x4d
+#define		ENWIDE_RES      	0x04
+#define		ENSDONE         	0x02
+#define		ENDMADONE       	0x01
+
 #define	SSTAT2          		0x4d
 #define		BUSFREETIME     	0xc0
 #define		NONPACKREQ      	0x20
@@ -1662,11 +792,6 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		BUSFREE_DFF0    	0x80
 #define		BUSFREE_LQO     	0x40
 
-#define	SIMODE2         		0x4d
-#define		ENWIDE_RES      	0x04
-#define		ENSDONE         	0x02
-#define		ENDMADONE       	0x01
-
 #define	CLRSINT2        		0x4d
 #define		CLRNONPACKREQ   	0x20
 #define		CLRWIDE_RES     	0x04
@@ -1685,10 +810,10 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 
 #define	LQISTATE        		0x4e
 
-#define	SOFFCNT         		0x4f
-
 #define	LQOSTATE        		0x4f
 
+#define	SOFFCNT         		0x4f
+
 #define	LQISTAT0        		0x50
 #define		LQIATNQAS       	0x20
 #define		LQICRCT1        	0x10
@@ -1697,14 +822,6 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		LQIATNLQ        	0x02
 #define		LQIATNCMD       	0x01
 
-#define	CLRLQIINT0      		0x50
-#define		CLRLQIATNQAS    	0x20
-#define		CLRLQICRCT1     	0x10
-#define		CLRLQICRCT2     	0x08
-#define		CLRLQIBADLQT    	0x04
-#define		CLRLQIATNLQ     	0x02
-#define		CLRLQIATNCMD    	0x01
-
 #define	LQIMODE0        		0x50
 #define		ENLQIATNQASK    	0x20
 #define		ENLQICRCT1      	0x10
@@ -1713,6 +830,14 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		ENLQIATNLQ      	0x02
 #define		ENLQIATNCMD     	0x01
 
+#define	CLRLQIINT0      		0x50
+#define		CLRLQIATNQAS    	0x20
+#define		CLRLQICRCT1     	0x10
+#define		CLRLQICRCT2     	0x08
+#define		CLRLQIBADLQT    	0x04
+#define		CLRLQIATNLQ     	0x02
+#define		CLRLQIATNCMD    	0x01
+
 #define	LQIMODE1        		0x51
 #define		ENLQIPHASE_LQ   	0x80
 #define		ENLQIPHASE_NLQ  	0x40
@@ -1753,25 +878,18 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		LQISTOPCMD      	0x02
 #define		LQIGSAVAIL      	0x01
 
-#define	SSTAT3          		0x53
-#define		NTRAMPERR       	0x02
-#define		OSRAMPERR       	0x01
-
 #define	SIMODE3         		0x53
 #define		ENNTRAMPERR     	0x02
 #define		ENOSRAMPERR     	0x01
 
+#define	SSTAT3          		0x53
+#define		NTRAMPERR       	0x02
+#define		OSRAMPERR       	0x01
+
 #define	CLRSINT3        		0x53
 #define		CLRNTRAMPERR    	0x02
 #define		CLROSRAMPERR    	0x01
 
-#define	LQOSTAT0        		0x54
-#define		LQOTARGSCBPERR  	0x10
-#define		LQOSTOPT2       	0x08
-#define		LQOATNLQ        	0x04
-#define		LQOATNPKT       	0x02
-#define		LQOTCRC         	0x01
-
 #define	CLRLQOINT0      		0x54
 #define		CLRLQOTARGSCBPERR	0x10
 #define		CLRLQOSTOPT2    	0x08
@@ -1779,6 +897,13 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		CLRLQOATNPKT    	0x02
 #define		CLRLQOTCRC      	0x01
 
+#define	LQOSTAT0        		0x54
+#define		LQOTARGSCBPERR  	0x10
+#define		LQOSTOPT2       	0x08
+#define		LQOATNLQ        	0x04
+#define		LQOATNPKT       	0x02
+#define		LQOTCRC         	0x01
+
 #define	LQOMODE0        		0x54
 #define		ENLQOTARGSCBPERR	0x10
 #define		ENLQOSTOPT2     	0x08
@@ -1793,13 +918,6 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		ENLQOBUSFREE    	0x02
 #define		ENLQOPHACHGINPKT	0x01
 
-#define	LQOSTAT1        		0x55
-#define		LQOINITSCBPERR  	0x10
-#define		LQOSTOPI2       	0x08
-#define		LQOBADQAS       	0x04
-#define		LQOBUSFREE      	0x02
-#define		LQOPHACHGINPKT  	0x01
-
 #define	CLRLQOINT1      		0x55
 #define		CLRLQOINITSCBPERR	0x10
 #define		CLRLQOSTOPI2    	0x08
@@ -1807,6 +925,13 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		CLRLQOBUSFREE   	0x02
 #define		CLRLQOPHACHGINPKT	0x01
 
+#define	LQOSTAT1        		0x55
+#define		LQOINITSCBPERR  	0x10
+#define		LQOSTOPI2       	0x08
+#define		LQOBADQAS       	0x04
+#define		LQOBUSFREE      	0x02
+#define		LQOPHACHGINPKT  	0x01
+
 #define	LQOSTAT2        		0x56
 #define		LQOPKT          	0xe0
 #define		LQOWAITFIFO     	0x10
@@ -1859,8 +984,6 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		CFG4ICMD        	0x02
 #define		CFG4TCMD        	0x01
 
-#define	CURRSCB         		0x5c
-
 #define	SEQIMODE        		0x5c
 #define		ENCTXTDONE      	0x40
 #define		ENSAVEPTRS      	0x20
@@ -1870,6 +993,11 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		ENCFG4ICMD      	0x02
 #define		ENCFG4TCMD      	0x01
 
+#define	CURRSCB         		0x5c
+
+#define	CRCCONTROL      		0x5d
+#define		CRCVALCHKEN     	0x40
+
 #define	MDFFSTAT        		0x5d
 #define		SHCNTNEGATIVE   	0x40
 #define		SHCNTMINUS1     	0x20
@@ -1879,34 +1007,31 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		DATAINFIFO      	0x02
 #define		FIFOFREE        	0x01
 
-#define	CRCCONTROL      		0x5d
-#define		CRCVALCHKEN     	0x40
-
 #define	DFFTAG          		0x5e
 
-#define	LASTSCB         		0x5e
-
 #define	SCSITEST        		0x5e
 #define		CNTRTEST        	0x08
 #define		SEL_TXPLL_DEBUG 	0x04
 
+#define	LASTSCB         		0x5e
+
 #define	IOPDNCTL        		0x5f
 #define		DISABLE_OE      	0x80
 #define		PDN_IDIST       	0x04
 #define		PDN_DIFFSENSE   	0x01
 
-#define	SHADDR          		0x60
+#define	DGRPCRCI        		0x60
 
 #define	NEGOADDR        		0x60
 
-#define	DGRPCRCI        		0x60
+#define	SHADDR          		0x60
 
 #define	NEGPERIOD       		0x61
 
-#define	PACKCRCI        		0x62
-
 #define	NEGOFFSET       		0x62
 
+#define	PACKCRCI        		0x62
+
 #define	NEGPPROPTS      		0x63
 #define		PPROPT_PACE     	0x08
 #define		PPROPT_QAS      	0x04
@@ -1942,16 +1067,18 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 
 #define	SHCNT           		0x68
 
-#define	TOWNID          		0x69
-
 #define	PLL960CTL1      		0x69
 
+#define	TOWNID          		0x69
+
 #define	PLL960CNT0      		0x6a
 
 #define	XSIG            		0x6a
 
 #define	SELOID          		0x6b
 
+#define	FAIRNESS        		0x6c
+
 #define	PLL400CTL0      		0x6c
 #define		PLL_VCOSEL      	0x80
 #define		PLL_PWDN        	0x40
@@ -1961,8 +1088,6 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		PLL_DLPF        	0x02
 #define		PLL_ENFBM       	0x01
 
-#define	FAIRNESS        		0x6c
-
 #define	PLL400CTL1      		0x6d
 #define		PLL_CNTEN       	0x80
 #define		PLL_CNTCLR      	0x40
@@ -1974,25 +1099,25 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 
 #define	HADDR           		0x70
 
+#define	HODMAADR        		0x70
+
 #define	PLLDELAY        		0x70
 #define		SPLIT_DROP_REQ  	0x80
 
-#define	HODMAADR        		0x70
+#define	HCNT            		0x78
 
 #define	HODMACNT        		0x78
 
-#define	HCNT            		0x78
-
 #define	HODMAEN         		0x7a
 
-#define	SCBHADDR        		0x7c
-
 #define	SGHADDR         		0x7c
 
-#define	SCBHCNT         		0x84
+#define	SCBHADDR        		0x7c
 
 #define	SGHCNT          		0x84
 
+#define	SCBHCNT         		0x84
+
 #define	DFF_THRSH       		0x88
 #define		WR_DFTHRSH      	0x70
 #define		RD_DFTHRSH      	0x07
@@ -2025,6 +1150,10 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 
 #define	CMCRXMSG0       		0x90
 
+#define	OVLYRXMSG0      		0x90
+
+#define	DCHRXMSG0       		0x90
+
 #define	ROENABLE        		0x90
 #define		MSIROEN         	0x20
 #define		OVLYROEN        	0x10
@@ -2033,11 +1162,11 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		DCH1ROEN        	0x02
 #define		DCH0ROEN        	0x01
 
-#define	OVLYRXMSG0      		0x90
+#define	OVLYRXMSG1      		0x91
 
-#define	DCHRXMSG0       		0x90
+#define	CMCRXMSG1       		0x91
 
-#define	OVLYRXMSG1      		0x91
+#define	DCHRXMSG1       		0x91
 
 #define	NSENABLE        		0x91
 #define		MSINSEN         	0x20
@@ -2047,10 +1176,6 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		DCH1NSEN        	0x02
 #define		DCH0NSEN        	0x01
 
-#define	CMCRXMSG1       		0x91
-
-#define	DCHRXMSG1       		0x91
-
 #define	DCHRXMSG2       		0x92
 
 #define	CMCRXMSG2       		0x92
@@ -2074,24 +1199,24 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		TSCSERREN       	0x02
 #define		CMPABCDIS       	0x01
 
+#define	CMCSEQBCNT      		0x94
+
 #define	OVLYSEQBCNT     		0x94
 
 #define	DCHSEQBCNT      		0x94
 
-#define	CMCSEQBCNT      		0x94
-
-#define	CMCSPLTSTAT0    		0x96
-
 #define	DCHSPLTSTAT0    		0x96
 
 #define	OVLYSPLTSTAT0   		0x96
 
-#define	CMCSPLTSTAT1    		0x97
+#define	CMCSPLTSTAT0    		0x96
 
 #define	OVLYSPLTSTAT1   		0x97
 
 #define	DCHSPLTSTAT1    		0x97
 
+#define	CMCSPLTSTAT1    		0x97
+
 #define	SGRXMSG0        		0x98
 #define		CDNUM           	0xf8
 #define		CFNUM           	0x07
@@ -2119,18 +1244,15 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		TAG_NUM         	0x1f
 #define		RLXORD          	0x10
 
-#define	SGSEQBCNT       		0x9c
-
 #define	SLVSPLTOUTATTR0 		0x9c
 #define		LOWER_BCNT      	0xff
 
+#define	SGSEQBCNT       		0x9c
+
 #define	SLVSPLTOUTATTR1 		0x9d
 #define		CMPLT_DNUM      	0xf8
 #define		CMPLT_FNUM      	0x07
 
-#define	SLVSPLTOUTATTR2 		0x9e
-#define		CMPLT_BNUM      	0xff
-
 #define	SGSPLTSTAT0     		0x9e
 #define		STAETERM        	0x80
 #define		SCBCERR         	0x40
@@ -2141,6 +1263,9 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		RXSCEMSG        	0x02
 #define		RXSPLTRSP       	0x01
 
+#define	SLVSPLTOUTATTR2 		0x9e
+#define		CMPLT_BNUM      	0xff
+
 #define	SGSPLTSTAT1     		0x9f
 #define		RXDATABUCKET    	0x01
 
@@ -2177,14 +1302,14 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		CLRPENDMSI      	0x08
 #define		DPR             	0x01
 
+#define	DATA_COUNT_ODD  		0xa7
+
 #define	TARGPCISTAT     		0xa7
 #define		DPE             	0x80
 #define		SSE             	0x40
 #define		STA             	0x08
 #define		TWATERR         	0x02
 
-#define	DATA_COUNT_ODD  		0xa7
-
 #define	SCBPTR          		0xa8
 
 #define	CCSCBACNT       		0xab
@@ -2196,10 +1321,10 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 
 #define	CCSGADDR        		0xac
 
-#define	CCSCBADR_BK     		0xac
-
 #define	CCSCBADDR       		0xac
 
+#define	CCSCBADR_BK     		0xac
+
 #define	CMC_RAMBIST     		0xad
 #define		SG_ELEMENT_SIZE 	0x80
 #define		SCBRAMBIST_FAIL 	0x40
@@ -2253,9 +1378,9 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define	SEEDAT          		0xbc
 
 #define	SEECTL          		0xbe
+#define		SEEOP_EWDS      	0x40
 #define		SEEOP_WALL      	0x40
 #define		SEEOP_EWEN      	0x40
-#define		SEEOP_EWDS      	0x40
 #define		SEEOPCODE       	0x70
 #define		SEERST          	0x02
 #define		SEESTART        	0x01
@@ -2272,25 +1397,25 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 
 #define	SCBCNT          		0xbf
 
-#define	DFWADDR         		0xc0
-
 #define	DSPFLTRCTL      		0xc0
 #define		FLTRDISABLE     	0x20
 #define		EDGESENSE       	0x10
 #define		DSPFCNTSEL      	0x0f
 
+#define	DFWADDR         		0xc0
+
 #define	DSPDATACTL      		0xc1
 #define		BYPASSENAB      	0x80
 #define		DESQDIS         	0x10
 #define		RCVROFFSTDIS    	0x04
 #define		XMITOFFSTDIS    	0x02
 
-#define	DFRADDR         		0xc2
-
 #define	DSPREQCTL       		0xc2
 #define		MANREQCTL       	0xc0
 #define		MANREQDLY       	0x3f
 
+#define	DFRADDR         		0xc2
+
 #define	DSPACKCTL       		0xc3
 #define		MANACKCTL       	0xc0
 #define		MANACKDLY       	0x3f
@@ -2311,14 +1436,14 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 
 #define	WRTBIASCALC     		0xc7
 
-#define	RCVRBIASCALC    		0xc8
-
 #define	DFPTRS          		0xc8
 
-#define	SKEWCALC        		0xc9
+#define	RCVRBIASCALC    		0xc8
 
 #define	DFBKPTR         		0xc9
 
+#define	SKEWCALC        		0xc9
+
 #define	DFDBCTL         		0xcb
 #define		DFF_CIO_WR_RDY  	0x20
 #define		DFF_CIO_RD_RDY  	0x10
@@ -2403,12 +1528,12 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 
 #define	ACCUM_SAVE      		0xfa
 
-#define	WAITING_SCB_TAILS		0x100
-
 #define	AHD_PCI_CONFIG_BASE		0x100
 
 #define	SRAM_BASE       		0x100
 
+#define	WAITING_SCB_TAILS		0x100
+
 #define	WAITING_TID_HEAD		0x120
 
 #define	WAITING_TID_TAIL		0x122
@@ -2437,8 +1562,8 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define		PRELOADEN       	0x80
 #define		WIDEODD         	0x40
 #define		SCSIEN          	0x20
-#define		SDMAEN          	0x10
 #define		SDMAENACK       	0x10
+#define		SDMAEN          	0x10
 #define		HDMAEN          	0x08
 #define		HDMAENACK       	0x08
 #define		DIRECTION       	0x04
@@ -2536,12 +1661,12 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 
 #define	MK_MESSAGE_SCSIID		0x162
 
-#define	SCB_BASE        		0x180
-
 #define	SCB_RESIDUAL_DATACNT		0x180
 #define	SCB_CDB_STORE   		0x180
 #define	SCB_HOST_CDB_PTR		0x180
 
+#define	SCB_BASE        		0x180
+
 #define	SCB_RESIDUAL_SGPTR		0x184
 #define		SG_ADDR_MASK    	0xf8
 #define		SG_OVERRUN_RESID	0x02
@@ -2609,77 +1734,77 @@ ahd_reg_print_t ahd_scb_disconnected_lists_print;
 #define	SCB_DISCONNECTED_LISTS		0x1b8
 
 
+#define	STIMESEL_SHIFT	0x03
+#define	STIMESEL_MIN	0x18
+#define	INVALID_ADDR	0x80
+#define	CMD_GROUP_CODE_SHIFT	0x05
+#define	AHD_PRECOMP_MASK	0x07
+#define	TARGET_DATA_IN	0x01
+#define	SEEOP_EWEN_ADDR	0xc0
+#define	NUMDSPS 	0x14
+#define	DST_MODE_SHIFT	0x04
+#define	CCSCBADDR_MAX	0x80
+#define	AHD_ANNEXCOL_PER_DEV0	0x04
+#define	TARGET_CMD_CMPLT	0xfe
+#define	SEEOP_WRAL_ADDR	0x40
+#define	BUS_8_BIT	0x00
 #define	AHD_TIMER_MAX_US	0x18ffe7
 #define	AHD_TIMER_MAX_TICKS	0xffff
 #define	AHD_SENSE_BUFSIZE	0x100
-#define	BUS_8_BIT	0x00
-#define	TARGET_CMD_CMPLT	0xfe
-#define	SEEOP_WRAL_ADDR	0x40
-#define	AHD_AMPLITUDE_DEF	0x07
-#define	AHD_PRECOMP_CUTBACK_37	0x07
 #define	AHD_PRECOMP_SHIFT	0x00
+#define	AHD_PRECOMP_CUTBACK_37	0x07
 #define	AHD_ANNEXCOL_PRECOMP_SLEW	0x04
-#define	AHD_TIMER_US_PER_TICK	0x19
-#define	SCB_TRANSFER_SIZE_FULL_LUN	0x38
+#define	AHD_AMPLITUDE_DEF	0x07
+#define	WRTBIASCTL_HP_DEFAULT	0x00
+#define	TID_SHIFT	0x04
 #define	STATUS_QUEUE_FULL	0x28
 #define	STATUS_BUSY	0x08
-#define	MAX_OFFSET_NON_PACED	0x7f
+#define	SEEOP_EWDS_ADDR	0x00
+#define	SCB_TRANSFER_SIZE_FULL_LUN	0x38
+#define	MK_MESSAGE_BIT_OFFSET	0x04
 #define	MAX_OFFSET_PACED	0xfe
-#define	BUS_32_BIT	0x02
+#define	MAX_OFFSET_NON_PACED	0x7f
+#define	LUNLEN_SINGLE_LEVEL_LUN	0x0f
 #define	CCSGADDR_MAX	0x80
-#define	TID_SHIFT	0x04
-#define	MK_MESSAGE_BIT_OFFSET	0x04
-#define	WRTBIASCTL_HP_DEFAULT	0x00
-#define	SEEOP_EWDS_ADDR	0x00
-#define	AHD_AMPLITUDE_SHIFT	0x00
-#define	AHD_AMPLITUDE_MASK	0x07
-#define	AHD_ANNEXCOL_AMPLITUDE	0x06
-#define	AHD_SLEWRATE_DEF_REVA	0x08
+#define	B_CURRFIFO_0	0x02
+#define	BUS_32_BIT	0x02
+#define	AHD_TIMER_US_PER_TICK	0x19
 #define	AHD_SLEWRATE_SHIFT	0x03
 #define	AHD_SLEWRATE_MASK	0x78
+#define	AHD_SLEWRATE_DEF_REVA	0x08
 #define	AHD_PRECOMP_CUTBACK_29	0x06
 #define	AHD_NUM_PER_DEV_ANNEXCOLS	0x04
-#define	B_CURRFIFO_0	0x02
-#define	LUNLEN_SINGLE_LEVEL_LUN	0x0f
-#define	NVRAM_SCB_OFFSET	0x2c
+#define	AHD_ANNEXCOL_AMPLITUDE	0x06
+#define	AHD_AMPLITUDE_SHIFT	0x00
+#define	AHD_AMPLITUDE_MASK	0x07
+#define	STIMESEL_BUG_ADJ	0x08
 #define	STATUS_PKT_SENSE	0xff
-#define	CMD_GROUP_CODE_SHIFT	0x05
+#define	SRC_MODE_SHIFT	0x00
+#define	SEEOP_ERAL_ADDR	0x80
+#define	NVRAM_SCB_OFFSET	0x2c
 #define	MAX_OFFSET_PACED_BUG	0x7f
-#define	STIMESEL_BUG_ADJ	0x08
-#define	STIMESEL_MIN	0x18
-#define	STIMESEL_SHIFT	0x03
 #define	CCSGRAM_MAXSEGS	0x10
-#define	INVALID_ADDR	0x80
-#define	SEEOP_ERAL_ADDR	0x80
 #define	AHD_SLEWRATE_DEF_REVB	0x08
 #define	AHD_PRECOMP_CUTBACK_17	0x04
-#define	AHD_PRECOMP_MASK	0x07
-#define	SRC_MODE_SHIFT	0x00
-#define	PKT_OVERRUN_BUFSIZE	0x200
 #define	SCB_TRANSFER_SIZE_1BYTE_LUN	0x30
-#define	TARGET_DATA_IN	0x01
-#define	HOST_MSG	0xff
+#define	PKT_OVERRUN_BUFSIZE	0x200
 #define	MAX_OFFSET	0xfe
+#define	HOST_MSG	0xff
 #define	BUS_16_BIT	0x01
-#define	CCSCBADDR_MAX	0x80
-#define	NUMDSPS 	0x14
-#define	SEEOP_EWEN_ADDR	0xc0
-#define	AHD_ANNEXCOL_PER_DEV0	0x04
-#define	DST_MODE_SHIFT	0x04
 
 
 /* Downloaded Constant Definitions */
+#define	SG_SIZEOF	0x04
+#define	SG_PREFETCH_ALIGN_MASK	0x02
+#define	SG_PREFETCH_CNT_LIMIT	0x01
 #define	CACHELINE_MASK	0x07
 #define	SCB_TRANSFER_SIZE	0x06
 #define	PKT_OVERRUN_BUFOFFSET	0x05
-#define	SG_SIZEOF	0x04
 #define	SG_PREFETCH_ADDR_MASK	0x03
-#define	SG_PREFETCH_ALIGN_MASK	0x02
-#define	SG_PREFETCH_CNT_LIMIT	0x01
 #define	SG_PREFETCH_CNT	0x00
 #define	DOWNLOAD_CONST_COUNT	0x08
 
 
 /* Exported Labels */
-#define	LABEL_seq_isr 	0x28f
 #define	LABEL_timer_isr	0x28b
+#define	LABEL_seq_isr 	0x28f
diff --git a/drivers/scsi/aic7xxx/aic79xx_reg_print.c_shipped b/drivers/scsi/aic7xxx/aic79xx_reg_print.c_shipped
index f5ea715d6ac3..2e0c58905b9e 100644
--- a/drivers/scsi/aic7xxx/aic79xx_reg_print.c_shipped
+++ b/drivers/scsi/aic7xxx/aic79xx_reg_print.c_shipped
@@ -234,6 +234,23 @@ ahd_selid_print(u_int regvalue, u_int *cur_col, u_int wrap)
 	    0x49, regvalue, cur_col, wrap));
 }
 
+static const ahd_reg_parse_entry_t SIMODE0_parse_table[] = {
+	{ "ENARBDO",		0x01, 0x01 },
+	{ "ENSPIORDY",		0x02, 0x02 },
+	{ "ENOVERRUN",		0x04, 0x04 },
+	{ "ENIOERR",		0x08, 0x08 },
+	{ "ENSELINGO",		0x10, 0x10 },
+	{ "ENSELDI",		0x20, 0x20 },
+	{ "ENSELDO",		0x40, 0x40 }
+};
+
+int
+ahd_simode0_print(u_int regvalue, u_int *cur_col, u_int wrap)
+{
+	return (ahd_print_register(SIMODE0_parse_table, 7, "SIMODE0",
+	    0x4b, regvalue, cur_col, wrap));
+}
+
 static const ahd_reg_parse_entry_t SSTAT0_parse_table[] = {
 	{ "ARBDO",		0x01, 0x01 },
 	{ "SPIORDY",		0x02, 0x02 },
@@ -252,23 +269,6 @@ ahd_sstat0_print(u_int regvalue, u_int *cur_col, u_int wrap)
 	    0x4b, regvalue, cur_col, wrap));
 }
 
-static const ahd_reg_parse_entry_t SIMODE0_parse_table[] = {
-	{ "ENARBDO",		0x01, 0x01 },
-	{ "ENSPIORDY",		0x02, 0x02 },
-	{ "ENOVERRUN",		0x04, 0x04 },
-	{ "ENIOERR",		0x08, 0x08 },
-	{ "ENSELINGO",		0x10, 0x10 },
-	{ "ENSELDI",		0x20, 0x20 },
-	{ "ENSELDO",		0x40, 0x40 }
-};
-
-int
-ahd_simode0_print(u_int regvalue, u_int *cur_col, u_int wrap)
-{
-	return (ahd_print_register(SIMODE0_parse_table, 7, "SIMODE0",
-	    0x4b, regvalue, cur_col, wrap));
-}
-
 static const ahd_reg_parse_entry_t SSTAT1_parse_table[] = {
 	{ "REQINIT",		0x01, 0x01 },
 	{ "STRB2FAST",		0x02, 0x02 },
diff --git a/drivers/scsi/aic7xxx/aic7xxx_core.c b/drivers/scsi/aic7xxx/aic7xxx_core.c
index 64ab9eaec428..381846164003 100644
--- a/drivers/scsi/aic7xxx/aic7xxx_core.c
+++ b/drivers/scsi/aic7xxx/aic7xxx_core.c
@@ -7340,7 +7340,6 @@ ahc_dump_card_state(struct ahc_softc *ahc)
 		printk("\n");
 	}
 
-	ahc_platform_dump_card_state(ahc);
 	printk("\n<<<<<<<<<<<<<<<<< Dump Card State Ends >>>>>>>>>>>>>>>>>>\n");
 	ahc_outb(ahc, SCBPTR, saved_scbptr);
 	if (paused == 0)
diff --git a/drivers/scsi/aic7xxx/aic7xxx_osm.c b/drivers/scsi/aic7xxx/aic7xxx_osm.c
index fc6a83188c1e..acd687f4554e 100644
--- a/drivers/scsi/aic7xxx/aic7xxx_osm.c
+++ b/drivers/scsi/aic7xxx/aic7xxx_osm.c
@@ -2329,11 +2329,6 @@ done:
 	return (retval);
 }
 
-void
-ahc_platform_dump_card_state(struct ahc_softc *ahc)
-{
-}
-
 static void ahc_linux_set_width(struct scsi_target *starget, int width)
 {
 	struct Scsi_Host *shost = dev_to_shost(starget->dev.parent);
diff --git a/drivers/scsi/aic7xxx/aic7xxx_osm.h b/drivers/scsi/aic7xxx/aic7xxx_osm.h
index 54c702864103..f8489078f003 100644
--- a/drivers/scsi/aic7xxx/aic7xxx_osm.h
+++ b/drivers/scsi/aic7xxx/aic7xxx_osm.h
@@ -688,7 +688,6 @@ void	ahc_done(struct ahc_softc*, struct scb*);
 void	ahc_send_async(struct ahc_softc *, char channel,
 		       u_int target, u_int lun, ac_code);
 void	ahc_print_path(struct ahc_softc *, struct scb *);
-void	ahc_platform_dump_card_state(struct ahc_softc *ahc);
 
 #ifdef CONFIG_PCI
 #define AHC_PCI_CONFIG 1
diff --git a/drivers/scsi/aic7xxx/aic7xxx_reg.h_shipped b/drivers/scsi/aic7xxx/aic7xxx_reg.h_shipped
index e821082a4f47..473039df0ed5 100644
--- a/drivers/scsi/aic7xxx/aic7xxx_reg.h_shipped
+++ b/drivers/scsi/aic7xxx/aic7xxx_reg.h_shipped
@@ -244,8 +244,6 @@ ahc_reg_print_t ahc_scb_tag_print;
 
 #define	SCSIDATH        		0x07
 
-#define	STCNT           		0x08
-
 #define	OPTIONMODE      		0x08
 #define		OPTIONMODE_DEFAULTS	0x03
 #define		AUTORATEEN      	0x80
@@ -257,6 +255,8 @@ ahc_reg_print_t ahc_scb_tag_print;
 #define		AUTO_MSGOUT_DE  	0x02
 #define		DIS_MSGIN_DUALEDGE	0x01
 
+#define	STCNT           		0x08
+
 #define	TARGCRCCNT      		0x0a
 
 #define	CLRSINT0        		0x0b
@@ -365,8 +365,6 @@ ahc_reg_print_t ahc_scb_tag_print;
 #define		ALTSTIM         	0x20
 #define		DFLTTID         	0x10
 
-#define	TARGID          		0x1b
-
 #define	SPIOCAP         		0x1b
 #define		SOFT1           	0x80
 #define		SOFT0           	0x40
@@ -377,12 +375,14 @@ ahc_reg_print_t ahc_scb_tag_print;
 #define		ROM             	0x02
 #define		SSPIOCPS        	0x01
 
+#define	TARGID          		0x1b
+
 #define	BRDCTL          		0x1d
 #define		BRDDAT7         	0x80
 #define		BRDDAT6         	0x40
 #define		BRDDAT5         	0x20
-#define		BRDDAT4         	0x10
 #define		BRDSTB          	0x10
+#define		BRDDAT4         	0x10
 #define		BRDDAT3         	0x08
 #define		BRDCS           	0x08
 #define		BRDDAT2         	0x04
@@ -406,8 +406,8 @@ ahc_reg_print_t ahc_scb_tag_print;
 #define		DIAGLEDEN       	0x80
 #define		DIAGLEDON       	0x40
 #define		AUTOFLUSHDIS    	0x20
-#define		ENAB40          	0x08
 #define		SELBUSB         	0x08
+#define		ENAB40          	0x08
 #define		ENAB20          	0x04
 #define		SELWIDE         	0x02
 #define		XCVR            	0x01
@@ -730,8 +730,8 @@ ahc_reg_print_t ahc_scb_tag_print;
 #define	SCB_BASE        		0xa0
 
 #define	SCB_CDB_PTR     		0xa0
-#define	SCB_RESIDUAL_DATACNT		0xa0
 #define	SCB_CDB_STORE   		0xa0
+#define	SCB_RESIDUAL_DATACNT		0xa0
 
 #define	SCB_RESIDUAL_SGPTR		0xa4
 
@@ -756,8 +756,8 @@ ahc_reg_print_t ahc_scb_tag_print;
 
 #define	SCB_CONTROL     		0xb8
 #define		SCB_TAG_TYPE    	0x03
-#define		STATUS_RCVD     	0x80
 #define		TARGET_SCB      	0x80
+#define		STATUS_RCVD     	0x80
 #define		DISCENB         	0x40
 #define		TAG_ENB         	0x20
 #define		MK_MESSAGE      	0x10
@@ -872,40 +872,40 @@ ahc_reg_print_t ahc_scb_tag_print;
 #define	SG_CACHE_PRE    		0xfc
 
 
+#define	TARGET_CMD_CMPLT	0xfe
 #define	MAX_OFFSET_ULTRA2	0x7f
 #define	MAX_OFFSET_16BIT	0x08
 #define	BUS_8_BIT	0x00
-#define	TARGET_CMD_CMPLT	0xfe
+#define	TID_SHIFT	0x04
 #define	STATUS_QUEUE_FULL	0x28
 #define	STATUS_BUSY	0x08
-#define	MAX_OFFSET_8BIT	0x0f
-#define	BUS_32_BIT	0x02
-#define	CCSGADDR_MAX	0x80
-#define	TID_SHIFT	0x04
 #define	SCB_DOWNLOAD_SIZE_64	0x30
+#define	MAX_OFFSET_8BIT	0x0f
 #define	HOST_MAILBOX_SHIFT	0x04
-#define	CMD_GROUP_CODE_SHIFT	0x05
-#define	CCSGRAM_MAXSEGS	0x10
-#define	SCB_LIST_NULL	0xff
+#define	CCSGADDR_MAX	0x80
+#define	BUS_32_BIT	0x02
 #define	SG_SIZEOF	0x08
-#define	SCB_DOWNLOAD_SIZE	0x20
 #define	SEQ_MAILBOX_SHIFT	0x00
+#define	SCB_LIST_NULL	0xff
+#define	SCB_DOWNLOAD_SIZE	0x20
+#define	CMD_GROUP_CODE_SHIFT	0x05
+#define	CCSGRAM_MAXSEGS	0x10
 #define	TARGET_DATA_IN	0x01
-#define	HOST_MSG	0xff
+#define	STACK_SIZE	0x04
+#define	SCB_UPLOAD_SIZE	0x20
 #define	MAX_OFFSET	0x7f
+#define	HOST_MSG	0xff
 #define	BUS_16_BIT	0x01
-#define	SCB_UPLOAD_SIZE	0x20
-#define	STACK_SIZE	0x04
 
 
 /* Downloaded Constant Definitions */
 #define	INVERTED_CACHESIZE_MASK	0x03
-#define	SG_PREFETCH_ADDR_MASK	0x06
 #define	SG_PREFETCH_ALIGN_MASK	0x05
+#define	SG_PREFETCH_ADDR_MASK	0x06
 #define	QOUTFIFO_OFFSET	0x00
 #define	SG_PREFETCH_CNT	0x04
-#define	CACHESIZE_MASK	0x02
 #define	QINFIFO_OFFSET	0x01
+#define	CACHESIZE_MASK	0x02
 #define	DOWNLOAD_CONST_COUNT	0x07
 
 
diff --git a/drivers/scsi/aic94xx/aic94xx_init.c b/drivers/scsi/aic94xx/aic94xx_init.c
index a14ba7a6b81e..6c838865ac5a 100644
--- a/drivers/scsi/aic94xx/aic94xx_init.c
+++ b/drivers/scsi/aic94xx/aic94xx_init.c
@@ -70,7 +70,7 @@ static struct scsi_host_template aic94xx_sht = {
 	.max_sectors		= SCSI_DEFAULT_MAX_SECTORS,
 	.use_clustering		= ENABLE_CLUSTERING,
 	.eh_device_reset_handler	= sas_eh_device_reset_handler,
-	.eh_bus_reset_handler	= sas_eh_bus_reset_handler,
+	.eh_target_reset_handler	= sas_eh_target_reset_handler,
 	.target_destroy		= sas_target_destroy,
 	.ioctl			= sas_ioctl,
 	.track_queue_depth	= 1,
@@ -956,11 +956,11 @@ static int asd_scan_finished(struct Scsi_Host *shost, unsigned long time)
 	return 1;
 }
 
-static ssize_t asd_version_show(struct device_driver *driver, char *buf)
+static ssize_t version_show(struct device_driver *driver, char *buf)
 {
 	return snprintf(buf, PAGE_SIZE, "%s\n", ASD_DRIVER_VERSION);
 }
-static DRIVER_ATTR(version, S_IRUGO, asd_version_show, NULL);
+static DRIVER_ATTR_RO(version);
 
 static int asd_create_driver_attrs(struct device_driver *driver)
 {
diff --git a/drivers/scsi/arcmsr/arcmsr_attr.c b/drivers/scsi/arcmsr/arcmsr_attr.c
index 9c86481f779f..259d9c20bf25 100644
--- a/drivers/scsi/arcmsr/arcmsr_attr.c
+++ b/drivers/scsi/arcmsr/arcmsr_attr.c
@@ -190,7 +190,7 @@ static ssize_t arcmsr_sysfs_iop_message_clear(struct file *filp,
 	return 1;
 }
 
-static struct bin_attribute arcmsr_sysfs_message_read_attr = {
+static const struct bin_attribute arcmsr_sysfs_message_read_attr = {
 	.attr = {
 		.name = "mu_read",
 		.mode = S_IRUSR ,
@@ -199,7 +199,7 @@ static struct bin_attribute arcmsr_sysfs_message_read_attr = {
 	.read = arcmsr_sysfs_iop_message_read,
 };
 
-static struct bin_attribute arcmsr_sysfs_message_write_attr = {
+static const struct bin_attribute arcmsr_sysfs_message_write_attr = {
 	.attr = {
 		.name = "mu_write",
 		.mode = S_IWUSR,
@@ -208,7 +208,7 @@ static struct bin_attribute arcmsr_sysfs_message_write_attr = {
 	.write = arcmsr_sysfs_iop_message_write,
 };
 
-static struct bin_attribute arcmsr_sysfs_message_clear_attr = {
+static const struct bin_attribute arcmsr_sysfs_message_clear_attr = {
 	.attr = {
 		.name = "mu_clear",
 		.mode = S_IWUSR,
diff --git a/drivers/scsi/arm/acornscsi.c b/drivers/scsi/arm/acornscsi.c
index 12b88294d667..421fe869a11e 100644
--- a/drivers/scsi/arm/acornscsi.c
+++ b/drivers/scsi/arm/acornscsi.c
@@ -2725,7 +2725,7 @@ int acornscsi_abort(struct scsi_cmnd *SCpnt)
  * Params   : SCpnt  - command causing reset
  * Returns  : one of SCSI_RESET_ macros
  */
-int acornscsi_bus_reset(struct scsi_cmnd *SCpnt)
+int acornscsi_host_reset(struct scsi_cmnd *SCpnt)
 {
 	AS_Host *host = (AS_Host *)SCpnt->device->host->hostdata;
 	struct scsi_cmnd *SCptr;
@@ -2734,14 +2734,15 @@ int acornscsi_bus_reset(struct scsi_cmnd *SCpnt)
 
 #if (DEBUG & DEBUG_RESET)
     {
-	int asr, ssr;
+	int asr, ssr, devidx;
 
 	asr = sbic_arm_read(host, SBIC_ASR);
 	ssr = sbic_arm_read(host, SBIC_SSR);
 
 	printk(KERN_WARNING "acornscsi_reset: ");
 	print_sbic_status(asr, ssr, host->scsi.phase);
-	acornscsi_dumplog(host, SCpnt->device->id);
+	for (devidx = 0; devidx < 9; devidx++)
+	    acornscsi_dumplog(host, devidx);
     }
 #endif
 
@@ -2884,7 +2885,7 @@ static struct scsi_host_template acornscsi_template = {
 	.info			= acornscsi_info,
 	.queuecommand		= acornscsi_queuecmd,
 	.eh_abort_handler	= acornscsi_abort,
-	.eh_bus_reset_handler	= acornscsi_bus_reset,
+	.eh_host_reset_handler	= acornscsi_host_reset,
 	.can_queue		= 16,
 	.this_id		= 7,
 	.sg_tablesize		= SG_ALL,
diff --git a/drivers/scsi/arm/cumana_1.c b/drivers/scsi/arm/cumana_1.c
index a87b99c7fb9a..ae1d809904fb 100644
--- a/drivers/scsi/arm/cumana_1.c
+++ b/drivers/scsi/arm/cumana_1.c
@@ -216,7 +216,7 @@ static struct scsi_host_template cumanascsi_template = {
 	.info			= cumanascsi_info,
 	.queuecommand		= cumanascsi_queue_command,
 	.eh_abort_handler	= NCR5380_abort,
-	.eh_bus_reset_handler	= NCR5380_bus_reset,
+	.eh_host_reset_handler	= NCR5380_host_reset,
 	.can_queue		= 16,
 	.this_id		= 7,
 	.sg_tablesize		= SG_ALL,
diff --git a/drivers/scsi/arm/oak.c b/drivers/scsi/arm/oak.c
index 6be6666534d4..05b7f755499b 100644
--- a/drivers/scsi/arm/oak.c
+++ b/drivers/scsi/arm/oak.c
@@ -105,7 +105,7 @@ static struct scsi_host_template oakscsi_template = {
 	.info			= oakscsi_info,
 	.queuecommand		= oakscsi_queue_command,
 	.eh_abort_handler	= NCR5380_abort,
-	.eh_bus_reset_handler	= NCR5380_bus_reset,
+	.eh_host_reset_handler	= NCR5380_host_reset,
 	.can_queue		= 16,
 	.this_id		= 7,
 	.sg_tablesize		= SG_ALL,
diff --git a/drivers/scsi/atari_scsi.c b/drivers/scsi/atari_scsi.c
index a75feebe6ad6..89f5154c40b6 100644
--- a/drivers/scsi/atari_scsi.c
+++ b/drivers/scsi/atari_scsi.c
@@ -671,7 +671,7 @@ static void atari_scsi_falcon_reg_write(unsigned int reg, u8 value)
 
 #include "NCR5380.c"
 
-static int atari_scsi_bus_reset(struct scsi_cmnd *cmd)
+static int atari_scsi_host_reset(struct scsi_cmnd *cmd)
 {
 	int rv;
 	unsigned long flags;
@@ -688,7 +688,7 @@ static int atari_scsi_bus_reset(struct scsi_cmnd *cmd)
 		atari_dma_orig_addr = NULL;
 	}
 
-	rv = NCR5380_bus_reset(cmd);
+	rv = NCR5380_host_reset(cmd);
 
 	/* The 5380 raises its IRQ line while _RST is active but the ST DMA
 	 * "lock" has been released so this interrupt may end up handled by
@@ -711,7 +711,7 @@ static struct scsi_host_template atari_scsi_template = {
 	.info			= atari_scsi_info,
 	.queuecommand		= atari_scsi_queue_command,
 	.eh_abort_handler	= atari_scsi_abort,
-	.eh_bus_reset_handler	= atari_scsi_bus_reset,
+	.eh_host_reset_handler	= atari_scsi_host_reset,
 	.this_id		= 7,
 	.cmd_per_lun		= 2,
 	.use_clustering		= DISABLE_CLUSTERING,
diff --git a/drivers/scsi/be2iscsi/be_iscsi.c b/drivers/scsi/be2iscsi/be_iscsi.c
index 97dca4681784..43a80ce5ce6a 100644
--- a/drivers/scsi/be2iscsi/be_iscsi.c
+++ b/drivers/scsi/be2iscsi/be_iscsi.c
@@ -82,8 +82,8 @@ struct iscsi_cls_session *beiscsi_session_create(struct iscsi_endpoint *ep,
 		return NULL;
 	sess = cls_session->dd_data;
 	beiscsi_sess = sess->dd_data;
-	beiscsi_sess->bhs_pool =  pci_pool_create("beiscsi_bhs_pool",
-						   phba->pcidev,
+	beiscsi_sess->bhs_pool =  dma_pool_create("beiscsi_bhs_pool",
+						   &phba->pcidev->dev,
 						   sizeof(struct be_cmd_bhs),
 						   64, 0);
 	if (!beiscsi_sess->bhs_pool)
@@ -108,7 +108,7 @@ void beiscsi_session_destroy(struct iscsi_cls_session *cls_session)
 	struct beiscsi_session *beiscsi_sess = sess->dd_data;
 
 	printk(KERN_INFO "In beiscsi_session_destroy\n");
-	pci_pool_destroy(beiscsi_sess->bhs_pool);
+	dma_pool_destroy(beiscsi_sess->bhs_pool);
 	iscsi_session_teardown(cls_session);
 }
 
diff --git a/drivers/scsi/be2iscsi/be_main.c b/drivers/scsi/be2iscsi/be_main.c
index f862332261f8..b4542e7e2ad5 100644
--- a/drivers/scsi/be2iscsi/be_main.c
+++ b/drivers/scsi/be2iscsi/be_main.c
@@ -4257,7 +4257,7 @@ static void beiscsi_cleanup_task(struct iscsi_task *task)
 	pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
 
 	if (io_task->cmd_bhs) {
-		pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
+		dma_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
 			      io_task->bhs_pa.u.a64.address);
 		io_task->cmd_bhs = NULL;
 		task->hdr = NULL;
@@ -4374,7 +4374,7 @@ static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
 	struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
 	dma_addr_t paddr;
 
-	io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool,
+	io_task->cmd_bhs = dma_pool_alloc(beiscsi_sess->bhs_pool,
 					  GFP_ATOMIC, &paddr);
 	if (!io_task->cmd_bhs)
 		return -ENOMEM;
@@ -4501,7 +4501,7 @@ free_hndls:
 	if (io_task->pwrb_handle)
 		free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
 	io_task->pwrb_handle = NULL;
-	pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
+	dma_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
 		      io_task->bhs_pa.u.a64.address);
 	io_task->cmd_bhs = NULL;
 	return -ENOMEM;
diff --git a/drivers/scsi/be2iscsi/be_main.h b/drivers/scsi/be2iscsi/be_main.h
index 338dbe0800c1..81ce3ffda968 100644
--- a/drivers/scsi/be2iscsi/be_main.h
+++ b/drivers/scsi/be2iscsi/be_main.h
@@ -438,7 +438,7 @@ struct beiscsi_hba {
 	 test_bit(BEISCSI_HBA_ONLINE, &phba->state))
 
 struct beiscsi_session {
-	struct pci_pool *bhs_pool;
+	struct dma_pool *bhs_pool;
 };
 
 /**
diff --git a/drivers/scsi/bfa/bfad_im.c b/drivers/scsi/bfa/bfad_im.c
index 7eb0eef18fdd..24e657a4ec80 100644
--- a/drivers/scsi/bfa/bfad_im.c
+++ b/drivers/scsi/bfa/bfad_im.c
@@ -373,32 +373,28 @@ out:
 }
 
 /*
- * Scsi_Host template entry, resets the bus and abort all commands.
+ * Scsi_Host template entry, resets the target and abort all commands.
  */
 static int
-bfad_im_reset_bus_handler(struct scsi_cmnd *cmnd)
+bfad_im_reset_target_handler(struct scsi_cmnd *cmnd)
 {
 	struct Scsi_Host *shost = cmnd->device->host;
+	struct scsi_target *starget = scsi_target(cmnd->device);
 	struct bfad_im_port_s *im_port =
 				(struct bfad_im_port_s *) shost->hostdata[0];
 	struct bfad_s         *bfad = im_port->bfad;
 	struct bfad_itnim_s   *itnim;
 	unsigned long   flags;
-	u32        i, rc, err_cnt = 0;
+	u32        rc, rtn = FAILED;
 	DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
 	enum bfi_tskim_status task_status;
 
 	spin_lock_irqsave(&bfad->bfad_lock, flags);
-	for (i = 0; i < MAX_FCP_TARGET; i++) {
-		itnim = bfad_get_itnim(im_port, i);
-		if (itnim) {
-			cmnd->SCp.ptr = (char *)&wq;
-			rc = bfad_im_target_reset_send(bfad, cmnd, itnim);
-			if (rc != BFA_STATUS_OK) {
-				err_cnt++;
-				continue;
-			}
-
+	itnim = bfad_get_itnim(im_port, starget->id);
+	if (itnim) {
+		cmnd->SCp.ptr = (char *)&wq;
+		rc = bfad_im_target_reset_send(bfad, cmnd, itnim);
+		if (rc == BFA_STATUS_OK) {
 			/* wait target reset to complete */
 			spin_unlock_irqrestore(&bfad->bfad_lock, flags);
 			wait_event(wq, test_bit(IO_DONE_BIT,
@@ -406,20 +402,17 @@ bfad_im_reset_bus_handler(struct scsi_cmnd *cmnd)
 			spin_lock_irqsave(&bfad->bfad_lock, flags);
 
 			task_status = cmnd->SCp.Status >> 1;
-			if (task_status != BFI_TSKIM_STS_OK) {
+			if (task_status != BFI_TSKIM_STS_OK)
 				BFA_LOG(KERN_ERR, bfad, bfa_log_level,
 					"target reset failure,"
 					" status: %d\n", task_status);
-				err_cnt++;
-			}
+			else
+				rtn = SUCCESS;
 		}
 	}
 	spin_unlock_irqrestore(&bfad->bfad_lock, flags);
 
-	if (err_cnt)
-		return FAILED;
-
-	return SUCCESS;
+	return rtn;
 }
 
 /*
@@ -816,7 +809,7 @@ struct scsi_host_template bfad_im_scsi_host_template = {
 	.eh_timed_out = fc_eh_timed_out,
 	.eh_abort_handler = bfad_im_abort_handler,
 	.eh_device_reset_handler = bfad_im_reset_lun_handler,
-	.eh_bus_reset_handler = bfad_im_reset_bus_handler,
+	.eh_target_reset_handler = bfad_im_reset_target_handler,
 
 	.slave_alloc = bfad_im_slave_alloc,
 	.slave_configure = bfad_im_slave_configure,
@@ -839,7 +832,7 @@ struct scsi_host_template bfad_im_vport_template = {
 	.eh_timed_out = fc_eh_timed_out,
 	.eh_abort_handler = bfad_im_abort_handler,
 	.eh_device_reset_handler = bfad_im_reset_lun_handler,
-	.eh_bus_reset_handler = bfad_im_reset_bus_handler,
+	.eh_target_reset_handler = bfad_im_reset_target_handler,
 
 	.slave_alloc = bfad_im_slave_alloc,
 	.slave_configure = bfad_im_slave_configure,
diff --git a/drivers/scsi/bnx2fc/bnx2fc.h b/drivers/scsi/bnx2fc/bnx2fc.h
index 7e007e142aab..901a31632493 100644
--- a/drivers/scsi/bnx2fc/bnx2fc.h
+++ b/drivers/scsi/bnx2fc/bnx2fc.h
@@ -539,7 +539,6 @@ void bnx2fc_init_task(struct bnx2fc_cmd *io_req,
 void bnx2fc_add_2_sq(struct bnx2fc_rport *tgt, u16 xid);
 void bnx2fc_ring_doorbell(struct bnx2fc_rport *tgt);
 int bnx2fc_eh_abort(struct scsi_cmnd *sc_cmd);
-int bnx2fc_eh_host_reset(struct scsi_cmnd *sc_cmd);
 int bnx2fc_eh_target_reset(struct scsi_cmnd *sc_cmd);
 int bnx2fc_eh_device_reset(struct scsi_cmnd *sc_cmd);
 void bnx2fc_rport_event_handler(struct fc_lport *lport,
diff --git a/drivers/scsi/ch.c b/drivers/scsi/ch.c
index dad959fcf6d8..c535c52e72e5 100644
--- a/drivers/scsi/ch.c
+++ b/drivers/scsi/ch.c
@@ -105,6 +105,7 @@ do {									\
 static struct class * ch_sysfs_class;
 
 typedef struct {
+	struct kref         ref;
 	struct list_head    list;
 	int                 minor;
 	char                name[8];
@@ -563,13 +564,23 @@ static int ch_gstatus(scsi_changer *ch, int type, unsigned char __user *dest)
 
 /* ------------------------------------------------------------------------ */
 
+static void ch_destroy(struct kref *ref)
+{
+	scsi_changer *ch = container_of(ref, scsi_changer, ref);
+
+	kfree(ch->dt);
+	kfree(ch);
+}
+
 static int
 ch_release(struct inode *inode, struct file *file)
 {
 	scsi_changer *ch = file->private_data;
 
 	scsi_device_put(ch->device);
+	ch->device = NULL;
 	file->private_data = NULL;
+	kref_put(&ch->ref, ch_destroy);
 	return 0;
 }
 
@@ -588,6 +599,7 @@ ch_open(struct inode *inode, struct file *file)
 		mutex_unlock(&ch_mutex);
 		return -ENXIO;
 	}
+	kref_get(&ch->ref);
 	spin_unlock(&ch_index_lock);
 
 	file->private_data = ch;
@@ -935,8 +947,11 @@ static int ch_probe(struct device *dev)
 	}
 
 	mutex_init(&ch->lock);
+	kref_init(&ch->ref);
 	ch->device = sd;
-	ch_readconfig(ch);
+	ret = ch_readconfig(ch);
+	if (ret)
+		goto destroy_dev;
 	if (init)
 		ch_init_elem(ch);
 
@@ -944,6 +959,8 @@ static int ch_probe(struct device *dev)
 	sdev_printk(KERN_INFO, sd, "Attached scsi changer %s\n", ch->name);
 
 	return 0;
+destroy_dev:
+	device_destroy(ch_sysfs_class, MKDEV(SCSI_CHANGER_MAJOR, ch->minor));
 remove_idr:
 	idr_remove(&ch_index_idr, ch->minor);
 free_ch:
@@ -960,8 +977,7 @@ static int ch_remove(struct device *dev)
 	spin_unlock(&ch_index_lock);
 
 	device_destroy(ch_sysfs_class, MKDEV(SCSI_CHANGER_MAJOR,ch->minor));
-	kfree(ch->dt);
-	kfree(ch);
+	kref_put(&ch->ref, ch_destroy);
 	return 0;
 }
 
diff --git a/drivers/scsi/csiostor/csio_hw.c b/drivers/scsi/csiostor/csio_hw.c
index 2029ad225121..5be0086142ca 100644
--- a/drivers/scsi/csiostor/csio_hw.c
+++ b/drivers/scsi/csiostor/csio_hw.c
@@ -3845,8 +3845,10 @@ csio_hw_start(struct csio_hw *hw)
 
 	if (csio_is_hw_ready(hw))
 		return 0;
-	else
+	else if (csio_match_state(hw, csio_hws_uninit))
 		return -EINVAL;
+	else
+		return -ENODEV;
 }
 
 int
diff --git a/drivers/scsi/csiostor/csio_hw.h b/drivers/scsi/csiostor/csio_hw.h
index 9acb89538e29..667046419b19 100644
--- a/drivers/scsi/csiostor/csio_hw.h
+++ b/drivers/scsi/csiostor/csio_hw.h
@@ -465,7 +465,7 @@ struct csio_hw {
 	struct csio_pport	pport[CSIO_MAX_PPORTS];	/* Ports (XGMACs) */
 	struct csio_hw_params	params;			/* Hw parameters */
 
-	struct pci_pool		*scsi_pci_pool;		/* PCI pool for SCSI */
+	struct dma_pool		*scsi_dma_pool;		/* DMA pool for SCSI */
 	mempool_t		*mb_mempool;		/* Mailbox memory pool*/
 	mempool_t		*rnode_mempool;		/* rnode memory pool */
 
diff --git a/drivers/scsi/csiostor/csio_init.c b/drivers/scsi/csiostor/csio_init.c
index ea0c31086cc6..28a9c7d706cb 100644
--- a/drivers/scsi/csiostor/csio_init.c
+++ b/drivers/scsi/csiostor/csio_init.c
@@ -485,9 +485,10 @@ csio_resource_alloc(struct csio_hw *hw)
 	if (!hw->rnode_mempool)
 		goto err_free_mb_mempool;
 
-	hw->scsi_pci_pool = pci_pool_create("csio_scsi_pci_pool", hw->pdev,
-					    CSIO_SCSI_RSP_LEN, 8, 0);
-	if (!hw->scsi_pci_pool)
+	hw->scsi_dma_pool = dma_pool_create("csio_scsi_dma_pool",
+					    &hw->pdev->dev, CSIO_SCSI_RSP_LEN,
+					    8, 0);
+	if (!hw->scsi_dma_pool)
 		goto err_free_rn_pool;
 
 	return 0;
@@ -505,8 +506,8 @@ err:
 static void
 csio_resource_free(struct csio_hw *hw)
 {
-	pci_pool_destroy(hw->scsi_pci_pool);
-	hw->scsi_pci_pool = NULL;
+	dma_pool_destroy(hw->scsi_dma_pool);
+	hw->scsi_dma_pool = NULL;
 	mempool_destroy(hw->rnode_mempool);
 	hw->rnode_mempool = NULL;
 	mempool_destroy(hw->mb_mempool);
@@ -969,10 +970,14 @@ static int csio_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 
 	pci_set_drvdata(pdev, hw);
 
-	if (csio_hw_start(hw) != 0) {
-		dev_err(&pdev->dev,
-			"Failed to start FW, continuing in debug mode.\n");
-		return 0;
+	rv = csio_hw_start(hw);
+	if (rv) {
+		if (rv == -EINVAL) {
+			dev_err(&pdev->dev,
+				"Failed to start FW, continuing in debug mode.\n");
+			return 0;
+		}
+		goto err_lnode_exit;
 	}
 
 	sprintf(hw->fwrev_str, "%u.%u.%u.%u\n",
diff --git a/drivers/scsi/csiostor/csio_scsi.c b/drivers/scsi/csiostor/csio_scsi.c
index a1ff75f1384f..dab0d3f9bee1 100644
--- a/drivers/scsi/csiostor/csio_scsi.c
+++ b/drivers/scsi/csiostor/csio_scsi.c
@@ -2445,7 +2445,7 @@ csio_scsim_init(struct csio_scsim *scm, struct csio_hw *hw)
 
 		/* Allocate Dma buffers for Response Payload */
 		dma_buf = &ioreq->dma_buf;
-		dma_buf->vaddr = pci_pool_alloc(hw->scsi_pci_pool, GFP_KERNEL,
+		dma_buf->vaddr = dma_pool_alloc(hw->scsi_dma_pool, GFP_KERNEL,
 						&dma_buf->paddr);
 		if (!dma_buf->vaddr) {
 			csio_err(hw,
@@ -2485,7 +2485,7 @@ free_ioreq:
 		ioreq = (struct csio_ioreq *)tmp;
 
 		dma_buf = &ioreq->dma_buf;
-		pci_pool_free(hw->scsi_pci_pool, dma_buf->vaddr,
+		dma_pool_free(hw->scsi_dma_pool, dma_buf->vaddr,
 			      dma_buf->paddr);
 
 		kfree(ioreq);
@@ -2516,7 +2516,7 @@ csio_scsim_exit(struct csio_scsim *scm)
 		ioreq = (struct csio_ioreq *)tmp;
 
 		dma_buf = &ioreq->dma_buf;
-		pci_pool_free(scm->hw->scsi_pci_pool, dma_buf->vaddr,
+		dma_pool_free(scm->hw->scsi_dma_pool, dma_buf->vaddr,
 			      dma_buf->paddr);
 
 		kfree(ioreq);
diff --git a/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c
index a69a9ac836f5..1d02cf9fe06c 100644
--- a/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c
+++ b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c
@@ -1635,6 +1635,9 @@ static int init_act_open(struct cxgbi_sock *csk)
 		goto rel_resource;
 	}
 
+	if (!(n->nud_state & NUD_VALID))
+		neigh_event_send(n, NULL);
+
 	csk->atid = cxgb4_alloc_atid(lldi->tids, csk);
 	if (csk->atid < 0) {
 		pr_err("%s, NO atid available.\n", ndev->name);
diff --git a/drivers/scsi/cxgbi/libcxgbi.c b/drivers/scsi/cxgbi/libcxgbi.c
index 1a4cfa562a60..512c8f1ea5b0 100644
--- a/drivers/scsi/cxgbi/libcxgbi.c
+++ b/drivers/scsi/cxgbi/libcxgbi.c
@@ -585,19 +585,21 @@ static struct cxgbi_sock *cxgbi_sock_create(struct cxgbi_device *cdev)
 
 static struct rtable *find_route_ipv4(struct flowi4 *fl4,
 				      __be32 saddr, __be32 daddr,
-				      __be16 sport, __be16 dport, u8 tos)
+				      __be16 sport, __be16 dport, u8 tos,
+				      int ifindex)
 {
 	struct rtable *rt;
 
 	rt = ip_route_output_ports(&init_net, fl4, NULL, daddr, saddr,
-				   dport, sport, IPPROTO_TCP, tos, 0);
+				   dport, sport, IPPROTO_TCP, tos, ifindex);
 	if (IS_ERR(rt))
 		return NULL;
 
 	return rt;
 }
 
-static struct cxgbi_sock *cxgbi_check_route(struct sockaddr *dst_addr)
+static struct cxgbi_sock *
+cxgbi_check_route(struct sockaddr *dst_addr, int ifindex)
 {
 	struct sockaddr_in *daddr = (struct sockaddr_in *)dst_addr;
 	struct dst_entry *dst;
@@ -611,7 +613,8 @@ static struct cxgbi_sock *cxgbi_check_route(struct sockaddr *dst_addr)
 	int port = 0xFFFF;
 	int err = 0;
 
-	rt = find_route_ipv4(&fl4, 0, daddr->sin_addr.s_addr, 0, daddr->sin_port, 0);
+	rt = find_route_ipv4(&fl4, 0, daddr->sin_addr.s_addr, 0,
+			     daddr->sin_port, 0, ifindex);
 	if (!rt) {
 		pr_info("no route to ipv4 0x%x, port %u.\n",
 			be32_to_cpu(daddr->sin_addr.s_addr),
@@ -693,11 +696,13 @@ err_out:
 
 #if IS_ENABLED(CONFIG_IPV6)
 static struct rt6_info *find_route_ipv6(const struct in6_addr *saddr,
-					const struct in6_addr *daddr)
+					const struct in6_addr *daddr,
+					int ifindex)
 {
 	struct flowi6 fl;
 
 	memset(&fl, 0, sizeof(fl));
+	fl.flowi6_oif = ifindex;
 	if (saddr)
 		memcpy(&fl.saddr, saddr, sizeof(struct in6_addr));
 	if (daddr)
@@ -705,7 +710,8 @@ static struct rt6_info *find_route_ipv6(const struct in6_addr *saddr,
 	return (struct rt6_info *)ip6_route_output(&init_net, NULL, &fl);
 }
 
-static struct cxgbi_sock *cxgbi_check_route6(struct sockaddr *dst_addr)
+static struct cxgbi_sock *
+cxgbi_check_route6(struct sockaddr *dst_addr, int ifindex)
 {
 	struct sockaddr_in6 *daddr6 = (struct sockaddr_in6 *)dst_addr;
 	struct dst_entry *dst;
@@ -719,7 +725,7 @@ static struct cxgbi_sock *cxgbi_check_route6(struct sockaddr *dst_addr)
 	int port = 0xFFFF;
 	int err = 0;
 
-	rt = find_route_ipv6(NULL, &daddr6->sin6_addr);
+	rt = find_route_ipv6(NULL, &daddr6->sin6_addr, ifindex);
 
 	if (!rt) {
 		pr_info("no route to ipv6 %pI6 port %u\n",
@@ -2536,6 +2542,7 @@ struct iscsi_endpoint *cxgbi_ep_connect(struct Scsi_Host *shost,
 	struct cxgbi_endpoint *cep;
 	struct cxgbi_hba *hba = NULL;
 	struct cxgbi_sock *csk;
+	int ifindex = 0;
 	int err = -EINVAL;
 
 	log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK,
@@ -2548,13 +2555,15 @@ struct iscsi_endpoint *cxgbi_ep_connect(struct Scsi_Host *shost,
 			pr_info("shost 0x%p, priv NULL.\n", shost);
 			goto err_out;
 		}
+
+		ifindex = hba->ndev->ifindex;
 	}
 
 	if (dst_addr->sa_family == AF_INET) {
-		csk = cxgbi_check_route(dst_addr);
+		csk = cxgbi_check_route(dst_addr, ifindex);
 #if IS_ENABLED(CONFIG_IPV6)
 	} else if (dst_addr->sa_family == AF_INET6) {
-		csk = cxgbi_check_route6(dst_addr);
+		csk = cxgbi_check_route6(dst_addr, ifindex);
 #endif
 	} else {
 		pr_info("address family 0x%x NOT supported.\n",
diff --git a/drivers/scsi/cxlflash/main.c b/drivers/scsi/cxlflash/main.c
index 6a4367cc9caa..76b8b7eed0c0 100644
--- a/drivers/scsi/cxlflash/main.c
+++ b/drivers/scsi/cxlflash/main.c
@@ -820,8 +820,7 @@ static void term_afu(struct cxlflash_cfg *cfg)
 	for (k = cfg->afu->num_hwqs - 1; k >= 0; k--)
 		term_intr(cfg, UNMAP_THREE, k);
 
-	if (cfg->afu)
-		stop_afu(cfg);
+	stop_afu(cfg);
 
 	for (k = cfg->afu->num_hwqs - 1; k >= 0; k--)
 		term_mc(cfg, k);
diff --git a/drivers/scsi/cxlflash/superpipe.c b/drivers/scsi/cxlflash/superpipe.c
index ad0f9968ccfb..ed46e8df2e42 100644
--- a/drivers/scsi/cxlflash/superpipe.c
+++ b/drivers/scsi/cxlflash/superpipe.c
@@ -1390,6 +1390,7 @@ static int cxlflash_disk_attach(struct scsi_device *sdev,
 	if (unlikely(!ctxi)) {
 		dev_err(dev, "%s: Failed to create context ctxid=%d\n",
 			__func__, ctxid);
+		rc = -ENOMEM;
 		goto err;
 	}
 
@@ -1650,6 +1651,7 @@ static int cxlflash_afu_recover(struct scsi_device *sdev,
 	u64 ctxid = DECODE_CTXID(recover->context_id),
 	    rctxid = recover->context_id;
 	long reg;
+	bool locked = true;
 	int lretry = 20; /* up to 2 seconds */
 	int new_adap_fd = -1;
 	int rc = 0;
@@ -1658,8 +1660,11 @@ static int cxlflash_afu_recover(struct scsi_device *sdev,
 	up_read(&cfg->ioctl_rwsem);
 	rc = mutex_lock_interruptible(mutex);
 	down_read(&cfg->ioctl_rwsem);
-	if (rc)
+	if (rc) {
+		locked = false;
 		goto out;
+	}
+
 	rc = check_state(cfg);
 	if (rc) {
 		dev_err(dev, "%s: Failed state rc=%d\n", __func__, rc);
@@ -1693,8 +1698,10 @@ retry_recover:
 				mutex_unlock(mutex);
 				msleep(100);
 				rc = mutex_lock_interruptible(mutex);
-				if (rc)
+				if (rc) {
+					locked = false;
 					goto out;
+				}
 				goto retry_recover;
 			}
 
@@ -1738,7 +1745,8 @@ retry_recover:
 out:
 	if (likely(ctxi))
 		put_context(ctxi);
-	mutex_unlock(mutex);
+	if (locked)
+		mutex_unlock(mutex);
 	atomic_dec_if_positive(&cfg->recovery_threads);
 	return rc;
 }
diff --git a/drivers/scsi/cxlflash/vlun.c b/drivers/scsi/cxlflash/vlun.c
index bdfb93061460..703bf1e9a64a 100644
--- a/drivers/scsi/cxlflash/vlun.c
+++ b/drivers/scsi/cxlflash/vlun.c
@@ -694,11 +694,7 @@ static int shrink_lxt(struct afu *afu,
 	/* Free LBAs allocated to freed chunks */
 	mutex_lock(&blka->mutex);
 	for (i = delta - 1; i >= 0; i--) {
-		/* Mask the higher 48 bits before shifting, even though
-		 * it is a noop
-		 */
-		aun = (lxt_old[my_new_size + i].rlba_base & SISL_ASTATUS_MASK);
-		aun = (aun >> MC_CHUNK_SHIFT);
+		aun = lxt_old[my_new_size + i].rlba_base >> MC_CHUNK_SHIFT;
 		if (needs_ws)
 			write_same16(sdev, aun, MC_CHUNK_SIZE);
 		ba_free(&blka->ba_lun, aun);
diff --git a/drivers/scsi/dmx3191d.c b/drivers/scsi/dmx3191d.c
index 6af3394d051d..003c3d726238 100644
--- a/drivers/scsi/dmx3191d.c
+++ b/drivers/scsi/dmx3191d.c
@@ -58,7 +58,7 @@ static struct scsi_host_template dmx3191d_driver_template = {
 	.info			= NCR5380_info,
 	.queuecommand		= NCR5380_queue_command,
 	.eh_abort_handler	= NCR5380_abort,
-	.eh_bus_reset_handler	= NCR5380_bus_reset,
+	.eh_host_reset_handler	= NCR5380_host_reset,
 	.can_queue		= 32,
 	.this_id		= 7,
 	.sg_tablesize		= SG_ALL,
diff --git a/drivers/scsi/dpt_i2o.c b/drivers/scsi/dpt_i2o.c
index 256dd6791fcc..fd172b0890d3 100644
--- a/drivers/scsi/dpt_i2o.c
+++ b/drivers/scsi/dpt_i2o.c
@@ -1169,11 +1169,6 @@ static struct adpt_device* adpt_find_device(adpt_hba* pHba, u32 chan, u32 id, u6
 	if(chan < 0 || chan >= MAX_CHANNEL)
 		return NULL;
 	
-	if( pHba->channel[chan].device == NULL){
-		printk(KERN_DEBUG"Adaptec I2O RAID: Trying to find device before they are allocated\n");
-		return NULL;
-	}
-
 	d = pHba->channel[chan].device[id];
 	if(!d || d->tid == 0) {
 		return NULL;
diff --git a/drivers/scsi/eata.c b/drivers/scsi/eata.c
index 227dd2c2ec2f..6501c330d8c8 100644
--- a/drivers/scsi/eata.c
+++ b/drivers/scsi/eata.c
@@ -1899,7 +1899,6 @@ static int eata2x_eh_abort(struct scsi_cmnd *SCarg)
 static int eata2x_eh_host_reset(struct scsi_cmnd *SCarg)
 {
 	unsigned int i, time, k, c, limit = 0;
-	int arg_done = 0;
 	struct scsi_cmnd *SCpnt;
 	struct Scsi_Host *shost = SCarg->device->host;
 	struct hostdata *ha = (struct hostdata *)shost->hostdata;
@@ -1967,9 +1966,6 @@ static int eata2x_eh_host_reset(struct scsi_cmnd *SCarg)
 		if (SCpnt->scsi_done == NULL)
 			panic("%s: reset, mbox %d, SCpnt->scsi_done == NULL.\n",
 			      ha->board_name, i);
-
-		if (SCpnt == SCarg)
-			arg_done = 1;
 	}
 
 	if (do_dma(shost->io_port, 0, RESET_PIO)) {
@@ -2037,10 +2033,7 @@ static int eata2x_eh_host_reset(struct scsi_cmnd *SCarg)
 	ha->in_reset = 0;
 	do_trace = 0;
 
-	if (arg_done)
-		printk("%s: reset, exit, done.\n", ha->board_name);
-	else
-		printk("%s: reset, exit.\n", ha->board_name);
+	printk("%s: reset, exit.\n", ha->board_name);
 
 	spin_unlock_irq(shost->host_lock);
 	return SUCCESS;
diff --git a/drivers/scsi/esas2r/esas2r_main.c b/drivers/scsi/esas2r/esas2r_main.c
index f2e9d8aa979c..81f226be3e3b 100644
--- a/drivers/scsi/esas2r/esas2r_main.c
+++ b/drivers/scsi/esas2r/esas2r_main.c
@@ -309,7 +309,7 @@ MODULE_PARM_DESC(interrupt_mode,
 		 "Defines the interrupt mode to use.  0 for legacy"
 		 ", 1 for MSI.  Default is MSI (1).");
 
-static struct pci_device_id
+static const struct pci_device_id
 	esas2r_pci_table[] = {
 	{ ATTO_VENDOR_ID, 0x0049,	  ATTO_VENDOR_ID, 0x0049,
 	  0,
diff --git a/drivers/scsi/esp_scsi.c b/drivers/scsi/esp_scsi.c
index 71cb05b1c3eb..c3fc34b9964d 100644
--- a/drivers/scsi/esp_scsi.c
+++ b/drivers/scsi/esp_scsi.c
@@ -597,14 +597,12 @@ static int esp_alloc_lun_tag(struct esp_cmd_entry *ent,
 
 		lp->non_tagged_cmd = ent;
 		return 0;
-	} else {
-		/* Tagged command, see if blocked by a
-		 * non-tagged one.
-		 */
-		if (lp->non_tagged_cmd || lp->hold)
-			return -EBUSY;
 	}
 
+	/* Tagged command. Check that it isn't blocked by a non-tagged one. */
+	if (lp->non_tagged_cmd || lp->hold)
+		return -EBUSY;
+
 	BUG_ON(lp->tagged_cmds[ent->orig_tag[1]]);
 
 	lp->tagged_cmds[ent->orig_tag[1]] = ent;
@@ -1210,12 +1208,6 @@ static int esp_reconnect(struct esp *esp)
 
 	esp->active_cmd = ent;
 
-	if (ent->flags & ESP_CMD_FLAG_ABORT) {
-		esp->msg_out[0] = ABORT_TASK_SET;
-		esp->msg_out_len = 1;
-		scsi_esp_cmd(esp, ESP_CMD_SATN);
-	}
-
 	esp_event(esp, ESP_EVENT_CHECK_PHASE);
 	esp_restore_pointers(esp, ent);
 	esp->flags |= ESP_FLAG_QUICKIRQ_CHECK;
@@ -1230,9 +1222,6 @@ static int esp_finish_select(struct esp *esp)
 {
 	struct esp_cmd_entry *ent;
 	struct scsi_cmnd *cmd;
-	u8 orig_select_state;
-
-	orig_select_state = esp->select_state;
 
 	/* No longer selecting.  */
 	esp->select_state = ESP_SELECT_NONE;
@@ -1496,9 +1485,8 @@ static void esp_msgin_reject(struct esp *esp)
 		return;
 	}
 
-	esp->msg_out[0] = ABORT_TASK_SET;
-	esp->msg_out_len = 1;
-	scsi_esp_cmd(esp, ESP_CMD_SATN);
+	shost_printk(KERN_INFO, esp->host, "Unexpected MESSAGE REJECT\n");
+	esp_schedule_reset(esp);
 }
 
 static void esp_msgin_sdtr(struct esp *esp, struct esp_target_data *tp)
@@ -1621,7 +1609,7 @@ static void esp_msgin_extended(struct esp *esp)
 	shost_printk(KERN_INFO, esp->host,
 		     "Unexpected extended msg type %x\n", esp->msg_in[2]);
 
-	esp->msg_out[0] = ABORT_TASK_SET;
+	esp->msg_out[0] = MESSAGE_REJECT;
 	esp->msg_out_len = 1;
 	scsi_esp_cmd(esp, ESP_CMD_SATN);
 }
@@ -1745,7 +1733,6 @@ again:
 			return 0;
 		}
 		goto again;
-		break;
 
 	case ESP_EVENT_DATA_IN:
 		write = 1;
@@ -1956,12 +1943,16 @@ again:
 		} else {
 			if (esp->msg_out_len > 1)
 				esp->ops->dma_invalidate(esp);
-		}
 
-		if (!(esp->ireg & ESP_INTR_DC)) {
-			if (esp->rev != FASHME)
+			/* XXX if the chip went into disconnected mode,
+			 * we can't run the phase state machine anyway.
+			 */
+			if (!(esp->ireg & ESP_INTR_DC))
 				scsi_esp_cmd(esp, ESP_CMD_NULL);
 		}
+
+		esp->msg_out_len = 0;
+
 		esp_event(esp, ESP_EVENT_CHECK_PHASE);
 		goto again;
 	case ESP_EVENT_MSGIN:
@@ -1998,6 +1989,10 @@ again:
 
 			scsi_esp_cmd(esp, ESP_CMD_MOK);
 
+			/* Check whether a bus reset is to be done next */
+			if (esp->event == ESP_EVENT_RESET)
+				return 0;
+
 			if (esp->event != ESP_EVENT_FREE_BUS)
 				esp_event(esp, ESP_EVENT_CHECK_PHASE);
 		} else {
@@ -2022,7 +2017,6 @@ again:
 		}
 		esp_schedule_reset(esp);
 		return 0;
-		break;
 
 	case ESP_EVENT_RESET:
 		scsi_esp_cmd(esp, ESP_CMD_RS);
@@ -2033,7 +2027,6 @@ again:
 			     "Unexpected event %x, resetting\n", esp->event);
 		esp_schedule_reset(esp);
 		return 0;
-		break;
 	}
 	return 1;
 }
@@ -2170,14 +2163,14 @@ static void __esp_interrupt(struct esp *esp)
 
 		esp_schedule_reset(esp);
 	} else {
-		if (!(esp->ireg & ESP_INTR_RSEL)) {
-			/* Some combination of FDONE, BSERV, DC.  */
-			if (esp->select_state != ESP_SELECT_NONE)
-				intr_done = esp_finish_select(esp);
-		} else if (esp->ireg & ESP_INTR_RSEL) {
+		if (esp->ireg & ESP_INTR_RSEL) {
 			if (esp->active_cmd)
 				(void) esp_finish_select(esp);
 			intr_done = esp_reconnect(esp);
+		} else {
+			/* Some combination of FDONE, BSERV, DC. */
+			if (esp->select_state != ESP_SELECT_NONE)
+				intr_done = esp_finish_select(esp);
 		}
 	}
 	while (!intr_done)
diff --git a/drivers/scsi/esp_scsi.h b/drivers/scsi/esp_scsi.h
index 84dcbe4a6268..7e8932ae91f8 100644
--- a/drivers/scsi/esp_scsi.h
+++ b/drivers/scsi/esp_scsi.h
@@ -281,7 +281,6 @@ struct esp_cmd_entry {
 
 	u8			flags;
 #define ESP_CMD_FLAG_WRITE	0x01 /* DMA is a write */
-#define ESP_CMD_FLAG_ABORT	0x02 /* being aborted */
 #define ESP_CMD_FLAG_AUTOSENSE	0x04 /* Doing automatic REQUEST_SENSE */
 #define ESP_CMD_FLAG_RESIDUAL	0x08 /* AM53c974 BLAST residual */
 
diff --git a/drivers/scsi/fcoe/fcoe_sysfs.c b/drivers/scsi/fcoe/fcoe_sysfs.c
index 9cf3d56296ab..5c8310bade61 100644
--- a/drivers/scsi/fcoe/fcoe_sysfs.c
+++ b/drivers/scsi/fcoe/fcoe_sysfs.c
@@ -659,13 +659,13 @@ static void fcoe_fcf_device_release(struct device *dev)
 	kfree(fcf);
 }
 
-static struct device_type fcoe_ctlr_device_type = {
+static const struct device_type fcoe_ctlr_device_type = {
 	.name = "fcoe_ctlr",
 	.groups = fcoe_ctlr_attr_groups,
 	.release = fcoe_ctlr_device_release,
 };
 
-static struct device_type fcoe_fcf_device_type = {
+static const struct device_type fcoe_fcf_device_type = {
 	.name = "fcoe_fcf",
 	.groups = fcoe_fcf_attr_groups,
 	.release = fcoe_fcf_device_release,
diff --git a/drivers/scsi/fdomain.c b/drivers/scsi/fdomain.c
index b87ab38a4530..ebbe5a3e665d 100644
--- a/drivers/scsi/fdomain.c
+++ b/drivers/scsi/fdomain.c
@@ -933,7 +933,7 @@ struct Scsi_Host *__fdomain_16x0_detect(struct scsi_host_template *tpnt )
       }
    }
 
-   fdomain_16x0_bus_reset(NULL);
+   fdomain_16x0_host_reset(NULL);
 
    if (fdomain_test_loopback()) {
       printk(KERN_ERR  "scsi: <fdomain> Detection failed (loopback test failed at port base 0x%x)\n", port_base);
@@ -1568,7 +1568,7 @@ static int fdomain_16x0_abort(struct scsi_cmnd *SCpnt)
    return SUCCESS;
 }
 
-int fdomain_16x0_bus_reset(struct scsi_cmnd *SCpnt)
+int fdomain_16x0_host_reset(struct scsi_cmnd *SCpnt)
 {
    unsigned long flags;
 
@@ -1758,7 +1758,7 @@ struct scsi_host_template fdomain_driver_template = {
 	.info			= fdomain_16x0_info,
 	.queuecommand		= fdomain_16x0_queue,
 	.eh_abort_handler	= fdomain_16x0_abort,
-	.eh_bus_reset_handler	= fdomain_16x0_bus_reset,
+	.eh_host_reset_handler	= fdomain_16x0_host_reset,
 	.bios_param		= fdomain_16x0_biosparam,
 	.release		= fdomain_16x0_release,
 	.can_queue		= 1,
diff --git a/drivers/scsi/fdomain.h b/drivers/scsi/fdomain.h
index 47021d9d4fe4..5cbe86b573ae 100644
--- a/drivers/scsi/fdomain.h
+++ b/drivers/scsi/fdomain.h
@@ -21,4 +21,4 @@
 extern struct scsi_host_template fdomain_driver_template;
 extern int fdomain_setup(char *str);
 extern struct Scsi_Host *__fdomain_16x0_detect(struct  scsi_host_template *tpnt );
-extern int fdomain_16x0_bus_reset(struct scsi_cmnd *SCpnt);
+extern int fdomain_16x0_host_reset(struct scsi_cmnd *SCpnt);
diff --git a/drivers/scsi/fnic/fnic.h b/drivers/scsi/fnic/fnic.h
index 67aab965c0f4..d094ba59ed15 100644
--- a/drivers/scsi/fnic/fnic.h
+++ b/drivers/scsi/fnic/fnic.h
@@ -180,7 +180,7 @@ enum fnic_msix_intr_index {
 
 struct fnic_msix_entry {
 	int requested;
-	char devname[IFNAMSIZ];
+	char devname[IFNAMSIZ + 11];
 	irqreturn_t (*isr)(int, void *);
 	void *devid;
 };
diff --git a/drivers/scsi/fnic/fnic_scsi.c b/drivers/scsi/fnic/fnic_scsi.c
index 6c0646d62dfb..242e2ee494a1 100644
--- a/drivers/scsi/fnic/fnic_scsi.c
+++ b/drivers/scsi/fnic/fnic_scsi.c
@@ -1990,10 +1990,6 @@ int fnic_abort_cmd(struct scsi_cmnd *sc)
 		FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,
 			"Issuing Host reset due to out of order IO\n");
 
-		if (fnic_host_reset(sc) == FAILED) {
-			FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,
-				"fnic_host_reset failed.\n");
-		}
 		ret = FAILED;
 		goto fnic_abort_cmd_end;
 	}
diff --git a/drivers/scsi/g_NCR5380.c b/drivers/scsi/g_NCR5380.c
index c34fc91ba486..fc538181f8df 100644
--- a/drivers/scsi/g_NCR5380.c
+++ b/drivers/scsi/g_NCR5380.c
@@ -1,17 +1,17 @@
 /*
  * Generic Generic NCR5380 driver
- *	
+ *
  * Copyright 1993, Drew Eckhardt
- *	Visionary Computing
- *	(Unix and Linux consulting and custom programming)
- *	drew@colorado.edu
- *      +1 (303) 440-4894
+ * Visionary Computing
+ * (Unix and Linux consulting and custom programming)
+ * drew@colorado.edu
+ * +1 (303) 440-4894
  *
  * NCR53C400 extensions (c) 1994,1995,1996, Kevin Lentin
- *    K.Lentin@cs.monash.edu.au
+ * K.Lentin@cs.monash.edu.au
  *
  * NCR53C400A extensions (c) 1996, Ingmar Baumgart
- *    ingmar@gonzo.schwaben.de
+ * ingmar@gonzo.schwaben.de
  *
  * DTC3181E extensions (c) 1997, Ronald van Cuijlenborg
  * ronald.van.cuijlenborg@tip.nl or nutty@dds.nl
@@ -44,17 +44,19 @@
 	int c400_ctl_status; \
 	int c400_blk_cnt; \
 	int c400_host_buf; \
-	int io_width
+	int io_width; \
+	int pdma_residual; \
+	int board
 
 #define NCR5380_dma_xfer_len            generic_NCR5380_dma_xfer_len
-#define NCR5380_dma_recv_setup          generic_NCR5380_pread
-#define NCR5380_dma_send_setup          generic_NCR5380_pwrite
-#define NCR5380_dma_residual            NCR5380_dma_residual_none
+#define NCR5380_dma_recv_setup          generic_NCR5380_precv
+#define NCR5380_dma_send_setup          generic_NCR5380_psend
+#define NCR5380_dma_residual            generic_NCR5380_dma_residual
 
 #define NCR5380_intr                    generic_NCR5380_intr
 #define NCR5380_queue_command           generic_NCR5380_queue_command
 #define NCR5380_abort                   generic_NCR5380_abort
-#define NCR5380_bus_reset               generic_NCR5380_bus_reset
+#define NCR5380_host_reset              generic_NCR5380_host_reset
 #define NCR5380_info                    generic_NCR5380_info
 
 #define NCR5380_io_delay(x)             udelay(x)
@@ -76,6 +78,7 @@
 #define IRQ_AUTO 254
 
 #define MAX_CARDS 8
+#define DMA_MAX_SIZE 32768
 
 /* old-style parameters for compatibility */
 static int ncr_irq = -1;
@@ -314,6 +317,7 @@ static int generic_NCR5380_init_one(struct scsi_host_template *tpnt,
 	}
 	hostdata = shost_priv(instance);
 
+	hostdata->board = board;
 	hostdata->io = iomem;
 	hostdata->region_size = region_size;
 
@@ -478,180 +482,210 @@ static void generic_NCR5380_release_resources(struct Scsi_Host *instance)
 		release_mem_region(base, region_size);
 }
 
+/* wait_for_53c80_access - wait for 53C80 registers to become accessible
+ * @hostdata: scsi host private data
+ *
+ * The registers within the 53C80 logic block are inaccessible until
+ * bit 7 in the 53C400 control status register gets asserted.
+ */
+
+static void wait_for_53c80_access(struct NCR5380_hostdata *hostdata)
+{
+	int count = 10000;
+
+	do {
+		if (hostdata->board == BOARD_DTC3181E)
+			udelay(4); /* DTC436 chip hangs without this */
+		if (NCR5380_read(hostdata->c400_ctl_status) & CSR_53C80_REG)
+			return;
+	} while (--count > 0);
+
+	scmd_printk(KERN_ERR, hostdata->connected,
+	            "53c80 registers not accessible, device will be reset\n");
+	NCR5380_write(hostdata->c400_ctl_status, CSR_RESET);
+	NCR5380_write(hostdata->c400_ctl_status, CSR_BASE);
+}
+
 /**
- *	generic_NCR5380_pread - pseudo DMA read
- *	@hostdata: scsi host private data
- *	@dst: buffer to read into
- *	@len: buffer length
+ * generic_NCR5380_precv - pseudo DMA receive
+ * @hostdata: scsi host private data
+ * @dst: buffer to write into
+ * @len: transfer size
  *
- *	Perform a pseudo DMA mode read from an NCR53C400 or equivalent
- *	controller
+ * Perform a pseudo DMA mode receive from a 53C400 or equivalent device.
  */
- 
-static inline int generic_NCR5380_pread(struct NCR5380_hostdata *hostdata,
+
+static inline int generic_NCR5380_precv(struct NCR5380_hostdata *hostdata,
                                         unsigned char *dst, int len)
 {
-	int blocks = len / 128;
+	int residual;
 	int start = 0;
 
 	NCR5380_write(hostdata->c400_ctl_status, CSR_BASE | CSR_TRANS_DIR);
-	NCR5380_write(hostdata->c400_blk_cnt, blocks);
-	while (1) {
-		if (NCR5380_read(hostdata->c400_blk_cnt) == 0)
-			break;
-		if (NCR5380_read(hostdata->c400_ctl_status) & CSR_GATED_53C80_IRQ) {
-			printk(KERN_ERR "53C400r: Got 53C80_IRQ start=%d, blocks=%d\n", start, blocks);
-			return -1;
+	NCR5380_write(hostdata->c400_blk_cnt, len / 128);
+
+	do {
+		if (start == len - 128) {
+			/* Ignore End of DMA interrupt for the final buffer */
+			if (NCR5380_poll_politely(hostdata, hostdata->c400_ctl_status,
+			                          CSR_HOST_BUF_NOT_RDY, 0, HZ / 64) < 0)
+				break;
+		} else {
+			if (NCR5380_poll_politely2(hostdata, hostdata->c400_ctl_status,
+			                           CSR_HOST_BUF_NOT_RDY, 0,
+			                           hostdata->c400_ctl_status,
+			                           CSR_GATED_53C80_IRQ,
+			                           CSR_GATED_53C80_IRQ, HZ / 64) < 0 ||
+			    NCR5380_read(hostdata->c400_ctl_status) & CSR_HOST_BUF_NOT_RDY)
+				break;
 		}
-		while (NCR5380_read(hostdata->c400_ctl_status) & CSR_HOST_BUF_NOT_RDY)
-			; /* FIXME - no timeout */
 
 		if (hostdata->io_port && hostdata->io_width == 2)
 			insw(hostdata->io_port + hostdata->c400_host_buf,
-							dst + start, 64);
+			     dst + start, 64);
 		else if (hostdata->io_port)
 			insb(hostdata->io_port + hostdata->c400_host_buf,
-							dst + start, 128);
+			     dst + start, 128);
 		else
 			memcpy_fromio(dst + start,
 				hostdata->io + NCR53C400_host_buffer, 128);
-
 		start += 128;
-		blocks--;
-	}
+	} while (start < len);
 
-	if (blocks) {
-		while (NCR5380_read(hostdata->c400_ctl_status) & CSR_HOST_BUF_NOT_RDY)
-			; /* FIXME - no timeout */
+	residual = len - start;
 
-		if (hostdata->io_port && hostdata->io_width == 2)
-			insw(hostdata->io_port + hostdata->c400_host_buf,
-							dst + start, 64);
-		else if (hostdata->io_port)
-			insb(hostdata->io_port + hostdata->c400_host_buf,
-							dst + start, 128);
-		else
-			memcpy_fromio(dst + start,
-				hostdata->io + NCR53C400_host_buffer, 128);
-
-		start += 128;
-		blocks--;
+	if (residual != 0) {
+		/* 53c80 interrupt or transfer timeout. Reset 53c400 logic. */
+		NCR5380_write(hostdata->c400_ctl_status, CSR_RESET);
+		NCR5380_write(hostdata->c400_ctl_status, CSR_BASE);
 	}
+	wait_for_53c80_access(hostdata);
 
-	if (!(NCR5380_read(hostdata->c400_ctl_status) & CSR_GATED_53C80_IRQ))
-		printk("53C400r: no 53C80 gated irq after transfer");
+	if (residual == 0 && NCR5380_poll_politely(hostdata, BUS_AND_STATUS_REG,
+	                                           BASR_END_DMA_TRANSFER,
+	                                           BASR_END_DMA_TRANSFER,
+	                                           HZ / 64) < 0)
+		scmd_printk(KERN_ERR, hostdata->connected, "%s: End of DMA timeout\n",
+		            __func__);
 
-	/* wait for 53C80 registers to be available */
-	while (!(NCR5380_read(hostdata->c400_ctl_status) & CSR_53C80_REG))
-		;
+	hostdata->pdma_residual = residual;
 
-	if (!(NCR5380_read(BUS_AND_STATUS_REG) & BASR_END_DMA_TRANSFER))
-		printk(KERN_ERR "53C400r: no end dma signal\n");
-		
 	return 0;
 }
 
 /**
- *	generic_NCR5380_pwrite - pseudo DMA write
- *	@hostdata: scsi host private data
- *	@dst: buffer to read into
- *	@len: buffer length
+ * generic_NCR5380_psend - pseudo DMA send
+ * @hostdata: scsi host private data
+ * @src: buffer to read from
+ * @len: transfer size
  *
- *	Perform a pseudo DMA mode read from an NCR53C400 or equivalent
- *	controller
+ * Perform a pseudo DMA mode send to a 53C400 or equivalent device.
  */
 
-static inline int generic_NCR5380_pwrite(struct NCR5380_hostdata *hostdata,
-                                         unsigned char *src, int len)
+static inline int generic_NCR5380_psend(struct NCR5380_hostdata *hostdata,
+                                        unsigned char *src, int len)
 {
-	int blocks = len / 128;
+	int residual;
 	int start = 0;
 
 	NCR5380_write(hostdata->c400_ctl_status, CSR_BASE);
-	NCR5380_write(hostdata->c400_blk_cnt, blocks);
-	while (1) {
-		if (NCR5380_read(hostdata->c400_ctl_status) & CSR_GATED_53C80_IRQ) {
-			printk(KERN_ERR "53C400w: Got 53C80_IRQ start=%d, blocks=%d\n", start, blocks);
-			return -1;
+	NCR5380_write(hostdata->c400_blk_cnt, len / 128);
+
+	do {
+		if (NCR5380_poll_politely2(hostdata, hostdata->c400_ctl_status,
+		                           CSR_HOST_BUF_NOT_RDY, 0,
+		                           hostdata->c400_ctl_status,
+		                           CSR_GATED_53C80_IRQ,
+		                           CSR_GATED_53C80_IRQ, HZ / 64) < 0 ||
+		    NCR5380_read(hostdata->c400_ctl_status) & CSR_HOST_BUF_NOT_RDY) {
+			/* Both 128 B buffers are in use */
+			if (start >= 128)
+				start -= 128;
+			if (start >= 128)
+				start -= 128;
+			break;
 		}
 
-		if (NCR5380_read(hostdata->c400_blk_cnt) == 0)
+		if (start >= len && NCR5380_read(hostdata->c400_blk_cnt) == 0)
 			break;
-		while (NCR5380_read(hostdata->c400_ctl_status) & CSR_HOST_BUF_NOT_RDY)
-			; // FIXME - timeout
 
-		if (hostdata->io_port && hostdata->io_width == 2)
-			outsw(hostdata->io_port + hostdata->c400_host_buf,
-							src + start, 64);
-		else if (hostdata->io_port)
-			outsb(hostdata->io_port + hostdata->c400_host_buf,
-							src + start, 128);
-		else
-			memcpy_toio(hostdata->io + NCR53C400_host_buffer,
-			            src + start, 128);
+		if (NCR5380_read(hostdata->c400_ctl_status) & CSR_GATED_53C80_IRQ) {
+			/* Host buffer is empty, other one is in use */
+			if (start >= 128)
+				start -= 128;
+			break;
+		}
 
-		start += 128;
-		blocks--;
-	}
-	if (blocks) {
-		while (NCR5380_read(hostdata->c400_ctl_status) & CSR_HOST_BUF_NOT_RDY)
-			; // FIXME - no timeout
+		if (start >= len)
+			continue;
 
 		if (hostdata->io_port && hostdata->io_width == 2)
 			outsw(hostdata->io_port + hostdata->c400_host_buf,
-							src + start, 64);
+			      src + start, 64);
 		else if (hostdata->io_port)
 			outsb(hostdata->io_port + hostdata->c400_host_buf,
-							src + start, 128);
+			      src + start, 128);
 		else
 			memcpy_toio(hostdata->io + NCR53C400_host_buffer,
 			            src + start, 128);
-
 		start += 128;
-		blocks--;
-	}
+	} while (1);
 
-	/* wait for 53C80 registers to be available */
-	while (!(NCR5380_read(hostdata->c400_ctl_status) & CSR_53C80_REG)) {
-		udelay(4); /* DTC436 chip hangs without this */
-		/* FIXME - no timeout */
-	}
+	residual = len - start;
 
-	if (!(NCR5380_read(BUS_AND_STATUS_REG) & BASR_END_DMA_TRANSFER)) {
-		printk(KERN_ERR "53C400w: no end dma signal\n");
+	if (residual != 0) {
+		/* 53c80 interrupt or transfer timeout. Reset 53c400 logic. */
+		NCR5380_write(hostdata->c400_ctl_status, CSR_RESET);
+		NCR5380_write(hostdata->c400_ctl_status, CSR_BASE);
 	}
+	wait_for_53c80_access(hostdata);
+
+	if (residual == 0) {
+		if (NCR5380_poll_politely(hostdata, TARGET_COMMAND_REG,
+		                          TCR_LAST_BYTE_SENT, TCR_LAST_BYTE_SENT,
+		                          HZ / 64) < 0)
+			scmd_printk(KERN_ERR, hostdata->connected,
+			            "%s: Last Byte Sent timeout\n", __func__);
+
+		if (NCR5380_poll_politely(hostdata, BUS_AND_STATUS_REG,
+		                          BASR_END_DMA_TRANSFER, BASR_END_DMA_TRANSFER,
+		                          HZ / 64) < 0)
+			scmd_printk(KERN_ERR, hostdata->connected, "%s: End of DMA timeout\n",
+			            __func__);
+	}
+
+	hostdata->pdma_residual = residual;
 
-	while (!(NCR5380_read(TARGET_COMMAND_REG) & TCR_LAST_BYTE_SENT))
-		; 	// TIMEOUT
 	return 0;
 }
 
 static int generic_NCR5380_dma_xfer_len(struct NCR5380_hostdata *hostdata,
                                         struct scsi_cmnd *cmd)
 {
-	int transfersize = cmd->transfersize;
+	int transfersize = cmd->SCp.this_residual;
 
 	if (hostdata->flags & FLAG_NO_PSEUDO_DMA)
 		return 0;
 
-	/* Limit transfers to 32K, for xx400 & xx406
-	 * pseudoDMA that transfers in 128 bytes blocks.
-	 */
-	if (transfersize > 32 * 1024 && cmd->SCp.this_residual &&
-	    !(cmd->SCp.this_residual % transfersize))
-		transfersize = 32 * 1024;
-
 	/* 53C400 datasheet: non-modulo-128-byte transfers should use PIO */
 	if (transfersize % 128)
-		transfersize = 0;
+		return 0;
+
+	/* Limit PDMA send to 512 B to avoid random corruption on DTC3181E */
+	if (hostdata->board == BOARD_DTC3181E &&
+	    cmd->sc_data_direction == DMA_TO_DEVICE)
+		transfersize = min(cmd->SCp.this_residual, 512);
 
-	return transfersize;
+	return min(transfersize, DMA_MAX_SIZE);
 }
 
-/*
- *	Include the NCR5380 core code that we build our driver around	
- */
- 
+static int generic_NCR5380_dma_residual(struct NCR5380_hostdata *hostdata)
+{
+	return hostdata->pdma_residual;
+}
+
+/* Include the core driver code. */
+
 #include "NCR5380.c"
 
 static struct scsi_host_template driver_template = {
@@ -661,7 +695,7 @@ static struct scsi_host_template driver_template = {
 	.info			= generic_NCR5380_info,
 	.queuecommand		= generic_NCR5380_queue_command,
 	.eh_abort_handler	= generic_NCR5380_abort,
-	.eh_bus_reset_handler	= generic_NCR5380_bus_reset,
+	.eh_host_reset_handler	= generic_NCR5380_host_reset,
 	.can_queue		= 16,
 	.this_id		= 7,
 	.sg_tablesize		= SG_ALL,
@@ -671,11 +705,10 @@ static struct scsi_host_template driver_template = {
 	.max_sectors		= 128,
 };
 
-
 static int generic_NCR5380_isa_match(struct device *pdev, unsigned int ndev)
 {
 	int ret = generic_NCR5380_init_one(&driver_template, pdev, base[ndev],
-					  irq[ndev], card[ndev]);
+	                                   irq[ndev], card[ndev]);
 	if (ret) {
 		if (base[ndev])
 			printk(KERN_WARNING "Card not found at address 0x%03x\n",
@@ -687,7 +720,7 @@ static int generic_NCR5380_isa_match(struct device *pdev, unsigned int ndev)
 }
 
 static int generic_NCR5380_isa_remove(struct device *pdev,
-				   unsigned int ndev)
+                                      unsigned int ndev)
 {
 	generic_NCR5380_release_resources(dev_get_drvdata(pdev));
 	dev_set_drvdata(pdev, NULL);
@@ -703,14 +736,14 @@ static struct isa_driver generic_NCR5380_isa_driver = {
 };
 
 #ifdef CONFIG_PNP
-static struct pnp_device_id generic_NCR5380_pnp_ids[] = {
+static const struct pnp_device_id generic_NCR5380_pnp_ids[] = {
 	{ .id = "DTC436e", .driver_data = BOARD_DTC3181E },
 	{ .id = "" }
 };
 MODULE_DEVICE_TABLE(pnp, generic_NCR5380_pnp_ids);
 
 static int generic_NCR5380_pnp_probe(struct pnp_dev *pdev,
-			       const struct pnp_device_id *id)
+                                     const struct pnp_device_id *id)
 {
 	int base, irq;
 
@@ -721,7 +754,7 @@ static int generic_NCR5380_pnp_probe(struct pnp_dev *pdev,
 	irq = pnp_irq(pdev, 0);
 
 	return generic_NCR5380_init_one(&driver_template, &pdev->dev, base, irq,
-				       id->driver_data);
+	                                id->driver_data);
 }
 
 static void generic_NCR5380_pnp_remove(struct pnp_dev *pdev)
diff --git a/drivers/scsi/gdth.c b/drivers/scsi/gdth.c
index facc7271f932..a4473356a9dc 100644
--- a/drivers/scsi/gdth.c
+++ b/drivers/scsi/gdth.c
@@ -2354,7 +2354,7 @@ static int gdth_internal_cache_cmd(gdth_ha_str *ha, Scsi_Cmnd *scp)
         inq.resp_aenc = 2;
         inq.add_length= 32;
         strcpy(inq.vendor,ha->oem_name);
-        sprintf(inq.product,"Host Drive  #%02d",t);
+        snprintf(inq.product, sizeof(inq.product), "Host Drive  #%02d",t);
         strcpy(inq.revision,"   ");
         gdth_copy_internal_data(ha, scp, (char*)&inq, sizeof(gdth_inq_data));
         break;
diff --git a/drivers/scsi/gdth_proc.c b/drivers/scsi/gdth_proc.c
index be609db66807..d08b2716752c 100644
--- a/drivers/scsi/gdth_proc.c
+++ b/drivers/scsi/gdth_proc.c
@@ -147,7 +147,7 @@ int gdth_show_info(struct seq_file *m, struct Scsi_Host *host)
 
     gdth_cmd_str *gdtcmd;
     gdth_evt_str *estr;
-    char hrec[161];
+    char hrec[277];
 
     char *buf;
     gdth_dskstat_str *pds;
diff --git a/drivers/scsi/gvp11.c b/drivers/scsi/gvp11.c
index 3b6f83ffddc4..a27fc49ebd3a 100644
--- a/drivers/scsi/gvp11.c
+++ b/drivers/scsi/gvp11.c
@@ -171,23 +171,6 @@ static void dma_stop(struct Scsi_Host *instance, struct scsi_cmnd *SCpnt,
 	}
 }
 
-static int gvp11_bus_reset(struct scsi_cmnd *cmd)
-{
-	struct Scsi_Host *instance = cmd->device->host;
-
-	/* FIXME perform bus-specific reset */
-
-	/* FIXME 2: shouldn't we no-op this function (return
-	   FAILED), and fall back to host reset function,
-	   wd33c93_host_reset ? */
-
-	spin_lock_irq(instance->host_lock);
-	wd33c93_host_reset(cmd);
-	spin_unlock_irq(instance->host_lock);
-
-	return SUCCESS;
-}
-
 static struct scsi_host_template gvp11_scsi_template = {
 	.module			= THIS_MODULE,
 	.name			= "GVP Series II SCSI",
@@ -196,7 +179,6 @@ static struct scsi_host_template gvp11_scsi_template = {
 	.proc_name		= "GVP11",
 	.queuecommand		= wd33c93_queuecommand,
 	.eh_abort_handler	= wd33c93_abort,
-	.eh_bus_reset_handler	= gvp11_bus_reset,
 	.eh_host_reset_handler	= wd33c93_host_reset,
 	.can_queue		= CAN_QUEUE,
 	.this_id		= 7,
diff --git a/drivers/scsi/hisi_sas/hisi_sas.h b/drivers/scsi/hisi_sas/hisi_sas.h
index a722f2bd72ab..07f4a4cfbec1 100644
--- a/drivers/scsi/hisi_sas/hisi_sas.h
+++ b/drivers/scsi/hisi_sas/hisi_sas.h
@@ -15,6 +15,7 @@
 #include <linux/acpi.h>
 #include <linux/clk.h>
 #include <linux/dmapool.h>
+#include <linux/iopoll.h>
 #include <linux/mfd/syscon.h>
 #include <linux/module.h>
 #include <linux/of_address.h>
@@ -25,14 +26,13 @@
 #include <scsi/sas_ata.h>
 #include <scsi/libsas.h>
 
-#define DRV_VERSION "v1.6"
-
 #define HISI_SAS_MAX_PHYS	9
 #define HISI_SAS_MAX_QUEUES	32
 #define HISI_SAS_QUEUE_SLOTS 512
 #define HISI_SAS_MAX_ITCT_ENTRIES 2048
 #define HISI_SAS_MAX_DEVICES HISI_SAS_MAX_ITCT_ENTRIES
 #define HISI_SAS_RESET_BIT	0
+#define HISI_SAS_REJECT_CMD_BIT	1
 
 #define HISI_SAS_STATUS_BUF_SZ (sizeof(struct hisi_sas_status_buffer))
 #define HISI_SAS_COMMAND_TABLE_SZ (sizeof(union hisi_sas_command_table))
@@ -90,6 +90,14 @@ enum hisi_sas_dev_type {
 	HISI_SAS_DEV_TYPE_SATA,
 };
 
+struct hisi_sas_hw_error {
+	u32 irq_msk;
+	u32 msk;
+	int shift;
+	const char *msg;
+	int reg;
+};
+
 struct hisi_sas_phy {
 	struct hisi_hba	*hisi_hba;
 	struct hisi_sas_port	*port;
@@ -132,6 +140,7 @@ struct hisi_sas_dq {
 struct hisi_sas_device {
 	struct hisi_hba		*hisi_hba;
 	struct domain_device	*sas_device;
+	struct completion *completion;
 	struct hisi_sas_dq	*dq;
 	struct list_head	list;
 	u64 attached_phy;
@@ -192,6 +201,7 @@ struct hisi_sas_hw {
 	void (*phy_enable)(struct hisi_hba *hisi_hba, int phy_no);
 	void (*phy_disable)(struct hisi_hba *hisi_hba, int phy_no);
 	void (*phy_hard_reset)(struct hisi_hba *hisi_hba, int phy_no);
+	void (*get_events)(struct hisi_hba *hisi_hba, int phy_no);
 	void (*phy_set_linkrate)(struct hisi_hba *hisi_hba, int phy_no,
 			struct sas_phy_linkrates *linkrates);
 	enum sas_linkrate (*phy_get_max_linkrate)(void);
@@ -201,6 +211,7 @@ struct hisi_sas_hw {
 	void (*dereg_device)(struct hisi_hba *hisi_hba,
 				struct domain_device *device);
 	int (*soft_reset)(struct hisi_hba *hisi_hba);
+	u32 (*get_phys_state)(struct hisi_hba *hisi_hba);
 	int max_command_entries;
 	int complete_hdr_size;
 };
@@ -390,6 +401,7 @@ struct hisi_sas_slot_buf_table {
 extern struct scsi_transport_template *hisi_sas_stt;
 extern struct scsi_host_template *hisi_sas_sht;
 
+extern void hisi_sas_stop_phys(struct hisi_hba *hisi_hba);
 extern void hisi_sas_init_add(struct hisi_hba *hisi_hba);
 extern int hisi_sas_alloc(struct hisi_hba *hisi_hba, struct Scsi_Host *shost);
 extern void hisi_sas_free(struct hisi_hba *hisi_hba);
@@ -408,6 +420,4 @@ extern void hisi_sas_slot_task_free(struct hisi_hba *hisi_hba,
 				    struct sas_task *task,
 				    struct hisi_sas_slot *slot);
 extern void hisi_sas_init_mem(struct hisi_hba *hisi_hba);
-extern void hisi_sas_rescan_topology(struct hisi_hba *hisi_hba, u32 old_state,
-				     u32 state);
 #endif
diff --git a/drivers/scsi/hisi_sas/hisi_sas_main.c b/drivers/scsi/hisi_sas/hisi_sas_main.c
index 4022c3f8295f..16664f2e15fb 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_main.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_main.c
@@ -61,6 +61,7 @@ u8 hisi_sas_get_ata_protocol(u8 cmd, int direction)
 	case ATA_CMD_WRITE_QUEUED:
 	case ATA_CMD_WRITE_LOG_DMA_EXT:
 	case ATA_CMD_WRITE_STREAM_DMA_EXT:
+	case ATA_CMD_ZAC_MGMT_IN:
 	return HISI_SAS_SATA_PROTOCOL_DMA;
 
 	case ATA_CMD_CHK_POWER:
@@ -73,6 +74,7 @@ u8 hisi_sas_get_ata_protocol(u8 cmd, int direction)
 	case ATA_CMD_SET_FEATURES:
 	case ATA_CMD_STANDBY:
 	case ATA_CMD_STANDBYNOW1:
+	case ATA_CMD_ZAC_MGMT_OUT:
 	return HISI_SAS_SATA_PROTOCOL_NONDATA;
 	default:
 		if (direction == DMA_NONE)
@@ -125,6 +127,15 @@ struct hisi_sas_port *to_hisi_sas_port(struct asd_sas_port *sas_port)
 }
 EXPORT_SYMBOL_GPL(to_hisi_sas_port);
 
+void hisi_sas_stop_phys(struct hisi_hba *hisi_hba)
+{
+	int phy_no;
+
+	for (phy_no = 0; phy_no < hisi_hba->n_phy; phy_no++)
+		hisi_hba->hw->phy_disable(hisi_hba, phy_no);
+}
+EXPORT_SYMBOL_GPL(hisi_sas_stop_phys);
+
 static void hisi_sas_slot_index_clear(struct hisi_hba *hisi_hba, int slot_idx)
 {
 	void *bitmap = hisi_hba->slot_index_tags;
@@ -433,7 +444,7 @@ static int hisi_sas_task_exec(struct sas_task *task, gfp_t gfp_flags,
 	struct hisi_sas_device *sas_dev = device->lldd_dev;
 	struct hisi_sas_dq *dq = sas_dev->dq;
 
-	if (unlikely(test_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags)))
+	if (unlikely(test_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags)))
 		return -EINVAL;
 
 	/* protect task_prep and start_delivery sequence */
@@ -716,7 +727,6 @@ static void hisi_sas_dev_gone(struct domain_device *device)
 	struct hisi_sas_device *sas_dev = device->lldd_dev;
 	struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
 	struct device *dev = hisi_hba->dev;
-	int dev_id = sas_dev->device_id;
 
 	dev_info(dev, "found dev[%d:%x] is gone\n",
 		 sas_dev->device_id, sas_dev->dev_type);
@@ -729,9 +739,7 @@ static void hisi_sas_dev_gone(struct domain_device *device)
 	hisi_hba->hw->free_device(hisi_hba, sas_dev);
 	device->lldd_dev = NULL;
 	memset(sas_dev, 0, sizeof(*sas_dev));
-	sas_dev->device_id = dev_id;
 	sas_dev->dev_type = SAS_PHY_UNUSED;
-	sas_dev->dev_status = HISI_SAS_DEV_NORMAL;
 }
 
 static int hisi_sas_queue_command(struct sas_task *task, gfp_t gfp_flags)
@@ -764,7 +772,12 @@ static int hisi_sas_control_phy(struct asd_sas_phy *sas_phy, enum phy_func func,
 	case PHY_FUNC_SET_LINK_RATE:
 		hisi_hba->hw->phy_set_linkrate(hisi_hba, phy_no, funcdata);
 		break;
-
+	case PHY_FUNC_GET_EVENTS:
+		if (hisi_hba->hw->get_events) {
+			hisi_hba->hw->get_events(hisi_hba, phy_no);
+			break;
+		}
+		/* fallthru */
 	case PHY_FUNC_RELEASE_SPINUP_HOLD:
 	default:
 		return -EOPNOTSUPP;
@@ -967,37 +980,117 @@ static int hisi_sas_debug_issue_ssp_tmf(struct domain_device *device,
 				sizeof(ssp_task), tmf);
 }
 
+static void hisi_sas_refresh_port_id(struct hisi_hba *hisi_hba,
+		struct asd_sas_port *sas_port, enum sas_linkrate linkrate)
+{
+	struct hisi_sas_device	*sas_dev;
+	struct domain_device *device;
+	int i;
+
+	for (i = 0; i < HISI_SAS_MAX_DEVICES; i++) {
+		sas_dev = &hisi_hba->devices[i];
+		device = sas_dev->sas_device;
+		if ((sas_dev->dev_type == SAS_PHY_UNUSED)
+				|| !device || (device->port != sas_port))
+			continue;
+
+		hisi_hba->hw->free_device(hisi_hba, sas_dev);
+
+		/* Update linkrate of directly attached device. */
+		if (!device->parent)
+			device->linkrate = linkrate;
+
+		hisi_hba->hw->setup_itct(hisi_hba, sas_dev);
+	}
+}
+
+static void hisi_sas_rescan_topology(struct hisi_hba *hisi_hba, u32 old_state,
+			      u32 state)
+{
+	struct sas_ha_struct *sas_ha = &hisi_hba->sha;
+	struct asd_sas_port *_sas_port = NULL;
+	int phy_no;
+
+	for (phy_no = 0; phy_no < hisi_hba->n_phy; phy_no++) {
+		struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
+		struct asd_sas_phy *sas_phy = &phy->sas_phy;
+		struct asd_sas_port *sas_port = sas_phy->port;
+		struct hisi_sas_port *port = to_hisi_sas_port(sas_port);
+		bool do_port_check = !!(_sas_port != sas_port);
+
+		if (!sas_phy->phy->enabled)
+			continue;
+
+		/* Report PHY state change to libsas */
+		if (state & (1 << phy_no)) {
+			if (do_port_check && sas_port) {
+				struct domain_device *dev = sas_port->port_dev;
+
+				_sas_port = sas_port;
+				port->id = phy->port_id;
+				hisi_sas_refresh_port_id(hisi_hba,
+						sas_port, sas_phy->linkrate);
+
+				if (DEV_IS_EXPANDER(dev->dev_type))
+					sas_ha->notify_port_event(sas_phy,
+							PORTE_BROADCAST_RCVD);
+			}
+		} else if (old_state & (1 << phy_no))
+			/* PHY down but was up before */
+			hisi_sas_phy_down(hisi_hba, phy_no, 0);
+
+	}
+
+	drain_workqueue(hisi_hba->shost->work_q);
+}
+
 static int hisi_sas_controller_reset(struct hisi_hba *hisi_hba)
 {
+	struct sas_ha_struct *sas_ha = &hisi_hba->sha;
+	struct device *dev = hisi_hba->dev;
+	struct Scsi_Host *shost = hisi_hba->shost;
+	u32 old_state, state;
+	unsigned long flags;
 	int rc;
 
 	if (!hisi_hba->hw->soft_reset)
 		return -1;
 
-	if (!test_and_set_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags)) {
-		struct device *dev = hisi_hba->dev;
-		struct sas_ha_struct *sas_ha = &hisi_hba->sha;
-		unsigned long flags;
+	if (test_and_set_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags))
+		return -1;
 
-		dev_dbg(dev, "controller reset begins!\n");
-		scsi_block_requests(hisi_hba->shost);
-		rc = hisi_hba->hw->soft_reset(hisi_hba);
-		if (rc) {
-			dev_warn(dev, "controller reset failed (%d)\n", rc);
-			goto out;
-		}
-		spin_lock_irqsave(&hisi_hba->lock, flags);
-		hisi_sas_release_tasks(hisi_hba);
-		spin_unlock_irqrestore(&hisi_hba->lock, flags);
+	dev_dbg(dev, "controller resetting...\n");
+	old_state = hisi_hba->hw->get_phys_state(hisi_hba);
 
-		sas_ha->notify_ha_event(sas_ha, HAE_RESET);
-		dev_dbg(dev, "controller reset successful!\n");
-	} else
-		return -1;
+	scsi_block_requests(shost);
+	set_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
+	rc = hisi_hba->hw->soft_reset(hisi_hba);
+	if (rc) {
+		dev_warn(dev, "controller reset failed (%d)\n", rc);
+		clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
+		goto out;
+	}
+	spin_lock_irqsave(&hisi_hba->lock, flags);
+	hisi_sas_release_tasks(hisi_hba);
+	spin_unlock_irqrestore(&hisi_hba->lock, flags);
+
+	sas_ha->notify_ha_event(sas_ha, HAE_RESET);
+	clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
+
+	/* Init and wait for PHYs to come up and all libsas event finished. */
+	hisi_hba->hw->phys_init(hisi_hba);
+	msleep(1000);
+	drain_workqueue(hisi_hba->wq);
+	drain_workqueue(shost->work_q);
+
+	state = hisi_hba->hw->get_phys_state(hisi_hba);
+	hisi_sas_rescan_topology(hisi_hba, old_state, state);
+	dev_dbg(dev, "controller reset complete\n");
 
 out:
-	scsi_unblock_requests(hisi_hba->shost);
+	scsi_unblock_requests(shost);
 	clear_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags);
+
 	return rc;
 }
 
@@ -1241,7 +1334,7 @@ hisi_sas_internal_abort_task_exec(struct hisi_hba *hisi_hba, int device_id,
 	int dlvry_queue_slot, dlvry_queue, n_elem = 0, rc, slot_idx;
 	unsigned long flags, flags_dq;
 
-	if (unlikely(test_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags)))
+	if (unlikely(test_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags)))
 		return -EINVAL;
 
 	if (!device->port)
@@ -1279,12 +1372,21 @@ hisi_sas_internal_abort_task_exec(struct hisi_hba *hisi_hba, int device_id,
 	slot->port = port;
 	task->lldd_task = slot;
 
+	slot->buf = dma_pool_alloc(hisi_hba->buffer_pool,
+			GFP_ATOMIC, &slot->buf_dma);
+	if (!slot->buf) {
+		rc = -ENOMEM;
+		goto err_out_tag;
+	}
+
 	memset(slot->cmd_hdr, 0, sizeof(struct hisi_sas_cmd_hdr));
+	memset(hisi_sas_cmd_hdr_addr_mem(slot), 0, HISI_SAS_COMMAND_TABLE_SZ);
+	memset(hisi_sas_status_buf_addr_mem(slot), 0, HISI_SAS_STATUS_BUF_SZ);
 
 	rc = hisi_sas_task_prep_abort(hisi_hba, slot, device_id,
 				      abort_flag, task_tag);
 	if (rc)
-		goto err_out_tag;
+		goto err_out_buf;
 
 
 	list_add_tail(&slot->entry, &sas_dev->list);
@@ -1302,6 +1404,9 @@ hisi_sas_internal_abort_task_exec(struct hisi_hba *hisi_hba, int device_id,
 
 	return 0;
 
+err_out_buf:
+	dma_pool_free(hisi_hba->buffer_pool, slot->buf,
+		slot->buf_dma);
 err_out_tag:
 	spin_lock_irqsave(&hisi_hba->lock, flags);
 	hisi_sas_slot_index_free(hisi_hba, slot_idx);
@@ -1437,36 +1542,6 @@ void hisi_sas_phy_down(struct hisi_hba *hisi_hba, int phy_no, int rdy)
 }
 EXPORT_SYMBOL_GPL(hisi_sas_phy_down);
 
-void hisi_sas_rescan_topology(struct hisi_hba *hisi_hba, u32 old_state,
-			      u32 state)
-{
-	struct sas_ha_struct *sas_ha = &hisi_hba->sha;
-	int phy_no;
-
-	for (phy_no = 0; phy_no < hisi_hba->n_phy; phy_no++) {
-		struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
-		struct asd_sas_phy *sas_phy = &phy->sas_phy;
-		struct asd_sas_port *sas_port = sas_phy->port;
-		struct domain_device *dev;
-
-		if (sas_phy->enabled) {
-			/* Report PHY state change to libsas */
-			if (state & (1 << phy_no))
-				continue;
-
-			if (old_state & (1 << phy_no))
-				/* PHY down but was up before */
-				hisi_sas_phy_down(hisi_hba, phy_no, 0);
-		}
-		if (!sas_port)
-			continue;
-		dev = sas_port->port_dev;
-
-		if (DEV_IS_EXPANDER(dev->dev_type))
-			sas_ha->notify_phy_event(sas_phy, PORTE_BROADCAST_RCVD);
-	}
-}
-EXPORT_SYMBOL_GPL(hisi_sas_rescan_topology);
 
 struct scsi_transport_template *hisi_sas_stt;
 EXPORT_SYMBOL_GPL(hisi_sas_stt);
@@ -1487,7 +1562,7 @@ static struct scsi_host_template _hisi_sas_sht = {
 	.max_sectors		= SCSI_DEFAULT_MAX_SECTORS,
 	.use_clustering		= ENABLE_CLUSTERING,
 	.eh_device_reset_handler = sas_eh_device_reset_handler,
-	.eh_bus_reset_handler	= sas_eh_bus_reset_handler,
+	.eh_target_reset_handler = sas_eh_target_reset_handler,
 	.target_destroy		= sas_target_destroy,
 	.ioctl			= sas_ioctl,
 };
@@ -1825,7 +1900,7 @@ static struct Scsi_Host *hisi_sas_shost_alloc(struct platform_device *pdev,
 
 	return shost;
 err_out:
-	kfree(shost);
+	scsi_host_put(shost);
 	dev_err(dev, "shost alloc failed\n");
 	return NULL;
 }
@@ -1916,7 +1991,7 @@ err_out_register_ha:
 	scsi_remove_host(shost);
 err_out_ha:
 	hisi_sas_free(hisi_hba);
-	kfree(shost);
+	scsi_host_put(shost);
 	return rc;
 }
 EXPORT_SYMBOL_GPL(hisi_sas_probe);
@@ -1931,15 +2006,13 @@ int hisi_sas_remove(struct platform_device *pdev)
 	sas_remove_host(sha->core.shost);
 
 	hisi_sas_free(hisi_hba);
-	kfree(shost);
+	scsi_host_put(shost);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(hisi_sas_remove);
 
 static __init int hisi_sas_init(void)
 {
-	pr_info("hisi_sas: driver version %s\n", DRV_VERSION);
-
 	hisi_sas_stt = sas_domain_attach_transport(&hisi_sas_transport_ops);
 	if (!hisi_sas_stt)
 		return -ENOMEM;
@@ -1955,7 +2028,6 @@ static __exit void hisi_sas_exit(void)
 module_init(hisi_sas_init);
 module_exit(hisi_sas_exit);
 
-MODULE_VERSION(DRV_VERSION);
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("John Garry <john.garry@huawei.com>");
 MODULE_DESCRIPTION("HISILICON SAS controller driver");
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
index 2bfea7082e3a..779af979b6db 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
@@ -256,6 +256,8 @@
 #define LINK_DFX2_RCVR_HOLD_STS_MSK	(0x1 << LINK_DFX2_RCVR_HOLD_STS_OFF)
 #define LINK_DFX2_SEND_HOLD_STS_OFF	10
 #define LINK_DFX2_SEND_HOLD_STS_MSK	(0x1 << LINK_DFX2_SEND_HOLD_STS_OFF)
+#define SAS_ERR_CNT4_REG		(PORT_BASE + 0x290)
+#define SAS_ERR_CNT6_REG		(PORT_BASE + 0x298)
 #define PHY_CTRL_RDY_MSK		(PORT_BASE + 0x2b0)
 #define PHYCTRL_NOT_RDY_MSK		(PORT_BASE + 0x2b4)
 #define PHYCTRL_DWS_RESET_MSK		(PORT_BASE + 0x2b8)
@@ -399,6 +401,172 @@ struct hisi_sas_err_record_v2 {
 	__le32 dma_rx_err_type;
 };
 
+static const struct hisi_sas_hw_error one_bit_ecc_errors[] = {
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_DQE_ECC_1B_OFF),
+		.msk = HGC_DQE_ECC_1B_ADDR_MSK,
+		.shift = HGC_DQE_ECC_1B_ADDR_OFF,
+		.msg = "hgc_dqe_acc1b_intr found: \
+				Ram address is 0x%08X\n",
+		.reg = HGC_DQE_ECC_ADDR,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_IOST_ECC_1B_OFF),
+		.msk = HGC_IOST_ECC_1B_ADDR_MSK,
+		.shift = HGC_IOST_ECC_1B_ADDR_OFF,
+		.msg = "hgc_iost_acc1b_intr found: \
+				Ram address is 0x%08X\n",
+		.reg = HGC_IOST_ECC_ADDR,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_ITCT_ECC_1B_OFF),
+		.msk = HGC_ITCT_ECC_1B_ADDR_MSK,
+		.shift = HGC_ITCT_ECC_1B_ADDR_OFF,
+		.msg = "hgc_itct_acc1b_intr found: \
+				Ram address is 0x%08X\n",
+		.reg = HGC_ITCT_ECC_ADDR,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_IOSTLIST_ECC_1B_OFF),
+		.msk = HGC_LM_DFX_STATUS2_IOSTLIST_MSK,
+		.shift = HGC_LM_DFX_STATUS2_IOSTLIST_OFF,
+		.msg = "hgc_iostl_acc1b_intr found:  \
+				memory address is 0x%08X\n",
+		.reg = HGC_LM_DFX_STATUS2,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_ITCTLIST_ECC_1B_OFF),
+		.msk = HGC_LM_DFX_STATUS2_ITCTLIST_MSK,
+		.shift = HGC_LM_DFX_STATUS2_ITCTLIST_OFF,
+		.msg = "hgc_itctl_acc1b_intr found: \
+				memory address is 0x%08X\n",
+		.reg = HGC_LM_DFX_STATUS2,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_CQE_ECC_1B_OFF),
+		.msk = HGC_CQE_ECC_1B_ADDR_MSK,
+		.shift = HGC_CQE_ECC_1B_ADDR_OFF,
+		.msg = "hgc_cqe_acc1b_intr found: \
+				Ram address is 0x%08X\n",
+		.reg = HGC_CQE_ECC_ADDR,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_NCQ_MEM0_ECC_1B_OFF),
+		.msk = HGC_RXM_DFX_STATUS14_MEM0_MSK,
+		.shift = HGC_RXM_DFX_STATUS14_MEM0_OFF,
+		.msg = "rxm_mem0_acc1b_intr found: \
+				memory address is 0x%08X\n",
+		.reg = HGC_RXM_DFX_STATUS14,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_NCQ_MEM1_ECC_1B_OFF),
+		.msk = HGC_RXM_DFX_STATUS14_MEM1_MSK,
+		.shift = HGC_RXM_DFX_STATUS14_MEM1_OFF,
+		.msg = "rxm_mem1_acc1b_intr found: \
+				memory address is 0x%08X\n",
+		.reg = HGC_RXM_DFX_STATUS14,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_NCQ_MEM2_ECC_1B_OFF),
+		.msk = HGC_RXM_DFX_STATUS14_MEM2_MSK,
+		.shift = HGC_RXM_DFX_STATUS14_MEM2_OFF,
+		.msg = "rxm_mem2_acc1b_intr found: \
+				memory address is 0x%08X\n",
+		.reg = HGC_RXM_DFX_STATUS14,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_NCQ_MEM3_ECC_1B_OFF),
+		.msk = HGC_RXM_DFX_STATUS15_MEM3_MSK,
+		.shift = HGC_RXM_DFX_STATUS15_MEM3_OFF,
+		.msg = "rxm_mem3_acc1b_intr found: \
+				memory address is 0x%08X\n",
+		.reg = HGC_RXM_DFX_STATUS15,
+	},
+};
+
+static const struct hisi_sas_hw_error multi_bit_ecc_errors[] = {
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_DQE_ECC_MB_OFF),
+		.msk = HGC_DQE_ECC_MB_ADDR_MSK,
+		.shift = HGC_DQE_ECC_MB_ADDR_OFF,
+		.msg = "hgc_dqe_accbad_intr (0x%x) found: \
+				Ram address is 0x%08X\n",
+		.reg = HGC_DQE_ECC_ADDR,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_IOST_ECC_MB_OFF),
+		.msk = HGC_IOST_ECC_MB_ADDR_MSK,
+		.shift = HGC_IOST_ECC_MB_ADDR_OFF,
+		.msg = "hgc_iost_accbad_intr (0x%x) found: \
+				Ram address is 0x%08X\n",
+		.reg = HGC_IOST_ECC_ADDR,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_ITCT_ECC_MB_OFF),
+		.msk = HGC_ITCT_ECC_MB_ADDR_MSK,
+		.shift = HGC_ITCT_ECC_MB_ADDR_OFF,
+		.msg = "hgc_itct_accbad_intr (0x%x) found: \
+				Ram address is 0x%08X\n",
+		.reg = HGC_ITCT_ECC_ADDR,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_IOSTLIST_ECC_MB_OFF),
+		.msk = HGC_LM_DFX_STATUS2_IOSTLIST_MSK,
+		.shift = HGC_LM_DFX_STATUS2_IOSTLIST_OFF,
+		.msg = "hgc_iostl_accbad_intr (0x%x) found: \
+				memory address is 0x%08X\n",
+		.reg = HGC_LM_DFX_STATUS2,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_ITCTLIST_ECC_MB_OFF),
+		.msk = HGC_LM_DFX_STATUS2_ITCTLIST_MSK,
+		.shift = HGC_LM_DFX_STATUS2_ITCTLIST_OFF,
+		.msg = "hgc_itctl_accbad_intr (0x%x) found: \
+				memory address is 0x%08X\n",
+		.reg = HGC_LM_DFX_STATUS2,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_CQE_ECC_MB_OFF),
+		.msk = HGC_CQE_ECC_MB_ADDR_MSK,
+		.shift = HGC_CQE_ECC_MB_ADDR_OFF,
+		.msg = "hgc_cqe_accbad_intr (0x%x) found: \
+				Ram address is 0x%08X\n",
+		.reg = HGC_CQE_ECC_ADDR,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_NCQ_MEM0_ECC_MB_OFF),
+		.msk = HGC_RXM_DFX_STATUS14_MEM0_MSK,
+		.shift = HGC_RXM_DFX_STATUS14_MEM0_OFF,
+		.msg = "rxm_mem0_accbad_intr (0x%x) found: \
+			memory address is 0x%08X\n",
+		.reg = HGC_RXM_DFX_STATUS14,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_NCQ_MEM1_ECC_MB_OFF),
+		.msk = HGC_RXM_DFX_STATUS14_MEM1_MSK,
+		.shift = HGC_RXM_DFX_STATUS14_MEM1_OFF,
+		.msg = "rxm_mem1_accbad_intr (0x%x) found: \
+			memory address is 0x%08X\n",
+		.reg = HGC_RXM_DFX_STATUS14,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_NCQ_MEM2_ECC_MB_OFF),
+		.msk = HGC_RXM_DFX_STATUS14_MEM2_MSK,
+		.shift = HGC_RXM_DFX_STATUS14_MEM2_OFF,
+		.msg = "rxm_mem2_accbad_intr (0x%x) found: \
+				memory address is 0x%08X\n",
+		.reg = HGC_RXM_DFX_STATUS14,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_NCQ_MEM3_ECC_MB_OFF),
+		.msk = HGC_RXM_DFX_STATUS15_MEM3_MSK,
+		.shift = HGC_RXM_DFX_STATUS15_MEM3_OFF,
+		.msg = "rxm_mem3_accbad_intr (0x%x) found: \
+				memory address is 0x%08X\n",
+		.reg = HGC_RXM_DFX_STATUS15,
+	},
+};
+
 enum {
 	HISI_SAS_PHY_PHY_UPDOWN,
 	HISI_SAS_PHY_CHNL_INT,
@@ -806,12 +974,14 @@ static void setup_itct_v2_hw(struct hisi_hba *hisi_hba,
 static void free_device_v2_hw(struct hisi_hba *hisi_hba,
 			      struct hisi_sas_device *sas_dev)
 {
+	DECLARE_COMPLETION_ONSTACK(completion);
 	u64 dev_id = sas_dev->device_id;
-	struct device *dev = hisi_hba->dev;
 	struct hisi_sas_itct *itct = &hisi_hba->itct[dev_id];
 	u32 reg_val = hisi_sas_read32(hisi_hba, ENT_INT_SRC3);
 	int i;
 
+	sas_dev->completion = &completion;
+
 	/* SoC bug workaround */
 	if (dev_is_sata(sas_dev->sas_device))
 		clear_bit(sas_dev->sata_idx, hisi_hba->sata_dev_bitmap);
@@ -821,28 +991,12 @@ static void free_device_v2_hw(struct hisi_hba *hisi_hba,
 		hisi_sas_write32(hisi_hba, ENT_INT_SRC3,
 				 ENT_INT_SRC3_ITC_INT_MSK);
 
-	/* clear the itct int*/
 	for (i = 0; i < 2; i++) {
-		/* clear the itct table*/
-		reg_val = hisi_sas_read32(hisi_hba, ITCT_CLR);
-		reg_val |= ITCT_CLR_EN_MSK | (dev_id & ITCT_DEV_MSK);
+		reg_val = ITCT_CLR_EN_MSK | (dev_id & ITCT_DEV_MSK);
 		hisi_sas_write32(hisi_hba, ITCT_CLR, reg_val);
+		wait_for_completion(sas_dev->completion);
 
-		udelay(10);
-		reg_val = hisi_sas_read32(hisi_hba, ENT_INT_SRC3);
-		if (ENT_INT_SRC3_ITC_INT_MSK & reg_val) {
-			dev_dbg(dev, "got clear ITCT done interrupt\n");
-
-			/* invalid the itct state*/
-			memset(itct, 0, sizeof(struct hisi_sas_itct));
-			hisi_sas_write32(hisi_hba, ENT_INT_SRC3,
-					 ENT_INT_SRC3_ITC_INT_MSK);
-
-			/* clear the itct */
-			hisi_sas_write32(hisi_hba, ITCT_CLR, 0);
-			dev_dbg(dev, "clear ITCT ok\n");
-			break;
-		}
+		memset(itct, 0, sizeof(struct hisi_sas_itct));
 	}
 }
 
@@ -1023,7 +1177,7 @@ static void init_reg_v2_hw(struct hisi_hba *hisi_hba)
 	hisi_sas_write32(hisi_hba, ENT_INT_SRC3, 0xffffffff);
 	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0x7efefefe);
 	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0x7efefefe);
-	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0x7ffffffe);
+	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0x7ffe20fe);
 	hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0xfff00c30);
 	for (i = 0; i < hisi_hba->queue_count; i++)
 		hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK+0x4*i, 0);
@@ -1332,25 +1486,12 @@ static void start_phy_v2_hw(struct hisi_hba *hisi_hba, int phy_no)
 	enable_phy_v2_hw(hisi_hba, phy_no);
 }
 
-static void stop_phy_v2_hw(struct hisi_hba *hisi_hba, int phy_no)
-{
-	disable_phy_v2_hw(hisi_hba, phy_no);
-}
-
-static void stop_phys_v2_hw(struct hisi_hba *hisi_hba)
-{
-	int i;
-
-	for (i = 0; i < hisi_hba->n_phy; i++)
-		stop_phy_v2_hw(hisi_hba, i);
-}
-
 static void phy_hard_reset_v2_hw(struct hisi_hba *hisi_hba, int phy_no)
 {
 	struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
 	u32 txid_auto;
 
-	stop_phy_v2_hw(hisi_hba, phy_no);
+	disable_phy_v2_hw(hisi_hba, phy_no);
 	if (phy->identify.device_type == SAS_END_DEVICE) {
 		txid_auto = hisi_sas_phy_read32(hisi_hba, phy_no, TXID_AUTO);
 		hisi_sas_phy_write32(hisi_hba, phy_no, TXID_AUTO,
@@ -1360,17 +1501,38 @@ static void phy_hard_reset_v2_hw(struct hisi_hba *hisi_hba, int phy_no)
 	start_phy_v2_hw(hisi_hba, phy_no);
 }
 
-static void start_phys_v2_hw(struct hisi_hba *hisi_hba)
+static void phy_get_events_v2_hw(struct hisi_hba *hisi_hba, int phy_no)
 {
-	int i;
+	struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
+	struct asd_sas_phy *sas_phy = &phy->sas_phy;
+	struct sas_phy *sphy = sas_phy->phy;
+	u32 err4_reg_val, err6_reg_val;
 
-	for (i = 0; i < hisi_hba->n_phy; i++)
-		start_phy_v2_hw(hisi_hba, i);
+	/* loss dword syn, phy reset problem */
+	err4_reg_val = hisi_sas_phy_read32(hisi_hba, phy_no, SAS_ERR_CNT4_REG);
+
+	/* disparity err, invalid dword */
+	err6_reg_val = hisi_sas_phy_read32(hisi_hba, phy_no, SAS_ERR_CNT6_REG);
+
+	sphy->loss_of_dword_sync_count += (err4_reg_val >> 16) & 0xFFFF;
+	sphy->phy_reset_problem_count += err4_reg_val & 0xFFFF;
+	sphy->invalid_dword_count += (err6_reg_val & 0xFF0000) >> 16;
+	sphy->running_disparity_error_count += err6_reg_val & 0xFF;
 }
 
 static void phys_init_v2_hw(struct hisi_hba *hisi_hba)
 {
-	start_phys_v2_hw(hisi_hba);
+	int i;
+
+	for (i = 0; i < hisi_hba->n_phy; i++) {
+		struct hisi_sas_phy *phy = &hisi_hba->phy[i];
+		struct asd_sas_phy *sas_phy = &phy->sas_phy;
+
+		if (!sas_phy->phy->enabled)
+			continue;
+
+		start_phy_v2_hw(hisi_hba, i);
+	}
 }
 
 static void sl_notify_v2_hw(struct hisi_hba *hisi_hba, int phy_no)
@@ -1965,7 +2127,7 @@ static void slot_err_v2_hw(struct hisi_hba *hisi_hba,
 		}
 		case DMA_RX_DATA_LEN_UNDERFLOW:
 		{
-			ts->residual = dma_rx_err_type;
+			ts->residual = trans_tx_fail_type;
 			ts->stat = SAS_DATA_UNDERRUN;
 			break;
 		}
@@ -2091,7 +2253,7 @@ static void slot_err_v2_hw(struct hisi_hba *hisi_hba,
 		}
 		case DMA_RX_DATA_LEN_UNDERFLOW:
 		{
-			ts->residual = dma_rx_err_type;
+			ts->residual = trans_tx_fail_type;
 			ts->stat = SAS_DATA_UNDERRUN;
 			break;
 		}
@@ -2599,6 +2761,7 @@ static irqreturn_t int_phy_updown_v2_hw(int irq_no, void *p)
 	struct hisi_hba *hisi_hba = p;
 	u32 irq_msk;
 	int phy_no = 0;
+	irqreturn_t res = IRQ_NONE;
 
 	irq_msk = (hisi_sas_read32(hisi_hba, HGC_INVLD_DQE_INFO)
 		   >> HGC_INVLD_DQE_INFO_FB_CH0_OFF) & 0x1ff;
@@ -2613,15 +2776,15 @@ static irqreturn_t int_phy_updown_v2_hw(int irq_no, void *p)
 			case CHL_INT0_SL_PHY_ENABLE_MSK:
 				/* phy up */
 				if (phy_up_v2_hw(phy_no, hisi_hba) ==
-				    IRQ_NONE)
-					return IRQ_NONE;
+				    IRQ_HANDLED)
+					res = IRQ_HANDLED;
 				break;
 
 			case CHL_INT0_NOT_RDY_MSK:
 				/* phy down */
 				if (phy_down_v2_hw(phy_no, hisi_hba) ==
-				    IRQ_NONE)
-					return IRQ_NONE;
+				    IRQ_HANDLED)
+					res = IRQ_HANDLED;
 				break;
 
 			case (CHL_INT0_NOT_RDY_MSK |
@@ -2631,13 +2794,13 @@ static irqreturn_t int_phy_updown_v2_hw(int irq_no, void *p)
 				if (reg_value & BIT(phy_no)) {
 					/* phy up */
 					if (phy_up_v2_hw(phy_no, hisi_hba) ==
-					    IRQ_NONE)
-						return IRQ_NONE;
+					    IRQ_HANDLED)
+						res = IRQ_HANDLED;
 				} else {
 					/* phy down */
 					if (phy_down_v2_hw(phy_no, hisi_hba) ==
-					    IRQ_NONE)
-						return IRQ_NONE;
+					    IRQ_HANDLED)
+						res = IRQ_HANDLED;
 				}
 				break;
 
@@ -2650,7 +2813,7 @@ static irqreturn_t int_phy_updown_v2_hw(int irq_no, void *p)
 		phy_no++;
 	}
 
-	return IRQ_HANDLED;
+	return res;
 }
 
 static void phy_bcast_v2_hw(int phy_no, struct hisi_hba *hisi_hba)
@@ -2733,194 +2896,38 @@ static void
 one_bit_ecc_error_process_v2_hw(struct hisi_hba *hisi_hba, u32 irq_value)
 {
 	struct device *dev = hisi_hba->dev;
-	u32 reg_val;
-
-	if (irq_value & BIT(SAS_ECC_INTR_DQE_ECC_1B_OFF)) {
-		reg_val = hisi_sas_read32(hisi_hba, HGC_DQE_ECC_ADDR);
-		dev_warn(dev, "hgc_dqe_acc1b_intr found: \
-			       Ram address is 0x%08X\n",
-			 (reg_val & HGC_DQE_ECC_1B_ADDR_MSK) >>
-			 HGC_DQE_ECC_1B_ADDR_OFF);
-	}
-
-	if (irq_value & BIT(SAS_ECC_INTR_IOST_ECC_1B_OFF)) {
-		reg_val = hisi_sas_read32(hisi_hba, HGC_IOST_ECC_ADDR);
-		dev_warn(dev, "hgc_iost_acc1b_intr found: \
-			       Ram address is 0x%08X\n",
-			 (reg_val & HGC_IOST_ECC_1B_ADDR_MSK) >>
-			 HGC_IOST_ECC_1B_ADDR_OFF);
-	}
-
-	if (irq_value & BIT(SAS_ECC_INTR_ITCT_ECC_1B_OFF)) {
-		reg_val = hisi_sas_read32(hisi_hba, HGC_ITCT_ECC_ADDR);
-		dev_warn(dev, "hgc_itct_acc1b_intr found: \
-			       Ram address is 0x%08X\n",
-			 (reg_val & HGC_ITCT_ECC_1B_ADDR_MSK) >>
-			 HGC_ITCT_ECC_1B_ADDR_OFF);
-	}
-
-	if (irq_value & BIT(SAS_ECC_INTR_IOSTLIST_ECC_1B_OFF)) {
-		reg_val = hisi_sas_read32(hisi_hba, HGC_LM_DFX_STATUS2);
-		dev_warn(dev, "hgc_iostl_acc1b_intr found: \
-			       memory address is 0x%08X\n",
-			 (reg_val & HGC_LM_DFX_STATUS2_IOSTLIST_MSK) >>
-			 HGC_LM_DFX_STATUS2_IOSTLIST_OFF);
-	}
-
-	if (irq_value & BIT(SAS_ECC_INTR_ITCTLIST_ECC_1B_OFF)) {
-		reg_val = hisi_sas_read32(hisi_hba, HGC_LM_DFX_STATUS2);
-		dev_warn(dev, "hgc_itctl_acc1b_intr found: \
-			       memory address is 0x%08X\n",
-			 (reg_val & HGC_LM_DFX_STATUS2_ITCTLIST_MSK) >>
-			 HGC_LM_DFX_STATUS2_ITCTLIST_OFF);
-	}
-
-	if (irq_value & BIT(SAS_ECC_INTR_CQE_ECC_1B_OFF)) {
-		reg_val = hisi_sas_read32(hisi_hba, HGC_CQE_ECC_ADDR);
-		dev_warn(dev, "hgc_cqe_acc1b_intr found: \
-			       Ram address is 0x%08X\n",
-			 (reg_val & HGC_CQE_ECC_1B_ADDR_MSK) >>
-			 HGC_CQE_ECC_1B_ADDR_OFF);
-	}
-
-	if (irq_value & BIT(SAS_ECC_INTR_NCQ_MEM0_ECC_1B_OFF)) {
-		reg_val = hisi_sas_read32(hisi_hba, HGC_RXM_DFX_STATUS14);
-		dev_warn(dev, "rxm_mem0_acc1b_intr found: \
-			       memory address is 0x%08X\n",
-			 (reg_val & HGC_RXM_DFX_STATUS14_MEM0_MSK) >>
-			 HGC_RXM_DFX_STATUS14_MEM0_OFF);
-	}
-
-	if (irq_value & BIT(SAS_ECC_INTR_NCQ_MEM1_ECC_1B_OFF)) {
-		reg_val = hisi_sas_read32(hisi_hba, HGC_RXM_DFX_STATUS14);
-		dev_warn(dev, "rxm_mem1_acc1b_intr found: \
-			       memory address is 0x%08X\n",
-			 (reg_val & HGC_RXM_DFX_STATUS14_MEM1_MSK) >>
-			 HGC_RXM_DFX_STATUS14_MEM1_OFF);
-	}
-
-	if (irq_value & BIT(SAS_ECC_INTR_NCQ_MEM2_ECC_1B_OFF)) {
-		reg_val = hisi_sas_read32(hisi_hba, HGC_RXM_DFX_STATUS14);
-		dev_warn(dev, "rxm_mem2_acc1b_intr found: \
-			       memory address is 0x%08X\n",
-			 (reg_val & HGC_RXM_DFX_STATUS14_MEM2_MSK) >>
-			 HGC_RXM_DFX_STATUS14_MEM2_OFF);
-	}
+	const struct hisi_sas_hw_error *ecc_error;
+	u32 val;
+	int i;
 
-	if (irq_value & BIT(SAS_ECC_INTR_NCQ_MEM3_ECC_1B_OFF)) {
-		reg_val = hisi_sas_read32(hisi_hba, HGC_RXM_DFX_STATUS15);
-		dev_warn(dev, "rxm_mem3_acc1b_intr found: \
-			       memory address is 0x%08X\n",
-			 (reg_val & HGC_RXM_DFX_STATUS15_MEM3_MSK) >>
-			 HGC_RXM_DFX_STATUS15_MEM3_OFF);
+	for (i = 0; i < ARRAY_SIZE(one_bit_ecc_errors); i++) {
+		ecc_error = &one_bit_ecc_errors[i];
+		if (irq_value & ecc_error->irq_msk) {
+			val = hisi_sas_read32(hisi_hba, ecc_error->reg);
+			val &= ecc_error->msk;
+			val >>= ecc_error->shift;
+			dev_warn(dev, ecc_error->msg, val);
+		}
 	}
-
 }
 
 static void multi_bit_ecc_error_process_v2_hw(struct hisi_hba *hisi_hba,
 		u32 irq_value)
 {
-	u32 reg_val;
 	struct device *dev = hisi_hba->dev;
+	const struct hisi_sas_hw_error *ecc_error;
+	u32 val;
+	int i;
 
-	if (irq_value & BIT(SAS_ECC_INTR_DQE_ECC_MB_OFF)) {
-		reg_val = hisi_sas_read32(hisi_hba, HGC_DQE_ECC_ADDR);
-		dev_warn(dev, "hgc_dqe_accbad_intr (0x%x) found: \
-		       Ram address is 0x%08X\n",
-		      irq_value,
-		      (reg_val & HGC_DQE_ECC_MB_ADDR_MSK) >>
-		      HGC_DQE_ECC_MB_ADDR_OFF);
-		queue_work(hisi_hba->wq, &hisi_hba->rst_work);
-	}
-
-	if (irq_value & BIT(SAS_ECC_INTR_IOST_ECC_MB_OFF)) {
-		reg_val = hisi_sas_read32(hisi_hba, HGC_IOST_ECC_ADDR);
-		dev_warn(dev, "hgc_iost_accbad_intr (0x%x) found: \
-		       Ram address is 0x%08X\n",
-		      irq_value,
-		      (reg_val & HGC_IOST_ECC_MB_ADDR_MSK) >>
-		      HGC_IOST_ECC_MB_ADDR_OFF);
-		queue_work(hisi_hba->wq, &hisi_hba->rst_work);
-	}
-
-	if (irq_value & BIT(SAS_ECC_INTR_ITCT_ECC_MB_OFF)) {
-		reg_val = hisi_sas_read32(hisi_hba, HGC_ITCT_ECC_ADDR);
-		dev_warn(dev,"hgc_itct_accbad_intr (0x%x) found: \
-		       Ram address is 0x%08X\n",
-		      irq_value,
-		      (reg_val & HGC_ITCT_ECC_MB_ADDR_MSK) >>
-		      HGC_ITCT_ECC_MB_ADDR_OFF);
-		queue_work(hisi_hba->wq, &hisi_hba->rst_work);
-	}
-
-	if (irq_value & BIT(SAS_ECC_INTR_IOSTLIST_ECC_MB_OFF)) {
-		reg_val = hisi_sas_read32(hisi_hba, HGC_LM_DFX_STATUS2);
-		dev_warn(dev, "hgc_iostl_accbad_intr (0x%x) found: \
-		       memory address is 0x%08X\n",
-		      irq_value,
-		      (reg_val & HGC_LM_DFX_STATUS2_IOSTLIST_MSK) >>
-		      HGC_LM_DFX_STATUS2_IOSTLIST_OFF);
-		queue_work(hisi_hba->wq, &hisi_hba->rst_work);
-	}
-
-	if (irq_value & BIT(SAS_ECC_INTR_ITCTLIST_ECC_MB_OFF)) {
-		reg_val = hisi_sas_read32(hisi_hba, HGC_LM_DFX_STATUS2);
-		dev_warn(dev, "hgc_itctl_accbad_intr (0x%x) found: \
-		       memory address is 0x%08X\n",
-		      irq_value,
-		      (reg_val & HGC_LM_DFX_STATUS2_ITCTLIST_MSK) >>
-		      HGC_LM_DFX_STATUS2_ITCTLIST_OFF);
-		queue_work(hisi_hba->wq, &hisi_hba->rst_work);
-	}
-
-	if (irq_value & BIT(SAS_ECC_INTR_CQE_ECC_MB_OFF)) {
-		reg_val = hisi_sas_read32(hisi_hba, HGC_CQE_ECC_ADDR);
-		dev_warn(dev, "hgc_cqe_accbad_intr (0x%x) found: \
-		       Ram address is 0x%08X\n",
-		      irq_value,
-		      (reg_val & HGC_CQE_ECC_MB_ADDR_MSK) >>
-		      HGC_CQE_ECC_MB_ADDR_OFF);
-		queue_work(hisi_hba->wq, &hisi_hba->rst_work);
-	}
-
-	if (irq_value & BIT(SAS_ECC_INTR_NCQ_MEM0_ECC_MB_OFF)) {
-		reg_val = hisi_sas_read32(hisi_hba, HGC_RXM_DFX_STATUS14);
-		dev_warn(dev, "rxm_mem0_accbad_intr (0x%x) found: \
-		       memory address is 0x%08X\n",
-		      irq_value,
-		      (reg_val & HGC_RXM_DFX_STATUS14_MEM0_MSK) >>
-		      HGC_RXM_DFX_STATUS14_MEM0_OFF);
-		queue_work(hisi_hba->wq, &hisi_hba->rst_work);
-	}
-
-	if (irq_value & BIT(SAS_ECC_INTR_NCQ_MEM1_ECC_MB_OFF)) {
-		reg_val = hisi_sas_read32(hisi_hba, HGC_RXM_DFX_STATUS14);
-		dev_warn(dev, "rxm_mem1_accbad_intr (0x%x) found: \
-		       memory address is 0x%08X\n",
-		      irq_value,
-		      (reg_val & HGC_RXM_DFX_STATUS14_MEM1_MSK) >>
-		      HGC_RXM_DFX_STATUS14_MEM1_OFF);
-		queue_work(hisi_hba->wq, &hisi_hba->rst_work);
-	}
-
-	if (irq_value & BIT(SAS_ECC_INTR_NCQ_MEM2_ECC_MB_OFF)) {
-		reg_val = hisi_sas_read32(hisi_hba, HGC_RXM_DFX_STATUS14);
-		dev_warn(dev, "rxm_mem2_accbad_intr (0x%x) found: \
-		       memory address is 0x%08X\n",
-		      irq_value,
-		      (reg_val & HGC_RXM_DFX_STATUS14_MEM2_MSK) >>
-		      HGC_RXM_DFX_STATUS14_MEM2_OFF);
-		queue_work(hisi_hba->wq, &hisi_hba->rst_work);
-	}
-
-	if (irq_value & BIT(SAS_ECC_INTR_NCQ_MEM3_ECC_MB_OFF)) {
-		reg_val = hisi_sas_read32(hisi_hba, HGC_RXM_DFX_STATUS15);
-		dev_warn(dev, "rxm_mem3_accbad_intr (0x%x) found: \
-		       memory address is 0x%08X\n",
-		      irq_value,
-		      (reg_val & HGC_RXM_DFX_STATUS15_MEM3_MSK) >>
-		      HGC_RXM_DFX_STATUS15_MEM3_OFF);
-		queue_work(hisi_hba->wq, &hisi_hba->rst_work);
+	for (i = 0; i < ARRAY_SIZE(multi_bit_ecc_errors); i++) {
+		ecc_error = &multi_bit_ecc_errors[i];
+		if (irq_value & ecc_error->irq_msk) {
+			val = hisi_sas_read32(hisi_hba, ecc_error->reg);
+			val &= ecc_error->msk;
+			val >>= ecc_error->shift;
+			dev_warn(dev, ecc_error->msg, irq_value, val);
+			queue_work(hisi_hba->wq, &hisi_hba->rst_work);
+		}
 	}
 
 	return;
@@ -3053,8 +3060,20 @@ static irqreturn_t fatal_axi_int_v2_hw(int irq_no, void *p)
 			      irq_value);
 			queue_work(hisi_hba->wq, &hisi_hba->rst_work);
 		}
+
+		if (irq_value & BIT(ENT_INT_SRC3_ITC_INT_OFF)) {
+			u32 reg_val = hisi_sas_read32(hisi_hba, ITCT_CLR);
+			u32 dev_id = reg_val & ITCT_DEV_MSK;
+			struct hisi_sas_device *sas_dev =
+					&hisi_hba->devices[dev_id];
+
+			hisi_sas_write32(hisi_hba, ITCT_CLR, 0);
+			dev_dbg(dev, "clear ITCT ok\n");
+			complete(sas_dev->completion);
+		}
 	}
 
+	hisi_sas_write32(hisi_hba, ENT_INT_SRC3, irq_value);
 	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, irq_msk);
 
 	return IRQ_HANDLED;
@@ -3251,97 +3270,92 @@ static int interrupt_init_v2_hw(struct hisi_hba *hisi_hba)
 {
 	struct platform_device *pdev = hisi_hba->platform_dev;
 	struct device *dev = &pdev->dev;
-	int i, irq, rc, irq_map[128];
-
+	int irq, rc, irq_map[128];
+	int i, phy_no, fatal_no, queue_no, k;
 
 	for (i = 0; i < 128; i++)
 		irq_map[i] = platform_get_irq(pdev, i);
 
 	for (i = 0; i < HISI_SAS_PHY_INT_NR; i++) {
-		int idx = i;
-
-		irq = irq_map[idx + 1]; /* Phy up/down is irq1 */
-		if (!irq) {
-			dev_err(dev, "irq init: fail map phy interrupt %d\n",
-				idx);
-			return -ENOENT;
-		}
-
+		irq = irq_map[i + 1]; /* Phy up/down is irq1 */
 		rc = devm_request_irq(dev, irq, phy_interrupts[i], 0,
 				      DRV_NAME " phy", hisi_hba);
 		if (rc) {
 			dev_err(dev, "irq init: could not request "
 				"phy interrupt %d, rc=%d\n",
 				irq, rc);
-			return -ENOENT;
+			rc = -ENOENT;
+			goto free_phy_int_irqs;
 		}
 	}
 
-	for (i = 0; i < hisi_hba->n_phy; i++) {
-		struct hisi_sas_phy *phy = &hisi_hba->phy[i];
-		int idx = i + 72; /* First SATA interrupt is irq72 */
-
-		irq = irq_map[idx];
-		if (!irq) {
-			dev_err(dev, "irq init: fail map phy interrupt %d\n",
-				idx);
-			return -ENOENT;
-		}
+	for (phy_no = 0; phy_no < hisi_hba->n_phy; phy_no++) {
+		struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
 
+		irq = irq_map[phy_no + 72];
 		rc = devm_request_irq(dev, irq, sata_int_v2_hw, 0,
 				      DRV_NAME " sata", phy);
 		if (rc) {
 			dev_err(dev, "irq init: could not request "
 				"sata interrupt %d, rc=%d\n",
 				irq, rc);
-			return -ENOENT;
+			rc = -ENOENT;
+			goto free_sata_int_irqs;
 		}
 	}
 
-	for (i = 0; i < HISI_SAS_FATAL_INT_NR; i++) {
-		int idx = i;
-
-		irq = irq_map[idx + 81];
-		if (!irq) {
-			dev_err(dev, "irq init: fail map fatal interrupt %d\n",
-				idx);
-			return -ENOENT;
-		}
-
-		rc = devm_request_irq(dev, irq, fatal_interrupts[i], 0,
+	for (fatal_no = 0; fatal_no < HISI_SAS_FATAL_INT_NR; fatal_no++) {
+		irq = irq_map[fatal_no + 81];
+		rc = devm_request_irq(dev, irq, fatal_interrupts[fatal_no], 0,
 				      DRV_NAME " fatal", hisi_hba);
 		if (rc) {
 			dev_err(dev,
 				"irq init: could not request fatal interrupt %d, rc=%d\n",
 				irq, rc);
-			return -ENOENT;
+			rc = -ENOENT;
+			goto free_fatal_int_irqs;
 		}
 	}
 
-	for (i = 0; i < hisi_hba->queue_count; i++) {
-		int idx = i + 96; /* First cq interrupt is irq96 */
-		struct hisi_sas_cq *cq = &hisi_hba->cq[i];
+	for (queue_no = 0; queue_no < hisi_hba->queue_count; queue_no++) {
+		struct hisi_sas_cq *cq = &hisi_hba->cq[queue_no];
 		struct tasklet_struct *t = &cq->tasklet;
 
-		irq = irq_map[idx];
-		if (!irq) {
-			dev_err(dev,
-				"irq init: could not map cq interrupt %d\n",
-				idx);
-			return -ENOENT;
-		}
+		irq = irq_map[queue_no + 96];
 		rc = devm_request_irq(dev, irq, cq_interrupt_v2_hw, 0,
-				      DRV_NAME " cq", &hisi_hba->cq[i]);
+				      DRV_NAME " cq", cq);
 		if (rc) {
 			dev_err(dev,
 				"irq init: could not request cq interrupt %d, rc=%d\n",
 				irq, rc);
-			return -ENOENT;
+			rc = -ENOENT;
+			goto free_cq_int_irqs;
 		}
 		tasklet_init(t, cq_tasklet_v2_hw, (unsigned long)cq);
 	}
 
 	return 0;
+
+free_cq_int_irqs:
+	for (k = 0; k < queue_no; k++) {
+		struct hisi_sas_cq *cq = &hisi_hba->cq[k];
+
+		free_irq(irq_map[k + 96], cq);
+		tasklet_kill(&cq->tasklet);
+	}
+free_fatal_int_irqs:
+	for (k = 0; k < fatal_no; k++)
+		free_irq(irq_map[k + 81], hisi_hba);
+free_sata_int_irqs:
+	for (k = 0; k < phy_no; k++) {
+		struct hisi_sas_phy *phy = &hisi_hba->phy[k];
+
+		free_irq(irq_map[k + 72], phy);
+	}
+free_phy_int_irqs:
+	for (k = 0; k < i; k++)
+		free_irq(irq_map[k + 1], hisi_hba);
+	return rc;
 }
 
 static int hisi_sas_v2_init(struct hisi_hba *hisi_hba)
@@ -3383,19 +3397,21 @@ static void interrupt_disable_v2_hw(struct hisi_hba *hisi_hba)
 		synchronize_irq(platform_get_irq(pdev, i));
 }
 
+
+static u32 get_phys_state_v2_hw(struct hisi_hba *hisi_hba)
+{
+	return hisi_sas_read32(hisi_hba, PHY_STATE);
+}
+
 static int soft_reset_v2_hw(struct hisi_hba *hisi_hba)
 {
 	struct device *dev = hisi_hba->dev;
-	u32 old_state, state;
 	int rc, cnt;
-	int phy_no;
-
-	old_state = hisi_sas_read32(hisi_hba, PHY_STATE);
 
 	interrupt_disable_v2_hw(hisi_hba);
 	hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0x0);
 
-	stop_phys_v2_hw(hisi_hba);
+	hisi_sas_stop_phys(hisi_hba);
 
 	mdelay(10);
 
@@ -3425,22 +3441,6 @@ static int soft_reset_v2_hw(struct hisi_hba *hisi_hba)
 
 	phys_reject_stp_links_v2_hw(hisi_hba);
 
-	/* Re-enable the PHYs */
-	for (phy_no = 0; phy_no < hisi_hba->n_phy; phy_no++) {
-		struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
-		struct asd_sas_phy *sas_phy = &phy->sas_phy;
-
-		if (sas_phy->enabled)
-			start_phy_v2_hw(hisi_hba, phy_no);
-	}
-
-	/* Wait for the PHYs to come up and read the PHY state */
-	msleep(1000);
-
-	state = hisi_sas_read32(hisi_hba, PHY_STATE);
-
-	hisi_sas_rescan_topology(hisi_hba, old_state, state);
-
 	return 0;
 }
 
@@ -3463,11 +3463,13 @@ static const struct hisi_sas_hw hisi_sas_v2_hw = {
 	.phy_enable = enable_phy_v2_hw,
 	.phy_disable = disable_phy_v2_hw,
 	.phy_hard_reset = phy_hard_reset_v2_hw,
+	.get_events = phy_get_events_v2_hw,
 	.phy_set_linkrate = phy_set_linkrate_v2_hw,
 	.phy_get_max_linkrate = phy_get_max_linkrate_v2_hw,
 	.max_command_entries = HISI_SAS_COMMAND_ENTRIES_V2_HW,
 	.complete_hdr_size = sizeof(struct hisi_sas_complete_v2_hdr),
 	.soft_reset = soft_reset_v2_hw,
+	.get_phys_state = get_phys_state_v2_hw,
 };
 
 static int hisi_sas_v2_probe(struct platform_device *pdev)
@@ -3491,10 +3493,17 @@ static int hisi_sas_v2_remove(struct platform_device *pdev)
 {
 	struct sas_ha_struct *sha = platform_get_drvdata(pdev);
 	struct hisi_hba *hisi_hba = sha->lldd_ha;
+	int i;
 
 	if (timer_pending(&hisi_hba->timer))
 		del_timer(&hisi_hba->timer);
 
+	for (i = 0; i < hisi_hba->queue_count; i++) {
+		struct hisi_sas_cq *cq = &hisi_hba->cq[i];
+
+		tasklet_kill(&cq->tasklet);
+	}
+
 	return hisi_sas_remove(pdev);
 }
 
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
index 83d2dca1c650..2e5fa9717be8 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
@@ -23,14 +23,11 @@
 #define PHY_STATE			0x24
 #define PHY_PORT_NUM_MA			0x28
 #define PHY_CONN_RATE			0x30
-#define AXI_AHB_CLK_CFG			0x3c
 #define ITCT_CLR			0x44
 #define ITCT_CLR_EN_OFF			16
 #define ITCT_CLR_EN_MSK			(0x1 << ITCT_CLR_EN_OFF)
 #define ITCT_DEV_OFF			0
 #define ITCT_DEV_MSK			(0x7ff << ITCT_DEV_OFF)
-#define AXI_USER1			0x48
-#define AXI_USER2			0x4c
 #define IO_SATA_BROKEN_MSG_ADDR_LO	0x58
 #define IO_SATA_BROKEN_MSG_ADDR_HI	0x5c
 #define SATA_INITI_D2H_STORE_ADDR_LO	0x60
@@ -137,6 +134,7 @@
 #define TX_HARDRST_MSK          (0x1 << TX_HARDRST_OFF)
 #define RX_IDAF_DWORD0			(PORT_BASE + 0xc4)
 #define RXOP_CHECK_CFG_H		(PORT_BASE + 0xfc)
+#define STP_LINK_TIMER			(PORT_BASE + 0x120)
 #define SAS_SSP_CON_TIMER_CFG		(PORT_BASE + 0x134)
 #define SAS_SMP_CON_TIMER_CFG		(PORT_BASE + 0x138)
 #define SAS_STP_CON_TIMER_CFG		(PORT_BASE + 0x13c)
@@ -167,6 +165,31 @@
 #define PHYCTRL_PHY_ENA_MSK		(PORT_BASE + 0x2bc)
 #define SL_RX_BCAST_CHK_MSK		(PORT_BASE + 0x2c0)
 #define PHYCTRL_OOB_RESTART_MSK		(PORT_BASE + 0x2c4)
+#define DMA_TX_STATUS			(PORT_BASE + 0x2d0)
+#define DMA_TX_STATUS_BUSY_OFF		0
+#define DMA_TX_STATUS_BUSY_MSK		(0x1 << DMA_TX_STATUS_BUSY_OFF)
+#define DMA_RX_STATUS			(PORT_BASE + 0x2e8)
+#define DMA_RX_STATUS_BUSY_OFF		0
+#define DMA_RX_STATUS_BUSY_MSK		(0x1 << DMA_RX_STATUS_BUSY_OFF)
+
+#define MAX_ITCT_HW			4096 /* max the hw can support */
+#define DEFAULT_ITCT_HW		2048 /* reset value, not reprogrammed */
+#if (HISI_SAS_MAX_DEVICES > DEFAULT_ITCT_HW)
+#error Max ITCT exceeded
+#endif
+
+#define AXI_MASTER_CFG_BASE		(0x5000)
+#define AM_CTRL_GLOBAL			(0x0)
+#define AM_CURR_TRANS_RETURN	(0x150)
+
+#define AM_CFG_MAX_TRANS		(0x5010)
+#define AM_CFG_SINGLE_PORT_MAX_TRANS	(0x5014)
+#define AXI_CFG					(0x5100)
+#define AM_ROB_ECC_ERR_ADDR		(0x510c)
+#define AM_ROB_ECC_ONEBIT_ERR_ADDR_OFF	0
+#define AM_ROB_ECC_ONEBIT_ERR_ADDR_MSK	(0xff << AM_ROB_ECC_ONEBIT_ERR_ADDR_OFF)
+#define AM_ROB_ECC_MULBIT_ERR_ADDR_OFF	8
+#define AM_ROB_ECC_MULBIT_ERR_ADDR_MSK	(0xff << AM_ROB_ECC_MULBIT_ERR_ADDR_OFF)
 
 /* HW dma structures */
 /* Delivery queue header */
@@ -354,8 +377,6 @@ static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
 	/* Global registers init */
 	hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE,
 			 (u32)((1ULL << hisi_hba->queue_count) - 1));
-	hisi_sas_write32(hisi_hba, AXI_USER1, 0x0);
-	hisi_sas_write32(hisi_hba, AXI_USER2, 0x40000060);
 	hisi_sas_write32(hisi_hba, HGC_SAS_TXFAIL_RETRY_CTRL, 0x108);
 	hisi_sas_write32(hisi_hba, CFG_1US_TIMER_TRSH, 0xd);
 	hisi_sas_write32(hisi_hba, INT_COAL_EN, 0x1);
@@ -371,15 +392,14 @@ static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
 	hisi_sas_write32(hisi_hba, CHNL_PHYUPDOWN_INT_MSK, 0x0);
 	hisi_sas_write32(hisi_hba, CHNL_ENT_INT_MSK, 0x0);
 	hisi_sas_write32(hisi_hba, HGC_COM_INT_MSK, 0x0);
-	hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0xfff00c30);
+	hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0x0);
 	hisi_sas_write32(hisi_hba, AWQOS_AWCACHE_CFG, 0xf0f0);
 	hisi_sas_write32(hisi_hba, ARQOS_ARCACHE_CFG, 0xf0f0);
 	for (i = 0; i < hisi_hba->queue_count; i++)
 		hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK+0x4*i, 0);
 
-	hisi_sas_write32(hisi_hba, AXI_AHB_CLK_CFG, 1);
 	hisi_sas_write32(hisi_hba, HYPER_STREAM_ID_EN_CFG, 1);
-	hisi_sas_write32(hisi_hba, CFG_MAX_TAG, 0xfff07fff);
+	hisi_sas_write32(hisi_hba, AXI_MASTER_CFG_BASE, 0x30000);
 
 	for (i = 0; i < hisi_hba->n_phy; i++) {
 		hisi_sas_phy_write32(hisi_hba, i, PROG_PHY_LINK_RATE, 0x801);
@@ -389,7 +409,6 @@ static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
 		hisi_sas_phy_write32(hisi_hba, i, RXOP_CHECK_CFG_H, 0x1000);
 		hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK, 0xffffffff);
 		hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0x8ffffbff);
-		hisi_sas_phy_write32(hisi_hba, i, SL_CFG, 0x83f801fc);
 		hisi_sas_phy_write32(hisi_hba, i, PHY_CTRL_RDY_MSK, 0x0);
 		hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_NOT_RDY_MSK, 0x0);
 		hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_DWS_RESET_MSK, 0x0);
@@ -398,9 +417,11 @@ static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
 		hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_OOB_RESTART_MSK, 0x0);
 		hisi_sas_phy_write32(hisi_hba, i, PHY_CTRL, 0x199b4fa);
 		hisi_sas_phy_write32(hisi_hba, i, SAS_SSP_CON_TIMER_CFG,
-				     0xa0064);
+				     0xa03e8);
 		hisi_sas_phy_write32(hisi_hba, i, SAS_STP_CON_TIMER_CFG,
-				     0xa0064);
+				     0xa03e8);
+		hisi_sas_phy_write32(hisi_hba, i, STP_LINK_TIMER,
+				     0x7f7a120);
 	}
 	for (i = 0; i < hisi_hba->queue_count; i++) {
 		/* Delivery queue */
@@ -578,8 +599,6 @@ static void free_device_v3_hw(struct hisi_hba *hisi_hba,
 		memset(itct, 0, sizeof(struct hisi_sas_itct));
 		hisi_sas_write32(hisi_hba, ENT_INT_SRC3,
 				 ENT_INT_SRC3_ITC_INT_MSK);
-		hisi_hba->devices[dev_id].dev_type = SAS_PHY_UNUSED;
-		hisi_hba->devices[dev_id].dev_status = HISI_SAS_DEV_NORMAL;
 
 		/* clear the itct */
 		hisi_sas_write32(hisi_hba, ITCT_CLR, 0);
@@ -610,8 +629,52 @@ static void dereg_device_v3_hw(struct hisi_hba *hisi_hba,
 					1 << CFG_ABT_SET_IPTT_DONE_OFF);
 }
 
+static int reset_hw_v3_hw(struct hisi_hba *hisi_hba)
+{
+	struct device *dev = hisi_hba->dev;
+	int ret;
+	u32 val;
+
+	hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0);
+
+	/* Disable all of the PHYs */
+	hisi_sas_stop_phys(hisi_hba);
+	udelay(50);
+
+	/* Ensure axi bus idle */
+	ret = readl_poll_timeout(hisi_hba->regs + AXI_CFG, val, !val,
+			20000, 1000000);
+	if (ret) {
+		dev_err(dev, "axi bus is not idle, ret = %d!\n", ret);
+		return -EIO;
+	}
+
+	if (ACPI_HANDLE(dev)) {
+		acpi_status s;
+
+		s = acpi_evaluate_object(ACPI_HANDLE(dev), "_RST", NULL, NULL);
+		if (ACPI_FAILURE(s)) {
+			dev_err(dev, "Reset failed\n");
+			return -EIO;
+		}
+	} else
+		dev_err(dev, "no reset method!\n");
+
+	return 0;
+}
+
 static int hw_init_v3_hw(struct hisi_hba *hisi_hba)
 {
+	struct device *dev = hisi_hba->dev;
+	int rc;
+
+	rc = reset_hw_v3_hw(hisi_hba);
+	if (rc) {
+		dev_err(dev, "hisi_sas_reset_hw failed, rc=%d", rc);
+		return rc;
+	}
+
+	msleep(100);
 	init_reg_v3_hw(hisi_hba);
 
 	return 0;
@@ -640,25 +703,12 @@ static void start_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 	enable_phy_v3_hw(hisi_hba, phy_no);
 }
 
-static void stop_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
-{
-	disable_phy_v3_hw(hisi_hba, phy_no);
-}
-
-static void start_phys_v3_hw(struct hisi_hba *hisi_hba)
-{
-	int i;
-
-	for (i = 0; i < hisi_hba->n_phy; i++)
-		start_phy_v3_hw(hisi_hba, i);
-}
-
 static void phy_hard_reset_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 {
 	struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
 	u32 txid_auto;
 
-	stop_phy_v3_hw(hisi_hba, phy_no);
+	disable_phy_v3_hw(hisi_hba, phy_no);
 	if (phy->identify.device_type == SAS_END_DEVICE) {
 		txid_auto = hisi_sas_phy_read32(hisi_hba, phy_no, TXID_AUTO);
 		hisi_sas_phy_write32(hisi_hba, phy_no, TXID_AUTO,
@@ -675,7 +725,17 @@ enum sas_linkrate phy_get_max_linkrate_v3_hw(void)
 
 static void phys_init_v3_hw(struct hisi_hba *hisi_hba)
 {
-	start_phys_v3_hw(hisi_hba);
+	int i;
+
+	for (i = 0; i < hisi_hba->n_phy; i++) {
+		struct hisi_sas_phy *phy = &hisi_hba->phy[i];
+		struct asd_sas_phy *sas_phy = &phy->sas_phy;
+
+		if (!sas_phy->phy->enabled)
+			continue;
+
+		start_phy_v3_hw(hisi_hba, i);
+	}
 }
 
 static void sl_notify_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
@@ -1140,7 +1200,6 @@ end:
 
 static int phy_down_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
 {
-	int res = 0;
 	u32 phy_state, sl_ctrl, txid_auto;
 	struct device *dev = hisi_hba->dev;
 
@@ -1161,7 +1220,7 @@ static int phy_down_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
 	hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0, CHL_INT0_NOT_RDY_MSK);
 	hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_NOT_RDY_MSK, 0);
 
-	return res;
+	return 0;
 }
 
 static void phy_bcast_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
@@ -1259,7 +1318,7 @@ static irqreturn_t int_chnl_int_v3_hw(int irq_no, void *p)
 		if (irq_msk & (2 << (phy_no * 4)) && irq_value0) {
 			hisi_sas_phy_write32(hisi_hba, phy_no,
 					CHL_INT0, irq_value0
-					& (~CHL_INT0_HOTPLUG_TOUT_MSK)
+					& (~CHL_INT0_SL_RX_BCST_ACK_MSK)
 					& (~CHL_INT0_SL_PHY_ENABLE_MSK)
 					& (~CHL_INT0_NOT_RDY_MSK));
 		}
@@ -1620,6 +1679,104 @@ static int hisi_sas_v3_init(struct hisi_hba *hisi_hba)
 	return 0;
 }
 
+static void phy_set_linkrate_v3_hw(struct hisi_hba *hisi_hba, int phy_no,
+		struct sas_phy_linkrates *r)
+{
+	u32 prog_phy_link_rate =
+		hisi_sas_phy_read32(hisi_hba, phy_no, PROG_PHY_LINK_RATE);
+	struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
+	struct asd_sas_phy *sas_phy = &phy->sas_phy;
+	int i;
+	enum sas_linkrate min, max;
+	u32 rate_mask = 0;
+
+	if (r->maximum_linkrate == SAS_LINK_RATE_UNKNOWN) {
+		max = sas_phy->phy->maximum_linkrate;
+		min = r->minimum_linkrate;
+	} else if (r->minimum_linkrate == SAS_LINK_RATE_UNKNOWN) {
+		max = r->maximum_linkrate;
+		min = sas_phy->phy->minimum_linkrate;
+	} else
+		return;
+
+	sas_phy->phy->maximum_linkrate = max;
+	sas_phy->phy->minimum_linkrate = min;
+
+	min -= SAS_LINK_RATE_1_5_GBPS;
+	max -= SAS_LINK_RATE_1_5_GBPS;
+
+	for (i = 0; i <= max; i++)
+		rate_mask |= 1 << (i * 2);
+
+	prog_phy_link_rate &= ~0xff;
+	prog_phy_link_rate |= rate_mask;
+
+	hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE,
+			prog_phy_link_rate);
+
+	phy_hard_reset_v3_hw(hisi_hba, phy_no);
+}
+
+static void interrupt_disable_v3_hw(struct hisi_hba *hisi_hba)
+{
+	struct pci_dev *pdev = hisi_hba->pci_dev;
+	int i;
+
+	synchronize_irq(pci_irq_vector(pdev, 1));
+	synchronize_irq(pci_irq_vector(pdev, 2));
+	synchronize_irq(pci_irq_vector(pdev, 11));
+	for (i = 0; i < hisi_hba->queue_count; i++) {
+		hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK + 0x4 * i, 0x1);
+		synchronize_irq(pci_irq_vector(pdev, i + 16));
+	}
+
+	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0xffffffff);
+	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0xffffffff);
+	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xffffffff);
+	hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0xffffffff);
+
+	for (i = 0; i < hisi_hba->n_phy; i++) {
+		hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK, 0xffffffff);
+		hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0xffffffff);
+		hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_NOT_RDY_MSK, 0x1);
+		hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_PHY_ENA_MSK, 0x1);
+		hisi_sas_phy_write32(hisi_hba, i, SL_RX_BCAST_CHK_MSK, 0x1);
+	}
+}
+
+static u32 get_phys_state_v3_hw(struct hisi_hba *hisi_hba)
+{
+	return hisi_sas_read32(hisi_hba, PHY_STATE);
+}
+
+static int soft_reset_v3_hw(struct hisi_hba *hisi_hba)
+{
+	struct device *dev = hisi_hba->dev;
+	int rc;
+	u32 status;
+
+	interrupt_disable_v3_hw(hisi_hba);
+	hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0x0);
+
+	hisi_sas_stop_phys(hisi_hba);
+
+	mdelay(10);
+
+	hisi_sas_write32(hisi_hba, AXI_MASTER_CFG_BASE + AM_CTRL_GLOBAL, 0x1);
+
+	/* wait until bus idle */
+	rc = readl_poll_timeout(hisi_hba->regs + AXI_MASTER_CFG_BASE +
+		AM_CURR_TRANS_RETURN, status, status == 0x3, 10, 100);
+	if (rc) {
+		dev_err(dev, "axi bus is not idle, rc = %d\n", rc);
+		return rc;
+	}
+
+	hisi_sas_init_mem(hisi_hba);
+
+	return hw_init_v3_hw(hisi_hba);
+}
+
 static const struct hisi_sas_hw hisi_sas_v3_hw = {
 	.hw_init = hisi_sas_v3_init,
 	.setup_itct = setup_itct_v3_hw,
@@ -1640,7 +1797,10 @@ static const struct hisi_sas_hw hisi_sas_v3_hw = {
 	.phy_disable = disable_phy_v3_hw,
 	.phy_hard_reset = phy_hard_reset_v3_hw,
 	.phy_get_max_linkrate = phy_get_max_linkrate_v3_hw,
+	.phy_set_linkrate = phy_set_linkrate_v3_hw,
 	.dereg_device = dereg_device_v3_hw,
+	.soft_reset = soft_reset_v3_hw,
+	.get_phys_state = get_phys_state_v3_hw,
 };
 
 static struct Scsi_Host *
@@ -1651,8 +1811,10 @@ hisi_sas_shost_alloc_pci(struct pci_dev *pdev)
 	struct device *dev = &pdev->dev;
 
 	shost = scsi_host_alloc(hisi_sas_sht, sizeof(*hisi_hba));
-	if (!shost)
-		goto err_out;
+	if (!shost) {
+		dev_err(dev, "shost alloc failed\n");
+		return NULL;
+	}
 	hisi_hba = shost_priv(shost);
 
 	hisi_hba->hw = &hisi_sas_v3_hw;
@@ -1673,6 +1835,7 @@ hisi_sas_shost_alloc_pci(struct pci_dev *pdev)
 
 	return shost;
 err_out:
+	scsi_host_put(shost);
 	dev_err(dev, "shost alloc failed\n");
 	return NULL;
 }
@@ -1781,7 +1944,7 @@ hisi_sas_v3_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 err_out_register_ha:
 	scsi_remove_host(shost);
 err_out_ha:
-	kfree(shost);
+	scsi_host_put(shost);
 err_out_regions:
 	pci_release_regions(pdev);
 err_out_disable_device:
@@ -1801,6 +1964,7 @@ hisi_sas_v3_destroy_irqs(struct pci_dev *pdev, struct hisi_hba *hisi_hba)
 		struct hisi_sas_cq *cq = &hisi_hba->cq[i];
 
 		free_irq(pci_irq_vector(pdev, i+16), cq);
+		tasklet_kill(&cq->tasklet);
 	}
 	pci_free_irq_vectors(pdev);
 }
@@ -1810,14 +1974,16 @@ static void hisi_sas_v3_remove(struct pci_dev *pdev)
 	struct device *dev = &pdev->dev;
 	struct sas_ha_struct *sha = dev_get_drvdata(dev);
 	struct hisi_hba *hisi_hba = sha->lldd_ha;
+	struct Scsi_Host *shost = sha->core.shost;
 
 	sas_unregister_ha(sha);
 	sas_remove_host(sha->core.shost);
 
-	hisi_sas_free(hisi_hba);
 	hisi_sas_v3_destroy_irqs(pdev, hisi_hba);
 	pci_release_regions(pdev);
 	pci_disable_device(pdev);
+	hisi_sas_free(hisi_hba);
+	scsi_host_put(shost);
 }
 
 enum {
@@ -1839,7 +2005,6 @@ static struct pci_driver sas_v3_pci_driver = {
 
 module_pci_driver(sas_v3_pci_driver);
 
-MODULE_VERSION(DRV_VERSION);
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("John Garry <john.garry@huawei.com>");
 MODULE_DESCRIPTION("HISILICON SAS controller v3 hw driver based on pci device");
diff --git a/drivers/scsi/hosts.c b/drivers/scsi/hosts.c
index 831a1c8b9f89..fe3a0da3ec97 100644
--- a/drivers/scsi/hosts.c
+++ b/drivers/scsi/hosts.c
@@ -315,8 +315,6 @@ static void scsi_host_dev_release(struct device *dev)
 {
 	struct Scsi_Host *shost = dev_to_shost(dev);
 	struct device *parent = dev->parent;
-	struct request_queue *q;
-	void *queuedata;
 
 	scsi_proc_hostdir_rm(shost->hostt);
 
@@ -326,12 +324,6 @@ static void scsi_host_dev_release(struct device *dev)
 		kthread_stop(shost->ehandler);
 	if (shost->work_q)
 		destroy_workqueue(shost->work_q);
-	q = shost->uspace_req_q;
-	if (q) {
-		queuedata = q->queuedata;
-		blk_cleanup_queue(q);
-		kfree(queuedata);
-	}
 
 	if (shost->shost_state == SHOST_CREATED) {
 		/*
diff --git a/drivers/scsi/hpsa.c b/drivers/scsi/hpsa.c
index 4f7cdb28bd38..9abe81021484 100644
--- a/drivers/scsi/hpsa.c
+++ b/drivers/scsi/hpsa.c
@@ -81,11 +81,8 @@ MODULE_DESCRIPTION("Driver for HP Smart Array Controller version " \
 MODULE_SUPPORTED_DEVICE("HP Smart Array Controllers");
 MODULE_VERSION(HPSA_DRIVER_VERSION);
 MODULE_LICENSE("GPL");
+MODULE_ALIAS("cciss");
 
-static int hpsa_allow_any;
-module_param(hpsa_allow_any, int, S_IRUGO|S_IWUSR);
-MODULE_PARM_DESC(hpsa_allow_any,
-		"Allow hpsa driver to access unknown HP Smart Array hardware");
 static int hpsa_simple_mode;
 module_param(hpsa_simple_mode, int, S_IRUGO|S_IWUSR);
 MODULE_PARM_DESC(hpsa_simple_mode,
@@ -148,6 +145,8 @@ static const struct pci_device_id hpsa_pci_device_id[] = {
 	{PCI_VENDOR_ID_HP, 0x333f, 0x103c, 0x333f},
 	{PCI_VENDOR_ID_HP,     PCI_ANY_ID,	PCI_ANY_ID, PCI_ANY_ID,
 		PCI_CLASS_STORAGE_RAID << 8, 0xffff << 8, 0},
+	{PCI_VENDOR_ID_COMPAQ,     PCI_ANY_ID,	PCI_ANY_ID, PCI_ANY_ID,
+		PCI_CLASS_STORAGE_RAID << 8, 0xffff << 8, 0},
 	{0,}
 };
 
@@ -158,6 +157,26 @@ MODULE_DEVICE_TABLE(pci, hpsa_pci_device_id);
  *  access = Address of the struct of function pointers
  */
 static struct board_type products[] = {
+	{0x40700E11, "Smart Array 5300", &SA5A_access},
+	{0x40800E11, "Smart Array 5i", &SA5B_access},
+	{0x40820E11, "Smart Array 532", &SA5B_access},
+	{0x40830E11, "Smart Array 5312", &SA5B_access},
+	{0x409A0E11, "Smart Array 641", &SA5A_access},
+	{0x409B0E11, "Smart Array 642", &SA5A_access},
+	{0x409C0E11, "Smart Array 6400", &SA5A_access},
+	{0x409D0E11, "Smart Array 6400 EM", &SA5A_access},
+	{0x40910E11, "Smart Array 6i", &SA5A_access},
+	{0x3225103C, "Smart Array P600", &SA5A_access},
+	{0x3223103C, "Smart Array P800", &SA5A_access},
+	{0x3234103C, "Smart Array P400", &SA5A_access},
+	{0x3235103C, "Smart Array P400i", &SA5A_access},
+	{0x3211103C, "Smart Array E200i", &SA5A_access},
+	{0x3212103C, "Smart Array E200", &SA5A_access},
+	{0x3213103C, "Smart Array E200i", &SA5A_access},
+	{0x3214103C, "Smart Array E200i", &SA5A_access},
+	{0x3215103C, "Smart Array E200i", &SA5A_access},
+	{0x3237103C, "Smart Array E500", &SA5A_access},
+	{0x323D103C, "Smart Array P700m", &SA5A_access},
 	{0x3241103C, "Smart Array P212", &SA5_access},
 	{0x3243103C, "Smart Array P410", &SA5_access},
 	{0x3245103C, "Smart Array P410i", &SA5_access},
@@ -278,7 +297,8 @@ static int hpsa_find_cfg_addrs(struct pci_dev *pdev, void __iomem *vaddr,
 			       u64 *cfg_offset);
 static int hpsa_pci_find_memory_BAR(struct pci_dev *pdev,
 				    unsigned long *memory_bar);
-static int hpsa_lookup_board_id(struct pci_dev *pdev, u32 *board_id);
+static int hpsa_lookup_board_id(struct pci_dev *pdev, u32 *board_id,
+				bool *legacy_board);
 static int wait_for_device_to_become_ready(struct ctlr_info *h,
 					   unsigned char lunaddr[],
 					   int reply_queue);
@@ -866,6 +886,16 @@ static ssize_t host_show_ctlr_num(struct device *dev,
 	return snprintf(buf, 20, "%d\n", h->ctlr);
 }
 
+static ssize_t host_show_legacy_board(struct device *dev,
+	struct device_attribute *attr, char *buf)
+{
+	struct ctlr_info *h;
+	struct Scsi_Host *shost = class_to_shost(dev);
+
+	h = shost_to_hba(shost);
+	return snprintf(buf, 20, "%d\n", h->legacy_board ? 1 : 0);
+}
+
 static DEVICE_ATTR(raid_level, S_IRUGO, raid_level_show, NULL);
 static DEVICE_ATTR(lunid, S_IRUGO, lunid_show, NULL);
 static DEVICE_ATTR(unique_id, S_IRUGO, unique_id_show, NULL);
@@ -891,6 +921,8 @@ static DEVICE_ATTR(lockup_detected, S_IRUGO,
 	host_show_lockup_detected, NULL);
 static DEVICE_ATTR(ctlr_num, S_IRUGO,
 	host_show_ctlr_num, NULL);
+static DEVICE_ATTR(legacy_board, S_IRUGO,
+	host_show_legacy_board, NULL);
 
 static struct device_attribute *hpsa_sdev_attrs[] = {
 	&dev_attr_raid_level,
@@ -912,6 +944,7 @@ static struct device_attribute *hpsa_shost_attrs[] = {
 	&dev_attr_raid_offload_debug,
 	&dev_attr_lockup_detected,
 	&dev_attr_ctlr_num,
+	&dev_attr_legacy_board,
 	NULL,
 };
 
@@ -3565,7 +3598,7 @@ static int hpsa_scsi_do_report_luns(struct ctlr_info *h, int logical,
 	memset(scsi3addr, 0, sizeof(scsi3addr));
 	if (fill_cmd(c, logical ? HPSA_REPORT_LOG : HPSA_REPORT_PHYS, h,
 		buf, bufsize, 0, scsi3addr, TYPE_CMD)) {
-		rc = -1;
+		rc = -EAGAIN;
 		goto out;
 	}
 	if (extended_response)
@@ -3578,16 +3611,19 @@ static int hpsa_scsi_do_report_luns(struct ctlr_info *h, int logical,
 	if (ei->CommandStatus != 0 &&
 	    ei->CommandStatus != CMD_DATA_UNDERRUN) {
 		hpsa_scsi_interpret_error(h, c);
-		rc = -1;
+		rc = -EIO;
 	} else {
 		struct ReportLUNdata *rld = buf;
 
 		if (rld->extended_response_flag != extended_response) {
-			dev_err(&h->pdev->dev,
-				"report luns requested format %u, got %u\n",
-				extended_response,
-				rld->extended_response_flag);
-			rc = -1;
+			if (!h->legacy_board) {
+				dev_err(&h->pdev->dev,
+					"report luns requested format %u, got %u\n",
+					extended_response,
+					rld->extended_response_flag);
+				rc = -EINVAL;
+			} else
+				rc = -EOPNOTSUPP;
 		}
 	}
 out:
@@ -3603,7 +3639,7 @@ static inline int hpsa_scsi_do_report_phys_luns(struct ctlr_info *h,
 
 	rc = hpsa_scsi_do_report_luns(h, 0, buf, bufsize,
 				      HPSA_REPORT_PHYS_EXTENDED);
-	if (!rc || !hpsa_allow_any)
+	if (!rc || rc != -EOPNOTSUPP)
 		return rc;
 
 	/* REPORT PHYS EXTENDED is not supported */
@@ -3791,7 +3827,7 @@ static int hpsa_update_device_info(struct ctlr_info *h,
 	memset(this_device->device_id, 0,
 		sizeof(this_device->device_id));
 	if (hpsa_get_device_id(h, scsi3addr, this_device->device_id, 8,
-		sizeof(this_device->device_id)))
+		sizeof(this_device->device_id)) < 0)
 		dev_err(&h->pdev->dev,
 			"hpsa%d: %s: can't get device id for host %d:C0:T%d:L%d\t%s\t%.16s\n",
 			h->ctlr, __func__,
@@ -3809,6 +3845,16 @@ static int hpsa_update_device_info(struct ctlr_info *h,
 		if (h->fw_support & MISC_FW_RAID_OFFLOAD_BASIC)
 			hpsa_get_ioaccel_status(h, scsi3addr, this_device);
 		volume_offline = hpsa_volume_offline(h, scsi3addr);
+		if (volume_offline == HPSA_VPD_LV_STATUS_UNSUPPORTED &&
+		    h->legacy_board) {
+			/*
+			 * Legacy boards might not support volume status
+			 */
+			dev_info(&h->pdev->dev,
+				 "C0:T%d:L%d Volume status not available, assuming online.\n",
+				 this_device->target, this_device->lun);
+			volume_offline = 0;
+		}
 		this_device->volume_offline = volume_offline;
 		if (volume_offline == HPSA_LV_FAILED) {
 			rc = HPSA_LV_FAILED;
@@ -6571,7 +6617,6 @@ static int fill_cmd(struct CommandList *c, u8 cmd, struct ctlr_info *h,
 		default:
 			dev_warn(&h->pdev->dev, "unknown command 0x%c\n", cmd);
 			BUG();
-			return -1;
 		}
 	} else if (cmd_type == TYPE_MSG) {
 		switch (cmd) {
@@ -7232,7 +7277,8 @@ static int hpsa_interrupt_mode(struct ctlr_info *h)
 	return 0;
 }
 
-static int hpsa_lookup_board_id(struct pci_dev *pdev, u32 *board_id)
+static int hpsa_lookup_board_id(struct pci_dev *pdev, u32 *board_id,
+				bool *legacy_board)
 {
 	int i;
 	u32 subsystem_vendor_id, subsystem_device_id;
@@ -7242,17 +7288,24 @@ static int hpsa_lookup_board_id(struct pci_dev *pdev, u32 *board_id)
 	*board_id = ((subsystem_device_id << 16) & 0xffff0000) |
 		    subsystem_vendor_id;
 
+	if (legacy_board)
+		*legacy_board = false;
 	for (i = 0; i < ARRAY_SIZE(products); i++)
-		if (*board_id == products[i].board_id)
+		if (*board_id == products[i].board_id) {
+			if (products[i].access != &SA5A_access &&
+			    products[i].access != &SA5B_access)
+				return i;
+			dev_warn(&pdev->dev,
+				 "legacy board ID: 0x%08x\n",
+				 *board_id);
+			if (legacy_board)
+			    *legacy_board = true;
 			return i;
+		}
 
-	if ((subsystem_vendor_id != PCI_VENDOR_ID_HP &&
-		subsystem_vendor_id != PCI_VENDOR_ID_COMPAQ) ||
-		!hpsa_allow_any) {
-		dev_warn(&pdev->dev, "unrecognized board ID: "
-			"0x%08x, ignoring.\n", *board_id);
-			return -ENODEV;
-	}
+	dev_warn(&pdev->dev, "unrecognized board ID: 0x%08x\n", *board_id);
+	if (legacy_board)
+		*legacy_board = true;
 	return ARRAY_SIZE(products) - 1; /* generic unknown smart array */
 }
 
@@ -7555,13 +7608,14 @@ static void hpsa_free_pci_init(struct ctlr_info *h)
 static int hpsa_pci_init(struct ctlr_info *h)
 {
 	int prod_index, err;
+	bool legacy_board;
 
-	prod_index = hpsa_lookup_board_id(h->pdev, &h->board_id);
+	prod_index = hpsa_lookup_board_id(h->pdev, &h->board_id, &legacy_board);
 	if (prod_index < 0)
 		return prod_index;
 	h->product_name = products[prod_index].product_name;
 	h->access = *(products[prod_index].access);
-
+	h->legacy_board = legacy_board;
 	pci_disable_link_state(h->pdev, PCIE_LINK_STATE_L0S |
 			       PCIE_LINK_STATE_L1 | PCIE_LINK_STATE_CLKPM);
 
@@ -8241,7 +8295,7 @@ static int hpsa_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if (number_of_controllers == 0)
 		printk(KERN_INFO DRIVER_NAME "\n");
 
-	rc = hpsa_lookup_board_id(pdev, &board_id);
+	rc = hpsa_lookup_board_id(pdev, &board_id, NULL);
 	if (rc < 0) {
 		dev_warn(&pdev->dev, "Board ID not found\n");
 		return rc;
@@ -9443,14 +9497,6 @@ hpsa_sas_phy_speed(struct sas_phy *phy, struct sas_phy_linkrates *rates)
 	return -EINVAL;
 }
 
-/* SMP = Serial Management Protocol */
-static int
-hpsa_sas_smp_handler(struct Scsi_Host *shost, struct sas_rphy *rphy,
-struct request *req)
-{
-	return -EINVAL;
-}
-
 static struct sas_function_template hpsa_sas_transport_functions = {
 	.get_linkerrors = hpsa_sas_get_linkerrors,
 	.get_enclosure_identifier = hpsa_sas_get_enclosure_identifier,
@@ -9460,7 +9506,6 @@ static struct sas_function_template hpsa_sas_transport_functions = {
 	.phy_setup = hpsa_sas_phy_setup,
 	.phy_release = hpsa_sas_phy_release,
 	.set_phy_speed = hpsa_sas_phy_speed,
-	.smp_handler = hpsa_sas_smp_handler,
 };
 
 /*
diff --git a/drivers/scsi/hpsa.h b/drivers/scsi/hpsa.h
index 1c49741bc639..018f980a701c 100644
--- a/drivers/scsi/hpsa.h
+++ b/drivers/scsi/hpsa.h
@@ -293,6 +293,7 @@ struct ctlr_info {
 	int	drv_req_rescan;
 	int	raid_offload_debug;
 	int     discovery_polling;
+	int     legacy_board;
 	struct  ReportLUNdata *lastlogicals;
 	int	needs_abort_tags_swizzled;
 	struct workqueue_struct *resubmit_wq;
@@ -447,6 +448,23 @@ static void SA5_intr_mask(struct ctlr_info *h, unsigned long val)
 	}
 }
 
+/*
+ *  Variant of the above; 0x04 turns interrupts off...
+ */
+static void SA5B_intr_mask(struct ctlr_info *h, unsigned long val)
+{
+	if (val) { /* Turn interrupts on */
+		h->interrupts_enabled = 1;
+		writel(0, h->vaddr + SA5_REPLY_INTR_MASK_OFFSET);
+		(void) readl(h->vaddr + SA5_REPLY_INTR_MASK_OFFSET);
+	} else { /* Turn them off */
+		h->interrupts_enabled = 0;
+		writel(SA5B_INTR_OFF,
+		       h->vaddr + SA5_REPLY_INTR_MASK_OFFSET);
+		(void) readl(h->vaddr + SA5_REPLY_INTR_MASK_OFFSET);
+	}
+}
+
 static void SA5_performant_intr_mask(struct ctlr_info *h, unsigned long val)
 {
 	if (val) { /* turn on interrupts */
@@ -549,6 +567,14 @@ static bool SA5_ioaccel_mode1_intr_pending(struct ctlr_info *h)
 		true : false;
 }
 
+/*
+ *      Returns true if an interrupt is pending..
+ */
+static bool SA5B_intr_pending(struct ctlr_info *h)
+{
+	return readl(h->vaddr + SA5_INTR_STATUS) & SA5B_INTR_PENDING;
+}
+
 #define IOACCEL_MODE1_REPLY_QUEUE_INDEX  0x1A0
 #define IOACCEL_MODE1_PRODUCER_INDEX     0x1B8
 #define IOACCEL_MODE1_CONSUMER_INDEX     0x1BC
@@ -581,38 +607,53 @@ static unsigned long SA5_ioaccel_mode1_completed(struct ctlr_info *h, u8 q)
 }
 
 static struct access_method SA5_access = {
-	.submit_command = SA5_submit_command,
-	.set_intr_mask = SA5_intr_mask,
-	.intr_pending = SA5_intr_pending,
-	.command_completed = SA5_completed,
+	.submit_command =	SA5_submit_command,
+	.set_intr_mask =	SA5_intr_mask,
+	.intr_pending =		SA5_intr_pending,
+	.command_completed =	SA5_completed,
+};
+
+/* Duplicate entry of the above to mark unsupported boards */
+static struct access_method SA5A_access = {
+	.submit_command =	SA5_submit_command,
+	.set_intr_mask =	SA5_intr_mask,
+	.intr_pending =		SA5_intr_pending,
+	.command_completed =	SA5_completed,
+};
+
+static struct access_method SA5B_access = {
+	.submit_command =	SA5_submit_command,
+	.set_intr_mask =	SA5B_intr_mask,
+	.intr_pending =		SA5B_intr_pending,
+	.command_completed =	SA5_completed,
 };
 
 static struct access_method SA5_ioaccel_mode1_access = {
-	.submit_command = SA5_submit_command,
-	.set_intr_mask = SA5_performant_intr_mask,
-	.intr_pending = SA5_ioaccel_mode1_intr_pending,
-	.command_completed = SA5_ioaccel_mode1_completed,
+	.submit_command =	SA5_submit_command,
+	.set_intr_mask =	SA5_performant_intr_mask,
+	.intr_pending =		SA5_ioaccel_mode1_intr_pending,
+	.command_completed =	SA5_ioaccel_mode1_completed,
 };
 
 static struct access_method SA5_ioaccel_mode2_access = {
-	.submit_command = SA5_submit_command_ioaccel2,
-	.set_intr_mask = SA5_performant_intr_mask,
-	.intr_pending = SA5_performant_intr_pending,
-	.command_completed = SA5_performant_completed,
+	.submit_command =	SA5_submit_command_ioaccel2,
+	.set_intr_mask =	SA5_performant_intr_mask,
+	.intr_pending =		SA5_performant_intr_pending,
+	.command_completed =	SA5_performant_completed,
 };
 
 static struct access_method SA5_performant_access = {
-	.submit_command = SA5_submit_command,
-	.set_intr_mask = SA5_performant_intr_mask,
-	.intr_pending = SA5_performant_intr_pending,
-	.command_completed = SA5_performant_completed,
+	.submit_command =	SA5_submit_command,
+	.set_intr_mask =	SA5_performant_intr_mask,
+	.intr_pending =		SA5_performant_intr_pending,
+	.command_completed =	SA5_performant_completed,
 };
 
 static struct access_method SA5_performant_access_no_read = {
-	.submit_command = SA5_submit_command_no_read,
-	.set_intr_mask = SA5_performant_intr_mask,
-	.intr_pending = SA5_performant_intr_pending,
-	.command_completed = SA5_performant_completed,
+	.submit_command =	SA5_submit_command_no_read,
+	.set_intr_mask =	SA5_performant_intr_mask,
+	.intr_pending =		SA5_performant_intr_pending,
+	.command_completed =	SA5_performant_completed,
 };
 
 struct board_type {
diff --git a/drivers/scsi/hptiop.c b/drivers/scsi/hptiop.c
index 7226226f7383..2fad7f03aa02 100644
--- a/drivers/scsi/hptiop.c
+++ b/drivers/scsi/hptiop.c
@@ -1106,12 +1106,10 @@ static int hptiop_reset_hba(struct hptiop_hba *hba)
 
 static int hptiop_reset(struct scsi_cmnd *scp)
 {
-	struct Scsi_Host * host = scp->device->host;
-	struct hptiop_hba * hba = (struct hptiop_hba *)host->hostdata;
+	struct hptiop_hba * hba = (struct hptiop_hba *)scp->device->host->hostdata;
 
-	printk(KERN_WARNING "hptiop_reset(%d/%d/%d) scp=%p\n",
-			scp->device->host->host_no, scp->device->channel,
-			scp->device->id, scp);
+	printk(KERN_WARNING "hptiop_reset(%d/%d/%d)\n",
+	       scp->device->host->host_no, -1, -1);
 
 	return hptiop_reset_hba(hba)? FAILED : SUCCESS;
 }
@@ -1179,8 +1177,7 @@ static struct scsi_host_template driver_template = {
 	.module                     = THIS_MODULE,
 	.name                       = driver_name,
 	.queuecommand               = hptiop_queuecommand,
-	.eh_device_reset_handler    = hptiop_reset,
-	.eh_bus_reset_handler       = hptiop_reset,
+	.eh_host_reset_handler      = hptiop_reset,
 	.info                       = hptiop_info,
 	.emulated                   = 0,
 	.use_clustering             = ENABLE_CLUSTERING,
diff --git a/drivers/scsi/ibmvscsi/ibmvfc.c b/drivers/scsi/ibmvscsi/ibmvfc.c
index cc4e05be8d4a..b491af31a5f8 100644
--- a/drivers/scsi/ibmvscsi/ibmvfc.c
+++ b/drivers/scsi/ibmvscsi/ibmvfc.c
@@ -2528,16 +2528,12 @@ static int ibmvfc_eh_target_reset_handler(struct scsi_cmnd *cmd)
  **/
 static int ibmvfc_eh_host_reset_handler(struct scsi_cmnd *cmd)
 {
-	int rc, block_rc;
+	int rc;
 	struct ibmvfc_host *vhost = shost_priv(cmd->device->host);
 
-	block_rc = fc_block_scsi_eh(cmd);
 	dev_err(vhost->dev, "Resetting connection due to error recovery\n");
 	rc = ibmvfc_issue_fc_host_lip(vhost->host);
 
-	if (block_rc == FAST_IO_FAIL)
-		return FAST_IO_FAIL;
-
 	return rc ? FAILED : SUCCESS;
 }
 
@@ -4929,7 +4925,7 @@ static unsigned long ibmvfc_get_desired_dma(struct vio_dev *vdev)
 	return pool_dma + ((512 * 1024) * driver_template.cmd_per_lun);
 }
 
-static struct vio_device_id ibmvfc_device_table[] = {
+static const struct vio_device_id ibmvfc_device_table[] = {
 	{"fcp", "IBM,vfc-client"},
 	{ "", "" }
 };
diff --git a/drivers/scsi/ibmvscsi/ibmvscsi.c b/drivers/scsi/ibmvscsi/ibmvscsi.c
index da22b3665cb0..7d156b161482 100644
--- a/drivers/scsi/ibmvscsi/ibmvscsi.c
+++ b/drivers/scsi/ibmvscsi/ibmvscsi.c
@@ -2330,7 +2330,7 @@ static int ibmvscsi_resume(struct device *dev)
  * ibmvscsi_device_table: Used by vio.c to match devices in the device tree we 
  * support.
  */
-static struct vio_device_id ibmvscsi_device_table[] = {
+static const struct vio_device_id ibmvscsi_device_table[] = {
 	{"vscsi", "IBM,v-scsi"},
 	{ "", "" }
 };
diff --git a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
index 1f75d0380516..785fb42f6650 100644
--- a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
+++ b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
@@ -4086,7 +4086,7 @@ static struct class ibmvscsis_class = {
 	.dev_groups	= ibmvscsis_dev_groups,
 };
 
-static struct vio_device_id ibmvscsis_device_table[] = {
+static const struct vio_device_id ibmvscsis_device_table[] = {
 	{ "v-scsi-host", "IBM,v-scsi-host" },
 	{ "", "" }
 };
diff --git a/drivers/scsi/imm.c b/drivers/scsi/imm.c
index 9164ce1249c1..87c94191033b 100644
--- a/drivers/scsi/imm.c
+++ b/drivers/scsi/imm.c
@@ -1106,7 +1106,6 @@ static struct scsi_host_template imm_template = {
 	.name			= "Iomega VPI2 (imm) interface",
 	.queuecommand		= imm_queuecommand,
 	.eh_abort_handler	= imm_abort,
-	.eh_bus_reset_handler	= imm_reset,
 	.eh_host_reset_handler	= imm_reset,
 	.bios_param		= imm_biosparam,
 	.this_id		= 7,
diff --git a/drivers/scsi/ipr.c b/drivers/scsi/ipr.c
index b0c68d24db01..f838bd73befa 100644
--- a/drivers/scsi/ipr.c
+++ b/drivers/scsi/ipr.c
@@ -3351,6 +3351,16 @@ static void ipr_worker_thread(struct work_struct *work)
 		return;
 	}
 
+	if (ioa_cfg->scsi_unblock) {
+		ioa_cfg->scsi_unblock = 0;
+		ioa_cfg->scsi_blocked = 0;
+		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		scsi_unblock_requests(ioa_cfg->host);
+		spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+		if (ioa_cfg->scsi_blocked)
+			scsi_block_requests(ioa_cfg->host);
+	}
+
 	if (!ioa_cfg->scan_enabled) {
 		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
 		return;
@@ -4935,6 +4945,7 @@ static int ipr_slave_configure(struct scsi_device *sdev)
 		}
 		if (ipr_is_vset_device(res)) {
 			sdev->scsi_level = SCSI_SPC_3;
+			sdev->no_report_opcodes = 1;
 			blk_queue_rq_timeout(sdev->request_queue,
 					     IPR_VSET_RW_TIMEOUT);
 			blk_queue_max_hw_sectors(sdev->request_queue, IPR_VSET_MAX_SECTORS);
@@ -7211,9 +7222,8 @@ static int ipr_ioa_bringdown_done(struct ipr_cmnd *ipr_cmd)
 	ENTER;
 	if (!ioa_cfg->hrrq[IPR_INIT_HRRQ].removing_ioa) {
 		ipr_trace;
-		spin_unlock_irq(ioa_cfg->host->host_lock);
-		scsi_unblock_requests(ioa_cfg->host);
-		spin_lock_irq(ioa_cfg->host->host_lock);
+		ioa_cfg->scsi_unblock = 1;
+		schedule_work(&ioa_cfg->work_q);
 	}
 
 	ioa_cfg->in_reset_reload = 0;
@@ -7287,13 +7297,7 @@ static int ipr_ioa_reset_done(struct ipr_cmnd *ipr_cmd)
 	list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_free_q);
 	wake_up_all(&ioa_cfg->reset_wait_q);
 
-	spin_unlock(ioa_cfg->host->host_lock);
-	scsi_unblock_requests(ioa_cfg->host);
-	spin_lock(ioa_cfg->host->host_lock);
-
-	if (!ioa_cfg->hrrq[IPR_INIT_HRRQ].allow_cmds)
-		scsi_block_requests(ioa_cfg->host);
-
+	ioa_cfg->scsi_unblock = 1;
 	schedule_work(&ioa_cfg->work_q);
 	LEAVE;
 	return IPR_RC_JOB_RETURN;
@@ -9249,8 +9253,11 @@ static void _ipr_initiate_ioa_reset(struct ipr_ioa_cfg *ioa_cfg,
 		spin_unlock(&ioa_cfg->hrrq[i]._lock);
 	}
 	wmb();
-	if (!ioa_cfg->hrrq[IPR_INIT_HRRQ].removing_ioa)
+	if (!ioa_cfg->hrrq[IPR_INIT_HRRQ].removing_ioa) {
+		ioa_cfg->scsi_unblock = 0;
+		ioa_cfg->scsi_blocked = 1;
 		scsi_block_requests(ioa_cfg->host);
+	}
 
 	ipr_cmd = ipr_get_free_ipr_cmnd(ioa_cfg);
 	ioa_cfg->reset_cmd = ipr_cmd;
@@ -9306,9 +9313,8 @@ static void ipr_initiate_ioa_reset(struct ipr_ioa_cfg *ioa_cfg,
 			wake_up_all(&ioa_cfg->reset_wait_q);
 
 			if (!ioa_cfg->hrrq[IPR_INIT_HRRQ].removing_ioa) {
-				spin_unlock_irq(ioa_cfg->host->host_lock);
-				scsi_unblock_requests(ioa_cfg->host);
-				spin_lock_irq(ioa_cfg->host->host_lock);
+				ioa_cfg->scsi_unblock = 1;
+				schedule_work(&ioa_cfg->work_q);
 			}
 			return;
 		} else {
diff --git a/drivers/scsi/ipr.h b/drivers/scsi/ipr.h
index e98a87a65335..c7f0e9e3cd7d 100644
--- a/drivers/scsi/ipr.h
+++ b/drivers/scsi/ipr.h
@@ -1488,6 +1488,8 @@ struct ipr_ioa_cfg {
 	u8 cfg_locked:1;
 	u8 clear_isr:1;
 	u8 probe_done:1;
+	u8 scsi_unblock:1;
+	u8 scsi_blocked:1;
 
 	u8 revid;
 
diff --git a/drivers/scsi/isci/init.c b/drivers/scsi/isci/init.c
index 45371179ab87..922e3e56c90d 100644
--- a/drivers/scsi/isci/init.c
+++ b/drivers/scsi/isci/init.c
@@ -166,7 +166,7 @@ static struct scsi_host_template isci_sht = {
 	.use_clustering			= ENABLE_CLUSTERING,
 	.eh_abort_handler		= sas_eh_abort_handler,
 	.eh_device_reset_handler        = sas_eh_device_reset_handler,
-	.eh_bus_reset_handler           = sas_eh_bus_reset_handler,
+	.eh_target_reset_handler        = sas_eh_target_reset_handler,
 	.target_destroy			= sas_target_destroy,
 	.ioctl				= sas_ioctl,
 	.shost_attrs			= isci_host_attrs,
diff --git a/drivers/scsi/iscsi_tcp.c b/drivers/scsi/iscsi_tcp.c
index 4842fc0e809d..4d934d6c3e13 100644
--- a/drivers/scsi/iscsi_tcp.c
+++ b/drivers/scsi/iscsi_tcp.c
@@ -163,7 +163,6 @@ static void iscsi_sw_tcp_state_change(struct sock *sk)
 	struct iscsi_tcp_conn *tcp_conn;
 	struct iscsi_sw_tcp_conn *tcp_sw_conn;
 	struct iscsi_conn *conn;
-	struct iscsi_session *session;
 	void (*old_state_change)(struct sock *);
 
 	read_lock_bh(&sk->sk_callback_lock);
@@ -172,7 +171,6 @@ static void iscsi_sw_tcp_state_change(struct sock *sk)
 		read_unlock_bh(&sk->sk_callback_lock);
 		return;
 	}
-	session = conn->session;
 
 	iscsi_sw_sk_state_check(sk);
 
diff --git a/drivers/scsi/lasi700.c b/drivers/scsi/lasi700.c
index 5c4ded997265..dc839279bbd9 100644
--- a/drivers/scsi/lasi700.c
+++ b/drivers/scsi/lasi700.c
@@ -81,7 +81,7 @@ MODULE_LICENSE("GPL");
 #define LASI710_CLOCK	40
 #define LASI_SCSI_CORE_OFFSET 0x100
 
-static struct parisc_device_id lasi700_ids[] = {
+static const struct parisc_device_id lasi700_ids[] __initconst = {
 	LASI700_ID_TABLE,
 	LASI710_ID_TABLE,
 	{ 0 }
@@ -164,11 +164,11 @@ lasi700_driver_remove(struct parisc_device *dev)
 	return 0;
 }
 
-static struct parisc_driver lasi700_driver = {
+static struct parisc_driver lasi700_driver __refdata = {
 	.name =		"lasi_scsi",
 	.id_table =	lasi700_ids,
 	.probe =	lasi700_probe,
-	.remove =	lasi700_driver_remove,
+	.remove =	__exit_p(lasi700_driver_remove),
 };
 
 static int __init
diff --git a/drivers/scsi/libfc/fc_fcp.c b/drivers/scsi/libfc/fc_fcp.c
index 234352da5c3c..772c35a5c49e 100644
--- a/drivers/scsi/libfc/fc_fcp.c
+++ b/drivers/scsi/libfc/fc_fcp.c
@@ -2222,8 +2222,6 @@ int fc_eh_host_reset(struct scsi_cmnd *sc_cmd)
 
 	FC_SCSI_DBG(lport, "Resetting host\n");
 
-	fc_block_scsi_eh(sc_cmd);
-
 	fc_lport_reset(lport);
 	wait_tmo = jiffies + FC_HOST_RESET_TIMEOUT;
 	while (!fc_fcp_lport_queue_ready(lport) && time_before(jiffies,
diff --git a/drivers/scsi/libiscsi.c b/drivers/scsi/libiscsi.c
index 42381adf0769..bd4605a34f54 100644
--- a/drivers/scsi/libiscsi.c
+++ b/drivers/scsi/libiscsi.c
@@ -1078,7 +1078,7 @@ static int iscsi_handle_reject(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
 		if (opcode != ISCSI_OP_NOOP_OUT)
 			return 0;
 
-		 if (rejected_pdu.itt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
+		if (rejected_pdu.itt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
 			/*
 			 * nop-out in response to target's nop-out rejected.
 			 * Just resend.
diff --git a/drivers/scsi/libsas/Kconfig b/drivers/scsi/libsas/Kconfig
index 9dafe64e7c7a..13739bfacc67 100644
--- a/drivers/scsi/libsas/Kconfig
+++ b/drivers/scsi/libsas/Kconfig
@@ -26,6 +26,7 @@ config SCSI_SAS_LIBSAS
 	tristate "SAS Domain Transport Attributes"
 	depends on SCSI
 	select SCSI_SAS_ATTRS
+	select BLK_DEV_BSGLIB
 	help
 	  This provides transport specific helpers for SAS drivers which
 	  use the domain device construct (like the aic94xxx).
diff --git a/drivers/scsi/libsas/sas_ata.c b/drivers/scsi/libsas/sas_ata.c
index 87f5e694dbed..70be4425ae0b 100644
--- a/drivers/scsi/libsas/sas_ata.c
+++ b/drivers/scsi/libsas/sas_ata.c
@@ -343,6 +343,7 @@ static int smp_ata_check_ready(struct ata_link *link)
 	case SAS_END_DEVICE:
 		if (ex_phy->attached_sata_dev)
 			return sas_ata_clear_pending(dev, ex_phy);
+		/* fall through */
 	default:
 		return -ENODEV;
 	}
diff --git a/drivers/scsi/libsas/sas_expander.c b/drivers/scsi/libsas/sas_expander.c
index 570b2cb2da43..6b4fd2375178 100644
--- a/drivers/scsi/libsas/sas_expander.c
+++ b/drivers/scsi/libsas/sas_expander.c
@@ -64,8 +64,8 @@ static void smp_task_done(struct sas_task *task)
 /* Give it some long enough timeout. In seconds. */
 #define SMP_TIMEOUT 10
 
-static int smp_execute_task(struct domain_device *dev, void *req, int req_size,
-			    void *resp, int resp_size)
+static int smp_execute_task_sg(struct domain_device *dev,
+		struct scatterlist *req, struct scatterlist *resp)
 {
 	int res, retry;
 	struct sas_task *task = NULL;
@@ -86,8 +86,8 @@ static int smp_execute_task(struct domain_device *dev, void *req, int req_size,
 		}
 		task->dev = dev;
 		task->task_proto = dev->tproto;
-		sg_init_one(&task->smp_task.smp_req, req, req_size);
-		sg_init_one(&task->smp_task.smp_resp, resp, resp_size);
+		task->smp_task.smp_req = *req;
+		task->smp_task.smp_resp = *resp;
 
 		task->task_done = smp_task_done;
 
@@ -151,6 +151,17 @@ static int smp_execute_task(struct domain_device *dev, void *req, int req_size,
 	return res;
 }
 
+static int smp_execute_task(struct domain_device *dev, void *req, int req_size,
+			    void *resp, int resp_size)
+{
+	struct scatterlist req_sg;
+	struct scatterlist resp_sg;
+
+	sg_init_one(&req_sg, req, req_size);
+	sg_init_one(&resp_sg, resp, resp_size);
+	return smp_execute_task_sg(dev, &req_sg, &resp_sg);
+}
+
 /* ---------- Allocations ---------- */
 
 static inline void *alloc_smp_req(int size)
@@ -2130,57 +2141,50 @@ int sas_ex_revalidate_domain(struct domain_device *port_dev)
 	return res;
 }
 
-int sas_smp_handler(struct Scsi_Host *shost, struct sas_rphy *rphy,
-		    struct request *req)
+void sas_smp_handler(struct bsg_job *job, struct Scsi_Host *shost,
+		struct sas_rphy *rphy)
 {
 	struct domain_device *dev;
-	int ret, type;
-	struct request *rsp = req->next_rq;
-
-	if (!rsp) {
-		printk("%s: space for a smp response is missing\n",
-		       __func__);
-		return -EINVAL;
-	}
+	unsigned int reslen = 0;
+	int ret = -EINVAL;
 
 	/* no rphy means no smp target support (ie aic94xx host) */
 	if (!rphy)
-		return sas_smp_host_handler(shost, req, rsp);
-
-	type = rphy->identify.device_type;
+		return sas_smp_host_handler(job, shost);
 
-	if (type != SAS_EDGE_EXPANDER_DEVICE &&
-	    type != SAS_FANOUT_EXPANDER_DEVICE) {
+	switch (rphy->identify.device_type) {
+	case SAS_EDGE_EXPANDER_DEVICE:
+	case SAS_FANOUT_EXPANDER_DEVICE:
+		break;
+	default:
 		printk("%s: can we send a smp request to a device?\n",
 		       __func__);
-		return -EINVAL;
+		goto out;
 	}
 
 	dev = sas_find_dev_by_rphy(rphy);
 	if (!dev) {
 		printk("%s: fail to find a domain_device?\n", __func__);
-		return -EINVAL;
+		goto out;
 	}
 
 	/* do we need to support multiple segments? */
-	if (bio_multiple_segments(req->bio) ||
-	    bio_multiple_segments(rsp->bio)) {
+	if (job->request_payload.sg_cnt > 1 ||
+	    job->reply_payload.sg_cnt > 1) {
 		printk("%s: multiple segments req %u, rsp %u\n",
-		       __func__, blk_rq_bytes(req), blk_rq_bytes(rsp));
-		return -EINVAL;
+		       __func__, job->request_payload.payload_len,
+		       job->reply_payload.payload_len);
+		goto out;
 	}
 
-	ret = smp_execute_task(dev, bio_data(req->bio), blk_rq_bytes(req),
-			       bio_data(rsp->bio), blk_rq_bytes(rsp));
+	ret = smp_execute_task_sg(dev, job->request_payload.sg_list,
+			job->reply_payload.sg_list);
 	if (ret > 0) {
 		/* positive number is the untransferred residual */
-		scsi_req(rsp)->resid_len = ret;
-		scsi_req(req)->resid_len = 0;
+		reslen = ret;
 		ret = 0;
-	} else if (ret == 0) {
-		scsi_req(rsp)->resid_len = 0;
-		scsi_req(req)->resid_len = 0;
 	}
 
-	return ret;
+out:
+	bsg_job_done(job, ret, reslen);
 }
diff --git a/drivers/scsi/libsas/sas_host_smp.c b/drivers/scsi/libsas/sas_host_smp.c
index 45cbbc44f4d7..9ead93df3a6e 100644
--- a/drivers/scsi/libsas/sas_host_smp.c
+++ b/drivers/scsi/libsas/sas_host_smp.c
@@ -225,47 +225,36 @@ static void sas_phy_control(struct sas_ha_struct *sas_ha, u8 phy_id,
 		resp_data[2] = SMP_RESP_FUNC_ACC;
 }
 
-int sas_smp_host_handler(struct Scsi_Host *shost, struct request *req,
-			 struct request *rsp)
+void sas_smp_host_handler(struct bsg_job *job, struct Scsi_Host *shost)
 {
-	u8 *req_data = NULL, *resp_data = NULL, *buf;
 	struct sas_ha_struct *sas_ha = SHOST_TO_SAS_HA(shost);
+	u8 *req_data, *resp_data;
+	unsigned int reslen = 0;
 	int error = -EINVAL;
 
 	/* eight is the minimum size for request and response frames */
-	if (blk_rq_bytes(req) < 8 || blk_rq_bytes(rsp) < 8)
+	if (job->request_payload.payload_len < 8 ||
+	    job->reply_payload.payload_len < 8)
 		goto out;
 
-	if (bio_offset(req->bio) + blk_rq_bytes(req) > PAGE_SIZE ||
-	    bio_offset(rsp->bio) + blk_rq_bytes(rsp) > PAGE_SIZE) {
-		shost_printk(KERN_ERR, shost,
-			"SMP request/response frame crosses page boundary");
+	error = -ENOMEM;
+	req_data = kzalloc(job->request_payload.payload_len, GFP_KERNEL);
+	if (!req_data)
 		goto out;
-	}
-
-	req_data = kzalloc(blk_rq_bytes(req), GFP_KERNEL);
+	sg_copy_to_buffer(job->request_payload.sg_list,
+			  job->request_payload.sg_cnt, req_data,
+			  job->request_payload.payload_len);
 
 	/* make sure frame can always be built ... we copy
 	 * back only the requested length */
-	resp_data = kzalloc(max(blk_rq_bytes(rsp), 128U), GFP_KERNEL);
-
-	if (!req_data || !resp_data) {
-		error = -ENOMEM;
-		goto out;
-	}
-
-	local_irq_disable();
-	buf = kmap_atomic(bio_page(req->bio));
-	memcpy(req_data, buf, blk_rq_bytes(req));
-	kunmap_atomic(buf - bio_offset(req->bio));
-	local_irq_enable();
+	resp_data = kzalloc(max(job->reply_payload.payload_len, 128U),
+			GFP_KERNEL);
+	if (!resp_data)
+		goto out_free_req;
 
+	error = -EINVAL;
 	if (req_data[0] != SMP_REQUEST)
-		goto out;
-
-	/* always succeeds ... even if we can't process the request
-	 * the result is in the response frame */
-	error = 0;
+		goto out_free_resp;
 
 	/* set up default don't know response */
 	resp_data[0] = SMP_RESPONSE;
@@ -274,20 +263,18 @@ int sas_smp_host_handler(struct Scsi_Host *shost, struct request *req,
 
 	switch (req_data[1]) {
 	case SMP_REPORT_GENERAL:
-		scsi_req(req)->resid_len -= 8;
-		scsi_req(rsp)->resid_len -= 32;
 		resp_data[2] = SMP_RESP_FUNC_ACC;
 		resp_data[9] = sas_ha->num_phys;
+		reslen = 32;
 		break;
 
 	case SMP_REPORT_MANUF_INFO:
-		scsi_req(req)->resid_len -= 8;
-		scsi_req(rsp)->resid_len -= 64;
 		resp_data[2] = SMP_RESP_FUNC_ACC;
 		memcpy(resp_data + 12, shost->hostt->name,
 		       SAS_EXPANDER_VENDOR_ID_LEN);
 		memcpy(resp_data + 20, "libsas virt phy",
 		       SAS_EXPANDER_PRODUCT_ID_LEN);
+		reslen = 64;
 		break;
 
 	case SMP_READ_GPIO_REG:
@@ -295,14 +282,10 @@ int sas_smp_host_handler(struct Scsi_Host *shost, struct request *req,
 		break;
 
 	case SMP_DISCOVER:
-		scsi_req(req)->resid_len -= 16;
-		if ((int)scsi_req(req)->resid_len < 0) {
-			scsi_req(req)->resid_len = 0;
-			error = -EINVAL;
-			goto out;
-		}
-		scsi_req(rsp)->resid_len -= 56;
+		if (job->request_payload.payload_len < 16)
+			goto out_free_resp;
 		sas_host_smp_discover(sas_ha, resp_data, req_data[9]);
+		reslen = 56;
 		break;
 
 	case SMP_REPORT_PHY_ERR_LOG:
@@ -311,14 +294,10 @@ int sas_smp_host_handler(struct Scsi_Host *shost, struct request *req,
 		break;
 
 	case SMP_REPORT_PHY_SATA:
-		scsi_req(req)->resid_len -= 16;
-		if ((int)scsi_req(req)->resid_len < 0) {
-			scsi_req(req)->resid_len = 0;
-			error = -EINVAL;
-			goto out;
-		}
-		scsi_req(rsp)->resid_len -= 60;
+		if (job->request_payload.payload_len < 16)
+			goto out_free_resp;
 		sas_report_phy_sata(sas_ha, resp_data, req_data[9]);
+		reslen = 60;
 		break;
 
 	case SMP_REPORT_ROUTE_INFO:
@@ -330,16 +309,15 @@ int sas_smp_host_handler(struct Scsi_Host *shost, struct request *req,
 		const int base_frame_size = 11;
 		int to_write = req_data[4];
 
-		if (blk_rq_bytes(req) < base_frame_size + to_write * 4 ||
-		    scsi_req(req)->resid_len < base_frame_size + to_write * 4) {
+		if (job->request_payload.payload_len <
+				base_frame_size + to_write * 4) {
 			resp_data[2] = SMP_RESP_INV_FRM_LEN;
 			break;
 		}
 
 		to_write = sas_host_smp_write_gpio(sas_ha, resp_data, req_data[2],
 						   req_data[3], to_write, &req_data[8]);
-		scsi_req(req)->resid_len -= base_frame_size + to_write * 4;
-		scsi_req(rsp)->resid_len -= 8;
+		reslen = 8;
 		break;
 	}
 
@@ -348,16 +326,12 @@ int sas_smp_host_handler(struct Scsi_Host *shost, struct request *req,
 		break;
 
 	case SMP_PHY_CONTROL:
-		scsi_req(req)->resid_len -= 44;
-		if ((int)scsi_req(req)->resid_len < 0) {
-			scsi_req(req)->resid_len = 0;
-			error = -EINVAL;
-			goto out;
-		}
-		scsi_req(rsp)->resid_len -= 8;
+		if (job->request_payload.payload_len < 44)
+			goto out_free_resp;
 		sas_phy_control(sas_ha, req_data[9], req_data[10],
 				req_data[32] >> 4, req_data[33] >> 4,
 				resp_data);
+		reslen = 8;
 		break;
 
 	case SMP_PHY_TEST_FUNCTION:
@@ -369,15 +343,15 @@ int sas_smp_host_handler(struct Scsi_Host *shost, struct request *req,
 		break;
 	}
 
-	local_irq_disable();
-	buf = kmap_atomic(bio_page(rsp->bio));
-	memcpy(buf, resp_data, blk_rq_bytes(rsp));
-	flush_kernel_dcache_page(bio_page(rsp->bio));
-	kunmap_atomic(buf - bio_offset(rsp->bio));
-	local_irq_enable();
+	sg_copy_from_buffer(job->reply_payload.sg_list,
+			    job->reply_payload.sg_cnt, resp_data,
+			    job->reply_payload.payload_len);
 
- out:
-	kfree(req_data);
+	error = 0;
+out_free_resp:
 	kfree(resp_data);
-	return error;
+out_free_req:
+	kfree(req_data);
+out:
+	bsg_job_done(job, error, reslen);
 }
diff --git a/drivers/scsi/libsas/sas_internal.h b/drivers/scsi/libsas/sas_internal.h
index a216c957b639..c07e08136491 100644
--- a/drivers/scsi/libsas/sas_internal.h
+++ b/drivers/scsi/libsas/sas_internal.h
@@ -81,6 +81,8 @@ int sas_queue_work(struct sas_ha_struct *ha, struct sas_work *sw);
 int sas_notify_lldd_dev_found(struct domain_device *);
 void sas_notify_lldd_dev_gone(struct domain_device *);
 
+void sas_smp_handler(struct bsg_job *job, struct Scsi_Host *shost,
+		struct sas_rphy *rphy);
 int sas_smp_phy_control(struct domain_device *dev, int phy_id,
 			enum phy_func phy_func, struct sas_phy_linkrates *);
 int sas_smp_get_phy_events(struct sas_phy *phy);
@@ -98,16 +100,14 @@ void sas_hae_reset(struct work_struct *work);
 void sas_free_device(struct kref *kref);
 
 #ifdef CONFIG_SCSI_SAS_HOST_SMP
-extern int sas_smp_host_handler(struct Scsi_Host *shost, struct request *req,
-				struct request *rsp);
+extern void sas_smp_host_handler(struct bsg_job *job, struct Scsi_Host *shost);
 #else
-static inline int sas_smp_host_handler(struct Scsi_Host *shost,
-				       struct request *req,
-				       struct request *rsp)
+static inline void sas_smp_host_handler(struct bsg_job *job,
+		struct Scsi_Host *shost)
 {
 	shost_printk(KERN_ERR, shost,
 		"Cannot send SMP to a sas host (not enabled in CONFIG)\n");
-	return -EINVAL;
+	bsg_job_done(job, -EINVAL, 0);
 }
 #endif
 
diff --git a/drivers/scsi/libsas/sas_scsi_host.c b/drivers/scsi/libsas/sas_scsi_host.c
index 87e5079d816b..ea8ad06ff582 100644
--- a/drivers/scsi/libsas/sas_scsi_host.c
+++ b/drivers/scsi/libsas/sas_scsi_host.c
@@ -526,7 +526,7 @@ int sas_eh_device_reset_handler(struct scsi_cmnd *cmd)
 	return FAILED;
 }
 
-int sas_eh_bus_reset_handler(struct scsi_cmnd *cmd)
+int sas_eh_target_reset_handler(struct scsi_cmnd *cmd)
 {
 	int res;
 	struct Scsi_Host *host = cmd->device->host;
@@ -554,15 +554,15 @@ static int try_to_reset_cmd_device(struct scsi_cmnd *cmd)
 	struct Scsi_Host *shost = cmd->device->host;
 
 	if (!shost->hostt->eh_device_reset_handler)
-		goto try_bus_reset;
+		goto try_target_reset;
 
 	res = shost->hostt->eh_device_reset_handler(cmd);
 	if (res == SUCCESS)
 		return res;
 
-try_bus_reset:
-	if (shost->hostt->eh_bus_reset_handler)
-		return shost->hostt->eh_bus_reset_handler(cmd);
+try_target_reset:
+	if (shost->hostt->eh_target_reset_handler)
+		return shost->hostt->eh_target_reset_handler(cmd);
 
 	return FAILED;
 }
@@ -855,7 +855,6 @@ int sas_target_alloc(struct scsi_target *starget)
 int sas_slave_configure(struct scsi_device *scsi_dev)
 {
 	struct domain_device *dev = sdev_to_domain_dev(scsi_dev);
-	struct sas_ha_struct *sas_ha;
 
 	BUG_ON(dev->rphy->identify.device_type != SAS_END_DEVICE);
 
@@ -864,8 +863,6 @@ int sas_slave_configure(struct scsi_device *scsi_dev)
 		return 0;
 	}
 
-	sas_ha = dev->port->ha;
-
 	sas_read_port_mode_page(scsi_dev);
 
 	if (scsi_dev->tagged_supported) {
@@ -996,6 +993,6 @@ EXPORT_SYMBOL_GPL(sas_bios_param);
 EXPORT_SYMBOL_GPL(sas_task_abort);
 EXPORT_SYMBOL_GPL(sas_phy_reset);
 EXPORT_SYMBOL_GPL(sas_eh_device_reset_handler);
-EXPORT_SYMBOL_GPL(sas_eh_bus_reset_handler);
+EXPORT_SYMBOL_GPL(sas_eh_target_reset_handler);
 EXPORT_SYMBOL_GPL(sas_target_destroy);
 EXPORT_SYMBOL_GPL(sas_ioctl);
diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h
index 562dc0139735..8eb3f96fe068 100644
--- a/drivers/scsi/lpfc/lpfc.h
+++ b/drivers/scsi/lpfc/lpfc.h
@@ -733,7 +733,6 @@ struct lpfc_hba {
 	uint32_t fc_rttov;	/* R_T_TOV timer value */
 	uint32_t fc_altov;	/* AL_TOV timer value */
 	uint32_t fc_crtov;	/* C_R_TOV timer value */
-	uint32_t fc_citov;	/* C_I_TOV timer value */
 
 	struct serv_parm fc_fabparam;	/* fabric service parameters buffer */
 	uint8_t alpa_map[128];	/* AL_PA map from READ_LA */
@@ -757,6 +756,7 @@ struct lpfc_hba {
 #define LPFC_NVMET_MAX_PORTS	32
 	uint8_t  mds_diags_support;
 	uint32_t initial_imax;
+	uint8_t  bbcredit_support;
 
 	/* HBA Config Parameters */
 	uint32_t cfg_ack0;
@@ -836,6 +836,7 @@ struct lpfc_hba {
 	uint32_t cfg_enable_SmartSAN;
 	uint32_t cfg_enable_mds_diags;
 	uint32_t cfg_enable_fc4_type;
+	uint32_t cfg_enable_bbcr;	/*Enable BB Credit Recovery*/
 	uint32_t cfg_xri_split;
 #define LPFC_ENABLE_FCP  1
 #define LPFC_ENABLE_NVME 2
@@ -946,14 +947,14 @@ struct lpfc_hba {
 	struct list_head active_rrq_list;
 	spinlock_t hbalock;
 
-	/* pci_mem_pools */
-	struct pci_pool *lpfc_sg_dma_buf_pool;
-	struct pci_pool *lpfc_mbuf_pool;
-	struct pci_pool *lpfc_hrb_pool;	/* header receive buffer pool */
-	struct pci_pool *lpfc_drb_pool; /* data receive buffer pool */
-	struct pci_pool *lpfc_nvmet_drb_pool; /* data receive buffer pool */
-	struct pci_pool *lpfc_hbq_pool;	/* SLI3 hbq buffer pool */
-	struct pci_pool *txrdy_payload_pool;
+	/* dma_mem_pools */
+	struct dma_pool *lpfc_sg_dma_buf_pool;
+	struct dma_pool *lpfc_mbuf_pool;
+	struct dma_pool *lpfc_hrb_pool;	/* header receive buffer pool */
+	struct dma_pool *lpfc_drb_pool; /* data receive buffer pool */
+	struct dma_pool *lpfc_nvmet_drb_pool; /* data receive buffer pool */
+	struct dma_pool *lpfc_hbq_pool;	/* SLI3 hbq buffer pool */
+	struct dma_pool *txrdy_payload_pool;
 	struct lpfc_dma_pool lpfc_mbuf_safety_pool;
 
 	mempool_t *mbox_mem_pool;
diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c
index 7ee1a94c0b33..c17677f494af 100644
--- a/drivers/scsi/lpfc/lpfc_attr.c
+++ b/drivers/scsi/lpfc/lpfc_attr.c
@@ -247,13 +247,10 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr,
 				atomic_read(&tgtp->xmt_abort_rsp),
 				atomic_read(&tgtp->xmt_abort_rsp_error));
 
-		spin_lock(&phba->sli4_hba.nvmet_ctx_get_lock);
-		spin_lock(&phba->sli4_hba.nvmet_ctx_put_lock);
-		tot = phba->sli4_hba.nvmet_xri_cnt -
-			(phba->sli4_hba.nvmet_ctx_get_cnt +
-			phba->sli4_hba.nvmet_ctx_put_cnt);
-		spin_unlock(&phba->sli4_hba.nvmet_ctx_put_lock);
-		spin_unlock(&phba->sli4_hba.nvmet_ctx_get_lock);
+		/* Calculate outstanding IOs */
+		tot = atomic_read(&tgtp->rcv_fcp_cmd_drop);
+		tot += atomic_read(&tgtp->xmt_fcp_release);
+		tot = atomic_read(&tgtp->rcv_fcp_cmd_in) - tot;
 
 		len += snprintf(buf + len, PAGE_SIZE - len,
 				"IO_CTX: %08x  WAIT: cur %08x tot %08x\n"
@@ -1894,6 +1891,36 @@ static inline bool lpfc_rangecheck(uint val, uint min, uint max)
 }
 
 /**
+ * lpfc_enable_bbcr_set: Sets an attribute value.
+ * @phba: pointer the the adapter structure.
+ * @val: integer attribute value.
+ *
+ * Description:
+ * Validates the min and max values then sets the
+ * adapter config field if in the valid range. prints error message
+ * and does not set the parameter if invalid.
+ *
+ * Returns:
+ * zero on success
+ * -EINVAL if val is invalid
+ */
+static ssize_t
+lpfc_enable_bbcr_set(struct lpfc_hba *phba, uint val)
+{
+	if (lpfc_rangecheck(val, 0, 1) && phba->sli_rev == LPFC_SLI_REV4) {
+		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+				"3068 %s_enable_bbcr changed from %d to %d\n",
+				LPFC_DRIVER_NAME, phba->cfg_enable_bbcr, val);
+		phba->cfg_enable_bbcr = val;
+		return 0;
+	}
+	lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+			"0451 %s_enable_bbcr cannot set to %d, range is 0, 1\n",
+			LPFC_DRIVER_NAME, val);
+	return -EINVAL;
+}
+
+/**
  * lpfc_param_show - Return a cfg attribute value in decimal
  *
  * Description:
@@ -5116,6 +5143,14 @@ LPFC_ATTR_R(sg_seg_cnt, LPFC_DEFAULT_SG_SEG_CNT, LPFC_DEFAULT_SG_SEG_CNT,
  */
 LPFC_ATTR_R(enable_mds_diags, 0, 0, 1, "Enable MDS Diagnostics");
 
+/*
+ * lpfc_enable_bbcr: Enable BB Credit Recovery
+ *       0  = BB Credit Recovery disabled
+ *       1  = BB Credit Recovery enabled (default)
+ * Value range is [0,1]. Default value is 1.
+ */
+LPFC_BBCR_ATTR_RW(enable_bbcr, 1, 0, 1, "Enable BBC Recovery");
+
 struct device_attribute *lpfc_hba_attrs[] = {
 	&dev_attr_nvme_info,
 	&dev_attr_bg_info,
@@ -5223,6 +5258,7 @@ struct device_attribute *lpfc_hba_attrs[] = {
 	&dev_attr_protocol,
 	&dev_attr_lpfc_xlane_supported,
 	&dev_attr_lpfc_enable_mds_diags,
+	&dev_attr_lpfc_enable_bbcr,
 	NULL,
 };
 
@@ -6234,11 +6270,13 @@ lpfc_get_cfgparam(struct lpfc_hba *phba)
 	lpfc_nvmet_fb_size_init(phba, lpfc_nvmet_fb_size);
 	lpfc_fcp_io_channel_init(phba, lpfc_fcp_io_channel);
 	lpfc_nvme_io_channel_init(phba, lpfc_nvme_io_channel);
+	lpfc_enable_bbcr_init(phba, lpfc_enable_bbcr);
 
 	if (phba->sli_rev != LPFC_SLI_REV4) {
 		/* NVME only supported on SLI4 */
 		phba->nvmet_support = 0;
 		phba->cfg_enable_fc4_type = LPFC_ENABLE_FCP;
+		phba->cfg_enable_bbcr = 0;
 	} else {
 		/* We MUST have FCP support */
 		if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP))
diff --git a/drivers/scsi/lpfc/lpfc_attr.h b/drivers/scsi/lpfc/lpfc_attr.h
index d56dafcdd563..931db52692f5 100644
--- a/drivers/scsi/lpfc/lpfc_attr.h
+++ b/drivers/scsi/lpfc/lpfc_attr.h
@@ -46,6 +46,16 @@ lpfc_param_store(name)\
 static DEVICE_ATTR(lpfc_##name, S_IRUGO | S_IWUSR,\
 		   lpfc_##name##_show, lpfc_##name##_store)
 
+#define LPFC_BBCR_ATTR_RW(name, defval, minval, maxval, desc) \
+static uint lpfc_##name = defval;\
+module_param(lpfc_##name, uint, 0444);\
+MODULE_PARM_DESC(lpfc_##name, desc);\
+lpfc_param_show(name)\
+lpfc_param_init(name, defval, minval, maxval)\
+lpfc_param_store(name)\
+static DEVICE_ATTR(lpfc_##name, 0444 | 0644,\
+		   lpfc_##name##_show, lpfc_##name##_store)
+
 #define LPFC_ATTR_HEX_R(name, defval, minval, maxval, desc) \
 static uint lpfc_##name = defval;\
 module_param(lpfc_##name, uint, S_IRUGO);\
diff --git a/drivers/scsi/lpfc/lpfc_bsg.c b/drivers/scsi/lpfc/lpfc_bsg.c
index a1686c2d863c..fe9e1c079c20 100644
--- a/drivers/scsi/lpfc/lpfc_bsg.c
+++ b/drivers/scsi/lpfc/lpfc_bsg.c
@@ -2384,20 +2384,17 @@ lpfc_sli4_bsg_link_diag_test(struct bsg_job *job)
 		goto job_error;
 
 	pmboxq = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
-	if (!pmboxq) {
-		rc = -ENOMEM;
+	if (!pmboxq)
 		goto link_diag_test_exit;
-	}
 
 	req_len = (sizeof(struct lpfc_mbx_set_link_diag_state) -
 		   sizeof(struct lpfc_sli4_cfg_mhdr));
 	alloc_len = lpfc_sli4_config(phba, pmboxq, LPFC_MBOX_SUBSYSTEM_FCOE,
 				     LPFC_MBOX_OPCODE_FCOE_LINK_DIAG_STATE,
 				     req_len, LPFC_SLI4_MBX_EMBED);
-	if (alloc_len != req_len) {
-		rc = -ENOMEM;
+	if (alloc_len != req_len)
 		goto link_diag_test_exit;
-	}
+
 	run_link_diag_test = &pmboxq->u.mqe.un.link_diag_test;
 	bf_set(lpfc_mbx_run_diag_test_link_num, &run_link_diag_test->u.req,
 	       phba->sli4_hba.lnk_info.lnk_no);
diff --git a/drivers/scsi/lpfc/lpfc_crtn.h b/drivers/scsi/lpfc/lpfc_crtn.h
index da669dce12fe..7e300734b345 100644
--- a/drivers/scsi/lpfc/lpfc_crtn.h
+++ b/drivers/scsi/lpfc/lpfc_crtn.h
@@ -556,9 +556,8 @@ int lpfc_nvmet_update_targetport(struct lpfc_hba *phba);
 void lpfc_nvmet_destroy_targetport(struct lpfc_hba *phba);
 void lpfc_nvmet_unsol_ls_event(struct lpfc_hba *phba,
 			struct lpfc_sli_ring *pring, struct lpfc_iocbq *piocb);
-void lpfc_nvmet_unsol_fcp_event(struct lpfc_hba *phba,
-			struct lpfc_sli_ring *pring,
-			struct rqb_dmabuf *nvmebuf, uint64_t isr_ts);
+void lpfc_nvmet_unsol_fcp_event(struct lpfc_hba *phba, uint32_t idx,
+				struct rqb_dmabuf *nvmebuf, uint64_t isr_ts);
 void lpfc_nvme_mod_param_dep(struct lpfc_hba *phba);
 void lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba,
 				struct lpfc_iocbq *cmdiocb,
diff --git a/drivers/scsi/lpfc/lpfc_ct.c b/drivers/scsi/lpfc/lpfc_ct.c
index 9c0c1463057d..33417681f5d4 100644
--- a/drivers/scsi/lpfc/lpfc_ct.c
+++ b/drivers/scsi/lpfc/lpfc_ct.c
@@ -955,7 +955,7 @@ lpfc_cmpl_ct_cmd_gft_id(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 		CTrsp = (struct lpfc_sli_ct_request *)outp->virt;
 		fc4_data_0 = be32_to_cpu(CTrsp->un.gft_acc.fc4_types[0]);
 		fc4_data_1 = be32_to_cpu(CTrsp->un.gft_acc.fc4_types[1]);
-		lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+		lpfc_printf_vlog(vport, KERN_INFO, LOG_DISCOVERY,
 				 "3062 DID x%06x GFT Wd0 x%08x Wd1 x%08x\n",
 				 did, fc4_data_0, fc4_data_1);
 
@@ -969,7 +969,7 @@ lpfc_cmpl_ct_cmd_gft_id(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 				ndlp->nlp_fc4_type |= NLP_FC4_FCP;
 			if (fc4_data_1 &  LPFC_FC4_TYPE_BITMASK)
 				ndlp->nlp_fc4_type |= NLP_FC4_NVME;
-			lpfc_printf_vlog(vport, KERN_ERR, LOG_DISCOVERY,
+			lpfc_printf_vlog(vport, KERN_INFO, LOG_DISCOVERY,
 					 "3064 Setting ndlp %p, DID x%06x with "
 					 "FC4 x%08x, Data: x%08x x%08x\n",
 					 ndlp, did, ndlp->nlp_fc4_type,
diff --git a/drivers/scsi/lpfc/lpfc_debugfs.c b/drivers/scsi/lpfc/lpfc_debugfs.c
index 744f3f395b64..d50c481ec41c 100644
--- a/drivers/scsi/lpfc/lpfc_debugfs.c
+++ b/drivers/scsi/lpfc/lpfc_debugfs.c
@@ -851,13 +851,10 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size)
 			spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock);
 		}
 
-		spin_lock(&phba->sli4_hba.nvmet_ctx_get_lock);
-		spin_lock(&phba->sli4_hba.nvmet_ctx_put_lock);
-		tot = phba->sli4_hba.nvmet_xri_cnt -
-			(phba->sli4_hba.nvmet_ctx_get_cnt +
-			phba->sli4_hba.nvmet_ctx_put_cnt);
-		spin_unlock(&phba->sli4_hba.nvmet_ctx_put_lock);
-		spin_unlock(&phba->sli4_hba.nvmet_ctx_get_lock);
+		/* Calculate outstanding IOs */
+		tot = atomic_read(&tgtp->rcv_fcp_cmd_drop);
+		tot += atomic_read(&tgtp->xmt_fcp_release);
+		tot = atomic_read(&tgtp->rcv_fcp_cmd_in) - tot;
 
 		len += snprintf(buf + len, size - len,
 				"IO_CTX: %08x  WAIT: cur %08x tot %08x\n"
diff --git a/drivers/scsi/lpfc/lpfc_debugfs.h b/drivers/scsi/lpfc/lpfc_debugfs.h
index 7b7d314af0e0..c4edd87bfc65 100644
--- a/drivers/scsi/lpfc/lpfc_debugfs.h
+++ b/drivers/scsi/lpfc/lpfc_debugfs.h
@@ -478,16 +478,16 @@ lpfc_debug_dump_cq(struct lpfc_hba *phba, int qtype, int wqidx)
 		return;
 
 	for (eqidx = 0; eqidx < phba->io_channel_irqs; eqidx++) {
-		eq = phba->sli4_hba.hba_eq[eqidx];
-		if (cq->assoc_qid == eq->queue_id)
+		if (cq->assoc_qid == phba->sli4_hba.hba_eq[eqidx]->queue_id)
 			break;
 	}
 	if (eqidx == phba->io_channel_irqs) {
 		pr_err("Couldn't find EQ for CQ. Using EQ[0]\n");
 		eqidx = 0;
-		eq = phba->sli4_hba.hba_eq[0];
 	}
 
+	eq = phba->sli4_hba.hba_eq[eqidx];
+
 	if (qtype == DUMP_FCP || qtype == DUMP_NVME)
 		pr_err("%s CQ: WQ[Idx:%d|Qid%d]->CQ[Idx%d|Qid%d]"
 			"->EQ[Idx:%d|Qid:%d]:\n",
diff --git a/drivers/scsi/lpfc/lpfc_disc.h b/drivers/scsi/lpfc/lpfc_disc.h
index 094c97b9e5f7..f9a566eaef04 100644
--- a/drivers/scsi/lpfc/lpfc_disc.h
+++ b/drivers/scsi/lpfc/lpfc_disc.h
@@ -159,6 +159,7 @@ struct lpfc_node_rrq {
 #define NLP_RNID_SND       0x00000400	/* sent RNID request for this entry */
 #define NLP_ELS_SND_MASK   0x000007e0	/* sent ELS request for this entry */
 #define NLP_NVMET_RECOV    0x00001000   /* NVMET auditing node for recovery. */
+#define NLP_FCP_PRLI_RJT   0x00002000   /* Rport does not support FCP PRLI. */
 #define NLP_DEFER_RM       0x00010000	/* Remove this ndlp if no longer used */
 #define NLP_DELAY_TMO      0x00020000	/* delay timeout is running for node */
 #define NLP_NPR_2B_DISC    0x00040000	/* node is included in num_disc_nodes */
diff --git a/drivers/scsi/lpfc/lpfc_els.c b/drivers/scsi/lpfc/lpfc_els.c
index 6d1d6f691df4..468a66371de9 100644
--- a/drivers/scsi/lpfc/lpfc_els.c
+++ b/drivers/scsi/lpfc/lpfc_els.c
@@ -1527,6 +1527,7 @@ lpfc_plogi_confirm_nport(struct lpfc_hba *phba, uint32_t *prsp,
 	uint8_t  name[sizeof(struct lpfc_name)];
 	uint32_t rc, keepDID = 0, keep_nlp_flag = 0;
 	uint16_t keep_nlp_state;
+	struct lpfc_nvme_rport *keep_nrport = NULL;
 	int  put_node;
 	int  put_rport;
 	unsigned long *active_rrqs_xri_bitmap = NULL;
@@ -1624,6 +1625,10 @@ lpfc_plogi_confirm_nport(struct lpfc_hba *phba, uint32_t *prsp,
 	keep_nlp_state = new_ndlp->nlp_state;
 	lpfc_nlp_set_state(vport, new_ndlp, ndlp->nlp_state);
 
+	/* interchange the nvme remoteport structs */
+	keep_nrport = new_ndlp->nrport;
+	new_ndlp->nrport = ndlp->nrport;
+
 	/* Move this back to NPR state */
 	if (memcmp(&ndlp->nlp_portname, name, sizeof(struct lpfc_name)) == 0) {
 		/* The new_ndlp is replacing ndlp totally, so we need
@@ -1646,6 +1651,13 @@ lpfc_plogi_confirm_nport(struct lpfc_hba *phba, uint32_t *prsp,
 			}
 			new_ndlp->nlp_type = ndlp->nlp_type;
 		}
+
+		/* Fix up the nvme rport */
+		if (ndlp->nrport) {
+			ndlp->nrport = NULL;
+			lpfc_nlp_put(ndlp);
+		}
+
 		/* We shall actually free the ndlp with both nlp_DID and
 		 * nlp_portname fields equals 0 to avoid any ndlp on the
 		 * nodelist never to be used.
@@ -1690,6 +1702,14 @@ lpfc_plogi_confirm_nport(struct lpfc_hba *phba, uint32_t *prsp,
 			keep_nlp_state = NLP_STE_NPR_NODE;
 		lpfc_nlp_set_state(vport, ndlp, keep_nlp_state);
 
+		/* Previous ndlp no longer active with nvme host transport.
+		 * Remove reference from earlier registration unless the
+		 * nvme host took care of it.
+		 */
+		if (ndlp->nrport)
+			lpfc_nlp_put(ndlp);
+		ndlp->nrport = keep_nrport;
+
 		/* Fix up the rport accordingly */
 		rport = ndlp->rport;
 		if (rport) {
@@ -1966,6 +1986,7 @@ int
 lpfc_issue_els_plogi(struct lpfc_vport *vport, uint32_t did, uint8_t retry)
 {
 	struct lpfc_hba  *phba = vport->phba;
+	struct Scsi_Host *shost;
 	struct serv_parm *sp;
 	struct lpfc_nodelist *ndlp;
 	struct lpfc_iocbq *elsiocb;
@@ -1984,6 +2005,11 @@ lpfc_issue_els_plogi(struct lpfc_vport *vport, uint32_t did, uint8_t retry)
 	if (!elsiocb)
 		return 1;
 
+	shost = lpfc_shost_from_vport(vport);
+	spin_lock_irq(shost->host_lock);
+	ndlp->nlp_flag &= ~NLP_FCP_PRLI_RJT;
+	spin_unlock_irq(shost->host_lock);
+
 	pcmd = (uint8_t *) (((struct lpfc_dmabuf *) elsiocb->context2)->virt);
 
 	/* For PLOGI request, remainder of payload is service parameters */
@@ -2007,6 +2033,7 @@ lpfc_issue_els_plogi(struct lpfc_vport *vport, uint32_t did, uint8_t retry)
 
 	sp->cmn.valid_vendor_ver_level = 0;
 	memset(sp->un.vendorVersion, 0, sizeof(sp->un.vendorVersion));
+	sp->cmn.bbRcvSizeMsb &= 0xF;
 
 	lpfc_debugfs_disc_trc(vport, LPFC_DISC_TRC_ELS_CMD,
 		"Issue PLOGI:     did:x%x",
@@ -2151,6 +2178,16 @@ lpfc_issue_els_prli(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
 	uint16_t cmdsize;
 	u32 local_nlp_type, elscmd;
 
+	/*
+	 * If we are in RSCN mode, the FC4 types supported from a
+	 * previous GFT_ID command may not be accurate. So, if we
+	 * are a NVME Initiator, always look for the possibility of
+	 * the remote NPort beng a NVME Target.
+	 */
+	if (phba->sli_rev == LPFC_SLI_REV4 &&
+	    vport->fc_flag & FC_RSCN_MODE &&
+	    vport->nvmei_support)
+		ndlp->nlp_fc4_type |= NLP_FC4_NVME;
 	local_nlp_type = ndlp->nlp_fc4_type;
 
  send_next_prli:
@@ -3420,8 +3457,18 @@ lpfc_els_retry(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 				maxretry = 3;
 				delay = 1000;
 				retry = 1;
-				break;
+			} else if (cmd == ELS_CMD_FLOGI &&
+				   stat.un.b.lsRjtRsnCodeExp ==
+						LSEXP_NOTHING_MORE) {
+				vport->fc_sparam.cmn.bbRcvSizeMsb &= 0xf;
+				retry = 1;
+				lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
+						 "0820 FLOGI Failed (x%x). "
+						 "BBCredit Not Supported\n",
+						 stat.un.lsRjtError);
 			}
+			break;
+
 		case LSRJT_PROTOCOL_ERR:
 			if ((phba->sli3_options & LPFC_SLI3_NPIV_ENABLED) &&
 			  (cmd == ELS_CMD_FDISC) &&
@@ -3442,6 +3489,21 @@ lpfc_els_retry(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 				goto out_retry;
 			}
 			break;
+		case LSRJT_CMD_UNSUPPORTED:
+			/* lpfc nvmet returns this type of LS_RJT when it
+			 * receives an FCP PRLI because lpfc nvmet only
+			 * support NVME.  ELS request is terminated for FCP4
+			 * on this rport.
+			 */
+			if (stat.un.b.lsRjtRsnCodeExp ==
+			    LSEXP_REQ_UNSUPPORTED && cmd == ELS_CMD_PRLI) {
+				spin_lock_irq(shost->host_lock);
+				ndlp->nlp_flag |= NLP_FCP_PRLI_RJT;
+				spin_unlock_irq(shost->host_lock);
+				retry = 0;
+				goto out_retry;
+			}
+			break;
 		}
 		break;
 
@@ -3930,7 +3992,25 @@ lpfc_cmpl_els_rsp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 	if (mbox) {
 		if ((rspiocb->iocb.ulpStatus == 0)
 		    && (ndlp->nlp_flag & NLP_ACC_REGLOGIN)) {
-			lpfc_unreg_rpi(vport, ndlp);
+			if (!lpfc_unreg_rpi(vport, ndlp) &&
+			    (ndlp->nlp_state ==  NLP_STE_PLOGI_ISSUE ||
+			     ndlp->nlp_state == NLP_STE_REG_LOGIN_ISSUE)) {
+				lpfc_printf_vlog(vport, KERN_INFO,
+					LOG_DISCOVERY,
+					"0314 PLOGI recov DID x%x "
+					"Data: x%x x%x x%x\n",
+					ndlp->nlp_DID, ndlp->nlp_state,
+					ndlp->nlp_rpi, ndlp->nlp_flag);
+				mp = mbox->context1;
+				if (mp) {
+					lpfc_mbuf_free(phba, mp->virt,
+						       mp->phys);
+					kfree(mp);
+				}
+				mempool_free(mbox, phba->mbox_mem_pool);
+				goto out;
+			}
+
 			/* Increment reference count to ndlp to hold the
 			 * reference to ndlp for the callback function.
 			 */
@@ -4132,6 +4212,7 @@ lpfc_els_rsp_acc(struct lpfc_vport *vport, uint32_t flag,
 			sp->cmn.valid_vendor_ver_level = 0;
 			memset(sp->un.vendorVersion, 0,
 			       sizeof(sp->un.vendorVersion));
+			sp->cmn.bbRcvSizeMsb &= 0xF;
 
 			/* If our firmware supports this feature, convey that
 			 * info to the target using the vendor specific field.
@@ -7989,6 +8070,13 @@ lpfc_els_unsol_buffer(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
 			rjt_exp = LSEXP_NOTHING_MORE;
 			break;
 		}
+
+		/* NVMET accepts NVME PRLI only.  Reject FCP PRLI */
+		if (cmd == ELS_CMD_PRLI && phba->nvmet_support) {
+			rjt_err = LSRJT_CMD_UNSUPPORTED;
+			rjt_exp = LSEXP_REQ_UNSUPPORTED;
+			break;
+		}
 		lpfc_disc_state_machine(vport, ndlp, elsiocb, NLP_EVT_RCV_PRLI);
 		break;
 	case ELS_CMD_LIRR:
@@ -8784,6 +8872,7 @@ lpfc_issue_els_fdisc(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
 	pcmd += sizeof(uint32_t); /* Node Name */
 	pcmd += sizeof(uint32_t); /* Node Name */
 	memcpy(pcmd, &vport->fc_nodename, 8);
+	sp->cmn.valid_vendor_ver_level = 0;
 	memset(sp->un.vendorVersion, 0, sizeof(sp->un.vendorVersion));
 	lpfc_set_disctmo(vport);
 
diff --git a/drivers/scsi/lpfc/lpfc_hbadisc.c b/drivers/scsi/lpfc/lpfc_hbadisc.c
index aa5e5ff56dfb..20808349a80e 100644
--- a/drivers/scsi/lpfc/lpfc_hbadisc.c
+++ b/drivers/scsi/lpfc/lpfc_hbadisc.c
@@ -1108,6 +1108,7 @@ void
 lpfc_mbx_cmpl_local_config_link(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
 {
 	struct lpfc_vport *vport = pmb->vport;
+	uint8_t bbscn = 0;
 
 	if (pmb->u.mb.mbxStatus)
 		goto out;
@@ -1134,10 +1135,17 @@ lpfc_mbx_cmpl_local_config_link(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
 	/* Start discovery by sending a FLOGI. port_state is identically
 	 * LPFC_FLOGI while waiting for FLOGI cmpl
 	 */
-	if (vport->port_state != LPFC_FLOGI)
+	if (vport->port_state != LPFC_FLOGI) {
+		if (phba->bbcredit_support && phba->cfg_enable_bbcr) {
+			bbscn = bf_get(lpfc_bbscn_def,
+				       &phba->sli4_hba.bbscn_params);
+			vport->fc_sparam.cmn.bbRcvSizeMsb &= 0xf;
+			vport->fc_sparam.cmn.bbRcvSizeMsb |= (bbscn << 4);
+		}
 		lpfc_initial_flogi(vport);
-	else if (vport->fc_flag & FC_PT2PT)
+	} else if (vport->fc_flag & FC_PT2PT) {
 		lpfc_disc_start(vport);
+	}
 	return;
 
 out:
diff --git a/drivers/scsi/lpfc/lpfc_hw.h b/drivers/scsi/lpfc/lpfc_hw.h
index 26a5647e057e..bdc1f184f67a 100644
--- a/drivers/scsi/lpfc/lpfc_hw.h
+++ b/drivers/scsi/lpfc/lpfc_hw.h
@@ -2293,15 +2293,27 @@ typedef struct {
 	uint32_t rttov;
 	uint32_t altov;
 	uint32_t crtov;
-	uint32_t citov;
+
+#ifdef __BIG_ENDIAN_BITFIELD
+	uint32_t rsvd4:19;
+	uint32_t cscn:1;
+	uint32_t bbscn:4;
+	uint32_t rsvd3:8;
+#else	/*  __LITTLE_ENDIAN_BITFIELD */
+	uint32_t rsvd3:8;
+	uint32_t bbscn:4;
+	uint32_t cscn:1;
+	uint32_t rsvd4:19;
+#endif
+
 #ifdef __BIG_ENDIAN_BITFIELD
 	uint32_t rrq_enable:1;
 	uint32_t rrq_immed:1;
-	uint32_t rsvd4:29;
+	uint32_t rsvd5:29;
 	uint32_t ack0_enable:1;
 #else	/*  __LITTLE_ENDIAN_BITFIELD */
 	uint32_t ack0_enable:1;
-	uint32_t rsvd4:29;
+	uint32_t rsvd5:29;
 	uint32_t rrq_immed:1;
 	uint32_t rrq_enable:1;
 #endif
diff --git a/drivers/scsi/lpfc/lpfc_hw4.h b/drivers/scsi/lpfc/lpfc_hw4.h
index bb4715705fa3..1db0a38683f4 100644
--- a/drivers/scsi/lpfc/lpfc_hw4.h
+++ b/drivers/scsi/lpfc/lpfc_hw4.h
@@ -2217,9 +2217,15 @@ struct lpfc_mbx_reg_vfi {
 	uint32_t e_d_tov;
 	uint32_t r_a_tov;
 	uint32_t word10;
-#define lpfc_reg_vfi_nport_id_SHIFT		0
-#define lpfc_reg_vfi_nport_id_MASK		0x00FFFFFF
-#define lpfc_reg_vfi_nport_id_WORD		word10
+#define lpfc_reg_vfi_nport_id_SHIFT	0
+#define lpfc_reg_vfi_nport_id_MASK	0x00FFFFFF
+#define lpfc_reg_vfi_nport_id_WORD	word10
+#define lpfc_reg_vfi_bbcr_SHIFT		27
+#define lpfc_reg_vfi_bbcr_MASK		0x00000001
+#define lpfc_reg_vfi_bbcr_WORD		word10
+#define lpfc_reg_vfi_bbscn_SHIFT	28
+#define lpfc_reg_vfi_bbscn_MASK		0x0000000F
+#define lpfc_reg_vfi_bbscn_WORD		word10
 };
 
 struct lpfc_mbx_init_vpi {
@@ -2646,7 +2652,16 @@ struct lpfc_mbx_read_config {
 #define lpfc_mbx_rd_conf_link_speed_MASK	0x0000FFFF
 #define lpfc_mbx_rd_conf_link_speed_WORD	word6
 	uint32_t rsvd_7;
-	uint32_t rsvd_8;
+	uint32_t word8;
+#define lpfc_mbx_rd_conf_bbscn_min_SHIFT	0
+#define lpfc_mbx_rd_conf_bbscn_min_MASK		0x0000000F
+#define lpfc_mbx_rd_conf_bbscn_min_WORD		word8
+#define lpfc_mbx_rd_conf_bbscn_max_SHIFT	4
+#define lpfc_mbx_rd_conf_bbscn_max_MASK		0x0000000F
+#define lpfc_mbx_rd_conf_bbscn_max_WORD		word8
+#define lpfc_mbx_rd_conf_bbscn_def_SHIFT	8
+#define lpfc_mbx_rd_conf_bbscn_def_MASK		0x0000000F
+#define lpfc_mbx_rd_conf_bbscn_def_WORD		word8
 	uint32_t word9;
 #define lpfc_mbx_rd_conf_lmt_SHIFT		0
 #define lpfc_mbx_rd_conf_lmt_MASK		0x0000FFFF
diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c
index 491aa95eb0f6..7e7ae786121b 100644
--- a/drivers/scsi/lpfc/lpfc_init.c
+++ b/drivers/scsi/lpfc/lpfc_init.c
@@ -37,6 +37,7 @@
 #include <linux/miscdevice.h>
 #include <linux/percpu.h>
 #include <linux/msi.h>
+#include <linux/bitops.h>
 
 #include <scsi/scsi.h>
 #include <scsi/scsi_device.h>
@@ -1253,6 +1254,7 @@ lpfc_hb_timeout_handler(struct lpfc_hba *phba)
 	unsigned long time_elapsed;
 	uint32_t tick_cqe, max_cqe, val;
 	uint64_t tot, data1, data2, data3;
+	struct lpfc_nvmet_tgtport *tgtp;
 	struct lpfc_register reg_data;
 	void __iomem *eqdreg = phba->sli4_hba.u.if_type2.EQDregaddr;
 
@@ -1281,13 +1283,11 @@ lpfc_hb_timeout_handler(struct lpfc_hba *phba)
 		/* Check outstanding IO count */
 		if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) {
 			if (phba->nvmet_support) {
-				spin_lock(&phba->sli4_hba.nvmet_ctx_get_lock);
-				spin_lock(&phba->sli4_hba.nvmet_ctx_put_lock);
-				tot = phba->sli4_hba.nvmet_xri_cnt -
-					(phba->sli4_hba.nvmet_ctx_get_cnt +
-					phba->sli4_hba.nvmet_ctx_put_cnt);
-				spin_unlock(&phba->sli4_hba.nvmet_ctx_put_lock);
-				spin_unlock(&phba->sli4_hba.nvmet_ctx_get_lock);
+				tgtp = phba->targetport->private;
+				/* Calculate outstanding IOs */
+				tot = atomic_read(&tgtp->rcv_fcp_cmd_drop);
+				tot += atomic_read(&tgtp->xmt_fcp_release);
+				tot = atomic_read(&tgtp->rcv_fcp_cmd_in) - tot;
 			} else {
 				tot = atomic_read(&phba->fc4NvmeIoCmpls);
 				data1 = atomic_read(
@@ -3048,7 +3048,7 @@ lpfc_online(struct lpfc_hba *phba)
 {
 	struct lpfc_vport *vport;
 	struct lpfc_vport **vports;
-	int i;
+	int i, error = 0;
 	bool vpis_cleared = false;
 
 	if (!phba)
@@ -3072,6 +3072,18 @@ lpfc_online(struct lpfc_hba *phba)
 		if (!phba->sli4_hba.max_cfg_param.vpi_used)
 			vpis_cleared = true;
 		spin_unlock_irq(&phba->hbalock);
+
+		/* Reestablish the local initiator port.
+		 * The offline process destroyed the previous lport.
+		 */
+		if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME &&
+				!phba->nvmet_support) {
+			error = lpfc_nvme_create_localport(phba->pport);
+			if (error)
+				lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+					"6132 NVME restore reg failed "
+					"on nvmei error x%x\n", error);
+		}
 	} else {
 		lpfc_sli_queue_init(phba);
 		if (lpfc_sli_hba_setup(phba)) {	/* Initialize SLI2/SLI3 HBA */
@@ -3226,6 +3238,13 @@ lpfc_offline(struct lpfc_hba *phba)
 
 	/* stop port and all timers associated with this hba */
 	lpfc_stop_port(phba);
+
+	/* Tear down the local and target port registrations.  The
+	 * nvme transports need to cleanup.
+	 */
+	lpfc_nvmet_destroy_targetport(phba);
+	lpfc_nvme_destroy_localport(phba->pport);
+
 	vports = lpfc_create_vport_work_array(phba);
 	if (vports != NULL)
 		for (i = 0; i <= phba->max_vports && vports[i] != NULL; i++)
@@ -3275,7 +3294,7 @@ lpfc_scsi_free(struct lpfc_hba *phba)
 	list_for_each_entry_safe(sb, sb_next, &phba->lpfc_scsi_buf_list_put,
 				 list) {
 		list_del(&sb->list);
-		pci_pool_free(phba->lpfc_sg_dma_buf_pool, sb->data,
+		dma_pool_free(phba->lpfc_sg_dma_buf_pool, sb->data,
 			      sb->dma_handle);
 		kfree(sb);
 		phba->total_scsi_bufs--;
@@ -3286,7 +3305,7 @@ lpfc_scsi_free(struct lpfc_hba *phba)
 	list_for_each_entry_safe(sb, sb_next, &phba->lpfc_scsi_buf_list_get,
 				 list) {
 		list_del(&sb->list);
-		pci_pool_free(phba->lpfc_sg_dma_buf_pool, sb->data,
+		dma_pool_free(phba->lpfc_sg_dma_buf_pool, sb->data,
 			      sb->dma_handle);
 		kfree(sb);
 		phba->total_scsi_bufs--;
@@ -3317,7 +3336,7 @@ lpfc_nvme_free(struct lpfc_hba *phba)
 	list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next,
 				 &phba->lpfc_nvme_buf_list_put, list) {
 		list_del(&lpfc_ncmd->list);
-		pci_pool_free(phba->lpfc_sg_dma_buf_pool, lpfc_ncmd->data,
+		dma_pool_free(phba->lpfc_sg_dma_buf_pool, lpfc_ncmd->data,
 			      lpfc_ncmd->dma_handle);
 		kfree(lpfc_ncmd);
 		phba->total_nvme_bufs--;
@@ -3328,7 +3347,7 @@ lpfc_nvme_free(struct lpfc_hba *phba)
 	list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next,
 				 &phba->lpfc_nvme_buf_list_get, list) {
 		list_del(&lpfc_ncmd->list);
-		pci_pool_free(phba->lpfc_sg_dma_buf_pool, lpfc_ncmd->data,
+		dma_pool_free(phba->lpfc_sg_dma_buf_pool, lpfc_ncmd->data,
 			      lpfc_ncmd->dma_handle);
 		kfree(lpfc_ncmd);
 		phba->total_nvme_bufs--;
@@ -3640,7 +3659,7 @@ lpfc_sli4_scsi_sgl_update(struct lpfc_hba *phba)
 			list_remove_head(&scsi_sgl_list, psb,
 					 struct lpfc_scsi_buf, list);
 			if (psb) {
-				pci_pool_free(phba->lpfc_sg_dma_buf_pool,
+				dma_pool_free(phba->lpfc_sg_dma_buf_pool,
 					      psb->data, psb->dma_handle);
 				kfree(psb);
 			}
@@ -3710,9 +3729,7 @@ lpfc_get_wwpn(struct lpfc_hba *phba)
 	if (phba->sli_rev == LPFC_SLI_REV4)
 		return be64_to_cpu(wwn);
 	else
-		return (((wwn & 0xffffffff00000000) >> 32) |
-			((wwn & 0x00000000ffffffff) << 32));
-
+		return rol64(wwn, 32);
 }
 
 /**
@@ -3774,7 +3791,7 @@ lpfc_sli4_nvme_sgl_update(struct lpfc_hba *phba)
 			list_remove_head(&nvme_sgl_list, lpfc_ncmd,
 					 struct lpfc_nvme_buf, list);
 			if (lpfc_ncmd) {
-				pci_pool_free(phba->lpfc_sg_dma_buf_pool,
+				dma_pool_free(phba->lpfc_sg_dma_buf_pool,
 					      lpfc_ncmd->data,
 					      lpfc_ncmd->dma_handle);
 				kfree(lpfc_ncmd);
@@ -5930,8 +5947,6 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba)
 		spin_lock_init(&phba->sli4_hba.abts_nvme_buf_list_lock);
 		INIT_LIST_HEAD(&phba->sli4_hba.lpfc_abts_nvme_buf_list);
 		INIT_LIST_HEAD(&phba->sli4_hba.lpfc_abts_nvmet_ctx_list);
-		INIT_LIST_HEAD(&phba->sli4_hba.lpfc_nvmet_ctx_get_list);
-		INIT_LIST_HEAD(&phba->sli4_hba.lpfc_nvmet_ctx_put_list);
 		INIT_LIST_HEAD(&phba->sli4_hba.lpfc_nvmet_io_wait_list);
 
 		/* Fast-path XRI aborted CQ Event work queue list */
@@ -5940,8 +5955,6 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba)
 
 	/* This abort list used by worker thread */
 	spin_lock_init(&phba->sli4_hba.sgl_list_lock);
-	spin_lock_init(&phba->sli4_hba.nvmet_ctx_get_lock);
-	spin_lock_init(&phba->sli4_hba.nvmet_ctx_put_lock);
 	spin_lock_init(&phba->sli4_hba.nvmet_io_wait_lock);
 
 	/*
@@ -6516,6 +6529,12 @@ lpfc_free_nvmet_sgl_list(struct lpfc_hba *phba)
 		lpfc_nvmet_buf_free(phba, sglq_entry->virt, sglq_entry->phys);
 		kfree(sglq_entry);
 	}
+
+	/* Update the nvmet_xri_cnt to reflect no current sgls.
+	 * The next initialization cycle sets the count and allocates
+	 * the sgls over again.
+	 */
+	phba->sli4_hba.nvmet_xri_cnt = 0;
 }
 
 /**
@@ -6846,8 +6865,8 @@ lpfc_create_shost(struct lpfc_hba *phba)
 	if (phba->nvmet_support) {
 		/* Only 1 vport (pport) will support NVME target */
 		if (phba->txrdy_payload_pool == NULL) {
-			phba->txrdy_payload_pool = pci_pool_create(
-				"txrdy_pool", phba->pcidev,
+			phba->txrdy_payload_pool = dma_pool_create(
+				"txrdy_pool", &phba->pcidev->dev,
 				TXRDY_PAYLOAD_LEN, 16, 0);
 			if (phba->txrdy_payload_pool) {
 				phba->targetport = NULL;
@@ -7605,6 +7624,11 @@ lpfc_sli4_read_config(struct lpfc_hba *phba)
 			lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
 					"3082 Mailbox (x%x) returned ldv:x0\n",
 					bf_get(lpfc_mqe_command, &pmb->u.mqe));
+		if (bf_get(lpfc_mbx_rd_conf_bbscn_def, rd_config)) {
+			phba->bbcredit_support = 1;
+			phba->sli4_hba.bbscn_params.word0 = rd_config->word8;
+		}
+
 		phba->sli4_hba.extents_in_use =
 			bf_get(lpfc_mbx_rd_conf_extnts_inuse, rd_config);
 		phba->sli4_hba.max_cfg_param.max_xri =
@@ -8301,6 +8325,9 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba)
 				goto out_error;
 			}
 
+			/* Put list in known state in case driver load fails. */
+			INIT_LIST_HEAD(&qdesc->rqbp->rqb_buffer_list);
+
 			/* Create NVMET Receive Queue for data */
 			qdesc = lpfc_sli4_queue_alloc(phba,
 						      phba->sli4_hba.rq_esize,
diff --git a/drivers/scsi/lpfc/lpfc_mbox.c b/drivers/scsi/lpfc/lpfc_mbox.c
index ce25a18367b5..81fb92967b11 100644
--- a/drivers/scsi/lpfc/lpfc_mbox.c
+++ b/drivers/scsi/lpfc/lpfc_mbox.c
@@ -376,7 +376,12 @@ lpfc_config_link(struct lpfc_hba * phba, LPFC_MBOXQ_t * pmb)
 	mb->un.varCfgLnk.rttov = phba->fc_rttov;
 	mb->un.varCfgLnk.altov = phba->fc_altov;
 	mb->un.varCfgLnk.crtov = phba->fc_crtov;
-	mb->un.varCfgLnk.citov = phba->fc_citov;
+	mb->un.varCfgLnk.cscn = 0;
+	if (phba->bbcredit_support && phba->cfg_enable_bbcr) {
+		mb->un.varCfgLnk.cscn = 1;
+		mb->un.varCfgLnk.bbscn = bf_get(lpfc_bbscn_def,
+						 &phba->sli4_hba.bbscn_params);
+	}
 
 	if (phba->cfg_ack0 && (phba->sli_rev < LPFC_SLI_REV4))
 		mb->un.varCfgLnk.ack0_enable = 1;
@@ -2139,6 +2144,7 @@ lpfc_reg_vfi(struct lpfcMboxq *mbox, struct lpfc_vport *vport, dma_addr_t phys)
 {
 	struct lpfc_mbx_reg_vfi *reg_vfi;
 	struct lpfc_hba *phba = vport->phba;
+	uint8_t bbscn_fabric = 0, bbscn_max = 0, bbscn_def = 0;
 
 	memset(mbox, 0, sizeof(*mbox));
 	reg_vfi = &mbox->u.mqe.un.reg_vfi;
@@ -2168,16 +2174,39 @@ lpfc_reg_vfi(struct lpfcMboxq *mbox, struct lpfc_vport *vport, dma_addr_t phys)
 		bf_set(lpfc_reg_vfi_vp, reg_vfi, 0);
 		bf_set(lpfc_reg_vfi_upd, reg_vfi, 1);
 	}
+
+	bf_set(lpfc_reg_vfi_bbcr, reg_vfi, 0);
+	bf_set(lpfc_reg_vfi_bbscn, reg_vfi, 0);
+	bbscn_fabric = (phba->fc_fabparam.cmn.bbRcvSizeMsb >> 4) & 0xF;
+
+	if (phba->bbcredit_support && phba->cfg_enable_bbcr  &&
+	    bbscn_fabric != 0) {
+		bbscn_max = bf_get(lpfc_bbscn_max,
+				   &phba->sli4_hba.bbscn_params);
+		if (bbscn_fabric <= bbscn_max) {
+			bbscn_def = bf_get(lpfc_bbscn_def,
+					   &phba->sli4_hba.bbscn_params);
+
+			if (bbscn_fabric > bbscn_def)
+				bf_set(lpfc_reg_vfi_bbscn, reg_vfi,
+				       bbscn_fabric);
+			else
+				bf_set(lpfc_reg_vfi_bbscn, reg_vfi, bbscn_def);
+
+			bf_set(lpfc_reg_vfi_bbcr, reg_vfi, 1);
+		}
+	}
 	lpfc_printf_vlog(vport, KERN_INFO, LOG_MBOX,
 			"3134 Register VFI, mydid:x%x, fcfi:%d, "
 			" vfi:%d, vpi:%d, fc_pname:%x%x fc_flag:x%x"
-			" port_state:x%x topology chg:%d\n",
+			" port_state:x%x topology chg:%d bbscn_fabric :%d\n",
 			vport->fc_myDID,
 			phba->fcf.fcfi,
 			phba->sli4_hba.vfi_ids[vport->vfi],
 			phba->vpi_ids[vport->vpi],
 			reg_vfi->wwn[0], reg_vfi->wwn[1], vport->fc_flag,
-			vport->port_state, phba->fc_topology_changed);
+			vport->port_state, phba->fc_topology_changed,
+			bbscn_fabric);
 }
 
 /**
diff --git a/drivers/scsi/lpfc/lpfc_mem.c b/drivers/scsi/lpfc/lpfc_mem.c
index fcc05a1517c2..56faeb049b4a 100644
--- a/drivers/scsi/lpfc/lpfc_mem.c
+++ b/drivers/scsi/lpfc/lpfc_mem.c
@@ -97,8 +97,8 @@ lpfc_mem_alloc(struct lpfc_hba *phba, int align)
 			i = SLI4_PAGE_SIZE;
 
 		phba->lpfc_sg_dma_buf_pool =
-			pci_pool_create("lpfc_sg_dma_buf_pool",
-					phba->pcidev,
+			dma_pool_create("lpfc_sg_dma_buf_pool",
+					&phba->pcidev->dev,
 					phba->cfg_sg_dma_buf_size,
 					i, 0);
 		if (!phba->lpfc_sg_dma_buf_pool)
@@ -106,15 +106,15 @@ lpfc_mem_alloc(struct lpfc_hba *phba, int align)
 
 	} else {
 		phba->lpfc_sg_dma_buf_pool =
-			pci_pool_create("lpfc_sg_dma_buf_pool",
-					phba->pcidev, phba->cfg_sg_dma_buf_size,
+			dma_pool_create("lpfc_sg_dma_buf_pool",
+					&phba->pcidev->dev, phba->cfg_sg_dma_buf_size,
 					align, 0);
 
 		if (!phba->lpfc_sg_dma_buf_pool)
 			goto fail;
 	}
 
-	phba->lpfc_mbuf_pool = pci_pool_create("lpfc_mbuf_pool", phba->pcidev,
+	phba->lpfc_mbuf_pool = dma_pool_create("lpfc_mbuf_pool", &phba->pcidev->dev,
 							LPFC_BPL_SIZE,
 							align, 0);
 	if (!phba->lpfc_mbuf_pool)
@@ -128,7 +128,7 @@ lpfc_mem_alloc(struct lpfc_hba *phba, int align)
 	pool->max_count = 0;
 	pool->current_count = 0;
 	for ( i = 0; i < LPFC_MBUF_POOL_SIZE; i++) {
-		pool->elements[i].virt = pci_pool_alloc(phba->lpfc_mbuf_pool,
+		pool->elements[i].virt = dma_pool_alloc(phba->lpfc_mbuf_pool,
 				       GFP_KERNEL, &pool->elements[i].phys);
 		if (!pool->elements[i].virt)
 			goto fail_free_mbuf_pool;
@@ -152,21 +152,21 @@ lpfc_mem_alloc(struct lpfc_hba *phba, int align)
 						sizeof(struct lpfc_node_rrq));
 		if (!phba->rrq_pool)
 			goto fail_free_nlp_mem_pool;
-		phba->lpfc_hrb_pool = pci_pool_create("lpfc_hrb_pool",
-					      phba->pcidev,
+		phba->lpfc_hrb_pool = dma_pool_create("lpfc_hrb_pool",
+					      &phba->pcidev->dev,
 					      LPFC_HDR_BUF_SIZE, align, 0);
 		if (!phba->lpfc_hrb_pool)
 			goto fail_free_rrq_mem_pool;
 
-		phba->lpfc_drb_pool = pci_pool_create("lpfc_drb_pool",
-					      phba->pcidev,
+		phba->lpfc_drb_pool = dma_pool_create("lpfc_drb_pool",
+					      &phba->pcidev->dev,
 					      LPFC_DATA_BUF_SIZE, align, 0);
 		if (!phba->lpfc_drb_pool)
 			goto fail_free_hrb_pool;
 		phba->lpfc_hbq_pool = NULL;
 	} else {
-		phba->lpfc_hbq_pool = pci_pool_create("lpfc_hbq_pool",
-			phba->pcidev, LPFC_BPL_SIZE, align, 0);
+		phba->lpfc_hbq_pool = dma_pool_create("lpfc_hbq_pool",
+			&phba->pcidev->dev, LPFC_BPL_SIZE, align, 0);
 		if (!phba->lpfc_hbq_pool)
 			goto fail_free_nlp_mem_pool;
 		phba->lpfc_hrb_pool = NULL;
@@ -185,10 +185,10 @@ lpfc_mem_alloc(struct lpfc_hba *phba, int align)
 
 	return 0;
 fail_free_drb_pool:
-	pci_pool_destroy(phba->lpfc_drb_pool);
+	dma_pool_destroy(phba->lpfc_drb_pool);
 	phba->lpfc_drb_pool = NULL;
  fail_free_hrb_pool:
-	pci_pool_destroy(phba->lpfc_hrb_pool);
+	dma_pool_destroy(phba->lpfc_hrb_pool);
 	phba->lpfc_hrb_pool = NULL;
  fail_free_rrq_mem_pool:
 	mempool_destroy(phba->rrq_pool);
@@ -201,14 +201,14 @@ fail_free_drb_pool:
 	phba->mbox_mem_pool = NULL;
  fail_free_mbuf_pool:
 	while (i--)
-		pci_pool_free(phba->lpfc_mbuf_pool, pool->elements[i].virt,
+		dma_pool_free(phba->lpfc_mbuf_pool, pool->elements[i].virt,
 						 pool->elements[i].phys);
 	kfree(pool->elements);
  fail_free_lpfc_mbuf_pool:
-	pci_pool_destroy(phba->lpfc_mbuf_pool);
+	dma_pool_destroy(phba->lpfc_mbuf_pool);
 	phba->lpfc_mbuf_pool = NULL;
  fail_free_dma_buf_pool:
-	pci_pool_destroy(phba->lpfc_sg_dma_buf_pool);
+	dma_pool_destroy(phba->lpfc_sg_dma_buf_pool);
 	phba->lpfc_sg_dma_buf_pool = NULL;
  fail:
 	return -ENOMEM;
@@ -218,8 +218,8 @@ int
 lpfc_nvmet_mem_alloc(struct lpfc_hba *phba)
 {
 	phba->lpfc_nvmet_drb_pool =
-		pci_pool_create("lpfc_nvmet_drb_pool",
-				phba->pcidev, LPFC_NVMET_DATA_BUF_SIZE,
+		dma_pool_create("lpfc_nvmet_drb_pool",
+				&phba->pcidev->dev, LPFC_NVMET_DATA_BUF_SIZE,
 				SGL_ALIGN_SZ, 0);
 	if (!phba->lpfc_nvmet_drb_pool) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
@@ -248,20 +248,20 @@ lpfc_mem_free(struct lpfc_hba *phba)
 	/* Free HBQ pools */
 	lpfc_sli_hbqbuf_free_all(phba);
 	if (phba->lpfc_nvmet_drb_pool)
-		pci_pool_destroy(phba->lpfc_nvmet_drb_pool);
+		dma_pool_destroy(phba->lpfc_nvmet_drb_pool);
 	phba->lpfc_nvmet_drb_pool = NULL;
 	if (phba->lpfc_drb_pool)
-		pci_pool_destroy(phba->lpfc_drb_pool);
+		dma_pool_destroy(phba->lpfc_drb_pool);
 	phba->lpfc_drb_pool = NULL;
 	if (phba->lpfc_hrb_pool)
-		pci_pool_destroy(phba->lpfc_hrb_pool);
+		dma_pool_destroy(phba->lpfc_hrb_pool);
 	phba->lpfc_hrb_pool = NULL;
 	if (phba->txrdy_payload_pool)
-		pci_pool_destroy(phba->txrdy_payload_pool);
+		dma_pool_destroy(phba->txrdy_payload_pool);
 	phba->txrdy_payload_pool = NULL;
 
 	if (phba->lpfc_hbq_pool)
-		pci_pool_destroy(phba->lpfc_hbq_pool);
+		dma_pool_destroy(phba->lpfc_hbq_pool);
 	phba->lpfc_hbq_pool = NULL;
 
 	if (phba->rrq_pool)
@@ -282,15 +282,15 @@ lpfc_mem_free(struct lpfc_hba *phba)
 
 	/* Free MBUF memory pool */
 	for (i = 0; i < pool->current_count; i++)
-		pci_pool_free(phba->lpfc_mbuf_pool, pool->elements[i].virt,
+		dma_pool_free(phba->lpfc_mbuf_pool, pool->elements[i].virt,
 			      pool->elements[i].phys);
 	kfree(pool->elements);
 
-	pci_pool_destroy(phba->lpfc_mbuf_pool);
+	dma_pool_destroy(phba->lpfc_mbuf_pool);
 	phba->lpfc_mbuf_pool = NULL;
 
 	/* Free DMA buffer memory pool */
-	pci_pool_destroy(phba->lpfc_sg_dma_buf_pool);
+	dma_pool_destroy(phba->lpfc_sg_dma_buf_pool);
 	phba->lpfc_sg_dma_buf_pool = NULL;
 
 	/* Free Device Data memory pool */
@@ -379,7 +379,7 @@ lpfc_mem_free_all(struct lpfc_hba *phba)
  * @handle: used to return the DMA-mapped address of the mbuf
  *
  * Description: Allocates a DMA-mapped buffer from the lpfc_mbuf_pool PCI pool.
- * Allocates from generic pci_pool_alloc function first and if that fails and
+ * Allocates from generic dma_pool_alloc function first and if that fails and
  * mem_flags has MEM_PRI set (the only defined flag), returns an mbuf from the
  * HBA's pool.
  *
@@ -397,7 +397,7 @@ lpfc_mbuf_alloc(struct lpfc_hba *phba, int mem_flags, dma_addr_t *handle)
 	unsigned long iflags;
 	void *ret;
 
-	ret = pci_pool_alloc(phba->lpfc_mbuf_pool, GFP_KERNEL, handle);
+	ret = dma_pool_alloc(phba->lpfc_mbuf_pool, GFP_KERNEL, handle);
 
 	spin_lock_irqsave(&phba->hbalock, iflags);
 	if (!ret && (mem_flags & MEM_PRI) && pool->current_count) {
@@ -433,7 +433,7 @@ __lpfc_mbuf_free(struct lpfc_hba * phba, void *virt, dma_addr_t dma)
 		pool->elements[pool->current_count].phys = dma;
 		pool->current_count++;
 	} else {
-		pci_pool_free(phba->lpfc_mbuf_pool, virt, dma);
+		dma_pool_free(phba->lpfc_mbuf_pool, virt, dma);
 	}
 	return;
 }
@@ -470,7 +470,7 @@ lpfc_mbuf_free(struct lpfc_hba * phba, void *virt, dma_addr_t dma)
  * @handle: used to return the DMA-mapped address of the nvmet_buf
  *
  * Description: Allocates a DMA-mapped buffer from the lpfc_sg_dma_buf_pool
- * PCI pool.  Allocates from generic pci_pool_alloc function.
+ * PCI pool.  Allocates from generic dma_pool_alloc function.
  *
  * Returns:
  *   pointer to the allocated nvmet_buf on success
@@ -481,7 +481,7 @@ lpfc_nvmet_buf_alloc(struct lpfc_hba *phba, int mem_flags, dma_addr_t *handle)
 {
 	void *ret;
 
-	ret = pci_pool_alloc(phba->lpfc_sg_dma_buf_pool, GFP_KERNEL, handle);
+	ret = dma_pool_alloc(phba->lpfc_sg_dma_buf_pool, GFP_KERNEL, handle);
 	return ret;
 }
 
@@ -497,7 +497,7 @@ lpfc_nvmet_buf_alloc(struct lpfc_hba *phba, int mem_flags, dma_addr_t *handle)
 void
 lpfc_nvmet_buf_free(struct lpfc_hba *phba, void *virt, dma_addr_t dma)
 {
-	pci_pool_free(phba->lpfc_sg_dma_buf_pool, virt, dma);
+	dma_pool_free(phba->lpfc_sg_dma_buf_pool, virt, dma);
 }
 
 /**
@@ -522,7 +522,7 @@ lpfc_els_hbq_alloc(struct lpfc_hba *phba)
 	if (!hbqbp)
 		return NULL;
 
-	hbqbp->dbuf.virt = pci_pool_alloc(phba->lpfc_hbq_pool, GFP_KERNEL,
+	hbqbp->dbuf.virt = dma_pool_alloc(phba->lpfc_hbq_pool, GFP_KERNEL,
 					  &hbqbp->dbuf.phys);
 	if (!hbqbp->dbuf.virt) {
 		kfree(hbqbp);
@@ -547,7 +547,7 @@ lpfc_els_hbq_alloc(struct lpfc_hba *phba)
 void
 lpfc_els_hbq_free(struct lpfc_hba *phba, struct hbq_dmabuf *hbqbp)
 {
-	pci_pool_free(phba->lpfc_hbq_pool, hbqbp->dbuf.virt, hbqbp->dbuf.phys);
+	dma_pool_free(phba->lpfc_hbq_pool, hbqbp->dbuf.virt, hbqbp->dbuf.phys);
 	kfree(hbqbp);
 	return;
 }
@@ -574,16 +574,16 @@ lpfc_sli4_rb_alloc(struct lpfc_hba *phba)
 	if (!dma_buf)
 		return NULL;
 
-	dma_buf->hbuf.virt = pci_pool_alloc(phba->lpfc_hrb_pool, GFP_KERNEL,
+	dma_buf->hbuf.virt = dma_pool_alloc(phba->lpfc_hrb_pool, GFP_KERNEL,
 					    &dma_buf->hbuf.phys);
 	if (!dma_buf->hbuf.virt) {
 		kfree(dma_buf);
 		return NULL;
 	}
-	dma_buf->dbuf.virt = pci_pool_alloc(phba->lpfc_drb_pool, GFP_KERNEL,
+	dma_buf->dbuf.virt = dma_pool_alloc(phba->lpfc_drb_pool, GFP_KERNEL,
 					    &dma_buf->dbuf.phys);
 	if (!dma_buf->dbuf.virt) {
-		pci_pool_free(phba->lpfc_hrb_pool, dma_buf->hbuf.virt,
+		dma_pool_free(phba->lpfc_hrb_pool, dma_buf->hbuf.virt,
 			      dma_buf->hbuf.phys);
 		kfree(dma_buf);
 		return NULL;
@@ -607,8 +607,8 @@ lpfc_sli4_rb_alloc(struct lpfc_hba *phba)
 void
 lpfc_sli4_rb_free(struct lpfc_hba *phba, struct hbq_dmabuf *dmab)
 {
-	pci_pool_free(phba->lpfc_hrb_pool, dmab->hbuf.virt, dmab->hbuf.phys);
-	pci_pool_free(phba->lpfc_drb_pool, dmab->dbuf.virt, dmab->dbuf.phys);
+	dma_pool_free(phba->lpfc_hrb_pool, dmab->hbuf.virt, dmab->hbuf.phys);
+	dma_pool_free(phba->lpfc_drb_pool, dmab->dbuf.virt, dmab->dbuf.phys);
 	kfree(dmab);
 }
 
@@ -634,16 +634,16 @@ lpfc_sli4_nvmet_alloc(struct lpfc_hba *phba)
 	if (!dma_buf)
 		return NULL;
 
-	dma_buf->hbuf.virt = pci_pool_alloc(phba->lpfc_hrb_pool, GFP_KERNEL,
+	dma_buf->hbuf.virt = dma_pool_alloc(phba->lpfc_hrb_pool, GFP_KERNEL,
 					    &dma_buf->hbuf.phys);
 	if (!dma_buf->hbuf.virt) {
 		kfree(dma_buf);
 		return NULL;
 	}
-	dma_buf->dbuf.virt = pci_pool_alloc(phba->lpfc_nvmet_drb_pool,
+	dma_buf->dbuf.virt = dma_pool_alloc(phba->lpfc_nvmet_drb_pool,
 					    GFP_KERNEL, &dma_buf->dbuf.phys);
 	if (!dma_buf->dbuf.virt) {
-		pci_pool_free(phba->lpfc_hrb_pool, dma_buf->hbuf.virt,
+		dma_pool_free(phba->lpfc_hrb_pool, dma_buf->hbuf.virt,
 			      dma_buf->hbuf.phys);
 		kfree(dma_buf);
 		return NULL;
@@ -667,8 +667,8 @@ lpfc_sli4_nvmet_alloc(struct lpfc_hba *phba)
 void
 lpfc_sli4_nvmet_free(struct lpfc_hba *phba, struct rqb_dmabuf *dmab)
 {
-	pci_pool_free(phba->lpfc_hrb_pool, dmab->hbuf.virt, dmab->hbuf.phys);
-	pci_pool_free(phba->lpfc_nvmet_drb_pool,
+	dma_pool_free(phba->lpfc_hrb_pool, dmab->hbuf.virt, dmab->hbuf.phys);
+	dma_pool_free(phba->lpfc_nvmet_drb_pool,
 		      dmab->dbuf.virt, dmab->dbuf.phys);
 	kfree(dmab);
 }
diff --git a/drivers/scsi/lpfc/lpfc_nportdisc.c b/drivers/scsi/lpfc/lpfc_nportdisc.c
index f74cb0142fd4..f3ad7cac355d 100644
--- a/drivers/scsi/lpfc/lpfc_nportdisc.c
+++ b/drivers/scsi/lpfc/lpfc_nportdisc.c
@@ -1724,6 +1724,9 @@ lpfc_cmpl_reglogin_reglogin_issue(struct lpfc_vport *vport,
 				lpfc_nvme_update_localport(vport);
 			}
 
+		} else if (phba->fc_topology == LPFC_TOPOLOGY_LOOP) {
+			ndlp->nlp_fc4_type |= NLP_FC4_FCP;
+
 		} else if (ndlp->nlp_fc4_type == 0) {
 			rc = lpfc_ns_cmd(vport, SLI_CTNS_GFT_ID,
 					 0, ndlp->nlp_DID);
@@ -1892,6 +1895,15 @@ lpfc_cmpl_prli_prli_issue(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
 			goto out;
 		}
 
+		/* When the rport rejected the FCP PRLI as unsupported.
+		 * This should only happen in Pt2Pt so an NVME PRLI
+		 * should be outstanding still.
+		 */
+		if (npr && ndlp->nlp_flag & NLP_FCP_PRLI_RJT) {
+			ndlp->nlp_fc4_type &= ~NLP_FC4_FCP;
+			goto out_err;
+		}
+
 		/* The LS Req had some error.  Don't let this be a
 		 * target.
 		 */
@@ -2189,12 +2201,15 @@ lpfc_device_rm_logo_issue(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
 			  void *arg, uint32_t evt)
 {
 	/*
-	 * Take no action.  If a LOGO is outstanding, then possibly DevLoss has
-	 * timed out and is calling for Device Remove.  In this case, the LOGO
-	 * must be allowed to complete in state LOGO_ISSUE so that the rpi
-	 * and other NLP flags are correctly cleaned up.
+	 * DevLoss has timed out and is calling for Device Remove.
+	 * In this case, abort the LOGO and cleanup the ndlp
 	 */
-	return ndlp->nlp_state;
+
+	lpfc_unreg_rpi(vport, ndlp);
+	/* software abort outstanding PLOGI */
+	lpfc_els_abort(vport->phba, ndlp);
+	lpfc_drop_node(vport, ndlp);
+	return NLP_STE_FREED_NODE;
 }
 
 static uint32_t
diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c
index 0a0a1b92d01d..79ba3ce063a4 100644
--- a/drivers/scsi/lpfc/lpfc_nvme.c
+++ b/drivers/scsi/lpfc/lpfc_nvme.c
@@ -110,7 +110,7 @@ lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport,
 		qhandle->index = qidx;
 	}
 
-	lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME,
+	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
 			 "6073 Binding %s HdwQueue %d  (cpu %d) to "
 			 "io_channel %d qhandle %p\n", str,
 			 qidx, qhandle->cpu_id, qhandle->index, qhandle);
@@ -364,7 +364,7 @@ lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp,
 			 genwqe->sli4_xritag, genwqe->iotag, ndlp->nlp_DID);
 
 	rc = lpfc_sli4_issue_wqe(phba, LPFC_ELS_RING, genwqe);
-	if (rc == WQE_ERROR) {
+	if (rc) {
 		lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
 				 "6045 Issue GEN REQ WQE to NPORT x%x "
 				 "Data: x%x x%x\n",
@@ -1270,7 +1270,7 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
 	 * not exceed the programmed depth.
 	 */
 	if (atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) {
-		ret = -EAGAIN;
+		ret = -EBUSY;
 		goto out_fail;
 	}
 
@@ -1279,7 +1279,7 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
 				 "6065 driver's buffer pool is empty, "
 				 "IO failed\n");
-		ret = -ENOMEM;
+		ret = -EBUSY;
 		goto out_fail;
 	}
 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
@@ -1332,7 +1332,6 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
 				 "sid: x%x did: x%x oxid: x%x\n",
 				 ret, vport->fc_myDID, ndlp->nlp_DID,
 				 lpfc_ncmd->cur_iocbq.sli4_xritag);
-		ret = -EBUSY;
 		goto out_free_nvme_buf;
 	}
 
@@ -1576,7 +1575,7 @@ lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport,
 	abts_buf->wqe_cmpl = lpfc_nvme_abort_fcreq_cmpl;
 	ret_val = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, abts_buf);
 	spin_unlock_irqrestore(&phba->hbalock, flags);
-	if (ret_val == IOCB_ERROR) {
+	if (ret_val) {
 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
 				 "6137 Failed abts issue_wqe with status x%x "
 				 "for nvme_fcreq %p.\n",
@@ -1939,7 +1938,7 @@ lpfc_new_nvme_buf(struct lpfc_vport *vport, int num_to_alloc)
 		 * pci bus space for an I/O. The DMA buffer includes the
 		 * number of SGE's necessary to support the sg_tablesize.
 		 */
-		lpfc_ncmd->data = pci_pool_alloc(phba->lpfc_sg_dma_buf_pool,
+		lpfc_ncmd->data = dma_pool_alloc(phba->lpfc_sg_dma_buf_pool,
 						 GFP_KERNEL,
 						 &lpfc_ncmd->dma_handle);
 		if (!lpfc_ncmd->data) {
@@ -1950,7 +1949,7 @@ lpfc_new_nvme_buf(struct lpfc_vport *vport, int num_to_alloc)
 
 		lxri = lpfc_sli4_next_xritag(phba);
 		if (lxri == NO_XRI) {
-			pci_pool_free(phba->lpfc_sg_dma_buf_pool,
+			dma_pool_free(phba->lpfc_sg_dma_buf_pool,
 				      lpfc_ncmd->data, lpfc_ncmd->dma_handle);
 			kfree(lpfc_ncmd);
 			break;
@@ -1961,7 +1960,7 @@ lpfc_new_nvme_buf(struct lpfc_vport *vport, int num_to_alloc)
 		/* Allocate iotag for lpfc_ncmd->cur_iocbq. */
 		iotag = lpfc_sli_next_iotag(phba, pwqeq);
 		if (iotag == 0) {
-			pci_pool_free(phba->lpfc_sg_dma_buf_pool,
+			dma_pool_free(phba->lpfc_sg_dma_buf_pool,
 				      lpfc_ncmd->data, lpfc_ncmd->dma_handle);
 			kfree(lpfc_ncmd);
 			lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
@@ -2182,8 +2181,15 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport)
 		vport->localport = localport;
 		lport->vport = vport;
 		vport->nvmei_support = 1;
-		len  = lpfc_new_nvme_buf(vport, phba->sli4_hba.nvme_xri_max);
-		vport->phba->total_nvme_bufs += len;
+
+		/* Don't post more new bufs if repost already recovered
+		 * the nvme sgls.
+		 */
+		if (phba->sli4_hba.nvme_xri_cnt == 0) {
+			len  = lpfc_new_nvme_buf(vport,
+						 phba->sli4_hba.nvme_xri_max);
+			vport->phba->total_nvme_bufs += len;
+		}
 	}
 
 	return ret;
@@ -2296,6 +2302,9 @@ lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
 			 ndlp->nlp_DID, ndlp->nlp_type);
 
 	localport = vport->localport;
+	if (!localport)
+		return 0;
+
 	lport = (struct lpfc_nvme_lport *)localport->private;
 
 	/* NVME rports are not preserved across devloss.
diff --git a/drivers/scsi/lpfc/lpfc_nvmet.c b/drivers/scsi/lpfc/lpfc_nvmet.c
index bbbd0f84160d..0b7c1a49e203 100644
--- a/drivers/scsi/lpfc/lpfc_nvmet.c
+++ b/drivers/scsi/lpfc/lpfc_nvmet.c
@@ -170,12 +170,14 @@ lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf)
 	struct lpfc_nvmet_tgtport *tgtp;
 	struct fc_frame_header *fc_hdr;
 	struct rqb_dmabuf *nvmebuf;
+	struct lpfc_nvmet_ctx_info *infop;
 	uint32_t *payload;
 	uint32_t size, oxid, sid, rc;
+	int cpu;
 	unsigned long iflag;
 
 	if (ctxp->txrdy) {
-		pci_pool_free(phba->txrdy_payload_pool, ctxp->txrdy,
+		dma_pool_free(phba->txrdy_payload_pool, ctxp->txrdy,
 			      ctxp->txrdy_phys);
 		ctxp->txrdy = NULL;
 		ctxp->txrdy_phys = 0;
@@ -267,11 +269,16 @@ lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf)
 	}
 	spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock, iflag);
 
-	spin_lock_irqsave(&phba->sli4_hba.nvmet_ctx_put_lock, iflag);
-	list_add_tail(&ctx_buf->list,
-		      &phba->sli4_hba.lpfc_nvmet_ctx_put_list);
-	phba->sli4_hba.nvmet_ctx_put_cnt++;
-	spin_unlock_irqrestore(&phba->sli4_hba.nvmet_ctx_put_lock, iflag);
+	/*
+	 * Use the CPU context list, from the MRQ the IO was received on
+	 * (ctxp->idx), to save context structure.
+	 */
+	cpu = smp_processor_id();
+	infop = lpfc_get_ctx_list(phba, cpu, ctxp->idx);
+	spin_lock_irqsave(&infop->nvmet_ctx_list_lock, iflag);
+	list_add_tail(&ctx_buf->list, &infop->nvmet_ctx_list);
+	infop->nvmet_ctx_list_cnt++;
+	spin_unlock_irqrestore(&infop->nvmet_ctx_list_lock, iflag);
 #endif
 }
 
@@ -552,7 +559,7 @@ lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
 		/* lpfc_nvmet_xmt_fcp_release() will recycle the context */
 	} else {
 		ctxp->entry_cnt++;
-		start_clean = offsetof(struct lpfc_iocbq, wqe);
+		start_clean = offsetof(struct lpfc_iocbq, iocb_flag);
 		memset(((char *)cmdwqe) + start_clean, 0,
 		       (sizeof(struct lpfc_iocbq) - start_clean));
 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
@@ -879,51 +886,54 @@ static struct nvmet_fc_target_template lpfc_tgttemplate = {
 };
 
 static void
-lpfc_nvmet_cleanup_io_context(struct lpfc_hba *phba)
+__lpfc_nvmet_clean_io_for_cpu(struct lpfc_hba *phba,
+		struct lpfc_nvmet_ctx_info *infop)
 {
 	struct lpfc_nvmet_ctxbuf *ctx_buf, *next_ctx_buf;
 	unsigned long flags;
 
-	spin_lock_irqsave(&phba->sli4_hba.nvmet_ctx_get_lock, flags);
-	spin_lock(&phba->sli4_hba.nvmet_ctx_put_lock);
+	spin_lock_irqsave(&infop->nvmet_ctx_list_lock, flags);
 	list_for_each_entry_safe(ctx_buf, next_ctx_buf,
-			&phba->sli4_hba.lpfc_nvmet_ctx_get_list, list) {
+				&infop->nvmet_ctx_list, list) {
 		spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock);
 		list_del_init(&ctx_buf->list);
 		spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock);
-		__lpfc_clear_active_sglq(phba,
-					 ctx_buf->sglq->sli4_lxritag);
+
+		__lpfc_clear_active_sglq(phba, ctx_buf->sglq->sli4_lxritag);
 		ctx_buf->sglq->state = SGL_FREED;
 		ctx_buf->sglq->ndlp = NULL;
 
 		spin_lock(&phba->sli4_hba.sgl_list_lock);
 		list_add_tail(&ctx_buf->sglq->list,
-			      &phba->sli4_hba.lpfc_nvmet_sgl_list);
+				&phba->sli4_hba.lpfc_nvmet_sgl_list);
 		spin_unlock(&phba->sli4_hba.sgl_list_lock);
 
 		lpfc_sli_release_iocbq(phba, ctx_buf->iocbq);
 		kfree(ctx_buf->context);
 	}
-	list_for_each_entry_safe(ctx_buf, next_ctx_buf,
-			&phba->sli4_hba.lpfc_nvmet_ctx_put_list, list) {
-		spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock);
-		list_del_init(&ctx_buf->list);
-		spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock);
-		__lpfc_clear_active_sglq(phba,
-					 ctx_buf->sglq->sli4_lxritag);
-		ctx_buf->sglq->state = SGL_FREED;
-		ctx_buf->sglq->ndlp = NULL;
+	spin_unlock_irqrestore(&infop->nvmet_ctx_list_lock, flags);
+}
 
-		spin_lock(&phba->sli4_hba.sgl_list_lock);
-		list_add_tail(&ctx_buf->sglq->list,
-			      &phba->sli4_hba.lpfc_nvmet_sgl_list);
-		spin_unlock(&phba->sli4_hba.sgl_list_lock);
+static void
+lpfc_nvmet_cleanup_io_context(struct lpfc_hba *phba)
+{
+	struct lpfc_nvmet_ctx_info *infop;
+	int i, j;
 
-		lpfc_sli_release_iocbq(phba, ctx_buf->iocbq);
-		kfree(ctx_buf->context);
+	/* The first context list, MRQ 0 CPU 0 */
+	infop = phba->sli4_hba.nvmet_ctx_info;
+	if (!infop)
+		return;
+
+	/* Cycle the the entire CPU context list for every MRQ */
+	for (i = 0; i < phba->cfg_nvmet_mrq; i++) {
+		for (j = 0; j < phba->sli4_hba.num_present_cpu; j++) {
+			__lpfc_nvmet_clean_io_for_cpu(phba, infop);
+			infop++; /* next */
+		}
 	}
-	spin_unlock(&phba->sli4_hba.nvmet_ctx_put_lock);
-	spin_unlock_irqrestore(&phba->sli4_hba.nvmet_ctx_get_lock, flags);
+	kfree(phba->sli4_hba.nvmet_ctx_info);
+	phba->sli4_hba.nvmet_ctx_info = NULL;
 }
 
 static int
@@ -932,15 +942,71 @@ lpfc_nvmet_setup_io_context(struct lpfc_hba *phba)
 	struct lpfc_nvmet_ctxbuf *ctx_buf;
 	struct lpfc_iocbq *nvmewqe;
 	union lpfc_wqe128 *wqe;
-	int i;
+	struct lpfc_nvmet_ctx_info *last_infop;
+	struct lpfc_nvmet_ctx_info *infop;
+	int i, j, idx;
 
 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME,
 			"6403 Allocate NVMET resources for %d XRIs\n",
 			phba->sli4_hba.nvmet_xri_cnt);
 
+	phba->sli4_hba.nvmet_ctx_info = kcalloc(
+		phba->sli4_hba.num_present_cpu * phba->cfg_nvmet_mrq,
+		sizeof(struct lpfc_nvmet_ctx_info), GFP_KERNEL);
+	if (!phba->sli4_hba.nvmet_ctx_info) {
+		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+				"6419 Failed allocate memory for "
+				"nvmet context lists\n");
+		return -ENOMEM;
+	}
+
+	/*
+	 * Assuming X CPUs in the system, and Y MRQs, allocate some
+	 * lpfc_nvmet_ctx_info structures as follows:
+	 *
+	 * cpu0/mrq0 cpu1/mrq0 ... cpuX/mrq0
+	 * cpu0/mrq1 cpu1/mrq1 ... cpuX/mrq1
+	 * ...
+	 * cpuX/mrqY cpuX/mrqY ... cpuX/mrqY
+	 *
+	 * Each line represents a MRQ "silo" containing an entry for
+	 * every CPU.
+	 *
+	 * MRQ X is initially assumed to be associated with CPU X, thus
+	 * contexts are initially distributed across all MRQs using
+	 * the MRQ index (N) as follows cpuN/mrqN. When contexts are
+	 * freed, the are freed to the MRQ silo based on the CPU number
+	 * of the IO completion. Thus a context that was allocated for MRQ A
+	 * whose IO completed on CPU B will be freed to cpuB/mrqA.
+	 */
+	infop = phba->sli4_hba.nvmet_ctx_info;
+	for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) {
+		for (j = 0; j < phba->cfg_nvmet_mrq; j++) {
+			INIT_LIST_HEAD(&infop->nvmet_ctx_list);
+			spin_lock_init(&infop->nvmet_ctx_list_lock);
+			infop->nvmet_ctx_list_cnt = 0;
+			infop++;
+		}
+	}
+
+	/*
+	 * Setup the next CPU context info ptr for each MRQ.
+	 * MRQ 0 will cycle thru CPUs 0 - X separately from
+	 * MRQ 1 cycling thru CPUs 0 - X, and so on.
+	 */
+	for (j = 0; j < phba->cfg_nvmet_mrq; j++) {
+		last_infop = lpfc_get_ctx_list(phba, 0, j);
+		for (i = phba->sli4_hba.num_present_cpu - 1;  i >= 0; i--) {
+			infop = lpfc_get_ctx_list(phba, i, j);
+			infop->nvmet_ctx_next_cpu = last_infop;
+			last_infop = infop;
+		}
+	}
+
 	/* For all nvmet xris, allocate resources needed to process a
 	 * received command on a per xri basis.
 	 */
+	idx = 0;
 	for (i = 0; i < phba->sli4_hba.nvmet_xri_cnt; i++) {
 		ctx_buf = kzalloc(sizeof(*ctx_buf), GFP_KERNEL);
 		if (!ctx_buf) {
@@ -977,7 +1043,6 @@ lpfc_nvmet_setup_io_context(struct lpfc_hba *phba)
 		/* Word 7 */
 		bf_set(wqe_ct, &wqe->generic.wqe_com, SLI4_CT_RPI);
 		bf_set(wqe_class, &wqe->generic.wqe_com, CLASS3);
-		bf_set(wqe_pu, &wqe->generic.wqe_com, 1);
 		/* Word 10 */
 		bf_set(wqe_nvme, &wqe->fcp_tsend.wqe_com, 1);
 		bf_set(wqe_ebde_cnt, &wqe->generic.wqe_com, 0);
@@ -995,12 +1060,35 @@ lpfc_nvmet_setup_io_context(struct lpfc_hba *phba)
 					"6407 Ran out of NVMET XRIs\n");
 			return -ENOMEM;
 		}
-		spin_lock(&phba->sli4_hba.nvmet_ctx_get_lock);
-		list_add_tail(&ctx_buf->list,
-			      &phba->sli4_hba.lpfc_nvmet_ctx_get_list);
-		spin_unlock(&phba->sli4_hba.nvmet_ctx_get_lock);
+
+		/*
+		 * Add ctx to MRQidx context list. Our initial assumption
+		 * is MRQidx will be associated with CPUidx. This association
+		 * can change on the fly.
+		 */
+		infop = lpfc_get_ctx_list(phba, idx, idx);
+		spin_lock(&infop->nvmet_ctx_list_lock);
+		list_add_tail(&ctx_buf->list, &infop->nvmet_ctx_list);
+		infop->nvmet_ctx_list_cnt++;
+		spin_unlock(&infop->nvmet_ctx_list_lock);
+
+		/* Spread ctx structures evenly across all MRQs */
+		idx++;
+		if (idx >= phba->cfg_nvmet_mrq)
+			idx = 0;
+	}
+
+	infop = phba->sli4_hba.nvmet_ctx_info;
+	for (j = 0; j < phba->cfg_nvmet_mrq; j++) {
+		for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) {
+			lpfc_printf_log(phba, KERN_INFO, LOG_NVME | LOG_INIT,
+					"6408 TOTAL NVMET ctx for CPU %d "
+					"MRQ %d: cnt %d nextcpu %p\n",
+					i, j, infop->nvmet_ctx_list_cnt,
+					infop->nvmet_ctx_next_cpu);
+			infop++;
+		}
 	}
-	phba->sli4_hba.nvmet_ctx_get_cnt = phba->sli4_hba.nvmet_xri_cnt;
 	return 0;
 }
 
@@ -1365,10 +1453,65 @@ dropit:
 #endif
 }
 
+static struct lpfc_nvmet_ctxbuf *
+lpfc_nvmet_replenish_context(struct lpfc_hba *phba,
+			     struct lpfc_nvmet_ctx_info *current_infop)
+{
+	struct lpfc_nvmet_ctxbuf *ctx_buf = NULL;
+	struct lpfc_nvmet_ctx_info *get_infop;
+	int i;
+
+	/*
+	 * The current_infop for the MRQ a NVME command IU was received
+	 * on is empty. Our goal is to replenish this MRQs context
+	 * list from a another CPUs.
+	 *
+	 * First we need to pick a context list to start looking on.
+	 * nvmet_ctx_start_cpu has available context the last time
+	 * we needed to replenish this CPU where nvmet_ctx_next_cpu
+	 * is just the next sequential CPU for this MRQ.
+	 */
+	if (current_infop->nvmet_ctx_start_cpu)
+		get_infop = current_infop->nvmet_ctx_start_cpu;
+	else
+		get_infop = current_infop->nvmet_ctx_next_cpu;
+
+	for (i = 0; i < phba->sli4_hba.num_present_cpu; i++) {
+		if (get_infop == current_infop) {
+			get_infop = get_infop->nvmet_ctx_next_cpu;
+			continue;
+		}
+		spin_lock(&get_infop->nvmet_ctx_list_lock);
+
+		/* Just take the entire context list, if there are any */
+		if (get_infop->nvmet_ctx_list_cnt) {
+			list_splice_init(&get_infop->nvmet_ctx_list,
+				    &current_infop->nvmet_ctx_list);
+			current_infop->nvmet_ctx_list_cnt =
+				get_infop->nvmet_ctx_list_cnt - 1;
+			get_infop->nvmet_ctx_list_cnt = 0;
+			spin_unlock(&get_infop->nvmet_ctx_list_lock);
+
+			current_infop->nvmet_ctx_start_cpu = get_infop;
+			list_remove_head(&current_infop->nvmet_ctx_list,
+					 ctx_buf, struct lpfc_nvmet_ctxbuf,
+					 list);
+			return ctx_buf;
+		}
+
+		/* Otherwise, move on to the next CPU for this MRQ */
+		spin_unlock(&get_infop->nvmet_ctx_list_lock);
+		get_infop = get_infop->nvmet_ctx_next_cpu;
+	}
+
+	/* Nothing found, all contexts for the MRQ are in-flight */
+	return NULL;
+}
+
 /**
  * lpfc_nvmet_unsol_fcp_buffer - Process an unsolicited event data buffer
  * @phba: pointer to lpfc hba data structure.
- * @pring: pointer to a SLI ring.
+ * @idx: relative index of MRQ vector
  * @nvmebuf: pointer to lpfc nvme command HBQ data structure.
  *
  * This routine is used for processing the WQE associated with a unsolicited
@@ -1380,22 +1523,26 @@ dropit:
  **/
 static void
 lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba,
-			    struct lpfc_sli_ring *pring,
+			    uint32_t idx,
 			    struct rqb_dmabuf *nvmebuf,
 			    uint64_t isr_timestamp)
 {
-#if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
 	struct lpfc_nvmet_rcv_ctx *ctxp;
 	struct lpfc_nvmet_tgtport *tgtp;
 	struct fc_frame_header *fc_hdr;
 	struct lpfc_nvmet_ctxbuf *ctx_buf;
+	struct lpfc_nvmet_ctx_info *current_infop;
 	uint32_t *payload;
 	uint32_t size, oxid, sid, rc, qno;
 	unsigned long iflag;
+	int current_cpu;
 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 	uint32_t id;
 #endif
 
+	if (!IS_ENABLED(CONFIG_NVME_TARGET_FC))
+		return;
+
 	ctx_buf = NULL;
 	if (!nvmebuf || !phba->targetport) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
@@ -1407,31 +1554,24 @@ lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba,
 		goto dropit;
 	}
 
-	spin_lock_irqsave(&phba->sli4_hba.nvmet_ctx_get_lock, iflag);
-	if (phba->sli4_hba.nvmet_ctx_get_cnt) {
-		list_remove_head(&phba->sli4_hba.lpfc_nvmet_ctx_get_list,
+	/*
+	 * Get a pointer to the context list for this MRQ based on
+	 * the CPU this MRQ IRQ is associated with. If the CPU association
+	 * changes from our initial assumption, the context list could
+	 * be empty, thus it would need to be replenished with the
+	 * context list from another CPU for this MRQ.
+	 */
+	current_cpu = smp_processor_id();
+	current_infop = lpfc_get_ctx_list(phba, current_cpu, idx);
+	spin_lock_irqsave(&current_infop->nvmet_ctx_list_lock, iflag);
+	if (current_infop->nvmet_ctx_list_cnt) {
+		list_remove_head(&current_infop->nvmet_ctx_list,
 				 ctx_buf, struct lpfc_nvmet_ctxbuf, list);
-		phba->sli4_hba.nvmet_ctx_get_cnt--;
+		current_infop->nvmet_ctx_list_cnt--;
 	} else {
-		spin_lock(&phba->sli4_hba.nvmet_ctx_put_lock);
-		if (phba->sli4_hba.nvmet_ctx_put_cnt) {
-			list_splice(&phba->sli4_hba.lpfc_nvmet_ctx_put_list,
-				    &phba->sli4_hba.lpfc_nvmet_ctx_get_list);
-			INIT_LIST_HEAD(&phba->sli4_hba.lpfc_nvmet_ctx_put_list);
-			phba->sli4_hba.nvmet_ctx_get_cnt =
-				phba->sli4_hba.nvmet_ctx_put_cnt;
-			phba->sli4_hba.nvmet_ctx_put_cnt = 0;
-			spin_unlock(&phba->sli4_hba.nvmet_ctx_put_lock);
-
-			list_remove_head(
-				&phba->sli4_hba.lpfc_nvmet_ctx_get_list,
-				ctx_buf, struct lpfc_nvmet_ctxbuf, list);
-			phba->sli4_hba.nvmet_ctx_get_cnt--;
-		} else {
-			spin_unlock(&phba->sli4_hba.nvmet_ctx_put_lock);
-		}
+		ctx_buf = lpfc_nvmet_replenish_context(phba, current_infop);
 	}
-	spin_unlock_irqrestore(&phba->sli4_hba.nvmet_ctx_get_lock, iflag);
+	spin_unlock_irqrestore(&current_infop->nvmet_ctx_list_lock, iflag);
 
 	fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt);
 	oxid = be16_to_cpu(fc_hdr->fh_ox_id);
@@ -1483,6 +1623,7 @@ lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba,
 	ctxp->size = size;
 	ctxp->oxid = oxid;
 	ctxp->sid = sid;
+	ctxp->idx = idx;
 	ctxp->state = LPFC_NVMET_STE_RCV;
 	ctxp->entry_cnt = 1;
 	ctxp->flag = 0;
@@ -1556,7 +1697,6 @@ dropit:
 
 	if (nvmebuf)
 		lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */
-#endif
 }
 
 /**
@@ -1591,7 +1731,7 @@ lpfc_nvmet_unsol_ls_event(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
 /**
  * lpfc_nvmet_unsol_fcp_event - Process an unsolicited event from an nvme nport
  * @phba: pointer to lpfc hba data structure.
- * @pring: pointer to a SLI ring.
+ * @idx: relative index of MRQ vector
  * @nvmebuf: pointer to received nvme data structure.
  *
  * This routine is used to process an unsolicited event received from a SLI
@@ -1602,7 +1742,7 @@ lpfc_nvmet_unsol_ls_event(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
  **/
 void
 lpfc_nvmet_unsol_fcp_event(struct lpfc_hba *phba,
-			   struct lpfc_sli_ring *pring,
+			   uint32_t idx,
 			   struct rqb_dmabuf *nvmebuf,
 			   uint64_t isr_timestamp)
 {
@@ -1610,7 +1750,7 @@ lpfc_nvmet_unsol_fcp_event(struct lpfc_hba *phba,
 		lpfc_rq_buf_free(phba, &nvmebuf->hbuf);
 		return;
 	}
-	lpfc_nvmet_unsol_fcp_buffer(phba, pring, nvmebuf,
+	lpfc_nvmet_unsol_fcp_buffer(phba, idx, nvmebuf,
 				    isr_timestamp);
 }
 
@@ -1863,6 +2003,7 @@ lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,
 		       nvmewqe->sli4_xritag);
 
 		/* Word 7 */
+		bf_set(wqe_pu, &wqe->fcp_tsend.wqe_com, 1);
 		bf_set(wqe_cmnd, &wqe->fcp_tsend.wqe_com, CMD_FCP_TSEND64_WQE);
 
 		/* Word 8 */
@@ -1939,7 +2080,7 @@ lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,
 
 	case NVMET_FCOP_WRITEDATA:
 		/* Words 0 - 2 : The first sg segment */
-		txrdy = pci_pool_alloc(phba->txrdy_payload_pool,
+		txrdy = dma_pool_alloc(phba->txrdy_payload_pool,
 				       GFP_KERNEL, &physaddr);
 		if (!txrdy) {
 			lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
@@ -1971,6 +2112,7 @@ lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,
 		       nvmewqe->sli4_xritag);
 
 		/* Word 7 */
+		bf_set(wqe_pu, &wqe->fcp_treceive.wqe_com, 1);
 		bf_set(wqe_ar, &wqe->fcp_treceive.wqe_com, 0);
 		bf_set(wqe_cmnd, &wqe->fcp_treceive.wqe_com,
 		       CMD_FCP_TRECEIVE64_WQE);
@@ -2054,6 +2196,7 @@ lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,
 		       nvmewqe->sli4_xritag);
 
 		/* Word 7 */
+		bf_set(wqe_pu, &wqe->fcp_trsp.wqe_com, 0);
 		bf_set(wqe_ag, &wqe->fcp_trsp.wqe_com, 1);
 		bf_set(wqe_cmnd, &wqe->fcp_trsp.wqe_com, CMD_FCP_TRSP64_WQE);
 
diff --git a/drivers/scsi/lpfc/lpfc_nvmet.h b/drivers/scsi/lpfc/lpfc_nvmet.h
index 48a76788b003..25a65b0bb7f3 100644
--- a/drivers/scsi/lpfc/lpfc_nvmet.h
+++ b/drivers/scsi/lpfc/lpfc_nvmet.h
@@ -74,6 +74,19 @@ struct lpfc_nvmet_tgtport {
 	atomic_t xmt_abort_rsp_error;
 };
 
+struct lpfc_nvmet_ctx_info {
+	struct list_head nvmet_ctx_list;
+	spinlock_t	nvmet_ctx_list_lock; /* lock per CPU */
+	struct lpfc_nvmet_ctx_info *nvmet_ctx_next_cpu;
+	struct lpfc_nvmet_ctx_info *nvmet_ctx_start_cpu;
+	uint16_t	nvmet_ctx_list_cnt;
+	char pad[16];  /* pad to a cache-line */
+};
+
+/* This retrieves the context info associated with the specified cpu / mrq */
+#define lpfc_get_ctx_list(phba, cpu, mrq)  \
+	(phba->sli4_hba.nvmet_ctx_info + ((cpu * phba->cfg_nvmet_mrq) + mrq))
+
 struct lpfc_nvmet_rcv_ctx {
 	union {
 		struct nvmefc_tgt_ls_req ls_req;
@@ -92,6 +105,7 @@ struct lpfc_nvmet_rcv_ctx {
 	uint16_t size;
 	uint16_t entry_cnt;
 	uint16_t cpu;
+	uint16_t idx;
 	uint16_t state;
 	/* States */
 #define LPFC_NVMET_STE_LS_RCV		1
diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c
index adc784539061..1a6f122bb25d 100644
--- a/drivers/scsi/lpfc/lpfc_scsi.c
+++ b/drivers/scsi/lpfc/lpfc_scsi.c
@@ -416,7 +416,7 @@ lpfc_new_scsi_buf_s3(struct lpfc_vport *vport, int num_to_alloc)
 		 * struct fcp_cmnd, struct fcp_rsp and the number of bde's
 		 * necessary to support the sg_tablesize.
 		 */
-		psb->data = pci_pool_zalloc(phba->lpfc_sg_dma_buf_pool,
+		psb->data = dma_pool_zalloc(phba->lpfc_sg_dma_buf_pool,
 					GFP_KERNEL, &psb->dma_handle);
 		if (!psb->data) {
 			kfree(psb);
@@ -427,7 +427,7 @@ lpfc_new_scsi_buf_s3(struct lpfc_vport *vport, int num_to_alloc)
 		/* Allocate iotag for psb->cur_iocbq. */
 		iotag = lpfc_sli_next_iotag(phba, &psb->cur_iocbq);
 		if (iotag == 0) {
-			pci_pool_free(phba->lpfc_sg_dma_buf_pool,
+			dma_pool_free(phba->lpfc_sg_dma_buf_pool,
 				      psb->data, psb->dma_handle);
 			kfree(psb);
 			break;
@@ -826,7 +826,7 @@ lpfc_new_scsi_buf_s4(struct lpfc_vport *vport, int num_to_alloc)
 		 * for the struct fcp_cmnd, struct fcp_rsp and the number
 		 * of bde's necessary to support the sg_tablesize.
 		 */
-		psb->data = pci_pool_zalloc(phba->lpfc_sg_dma_buf_pool,
+		psb->data = dma_pool_zalloc(phba->lpfc_sg_dma_buf_pool,
 						GFP_KERNEL, &psb->dma_handle);
 		if (!psb->data) {
 			kfree(psb);
@@ -839,7 +839,7 @@ lpfc_new_scsi_buf_s4(struct lpfc_vport *vport, int num_to_alloc)
 		 */
 		if (phba->cfg_enable_bg  && (((unsigned long)(psb->data) &
 		    (unsigned long)(SLI4_PAGE_SIZE - 1)) != 0)) {
-			pci_pool_free(phba->lpfc_sg_dma_buf_pool,
+			dma_pool_free(phba->lpfc_sg_dma_buf_pool,
 				      psb->data, psb->dma_handle);
 			kfree(psb);
 			break;
@@ -848,7 +848,7 @@ lpfc_new_scsi_buf_s4(struct lpfc_vport *vport, int num_to_alloc)
 
 		lxri = lpfc_sli4_next_xritag(phba);
 		if (lxri == NO_XRI) {
-			pci_pool_free(phba->lpfc_sg_dma_buf_pool,
+			dma_pool_free(phba->lpfc_sg_dma_buf_pool,
 				      psb->data, psb->dma_handle);
 			kfree(psb);
 			break;
@@ -857,7 +857,7 @@ lpfc_new_scsi_buf_s4(struct lpfc_vport *vport, int num_to_alloc)
 		/* Allocate iotag for psb->cur_iocbq. */
 		iotag = lpfc_sli_next_iotag(phba, &psb->cur_iocbq);
 		if (iotag == 0) {
-			pci_pool_free(phba->lpfc_sg_dma_buf_pool,
+			dma_pool_free(phba->lpfc_sg_dma_buf_pool,
 				      psb->data, psb->dma_handle);
 			kfree(psb);
 			lpfc_printf_log(phba, KERN_ERR, LOG_FCP,
diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c
index e948ea05fd33..8b119f87b51d 100644
--- a/drivers/scsi/lpfc/lpfc_sli.c
+++ b/drivers/scsi/lpfc/lpfc_sli.c
@@ -80,8 +80,8 @@ static int lpfc_sli4_fp_handle_cqe(struct lpfc_hba *, struct lpfc_queue *,
 				    struct lpfc_cqe *);
 static int lpfc_sli4_post_sgl_list(struct lpfc_hba *, struct list_head *,
 				       int);
-static void lpfc_sli4_hba_handle_eqe(struct lpfc_hba *, struct lpfc_eqe *,
-			uint32_t);
+static int lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba,
+				    struct lpfc_eqe *eqe, uint32_t qidx);
 static bool lpfc_sli4_mbox_completions_pending(struct lpfc_hba *phba);
 static bool lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba);
 static int lpfc_sli4_abort_nvme_io(struct lpfc_hba *phba,
@@ -106,7 +106,7 @@ lpfc_get_iocb_from_iocbq(struct lpfc_iocbq *iocbq)
  * -ENOMEM.
  * The caller is expected to hold the hbalock when calling this routine.
  **/
-static uint32_t
+static int
 lpfc_sli4_wq_put(struct lpfc_queue *q, union lpfc_wqe *wqe)
 {
 	union lpfc_wqe *temp_wqe;
@@ -123,7 +123,7 @@ lpfc_sli4_wq_put(struct lpfc_queue *q, union lpfc_wqe *wqe)
 	idx = ((q->host_index + 1) % q->entry_count);
 	if (idx == q->hba_index) {
 		q->WQ_overflow++;
-		return -ENOMEM;
+		return -EBUSY;
 	}
 	q->WQ_posted++;
 	/* set consumption flag every once in a while */
@@ -10741,7 +10741,7 @@ lpfc_sli4_abort_nvme_io(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
 	abtsiocbp->vport = vport;
 	abtsiocbp->wqe_cmpl = lpfc_nvme_abort_fcreq_cmpl;
 	retval = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, abtsiocbp);
-	if (retval == IOCB_ERROR) {
+	if (retval) {
 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME,
 				 "6147 Failed abts issue_wqe with status x%x "
 				 "for oxid x%x\n",
@@ -13010,7 +13010,7 @@ lpfc_sli4_sp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
  * completion queue, and then return.
  *
  **/
-static void
+static int
 lpfc_sli4_sp_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe,
 	struct lpfc_queue *speq)
 {
@@ -13034,7 +13034,7 @@ lpfc_sli4_sp_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe,
 			lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
 					"0365 Slow-path CQ identifier "
 					"(%d) does not exist\n", cqid);
-		return;
+		return 0;
 	}
 
 	/* Save EQ associated with this CQ */
@@ -13071,7 +13071,7 @@ lpfc_sli4_sp_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe,
 		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
 				"0370 Invalid completion queue type (%d)\n",
 				cq->type);
-		return;
+		return 0;
 	}
 
 	/* Catch the no cq entry condition, log an error */
@@ -13086,6 +13086,8 @@ lpfc_sli4_sp_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe,
 	/* wake up worker thread if there are works to be done */
 	if (workposted)
 		lpfc_worker_wake_up(phba);
+
+	return ecount;
 }
 
 /**
@@ -13289,7 +13291,7 @@ lpfc_sli4_nvmet_handle_rcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
 		if (fc_hdr->fh_type == FC_TYPE_FCP) {
 			dma_buf->bytes_recv = bf_get(lpfc_rcqe_length,  rcqe);
 			lpfc_nvmet_unsol_fcp_event(
-				phba, phba->sli4_hba.els_wq->pring, dma_buf,
+				phba, idx, dma_buf,
 				cq->assoc_qp->isr_timestamp);
 			return false;
 		}
@@ -13393,7 +13395,7 @@ lpfc_sli4_fp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
  * queue and process all the entries on the completion queue, rearm the
  * completion queue, and then return.
  **/
-static void
+static int
 lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe,
 			uint32_t qidx)
 {
@@ -13409,7 +13411,7 @@ lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe,
 				"event: majorcode=x%x, minorcode=x%x\n",
 				bf_get_le32(lpfc_eqe_major_code, eqe),
 				bf_get_le32(lpfc_eqe_minor_code, eqe));
-		return;
+		return 0;
 	}
 
 	/* Get the reference to the corresponding CQ */
@@ -13446,8 +13448,9 @@ lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe,
 
 	/* Otherwise this is a Slow path event */
 	if (cq == NULL) {
-		lpfc_sli4_sp_handle_eqe(phba, eqe, phba->sli4_hba.hba_eq[qidx]);
-		return;
+		ecount = lpfc_sli4_sp_handle_eqe(phba, eqe,
+						 phba->sli4_hba.hba_eq[qidx]);
+		return ecount;
 	}
 
 process_cq:
@@ -13456,7 +13459,7 @@ process_cq:
 				"0368 Miss-matched fast-path completion "
 				"queue identifier: eqcqid=%d, fcpcqid=%d\n",
 				cqid, cq->queue_id);
-		return;
+		return 0;
 	}
 
 	/* Save EQ associated with this CQ */
@@ -13486,6 +13489,8 @@ process_cq:
 	/* wake up worker thread if there are works to be done */
 	if (workposted)
 		lpfc_worker_wake_up(phba);
+
+	return ecount;
 }
 
 static void
@@ -13706,6 +13711,7 @@ lpfc_sli4_hba_intr_handler(int irq, void *dev_id)
 	struct lpfc_eqe *eqe;
 	unsigned long iflag;
 	int ecount = 0;
+	int ccount = 0;
 	int hba_eqidx;
 
 	/* Get the driver's phba structure from the dev_id */
@@ -13757,8 +13763,9 @@ lpfc_sli4_hba_intr_handler(int irq, void *dev_id)
 		if (eqe == NULL)
 			break;
 
-		lpfc_sli4_hba_handle_eqe(phba, eqe, hba_eqidx);
-		if (!(++ecount % fpeq->entry_repost))
+		ccount += lpfc_sli4_hba_handle_eqe(phba, eqe, hba_eqidx);
+		if (!(++ecount % fpeq->entry_repost) ||
+		    ccount > LPFC_MAX_ISR_CQE)
 			break;
 		fpeq->EQ_processed++;
 	}
@@ -17051,7 +17058,7 @@ lpfc_sli4_mds_loopback_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 	struct lpfc_dmabuf *pcmd = cmdiocb->context2;
 
 	if (pcmd && pcmd->virt)
-		pci_pool_free(phba->lpfc_drb_pool, pcmd->virt, pcmd->phys);
+		dma_pool_free(phba->lpfc_drb_pool, pcmd->virt, pcmd->phys);
 	kfree(pcmd);
 	lpfc_sli_release_iocbq(phba, cmdiocb);
 }
@@ -17079,7 +17086,7 @@ lpfc_sli4_handle_mds_loopback(struct lpfc_vport *vport,
 	/* Allocate buffer for command payload */
 	pcmd = kmalloc(sizeof(struct lpfc_dmabuf), GFP_KERNEL);
 	if (pcmd)
-		pcmd->virt = pci_pool_alloc(phba->lpfc_drb_pool, GFP_KERNEL,
+		pcmd->virt = dma_pool_alloc(phba->lpfc_drb_pool, GFP_KERNEL,
 					    &pcmd->phys);
 	if (!pcmd || !pcmd->virt)
 		goto exit;
@@ -17128,7 +17135,7 @@ exit:
 	lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
 			"2023 Unable to process MDS loopback frame\n");
 	if (pcmd && pcmd->virt)
-		pci_pool_free(phba->lpfc_drb_pool, pcmd->virt, pcmd->phys);
+		dma_pool_free(phba->lpfc_drb_pool, pcmd->virt, pcmd->phys);
 	kfree(pcmd);
 	lpfc_sli_release_iocbq(phba, iocbq);
 	lpfc_in_buf_free(phba, &dmabuf->dbuf);
@@ -18888,6 +18895,7 @@ lpfc_sli4_issue_wqe(struct lpfc_hba *phba, uint32_t ring_number,
 	struct lpfc_sglq *sglq;
 	struct lpfc_sli_ring *pring;
 	unsigned long iflags;
+	uint32_t ret = 0;
 
 	/* NVME_LS and NVME_LS ABTS requests. */
 	if (pwqe->iocb_flag & LPFC_IO_NVME_LS) {
@@ -18906,10 +18914,12 @@ lpfc_sli4_issue_wqe(struct lpfc_hba *phba, uint32_t ring_number,
 		}
 		bf_set(wqe_xri_tag, &pwqe->wqe.xmit_bls_rsp.wqe_com,
 		       pwqe->sli4_xritag);
-		if (lpfc_sli4_wq_put(phba->sli4_hba.nvmels_wq, wqe)) {
+		ret = lpfc_sli4_wq_put(phba->sli4_hba.nvmels_wq, wqe);
+		if (ret) {
 			spin_unlock_irqrestore(&pring->ring_lock, iflags);
-			return WQE_ERROR;
+			return ret;
 		}
+
 		lpfc_sli_ringtxcmpl_put(phba, pring, pwqe);
 		spin_unlock_irqrestore(&pring->ring_lock, iflags);
 		return 0;
@@ -18924,9 +18934,10 @@ lpfc_sli4_issue_wqe(struct lpfc_hba *phba, uint32_t ring_number,
 		wq = phba->sli4_hba.nvme_wq[pwqe->hba_wqidx];
 		bf_set(wqe_cqid, &wqe->generic.wqe_com,
 		      phba->sli4_hba.nvme_cq[pwqe->hba_wqidx]->queue_id);
-		if (lpfc_sli4_wq_put(wq, wqe)) {
+		ret = lpfc_sli4_wq_put(wq, wqe);
+		if (ret) {
 			spin_unlock_irqrestore(&pring->ring_lock, iflags);
-			return WQE_ERROR;
+			return ret;
 		}
 		lpfc_sli_ringtxcmpl_put(phba, pring, pwqe);
 		spin_unlock_irqrestore(&pring->ring_lock, iflags);
@@ -18950,9 +18961,10 @@ lpfc_sli4_issue_wqe(struct lpfc_hba *phba, uint32_t ring_number,
 		wq = phba->sli4_hba.nvme_wq[pwqe->hba_wqidx];
 		bf_set(wqe_cqid, &wqe->generic.wqe_com,
 		      phba->sli4_hba.nvme_cq[pwqe->hba_wqidx]->queue_id);
-		if (lpfc_sli4_wq_put(wq, wqe)) {
+		ret = lpfc_sli4_wq_put(wq, wqe);
+		if (ret) {
 			spin_unlock_irqrestore(&pring->ring_lock, iflags);
-			return WQE_ERROR;
+			return ret;
 		}
 		lpfc_sli_ringtxcmpl_put(phba, pring, pwqe);
 		spin_unlock_irqrestore(&pring->ring_lock, iflags);
diff --git a/drivers/scsi/lpfc/lpfc_sli4.h b/drivers/scsi/lpfc/lpfc_sli4.h
index 7a1d74e9e877..60200385fe00 100644
--- a/drivers/scsi/lpfc/lpfc_sli4.h
+++ b/drivers/scsi/lpfc/lpfc_sli4.h
@@ -158,6 +158,7 @@ struct lpfc_queue {
 #define LPFC_MQ_REPOST		8
 #define LPFC_CQ_REPOST		64
 #define LPFC_RQ_REPOST		64
+#define LPFC_MAX_ISR_CQE	64
 #define LPFC_RELEASE_NOTIFICATION_INTERVAL	32  /* For WQs */
 	uint32_t queue_id;	/* Queue ID assigned by the hardware */
 	uint32_t assoc_qid;     /* Queue ID associated with, for CQ/WQ/MQ */
@@ -419,6 +420,20 @@ struct lpfc_hba_eq_hdl {
 #define LPFC_MULTI_CPU_AFFINITY 0xffffffff
 };
 
+/*BB Credit recovery value*/
+struct lpfc_bbscn_params {
+	uint32_t word0;
+#define lpfc_bbscn_min_SHIFT		0
+#define lpfc_bbscn_min_MASK		0x0000000F
+#define lpfc_bbscn_min_WORD		word0
+#define lpfc_bbscn_max_SHIFT		4
+#define lpfc_bbscn_max_MASK		0x0000000F
+#define lpfc_bbscn_max_WORD		word0
+#define lpfc_bbscn_def_SHIFT		8
+#define lpfc_bbscn_def_MASK		0x0000000F
+#define lpfc_bbscn_def_WORD		word0
+};
+
 /* Port Capabilities for SLI4 Parameters */
 struct lpfc_pc_sli4_params {
 	uint32_t supported;
@@ -550,6 +565,7 @@ struct lpfc_sli4_hba {
 	uint32_t ue_to_rp;
 	struct lpfc_register sli_intf;
 	struct lpfc_pc_sli4_params pc_sli4_params;
+	struct lpfc_bbscn_params bbscn_params;
 	struct lpfc_hba_eq_hdl *hba_eq_hdl; /* HBA per-WQ handle */
 
 	/* Pointers to the constructed SLI4 queues */
@@ -621,8 +637,6 @@ struct lpfc_sli4_hba {
 	uint16_t scsi_xri_start;
 	uint16_t els_xri_cnt;
 	uint16_t nvmet_xri_cnt;
-	uint16_t nvmet_ctx_get_cnt;
-	uint16_t nvmet_ctx_put_cnt;
 	uint16_t nvmet_io_wait_cnt;
 	uint16_t nvmet_io_wait_total;
 	struct list_head lpfc_els_sgl_list;
@@ -631,9 +645,8 @@ struct lpfc_sli4_hba {
 	struct list_head lpfc_abts_nvmet_ctx_list;
 	struct list_head lpfc_abts_scsi_buf_list;
 	struct list_head lpfc_abts_nvme_buf_list;
-	struct list_head lpfc_nvmet_ctx_get_list;
-	struct list_head lpfc_nvmet_ctx_put_list;
 	struct list_head lpfc_nvmet_io_wait_list;
+	struct lpfc_nvmet_ctx_info *nvmet_ctx_info;
 	struct lpfc_sglq **lpfc_sglq_active_list;
 	struct list_head lpfc_rpi_hdr_list;
 	unsigned long *rpi_bmask;
@@ -664,8 +677,6 @@ struct lpfc_sli4_hba {
 	spinlock_t abts_nvme_buf_list_lock; /* list of aborted SCSI IOs */
 	spinlock_t abts_scsi_buf_list_lock; /* list of aborted SCSI IOs */
 	spinlock_t sgl_list_lock; /* list of aborted els IOs */
-	spinlock_t nvmet_ctx_get_lock; /* list of avail XRI contexts */
-	spinlock_t nvmet_ctx_put_lock; /* list of avail XRI contexts */
 	spinlock_t nvmet_io_wait_lock; /* IOs waiting for ctx resources */
 	uint32_t physical_port;
 
diff --git a/drivers/scsi/lpfc/lpfc_version.h b/drivers/scsi/lpfc/lpfc_version.h
index c6a24c3e2d5e..6aa192b3e4bf 100644
--- a/drivers/scsi/lpfc/lpfc_version.h
+++ b/drivers/scsi/lpfc/lpfc_version.h
@@ -20,7 +20,7 @@
  * included with this package.                                     *
  *******************************************************************/
 
-#define LPFC_DRIVER_VERSION "11.4.0.1"
+#define LPFC_DRIVER_VERSION "11.4.0.3"
 #define LPFC_DRIVER_NAME		"lpfc"
 
 /* Used for SLI 2/3 */
diff --git a/drivers/scsi/mac53c94.c b/drivers/scsi/mac53c94.c
index a6682c508c4c..8c4d3003b68b 100644
--- a/drivers/scsi/mac53c94.c
+++ b/drivers/scsi/mac53c94.c
@@ -448,15 +448,14 @@ static int mac53c94_probe(struct macio_dev *mdev, const struct of_device_id *mat
 		ioremap(macio_resource_start(mdev, 1), 0x1000);
 	state->dmaintr = macio_irq(mdev, 1);
 	if (state->regs == NULL || state->dma == NULL) {
-		printk(KERN_ERR "mac53c94: ioremap failed for %s\n",
-		       node->full_name);
+		printk(KERN_ERR "mac53c94: ioremap failed for %pOF\n", node);
 		goto out_free;
 	}
 
 	clkprop = of_get_property(node, "clock-frequency", &proplen);
        	if (clkprop == NULL || proplen != sizeof(int)) {
-       		printk(KERN_ERR "%s: can't get clock frequency, "
-       		       "assuming 25MHz\n", node->full_name);
+       		printk(KERN_ERR "%pOF: can't get clock frequency, "
+       		       "assuming 25MHz\n", node);
        		state->clk_freq = 25000000;
        	} else
        		state->clk_freq = *(int *)clkprop;
@@ -469,7 +468,7 @@ static int mac53c94_probe(struct macio_dev *mdev, const struct of_device_id *mat
        				sizeof(struct dbdma_cmd), GFP_KERNEL);
        	if (dma_cmd_space == 0) {
        		printk(KERN_ERR "mac53c94: couldn't allocate dma "
-       		       "command space for %s\n", node->full_name);
+       		       "command space for %pOF\n", node);
 		rc = -ENOMEM;
        		goto out_free;
        	}
@@ -481,8 +480,8 @@ static int mac53c94_probe(struct macio_dev *mdev, const struct of_device_id *mat
 	mac53c94_init(state);
 
 	if (request_irq(state->intr, do_mac53c94_interrupt, 0, "53C94",state)) {
-		printk(KERN_ERR "mac53C94: can't get irq %d for %s\n",
-		       state->intr, node->full_name);
+		printk(KERN_ERR "mac53C94: can't get irq %d for %pOF\n",
+		       state->intr, node);
 		goto out_free_dma;
 	}
 
diff --git a/drivers/scsi/mac_esp.c b/drivers/scsi/mac_esp.c
index cdb61eaa2d1f..eb551f3cc471 100644
--- a/drivers/scsi/mac_esp.c
+++ b/drivers/scsi/mac_esp.c
@@ -348,26 +348,24 @@ static void mac_esp_send_pio_cmd(struct esp *esp, u32 addr, u32 esp_count,
 				 u32 dma_count, int write, u8 cmd)
 {
 	struct mac_esp_priv *mep = MAC_ESP_GET_PRIV(esp);
-	u8 *fifo = esp->regs + ESP_FDATA * 16;
+	u8 __iomem *fifo = esp->regs + ESP_FDATA * 16;
+	u8 phase = esp->sreg & ESP_STAT_PMASK;
 
 	cmd &= ~ESP_CMD_DMA;
 	mep->error = 0;
 
 	if (write) {
+		u8 *dst = (u8 *)addr;
+		u8 mask = ~(phase == ESP_MIP ? ESP_INTR_FDONE : ESP_INTR_BSERV);
+
 		scsi_esp_cmd(esp, cmd);
 
 		while (1) {
-			unsigned int n;
-
-			n = mac_esp_wait_for_fifo(esp);
-			if (!n)
+			if (!mac_esp_wait_for_fifo(esp))
 				break;
 
-			if (n > esp_count)
-				n = esp_count;
-			esp_count -= n;
-
-			MAC_ESP_PIO_LOOP("%2@,%0@+", n);
+			*dst++ = esp_read8(ESP_FDATA);
+			--esp_count;
 
 			if (!esp_count)
 				break;
@@ -375,14 +373,17 @@ static void mac_esp_send_pio_cmd(struct esp *esp, u32 addr, u32 esp_count,
 			if (mac_esp_wait_for_intr(esp))
 				break;
 
-			if (((esp->sreg & ESP_STAT_PMASK) != ESP_DIP) &&
-			    ((esp->sreg & ESP_STAT_PMASK) != ESP_MIP))
+			if ((esp->sreg & ESP_STAT_PMASK) != phase)
 				break;
 
 			esp->ireg = esp_read8(ESP_INTRPT);
-			if ((esp->ireg & (ESP_INTR_DC | ESP_INTR_BSERV)) !=
-			    ESP_INTR_BSERV)
+			if (esp->ireg & mask) {
+				mep->error = 1;
 				break;
+			}
+
+			if (phase == ESP_MIP)
+				scsi_esp_cmd(esp, ESP_CMD_MOK);
 
 			scsi_esp_cmd(esp, ESP_CMD_TI);
 		}
@@ -402,14 +403,14 @@ static void mac_esp_send_pio_cmd(struct esp *esp, u32 addr, u32 esp_count,
 			if (mac_esp_wait_for_intr(esp))
 				break;
 
-			if (((esp->sreg & ESP_STAT_PMASK) != ESP_DOP) &&
-			    ((esp->sreg & ESP_STAT_PMASK) != ESP_MOP))
+			if ((esp->sreg & ESP_STAT_PMASK) != phase)
 				break;
 
 			esp->ireg = esp_read8(ESP_INTRPT);
-			if ((esp->ireg & (ESP_INTR_DC | ESP_INTR_BSERV)) !=
-			    ESP_INTR_BSERV)
+			if (esp->ireg & ~ESP_INTR_BSERV) {
+				mep->error = 1;
 				break;
+			}
 
 			n = MAC_ESP_FIFO_SIZE -
 			    (esp_read8(ESP_FFLAGS) & ESP_FF_FBYTES);
diff --git a/drivers/scsi/mac_scsi.c b/drivers/scsi/mac_scsi.c
index 196acc79714b..dd6057359d7c 100644
--- a/drivers/scsi/mac_scsi.c
+++ b/drivers/scsi/mac_scsi.c
@@ -41,7 +41,7 @@
 #define NCR5380_intr                    macscsi_intr
 #define NCR5380_queue_command           macscsi_queue_command
 #define NCR5380_abort                   macscsi_abort
-#define NCR5380_bus_reset               macscsi_bus_reset
+#define NCR5380_host_reset              macscsi_host_reset
 #define NCR5380_info                    macscsi_info
 
 #include "NCR5380.h"
@@ -328,7 +328,7 @@ static struct scsi_host_template mac_scsi_template = {
 	.info			= macscsi_info,
 	.queuecommand		= macscsi_queue_command,
 	.eh_abort_handler	= macscsi_abort,
-	.eh_bus_reset_handler	= macscsi_bus_reset,
+	.eh_host_reset_handler	= macscsi_host_reset,
 	.can_queue		= 16,
 	.this_id		= 7,
 	.sg_tablesize		= 1,
diff --git a/drivers/scsi/megaraid.c b/drivers/scsi/megaraid.c
index 3c63c292cb92..7195cff51d4c 100644
--- a/drivers/scsi/megaraid.c
+++ b/drivers/scsi/megaraid.c
@@ -311,13 +311,15 @@ mega_query_adapter(adapter_t *adapter)
 	   right 8 bits making them zero. This 0 value was hardcoded to fix
 	   sparse warnings. */
 	if (adapter->product_info.subsysvid == PCI_VENDOR_ID_HP) {
-		sprintf (adapter->fw_version, "%c%d%d.%d%d",
+		snprintf(adapter->fw_version, sizeof(adapter->fw_version),
+			 "%c%d%d.%d%d",
 			 adapter->product_info.fw_version[2],
 			 0,
 			 adapter->product_info.fw_version[1] & 0x0f,
 			 0,
 			 adapter->product_info.fw_version[0] & 0x0f);
-		sprintf (adapter->bios_version, "%c%d%d.%d%d",
+		snprintf(adapter->bios_version, sizeof(adapter->fw_version),
+			 "%c%d%d.%d%d",
 			 adapter->product_info.bios_version[2],
 			 0,
 			 adapter->product_info.bios_version[1] & 0x0f,
diff --git a/drivers/scsi/megaraid/megaraid_mbox.c b/drivers/scsi/megaraid/megaraid_mbox.c
index f0987f22ea70..ec3c43854978 100644
--- a/drivers/scsi/megaraid/megaraid_mbox.c
+++ b/drivers/scsi/megaraid/megaraid_mbox.c
@@ -341,8 +341,6 @@ static struct scsi_host_template megaraid_template_g = {
 	.proc_name			= "megaraid",
 	.queuecommand			= megaraid_queue_command,
 	.eh_abort_handler		= megaraid_abort_handler,
-	.eh_device_reset_handler	= megaraid_reset_handler,
-	.eh_bus_reset_handler		= megaraid_reset_handler,
 	.eh_host_reset_handler		= megaraid_reset_handler,
 	.change_queue_depth		= scsi_change_queue_depth,
 	.use_clustering			= ENABLE_CLUSTERING,
@@ -1153,8 +1151,8 @@ megaraid_mbox_setup_dma_pools(adapter_t *adapter)
 
 
 	// Allocate memory for 16-bytes aligned mailboxes
-	raid_dev->mbox_pool_handle = pci_pool_create("megaraid mbox pool",
-						adapter->pdev,
+	raid_dev->mbox_pool_handle = dma_pool_create("megaraid mbox pool",
+						&adapter->pdev->dev,
 						sizeof(mbox64_t) + 16,
 						16, 0);
 
@@ -1164,7 +1162,7 @@ megaraid_mbox_setup_dma_pools(adapter_t *adapter)
 
 	mbox_pci_blk = raid_dev->mbox_pool;
 	for (i = 0; i < MBOX_MAX_SCSI_CMDS; i++) {
-		mbox_pci_blk[i].vaddr = pci_pool_alloc(
+		mbox_pci_blk[i].vaddr = dma_pool_alloc(
 						raid_dev->mbox_pool_handle,
 						GFP_KERNEL,
 						&mbox_pci_blk[i].dma_addr);
@@ -1181,8 +1179,8 @@ megaraid_mbox_setup_dma_pools(adapter_t *adapter)
 	 * share common memory pool. Passthru structures piggyback on memory
 	 * allocted to extended passthru since passthru is smaller of the two
 	 */
-	raid_dev->epthru_pool_handle = pci_pool_create("megaraid mbox pthru",
-			adapter->pdev, sizeof(mraid_epassthru_t), 128, 0);
+	raid_dev->epthru_pool_handle = dma_pool_create("megaraid mbox pthru",
+			&adapter->pdev->dev, sizeof(mraid_epassthru_t), 128, 0);
 
 	if (raid_dev->epthru_pool_handle == NULL) {
 		goto fail_setup_dma_pool;
@@ -1190,7 +1188,7 @@ megaraid_mbox_setup_dma_pools(adapter_t *adapter)
 
 	epthru_pci_blk = raid_dev->epthru_pool;
 	for (i = 0; i < MBOX_MAX_SCSI_CMDS; i++) {
-		epthru_pci_blk[i].vaddr = pci_pool_alloc(
+		epthru_pci_blk[i].vaddr = dma_pool_alloc(
 						raid_dev->epthru_pool_handle,
 						GFP_KERNEL,
 						&epthru_pci_blk[i].dma_addr);
@@ -1202,8 +1200,8 @@ megaraid_mbox_setup_dma_pools(adapter_t *adapter)
 
 	// Allocate memory for each scatter-gather list. Request for 512 bytes
 	// alignment for each sg list
-	raid_dev->sg_pool_handle = pci_pool_create("megaraid mbox sg",
-					adapter->pdev,
+	raid_dev->sg_pool_handle = dma_pool_create("megaraid mbox sg",
+					&adapter->pdev->dev,
 					sizeof(mbox_sgl64) * MBOX_MAX_SG_SIZE,
 					512, 0);
 
@@ -1213,7 +1211,7 @@ megaraid_mbox_setup_dma_pools(adapter_t *adapter)
 
 	sg_pci_blk = raid_dev->sg_pool;
 	for (i = 0; i < MBOX_MAX_SCSI_CMDS; i++) {
-		sg_pci_blk[i].vaddr = pci_pool_alloc(
+		sg_pci_blk[i].vaddr = dma_pool_alloc(
 						raid_dev->sg_pool_handle,
 						GFP_KERNEL,
 						&sg_pci_blk[i].dma_addr);
@@ -1249,29 +1247,29 @@ megaraid_mbox_teardown_dma_pools(adapter_t *adapter)
 
 	sg_pci_blk = raid_dev->sg_pool;
 	for (i = 0; i < MBOX_MAX_SCSI_CMDS && sg_pci_blk[i].vaddr; i++) {
-		pci_pool_free(raid_dev->sg_pool_handle, sg_pci_blk[i].vaddr,
+		dma_pool_free(raid_dev->sg_pool_handle, sg_pci_blk[i].vaddr,
 			sg_pci_blk[i].dma_addr);
 	}
 	if (raid_dev->sg_pool_handle)
-		pci_pool_destroy(raid_dev->sg_pool_handle);
+		dma_pool_destroy(raid_dev->sg_pool_handle);
 
 
 	epthru_pci_blk = raid_dev->epthru_pool;
 	for (i = 0; i < MBOX_MAX_SCSI_CMDS && epthru_pci_blk[i].vaddr; i++) {
-		pci_pool_free(raid_dev->epthru_pool_handle,
+		dma_pool_free(raid_dev->epthru_pool_handle,
 			epthru_pci_blk[i].vaddr, epthru_pci_blk[i].dma_addr);
 	}
 	if (raid_dev->epthru_pool_handle)
-		pci_pool_destroy(raid_dev->epthru_pool_handle);
+		dma_pool_destroy(raid_dev->epthru_pool_handle);
 
 
 	mbox_pci_blk = raid_dev->mbox_pool;
 	for (i = 0; i < MBOX_MAX_SCSI_CMDS && mbox_pci_blk[i].vaddr; i++) {
-		pci_pool_free(raid_dev->mbox_pool_handle,
+		dma_pool_free(raid_dev->mbox_pool_handle,
 			mbox_pci_blk[i].vaddr, mbox_pci_blk[i].dma_addr);
 	}
 	if (raid_dev->mbox_pool_handle)
-		pci_pool_destroy(raid_dev->mbox_pool_handle);
+		dma_pool_destroy(raid_dev->mbox_pool_handle);
 
 	return;
 }
diff --git a/drivers/scsi/megaraid/megaraid_mm.c b/drivers/scsi/megaraid/megaraid_mm.c
index 544d6f7e6138..65b6f6ace3a5 100644
--- a/drivers/scsi/megaraid/megaraid_mm.c
+++ b/drivers/scsi/megaraid/megaraid_mm.c
@@ -574,7 +574,7 @@ mraid_mm_attach_buf(mraid_mmadp_t *adp, uioc_t *kioc, int xferlen)
 
 	kioc->pool_index	= right_pool;
 	kioc->free_buf		= 1;
-	kioc->buf_vaddr		= pci_pool_alloc(pool->handle, GFP_ATOMIC,
+	kioc->buf_vaddr		= dma_pool_alloc(pool->handle, GFP_ATOMIC,
 							&kioc->buf_paddr);
 	spin_unlock_irqrestore(&pool->lock, flags);
 
@@ -658,7 +658,7 @@ mraid_mm_dealloc_kioc(mraid_mmadp_t *adp, uioc_t *kioc)
 		 * not in use
 		 */
 		if (kioc->free_buf == 1)
-			pci_pool_free(pool->handle, kioc->buf_vaddr, 
+			dma_pool_free(pool->handle, kioc->buf_vaddr, 
 							kioc->buf_paddr);
 		else
 			pool->in_use = 0;
@@ -940,8 +940,8 @@ mraid_mm_register_adp(mraid_mmadp_t *lld_adp)
 						GFP_KERNEL);
 	adapter->mbox_list	= kmalloc(sizeof(mbox64_t) * lld_adp->max_kioc,
 						GFP_KERNEL);
-	adapter->pthru_dma_pool = pci_pool_create("megaraid mm pthru pool",
-						adapter->pdev,
+	adapter->pthru_dma_pool = dma_pool_create("megaraid mm pthru pool",
+						&adapter->pdev->dev,
 						sizeof(mraid_passthru_t),
 						16, 0);
 
@@ -970,7 +970,7 @@ mraid_mm_register_adp(mraid_mmadp_t *lld_adp)
 
 		kioc		= adapter->kioc_list + i;
 		kioc->cmdbuf	= (uint64_t)(unsigned long)(mbox_list + i);
-		kioc->pthru32	= pci_pool_alloc(adapter->pthru_dma_pool,
+		kioc->pthru32	= dma_pool_alloc(adapter->pthru_dma_pool,
 						GFP_KERNEL, &kioc->pthru32_h);
 
 		if (!kioc->pthru32) {
@@ -1006,7 +1006,7 @@ pthru_dma_pool_error:
 	for (i = 0; i < lld_adp->max_kioc; i++) {
 		kioc = adapter->kioc_list + i;
 		if (kioc->pthru32) {
-			pci_pool_free(adapter->pthru_dma_pool, kioc->pthru32,
+			dma_pool_free(adapter->pthru_dma_pool, kioc->pthru32,
 				kioc->pthru32_h);
 		}
 	}
@@ -1017,7 +1017,7 @@ memalloc_error:
 	kfree(adapter->mbox_list);
 
 	if (adapter->pthru_dma_pool)
-		pci_pool_destroy(adapter->pthru_dma_pool);
+		dma_pool_destroy(adapter->pthru_dma_pool);
 
 	kfree(adapter);
 
@@ -1086,14 +1086,15 @@ mraid_mm_setup_dma_pools(mraid_mmadp_t *adp)
 		pool->buf_size = bufsize;
 		spin_lock_init(&pool->lock);
 
-		pool->handle = pci_pool_create("megaraid mm data buffer",
-						adp->pdev, bufsize, 16, 0);
+		pool->handle = dma_pool_create("megaraid mm data buffer",
+						&adp->pdev->dev, bufsize,
+						16, 0);
 
 		if (!pool->handle) {
 			goto dma_pool_setup_error;
 		}
 
-		pool->vaddr = pci_pool_alloc(pool->handle, GFP_KERNEL,
+		pool->vaddr = dma_pool_alloc(pool->handle, GFP_KERNEL,
 							&pool->paddr);
 
 		if (!pool->vaddr)
@@ -1163,14 +1164,14 @@ mraid_mm_free_adp_resources(mraid_mmadp_t *adp)
 
 		kioc = adp->kioc_list + i;
 
-		pci_pool_free(adp->pthru_dma_pool, kioc->pthru32,
+		dma_pool_free(adp->pthru_dma_pool, kioc->pthru32,
 				kioc->pthru32_h);
 	}
 
 	kfree(adp->kioc_list);
 	kfree(adp->mbox_list);
 
-	pci_pool_destroy(adp->pthru_dma_pool);
+	dma_pool_destroy(adp->pthru_dma_pool);
 
 
 	return;
@@ -1194,10 +1195,10 @@ mraid_mm_teardown_dma_pools(mraid_mmadp_t *adp)
 		if (pool->handle) {
 
 			if (pool->vaddr)
-				pci_pool_free(pool->handle, pool->vaddr,
+				dma_pool_free(pool->handle, pool->vaddr,
 							pool->paddr);
 
-			pci_pool_destroy(pool->handle);
+			dma_pool_destroy(pool->handle);
 			pool->handle = NULL;
 		}
 	}
diff --git a/drivers/scsi/megaraid/megaraid_sas.h b/drivers/scsi/megaraid/megaraid_sas.h
index 2b209bbb4c91..a6722c93a295 100644
--- a/drivers/scsi/megaraid/megaraid_sas.h
+++ b/drivers/scsi/megaraid/megaraid_sas.h
@@ -35,8 +35,8 @@
 /*
  * MegaRAID SAS Driver meta data
  */
-#define MEGASAS_VERSION				"07.701.17.00-rc1"
-#define MEGASAS_RELDATE				"March 2, 2017"
+#define MEGASAS_VERSION				"07.702.06.00-rc1"
+#define MEGASAS_RELDATE				"June 21, 2017"
 
 /*
  * Device IDs
@@ -2115,7 +2115,6 @@ struct megasas_instance {
 	u32 *crash_dump_buf;
 	dma_addr_t crash_dump_h;
 	void *crash_buf[MAX_CRASH_DUMP_SIZE];
-	u32 crash_buf_pages;
 	unsigned int    fw_crash_buffer_size;
 	unsigned int    fw_crash_state;
 	unsigned int    fw_crash_buffer_offset;
diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c
index 316c3df0c3fd..e518dadc8161 100644
--- a/drivers/scsi/megaraid/megaraid_sas_base.c
+++ b/drivers/scsi/megaraid/megaraid_sas_base.c
@@ -49,6 +49,7 @@
 #include <linux/blkdev.h>
 #include <linux/mutex.h>
 #include <linux/poll.h>
+#include <linux/vmalloc.h>
 
 #include <scsi/scsi.h>
 #include <scsi/scsi_cmnd.h>
@@ -1995,9 +1996,12 @@ static void megasas_complete_outstanding_ioctls(struct megasas_instance *instanc
 			if (cmd_fusion->sync_cmd_idx != (u32)ULONG_MAX) {
 				cmd_mfi = instance->cmd_list[cmd_fusion->sync_cmd_idx];
 				if (cmd_mfi->sync_cmd &&
-					cmd_mfi->frame->hdr.cmd != MFI_CMD_ABORT)
+				    (cmd_mfi->frame->hdr.cmd != MFI_CMD_ABORT)) {
+					cmd_mfi->frame->hdr.cmd_status =
+							MFI_STAT_WRONG_STATE;
 					megasas_complete_cmd(instance,
 							     cmd_mfi, DID_OK);
+				}
 			}
 		}
 	} else {
@@ -2791,7 +2795,7 @@ static int megasas_reset_bus_host(struct scsi_cmnd *scmd)
 		cmd = (struct megasas_cmd_fusion *)scmd->SCp.ptr;
 		if (cmd)
 			megasas_dump_frame(cmd->io_request,
-				sizeof(struct MPI2_RAID_SCSI_IO_REQUEST));
+				MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE);
 		ret = megasas_reset_fusion(scmd->device->host,
 				SCSIIO_TIMEOUT_OCR);
 	} else
@@ -3862,19 +3866,19 @@ static void megasas_teardown_frame_pool(struct megasas_instance *instance)
 		cmd = instance->cmd_list[i];
 
 		if (cmd->frame)
-			pci_pool_free(instance->frame_dma_pool, cmd->frame,
+			dma_pool_free(instance->frame_dma_pool, cmd->frame,
 				      cmd->frame_phys_addr);
 
 		if (cmd->sense)
-			pci_pool_free(instance->sense_dma_pool, cmd->sense,
+			dma_pool_free(instance->sense_dma_pool, cmd->sense,
 				      cmd->sense_phys_addr);
 	}
 
 	/*
 	 * Now destroy the pool itself
 	 */
-	pci_pool_destroy(instance->frame_dma_pool);
-	pci_pool_destroy(instance->sense_dma_pool);
+	dma_pool_destroy(instance->frame_dma_pool);
+	dma_pool_destroy(instance->sense_dma_pool);
 
 	instance->frame_dma_pool = NULL;
 	instance->sense_dma_pool = NULL;
@@ -3925,22 +3929,23 @@ static int megasas_create_frame_pool(struct megasas_instance *instance)
 	/*
 	 * Use DMA pool facility provided by PCI layer
 	 */
-	instance->frame_dma_pool = pci_pool_create("megasas frame pool",
-					instance->pdev, instance->mfi_frame_size,
-					256, 0);
+	instance->frame_dma_pool = dma_pool_create("megasas frame pool",
+					&instance->pdev->dev,
+					instance->mfi_frame_size, 256, 0);
 
 	if (!instance->frame_dma_pool) {
 		dev_printk(KERN_DEBUG, &instance->pdev->dev, "failed to setup frame pool\n");
 		return -ENOMEM;
 	}
 
-	instance->sense_dma_pool = pci_pool_create("megasas sense pool",
-						   instance->pdev, 128, 4, 0);
+	instance->sense_dma_pool = dma_pool_create("megasas sense pool",
+						   &instance->pdev->dev, 128,
+						   4, 0);
 
 	if (!instance->sense_dma_pool) {
 		dev_printk(KERN_DEBUG, &instance->pdev->dev, "failed to setup sense pool\n");
 
-		pci_pool_destroy(instance->frame_dma_pool);
+		dma_pool_destroy(instance->frame_dma_pool);
 		instance->frame_dma_pool = NULL;
 
 		return -ENOMEM;
@@ -3955,10 +3960,10 @@ static int megasas_create_frame_pool(struct megasas_instance *instance)
 
 		cmd = instance->cmd_list[i];
 
-		cmd->frame = pci_pool_alloc(instance->frame_dma_pool,
+		cmd->frame = dma_pool_alloc(instance->frame_dma_pool,
 					    GFP_KERNEL, &cmd->frame_phys_addr);
 
-		cmd->sense = pci_pool_alloc(instance->sense_dma_pool,
+		cmd->sense = dma_pool_alloc(instance->sense_dma_pool,
 					    GFP_KERNEL, &cmd->sense_phys_addr);
 
 		/*
@@ -3966,7 +3971,7 @@ static int megasas_create_frame_pool(struct megasas_instance *instance)
 		 * whatever has been allocated
 		 */
 		if (!cmd->frame || !cmd->sense) {
-			dev_printk(KERN_DEBUG, &instance->pdev->dev, "pci_pool_alloc failed\n");
+			dev_printk(KERN_DEBUG, &instance->pdev->dev, "dma_pool_alloc failed\n");
 			megasas_teardown_frame_pool(instance);
 			return -ENOMEM;
 		}
@@ -5478,7 +5483,8 @@ static int megasas_init_fw(struct megasas_instance *instance)
 		instance->throttlequeuedepth =
 				MEGASAS_THROTTLE_QUEUE_DEPTH;
 
-	if (resetwaittime > MEGASAS_RESET_WAIT_TIME)
+	if ((resetwaittime < 1) ||
+	    (resetwaittime > MEGASAS_RESET_WAIT_TIME))
 		resetwaittime = MEGASAS_RESET_WAIT_TIME;
 
 	if ((scmd_timeout < 10) || (scmd_timeout > MEGASAS_DEFAULT_CMD_TIMEOUT))
@@ -5649,6 +5655,14 @@ megasas_register_aen(struct megasas_instance *instance, u32 seq_num,
 		prev_aen.word =
 			le32_to_cpu(instance->aen_cmd->frame->dcmd.mbox.w[1]);
 
+		if ((curr_aen.members.class < MFI_EVT_CLASS_DEBUG) ||
+		    (curr_aen.members.class > MFI_EVT_CLASS_DEAD)) {
+			dev_info(&instance->pdev->dev,
+				 "%s %d out of range class %d send by application\n",
+				 __func__, __LINE__, curr_aen.members.class);
+			return 0;
+		}
+
 		/*
 		 * A class whose enum value is smaller is inclusive of all
 		 * higher values. If a PROGRESS (= -1) was previously
@@ -6096,14 +6110,12 @@ static int megasas_probe_one(struct pci_dev *pdev,
 		instance->pd_info = pci_alloc_consistent(pdev,
 			sizeof(struct MR_PD_INFO), &instance->pd_info_h);
 
-		instance->pd_info = pci_alloc_consistent(pdev,
-			sizeof(struct MR_PD_INFO), &instance->pd_info_h);
-		instance->tgt_prop = pci_alloc_consistent(pdev,
-			sizeof(struct MR_TARGET_PROPERTIES), &instance->tgt_prop_h);
-
 		if (!instance->pd_info)
 			dev_err(&instance->pdev->dev, "Failed to alloc mem for pd_info\n");
 
+		instance->tgt_prop = pci_alloc_consistent(pdev,
+			sizeof(struct MR_TARGET_PROPERTIES), &instance->tgt_prop_h);
+
 		if (!instance->tgt_prop)
 			dev_err(&instance->pdev->dev, "Failed to alloc mem for tgt_prop\n");
 
@@ -6228,8 +6240,8 @@ static int megasas_probe_one(struct pci_dev *pdev,
 fail_start_aen:
 fail_io_attach:
 	megasas_mgmt_info.count--;
-	megasas_mgmt_info.instance[megasas_mgmt_info.max_index] = NULL;
 	megasas_mgmt_info.max_index--;
+	megasas_mgmt_info.instance[megasas_mgmt_info.max_index] = NULL;
 
 	instance->instancet->disable_intr(instance);
 	megasas_destroy_irqs(instance);
@@ -6663,9 +6675,14 @@ skip_firing_dcmds:
 						  fusion->max_map_sz,
 						  fusion->ld_map[i],
 						  fusion->ld_map_phys[i]);
-			if (fusion->ld_drv_map[i])
-				free_pages((ulong)fusion->ld_drv_map[i],
-					fusion->drv_map_pages);
+			if (fusion->ld_drv_map[i]) {
+				if (is_vmalloc_addr(fusion->ld_drv_map[i]))
+					vfree(fusion->ld_drv_map[i]);
+				else
+					free_pages((ulong)fusion->ld_drv_map[i],
+						   fusion->drv_map_pages);
+			}
+
 			if (fusion->pd_seq_sync[i])
 				dma_free_coherent(&instance->pdev->dev,
 					pd_seq_map_sz,
@@ -6866,6 +6883,7 @@ megasas_mgmt_fw_ioctl(struct megasas_instance *instance,
 	void *sense = NULL;
 	dma_addr_t sense_handle;
 	unsigned long *sense_ptr;
+	u32 opcode;
 
 	memset(kbuff_arr, 0, sizeof(kbuff_arr));
 
@@ -6893,15 +6911,16 @@ megasas_mgmt_fw_ioctl(struct megasas_instance *instance,
 	cmd->frame->hdr.flags &= cpu_to_le16(~(MFI_FRAME_IEEE |
 					       MFI_FRAME_SGL64 |
 					       MFI_FRAME_SENSE64));
+	opcode = le32_to_cpu(cmd->frame->dcmd.opcode);
 
-	if (cmd->frame->dcmd.opcode == MR_DCMD_CTRL_SHUTDOWN) {
+	if (opcode == MR_DCMD_CTRL_SHUTDOWN) {
 		if (megasas_get_ctrl_info(instance) != DCMD_SUCCESS) {
 			megasas_return_cmd(instance, cmd);
 			return -1;
 		}
 	}
 
-	if (cmd->frame->dcmd.opcode == MR_DRIVER_SET_APP_CRASHDUMP_MODE) {
+	if (opcode == MR_DRIVER_SET_APP_CRASHDUMP_MODE) {
 		error = megasas_set_crash_dump_params_ioctl(cmd);
 		megasas_return_cmd(instance, cmd);
 		return error;
@@ -6975,8 +6994,7 @@ megasas_mgmt_fw_ioctl(struct megasas_instance *instance,
 		cmd->sync_cmd = 0;
 		dev_err(&instance->pdev->dev,
 			"return -EBUSY from %s %d opcode 0x%x cmd->cmd_status_drv 0x%x\n",
-			__func__, __LINE__, cmd->frame->dcmd.opcode,
-			cmd->cmd_status_drv);
+			__func__, __LINE__, opcode,	cmd->cmd_status_drv);
 		return -EBUSY;
 	}
 
@@ -7323,49 +7341,39 @@ static struct pci_driver megasas_pci_driver = {
 /*
  * Sysfs driver attributes
  */
-static ssize_t megasas_sysfs_show_version(struct device_driver *dd, char *buf)
+static ssize_t version_show(struct device_driver *dd, char *buf)
 {
 	return snprintf(buf, strlen(MEGASAS_VERSION) + 2, "%s\n",
 			MEGASAS_VERSION);
 }
+static DRIVER_ATTR_RO(version);
 
-static DRIVER_ATTR(version, S_IRUGO, megasas_sysfs_show_version, NULL);
-
-static ssize_t
-megasas_sysfs_show_release_date(struct device_driver *dd, char *buf)
+static ssize_t release_date_show(struct device_driver *dd, char *buf)
 {
 	return snprintf(buf, strlen(MEGASAS_RELDATE) + 2, "%s\n",
 		MEGASAS_RELDATE);
 }
+static DRIVER_ATTR_RO(release_date);
 
-static DRIVER_ATTR(release_date, S_IRUGO, megasas_sysfs_show_release_date, NULL);
-
-static ssize_t
-megasas_sysfs_show_support_poll_for_event(struct device_driver *dd, char *buf)
+static ssize_t support_poll_for_event_show(struct device_driver *dd, char *buf)
 {
 	return sprintf(buf, "%u\n", support_poll_for_event);
 }
+static DRIVER_ATTR_RO(support_poll_for_event);
 
-static DRIVER_ATTR(support_poll_for_event, S_IRUGO,
-			megasas_sysfs_show_support_poll_for_event, NULL);
-
- static ssize_t
-megasas_sysfs_show_support_device_change(struct device_driver *dd, char *buf)
+static ssize_t support_device_change_show(struct device_driver *dd, char *buf)
 {
 	return sprintf(buf, "%u\n", support_device_change);
 }
+static DRIVER_ATTR_RO(support_device_change);
 
-static DRIVER_ATTR(support_device_change, S_IRUGO,
-			megasas_sysfs_show_support_device_change, NULL);
-
-static ssize_t
-megasas_sysfs_show_dbg_lvl(struct device_driver *dd, char *buf)
+static ssize_t dbg_lvl_show(struct device_driver *dd, char *buf)
 {
 	return sprintf(buf, "%u\n", megasas_dbg_lvl);
 }
 
-static ssize_t
-megasas_sysfs_set_dbg_lvl(struct device_driver *dd, const char *buf, size_t count)
+static ssize_t dbg_lvl_store(struct device_driver *dd, const char *buf,
+			     size_t count)
 {
 	int retval = count;
 
@@ -7375,9 +7383,7 @@ megasas_sysfs_set_dbg_lvl(struct device_driver *dd, const char *buf, size_t coun
 	}
 	return retval;
 }
-
-static DRIVER_ATTR(dbg_lvl, S_IRUGO|S_IWUSR, megasas_sysfs_show_dbg_lvl,
-		megasas_sysfs_set_dbg_lvl);
+static DRIVER_ATTR_RW(dbg_lvl);
 
 static inline void megasas_remove_scsi_device(struct scsi_device *sdev)
 {
diff --git a/drivers/scsi/megaraid/megaraid_sas_fp.c b/drivers/scsi/megaraid/megaraid_sas_fp.c
index 62affa76133d..ecc699a65bac 100644
--- a/drivers/scsi/megaraid/megaraid_sas_fp.c
+++ b/drivers/scsi/megaraid/megaraid_sas_fp.c
@@ -67,16 +67,6 @@ MODULE_PARM_DESC(lb_pending_cmds, "Change raid-1 load balancing outstanding "
 #define ABS_DIFF(a, b)   (((a) > (b)) ? ((a) - (b)) : ((b) - (a)))
 #define MR_LD_STATE_OPTIMAL 3
 
-#ifdef FALSE
-#undef FALSE
-#endif
-#define FALSE 0
-
-#ifdef TRUE
-#undef TRUE
-#endif
-#define TRUE 1
-
 #define SPAN_ROW_SIZE(map, ld, index_) (MR_LdSpanPtrGet(ld, index_, map)->spanRowSize)
 #define SPAN_ROW_DATA_SIZE(map_, ld, index_)   (MR_LdSpanPtrGet(ld, index_, map)->spanRowDataSize)
 #define SPAN_INVALID  0xff
@@ -709,7 +699,7 @@ static u8 mr_spanset_get_phy_params(struct megasas_instance *instance, u32 ld,
 	u32     pd, arRef, r1_alt_pd;
 	u8      physArm, span;
 	u64     row;
-	u8	retval = TRUE;
+	u8	retval = true;
 	u64	*pdBlock = &io_info->pdBlock;
 	__le16	*pDevHandle = &io_info->devHandle;
 	u8	*pPdInterface = &io_info->pd_interface;
@@ -727,7 +717,7 @@ static u8 mr_spanset_get_phy_params(struct megasas_instance *instance, u32 ld,
 	if (raid->level == 6) {
 		logArm = get_arm_from_strip(instance, ld, stripRow, map);
 		if (logArm == -1U)
-			return FALSE;
+			return false;
 		rowMod = mega_mod64(row, SPAN_ROW_SIZE(map, ld, span));
 		armQ = SPAN_ROW_SIZE(map, ld, span) - 1 - rowMod;
 		arm = armQ + 1 + logArm;
@@ -738,7 +728,7 @@ static u8 mr_spanset_get_phy_params(struct megasas_instance *instance, u32 ld,
 		/* Calculate the arm */
 		physArm = get_arm(instance, ld, span, stripRow, map);
 	if (physArm == 0xFF)
-		return FALSE;
+		return false;
 
 	arRef       = MR_LdSpanArrayGet(ld, span, map);
 	pd          = MR_ArPdGet(arRef, physArm, map);
@@ -812,7 +802,7 @@ u8 MR_GetPhyParams(struct megasas_instance *instance, u32 ld, u64 stripRow,
 	u32         pd, arRef, r1_alt_pd;
 	u8          physArm, span;
 	u64         row;
-	u8	    retval = TRUE;
+	u8	    retval = true;
 	u64	    *pdBlock = &io_info->pdBlock;
 	__le16	    *pDevHandle = &io_info->devHandle;
 	u8	    *pPdInterface = &io_info->pd_interface;
@@ -829,7 +819,7 @@ u8 MR_GetPhyParams(struct megasas_instance *instance, u32 ld, u64 stripRow,
 		u32 rowMod, armQ, arm;
 
 		if (raid->rowSize == 0)
-			return FALSE;
+			return false;
 		/* get logical row mod */
 		rowMod = mega_mod64(row, raid->rowSize);
 		armQ = raid->rowSize-1-rowMod; /* index of Q drive */
@@ -839,7 +829,7 @@ u8 MR_GetPhyParams(struct megasas_instance *instance, u32 ld, u64 stripRow,
 		physArm = (u8)arm;
 	} else  {
 		if (raid->modFactor == 0)
-			return FALSE;
+			return false;
 		physArm = MR_LdDataArmGet(ld,  mega_mod64(stripRow,
 							  raid->modFactor),
 					  map);
@@ -851,7 +841,7 @@ u8 MR_GetPhyParams(struct megasas_instance *instance, u32 ld, u64 stripRow,
 	} else {
 		span = (u8)MR_GetSpanBlock(ld, row, pdBlock, map);
 		if (span == SPAN_INVALID)
-			return FALSE;
+			return false;
 	}
 
 	/* Get the array on which this span is present */
@@ -954,7 +944,7 @@ MR_BuildRaidContext(struct megasas_instance *instance,
 	 */
 	if (raid->rowDataSize == 0) {
 		if (MR_LdSpanPtrGet(ld, 0, map)->spanRowDataSize == 0)
-			return FALSE;
+			return false;
 		else if (instance->UnevenSpanSupport) {
 			io_info->IoforUnevenSpan = 1;
 		} else {
@@ -963,7 +953,7 @@ MR_BuildRaidContext(struct megasas_instance *instance,
 				"rowDataSize = 0x%0x,"
 				"but there is _NO_ UnevenSpanSupport\n",
 				MR_LdSpanPtrGet(ld, 0, map)->spanRowDataSize);
-			return FALSE;
+			return false;
 		}
 	}
 
@@ -988,7 +978,7 @@ MR_BuildRaidContext(struct megasas_instance *instance,
 			dev_info(&instance->pdev->dev, "return from %s %d."
 				"Send IO w/o region lock.\n",
 				__func__, __LINE__);
-			return FALSE;
+			return false;
 		}
 
 		if (raid->spanDepth == 1) {
@@ -1004,7 +994,7 @@ MR_BuildRaidContext(struct megasas_instance *instance,
 				(unsigned long long)start_row,
 				(unsigned long long)start_strip,
 				(unsigned long long)endStrip);
-			return FALSE;
+			return false;
 		}
 		io_info->start_span	= startlba_span;
 		io_info->start_row	= start_row;
@@ -1038,7 +1028,7 @@ MR_BuildRaidContext(struct megasas_instance *instance,
 					       raid->capability.
 					       fpWriteAcrossStripe));
 	} else
-		io_info->fpOkForIo = FALSE;
+		io_info->fpOkForIo = false;
 
 	if (numRows == 1) {
 		/* single-strip IOs can always lock only the data needed */
@@ -1124,7 +1114,7 @@ MR_BuildRaidContext(struct megasas_instance *instance,
 					pRAID_Context, map);
 		/* If IO on an invalid Pd, then FP is not possible.*/
 		if (io_info->devHandle == MR_DEVHANDLE_INVALID)
-			io_info->fpOkForIo = FALSE;
+			io_info->fpOkForIo = false;
 		return retval;
 	} else if (isRead) {
 		uint stripIdx;
@@ -1138,10 +1128,10 @@ MR_BuildRaidContext(struct megasas_instance *instance,
 				    start_strip + stripIdx, ref_in_start_stripe,
 				    io_info, pRAID_Context, map);
 			if (!retval)
-				return TRUE;
+				return true;
 		}
 	}
-	return TRUE;
+	return true;
 }
 
 /*
diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.c b/drivers/scsi/megaraid/megaraid_sas_fusion.c
index 985510628f56..11bd2e698b84 100644
--- a/drivers/scsi/megaraid/megaraid_sas_fusion.c
+++ b/drivers/scsi/megaraid/megaraid_sas_fusion.c
@@ -313,20 +313,20 @@ megasas_free_cmds_fusion(struct megasas_instance *instance)
 		cmd = fusion->cmd_list[i];
 		if (cmd) {
 			if (cmd->sg_frame)
-				pci_pool_free(fusion->sg_dma_pool, cmd->sg_frame,
+				dma_pool_free(fusion->sg_dma_pool, cmd->sg_frame,
 				      cmd->sg_frame_phys_addr);
 			if (cmd->sense)
-				pci_pool_free(fusion->sense_dma_pool, cmd->sense,
+				dma_pool_free(fusion->sense_dma_pool, cmd->sense,
 				      cmd->sense_phys_addr);
 		}
 	}
 
 	if (fusion->sg_dma_pool) {
-		pci_pool_destroy(fusion->sg_dma_pool);
+		dma_pool_destroy(fusion->sg_dma_pool);
 		fusion->sg_dma_pool = NULL;
 	}
 	if (fusion->sense_dma_pool) {
-		pci_pool_destroy(fusion->sense_dma_pool);
+		dma_pool_destroy(fusion->sense_dma_pool);
 		fusion->sense_dma_pool = NULL;
 	}
 
@@ -343,11 +343,11 @@ megasas_free_cmds_fusion(struct megasas_instance *instance)
 			fusion->request_alloc_sz, fusion->req_frames_desc,
 			fusion->req_frames_desc_phys);
 	if (fusion->io_request_frames)
-		pci_pool_free(fusion->io_request_frames_pool,
+		dma_pool_free(fusion->io_request_frames_pool,
 			fusion->io_request_frames,
 			fusion->io_request_frames_phys);
 	if (fusion->io_request_frames_pool) {
-		pci_pool_destroy(fusion->io_request_frames_pool);
+		dma_pool_destroy(fusion->io_request_frames_pool);
 		fusion->io_request_frames_pool = NULL;
 	}
 
@@ -376,12 +376,12 @@ static int megasas_create_sg_sense_fusion(struct megasas_instance *instance)
 
 
 	fusion->sg_dma_pool =
-			pci_pool_create("mr_sg", instance->pdev,
+			dma_pool_create("mr_sg", &instance->pdev->dev,
 				instance->max_chain_frame_sz,
 				MR_DEFAULT_NVME_PAGE_SIZE, 0);
 	/* SCSI_SENSE_BUFFERSIZE  = 96 bytes */
 	fusion->sense_dma_pool =
-			pci_pool_create("mr_sense", instance->pdev,
+			dma_pool_create("mr_sense", &instance->pdev->dev,
 				SCSI_SENSE_BUFFERSIZE, 64, 0);
 
 	if (!fusion->sense_dma_pool || !fusion->sg_dma_pool) {
@@ -395,10 +395,10 @@ static int megasas_create_sg_sense_fusion(struct megasas_instance *instance)
 	 */
 	for (i = 0; i < max_cmd; i++) {
 		cmd = fusion->cmd_list[i];
-		cmd->sg_frame = pci_pool_alloc(fusion->sg_dma_pool,
+		cmd->sg_frame = dma_pool_alloc(fusion->sg_dma_pool,
 					GFP_KERNEL, &cmd->sg_frame_phys_addr);
 
-		cmd->sense = pci_pool_alloc(fusion->sense_dma_pool,
+		cmd->sense = dma_pool_alloc(fusion->sense_dma_pool,
 					GFP_KERNEL, &cmd->sense_phys_addr);
 		if (!cmd->sg_frame || !cmd->sense) {
 			dev_err(&instance->pdev->dev,
@@ -410,7 +410,7 @@ static int megasas_create_sg_sense_fusion(struct megasas_instance *instance)
 	/* create sense buffer for the raid 1/10 fp */
 	for (i = max_cmd; i < instance->max_mpt_cmds; i++) {
 		cmd = fusion->cmd_list[i];
-		cmd->sense = pci_pool_alloc(fusion->sense_dma_pool,
+		cmd->sense = dma_pool_alloc(fusion->sense_dma_pool,
 			GFP_KERNEL, &cmd->sense_phys_addr);
 		if (!cmd->sense) {
 			dev_err(&instance->pdev->dev,
@@ -479,7 +479,7 @@ megasas_alloc_request_fusion(struct megasas_instance *instance)
 	}
 
 	fusion->io_request_frames_pool =
-			pci_pool_create("mr_ioreq", instance->pdev,
+			dma_pool_create("mr_ioreq", &instance->pdev->dev,
 				fusion->io_frames_alloc_sz, 16, 0);
 
 	if (!fusion->io_request_frames_pool) {
@@ -489,7 +489,7 @@ megasas_alloc_request_fusion(struct megasas_instance *instance)
 	}
 
 	fusion->io_request_frames =
-			pci_pool_alloc(fusion->io_request_frames_pool,
+			dma_pool_alloc(fusion->io_request_frames_pool,
 				GFP_KERNEL, &fusion->io_request_frames_phys);
 	if (!fusion->io_request_frames) {
 		dev_err(&instance->pdev->dev,
@@ -509,7 +509,7 @@ megasas_alloc_reply_fusion(struct megasas_instance *instance)
 
 	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
 	fusion->reply_frames_desc_pool =
-			pci_pool_create("mr_reply", instance->pdev,
+			dma_pool_create("mr_reply", &instance->pdev->dev,
 				fusion->reply_alloc_sz * count, 16, 0);
 
 	if (!fusion->reply_frames_desc_pool) {
@@ -519,7 +519,7 @@ megasas_alloc_reply_fusion(struct megasas_instance *instance)
 	}
 
 	fusion->reply_frames_desc[0] =
-		pci_pool_alloc(fusion->reply_frames_desc_pool,
+		dma_pool_alloc(fusion->reply_frames_desc_pool,
 			GFP_KERNEL, &fusion->reply_frames_desc_phys[0]);
 	if (!fusion->reply_frames_desc[0]) {
 		dev_err(&instance->pdev->dev,
@@ -562,8 +562,10 @@ megasas_alloc_rdpq_fusion(struct megasas_instance *instance)
 	memset(fusion->rdpq_virt, 0,
 			sizeof(struct MPI2_IOC_INIT_RDPQ_ARRAY_ENTRY) * MAX_MSIX_QUEUES_FUSION);
 	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
-	fusion->reply_frames_desc_pool = pci_pool_create("mr_rdpq",
-							 instance->pdev, fusion->reply_alloc_sz, 16, 0);
+	fusion->reply_frames_desc_pool = dma_pool_create("mr_rdpq",
+							 &instance->pdev->dev,
+							 fusion->reply_alloc_sz,
+							 16, 0);
 
 	if (!fusion->reply_frames_desc_pool) {
 		dev_err(&instance->pdev->dev,
@@ -573,7 +575,7 @@ megasas_alloc_rdpq_fusion(struct megasas_instance *instance)
 
 	for (i = 0; i < count; i++) {
 		fusion->reply_frames_desc[i] =
-				pci_pool_alloc(fusion->reply_frames_desc_pool,
+				dma_pool_alloc(fusion->reply_frames_desc_pool,
 					GFP_KERNEL, &fusion->reply_frames_desc_phys[i]);
 		if (!fusion->reply_frames_desc[i]) {
 			dev_err(&instance->pdev->dev,
@@ -601,13 +603,13 @@ megasas_free_rdpq_fusion(struct megasas_instance *instance) {
 
 	for (i = 0; i < MAX_MSIX_QUEUES_FUSION; i++) {
 		if (fusion->reply_frames_desc[i])
-			pci_pool_free(fusion->reply_frames_desc_pool,
+			dma_pool_free(fusion->reply_frames_desc_pool,
 				fusion->reply_frames_desc[i],
 				fusion->reply_frames_desc_phys[i]);
 	}
 
 	if (fusion->reply_frames_desc_pool)
-		pci_pool_destroy(fusion->reply_frames_desc_pool);
+		dma_pool_destroy(fusion->reply_frames_desc_pool);
 
 	if (fusion->rdpq_virt)
 		pci_free_consistent(instance->pdev,
@@ -623,12 +625,12 @@ megasas_free_reply_fusion(struct megasas_instance *instance) {
 	fusion = instance->ctrl_context;
 
 	if (fusion->reply_frames_desc[0])
-		pci_pool_free(fusion->reply_frames_desc_pool,
+		dma_pool_free(fusion->reply_frames_desc_pool,
 			fusion->reply_frames_desc[0],
 			fusion->reply_frames_desc_phys[0]);
 
 	if (fusion->reply_frames_desc_pool)
-		pci_pool_destroy(fusion->reply_frames_desc_pool);
+		dma_pool_destroy(fusion->reply_frames_desc_pool);
 
 }
 
@@ -914,7 +916,6 @@ megasas_ioc_init_fusion(struct megasas_instance *instance)
 		ret = 1;
 		goto fail_fw_init;
 	}
-	dev_info(&instance->pdev->dev, "Init cmd success\n");
 
 	ret = 0;
 
@@ -925,6 +926,10 @@ fail_fw_init:
 				  sizeof(struct MPI2_IOC_INIT_REQUEST),
 				  IOCInitMessage, ioc_init_handle);
 fail_get_cmd:
+	dev_err(&instance->pdev->dev,
+		"Init cmd return status %s for SCSI host %d\n",
+		ret ? "FAILED" : "SUCCESS", instance->host->host_no);
+
 	return ret;
 }
 
@@ -1261,6 +1266,80 @@ megasas_display_intel_branding(struct megasas_instance *instance)
 }
 
 /**
+ * megasas_allocate_raid_maps -	Allocate memory for RAID maps
+ * @instance:				Adapter soft state
+ *
+ * return:				if success: return 0
+ *					failed:  return -ENOMEM
+ */
+static inline int megasas_allocate_raid_maps(struct megasas_instance *instance)
+{
+	struct fusion_context *fusion;
+	int i = 0;
+
+	fusion = instance->ctrl_context;
+
+	fusion->drv_map_pages = get_order(fusion->drv_map_sz);
+
+	for (i = 0; i < 2; i++) {
+		fusion->ld_map[i] = NULL;
+
+		fusion->ld_drv_map[i] = (void *)
+			__get_free_pages(__GFP_ZERO | GFP_KERNEL,
+					 fusion->drv_map_pages);
+
+		if (!fusion->ld_drv_map[i]) {
+			fusion->ld_drv_map[i] = vzalloc(fusion->drv_map_sz);
+
+			if (!fusion->ld_drv_map[i]) {
+				dev_err(&instance->pdev->dev,
+					"Could not allocate memory for local map"
+					" size requested: %d\n",
+					fusion->drv_map_sz);
+				goto ld_drv_map_alloc_fail;
+			}
+		}
+	}
+
+	for (i = 0; i < 2; i++) {
+		fusion->ld_map[i] = dma_alloc_coherent(&instance->pdev->dev,
+						       fusion->max_map_sz,
+						       &fusion->ld_map_phys[i],
+						       GFP_KERNEL);
+		if (!fusion->ld_map[i]) {
+			dev_err(&instance->pdev->dev,
+				"Could not allocate memory for map info %s:%d\n",
+				__func__, __LINE__);
+			goto ld_map_alloc_fail;
+		}
+	}
+
+	return 0;
+
+ld_map_alloc_fail:
+	for (i = 0; i < 2; i++) {
+		if (fusion->ld_map[i])
+			dma_free_coherent(&instance->pdev->dev,
+					  fusion->max_map_sz,
+					  fusion->ld_map[i],
+					  fusion->ld_map_phys[i]);
+	}
+
+ld_drv_map_alloc_fail:
+	for (i = 0; i < 2; i++) {
+		if (fusion->ld_drv_map[i]) {
+			if (is_vmalloc_addr(fusion->ld_drv_map[i]))
+				vfree(fusion->ld_drv_map[i]);
+			else
+				free_pages((ulong)fusion->ld_drv_map[i],
+					   fusion->drv_map_pages);
+		}
+	}
+
+	return -ENOMEM;
+}
+
+/**
  * megasas_init_adapter_fusion -	Initializes the FW
  * @instance:		Adapter soft state
  *
@@ -1379,45 +1458,14 @@ megasas_init_adapter_fusion(struct megasas_instance *instance)
 	instance->r1_ldio_hint_default =  MR_R1_LDIO_PIGGYBACK_DEFAULT;
 	fusion->fast_path_io = 0;
 
-	fusion->drv_map_pages = get_order(fusion->drv_map_sz);
-	for (i = 0; i < 2; i++) {
-		fusion->ld_map[i] = NULL;
-		fusion->ld_drv_map[i] = (void *)__get_free_pages(GFP_KERNEL,
-			fusion->drv_map_pages);
-		if (!fusion->ld_drv_map[i]) {
-			dev_err(&instance->pdev->dev, "Could not allocate "
-				"memory for local map info for %d pages\n",
-				fusion->drv_map_pages);
-			if (i == 1)
-				free_pages((ulong)fusion->ld_drv_map[0],
-					fusion->drv_map_pages);
-			goto fail_ioc_init;
-		}
-		memset(fusion->ld_drv_map[i], 0,
-			((1 << PAGE_SHIFT) << fusion->drv_map_pages));
-	}
-
-	for (i = 0; i < 2; i++) {
-		fusion->ld_map[i] = dma_alloc_coherent(&instance->pdev->dev,
-						       fusion->max_map_sz,
-						       &fusion->ld_map_phys[i],
-						       GFP_KERNEL);
-		if (!fusion->ld_map[i]) {
-			dev_err(&instance->pdev->dev, "Could not allocate memory "
-			       "for map info\n");
-			goto fail_map_info;
-		}
-	}
+	if (megasas_allocate_raid_maps(instance))
+		goto fail_ioc_init;
 
 	if (!megasas_get_map_info(instance))
 		megasas_sync_map_info(instance);
 
 	return 0;
 
-fail_map_info:
-	if (i == 1)
-		dma_free_coherent(&instance->pdev->dev, fusion->max_map_sz,
-				  fusion->ld_map[0], fusion->ld_map_phys[0]);
 fail_ioc_init:
 	megasas_free_cmds_fusion(instance);
 fail_alloc_cmds:
@@ -3287,7 +3335,7 @@ build_mpt_mfi_pass_thru(struct megasas_instance *instance,
 	mpi25_ieee_chain->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT |
 		MPI2_IEEE_SGE_FLAGS_IOCPLBNTA_ADDR;
 
-	mpi25_ieee_chain->Length = cpu_to_le32(instance->max_chain_frame_sz);
+	mpi25_ieee_chain->Length = cpu_to_le32(instance->mfi_frame_size);
 }
 
 /**
@@ -3369,17 +3417,13 @@ megasas_alloc_host_crash_buffer(struct megasas_instance *instance)
 {
 	unsigned int i;
 
-	instance->crash_buf_pages = get_order(CRASH_DMA_BUF_SIZE);
 	for (i = 0; i < MAX_CRASH_DUMP_SIZE; i++) {
-		instance->crash_buf[i] = (void	*)__get_free_pages(GFP_KERNEL,
-				instance->crash_buf_pages);
+		instance->crash_buf[i] = vzalloc(CRASH_DMA_BUF_SIZE);
 		if (!instance->crash_buf[i]) {
 			dev_info(&instance->pdev->dev, "Firmware crash dump "
 				"memory allocation failed at index %d\n", i);
 			break;
 		}
-		memset(instance->crash_buf[i], 0,
-			((1 << PAGE_SHIFT) << instance->crash_buf_pages));
 	}
 	instance->drv_buf_alloc = i;
 }
@@ -3391,12 +3435,10 @@ megasas_alloc_host_crash_buffer(struct megasas_instance *instance)
 void
 megasas_free_host_crash_buffer(struct megasas_instance *instance)
 {
-	unsigned int i
-;
+	unsigned int i;
 	for (i = 0; i < instance->drv_buf_alloc; i++) {
 		if (instance->crash_buf[i])
-			free_pages((ulong)instance->crash_buf[i],
-					instance->crash_buf_pages);
+			vfree(instance->crash_buf[i]);
 	}
 	instance->drv_buf_index = 0;
 	instance->drv_buf_alloc = 0;
@@ -3556,6 +3598,7 @@ int megasas_wait_for_outstanding_fusion(struct megasas_instance *instance,
 			}
 		}
 
+		megasas_complete_cmd_dpc_fusion((unsigned long)instance);
 		outstanding = atomic_read(&instance->fw_outstanding);
 		if (!outstanding)
 			goto out;
@@ -3564,8 +3607,6 @@ int megasas_wait_for_outstanding_fusion(struct megasas_instance *instance,
 			dev_notice(&instance->pdev->dev, "[%2d]waiting for %d "
 			       "commands to complete for scsi%d\n", i,
 			       outstanding, instance->host->host_no);
-			megasas_complete_cmd_dpc_fusion(
-				(unsigned long)instance);
 		}
 		msleep(1000);
 	}
@@ -3623,6 +3664,15 @@ void megasas_refire_mgmt_cmd(struct megasas_instance *instance)
 
 		if (!smid)
 			continue;
+
+		/* Do not refire shutdown command */
+		if (le32_to_cpu(cmd_mfi->frame->dcmd.opcode) ==
+			MR_DCMD_CTRL_SHUTDOWN) {
+			cmd_mfi->frame->dcmd.cmd_status = MFI_STAT_OK;
+			megasas_complete_cmd(instance, cmd_mfi, DID_OK);
+			continue;
+		}
+
 		req_desc = megasas_get_request_descriptor
 					(instance, smid - 1);
 		refire_cmd = req_desc && ((cmd_mfi->frame->dcmd.opcode !=
@@ -3750,7 +3800,7 @@ megasas_issue_tm(struct megasas_instance *instance, u16 device_handle,
 	struct megasas_cmd_fusion *cmd_fusion;
 	struct megasas_cmd *cmd_mfi;
 	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
-	struct fusion_context *fusion;
+	struct fusion_context *fusion = NULL;
 	struct megasas_cmd_fusion *scsi_lookup;
 	int rc;
 	struct MPI2_SCSI_TASK_MANAGE_REPLY *mpi_reply;
@@ -3777,8 +3827,6 @@ megasas_issue_tm(struct megasas_instance *instance, u16 device_handle,
 	cmd_fusion->request_desc = req_desc;
 	req_desc->Words = 0;
 
-	scsi_lookup = fusion->cmd_list[smid_task - 1];
-
 	mr_request = (struct MR_TASK_MANAGE_REQUEST *) cmd_fusion->io_request;
 	memset(mr_request, 0, sizeof(struct MR_TASK_MANAGE_REQUEST));
 	mpi_request = (struct MPI2_SCSI_TASK_MANAGE_REQUEST *) &mr_request->TmRequest;
@@ -3825,13 +3873,13 @@ megasas_issue_tm(struct megasas_instance *instance, u16 device_handle,
 	rc = SUCCESS;
 	switch (type) {
 	case MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK:
+		scsi_lookup = fusion->cmd_list[smid_task - 1];
+
 		if (scsi_lookup->scmd == NULL)
 			break;
 		else {
 			instance->instancet->disable_intr(instance);
 			megasas_sync_irqs((unsigned long)instance);
-			megasas_complete_cmd_dpc_fusion
-					((unsigned long)instance);
 			instance->instancet->enable_intr(instance);
 			if (scsi_lookup->scmd == NULL)
 				break;
@@ -3843,9 +3891,7 @@ megasas_issue_tm(struct megasas_instance *instance, u16 device_handle,
 		if ((channel == 0xFFFFFFFF) && (id == 0xFFFFFFFF))
 			break;
 		instance->instancet->disable_intr(instance);
-		msleep(1000);
-		megasas_complete_cmd_dpc_fusion
-				((unsigned long)instance);
+		megasas_sync_irqs((unsigned long)instance);
 		rc = megasas_track_scsiio(instance, id, channel);
 		instance->instancet->enable_intr(instance);
 
@@ -4271,9 +4317,6 @@ transition_to_ready:
 			megasas_fusion_update_can_queue(instance, OCR_CONTEXT);
 
 			if (megasas_ioc_init_fusion(instance)) {
-				dev_warn(&instance->pdev->dev,
-				       "megasas_ioc_init_fusion() failed! for "
-				       "scsi%d\n", instance->host->host_no);
 				if (instance->requestorId && !reason)
 					goto fail_kill_adapter;
 				else
@@ -4319,6 +4362,10 @@ transition_to_ready:
 			instance->instancet->enable_intr(instance);
 			atomic_set(&instance->adprecovery, MEGASAS_HBA_OPERATIONAL);
 
+			dev_info(&instance->pdev->dev, "Interrupts are enabled and"
+				" controller is OPERATIONAL for scsi:%d\n",
+				instance->host->host_no);
+
 			/* Restart SR-IOV heartbeat */
 			if (instance->requestorId) {
 				if (!megasas_sriov_start_heartbeat(instance, 0))
@@ -4330,11 +4377,6 @@ transition_to_ready:
 					instance->skip_heartbeat_timer_del = 1;
 			}
 
-			/* Adapter reset completed successfully */
-			dev_warn(&instance->pdev->dev, "Reset "
-			       "successful for scsi%d.\n",
-				instance->host->host_no);
-
 			if (instance->crash_dump_drv_support &&
 				instance->crash_dump_app_support)
 				megasas_set_crash_dump_params(instance,
@@ -4344,6 +4386,12 @@ transition_to_ready:
 					MR_CRASH_BUF_TURN_OFF);
 
 			retval = SUCCESS;
+
+			/* Adapter reset completed successfully */
+			dev_warn(&instance->pdev->dev,
+				 "Reset successful for scsi%d.\n",
+				 instance->host->host_no);
+
 			goto out;
 		}
 fail_kill_adapter:
diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c
index 18039bba26c4..87999905bca3 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_base.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_base.c
@@ -615,9 +615,9 @@ _base_display_event_data(struct MPT3SAS_ADAPTER *ioc,
 		    (event_data->ReasonCode == MPI2_EVENT_SAS_DISC_RC_STARTED) ?
 		    "start" : "stop");
 		if (event_data->DiscoveryStatus)
-			pr_info("discovery_status(0x%08x)",
+			pr_cont(" discovery_status(0x%08x)",
 			    le32_to_cpu(event_data->DiscoveryStatus));
-			pr_info("\n");
+		pr_cont("\n");
 		return;
 	}
 	case MPI2_EVENT_SAS_BROADCAST_PRIMITIVE:
@@ -3198,9 +3198,8 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 	}
 
 	if (ioc->sense) {
-		pci_pool_free(ioc->sense_dma_pool, ioc->sense, ioc->sense_dma);
-		if (ioc->sense_dma_pool)
-			pci_pool_destroy(ioc->sense_dma_pool);
+		dma_pool_free(ioc->sense_dma_pool, ioc->sense, ioc->sense_dma);
+		dma_pool_destroy(ioc->sense_dma_pool);
 		dexitprintk(ioc, pr_info(MPT3SAS_FMT
 			"sense_pool(0x%p): free\n",
 			ioc->name, ioc->sense));
@@ -3208,9 +3207,8 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 	}
 
 	if (ioc->reply) {
-		pci_pool_free(ioc->reply_dma_pool, ioc->reply, ioc->reply_dma);
-		if (ioc->reply_dma_pool)
-			pci_pool_destroy(ioc->reply_dma_pool);
+		dma_pool_free(ioc->reply_dma_pool, ioc->reply, ioc->reply_dma);
+		dma_pool_destroy(ioc->reply_dma_pool);
 		dexitprintk(ioc, pr_info(MPT3SAS_FMT
 			"reply_pool(0x%p): free\n",
 			ioc->name, ioc->reply));
@@ -3218,10 +3216,9 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 	}
 
 	if (ioc->reply_free) {
-		pci_pool_free(ioc->reply_free_dma_pool, ioc->reply_free,
+		dma_pool_free(ioc->reply_free_dma_pool, ioc->reply_free,
 		    ioc->reply_free_dma);
-		if (ioc->reply_free_dma_pool)
-			pci_pool_destroy(ioc->reply_free_dma_pool);
+		dma_pool_destroy(ioc->reply_free_dma_pool);
 		dexitprintk(ioc, pr_info(MPT3SAS_FMT
 			"reply_free_pool(0x%p): free\n",
 			ioc->name, ioc->reply_free));
@@ -3232,7 +3229,7 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 		do {
 			rps = &ioc->reply_post[i];
 			if (rps->reply_post_free) {
-				pci_pool_free(
+				dma_pool_free(
 				    ioc->reply_post_free_dma_pool,
 				    rps->reply_post_free,
 				    rps->reply_post_free_dma);
@@ -3244,8 +3241,7 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 		} while (ioc->rdpq_array_enable &&
 			   (++i < ioc->reply_queue_count));
 
-		if (ioc->reply_post_free_dma_pool)
-			pci_pool_destroy(ioc->reply_post_free_dma_pool);
+		dma_pool_destroy(ioc->reply_post_free_dma_pool);
 		kfree(ioc->reply_post);
 	}
 
@@ -3266,12 +3262,11 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 	if (ioc->chain_lookup) {
 		for (i = 0; i < ioc->chain_depth; i++) {
 			if (ioc->chain_lookup[i].chain_buffer)
-				pci_pool_free(ioc->chain_dma_pool,
+				dma_pool_free(ioc->chain_dma_pool,
 				    ioc->chain_lookup[i].chain_buffer,
 				    ioc->chain_lookup[i].chain_buffer_dma);
 		}
-		if (ioc->chain_dma_pool)
-			pci_pool_destroy(ioc->chain_dma_pool);
+		dma_pool_destroy(ioc->chain_dma_pool);
 		free_pages((ulong)ioc->chain_lookup, ioc->chain_pages);
 		ioc->chain_lookup = NULL;
 	}
@@ -3446,23 +3441,23 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 			ioc->name);
 		goto out;
 	}
-	ioc->reply_post_free_dma_pool = pci_pool_create("reply_post_free pool",
-	    ioc->pdev, sz, 16, 0);
+	ioc->reply_post_free_dma_pool = dma_pool_create("reply_post_free pool",
+	    &ioc->pdev->dev, sz, 16, 0);
 	if (!ioc->reply_post_free_dma_pool) {
 		pr_err(MPT3SAS_FMT
-		 "reply_post_free pool: pci_pool_create failed\n",
+		 "reply_post_free pool: dma_pool_create failed\n",
 		 ioc->name);
 		goto out;
 	}
 	i = 0;
 	do {
 		ioc->reply_post[i].reply_post_free =
-		    pci_pool_alloc(ioc->reply_post_free_dma_pool,
+		    dma_pool_alloc(ioc->reply_post_free_dma_pool,
 		    GFP_KERNEL,
 		    &ioc->reply_post[i].reply_post_free_dma);
 		if (!ioc->reply_post[i].reply_post_free) {
 			pr_err(MPT3SAS_FMT
-			"reply_post_free pool: pci_pool_alloc failed\n",
+			"reply_post_free pool: dma_pool_alloc failed\n",
 			ioc->name);
 			goto out;
 		}
@@ -3577,15 +3572,15 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 			ioc->name);
 		goto out;
 	}
-	ioc->chain_dma_pool = pci_pool_create("chain pool", ioc->pdev,
+	ioc->chain_dma_pool = dma_pool_create("chain pool", &ioc->pdev->dev,
 	    ioc->chain_segment_sz, 16, 0);
 	if (!ioc->chain_dma_pool) {
-		pr_err(MPT3SAS_FMT "chain_dma_pool: pci_pool_create failed\n",
+		pr_err(MPT3SAS_FMT "chain_dma_pool: dma_pool_create failed\n",
 			ioc->name);
 		goto out;
 	}
 	for (i = 0; i < ioc->chain_depth; i++) {
-		ioc->chain_lookup[i].chain_buffer = pci_pool_alloc(
+		ioc->chain_lookup[i].chain_buffer = dma_pool_alloc(
 		    ioc->chain_dma_pool , GFP_KERNEL,
 		    &ioc->chain_lookup[i].chain_buffer_dma);
 		if (!ioc->chain_lookup[i].chain_buffer) {
@@ -3630,17 +3625,17 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 
 	/* sense buffers, 4 byte align */
 	sz = ioc->scsiio_depth * SCSI_SENSE_BUFFERSIZE;
-	ioc->sense_dma_pool = pci_pool_create("sense pool", ioc->pdev, sz, 4,
-	    0);
+	ioc->sense_dma_pool = dma_pool_create("sense pool", &ioc->pdev->dev, sz,
+					      4, 0);
 	if (!ioc->sense_dma_pool) {
-		pr_err(MPT3SAS_FMT "sense pool: pci_pool_create failed\n",
+		pr_err(MPT3SAS_FMT "sense pool: dma_pool_create failed\n",
 		    ioc->name);
 		goto out;
 	}
-	ioc->sense = pci_pool_alloc(ioc->sense_dma_pool , GFP_KERNEL,
+	ioc->sense = dma_pool_alloc(ioc->sense_dma_pool, GFP_KERNEL,
 	    &ioc->sense_dma);
 	if (!ioc->sense) {
-		pr_err(MPT3SAS_FMT "sense pool: pci_pool_alloc failed\n",
+		pr_err(MPT3SAS_FMT "sense pool: dma_pool_alloc failed\n",
 		    ioc->name);
 		goto out;
 	}
@@ -3654,17 +3649,17 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 
 	/* reply pool, 4 byte align */
 	sz = ioc->reply_free_queue_depth * ioc->reply_sz;
-	ioc->reply_dma_pool = pci_pool_create("reply pool", ioc->pdev, sz, 4,
-	    0);
+	ioc->reply_dma_pool = dma_pool_create("reply pool", &ioc->pdev->dev, sz,
+					      4, 0);
 	if (!ioc->reply_dma_pool) {
-		pr_err(MPT3SAS_FMT "reply pool: pci_pool_create failed\n",
+		pr_err(MPT3SAS_FMT "reply pool: dma_pool_create failed\n",
 		    ioc->name);
 		goto out;
 	}
-	ioc->reply = pci_pool_alloc(ioc->reply_dma_pool , GFP_KERNEL,
+	ioc->reply = dma_pool_alloc(ioc->reply_dma_pool, GFP_KERNEL,
 	    &ioc->reply_dma);
 	if (!ioc->reply) {
-		pr_err(MPT3SAS_FMT "reply pool: pci_pool_alloc failed\n",
+		pr_err(MPT3SAS_FMT "reply pool: dma_pool_alloc failed\n",
 		    ioc->name);
 		goto out;
 	}
@@ -3680,17 +3675,17 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 
 	/* reply free queue, 16 byte align */
 	sz = ioc->reply_free_queue_depth * 4;
-	ioc->reply_free_dma_pool = pci_pool_create("reply_free pool",
-	    ioc->pdev, sz, 16, 0);
+	ioc->reply_free_dma_pool = dma_pool_create("reply_free pool",
+	    &ioc->pdev->dev, sz, 16, 0);
 	if (!ioc->reply_free_dma_pool) {
-		pr_err(MPT3SAS_FMT "reply_free pool: pci_pool_create failed\n",
+		pr_err(MPT3SAS_FMT "reply_free pool: dma_pool_create failed\n",
 			ioc->name);
 		goto out;
 	}
-	ioc->reply_free = pci_pool_alloc(ioc->reply_free_dma_pool , GFP_KERNEL,
+	ioc->reply_free = dma_pool_alloc(ioc->reply_free_dma_pool, GFP_KERNEL,
 	    &ioc->reply_free_dma);
 	if (!ioc->reply_free) {
-		pr_err(MPT3SAS_FMT "reply_free pool: pci_pool_alloc failed\n",
+		pr_err(MPT3SAS_FMT "reply_free pool: dma_pool_alloc failed\n",
 			ioc->name);
 		goto out;
 	}
@@ -3708,7 +3703,7 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
 	    ioc->config_page_sz, &ioc->config_page_dma);
 	if (!ioc->config_page) {
 		pr_err(MPT3SAS_FMT
-			"config page: pci_pool_alloc failed\n",
+			"config page: dma_pool_alloc failed\n",
 			ioc->name);
 		goto out;
 	}
@@ -5499,10 +5494,10 @@ mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc)
 	ioc->ctl_cmds.status = MPT3_CMD_NOT_USED;
 	mutex_init(&ioc->ctl_cmds.mutex);
 
-	if (!ioc->base_cmds.reply || !ioc->transport_cmds.reply ||
-	    !ioc->scsih_cmds.reply || !ioc->tm_cmds.reply ||
-	    !ioc->config_cmds.reply || !ioc->ctl_cmds.reply ||
-	    !ioc->ctl_cmds.sense) {
+	if (!ioc->base_cmds.reply || !ioc->port_enable_cmds.reply ||
+	    !ioc->transport_cmds.reply || !ioc->scsih_cmds.reply ||
+	    !ioc->tm_cmds.reply || !ioc->config_cmds.reply ||
+	    !ioc->ctl_cmds.reply || !ioc->ctl_cmds.sense) {
 		r = -ENOMEM;
 		goto out_free_resources;
 	}
diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.h b/drivers/scsi/mpt3sas/mpt3sas_base.h
index 099ab4ca7edf..a77bb7dc12b1 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_base.h
+++ b/drivers/scsi/mpt3sas/mpt3sas_base.h
@@ -970,7 +970,7 @@ struct MPT3SAS_ADAPTER {
 	u8		id;
 	int		cpu_count;
 	char		name[MPT_NAME_LENGTH];
-	char		driver_name[MPT_NAME_LENGTH];
+	char		driver_name[MPT_NAME_LENGTH - 8];
 	char		tmp_string[MPT_STRING_LENGTH];
 	struct pci_dev	*pdev;
 	Mpi2SystemInterfaceRegs_t __iomem *chip;
diff --git a/drivers/scsi/mpt3sas/mpt3sas_transport.c b/drivers/scsi/mpt3sas/mpt3sas_transport.c
index e7a7a704a315..d3940c5d079d 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_transport.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_transport.c
@@ -1870,6 +1870,38 @@ _transport_phy_speed(struct sas_phy *phy, struct sas_phy_linkrates *rates)
 	return rc;
 }
 
+static int
+_transport_map_smp_buffer(struct device *dev, struct bsg_buffer *buf,
+		dma_addr_t *dma_addr, size_t *dma_len, void **p)
+{
+	/* Check if the request is split across multiple segments */
+	if (buf->sg_cnt > 1) {
+		*p = dma_alloc_coherent(dev, buf->payload_len, dma_addr,
+				GFP_KERNEL);
+		if (!*p)
+			return -ENOMEM;
+		*dma_len = buf->payload_len;
+	} else {
+		if (!dma_map_sg(dev, buf->sg_list, 1, DMA_BIDIRECTIONAL))
+			return -ENOMEM;
+		*dma_addr = sg_dma_address(buf->sg_list);
+		*dma_len = sg_dma_len(buf->sg_list);
+		*p = NULL;
+	}
+
+	return 0;
+}
+
+static void
+_transport_unmap_smp_buffer(struct device *dev, struct bsg_buffer *buf,
+		dma_addr_t dma_addr, void *p)
+{
+	if (p)
+		dma_free_coherent(dev, buf->payload_len, p, dma_addr);
+	else
+		dma_unmap_sg(dev, buf->sg_list, 1, DMA_BIDIRECTIONAL);
+}
+
 /**
  * _transport_smp_handler - transport portal for smp passthru
  * @shost: shost object
@@ -1880,9 +1912,9 @@ _transport_phy_speed(struct sas_phy *phy, struct sas_phy_linkrates *rates)
  * Example:
  *           smp_rep_general /sys/class/bsg/expander-5:0
  */
-static int
-_transport_smp_handler(struct Scsi_Host *shost, struct sas_rphy *rphy,
-	struct request *req)
+static void
+_transport_smp_handler(struct bsg_job *job, struct Scsi_Host *shost,
+		struct sas_rphy *rphy)
 {
 	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
 	Mpi2SmpPassthroughRequest_t *mpi_request;
@@ -1891,33 +1923,25 @@ _transport_smp_handler(struct Scsi_Host *shost, struct sas_rphy *rphy,
 	u16 smid;
 	u32 ioc_state;
 	void *psge;
-	u8 issue_reset = 0;
-	dma_addr_t dma_addr_in = 0;
-	dma_addr_t dma_addr_out = 0;
-	dma_addr_t pci_dma_in = 0;
-	dma_addr_t pci_dma_out = 0;
-	void *pci_addr_in = NULL;
-	void *pci_addr_out = NULL;
+	dma_addr_t dma_addr_in;
+	dma_addr_t dma_addr_out;
+	void *addr_in = NULL;
+	void *addr_out = NULL;
+	size_t dma_len_in;
+	size_t dma_len_out;
 	u16 wait_state_count;
-	struct request *rsp = req->next_rq;
-	struct bio_vec bvec;
-	struct bvec_iter iter;
-
-	if (!rsp) {
-		pr_err(MPT3SAS_FMT "%s: the smp response space is missing\n",
-			ioc->name, __func__);
-		return -EINVAL;
-	}
+	unsigned int reslen = 0;
 
 	if (ioc->shost_recovery || ioc->pci_error_recovery) {
 		pr_info(MPT3SAS_FMT "%s: host reset in progress!\n",
 		    __func__, ioc->name);
-		return -EFAULT;
+		rc = -EFAULT;
+		goto out;
 	}
 
 	rc = mutex_lock_interruptible(&ioc->transport_cmds.mutex);
 	if (rc)
-		return rc;
+		goto out;
 
 	if (ioc->transport_cmds.status != MPT3_CMD_NOT_USED) {
 		pr_err(MPT3SAS_FMT "%s: transport_cmds in use\n", ioc->name,
@@ -1927,58 +1951,20 @@ _transport_smp_handler(struct Scsi_Host *shost, struct sas_rphy *rphy,
 	}
 	ioc->transport_cmds.status = MPT3_CMD_PENDING;
 
-	/* Check if the request is split across multiple segments */
-	if (bio_multiple_segments(req->bio)) {
-		u32 offset = 0;
-
-		/* Allocate memory and copy the request */
-		pci_addr_out = pci_alloc_consistent(ioc->pdev,
-		    blk_rq_bytes(req), &pci_dma_out);
-		if (!pci_addr_out) {
-			pr_info(MPT3SAS_FMT "%s(): PCI Addr out = NULL\n",
-			    ioc->name, __func__);
-			rc = -ENOMEM;
-			goto out;
-		}
-
-		bio_for_each_segment(bvec, req->bio, iter) {
-			memcpy(pci_addr_out + offset,
-			    page_address(bvec.bv_page) + bvec.bv_offset,
-			    bvec.bv_len);
-			offset += bvec.bv_len;
-		}
-	} else {
-		dma_addr_out = pci_map_single(ioc->pdev, bio_data(req->bio),
-		    blk_rq_bytes(req), PCI_DMA_BIDIRECTIONAL);
-		if (pci_dma_mapping_error(ioc->pdev, dma_addr_out)) {
-			pr_info(MPT3SAS_FMT "%s(): DMA Addr out = NULL\n",
-			    ioc->name, __func__);
-			rc = -ENOMEM;
-			goto free_pci;
-		}
+	rc = _transport_map_smp_buffer(&ioc->pdev->dev, &job->request_payload,
+			&dma_addr_out, &dma_len_out, &addr_out);
+	if (rc)
+		goto out;
+	if (addr_out) {
+		sg_copy_to_buffer(job->request_payload.sg_list,
+				job->request_payload.sg_cnt, addr_out,
+				job->request_payload.payload_len);
 	}
 
-	/* Check if the response needs to be populated across
-	 * multiple segments */
-	if (bio_multiple_segments(rsp->bio)) {
-		pci_addr_in = pci_alloc_consistent(ioc->pdev, blk_rq_bytes(rsp),
-		    &pci_dma_in);
-		if (!pci_addr_in) {
-			pr_info(MPT3SAS_FMT "%s(): PCI Addr in = NULL\n",
-			    ioc->name, __func__);
-			rc = -ENOMEM;
-			goto unmap;
-		}
-	} else {
-		dma_addr_in =  pci_map_single(ioc->pdev, bio_data(rsp->bio),
-		    blk_rq_bytes(rsp), PCI_DMA_BIDIRECTIONAL);
-		if (pci_dma_mapping_error(ioc->pdev, dma_addr_in)) {
-			pr_info(MPT3SAS_FMT "%s(): DMA Addr in = NULL\n",
-			    ioc->name, __func__);
-			rc = -ENOMEM;
-			goto unmap;
-		}
-	}
+	rc = _transport_map_smp_buffer(&ioc->pdev->dev, &job->reply_payload,
+			&dma_addr_in, &dma_len_in, &addr_in);
+	if (rc)
+		goto unmap_out;
 
 	wait_state_count = 0;
 	ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
@@ -1988,7 +1974,7 @@ _transport_smp_handler(struct Scsi_Host *shost, struct sas_rphy *rphy,
 			    "%s: failed due to ioc not operational\n",
 			    ioc->name, __func__);
 			rc = -EFAULT;
-			goto unmap;
+			goto unmap_in;
 		}
 		ssleep(1);
 		ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
@@ -2005,7 +1991,7 @@ _transport_smp_handler(struct Scsi_Host *shost, struct sas_rphy *rphy,
 		pr_err(MPT3SAS_FMT "%s: failed obtaining a smid\n",
 		    ioc->name, __func__);
 		rc = -EAGAIN;
-		goto unmap;
+		goto unmap_in;
 	}
 
 	rc = 0;
@@ -2018,15 +2004,11 @@ _transport_smp_handler(struct Scsi_Host *shost, struct sas_rphy *rphy,
 	mpi_request->SASAddress = (rphy) ?
 	    cpu_to_le64(rphy->identify.sas_address) :
 	    cpu_to_le64(ioc->sas_hba.sas_address);
-	mpi_request->RequestDataLength = cpu_to_le16(blk_rq_bytes(req) - 4);
+	mpi_request->RequestDataLength = cpu_to_le16(dma_len_out - 4);
 	psge = &mpi_request->SGL;
 
-	if (bio_multiple_segments(req->bio))
-		ioc->build_sg(ioc, psge, pci_dma_out, (blk_rq_bytes(req) - 4),
-		    pci_dma_in, (blk_rq_bytes(rsp) + 4));
-	else
-		ioc->build_sg(ioc, psge, dma_addr_out, (blk_rq_bytes(req) - 4),
-		    dma_addr_in, (blk_rq_bytes(rsp) + 4));
+	ioc->build_sg(ioc, psge, dma_addr_out, dma_len_out - 4, dma_addr_in,
+			dma_len_in - 4);
 
 	dtransportprintk(ioc, pr_info(MPT3SAS_FMT
 		"%s - sending smp request\n", ioc->name, __func__));
@@ -2040,83 +2022,51 @@ _transport_smp_handler(struct Scsi_Host *shost, struct sas_rphy *rphy,
 		    __func__, ioc->name);
 		_debug_dump_mf(mpi_request,
 		    sizeof(Mpi2SmpPassthroughRequest_t)/4);
-		if (!(ioc->transport_cmds.status & MPT3_CMD_RESET))
-			issue_reset = 1;
-		goto issue_host_reset;
+		if (!(ioc->transport_cmds.status & MPT3_CMD_RESET)) {
+			mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
+			rc = -ETIMEDOUT;
+			goto unmap_in;
+		}
 	}
 
 	dtransportprintk(ioc, pr_info(MPT3SAS_FMT
 		"%s - complete\n", ioc->name, __func__));
 
-	if (ioc->transport_cmds.status & MPT3_CMD_REPLY_VALID) {
-
-		mpi_reply = ioc->transport_cmds.reply;
-
-		dtransportprintk(ioc, pr_info(MPT3SAS_FMT
-		    "%s - reply data transfer size(%d)\n",
-		    ioc->name, __func__,
-		    le16_to_cpu(mpi_reply->ResponseDataLength)));
-
-		memcpy(scsi_req(req)->sense, mpi_reply, sizeof(*mpi_reply));
-		scsi_req(req)->sense_len = sizeof(*mpi_reply);
-		scsi_req(req)->resid_len = 0;
-		scsi_req(rsp)->resid_len -=
-		    le16_to_cpu(mpi_reply->ResponseDataLength);
-
-		/* check if the resp needs to be copied from the allocated
-		 * pci mem */
-		if (bio_multiple_segments(rsp->bio)) {
-			u32 offset = 0;
-			u32 bytes_to_copy =
-			    le16_to_cpu(mpi_reply->ResponseDataLength);
-			bio_for_each_segment(bvec, rsp->bio, iter) {
-				if (bytes_to_copy <= bvec.bv_len) {
-					memcpy(page_address(bvec.bv_page) +
-					    bvec.bv_offset, pci_addr_in +
-					    offset, bytes_to_copy);
-					break;
-				} else {
-					memcpy(page_address(bvec.bv_page) +
-					    bvec.bv_offset, pci_addr_in +
-					    offset, bvec.bv_len);
-					bytes_to_copy -= bvec.bv_len;
-				}
-				offset += bvec.bv_len;
-			}
-		}
-	} else {
+	if (!(ioc->transport_cmds.status & MPT3_CMD_REPLY_VALID)) {
 		dtransportprintk(ioc, pr_info(MPT3SAS_FMT
 		    "%s - no reply\n", ioc->name, __func__));
 		rc = -ENXIO;
+		goto unmap_in;
 	}
 
- issue_host_reset:
-	if (issue_reset) {
-		mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
-		rc = -ETIMEDOUT;
-	}
+	mpi_reply = ioc->transport_cmds.reply;
 
- unmap:
-	if (dma_addr_out)
-		pci_unmap_single(ioc->pdev, dma_addr_out, blk_rq_bytes(req),
-		    PCI_DMA_BIDIRECTIONAL);
-	if (dma_addr_in)
-		pci_unmap_single(ioc->pdev, dma_addr_in, blk_rq_bytes(rsp),
-		    PCI_DMA_BIDIRECTIONAL);
+	dtransportprintk(ioc,
+		pr_info(MPT3SAS_FMT "%s - reply data transfer size(%d)\n",
+			ioc->name, __func__,
+			le16_to_cpu(mpi_reply->ResponseDataLength)));
 
- free_pci:
-	if (pci_addr_out)
-		pci_free_consistent(ioc->pdev, blk_rq_bytes(req), pci_addr_out,
-		    pci_dma_out);
+	memcpy(job->reply, mpi_reply, sizeof(*mpi_reply));
+	job->reply_len = sizeof(*mpi_reply);
+	reslen = le16_to_cpu(mpi_reply->ResponseDataLength);
 
-	if (pci_addr_in)
-		pci_free_consistent(ioc->pdev, blk_rq_bytes(rsp), pci_addr_in,
-		    pci_dma_in);
+	if (addr_in) {
+		sg_copy_to_buffer(job->reply_payload.sg_list,
+				job->reply_payload.sg_cnt, addr_in,
+				job->reply_payload.payload_len);
+	}
 
+	rc = 0;
+ unmap_in:
+	_transport_unmap_smp_buffer(&ioc->pdev->dev, &job->reply_payload,
+			dma_addr_in, addr_in);
+ unmap_out:
+	_transport_unmap_smp_buffer(&ioc->pdev->dev, &job->request_payload,
+			dma_addr_out, addr_out);
  out:
 	ioc->transport_cmds.status = MPT3_CMD_NOT_USED;
 	mutex_unlock(&ioc->transport_cmds.mutex);
-	return rc;
+	bsg_job_done(job, rc, reslen);
 }
 
 struct sas_function_template mpt3sas_transport_functions = {
diff --git a/drivers/scsi/mvme147.c b/drivers/scsi/mvme147.c
index e7f6661a8862..4f515700bdc3 100644
--- a/drivers/scsi/mvme147.c
+++ b/drivers/scsi/mvme147.c
@@ -121,21 +121,6 @@ err_out:
 	return 0;
 }
 
-static int mvme147_bus_reset(struct scsi_cmnd *cmd)
-{
-	/* FIXME perform bus-specific reset */
-
-	/* FIXME 2: kill this function, and let midlayer fallback to
-	   the same result, calling wd33c93_host_reset() */
-
-	spin_lock_irq(cmd->device->host->host_lock);
-	wd33c93_host_reset(cmd);
-	spin_unlock_irq(cmd->device->host->host_lock);
-
-	return SUCCESS;
-}
-
-
 static struct scsi_host_template driver_template = {
 	.proc_name		= "MVME147",
 	.name			= "MVME147 built-in SCSI",
@@ -143,7 +128,6 @@ static struct scsi_host_template driver_template = {
 	.release		= mvme147_release,
 	.queuecommand		= wd33c93_queuecommand,
 	.eh_abort_handler	= wd33c93_abort,
-	.eh_bus_reset_handler	= mvme147_bus_reset,
 	.eh_host_reset_handler	= wd33c93_host_reset,
 	.can_queue		= CAN_QUEUE,
 	.this_id		= 7,
diff --git a/drivers/scsi/mvsas/mv_init.c b/drivers/scsi/mvsas/mv_init.c
index 4e047b5001a6..718c88de328b 100644
--- a/drivers/scsi/mvsas/mv_init.c
+++ b/drivers/scsi/mvsas/mv_init.c
@@ -61,7 +61,7 @@ static struct scsi_host_template mvs_sht = {
 	.max_sectors		= SCSI_DEFAULT_MAX_SECTORS,
 	.use_clustering		= ENABLE_CLUSTERING,
 	.eh_device_reset_handler = sas_eh_device_reset_handler,
-	.eh_bus_reset_handler	= sas_eh_bus_reset_handler,
+	.eh_target_reset_handler = sas_eh_target_reset_handler,
 	.target_destroy		= sas_target_destroy,
 	.ioctl			= sas_ioctl,
 	.shost_attrs		= mvst_host_attrs,
@@ -125,8 +125,7 @@ static void mvs_free(struct mvs_info *mvi)
 	else
 		slot_nr = MVS_CHIP_SLOT_SZ;
 
-	if (mvi->dma_pool)
-		pci_pool_destroy(mvi->dma_pool);
+	dma_pool_destroy(mvi->dma_pool);
 
 	if (mvi->tx)
 		dma_free_coherent(mvi->dev,
@@ -296,7 +295,8 @@ static int mvs_alloc(struct mvs_info *mvi, struct Scsi_Host *shost)
 		goto err_out;
 
 	sprintf(pool_name, "%s%d", "mvs_dma_pool", mvi->id);
-	mvi->dma_pool = pci_pool_create(pool_name, mvi->pdev, MVS_SLOT_BUF_SZ, 16, 0);
+	mvi->dma_pool = dma_pool_create(pool_name, &mvi->pdev->dev,
+					MVS_SLOT_BUF_SZ, 16, 0);
 	if (!mvi->dma_pool) {
 			printk(KERN_DEBUG "failed to create dma pool %s.\n", pool_name);
 			goto err_out;
@@ -557,14 +557,14 @@ static int mvs_pci_init(struct pci_dev *pdev, const struct pci_device_id *ent)
 	SHOST_TO_SAS_HA(shost) =
 		kcalloc(1, sizeof(struct sas_ha_struct), GFP_KERNEL);
 	if (!SHOST_TO_SAS_HA(shost)) {
-		kfree(shost);
+		scsi_host_put(shost);
 		rc = -ENOMEM;
 		goto err_out_regions;
 	}
 
 	rc = mvs_prep_sas_ha_init(shost, chip);
 	if (rc) {
-		kfree(shost);
+		scsi_host_put(shost);
 		rc = -ENOMEM;
 		goto err_out_regions;
 	}
diff --git a/drivers/scsi/mvsas/mv_sas.c b/drivers/scsi/mvsas/mv_sas.c
index c7cc8035eacb..ee81d10252e0 100644
--- a/drivers/scsi/mvsas/mv_sas.c
+++ b/drivers/scsi/mvsas/mv_sas.c
@@ -790,7 +790,7 @@ static int mvs_task_prep(struct sas_task *task, struct mvs_info *mvi, int is_tmf
 	slot->n_elem = n_elem;
 	slot->slot_tag = tag;
 
-	slot->buf = pci_pool_alloc(mvi->dma_pool, GFP_ATOMIC, &slot->buf_dma);
+	slot->buf = dma_pool_alloc(mvi->dma_pool, GFP_ATOMIC, &slot->buf_dma);
 	if (!slot->buf) {
 		rc = -ENOMEM;
 		goto err_out_tag;
@@ -840,7 +840,7 @@ static int mvs_task_prep(struct sas_task *task, struct mvs_info *mvi, int is_tmf
 	return rc;
 
 err_out_slot_buf:
-	pci_pool_free(mvi->dma_pool, slot->buf, slot->buf_dma);
+	dma_pool_free(mvi->dma_pool, slot->buf, slot->buf_dma);
 err_out_tag:
 	mvs_tag_free(mvi, tag);
 err_out:
@@ -918,7 +918,7 @@ static void mvs_slot_task_free(struct mvs_info *mvi, struct sas_task *task,
 	}
 
 	if (slot->buf) {
-		pci_pool_free(mvi->dma_pool, slot->buf, slot->buf_dma);
+		dma_pool_free(mvi->dma_pool, slot->buf, slot->buf_dma);
 		slot->buf = NULL;
 	}
 	list_del_init(&slot->entry);
diff --git a/drivers/scsi/nsp32.c b/drivers/scsi/nsp32.c
index 53c84771f0e8..107e191bf023 100644
--- a/drivers/scsi/nsp32.c
+++ b/drivers/scsi/nsp32.c
@@ -201,7 +201,6 @@ static int         nsp32_release     (struct Scsi_Host *);
 
 /* SCSI error handler */
 static int         nsp32_eh_abort     (struct scsi_cmnd *);
-static int         nsp32_eh_bus_reset (struct scsi_cmnd *);
 static int         nsp32_eh_host_reset(struct scsi_cmnd *);
 
 /* generate SCSI message */
@@ -276,8 +275,7 @@ static struct scsi_host_template nsp32_template = {
 	.max_sectors			= 128,
 	.this_id			= NSP32_HOST_SCSIID,
 	.use_clustering			= DISABLE_CLUSTERING,
-	.eh_abort_handler       	= nsp32_eh_abort,
-	.eh_bus_reset_handler		= nsp32_eh_bus_reset,
+	.eh_abort_handler		= nsp32_eh_abort,
 	.eh_host_reset_handler		= nsp32_eh_host_reset,
 /*	.highmem_io			= 1, */
 };
@@ -2845,24 +2843,6 @@ static int nsp32_eh_abort(struct scsi_cmnd *SCpnt)
 	return SUCCESS;
 }
 
-static int nsp32_eh_bus_reset(struct scsi_cmnd *SCpnt)
-{
-	nsp32_hw_data *data = (nsp32_hw_data *)SCpnt->device->host->hostdata;
-	unsigned int   base = SCpnt->device->host->io_port;
-
-	spin_lock_irq(SCpnt->device->host->host_lock);
-
-	nsp32_msg(KERN_INFO, "Bus Reset");	
-	nsp32_dbg(NSP32_DEBUG_BUSRESET, "SCpnt=0x%x", SCpnt);
-
-	nsp32_write2(base, IRQ_CONTROL, IRQ_CONTROL_ALL_IRQ_MASK);
-	nsp32_do_bus_reset(data);
-	nsp32_write2(base, IRQ_CONTROL, 0);
-
-	spin_unlock_irq(SCpnt->device->host->host_lock);
-	return SUCCESS;	/* SCSI bus reset is succeeded at any time. */
-}
-
 static void nsp32_do_bus_reset(nsp32_hw_data *data)
 {
 	unsigned int   base = data->BaseAddress;
diff --git a/drivers/scsi/osst.c b/drivers/scsi/osst.c
index 929ee7e88120..20ec1c01dbd5 100644
--- a/drivers/scsi/osst.c
+++ b/drivers/scsi/osst.c
@@ -619,7 +619,7 @@ static int osst_verify_frame(struct osst_tape * STp, int frame_seq_number, int q
 	os_aux_t           * aux  = STp->buffer->aux;
 	os_partition_t     * par  = &(aux->partition);
 	struct st_partstat * STps = &(STp->ps[STp->partition]);
-	int		     blk_cnt, blk_sz, i;
+	unsigned int	     blk_cnt, blk_sz, i;
 
 	if (STp->raw) {
 		if (STp->buffer->syscall_result) {
@@ -5434,7 +5434,7 @@ static int append_to_buffer(const char __user *ubp, struct osst_buffer *st_bp, i
 
 	for (i=0, offset=st_bp->buffer_bytes;
 	     i < st_bp->sg_segs && offset >= st_bp->sg[i].length; i++)
-	offset -= st_bp->sg[i].length;
+		offset -= st_bp->sg[i].length;
 	if (i == st_bp->sg_segs) {  /* Should never happen */
 		printk(KERN_WARNING "osst :A: Append_to_buffer offset overflow.\n");
 		return (-EIO);
@@ -5667,12 +5667,12 @@ static	struct	osst_support_data support_list[] = {
  * sysfs support for osst driver parameter information
  */
 
-static ssize_t osst_version_show(struct device_driver *ddd, char *buf)
+static ssize_t version_show(struct device_driver *ddd, char *buf)
 {
 	return snprintf(buf, PAGE_SIZE, "%s\n", osst_version);
 }
 
-static DRIVER_ATTR(version, S_IRUGO, osst_version_show, NULL);
+static DRIVER_ATTR_RO(version);
 
 static int osst_create_sysfs_files(struct device_driver *sysfs)
 {
diff --git a/drivers/scsi/pcmcia/fdomain_stub.c b/drivers/scsi/pcmcia/fdomain_stub.c
index 714b248f5d5e..953a792150ae 100644
--- a/drivers/scsi/pcmcia/fdomain_stub.c
+++ b/drivers/scsi/pcmcia/fdomain_stub.c
@@ -173,7 +173,7 @@ static void fdomain_release(struct pcmcia_device *link)
 
 static int fdomain_resume(struct pcmcia_device *link)
 {
-	fdomain_16x0_bus_reset(NULL);
+	fdomain_16x0_host_reset(NULL);
 
 	return 0;
 }
diff --git a/drivers/scsi/pcmcia/qlogic_stub.c b/drivers/scsi/pcmcia/qlogic_stub.c
index c670dc704c74..0556054764dc 100644
--- a/drivers/scsi/pcmcia/qlogic_stub.c
+++ b/drivers/scsi/pcmcia/qlogic_stub.c
@@ -67,7 +67,7 @@ static struct scsi_host_template qlogicfas_driver_template = {
 	.info			= qlogicfas408_info,
 	.queuecommand		= qlogicfas408_queuecommand,
 	.eh_abort_handler	= qlogicfas408_abort,
-	.eh_bus_reset_handler	= qlogicfas408_bus_reset,
+	.eh_host_reset_handler	= qlogicfas408_host_reset,
 	.bios_param		= qlogicfas408_biosparam,
 	.can_queue		= 1,
 	.this_id		= -1,
@@ -264,7 +264,7 @@ static int qlogic_resume(struct pcmcia_device *link)
 		outb(0x04, link->resource[0]->start + 0xd);
 	}
 	/* Ugggglllyyyy!!! */
-	qlogicfas408_bus_reset(NULL);
+	qlogicfas408_host_reset(NULL);
 
 	return 0;
 }
diff --git a/drivers/scsi/pm8001/pm8001_init.c b/drivers/scsi/pm8001/pm8001_init.c
index 034b2f7d1135..0e013f76b582 100644
--- a/drivers/scsi/pm8001/pm8001_init.c
+++ b/drivers/scsi/pm8001/pm8001_init.c
@@ -86,7 +86,7 @@ static struct scsi_host_template pm8001_sht = {
 	.max_sectors		= SCSI_DEFAULT_MAX_SECTORS,
 	.use_clustering		= ENABLE_CLUSTERING,
 	.eh_device_reset_handler = sas_eh_device_reset_handler,
-	.eh_bus_reset_handler	= sas_eh_bus_reset_handler,
+	.eh_target_reset_handler = sas_eh_target_reset_handler,
 	.target_destroy		= sas_target_destroy,
 	.ioctl			= sas_ioctl,
 	.shost_attrs		= pm8001_host_attrs,
@@ -160,8 +160,6 @@ static void pm8001_free(struct pm8001_hba_info *pm8001_ha)
 			}
 	}
 	PM8001_CHIP_DISP->chip_iounmap(pm8001_ha);
-	if (pm8001_ha->shost)
-		scsi_host_put(pm8001_ha->shost);
 	flush_workqueue(pm8001_wq);
 	kfree(pm8001_ha->tags);
 	kfree(pm8001_ha);
@@ -1073,7 +1071,7 @@ err_out_ha_free:
 err_out_free:
 	kfree(SHOST_TO_SAS_HA(shost));
 err_out_free_host:
-	kfree(shost);
+	scsi_host_put(shost);
 err_out_regions:
 	pci_release_regions(pdev);
 err_out_disable:
@@ -1112,6 +1110,7 @@ static void pm8001_pci_remove(struct pci_dev *pdev)
 		for (j = 0; j < PM8001_MAX_MSIX_VEC; j++)
 			tasklet_kill(&pm8001_ha->tasklet[j]);
 #endif
+	scsi_host_put(pm8001_ha->shost);
 	pm8001_free(pm8001_ha);
 	kfree(sha->sas_phy);
 	kfree(sha->sas_port);
diff --git a/drivers/scsi/pmcraid.c b/drivers/scsi/pmcraid.c
index 1cc814f1505a..b4d6cd8cd1ad 100644
--- a/drivers/scsi/pmcraid.c
+++ b/drivers/scsi/pmcraid.c
@@ -1595,12 +1595,7 @@ static void pmcraid_handle_config_change(struct pmcraid_instance *pinstance)
 	if (pinstance->ccn.hcam->notification_type ==
 	    NOTIFICATION_TYPE_ENTRY_CHANGED &&
 	    cfg_entry->resource_type == RES_TYPE_VSET) {
-
-		if (fw_version <= PMCRAID_FW_VERSION_1)
-			hidden_entry = (cfg_entry->unique_flags1 & 0x80) != 0;
-		else
-			hidden_entry = (cfg_entry->unique_flags1 & 0x80) != 0;
-
+		hidden_entry = (cfg_entry->unique_flags1 & 0x80) != 0;
 	} else if (!pmcraid_expose_resource(fw_version, cfg_entry)) {
 		goto out_notify_apps;
 	}
@@ -4655,13 +4650,13 @@ pmcraid_release_control_blocks(
 		return;
 
 	for (i = 0; i < max_index; i++) {
-		pci_pool_free(pinstance->control_pool,
+		dma_pool_free(pinstance->control_pool,
 			      pinstance->cmd_list[i]->ioa_cb,
 			      pinstance->cmd_list[i]->ioa_cb_bus_addr);
 		pinstance->cmd_list[i]->ioa_cb = NULL;
 		pinstance->cmd_list[i]->ioa_cb_bus_addr = 0;
 	}
-	pci_pool_destroy(pinstance->control_pool);
+	dma_pool_destroy(pinstance->control_pool);
 	pinstance->control_pool = NULL;
 }
 
@@ -4718,8 +4713,8 @@ static int pmcraid_allocate_control_blocks(struct pmcraid_instance *pinstance)
 		pinstance->host->unique_id);
 
 	pinstance->control_pool =
-		pci_pool_create(pinstance->ctl_pool_name,
-				pinstance->pdev,
+		dma_pool_create(pinstance->ctl_pool_name,
+				&pinstance->pdev->dev,
 				sizeof(struct pmcraid_control_block),
 				PMCRAID_IOARCB_ALIGNMENT, 0);
 
@@ -4728,7 +4723,7 @@ static int pmcraid_allocate_control_blocks(struct pmcraid_instance *pinstance)
 
 	for (i = 0; i < PMCRAID_MAX_CMD; i++) {
 		pinstance->cmd_list[i]->ioa_cb =
-			pci_pool_alloc(
+			dma_pool_alloc(
 				pinstance->control_pool,
 				GFP_KERNEL,
 				&(pinstance->cmd_list[i]->ioa_cb_bus_addr));
diff --git a/drivers/scsi/pmcraid.h b/drivers/scsi/pmcraid.h
index 01eb2bc16dc1..8bfac72a242b 100644
--- a/drivers/scsi/pmcraid.h
+++ b/drivers/scsi/pmcraid.h
@@ -755,7 +755,7 @@ struct pmcraid_instance {
 
 	/* structures related to command blocks */
 	struct kmem_cache *cmd_cachep;		/* cache for cmd blocks */
-	struct pci_pool *control_pool;		/* pool for control blocks */
+	struct dma_pool *control_pool;		/* pool for control blocks */
 	char   cmd_pool_name[64];		/* name of cmd cache */
 	char   ctl_pool_name[64];		/* name of control cache */
 
diff --git a/drivers/scsi/ppa.c b/drivers/scsi/ppa.c
index f6ad579280d4..7be5823ab036 100644
--- a/drivers/scsi/ppa.c
+++ b/drivers/scsi/ppa.c
@@ -970,7 +970,6 @@ static struct scsi_host_template ppa_template = {
 	.name			= "Iomega VPI0 (ppa) interface",
 	.queuecommand		= ppa_queuecommand,
 	.eh_abort_handler	= ppa_abort,
-	.eh_bus_reset_handler	= ppa_reset,
 	.eh_host_reset_handler	= ppa_reset,
 	.bios_param		= ppa_biosparam,
 	.this_id		= -1,
diff --git a/drivers/scsi/qedf/qedf.h b/drivers/scsi/qedf/qedf.h
index 351f06dfc5a0..9bf7b227e69a 100644
--- a/drivers/scsi/qedf/qedf.h
+++ b/drivers/scsi/qedf/qedf.h
@@ -300,7 +300,6 @@ struct qedf_ctx {
 #define QEDF_FALLBACK_VLAN	1002
 #define QEDF_DEFAULT_PRIO	3
 	int vlan_id;
-	uint vlan_hw_insert:1;
 	struct qed_dev *cdev;
 	struct qed_dev_fcoe_info dev_info;
 	struct qed_int_info int_info;
@@ -443,7 +442,6 @@ extern void qedf_cmd_mgr_free(struct qedf_cmd_mgr *cmgr);
 extern int qedf_queuecommand(struct Scsi_Host *host,
 	struct scsi_cmnd *sc_cmd);
 extern void qedf_fip_send(struct fcoe_ctlr *fip, struct sk_buff *skb);
-extern void qedf_update_src_mac(struct fc_lport *lport, u8 *addr);
 extern u8 *qedf_get_src_mac(struct fc_lport *lport);
 extern void qedf_fip_recv(struct qedf_ctx *qedf, struct sk_buff *skb);
 extern void qedf_fcoe_send_vlan_req(struct qedf_ctx *qedf);
diff --git a/drivers/scsi/qedf/qedf_els.c b/drivers/scsi/qedf/qedf_els.c
index eb07f1de8afa..59c18ca4cda9 100644
--- a/drivers/scsi/qedf/qedf_els.c
+++ b/drivers/scsi/qedf/qedf_els.c
@@ -489,7 +489,7 @@ static void qedf_srr_compl(struct qedf_els_cb_arg *cb_arg)
 
 	/* If a SRR times out, simply free resources */
 	if (srr_req->event == QEDF_IOREQ_EV_ELS_TMO)
-		goto out_free;
+		goto out_put;
 
 	/* Normalize response data into struct fc_frame */
 	mp_req = &(srr_req->mp_req);
@@ -501,7 +501,7 @@ static void qedf_srr_compl(struct qedf_els_cb_arg *cb_arg)
 	if (!fp) {
 		QEDF_ERR(&(qedf->dbg_ctx),
 		    "fc_frame_alloc failure.\n");
-		goto out_free;
+		goto out_put;
 	}
 
 	/* Copy frame header from firmware into fp */
@@ -526,9 +526,10 @@ static void qedf_srr_compl(struct qedf_els_cb_arg *cb_arg)
 	}
 
 	fc_frame_free(fp);
-out_free:
+out_put:
 	/* Put reference for original command since SRR completed */
 	kref_put(&orig_io_req->refcount, qedf_release_cmd);
+out_free:
 	kfree(cb_arg);
 }
 
@@ -780,7 +781,7 @@ static void qedf_rec_compl(struct qedf_els_cb_arg *cb_arg)
 
 	/* If a REC times out, free resources */
 	if (rec_req->event == QEDF_IOREQ_EV_ELS_TMO)
-		goto out_free;
+		goto out_put;
 
 	/* Normalize response data into struct fc_frame */
 	mp_req = &(rec_req->mp_req);
@@ -792,7 +793,7 @@ static void qedf_rec_compl(struct qedf_els_cb_arg *cb_arg)
 	if (!fp) {
 		QEDF_ERR(&(qedf->dbg_ctx),
 		    "fc_frame_alloc failure.\n");
-		goto out_free;
+		goto out_put;
 	}
 
 	/* Copy frame header from firmware into fp */
@@ -884,9 +885,10 @@ static void qedf_rec_compl(struct qedf_els_cb_arg *cb_arg)
 
 out_free_frame:
 	fc_frame_free(fp);
-out_free:
+out_put:
 	/* Put reference for original command since REC completed */
 	kref_put(&orig_io_req->refcount, qedf_release_cmd);
+out_free:
 	kfree(cb_arg);
 }
 
diff --git a/drivers/scsi/qedf/qedf_fip.c b/drivers/scsi/qedf/qedf_fip.c
index aefd24ca9604..773558fc0697 100644
--- a/drivers/scsi/qedf/qedf_fip.c
+++ b/drivers/scsi/qedf/qedf_fip.c
@@ -108,7 +108,6 @@ void qedf_fip_send(struct fcoe_ctlr *fip, struct sk_buff *skb)
 {
 	struct qedf_ctx *qedf = container_of(fip, struct qedf_ctx, ctlr);
 	struct ethhdr *eth_hdr;
-	struct vlan_ethhdr *vlan_hdr;
 	struct fip_header *fiph;
 	u16 op, vlan_tci = 0;
 	u8 sub;
@@ -124,16 +123,14 @@ void qedf_fip_send(struct fcoe_ctlr *fip, struct sk_buff *skb)
 	op = ntohs(fiph->fip_op);
 	sub = fiph->fip_subcode;
 
-	if (!qedf->vlan_hw_insert) {
-		vlan_hdr = skb_push(skb, sizeof(*vlan_hdr) - sizeof(*eth_hdr));
-		memcpy(vlan_hdr, eth_hdr, 2 * ETH_ALEN);
-		vlan_hdr->h_vlan_proto = htons(ETH_P_8021Q);
-		vlan_hdr->h_vlan_encapsulated_proto = eth_hdr->h_proto;
-		vlan_hdr->h_vlan_TCI = vlan_tci =  htons(qedf->vlan_id);
-	}
+	/*
+	 * Add VLAN tag to non-offload FIP frame based on current stored VLAN
+	 * for FIP/FCoE traffic.
+	 */
+	__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), qedf->vlan_id);
 
-	/* Update eth_hdr since we added a VLAN tag */
-	eth_hdr = (struct ethhdr *)skb_mac_header(skb);
+	/* Get VLAN ID from skb for printing purposes */
+	__vlan_hwaccel_get_tag(skb, &vlan_tci);
 
 	QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_LL2, "FIP frame send: "
 	    "dest=%pM op=%x sub=%x vlan=%04x.", eth_hdr->h_dest, op, sub,
@@ -174,7 +171,6 @@ void qedf_fip_recv(struct qedf_ctx *qedf, struct sk_buff *skb)
 	/* Handle FIP VLAN resp in the driver */
 	if (op == FIP_OP_VLAN && sub == FIP_SC_VL_NOTE) {
 		qedf_fcoe_process_vlan_resp(qedf, skb);
-		qedf->vlan_hw_insert = 0;
 		kfree_skb(skb);
 	} else if (op == FIP_OP_CTRL && sub == FIP_SC_CLR_VLINK) {
 		QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC, "Clear virtual "
@@ -242,26 +238,9 @@ void qedf_fip_recv(struct qedf_ctx *qedf, struct sk_buff *skb)
 	}
 }
 
-void qedf_update_src_mac(struct fc_lport *lport, u8 *addr)
-{
-	struct qedf_ctx *qedf = lport_priv(lport);
-
-	QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
-	    "Setting data_src_addr=%pM.\n", addr);
-	ether_addr_copy(qedf->data_src_addr, addr);
-}
-
 u8 *qedf_get_src_mac(struct fc_lport *lport)
 {
-	u8 mac[ETH_ALEN];
-	u8 port_id[3];
 	struct qedf_ctx *qedf = lport_priv(lport);
 
-	/* We need to use the lport port_id to create the data_src_addr */
-	if (is_zero_ether_addr(qedf->data_src_addr)) {
-		hton24(port_id, lport->port_id);
-		fc_fcoe_set_mac(mac, port_id);
-		qedf->ctlr.update_mac(lport, mac);
-	}
 	return qedf->data_src_addr;
 }
diff --git a/drivers/scsi/qedf/qedf_main.c b/drivers/scsi/qedf/qedf_main.c
index 1d13c9ca517d..7c0064500cc5 100644
--- a/drivers/scsi/qedf/qedf_main.c
+++ b/drivers/scsi/qedf/qedf_main.c
@@ -18,6 +18,7 @@
 #include <linux/kthread.h>
 #include <scsi/libfc.h>
 #include <scsi/scsi_host.h>
+#include <scsi/fc_frame.h>
 #include <linux/if_ether.h>
 #include <linux/if_vlan.h>
 #include <linux/cpu.h>
@@ -42,7 +43,7 @@ MODULE_PARM_DESC(dev_loss_tmo,  " dev_loss_tmo setting for attached "
 
 uint qedf_debug = QEDF_LOG_INFO;
 module_param_named(debug, qedf_debug, uint, S_IRUGO);
-MODULE_PARM_DESC(qedf_debug, " Debug mask. Pass '1' to enable default debugging"
+MODULE_PARM_DESC(debug, " Debug mask. Pass '1' to enable default debugging"
 	" mask");
 
 static uint qedf_fipvlan_retries = 30;
@@ -163,7 +164,7 @@ static void qedf_handle_link_update(struct work_struct *work)
 		QEDF_WARN(&(qedf->dbg_ctx), "Did not receive FIP VLAN "
 			   "response, falling back to default VLAN %d.\n",
 			   qedf_fallback_vlan);
-		qedf_set_vlan_id(qedf, QEDF_FALLBACK_VLAN);
+		qedf_set_vlan_id(qedf, qedf_fallback_vlan);
 
 		/*
 		 * Zero out data_src_addr so we'll update it with the new
@@ -187,6 +188,50 @@ static void qedf_handle_link_update(struct work_struct *work)
 	}
 }
 
+#define	QEDF_FCOE_MAC_METHOD_GRANGED_MAC		1
+#define QEDF_FCOE_MAC_METHOD_FCF_MAP			2
+#define QEDF_FCOE_MAC_METHOD_FCOE_SET_MAC		3
+static void qedf_set_data_src_addr(struct qedf_ctx *qedf, struct fc_frame *fp)
+{
+	u8 *granted_mac;
+	struct fc_frame_header *fh = fc_frame_header_get(fp);
+	u8 fc_map[3];
+	int method = 0;
+
+	/* Get granted MAC address from FIP FLOGI payload */
+	granted_mac = fr_cb(fp)->granted_mac;
+
+	/*
+	 * We set the source MAC for FCoE traffic based on the Granted MAC
+	 * address from the switch.
+	 *
+	 * If granted_mac is non-zero, we used that.
+	 * If the granted_mac is zeroed out, created the FCoE MAC based on
+	 * the sel_fcf->fc_map and the d_id fo the FLOGI frame.
+	 * If sel_fcf->fc_map is 0 then we use the default FCF-MAC plus the
+	 * d_id of the FLOGI frame.
+	 */
+	if (!is_zero_ether_addr(granted_mac)) {
+		ether_addr_copy(qedf->data_src_addr, granted_mac);
+		method = QEDF_FCOE_MAC_METHOD_GRANGED_MAC;
+	} else if (qedf->ctlr.sel_fcf->fc_map != 0) {
+		hton24(fc_map, qedf->ctlr.sel_fcf->fc_map);
+		qedf->data_src_addr[0] = fc_map[0];
+		qedf->data_src_addr[1] = fc_map[1];
+		qedf->data_src_addr[2] = fc_map[2];
+		qedf->data_src_addr[3] = fh->fh_d_id[0];
+		qedf->data_src_addr[4] = fh->fh_d_id[1];
+		qedf->data_src_addr[5] = fh->fh_d_id[2];
+		method = QEDF_FCOE_MAC_METHOD_FCF_MAP;
+	} else {
+		fc_fcoe_set_mac(qedf->data_src_addr, fh->fh_d_id);
+		method = QEDF_FCOE_MAC_METHOD_FCOE_SET_MAC;
+	}
+
+	QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
+	    "QEDF data_src_mac=%pM method=%d.\n", qedf->data_src_addr, method);
+}
+
 static void qedf_flogi_resp(struct fc_seq *seq, struct fc_frame *fp,
 	void *arg)
 {
@@ -212,6 +257,10 @@ static void qedf_flogi_resp(struct fc_seq *seq, struct fc_frame *fp,
 	/* Log stats for FLOGI reject */
 	if (fc_frame_payload_op(fp) == ELS_LS_RJT)
 		qedf->flogi_failed++;
+	else if (fc_frame_payload_op(fp) == ELS_LS_ACC) {
+		/* Set the source MAC we will use for FCoE traffic */
+		qedf_set_data_src_addr(qedf, fp);
+	}
 
 	/* Complete flogi_compl so we can proceed to sending ADISCs */
 	complete(&qedf->flogi_compl);
@@ -312,8 +361,9 @@ static void qedf_link_recovery(struct work_struct *work)
 	/* Since the link when down and up to verify which vlan we're on */
 	qedf->fipvlan_retries = qedf_fipvlan_retries;
 	rc = qedf_initiate_fipvlan_req(qedf);
+	/* If getting the VLAN fails, set the VLAN to the fallback one */
 	if (!rc)
-		return;
+		qedf_set_vlan_id(qedf, qedf_fallback_vlan);
 
 	/*
 	 * We need to wait for an FCF to be selected due to the
@@ -629,16 +679,6 @@ static int qedf_eh_device_reset(struct scsi_cmnd *sc_cmd)
 	return qedf_initiate_tmf(sc_cmd, FCP_TMF_LUN_RESET);
 }
 
-static int qedf_eh_bus_reset(struct scsi_cmnd *sc_cmd)
-{
-	QEDF_ERR(NULL, "BUS RESET Issued...\n");
-	/*
-	 * Essentially a no-op but return SUCCESS to prevent
-	 * unnecessary escalation to the host reset handler.
-	 */
-	return SUCCESS;
-}
-
 void qedf_wait_for_upload(struct qedf_ctx *qedf)
 {
 	while (1) {
@@ -716,7 +756,6 @@ static struct scsi_host_template qedf_host_template = {
 	.eh_abort_handler	= qedf_eh_abort,
 	.eh_device_reset_handler = qedf_eh_device_reset, /* lun reset */
 	.eh_target_reset_handler = qedf_eh_target_reset, /* target reset */
-	.eh_bus_reset_handler = qedf_eh_bus_reset,
 	.eh_host_reset_handler  = qedf_eh_host_reset,
 	.slave_configure	= qedf_slave_configure,
 	.dma_boundary = QED_HW_DMA_BOUNDARY,
@@ -915,6 +954,10 @@ static int qedf_xmit(struct fc_lport *lport, struct fc_frame *fp)
 	skb->mac_len = elen;
 	skb->protocol = htons(ETH_P_FCOE);
 
+	/*
+	 * Add VLAN tag to non-offload FCoE frame based on current stored VLAN
+	 * for FIP/FCoE traffic.
+	 */
 	__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), qedf->vlan_id);
 
 	/* fill up mac and fcoe headers */
@@ -927,7 +970,7 @@ static int qedf_xmit(struct fc_lport *lport, struct fc_frame *fp)
 		ether_addr_copy(eh->h_dest, qedf->ctlr.dest_addr);
 
 	/* Set the source MAC address */
-	fc_fcoe_set_mac(eh->h_source, fh->fh_s_id);
+	ether_addr_copy(eh->h_source, qedf->data_src_addr);
 
 	hp = (struct fcoe_hdr *)(eh + 1);
 	memset(hp, 0, sizeof(*hp));
@@ -1025,7 +1068,6 @@ static int qedf_offload_connection(struct qedf_ctx *qedf,
 {
 	struct qed_fcoe_params_offload conn_info;
 	u32 port_id;
-	u8 lport_src_id[3];
 	int rval;
 	uint16_t total_sqe = (fcport->sq_mem_size / sizeof(struct fcoe_wqe));
 
@@ -1054,11 +1096,7 @@ static int qedf_offload_connection(struct qedf_ctx *qedf,
 	    (dma_addr_t)(*(u64 *)(fcport->sq_pbl + 8));
 
 	/* Need to use our FCoE MAC for the offload session */
-	port_id = fc_host_port_id(qedf->lport->host);
-	lport_src_id[2] = (port_id & 0x000000FF);
-	lport_src_id[1] = (port_id & 0x0000FF00) >> 8;
-	lport_src_id[0] = (port_id & 0x00FF0000) >> 16;
-	fc_fcoe_set_mac(conn_info.src_mac, lport_src_id);
+	ether_addr_copy(conn_info.src_mac, qedf->data_src_addr);
 
 	ether_addr_copy(conn_info.dst_mac, qedf->ctlr.dest_addr);
 
@@ -1347,7 +1385,6 @@ static void qedf_fcoe_ctlr_setup(struct qedf_ctx *qedf)
 	fcoe_ctlr_init(&qedf->ctlr, FIP_ST_AUTO);
 
 	qedf->ctlr.send = qedf_fip_send;
-	qedf->ctlr.update_mac = qedf_update_src_mac;
 	qedf->ctlr.get_src_addr = qedf_get_src_mac;
 	ether_addr_copy(qedf->ctlr.ctl_src_addr, qedf->mac);
 }
@@ -2939,7 +2976,7 @@ static int __qedf_probe(struct pci_dev *pdev, int mode)
 
 	sprintf(host_buf, "qedf_%u_link",
 	    qedf->lport->host->host_no);
-	qedf->link_update_wq = create_singlethread_workqueue(host_buf);
+	qedf->link_update_wq = create_workqueue(host_buf);
 	INIT_DELAYED_WORK(&qedf->link_update, qedf_handle_link_update);
 	INIT_DELAYED_WORK(&qedf->link_recovery, qedf_link_recovery);
 
@@ -3056,9 +3093,24 @@ static int __qedf_probe(struct pci_dev *pdev, int mode)
 	QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC, "MAC address is %pM.\n",
 		   qedf->mac);
 
-	/* Set the WWNN and WWPN based on the MAC address */
-	qedf->wwnn = fcoe_wwn_from_mac(qedf->mac, 1, 0);
-	qedf->wwpn = fcoe_wwn_from_mac(qedf->mac, 2, 0);
+	/*
+	 * Set the WWNN and WWPN in the following way:
+	 *
+	 * If the info we get from qed is non-zero then use that to set the
+	 * WWPN and WWNN. Otherwise fall back to use fcoe_wwn_from_mac() based
+	 * on the MAC address.
+	 */
+	if (qedf->dev_info.wwnn != 0 && qedf->dev_info.wwpn != 0) {
+		QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
+		    "Setting WWPN and WWNN from qed dev_info.\n");
+		qedf->wwnn = qedf->dev_info.wwnn;
+		qedf->wwpn = qedf->dev_info.wwpn;
+	} else {
+		QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
+		    "Setting WWPN and WWNN using fcoe_wwn_from_mac().\n");
+		qedf->wwnn = fcoe_wwn_from_mac(qedf->mac, 1, 0);
+		qedf->wwpn = fcoe_wwn_from_mac(qedf->mac, 2, 0);
+	}
 	QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,  "WWNN=%016llx "
 		   "WWPN=%016llx.\n", qedf->wwnn, qedf->wwpn);
 
@@ -3094,7 +3146,7 @@ static int __qedf_probe(struct pci_dev *pdev, int mode)
 	/* Start LL2 processing thread */
 	snprintf(host_buf, 20, "qedf_%d_ll2", host->host_no);
 	qedf->ll2_recv_wq =
-		create_singlethread_workqueue(host_buf);
+		create_workqueue(host_buf);
 	if (!qedf->ll2_recv_wq) {
 		QEDF_ERR(&(qedf->dbg_ctx), "Failed to LL2 workqueue.\n");
 		goto err7;
@@ -3114,8 +3166,7 @@ static int __qedf_probe(struct pci_dev *pdev, int mode)
 	}
 	set_bit(QEDF_LL2_STARTED, &qedf->flags);
 
-	/* hw will be insterting vlan tag*/
-	qedf->vlan_hw_insert = 1;
+	/* Set initial FIP/FCoE VLAN to NULL */
 	qedf->vlan_id = 0;
 
 	/*
@@ -3137,7 +3188,7 @@ static int __qedf_probe(struct pci_dev *pdev, int mode)
 
 	sprintf(host_buf, "qedf_%u_timer", qedf->lport->host->host_no);
 	qedf->timer_work_queue =
-		create_singlethread_workqueue(host_buf);
+		create_workqueue(host_buf);
 	if (!qedf->timer_work_queue) {
 		QEDF_ERR(&(qedf->dbg_ctx), "Failed to start timer "
 			  "workqueue.\n");
@@ -3148,7 +3199,7 @@ static int __qedf_probe(struct pci_dev *pdev, int mode)
 	if (mode != QEDF_MODE_RECOVERY) {
 		sprintf(host_buf, "qedf_%u_dpc",
 		    qedf->lport->host->host_no);
-		qedf->dpc_wq = create_singlethread_workqueue(host_buf);
+		qedf->dpc_wq = create_workqueue(host_buf);
 	}
 
 	/*
diff --git a/drivers/scsi/qedf/qedf_version.h b/drivers/scsi/qedf/qedf_version.h
index 6fa442061c32..397b3b8ee51a 100644
--- a/drivers/scsi/qedf/qedf_version.h
+++ b/drivers/scsi/qedf/qedf_version.h
@@ -7,9 +7,9 @@
  *  this source tree.
  */
 
-#define QEDF_VERSION		"8.18.22.0"
+#define QEDF_VERSION		"8.20.5.0"
 #define QEDF_DRIVER_MAJOR_VER		8
-#define QEDF_DRIVER_MINOR_VER		18
-#define QEDF_DRIVER_REV_VER		22
+#define QEDF_DRIVER_MINOR_VER		20
+#define QEDF_DRIVER_REV_VER		5
 #define QEDF_DRIVER_ENG_VER		0
 
diff --git a/drivers/scsi/qedi/qedi.h b/drivers/scsi/qedi/qedi.h
index 91d2f51c351b..b8b22ce60ecc 100644
--- a/drivers/scsi/qedi/qedi.h
+++ b/drivers/scsi/qedi/qedi.h
@@ -54,8 +54,8 @@ struct qedi_endpoint;
 /* MAX Length for cached SGL */
 #define MAX_SGLEN_FOR_CACHESGL	((1U << 16) - 1)
 
-#define MAX_NUM_MSIX_PF         8
-#define MIN_NUM_CPUS_MSIX(x)	min((x)->msix_count, num_online_cpus())
+#define MIN_NUM_CPUS_MSIX(x)	min_t(u32, x->dev_info.num_cqs, \
+					num_online_cpus())
 
 #define QEDI_LOCAL_PORT_MIN     60000
 #define QEDI_LOCAL_PORT_MAX     61024
@@ -301,7 +301,6 @@ struct qedi_ctx {
 	u16 bdq_prod_idx;
 	u16 rq_num_entries;
 
-	u32 msix_count;
 	u32 max_sqes;
 	u8 num_queues;
 	u32 max_active_conns;
diff --git a/drivers/scsi/qedi/qedi_iscsi.c b/drivers/scsi/qedi/qedi_iscsi.c
index 37da9a8b43b1..a02b34ea5cab 100644
--- a/drivers/scsi/qedi/qedi_iscsi.c
+++ b/drivers/scsi/qedi/qedi_iscsi.c
@@ -534,7 +534,7 @@ static int qedi_iscsi_offload_conn(struct qedi_endpoint *qedi_ep)
 	SET_FIELD(conn_info->tcp_flags, TCP_OFFLOAD_PARAMS_DA_CNT_EN, 1);
 	SET_FIELD(conn_info->tcp_flags, TCP_OFFLOAD_PARAMS_KA_EN, 1);
 
-	conn_info->default_cq = (qedi_ep->fw_cid % 8);
+	conn_info->default_cq = (qedi_ep->fw_cid % qedi->num_queues);
 
 	conn_info->ka_max_probe_cnt = DEF_KA_MAX_PROBE_COUNT;
 	conn_info->dup_ack_theshold = 3;
diff --git a/drivers/scsi/qedi/qedi_main.c b/drivers/scsi/qedi/qedi_main.c
index 2c3783684815..cccc34adc0e0 100644
--- a/drivers/scsi/qedi/qedi_main.c
+++ b/drivers/scsi/qedi/qedi_main.c
@@ -794,13 +794,14 @@ static int qedi_set_iscsi_pf_param(struct qedi_ctx *qedi)
 	u32 log_page_size;
 	int rval = 0;
 
-	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC, "Min number of MSIX %d\n",
-		  MIN_NUM_CPUS_MSIX(qedi));
 
 	num_sq_pages = (MAX_OUSTANDING_TASKS_PER_CON * 8) / PAGE_SIZE;
 
 	qedi->num_queues = MIN_NUM_CPUS_MSIX(qedi);
 
+	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
+		  "Number of CQ count is %d\n", qedi->num_queues);
+
 	memset(&qedi->pf_params.iscsi_pf_params, 0,
 	       sizeof(qedi->pf_params.iscsi_pf_params));
 
@@ -1575,7 +1576,7 @@ struct qedi_cmd *qedi_get_cmd_from_tid(struct qedi_ctx *qedi, u32 tid)
 {
 	struct qedi_cmd *cmd = NULL;
 
-	if (tid > MAX_ISCSI_TASK_ENTRIES)
+	if (tid >= MAX_ISCSI_TASK_ENTRIES)
 		return NULL;
 
 	cmd = qedi->itt_map[tid].p_cmd;
@@ -2179,9 +2180,12 @@ static int __qedi_probe(struct pci_dev *pdev, int mode)
 		goto free_host;
 	}
 
-	qedi->msix_count = MAX_NUM_MSIX_PF;
 	atomic_set(&qedi->link_state, QEDI_LINK_DOWN);
 
+	rc = qedi_ops->fill_dev_info(qedi->cdev, &qedi->dev_info);
+	if (rc)
+		goto free_host;
+
 	if (mode != QEDI_MODE_RECOVERY) {
 		rc = qedi_set_iscsi_pf_param(qedi);
 		if (rc) {
diff --git a/drivers/scsi/qla2xxx/qla_attr.c b/drivers/scsi/qla2xxx/qla_attr.c
index 08a1feb3a195..9ce28c4f9812 100644
--- a/drivers/scsi/qla2xxx/qla_attr.c
+++ b/drivers/scsi/qla2xxx/qla_attr.c
@@ -318,6 +318,8 @@ qla2x00_sysfs_write_optrom_ctl(struct file *filp, struct kobject *kobj,
 		return -EINVAL;
 	if (start > ha->optrom_size)
 		return -EINVAL;
+	if (size > ha->optrom_size - start)
+		size = ha->optrom_size - start;
 
 	mutex_lock(&ha->optrom_mutex);
 	switch (val) {
@@ -343,8 +345,7 @@ qla2x00_sysfs_write_optrom_ctl(struct file *filp, struct kobject *kobj,
 		}
 
 		ha->optrom_region_start = start;
-		ha->optrom_region_size = start + size > ha->optrom_size ?
-		    ha->optrom_size - start : size;
+		ha->optrom_region_size = start + size;
 
 		ha->optrom_state = QLA_SREADING;
 		ha->optrom_buffer = vmalloc(ha->optrom_region_size);
@@ -417,8 +418,7 @@ qla2x00_sysfs_write_optrom_ctl(struct file *filp, struct kobject *kobj,
 		}
 
 		ha->optrom_region_start = start;
-		ha->optrom_region_size = start + size > ha->optrom_size ?
-		    ha->optrom_size - start : size;
+		ha->optrom_region_size = start + size;
 
 		ha->optrom_state = QLA_SWRITING;
 		ha->optrom_buffer = vmalloc(ha->optrom_region_size);
@@ -565,47 +565,17 @@ qla2x00_sysfs_read_sfp(struct file *filp, struct kobject *kobj,
 {
 	struct scsi_qla_host *vha = shost_priv(dev_to_shost(container_of(kobj,
 	    struct device, kobj)));
-	struct qla_hw_data *ha = vha->hw;
-	uint16_t iter, addr, offset;
 	int rval;
 
-	if (!capable(CAP_SYS_ADMIN) || off != 0 || count != SFP_DEV_SIZE * 2)
+	if (!capable(CAP_SYS_ADMIN) || off != 0 || count < SFP_DEV_SIZE)
 		return 0;
 
-	if (ha->sfp_data)
-		goto do_read;
-
-	ha->sfp_data = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL,
-	    &ha->sfp_data_dma);
-	if (!ha->sfp_data) {
-		ql_log(ql_log_warn, vha, 0x706c,
-		    "Unable to allocate memory for SFP read-data.\n");
+	if (qla2x00_reset_active(vha))
 		return 0;
-	}
-
-do_read:
-	memset(ha->sfp_data, 0, SFP_BLOCK_SIZE);
-	addr = 0xa0;
-	for (iter = 0, offset = 0; iter < (SFP_DEV_SIZE * 2) / SFP_BLOCK_SIZE;
-	    iter++, offset += SFP_BLOCK_SIZE) {
-		if (iter == 4) {
-			/* Skip to next device address. */
-			addr = 0xa2;
-			offset = 0;
-		}
-
-		rval = qla2x00_read_sfp(vha, ha->sfp_data_dma, ha->sfp_data,
-		    addr, offset, SFP_BLOCK_SIZE, BIT_1);
-		if (rval != QLA_SUCCESS) {
-			ql_log(ql_log_warn, vha, 0x706d,
-			    "Unable to read SFP data (%x/%x/%x).\n", rval,
-			    addr, offset);
 
-			return -EIO;
-		}
-		memcpy(buf, ha->sfp_data, SFP_BLOCK_SIZE);
-		buf += SFP_BLOCK_SIZE;
-	}
+	rval = qla2x00_read_sfp_dev(vha, buf, count);
+	if (rval)
+		return -EIO;
 
 	return count;
 }
@@ -615,7 +585,7 @@ static struct bin_attribute sysfs_sfp_attr = {
 		.name = "sfp",
 		.mode = S_IRUSR | S_IWUSR,
 	},
-	.size = SFP_DEV_SIZE * 2,
+	.size = SFP_DEV_SIZE,
 	.read = qla2x00_sysfs_read_sfp,
 };
 
@@ -1511,6 +1481,38 @@ qla2x00_pep_version_show(struct device *dev, struct device_attribute *attr,
 	    ha->pep_version[0], ha->pep_version[1], ha->pep_version[2]);
 }
 
+static ssize_t
+qla2x00_min_link_speed_show(struct device *dev, struct device_attribute *attr,
+    char *buf)
+{
+	scsi_qla_host_t *vha = shost_priv(class_to_shost(dev));
+	struct qla_hw_data *ha = vha->hw;
+
+	if (!IS_QLA27XX(ha))
+		return scnprintf(buf, PAGE_SIZE, "\n");
+
+	return scnprintf(buf, PAGE_SIZE, "%s\n",
+	    ha->min_link_speed == 5 ? "32Gps" :
+	    ha->min_link_speed == 4 ? "16Gps" :
+	    ha->min_link_speed == 3 ? "8Gps" :
+	    ha->min_link_speed == 2 ? "4Gps" :
+	    ha->min_link_speed != 0 ? "unknown" : "");
+}
+
+static ssize_t
+qla2x00_max_speed_sup_show(struct device *dev, struct device_attribute *attr,
+    char *buf)
+{
+	scsi_qla_host_t *vha = shost_priv(class_to_shost(dev));
+	struct qla_hw_data *ha = vha->hw;
+
+	if (!IS_QLA27XX(ha))
+		return scnprintf(buf, PAGE_SIZE, "\n");
+
+	return scnprintf(buf, PAGE_SIZE, "%s\n",
+	    ha->max_speed_sup ? "32Gps" : "16Gps");
+}
+
 static DEVICE_ATTR(driver_version, S_IRUGO, qla2x00_drvr_version_show, NULL);
 static DEVICE_ATTR(fw_version, S_IRUGO, qla2x00_fw_version_show, NULL);
 static DEVICE_ATTR(serial_num, S_IRUGO, qla2x00_serial_num_show, NULL);
@@ -1556,6 +1558,8 @@ static DEVICE_ATTR(allow_cna_fw_dump, S_IRUGO | S_IWUSR,
 		   qla2x00_allow_cna_fw_dump_show,
 		   qla2x00_allow_cna_fw_dump_store);
 static DEVICE_ATTR(pep_version, S_IRUGO, qla2x00_pep_version_show, NULL);
+static DEVICE_ATTR(min_link_speed, S_IRUGO, qla2x00_min_link_speed_show, NULL);
+static DEVICE_ATTR(max_speed_sup, S_IRUGO, qla2x00_max_speed_sup_show, NULL);
 
 struct device_attribute *qla2x00_host_attrs[] = {
 	&dev_attr_driver_version,
@@ -1590,6 +1594,8 @@ struct device_attribute *qla2x00_host_attrs[] = {
 	&dev_attr_fw_dump_size,
 	&dev_attr_allow_cna_fw_dump,
 	&dev_attr_pep_version,
+	&dev_attr_min_link_speed,
+	&dev_attr_max_speed_sup,
 	NULL,
 };
 
diff --git a/drivers/scsi/qla2xxx/qla_dbg.c b/drivers/scsi/qla2xxx/qla_dbg.c
index 26751d34bcf2..3e9dc54b89a3 100644
--- a/drivers/scsi/qla2xxx/qla_dbg.c
+++ b/drivers/scsi/qla2xxx/qla_dbg.c
@@ -14,7 +14,7 @@
  * | Module Init and Probe        |       0x0193       | 0x0146         |
  * |                              |                    | 0x015b-0x0160	|
  * |                              |                    | 0x016e		|
- * | Mailbox commands             |       0x1199       | 0x1193		|
+ * | Mailbox commands             |       0x1205       | 0x11a2-0x11ff	|
  * | Device Discovery             |       0x2134       | 0x210e-0x2116  |
  * |				  | 		       | 0x211a         |
  * |                              |                    | 0x211c-0x2128  |
@@ -41,7 +41,7 @@
  * |                              |                    | 0x70ad-0x70ae  |
  * |                              |                    | 0x70d0-0x70d6	|
  * |                              |                    | 0x70d7-0x70db  |
- * | Task Management              |       0x8042       | 0x8000,0x800b  |
+ * | Task Management              |       0x8042       | 0x8000         |
  * |                              |                    | 0x8019         |
  * |                              |                    | 0x8025,0x8026  |
  * |                              |                    | 0x8031,0x8032  |
@@ -2520,8 +2520,6 @@ qla83xx_fw_dump_failed:
 static inline int
 ql_mask_match(uint32_t level)
 {
-	if (ql2xextended_error_logging == 1)
-		ql2xextended_error_logging = QL_DBG_DEFAULT1_MASK;
 	return (level & ql2xextended_error_logging) == level;
 }
 
@@ -2738,9 +2736,9 @@ ql_dump_regs(uint32_t level, scsi_qla_host_t *vha, int32_t id)
 		mbx_reg = MAILBOX_REG(ha, reg, 0);
 
 	ql_dbg(level, vha, id, "Mailbox registers:\n");
-	for (i = 0; i < 6; i++)
+	for (i = 0; i < 6; i++, mbx_reg++)
 		ql_dbg(level, vha, id,
-		    "mbox[%d] 0x%04x\n", i, RD_REG_WORD(mbx_reg++));
+		    "mbox[%d] 0x%04x\n", i, RD_REG_WORD(mbx_reg));
 }
 
 
diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h
index 0730b10b4280..486c075998f6 100644
--- a/drivers/scsi/qla2xxx/qla_def.h
+++ b/drivers/scsi/qla2xxx/qla_def.h
@@ -427,6 +427,7 @@ struct srb_iocb {
 			enum nvmefc_fcp_datadir dir;
 			uint32_t dl;
 			uint32_t timeout_sec;
+			struct	list_head   entry;
 		} nvme;
 	} u;
 
@@ -470,7 +471,7 @@ typedef struct srb {
 	uint8_t cmd_type;
 	uint8_t pad[3];
 	atomic_t ref_count;
-	wait_queue_head_t nvme_ls_waitQ;
+	wait_queue_head_t nvme_ls_waitq;
 	struct fc_port *fcport;
 	struct scsi_qla_host *vha;
 	uint32_t handle;
@@ -901,6 +902,7 @@ struct mbx_cmd_32 {
 #define MBA_SHUTDOWN_REQUESTED	0x8062	/* Shutdown Requested */
 #define MBA_TEMPERATURE_ALERT	0x8070	/* Temperature Alert */
 #define MBA_DPORT_DIAGNOSTICS	0x8080	/* D-port Diagnostics */
+#define MBA_TRANS_INSERT	0x8130	/* Transceiver Insertion */
 #define MBA_FW_INIT_FAILURE	0x8401	/* Firmware initialization failure */
 #define MBA_MIRROR_LUN_CHANGE	0x8402	/* Mirror LUN State Change
 					   Notification */
@@ -977,6 +979,7 @@ struct mbx_cmd_32 {
 #define MBC_ABORT_TARGET		0x17	/* Abort target (ID). */
 #define MBC_RESET			0x18	/* Reset. */
 #define MBC_GET_ADAPTER_LOOP_ID		0x20	/* Get loop id of ISP2200. */
+#define MBC_GET_SET_ZIO_THRESHOLD	0x21	/* Get/SET ZIO THRESHOLD. */
 #define MBC_GET_RETRY_COUNT		0x22	/* Get f/w retry cnt/delay. */
 #define MBC_DISABLE_VI			0x24	/* Disable VI operation. */
 #define MBC_ENABLE_VI			0x25	/* Enable VI operation. */
@@ -2301,8 +2304,7 @@ typedef struct fc_port {
 	unsigned int login_succ:1;
 
 	struct work_struct nvme_del_work;
-	atomic_t nvme_ref_count;
-	wait_queue_head_t nvme_waitQ;
+	struct completion nvme_del_done;
 	uint32_t nvme_prli_service_param;
 #define NVME_PRLI_SP_CONF       BIT_7
 #define NVME_PRLI_SP_INITIATOR  BIT_5
@@ -3338,6 +3340,7 @@ struct qla_qpair {
 	struct work_struct q_work;
 	struct list_head qp_list_elem; /* vha->qp_list */
 	struct list_head hints_list;
+	struct list_head nvme_done_list;
 	uint16_t cpuid;
 	struct qla_tgt_counters tgt_counters;
 };
@@ -3463,8 +3466,15 @@ struct qla_hw_data {
 		uint32_t	n2n_ae:1;
 		uint32_t	fw_started:1;
 		uint32_t	fw_init_done:1;
+
+		uint32_t	detected_lr_sfp:1;
+		uint32_t	using_lr_setting:1;
 	} flags;
 
+	uint16_t long_range_distance;	/* 32G & above */
+#define LR_DISTANCE_5K  1
+#define LR_DISTANCE_10K 0
+
 	/* This spinlock is used to protect "io transactions", you must
 	* acquire it before doing any IO to the card, eg with RD_REG*() and
 	* WRT_REG*() for the duration of your entire commandtransaction.
@@ -3712,7 +3722,7 @@ struct qla_hw_data {
 	struct sns_cmd_pkt	*sns_cmd;
 	dma_addr_t		sns_cmd_dma;
 
-#define SFP_DEV_SIZE    256
+#define SFP_DEV_SIZE    512
 #define SFP_BLOCK_SIZE  64
 	void		*sfp_data;
 	dma_addr_t	sfp_data_dma;
@@ -4017,8 +4027,20 @@ struct qla_hw_data {
 
 	struct qlt_hw_data tgt;
 	int	allow_cna_fw_dump;
+	uint32_t fw_ability_mask;
+	uint16_t min_link_speed;
+	uint16_t max_speed_sup;
+
+	atomic_t        nvme_active_aen_cnt;
+	uint16_t        nvme_last_rptd_aen;             /* Last recorded aen count */
 };
 
+#define FW_ABILITY_MAX_SPEED_MASK	0xFUL
+#define FW_ABILITY_MAX_SPEED_16G	0x0
+#define FW_ABILITY_MAX_SPEED_32G	0x1
+#define FW_ABILITY_MAX_SPEED(ha)	\
+	(ha->fw_ability_mask & FW_ABILITY_MAX_SPEED_MASK)
+
 /*
  * Qlogic scsi host structure
  */
@@ -4089,6 +4111,8 @@ typedef struct scsi_qla_host {
 #define FX00_CRITEMP_RECOVERY	25
 #define FX00_HOST_INFO_RESEND	26
 #define QPAIR_ONLINE_CHECK_NEEDED	27
+#define SET_ZIO_THRESHOLD_NEEDED	28
+#define DETECT_SFP_CHANGE	29
 
 	unsigned long	pci_flags;
 #define PFLG_DISCONNECTED	0	/* PCI device removed */
@@ -4129,8 +4153,7 @@ typedef struct scsi_qla_host {
 	uint8_t		fabric_node_name[WWN_SIZE];
 
 	struct		nvme_fc_local_port *nvme_local_port;
-	atomic_t	nvme_ref_count;
-	wait_queue_head_t nvme_waitQ;
+	struct completion nvme_del_done;
 	struct list_head nvme_rport_list;
 	atomic_t 	nvme_active_aen_cnt;
 	uint16_t	nvme_last_rptd_aen;
@@ -4199,6 +4222,7 @@ typedef struct scsi_qla_host {
 	int fcport_count;
 	wait_queue_head_t fcport_waitQ;
 	wait_queue_head_t vref_waitq;
+	uint8_t min_link_speed_feat;
 } scsi_qla_host_t;
 
 struct qla27xx_image_status {
@@ -4373,6 +4397,88 @@ enum nexus_wait_type {
 	WAIT_LUN,
 };
 
+/* Refer to SNIA SFF 8247 */
+struct sff_8247_a0 {
+	u8 txid;	/* transceiver id */
+	u8 ext_txid;
+	u8 connector;
+	/* compliance code */
+	u8 eth_infi_cc3;	/* ethernet, inifiband */
+	u8 sonet_cc4[2];
+	u8 eth_cc6;
+	/* link length */
+#define FC_LL_VL BIT_7	/* very long */
+#define FC_LL_S  BIT_6	/* Short */
+#define FC_LL_I  BIT_5	/* Intermidiate*/
+#define FC_LL_L  BIT_4	/* Long */
+#define FC_LL_M  BIT_3	/* Medium */
+#define FC_LL_SA BIT_2	/* ShortWave laser */
+#define FC_LL_LC BIT_1	/* LongWave laser */
+#define FC_LL_EL BIT_0	/* Electrical inter enclosure */
+	u8 fc_ll_cc7;
+	/* FC technology */
+#define FC_TEC_EL BIT_7	/* Electrical inter enclosure */
+#define FC_TEC_SN BIT_6	/* short wave w/o OFC */
+#define FC_TEC_SL BIT_5	/* short wave with OFC */
+#define FC_TEC_LL BIT_4	/* Longwave Laser */
+#define FC_TEC_ACT BIT_3	/* Active cable */
+#define FC_TEC_PAS BIT_2	/* Passive cable */
+	u8 fc_tec_cc8;
+	/* Transmission Media */
+#define FC_MED_TW BIT_7	/* Twin Ax */
+#define FC_MED_TP BIT_6	/* Twited Pair */
+#define FC_MED_MI BIT_5	/* Min Coax */
+#define FC_MED_TV BIT_4	/* Video Coax */
+#define FC_MED_M6 BIT_3	/* Multimode, 62.5um */
+#define FC_MED_M5 BIT_2	/* Multimode, 50um */
+#define FC_MED_SM BIT_0	/* Single Mode */
+	u8 fc_med_cc9;
+	/* speed FC_SP_12: 12*100M = 1200 MB/s */
+#define FC_SP_12 BIT_7
+#define FC_SP_8  BIT_6
+#define FC_SP_16 BIT_5
+#define FC_SP_4  BIT_4
+#define FC_SP_32 BIT_3
+#define FC_SP_2  BIT_2
+#define FC_SP_1  BIT_0
+	u8 fc_sp_cc10;
+	u8 encode;
+	u8 bitrate;
+	u8 rate_id;
+	u8 length_km;		/* offset 14/eh */
+	u8 length_100m;
+	u8 length_50um_10m;
+	u8 length_62um_10m;
+	u8 length_om4_10m;
+	u8 length_om3_10m;
+#define SFF_VEN_NAME_LEN 16
+	u8 vendor_name[SFF_VEN_NAME_LEN];	/* offset 20/14h */
+	u8 tx_compat;
+	u8 vendor_oui[3];
+#define SFF_PART_NAME_LEN 16
+	u8 vendor_pn[SFF_PART_NAME_LEN];	/* part number */
+	u8 vendor_rev[4];
+	u8 wavelength[2];
+	u8 resv;
+	u8 cc_base;
+	u8 options[2];	/* offset 64 */
+	u8 br_max;
+	u8 br_min;
+	u8 vendor_sn[16];
+	u8 date_code[8];
+	u8 diag;
+	u8 enh_options;
+	u8 sff_revision;
+	u8 cc_ext;
+	u8 vendor_specific[32];
+	u8 resv2[128];
+};
+
+#define AUTO_DETECT_SFP_SUPPORT(_vha)\
+	(ql2xautodetectsfp && !_vha->vp_idx &&		\
+	(IS_QLA25XX(_vha->hw) || IS_QLA81XX(_vha->hw) ||\
+	IS_QLA83XX(_vha->hw) || IS_QLA27XX(_vha->hw)))
+
 #define USER_CTRL_IRQ(_ha) (ql2xuctrlirq && QLA_TGT_MODE_ENABLED() && \
 	(IS_QLA27XX(_ha) || IS_QLA83XX(_ha)))
 
diff --git a/drivers/scsi/qla2xxx/qla_fw.h b/drivers/scsi/qla2xxx/qla_fw.h
index b9c9886e8b1d..bec641aae7b3 100644
--- a/drivers/scsi/qla2xxx/qla_fw.h
+++ b/drivers/scsi/qla2xxx/qla_fw.h
@@ -1699,6 +1699,15 @@ struct access_chip_rsp_84xx {
 #define FAC_OPT_CMD_UNLOCK_SEMAPHORE	0x04
 #define FAC_OPT_CMD_GET_SECTOR_SIZE	0x05
 
+/* enhanced features bit definitions */
+#define NEF_LR_DIST_ENABLE	BIT_0
+
+/* LR Distance bit positions */
+#define LR_DIST_NV_POS		2
+#define LR_DIST_FW_POS		12
+#define LR_DIST_FW_SHIFT	(LR_DIST_FW_POS - LR_DIST_NV_POS)
+#define LR_DIST_FW_FIELD(x)	((x) << LR_DIST_FW_SHIFT & 0xf000)
+
 struct nvram_81xx {
 	/* NVRAM header. */
 	uint8_t id[4];
@@ -1745,7 +1754,9 @@ struct nvram_81xx {
 	uint16_t reserved_6_3[14];
 
 	/* Offset 192. */
-	uint16_t reserved_7[32];
+	uint8_t min_link_speed;
+	uint8_t reserved_7_0;
+	uint16_t reserved_7[31];
 
 	/*
 	 * BIT 0  = Enable spinup delay
@@ -1839,16 +1850,13 @@ struct nvram_81xx {
 	uint8_t reserved_21[16];
 	uint16_t reserved_22[3];
 
-	/*
-	 * BIT 0 = Extended BB credits for LR
-	 * BIT 1 = Virtual Fabric Enable
-	 * BIT 2 = Enhanced Features Unused
-	 * BIT 3-7 = Enhanced Features Reserved
+	/* Offset 406 (0x196) Enhanced Features
+	 * BIT 0    = Extended BB credits for LR
+	 * BIT 1    = Virtual Fabric Enable
+	 * BIT 2-5  = Distance Support if BIT 0 is on
+	 * BIT 6-15 = Unused
 	 */
-	/* Enhanced Features */
-	uint8_t enhanced_features;
-
-	uint8_t reserved_23;
+	uint16_t enhanced_features;
 	uint16_t reserved_24[4];
 
 	/* Offset 416. */
diff --git a/drivers/scsi/qla2xxx/qla_gbl.h b/drivers/scsi/qla2xxx/qla_gbl.h
index cadb6e3baacc..f852ca60c49f 100644
--- a/drivers/scsi/qla2xxx/qla_gbl.h
+++ b/drivers/scsi/qla2xxx/qla_gbl.h
@@ -10,17 +10,6 @@
 #include <linux/interrupt.h>
 
 /*
- * Global functions prototype in qla_nvme.c source file.
- */
-extern void qla_nvme_register_hba(scsi_qla_host_t *);
-extern int  qla_nvme_register_remote(scsi_qla_host_t *, fc_port_t *);
-extern void qla_nvme_delete(scsi_qla_host_t *);
-extern void qla_nvme_abort(struct qla_hw_data *, srb_t *sp);
-extern void qla24xx_nvme_ls4_iocb(scsi_qla_host_t *, struct pt_ls4_request *,
-    struct req_que *);
-extern void qla24xx_async_gffid_sp_done(void *, int);
-
-/*
  * Global Function Prototypes in qla_init.c source file.
  */
 extern int qla2x00_initialize_adapter(scsi_qla_host_t *);
@@ -116,13 +105,15 @@ int qla24xx_async_notify_ack(scsi_qla_host_t *, fc_port_t *,
 int qla24xx_post_newsess_work(struct scsi_qla_host *, port_id_t *, u8 *,
     void *);
 int qla24xx_fcport_handle_login(struct scsi_qla_host *, fc_port_t *);
-
+int qla24xx_detect_sfp(scsi_qla_host_t *vha);
+int qla24xx_post_gpdb_work(struct scsi_qla_host *, fc_port_t *, u8);
 /*
  * Global Data in qla_os.c source file.
  */
 extern char qla2x00_version_str[];
 
 extern struct kmem_cache *srb_cachep;
+extern struct kmem_cache *qla_tgt_plogi_cachep;
 
 extern int ql2xlogintimeout;
 extern int qlport_down_retry;
@@ -153,6 +144,7 @@ extern int ql2xfwholdabts;
 extern int ql2xmvasynctoatio;
 extern int ql2xuctrlirq;
 extern int ql2xnvmeenable;
+extern int ql2xautodetectsfp;
 
 extern int qla2x00_loop_reset(scsi_qla_host_t *);
 extern void qla2x00_abort_all_cmds(scsi_qla_host_t *, int);
@@ -495,6 +487,9 @@ int qla24xx_gidlist_wait(struct scsi_qla_host *, void *, dma_addr_t,
 int __qla24xx_parse_gpdb(struct scsi_qla_host *, fc_port_t *,
 	struct port_database_24xx *);
 
+extern int qla27xx_get_zio_threshold(scsi_qla_host_t *, uint16_t *);
+extern int qla27xx_set_zio_threshold(scsi_qla_host_t *, uint16_t);
+
 /*
  * Global Function Prototypes in qla_isr.c source file.
  */
@@ -804,6 +799,7 @@ extern char *qdev_state(uint32_t);
 extern void qla82xx_clear_pending_mbx(scsi_qla_host_t *);
 extern int qla82xx_read_temperature(scsi_qla_host_t *);
 extern int qla8044_read_temperature(scsi_qla_host_t *);
+extern int qla2x00_read_sfp_dev(struct scsi_qla_host *, char *, int);
 
 /* BSG related functions */
 extern int qla24xx_bsg_request(struct bsg_job *);
@@ -873,4 +869,6 @@ void qlt_update_host_map(struct scsi_qla_host *, port_id_t);
 void qlt_remove_target_resources(struct qla_hw_data *);
 void qlt_clr_qp_table(struct scsi_qla_host *vha);
 
+void qla_nvme_cmpl_io(struct srb_iocb *);
+
 #endif /* _QLA_GBL_H */
diff --git a/drivers/scsi/qla2xxx/qla_gs.c b/drivers/scsi/qla2xxx/qla_gs.c
index b323a7c71eda..bc3db6abc9a0 100644
--- a/drivers/scsi/qla2xxx/qla_gs.c
+++ b/drivers/scsi/qla2xxx/qla_gs.c
@@ -2816,13 +2816,19 @@ void qla24xx_handle_gidpn_event(scsi_qla_host_t *vha, struct event_arg *ea)
 				case MODE_INITIATOR:
 				case MODE_DUAL:
 				default:
+					ql_dbg(ql_dbg_disc, vha, 0x201f,
+					    "%s %d %8phC post %s\n", __func__,
+					    __LINE__, fcport->port_name,
+					    (atomic_read(&fcport->state) ==
+					    FCS_ONLINE) ? "gpdb" : "gnl");
+
 					if (atomic_read(&fcport->state) ==
 					    FCS_ONLINE)
-						break;
-					ql_dbg(ql_dbg_disc, vha, 0x201f,
-					    "%s %d %8phC post gnl\n",
-					    __func__, __LINE__, fcport->port_name);
-					qla24xx_post_gnl_work(vha, fcport);
+						qla24xx_post_gpdb_work(vha,
+						    fcport, PDO_FORCE_ADISC);
+					else
+						qla24xx_post_gnl_work(vha,
+						    fcport);
 					break;
 				}
 			} else { /* fcport->d_id.b24 != ea->id.b24 */
@@ -3080,7 +3086,7 @@ int qla24xx_async_gpsc(scsi_qla_host_t *vha, fc_port_t *fcport)
 		GPSC_RSP_SIZE);
 
 	/* GPSC req */
-	memcpy(ct_req->req.gpsc.port_name, fcport->port_name,
+	memcpy(ct_req->req.gpsc.port_name, fcport->fabric_port_name,
 		WWN_SIZE);
 
 	sp->u.iocb_cmd.u.ctarg.req = fcport->ct_desc.ct_sns;
diff --git a/drivers/scsi/qla2xxx/qla_init.c b/drivers/scsi/qla2xxx/qla_init.c
index 072ad1aa5505..b5b48ddca962 100644
--- a/drivers/scsi/qla2xxx/qla_init.c
+++ b/drivers/scsi/qla2xxx/qla_init.c
@@ -36,7 +36,6 @@ static int qla2x00_restart_isp(scsi_qla_host_t *);
 static struct qla_chip_state_84xx *qla84xx_get_chip(struct scsi_qla_host *);
 static int qla84xx_init_chip(scsi_qla_host_t *);
 static int qla25xx_init_queues(struct qla_hw_data *);
-static int qla24xx_post_gpdb_work(struct scsi_qla_host *, fc_port_t *, u8);
 static int qla24xx_post_prli_work(struct scsi_qla_host*, fc_port_t *);
 static void qla24xx_handle_plogi_done_event(struct scsi_qla_host *,
     struct event_arg *);
@@ -774,8 +773,7 @@ done_free_sp:
 	return rval;
 }
 
-static int qla24xx_post_gpdb_work(struct scsi_qla_host *vha, fc_port_t *fcport,
-    u8 opt)
+int qla24xx_post_gpdb_work(struct scsi_qla_host *vha, fc_port_t *fcport, u8 opt)
 {
 	struct qla_work_evt *e;
 
@@ -808,6 +806,12 @@ int qla24xx_async_gpdb(struct scsi_qla_host *vha, fc_port_t *fcport, u8 opt)
 	if (!sp)
 		goto done;
 
+	sp->type = SRB_MB_IOCB;
+	sp->name = "gpdb";
+	sp->gen1 = fcport->rscn_gen;
+	sp->gen2 = fcport->login_gen;
+	qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
+
 	pd = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &pd_dma);
 	if (pd == NULL) {
 		ql_log(ql_log_warn, vha, 0xd043,
@@ -816,12 +820,6 @@ int qla24xx_async_gpdb(struct scsi_qla_host *vha, fc_port_t *fcport, u8 opt)
 	}
 	memset(pd, 0, max(PORT_DATABASE_SIZE, PORT_DATABASE_24XX_SIZE));
 
-	sp->type = SRB_MB_IOCB;
-	sp->name = "gpdb";
-	sp->gen1 = fcport->rscn_gen;
-	sp->gen2 = fcport->login_gen;
-	qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
-
 	mb = sp->u.iocb_cmd.u.mbx.out_mb;
 	mb[0] = MBC_GET_PORT_DATABASE;
 	mb[1] = fcport->loop_id;
@@ -1466,6 +1464,7 @@ qla24xx_handle_plogi_done_event(struct scsi_qla_host *vha, struct event_arg *ea)
 				__func__, __LINE__, ea->fcport->port_name);
 			ea->fcport->chip_reset = vha->hw->base_qpair->chip_reset;
 			ea->fcport->logout_on_delete = 1;
+			ea->fcport->send_els_logo = 0;
 			qla24xx_post_gpdb_work(vha, ea->fcport, 0);
 		}
 		break;
@@ -2823,6 +2822,147 @@ qla2x00_alloc_outstanding_cmds(struct qla_hw_data *ha, struct req_que *req)
 	return QLA_SUCCESS;
 }
 
+#define PRINT_FIELD(_field, _flag, _str) {		\
+	if (a0->_field & _flag) {\
+		if (p) {\
+			strcat(ptr, "|");\
+			ptr++;\
+			leftover--;\
+		} \
+		len = snprintf(ptr, leftover, "%s", _str);	\
+		p = 1;\
+		leftover -= len;\
+		ptr += len; \
+	} \
+}
+
+static void qla2xxx_print_sfp_info(struct scsi_qla_host *vha)
+{
+#define STR_LEN 64
+	struct sff_8247_a0 *a0 = (struct sff_8247_a0 *)vha->hw->sfp_data;
+	u8 str[STR_LEN], *ptr, p;
+	int leftover, len;
+
+	memset(str, 0, STR_LEN);
+	snprintf(str, SFF_VEN_NAME_LEN+1, a0->vendor_name);
+	ql_dbg(ql_dbg_init, vha, 0x015a,
+	    "SFP MFG Name: %s\n", str);
+
+	memset(str, 0, STR_LEN);
+	snprintf(str, SFF_PART_NAME_LEN+1, a0->vendor_pn);
+	ql_dbg(ql_dbg_init, vha, 0x015c,
+	    "SFP Part Name: %s\n", str);
+
+	/* media */
+	memset(str, 0, STR_LEN);
+	ptr = str;
+	leftover = STR_LEN;
+	p = len = 0;
+	PRINT_FIELD(fc_med_cc9, FC_MED_TW, "Twin AX");
+	PRINT_FIELD(fc_med_cc9, FC_MED_TP, "Twisted Pair");
+	PRINT_FIELD(fc_med_cc9, FC_MED_MI, "Min Coax");
+	PRINT_FIELD(fc_med_cc9, FC_MED_TV, "Video Coax");
+	PRINT_FIELD(fc_med_cc9, FC_MED_M6, "MultiMode 62.5um");
+	PRINT_FIELD(fc_med_cc9, FC_MED_M5, "MultiMode 50um");
+	PRINT_FIELD(fc_med_cc9, FC_MED_SM, "SingleMode");
+	ql_dbg(ql_dbg_init, vha, 0x0160,
+	    "SFP Media: %s\n", str);
+
+	/* link length */
+	memset(str, 0, STR_LEN);
+	ptr = str;
+	leftover = STR_LEN;
+	p = len = 0;
+	PRINT_FIELD(fc_ll_cc7, FC_LL_VL, "Very Long");
+	PRINT_FIELD(fc_ll_cc7, FC_LL_S, "Short");
+	PRINT_FIELD(fc_ll_cc7, FC_LL_I, "Intermediate");
+	PRINT_FIELD(fc_ll_cc7, FC_LL_L, "Long");
+	PRINT_FIELD(fc_ll_cc7, FC_LL_M, "Medium");
+	ql_dbg(ql_dbg_init, vha, 0x0196,
+	    "SFP Link Length: %s\n", str);
+
+	memset(str, 0, STR_LEN);
+	ptr = str;
+	leftover = STR_LEN;
+	p = len = 0;
+	PRINT_FIELD(fc_ll_cc7, FC_LL_SA, "Short Wave (SA)");
+	PRINT_FIELD(fc_ll_cc7, FC_LL_LC, "Long Wave(LC)");
+	PRINT_FIELD(fc_tec_cc8, FC_TEC_SN, "Short Wave (SN)");
+	PRINT_FIELD(fc_tec_cc8, FC_TEC_SL, "Short Wave (SL)");
+	PRINT_FIELD(fc_tec_cc8, FC_TEC_LL, "Long Wave (LL)");
+	ql_dbg(ql_dbg_init, vha, 0x016e,
+	    "SFP FC Link Tech: %s\n", str);
+
+	if (a0->length_km)
+		ql_dbg(ql_dbg_init, vha, 0x016f,
+		    "SFP Distant: %d km\n", a0->length_km);
+	if (a0->length_100m)
+		ql_dbg(ql_dbg_init, vha, 0x0170,
+		    "SFP Distant: %d m\n", a0->length_100m*100);
+	if (a0->length_50um_10m)
+		ql_dbg(ql_dbg_init, vha, 0x0189,
+		    "SFP Distant (WL=50um): %d m\n", a0->length_50um_10m * 10);
+	if (a0->length_62um_10m)
+		ql_dbg(ql_dbg_init, vha, 0x018a,
+		  "SFP Distant (WL=62.5um): %d m\n", a0->length_62um_10m * 10);
+	if (a0->length_om4_10m)
+		ql_dbg(ql_dbg_init, vha, 0x0194,
+		    "SFP Distant (OM4): %d m\n", a0->length_om4_10m * 10);
+	if (a0->length_om3_10m)
+		ql_dbg(ql_dbg_init, vha, 0x0195,
+		    "SFP Distant (OM3): %d m\n", a0->length_om3_10m * 10);
+}
+
+
+/*
+ * Return Code:
+ *   QLA_SUCCESS: no action
+ *   QLA_INTERFACE_ERROR: SFP is not there.
+ *   QLA_FUNCTION_FAILED: detected New SFP
+ */
+int
+qla24xx_detect_sfp(scsi_qla_host_t *vha)
+{
+	int rc = QLA_SUCCESS;
+	struct sff_8247_a0 *a;
+	struct qla_hw_data *ha = vha->hw;
+
+	if (!AUTO_DETECT_SFP_SUPPORT(vha))
+		goto out;
+
+	rc = qla2x00_read_sfp_dev(vha, NULL, 0);
+	if (rc)
+		goto out;
+
+	a = (struct sff_8247_a0 *)vha->hw->sfp_data;
+	qla2xxx_print_sfp_info(vha);
+
+	if (a->fc_ll_cc7 & FC_LL_VL || a->fc_ll_cc7 & FC_LL_L) {
+		/* long range */
+		ha->flags.detected_lr_sfp = 1;
+
+		if (a->length_km > 5 || a->length_100m > 50)
+			ha->long_range_distance = LR_DISTANCE_10K;
+		else
+			ha->long_range_distance = LR_DISTANCE_5K;
+
+		if (ha->flags.detected_lr_sfp != ha->flags.using_lr_setting)
+			ql_dbg(ql_dbg_async, vha, 0x507b,
+			    "Detected Long Range SFP.\n");
+	} else {
+		/* short range */
+		ha->flags.detected_lr_sfp = 0;
+		if (ha->flags.using_lr_setting)
+			ql_dbg(ql_dbg_async, vha, 0x5084,
+			    "Detected Short Range SFP.\n");
+	}
+
+	if (!vha->flags.init_done)
+		rc = QLA_SUCCESS;
+out:
+	return rc;
+}
+
 /**
  * qla2x00_setup_chip() - Load and start RISC firmware.
  * @ha: HA context
@@ -2879,6 +3019,8 @@ qla2x00_setup_chip(scsi_qla_host_t *vha)
 			rval = qla2x00_execute_fw(vha, srisc_address);
 			/* Retrieve firmware information. */
 			if (rval == QLA_SUCCESS) {
+				qla24xx_detect_sfp(vha);
+
 				rval = qla2x00_set_exlogins_buffer(vha);
 				if (rval != QLA_SUCCESS)
 					goto failed;
@@ -4609,24 +4751,16 @@ qla2x00_configure_fabric(scsi_qla_host_t *vha)
 			qla2x00_fdmi_register(vha);
 
 		/* Ensure we are logged into the SNS. */
-		if (IS_FWI2_CAPABLE(ha))
-			loop_id = NPH_SNS;
-		else
-			loop_id = SIMPLE_NAME_SERVER;
+		loop_id = NPH_SNS_LID(ha);
 		rval = ha->isp_ops->fabric_login(vha, loop_id, 0xff, 0xff,
 		    0xfc, mb, BIT_1|BIT_0);
-		if (rval != QLA_SUCCESS) {
+		if (rval != QLA_SUCCESS || mb[0] != MBS_COMMAND_COMPLETE) {
+			ql_dbg(ql_dbg_disc, vha, 0x20a1,
+			    "Failed SNS login: loop_id=%x mb[0]=%x mb[1]=%x mb[2]=%x mb[6]=%x mb[7]=%x (%x).\n",
+			    loop_id, mb[0], mb[1], mb[2], mb[6], mb[7], rval);
 			set_bit(LOOP_RESYNC_NEEDED, &vha->dpc_flags);
 			return rval;
 		}
-		if (mb[0] != MBS_COMMAND_COMPLETE) {
-			ql_dbg(ql_dbg_disc, vha, 0x20a1,
-			    "Failed SNS login: loop_id=%x mb[0]=%x mb[1]=%x mb[2]=%x "
-			    "mb[6]=%x mb[7]=%x.\n", loop_id, mb[0], mb[1],
-			    mb[2], mb[6], mb[7]);
-			return (QLA_SUCCESS);
-		}
-
 		if (test_and_clear_bit(REGISTER_FC4_NEEDED, &vha->dpc_flags)) {
 			if (qla2x00_rft_id(vha)) {
 				/* EMPTY */
@@ -4804,6 +4938,7 @@ qla2x00_find_all_fabric_devs(scsi_qla_host_t *vha)
 				new_fcport->fc4_type = swl[swl_idx].fc4_type;
 
 				new_fcport->nvme_flag = 0;
+				new_fcport->fc4f_nvme = 0;
 				if (vha->flags.nvme_enabled &&
 				    swl[swl_idx].fc4f_nvme) {
 					new_fcport->fc4f_nvme =
@@ -5913,7 +6048,7 @@ qla2x00_abort_isp(scsi_qla_host_t *vha)
 
 	if (!status) {
 		ql_dbg(ql_dbg_taskm, vha, 0x8022, "%s succeeded.\n", __func__);
-
+		qla2x00_configure_hba(vha);
 		spin_lock_irqsave(&ha->vport_slock, flags);
 		list_for_each_entry(vp, &ha->vp_list, list) {
 			if (vp->vp_idx) {
@@ -7804,7 +7939,9 @@ struct qla_qpair *qla2xxx_create_qpair(struct scsi_qla_host *vha, int qos,
 		ha->queue_pair_map[qpair_id] = qpair;
 		qpair->id = qpair_id;
 		qpair->vp_idx = vp_idx;
+		qpair->fw_started = ha->flags.fw_started;
 		INIT_LIST_HEAD(&qpair->hints_list);
+		INIT_LIST_HEAD(&qpair->nvme_done_list);
 		qpair->chip_reset = ha->base_qpair->chip_reset;
 		qpair->enable_class_2 = ha->base_qpair->enable_class_2;
 		qpair->enable_explicit_conf =
diff --git a/drivers/scsi/qla2xxx/qla_iocb.c b/drivers/scsi/qla2xxx/qla_iocb.c
index a36c485fae50..2f94159186d7 100644
--- a/drivers/scsi/qla2xxx/qla_iocb.c
+++ b/drivers/scsi/qla2xxx/qla_iocb.c
@@ -2682,12 +2682,12 @@ qla24xx_ct_iocb(srb_t *sp, struct ct_entry_24xx *ct_iocb)
 	uint32_t        *cur_dsd;
 	struct scatterlist *sg;
 	int index;
-	uint16_t tot_dsds;
+	uint16_t cmd_dsds, rsp_dsds;
 	scsi_qla_host_t *vha = sp->vha;
 	struct qla_hw_data *ha = vha->hw;
 	struct bsg_job *bsg_job = sp->u.bsg_job;
-	int loop_iterartion = 0;
 	int entry_count = 1;
+	cont_a64_entry_t *cont_pkt = NULL;
 
 	ct_iocb->entry_type = CT_IOCB_TYPE;
         ct_iocb->entry_status = 0;
@@ -2698,30 +2698,46 @@ qla24xx_ct_iocb(srb_t *sp, struct ct_entry_24xx *ct_iocb)
 	ct_iocb->vp_index = sp->vha->vp_idx;
 	ct_iocb->comp_status = cpu_to_le16(0);
 
-	ct_iocb->cmd_dsd_count =
-		cpu_to_le16(bsg_job->request_payload.sg_cnt);
+	cmd_dsds = bsg_job->request_payload.sg_cnt;
+	rsp_dsds = bsg_job->reply_payload.sg_cnt;
+
+	ct_iocb->cmd_dsd_count = cpu_to_le16(cmd_dsds);
         ct_iocb->timeout = 0;
-        ct_iocb->rsp_dsd_count =
-		cpu_to_le16(bsg_job->reply_payload.sg_cnt);
-        ct_iocb->rsp_byte_count =
-            cpu_to_le32(bsg_job->reply_payload.payload_len);
+	ct_iocb->rsp_dsd_count = cpu_to_le16(rsp_dsds);
         ct_iocb->cmd_byte_count =
             cpu_to_le32(bsg_job->request_payload.payload_len);
-        ct_iocb->dseg_0_address[0] = cpu_to_le32(LSD(sg_dma_address
-            (bsg_job->request_payload.sg_list)));
-        ct_iocb->dseg_0_address[1] = cpu_to_le32(MSD(sg_dma_address
-           (bsg_job->request_payload.sg_list)));
-        ct_iocb->dseg_0_len = cpu_to_le32(sg_dma_len
-            (bsg_job->request_payload.sg_list));
 
-	avail_dsds = 1;
-	cur_dsd = (uint32_t *)ct_iocb->dseg_1_address;
+	avail_dsds = 2;
+	cur_dsd = (uint32_t *)ct_iocb->dseg_0_address;
 	index = 0;
-	tot_dsds = bsg_job->reply_payload.sg_cnt;
 
-	for_each_sg(bsg_job->reply_payload.sg_list, sg, tot_dsds, index) {
+	for_each_sg(bsg_job->request_payload.sg_list, sg, cmd_dsds, index) {
+		dma_addr_t       sle_dma;
+
+		/* Allocate additional continuation packets? */
+		if (avail_dsds == 0) {
+			/*
+			 * Five DSDs are available in the Cont.
+			 * Type 1 IOCB.
+			 */
+			cont_pkt = qla2x00_prep_cont_type1_iocb(
+			    vha, ha->req_q_map[0]);
+			cur_dsd = (uint32_t *) cont_pkt->dseg_0_address;
+			avail_dsds = 5;
+			entry_count++;
+		}
+
+		sle_dma = sg_dma_address(sg);
+		*cur_dsd++   = cpu_to_le32(LSD(sle_dma));
+		*cur_dsd++   = cpu_to_le32(MSD(sle_dma));
+		*cur_dsd++   = cpu_to_le32(sg_dma_len(sg));
+		avail_dsds--;
+	}
+
+	index = 0;
+
+	for_each_sg(bsg_job->reply_payload.sg_list, sg, rsp_dsds, index) {
 		dma_addr_t       sle_dma;
-		cont_a64_entry_t *cont_pkt;
 
 		/* Allocate additional continuation packets? */
 		if (avail_dsds == 0) {
@@ -2740,7 +2756,6 @@ qla24xx_ct_iocb(srb_t *sp, struct ct_entry_24xx *ct_iocb)
 		*cur_dsd++   = cpu_to_le32(LSD(sle_dma));
 		*cur_dsd++   = cpu_to_le32(MSD(sle_dma));
 		*cur_dsd++   = cpu_to_le32(sg_dma_len(sg));
-		loop_iterartion++;
 		avail_dsds--;
 	}
         ct_iocb->entry_count = entry_count;
diff --git a/drivers/scsi/qla2xxx/qla_isr.c b/drivers/scsi/qla2xxx/qla_isr.c
index 7b3b702ef622..9d9668aac6f6 100644
--- a/drivers/scsi/qla2xxx/qla_isr.c
+++ b/drivers/scsi/qla2xxx/qla_isr.c
@@ -14,6 +14,8 @@
 #include <scsi/scsi_tcq.h>
 #include <scsi/scsi_bsg_fc.h>
 #include <scsi/scsi_eh.h>
+#include <scsi/fc/fc_fs.h>
+#include <linux/nvme-fc-driver.h>
 
 static void qla2x00_mbx_completion(scsi_qla_host_t *, uint16_t);
 static void qla2x00_status_entry(scsi_qla_host_t *, struct rsp_que *, void *);
@@ -452,7 +454,7 @@ qla83xx_handle_8200_aen(scsi_qla_host_t *vha, uint16_t *mb)
 			uint16_t peg_fw_state, nw_interface_link_up;
 			uint16_t nw_interface_signal_detect, sfp_status;
 			uint16_t htbt_counter, htbt_monitor_enable;
-			uint16_t sfp_additonal_info, sfp_multirate;
+			uint16_t sfp_additional_info, sfp_multirate;
 			uint16_t sfp_tx_fault, link_speed, dcbx_status;
 
 			/*
@@ -492,7 +494,7 @@ qla83xx_handle_8200_aen(scsi_qla_host_t *vha, uint16_t *mb)
 			sfp_status = ((mb[2] & 0x0c00) >> 10);
 			htbt_counter = ((mb[2] & 0x7000) >> 12);
 			htbt_monitor_enable = ((mb[2] & 0x8000) >> 15);
-			sfp_additonal_info = (mb[6] & 0x0003);
+			sfp_additional_info = (mb[6] & 0x0003);
 			sfp_multirate = ((mb[6] & 0x0004) >> 2);
 			sfp_tx_fault = ((mb[6] & 0x0008) >> 3);
 			link_speed = ((mb[6] & 0x0070) >> 4);
@@ -507,9 +509,9 @@ qla83xx_handle_8200_aen(scsi_qla_host_t *vha, uint16_t *mb)
 			    sfp_status);
 			ql_log(ql_log_warn, vha, 0x5067,
 			    "htbt_counter=0x%x, htbt_monitor_enable=0x%x, "
-			    "sfp_additonal_info=0x%x, sfp_multirate=0x%x.\n ",
+			    "sfp_additional_info=0x%x, sfp_multirate=0x%x.\n ",
 			    htbt_counter, htbt_monitor_enable,
-			    sfp_additonal_info, sfp_multirate);
+			    sfp_additional_info, sfp_multirate);
 			ql_log(ql_log_warn, vha, 0x5068,
 			    "sfp_tx_fault=0x%x, link_state=0x%x, "
 			    "dcbx_status=0x%x.\n", sfp_tx_fault, link_speed,
@@ -799,6 +801,11 @@ skip_rio:
 
 		vha->flags.management_server_logged_in = 0;
 		qla2x00_post_aen_work(vha, FCH_EVT_LINKUP, ha->link_data_rate);
+
+		if (AUTO_DETECT_SFP_SUPPORT(vha)) {
+			set_bit(DETECT_SFP_CHANGE, &vha->dpc_flags);
+			qla2xxx_wake_dpc(vha);
+		}
 		break;
 
 	case MBA_LOOP_DOWN:		/* Loop Down Event */
@@ -1228,6 +1235,11 @@ global_port_update:
 			schedule_work(&ha->board_disable);
 		break;
 
+	case MBA_TRANS_INSERT:
+		ql_dbg(ql_dbg_async, vha, 0x5091,
+		    "Transceiver Insertion: %04x\n", mb[1]);
+		break;
+
 	default:
 		ql_dbg(ql_dbg_async, vha, 0x5057,
 		    "Unknown AEN:%04x %04x %04x %04x\n",
@@ -1537,8 +1549,6 @@ qla24xx_els_ct_entry(scsi_qla_host_t *vha, struct req_que *req,
 	sp = qla2x00_get_sp_from_handle(vha, func, req, pkt);
 	if (!sp)
 		return;
-	bsg_job = sp->u.bsg_job;
-	bsg_reply = bsg_job->reply;
 
 	type = NULL;
 	switch (sp->type) {
@@ -1577,6 +1587,8 @@ qla24xx_els_ct_entry(scsi_qla_host_t *vha, struct req_que *req,
 	/* return FC_CTELS_STATUS_OK and leave the decoding of the ELS/CT
 	 * fc payload  to the caller
 	 */
+	bsg_job = sp->u.bsg_job;
+	bsg_reply = bsg_job->reply;
 	bsg_reply->reply_data.ctels_reply.status = FC_CTELS_STATUS_OK;
 	bsg_job->reply_len = sizeof(struct fc_bsg_reply) + sizeof(fw_status);
 
@@ -1823,7 +1835,7 @@ qla24xx_nvme_iocb_entry(scsi_qla_host_t *vha, struct req_que *req, void *tsk)
 	nvme = &sp->u.iocb_cmd;
 
 	if (unlikely(nvme->u.nvme.aen_op))
-		atomic_dec(&sp->vha->nvme_active_aen_cnt);
+		atomic_dec(&sp->vha->hw->nvme_active_aen_cnt);
 
 	/*
 	 * State flags: Bit 6 and 0.
@@ -1856,17 +1868,42 @@ qla24xx_nvme_iocb_entry(scsi_qla_host_t *vha, struct req_que *req, void *tsk)
 	fd->transferred_length = fd->payload_length -
 	    le32_to_cpu(sts->residual_len);
 
+	/*
+	 * If transport error then Failure (HBA rejects request)
+	 * otherwise transport will handle.
+	 */
 	if (sts->entry_status) {
 		ql_log(ql_log_warn, fcport->vha, 0x5038,
 		    "NVME-%s error - hdl=%x entry-status(%x).\n",
 		    sp->name, sp->handle, sts->entry_status);
 		ret = QLA_FUNCTION_FAILED;
-	} else if (sts->comp_status != cpu_to_le16(CS_COMPLETE)) {
-		ql_log(ql_log_warn, fcport->vha, 0x5039,
-		    "NVME-%s error - hdl=%x completion status(%x) resid=%x  ox_id=%x\n",
-		    sp->name, sp->handle, sts->comp_status,
-		    le32_to_cpu(sts->residual_len), sts->ox_id);
-		ret = QLA_FUNCTION_FAILED;
+	} else  {
+		switch (le16_to_cpu(sts->comp_status)) {
+			case CS_COMPLETE:
+				ret = 0;
+			break;
+
+			case CS_ABORTED:
+			case CS_RESET:
+			case CS_PORT_UNAVAILABLE:
+			case CS_PORT_LOGGED_OUT:
+			case CS_PORT_BUSY:
+				ql_log(ql_log_warn, fcport->vha, 0x5060,
+				"NVME-%s ERR Handling - hdl=%x completion status(%x) resid=%x  ox_id=%x\n",
+				sp->name, sp->handle, sts->comp_status,
+				le32_to_cpu(sts->residual_len), sts->ox_id);
+				fd->transferred_length = fd->payload_length;
+				ret = QLA_ABORTED;
+			break;
+
+			default:
+				ql_log(ql_log_warn, fcport->vha, 0x5060,
+				"NVME-%s error - hdl=%x completion status(%x) resid=%x  ox_id=%x\n",
+				sp->name, sp->handle, sts->comp_status,
+				le32_to_cpu(sts->residual_len), sts->ox_id);
+				ret = QLA_FUNCTION_FAILED;
+				break;
+		}
 	}
 	sp->done(sp, ret);
 }
@@ -2827,8 +2864,8 @@ qla24xx_abort_iocb_entry(scsi_qla_host_t *vha, struct req_que *req,
 	sp->done(sp, 0);
 }
 
-void qla24xx_nvme_ls4_iocb(scsi_qla_host_t *vha, struct pt_ls4_request *pkt,
-    struct req_que *req)
+void qla24xx_nvme_ls4_iocb(struct scsi_qla_host *vha,
+    struct pt_ls4_request *pkt, struct req_que *req)
 {
 	srb_t *sp;
 	const char func[] = "LS4_IOCB";
@@ -3132,7 +3169,6 @@ qla24xx_msix_rsp_q(int irq, void *dev_id)
 	struct device_reg_24xx __iomem *reg;
 	struct scsi_qla_host *vha;
 	unsigned long flags;
-	uint32_t stat = 0;
 
 	rsp = (struct rsp_que *) dev_id;
 	if (!rsp) {
@@ -3146,19 +3182,11 @@ qla24xx_msix_rsp_q(int irq, void *dev_id)
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 
 	vha = pci_get_drvdata(ha->pdev);
-	/*
-	 * Use host_status register to check to PCI disconnection before we
-	 * we process the response queue.
-	 */
-	stat = RD_REG_DWORD(&reg->host_status);
-	if (qla2x00_check_reg32_for_disconnect(vha, stat))
-		goto out;
 	qla24xx_process_response_queue(vha, rsp);
 	if (!ha->flags.disable_msix_handshake) {
 		WRT_REG_DWORD(&reg->hccr, HCCRX_CLR_RISC_INT);
 		RD_REG_DWORD_RELAXED(&reg->hccr);
 	}
-out:
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 
 	return IRQ_HANDLED;
@@ -3429,7 +3457,7 @@ msix_register_fail:
 	}
 
 	/* Enable MSI-X vector for response queue update for queue 0 */
-	if (IS_QLA25XX(ha) || IS_QLA83XX(ha) || IS_QLA27XX(ha)) {
+	if (IS_QLA83XX(ha) || IS_QLA27XX(ha)) {
 		if (ha->msixbase && ha->mqiobase &&
 		    (ha->max_rsp_queues > 1 || ha->max_req_queues > 1 ||
 		     ql2xmqsupport))
diff --git a/drivers/scsi/qla2xxx/qla_mbx.c b/drivers/scsi/qla2xxx/qla_mbx.c
index 7c6d1a404011..99502fa90810 100644
--- a/drivers/scsi/qla2xxx/qla_mbx.c
+++ b/drivers/scsi/qla2xxx/qla_mbx.c
@@ -54,6 +54,10 @@ static struct rom_cmd {
 	{ MBC_GET_MEM_OFFLOAD_CNTRL_STAT },
 	{ MBC_GET_RETRY_COUNT },
 	{ MBC_TRACE_CONTROL },
+	{ MBC_INITIALIZE_MULTIQ },
+	{ MBC_IOCB_COMMAND_A64 },
+	{ MBC_GET_ADAPTER_LOOP_ID },
+	{ MBC_READ_SFP },
 };
 
 static int is_rom_cmd(uint16_t cmd)
@@ -102,7 +106,6 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp)
 	uint16_t __iomem *optr;
 	uint32_t	cnt;
 	uint32_t	mboxes;
-	uint16_t __iomem *mbx_reg;
 	unsigned long	wait_time;
 	struct qla_hw_data *ha = vha->hw;
 	scsi_qla_host_t *base_vha = pci_get_drvdata(ha->pdev);
@@ -486,21 +489,24 @@ premature_exit:
 
 mbx_done:
 	if (rval) {
-		ql_dbg(ql_dbg_disc, base_vha, 0x1020,
-		    "**** Failed mbx[0]=%x, mb[1]=%x, mb[2]=%x, mb[3]=%x, cmd=%x ****.\n",
-		    mcp->mb[0], mcp->mb[1], mcp->mb[2], mcp->mb[3], command);
-
+		if (ql2xextended_error_logging & (ql_dbg_disc|ql_dbg_mbx)) {
+			pr_warn("%s [%s]-%04x:%ld: **** Failed", QL_MSGHDR,
+			    dev_name(&ha->pdev->dev), 0x1020+0x800,
+			    vha->host_no);
+			mboxes = mcp->in_mb;
+			cnt = 4;
+			for (i = 0; i < ha->mbx_count && cnt; i++, mboxes >>= 1)
+				if (mboxes & BIT_0) {
+					printk(" mb[%u]=%x", i, mcp->mb[i]);
+					cnt--;
+				}
+			pr_warn(" cmd=%x ****\n", command);
+		}
 		ql_dbg(ql_dbg_mbx, vha, 0x1198,
-		    "host status: 0x%x, flags:0x%lx, intr ctrl reg:0x%x, intr status:0x%x\n",
+		    "host_status=%#x intr_ctrl=%#x intr_status=%#x\n",
 		    RD_REG_DWORD(&reg->isp24.host_status),
-		    ha->fw_dump_cap_flags,
 		    RD_REG_DWORD(&reg->isp24.ictrl),
 		    RD_REG_DWORD(&reg->isp24.istatus));
-
-		mbx_reg = &reg->isp24.mailbox0;
-		for (i = 0; i < 6; i++)
-			ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1199,
-			    "mbox[%d] 0x%04x\n", i, RD_REG_WORD(mbx_reg++));
 	} else {
 		ql_dbg(ql_dbg_mbx, base_vha, 0x1021, "Done %s.\n", __func__);
 	}
@@ -561,6 +567,28 @@ qla2x00_load_ram(scsi_qla_host_t *vha, dma_addr_t req_dma, uint32_t risc_addr,
 
 #define	EXTENDED_BB_CREDITS	BIT_0
 #define	NVME_ENABLE_FLAG	BIT_3
+static inline uint16_t qla25xx_set_sfp_lr_dist(struct qla_hw_data *ha)
+{
+	uint16_t mb4 = BIT_0;
+
+	if (IS_QLA83XX(ha) || IS_QLA27XX(ha))
+		mb4 |= ha->long_range_distance << LR_DIST_FW_POS;
+
+	return mb4;
+}
+
+static inline uint16_t qla25xx_set_nvr_lr_dist(struct qla_hw_data *ha)
+{
+	uint16_t mb4 = BIT_0;
+
+	if (IS_QLA83XX(ha) || IS_QLA27XX(ha)) {
+		struct nvram_81xx *nv = ha->nvram;
+
+		mb4 |= LR_DIST_FW_FIELD(nv->enhanced_features);
+	}
+
+	return mb4;
+}
 
 /*
  * qla2x00_execute_fw
@@ -595,17 +623,44 @@ qla2x00_execute_fw(scsi_qla_host_t *vha, uint32_t risc_addr)
 		mcp->mb[1] = MSW(risc_addr);
 		mcp->mb[2] = LSW(risc_addr);
 		mcp->mb[3] = 0;
+		mcp->mb[4] = 0;
+		ha->flags.using_lr_setting = 0;
 		if (IS_QLA25XX(ha) || IS_QLA81XX(ha) || IS_QLA83XX(ha) ||
 		    IS_QLA27XX(ha)) {
-			struct nvram_81xx *nv = ha->nvram;
-			mcp->mb[4] = (nv->enhanced_features &
-			    EXTENDED_BB_CREDITS);
-		} else
-			mcp->mb[4] = 0;
+			if (ql2xautodetectsfp) {
+				if (ha->flags.detected_lr_sfp) {
+					mcp->mb[4] |=
+					    qla25xx_set_sfp_lr_dist(ha);
+					ha->flags.using_lr_setting = 1;
+				}
+			} else {
+				struct nvram_81xx *nv = ha->nvram;
+				/* set LR distance if specified in nvram */
+				if (nv->enhanced_features &
+				    NEF_LR_DIST_ENABLE) {
+					mcp->mb[4] |=
+					    qla25xx_set_nvr_lr_dist(ha);
+					ha->flags.using_lr_setting = 1;
+				}
+			}
+		}
 
 		if (ql2xnvmeenable && IS_QLA27XX(ha))
 			mcp->mb[4] |= NVME_ENABLE_FLAG;
 
+		if (IS_QLA83XX(ha) || IS_QLA27XX(ha)) {
+			struct nvram_81xx *nv = ha->nvram;
+			/* set minimum speed if specified in nvram */
+			if (nv->min_link_speed >= 2 &&
+			    nv->min_link_speed <= 5) {
+				mcp->mb[4] |= BIT_4;
+				mcp->mb[11] = nv->min_link_speed;
+				mcp->out_mb |= MBX_11;
+				mcp->in_mb |= BIT_5;
+				vha->min_link_speed_feat = nv->min_link_speed;
+			}
+		}
+
 		if (ha->flags.exlogins_enabled)
 			mcp->mb[4] |= ENABLE_EXTENDED_LOGIN;
 
@@ -613,7 +668,7 @@ qla2x00_execute_fw(scsi_qla_host_t *vha, uint32_t risc_addr)
 			mcp->mb[4] |= ENABLE_EXCHANGE_OFFLD;
 
 		mcp->out_mb |= MBX_4|MBX_3|MBX_2|MBX_1;
-		mcp->in_mb |= MBX_1;
+		mcp->in_mb |= MBX_3 | MBX_2 | MBX_1;
 	} else {
 		mcp->mb[1] = LSW(risc_addr);
 		mcp->out_mb |= MBX_1;
@@ -632,12 +687,30 @@ qla2x00_execute_fw(scsi_qla_host_t *vha, uint32_t risc_addr)
 		    "Failed=%x mb[0]=%x.\n", rval, mcp->mb[0]);
 	} else {
 		if (IS_FWI2_CAPABLE(ha)) {
-			ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1027,
-			    "Done exchanges=%x.\n", mcp->mb[1]);
-		} else {
-			ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1028,
-			    "Done %s.\n", __func__);
+			ha->fw_ability_mask = mcp->mb[3] << 16 | mcp->mb[2];
+			ql_dbg(ql_dbg_mbx, vha, 0x119a,
+			    "fw_ability_mask=%x.\n", ha->fw_ability_mask);
+			ql_dbg(ql_dbg_mbx, vha, 0x1027,
+			    "exchanges=%x.\n", mcp->mb[1]);
+			if (IS_QLA83XX(ha) || IS_QLA27XX(ha)) {
+				ha->max_speed_sup = mcp->mb[2] & BIT_0;
+				ql_dbg(ql_dbg_mbx, vha, 0x119b,
+				    "Maximum speed supported=%s.\n",
+				    ha->max_speed_sup ? "32Gps" : "16Gps");
+				if (vha->min_link_speed_feat) {
+					ha->min_link_speed = mcp->mb[5];
+					ql_dbg(ql_dbg_mbx, vha, 0x119c,
+					    "Minimum speed set=%s.\n",
+					    mcp->mb[5] == 5 ? "32Gps" :
+					    mcp->mb[5] == 4 ? "16Gps" :
+					    mcp->mb[5] == 3 ? "8Gps" :
+					    mcp->mb[5] == 2 ? "4Gps" :
+						"unknown");
+				}
+			}
 		}
+		ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1028,
+		    "Done.\n");
 	}
 
 	return rval;
@@ -947,20 +1020,12 @@ qla2x00_get_fw_version(scsi_qla_host_t *vha)
 			    "%s: Firmware supports Exchange Offload 0x%x\n",
 			    __func__, ha->fw_attributes_h);
 
-		/* bit 26 of fw_attributes */
-		if ((ha->fw_attributes_h & 0x400) && ql2xnvmeenable) {
-			struct init_cb_24xx *icb;
-
-			icb = (struct init_cb_24xx *)ha->init_cb;
-			/*
-			 * fw supports nvme and driver load
-			 * parameter requested nvme
-			 */
+		/*
+		 * FW supports nvme and driver load parameter requested nvme.
+		 * BIT 26 of fw_attributes indicates NVMe support.
+		 */
+		if ((ha->fw_attributes_h & 0x400) && ql2xnvmeenable)
 			vha->flags.nvme_enabled = 1;
-			icb->firmware_options_2 &= cpu_to_le32(~0xf);
-			ha->zio_mode = 0;
-			ha->zio_timer = 0;
-		}
 
 	}
 
@@ -1673,7 +1738,11 @@ qla2x00_init_firmware(scsi_qla_host_t *vha, uint16_t size)
 		    "Failed=%x mb[0]=%x, mb[1]=%x, mb[2]=%x, mb[3]=%x,.\n",
 		    rval, mcp->mb[0], mcp->mb[1], mcp->mb[2], mcp->mb[3]);
 	} else {
-		/*EMPTY*/
+		if (IS_QLA27XX(ha)) {
+			if (mcp->mb[2] == 6 || mcp->mb[3] == 2)
+				ql_dbg(ql_dbg_mbx, vha, 0x119d,
+				    "Invalid SFP/Validation Failed\n");
+		}
 		ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x104e,
 		    "Done %s.\n", __func__);
 	}
@@ -1878,6 +1947,7 @@ qla2x00_get_firmware_state(scsi_qla_host_t *vha, uint16_t *states)
 	int rval;
 	mbx_cmd_t mc;
 	mbx_cmd_t *mcp = &mc;
+	struct qla_hw_data *ha = vha->hw;
 
 	ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1054,
 	    "Entered %s.\n", __func__);
@@ -1906,7 +1976,11 @@ qla2x00_get_firmware_state(scsi_qla_host_t *vha, uint16_t *states)
 		/*EMPTY*/
 		ql_dbg(ql_dbg_mbx, vha, 0x1055, "Failed=%x.\n", rval);
 	} else {
-		/*EMPTY*/
+		if (IS_QLA27XX(ha)) {
+			if (mcp->mb[2] == 6 || mcp->mb[3] == 2)
+				ql_dbg(ql_dbg_mbx, vha, 0x119e,
+				    "Invalid SFP/Validation Failed\n");
+		}
 		ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1056,
 		    "Done %s.\n", __func__);
 	}
@@ -3689,7 +3763,7 @@ qla24xx_report_id_acquisition(scsi_qla_host_t *vha,
 				if (qla_ini_mode_enabled(vha) &&
 				    ha->flags.fawwpn_enabled &&
 				    (rptid_entry->u.f1.flags &
-				     VP_FLAGS_NAME_VALID)) {
+				     BIT_6)) {
 					memcpy(vha->port_name,
 					    rptid_entry->u.f1.port_name,
 					    WWN_SIZE);
@@ -4590,6 +4664,10 @@ qla2x00_read_sfp(scsi_qla_host_t *vha, dma_addr_t sfp_dma, uint8_t *sfp,
 	if (rval != QLA_SUCCESS) {
 		ql_dbg(ql_dbg_mbx, vha, 0x10e9,
 		    "Failed=%x mb[0]=%x.\n", rval, mcp->mb[0]);
+		if (mcp->mb[0] == MBS_COMMAND_ERROR &&
+		    mcp->mb[1] == 0x22)
+			/* sfp is not there */
+			rval = QLA_INTERFACE_ERROR;
 	} else {
 		ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x10ea,
 		    "Done %s.\n", __func__);
@@ -5817,7 +5895,7 @@ qla26xx_dport_diagnostics(scsi_qla_host_t *vha,
 
 	dd_dma = dma_map_single(&vha->hw->pdev->dev,
 	    dd_buf, size, DMA_FROM_DEVICE);
-	if (!dd_dma) {
+	if (dma_mapping_error(&vha->hw->pdev->dev, dd_dma)) {
 		ql_log(ql_log_warn, vha, 0x1194, "Failed to map dma buffer.\n");
 		return QLA_MEMORY_ALLOC_FAILED;
 	}
@@ -6085,3 +6163,108 @@ int qla24xx_gidlist_wait(struct scsi_qla_host *vha,
 done:
 	return rval;
 }
+
+int qla27xx_set_zio_threshold(scsi_qla_host_t *vha, uint16_t value)
+{
+	int rval;
+	mbx_cmd_t	mc;
+	mbx_cmd_t	*mcp = &mc;
+
+	ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1200,
+	    "Entered %s\n", __func__);
+
+	memset(mcp->mb, 0 , sizeof(mcp->mb));
+	mcp->mb[0] = MBC_GET_SET_ZIO_THRESHOLD;
+	mcp->mb[1] = cpu_to_le16(1);
+	mcp->mb[2] = cpu_to_le16(value);
+	mcp->out_mb = MBX_2 | MBX_1 | MBX_0;
+	mcp->in_mb = MBX_2 | MBX_0;
+	mcp->tov = MBX_TOV_SECONDS;
+	mcp->flags = 0;
+
+	rval = qla2x00_mailbox_command(vha, mcp);
+
+	ql_dbg(ql_dbg_mbx, vha, 0x1201, "%s %x\n",
+	    (rval != QLA_SUCCESS) ? "Failed"  : "Done", rval);
+
+	return rval;
+}
+
+int qla27xx_get_zio_threshold(scsi_qla_host_t *vha, uint16_t *value)
+{
+	int rval;
+	mbx_cmd_t	mc;
+	mbx_cmd_t	*mcp = &mc;
+
+	ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1203,
+	    "Entered %s\n", __func__);
+
+	memset(mcp->mb, 0, sizeof(mcp->mb));
+	mcp->mb[0] = MBC_GET_SET_ZIO_THRESHOLD;
+	mcp->mb[1] = cpu_to_le16(0);
+	mcp->out_mb = MBX_1 | MBX_0;
+	mcp->in_mb = MBX_2 | MBX_0;
+	mcp->tov = MBX_TOV_SECONDS;
+	mcp->flags = 0;
+
+	rval = qla2x00_mailbox_command(vha, mcp);
+	if (rval == QLA_SUCCESS)
+		*value = mc.mb[2];
+
+	ql_dbg(ql_dbg_mbx, vha, 0x1205, "%s %x\n",
+	    (rval != QLA_SUCCESS) ? "Failed" : "Done", rval);
+
+	return rval;
+}
+
+int
+qla2x00_read_sfp_dev(struct scsi_qla_host *vha, char *buf, int count)
+{
+	struct qla_hw_data *ha = vha->hw;
+	uint16_t iter, addr, offset;
+	dma_addr_t phys_addr;
+	int rval, c;
+	u8 *sfp_data;
+
+	memset(ha->sfp_data, 0, SFP_DEV_SIZE);
+	addr = 0xa0;
+	phys_addr = ha->sfp_data_dma;
+	sfp_data = ha->sfp_data;
+	offset = c = 0;
+
+	for (iter = 0; iter < SFP_DEV_SIZE / SFP_BLOCK_SIZE; iter++) {
+		if (iter == 4) {
+			/* Skip to next device address. */
+			addr = 0xa2;
+			offset = 0;
+		}
+
+		rval = qla2x00_read_sfp(vha, phys_addr, sfp_data,
+		    addr, offset, SFP_BLOCK_SIZE, BIT_1);
+		if (rval != QLA_SUCCESS) {
+			ql_log(ql_log_warn, vha, 0x706d,
+			    "Unable to read SFP data (%x/%x/%x).\n", rval,
+			    addr, offset);
+
+			return rval;
+		}
+
+		if (buf && (c < count)) {
+			u16 sz;
+
+			if ((count - c) >= SFP_BLOCK_SIZE)
+				sz = SFP_BLOCK_SIZE;
+			else
+				sz = count - c;
+
+			memcpy(buf, sfp_data, sz);
+			buf += SFP_BLOCK_SIZE;
+			c += sz;
+		}
+		phys_addr += SFP_BLOCK_SIZE;
+		sfp_data  += SFP_BLOCK_SIZE;
+		offset += SFP_BLOCK_SIZE;
+	}
+
+	return rval;
+}
diff --git a/drivers/scsi/qla2xxx/qla_mid.c b/drivers/scsi/qla2xxx/qla_mid.c
index f0605cd196fb..c0f8f6c17b79 100644
--- a/drivers/scsi/qla2xxx/qla_mid.c
+++ b/drivers/scsi/qla2xxx/qla_mid.c
@@ -74,7 +74,7 @@ qla24xx_deallocate_vp_id(scsi_qla_host_t *vha)
 	 * ensures no active vp_list traversal while the vport is removed
 	 * from the queue)
 	 */
-	wait_event_timeout(vha->vref_waitq, atomic_read(&vha->vref_count),
+	wait_event_timeout(vha->vref_waitq, !atomic_read(&vha->vref_count),
 	    10*HZ);
 
 	spin_lock_irqsave(&ha->vport_slock, flags);
@@ -187,6 +187,11 @@ qla24xx_enable_vp(scsi_qla_host_t *vha)
 		!(ha->current_topology & ISP_CFG_F)) {
 		vha->vp_err_state =  VP_ERR_PORTDWN;
 		fc_vport_set_state(vha->fc_vport, FC_VPORT_LINKDOWN);
+		ql_dbg(ql_dbg_taskm, vha, 0x800b,
+		    "%s skip enable. loop_state %x topo %x\n",
+		    __func__, base_vha->loop_state.counter,
+		    ha->current_topology);
+
 		goto enable_failed;
 	}
 
@@ -759,11 +764,18 @@ static void qla_do_work(struct work_struct *work)
 	struct qla_qpair *qpair = container_of(work, struct qla_qpair, q_work);
 	struct scsi_qla_host *vha;
 	struct qla_hw_data *ha = qpair->hw;
+	struct srb_iocb	*nvme, *nxt_nvme;
 
 	spin_lock_irqsave(&qpair->qp_lock, flags);
 	vha = pci_get_drvdata(ha->pdev);
 	qla24xx_process_response_queue(vha, qpair->rsp);
 	spin_unlock_irqrestore(&qpair->qp_lock, flags);
+
+	list_for_each_entry_safe(nvme, nxt_nvme, &qpair->nvme_done_list,
+		    u.nvme.entry) {
+		list_del_init(&nvme->u.nvme.entry);
+		qla_nvme_cmpl_io(nvme);
+	}
 }
 
 /* create response queue */
diff --git a/drivers/scsi/qla2xxx/qla_mr.c b/drivers/scsi/qla2xxx/qla_mr.c
index 10b742d27e16..e23a3d4c36f3 100644
--- a/drivers/scsi/qla2xxx/qla_mr.c
+++ b/drivers/scsi/qla2xxx/qla_mr.c
@@ -1819,6 +1819,10 @@ qlafx00_fx_disc(scsi_qla_host_t *vha, fc_port_t *fcport, uint16_t fx_type)
 	if (!sp)
 		goto done;
 
+	sp->type = SRB_FXIOCB_DCMD;
+	sp->name = "fxdisc";
+	qla2x00_init_timer(sp, FXDISC_TIMEOUT);
+
 	fdisc = &sp->u.iocb_cmd;
 	switch (fx_type) {
 	case FXDISC_GET_CONFIG_INFO:
@@ -1920,9 +1924,6 @@ qlafx00_fx_disc(scsi_qla_host_t *vha, fc_port_t *fcport, uint16_t fx_type)
 			goto done_unmap_req;
 	}
 
-	sp->type = SRB_FXIOCB_DCMD;
-	sp->name = "fxdisc";
-	qla2x00_init_timer(sp, FXDISC_TIMEOUT);
 	fdisc->timeout = qla2x00_fxdisc_iocb_timeout;
 	fdisc->u.fxiocb.req_func_type = cpu_to_le16(fx_type);
 	sp->done = qla2x00_fxdisc_sp_done;
diff --git a/drivers/scsi/qla2xxx/qla_nvme.c b/drivers/scsi/qla2xxx/qla_nvme.c
index f3710a75fe1f..1f59e7a74c7b 100644
--- a/drivers/scsi/qla2xxx/qla_nvme.c
+++ b/drivers/scsi/qla2xxx/qla_nvme.c
@@ -5,7 +5,6 @@
  * See LICENSE.qla2xxx for copyright and licensing details.
  */
 #include "qla_nvme.h"
-#include "qla_def.h"
 #include <linux/scatterlist.h>
 #include <linux/delay.h>
 #include <linux/nvme.h>
@@ -15,7 +14,7 @@ static struct nvme_fc_port_template qla_nvme_fc_transport;
 
 static void qla_nvme_unregister_remote_port(struct work_struct *);
 
-int qla_nvme_register_remote(scsi_qla_host_t *vha, fc_port_t *fcport)
+int qla_nvme_register_remote(struct scsi_qla_host *vha, struct fc_port *fcport)
 {
 	struct nvme_rport *rport;
 	int ret;
@@ -61,8 +60,8 @@ int qla_nvme_register_remote(scsi_qla_host_t *vha, fc_port_t *fcport)
 	rport->req.port_id = fcport->d_id.b24;
 
 	ql_log(ql_log_info, vha, 0x2102,
-	    "%s: traddr=pn-0x%016llx:nn-0x%016llx PortID:%06x\n",
-	    __func__, rport->req.port_name, rport->req.node_name,
+	    "%s: traddr=nn-0x%016llx:pn-0x%016llx PortID:%06x\n",
+	    __func__, rport->req.node_name, rport->req.port_name,
 	    rport->req.port_id);
 
 	ret = nvme_fc_register_remoteport(vha->nvme_local_port, &rport->req,
@@ -76,16 +75,14 @@ int qla_nvme_register_remote(scsi_qla_host_t *vha, fc_port_t *fcport)
 
 	fcport->nvme_remote_port->private = fcport;
 	fcport->nvme_flag |= NVME_FLAG_REGISTERED;
-	atomic_set(&fcport->nvme_ref_count, 1);
-	init_waitqueue_head(&fcport->nvme_waitQ);
 	rport->fcport = fcport;
 	list_add_tail(&rport->list, &vha->nvme_rport_list);
 	return 0;
 }
 
 /* Allocate a queue for NVMe traffic */
-static int qla_nvme_alloc_queue(struct nvme_fc_local_port *lport, unsigned int qidx,
-    u16 qsize, void **handle)
+static int qla_nvme_alloc_queue(struct nvme_fc_local_port *lport,
+    unsigned int qidx, u16 qsize, void **handle)
 {
 	struct scsi_qla_host *vha;
 	struct qla_hw_data *ha;
@@ -157,6 +154,16 @@ static void qla_nvme_sp_ls_done(void *ptr, int res)
 	qla2x00_rel_sp(sp);
 }
 
+void qla_nvme_cmpl_io(struct srb_iocb *nvme)
+{
+	srb_t *sp;
+	struct nvmefc_fcp_req *fd = nvme->u.nvme.desc;
+
+	sp = container_of(nvme, srb_t, u.iocb_cmd);
+	fd->done(fd);
+	qla2xxx_rel_qpair_sp(sp->qpair, sp);
+}
+
 static void qla_nvme_sp_done(void *ptr, int res)
 {
 	srb_t *sp = ptr;
@@ -172,13 +179,14 @@ static void qla_nvme_sp_done(void *ptr, int res)
 	if (!(sp->fcport->nvme_flag & NVME_FLAG_REGISTERED))
 		goto rel;
 
-	if (unlikely(nvme->u.nvme.comp_status || res))
-		fd->status = -EINVAL;
+	if (unlikely(res == QLA_FUNCTION_FAILED))
+		fd->status = NVME_SC_FC_TRANSPORT_ERROR;
 	else
 		fd->status = 0;
 
 	fd->rcv_rsplen = nvme->u.nvme.rsp_pyld_len;
-	fd->done(fd);
+	list_add_tail(&nvme->u.nvme.entry, &sp->qpair->nvme_done_list);
+	return;
 rel:
 	qla2xxx_rel_qpair_sp(sp->qpair, sp);
 }
@@ -193,13 +201,11 @@ static void qla_nvme_ls_abort(struct nvme_fc_local_port *lport,
 	struct qla_hw_data *ha = fcport->vha->hw;
 
 	rval = ha->isp_ops->abort_command(sp);
-	if (rval != QLA_SUCCESS)
-		ql_log(ql_log_warn, fcport->vha, 0x2125,
-		    "%s: failed to abort LS command for SP:%p rval=%x\n",
-		    __func__, sp, rval);
 
 	ql_dbg(ql_dbg_io, fcport->vha, 0x212b,
-	    "%s: aborted sp:%p on fcport:%p\n", __func__, sp, fcport);
+	    "%s: %s LS command for sp=%p on fcport=%p rval=%x\n", __func__,
+	    (rval != QLA_SUCCESS) ? "Failed to abort" : "Aborted",
+	    sp, fcport, rval);
 }
 
 static void qla_nvme_ls_complete(struct work_struct *work)
@@ -214,7 +220,7 @@ static void qla_nvme_ls_complete(struct work_struct *work)
 static int qla_nvme_ls_req(struct nvme_fc_local_port *lport,
     struct nvme_fc_remote_port *rport, struct nvmefc_ls_req *fd)
 {
-	fc_port_t *fcport = (fc_port_t *)rport->private;
+	fc_port_t *fcport = rport->private;
 	struct srb_iocb   *nvme;
 	struct nvme_private *priv = fd->private;
 	struct scsi_qla_host *vha;
@@ -236,7 +242,6 @@ static int qla_nvme_ls_req(struct nvme_fc_local_port *lport,
 	sp->name = "nvme_ls";
 	sp->done = qla_nvme_sp_ls_done;
 	atomic_set(&sp->ref_count, 1);
-	init_waitqueue_head(&sp->nvme_ls_waitQ);
 	nvme = &sp->u.iocb_cmd;
 	priv->sp = sp;
 	priv->fd = fd;
@@ -258,7 +263,7 @@ static int qla_nvme_ls_req(struct nvme_fc_local_port *lport,
 		ql_log(ql_log_warn, vha, 0x700e,
 		    "qla2x00_start_sp failed = %d\n", rval);
 		atomic_dec(&sp->ref_count);
-		wake_up(&sp->nvme_ls_waitQ);
+		wake_up(&sp->nvme_ls_waitq);
 		return rval;
 	}
 
@@ -276,20 +281,18 @@ static void qla_nvme_fcp_abort(struct nvme_fc_local_port *lport,
 	struct qla_hw_data *ha = fcport->vha->hw;
 
 	rval = ha->isp_ops->abort_command(sp);
-	if (!rval)
-		ql_log(ql_log_warn, fcport->vha, 0x2127,
-		    "%s: failed to abort command for SP:%p rval=%x\n",
-		    __func__, sp, rval);
 
-	ql_dbg(ql_dbg_io, fcport->vha, 0x2126,
-	    "%s: aborted sp:%p on fcport:%p\n", __func__, sp, fcport);
+	ql_dbg(ql_dbg_io, fcport->vha, 0x2127,
+	    "%s: %s command for sp=%p on fcport=%p rval=%x\n", __func__,
+	    (rval != QLA_SUCCESS) ? "Failed to abort" : "Aborted",
+	    sp, fcport, rval);
 }
 
 static void qla_nvme_poll(struct nvme_fc_local_port *lport, void *hw_queue_handle)
 {
 	struct scsi_qla_host *vha = lport->private;
 	unsigned long flags;
-	struct qla_qpair *qpair = (struct qla_qpair *)hw_queue_handle;
+	struct qla_qpair *qpair = hw_queue_handle;
 
 	/* Acquire ring specific lock */
 	spin_lock_irqsave(&qpair->qp_lock, flags);
@@ -310,6 +313,7 @@ static int qla2x00_start_nvme_mq(srb_t *sp)
 	uint16_t	avail_dsds;
 	uint32_t	*cur_dsd;
 	struct req_que *req = NULL;
+	struct rsp_que *rsp = NULL;
 	struct scsi_qla_host *vha = sp->fcport->vha;
 	struct qla_hw_data *ha = vha->hw;
 	struct qla_qpair *qpair = sp->qpair;
@@ -318,13 +322,15 @@ static int qla2x00_start_nvme_mq(srb_t *sp)
 	struct nvmefc_fcp_req *fd = nvme->u.nvme.desc;
 	uint32_t        rval = QLA_SUCCESS;
 
-	/* Setup qpair pointers */
-	req = qpair->req;
 	tot_dsds = fd->sg_cnt;
 
 	/* Acquire qpair specific lock */
 	spin_lock_irqsave(&qpair->qp_lock, flags);
 
+	/* Setup qpair pointers */
+	req = qpair->req;
+	rsp = qpair->rsp;
+
 	/* Check for room in outstanding command list. */
 	handle = req->current_outstanding_cmd;
 	for (index = 1; index < req->num_outstanding_cmds; index++) {
@@ -359,7 +365,7 @@ static int qla2x00_start_nvme_mq(srb_t *sp)
 		struct nvme_fc_cmd_iu *cmd = fd->cmdaddr;
 		if (cmd->sqe.common.opcode == nvme_admin_async_event) {
 			nvme->u.nvme.aen_op = 1;
-			atomic_inc(&vha->nvme_active_aen_cnt);
+			atomic_inc(&vha->hw->nvme_active_aen_cnt);
 		}
 	}
 
@@ -472,6 +478,11 @@ static int qla2x00_start_nvme_mq(srb_t *sp)
 	/* Set chip new ring index. */
 	WRT_REG_DWORD(req->req_q_in, req->ring_index);
 
+	/* Manage unprocessed RIO/ZIO commands in response queue. */
+	if (vha->flags.process_response_queue &&
+	    rsp->ring_ptr->signature != RESPONSE_PROCESSED)
+		qla24xx_process_response_queue(vha, rsp);
+
 queuing_error:
 	spin_unlock_irqrestore(&qpair->qp_lock, flags);
 	return rval;
@@ -487,7 +498,7 @@ static int qla_nvme_post_cmd(struct nvme_fc_local_port *lport,
 	struct scsi_qla_host *vha;
 	int rval = QLA_FUNCTION_FAILED;
 	srb_t *sp;
-	struct qla_qpair *qpair = (struct qla_qpair *)hw_queue_handle;
+	struct qla_qpair *qpair = hw_queue_handle;
 	struct nvme_private *priv;
 
 	if (!fd) {
@@ -496,7 +507,7 @@ static int qla_nvme_post_cmd(struct nvme_fc_local_port *lport,
 	}
 
 	priv = fd->private;
-	fcport = (fc_port_t *)rport->private;
+	fcport = rport->private;
 	if (!fcport) {
 		ql_log(ql_log_warn, NULL, 0x210e, "No fcport ptr\n");
 		return rval;
@@ -512,7 +523,7 @@ static int qla_nvme_post_cmd(struct nvme_fc_local_port *lport,
 		return -EIO;
 
 	atomic_set(&sp->ref_count, 1);
-	init_waitqueue_head(&sp->nvme_ls_waitQ);
+	init_waitqueue_head(&sp->nvme_ls_waitq);
 	priv->sp = sp;
 	sp->type = SRB_NVME_CMD;
 	sp->name = "nvme_cmd";
@@ -526,7 +537,7 @@ static int qla_nvme_post_cmd(struct nvme_fc_local_port *lport,
 		ql_log(ql_log_warn, vha, 0x212d,
 		    "qla2x00_start_nvme_mq failed = %d\n", rval);
 		atomic_dec(&sp->ref_count);
-		wake_up(&sp->nvme_ls_waitQ);
+		wake_up(&sp->nvme_ls_waitq);
 		return -EIO;
 	}
 
@@ -537,12 +548,10 @@ static void qla_nvme_localport_delete(struct nvme_fc_local_port *lport)
 {
 	struct scsi_qla_host *vha = lport->private;
 
-	atomic_dec(&vha->nvme_ref_count);
-	wake_up_all(&vha->nvme_waitQ);
-
 	ql_log(ql_log_info, vha, 0x210f,
 	    "localport delete of %p completed.\n", vha->nvme_local_port);
 	vha->nvme_local_port = NULL;
+	complete(&vha->nvme_del_done);
 }
 
 static void qla_nvme_remoteport_delete(struct nvme_fc_remote_port *rport)
@@ -550,11 +559,9 @@ static void qla_nvme_remoteport_delete(struct nvme_fc_remote_port *rport)
 	fc_port_t *fcport;
 	struct nvme_rport *r_port, *trport;
 
-	fcport = (fc_port_t *)rport->private;
+	fcport = rport->private;
 	fcport->nvme_remote_port = NULL;
 	fcport->nvme_flag &= ~NVME_FLAG_REGISTERED;
-	atomic_dec(&fcport->nvme_ref_count);
-	wake_up_all(&fcport->nvme_waitQ);
 
 	list_for_each_entry_safe(r_port, trport,
 	    &fcport->vha->nvme_rport_list, list) {
@@ -564,6 +571,7 @@ static void qla_nvme_remoteport_delete(struct nvme_fc_remote_port *rport)
 		}
 	}
 	kfree(r_port);
+	complete(&fcport->nvme_del_done);
 
 	ql_log(ql_log_info, fcport->vha, 0x2110,
 	    "remoteport_delete of %p completed.\n", fcport);
@@ -594,7 +602,7 @@ static int qla_nvme_wait_on_command(srb_t *sp)
 {
 	int ret = QLA_SUCCESS;
 
-	wait_event_timeout(sp->nvme_ls_waitQ, (atomic_read(&sp->ref_count) > 1),
+	wait_event_timeout(sp->nvme_ls_waitq, (atomic_read(&sp->ref_count) > 1),
 	    NVME_ABORT_POLLING_PERIOD*HZ);
 
 	if (atomic_read(&sp->ref_count) > 1)
@@ -606,12 +614,11 @@ static int qla_nvme_wait_on_command(srb_t *sp)
 static int qla_nvme_wait_on_rport_del(fc_port_t *fcport)
 {
 	int ret = QLA_SUCCESS;
+	int timeout;
 
-	wait_event_timeout(fcport->nvme_waitQ,
-	    atomic_read(&fcport->nvme_ref_count),
-	    NVME_ABORT_POLLING_PERIOD*HZ);
-
-	if (atomic_read(&fcport->nvme_ref_count)) {
+	timeout = wait_for_completion_timeout(&fcport->nvme_del_done,
+	    msecs_to_jiffies(2000));
+	if (!timeout) {
 		ret = QLA_FUNCTION_FAILED;
 		ql_log(ql_log_info, fcport->vha, 0x2111,
 		    "timed out waiting for fcport=%p to delete\n", fcport);
@@ -620,50 +627,14 @@ static int qla_nvme_wait_on_rport_del(fc_port_t *fcport)
 	return ret;
 }
 
-void qla_nvme_abort(struct qla_hw_data *ha, srb_t *sp)
+void qla_nvme_abort(struct qla_hw_data *ha, struct srb *sp)
 {
 	int rval;
 
 	rval = ha->isp_ops->abort_command(sp);
-	if (!rval) {
-		if (!qla_nvme_wait_on_command(sp))
-			ql_log(ql_log_warn, NULL, 0x2112,
-			    "nvme_wait_on_command timed out waiting on sp=%p\n",
-			    sp);
-	}
-}
-
-static void qla_nvme_abort_all(fc_port_t *fcport)
-{
-	int que, cnt;
-	unsigned long flags;
-	srb_t *sp;
-	struct qla_hw_data *ha = fcport->vha->hw;
-	struct req_que *req;
-
-	spin_lock_irqsave(&ha->hardware_lock, flags);
-	for (que = 0; que < ha->max_req_queues; que++) {
-		req = ha->req_q_map[que];
-		if (!req)
-			continue;
-		if (!req->outstanding_cmds)
-			continue;
-		for (cnt = 1; cnt < req->num_outstanding_cmds; cnt++) {
-			sp = req->outstanding_cmds[cnt];
-			if ((sp) && ((sp->type == SRB_NVME_CMD) ||
-			    (sp->type == SRB_NVME_LS)) &&
-				(sp->fcport == fcport)) {
-				atomic_inc(&sp->ref_count);
-				spin_unlock_irqrestore(&ha->hardware_lock,
-				    flags);
-				qla_nvme_abort(ha, sp);
-				spin_lock_irqsave(&ha->hardware_lock, flags);
-				req->outstanding_cmds[cnt] = NULL;
-				sp->done(sp, 1);
-			}
-		}
-	}
-	spin_unlock_irqrestore(&ha->hardware_lock, flags);
+	if (!rval && !qla_nvme_wait_on_command(sp))
+		ql_log(ql_log_warn, NULL, 0x2112,
+		    "nvme_wait_on_comand timed out waiting on sp=%p\n", sp);
 }
 
 static void qla_nvme_unregister_remote_port(struct work_struct *work)
@@ -675,18 +646,23 @@ static void qla_nvme_unregister_remote_port(struct work_struct *work)
 	if (!IS_ENABLED(CONFIG_NVME_FC))
 		return;
 
+	ql_log(ql_log_warn, NULL, 0x2112,
+	    "%s: unregister remoteport on %p\n",__func__, fcport);
+
 	list_for_each_entry_safe(rport, trport,
 	    &fcport->vha->nvme_rport_list, list) {
 		if (rport->fcport == fcport) {
 			ql_log(ql_log_info, fcport->vha, 0x2113,
 			    "%s: fcport=%p\n", __func__, fcport);
+			init_completion(&fcport->nvme_del_done);
 			nvme_fc_unregister_remoteport(
 			    fcport->nvme_remote_port);
+			qla_nvme_wait_on_rport_del(fcport);
 		}
 	}
 }
 
-void qla_nvme_delete(scsi_qla_host_t *vha)
+void qla_nvme_delete(struct scsi_qla_host *vha)
 {
 	struct nvme_rport *rport, *trport;
 	fc_port_t *fcport;
@@ -701,12 +677,13 @@ void qla_nvme_delete(scsi_qla_host_t *vha)
 		ql_log(ql_log_info, fcport->vha, 0x2114, "%s: fcport=%p\n",
 		    __func__, fcport);
 
+		init_completion(&fcport->nvme_del_done);
 		nvme_fc_unregister_remoteport(fcport->nvme_remote_port);
 		qla_nvme_wait_on_rport_del(fcport);
-		qla_nvme_abort_all(fcport);
 	}
 
 	if (vha->nvme_local_port) {
+		init_completion(&vha->nvme_del_done);
 		nv_ret = nvme_fc_unregister_localport(vha->nvme_local_port);
 		if (nv_ret == 0)
 			ql_log(ql_log_info, vha, 0x2116,
@@ -715,10 +692,12 @@ void qla_nvme_delete(scsi_qla_host_t *vha)
 		else
 			ql_log(ql_log_info, vha, 0x2115,
 			    "Unregister of localport failed\n");
+		wait_for_completion_timeout(&vha->nvme_del_done,
+		    msecs_to_jiffies(5000));
 	}
 }
 
-void qla_nvme_register_hba(scsi_qla_host_t *vha)
+void qla_nvme_register_hba(struct scsi_qla_host *vha)
 {
 	struct nvme_fc_port_template *tmpl;
 	struct qla_hw_data *ha;
@@ -744,8 +723,8 @@ void qla_nvme_register_hba(scsi_qla_host_t *vha)
 	pinfo.port_id = vha->d_id.b24;
 
 	ql_log(ql_log_info, vha, 0xffff,
-	    "register_localport: host-traddr=pn-0x%llx:nn-0x%llx on portID:%x\n",
-	    pinfo.port_name, pinfo.node_name, pinfo.port_id);
+	    "register_localport: host-traddr=nn-0x%llx:pn-0x%llx on portID:%x\n",
+	    pinfo.node_name, pinfo.port_name, pinfo.port_id);
 	qla_nvme_fc_transport.dma_boundary = vha->host->dma_boundary;
 
 	ret = nvme_fc_register_localport(&pinfo, tmpl,
@@ -755,7 +734,5 @@ void qla_nvme_register_hba(scsi_qla_host_t *vha)
 		    "register_localport failed: ret=%x\n", ret);
 		return;
 	}
-	atomic_set(&vha->nvme_ref_count, 1);
 	vha->nvme_local_port->private = vha;
-	init_waitqueue_head(&vha->nvme_waitQ);
 }
diff --git a/drivers/scsi/qla2xxx/qla_nvme.h b/drivers/scsi/qla2xxx/qla_nvme.h
index dfe56f207b28..7f05fa1c77db 100644
--- a/drivers/scsi/qla2xxx/qla_nvme.h
+++ b/drivers/scsi/qla2xxx/qla_nvme.h
@@ -12,12 +12,18 @@
 #include <uapi/scsi/fc/fc_els.h>
 #include <linux/nvme-fc-driver.h>
 
+#include "qla_def.h"
+
 #define NVME_ATIO_CMD_OFF 32
 #define NVME_FIRST_PACKET_CMDLEN (64 - NVME_ATIO_CMD_OFF)
 #define Q2T_NVME_NUM_TAGS 2048
 #define QLA_MAX_FC_SEGMENTS 64
 
+struct scsi_qla_host;
+struct qla_hw_data;
+struct req_que;
 struct srb;
+
 struct nvme_private {
 	struct srb	*sp;
 	struct nvmefc_ls_req *fd;
@@ -129,4 +135,15 @@ struct pt_ls4_rx_unsol {
 	uint32_t desc_len;
 	uint32_t payload[3];
 };
+
+/*
+ * Global functions prototype in qla_nvme.c source file.
+ */
+void qla_nvme_register_hba(struct scsi_qla_host *);
+int  qla_nvme_register_remote(struct scsi_qla_host *, struct fc_port *);
+void qla_nvme_delete(struct scsi_qla_host *);
+void qla_nvme_abort(struct qla_hw_data *, struct srb *sp);
+void qla24xx_nvme_ls4_iocb(struct scsi_qla_host *, struct pt_ls4_request *,
+    struct req_que *);
+void qla24xx_async_gffid_sp_done(void *, int);
 #endif
diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c
index df57655779ed..5b2437a5ea44 100644
--- a/drivers/scsi/qla2xxx/qla_os.c
+++ b/drivers/scsi/qla2xxx/qla_os.c
@@ -113,12 +113,12 @@ MODULE_PARM_DESC(ql2xfdmienable,
 		"Enables FDMI registrations. "
 		"0 - no FDMI. Default is 1 - perform FDMI.");
 
-#define MAX_Q_DEPTH	32
+#define MAX_Q_DEPTH	64
 static int ql2xmaxqdepth = MAX_Q_DEPTH;
 module_param(ql2xmaxqdepth, int, S_IRUGO|S_IWUSR);
 MODULE_PARM_DESC(ql2xmaxqdepth,
 		"Maximum queue depth to set for each LUN. "
-		"Default is 32.");
+		"Default is 64.");
 
 #if (IS_ENABLED(CONFIG_NVME_FC))
 int ql2xenabledif;
@@ -200,7 +200,7 @@ MODULE_PARM_DESC(ql2xgffidenable,
 		"Enables GFF_ID checks of port type. "
 		"Default is 0 - Do not use GFF_ID information.");
 
-int ql2xasynctmfenable;
+int ql2xasynctmfenable = 1;
 module_param(ql2xasynctmfenable, int, S_IRUGO);
 MODULE_PARM_DESC(ql2xasynctmfenable,
 		"Enables issue of TM IOCBs asynchronously via IOCB mechanism"
@@ -262,6 +262,12 @@ MODULE_PARM_DESC(ql2xmvasynctoatio,
 		"0 (Default). Do not move IOCBs"
 		"1 - Move IOCBs.");
 
+int ql2xautodetectsfp = 1;
+module_param(ql2xautodetectsfp, int, 0444);
+MODULE_PARM_DESC(ql2xautodetectsfp,
+		 "Detect SFP range and set appropriate distance.\n"
+		 "1 (Default): Enable\n");
+
 /*
  * SCSI host template entry points
  */
@@ -379,6 +385,7 @@ static void qla_init_base_qpair(struct scsi_qla_host *vha, struct req_que *req,
 	ha->base_qpair->use_shadow_reg = IS_SHADOW_REG_CAPABLE(ha) ? 1 : 0;
 	ha->base_qpair->msix = &ha->msix_entries[QLA_MSIX_RSP_Q];
 	INIT_LIST_HEAD(&ha->base_qpair->hints_list);
+	INIT_LIST_HEAD(&ha->base_qpair->nvme_done_list);
 	ha->base_qpair->enable_class_2 = ql2xenableclass2;
 	/* init qpair to this cpu. Will adjust at run time. */
 	qla_cpu_update(rsp->qpair, smp_processor_id());
@@ -710,7 +717,7 @@ qla2x00_sp_free_dma(void *ptr)
 	}
 
 end:
-	if ((sp->type != SRB_NVME_CMD) && (sp->type != SRB_NVME_LS)) {
+	if (sp->type != SRB_NVME_CMD && sp->type != SRB_NVME_LS) {
 		CMD_SP(cmd) = NULL;
 		qla2x00_rel_sp(sp);
 	}
@@ -735,7 +742,7 @@ qla2x00_sp_compl(void *ptr, int res)
 	if (!atomic_dec_and_test(&sp->ref_count))
 		return;
 
-	qla2x00_sp_free_dma(sp);
+	sp->free(sp);
 	cmd->scsi_done(cmd);
 }
 
@@ -807,7 +814,7 @@ qla2xxx_qpair_sp_compl(void *ptr, int res)
 	if (!atomic_dec_and_test(&sp->ref_count))
 		return;
 
-	qla2xxx_qpair_sp_free_dma(sp);
+	sp->free(sp);
 	cmd->scsi_done(cmd);
 }
 
@@ -928,7 +935,7 @@ qla2xxx_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
 	return 0;
 
 qc24_host_busy_free_sp:
-	qla2x00_sp_free_dma(sp);
+	sp->free(sp);
 
 qc24_host_busy:
 	return SCSI_MLQUEUE_HOST_BUSY;
@@ -1017,7 +1024,7 @@ qla2xxx_mqueuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd,
 	return 0;
 
 qc24_host_busy_free_sp:
-	qla2xxx_qpair_sp_free_dma(sp);
+	sp->free(sp);
 
 qc24_host_busy:
 	return SCSI_MLQUEUE_HOST_BUSY;
@@ -1134,7 +1141,7 @@ qla2x00_wait_for_sess_deletion(scsi_qla_host_t *vha)
 {
 	qla2x00_mark_all_devices_lost(vha, 0);
 
-	wait_event(vha->fcport_waitQ, test_fcport_count(vha));
+	wait_event_timeout(vha->fcport_waitQ, test_fcport_count(vha), 10*HZ);
 }
 
 /*
@@ -1715,8 +1722,8 @@ qla2x00_abort_all_cmds(scsi_qla_host_t *vha, int res)
 			if (sp) {
 				req->outstanding_cmds[cnt] = NULL;
 				if (sp->cmd_type == TYPE_SRB) {
-					if ((sp->type == SRB_NVME_CMD) ||
-					    (sp->type == SRB_NVME_LS)) {
+					if (sp->type == SRB_NVME_CMD ||
+					    sp->type == SRB_NVME_LS) {
 						sp_get(sp);
 						spin_unlock_irqrestore(
 						    &ha->hardware_lock, flags);
@@ -1725,6 +1732,8 @@ qla2x00_abort_all_cmds(scsi_qla_host_t *vha, int res)
 						    &ha->hardware_lock, flags);
 					} else if (GET_CMD_SP(sp) &&
 					    !ha->flags.eeh_busy &&
+					    (!test_bit(ABORT_ISP_ACTIVE,
+						&vha->dpc_flags)) &&
 					    (sp->type == SRB_SCSI_CMD)) {
 						/*
 						 * Don't abort commands in
@@ -2751,6 +2760,7 @@ qla2x00_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	spin_lock_init(&ha->tgt.sess_lock);
 	spin_lock_init(&ha->tgt.atio_lock);
 
+	atomic_set(&ha->nvme_active_aen_cnt, 0);
 
 	/* Clear our data area */
 	ha->bars = bars;
@@ -3168,7 +3178,7 @@ qla2x00_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	if (ha->mqenable) {
 		bool mq = false;
 		bool startit = false;
-		ha->wq = alloc_workqueue("qla2xxx_wq", WQ_MEM_RECLAIM, 1);
+		ha->wq = alloc_workqueue("qla2xxx_wq", WQ_MEM_RECLAIM, 0);
 
 		if (QLA_TGT_MODE_ENABLED()) {
 			mq = true;
@@ -3328,6 +3338,13 @@ skip_dpc:
 	if (test_bit(UNLOADING, &base_vha->dpc_flags))
 		return -ENODEV;
 
+	if (ha->flags.detected_lr_sfp) {
+		ql_log(ql_log_info, base_vha, 0xffff,
+		    "Reset chip to pick up LR SFP setting\n");
+		set_bit(ISP_ABORT_NEEDED, &base_vha->dpc_flags);
+		qla2xxx_wake_dpc(base_vha);
+	}
+
 	return 0;
 
 probe_init_failed:
@@ -3383,12 +3400,22 @@ qla2x00_shutdown(struct pci_dev *pdev)
 	scsi_qla_host_t *vha;
 	struct qla_hw_data  *ha;
 
-	if (!atomic_read(&pdev->enable_cnt))
-		return;
-
 	vha = pci_get_drvdata(pdev);
 	ha = vha->hw;
 
+	ql_log(ql_log_info, vha, 0xfffa,
+		"Adapter shutdown\n");
+
+	/*
+	 * Prevent future board_disable and wait
+	 * until any pending board_disable has completed.
+	 */
+	set_bit(PFLG_DRIVER_REMOVING, &vha->pci_flags);
+	cancel_work_sync(&ha->board_disable);
+
+	if (!atomic_read(&pdev->enable_cnt))
+		return;
+
 	/* Notify ISPFX00 firmware */
 	if (IS_QLAFX00(ha))
 		qlafx00_driver_shutdown(vha, 20);
@@ -3419,8 +3446,9 @@ qla2x00_shutdown(struct pci_dev *pdev)
 
 	qla2x00_free_fw_dump(ha);
 
-	pci_disable_pcie_error_reporting(pdev);
 	pci_disable_device(pdev);
+	ql_log(ql_log_info, vha, 0xfffe,
+		"Adapter shutdown successfully.\n");
 }
 
 /* Deletes all the virtual ports for a given ha */
@@ -4006,8 +4034,18 @@ qla2x00_mem_alloc(struct qla_hw_data *ha, uint16_t req_len, uint16_t rsp_len,
 		    "loop_id_map=%p.\n", ha->loop_id_map);
 	}
 
+	ha->sfp_data = dma_alloc_coherent(&ha->pdev->dev,
+	    SFP_DEV_SIZE, &ha->sfp_data_dma, GFP_KERNEL);
+	if (!ha->sfp_data) {
+		ql_dbg_pci(ql_dbg_init, ha->pdev, 0x011b,
+		    "Unable to allocate memory for SFP read-data.\n");
+		goto fail_sfp_data;
+	}
+
 	return 0;
 
+fail_sfp_data:
+	kfree(ha->loop_id_map);
 fail_loop_id_map:
 	dma_pool_free(ha->s_dma_pool, ha->async_pd, ha->async_pd_dma);
 fail_async_pd:
@@ -4345,7 +4383,8 @@ qla2x00_mem_free(struct qla_hw_data *ha)
 		ha->ct_sns, ha->ct_sns_dma);
 
 	if (ha->sfp_data)
-		dma_pool_free(ha->s_dma_pool, ha->sfp_data, ha->sfp_data_dma);
+		dma_free_coherent(&ha->pdev->dev, SFP_DEV_SIZE, ha->sfp_data,
+		    ha->sfp_data_dma);
 
 	if (ha->ms_iocb)
 		dma_pool_free(ha->s_dma_pool, ha->ms_iocb, ha->ms_iocb_dma);
@@ -4638,9 +4677,10 @@ static
 void qla24xx_create_new_sess(struct scsi_qla_host *vha, struct qla_work_evt *e)
 {
 	unsigned long flags;
-	fc_port_t *fcport =  NULL;
+	fc_port_t *fcport =  NULL, *tfcp;
 	struct qlt_plogi_ack_t *pla =
 	    (struct qlt_plogi_ack_t *)e->u.new_sess.pla;
+	uint8_t free_fcport = 0;
 
 	spin_lock_irqsave(&vha->hw->tgt.sess_lock, flags);
 	fcport = qla2x00_find_fcport_by_wwpn(vha, e->u.new_sess.port_name, 1);
@@ -4655,6 +4695,7 @@ void qla24xx_create_new_sess(struct scsi_qla_host *vha, struct qla_work_evt *e)
 			pla->ref_count--;
 		}
 	} else {
+		spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
 		fcport = qla2x00_alloc_fcport(vha, GFP_KERNEL);
 		if (fcport) {
 			fcport->d_id = e->u.new_sess.id;
@@ -4664,6 +4705,29 @@ void qla24xx_create_new_sess(struct scsi_qla_host *vha, struct qla_work_evt *e)
 
 			memcpy(fcport->port_name, e->u.new_sess.port_name,
 			    WWN_SIZE);
+		} else {
+			ql_dbg(ql_dbg_disc, vha, 0xffff,
+				   "%s %8phC mem alloc fail.\n",
+				   __func__, e->u.new_sess.port_name);
+
+			if (pla)
+				kmem_cache_free(qla_tgt_plogi_cachep, pla);
+			return;
+		}
+
+		spin_lock_irqsave(&vha->hw->tgt.sess_lock, flags);
+		/* search again to make sure one else got ahead */
+		tfcp = qla2x00_find_fcport_by_wwpn(vha,
+		    e->u.new_sess.port_name, 1);
+		if (tfcp) {
+			/* should rarily happen */
+			ql_dbg(ql_dbg_disc, vha, 0xffff,
+			    "%s %8phC found existing fcport b4 add. DS %d LS %d\n",
+			    __func__, tfcp->port_name, tfcp->disc_state,
+			    tfcp->fw_login_state);
+
+			free_fcport = 1;
+		} else {
 			list_add_tail(&fcport->list, &vha->vp_fcports);
 
 			if (pla) {
@@ -4681,6 +4745,12 @@ void qla24xx_create_new_sess(struct scsi_qla_host *vha, struct qla_work_evt *e)
 		else
 			qla24xx_async_gnl(vha, fcport);
 	}
+
+	if (free_fcport) {
+		qla2x00_free_fcport(fcport);
+		if (pla)
+			kmem_cache_free(qla_tgt_plogi_cachep, pla);
+	}
 }
 
 void
@@ -5493,6 +5563,13 @@ qla2x00_disable_board_on_pci_error(struct work_struct *work)
 	ql_log(ql_log_warn, base_vha, 0x015b,
 	    "Disabling adapter.\n");
 
+	if (!atomic_read(&pdev->enable_cnt)) {
+		ql_log(ql_log_info, base_vha, 0xfffc,
+		    "PCI device disabled, no action req for PCI error=%lx\n",
+		    base_vha->pci_flags);
+		return;
+	}
+
 	qla2x00_wait_for_sess_deletion(base_vha);
 
 	set_bit(UNLOADING, &base_vha->dpc_flags);
@@ -5687,6 +5764,16 @@ qla2x00_do_dpc(void *data)
 			}
 		}
 
+		if (test_and_clear_bit(DETECT_SFP_CHANGE,
+			&base_vha->dpc_flags) &&
+		    !test_bit(ISP_ABORT_NEEDED, &base_vha->dpc_flags)) {
+			qla24xx_detect_sfp(base_vha);
+
+			if (ha->flags.detected_lr_sfp !=
+			    ha->flags.using_lr_setting)
+				set_bit(ISP_ABORT_NEEDED, &base_vha->dpc_flags);
+		}
+
 		if (test_and_clear_bit(ISP_ABORT_NEEDED,
 						&base_vha->dpc_flags)) {
 
@@ -5828,6 +5915,17 @@ intr_on_check:
 			mutex_unlock(&ha->mq_lock);
 		}
 
+		if (test_and_clear_bit(SET_ZIO_THRESHOLD_NEEDED, &base_vha->dpc_flags)) {
+			ql_log(ql_log_info, base_vha, 0xffffff,
+				"nvme: SET ZIO Activity exchange threshold to %d.\n",
+						ha->nvme_last_rptd_aen);
+			if (qla27xx_set_zio_threshold(base_vha, ha->nvme_last_rptd_aen)) {
+				ql_log(ql_log_info, base_vha, 0xffffff,
+					"nvme: Unable to SET ZIO Activity exchange threshold to %d.\n",
+						ha->nvme_last_rptd_aen);
+			}
+		}
+
 		if (!IS_QLAFX00(ha))
 			qla2x00_do_dpc_all_vps(base_vha);
 
@@ -6025,12 +6123,15 @@ qla2x00_timer(scsi_qla_host_t *vha)
 	 * FC-NVME
 	 * see if the active AEN count has changed from what was last reported.
 	 */
-	if (atomic_read(&vha->nvme_active_aen_cnt) != vha->nvme_last_rptd_aen) {
-		vha->nvme_last_rptd_aen =
-		    atomic_read(&vha->nvme_active_aen_cnt);
+	if (!vha->vp_idx &&
+		atomic_read(&ha->nvme_active_aen_cnt) != ha->nvme_last_rptd_aen &&
+		ha->zio_mode == QLA_ZIO_MODE_6) {
 		ql_log(ql_log_info, vha, 0x3002,
-		    "reporting new aen count of %d to the fw\n",
-		    vha->nvme_last_rptd_aen);
+			"nvme: Sched: Set ZIO exchange threshold to %d.\n",
+			ha->nvme_last_rptd_aen);
+		ha->nvme_last_rptd_aen = atomic_read(&ha->nvme_active_aen_cnt);
+		set_bit(SET_ZIO_THRESHOLD_NEEDED, &vha->dpc_flags);
+		start_dpc++;
 	}
 
 	/* Schedule the DPC routine if needed */
@@ -6181,6 +6282,12 @@ qla2xxx_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
 	ql_dbg(ql_dbg_aer, vha, 0x9000,
 	    "PCI error detected, state %x.\n", state);
 
+	if (!atomic_read(&pdev->enable_cnt)) {
+		ql_log(ql_log_info, vha, 0xffff,
+			"PCI device is disabled,state %x\n", state);
+		return PCI_ERS_RESULT_NEED_RESET;
+	}
+
 	switch (state) {
 	case pci_channel_io_normal:
 		ha->flags.eeh_busy = 0;
@@ -6574,6 +6681,8 @@ qla2x00_module_init(void)
 	strcpy(qla2x00_version_str, QLA2XXX_VERSION);
 	if (ql2xextended_error_logging)
 		strcat(qla2x00_version_str, "-debug");
+	if (ql2xextended_error_logging == 1)
+		ql2xextended_error_logging = QL_DBG_DEFAULT1_MASK;
 
 	qla2xxx_transport_template =
 	    fc_attach_transport(&qla2xxx_transport_functions);
diff --git a/drivers/scsi/qla2xxx/qla_target.c b/drivers/scsi/qla2xxx/qla_target.c
index e101cd3043b9..f05cfc83c9c8 100644
--- a/drivers/scsi/qla2xxx/qla_target.c
+++ b/drivers/scsi/qla2xxx/qla_target.c
@@ -145,7 +145,7 @@ static void qlt_send_busy(struct qla_qpair *, struct atio_from_isp *,
  * Global Variables
  */
 static struct kmem_cache *qla_tgt_mgmt_cmd_cachep;
-static struct kmem_cache *qla_tgt_plogi_cachep;
+struct kmem_cache *qla_tgt_plogi_cachep;
 static mempool_t *qla_tgt_mgmt_cmd_mempool;
 static struct workqueue_struct *qla_tgt_wq;
 static DEFINE_MUTEX(qla_tgt_mutex);
@@ -585,11 +585,13 @@ void qla2x00_async_nack_sp_done(void *s, int res)
 		sp->fcport->fw_login_state = DSC_LS_PLOGI_COMP;
 		sp->fcport->logout_on_delete = 1;
 		sp->fcport->plogi_nack_done_deadline = jiffies + HZ;
+		sp->fcport->send_els_logo = 0;
 		break;
 
 	case SRB_NACK_PRLI:
 		sp->fcport->fw_login_state = DSC_LS_PRLI_COMP;
 		sp->fcport->deleted = 0;
+		sp->fcport->send_els_logo = 0;
 
 		if (!sp->fcport->login_succ &&
 		    !IS_SW_RESV_ADDR(sp->fcport->d_id)) {
@@ -1479,7 +1481,7 @@ int qlt_stop_phase1(struct qla_tgt *tgt)
 	ql_dbg(ql_dbg_tgt_mgt, vha, 0xf00a,
 	    "Waiting for tgt %p: sess_count=%d\n", tgt, tgt->sess_count);
 
-	wait_event(tgt->waitQ, test_tgt_sess_count(tgt));
+	wait_event_timeout(tgt->waitQ, test_tgt_sess_count(tgt), 10*HZ);
 
 	/* Big hammer */
 	if (!ha->flags.host_shutting_down &&
@@ -1487,7 +1489,7 @@ int qlt_stop_phase1(struct qla_tgt *tgt)
 		qlt_disable_vha(vha);
 
 	/* Wait for sessions to clear out (just in case) */
-	wait_event(tgt->waitQ, test_tgt_sess_count(tgt));
+	wait_event_timeout(tgt->waitQ, test_tgt_sess_count(tgt), 10*HZ);
 	return 0;
 }
 EXPORT_SYMBOL(qlt_stop_phase1);
@@ -1528,6 +1530,7 @@ static void qlt_release(struct qla_tgt *tgt)
 	u64 key = 0;
 	u16 i;
 	struct qla_qpair_hint *h;
+	struct qla_hw_data *ha = vha->hw;
 
 	if ((vha->vha_tgt.qla_tgt != NULL) && !tgt->tgt_stop &&
 	    !tgt->tgt_stopped)
@@ -1548,12 +1551,18 @@ static void qlt_release(struct qla_tgt *tgt)
 		}
 	}
 	kfree(tgt->qphints);
+	mutex_lock(&qla_tgt_mutex);
+	list_del(&vha->vha_tgt.qla_tgt->tgt_list_entry);
+	mutex_unlock(&qla_tgt_mutex);
 
 	btree_for_each_safe64(&tgt->lun_qpair_map, key, node)
 		btree_remove64(&tgt->lun_qpair_map, key);
 
 	btree_destroy64(&tgt->lun_qpair_map);
 
+	if (ha->tgt.tgt_ops && ha->tgt.tgt_ops->remove_target)
+		ha->tgt.tgt_ops->remove_target(vha);
+
 	vha->vha_tgt.qla_tgt = NULL;
 
 	ql_dbg(ql_dbg_tgt_mgt, vha, 0xf00d,
@@ -1901,6 +1910,7 @@ static int __qlt_24xx_handle_abts(struct scsi_qla_host *vha,
 	mcmd->reset_count = ha->base_qpair->chip_reset;
 	mcmd->tmr_func = QLA_TGT_ABTS;
 	mcmd->qpair = ha->base_qpair;
+	mcmd->vha = vha;
 
 	/*
 	 * LUN is looked up by target-core internally based on the passed
@@ -2003,7 +2013,7 @@ static void qlt_24xx_handle_abts(struct scsi_qla_host *vha,
 static void qlt_24xx_send_task_mgmt_ctio(struct qla_qpair *qpair,
 	struct qla_tgt_mgmt_cmd *mcmd, uint32_t resp_code)
 {
-	struct scsi_qla_host *ha = qpair->vha;
+	struct scsi_qla_host *ha = mcmd->vha;
 	struct atio_from_isp *atio = &mcmd->orig_iocb.atio;
 	struct ctio7_to_24xx *ctio;
 	uint16_t temp;
@@ -3464,6 +3474,9 @@ static int __qlt_send_term_exchange(struct qla_qpair *qpair,
 
 	ql_dbg(ql_dbg_tgt, vha, 0xe009, "Sending TERM EXCH CTIO (ha=%p)\n", ha);
 
+	if (cmd)
+		vha = cmd->vha;
+
 	pkt = (request_t *)qla2x00_alloc_iocbs_ready(qpair, NULL);
 	if (pkt == NULL) {
 		ql_dbg(ql_dbg_tgt, vha, 0xe050,
@@ -4379,6 +4392,7 @@ static int qlt_issue_task_mgmt(struct fc_port *sess, u64 lun,
 	mcmd->flags = flags;
 	mcmd->reset_count = ha->base_qpair->chip_reset;
 	mcmd->qpair = ha->base_qpair;
+	mcmd->vha = vha;
 
 	switch (fn) {
 	case QLA_TGT_LUN_RESET:
@@ -6170,10 +6184,6 @@ int qlt_remove_target(struct qla_hw_data *ha, struct scsi_qla_host *vha)
 	/* free left over qfull cmds */
 	qlt_init_term_exchange(vha);
 
-	mutex_lock(&qla_tgt_mutex);
-	list_del(&vha->vha_tgt.qla_tgt->tgt_list_entry);
-	mutex_unlock(&qla_tgt_mutex);
-
 	ql_dbg(ql_dbg_tgt, vha, 0xe03c, "Unregistering target for host %ld(%p)",
 	    vha->host_no, ha);
 	qlt_release(vha->vha_tgt.qla_tgt);
@@ -6530,7 +6540,6 @@ qlt_24xx_process_atio_queue(struct scsi_qla_host *vha, uint8_t ha_locked)
 
 	/* Adjust ring index */
 	WRT_REG_DWORD(ISP_ATIO_Q_OUT(vha), ha->tgt.atio_ring_index);
-	RD_REG_DWORD_RELAXED(ISP_ATIO_Q_OUT(vha));
 }
 
 void
@@ -6796,7 +6805,7 @@ qlt_probe_one_stage1(struct scsi_qla_host *base_vha, struct qla_hw_data *ha)
 	if (!QLA_TGT_MODE_ENABLED())
 		return;
 
-	if  (ha->mqenable || IS_QLA83XX(ha) || IS_QLA27XX(ha)) {
+	if  (IS_QLA83XX(ha) || IS_QLA27XX(ha)) {
 		ISP_ATIO_Q_IN(base_vha) = &ha->mqiobase->isp25mq.atio_q_in;
 		ISP_ATIO_Q_OUT(base_vha) = &ha->mqiobase->isp25mq.atio_q_out;
 	} else {
diff --git a/drivers/scsi/qla2xxx/qla_target.h b/drivers/scsi/qla2xxx/qla_target.h
index 7fe02d036bdf..aba58d3848a6 100644
--- a/drivers/scsi/qla2xxx/qla_target.h
+++ b/drivers/scsi/qla2xxx/qla_target.h
@@ -705,6 +705,7 @@ struct qla_tgt_func_tmpl {
 	int (*get_dif_tags)(struct qla_tgt_cmd *cmd, uint16_t *pfw_prot_opts);
 	int (*chk_dif_tags)(uint32_t tag);
 	void (*add_target)(struct scsi_qla_host *);
+	void (*remove_target)(struct scsi_qla_host *);
 };
 
 int qla2x00_wait_for_hba_online(struct scsi_qla_host *);
@@ -959,6 +960,7 @@ struct qla_tgt_mgmt_cmd {
 	uint8_t fc_tm_rsp;
 	struct fc_port *sess;
 	struct qla_qpair *qpair;
+	struct scsi_qla_host *vha;
 	struct se_cmd se_cmd;
 	struct work_struct free_work;
 	unsigned int flags;
diff --git a/drivers/scsi/qla2xxx/qla_tmpl.c b/drivers/scsi/qla2xxx/qla_tmpl.c
index 33142610882f..733e8dcccf5c 100644
--- a/drivers/scsi/qla2xxx/qla_tmpl.c
+++ b/drivers/scsi/qla2xxx/qla_tmpl.c
@@ -401,9 +401,6 @@ qla27xx_fwdt_entry_t263(struct scsi_qla_host *vha,
 		for (i = 0; i < vha->hw->max_req_queues; i++) {
 			struct req_que *req = vha->hw->req_q_map[i];
 
-			if (!test_bit(i, vha->hw->req_qid_map))
-				continue;
-
 			if (req || !buf) {
 				length = req ?
 				    req->length : REQUEST_ENTRY_CNT_24XX;
@@ -418,9 +415,6 @@ qla27xx_fwdt_entry_t263(struct scsi_qla_host *vha,
 		for (i = 0; i < vha->hw->max_rsp_queues; i++) {
 			struct rsp_que *rsp = vha->hw->rsp_q_map[i];
 
-			if (!test_bit(i, vha->hw->rsp_qid_map))
-				continue;
-
 			if (rsp || !buf) {
 				length = rsp ?
 				    rsp->length : RESPONSE_ENTRY_CNT_MQ;
@@ -449,8 +443,12 @@ qla27xx_fwdt_entry_t263(struct scsi_qla_host *vha,
 		qla27xx_skip_entry(ent, buf);
 	}
 
-	if (buf)
-		ent->t263.num_queues = count;
+	if (buf) {
+		if (count)
+			ent->t263.num_queues = count;
+		else
+			qla27xx_skip_entry(ent, buf);
+	}
 
 	return false;
 }
@@ -660,9 +658,6 @@ qla27xx_fwdt_entry_t274(struct scsi_qla_host *vha,
 		for (i = 0; i < vha->hw->max_req_queues; i++) {
 			struct req_que *req = vha->hw->req_q_map[i];
 
-			if (!test_bit(i, vha->hw->req_qid_map))
-				continue;
-
 			if (req || !buf) {
 				qla27xx_insert16(i, buf, len);
 				qla27xx_insert16(1, buf, len);
@@ -675,9 +670,6 @@ qla27xx_fwdt_entry_t274(struct scsi_qla_host *vha,
 		for (i = 0; i < vha->hw->max_rsp_queues; i++) {
 			struct rsp_que *rsp = vha->hw->rsp_q_map[i];
 
-			if (!test_bit(i, vha->hw->rsp_qid_map))
-				continue;
-
 			if (rsp || !buf) {
 				qla27xx_insert16(i, buf, len);
 				qla27xx_insert16(1, buf, len);
@@ -704,11 +696,12 @@ qla27xx_fwdt_entry_t274(struct scsi_qla_host *vha,
 		qla27xx_skip_entry(ent, buf);
 	}
 
-	if (buf)
-		ent->t274.num_queues = count;
-
-	if (!count)
-		qla27xx_skip_entry(ent, buf);
+	if (buf) {
+		if (count)
+			ent->t274.num_queues = count;
+		else
+			qla27xx_skip_entry(ent, buf);
+	}
 
 	return false;
 }
diff --git a/drivers/scsi/qla2xxx/qla_version.h b/drivers/scsi/qla2xxx/qla_version.h
index 005a378f7fab..8c4b505c9f66 100644
--- a/drivers/scsi/qla2xxx/qla_version.h
+++ b/drivers/scsi/qla2xxx/qla_version.h
@@ -7,7 +7,7 @@
 /*
  * Driver version
  */
-#define QLA2XXX_VERSION      "10.00.00.00-k"
+#define QLA2XXX_VERSION      "10.00.00.01-k"
 
 #define QLA_DRIVER_MAJOR_VER	10
 #define QLA_DRIVER_MINOR_VER	0
diff --git a/drivers/scsi/qlogicfas.c b/drivers/scsi/qlogicfas.c
index 840823b99e51..95431d605c24 100644
--- a/drivers/scsi/qlogicfas.c
+++ b/drivers/scsi/qlogicfas.c
@@ -188,7 +188,7 @@ static struct scsi_host_template qlogicfas_driver_template = {
 	.info			= qlogicfas408_info,
 	.queuecommand		= qlogicfas408_queuecommand,
 	.eh_abort_handler	= qlogicfas408_abort,
-	.eh_bus_reset_handler	= qlogicfas408_bus_reset,
+	.eh_host_reset_handler	= qlogicfas408_host_reset,
 	.bios_param		= qlogicfas408_biosparam,
 	.can_queue		= 1,
 	.this_id		= -1,
diff --git a/drivers/scsi/qlogicfas408.c b/drivers/scsi/qlogicfas408.c
index c3a9151ca823..8b471a925b43 100644
--- a/drivers/scsi/qlogicfas408.c
+++ b/drivers/scsi/qlogicfas408.c
@@ -496,13 +496,13 @@ int qlogicfas408_abort(struct scsi_cmnd *cmd)
 	return SUCCESS;
 }
 
-/* 
+/*
  *	Reset SCSI bus
  *	FIXME: This function is invoked with cmd = NULL directly by
  *	the PCMCIA qlogic_stub code. This wants fixing
  */
 
-int qlogicfas408_bus_reset(struct scsi_cmnd *cmd)
+int qlogicfas408_host_reset(struct scsi_cmnd *cmd)
 {
 	struct qlogicfas408_priv *priv = get_priv_by_cmd(cmd);
 	unsigned long flags;
@@ -607,7 +607,7 @@ module_exit(qlogicfas408_exit);
 EXPORT_SYMBOL(qlogicfas408_info);
 EXPORT_SYMBOL(qlogicfas408_queuecommand);
 EXPORT_SYMBOL(qlogicfas408_abort);
-EXPORT_SYMBOL(qlogicfas408_bus_reset);
+EXPORT_SYMBOL(qlogicfas408_host_reset);
 EXPORT_SYMBOL(qlogicfas408_biosparam);
 EXPORT_SYMBOL(qlogicfas408_ihandl);
 EXPORT_SYMBOL(qlogicfas408_get_chip_type);
diff --git a/drivers/scsi/qlogicfas408.h b/drivers/scsi/qlogicfas408.h
index 2f6c0a166200..f6b1216af79f 100644
--- a/drivers/scsi/qlogicfas408.h
+++ b/drivers/scsi/qlogicfas408.h
@@ -108,7 +108,7 @@ int qlogicfas408_biosparam(struct scsi_device * disk,
 			   struct block_device *dev,
 			   sector_t capacity, int ip[]);
 int qlogicfas408_abort(struct scsi_cmnd * cmd);
-int qlogicfas408_bus_reset(struct scsi_cmnd * cmd);
+extern int qlogicfas408_host_reset(struct scsi_cmnd *cmd);
 const char *qlogicfas408_info(struct Scsi_Host *host);
 int qlogicfas408_get_chip_type(int qbase, int int_type);
 void qlogicfas408_setup(int qbase, int id, int int_type);
diff --git a/drivers/scsi/qlogicpti.c b/drivers/scsi/qlogicpti.c
index 69bfc0a1aea3..cec9a14982e6 100644
--- a/drivers/scsi/qlogicpti.c
+++ b/drivers/scsi/qlogicpti.c
@@ -1283,7 +1283,7 @@ static struct scsi_host_template qpti_template = {
 	.queuecommand		= qlogicpti_queuecommand,
 	.slave_configure	= qlogicpti_slave_configure,
 	.eh_abort_handler	= qlogicpti_abort,
-	.eh_bus_reset_handler	= qlogicpti_reset,
+	.eh_host_reset_handler	= qlogicpti_reset,
 	.can_queue		= QLOGICPTI_REQ_QUEUE_LEN,
 	.this_id		= 7,
 	.sg_tablesize		= QLOGICPTI_MAX_SG(QLOGICPTI_REQ_QUEUE_LEN),
diff --git a/drivers/scsi/scsi.c b/drivers/scsi/scsi.c
index 3d38c6d463b8..a7e4fba724b7 100644
--- a/drivers/scsi/scsi.c
+++ b/drivers/scsi/scsi.c
@@ -412,6 +412,57 @@ int scsi_get_vpd_page(struct scsi_device *sdev, u8 page, unsigned char *buf,
 EXPORT_SYMBOL_GPL(scsi_get_vpd_page);
 
 /**
+ * scsi_get_vpd_buf - Get Vital Product Data from a SCSI device
+ * @sdev: The device to ask
+ * @page: Which Vital Product Data to return
+ *
+ * Returns %NULL upon failure.
+ */
+static struct scsi_vpd *scsi_get_vpd_buf(struct scsi_device *sdev, u8 page)
+{
+	struct scsi_vpd *vpd_buf;
+	int vpd_len = SCSI_VPD_PG_LEN, result;
+
+retry_pg:
+	vpd_buf = kmalloc(sizeof(*vpd_buf) + vpd_len, GFP_KERNEL);
+	if (!vpd_buf)
+		return NULL;
+
+	result = scsi_vpd_inquiry(sdev, vpd_buf->data, page, vpd_len);
+	if (result < 0) {
+		kfree(vpd_buf);
+		return NULL;
+	}
+	if (result > vpd_len) {
+		vpd_len = result;
+		kfree(vpd_buf);
+		goto retry_pg;
+	}
+
+	vpd_buf->len = result;
+
+	return vpd_buf;
+}
+
+static void scsi_update_vpd_page(struct scsi_device *sdev, u8 page,
+				 struct scsi_vpd __rcu **sdev_vpd_buf)
+{
+	struct scsi_vpd *vpd_buf;
+
+	vpd_buf = scsi_get_vpd_buf(sdev, page);
+	if (!vpd_buf)
+		return;
+
+	mutex_lock(&sdev->inquiry_mutex);
+	rcu_swap_protected(*sdev_vpd_buf, vpd_buf,
+			   lockdep_is_held(&sdev->inquiry_mutex));
+	mutex_unlock(&sdev->inquiry_mutex);
+
+	if (vpd_buf)
+		kfree_rcu(vpd_buf, rcu);
+}
+
+/**
  * scsi_attach_vpd - Attach Vital Product Data to a SCSI device structure
  * @sdev: The device to ask
  *
@@ -422,95 +473,24 @@ EXPORT_SYMBOL_GPL(scsi_get_vpd_page);
  */
 void scsi_attach_vpd(struct scsi_device *sdev)
 {
-	int result, i;
-	int vpd_len = SCSI_VPD_PG_LEN;
-	int pg80_supported = 0;
-	int pg83_supported = 0;
-	unsigned char __rcu *vpd_buf, *orig_vpd_buf = NULL;
+	int i;
+	struct scsi_vpd *vpd_buf;
 
 	if (!scsi_device_supports_vpd(sdev))
 		return;
 
-retry_pg0:
-	vpd_buf = kmalloc(vpd_len, GFP_KERNEL);
-	if (!vpd_buf)
-		return;
-
 	/* Ask for all the pages supported by this device */
-	result = scsi_vpd_inquiry(sdev, vpd_buf, 0, vpd_len);
-	if (result < 0) {
-		kfree(vpd_buf);
+	vpd_buf = scsi_get_vpd_buf(sdev, 0);
+	if (!vpd_buf)
 		return;
-	}
-	if (result > vpd_len) {
-		vpd_len = result;
-		kfree(vpd_buf);
-		goto retry_pg0;
-	}
 
-	for (i = 4; i < result; i++) {
-		if (vpd_buf[i] == 0x80)
-			pg80_supported = 1;
-		if (vpd_buf[i] == 0x83)
-			pg83_supported = 1;
+	for (i = 4; i < vpd_buf->len; i++) {
+		if (vpd_buf->data[i] == 0x80)
+			scsi_update_vpd_page(sdev, 0x80, &sdev->vpd_pg80);
+		if (vpd_buf->data[i] == 0x83)
+			scsi_update_vpd_page(sdev, 0x83, &sdev->vpd_pg83);
 	}
 	kfree(vpd_buf);
-	vpd_len = SCSI_VPD_PG_LEN;
-
-	if (pg80_supported) {
-retry_pg80:
-		vpd_buf = kmalloc(vpd_len, GFP_KERNEL);
-		if (!vpd_buf)
-			return;
-
-		result = scsi_vpd_inquiry(sdev, vpd_buf, 0x80, vpd_len);
-		if (result < 0) {
-			kfree(vpd_buf);
-			return;
-		}
-		if (result > vpd_len) {
-			vpd_len = result;
-			kfree(vpd_buf);
-			goto retry_pg80;
-		}
-		mutex_lock(&sdev->inquiry_mutex);
-		orig_vpd_buf = sdev->vpd_pg80;
-		sdev->vpd_pg80_len = result;
-		rcu_assign_pointer(sdev->vpd_pg80, vpd_buf);
-		mutex_unlock(&sdev->inquiry_mutex);
-		synchronize_rcu();
-		if (orig_vpd_buf) {
-			kfree(orig_vpd_buf);
-			orig_vpd_buf = NULL;
-		}
-		vpd_len = SCSI_VPD_PG_LEN;
-	}
-
-	if (pg83_supported) {
-retry_pg83:
-		vpd_buf = kmalloc(vpd_len, GFP_KERNEL);
-		if (!vpd_buf)
-			return;
-
-		result = scsi_vpd_inquiry(sdev, vpd_buf, 0x83, vpd_len);
-		if (result < 0) {
-			kfree(vpd_buf);
-			return;
-		}
-		if (result > vpd_len) {
-			vpd_len = result;
-			kfree(vpd_buf);
-			goto retry_pg83;
-		}
-		mutex_lock(&sdev->inquiry_mutex);
-		orig_vpd_buf = sdev->vpd_pg83;
-		sdev->vpd_pg83_len = result;
-		rcu_assign_pointer(sdev->vpd_pg83, vpd_buf);
-		mutex_unlock(&sdev->inquiry_mutex);
-		synchronize_rcu();
-		if (orig_vpd_buf)
-			kfree(orig_vpd_buf);
-	}
 }
 
 /**
@@ -800,7 +780,11 @@ MODULE_LICENSE("GPL");
 module_param(scsi_logging_level, int, S_IRUGO|S_IWUSR);
 MODULE_PARM_DESC(scsi_logging_level, "a bit mask of logging levels");
 
+#ifdef CONFIG_SCSI_MQ_DEFAULT
 bool scsi_use_blk_mq = true;
+#else
+bool scsi_use_blk_mq = false;
+#endif
 module_param_named(use_blk_mq, scsi_use_blk_mq, bool, S_IWUSR | S_IRUGO);
 
 static int __init init_scsi(void)
diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c
index 3be980d47268..09ba494f8896 100644
--- a/drivers/scsi/scsi_debug.c
+++ b/drivers/scsi/scsi_debug.c
@@ -2261,7 +2261,7 @@ static int resp_ie_l_pg(unsigned char * arr)
 static int resp_log_sense(struct scsi_cmnd * scp,
                           struct sdebug_dev_info * devip)
 {
-	int ppc, sp, pcontrol, pcode, subpcode, alloc_len, len, n;
+	int ppc, sp, pcode, subpcode, alloc_len, len, n;
 	unsigned char arr[SDEBUG_MAX_LSENSE_SZ];
 	unsigned char *cmd = scp->cmnd;
 
@@ -2272,7 +2272,6 @@ static int resp_log_sense(struct scsi_cmnd * scp,
 		mk_sense_invalid_fld(scp, SDEB_IN_CDB, 1, ppc ? 1 : 0);
 		return check_condition_result;
 	}
-	pcontrol = (cmd[2] & 0xc0) >> 6;
 	pcode = cmd[2] & 0x3f;
 	subpcode = cmd[3] & 0xff;
 	alloc_len = get_unaligned_be16(cmd + 7);
@@ -5466,7 +5465,7 @@ static int sdebug_driver_probe(struct device * dev)
 		return error;
 	}
 	if (submit_queues > nr_cpu_ids) {
-		pr_warn("%s: trim submit_queues (was %d) to nr_cpu_ids=%d\n",
+		pr_warn("%s: trim submit_queues (was %d) to nr_cpu_ids=%u\n",
 			my_name, submit_queues, nr_cpu_ids);
 		submit_queues = nr_cpu_ids;
 	}
diff --git a/drivers/scsi/scsi_debugfs.c b/drivers/scsi/scsi_debugfs.c
index a97c9507103d..5e9755008aed 100644
--- a/drivers/scsi/scsi_debugfs.c
+++ b/drivers/scsi/scsi_debugfs.c
@@ -6,8 +6,10 @@
 void scsi_show_rq(struct seq_file *m, struct request *rq)
 {
 	struct scsi_cmnd *cmd = container_of(scsi_req(rq), typeof(*cmd), req);
+	int msecs = jiffies_to_msecs(jiffies - cmd->jiffies_at_alloc);
 	char buf[80];
 
 	__scsi_format_command(buf, sizeof(buf), cmd->cmnd, cmd->cmd_len);
-	seq_printf(m, ", .cmd=%s", buf);
+	seq_printf(m, ", .cmd=%s, .retries=%d, allocated %d.%03d s ago", buf,
+		   cmd->retries, msecs / 1000, msecs % 1000);
 }
diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c
index ea9f40e51f68..38942050b265 100644
--- a/drivers/scsi/scsi_error.c
+++ b/drivers/scsi/scsi_error.c
@@ -259,7 +259,7 @@ void scsi_eh_scmd_add(struct scsi_cmnd *scmd)
  */
 enum blk_eh_timer_return scsi_times_out(struct request *req)
 {
-	struct scsi_cmnd *scmd = req->special;
+	struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(req);
 	enum blk_eh_timer_return rtn = BLK_EH_NOT_HANDLED;
 	struct Scsi_Host *host = scmd->device->host;
 
@@ -552,6 +552,7 @@ int scsi_check_sense(struct scsi_cmnd *scmd)
 			set_host_byte(scmd, DID_ALLOC_FAILURE);
 			return SUCCESS;
 		}
+		/* FALLTHROUGH */
 	case COPY_ABORTED:
 	case VOLUME_OVERFLOW:
 	case MISCOMPARE:
@@ -573,6 +574,7 @@ int scsi_check_sense(struct scsi_cmnd *scmd)
 			return ADD_TO_MLQUEUE;
 		else
 			set_host_byte(scmd, DID_TARGET_FAILURE);
+		/* FALLTHROUGH */
 
 	case ILLEGAL_REQUEST:
 		if (sshdr.asc == 0x20 || /* Invalid command operation code */
@@ -683,6 +685,7 @@ static int scsi_eh_completed_normally(struct scsi_cmnd *scmd)
 	switch (status_byte(scmd->result)) {
 	case GOOD:
 		scsi_handle_queue_ramp_up(scmd->device);
+		/* FALLTHROUGH */
 	case COMMAND_TERMINATED:
 		return SUCCESS;
 	case CHECK_CONDITION:
@@ -1734,6 +1737,7 @@ int scsi_decide_disposition(struct scsi_cmnd *scmd)
 			set_host_byte(scmd, DID_TIME_OUT);
 			return SUCCESS;
 		}
+		/* FALLTHROUGH */
 	case DID_NO_CONNECT:
 	case DID_BAD_TARGET:
 		/*
@@ -1819,6 +1823,7 @@ int scsi_decide_disposition(struct scsi_cmnd *scmd)
 		 * the case of trying to send too many commands to a
 		 * tagged queueing device.
 		 */
+		/* FALLTHROUGH */
 	case BUSY:
 		/*
 		 * device can't talk to us at the moment.  Should only
@@ -1831,6 +1836,7 @@ int scsi_decide_disposition(struct scsi_cmnd *scmd)
 		if (scmd->cmnd[0] == REPORT_LUNS)
 			scmd->device->sdev_target->expecting_lun_change = 0;
 		scsi_handle_queue_ramp_up(scmd->device);
+		/* FALLTHROUGH */
 	case COMMAND_TERMINATED:
 		return SUCCESS;
 	case TASK_ABORTED:
@@ -2320,8 +2326,8 @@ scsi_ioctl_reset(struct scsi_device *dev, int __user *arg)
 		rtn = scsi_try_host_reset(scmd);
 		if (rtn == SUCCESS)
 			break;
-	default:
 		/* FALLTHROUGH */
+	default:
 		rtn = FAILED;
 		break;
 	}
diff --git a/drivers/scsi/scsi_ioctl.c b/drivers/scsi/scsi_ioctl.c
index b6bf3f29a12a..0a875491f5a7 100644
--- a/drivers/scsi/scsi_ioctl.c
+++ b/drivers/scsi/scsi_ioctl.c
@@ -116,13 +116,15 @@ static int ioctl_internal_command(struct scsi_device *sdev, char *cmd,
 		case NOT_READY:	/* This happens if there is no disc in drive */
 			if (sdev->removable)
 				break;
+			/* FALLTHROUGH */
 		case UNIT_ATTENTION:
 			if (sdev->removable) {
 				sdev->changed = 1;
 				result = 0;	/* This is no longer considered an error */
 				break;
 			}
-		default:	/* Fall through for non-removable media */
+			/* FALLTHROUGH -- for non-removable media */
+		default:
 			sdev_printk(KERN_INFO, sdev,
 				    "ioctl_internal_command return code = %x\n",
 				    result);
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index f6097b89d5d3..9cf6a80fe297 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -44,6 +44,8 @@ static struct kmem_cache *scsi_sense_cache;
 static struct kmem_cache *scsi_sense_isadma_cache;
 static DEFINE_MUTEX(scsi_sense_cache_mutex);
 
+static void scsi_mq_uninit_cmd(struct scsi_cmnd *cmd);
+
 static inline struct kmem_cache *
 scsi_select_sense_cache(bool unchecked_isa_dma)
 {
@@ -140,6 +142,12 @@ static void scsi_mq_requeue_cmd(struct scsi_cmnd *cmd)
 {
 	struct scsi_device *sdev = cmd->device;
 
+	if (cmd->request->rq_flags & RQF_DONTPREP) {
+		cmd->request->rq_flags &= ~RQF_DONTPREP;
+		scsi_mq_uninit_cmd(cmd);
+	} else {
+		WARN_ON_ONCE(true);
+	}
 	blk_mq_requeue_request(cmd->request, true);
 	put_device(&sdev->sdev_gendev);
 }
@@ -627,7 +635,7 @@ static void scsi_release_bidi_buffers(struct scsi_cmnd *cmd)
 static bool scsi_end_request(struct request *req, blk_status_t error,
 		unsigned int bytes, unsigned int bidi_bytes)
 {
-	struct scsi_cmnd *cmd = req->special;
+	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
 	struct scsi_device *sdev = cmd->device;
 	struct request_queue *q = sdev->request_queue;
 
@@ -642,6 +650,11 @@ static bool scsi_end_request(struct request *req, blk_status_t error,
 	if (blk_queue_add_random(q))
 		add_disk_randomness(req->rq_disk);
 
+	if (!blk_rq_is_scsi(req)) {
+		WARN_ON_ONCE(!(cmd->flags & SCMD_INITIALIZED));
+		cmd->flags &= ~SCMD_INITIALIZED;
+	}
+
 	if (req->mq_ctx) {
 		/*
 		 * In the MQ case the command gets freed by __blk_mq_end_request,
@@ -977,8 +990,6 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
 		 * A new command will be prepared and issued.
 		 */
 		if (q->mq_ops) {
-			cmd->request->rq_flags &= ~RQF_DONTPREP;
-			scsi_mq_uninit_cmd(cmd);
 			scsi_mq_requeue_cmd(cmd);
 		} else {
 			scsi_release_buffers(cmd);
@@ -1107,15 +1118,23 @@ err_exit:
 EXPORT_SYMBOL(scsi_init_io);
 
 /**
- * scsi_initialize_rq - initialize struct scsi_cmnd.req
+ * scsi_initialize_rq - initialize struct scsi_cmnd partially
+ * @rq: Request associated with the SCSI command to be initialized.
+ *
+ * This function initializes the members of struct scsi_cmnd that must be
+ * initialized before request processing starts and that won't be
+ * reinitialized if a SCSI command is requeued.
  *
- * Called from inside blk_get_request().
+ * Called from inside blk_get_request() for pass-through requests and from
+ * inside scsi_init_command() for filesystem requests.
  */
 void scsi_initialize_rq(struct request *rq)
 {
 	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
 
 	scsi_req_init(&cmd->req);
+	cmd->jiffies_at_alloc = jiffies;
+	cmd->retries = 0;
 }
 EXPORT_SYMBOL(scsi_initialize_rq);
 
@@ -1153,8 +1172,18 @@ void scsi_init_command(struct scsi_device *dev, struct scsi_cmnd *cmd)
 {
 	void *buf = cmd->sense_buffer;
 	void *prot = cmd->prot_sdb;
-	unsigned int unchecked_isa_dma = cmd->flags & SCMD_UNCHECKED_ISA_DMA;
+	struct request *rq = blk_mq_rq_from_pdu(cmd);
+	unsigned int flags = cmd->flags & SCMD_PRESERVED_FLAGS;
+	unsigned long jiffies_at_alloc;
+	int retries;
+
+	if (!blk_rq_is_scsi(rq) && !(flags & SCMD_INITIALIZED)) {
+		flags |= SCMD_INITIALIZED;
+		scsi_initialize_rq(rq);
+	}
 
+	jiffies_at_alloc = cmd->jiffies_at_alloc;
+	retries = cmd->retries;
 	/* zero out the cmd, except for the embedded scsi_request */
 	memset((char *)cmd + sizeof(cmd->req), 0,
 		sizeof(*cmd) - sizeof(cmd->req) + dev->host->hostt->cmd_size);
@@ -1162,16 +1191,17 @@ void scsi_init_command(struct scsi_device *dev, struct scsi_cmnd *cmd)
 	cmd->device = dev;
 	cmd->sense_buffer = buf;
 	cmd->prot_sdb = prot;
-	cmd->flags = unchecked_isa_dma;
+	cmd->flags = flags;
 	INIT_DELAYED_WORK(&cmd->abort_work, scmd_eh_abort_handler);
-	cmd->jiffies_at_alloc = jiffies;
+	cmd->jiffies_at_alloc = jiffies_at_alloc;
+	cmd->retries = retries;
 
 	scsi_add_cmd_to_list(cmd);
 }
 
 static int scsi_setup_scsi_cmnd(struct scsi_device *sdev, struct request *req)
 {
-	struct scsi_cmnd *cmd = req->special;
+	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
 
 	/*
 	 * Passthrough requests may transfer data, in which case they must
@@ -1202,7 +1232,7 @@ static int scsi_setup_scsi_cmnd(struct scsi_device *sdev, struct request *req)
  */
 static int scsi_setup_fs_cmnd(struct scsi_device *sdev, struct request *req)
 {
-	struct scsi_cmnd *cmd = req->special;
+	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
 
 	if (unlikely(sdev->handler && sdev->handler->prep_fn)) {
 		int ret = sdev->handler->prep_fn(sdev, req);
@@ -1217,7 +1247,7 @@ static int scsi_setup_fs_cmnd(struct scsi_device *sdev, struct request *req)
 
 static int scsi_setup_cmnd(struct scsi_device *sdev, struct request *req)
 {
-	struct scsi_cmnd *cmd = req->special;
+	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
 
 	if (!blk_rq_bytes(req))
 		cmd->sc_data_direction = DMA_NONE;
@@ -1349,12 +1379,14 @@ static int scsi_prep_fn(struct request_queue *q, struct request *req)
 
 	ret = scsi_setup_cmnd(sdev, req);
 out:
+	if (ret != BLKPREP_OK)
+		cmd->flags &= ~SCMD_INITIALIZED;
 	return scsi_prep_return(q, req, ret);
 }
 
 static void scsi_unprep_fn(struct request_queue *q, struct request *req)
 {
-	scsi_uninit_cmd(req->special);
+	scsi_uninit_cmd(blk_mq_rq_to_pdu(req));
 }
 
 /*
@@ -1545,7 +1577,7 @@ static int scsi_lld_busy(struct request_queue *q)
  */
 static void scsi_kill_request(struct request *req, struct request_queue *q)
 {
-	struct scsi_cmnd *cmd = req->special;
+	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
 	struct scsi_device *sdev;
 	struct scsi_target *starget;
 	struct Scsi_Host *shost;
@@ -1576,7 +1608,7 @@ static void scsi_kill_request(struct request *req, struct request_queue *q)
 
 static void scsi_softirq_done(struct request *rq)
 {
-	struct scsi_cmnd *cmd = rq->special;
+	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
 	unsigned long wait_for = (cmd->allowed + 1) * rq->timeout;
 	int disposition;
 
@@ -1764,8 +1796,8 @@ static void scsi_request_fn(struct request_queue *q)
 			blk_start_request(req);
 
 		spin_unlock_irq(q->queue_lock);
-		cmd = req->special;
-		if (unlikely(cmd == NULL)) {
+		cmd = blk_mq_rq_to_pdu(req);
+		if (cmd != req->special) {
 			printk(KERN_CRIT "impossible request in %s.\n"
 					 "please mail a stack trace to "
 					 "linux-scsi@vger.kernel.org\n",
@@ -1868,6 +1900,7 @@ static int scsi_mq_prep_fn(struct request *req)
 	struct scsi_device *sdev = req->q->queuedata;
 	struct Scsi_Host *shost = sdev->host;
 	struct scatterlist *sg;
+	int ret;
 
 	scsi_init_command(sdev, cmd);
 
@@ -1901,7 +1934,10 @@ static int scsi_mq_prep_fn(struct request *req)
 
 	blk_mq_start_request(req);
 
-	return scsi_setup_cmnd(sdev, req);
+	ret = scsi_setup_cmnd(sdev, req);
+	if (ret != BLK_STS_OK)
+		cmd->flags &= ~SCMD_INITIALIZED;
+	return ret;
 }
 
 static void scsi_mq_done(struct scsi_cmnd *cmd)
@@ -2001,8 +2037,8 @@ static enum blk_eh_timer_return scsi_timeout(struct request *req,
 	return scsi_times_out(req);
 }
 
-static int scsi_init_request(struct blk_mq_tag_set *set, struct request *rq,
-		unsigned int hctx_idx, unsigned int numa_node)
+static int scsi_mq_init_request(struct blk_mq_tag_set *set, struct request *rq,
+				unsigned int hctx_idx, unsigned int numa_node)
 {
 	struct Scsi_Host *shost = set->driver_data;
 	const bool unchecked_isa_dma = shost->unchecked_isa_dma;
@@ -2026,8 +2062,8 @@ static int scsi_init_request(struct blk_mq_tag_set *set, struct request *rq,
 	return 0;
 }
 
-static void scsi_exit_request(struct blk_mq_tag_set *set, struct request *rq,
-		unsigned int hctx_idx)
+static void scsi_mq_exit_request(struct blk_mq_tag_set *set, struct request *rq,
+				 unsigned int hctx_idx)
 {
 	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
 
@@ -2104,7 +2140,8 @@ void __scsi_init_queue(struct Scsi_Host *shost, struct request_queue *q)
 }
 EXPORT_SYMBOL_GPL(__scsi_init_queue);
 
-static int scsi_init_rq(struct request_queue *q, struct request *rq, gfp_t gfp)
+static int scsi_old_init_rq(struct request_queue *q, struct request *rq,
+			    gfp_t gfp)
 {
 	struct Scsi_Host *shost = q->rq_alloc_data;
 	const bool unchecked_isa_dma = shost->unchecked_isa_dma;
@@ -2134,7 +2171,7 @@ fail:
 	return -ENOMEM;
 }
 
-static void scsi_exit_rq(struct request_queue *q, struct request *rq)
+static void scsi_old_exit_rq(struct request_queue *q, struct request *rq)
 {
 	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
 
@@ -2144,7 +2181,7 @@ static void scsi_exit_rq(struct request_queue *q, struct request *rq)
 			       cmd->sense_buffer);
 }
 
-struct request_queue *scsi_alloc_queue(struct scsi_device *sdev)
+struct request_queue *scsi_old_alloc_queue(struct scsi_device *sdev)
 {
 	struct Scsi_Host *shost = sdev->host;
 	struct request_queue *q;
@@ -2155,8 +2192,8 @@ struct request_queue *scsi_alloc_queue(struct scsi_device *sdev)
 	q->cmd_size = sizeof(struct scsi_cmnd) + shost->hostt->cmd_size;
 	q->rq_alloc_data = shost;
 	q->request_fn = scsi_request_fn;
-	q->init_rq_fn = scsi_init_rq;
-	q->exit_rq_fn = scsi_exit_rq;
+	q->init_rq_fn = scsi_old_init_rq;
+	q->exit_rq_fn = scsi_old_exit_rq;
 	q->initialize_rq_fn = scsi_initialize_rq;
 
 	if (blk_init_allocated_queue(q) < 0) {
@@ -2180,8 +2217,8 @@ static const struct blk_mq_ops scsi_mq_ops = {
 #ifdef CONFIG_BLK_DEBUG_FS
 	.show_rq	= scsi_show_rq,
 #endif
-	.init_request	= scsi_init_request,
-	.exit_request	= scsi_exit_request,
+	.init_request	= scsi_mq_init_request,
+	.exit_request	= scsi_mq_exit_request,
 	.initialize_rq_fn = scsi_initialize_rq,
 	.map_queues	= scsi_map_queues,
 };
@@ -2547,7 +2584,7 @@ EXPORT_SYMBOL(scsi_test_unit_ready);
  *	@sdev:	scsi device to change the state of.
  *	@state:	state to change to.
  *
- *	Returns zero if unsuccessful or an error if the requested 
+ *	Returns zero if successful or an error if the requested
  *	transition is illegal.
  */
 int
@@ -2654,6 +2691,7 @@ scsi_device_set_state(struct scsi_device *sdev, enum scsi_device_state state)
 
 	}
 	sdev->sdev_state = state;
+	sysfs_notify(&sdev->sdev_gendev.kobj, NULL, "state");
 	return 0;
 
  illegal:
@@ -3073,19 +3111,26 @@ int scsi_internal_device_unblock_nowait(struct scsi_device *sdev,
 	 * Try to transition the scsi device to SDEV_RUNNING or one of the
 	 * offlined states and goose the device queue if successful.
 	 */
-	if ((sdev->sdev_state == SDEV_BLOCK) ||
-	    (sdev->sdev_state == SDEV_TRANSPORT_OFFLINE))
+	switch (sdev->sdev_state) {
+	case SDEV_BLOCK:
+	case SDEV_TRANSPORT_OFFLINE:
 		sdev->sdev_state = new_state;
-	else if (sdev->sdev_state == SDEV_CREATED_BLOCK) {
+		sysfs_notify(&sdev->sdev_gendev.kobj, NULL, "state");
+		break;
+	case SDEV_CREATED_BLOCK:
 		if (new_state == SDEV_TRANSPORT_OFFLINE ||
 		    new_state == SDEV_OFFLINE)
 			sdev->sdev_state = new_state;
 		else
 			sdev->sdev_state = SDEV_CREATED;
-	} else if (sdev->sdev_state != SDEV_CANCEL &&
-		 sdev->sdev_state != SDEV_OFFLINE)
+		sysfs_notify(&sdev->sdev_gendev.kobj, NULL, "state");
+		break;
+	case SDEV_CANCEL:
+	case SDEV_OFFLINE:
+		break;
+	default:
 		return -EINVAL;
-
+	}
 	scsi_start_queue(sdev);
 
 	return 0;
@@ -3262,8 +3307,8 @@ int scsi_vpd_lun_id(struct scsi_device *sdev, char *id, size_t id_len)
 {
 	u8 cur_id_type = 0xff;
 	u8 cur_id_size = 0;
-	unsigned char *d, *cur_id_str;
-	unsigned char __rcu *vpd_pg83;
+	const unsigned char *d, *cur_id_str;
+	const struct scsi_vpd *vpd_pg83;
 	int id_size = -EINVAL;
 
 	rcu_read_lock();
@@ -3294,8 +3339,8 @@ int scsi_vpd_lun_id(struct scsi_device *sdev, char *id, size_t id_len)
 	}
 
 	memset(id, 0, id_len);
-	d = vpd_pg83 + 4;
-	while (d < vpd_pg83 + sdev->vpd_pg83_len) {
+	d = vpd_pg83->data + 4;
+	while (d < vpd_pg83->data + vpd_pg83->len) {
 		/* Skip designators not referring to the LUN */
 		if ((d[1] & 0x30) != 0x00)
 			goto next_desig;
@@ -3411,8 +3456,8 @@ EXPORT_SYMBOL(scsi_vpd_lun_id);
  */
 int scsi_vpd_tpg_id(struct scsi_device *sdev, int *rel_id)
 {
-	unsigned char *d;
-	unsigned char __rcu *vpd_pg83;
+	const unsigned char *d;
+	const struct scsi_vpd *vpd_pg83;
 	int group_id = -EAGAIN, rel_port = -1;
 
 	rcu_read_lock();
@@ -3422,8 +3467,8 @@ int scsi_vpd_tpg_id(struct scsi_device *sdev, int *rel_id)
 		return -ENXIO;
 	}
 
-	d = sdev->vpd_pg83 + 4;
-	while (d < sdev->vpd_pg83 + sdev->vpd_pg83_len) {
+	d = vpd_pg83->data + 4;
+	while (d < vpd_pg83->data + vpd_pg83->len) {
 		switch (d[1] & 0xf) {
 		case 0x4:
 			/* Relative target port */
diff --git a/drivers/scsi/scsi_priv.h b/drivers/scsi/scsi_priv.h
index c11c1f9c912c..5c6d016a5ae9 100644
--- a/drivers/scsi/scsi_priv.h
+++ b/drivers/scsi/scsi_priv.h
@@ -88,7 +88,7 @@ extern void scsi_queue_insert(struct scsi_cmnd *cmd, int reason);
 extern void scsi_io_completion(struct scsi_cmnd *, unsigned int);
 extern void scsi_run_host_queues(struct Scsi_Host *shost);
 extern void scsi_requeue_run_queue(struct work_struct *work);
-extern struct request_queue *scsi_alloc_queue(struct scsi_device *sdev);
+extern struct request_queue *scsi_old_alloc_queue(struct scsi_device *sdev);
 extern struct request_queue *scsi_mq_alloc_queue(struct scsi_device *sdev);
 extern void scsi_start_queue(struct scsi_device *sdev);
 extern int scsi_mq_setup_tags(struct Scsi_Host *shost);
diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c
index fd88dabd599d..e7818afeda2b 100644
--- a/drivers/scsi/scsi_scan.c
+++ b/drivers/scsi/scsi_scan.c
@@ -268,7 +268,7 @@ static struct scsi_device *scsi_alloc_sdev(struct scsi_target *starget,
 	if (shost_use_blk_mq(shost))
 		sdev->request_queue = scsi_mq_alloc_queue(sdev);
 	else
-		sdev->request_queue = scsi_alloc_queue(sdev);
+		sdev->request_queue = scsi_old_alloc_queue(sdev);
 	if (!sdev->request_queue) {
 		/* release fn is set up in scsi_sysfs_device_initialise, so
 		 * have to free and put manually here */
diff --git a/drivers/scsi/scsi_sysfs.c b/drivers/scsi/scsi_sysfs.c
index d6984df71f1c..bf53356f41f0 100644
--- a/drivers/scsi/scsi_sysfs.c
+++ b/drivers/scsi/scsi_sysfs.c
@@ -114,7 +114,7 @@ static int check_set(unsigned long long *val, char *src)
 {
 	char *last;
 
-	if (strncmp(src, "-", 20) == 0) {
+	if (strcmp(src, "-") == 0) {
 		*val = SCAN_WILD_CARD;
 	} else {
 		/*
@@ -303,6 +303,8 @@ store_host_reset(struct device *dev, struct device_attribute *attr,
 
 	if (sht->host_reset)
 		ret = sht->host_reset(shost, type);
+	else
+		ret = -EOPNOTSUPP;
 
 exit_store_host_reset:
 	if (ret == 0)
@@ -426,6 +428,7 @@ static void scsi_device_dev_release_usercontext(struct work_struct *work)
 	struct scsi_device *sdev;
 	struct device *parent;
 	struct list_head *this, *tmp;
+	struct scsi_vpd *vpd_pg80 = NULL, *vpd_pg83 = NULL;
 	unsigned long flags;
 
 	sdev = container_of(work, struct scsi_device, ew.work);
@@ -454,8 +457,17 @@ static void scsi_device_dev_release_usercontext(struct work_struct *work)
 	/* NULL queue means the device can't be used */
 	sdev->request_queue = NULL;
 
-	kfree(sdev->vpd_pg83);
-	kfree(sdev->vpd_pg80);
+	mutex_lock(&sdev->inquiry_mutex);
+	rcu_swap_protected(sdev->vpd_pg80, vpd_pg80,
+			   lockdep_is_held(&sdev->inquiry_mutex));
+	rcu_swap_protected(sdev->vpd_pg83, vpd_pg83,
+			   lockdep_is_held(&sdev->inquiry_mutex));
+	mutex_unlock(&sdev->inquiry_mutex);
+
+	if (vpd_pg83)
+		kfree_rcu(vpd_pg83, rcu);
+	if (vpd_pg80)
+		kfree_rcu(vpd_pg80, rcu);
 	kfree(sdev->inquiry);
 	kfree(sdev);
 
@@ -793,15 +805,16 @@ show_vpd_##_page(struct file *filp, struct kobject *kobj,	\
 {									\
 	struct device *dev = container_of(kobj, struct device, kobj);	\
 	struct scsi_device *sdev = to_scsi_device(dev);			\
-	int ret;							\
-	if (!sdev->vpd_##_page)						\
-		return -EINVAL;						\
+	struct scsi_vpd *vpd_page;					\
+	int ret = -EINVAL;						\
+									\
 	rcu_read_lock();						\
-	ret = memory_read_from_buffer(buf, count, &off,			\
-				      rcu_dereference(sdev->vpd_##_page), \
-				       sdev->vpd_##_page##_len);	\
+	vpd_page = rcu_dereference(sdev->vpd_##_page);			\
+	if (vpd_page)							\
+		ret = memory_read_from_buffer(buf, count, &off,		\
+				vpd_page->data, vpd_page->len);		\
 	rcu_read_unlock();						\
-	return ret;						\
+	return ret;							\
 }									\
 static struct bin_attribute dev_attr_vpd_##_page = {		\
 	.attr =	{.name = __stringify(vpd_##_page), .mode = S_IRUGO },	\
diff --git a/drivers/scsi/scsi_transport_fc.c b/drivers/scsi/scsi_transport_fc.c
index 892fbd9800d9..ba9d70f8a6a1 100644
--- a/drivers/scsi/scsi_transport_fc.c
+++ b/drivers/scsi/scsi_transport_fc.c
@@ -3276,8 +3276,8 @@ fc_scsi_scan_rport(struct work_struct *work)
 }
 
 /**
- * fc_block_scsi_eh - Block SCSI eh thread for blocked fc_rport
- * @cmnd: SCSI command that scsi_eh is trying to recover
+ * fc_block_rport() - Block SCSI eh thread for blocked fc_rport.
+ * @rport: Remote port that scsi_eh is trying to recover.
  *
  * This routine can be called from a FC LLD scsi_eh callback. It
  * blocks the scsi_eh thread until the fc_rport leaves the
@@ -3289,10 +3289,9 @@ fc_scsi_scan_rport(struct work_struct *work)
  *	    FAST_IO_FAIL if the fast_io_fail_tmo fired, this should be
  *	    passed back to scsi_eh.
  */
-int fc_block_scsi_eh(struct scsi_cmnd *cmnd)
+int fc_block_rport(struct fc_rport *rport)
 {
-	struct Scsi_Host *shost = cmnd->device->host;
-	struct fc_rport *rport = starget_to_rport(scsi_target(cmnd->device));
+	struct Scsi_Host *shost = rport_to_shost(rport);
 	unsigned long flags;
 
 	spin_lock_irqsave(shost->host_lock, flags);
@@ -3309,6 +3308,28 @@ int fc_block_scsi_eh(struct scsi_cmnd *cmnd)
 
 	return 0;
 }
+EXPORT_SYMBOL(fc_block_rport);
+
+/**
+ * fc_block_scsi_eh - Block SCSI eh thread for blocked fc_rport
+ * @cmnd: SCSI command that scsi_eh is trying to recover
+ *
+ * This routine can be called from a FC LLD scsi_eh callback. It
+ * blocks the scsi_eh thread until the fc_rport leaves the
+ * FC_PORTSTATE_BLOCKED, or the fast_io_fail_tmo fires. This is
+ * necessary to avoid the scsi_eh failing recovery actions for blocked
+ * rports which would lead to offlined SCSI devices.
+ *
+ * Returns: 0 if the fc_rport left the state FC_PORTSTATE_BLOCKED.
+ *	    FAST_IO_FAIL if the fast_io_fail_tmo fired, this should be
+ *	    passed back to scsi_eh.
+ */
+int fc_block_scsi_eh(struct scsi_cmnd *cmnd)
+{
+	struct fc_rport *rport = starget_to_rport(scsi_target(cmnd->device));
+
+	return fc_block_rport(rport);
+}
 EXPORT_SYMBOL(fc_block_scsi_eh);
 
 /**
@@ -3550,7 +3571,7 @@ fc_vport_sched_delete(struct work_struct *work)
 static enum blk_eh_timer_return
 fc_bsg_job_timeout(struct request *req)
 {
-	struct bsg_job *job = (void *) req->special;
+	struct bsg_job *job = blk_mq_rq_to_pdu(req);
 	struct Scsi_Host *shost = fc_bsg_to_shost(job);
 	struct fc_rport *rport = fc_bsg_to_rport(job);
 	struct fc_internal *i = to_fc_internal(shost->transportt);
@@ -3763,7 +3784,8 @@ fc_bsg_hostadd(struct Scsi_Host *shost, struct fc_host_attrs *fc_host)
 	snprintf(bsg_name, sizeof(bsg_name),
 		 "fc_host%d", shost->host_no);
 
-	q = bsg_setup_queue(dev, bsg_name, fc_bsg_dispatch, i->f->dd_bsg_size);
+	q = bsg_setup_queue(dev, bsg_name, fc_bsg_dispatch, i->f->dd_bsg_size,
+			NULL);
 	if (IS_ERR(q)) {
 		dev_err(dev,
 			"fc_host%d: bsg interface failed to initialize - setup queue\n",
@@ -3808,7 +3830,8 @@ fc_bsg_rportadd(struct Scsi_Host *shost, struct fc_rport *rport)
 	if (!i->f->bsg_request)
 		return -ENOTSUPP;
 
-	q = bsg_setup_queue(dev, NULL, fc_bsg_dispatch, i->f->dd_bsg_size);
+	q = bsg_setup_queue(dev, NULL, fc_bsg_dispatch, i->f->dd_bsg_size,
+			NULL);
 	if (IS_ERR(q)) {
 		dev_err(dev, "failed to setup bsg queue\n");
 		return PTR_ERR(q);
diff --git a/drivers/scsi/scsi_transport_iscsi.c b/drivers/scsi/scsi_transport_iscsi.c
index a424eaeafeb0..8934f19bce8e 100644
--- a/drivers/scsi/scsi_transport_iscsi.c
+++ b/drivers/scsi/scsi_transport_iscsi.c
@@ -1009,7 +1009,7 @@ static void iscsi_flashnode_sess_release(struct device *dev)
 	kfree(fnode_sess);
 }
 
-static struct device_type iscsi_flashnode_sess_dev_type = {
+static const struct device_type iscsi_flashnode_sess_dev_type = {
 	.name = "iscsi_flashnode_sess_dev_type",
 	.groups = iscsi_flashnode_sess_attr_groups,
 	.release = iscsi_flashnode_sess_release,
@@ -1195,7 +1195,7 @@ static void iscsi_flashnode_conn_release(struct device *dev)
 	kfree(fnode_conn);
 }
 
-static struct device_type iscsi_flashnode_conn_dev_type = {
+static const struct device_type iscsi_flashnode_conn_dev_type = {
 	.name = "iscsi_flashnode_conn_dev_type",
 	.groups = iscsi_flashnode_conn_attr_groups,
 	.release = iscsi_flashnode_conn_release,
@@ -1542,7 +1542,7 @@ iscsi_bsg_host_add(struct Scsi_Host *shost, struct iscsi_cls_host *ihost)
 		return -ENOTSUPP;
 
 	snprintf(bsg_name, sizeof(bsg_name), "iscsi_host%d", shost->host_no);
-	q = bsg_setup_queue(dev, bsg_name, iscsi_bsg_host_dispatch, 0);
+	q = bsg_setup_queue(dev, bsg_name, iscsi_bsg_host_dispatch, 0, NULL);
 	if (IS_ERR(q)) {
 		shost_printk(KERN_ERR, shost, "bsg interface failed to "
 			     "initialize - no request queue\n");
diff --git a/drivers/scsi/scsi_transport_sas.c b/drivers/scsi/scsi_transport_sas.c
index 5006a656e16a..319dff970237 100644
--- a/drivers/scsi/scsi_transport_sas.c
+++ b/drivers/scsi/scsi_transport_sas.c
@@ -169,39 +169,22 @@ static struct sas_end_device *sas_sdev_to_rdev(struct scsi_device *sdev)
 	return rdev;
 }
 
-static void sas_smp_request(struct request_queue *q, struct Scsi_Host *shost,
-			    struct sas_rphy *rphy)
+static int sas_smp_dispatch(struct bsg_job *job)
 {
-	struct request *req;
-	blk_status_t ret;
-	int (*handler)(struct Scsi_Host *, struct sas_rphy *, struct request *);
+	struct Scsi_Host *shost = dev_to_shost(job->dev);
+	struct sas_rphy *rphy = NULL;
 
-	while ((req = blk_fetch_request(q)) != NULL) {
-		spin_unlock_irq(q->queue_lock);
+	if (!scsi_is_host_device(job->dev))
+		rphy = dev_to_rphy(job->dev);
 
-		scsi_req(req)->resid_len = blk_rq_bytes(req);
-		if (req->next_rq)
-			scsi_req(req->next_rq)->resid_len =
-				blk_rq_bytes(req->next_rq);
-		handler = to_sas_internal(shost->transportt)->f->smp_handler;
-		ret = handler(shost, rphy, req);
-		scsi_req(req)->result = ret;
-
-		blk_end_request_all(req, 0);
-
-		spin_lock_irq(q->queue_lock);
+	if (!job->req->next_rq) {
+		dev_warn(job->dev, "space for a smp response is missing\n");
+		bsg_job_done(job, -EINVAL, 0);
+		return 0;
 	}
-}
-
-static void sas_host_smp_request(struct request_queue *q)
-{
-	sas_smp_request(q, (struct Scsi_Host *)q->queuedata, NULL);
-}
 
-static void sas_non_host_smp_request(struct request_queue *q)
-{
-	struct sas_rphy *rphy = q->queuedata;
-	sas_smp_request(q, rphy_to_shost(rphy), rphy);
+	to_sas_internal(shost->transportt)->f->smp_handler(job, shost, rphy);
+	return 0;
 }
 
 static void sas_host_release(struct device *dev)
@@ -217,81 +200,36 @@ static void sas_host_release(struct device *dev)
 static int sas_bsg_initialize(struct Scsi_Host *shost, struct sas_rphy *rphy)
 {
 	struct request_queue *q;
-	int error;
-	struct device *dev;
-	char namebuf[20];
-	const char *name;
-	void (*release)(struct device *);
 
 	if (!to_sas_internal(shost->transportt)->f->smp_handler) {
 		printk("%s can't handle SMP requests\n", shost->hostt->name);
 		return 0;
 	}
 
-	q = blk_alloc_queue(GFP_KERNEL);
-	if (!q)
-		return -ENOMEM;
-	q->initialize_rq_fn = scsi_initialize_rq;
-	q->cmd_size = sizeof(struct scsi_request);
-
 	if (rphy) {
-		q->request_fn = sas_non_host_smp_request;
-		dev = &rphy->dev;
-		name = dev_name(dev);
-		release = NULL;
+		q = bsg_setup_queue(&rphy->dev, dev_name(&rphy->dev),
+				sas_smp_dispatch, 0, NULL);
+		if (IS_ERR(q))
+			return PTR_ERR(q);
+		rphy->q = q;
 	} else {
-		q->request_fn = sas_host_smp_request;
-		dev = &shost->shost_gendev;
-		snprintf(namebuf, sizeof(namebuf),
-			 "sas_host%d", shost->host_no);
-		name = namebuf;
-		release = sas_host_release;
+		char name[20];
+
+		snprintf(name, sizeof(name), "sas_host%d", shost->host_no);
+		q = bsg_setup_queue(&shost->shost_gendev, name,
+				sas_smp_dispatch, 0, sas_host_release);
+		if (IS_ERR(q))
+			return PTR_ERR(q);
+		to_sas_host_attrs(shost)->q = q;
 	}
-	error = blk_init_allocated_queue(q);
-	if (error)
-		goto out_cleanup_queue;
 
 	/*
 	 * by default assume old behaviour and bounce for any highmem page
 	 */
 	blk_queue_bounce_limit(q, BLK_BOUNCE_HIGH);
-
-	error = bsg_register_queue(q, dev, name, release);
-	if (error)
-		goto out_cleanup_queue;
-
-	if (rphy)
-		rphy->q = q;
-	else
-		to_sas_host_attrs(shost)->q = q;
-
-	if (rphy)
-		q->queuedata = rphy;
-	else
-		q->queuedata = shost;
-
 	queue_flag_set_unlocked(QUEUE_FLAG_BIDI, q);
 	queue_flag_set_unlocked(QUEUE_FLAG_SCSI_PASSTHROUGH, q);
 	return 0;
-
-out_cleanup_queue:
-	blk_cleanup_queue(q);
-	return error;
-}
-
-static void sas_bsg_remove(struct Scsi_Host *shost, struct sas_rphy *rphy)
-{
-	struct request_queue *q;
-
-	if (rphy)
-		q = rphy->q;
-	else
-		q = to_sas_host_attrs(shost)->q;
-
-	if (!q)
-		return;
-
-	bsg_unregister_queue(q);
 }
 
 /*
@@ -321,9 +259,10 @@ static int sas_host_remove(struct transport_container *tc, struct device *dev,
 			   struct device *cdev)
 {
 	struct Scsi_Host *shost = dev_to_shost(dev);
+	struct request_queue *q = to_sas_host_attrs(shost)->q;
 
-	sas_bsg_remove(shost, NULL);
-
+	if (q)
+		bsg_unregister_queue(q);
 	return 0;
 }
 
@@ -421,6 +360,9 @@ sas_tlr_supported(struct scsi_device *sdev)
 	char *buffer = kzalloc(vpd_len, GFP_KERNEL);
 	int ret = 0;
 
+	if (!buffer)
+		goto out;
+
 	if (scsi_get_vpd_page(sdev, 0x90, buffer, vpd_len))
 		goto out;
 
@@ -1710,7 +1652,8 @@ sas_rphy_remove(struct sas_rphy *rphy)
 	}
 
 	sas_rphy_unlink(rphy);
-	sas_bsg_remove(NULL, rphy);
+	if (rphy->q)
+		bsg_unregister_queue(rphy->q);
 	transport_remove_device(dev);
 	device_del(dev);
 }
diff --git a/drivers/scsi/scsi_transport_srp.c b/drivers/scsi/scsi_transport_srp.c
index f617021c94f7..4f6f01cf9968 100644
--- a/drivers/scsi/scsi_transport_srp.c
+++ b/drivers/scsi/scsi_transport_srp.c
@@ -78,7 +78,7 @@ static inline struct srp_rport *shost_to_rport(struct Scsi_Host *shost)
  * parameters must be such that multipath can detect failed paths timely.
  * Hence do not allow all three parameters to be disabled simultaneously.
  */
-int srp_tmo_valid(int reconnect_delay, int fast_io_fail_tmo, int dev_loss_tmo)
+int srp_tmo_valid(int reconnect_delay, int fast_io_fail_tmo, long dev_loss_tmo)
 {
 	if (reconnect_delay < 0 && fast_io_fail_tmo < 0 && dev_loss_tmo < 0)
 		return -EINVAL;
@@ -556,8 +556,11 @@ int srp_reconnect_rport(struct srp_rport *rport)
 		 */
 		shost_for_each_device(sdev, shost) {
 			mutex_lock(&sdev->state_mutex);
-			if (sdev->sdev_state == SDEV_OFFLINE)
+			if (sdev->sdev_state == SDEV_OFFLINE) {
 				sdev->sdev_state = SDEV_RUNNING;
+				sysfs_notify(&sdev->sdev_gendev.kobj,
+					     NULL, "state");
+			}
 			mutex_unlock(&sdev->state_mutex);
 		}
 	} else if (rport->state == SRP_RPORT_RUNNING) {
diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c
index bea36adeee17..fb9f8b5f4673 100644
--- a/drivers/scsi/sd.c
+++ b/drivers/scsi/sd.c
@@ -312,7 +312,7 @@ protection_type_store(struct device *dev, struct device_attribute *attr,
 	if (err)
 		return err;
 
-	if (val >= 0 && val <= T10_PI_TYPE3_PROTECTION)
+	if (val <= T10_PI_TYPE3_PROTECTION)
 		sdkp->protection_type = val;
 
 	return count;
@@ -1013,7 +1013,7 @@ static int sd_setup_read_write_cmnd(struct scsi_cmnd *SCpnt)
 	ret = scsi_init_io(SCpnt);
 	if (ret != BLKPREP_OK)
 		goto out;
-	SCpnt = rq->special;
+	WARN_ON_ONCE(SCpnt != rq->special);
 
 	/* from here on until we're complete, any goto out
 	 * is used for a killable error condition */
@@ -1277,6 +1277,9 @@ static void sd_uninit_command(struct scsi_cmnd *SCpnt)
 {
 	struct request *rq = SCpnt->request;
 
+	if (SCpnt->flags & SCMD_ZONE_WRITE_LOCK)
+		sd_zbc_write_unlock_zone(SCpnt);
+
 	if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
 		__free_page(rq->special_vec.bv_page);
 
@@ -2912,8 +2915,6 @@ static void sd_read_block_limits(struct scsi_disk *sdkp)
 				sd_config_discard(sdkp, SD_LBP_WS16);
 			else if (sdkp->lbpws10)
 				sd_config_discard(sdkp, SD_LBP_WS10);
-			else if (sdkp->lbpu && sdkp->max_unmap_blocks)
-				sd_config_discard(sdkp, SD_LBP_UNMAP);
 			else
 				sd_config_discard(sdkp, SD_LBP_DISABLE);
 		}
@@ -3223,7 +3224,6 @@ static void sd_probe_async(void *data, async_cookie_t cookie)
 
 	gd->major = sd_major((index & 0xf0) >> 4);
 	gd->first_minor = ((index & 0xf) << 4) | (index & 0xfff00);
-	gd->minors = SD_MINORS;
 
 	gd->fops = &sd_fops;
 	gd->private_data = &sdkp->driver;
@@ -3537,7 +3537,7 @@ static int sd_suspend_common(struct device *dev, bool ignore_stop_errors)
 			 * doesn't support sync. There's not much to do and
 			 * suspend shouldn't fail.
 			 */
-			 ret = 0;
+			ret = 0;
 		}
 	}
 
diff --git a/drivers/scsi/sd_zbc.c b/drivers/scsi/sd_zbc.c
index 96855df9f49d..8aa54779aac1 100644
--- a/drivers/scsi/sd_zbc.c
+++ b/drivers/scsi/sd_zbc.c
@@ -294,6 +294,9 @@ int sd_zbc_write_lock_zone(struct scsi_cmnd *cmd)
 	    test_and_set_bit(zno, sdkp->zones_wlock))
 		return BLKPREP_DEFER;
 
+	WARN_ON_ONCE(cmd->flags & SCMD_ZONE_WRITE_LOCK);
+	cmd->flags |= SCMD_ZONE_WRITE_LOCK;
+
 	return BLKPREP_OK;
 }
 
@@ -302,9 +305,10 @@ void sd_zbc_write_unlock_zone(struct scsi_cmnd *cmd)
 	struct request *rq = cmd->request;
 	struct scsi_disk *sdkp = scsi_disk(rq->rq_disk);
 
-	if (sdkp->zones_wlock) {
+	if (sdkp->zones_wlock && cmd->flags & SCMD_ZONE_WRITE_LOCK) {
 		unsigned int zno = sd_zbc_zone_no(sdkp, blk_rq_pos(rq));
 		WARN_ON_ONCE(!test_bit(zno, sdkp->zones_wlock));
+		cmd->flags &= ~SCMD_ZONE_WRITE_LOCK;
 		clear_bit_unlock(zno, sdkp->zones_wlock);
 		smp_mb__after_atomic();
 	}
@@ -335,9 +339,6 @@ void sd_zbc_complete(struct scsi_cmnd *cmd,
 	case REQ_OP_WRITE_ZEROES:
 	case REQ_OP_WRITE_SAME:
 
-		/* Unlock the zone */
-		sd_zbc_write_unlock_zone(cmd);
-
 		if (result &&
 		    sshdr->sense_key == ILLEGAL_REQUEST &&
 		    sshdr->asc == 0x21)
diff --git a/drivers/scsi/ses.c b/drivers/scsi/ses.c
index f1cdf32d7514..11826c5c2dd4 100644
--- a/drivers/scsi/ses.c
+++ b/drivers/scsi/ses.c
@@ -51,6 +51,13 @@ struct ses_component {
 	u64 addr;
 };
 
+static bool ses_page2_supported(struct enclosure_device *edev)
+{
+	struct ses_device *ses_dev = edev->scratch;
+
+	return (ses_dev->page2 != NULL);
+}
+
 static int ses_probe(struct device *dev)
 {
 	struct scsi_device *sdev = to_scsi_device(dev);
@@ -99,7 +106,7 @@ static int ses_recv_diag(struct scsi_device *sdev, int page_code,
 
 	ret =  scsi_execute_req(sdev, cmd, DMA_FROM_DEVICE, buf, bufflen,
 				NULL, SES_TIMEOUT, SES_RETRIES, NULL);
-	if (unlikely(!ret))
+	if (unlikely(ret))
 		return ret;
 
 	recv_page_code = ((unsigned char *)buf)[0];
@@ -179,7 +186,8 @@ static unsigned char *ses_get_page2_descriptor(struct enclosure_device *edev,
 	unsigned char *type_ptr = ses_dev->page1_types;
 	unsigned char *desc_ptr = ses_dev->page2 + 8;
 
-	ses_recv_diag(sdev, 2, ses_dev->page2, ses_dev->page2_len);
+	if (ses_recv_diag(sdev, 2, ses_dev->page2, ses_dev->page2_len) < 0)
+		return NULL;
 
 	for (i = 0; i < ses_dev->page1_num_types; i++, type_ptr += 4) {
 		for (j = 0; j < type_ptr[1]; j++) {
@@ -203,6 +211,10 @@ static void ses_get_fault(struct enclosure_device *edev,
 {
 	unsigned char *desc;
 
+	if (!ses_page2_supported(edev)) {
+		ecomp->fault = 0;
+		return;
+	}
 	desc = ses_get_page2_descriptor(edev, ecomp);
 	if (desc)
 		ecomp->fault = (desc[3] & 0x60) >> 4;
@@ -215,6 +227,9 @@ static int ses_set_fault(struct enclosure_device *edev,
 	unsigned char desc[4];
 	unsigned char *desc_ptr;
 
+	if (!ses_page2_supported(edev))
+		return -EINVAL;
+
 	desc_ptr = ses_get_page2_descriptor(edev, ecomp);
 
 	if (!desc_ptr)
@@ -242,6 +257,10 @@ static void ses_get_status(struct enclosure_device *edev,
 {
 	unsigned char *desc;
 
+	if (!ses_page2_supported(edev)) {
+		ecomp->status = 0;
+		return;
+	}
 	desc = ses_get_page2_descriptor(edev, ecomp);
 	if (desc)
 		ecomp->status = (desc[0] & 0x0f);
@@ -252,6 +271,10 @@ static void ses_get_locate(struct enclosure_device *edev,
 {
 	unsigned char *desc;
 
+	if (!ses_page2_supported(edev)) {
+		ecomp->locate = 0;
+		return;
+	}
 	desc = ses_get_page2_descriptor(edev, ecomp);
 	if (desc)
 		ecomp->locate = (desc[2] & 0x02) ? 1 : 0;
@@ -264,6 +287,9 @@ static int ses_set_locate(struct enclosure_device *edev,
 	unsigned char desc[4];
 	unsigned char *desc_ptr;
 
+	if (!ses_page2_supported(edev))
+		return -EINVAL;
+
 	desc_ptr = ses_get_page2_descriptor(edev, ecomp);
 
 	if (!desc_ptr)
@@ -292,6 +318,9 @@ static int ses_set_active(struct enclosure_device *edev,
 	unsigned char desc[4];
 	unsigned char *desc_ptr;
 
+	if (!ses_page2_supported(edev))
+		return -EINVAL;
+
 	desc_ptr = ses_get_page2_descriptor(edev, ecomp);
 
 	if (!desc_ptr)
@@ -328,6 +357,11 @@ static void ses_get_power_status(struct enclosure_device *edev,
 {
 	unsigned char *desc;
 
+	if (!ses_page2_supported(edev)) {
+		ecomp->power_status = 0;
+		return;
+	}
+
 	desc = ses_get_page2_descriptor(edev, ecomp);
 	if (desc)
 		ecomp->power_status = (desc[3] & 0x10) ? 0 : 1;
@@ -340,6 +374,9 @@ static int ses_set_power_status(struct enclosure_device *edev,
 	unsigned char desc[4];
 	unsigned char *desc_ptr;
 
+	if (!ses_page2_supported(edev))
+		return -EINVAL;
+
 	desc_ptr = ses_get_page2_descriptor(edev, ecomp);
 
 	if (!desc_ptr)
@@ -601,7 +638,7 @@ static int ses_intf_add(struct device *cdev,
 {
 	struct scsi_device *sdev = to_scsi_device(cdev->parent);
 	struct scsi_device *tmp_sdev;
-	unsigned char *buf = NULL, *hdr_buf, *type_ptr;
+	unsigned char *buf = NULL, *hdr_buf, *type_ptr, page;
 	struct ses_device *ses_dev;
 	u32 result;
 	int i, types, len, components = 0;
@@ -630,7 +667,8 @@ static int ses_intf_add(struct device *cdev,
 	if (!hdr_buf || !ses_dev)
 		goto err_init_free;
 
-	result = ses_recv_diag(sdev, 1, hdr_buf, INIT_ALLOC_SIZE);
+	page = 1;
+	result = ses_recv_diag(sdev, page, hdr_buf, INIT_ALLOC_SIZE);
 	if (result)
 		goto recv_failed;
 
@@ -639,7 +677,7 @@ static int ses_intf_add(struct device *cdev,
 	if (!buf)
 		goto err_free;
 
-	result = ses_recv_diag(sdev, 1, buf, len);
+	result = ses_recv_diag(sdev, page, buf, len);
 	if (result)
 		goto recv_failed;
 
@@ -669,9 +707,10 @@ static int ses_intf_add(struct device *cdev,
 	ses_dev->page1_len = len;
 	buf = NULL;
 
-	result = ses_recv_diag(sdev, 2, hdr_buf, INIT_ALLOC_SIZE);
+	page = 2;
+	result = ses_recv_diag(sdev, page, hdr_buf, INIT_ALLOC_SIZE);
 	if (result)
-		goto recv_failed;
+		goto page2_not_supported;
 
 	len = (hdr_buf[2] << 8) + hdr_buf[3] + 4;
 	buf = kzalloc(len, GFP_KERNEL);
@@ -688,7 +727,8 @@ static int ses_intf_add(struct device *cdev,
 
 	/* The additional information page --- allows us
 	 * to match up the devices */
-	result = ses_recv_diag(sdev, 10, hdr_buf, INIT_ALLOC_SIZE);
+	page = 10;
+	result = ses_recv_diag(sdev, page, hdr_buf, INIT_ALLOC_SIZE);
 	if (!result) {
 
 		len = (hdr_buf[2] << 8) + hdr_buf[3] + 4;
@@ -696,13 +736,14 @@ static int ses_intf_add(struct device *cdev,
 		if (!buf)
 			goto err_free;
 
-		result = ses_recv_diag(sdev, 10, buf, len);
+		result = ses_recv_diag(sdev, page, buf, len);
 		if (result)
 			goto recv_failed;
 		ses_dev->page10 = buf;
 		ses_dev->page10_len = len;
 		buf = NULL;
 	}
+page2_not_supported:
 	scomp = kzalloc(sizeof(struct ses_component) * components, GFP_KERNEL);
 	if (!scomp)
 		goto err_free;
@@ -734,7 +775,7 @@ static int ses_intf_add(struct device *cdev,
 
  recv_failed:
 	sdev_printk(KERN_ERR, sdev, "Failed to get diagnostic page 0x%x\n",
-		    result);
+		    page);
 	err = -ENODEV;
  err_free:
 	kfree(buf);
@@ -777,8 +818,6 @@ static void ses_intf_remove_enclosure(struct scsi_device *sdev)
 	if (!edev)
 		return;
 
-	enclosure_unregister(edev);
-
 	ses_dev = edev->scratch;
 	edev->scratch = NULL;
 
@@ -790,6 +829,7 @@ static void ses_intf_remove_enclosure(struct scsi_device *sdev)
 	kfree(edev->component[0].scratch);
 
 	put_device(&edev->edev);
+	enclosure_unregister(edev);
 }
 
 static void ses_intf_remove(struct device *cdev,
diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c
index d7ff71e0c85c..0419c2298eab 100644
--- a/drivers/scsi/sg.c
+++ b/drivers/scsi/sg.c
@@ -828,6 +828,39 @@ static int max_sectors_bytes(struct request_queue *q)
 	return max_sectors << 9;
 }
 
+static void
+sg_fill_request_table(Sg_fd *sfp, sg_req_info_t *rinfo)
+{
+	Sg_request *srp;
+	int val;
+	unsigned int ms;
+
+	val = 0;
+	list_for_each_entry(srp, &sfp->rq_list, entry) {
+		if (val > SG_MAX_QUEUE)
+			break;
+		rinfo[val].req_state = srp->done + 1;
+		rinfo[val].problem =
+			srp->header.masked_status &
+			srp->header.host_status &
+			srp->header.driver_status;
+		if (srp->done)
+			rinfo[val].duration =
+				srp->header.duration;
+		else {
+			ms = jiffies_to_msecs(jiffies);
+			rinfo[val].duration =
+				(ms > srp->header.duration) ?
+				(ms - srp->header.duration) : 0;
+		}
+		rinfo[val].orphan = srp->orphan;
+		rinfo[val].sg_io_owned = srp->sg_io_owned;
+		rinfo[val].pack_id = srp->header.pack_id;
+		rinfo[val].usr_ptr = srp->header.usr_ptr;
+		val++;
+	}
+}
+
 static long
 sg_ioctl(struct file *filp, unsigned int cmd_in, unsigned long arg)
 {
@@ -1012,38 +1045,13 @@ sg_ioctl(struct file *filp, unsigned int cmd_in, unsigned long arg)
 			return -EFAULT;
 		else {
 			sg_req_info_t *rinfo;
-			unsigned int ms;
 
-			rinfo = kmalloc(SZ_SG_REQ_INFO * SG_MAX_QUEUE,
-								GFP_KERNEL);
+			rinfo = kzalloc(SZ_SG_REQ_INFO * SG_MAX_QUEUE,
+					GFP_KERNEL);
 			if (!rinfo)
 				return -ENOMEM;
 			read_lock_irqsave(&sfp->rq_list_lock, iflags);
-			val = 0;
-			list_for_each_entry(srp, &sfp->rq_list, entry) {
-				if (val > SG_MAX_QUEUE)
-					break;
-				memset(&rinfo[val], 0, SZ_SG_REQ_INFO);
-				rinfo[val].req_state = srp->done + 1;
-				rinfo[val].problem =
-					srp->header.masked_status &
-					srp->header.host_status &
-					srp->header.driver_status;
-				if (srp->done)
-					rinfo[val].duration =
-						srp->header.duration;
-				else {
-					ms = jiffies_to_msecs(jiffies);
-					rinfo[val].duration =
-						(ms > srp->header.duration) ?
-						(ms - srp->header.duration) : 0;
-				}
-				rinfo[val].orphan = srp->orphan;
-				rinfo[val].sg_io_owned = srp->sg_io_owned;
-				rinfo[val].pack_id = srp->header.pack_id;
-				rinfo[val].usr_ptr = srp->header.usr_ptr;
-				val++;
-			}
+			sg_fill_request_table(sfp, rinfo);
 			read_unlock_irqrestore(&sfp->rq_list_lock, iflags);
 			result = __copy_to_user(p, rinfo,
 						SZ_SG_REQ_INFO * SG_MAX_QUEUE);
@@ -1081,8 +1089,7 @@ sg_ioctl(struct file *filp, unsigned int cmd_in, unsigned long arg)
 		return blk_trace_setup(sdp->device->request_queue,
 				       sdp->disk->disk_name,
 				       MKDEV(SCSI_GENERIC_MAJOR, sdp->index),
-				       NULL,
-				       (char *)arg);
+				       NULL, p);
 	case BLKTRACESTART:
 		return blk_trace_startstop(sdp->device->request_queue, 1);
 	case BLKTRACESTOP:
@@ -1233,6 +1240,7 @@ sg_mmap(struct file *filp, struct vm_area_struct *vma)
 	unsigned long req_sz, len, sa;
 	Sg_scatter_hold *rsv_schp;
 	int k, length;
+	int ret = 0;
 
 	if ((!filp) || (!vma) || (!(sfp = (Sg_fd *) filp->private_data)))
 		return -ENXIO;
@@ -1243,8 +1251,11 @@ sg_mmap(struct file *filp, struct vm_area_struct *vma)
 	if (vma->vm_pgoff)
 		return -EINVAL;	/* want no offset */
 	rsv_schp = &sfp->reserve;
-	if (req_sz > rsv_schp->bufflen)
-		return -ENOMEM;	/* cannot map more than reserved buffer */
+	mutex_lock(&sfp->f_mutex);
+	if (req_sz > rsv_schp->bufflen) {
+		ret = -ENOMEM;	/* cannot map more than reserved buffer */
+		goto out;
+	}
 
 	sa = vma->vm_start;
 	length = 1 << (PAGE_SHIFT + rsv_schp->page_order);
@@ -1258,7 +1269,9 @@ sg_mmap(struct file *filp, struct vm_area_struct *vma)
 	vma->vm_flags |= VM_IO | VM_DONTEXPAND | VM_DONTDUMP;
 	vma->vm_private_data = sfp;
 	vma->vm_ops = &sg_mmap_vm_ops;
-	return 0;
+out:
+	mutex_unlock(&sfp->f_mutex);
+	return ret;
 }
 
 static void
@@ -1735,9 +1748,12 @@ sg_start_req(Sg_request *srp, unsigned char *cmd)
 		    !sfp->res_in_use) {
 			sfp->res_in_use = 1;
 			sg_link_reserve(sfp, srp, dxfer_len);
-		} else if ((hp->flags & SG_FLAG_MMAP_IO) && sfp->res_in_use) {
+		} else if (hp->flags & SG_FLAG_MMAP_IO) {
+			res = -EBUSY; /* sfp->res_in_use == 1 */
+			if (dxfer_len > rsv_schp->bufflen)
+				res = -ENOMEM;
 			mutex_unlock(&sfp->f_mutex);
-			return -EBUSY;
+			return res;
 		} else {
 			res = sg_build_indirect(req_schp, sfp, dxfer_len);
 			if (res) {
diff --git a/drivers/scsi/sgiwd93.c b/drivers/scsi/sgiwd93.c
index 80cfa93e407c..5ed696dc9bbd 100644
--- a/drivers/scsi/sgiwd93.c
+++ b/drivers/scsi/sgiwd93.c
@@ -192,20 +192,6 @@ static inline void init_hpc_chain(struct ip22_hostdata *hdata)
 	hcp->desc.pnext = hdata->dma;
 }
 
-static int sgiwd93_bus_reset(struct scsi_cmnd *cmd)
-{
-	/* FIXME perform bus-specific reset */
-
-	/* FIXME 2: kill this function, and let midlayer fallback
-	   to the same result, calling wd33c93_host_reset() */
-
-	spin_lock_irq(cmd->device->host->host_lock);
-	wd33c93_host_reset(cmd);
-	spin_unlock_irq(cmd->device->host->host_lock);
-
-	return SUCCESS;
-}
-
 /*
  * Kludge alert - the SCSI code calls the abort and reset method with int
  * arguments not with pointers.  So this is going to blow up beautyfully
@@ -217,7 +203,6 @@ static struct scsi_host_template sgiwd93_template = {
 	.name			= "SGI WD93",
 	.queuecommand		= wd33c93_queuecommand,
 	.eh_abort_handler	= wd33c93_abort,
-	.eh_bus_reset_handler	= sgiwd93_bus_reset,
 	.eh_host_reset_handler	= wd33c93_host_reset,
 	.can_queue		= 16,
 	.this_id		= 7,
diff --git a/drivers/scsi/smartpqi/smartpqi.h b/drivers/scsi/smartpqi/smartpqi.h
index e164ffade38a..dc3a0542a2e8 100644
--- a/drivers/scsi/smartpqi/smartpqi.h
+++ b/drivers/scsi/smartpqi/smartpqi.h
@@ -688,6 +688,28 @@ struct pqi_config_table_heartbeat {
 	__le32	heartbeat_counter;
 };
 
+union pqi_reset_register {
+	struct {
+		u32	reset_type : 3;
+		u32	reserved : 2;
+		u32	reset_action : 3;
+		u32	hold_in_pd1 : 1;
+		u32	reserved2 : 23;
+	} bits;
+	u32	all_bits;
+};
+
+#define PQI_RESET_ACTION_RESET		0x1
+
+#define PQI_RESET_TYPE_NO_RESET		0x0
+#define PQI_RESET_TYPE_SOFT_RESET	0x1
+#define PQI_RESET_TYPE_FIRM_RESET	0x2
+#define PQI_RESET_TYPE_HARD_RESET	0x3
+
+#define PQI_RESET_ACTION_COMPLETED	0x2
+
+#define PQI_RESET_POLL_INTERVAL_MSECS	100
+
 #define PQI_MAX_OUTSTANDING_REQUESTS		((u32)~0)
 #define PQI_MAX_OUTSTANDING_REQUESTS_KDUMP	32
 #define PQI_MAX_TRANSFER_SIZE			(1024U * 1024U)
@@ -995,6 +1017,7 @@ struct pqi_ctrl_info {
 	u8		inbound_spanning_supported : 1;
 	u8		outbound_spanning_supported : 1;
 	u8		pqi_mode_enabled : 1;
+	u8		pqi_reset_quiesce_supported : 1;
 
 	struct list_head scsi_device_list;
 	spinlock_t	scsi_device_list_lock;
@@ -1056,9 +1079,9 @@ enum pqi_ctrl_mode {
 #define BMIC_SENSE_CONTROLLER_PARAMETERS	0x64
 #define BMIC_SENSE_SUBSYSTEM_INFORMATION	0x66
 #define BMIC_WRITE_HOST_WELLNESS		0xa5
-#define BMIC_CACHE_FLUSH			0xc2
+#define BMIC_FLUSH_CACHE			0xc2
 
-#define SA_CACHE_FLUSH				0x1
+#define SA_FLUSH_CACHE				0x1
 
 #define MASKED_DEVICE(lunid)			((lunid)[3] & 0xc0)
 #define CISS_GET_LEVEL_2_BUS(lunid)		((lunid)[7] & 0x3f)
@@ -1164,6 +1187,23 @@ struct bmic_identify_physical_device {
 	u8	padding_to_multiple_of_512[9];
 };
 
+struct bmic_flush_cache {
+	u8	disable_flag;
+	u8	system_power_action;
+	u8	ndu_flush;
+	u8	shutdown_event;
+	u8	reserved[28];
+};
+
+/* for shutdown_event member of struct bmic_flush_cache */
+enum bmic_flush_cache_shutdown_event {
+	NONE_CACHE_FLUSH_ONLY = 0,
+	SHUTDOWN = 1,
+	HIBERNATE = 2,
+	SUSPEND = 3,
+	RESTART = 4
+};
+
 #pragma pack()
 
 int pqi_add_sas_host(struct Scsi_Host *shost, struct pqi_ctrl_info *ctrl_info);
diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c
index cb8f886e705c..83bdbd84eb01 100644
--- a/drivers/scsi/smartpqi/smartpqi_init.c
+++ b/drivers/scsi/smartpqi/smartpqi_init.c
@@ -40,11 +40,11 @@
 #define BUILD_TIMESTAMP
 #endif
 
-#define DRIVER_VERSION		"1.0.4-100"
+#define DRIVER_VERSION		"1.1.2-125"
 #define DRIVER_MAJOR		1
-#define DRIVER_MINOR		0
-#define DRIVER_RELEASE		4
-#define DRIVER_REVISION		100
+#define DRIVER_MINOR		1
+#define DRIVER_RELEASE		2
+#define DRIVER_REVISION		125
 
 #define DRIVER_NAME		"Microsemi PQI Driver (v" \
 				DRIVER_VERSION BUILD_TIMESTAMP ")"
@@ -431,10 +431,10 @@ static int pqi_build_raid_path_request(struct pqi_ctrl_info *ctrl_info,
 		cdb[1] = CISS_GET_RAID_MAP;
 		put_unaligned_be32(buffer_length, &cdb[6]);
 		break;
-	case SA_CACHE_FLUSH:
+	case SA_FLUSH_CACHE:
 		request->data_direction = SOP_WRITE_FLAG;
 		cdb[0] = BMIC_WRITE;
-		cdb[6] = BMIC_CACHE_FLUSH;
+		cdb[6] = BMIC_FLUSH_CACHE;
 		put_unaligned_be16(buffer_length, &cdb[7]);
 		break;
 	case BMIC_IDENTIFY_CONTROLLER:
@@ -585,14 +585,13 @@ static int pqi_identify_physical_device(struct pqi_ctrl_info *ctrl_info,
 	return rc;
 }
 
-#define SA_CACHE_FLUSH_BUFFER_LENGTH	4
-
-static int pqi_flush_cache(struct pqi_ctrl_info *ctrl_info)
+static int pqi_flush_cache(struct pqi_ctrl_info *ctrl_info,
+	enum bmic_flush_cache_shutdown_event shutdown_event)
 {
 	int rc;
 	struct pqi_raid_path_request request;
 	int pci_direction;
-	u8 *buffer;
+	struct bmic_flush_cache *flush_cache;
 
 	/*
 	 * Don't bother trying to flush the cache if the controller is
@@ -601,13 +600,15 @@ static int pqi_flush_cache(struct pqi_ctrl_info *ctrl_info)
 	if (pqi_ctrl_offline(ctrl_info))
 		return -ENXIO;
 
-	buffer = kzalloc(SA_CACHE_FLUSH_BUFFER_LENGTH, GFP_KERNEL);
-	if (!buffer)
+	flush_cache = kzalloc(sizeof(*flush_cache), GFP_KERNEL);
+	if (!flush_cache)
 		return -ENOMEM;
 
+	flush_cache->shutdown_event = shutdown_event;
+
 	rc = pqi_build_raid_path_request(ctrl_info, &request,
-		SA_CACHE_FLUSH, RAID_CTLR_LUNID, buffer,
-		SA_CACHE_FLUSH_BUFFER_LENGTH, 0, &pci_direction);
+		SA_FLUSH_CACHE, RAID_CTLR_LUNID, flush_cache,
+		sizeof(*flush_cache), 0, &pci_direction);
 	if (rc)
 		goto out;
 
@@ -618,7 +619,7 @@ static int pqi_flush_cache(struct pqi_ctrl_info *ctrl_info)
 		pci_direction);
 
 out:
-	kfree(buffer);
+	kfree(flush_cache);
 
 	return rc;
 }
@@ -3007,11 +3008,9 @@ static void pqi_change_irq_mode(struct pqi_ctrl_info *ctrl_info,
 			break;
 		case IRQ_MODE_INTX:
 			pqi_configure_legacy_intx(ctrl_info, true);
-			sis_disable_msix(ctrl_info);
 			sis_enable_intx(ctrl_info);
 			break;
 		case IRQ_MODE_NONE:
-			sis_disable_msix(ctrl_info);
 			break;
 		}
 		break;
@@ -3019,14 +3018,12 @@ static void pqi_change_irq_mode(struct pqi_ctrl_info *ctrl_info,
 		switch (new_mode) {
 		case IRQ_MODE_MSIX:
 			pqi_configure_legacy_intx(ctrl_info, false);
-			sis_disable_intx(ctrl_info);
 			sis_enable_msix(ctrl_info);
 			break;
 		case IRQ_MODE_INTX:
 			break;
 		case IRQ_MODE_NONE:
 			pqi_configure_legacy_intx(ctrl_info, false);
-			sis_disable_intx(ctrl_info);
 			break;
 		}
 		break;
@@ -5498,6 +5495,7 @@ static int pqi_passthru_ioctl(struct pqi_ctrl_info *ctrl_info, void __user *arg)
 	case XFER_NONE:
 	case XFER_WRITE:
 	case XFER_READ:
+	case XFER_READ | XFER_WRITE:
 		break;
 	default:
 		return -EINVAL;
@@ -5538,6 +5536,9 @@ static int pqi_passthru_ioctl(struct pqi_ctrl_info *ctrl_info, void __user *arg)
 	case XFER_READ:
 		request.data_direction = SOP_READ_FLAG;
 		break;
+	case XFER_READ | XFER_WRITE:
+		request.data_direction = SOP_BIDIRECTIONAL;
+		break;
 	}
 
 	request.task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
@@ -5889,28 +5890,62 @@ static void pqi_unregister_scsi(struct pqi_ctrl_info *ctrl_info)
 	scsi_host_put(shost);
 }
 
-#define PQI_RESET_ACTION_RESET		0x1
+static int pqi_wait_for_pqi_reset_completion(struct pqi_ctrl_info *ctrl_info)
+{
+	int rc = 0;
+	struct pqi_device_registers __iomem *pqi_registers;
+	unsigned long timeout;
+	unsigned int timeout_msecs;
+	union pqi_reset_register reset_reg;
 
-#define PQI_RESET_TYPE_NO_RESET		0x0
-#define PQI_RESET_TYPE_SOFT_RESET	0x1
-#define PQI_RESET_TYPE_FIRM_RESET	0x2
-#define PQI_RESET_TYPE_HARD_RESET	0x3
+	pqi_registers = ctrl_info->pqi_registers;
+	timeout_msecs = readw(&pqi_registers->max_reset_timeout) * 100;
+	timeout = msecs_to_jiffies(timeout_msecs) + jiffies;
+
+	while (1) {
+		msleep(PQI_RESET_POLL_INTERVAL_MSECS);
+		reset_reg.all_bits = readl(&pqi_registers->device_reset);
+		if (reset_reg.bits.reset_action == PQI_RESET_ACTION_COMPLETED)
+			break;
+		pqi_check_ctrl_health(ctrl_info);
+		if (pqi_ctrl_offline(ctrl_info)) {
+			rc = -ENXIO;
+			break;
+		}
+		if (time_after(jiffies, timeout)) {
+			rc = -ETIMEDOUT;
+			break;
+		}
+	}
+
+	return rc;
+}
 
 static int pqi_reset(struct pqi_ctrl_info *ctrl_info)
 {
 	int rc;
-	u32 reset_params;
+	union pqi_reset_register reset_reg;
+
+	if (ctrl_info->pqi_reset_quiesce_supported) {
+		rc = sis_pqi_reset_quiesce(ctrl_info);
+		if (rc) {
+			dev_err(&ctrl_info->pci_dev->dev,
+				"PQI reset failed during quiesce with error %d\n",
+				rc);
+			return rc;
+		}
+	}
 
-	reset_params = (PQI_RESET_ACTION_RESET << 5) |
-		PQI_RESET_TYPE_HARD_RESET;
+	reset_reg.all_bits = 0;
+	reset_reg.bits.reset_type = PQI_RESET_TYPE_HARD_RESET;
+	reset_reg.bits.reset_action = PQI_RESET_ACTION_RESET;
 
-	writel(reset_params,
-		&ctrl_info->pqi_registers->device_reset);
+	writel(reset_reg.all_bits, &ctrl_info->pqi_registers->device_reset);
 
-	rc = pqi_wait_for_pqi_mode_ready(ctrl_info);
+	rc = pqi_wait_for_pqi_reset_completion(ctrl_info);
 	if (rc)
 		dev_err(&ctrl_info->pci_dev->dev,
-			"PQI reset failed\n");
+			"PQI reset failed with error %d\n", rc);
 
 	return rc;
 }
@@ -6007,7 +6042,12 @@ static int pqi_revert_to_sis_mode(struct pqi_ctrl_info *ctrl_info)
 	rc = pqi_reset(ctrl_info);
 	if (rc)
 		return rc;
-	sis_reenable_sis_mode(ctrl_info);
+	rc = sis_reenable_sis_mode(ctrl_info);
+	if (rc) {
+		dev_err(&ctrl_info->pci_dev->dev,
+			"re-enabling SIS mode failed with error %d\n", rc);
+		return rc;
+	}
 	pqi_save_ctrl_mode(ctrl_info, SIS_MODE);
 
 	return 0;
@@ -6659,7 +6699,8 @@ static void pqi_shutdown(struct pci_dev *pci_dev)
 	 * Write all data in the controller's battery-backed cache to
 	 * storage.
 	 */
-	rc = pqi_flush_cache(ctrl_info);
+	rc = pqi_flush_cache(ctrl_info, SHUTDOWN);
+	pqi_reset(ctrl_info);
 	if (rc == 0)
 		return;
 
@@ -6703,7 +6744,7 @@ static __maybe_unused int pqi_suspend(struct pci_dev *pci_dev, pm_message_t stat
 	pqi_cancel_rescan_worker(ctrl_info);
 	pqi_wait_until_scan_finished(ctrl_info);
 	pqi_wait_until_lun_reset_finished(ctrl_info);
-	pqi_flush_cache(ctrl_info);
+	pqi_flush_cache(ctrl_info, SUSPEND);
 	pqi_ctrl_block_requests(ctrl_info);
 	pqi_ctrl_wait_until_quiesced(ctrl_info);
 	pqi_wait_until_inbound_queues_empty(ctrl_info);
@@ -6781,7 +6822,7 @@ static const struct pci_device_id pqi_pci_id_table[] = {
 	},
 	{
 		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
-			       PCI_VENDOR_ID_ADAPTEC2, 0x0605)
+			       PCI_VENDOR_ID_ADAPTEC2, 0x0608)
 	},
 	{
 		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
@@ -6813,6 +6854,10 @@ static const struct pci_device_id pqi_pci_id_table[] = {
 	},
 	{
 		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_ADAPTEC2, 0x0807)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
 			       PCI_VENDOR_ID_ADAPTEC2, 0x0900)
 	},
 	{
@@ -6849,6 +6894,10 @@ static const struct pci_device_id pqi_pci_id_table[] = {
 	},
 	{
 		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_ADAPTEC2, 0x090a)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
 			       PCI_VENDOR_ID_ADAPTEC2, 0x1200)
 	},
 	{
@@ -6881,6 +6930,10 @@ static const struct pci_device_id pqi_pci_id_table[] = {
 	},
 	{
 		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       PCI_VENDOR_ID_DELL, 0x1fe0)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
 			       PCI_VENDOR_ID_HP, 0x0600)
 	},
 	{
@@ -6897,11 +6950,7 @@ static const struct pci_device_id pqi_pci_id_table[] = {
 	},
 	{
 		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
-			       PCI_VENDOR_ID_HP, 0x0604)
-	},
-	{
-		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
-			       PCI_VENDOR_ID_HP, 0x0606)
+			       PCI_VENDOR_ID_HP, 0x0609)
 	},
 	{
 		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
@@ -6929,14 +6978,6 @@ static const struct pci_device_id pqi_pci_id_table[] = {
 	},
 	{
 		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
-			       PCI_VENDOR_ID_HP, 0x0656)
-	},
-	{
-		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
-			       PCI_VENDOR_ID_HP, 0x0657)
-	},
-	{
-		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
 			       PCI_VENDOR_ID_HP, 0x0700)
 	},
 	{
@@ -6957,14 +6998,6 @@ static const struct pci_device_id pqi_pci_id_table[] = {
 	},
 	{
 		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
-			       PCI_VENDOR_ID_HP, 0x1102)
-	},
-	{
-		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
-			       PCI_VENDOR_ID_HP, 0x1150)
-	},
-	{
-		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
 			       PCI_ANY_ID, PCI_ANY_ID)
 	},
 	{ 0 }
diff --git a/drivers/scsi/smartpqi/smartpqi_sas_transport.c b/drivers/scsi/smartpqi/smartpqi_sas_transport.c
index 0d89d3728b43..b209a35e482e 100644
--- a/drivers/scsi/smartpqi/smartpqi_sas_transport.c
+++ b/drivers/scsi/smartpqi/smartpqi_sas_transport.c
@@ -329,14 +329,6 @@ static int pqi_sas_phy_speed(struct sas_phy *phy,
 	return -EINVAL;
 }
 
-/* SMP = Serial Management Protocol */
-
-static int pqi_sas_smp_handler(struct Scsi_Host *shost, struct sas_rphy *rphy,
-	struct request *req)
-{
-	return -EINVAL;
-}
-
 struct sas_function_template pqi_sas_transport_functions = {
 	.get_linkerrors = pqi_sas_get_linkerrors,
 	.get_enclosure_identifier = pqi_sas_get_enclosure_identifier,
@@ -346,5 +338,4 @@ struct sas_function_template pqi_sas_transport_functions = {
 	.phy_setup = pqi_sas_phy_setup,
 	.phy_release = pqi_sas_phy_release,
 	.set_phy_speed = pqi_sas_phy_speed,
-	.smp_handler = pqi_sas_smp_handler,
 };
diff --git a/drivers/scsi/smartpqi/smartpqi_sis.c b/drivers/scsi/smartpqi/smartpqi_sis.c
index e55dfcf200e5..5141bd4c9f06 100644
--- a/drivers/scsi/smartpqi/smartpqi_sis.c
+++ b/drivers/scsi/smartpqi/smartpqi_sis.c
@@ -34,11 +34,13 @@
 #define SIS_REENABLE_SIS_MODE			0x1
 #define SIS_ENABLE_MSIX				0x40
 #define SIS_ENABLE_INTX				0x80
-#define SIS_SOFT_RESET				0x100
-#define SIS_TRIGGER_SHUTDOWN			0x800000
 #define SIS_CMD_READY				0x200
+#define SIS_TRIGGER_SHUTDOWN			0x800000
+#define SIS_PQI_RESET_QUIESCE			0x1000000
+
 #define SIS_CMD_COMPLETE			0x1000
 #define SIS_CLEAR_CTRL_TO_HOST_DOORBELL		0x1000
+
 #define SIS_CMD_STATUS_SUCCESS			0x1
 #define SIS_CMD_COMPLETE_TIMEOUT_SECS		30
 #define SIS_CMD_COMPLETE_POLL_INTERVAL_MSECS	10
@@ -47,6 +49,7 @@
 #define SIS_EXTENDED_PROPERTIES_SUPPORTED	0x800000
 #define SIS_SMARTARRAY_FEATURES_SUPPORTED	0x2
 #define SIS_PQI_MODE_SUPPORTED			0x4
+#define SIS_PQI_RESET_QUIESCE_SUPPORTED		0x8
 #define SIS_REQUIRED_EXTENDED_PROPERTIES	\
 	(SIS_SMARTARRAY_FEATURES_SUPPORTED | SIS_PQI_MODE_SUPPORTED)
 
@@ -258,6 +261,9 @@ int sis_get_ctrl_properties(struct pqi_ctrl_info *ctrl_info)
 		SIS_REQUIRED_EXTENDED_PROPERTIES)
 		return -ENODEV;
 
+	if (extended_properties & SIS_PQI_RESET_QUIESCE_SUPPORTED)
+		ctrl_info->pqi_reset_quiesce_supported = true;
+
 	return 0;
 }
 
@@ -336,9 +342,10 @@ out:
 
 #define SIS_DOORBELL_BIT_CLEAR_TIMEOUT_SECS	30
 
-static void sis_wait_for_doorbell_bit_to_clear(
+static int sis_wait_for_doorbell_bit_to_clear(
 	struct pqi_ctrl_info *ctrl_info, u32 bit)
 {
+	int rc = 0;
 	u32 doorbell_register;
 	unsigned long timeout;
 
@@ -350,78 +357,38 @@ static void sis_wait_for_doorbell_bit_to_clear(
 		if ((doorbell_register & bit) == 0)
 			break;
 		if (readl(&ctrl_info->registers->sis_firmware_status) &
-			SIS_CTRL_KERNEL_PANIC)
+			SIS_CTRL_KERNEL_PANIC) {
+			rc = -ENODEV;
 			break;
+		}
 		if (time_after(jiffies, timeout)) {
 			dev_err(&ctrl_info->pci_dev->dev,
 				"doorbell register bit 0x%x not cleared\n",
 				bit);
+			rc = -ETIMEDOUT;
 			break;
 		}
 		usleep_range(1000, 2000);
 	}
-}
-
-/* Enable MSI-X interrupts on the controller. */
-
-void sis_enable_msix(struct pqi_ctrl_info *ctrl_info)
-{
-	u32 doorbell_register;
-
-	doorbell_register =
-		readl(&ctrl_info->registers->sis_host_to_ctrl_doorbell);
-	doorbell_register |= SIS_ENABLE_MSIX;
-
-	writel(doorbell_register,
-		&ctrl_info->registers->sis_host_to_ctrl_doorbell);
 
-	sis_wait_for_doorbell_bit_to_clear(ctrl_info, SIS_ENABLE_MSIX);
+	return rc;
 }
 
-/* Disable MSI-X interrupts on the controller. */
-
-void sis_disable_msix(struct pqi_ctrl_info *ctrl_info)
+static inline int sis_set_doorbell_bit(struct pqi_ctrl_info *ctrl_info, u32 bit)
 {
-	u32 doorbell_register;
-
-	doorbell_register =
-		readl(&ctrl_info->registers->sis_host_to_ctrl_doorbell);
-	doorbell_register &= ~SIS_ENABLE_MSIX;
+	writel(bit, &ctrl_info->registers->sis_host_to_ctrl_doorbell);
 
-	writel(doorbell_register,
-		&ctrl_info->registers->sis_host_to_ctrl_doorbell);
+	return sis_wait_for_doorbell_bit_to_clear(ctrl_info, bit);
 }
 
-void sis_enable_intx(struct pqi_ctrl_info *ctrl_info)
+void sis_enable_msix(struct pqi_ctrl_info *ctrl_info)
 {
-	u32 doorbell_register;
-
-	doorbell_register =
-		readl(&ctrl_info->registers->sis_host_to_ctrl_doorbell);
-	doorbell_register |= SIS_ENABLE_INTX;
-
-	writel(doorbell_register,
-		&ctrl_info->registers->sis_host_to_ctrl_doorbell);
-
-	sis_wait_for_doorbell_bit_to_clear(ctrl_info, SIS_ENABLE_INTX);
+	sis_set_doorbell_bit(ctrl_info, SIS_ENABLE_MSIX);
 }
 
-void sis_disable_intx(struct pqi_ctrl_info *ctrl_info)
-{
-	u32 doorbell_register;
-
-	doorbell_register =
-		readl(&ctrl_info->registers->sis_host_to_ctrl_doorbell);
-	doorbell_register &= ~SIS_ENABLE_INTX;
-
-	writel(doorbell_register,
-		&ctrl_info->registers->sis_host_to_ctrl_doorbell);
-}
-
-void sis_soft_reset(struct pqi_ctrl_info *ctrl_info)
+void sis_enable_intx(struct pqi_ctrl_info *ctrl_info)
 {
-	writel(SIS_SOFT_RESET,
-		&ctrl_info->registers->sis_host_to_ctrl_doorbell);
+	sis_set_doorbell_bit(ctrl_info, SIS_ENABLE_INTX);
 }
 
 void sis_shutdown_ctrl(struct pqi_ctrl_info *ctrl_info)
@@ -434,38 +401,14 @@ void sis_shutdown_ctrl(struct pqi_ctrl_info *ctrl_info)
 		&ctrl_info->registers->sis_host_to_ctrl_doorbell);
 }
 
-#define SIS_MODE_READY_TIMEOUT_SECS	30
+int sis_pqi_reset_quiesce(struct pqi_ctrl_info *ctrl_info)
+{
+	return sis_set_doorbell_bit(ctrl_info, SIS_PQI_RESET_QUIESCE);
+}
 
 int sis_reenable_sis_mode(struct pqi_ctrl_info *ctrl_info)
 {
-	int rc;
-	unsigned long timeout;
-	struct pqi_ctrl_registers __iomem *registers;
-	u32 doorbell;
-
-	registers = ctrl_info->registers;
-
-	writel(SIS_REENABLE_SIS_MODE,
-		&registers->sis_host_to_ctrl_doorbell);
-
-	rc = 0;
-	timeout = (SIS_MODE_READY_TIMEOUT_SECS * HZ) + jiffies;
-
-	while (1) {
-		doorbell = readl(&registers->sis_ctrl_to_host_doorbell);
-		if ((doorbell & SIS_REENABLE_SIS_MODE) == 0)
-			break;
-		if (time_after(jiffies, timeout)) {
-			rc = -ETIMEDOUT;
-			break;
-		}
-	}
-
-	if (rc)
-		dev_err(&ctrl_info->pci_dev->dev,
-			"re-enabling SIS mode failed\n");
-
-	return rc;
+	return sis_set_doorbell_bit(ctrl_info, SIS_REENABLE_SIS_MODE);
 }
 
 void sis_write_driver_scratch(struct pqi_ctrl_info *ctrl_info, u32 value)
diff --git a/drivers/scsi/smartpqi/smartpqi_sis.h b/drivers/scsi/smartpqi/smartpqi_sis.h
index 983184b69373..2bf889dbf5ab 100644
--- a/drivers/scsi/smartpqi/smartpqi_sis.h
+++ b/drivers/scsi/smartpqi/smartpqi_sis.h
@@ -27,11 +27,9 @@ int sis_get_ctrl_properties(struct pqi_ctrl_info *ctrl_info);
 int sis_get_pqi_capabilities(struct pqi_ctrl_info *ctrl_info);
 int sis_init_base_struct_addr(struct pqi_ctrl_info *ctrl_info);
 void sis_enable_msix(struct pqi_ctrl_info *ctrl_info);
-void sis_disable_msix(struct pqi_ctrl_info *ctrl_info);
 void sis_enable_intx(struct pqi_ctrl_info *ctrl_info);
-void sis_disable_intx(struct pqi_ctrl_info *ctrl_info);
-void sis_soft_reset(struct pqi_ctrl_info *ctrl_info);
 void sis_shutdown_ctrl(struct pqi_ctrl_info *ctrl_info);
+int sis_pqi_reset_quiesce(struct pqi_ctrl_info *ctrl_info);
 int sis_reenable_sis_mode(struct pqi_ctrl_info *ctrl_info);
 void sis_write_driver_scratch(struct pqi_ctrl_info *ctrl_info, u32 value);
 u32 sis_read_driver_scratch(struct pqi_ctrl_info *ctrl_info);
diff --git a/drivers/scsi/sr.c b/drivers/scsi/sr.c
index a8f630213a1a..9be34d37c356 100644
--- a/drivers/scsi/sr.c
+++ b/drivers/scsi/sr.c
@@ -393,7 +393,7 @@ static int sr_init_command(struct scsi_cmnd *SCpnt)
 	ret = scsi_init_io(SCpnt);
 	if (ret != BLKPREP_OK)
 		goto out;
-	SCpnt = rq->special;
+	WARN_ON_ONCE(SCpnt != rq->special);
 	cd = scsi_cd(rq->rq_disk);
 
 	/* from here on until we're complete, any goto out
diff --git a/drivers/scsi/st.c b/drivers/scsi/st.c
index 8e5013d9cad4..94e402ed30f6 100644
--- a/drivers/scsi/st.c
+++ b/drivers/scsi/st.c
@@ -4299,11 +4299,11 @@ static int st_probe(struct device *dev)
 	kref_init(&tpnt->kref);
 	tpnt->disk = disk;
 	disk->private_data = &tpnt->driver;
-	disk->queue = SDp->request_queue;
 	/* SCSI tape doesn't register this gendisk via add_disk().  Manually
 	 * take queue reference that release_disk() expects. */
-	if (!blk_get_queue(disk->queue))
+	if (!blk_get_queue(SDp->request_queue))
 		goto out_put_disk;
+	disk->queue = SDp->request_queue;
 	tpnt->driver = &st_template;
 
 	tpnt->device = SDp;
diff --git a/drivers/scsi/storvsc_drv.c b/drivers/scsi/storvsc_drv.c
index 3cc8d67783a1..5e7200f05873 100644
--- a/drivers/scsi/storvsc_drv.c
+++ b/drivers/scsi/storvsc_drv.c
@@ -1640,6 +1640,8 @@ static int storvsc_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *scmnd)
 	put_cpu();
 
 	if (ret == -EAGAIN) {
+		if (payload_sz > sizeof(cmd_request->mpb))
+			kfree(payload);
 		/* no more space */
 		return SCSI_MLQUEUE_DEVICE_BUSY;
 	}
diff --git a/drivers/scsi/sun3_scsi.c b/drivers/scsi/sun3_scsi.c
index e64b0c542f95..9492638296c8 100644
--- a/drivers/scsi/sun3_scsi.c
+++ b/drivers/scsi/sun3_scsi.c
@@ -46,7 +46,7 @@
 #define NCR5380_write(reg, value)       out_8(hostdata->io + (reg), value)
 
 #define NCR5380_queue_command           sun3scsi_queue_command
-#define NCR5380_bus_reset               sun3scsi_bus_reset
+#define NCR5380_host_reset              sun3scsi_host_reset
 #define NCR5380_abort                   sun3scsi_abort
 #define NCR5380_info                    sun3scsi_info
 
@@ -495,7 +495,7 @@ static struct scsi_host_template sun3_scsi_template = {
 	.info			= sun3scsi_info,
 	.queuecommand		= sun3scsi_queue_command,
 	.eh_abort_handler	= sun3scsi_abort,
-	.eh_bus_reset_handler	= sun3scsi_bus_reset,
+	.eh_host_reset_handler	= sun3scsi_host_reset,
 	.can_queue		= 16,
 	.this_id		= 7,
 	.sg_tablesize		= SG_NONE,
diff --git a/drivers/scsi/sym53c8xx_2/sym_hipd.c b/drivers/scsi/sym53c8xx_2/sym_hipd.c
index 6b349e301869..ca360daa6a25 100644
--- a/drivers/scsi/sym53c8xx_2/sym_hipd.c
+++ b/drivers/scsi/sym53c8xx_2/sym_hipd.c
@@ -759,7 +759,7 @@ static int sym_prepare_setting(struct Scsi_Host *shost, struct sym_hcb *np, stru
 	/*
 	 * Maximum synchronous period factor supported by the chip.
 	 */
-	period = (11 * div_10M[np->clock_divn - 1]) / (4 * np->clock_khz);
+	period = div64_ul(11 * div_10M[np->clock_divn - 1], 4 * np->clock_khz);
 	np->maxsync = period > 2540 ? 254 : period / 10;
 
 	/*
@@ -4985,13 +4985,10 @@ struct sym_lcb *sym_alloc_lcb (struct sym_hcb *np, u_char tn, u_char ln)
 	 *  Compute the bus address of this table.
 	 */
 	if (ln && !tp->luntbl) {
-		int i;
-
 		tp->luntbl = sym_calloc_dma(256, "LUNTBL");
 		if (!tp->luntbl)
 			goto fail;
-		for (i = 0 ; i < 64 ; i++)
-			tp->luntbl[i] = cpu_to_scr(vtobus(&np->badlun_sa));
+		memset32(tp->luntbl, cpu_to_scr(vtobus(&np->badlun_sa)), 64);
 		tp->head.luntbl_sa = cpu_to_scr(vtobus(tp->luntbl));
 	}
 
@@ -5077,8 +5074,7 @@ static void sym_alloc_lcb_tags (struct sym_hcb *np, u_char tn, u_char ln)
 	/*
 	 *  Initialize the task table with invalid entries.
 	 */
-	for (i = 0 ; i < SYM_CONF_MAX_TASK ; i++)
-		lp->itlq_tbl[i] = cpu_to_scr(np->notask_ba);
+	memset32(lp->itlq_tbl, cpu_to_scr(np->notask_ba), SYM_CONF_MAX_TASK);
 
 	/*
 	 *  Fill up the tag buffer with tag numbers.
@@ -5764,8 +5760,7 @@ int sym_hcb_attach(struct Scsi_Host *shost, struct sym_fw *fw, struct sym_nvram
 		goto attach_failed;
 
 	np->badlun_sa = cpu_to_scr(SCRIPTB_BA(np, resel_bad_lun));
-	for (i = 0 ; i < 64 ; i++)	/* 64 luns/target, no less */
-		np->badluntbl[i] = cpu_to_scr(vtobus(&np->badlun_sa));
+	memset32(np->badluntbl, cpu_to_scr(vtobus(&np->badlun_sa)), 64);
 
 	/*
 	 *  Prepare the bus address array that contains the bus 
diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c
index 5bc9dc14e075..794a4600e952 100644
--- a/drivers/scsi/ufs/ufshcd.c
+++ b/drivers/scsi/ufs/ufshcd.c
@@ -438,7 +438,7 @@ static void ufshcd_print_host_state(struct ufs_hba *hba)
 {
 	dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
 	dev_err(hba->dev, "lrb in use=0x%lx, outstanding reqs=0x%lx tasks=0x%lx\n",
-		hba->lrb_in_use, hba->outstanding_tasks, hba->outstanding_reqs);
+		hba->lrb_in_use, hba->outstanding_reqs, hba->outstanding_tasks);
 	dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n",
 		hba->saved_err, hba->saved_uic_err);
 	dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n",
diff --git a/drivers/scsi/virtio_scsi.c b/drivers/scsi/virtio_scsi.c
index 9be211d68b15..7c28e8d4955a 100644
--- a/drivers/scsi/virtio_scsi.c
+++ b/drivers/scsi/virtio_scsi.c
@@ -818,7 +818,6 @@ static struct scsi_host_template virtscsi_host_template_single = {
 	.eh_timed_out = virtscsi_eh_timed_out,
 	.slave_alloc = virtscsi_device_alloc,
 
-	.can_queue = 1024,
 	.dma_boundary = UINT_MAX,
 	.use_clustering = ENABLE_CLUSTERING,
 	.target_alloc = virtscsi_target_alloc,
@@ -839,7 +838,6 @@ static struct scsi_host_template virtscsi_host_template_multi = {
 	.eh_timed_out = virtscsi_eh_timed_out,
 	.slave_alloc = virtscsi_device_alloc,
 
-	.can_queue = 1024,
 	.dma_boundary = UINT_MAX,
 	.use_clustering = ENABLE_CLUSTERING,
 	.target_alloc = virtscsi_target_alloc,
@@ -972,6 +970,8 @@ static int virtscsi_probe(struct virtio_device *vdev)
 	if (err)
 		goto virtscsi_init_failed;
 
+	shost->can_queue = virtqueue_get_vring_size(vscsi->req_vqs[0].vq);
+
 	cmd_per_lun = virtscsi_config_get(vdev, cmd_per_lun) ?: 1;
 	shost->cmd_per_lun = min_t(u32, cmd_per_lun, shost->can_queue);
 	shost->max_sectors = virtscsi_config_get(vdev, max_sectors) ?: 0xFFFF;
diff --git a/drivers/scsi/wd33c93.c b/drivers/scsi/wd33c93.c
index 9e09da412b92..74be04f2357c 100644
--- a/drivers/scsi/wd33c93.c
+++ b/drivers/scsi/wd33c93.c
@@ -1578,6 +1578,7 @@ wd33c93_host_reset(struct scsi_cmnd * SCpnt)
 	int i;
 
 	instance = SCpnt->device->host;
+	spin_lock_irq(instance->host_lock);
 	hostdata = (struct WD33C93_hostdata *) instance->hostdata;
 
 	printk("scsi%d: reset. ", instance->host_no);
@@ -1603,6 +1604,7 @@ wd33c93_host_reset(struct scsi_cmnd * SCpnt)
 	reset_wd33c93(instance);
 	SCpnt->result = DID_RESET << 16;
 	enable_irq(instance->irq);
+	spin_unlock_irq(instance->host_lock);
 	return SUCCESS;
 }
 
diff --git a/drivers/scsi/zalon.c b/drivers/scsi/zalon.c
index b2cf1faa819d..4722660958f8 100644
--- a/drivers/scsi/zalon.c
+++ b/drivers/scsi/zalon.c
@@ -160,14 +160,14 @@ zalon_probe(struct parisc_device *dev)
 	return error;
 }
 
-static struct parisc_device_id zalon_tbl[] = {
+static const struct parisc_device_id zalon_tbl[] __initconst = {
 	{ HPHW_A_DMA, HVERSION_REV_ANY_ID, HVERSION_ANY_ID, 0x00089 }, 
 	{ 0, }
 };
 
 MODULE_DEVICE_TABLE(parisc, zalon_tbl);
 
-static int zalon_remove(struct parisc_device *dev)
+static int __exit zalon_remove(struct parisc_device *dev)
 {
 	struct Scsi_Host *host = dev_get_drvdata(&dev->dev);
 
@@ -178,11 +178,11 @@ static int zalon_remove(struct parisc_device *dev)
 	return 0;
 }
 
-static struct parisc_driver zalon_driver = {
+static struct parisc_driver zalon_driver __refdata = {
 	.name =		"zalon",
 	.id_table =	zalon_tbl,
 	.probe =	zalon_probe,
-	.remove =	zalon_remove,
+	.remove =	__exit_p(zalon_remove),
 };
 
 static int __init zalon7xx_init(void)
diff --git a/drivers/sfi/sfi_core.c b/drivers/sfi/sfi_core.c
index 296db7a69c27..153b3f3cc795 100644
--- a/drivers/sfi/sfi_core.c
+++ b/drivers/sfi/sfi_core.c
@@ -68,6 +68,7 @@
 #include <linux/init.h>
 #include <linux/sfi.h>
 #include <linux/slab.h>
+#include <linux/io.h>
 
 #include "sfi_core.h"
 
@@ -86,13 +87,13 @@ static struct sfi_table_simple *syst_va __read_mostly;
 /*
  * FW creates and saves the SFI tables in memory. When these tables get
  * used, they may need to be mapped to virtual address space, and the mapping
- * can happen before or after the ioremap() is ready, so a flag is needed
+ * can happen before or after the memremap() is ready, so a flag is needed
  * to indicating this
  */
-static u32 sfi_use_ioremap __read_mostly;
+static u32 sfi_use_memremap __read_mostly;
 
 /*
- * sfi_un/map_memory calls early_ioremap/iounmap which is a __init function
+ * sfi_un/map_memory calls early_memremap/memunmap which is a __init function
  * and introduces section mismatch. So use __ref to make it calm.
  */
 static void __iomem * __ref sfi_map_memory(u64 phys, u32 size)
@@ -100,10 +101,10 @@ static void __iomem * __ref sfi_map_memory(u64 phys, u32 size)
 	if (!phys || !size)
 		return NULL;
 
-	if (sfi_use_ioremap)
-		return ioremap_cache(phys, size);
+	if (sfi_use_memremap)
+		return memremap(phys, size, MEMREMAP_WB);
 	else
-		return early_ioremap(phys, size);
+		return early_memremap(phys, size);
 }
 
 static void __ref sfi_unmap_memory(void __iomem *virt, u32 size)
@@ -111,10 +112,10 @@ static void __ref sfi_unmap_memory(void __iomem *virt, u32 size)
 	if (!virt || !size)
 		return;
 
-	if (sfi_use_ioremap)
-		iounmap(virt);
+	if (sfi_use_memremap)
+		memunmap(virt);
 	else
-		early_iounmap(virt, size);
+		early_memunmap(virt, size);
 }
 
 static void sfi_print_table_header(unsigned long long pa,
@@ -507,8 +508,8 @@ void __init sfi_init_late(void)
 	length = syst_va->header.len;
 	sfi_unmap_memory(syst_va, sizeof(struct sfi_table_simple));
 
-	/* Use ioremap now after it is ready */
-	sfi_use_ioremap = 1;
+	/* Use memremap now after it is ready */
+	sfi_use_memremap = 1;
 	syst_va = sfi_map_memory(syst_pa, length);
 
 	sfi_acpi_init();
diff --git a/drivers/soc/Kconfig b/drivers/soc/Kconfig
index 07fc0ac51c52..fc9e98047421 100644
--- a/drivers/soc/Kconfig
+++ b/drivers/soc/Kconfig
@@ -1,6 +1,7 @@
 menu "SOC (System On Chip) specific Drivers"
 
 source "drivers/soc/actions/Kconfig"
+source "drivers/soc/amlogic/Kconfig"
 source "drivers/soc/atmel/Kconfig"
 source "drivers/soc/bcm/Kconfig"
 source "drivers/soc/fsl/Kconfig"
diff --git a/drivers/soc/Makefile b/drivers/soc/Makefile
index 9241125416ba..2fcaff864584 100644
--- a/drivers/soc/Makefile
+++ b/drivers/soc/Makefile
@@ -9,7 +9,9 @@ obj-$(CONFIG_ARCH_DOVE)		+= dove/
 obj-$(CONFIG_MACH_DOVE)		+= dove/
 obj-y				+= fsl/
 obj-$(CONFIG_ARCH_MXC)		+= imx/
+obj-$(CONFIG_SOC_XWAY)		+= lantiq/
 obj-$(CONFIG_ARCH_MEDIATEK)	+= mediatek/
+obj-$(CONFIG_ARCH_MESON)	+= amlogic/
 obj-$(CONFIG_ARCH_QCOM)		+= qcom/
 obj-y				+= renesas/
 obj-$(CONFIG_ARCH_ROCKCHIP)	+= rockchip/
diff --git a/drivers/soc/amlogic/Kconfig b/drivers/soc/amlogic/Kconfig
new file mode 100644
index 000000000000..22acf064531f
--- /dev/null
+++ b/drivers/soc/amlogic/Kconfig
@@ -0,0 +1,12 @@
+menu "Amlogic SoC drivers"
+
+config MESON_GX_SOCINFO
+	bool "Amlogic Meson GX SoC Information driver"
+	depends on ARCH_MESON || COMPILE_TEST
+	default ARCH_MESON
+	select SOC_BUS
+	help
+	  Say yes to support decoding of Amlogic Meson GX SoC family
+	  information about the type, package and version.
+
+endmenu
diff --git a/drivers/soc/amlogic/Makefile b/drivers/soc/amlogic/Makefile
new file mode 100644
index 000000000000..3e85fc462c21
--- /dev/null
+++ b/drivers/soc/amlogic/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_MESON_GX_SOCINFO) += meson-gx-socinfo.o
diff --git a/drivers/soc/amlogic/meson-gx-socinfo.c b/drivers/soc/amlogic/meson-gx-socinfo.c
new file mode 100644
index 000000000000..89f4cf507be6
--- /dev/null
+++ b/drivers/soc/amlogic/meson-gx-socinfo.c
@@ -0,0 +1,177 @@
+/*
+ * Copyright (c) 2017 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ *
+ * SPDX-License-Identifier: GPL-2.0+
+ */
+
+#include <linux/io.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/sys_soc.h>
+#include <linux/bitfield.h>
+#include <linux/regmap.h>
+#include <linux/mfd/syscon.h>
+
+#define AO_SEC_SD_CFG8		0xe0
+#define AO_SEC_SOCINFO_OFFSET	AO_SEC_SD_CFG8
+
+#define SOCINFO_MAJOR	GENMASK(31, 24)
+#define SOCINFO_MINOR	GENMASK(23, 16)
+#define SOCINFO_PACK	GENMASK(15, 8)
+#define SOCINFO_MISC	GENMASK(7, 0)
+
+static const struct meson_gx_soc_id {
+	const char *name;
+	unsigned int id;
+} soc_ids[] = {
+	{ "GXBB", 0x1f },
+	{ "GXTVBB", 0x20 },
+	{ "GXL", 0x21 },
+	{ "GXM", 0x22 },
+	{ "TXL", 0x23 },
+};
+
+static const struct meson_gx_package_id {
+	const char *name;
+	unsigned int major_id;
+	unsigned int pack_id;
+} soc_packages[] = {
+	{ "S905", 0x1f, 0 },
+	{ "S905M", 0x1f, 0x20 },
+	{ "S905D", 0x21, 0 },
+	{ "S905X", 0x21, 0x80 },
+	{ "S905L", 0x21, 0xc0 },
+	{ "S905M2", 0x21, 0xe0 },
+	{ "S912", 0x22, 0 },
+};
+
+static inline unsigned int socinfo_to_major(u32 socinfo)
+{
+	return FIELD_GET(SOCINFO_MAJOR, socinfo);
+}
+
+static inline unsigned int socinfo_to_minor(u32 socinfo)
+{
+	return FIELD_GET(SOCINFO_MINOR, socinfo);
+}
+
+static inline unsigned int socinfo_to_pack(u32 socinfo)
+{
+	return FIELD_GET(SOCINFO_PACK, socinfo);
+}
+
+static inline unsigned int socinfo_to_misc(u32 socinfo)
+{
+	return FIELD_GET(SOCINFO_MISC, socinfo);
+}
+
+static const char *socinfo_to_package_id(u32 socinfo)
+{
+	unsigned int pack = socinfo_to_pack(socinfo) & 0xf0;
+	unsigned int major = socinfo_to_major(socinfo);
+	int i;
+
+	for (i = 0 ; i < ARRAY_SIZE(soc_packages) ; ++i) {
+		if (soc_packages[i].major_id == major &&
+		    soc_packages[i].pack_id == pack)
+			return soc_packages[i].name;
+	}
+
+	return "Unknown";
+}
+
+static const char *socinfo_to_soc_id(u32 socinfo)
+{
+	unsigned int id = socinfo_to_major(socinfo);
+	int i;
+
+	for (i = 0 ; i < ARRAY_SIZE(soc_ids) ; ++i) {
+		if (soc_ids[i].id == id)
+			return soc_ids[i].name;
+	}
+
+	return "Unknown";
+}
+
+int __init meson_gx_socinfo_init(void)
+{
+	struct soc_device_attribute *soc_dev_attr;
+	struct soc_device *soc_dev;
+	struct device_node *np;
+	struct regmap *regmap;
+	unsigned int socinfo;
+	struct device *dev;
+	int ret;
+
+	/* look up for chipid node */
+	np = of_find_compatible_node(NULL, NULL, "amlogic,meson-gx-ao-secure");
+	if (!np)
+		return -ENODEV;
+
+	/* check if interface is enabled */
+	if (!of_device_is_available(np))
+		return -ENODEV;
+
+	/* check if chip-id is available */
+	if (!of_property_read_bool(np, "amlogic,has-chip-id"))
+		return -ENODEV;
+
+	/* node should be a syscon */
+	regmap = syscon_node_to_regmap(np);
+	of_node_put(np);
+	if (IS_ERR(regmap)) {
+		pr_err("%s: failed to get regmap\n", __func__);
+		return -ENODEV;
+	}
+
+	ret = regmap_read(regmap, AO_SEC_SOCINFO_OFFSET, &socinfo);
+	if (ret < 0)
+		return ret;
+
+	if (!socinfo) {
+		pr_err("%s: invalid chipid value\n", __func__);
+		return -EINVAL;
+	}
+
+	soc_dev_attr = kzalloc(sizeof(*soc_dev_attr), GFP_KERNEL);
+	if (!soc_dev_attr)
+		return -ENODEV;
+
+	soc_dev_attr->family = "Amlogic Meson";
+
+	np = of_find_node_by_path("/");
+	of_property_read_string(np, "model", &soc_dev_attr->machine);
+	of_node_put(np);
+
+	soc_dev_attr->revision = kasprintf(GFP_KERNEL, "%x:%x - %x:%x",
+					   socinfo_to_major(socinfo),
+					   socinfo_to_minor(socinfo),
+					   socinfo_to_pack(socinfo),
+					   socinfo_to_misc(socinfo));
+	soc_dev_attr->soc_id = kasprintf(GFP_KERNEL, "%s (%s)",
+					 socinfo_to_soc_id(socinfo),
+					 socinfo_to_package_id(socinfo));
+
+	soc_dev = soc_device_register(soc_dev_attr);
+	if (IS_ERR(soc_dev)) {
+		kfree(soc_dev_attr->revision);
+		kfree_const(soc_dev_attr->soc_id);
+		kfree(soc_dev_attr);
+		return PTR_ERR(soc_dev);
+	}
+	dev = soc_device_to_device(soc_dev);
+
+	dev_info(dev, "Amlogic Meson %s Revision %x:%x (%x:%x) Detected\n",
+			soc_dev_attr->soc_id,
+			socinfo_to_major(socinfo),
+			socinfo_to_minor(socinfo),
+			socinfo_to_pack(socinfo),
+			socinfo_to_misc(socinfo));
+
+	return 0;
+}
+device_initcall(meson_gx_socinfo_init);
diff --git a/drivers/soc/fsl/qbman/bman_ccsr.c b/drivers/soc/fsl/qbman/bman_ccsr.c
index a8e8389a6894..eaa9585c7347 100644
--- a/drivers/soc/fsl/qbman/bman_ccsr.c
+++ b/drivers/soc/fsl/qbman/bman_ccsr.c
@@ -177,8 +177,8 @@ static int fsl_bman_probe(struct platform_device *pdev)
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	if (!res) {
-		dev_err(dev, "Can't get %s property 'IORESOURCE_MEM'\n",
-			node->full_name);
+		dev_err(dev, "Can't get %pOF property 'IORESOURCE_MEM'\n",
+			node);
 		return -ENXIO;
 	}
 	bm_ccsr_start = devm_ioremap(dev, res->start, resource_size(res));
@@ -205,14 +205,14 @@ static int fsl_bman_probe(struct platform_device *pdev)
 
 	err_irq = platform_get_irq(pdev, 0);
 	if (err_irq <= 0) {
-		dev_info(dev, "Can't get %s IRQ\n", node->full_name);
+		dev_info(dev, "Can't get %pOF IRQ\n", node);
 		return -ENODEV;
 	}
 	ret = devm_request_irq(dev, err_irq, bman_isr, IRQF_SHARED, "bman-err",
 			       dev);
 	if (ret)  {
-		dev_err(dev, "devm_request_irq() failed %d for '%s'\n",
-			ret, node->full_name);
+		dev_err(dev, "devm_request_irq() failed %d for '%pOF'\n",
+			ret, node);
 		return ret;
 	}
 	/* Disable Buffer Pool State Change */
diff --git a/drivers/soc/fsl/qbman/bman_portal.c b/drivers/soc/fsl/qbman/bman_portal.c
index 8354d4dabdad..39b39c8f1399 100644
--- a/drivers/soc/fsl/qbman/bman_portal.c
+++ b/drivers/soc/fsl/qbman/bman_portal.c
@@ -103,16 +103,14 @@ static int bman_portal_probe(struct platform_device *pdev)
 	addr_phys[0] = platform_get_resource(pdev, IORESOURCE_MEM,
 					     DPAA_PORTAL_CE);
 	if (!addr_phys[0]) {
-		dev_err(dev, "Can't get %s property 'reg::CE'\n",
-			node->full_name);
+		dev_err(dev, "Can't get %pOF property 'reg::CE'\n", node);
 		return -ENXIO;
 	}
 
 	addr_phys[1] = platform_get_resource(pdev, IORESOURCE_MEM,
 					     DPAA_PORTAL_CI);
 	if (!addr_phys[1]) {
-		dev_err(dev, "Can't get %s property 'reg::CI'\n",
-			node->full_name);
+		dev_err(dev, "Can't get %pOF property 'reg::CI'\n", node);
 		return -ENXIO;
 	}
 
@@ -120,7 +118,7 @@ static int bman_portal_probe(struct platform_device *pdev)
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq <= 0) {
-		dev_err(dev, "Can't get %s IRQ'\n", node->full_name);
+		dev_err(dev, "Can't get %pOF IRQ'\n", node);
 		return -ENXIO;
 	}
 	pcfg->irq = irq;
diff --git a/drivers/soc/fsl/qbman/qman_ccsr.c b/drivers/soc/fsl/qbman/qman_ccsr.c
index 90bc40c48675..835ce947ffca 100644
--- a/drivers/soc/fsl/qbman/qman_ccsr.c
+++ b/drivers/soc/fsl/qbman/qman_ccsr.c
@@ -695,8 +695,8 @@ static int fsl_qman_probe(struct platform_device *pdev)
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	if (!res) {
-		dev_err(dev, "Can't get %s property 'IORESOURCE_MEM'\n",
-			node->full_name);
+		dev_err(dev, "Can't get %pOF property 'IORESOURCE_MEM'\n",
+			node);
 		return -ENXIO;
 	}
 	qm_ccsr_start = devm_ioremap(dev, res->start, resource_size(res));
@@ -740,15 +740,15 @@ static int fsl_qman_probe(struct platform_device *pdev)
 
 	err_irq = platform_get_irq(pdev, 0);
 	if (err_irq <= 0) {
-		dev_info(dev, "Can't get %s property 'interrupts'\n",
-			 node->full_name);
+		dev_info(dev, "Can't get %pOF property 'interrupts'\n",
+			 node);
 		return -ENODEV;
 	}
 	ret = devm_request_irq(dev, err_irq, qman_isr, IRQF_SHARED, "qman-err",
 			       dev);
 	if (ret)  {
-		dev_err(dev, "devm_request_irq() failed %d for '%s'\n",
-			ret, node->full_name);
+		dev_err(dev, "devm_request_irq() failed %d for '%pOF'\n",
+			ret, node);
 		return ret;
 	}
 
diff --git a/drivers/soc/fsl/qbman/qman_portal.c b/drivers/soc/fsl/qbman/qman_portal.c
index adbaa30d3c5a..cbacdf4f98ed 100644
--- a/drivers/soc/fsl/qbman/qman_portal.c
+++ b/drivers/soc/fsl/qbman/qman_portal.c
@@ -237,30 +237,27 @@ static int qman_portal_probe(struct platform_device *pdev)
 	addr_phys[0] = platform_get_resource(pdev, IORESOURCE_MEM,
 					     DPAA_PORTAL_CE);
 	if (!addr_phys[0]) {
-		dev_err(dev, "Can't get %s property 'reg::CE'\n",
-			node->full_name);
+		dev_err(dev, "Can't get %pOF property 'reg::CE'\n", node);
 		return -ENXIO;
 	}
 
 	addr_phys[1] = platform_get_resource(pdev, IORESOURCE_MEM,
 					     DPAA_PORTAL_CI);
 	if (!addr_phys[1]) {
-		dev_err(dev, "Can't get %s property 'reg::CI'\n",
-			node->full_name);
+		dev_err(dev, "Can't get %pOF property 'reg::CI'\n", node);
 		return -ENXIO;
 	}
 
 	err = of_property_read_u32(node, "cell-index", &val);
 	if (err) {
-		dev_err(dev, "Can't get %s property 'cell-index'\n",
-			node->full_name);
+		dev_err(dev, "Can't get %pOF property 'cell-index'\n", node);
 		return err;
 	}
 	pcfg->channel = val;
 	pcfg->cpu = -1;
 	irq = platform_get_irq(pdev, 0);
 	if (irq <= 0) {
-		dev_err(dev, "Can't get %s IRQ\n", node->full_name);
+		dev_err(dev, "Can't get %pOF IRQ\n", node);
 		return -ENXIO;
 	}
 	pcfg->irq = irq;
diff --git a/drivers/soc/fsl/qe/gpio.c b/drivers/soc/fsl/qe/gpio.c
index 0aaf429f31d5..3b27075c21a7 100644
--- a/drivers/soc/fsl/qe/gpio.c
+++ b/drivers/soc/fsl/qe/gpio.c
@@ -304,8 +304,8 @@ static int __init qe_add_gpiochips(void)
 			goto err;
 		continue;
 err:
-		pr_err("%s: registration failed with status %d\n",
-		       np->full_name, ret);
+		pr_err("%pOF: registration failed with status %d\n",
+		       np, ret);
 		kfree(qe_gc);
 		/* try others anyway */
 	}
diff --git a/drivers/soc/imx/gpcv2.c b/drivers/soc/imx/gpcv2.c
index 3039072911a5..afc7ecc3c187 100644
--- a/drivers/soc/imx/gpcv2.c
+++ b/drivers/soc/imx/gpcv2.c
@@ -200,16 +200,11 @@ static int imx7_pgc_domain_probe(struct platform_device *pdev)
 
 	domain->dev = &pdev->dev;
 
-	ret = pm_genpd_init(&domain->genpd, NULL, true);
-	if (ret) {
-		dev_err(domain->dev, "Failed to init power domain\n");
-		return ret;
-	}
-
 	domain->regulator = devm_regulator_get_optional(domain->dev, "power");
 	if (IS_ERR(domain->regulator)) {
 		if (PTR_ERR(domain->regulator) != -ENODEV) {
-			dev_err(domain->dev, "Failed to get domain's regulator\n");
+			if (PTR_ERR(domain->regulator) != -EPROBE_DEFER)
+				dev_err(domain->dev, "Failed to get domain's regulator\n");
 			return PTR_ERR(domain->regulator);
 		}
 	} else {
@@ -217,6 +212,12 @@ static int imx7_pgc_domain_probe(struct platform_device *pdev)
 				      domain->voltage, domain->voltage);
 	}
 
+	ret = pm_genpd_init(&domain->genpd, NULL, true);
+	if (ret) {
+		dev_err(domain->dev, "Failed to init power domain\n");
+		return ret;
+	}
+
 	ret = of_genpd_add_provider_simple(domain->dev->of_node,
 					   &domain->genpd);
 	if (ret) {
diff --git a/drivers/soc/lantiq/Makefile b/drivers/soc/lantiq/Makefile
new file mode 100644
index 000000000000..be9e866d53e5
--- /dev/null
+++ b/drivers/soc/lantiq/Makefile
@@ -0,0 +1,2 @@
+obj-y				+= fpi-bus.o
+obj-$(CONFIG_XRX200_PHY_FW)	+= gphy.o
diff --git a/drivers/soc/lantiq/fpi-bus.c b/drivers/soc/lantiq/fpi-bus.c
new file mode 100644
index 000000000000..a671c9984c4c
--- /dev/null
+++ b/drivers/soc/lantiq/fpi-bus.c
@@ -0,0 +1,87 @@
+/*
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License version 2 as published
+ *  by the Free Software Foundation.
+ *
+ *  Copyright (C) 2011-2015 John Crispin <blogic@phrozen.org>
+ *  Copyright (C) 2015 Martin Blumenstingl <martin.blumenstingl@googlemail.com>
+ *  Copyright (C) 2017 Hauke Mehrtens <hauke@hauke-m.de>
+ */
+
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+
+#include <lantiq_soc.h>
+
+#define XBAR_ALWAYS_LAST	0x430
+#define XBAR_FPI_BURST_EN	BIT(1)
+#define XBAR_AHB_BURST_EN	BIT(2)
+
+#define RCU_VR9_BE_AHB1S	0x00000008
+
+static int ltq_fpi_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct resource *res_xbar;
+	struct regmap *rcu_regmap;
+	void __iomem *xbar_membase;
+	u32 rcu_ahb_endianness_reg_offset;
+	int ret;
+
+	res_xbar = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	xbar_membase = devm_ioremap_resource(dev, res_xbar);
+	if (IS_ERR(xbar_membase))
+		return PTR_ERR(xbar_membase);
+
+	/* RCU configuration is optional */
+	rcu_regmap = syscon_regmap_lookup_by_phandle(np, "lantiq,rcu");
+	if (IS_ERR(rcu_regmap))
+		return PTR_ERR(rcu_regmap);
+
+	ret = device_property_read_u32(dev, "lantiq,offset-endianness",
+				       &rcu_ahb_endianness_reg_offset);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to get RCU reg offset\n");
+		return ret;
+	}
+
+	ret = regmap_update_bits(rcu_regmap, rcu_ahb_endianness_reg_offset,
+				 RCU_VR9_BE_AHB1S, RCU_VR9_BE_AHB1S);
+	if (ret) {
+		dev_warn(&pdev->dev,
+			 "Failed to configure RCU AHB endianness\n");
+		return ret;
+	}
+
+	/* disable fpi burst */
+	ltq_w32_mask(XBAR_FPI_BURST_EN, 0, xbar_membase + XBAR_ALWAYS_LAST);
+
+	return of_platform_populate(dev->of_node, NULL, NULL, dev);
+}
+
+static const struct of_device_id ltq_fpi_match[] = {
+	{ .compatible = "lantiq,xrx200-fpi" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, ltq_fpi_match);
+
+static struct platform_driver ltq_fpi_driver = {
+	.probe = ltq_fpi_probe,
+	.driver = {
+		.name = "fpi-xway",
+		.of_match_table = ltq_fpi_match,
+	},
+};
+
+module_platform_driver(ltq_fpi_driver);
+
+MODULE_DESCRIPTION("Lantiq FPI bus driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/soc/lantiq/gphy.c b/drivers/soc/lantiq/gphy.c
new file mode 100644
index 000000000000..8d8659463b3e
--- /dev/null
+++ b/drivers/soc/lantiq/gphy.c
@@ -0,0 +1,260 @@
+/*
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License version 2 as published
+ *  by the Free Software Foundation.
+ *
+ *  Copyright (C) 2012 John Crispin <blogic@phrozen.org>
+ *  Copyright (C) 2016 Martin Blumenstingl <martin.blumenstingl@googlemail.com>
+ *  Copyright (C) 2017 Hauke Mehrtens <hauke@hauke-m.de>
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/firmware.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/reboot.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+#include <linux/of_device.h>
+#include <linux/of_platform.h>
+#include <linux/property.h>
+#include <dt-bindings/mips/lantiq_rcu_gphy.h>
+
+#include <lantiq_soc.h>
+
+#define XRX200_GPHY_FW_ALIGN	(16 * 1024)
+
+struct xway_gphy_priv {
+	struct clk *gphy_clk_gate;
+	struct reset_control *gphy_reset;
+	struct reset_control *gphy_reset2;
+	struct notifier_block gphy_reboot_nb;
+	void __iomem *membase;
+	char *fw_name;
+};
+
+struct xway_gphy_match_data {
+	char *fe_firmware_name;
+	char *ge_firmware_name;
+};
+
+static const struct xway_gphy_match_data xrx200a1x_gphy_data = {
+	.fe_firmware_name = "lantiq/xrx200_phy22f_a14.bin",
+	.ge_firmware_name = "lantiq/xrx200_phy11g_a14.bin",
+};
+
+static const struct xway_gphy_match_data xrx200a2x_gphy_data = {
+	.fe_firmware_name = "lantiq/xrx200_phy22f_a22.bin",
+	.ge_firmware_name = "lantiq/xrx200_phy11g_a22.bin",
+};
+
+static const struct xway_gphy_match_data xrx300_gphy_data = {
+	.fe_firmware_name = "lantiq/xrx300_phy22f_a21.bin",
+	.ge_firmware_name = "lantiq/xrx300_phy11g_a21.bin",
+};
+
+static const struct of_device_id xway_gphy_match[] = {
+	{ .compatible = "lantiq,xrx200a1x-gphy", .data = &xrx200a1x_gphy_data },
+	{ .compatible = "lantiq,xrx200a2x-gphy", .data = &xrx200a2x_gphy_data },
+	{ .compatible = "lantiq,xrx300-gphy", .data = &xrx300_gphy_data },
+	{ .compatible = "lantiq,xrx330-gphy", .data = &xrx300_gphy_data },
+	{},
+};
+MODULE_DEVICE_TABLE(of, xway_gphy_match);
+
+static struct xway_gphy_priv *to_xway_gphy_priv(struct notifier_block *nb)
+{
+	return container_of(nb, struct xway_gphy_priv, gphy_reboot_nb);
+}
+
+static int xway_gphy_reboot_notify(struct notifier_block *reboot_nb,
+				   unsigned long code, void *unused)
+{
+	struct xway_gphy_priv *priv = to_xway_gphy_priv(reboot_nb);
+
+	if (priv) {
+		reset_control_assert(priv->gphy_reset);
+		reset_control_assert(priv->gphy_reset2);
+	}
+
+	return NOTIFY_DONE;
+}
+
+static int xway_gphy_load(struct device *dev, struct xway_gphy_priv *priv,
+			  dma_addr_t *dev_addr)
+{
+	const struct firmware *fw;
+	void *fw_addr;
+	dma_addr_t dma_addr;
+	size_t size;
+	int ret;
+
+	ret = request_firmware(&fw, priv->fw_name, dev);
+	if (ret) {
+		dev_err(dev, "failed to load firmware: %s, error: %i\n",
+			priv->fw_name, ret);
+		return ret;
+	}
+
+	/*
+	 * GPHY cores need the firmware code in a persistent and contiguous
+	 * memory area with a 16 kB boundary aligned start address.
+	 */
+	size = fw->size + XRX200_GPHY_FW_ALIGN;
+
+	fw_addr = dmam_alloc_coherent(dev, size, &dma_addr, GFP_KERNEL);
+	if (fw_addr) {
+		fw_addr = PTR_ALIGN(fw_addr, XRX200_GPHY_FW_ALIGN);
+		*dev_addr = ALIGN(dma_addr, XRX200_GPHY_FW_ALIGN);
+		memcpy(fw_addr, fw->data, fw->size);
+	} else {
+		dev_err(dev, "failed to alloc firmware memory\n");
+		ret = -ENOMEM;
+	}
+
+	release_firmware(fw);
+
+	return ret;
+}
+
+static int xway_gphy_of_probe(struct platform_device *pdev,
+			      struct xway_gphy_priv *priv)
+{
+	struct device *dev = &pdev->dev;
+	const struct xway_gphy_match_data *gphy_fw_name_cfg;
+	u32 gphy_mode;
+	int ret;
+	struct resource *res_gphy;
+
+	gphy_fw_name_cfg = of_device_get_match_data(dev);
+
+	priv->gphy_clk_gate = devm_clk_get(dev, NULL);
+	if (IS_ERR(priv->gphy_clk_gate)) {
+		dev_err(dev, "Failed to lookup gate clock\n");
+		return PTR_ERR(priv->gphy_clk_gate);
+	}
+
+	res_gphy = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	priv->membase = devm_ioremap_resource(dev, res_gphy);
+	if (IS_ERR(priv->membase))
+		return PTR_ERR(priv->membase);
+
+	priv->gphy_reset = devm_reset_control_get(dev, "gphy");
+	if (IS_ERR(priv->gphy_reset)) {
+		if (PTR_ERR(priv->gphy_reset) != -EPROBE_DEFER)
+			dev_err(dev, "Failed to lookup gphy reset\n");
+		return PTR_ERR(priv->gphy_reset);
+	}
+
+	priv->gphy_reset2 = devm_reset_control_get_optional(dev, "gphy2");
+	if (IS_ERR(priv->gphy_reset2))
+		return PTR_ERR(priv->gphy_reset2);
+
+	ret = device_property_read_u32(dev, "lantiq,gphy-mode", &gphy_mode);
+	/* Default to GE mode */
+	if (ret)
+		gphy_mode = GPHY_MODE_GE;
+
+	switch (gphy_mode) {
+	case GPHY_MODE_FE:
+		priv->fw_name = gphy_fw_name_cfg->fe_firmware_name;
+		break;
+	case GPHY_MODE_GE:
+		priv->fw_name = gphy_fw_name_cfg->ge_firmware_name;
+		break;
+	default:
+		dev_err(dev, "Unknown GPHY mode %d\n", gphy_mode);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int xway_gphy_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct xway_gphy_priv *priv;
+	dma_addr_t fw_addr = 0;
+	int ret;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	ret = xway_gphy_of_probe(pdev, priv);
+	if (ret)
+		return ret;
+
+	ret = clk_prepare_enable(priv->gphy_clk_gate);
+	if (ret)
+		return ret;
+
+	ret = xway_gphy_load(dev, priv, &fw_addr);
+	if (ret) {
+		clk_disable_unprepare(priv->gphy_clk_gate);
+		return ret;
+	}
+
+	reset_control_assert(priv->gphy_reset);
+	reset_control_assert(priv->gphy_reset2);
+
+	iowrite32be(fw_addr, priv->membase);
+
+	reset_control_deassert(priv->gphy_reset);
+	reset_control_deassert(priv->gphy_reset2);
+
+	/* assert the gphy reset because it can hang after a reboot: */
+	priv->gphy_reboot_nb.notifier_call = xway_gphy_reboot_notify;
+	priv->gphy_reboot_nb.priority = -1;
+
+	ret = register_reboot_notifier(&priv->gphy_reboot_nb);
+	if (ret)
+		dev_warn(dev, "Failed to register reboot notifier\n");
+
+	platform_set_drvdata(pdev, priv);
+
+	return ret;
+}
+
+static int xway_gphy_remove(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct xway_gphy_priv *priv = platform_get_drvdata(pdev);
+	int ret;
+
+	reset_control_assert(priv->gphy_reset);
+	reset_control_assert(priv->gphy_reset2);
+
+	iowrite32be(0, priv->membase);
+
+	clk_disable_unprepare(priv->gphy_clk_gate);
+
+	ret = unregister_reboot_notifier(&priv->gphy_reboot_nb);
+	if (ret)
+		dev_warn(dev, "Failed to unregister reboot notifier\n");
+
+	return 0;
+}
+
+static struct platform_driver xway_gphy_driver = {
+	.probe = xway_gphy_probe,
+	.remove = xway_gphy_remove,
+	.driver = {
+		.name = "xway-rcu-gphy",
+		.of_match_table = xway_gphy_match,
+	},
+};
+
+module_platform_driver(xway_gphy_driver);
+
+MODULE_FIRMWARE("lantiq/xrx300_phy11g_a21.bin");
+MODULE_FIRMWARE("lantiq/xrx300_phy22f_a21.bin");
+MODULE_FIRMWARE("lantiq/xrx200_phy11g_a14.bin");
+MODULE_FIRMWARE("lantiq/xrx200_phy11g_a22.bin");
+MODULE_FIRMWARE("lantiq/xrx200_phy22f_a14.bin");
+MODULE_FIRMWARE("lantiq/xrx200_phy22f_a22.bin");
+MODULE_AUTHOR("Martin Blumenstingl <martin.blumenstingl@googlemail.com>");
+MODULE_DESCRIPTION("Lantiq XWAY GPHY Firmware Loader");
+MODULE_LICENSE("GPL");
diff --git a/drivers/soc/mediatek/mtk-pmic-wrap.c b/drivers/soc/mediatek/mtk-pmic-wrap.c
index c80a04e1b2b1..c2048382830f 100644
--- a/drivers/soc/mediatek/mtk-pmic-wrap.c
+++ b/drivers/soc/mediatek/mtk-pmic-wrap.c
@@ -1067,7 +1067,7 @@ static const struct pmic_wrapper_type pwrap_mt2701 = {
 	.init_soc_specific = pwrap_mt2701_init_soc_specific,
 };
 
-static struct pmic_wrapper_type pwrap_mt8135 = {
+static const struct pmic_wrapper_type pwrap_mt8135 = {
 	.regs = mt8135_regs,
 	.type = PWRAP_MT8135,
 	.arb_en_all = 0x1ff,
@@ -1079,7 +1079,7 @@ static struct pmic_wrapper_type pwrap_mt8135 = {
 	.init_soc_specific = pwrap_mt8135_init_soc_specific,
 };
 
-static struct pmic_wrapper_type pwrap_mt8173 = {
+static const struct pmic_wrapper_type pwrap_mt8173 = {
 	.regs = mt8173_regs,
 	.type = PWRAP_MT8173,
 	.arb_en_all = 0x3f,
@@ -1091,7 +1091,7 @@ static struct pmic_wrapper_type pwrap_mt8173 = {
 	.init_soc_specific = pwrap_mt8173_init_soc_specific,
 };
 
-static struct of_device_id of_pwrap_match_tbl[] = {
+static const struct of_device_id of_pwrap_match_tbl[] = {
 	{
 		.compatible = "mediatek,mt2701-pwrap",
 		.data = &pwrap_mt2701,
@@ -1233,8 +1233,8 @@ static int pwrap_probe(struct platform_device *pdev)
 
 	ret = of_platform_populate(np, NULL, NULL, wrp->dev);
 	if (ret) {
-		dev_dbg(wrp->dev, "failed to create child devices at %s\n",
-				np->full_name);
+		dev_dbg(wrp->dev, "failed to create child devices at %pOF\n",
+				np);
 		goto err_out2;
 	}
 
diff --git a/drivers/soc/mediatek/mtk-scpsys.c b/drivers/soc/mediatek/mtk-scpsys.c
index ceb2cc495cd0..e1ce8b1b5090 100644
--- a/drivers/soc/mediatek/mtk-scpsys.c
+++ b/drivers/soc/mediatek/mtk-scpsys.c
@@ -22,6 +22,7 @@
 
 #include <dt-bindings/power/mt2701-power.h>
 #include <dt-bindings/power/mt6797-power.h>
+#include <dt-bindings/power/mt7622-power.h>
 #include <dt-bindings/power/mt8173-power.h>
 
 #define SPM_VDE_PWR_CON			0x0210
@@ -39,6 +40,11 @@
 #define SPM_MFG_2D_PWR_CON		0x02c0
 #define SPM_MFG_ASYNC_PWR_CON		0x02c4
 #define SPM_USB_PWR_CON			0x02cc
+#define SPM_ETHSYS_PWR_CON		0x02e0	/* MT7622 */
+#define SPM_HIF0_PWR_CON		0x02e4	/* MT7622 */
+#define SPM_HIF1_PWR_CON		0x02e8	/* MT7622 */
+#define SPM_WB_PWR_CON			0x02ec	/* MT7622 */
+
 
 #define SPM_PWR_STATUS			0x060c
 #define SPM_PWR_STATUS_2ND		0x0610
@@ -64,6 +70,10 @@
 #define PWR_STATUS_MFG_ASYNC		BIT(23)
 #define PWR_STATUS_AUDIO		BIT(24)
 #define PWR_STATUS_USB			BIT(25)
+#define PWR_STATUS_ETHSYS		BIT(24)	/* MT7622 */
+#define PWR_STATUS_HIF0			BIT(25)	/* MT7622 */
+#define PWR_STATUS_HIF1			BIT(26)	/* MT7622 */
+#define PWR_STATUS_WB			BIT(27)	/* MT7622 */
 
 enum clk_id {
 	CLK_NONE,
@@ -73,6 +83,7 @@ enum clk_id {
 	CLK_VENC_LT,
 	CLK_ETHIF,
 	CLK_VDEC,
+	CLK_HIFSEL,
 	CLK_MAX,
 };
 
@@ -84,6 +95,7 @@ static const char * const clk_names[] = {
 	"venc_lt",
 	"ethif",
 	"vdec",
+	"hif_sel",
 	NULL,
 };
 
@@ -124,6 +136,19 @@ struct scp {
 	struct scp_ctrl_reg ctrl_reg;
 };
 
+struct scp_subdomain {
+	int origin;
+	int subdomain;
+};
+
+struct scp_soc_data {
+	const struct scp_domain_data *domains;
+	int num_domains;
+	const struct scp_subdomain *subdomains;
+	int num_subdomains;
+	const struct scp_ctrl_reg regs;
+};
+
 static int scpsys_domain_is_on(struct scp_domain *scpd)
 {
 	struct scp *scp = scpd->scp;
@@ -357,7 +382,7 @@ static void init_clks(struct platform_device *pdev, struct clk **clk)
 
 static struct scp *init_scp(struct platform_device *pdev,
 			const struct scp_domain_data *scp_domain_data, int num,
-			struct scp_ctrl_reg *scp_ctrl_reg)
+			const struct scp_ctrl_reg *scp_ctrl_reg)
 {
 	struct genpd_onecell_data *pd_data;
 	struct resource *res;
@@ -565,26 +590,6 @@ static const struct scp_domain_data scp_domain_data_mt2701[] = {
 	},
 };
 
-#define NUM_DOMAINS_MT2701	ARRAY_SIZE(scp_domain_data_mt2701)
-
-static int __init scpsys_probe_mt2701(struct platform_device *pdev)
-{
-	struct scp *scp;
-	struct scp_ctrl_reg scp_reg;
-
-	scp_reg.pwr_sta_offs = SPM_PWR_STATUS;
-	scp_reg.pwr_sta2nd_offs = SPM_PWR_STATUS_2ND;
-
-	scp = init_scp(pdev, scp_domain_data_mt2701, NUM_DOMAINS_MT2701,
-		       &scp_reg);
-	if (IS_ERR(scp))
-		return PTR_ERR(scp);
-
-	mtk_register_power_domains(pdev, scp, NUM_DOMAINS_MT2701);
-
-	return 0;
-}
-
 /*
  * MT6797 power domain support
  */
@@ -649,51 +654,62 @@ static const struct scp_domain_data scp_domain_data_mt6797[] = {
 	},
 };
 
-#define NUM_DOMAINS_MT6797	ARRAY_SIZE(scp_domain_data_mt6797)
 #define SPM_PWR_STATUS_MT6797		0x0180
 #define SPM_PWR_STATUS_2ND_MT6797	0x0184
 
-static int __init scpsys_probe_mt6797(struct platform_device *pdev)
-{
-	struct scp *scp;
-	struct genpd_onecell_data *pd_data;
-	int ret;
-	struct scp_ctrl_reg scp_reg;
-
-	scp_reg.pwr_sta_offs = SPM_PWR_STATUS_MT6797;
-	scp_reg.pwr_sta2nd_offs = SPM_PWR_STATUS_2ND_MT6797;
-
-	scp = init_scp(pdev, scp_domain_data_mt6797, NUM_DOMAINS_MT6797,
-		       &scp_reg);
-	if (IS_ERR(scp))
-		return PTR_ERR(scp);
-
-	mtk_register_power_domains(pdev, scp, NUM_DOMAINS_MT6797);
-
-	pd_data = &scp->pd_data;
-
-	ret = pm_genpd_add_subdomain(pd_data->domains[MT6797_POWER_DOMAIN_MM],
-				     pd_data->domains[MT6797_POWER_DOMAIN_VDEC]);
-	if (ret && IS_ENABLED(CONFIG_PM))
-		dev_err(&pdev->dev, "Failed to add subdomain: %d\n", ret);
-
-	ret = pm_genpd_add_subdomain(pd_data->domains[MT6797_POWER_DOMAIN_MM],
-				     pd_data->domains[MT6797_POWER_DOMAIN_ISP]);
-	if (ret && IS_ENABLED(CONFIG_PM))
-		dev_err(&pdev->dev, "Failed to add subdomain: %d\n", ret);
-
-	ret = pm_genpd_add_subdomain(pd_data->domains[MT6797_POWER_DOMAIN_MM],
-				     pd_data->domains[MT6797_POWER_DOMAIN_VENC]);
-	if (ret && IS_ENABLED(CONFIG_PM))
-		dev_err(&pdev->dev, "Failed to add subdomain: %d\n", ret);
+static const struct scp_subdomain scp_subdomain_mt6797[] = {
+	{MT6797_POWER_DOMAIN_MM, MT6797_POWER_DOMAIN_VDEC},
+	{MT6797_POWER_DOMAIN_MM, MT6797_POWER_DOMAIN_ISP},
+	{MT6797_POWER_DOMAIN_MM, MT6797_POWER_DOMAIN_VENC},
+	{MT6797_POWER_DOMAIN_MM, MT6797_POWER_DOMAIN_MJC},
+};
 
-	ret = pm_genpd_add_subdomain(pd_data->domains[MT6797_POWER_DOMAIN_MM],
-				     pd_data->domains[MT6797_POWER_DOMAIN_MJC]);
-	if (ret && IS_ENABLED(CONFIG_PM))
-		dev_err(&pdev->dev, "Failed to add subdomain: %d\n", ret);
+/*
+ * MT7622 power domain support
+ */
 
-	return 0;
-}
+static const struct scp_domain_data scp_domain_data_mt7622[] = {
+	[MT7622_POWER_DOMAIN_ETHSYS] = {
+		.name = "ethsys",
+		.sta_mask = PWR_STATUS_ETHSYS,
+		.ctl_offs = SPM_ETHSYS_PWR_CON,
+		.sram_pdn_bits = GENMASK(11, 8),
+		.sram_pdn_ack_bits = GENMASK(15, 12),
+		.clk_id = {CLK_NONE},
+		.bus_prot_mask = MT7622_TOP_AXI_PROT_EN_ETHSYS,
+		.active_wakeup = true,
+	},
+	[MT7622_POWER_DOMAIN_HIF0] = {
+		.name = "hif0",
+		.sta_mask = PWR_STATUS_HIF0,
+		.ctl_offs = SPM_HIF0_PWR_CON,
+		.sram_pdn_bits = GENMASK(11, 8),
+		.sram_pdn_ack_bits = GENMASK(15, 12),
+		.clk_id = {CLK_HIFSEL},
+		.bus_prot_mask = MT7622_TOP_AXI_PROT_EN_HIF0,
+		.active_wakeup = true,
+	},
+	[MT7622_POWER_DOMAIN_HIF1] = {
+		.name = "hif1",
+		.sta_mask = PWR_STATUS_HIF1,
+		.ctl_offs = SPM_HIF1_PWR_CON,
+		.sram_pdn_bits = GENMASK(11, 8),
+		.sram_pdn_ack_bits = GENMASK(15, 12),
+		.clk_id = {CLK_HIFSEL},
+		.bus_prot_mask = MT7622_TOP_AXI_PROT_EN_HIF1,
+		.active_wakeup = true,
+	},
+	[MT7622_POWER_DOMAIN_WB] = {
+		.name = "wb",
+		.sta_mask = PWR_STATUS_WB,
+		.ctl_offs = SPM_WB_PWR_CON,
+		.sram_pdn_bits = 0,
+		.sram_pdn_ack_bits = 0,
+		.clk_id = {CLK_NONE},
+		.bus_prot_mask = MT7622_TOP_AXI_PROT_EN_WB,
+		.active_wakeup = true,
+	},
+};
 
 /*
  * MT8173 power domain support
@@ -789,39 +805,50 @@ static const struct scp_domain_data scp_domain_data_mt8173[] = {
 	},
 };
 
-#define NUM_DOMAINS_MT8173	ARRAY_SIZE(scp_domain_data_mt8173)
-
-static int __init scpsys_probe_mt8173(struct platform_device *pdev)
-{
-	struct scp *scp;
-	struct genpd_onecell_data *pd_data;
-	int ret;
-	struct scp_ctrl_reg scp_reg;
-
-	scp_reg.pwr_sta_offs = SPM_PWR_STATUS;
-	scp_reg.pwr_sta2nd_offs = SPM_PWR_STATUS_2ND;
-
-	scp = init_scp(pdev, scp_domain_data_mt8173, NUM_DOMAINS_MT8173,
-		       &scp_reg);
-	if (IS_ERR(scp))
-		return PTR_ERR(scp);
-
-	mtk_register_power_domains(pdev, scp, NUM_DOMAINS_MT8173);
+static const struct scp_subdomain scp_subdomain_mt8173[] = {
+	{MT8173_POWER_DOMAIN_MFG_ASYNC, MT8173_POWER_DOMAIN_MFG_2D},
+	{MT8173_POWER_DOMAIN_MFG_2D, MT8173_POWER_DOMAIN_MFG},
+};
 
-	pd_data = &scp->pd_data;
+static const struct scp_soc_data mt2701_data = {
+	.domains = scp_domain_data_mt2701,
+	.num_domains = ARRAY_SIZE(scp_domain_data_mt2701),
+	.regs = {
+		.pwr_sta_offs = SPM_PWR_STATUS,
+		.pwr_sta2nd_offs = SPM_PWR_STATUS_2ND
+	}
+};
 
-	ret = pm_genpd_add_subdomain(pd_data->domains[MT8173_POWER_DOMAIN_MFG_ASYNC],
-		pd_data->domains[MT8173_POWER_DOMAIN_MFG_2D]);
-	if (ret && IS_ENABLED(CONFIG_PM))
-		dev_err(&pdev->dev, "Failed to add subdomain: %d\n", ret);
+static const struct scp_soc_data mt6797_data = {
+	.domains = scp_domain_data_mt6797,
+	.num_domains = ARRAY_SIZE(scp_domain_data_mt6797),
+	.subdomains = scp_subdomain_mt6797,
+	.num_subdomains = ARRAY_SIZE(scp_subdomain_mt6797),
+	.regs = {
+		.pwr_sta_offs = SPM_PWR_STATUS_MT6797,
+		.pwr_sta2nd_offs = SPM_PWR_STATUS_2ND_MT6797
+	}
+};
 
-	ret = pm_genpd_add_subdomain(pd_data->domains[MT8173_POWER_DOMAIN_MFG_2D],
-		pd_data->domains[MT8173_POWER_DOMAIN_MFG]);
-	if (ret && IS_ENABLED(CONFIG_PM))
-		dev_err(&pdev->dev, "Failed to add subdomain: %d\n", ret);
+static const struct scp_soc_data mt7622_data = {
+	.domains = scp_domain_data_mt7622,
+	.num_domains = ARRAY_SIZE(scp_domain_data_mt7622),
+	.regs = {
+		.pwr_sta_offs = SPM_PWR_STATUS,
+		.pwr_sta2nd_offs = SPM_PWR_STATUS_2ND
+	}
+};
 
-	return 0;
-}
+static const struct scp_soc_data mt8173_data = {
+	.domains = scp_domain_data_mt8173,
+	.num_domains = ARRAY_SIZE(scp_domain_data_mt8173),
+	.subdomains = scp_subdomain_mt8173,
+	.num_subdomains = ARRAY_SIZE(scp_subdomain_mt8173),
+	.regs = {
+		.pwr_sta_offs = SPM_PWR_STATUS,
+		.pwr_sta2nd_offs = SPM_PWR_STATUS_2ND
+	}
+};
 
 /*
  * scpsys driver init
@@ -830,13 +857,16 @@ static int __init scpsys_probe_mt8173(struct platform_device *pdev)
 static const struct of_device_id of_scpsys_match_tbl[] = {
 	{
 		.compatible = "mediatek,mt2701-scpsys",
-		.data = scpsys_probe_mt2701,
+		.data = &mt2701_data,
 	}, {
 		.compatible = "mediatek,mt6797-scpsys",
-		.data = scpsys_probe_mt6797,
+		.data = &mt6797_data,
+	}, {
+		.compatible = "mediatek,mt7622-scpsys",
+		.data = &mt7622_data,
 	}, {
 		.compatible = "mediatek,mt8173-scpsys",
-		.data = scpsys_probe_mt8173,
+		.data = &mt8173_data,
 	}, {
 		/* sentinel */
 	}
@@ -844,16 +874,33 @@ static const struct of_device_id of_scpsys_match_tbl[] = {
 
 static int scpsys_probe(struct platform_device *pdev)
 {
-	int (*probe)(struct platform_device *);
-	const struct of_device_id *of_id;
+	const struct of_device_id *match;
+	const struct scp_subdomain *sd;
+	const struct scp_soc_data *soc;
+	struct scp *scp;
+	struct genpd_onecell_data *pd_data;
+	int i, ret;
 
-	of_id = of_match_node(of_scpsys_match_tbl, pdev->dev.of_node);
-	if (!of_id || !of_id->data)
-		return -EINVAL;
+	match = of_match_device(of_scpsys_match_tbl, &pdev->dev);
+	soc = (const struct scp_soc_data *)match->data;
+
+	scp = init_scp(pdev, soc->domains, soc->num_domains, &soc->regs);
+	if (IS_ERR(scp))
+		return PTR_ERR(scp);
 
-	probe = of_id->data;
+	mtk_register_power_domains(pdev, scp, soc->num_domains);
 
-	return probe(pdev);
+	pd_data = &scp->pd_data;
+
+	for (i = 0, sd = soc->subdomains ; i < soc->num_subdomains ; i++) {
+		ret = pm_genpd_add_subdomain(pd_data->domains[sd->origin],
+					     pd_data->domains[sd->subdomain]);
+		if (ret && IS_ENABLED(CONFIG_PM))
+			dev_err(&pdev->dev, "Failed to add subdomain: %d\n",
+				ret);
+	}
+
+	return 0;
 }
 
 static struct platform_driver scpsys_drv = {
diff --git a/drivers/soc/qcom/Kconfig b/drivers/soc/qcom/Kconfig
index 9fca977ef18d..b00bccddcd3b 100644
--- a/drivers/soc/qcom/Kconfig
+++ b/drivers/soc/qcom/Kconfig
@@ -1,6 +1,17 @@
 #
 # QCOM Soc drivers
 #
+menu "Qualcomm SoC drivers"
+
+config QCOM_GLINK_SSR
+	tristate "Qualcomm Glink SSR driver"
+	depends on RPMSG
+	depends on QCOM_RPROC_COMMON
+	help
+	  Say y here to enable GLINK SSR support. The GLINK SSR driver
+	  implements the SSR protocol for notifying the remote processor about
+	  neighboring subsystems going up or down.
+
 config QCOM_GSBI
         tristate "QCOM General Serial Bus Interface"
         depends on ARCH_QCOM
@@ -74,3 +85,5 @@ config QCOM_WCNSS_CTRL
 	help
 	  Client driver for the WCNSS_CTRL SMD channel, used to download nv
 	  firmware to a newly booted WCNSS chip.
+
+endmenu
diff --git a/drivers/soc/qcom/Makefile b/drivers/soc/qcom/Makefile
index 414f0de274fa..f151de41eb93 100644
--- a/drivers/soc/qcom/Makefile
+++ b/drivers/soc/qcom/Makefile
@@ -1,3 +1,4 @@
+obj-$(CONFIG_QCOM_GLINK_SSR) +=	glink_ssr.o
 obj-$(CONFIG_QCOM_GSBI)	+=	qcom_gsbi.o
 obj-$(CONFIG_QCOM_MDT_LOADER)	+= mdt_loader.o
 obj-$(CONFIG_QCOM_PM)	+=	spm.o
diff --git a/drivers/soc/qcom/glink_ssr.c b/drivers/soc/qcom/glink_ssr.c
new file mode 100644
index 000000000000..19c7399eddb5
--- /dev/null
+++ b/drivers/soc/qcom/glink_ssr.c
@@ -0,0 +1,164 @@
+/*
+ * Copyright (c) 2014-2017, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2017, Linaro Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/completion.h>
+#include <linux/module.h>
+#include <linux/notifier.h>
+#include <linux/rpmsg.h>
+#include <linux/remoteproc/qcom_rproc.h>
+
+/**
+ * struct do_cleanup_msg - The data structure for an SSR do_cleanup message
+ * version:     The G-Link SSR protocol version
+ * command:     The G-Link SSR command - do_cleanup
+ * seq_num:     Sequence number
+ * name_len:    Length of the name of the subsystem being restarted
+ * name:        G-Link edge name of the subsystem being restarted
+ */
+struct do_cleanup_msg {
+	__le32 version;
+	__le32 command;
+	__le32 seq_num;
+	__le32 name_len;
+	char name[32];
+};
+
+/**
+ * struct cleanup_done_msg - The data structure for an SSR cleanup_done message
+ * version:     The G-Link SSR protocol version
+ * response:    The G-Link SSR response to a do_cleanup command, cleanup_done
+ * seq_num:     Sequence number
+ */
+struct cleanup_done_msg {
+	__le32 version;
+	__le32 response;
+	__le32 seq_num;
+};
+
+/**
+ * G-Link SSR protocol commands
+ */
+#define GLINK_SSR_DO_CLEANUP	0
+#define GLINK_SSR_CLEANUP_DONE	1
+
+struct glink_ssr {
+	struct device *dev;
+	struct rpmsg_endpoint *ept;
+
+	struct notifier_block nb;
+
+	u32 seq_num;
+	struct completion completion;
+};
+
+static int qcom_glink_ssr_callback(struct rpmsg_device *rpdev,
+				   void *data, int len, void *priv, u32 addr)
+{
+	struct cleanup_done_msg *msg = data;
+	struct glink_ssr *ssr = dev_get_drvdata(&rpdev->dev);
+
+	if (len < sizeof(*msg)) {
+		dev_err(ssr->dev, "message too short\n");
+		return -EINVAL;
+	}
+
+	if (le32_to_cpu(msg->version) != 0)
+		return -EINVAL;
+
+	if (le32_to_cpu(msg->response) != GLINK_SSR_CLEANUP_DONE)
+		return 0;
+
+	if (le32_to_cpu(msg->seq_num) != ssr->seq_num) {
+		dev_err(ssr->dev, "invalid sequence number of response\n");
+		return -EINVAL;
+	}
+
+	complete(&ssr->completion);
+
+	return 0;
+}
+
+static int qcom_glink_ssr_notify(struct notifier_block *nb, unsigned long event,
+				 void *data)
+{
+	struct glink_ssr *ssr = container_of(nb, struct glink_ssr, nb);
+	struct do_cleanup_msg msg;
+	char *ssr_name = data;
+	int ret;
+
+	ssr->seq_num++;
+	reinit_completion(&ssr->completion);
+
+	memset(&msg, 0, sizeof(msg));
+	msg.command = cpu_to_le32(GLINK_SSR_DO_CLEANUP);
+	msg.seq_num = cpu_to_le32(ssr->seq_num);
+	msg.name_len = cpu_to_le32(strlen(ssr_name));
+	strlcpy(msg.name, ssr_name, sizeof(msg.name));
+
+	ret = rpmsg_send(ssr->ept, &msg, sizeof(msg));
+	if (ret < 0)
+		dev_err(ssr->dev, "failed to send cleanup message\n");
+
+	ret = wait_for_completion_timeout(&ssr->completion, HZ);
+	if (!ret)
+		dev_err(ssr->dev, "timeout waiting for cleanup done message\n");
+
+	return NOTIFY_DONE;
+}
+
+static int qcom_glink_ssr_probe(struct rpmsg_device *rpdev)
+{
+	struct glink_ssr *ssr;
+
+	ssr = devm_kzalloc(&rpdev->dev, sizeof(*ssr), GFP_KERNEL);
+	if (!ssr)
+		return -ENOMEM;
+
+	init_completion(&ssr->completion);
+
+	ssr->dev = &rpdev->dev;
+	ssr->ept = rpdev->ept;
+	ssr->nb.notifier_call = qcom_glink_ssr_notify;
+
+	dev_set_drvdata(&rpdev->dev, ssr);
+
+	return qcom_register_ssr_notifier(&ssr->nb);
+}
+
+static void qcom_glink_ssr_remove(struct rpmsg_device *rpdev)
+{
+	struct glink_ssr *ssr = dev_get_drvdata(&rpdev->dev);
+
+	qcom_unregister_ssr_notifier(&ssr->nb);
+}
+
+static const struct rpmsg_device_id qcom_glink_ssr_match[] = {
+	{ "glink_ssr" },
+	{}
+};
+
+static struct rpmsg_driver qcom_glink_ssr_driver = {
+	.probe = qcom_glink_ssr_probe,
+	.remove = qcom_glink_ssr_remove,
+	.callback = qcom_glink_ssr_callback,
+	.id_table = qcom_glink_ssr_match,
+	.drv = {
+		.name = "qcom_glink_ssr",
+	},
+};
+module_rpmsg_driver(qcom_glink_ssr_driver);
+
+MODULE_ALIAS("rpmsg:glink_ssr");
+MODULE_DESCRIPTION("Qualcomm GLINK SSR notifier");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/soc/qcom/mdt_loader.c b/drivers/soc/qcom/mdt_loader.c
index bd63df0d14e0..08bd8549242a 100644
--- a/drivers/soc/qcom/mdt_loader.c
+++ b/drivers/soc/qcom/mdt_loader.c
@@ -178,14 +178,13 @@ int qcom_mdt_load(struct device *dev, const struct firmware *fw,
 
 		if (phdr->p_filesz) {
 			sprintf(fw_name + fw_name_len - 3, "b%02d", i);
-			ret = request_firmware(&seg_fw, fw_name, dev);
+			ret = request_firmware_into_buf(&seg_fw, fw_name, dev,
+							ptr, phdr->p_filesz);
 			if (ret) {
 				dev_err(dev, "failed to load %s\n", fw_name);
 				break;
 			}
 
-			memcpy(ptr, seg_fw->data, seg_fw->size);
-
 			release_firmware(seg_fw);
 		}
 
diff --git a/drivers/soc/qcom/smsm.c b/drivers/soc/qcom/smsm.c
index dc540ea92e9d..403bea9d546b 100644
--- a/drivers/soc/qcom/smsm.c
+++ b/drivers/soc/qcom/smsm.c
@@ -496,7 +496,8 @@ static int qcom_smsm_probe(struct platform_device *pdev)
 	if (!smsm->hosts)
 		return -ENOMEM;
 
-	local_node = of_find_node_with_property(pdev->dev.of_node, "#qcom,smem-state-cells");
+	local_node = of_find_node_with_property(of_node_get(pdev->dev.of_node),
+						"#qcom,smem-state-cells");
 	if (!local_node) {
 		dev_err(&pdev->dev, "no state entry\n");
 		return -EINVAL;
diff --git a/drivers/soc/qcom/wcnss_ctrl.c b/drivers/soc/qcom/wcnss_ctrl.c
index b9069184df19..d008e5b82db4 100644
--- a/drivers/soc/qcom/wcnss_ctrl.c
+++ b/drivers/soc/qcom/wcnss_ctrl.c
@@ -347,6 +347,7 @@ static const struct of_device_id wcnss_ctrl_of_match[] = {
 	{ .compatible = "qcom,wcnss", },
 	{}
 };
+MODULE_DEVICE_TABLE(of, wcnss_ctrl_of_match);
 
 static struct rpmsg_driver wcnss_ctrl_driver = {
 	.probe = wcnss_ctrl_probe,
diff --git a/drivers/soc/renesas/Kconfig b/drivers/soc/renesas/Kconfig
index 87a4be46bd98..567414cb42ba 100644
--- a/drivers/soc/renesas/Kconfig
+++ b/drivers/soc/renesas/Kconfig
@@ -3,7 +3,7 @@ config SOC_RENESAS
 	default y if ARCH_RENESAS
 	select SOC_BUS
 	select RST_RCAR if ARCH_RCAR_GEN1 || ARCH_RCAR_GEN2 || \
-			   ARCH_R8A7795 || ARCH_R8A7796
+			   ARCH_R8A7795 || ARCH_R8A7796 || ARCH_R8A77995
 	select SYSC_R8A7743 if ARCH_R8A7743
 	select SYSC_R8A7745 if ARCH_R8A7745
 	select SYSC_R8A7779 if ARCH_R8A7779
@@ -13,6 +13,7 @@ config SOC_RENESAS
 	select SYSC_R8A7794 if ARCH_R8A7794
 	select SYSC_R8A7795 if ARCH_R8A7795
 	select SYSC_R8A7796 if ARCH_R8A7796
+	select SYSC_R8A77995 if ARCH_R8A77995
 
 if SOC_RENESAS
 
@@ -53,6 +54,10 @@ config SYSC_R8A7796
 	bool "R-Car M3-W System Controller support" if COMPILE_TEST
 	select SYSC_RCAR
 
+config SYSC_R8A77995
+	bool "R-Car D3 System Controller support" if COMPILE_TEST
+	select SYSC_RCAR
+
 # Family
 config RST_RCAR
 	bool "R-Car Reset Controller support" if COMPILE_TEST
diff --git a/drivers/soc/renesas/Makefile b/drivers/soc/renesas/Makefile
index 1a1a297b26a7..6b6e7f16104c 100644
--- a/drivers/soc/renesas/Makefile
+++ b/drivers/soc/renesas/Makefile
@@ -11,6 +11,7 @@ obj-$(CONFIG_SYSC_R8A7792)	+= r8a7792-sysc.o
 obj-$(CONFIG_SYSC_R8A7794)	+= r8a7794-sysc.o
 obj-$(CONFIG_SYSC_R8A7795)	+= r8a7795-sysc.o
 obj-$(CONFIG_SYSC_R8A7796)	+= r8a7796-sysc.o
+obj-$(CONFIG_SYSC_R8A77995)	+= r8a77995-sysc.o
 
 # Family
 obj-$(CONFIG_RST_RCAR)		+= rcar-rst.o
diff --git a/drivers/soc/renesas/r8a77995-sysc.c b/drivers/soc/renesas/r8a77995-sysc.c
new file mode 100644
index 000000000000..f718429cab02
--- /dev/null
+++ b/drivers/soc/renesas/r8a77995-sysc.c
@@ -0,0 +1,31 @@
+/*
+ * Renesas R-Car D3 System Controller
+ *
+ * Copyright (C) 2017 Glider bvba
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ */
+
+#include <linux/bug.h>
+#include <linux/kernel.h>
+#include <linux/sys_soc.h>
+
+#include <dt-bindings/power/r8a77995-sysc.h>
+
+#include "rcar-sysc.h"
+
+static struct rcar_sysc_area r8a77995_areas[] __initdata = {
+	{ "always-on",     0, 0, R8A77995_PD_ALWAYS_ON, -1, PD_ALWAYS_ON },
+	{ "ca53-scu",  0x140, 0, R8A77995_PD_CA53_SCU,  R8A77995_PD_ALWAYS_ON,
+	  PD_SCU },
+	{ "ca53-cpu0", 0x200, 0, R8A77995_PD_CA53_CPU0, R8A77995_PD_CA53_SCU,
+	  PD_CPU_NOCR },
+};
+
+
+const struct rcar_sysc_info r8a77995_sysc_info __initconst = {
+	.areas = r8a77995_areas,
+	.num_areas = ARRAY_SIZE(r8a77995_areas),
+};
diff --git a/drivers/soc/renesas/rcar-rst.c b/drivers/soc/renesas/rcar-rst.c
index a6d1c26d3167..baa47014e96b 100644
--- a/drivers/soc/renesas/rcar-rst.c
+++ b/drivers/soc/renesas/rcar-rst.c
@@ -41,6 +41,7 @@ static const struct of_device_id rcar_rst_matches[] __initconst = {
 	/* R-Car Gen3 is handled like R-Car Gen2 */
 	{ .compatible = "renesas,r8a7795-rst", .data = &rcar_rst_gen2 },
 	{ .compatible = "renesas,r8a7796-rst", .data = &rcar_rst_gen2 },
+	{ .compatible = "renesas,r8a77995-rst", .data = &rcar_rst_gen2 },
 	{ /* sentinel */ }
 };
 
@@ -61,7 +62,7 @@ static int __init rcar_rst_init(void)
 
 	base = of_iomap(np, 0);
 	if (!base) {
-		pr_warn("%s: Cannot map regs\n", np->full_name);
+		pr_warn("%pOF: Cannot map regs\n", np);
 		error = -ENOMEM;
 		goto out_put;
 	}
@@ -70,7 +71,7 @@ static int __init rcar_rst_init(void)
 	cfg = match->data;
 	saved_mode = ioread32(base + cfg->modemr);
 
-	pr_debug("%s: MODE = 0x%08x\n", np->full_name, saved_mode);
+	pr_debug("%pOF: MODE = 0x%08x\n", np, saved_mode);
 
 out_put:
 	of_node_put(np);
diff --git a/drivers/soc/renesas/rcar-sysc.c b/drivers/soc/renesas/rcar-sysc.c
index 7c8da3c90011..c8406e81640f 100644
--- a/drivers/soc/renesas/rcar-sysc.c
+++ b/drivers/soc/renesas/rcar-sysc.c
@@ -284,6 +284,9 @@ static const struct of_device_id rcar_sysc_matches[] = {
 #ifdef CONFIG_SYSC_R8A7796
 	{ .compatible = "renesas,r8a7796-sysc", .data = &r8a7796_sysc_info },
 #endif
+#ifdef CONFIG_SYSC_R8A77995
+	{ .compatible = "renesas,r8a77995-sysc", .data = &r8a77995_sysc_info },
+#endif
 	{ /* sentinel */ }
 };
 
@@ -323,7 +326,7 @@ static int __init rcar_sysc_pd_init(void)
 
 	base = of_iomap(np, 0);
 	if (!base) {
-		pr_warn("%s: Cannot map regs\n", np->full_name);
+		pr_warn("%pOF: Cannot map regs\n", np);
 		error = -ENOMEM;
 		goto out_put;
 	}
@@ -348,13 +351,13 @@ static int __init rcar_sysc_pd_init(void)
 	 */
 	syscimr = ioread32(base + SYSCIMR);
 	syscimr |= syscier;
-	pr_debug("%s: syscimr = 0x%08x\n", np->full_name, syscimr);
+	pr_debug("%pOF: syscimr = 0x%08x\n", np, syscimr);
 	iowrite32(syscimr, base + SYSCIMR);
 
 	/*
 	 * SYSC needs all interrupt sources enabled to control power.
 	 */
-	pr_debug("%s: syscier = 0x%08x\n", np->full_name, syscier);
+	pr_debug("%pOF: syscier = 0x%08x\n", np, syscier);
 	iowrite32(syscier, base + SYSCIER);
 
 	for (i = 0; i < info->num_areas; i++) {
diff --git a/drivers/soc/renesas/rcar-sysc.h b/drivers/soc/renesas/rcar-sysc.h
index 1a5bebaf54ba..2f524922c4d2 100644
--- a/drivers/soc/renesas/rcar-sysc.h
+++ b/drivers/soc/renesas/rcar-sysc.h
@@ -58,6 +58,7 @@ extern const struct rcar_sysc_info r8a7792_sysc_info;
 extern const struct rcar_sysc_info r8a7794_sysc_info;
 extern const struct rcar_sysc_info r8a7795_sysc_info;
 extern const struct rcar_sysc_info r8a7796_sysc_info;
+extern const struct rcar_sysc_info r8a77995_sysc_info;
 
 
     /*
diff --git a/drivers/soc/renesas/renesas-soc.c b/drivers/soc/renesas/renesas-soc.c
index ca26f13d399c..90d6b7a4340a 100644
--- a/drivers/soc/renesas/renesas-soc.c
+++ b/drivers/soc/renesas/renesas-soc.c
@@ -144,6 +144,11 @@ static const struct renesas_soc soc_rcar_m3_w __initconst __maybe_unused = {
 	.id	= 0x52,
 };
 
+static const struct renesas_soc soc_rcar_d3 __initconst __maybe_unused = {
+	.family	= &fam_rcar_gen3,
+	.id	= 0x58,
+};
+
 static const struct renesas_soc soc_shmobile_ag5 __initconst __maybe_unused = {
 	.family	= &fam_shmobile,
 	.id	= 0x37,
@@ -199,6 +204,9 @@ static const struct of_device_id renesas_socs[] __initconst = {
 #ifdef CONFIG_ARCH_R8A7796
 	{ .compatible = "renesas,r8a7796",	.data = &soc_rcar_m3_w },
 #endif
+#ifdef CONFIG_ARCH_R8A77995
+	{ .compatible = "renesas,r8a77995",	.data = &soc_rcar_d3 },
+#endif
 #ifdef CONFIG_ARCH_SH73A0
 	{ .compatible = "renesas,sh73a0",	.data = &soc_shmobile_ag5 },
 #endif
diff --git a/drivers/soc/rockchip/grf.c b/drivers/soc/rockchip/grf.c
index d61db34ad6dd..15e71fd6c513 100644
--- a/drivers/soc/rockchip/grf.c
+++ b/drivers/soc/rockchip/grf.c
@@ -54,6 +54,17 @@ static const struct rockchip_grf_info rk3288_grf __initconst = {
 	.num_values = ARRAY_SIZE(rk3288_defaults),
 };
 
+#define RK3328_GRF_SOC_CON4		0x410
+
+static const struct rockchip_grf_value rk3328_defaults[] __initconst = {
+	{ "jtag switching", RK3328_GRF_SOC_CON4, HIWORD_UPDATE(0, 1, 12) },
+};
+
+static const struct rockchip_grf_info rk3328_grf __initconst = {
+	.values = rk3328_defaults,
+	.num_values = ARRAY_SIZE(rk3328_defaults),
+};
+
 #define RK3368_GRF_SOC_CON15		0x43c
 
 static const struct rockchip_grf_value rk3368_defaults[] __initconst = {
@@ -84,6 +95,9 @@ static const struct of_device_id rockchip_grf_dt_match[] __initconst = {
 		.compatible = "rockchip,rk3288-grf",
 		.data = (void *)&rk3288_grf,
 	}, {
+		.compatible = "rockchip,rk3328-grf",
+		.data = (void *)&rk3328_grf,
+	}, {
 		.compatible = "rockchip,rk3368-grf",
 		.data = (void *)&rk3368_grf,
 	}, {
diff --git a/drivers/soc/rockchip/pm_domains.c b/drivers/soc/rockchip/pm_domains.c
index 796c46a6cbe7..40b75748835f 100644
--- a/drivers/soc/rockchip/pm_domains.c
+++ b/drivers/soc/rockchip/pm_domains.c
@@ -20,6 +20,7 @@
 #include <linux/mfd/syscon.h>
 #include <dt-bindings/power/rk3288-power.h>
 #include <dt-bindings/power/rk3328-power.h>
+#include <dt-bindings/power/rk3366-power.h>
 #include <dt-bindings/power/rk3368-power.h>
 #include <dt-bindings/power/rk3399-power.h>
 
@@ -730,6 +731,16 @@ static const struct rockchip_domain_info rk3328_pm_domains[] = {
 	[RK3328_PD_VPU]		= DOMAIN_RK3328(-1, 9, 9, false),
 };
 
+static const struct rockchip_domain_info rk3366_pm_domains[] = {
+	[RK3366_PD_PERI]	= DOMAIN_RK3368(10, 10, 6, true),
+	[RK3366_PD_VIO]		= DOMAIN_RK3368(14, 14, 8, false),
+	[RK3366_PD_VIDEO]	= DOMAIN_RK3368(13, 13, 7, false),
+	[RK3366_PD_RKVDEC]	= DOMAIN_RK3368(11, 11, 7, false),
+	[RK3366_PD_WIFIBT]	= DOMAIN_RK3368(8, 8, 9, false),
+	[RK3366_PD_VPU]		= DOMAIN_RK3368(12, 12, 7, false),
+	[RK3366_PD_GPU]		= DOMAIN_RK3368(15, 15, 2, false),
+};
+
 static const struct rockchip_domain_info rk3368_pm_domains[] = {
 	[RK3368_PD_PERI]	= DOMAIN_RK3368(13, 12, 6, true),
 	[RK3368_PD_VIO]		= DOMAIN_RK3368(15, 14, 8, false),
@@ -794,6 +805,23 @@ static const struct rockchip_pmu_info rk3328_pmu = {
 	.domain_info = rk3328_pm_domains,
 };
 
+static const struct rockchip_pmu_info rk3366_pmu = {
+	.pwr_offset = 0x0c,
+	.status_offset = 0x10,
+	.req_offset = 0x3c,
+	.idle_offset = 0x40,
+	.ack_offset = 0x40,
+
+	.core_pwrcnt_offset = 0x48,
+	.gpu_pwrcnt_offset = 0x50,
+
+	.core_power_transition_time = 24,
+	.gpu_power_transition_time = 24,
+
+	.num_domains = ARRAY_SIZE(rk3366_pm_domains),
+	.domain_info = rk3366_pm_domains,
+};
+
 static const struct rockchip_pmu_info rk3368_pmu = {
 	.pwr_offset = 0x0c,
 	.status_offset = 0x10,
@@ -834,6 +862,10 @@ static const struct of_device_id rockchip_pm_domain_dt_match[] = {
 		.data = (void *)&rk3328_pmu,
 	},
 	{
+		.compatible = "rockchip,rk3366-power-controller",
+		.data = (void *)&rk3366_pmu,
+	},
+	{
 		.compatible = "rockchip,rk3368-power-controller",
 		.data = (void *)&rk3368_pmu,
 	},
diff --git a/drivers/soc/samsung/pm_domains.c b/drivers/soc/samsung/pm_domains.c
index a6a5d807cc2b..7c4fec1f93b5 100644
--- a/drivers/soc/samsung/pm_domains.c
+++ b/drivers/soc/samsung/pm_domains.c
@@ -147,7 +147,7 @@ static __init const char *exynos_get_domain_name(struct device_node *node)
 	const char *name;
 
 	if (of_property_read_string(node, "label", &name) < 0)
-		name = strrchr(node->full_name, '/') + 1;
+		name = kbasename(node->full_name);
 	return kstrdup_const(name, GFP_KERNEL);
 }
 
@@ -237,11 +237,11 @@ no_clk:
 			continue;
 
 		if (of_genpd_add_subdomain(&parent, &child))
-			pr_warn("%s failed to add subdomain: %s\n",
-				parent.np->full_name, child.np->full_name);
+			pr_warn("%pOF failed to add subdomain: %pOF\n",
+				parent.np, child.np);
 		else
-			pr_info("%s has as child subdomain: %s.\n",
-				parent.np->full_name, child.np->full_name);
+			pr_info("%pOF has as child subdomain: %pOF.\n",
+				parent.np, child.np);
 	}
 
 	return 0;
diff --git a/drivers/soc/sunxi/sunxi_sram.c b/drivers/soc/sunxi/sunxi_sram.c
index 99e354c8f53f..882be5ed7e84 100644
--- a/drivers/soc/sunxi/sunxi_sram.c
+++ b/drivers/soc/sunxi/sunxi_sram.c
@@ -23,6 +23,7 @@
 struct sunxi_sram_func {
 	char	*func;
 	u8	val;
+	u32	reg_val;
 };
 
 struct sunxi_sram_data {
@@ -39,10 +40,11 @@ struct sunxi_sram_desc {
 	bool			claimed;
 };
 
-#define SUNXI_SRAM_MAP(_val, _func)				\
+#define SUNXI_SRAM_MAP(_reg_val, _val, _func)			\
 	{							\
 		.func = _func,					\
 		.val = _val,					\
+		.reg_val = _reg_val,				\
 	}
 
 #define SUNXI_SRAM_DATA(_name, _reg, _off, _width, ...)		\
@@ -57,14 +59,20 @@ struct sunxi_sram_desc {
 
 static struct sunxi_sram_desc sun4i_a10_sram_a3_a4 = {
 	.data	= SUNXI_SRAM_DATA("A3-A4", 0x4, 0x4, 2,
-				  SUNXI_SRAM_MAP(0, "cpu"),
-				  SUNXI_SRAM_MAP(1, "emac")),
+				  SUNXI_SRAM_MAP(0, 0, "cpu"),
+				  SUNXI_SRAM_MAP(1, 1, "emac")),
 };
 
 static struct sunxi_sram_desc sun4i_a10_sram_d = {
 	.data	= SUNXI_SRAM_DATA("D", 0x4, 0x0, 1,
-				  SUNXI_SRAM_MAP(0, "cpu"),
-				  SUNXI_SRAM_MAP(1, "usb-otg")),
+				  SUNXI_SRAM_MAP(0, 0, "cpu"),
+				  SUNXI_SRAM_MAP(1, 1, "usb-otg")),
+};
+
+static struct sunxi_sram_desc sun50i_a64_sram_c = {
+	.data	= SUNXI_SRAM_DATA("C", 0x4, 24, 1,
+				  SUNXI_SRAM_MAP(0, 1, "cpu"),
+				  SUNXI_SRAM_MAP(1, 0, "de2")),
 };
 
 static const struct of_device_id sunxi_sram_dt_ids[] = {
@@ -76,6 +84,10 @@ static const struct of_device_id sunxi_sram_dt_ids[] = {
 		.compatible	= "allwinner,sun4i-a10-sram-d",
 		.data		= &sun4i_a10_sram_d.data,
 	},
+	{
+		.compatible	= "allwinner,sun50i-a64-sram-c",
+		.data		= &sun50i_a64_sram_c.data,
+	},
 	{}
 };
 
@@ -121,7 +133,8 @@ static int sunxi_sram_show(struct seq_file *s, void *data)
 
 			for (func = sram_data->func; func->func; func++) {
 				seq_printf(s, "\t\t%s%c\n", func->func,
-					   func->val == val ? '*' : ' ');
+					   func->reg_val == val ?
+					   '*' : ' ');
 			}
 		}
 
@@ -149,10 +162,13 @@ static inline struct sunxi_sram_desc *to_sram_desc(const struct sunxi_sram_data
 }
 
 static const struct sunxi_sram_data *sunxi_sram_of_parse(struct device_node *node,
-							 unsigned int *value)
+							 unsigned int *reg_value)
 {
 	const struct of_device_id *match;
+	const struct sunxi_sram_data *data;
+	struct sunxi_sram_func *func;
 	struct of_phandle_args args;
+	u8 val;
 	int ret;
 
 	ret = of_parse_phandle_with_fixed_args(node, "allwinner,sram", 1, 0,
@@ -165,8 +181,7 @@ static const struct sunxi_sram_data *sunxi_sram_of_parse(struct device_node *nod
 		goto err;
 	}
 
-	if (value)
-		*value = args.args[0];
+	val = args.args[0];
 
 	match = of_match_node(sunxi_sram_dt_ids, args.np);
 	if (!match) {
@@ -174,6 +189,26 @@ static const struct sunxi_sram_data *sunxi_sram_of_parse(struct device_node *nod
 		goto err;
 	}
 
+	data = match->data;
+	if (!data) {
+		ret = -EINVAL;
+		goto err;
+	};
+
+	for (func = data->func; func->func; func++) {
+		if (val == func->val) {
+			if (reg_value)
+				*reg_value = func->reg_val;
+
+			break;
+		}
+	}
+
+	if (!func->func) {
+		ret = -EINVAL;
+		goto err;
+	}
+
 	of_node_put(args.np);
 	return match->data;
 
@@ -190,6 +225,9 @@ int sunxi_sram_claim(struct device *dev)
 	u32 val, mask;
 
 	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	if (!base)
 		return -EPROBE_DEFER;
 
 	if (!dev || !dev->of_node)
@@ -267,6 +305,7 @@ static int sunxi_sram_probe(struct platform_device *pdev)
 
 static const struct of_device_id sunxi_sram_dt_match[] = {
 	{ .compatible = "allwinner,sun4i-a10-sram-controller" },
+	{ .compatible = "allwinner,sun50i-a64-sram-controller" },
 	{ },
 };
 MODULE_DEVICE_TABLE(of, sunxi_sram_dt_match);
diff --git a/drivers/soc/tegra/Kconfig b/drivers/soc/tegra/Kconfig
index 1beb7c347344..e9e277178c94 100644
--- a/drivers/soc/tegra/Kconfig
+++ b/drivers/soc/tegra/Kconfig
@@ -107,6 +107,11 @@ config ARCH_TEGRA_186_SOC
 endif
 endif
 
+config SOC_TEGRA_FUSE
+	def_bool y
+	depends on ARCH_TEGRA
+	select SOC_BUS
+
 config SOC_TEGRA_FLOWCTRL
 	bool
 
diff --git a/drivers/soc/tegra/fuse/fuse-tegra.c b/drivers/soc/tegra/fuse/fuse-tegra.c
index 7413f60fa855..b7c552e3133c 100644
--- a/drivers/soc/tegra/fuse/fuse-tegra.c
+++ b/drivers/soc/tegra/fuse/fuse-tegra.c
@@ -19,10 +19,12 @@
 #include <linux/device.h>
 #include <linux/kobject.h>
 #include <linux/init.h>
-#include <linux/platform_device.h>
+#include <linux/io.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
-#include <linux/io.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/sys_soc.h>
 
 #include <soc/tegra/common.h>
 #include <soc/tegra/fuse.h>
@@ -210,6 +212,31 @@ static void tegra_enable_fuse_clk(void __iomem *base)
 	writel(reg, base + 0x14);
 }
 
+struct device * __init tegra_soc_device_register(void)
+{
+	struct soc_device_attribute *attr;
+	struct soc_device *dev;
+
+	attr = kzalloc(sizeof(*attr), GFP_KERNEL);
+	if (!attr)
+		return NULL;
+
+	attr->family = kasprintf(GFP_KERNEL, "Tegra");
+	attr->revision = kasprintf(GFP_KERNEL, "%d", tegra_sku_info.revision);
+	attr->soc_id = kasprintf(GFP_KERNEL, "%u", tegra_get_chip_id());
+
+	dev = soc_device_register(attr);
+	if (IS_ERR(dev)) {
+		kfree(attr->soc_id);
+		kfree(attr->revision);
+		kfree(attr->family);
+		kfree(attr);
+		return ERR_CAST(dev);
+	}
+
+	return soc_device_to_device(dev);
+}
+
 static int __init tegra_init_fuse(void)
 {
 	const struct of_device_id *match;
@@ -311,6 +338,31 @@ static int __init tegra_init_fuse(void)
 	pr_debug("Tegra CPU Speedo ID %d, SoC Speedo ID %d\n",
 		 tegra_sku_info.cpu_speedo_id, tegra_sku_info.soc_speedo_id);
 
+
 	return 0;
 }
 early_initcall(tegra_init_fuse);
+
+#ifdef CONFIG_ARM64
+static int __init tegra_init_soc(void)
+{
+	struct device_node *np;
+	struct device *soc;
+
+	/* make sure we're running on Tegra */
+	np = of_find_matching_node(NULL, tegra_fuse_match);
+	if (!np)
+		return 0;
+
+	of_node_put(np);
+
+	soc = tegra_soc_device_register();
+	if (IS_ERR(soc)) {
+		pr_err("failed to register SoC device: %ld\n", PTR_ERR(soc));
+		return PTR_ERR(soc);
+	}
+
+	return 0;
+}
+device_initcall(tegra_init_soc);
+#endif
diff --git a/drivers/soc/tegra/pmc.c b/drivers/soc/tegra/pmc.c
index e233dd5dcab3..0453ff6839a7 100644
--- a/drivers/soc/tegra/pmc.c
+++ b/drivers/soc/tegra/pmc.c
@@ -918,10 +918,8 @@ static void tegra_powergate_init(struct tegra_pmc *pmc,
 	if (!np)
 		return;
 
-	for_each_child_of_node(np, child) {
+	for_each_child_of_node(np, child)
 		tegra_powergate_add(pmc, child);
-		of_node_put(child);
-	}
 
 	of_node_put(np);
 }
diff --git a/drivers/soc/ti/knav_qmss_queue.c b/drivers/soc/ti/knav_qmss_queue.c
index 279e7c5551dd..39225de9d7f1 100644
--- a/drivers/soc/ti/knav_qmss_queue.c
+++ b/drivers/soc/ti/knav_qmss_queue.c
@@ -745,6 +745,9 @@ void *knav_pool_create(const char *name,
 	bool slot_found;
 	int ret;
 
+	if (!kdev)
+		return ERR_PTR(-EPROBE_DEFER);
+
 	if (!kdev->dev)
 		return ERR_PTR(-ENODEV);
 
diff --git a/drivers/soc/ti/ti_sci_pm_domains.c b/drivers/soc/ti/ti_sci_pm_domains.c
index b0b283810e72..de31b9389e2e 100644
--- a/drivers/soc/ti/ti_sci_pm_domains.c
+++ b/drivers/soc/ti/ti_sci_pm_domains.c
@@ -176,6 +176,8 @@ static int ti_sci_pm_domain_probe(struct platform_device *pdev)
 
 	ti_sci_pd->dev = dev;
 
+	ti_sci_pd->pd.name = "ti_sci_pd";
+
 	ti_sci_pd->pd.attach_dev = ti_sci_pd_attach_dev;
 	ti_sci_pd->pd.detach_dev = ti_sci_pd_detach_dev;
 
diff --git a/drivers/soc/versatile/soc-realview.c b/drivers/soc/versatile/soc-realview.c
index 282e371378ce..caf698e5f0b0 100644
--- a/drivers/soc/versatile/soc-realview.c
+++ b/drivers/soc/versatile/soc-realview.c
@@ -85,7 +85,7 @@ static struct device_attribute realview_build_attr =
 
 static int realview_soc_probe(struct platform_device *pdev)
 {
-	static struct regmap *syscon_regmap;
+	struct regmap *syscon_regmap;
 	struct soc_device *soc_dev;
 	struct soc_device_attribute *soc_dev_attr;
 	struct device_node *np = pdev->dev.of_node;
diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig
index 9b31351fe429..a75f2a2cf780 100644
--- a/drivers/spi/Kconfig
+++ b/drivers/spi/Kconfig
@@ -55,7 +55,6 @@ comment "SPI Master Controller Drivers"
 
 config SPI_ALTERA
 	tristate "Altera SPI Controller"
-	select SPI_BITBANG
 	help
 	  This is the driver for the Altera SPI Controller.
 
@@ -518,8 +517,8 @@ config SPI_PPC4xx
 
 config SPI_PXA2XX
 	tristate "PXA2xx SSP SPI master"
-	depends on (ARCH_PXA || PCI || ACPI)
-	select PXA_SSP if ARCH_PXA
+	depends on (ARCH_PXA || ARCH_MMP || PCI || ACPI)
+	select PXA_SSP if ARCH_PXA || ARCH_MMP
 	help
 	  This enables using a PXA2xx or Sodaville SSP port as a SPI master
 	  controller. The driver can be configured to use any SSP port and
diff --git a/drivers/spi/spi-altera.c b/drivers/spi/spi-altera.c
index b95010e72452..a5adf0d868fc 100644
--- a/drivers/spi/spi-altera.c
+++ b/drivers/spi/spi-altera.c
@@ -18,7 +18,6 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/spi/spi.h>
-#include <linux/spi/spi_bitbang.h>
 #include <linux/io.h>
 #include <linux/of.h>
 
@@ -45,10 +44,6 @@
 #define ALTERA_SPI_CONTROL_SSO_MSK	0x400
 
 struct altera_spi {
-	/* bitbang has to be first */
-	struct spi_bitbang bitbang;
-	struct completion done;
-
 	void __iomem *base;
 	int irq;
 	int len;
@@ -66,59 +61,64 @@ static inline struct altera_spi *altera_spi_to_hw(struct spi_device *sdev)
 	return spi_master_get_devdata(sdev->master);
 }
 
-static void altera_spi_chipsel(struct spi_device *spi, int value)
+static void altera_spi_set_cs(struct spi_device *spi, bool is_high)
 {
 	struct altera_spi *hw = altera_spi_to_hw(spi);
 
-	if (spi->mode & SPI_CS_HIGH) {
-		switch (value) {
-		case BITBANG_CS_INACTIVE:
-			writel(1 << spi->chip_select,
-			       hw->base + ALTERA_SPI_SLAVE_SEL);
-			hw->imr |= ALTERA_SPI_CONTROL_SSO_MSK;
-			writel(hw->imr, hw->base + ALTERA_SPI_CONTROL);
-			break;
-
-		case BITBANG_CS_ACTIVE:
-			hw->imr &= ~ALTERA_SPI_CONTROL_SSO_MSK;
-			writel(hw->imr, hw->base + ALTERA_SPI_CONTROL);
-			writel(0, hw->base + ALTERA_SPI_SLAVE_SEL);
-			break;
-		}
+	if (is_high) {
+		hw->imr &= ~ALTERA_SPI_CONTROL_SSO_MSK;
+		writel(hw->imr, hw->base + ALTERA_SPI_CONTROL);
+		writel(0, hw->base + ALTERA_SPI_SLAVE_SEL);
 	} else {
-		switch (value) {
-		case BITBANG_CS_INACTIVE:
-			hw->imr &= ~ALTERA_SPI_CONTROL_SSO_MSK;
-			writel(hw->imr, hw->base + ALTERA_SPI_CONTROL);
-			break;
+		writel(BIT(spi->chip_select), hw->base + ALTERA_SPI_SLAVE_SEL);
+		hw->imr |= ALTERA_SPI_CONTROL_SSO_MSK;
+		writel(hw->imr, hw->base + ALTERA_SPI_CONTROL);
+	}
+}
+
+static void altera_spi_tx_word(struct altera_spi *hw)
+{
+	unsigned int txd = 0;
 
-		case BITBANG_CS_ACTIVE:
-			writel(1 << spi->chip_select,
-			       hw->base + ALTERA_SPI_SLAVE_SEL);
-			hw->imr |= ALTERA_SPI_CONTROL_SSO_MSK;
-			writel(hw->imr, hw->base + ALTERA_SPI_CONTROL);
+	if (hw->tx) {
+		switch (hw->bytes_per_word) {
+		case 1:
+			txd = hw->tx[hw->count];
+			break;
+		case 2:
+			txd = (hw->tx[hw->count * 2]
+				| (hw->tx[hw->count * 2 + 1] << 8));
 			break;
 		}
 	}
+
+	writel(txd, hw->base + ALTERA_SPI_TXDATA);
 }
 
-static inline unsigned int hw_txbyte(struct altera_spi *hw, int count)
+static void altera_spi_rx_word(struct altera_spi *hw)
 {
-	if (hw->tx) {
+	unsigned int rxd;
+
+	rxd = readl(hw->base + ALTERA_SPI_RXDATA);
+	if (hw->rx) {
 		switch (hw->bytes_per_word) {
 		case 1:
-			return hw->tx[count];
+			hw->rx[hw->count] = rxd;
+			break;
 		case 2:
-			return (hw->tx[count * 2]
-				| (hw->tx[count * 2 + 1] << 8));
+			hw->rx[hw->count * 2] = rxd;
+			hw->rx[hw->count * 2 + 1] = rxd >> 8;
+			break;
 		}
 	}
-	return 0;
+
+	hw->count++;
 }
 
-static int altera_spi_txrx(struct spi_device *spi, struct spi_transfer *t)
+static int altera_spi_txrx(struct spi_master *master,
+	struct spi_device *spi, struct spi_transfer *t)
 {
-	struct altera_spi *hw = altera_spi_to_hw(spi);
+	struct altera_spi *hw = spi_master_get_devdata(master);
 
 	hw->tx = t->tx_buf;
 	hw->rx = t->rx_buf;
@@ -132,67 +132,39 @@ static int altera_spi_txrx(struct spi_device *spi, struct spi_transfer *t)
 		writel(hw->imr, hw->base + ALTERA_SPI_CONTROL);
 
 		/* send the first byte */
-		writel(hw_txbyte(hw, 0), hw->base + ALTERA_SPI_TXDATA);
-
-		wait_for_completion(&hw->done);
-		/* disable receive interrupt */
-		hw->imr &= ~ALTERA_SPI_CONTROL_IRRDY_MSK;
-		writel(hw->imr, hw->base + ALTERA_SPI_CONTROL);
+		altera_spi_tx_word(hw);
 	} else {
 		while (hw->count < hw->len) {
-			unsigned int rxd;
-
-			writel(hw_txbyte(hw, hw->count),
-			       hw->base + ALTERA_SPI_TXDATA);
+			altera_spi_tx_word(hw);
 
 			while (!(readl(hw->base + ALTERA_SPI_STATUS) &
 				 ALTERA_SPI_STATUS_RRDY_MSK))
 				cpu_relax();
 
-			rxd = readl(hw->base + ALTERA_SPI_RXDATA);
-			if (hw->rx) {
-				switch (hw->bytes_per_word) {
-				case 1:
-					hw->rx[hw->count] = rxd;
-					break;
-				case 2:
-					hw->rx[hw->count * 2] = rxd;
-					hw->rx[hw->count * 2 + 1] = rxd >> 8;
-					break;
-				}
-			}
-
-			hw->count++;
+			altera_spi_rx_word(hw);
 		}
+		spi_finalize_current_transfer(master);
 	}
 
-	return hw->count * hw->bytes_per_word;
+	return t->len;
 }
 
 static irqreturn_t altera_spi_irq(int irq, void *dev)
 {
-	struct altera_spi *hw = dev;
-	unsigned int rxd;
+	struct spi_master *master = dev;
+	struct altera_spi *hw = spi_master_get_devdata(master);
 
-	rxd = readl(hw->base + ALTERA_SPI_RXDATA);
-	if (hw->rx) {
-		switch (hw->bytes_per_word) {
-		case 1:
-			hw->rx[hw->count] = rxd;
-			break;
-		case 2:
-			hw->rx[hw->count * 2] = rxd;
-			hw->rx[hw->count * 2 + 1] = rxd >> 8;
-			break;
-		}
-	}
+	altera_spi_rx_word(hw);
 
-	hw->count++;
+	if (hw->count < hw->len) {
+		altera_spi_tx_word(hw);
+	} else {
+		/* disable receive interrupt */
+		hw->imr &= ~ALTERA_SPI_CONTROL_IRRDY_MSK;
+		writel(hw->imr, hw->base + ALTERA_SPI_CONTROL);
 
-	if (hw->count < hw->len)
-		writel(hw_txbyte(hw, hw->count), hw->base + ALTERA_SPI_TXDATA);
-	else
-		complete(&hw->done);
+		spi_finalize_current_transfer(master);
+	}
 
 	return IRQ_HANDLED;
 }
@@ -214,14 +186,10 @@ static int altera_spi_probe(struct platform_device *pdev)
 	master->mode_bits = SPI_CS_HIGH;
 	master->bits_per_word_mask = SPI_BPW_RANGE_MASK(1, 16);
 	master->dev.of_node = pdev->dev.of_node;
+	master->transfer_one = altera_spi_txrx;
+	master->set_cs = altera_spi_set_cs;
 
 	hw = spi_master_get_devdata(master);
-	platform_set_drvdata(pdev, hw);
-
-	/* setup the state for the bitbang driver */
-	hw->bitbang.master = master;
-	hw->bitbang.chipselect = altera_spi_chipsel;
-	hw->bitbang.txrx_bufs = altera_spi_txrx;
 
 	/* find and map our resources */
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
@@ -239,15 +207,13 @@ static int altera_spi_probe(struct platform_device *pdev)
 	/* irq is optional */
 	hw->irq = platform_get_irq(pdev, 0);
 	if (hw->irq >= 0) {
-		init_completion(&hw->done);
 		err = devm_request_irq(&pdev->dev, hw->irq, altera_spi_irq, 0,
-				       pdev->name, hw);
+				       pdev->name, master);
 		if (err)
 			goto exit;
 	}
 
-	/* register our spi controller */
-	err = spi_bitbang_start(&hw->bitbang);
+	err = devm_spi_register_master(&pdev->dev, master);
 	if (err)
 		goto exit;
 	dev_info(&pdev->dev, "base %p, irq %d\n", hw->base, hw->irq);
@@ -258,16 +224,6 @@ exit:
 	return err;
 }
 
-static int altera_spi_remove(struct platform_device *dev)
-{
-	struct altera_spi *hw = platform_get_drvdata(dev);
-	struct spi_master *master = hw->bitbang.master;
-
-	spi_bitbang_stop(&hw->bitbang);
-	spi_master_put(master);
-	return 0;
-}
-
 #ifdef CONFIG_OF
 static const struct of_device_id altera_spi_match[] = {
 	{ .compatible = "ALTR,spi-1.0", },
@@ -279,7 +235,6 @@ MODULE_DEVICE_TABLE(of, altera_spi_match);
 
 static struct platform_driver altera_spi_driver = {
 	.probe = altera_spi_probe,
-	.remove = altera_spi_remove,
 	.driver = {
 		.name = DRV_NAME,
 		.pm = NULL,
diff --git a/drivers/spi/spi-ath79.c b/drivers/spi/spi-ath79.c
index b89cee11f418..0719bd484891 100644
--- a/drivers/spi/spi-ath79.c
+++ b/drivers/spi/spi-ath79.c
@@ -39,15 +39,15 @@ struct ath79_spi {
 	u32			reg_ctrl;
 	void __iomem		*base;
 	struct clk		*clk;
-	unsigned		rrw_delay;
+	unsigned int		rrw_delay;
 };
 
-static inline u32 ath79_spi_rr(struct ath79_spi *sp, unsigned reg)
+static inline u32 ath79_spi_rr(struct ath79_spi *sp, unsigned int reg)
 {
 	return ioread32(sp->base + reg);
 }
 
-static inline void ath79_spi_wr(struct ath79_spi *sp, unsigned reg, u32 val)
+static inline void ath79_spi_wr(struct ath79_spi *sp, unsigned int reg, u32 val)
 {
 	iowrite32(val, sp->base + reg);
 }
@@ -57,7 +57,7 @@ static inline struct ath79_spi *ath79_spidev_to_sp(struct spi_device *spi)
 	return spi_master_get_devdata(spi->master);
 }
 
-static inline void ath79_spi_delay(struct ath79_spi *sp, unsigned nsecs)
+static inline void ath79_spi_delay(struct ath79_spi *sp, unsigned int nsecs)
 {
 	if (nsecs > sp->rrw_delay)
 		ndelay(nsecs - sp->rrw_delay);
@@ -148,9 +148,8 @@ static int ath79_spi_setup_cs(struct spi_device *spi)
 
 static void ath79_spi_cleanup_cs(struct spi_device *spi)
 {
-	if (gpio_is_valid(spi->cs_gpio)) {
+	if (gpio_is_valid(spi->cs_gpio))
 		gpio_free(spi->cs_gpio);
-	}
 }
 
 static int ath79_spi_setup(struct spi_device *spi)
@@ -176,7 +175,7 @@ static void ath79_spi_cleanup(struct spi_device *spi)
 	spi_bitbang_cleanup(spi);
 }
 
-static u32 ath79_spi_txrx_mode0(struct spi_device *spi, unsigned nsecs,
+static u32 ath79_spi_txrx_mode0(struct spi_device *spi, unsigned int nsecs,
 			       u32 word, u8 bits)
 {
 	struct ath79_spi *sp = ath79_spidev_to_sp(spi);
diff --git a/drivers/spi/spi-bcm-qspi.c b/drivers/spi/spi-bcm-qspi.c
index b19722ba908c..6ef6c44f39f5 100644
--- a/drivers/spi/spi-bcm-qspi.c
+++ b/drivers/spi/spi-bcm-qspi.c
@@ -25,7 +25,6 @@
 #include <linux/ioport.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/mtd/spi-nor.h>
 #include <linux/of.h>
 #include <linux/of_irq.h>
 #include <linux/platform_device.h>
@@ -349,76 +348,60 @@ static void bcm_qspi_bspi_set_xfer_params(struct bcm_qspi *qspi, u8 cmd_byte,
 	bcm_qspi_write(qspi, BSPI, BSPI_FLEX_MODE_ENABLE, flex_mode);
 }
 
-static int bcm_qspi_bspi_set_flex_mode(struct bcm_qspi *qspi, int width,
-				       int addrlen, int hp)
+static int bcm_qspi_bspi_set_flex_mode(struct bcm_qspi *qspi,
+				       struct spi_flash_read_message *msg,
+				       int hp)
 {
 	int bpc = 0, bpp = 0;
-	u8 command = SPINOR_OP_READ_FAST;
-	int flex_mode = 1, rv = 0;
-	bool spans_4byte = false;
+	u8 command = msg->read_opcode;
+	int width  = msg->data_nbits ? msg->data_nbits : SPI_NBITS_SINGLE;
+	int addrlen = msg->addr_width;
+	int addr_nbits = msg->addr_nbits ? msg->addr_nbits : SPI_NBITS_SINGLE;
+	int flex_mode = 1;
 
 	dev_dbg(&qspi->pdev->dev, "set flex mode w %x addrlen %x hp %d\n",
 		width, addrlen, hp);
 
-	if (addrlen == BSPI_ADDRLEN_4BYTES) {
+	if (addrlen == BSPI_ADDRLEN_4BYTES)
 		bpp = BSPI_BPP_ADDR_SELECT_MASK;
-		spans_4byte = true;
-	}
 
-	bpp |= 8;
+	bpp |= msg->dummy_bytes * (8/addr_nbits);
 
 	switch (width) {
 	case SPI_NBITS_SINGLE:
 		if (addrlen == BSPI_ADDRLEN_3BYTES)
 			/* default mode, does not need flex_cmd */
 			flex_mode = 0;
-		else
-			command = SPINOR_OP_READ_FAST_4B;
 		break;
 	case SPI_NBITS_DUAL:
 		bpc = 0x00000001;
 		if (hp) {
 			bpc |= 0x00010100; /* address and mode are 2-bit */
 			bpp = BSPI_BPP_MODE_SELECT_MASK;
-			command = OPCODE_DIOR;
-			if (spans_4byte)
-				command = OPCODE_DIOR_4B;
-		} else {
-			command = SPINOR_OP_READ_1_1_2;
-			if (spans_4byte)
-				command = SPINOR_OP_READ_1_1_2_4B;
 		}
 		break;
 	case SPI_NBITS_QUAD:
 		bpc = 0x00000002;
 		if (hp) {
 			bpc |= 0x00020200; /* address and mode are 4-bit */
-			bpp = 4; /* dummy cycles */
-			bpp |= BSPI_BPP_ADDR_SELECT_MASK;
-			command = OPCODE_QIOR;
-			if (spans_4byte)
-				command = OPCODE_QIOR_4B;
-		} else {
-			command = SPINOR_OP_READ_1_1_4;
-			if (spans_4byte)
-				command = SPINOR_OP_READ_1_1_4_4B;
+			bpp |= BSPI_BPP_MODE_SELECT_MASK;
 		}
 		break;
 	default:
-		rv = -EINVAL;
-		break;
+		return -EINVAL;
 	}
 
-	if (rv == 0)
-		bcm_qspi_bspi_set_xfer_params(qspi, command, bpp, bpc,
-					      flex_mode);
+	bcm_qspi_bspi_set_xfer_params(qspi, command, bpp, bpc, flex_mode);
 
-	return rv;
+	return 0;
 }
 
-static int bcm_qspi_bspi_set_override(struct bcm_qspi *qspi, int width,
-				      int addrlen, int hp)
+static int bcm_qspi_bspi_set_override(struct bcm_qspi *qspi,
+				      struct spi_flash_read_message *msg,
+				      int hp)
 {
+	int width = msg->data_nbits ? msg->data_nbits : SPI_NBITS_SINGLE;
+	int addrlen = msg->addr_width;
 	u32 data = bcm_qspi_read(qspi, BSPI, BSPI_STRAP_OVERRIDE_CTRL);
 
 	dev_dbg(&qspi->pdev->dev, "set override mode w %x addrlen %x hp %d\n",
@@ -430,7 +413,6 @@ static int bcm_qspi_bspi_set_override(struct bcm_qspi *qspi, int width,
 		data &= ~(BSPI_STRAP_OVERRIDE_CTRL_DATA_QUAD |
 			  BSPI_STRAP_OVERRIDE_CTRL_DATA_DUAL);
 		break;
-
 	case SPI_NBITS_QUAD:
 		/* clear dual mode and set quad mode */
 		data &= ~BSPI_STRAP_OVERRIDE_CTRL_DATA_DUAL;
@@ -455,15 +437,17 @@ static int bcm_qspi_bspi_set_override(struct bcm_qspi *qspi, int width,
 	/* set the override mode */
 	data |=	BSPI_STRAP_OVERRIDE_CTRL_OVERRIDE;
 	bcm_qspi_write(qspi, BSPI, BSPI_STRAP_OVERRIDE_CTRL, data);
-	bcm_qspi_bspi_set_xfer_params(qspi, SPINOR_OP_READ_FAST, 0, 0, 0);
+	bcm_qspi_bspi_set_xfer_params(qspi, msg->read_opcode, 0, 0, 0);
 
 	return 0;
 }
 
 static int bcm_qspi_bspi_set_mode(struct bcm_qspi *qspi,
-				  int width, int addrlen, int hp)
+				  struct spi_flash_read_message *msg, int hp)
 {
 	int error = 0;
+	int width = msg->data_nbits ? msg->data_nbits : SPI_NBITS_SINGLE;
+	int addrlen = msg->addr_width;
 
 	/* default mode */
 	qspi->xfer_mode.flex_mode = true;
@@ -475,23 +459,13 @@ static int bcm_qspi_bspi_set_mode(struct bcm_qspi *qspi,
 		mask = BSPI_STRAP_OVERRIDE_CTRL_OVERRIDE;
 		if (val & mask || qspi->s3_strap_override_ctrl & mask) {
 			qspi->xfer_mode.flex_mode = false;
-			bcm_qspi_write(qspi, BSPI, BSPI_FLEX_MODE_ENABLE,
-				       0);
-
-			if ((val | qspi->s3_strap_override_ctrl) &
-			    BSPI_STRAP_OVERRIDE_CTRL_DATA_DUAL)
-				width = SPI_NBITS_DUAL;
-			else if ((val |  qspi->s3_strap_override_ctrl) &
-				 BSPI_STRAP_OVERRIDE_CTRL_DATA_QUAD)
-				width = SPI_NBITS_QUAD;
-
-			error = bcm_qspi_bspi_set_override(qspi, width, addrlen,
-							   hp);
+			bcm_qspi_write(qspi, BSPI, BSPI_FLEX_MODE_ENABLE, 0);
+			error = bcm_qspi_bspi_set_override(qspi, msg, hp);
 		}
 	}
 
 	if (qspi->xfer_mode.flex_mode)
-		error = bcm_qspi_bspi_set_flex_mode(qspi, width, addrlen, hp);
+		error = bcm_qspi_bspi_set_flex_mode(qspi, msg, hp);
 
 	if (error) {
 		dev_warn(&qspi->pdev->dev,
@@ -981,7 +955,7 @@ static int bcm_qspi_flash_read(struct spi_device *spi,
 	struct bcm_qspi *qspi = spi_master_get_devdata(spi->master);
 	int ret = 0;
 	bool mspi_read = false;
-	u32 io_width, addrlen, addr, len;
+	u32 addr, len;
 	u_char *buf;
 
 	buf = msg->buf;
@@ -1010,9 +984,7 @@ static int bcm_qspi_flash_read(struct spi_device *spi,
 	if (mspi_read)
 		return bcm_qspi_mspi_flash_read(spi, msg);
 
-	io_width = msg->data_nbits ? msg->data_nbits : SPI_NBITS_SINGLE;
-	addrlen = msg->addr_width;
-	ret = bcm_qspi_bspi_set_mode(qspi, io_width, addrlen, -1);
+	ret = bcm_qspi_bspi_set_mode(qspi, msg, -1);
 
 	if (!ret)
 		ret = bcm_qspi_bspi_flash_read(spi, msg);
@@ -1422,6 +1394,11 @@ static int __maybe_unused bcm_qspi_suspend(struct device *dev)
 {
 	struct bcm_qspi *qspi = dev_get_drvdata(dev);
 
+	/* store the override strap value */
+	if (!bcm_qspi_bspi_ver_three(qspi))
+		qspi->s3_strap_override_ctrl =
+			bcm_qspi_read(qspi, BSPI, BSPI_STRAP_OVERRIDE_CTRL);
+
 	spi_master_suspend(qspi->master);
 	clk_disable(qspi->clk);
 	bcm_qspi_hw_uninit(qspi);
diff --git a/drivers/spi/spi-bcm63xx-hsspi.c b/drivers/spi/spi-bcm63xx-hsspi.c
index 4da2d4a524ca..cbcba614b253 100644
--- a/drivers/spi/spi-bcm63xx-hsspi.c
+++ b/drivers/spi/spi-bcm63xx-hsspi.c
@@ -108,7 +108,7 @@ struct bcm63xx_hsspi {
 	u8 cs_polarity;
 };
 
-static void bcm63xx_hsspi_set_cs(struct bcm63xx_hsspi *bs, unsigned cs,
+static void bcm63xx_hsspi_set_cs(struct bcm63xx_hsspi *bs, unsigned int cs,
 				 bool active)
 {
 	u32 reg;
@@ -127,7 +127,7 @@ static void bcm63xx_hsspi_set_cs(struct bcm63xx_hsspi *bs, unsigned cs,
 static void bcm63xx_hsspi_set_clk(struct bcm63xx_hsspi *bs,
 				  struct spi_device *spi, int hz)
 {
-	unsigned profile = spi->chip_select;
+	unsigned int profile = spi->chip_select;
 	u32 reg;
 
 	reg = DIV_ROUND_UP(2048, DIV_ROUND_UP(bs->speed_hz, hz));
@@ -154,7 +154,7 @@ static void bcm63xx_hsspi_set_clk(struct bcm63xx_hsspi *bs,
 static int bcm63xx_hsspi_do_txrx(struct spi_device *spi, struct spi_transfer *t)
 {
 	struct bcm63xx_hsspi *bs = spi_master_get_devdata(spi->master);
-	unsigned chip_select = spi->chip_select;
+	unsigned int chip_select = spi->chip_select;
 	u16 opcode = 0;
 	int pending = t->len;
 	int step_size = HSSPI_BUFFER_LEN;
@@ -338,8 +338,8 @@ static int bcm63xx_hsspi_probe(struct platform_device *pdev)
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
-		dev_err(dev, "no irq\n");
-		return -ENXIO;
+		dev_err(dev, "no irq: %d\n", irq);
+		return irq;
 	}
 
 	res_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
diff --git a/drivers/spi/spi-bcm63xx.c b/drivers/spi/spi-bcm63xx.c
index 84c7356ce5b4..bfe5754768f9 100644
--- a/drivers/spi/spi-bcm63xx.c
+++ b/drivers/spi/spi-bcm63xx.c
@@ -530,8 +530,8 @@ static int bcm63xx_spi_probe(struct platform_device *pdev)
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
-		dev_err(dev, "no irq\n");
-		return -ENXIO;
+		dev_err(dev, "no irq: %d\n", irq);
+		return irq;
 	}
 
 	clk = devm_clk_get(dev, "spi");
diff --git a/drivers/spi/spi-cadence.c b/drivers/spi/spi-cadence.c
index f0b5c7b91f37..5c9516ae4942 100644
--- a/drivers/spi/spi-cadence.c
+++ b/drivers/spi/spi-cadence.c
@@ -576,10 +576,10 @@ static int cdns_spi_probe(struct platform_device *pdev)
 		goto clk_dis_apb;
 	}
 
-	pm_runtime_enable(&pdev->dev);
 	pm_runtime_use_autosuspend(&pdev->dev);
 	pm_runtime_set_autosuspend_delay(&pdev->dev, SPI_AUTOSUSPEND_TIMEOUT);
 	pm_runtime_set_active(&pdev->dev);
+	pm_runtime_enable(&pdev->dev);
 
 	ret = of_property_read_u32(pdev->dev.of_node, "num-cs", &num_cs);
 	if (ret < 0)
@@ -704,7 +704,9 @@ static int __maybe_unused cdns_spi_resume(struct device *dev)
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct spi_master *master = platform_get_drvdata(pdev);
+	struct cdns_spi *xspi = spi_master_get_devdata(master);
 
+	cdns_spi_init_hw(xspi);
 	return spi_master_resume(master);
 }
 
diff --git a/drivers/spi/spi-ep93xx.c b/drivers/spi/spi-ep93xx.c
index b5d766064b7b..e5cc07357746 100644
--- a/drivers/spi/spi-ep93xx.c
+++ b/drivers/spi/spi-ep93xx.c
@@ -70,12 +70,9 @@
 
 /**
  * struct ep93xx_spi - EP93xx SPI controller structure
- * @pdev: pointer to platform device
  * @clk: clock for the controller
- * @regs_base: pointer to ioremap()'d registers
+ * @mmio: pointer to ioremap()'d registers
  * @sspdr_phys: physical address of the SSPDR register
- * @wait: wait here until given transfer is completed
- * @current_msg: message that is currently processed (or %NULL if none)
  * @tx: current byte in transfer to transmit
  * @rx: current byte in transfer to receive
  * @fifo_level: how full is FIFO (%0..%SPI_FIFO_SIZE - %1). Receiving one
@@ -90,12 +87,9 @@
  *            the client
  */
 struct ep93xx_spi {
-	const struct platform_device	*pdev;
 	struct clk			*clk;
-	void __iomem			*regs_base;
+	void __iomem			*mmio;
 	unsigned long			sspdr_phys;
-	struct completion		wait;
-	struct spi_message		*current_msg;
 	size_t				tx;
 	size_t				rx;
 	size_t				fifo_level;
@@ -111,91 +105,23 @@ struct ep93xx_spi {
 /* converts bits per word to CR0.DSS value */
 #define bits_per_word_to_dss(bpw)	((bpw) - 1)
 
-static void ep93xx_spi_write_u8(const struct ep93xx_spi *espi,
-				u16 reg, u8 value)
-{
-	writeb(value, espi->regs_base + reg);
-}
-
-static u8 ep93xx_spi_read_u8(const struct ep93xx_spi *spi, u16 reg)
-{
-	return readb(spi->regs_base + reg);
-}
-
-static void ep93xx_spi_write_u16(const struct ep93xx_spi *espi,
-				 u16 reg, u16 value)
-{
-	writew(value, espi->regs_base + reg);
-}
-
-static u16 ep93xx_spi_read_u16(const struct ep93xx_spi *spi, u16 reg)
-{
-	return readw(spi->regs_base + reg);
-}
-
-static int ep93xx_spi_enable(const struct ep93xx_spi *espi)
-{
-	u8 regval;
-	int err;
-
-	err = clk_enable(espi->clk);
-	if (err)
-		return err;
-
-	regval = ep93xx_spi_read_u8(espi, SSPCR1);
-	regval |= SSPCR1_SSE;
-	ep93xx_spi_write_u8(espi, SSPCR1, regval);
-
-	return 0;
-}
-
-static void ep93xx_spi_disable(const struct ep93xx_spi *espi)
-{
-	u8 regval;
-
-	regval = ep93xx_spi_read_u8(espi, SSPCR1);
-	regval &= ~SSPCR1_SSE;
-	ep93xx_spi_write_u8(espi, SSPCR1, regval);
-
-	clk_disable(espi->clk);
-}
-
-static void ep93xx_spi_enable_interrupts(const struct ep93xx_spi *espi)
-{
-	u8 regval;
-
-	regval = ep93xx_spi_read_u8(espi, SSPCR1);
-	regval |= (SSPCR1_RORIE | SSPCR1_TIE | SSPCR1_RIE);
-	ep93xx_spi_write_u8(espi, SSPCR1, regval);
-}
-
-static void ep93xx_spi_disable_interrupts(const struct ep93xx_spi *espi)
-{
-	u8 regval;
-
-	regval = ep93xx_spi_read_u8(espi, SSPCR1);
-	regval &= ~(SSPCR1_RORIE | SSPCR1_TIE | SSPCR1_RIE);
-	ep93xx_spi_write_u8(espi, SSPCR1, regval);
-}
-
 /**
  * ep93xx_spi_calc_divisors() - calculates SPI clock divisors
- * @espi: ep93xx SPI controller struct
+ * @master: SPI master
  * @rate: desired SPI output clock rate
  * @div_cpsr: pointer to return the cpsr (pre-scaler) divider
  * @div_scr: pointer to return the scr divider
  */
-static int ep93xx_spi_calc_divisors(const struct ep93xx_spi *espi,
+static int ep93xx_spi_calc_divisors(struct spi_master *master,
 				    u32 rate, u8 *div_cpsr, u8 *div_scr)
 {
-	struct spi_master *master = platform_get_drvdata(espi->pdev);
+	struct ep93xx_spi *espi = spi_master_get_devdata(master);
 	unsigned long spi_clk_rate = clk_get_rate(espi->clk);
 	int cpsr, scr;
 
 	/*
 	 * Make sure that max value is between values supported by the
-	 * controller. Note that minimum value is already checked in
-	 * ep93xx_spi_transfer_one_message().
+	 * controller.
 	 */
 	rate = clamp(rate, master->min_speed_hz, master->max_speed_hz);
 
@@ -220,26 +146,18 @@ static int ep93xx_spi_calc_divisors(const struct ep93xx_spi *espi,
 	return -EINVAL;
 }
 
-static void ep93xx_spi_cs_control(struct spi_device *spi, bool enable)
-{
-	if (spi->mode & SPI_CS_HIGH)
-		enable = !enable;
-
-	if (gpio_is_valid(spi->cs_gpio))
-		gpio_set_value(spi->cs_gpio, !enable);
-}
-
-static int ep93xx_spi_chip_setup(const struct ep93xx_spi *espi,
+static int ep93xx_spi_chip_setup(struct spi_master *master,
 				 struct spi_device *spi,
 				 struct spi_transfer *xfer)
 {
+	struct ep93xx_spi *espi = spi_master_get_devdata(master);
 	u8 dss = bits_per_word_to_dss(xfer->bits_per_word);
 	u8 div_cpsr = 0;
 	u8 div_scr = 0;
 	u16 cr0;
 	int err;
 
-	err = ep93xx_spi_calc_divisors(espi, xfer->speed_hz,
+	err = ep93xx_spi_calc_divisors(master, xfer->speed_hz,
 				       &div_cpsr, &div_scr);
 	if (err)
 		return err;
@@ -248,51 +166,49 @@ static int ep93xx_spi_chip_setup(const struct ep93xx_spi *espi,
 	cr0 |= (spi->mode & (SPI_CPHA | SPI_CPOL)) << SSPCR0_MODE_SHIFT;
 	cr0 |= dss;
 
-	dev_dbg(&espi->pdev->dev, "setup: mode %d, cpsr %d, scr %d, dss %d\n",
+	dev_dbg(&master->dev, "setup: mode %d, cpsr %d, scr %d, dss %d\n",
 		spi->mode, div_cpsr, div_scr, dss);
-	dev_dbg(&espi->pdev->dev, "setup: cr0 %#x\n", cr0);
+	dev_dbg(&master->dev, "setup: cr0 %#x\n", cr0);
 
-	ep93xx_spi_write_u8(espi, SSPCPSR, div_cpsr);
-	ep93xx_spi_write_u16(espi, SSPCR0, cr0);
+	writel(div_cpsr, espi->mmio + SSPCPSR);
+	writel(cr0, espi->mmio + SSPCR0);
 
 	return 0;
 }
 
-static void ep93xx_do_write(struct ep93xx_spi *espi, struct spi_transfer *t)
+static void ep93xx_do_write(struct spi_master *master)
 {
-	if (t->bits_per_word > 8) {
-		u16 tx_val = 0;
+	struct ep93xx_spi *espi = spi_master_get_devdata(master);
+	struct spi_transfer *xfer = master->cur_msg->state;
+	u32 val = 0;
 
-		if (t->tx_buf)
-			tx_val = ((u16 *)t->tx_buf)[espi->tx];
-		ep93xx_spi_write_u16(espi, SSPDR, tx_val);
-		espi->tx += sizeof(tx_val);
+	if (xfer->bits_per_word > 8) {
+		if (xfer->tx_buf)
+			val = ((u16 *)xfer->tx_buf)[espi->tx];
+		espi->tx += 2;
 	} else {
-		u8 tx_val = 0;
-
-		if (t->tx_buf)
-			tx_val = ((u8 *)t->tx_buf)[espi->tx];
-		ep93xx_spi_write_u8(espi, SSPDR, tx_val);
-		espi->tx += sizeof(tx_val);
+		if (xfer->tx_buf)
+			val = ((u8 *)xfer->tx_buf)[espi->tx];
+		espi->tx += 1;
 	}
+	writel(val, espi->mmio + SSPDR);
 }
 
-static void ep93xx_do_read(struct ep93xx_spi *espi, struct spi_transfer *t)
+static void ep93xx_do_read(struct spi_master *master)
 {
-	if (t->bits_per_word > 8) {
-		u16 rx_val;
-
-		rx_val = ep93xx_spi_read_u16(espi, SSPDR);
-		if (t->rx_buf)
-			((u16 *)t->rx_buf)[espi->rx] = rx_val;
-		espi->rx += sizeof(rx_val);
+	struct ep93xx_spi *espi = spi_master_get_devdata(master);
+	struct spi_transfer *xfer = master->cur_msg->state;
+	u32 val;
+
+	val = readl(espi->mmio + SSPDR);
+	if (xfer->bits_per_word > 8) {
+		if (xfer->rx_buf)
+			((u16 *)xfer->rx_buf)[espi->rx] = val;
+		espi->rx += 2;
 	} else {
-		u8 rx_val;
-
-		rx_val = ep93xx_spi_read_u8(espi, SSPDR);
-		if (t->rx_buf)
-			((u8 *)t->rx_buf)[espi->rx] = rx_val;
-		espi->rx += sizeof(rx_val);
+		if (xfer->rx_buf)
+			((u8 *)xfer->rx_buf)[espi->rx] = val;
+		espi->rx += 1;
 	}
 }
 
@@ -307,44 +223,32 @@ static void ep93xx_do_read(struct ep93xx_spi *espi, struct spi_transfer *t)
  * When this function is finished, RX FIFO should be empty and TX FIFO should be
  * full.
  */
-static int ep93xx_spi_read_write(struct ep93xx_spi *espi)
+static int ep93xx_spi_read_write(struct spi_master *master)
 {
-	struct spi_message *msg = espi->current_msg;
-	struct spi_transfer *t = msg->state;
+	struct ep93xx_spi *espi = spi_master_get_devdata(master);
+	struct spi_transfer *xfer = master->cur_msg->state;
 
 	/* read as long as RX FIFO has frames in it */
-	while ((ep93xx_spi_read_u8(espi, SSPSR) & SSPSR_RNE)) {
-		ep93xx_do_read(espi, t);
+	while ((readl(espi->mmio + SSPSR) & SSPSR_RNE)) {
+		ep93xx_do_read(master);
 		espi->fifo_level--;
 	}
 
 	/* write as long as TX FIFO has room */
-	while (espi->fifo_level < SPI_FIFO_SIZE && espi->tx < t->len) {
-		ep93xx_do_write(espi, t);
+	while (espi->fifo_level < SPI_FIFO_SIZE && espi->tx < xfer->len) {
+		ep93xx_do_write(master);
 		espi->fifo_level++;
 	}
 
-	if (espi->rx == t->len)
+	if (espi->rx == xfer->len)
 		return 0;
 
 	return -EINPROGRESS;
 }
 
-static void ep93xx_spi_pio_transfer(struct ep93xx_spi *espi)
-{
-	/*
-	 * Now everything is set up for the current transfer. We prime the TX
-	 * FIFO, enable interrupts, and wait for the transfer to complete.
-	 */
-	if (ep93xx_spi_read_write(espi)) {
-		ep93xx_spi_enable_interrupts(espi);
-		wait_for_completion(&espi->wait);
-	}
-}
-
 /**
  * ep93xx_spi_dma_prepare() - prepares a DMA transfer
- * @espi: ep93xx SPI controller struct
+ * @master: SPI master
  * @dir: DMA transfer direction
  *
  * Function configures the DMA, maps the buffer and prepares the DMA
@@ -352,9 +256,11 @@ static void ep93xx_spi_pio_transfer(struct ep93xx_spi *espi)
  * in case of failure.
  */
 static struct dma_async_tx_descriptor *
-ep93xx_spi_dma_prepare(struct ep93xx_spi *espi, enum dma_transfer_direction dir)
+ep93xx_spi_dma_prepare(struct spi_master *master,
+		       enum dma_transfer_direction dir)
 {
-	struct spi_transfer *t = espi->current_msg->state;
+	struct ep93xx_spi *espi = spi_master_get_devdata(master);
+	struct spi_transfer *xfer = master->cur_msg->state;
 	struct dma_async_tx_descriptor *txd;
 	enum dma_slave_buswidth buswidth;
 	struct dma_slave_config conf;
@@ -362,10 +268,10 @@ ep93xx_spi_dma_prepare(struct ep93xx_spi *espi, enum dma_transfer_direction dir)
 	struct sg_table *sgt;
 	struct dma_chan *chan;
 	const void *buf, *pbuf;
-	size_t len = t->len;
+	size_t len = xfer->len;
 	int i, ret, nents;
 
-	if (t->bits_per_word > 8)
+	if (xfer->bits_per_word > 8)
 		buswidth = DMA_SLAVE_BUSWIDTH_2_BYTES;
 	else
 		buswidth = DMA_SLAVE_BUSWIDTH_1_BYTE;
@@ -375,14 +281,14 @@ ep93xx_spi_dma_prepare(struct ep93xx_spi *espi, enum dma_transfer_direction dir)
 
 	if (dir == DMA_DEV_TO_MEM) {
 		chan = espi->dma_rx;
-		buf = t->rx_buf;
+		buf = xfer->rx_buf;
 		sgt = &espi->rx_sgt;
 
 		conf.src_addr = espi->sspdr_phys;
 		conf.src_addr_width = buswidth;
 	} else {
 		chan = espi->dma_tx;
-		buf = t->tx_buf;
+		buf = xfer->tx_buf;
 		sgt = &espi->tx_sgt;
 
 		conf.dst_addr = espi->sspdr_phys;
@@ -429,7 +335,7 @@ ep93xx_spi_dma_prepare(struct ep93xx_spi *espi, enum dma_transfer_direction dir)
 	}
 
 	if (WARN_ON(len)) {
-		dev_warn(&espi->pdev->dev, "len = %zu expected 0!\n", len);
+		dev_warn(&master->dev, "len = %zu expected 0!\n", len);
 		return ERR_PTR(-EINVAL);
 	}
 
@@ -447,15 +353,16 @@ ep93xx_spi_dma_prepare(struct ep93xx_spi *espi, enum dma_transfer_direction dir)
 
 /**
  * ep93xx_spi_dma_finish() - finishes with a DMA transfer
- * @espi: ep93xx SPI controller struct
+ * @master: SPI master
  * @dir: DMA transfer direction
  *
  * Function finishes with the DMA transfer. After this, the DMA buffer is
  * unmapped.
  */
-static void ep93xx_spi_dma_finish(struct ep93xx_spi *espi,
+static void ep93xx_spi_dma_finish(struct spi_master *master,
 				  enum dma_transfer_direction dir)
 {
+	struct ep93xx_spi *espi = spi_master_get_devdata(master);
 	struct dma_chan *chan;
 	struct sg_table *sgt;
 
@@ -472,72 +379,107 @@ static void ep93xx_spi_dma_finish(struct ep93xx_spi *espi,
 
 static void ep93xx_spi_dma_callback(void *callback_param)
 {
-	complete(callback_param);
+	struct spi_master *master = callback_param;
+
+	ep93xx_spi_dma_finish(master, DMA_MEM_TO_DEV);
+	ep93xx_spi_dma_finish(master, DMA_DEV_TO_MEM);
+
+	spi_finalize_current_transfer(master);
 }
 
-static void ep93xx_spi_dma_transfer(struct ep93xx_spi *espi)
+static int ep93xx_spi_dma_transfer(struct spi_master *master)
 {
-	struct spi_message *msg = espi->current_msg;
+	struct ep93xx_spi *espi = spi_master_get_devdata(master);
 	struct dma_async_tx_descriptor *rxd, *txd;
 
-	rxd = ep93xx_spi_dma_prepare(espi, DMA_DEV_TO_MEM);
+	rxd = ep93xx_spi_dma_prepare(master, DMA_DEV_TO_MEM);
 	if (IS_ERR(rxd)) {
-		dev_err(&espi->pdev->dev, "DMA RX failed: %ld\n", PTR_ERR(rxd));
-		msg->status = PTR_ERR(rxd);
-		return;
+		dev_err(&master->dev, "DMA RX failed: %ld\n", PTR_ERR(rxd));
+		return PTR_ERR(rxd);
 	}
 
-	txd = ep93xx_spi_dma_prepare(espi, DMA_MEM_TO_DEV);
+	txd = ep93xx_spi_dma_prepare(master, DMA_MEM_TO_DEV);
 	if (IS_ERR(txd)) {
-		ep93xx_spi_dma_finish(espi, DMA_DEV_TO_MEM);
-		dev_err(&espi->pdev->dev, "DMA TX failed: %ld\n", PTR_ERR(txd));
-		msg->status = PTR_ERR(txd);
-		return;
+		ep93xx_spi_dma_finish(master, DMA_DEV_TO_MEM);
+		dev_err(&master->dev, "DMA TX failed: %ld\n", PTR_ERR(txd));
+		return PTR_ERR(txd);
 	}
 
 	/* We are ready when RX is done */
 	rxd->callback = ep93xx_spi_dma_callback;
-	rxd->callback_param = &espi->wait;
+	rxd->callback_param = master;
 
-	/* Now submit both descriptors and wait while they finish */
+	/* Now submit both descriptors and start DMA */
 	dmaengine_submit(rxd);
 	dmaengine_submit(txd);
 
 	dma_async_issue_pending(espi->dma_rx);
 	dma_async_issue_pending(espi->dma_tx);
 
-	wait_for_completion(&espi->wait);
-
-	ep93xx_spi_dma_finish(espi, DMA_MEM_TO_DEV);
-	ep93xx_spi_dma_finish(espi, DMA_DEV_TO_MEM);
+	/* signal that we need to wait for completion */
+	return 1;
 }
 
-/**
- * ep93xx_spi_process_transfer() - processes one SPI transfer
- * @espi: ep93xx SPI controller struct
- * @msg: current message
- * @t: transfer to process
- *
- * This function processes one SPI transfer given in @t. Function waits until
- * transfer is complete (may sleep) and updates @msg->status based on whether
- * transfer was successfully processed or not.
- */
-static void ep93xx_spi_process_transfer(struct ep93xx_spi *espi,
-					struct spi_message *msg,
-					struct spi_transfer *t)
+static irqreturn_t ep93xx_spi_interrupt(int irq, void *dev_id)
 {
-	int err;
+	struct spi_master *master = dev_id;
+	struct ep93xx_spi *espi = spi_master_get_devdata(master);
+	u32 val;
+
+	/*
+	 * If we got ROR (receive overrun) interrupt we know that something is
+	 * wrong. Just abort the message.
+	 */
+	if (readl(espi->mmio + SSPIIR) & SSPIIR_RORIS) {
+		/* clear the overrun interrupt */
+		writel(0, espi->mmio + SSPICR);
+		dev_warn(&master->dev,
+			 "receive overrun, aborting the message\n");
+		master->cur_msg->status = -EIO;
+	} else {
+		/*
+		 * Interrupt is either RX (RIS) or TX (TIS). For both cases we
+		 * simply execute next data transfer.
+		 */
+		if (ep93xx_spi_read_write(master)) {
+			/*
+			 * In normal case, there still is some processing left
+			 * for current transfer. Let's wait for the next
+			 * interrupt then.
+			 */
+			return IRQ_HANDLED;
+		}
+	}
 
-	msg->state = t;
+	/*
+	 * Current transfer is finished, either with error or with success. In
+	 * any case we disable interrupts and notify the worker to handle
+	 * any post-processing of the message.
+	 */
+	val = readl(espi->mmio + SSPCR1);
+	val &= ~(SSPCR1_RORIE | SSPCR1_TIE | SSPCR1_RIE);
+	writel(val, espi->mmio + SSPCR1);
+
+	spi_finalize_current_transfer(master);
+
+	return IRQ_HANDLED;
+}
 
-	err = ep93xx_spi_chip_setup(espi, msg->spi, t);
-	if (err) {
-		dev_err(&espi->pdev->dev,
-			"failed to setup chip for transfer\n");
-		msg->status = err;
-		return;
+static int ep93xx_spi_transfer_one(struct spi_master *master,
+				   struct spi_device *spi,
+				   struct spi_transfer *xfer)
+{
+	struct ep93xx_spi *espi = spi_master_get_devdata(master);
+	u32 val;
+	int ret;
+
+	ret = ep93xx_spi_chip_setup(master, spi, xfer);
+	if (ret) {
+		dev_err(&master->dev, "failed to setup chip for transfer\n");
+		return ret;
 	}
 
+	master->cur_msg->state = xfer;
 	espi->rx = 0;
 	espi->tx = 0;
 
@@ -546,83 +488,37 @@ static void ep93xx_spi_process_transfer(struct ep93xx_spi *espi,
 	 * fit into the FIFO and can be transferred with a single interrupt.
 	 * So in these cases we will be using PIO and don't bother for DMA.
 	 */
-	if (espi->dma_rx && t->len > SPI_FIFO_SIZE)
-		ep93xx_spi_dma_transfer(espi);
-	else
-		ep93xx_spi_pio_transfer(espi);
+	if (espi->dma_rx && xfer->len > SPI_FIFO_SIZE)
+		return ep93xx_spi_dma_transfer(master);
 
-	/*
-	 * In case of error during transmit, we bail out from processing
-	 * the message.
-	 */
-	if (msg->status)
-		return;
+	/* Using PIO so prime the TX FIFO and enable interrupts */
+	ep93xx_spi_read_write(master);
 
-	msg->actual_length += t->len;
+	val = readl(espi->mmio + SSPCR1);
+	val |= (SSPCR1_RORIE | SSPCR1_TIE | SSPCR1_RIE);
+	writel(val, espi->mmio + SSPCR1);
 
-	/*
-	 * After this transfer is finished, perform any possible
-	 * post-transfer actions requested by the protocol driver.
-	 */
-	if (t->delay_usecs) {
-		set_current_state(TASK_UNINTERRUPTIBLE);
-		schedule_timeout(usecs_to_jiffies(t->delay_usecs));
-	}
-	if (t->cs_change) {
-		if (!list_is_last(&t->transfer_list, &msg->transfers)) {
-			/*
-			 * In case protocol driver is asking us to drop the
-			 * chipselect briefly, we let the scheduler to handle
-			 * any "delay" here.
-			 */
-			ep93xx_spi_cs_control(msg->spi, false);
-			cond_resched();
-			ep93xx_spi_cs_control(msg->spi, true);
-		}
-	}
+	/* signal that we need to wait for completion */
+	return 1;
 }
 
-/*
- * ep93xx_spi_process_message() - process one SPI message
- * @espi: ep93xx SPI controller struct
- * @msg: message to process
- *
- * This function processes a single SPI message. We go through all transfers in
- * the message and pass them to ep93xx_spi_process_transfer(). Chipselect is
- * asserted during the whole message (unless per transfer cs_change is set).
- *
- * @msg->status contains %0 in case of success or negative error code in case of
- * failure.
- */
-static void ep93xx_spi_process_message(struct ep93xx_spi *espi,
-				       struct spi_message *msg)
+static int ep93xx_spi_prepare_message(struct spi_master *master,
+				      struct spi_message *msg)
 {
+	struct ep93xx_spi *espi = spi_master_get_devdata(master);
 	unsigned long timeout;
-	struct spi_transfer *t;
-	int err;
-
-	/*
-	 * Enable the SPI controller and its clock.
-	 */
-	err = ep93xx_spi_enable(espi);
-	if (err) {
-		dev_err(&espi->pdev->dev, "failed to enable SPI controller\n");
-		msg->status = err;
-		return;
-	}
 
 	/*
 	 * Just to be sure: flush any data from RX FIFO.
 	 */
 	timeout = jiffies + msecs_to_jiffies(SPI_TIMEOUT);
-	while (ep93xx_spi_read_u16(espi, SSPSR) & SSPSR_RNE) {
+	while (readl(espi->mmio + SSPSR) & SSPSR_RNE) {
 		if (time_after(jiffies, timeout)) {
-			dev_warn(&espi->pdev->dev,
+			dev_warn(&master->dev,
 				 "timeout while flushing RX FIFO\n");
-			msg->status = -ETIMEDOUT;
-			return;
+			return -ETIMEDOUT;
 		}
-		ep93xx_spi_read_u16(espi, SSPDR);
+		readl(espi->mmio + SSPDR);
 	}
 
 	/*
@@ -631,81 +527,38 @@ static void ep93xx_spi_process_message(struct ep93xx_spi *espi,
 	 */
 	espi->fifo_level = 0;
 
-	/*
-	 * Assert the chipselect.
-	 */
-	ep93xx_spi_cs_control(msg->spi, true);
-
-	list_for_each_entry(t, &msg->transfers, transfer_list) {
-		ep93xx_spi_process_transfer(espi, msg, t);
-		if (msg->status)
-			break;
-	}
-
-	/*
-	 * Now the whole message is transferred (or failed for some reason). We
-	 * deselect the device and disable the SPI controller.
-	 */
-	ep93xx_spi_cs_control(msg->spi, false);
-	ep93xx_spi_disable(espi);
+	return 0;
 }
 
-static int ep93xx_spi_transfer_one_message(struct spi_master *master,
-					   struct spi_message *msg)
+static int ep93xx_spi_prepare_hardware(struct spi_master *master)
 {
 	struct ep93xx_spi *espi = spi_master_get_devdata(master);
+	u32 val;
+	int ret;
 
-	msg->state = NULL;
-	msg->status = 0;
-	msg->actual_length = 0;
-
-	espi->current_msg = msg;
-	ep93xx_spi_process_message(espi, msg);
-	espi->current_msg = NULL;
+	ret = clk_enable(espi->clk);
+	if (ret)
+		return ret;
 
-	spi_finalize_current_message(master);
+	val = readl(espi->mmio + SSPCR1);
+	val |= SSPCR1_SSE;
+	writel(val, espi->mmio + SSPCR1);
 
 	return 0;
 }
 
-static irqreturn_t ep93xx_spi_interrupt(int irq, void *dev_id)
+static int ep93xx_spi_unprepare_hardware(struct spi_master *master)
 {
-	struct ep93xx_spi *espi = dev_id;
-	u8 irq_status = ep93xx_spi_read_u8(espi, SSPIIR);
+	struct ep93xx_spi *espi = spi_master_get_devdata(master);
+	u32 val;
 
-	/*
-	 * If we got ROR (receive overrun) interrupt we know that something is
-	 * wrong. Just abort the message.
-	 */
-	if (unlikely(irq_status & SSPIIR_RORIS)) {
-		/* clear the overrun interrupt */
-		ep93xx_spi_write_u8(espi, SSPICR, 0);
-		dev_warn(&espi->pdev->dev,
-			 "receive overrun, aborting the message\n");
-		espi->current_msg->status = -EIO;
-	} else {
-		/*
-		 * Interrupt is either RX (RIS) or TX (TIS). For both cases we
-		 * simply execute next data transfer.
-		 */
-		if (ep93xx_spi_read_write(espi)) {
-			/*
-			 * In normal case, there still is some processing left
-			 * for current transfer. Let's wait for the next
-			 * interrupt then.
-			 */
-			return IRQ_HANDLED;
-		}
-	}
+	val = readl(espi->mmio + SSPCR1);
+	val &= ~SSPCR1_SSE;
+	writel(val, espi->mmio + SSPCR1);
 
-	/*
-	 * Current transfer is finished, either with error or with success. In
-	 * any case we disable interrupts and notify the worker to handle
-	 * any post-processing of the message.
-	 */
-	ep93xx_spi_disable_interrupts(espi);
-	complete(&espi->wait);
-	return IRQ_HANDLED;
+	clk_disable(espi->clk);
+
+	return 0;
 }
 
 static bool ep93xx_spi_dma_filter(struct dma_chan *chan, void *filter_param)
@@ -809,7 +662,10 @@ static int ep93xx_spi_probe(struct platform_device *pdev)
 	if (!master)
 		return -ENOMEM;
 
-	master->transfer_one_message = ep93xx_spi_transfer_one_message;
+	master->prepare_transfer_hardware = ep93xx_spi_prepare_hardware;
+	master->unprepare_transfer_hardware = ep93xx_spi_unprepare_hardware;
+	master->prepare_message = ep93xx_spi_prepare_message;
+	master->transfer_one = ep93xx_spi_transfer_one;
 	master->bus_num = pdev->id;
 	master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH;
 	master->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 16);
@@ -850,26 +706,23 @@ static int ep93xx_spi_probe(struct platform_device *pdev)
 		goto fail_release_master;
 	}
 
-	init_completion(&espi->wait);
-
 	/*
 	 * Calculate maximum and minimum supported clock rates
 	 * for the controller.
 	 */
 	master->max_speed_hz = clk_get_rate(espi->clk) / 2;
 	master->min_speed_hz = clk_get_rate(espi->clk) / (254 * 256);
-	espi->pdev = pdev;
 
 	espi->sspdr_phys = res->start + SSPDR;
 
-	espi->regs_base = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(espi->regs_base)) {
-		error = PTR_ERR(espi->regs_base);
+	espi->mmio = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(espi->mmio)) {
+		error = PTR_ERR(espi->mmio);
 		goto fail_release_master;
 	}
 
 	error = devm_request_irq(&pdev->dev, irq, ep93xx_spi_interrupt,
-				0, "ep93xx-spi", espi);
+				0, "ep93xx-spi", master);
 	if (error) {
 		dev_err(&pdev->dev, "failed to request irq\n");
 		goto fail_release_master;
@@ -879,7 +732,7 @@ static int ep93xx_spi_probe(struct platform_device *pdev)
 		dev_warn(&pdev->dev, "DMA setup failed. Falling back to PIO\n");
 
 	/* make sure that the hardware is disabled */
-	ep93xx_spi_write_u8(espi, SSPCR1, 0);
+	writel(0, espi->mmio + SSPCR1);
 
 	error = devm_spi_register_master(&pdev->dev, master);
 	if (error) {
diff --git a/drivers/spi/spi-falcon.c b/drivers/spi/spi-falcon.c
index 286b2c81fc6b..f8638e82e5db 100644
--- a/drivers/spi/spi-falcon.c
+++ b/drivers/spi/spi-falcon.c
@@ -395,11 +395,6 @@ static int falcon_sflash_probe(struct platform_device *pdev)
 	struct spi_master *master;
 	int ret;
 
-	if (ltq_boot_select() != BS_SPI) {
-		dev_err(&pdev->dev, "invalid bootstrap options\n");
-		return -ENODEV;
-	}
-
 	master = spi_alloc_master(&pdev->dev, sizeof(*priv));
 	if (!master)
 		return -ENOMEM;
diff --git a/drivers/spi/spi-imx.c b/drivers/spi/spi-imx.c
index f9698b7aeb3b..babb15f07995 100644
--- a/drivers/spi/spi-imx.c
+++ b/drivers/spi/spi-imx.c
@@ -56,6 +56,7 @@
 
 /* The maximum  bytes that a sdma BD can transfer.*/
 #define MAX_SDMA_BD_BYTES  (1 << 15)
+#define MX51_ECSPI_CTRL_MAX_BURST	512
 
 enum spi_imx_devtype {
 	IMX1_CSPI,
@@ -63,7 +64,8 @@ enum spi_imx_devtype {
 	IMX27_CSPI,
 	IMX31_CSPI,
 	IMX35_CSPI,	/* CSPI on all i.mx except above */
-	IMX51_ECSPI,	/* ECSPI on i.mx51 and later */
+	IMX51_ECSPI,	/* ECSPI on i.mx51 */
+	IMX53_ECSPI,	/* ECSPI on i.mx53 and later */
 };
 
 struct spi_imx_data;
@@ -74,6 +76,9 @@ struct spi_imx_devtype_data {
 	void (*trigger)(struct spi_imx_data *);
 	int (*rx_available)(struct spi_imx_data *);
 	void (*reset)(struct spi_imx_data *);
+	bool has_dmamode;
+	unsigned int fifo_size;
+	bool dynamic_burst;
 	enum spi_imx_devtype devtype;
 };
 
@@ -94,12 +99,14 @@ struct spi_imx_data {
 	unsigned int bits_per_word;
 	unsigned int spi_drctl;
 
-	unsigned int count;
+	unsigned int count, remainder;
 	void (*tx)(struct spi_imx_data *);
 	void (*rx)(struct spi_imx_data *);
 	void *rx_buf;
 	const void *tx_buf;
 	unsigned int txfifo; /* number of words pushed in tx FIFO */
+	unsigned int dynamic_burst, read_u32;
+	unsigned int word_mask;
 
 	/* DMA */
 	bool usedma;
@@ -125,9 +132,9 @@ static inline int is_imx51_ecspi(struct spi_imx_data *d)
 	return d->devtype_data->devtype == IMX51_ECSPI;
 }
 
-static inline unsigned spi_imx_get_fifosize(struct spi_imx_data *d)
+static inline int is_imx53_ecspi(struct spi_imx_data *d)
 {
-	return is_imx51_ecspi(d) ? 64 : 8;
+	return d->devtype_data->devtype == IMX53_ECSPI;
 }
 
 #define MXC_SPI_BUF_RX(type)						\
@@ -219,7 +226,7 @@ static bool spi_imx_can_dma(struct spi_master *master, struct spi_device *spi,
 	if (bytes_per_word != 1 && bytes_per_word != 2 && bytes_per_word != 4)
 		return false;
 
-	for (i = spi_imx_get_fifosize(spi_imx) / 2; i > 0; i--) {
+	for (i = spi_imx->devtype_data->fifo_size / 2; i > 0; i--) {
 		if (!(transfer->len % (i * bytes_per_word)))
 			break;
 	}
@@ -228,6 +235,7 @@ static bool spi_imx_can_dma(struct spi_master *master, struct spi_device *spi,
 		return false;
 
 	spi_imx->wml = i;
+	spi_imx->dynamic_burst = 0;
 
 	return true;
 }
@@ -242,6 +250,7 @@ static bool spi_imx_can_dma(struct spi_master *master, struct spi_device *spi,
 #define MX51_ECSPI_CTRL_PREDIV_OFFSET	12
 #define MX51_ECSPI_CTRL_CS(cs)		((cs) << 18)
 #define MX51_ECSPI_CTRL_BL_OFFSET	20
+#define MX51_ECSPI_CTRL_BL_MASK		(0xfff << 20)
 
 #define MX51_ECSPI_CONFIG	0x0c
 #define MX51_ECSPI_CONFIG_SCLKPHA(cs)	(1 << ((cs) +  0))
@@ -269,6 +278,106 @@ static bool spi_imx_can_dma(struct spi_master *master, struct spi_device *spi,
 #define MX51_ECSPI_TESTREG	0x20
 #define MX51_ECSPI_TESTREG_LBC	BIT(31)
 
+static void spi_imx_buf_rx_swap_u32(struct spi_imx_data *spi_imx)
+{
+	unsigned int val = readl(spi_imx->base + MXC_CSPIRXDATA);
+#ifdef __LITTLE_ENDIAN
+	unsigned int bytes_per_word;
+#endif
+
+	if (spi_imx->rx_buf) {
+#ifdef __LITTLE_ENDIAN
+		bytes_per_word = spi_imx_bytes_per_word(spi_imx->bits_per_word);
+		if (bytes_per_word == 1)
+			val = cpu_to_be32(val);
+		else if (bytes_per_word == 2)
+			val = (val << 16) | (val >> 16);
+#endif
+		val &= spi_imx->word_mask;
+		*(u32 *)spi_imx->rx_buf = val;
+		spi_imx->rx_buf += sizeof(u32);
+	}
+}
+
+static void spi_imx_buf_rx_swap(struct spi_imx_data *spi_imx)
+{
+	unsigned int bytes_per_word;
+
+	bytes_per_word = spi_imx_bytes_per_word(spi_imx->bits_per_word);
+	if (spi_imx->read_u32) {
+		spi_imx_buf_rx_swap_u32(spi_imx);
+		return;
+	}
+
+	if (bytes_per_word == 1)
+		spi_imx_buf_rx_u8(spi_imx);
+	else if (bytes_per_word == 2)
+		spi_imx_buf_rx_u16(spi_imx);
+}
+
+static void spi_imx_buf_tx_swap_u32(struct spi_imx_data *spi_imx)
+{
+	u32 val = 0;
+#ifdef __LITTLE_ENDIAN
+	unsigned int bytes_per_word;
+#endif
+
+	if (spi_imx->tx_buf) {
+		val = *(u32 *)spi_imx->tx_buf;
+		val &= spi_imx->word_mask;
+		spi_imx->tx_buf += sizeof(u32);
+	}
+
+	spi_imx->count -= sizeof(u32);
+#ifdef __LITTLE_ENDIAN
+	bytes_per_word = spi_imx_bytes_per_word(spi_imx->bits_per_word);
+
+	if (bytes_per_word == 1)
+		val = cpu_to_be32(val);
+	else if (bytes_per_word == 2)
+		val = (val << 16) | (val >> 16);
+#endif
+	writel(val, spi_imx->base + MXC_CSPITXDATA);
+}
+
+static void spi_imx_buf_tx_swap(struct spi_imx_data *spi_imx)
+{
+	u32 ctrl, val;
+	unsigned int bytes_per_word;
+
+	if (spi_imx->count == spi_imx->remainder) {
+		ctrl = readl(spi_imx->base + MX51_ECSPI_CTRL);
+		ctrl &= ~MX51_ECSPI_CTRL_BL_MASK;
+		if (spi_imx->count > MX51_ECSPI_CTRL_MAX_BURST) {
+			spi_imx->remainder = spi_imx->count %
+					     MX51_ECSPI_CTRL_MAX_BURST;
+			val = MX51_ECSPI_CTRL_MAX_BURST * 8 - 1;
+		} else if (spi_imx->count >= sizeof(u32)) {
+			spi_imx->remainder = spi_imx->count % sizeof(u32);
+			val = (spi_imx->count - spi_imx->remainder) * 8 - 1;
+		} else {
+			spi_imx->remainder = 0;
+			val = spi_imx->bits_per_word - 1;
+			spi_imx->read_u32 = 0;
+		}
+
+		ctrl |= (val << MX51_ECSPI_CTRL_BL_OFFSET);
+		writel(ctrl, spi_imx->base + MX51_ECSPI_CTRL);
+	}
+
+	if (spi_imx->count >= sizeof(u32)) {
+		spi_imx_buf_tx_swap_u32(spi_imx);
+		return;
+	}
+
+	bytes_per_word = spi_imx_bytes_per_word(spi_imx->bits_per_word);
+
+	if (bytes_per_word == 1)
+		spi_imx_buf_tx_u8(spi_imx);
+	else if (bytes_per_word == 2)
+		spi_imx_buf_tx_u16(spi_imx);
+}
+
 /* MX51 eCSPI */
 static unsigned int mx51_ecspi_clkdiv(struct spi_imx_data *spi_imx,
 				      unsigned int fspi, unsigned int *fres)
@@ -513,8 +622,8 @@ static int mx31_config(struct spi_device *spi)
 		reg |= MX31_CSPICTRL_POL;
 	if (spi->mode & SPI_CS_HIGH)
 		reg |= MX31_CSPICTRL_SSPOL;
-	if (spi->cs_gpio < 0)
-		reg |= (spi->cs_gpio + 32) <<
+	if (!gpio_is_valid(spi->cs_gpio))
+		reg |= (spi->chip_select) <<
 			(is_imx35_cspi(spi_imx) ? MX35_CSPICTRL_CS_SHIFT :
 						  MX31_CSPICTRL_CS_SHIFT);
 
@@ -605,8 +714,8 @@ static int mx21_config(struct spi_device *spi)
 		reg |= MX21_CSPICTRL_POL;
 	if (spi->mode & SPI_CS_HIGH)
 		reg |= MX21_CSPICTRL_SSPOL;
-	if (spi->cs_gpio < 0)
-		reg |= (spi->cs_gpio + 32) << MX21_CSPICTRL_CS_SHIFT;
+	if (!gpio_is_valid(spi->cs_gpio))
+		reg |= spi->chip_select << MX21_CSPICTRL_CS_SHIFT;
 
 	writel(reg, spi_imx->base + MXC_CSPICTRL);
 
@@ -693,6 +802,9 @@ static struct spi_imx_devtype_data imx1_cspi_devtype_data = {
 	.trigger = mx1_trigger,
 	.rx_available = mx1_rx_available,
 	.reset = mx1_reset,
+	.fifo_size = 8,
+	.has_dmamode = false,
+	.dynamic_burst = false,
 	.devtype = IMX1_CSPI,
 };
 
@@ -702,6 +814,9 @@ static struct spi_imx_devtype_data imx21_cspi_devtype_data = {
 	.trigger = mx21_trigger,
 	.rx_available = mx21_rx_available,
 	.reset = mx21_reset,
+	.fifo_size = 8,
+	.has_dmamode = false,
+	.dynamic_burst = false,
 	.devtype = IMX21_CSPI,
 };
 
@@ -712,6 +827,9 @@ static struct spi_imx_devtype_data imx27_cspi_devtype_data = {
 	.trigger = mx21_trigger,
 	.rx_available = mx21_rx_available,
 	.reset = mx21_reset,
+	.fifo_size = 8,
+	.has_dmamode = false,
+	.dynamic_burst = false,
 	.devtype = IMX27_CSPI,
 };
 
@@ -721,6 +839,9 @@ static struct spi_imx_devtype_data imx31_cspi_devtype_data = {
 	.trigger = mx31_trigger,
 	.rx_available = mx31_rx_available,
 	.reset = mx31_reset,
+	.fifo_size = 8,
+	.has_dmamode = false,
+	.dynamic_burst = false,
 	.devtype = IMX31_CSPI,
 };
 
@@ -731,6 +852,9 @@ static struct spi_imx_devtype_data imx35_cspi_devtype_data = {
 	.trigger = mx31_trigger,
 	.rx_available = mx31_rx_available,
 	.reset = mx31_reset,
+	.fifo_size = 8,
+	.has_dmamode = true,
+	.dynamic_burst = false,
 	.devtype = IMX35_CSPI,
 };
 
@@ -740,9 +864,23 @@ static struct spi_imx_devtype_data imx51_ecspi_devtype_data = {
 	.trigger = mx51_ecspi_trigger,
 	.rx_available = mx51_ecspi_rx_available,
 	.reset = mx51_ecspi_reset,
+	.fifo_size = 64,
+	.has_dmamode = true,
+	.dynamic_burst = true,
 	.devtype = IMX51_ECSPI,
 };
 
+static struct spi_imx_devtype_data imx53_ecspi_devtype_data = {
+	.intctrl = mx51_ecspi_intctrl,
+	.config = mx51_ecspi_config,
+	.trigger = mx51_ecspi_trigger,
+	.rx_available = mx51_ecspi_rx_available,
+	.reset = mx51_ecspi_reset,
+	.fifo_size = 64,
+	.has_dmamode = true,
+	.devtype = IMX53_ECSPI,
+};
+
 static const struct platform_device_id spi_imx_devtype[] = {
 	{
 		.name = "imx1-cspi",
@@ -763,6 +901,9 @@ static const struct platform_device_id spi_imx_devtype[] = {
 		.name = "imx51-ecspi",
 		.driver_data = (kernel_ulong_t) &imx51_ecspi_devtype_data,
 	}, {
+		.name = "imx53-ecspi",
+		.driver_data = (kernel_ulong_t) &imx53_ecspi_devtype_data,
+	}, {
 		/* sentinel */
 	}
 };
@@ -774,6 +915,7 @@ static const struct of_device_id spi_imx_dt_ids[] = {
 	{ .compatible = "fsl,imx31-cspi", .data = &imx31_cspi_devtype_data, },
 	{ .compatible = "fsl,imx35-cspi", .data = &imx35_cspi_devtype_data, },
 	{ .compatible = "fsl,imx51-ecspi", .data = &imx51_ecspi_devtype_data, },
+	{ .compatible = "fsl,imx53-ecspi", .data = &imx53_ecspi_devtype_data, },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, spi_imx_dt_ids);
@@ -783,6 +925,9 @@ static void spi_imx_chipselect(struct spi_device *spi, int is_active)
 	int active = is_active != BITBANG_CS_INACTIVE;
 	int dev_is_lowactive = !(spi->mode & SPI_CS_HIGH);
 
+	if (spi->mode & SPI_NO_CS)
+		return;
+
 	if (!gpio_is_valid(spi->cs_gpio))
 		return;
 
@@ -791,9 +936,11 @@ static void spi_imx_chipselect(struct spi_device *spi, int is_active)
 
 static void spi_imx_push(struct spi_imx_data *spi_imx)
 {
-	while (spi_imx->txfifo < spi_imx_get_fifosize(spi_imx)) {
+	while (spi_imx->txfifo < spi_imx->devtype_data->fifo_size) {
 		if (!spi_imx->count)
 			break;
+		if (spi_imx->txfifo && (spi_imx->count == spi_imx->remainder))
+			break;
 		spi_imx->tx(spi_imx);
 		spi_imx->txfifo++;
 	}
@@ -887,15 +1034,37 @@ static int spi_imx_setupxfer(struct spi_device *spi,
 	spi_imx->speed_hz  = t->speed_hz;
 
 	/* Initialize the functions for transfer */
-	if (spi_imx->bits_per_word <= 8) {
-		spi_imx->rx = spi_imx_buf_rx_u8;
-		spi_imx->tx = spi_imx_buf_tx_u8;
-	} else if (spi_imx->bits_per_word <= 16) {
-		spi_imx->rx = spi_imx_buf_rx_u16;
-		spi_imx->tx = spi_imx_buf_tx_u16;
+	if (spi_imx->devtype_data->dynamic_burst) {
+		u32 mask;
+
+		spi_imx->dynamic_burst = 0;
+		spi_imx->remainder = 0;
+		spi_imx->read_u32  = 1;
+
+		mask = (1 << spi_imx->bits_per_word) - 1;
+		spi_imx->rx = spi_imx_buf_rx_swap;
+		spi_imx->tx = spi_imx_buf_tx_swap;
+		spi_imx->dynamic_burst = 1;
+		spi_imx->remainder = t->len;
+
+		if (spi_imx->bits_per_word <= 8)
+			spi_imx->word_mask = mask << 24 | mask << 16
+					     | mask << 8 | mask;
+		else if (spi_imx->bits_per_word <= 16)
+			spi_imx->word_mask = mask << 16 | mask;
+		else
+			spi_imx->word_mask = mask;
 	} else {
-		spi_imx->rx = spi_imx_buf_rx_u32;
-		spi_imx->tx = spi_imx_buf_tx_u32;
+		if (spi_imx->bits_per_word <= 8) {
+			spi_imx->rx = spi_imx_buf_rx_u8;
+			spi_imx->tx = spi_imx_buf_tx_u8;
+		} else if (spi_imx->bits_per_word <= 16) {
+			spi_imx->rx = spi_imx_buf_rx_u16;
+			spi_imx->tx = spi_imx_buf_tx_u16;
+		} else {
+			spi_imx->rx = spi_imx_buf_rx_u32;
+			spi_imx->tx = spi_imx_buf_tx_u32;
+		}
 	}
 
 	if (spi_imx_can_dma(spi_imx->bitbang.master, spi, t))
@@ -938,7 +1107,7 @@ static int spi_imx_sdma_init(struct device *dev, struct spi_imx_data *spi_imx,
 	if (of_machine_is_compatible("fsl,imx6dl"))
 		return 0;
 
-	spi_imx->wml = spi_imx_get_fifosize(spi_imx) / 2;
+	spi_imx->wml = spi_imx->devtype_data->fifo_size / 2;
 
 	/* Prepare for TX DMA: */
 	master->dma_tx = dma_request_slave_channel_reason(dev, "tx");
@@ -1109,6 +1278,9 @@ static int spi_imx_setup(struct spi_device *spi)
 	dev_dbg(&spi->dev, "%s: mode %d, %u bpw, %d hz\n", __func__,
 		 spi->mode, spi->bits_per_word, spi->max_speed_hz);
 
+	if (spi->mode & SPI_NO_CS)
+		return 0;
+
 	if (gpio_is_valid(spi->cs_gpio))
 		gpio_direction_output(spi->cs_gpio,
 				      spi->mode & SPI_CS_HIGH ? 0 : 1);
@@ -1208,8 +1380,10 @@ static int spi_imx_probe(struct platform_device *pdev)
 	spi_imx->bitbang.master->cleanup = spi_imx_cleanup;
 	spi_imx->bitbang.master->prepare_message = spi_imx_prepare_message;
 	spi_imx->bitbang.master->unprepare_message = spi_imx_unprepare_message;
-	spi_imx->bitbang.master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH;
-	if (is_imx35_cspi(spi_imx) || is_imx51_ecspi(spi_imx))
+	spi_imx->bitbang.master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH \
+					     | SPI_NO_CS;
+	if (is_imx35_cspi(spi_imx) || is_imx51_ecspi(spi_imx) ||
+	    is_imx53_ecspi(spi_imx))
 		spi_imx->bitbang.master->mode_bits |= SPI_LOOP | SPI_READY;
 
 	spi_imx->spi_drctl = spi_drctl;
@@ -1262,7 +1436,7 @@ static int spi_imx_probe(struct platform_device *pdev)
 	 * Only validated on i.mx35 and i.mx6 now, can remove the constraint
 	 * if validated on other chips.
 	 */
-	if (is_imx35_cspi(spi_imx) || is_imx51_ecspi(spi_imx)) {
+	if (spi_imx->devtype_data->has_dmamode) {
 		ret = spi_imx_sdma_init(&pdev->dev, spi_imx, master);
 		if (ret == -EPROBE_DEFER)
 			goto out_clk_put;
diff --git a/drivers/spi/spi-loopback-test.c b/drivers/spi/spi-loopback-test.c
index 3459965004f8..bed7403bb6b3 100644
--- a/drivers/spi/spi-loopback-test.c
+++ b/drivers/spi/spi-loopback-test.c
@@ -32,39 +32,50 @@
 #include "spi-test.h"
 
 /* flag to only simulate transfers */
-int simulate_only;
+static int simulate_only;
 module_param(simulate_only, int, 0);
 MODULE_PARM_DESC(simulate_only, "if not 0 do not execute the spi message");
 
 /* dump spi messages */
-int dump_messages;
+static int dump_messages;
 module_param(dump_messages, int, 0);
 MODULE_PARM_DESC(dump_messages,
 		 "=1 dump the basic spi_message_structure, " \
 		 "=2 dump the spi_message_structure including data, " \
 		 "=3 dump the spi_message structure before and after execution");
 /* the device is jumpered for loopback - enabling some rx_buf tests */
-int loopback;
+static int loopback;
 module_param(loopback, int, 0);
 MODULE_PARM_DESC(loopback,
 		 "if set enable loopback mode, where the rx_buf "	\
 		 "is checked to match tx_buf after the spi_message "	\
 		 "is executed");
 
+static int loop_req;
+module_param(loop_req, int, 0);
+MODULE_PARM_DESC(loop_req,
+		 "if set controller will be asked to enable test loop mode. " \
+		 "If controller supported it, MISO and MOSI will be connected");
+
+static int no_cs;
+module_param(no_cs, int, 0);
+MODULE_PARM_DESC(no_cs,
+		 "if set Chip Select (CS) will not be used");
+
 /* run only a specific test */
-int run_only_test = -1;
+static int run_only_test = -1;
 module_param(run_only_test, int, 0);
 MODULE_PARM_DESC(run_only_test,
 		 "only run the test with this number (0-based !)");
 
 /* use vmalloc'ed buffers */
-int use_vmalloc;
+static int use_vmalloc;
 module_param(use_vmalloc, int, 0644);
 MODULE_PARM_DESC(use_vmalloc,
 		 "use vmalloc'ed buffers instead of kmalloc'ed");
 
 /* check rx ranges */
-int check_ranges = 1;
+static int check_ranges = 1;
 module_param(check_ranges, int, 0644);
 MODULE_PARM_DESC(check_ranges,
 		 "checks rx_buffer pattern are valid");
@@ -313,6 +324,17 @@ static int spi_loopback_test_probe(struct spi_device *spi)
 {
 	int ret;
 
+	if (loop_req || no_cs) {
+		spi->mode |= loop_req ? SPI_LOOP : 0;
+		spi->mode |= no_cs ? SPI_NO_CS : 0;
+		ret = spi_setup(spi);
+		if (ret) {
+			dev_err(&spi->dev, "SPI setup with SPI_LOOP or SPI_NO_CS failed (%d)\n",
+				ret);
+			return ret;
+		}
+	}
+
 	dev_info(&spi->dev, "Executing spi-loopback-tests\n");
 
 	ret = spi_test_run_tests(spi, spi_tests);
diff --git a/drivers/spi/spi-omap2-mcspi.c b/drivers/spi/spi-omap2-mcspi.c
index e048268d8ba2..9bf64e6eca9b 100644
--- a/drivers/spi/spi-omap2-mcspi.c
+++ b/drivers/spi/spi-omap2-mcspi.c
@@ -1338,7 +1338,6 @@ static int omap2_mcspi_probe(struct platform_device *pdev)
 	struct resource		*r;
 	int			status = 0, i;
 	u32			regs_offset = 0;
-	static int		bus_num = 1;
 	struct device_node	*node = pdev->dev.of_node;
 	const struct of_device_id *match;
 
@@ -1374,14 +1373,11 @@ static int omap2_mcspi_probe(struct platform_device *pdev)
 
 		of_property_read_u32(node, "ti,spi-num-cs", &num_cs);
 		master->num_chipselect = num_cs;
-		master->bus_num = bus_num++;
 		if (of_get_property(node, "ti,pindir-d0-out-d1-in", NULL))
 			mcspi->pin_dir = MCSPI_PINDIR_D0_OUT_D1_IN;
 	} else {
 		pdata = dev_get_platdata(&pdev->dev);
 		master->num_chipselect = pdata->num_cs;
-		if (pdev->id != -1)
-			master->bus_num = pdev->id;
 		mcspi->pin_dir = pdata->pin_dir;
 	}
 	regs_offset = pdata->regs_offset;
diff --git a/drivers/spi/spi-orion.c b/drivers/spi/spi-orion.c
index 28fc9f161b9d..4b6dd73b80da 100644
--- a/drivers/spi/spi-orion.c
+++ b/drivers/spi/spi-orion.c
@@ -669,8 +669,8 @@ static int orion_spi_probe(struct platform_device *pdev)
 		status = of_property_read_u32(np, "reg", &cs);
 		if (status) {
 			dev_err(&pdev->dev,
-				"%s has no valid 'reg' property (%d)\n",
-				np->full_name, status);
+				"%pOF has no valid 'reg' property (%d)\n",
+				np, status);
 			status = 0;
 			continue;
 		}
diff --git a/drivers/spi/spi-pic32.c b/drivers/spi/spi-pic32.c
index fefb688a3432..f8a45af1fa9f 100644
--- a/drivers/spi/spi-pic32.c
+++ b/drivers/spi/spi-pic32.c
@@ -52,14 +52,14 @@ struct pic32_spi_regs {
 
 /* Bit fields of SPI Control Register */
 #define CTRL_RX_INT_SHIFT	0  /* Rx interrupt generation */
-#define  RX_FIFO_EMTPY		0
+#define  RX_FIFO_EMPTY		0
 #define  RX_FIFO_NOT_EMPTY	1 /* not empty */
 #define  RX_FIFO_HALF_FULL	2 /* full by half or more */
 #define  RX_FIFO_FULL		3 /* completely full */
 
 #define CTRL_TX_INT_SHIFT	2  /* TX interrupt generation */
 #define  TX_FIFO_ALL_EMPTY	0 /* completely empty */
-#define  TX_FIFO_EMTPY		1 /* empty */
+#define  TX_FIFO_EMPTY		1 /* empty */
 #define  TX_FIFO_HALF_EMPTY	2 /* empty by half or more */
 #define  TX_FIFO_NOT_FULL	3 /* atleast one empty */
 
diff --git a/drivers/spi/spi-pl022.c b/drivers/spi/spi-pl022.c
index 2f76e022cc59..4797c57f4263 100644
--- a/drivers/spi/spi-pl022.c
+++ b/drivers/spi/spi-pl022.c
@@ -2429,7 +2429,7 @@ static struct vendor_data vendor_lsi = {
 	.internal_cs_ctrl = true,
 };
 
-static struct amba_id pl022_ids[] = {
+static const struct amba_id pl022_ids[] = {
 	{
 		/*
 		 * ARM PL022 variant, this has a 16bit wide
diff --git a/drivers/spi/spi-pxa2xx.c b/drivers/spi/spi-pxa2xx.c
index 38d053682892..4cb515a3104c 100644
--- a/drivers/spi/spi-pxa2xx.c
+++ b/drivers/spi/spi-pxa2xx.c
@@ -402,8 +402,8 @@ static void cs_assert(struct driver_data *drv_data)
 		return;
 	}
 
-	if (gpio_is_valid(chip->gpio_cs)) {
-		gpio_set_value(chip->gpio_cs, chip->gpio_cs_inverted);
+	if (chip->gpiod_cs) {
+		gpiod_set_value(chip->gpiod_cs, chip->gpio_cs_inverted);
 		return;
 	}
 
@@ -424,8 +424,8 @@ static void cs_deassert(struct driver_data *drv_data)
 		return;
 	}
 
-	if (gpio_is_valid(chip->gpio_cs)) {
-		gpio_set_value(chip->gpio_cs, !chip->gpio_cs_inverted);
+	if (chip->gpiod_cs) {
+		gpiod_set_value(chip->gpiod_cs, !chip->gpio_cs_inverted);
 		return;
 	}
 
@@ -1213,17 +1213,16 @@ static int setup_cs(struct spi_device *spi, struct chip_data *chip,
 		    struct pxa2xx_spi_chip *chip_info)
 {
 	struct driver_data *drv_data = spi_master_get_devdata(spi->master);
+	struct gpio_desc *gpiod;
 	int err = 0;
 
 	if (chip == NULL)
 		return 0;
 
 	if (drv_data->cs_gpiods) {
-		struct gpio_desc *gpiod;
-
 		gpiod = drv_data->cs_gpiods[spi->chip_select];
 		if (gpiod) {
-			chip->gpio_cs = desc_to_gpio(gpiod);
+			chip->gpiod_cs = gpiod;
 			chip->gpio_cs_inverted = spi->mode & SPI_CS_HIGH;
 			gpiod_set_value(gpiod, chip->gpio_cs_inverted);
 		}
@@ -1237,8 +1236,10 @@ static int setup_cs(struct spi_device *spi, struct chip_data *chip,
 	/* NOTE: setup() can be called multiple times, possibly with
 	 * different chip_info, release previously requested GPIO
 	 */
-	if (gpio_is_valid(chip->gpio_cs))
-		gpio_free(chip->gpio_cs);
+	if (chip->gpiod_cs) {
+		gpio_free(desc_to_gpio(chip->gpiod_cs));
+		chip->gpiod_cs = NULL;
+	}
 
 	/* If (*cs_control) is provided, ignore GPIO chip select */
 	if (chip_info->cs_control) {
@@ -1254,11 +1255,11 @@ static int setup_cs(struct spi_device *spi, struct chip_data *chip,
 			return err;
 		}
 
-		chip->gpio_cs = chip_info->gpio_cs;
+		gpiod = gpio_to_desc(chip_info->gpio_cs);
+		chip->gpiod_cs = gpiod;
 		chip->gpio_cs_inverted = spi->mode & SPI_CS_HIGH;
 
-		err = gpio_direction_output(chip->gpio_cs,
-					!chip->gpio_cs_inverted);
+		err = gpiod_direction_output(gpiod, !chip->gpio_cs_inverted);
 	}
 
 	return err;
@@ -1317,8 +1318,7 @@ static int setup(struct spi_device *spi)
 			}
 
 			chip->frm = spi->chip_select;
-		} else
-			chip->gpio_cs = -1;
+		}
 		chip->enable_dma = drv_data->master_info->enable_dma;
 		chip->timeout = TIMOUT_DFLT;
 	}
@@ -1416,8 +1416,8 @@ static void cleanup(struct spi_device *spi)
 		return;
 
 	if (drv_data->ssp_type != CE4100_SSP && !drv_data->cs_gpiods &&
-	    gpio_is_valid(chip->gpio_cs))
-		gpio_free(chip->gpio_cs);
+	    chip->gpiod_cs)
+		gpio_free(desc_to_gpio(chip->gpiod_cs));
 
 	kfree(chip);
 }
@@ -1769,8 +1769,7 @@ static int pxa2xx_spi_probe(struct platform_device *pdev)
 		for (i = 0; i < master->num_chipselect; i++) {
 			struct gpio_desc *gpiod;
 
-			gpiod = devm_gpiod_get_index(dev, "cs", i,
-						     GPIOD_OUT_HIGH);
+			gpiod = devm_gpiod_get_index(dev, "cs", i, GPIOD_ASIS);
 			if (IS_ERR(gpiod)) {
 				/* Means use native chip select */
 				if (PTR_ERR(gpiod) == -ENOENT)
diff --git a/drivers/spi/spi-pxa2xx.h b/drivers/spi/spi-pxa2xx.h
index 2823a00a9405..94f7b0713281 100644
--- a/drivers/spi/spi-pxa2xx.h
+++ b/drivers/spi/spi-pxa2xx.h
@@ -83,7 +83,7 @@ struct chip_data {
 	u16 lpss_tx_threshold;
 	u8 enable_dma;
 	union {
-		int gpio_cs;
+		struct gpio_desc *gpiod_cs;
 		unsigned int frm;
 	};
 	int gpio_cs_inverted;
diff --git a/drivers/spi/spi-qup.c b/drivers/spi/spi-qup.c
index 1bfa889b8427..974a8ce58b68 100644
--- a/drivers/spi/spi-qup.c
+++ b/drivers/spi/spi-qup.c
@@ -19,6 +19,7 @@
 #include <linux/list.h>
 #include <linux/module.h>
 #include <linux/of.h>
+#include <linux/of_device.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
 #include <linux/spi/spi.h>
@@ -82,6 +83,8 @@
 #define QUP_IO_M_MODE_BAM		3
 
 /* QUP_OPERATIONAL fields */
+#define QUP_OP_IN_BLOCK_READ_REQ	BIT(13)
+#define QUP_OP_OUT_BLOCK_WRITE_REQ	BIT(12)
 #define QUP_OP_MAX_INPUT_DONE_FLAG	BIT(11)
 #define QUP_OP_MAX_OUTPUT_DONE_FLAG	BIT(10)
 #define QUP_OP_IN_SERVICE_FLAG		BIT(9)
@@ -118,7 +121,7 @@
 
 #define SPI_NUM_CHIPSELECTS		4
 
-#define SPI_MAX_DMA_XFER		(SZ_64K - 64)
+#define SPI_MAX_XFER			(SZ_64K - 64)
 
 /* high speed mode is when bus rate is greater then 26MHz */
 #define SPI_HS_MIN_RATE			26000000
@@ -147,13 +150,37 @@ struct spi_qup {
 	int			n_words;
 	int			tx_bytes;
 	int			rx_bytes;
+	const u8		*tx_buf;
+	u8			*rx_buf;
 	int			qup_v1;
 
-	int			use_dma;
+	int			mode;
 	struct dma_slave_config	rx_conf;
 	struct dma_slave_config	tx_conf;
 };
 
+static int spi_qup_io_config(struct spi_device *spi, struct spi_transfer *xfer);
+
+static inline bool spi_qup_is_flag_set(struct spi_qup *controller, u32 flag)
+{
+	u32 opflag = readl_relaxed(controller->base + QUP_OPERATIONAL);
+
+	return (opflag & flag) != 0;
+}
+
+static inline bool spi_qup_is_dma_xfer(int mode)
+{
+	if (mode == QUP_IO_M_MODE_DMOV || mode == QUP_IO_M_MODE_BAM)
+		return true;
+
+	return false;
+}
+
+/* get's the transaction size length */
+static inline unsigned int spi_qup_len(struct spi_qup *controller)
+{
+	return controller->n_words * controller->w_size;
+}
 
 static inline bool spi_qup_is_valid_state(struct spi_qup *controller)
 {
@@ -207,29 +234,26 @@ static int spi_qup_set_state(struct spi_qup *controller, u32 state)
 	return 0;
 }
 
-static void spi_qup_fifo_read(struct spi_qup *controller,
-			    struct spi_transfer *xfer)
+static void spi_qup_read_from_fifo(struct spi_qup *controller, u32 num_words)
 {
-	u8 *rx_buf = xfer->rx_buf;
-	u32 word, state;
-	int idx, shift, w_size;
-
-	w_size = controller->w_size;
+	u8 *rx_buf = controller->rx_buf;
+	int i, shift, num_bytes;
+	u32 word;
 
-	while (controller->rx_bytes < xfer->len) {
-
-		state = readl_relaxed(controller->base + QUP_OPERATIONAL);
-		if (0 == (state & QUP_OP_IN_FIFO_NOT_EMPTY))
-			break;
+	for (; num_words; num_words--) {
 
 		word = readl_relaxed(controller->base + QUP_INPUT_FIFO);
 
+		num_bytes = min_t(int, spi_qup_len(controller) -
+				       controller->rx_bytes,
+				       controller->w_size);
+
 		if (!rx_buf) {
-			controller->rx_bytes += w_size;
+			controller->rx_bytes += num_bytes;
 			continue;
 		}
 
-		for (idx = 0; idx < w_size; idx++, controller->rx_bytes++) {
+		for (i = 0; i < num_bytes; i++, controller->rx_bytes++) {
 			/*
 			 * The data format depends on bytes per SPI word:
 			 *  4 bytes: 0x12345678
@@ -237,38 +261,81 @@ static void spi_qup_fifo_read(struct spi_qup *controller,
 			 *  1 byte : 0x00000012
 			 */
 			shift = BITS_PER_BYTE;
-			shift *= (w_size - idx - 1);
+			shift *= (controller->w_size - i - 1);
 			rx_buf[controller->rx_bytes] = word >> shift;
 		}
 	}
 }
 
-static void spi_qup_fifo_write(struct spi_qup *controller,
-			    struct spi_transfer *xfer)
+static void spi_qup_read(struct spi_qup *controller, u32 *opflags)
 {
-	const u8 *tx_buf = xfer->tx_buf;
-	u32 word, state, data;
-	int idx, w_size;
+	u32 remainder, words_per_block, num_words;
+	bool is_block_mode = controller->mode == QUP_IO_M_MODE_BLOCK;
+
+	remainder = DIV_ROUND_UP(spi_qup_len(controller) - controller->rx_bytes,
+				 controller->w_size);
+	words_per_block = controller->in_blk_sz >> 2;
+
+	do {
+		/* ACK by clearing service flag */
+		writel_relaxed(QUP_OP_IN_SERVICE_FLAG,
+			       controller->base + QUP_OPERATIONAL);
+
+		if (is_block_mode) {
+			num_words = (remainder > words_per_block) ?
+					words_per_block : remainder;
+		} else {
+			if (!spi_qup_is_flag_set(controller,
+						 QUP_OP_IN_FIFO_NOT_EMPTY))
+				break;
 
-	w_size = controller->w_size;
+			num_words = 1;
+		}
 
-	while (controller->tx_bytes < xfer->len) {
+		/* read up to the maximum transfer size available */
+		spi_qup_read_from_fifo(controller, num_words);
 
-		state = readl_relaxed(controller->base + QUP_OPERATIONAL);
-		if (state & QUP_OP_OUT_FIFO_FULL)
+		remainder -= num_words;
+
+		/* if block mode, check to see if next block is available */
+		if (is_block_mode && !spi_qup_is_flag_set(controller,
+					QUP_OP_IN_BLOCK_READ_REQ))
 			break;
 
+	} while (remainder);
+
+	/*
+	 * Due to extra stickiness of the QUP_OP_IN_SERVICE_FLAG during block
+	 * reads, it has to be cleared again at the very end.  However, be sure
+	 * to refresh opflags value because MAX_INPUT_DONE_FLAG may now be
+	 * present and this is used to determine if transaction is complete
+	 */
+	*opflags = readl_relaxed(controller->base + QUP_OPERATIONAL);
+	if (is_block_mode && *opflags & QUP_OP_MAX_INPUT_DONE_FLAG)
+		writel_relaxed(QUP_OP_IN_SERVICE_FLAG,
+			       controller->base + QUP_OPERATIONAL);
+
+}
+
+static void spi_qup_write_to_fifo(struct spi_qup *controller, u32 num_words)
+{
+	const u8 *tx_buf = controller->tx_buf;
+	int i, num_bytes;
+	u32 word, data;
+
+	for (; num_words; num_words--) {
 		word = 0;
-		for (idx = 0; idx < w_size; idx++, controller->tx_bytes++) {
 
-			if (!tx_buf) {
-				controller->tx_bytes += w_size;
-				break;
+		num_bytes = min_t(int, spi_qup_len(controller) -
+				       controller->tx_bytes,
+				       controller->w_size);
+		if (tx_buf)
+			for (i = 0; i < num_bytes; i++) {
+				data = tx_buf[controller->tx_bytes + i];
+				word |= data << (BITS_PER_BYTE * (3 - i));
 			}
 
-			data = tx_buf[controller->tx_bytes];
-			word |= data << (BITS_PER_BYTE * (3 - idx));
-		}
+		controller->tx_bytes += num_bytes;
 
 		writel_relaxed(word, controller->base + QUP_OUTPUT_FIFO);
 	}
@@ -281,31 +348,61 @@ static void spi_qup_dma_done(void *data)
 	complete(&qup->done);
 }
 
-static int spi_qup_prep_sg(struct spi_master *master, struct spi_transfer *xfer,
-			   enum dma_transfer_direction dir,
+static void spi_qup_write(struct spi_qup *controller)
+{
+	bool is_block_mode = controller->mode == QUP_IO_M_MODE_BLOCK;
+	u32 remainder, words_per_block, num_words;
+
+	remainder = DIV_ROUND_UP(spi_qup_len(controller) - controller->tx_bytes,
+				 controller->w_size);
+	words_per_block = controller->out_blk_sz >> 2;
+
+	do {
+		/* ACK by clearing service flag */
+		writel_relaxed(QUP_OP_OUT_SERVICE_FLAG,
+			       controller->base + QUP_OPERATIONAL);
+
+		if (is_block_mode) {
+			num_words = (remainder > words_per_block) ?
+				words_per_block : remainder;
+		} else {
+			if (spi_qup_is_flag_set(controller,
+						QUP_OP_OUT_FIFO_FULL))
+				break;
+
+			num_words = 1;
+		}
+
+		spi_qup_write_to_fifo(controller, num_words);
+
+		remainder -= num_words;
+
+		/* if block mode, check to see if next block is available */
+		if (is_block_mode && !spi_qup_is_flag_set(controller,
+					QUP_OP_OUT_BLOCK_WRITE_REQ))
+			break;
+
+	} while (remainder);
+}
+
+static int spi_qup_prep_sg(struct spi_master *master, struct scatterlist *sgl,
+			   unsigned int nents, enum dma_transfer_direction dir,
 			   dma_async_tx_callback callback)
 {
 	struct spi_qup *qup = spi_master_get_devdata(master);
 	unsigned long flags = DMA_PREP_INTERRUPT | DMA_PREP_FENCE;
 	struct dma_async_tx_descriptor *desc;
-	struct scatterlist *sgl;
 	struct dma_chan *chan;
 	dma_cookie_t cookie;
-	unsigned int nents;
 
-	if (dir == DMA_MEM_TO_DEV) {
+	if (dir == DMA_MEM_TO_DEV)
 		chan = master->dma_tx;
-		nents = xfer->tx_sg.nents;
-		sgl = xfer->tx_sg.sgl;
-	} else {
+	else
 		chan = master->dma_rx;
-		nents = xfer->rx_sg.nents;
-		sgl = xfer->rx_sg.sgl;
-	}
 
 	desc = dmaengine_prep_slave_sg(chan, sgl, nents, dir, flags);
-	if (!desc)
-		return -EINVAL;
+	if (IS_ERR_OR_NULL(desc))
+		return desc ? PTR_ERR(desc) : -EINVAL;
 
 	desc->callback = callback;
 	desc->callback_param = qup;
@@ -324,9 +421,33 @@ static void spi_qup_dma_terminate(struct spi_master *master,
 		dmaengine_terminate_all(master->dma_rx);
 }
 
-static int spi_qup_do_dma(struct spi_master *master, struct spi_transfer *xfer)
+static u32 spi_qup_sgl_get_nents_len(struct scatterlist *sgl, u32 max,
+				     u32 *nents)
+{
+	struct scatterlist *sg;
+	u32 total = 0;
+
+	for (sg = sgl; sg; sg = sg_next(sg)) {
+		unsigned int len = sg_dma_len(sg);
+
+		/* check for overflow as well as limit */
+		if (((total + len) < total) || ((total + len) > max))
+			break;
+
+		total += len;
+		(*nents)++;
+	}
+
+	return total;
+}
+
+static int spi_qup_do_dma(struct spi_device *spi, struct spi_transfer *xfer,
+			  unsigned long timeout)
 {
 	dma_async_tx_callback rx_done = NULL, tx_done = NULL;
+	struct spi_master *master = spi->master;
+	struct spi_qup *qup = spi_master_get_devdata(master);
+	struct scatterlist *tx_sgl, *rx_sgl;
 	int ret;
 
 	if (xfer->rx_buf)
@@ -334,43 +455,122 @@ static int spi_qup_do_dma(struct spi_master *master, struct spi_transfer *xfer)
 	else if (xfer->tx_buf)
 		tx_done = spi_qup_dma_done;
 
-	if (xfer->rx_buf) {
-		ret = spi_qup_prep_sg(master, xfer, DMA_DEV_TO_MEM, rx_done);
-		if (ret)
-			return ret;
+	rx_sgl = xfer->rx_sg.sgl;
+	tx_sgl = xfer->tx_sg.sgl;
 
-		dma_async_issue_pending(master->dma_rx);
-	}
+	do {
+		u32 rx_nents = 0, tx_nents = 0;
 
-	if (xfer->tx_buf) {
-		ret = spi_qup_prep_sg(master, xfer, DMA_MEM_TO_DEV, tx_done);
+		if (rx_sgl)
+			qup->n_words = spi_qup_sgl_get_nents_len(rx_sgl,
+					SPI_MAX_XFER, &rx_nents) / qup->w_size;
+		if (tx_sgl)
+			qup->n_words = spi_qup_sgl_get_nents_len(tx_sgl,
+					SPI_MAX_XFER, &tx_nents) / qup->w_size;
+		if (!qup->n_words)
+			return -EIO;
+
+		ret = spi_qup_io_config(spi, xfer);
 		if (ret)
 			return ret;
 
-		dma_async_issue_pending(master->dma_tx);
-	}
+		/* before issuing the descriptors, set the QUP to run */
+		ret = spi_qup_set_state(qup, QUP_STATE_RUN);
+		if (ret) {
+			dev_warn(qup->dev, "cannot set RUN state\n");
+			return ret;
+		}
+		if (rx_sgl) {
+			ret = spi_qup_prep_sg(master, rx_sgl, rx_nents,
+					      DMA_DEV_TO_MEM, rx_done);
+			if (ret)
+				return ret;
+			dma_async_issue_pending(master->dma_rx);
+		}
+
+		if (tx_sgl) {
+			ret = spi_qup_prep_sg(master, tx_sgl, tx_nents,
+					      DMA_MEM_TO_DEV, tx_done);
+			if (ret)
+				return ret;
+
+			dma_async_issue_pending(master->dma_tx);
+		}
+
+		if (!wait_for_completion_timeout(&qup->done, timeout))
+			return -ETIMEDOUT;
+
+		for (; rx_sgl && rx_nents--; rx_sgl = sg_next(rx_sgl))
+			;
+		for (; tx_sgl && tx_nents--; tx_sgl = sg_next(tx_sgl))
+			;
+
+	} while (rx_sgl || tx_sgl);
 
 	return 0;
 }
 
-static int spi_qup_do_pio(struct spi_master *master, struct spi_transfer *xfer)
+static int spi_qup_do_pio(struct spi_device *spi, struct spi_transfer *xfer,
+			  unsigned long timeout)
 {
+	struct spi_master *master = spi->master;
 	struct spi_qup *qup = spi_master_get_devdata(master);
-	int ret;
+	int ret, n_words, iterations, offset = 0;
 
-	ret = spi_qup_set_state(qup, QUP_STATE_RUN);
-	if (ret) {
-		dev_warn(qup->dev, "cannot set RUN state\n");
-		return ret;
-	}
+	n_words = qup->n_words;
+	iterations = n_words / SPI_MAX_XFER; /* round down */
+	qup->rx_buf = xfer->rx_buf;
+	qup->tx_buf = xfer->tx_buf;
 
-	ret = spi_qup_set_state(qup, QUP_STATE_PAUSE);
-	if (ret) {
-		dev_warn(qup->dev, "cannot set PAUSE state\n");
-		return ret;
-	}
+	do {
+		if (iterations)
+			qup->n_words = SPI_MAX_XFER;
+		else
+			qup->n_words = n_words % SPI_MAX_XFER;
+
+		if (qup->tx_buf && offset)
+			qup->tx_buf = xfer->tx_buf + offset * SPI_MAX_XFER;
+
+		if (qup->rx_buf && offset)
+			qup->rx_buf = xfer->rx_buf + offset * SPI_MAX_XFER;
+
+		/*
+		 * if the transaction is small enough, we need
+		 * to fallback to FIFO mode
+		 */
+		if (qup->n_words <= (qup->in_fifo_sz / sizeof(u32)))
+			qup->mode = QUP_IO_M_MODE_FIFO;
 
-	spi_qup_fifo_write(qup, xfer);
+		ret = spi_qup_io_config(spi, xfer);
+		if (ret)
+			return ret;
+
+		ret = spi_qup_set_state(qup, QUP_STATE_RUN);
+		if (ret) {
+			dev_warn(qup->dev, "cannot set RUN state\n");
+			return ret;
+		}
+
+		ret = spi_qup_set_state(qup, QUP_STATE_PAUSE);
+		if (ret) {
+			dev_warn(qup->dev, "cannot set PAUSE state\n");
+			return ret;
+		}
+
+		if (qup->mode == QUP_IO_M_MODE_FIFO)
+			spi_qup_write(qup);
+
+		ret = spi_qup_set_state(qup, QUP_STATE_RUN);
+		if (ret) {
+			dev_warn(qup->dev, "cannot set RUN state\n");
+			return ret;
+		}
+
+		if (!wait_for_completion_timeout(&qup->done, timeout))
+			return -ETIMEDOUT;
+
+		offset++;
+	} while (iterations--);
 
 	return 0;
 }
@@ -378,29 +578,15 @@ static int spi_qup_do_pio(struct spi_master *master, struct spi_transfer *xfer)
 static irqreturn_t spi_qup_qup_irq(int irq, void *dev_id)
 {
 	struct spi_qup *controller = dev_id;
-	struct spi_transfer *xfer;
 	u32 opflags, qup_err, spi_err;
-	unsigned long flags;
 	int error = 0;
 
-	spin_lock_irqsave(&controller->lock, flags);
-	xfer = controller->xfer;
-	controller->xfer = NULL;
-	spin_unlock_irqrestore(&controller->lock, flags);
-
 	qup_err = readl_relaxed(controller->base + QUP_ERROR_FLAGS);
 	spi_err = readl_relaxed(controller->base + SPI_ERROR_FLAGS);
 	opflags = readl_relaxed(controller->base + QUP_OPERATIONAL);
 
 	writel_relaxed(qup_err, controller->base + QUP_ERROR_FLAGS);
 	writel_relaxed(spi_err, controller->base + SPI_ERROR_FLAGS);
-	writel_relaxed(opflags, controller->base + QUP_OPERATIONAL);
-
-	if (!xfer) {
-		dev_err_ratelimited(controller->dev, "unexpected irq %08x %08x %08x\n",
-				    qup_err, spi_err, opflags);
-		return IRQ_HANDLED;
-	}
 
 	if (qup_err) {
 		if (qup_err & QUP_ERROR_OUTPUT_OVER_RUN)
@@ -424,54 +610,27 @@ static irqreturn_t spi_qup_qup_irq(int irq, void *dev_id)
 		error = -EIO;
 	}
 
-	if (!controller->use_dma) {
+	if (spi_qup_is_dma_xfer(controller->mode)) {
+		writel_relaxed(opflags, controller->base + QUP_OPERATIONAL);
+	} else {
 		if (opflags & QUP_OP_IN_SERVICE_FLAG)
-			spi_qup_fifo_read(controller, xfer);
+			spi_qup_read(controller, &opflags);
 
 		if (opflags & QUP_OP_OUT_SERVICE_FLAG)
-			spi_qup_fifo_write(controller, xfer);
+			spi_qup_write(controller);
 	}
 
-	spin_lock_irqsave(&controller->lock, flags);
-	controller->error = error;
-	controller->xfer = xfer;
-	spin_unlock_irqrestore(&controller->lock, flags);
-
-	if (controller->rx_bytes == xfer->len || error)
+	if ((opflags & QUP_OP_MAX_INPUT_DONE_FLAG) || error)
 		complete(&controller->done);
 
 	return IRQ_HANDLED;
 }
 
-static u32
-spi_qup_get_mode(struct spi_master *master, struct spi_transfer *xfer)
-{
-	struct spi_qup *qup = spi_master_get_devdata(master);
-	u32 mode;
-
-	qup->w_size = 4;
-
-	if (xfer->bits_per_word <= 8)
-		qup->w_size = 1;
-	else if (xfer->bits_per_word <= 16)
-		qup->w_size = 2;
-
-	qup->n_words = xfer->len / qup->w_size;
-
-	if (qup->n_words <= (qup->in_fifo_sz / sizeof(u32)))
-		mode = QUP_IO_M_MODE_FIFO;
-	else
-		mode = QUP_IO_M_MODE_BLOCK;
-
-	return mode;
-}
-
-/* set clock freq ... bits per word */
-static int spi_qup_io_config(struct spi_device *spi, struct spi_transfer *xfer)
+/* set clock freq ... bits per word, determine mode */
+static int spi_qup_io_prep(struct spi_device *spi, struct spi_transfer *xfer)
 {
 	struct spi_qup *controller = spi_master_get_devdata(spi->master);
-	u32 config, iomode, mode, control;
-	int ret, n_words;
+	int ret;
 
 	if (spi->mode & SPI_LOOP && xfer->len > controller->in_fifo_sz) {
 		dev_err(controller->dev, "too big size for loopback %d > %d\n",
@@ -486,30 +645,59 @@ static int spi_qup_io_config(struct spi_device *spi, struct spi_transfer *xfer)
 		return -EIO;
 	}
 
+	controller->w_size = DIV_ROUND_UP(xfer->bits_per_word, 8);
+	controller->n_words = xfer->len / controller->w_size;
+
+	if (controller->n_words <= (controller->in_fifo_sz / sizeof(u32)))
+		controller->mode = QUP_IO_M_MODE_FIFO;
+	else if (spi->master->can_dma &&
+		 spi->master->can_dma(spi->master, spi, xfer) &&
+		 spi->master->cur_msg_mapped)
+		controller->mode = QUP_IO_M_MODE_BAM;
+	else
+		controller->mode = QUP_IO_M_MODE_BLOCK;
+
+	return 0;
+}
+
+/* prep qup for another spi transaction of specific type */
+static int spi_qup_io_config(struct spi_device *spi, struct spi_transfer *xfer)
+{
+	struct spi_qup *controller = spi_master_get_devdata(spi->master);
+	u32 config, iomode, control;
+	unsigned long flags;
+
+	spin_lock_irqsave(&controller->lock, flags);
+	controller->xfer     = xfer;
+	controller->error    = 0;
+	controller->rx_bytes = 0;
+	controller->tx_bytes = 0;
+	spin_unlock_irqrestore(&controller->lock, flags);
+
+
 	if (spi_qup_set_state(controller, QUP_STATE_RESET)) {
 		dev_err(controller->dev, "cannot set RESET state\n");
 		return -EIO;
 	}
 
-	mode = spi_qup_get_mode(spi->master, xfer);
-	n_words = controller->n_words;
-
-	if (mode == QUP_IO_M_MODE_FIFO) {
-		writel_relaxed(n_words, controller->base + QUP_MX_READ_CNT);
-		writel_relaxed(n_words, controller->base + QUP_MX_WRITE_CNT);
+	switch (controller->mode) {
+	case QUP_IO_M_MODE_FIFO:
+		writel_relaxed(controller->n_words,
+			       controller->base + QUP_MX_READ_CNT);
+		writel_relaxed(controller->n_words,
+			       controller->base + QUP_MX_WRITE_CNT);
 		/* must be zero for FIFO */
 		writel_relaxed(0, controller->base + QUP_MX_INPUT_CNT);
 		writel_relaxed(0, controller->base + QUP_MX_OUTPUT_CNT);
-	} else if (!controller->use_dma) {
-		writel_relaxed(n_words, controller->base + QUP_MX_INPUT_CNT);
-		writel_relaxed(n_words, controller->base + QUP_MX_OUTPUT_CNT);
+		break;
+	case QUP_IO_M_MODE_BAM:
+		writel_relaxed(controller->n_words,
+			       controller->base + QUP_MX_INPUT_CNT);
+		writel_relaxed(controller->n_words,
+			       controller->base + QUP_MX_OUTPUT_CNT);
 		/* must be zero for BLOCK and BAM */
 		writel_relaxed(0, controller->base + QUP_MX_READ_CNT);
 		writel_relaxed(0, controller->base + QUP_MX_WRITE_CNT);
-	} else {
-		mode = QUP_IO_M_MODE_BAM;
-		writel_relaxed(0, controller->base + QUP_MX_READ_CNT);
-		writel_relaxed(0, controller->base + QUP_MX_WRITE_CNT);
 
 		if (!controller->qup_v1) {
 			void __iomem *input_cnt;
@@ -524,23 +712,38 @@ static int spi_qup_io_config(struct spi_device *spi, struct spi_transfer *xfer)
 			if (xfer->tx_buf)
 				writel_relaxed(0, input_cnt);
 			else
-				writel_relaxed(n_words, input_cnt);
+				writel_relaxed(controller->n_words, input_cnt);
 
 			writel_relaxed(0, controller->base + QUP_MX_OUTPUT_CNT);
 		}
+		break;
+	case QUP_IO_M_MODE_BLOCK:
+		reinit_completion(&controller->done);
+		writel_relaxed(controller->n_words,
+			       controller->base + QUP_MX_INPUT_CNT);
+		writel_relaxed(controller->n_words,
+			       controller->base + QUP_MX_OUTPUT_CNT);
+		/* must be zero for BLOCK and BAM */
+		writel_relaxed(0, controller->base + QUP_MX_READ_CNT);
+		writel_relaxed(0, controller->base + QUP_MX_WRITE_CNT);
+		break;
+	default:
+		dev_err(controller->dev, "unknown mode = %d\n",
+				controller->mode);
+		return -EIO;
 	}
 
 	iomode = readl_relaxed(controller->base + QUP_IO_M_MODES);
 	/* Set input and output transfer mode */
 	iomode &= ~(QUP_IO_M_INPUT_MODE_MASK | QUP_IO_M_OUTPUT_MODE_MASK);
 
-	if (!controller->use_dma)
+	if (!spi_qup_is_dma_xfer(controller->mode))
 		iomode &= ~(QUP_IO_M_PACK_EN | QUP_IO_M_UNPACK_EN);
 	else
 		iomode |= QUP_IO_M_PACK_EN | QUP_IO_M_UNPACK_EN;
 
-	iomode |= (mode << QUP_IO_M_OUTPUT_MODE_MASK_SHIFT);
-	iomode |= (mode << QUP_IO_M_INPUT_MODE_MASK_SHIFT);
+	iomode |= (controller->mode << QUP_IO_M_OUTPUT_MODE_MASK_SHIFT);
+	iomode |= (controller->mode << QUP_IO_M_INPUT_MODE_MASK_SHIFT);
 
 	writel_relaxed(iomode, controller->base + QUP_IO_M_MODES);
 
@@ -581,7 +784,7 @@ static int spi_qup_io_config(struct spi_device *spi, struct spi_transfer *xfer)
 	config |= xfer->bits_per_word - 1;
 	config |= QUP_CONFIG_SPI_MODE;
 
-	if (controller->use_dma) {
+	if (spi_qup_is_dma_xfer(controller->mode)) {
 		if (!xfer->tx_buf)
 			config |= QUP_CONFIG_NO_OUTPUT;
 		if (!xfer->rx_buf)
@@ -599,7 +802,7 @@ static int spi_qup_io_config(struct spi_device *spi, struct spi_transfer *xfer)
 		 * status change in BAM mode
 		 */
 
-		if (mode == QUP_IO_M_MODE_BAM)
+		if (spi_qup_is_dma_xfer(controller->mode))
 			mask = QUP_OP_IN_SERVICE_FLAG | QUP_OP_OUT_SERVICE_FLAG;
 
 		writel_relaxed(mask, controller->base + QUP_OPERATIONAL_MASK);
@@ -616,12 +819,13 @@ static int spi_qup_transfer_one(struct spi_master *master,
 	unsigned long timeout, flags;
 	int ret = -EIO;
 
-	ret = spi_qup_io_config(spi, xfer);
+	ret = spi_qup_io_prep(spi, xfer);
 	if (ret)
 		return ret;
 
 	timeout = DIV_ROUND_UP(xfer->speed_hz, MSEC_PER_SEC);
-	timeout = DIV_ROUND_UP(xfer->len * 8, timeout);
+	timeout = DIV_ROUND_UP(min_t(unsigned long, SPI_MAX_XFER,
+				     xfer->len) * 8, timeout);
 	timeout = 100 * msecs_to_jiffies(timeout);
 
 	reinit_completion(&controller->done);
@@ -633,31 +837,22 @@ static int spi_qup_transfer_one(struct spi_master *master,
 	controller->tx_bytes = 0;
 	spin_unlock_irqrestore(&controller->lock, flags);
 
-	if (controller->use_dma)
-		ret = spi_qup_do_dma(master, xfer);
+	if (spi_qup_is_dma_xfer(controller->mode))
+		ret = spi_qup_do_dma(spi, xfer, timeout);
 	else
-		ret = spi_qup_do_pio(master, xfer);
+		ret = spi_qup_do_pio(spi, xfer, timeout);
 
 	if (ret)
 		goto exit;
 
-	if (spi_qup_set_state(controller, QUP_STATE_RUN)) {
-		dev_warn(controller->dev, "cannot set EXECUTE state\n");
-		goto exit;
-	}
-
-	if (!wait_for_completion_timeout(&controller->done, timeout))
-		ret = -ETIMEDOUT;
-
 exit:
 	spi_qup_set_state(controller, QUP_STATE_RESET);
 	spin_lock_irqsave(&controller->lock, flags);
-	controller->xfer = NULL;
 	if (!ret)
 		ret = controller->error;
 	spin_unlock_irqrestore(&controller->lock, flags);
 
-	if (ret && controller->use_dma)
+	if (ret && spi_qup_is_dma_xfer(controller->mode))
 		spi_qup_dma_terminate(master, xfer);
 
 	return ret;
@@ -668,26 +863,28 @@ static bool spi_qup_can_dma(struct spi_master *master, struct spi_device *spi,
 {
 	struct spi_qup *qup = spi_master_get_devdata(master);
 	size_t dma_align = dma_get_cache_alignment();
-	u32 mode;
-
-	qup->use_dma = 0;
+	int n_words;
 
-	if (xfer->rx_buf && (xfer->len % qup->in_blk_sz ||
-	    IS_ERR_OR_NULL(master->dma_rx) ||
-	    !IS_ALIGNED((size_t)xfer->rx_buf, dma_align)))
-		return false;
+	if (xfer->rx_buf) {
+		if (!IS_ALIGNED((size_t)xfer->rx_buf, dma_align) ||
+		    IS_ERR_OR_NULL(master->dma_rx))
+			return false;
+		if (qup->qup_v1 && (xfer->len % qup->in_blk_sz))
+			return false;
+	}
 
-	if (xfer->tx_buf && (xfer->len % qup->out_blk_sz ||
-	    IS_ERR_OR_NULL(master->dma_tx) ||
-	    !IS_ALIGNED((size_t)xfer->tx_buf, dma_align)))
-		return false;
+	if (xfer->tx_buf) {
+		if (!IS_ALIGNED((size_t)xfer->tx_buf, dma_align) ||
+		    IS_ERR_OR_NULL(master->dma_tx))
+			return false;
+		if (qup->qup_v1 && (xfer->len % qup->out_blk_sz))
+			return false;
+	}
 
-	mode = spi_qup_get_mode(master, xfer);
-	if (mode == QUP_IO_M_MODE_FIFO)
+	n_words = xfer->len / DIV_ROUND_UP(xfer->bits_per_word, 8);
+	if (n_words <= (qup->in_fifo_sz / sizeof(u32)))
 		return false;
 
-	qup->use_dma = 1;
-
 	return true;
 }
 
@@ -750,6 +947,24 @@ err_tx:
 	return ret;
 }
 
+static void spi_qup_set_cs(struct spi_device *spi, bool val)
+{
+	struct spi_qup *controller;
+	u32 spi_ioc;
+	u32 spi_ioc_orig;
+
+	controller = spi_master_get_devdata(spi->master);
+	spi_ioc = readl_relaxed(controller->base + SPI_IO_CONTROL);
+	spi_ioc_orig = spi_ioc;
+	if (!val)
+		spi_ioc |= SPI_IO_C_FORCE_CS;
+	else
+		spi_ioc &= ~SPI_IO_C_FORCE_CS;
+
+	if (spi_ioc != spi_ioc_orig)
+		writel_relaxed(spi_ioc, controller->base + SPI_IO_CONTROL);
+}
+
 static int spi_qup_probe(struct platform_device *pdev)
 {
 	struct spi_master *master;
@@ -824,7 +1039,7 @@ static int spi_qup_probe(struct platform_device *pdev)
 	master->dev.of_node = pdev->dev.of_node;
 	master->auto_runtime_pm = true;
 	master->dma_alignment = dma_get_cache_alignment();
-	master->max_dma_len = SPI_MAX_DMA_XFER;
+	master->max_dma_len = SPI_MAX_XFER;
 
 	platform_set_drvdata(pdev, master);
 
@@ -842,9 +1057,10 @@ static int spi_qup_probe(struct platform_device *pdev)
 	else if (!ret)
 		master->can_dma = spi_qup_can_dma;
 
-	/* set v1 flag if device is version 1 */
-	if (of_device_is_compatible(dev->of_node, "qcom,spi-qup-v1.1.1"))
-		controller->qup_v1 = 1;
+	controller->qup_v1 = (uintptr_t)of_device_get_match_data(dev);
+
+	if (!controller->qup_v1)
+		master->set_cs = spi_qup_set_cs;
 
 	spin_lock_init(&controller->lock);
 	init_completion(&controller->done);
@@ -1037,7 +1253,7 @@ static int spi_qup_remove(struct platform_device *pdev)
 }
 
 static const struct of_device_id spi_qup_dt_match[] = {
-	{ .compatible = "qcom,spi-qup-v1.1.1", },
+	{ .compatible = "qcom,spi-qup-v1.1.1", .data = (void *)1, },
 	{ .compatible = "qcom,spi-qup-v2.1.1", },
 	{ .compatible = "qcom,spi-qup-v2.2.1", },
 	{ }
diff --git a/drivers/spi/spi-rockchip.c b/drivers/spi/spi-rockchip.c
index 0b4a52b3e1dc..fdcf3076681b 100644
--- a/drivers/spi/spi-rockchip.c
+++ b/drivers/spi/spi-rockchip.c
@@ -568,7 +568,13 @@ static void rockchip_spi_config(struct rockchip_spi *rs)
 
 	writel_relaxed(cr0, rs->regs + ROCKCHIP_SPI_CTRLR0);
 
-	writel_relaxed(rs->len - 1, rs->regs + ROCKCHIP_SPI_CTRLR1);
+	if (rs->n_bytes == 1)
+		writel_relaxed(rs->len - 1, rs->regs + ROCKCHIP_SPI_CTRLR1);
+	else if (rs->n_bytes == 2)
+		writel_relaxed((rs->len / 2) - 1, rs->regs + ROCKCHIP_SPI_CTRLR1);
+	else
+		writel_relaxed((rs->len * 2) - 1, rs->regs + ROCKCHIP_SPI_CTRLR1);
+
 	writel_relaxed(rs->fifo_len / 2 - 1, rs->regs + ROCKCHIP_SPI_TXFTLR);
 	writel_relaxed(rs->fifo_len / 2 - 1, rs->regs + ROCKCHIP_SPI_RXFTLR);
 
@@ -666,7 +672,7 @@ static bool rockchip_spi_can_dma(struct spi_master *master,
 
 static int rockchip_spi_probe(struct platform_device *pdev)
 {
-	int ret = 0;
+	int ret;
 	struct rockchip_spi *rs;
 	struct spi_master *master;
 	struct resource *mem;
@@ -703,13 +709,13 @@ static int rockchip_spi_probe(struct platform_device *pdev)
 	}
 
 	ret = clk_prepare_enable(rs->apb_pclk);
-	if (ret) {
+	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to enable apb_pclk\n");
 		goto err_put_master;
 	}
 
 	ret = clk_prepare_enable(rs->spiclk);
-	if (ret) {
+	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to enable spi_clk\n");
 		goto err_disable_apbclk;
 	}
@@ -786,7 +792,7 @@ static int rockchip_spi_probe(struct platform_device *pdev)
 	}
 
 	ret = devm_spi_register_master(&pdev->dev, master);
-	if (ret) {
+	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to register master\n");
 		goto err_free_dma_rx;
 	}
@@ -816,11 +822,15 @@ static int rockchip_spi_remove(struct platform_device *pdev)
 	struct spi_master *master = spi_master_get(platform_get_drvdata(pdev));
 	struct rockchip_spi *rs = spi_master_get_devdata(master);
 
-	pm_runtime_disable(&pdev->dev);
+	pm_runtime_get_sync(&pdev->dev);
 
 	clk_disable_unprepare(rs->spiclk);
 	clk_disable_unprepare(rs->apb_pclk);
 
+	pm_runtime_put_noidle(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+	pm_runtime_set_suspended(&pdev->dev);
+
 	if (rs->dma_tx.ch)
 		dma_release_channel(rs->dma_tx.ch);
 	if (rs->dma_rx.ch)
@@ -834,43 +844,34 @@ static int rockchip_spi_remove(struct platform_device *pdev)
 #ifdef CONFIG_PM_SLEEP
 static int rockchip_spi_suspend(struct device *dev)
 {
-	int ret = 0;
+	int ret;
 	struct spi_master *master = dev_get_drvdata(dev);
 	struct rockchip_spi *rs = spi_master_get_devdata(master);
 
 	ret = spi_master_suspend(rs->master);
-	if (ret)
+	if (ret < 0)
 		return ret;
 
-	if (!pm_runtime_suspended(dev)) {
-		clk_disable_unprepare(rs->spiclk);
-		clk_disable_unprepare(rs->apb_pclk);
-	}
+	ret = pm_runtime_force_suspend(dev);
+	if (ret < 0)
+		return ret;
 
 	pinctrl_pm_select_sleep_state(dev);
 
-	return ret;
+	return 0;
 }
 
 static int rockchip_spi_resume(struct device *dev)
 {
-	int ret = 0;
+	int ret;
 	struct spi_master *master = dev_get_drvdata(dev);
 	struct rockchip_spi *rs = spi_master_get_devdata(master);
 
 	pinctrl_pm_select_default_state(dev);
 
-	if (!pm_runtime_suspended(dev)) {
-		ret = clk_prepare_enable(rs->apb_pclk);
-		if (ret < 0)
-			return ret;
-
-		ret = clk_prepare_enable(rs->spiclk);
-		if (ret < 0) {
-			clk_disable_unprepare(rs->apb_pclk);
-			return ret;
-		}
-	}
+	ret = pm_runtime_force_resume(dev);
+	if (ret < 0)
+		return ret;
 
 	ret = spi_master_resume(rs->master);
 	if (ret < 0) {
@@ -878,7 +879,7 @@ static int rockchip_spi_resume(struct device *dev)
 		clk_disable_unprepare(rs->apb_pclk);
 	}
 
-	return ret;
+	return 0;
 }
 #endif /* CONFIG_PM_SLEEP */
 
@@ -901,14 +902,14 @@ static int rockchip_spi_runtime_resume(struct device *dev)
 	struct rockchip_spi *rs = spi_master_get_devdata(master);
 
 	ret = clk_prepare_enable(rs->apb_pclk);
-	if (ret)
+	if (ret < 0)
 		return ret;
 
 	ret = clk_prepare_enable(rs->spiclk);
-	if (ret)
+	if (ret < 0)
 		clk_disable_unprepare(rs->apb_pclk);
 
-	return ret;
+	return 0;
 }
 #endif /* CONFIG_PM */
 
@@ -919,6 +920,7 @@ static const struct dev_pm_ops rockchip_spi_pm = {
 };
 
 static const struct of_device_id rockchip_spi_dt_match[] = {
+	{ .compatible = "rockchip,rv1108-spi", },
 	{ .compatible = "rockchip,rk3036-spi", },
 	{ .compatible = "rockchip,rk3066-spi", },
 	{ .compatible = "rockchip,rk3188-spi", },
diff --git a/drivers/spi/spi-sh-msiof.c b/drivers/spi/spi-sh-msiof.c
index c304c7167866..0eb1e9583485 100644
--- a/drivers/spi/spi-sh-msiof.c
+++ b/drivers/spi/spi-sh-msiof.c
@@ -38,6 +38,7 @@ struct sh_msiof_chipdata {
 	u16 tx_fifo_size;
 	u16 rx_fifo_size;
 	u16 master_flags;
+	u16 min_div;
 };
 
 struct sh_msiof_spi_priv {
@@ -49,6 +50,7 @@ struct sh_msiof_spi_priv {
 	struct completion done;
 	unsigned int tx_fifo_size;
 	unsigned int rx_fifo_size;
+	unsigned int min_div;
 	void *tx_dma_page;
 	void *rx_dma_page;
 	dma_addr_t tx_dma_addr;
@@ -261,6 +263,8 @@ static void sh_msiof_spi_set_clk_regs(struct sh_msiof_spi_priv *p,
 	if (!WARN_ON(!spi_hz || !parent_rate))
 		div = DIV_ROUND_UP(parent_rate, spi_hz);
 
+	div = max_t(unsigned long, div, p->min_div);
+
 	for (k = 0; k < ARRAY_SIZE(sh_msiof_spi_div_table); k++) {
 		brps = DIV_ROUND_UP(div, sh_msiof_spi_div_table[k].div);
 		/* SCR_BRDV_DIV_1 is valid only if BRPS is x 1/1 or x 1/2 */
@@ -998,24 +1002,33 @@ static const struct sh_msiof_chipdata sh_data = {
 	.tx_fifo_size = 64,
 	.rx_fifo_size = 64,
 	.master_flags = 0,
+	.min_div = 1,
+};
+
+static const struct sh_msiof_chipdata rcar_gen2_data = {
+	.tx_fifo_size = 64,
+	.rx_fifo_size = 64,
+	.master_flags = SPI_MASTER_MUST_TX,
+	.min_div = 1,
 };
 
-static const struct sh_msiof_chipdata r8a779x_data = {
+static const struct sh_msiof_chipdata rcar_gen3_data = {
 	.tx_fifo_size = 64,
 	.rx_fifo_size = 64,
 	.master_flags = SPI_MASTER_MUST_TX,
+	.min_div = 2,
 };
 
 static const struct of_device_id sh_msiof_match[] = {
 	{ .compatible = "renesas,sh-mobile-msiof", .data = &sh_data },
-	{ .compatible = "renesas,msiof-r8a7790",   .data = &r8a779x_data },
-	{ .compatible = "renesas,msiof-r8a7791",   .data = &r8a779x_data },
-	{ .compatible = "renesas,msiof-r8a7792",   .data = &r8a779x_data },
-	{ .compatible = "renesas,msiof-r8a7793",   .data = &r8a779x_data },
-	{ .compatible = "renesas,msiof-r8a7794",   .data = &r8a779x_data },
-	{ .compatible = "renesas,rcar-gen2-msiof", .data = &r8a779x_data },
-	{ .compatible = "renesas,msiof-r8a7796",   .data = &r8a779x_data },
-	{ .compatible = "renesas,rcar-gen3-msiof", .data = &r8a779x_data },
+	{ .compatible = "renesas,msiof-r8a7790",   .data = &rcar_gen2_data },
+	{ .compatible = "renesas,msiof-r8a7791",   .data = &rcar_gen2_data },
+	{ .compatible = "renesas,msiof-r8a7792",   .data = &rcar_gen2_data },
+	{ .compatible = "renesas,msiof-r8a7793",   .data = &rcar_gen2_data },
+	{ .compatible = "renesas,msiof-r8a7794",   .data = &rcar_gen2_data },
+	{ .compatible = "renesas,rcar-gen2-msiof", .data = &rcar_gen2_data },
+	{ .compatible = "renesas,msiof-r8a7796",   .data = &rcar_gen3_data },
+	{ .compatible = "renesas,rcar-gen3-msiof", .data = &rcar_gen3_data },
 	{ .compatible = "renesas,sh-msiof",        .data = &sh_data }, /* Deprecated */
 	{},
 };
@@ -1230,6 +1243,7 @@ static int sh_msiof_spi_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, p);
 	p->master = master;
 	p->info = info;
+	p->min_div = chipdata->min_div;
 
 	init_completion(&p->done);
 
diff --git a/drivers/spi/spi-sh.c b/drivers/spi/spi-sh.c
index 2bf53f0e27d9..50e0ea9acf8b 100644
--- a/drivers/spi/spi-sh.c
+++ b/drivers/spi/spi-sh.c
@@ -446,8 +446,8 @@ static int spi_sh_probe(struct platform_device *pdev)
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
-		dev_err(&pdev->dev, "platform_get_irq error\n");
-		return -ENODEV;
+		dev_err(&pdev->dev, "platform_get_irq error: %d\n", irq);
+		return irq;
 	}
 
 	master = spi_alloc_master(&pdev->dev, sizeof(struct spi_sh_data));
diff --git a/drivers/spi/spi-stm32.c b/drivers/spi/spi-stm32.c
index 75644bcd938b..680cdf549506 100644
--- a/drivers/spi/spi-stm32.c
+++ b/drivers/spi/spi-stm32.c
@@ -1132,7 +1132,7 @@ static int stm32_spi_probe(struct platform_device *pdev)
 		goto err_master_put;
 	}
 
-	spi->rst = devm_reset_control_get(&pdev->dev, NULL);
+	spi->rst = devm_reset_control_get_exclusive(&pdev->dev, NULL);
 	if (!IS_ERR(spi->rst)) {
 		reset_control_assert(spi->rst);
 		udelay(2);
diff --git a/drivers/spi/spi-sun6i.c b/drivers/spi/spi-sun6i.c
index 03a773a9531a..fb38234249a8 100644
--- a/drivers/spi/spi-sun6i.c
+++ b/drivers/spi/spi-sun6i.c
@@ -502,7 +502,7 @@ static int sun6i_spi_probe(struct platform_device *pdev)
 
 	init_completion(&sspi->done);
 
-	sspi->rstc = devm_reset_control_get(&pdev->dev, NULL);
+	sspi->rstc = devm_reset_control_get_exclusive(&pdev->dev, NULL);
 	if (IS_ERR(sspi->rstc)) {
 		dev_err(&pdev->dev, "Couldn't get reset controller\n");
 		ret = PTR_ERR(sspi->rstc);
diff --git a/drivers/spi/spi-tegra114.c b/drivers/spi/spi-tegra114.c
index 08012ae5aa66..44550182a4a3 100644
--- a/drivers/spi/spi-tegra114.c
+++ b/drivers/spi/spi-tegra114.c
@@ -1083,7 +1083,7 @@ static int tegra_spi_probe(struct platform_device *pdev)
 		goto exit_free_irq;
 	}
 
-	tspi->rst = devm_reset_control_get(&pdev->dev, "spi");
+	tspi->rst = devm_reset_control_get_exclusive(&pdev->dev, "spi");
 	if (IS_ERR(tspi->rst)) {
 		dev_err(&pdev->dev, "can not get reset\n");
 		ret = PTR_ERR(tspi->rst);
diff --git a/drivers/spi/spi-tegra20-sflash.c b/drivers/spi/spi-tegra20-sflash.c
index 2c797ee2664d..22893a7e0aa0 100644
--- a/drivers/spi/spi-tegra20-sflash.c
+++ b/drivers/spi/spi-tegra20-sflash.c
@@ -485,7 +485,7 @@ static int tegra_sflash_probe(struct platform_device *pdev)
 		goto exit_free_irq;
 	}
 
-	tsd->rst = devm_reset_control_get(&pdev->dev, "spi");
+	tsd->rst = devm_reset_control_get_exclusive(&pdev->dev, "spi");
 	if (IS_ERR(tsd->rst)) {
 		dev_err(&pdev->dev, "can not get reset\n");
 		ret = PTR_ERR(tsd->rst);
diff --git a/drivers/spi/spi-tegra20-slink.c b/drivers/spi/spi-tegra20-slink.c
index 0c06ce424210..3e12d5f87ee4 100644
--- a/drivers/spi/spi-tegra20-slink.c
+++ b/drivers/spi/spi-tegra20-slink.c
@@ -1081,7 +1081,7 @@ static int tegra_slink_probe(struct platform_device *pdev)
 		goto exit_free_irq;
 	}
 
-	tspi->rst = devm_reset_control_get(&pdev->dev, "spi");
+	tspi->rst = devm_reset_control_get_exclusive(&pdev->dev, "spi");
 	if (IS_ERR(tspi->rst)) {
 		dev_err(&pdev->dev, "can not get reset\n");
 		ret = PTR_ERR(tspi->rst);
diff --git a/drivers/spi/spi-xlp.c b/drivers/spi/spi-xlp.c
index 80cb4d6af892..74a01b09a8a5 100644
--- a/drivers/spi/spi-xlp.c
+++ b/drivers/spi/spi-xlp.c
@@ -393,8 +393,8 @@ static int xlp_spi_probe(struct platform_device *pdev)
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
-		dev_err(&pdev->dev, "no IRQ resource found\n");
-		return -EINVAL;
+		dev_err(&pdev->dev, "no IRQ resource found: %d\n", irq);
+		return irq;
 	}
 	err = devm_request_irq(&pdev->dev, irq, xlp_spi_interrupt, 0,
 			pdev->name, xspi);
diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c
index 4fcbb0aa71d3..6e65524cbfd9 100644
--- a/drivers/spi/spi.c
+++ b/drivers/spi/spi.c
@@ -40,9 +40,14 @@
 #include <linux/ioport.h>
 #include <linux/acpi.h>
 #include <linux/highmem.h>
+#include <linux/idr.h>
+#include <linux/platform_data/x86/apple.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/spi.h>
+#define SPI_DYN_FIRST_BUS_NUM 0
+
+static DEFINE_IDR(spi_master_idr);
 
 static void spidev_release(struct device *dev)
 {
@@ -321,8 +326,7 @@ static int spi_uevent(struct device *dev, struct kobj_uevent_env *env)
 	if (rc != -ENODEV)
 		return rc;
 
-	add_uevent_var(env, "MODALIAS=%s%s", SPI_MODULE_PREFIX, spi->modalias);
-	return 0;
+	return add_uevent_var(env, "MODALIAS=%s%s", SPI_MODULE_PREFIX, spi->modalias);
 }
 
 struct bus_type spi_bus_type = {
@@ -421,6 +425,7 @@ static LIST_HEAD(spi_controller_list);
 /*
  * Used to protect add/del opertion for board_info list and
  * spi_controller list, and their matching process
+ * also used to protect object of type struct idr
  */
 static DEFINE_MUTEX(board_lock);
 
@@ -1533,15 +1538,15 @@ static int of_spi_parse_dt(struct spi_controller *ctlr, struct spi_device *spi,
 	int rc;
 
 	/* Mode (clock phase/polarity/etc.) */
-	if (of_find_property(nc, "spi-cpha", NULL))
+	if (of_property_read_bool(nc, "spi-cpha"))
 		spi->mode |= SPI_CPHA;
-	if (of_find_property(nc, "spi-cpol", NULL))
+	if (of_property_read_bool(nc, "spi-cpol"))
 		spi->mode |= SPI_CPOL;
-	if (of_find_property(nc, "spi-cs-high", NULL))
+	if (of_property_read_bool(nc, "spi-cs-high"))
 		spi->mode |= SPI_CS_HIGH;
-	if (of_find_property(nc, "spi-3wire", NULL))
+	if (of_property_read_bool(nc, "spi-3wire"))
 		spi->mode |= SPI_3WIRE;
-	if (of_find_property(nc, "spi-lsb-first", NULL))
+	if (of_property_read_bool(nc, "spi-lsb-first"))
 		spi->mode |= SPI_LSB_FIRST;
 
 	/* Device DUAL/QUAD mode */
@@ -1583,8 +1588,8 @@ static int of_spi_parse_dt(struct spi_controller *ctlr, struct spi_device *spi,
 
 	if (spi_controller_is_slave(ctlr)) {
 		if (strcmp(nc->name, "slave")) {
-			dev_err(&ctlr->dev, "%s is not called 'slave'\n",
-				nc->full_name);
+			dev_err(&ctlr->dev, "%pOF is not called 'slave'\n",
+				nc);
 			return -EINVAL;
 		}
 		return 0;
@@ -1593,8 +1598,8 @@ static int of_spi_parse_dt(struct spi_controller *ctlr, struct spi_device *spi,
 	/* Device address */
 	rc = of_property_read_u32(nc, "reg", &value);
 	if (rc) {
-		dev_err(&ctlr->dev, "%s has no valid 'reg' property (%d)\n",
-			nc->full_name, rc);
+		dev_err(&ctlr->dev, "%pOF has no valid 'reg' property (%d)\n",
+			nc, rc);
 		return rc;
 	}
 	spi->chip_select = value;
@@ -1603,8 +1608,7 @@ static int of_spi_parse_dt(struct spi_controller *ctlr, struct spi_device *spi,
 	rc = of_property_read_u32(nc, "spi-max-frequency", &value);
 	if (rc) {
 		dev_err(&ctlr->dev,
-			"%s has no valid 'spi-max-frequency' property (%d)\n",
-			nc->full_name, rc);
+			"%pOF has no valid 'spi-max-frequency' property (%d)\n", nc, rc);
 		return rc;
 	}
 	spi->max_speed_hz = value;
@@ -1621,8 +1625,7 @@ of_register_spi_device(struct spi_controller *ctlr, struct device_node *nc)
 	/* Alloc an spi_device */
 	spi = spi_alloc_device(ctlr);
 	if (!spi) {
-		dev_err(&ctlr->dev, "spi_device alloc error for %s\n",
-			nc->full_name);
+		dev_err(&ctlr->dev, "spi_device alloc error for %pOF\n", nc);
 		rc = -ENOMEM;
 		goto err_out;
 	}
@@ -1631,8 +1634,7 @@ of_register_spi_device(struct spi_controller *ctlr, struct device_node *nc)
 	rc = of_modalias_node(nc, spi->modalias,
 				sizeof(spi->modalias));
 	if (rc < 0) {
-		dev_err(&ctlr->dev, "cannot find modalias for %s\n",
-			nc->full_name);
+		dev_err(&ctlr->dev, "cannot find modalias for %pOF\n", nc);
 		goto err_out;
 	}
 
@@ -1647,8 +1649,7 @@ of_register_spi_device(struct spi_controller *ctlr, struct device_node *nc)
 	/* Register the new device */
 	rc = spi_add_device(spi);
 	if (rc) {
-		dev_err(&ctlr->dev, "spi_device register error %s\n",
-			nc->full_name);
+		dev_err(&ctlr->dev, "spi_device register error %pOF\n", nc);
 		goto err_of_node_put;
 	}
 
@@ -1682,8 +1683,7 @@ static void of_register_spi_devices(struct spi_controller *ctlr)
 		spi = of_register_spi_device(ctlr, nc);
 		if (IS_ERR(spi)) {
 			dev_warn(&ctlr->dev,
-				 "Failed to create SPI device for %s\n",
-				 nc->full_name);
+				 "Failed to create SPI device for %pOF\n", nc);
 			of_node_clear_flag(nc, OF_POPULATED);
 		}
 	}
@@ -1693,6 +1693,35 @@ static void of_register_spi_devices(struct spi_controller *ctlr) { }
 #endif
 
 #ifdef CONFIG_ACPI
+static void acpi_spi_parse_apple_properties(struct spi_device *spi)
+{
+	struct acpi_device *dev = ACPI_COMPANION(&spi->dev);
+	const union acpi_object *obj;
+
+	if (!x86_apple_machine)
+		return;
+
+	if (!acpi_dev_get_property(dev, "spiSclkPeriod", ACPI_TYPE_BUFFER, &obj)
+	    && obj->buffer.length >= 4)
+		spi->max_speed_hz  = NSEC_PER_SEC / *(u32 *)obj->buffer.pointer;
+
+	if (!acpi_dev_get_property(dev, "spiWordSize", ACPI_TYPE_BUFFER, &obj)
+	    && obj->buffer.length == 8)
+		spi->bits_per_word = *(u64 *)obj->buffer.pointer;
+
+	if (!acpi_dev_get_property(dev, "spiBitOrder", ACPI_TYPE_BUFFER, &obj)
+	    && obj->buffer.length == 8 && !*(u64 *)obj->buffer.pointer)
+		spi->mode |= SPI_LSB_FIRST;
+
+	if (!acpi_dev_get_property(dev, "spiSPO", ACPI_TYPE_BUFFER, &obj)
+	    && obj->buffer.length == 8 &&  *(u64 *)obj->buffer.pointer)
+		spi->mode |= SPI_CPOL;
+
+	if (!acpi_dev_get_property(dev, "spiSPH", ACPI_TYPE_BUFFER, &obj)
+	    && obj->buffer.length == 8 &&  *(u64 *)obj->buffer.pointer)
+		spi->mode |= SPI_CPHA;
+}
+
 static int acpi_spi_add_resource(struct acpi_resource *ares, void *data)
 {
 	struct spi_device *spi = data;
@@ -1766,6 +1795,8 @@ static acpi_status acpi_register_spi_device(struct spi_controller *ctlr,
 				     acpi_spi_add_resource, spi);
 	acpi_dev_free_resource_list(&resource_list);
 
+	acpi_spi_parse_apple_properties(spi);
+
 	if (ret < 0 || !spi->max_speed_hz) {
 		spi_dev_put(spi);
 		return AE_OK;
@@ -2052,11 +2083,10 @@ static int of_spi_register_master(struct spi_controller *ctlr)
  */
 int spi_register_controller(struct spi_controller *ctlr)
 {
-	static atomic_t		dyn_bus_id = ATOMIC_INIT((1<<15) - 1);
 	struct device		*dev = ctlr->dev.parent;
 	struct boardinfo	*bi;
 	int			status = -ENODEV;
-	int			dynamic = 0;
+	int			id;
 
 	if (!dev)
 		return -ENODEV;
@@ -2072,19 +2102,28 @@ int spi_register_controller(struct spi_controller *ctlr)
 	 */
 	if (ctlr->num_chipselect == 0)
 		return -EINVAL;
-
-	if ((ctlr->bus_num < 0) && ctlr->dev.of_node)
-		ctlr->bus_num = of_alias_get_id(ctlr->dev.of_node, "spi");
-
-	/* convention:  dynamically assigned bus IDs count down from the max */
+	/* allocate dynamic bus number using Linux idr */
+	if ((ctlr->bus_num < 0) && ctlr->dev.of_node) {
+		id = of_alias_get_id(ctlr->dev.of_node, "spi");
+		if (id >= 0) {
+			ctlr->bus_num = id;
+			mutex_lock(&board_lock);
+			id = idr_alloc(&spi_master_idr, ctlr, ctlr->bus_num,
+				       ctlr->bus_num + 1, GFP_KERNEL);
+			mutex_unlock(&board_lock);
+			if (WARN(id < 0, "couldn't get idr"))
+				return id == -ENOSPC ? -EBUSY : id;
+		}
+	}
 	if (ctlr->bus_num < 0) {
-		/* FIXME switch to an IDR based scheme, something like
-		 * I2C now uses, so we can't run out of "dynamic" IDs
-		 */
-		ctlr->bus_num = atomic_dec_return(&dyn_bus_id);
-		dynamic = 1;
+		mutex_lock(&board_lock);
+		id = idr_alloc(&spi_master_idr, ctlr, SPI_DYN_FIRST_BUS_NUM, 0,
+			       GFP_KERNEL);
+		mutex_unlock(&board_lock);
+		if (WARN(id < 0, "couldn't get idr"))
+			return id;
+		ctlr->bus_num = id;
 	}
-
 	INIT_LIST_HEAD(&ctlr->queue);
 	spin_lock_init(&ctlr->queue_lock);
 	spin_lock_init(&ctlr->bus_lock_spinlock);
@@ -2100,11 +2139,16 @@ int spi_register_controller(struct spi_controller *ctlr)
 	 */
 	dev_set_name(&ctlr->dev, "spi%u", ctlr->bus_num);
 	status = device_add(&ctlr->dev);
-	if (status < 0)
+	if (status < 0) {
+		/* free bus id */
+		mutex_lock(&board_lock);
+		idr_remove(&spi_master_idr, ctlr->bus_num);
+		mutex_unlock(&board_lock);
 		goto done;
-	dev_dbg(dev, "registered %s %s%s\n",
+	}
+	dev_dbg(dev, "registered %s %s\n",
 			spi_controller_is_slave(ctlr) ? "slave" : "master",
-			dev_name(&ctlr->dev), dynamic ? " (dynamic)" : "");
+			dev_name(&ctlr->dev));
 
 	/* If we're using a queued driver, start the queue */
 	if (ctlr->transfer)
@@ -2113,6 +2157,10 @@ int spi_register_controller(struct spi_controller *ctlr)
 		status = spi_controller_initialize_queue(ctlr);
 		if (status) {
 			device_del(&ctlr->dev);
+			/* free bus id */
+			mutex_lock(&board_lock);
+			idr_remove(&spi_master_idr, ctlr->bus_num);
+			mutex_unlock(&board_lock);
 			goto done;
 		}
 	}
@@ -2191,19 +2239,33 @@ static int __unregister(struct device *dev, void *null)
  */
 void spi_unregister_controller(struct spi_controller *ctlr)
 {
+	struct spi_controller *found;
 	int dummy;
 
+	/* First make sure that this controller was ever added */
+	mutex_lock(&board_lock);
+	found = idr_find(&spi_master_idr, ctlr->bus_num);
+	mutex_unlock(&board_lock);
+	if (found != ctlr) {
+		dev_dbg(&ctlr->dev,
+			"attempting to delete unregistered controller [%s]\n",
+			dev_name(&ctlr->dev));
+		return;
+	}
 	if (ctlr->queued) {
 		if (spi_destroy_queue(ctlr))
 			dev_err(&ctlr->dev, "queue remove failed\n");
 	}
-
 	mutex_lock(&board_lock);
 	list_del(&ctlr->list);
 	mutex_unlock(&board_lock);
 
 	dummy = device_for_each_child(&ctlr->dev, NULL, __unregister);
 	device_unregister(&ctlr->dev);
+	/* free bus id */
+	mutex_lock(&board_lock);
+	idr_remove(&spi_master_idr, ctlr->bus_num);
+	mutex_unlock(&board_lock);
 }
 EXPORT_SYMBOL_GPL(spi_unregister_controller);
 
@@ -3311,8 +3373,8 @@ static int of_spi_notify(struct notifier_block *nb, unsigned long action,
 		put_device(&ctlr->dev);
 
 		if (IS_ERR(spi)) {
-			pr_err("%s: failed to create for '%s'\n",
-					__func__, rd->dn->full_name);
+			pr_err("%s: failed to create for '%pOF'\n",
+					__func__, rd->dn);
 			of_node_clear_flag(rd->dn, OF_POPULATED);
 			return notifier_from_errno(PTR_ERR(spi));
 		}
diff --git a/drivers/spmi/spmi-pmic-arb.c b/drivers/spmi/spmi-pmic-arb.c
index f4b7a98a7913..360b8218f322 100644
--- a/drivers/spmi/spmi-pmic-arb.c
+++ b/drivers/spmi/spmi-pmic-arb.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012-2015, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2012-2015, 2017, The Linux Foundation. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 and
@@ -29,6 +29,7 @@
 #define PMIC_ARB_VERSION		0x0000
 #define PMIC_ARB_VERSION_V2_MIN		0x20010000
 #define PMIC_ARB_VERSION_V3_MIN		0x30000000
+#define PMIC_ARB_VERSION_V5_MIN		0x50000000
 #define PMIC_ARB_INT_EN			0x0004
 
 /* PMIC Arbiter channel registers offsets */
@@ -39,7 +40,6 @@
 #define PMIC_ARB_WDATA1			0x14
 #define PMIC_ARB_RDATA0			0x18
 #define PMIC_ARB_RDATA1			0x1C
-#define PMIC_ARB_REG_CHNL(N)		(0x800 + 0x4 * (N))
 
 /* Mapping Table */
 #define SPMI_MAPPING_TABLE_REG(N)	(0x0B00 + (4 * (N)))
@@ -51,7 +51,9 @@
 
 #define SPMI_MAPPING_TABLE_TREE_DEPTH	16	/* Maximum of 16-bits */
 #define PMIC_ARB_MAX_PPID		BIT(12) /* PPID is 12bit */
-#define PMIC_ARB_CHAN_VALID		BIT(15)
+#define PMIC_ARB_APID_VALID		BIT(15)
+#define PMIC_ARB_CHAN_IS_IRQ_OWNER(reg)	((reg) & BIT(24))
+#define INVALID_EE				0xFF
 
 /* Ownership Table */
 #define SPMI_OWNERSHIP_TABLE_REG(N)	(0x0700 + (4 * (N)))
@@ -86,6 +88,15 @@ enum pmic_arb_cmd_op_code {
 	PMIC_ARB_OP_ZERO_WRITE = 16,
 };
 
+/*
+ * PMIC arbiter version 5 uses different register offsets for read/write vs
+ * observer channels.
+ */
+enum pmic_arb_channel {
+	PMIC_ARB_CHANNEL_RW,
+	PMIC_ARB_CHANNEL_OBS,
+};
+
 /* Maximum number of support PMIC peripherals */
 #define PMIC_ARB_MAX_PERIPHS		512
 #define PMIC_ARB_TIMEOUT_US		100
@@ -97,22 +108,23 @@ enum pmic_arb_cmd_op_code {
 /* interrupt enable bit */
 #define SPMI_PIC_ACC_ENABLE_BIT		BIT(0)
 
-#define HWIRQ(slave_id, periph_id, irq_id, apid) \
+#define spec_to_hwirq(slave_id, periph_id, irq_id, apid) \
 	((((slave_id) & 0xF)   << 28) | \
 	(((periph_id) & 0xFF)  << 20) | \
 	(((irq_id)    & 0x7)   << 16) | \
 	(((apid)      & 0x1FF) << 0))
 
-#define HWIRQ_SID(hwirq)  (((hwirq) >> 28) & 0xF)
-#define HWIRQ_PER(hwirq)  (((hwirq) >> 20) & 0xFF)
-#define HWIRQ_IRQ(hwirq)  (((hwirq) >> 16) & 0x7)
-#define HWIRQ_APID(hwirq) (((hwirq) >> 0)  & 0x1FF)
+#define hwirq_to_sid(hwirq)  (((hwirq) >> 28) & 0xF)
+#define hwirq_to_per(hwirq)  (((hwirq) >> 20) & 0xFF)
+#define hwirq_to_irq(hwirq)  (((hwirq) >> 16) & 0x7)
+#define hwirq_to_apid(hwirq) (((hwirq) >> 0)  & 0x1FF)
 
 struct pmic_arb_ver_ops;
 
 struct apid_data {
 	u16		ppid;
-	u8		owner;
+	u8		write_ee;
+	u8		irq_ee;
 };
 
 /**
@@ -128,12 +140,11 @@ struct apid_data {
  * @ee:			the current Execution Environment
  * @min_apid:		minimum APID (used for bounding IRQ search)
  * @max_apid:		maximum APID
- * @max_periph:		maximum number of PMIC peripherals supported by HW.
  * @mapping_table:	in-memory copy of PPID -> APID mapping table.
  * @domain:		irq domain object for PMIC IRQ domain
  * @spmic:		SPMI controller object
  * @ver_ops:		version dependent operations.
- * @ppid_to_apid	in-memory copy of PPID -> channel (APID) mapping table.
+ * @ppid_to_apid	in-memory copy of PPID -> APID mapping table.
  */
 struct spmi_pmic_arb {
 	void __iomem		*rd_base;
@@ -148,7 +159,6 @@ struct spmi_pmic_arb {
 	u8			ee;
 	u16			min_apid;
 	u16			max_apid;
-	u16			max_periph;
 	u32			*mapping_table;
 	DECLARE_BITMAP(mapping_table_valid, PMIC_ARB_MAX_PERIPHS);
 	struct irq_domain	*domain;
@@ -164,92 +174,94 @@ struct spmi_pmic_arb {
  *
  * @ver_str:		version string.
  * @ppid_to_apid:	finds the apid for a given ppid.
- * @mode:		access rights to specified pmic peripheral.
  * @non_data_cmd:	on v1 issues an spmi non-data command.
  *			on v2 no HW support, returns -EOPNOTSUPP.
  * @offset:		on v1 offset of per-ee channel.
  *			on v2 offset of per-ee and per-ppid channel.
  * @fmt_cmd:		formats a GENI/SPMI command.
- * @owner_acc_status:	on v1 offset of PMIC_ARB_SPMI_PIC_OWNERm_ACC_STATUSn
- *			on v2 offset of SPMI_PIC_OWNERm_ACC_STATUSn.
- * @acc_enable:		on v1 offset of PMIC_ARB_SPMI_PIC_ACC_ENABLEn
- *			on v2 offset of SPMI_PIC_ACC_ENABLEn.
- * @irq_status:		on v1 offset of PMIC_ARB_SPMI_PIC_IRQ_STATUSn
- *			on v2 offset of SPMI_PIC_IRQ_STATUSn.
- * @irq_clear:		on v1 offset of PMIC_ARB_SPMI_PIC_IRQ_CLEARn
- *			on v2 offset of SPMI_PIC_IRQ_CLEARn.
+ * @owner_acc_status:	on v1 address of PMIC_ARB_SPMI_PIC_OWNERm_ACC_STATUSn
+ *			on v2 address of SPMI_PIC_OWNERm_ACC_STATUSn.
+ * @acc_enable:		on v1 address of PMIC_ARB_SPMI_PIC_ACC_ENABLEn
+ *			on v2 address of SPMI_PIC_ACC_ENABLEn.
+ * @irq_status:		on v1 address of PMIC_ARB_SPMI_PIC_IRQ_STATUSn
+ *			on v2 address of SPMI_PIC_IRQ_STATUSn.
+ * @irq_clear:		on v1 address of PMIC_ARB_SPMI_PIC_IRQ_CLEARn
+ *			on v2 address of SPMI_PIC_IRQ_CLEARn.
+ * @apid_map_offset:	offset of PMIC_ARB_REG_CHNLn
  */
 struct pmic_arb_ver_ops {
 	const char *ver_str;
-	int (*ppid_to_apid)(struct spmi_pmic_arb *pa, u8 sid, u16 addr,
-			u16 *apid);
-	int (*mode)(struct spmi_pmic_arb *dev, u8 sid, u16 addr,
-			mode_t *mode);
+	int (*ppid_to_apid)(struct spmi_pmic_arb *pmic_arb, u16 ppid);
 	/* spmi commands (read_cmd, write_cmd, cmd) functionality */
-	int (*offset)(struct spmi_pmic_arb *dev, u8 sid, u16 addr,
-		      u32 *offset);
+	int (*offset)(struct spmi_pmic_arb *pmic_arb, u8 sid, u16 addr,
+			enum pmic_arb_channel ch_type);
 	u32 (*fmt_cmd)(u8 opc, u8 sid, u16 addr, u8 bc);
 	int (*non_data_cmd)(struct spmi_controller *ctrl, u8 opc, u8 sid);
 	/* Interrupts controller functionality (offset of PIC registers) */
-	u32 (*owner_acc_status)(u8 m, u16 n);
-	u32 (*acc_enable)(u16 n);
-	u32 (*irq_status)(u16 n);
-	u32 (*irq_clear)(u16 n);
+	void __iomem *(*owner_acc_status)(struct spmi_pmic_arb *pmic_arb, u8 m,
+					  u16 n);
+	void __iomem *(*acc_enable)(struct spmi_pmic_arb *pmic_arb, u16 n);
+	void __iomem *(*irq_status)(struct spmi_pmic_arb *pmic_arb, u16 n);
+	void __iomem *(*irq_clear)(struct spmi_pmic_arb *pmic_arb, u16 n);
+	u32 (*apid_map_offset)(u16 n);
 };
 
-static inline void pmic_arb_base_write(struct spmi_pmic_arb *pa,
+static inline void pmic_arb_base_write(struct spmi_pmic_arb *pmic_arb,
 				       u32 offset, u32 val)
 {
-	writel_relaxed(val, pa->wr_base + offset);
+	writel_relaxed(val, pmic_arb->wr_base + offset);
 }
 
-static inline void pmic_arb_set_rd_cmd(struct spmi_pmic_arb *pa,
+static inline void pmic_arb_set_rd_cmd(struct spmi_pmic_arb *pmic_arb,
 				       u32 offset, u32 val)
 {
-	writel_relaxed(val, pa->rd_base + offset);
+	writel_relaxed(val, pmic_arb->rd_base + offset);
 }
 
 /**
- * pa_read_data: reads pmic-arb's register and copy 1..4 bytes to buf
+ * pmic_arb_read_data: reads pmic-arb's register and copy 1..4 bytes to buf
  * @bc:		byte count -1. range: 0..3
  * @reg:	register's address
  * @buf:	output parameter, length must be bc + 1
  */
-static void pa_read_data(struct spmi_pmic_arb *pa, u8 *buf, u32 reg, u8 bc)
+static void
+pmic_arb_read_data(struct spmi_pmic_arb *pmic_arb, u8 *buf, u32 reg, u8 bc)
 {
-	u32 data = __raw_readl(pa->rd_base + reg);
+	u32 data = __raw_readl(pmic_arb->rd_base + reg);
 
 	memcpy(buf, &data, (bc & 3) + 1);
 }
 
 /**
- * pa_write_data: write 1..4 bytes from buf to pmic-arb's register
+ * pmic_arb_write_data: write 1..4 bytes from buf to pmic-arb's register
  * @bc:		byte-count -1. range: 0..3.
  * @reg:	register's address.
  * @buf:	buffer to write. length must be bc + 1.
  */
-static void
-pa_write_data(struct spmi_pmic_arb *pa, const u8 *buf, u32 reg, u8 bc)
+static void pmic_arb_write_data(struct spmi_pmic_arb *pmic_arb, const u8 *buf,
+				u32 reg, u8 bc)
 {
 	u32 data = 0;
 
 	memcpy(&data, buf, (bc & 3) + 1);
-	pmic_arb_base_write(pa, reg, data);
+	__raw_writel(data, pmic_arb->wr_base + reg);
 }
 
 static int pmic_arb_wait_for_done(struct spmi_controller *ctrl,
-				  void __iomem *base, u8 sid, u16 addr)
+				  void __iomem *base, u8 sid, u16 addr,
+				  enum pmic_arb_channel ch_type)
 {
-	struct spmi_pmic_arb *pa = spmi_controller_get_drvdata(ctrl);
+	struct spmi_pmic_arb *pmic_arb = spmi_controller_get_drvdata(ctrl);
 	u32 status = 0;
 	u32 timeout = PMIC_ARB_TIMEOUT_US;
 	u32 offset;
 	int rc;
 
-	rc = pa->ver_ops->offset(pa, sid, addr, &offset);
-	if (rc)
+	rc = pmic_arb->ver_ops->offset(pmic_arb, sid, addr, ch_type);
+	if (rc < 0)
 		return rc;
 
+	offset = rc;
 	offset += PMIC_ARB_STATUS;
 
 	while (timeout--) {
@@ -257,22 +269,19 @@ static int pmic_arb_wait_for_done(struct spmi_controller *ctrl,
 
 		if (status & PMIC_ARB_STATUS_DONE) {
 			if (status & PMIC_ARB_STATUS_DENIED) {
-				dev_err(&ctrl->dev,
-					"%s: transaction denied (0x%x)\n",
+				dev_err(&ctrl->dev, "%s: transaction denied (0x%x)\n",
 					__func__, status);
 				return -EPERM;
 			}
 
 			if (status & PMIC_ARB_STATUS_FAILURE) {
-				dev_err(&ctrl->dev,
-					"%s: transaction failed (0x%x)\n",
+				dev_err(&ctrl->dev, "%s: transaction failed (0x%x)\n",
 					__func__, status);
 				return -EIO;
 			}
 
 			if (status & PMIC_ARB_STATUS_DROPPED) {
-				dev_err(&ctrl->dev,
-					"%s: transaction dropped (0x%x)\n",
+				dev_err(&ctrl->dev, "%s: transaction dropped (0x%x)\n",
 					__func__, status);
 				return -EIO;
 			}
@@ -282,8 +291,7 @@ static int pmic_arb_wait_for_done(struct spmi_controller *ctrl,
 		udelay(1);
 	}
 
-	dev_err(&ctrl->dev,
-		"%s: timeout, status 0x%x\n",
+	dev_err(&ctrl->dev, "%s: timeout, status 0x%x\n",
 		__func__, status);
 	return -ETIMEDOUT;
 }
@@ -291,22 +299,24 @@ static int pmic_arb_wait_for_done(struct spmi_controller *ctrl,
 static int
 pmic_arb_non_data_cmd_v1(struct spmi_controller *ctrl, u8 opc, u8 sid)
 {
-	struct spmi_pmic_arb *pa = spmi_controller_get_drvdata(ctrl);
+	struct spmi_pmic_arb *pmic_arb = spmi_controller_get_drvdata(ctrl);
 	unsigned long flags;
 	u32 cmd;
 	int rc;
 	u32 offset;
 
-	rc = pa->ver_ops->offset(pa, sid, 0, &offset);
-	if (rc)
+	rc = pmic_arb->ver_ops->offset(pmic_arb, sid, 0, PMIC_ARB_CHANNEL_RW);
+	if (rc < 0)
 		return rc;
 
+	offset = rc;
 	cmd = ((opc | 0x40) << 27) | ((sid & 0xf) << 20);
 
-	raw_spin_lock_irqsave(&pa->lock, flags);
-	pmic_arb_base_write(pa, offset + PMIC_ARB_CMD, cmd);
-	rc = pmic_arb_wait_for_done(ctrl, pa->wr_base, sid, 0);
-	raw_spin_unlock_irqrestore(&pa->lock, flags);
+	raw_spin_lock_irqsave(&pmic_arb->lock, flags);
+	pmic_arb_base_write(pmic_arb, offset + PMIC_ARB_CMD, cmd);
+	rc = pmic_arb_wait_for_done(ctrl, pmic_arb->wr_base, sid, 0,
+				    PMIC_ARB_CHANNEL_RW);
+	raw_spin_unlock_irqrestore(&pmic_arb->lock, flags);
 
 	return rc;
 }
@@ -320,7 +330,7 @@ pmic_arb_non_data_cmd_v2(struct spmi_controller *ctrl, u8 opc, u8 sid)
 /* Non-data command */
 static int pmic_arb_cmd(struct spmi_controller *ctrl, u8 opc, u8 sid)
 {
-	struct spmi_pmic_arb *pa = spmi_controller_get_drvdata(ctrl);
+	struct spmi_pmic_arb *pmic_arb = spmi_controller_get_drvdata(ctrl);
 
 	dev_dbg(&ctrl->dev, "cmd op:0x%x sid:%d\n", opc, sid);
 
@@ -328,38 +338,27 @@ static int pmic_arb_cmd(struct spmi_controller *ctrl, u8 opc, u8 sid)
 	if (opc < SPMI_CMD_RESET || opc > SPMI_CMD_WAKEUP)
 		return -EINVAL;
 
-	return pa->ver_ops->non_data_cmd(ctrl, opc, sid);
+	return pmic_arb->ver_ops->non_data_cmd(ctrl, opc, sid);
 }
 
 static int pmic_arb_read_cmd(struct spmi_controller *ctrl, u8 opc, u8 sid,
 			     u16 addr, u8 *buf, size_t len)
 {
-	struct spmi_pmic_arb *pa = spmi_controller_get_drvdata(ctrl);
+	struct spmi_pmic_arb *pmic_arb = spmi_controller_get_drvdata(ctrl);
 	unsigned long flags;
 	u8 bc = len - 1;
 	u32 cmd;
 	int rc;
 	u32 offset;
-	mode_t mode;
-
-	rc = pa->ver_ops->offset(pa, sid, addr, &offset);
-	if (rc)
-		return rc;
 
-	rc = pa->ver_ops->mode(pa, sid, addr, &mode);
-	if (rc)
+	rc = pmic_arb->ver_ops->offset(pmic_arb, sid, addr,
+				       PMIC_ARB_CHANNEL_OBS);
+	if (rc < 0)
 		return rc;
 
-	if (!(mode & S_IRUSR)) {
-		dev_err(&pa->spmic->dev,
-			"error: impermissible read from peripheral sid:%d addr:0x%x\n",
-			sid, addr);
-		return -EPERM;
-	}
-
+	offset = rc;
 	if (bc >= PMIC_ARB_MAX_TRANS_BYTES) {
-		dev_err(&ctrl->dev,
-			"pmic-arb supports 1..%d bytes per trans, but:%zu requested",
+		dev_err(&ctrl->dev, "pmic-arb supports 1..%d bytes per trans, but:%zu requested",
 			PMIC_ARB_MAX_TRANS_BYTES, len);
 		return  -EINVAL;
 	}
@@ -374,54 +373,45 @@ static int pmic_arb_read_cmd(struct spmi_controller *ctrl, u8 opc, u8 sid,
 	else
 		return -EINVAL;
 
-	cmd = pa->ver_ops->fmt_cmd(opc, sid, addr, bc);
+	cmd = pmic_arb->ver_ops->fmt_cmd(opc, sid, addr, bc);
 
-	raw_spin_lock_irqsave(&pa->lock, flags);
-	pmic_arb_set_rd_cmd(pa, offset + PMIC_ARB_CMD, cmd);
-	rc = pmic_arb_wait_for_done(ctrl, pa->rd_base, sid, addr);
+	raw_spin_lock_irqsave(&pmic_arb->lock, flags);
+	pmic_arb_set_rd_cmd(pmic_arb, offset + PMIC_ARB_CMD, cmd);
+	rc = pmic_arb_wait_for_done(ctrl, pmic_arb->rd_base, sid, addr,
+				    PMIC_ARB_CHANNEL_OBS);
 	if (rc)
 		goto done;
 
-	pa_read_data(pa, buf, offset + PMIC_ARB_RDATA0,
+	pmic_arb_read_data(pmic_arb, buf, offset + PMIC_ARB_RDATA0,
 		     min_t(u8, bc, 3));
 
 	if (bc > 3)
-		pa_read_data(pa, buf + 4, offset + PMIC_ARB_RDATA1, bc - 4);
+		pmic_arb_read_data(pmic_arb, buf + 4, offset + PMIC_ARB_RDATA1,
+					bc - 4);
 
 done:
-	raw_spin_unlock_irqrestore(&pa->lock, flags);
+	raw_spin_unlock_irqrestore(&pmic_arb->lock, flags);
 	return rc;
 }
 
 static int pmic_arb_write_cmd(struct spmi_controller *ctrl, u8 opc, u8 sid,
-			      u16 addr, const u8 *buf, size_t len)
+			u16 addr, const u8 *buf, size_t len)
 {
-	struct spmi_pmic_arb *pa = spmi_controller_get_drvdata(ctrl);
+	struct spmi_pmic_arb *pmic_arb = spmi_controller_get_drvdata(ctrl);
 	unsigned long flags;
 	u8 bc = len - 1;
 	u32 cmd;
 	int rc;
 	u32 offset;
-	mode_t mode;
-
-	rc = pa->ver_ops->offset(pa, sid, addr, &offset);
-	if (rc)
-		return rc;
 
-	rc = pa->ver_ops->mode(pa, sid, addr, &mode);
-	if (rc)
+	rc = pmic_arb->ver_ops->offset(pmic_arb, sid, addr,
+					PMIC_ARB_CHANNEL_RW);
+	if (rc < 0)
 		return rc;
 
-	if (!(mode & S_IWUSR)) {
-		dev_err(&pa->spmic->dev,
-			"error: impermissible write to peripheral sid:%d addr:0x%x\n",
-			sid, addr);
-		return -EPERM;
-	}
-
+	offset = rc;
 	if (bc >= PMIC_ARB_MAX_TRANS_BYTES) {
-		dev_err(&ctrl->dev,
-			"pmic-arb supports 1..%d bytes per trans, but:%zu requested",
+		dev_err(&ctrl->dev, "pmic-arb supports 1..%d bytes per trans, but:%zu requested",
 			PMIC_ARB_MAX_TRANS_BYTES, len);
 		return  -EINVAL;
 	}
@@ -429,7 +419,7 @@ static int pmic_arb_write_cmd(struct spmi_controller *ctrl, u8 opc, u8 sid,
 	/* Check the opcode */
 	if (opc >= 0x40 && opc <= 0x5F)
 		opc = PMIC_ARB_OP_WRITE;
-	else if (opc >= 0x00 && opc <= 0x0F)
+	else if (opc <= 0x0F)
 		opc = PMIC_ARB_OP_EXT_WRITE;
 	else if (opc >= 0x30 && opc <= 0x37)
 		opc = PMIC_ARB_OP_EXT_WRITEL;
@@ -438,18 +428,21 @@ static int pmic_arb_write_cmd(struct spmi_controller *ctrl, u8 opc, u8 sid,
 	else
 		return -EINVAL;
 
-	cmd = pa->ver_ops->fmt_cmd(opc, sid, addr, bc);
+	cmd = pmic_arb->ver_ops->fmt_cmd(opc, sid, addr, bc);
 
 	/* Write data to FIFOs */
-	raw_spin_lock_irqsave(&pa->lock, flags);
-	pa_write_data(pa, buf, offset + PMIC_ARB_WDATA0, min_t(u8, bc, 3));
+	raw_spin_lock_irqsave(&pmic_arb->lock, flags);
+	pmic_arb_write_data(pmic_arb, buf, offset + PMIC_ARB_WDATA0,
+				min_t(u8, bc, 3));
 	if (bc > 3)
-		pa_write_data(pa, buf + 4, offset + PMIC_ARB_WDATA1, bc - 4);
+		pmic_arb_write_data(pmic_arb, buf + 4, offset + PMIC_ARB_WDATA1,
+					bc - 4);
 
 	/* Start the transaction */
-	pmic_arb_base_write(pa, offset + PMIC_ARB_CMD, cmd);
-	rc = pmic_arb_wait_for_done(ctrl, pa->wr_base, sid, addr);
-	raw_spin_unlock_irqrestore(&pa->lock, flags);
+	pmic_arb_base_write(pmic_arb, offset + PMIC_ARB_CMD, cmd);
+	rc = pmic_arb_wait_for_done(ctrl, pmic_arb->wr_base, sid, addr,
+				    PMIC_ARB_CHANNEL_RW);
+	raw_spin_unlock_irqrestore(&pmic_arb->lock, flags);
 
 	return rc;
 }
@@ -475,67 +468,64 @@ struct spmi_pmic_arb_qpnpint_type {
 static void qpnpint_spmi_write(struct irq_data *d, u8 reg, void *buf,
 			       size_t len)
 {
-	struct spmi_pmic_arb *pa = irq_data_get_irq_chip_data(d);
-	u8 sid = HWIRQ_SID(d->hwirq);
-	u8 per = HWIRQ_PER(d->hwirq);
+	struct spmi_pmic_arb *pmic_arb = irq_data_get_irq_chip_data(d);
+	u8 sid = hwirq_to_sid(d->hwirq);
+	u8 per = hwirq_to_per(d->hwirq);
 
-	if (pmic_arb_write_cmd(pa->spmic, SPMI_CMD_EXT_WRITEL, sid,
+	if (pmic_arb_write_cmd(pmic_arb->spmic, SPMI_CMD_EXT_WRITEL, sid,
 			       (per << 8) + reg, buf, len))
-		dev_err_ratelimited(&pa->spmic->dev,
-				"failed irqchip transaction on %x\n",
+		dev_err_ratelimited(&pmic_arb->spmic->dev, "failed irqchip transaction on %x\n",
 				    d->irq);
 }
 
 static void qpnpint_spmi_read(struct irq_data *d, u8 reg, void *buf, size_t len)
 {
-	struct spmi_pmic_arb *pa = irq_data_get_irq_chip_data(d);
-	u8 sid = HWIRQ_SID(d->hwirq);
-	u8 per = HWIRQ_PER(d->hwirq);
+	struct spmi_pmic_arb *pmic_arb = irq_data_get_irq_chip_data(d);
+	u8 sid = hwirq_to_sid(d->hwirq);
+	u8 per = hwirq_to_per(d->hwirq);
 
-	if (pmic_arb_read_cmd(pa->spmic, SPMI_CMD_EXT_READL, sid,
+	if (pmic_arb_read_cmd(pmic_arb->spmic, SPMI_CMD_EXT_READL, sid,
 			      (per << 8) + reg, buf, len))
-		dev_err_ratelimited(&pa->spmic->dev,
-				"failed irqchip transaction on %x\n",
+		dev_err_ratelimited(&pmic_arb->spmic->dev, "failed irqchip transaction on %x\n",
 				    d->irq);
 }
 
-static void cleanup_irq(struct spmi_pmic_arb *pa, u16 apid, int id)
+static void cleanup_irq(struct spmi_pmic_arb *pmic_arb, u16 apid, int id)
 {
-	u16 ppid = pa->apid_data[apid].ppid;
+	u16 ppid = pmic_arb->apid_data[apid].ppid;
 	u8 sid = ppid >> 8;
 	u8 per = ppid & 0xFF;
 	u8 irq_mask = BIT(id);
 
-	writel_relaxed(irq_mask, pa->intr + pa->ver_ops->irq_clear(apid));
+	writel_relaxed(irq_mask, pmic_arb->ver_ops->irq_clear(pmic_arb, apid));
 
-	if (pmic_arb_write_cmd(pa->spmic, SPMI_CMD_EXT_WRITEL, sid,
+	if (pmic_arb_write_cmd(pmic_arb->spmic, SPMI_CMD_EXT_WRITEL, sid,
 			(per << 8) + QPNPINT_REG_LATCHED_CLR, &irq_mask, 1))
-		dev_err_ratelimited(&pa->spmic->dev,
-				"failed to ack irq_mask = 0x%x for ppid = %x\n",
+		dev_err_ratelimited(&pmic_arb->spmic->dev, "failed to ack irq_mask = 0x%x for ppid = %x\n",
 				irq_mask, ppid);
 
-	if (pmic_arb_write_cmd(pa->spmic, SPMI_CMD_EXT_WRITEL, sid,
+	if (pmic_arb_write_cmd(pmic_arb->spmic, SPMI_CMD_EXT_WRITEL, sid,
 			       (per << 8) + QPNPINT_REG_EN_CLR, &irq_mask, 1))
-		dev_err_ratelimited(&pa->spmic->dev,
-				"failed to ack irq_mask = 0x%x for ppid = %x\n",
+		dev_err_ratelimited(&pmic_arb->spmic->dev, "failed to ack irq_mask = 0x%x for ppid = %x\n",
 				irq_mask, ppid);
 }
 
-static void periph_interrupt(struct spmi_pmic_arb *pa, u16 apid)
+static void periph_interrupt(struct spmi_pmic_arb *pmic_arb, u16 apid)
 {
 	unsigned int irq;
 	u32 status;
 	int id;
-	u8 sid = (pa->apid_data[apid].ppid >> 8) & 0xF;
-	u8 per = pa->apid_data[apid].ppid & 0xFF;
+	u8 sid = (pmic_arb->apid_data[apid].ppid >> 8) & 0xF;
+	u8 per = pmic_arb->apid_data[apid].ppid & 0xFF;
 
-	status = readl_relaxed(pa->intr + pa->ver_ops->irq_status(apid));
+	status = readl_relaxed(pmic_arb->ver_ops->irq_status(pmic_arb, apid));
 	while (status) {
 		id = ffs(status) - 1;
 		status &= ~BIT(id);
-		irq = irq_find_mapping(pa->domain, HWIRQ(sid, per, id, apid));
+		irq = irq_find_mapping(pmic_arb->domain,
+					spec_to_hwirq(sid, per, id, apid));
 		if (irq == 0) {
-			cleanup_irq(pa, apid, id);
+			cleanup_irq(pmic_arb, apid, id);
 			continue;
 		}
 		generic_handle_irq(irq);
@@ -544,27 +534,28 @@ static void periph_interrupt(struct spmi_pmic_arb *pa, u16 apid)
 
 static void pmic_arb_chained_irq(struct irq_desc *desc)
 {
-	struct spmi_pmic_arb *pa = irq_desc_get_handler_data(desc);
+	struct spmi_pmic_arb *pmic_arb = irq_desc_get_handler_data(desc);
+	const struct pmic_arb_ver_ops *ver_ops = pmic_arb->ver_ops;
 	struct irq_chip *chip = irq_desc_get_chip(desc);
-	void __iomem *intr = pa->intr;
-	int first = pa->min_apid >> 5;
-	int last = pa->max_apid >> 5;
+	int first = pmic_arb->min_apid >> 5;
+	int last = pmic_arb->max_apid >> 5;
+	u8 ee = pmic_arb->ee;
 	u32 status, enable;
 	int i, id, apid;
 
 	chained_irq_enter(chip, desc);
 
 	for (i = first; i <= last; ++i) {
-		status = readl_relaxed(intr +
-				      pa->ver_ops->owner_acc_status(pa->ee, i));
+		status = readl_relaxed(
+				ver_ops->owner_acc_status(pmic_arb, ee, i));
 		while (status) {
 			id = ffs(status) - 1;
 			status &= ~BIT(id);
 			apid = id + i * 32;
-			enable = readl_relaxed(intr +
-					pa->ver_ops->acc_enable(apid));
+			enable = readl_relaxed(
+					ver_ops->acc_enable(pmic_arb, apid));
 			if (enable & SPMI_PIC_ACC_ENABLE_BIT)
-				periph_interrupt(pa, apid);
+				periph_interrupt(pmic_arb, apid);
 		}
 	}
 
@@ -573,12 +564,12 @@ static void pmic_arb_chained_irq(struct irq_desc *desc)
 
 static void qpnpint_irq_ack(struct irq_data *d)
 {
-	struct spmi_pmic_arb *pa = irq_data_get_irq_chip_data(d);
-	u8 irq = HWIRQ_IRQ(d->hwirq);
-	u16 apid = HWIRQ_APID(d->hwirq);
+	struct spmi_pmic_arb *pmic_arb = irq_data_get_irq_chip_data(d);
+	u8 irq = hwirq_to_irq(d->hwirq);
+	u16 apid = hwirq_to_apid(d->hwirq);
 	u8 data;
 
-	writel_relaxed(BIT(irq), pa->intr + pa->ver_ops->irq_clear(apid));
+	writel_relaxed(BIT(irq), pmic_arb->ver_ops->irq_clear(pmic_arb, apid));
 
 	data = BIT(irq);
 	qpnpint_spmi_write(d, QPNPINT_REG_LATCHED_CLR, &data, 1);
@@ -586,7 +577,7 @@ static void qpnpint_irq_ack(struct irq_data *d)
 
 static void qpnpint_irq_mask(struct irq_data *d)
 {
-	u8 irq = HWIRQ_IRQ(d->hwirq);
+	u8 irq = hwirq_to_irq(d->hwirq);
 	u8 data = BIT(irq);
 
 	qpnpint_spmi_write(d, QPNPINT_REG_EN_CLR, &data, 1);
@@ -594,13 +585,14 @@ static void qpnpint_irq_mask(struct irq_data *d)
 
 static void qpnpint_irq_unmask(struct irq_data *d)
 {
-	struct spmi_pmic_arb *pa = irq_data_get_irq_chip_data(d);
-	u8 irq = HWIRQ_IRQ(d->hwirq);
-	u16 apid = HWIRQ_APID(d->hwirq);
+	struct spmi_pmic_arb *pmic_arb = irq_data_get_irq_chip_data(d);
+	const struct pmic_arb_ver_ops *ver_ops = pmic_arb->ver_ops;
+	u8 irq = hwirq_to_irq(d->hwirq);
+	u16 apid = hwirq_to_apid(d->hwirq);
 	u8 buf[2];
 
 	writel_relaxed(SPMI_PIC_ACC_ENABLE_BIT,
-		pa->intr + pa->ver_ops->acc_enable(apid));
+			ver_ops->acc_enable(pmic_arb, apid));
 
 	qpnpint_spmi_read(d, QPNPINT_REG_EN_SET, &buf[0], 1);
 	if (!(buf[0] & BIT(irq))) {
@@ -618,44 +610,51 @@ static void qpnpint_irq_unmask(struct irq_data *d)
 static int qpnpint_irq_set_type(struct irq_data *d, unsigned int flow_type)
 {
 	struct spmi_pmic_arb_qpnpint_type type;
-	u8 irq = HWIRQ_IRQ(d->hwirq);
-	u8 bit_mask_irq = BIT(irq);
+	irq_flow_handler_t flow_handler;
+	u8 irq = hwirq_to_irq(d->hwirq);
 
 	qpnpint_spmi_read(d, QPNPINT_REG_SET_TYPE, &type, sizeof(type));
 
 	if (flow_type & (IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING)) {
-		type.type |= bit_mask_irq;
+		type.type |= BIT(irq);
 		if (flow_type & IRQF_TRIGGER_RISING)
-			type.polarity_high |= bit_mask_irq;
+			type.polarity_high |= BIT(irq);
 		if (flow_type & IRQF_TRIGGER_FALLING)
-			type.polarity_low  |= bit_mask_irq;
+			type.polarity_low  |= BIT(irq);
+
+		flow_handler = handle_edge_irq;
 	} else {
 		if ((flow_type & (IRQF_TRIGGER_HIGH)) &&
 		    (flow_type & (IRQF_TRIGGER_LOW)))
 			return -EINVAL;
 
-		type.type &= ~bit_mask_irq; /* level trig */
+		type.type &= ~BIT(irq); /* level trig */
 		if (flow_type & IRQF_TRIGGER_HIGH)
-			type.polarity_high |= bit_mask_irq;
+			type.polarity_high |= BIT(irq);
 		else
-			type.polarity_low  |= bit_mask_irq;
+			type.polarity_low  |= BIT(irq);
+
+		flow_handler = handle_level_irq;
 	}
 
 	qpnpint_spmi_write(d, QPNPINT_REG_SET_TYPE, &type, sizeof(type));
-
-	if (flow_type & IRQ_TYPE_EDGE_BOTH)
-		irq_set_handler_locked(d, handle_edge_irq);
-	else
-		irq_set_handler_locked(d, handle_level_irq);
+	irq_set_handler_locked(d, flow_handler);
 
 	return 0;
 }
 
+static int qpnpint_irq_set_wake(struct irq_data *d, unsigned int on)
+{
+	struct spmi_pmic_arb *pmic_arb = irq_data_get_irq_chip_data(d);
+
+	return irq_set_irq_wake(pmic_arb->irq, on);
+}
+
 static int qpnpint_get_irqchip_state(struct irq_data *d,
 				     enum irqchip_irq_state which,
 				     bool *state)
 {
-	u8 irq = HWIRQ_IRQ(d->hwirq);
+	u8 irq = hwirq_to_irq(d->hwirq);
 	u8 status = 0;
 
 	if (which != IRQCHIP_STATE_LINE_LEVEL)
@@ -667,15 +666,34 @@ static int qpnpint_get_irqchip_state(struct irq_data *d,
 	return 0;
 }
 
+static int qpnpint_irq_request_resources(struct irq_data *d)
+{
+	struct spmi_pmic_arb *pmic_arb = irq_data_get_irq_chip_data(d);
+	u16 periph = hwirq_to_per(d->hwirq);
+	u16 apid = hwirq_to_apid(d->hwirq);
+	u16 sid = hwirq_to_sid(d->hwirq);
+	u16 irq = hwirq_to_irq(d->hwirq);
+
+	if (pmic_arb->apid_data[apid].irq_ee != pmic_arb->ee) {
+		dev_err(&pmic_arb->spmic->dev, "failed to xlate sid = %#x, periph = %#x, irq = %u: ee=%u but owner=%u\n",
+			sid, periph, irq, pmic_arb->ee,
+			pmic_arb->apid_data[apid].irq_ee);
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
 static struct irq_chip pmic_arb_irqchip = {
 	.name		= "pmic_arb",
 	.irq_ack	= qpnpint_irq_ack,
 	.irq_mask	= qpnpint_irq_mask,
 	.irq_unmask	= qpnpint_irq_unmask,
 	.irq_set_type	= qpnpint_irq_set_type,
+	.irq_set_wake	= qpnpint_irq_set_wake,
 	.irq_get_irqchip_state	= qpnpint_get_irqchip_state,
-	.flags		= IRQCHIP_MASK_ON_SUSPEND
-			| IRQCHIP_SKIP_SET_WAKE,
+	.irq_request_resources = qpnpint_irq_request_resources,
+	.flags		= IRQCHIP_MASK_ON_SUSPEND,
 };
 
 static int qpnpint_irq_domain_dt_translate(struct irq_domain *d,
@@ -685,12 +703,11 @@ static int qpnpint_irq_domain_dt_translate(struct irq_domain *d,
 					   unsigned long *out_hwirq,
 					   unsigned int *out_type)
 {
-	struct spmi_pmic_arb *pa = d->host_data;
+	struct spmi_pmic_arb *pmic_arb = d->host_data;
+	u16 apid, ppid;
 	int rc;
-	u16 apid;
 
-	dev_dbg(&pa->spmic->dev,
-		"intspec[0] 0x%1x intspec[1] 0x%02x intspec[2] 0x%02x\n",
+	dev_dbg(&pmic_arb->spmic->dev, "intspec[0] 0x%1x intspec[1] 0x%02x intspec[2] 0x%02x\n",
 		intspec[0], intspec[1], intspec[2]);
 
 	if (irq_domain_get_of_node(d) != controller)
@@ -700,25 +717,25 @@ static int qpnpint_irq_domain_dt_translate(struct irq_domain *d,
 	if (intspec[0] > 0xF || intspec[1] > 0xFF || intspec[2] > 0x7)
 		return -EINVAL;
 
-	rc = pa->ver_ops->ppid_to_apid(pa, intspec[0],
-			(intspec[1] << 8), &apid);
+	ppid = intspec[0] << 8 | intspec[1];
+	rc = pmic_arb->ver_ops->ppid_to_apid(pmic_arb, ppid);
 	if (rc < 0) {
-		dev_err(&pa->spmic->dev,
-		"failed to xlate sid = 0x%x, periph = 0x%x, irq = %x rc = %d\n",
+		dev_err(&pmic_arb->spmic->dev, "failed to xlate sid = %#x, periph = %#x, irq = %u rc = %d\n",
 		intspec[0], intspec[1], intspec[2], rc);
 		return rc;
 	}
 
+	apid = rc;
 	/* Keep track of {max,min}_apid for bounding search during interrupt */
-	if (apid > pa->max_apid)
-		pa->max_apid = apid;
-	if (apid < pa->min_apid)
-		pa->min_apid = apid;
+	if (apid > pmic_arb->max_apid)
+		pmic_arb->max_apid = apid;
+	if (apid < pmic_arb->min_apid)
+		pmic_arb->min_apid = apid;
 
-	*out_hwirq = HWIRQ(intspec[0], intspec[1], intspec[2], apid);
+	*out_hwirq = spec_to_hwirq(intspec[0], intspec[1], intspec[2], apid);
 	*out_type  = intspec[3] & IRQ_TYPE_SENSE_MASK;
 
-	dev_dbg(&pa->spmic->dev, "out_hwirq = %lu\n", *out_hwirq);
+	dev_dbg(&pmic_arb->spmic->dev, "out_hwirq = %lu\n", *out_hwirq);
 
 	return 0;
 }
@@ -727,9 +744,9 @@ static int qpnpint_irq_domain_map(struct irq_domain *d,
 				  unsigned int virq,
 				  irq_hw_number_t hwirq)
 {
-	struct spmi_pmic_arb *pa = d->host_data;
+	struct spmi_pmic_arb *pmic_arb = d->host_data;
 
-	dev_dbg(&pa->spmic->dev, "virq = %u, hwirq = %lu\n", virq, hwirq);
+	dev_dbg(&pmic_arb->spmic->dev, "virq = %u, hwirq = %lu\n", virq, hwirq);
 
 	irq_set_chip_and_handler(virq, &pmic_arb_irqchip, handle_level_irq);
 	irq_set_chip_data(virq, d->host_data);
@@ -737,24 +754,23 @@ static int qpnpint_irq_domain_map(struct irq_domain *d,
 	return 0;
 }
 
-static int
-pmic_arb_ppid_to_apid_v1(struct spmi_pmic_arb *pa, u8 sid, u16 addr, u16 *apid)
+static int pmic_arb_ppid_to_apid_v1(struct spmi_pmic_arb *pmic_arb, u16 ppid)
 {
-	u16 ppid = sid << 8 | ((addr >> 8) & 0xFF);
-	u32 *mapping_table = pa->mapping_table;
+	u32 *mapping_table = pmic_arb->mapping_table;
 	int index = 0, i;
 	u16 apid_valid;
+	u16 apid;
 	u32 data;
 
-	apid_valid = pa->ppid_to_apid[ppid];
-	if (apid_valid & PMIC_ARB_CHAN_VALID) {
-		*apid = (apid_valid & ~PMIC_ARB_CHAN_VALID);
-		return 0;
+	apid_valid = pmic_arb->ppid_to_apid[ppid];
+	if (apid_valid & PMIC_ARB_APID_VALID) {
+		apid = apid_valid & ~PMIC_ARB_APID_VALID;
+		return apid;
 	}
 
 	for (i = 0; i < SPMI_MAPPING_TABLE_TREE_DEPTH; ++i) {
-		if (!test_and_set_bit(index, pa->mapping_table_valid))
-			mapping_table[index] = readl_relaxed(pa->cnfg +
+		if (!test_and_set_bit(index, pmic_arb->mapping_table_valid))
+			mapping_table[index] = readl_relaxed(pmic_arb->cnfg +
 						SPMI_MAPPING_TABLE_REG(index));
 
 		data = mapping_table[index];
@@ -763,21 +779,21 @@ pmic_arb_ppid_to_apid_v1(struct spmi_pmic_arb *pa, u8 sid, u16 addr, u16 *apid)
 			if (SPMI_MAPPING_BIT_IS_1_FLAG(data)) {
 				index = SPMI_MAPPING_BIT_IS_1_RESULT(data);
 			} else {
-				*apid = SPMI_MAPPING_BIT_IS_1_RESULT(data);
-				pa->ppid_to_apid[ppid]
-					= *apid | PMIC_ARB_CHAN_VALID;
-				pa->apid_data[*apid].ppid = ppid;
-				return 0;
+				apid = SPMI_MAPPING_BIT_IS_1_RESULT(data);
+				pmic_arb->ppid_to_apid[ppid]
+					= apid | PMIC_ARB_APID_VALID;
+				pmic_arb->apid_data[apid].ppid = ppid;
+				return apid;
 			}
 		} else {
 			if (SPMI_MAPPING_BIT_IS_0_FLAG(data)) {
 				index = SPMI_MAPPING_BIT_IS_0_RESULT(data);
 			} else {
-				*apid = SPMI_MAPPING_BIT_IS_0_RESULT(data);
-				pa->ppid_to_apid[ppid]
-					= *apid | PMIC_ARB_CHAN_VALID;
-				pa->apid_data[*apid].ppid = ppid;
-				return 0;
+				apid = SPMI_MAPPING_BIT_IS_0_RESULT(data);
+				pmic_arb->ppid_to_apid[ppid]
+					= apid | PMIC_ARB_APID_VALID;
+				pmic_arb->apid_data[apid].ppid = ppid;
+				return apid;
 			}
 		}
 	}
@@ -785,105 +801,178 @@ pmic_arb_ppid_to_apid_v1(struct spmi_pmic_arb *pa, u8 sid, u16 addr, u16 *apid)
 	return -ENODEV;
 }
 
-static int
-pmic_arb_mode_v1_v3(struct spmi_pmic_arb *pa, u8 sid, u16 addr, mode_t *mode)
-{
-	*mode = S_IRUSR | S_IWUSR;
-	return 0;
-}
-
 /* v1 offset per ee */
-static int
-pmic_arb_offset_v1(struct spmi_pmic_arb *pa, u8 sid, u16 addr, u32 *offset)
+static int pmic_arb_offset_v1(struct spmi_pmic_arb *pmic_arb, u8 sid, u16 addr,
+			enum pmic_arb_channel ch_type)
 {
-	*offset = 0x800 + 0x80 * pa->channel;
-	return 0;
+	return 0x800 + 0x80 * pmic_arb->channel;
 }
 
-static u16 pmic_arb_find_apid(struct spmi_pmic_arb *pa, u16 ppid)
+static u16 pmic_arb_find_apid(struct spmi_pmic_arb *pmic_arb, u16 ppid)
 {
+	struct apid_data *apidd = &pmic_arb->apid_data[pmic_arb->last_apid];
 	u32 regval, offset;
-	u16 apid;
-	u16 id;
-
-	/*
-	 * PMIC_ARB_REG_CHNL is a table in HW mapping channel to ppid.
-	 * ppid_to_apid is an in-memory invert of that table.
-	 */
-	for (apid = pa->last_apid; apid < pa->max_periph; apid++) {
-		regval = readl_relaxed(pa->cnfg +
-				      SPMI_OWNERSHIP_TABLE_REG(apid));
-		pa->apid_data[apid].owner = SPMI_OWNERSHIP_PERIPH2OWNER(regval);
+	u16 id, apid;
 
-		offset = PMIC_ARB_REG_CHNL(apid);
-		if (offset >= pa->core_size)
+	for (apid = pmic_arb->last_apid; ; apid++, apidd++) {
+		offset = pmic_arb->ver_ops->apid_map_offset(apid);
+		if (offset >= pmic_arb->core_size)
 			break;
 
-		regval = readl_relaxed(pa->core + offset);
+		regval = readl_relaxed(pmic_arb->cnfg +
+				      SPMI_OWNERSHIP_TABLE_REG(apid));
+		apidd->irq_ee = SPMI_OWNERSHIP_PERIPH2OWNER(regval);
+		apidd->write_ee = apidd->irq_ee;
+
+		regval = readl_relaxed(pmic_arb->core + offset);
 		if (!regval)
 			continue;
 
 		id = (regval >> 8) & PMIC_ARB_PPID_MASK;
-		pa->ppid_to_apid[id] = apid | PMIC_ARB_CHAN_VALID;
-		pa->apid_data[apid].ppid = id;
+		pmic_arb->ppid_to_apid[id] = apid | PMIC_ARB_APID_VALID;
+		apidd->ppid = id;
 		if (id == ppid) {
-			apid |= PMIC_ARB_CHAN_VALID;
+			apid |= PMIC_ARB_APID_VALID;
 			break;
 		}
 	}
-	pa->last_apid = apid & ~PMIC_ARB_CHAN_VALID;
+	pmic_arb->last_apid = apid & ~PMIC_ARB_APID_VALID;
 
 	return apid;
 }
 
-
-static int
-pmic_arb_ppid_to_apid_v2(struct spmi_pmic_arb *pa, u8 sid, u16 addr, u16 *apid)
+static int pmic_arb_ppid_to_apid_v2(struct spmi_pmic_arb *pmic_arb, u16 ppid)
 {
-	u16 ppid = (sid << 8) | (addr >> 8);
 	u16 apid_valid;
 
-	apid_valid = pa->ppid_to_apid[ppid];
-	if (!(apid_valid & PMIC_ARB_CHAN_VALID))
-		apid_valid = pmic_arb_find_apid(pa, ppid);
-	if (!(apid_valid & PMIC_ARB_CHAN_VALID))
+	apid_valid = pmic_arb->ppid_to_apid[ppid];
+	if (!(apid_valid & PMIC_ARB_APID_VALID))
+		apid_valid = pmic_arb_find_apid(pmic_arb, ppid);
+	if (!(apid_valid & PMIC_ARB_APID_VALID))
 		return -ENODEV;
 
-	*apid = (apid_valid & ~PMIC_ARB_CHAN_VALID);
+	return apid_valid & ~PMIC_ARB_APID_VALID;
+}
+
+static int pmic_arb_read_apid_map_v5(struct spmi_pmic_arb *pmic_arb)
+{
+	struct apid_data *apidd = pmic_arb->apid_data;
+	struct apid_data *prev_apidd;
+	u16 i, apid, ppid;
+	bool valid, is_irq_ee;
+	u32 regval, offset;
+
+	/*
+	 * In order to allow multiple EEs to write to a single PPID in arbiter
+	 * version 5, there is more than one APID mapped to each PPID.
+	 * The owner field for each of these mappings specifies the EE which is
+	 * allowed to write to the APID.  The owner of the last (highest) APID
+	 * for a given PPID will receive interrupts from the PPID.
+	 */
+	for (i = 0; ; i++, apidd++) {
+		offset = pmic_arb->ver_ops->apid_map_offset(i);
+		if (offset >= pmic_arb->core_size)
+			break;
+
+		regval = readl_relaxed(pmic_arb->core + offset);
+		if (!regval)
+			continue;
+		ppid = (regval >> 8) & PMIC_ARB_PPID_MASK;
+		is_irq_ee = PMIC_ARB_CHAN_IS_IRQ_OWNER(regval);
+
+		regval = readl_relaxed(pmic_arb->cnfg +
+				      SPMI_OWNERSHIP_TABLE_REG(i));
+		apidd->write_ee = SPMI_OWNERSHIP_PERIPH2OWNER(regval);
+
+		apidd->irq_ee = is_irq_ee ? apidd->write_ee : INVALID_EE;
+
+		valid = pmic_arb->ppid_to_apid[ppid] & PMIC_ARB_APID_VALID;
+		apid = pmic_arb->ppid_to_apid[ppid] & ~PMIC_ARB_APID_VALID;
+		prev_apidd = &pmic_arb->apid_data[apid];
+
+		if (valid && is_irq_ee &&
+				prev_apidd->write_ee == pmic_arb->ee) {
+			/*
+			 * Duplicate PPID mapping after the one for this EE;
+			 * override the irq owner
+			 */
+			prev_apidd->irq_ee = apidd->irq_ee;
+		} else if (!valid || is_irq_ee) {
+			/* First PPID mapping or duplicate for another EE */
+			pmic_arb->ppid_to_apid[ppid] = i | PMIC_ARB_APID_VALID;
+		}
+
+		apidd->ppid = ppid;
+		pmic_arb->last_apid = i;
+	}
+
+	/* Dump the mapping table for debug purposes. */
+	dev_dbg(&pmic_arb->spmic->dev, "PPID APID Write-EE IRQ-EE\n");
+	for (ppid = 0; ppid < PMIC_ARB_MAX_PPID; ppid++) {
+		apid = pmic_arb->ppid_to_apid[ppid];
+		if (apid & PMIC_ARB_APID_VALID) {
+			apid &= ~PMIC_ARB_APID_VALID;
+			apidd = &pmic_arb->apid_data[apid];
+			dev_dbg(&pmic_arb->spmic->dev, "%#03X %3u %2u %2u\n",
+			      ppid, apid, apidd->write_ee, apidd->irq_ee);
+		}
+	}
+
 	return 0;
 }
 
-static int
-pmic_arb_mode_v2(struct spmi_pmic_arb *pa, u8 sid, u16 addr, mode_t *mode)
+static int pmic_arb_ppid_to_apid_v5(struct spmi_pmic_arb *pmic_arb, u16 ppid)
+{
+	if (!(pmic_arb->ppid_to_apid[ppid] & PMIC_ARB_APID_VALID))
+		return -ENODEV;
+
+	return pmic_arb->ppid_to_apid[ppid] & ~PMIC_ARB_APID_VALID;
+}
+
+/* v2 offset per ppid and per ee */
+static int pmic_arb_offset_v2(struct spmi_pmic_arb *pmic_arb, u8 sid, u16 addr,
+			   enum pmic_arb_channel ch_type)
 {
 	u16 apid;
+	u16 ppid;
 	int rc;
 
-	rc = pmic_arb_ppid_to_apid_v2(pa, sid, addr, &apid);
+	ppid = sid << 8 | ((addr >> 8) & 0xFF);
+	rc = pmic_arb_ppid_to_apid_v2(pmic_arb, ppid);
 	if (rc < 0)
 		return rc;
 
-	*mode = 0;
-	*mode |= S_IRUSR;
-
-	if (pa->ee == pa->apid_data[apid].owner)
-		*mode |= S_IWUSR;
-	return 0;
+	apid = rc;
+	return 0x1000 * pmic_arb->ee + 0x8000 * apid;
 }
 
-/* v2 offset per ppid and per ee */
-static int
-pmic_arb_offset_v2(struct spmi_pmic_arb *pa, u8 sid, u16 addr, u32 *offset)
+/*
+ * v5 offset per ee and per apid for observer channels and per apid for
+ * read/write channels.
+ */
+static int pmic_arb_offset_v5(struct spmi_pmic_arb *pmic_arb, u8 sid, u16 addr,
+			   enum pmic_arb_channel ch_type)
 {
 	u16 apid;
 	int rc;
+	u32 offset = 0;
+	u16 ppid = (sid << 8) | (addr >> 8);
 
-	rc = pmic_arb_ppid_to_apid_v2(pa, sid, addr, &apid);
+	rc = pmic_arb_ppid_to_apid_v5(pmic_arb, ppid);
 	if (rc < 0)
 		return rc;
 
-	*offset = 0x1000 * pa->ee + 0x8000 * apid;
-	return 0;
+	apid = rc;
+	switch (ch_type) {
+	case PMIC_ARB_CHANNEL_OBS:
+		offset = 0x10000 * pmic_arb->ee + 0x80 * apid;
+		break;
+	case PMIC_ARB_CHANNEL_RW:
+		offset = 0x10000 * apid;
+		break;
+	}
+
+	return offset;
 }
 
 static u32 pmic_arb_fmt_cmd_v1(u8 opc, u8 sid, u16 addr, u8 bc)
@@ -896,55 +985,97 @@ static u32 pmic_arb_fmt_cmd_v2(u8 opc, u8 sid, u16 addr, u8 bc)
 	return (opc << 27) | ((addr & 0xff) << 4) | (bc & 0x7);
 }
 
-static u32 pmic_arb_owner_acc_status_v1(u8 m, u16 n)
+static void __iomem *
+pmic_arb_owner_acc_status_v1(struct spmi_pmic_arb *pmic_arb, u8 m, u16 n)
+{
+	return pmic_arb->intr + 0x20 * m + 0x4 * n;
+}
+
+static void __iomem *
+pmic_arb_owner_acc_status_v2(struct spmi_pmic_arb *pmic_arb, u8 m, u16 n)
+{
+	return pmic_arb->intr + 0x100000 + 0x1000 * m + 0x4 * n;
+}
+
+static void __iomem *
+pmic_arb_owner_acc_status_v3(struct spmi_pmic_arb *pmic_arb, u8 m, u16 n)
+{
+	return pmic_arb->intr + 0x200000 + 0x1000 * m + 0x4 * n;
+}
+
+static void __iomem *
+pmic_arb_owner_acc_status_v5(struct spmi_pmic_arb *pmic_arb, u8 m, u16 n)
+{
+	return pmic_arb->intr + 0x10000 * m + 0x4 * n;
+}
+
+static void __iomem *
+pmic_arb_acc_enable_v1(struct spmi_pmic_arb *pmic_arb, u16 n)
+{
+	return pmic_arb->intr + 0x200 + 0x4 * n;
+}
+
+static void __iomem *
+pmic_arb_acc_enable_v2(struct spmi_pmic_arb *pmic_arb, u16 n)
+{
+	return pmic_arb->intr + 0x1000 * n;
+}
+
+static void __iomem *
+pmic_arb_acc_enable_v5(struct spmi_pmic_arb *pmic_arb, u16 n)
 {
-	return 0x20 * m + 0x4 * n;
+	return pmic_arb->wr_base + 0x100 + 0x10000 * n;
 }
 
-static u32 pmic_arb_owner_acc_status_v2(u8 m, u16 n)
+static void __iomem *
+pmic_arb_irq_status_v1(struct spmi_pmic_arb *pmic_arb, u16 n)
 {
-	return 0x100000 + 0x1000 * m + 0x4 * n;
+	return pmic_arb->intr + 0x600 + 0x4 * n;
 }
 
-static u32 pmic_arb_owner_acc_status_v3(u8 m, u16 n)
+static void __iomem *
+pmic_arb_irq_status_v2(struct spmi_pmic_arb *pmic_arb, u16 n)
 {
-	return 0x200000 + 0x1000 * m + 0x4 * n;
+	return pmic_arb->intr + 0x4 + 0x1000 * n;
 }
 
-static u32 pmic_arb_acc_enable_v1(u16 n)
+static void __iomem *
+pmic_arb_irq_status_v5(struct spmi_pmic_arb *pmic_arb, u16 n)
 {
-	return 0x200 + 0x4 * n;
+	return pmic_arb->wr_base + 0x104 + 0x10000 * n;
 }
 
-static u32 pmic_arb_acc_enable_v2(u16 n)
+static void __iomem *
+pmic_arb_irq_clear_v1(struct spmi_pmic_arb *pmic_arb, u16 n)
 {
-	return 0x1000 * n;
+	return pmic_arb->intr + 0xA00 + 0x4 * n;
 }
 
-static u32 pmic_arb_irq_status_v1(u16 n)
+static void __iomem *
+pmic_arb_irq_clear_v2(struct spmi_pmic_arb *pmic_arb, u16 n)
 {
-	return 0x600 + 0x4 * n;
+	return pmic_arb->intr + 0x8 + 0x1000 * n;
 }
 
-static u32 pmic_arb_irq_status_v2(u16 n)
+static void __iomem *
+pmic_arb_irq_clear_v5(struct spmi_pmic_arb *pmic_arb, u16 n)
 {
-	return 0x4 + 0x1000 * n;
+	return pmic_arb->wr_base + 0x108 + 0x10000 * n;
 }
 
-static u32 pmic_arb_irq_clear_v1(u16 n)
+static u32 pmic_arb_apid_map_offset_v2(u16 n)
 {
-	return 0xA00 + 0x4 * n;
+	return 0x800 + 0x4 * n;
 }
 
-static u32 pmic_arb_irq_clear_v2(u16 n)
+static u32 pmic_arb_apid_map_offset_v5(u16 n)
 {
-	return 0x8 + 0x1000 * n;
+	return 0x900 + 0x4 * n;
 }
 
 static const struct pmic_arb_ver_ops pmic_arb_v1 = {
 	.ver_str		= "v1",
 	.ppid_to_apid		= pmic_arb_ppid_to_apid_v1,
-	.mode			= pmic_arb_mode_v1_v3,
 	.non_data_cmd		= pmic_arb_non_data_cmd_v1,
 	.offset			= pmic_arb_offset_v1,
 	.fmt_cmd		= pmic_arb_fmt_cmd_v1,
@@ -952,12 +1083,12 @@ static const struct pmic_arb_ver_ops pmic_arb_v1 = {
 	.acc_enable		= pmic_arb_acc_enable_v1,
 	.irq_status		= pmic_arb_irq_status_v1,
 	.irq_clear		= pmic_arb_irq_clear_v1,
+	.apid_map_offset	= pmic_arb_apid_map_offset_v2,
 };
 
 static const struct pmic_arb_ver_ops pmic_arb_v2 = {
 	.ver_str		= "v2",
 	.ppid_to_apid		= pmic_arb_ppid_to_apid_v2,
-	.mode			= pmic_arb_mode_v2,
 	.non_data_cmd		= pmic_arb_non_data_cmd_v2,
 	.offset			= pmic_arb_offset_v2,
 	.fmt_cmd		= pmic_arb_fmt_cmd_v2,
@@ -965,12 +1096,12 @@ static const struct pmic_arb_ver_ops pmic_arb_v2 = {
 	.acc_enable		= pmic_arb_acc_enable_v2,
 	.irq_status		= pmic_arb_irq_status_v2,
 	.irq_clear		= pmic_arb_irq_clear_v2,
+	.apid_map_offset	= pmic_arb_apid_map_offset_v2,
 };
 
 static const struct pmic_arb_ver_ops pmic_arb_v3 = {
 	.ver_str		= "v3",
 	.ppid_to_apid		= pmic_arb_ppid_to_apid_v2,
-	.mode			= pmic_arb_mode_v1_v3,
 	.non_data_cmd		= pmic_arb_non_data_cmd_v2,
 	.offset			= pmic_arb_offset_v2,
 	.fmt_cmd		= pmic_arb_fmt_cmd_v2,
@@ -978,6 +1109,20 @@ static const struct pmic_arb_ver_ops pmic_arb_v3 = {
 	.acc_enable		= pmic_arb_acc_enable_v2,
 	.irq_status		= pmic_arb_irq_status_v2,
 	.irq_clear		= pmic_arb_irq_clear_v2,
+	.apid_map_offset	= pmic_arb_apid_map_offset_v2,
+};
+
+static const struct pmic_arb_ver_ops pmic_arb_v5 = {
+	.ver_str		= "v5",
+	.ppid_to_apid		= pmic_arb_ppid_to_apid_v5,
+	.non_data_cmd		= pmic_arb_non_data_cmd_v2,
+	.offset			= pmic_arb_offset_v5,
+	.fmt_cmd		= pmic_arb_fmt_cmd_v2,
+	.owner_acc_status	= pmic_arb_owner_acc_status_v5,
+	.acc_enable		= pmic_arb_acc_enable_v5,
+	.irq_status		= pmic_arb_irq_status_v5,
+	.irq_clear		= pmic_arb_irq_clear_v5,
+	.apid_map_offset	= pmic_arb_apid_map_offset_v5,
 };
 
 static const struct irq_domain_ops pmic_arb_irq_domain_ops = {
@@ -987,37 +1132,34 @@ static const struct irq_domain_ops pmic_arb_irq_domain_ops = {
 
 static int spmi_pmic_arb_probe(struct platform_device *pdev)
 {
-	struct spmi_pmic_arb *pa;
+	struct spmi_pmic_arb *pmic_arb;
 	struct spmi_controller *ctrl;
 	struct resource *res;
 	void __iomem *core;
+	u32 *mapping_table;
 	u32 channel, ee, hw_ver;
 	int err;
 
-	ctrl = spmi_controller_alloc(&pdev->dev, sizeof(*pa));
+	ctrl = spmi_controller_alloc(&pdev->dev, sizeof(*pmic_arb));
 	if (!ctrl)
 		return -ENOMEM;
 
-	pa = spmi_controller_get_drvdata(ctrl);
-	pa->spmic = ctrl;
+	pmic_arb = spmi_controller_get_drvdata(ctrl);
+	pmic_arb->spmic = ctrl;
 
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "core");
-	pa->core_size = resource_size(res);
-	if (pa->core_size <= 0x800) {
-		dev_err(&pdev->dev, "core_size is smaller than 0x800. Failing Probe\n");
-		err = -EINVAL;
-		goto err_put_ctrl;
-	}
-
 	core = devm_ioremap_resource(&ctrl->dev, res);
 	if (IS_ERR(core)) {
 		err = PTR_ERR(core);
 		goto err_put_ctrl;
 	}
 
-	pa->ppid_to_apid = devm_kcalloc(&ctrl->dev, PMIC_ARB_MAX_PPID,
-					sizeof(*pa->ppid_to_apid), GFP_KERNEL);
-	if (!pa->ppid_to_apid) {
+	pmic_arb->core_size = resource_size(res);
+
+	pmic_arb->ppid_to_apid = devm_kcalloc(&ctrl->dev, PMIC_ARB_MAX_PPID,
+					      sizeof(*pmic_arb->ppid_to_apid),
+					      GFP_KERNEL);
+	if (!pmic_arb->ppid_to_apid) {
 		err = -ENOMEM;
 		goto err_put_ctrl;
 	}
@@ -1025,57 +1167,56 @@ static int spmi_pmic_arb_probe(struct platform_device *pdev)
 	hw_ver = readl_relaxed(core + PMIC_ARB_VERSION);
 
 	if (hw_ver < PMIC_ARB_VERSION_V2_MIN) {
-		pa->ver_ops = &pmic_arb_v1;
-		pa->wr_base = core;
-		pa->rd_base = core;
+		pmic_arb->ver_ops = &pmic_arb_v1;
+		pmic_arb->wr_base = core;
+		pmic_arb->rd_base = core;
 	} else {
-		pa->core = core;
+		pmic_arb->core = core;
 
 		if (hw_ver < PMIC_ARB_VERSION_V3_MIN)
-			pa->ver_ops = &pmic_arb_v2;
+			pmic_arb->ver_ops = &pmic_arb_v2;
+		else if (hw_ver < PMIC_ARB_VERSION_V5_MIN)
+			pmic_arb->ver_ops = &pmic_arb_v3;
 		else
-			pa->ver_ops = &pmic_arb_v3;
-
-		/* the apid to ppid table starts at PMIC_ARB_REG_CHNL(0) */
-		pa->max_periph = (pa->core_size - PMIC_ARB_REG_CHNL(0)) / 4;
+			pmic_arb->ver_ops = &pmic_arb_v5;
 
 		res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
 						   "obsrvr");
-		pa->rd_base = devm_ioremap_resource(&ctrl->dev, res);
-		if (IS_ERR(pa->rd_base)) {
-			err = PTR_ERR(pa->rd_base);
+		pmic_arb->rd_base = devm_ioremap_resource(&ctrl->dev, res);
+		if (IS_ERR(pmic_arb->rd_base)) {
+			err = PTR_ERR(pmic_arb->rd_base);
 			goto err_put_ctrl;
 		}
 
 		res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
 						   "chnls");
-		pa->wr_base = devm_ioremap_resource(&ctrl->dev, res);
-		if (IS_ERR(pa->wr_base)) {
-			err = PTR_ERR(pa->wr_base);
+		pmic_arb->wr_base = devm_ioremap_resource(&ctrl->dev, res);
+		if (IS_ERR(pmic_arb->wr_base)) {
+			err = PTR_ERR(pmic_arb->wr_base);
 			goto err_put_ctrl;
 		}
 	}
 
 	dev_info(&ctrl->dev, "PMIC arbiter version %s (0x%x)\n",
-		 pa->ver_ops->ver_str, hw_ver);
+		 pmic_arb->ver_ops->ver_str, hw_ver);
 
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "intr");
-	pa->intr = devm_ioremap_resource(&ctrl->dev, res);
-	if (IS_ERR(pa->intr)) {
-		err = PTR_ERR(pa->intr);
+	pmic_arb->intr = devm_ioremap_resource(&ctrl->dev, res);
+	if (IS_ERR(pmic_arb->intr)) {
+		err = PTR_ERR(pmic_arb->intr);
 		goto err_put_ctrl;
 	}
 
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cnfg");
-	pa->cnfg = devm_ioremap_resource(&ctrl->dev, res);
-	if (IS_ERR(pa->cnfg)) {
-		err = PTR_ERR(pa->cnfg);
+	pmic_arb->cnfg = devm_ioremap_resource(&ctrl->dev, res);
+	if (IS_ERR(pmic_arb->cnfg)) {
+		err = PTR_ERR(pmic_arb->cnfg);
 		goto err_put_ctrl;
 	}
 
-	pa->irq = platform_get_irq_byname(pdev, "periph_irq");
-	if (pa->irq < 0) {
-		err = pa->irq;
+	pmic_arb->irq = platform_get_irq_byname(pdev, "periph_irq");
+	if (pmic_arb->irq < 0) {
+		err = pmic_arb->irq;
 		goto err_put_ctrl;
 	}
 
@@ -1092,7 +1233,7 @@ static int spmi_pmic_arb_probe(struct platform_device *pdev)
 		goto err_put_ctrl;
 	}
 
-	pa->channel = channel;
+	pmic_arb->channel = channel;
 
 	err = of_property_read_u32(pdev->dev.of_node, "qcom,ee", &ee);
 	if (err) {
@@ -1106,39 +1247,47 @@ static int spmi_pmic_arb_probe(struct platform_device *pdev)
 		goto err_put_ctrl;
 	}
 
-	pa->ee = ee;
-
-	pa->mapping_table = devm_kcalloc(&ctrl->dev, PMIC_ARB_MAX_PERIPHS - 1,
-					sizeof(*pa->mapping_table), GFP_KERNEL);
-	if (!pa->mapping_table) {
+	pmic_arb->ee = ee;
+	mapping_table = devm_kcalloc(&ctrl->dev, PMIC_ARB_MAX_PERIPHS,
+					sizeof(*mapping_table), GFP_KERNEL);
+	if (!mapping_table) {
 		err = -ENOMEM;
 		goto err_put_ctrl;
 	}
 
+	pmic_arb->mapping_table = mapping_table;
 	/* Initialize max_apid/min_apid to the opposite bounds, during
 	 * the irq domain translation, we are sure to update these */
-	pa->max_apid = 0;
-	pa->min_apid = PMIC_ARB_MAX_PERIPHS - 1;
+	pmic_arb->max_apid = 0;
+	pmic_arb->min_apid = PMIC_ARB_MAX_PERIPHS - 1;
 
 	platform_set_drvdata(pdev, ctrl);
-	raw_spin_lock_init(&pa->lock);
+	raw_spin_lock_init(&pmic_arb->lock);
 
 	ctrl->cmd = pmic_arb_cmd;
 	ctrl->read_cmd = pmic_arb_read_cmd;
 	ctrl->write_cmd = pmic_arb_write_cmd;
 
+	if (hw_ver >= PMIC_ARB_VERSION_V5_MIN) {
+		err = pmic_arb_read_apid_map_v5(pmic_arb);
+		if (err) {
+			dev_err(&pdev->dev, "could not read APID->PPID mapping table, rc= %d\n",
+				err);
+			goto err_put_ctrl;
+		}
+	}
+
 	dev_dbg(&pdev->dev, "adding irq domain\n");
-	pa->domain = irq_domain_add_tree(pdev->dev.of_node,
-					 &pmic_arb_irq_domain_ops, pa);
-	if (!pa->domain) {
+	pmic_arb->domain = irq_domain_add_tree(pdev->dev.of_node,
+					 &pmic_arb_irq_domain_ops, pmic_arb);
+	if (!pmic_arb->domain) {
 		dev_err(&pdev->dev, "unable to create irq_domain\n");
 		err = -ENOMEM;
 		goto err_put_ctrl;
 	}
 
-	irq_set_chained_handler_and_data(pa->irq, pmic_arb_chained_irq, pa);
-	enable_irq_wake(pa->irq);
-
+	irq_set_chained_handler_and_data(pmic_arb->irq, pmic_arb_chained_irq,
+					pmic_arb);
 	err = spmi_controller_add(ctrl);
 	if (err)
 		goto err_domain_remove;
@@ -1146,8 +1295,8 @@ static int spmi_pmic_arb_probe(struct platform_device *pdev)
 	return 0;
 
 err_domain_remove:
-	irq_set_chained_handler_and_data(pa->irq, NULL, NULL);
-	irq_domain_remove(pa->domain);
+	irq_set_chained_handler_and_data(pmic_arb->irq, NULL, NULL);
+	irq_domain_remove(pmic_arb->domain);
 err_put_ctrl:
 	spmi_controller_put(ctrl);
 	return err;
@@ -1156,10 +1305,10 @@ err_put_ctrl:
 static int spmi_pmic_arb_remove(struct platform_device *pdev)
 {
 	struct spmi_controller *ctrl = platform_get_drvdata(pdev);
-	struct spmi_pmic_arb *pa = spmi_controller_get_drvdata(ctrl);
+	struct spmi_pmic_arb *pmic_arb = spmi_controller_get_drvdata(ctrl);
 	spmi_controller_remove(ctrl);
-	irq_set_chained_handler_and_data(pa->irq, NULL, NULL);
-	irq_domain_remove(pa->domain);
+	irq_set_chained_handler_and_data(pmic_arb->irq, NULL, NULL);
+	irq_domain_remove(pmic_arb->domain);
 	spmi_controller_put(ctrl);
 	return 0;
 }
diff --git a/drivers/spmi/spmi.c b/drivers/spmi/spmi.c
index 6d23226e5f69..aa3edabc2b0f 100644
--- a/drivers/spmi/spmi.c
+++ b/drivers/spmi/spmi.c
@@ -466,27 +466,25 @@ static void of_spmi_register_devices(struct spmi_controller *ctrl)
 		struct spmi_device *sdev;
 		u32 reg[2];
 
-		dev_dbg(&ctrl->dev, "adding child %s\n", node->full_name);
+		dev_dbg(&ctrl->dev, "adding child %pOF\n", node);
 
 		err = of_property_read_u32_array(node, "reg", reg, 2);
 		if (err) {
 			dev_err(&ctrl->dev,
-				"node %s err (%d) does not have 'reg' property\n",
-				node->full_name, err);
+				"node %pOF err (%d) does not have 'reg' property\n",
+				node, err);
 			continue;
 		}
 
 		if (reg[1] != SPMI_USID) {
 			dev_err(&ctrl->dev,
-				"node %s contains unsupported 'reg' entry\n",
-				node->full_name);
+				"node %pOF contains unsupported 'reg' entry\n",
+				node);
 			continue;
 		}
 
 		if (reg[0] >= SPMI_MAX_SLAVE_ID) {
-			dev_err(&ctrl->dev,
-				"invalid usid on node %s\n",
-				node->full_name);
+			dev_err(&ctrl->dev, "invalid usid on node %pOF\n", node);
 			continue;
 		}
 
diff --git a/drivers/staging/Kconfig b/drivers/staging/Kconfig
index ef28a1cb64ae..554683912cff 100644
--- a/drivers/staging/Kconfig
+++ b/drivers/staging/Kconfig
@@ -24,6 +24,8 @@ menuconfig STAGING
 
 if STAGING
 
+source "drivers/staging/irda/net/Kconfig"
+
 source "drivers/staging/wlan-ng/Kconfig"
 
 source "drivers/staging/comedi/Kconfig"
@@ -40,6 +42,8 @@ source "drivers/staging/rtl8712/Kconfig"
 
 source "drivers/staging/rtl8188eu/Kconfig"
 
+source "drivers/staging/rtlwifi/Kconfig"
+
 source "drivers/staging/rts5208/Kconfig"
 
 source "drivers/staging/octeon/Kconfig"
@@ -112,4 +116,6 @@ source "drivers/staging/typec/Kconfig"
 
 source "drivers/staging/vboxvideo/Kconfig"
 
+source "drivers/staging/pi433/Kconfig"
+
 endif # STAGING
diff --git a/drivers/staging/Makefile b/drivers/staging/Makefile
index 2918580bdb9e..8951c37d8d80 100644
--- a/drivers/staging/Makefile
+++ b/drivers/staging/Makefile
@@ -2,6 +2,8 @@
 
 obj-y				+= media/
 obj-y				+= typec/
+obj-$(CONFIG_IRDA)		+= irda/net/
+obj-$(CONFIG_IRDA)		+= irda/drivers/
 obj-$(CONFIG_PRISM2_USB)	+= wlan-ng/
 obj-$(CONFIG_COMEDI)		+= comedi/
 obj-$(CONFIG_FB_OLPC_DCON)	+= olpc_dcon/
@@ -10,6 +12,7 @@ obj-$(CONFIG_RTL8192E)		+= rtl8192e/
 obj-$(CONFIG_RTL8723BS)		+= rtl8723bs/
 obj-$(CONFIG_R8712U)		+= rtl8712/
 obj-$(CONFIG_R8188EU)		+= rtl8188eu/
+obj-$(CONFIG_R8822BE)		+= rtlwifi/
 obj-$(CONFIG_RTS5208)		+= rts5208/
 obj-$(CONFIG_NETLOGIC_XLR_NET)	+= netlogic/
 obj-$(CONFIG_OCTEON_ETHERNET)	+= octeon/
@@ -45,3 +48,4 @@ obj-$(CONFIG_GREYBUS)		+= greybus/
 obj-$(CONFIG_BCM2835_VCHIQ)	+= vc04_services/
 obj-$(CONFIG_CRYPTO_DEV_CCREE)	+= ccree/
 obj-$(CONFIG_DRM_VBOXVIDEO)	+= vboxvideo/
+obj-$(CONFIG_PI433)		+= pi433/
diff --git a/drivers/staging/android/ashmem.c b/drivers/staging/android/ashmem.c
index 6ba270e0494d..0f695df14c9d 100644
--- a/drivers/staging/android/ashmem.c
+++ b/drivers/staging/android/ashmem.c
@@ -294,19 +294,9 @@ static int ashmem_release(struct inode *ignored, struct file *file)
 	return 0;
 }
 
-/**
- * ashmem_read() - Reads a set of bytes from an Ashmem-enabled file
- * @file:	   The associated backing file.
- * @buf:	   The buffer of data being written to
- * @len:	   The number of bytes being read
- * @pos:	   The position of the first byte to read.
- *
- * Return: 0 if successful, or another return code if not.
- */
-static ssize_t ashmem_read(struct file *file, char __user *buf,
-			   size_t len, loff_t *pos)
+static ssize_t ashmem_read_iter(struct kiocb *iocb, struct iov_iter *iter)
 {
-	struct ashmem_area *asma = file->private_data;
+	struct ashmem_area *asma = iocb->ki_filp->private_data;
 	int ret = 0;
 
 	mutex_lock(&ashmem_mutex);
@@ -320,20 +310,17 @@ static ssize_t ashmem_read(struct file *file, char __user *buf,
 		goto out_unlock;
 	}
 
-	mutex_unlock(&ashmem_mutex);
-
 	/*
 	 * asma and asma->file are used outside the lock here.  We assume
 	 * once asma->file is set it will never be changed, and will not
 	 * be destroyed until all references to the file are dropped and
 	 * ashmem_release is called.
 	 */
-	ret = __vfs_read(asma->file, buf, len, pos);
-	if (ret >= 0)
-		/** Update backing file pos, since f_ops->read() doesn't */
-		asma->file->f_pos = *pos;
-	return ret;
-
+	mutex_unlock(&ashmem_mutex);
+	ret = vfs_iter_read(asma->file, iter, &iocb->ki_pos, 0);
+	mutex_lock(&ashmem_mutex);
+	if (ret > 0)
+		asma->file->f_pos = iocb->ki_pos;
 out_unlock:
 	mutex_unlock(&ashmem_mutex);
 	return ret;
@@ -834,7 +821,7 @@ static const struct file_operations ashmem_fops = {
 	.owner = THIS_MODULE,
 	.open = ashmem_open,
 	.release = ashmem_release,
-	.read = ashmem_read,
+	.read_iter = ashmem_read_iter,
 	.llseek = ashmem_llseek,
 	.mmap = ashmem_mmap,
 	.unlocked_ioctl = ashmem_ioctl,
diff --git a/drivers/staging/android/ion/ion.h b/drivers/staging/android/ion/ion.h
index fa9ed81ab972..621e5f7ceacb 100644
--- a/drivers/staging/android/ion/ion.h
+++ b/drivers/staging/android/ion/ion.h
@@ -135,7 +135,7 @@ struct ion_heap_ops {
 /**
  * heap flags - flags between the heaps and core ion code
  */
-#define ION_HEAP_FLAG_DEFER_FREE (1 << 0)
+#define ION_HEAP_FLAG_DEFER_FREE BIT(0)
 
 /**
  * private flags - flags internal to ion
@@ -146,7 +146,7 @@ struct ion_heap_ops {
  * any buffer storage that came from the system allocator will be
  * returned to the system allocator.
  */
-#define ION_PRIV_FLAG_SHRINKER_FREE (1 << 0)
+#define ION_PRIV_FLAG_SHRINKER_FREE BIT(0)
 
 /**
  * struct ion_heap - represents a heap in the system
@@ -226,8 +226,8 @@ int ion_heap_buffer_zero(struct ion_buffer *buffer);
 int ion_heap_pages_zero(struct page *page, size_t size, pgprot_t pgprot);
 
 int ion_alloc(size_t len,
-				unsigned int heap_id_mask,
-				unsigned int flags);
+	      unsigned int heap_id_mask,
+	      unsigned int flags);
 
 /**
  * ion_heap_init_shrinker
@@ -291,7 +291,7 @@ size_t ion_heap_freelist_drain(struct ion_heap *heap, size_t size);
  * flag.
  */
 size_t ion_heap_freelist_shrink(struct ion_heap *heap,
-					size_t size);
+				size_t size);
 
 /**
  * ion_heap_freelist_size - returns the size of the freelist in bytes
@@ -352,7 +352,7 @@ void ion_page_pool_free(struct ion_page_pool *pool, struct page *page);
  * returns the number of items freed in pages
  */
 int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask,
-			  int nr_to_scan);
+			 int nr_to_scan);
 
 long ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
 
diff --git a/drivers/staging/android/ion/ion_cma_heap.c b/drivers/staging/android/ion/ion_cma_heap.c
index a0949bc0dcf4..dd5545d9990a 100644
--- a/drivers/staging/android/ion/ion_cma_heap.c
+++ b/drivers/staging/android/ion/ion_cma_heap.c
@@ -31,7 +31,6 @@ struct ion_cma_heap {
 
 #define to_cma_heap(x) container_of(x, struct ion_cma_heap, heap)
 
-
 /* ION CMA heap operations functions */
 static int ion_cma_allocate(struct ion_heap *heap, struct ion_buffer *buffer,
 			    unsigned long len,
@@ -46,7 +45,7 @@ static int ion_cma_allocate(struct ion_heap *heap, struct ion_buffer *buffer,
 	if (!pages)
 		return -ENOMEM;
 
-	table = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
+	table = kmalloc(sizeof(*table), GFP_KERNEL);
 	if (!table)
 		goto err;
 
@@ -106,7 +105,7 @@ static struct ion_heap *__ion_cma_heap_create(struct cma *cma)
 	return &cma_heap->heap;
 }
 
-int __ion_add_cma_heaps(struct cma *cma, void *data)
+static int __ion_add_cma_heaps(struct cma *cma, void *data)
 {
 	struct ion_heap *heap;
 
diff --git a/drivers/staging/android/ion/ion_system_heap.c b/drivers/staging/android/ion/ion_system_heap.c
index 5964bf21fd80..4dc5d7a589c2 100644
--- a/drivers/staging/android/ion/ion_system_heap.c
+++ b/drivers/staging/android/ion/ion_system_heap.c
@@ -98,7 +98,6 @@ static void free_buffer_page(struct ion_system_heap *heap,
 	ion_page_pool_free(pool, page);
 }
 
-
 static struct page *alloc_largest_available(struct ion_system_heap *heap,
 					    struct ion_buffer *buffer,
 					    unsigned long size,
@@ -256,7 +255,6 @@ static struct ion_heap_ops system_heap_ops = {
 static int ion_system_heap_debug_show(struct ion_heap *heap, struct seq_file *s,
 				      void *unused)
 {
-
 	struct ion_system_heap *sys_heap = container_of(heap,
 							struct ion_system_heap,
 							heap);
diff --git a/drivers/staging/ccree/Kconfig b/drivers/staging/ccree/Kconfig
index 36a87c686a2a..0b3092ba2fcb 100644
--- a/drivers/staging/ccree/Kconfig
+++ b/drivers/staging/ccree/Kconfig
@@ -23,12 +23,3 @@ config CRYPTO_DEV_CCREE
 	  Choose this if you wish to use hardware acceleration of
 	  cryptographic operations on the system REE.
 	  If unsure say Y.
-
-config CCREE_FIPS_SUPPORT
-	bool "Turn on CryptoCell 7XX REE FIPS mode support"
-	depends on CRYPTO_DEV_CCREE
-	default n
-	help
-	  Say 'Y' to enable support for FIPS compliant mode by the
-	  CCREE driver.
-	  If unsure say N.
diff --git a/drivers/staging/ccree/Makefile b/drivers/staging/ccree/Makefile
index 318c2b39acf6..ae702f3b5369 100644
--- a/drivers/staging/ccree/Makefile
+++ b/drivers/staging/ccree/Makefile
@@ -1,3 +1,3 @@
 obj-$(CONFIG_CRYPTO_DEV_CCREE) := ccree.o
 ccree-y := ssi_driver.o ssi_sysfs.o ssi_buffer_mgr.o ssi_request_mgr.o ssi_cipher.o ssi_hash.o ssi_aead.o ssi_ivgen.o ssi_sram_mgr.o ssi_pm.o
-ccree-$(CCREE_FIPS_SUPPORT) += ssi_fips.o ssi_fips_ll.o ssi_fips_ext.o ssi_fips_local.o
+ccree-$(CONFIG_CRYPTO_FIPS) += ssi_fips.o
diff --git a/drivers/staging/ccree/cc_hw_queue_defs.h b/drivers/staging/ccree/cc_hw_queue_defs.h
index e6b8cea3f88d..2ae0f655e7a0 100644
--- a/drivers/staging/ccree/cc_hw_queue_defs.h
+++ b/drivers/staging/ccree/cc_hw_queue_defs.h
@@ -27,7 +27,8 @@
  ******************************************************************************/
 
 #define HW_DESC_SIZE_WORDS		6
-#define HW_QUEUE_SLOTS_MAX              15 /* Max. available slots in HW queue */
+/* Define max. available slots in HW queue */
+#define HW_QUEUE_SLOTS_MAX              15
 
 #define CC_REG_NAME(word, name) DX_DSCRPTR_QUEUE_WORD ## word ## _ ## name
 
diff --git a/drivers/staging/ccree/ssi_aead.c b/drivers/staging/ccree/ssi_aead.c
index 1fc0b05ea0d5..5abe6b24ff8c 100644
--- a/drivers/staging/ccree/ssi_aead.c
+++ b/drivers/staging/ccree/ssi_aead.c
@@ -36,7 +36,6 @@
 #include "ssi_hash.h"
 #include "ssi_sysfs.h"
 #include "ssi_sram_mgr.h"
-#include "ssi_fips_local.h"
 
 #define template_aead	template_u.aead
 
@@ -57,22 +56,26 @@ struct ssi_aead_handle {
 	struct list_head aead_list;
 };
 
+struct cc_hmac_s {
+	u8 *padded_authkey;
+	u8 *ipad_opad; /* IPAD, OPAD*/
+	dma_addr_t padded_authkey_dma_addr;
+	dma_addr_t ipad_opad_dma_addr;
+};
+
+struct cc_xcbc_s {
+	u8 *xcbc_keys; /* K1,K2,K3 */
+	dma_addr_t xcbc_keys_dma_addr;
+};
+
 struct ssi_aead_ctx {
 	struct ssi_drvdata *drvdata;
 	u8 ctr_nonce[MAX_NONCE_SIZE]; /* used for ctr3686 iv and aes ccm */
 	u8 *enckey;
 	dma_addr_t enckey_dma_addr;
 	union {
-		struct {
-			u8 *padded_authkey;
-			u8 *ipad_opad; /* IPAD, OPAD*/
-			dma_addr_t padded_authkey_dma_addr;
-			dma_addr_t ipad_opad_dma_addr;
-		} hmac;
-		struct {
-			u8 *xcbc_keys; /* K1,K2,K3 */
-			dma_addr_t xcbc_keys_dma_addr;
-		} xcbc;
+		struct cc_hmac_s hmac;
+		struct cc_xcbc_s xcbc;
 	} auth_state;
 	unsigned int enc_keylen;
 	unsigned int auth_keylen;
@@ -93,46 +96,50 @@ static void ssi_aead_exit(struct crypto_aead *tfm)
 	struct ssi_aead_ctx *ctx = crypto_aead_ctx(tfm);
 
 	SSI_LOG_DEBUG("Clearing context @%p for %s\n",
-		crypto_aead_ctx(tfm), crypto_tfm_alg_name(&(tfm->base)));
+		      crypto_aead_ctx(tfm), crypto_tfm_alg_name(&tfm->base));
 
 	dev = &ctx->drvdata->plat_dev->dev;
 	/* Unmap enckey buffer */
 	if (ctx->enckey) {
 		dma_free_coherent(dev, AES_MAX_KEY_SIZE, ctx->enckey, ctx->enckey_dma_addr);
-		SSI_LOG_DEBUG("Freed enckey DMA buffer enckey_dma_addr=0x%llX\n",
-			(unsigned long long)ctx->enckey_dma_addr);
+		SSI_LOG_DEBUG("Freed enckey DMA buffer enckey_dma_addr=%pad\n",
+			      ctx->enckey_dma_addr);
 		ctx->enckey_dma_addr = 0;
 		ctx->enckey = NULL;
 	}
 
 	if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */
-		if (ctx->auth_state.xcbc.xcbc_keys) {
+		struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc;
+
+		if (xcbc->xcbc_keys) {
 			dma_free_coherent(dev, CC_AES_128_BIT_KEY_SIZE * 3,
-				ctx->auth_state.xcbc.xcbc_keys,
-				ctx->auth_state.xcbc.xcbc_keys_dma_addr);
+					  xcbc->xcbc_keys,
+					  xcbc->xcbc_keys_dma_addr);
 		}
-		SSI_LOG_DEBUG("Freed xcbc_keys DMA buffer xcbc_keys_dma_addr=0x%llX\n",
-			(unsigned long long)ctx->auth_state.xcbc.xcbc_keys_dma_addr);
-		ctx->auth_state.xcbc.xcbc_keys_dma_addr = 0;
-		ctx->auth_state.xcbc.xcbc_keys = NULL;
+		SSI_LOG_DEBUG("Freed xcbc_keys DMA buffer xcbc_keys_dma_addr=%pad\n",
+			      xcbc->xcbc_keys_dma_addr);
+		xcbc->xcbc_keys_dma_addr = 0;
+		xcbc->xcbc_keys = NULL;
 	} else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC auth. */
-		if (ctx->auth_state.hmac.ipad_opad) {
+		struct cc_hmac_s *hmac = &ctx->auth_state.hmac;
+
+		if (hmac->ipad_opad) {
 			dma_free_coherent(dev, 2 * MAX_HMAC_DIGEST_SIZE,
-				ctx->auth_state.hmac.ipad_opad,
-				ctx->auth_state.hmac.ipad_opad_dma_addr);
-			SSI_LOG_DEBUG("Freed ipad_opad DMA buffer ipad_opad_dma_addr=0x%llX\n",
-				(unsigned long long)ctx->auth_state.hmac.ipad_opad_dma_addr);
-			ctx->auth_state.hmac.ipad_opad_dma_addr = 0;
-			ctx->auth_state.hmac.ipad_opad = NULL;
+					  hmac->ipad_opad,
+					  hmac->ipad_opad_dma_addr);
+			SSI_LOG_DEBUG("Freed ipad_opad DMA buffer ipad_opad_dma_addr=%pad\n",
+				      hmac->ipad_opad_dma_addr);
+			hmac->ipad_opad_dma_addr = 0;
+			hmac->ipad_opad = NULL;
 		}
-		if (ctx->auth_state.hmac.padded_authkey) {
+		if (hmac->padded_authkey) {
 			dma_free_coherent(dev, MAX_HMAC_BLOCK_SIZE,
-				ctx->auth_state.hmac.padded_authkey,
-				ctx->auth_state.hmac.padded_authkey_dma_addr);
-			SSI_LOG_DEBUG("Freed padded_authkey DMA buffer padded_authkey_dma_addr=0x%llX\n",
-				(unsigned long long)ctx->auth_state.hmac.padded_authkey_dma_addr);
-			ctx->auth_state.hmac.padded_authkey_dma_addr = 0;
-			ctx->auth_state.hmac.padded_authkey = NULL;
+					  hmac->padded_authkey,
+					  hmac->padded_authkey_dma_addr);
+			SSI_LOG_DEBUG("Freed padded_authkey DMA buffer padded_authkey_dma_addr=%pad\n",
+				      hmac->padded_authkey_dma_addr);
+			hmac->padded_authkey_dma_addr = 0;
+			hmac->padded_authkey = NULL;
 		}
 	}
 }
@@ -144,9 +151,7 @@ static int ssi_aead_init(struct crypto_aead *tfm)
 	struct ssi_aead_ctx *ctx = crypto_aead_ctx(tfm);
 	struct ssi_crypto_alg *ssi_alg =
 			container_of(alg, struct ssi_crypto_alg, aead_alg);
-	SSI_LOG_DEBUG("Initializing context @%p for %s\n", ctx, crypto_tfm_alg_name(&(tfm->base)));
-
-	CHECK_AND_RETURN_UPON_FIPS_ERROR();
+	SSI_LOG_DEBUG("Initializing context @%p for %s\n", ctx, crypto_tfm_alg_name(&tfm->base));
 
 	/* Initialize modes in instance */
 	ctx->cipher_mode = ssi_alg->cipher_mode;
@@ -158,7 +163,7 @@ static int ssi_aead_init(struct crypto_aead *tfm)
 
 	/* Allocate key buffer, cache line aligned */
 	ctx->enckey = dma_alloc_coherent(dev, AES_MAX_KEY_SIZE,
-		&ctx->enckey_dma_addr, GFP_KERNEL);
+					 &ctx->enckey_dma_addr, GFP_KERNEL);
 	if (!ctx->enckey) {
 		SSI_LOG_ERR("Failed allocating key buffer\n");
 		goto init_failed;
@@ -168,31 +173,42 @@ static int ssi_aead_init(struct crypto_aead *tfm)
 	/* Set default authlen value */
 
 	if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */
+		struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc;
+		const unsigned int key_size = CC_AES_128_BIT_KEY_SIZE * 3;
+
 		/* Allocate dma-coherent buffer for XCBC's K1+K2+K3 */
 		/* (and temporary for user key - up to 256b) */
-		ctx->auth_state.xcbc.xcbc_keys = dma_alloc_coherent(dev,
-			CC_AES_128_BIT_KEY_SIZE * 3,
-			&ctx->auth_state.xcbc.xcbc_keys_dma_addr, GFP_KERNEL);
-		if (!ctx->auth_state.xcbc.xcbc_keys) {
+		xcbc->xcbc_keys = dma_alloc_coherent(dev, key_size,
+						     &xcbc->xcbc_keys_dma_addr,
+						     GFP_KERNEL);
+		if (!xcbc->xcbc_keys) {
 			SSI_LOG_ERR("Failed allocating buffer for XCBC keys\n");
 			goto init_failed;
 		}
 	} else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC authentication */
+		struct cc_hmac_s *hmac = &ctx->auth_state.hmac;
+		const unsigned int digest_size = 2 * MAX_HMAC_DIGEST_SIZE;
+		dma_addr_t *pkey_dma = &hmac->padded_authkey_dma_addr;
+
 		/* Allocate dma-coherent buffer for IPAD + OPAD */
-		ctx->auth_state.hmac.ipad_opad = dma_alloc_coherent(dev,
-			2 * MAX_HMAC_DIGEST_SIZE,
-			&ctx->auth_state.hmac.ipad_opad_dma_addr, GFP_KERNEL);
-		if (!ctx->auth_state.hmac.ipad_opad) {
+		hmac->ipad_opad = dma_alloc_coherent(dev, digest_size,
+						     &hmac->ipad_opad_dma_addr,
+						     GFP_KERNEL);
+
+		if (!hmac->ipad_opad) {
 			SSI_LOG_ERR("Failed allocating IPAD/OPAD buffer\n");
 			goto init_failed;
 		}
+
 		SSI_LOG_DEBUG("Allocated authkey buffer in context ctx->authkey=@%p\n",
-			ctx->auth_state.hmac.ipad_opad);
+			      hmac->ipad_opad);
+
+		hmac->padded_authkey = dma_alloc_coherent(dev,
+							  MAX_HMAC_BLOCK_SIZE,
+							  pkey_dma,
+							  GFP_KERNEL);
 
-		ctx->auth_state.hmac.padded_authkey = dma_alloc_coherent(dev,
-			MAX_HMAC_BLOCK_SIZE,
-			&ctx->auth_state.hmac.padded_authkey_dma_addr, GFP_KERNEL);
-		if (!ctx->auth_state.hmac.padded_authkey) {
+		if (!hmac->padded_authkey) {
 			SSI_LOG_ERR("failed to allocate padded_authkey\n");
 			goto init_failed;
 		}
@@ -223,7 +239,7 @@ static void ssi_aead_complete(struct device *dev, void *ssi_req, void __iomem *c
 
 	if (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
 		if (memcmp(areq_ctx->mac_buf, areq_ctx->icv_virt_addr,
-			ctx->authsize) != 0) {
+			   ctx->authsize) != 0) {
 			SSI_LOG_DEBUG("Payload authentication failure, "
 				"(auth-size=%d, cipher=%d).\n",
 				ctx->authsize, ctx->cipher_mode);
@@ -236,8 +252,8 @@ static void ssi_aead_complete(struct device *dev, void *ssi_req, void __iomem *c
 	} else { /*ENCRYPT*/
 		if (unlikely(areq_ctx->is_icv_fragmented))
 			ssi_buffer_mgr_copy_scatterlist_portion(
-				areq_ctx->mac_buf, areq_ctx->dstSgl, areq->cryptlen + areq_ctx->dstOffset,
-				areq->cryptlen + areq_ctx->dstOffset + ctx->authsize, SSI_SG_FROM_BUF);
+				areq_ctx->mac_buf, areq_ctx->dst_sgl, areq->cryptlen + areq_ctx->dst_offset,
+				areq->cryptlen + areq_ctx->dst_offset + ctx->authsize, SSI_SG_FROM_BUF);
 
 		/* If an IV was generated, copy it back to the user provided buffer. */
 		if (areq_ctx->backup_giv) {
@@ -292,12 +308,13 @@ static int xcbc_setkey(struct cc_hw_desc *desc, struct ssi_aead_ctx *ctx)
 
 static int hmac_setkey(struct cc_hw_desc *desc, struct ssi_aead_ctx *ctx)
 {
-	unsigned int hmacPadConst[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST };
+	unsigned int hmac_pad_const[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST };
 	unsigned int digest_ofs = 0;
 	unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
 			DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
 	unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ?
 			CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE;
+	struct cc_hmac_s *hmac = &ctx->auth_state.hmac;
 
 	int idx = 0;
 	int i;
@@ -325,7 +342,7 @@ static int hmac_setkey(struct cc_hw_desc *desc, struct ssi_aead_ctx *ctx)
 
 		/* Prepare ipad key */
 		hw_desc_init(&desc[idx]);
-		set_xor_val(&desc[idx], hmacPadConst[i]);
+		set_xor_val(&desc[idx], hmac_pad_const[i]);
 		set_cipher_mode(&desc[idx], hash_mode);
 		set_flow_mode(&desc[idx], S_DIN_to_HASH);
 		set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
@@ -334,7 +351,7 @@ static int hmac_setkey(struct cc_hw_desc *desc, struct ssi_aead_ctx *ctx)
 		/* Perform HASH update */
 		hw_desc_init(&desc[idx]);
 		set_din_type(&desc[idx], DMA_DLLI,
-			     ctx->auth_state.hmac.padded_authkey_dma_addr,
+			     hmac->padded_authkey_dma_addr,
 			     SHA256_BLOCK_SIZE, NS_BIT);
 		set_cipher_mode(&desc[idx], hash_mode);
 		set_xor_active(&desc[idx]);
@@ -345,8 +362,8 @@ static int hmac_setkey(struct cc_hw_desc *desc, struct ssi_aead_ctx *ctx)
 		hw_desc_init(&desc[idx]);
 		set_cipher_mode(&desc[idx], hash_mode);
 		set_dout_dlli(&desc[idx],
-			      (ctx->auth_state.hmac.ipad_opad_dma_addr +
-			       digest_ofs), digest_size, NS_BIT, 0);
+			      (hmac->ipad_opad_dma_addr + digest_ofs),
+			      digest_size, NS_BIT, 0);
 		set_flow_mode(&desc[idx], S_HASH_to_DOUT);
 		set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
 		set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
@@ -361,7 +378,7 @@ static int hmac_setkey(struct cc_hw_desc *desc, struct ssi_aead_ctx *ctx)
 static int validate_keys_sizes(struct ssi_aead_ctx *ctx)
 {
 	SSI_LOG_DEBUG("enc_keylen=%u  authkeylen=%u\n",
-		ctx->enc_keylen, ctx->auth_keylen);
+		      ctx->enc_keylen, ctx->auth_keylen);
 
 	switch (ctx->auth_mode) {
 	case DRV_HASH_SHA1:
@@ -385,7 +402,7 @@ static int validate_keys_sizes(struct ssi_aead_ctx *ctx)
 	if (unlikely(ctx->flow_mode == S_DIN_to_DES)) {
 		if (ctx->enc_keylen != DES3_EDE_KEY_SIZE) {
 			SSI_LOG_ERR("Invalid cipher(3DES) key size: %u\n",
-				ctx->enc_keylen);
+				    ctx->enc_keylen);
 			return -EINVAL;
 		}
 	} else { /* Default assumed to be AES ciphers */
@@ -393,7 +410,7 @@ static int validate_keys_sizes(struct ssi_aead_ctx *ctx)
 		    (ctx->enc_keylen != AES_KEYSIZE_192) &&
 		    (ctx->enc_keylen != AES_KEYSIZE_256)) {
 			SSI_LOG_ERR("Invalid cipher(AES) key size: %u\n",
-				ctx->enc_keylen);
+				    ctx->enc_keylen);
 			return -EINVAL;
 		}
 	}
@@ -536,9 +553,9 @@ ssi_aead_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int keylen)
 	int seq_len = 0, rc = -EINVAL;
 
 	SSI_LOG_DEBUG("Setting key in context @%p for %s. key=%p keylen=%u\n",
-		ctx, crypto_tfm_alg_name(crypto_aead_tfm(tfm)), key, keylen);
+		      ctx, crypto_tfm_alg_name(crypto_aead_tfm(tfm)),
+		      key, keylen);
 
-	CHECK_AND_RETURN_UPON_FIPS_ERROR();
 	/* STAT_PHASE_0: Init and sanity checks */
 
 	if (ctx->auth_mode != DRV_HASH_NULL) { /* authenc() alg. */
@@ -654,7 +671,6 @@ static int ssi_aead_setauthsize(
 {
 	struct ssi_aead_ctx *ctx = crypto_aead_ctx(authenc);
 
-	CHECK_AND_RETURN_UPON_FIPS_ERROR();
 	/* Unsupported auth. sizes */
 	if ((authsize == 0) ||
 	    (authsize > crypto_aead_maxauthsize(authenc))) {
@@ -669,7 +685,7 @@ static int ssi_aead_setauthsize(
 
 #if SSI_CC_HAS_AES_CCM
 static int ssi_rfc4309_ccm_setauthsize(struct crypto_aead *authenc,
-				      unsigned int authsize)
+				       unsigned int authsize)
 {
 	switch (authsize) {
 	case 8:
@@ -684,7 +700,7 @@ static int ssi_rfc4309_ccm_setauthsize(struct crypto_aead *authenc,
 }
 
 static int ssi_ccm_setauthsize(struct crypto_aead *authenc,
-				      unsigned int authsize)
+			       unsigned int authsize)
 {
 	switch (authsize) {
 	case 4:
@@ -762,11 +778,11 @@ ssi_aead_process_authenc_data_desc(
 	{
 		struct scatterlist *cipher =
 			(direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ?
-			areq_ctx->dstSgl : areq_ctx->srcSgl;
+			areq_ctx->dst_sgl : areq_ctx->src_sgl;
 
 		unsigned int offset =
 			(direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ?
-			areq_ctx->dstOffset : areq_ctx->srcOffset;
+			areq_ctx->dst_offset : areq_ctx->src_offset;
 		SSI_LOG_DEBUG("AUTHENC: SRC/DST buffer type DLLI\n");
 		hw_desc_init(&desc[idx]);
 		set_din_type(&desc[idx], DMA_DLLI,
@@ -828,11 +844,11 @@ ssi_aead_process_cipher_data_desc(
 		SSI_LOG_DEBUG("CIPHER: SRC/DST buffer type DLLI\n");
 		hw_desc_init(&desc[idx]);
 		set_din_type(&desc[idx], DMA_DLLI,
-			     (sg_dma_address(areq_ctx->srcSgl) +
-			      areq_ctx->srcOffset), areq_ctx->cryptlen, NS_BIT);
+			     (sg_dma_address(areq_ctx->src_sgl) +
+			      areq_ctx->src_offset), areq_ctx->cryptlen, NS_BIT);
 		set_dout_dlli(&desc[idx],
-			      (sg_dma_address(areq_ctx->dstSgl) +
-			       areq_ctx->dstOffset),
+			      (sg_dma_address(areq_ctx->dst_sgl) +
+			       areq_ctx->dst_offset),
 			      areq_ctx->cryptlen, NS_BIT, 0);
 		set_flow_mode(&desc[idx], flow_mode);
 		break;
@@ -1168,8 +1184,8 @@ static inline void ssi_aead_load_mlli_to_sram(
 		(req_ctx->data_buff_type == SSI_DMA_BUF_MLLI) ||
 		!req_ctx->is_single_pass)) {
 		SSI_LOG_DEBUG("Copy-to-sram: mlli_dma=%08x, mlli_size=%u\n",
-			(unsigned int)ctx->drvdata->mlli_sram_addr,
-			req_ctx->mlli_params.mlli_len);
+			      (unsigned int)ctx->drvdata->mlli_sram_addr,
+			      req_ctx->mlli_params.mlli_len);
 		/* Copy MLLI table host-to-sram */
 		hw_desc_init(&desc[*seq_size]);
 		set_din_type(&desc[*seq_size], DMA_DLLI,
@@ -1313,7 +1329,8 @@ ssi_aead_xcbc_authenc(
 }
 
 static int validate_data_size(struct ssi_aead_ctx *ctx,
-	enum drv_crypto_direction direct, struct aead_request *req)
+			      enum drv_crypto_direction direct,
+			      struct aead_request *req)
 {
 	struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
 	unsigned int assoclen = req->assoclen;
@@ -1321,7 +1338,7 @@ static int validate_data_size(struct ssi_aead_ctx *ctx,
 			(req->cryptlen - ctx->authsize) : req->cryptlen;
 
 	if (unlikely((direct == DRV_CRYPTO_DIRECTION_DECRYPT) &&
-		(req->cryptlen < ctx->authsize)))
+		     (req->cryptlen < ctx->authsize)))
 		goto data_size_err;
 
 	areq_ctx->is_single_pass = true; /*defaulted to fast flow*/
@@ -1329,7 +1346,7 @@ static int validate_data_size(struct ssi_aead_ctx *ctx,
 	switch (ctx->flow_mode) {
 	case S_DIN_to_AES:
 		if (unlikely((ctx->cipher_mode == DRV_CIPHER_CBC) &&
-			!IS_ALIGNED(cipherlen, AES_BLOCK_SIZE)))
+			     !IS_ALIGNED(cipherlen, AES_BLOCK_SIZE)))
 			goto data_size_err;
 		if (ctx->cipher_mode == DRV_CIPHER_CCM)
 			break;
@@ -1365,27 +1382,27 @@ data_size_err:
 }
 
 #if SSI_CC_HAS_AES_CCM
-static unsigned int format_ccm_a0(u8 *pA0Buff, u32 headerSize)
+static unsigned int format_ccm_a0(u8 *pa0_buff, u32 header_size)
 {
 	unsigned int len = 0;
 
-	if (headerSize == 0)
+	if (header_size == 0)
 		return 0;
 
-	if (headerSize < ((1UL << 16) - (1UL << 8))) {
+	if (header_size < ((1UL << 16) - (1UL << 8))) {
 		len = 2;
 
-		pA0Buff[0] = (headerSize >> 8) & 0xFF;
-		pA0Buff[1] = headerSize & 0xFF;
+		pa0_buff[0] = (header_size >> 8) & 0xFF;
+		pa0_buff[1] = header_size & 0xFF;
 	} else {
 		len = 6;
 
-		pA0Buff[0] = 0xFF;
-		pA0Buff[1] = 0xFE;
-		pA0Buff[2] = (headerSize >> 24) & 0xFF;
-		pA0Buff[3] = (headerSize >> 16) & 0xFF;
-		pA0Buff[4] = (headerSize >> 8) & 0xFF;
-		pA0Buff[5] = headerSize & 0xFF;
+		pa0_buff[0] = 0xFF;
+		pa0_buff[1] = 0xFE;
+		pa0_buff[2] = (header_size >> 24) & 0xFF;
+		pa0_buff[3] = (header_size >> 16) & 0xFF;
+		pa0_buff[4] = (header_size >> 8) & 0xFF;
+		pa0_buff[5] = header_size & 0xFF;
 	}
 
 	return len;
@@ -1557,7 +1574,7 @@ static int config_ccm_adata(struct aead_request *req)
 
 	/* taken from crypto/ccm.c */
 	/* 2 <= L <= 8, so 1 <= L' <= 7. */
-	if (2 > l || l > 8) {
+	if (l < 2 || l > 8) {
 		SSI_LOG_ERR("illegal iv value %X\n", req->iv[0]);
 		return -EINVAL;
 	}
@@ -1848,8 +1865,9 @@ static inline void ssi_aead_dump_gcm(
 		SSI_LOG_DEBUG("%s\n", title);
 	}
 
-	SSI_LOG_DEBUG("cipher_mode %d, authsize %d, enc_keylen %d, assoclen %d, cryptlen %d\n", \
-				 ctx->cipher_mode, ctx->authsize, ctx->enc_keylen, req->assoclen, req_ctx->cryptlen);
+	SSI_LOG_DEBUG("cipher_mode %d, authsize %d, enc_keylen %d, assoclen %d, cryptlen %d\n",
+		      ctx->cipher_mode, ctx->authsize, ctx->enc_keylen,
+		      req->assoclen, req_ctx->cryptlen);
 
 	if (ctx->enckey)
 		dump_byte_array("mac key", ctx->enckey, 16);
@@ -1864,7 +1882,7 @@ static inline void ssi_aead_dump_gcm(
 
 	dump_byte_array("mac_buf", req_ctx->mac_buf, AES_BLOCK_SIZE);
 
-	dump_byte_array("gcm_len_block", req_ctx->gcm_len_block.lenA, AES_BLOCK_SIZE);
+	dump_byte_array("gcm_len_block", req_ctx->gcm_len_block.len_a, AES_BLOCK_SIZE);
 
 	if (req->src && req->cryptlen)
 		dump_byte_array("req->src", sg_virt(req->src), req->cryptlen + req->assoclen);
@@ -1886,7 +1904,7 @@ static int config_gcm_context(struct aead_request *req)
 				(req->cryptlen - ctx->authsize);
 	__be32 counter = cpu_to_be32(2);
 
-	SSI_LOG_DEBUG("config_gcm_context() cryptlen = %d, req->assoclen = %d ctx->authsize = %d\n", cryptlen, req->assoclen, ctx->authsize);
+	SSI_LOG_DEBUG("%s() cryptlen = %d, req->assoclen = %d ctx->authsize = %d\n", __func__, cryptlen, req->assoclen, ctx->authsize);
 
 	memset(req_ctx->hkey, 0, AES_BLOCK_SIZE);
 
@@ -1903,16 +1921,16 @@ static int config_gcm_context(struct aead_request *req)
 		__be64 temp64;
 
 		temp64 = cpu_to_be64(req->assoclen * 8);
-		memcpy(&req_ctx->gcm_len_block.lenA, &temp64, sizeof(temp64));
+		memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64));
 		temp64 = cpu_to_be64(cryptlen * 8);
-		memcpy(&req_ctx->gcm_len_block.lenC, &temp64, 8);
+		memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8);
 	} else { //rfc4543=>  all data(AAD,IV,Plain) are considered additional data that is nothing is encrypted.
 		__be64 temp64;
 
 		temp64 = cpu_to_be64((req->assoclen + GCM_BLOCK_RFC4_IV_SIZE + cryptlen) * 8);
-		memcpy(&req_ctx->gcm_len_block.lenA, &temp64, sizeof(temp64));
+		memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64));
 		temp64 = 0;
-		memcpy(&req_ctx->gcm_len_block.lenC, &temp64, 8);
+		memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8);
 	}
 
 	return 0;
@@ -1944,16 +1962,16 @@ static int ssi_aead_process(struct aead_request *req, enum drv_crypto_direction
 	struct ssi_crypto_req ssi_req = {};
 
 	SSI_LOG_DEBUG("%s context=%p req=%p iv=%p src=%p src_ofs=%d dst=%p dst_ofs=%d cryptolen=%d\n",
-		((direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? "Encrypt" : "Decrypt"), ctx, req, req->iv,
-		sg_virt(req->src), req->src->offset, sg_virt(req->dst), req->dst->offset, req->cryptlen);
-	CHECK_AND_RETURN_UPON_FIPS_ERROR();
+		      ((direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? "Encrypt" : "Decrypt"),
+		      ctx, req, req->iv, sg_virt(req->src), req->src->offset,
+		      sg_virt(req->dst), req->dst->offset, req->cryptlen);
 
 	/* STAT_PHASE_0: Init and sanity checks */
 
 	/* Check data length according to mode */
 	if (unlikely(validate_data_size(ctx, direct, req) != 0)) {
 		SSI_LOG_ERR("Unsupported crypt/assoc len %d/%d.\n",
-				req->cryptlen, req->assoclen);
+			    req->cryptlen, req->assoclen);
 		crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_BLOCK_LEN);
 		return -EINVAL;
 	}
@@ -1976,7 +1994,7 @@ static int ssi_aead_process(struct aead_request *req, enum drv_crypto_direction
 		memcpy(areq_ctx->ctr_iv, ctx->ctr_nonce, CTR_RFC3686_NONCE_SIZE);
 		if (!areq_ctx->backup_giv) /*User none-generated IV*/
 			memcpy(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE,
-				req->iv, CTR_RFC3686_IV_SIZE);
+			       req->iv, CTR_RFC3686_IV_SIZE);
 		/* Initialize counter portion of counter block */
 		*(__be32 *)(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE +
 			    CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
@@ -2198,7 +2216,7 @@ static int ssi_rfc4106_gcm_setkey(struct crypto_aead *tfm, const u8 *key, unsign
 	struct ssi_aead_ctx *ctx = crypto_aead_ctx(tfm);
 	int rc = 0;
 
-	SSI_LOG_DEBUG("ssi_rfc4106_gcm_setkey()  keylen %d, key %p\n", keylen, key);
+	SSI_LOG_DEBUG("%s()  keylen %d, key %p\n", __func__, keylen, key);
 
 	if (keylen < 4)
 		return -EINVAL;
@@ -2216,7 +2234,7 @@ static int ssi_rfc4543_gcm_setkey(struct crypto_aead *tfm, const u8 *key, unsign
 	struct ssi_aead_ctx *ctx = crypto_aead_ctx(tfm);
 	int rc = 0;
 
-	SSI_LOG_DEBUG("ssi_rfc4543_gcm_setkey()  keylen %d, key %p\n", keylen, key);
+	SSI_LOG_DEBUG("%s()  keylen %d, key %p\n", __func__, keylen, key);
 
 	if (keylen < 4)
 		return -EINVAL;
@@ -2230,7 +2248,7 @@ static int ssi_rfc4543_gcm_setkey(struct crypto_aead *tfm, const u8 *key, unsign
 }
 
 static int ssi_gcm_setauthsize(struct crypto_aead *authenc,
-				      unsigned int authsize)
+			       unsigned int authsize)
 {
 	switch (authsize) {
 	case 4:
@@ -2249,9 +2267,9 @@ static int ssi_gcm_setauthsize(struct crypto_aead *authenc,
 }
 
 static int ssi_rfc4106_gcm_setauthsize(struct crypto_aead *authenc,
-				      unsigned int authsize)
+				       unsigned int authsize)
 {
-	SSI_LOG_DEBUG("ssi_rfc4106_gcm_setauthsize()  authsize %d\n", authsize);
+	SSI_LOG_DEBUG("authsize %d\n", authsize);
 
 	switch (authsize) {
 	case 8:
@@ -2268,7 +2286,7 @@ static int ssi_rfc4106_gcm_setauthsize(struct crypto_aead *authenc,
 static int ssi_rfc4543_gcm_setauthsize(struct crypto_aead *authenc,
 				       unsigned int authsize)
 {
-	SSI_LOG_DEBUG("ssi_rfc4543_gcm_setauthsize()  authsize %d\n", authsize);
+	SSI_LOG_DEBUG("authsize %d\n", authsize);
 
 	if (authsize != 16)
 		return -EINVAL;
@@ -2641,7 +2659,7 @@ static struct ssi_crypto_alg *ssi_aead_create_alg(struct ssi_alg_template *templ
 	struct ssi_crypto_alg *t_alg;
 	struct aead_alg *alg;
 
-	t_alg = kzalloc(sizeof(struct ssi_crypto_alg), GFP_KERNEL);
+	t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
 	if (!t_alg) {
 		SSI_LOG_ERR("failed to allocate t_alg\n");
 		return ERR_PTR(-ENOMEM);
@@ -2696,7 +2714,7 @@ int ssi_aead_alloc(struct ssi_drvdata *drvdata)
 	int rc = -ENOMEM;
 	int alg;
 
-	aead_handle = kmalloc(sizeof(struct ssi_aead_handle), GFP_KERNEL);
+	aead_handle = kmalloc(sizeof(*aead_handle), GFP_KERNEL);
 	if (!aead_handle) {
 		rc = -ENOMEM;
 		goto fail0;
@@ -2720,14 +2738,14 @@ int ssi_aead_alloc(struct ssi_drvdata *drvdata)
 		if (IS_ERR(t_alg)) {
 			rc = PTR_ERR(t_alg);
 			SSI_LOG_ERR("%s alg allocation failed\n",
-				 aead_algs[alg].driver_name);
+				    aead_algs[alg].driver_name);
 			goto fail1;
 		}
 		t_alg->drvdata = drvdata;
 		rc = crypto_register_aead(&t_alg->aead_alg);
 		if (unlikely(rc != 0)) {
 			SSI_LOG_ERR("%s alg registration failed\n",
-				t_alg->aead_alg.base.cra_driver_name);
+				    t_alg->aead_alg.base.cra_driver_name);
 			goto fail2;
 		} else {
 			list_add_tail(&t_alg->entry, &aead_handle->aead_list);
diff --git a/drivers/staging/ccree/ssi_aead.h b/drivers/staging/ccree/ssi_aead.h
index 39cc633a3ffa..e85bcd917e7b 100644
--- a/drivers/staging/ccree/ssi_aead.h
+++ b/drivers/staging/ccree/ssi_aead.h
@@ -69,8 +69,8 @@ struct aead_req_ctx {
 	u8 gcm_iv_inc2[AES_BLOCK_SIZE] ____cacheline_aligned;
 	u8 hkey[AES_BLOCK_SIZE] ____cacheline_aligned;
 	struct {
-		u8 lenA[GCM_BLOCK_LEN_SIZE] ____cacheline_aligned;
-		u8 lenC[GCM_BLOCK_LEN_SIZE];
+		u8 len_a[GCM_BLOCK_LEN_SIZE] ____cacheline_aligned;
+		u8 len_c[GCM_BLOCK_LEN_SIZE];
 	} gcm_len_block;
 
 	u8 ccm_config[CCM_CONFIG_BUF_SIZE] ____cacheline_aligned;
@@ -94,10 +94,10 @@ struct aead_req_ctx {
 	struct ssi_mlli assoc;
 	struct ssi_mlli src;
 	struct ssi_mlli dst;
-	struct scatterlist *srcSgl;
-	struct scatterlist *dstSgl;
-	unsigned int srcOffset;
-	unsigned int dstOffset;
+	struct scatterlist *src_sgl;
+	struct scatterlist *dst_sgl;
+	unsigned int src_offset;
+	unsigned int dst_offset;
 	enum ssi_req_dma_buf_type assoc_buff_type;
 	enum ssi_req_dma_buf_type data_buff_type;
 	struct mlli_params mlli_params;
diff --git a/drivers/staging/ccree/ssi_buffer_mgr.c b/drivers/staging/ccree/ssi_buffer_mgr.c
index b35871eeabd1..63936091d524 100644
--- a/drivers/staging/ccree/ssi_buffer_mgr.c
+++ b/drivers/staging/ccree/ssi_buffer_mgr.c
@@ -150,7 +150,7 @@ static inline int ssi_buffer_mgr_render_buff_to_mlli(
 	u32 **mlli_entry_pp)
 {
 	u32 *mlli_entry_p = *mlli_entry_pp;
-	u32 new_nents;;
+	u32 new_nents;
 
 	/* Verify there is no memory overflow*/
 	new_nents = (*curr_nents + buff_size / CC_MAX_MLLI_ENTRY_SIZE + 1);
@@ -162,8 +162,8 @@ static inline int ssi_buffer_mgr_render_buff_to_mlli(
 		cc_lli_set_addr(mlli_entry_p, buff_dma);
 		cc_lli_set_size(mlli_entry_p, CC_MAX_MLLI_ENTRY_SIZE);
 		SSI_LOG_DEBUG("entry[%d]: single_buff=0x%08X size=%08X\n", *curr_nents,
-			   mlli_entry_p[LLI_WORD0_OFFSET],
-			   mlli_entry_p[LLI_WORD1_OFFSET]);
+			      mlli_entry_p[LLI_WORD0_OFFSET],
+			      mlli_entry_p[LLI_WORD1_OFFSET]);
 		buff_dma += CC_MAX_MLLI_ENTRY_SIZE;
 		buff_size -= CC_MAX_MLLI_ENTRY_SIZE;
 		mlli_entry_p = mlli_entry_p + 2;
@@ -173,8 +173,8 @@ static inline int ssi_buffer_mgr_render_buff_to_mlli(
 	cc_lli_set_addr(mlli_entry_p, buff_dma);
 	cc_lli_set_size(mlli_entry_p, buff_size);
 	SSI_LOG_DEBUG("entry[%d]: single_buff=0x%08X size=%08X\n", *curr_nents,
-		   mlli_entry_p[LLI_WORD0_OFFSET],
-		   mlli_entry_p[LLI_WORD1_OFFSET]);
+		      mlli_entry_p[LLI_WORD0_OFFSET],
+		      mlli_entry_p[LLI_WORD1_OFFSET]);
 	mlli_entry_p = mlli_entry_p + 2;
 	*mlli_entry_pp = mlli_entry_p;
 	(*curr_nents)++;
@@ -182,8 +182,8 @@ static inline int ssi_buffer_mgr_render_buff_to_mlli(
 }
 
 static inline int ssi_buffer_mgr_render_scatterlist_to_mlli(
-	struct scatterlist *sgl, u32 sgl_data_len, u32 sglOffset, u32 *curr_nents,
-	u32 **mlli_entry_pp)
+	struct scatterlist *sgl, u32 sgl_data_len, u32 sgl_offset,
+	u32 *curr_nents, u32 **mlli_entry_pp)
 {
 	struct scatterlist *curr_sgl = sgl;
 	u32 *mlli_entry_p = *mlli_entry_pp;
@@ -192,16 +192,17 @@ static inline int ssi_buffer_mgr_render_scatterlist_to_mlli(
 	for ( ; (curr_sgl) && (sgl_data_len != 0);
 	      curr_sgl = sg_next(curr_sgl)) {
 		u32 entry_data_len =
-			(sgl_data_len > sg_dma_len(curr_sgl) - sglOffset) ?
-				sg_dma_len(curr_sgl) - sglOffset : sgl_data_len;
+			(sgl_data_len > sg_dma_len(curr_sgl) - sgl_offset) ?
+				sg_dma_len(curr_sgl) - sgl_offset :
+				sgl_data_len;
 		sgl_data_len -= entry_data_len;
 		rc = ssi_buffer_mgr_render_buff_to_mlli(
-			sg_dma_address(curr_sgl) + sglOffset, entry_data_len, curr_nents,
-			&mlli_entry_p);
+			sg_dma_address(curr_sgl) + sgl_offset, entry_data_len,
+			curr_nents, &mlli_entry_p);
 		if (rc != 0)
 			return rc;
 
-		sglOffset = 0;
+		sgl_offset = 0;
 	}
 	*mlli_entry_pp = mlli_entry_p;
 	return 0;
@@ -221,7 +222,7 @@ static int ssi_buffer_mgr_generate_mlli(
 	/* Allocate memory from the pointed pool */
 	mlli_params->mlli_virt_addr = dma_pool_alloc(
 			mlli_params->curr_pool, GFP_KERNEL,
-			&(mlli_params->mlli_dma_addr));
+			&mlli_params->mlli_dma_addr);
 	if (unlikely(!mlli_params->mlli_virt_addr)) {
 		SSI_LOG_ERR("dma_pool_alloc() failed\n");
 		rc = -ENOMEM;
@@ -249,7 +250,7 @@ static int ssi_buffer_mgr_generate_mlli(
 			/*Calculate the current MLLI table length for the
 			 *length field in the descriptor
 			 */
-			*(sg_data->mlli_nents[i]) +=
+			*sg_data->mlli_nents[i] +=
 				(total_nents - prev_total_nents);
 			prev_total_nents = total_nents;
 		}
@@ -259,9 +260,9 @@ static int ssi_buffer_mgr_generate_mlli(
 	mlli_params->mlli_len = (total_nents * LLI_ENTRY_BYTE_SIZE);
 
 	SSI_LOG_DEBUG("MLLI params: "
-		     "virt_addr=%pK dma_addr=0x%llX mlli_len=0x%X\n",
+		     "virt_addr=%pK dma_addr=%pad mlli_len=0x%X\n",
 		   mlli_params->mlli_virt_addr,
-		   (unsigned long long)mlli_params->mlli_dma_addr,
+		   mlli_params->mlli_dma_addr,
 		   mlli_params->mlli_len);
 
 build_mlli_exit:
@@ -275,9 +276,9 @@ static inline void ssi_buffer_mgr_add_buffer_entry(
 {
 	unsigned int index = sgl_data->num_of_buffers;
 
-	SSI_LOG_DEBUG("index=%u single_buff=0x%llX "
+	SSI_LOG_DEBUG("index=%u single_buff=%pad "
 		     "buffer_len=0x%08X is_last=%d\n",
-		     index, (unsigned long long)buffer_dma, buffer_len, is_last_entry);
+		     index, buffer_dma, buffer_len, is_last_entry);
 	sgl_data->nents[index] = 1;
 	sgl_data->entry[index].buffer_dma = buffer_dma;
 	sgl_data->offset[index] = 0;
@@ -302,7 +303,7 @@ static inline void ssi_buffer_mgr_add_scatterlist_entry(
 	unsigned int index = sgl_data->num_of_buffers;
 
 	SSI_LOG_DEBUG("index=%u nents=%u sgl=%pK data_len=0x%08X is_last=%d\n",
-		     index, nents, sgl, data_len, is_last_table);
+		      index, nents, sgl, data_len, is_last_table);
 	sgl_data->nents[index] = nents;
 	sgl_data->entry[index].sgl = sgl;
 	sgl_data->offset[index] = data_offset;
@@ -317,7 +318,7 @@ static inline void ssi_buffer_mgr_add_scatterlist_entry(
 
 static int
 ssi_buffer_mgr_dma_map_sg(struct device *dev, struct scatterlist *sg, u32 nents,
-			 enum dma_data_direction direction)
+			  enum dma_data_direction direction)
 {
 	u32 i, j;
 	struct scatterlist *l_sg = sg;
@@ -358,10 +359,10 @@ static int ssi_buffer_mgr_map_scatterlist(
 			SSI_LOG_ERR("dma_map_sg() single buffer failed\n");
 			return -ENOMEM;
 		}
-		SSI_LOG_DEBUG("Mapped sg: dma_address=0x%llX "
+		SSI_LOG_DEBUG("Mapped sg: dma_address=%pad "
 			     "page=%p addr=%pK offset=%u "
 			     "length=%u\n",
-			     (unsigned long long)sg_dma_address(sg),
+			     sg_dma_address(sg),
 			     sg_page(sg),
 			     sg_virt(sg),
 			     sg->offset, sg->length);
@@ -370,11 +371,11 @@ static int ssi_buffer_mgr_map_scatterlist(
 		*mapped_nents = 1;
 	} else {  /*sg_is_last*/
 		*nents = ssi_buffer_mgr_get_sgl_nents(sg, nbytes, lbytes,
-						     &is_chained);
+						      &is_chained);
 		if (*nents > max_sg_nents) {
 			*nents = 0;
 			SSI_LOG_ERR("Too many fragments. current %d max %d\n",
-				   *nents, max_sg_nents);
+				    *nents, max_sg_nents);
 			return -ENOMEM;
 		}
 		if (!is_chained) {
@@ -392,9 +393,9 @@ static int ssi_buffer_mgr_map_scatterlist(
 			 * must have the same nents before and after map
 			 */
 			*mapped_nents = ssi_buffer_mgr_dma_map_sg(dev,
-								 sg,
-								 *nents,
-								 direction);
+								  sg,
+								  *nents,
+								  direction);
 			if (unlikely(*mapped_nents != *nents)) {
 				*nents = *mapped_nents;
 				SSI_LOG_ERR("dma_map_sg() sg buffer failed\n");
@@ -408,10 +409,10 @@ static int ssi_buffer_mgr_map_scatterlist(
 
 static inline int
 ssi_aead_handle_config_buf(struct device *dev,
-	struct aead_req_ctx *areq_ctx,
-	u8 *config_data,
-	struct buffer_array *sg_data,
-	unsigned int assoclen)
+			   struct aead_req_ctx *areq_ctx,
+			   u8 *config_data,
+			   struct buffer_array *sg_data,
+			   unsigned int assoclen)
 {
 	SSI_LOG_DEBUG(" handle additional data config set to   DLLI\n");
 	/* create sg for the current buffer */
@@ -422,10 +423,10 @@ ssi_aead_handle_config_buf(struct device *dev,
 			   "config buffer failed\n");
 			return -ENOMEM;
 	}
-	SSI_LOG_DEBUG("Mapped curr_buff: dma_address=0x%llX "
+	SSI_LOG_DEBUG("Mapped curr_buff: dma_address=%pad "
 		     "page=%p addr=%pK "
 		     "offset=%u length=%u\n",
-		     (unsigned long long)sg_dma_address(&areq_ctx->ccm_adata_sg),
+		     sg_dma_address(&areq_ctx->ccm_adata_sg),
 		     sg_page(&areq_ctx->ccm_adata_sg),
 		     sg_virt(&areq_ctx->ccm_adata_sg),
 		     areq_ctx->ccm_adata_sg.offset,
@@ -433,19 +434,18 @@ ssi_aead_handle_config_buf(struct device *dev,
 	/* prepare for case of MLLI */
 	if (assoclen > 0) {
 		ssi_buffer_mgr_add_scatterlist_entry(sg_data, 1,
-						    &areq_ctx->ccm_adata_sg,
-						    (AES_BLOCK_SIZE +
-						    areq_ctx->ccm_hdr_size), 0,
-						    false, NULL);
+						     &areq_ctx->ccm_adata_sg,
+						     (AES_BLOCK_SIZE + areq_ctx->ccm_hdr_size),
+						     0, false, NULL);
 	}
 	return 0;
 }
 
 static inline int ssi_ahash_handle_curr_buf(struct device *dev,
-					   struct ahash_req_ctx *areq_ctx,
-					   u8 *curr_buff,
-					   u32 curr_buff_cnt,
-					   struct buffer_array *sg_data)
+					    struct ahash_req_ctx *areq_ctx,
+					    u8 *curr_buff,
+					    u32 curr_buff_cnt,
+					    struct buffer_array *sg_data)
 {
 	SSI_LOG_DEBUG(" handle curr buff %x set to   DLLI\n", curr_buff_cnt);
 	/* create sg for the current buffer */
@@ -456,10 +456,10 @@ static inline int ssi_ahash_handle_curr_buf(struct device *dev,
 			   "src buffer failed\n");
 			return -ENOMEM;
 	}
-	SSI_LOG_DEBUG("Mapped curr_buff: dma_address=0x%llX "
+	SSI_LOG_DEBUG("Mapped curr_buff: dma_address=%pad "
 		     "page=%p addr=%pK "
 		     "offset=%u length=%u\n",
-		     (unsigned long long)sg_dma_address(areq_ctx->buff_sg),
+		     sg_dma_address(areq_ctx->buff_sg),
 		     sg_page(areq_ctx->buff_sg),
 		     sg_virt(areq_ctx->buff_sg),
 		     areq_ctx->buff_sg->offset,
@@ -469,7 +469,7 @@ static inline int ssi_ahash_handle_curr_buf(struct device *dev,
 	areq_ctx->in_nents = 0;
 	/* prepare for case of MLLI */
 	ssi_buffer_mgr_add_scatterlist_entry(sg_data, 1, areq_ctx->buff_sg,
-				curr_buff_cnt, 0, false, NULL);
+					     curr_buff_cnt, 0, false, NULL);
 	return 0;
 }
 
@@ -483,9 +483,9 @@ void ssi_buffer_mgr_unmap_blkcipher_request(
 	struct blkcipher_req_ctx *req_ctx = (struct blkcipher_req_ctx *)ctx;
 
 	if (likely(req_ctx->gen_ctx.iv_dma_addr != 0)) {
-		SSI_LOG_DEBUG("Unmapped iv: iv_dma_addr=0x%llX iv_size=%u\n",
-			(unsigned long long)req_ctx->gen_ctx.iv_dma_addr,
-			ivsize);
+		SSI_LOG_DEBUG("Unmapped iv: iv_dma_addr=%pad iv_size=%u\n",
+			      req_ctx->gen_ctx.iv_dma_addr,
+			      ivsize);
 		dma_unmap_single(dev, req_ctx->gen_ctx.iv_dma_addr,
 				 ivsize,
 				 req_ctx->is_giv ? DMA_BIDIRECTIONAL :
@@ -498,16 +498,12 @@ void ssi_buffer_mgr_unmap_blkcipher_request(
 			      req_ctx->mlli_params.mlli_dma_addr);
 	}
 
-	dma_unmap_sg(dev, src, req_ctx->in_nents,
-		DMA_BIDIRECTIONAL);
-	SSI_LOG_DEBUG("Unmapped req->src=%pK\n",
-		     sg_virt(src));
+	dma_unmap_sg(dev, src, req_ctx->in_nents, DMA_BIDIRECTIONAL);
+	SSI_LOG_DEBUG("Unmapped req->src=%pK\n", sg_virt(src));
 
 	if (src != dst) {
-		dma_unmap_sg(dev, dst, req_ctx->out_nents,
-			DMA_BIDIRECTIONAL);
-		SSI_LOG_DEBUG("Unmapped req->dst=%pK\n",
-			sg_virt(dst));
+		dma_unmap_sg(dev, dst, req_ctx->out_nents, DMA_BIDIRECTIONAL);
+		SSI_LOG_DEBUG("Unmapped req->dst=%pK\n", sg_virt(dst));
 	}
 }
 
@@ -542,22 +538,24 @@ int ssi_buffer_mgr_map_blkcipher_request(
 				       req_ctx->is_giv ? DMA_BIDIRECTIONAL :
 				       DMA_TO_DEVICE);
 		if (unlikely(dma_mapping_error(dev,
-					req_ctx->gen_ctx.iv_dma_addr))) {
+					       req_ctx->gen_ctx.iv_dma_addr))) {
 			SSI_LOG_ERR("Mapping iv %u B at va=%pK "
 				   "for DMA failed\n", ivsize, info);
 			return -ENOMEM;
 		}
-		SSI_LOG_DEBUG("Mapped iv %u B at va=%pK to dma=0x%llX\n",
-			ivsize, info,
-			(unsigned long long)req_ctx->gen_ctx.iv_dma_addr);
+		SSI_LOG_DEBUG("Mapped iv %u B at va=%pK to dma=%pad\n",
+			      ivsize, info,
+			      req_ctx->gen_ctx.iv_dma_addr);
 	} else {
 		req_ctx->gen_ctx.iv_dma_addr = 0;
 	}
 
 	/* Map the src SGL */
 	rc = ssi_buffer_mgr_map_scatterlist(dev, src,
-		nbytes, DMA_BIDIRECTIONAL, &req_ctx->in_nents,
-		LLI_MAX_NUM_OF_DATA_ENTRIES, &dummy, &mapped_nents);
+					    nbytes, DMA_BIDIRECTIONAL,
+					    &req_ctx->in_nents,
+					    LLI_MAX_NUM_OF_DATA_ENTRIES, &dummy,
+					    &mapped_nents);
 	if (unlikely(rc != 0)) {
 		rc = -ENOMEM;
 		goto ablkcipher_exit;
@@ -570,8 +568,10 @@ int ssi_buffer_mgr_map_blkcipher_request(
 		if (unlikely(req_ctx->dma_buf_type == SSI_DMA_BUF_MLLI)) {
 			req_ctx->out_nents = 0;
 			ssi_buffer_mgr_add_scatterlist_entry(&sg_data,
-				req_ctx->in_nents, src,
-				nbytes, 0, true, &req_ctx->in_mlli_nents);
+							     req_ctx->in_nents,
+							     src, nbytes, 0,
+							     true,
+							     &req_ctx->in_mlli_nents);
 		}
 	} else {
 		/* Map the dst sg */
@@ -588,13 +588,15 @@ int ssi_buffer_mgr_map_blkcipher_request(
 
 		if (unlikely((req_ctx->dma_buf_type == SSI_DMA_BUF_MLLI))) {
 			ssi_buffer_mgr_add_scatterlist_entry(&sg_data,
-				req_ctx->in_nents, src,
-				nbytes, 0, true,
-				&req_ctx->in_mlli_nents);
+							     req_ctx->in_nents,
+							     src, nbytes, 0,
+							     true,
+							     &req_ctx->in_mlli_nents);
 			ssi_buffer_mgr_add_scatterlist_entry(&sg_data,
-				req_ctx->out_nents, dst,
-				nbytes, 0, true,
-				&req_ctx->out_mlli_nents);
+							     req_ctx->out_nents,
+							     dst, nbytes, 0,
+							     true,
+							     &req_ctx->out_mlli_nents);
 		}
 	}
 
@@ -606,7 +608,7 @@ int ssi_buffer_mgr_map_blkcipher_request(
 	}
 
 	SSI_LOG_DEBUG("areq_ctx->dma_buf_type = %s\n",
-		GET_DMA_BUFFER_TYPE(req_ctx->dma_buf_type));
+		      GET_DMA_BUFFER_TYPE(req_ctx->dma_buf_type));
 
 	return 0;
 
@@ -628,7 +630,7 @@ void ssi_buffer_mgr_unmap_aead_request(
 
 	if (areq_ctx->mac_buf_dma_addr != 0) {
 		dma_unmap_single(dev, areq_ctx->mac_buf_dma_addr,
-			MAX_MAC_SIZE, DMA_BIDIRECTIONAL);
+				 MAX_MAC_SIZE, DMA_BIDIRECTIONAL);
 	}
 
 #if SSI_CC_HAS_AES_GCM
@@ -645,12 +647,12 @@ void ssi_buffer_mgr_unmap_aead_request(
 
 		if (areq_ctx->gcm_iv_inc1_dma_addr != 0) {
 			dma_unmap_single(dev, areq_ctx->gcm_iv_inc1_dma_addr,
-				AES_BLOCK_SIZE, DMA_TO_DEVICE);
+					 AES_BLOCK_SIZE, DMA_TO_DEVICE);
 		}
 
 		if (areq_ctx->gcm_iv_inc2_dma_addr != 0) {
 			dma_unmap_single(dev, areq_ctx->gcm_iv_inc2_dma_addr,
-				AES_BLOCK_SIZE, DMA_TO_DEVICE);
+					 AES_BLOCK_SIZE, DMA_TO_DEVICE);
 		}
 	}
 #endif
@@ -658,7 +660,7 @@ void ssi_buffer_mgr_unmap_aead_request(
 	if (areq_ctx->ccm_hdr_size != ccm_header_size_null) {
 		if (areq_ctx->ccm_iv0_dma_addr != 0) {
 			dma_unmap_single(dev, areq_ctx->ccm_iv0_dma_addr,
-				AES_BLOCK_SIZE, DMA_TO_DEVICE);
+					 AES_BLOCK_SIZE, DMA_TO_DEVICE);
 		}
 
 		dma_unmap_sg(dev, &areq_ctx->ccm_adata_sg, 1, DMA_TO_DEVICE);
@@ -672,9 +674,9 @@ void ssi_buffer_mgr_unmap_aead_request(
 	 *allocated and should be released
 	 */
 	if (areq_ctx->mlli_params.curr_pool) {
-		SSI_LOG_DEBUG("free MLLI buffer: dma=0x%08llX virt=%pK\n",
-			(unsigned long long)areq_ctx->mlli_params.mlli_dma_addr,
-			areq_ctx->mlli_params.mlli_virt_addr);
+		SSI_LOG_DEBUG("free MLLI buffer: dma=%pad virt=%pK\n",
+			      areq_ctx->mlli_params.mlli_dma_addr,
+			      areq_ctx->mlli_params.mlli_virt_addr);
 		dma_pool_free(areq_ctx->mlli_params.curr_pool,
 			      areq_ctx->mlli_params.mlli_virt_addr,
 			      areq_ctx->mlli_params.mlli_dma_addr);
@@ -690,14 +692,17 @@ void ssi_buffer_mgr_unmap_aead_request(
 	dma_unmap_sg(dev, req->src, ssi_buffer_mgr_get_sgl_nents(req->src, size_to_unmap, &dummy, &chained), DMA_BIDIRECTIONAL);
 	if (unlikely(req->src != req->dst)) {
 		SSI_LOG_DEBUG("Unmapping dst sgl: req->dst=%pK\n",
-			sg_virt(req->dst));
-		dma_unmap_sg(dev, req->dst, ssi_buffer_mgr_get_sgl_nents(req->dst, size_to_unmap, &dummy, &chained),
-			DMA_BIDIRECTIONAL);
+			      sg_virt(req->dst));
+		dma_unmap_sg(dev, req->dst,
+			     ssi_buffer_mgr_get_sgl_nents(req->dst,
+							  size_to_unmap,
+							  &dummy,
+							  &chained),
+			     DMA_BIDIRECTIONAL);
 	}
 	if (drvdata->coherent &&
 	    (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) &&
-	    likely(req->src == req->dst))
-	{
+	    likely(req->src == req->dst)) {
 		u32 size_to_skip = req->assoclen;
 
 		if (areq_ctx->is_gcm4543)
@@ -753,11 +758,11 @@ static inline int ssi_buffer_mgr_get_aead_icv_nents(
 		*is_icv_fragmented = true;
 	} else {
 		SSI_LOG_ERR("Unsupported num. of ICV fragments (> %d)\n",
-			MAX_ICV_NENTS_SUPPORTED);
+			    MAX_ICV_NENTS_SUPPORTED);
 		nents = -1; /*unsupported*/
 	}
 	SSI_LOG_DEBUG("is_frag=%s icv_nents=%u\n",
-		(*is_icv_fragmented ? "true" : "false"), nents);
+		      (*is_icv_fragmented ? "true" : "false"), nents);
 
 	return nents;
 }
@@ -778,18 +783,18 @@ static inline int ssi_buffer_mgr_aead_chain_iv(
 		goto chain_iv_exit;
 	}
 
-	areq_ctx->gen_ctx.iv_dma_addr = dma_map_single(dev, req->iv,
-		hw_iv_size, DMA_BIDIRECTIONAL);
+	areq_ctx->gen_ctx.iv_dma_addr = dma_map_single(dev, req->iv, hw_iv_size,
+						       DMA_BIDIRECTIONAL);
 	if (unlikely(dma_mapping_error(dev, areq_ctx->gen_ctx.iv_dma_addr))) {
 		SSI_LOG_ERR("Mapping iv %u B at va=%pK for DMA failed\n",
-			hw_iv_size, req->iv);
+			    hw_iv_size, req->iv);
 		rc = -ENOMEM;
 		goto chain_iv_exit;
 	}
 
-	SSI_LOG_DEBUG("Mapped iv %u B at va=%pK to dma=0x%llX\n",
-		hw_iv_size, req->iv,
-		(unsigned long long)areq_ctx->gen_ctx.iv_dma_addr);
+	SSI_LOG_DEBUG("Mapped iv %u B at va=%pK to dma=%pad\n",
+		      hw_iv_size, req->iv,
+		      areq_ctx->gen_ctx.iv_dma_addr);
 	if (do_chain && areq_ctx->plaintext_authenticate_only) {  // TODO: what about CTR?? ask Ron
 		struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 		unsigned int iv_size_to_authenc = crypto_aead_ivsize(tfm);
@@ -833,8 +838,8 @@ static inline int ssi_buffer_mgr_aead_chain_assoc(
 		areq_ctx->assoc.nents = 0;
 		areq_ctx->assoc.mlli_nents = 0;
 		SSI_LOG_DEBUG("Chain assoc of length 0: buff_type=%s nents=%u\n",
-			GET_DMA_BUFFER_TYPE(areq_ctx->assoc_buff_type),
-			areq_ctx->assoc.nents);
+			      GET_DMA_BUFFER_TYPE(areq_ctx->assoc_buff_type),
+			      areq_ctx->assoc.nents);
 		goto chain_assoc_exit;
 	}
 
@@ -868,10 +873,9 @@ static inline int ssi_buffer_mgr_aead_chain_assoc(
 	if (areq_ctx->ccm_hdr_size != ccm_header_size_null) {
 		if (unlikely((mapped_nents + 1) >
 			LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES)) {
-			SSI_LOG_ERR("CCM case.Too many fragments. "
-				"Current %d max %d\n",
-				(areq_ctx->assoc.nents + 1),
-				LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES);
+			SSI_LOG_ERR("CCM case.Too many fragments. Current %d max %d\n",
+				    (areq_ctx->assoc.nents + 1),
+				    LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES);
 			rc = -ENOMEM;
 			goto chain_assoc_exit;
 		}
@@ -884,10 +888,10 @@ static inline int ssi_buffer_mgr_aead_chain_assoc(
 		areq_ctx->assoc_buff_type = SSI_DMA_BUF_MLLI;
 
 	if (unlikely((do_chain) ||
-		(areq_ctx->assoc_buff_type == SSI_DMA_BUF_MLLI))) {
+		     (areq_ctx->assoc_buff_type == SSI_DMA_BUF_MLLI))) {
 		SSI_LOG_DEBUG("Chain assoc: buff_type=%s nents=%u\n",
-			GET_DMA_BUFFER_TYPE(areq_ctx->assoc_buff_type),
-			areq_ctx->assoc.nents);
+			      GET_DMA_BUFFER_TYPE(areq_ctx->assoc_buff_type),
+			      areq_ctx->assoc.nents);
 		ssi_buffer_mgr_add_scatterlist_entry(
 			sg_data, areq_ctx->assoc.nents,
 			req->src, req->assoclen, 0, is_last,
@@ -911,26 +915,26 @@ static inline void ssi_buffer_mgr_prepare_aead_data_dlli(
 	if (likely(req->src == req->dst)) {
 		/*INPLACE*/
 		areq_ctx->icv_dma_addr = sg_dma_address(
-			areq_ctx->srcSgl) +
+			areq_ctx->src_sgl) +
 			(*src_last_bytes - authsize);
 		areq_ctx->icv_virt_addr = sg_virt(
-			areq_ctx->srcSgl) +
+			areq_ctx->src_sgl) +
 			(*src_last_bytes - authsize);
 	} else if (direct == DRV_CRYPTO_DIRECTION_DECRYPT) {
 		/*NON-INPLACE and DECRYPT*/
 		areq_ctx->icv_dma_addr = sg_dma_address(
-			areq_ctx->srcSgl) +
+			areq_ctx->src_sgl) +
 			(*src_last_bytes - authsize);
 		areq_ctx->icv_virt_addr = sg_virt(
-			areq_ctx->srcSgl) +
+			areq_ctx->src_sgl) +
 			(*src_last_bytes - authsize);
 	} else {
 		/*NON-INPLACE and ENCRYPT*/
 		areq_ctx->icv_dma_addr = sg_dma_address(
-			areq_ctx->dstSgl) +
+			areq_ctx->dst_sgl) +
 			(*dst_last_bytes - authsize);
 		areq_ctx->icv_virt_addr = sg_virt(
-			areq_ctx->dstSgl) +
+			areq_ctx->dst_sgl) +
 			(*dst_last_bytes - authsize);
 	}
 }
@@ -951,13 +955,18 @@ static inline int ssi_buffer_mgr_prepare_aead_data_mlli(
 	if (likely(req->src == req->dst)) {
 		/*INPLACE*/
 		ssi_buffer_mgr_add_scatterlist_entry(sg_data,
-			areq_ctx->src.nents, areq_ctx->srcSgl,
-			areq_ctx->cryptlen, areq_ctx->srcOffset, is_last_table,
-			&areq_ctx->src.mlli_nents);
-
-		icv_nents = ssi_buffer_mgr_get_aead_icv_nents(areq_ctx->srcSgl,
-			areq_ctx->src.nents, authsize, *src_last_bytes,
-			&areq_ctx->is_icv_fragmented);
+						     areq_ctx->src.nents,
+						     areq_ctx->src_sgl,
+						     areq_ctx->cryptlen,
+						     areq_ctx->src_offset,
+						     is_last_table,
+						     &areq_ctx->src.mlli_nents);
+
+		icv_nents = ssi_buffer_mgr_get_aead_icv_nents(areq_ctx->src_sgl,
+							      areq_ctx->src.nents,
+							      authsize,
+							      *src_last_bytes,
+							      &areq_ctx->is_icv_fragmented);
 		if (unlikely(icv_nents < 0)) {
 			rc = -ENOTSUPP;
 			goto prepare_data_mlli_exit;
@@ -995,27 +1004,35 @@ static inline int ssi_buffer_mgr_prepare_aead_data_mlli(
 		} else { /* Contig. ICV */
 			/*Should hanlde if the sg is not contig.*/
 			areq_ctx->icv_dma_addr = sg_dma_address(
-				&areq_ctx->srcSgl[areq_ctx->src.nents - 1]) +
+				&areq_ctx->src_sgl[areq_ctx->src.nents - 1]) +
 				(*src_last_bytes - authsize);
 			areq_ctx->icv_virt_addr = sg_virt(
-				&areq_ctx->srcSgl[areq_ctx->src.nents - 1]) +
+				&areq_ctx->src_sgl[areq_ctx->src.nents - 1]) +
 				(*src_last_bytes - authsize);
 		}
 
 	} else if (direct == DRV_CRYPTO_DIRECTION_DECRYPT) {
 		/*NON-INPLACE and DECRYPT*/
 		ssi_buffer_mgr_add_scatterlist_entry(sg_data,
-			areq_ctx->src.nents, areq_ctx->srcSgl,
-			areq_ctx->cryptlen, areq_ctx->srcOffset, is_last_table,
-			&areq_ctx->src.mlli_nents);
+						     areq_ctx->src.nents,
+						     areq_ctx->src_sgl,
+						     areq_ctx->cryptlen,
+						     areq_ctx->src_offset,
+						     is_last_table,
+						     &areq_ctx->src.mlli_nents);
 		ssi_buffer_mgr_add_scatterlist_entry(sg_data,
-			areq_ctx->dst.nents, areq_ctx->dstSgl,
-			areq_ctx->cryptlen, areq_ctx->dstOffset, is_last_table,
-			&areq_ctx->dst.mlli_nents);
-
-		icv_nents = ssi_buffer_mgr_get_aead_icv_nents(areq_ctx->srcSgl,
-			areq_ctx->src.nents, authsize, *src_last_bytes,
-			&areq_ctx->is_icv_fragmented);
+						     areq_ctx->dst.nents,
+						     areq_ctx->dst_sgl,
+						     areq_ctx->cryptlen,
+						     areq_ctx->dst_offset,
+						     is_last_table,
+						     &areq_ctx->dst.mlli_nents);
+
+		icv_nents = ssi_buffer_mgr_get_aead_icv_nents(areq_ctx->src_sgl,
+							      areq_ctx->src.nents,
+							      authsize,
+							      *src_last_bytes,
+							      &areq_ctx->is_icv_fragmented);
 		if (unlikely(icv_nents < 0)) {
 			rc = -ENOTSUPP;
 			goto prepare_data_mlli_exit;
@@ -1039,26 +1056,34 @@ static inline int ssi_buffer_mgr_prepare_aead_data_mlli(
 		} else { /* Contig. ICV */
 			/*Should hanlde if the sg is not contig.*/
 			areq_ctx->icv_dma_addr = sg_dma_address(
-				&areq_ctx->srcSgl[areq_ctx->src.nents - 1]) +
+				&areq_ctx->src_sgl[areq_ctx->src.nents - 1]) +
 				(*src_last_bytes - authsize);
 			areq_ctx->icv_virt_addr = sg_virt(
-				&areq_ctx->srcSgl[areq_ctx->src.nents - 1]) +
+				&areq_ctx->src_sgl[areq_ctx->src.nents - 1]) +
 				(*src_last_bytes - authsize);
 		}
 
 	} else {
 		/*NON-INPLACE and ENCRYPT*/
 		ssi_buffer_mgr_add_scatterlist_entry(sg_data,
-			areq_ctx->dst.nents, areq_ctx->dstSgl,
-			areq_ctx->cryptlen, areq_ctx->dstOffset, is_last_table,
-			&areq_ctx->dst.mlli_nents);
+						     areq_ctx->dst.nents,
+						     areq_ctx->dst_sgl,
+						     areq_ctx->cryptlen,
+						     areq_ctx->dst_offset,
+						     is_last_table,
+						     &areq_ctx->dst.mlli_nents);
 		ssi_buffer_mgr_add_scatterlist_entry(sg_data,
-			areq_ctx->src.nents, areq_ctx->srcSgl,
-			areq_ctx->cryptlen, areq_ctx->srcOffset, is_last_table,
-			&areq_ctx->src.mlli_nents);
-
-		icv_nents = ssi_buffer_mgr_get_aead_icv_nents(areq_ctx->dstSgl,
-			areq_ctx->dst.nents, authsize, *dst_last_bytes,
+						     areq_ctx->src.nents,
+						     areq_ctx->src_sgl,
+						     areq_ctx->cryptlen,
+						     areq_ctx->src_offset,
+						     is_last_table,
+						     &areq_ctx->src.mlli_nents);
+
+		icv_nents = ssi_buffer_mgr_get_aead_icv_nents(areq_ctx->dst_sgl,
+							      areq_ctx->dst.nents,
+							      authsize,
+							      *dst_last_bytes,
 			&areq_ctx->is_icv_fragmented);
 		if (unlikely(icv_nents < 0)) {
 			rc = -ENOTSUPP;
@@ -1068,10 +1093,10 @@ static inline int ssi_buffer_mgr_prepare_aead_data_mlli(
 		if (likely(!areq_ctx->is_icv_fragmented)) {
 			/* Contig. ICV */
 			areq_ctx->icv_dma_addr = sg_dma_address(
-				&areq_ctx->dstSgl[areq_ctx->dst.nents - 1]) +
+				&areq_ctx->dst_sgl[areq_ctx->dst.nents - 1]) +
 				(*dst_last_bytes - authsize);
 			areq_ctx->icv_virt_addr = sg_virt(
-				&areq_ctx->dstSgl[areq_ctx->dst.nents - 1]) +
+				&areq_ctx->dst_sgl[areq_ctx->dst.nents - 1]) +
 				(*dst_last_bytes - authsize);
 		} else {
 			areq_ctx->icv_dma_addr = areq_ctx->mac_buf_dma_addr;
@@ -1113,37 +1138,36 @@ static inline int ssi_buffer_mgr_aead_chain_data(
 		rc = -EINVAL;
 		goto chain_data_exit;
 	}
-	areq_ctx->srcSgl = req->src;
-	areq_ctx->dstSgl = req->dst;
+	areq_ctx->src_sgl = req->src;
+	areq_ctx->dst_sgl = req->dst;
 
 	if (is_gcm4543)
 		size_for_map += crypto_aead_ivsize(tfm);
 
 	size_for_map += (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? authsize : 0;
 	src_mapped_nents = ssi_buffer_mgr_get_sgl_nents(req->src, size_for_map, &src_last_bytes, &chained);
-	sg_index = areq_ctx->srcSgl->length;
+	sg_index = areq_ctx->src_sgl->length;
 	//check where the data starts
 	while (sg_index <= size_to_skip) {
-		offset -= areq_ctx->srcSgl->length;
-		areq_ctx->srcSgl = sg_next(areq_ctx->srcSgl);
+		offset -= areq_ctx->src_sgl->length;
+		areq_ctx->src_sgl = sg_next(areq_ctx->src_sgl);
 		//if have reached the end of the sgl, then this is unexpected
-		if (!areq_ctx->srcSgl) {
+		if (!areq_ctx->src_sgl) {
 			SSI_LOG_ERR("reached end of sg list. unexpected\n");
 			BUG();
 		}
-		sg_index += areq_ctx->srcSgl->length;
+		sg_index += areq_ctx->src_sgl->length;
 		src_mapped_nents--;
 	}
-	if (unlikely(src_mapped_nents > LLI_MAX_NUM_OF_DATA_ENTRIES))
-	{
+	if (unlikely(src_mapped_nents > LLI_MAX_NUM_OF_DATA_ENTRIES)) {
 		SSI_LOG_ERR("Too many fragments. current %d max %d\n",
-				src_mapped_nents, LLI_MAX_NUM_OF_DATA_ENTRIES);
+			    src_mapped_nents, LLI_MAX_NUM_OF_DATA_ENTRIES);
 			return -ENOMEM;
 	}
 
 	areq_ctx->src.nents = src_mapped_nents;
 
-	areq_ctx->srcOffset = offset;
+	areq_ctx->src_offset = offset;
 
 	if (req->src != req->dst) {
 		size_for_map = req->assoclen + req->cryptlen;
@@ -1152,9 +1176,11 @@ static inline int ssi_buffer_mgr_aead_chain_data(
 			size_for_map += crypto_aead_ivsize(tfm);
 
 		rc = ssi_buffer_mgr_map_scatterlist(dev, req->dst, size_for_map,
-			 DMA_BIDIRECTIONAL, &(areq_ctx->dst.nents),
-			 LLI_MAX_NUM_OF_DATA_ENTRIES, &dst_last_bytes,
-						   &dst_mapped_nents);
+						    DMA_BIDIRECTIONAL,
+						    &areq_ctx->dst.nents,
+						    LLI_MAX_NUM_OF_DATA_ENTRIES,
+						    &dst_last_bytes,
+						    &dst_mapped_nents);
 		if (unlikely(rc != 0)) {
 			rc = -ENOMEM;
 			goto chain_data_exit;
@@ -1162,35 +1188,37 @@ static inline int ssi_buffer_mgr_aead_chain_data(
 	}
 
 	dst_mapped_nents = ssi_buffer_mgr_get_sgl_nents(req->dst, size_for_map, &dst_last_bytes, &chained);
-	sg_index = areq_ctx->dstSgl->length;
+	sg_index = areq_ctx->dst_sgl->length;
 	offset = size_to_skip;
 
 	//check where the data starts
 	while (sg_index <= size_to_skip) {
-		offset -= areq_ctx->dstSgl->length;
-		areq_ctx->dstSgl = sg_next(areq_ctx->dstSgl);
+		offset -= areq_ctx->dst_sgl->length;
+		areq_ctx->dst_sgl = sg_next(areq_ctx->dst_sgl);
 		//if have reached the end of the sgl, then this is unexpected
-		if (!areq_ctx->dstSgl) {
+		if (!areq_ctx->dst_sgl) {
 			SSI_LOG_ERR("reached end of sg list. unexpected\n");
 			BUG();
 		}
-		sg_index += areq_ctx->dstSgl->length;
+		sg_index += areq_ctx->dst_sgl->length;
 		dst_mapped_nents--;
 	}
-	if (unlikely(dst_mapped_nents > LLI_MAX_NUM_OF_DATA_ENTRIES))
-	{
+	if (unlikely(dst_mapped_nents > LLI_MAX_NUM_OF_DATA_ENTRIES)) {
 		SSI_LOG_ERR("Too many fragments. current %d max %d\n",
 			    dst_mapped_nents, LLI_MAX_NUM_OF_DATA_ENTRIES);
 		return -ENOMEM;
 	}
 	areq_ctx->dst.nents = dst_mapped_nents;
-	areq_ctx->dstOffset = offset;
+	areq_ctx->dst_offset = offset;
 	if ((src_mapped_nents > 1) ||
 	    (dst_mapped_nents  > 1) ||
 	    do_chain) {
 		areq_ctx->data_buff_type = SSI_DMA_BUF_MLLI;
-		rc = ssi_buffer_mgr_prepare_aead_data_mlli(drvdata, req, sg_data,
-			&src_last_bytes, &dst_last_bytes, is_last_table);
+		rc = ssi_buffer_mgr_prepare_aead_data_mlli(drvdata, req,
+							   sg_data,
+							   &src_last_bytes,
+							   &dst_last_bytes,
+							   is_last_table);
 	} else {
 		areq_ctx->data_buff_type = SSI_DMA_BUF_DLLI;
 		ssi_buffer_mgr_prepare_aead_data_dlli(
@@ -1202,7 +1230,7 @@ chain_data_exit:
 }
 
 static void ssi_buffer_mgr_update_aead_mlli_nents(struct ssi_drvdata *drvdata,
-					   struct aead_request *req)
+						  struct aead_request *req)
 {
 	struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
 	u32 curr_mlli_size = 0;
@@ -1274,8 +1302,7 @@ int ssi_buffer_mgr_map_aead_request(
 
 	if (drvdata->coherent &&
 	    (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) &&
-	    likely(req->src == req->dst))
-	{
+	    likely(req->src == req->dst)) {
 		u32 size_to_skip = req->assoclen;
 
 		if (is_gcm4543)
@@ -1296,19 +1323,21 @@ int ssi_buffer_mgr_map_aead_request(
 				req->cryptlen :
 				(req->cryptlen - authsize);
 
-	areq_ctx->mac_buf_dma_addr = dma_map_single(dev,
-		areq_ctx->mac_buf, MAX_MAC_SIZE, DMA_BIDIRECTIONAL);
+	areq_ctx->mac_buf_dma_addr = dma_map_single(dev, areq_ctx->mac_buf,
+						    MAX_MAC_SIZE,
+						    DMA_BIDIRECTIONAL);
 	if (unlikely(dma_mapping_error(dev, areq_ctx->mac_buf_dma_addr))) {
 		SSI_LOG_ERR("Mapping mac_buf %u B at va=%pK for DMA failed\n",
-			MAX_MAC_SIZE, areq_ctx->mac_buf);
+			    MAX_MAC_SIZE, areq_ctx->mac_buf);
 		rc = -ENOMEM;
 		goto aead_map_failure;
 	}
 
 	if (areq_ctx->ccm_hdr_size != ccm_header_size_null) {
 		areq_ctx->ccm_iv0_dma_addr = dma_map_single(dev,
-			(areq_ctx->ccm_config + CCM_CTR_COUNT_0_OFFSET),
-			AES_BLOCK_SIZE, DMA_TO_DEVICE);
+							    (areq_ctx->ccm_config + CCM_CTR_COUNT_0_OFFSET),
+							    AES_BLOCK_SIZE,
+							    DMA_TO_DEVICE);
 
 		if (unlikely(dma_mapping_error(dev, areq_ctx->ccm_iv0_dma_addr))) {
 			SSI_LOG_ERR("Mapping mac_buf %u B at va=%pK "
@@ -1319,7 +1348,8 @@ int ssi_buffer_mgr_map_aead_request(
 			goto aead_map_failure;
 		}
 		if (ssi_aead_handle_config_buf(dev, areq_ctx,
-			areq_ctx->ccm_config, &sg_data, req->assoclen) != 0) {
+					       areq_ctx->ccm_config, &sg_data,
+					       req->assoclen) != 0) {
 			rc = -ENOMEM;
 			goto aead_map_failure;
 		}
@@ -1328,26 +1358,31 @@ int ssi_buffer_mgr_map_aead_request(
 #if SSI_CC_HAS_AES_GCM
 	if (areq_ctx->cipher_mode == DRV_CIPHER_GCTR) {
 		areq_ctx->hkey_dma_addr = dma_map_single(dev,
-			areq_ctx->hkey, AES_BLOCK_SIZE, DMA_BIDIRECTIONAL);
+							 areq_ctx->hkey,
+							 AES_BLOCK_SIZE,
+							 DMA_BIDIRECTIONAL);
 		if (unlikely(dma_mapping_error(dev, areq_ctx->hkey_dma_addr))) {
 			SSI_LOG_ERR("Mapping hkey %u B at va=%pK for DMA failed\n",
-				AES_BLOCK_SIZE, areq_ctx->hkey);
+				    AES_BLOCK_SIZE, areq_ctx->hkey);
 			rc = -ENOMEM;
 			goto aead_map_failure;
 		}
 
 		areq_ctx->gcm_block_len_dma_addr = dma_map_single(dev,
-			&areq_ctx->gcm_len_block, AES_BLOCK_SIZE, DMA_TO_DEVICE);
+								  &areq_ctx->gcm_len_block,
+								  AES_BLOCK_SIZE,
+								  DMA_TO_DEVICE);
 		if (unlikely(dma_mapping_error(dev, areq_ctx->gcm_block_len_dma_addr))) {
 			SSI_LOG_ERR("Mapping gcm_len_block %u B at va=%pK for DMA failed\n",
-				AES_BLOCK_SIZE, &areq_ctx->gcm_len_block);
+				    AES_BLOCK_SIZE, &areq_ctx->gcm_len_block);
 			rc = -ENOMEM;
 			goto aead_map_failure;
 		}
 
 		areq_ctx->gcm_iv_inc1_dma_addr = dma_map_single(dev,
-			areq_ctx->gcm_iv_inc1,
-			AES_BLOCK_SIZE, DMA_TO_DEVICE);
+								areq_ctx->gcm_iv_inc1,
+								AES_BLOCK_SIZE,
+								DMA_TO_DEVICE);
 
 		if (unlikely(dma_mapping_error(dev, areq_ctx->gcm_iv_inc1_dma_addr))) {
 			SSI_LOG_ERR("Mapping gcm_iv_inc1 %u B at va=%pK "
@@ -1359,8 +1394,9 @@ int ssi_buffer_mgr_map_aead_request(
 		}
 
 		areq_ctx->gcm_iv_inc2_dma_addr = dma_map_single(dev,
-			areq_ctx->gcm_iv_inc2,
-			AES_BLOCK_SIZE, DMA_TO_DEVICE);
+								areq_ctx->gcm_iv_inc2,
+								AES_BLOCK_SIZE,
+								DMA_TO_DEVICE);
 
 		if (unlikely(dma_mapping_error(dev, areq_ctx->gcm_iv_inc2_dma_addr))) {
 			SSI_LOG_ERR("Mapping gcm_iv_inc2 %u B at va=%pK "
@@ -1380,7 +1416,7 @@ int ssi_buffer_mgr_map_aead_request(
 	if (is_gcm4543)
 		size_to_map += crypto_aead_ivsize(tfm);
 	rc = ssi_buffer_mgr_map_scatterlist(dev, req->src,
-					    size_to_map, DMA_BIDIRECTIONAL, &(areq_ctx->src.nents),
+					    size_to_map, DMA_BIDIRECTIONAL, &areq_ctx->src.nents,
 					    LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES + LLI_MAX_NUM_OF_DATA_ENTRIES, &dummy, &mapped_nents);
 	if (unlikely(rc != 0)) {
 		rc = -ENOMEM;
@@ -1491,18 +1527,18 @@ int ssi_buffer_mgr_map_hash_request_final(
 	/* map the previous buffer */
 	if (*curr_buff_cnt != 0) {
 		if (ssi_ahash_handle_curr_buf(dev, areq_ctx, curr_buff,
-					    *curr_buff_cnt, &sg_data) != 0) {
+					      *curr_buff_cnt, &sg_data) != 0) {
 			return -ENOMEM;
 		}
 	}
 
 	if (src && (nbytes > 0) && do_update) {
-		if (unlikely(ssi_buffer_mgr_map_scatterlist(dev, src,
-					  nbytes,
-					  DMA_TO_DEVICE,
-					  &areq_ctx->in_nents,
-					  LLI_MAX_NUM_OF_DATA_ENTRIES,
-					  &dummy, &mapped_nents))){
+		if (unlikely(ssi_buffer_mgr_map_scatterlist(dev, src, nbytes,
+							    DMA_TO_DEVICE,
+							    &areq_ctx->in_nents,
+							    LLI_MAX_NUM_OF_DATA_ENTRIES,
+							    &dummy,
+							    &mapped_nents))){
 			goto unmap_curr_buff;
 		}
 		if (src && (mapped_nents == 1)
@@ -1522,19 +1558,18 @@ int ssi_buffer_mgr_map_hash_request_final(
 		mlli_params->curr_pool = buff_mgr->mlli_buffs_pool;
 		/* add the src data to the sg_data */
 		ssi_buffer_mgr_add_scatterlist_entry(&sg_data,
-					areq_ctx->in_nents,
-					src,
-					nbytes, 0,
-					true, &areq_ctx->mlli_nents);
+						     areq_ctx->in_nents,
+						     src, nbytes, 0, true,
+						     &areq_ctx->mlli_nents);
 		if (unlikely(ssi_buffer_mgr_generate_mlli(dev, &sg_data,
-						  mlli_params) != 0)) {
+							  mlli_params) != 0)) {
 			goto fail_unmap_din;
 		}
 	}
 	/* change the buffer index for the unmap function */
 	areq_ctx->buff_index = (areq_ctx->buff_index ^ 1);
 	SSI_LOG_DEBUG("areq_ctx->data_dma_buf_type = %s\n",
-		GET_DMA_BUFFER_TYPE(areq_ctx->data_dma_buf_type));
+		      GET_DMA_BUFFER_TYPE(areq_ctx->data_dma_buf_type));
 	return 0;
 
 fail_unmap_din:
@@ -1588,8 +1623,8 @@ int ssi_buffer_mgr_map_hash_request_update(
 			&curr_buff[*curr_buff_cnt]);
 		areq_ctx->in_nents =
 			ssi_buffer_mgr_get_sgl_nents(src,
-						    nbytes,
-						    &dummy, NULL);
+						     nbytes,
+						     &dummy, NULL);
 		sg_copy_to_buffer(src, areq_ctx->in_nents,
 				  &curr_buff[*curr_buff_cnt], nbytes);
 		*curr_buff_cnt += nbytes;
@@ -1612,15 +1647,15 @@ int ssi_buffer_mgr_map_hash_request_update(
 			     (update_data_len - *curr_buff_cnt),
 			     *next_buff_cnt);
 		ssi_buffer_mgr_copy_scatterlist_portion(next_buff, src,
-			     (update_data_len - *curr_buff_cnt),
-			     nbytes, SSI_SG_TO_BUF);
+							(update_data_len - *curr_buff_cnt),
+							nbytes, SSI_SG_TO_BUF);
 		/* change the buffer index for next operation */
 		swap_index = 1;
 	}
 
 	if (*curr_buff_cnt != 0) {
 		if (ssi_ahash_handle_curr_buf(dev, areq_ctx, curr_buff,
-					    *curr_buff_cnt, &sg_data) != 0) {
+					      *curr_buff_cnt, &sg_data) != 0) {
 			return -ENOMEM;
 		}
 		/* change the buffer index for next operation */
@@ -1629,11 +1664,12 @@ int ssi_buffer_mgr_map_hash_request_update(
 
 	if (update_data_len > *curr_buff_cnt) {
 		if (unlikely(ssi_buffer_mgr_map_scatterlist(dev, src,
-					  (update_data_len - *curr_buff_cnt),
-					  DMA_TO_DEVICE,
-					  &areq_ctx->in_nents,
-					  LLI_MAX_NUM_OF_DATA_ENTRIES,
-					  &dummy, &mapped_nents))){
+							    (update_data_len - *curr_buff_cnt),
+							    DMA_TO_DEVICE,
+							    &areq_ctx->in_nents,
+							    LLI_MAX_NUM_OF_DATA_ENTRIES,
+							    &dummy,
+							    &mapped_nents))){
 			goto unmap_curr_buff;
 		}
 		if ((mapped_nents == 1)
@@ -1653,12 +1689,14 @@ int ssi_buffer_mgr_map_hash_request_update(
 		mlli_params->curr_pool = buff_mgr->mlli_buffs_pool;
 		/* add the src data to the sg_data */
 		ssi_buffer_mgr_add_scatterlist_entry(&sg_data,
-					areq_ctx->in_nents,
-					src,
-					(update_data_len - *curr_buff_cnt), 0,
-					true, &areq_ctx->mlli_nents);
+						     areq_ctx->in_nents,
+						     src,
+						     (update_data_len - *curr_buff_cnt),
+						     0,
+						     true,
+						     &areq_ctx->mlli_nents);
 		if (unlikely(ssi_buffer_mgr_generate_mlli(dev, &sg_data,
-						  mlli_params) != 0)) {
+							  mlli_params) != 0)) {
 			goto fail_unmap_din;
 		}
 	}
@@ -1687,28 +1725,28 @@ void ssi_buffer_mgr_unmap_hash_request(
 	 *allocated and should be released
 	 */
 	if (areq_ctx->mlli_params.curr_pool) {
-		SSI_LOG_DEBUG("free MLLI buffer: dma=0x%llX virt=%pK\n",
-			     (unsigned long long)areq_ctx->mlli_params.mlli_dma_addr,
-			     areq_ctx->mlli_params.mlli_virt_addr);
+		SSI_LOG_DEBUG("free MLLI buffer: dma=%pad virt=%pK\n",
+			      areq_ctx->mlli_params.mlli_dma_addr,
+			      areq_ctx->mlli_params.mlli_virt_addr);
 		dma_pool_free(areq_ctx->mlli_params.curr_pool,
 			      areq_ctx->mlli_params.mlli_virt_addr,
 			      areq_ctx->mlli_params.mlli_dma_addr);
 	}
 
 	if ((src) && likely(areq_ctx->in_nents != 0)) {
-		SSI_LOG_DEBUG("Unmapped sg src: virt=%pK dma=0x%llX len=0x%X\n",
-			     sg_virt(src),
-			     (unsigned long long)sg_dma_address(src),
-			     sg_dma_len(src));
+		SSI_LOG_DEBUG("Unmapped sg src: virt=%pK dma=%pad len=0x%X\n",
+			      sg_virt(src),
+			      sg_dma_address(src),
+			      sg_dma_len(src));
 		dma_unmap_sg(dev, src,
 			     areq_ctx->in_nents, DMA_TO_DEVICE);
 	}
 
 	if (*prev_len != 0) {
 		SSI_LOG_DEBUG("Unmapped buffer: areq_ctx->buff_sg=%pK"
-			     " dma=0x%llX len 0x%X\n",
+			     " dma=%pad len 0x%X\n",
 				sg_virt(areq_ctx->buff_sg),
-				(unsigned long long)sg_dma_address(areq_ctx->buff_sg),
+				sg_dma_address(areq_ctx->buff_sg),
 				sg_dma_len(areq_ctx->buff_sg));
 		dma_unmap_sg(dev, areq_ctx->buff_sg, 1, DMA_TO_DEVICE);
 		if (!do_revert) {
@@ -1725,8 +1763,7 @@ int ssi_buffer_mgr_init(struct ssi_drvdata *drvdata)
 	struct buff_mgr_handle *buff_mgr_handle;
 	struct device *dev = &drvdata->plat_dev->dev;
 
-	buff_mgr_handle = (struct buff_mgr_handle *)
-		kmalloc(sizeof(struct buff_mgr_handle), GFP_KERNEL);
+	buff_mgr_handle = kmalloc(sizeof(*buff_mgr_handle), GFP_KERNEL);
 	if (!buff_mgr_handle)
 		return -ENOMEM;
 
diff --git a/drivers/staging/ccree/ssi_cipher.c b/drivers/staging/ccree/ssi_cipher.c
index cd2eafc04232..8d31a93fd8b7 100644
--- a/drivers/staging/ccree/ssi_cipher.c
+++ b/drivers/staging/ccree/ssi_cipher.c
@@ -23,6 +23,8 @@
 #include <crypto/aes.h>
 #include <crypto/ctr.h>
 #include <crypto/des.h>
+#include <crypto/xts.h>
+#include <crypto/scatterwalk.h>
 
 #include "ssi_config.h"
 #include "ssi_driver.h"
@@ -31,7 +33,6 @@
 #include "ssi_cipher.h"
 #include "ssi_request_mgr.h"
 #include "ssi_sysfs.h"
-#include "ssi_fips_local.h"
 
 #define MAX_ABLKCIPHER_SEQ_LEN 6
 
@@ -68,7 +69,8 @@ struct ssi_ablkcipher_ctx {
 
 static void ssi_ablkcipher_complete(struct device *dev, void *ssi_req, void __iomem *cc_base);
 
-static int validate_keys_sizes(struct ssi_ablkcipher_ctx *ctx_p, u32 size) {
+static int validate_keys_sizes(struct ssi_ablkcipher_ctx *ctx_p, u32 size)
+{
 	switch (ctx_p->flow_mode) {
 	case S_DIN_to_AES:
 		switch (size) {
@@ -92,8 +94,7 @@ static int validate_keys_sizes(struct ssi_ablkcipher_ctx *ctx_p, u32 size) {
 			break;
 		}
 	case S_DIN_to_DES:
-		if (likely(size == DES3_EDE_KEY_SIZE ||
-		    size == DES_KEY_SIZE))
+		if (likely(size == DES3_EDE_KEY_SIZE || size == DES_KEY_SIZE))
 			return 0;
 		break;
 #if SSI_CC_HAS_MULTI2
@@ -108,7 +109,8 @@ static int validate_keys_sizes(struct ssi_ablkcipher_ctx *ctx_p, u32 size) {
 	return -EINVAL;
 }
 
-static int validate_data_size(struct ssi_ablkcipher_ctx *ctx_p, unsigned int size) {
+static int validate_data_size(struct ssi_ablkcipher_ctx *ctx_p, unsigned int size)
+{
 	switch (ctx_p->flow_mode) {
 	case S_DIN_to_AES:
 		switch (ctx_p->cipher_mode) {
@@ -183,10 +185,9 @@ static int ssi_blkcipher_init(struct crypto_tfm *tfm)
 	int rc = 0;
 	unsigned int max_key_buf_size = get_max_keysize(tfm);
 
-	SSI_LOG_DEBUG("Initializing context @%p for %s\n", ctx_p,
-						crypto_tfm_alg_name(tfm));
+	SSI_LOG_DEBUG("Initializing context @%p for %s\n",
+		      ctx_p, crypto_tfm_alg_name(tfm));
 
-	CHECK_AND_RETURN_UPON_FIPS_ERROR();
 	ctx_p->cipher_mode = ssi_alg->cipher_mode;
 	ctx_p->flow_mode = ssi_alg->flow_mode;
 	ctx_p->drvdata = ssi_alg->drvdata;
@@ -203,15 +204,16 @@ static int ssi_blkcipher_init(struct crypto_tfm *tfm)
 
 	/* Map key buffer */
 	ctx_p->user.key_dma_addr = dma_map_single(dev, (void *)ctx_p->user.key,
-					     max_key_buf_size, DMA_TO_DEVICE);
+						  max_key_buf_size,
+						  DMA_TO_DEVICE);
 	if (dma_mapping_error(dev, ctx_p->user.key_dma_addr)) {
 		SSI_LOG_ERR("Mapping Key %u B at va=%pK for DMA failed\n",
-			max_key_buf_size, ctx_p->user.key);
+			    max_key_buf_size, ctx_p->user.key);
 		return -ENOMEM;
 	}
-	SSI_LOG_DEBUG("Mapped key %u B at va=%pK to dma=0x%llX\n",
-		max_key_buf_size, ctx_p->user.key,
-		(unsigned long long)ctx_p->user.key_dma_addr);
+	SSI_LOG_DEBUG("Mapped key %u B at va=%pK to dma=%pad\n",
+		      max_key_buf_size, ctx_p->user.key,
+		      ctx_p->user.key_dma_addr);
 
 	if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) {
 		/* Alloc hash tfm for essiv */
@@ -232,7 +234,7 @@ static void ssi_blkcipher_exit(struct crypto_tfm *tfm)
 	unsigned int max_key_buf_size = get_max_keysize(tfm);
 
 	SSI_LOG_DEBUG("Clearing context @%p for %s\n",
-		crypto_tfm_ctx(tfm), crypto_tfm_alg_name(tfm));
+		      crypto_tfm_ctx(tfm), crypto_tfm_alg_name(tfm));
 
 	if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) {
 		/* Free hash tfm for essiv */
@@ -242,9 +244,9 @@ static void ssi_blkcipher_exit(struct crypto_tfm *tfm)
 
 	/* Unmap key buffer */
 	dma_unmap_single(dev, ctx_p->user.key_dma_addr, max_key_buf_size,
-								DMA_TO_DEVICE);
-	SSI_LOG_DEBUG("Unmapped key buffer key_dma_addr=0x%llX\n",
-		(unsigned long long)ctx_p->user.key_dma_addr);
+			 DMA_TO_DEVICE);
+	SSI_LOG_DEBUG("Unmapped key buffer key_dma_addr=%pad\n",
+		      ctx_p->user.key_dma_addr);
 
 	/* Free key buffer in context */
 	kfree(ctx_p->user.key);
@@ -263,31 +265,15 @@ static const u8 zero_buff[] = {	0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
 				0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
 
 /* The function verifies that tdes keys are not weak.*/
-static int ssi_fips_verify_3des_keys(const u8 *key, unsigned int keylen)
+static int ssi_verify_3des_keys(const u8 *key, unsigned int keylen)
 {
-#ifdef CCREE_FIPS_SUPPORT
 	struct tdes_keys *tdes_key = (struct tdes_keys *)key;
 
 	/* verify key1 != key2 and key3 != key2*/
 	if (unlikely((memcmp((u8 *)tdes_key->key1, (u8 *)tdes_key->key2, sizeof(tdes_key->key1)) == 0) ||
-		      (memcmp((u8 *)tdes_key->key3, (u8 *)tdes_key->key2, sizeof(tdes_key->key3)) == 0))) {
+		     (memcmp((u8 *)tdes_key->key3, (u8 *)tdes_key->key2, sizeof(tdes_key->key3)) == 0))) {
 		return -ENOEXEC;
 	}
-#endif /* CCREE_FIPS_SUPPORT */
-
-	return 0;
-}
-
-/* The function verifies that xts keys are not weak.*/
-static int ssi_fips_verify_xts_keys(const u8 *key, unsigned int keylen)
-{
-#ifdef CCREE_FIPS_SUPPORT
-	/* Weak key is define as key that its first half (128/256 lsb) equals its second half (128/256 msb) */
-	int singleKeySize = keylen >> 1;
-
-	if (unlikely(memcmp(key, &key[singleKeySize], singleKeySize) == 0))
-		return -ENOEXEC;
-#endif /* CCREE_FIPS_SUPPORT */
 
 	return 0;
 }
@@ -317,12 +303,10 @@ static int ssi_blkcipher_setkey(struct crypto_tfm *tfm,
 	unsigned int max_key_buf_size = get_max_keysize(tfm);
 
 	SSI_LOG_DEBUG("Setting key in context @%p for %s. keylen=%u\n",
-		ctx_p, crypto_tfm_alg_name(tfm), keylen);
+		      ctx_p, crypto_tfm_alg_name(tfm), keylen);
 	dump_byte_array("key", (u8 *)key, keylen);
 
-	CHECK_AND_RETURN_UPON_FIPS_ERROR();
-
-	SSI_LOG_DEBUG("ssi_blkcipher_setkey: after FIPS check");
+	SSI_LOG_DEBUG("after FIPS check");
 
 	/* STAT_PHASE_0: Init and sanity checks */
 
@@ -368,7 +352,7 @@ static int ssi_blkcipher_setkey(struct crypto_tfm *tfm,
 		}
 
 		ctx_p->keylen = keylen;
-		SSI_LOG_DEBUG("ssi_blkcipher_setkey: ssi_is_hw_key ret 0");
+		SSI_LOG_DEBUG("ssi_is_hw_key ret 0");
 
 		return 0;
 	}
@@ -378,25 +362,25 @@ static int ssi_blkcipher_setkey(struct crypto_tfm *tfm,
 		if (unlikely(!des_ekey(tmp, key)) &&
 		    (crypto_tfm_get_flags(tfm) & CRYPTO_TFM_REQ_WEAK_KEY)) {
 			tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
-			SSI_LOG_DEBUG("ssi_blkcipher_setkey:  weak DES key");
+			SSI_LOG_DEBUG("weak DES key");
 			return -EINVAL;
 		}
 	}
 	if ((ctx_p->cipher_mode == DRV_CIPHER_XTS) &&
-	    ssi_fips_verify_xts_keys(key, keylen) != 0) {
-		SSI_LOG_DEBUG("ssi_blkcipher_setkey: weak XTS key");
+	    xts_check_key(tfm, key, keylen) != 0) {
+		SSI_LOG_DEBUG("weak XTS key");
 		return -EINVAL;
 	}
 	if ((ctx_p->flow_mode == S_DIN_to_DES) &&
 	    (keylen == DES3_EDE_KEY_SIZE) &&
-	    ssi_fips_verify_3des_keys(key, keylen) != 0) {
-		SSI_LOG_DEBUG("ssi_blkcipher_setkey: weak 3DES key");
+	    ssi_verify_3des_keys(key, keylen) != 0) {
+		SSI_LOG_DEBUG("weak 3DES key");
 		return -EINVAL;
 	}
 
 	/* STAT_PHASE_1: Copy key to ctx */
 	dma_sync_single_for_cpu(dev, ctx_p->user.key_dma_addr,
-					max_key_buf_size, DMA_TO_DEVICE);
+				max_key_buf_size, DMA_TO_DEVICE);
 
 	if (ctx_p->flow_mode == S_DIN_to_MULTI2) {
 #if SSI_CC_HAS_MULTI2
@@ -405,7 +389,7 @@ static int ssi_blkcipher_setkey(struct crypto_tfm *tfm,
 		if (ctx_p->key_round_number < CC_MULTI2_MIN_NUM_ROUNDS ||
 		    ctx_p->key_round_number > CC_MULTI2_MAX_NUM_ROUNDS) {
 			crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
-			SSI_LOG_DEBUG("ssi_blkcipher_setkey: SSI_CC_HAS_MULTI2 einval");
+			SSI_LOG_DEBUG("SSI_CC_HAS_MULTI2 einval");
 			return -EINVAL;
 #endif /*SSI_CC_HAS_MULTI2*/
 	} else {
@@ -429,10 +413,10 @@ static int ssi_blkcipher_setkey(struct crypto_tfm *tfm,
 		}
 	}
 	dma_sync_single_for_device(dev, ctx_p->user.key_dma_addr,
-					max_key_buf_size, DMA_TO_DEVICE);
+				   max_key_buf_size, DMA_TO_DEVICE);
 	ctx_p->keylen = keylen;
 
-	 SSI_LOG_DEBUG("ssi_blkcipher_setkey: return safely");
+	 SSI_LOG_DEBUG("return safely");
 	return 0;
 }
 
@@ -632,17 +616,15 @@ ssi_blkcipher_create_data_desc(
 		break;
 #endif /*SSI_CC_HAS_MULTI2*/
 	default:
-		SSI_LOG_ERR("invalid flow mode, flow_mode = %d \n", flow_mode);
+		SSI_LOG_ERR("invalid flow mode, flow_mode = %d\n", flow_mode);
 		return;
 	}
 	/* Process */
 	if (likely(req_ctx->dma_buf_type == SSI_DMA_BUF_DLLI)) {
-		SSI_LOG_DEBUG(" data params addr 0x%llX length 0x%X \n",
-			     (unsigned long long)sg_dma_address(src),
-			     nbytes);
-		SSI_LOG_DEBUG(" data params addr 0x%llX length 0x%X \n",
-			     (unsigned long long)sg_dma_address(dst),
-			     nbytes);
+		SSI_LOG_DEBUG(" data params addr %pad length 0x%X\n",
+			      sg_dma_address(src), nbytes);
+		SSI_LOG_DEBUG(" data params addr %pad length 0x%X\n",
+			      sg_dma_address(dst), nbytes);
 		hw_desc_init(&desc[*seq_size]);
 		set_din_type(&desc[*seq_size], DMA_DLLI, sg_dma_address(src),
 			     nbytes, NS_BIT);
@@ -655,9 +637,9 @@ ssi_blkcipher_create_data_desc(
 		(*seq_size)++;
 	} else {
 		/* bypass */
-		SSI_LOG_DEBUG(" bypass params addr 0x%llX "
+		SSI_LOG_DEBUG(" bypass params addr %pad "
 			     "length 0x%X addr 0x%08X\n",
-			(unsigned long long)req_ctx->mlli_params.mlli_dma_addr,
+			req_ctx->mlli_params.mlli_dma_addr,
 			req_ctx->mlli_params.mlli_len,
 			(unsigned int)ctx_p->drvdata->mlli_sram_addr);
 		hw_desc_init(&desc[*seq_size]);
@@ -706,16 +688,17 @@ ssi_blkcipher_create_data_desc(
 }
 
 static int ssi_blkcipher_complete(struct device *dev,
-				struct ssi_ablkcipher_ctx *ctx_p,
-				struct blkcipher_req_ctx *req_ctx,
-				struct scatterlist *dst,
-				struct scatterlist *src,
-				unsigned int ivsize,
-				void *areq,
-				void __iomem *cc_base)
+				  struct ssi_ablkcipher_ctx *ctx_p,
+				  struct blkcipher_req_ctx *req_ctx,
+				  struct scatterlist *dst,
+				  struct scatterlist *src,
+				  unsigned int ivsize,
+				  void *areq,
+				  void __iomem *cc_base)
 {
 	int completion_error = 0;
 	u32 inflight_counter;
+	struct ablkcipher_request *req = (struct ablkcipher_request *)areq;
 
 	ssi_buffer_mgr_unmap_blkcipher_request(dev, req_ctx, ivsize, src, dst);
 
@@ -726,6 +709,22 @@ static int ssi_blkcipher_complete(struct device *dev,
 		ctx_p->drvdata->inflight_counter--;
 
 	if (areq) {
+		/*
+		 * The crypto API expects us to set the req->info to the last
+		 * ciphertext block. For encrypt, simply copy from the result.
+		 * For decrypt, we must copy from a saved buffer since this
+		 * could be an in-place decryption operation and the src is
+		 * lost by this point.
+		 */
+		if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT)  {
+			memcpy(req->info, req_ctx->backup_info, ivsize);
+			kfree(req_ctx->backup_info);
+		} else {
+			scatterwalk_map_and_copy(req->info, req->dst,
+						 (req->nbytes - ivsize),
+						 ivsize, 0);
+		}
+
 		ablkcipher_request_complete(areq, completion_error);
 		return 0;
 	}
@@ -749,21 +748,22 @@ static int ssi_blkcipher_process(
 	int rc, seq_len = 0, cts_restore_flag = 0;
 
 	SSI_LOG_DEBUG("%s areq=%p info=%p nbytes=%d\n",
-		((direction == DRV_CRYPTO_DIRECTION_ENCRYPT) ? "Encrypt" : "Decrypt"),
-		     areq, info, nbytes);
+		      ((direction == DRV_CRYPTO_DIRECTION_ENCRYPT) ? "Encrypt" : "Decrypt"),
+		      areq, info, nbytes);
 
-	CHECK_AND_RETURN_UPON_FIPS_ERROR();
 	/* STAT_PHASE_0: Init and sanity checks */
 
 	/* TODO: check data length according to mode */
 	if (unlikely(validate_data_size(ctx_p, nbytes))) {
 		SSI_LOG_ERR("Unsupported data size %d.\n", nbytes);
 		crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_BLOCK_LEN);
-		return -EINVAL;
+		rc = -EINVAL;
+		goto exit_process;
 	}
 	if (nbytes == 0) {
 		/* No data to process is valid */
-		return 0;
+		rc = 0;
+		goto exit_process;
 	}
 	/*For CTS in case of data size aligned to 16 use CBC mode*/
 	if (((nbytes % AES_BLOCK_SIZE) == 0) && (ctx_p->cipher_mode == DRV_CIPHER_CBC_CTS)) {
@@ -804,12 +804,8 @@ static int ssi_blkcipher_process(
 		ssi_blkcipher_create_setup_desc(tfm, req_ctx, ivsize, nbytes,
 						desc, &seq_len);
 	/* Data processing */
-	ssi_blkcipher_create_data_desc(tfm,
-			      req_ctx,
-			      dst, src,
-			      nbytes,
-			      areq,
-			      desc, &seq_len);
+	ssi_blkcipher_create_data_desc(tfm, req_ctx, dst, src, nbytes, areq,
+				       desc, &seq_len);
 
 	/* do we need to generate IV? */
 	if (req_ctx->is_giv) {
@@ -842,6 +838,9 @@ exit_process:
 	if (cts_restore_flag != 0)
 		ctx_p->cipher_mode = DRV_CIPHER_CBC_CTS;
 
+	if (rc != -EINPROGRESS)
+		kfree(req_ctx->backup_info);
+
 	return rc;
 }
 
@@ -853,8 +852,6 @@ static void ssi_ablkcipher_complete(struct device *dev, void *ssi_req, void __io
 	struct ssi_ablkcipher_ctx *ctx_p = crypto_ablkcipher_ctx(tfm);
 	unsigned int ivsize = crypto_ablkcipher_ivsize(tfm);
 
-	CHECK_AND_RETURN_VOID_UPON_FIPS_ERROR();
-
 	ssi_blkcipher_complete(dev, ctx_p, req_ctx, areq->dst, areq->src,
 			       ivsize, areq, cc_base);
 }
@@ -871,8 +868,8 @@ static int ssi_ablkcipher_init(struct crypto_tfm *tfm)
 }
 
 static int ssi_ablkcipher_setkey(struct crypto_ablkcipher *tfm,
-				const u8 *key,
-				unsigned int keylen)
+				 const u8 *key,
+				 unsigned int keylen)
 {
 	return ssi_blkcipher_setkey(crypto_ablkcipher_tfm(tfm), key, keylen);
 }
@@ -884,7 +881,6 @@ static int ssi_ablkcipher_encrypt(struct ablkcipher_request *req)
 	struct blkcipher_req_ctx *req_ctx = ablkcipher_request_ctx(req);
 	unsigned int ivsize = crypto_ablkcipher_ivsize(ablk_tfm);
 
-	req_ctx->backup_info = req->info;
 	req_ctx->is_giv = false;
 
 	return ssi_blkcipher_process(tfm, req_ctx, req->dst, req->src, req->nbytes, req->info, ivsize, (void *)req, DRV_CRYPTO_DIRECTION_ENCRYPT);
@@ -897,8 +893,18 @@ static int ssi_ablkcipher_decrypt(struct ablkcipher_request *req)
 	struct blkcipher_req_ctx *req_ctx = ablkcipher_request_ctx(req);
 	unsigned int ivsize = crypto_ablkcipher_ivsize(ablk_tfm);
 
-	req_ctx->backup_info = req->info;
+	/*
+	 * Allocate and save the last IV sized bytes of the source, which will
+	 * be lost in case of in-place decryption and might be needed for CTS.
+	 */
+	req_ctx->backup_info = kmalloc(ivsize, GFP_KERNEL);
+	if (!req_ctx->backup_info)
+		return -ENOMEM;
+
+	scatterwalk_map_and_copy(req_ctx->backup_info, req->src,
+				 (req->nbytes - ivsize), ivsize, 0);
 	req_ctx->is_giv = false;
+
 	return ssi_blkcipher_process(tfm, req_ctx, req->dst, req->src, req->nbytes, req->info, ivsize, (void *)req, DRV_CRYPTO_DIRECTION_DECRYPT);
 }
 
@@ -1244,7 +1250,7 @@ struct ssi_crypto_alg *ssi_ablkcipher_create_alg(struct ssi_alg_template *templa
 	struct ssi_crypto_alg *t_alg;
 	struct crypto_alg *alg;
 
-	t_alg = kzalloc(sizeof(struct ssi_crypto_alg), GFP_KERNEL);
+	t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
 	if (!t_alg) {
 		SSI_LOG_ERR("failed to allocate t_alg\n");
 		return ERR_PTR(-ENOMEM);
@@ -1286,7 +1292,7 @@ int ssi_ablkcipher_free(struct ssi_drvdata *drvdata)
 	if (blkcipher_handle) {
 		/* Remove registered algs */
 		list_for_each_entry_safe(t_alg, n,
-				&blkcipher_handle->blkcipher_alg_list,
+					 &blkcipher_handle->blkcipher_alg_list,
 					 entry) {
 			crypto_unregister_alg(&t_alg->crypto_alg);
 			list_del(&t_alg->entry);
@@ -1305,8 +1311,7 @@ int ssi_ablkcipher_alloc(struct ssi_drvdata *drvdata)
 	int rc = -ENOMEM;
 	int alg;
 
-	ablkcipher_handle = kmalloc(sizeof(struct ssi_blkcipher_handle),
-		GFP_KERNEL);
+	ablkcipher_handle = kmalloc(sizeof(*ablkcipher_handle), GFP_KERNEL);
 	if (!ablkcipher_handle)
 		return -ENOMEM;
 
@@ -1322,7 +1327,7 @@ int ssi_ablkcipher_alloc(struct ssi_drvdata *drvdata)
 		if (IS_ERR(t_alg)) {
 			rc = PTR_ERR(t_alg);
 			SSI_LOG_ERR("%s alg allocation failed\n",
-				 blkcipher_algs[alg].driver_name);
+				    blkcipher_algs[alg].driver_name);
 			goto fail0;
 		}
 		t_alg->drvdata = drvdata;
@@ -1330,17 +1335,17 @@ int ssi_ablkcipher_alloc(struct ssi_drvdata *drvdata)
 		SSI_LOG_DEBUG("registering %s\n", blkcipher_algs[alg].driver_name);
 		rc = crypto_register_alg(&t_alg->crypto_alg);
 		SSI_LOG_DEBUG("%s alg registration rc = %x\n",
-			t_alg->crypto_alg.cra_driver_name, rc);
+			      t_alg->crypto_alg.cra_driver_name, rc);
 		if (unlikely(rc != 0)) {
 			SSI_LOG_ERR("%s alg registration failed\n",
-				t_alg->crypto_alg.cra_driver_name);
+				    t_alg->crypto_alg.cra_driver_name);
 			kfree(t_alg);
 			goto fail0;
 		} else {
 			list_add_tail(&t_alg->entry,
 				      &ablkcipher_handle->blkcipher_alg_list);
 			SSI_LOG_DEBUG("Registered %s\n",
-					t_alg->crypto_alg.cra_driver_name);
+				      t_alg->crypto_alg.cra_driver_name);
 		}
 	}
 	return 0;
diff --git a/drivers/staging/ccree/ssi_driver.c b/drivers/staging/ccree/ssi_driver.c
index 78709b92736d..9c6f1200c130 100644
--- a/drivers/staging/ccree/ssi_driver.c
+++ b/drivers/staging/ccree/ssi_driver.c
@@ -71,7 +71,7 @@
 #include "ssi_ivgen.h"
 #include "ssi_sram_mgr.h"
 #include "ssi_pm.h"
-#include "ssi_fips_local.h"
+#include "ssi_fips.h"
 
 #ifdef DX_DUMP_BYTES
 void dump_byte_array(const char *name, const u8 *the_array, unsigned long size)
@@ -81,12 +81,11 @@ void dump_byte_array(const char *name, const u8 *the_array, unsigned long size)
 	char line_buf[80];
 
 	if (!the_array) {
-		SSI_LOG_ERR("cannot dump_byte_array - NULL pointer\n");
+		SSI_LOG_ERR("cannot dump array - NULL pointer\n");
 		return;
 	}
 
-	ret = snprintf(line_buf, sizeof(line_buf), "%s[%lu]: ",
-		name, size);
+	ret = snprintf(line_buf, sizeof(line_buf), "%s[%lu]: ", name, size);
 	if (ret < 0) {
 		SSI_LOG_ERR("snprintf returned %d . aborting buffer array dump\n", ret);
 		return;
@@ -95,8 +94,8 @@ void dump_byte_array(const char *name, const u8 *the_array, unsigned long size)
 	for (i = 0, cur_byte = the_array;
 	     (i < size) && (line_offset < sizeof(line_buf)); i++, cur_byte++) {
 			ret = snprintf(line_buf + line_offset,
-					sizeof(line_buf) - line_offset,
-					"0x%02X ", *cur_byte);
+				       sizeof(line_buf) - line_offset,
+				       "0x%02X ", *cur_byte);
 		if (ret < 0) {
 			SSI_LOG_ERR("snprintf returned %d . aborting buffer array dump\n", ret);
 			return;
@@ -193,11 +192,11 @@ int init_cc_regs(struct ssi_drvdata *drvdata, bool is_probe)
 #ifdef DX_IRQ_DELAY
 	/* Set CC IRQ delay */
 	CC_HAL_WRITE_REGISTER(CC_REG_OFFSET(HOST_RGF, HOST_IRQ_TIMER_INIT_VAL),
-		DX_IRQ_DELAY);
+			      DX_IRQ_DELAY);
 #endif
 	if (CC_HAL_READ_REGISTER(CC_REG_OFFSET(HOST_RGF, HOST_IRQ_TIMER_INIT_VAL)) > 0) {
 		SSI_LOG_DEBUG("irq_delay=%d CC cycles\n",
-			CC_HAL_READ_REGISTER(CC_REG_OFFSET(HOST_RGF, HOST_IRQ_TIMER_INIT_VAL)));
+			      CC_HAL_READ_REGISTER(CC_REG_OFFSET(HOST_RGF, HOST_IRQ_TIMER_INIT_VAL)));
 	}
 #endif
 
@@ -224,7 +223,8 @@ static int init_cc_resources(struct platform_device *plat_dev)
 	struct resource *req_mem_cc_regs = NULL;
 	void __iomem *cc_base = NULL;
 	bool irq_registered = false;
-	struct ssi_drvdata *new_drvdata = kzalloc(sizeof(struct ssi_drvdata), GFP_KERNEL);
+	struct ssi_drvdata *new_drvdata = kzalloc(sizeof(*new_drvdata),
+						  GFP_KERNEL);
 	struct device *dev = &plat_dev->dev;
 	struct device_node *np = dev->of_node;
 	u32 signature_val;
@@ -251,10 +251,10 @@ static int init_cc_resources(struct platform_device *plat_dev)
 		rc = -ENODEV;
 		goto init_cc_res_err;
 	}
-	SSI_LOG_DEBUG("Got MEM resource (%s): start=0x%llX end=0x%llX\n",
-		new_drvdata->res_mem->name,
-		(unsigned long long)new_drvdata->res_mem->start,
-		(unsigned long long)new_drvdata->res_mem->end);
+	SSI_LOG_DEBUG("Got MEM resource (%s): start=%pad end=%pad\n",
+		      new_drvdata->res_mem->name,
+		      new_drvdata->res_mem->start,
+		      new_drvdata->res_mem->end);
 	/* Map registers space */
 	req_mem_cc_regs = request_mem_region(new_drvdata->res_mem->start, resource_size(new_drvdata->res_mem), "arm_cc7x_regs");
 	if (unlikely(!req_mem_cc_regs)) {
@@ -266,7 +266,8 @@ static int init_cc_resources(struct platform_device *plat_dev)
 	cc_base = ioremap(new_drvdata->res_mem->start, resource_size(new_drvdata->res_mem));
 	if (unlikely(!cc_base)) {
 		SSI_LOG_ERR("ioremap[CC](0x%08X,0x%08X) failed\n",
-			(unsigned int)new_drvdata->res_mem->start, (unsigned int)resource_size(new_drvdata->res_mem));
+			    (unsigned int)new_drvdata->res_mem->start,
+			    (unsigned int)resource_size(new_drvdata->res_mem));
 		rc = -ENOMEM;
 		goto init_cc_res_err;
 	}
@@ -284,15 +285,15 @@ static int init_cc_resources(struct platform_device *plat_dev)
 			 IRQF_SHARED, "arm_cc7x", new_drvdata);
 	if (unlikely(rc != 0)) {
 		SSI_LOG_ERR("Could not register to interrupt %llu\n",
-			(unsigned long long)new_drvdata->res_irq->start);
+			    (unsigned long long)new_drvdata->res_irq->start);
 		goto init_cc_res_err;
 	}
 	init_completion(&new_drvdata->icache_setup_completion);
 
 	irq_registered = true;
 	SSI_LOG_DEBUG("Registered to IRQ (%s) %llu\n",
-		new_drvdata->res_irq->name,
-		(unsigned long long)new_drvdata->res_irq->start);
+		      new_drvdata->res_irq->name,
+		      (unsigned long long)new_drvdata->res_irq->start);
 
 	new_drvdata->plat_dev = plat_dev;
 
@@ -301,19 +302,16 @@ static int init_cc_resources(struct platform_device *plat_dev)
 		goto init_cc_res_err;
 
 	if (!new_drvdata->plat_dev->dev.dma_mask)
-	{
 		new_drvdata->plat_dev->dev.dma_mask = &new_drvdata->plat_dev->dev.coherent_dma_mask;
-	}
+
 	if (!new_drvdata->plat_dev->dev.coherent_dma_mask)
-	{
 		new_drvdata->plat_dev->dev.coherent_dma_mask = DMA_BIT_MASK(DMA_BIT_MASK_LEN);
-	}
 
 	/* Verify correct mapping */
 	signature_val = CC_HAL_READ_REGISTER(CC_REG_OFFSET(HOST_RGF, HOST_SIGNATURE));
 	if (signature_val != DX_DEV_SIGNATURE) {
 		SSI_LOG_ERR("Invalid CC signature: SIGNATURE=0x%08X != expected=0x%08X\n",
-			signature_val, (u32)DX_DEV_SIGNATURE);
+			    signature_val, (u32)DX_DEV_SIGNATURE);
 		rc = -EINVAL;
 		goto init_cc_res_err;
 	}
@@ -330,7 +328,7 @@ static int init_cc_resources(struct platform_device *plat_dev)
 	}
 
 #ifdef ENABLE_CC_SYSFS
-	rc = ssi_sysfs_init(&(plat_dev->dev.kobj), new_drvdata);
+	rc = ssi_sysfs_init(&plat_dev->dev.kobj, new_drvdata);
 	if (unlikely(rc != 0)) {
 		SSI_LOG_ERR("init_stat_db failed\n");
 		goto init_cc_res_err;
@@ -401,6 +399,12 @@ static int init_cc_resources(struct platform_device *plat_dev)
 		goto init_cc_res_err;
 	}
 
+	/* If we got here and FIPS mode is enabled
+	 * it means all FIPS test passed, so let TEE
+	 * know we're good.
+	 */
+	cc_set_ree_fips_status(new_drvdata, true);
+
 	return 0;
 
 init_cc_res_err:
@@ -428,7 +432,7 @@ init_cc_res_err:
 				new_drvdata->cc_base = NULL;
 			}
 			release_mem_region(new_drvdata->res_mem->start,
-				resource_size(new_drvdata->res_mem));
+					   resource_size(new_drvdata->res_mem));
 			new_drvdata->res_mem = NULL;
 		}
 		kfree(new_drvdata);
@@ -471,7 +475,7 @@ static void cleanup_cc_resources(struct platform_device *plat_dev)
 	if (drvdata->cc_base) {
 		iounmap(drvdata->cc_base);
 		release_mem_region(drvdata->res_mem->start,
-			resource_size(drvdata->res_mem));
+				   resource_size(drvdata->res_mem));
 		drvdata->cc_base = NULL;
 		drvdata->res_mem = NULL;
 	}
@@ -516,12 +520,12 @@ static int cc7x_probe(struct platform_device *plat_dev)
 	asm volatile("mrc p15, 0, %0, c0, c0, 1" : "=r" (ctr));
 	cacheline_size =  4 << ((ctr >> 16) & 0xf);
 	SSI_LOG_DEBUG("CP15(L1_CACHE_BYTES) = %u , Kconfig(L1_CACHE_BYTES) = %u\n",
-		cacheline_size, L1_CACHE_BYTES);
+		      cacheline_size, L1_CACHE_BYTES);
 
 	asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (ctr));
-	SSI_LOG_DEBUG("Main ID register (MIDR): Implementer 0x%02X, Arch 0x%01X,"
-		     " Part 0x%03X, Rev r%dp%d\n",
-		(ctr >> 24), (ctr >> 16) & 0xF, (ctr >> 4) & 0xFFF, (ctr >> 20) & 0xF, ctr & 0xF);
+	SSI_LOG_DEBUG("Main ID register (MIDR): Implementer 0x%02X, Arch 0x%01X, Part 0x%03X, Rev r%dp%d\n",
+		      (ctr >> 24), (ctr >> 16) & 0xF, (ctr >> 4) & 0xFFF,
+		      (ctr >> 20) & 0xF, ctr & 0xF);
 #endif
 
 	/* Map registers space */
@@ -546,7 +550,7 @@ static int cc7x_remove(struct platform_device *plat_dev)
 }
 
 #if defined(CONFIG_PM_RUNTIME) || defined(CONFIG_PM_SLEEP)
-static struct dev_pm_ops arm_cc7x_driver_pm = {
+static const struct dev_pm_ops arm_cc7x_driver_pm = {
 	SET_RUNTIME_PM_OPS(ssi_power_mgr_runtime_suspend, ssi_power_mgr_runtime_resume, NULL)
 };
 #endif
diff --git a/drivers/staging/ccree/ssi_driver.h b/drivers/staging/ccree/ssi_driver.h
index c1ed61f1a202..b6ad89ae9bee 100644
--- a/drivers/staging/ccree/ssi_driver.h
+++ b/drivers/staging/ccree/ssi_driver.h
@@ -48,7 +48,6 @@
 #include "cc_crypto_ctx.h"
 #include "ssi_sysfs.h"
 #include "hash_defs.h"
-#include "ssi_fips_local.h"
 #include "cc_hw_queue_defs.h"
 #include "ssi_sram_mgr.h"
 
diff --git a/drivers/staging/ccree/ssi_fips.c b/drivers/staging/ccree/ssi_fips.c
index fdc40f38332a..33d53d64603d 100644
--- a/drivers/staging/ccree/ssi_fips.c
+++ b/drivers/staging/ccree/ssi_fips.c
@@ -14,48 +14,115 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-/**************************************************************
- * This file defines the driver FIPS APIs                     *
- **************************************************************/
+#include <linux/kernel.h>
+#include <linux/fips.h>
 
-#include <linux/module.h>
+#include "ssi_config.h"
+#include "ssi_driver.h"
+#include "cc_hal.h"
 #include "ssi_fips.h"
 
-extern int ssi_fips_ext_get_state(enum cc_fips_state_t *p_state);
-extern int ssi_fips_ext_get_error(enum cc_fips_error *p_err);
+static void fips_dsr(unsigned long devarg);
+
+struct ssi_fips_handle {
+	struct tasklet_struct tasklet;
+};
+
+/* The function called once at driver entry point to check
+ * whether TEE FIPS error occurred.
+ */
+static bool cc_get_tee_fips_status(struct ssi_drvdata *drvdata)
+{
+	u32 reg;
+	void __iomem *cc_base = drvdata->cc_base;
+
+	reg = CC_HAL_READ_REGISTER(CC_REG_OFFSET(HOST_RGF, GPR_HOST));
+	return (reg == (CC_FIPS_SYNC_TEE_STATUS | CC_FIPS_SYNC_MODULE_OK));
+}
 
 /*
- * This function returns the REE FIPS state.
- * It should be called by kernel module.
+ * This function should push the FIPS REE library status towards the TEE library
+ * by writing the error state to HOST_GPR0 register.
  */
-int ssi_fips_get_state(enum cc_fips_state_t *p_state)
+void cc_set_ree_fips_status(struct ssi_drvdata *drvdata, bool status)
+{
+	void __iomem *cc_base = drvdata->cc_base;
+	int val = CC_FIPS_SYNC_REE_STATUS;
+
+	val |= (status ? CC_FIPS_SYNC_MODULE_OK : CC_FIPS_SYNC_MODULE_ERROR);
+
+	CC_HAL_WRITE_REGISTER(CC_REG_OFFSET(HOST_RGF, HOST_GPR0), val);
+}
+
+void ssi_fips_fini(struct ssi_drvdata *drvdata)
 {
-	int rc = 0;
+	struct ssi_fips_handle *fips_h = drvdata->fips_handle;
 
-	if (!p_state)
-		return -EINVAL;
+	if (!fips_h)
+		return; /* Not allocated */
 
-	rc = ssi_fips_ext_get_state(p_state);
+	/* Kill tasklet */
+	tasklet_kill(&fips_h->tasklet);
 
-	return rc;
+	kfree(fips_h);
+	drvdata->fips_handle = NULL;
 }
 
-EXPORT_SYMBOL(ssi_fips_get_state);
+void fips_handler(struct ssi_drvdata *drvdata)
+{
+	struct ssi_fips_handle *fips_handle_ptr =
+		drvdata->fips_handle;
 
-/*
- * This function returns the REE FIPS error.
- * It should be called by kernel module.
- */
-int ssi_fips_get_error(enum cc_fips_error *p_err)
+	tasklet_schedule(&fips_handle_ptr->tasklet);
+}
+
+static inline void tee_fips_error(void)
 {
-	int rc = 0;
+	if (fips_enabled)
+		panic("ccree: TEE reported cryptographic error in fips mode!\n");
+	else
+		SSI_LOG_ERR("TEE reported error!\n");
+}
 
-	if (!p_err)
-		return -EINVAL;
+/* Deferred service handler, run as interrupt-fired tasklet */
+static void fips_dsr(unsigned long devarg)
+{
+	struct ssi_drvdata *drvdata = (struct ssi_drvdata *)devarg;
+	void __iomem *cc_base = drvdata->cc_base;
+	u32 irq, state, val;
 
-	rc = ssi_fips_ext_get_error(p_err);
+	irq = (drvdata->irq & (SSI_GPR0_IRQ_MASK));
 
-	return rc;
+	if (irq) {
+		state = CC_HAL_READ_REGISTER(CC_REG_OFFSET(HOST_RGF, GPR_HOST));
+
+		if (state != (CC_FIPS_SYNC_TEE_STATUS | CC_FIPS_SYNC_MODULE_OK))
+			tee_fips_error();
+	}
+
+	/* after verifing that there is nothing to do,
+	 * unmask AXI completion interrupt.
+	 */
+	val = (CC_REG_OFFSET(HOST_RGF, HOST_IMR) & ~irq);
+	CC_HAL_WRITE_REGISTER(CC_REG_OFFSET(HOST_RGF, HOST_IMR), val);
 }
 
-EXPORT_SYMBOL(ssi_fips_get_error);
+/* The function called once at driver entry point .*/
+int ssi_fips_init(struct ssi_drvdata *p_drvdata)
+{
+	struct ssi_fips_handle *fips_h;
+
+	fips_h = kzalloc(sizeof(*fips_h), GFP_KERNEL);
+	if (!fips_h)
+		return -ENOMEM;
+
+	p_drvdata->fips_handle = fips_h;
+
+	SSI_LOG_DEBUG("Initializing fips tasklet\n");
+	tasklet_init(&fips_h->tasklet, fips_dsr, (unsigned long)p_drvdata);
+
+	if (!cc_get_tee_fips_status(p_drvdata))
+		tee_fips_error();
+
+	return 0;
+}
diff --git a/drivers/staging/ccree/ssi_fips.h b/drivers/staging/ccree/ssi_fips.h
index 4f5c6a9a8363..369ddf9478e7 100644
--- a/drivers/staging/ccree/ssi_fips.h
+++ b/drivers/staging/ccree/ssi_fips.h
@@ -17,45 +17,33 @@
 #ifndef __SSI_FIPS_H__
 #define __SSI_FIPS_H__
 
-/*!
- * @file
- * @brief This file contains FIPS related defintions and APIs.
- */
+#ifdef CONFIG_CRYPTO_FIPS
 
-enum cc_fips_state {
-	CC_FIPS_STATE_NOT_SUPPORTED = 0,
-	CC_FIPS_STATE_SUPPORTED,
-	CC_FIPS_STATE_ERROR,
-	CC_FIPS_STATE_RESERVE32B = S32_MAX
+enum cc_fips_status {
+	CC_FIPS_SYNC_MODULE_OK = 0x0,
+	CC_FIPS_SYNC_MODULE_ERROR = 0x1,
+	CC_FIPS_SYNC_REE_STATUS = 0x4,
+	CC_FIPS_SYNC_TEE_STATUS = 0x8,
+	CC_FIPS_SYNC_STATUS_RESERVE32B = S32_MAX
 };
 
-enum cc_fips_error {
-	CC_REE_FIPS_ERROR_OK = 0,
-	CC_REE_FIPS_ERROR_GENERAL,
-	CC_REE_FIPS_ERROR_FROM_TEE,
-	CC_REE_FIPS_ERROR_AES_ECB_PUT,
-	CC_REE_FIPS_ERROR_AES_CBC_PUT,
-	CC_REE_FIPS_ERROR_AES_OFB_PUT,
-	CC_REE_FIPS_ERROR_AES_CTR_PUT,
-	CC_REE_FIPS_ERROR_AES_CBC_CTS_PUT,
-	CC_REE_FIPS_ERROR_AES_XTS_PUT,
-	CC_REE_FIPS_ERROR_AES_CMAC_PUT,
-	CC_REE_FIPS_ERROR_AESCCM_PUT,
-	CC_REE_FIPS_ERROR_AESGCM_PUT,
-	CC_REE_FIPS_ERROR_DES_ECB_PUT,
-	CC_REE_FIPS_ERROR_DES_CBC_PUT,
-	CC_REE_FIPS_ERROR_SHA1_PUT,
-	CC_REE_FIPS_ERROR_SHA256_PUT,
-	CC_REE_FIPS_ERROR_SHA512_PUT,
-	CC_REE_FIPS_ERROR_HMAC_SHA1_PUT,
-	CC_REE_FIPS_ERROR_HMAC_SHA256_PUT,
-	CC_REE_FIPS_ERROR_HMAC_SHA512_PUT,
-	CC_REE_FIPS_ERROR_ROM_CHECKSUM,
-	CC_REE_FIPS_ERROR_RESERVE32B = S32_MAX
-};
+int ssi_fips_init(struct ssi_drvdata *p_drvdata);
+void ssi_fips_fini(struct ssi_drvdata *drvdata);
+void fips_handler(struct ssi_drvdata *drvdata);
+void cc_set_ree_fips_status(struct ssi_drvdata *drvdata, bool ok);
+
+#else  /* CONFIG_CRYPTO_FIPS */
+
+static inline int ssi_fips_init(struct ssi_drvdata *p_drvdata)
+{
+	return 0;
+}
+
+static inline void ssi_fips_fini(struct ssi_drvdata *drvdata) {}
+void cc_set_ree_fips_status(struct ssi_drvdata *drvdata, bool ok) {}
+void fips_handler(struct ssi_drvdata *drvdata) {}
 
-int ssi_fips_get_state(enum cc_fips_state *p_state);
-int ssi_fips_get_error(enum cc_fips_error *p_err);
+#endif /* CONFIG_CRYPTO_FIPS */
 
 #endif  /*__SSI_FIPS_H__*/
 
diff --git a/drivers/staging/ccree/ssi_fips_data.h b/drivers/staging/ccree/ssi_fips_data.h
deleted file mode 100644
index c41671dbee40..000000000000
--- a/drivers/staging/ccree/ssi_fips_data.h
+++ /dev/null
@@ -1,306 +0,0 @@
-/*
- * Copyright (C) 2012-2017 ARM Limited or its affiliates.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-/*
- * The test vectors were taken from:
- *
- * * AES
- * NIST Special Publication 800-38A 2001 Edition
- * Recommendation for Block Cipher Modes of Operation
- * http://csrc.nist.gov/publications/nistpubs/800-38a/sp800-38a.pdf
- * Appendix F: Example Vectors for Modes of Operation of the AES
- *
- * * AES CTS
- * Advanced Encryption Standard (AES) Encryption for Kerberos 5
- * February 2005
- * https://tools.ietf.org/html/rfc3962#appendix-B
- * B.  Sample Test Vectors
- *
- * * AES XTS
- * http://csrc.nist.gov/groups/STM/cavp/#08
- * http://csrc.nist.gov/groups/STM/cavp/documents/aes/XTSTestVectors.zip
- *
- * * AES CMAC
- * http://csrc.nist.gov/groups/STM/cavp/index.html#07
- * http://csrc.nist.gov/groups/STM/cavp/documents/mac/cmactestvectors.zip
- *
- * * AES-CCM
- * http://csrc.nist.gov/groups/STM/cavp/#07
- * http://csrc.nist.gov/groups/STM/cavp/documents/mac/ccmtestvectors.zip
- *
- * * AES-GCM
- * http://csrc.nist.gov/groups/STM/cavp/documents/mac/gcmtestvectors.zip
- *
- * * Triple-DES
- * NIST Special Publication 800-67 January 2012
- * Recommendation for the Triple Data Encryption Algorithm (TDEA) Block Cipher
- * http://csrc.nist.gov/publications/nistpubs/800-67-Rev1/SP-800-67-Rev1.pdf
- * APPENDIX B: EXAMPLE OF TDEA FORWARD AND INVERSE CIPHER OPERATIONS
- * and
- * http://csrc.nist.gov/groups/STM/cavp/#01
- * http://csrc.nist.gov/groups/STM/cavp/documents/des/tdesmct_intermediate.zip
- *
- * * HASH
- * http://csrc.nist.gov/groups/STM/cavp/#03
- * http://csrc.nist.gov/groups/STM/cavp/documents/shs/shabytetestvectors.zip
- *
- * * HMAC
- * http://csrc.nist.gov/groups/STM/cavp/#07
- * http://csrc.nist.gov/groups/STM/cavp/documents/mac/hmactestvectors.zip
- */
-
-/* NIST AES */
-#define AES_128_BIT_KEY_SIZE    16
-#define AES_192_BIT_KEY_SIZE    24
-#define AES_256_BIT_KEY_SIZE    32
-#define AES_512_BIT_KEY_SIZE    64
-
-#define NIST_AES_IV_SIZE        16
-
-#define NIST_AES_128_KEY        { 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c }
-#define NIST_AES_192_KEY        { 0x8e, 0x73, 0xb0, 0xf7, 0xda, 0x0e, 0x64, 0x52, 0xc8, 0x10, 0xf3, 0x2b, 0x80, 0x90, 0x79, 0xe5, \
-				  0x62, 0xf8, 0xea, 0xd2, 0x52, 0x2c, 0x6b, 0x7b }
-#define NIST_AES_256_KEY        { 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, 0x2b, 0x73, 0xae, 0xf0, 0x85, 0x7d, 0x77, 0x81, \
-				  0x1f, 0x35, 0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, 0x2d, 0x98, 0x10, 0xa3, 0x09, 0x14, 0xdf, 0xf4 }
-#define NIST_AES_VECTOR_SIZE    16
-#define NIST_AES_PLAIN_DATA     { 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a }
-
-#define NIST_AES_ECB_IV         { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }
-#define NIST_AES_128_ECB_CIPHER { 0x3a, 0xd7, 0x7b, 0xb4, 0x0d, 0x7a, 0x36, 0x60, 0xa8, 0x9e, 0xca, 0xf3, 0x24, 0x66, 0xef, 0x97 }
-#define NIST_AES_192_ECB_CIPHER { 0xbd, 0x33, 0x4f, 0x1d, 0x6e, 0x45, 0xf2, 0x5f, 0xf7, 0x12, 0xa2, 0x14, 0x57, 0x1f, 0xa5, 0xcc }
-#define NIST_AES_256_ECB_CIPHER { 0xf3, 0xee, 0xd1, 0xbd, 0xb5, 0xd2, 0xa0, 0x3c, 0x06, 0x4b, 0x5a, 0x7e, 0x3d, 0xb1, 0x81, 0xf8 }
-
-#define NIST_AES_CBC_IV         { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }
-#define NIST_AES_128_CBC_CIPHER { 0x76, 0x49, 0xab, 0xac, 0x81, 0x19, 0xb2, 0x46, 0xce, 0xe9, 0x8e, 0x9b, 0x12, 0xe9, 0x19, 0x7d }
-#define NIST_AES_192_CBC_CIPHER { 0x4f, 0x02, 0x1d, 0xb2, 0x43, 0xbc, 0x63, 0x3d, 0x71, 0x78, 0x18, 0x3a, 0x9f, 0xa0, 0x71, 0xe8 }
-#define NIST_AES_256_CBC_CIPHER { 0xf5, 0x8c, 0x4c, 0x04, 0xd6, 0xe5, 0xf1, 0xba, 0x77, 0x9e, 0xab, 0xfb, 0x5f, 0x7b, 0xfb, 0xd6 }
-
-#define NIST_AES_OFB_IV         { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }
-#define NIST_AES_128_OFB_CIPHER { 0x3b, 0x3f, 0xd9, 0x2e, 0xb7, 0x2d, 0xad, 0x20, 0x33, 0x34, 0x49, 0xf8, 0xe8, 0x3c, 0xfb, 0x4a }
-#define NIST_AES_192_OFB_CIPHER { 0xcd, 0xc8, 0x0d, 0x6f, 0xdd, 0xf1, 0x8c, 0xab, 0x34, 0xc2, 0x59, 0x09, 0xc9, 0x9a, 0x41, 0x74 }
-#define NIST_AES_256_OFB_CIPHER { 0xdc, 0x7e, 0x84, 0xbf, 0xda, 0x79, 0x16, 0x4b, 0x7e, 0xcd, 0x84, 0x86, 0x98, 0x5d, 0x38, 0x60 }
-
-#define NIST_AES_CTR_IV         { 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff }
-#define NIST_AES_128_CTR_CIPHER { 0x87, 0x4d, 0x61, 0x91, 0xb6, 0x20, 0xe3, 0x26, 0x1b, 0xef, 0x68, 0x64, 0x99, 0x0d, 0xb6, 0xce }
-#define NIST_AES_192_CTR_CIPHER { 0x1a, 0xbc, 0x93, 0x24, 0x17, 0x52, 0x1c, 0xa2, 0x4f, 0x2b, 0x04, 0x59, 0xfe, 0x7e, 0x6e, 0x0b }
-#define NIST_AES_256_CTR_CIPHER { 0x60, 0x1e, 0xc3, 0x13, 0x77, 0x57, 0x89, 0xa5, 0xb7, 0xa7, 0xf5, 0x04, 0xbb, 0xf3, 0xd2, 0x28 }
-
-#define RFC3962_AES_128_KEY            { 0x63, 0x68, 0x69, 0x63, 0x6b, 0x65, 0x6e, 0x20, 0x74, 0x65, 0x72, 0x69, 0x79, 0x61, 0x6b, 0x69 }
-#define RFC3962_AES_VECTOR_SIZE        17
-#define RFC3962_AES_PLAIN_DATA         { 0x49, 0x20, 0x77, 0x6f, 0x75, 0x6c, 0x64, 0x20, 0x6c, 0x69, 0x6b, 0x65, 0x20, 0x74, 0x68, 0x65, 0x20 }
-#define RFC3962_AES_CBC_CTS_IV         { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }
-#define RFC3962_AES_128_CBC_CTS_CIPHER { 0xc6, 0x35, 0x35, 0x68, 0xf2, 0xbf, 0x8c, 0xb4, 0xd8, 0xa5, 0x80, 0x36, 0x2d, 0xa7, 0xff, 0x7f, 0x97 }
-
-#define NIST_AES_256_XTS_KEY            { 0xa1, 0xb9, 0x0c, 0xba, 0x3f, 0x06, 0xac, 0x35, 0x3b, 0x2c, 0x34, 0x38, 0x76, 0x08, 0x17, 0x62, \
-					  0x09, 0x09, 0x23, 0x02, 0x6e, 0x91, 0x77, 0x18, 0x15, 0xf2, 0x9d, 0xab, 0x01, 0x93, 0x2f, 0x2f }
-#define NIST_AES_256_XTS_IV             { 0x4f, 0xae, 0xf7, 0x11, 0x7c, 0xda, 0x59, 0xc6, 0x6e, 0x4b, 0x92, 0x01, 0x3e, 0x76, 0x8a, 0xd5 }
-#define NIST_AES_256_XTS_VECTOR_SIZE    16
-#define NIST_AES_256_XTS_PLAIN          { 0xeb, 0xab, 0xce, 0x95, 0xb1, 0x4d, 0x3c, 0x8d, 0x6f, 0xb3, 0x50, 0x39, 0x07, 0x90, 0x31, 0x1c }
-#define NIST_AES_256_XTS_CIPHER         { 0x77, 0x8a, 0xe8, 0xb4, 0x3c, 0xb9, 0x8d, 0x5a, 0x82, 0x50, 0x81, 0xd5, 0xbe, 0x47, 0x1c, 0x63 }
-
-#define NIST_AES_512_XTS_KEY            { 0x1e, 0xa6, 0x61, 0xc5, 0x8d, 0x94, 0x3a, 0x0e, 0x48, 0x01, 0xe4, 0x2f, 0x4b, 0x09, 0x47, 0x14, \
-					  0x9e, 0x7f, 0x9f, 0x8e, 0x3e, 0x68, 0xd0, 0xc7, 0x50, 0x52, 0x10, 0xbd, 0x31, 0x1a, 0x0e, 0x7c, \
-					  0xd6, 0xe1, 0x3f, 0xfd, 0xf2, 0x41, 0x8d, 0x8d, 0x19, 0x11, 0xc0, 0x04, 0xcd, 0xa5, 0x8d, 0xa3, \
-					  0xd6, 0x19, 0xb7, 0xe2, 0xb9, 0x14, 0x1e, 0x58, 0x31, 0x8e, 0xea, 0x39, 0x2c, 0xf4, 0x1b, 0x08 }
-#define NIST_AES_512_XTS_IV             { 0xad, 0xf8, 0xd9, 0x26, 0x27, 0x46, 0x4a, 0xd2, 0xf0, 0x42, 0x8e, 0x84, 0xa9, 0xf8, 0x75, 0x64,  }
-#define NIST_AES_512_XTS_VECTOR_SIZE    32
-#define NIST_AES_512_XTS_PLAIN          { 0x2e, 0xed, 0xea, 0x52, 0xcd, 0x82, 0x15, 0xe1, 0xac, 0xc6, 0x47, 0xe8, 0x10, 0xbb, 0xc3, 0x64, \
-					  0x2e, 0x87, 0x28, 0x7f, 0x8d, 0x2e, 0x57, 0xe3, 0x6c, 0x0a, 0x24, 0xfb, 0xc1, 0x2a, 0x20, 0x2e }
-#define NIST_AES_512_XTS_CIPHER         { 0xcb, 0xaa, 0xd0, 0xe2, 0xf6, 0xce, 0xa3, 0xf5, 0x0b, 0x37, 0xf9, 0x34, 0xd4, 0x6a, 0x9b, 0x13, \
-					  0x0b, 0x9d, 0x54, 0xf0, 0x7e, 0x34, 0xf3, 0x6a, 0xf7, 0x93, 0xe8, 0x6f, 0x73, 0xc6, 0xd7, 0xdb }
-
-/* NIST AES-CMAC */
-#define NIST_AES_128_CMAC_KEY           { 0x67, 0x08, 0xc9, 0x88, 0x7b, 0x84, 0x70, 0x84, 0xf1, 0x23, 0xd3, 0xdd, 0x9c, 0x3a, 0x81, 0x36 }
-#define NIST_AES_128_CMAC_PLAIN_DATA    { 0xa8, 0xde, 0x55, 0x17, 0x0c, 0x6d, 0xc0, 0xd8, 0x0d, 0xe3, 0x2f, 0x50, 0x8b, 0xf4, 0x9b, 0x70 }
-#define NIST_AES_128_CMAC_MAC           { 0xcf, 0xef, 0x9b, 0x78, 0x39, 0x84, 0x1f, 0xdb, 0xcc, 0xbb, 0x6c, 0x2c, 0xf2, 0x38, 0xf7 }
-#define NIST_AES_128_CMAC_VECTOR_SIZE   16
-#define NIST_AES_128_CMAC_OUTPUT_SIZE   15
-
-#define NIST_AES_192_CMAC_KEY           { 0x20, 0x51, 0xaf, 0x34, 0x76, 0x2e, 0xbe, 0x55, 0x6f, 0x72, 0xa5, 0xc6, 0xed, 0xc7, 0x77, 0x1e, \
-					  0xb9, 0x24, 0x5f, 0xad, 0x76, 0xf0, 0x34, 0xbe }
-#define NIST_AES_192_CMAC_PLAIN_DATA    { 0xae, 0x8e, 0x93, 0xc9, 0xc9, 0x91, 0xcf, 0x89, 0x6a, 0x49, 0x1a, 0x89, 0x07, 0xdf, 0x4e, 0x4b, \
-					  0xe5, 0x18, 0x6a, 0xe4, 0x96, 0xcd, 0x34, 0x0d, 0xc1, 0x9b, 0x23, 0x78, 0x21, 0xdb, 0x7b, 0x60 }
-#define NIST_AES_192_CMAC_MAC           { 0x74, 0xf7, 0x46, 0x08, 0xc0, 0x4f, 0x0f, 0x4e, 0x47, 0xfa, 0x64, 0x04, 0x33, 0xb6, 0xe6, 0xfb }
-#define NIST_AES_192_CMAC_VECTOR_SIZE   32
-#define NIST_AES_192_CMAC_OUTPUT_SIZE   16
-
-#define NIST_AES_256_CMAC_KEY           { 0x3a, 0x75, 0xa9, 0xd2, 0xbd, 0xb8, 0xc8, 0x04, 0xba, 0x4a, 0xb4, 0x98, 0x35, 0x73, 0xa6, 0xb2, \
-					  0x53, 0x16, 0x0d, 0xd9, 0x0f, 0x8e, 0xdd, 0xfb, 0x2f, 0xdc, 0x2a, 0xb1, 0x76, 0x04, 0xf5, 0xc5 }
-#define NIST_AES_256_CMAC_PLAIN_DATA    { 0x42, 0xf3, 0x5d, 0x5a, 0xa5, 0x33, 0xa7, 0xa0, 0xa5, 0xf7, 0x4e, 0x14, 0x4f, 0x2a, 0x5f, 0x20 }
-#define NIST_AES_256_CMAC_MAC           { 0xf1, 0x53, 0x2f, 0x87, 0x32, 0xd9, 0xf5, 0x90, 0x30, 0x07 }
-#define NIST_AES_256_CMAC_VECTOR_SIZE   16
-#define NIST_AES_256_CMAC_OUTPUT_SIZE   10
-
-/* NIST TDES */
-#define TDES_NUM_OF_KEYS                3
-#define NIST_TDES_VECTOR_SIZE           8
-#define NIST_TDES_IV_SIZE               8
-
-#define NIST_TDES_ECB_IV		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }
-
-#define NIST_TDES_ECB3_KEY		{ 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, \
-					  0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0x01, \
-					  0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0x01, 0x23 }
-#define NIST_TDES_ECB3_PLAIN_DATA	{ 0x54, 0x68, 0x65, 0x20, 0x71, 0x75, 0x66, 0x63 }
-#define NIST_TDES_ECB3_CIPHER		{ 0xa8, 0x26, 0xfd, 0x8c, 0xe5, 0x3b, 0x85, 0x5f }
-
-#define NIST_TDES_CBC3_IV		{ 0xf8, 0xee, 0xe1, 0x35, 0x9c, 0x6e, 0x54, 0x40 }
-#define NIST_TDES_CBC3_KEY		{ 0xe9, 0xda, 0x37, 0xf8, 0xdc, 0x97, 0x6d, 0x5b, \
-					  0xb6, 0x8c, 0x04, 0xe3, 0xec, 0x98, 0x20, 0x15, \
-					  0xf4, 0x0e, 0x08, 0xb5, 0x97, 0x29, 0xf2, 0x8f }
-#define NIST_TDES_CBC3_PLAIN_DATA	{ 0x3b, 0xb7, 0xa7, 0xdb, 0xa3, 0xd5, 0x92, 0x91 }
-#define NIST_TDES_CBC3_CIPHER		{ 0x5b, 0x84, 0x24, 0xd2, 0x39, 0x3e, 0x55, 0xa2 }
-
-/* NIST AES-CCM */
-#define NIST_AESCCM_128_BIT_KEY_SIZE    16
-#define NIST_AESCCM_192_BIT_KEY_SIZE    24
-#define NIST_AESCCM_256_BIT_KEY_SIZE    32
-
-#define NIST_AESCCM_B0_VAL              0x79  /* L'[0:2]=1 , M'[3-5]=7 , Adata[6]=1, reserved[7]=0 */
-#define NIST_AESCCM_NONCE_SIZE          13
-#define NIST_AESCCM_IV_SIZE             16
-#define NIST_AESCCM_ADATA_SIZE          32
-#define NIST_AESCCM_TEXT_SIZE           16
-#define NIST_AESCCM_TAG_SIZE            16
-
-#define NIST_AESCCM_128_KEY             { 0x70, 0x01, 0x0e, 0xd9, 0x0e, 0x61, 0x86, 0xec, 0xad, 0x41, 0xf0, 0xd3, 0xc7, 0xc4, 0x2f, 0xf8 }
-#define NIST_AESCCM_128_NONCE           { 0xa5, 0xf4, 0xf4, 0x98, 0x6e, 0x98, 0x47, 0x29, 0x65, 0xf5, 0xab, 0xcc, 0x4b }
-#define NIST_AESCCM_128_ADATA           { 0x3f, 0xec, 0x0e, 0x5c, 0xc2, 0x4d, 0x67, 0x13, 0x94, 0x37, 0xcb, 0xc8, 0x11, 0x24, 0x14, 0xfc, \
-					  0x8d, 0xac, 0xcd, 0x1a, 0x94, 0xb4, 0x9a, 0x4c, 0x76, 0xe2, 0xd3, 0x93, 0x03, 0x54, 0x73, 0x17 }
-#define NIST_AESCCM_128_PLAIN_TEXT      { 0xbe, 0x32, 0x2f, 0x58, 0xef, 0xa7, 0xf8, 0xc6, 0x8a, 0x63, 0x5e, 0x0b, 0x9c, 0xce, 0x77, 0xf2 }
-#define NIST_AESCCM_128_CIPHER          { 0x8e, 0x44, 0x25, 0xae, 0x57, 0x39, 0x74, 0xf0, 0xf0, 0x69, 0x3a, 0x18, 0x8b, 0x52, 0x58, 0x12 }
-#define NIST_AESCCM_128_MAC             { 0xee, 0xf0, 0x8e, 0x3f, 0xb1, 0x5f, 0x42, 0x27, 0xe0, 0xd9, 0x89, 0xa4, 0xd5, 0x87, 0xa8, 0xcf }
-
-#define NIST_AESCCM_192_KEY             { 0x68, 0x73, 0xf1, 0xc6, 0xc3, 0x09, 0x75, 0xaf, 0xf6, 0xf0, 0x84, 0x70, 0x26, 0x43, 0x21, 0x13, \
-					  0x0a, 0x6e, 0x59, 0x84, 0xad, 0xe3, 0x24, 0xe9 }
-#define NIST_AESCCM_192_NONCE           { 0x7c, 0x4d, 0x2f, 0x7c, 0xec, 0x04, 0x36, 0x1f, 0x18, 0x7f, 0x07, 0x26, 0xd5 }
-#define NIST_AESCCM_192_ADATA           { 0x77, 0x74, 0x3b, 0x5d, 0x83, 0xa0, 0x0d, 0x2c, 0x8d, 0x5f, 0x7e, 0x10, 0x78, 0x15, 0x31, 0xb4, \
-					  0x96, 0xe0, 0x9f, 0x3b, 0xc9, 0x29, 0x5d, 0x7a, 0xe9, 0x79, 0x9e, 0x64, 0x66, 0x8e, 0xf8, 0xc5 }
-#define NIST_AESCCM_192_PLAIN_TEXT      { 0x50, 0x51, 0xa0, 0xb0, 0xb6, 0x76, 0x6c, 0xd6, 0xea, 0x29, 0xa6, 0x72, 0x76, 0x9d, 0x40, 0xfe }
-#define NIST_AESCCM_192_CIPHER          { 0x0c, 0xe5, 0xac, 0x8d, 0x6b, 0x25, 0x6f, 0xb7, 0x58, 0x0b, 0xf6, 0xac, 0xc7, 0x64, 0x26, 0xaf }
-#define NIST_AESCCM_192_MAC             { 0x40, 0xbc, 0xe5, 0x8f, 0xd4, 0xcd, 0x65, 0x48, 0xdf, 0x90, 0xa0, 0x33, 0x7c, 0x84, 0x20, 0x04 }
-
-#define NIST_AESCCM_256_KEY             { 0xee, 0x8c, 0xe1, 0x87, 0x16, 0x97, 0x79, 0xd1, 0x3e, 0x44, 0x3d, 0x64, 0x28, 0xe3, 0x8b, 0x38, \
-					  0xb5, 0x5d, 0xfb, 0x90, 0xf0, 0x22, 0x8a, 0x8a, 0x4e, 0x62, 0xf8, 0xf5, 0x35, 0x80, 0x6e, 0x62 }
-#define NIST_AESCCM_256_NONCE           { 0x12, 0x16, 0x42, 0xc4, 0x21, 0x8b, 0x39, 0x1c, 0x98, 0xe6, 0x26, 0x9c, 0x8a }
-#define NIST_AESCCM_256_ADATA           { 0x71, 0x8d, 0x13, 0xe4, 0x75, 0x22, 0xac, 0x4c, 0xdf, 0x3f, 0x82, 0x80, 0x63, 0x98, 0x0b, 0x6d, \
-					  0x45, 0x2f, 0xcd, 0xcd, 0x6e, 0x1a, 0x19, 0x04, 0xbf, 0x87, 0xf5, 0x48, 0xa5, 0xfd, 0x5a, 0x05 }
-#define NIST_AESCCM_256_PLAIN_TEXT      { 0xd1, 0x5f, 0x98, 0xf2, 0xc6, 0xd6, 0x70, 0xf5, 0x5c, 0x78, 0xa0, 0x66, 0x48, 0x33, 0x2b, 0xc9 }
-#define NIST_AESCCM_256_CIPHER          { 0xcc, 0x17, 0xbf, 0x87, 0x94, 0xc8, 0x43, 0x45, 0x7d, 0x89, 0x93, 0x91, 0x89, 0x8e, 0xd2, 0x2a }
-#define NIST_AESCCM_256_MAC             { 0x6f, 0x9d, 0x28, 0xfc, 0xb6, 0x42, 0x34, 0xe1, 0xcd, 0x79, 0x3c, 0x41, 0x44, 0xf1, 0xda, 0x50 }
-
-/* NIST AES-GCM */
-#define NIST_AESGCM_128_BIT_KEY_SIZE    16
-#define NIST_AESGCM_192_BIT_KEY_SIZE    24
-#define NIST_AESGCM_256_BIT_KEY_SIZE    32
-
-#define NIST_AESGCM_IV_SIZE             12
-#define NIST_AESGCM_ADATA_SIZE          16
-#define NIST_AESGCM_TEXT_SIZE           16
-#define NIST_AESGCM_TAG_SIZE            16
-
-#define NIST_AESGCM_128_KEY             { 0x81, 0x6e, 0x39, 0x07, 0x04, 0x10, 0xcf, 0x21, 0x84, 0x90, 0x4d, 0xa0, 0x3e, 0xa5, 0x07, 0x5a }
-#define NIST_AESGCM_128_IV              { 0x32, 0xc3, 0x67, 0xa3, 0x36, 0x26, 0x13, 0xb2, 0x7f, 0xc3, 0xe6, 0x7e }
-#define NIST_AESGCM_128_ADATA           { 0xf2, 0xa3, 0x07, 0x28, 0xed, 0x87, 0x4e, 0xe0, 0x29, 0x83, 0xc2, 0x94, 0x43, 0x5d, 0x3c, 0x16 }
-#define NIST_AESGCM_128_PLAIN_TEXT      { 0xec, 0xaf, 0xe9, 0x6c, 0x67, 0xa1, 0x64, 0x67, 0x44, 0xf1, 0xc8, 0x91, 0xf5, 0xe6, 0x94, 0x27 }
-#define NIST_AESGCM_128_CIPHER          { 0x55, 0x2e, 0xbe, 0x01, 0x2e, 0x7b, 0xcf, 0x90, 0xfc, 0xef, 0x71, 0x2f, 0x83, 0x44, 0xe8, 0xf1 }
-#define NIST_AESGCM_128_MAC             { 0xec, 0xaa, 0xe9, 0xfc, 0x68, 0x27, 0x6a, 0x45, 0xab, 0x0c, 0xa3, 0xcb, 0x9d, 0xd9, 0x53, 0x9f }
-
-#define NIST_AESGCM_192_KEY             { 0x0c, 0x44, 0xd6, 0xc9, 0x28, 0xee, 0x11, 0x2c, 0xe6, 0x65, 0xfe, 0x54, 0x7e, 0xbd, 0x38, 0x72, \
-					  0x98, 0xa9, 0x54, 0xb4, 0x62, 0xf6, 0x95, 0xd8 }
-#define NIST_AESGCM_192_IV              { 0x18, 0xb8, 0xf3, 0x20, 0xfe, 0xf4, 0xae, 0x8c, 0xcb, 0xe8, 0xf9, 0x52 }
-#define NIST_AESGCM_192_ADATA           { 0x73, 0x41, 0xd4, 0x3f, 0x98, 0xcf, 0x38, 0x82, 0x21, 0x18, 0x09, 0x41, 0x97, 0x03, 0x76, 0xe8 }
-#define NIST_AESGCM_192_PLAIN_TEXT      { 0x96, 0xad, 0x07, 0xf9, 0xb6, 0x28, 0xb6, 0x52, 0xcf, 0x86, 0xcb, 0x73, 0x17, 0x88, 0x6f, 0x51 }
-#define NIST_AESGCM_192_CIPHER          { 0xa6, 0x64, 0x07, 0x81, 0x33, 0x40, 0x5e, 0xb9, 0x09, 0x4d, 0x36, 0xf7, 0xe0, 0x70, 0x19, 0x1f }
-#define NIST_AESGCM_192_MAC             { 0xe8, 0xf9, 0xc3, 0x17, 0x84, 0x7c, 0xe3, 0xf3, 0xc2, 0x39, 0x94, 0xa4, 0x02, 0xf0, 0x65, 0x81 }
-
-#define NIST_AESGCM_256_KEY             { 0x54, 0xe3, 0x52, 0xea, 0x1d, 0x84, 0xbf, 0xe6, 0x4a, 0x10, 0x11, 0x09, 0x61, 0x11, 0xfb, 0xe7, \
-					  0x66, 0x8a, 0xd2, 0x20, 0x3d, 0x90, 0x2a, 0x01, 0x45, 0x8c, 0x3b, 0xbd, 0x85, 0xbf, 0xce, 0x14 }
-#define NIST_AESGCM_256_IV              { 0xdf, 0x7c, 0x3b, 0xca, 0x00, 0x39, 0x6d, 0x0c, 0x01, 0x84, 0x95, 0xd9 }
-#define NIST_AESGCM_256_ADATA           { 0x7e, 0x96, 0x8d, 0x71, 0xb5, 0x0c, 0x1f, 0x11, 0xfd, 0x00, 0x1f, 0x3f, 0xef, 0x49, 0xd0, 0x45 }
-#define NIST_AESGCM_256_PLAIN_TEXT      { 0x85, 0xfc, 0x3d, 0xfa, 0xd9, 0xb5, 0xa8, 0xd3, 0x25, 0x8e, 0x4f, 0xc4, 0x45, 0x71, 0xbd, 0x3b }
-#define NIST_AESGCM_256_CIPHER          { 0x42, 0x6e, 0x0e, 0xfc, 0x69, 0x3b, 0x7b, 0xe1, 0xf3, 0x01, 0x8d, 0xb7, 0xdd, 0xbb, 0x7e, 0x4d }
-#define NIST_AESGCM_256_MAC             { 0xee, 0x82, 0x57, 0x79, 0x5b, 0xe6, 0xa1, 0x16, 0x4d, 0x7e, 0x1d, 0x2d, 0x6c, 0xac, 0x77, 0xa7 }
-
-/* NIST HASH */
-#define NIST_SHA_MSG_SIZE               16
-
-#define NIST_SHA_1_MSG                  { 0x35, 0x52, 0x69, 0x4c, 0xdf, 0x66, 0x3f, 0xd9, 0x4b, 0x22, 0x47, 0x47, 0xac, 0x40, 0x6a, 0xaf }
-#define NIST_SHA_1_MD                   { 0xa1, 0x50, 0xde, 0x92, 0x74, 0x54, 0x20, 0x2d, 0x94, 0xe6, 0x56, 0xde, 0x4c, 0x7c, 0x0c, 0xa6, \
-					  0x91, 0xde, 0x95, 0x5d }
-
-#define NIST_SHA_256_MSG                { 0x0a, 0x27, 0x84, 0x7c, 0xdc, 0x98, 0xbd, 0x6f, 0x62, 0x22, 0x0b, 0x04, 0x6e, 0xdd, 0x76, 0x2b }
-#define NIST_SHA_256_MD                 { 0x80, 0xc2, 0x5e, 0xc1, 0x60, 0x05, 0x87, 0xe7, 0xf2, 0x8b, 0x18, 0xb1, 0xb1, 0x8e, 0x3c, 0xdc, \
-					  0x89, 0x92, 0x8e, 0x39, 0xca, 0xb3, 0xbc, 0x25, 0xe4, 0xd4, 0xa4, 0xc1, 0x39, 0xbc, 0xed, 0xc4 }
-
-#define NIST_SHA_512_MSG                { 0xcd, 0x67, 0xbd, 0x40, 0x54, 0xaa, 0xa3, 0xba, 0xa0, 0xdb, 0x17, 0x8c, 0xe2, 0x32, 0xfd, 0x5a }
-#define NIST_SHA_512_MD                 { 0x0d, 0x85, 0x21, 0xf8, 0xf2, 0xf3, 0x90, 0x03, 0x32, 0xd1, 0xa1, 0xa5, 0x5c, 0x60, 0xba, 0x81, \
-					  0xd0, 0x4d, 0x28, 0xdf, 0xe8, 0xc5, 0x04, 0xb6, 0x32, 0x8a, 0xe7, 0x87, 0x92, 0x5f, 0xe0, 0x18, \
-					  0x8f, 0x2b, 0xa9, 0x1c, 0x3a, 0x9f, 0x0c, 0x16, 0x53, 0xc4, 0xbf, 0x0a, 0xda, 0x35, 0x64, 0x55, \
-					  0xea, 0x36, 0xfd, 0x31, 0xf8, 0xe7, 0x3e, 0x39, 0x51, 0xca, 0xd4, 0xeb, 0xba, 0x8c, 0x6e, 0x04 }
-
-/* NIST HMAC */
-#define NIST_HMAC_MSG_SIZE              128
-
-#define NIST_HMAC_SHA1_KEY_SIZE         10
-#define NIST_HMAC_SHA1_KEY		{ 0x59, 0x78, 0x59, 0x28, 0xd7, 0x25, 0x16, 0xe3, 0x12, 0x72 }
-#define NIST_HMAC_SHA1_MSG		{ 0xa3, 0xce, 0x88, 0x99, 0xdf, 0x10, 0x22, 0xe8, 0xd2, 0xd5, 0x39, 0xb4, 0x7b, 0xf0, 0xe3, 0x09, \
-					  0xc6, 0x6f, 0x84, 0x09, 0x5e, 0x21, 0x43, 0x8e, 0xc3, 0x55, 0xbf, 0x11, 0x9c, 0xe5, 0xfd, 0xcb, \
-					  0x4e, 0x73, 0xa6, 0x19, 0xcd, 0xf3, 0x6f, 0x25, 0xb3, 0x69, 0xd8, 0xc3, 0x8f, 0xf4, 0x19, 0x99, \
-					  0x7f, 0x0c, 0x59, 0x83, 0x01, 0x08, 0x22, 0x36, 0x06, 0xe3, 0x12, 0x23, 0x48, 0x3f, 0xd3, 0x9e, \
-					  0xde, 0xaa, 0x4d, 0x3f, 0x0d, 0x21, 0x19, 0x88, 0x62, 0xd2, 0x39, 0xc9, 0xfd, 0x26, 0x07, 0x41, \
-					  0x30, 0xff, 0x6c, 0x86, 0x49, 0x3f, 0x52, 0x27, 0xab, 0x89, 0x5c, 0x8f, 0x24, 0x4b, 0xd4, 0x2c, \
-					  0x7a, 0xfc, 0xe5, 0xd1, 0x47, 0xa2, 0x0a, 0x59, 0x07, 0x98, 0xc6, 0x8e, 0x70, 0x8e, 0x96, 0x49, \
-					  0x02, 0xd1, 0x24, 0xda, 0xde, 0xcd, 0xbd, 0xa9, 0xdb, 0xd0, 0x05, 0x1e, 0xd7, 0x10, 0xe9, 0xbf }
-#define NIST_HMAC_SHA1_MD               { 0x3c, 0x81, 0x62, 0x58, 0x9a, 0xaf, 0xae, 0xe0, 0x24, 0xfc, 0x9a, 0x5c, 0xa5, 0x0d, 0xd2, 0x33, \
-					  0x6f, 0xe3, 0xeb, 0x28 }
-
-#define NIST_HMAC_SHA256_KEY_SIZE       40
-#define NIST_HMAC_SHA256_KEY		{ 0x97, 0x79, 0xd9, 0x12, 0x06, 0x42, 0x79, 0x7f, 0x17, 0x47, 0x02, 0x5d, 0x5b, 0x22, 0xb7, 0xac, \
-					  0x60, 0x7c, 0xab, 0x08, 0xe1, 0x75, 0x8f, 0x2f, 0x3a, 0x46, 0xc8, 0xbe, 0x1e, 0x25, 0xc5, 0x3b, \
-					  0x8c, 0x6a, 0x8f, 0x58, 0xff, 0xef, 0xa1, 0x76 }
-#define NIST_HMAC_SHA256_MSG		{ 0xb1, 0x68, 0x9c, 0x25, 0x91, 0xea, 0xf3, 0xc9, 0xe6, 0x60, 0x70, 0xf8, 0xa7, 0x79, 0x54, 0xff, \
-					  0xb8, 0x17, 0x49, 0xf1, 0xb0, 0x03, 0x46, 0xf9, 0xdf, 0xe0, 0xb2, 0xee, 0x90, 0x5d, 0xcc, 0x28, \
-					  0x8b, 0xaf, 0x4a, 0x92, 0xde, 0x3f, 0x40, 0x01, 0xdd, 0x9f, 0x44, 0xc4, 0x68, 0xc3, 0xd0, 0x7d, \
-					  0x6c, 0x6e, 0xe8, 0x2f, 0xac, 0xea, 0xfc, 0x97, 0xc2, 0xfc, 0x0f, 0xc0, 0x60, 0x17, 0x19, 0xd2, \
-					  0xdc, 0xd0, 0xaa, 0x2a, 0xec, 0x92, 0xd1, 0xb0, 0xae, 0x93, 0x3c, 0x65, 0xeb, 0x06, 0xa0, 0x3c, \
-					  0x9c, 0x93, 0x5c, 0x2b, 0xad, 0x04, 0x59, 0x81, 0x02, 0x41, 0x34, 0x7a, 0xb8, 0x7e, 0x9f, 0x11, \
-					  0xad, 0xb3, 0x04, 0x15, 0x42, 0x4c, 0x6c, 0x7f, 0x5f, 0x22, 0xa0, 0x03, 0xb8, 0xab, 0x8d, 0xe5, \
-					  0x4f, 0x6d, 0xed, 0x0e, 0x3a, 0xb9, 0x24, 0x5f, 0xa7, 0x95, 0x68, 0x45, 0x1d, 0xfa, 0x25, 0x8e }
-#define NIST_HMAC_SHA256_MD             { 0x76, 0x9f, 0x00, 0xd3, 0xe6, 0xa6, 0xcc, 0x1f, 0xb4, 0x26, 0xa1, 0x4a, 0x4f, 0x76, 0xc6, 0x46, \
-					  0x2e, 0x61, 0x49, 0x72, 0x6e, 0x0d, 0xee, 0x0e, 0xc0, 0xcf, 0x97, 0xa1, 0x66, 0x05, 0xac, 0x8b }
-
-#define NIST_HMAC_SHA512_KEY_SIZE       100
-#define NIST_HMAC_SHA512_KEY		{ 0x57, 0xc2, 0xeb, 0x67, 0x7b, 0x50, 0x93, 0xb9, 0xe8, 0x29, 0xea, 0x4b, 0xab, 0xb5, 0x0b, 0xde, \
-					  0x55, 0xd0, 0xad, 0x59, 0xfe, 0xc3, 0x4a, 0x61, 0x89, 0x73, 0x80, 0x2b, 0x2a, 0xd9, 0xb7, 0x8e, \
-					  0x26, 0xb2, 0x04, 0x5d, 0xda, 0x78, 0x4d, 0xf3, 0xff, 0x90, 0xae, 0x0f, 0x2c, 0xc5, 0x1c, 0xe3, \
-					  0x9c, 0xf5, 0x48, 0x67, 0x32, 0x0a, 0xc6, 0xf3, 0xba, 0x2c, 0x6f, 0x0d, 0x72, 0x36, 0x04, 0x80, \
-					  0xc9, 0x66, 0x14, 0xae, 0x66, 0x58, 0x1f, 0x26, 0x6c, 0x35, 0xfb, 0x79, 0xfd, 0x28, 0x77, 0x4a, \
-					  0xfd, 0x11, 0x3f, 0xa5, 0x18, 0x7e, 0xff, 0x92, 0x06, 0xd7, 0xcb, 0xe9, 0x0d, 0xd8, 0xbf, 0x67, \
-					  0xc8, 0x44, 0xe2, 0x02 }
-#define NIST_HMAC_SHA512_MSG		{ 0x24, 0x23, 0xdf, 0xf4, 0x8b, 0x31, 0x2b, 0xe8, 0x64, 0xcb, 0x34, 0x90, 0x64, 0x1f, 0x79, 0x3d, \
-					  0x2b, 0x9f, 0xb6, 0x8a, 0x77, 0x63, 0xb8, 0xe2, 0x98, 0xc8, 0x6f, 0x42, 0x24, 0x5e, 0x45, 0x40, \
-					  0xeb, 0x01, 0xae, 0x4d, 0x2d, 0x45, 0x00, 0x37, 0x0b, 0x18, 0x86, 0xf2, 0x3c, 0xa2, 0xcf, 0x97, \
-					  0x01, 0x70, 0x4c, 0xad, 0x5b, 0xd2, 0x1b, 0xa8, 0x7b, 0x81, 0x1d, 0xaf, 0x7a, 0x85, 0x4e, 0xa2, \
-					  0x4a, 0x56, 0x56, 0x5c, 0xed, 0x42, 0x5b, 0x35, 0xe4, 0x0e, 0x1a, 0xcb, 0xeb, 0xe0, 0x36, 0x03, \
-					  0xe3, 0x5d, 0xcf, 0x4a, 0x10, 0x0e, 0x57, 0x21, 0x84, 0x08, 0xa1, 0xd8, 0xdb, 0xcc, 0x3b, 0x99, \
-					  0x29, 0x6c, 0xfe, 0xa9, 0x31, 0xef, 0xe3, 0xeb, 0xd8, 0xf7, 0x19, 0xa6, 0xd9, 0xa1, 0x54, 0x87, \
-					  0xb9, 0xad, 0x67, 0xea, 0xfe, 0xdf, 0x15, 0x55, 0x9c, 0xa4, 0x24, 0x45, 0xb0, 0xf9, 0xb4, 0x2e }
-#define NIST_HMAC_SHA512_MD             { 0x33, 0xc5, 0x11, 0xe9, 0xbc, 0x23, 0x07, 0xc6, 0x27, 0x58, 0xdf, 0x61, 0x12, 0x5a, 0x98, 0x0e, \
-					  0xe6, 0x4c, 0xef, 0xeb, 0xd9, 0x09, 0x31, 0xcb, 0x91, 0xc1, 0x37, 0x42, 0xd4, 0x71, 0x4c, 0x06, \
-					  0xde, 0x40, 0x03, 0xfa, 0xf3, 0xc4, 0x1c, 0x06, 0xae, 0xfc, 0x63, 0x8a, 0xd4, 0x7b, 0x21, 0x90, \
-					  0x6e, 0x6b, 0x10, 0x48, 0x16, 0xb7, 0x2d, 0xe6, 0x26, 0x9e, 0x04, 0x5a, 0x1f, 0x44, 0x29, 0xd4 }
-
diff --git a/drivers/staging/ccree/ssi_fips_ext.c b/drivers/staging/ccree/ssi_fips_ext.c
deleted file mode 100644
index e7bf1843f60c..000000000000
--- a/drivers/staging/ccree/ssi_fips_ext.c
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Copyright (C) 2012-2017 ARM Limited or its affiliates.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-/**************************************************************
- * This file defines the driver FIPS functions that should be
- * implemented by the driver user. Current implementation is sample code only.
- ***************************************************************/
-
-#include <linux/module.h>
-#include "ssi_fips_local.h"
-#include "ssi_driver.h"
-
-static bool tee_error;
-module_param(tee_error, bool, 0644);
-MODULE_PARM_DESC(tee_error, "Simulate TEE library failure flag: 0 - no error (default), 1 - TEE error occured ");
-
-static enum cc_fips_state_t fips_state = CC_FIPS_STATE_NOT_SUPPORTED;
-static enum cc_fips_error fips_error = CC_REE_FIPS_ERROR_OK;
-
-/*
- * This function returns the FIPS REE state.
- * The function should be implemented by the driver user, depends on where
- * the state value is stored.
- * The reference code uses global variable.
- */
-int ssi_fips_ext_get_state(enum cc_fips_state_t *p_state)
-{
-	int rc = 0;
-
-	if (!p_state)
-		return -EINVAL;
-
-	*p_state = fips_state;
-
-	return rc;
-}
-
-/*
- * This function returns the FIPS REE error.
- * The function should be implemented by the driver user, depends on where
- * the error value is stored.
- * The reference code uses global variable.
- */
-int ssi_fips_ext_get_error(enum cc_fips_error *p_err)
-{
-	int rc = 0;
-
-	if (!p_err)
-		return -EINVAL;
-
-	*p_err = fips_error;
-
-	return rc;
-}
-
-/*
- * This function sets the FIPS REE state.
- * The function should be implemented by the driver user, depends on where
- * the state value is stored.
- * The reference code uses global variable.
- */
-int ssi_fips_ext_set_state(enum cc_fips_state_t state)
-{
-	fips_state = state;
-	return 0;
-}
-
-/*
- * This function sets the FIPS REE error.
- * The function should be implemented by the driver user, depends on where
- * the error value is stored.
- * The reference code uses global variable.
- */
-int ssi_fips_ext_set_error(enum cc_fips_error err)
-{
-	fips_error = err;
-	return 0;
-}
-
diff --git a/drivers/staging/ccree/ssi_fips_ll.c b/drivers/staging/ccree/ssi_fips_ll.c
deleted file mode 100644
index 3557e20c9e36..000000000000
--- a/drivers/staging/ccree/ssi_fips_ll.c
+++ /dev/null
@@ -1,1649 +0,0 @@
-/*
- * Copyright (C) 2012-2017 ARM Limited or its affiliates.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-/**************************************************************
- * This file defines the driver FIPS Low Level implmentaion functions,
- * that executes the KAT.
- ***************************************************************/
-#include <linux/kernel.h>
-
-#include "ssi_driver.h"
-#include "ssi_fips_local.h"
-#include "ssi_fips_data.h"
-#include "cc_crypto_ctx.h"
-#include "ssi_hash.h"
-#include "ssi_request_mgr.h"
-
-static const u32 digest_len_init[] = {
-	0x00000040, 0x00000000, 0x00000000, 0x00000000 };
-static const u32 sha1_init[] = {
-	SHA1_H4, SHA1_H3, SHA1_H2, SHA1_H1, SHA1_H0 };
-static const u32 sha256_init[] = {
-	SHA256_H7, SHA256_H6, SHA256_H5, SHA256_H4,
-	SHA256_H3, SHA256_H2, SHA256_H1, SHA256_H0 };
-#if (CC_SUPPORT_SHA > 256)
-static const u32 digest_len_sha512_init[] = {
-	0x00000080, 0x00000000, 0x00000000, 0x00000000 };
-static const u64 sha512_init[] = {
-	SHA512_H7, SHA512_H6, SHA512_H5, SHA512_H4,
-	SHA512_H3, SHA512_H2, SHA512_H1, SHA512_H0 };
-#endif
-
-#define NIST_CIPHER_AES_MAX_VECTOR_SIZE      32
-
-struct fips_cipher_ctx {
-	u8 iv[CC_AES_IV_SIZE];
-	u8 key[AES_512_BIT_KEY_SIZE];
-	u8 din[NIST_CIPHER_AES_MAX_VECTOR_SIZE];
-	u8 dout[NIST_CIPHER_AES_MAX_VECTOR_SIZE];
-};
-
-typedef struct _FipsCipherData {
-	u8                   isAes;
-	u8                   key[AES_512_BIT_KEY_SIZE];
-	size_t                    keySize;
-	u8                   iv[CC_AES_IV_SIZE];
-	enum drv_crypto_direction direction;
-	enum drv_cipher_mode      oprMode;
-	u8                   dataIn[NIST_CIPHER_AES_MAX_VECTOR_SIZE];
-	u8                   dataOut[NIST_CIPHER_AES_MAX_VECTOR_SIZE];
-	size_t                    dataInSize;
-} FipsCipherData;
-
-struct fips_cmac_ctx {
-	u8 key[AES_256_BIT_KEY_SIZE];
-	u8 din[NIST_CIPHER_AES_MAX_VECTOR_SIZE];
-	u8 mac_res[CC_DIGEST_SIZE_MAX];
-};
-
-typedef struct _FipsCmacData {
-	enum drv_crypto_direction direction;
-	u8                   key[AES_256_BIT_KEY_SIZE];
-	size_t                    key_size;
-	u8                   data_in[NIST_CIPHER_AES_MAX_VECTOR_SIZE];
-	size_t                    data_in_size;
-	u8                   mac_res[CC_DIGEST_SIZE_MAX];
-	size_t                    mac_res_size;
-} FipsCmacData;
-
-struct fips_hash_ctx {
-	u8 initial_digest[CC_DIGEST_SIZE_MAX];
-	u8 din[NIST_SHA_MSG_SIZE];
-	u8 mac_res[CC_DIGEST_SIZE_MAX];
-};
-
-typedef struct _FipsHashData {
-	enum drv_hash_mode    hash_mode;
-	u8               data_in[NIST_SHA_MSG_SIZE];
-	size_t                data_in_size;
-	u8               mac_res[CC_DIGEST_SIZE_MAX];
-} FipsHashData;
-
-/* note that the hmac key length must be equal or less than block size (block size is 64 up to sha256 and 128 for sha384/512) */
-struct fips_hmac_ctx {
-	u8 initial_digest[CC_DIGEST_SIZE_MAX];
-	u8 key[CC_HMAC_BLOCK_SIZE_MAX];
-	u8 k0[CC_HMAC_BLOCK_SIZE_MAX];
-	u8 digest_bytes_len[HASH_LEN_SIZE];
-	u8 tmp_digest[CC_DIGEST_SIZE_MAX];
-	u8 din[NIST_HMAC_MSG_SIZE];
-	u8 mac_res[CC_DIGEST_SIZE_MAX];
-};
-
-typedef struct _FipsHmacData {
-	enum drv_hash_mode    hash_mode;
-	u8               key[CC_HMAC_BLOCK_SIZE_MAX];
-	size_t                key_size;
-	u8               data_in[NIST_HMAC_MSG_SIZE];
-	size_t                data_in_size;
-	u8               mac_res[CC_DIGEST_SIZE_MAX];
-} FipsHmacData;
-
-#define FIPS_CCM_B0_A0_ADATA_SIZE   (NIST_AESCCM_IV_SIZE + NIST_AESCCM_IV_SIZE + NIST_AESCCM_ADATA_SIZE)
-
-struct fips_ccm_ctx {
-	u8 b0_a0_adata[FIPS_CCM_B0_A0_ADATA_SIZE];
-	u8 iv[NIST_AESCCM_IV_SIZE];
-	u8 ctr_cnt_0[NIST_AESCCM_IV_SIZE];
-	u8 key[CC_AES_KEY_SIZE_MAX];
-	u8 din[NIST_AESCCM_TEXT_SIZE];
-	u8 dout[NIST_AESCCM_TEXT_SIZE];
-	u8 mac_res[NIST_AESCCM_TAG_SIZE];
-};
-
-typedef struct _FipsCcmData {
-	enum drv_crypto_direction direction;
-	u8                   key[CC_AES_KEY_SIZE_MAX];
-	size_t                    keySize;
-	u8                   nonce[NIST_AESCCM_NONCE_SIZE];
-	u8                   adata[NIST_AESCCM_ADATA_SIZE];
-	size_t                    adataSize;
-	u8                   dataIn[NIST_AESCCM_TEXT_SIZE];
-	size_t                    dataInSize;
-	u8                   dataOut[NIST_AESCCM_TEXT_SIZE];
-	u8                   tagSize;
-	u8                   macResOut[NIST_AESCCM_TAG_SIZE];
-} FipsCcmData;
-
-struct fips_gcm_ctx {
-	u8 adata[NIST_AESGCM_ADATA_SIZE];
-	u8 key[CC_AES_KEY_SIZE_MAX];
-	u8 hkey[CC_AES_KEY_SIZE_MAX];
-	u8 din[NIST_AESGCM_TEXT_SIZE];
-	u8 dout[NIST_AESGCM_TEXT_SIZE];
-	u8 mac_res[NIST_AESGCM_TAG_SIZE];
-	u8 len_block[AES_BLOCK_SIZE];
-	u8 iv_inc1[AES_BLOCK_SIZE];
-	u8 iv_inc2[AES_BLOCK_SIZE];
-};
-
-typedef struct _FipsGcmData {
-	enum drv_crypto_direction direction;
-	u8                   key[CC_AES_KEY_SIZE_MAX];
-	size_t                    keySize;
-	u8                   iv[NIST_AESGCM_IV_SIZE];
-	u8                   adata[NIST_AESGCM_ADATA_SIZE];
-	size_t                    adataSize;
-	u8                   dataIn[NIST_AESGCM_TEXT_SIZE];
-	size_t                    dataInSize;
-	u8                   dataOut[NIST_AESGCM_TEXT_SIZE];
-	u8                   tagSize;
-	u8                   macResOut[NIST_AESGCM_TAG_SIZE];
-} FipsGcmData;
-
-typedef union _fips_ctx {
-	struct fips_cipher_ctx cipher;
-	struct fips_cmac_ctx cmac;
-	struct fips_hash_ctx hash;
-	struct fips_hmac_ctx hmac;
-	struct fips_ccm_ctx ccm;
-	struct fips_gcm_ctx gcm;
-} fips_ctx;
-
-/* test data tables */
-static const FipsCipherData FipsCipherDataTable[] = {
-	/* AES */
-	{ 1, NIST_AES_128_KEY, CC_AES_128_BIT_KEY_SIZE, NIST_AES_ECB_IV, DRV_CRYPTO_DIRECTION_ENCRYPT, DRV_CIPHER_ECB, NIST_AES_PLAIN_DATA, NIST_AES_128_ECB_CIPHER, NIST_AES_VECTOR_SIZE },
-	{ 1, NIST_AES_128_KEY, CC_AES_128_BIT_KEY_SIZE, NIST_AES_ECB_IV, DRV_CRYPTO_DIRECTION_DECRYPT, DRV_CIPHER_ECB, NIST_AES_128_ECB_CIPHER, NIST_AES_PLAIN_DATA, NIST_AES_VECTOR_SIZE },
-	{ 1, NIST_AES_192_KEY, CC_AES_192_BIT_KEY_SIZE, NIST_AES_ECB_IV, DRV_CRYPTO_DIRECTION_ENCRYPT, DRV_CIPHER_ECB, NIST_AES_PLAIN_DATA, NIST_AES_192_ECB_CIPHER, NIST_AES_VECTOR_SIZE },
-	{ 1, NIST_AES_192_KEY, CC_AES_192_BIT_KEY_SIZE, NIST_AES_ECB_IV, DRV_CRYPTO_DIRECTION_DECRYPT, DRV_CIPHER_ECB, NIST_AES_192_ECB_CIPHER, NIST_AES_PLAIN_DATA, NIST_AES_VECTOR_SIZE },
-	{ 1, NIST_AES_256_KEY, CC_AES_256_BIT_KEY_SIZE, NIST_AES_ECB_IV, DRV_CRYPTO_DIRECTION_ENCRYPT, DRV_CIPHER_ECB, NIST_AES_PLAIN_DATA, NIST_AES_256_ECB_CIPHER, NIST_AES_VECTOR_SIZE },
-	{ 1, NIST_AES_256_KEY, CC_AES_256_BIT_KEY_SIZE, NIST_AES_ECB_IV, DRV_CRYPTO_DIRECTION_DECRYPT, DRV_CIPHER_ECB, NIST_AES_256_ECB_CIPHER, NIST_AES_PLAIN_DATA, NIST_AES_VECTOR_SIZE },
-	{ 1, NIST_AES_128_KEY, CC_AES_128_BIT_KEY_SIZE, NIST_AES_CBC_IV, DRV_CRYPTO_DIRECTION_ENCRYPT, DRV_CIPHER_CBC, NIST_AES_PLAIN_DATA, NIST_AES_128_CBC_CIPHER, NIST_AES_VECTOR_SIZE },
-	{ 1, NIST_AES_128_KEY, CC_AES_128_BIT_KEY_SIZE, NIST_AES_CBC_IV, DRV_CRYPTO_DIRECTION_DECRYPT, DRV_CIPHER_CBC, NIST_AES_128_CBC_CIPHER, NIST_AES_PLAIN_DATA, NIST_AES_VECTOR_SIZE },
-	{ 1, NIST_AES_192_KEY, CC_AES_192_BIT_KEY_SIZE, NIST_AES_CBC_IV, DRV_CRYPTO_DIRECTION_ENCRYPT, DRV_CIPHER_CBC, NIST_AES_PLAIN_DATA, NIST_AES_192_CBC_CIPHER, NIST_AES_VECTOR_SIZE },
-	{ 1, NIST_AES_192_KEY, CC_AES_192_BIT_KEY_SIZE, NIST_AES_CBC_IV, DRV_CRYPTO_DIRECTION_DECRYPT, DRV_CIPHER_CBC, NIST_AES_192_CBC_CIPHER, NIST_AES_PLAIN_DATA, NIST_AES_VECTOR_SIZE },
-	{ 1, NIST_AES_256_KEY, CC_AES_256_BIT_KEY_SIZE, NIST_AES_CBC_IV, DRV_CRYPTO_DIRECTION_ENCRYPT, DRV_CIPHER_CBC, NIST_AES_PLAIN_DATA, NIST_AES_256_CBC_CIPHER, NIST_AES_VECTOR_SIZE },
-	{ 1, NIST_AES_256_KEY, CC_AES_256_BIT_KEY_SIZE, NIST_AES_CBC_IV, DRV_CRYPTO_DIRECTION_DECRYPT, DRV_CIPHER_CBC, NIST_AES_256_CBC_CIPHER, NIST_AES_PLAIN_DATA, NIST_AES_VECTOR_SIZE },
-	{ 1, NIST_AES_128_KEY, CC_AES_128_BIT_KEY_SIZE, NIST_AES_OFB_IV, DRV_CRYPTO_DIRECTION_ENCRYPT, DRV_CIPHER_OFB, NIST_AES_PLAIN_DATA, NIST_AES_128_OFB_CIPHER, NIST_AES_VECTOR_SIZE },
-	{ 1, NIST_AES_128_KEY, CC_AES_128_BIT_KEY_SIZE, NIST_AES_OFB_IV, DRV_CRYPTO_DIRECTION_ENCRYPT, DRV_CIPHER_OFB, NIST_AES_128_OFB_CIPHER, NIST_AES_PLAIN_DATA, NIST_AES_VECTOR_SIZE },
-	{ 1, NIST_AES_192_KEY, CC_AES_192_BIT_KEY_SIZE, NIST_AES_OFB_IV, DRV_CRYPTO_DIRECTION_ENCRYPT, DRV_CIPHER_OFB, NIST_AES_PLAIN_DATA, NIST_AES_192_OFB_CIPHER, NIST_AES_VECTOR_SIZE },
-	{ 1, NIST_AES_192_KEY, CC_AES_192_BIT_KEY_SIZE, NIST_AES_OFB_IV, DRV_CRYPTO_DIRECTION_DECRYPT, DRV_CIPHER_OFB, NIST_AES_192_OFB_CIPHER, NIST_AES_PLAIN_DATA, NIST_AES_VECTOR_SIZE },
-	{ 1, NIST_AES_256_KEY, CC_AES_256_BIT_KEY_SIZE, NIST_AES_OFB_IV, DRV_CRYPTO_DIRECTION_ENCRYPT, DRV_CIPHER_OFB, NIST_AES_PLAIN_DATA, NIST_AES_256_OFB_CIPHER, NIST_AES_VECTOR_SIZE },
-	{ 1, NIST_AES_256_KEY, CC_AES_256_BIT_KEY_SIZE, NIST_AES_OFB_IV, DRV_CRYPTO_DIRECTION_DECRYPT, DRV_CIPHER_OFB, NIST_AES_256_OFB_CIPHER, NIST_AES_PLAIN_DATA, NIST_AES_VECTOR_SIZE },
-	{ 1, NIST_AES_128_KEY, CC_AES_128_BIT_KEY_SIZE, NIST_AES_CTR_IV, DRV_CRYPTO_DIRECTION_ENCRYPT, DRV_CIPHER_CTR, NIST_AES_PLAIN_DATA, NIST_AES_128_CTR_CIPHER, NIST_AES_VECTOR_SIZE },
-	{ 1, NIST_AES_128_KEY, CC_AES_128_BIT_KEY_SIZE, NIST_AES_CTR_IV, DRV_CRYPTO_DIRECTION_DECRYPT, DRV_CIPHER_CTR, NIST_AES_128_CTR_CIPHER, NIST_AES_PLAIN_DATA, NIST_AES_VECTOR_SIZE },
-	{ 1, NIST_AES_192_KEY, CC_AES_192_BIT_KEY_SIZE, NIST_AES_CTR_IV, DRV_CRYPTO_DIRECTION_ENCRYPT, DRV_CIPHER_CTR, NIST_AES_PLAIN_DATA, NIST_AES_192_CTR_CIPHER, NIST_AES_VECTOR_SIZE },
-	{ 1, NIST_AES_192_KEY, CC_AES_192_BIT_KEY_SIZE, NIST_AES_CTR_IV, DRV_CRYPTO_DIRECTION_DECRYPT, DRV_CIPHER_CTR, NIST_AES_192_CTR_CIPHER, NIST_AES_PLAIN_DATA, NIST_AES_VECTOR_SIZE },
-	{ 1, NIST_AES_256_KEY, CC_AES_256_BIT_KEY_SIZE, NIST_AES_CTR_IV, DRV_CRYPTO_DIRECTION_ENCRYPT, DRV_CIPHER_CTR, NIST_AES_PLAIN_DATA, NIST_AES_256_CTR_CIPHER, NIST_AES_VECTOR_SIZE },
-	{ 1, NIST_AES_256_KEY, CC_AES_256_BIT_KEY_SIZE, NIST_AES_CTR_IV, DRV_CRYPTO_DIRECTION_DECRYPT, DRV_CIPHER_CTR, NIST_AES_256_CTR_CIPHER, NIST_AES_PLAIN_DATA, NIST_AES_VECTOR_SIZE },
-	{ 1, RFC3962_AES_128_KEY,  CC_AES_128_BIT_KEY_SIZE, RFC3962_AES_CBC_CTS_IV, DRV_CRYPTO_DIRECTION_ENCRYPT, DRV_CIPHER_CBC_CTS, RFC3962_AES_PLAIN_DATA, RFC3962_AES_128_CBC_CTS_CIPHER, RFC3962_AES_VECTOR_SIZE },
-	{ 1, RFC3962_AES_128_KEY,  CC_AES_128_BIT_KEY_SIZE, RFC3962_AES_CBC_CTS_IV, DRV_CRYPTO_DIRECTION_DECRYPT, DRV_CIPHER_CBC_CTS, RFC3962_AES_128_CBC_CTS_CIPHER, RFC3962_AES_PLAIN_DATA, RFC3962_AES_VECTOR_SIZE },
-	{ 1, NIST_AES_256_XTS_KEY, CC_AES_256_BIT_KEY_SIZE,   NIST_AES_256_XTS_IV,  DRV_CRYPTO_DIRECTION_ENCRYPT, DRV_CIPHER_XTS,     NIST_AES_256_XTS_PLAIN, NIST_AES_256_XTS_CIPHER, NIST_AES_256_XTS_VECTOR_SIZE },
-	{ 1, NIST_AES_256_XTS_KEY, CC_AES_256_BIT_KEY_SIZE,   NIST_AES_256_XTS_IV,  DRV_CRYPTO_DIRECTION_DECRYPT, DRV_CIPHER_XTS,     NIST_AES_256_XTS_CIPHER, NIST_AES_256_XTS_PLAIN, NIST_AES_256_XTS_VECTOR_SIZE },
-#if (CC_SUPPORT_SHA > 256)
-	{ 1, NIST_AES_512_XTS_KEY, 2 * CC_AES_256_BIT_KEY_SIZE, NIST_AES_512_XTS_IV,  DRV_CRYPTO_DIRECTION_ENCRYPT, DRV_CIPHER_XTS,     NIST_AES_512_XTS_PLAIN, NIST_AES_512_XTS_CIPHER, NIST_AES_512_XTS_VECTOR_SIZE },
-	{ 1, NIST_AES_512_XTS_KEY, 2 * CC_AES_256_BIT_KEY_SIZE, NIST_AES_512_XTS_IV,  DRV_CRYPTO_DIRECTION_DECRYPT, DRV_CIPHER_XTS,     NIST_AES_512_XTS_CIPHER, NIST_AES_512_XTS_PLAIN, NIST_AES_512_XTS_VECTOR_SIZE },
-#endif
-	/* DES */
-	{ 0, NIST_TDES_ECB3_KEY, CC_DRV_DES_TRIPLE_KEY_SIZE, NIST_TDES_ECB_IV, DRV_CRYPTO_DIRECTION_ENCRYPT, DRV_CIPHER_ECB, NIST_TDES_ECB3_PLAIN_DATA, NIST_TDES_ECB3_CIPHER, NIST_TDES_VECTOR_SIZE },
-	{ 0, NIST_TDES_ECB3_KEY, CC_DRV_DES_TRIPLE_KEY_SIZE, NIST_TDES_ECB_IV, DRV_CRYPTO_DIRECTION_DECRYPT, DRV_CIPHER_ECB, NIST_TDES_ECB3_CIPHER, NIST_TDES_ECB3_PLAIN_DATA, NIST_TDES_VECTOR_SIZE },
-	{ 0, NIST_TDES_CBC3_KEY, CC_DRV_DES_TRIPLE_KEY_SIZE, NIST_TDES_CBC3_IV, DRV_CRYPTO_DIRECTION_ENCRYPT, DRV_CIPHER_CBC, NIST_TDES_CBC3_PLAIN_DATA, NIST_TDES_CBC3_CIPHER, NIST_TDES_VECTOR_SIZE },
-	{ 0, NIST_TDES_CBC3_KEY, CC_DRV_DES_TRIPLE_KEY_SIZE, NIST_TDES_CBC3_IV, DRV_CRYPTO_DIRECTION_DECRYPT, DRV_CIPHER_CBC, NIST_TDES_CBC3_CIPHER, NIST_TDES_CBC3_PLAIN_DATA, NIST_TDES_VECTOR_SIZE },
-};
-
-#define FIPS_CIPHER_NUM_OF_TESTS        (sizeof(FipsCipherDataTable) / sizeof(FipsCipherData))
-
-static const FipsCmacData FipsCmacDataTable[] = {
-	{ DRV_CRYPTO_DIRECTION_ENCRYPT, NIST_AES_128_CMAC_KEY, AES_128_BIT_KEY_SIZE, NIST_AES_128_CMAC_PLAIN_DATA, NIST_AES_128_CMAC_VECTOR_SIZE, NIST_AES_128_CMAC_MAC, NIST_AES_128_CMAC_OUTPUT_SIZE },
-	{ DRV_CRYPTO_DIRECTION_ENCRYPT, NIST_AES_192_CMAC_KEY, AES_192_BIT_KEY_SIZE, NIST_AES_192_CMAC_PLAIN_DATA, NIST_AES_192_CMAC_VECTOR_SIZE, NIST_AES_192_CMAC_MAC, NIST_AES_192_CMAC_OUTPUT_SIZE },
-	{ DRV_CRYPTO_DIRECTION_ENCRYPT, NIST_AES_256_CMAC_KEY, AES_256_BIT_KEY_SIZE, NIST_AES_256_CMAC_PLAIN_DATA, NIST_AES_256_CMAC_VECTOR_SIZE, NIST_AES_256_CMAC_MAC, NIST_AES_256_CMAC_OUTPUT_SIZE },
-};
-
-#define FIPS_CMAC_NUM_OF_TESTS        (sizeof(FipsCmacDataTable) / sizeof(FipsCmacData))
-
-static const FipsHashData FipsHashDataTable[] = {
-	{ DRV_HASH_SHA1,   NIST_SHA_1_MSG,   NIST_SHA_MSG_SIZE, NIST_SHA_1_MD },
-	{ DRV_HASH_SHA256, NIST_SHA_256_MSG, NIST_SHA_MSG_SIZE, NIST_SHA_256_MD },
-#if (CC_SUPPORT_SHA > 256)
-//        { DRV_HASH_SHA512, NIST_SHA_512_MSG, NIST_SHA_MSG_SIZE, NIST_SHA_512_MD },
-#endif
-};
-
-#define FIPS_HASH_NUM_OF_TESTS        (sizeof(FipsHashDataTable) / sizeof(FipsHashData))
-
-static const FipsHmacData FipsHmacDataTable[] = {
-	{ DRV_HASH_SHA1,   NIST_HMAC_SHA1_KEY,   NIST_HMAC_SHA1_KEY_SIZE,   NIST_HMAC_SHA1_MSG,   NIST_HMAC_MSG_SIZE, NIST_HMAC_SHA1_MD },
-	{ DRV_HASH_SHA256, NIST_HMAC_SHA256_KEY, NIST_HMAC_SHA256_KEY_SIZE, NIST_HMAC_SHA256_MSG, NIST_HMAC_MSG_SIZE, NIST_HMAC_SHA256_MD },
-#if (CC_SUPPORT_SHA > 256)
-//        { DRV_HASH_SHA512, NIST_HMAC_SHA512_KEY, NIST_HMAC_SHA512_KEY_SIZE, NIST_HMAC_SHA512_MSG, NIST_HMAC_MSG_SIZE, NIST_HMAC_SHA512_MD },
-#endif
-};
-
-#define FIPS_HMAC_NUM_OF_TESTS        (sizeof(FipsHmacDataTable) / sizeof(FipsHmacData))
-
-static const FipsCcmData FipsCcmDataTable[] = {
-	{ DRV_CRYPTO_DIRECTION_ENCRYPT, NIST_AESCCM_128_KEY, NIST_AESCCM_128_BIT_KEY_SIZE, NIST_AESCCM_128_NONCE, NIST_AESCCM_128_ADATA, NIST_AESCCM_ADATA_SIZE, NIST_AESCCM_128_PLAIN_TEXT, NIST_AESCCM_TEXT_SIZE, NIST_AESCCM_128_CIPHER, NIST_AESCCM_TAG_SIZE, NIST_AESCCM_128_MAC },
-	{ DRV_CRYPTO_DIRECTION_DECRYPT, NIST_AESCCM_128_KEY, NIST_AESCCM_128_BIT_KEY_SIZE, NIST_AESCCM_128_NONCE, NIST_AESCCM_128_ADATA, NIST_AESCCM_ADATA_SIZE, NIST_AESCCM_128_CIPHER, NIST_AESCCM_TEXT_SIZE, NIST_AESCCM_128_PLAIN_TEXT, NIST_AESCCM_TAG_SIZE, NIST_AESCCM_128_MAC },
-	{ DRV_CRYPTO_DIRECTION_ENCRYPT, NIST_AESCCM_192_KEY, NIST_AESCCM_192_BIT_KEY_SIZE, NIST_AESCCM_192_NONCE, NIST_AESCCM_192_ADATA, NIST_AESCCM_ADATA_SIZE, NIST_AESCCM_192_PLAIN_TEXT, NIST_AESCCM_TEXT_SIZE, NIST_AESCCM_192_CIPHER, NIST_AESCCM_TAG_SIZE, NIST_AESCCM_192_MAC },
-	{ DRV_CRYPTO_DIRECTION_DECRYPT, NIST_AESCCM_192_KEY, NIST_AESCCM_192_BIT_KEY_SIZE, NIST_AESCCM_192_NONCE, NIST_AESCCM_192_ADATA, NIST_AESCCM_ADATA_SIZE, NIST_AESCCM_192_CIPHER, NIST_AESCCM_TEXT_SIZE, NIST_AESCCM_192_PLAIN_TEXT, NIST_AESCCM_TAG_SIZE, NIST_AESCCM_192_MAC },
-	{ DRV_CRYPTO_DIRECTION_ENCRYPT, NIST_AESCCM_256_KEY, NIST_AESCCM_256_BIT_KEY_SIZE, NIST_AESCCM_256_NONCE, NIST_AESCCM_256_ADATA, NIST_AESCCM_ADATA_SIZE, NIST_AESCCM_256_PLAIN_TEXT, NIST_AESCCM_TEXT_SIZE, NIST_AESCCM_256_CIPHER, NIST_AESCCM_TAG_SIZE, NIST_AESCCM_256_MAC },
-	{ DRV_CRYPTO_DIRECTION_DECRYPT, NIST_AESCCM_256_KEY, NIST_AESCCM_256_BIT_KEY_SIZE, NIST_AESCCM_256_NONCE, NIST_AESCCM_256_ADATA, NIST_AESCCM_ADATA_SIZE, NIST_AESCCM_256_CIPHER, NIST_AESCCM_TEXT_SIZE, NIST_AESCCM_256_PLAIN_TEXT, NIST_AESCCM_TAG_SIZE, NIST_AESCCM_256_MAC },
-};
-
-#define FIPS_CCM_NUM_OF_TESTS        (sizeof(FipsCcmDataTable) / sizeof(FipsCcmData))
-
-static const FipsGcmData FipsGcmDataTable[] = {
-	{ DRV_CRYPTO_DIRECTION_ENCRYPT, NIST_AESGCM_128_KEY, NIST_AESGCM_128_BIT_KEY_SIZE, NIST_AESGCM_128_IV, NIST_AESGCM_128_ADATA, NIST_AESGCM_ADATA_SIZE, NIST_AESGCM_128_PLAIN_TEXT, NIST_AESGCM_TEXT_SIZE, NIST_AESGCM_128_CIPHER, NIST_AESGCM_TAG_SIZE, NIST_AESGCM_128_MAC },
-	{ DRV_CRYPTO_DIRECTION_DECRYPT, NIST_AESGCM_128_KEY, NIST_AESGCM_128_BIT_KEY_SIZE, NIST_AESGCM_128_IV, NIST_AESGCM_128_ADATA, NIST_AESGCM_ADATA_SIZE, NIST_AESGCM_128_CIPHER, NIST_AESGCM_TEXT_SIZE, NIST_AESGCM_128_PLAIN_TEXT, NIST_AESGCM_TAG_SIZE, NIST_AESGCM_128_MAC },
-	{ DRV_CRYPTO_DIRECTION_ENCRYPT, NIST_AESGCM_192_KEY, NIST_AESGCM_192_BIT_KEY_SIZE, NIST_AESGCM_192_IV, NIST_AESGCM_192_ADATA, NIST_AESGCM_ADATA_SIZE, NIST_AESGCM_192_PLAIN_TEXT, NIST_AESGCM_TEXT_SIZE, NIST_AESGCM_192_CIPHER, NIST_AESGCM_TAG_SIZE, NIST_AESGCM_192_MAC },
-	{ DRV_CRYPTO_DIRECTION_DECRYPT, NIST_AESGCM_192_KEY, NIST_AESGCM_192_BIT_KEY_SIZE, NIST_AESGCM_192_IV, NIST_AESGCM_192_ADATA, NIST_AESGCM_ADATA_SIZE, NIST_AESGCM_192_CIPHER, NIST_AESGCM_TEXT_SIZE, NIST_AESGCM_192_PLAIN_TEXT, NIST_AESGCM_TAG_SIZE, NIST_AESGCM_192_MAC },
-	{ DRV_CRYPTO_DIRECTION_ENCRYPT, NIST_AESGCM_256_KEY, NIST_AESGCM_256_BIT_KEY_SIZE, NIST_AESGCM_256_IV, NIST_AESGCM_256_ADATA, NIST_AESGCM_ADATA_SIZE, NIST_AESGCM_256_PLAIN_TEXT, NIST_AESGCM_TEXT_SIZE, NIST_AESGCM_256_CIPHER, NIST_AESGCM_TAG_SIZE, NIST_AESGCM_256_MAC },
-	{ DRV_CRYPTO_DIRECTION_DECRYPT, NIST_AESGCM_256_KEY, NIST_AESGCM_256_BIT_KEY_SIZE, NIST_AESGCM_256_IV, NIST_AESGCM_256_ADATA, NIST_AESGCM_ADATA_SIZE, NIST_AESGCM_256_CIPHER, NIST_AESGCM_TEXT_SIZE, NIST_AESGCM_256_PLAIN_TEXT, NIST_AESGCM_TAG_SIZE, NIST_AESGCM_256_MAC },
-};
-
-#define FIPS_GCM_NUM_OF_TESTS        (sizeof(FipsGcmDataTable) / sizeof(FipsGcmData))
-
-static inline enum cc_fips_error
-FIPS_CipherToFipsError(enum drv_cipher_mode mode, bool is_aes)
-{
-	switch (mode)
-	{
-	case DRV_CIPHER_ECB:
-		return is_aes ? CC_REE_FIPS_ERROR_AES_ECB_PUT : CC_REE_FIPS_ERROR_DES_ECB_PUT;
-	case DRV_CIPHER_CBC:
-		return is_aes ? CC_REE_FIPS_ERROR_AES_CBC_PUT : CC_REE_FIPS_ERROR_DES_CBC_PUT;
-	case DRV_CIPHER_OFB:
-		return CC_REE_FIPS_ERROR_AES_OFB_PUT;
-	case DRV_CIPHER_CTR:
-		return CC_REE_FIPS_ERROR_AES_CTR_PUT;
-	case DRV_CIPHER_CBC_CTS:
-		return CC_REE_FIPS_ERROR_AES_CBC_CTS_PUT;
-	case DRV_CIPHER_XTS:
-		return CC_REE_FIPS_ERROR_AES_XTS_PUT;
-	default:
-		return CC_REE_FIPS_ERROR_GENERAL;
-	}
-
-	return CC_REE_FIPS_ERROR_GENERAL;
-}
-
-static inline int
-ssi_cipher_fips_run_test(struct ssi_drvdata *drvdata,
-			 bool is_aes,
-			 int cipher_mode,
-			 int direction,
-			 dma_addr_t key_dma_addr,
-			 size_t key_len,
-			 dma_addr_t iv_dma_addr,
-			 size_t iv_len,
-			 dma_addr_t din_dma_addr,
-			 dma_addr_t dout_dma_addr,
-			 size_t data_size)
-{
-	/* max number of descriptors used for the flow */
-	#define FIPS_CIPHER_MAX_SEQ_LEN 6
-
-	int rc;
-	struct ssi_crypto_req ssi_req = {0};
-	struct cc_hw_desc desc[FIPS_CIPHER_MAX_SEQ_LEN];
-	int idx = 0;
-	int s_flow_mode = is_aes ? S_DIN_to_AES : S_DIN_to_DES;
-
-	/* create setup descriptors */
-	switch (cipher_mode) {
-	case DRV_CIPHER_CBC:
-	case DRV_CIPHER_CBC_CTS:
-	case DRV_CIPHER_CTR:
-	case DRV_CIPHER_OFB:
-		/* Load cipher state */
-		hw_desc_init(&desc[idx]);
-		set_din_type(&desc[idx], DMA_DLLI,
-			     iv_dma_addr, iv_len, NS_BIT);
-		set_cipher_config0(&desc[idx], direction);
-		set_flow_mode(&desc[idx], s_flow_mode);
-		set_cipher_mode(&desc[idx], cipher_mode);
-		if ((cipher_mode == DRV_CIPHER_CTR) ||
-		    (cipher_mode == DRV_CIPHER_OFB)) {
-			set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
-		} else {
-			set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
-		}
-		idx++;
-		/*FALLTHROUGH*/
-	case DRV_CIPHER_ECB:
-		/* Load key */
-		hw_desc_init(&desc[idx]);
-		set_cipher_mode(&desc[idx], cipher_mode);
-		set_cipher_config0(&desc[idx], direction);
-		if (is_aes) {
-			set_din_type(&desc[idx], DMA_DLLI, key_dma_addr,
-				     ((key_len == 24) ? AES_MAX_KEY_SIZE :
-				      key_len), NS_BIT);
-			set_key_size_aes(&desc[idx], key_len);
-		} else {/*des*/
-			set_din_type(&desc[idx], DMA_DLLI, key_dma_addr,
-				     key_len, NS_BIT);
-			set_key_size_des(&desc[idx], key_len);
-		}
-		set_flow_mode(&desc[idx], s_flow_mode);
-		set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
-		idx++;
-		break;
-	case DRV_CIPHER_XTS:
-		/* Load AES key */
-		hw_desc_init(&desc[idx]);
-		set_cipher_mode(&desc[idx], cipher_mode);
-		set_cipher_config0(&desc[idx], direction);
-		set_din_type(&desc[idx], DMA_DLLI, key_dma_addr, (key_len / 2),
-			     NS_BIT);
-		set_key_size_aes(&desc[idx], (key_len / 2));
-		set_flow_mode(&desc[idx], s_flow_mode);
-		set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
-		idx++;
-
-		/* load XEX key */
-		hw_desc_init(&desc[idx]);
-		set_cipher_mode(&desc[idx], cipher_mode);
-		set_cipher_config0(&desc[idx], direction);
-		set_din_type(&desc[idx], DMA_DLLI,
-			     (key_dma_addr + (key_len / 2)),
-			     (key_len / 2), NS_BIT);
-		set_xex_data_unit_size(&desc[idx], data_size);
-		set_flow_mode(&desc[idx], s_flow_mode);
-		set_key_size_aes(&desc[idx], (key_len / 2));
-		set_setup_mode(&desc[idx], SETUP_LOAD_XEX_KEY);
-		idx++;
-
-		/* Set state */
-		hw_desc_init(&desc[idx]);
-		set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
-		set_cipher_mode(&desc[idx], cipher_mode);
-		set_cipher_config0(&desc[idx], direction);
-		set_key_size_aes(&desc[idx], (key_len / 2));
-		set_flow_mode(&desc[idx], s_flow_mode);
-		set_din_type(&desc[idx], DMA_DLLI, iv_dma_addr,
-			     CC_AES_BLOCK_SIZE, NS_BIT);
-		idx++;
-		break;
-	default:
-		FIPS_LOG("Unsupported cipher mode (%d)\n", cipher_mode);
-		BUG();
-	}
-
-	/* create data descriptor */
-	hw_desc_init(&desc[idx]);
-	set_din_type(&desc[idx], DMA_DLLI, din_dma_addr, data_size, NS_BIT);
-	set_dout_dlli(&desc[idx], dout_dma_addr, data_size, NS_BIT, 0);
-	set_flow_mode(&desc[idx], is_aes ? DIN_AES_DOUT : DIN_DES_DOUT);
-	idx++;
-
-	/* perform the operation - Lock HW and push sequence */
-	BUG_ON(idx > FIPS_CIPHER_MAX_SEQ_LEN);
-	rc = send_request(drvdata, &ssi_req, desc, idx, false);
-
-	// send_request returns error just in some corner cases which should not appear in this flow.
-	return rc;
-}
-
-enum cc_fips_error
-ssi_cipher_fips_power_up_tests(struct ssi_drvdata *drvdata, void *cpu_addr_buffer, dma_addr_t dma_coherent_buffer)
-{
-	enum cc_fips_error error = CC_REE_FIPS_ERROR_OK;
-	size_t i;
-	struct fips_cipher_ctx *virt_ctx = (struct fips_cipher_ctx *)cpu_addr_buffer;
-
-	/* set the phisical pointers for iv, key, din, dout */
-	dma_addr_t iv_dma_addr = dma_coherent_buffer + offsetof(struct fips_cipher_ctx, iv);
-	dma_addr_t key_dma_addr = dma_coherent_buffer + offsetof(struct fips_cipher_ctx, key);
-	dma_addr_t din_dma_addr = dma_coherent_buffer + offsetof(struct fips_cipher_ctx, din);
-	dma_addr_t dout_dma_addr = dma_coherent_buffer + offsetof(struct fips_cipher_ctx, dout);
-
-	for (i = 0; i < FIPS_CIPHER_NUM_OF_TESTS; ++i)
-	{
-		FipsCipherData *cipherData = (FipsCipherData *)&FipsCipherDataTable[i];
-		int rc = 0;
-		size_t iv_size = cipherData->isAes ? NIST_AES_IV_SIZE : NIST_TDES_IV_SIZE;
-
-		memset(cpu_addr_buffer, 0, sizeof(struct fips_cipher_ctx));
-
-		/* copy into the allocated buffer */
-		memcpy(virt_ctx->iv, cipherData->iv, iv_size);
-		memcpy(virt_ctx->key, cipherData->key, cipherData->keySize);
-		memcpy(virt_ctx->din, cipherData->dataIn, cipherData->dataInSize);
-
-		FIPS_DBG("ssi_cipher_fips_run_test -  (i = %d) \n", i);
-		rc = ssi_cipher_fips_run_test(drvdata,
-					      cipherData->isAes,
-					      cipherData->oprMode,
-					      cipherData->direction,
-					      key_dma_addr,
-					      cipherData->keySize,
-					      iv_dma_addr,
-					      iv_size,
-					      din_dma_addr,
-					      dout_dma_addr,
-					      cipherData->dataInSize);
-		if (rc != 0)
-		{
-			FIPS_LOG("ssi_cipher_fips_run_test %d returned error - rc = %d \n", i, rc);
-			error = FIPS_CipherToFipsError(cipherData->oprMode, cipherData->isAes);
-			break;
-		}
-
-		/* compare actual dout to expected */
-		if (memcmp(virt_ctx->dout, cipherData->dataOut, cipherData->dataInSize) != 0)
-		{
-			FIPS_LOG("dout comparison error %d - oprMode=%d, isAes=%d\n", i, cipherData->oprMode, cipherData->isAes);
-			FIPS_LOG("  i  expected   received \n");
-			FIPS_LOG("  i  0x%08x 0x%08x  (size=%d) \n", (size_t)cipherData->dataOut, (size_t)virt_ctx->dout, cipherData->dataInSize);
-			for (i = 0; i < cipherData->dataInSize; ++i)
-			{
-				FIPS_LOG("  %d    0x%02x     0x%02x \n", i, cipherData->dataOut[i], virt_ctx->dout[i]);
-			}
-
-			error = FIPS_CipherToFipsError(cipherData->oprMode, cipherData->isAes);
-			break;
-		}
-	}
-
-	return error;
-}
-
-static inline int
-ssi_cmac_fips_run_test(struct ssi_drvdata *drvdata,
-		       dma_addr_t key_dma_addr,
-		       size_t key_len,
-		       dma_addr_t din_dma_addr,
-		       size_t din_len,
-		       dma_addr_t digest_dma_addr,
-		       size_t digest_len)
-{
-	/* max number of descriptors used for the flow */
-	#define FIPS_CMAC_MAX_SEQ_LEN 4
-
-	int rc;
-	struct ssi_crypto_req ssi_req = {0};
-	struct cc_hw_desc desc[FIPS_CMAC_MAX_SEQ_LEN];
-	int idx = 0;
-
-	/* Setup CMAC Key */
-	hw_desc_init(&desc[idx]);
-	set_din_type(&desc[idx], DMA_DLLI, key_dma_addr,
-		     ((key_len == 24) ? AES_MAX_KEY_SIZE : key_len), NS_BIT);
-	set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
-	set_cipher_mode(&desc[idx], DRV_CIPHER_CMAC);
-	set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
-	set_key_size_aes(&desc[idx], key_len);
-	set_flow_mode(&desc[idx], S_DIN_to_AES);
-	idx++;
-
-	/* Load MAC state */
-	hw_desc_init(&desc[idx]);
-	set_din_type(&desc[idx], DMA_DLLI, digest_dma_addr, CC_AES_BLOCK_SIZE,
-		     NS_BIT);
-	set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
-	set_cipher_mode(&desc[idx], DRV_CIPHER_CMAC);
-	set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
-	set_key_size_aes(&desc[idx], key_len);
-	set_flow_mode(&desc[idx], S_DIN_to_AES);
-	idx++;
-
-	//ssi_hash_create_data_desc(state, ctx, DIN_AES_DOUT, desc, false, &idx);
-	hw_desc_init(&desc[idx]);
-	set_din_type(&desc[idx], DMA_DLLI, din_dma_addr, din_len, NS_BIT);
-	set_flow_mode(&desc[idx], DIN_AES_DOUT);
-	idx++;
-
-	/* Get final MAC result */
-	hw_desc_init(&desc[idx]);
-	set_dout_dlli(&desc[idx], digest_dma_addr, CC_AES_BLOCK_SIZE, NS_BIT,
-		      0);
-	set_flow_mode(&desc[idx], S_AES_to_DOUT);
-	set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
-	set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
-	set_cipher_mode(&desc[idx], DRV_CIPHER_CMAC);
-	idx++;
-
-	/* perform the operation - Lock HW and push sequence */
-	BUG_ON(idx > FIPS_CMAC_MAX_SEQ_LEN);
-	rc = send_request(drvdata, &ssi_req, desc, idx, false);
-
-	// send_request returns error just in some corner cases which should not appear in this flow.
-	return rc;
-}
-
-enum cc_fips_error
-ssi_cmac_fips_power_up_tests(struct ssi_drvdata *drvdata, void *cpu_addr_buffer, dma_addr_t dma_coherent_buffer)
-{
-	enum cc_fips_error error = CC_REE_FIPS_ERROR_OK;
-	size_t i;
-	struct fips_cmac_ctx *virt_ctx = (struct fips_cmac_ctx *)cpu_addr_buffer;
-
-	/* set the phisical pointers for key, din, dout */
-	dma_addr_t key_dma_addr = dma_coherent_buffer + offsetof(struct fips_cmac_ctx, key);
-	dma_addr_t din_dma_addr = dma_coherent_buffer + offsetof(struct fips_cmac_ctx, din);
-	dma_addr_t mac_res_dma_addr = dma_coherent_buffer + offsetof(struct fips_cmac_ctx, mac_res);
-
-	for (i = 0; i < FIPS_CMAC_NUM_OF_TESTS; ++i)
-	{
-		FipsCmacData *cmac_data = (FipsCmacData *)&FipsCmacDataTable[i];
-		int rc = 0;
-
-		memset(cpu_addr_buffer, 0, sizeof(struct fips_cmac_ctx));
-
-		/* copy into the allocated buffer */
-		memcpy(virt_ctx->key, cmac_data->key, cmac_data->key_size);
-		memcpy(virt_ctx->din, cmac_data->data_in, cmac_data->data_in_size);
-
-		BUG_ON(cmac_data->direction != DRV_CRYPTO_DIRECTION_ENCRYPT);
-
-		FIPS_DBG("ssi_cmac_fips_run_test -  (i = %d) \n", i);
-		rc = ssi_cmac_fips_run_test(drvdata,
-					    key_dma_addr,
-					    cmac_data->key_size,
-					    din_dma_addr,
-					    cmac_data->data_in_size,
-					    mac_res_dma_addr,
-					    cmac_data->mac_res_size);
-		if (rc != 0)
-		{
-			FIPS_LOG("ssi_cmac_fips_run_test %d returned error - rc = %d \n", i, rc);
-			error = CC_REE_FIPS_ERROR_AES_CMAC_PUT;
-			break;
-		}
-
-		/* compare actual mac result to expected */
-		if (memcmp(virt_ctx->mac_res, cmac_data->mac_res, cmac_data->mac_res_size) != 0)
-		{
-			FIPS_LOG("comparison error %d - digest_size=%d \n", i, cmac_data->mac_res_size);
-			FIPS_LOG("  i  expected   received \n");
-			FIPS_LOG("  i  0x%08x 0x%08x \n", (size_t)cmac_data->mac_res, (size_t)virt_ctx->mac_res);
-			for (i = 0; i < cmac_data->mac_res_size; ++i)
-			{
-				FIPS_LOG("  %d    0x%02x     0x%02x \n", i, cmac_data->mac_res[i], virt_ctx->mac_res[i]);
-			}
-
-			error = CC_REE_FIPS_ERROR_AES_CMAC_PUT;
-			break;
-		}
-	}
-
-	return error;
-}
-
-static inline enum cc_fips_error
-FIPS_HashToFipsError(enum drv_hash_mode hash_mode)
-{
-	switch (hash_mode) {
-	case DRV_HASH_SHA1:
-		return CC_REE_FIPS_ERROR_SHA1_PUT;
-	case DRV_HASH_SHA256:
-		return CC_REE_FIPS_ERROR_SHA256_PUT;
-#if (CC_SUPPORT_SHA > 256)
-	case DRV_HASH_SHA512:
-		return CC_REE_FIPS_ERROR_SHA512_PUT;
-#endif
-	default:
-		return CC_REE_FIPS_ERROR_GENERAL;
-	}
-
-	return CC_REE_FIPS_ERROR_GENERAL;
-}
-
-static inline int
-ssi_hash_fips_run_test(struct ssi_drvdata *drvdata,
-		       dma_addr_t initial_digest_dma_addr,
-		       dma_addr_t din_dma_addr,
-		       size_t data_in_size,
-		       dma_addr_t mac_res_dma_addr,
-		       enum drv_hash_mode hash_mode,
-		       enum drv_hash_hw_mode hw_mode,
-		       int digest_size,
-		       int inter_digestsize)
-{
-	/* max number of descriptors used for the flow */
-	#define FIPS_HASH_MAX_SEQ_LEN 4
-
-	int rc;
-	struct ssi_crypto_req ssi_req = {0};
-	struct cc_hw_desc desc[FIPS_HASH_MAX_SEQ_LEN];
-	int idx = 0;
-
-	/* Load initial digest */
-	hw_desc_init(&desc[idx]);
-	set_cipher_mode(&desc[idx], hw_mode);
-	set_din_type(&desc[idx], DMA_DLLI, initial_digest_dma_addr,
-		     inter_digestsize, NS_BIT);
-	set_flow_mode(&desc[idx], S_DIN_to_HASH);
-	set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
-	idx++;
-
-	/* Load the hash current length */
-	hw_desc_init(&desc[idx]);
-	set_cipher_mode(&desc[idx], hw_mode);
-	set_din_const(&desc[idx], 0, HASH_LEN_SIZE);
-	set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
-	set_flow_mode(&desc[idx], S_DIN_to_HASH);
-	set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
-	idx++;
-
-	/* data descriptor */
-	hw_desc_init(&desc[idx]);
-	set_din_type(&desc[idx], DMA_DLLI, din_dma_addr, data_in_size, NS_BIT);
-	set_flow_mode(&desc[idx], DIN_HASH);
-	idx++;
-
-	/* Get final MAC result */
-	hw_desc_init(&desc[idx]);
-	set_cipher_mode(&desc[idx], hw_mode);
-	set_dout_dlli(&desc[idx], mac_res_dma_addr, digest_size, NS_BIT, 0);
-	set_flow_mode(&desc[idx], S_HASH_to_DOUT);
-	set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
-	set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
-	if (unlikely((hash_mode == DRV_HASH_MD5) ||
-		     (hash_mode == DRV_HASH_SHA384) ||
-		     (hash_mode == DRV_HASH_SHA512))) {
-		set_bytes_swap(&desc[idx], 1);
-	} else {
-		set_cipher_config0(&desc[idx],
-				   HASH_DIGEST_RESULT_LITTLE_ENDIAN);
-	}
-	idx++;
-
-	/* perform the operation - Lock HW and push sequence */
-	BUG_ON(idx > FIPS_HASH_MAX_SEQ_LEN);
-	rc = send_request(drvdata, &ssi_req, desc, idx, false);
-
-	return rc;
-}
-
-enum cc_fips_error
-ssi_hash_fips_power_up_tests(struct ssi_drvdata *drvdata, void *cpu_addr_buffer, dma_addr_t dma_coherent_buffer)
-{
-	enum cc_fips_error error = CC_REE_FIPS_ERROR_OK;
-	size_t i;
-	struct fips_hash_ctx *virt_ctx = (struct fips_hash_ctx *)cpu_addr_buffer;
-
-	/* set the phisical pointers for initial_digest, din, mac_res */
-	dma_addr_t initial_digest_dma_addr = dma_coherent_buffer + offsetof(struct fips_hash_ctx, initial_digest);
-	dma_addr_t din_dma_addr = dma_coherent_buffer + offsetof(struct fips_hash_ctx, din);
-	dma_addr_t mac_res_dma_addr = dma_coherent_buffer + offsetof(struct fips_hash_ctx, mac_res);
-
-	for (i = 0; i < FIPS_HASH_NUM_OF_TESTS; ++i)
-	{
-		FipsHashData *hash_data = (FipsHashData *)&FipsHashDataTable[i];
-		int rc = 0;
-		enum drv_hash_hw_mode hw_mode = 0;
-		int digest_size = 0;
-		int inter_digestsize = 0;
-
-		memset(cpu_addr_buffer, 0, sizeof(struct fips_hash_ctx));
-
-		switch (hash_data->hash_mode) {
-		case DRV_HASH_SHA1:
-			hw_mode = DRV_HASH_HW_SHA1;
-			digest_size = CC_SHA1_DIGEST_SIZE;
-			inter_digestsize = CC_SHA1_DIGEST_SIZE;
-			/* copy the initial digest into the allocated cache coherent buffer */
-			memcpy(virt_ctx->initial_digest, (void *)sha1_init, CC_SHA1_DIGEST_SIZE);
-			break;
-		case DRV_HASH_SHA256:
-			hw_mode = DRV_HASH_HW_SHA256;
-			digest_size = CC_SHA256_DIGEST_SIZE;
-			inter_digestsize = CC_SHA256_DIGEST_SIZE;
-			memcpy(virt_ctx->initial_digest, (void *)sha256_init, CC_SHA256_DIGEST_SIZE);
-			break;
-#if (CC_SUPPORT_SHA > 256)
-		case DRV_HASH_SHA512:
-			hw_mode = DRV_HASH_HW_SHA512;
-			digest_size = CC_SHA512_DIGEST_SIZE;
-			inter_digestsize = CC_SHA512_DIGEST_SIZE;
-			memcpy(virt_ctx->initial_digest, (void *)sha512_init, CC_SHA512_DIGEST_SIZE);
-			break;
-#endif
-		default:
-			error = FIPS_HashToFipsError(hash_data->hash_mode);
-			break;
-		}
-
-		/* copy the din data into the allocated buffer */
-		memcpy(virt_ctx->din, hash_data->data_in, hash_data->data_in_size);
-
-		/* run the test on HW */
-		FIPS_DBG("ssi_hash_fips_run_test -  (i = %d) \n", i);
-		rc = ssi_hash_fips_run_test(drvdata,
-					    initial_digest_dma_addr,
-					    din_dma_addr,
-					    hash_data->data_in_size,
-					    mac_res_dma_addr,
-					    hash_data->hash_mode,
-					    hw_mode,
-					    digest_size,
-					    inter_digestsize);
-		if (rc != 0)
-		{
-			FIPS_LOG("ssi_hash_fips_run_test %d returned error - rc = %d \n", i, rc);
-			error = FIPS_HashToFipsError(hash_data->hash_mode);
-			break;
-		}
-
-		/* compare actual mac result to expected */
-		if (memcmp(virt_ctx->mac_res, hash_data->mac_res, digest_size) != 0)
-		{
-			FIPS_LOG("comparison error %d - hash_mode=%d digest_size=%d \n", i, hash_data->hash_mode, digest_size);
-			FIPS_LOG("  i  expected   received \n");
-			FIPS_LOG("  i  0x%08x 0x%08x \n", (size_t)hash_data->mac_res, (size_t)virt_ctx->mac_res);
-			for (i = 0; i < digest_size; ++i)
-			{
-				FIPS_LOG("  %d    0x%02x     0x%02x \n", i, hash_data->mac_res[i], virt_ctx->mac_res[i]);
-			}
-
-			error = FIPS_HashToFipsError(hash_data->hash_mode);
-			break;
-		}
-	}
-
-	return error;
-}
-
-static inline enum cc_fips_error
-FIPS_HmacToFipsError(enum drv_hash_mode hash_mode)
-{
-	switch (hash_mode) {
-	case DRV_HASH_SHA1:
-		return CC_REE_FIPS_ERROR_HMAC_SHA1_PUT;
-	case DRV_HASH_SHA256:
-		return CC_REE_FIPS_ERROR_HMAC_SHA256_PUT;
-#if (CC_SUPPORT_SHA > 256)
-	case DRV_HASH_SHA512:
-		return CC_REE_FIPS_ERROR_HMAC_SHA512_PUT;
-#endif
-	default:
-		return CC_REE_FIPS_ERROR_GENERAL;
-	}
-
-	return CC_REE_FIPS_ERROR_GENERAL;
-}
-
-static inline int
-ssi_hmac_fips_run_test(struct ssi_drvdata *drvdata,
-		       dma_addr_t initial_digest_dma_addr,
-		       dma_addr_t key_dma_addr,
-		       size_t key_size,
-		       dma_addr_t din_dma_addr,
-		       size_t data_in_size,
-		       dma_addr_t mac_res_dma_addr,
-		       enum drv_hash_mode hash_mode,
-		       enum drv_hash_hw_mode hw_mode,
-		       size_t digest_size,
-		       size_t inter_digestsize,
-		       size_t block_size,
-		       dma_addr_t k0_dma_addr,
-		       dma_addr_t tmp_digest_dma_addr,
-		       dma_addr_t digest_bytes_len_dma_addr)
-{
-	/* The implemented flow is not the same as the one implemented in ssi_hash.c (setkey + digest flows).
-	 * In this flow, there is no need to store and reload some of the intermidiate results.
-	 */
-
-	/* max number of descriptors used for the flow */
-	#define FIPS_HMAC_MAX_SEQ_LEN 12
-
-	int rc;
-	struct ssi_crypto_req ssi_req = {0};
-	struct cc_hw_desc desc[FIPS_HMAC_MAX_SEQ_LEN];
-	int idx = 0;
-	int i;
-	/* calc the hash opad first and ipad only afterwards (unlike the flow in ssi_hash.c) */
-	unsigned int hmacPadConst[2] = { HMAC_OPAD_CONST, HMAC_IPAD_CONST };
-
-	// assume (key_size <= block_size)
-	hw_desc_init(&desc[idx]);
-	set_din_type(&desc[idx], DMA_DLLI, key_dma_addr, key_size, NS_BIT);
-	set_flow_mode(&desc[idx], BYPASS);
-	set_dout_dlli(&desc[idx], k0_dma_addr, key_size, NS_BIT, 0);
-	idx++;
-
-	// if needed, append Key with zeros to create K0
-	if ((block_size - key_size) != 0) {
-		hw_desc_init(&desc[idx]);
-		set_din_const(&desc[idx], 0, (block_size - key_size));
-		set_flow_mode(&desc[idx], BYPASS);
-		set_dout_dlli(&desc[idx], (k0_dma_addr + key_size),
-			      (block_size - key_size), NS_BIT, 0);
-		idx++;
-	}
-
-	BUG_ON(idx > FIPS_HMAC_MAX_SEQ_LEN);
-	rc = send_request(drvdata, &ssi_req, desc, idx, 0);
-	if (unlikely(rc != 0)) {
-		SSI_LOG_ERR("send_request() failed (rc=%d)\n", rc);
-		return rc;
-	}
-	idx = 0;
-
-	/* calc derived HMAC key */
-	for (i = 0; i < 2; i++) {
-		/* Load hash initial state */
-		hw_desc_init(&desc[idx]);
-		set_cipher_mode(&desc[idx], hw_mode);
-		set_din_type(&desc[idx], DMA_DLLI, initial_digest_dma_addr,
-			     inter_digestsize, NS_BIT);
-		set_flow_mode(&desc[idx], S_DIN_to_HASH);
-		set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
-		idx++;
-
-		/* Load the hash current length*/
-		hw_desc_init(&desc[idx]);
-		set_cipher_mode(&desc[idx], hw_mode);
-		set_din_const(&desc[idx], 0, HASH_LEN_SIZE);
-		set_flow_mode(&desc[idx], S_DIN_to_HASH);
-		set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
-		idx++;
-
-		/* Prepare opad/ipad key */
-		hw_desc_init(&desc[idx]);
-		set_xor_val(&desc[idx], hmacPadConst[i]);
-		set_cipher_mode(&desc[idx], hw_mode);
-		set_flow_mode(&desc[idx], S_DIN_to_HASH);
-		set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
-		idx++;
-
-		/* Perform HASH update */
-		hw_desc_init(&desc[idx]);
-		set_din_type(&desc[idx], DMA_DLLI, k0_dma_addr, block_size,
-			     NS_BIT);
-		set_cipher_mode(&desc[idx], hw_mode);
-		set_xor_active(&desc[idx]);
-		set_flow_mode(&desc[idx], DIN_HASH);
-		idx++;
-
-		if (i == 0) {
-			/* First iteration - calc H(K0^opad) into tmp_digest_dma_addr */
-			hw_desc_init(&desc[idx]);
-			set_cipher_mode(&desc[idx], hw_mode);
-			set_dout_dlli(&desc[idx], tmp_digest_dma_addr,
-				      inter_digestsize, NS_BIT, 0);
-			set_flow_mode(&desc[idx], S_HASH_to_DOUT);
-			set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
-			idx++;
-
-			// is this needed?? or continue with current descriptors??
-			BUG_ON(idx > FIPS_HMAC_MAX_SEQ_LEN);
-			rc = send_request(drvdata, &ssi_req, desc, idx, 0);
-			if (unlikely(rc != 0)) {
-				SSI_LOG_ERR("send_request() failed (rc=%d)\n", rc);
-				return rc;
-			}
-			idx = 0;
-		}
-	}
-
-	/* data descriptor */
-	hw_desc_init(&desc[idx]);
-	set_din_type(&desc[idx], DMA_DLLI, din_dma_addr, data_in_size, NS_BIT);
-	set_flow_mode(&desc[idx], DIN_HASH);
-	idx++;
-
-	/* HW last hash block padding (aka. "DO_PAD") */
-	hw_desc_init(&desc[idx]);
-	set_cipher_mode(&desc[idx], hw_mode);
-	set_dout_dlli(&desc[idx], k0_dma_addr, HASH_LEN_SIZE, NS_BIT, 0);
-	set_flow_mode(&desc[idx], S_HASH_to_DOUT);
-	set_setup_mode(&desc[idx], SETUP_WRITE_STATE1);
-	set_cipher_do(&desc[idx], DO_PAD);
-	idx++;
-
-	/* store the hash digest result in the context */
-	hw_desc_init(&desc[idx]);
-	set_cipher_mode(&desc[idx], hw_mode);
-	set_dout_dlli(&desc[idx], k0_dma_addr, digest_size, NS_BIT, 0);
-	set_flow_mode(&desc[idx], S_HASH_to_DOUT);
-	if (unlikely((hash_mode == DRV_HASH_MD5) ||
-		     (hash_mode == DRV_HASH_SHA384) ||
-		     (hash_mode == DRV_HASH_SHA512))) {
-		set_bytes_swap(&desc[idx], 1);
-	} else {
-		set_cipher_config0(&desc[idx],
-				   HASH_DIGEST_RESULT_LITTLE_ENDIAN);
-	}
-	set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
-	idx++;
-
-	/* at this point:
-	 * tmp_digest = H(o_key_pad)
-	 * k0 = H(i_key_pad || m)
-	 */
-
-	/* Loading hash opad xor key state */
-	hw_desc_init(&desc[idx]);
-	set_cipher_mode(&desc[idx], hw_mode);
-	set_din_type(&desc[idx], DMA_DLLI, tmp_digest_dma_addr,
-		     inter_digestsize, NS_BIT);
-	set_flow_mode(&desc[idx], S_DIN_to_HASH);
-	set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
-	idx++;
-
-	/* Load the hash current length */
-	hw_desc_init(&desc[idx]);
-	set_cipher_mode(&desc[idx], hw_mode);
-	set_din_type(&desc[idx], DMA_DLLI, digest_bytes_len_dma_addr,
-		     HASH_LEN_SIZE, NS_BIT);
-	set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
-	set_flow_mode(&desc[idx], S_DIN_to_HASH);
-	set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
-	idx++;
-
-	/* Memory Barrier: wait for IPAD/OPAD axi write to complete */
-	hw_desc_init(&desc[idx]);
-	set_din_no_dma(&desc[idx], 0, 0xfffff0);
-	set_dout_no_dma(&desc[idx], 0, 0, 1);
-	idx++;
-
-	/* Perform HASH update */
-	hw_desc_init(&desc[idx]);
-	set_din_type(&desc[idx], DMA_DLLI, k0_dma_addr, digest_size, NS_BIT);
-	set_flow_mode(&desc[idx], DIN_HASH);
-	idx++;
-
-	/* Get final MAC result */
-	hw_desc_init(&desc[idx]);
-	set_cipher_mode(&desc[idx], hw_mode);
-	set_dout_dlli(&desc[idx], mac_res_dma_addr, digest_size, NS_BIT, 0);
-	set_flow_mode(&desc[idx], S_HASH_to_DOUT);
-	set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
-	set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
-	if (unlikely((hash_mode == DRV_HASH_MD5) ||
-		     (hash_mode == DRV_HASH_SHA384) ||
-		     (hash_mode == DRV_HASH_SHA512))) {
-		set_bytes_swap(&desc[idx], 1);
-	} else {
-		set_cipher_config0(&desc[idx],
-				   HASH_DIGEST_RESULT_LITTLE_ENDIAN);
-	}
-	idx++;
-
-	/* perform the operation - Lock HW and push sequence */
-	BUG_ON(idx > FIPS_HMAC_MAX_SEQ_LEN);
-	rc = send_request(drvdata, &ssi_req, desc, idx, false);
-
-	return rc;
-}
-
-enum cc_fips_error
-ssi_hmac_fips_power_up_tests(struct ssi_drvdata *drvdata, void *cpu_addr_buffer, dma_addr_t dma_coherent_buffer)
-{
-	enum cc_fips_error error = CC_REE_FIPS_ERROR_OK;
-	size_t i;
-	struct fips_hmac_ctx *virt_ctx = (struct fips_hmac_ctx *)cpu_addr_buffer;
-
-	/* set the phisical pointers */
-	dma_addr_t initial_digest_dma_addr = dma_coherent_buffer + offsetof(struct fips_hmac_ctx, initial_digest);
-	dma_addr_t key_dma_addr = dma_coherent_buffer + offsetof(struct fips_hmac_ctx, key);
-	dma_addr_t k0_dma_addr = dma_coherent_buffer + offsetof(struct fips_hmac_ctx, k0);
-	dma_addr_t tmp_digest_dma_addr = dma_coherent_buffer + offsetof(struct fips_hmac_ctx, tmp_digest);
-	dma_addr_t digest_bytes_len_dma_addr = dma_coherent_buffer + offsetof(struct fips_hmac_ctx, digest_bytes_len);
-	dma_addr_t din_dma_addr = dma_coherent_buffer + offsetof(struct fips_hmac_ctx, din);
-	dma_addr_t mac_res_dma_addr = dma_coherent_buffer + offsetof(struct fips_hmac_ctx, mac_res);
-
-	for (i = 0; i < FIPS_HMAC_NUM_OF_TESTS; ++i)
-	{
-		FipsHmacData *hmac_data = (FipsHmacData *)&FipsHmacDataTable[i];
-		int rc = 0;
-		enum drv_hash_hw_mode hw_mode = 0;
-		int digest_size = 0;
-		int block_size = 0;
-		int inter_digestsize = 0;
-
-		memset(cpu_addr_buffer, 0, sizeof(struct fips_hmac_ctx));
-
-		switch (hmac_data->hash_mode) {
-		case DRV_HASH_SHA1:
-			hw_mode = DRV_HASH_HW_SHA1;
-			digest_size = CC_SHA1_DIGEST_SIZE;
-			block_size = CC_SHA1_BLOCK_SIZE;
-			inter_digestsize = CC_SHA1_DIGEST_SIZE;
-			memcpy(virt_ctx->initial_digest, (void *)sha1_init, CC_SHA1_DIGEST_SIZE);
-			memcpy(virt_ctx->digest_bytes_len, digest_len_init, HASH_LEN_SIZE);
-			break;
-		case DRV_HASH_SHA256:
-			hw_mode = DRV_HASH_HW_SHA256;
-			digest_size = CC_SHA256_DIGEST_SIZE;
-			block_size = CC_SHA256_BLOCK_SIZE;
-			inter_digestsize = CC_SHA256_DIGEST_SIZE;
-			memcpy(virt_ctx->initial_digest, (void *)sha256_init, CC_SHA256_DIGEST_SIZE);
-			memcpy(virt_ctx->digest_bytes_len, digest_len_init, HASH_LEN_SIZE);
-			break;
-#if (CC_SUPPORT_SHA > 256)
-		case DRV_HASH_SHA512:
-			hw_mode = DRV_HASH_HW_SHA512;
-			digest_size = CC_SHA512_DIGEST_SIZE;
-			block_size = CC_SHA512_BLOCK_SIZE;
-			inter_digestsize = CC_SHA512_DIGEST_SIZE;
-			memcpy(virt_ctx->initial_digest, (void *)sha512_init, CC_SHA512_DIGEST_SIZE);
-			memcpy(virt_ctx->digest_bytes_len, digest_len_sha512_init, HASH_LEN_SIZE);
-			break;
-#endif
-		default:
-			error = FIPS_HmacToFipsError(hmac_data->hash_mode);
-			break;
-		}
-
-		/* copy into the allocated buffer */
-		memcpy(virt_ctx->key, hmac_data->key, hmac_data->key_size);
-		memcpy(virt_ctx->din, hmac_data->data_in, hmac_data->data_in_size);
-
-		/* run the test on HW */
-		FIPS_DBG("ssi_hmac_fips_run_test -  (i = %d) \n", i);
-		rc = ssi_hmac_fips_run_test(drvdata,
-					    initial_digest_dma_addr,
-					    key_dma_addr,
-					    hmac_data->key_size,
-					    din_dma_addr,
-					    hmac_data->data_in_size,
-					    mac_res_dma_addr,
-					    hmac_data->hash_mode,
-					    hw_mode,
-					    digest_size,
-					    inter_digestsize,
-					    block_size,
-					    k0_dma_addr,
-					    tmp_digest_dma_addr,
-					    digest_bytes_len_dma_addr);
-		if (rc != 0)
-		{
-			FIPS_LOG("ssi_hmac_fips_run_test %d returned error - rc = %d \n", i, rc);
-			error = FIPS_HmacToFipsError(hmac_data->hash_mode);
-			break;
-		}
-
-		/* compare actual mac result to expected */
-		if (memcmp(virt_ctx->mac_res, hmac_data->mac_res, digest_size) != 0)
-		{
-			FIPS_LOG("comparison error %d - hash_mode=%d digest_size=%d \n", i, hmac_data->hash_mode, digest_size);
-			FIPS_LOG("  i  expected   received \n");
-			FIPS_LOG("  i  0x%08x 0x%08x \n", (size_t)hmac_data->mac_res, (size_t)virt_ctx->mac_res);
-			for (i = 0; i < digest_size; ++i)
-			{
-				FIPS_LOG("  %d    0x%02x     0x%02x \n", i, hmac_data->mac_res[i], virt_ctx->mac_res[i]);
-			}
-
-			error = FIPS_HmacToFipsError(hmac_data->hash_mode);
-			break;
-		}
-	}
-
-	return error;
-}
-
-static inline int
-ssi_ccm_fips_run_test(struct ssi_drvdata *drvdata,
-		      enum drv_crypto_direction direction,
-		      dma_addr_t key_dma_addr,
-		      size_t key_size,
-		      dma_addr_t iv_dma_addr,
-		      dma_addr_t ctr_cnt_0_dma_addr,
-		      dma_addr_t b0_a0_adata_dma_addr,
-		      size_t b0_a0_adata_size,
-		      dma_addr_t din_dma_addr,
-		      size_t din_size,
-		      dma_addr_t dout_dma_addr,
-		      dma_addr_t mac_res_dma_addr)
-{
-	/* max number of descriptors used for the flow */
-	#define FIPS_CCM_MAX_SEQ_LEN 10
-
-	int rc;
-	struct ssi_crypto_req ssi_req = {0};
-	struct cc_hw_desc desc[FIPS_CCM_MAX_SEQ_LEN];
-	unsigned int idx = 0;
-	unsigned int cipher_flow_mode;
-
-	if (direction == DRV_CRYPTO_DIRECTION_DECRYPT) {
-		cipher_flow_mode = AES_to_HASH_and_DOUT;
-	} else { /* Encrypt */
-		cipher_flow_mode = AES_and_HASH;
-	}
-
-	/* load key */
-	hw_desc_init(&desc[idx]);
-	set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);
-	set_din_type(&desc[idx], DMA_DLLI, key_dma_addr,
-		     ((key_size == NIST_AESCCM_192_BIT_KEY_SIZE) ?
-		     CC_AES_KEY_SIZE_MAX : key_size), NS_BIT)
-	set_key_size_aes(&desc[idx], key_size);
-	set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
-	set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
-	set_flow_mode(&desc[idx], S_DIN_to_AES);
-	idx++;
-
-	/* load ctr state */
-	hw_desc_init(&desc[idx]);
-	set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);
-	set_key_size_aes(&desc[idx], key_size);
-	set_din_type(&desc[idx], DMA_DLLI, iv_dma_addr, AES_BLOCK_SIZE,
-		     NS_BIT);
-	set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
-	set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
-	set_flow_mode(&desc[idx], S_DIN_to_AES);
-	idx++;
-
-	/* load MAC key */
-	hw_desc_init(&desc[idx]);
-	set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);
-	set_din_type(&desc[idx], DMA_DLLI, key_dma_addr,
-		     ((key_size == NIST_AESCCM_192_BIT_KEY_SIZE) ?
-		     CC_AES_KEY_SIZE_MAX : key_size), NS_BIT);
-	set_key_size_aes(&desc[idx], key_size);
-	set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
-	set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
-	set_flow_mode(&desc[idx], S_DIN_to_HASH);
-	set_aes_not_hash_mode(&desc[idx]);
-	idx++;
-
-	/* load MAC state */
-	hw_desc_init(&desc[idx]);
-	set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);
-	set_key_size_aes(&desc[idx], key_size);
-	set_din_type(&desc[idx], DMA_DLLI, mac_res_dma_addr,
-		     NIST_AESCCM_TAG_SIZE, NS_BIT);
-	set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
-	set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
-	set_flow_mode(&desc[idx], S_DIN_to_HASH);
-	set_aes_not_hash_mode(&desc[idx]);
-	idx++;
-
-	/* prcess assoc data */
-	hw_desc_init(&desc[idx]);
-	set_din_type(&desc[idx], DMA_DLLI, b0_a0_adata_dma_addr,
-		     b0_a0_adata_size, NS_BIT);
-	set_flow_mode(&desc[idx], DIN_HASH);
-	idx++;
-
-	/* process the cipher */
-	hw_desc_init(&desc[idx]);
-	set_din_type(&desc[idx], DMA_DLLI, din_dma_addr, din_size, NS_BIT);
-	set_dout_dlli(&desc[idx], dout_dma_addr, din_size, NS_BIT, 0);
-	set_flow_mode(&desc[idx], cipher_flow_mode);
-	idx++;
-
-	/* Read temporal MAC */
-	hw_desc_init(&desc[idx]);
-	set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);
-	set_dout_dlli(&desc[idx], mac_res_dma_addr, NIST_AESCCM_TAG_SIZE,
-		      NS_BIT, 0);
-	set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
-	set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN);
-	set_flow_mode(&desc[idx], S_HASH_to_DOUT);
-	set_aes_not_hash_mode(&desc[idx]);
-	idx++;
-
-	/* load AES-CTR state (for last MAC calculation)*/
-	hw_desc_init(&desc[idx]);
-	set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);
-	set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
-	set_din_type(&desc[idx], DMA_DLLI, ctr_cnt_0_dma_addr, AES_BLOCK_SIZE,
-		     NS_BIT);
-	set_key_size_aes(&desc[idx], key_size);
-	set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
-	set_flow_mode(&desc[idx], S_DIN_to_AES);
-	idx++;
-
-	/* Memory Barrier */
-	hw_desc_init(&desc[idx]);
-	set_din_no_dma(&desc[idx], 0, 0xfffff0);
-	set_dout_no_dma(&desc[idx], 0, 0, 1);
-	idx++;
-
-	/* encrypt the "T" value and store MAC inplace */
-	hw_desc_init(&desc[idx]);
-	set_din_type(&desc[idx], DMA_DLLI, mac_res_dma_addr,
-		     NIST_AESCCM_TAG_SIZE, NS_BIT);
-	set_dout_dlli(&desc[idx], mac_res_dma_addr, NIST_AESCCM_TAG_SIZE,
-		      NS_BIT, 0);
-	set_flow_mode(&desc[idx], DIN_AES_DOUT);
-	idx++;
-
-	/* perform the operation - Lock HW and push sequence */
-	BUG_ON(idx > FIPS_CCM_MAX_SEQ_LEN);
-	rc = send_request(drvdata, &ssi_req, desc, idx, false);
-
-	return rc;
-}
-
-enum cc_fips_error
-ssi_ccm_fips_power_up_tests(struct ssi_drvdata *drvdata, void *cpu_addr_buffer, dma_addr_t dma_coherent_buffer)
-{
-	enum cc_fips_error error = CC_REE_FIPS_ERROR_OK;
-	size_t i;
-	struct fips_ccm_ctx *virt_ctx = (struct fips_ccm_ctx *)cpu_addr_buffer;
-
-	/* set the phisical pointers */
-	dma_addr_t b0_a0_adata_dma_addr = dma_coherent_buffer + offsetof(struct fips_ccm_ctx, b0_a0_adata);
-	dma_addr_t iv_dma_addr = dma_coherent_buffer + offsetof(struct fips_ccm_ctx, iv);
-	dma_addr_t ctr_cnt_0_dma_addr = dma_coherent_buffer + offsetof(struct fips_ccm_ctx, ctr_cnt_0);
-	dma_addr_t key_dma_addr = dma_coherent_buffer + offsetof(struct fips_ccm_ctx, key);
-	dma_addr_t din_dma_addr = dma_coherent_buffer + offsetof(struct fips_ccm_ctx, din);
-	dma_addr_t dout_dma_addr = dma_coherent_buffer + offsetof(struct fips_ccm_ctx, dout);
-	dma_addr_t mac_res_dma_addr = dma_coherent_buffer + offsetof(struct fips_ccm_ctx, mac_res);
-
-	for (i = 0; i < FIPS_CCM_NUM_OF_TESTS; ++i)
-	{
-		FipsCcmData *ccmData = (FipsCcmData *)&FipsCcmDataTable[i];
-		int rc = 0;
-
-		memset(cpu_addr_buffer, 0, sizeof(struct fips_ccm_ctx));
-
-		/* copy the nonce, key, adata, din data into the allocated buffer */
-		memcpy(virt_ctx->key, ccmData->key, ccmData->keySize);
-		memcpy(virt_ctx->din, ccmData->dataIn, ccmData->dataInSize);
-		{
-			/* build B0 -- B0, nonce, l(m) */
-			__be16 data = cpu_to_be16(NIST_AESCCM_TEXT_SIZE);
-
-			virt_ctx->b0_a0_adata[0] = NIST_AESCCM_B0_VAL;
-			memcpy(virt_ctx->b0_a0_adata + 1, ccmData->nonce, NIST_AESCCM_NONCE_SIZE);
-			memcpy(virt_ctx->b0_a0_adata + 14, (u8 *)&data, sizeof(__be16));
-			/* build A0+ADATA */
-			virt_ctx->b0_a0_adata[NIST_AESCCM_IV_SIZE + 0] = (ccmData->adataSize >> 8) & 0xFF;
-			virt_ctx->b0_a0_adata[NIST_AESCCM_IV_SIZE + 1] = ccmData->adataSize & 0xFF;
-			memcpy(virt_ctx->b0_a0_adata + NIST_AESCCM_IV_SIZE + 2, ccmData->adata, ccmData->adataSize);
-			/* iv */
-			virt_ctx->iv[0] = 1; /* L' */
-			memcpy(virt_ctx->iv + 1, ccmData->nonce, NIST_AESCCM_NONCE_SIZE);
-			virt_ctx->iv[15] = 1;
-			/* ctr_count_0 */
-			memcpy(virt_ctx->ctr_cnt_0, virt_ctx->iv, NIST_AESCCM_IV_SIZE);
-			virt_ctx->ctr_cnt_0[15] = 0;
-		}
-
-		FIPS_DBG("ssi_ccm_fips_run_test -  (i = %d) \n", i);
-		rc = ssi_ccm_fips_run_test(drvdata,
-					   ccmData->direction,
-					   key_dma_addr,
-					   ccmData->keySize,
-					   iv_dma_addr,
-					   ctr_cnt_0_dma_addr,
-					   b0_a0_adata_dma_addr,
-					   FIPS_CCM_B0_A0_ADATA_SIZE,
-					   din_dma_addr,
-					   ccmData->dataInSize,
-					   dout_dma_addr,
-					   mac_res_dma_addr);
-		if (rc != 0)
-		{
-			FIPS_LOG("ssi_ccm_fips_run_test %d returned error - rc = %d \n", i, rc);
-			error = CC_REE_FIPS_ERROR_AESCCM_PUT;
-			break;
-		}
-
-		/* compare actual dout to expected */
-		if (memcmp(virt_ctx->dout, ccmData->dataOut, ccmData->dataInSize) != 0)
-		{
-			FIPS_LOG("dout comparison error %d - size=%d \n", i, ccmData->dataInSize);
-			error = CC_REE_FIPS_ERROR_AESCCM_PUT;
-			break;
-		}
-
-		/* compare actual mac result to expected */
-		if (memcmp(virt_ctx->mac_res, ccmData->macResOut, ccmData->tagSize) != 0)
-		{
-			FIPS_LOG("mac_res comparison error %d - mac_size=%d \n", i, ccmData->tagSize);
-			FIPS_LOG("  i  expected   received \n");
-			FIPS_LOG("  i  0x%08x 0x%08x \n", (size_t)ccmData->macResOut, (size_t)virt_ctx->mac_res);
-			for (i = 0; i < ccmData->tagSize; ++i)
-			{
-				FIPS_LOG("  %d    0x%02x     0x%02x \n", i, ccmData->macResOut[i], virt_ctx->mac_res[i]);
-			}
-
-			error = CC_REE_FIPS_ERROR_AESCCM_PUT;
-			break;
-		}
-	}
-
-	return error;
-}
-
-static inline int
-ssi_gcm_fips_run_test(struct ssi_drvdata *drvdata,
-		      enum drv_crypto_direction direction,
-		      dma_addr_t key_dma_addr,
-		      size_t key_size,
-		      dma_addr_t hkey_dma_addr,
-		      dma_addr_t block_len_dma_addr,
-		      dma_addr_t iv_inc1_dma_addr,
-		      dma_addr_t iv_inc2_dma_addr,
-		      dma_addr_t adata_dma_addr,
-		      size_t adata_size,
-		      dma_addr_t din_dma_addr,
-		      size_t din_size,
-		      dma_addr_t dout_dma_addr,
-		      dma_addr_t mac_res_dma_addr)
-{
-	/* max number of descriptors used for the flow */
-	#define FIPS_GCM_MAX_SEQ_LEN 15
-
-	int rc;
-	struct ssi_crypto_req ssi_req = {0};
-	struct cc_hw_desc desc[FIPS_GCM_MAX_SEQ_LEN];
-	unsigned int idx = 0;
-	unsigned int cipher_flow_mode;
-
-	if (direction == DRV_CRYPTO_DIRECTION_DECRYPT) {
-		cipher_flow_mode = AES_and_HASH;
-	} else { /* Encrypt */
-		cipher_flow_mode = AES_to_HASH_and_DOUT;
-	}
-
-/////////////////////////////////   1   ////////////////////////////////////
-//	ssi_aead_gcm_setup_ghash_desc(req, desc, seq_size);
-/////////////////////////////////   1   ////////////////////////////////////
-
-	/* load key to AES */
-	hw_desc_init(&desc[idx]);
-	set_cipher_mode(&desc[idx], DRV_CIPHER_ECB);
-	set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
-	set_din_type(&desc[idx], DMA_DLLI, key_dma_addr, key_size, NS_BIT);
-	set_key_size_aes(&desc[idx], key_size);
-	set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
-	set_flow_mode(&desc[idx], S_DIN_to_AES);
-	idx++;
-
-	/* process one zero block to generate hkey */
-	hw_desc_init(&desc[idx]);
-	set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE);
-	set_dout_dlli(&desc[idx], hkey_dma_addr, AES_BLOCK_SIZE, NS_BIT, 0);
-	set_flow_mode(&desc[idx], DIN_AES_DOUT);
-	idx++;
-
-	/* Memory Barrier */
-	hw_desc_init(&desc[idx]);
-	set_din_no_dma(&desc[idx], 0, 0xfffff0);
-	set_dout_no_dma(&desc[idx], 0, 0, 1);
-	idx++;
-
-	/* Load GHASH subkey */
-	hw_desc_init(&desc[idx]);
-	set_din_type(&desc[idx], DMA_DLLI, hkey_dma_addr, AES_BLOCK_SIZE,
-		     NS_BIT);
-	set_dout_no_dma(&desc[idx], 0, 0, 1);
-	set_flow_mode(&desc[idx], S_DIN_to_HASH);
-	set_aes_not_hash_mode(&desc[idx]);
-	set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
-	set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
-	set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
-	idx++;
-
-	/* Configure Hash Engine to work with GHASH.
-	 * Since it was not possible to extend HASH submodes to add GHASH,
-	 * The following command is necessary in order to
-	 * select GHASH (according to HW designers)
-	 */
-	hw_desc_init(&desc[idx]);
-	set_din_no_dma(&desc[idx], 0, 0xfffff0);
-	set_dout_no_dma(&desc[idx], 0, 0, 1);
-	set_flow_mode(&desc[idx], S_DIN_to_HASH);
-	set_aes_not_hash_mode(&desc[idx]);
-	set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
-	set_cipher_do(&desc[idx], 1); //1=AES_SK RKEK
-	set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
-	set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
-	set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
-	idx++;
-
-	/* Load GHASH initial STATE (which is 0). (for any hash there is an initial state) */
-	hw_desc_init(&desc[idx]);
-	set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE);
-	set_dout_no_dma(&desc[idx], 0, 0, 1);
-	set_flow_mode(&desc[idx], S_DIN_to_HASH);
-	set_aes_not_hash_mode(&desc[idx]);
-	set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
-	set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
-	set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
-	idx++;
-
-/////////////////////////////////   2   ////////////////////////////////////
-	/* prcess(ghash) assoc data */
-//	if (req->assoclen > 0)
-//		ssi_aead_create_assoc_desc(req, DIN_HASH, desc, seq_size);
-/////////////////////////////////   2   ////////////////////////////////////
-
-	hw_desc_init(&desc[idx]);
-	set_din_type(&desc[idx], DMA_DLLI, adata_dma_addr, adata_size, NS_BIT);
-	set_flow_mode(&desc[idx], DIN_HASH);
-	idx++;
-
-/////////////////////////////////   3   ////////////////////////////////////
-//	ssi_aead_gcm_setup_gctr_desc(req, desc, seq_size);
-/////////////////////////////////   3   ////////////////////////////////////
-
-	/* load key to AES*/
-	hw_desc_init(&desc[idx]);
-	set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
-	set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
-	set_din_type(&desc[idx], DMA_DLLI, key_dma_addr, key_size, NS_BIT);
-	set_key_size_aes(&desc[idx], key_size);
-	set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
-	set_flow_mode(&desc[idx], S_DIN_to_AES);
-	idx++;
-
-	/* load AES/CTR initial CTR value inc by 2*/
-	hw_desc_init(&desc[idx]);
-	set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
-	set_key_size_aes(&desc[idx], key_size);
-	set_din_type(&desc[idx], DMA_DLLI, iv_inc2_dma_addr, AES_BLOCK_SIZE,
-		     NS_BIT);
-	set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
-	set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
-	set_flow_mode(&desc[idx], S_DIN_to_AES);
-	idx++;
-
-/////////////////////////////////   4   ////////////////////////////////////
-	/* process(gctr+ghash) */
-//	if (req_ctx->cryptlen != 0)
-//		ssi_aead_process_cipher_data_desc(req, cipher_flow_mode, desc, seq_size);
-/////////////////////////////////   4   ////////////////////////////////////
-
-	hw_desc_init(&desc[idx]);
-	set_din_type(&desc[idx], DMA_DLLI, din_dma_addr, din_size, NS_BIT);
-	set_dout_dlli(&desc[idx], dout_dma_addr, din_size, NS_BIT, 0);
-	set_flow_mode(&desc[idx], cipher_flow_mode);
-	idx++;
-
-/////////////////////////////////   5   ////////////////////////////////////
-//	ssi_aead_process_gcm_result_desc(req, desc, seq_size);
-/////////////////////////////////   5   ////////////////////////////////////
-
-	/* prcess(ghash) gcm_block_len */
-	hw_desc_init(&desc[idx]);
-	set_din_type(&desc[idx], DMA_DLLI, block_len_dma_addr, AES_BLOCK_SIZE,
-		     NS_BIT);
-	set_flow_mode(&desc[idx], DIN_HASH);
-	idx++;
-
-	/* Store GHASH state after GHASH(Associated Data + Cipher +LenBlock) */
-	hw_desc_init(&desc[idx]);
-	set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
-	set_din_no_dma(&desc[idx], 0, 0xfffff0);
-	set_dout_dlli(&desc[idx], mac_res_dma_addr, AES_BLOCK_SIZE, NS_BIT, 0);
-	set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
-	set_flow_mode(&desc[idx], S_HASH_to_DOUT);
-	set_aes_not_hash_mode(&desc[idx]);
-	idx++;
-
-	/* load AES/CTR initial CTR value inc by 1*/
-	hw_desc_init(&desc[idx]);
-	set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
-	set_key_size_aes(&desc[idx], key_size);
-	set_din_type(&desc[idx], DMA_DLLI, iv_inc1_dma_addr, AES_BLOCK_SIZE,
-		     NS_BIT);
-	set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
-	set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
-	set_flow_mode(&desc[idx], S_DIN_to_AES);
-	idx++;
-
-	/* Memory Barrier */
-	hw_desc_init(&desc[idx]);
-	set_din_no_dma(&desc[idx], 0, 0xfffff0);
-	set_dout_no_dma(&desc[idx], 0, 0, 1);
-	idx++;
-
-	/* process GCTR on stored GHASH and store MAC inplace */
-	hw_desc_init(&desc[idx]);
-	set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
-	set_din_type(&desc[idx], DMA_DLLI, mac_res_dma_addr, AES_BLOCK_SIZE,
-		     NS_BIT);
-	set_dout_dlli(&desc[idx], mac_res_dma_addr, AES_BLOCK_SIZE, NS_BIT, 0);
-	set_flow_mode(&desc[idx], DIN_AES_DOUT);
-	idx++;
-
-	/* perform the operation - Lock HW and push sequence */
-	BUG_ON(idx > FIPS_GCM_MAX_SEQ_LEN);
-	rc = send_request(drvdata, &ssi_req, desc, idx, false);
-
-	return rc;
-}
-
-enum cc_fips_error
-ssi_gcm_fips_power_up_tests(struct ssi_drvdata *drvdata, void *cpu_addr_buffer, dma_addr_t dma_coherent_buffer)
-{
-	enum cc_fips_error error = CC_REE_FIPS_ERROR_OK;
-	size_t i;
-	struct fips_gcm_ctx *virt_ctx = (struct fips_gcm_ctx *)cpu_addr_buffer;
-
-	/* set the phisical pointers */
-	dma_addr_t adata_dma_addr = dma_coherent_buffer + offsetof(struct fips_gcm_ctx, adata);
-	dma_addr_t key_dma_addr = dma_coherent_buffer + offsetof(struct fips_gcm_ctx, key);
-	dma_addr_t hkey_dma_addr = dma_coherent_buffer + offsetof(struct fips_gcm_ctx, hkey);
-	dma_addr_t din_dma_addr = dma_coherent_buffer + offsetof(struct fips_gcm_ctx, din);
-	dma_addr_t dout_dma_addr = dma_coherent_buffer + offsetof(struct fips_gcm_ctx, dout);
-	dma_addr_t mac_res_dma_addr = dma_coherent_buffer + offsetof(struct fips_gcm_ctx, mac_res);
-	dma_addr_t len_block_dma_addr = dma_coherent_buffer + offsetof(struct fips_gcm_ctx, len_block);
-	dma_addr_t iv_inc1_dma_addr = dma_coherent_buffer + offsetof(struct fips_gcm_ctx, iv_inc1);
-	dma_addr_t iv_inc2_dma_addr = dma_coherent_buffer + offsetof(struct fips_gcm_ctx, iv_inc2);
-
-	for (i = 0; i < FIPS_GCM_NUM_OF_TESTS; ++i)
-	{
-		FipsGcmData *gcmData = (FipsGcmData *)&FipsGcmDataTable[i];
-		int rc = 0;
-
-		memset(cpu_addr_buffer, 0, sizeof(struct fips_gcm_ctx));
-
-		/* copy the key, adata, din data - into the allocated buffer */
-		memcpy(virt_ctx->key, gcmData->key, gcmData->keySize);
-		memcpy(virt_ctx->adata, gcmData->adata, gcmData->adataSize);
-		memcpy(virt_ctx->din, gcmData->dataIn, gcmData->dataInSize);
-
-		/* len_block */
-		{
-			__be64 len_bits;
-
-			len_bits = cpu_to_be64(gcmData->adataSize * 8);
-			memcpy(virt_ctx->len_block, &len_bits, sizeof(len_bits));
-			len_bits = cpu_to_be64(gcmData->dataInSize * 8);
-			memcpy(virt_ctx->len_block + 8, &len_bits, sizeof(len_bits));
-		}
-		/* iv_inc1, iv_inc2 */
-		{
-			__be32 counter = cpu_to_be32(1);
-
-			memcpy(virt_ctx->iv_inc1, gcmData->iv, NIST_AESGCM_IV_SIZE);
-			memcpy(virt_ctx->iv_inc1 + NIST_AESGCM_IV_SIZE, &counter, sizeof(counter));
-			counter = cpu_to_be32(2);
-			memcpy(virt_ctx->iv_inc2, gcmData->iv, NIST_AESGCM_IV_SIZE);
-			memcpy(virt_ctx->iv_inc2 + NIST_AESGCM_IV_SIZE, &counter, sizeof(counter));
-		}
-
-		FIPS_DBG("ssi_gcm_fips_run_test -  (i = %d) \n", i);
-		rc = ssi_gcm_fips_run_test(drvdata,
-					   gcmData->direction,
-					   key_dma_addr,
-					   gcmData->keySize,
-					   hkey_dma_addr,
-					   len_block_dma_addr,
-					   iv_inc1_dma_addr,
-					   iv_inc2_dma_addr,
-					   adata_dma_addr,
-					   gcmData->adataSize,
-					   din_dma_addr,
-					   gcmData->dataInSize,
-					   dout_dma_addr,
-					   mac_res_dma_addr);
-		if (rc != 0)
-		{
-			FIPS_LOG("ssi_gcm_fips_run_test %d returned error - rc = %d \n", i, rc);
-			error = CC_REE_FIPS_ERROR_AESGCM_PUT;
-			break;
-		}
-
-		if (gcmData->direction == DRV_CRYPTO_DIRECTION_ENCRYPT) {
-			/* compare actual dout to expected */
-			if (memcmp(virt_ctx->dout, gcmData->dataOut, gcmData->dataInSize) != 0)
-			{
-				FIPS_LOG("dout comparison error %d - size=%d \n", i, gcmData->dataInSize);
-				FIPS_LOG("  i  expected   received \n");
-				FIPS_LOG("  i  0x%08x 0x%08x \n", (size_t)gcmData->dataOut, (size_t)virt_ctx->dout);
-				for (i = 0; i < gcmData->dataInSize; ++i)
-				{
-					FIPS_LOG("  %d    0x%02x     0x%02x \n", i, gcmData->dataOut[i], virt_ctx->dout[i]);
-				}
-
-				error = CC_REE_FIPS_ERROR_AESGCM_PUT;
-				break;
-			}
-		}
-
-		/* compare actual mac result to expected */
-		if (memcmp(virt_ctx->mac_res, gcmData->macResOut, gcmData->tagSize) != 0)
-		{
-			FIPS_LOG("mac_res comparison error %d - mac_size=%d \n", i, gcmData->tagSize);
-			FIPS_LOG("  i  expected   received \n");
-			FIPS_LOG("  i  0x%08x 0x%08x \n", (size_t)gcmData->macResOut, (size_t)virt_ctx->mac_res);
-			for (i = 0; i < gcmData->tagSize; ++i)
-			{
-				FIPS_LOG("  %d    0x%02x     0x%02x \n", i, gcmData->macResOut[i], virt_ctx->mac_res[i]);
-			}
-
-			error = CC_REE_FIPS_ERROR_AESGCM_PUT;
-			break;
-		}
-	}
-	return error;
-}
-
-size_t ssi_fips_max_mem_alloc_size(void)
-{
-	FIPS_DBG("sizeof(struct fips_cipher_ctx) %d \n", sizeof(struct fips_cipher_ctx));
-	FIPS_DBG("sizeof(struct fips_cmac_ctx) %d \n", sizeof(struct fips_cmac_ctx));
-	FIPS_DBG("sizeof(struct fips_hash_ctx) %d \n", sizeof(struct fips_hash_ctx));
-	FIPS_DBG("sizeof(struct fips_hmac_ctx) %d \n", sizeof(struct fips_hmac_ctx));
-	FIPS_DBG("sizeof(struct fips_ccm_ctx) %d \n", sizeof(struct fips_ccm_ctx));
-	FIPS_DBG("sizeof(struct fips_gcm_ctx) %d \n", sizeof(struct fips_gcm_ctx));
-
-	return sizeof(fips_ctx);
-}
-
diff --git a/drivers/staging/ccree/ssi_fips_local.c b/drivers/staging/ccree/ssi_fips_local.c
deleted file mode 100644
index aefb71dc9e9a..000000000000
--- a/drivers/staging/ccree/ssi_fips_local.c
+++ /dev/null
@@ -1,357 +0,0 @@
-/*
- * Copyright (C) 2012-2017 ARM Limited or its affiliates.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-/**************************************************************
- * This file defines the driver FIPS internal function, used by the driver itself.
- ***************************************************************/
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/platform_device.h>
-#include <crypto/des.h>
-
-#include "ssi_config.h"
-#include "ssi_driver.h"
-#include "cc_hal.h"
-
-#define FIPS_POWER_UP_TEST_CIPHER	1
-#define FIPS_POWER_UP_TEST_CMAC		1
-#define FIPS_POWER_UP_TEST_HASH		1
-#define FIPS_POWER_UP_TEST_HMAC		1
-#define FIPS_POWER_UP_TEST_CCM		1
-#define FIPS_POWER_UP_TEST_GCM		1
-
-static bool ssi_fips_support = 1;
-module_param(ssi_fips_support, bool, 0644);
-MODULE_PARM_DESC(ssi_fips_support, "FIPS supported flag: 0 - off , 1 - on (default)");
-
-static void fips_dsr(unsigned long devarg);
-
-struct ssi_fips_handle {
-#ifdef COMP_IN_WQ
-	struct workqueue_struct *workq;
-	struct delayed_work fipswork;
-#else
-	struct tasklet_struct fipstask;
-#endif
-};
-
-extern int ssi_fips_get_state(enum cc_fips_state_t *p_state);
-extern int ssi_fips_get_error(enum cc_fips_error *p_err);
-extern int ssi_fips_ext_set_state(enum cc_fips_state_t state);
-extern int ssi_fips_ext_set_error(enum cc_fips_error err);
-
-/* FIPS power-up tests */
-extern enum cc_fips_error ssi_cipher_fips_power_up_tests(struct ssi_drvdata *drvdata, void *cpu_addr_buffer, dma_addr_t dma_coherent_buffer);
-extern enum cc_fips_error ssi_cmac_fips_power_up_tests(struct ssi_drvdata *drvdata, void *cpu_addr_buffer, dma_addr_t dma_coherent_buffer);
-extern enum cc_fips_error ssi_hash_fips_power_up_tests(struct ssi_drvdata *drvdata, void *cpu_addr_buffer, dma_addr_t dma_coherent_buffer);
-extern enum cc_fips_error ssi_hmac_fips_power_up_tests(struct ssi_drvdata *drvdata, void *cpu_addr_buffer, dma_addr_t dma_coherent_buffer);
-extern enum cc_fips_error ssi_ccm_fips_power_up_tests(struct ssi_drvdata *drvdata, void *cpu_addr_buffer, dma_addr_t dma_coherent_buffer);
-extern enum cc_fips_error ssi_gcm_fips_power_up_tests(struct ssi_drvdata *drvdata, void *cpu_addr_buffer, dma_addr_t dma_coherent_buffer);
-extern size_t ssi_fips_max_mem_alloc_size(void);
-
-/* The function called once at driver entry point to check whether TEE FIPS error occured.*/
-static enum ssi_fips_error ssi_fips_get_tee_error(struct ssi_drvdata *drvdata)
-{
-	u32 regVal;
-	void __iomem *cc_base = drvdata->cc_base;
-
-	regVal = CC_HAL_READ_REGISTER(CC_REG_OFFSET(HOST_RGF, GPR_HOST));
-	if (regVal == (CC_FIPS_SYNC_TEE_STATUS | CC_FIPS_SYNC_MODULE_OK))
-		return CC_REE_FIPS_ERROR_OK;
-
-	return CC_REE_FIPS_ERROR_FROM_TEE;
-}
-
-/*
- * This function should push the FIPS REE library status towards the TEE library.
- * By writing the error state to HOST_GPR0 register. The function is called from
- * driver entry point so no need to protect by mutex.
- */
-static void ssi_fips_update_tee_upon_ree_status(struct ssi_drvdata *drvdata, enum cc_fips_error err)
-{
-	void __iomem *cc_base = drvdata->cc_base;
-
-	if (err == CC_REE_FIPS_ERROR_OK)
-		CC_HAL_WRITE_REGISTER(CC_REG_OFFSET(HOST_RGF, HOST_GPR0), (CC_FIPS_SYNC_REE_STATUS | CC_FIPS_SYNC_MODULE_OK));
-	else
-		CC_HAL_WRITE_REGISTER(CC_REG_OFFSET(HOST_RGF, HOST_GPR0), (CC_FIPS_SYNC_REE_STATUS | CC_FIPS_SYNC_MODULE_ERROR));
-}
-
-void ssi_fips_fini(struct ssi_drvdata *drvdata)
-{
-	struct ssi_fips_handle *fips_h = drvdata->fips_handle;
-
-	if (!fips_h)
-		return; /* Not allocated */
-
-#ifdef COMP_IN_WQ
-	if (fips_h->workq) {
-		flush_workqueue(fips_h->workq);
-		destroy_workqueue(fips_h->workq);
-	}
-#else
-	/* Kill tasklet */
-	tasklet_kill(&fips_h->fipstask);
-#endif
-	memset(fips_h, 0, sizeof(struct ssi_fips_handle));
-	kfree(fips_h);
-	drvdata->fips_handle = NULL;
-}
-
-void fips_handler(struct ssi_drvdata *drvdata)
-{
-	struct ssi_fips_handle *fips_handle_ptr =
-						drvdata->fips_handle;
-#ifdef COMP_IN_WQ
-	queue_delayed_work(fips_handle_ptr->workq, &fips_handle_ptr->fipswork, 0);
-#else
-	tasklet_schedule(&fips_handle_ptr->fipstask);
-#endif
-}
-
-#ifdef COMP_IN_WQ
-static void fips_wq_handler(struct work_struct *work)
-{
-	struct ssi_drvdata *drvdata =
-		container_of(work, struct ssi_drvdata, fipswork.work);
-
-	fips_dsr((unsigned long)drvdata);
-}
-#endif
-
-/* Deferred service handler, run as interrupt-fired tasklet */
-static void fips_dsr(unsigned long devarg)
-{
-	struct ssi_drvdata *drvdata = (struct ssi_drvdata *)devarg;
-	void __iomem *cc_base = drvdata->cc_base;
-	u32 irq;
-	u32 teeFipsError = 0;
-
-	irq = (drvdata->irq & (SSI_GPR0_IRQ_MASK));
-
-	if (irq & SSI_GPR0_IRQ_MASK) {
-		teeFipsError = CC_HAL_READ_REGISTER(CC_REG_OFFSET(HOST_RGF, GPR_HOST));
-		if (teeFipsError != (CC_FIPS_SYNC_TEE_STATUS | CC_FIPS_SYNC_MODULE_OK))
-			ssi_fips_set_error(drvdata, CC_REE_FIPS_ERROR_FROM_TEE);
-	}
-
-	/* after verifing that there is nothing to do, Unmask AXI completion interrupt */
-	CC_HAL_WRITE_REGISTER(CC_REG_OFFSET(HOST_RGF, HOST_IMR),
-		CC_HAL_READ_REGISTER(
-		CC_REG_OFFSET(HOST_RGF, HOST_IMR)) & ~irq);
-}
-
-enum cc_fips_error cc_fips_run_power_up_tests(struct ssi_drvdata *drvdata)
-{
-	enum cc_fips_error fips_error = CC_REE_FIPS_ERROR_OK;
-	void *cpu_addr_buffer = NULL;
-	dma_addr_t dma_handle;
-	size_t alloc_buff_size = ssi_fips_max_mem_alloc_size();
-	struct device *dev = &drvdata->plat_dev->dev;
-
-	// allocate memory using dma_alloc_coherent - for phisical, consecutive and cache coherent buffer (memory map is not needed)
-	// the return value is the virtual address - use it to copy data into the buffer
-	// the dma_handle is the returned phy address - use it in the HW descriptor
-	FIPS_DBG("dma_alloc_coherent \n");
-	cpu_addr_buffer = dma_alloc_coherent(dev, alloc_buff_size, &dma_handle, GFP_KERNEL);
-	if (!cpu_addr_buffer)
-		return CC_REE_FIPS_ERROR_GENERAL;
-
-	FIPS_DBG("allocated coherent buffer - addr 0x%08X , size = %d \n", (size_t)cpu_addr_buffer, alloc_buff_size);
-
-#if FIPS_POWER_UP_TEST_CIPHER
-	FIPS_DBG("ssi_cipher_fips_power_up_tests ...\n");
-	fips_error = ssi_cipher_fips_power_up_tests(drvdata, cpu_addr_buffer, dma_handle);
-	FIPS_DBG("ssi_cipher_fips_power_up_tests - done. (fips_error = %d) \n", fips_error);
-#endif
-#if FIPS_POWER_UP_TEST_CMAC
-	if (likely(fips_error == CC_REE_FIPS_ERROR_OK)) {
-		FIPS_DBG("ssi_cmac_fips_power_up_tests ...\n");
-		fips_error = ssi_cmac_fips_power_up_tests(drvdata, cpu_addr_buffer, dma_handle);
-		FIPS_DBG("ssi_cmac_fips_power_up_tests - done. (fips_error = %d) \n", fips_error);
-	}
-#endif
-#if FIPS_POWER_UP_TEST_HASH
-	if (likely(fips_error == CC_REE_FIPS_ERROR_OK)) {
-		FIPS_DBG("ssi_hash_fips_power_up_tests ...\n");
-		fips_error = ssi_hash_fips_power_up_tests(drvdata, cpu_addr_buffer, dma_handle);
-		FIPS_DBG("ssi_hash_fips_power_up_tests - done. (fips_error = %d) \n", fips_error);
-	}
-#endif
-#if FIPS_POWER_UP_TEST_HMAC
-	if (likely(fips_error == CC_REE_FIPS_ERROR_OK)) {
-		FIPS_DBG("ssi_hmac_fips_power_up_tests ...\n");
-		fips_error = ssi_hmac_fips_power_up_tests(drvdata, cpu_addr_buffer, dma_handle);
-		FIPS_DBG("ssi_hmac_fips_power_up_tests - done. (fips_error = %d) \n", fips_error);
-	}
-#endif
-#if FIPS_POWER_UP_TEST_CCM
-	if (likely(fips_error == CC_REE_FIPS_ERROR_OK)) {
-		FIPS_DBG("ssi_ccm_fips_power_up_tests ...\n");
-		fips_error = ssi_ccm_fips_power_up_tests(drvdata, cpu_addr_buffer, dma_handle);
-		FIPS_DBG("ssi_ccm_fips_power_up_tests - done. (fips_error = %d) \n", fips_error);
-	}
-#endif
-#if FIPS_POWER_UP_TEST_GCM
-	if (likely(fips_error == CC_REE_FIPS_ERROR_OK)) {
-		FIPS_DBG("ssi_gcm_fips_power_up_tests ...\n");
-		fips_error = ssi_gcm_fips_power_up_tests(drvdata, cpu_addr_buffer, dma_handle);
-		FIPS_DBG("ssi_gcm_fips_power_up_tests - done. (fips_error = %d) \n", fips_error);
-	}
-#endif
-	/* deallocate the buffer when all tests are done... */
-	FIPS_DBG("dma_free_coherent \n");
-	dma_free_coherent(dev, alloc_buff_size, cpu_addr_buffer, dma_handle);
-
-	return fips_error;
-}
-
-/* The function checks if FIPS supported and FIPS error exists.*
- * It should be used in every driver API.
- */
-int ssi_fips_check_fips_error(void)
-{
-	enum cc_fips_state_t fips_state;
-
-	if (ssi_fips_get_state(&fips_state) != 0) {
-		FIPS_LOG("ssi_fips_get_state FAILED, returning.. \n");
-		return -ENOEXEC;
-	}
-	if (fips_state == CC_FIPS_STATE_ERROR) {
-		FIPS_LOG("ssi_fips_get_state: fips_state is %d, returning.. \n", fips_state);
-		return -ENOEXEC;
-	}
-	return 0;
-}
-
-/* The function sets the REE FIPS state.*
- * It should be used while driver is being loaded.
- */
-int ssi_fips_set_state(enum cc_fips_state_t state)
-{
-	return ssi_fips_ext_set_state(state);
-}
-
-/* The function sets the REE FIPS error, and pushes the error to TEE library. *
- * It should be used when any of the KAT tests fails.
- */
-int ssi_fips_set_error(struct ssi_drvdata *p_drvdata, enum cc_fips_error err)
-{
-	int rc = 0;
-	enum cc_fips_error current_err;
-
-	FIPS_LOG("ssi_fips_set_error - fips_error = %d \n", err);
-
-	// setting no error is not allowed
-	if (err == CC_REE_FIPS_ERROR_OK)
-		return -ENOEXEC;
-
-	// If error exists, do not set new error
-	if (ssi_fips_get_error(&current_err) != 0)
-		return -ENOEXEC;
-
-	if (current_err != CC_REE_FIPS_ERROR_OK)
-		return -ENOEXEC;
-
-	// set REE internal error and state
-	rc = ssi_fips_ext_set_error(err);
-	if (rc != 0)
-		return -ENOEXEC;
-
-	rc = ssi_fips_ext_set_state(CC_FIPS_STATE_ERROR);
-	if (rc != 0)
-		return -ENOEXEC;
-
-	// push error towards TEE libraray, if it's not TEE error
-	if (err != CC_REE_FIPS_ERROR_FROM_TEE)
-		ssi_fips_update_tee_upon_ree_status(p_drvdata, err);
-
-	return rc;
-}
-
-/* The function called once at driver entry point .*/
-int ssi_fips_init(struct ssi_drvdata *p_drvdata)
-{
-	enum cc_fips_error rc = CC_REE_FIPS_ERROR_OK;
-	struct ssi_fips_handle *fips_h;
-
-	FIPS_DBG("CC FIPS code ..  (fips=%d) \n", ssi_fips_support);
-
-	fips_h = kzalloc(sizeof(struct ssi_fips_handle), GFP_KERNEL);
-	if (!fips_h) {
-		ssi_fips_set_error(p_drvdata, CC_REE_FIPS_ERROR_GENERAL);
-		return -ENOMEM;
-	}
-
-	p_drvdata->fips_handle = fips_h;
-
-#ifdef COMP_IN_WQ
-	SSI_LOG_DEBUG("Initializing fips workqueue\n");
-	fips_h->workq = create_singlethread_workqueue("arm_cc7x_fips_wq");
-	if (unlikely(!fips_h->workq)) {
-		SSI_LOG_ERR("Failed creating fips work queue\n");
-		ssi_fips_set_error(p_drvdata, CC_REE_FIPS_ERROR_GENERAL);
-		rc = -ENOMEM;
-		goto ssi_fips_init_err;
-	}
-	INIT_DELAYED_WORK(&fips_h->fipswork, fips_wq_handler);
-#else
-	SSI_LOG_DEBUG("Initializing fips tasklet\n");
-	tasklet_init(&fips_h->fipstask, fips_dsr, (unsigned long)p_drvdata);
-#endif
-
-	/* init fips driver data */
-	rc = ssi_fips_set_state((ssi_fips_support == 0) ? CC_FIPS_STATE_NOT_SUPPORTED : CC_FIPS_STATE_SUPPORTED);
-	if (unlikely(rc != 0)) {
-		ssi_fips_set_error(p_drvdata, CC_REE_FIPS_ERROR_GENERAL);
-		rc = -EAGAIN;
-		goto ssi_fips_init_err;
-	}
-
-	/* Run power up tests (before registration and operating the HW engines) */
-	FIPS_DBG("ssi_fips_get_tee_error \n");
-	rc = ssi_fips_get_tee_error(p_drvdata);
-	if (unlikely(rc != CC_REE_FIPS_ERROR_OK)) {
-		ssi_fips_set_error(p_drvdata, CC_REE_FIPS_ERROR_FROM_TEE);
-		rc = -EAGAIN;
-		goto ssi_fips_init_err;
-	}
-
-	FIPS_DBG("cc_fips_run_power_up_tests \n");
-	rc = cc_fips_run_power_up_tests(p_drvdata);
-	if (unlikely(rc != CC_REE_FIPS_ERROR_OK)) {
-		ssi_fips_set_error(p_drvdata, rc);
-		rc = -EAGAIN;
-		goto ssi_fips_init_err;
-	}
-	FIPS_LOG("cc_fips_run_power_up_tests - done  ...  fips_error = %d \n", rc);
-
-	/* when all tests passed, update TEE with fips OK status after power up tests */
-	ssi_fips_update_tee_upon_ree_status(p_drvdata, CC_REE_FIPS_ERROR_OK);
-
-	if (unlikely(rc != 0)) {
-		rc = -EAGAIN;
-		ssi_fips_set_error(p_drvdata, CC_REE_FIPS_ERROR_GENERAL);
-		goto ssi_fips_init_err;
-	}
-
-	return 0;
-
-ssi_fips_init_err:
-	ssi_fips_fini(p_drvdata);
-	return rc;
-}
-
diff --git a/drivers/staging/ccree/ssi_fips_local.h b/drivers/staging/ccree/ssi_fips_local.h
deleted file mode 100644
index 8c7994fe9fae..000000000000
--- a/drivers/staging/ccree/ssi_fips_local.h
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Copyright (C) 2012-2017 ARM Limited or its affiliates.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef __SSI_FIPS_LOCAL_H__
-#define __SSI_FIPS_LOCAL_H__
-
-#ifdef CONFIG_CCX7REE_FIPS_SUPPORT
-
-#include "ssi_fips.h"
-struct ssi_drvdata;
-
-#define CHECK_AND_RETURN_UPON_FIPS_ERROR() {\
-	if (ssi_fips_check_fips_error() != 0) {\
-		return -ENOEXEC;\
-	} \
-}
-
-#define CHECK_AND_RETURN_VOID_UPON_FIPS_ERROR() {\
-	if (ssi_fips_check_fips_error() != 0) {\
-		return;\
-	} \
-}
-
-#define SSI_FIPS_INIT(p_drvData)  (ssi_fips_init(p_drvData))
-#define SSI_FIPS_FINI(p_drvData)  (ssi_fips_fini(p_drvData))
-
-#define FIPS_LOG(...)	SSI_LOG(KERN_INFO, __VA_ARGS__)
-#define FIPS_DBG(...)	//SSI_LOG(KERN_INFO, __VA_ARGS__)
-
-/* FIPS functions */
-int ssi_fips_init(struct ssi_drvdata *p_drvdata);
-void ssi_fips_fini(struct ssi_drvdata *drvdata);
-int ssi_fips_check_fips_error(void);
-int ssi_fips_set_error(struct ssi_drvdata *p_drvdata, enum cc_fips_error err);
-void fips_handler(struct ssi_drvdata *drvdata);
-
-#else  /* CONFIG_CC7XXREE_FIPS_SUPPORT */
-
-#define CHECK_AND_RETURN_UPON_FIPS_ERROR()
-#define CHECK_AND_RETURN_VOID_UPON_FIPS_ERROR()
-
-static inline int ssi_fips_init(struct ssi_drvdata *p_drvdata)
-{
-	return 0;
-}
-
-static inline void ssi_fips_fini(struct ssi_drvdata *drvdata) {}
-
-void fips_handler(struct ssi_drvdata *drvdata);
-
-#endif  /* CONFIG_CC7XXREE_FIPS_SUPPORT */
-
-#endif  /*__SSI_FIPS_LOCAL_H__*/
-
diff --git a/drivers/staging/ccree/ssi_hash.c b/drivers/staging/ccree/ssi_hash.c
index ae8f36af3837..13291aeaf350 100644
--- a/drivers/staging/ccree/ssi_hash.c
+++ b/drivers/staging/ccree/ssi_hash.c
@@ -30,7 +30,6 @@
 #include "ssi_sysfs.h"
 #include "ssi_hash.h"
 #include "ssi_sram_mgr.h"
-#include "ssi_fips_local.h"
 
 #define SSI_MAX_AHASH_SEQ_LEN 12
 #define SSI_MAX_HASH_OPAD_TMP_KEYS_SIZE MAX(SSI_MAX_HASH_BLCK_SIZE, 3 * AES_BLOCK_SIZE)
@@ -71,8 +70,8 @@ static void ssi_hash_create_xcbc_setup(
 	unsigned int *seq_size);
 
 static void ssi_hash_create_cmac_setup(struct ahash_request *areq,
-				  struct cc_hw_desc desc[],
-				  unsigned int *seq_size);
+				       struct cc_hw_desc desc[],
+				       unsigned int *seq_size);
 
 struct ssi_hash_alg {
 	struct list_head entry;
@@ -118,8 +117,8 @@ static void ssi_hash_create_data_desc(
 static inline void ssi_set_hash_endianity(u32 mode, struct cc_hw_desc *desc)
 {
 	if (unlikely((mode == DRV_HASH_MD5) ||
-		(mode == DRV_HASH_SHA384) ||
-		(mode == DRV_HASH_SHA512))) {
+		     (mode == DRV_HASH_SHA384) ||
+		     (mode == DRV_HASH_SHA512))) {
 		set_bytes_swap(desc, 1);
 	} else {
 		set_cipher_config0(desc, HASH_DIGEST_RESULT_LITTLE_ENDIAN);
@@ -136,13 +135,13 @@ static int ssi_hash_map_result(struct device *dev,
 			       DMA_BIDIRECTIONAL);
 	if (unlikely(dma_mapping_error(dev, state->digest_result_dma_addr))) {
 		SSI_LOG_ERR("Mapping digest result buffer %u B for DMA failed\n",
-			digestsize);
+			    digestsize);
 		return -ENOMEM;
 	}
 	SSI_LOG_DEBUG("Mapped digest result buffer %u B "
-		     "at va=%pK to dma=0x%llX\n",
+		     "at va=%pK to dma=%pad\n",
 		digestsize, state->digest_result_buff,
-		(unsigned long long)state->digest_result_dma_addr);
+		state->digest_result_dma_addr);
 
 	return 0;
 }
@@ -201,12 +200,12 @@ static int ssi_hash_map_request(struct device *dev,
 	state->digest_buff_dma_addr = dma_map_single(dev, (void *)state->digest_buff, ctx->inter_digestsize, DMA_BIDIRECTIONAL);
 	if (dma_mapping_error(dev, state->digest_buff_dma_addr)) {
 		SSI_LOG_ERR("Mapping digest len %d B at va=%pK for DMA failed\n",
-		ctx->inter_digestsize, state->digest_buff);
+			    ctx->inter_digestsize, state->digest_buff);
 		goto fail3;
 	}
-	SSI_LOG_DEBUG("Mapped digest %d B at va=%pK to dma=0x%llX\n",
-		ctx->inter_digestsize, state->digest_buff,
-		(unsigned long long)state->digest_buff_dma_addr);
+	SSI_LOG_DEBUG("Mapped digest %d B at va=%pK to dma=%pad\n",
+		      ctx->inter_digestsize, state->digest_buff,
+		      state->digest_buff_dma_addr);
 
 	if (is_hmac) {
 		dma_sync_single_for_cpu(dev, ctx->digest_buff_dma_addr, ctx->inter_digestsize, DMA_BIDIRECTIONAL);
@@ -250,12 +249,12 @@ static int ssi_hash_map_request(struct device *dev,
 		state->digest_bytes_len_dma_addr = dma_map_single(dev, (void *)state->digest_bytes_len, HASH_LEN_SIZE, DMA_BIDIRECTIONAL);
 		if (dma_mapping_error(dev, state->digest_bytes_len_dma_addr)) {
 			SSI_LOG_ERR("Mapping digest len %u B at va=%pK for DMA failed\n",
-			HASH_LEN_SIZE, state->digest_bytes_len);
+				    HASH_LEN_SIZE, state->digest_bytes_len);
 			goto fail4;
 		}
-		SSI_LOG_DEBUG("Mapped digest len %u B at va=%pK to dma=0x%llX\n",
-			HASH_LEN_SIZE, state->digest_bytes_len,
-			(unsigned long long)state->digest_bytes_len_dma_addr);
+		SSI_LOG_DEBUG("Mapped digest len %u B at va=%pK to dma=%pad\n",
+			      HASH_LEN_SIZE, state->digest_bytes_len,
+			      state->digest_bytes_len_dma_addr);
 	} else {
 		state->digest_bytes_len_dma_addr = 0;
 	}
@@ -264,12 +263,13 @@ static int ssi_hash_map_request(struct device *dev,
 		state->opad_digest_dma_addr = dma_map_single(dev, (void *)state->opad_digest_buff, ctx->inter_digestsize, DMA_BIDIRECTIONAL);
 		if (dma_mapping_error(dev, state->opad_digest_dma_addr)) {
 			SSI_LOG_ERR("Mapping opad digest %d B at va=%pK for DMA failed\n",
-			ctx->inter_digestsize, state->opad_digest_buff);
+				    ctx->inter_digestsize,
+				    state->opad_digest_buff);
 			goto fail5;
 		}
-		SSI_LOG_DEBUG("Mapped opad digest %d B at va=%pK to dma=0x%llX\n",
-			ctx->inter_digestsize, state->opad_digest_buff,
-			(unsigned long long)state->opad_digest_dma_addr);
+		SSI_LOG_DEBUG("Mapped opad digest %d B at va=%pK to dma=%pad\n",
+			      ctx->inter_digestsize, state->opad_digest_buff,
+			      state->opad_digest_dma_addr);
 	} else {
 		state->opad_digest_dma_addr = 0;
 	}
@@ -297,20 +297,14 @@ fail2:
 fail1:
 	 kfree(state->digest_buff);
 fail_digest_result_buff:
-	 if (state->digest_result_buff) {
-		 kfree(state->digest_result_buff);
-	     state->digest_result_buff = NULL;
-	 }
+	kfree(state->digest_result_buff);
+	state->digest_result_buff = NULL;
 fail_buff1:
-	 if (state->buff1) {
-		 kfree(state->buff1);
-	     state->buff1 = NULL;
-	 }
+	kfree(state->buff1);
+	state->buff1 = NULL;
 fail_buff0:
-	 if (state->buff0) {
-		 kfree(state->buff0);
-	     state->buff0 = NULL;
-	 }
+	kfree(state->buff0);
+	state->buff0 = NULL;
 fail0:
 	return rc;
 }
@@ -322,22 +316,22 @@ static void ssi_hash_unmap_request(struct device *dev,
 	if (state->digest_buff_dma_addr != 0) {
 		dma_unmap_single(dev, state->digest_buff_dma_addr,
 				 ctx->inter_digestsize, DMA_BIDIRECTIONAL);
-		SSI_LOG_DEBUG("Unmapped digest-buffer: digest_buff_dma_addr=0x%llX\n",
-			(unsigned long long)state->digest_buff_dma_addr);
+		SSI_LOG_DEBUG("Unmapped digest-buffer: digest_buff_dma_addr=%pad\n",
+			      state->digest_buff_dma_addr);
 		state->digest_buff_dma_addr = 0;
 	}
 	if (state->digest_bytes_len_dma_addr != 0) {
 		dma_unmap_single(dev, state->digest_bytes_len_dma_addr,
 				 HASH_LEN_SIZE, DMA_BIDIRECTIONAL);
-		SSI_LOG_DEBUG("Unmapped digest-bytes-len buffer: digest_bytes_len_dma_addr=0x%llX\n",
-			(unsigned long long)state->digest_bytes_len_dma_addr);
+		SSI_LOG_DEBUG("Unmapped digest-bytes-len buffer: digest_bytes_len_dma_addr=%pad\n",
+			      state->digest_bytes_len_dma_addr);
 		state->digest_bytes_len_dma_addr = 0;
 	}
 	if (state->opad_digest_dma_addr != 0) {
 		dma_unmap_single(dev, state->opad_digest_dma_addr,
 				 ctx->inter_digestsize, DMA_BIDIRECTIONAL);
-		SSI_LOG_DEBUG("Unmapped opad-digest: opad_digest_dma_addr=0x%llX\n",
-			(unsigned long long)state->opad_digest_dma_addr);
+		SSI_LOG_DEBUG("Unmapped opad-digest: opad_digest_dma_addr=%pad\n",
+			      state->opad_digest_dma_addr);
 		state->opad_digest_dma_addr = 0;
 	}
 
@@ -359,9 +353,9 @@ static void ssi_hash_unmap_result(struct device *dev,
 				 digestsize,
 				  DMA_BIDIRECTIONAL);
 		SSI_LOG_DEBUG("unmpa digest result buffer "
-			     "va (%pK) pa (%llx) len %u\n",
+			     "va (%pK) pa (%pad) len %u\n",
 			     state->digest_result_buff,
-			     (unsigned long long)state->digest_result_dma_addr,
+			     state->digest_result_dma_addr,
 			     digestsize);
 		memcpy(result,
 		       state->digest_result_buff,
@@ -431,8 +425,6 @@ static int ssi_hash_digest(struct ahash_req_ctx *state,
 
 	SSI_LOG_DEBUG("===== %s-digest (%d) ====\n", is_hmac ? "hmac" : "hash", nbytes);
 
-	CHECK_AND_RETURN_UPON_FIPS_ERROR();
-
 	if (unlikely(ssi_hash_map_request(dev, state, ctx) != 0)) {
 		SSI_LOG_ERR("map_ahash_source() failed\n");
 		return -ENOMEM;
@@ -596,16 +588,16 @@ static int ssi_hash_update(struct ahash_req_ctx *state,
 	SSI_LOG_DEBUG("===== %s-update (%d) ====\n", ctx->is_hmac ?
 					"hmac" : "hash", nbytes);
 
-	CHECK_AND_RETURN_UPON_FIPS_ERROR();
 	if (nbytes == 0) {
 		/* no real updates required */
 		return 0;
 	}
 
-	if (unlikely(rc = ssi_buffer_mgr_map_hash_request_update(ctx->drvdata, state, src, nbytes, block_size))) {
+	rc = ssi_buffer_mgr_map_hash_request_update(ctx->drvdata, state, src, nbytes, block_size);
+	if (unlikely(rc)) {
 		if (rc == 1) {
 			SSI_LOG_DEBUG(" data size not require HW update %x\n",
-				     nbytes);
+				      nbytes);
 			/* No hardware updates are required */
 			return 0;
 		}
@@ -693,8 +685,6 @@ static int ssi_hash_finup(struct ahash_req_ctx *state,
 
 	SSI_LOG_DEBUG("===== %s-finup (%d) ====\n", is_hmac ? "hmac" : "hash", nbytes);
 
-	CHECK_AND_RETURN_UPON_FIPS_ERROR();
-
 	if (unlikely(ssi_buffer_mgr_map_hash_request_final(ctx->drvdata, state, src, nbytes, 1) != 0)) {
 		SSI_LOG_ERR("map_ahash_request_final() failed\n");
 		return -ENOMEM;
@@ -829,8 +819,6 @@ static int ssi_hash_final(struct ahash_req_ctx *state,
 
 	SSI_LOG_DEBUG("===== %s-final (%d) ====\n", is_hmac ? "hmac" : "hash", nbytes);
 
-	CHECK_AND_RETURN_UPON_FIPS_ERROR();
-
 	if (unlikely(ssi_buffer_mgr_map_hash_request_final(ctx->drvdata, state, src, nbytes, 0) != 0)) {
 		SSI_LOG_ERR("map_ahash_request_final() failed\n");
 		return -ENOMEM;
@@ -964,7 +952,6 @@ static int ssi_hash_init(struct ahash_req_ctx *state, struct ssi_hash_ctx *ctx)
 
 	state->xcbc_count = 0;
 
-	CHECK_AND_RETURN_UPON_FIPS_ERROR();
 	ssi_hash_map_request(dev, state, ctx);
 
 	return 0;
@@ -975,7 +962,7 @@ static int ssi_hash_setkey(void *hash,
 			   unsigned int keylen,
 			   bool synchronize)
 {
-	unsigned int hmacPadConst[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST };
+	unsigned int hmac_pad_const[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST };
 	struct ssi_crypto_req ssi_req = {};
 	struct ssi_hash_ctx *ctx = NULL;
 	int blocksize = 0;
@@ -984,9 +971,8 @@ static int ssi_hash_setkey(void *hash,
 	struct cc_hw_desc desc[SSI_MAX_AHASH_SEQ_LEN];
 	ssi_sram_addr_t larval_addr;
 
-	 SSI_LOG_DEBUG("ssi_hash_setkey: start keylen: %d", keylen);
+	 SSI_LOG_DEBUG("start keylen: %d", keylen);
 
-	CHECK_AND_RETURN_UPON_FIPS_ERROR();
 	ctx = crypto_ahash_ctx(((struct crypto_ahash *)hash));
 	blocksize = crypto_tfm_alg_blocksize(&((struct crypto_ahash *)hash)->base);
 	digestsize = crypto_ahash_digestsize(((struct crypto_ahash *)hash));
@@ -1012,9 +998,8 @@ static int ssi_hash_setkey(void *hash,
 				   " DMA failed\n", key, keylen);
 			return -ENOMEM;
 		}
-		SSI_LOG_DEBUG("mapping key-buffer: key_dma_addr=0x%llX "
-			     "keylen=%u\n",
-			     (unsigned long long)ctx->key_params.key_dma_addr,
+		SSI_LOG_DEBUG("mapping key-buffer: key_dma_addr=%pad "
+			     "keylen=%u\n", ctx->key_params.key_dma_addr,
 			     ctx->key_params.keylen);
 
 		if (keylen > blocksize) {
@@ -1118,7 +1103,7 @@ static int ssi_hash_setkey(void *hash,
 
 		/* Prepare ipad key */
 		hw_desc_init(&desc[idx]);
-		set_xor_val(&desc[idx], hmacPadConst[i]);
+		set_xor_val(&desc[idx], hmac_pad_const[i]);
 		set_cipher_mode(&desc[idx], ctx->hw_mode);
 		set_flow_mode(&desc[idx], S_DIN_to_HASH);
 		set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
@@ -1155,17 +1140,17 @@ out:
 
 	if (ctx->key_params.key_dma_addr) {
 		dma_unmap_single(&ctx->drvdata->plat_dev->dev,
-				ctx->key_params.key_dma_addr,
-				ctx->key_params.keylen, DMA_TO_DEVICE);
-		SSI_LOG_DEBUG("Unmapped key-buffer: key_dma_addr=0x%llX keylen=%u\n",
-				(unsigned long long)ctx->key_params.key_dma_addr,
-				ctx->key_params.keylen);
+				 ctx->key_params.key_dma_addr,
+				 ctx->key_params.keylen, DMA_TO_DEVICE);
+		SSI_LOG_DEBUG("Unmapped key-buffer: key_dma_addr=%pad keylen=%u\n",
+			      ctx->key_params.key_dma_addr,
+			      ctx->key_params.keylen);
 	}
 	return rc;
 }
 
 static int ssi_xcbc_setkey(struct crypto_ahash *ahash,
-			const u8 *key, unsigned int keylen)
+			   const u8 *key, unsigned int keylen)
 {
 	struct ssi_crypto_req ssi_req = {};
 	struct ssi_hash_ctx *ctx = crypto_ahash_ctx(ahash);
@@ -1173,15 +1158,14 @@ static int ssi_xcbc_setkey(struct crypto_ahash *ahash,
 	struct cc_hw_desc desc[SSI_MAX_AHASH_SEQ_LEN];
 
 	SSI_LOG_DEBUG("===== setkey (%d) ====\n", keylen);
-	CHECK_AND_RETURN_UPON_FIPS_ERROR();
 
 	switch (keylen) {
-		case AES_KEYSIZE_128:
-		case AES_KEYSIZE_192:
-		case AES_KEYSIZE_256:
-			break;
-		default:
-			return -EINVAL;
+	case AES_KEYSIZE_128:
+	case AES_KEYSIZE_192:
+	case AES_KEYSIZE_256:
+		break;
+	default:
+		return -EINVAL;
 	}
 
 	ctx->key_params.keylen = keylen;
@@ -1196,9 +1180,9 @@ static int ssi_xcbc_setkey(struct crypto_ahash *ahash,
 			   " DMA failed\n", key, keylen);
 		return -ENOMEM;
 	}
-	SSI_LOG_DEBUG("mapping key-buffer: key_dma_addr=0x%llX "
+	SSI_LOG_DEBUG("mapping key-buffer: key_dma_addr=%pad "
 		     "keylen=%u\n",
-		     (unsigned long long)ctx->key_params.key_dma_addr,
+		     ctx->key_params.key_dma_addr,
 		     ctx->key_params.keylen);
 
 	ctx->is_hmac = true;
@@ -1243,33 +1227,32 @@ static int ssi_xcbc_setkey(struct crypto_ahash *ahash,
 		crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN);
 
 	dma_unmap_single(&ctx->drvdata->plat_dev->dev,
-			ctx->key_params.key_dma_addr,
-			ctx->key_params.keylen, DMA_TO_DEVICE);
-	SSI_LOG_DEBUG("Unmapped key-buffer: key_dma_addr=0x%llX keylen=%u\n",
-			(unsigned long long)ctx->key_params.key_dma_addr,
-			ctx->key_params.keylen);
+			 ctx->key_params.key_dma_addr,
+			 ctx->key_params.keylen, DMA_TO_DEVICE);
+	SSI_LOG_DEBUG("Unmapped key-buffer: key_dma_addr=%pad keylen=%u\n",
+		      ctx->key_params.key_dma_addr,
+		      ctx->key_params.keylen);
 
 	return rc;
 }
 
 #if SSI_CC_HAS_CMAC
 static int ssi_cmac_setkey(struct crypto_ahash *ahash,
-			const u8 *key, unsigned int keylen)
+			   const u8 *key, unsigned int keylen)
 {
 	struct ssi_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 
 	SSI_LOG_DEBUG("===== setkey (%d) ====\n", keylen);
-	CHECK_AND_RETURN_UPON_FIPS_ERROR();
 
 	ctx->is_hmac = true;
 
 	switch (keylen) {
-		case AES_KEYSIZE_128:
-		case AES_KEYSIZE_192:
-		case AES_KEYSIZE_256:
-			break;
-		default:
-			return -EINVAL;
+	case AES_KEYSIZE_128:
+	case AES_KEYSIZE_192:
+	case AES_KEYSIZE_256:
+		break;
+	default:
+		return -EINVAL;
 	}
 
 	ctx->key_params.keylen = keylen;
@@ -1302,8 +1285,8 @@ static void ssi_hash_free_ctx(struct ssi_hash_ctx *ctx)
 		dma_unmap_single(dev, ctx->digest_buff_dma_addr,
 				 sizeof(ctx->digest_buff), DMA_BIDIRECTIONAL);
 		SSI_LOG_DEBUG("Unmapped digest-buffer: "
-			     "digest_buff_dma_addr=0x%llX\n",
-			(unsigned long long)ctx->digest_buff_dma_addr);
+			     "digest_buff_dma_addr=%pad\n",
+			      ctx->digest_buff_dma_addr);
 		ctx->digest_buff_dma_addr = 0;
 	}
 	if (ctx->opad_tmp_keys_dma_addr != 0) {
@@ -1311,8 +1294,8 @@ static void ssi_hash_free_ctx(struct ssi_hash_ctx *ctx)
 				 sizeof(ctx->opad_tmp_keys_buff),
 				 DMA_BIDIRECTIONAL);
 		SSI_LOG_DEBUG("Unmapped opad-digest: "
-			     "opad_tmp_keys_dma_addr=0x%llX\n",
-			(unsigned long long)ctx->opad_tmp_keys_dma_addr);
+			     "opad_tmp_keys_dma_addr=%pad\n",
+			      ctx->opad_tmp_keys_dma_addr);
 		ctx->opad_tmp_keys_dma_addr = 0;
 	}
 
@@ -1328,23 +1311,23 @@ static int ssi_hash_alloc_ctx(struct ssi_hash_ctx *ctx)
 	ctx->digest_buff_dma_addr = dma_map_single(dev, (void *)ctx->digest_buff, sizeof(ctx->digest_buff), DMA_BIDIRECTIONAL);
 	if (dma_mapping_error(dev, ctx->digest_buff_dma_addr)) {
 		SSI_LOG_ERR("Mapping digest len %zu B at va=%pK for DMA failed\n",
-			sizeof(ctx->digest_buff), ctx->digest_buff);
+			    sizeof(ctx->digest_buff), ctx->digest_buff);
 		goto fail;
 	}
-	SSI_LOG_DEBUG("Mapped digest %zu B at va=%pK to dma=0x%llX\n",
-		sizeof(ctx->digest_buff), ctx->digest_buff,
-		(unsigned long long)ctx->digest_buff_dma_addr);
+	SSI_LOG_DEBUG("Mapped digest %zu B at va=%pK to dma=%pad\n",
+		      sizeof(ctx->digest_buff), ctx->digest_buff,
+		      ctx->digest_buff_dma_addr);
 
 	ctx->opad_tmp_keys_dma_addr = dma_map_single(dev, (void *)ctx->opad_tmp_keys_buff, sizeof(ctx->opad_tmp_keys_buff), DMA_BIDIRECTIONAL);
 	if (dma_mapping_error(dev, ctx->opad_tmp_keys_dma_addr)) {
 		SSI_LOG_ERR("Mapping opad digest %zu B at va=%pK for DMA failed\n",
-			sizeof(ctx->opad_tmp_keys_buff),
-			ctx->opad_tmp_keys_buff);
+			    sizeof(ctx->opad_tmp_keys_buff),
+			    ctx->opad_tmp_keys_buff);
 		goto fail;
 	}
-	SSI_LOG_DEBUG("Mapped opad_tmp_keys %zu B at va=%pK to dma=0x%llX\n",
-		sizeof(ctx->opad_tmp_keys_buff), ctx->opad_tmp_keys_buff,
-		(unsigned long long)ctx->opad_tmp_keys_dma_addr);
+	SSI_LOG_DEBUG("Mapped opad_tmp_keys %zu B at va=%pK to dma=%pad\n",
+		      sizeof(ctx->opad_tmp_keys_buff), ctx->opad_tmp_keys_buff,
+		      ctx->opad_tmp_keys_dma_addr);
 
 	ctx->is_hmac = false;
 	return 0;
@@ -1364,9 +1347,8 @@ static int ssi_ahash_cra_init(struct crypto_tfm *tfm)
 	struct ssi_hash_alg *ssi_alg =
 			container_of(ahash_alg, struct ssi_hash_alg, ahash_alg);
 
-	CHECK_AND_RETURN_UPON_FIPS_ERROR();
 	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
-				sizeof(struct ahash_req_ctx));
+				 sizeof(struct ahash_req_ctx));
 
 	ctx->hash_mode = ssi_alg->hash_mode;
 	ctx->hw_mode = ssi_alg->hw_mode;
@@ -1396,7 +1378,6 @@ static int ssi_mac_update(struct ahash_request *req)
 	int rc;
 	u32 idx = 0;
 
-	CHECK_AND_RETURN_UPON_FIPS_ERROR();
 	if (req->nbytes == 0) {
 		/* no real updates required */
 		return 0;
@@ -1404,10 +1385,11 @@ static int ssi_mac_update(struct ahash_request *req)
 
 	state->xcbc_count++;
 
-	if (unlikely(rc = ssi_buffer_mgr_map_hash_request_update(ctx->drvdata, state, req->src, req->nbytes, block_size))) {
+	rc = ssi_buffer_mgr_map_hash_request_update(ctx->drvdata, state, req->src, req->nbytes, block_size);
+	if (unlikely(rc)) {
 		if (rc == 1) {
 			SSI_LOG_DEBUG(" data size not require HW update %x\n",
-				     req->nbytes);
+				      req->nbytes);
 			/* No hardware updates are required */
 			return 0;
 		}
@@ -1454,19 +1436,19 @@ static int ssi_mac_final(struct ahash_request *req)
 	struct cc_hw_desc desc[SSI_MAX_AHASH_SEQ_LEN];
 	int idx = 0;
 	int rc = 0;
-	u32 keySize, keyLen;
+	u32 key_size, key_len;
 	u32 digestsize = crypto_ahash_digestsize(tfm);
 
 	u32 rem_cnt = state->buff_index ? state->buff1_cnt :
 			state->buff0_cnt;
 
-	CHECK_AND_RETURN_UPON_FIPS_ERROR();
 	if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) {
-		keySize = CC_AES_128_BIT_KEY_SIZE;
-		keyLen  = CC_AES_128_BIT_KEY_SIZE;
+		key_size = CC_AES_128_BIT_KEY_SIZE;
+		key_len  = CC_AES_128_BIT_KEY_SIZE;
 	} else {
-		keySize = (ctx->key_params.keylen == 24) ? AES_MAX_KEY_SIZE : ctx->key_params.keylen;
-		keyLen =  ctx->key_params.keylen;
+		key_size = (ctx->key_params.keylen == 24) ? AES_MAX_KEY_SIZE :
+			ctx->key_params.keylen;
+		key_len =  ctx->key_params.keylen;
 	}
 
 	SSI_LOG_DEBUG("===== final  xcbc reminder (%d) ====\n", rem_cnt);
@@ -1492,8 +1474,8 @@ static int ssi_mac_final(struct ahash_request *req)
 		set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_DECRYPT);
 		set_din_type(&desc[idx], DMA_DLLI,
 			     (ctx->opad_tmp_keys_dma_addr +
-			      XCBC_MAC_K1_OFFSET), keySize, NS_BIT);
-		set_key_size_aes(&desc[idx], keyLen);
+			      XCBC_MAC_K1_OFFSET), key_size, NS_BIT);
+		set_key_size_aes(&desc[idx], key_len);
 		set_flow_mode(&desc[idx], S_DIN_to_AES);
 		set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
 		idx++;
@@ -1522,7 +1504,7 @@ static int ssi_mac_final(struct ahash_request *req)
 	if (state->xcbc_count == 0) {
 		hw_desc_init(&desc[idx]);
 		set_cipher_mode(&desc[idx], ctx->hw_mode);
-		set_key_size_aes(&desc[idx], keyLen);
+		set_key_size_aes(&desc[idx], key_len);
 		set_cmac_size0_mode(&desc[idx]);
 		set_flow_mode(&desc[idx], S_DIN_to_AES);
 		idx++;
@@ -1569,9 +1551,8 @@ static int ssi_mac_finup(struct ahash_request *req)
 	u32 digestsize = crypto_ahash_digestsize(tfm);
 
 	SSI_LOG_DEBUG("===== finup xcbc(%d) ====\n", req->nbytes);
-	CHECK_AND_RETURN_UPON_FIPS_ERROR();
 	if (state->xcbc_count > 0 && req->nbytes == 0) {
-		SSI_LOG_DEBUG("No data to update. Call to fdx_mac_final \n");
+		SSI_LOG_DEBUG("No data to update. Call to fdx_mac_final\n");
 		return ssi_mac_final(req);
 	}
 
@@ -1636,12 +1617,11 @@ static int ssi_mac_digest(struct ahash_request *req)
 	u32 digestsize = crypto_ahash_digestsize(tfm);
 	struct ssi_crypto_req ssi_req = {};
 	struct cc_hw_desc desc[SSI_MAX_AHASH_SEQ_LEN];
-	u32 keyLen;
+	u32 key_len;
 	int idx = 0;
 	int rc;
 
 	SSI_LOG_DEBUG("===== -digest mac (%d) ====\n",  req->nbytes);
-	CHECK_AND_RETURN_UPON_FIPS_ERROR();
 
 	if (unlikely(ssi_hash_map_request(dev, state, ctx) != 0)) {
 		SSI_LOG_ERR("map_ahash_source() failed\n");
@@ -1662,17 +1642,17 @@ static int ssi_mac_digest(struct ahash_request *req)
 	ssi_req.user_arg = (void *)req;
 
 	if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) {
-		keyLen = CC_AES_128_BIT_KEY_SIZE;
+		key_len = CC_AES_128_BIT_KEY_SIZE;
 		ssi_hash_create_xcbc_setup(req, desc, &idx);
 	} else {
-		keyLen = ctx->key_params.keylen;
+		key_len = ctx->key_params.keylen;
 		ssi_hash_create_cmac_setup(req, desc, &idx);
 	}
 
 	if (req->nbytes == 0) {
 		hw_desc_init(&desc[idx]);
 		set_cipher_mode(&desc[idx], ctx->hw_mode);
-		set_key_size_aes(&desc[idx], keyLen);
+		set_key_size_aes(&desc[idx], key_len);
 		set_cmac_size0_mode(&desc[idx]);
 		set_flow_mode(&desc[idx], S_DIN_to_AES);
 		idx++;
@@ -1764,8 +1744,6 @@ static int ssi_ahash_export(struct ahash_request *req, void *out)
 				state->buff0_cnt;
 	const u32 tmp = CC_EXPORT_MAGIC;
 
-	CHECK_AND_RETURN_UPON_FIPS_ERROR();
-
 	memcpy(out, &tmp, sizeof(u32));
 	out += sizeof(u32);
 
@@ -1805,8 +1783,6 @@ static int ssi_ahash_import(struct ahash_request *req, const void *in)
 	u32 tmp;
 	int rc;
 
-	CHECK_AND_RETURN_UPON_FIPS_ERROR();
-
 	memcpy(&tmp, in, sizeof(u32));
 	if (tmp != CC_EXPORT_MAGIC) {
 		rc = -EINVAL;
@@ -1856,7 +1832,7 @@ out:
 }
 
 static int ssi_ahash_setkey(struct crypto_ahash *ahash,
-			const u8 *key, unsigned int keylen)
+			    const u8 *key, unsigned int keylen)
 {
 	return ssi_hash_setkey((void *)ahash, key, keylen, false);
 }
@@ -2084,9 +2060,9 @@ ssi_hash_create_alg(struct ssi_hash_template *template, bool keyed)
 	struct crypto_alg *alg;
 	struct ahash_alg *halg;
 
-	t_crypto_alg = kzalloc(sizeof(struct ssi_hash_alg), GFP_KERNEL);
+	t_crypto_alg = kzalloc(sizeof(*t_crypto_alg), GFP_KERNEL);
 	if (!t_crypto_alg) {
-		SSI_LOG_ERR("failed to allocate t_alg\n");
+		SSI_LOG_ERR("failed to allocate t_crypto_alg\n");
 		return ERR_PTR(-ENOMEM);
 	}
 
@@ -2138,7 +2114,8 @@ int ssi_hash_init_sram_digest_consts(struct ssi_drvdata *drvdata)
 
 	/* Copy-to-sram digest-len */
 	ssi_sram_mgr_const2sram_desc(digest_len_init, sram_buff_ofs,
-		ARRAY_SIZE(digest_len_init), larval_seq, &larval_seq_len);
+				     ARRAY_SIZE(digest_len_init),
+				     larval_seq, &larval_seq_len);
 	rc = send_request_init(drvdata, larval_seq, larval_seq_len);
 	if (unlikely(rc != 0))
 		goto init_digest_const_err;
@@ -2149,7 +2126,8 @@ int ssi_hash_init_sram_digest_consts(struct ssi_drvdata *drvdata)
 #if (DX_DEV_SHA_MAX > 256)
 	/* Copy-to-sram digest-len for sha384/512 */
 	ssi_sram_mgr_const2sram_desc(digest_len_sha512_init, sram_buff_ofs,
-		ARRAY_SIZE(digest_len_sha512_init), larval_seq, &larval_seq_len);
+				     ARRAY_SIZE(digest_len_sha512_init),
+				     larval_seq, &larval_seq_len);
 	rc = send_request_init(drvdata, larval_seq, larval_seq_len);
 	if (unlikely(rc != 0))
 		goto init_digest_const_err;
@@ -2163,7 +2141,8 @@ int ssi_hash_init_sram_digest_consts(struct ssi_drvdata *drvdata)
 
 	/* Copy-to-sram initial SHA* digests */
 	ssi_sram_mgr_const2sram_desc(md5_init, sram_buff_ofs,
-		ARRAY_SIZE(md5_init), larval_seq, &larval_seq_len);
+				     ARRAY_SIZE(md5_init), larval_seq,
+				     &larval_seq_len);
 	rc = send_request_init(drvdata, larval_seq, larval_seq_len);
 	if (unlikely(rc != 0))
 		goto init_digest_const_err;
@@ -2171,7 +2150,8 @@ int ssi_hash_init_sram_digest_consts(struct ssi_drvdata *drvdata)
 	larval_seq_len = 0;
 
 	ssi_sram_mgr_const2sram_desc(sha1_init, sram_buff_ofs,
-		ARRAY_SIZE(sha1_init), larval_seq, &larval_seq_len);
+				     ARRAY_SIZE(sha1_init), larval_seq,
+				     &larval_seq_len);
 	rc = send_request_init(drvdata, larval_seq, larval_seq_len);
 	if (unlikely(rc != 0))
 		goto init_digest_const_err;
@@ -2179,7 +2159,8 @@ int ssi_hash_init_sram_digest_consts(struct ssi_drvdata *drvdata)
 	larval_seq_len = 0;
 
 	ssi_sram_mgr_const2sram_desc(sha224_init, sram_buff_ofs,
-		ARRAY_SIZE(sha224_init), larval_seq, &larval_seq_len);
+				     ARRAY_SIZE(sha224_init), larval_seq,
+				     &larval_seq_len);
 	rc = send_request_init(drvdata, larval_seq, larval_seq_len);
 	if (unlikely(rc != 0))
 		goto init_digest_const_err;
@@ -2187,7 +2168,8 @@ int ssi_hash_init_sram_digest_consts(struct ssi_drvdata *drvdata)
 	larval_seq_len = 0;
 
 	ssi_sram_mgr_const2sram_desc(sha256_init, sram_buff_ofs,
-		ARRAY_SIZE(sha256_init), larval_seq, &larval_seq_len);
+				     ARRAY_SIZE(sha256_init), larval_seq,
+				     &larval_seq_len);
 	rc = send_request_init(drvdata, larval_seq, larval_seq_len);
 	if (unlikely(rc != 0))
 		goto init_digest_const_err;
@@ -2201,10 +2183,10 @@ int ssi_hash_init_sram_digest_consts(struct ssi_drvdata *drvdata)
 		const u32 const1 = ((u32 *)((u64 *)&sha384_init[i]))[0];
 
 		ssi_sram_mgr_const2sram_desc(&const0, sram_buff_ofs, 1,
-			larval_seq, &larval_seq_len);
+					     larval_seq, &larval_seq_len);
 		sram_buff_ofs += sizeof(u32);
 		ssi_sram_mgr_const2sram_desc(&const1, sram_buff_ofs, 1,
-			larval_seq, &larval_seq_len);
+					     larval_seq, &larval_seq_len);
 		sram_buff_ofs += sizeof(u32);
 	}
 	rc = send_request_init(drvdata, larval_seq, larval_seq_len);
@@ -2219,10 +2201,10 @@ int ssi_hash_init_sram_digest_consts(struct ssi_drvdata *drvdata)
 		const u32 const1 = ((u32 *)((u64 *)&sha512_init[i]))[0];
 
 		ssi_sram_mgr_const2sram_desc(&const0, sram_buff_ofs, 1,
-			larval_seq, &larval_seq_len);
+					     larval_seq, &larval_seq_len);
 		sram_buff_ofs += sizeof(u32);
 		ssi_sram_mgr_const2sram_desc(&const1, sram_buff_ofs, 1,
-			larval_seq, &larval_seq_len);
+					     larval_seq, &larval_seq_len);
 		sram_buff_ofs += sizeof(u32);
 	}
 	rc = send_request_init(drvdata, larval_seq, larval_seq_len);
@@ -2244,10 +2226,10 @@ int ssi_hash_alloc(struct ssi_drvdata *drvdata)
 	int rc = 0;
 	int alg;
 
-	hash_handle = kzalloc(sizeof(struct ssi_hash_handle), GFP_KERNEL);
+	hash_handle = kzalloc(sizeof(*hash_handle), GFP_KERNEL);
 	if (!hash_handle) {
 		SSI_LOG_ERR("kzalloc failed to allocate %zu B\n",
-			sizeof(struct ssi_hash_handle));
+			    sizeof(*hash_handle));
 		rc = -ENOMEM;
 		goto fail;
 	}
@@ -2319,7 +2301,7 @@ int ssi_hash_alloc(struct ssi_drvdata *drvdata)
 		if (IS_ERR(t_alg)) {
 			rc = PTR_ERR(t_alg);
 			SSI_LOG_ERR("%s alg allocation failed\n",
-				 driver_hash[alg].driver_name);
+				    driver_hash[alg].driver_name);
 			goto fail;
 		}
 		t_alg->drvdata = drvdata;
@@ -2338,11 +2320,8 @@ int ssi_hash_alloc(struct ssi_drvdata *drvdata)
 	return 0;
 
 fail:
-
-	if (drvdata->hash_handle) {
-		kfree(drvdata->hash_handle);
-		drvdata->hash_handle = NULL;
-	}
+	kfree(drvdata->hash_handle);
+	drvdata->hash_handle = NULL;
 	return rc;
 }
 
@@ -2365,8 +2344,9 @@ int ssi_hash_free(struct ssi_drvdata *drvdata)
 }
 
 static void ssi_hash_create_xcbc_setup(struct ahash_request *areq,
-				  struct cc_hw_desc desc[],
-				  unsigned int *seq_size) {
+				       struct cc_hw_desc desc[],
+				       unsigned int *seq_size)
+{
 	unsigned int idx = *seq_size;
 	struct ahash_req_ctx *state = ahash_request_ctx(areq);
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
@@ -2422,8 +2402,8 @@ static void ssi_hash_create_xcbc_setup(struct ahash_request *areq,
 }
 
 static void ssi_hash_create_cmac_setup(struct ahash_request *areq,
-				  struct cc_hw_desc desc[],
-				  unsigned int *seq_size)
+				       struct cc_hw_desc desc[],
+				       unsigned int *seq_size)
 {
 	unsigned int idx = *seq_size;
 	struct ahash_req_ctx *state = ahash_request_ctx(areq);
diff --git a/drivers/staging/ccree/ssi_ivgen.c b/drivers/staging/ccree/ssi_ivgen.c
index 5ff3368c04d9..b01e03231947 100644
--- a/drivers/staging/ccree/ssi_ivgen.c
+++ b/drivers/staging/ccree/ssi_ivgen.c
@@ -158,7 +158,7 @@ int ssi_ivgen_init_sram_pool(struct ssi_drvdata *drvdata)
 void ssi_ivgen_fini(struct ssi_drvdata *drvdata)
 {
 	struct ssi_ivgen_ctx *ivgen_ctx = drvdata->ivgen_handle;
-	struct device *device = &(drvdata->plat_dev->dev);
+	struct device *device = &drvdata->plat_dev->dev;
 
 	if (!ivgen_ctx)
 		return;
@@ -166,7 +166,8 @@ void ssi_ivgen_fini(struct ssi_drvdata *drvdata)
 	if (ivgen_ctx->pool_meta) {
 		memset(ivgen_ctx->pool_meta, 0, SSI_IVPOOL_META_SIZE);
 		dma_free_coherent(device, SSI_IVPOOL_META_SIZE,
-			ivgen_ctx->pool_meta, ivgen_ctx->pool_meta_dma);
+				  ivgen_ctx->pool_meta,
+				  ivgen_ctx->pool_meta_dma);
 	}
 
 	ivgen_ctx->pool = NULL_SRAM_ADDR;
@@ -190,10 +191,11 @@ int ssi_ivgen_init(struct ssi_drvdata *drvdata)
 	int rc;
 
 	/* Allocate "this" context */
-	drvdata->ivgen_handle = kzalloc(sizeof(struct ssi_ivgen_ctx), GFP_KERNEL);
+	drvdata->ivgen_handle = kzalloc(sizeof(*drvdata->ivgen_handle),
+					GFP_KERNEL);
 	if (!drvdata->ivgen_handle) {
 		SSI_LOG_ERR("Not enough memory to allocate IVGEN context "
-			   "(%zu B)\n", sizeof(struct ssi_ivgen_ctx));
+			   "(%zu B)\n", sizeof(*drvdata->ivgen_handle));
 		rc = -ENOMEM;
 		goto out;
 	}
@@ -201,7 +203,8 @@ int ssi_ivgen_init(struct ssi_drvdata *drvdata)
 
 	/* Allocate pool's header for intial enc. key/IV */
 	ivgen_ctx->pool_meta = dma_alloc_coherent(device, SSI_IVPOOL_META_SIZE,
-			&ivgen_ctx->pool_meta_dma, GFP_KERNEL);
+						  &ivgen_ctx->pool_meta_dma,
+						  GFP_KERNEL);
 	if (!ivgen_ctx->pool_meta) {
 		SSI_LOG_ERR("Not enough memory to allocate DMA of pool_meta "
 			   "(%u B)\n", SSI_IVPOOL_META_SIZE);
diff --git a/drivers/staging/ccree/ssi_pm.c b/drivers/staging/ccree/ssi_pm.c
index 52a8ed579177..31325e6cd4b4 100644
--- a/drivers/staging/ccree/ssi_pm.c
+++ b/drivers/staging/ccree/ssi_pm.c
@@ -40,7 +40,7 @@ int ssi_power_mgr_runtime_suspend(struct device *dev)
 		(struct ssi_drvdata *)dev_get_drvdata(dev);
 	int rc;
 
-	SSI_LOG_DEBUG("ssi_power_mgr_runtime_suspend: set HOST_POWER_DOWN_EN\n");
+	SSI_LOG_DEBUG("set HOST_POWER_DOWN_EN\n");
 	WRITE_REGISTER(drvdata->cc_base + CC_REG_OFFSET(HOST_RGF, HOST_POWER_DOWN_EN), POWER_DOWN_ENABLE);
 	rc = ssi_request_mgr_runtime_suspend_queue(drvdata);
 	if (rc != 0) {
@@ -58,7 +58,7 @@ int ssi_power_mgr_runtime_resume(struct device *dev)
 	struct ssi_drvdata *drvdata =
 		(struct ssi_drvdata *)dev_get_drvdata(dev);
 
-	SSI_LOG_DEBUG("ssi_power_mgr_runtime_resume , unset HOST_POWER_DOWN_EN\n");
+	SSI_LOG_DEBUG("unset HOST_POWER_DOWN_EN\n");
 	WRITE_REGISTER(drvdata->cc_base + CC_REG_OFFSET(HOST_RGF, HOST_POWER_DOWN_EN), POWER_DOWN_DISABLE);
 
 	rc = cc_clk_on(drvdata);
diff --git a/drivers/staging/ccree/ssi_request_mgr.c b/drivers/staging/ccree/ssi_request_mgr.c
index 46d9396f9ff9..e5c2f92857f6 100644
--- a/drivers/staging/ccree/ssi_request_mgr.c
+++ b/drivers/staging/ccree/ssi_request_mgr.c
@@ -30,8 +30,6 @@
 #include "ssi_sysfs.h"
 #include "ssi_ivgen.h"
 #include "ssi_pm.h"
-#include "ssi_fips.h"
-#include "ssi_fips_local.h"
 
 #define SSI_MAX_POLL_ITER	10
 
@@ -102,7 +100,7 @@ int request_mgr_init(struct ssi_drvdata *drvdata)
 	struct ssi_request_mgr_handle *req_mgr_h;
 	int rc = 0;
 
-	req_mgr_h = kzalloc(sizeof(struct ssi_request_mgr_handle), GFP_KERNEL);
+	req_mgr_h = kzalloc(sizeof(*req_mgr_h), GFP_KERNEL);
 	if (!req_mgr_h) {
 		rc = -ENOMEM;
 		goto req_mgr_init_err;
@@ -129,7 +127,7 @@ int request_mgr_init(struct ssi_drvdata *drvdata)
 	SSI_LOG_DEBUG("hw_queue_size=0x%08X\n", req_mgr_h->hw_queue_size);
 	if (req_mgr_h->hw_queue_size < MIN_HW_QUEUE_SIZE) {
 		SSI_LOG_ERR("Invalid HW queue size = %u (Min. required is %u)\n",
-			req_mgr_h->hw_queue_size, MIN_HW_QUEUE_SIZE);
+			    req_mgr_h->hw_queue_size, MIN_HW_QUEUE_SIZE);
 		rc = -ENOMEM;
 		goto req_mgr_init_err;
 	}
@@ -138,7 +136,9 @@ int request_mgr_init(struct ssi_drvdata *drvdata)
 
 	/* Allocate DMA word for "dummy" completion descriptor use */
 	req_mgr_h->dummy_comp_buff = dma_alloc_coherent(&drvdata->plat_dev->dev,
-		sizeof(u32), &req_mgr_h->dummy_comp_buff_dma, GFP_KERNEL);
+							sizeof(u32),
+							&req_mgr_h->dummy_comp_buff_dma,
+							GFP_KERNEL);
 	if (!req_mgr_h->dummy_comp_buff) {
 		SSI_LOG_ERR("Not enough memory to allocate DMA (%zu) dropped "
 			   "buffer\n", sizeof(u32));
@@ -177,7 +177,8 @@ static inline void enqueue_seq(
 		writel_relaxed(seq[i].word[5], (volatile void __iomem *)(cc_base + CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_QUEUE_WORD0)));
 #ifdef DX_DUMP_DESCS
 		SSI_LOG_DEBUG("desc[%02d]: 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n", i,
-			seq[i].word[0], seq[i].word[1], seq[i].word[2], seq[i].word[3], seq[i].word[4], seq[i].word[5]);
+			      seq[i].word[0], seq[i].word[1], seq[i].word[2],
+			      seq[i].word[3], seq[i].word[4], seq[i].word[5]);
 #endif
 	}
 }
@@ -211,7 +212,7 @@ static inline int request_mgr_queues_status_check(
 		      (MAX_REQUEST_QUEUE_SIZE - 1)) ==
 		     req_mgr_h->req_queue_tail)) {
 		SSI_LOG_ERR("SW FIFO is full. req_queue_head=%d sw_fifo_len=%d\n",
-			   req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE);
+			    req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE);
 		return -EBUSY;
 	}
 
@@ -221,9 +222,8 @@ static inline int request_mgr_queues_status_check(
 	/* Wait for space in HW queue. Poll constant num of iterations. */
 	for (poll_queue = 0; poll_queue < SSI_MAX_POLL_ITER ; poll_queue++) {
 		req_mgr_h->q_free_slots =
-			CC_HAL_READ_REGISTER(
-				CC_REG_OFFSET(CRY_KERNEL,
-						 DSCRPTR_QUEUE_CONTENT));
+			CC_HAL_READ_REGISTER(CC_REG_OFFSET(CRY_KERNEL,
+							   DSCRPTR_QUEUE_CONTENT));
 		if (unlikely(req_mgr_h->q_free_slots <
 						req_mgr_h->min_free_hw_slots)) {
 			req_mgr_h->min_free_hw_slots = req_mgr_h->q_free_slots;
@@ -235,7 +235,7 @@ static inline int request_mgr_queues_status_check(
 		}
 
 		SSI_LOG_DEBUG("HW FIFO is full. q_free_slots=%d total_seq_len=%d\n",
-			req_mgr_h->q_free_slots, total_seq_len);
+			      req_mgr_h->q_free_slots, total_seq_len);
 	}
 	/* No room in the HW queue try again later */
 	SSI_LOG_DEBUG("HW FIFO full, timeout. req_queue_head=%d "
@@ -291,9 +291,8 @@ int send_request(
 		 * in case iv gen add the max size and in case of no dout add 1
 		 * for the internal completion descriptor
 		 */
-		rc = request_mgr_queues_status_check(req_mgr_h,
-					       cc_base,
-					       max_required_seq_len);
+		rc = request_mgr_queues_status_check(req_mgr_h, cc_base,
+						     max_required_seq_len);
 		if (likely(rc == 0))
 			/* There is enough place in the queue */
 			break;
@@ -320,21 +319,22 @@ int send_request(
 	if (!is_dout) {
 		init_completion(&ssi_req->seq_compl);
 		ssi_req->user_cb = request_mgr_complete;
-		ssi_req->user_arg = &(ssi_req->seq_compl);
+		ssi_req->user_arg = &ssi_req->seq_compl;
 		total_seq_len++;
 	}
 
 	if (ssi_req->ivgen_dma_addr_len > 0) {
-		SSI_LOG_DEBUG("Acquire IV from pool into %d DMA addresses 0x%llX, 0x%llX, 0x%llX, IV-size=%u\n",
-			ssi_req->ivgen_dma_addr_len,
-			(unsigned long long)ssi_req->ivgen_dma_addr[0],
-			(unsigned long long)ssi_req->ivgen_dma_addr[1],
-			(unsigned long long)ssi_req->ivgen_dma_addr[2],
-			ssi_req->ivgen_size);
+		SSI_LOG_DEBUG("Acquire IV from pool into %d DMA addresses %pad, %pad, %pad, IV-size=%u\n",
+			      ssi_req->ivgen_dma_addr_len,
+			      ssi_req->ivgen_dma_addr[0],
+			      ssi_req->ivgen_dma_addr[1],
+			      ssi_req->ivgen_dma_addr[2],
+			      ssi_req->ivgen_size);
 
 		/* Acquire IV from pool */
-		rc = ssi_ivgen_getiv(drvdata, ssi_req->ivgen_dma_addr, ssi_req->ivgen_dma_addr_len,
-			ssi_req->ivgen_size, iv_seq, &iv_seq_len);
+		rc = ssi_ivgen_getiv(drvdata, ssi_req->ivgen_dma_addr,
+				     ssi_req->ivgen_dma_addr_len,
+				     ssi_req->ivgen_size, iv_seq, &iv_seq_len);
 
 		if (unlikely(rc != 0)) {
 			SSI_LOG_ERR("Failed to generate IV (rc=%d)\n", rc);
@@ -418,9 +418,8 @@ int send_request_init(
 	enqueue_seq(cc_base, desc, len);
 
 	/* Update the free slots in HW queue */
-	req_mgr_h->q_free_slots = CC_HAL_READ_REGISTER(
-					CC_REG_OFFSET(CRY_KERNEL,
-					 DSCRPTR_QUEUE_CONTENT));
+	req_mgr_h->q_free_slots = CC_HAL_READ_REGISTER(CC_REG_OFFSET(CRY_KERNEL,
+								     DSCRPTR_QUEUE_CONTENT));
 
 	return 0;
 }
@@ -545,8 +544,7 @@ static void comp_handler(unsigned long devarg)
 	}
 	/* after verifing that there is nothing to do, Unmask AXI completion interrupt */
 	CC_HAL_WRITE_REGISTER(CC_REG_OFFSET(HOST_RGF, HOST_IMR),
-		CC_HAL_READ_REGISTER(
-		CC_REG_OFFSET(HOST_RGF, HOST_IMR)) & ~irq);
+			      CC_HAL_READ_REGISTER(CC_REG_OFFSET(HOST_RGF, HOST_IMR)) & ~irq);
 }
 
 /*
diff --git a/drivers/staging/ccree/ssi_sram_mgr.c b/drivers/staging/ccree/ssi_sram_mgr.c
index e05c0c13c2eb..f11116afe89a 100644
--- a/drivers/staging/ccree/ssi_sram_mgr.c
+++ b/drivers/staging/ccree/ssi_sram_mgr.c
@@ -58,7 +58,7 @@ int ssi_sram_mgr_init(struct ssi_drvdata *drvdata)
 			sizeof(struct ssi_sram_mgr_ctx), GFP_KERNEL);
 	if (!drvdata->sram_mgr_handle) {
 		SSI_LOG_ERR("Not enough memory to allocate SRAM_MGR ctx (%zu)\n",
-			sizeof(struct ssi_sram_mgr_ctx));
+			    sizeof(struct ssi_sram_mgr_ctx));
 		rc = -ENOMEM;
 		goto out;
 	}
@@ -90,12 +90,12 @@ ssi_sram_addr_t ssi_sram_mgr_alloc(struct ssi_drvdata *drvdata, u32 size)
 
 	if (unlikely((size & 0x3) != 0)) {
 		SSI_LOG_ERR("Requested buffer size (%u) is not multiple of 4",
-			size);
+			    size);
 		return NULL_SRAM_ADDR;
 	}
 	if (unlikely(size > (SSI_CC_SRAM_SIZE - smgr_ctx->sram_free_offset))) {
 		SSI_LOG_ERR("Not enough space to allocate %u B (at offset %llu)\n",
-			size, smgr_ctx->sram_free_offset);
+			    size, smgr_ctx->sram_free_offset);
 		return NULL_SRAM_ADDR;
 	}
 
diff --git a/drivers/staging/ccree/ssi_sysfs.c b/drivers/staging/ccree/ssi_sysfs.c
index dbcd1634aad1..0655658bba4d 100644
--- a/drivers/staging/ccree/ssi_sysfs.c
+++ b/drivers/staging/ccree/ssi_sysfs.c
@@ -40,8 +40,7 @@ struct stat_name {
 	const char *stat_phase_name[MAX_STAT_PHASES];
 };
 
-static struct stat_name stat_name_db[MAX_STAT_OP_TYPES] =
-{
+static struct stat_name stat_name_db[MAX_STAT_OP_TYPES] = {
 	{
 		/* STAT_OP_TYPE_NULL */
 		.op_type_name = "NULL",
@@ -144,8 +143,12 @@ static void display_db(struct stat_item item[MAX_STAT_OP_TYPES][MAX_STAT_PHASES]
 				avg = (u64)item[i][j].sum;
 				do_div(avg, item[i][j].count);
 				SSI_LOG_ERR("%s, %s: min=%d avg=%d max=%d sum=%lld count=%d\n",
-					stat_name_db[i].op_type_name, stat_name_db[i].stat_phase_name[j],
-					item[i][j].min, (int)avg, item[i][j].max, (long long)item[i][j].sum, item[i][j].count);
+					    stat_name_db[i].op_type_name,
+					    stat_name_db[i].stat_phase_name[j],
+					    item[i][j].min, (int)avg,
+					    item[i][j].max,
+					    (long long)item[i][j].sum,
+					    item[i][j].count);
 			}
 		}
 	}
@@ -156,21 +159,23 @@ static void display_db(struct stat_item item[MAX_STAT_OP_TYPES][MAX_STAT_PHASES]
  **************************************/
 
 static ssize_t ssi_sys_stats_host_db_clear(struct kobject *kobj,
-	struct kobj_attribute *attr, const char *buf, size_t count)
+					   struct kobj_attribute *attr,
+					   const char *buf, size_t count)
 {
 	init_db(stat_host_db);
 	return count;
 }
 
 static ssize_t ssi_sys_stats_cc_db_clear(struct kobject *kobj,
-	struct kobj_attribute *attr, const char *buf, size_t count)
+					 struct kobj_attribute *attr,
+					 const char *buf, size_t count)
 {
 	init_db(stat_cc_db);
 	return count;
 }
 
 static ssize_t ssi_sys_stat_host_db_show(struct kobject *kobj,
-		struct kobj_attribute *attr, char *buf)
+					 struct kobj_attribute *attr, char *buf)
 {
 	int i, j;
 	char line[512];
@@ -179,7 +184,7 @@ static ssize_t ssi_sys_stat_host_db_show(struct kobject *kobj,
 	ssize_t buf_len, tmp_len = 0;
 
 	buf_len = scnprintf(buf, PAGE_SIZE,
-		"phase\t\t\t\t\t\t\tmin[cy]\tavg[cy]\tmax[cy]\t#samples\n");
+			    "phase\t\t\t\t\t\t\tmin[cy]\tavg[cy]\tmax[cy]\t#samples\n");
 	if (buf_len < 0)/* scnprintf shouldn't return negative value according to its implementation*/
 		return buf_len;
 	for (i = STAT_OP_TYPE_ENCODE; i < MAX_STAT_OP_TYPES; i++) {
@@ -193,11 +198,11 @@ static ssize_t ssi_sys_stat_host_db_show(struct kobject *kobj,
 				avg = min_cyc = max_cyc = 0;
 			}
 			tmp_len = scnprintf(line, 512,
-				"%s::%s\t\t\t\t\t%6u\t%6u\t%6u\t%7u\n",
-				stat_name_db[i].op_type_name,
-				stat_name_db[i].stat_phase_name[j],
-				min_cyc, (unsigned int)avg, max_cyc,
-				stat_host_db[i][j].count);
+					    "%s::%s\t\t\t\t\t%6u\t%6u\t%6u\t%7u\n",
+					    stat_name_db[i].op_type_name,
+					    stat_name_db[i].stat_phase_name[j],
+					    min_cyc, (unsigned int)avg, max_cyc,
+					    stat_host_db[i][j].count);
 			if (tmp_len < 0)/* scnprintf shouldn't return negative value according to its implementation*/
 				return buf_len;
 			if (buf_len + tmp_len >= PAGE_SIZE)
@@ -210,7 +215,7 @@ static ssize_t ssi_sys_stat_host_db_show(struct kobject *kobj,
 }
 
 static ssize_t ssi_sys_stat_cc_db_show(struct kobject *kobj,
-		struct kobj_attribute *attr, char *buf)
+				       struct kobj_attribute *attr, char *buf)
 {
 	int i;
 	char line[256];
@@ -219,7 +224,7 @@ static ssize_t ssi_sys_stat_cc_db_show(struct kobject *kobj,
 	ssize_t buf_len, tmp_len = 0;
 
 	buf_len = scnprintf(buf, PAGE_SIZE,
-		"phase\tmin[cy]\tavg[cy]\tmax[cy]\t#samples\n");
+			    "phase\tmin[cy]\tavg[cy]\tmax[cy]\t#samples\n");
 	if (buf_len < 0)/* scnprintf shouldn't return negative value according to its implementation*/
 		return buf_len;
 	for (i = STAT_OP_TYPE_ENCODE; i < MAX_STAT_OP_TYPES; i++) {
@@ -231,13 +236,10 @@ static ssize_t ssi_sys_stat_cc_db_show(struct kobject *kobj,
 		} else {
 			avg = min_cyc = max_cyc = 0;
 		}
-		tmp_len = scnprintf(line, 256,
-			"%s\t%6u\t%6u\t%6u\t%7u\n",
-			stat_name_db[i].op_type_name,
-			min_cyc,
-			(unsigned int)avg,
-			max_cyc,
-			stat_cc_db[i][STAT_PHASE_6].count);
+		tmp_len = scnprintf(line, 256, "%s\t%6u\t%6u\t%6u\t%7u\n",
+				    stat_name_db[i].op_type_name, min_cyc,
+				    (unsigned int)avg, max_cyc,
+				    stat_cc_db[i][STAT_PHASE_6].count);
 
 		if (tmp_len < 0)/* scnprintf shouldn't return negative value according to its implementation*/
 			return buf_len;
@@ -255,7 +257,7 @@ void update_host_stat(unsigned int op_type, unsigned int phase, cycles_t result)
 	unsigned long flags;
 
 	spin_lock_irqsave(&stat_lock, flags);
-	update_db(&(stat_host_db[op_type][phase]), (unsigned int)result);
+	update_db(&stat_host_db[op_type][phase], (unsigned int)result);
 	spin_unlock_irqrestore(&stat_lock, flags);
 }
 
@@ -264,7 +266,7 @@ void update_cc_stat(
 	unsigned int phase,
 	unsigned int elapsed_cycles)
 {
-	update_db(&(stat_cc_db[op_type][phase]), elapsed_cycles);
+	update_db(&stat_cc_db[op_type][phase], elapsed_cycles);
 }
 
 void display_all_stat_db(void)
@@ -277,7 +279,7 @@ void display_all_stat_db(void)
 #endif /*CC_CYCLE_COUNT*/
 
 static ssize_t ssi_sys_regdump_show(struct kobject *kobj,
-		struct kobj_attribute *attr, char *buf)
+				    struct kobj_attribute *attr, char *buf)
 {
 	struct ssi_drvdata *drvdata = sys_get_drvdata();
 	u32 register_value;
@@ -285,20 +287,20 @@ static ssize_t ssi_sys_regdump_show(struct kobject *kobj,
 	int offset = 0;
 
 	register_value = CC_HAL_READ_REGISTER(CC_REG_OFFSET(HOST_RGF, HOST_SIGNATURE));
-	offset += scnprintf(buf + offset, PAGE_SIZE - offset, "%s \t(0x%lX)\t 0x%08X  \n", "HOST_SIGNATURE       ", DX_HOST_SIGNATURE_REG_OFFSET, register_value);
+	offset += scnprintf(buf + offset, PAGE_SIZE - offset, "%s \t(0x%lX)\t 0x%08X\n", "HOST_SIGNATURE       ", DX_HOST_SIGNATURE_REG_OFFSET, register_value);
 	register_value = CC_HAL_READ_REGISTER(CC_REG_OFFSET(HOST_RGF, HOST_IRR));
-	offset += scnprintf(buf + offset, PAGE_SIZE - offset, "%s \t(0x%lX)\t 0x%08X  \n", "HOST_IRR             ", DX_HOST_IRR_REG_OFFSET, register_value);
+	offset += scnprintf(buf + offset, PAGE_SIZE - offset, "%s \t(0x%lX)\t 0x%08X\n", "HOST_IRR             ", DX_HOST_IRR_REG_OFFSET, register_value);
 	register_value = CC_HAL_READ_REGISTER(CC_REG_OFFSET(HOST_RGF, HOST_POWER_DOWN_EN));
-	offset += scnprintf(buf + offset, PAGE_SIZE - offset, "%s \t(0x%lX)\t 0x%08X  \n", "HOST_POWER_DOWN_EN   ", DX_HOST_POWER_DOWN_EN_REG_OFFSET, register_value);
+	offset += scnprintf(buf + offset, PAGE_SIZE - offset, "%s \t(0x%lX)\t 0x%08X\n", "HOST_POWER_DOWN_EN   ", DX_HOST_POWER_DOWN_EN_REG_OFFSET, register_value);
 	register_value =  CC_HAL_READ_REGISTER(CC_REG_OFFSET(CRY_KERNEL, AXIM_MON_ERR));
-	offset += scnprintf(buf + offset, PAGE_SIZE - offset, "%s \t(0x%lX)\t 0x%08X  \n", "AXIM_MON_ERR         ", DX_AXIM_MON_ERR_REG_OFFSET, register_value);
+	offset += scnprintf(buf + offset, PAGE_SIZE - offset, "%s \t(0x%lX)\t 0x%08X\n", "AXIM_MON_ERR         ", DX_AXIM_MON_ERR_REG_OFFSET, register_value);
 	register_value = CC_HAL_READ_REGISTER(CC_REG_OFFSET(CRY_KERNEL, DSCRPTR_QUEUE_CONTENT));
-	offset += scnprintf(buf + offset, PAGE_SIZE - offset, "%s \t(0x%lX)\t 0x%08X  \n", "DSCRPTR_QUEUE_CONTENT", DX_DSCRPTR_QUEUE_CONTENT_REG_OFFSET, register_value);
+	offset += scnprintf(buf + offset, PAGE_SIZE - offset, "%s \t(0x%lX)\t 0x%08X\n", "DSCRPTR_QUEUE_CONTENT", DX_DSCRPTR_QUEUE_CONTENT_REG_OFFSET, register_value);
 	return offset;
 }
 
 static ssize_t ssi_sys_help_show(struct kobject *kobj,
-		struct kobj_attribute *attr, char *buf)
+				 struct kobj_attribute *attr, char *buf)
 {
 	char *help_str[] = {
 				"cat reg_dump              ", "Print several of CC register values",
@@ -357,8 +359,8 @@ static struct ssi_drvdata *sys_get_drvdata(void)
 }
 
 static int sys_init_dir(struct sys_dir *sys_dir, struct ssi_drvdata *drvdata,
-		 struct kobject *parent_dir_kobj, const char *dir_name,
-		 struct kobj_attribute *attrs, u32 num_of_attrs)
+			struct kobject *parent_dir_kobj, const char *dir_name,
+			struct kobj_attribute *attrs, u32 num_of_attrs)
 {
 	int i;
 
@@ -375,7 +377,7 @@ static int sys_init_dir(struct sys_dir *sys_dir, struct ssi_drvdata *drvdata,
 	/* allocate memory for directory's attributes list */
 	sys_dir->sys_dir_attr_list =
 		kzalloc(sizeof(struct attribute *) * (num_of_attrs + 1),
-				GFP_KERNEL);
+			GFP_KERNEL);
 
 	if (!(sys_dir->sys_dir_attr_list)) {
 		kobject_put(sys_dir->sys_dir_kobj);
@@ -386,7 +388,7 @@ static int sys_init_dir(struct sys_dir *sys_dir, struct ssi_drvdata *drvdata,
 
 	/* initialize attributes list */
 	for (i = 0; i < num_of_attrs; ++i)
-		sys_dir->sys_dir_attr_list[i] = &(attrs[i].attr);
+		sys_dir->sys_dir_attr_list[i] = &attrs[i].attr;
 
 	/* last list entry should be NULL */
 	sys_dir->sys_dir_attr_list[num_of_attrs] = NULL;
@@ -394,7 +396,7 @@ static int sys_init_dir(struct sys_dir *sys_dir, struct ssi_drvdata *drvdata,
 	sys_dir->sys_dir_attr_group.attrs = sys_dir->sys_dir_attr_list;
 
 	return sysfs_create_group(sys_dir->sys_dir_kobj,
-			&(sys_dir->sys_dir_attr_group));
+			&sys_dir->sys_dir_attr_group);
 }
 
 static void sys_free_dir(struct sys_dir *sys_dir)
@@ -421,9 +423,9 @@ int ssi_sysfs_init(struct kobject *sys_dev_obj, struct ssi_drvdata *drvdata)
 	SSI_LOG_ERR("setup sysfs under %s\n", sys_dev_obj->name);
 
 	/* Initialize top directory */
-	retval = sys_init_dir(&sys_top_dir, drvdata, sys_dev_obj,
-				"cc_info", ssi_sys_top_level_attrs,
-				ARRAY_SIZE(ssi_sys_top_level_attrs));
+	retval = sys_init_dir(&sys_top_dir, drvdata, sys_dev_obj, "cc_info",
+			      ssi_sys_top_level_attrs,
+			      ARRAY_SIZE(ssi_sys_top_level_attrs));
 	return retval;
 }
 
diff --git a/drivers/staging/comedi/comedi_buf.c b/drivers/staging/comedi/comedi_buf.c
index 8e9b30b26810..b455ff6714eb 100644
--- a/drivers/staging/comedi/comedi_buf.c
+++ b/drivers/staging/comedi/comedi_buf.c
@@ -165,7 +165,7 @@ int comedi_buf_map_put(struct comedi_buf_map *bm)
 int comedi_buf_map_access(struct comedi_buf_map *bm, unsigned long offset,
 			  void *buf, int len, int write)
 {
-	unsigned int pgoff = offset & ~PAGE_MASK;
+	unsigned int pgoff = offset_in_page(offset);
 	unsigned long pg = offset >> PAGE_SHIFT;
 	int done = 0;
 
diff --git a/drivers/staging/comedi/comedi_fops.c b/drivers/staging/comedi/comedi_fops.c
index 34ca7823255d..e19e395b0e44 100644
--- a/drivers/staging/comedi/comedi_fops.c
+++ b/drivers/staging/comedi/comedi_fops.c
@@ -745,7 +745,7 @@ static void do_become_nonbusy(struct comedi_device *dev,
 		wake_up_interruptible_all(&async->wait_head);
 	} else {
 		dev_err(dev->class_dev,
-			"BUG: (?) do_become_nonbusy called with async=NULL\n");
+			"BUG: (?) %s called with async=NULL\n", __func__);
 		s->busy = NULL;
 	}
 }
diff --git a/drivers/staging/comedi/drivers.c b/drivers/staging/comedi/drivers.c
index a5bf2cc165c0..0b43db6371c6 100644
--- a/drivers/staging/comedi/drivers.c
+++ b/drivers/staging/comedi/drivers.c
@@ -995,12 +995,12 @@ int comedi_auto_config(struct device *hardware_device,
 	int ret;
 
 	if (!hardware_device) {
-		pr_warn("BUG! comedi_auto_config called with NULL hardware_device\n");
+		pr_warn("BUG! %s called with NULL hardware_device\n", __func__);
 		return -EINVAL;
 	}
 	if (!driver) {
 		dev_warn(hardware_device,
-			 "BUG! comedi_auto_config called with NULL comedi driver\n");
+			 "BUG! %s called with NULL comedi driver\n", __func__);
 		return -EINVAL;
 	}
 
diff --git a/drivers/staging/comedi/drivers/ni_mio_common.c b/drivers/staging/comedi/drivers/ni_mio_common.c
index 2f7bfc1c59e5..398347fedc47 100644
--- a/drivers/staging/comedi/drivers/ni_mio_common.c
+++ b/drivers/staging/comedi/drivers/ni_mio_common.c
@@ -1962,7 +1962,8 @@ static unsigned int ni_timer_to_ns(const struct comedi_device *dev, int timer)
 static void ni_cmd_set_mite_transfer(struct mite_ring *ring,
 				     struct comedi_subdevice *sdev,
 				     const struct comedi_cmd *cmd,
-				     unsigned int max_count) {
+				     unsigned int max_count)
+{
 #ifdef PCIDMA
 	unsigned int nbytes = max_count;
 
diff --git a/drivers/staging/comedi/drivers/serial2002.c b/drivers/staging/comedi/drivers/serial2002.c
index 0d33e520f635..cc18e25103ca 100644
--- a/drivers/staging/comedi/drivers/serial2002.c
+++ b/drivers/staging/comedi/drivers/serial2002.c
@@ -106,16 +106,8 @@ static long serial2002_tty_ioctl(struct file *f, unsigned int op,
 
 static int serial2002_tty_write(struct file *f, unsigned char *buf, int count)
 {
-	const char __user *p = (__force const char __user *)buf;
-	int result;
-	loff_t offset = 0;
-	mm_segment_t oldfs;
-
-	oldfs = get_fs();
-	set_fs(KERNEL_DS);
-	result = __vfs_write(f, p, count, &offset);
-	set_fs(oldfs);
-	return result;
+	loff_t pos = 0;
+	return kernel_write(f, buf, count, &pos);
 }
 
 static void serial2002_tty_read_poll_wait(struct file *f, int timeout)
@@ -148,19 +140,14 @@ static int serial2002_tty_read(struct file *f, int timeout)
 {
 	unsigned char ch;
 	int result;
+	loff_t pos = 0;
 
 	result = -1;
 	if (!IS_ERR(f)) {
-		mm_segment_t oldfs;
-		char __user *p = (__force char __user *)&ch;
-		loff_t offset = 0;
-
-		oldfs = get_fs();
-		set_fs(KERNEL_DS);
 		if (f->f_op->poll) {
 			serial2002_tty_read_poll_wait(f, timeout);
 
-			if (__vfs_read(f, p, 1, &offset) == 1)
+			if (kernel_read(f, &ch, 1, &pos) == 1)
 				result = ch;
 		} else {
 			/* Device does not support poll, busy wait */
@@ -171,14 +158,13 @@ static int serial2002_tty_read(struct file *f, int timeout)
 				if (retries >= timeout)
 					break;
 
-				if (__vfs_read(f, p, 1, &offset) == 1) {
+				if (kernel_read(f, &ch, 1, &pos) == 1) {
 					result = ch;
 					break;
 				}
 				usleep_range(100, 1000);
 			}
 		}
-		set_fs(oldfs);
 	}
 	return result;
 }
diff --git a/drivers/staging/fbtft/fb_st7789v.c b/drivers/staging/fbtft/fb_st7789v.c
index 8935a97ec048..a5d7c87557f8 100644
--- a/drivers/staging/fbtft/fb_st7789v.c
+++ b/drivers/staging/fbtft/fb_st7789v.c
@@ -189,7 +189,7 @@ static int set_gamma(struct fbtft_par *par, u32 *curves)
 	 * The masks are the same for both positive and negative voltage
 	 * gamma curves.
 	 */
-	const u8 gamma_par_mask[] = {
+	static const u8 gamma_par_mask[] = {
 		0xFF, /* V63[3:0], V0[3:0]*/
 		0x3F, /* V1[5:0] */
 		0x3F, /* V2[5:0] */
diff --git a/drivers/staging/fbtft/fbtft-core.c b/drivers/staging/fbtft/fbtft-core.c
index b742ee786615..6d0363deba61 100644
--- a/drivers/staging/fbtft/fbtft-core.c
+++ b/drivers/staging/fbtft/fbtft-core.c
@@ -84,7 +84,7 @@ static unsigned long fbtft_request_gpios_match(struct fbtft_par *par,
 					       const struct fbtft_gpio *gpio)
 {
 	int ret;
-	long val;
+	unsigned int val;
 
 	fbtft_par_dbg(DEBUG_REQUEST_GPIOS_MATCH, par, "%s('%s')\n",
 		      __func__, gpio->name);
@@ -108,7 +108,7 @@ static unsigned long fbtft_request_gpios_match(struct fbtft_par *par,
 		par->gpio.latch = gpio->gpio;
 		return GPIOF_OUT_INIT_LOW;
 	} else if (gpio->name[0] == 'd' && gpio->name[1] == 'b') {
-		ret = kstrtol(&gpio->name[2], 10, &val);
+		ret = kstrtouint(&gpio->name[2], 10, &val);
 		if (ret == 0 && val < 16) {
 			par->gpio.db[val] = gpio->gpio;
 			return GPIOF_OUT_INIT_LOW;
diff --git a/drivers/staging/fsl-dpaa2/Kconfig b/drivers/staging/fsl-dpaa2/Kconfig
index 730fd6d4db33..dfff675b3055 100644
--- a/drivers/staging/fsl-dpaa2/Kconfig
+++ b/drivers/staging/fsl-dpaa2/Kconfig
@@ -4,7 +4,7 @@
 
 config FSL_DPAA2
 	bool "Freescale DPAA2 devices"
-	depends on FSL_MC_BUS
+	depends on FSL_MC_BUS && ARCH_LAYERSCAPE
 	---help---
 	  Build drivers for Freescale DataPath Acceleration
 	  Architecture (DPAA2) family of SoCs.
diff --git a/drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.c b/drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.c
index b9a0a315e6fb..26017fe9df93 100644
--- a/drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.c
+++ b/drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.c
@@ -616,7 +616,7 @@ static netdev_tx_t dpaa2_eth_tx(struct sk_buff *skb, struct net_device *net_dev)
 		free_tx_fd(priv, &fd, NULL);
 	} else {
 		percpu_stats->tx_packets++;
-		percpu_stats->tx_bytes += skb->len;
+		percpu_stats->tx_bytes += dpaa2_fd_get_len(&fd);
 	}
 
 	return NETDEV_TX_OK;
@@ -656,7 +656,7 @@ static void dpaa2_eth_tx_conf(struct dpaa2_eth_priv *priv,
 		has_fas_errors = (fd_errors & DPAA2_FD_CTRL_FAERR) &&
 				 !!(dpaa2_fd_get_frc(fd) & DPAA2_FD_FRC_FASV);
 		if (net_ratelimit())
-			netdev_dbg(priv->net_dev, "TX frame FD error: %x08\n",
+			netdev_dbg(priv->net_dev, "TX frame FD error: 0x%08x\n",
 				   fd_errors);
 	}
 
@@ -670,7 +670,7 @@ static void dpaa2_eth_tx_conf(struct dpaa2_eth_priv *priv,
 	percpu_stats->tx_errors++;
 
 	if (has_fas_errors && net_ratelimit())
-		netdev_dbg(priv->net_dev, "TX frame FAS error: %x08\n",
+		netdev_dbg(priv->net_dev, "TX frame FAS error: 0x%08x\n",
 			   status & DPAA2_FAS_TX_ERR_MASK);
 }
 
diff --git a/drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.h b/drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.h
index e6d28a249fc1..bfbabae1aad8 100644
--- a/drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.h
+++ b/drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.h
@@ -123,8 +123,8 @@ struct dpaa2_eth_swa {
 /* Error bits in FD CTRL */
 #define DPAA2_FD_CTRL_UFD		0x00000004
 #define DPAA2_FD_CTRL_SBE		0x00000008
-#define DPAA2_FD_CTRL_FSE		0x00000010
-#define DPAA2_FD_CTRL_FAERR		0x00000020
+#define DPAA2_FD_CTRL_FSE		0x00000020
+#define DPAA2_FD_CTRL_FAERR		0x00000040
 
 #define DPAA2_FD_RX_ERR_MASK		(DPAA2_FD_CTRL_SBE	| \
 					 DPAA2_FD_CTRL_FAERR)
diff --git a/drivers/staging/fsl-dpaa2/ethernet/dpaa2-ethtool.c b/drivers/staging/fsl-dpaa2/ethernet/dpaa2-ethtool.c
index 5312edc26f01..031179ab3a22 100644
--- a/drivers/staging/fsl-dpaa2/ethernet/dpaa2-ethtool.c
+++ b/drivers/staging/fsl-dpaa2/ethernet/dpaa2-ethtool.c
@@ -217,8 +217,6 @@ static void dpaa2_eth_get_ethtool_stats(struct net_device *net_dev,
 		case 2:
 			num_cnt = sizeof(dpni_stats.page_2) / sizeof(u64);
 			break;
-		default:
-			break;
 		}
 		for (k = 0; k < num_cnt; k++)
 			*(data + i++) = dpni_stats.raw.counter[k];
diff --git a/drivers/staging/fsl-dpaa2/ethernet/dpni.c b/drivers/staging/fsl-dpaa2/ethernet/dpni.c
index 5b9d4424e4fb..04a5b14bc1c5 100644
--- a/drivers/staging/fsl-dpaa2/ethernet/dpni.c
+++ b/drivers/staging/fsl-dpaa2/ethernet/dpni.c
@@ -1443,7 +1443,7 @@ int dpni_get_queue(struct fsl_mc_io *mc_io,
 	queue->destination.id = le32_to_cpu(rsp_params->dest_id);
 	queue->destination.priority = rsp_params->dest_prio;
 	queue->destination.type = dpni_get_field(rsp_params->flags,
-						     DEST_TYPE);
+						 DEST_TYPE);
 	queue->flc.stash_control = dpni_get_field(rsp_params->flags,
 						  STASH_CTRL);
 	queue->destination.hold_active = dpni_get_field(rsp_params->flags,
diff --git a/drivers/staging/fsl-mc/bus/Kconfig b/drivers/staging/fsl-mc/bus/Kconfig
index a10aaf03f314..504c987447f2 100644
--- a/drivers/staging/fsl-mc/bus/Kconfig
+++ b/drivers/staging/fsl-mc/bus/Kconfig
@@ -8,7 +8,7 @@
 
 config FSL_MC_BUS
 	bool "QorIQ DPAA2 fsl-mc bus driver"
-	depends on OF && ARCH_LAYERSCAPE
+	depends on OF && (ARCH_LAYERSCAPE || (COMPILE_TEST && (ARM || ARM64 || X86 || PPC)))
 	select GENERIC_MSI_IRQ_DOMAIN
 	help
 	  Driver to enable the bus infrastructure for the QorIQ DPAA2
@@ -18,7 +18,7 @@ config FSL_MC_BUS
 
 config FSL_MC_DPIO
         tristate "QorIQ DPAA2 DPIO driver"
-        depends on FSL_MC_BUS
+        depends on FSL_MC_BUS && ARCH_LAYERSCAPE
         help
 	  Driver for the DPAA2 DPIO object.  A DPIO provides queue and
 	  buffer management facilities for software to interact with
diff --git a/drivers/staging/fsl-mc/bus/dpio/qbman-portal.c b/drivers/staging/fsl-mc/bus/dpio/qbman-portal.c
index 7988612aaecf..163bdac6b051 100644
--- a/drivers/staging/fsl-mc/bus/dpio/qbman-portal.c
+++ b/drivers/staging/fsl-mc/bus/dpio/qbman-portal.c
@@ -136,18 +136,18 @@ static inline u32 qbman_set_swp_cfg(u8 max_fill, u8 wn,	u8 est, u8 rpm, u8 dcm,
 				    u8 epm, int sd, int sp, int se,
 				    int dp, int de, int ep)
 {
-	return cpu_to_le32 (max_fill << SWP_CFG_DQRR_MF_SHIFT |
-			    est << SWP_CFG_EST_SHIFT |
-			    wn << SWP_CFG_WN_SHIFT |
-			    rpm << SWP_CFG_RPM_SHIFT |
-			    dcm << SWP_CFG_DCM_SHIFT |
-			    epm << SWP_CFG_EPM_SHIFT |
-			    sd << SWP_CFG_SD_SHIFT |
-			    sp << SWP_CFG_SP_SHIFT |
-			    se << SWP_CFG_SE_SHIFT |
-			    dp << SWP_CFG_DP_SHIFT |
-			    de << SWP_CFG_DE_SHIFT |
-			    ep << SWP_CFG_EP_SHIFT);
+	return (max_fill << SWP_CFG_DQRR_MF_SHIFT |
+		est << SWP_CFG_EST_SHIFT |
+		wn << SWP_CFG_WN_SHIFT |
+		rpm << SWP_CFG_RPM_SHIFT |
+		dcm << SWP_CFG_DCM_SHIFT |
+		epm << SWP_CFG_EPM_SHIFT |
+		sd << SWP_CFG_SD_SHIFT |
+		sp << SWP_CFG_SP_SHIFT |
+		se << SWP_CFG_SE_SHIFT |
+		dp << SWP_CFG_DP_SHIFT |
+		de << SWP_CFG_DE_SHIFT |
+		ep << SWP_CFG_EP_SHIFT);
 }
 
 /**
diff --git a/drivers/staging/fsl-mc/bus/dprc-driver.c b/drivers/staging/fsl-mc/bus/dprc-driver.c
index 4cdd190a338b..06df528f4cfc 100644
--- a/drivers/staging/fsl-mc/bus/dprc-driver.c
+++ b/drivers/staging/fsl-mc/bus/dprc-driver.c
@@ -29,7 +29,7 @@ static bool fsl_mc_device_match(struct fsl_mc_device *mc_dev,
 				struct fsl_mc_obj_desc *obj_desc)
 {
 	return mc_dev->obj_desc.id == obj_desc->id &&
-	       !strcmp(mc_dev->obj_desc.type, obj_desc->type);
+	       strcmp(mc_dev->obj_desc.type, obj_desc->type) == 0;
 
 }
 
@@ -617,8 +617,7 @@ static int dprc_probe(struct fsl_mc_device *mc_dev)
 		if (WARN_ON(mc_dev->obj_desc.region_count == 0))
 			return -EINVAL;
 
-		region_size = mc_dev->regions[0].end -
-			      mc_dev->regions[0].start + 1;
+		region_size = resource_size(mc_dev->regions);
 
 		error = fsl_create_mc_io(&mc_dev->dev,
 					 mc_dev->regions[0].start,
diff --git a/drivers/staging/fsl-mc/bus/fsl-mc-allocator.c b/drivers/staging/fsl-mc/bus/fsl-mc-allocator.c
index b37a6f48225f..8ea3920400a0 100644
--- a/drivers/staging/fsl-mc/bus/fsl-mc-allocator.c
+++ b/drivers/staging/fsl-mc/bus/fsl-mc-allocator.c
@@ -16,9 +16,9 @@
 
 static bool __must_check fsl_mc_is_allocatable(const char *obj_type)
 {
-	return strcmp(obj_type, "dpbp") ||
-	       strcmp(obj_type, "dpmcp") ||
-	       strcmp(obj_type, "dpcon");
+	return strcmp(obj_type, "dpbp") == 0 ||
+	       strcmp(obj_type, "dpmcp") == 0 ||
+	       strcmp(obj_type, "dpcon") == 0;
 }
 
 /**
diff --git a/drivers/staging/fsl-mc/bus/fsl-mc-bus.c b/drivers/staging/fsl-mc/bus/fsl-mc-bus.c
index 19606e8d25dd..409f2b9e70ff 100644
--- a/drivers/staging/fsl-mc/bus/fsl-mc-bus.c
+++ b/drivers/staging/fsl-mc/bus/fsl-mc-bus.c
@@ -757,8 +757,8 @@ static int fsl_mc_bus_probe(struct platform_device *pdev)
 	error = of_address_to_resource(pdev->dev.of_node, 0, &res);
 	if (error < 0) {
 		dev_err(&pdev->dev,
-			"of_address_to_resource() failed for %s\n",
-			pdev->dev.of_node->full_name);
+			"of_address_to_resource() failed for %pOF\n",
+			pdev->dev.of_node);
 		return error;
 	}
 
diff --git a/drivers/staging/fsl-mc/bus/fsl-mc-msi.c b/drivers/staging/fsl-mc/bus/fsl-mc-msi.c
index c04a2f2b3409..038da4d1ebd0 100644
--- a/drivers/staging/fsl-mc/bus/fsl-mc-msi.c
+++ b/drivers/staging/fsl-mc/bus/fsl-mc-msi.c
@@ -11,13 +11,13 @@
 
 #include <linux/of_device.h>
 #include <linux/of_address.h>
-#include <linux/irqchip/arm-gic-v3.h>
 #include <linux/of_irq.h>
 #include <linux/irq.h>
 #include <linux/irqdomain.h>
 #include <linux/msi.h>
 #include "fsl-mc-private.h"
 
+#ifdef GENERIC_MSI_DOMAIN_OPS
 /*
  * Generate a unique ID identifying the interrupt (only used within the MSI
  * irqdomain.  Combine the icid with the interrupt index.
@@ -39,6 +39,9 @@ static void fsl_mc_msi_set_desc(msi_alloc_info_t *arg,
 	arg->hwirq = fsl_mc_domain_calc_hwirq(to_fsl_mc_device(desc->dev),
 					      desc);
 }
+#else
+#define fsl_mc_msi_set_desc NULL
+#endif
 
 static void fsl_mc_msi_update_dom_ops(struct msi_domain_info *info)
 {
@@ -183,8 +186,8 @@ int fsl_mc_find_msi_domain(struct device *mc_platform_dev,
 	msi_domain = of_msi_get_domain(mc_platform_dev, mc_of_node,
 				       DOMAIN_BUS_FSL_MC_MSI);
 	if (!msi_domain) {
-		pr_err("Unable to find fsl-mc MSI domain for %s\n",
-		       mc_of_node->full_name);
+		pr_err("Unable to find fsl-mc MSI domain for %pOF\n",
+		       mc_of_node);
 
 		return -ENOENT;
 	}
diff --git a/drivers/staging/fsl-mc/bus/irq-gic-v3-its-fsl-mc-msi.c b/drivers/staging/fsl-mc/bus/irq-gic-v3-its-fsl-mc-msi.c
index 865d38517508..123e4af58408 100644
--- a/drivers/staging/fsl-mc/bus/irq-gic-v3-its-fsl-mc-msi.c
+++ b/drivers/staging/fsl-mc/bus/irq-gic-v3-its-fsl-mc-msi.c
@@ -11,7 +11,6 @@
 
 #include <linux/of_device.h>
 #include <linux/of_address.h>
-#include <linux/irqchip/arm-gic-v3.h>
 #include <linux/irq.h>
 #include <linux/msi.h>
 #include <linux/of.h>
@@ -46,7 +45,9 @@ static int its_fsl_mc_msi_prepare(struct irq_domain *msi_domain,
 	 * NOTE: This device id corresponds to the IOMMU stream ID
 	 * associated with the DPRC object (ICID).
 	 */
+#ifdef GENERIC_MSI_DOMAIN_OPS
 	info->scratchpad[0].ul = mc_bus_dev->icid;
+#endif
 	msi_info = msi_get_domain_info(msi_domain->parent);
 	return msi_info->ops->msi_prepare(msi_domain->parent, dev, nvec, info);
 }
@@ -79,8 +80,7 @@ int __init its_fsl_mc_msi_init(void)
 
 		parent = irq_find_matching_host(np, DOMAIN_BUS_NEXUS);
 		if (!parent || !msi_get_domain_info(parent)) {
-			pr_err("%s: unable to locate ITS domain\n",
-			       np->full_name);
+			pr_err("%pOF: unable to locate ITS domain\n", np);
 			continue;
 		}
 
@@ -89,15 +89,14 @@ int __init its_fsl_mc_msi_init(void)
 						 &its_fsl_mc_msi_domain_info,
 						 parent);
 		if (!mc_msi_domain) {
-			pr_err("%s: unable to create fsl-mc domain\n",
-			       np->full_name);
+			pr_err("%pOF: unable to create fsl-mc domain\n", np);
 			continue;
 		}
 
 		WARN_ON(mc_msi_domain->host_data !=
 			&its_fsl_mc_msi_domain_info);
 
-		pr_info("fsl-mc MSI: %s domain created\n", np->full_name);
+		pr_info("fsl-mc MSI: %pOF domain created\n", np);
 	}
 
 	return 0;
diff --git a/drivers/staging/fsl-mc/bus/mc-io.c b/drivers/staging/fsl-mc/bus/mc-io.c
index 35221a17858b..f65c23ce83f1 100644
--- a/drivers/staging/fsl-mc/bus/mc-io.c
+++ b/drivers/staging/fsl-mc/bus/mc-io.c
@@ -129,8 +129,8 @@ int __must_check fsl_create_mc_io(struct device *dev,
 				      "mc_portal");
 	if (!res) {
 		dev_err(dev,
-			"devm_request_mem_region failed for MC portal %#llx\n",
-			mc_portal_phys_addr);
+			"devm_request_mem_region failed for MC portal %pa\n",
+			&mc_portal_phys_addr);
 		return -EBUSY;
 	}
 
@@ -139,8 +139,8 @@ int __must_check fsl_create_mc_io(struct device *dev,
 						   mc_portal_size);
 	if (!mc_portal_virt_addr) {
 		dev_err(dev,
-			"devm_ioremap_nocache failed for MC portal %#llx\n",
-			mc_portal_phys_addr);
+			"devm_ioremap_nocache failed for MC portal %pa\n",
+			&mc_portal_phys_addr);
 		return -ENXIO;
 	}
 
@@ -242,8 +242,7 @@ int __must_check fsl_mc_portal_allocate(struct fsl_mc_device *mc_dev,
 		goto error_cleanup_resource;
 
 	mc_portal_phys_addr = dpmcp_dev->regions[0].start;
-	mc_portal_size = dpmcp_dev->regions[0].end -
-			 dpmcp_dev->regions[0].start + 1;
+	mc_portal_size = resource_size(dpmcp_dev->regions);
 
 	if (WARN_ON(mc_portal_size != mc_bus_dev->mc_io->portal_size))
 		goto error_cleanup_resource;
diff --git a/drivers/staging/fsl-mc/bus/mc-sys.c b/drivers/staging/fsl-mc/bus/mc-sys.c
index a1704c3a6a78..7ce105bd3977 100644
--- a/drivers/staging/fsl-mc/bus/mc-sys.c
+++ b/drivers/staging/fsl-mc/bus/mc-sys.c
@@ -37,6 +37,7 @@
 #include <linux/ioport.h>
 #include <linux/device.h>
 #include <linux/io.h>
+#include <linux/io-64-nonatomic-hi-lo.h>
 #include "../include/mc.h"
 
 #include "dpmcp.h"
@@ -126,11 +127,15 @@ static inline void mc_write_command(struct mc_command __iomem *portal,
 
 	/* copy command parameters into the portal */
 	for (i = 0; i < MC_CMD_NUM_OF_PARAMS; i++)
-		__raw_writeq(cmd->params[i], &portal->params[i]);
-	__iowmb();
+		/*
+		 * Data is already in the expected LE byte-order. Do an
+		 * extra LE -> CPU conversion so that the CPU -> LE done in
+		 * the device io write api puts it back in the right order.
+		 */
+		writeq_relaxed(le64_to_cpu(cmd->params[i]), &portal->params[i]);
 
 	/* submit the command by writing the header */
-	__raw_writeq(cmd->header, &portal->header);
+	writeq(le64_to_cpu(cmd->header), &portal->header);
 }
 
 /**
@@ -150,17 +155,20 @@ static inline enum mc_cmd_status mc_read_response(struct mc_command __iomem *
 	enum mc_cmd_status status;
 
 	/* Copy command response header from MC portal: */
-	__iormb();
-	resp->header = __raw_readq(&portal->header);
-	__iormb();
+	resp->header = cpu_to_le64(readq_relaxed(&portal->header));
 	status = mc_cmd_hdr_read_status(resp);
 	if (status != MC_CMD_STATUS_OK)
 		return status;
 
 	/* Copy command response data from MC portal: */
 	for (i = 0; i < MC_CMD_NUM_OF_PARAMS; i++)
-		resp->params[i] = __raw_readq(&portal->params[i]);
-	__iormb();
+		/*
+		 * Data is expected to be in LE byte-order. Do an
+		 * extra CPU -> LE to revert the LE -> CPU done in
+		 * the device io read api.
+		 */
+		resp->params[i] =
+			cpu_to_le64(readq_relaxed(&portal->params[i]));
 
 	return status;
 }
@@ -198,8 +206,8 @@ static int mc_polling_wait_preemptible(struct fsl_mc_io *mc_io,
 
 		if (time_after_eq(jiffies, jiffies_until_timeout)) {
 			dev_dbg(mc_io->dev,
-				"MC command timed out (portal: %#llx, dprc handle: %#x, command: %#x)\n",
-				 mc_io->portal_phys_addr,
+				"MC command timed out (portal: %pa, dprc handle: %#x, command: %#x)\n",
+				 &mc_io->portal_phys_addr,
 				 (unsigned int)mc_cmd_hdr_read_token(cmd),
 				 (unsigned int)mc_cmd_hdr_read_cmdid(cmd));
 
@@ -238,8 +246,8 @@ static int mc_polling_wait_atomic(struct fsl_mc_io *mc_io,
 		timeout_usecs -= MC_CMD_COMPLETION_POLLING_MAX_SLEEP_USECS;
 		if (timeout_usecs == 0) {
 			dev_dbg(mc_io->dev,
-				"MC command timed out (portal: %#llx, dprc handle: %#x, command: %#x)\n",
-				 mc_io->portal_phys_addr,
+				"MC command timed out (portal: %pa, dprc handle: %#x, command: %#x)\n",
+				 &mc_io->portal_phys_addr,
 				 (unsigned int)mc_cmd_hdr_read_token(cmd),
 				 (unsigned int)mc_cmd_hdr_read_cmdid(cmd));
 
@@ -292,8 +300,8 @@ int mc_send_command(struct fsl_mc_io *mc_io, struct mc_command *cmd)
 
 	if (status != MC_CMD_STATUS_OK) {
 		dev_dbg(mc_io->dev,
-			"MC command failed: portal: %#llx, dprc handle: %#x, command: %#x, status: %s (%#x)\n",
-			 mc_io->portal_phys_addr,
+			"MC command failed: portal: %pa, dprc handle: %#x, command: %#x, status: %s (%#x)\n",
+			 &mc_io->portal_phys_addr,
 			 (unsigned int)mc_cmd_hdr_read_token(cmd),
 			 (unsigned int)mc_cmd_hdr_read_cmdid(cmd),
 			 mc_status_to_string(status),
diff --git a/drivers/staging/fsl-mc/include/dpaa2-io.h b/drivers/staging/fsl-mc/include/dpaa2-io.h
index 002829cecd75..c5646096c5d4 100644
--- a/drivers/staging/fsl-mc/include/dpaa2-io.h
+++ b/drivers/staging/fsl-mc/include/dpaa2-io.h
@@ -34,6 +34,7 @@
 
 #include <linux/types.h>
 #include <linux/cpumask.h>
+#include <linux/irqreturn.h>
 
 #include "dpaa2-fd.h"
 #include "dpaa2-global.h"
diff --git a/drivers/staging/goldfish/goldfish_nand.c b/drivers/staging/goldfish/goldfish_nand.c
index 8f92ff4ba4b8..52cc1363993e 100644
--- a/drivers/staging/goldfish/goldfish_nand.c
+++ b/drivers/staging/goldfish/goldfish_nand.c
@@ -153,12 +153,12 @@ static int goldfish_nand_read_oob(struct mtd_info *mtd, loff_t ofs,
 	ofs += mtd->writesize + ops->ooboffs;
 	if (ops->oobbuf)
 		ops->oobretlen = goldfish_nand_cmd(mtd, NAND_CMD_READ, ofs,
-						ops->ooblen, ops->oobbuf);
+						   ops->ooblen, ops->oobbuf);
 	return 0;
 
 invalid_arg:
-	pr_err("goldfish_nand_read_oob: invalid read, start %llx, len %zx, ooblen %zx, dev_size %llx, write_size %x\n",
-	       ofs, ops->len, ops->ooblen, mtd->size, mtd->writesize);
+	pr_err("%s: invalid read, start %llx, len %zx, ooblen %zx, dev_size %llx, write_size %x\n",
+	       __func__, ofs, ops->len, ops->ooblen, mtd->size, mtd->writesize);
 	return -EINVAL;
 }
 
@@ -185,12 +185,12 @@ static int goldfish_nand_write_oob(struct mtd_info *mtd, loff_t ofs,
 	ofs += mtd->writesize + ops->ooboffs;
 	if (ops->oobbuf)
 		ops->oobretlen = goldfish_nand_cmd(mtd, NAND_CMD_WRITE, ofs,
-						ops->ooblen, ops->oobbuf);
+						   ops->ooblen, ops->oobbuf);
 	return 0;
 
 invalid_arg:
-	pr_err("goldfish_nand_write_oob: invalid write, start %llx, len %zx, ooblen %zx, dev_size %llx, write_size %x\n",
-	       ofs, ops->len, ops->ooblen, mtd->size, mtd->writesize);
+	pr_err("%s: invalid write, start %llx, len %zx, ooblen %zx, dev_size %llx, write_size %x\n",
+	       __func__, ofs, ops->len, ops->ooblen, mtd->size, mtd->writesize);
 	return -EINVAL;
 }
 
@@ -211,8 +211,8 @@ static int goldfish_nand_read(struct mtd_info *mtd, loff_t from, size_t len,
 	return 0;
 
 invalid_arg:
-	pr_err("goldfish_nand_read: invalid read, start %llx, len %zx, dev_size %llx, write_size %x\n",
-	       from, len, mtd->size, mtd->writesize);
+	pr_err("%s: invalid read, start %llx, len %zx, dev_size %llx, write_size %x\n",
+	       __func__, from, len, mtd->size, mtd->writesize);
 	return -EINVAL;
 }
 
@@ -233,8 +233,8 @@ static int goldfish_nand_write(struct mtd_info *mtd, loff_t to, size_t len,
 	return 0;
 
 invalid_arg:
-	pr_err("goldfish_nand_write: invalid write, start %llx, len %zx, dev_size %llx, write_size %x\n",
-	       to, len, mtd->size, mtd->writesize);
+	pr_err("%s: invalid write, start %llx, len %zx, dev_size %llx, write_size %x\n",
+	       __func__, to, len, mtd->size, mtd->writesize);
 	return -EINVAL;
 }
 
@@ -340,8 +340,8 @@ static int goldfish_nand_init_device(struct platform_device *pdev,
 				   name);
 	if (result != name_len) {
 		dev_err(&pdev->dev,
-			"goldfish_nand_init_device failed to get dev name %d != %d\n",
-			       result, name_len);
+			"%s: failed to get dev name %d != %d\n",
+			       __func__, result, name_len);
 		return -ENODEV;
 	}
 	((char *)mtd->name)[name_len] = '\0';
diff --git a/drivers/staging/greybus/arche-platform.c b/drivers/staging/greybus/arche-platform.c
index eced2d26467b..21ac92d0f533 100644
--- a/drivers/staging/greybus/arche-platform.c
+++ b/drivers/staging/greybus/arche-platform.c
@@ -176,7 +176,10 @@ static irqreturn_t arche_platform_wd_irq(int irq, void *devid)
 				arche_platform_set_wake_detect_state(arche_pdata,
 								     WD_STATE_IDLE);
 			} else {
-				/* Check we are not in middle of irq thread already */
+				/*
+				 * Check we are not in middle of irq thread
+				 * already
+				 */
 				if (arche_pdata->wake_detect_state !=
 						WD_STATE_COLDBOOT_START) {
 					arche_platform_set_wake_detect_state(arche_pdata,
@@ -193,7 +196,7 @@ static irqreturn_t arche_platform_wd_irq(int irq, void *devid)
 		if (arche_pdata->wake_detect_state == WD_STATE_IDLE) {
 			arche_pdata->wake_detect_start = jiffies;
 			/*
-			 * In the begining, when wake/detect goes low
+			 * In the beginning, when wake/detect goes low
 			 * (first time), we assume it is meant for coldboot
 			 * and set the flag. If wake/detect line stays low
 			 * beyond 30msec, then it is coldboot else fallback
@@ -607,7 +610,6 @@ static int arche_platform_remove(struct platform_device *pdev)
 	device_remove_file(&pdev->dev, &dev_attr_state);
 	device_for_each_child(&pdev->dev, NULL, arche_remove_child);
 	arche_platform_poweroff_seq(arche_pdata);
-	platform_set_drvdata(pdev, NULL);
 
 	if (usb3613_hub_mode_ctrl(false))
 		dev_warn(arche_pdata->dev, "failed to control hub device\n");
@@ -657,12 +659,14 @@ static SIMPLE_DEV_PM_OPS(arche_platform_pm_ops,
 			arche_platform_resume);
 
 static const struct of_device_id arche_platform_of_match[] = {
-	{ .compatible = "google,arche-platform", }, /* Use PID/VID of SVC device */
+	/* Use PID/VID of SVC device */
+	{ .compatible = "google,arche-platform", },
 	{ },
 };
 
 static const struct of_device_id arche_combined_id[] = {
-	{ .compatible = "google,arche-platform", }, /* Use PID/VID of SVC device */
+	/* Use PID/VID of SVC device */
+	{ .compatible = "google,arche-platform", },
 	{ .compatible = "usbffff,2", },
 	{ },
 };
diff --git a/drivers/staging/greybus/audio_codec.c b/drivers/staging/greybus/audio_codec.c
index 25c8bb4cb0de..a6d01f0761f3 100644
--- a/drivers/staging/greybus/audio_codec.c
+++ b/drivers/staging/greybus/audio_codec.c
@@ -674,7 +674,7 @@ static int gbcodec_mute_stream(struct snd_soc_dai *dai, int mute, int stream)
 	return ret;
 }
 
-static struct snd_soc_dai_ops gbcodec_dai_ops = {
+static const struct snd_soc_dai_ops gbcodec_dai_ops = {
 	.startup = gbcodec_startup,
 	.shutdown = gbcodec_shutdown,
 	.hw_params = gbcodec_hw_params,
diff --git a/drivers/staging/greybus/gbphy.c b/drivers/staging/greybus/gbphy.c
index 603de6f21fd1..80c1da8224e6 100644
--- a/drivers/staging/greybus/gbphy.c
+++ b/drivers/staging/greybus/gbphy.c
@@ -66,7 +66,7 @@ static const struct dev_pm_ops gb_gbphy_pm_ops = {
 			   gb_gbphy_idle)
 };
 
-static struct device_type greybus_gbphy_dev_type = {
+static const struct device_type greybus_gbphy_dev_type = {
 	.name	 =	"gbphy_device",
 	.release =	gbphy_dev_release,
 	.pm	=	&gb_gbphy_pm_ops,
diff --git a/drivers/staging/greybus/interface.c b/drivers/staging/greybus/interface.c
index a4fd51632232..71e5cc234e78 100644
--- a/drivers/staging/greybus/interface.c
+++ b/drivers/staging/greybus/interface.c
@@ -47,7 +47,7 @@ static int gb_interface_hibernate_link(struct gb_interface *intf);
 static int gb_interface_refclk_set(struct gb_interface *intf, bool enable);
 
 static int gb_interface_dme_attr_get(struct gb_interface *intf,
-							u16 attr, u32 *val)
+				     u16 attr, u32 *val)
 {
 	return gb_svc_dme_peer_get(intf->hd->svc, intf->interface_id,
 					attr, DME_SELECTOR_INDEX_NULL, val);
@@ -64,7 +64,7 @@ static int gb_interface_read_ara_dme(struct gb_interface *intf)
 	 */
 	if (intf->ddbl1_manufacturer_id != TOSHIBA_DMID) {
 		dev_err(&intf->dev, "unknown manufacturer %08x\n",
-				intf->ddbl1_manufacturer_id);
+			intf->ddbl1_manufacturer_id);
 		return -ENODEV;
 	}
 
@@ -110,7 +110,7 @@ static int gb_interface_read_dme(struct gb_interface *intf)
 		return ret;
 
 	if (intf->ddbl1_manufacturer_id == TOSHIBA_DMID &&
-			intf->ddbl1_product_id == TOSHIBA_ES2_BRIDGE_DPID) {
+	    intf->ddbl1_product_id == TOSHIBA_ES2_BRIDGE_DPID) {
 		intf->quirks |= GB_INTERFACE_QUIRK_NO_GMP_IDS;
 		intf->quirks |= GB_INTERFACE_QUIRK_NO_INIT_STATUS;
 	}
@@ -144,7 +144,7 @@ static int gb_interface_route_create(struct gb_interface *intf)
 	ret = gb_svc_intf_device_id(svc, intf_id, device_id);
 	if (ret) {
 		dev_err(&intf->dev, "failed to set device id %u: %d\n",
-				device_id, ret);
+			device_id, ret);
 		goto err_ida_remove;
 	}
 
@@ -205,21 +205,21 @@ static int gb_interface_legacy_mode_switch(struct gb_interface *intf)
 }
 
 void gb_interface_mailbox_event(struct gb_interface *intf, u16 result,
-								u32 mailbox)
+				u32 mailbox)
 {
 	mutex_lock(&intf->mutex);
 
 	if (result) {
 		dev_warn(&intf->dev,
-				"mailbox event with UniPro error: 0x%04x\n",
-				result);
+			 "mailbox event with UniPro error: 0x%04x\n",
+			 result);
 		goto err_disable;
 	}
 
 	if (mailbox != GB_SVC_INTF_MAILBOX_GREYBUS) {
 		dev_warn(&intf->dev,
-				"mailbox event with unexpected value: 0x%08x\n",
-				mailbox);
+			 "mailbox event with unexpected value: 0x%08x\n",
+			 mailbox);
 		goto err_disable;
 	}
 
@@ -230,7 +230,7 @@ void gb_interface_mailbox_event(struct gb_interface *intf, u16 result,
 
 	if (!intf->mode_switch) {
 		dev_warn(&intf->dev, "unexpected mailbox event: 0x%08x\n",
-				mailbox);
+			 mailbox);
 		goto err_disable;
 	}
 
@@ -299,7 +299,7 @@ static void gb_interface_mode_switch_work(struct work_struct *work)
 		ret = gb_interface_enable(intf);
 		if (ret) {
 			dev_err(&intf->dev, "failed to re-enable interface: %d\n",
-					ret);
+				ret);
 			gb_interface_deactivate(intf);
 		}
 	}
@@ -619,7 +619,7 @@ static struct attribute *interface_common_attrs[] = {
 };
 
 static umode_t interface_unipro_is_visible(struct kobject *kobj,
-						struct attribute *attr, int n)
+					   struct attribute *attr, int n)
 {
 	struct device *dev = container_of(kobj, struct device, kobj);
 	struct gb_interface *intf = to_gb_interface(dev);
@@ -634,7 +634,7 @@ static umode_t interface_unipro_is_visible(struct kobject *kobj,
 }
 
 static umode_t interface_greybus_is_visible(struct kobject *kobj,
-						struct attribute *attr, int n)
+					    struct attribute *attr, int n)
 {
 	struct device *dev = container_of(kobj, struct device, kobj);
 	struct gb_interface *intf = to_gb_interface(dev);
@@ -648,7 +648,7 @@ static umode_t interface_greybus_is_visible(struct kobject *kobj,
 }
 
 static umode_t interface_power_is_visible(struct kobject *kobj,
-						struct attribute *attr, int n)
+					  struct attribute *attr, int n)
 {
 	struct device *dev = container_of(kobj, struct device, kobj);
 	struct gb_interface *intf = to_gb_interface(dev);
@@ -813,7 +813,7 @@ struct gb_interface *gb_interface_create(struct gb_module *module,
 	intf->dev.dma_mask = module->dev.dma_mask;
 	device_initialize(&intf->dev);
 	dev_set_name(&intf->dev, "%s.%u", dev_name(&module->dev),
-			interface_id);
+		     interface_id);
 
 	pm_runtime_set_autosuspend_delay(&intf->dev,
 					 GB_INTERFACE_AUTOSUSPEND_MS);
@@ -1083,7 +1083,7 @@ int gb_interface_enable(struct gb_interface *intf)
 	control = gb_control_create(intf);
 	if (IS_ERR(control)) {
 		dev_err(&intf->dev, "failed to create control device: %ld\n",
-				PTR_ERR(control));
+			PTR_ERR(control));
 		return PTR_ERR(control);
 	}
 	intf->control = control;
@@ -1228,17 +1228,17 @@ int gb_interface_add(struct gb_interface *intf)
 	trace_gb_interface_add(intf);
 
 	dev_info(&intf->dev, "Interface added (%s)\n",
-			gb_interface_type_string(intf));
+		 gb_interface_type_string(intf));
 
 	switch (intf->type) {
 	case GB_INTERFACE_TYPE_GREYBUS:
 		dev_info(&intf->dev, "GMP VID=0x%08x, PID=0x%08x\n",
-				intf->vendor_id, intf->product_id);
+			 intf->vendor_id, intf->product_id);
 		/* fall-through */
 	case GB_INTERFACE_TYPE_UNIPRO:
 		dev_info(&intf->dev, "DDBL1 Manufacturer=0x%08x, Product=0x%08x\n",
-				intf->ddbl1_manufacturer_id,
-				intf->ddbl1_product_id);
+			 intf->ddbl1_manufacturer_id,
+			 intf->ddbl1_product_id);
 		break;
 	default:
 		break;
diff --git a/drivers/staging/greybus/light.c b/drivers/staging/greybus/light.c
index 861a249e6ef1..3f4148c92308 100644
--- a/drivers/staging/greybus/light.c
+++ b/drivers/staging/greybus/light.c
@@ -58,6 +58,7 @@ struct gb_light {
 	bool			ready;
 #if IS_REACHABLE(CONFIG_V4L2_FLASH_LED_CLASS)
 	struct v4l2_flash	*v4l2_flash;
+	struct v4l2_flash	*v4l2_flash_ind;
 #endif
 };
 
@@ -534,26 +535,21 @@ static int gb_lights_light_v4l2_register(struct gb_light *light)
 {
 	struct gb_connection *connection = get_conn_from_light(light);
 	struct device *dev = &connection->bundle->dev;
-	struct v4l2_flash_config *sd_cfg;
+	struct v4l2_flash_config sd_cfg = { {0} }, sd_cfg_ind = { {0} };
 	struct led_classdev_flash *fled;
-	struct led_classdev_flash *iled = NULL;
+	struct led_classdev *iled = NULL;
 	struct gb_channel *channel_torch, *channel_ind, *channel_flash;
-	int ret = 0;
-
-	sd_cfg = kcalloc(1, sizeof(*sd_cfg), GFP_KERNEL);
-	if (!sd_cfg)
-		return -ENOMEM;
 
 	channel_torch = get_channel_from_mode(light, GB_CHANNEL_MODE_TORCH);
 	if (channel_torch)
 		__gb_lights_channel_v4l2_config(&channel_torch->intensity_uA,
-						&sd_cfg->torch_intensity);
+						&sd_cfg.intensity);
 
 	channel_ind = get_channel_from_mode(light, GB_CHANNEL_MODE_INDICATOR);
 	if (channel_ind) {
 		__gb_lights_channel_v4l2_config(&channel_ind->intensity_uA,
-						&sd_cfg->indicator_intensity);
-		iled = &channel_ind->fled;
+						&sd_cfg_ind.intensity);
+		iled = &channel_ind->fled.led_cdev;
 	}
 
 	channel_flash = get_channel_from_mode(light, GB_CHANNEL_MODE_FLASH);
@@ -561,31 +557,37 @@ static int gb_lights_light_v4l2_register(struct gb_light *light)
 
 	fled = &channel_flash->fled;
 
-	snprintf(sd_cfg->dev_name, sizeof(sd_cfg->dev_name), "%s", light->name);
+	snprintf(sd_cfg.dev_name, sizeof(sd_cfg.dev_name), "%s", light->name);
+	snprintf(sd_cfg_ind.dev_name, sizeof(sd_cfg_ind.dev_name),
+		 "%s indicator", light->name);
 
 	/* Set the possible values to faults, in our case all faults */
-	sd_cfg->flash_faults = LED_FAULT_OVER_VOLTAGE | LED_FAULT_TIMEOUT |
+	sd_cfg.flash_faults = LED_FAULT_OVER_VOLTAGE | LED_FAULT_TIMEOUT |
 		LED_FAULT_OVER_TEMPERATURE | LED_FAULT_SHORT_CIRCUIT |
 		LED_FAULT_OVER_CURRENT | LED_FAULT_INDICATOR |
 		LED_FAULT_UNDER_VOLTAGE | LED_FAULT_INPUT_VOLTAGE |
 		LED_FAULT_LED_OVER_TEMPERATURE;
 
-	light->v4l2_flash = v4l2_flash_init(dev, NULL, fled, iled,
-					    &v4l2_flash_ops, sd_cfg);
-	if (IS_ERR_OR_NULL(light->v4l2_flash)) {
-		ret = PTR_ERR(light->v4l2_flash);
-		goto out_free;
-	}
+	light->v4l2_flash = v4l2_flash_init(dev, NULL, fled, &v4l2_flash_ops,
+					    &sd_cfg);
+	if (IS_ERR(light->v4l2_flash))
+		return PTR_ERR(light->v4l2_flash);
 
-	return ret;
+	if (channel_ind) {
+		light->v4l2_flash_ind =
+			v4l2_flash_indicator_init(dev, NULL, iled, &sd_cfg_ind);
+		if (IS_ERR(light->v4l2_flash_ind)) {
+			v4l2_flash_release(light->v4l2_flash);
+			return PTR_ERR(light->v4l2_flash_ind);
+		}
+	}
 
-out_free:
-	kfree(sd_cfg);
-	return ret;
+	return 0;
 }
 
 static void gb_lights_light_v4l2_unregister(struct gb_light *light)
 {
+	v4l2_flash_release(light->v4l2_flash_ind);
 	v4l2_flash_release(light->v4l2_flash);
 }
 #else
diff --git a/drivers/staging/greybus/spilib.h b/drivers/staging/greybus/spilib.h
index 566d0dde7f79..cb6092578a92 100644
--- a/drivers/staging/greybus/spilib.h
+++ b/drivers/staging/greybus/spilib.h
@@ -18,7 +18,8 @@ struct spilib_ops {
 	void (*unprepare_transfer_hardware)(struct device *dev);
 };
 
-int gb_spilib_master_init(struct gb_connection *connection, struct device *dev, struct spilib_ops *ops);
+int gb_spilib_master_init(struct gb_connection *connection,
+			  struct device *dev, struct spilib_ops *ops);
 void gb_spilib_master_exit(struct gb_connection *connection);
 
 #endif /* __SPILIB_H */
diff --git a/drivers/staging/greybus/tools/loopback_test.c b/drivers/staging/greybus/tools/loopback_test.c
index 32a43693181c..fbe589fca840 100644
--- a/drivers/staging/greybus/tools/loopback_test.c
+++ b/drivers/staging/greybus/tools/loopback_test.c
@@ -420,10 +420,10 @@ void log_csv_error(int len, int err)
 }
 
 int format_output(struct loopback_test *t,
-			struct loopback_results *r,
-			const char *dev_name,
-			char *buf, int buf_len,
-			struct tm *tm)
+		  struct loopback_results *r,
+		  const char *dev_name,
+		  char *buf, int buf_len,
+		  struct tm *tm)
 {
 	int len = 0;
 
@@ -528,14 +528,14 @@ static int log_results(struct loopback_test *t)
 	tm = *localtime(&local_time);
 
 	/*
-	* file name will test_name_size_iteration_max.csv
-	* every time the same test with the same parameters is run we will then
-	* append to the same CSV with datestamp - representing each test
-	* dataset.
-	*/
+	 * file name will test_name_size_iteration_max.csv
+	 * every time the same test with the same parameters is run we will then
+	 * append to the same CSV with datestamp - representing each test
+	 * dataset.
+	 */
 	if (t->file_output && !t->porcelain) {
 		snprintf(file_name, sizeof(file_name), "%s_%d_%d.csv",
-			t->test_name, t->size, t->iteration_max);
+			 t->test_name, t->size, t->iteration_max);
 
 		fd = open(file_name, O_WRONLY | O_CREAT | O_APPEND, 0644);
 		if (fd < 0) {
@@ -549,8 +549,8 @@ static int log_results(struct loopback_test *t)
 			continue;
 
 		len = format_output(t, &t->devices[i].results,
-					t->devices[i].name,
-					data, sizeof(data), &tm);
+				    t->devices[i].name,
+				    data, sizeof(data), &tm);
 		if (t->file_output && !t->porcelain) {
 			ret = write(fd, data, len);
 			if (ret == -1)
@@ -562,7 +562,7 @@ static int log_results(struct loopback_test *t)
 
 	if (t->aggregate_output) {
 		len = format_output(t, &t->aggregate_results, "aggregate",
-					data, sizeof(data), &tm);
+				    data, sizeof(data), &tm);
 		if (t->file_output && !t->porcelain) {
 			ret = write(fd, data, len);
 			if (ret == -1)
@@ -623,14 +623,13 @@ int find_loopback_devices(struct loopback_test *t)
 		snprintf(d->name, MAX_STR_LEN, "gb_loopback%u", dev_id);
 
 		snprintf(d->sysfs_entry, MAX_SYSFS_PATH, "%s%s/",
-			t->sysfs_prefix, d->name);
+			 t->sysfs_prefix, d->name);
 
 		snprintf(d->debugfs_entry, MAX_SYSFS_PATH, "%sraw_latency_%s",
-			t->debugfs_prefix, d->name);
+			 t->debugfs_prefix, d->name);
 
 		if (t->debug)
-			printf("add %s %s\n", d->sysfs_entry,
-				d->debugfs_entry);
+			printf("add %s %s\n", d->sysfs_entry, d->debugfs_entry);
 	}
 
 	ret = 0;
@@ -779,7 +778,8 @@ static void prepare_devices(struct loopback_test *t)
 {
 	int i;
 
-	/* Cancel any running tests on enabled devices. If
+	/*
+	 * Cancel any running tests on enabled devices. If
 	 * stop_all option is given, stop test on all devices.
 	 */
 	for (i = 0; i < t->device_count; i++)
@@ -802,16 +802,14 @@ static void prepare_devices(struct loopback_test *t)
 				t->iteration_max);
 
 		if (t->use_async) {
+			write_sysfs_val(t->devices[i].sysfs_entry, "async", 1);
 			write_sysfs_val(t->devices[i].sysfs_entry,
-				"async", 1);
+					"timeout", t->async_timeout);
 			write_sysfs_val(t->devices[i].sysfs_entry,
-				"timeout", t->async_timeout);
-			write_sysfs_val(t->devices[i].sysfs_entry,
-				"outstanding_operations_max",
-				t->async_outstanding_operations);
+					"outstanding_operations_max",
+					t->async_outstanding_operations);
 		} else
-			write_sysfs_val(t->devices[i].sysfs_entry,
-				"async", 0);
+			write_sysfs_val(t->devices[i].sysfs_entry, "async", 0);
 	}
 }
 
diff --git a/drivers/staging/greybus/usb.c b/drivers/staging/greybus/usb.c
index ccadda084b76..f93a76d02de6 100644
--- a/drivers/staging/greybus/usb.c
+++ b/drivers/staging/greybus/usb.c
@@ -139,7 +139,7 @@ out:
 	return ret;
 }
 
-static struct hc_driver usb_gb_hc_driver = {
+static const struct hc_driver usb_gb_hc_driver = {
 	.description = "greybus-hcd",
 	.product_desc = "Greybus USB Host Controller",
 	.hcd_priv_size = sizeof(struct gb_usb_device),
diff --git a/drivers/staging/greybus/vibrator.c b/drivers/staging/greybus/vibrator.c
index 77a2365a55e6..5cd8a50d41ad 100644
--- a/drivers/staging/greybus/vibrator.c
+++ b/drivers/staging/greybus/vibrator.c
@@ -34,7 +34,7 @@ static int turn_off(struct gb_vibrator_device *vib)
 	int ret;
 
 	ret = gb_operation_sync(vib->connection, GB_VIBRATOR_TYPE_OFF,
-			NULL, 0, NULL, 0);
+				NULL, 0, NULL, 0);
 
 	gb_pm_runtime_put_autosuspend(bundle);
 
@@ -55,7 +55,7 @@ static int turn_on(struct gb_vibrator_device *vib, u16 timeout_ms)
 		turn_off(vib);
 
 	ret = gb_operation_sync(vib->connection, GB_VIBRATOR_TYPE_ON,
-			NULL, 0, NULL, 0);
+				NULL, 0, NULL, 0);
 	if (ret) {
 		gb_pm_runtime_put_autosuspend(bundle);
 		return ret;
@@ -116,7 +116,7 @@ static struct class vibrator_class = {
 static DEFINE_IDA(minors);
 
 static int gb_vibrator_probe(struct gb_bundle *bundle,
-					const struct greybus_bundle_id *id)
+			     const struct greybus_bundle_id *id)
 {
 	struct greybus_descriptor_cport *cport_desc;
 	struct gb_connection *connection;
@@ -136,7 +136,7 @@ static int gb_vibrator_probe(struct gb_bundle *bundle,
 		return -ENOMEM;
 
 	connection = gb_connection_create(bundle, le16_to_cpu(cport_desc->id),
-						NULL);
+					  NULL);
 	if (IS_ERR(connection)) {
 		retval = PTR_ERR(connection);
 		goto err_free_vib;
diff --git a/drivers/staging/gs_fpgaboot/gs_fpgaboot.c b/drivers/staging/gs_fpgaboot/gs_fpgaboot.c
index 19b550fff04b..bcbdc7340b55 100644
--- a/drivers/staging/gs_fpgaboot/gs_fpgaboot.c
+++ b/drivers/staging/gs_fpgaboot/gs_fpgaboot.c
@@ -23,6 +23,7 @@
 #include <linux/delay.h>
 #include <linux/io.h>
 #include <linux/firmware.h>
+#include <asm/unaligned.h>
 
 #include "gs_fpgaboot.h"
 #include "io.h"
@@ -41,16 +42,16 @@ static char	*file = "xlinx_fpga_firmware.bit";
 module_param(file, charp, 0444);
 MODULE_PARM_DESC(file, "Xilinx FPGA firmware file.");
 
-static void read_bitstream(char *bitdata, char *buf, int *offset, int rdsize)
+static void read_bitstream(u8 *bitdata, u8 *buf, int *offset, int rdsize)
 {
 	memcpy(buf, bitdata + *offset, rdsize);
 	*offset += rdsize;
 }
 
-static void readinfo_bitstream(char *bitdata, char *buf, int *offset)
+static int readinfo_bitstream(u8 *bitdata, u8 *buf, int size, int *offset)
 {
-	char tbuf[64];
-	s32 len;
+	u8 tbuf[2];
+	u16 len;
 
 	/* read section char */
 	read_bitstream(bitdata, tbuf, offset, 1);
@@ -58,18 +59,24 @@ static void readinfo_bitstream(char *bitdata, char *buf, int *offset)
 	/* read length */
 	read_bitstream(bitdata, tbuf, offset, 2);
 
-	len = tbuf[0] << 8 | tbuf[1];
+	len = get_unaligned_be16(tbuf);
+	if (len >= size) {
+		pr_err("error: readinfo buffer too small\n");
+		return -EINVAL;
+	}
 
 	read_bitstream(bitdata, buf, offset, len);
 	buf[len] = '\0';
+
+	return 0;
 }
 
 /*
  * read bitdata length
  */
-static int readlength_bitstream(char *bitdata, int *lendata, int *offset)
+static int readlength_bitstream(u8 *bitdata, int *lendata, int *offset)
 {
-	char tbuf[64];
+	u8 tbuf[4];
 
 	/* read section char */
 	read_bitstream(bitdata, tbuf, offset, 1);
@@ -77,14 +84,13 @@ static int readlength_bitstream(char *bitdata, int *lendata, int *offset)
 	/* make sure it is section 'e' */
 	if (tbuf[0] != 'e') {
 		pr_err("error: length section is not 'e', but %c\n", tbuf[0]);
-		return -1;
+		return -EINVAL;
 	}
 
 	/* read 4bytes length */
 	read_bitstream(bitdata, tbuf, offset, 4);
 
-	*lendata = tbuf[0] << 24 | tbuf[1] << 16 |
-		tbuf[2] << 8 | tbuf[3];
+	*lendata = get_unaligned_be32(tbuf);
 
 	return 0;
 }
@@ -92,16 +98,16 @@ static int readlength_bitstream(char *bitdata, int *lendata, int *offset)
 /*
  * read first 13 bytes to check bitstream magic number
  */
-static int readmagic_bitstream(char *bitdata, int *offset)
+static int readmagic_bitstream(u8 *bitdata, int *offset)
 {
-	char buf[13];
+	u8 buf[13];
 	int r;
 
 	read_bitstream(bitdata, buf, offset, 13);
 	r = memcmp(buf, bits_magic, 13);
 	if (r) {
 		pr_err("error: corrupted header");
-		return -1;
+		return -EINVAL;
 	}
 	pr_info("bitstream file magic number Ok\n");
 
@@ -113,7 +119,7 @@ static int readmagic_bitstream(char *bitdata, int *offset)
 /*
  * NOTE: supports only bitstream format
  */
-static enum fmt_image get_imageformat(struct fpgaimage *fimage)
+static enum fmt_image get_imageformat(void)
 {
 	return f_bit;
 }
@@ -127,38 +133,58 @@ static void gs_print_header(struct fpgaimage *fimage)
 	pr_info("lendata: %d\n", fimage->lendata);
 }
 
-static void gs_read_bitstream(struct fpgaimage *fimage)
+static int gs_read_bitstream(struct fpgaimage *fimage)
 {
-	char *bitdata;
+	u8 *bitdata;
 	int offset;
+	int err;
 
 	offset = 0;
-	bitdata = (char *)fimage->fw_entry->data;
+	bitdata = (u8 *)fimage->fw_entry->data;
+
+	err = readmagic_bitstream(bitdata, &offset);
+	if (err)
+		return err;
+
+	err = readinfo_bitstream(bitdata, fimage->filename, MAX_STR, &offset);
+	if (err)
+		return err;
+	err = readinfo_bitstream(bitdata, fimage->part, MAX_STR, &offset);
+	if (err)
+		return err;
+	err = readinfo_bitstream(bitdata, fimage->date, MAX_STR, &offset);
+	if (err)
+		return err;
+	err = readinfo_bitstream(bitdata, fimage->time, MAX_STR, &offset);
+	if (err)
+		return err;
 
-	readmagic_bitstream(bitdata, &offset);
-	readinfo_bitstream(bitdata, fimage->filename, &offset);
-	readinfo_bitstream(bitdata, fimage->part, &offset);
-	readinfo_bitstream(bitdata, fimage->date, &offset);
-	readinfo_bitstream(bitdata, fimage->time, &offset);
-	readlength_bitstream(bitdata, &fimage->lendata, &offset);
+	err = readlength_bitstream(bitdata, &fimage->lendata, &offset);
+	if (err)
+		return err;
 
 	fimage->fpgadata = bitdata + offset;
+
+	return 0;
 }
 
 static int gs_read_image(struct fpgaimage *fimage)
 {
 	int img_fmt;
+	int err;
 
-	img_fmt = get_imageformat(fimage);
+	img_fmt = get_imageformat();
 
 	switch (img_fmt) {
 	case f_bit:
 		pr_info("image is bitstream format\n");
-		gs_read_bitstream(fimage);
+		err = gs_read_bitstream(fimage);
+		if (err)
+			return err;
 		break;
 	default:
 		pr_err("unsupported fpga image format\n");
-		return -1;
+		return -EINVAL;
 	}
 
 	gs_print_header(fimage);
@@ -183,11 +209,11 @@ static int gs_load_image(struct fpgaimage *fimage, char *fw_file)
 
 static int gs_download_image(struct fpgaimage *fimage, enum wbus bus_bytes)
 {
-	char *bitdata;
+	u8 *bitdata;
 	int size, i, cnt;
 
 	cnt = 0;
-	bitdata = (char *)fimage->fpgadata;
+	bitdata = (u8 *)fimage->fpgadata;
 	size = fimage->lendata;
 
 #ifdef DEBUG_FPGA
@@ -197,7 +223,7 @@ static int gs_download_image(struct fpgaimage *fimage, enum wbus bus_bytes)
 	if (!xl_supported_prog_bus_width(bus_bytes)) {
 		pr_err("unsupported program bus width %d\n",
 		       bus_bytes);
-		return -1;
+		return -EINVAL;
 	}
 
 	/* Bring csi_b, rdwr_b Low and program_b High */
@@ -224,7 +250,7 @@ static int gs_download_image(struct fpgaimage *fimage, enum wbus bus_bytes)
 	/* Check INIT_B */
 	if (xl_get_init_b() == 0) {
 		pr_err("init_b 0\n");
-		return -1;
+		return -EIO;
 	}
 
 	while (xl_get_done_b() == 0) {
@@ -236,7 +262,7 @@ static int gs_download_image(struct fpgaimage *fimage, enum wbus bus_bytes)
 
 	if (cnt > MAX_WAIT_DONE) {
 		pr_err("fpga download fail\n");
-		return -1;
+		return -EIO;
 	}
 
 	pr_info("download fpgaimage\n");
@@ -325,7 +351,7 @@ err_out2:
 err_out1:
 	kfree(fimage);
 
-	return -1;
+	return err;
 }
 
 static int __init gs_fpgaboot_init(void)
diff --git a/drivers/staging/gs_fpgaboot/gs_fpgaboot.h b/drivers/staging/gs_fpgaboot/gs_fpgaboot.h
index cd1eb2c4c940..986e841f6b5e 100644
--- a/drivers/staging/gs_fpgaboot/gs_fpgaboot.h
+++ b/drivers/staging/gs_fpgaboot/gs_fpgaboot.h
@@ -47,5 +47,5 @@ struct fpgaimage {
 	char	date[MAX_STR];
 	char	time[MAX_STR];
 	int	lendata;
-	char	*fpgadata;
+	u8	*fpgadata;
 };
diff --git a/drivers/staging/gs_fpgaboot/io.c b/drivers/staging/gs_fpgaboot/io.c
index c9391198fbfb..83a13ca7259a 100644
--- a/drivers/staging/gs_fpgaboot/io.c
+++ b/drivers/staging/gs_fpgaboot/io.c
@@ -9,10 +9,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  *
  * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  */
 
 #include <linux/kernel.h>
diff --git a/drivers/staging/iio/adc/ad7280a.c b/drivers/staging/iio/adc/ad7280a.c
index d5ab83f0236d..f85dde9805e0 100644
--- a/drivers/staging/iio/adc/ad7280a.c
+++ b/drivers/staging/iio/adc/ad7280a.c
@@ -99,9 +99,14 @@
 #define AD7280A_DEVADDR_MASTER		0
 #define AD7280A_DEVADDR_ALL		0x1F
 /* 5-bit device address is sent LSB first */
-#define AD7280A_DEVADDR(addr)	(((addr & 0x1) << 4) | ((addr & 0x2) << 3) | \
-				(addr & 0x4) | ((addr & 0x8) >> 3) | \
-				((addr & 0x10) >> 4))
+static unsigned int ad7280a_devaddr(unsigned int addr)
+{
+	return ((addr & 0x1) << 4) |
+	       ((addr & 0x2) << 3) |
+	       (addr & 0x4) |
+	       ((addr & 0x8) >> 3) |
+	       ((addr & 0x10) >> 4);
+}
 
 /* During a read a valid write is mandatory.
  * So writing to the highest available address (Address 0x1F)
@@ -372,7 +377,7 @@ static int ad7280_chain_setup(struct ad7280_state *st)
 		if (ad7280_check_crc(st, val))
 			return -EIO;
 
-		if (n != AD7280A_DEVADDR(val >> 27))
+		if (n != ad7280a_devaddr(val >> 27))
 			return -EIO;
 	}
 
@@ -511,7 +516,7 @@ static int ad7280_channel_init(struct ad7280_state *st)
 			st->channels[cnt].info_mask_shared_by_type =
 				BIT(IIO_CHAN_INFO_SCALE);
 			st->channels[cnt].address =
-				AD7280A_DEVADDR(dev) << 8 | ch;
+				ad7280a_devaddr(dev) << 8 | ch;
 			st->channels[cnt].scan_index = cnt;
 			st->channels[cnt].scan_type.sign = 'u';
 			st->channels[cnt].scan_type.realbits = 12;
@@ -558,7 +563,7 @@ static int ad7280_attr_init(struct ad7280_state *st)
 		for (ch = AD7280A_CELL_VOLTAGE_1; ch <= AD7280A_CELL_VOLTAGE_6;
 			ch++, cnt++) {
 			st->iio_attr[cnt].address =
-				AD7280A_DEVADDR(dev) << 8 | ch;
+				ad7280a_devaddr(dev) << 8 | ch;
 			st->iio_attr[cnt].dev_attr.attr.mode =
 				0644;
 			st->iio_attr[cnt].dev_attr.show =
@@ -574,7 +579,7 @@ static int ad7280_attr_init(struct ad7280_state *st)
 				&st->iio_attr[cnt].dev_attr.attr;
 			cnt++;
 			st->iio_attr[cnt].address =
-				AD7280A_DEVADDR(dev) << 8 |
+				ad7280a_devaddr(dev) << 8 |
 				(AD7280A_CB1_TIMER + ch);
 			st->iio_attr[cnt].dev_attr.attr.mode =
 				0644;
@@ -918,7 +923,7 @@ static int ad7280_probe(struct spi_device *spi)
 		if (ret)
 			goto error_unregister;
 
-		ret = ad7280_write(st, AD7280A_DEVADDR(st->slave_num),
+		ret = ad7280_write(st, ad7280a_devaddr(st->slave_num),
 				   AD7280A_ALERT, 0,
 				   AD7280A_ALERT_GEN_STATIC_HIGH |
 				   (pdata->chain_last_alert_ignore & 0xF));
diff --git a/drivers/staging/iio/adc/ad7606_par.c b/drivers/staging/iio/adc/ad7606_par.c
index cd6c410c0484..3eb6f8f312dd 100644
--- a/drivers/staging/iio/adc/ad7606_par.c
+++ b/drivers/staging/iio/adc/ad7606_par.c
@@ -57,8 +57,8 @@ static int ad7606_par_probe(struct platform_device *pdev)
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
-		dev_err(&pdev->dev, "no irq\n");
-		return -ENODEV;
+		dev_err(&pdev->dev, "no irq: %d\n", irq);
+		return irq;
 	}
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
diff --git a/drivers/staging/iio/light/tsl2x7x.c b/drivers/staging/iio/light/tsl2x7x.c
index 146719928fb3..786e93f16ce9 100644
--- a/drivers/staging/iio/light/tsl2x7x.c
+++ b/drivers/staging/iio/light/tsl2x7x.c
@@ -285,35 +285,6 @@ static const u8 device_channel_config[] = {
 };
 
 /**
- * tsl2x7x_i2c_read() - Read a byte from a register.
- * @client:	i2c client
- * @reg:	device register to read from
- * @*val:	pointer to location to store register contents.
- *
- */
-static int
-tsl2x7x_i2c_read(struct i2c_client *client, u8 reg, u8 *val)
-{
-	int ret;
-
-	/* select register to write */
-	ret = i2c_smbus_write_byte(client, (TSL2X7X_CMD_REG | reg));
-	if (ret < 0) {
-		dev_err(&client->dev, "failed to write register %x\n", reg);
-		return ret;
-	}
-
-	/* read the data */
-	ret = i2c_smbus_read_byte(client);
-	if (ret >= 0)
-		*val = (u8)ret;
-	else
-		dev_err(&client->dev, "failed to read register %x\n", reg);
-
-	return ret;
-}
-
-/**
  * tsl2x7x_get_lux() - Reads and calculates current lux value.
  * @indio_dev:	pointer to IIO device
  *
@@ -352,15 +323,15 @@ static int tsl2x7x_get_lux(struct iio_dev *indio_dev)
 		goto out_unlock;
 	}
 
-	ret = tsl2x7x_i2c_read(chip->client,
-			       (TSL2X7X_CMD_REG | TSL2X7X_STATUS), &buf[0]);
+	ret = i2c_smbus_read_byte_data(chip->client,
+				       TSL2X7X_CMD_REG | TSL2X7X_STATUS);
 	if (ret < 0) {
 		dev_err(&chip->client->dev,
 			"%s: Failed to read STATUS Reg\n", __func__);
 		goto out_unlock;
 	}
 	/* is data new & valid */
-	if (!(buf[0] & TSL2X7X_STA_ADC_VALID)) {
+	if (!(ret & TSL2X7X_STA_ADC_VALID)) {
 		dev_err(&chip->client->dev,
 			"%s: data not valid yet\n", __func__);
 		ret = chip->als_cur_info.lux; /* return LAST VALUE */
@@ -368,14 +339,16 @@ static int tsl2x7x_get_lux(struct iio_dev *indio_dev)
 	}
 
 	for (i = 0; i < 4; i++) {
-		ret = tsl2x7x_i2c_read(chip->client,
-				       (TSL2X7X_CMD_REG |
-				       (TSL2X7X_ALS_CHAN0LO + i)), &buf[i]);
+		int reg = TSL2X7X_CMD_REG | (TSL2X7X_ALS_CHAN0LO + i);
+
+		ret = i2c_smbus_read_byte_data(chip->client, reg);
 		if (ret < 0) {
 			dev_err(&chip->client->dev,
 				"failed to read. err=%x\n", ret);
 			goto out_unlock;
 		}
+
+		buf[i] = ret;
 	}
 
 	/* clear any existing interrupt status */
@@ -475,7 +448,6 @@ static int tsl2x7x_get_prox(struct iio_dev *indio_dev)
 {
 	int i;
 	int ret;
-	u8 status;
 	u8 chdata[2];
 	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
 
@@ -485,8 +457,8 @@ static int tsl2x7x_get_prox(struct iio_dev *indio_dev)
 		return -EBUSY;
 	}
 
-	ret = tsl2x7x_i2c_read(chip->client,
-			       (TSL2X7X_CMD_REG | TSL2X7X_STATUS), &status);
+	ret = i2c_smbus_read_byte_data(chip->client,
+				       TSL2X7X_CMD_REG | TSL2X7X_STATUS);
 	if (ret < 0) {
 		dev_err(&chip->client->dev, "i2c err=%d\n", ret);
 		goto prox_poll_err;
@@ -498,7 +470,7 @@ static int tsl2x7x_get_prox(struct iio_dev *indio_dev)
 	case tmd2671:
 	case tsl2771:
 	case tmd2771:
-		if (!(status & TSL2X7X_STA_ADC_VALID))
+		if (!(ret & TSL2X7X_STA_ADC_VALID))
 			goto prox_poll_err;
 	break;
 	case tsl2572:
@@ -506,17 +478,19 @@ static int tsl2x7x_get_prox(struct iio_dev *indio_dev)
 	case tmd2672:
 	case tsl2772:
 	case tmd2772:
-		if (!(status & TSL2X7X_STA_PRX_VALID))
+		if (!(ret & TSL2X7X_STA_PRX_VALID))
 			goto prox_poll_err;
 	break;
 	}
 
 	for (i = 0; i < 2; i++) {
-		ret = tsl2x7x_i2c_read(chip->client,
-				       (TSL2X7X_CMD_REG |
-				       (TSL2X7X_PRX_LO + i)), &chdata[i]);
+		int reg = TSL2X7X_CMD_REG | (TSL2X7X_PRX_LO + i);
+
+		ret = i2c_smbus_read_byte_data(chip->client, reg);
 		if (ret < 0)
 			goto prox_poll_err;
+
+		chdata[i] = ret;
 	}
 
 	chip->prox_data =
@@ -568,39 +542,29 @@ static void tsl2x7x_defaults(struct tsl2X7X_chip *chip)
 static int tsl2x7x_als_calibrate(struct iio_dev *indio_dev)
 {
 	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
-	u8 reg_val;
 	int gain_trim_val;
 	int ret;
 	int lux_val;
 
-	ret = i2c_smbus_write_byte(chip->client,
-				   (TSL2X7X_CMD_REG | TSL2X7X_CNTRL));
+	ret = i2c_smbus_read_byte_data(chip->client,
+				       TSL2X7X_CMD_REG | TSL2X7X_CNTRL);
 	if (ret < 0) {
 		dev_err(&chip->client->dev,
-			"failed to write CNTRL register, ret=%d\n", ret);
+			"%s: failed to read from the CNTRL register\n",
+			__func__);
 		return ret;
 	}
 
-	reg_val = i2c_smbus_read_byte(chip->client);
-	if ((reg_val & (TSL2X7X_CNTL_ADC_ENBL | TSL2X7X_CNTL_PWR_ON))
-		!= (TSL2X7X_CNTL_ADC_ENBL | TSL2X7X_CNTL_PWR_ON)) {
-		dev_err(&chip->client->dev,
-			"%s: failed: ADC not enabled\n", __func__);
-		return -1;
-	}
-
-	ret = i2c_smbus_write_byte(chip->client,
-				   (TSL2X7X_CMD_REG | TSL2X7X_CNTRL));
-	if (ret < 0) {
+	if ((ret & (TSL2X7X_CNTL_ADC_ENBL | TSL2X7X_CNTL_PWR_ON))
+			!= (TSL2X7X_CNTL_ADC_ENBL | TSL2X7X_CNTL_PWR_ON)) {
 		dev_err(&chip->client->dev,
-			"failed to write ctrl reg: ret=%d\n", ret);
-		return ret;
-	}
-
-	reg_val = i2c_smbus_read_byte(chip->client);
-	if ((reg_val & TSL2X7X_STA_ADC_VALID) != TSL2X7X_STA_ADC_VALID) {
+			"%s: Device is not powered on and/or ADC is not enabled\n",
+			__func__);
+		return -EINVAL;
+	} else if ((ret & TSL2X7X_STA_ADC_VALID) != TSL2X7X_STA_ADC_VALID) {
 		dev_err(&chip->client->dev,
-			"%s: failed: STATUS - ADC not valid.\n", __func__);
+			"%s: The two ADC channels have not completed an integration cycle\n",
+			__func__);
 		return -ENODATA;
 	}
 
@@ -722,7 +686,8 @@ static int tsl2x7x_chip_on(struct iio_dev *indio_dev)
 		}
 	}
 
-	mdelay(3);	/* Power-on settling time */
+	/* Power-on settling time */
+	usleep_range(3000, 3500);
 
 	/*
 	 * NOW enable the ADC
@@ -806,22 +771,24 @@ int tsl2x7x_invoke_change(struct iio_dev *indio_dev)
 {
 	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
 	int device_status = chip->tsl2x7x_chip_status;
+	int ret;
 
 	mutex_lock(&chip->als_mutex);
 	mutex_lock(&chip->prox_mutex);
 
-	if (device_status == TSL2X7X_CHIP_WORKING)
-		tsl2x7x_chip_off(indio_dev);
-
-	tsl2x7x_chip_on(indio_dev);
+	if (device_status == TSL2X7X_CHIP_WORKING) {
+		ret = tsl2x7x_chip_off(indio_dev);
+		if (ret < 0)
+			goto unlock;
+	}
 
-	if (device_status != TSL2X7X_CHIP_WORKING)
-		tsl2x7x_chip_off(indio_dev);
+	ret = tsl2x7x_chip_on(indio_dev);
 
+unlock:
 	mutex_unlock(&chip->prox_mutex);
 	mutex_unlock(&chip->als_mutex);
 
-	return 0;
+	return ret;
 }
 
 static
@@ -889,7 +856,7 @@ static void tsl2x7x_prox_cal(struct iio_dev *indio_dev)
 
 	/*gather the samples*/
 	for (i = 0; i < chip->tsl2x7x_settings.prox_max_samples_cal; i++) {
-		mdelay(15);
+		usleep_range(15000, 17500);
 		tsl2x7x_get_prox(indio_dev);
 		prox_history[i] = chip->prox_data;
 		dev_info(&chip->client->dev, "2 i=%d prox data= %d\n",
@@ -915,33 +882,6 @@ static void tsl2x7x_prox_cal(struct iio_dev *indio_dev)
 		tsl2x7x_chip_on(indio_dev);
 }
 
-static ssize_t power_state_show(struct device *dev,
-					struct device_attribute *attr,
-					char *buf)
-{
-	struct tsl2X7X_chip *chip = iio_priv(dev_to_iio_dev(dev));
-
-	return snprintf(buf, PAGE_SIZE, "%d\n", chip->tsl2x7x_chip_status);
-}
-
-static ssize_t power_state_store(struct device *dev,
-					 struct device_attribute *attr,
-					 const char *buf, size_t len)
-{
-	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
-	bool value;
-
-	if (strtobool(buf, &value))
-		return -EINVAL;
-
-	if (value)
-		tsl2x7x_chip_on(indio_dev);
-	else
-		tsl2x7x_chip_off(indio_dev);
-
-	return len;
-}
-
 static ssize_t in_illuminance0_calibscale_available_show(struct device *dev,
 					   struct device_attribute *attr,
 					   char *buf)
@@ -1027,6 +967,7 @@ static ssize_t in_illuminance0_target_input_store(struct device *dev,
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
 	unsigned long value;
+	int ret;
 
 	if (kstrtoul(buf, 0, &value))
 		return -EINVAL;
@@ -1034,7 +975,9 @@ static ssize_t in_illuminance0_target_input_store(struct device *dev,
 	if (value)
 		chip->tsl2x7x_settings.als_cal_target = value;
 
-	tsl2x7x_invoke_change(indio_dev);
+	ret = tsl2x7x_invoke_change(indio_dev);
+	if (ret < 0)
+		return ret;
 
 	return len;
 }
@@ -1083,7 +1026,9 @@ static ssize_t in_intensity0_thresh_period_store(struct device *dev,
 	dev_info(&chip->client->dev, "%s: als persistence = %d",
 		 __func__, filter_delay);
 
-	tsl2x7x_invoke_change(indio_dev);
+	ret = tsl2x7x_invoke_change(indio_dev);
+	if (ret < 0)
+		return ret;
 
 	return IIO_VAL_INT_PLUS_MICRO;
 }
@@ -1131,7 +1076,10 @@ static ssize_t in_proximity0_thresh_period_store(struct device *dev,
 	dev_info(&chip->client->dev, "%s: prox persistence = %d",
 		 __func__, filter_delay);
 
-	tsl2x7x_invoke_change(indio_dev);
+	ret = tsl2x7x_invoke_change(indio_dev);
+	if (ret < 0)
+		return ret;
+
 
 	return IIO_VAL_INT_PLUS_MICRO;
 }
@@ -1142,6 +1090,7 @@ static ssize_t in_illuminance0_calibrate_store(struct device *dev,
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	bool value;
+	int ret;
 
 	if (strtobool(buf, &value))
 		return -EINVAL;
@@ -1149,7 +1098,9 @@ static ssize_t in_illuminance0_calibrate_store(struct device *dev,
 	if (value)
 		tsl2x7x_als_calibrate(indio_dev);
 
-	tsl2x7x_invoke_change(indio_dev);
+	ret = tsl2x7x_invoke_change(indio_dev);
+	if (ret < 0)
+		return ret;
 
 	return len;
 }
@@ -1189,7 +1140,7 @@ static ssize_t in_illuminance0_lux_table_store(struct device *dev,
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
 	int value[ARRAY_SIZE(chip->tsl2x7x_device_lux) * 3 + 1];
-	int n;
+	int n, ret;
 
 	get_options(buf, ARRAY_SIZE(value), value);
 
@@ -1217,7 +1168,9 @@ static ssize_t in_illuminance0_lux_table_store(struct device *dev,
 	memset(chip->tsl2x7x_device_lux, 0, sizeof(chip->tsl2x7x_device_lux));
 	memcpy(chip->tsl2x7x_device_lux, &value[1], (value[0] * 4));
 
-	tsl2x7x_invoke_change(indio_dev);
+	ret = tsl2x7x_invoke_change(indio_dev);
+	if (ret < 0)
+		return ret;
 
 	return len;
 }
@@ -1228,6 +1181,7 @@ static ssize_t in_proximity0_calibrate_store(struct device *dev,
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	bool value;
+	int ret;
 
 	if (strtobool(buf, &value))
 		return -EINVAL;
@@ -1235,7 +1189,9 @@ static ssize_t in_proximity0_calibrate_store(struct device *dev,
 	if (value)
 		tsl2x7x_prox_cal(indio_dev);
 
-	tsl2x7x_invoke_change(indio_dev);
+	ret = tsl2x7x_invoke_change(indio_dev);
+	if (ret < 0)
+		return ret;
 
 	return len;
 }
@@ -1263,6 +1219,7 @@ static int tsl2x7x_write_interrupt_config(struct iio_dev *indio_dev,
 					  int val)
 {
 	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
+	int ret;
 
 	if (chan->type == IIO_INTENSITY) {
 		if (val)
@@ -1276,83 +1233,108 @@ static int tsl2x7x_write_interrupt_config(struct iio_dev *indio_dev,
 			chip->tsl2x7x_settings.interrupts_en &= 0x10;
 	}
 
-	tsl2x7x_invoke_change(indio_dev);
+	ret = tsl2x7x_invoke_change(indio_dev);
+	if (ret < 0)
+		return ret;
 
 	return 0;
 }
 
-static int tsl2x7x_write_thresh(struct iio_dev *indio_dev,
-				const struct iio_chan_spec *chan,
-				enum iio_event_type type,
-				enum iio_event_direction dir,
-				enum iio_event_info info,
-				int val, int val2)
+static int tsl2x7x_write_event_value(struct iio_dev *indio_dev,
+				     const struct iio_chan_spec *chan,
+				     enum iio_event_type type,
+				     enum iio_event_direction dir,
+				     enum iio_event_info info,
+				     int val, int val2)
 {
 	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
+	int ret = -EINVAL;
 
-	if (chan->type == IIO_INTENSITY) {
-		switch (dir) {
-		case IIO_EV_DIR_RISING:
-			chip->tsl2x7x_settings.als_thresh_high = val;
-			break;
-		case IIO_EV_DIR_FALLING:
-			chip->tsl2x7x_settings.als_thresh_low = val;
-			break;
-		default:
-			return -EINVAL;
-		}
-	} else {
-		switch (dir) {
-		case IIO_EV_DIR_RISING:
-			chip->tsl2x7x_settings.prox_thres_high = val;
-			break;
-		case IIO_EV_DIR_FALLING:
-			chip->tsl2x7x_settings.prox_thres_low = val;
-			break;
-		default:
-			return -EINVAL;
+	switch (info) {
+	case IIO_EV_INFO_VALUE:
+		if (chan->type == IIO_INTENSITY) {
+			switch (dir) {
+			case IIO_EV_DIR_RISING:
+				chip->tsl2x7x_settings.als_thresh_high = val;
+				ret = 0;
+				break;
+			case IIO_EV_DIR_FALLING:
+				chip->tsl2x7x_settings.als_thresh_low = val;
+				ret = 0;
+				break;
+			default:
+				break;
+			}
+		} else {
+			switch (dir) {
+			case IIO_EV_DIR_RISING:
+				chip->tsl2x7x_settings.prox_thres_high = val;
+				ret = 0;
+				break;
+			case IIO_EV_DIR_FALLING:
+				chip->tsl2x7x_settings.prox_thres_low = val;
+				ret = 0;
+				break;
+			default:
+				break;
+			}
 		}
+		break;
+	default:
+		break;
 	}
 
-	tsl2x7x_invoke_change(indio_dev);
+	if (ret < 0)
+		return ret;
 
-	return 0;
+	return tsl2x7x_invoke_change(indio_dev);
 }
 
-static int tsl2x7x_read_thresh(struct iio_dev *indio_dev,
-			       const struct iio_chan_spec *chan,
-			       enum iio_event_type type,
-			       enum iio_event_direction dir,
-				   enum iio_event_info info,
-			       int *val, int *val2)
+static int tsl2x7x_read_event_value(struct iio_dev *indio_dev,
+				    const struct iio_chan_spec *chan,
+				    enum iio_event_type type,
+				    enum iio_event_direction dir,
+				    enum iio_event_info info,
+				    int *val, int *val2)
 {
 	struct tsl2X7X_chip *chip = iio_priv(indio_dev);
+	int ret = -EINVAL;
 
-	if (chan->type == IIO_INTENSITY) {
-		switch (dir) {
-		case IIO_EV_DIR_RISING:
-			*val = chip->tsl2x7x_settings.als_thresh_high;
-			break;
-		case IIO_EV_DIR_FALLING:
-			*val = chip->tsl2x7x_settings.als_thresh_low;
-			break;
-		default:
-			return -EINVAL;
-		}
-	} else {
-		switch (dir) {
-		case IIO_EV_DIR_RISING:
-			*val = chip->tsl2x7x_settings.prox_thres_high;
-			break;
-		case IIO_EV_DIR_FALLING:
-			*val = chip->tsl2x7x_settings.prox_thres_low;
-			break;
-		default:
-			return -EINVAL;
+	switch (info) {
+	case IIO_EV_INFO_VALUE:
+		if (chan->type == IIO_INTENSITY) {
+			switch (dir) {
+			case IIO_EV_DIR_RISING:
+				*val = chip->tsl2x7x_settings.als_thresh_high;
+				ret = IIO_VAL_INT;
+				break;
+			case IIO_EV_DIR_FALLING:
+				*val = chip->tsl2x7x_settings.als_thresh_low;
+				ret = IIO_VAL_INT;
+				break;
+			default:
+				break;
+			}
+		} else {
+			switch (dir) {
+			case IIO_EV_DIR_RISING:
+				*val = chip->tsl2x7x_settings.prox_thres_high;
+				ret = IIO_VAL_INT;
+				break;
+			case IIO_EV_DIR_FALLING:
+				*val = chip->tsl2x7x_settings.prox_thres_low;
+				ret = IIO_VAL_INT;
+				break;
+			default:
+				break;
+			}
 		}
+		break;
+	default:
+		break;
 	}
 
-	return IIO_VAL_INT;
+	return ret;
 }
 
 static int tsl2x7x_read_raw(struct iio_dev *indio_dev,
@@ -1489,13 +1471,9 @@ static int tsl2x7x_write_raw(struct iio_dev *indio_dev,
 		return -EINVAL;
 	}
 
-	tsl2x7x_invoke_change(indio_dev);
-
-	return 0;
+	return tsl2x7x_invoke_change(indio_dev);
 }
 
-static DEVICE_ATTR_RW(power_state);
-
 static DEVICE_ATTR_RO(in_proximity0_calibscale_available);
 
 static DEVICE_ATTR_RO(in_illuminance0_calibscale_available);
@@ -1515,7 +1493,7 @@ static DEVICE_ATTR_RW(in_intensity0_thresh_period);
 static DEVICE_ATTR_RW(in_proximity0_thresh_period);
 
 /* Use the default register values to identify the Taos device */
-static int tsl2x7x_device_id(unsigned char *id, int target)
+static int tsl2x7x_device_id(int *id, int target)
 {
 	switch (target) {
 	case tsl2571:
@@ -1580,7 +1558,6 @@ static irqreturn_t tsl2x7x_event_handler(int irq, void *private)
 }
 
 static struct attribute *tsl2x7x_ALS_device_attrs[] = {
-	&dev_attr_power_state.attr,
 	&dev_attr_in_illuminance0_calibscale_available.attr,
 	&dev_attr_in_illuminance0_integration_time.attr,
 	&iio_const_attr_in_illuminance0_integration_time_available.dev_attr.attr,
@@ -1591,13 +1568,11 @@ static struct attribute *tsl2x7x_ALS_device_attrs[] = {
 };
 
 static struct attribute *tsl2x7x_PRX_device_attrs[] = {
-	&dev_attr_power_state.attr,
 	&dev_attr_in_proximity0_calibrate.attr,
 	NULL
 };
 
 static struct attribute *tsl2x7x_ALSPRX_device_attrs[] = {
-	&dev_attr_power_state.attr,
 	&dev_attr_in_illuminance0_calibscale_available.attr,
 	&dev_attr_in_illuminance0_integration_time.attr,
 	&iio_const_attr_in_illuminance0_integration_time_available.dev_attr.attr,
@@ -1609,14 +1584,12 @@ static struct attribute *tsl2x7x_ALSPRX_device_attrs[] = {
 };
 
 static struct attribute *tsl2x7x_PRX2_device_attrs[] = {
-	&dev_attr_power_state.attr,
 	&dev_attr_in_proximity0_calibrate.attr,
 	&dev_attr_in_proximity0_calibscale_available.attr,
 	NULL
 };
 
 static struct attribute *tsl2x7x_ALSPRX2_device_attrs[] = {
-	&dev_attr_power_state.attr,
 	&dev_attr_in_illuminance0_calibscale_available.attr,
 	&dev_attr_in_illuminance0_integration_time.attr,
 	&iio_const_attr_in_illuminance0_integration_time_available.dev_attr.attr,
@@ -1684,8 +1657,8 @@ static const struct iio_info tsl2X7X_device_info[] = {
 		.driver_module = THIS_MODULE,
 		.read_raw = &tsl2x7x_read_raw,
 		.write_raw = &tsl2x7x_write_raw,
-		.read_event_value = &tsl2x7x_read_thresh,
-		.write_event_value = &tsl2x7x_write_thresh,
+		.read_event_value = &tsl2x7x_read_event_value,
+		.write_event_value = &tsl2x7x_write_event_value,
 		.read_event_config = &tsl2x7x_read_interrupt_config,
 		.write_event_config = &tsl2x7x_write_interrupt_config,
 	},
@@ -1695,8 +1668,8 @@ static const struct iio_info tsl2X7X_device_info[] = {
 		.driver_module = THIS_MODULE,
 		.read_raw = &tsl2x7x_read_raw,
 		.write_raw = &tsl2x7x_write_raw,
-		.read_event_value = &tsl2x7x_read_thresh,
-		.write_event_value = &tsl2x7x_write_thresh,
+		.read_event_value = &tsl2x7x_read_event_value,
+		.write_event_value = &tsl2x7x_write_event_value,
 		.read_event_config = &tsl2x7x_read_interrupt_config,
 		.write_event_config = &tsl2x7x_write_interrupt_config,
 	},
@@ -1706,8 +1679,8 @@ static const struct iio_info tsl2X7X_device_info[] = {
 		.driver_module = THIS_MODULE,
 		.read_raw = &tsl2x7x_read_raw,
 		.write_raw = &tsl2x7x_write_raw,
-		.read_event_value = &tsl2x7x_read_thresh,
-		.write_event_value = &tsl2x7x_write_thresh,
+		.read_event_value = &tsl2x7x_read_event_value,
+		.write_event_value = &tsl2x7x_write_event_value,
 		.read_event_config = &tsl2x7x_read_interrupt_config,
 		.write_event_config = &tsl2x7x_write_interrupt_config,
 	},
@@ -1717,8 +1690,8 @@ static const struct iio_info tsl2X7X_device_info[] = {
 		.driver_module = THIS_MODULE,
 		.read_raw = &tsl2x7x_read_raw,
 		.write_raw = &tsl2x7x_write_raw,
-		.read_event_value = &tsl2x7x_read_thresh,
-		.write_event_value = &tsl2x7x_write_thresh,
+		.read_event_value = &tsl2x7x_read_event_value,
+		.write_event_value = &tsl2x7x_write_event_value,
 		.read_event_config = &tsl2x7x_read_interrupt_config,
 		.write_event_config = &tsl2x7x_write_interrupt_config,
 	},
@@ -1728,8 +1701,8 @@ static const struct iio_info tsl2X7X_device_info[] = {
 		.driver_module = THIS_MODULE,
 		.read_raw = &tsl2x7x_read_raw,
 		.write_raw = &tsl2x7x_write_raw,
-		.read_event_value = &tsl2x7x_read_thresh,
-		.write_event_value = &tsl2x7x_write_thresh,
+		.read_event_value = &tsl2x7x_read_event_value,
+		.write_event_value = &tsl2x7x_write_event_value,
 		.read_event_config = &tsl2x7x_read_interrupt_config,
 		.write_event_config = &tsl2x7x_write_interrupt_config,
 	},
@@ -1877,7 +1850,6 @@ static int tsl2x7x_probe(struct i2c_client *clientp,
 			 const struct i2c_device_id *id)
 {
 	int ret;
-	unsigned char device_id;
 	struct iio_dev *indio_dev;
 	struct tsl2X7X_chip *chip;
 
@@ -1889,13 +1861,13 @@ static int tsl2x7x_probe(struct i2c_client *clientp,
 	chip->client = clientp;
 	i2c_set_clientdata(clientp, indio_dev);
 
-	ret = tsl2x7x_i2c_read(chip->client,
-			       TSL2X7X_CHIPID, &device_id);
+	ret = i2c_smbus_read_byte_data(chip->client,
+				       TSL2X7X_CMD_REG | TSL2X7X_CHIPID);
 	if (ret < 0)
 		return ret;
 
-	if ((!tsl2x7x_device_id(&device_id, id->driver_data)) ||
-	    (tsl2x7x_device_id(&device_id, id->driver_data) == -EINVAL)) {
+	if ((!tsl2x7x_device_id(&ret, id->driver_data)) ||
+	    (tsl2x7x_device_id(&ret, id->driver_data) == -EINVAL)) {
 		dev_info(&chip->client->dev,
 			 "%s: i2c device found does not match expected id\n",
 				__func__);
@@ -2026,6 +1998,21 @@ static struct i2c_device_id tsl2x7x_idtable[] = {
 
 MODULE_DEVICE_TABLE(i2c, tsl2x7x_idtable);
 
+static const struct of_device_id tsl2x7x_of_match[] = {
+	{ .compatible = "amstaos,tsl2571" },
+	{ .compatible = "amstaos,tsl2671" },
+	{ .compatible = "amstaos,tmd2671" },
+	{ .compatible = "amstaos,tsl2771" },
+	{ .compatible = "amstaos,tmd2771" },
+	{ .compatible = "amstaos,tsl2572" },
+	{ .compatible = "amstaos,tsl2672" },
+	{ .compatible = "amstaos,tmd2672" },
+	{ .compatible = "amstaos,tsl2772" },
+	{ .compatible = "amstaos,tmd2772" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, tsl2x7x_of_match);
+
 static const struct dev_pm_ops tsl2x7x_pm_ops = {
 	.suspend = tsl2x7x_suspend,
 	.resume  = tsl2x7x_resume,
@@ -2035,6 +2022,7 @@ static const struct dev_pm_ops tsl2x7x_pm_ops = {
 static struct i2c_driver tsl2x7x_driver = {
 	.driver = {
 		.name = "tsl2x7x",
+		.of_match_table = tsl2x7x_of_match,
 		.pm = &tsl2x7x_pm_ops,
 	},
 	.id_table = tsl2x7x_idtable,
diff --git a/drivers/staging/irda/TODO b/drivers/staging/irda/TODO
new file mode 100644
index 000000000000..7d98a5cffaff
--- /dev/null
+++ b/drivers/staging/irda/TODO
@@ -0,0 +1,4 @@
+The irda code will be removed soon from the kernel tree as it is old and
+obsolete and broken.
+
+Don't worry about fixing up anything here, it's not needed.
diff --git a/drivers/net/irda/Kconfig b/drivers/staging/irda/drivers/Kconfig
index e070e1222733..e070e1222733 100644
--- a/drivers/net/irda/Kconfig
+++ b/drivers/staging/irda/drivers/Kconfig
diff --git a/drivers/net/irda/Makefile b/drivers/staging/irda/drivers/Makefile
index 4c344433dae5..e2901b135528 100644
--- a/drivers/net/irda/Makefile
+++ b/drivers/staging/irda/drivers/Makefile
@@ -5,6 +5,8 @@
 # Rewritten to use lists instead of if-statements.
 #
 
+subdir-ccflags-y += -I$(srctree)/drivers/staging/irda/include
+
 # FIR drivers
 obj-$(CONFIG_USB_IRDA)		+= irda-usb.o
 obj-$(CONFIG_SIGMATEL_FIR)	+= stir4200.o
diff --git a/drivers/net/irda/act200l-sir.c b/drivers/staging/irda/drivers/act200l-sir.c
index e8917511e1aa..e8917511e1aa 100644
--- a/drivers/net/irda/act200l-sir.c
+++ b/drivers/staging/irda/drivers/act200l-sir.c
diff --git a/drivers/net/irda/actisys-sir.c b/drivers/staging/irda/drivers/actisys-sir.c
index e224b8b99517..e224b8b99517 100644
--- a/drivers/net/irda/actisys-sir.c
+++ b/drivers/staging/irda/drivers/actisys-sir.c
diff --git a/drivers/net/irda/ali-ircc.c b/drivers/staging/irda/drivers/ali-ircc.c
index 35f198d83701..35f198d83701 100644
--- a/drivers/net/irda/ali-ircc.c
+++ b/drivers/staging/irda/drivers/ali-ircc.c
diff --git a/drivers/net/irda/ali-ircc.h b/drivers/staging/irda/drivers/ali-ircc.h
index c2d9747a5108..c2d9747a5108 100644
--- a/drivers/net/irda/ali-ircc.h
+++ b/drivers/staging/irda/drivers/ali-ircc.h
diff --git a/drivers/net/irda/au1k_ir.c b/drivers/staging/irda/drivers/au1k_ir.c
index be4ea6aa57a9..be4ea6aa57a9 100644
--- a/drivers/net/irda/au1k_ir.c
+++ b/drivers/staging/irda/drivers/au1k_ir.c
diff --git a/drivers/net/irda/bfin_sir.c b/drivers/staging/irda/drivers/bfin_sir.c
index 3151b580dbd6..3151b580dbd6 100644
--- a/drivers/net/irda/bfin_sir.c
+++ b/drivers/staging/irda/drivers/bfin_sir.c
diff --git a/drivers/net/irda/bfin_sir.h b/drivers/staging/irda/drivers/bfin_sir.h
index d47cf14bb4a5..d47cf14bb4a5 100644
--- a/drivers/net/irda/bfin_sir.h
+++ b/drivers/staging/irda/drivers/bfin_sir.h
diff --git a/drivers/net/irda/donauboe.c b/drivers/staging/irda/drivers/donauboe.c
index b337e6d23a88..b337e6d23a88 100644
--- a/drivers/net/irda/donauboe.c
+++ b/drivers/staging/irda/drivers/donauboe.c
diff --git a/drivers/net/irda/donauboe.h b/drivers/staging/irda/drivers/donauboe.h
index d92d54e839b9..d92d54e839b9 100644
--- a/drivers/net/irda/donauboe.h
+++ b/drivers/staging/irda/drivers/donauboe.h
diff --git a/drivers/net/irda/esi-sir.c b/drivers/staging/irda/drivers/esi-sir.c
index 019a3e848bcb..019a3e848bcb 100644
--- a/drivers/net/irda/esi-sir.c
+++ b/drivers/staging/irda/drivers/esi-sir.c
diff --git a/drivers/net/irda/girbil-sir.c b/drivers/staging/irda/drivers/girbil-sir.c
index 7e0a5b8c6d53..7e0a5b8c6d53 100644
--- a/drivers/net/irda/girbil-sir.c
+++ b/drivers/staging/irda/drivers/girbil-sir.c
diff --git a/drivers/net/irda/irda-usb.c b/drivers/staging/irda/drivers/irda-usb.c
index 6f3c805f7211..723e49bc4baa 100644
--- a/drivers/net/irda/irda-usb.c
+++ b/drivers/staging/irda/drivers/irda-usb.c
@@ -72,7 +72,7 @@
 static int qos_mtt_bits = 0;
 
 /* These are the currently known IrDA USB dongles. Add new dongles here */
-static struct usb_device_id dongles[] = {
+static const struct usb_device_id dongles[] = {
 	/* ACTiSYS Corp.,  ACT-IR2000U FIR-USB Adapter */
 	{ USB_DEVICE(0x9c4, 0x011), .driver_info = IUC_SPEED_BUG | IUC_NO_WINDOW },
 	/* Look like ACTiSYS, Report : IBM Corp., IBM UltraPort IrDA */
diff --git a/drivers/net/irda/irda-usb.h b/drivers/staging/irda/drivers/irda-usb.h
index 8ac389fa9348..8ac389fa9348 100644
--- a/drivers/net/irda/irda-usb.h
+++ b/drivers/staging/irda/drivers/irda-usb.h
diff --git a/drivers/net/irda/irtty-sir.c b/drivers/staging/irda/drivers/irtty-sir.c
index 7a20a9a4663a..7a20a9a4663a 100644
--- a/drivers/net/irda/irtty-sir.c
+++ b/drivers/staging/irda/drivers/irtty-sir.c
diff --git a/drivers/net/irda/irtty-sir.h b/drivers/staging/irda/drivers/irtty-sir.h
index b132d8f6eb13..b132d8f6eb13 100644
--- a/drivers/net/irda/irtty-sir.h
+++ b/drivers/staging/irda/drivers/irtty-sir.h
diff --git a/drivers/net/irda/kingsun-sir.c b/drivers/staging/irda/drivers/kingsun-sir.c
index 24c0f169a7b1..4fd4ac2fe09f 100644
--- a/drivers/net/irda/kingsun-sir.c
+++ b/drivers/staging/irda/drivers/kingsun-sir.c
@@ -85,7 +85,7 @@
 #define KING_PRODUCT_ID 0x4200
 
 /* These are the currently known USB ids */
-static struct usb_device_id dongles[] = {
+static const struct usb_device_id dongles[] = {
     /* KingSun Co,Ltd  IrDA/USB Bridge */
     { USB_DEVICE(KING_VENDOR_ID, KING_PRODUCT_ID) },
     { }
diff --git a/drivers/net/irda/ks959-sir.c b/drivers/staging/irda/drivers/ks959-sir.c
index 3affded3e30d..8025741e7586 100644
--- a/drivers/net/irda/ks959-sir.c
+++ b/drivers/staging/irda/drivers/ks959-sir.c
@@ -133,7 +133,7 @@
 #define KS959_PRODUCT_ID 0x4959
 
 /* These are the currently known USB ids */
-static struct usb_device_id dongles[] = {
+static const struct usb_device_id dongles[] = {
 	/* KingSun Co,Ltd  IrDA/USB Bridge */
 	{USB_DEVICE(KS959_VENDOR_ID, KS959_PRODUCT_ID)},
 	{}
diff --git a/drivers/net/irda/ksdazzle-sir.c b/drivers/staging/irda/drivers/ksdazzle-sir.c
index 741452c7ce35..d2a0755df596 100644
--- a/drivers/net/irda/ksdazzle-sir.c
+++ b/drivers/staging/irda/drivers/ksdazzle-sir.c
@@ -97,7 +97,7 @@
 #define KSDAZZLE_PRODUCT_ID 0x4100
 
 /* These are the currently known USB ids */
-static struct usb_device_id dongles[] = {
+static const struct usb_device_id dongles[] = {
 	/* KingSun Co,Ltd  IrDA/USB Bridge */
 	{USB_DEVICE(KSDAZZLE_VENDOR_ID, KSDAZZLE_PRODUCT_ID)},
 	{}
diff --git a/drivers/net/irda/litelink-sir.c b/drivers/staging/irda/drivers/litelink-sir.c
index 8eefcb44bac3..8eefcb44bac3 100644
--- a/drivers/net/irda/litelink-sir.c
+++ b/drivers/staging/irda/drivers/litelink-sir.c
diff --git a/drivers/net/irda/ma600-sir.c b/drivers/staging/irda/drivers/ma600-sir.c
index a764817b47f1..a764817b47f1 100644
--- a/drivers/net/irda/ma600-sir.c
+++ b/drivers/staging/irda/drivers/ma600-sir.c
diff --git a/drivers/net/irda/mcp2120-sir.c b/drivers/staging/irda/drivers/mcp2120-sir.c
index 2e33f91bfe8f..2e33f91bfe8f 100644
--- a/drivers/net/irda/mcp2120-sir.c
+++ b/drivers/staging/irda/drivers/mcp2120-sir.c
diff --git a/drivers/net/irda/mcs7780.c b/drivers/staging/irda/drivers/mcs7780.c
index 765de3bedb88..c3f0b254b344 100644
--- a/drivers/net/irda/mcs7780.c
+++ b/drivers/staging/irda/drivers/mcs7780.c
@@ -66,7 +66,7 @@
 #define MCS_VENDOR_ID 0x9710
 #define MCS_PRODUCT_ID 0x7780
 
-static struct usb_device_id mcs_table[] = {
+static const struct usb_device_id mcs_table[] = {
 	/* MosChip Corp.,  MCS7780 FIR-USB Adapter */
 	{USB_DEVICE(MCS_VENDOR_ID, MCS_PRODUCT_ID)},
 	{},
diff --git a/drivers/net/irda/mcs7780.h b/drivers/staging/irda/drivers/mcs7780.h
index a6e8f7dbafc9..a6e8f7dbafc9 100644
--- a/drivers/net/irda/mcs7780.h
+++ b/drivers/staging/irda/drivers/mcs7780.h
diff --git a/drivers/net/irda/nsc-ircc.c b/drivers/staging/irda/drivers/nsc-ircc.c
index 7beae147be11..7beae147be11 100644
--- a/drivers/net/irda/nsc-ircc.c
+++ b/drivers/staging/irda/drivers/nsc-ircc.c
diff --git a/drivers/net/irda/nsc-ircc.h b/drivers/staging/irda/drivers/nsc-ircc.h
index 7be5acb56532..7be5acb56532 100644
--- a/drivers/net/irda/nsc-ircc.h
+++ b/drivers/staging/irda/drivers/nsc-ircc.h
diff --git a/drivers/net/irda/old_belkin-sir.c b/drivers/staging/irda/drivers/old_belkin-sir.c
index a7c2e990ae69..a7c2e990ae69 100644
--- a/drivers/net/irda/old_belkin-sir.c
+++ b/drivers/staging/irda/drivers/old_belkin-sir.c
diff --git a/drivers/net/irda/pxaficp_ir.c b/drivers/staging/irda/drivers/pxaficp_ir.c
index 1dba16bc7f8d..1dba16bc7f8d 100644
--- a/drivers/net/irda/pxaficp_ir.c
+++ b/drivers/staging/irda/drivers/pxaficp_ir.c
diff --git a/drivers/net/irda/sa1100_ir.c b/drivers/staging/irda/drivers/sa1100_ir.c
index b6e44ff4e373..b6e44ff4e373 100644
--- a/drivers/net/irda/sa1100_ir.c
+++ b/drivers/staging/irda/drivers/sa1100_ir.c
diff --git a/drivers/net/irda/sh_sir.c b/drivers/staging/irda/drivers/sh_sir.c
index fede6864c737..fede6864c737 100644
--- a/drivers/net/irda/sh_sir.c
+++ b/drivers/staging/irda/drivers/sh_sir.c
diff --git a/drivers/net/irda/sir-dev.h b/drivers/staging/irda/drivers/sir-dev.h
index f50b9c1c0639..f50b9c1c0639 100644
--- a/drivers/net/irda/sir-dev.h
+++ b/drivers/staging/irda/drivers/sir-dev.h
diff --git a/drivers/net/irda/sir_dev.c b/drivers/staging/irda/drivers/sir_dev.c
index 6af26a7d787c..6af26a7d787c 100644
--- a/drivers/net/irda/sir_dev.c
+++ b/drivers/staging/irda/drivers/sir_dev.c
diff --git a/drivers/net/irda/sir_dongle.c b/drivers/staging/irda/drivers/sir_dongle.c
index 7436f73ff1bb..7436f73ff1bb 100644
--- a/drivers/net/irda/sir_dongle.c
+++ b/drivers/staging/irda/drivers/sir_dongle.c
diff --git a/drivers/net/irda/smsc-ircc2.c b/drivers/staging/irda/drivers/smsc-ircc2.c
index 19a55cba6beb..19a55cba6beb 100644
--- a/drivers/net/irda/smsc-ircc2.c
+++ b/drivers/staging/irda/drivers/smsc-ircc2.c
diff --git a/drivers/net/irda/smsc-ircc2.h b/drivers/staging/irda/drivers/smsc-ircc2.h
index 4829fa22cb29..4829fa22cb29 100644
--- a/drivers/net/irda/smsc-ircc2.h
+++ b/drivers/staging/irda/drivers/smsc-ircc2.h
diff --git a/drivers/net/irda/smsc-sio.h b/drivers/staging/irda/drivers/smsc-sio.h
index 59e20e653ebe..59e20e653ebe 100644
--- a/drivers/net/irda/smsc-sio.h
+++ b/drivers/staging/irda/drivers/smsc-sio.h
diff --git a/drivers/net/irda/stir4200.c b/drivers/staging/irda/drivers/stir4200.c
index 7ee514879531..ee2cb70b688d 100644
--- a/drivers/net/irda/stir4200.c
+++ b/drivers/staging/irda/drivers/stir4200.c
@@ -183,7 +183,7 @@ struct stir_cb {
 
 
 /* These are the currently known USB ids */
-static struct usb_device_id dongles[] = {
+static const struct usb_device_id dongles[] = {
     /* SigmaTel, Inc,  STIr4200 IrDA/USB Bridge */
     { USB_DEVICE(0x066f, 0x4200) },
     { }
diff --git a/drivers/net/irda/tekram-sir.c b/drivers/staging/irda/drivers/tekram-sir.c
index 9dcf0c103b9d..9dcf0c103b9d 100644
--- a/drivers/net/irda/tekram-sir.c
+++ b/drivers/staging/irda/drivers/tekram-sir.c
diff --git a/drivers/net/irda/toim3232-sir.c b/drivers/staging/irda/drivers/toim3232-sir.c
index b977d6d33e74..b977d6d33e74 100644
--- a/drivers/net/irda/toim3232-sir.c
+++ b/drivers/staging/irda/drivers/toim3232-sir.c
diff --git a/drivers/net/irda/via-ircc.c b/drivers/staging/irda/drivers/via-ircc.c
index ca4442a9d631..ca4442a9d631 100644
--- a/drivers/net/irda/via-ircc.c
+++ b/drivers/staging/irda/drivers/via-ircc.c
diff --git a/drivers/net/irda/via-ircc.h b/drivers/staging/irda/drivers/via-ircc.h
index ac1525573398..ac1525573398 100644
--- a/drivers/net/irda/via-ircc.h
+++ b/drivers/staging/irda/drivers/via-ircc.h
diff --git a/drivers/net/irda/vlsi_ir.c b/drivers/staging/irda/drivers/vlsi_ir.c
index 6638784c082e..6638784c082e 100644
--- a/drivers/net/irda/vlsi_ir.c
+++ b/drivers/staging/irda/drivers/vlsi_ir.c
diff --git a/drivers/net/irda/vlsi_ir.h b/drivers/staging/irda/drivers/vlsi_ir.h
index f9db2ce4c5c6..f9db2ce4c5c6 100644
--- a/drivers/net/irda/vlsi_ir.h
+++ b/drivers/staging/irda/drivers/vlsi_ir.h
diff --git a/drivers/net/irda/w83977af.h b/drivers/staging/irda/drivers/w83977af.h
index 04476c2e9121..04476c2e9121 100644
--- a/drivers/net/irda/w83977af.h
+++ b/drivers/staging/irda/drivers/w83977af.h
diff --git a/drivers/net/irda/w83977af_ir.c b/drivers/staging/irda/drivers/w83977af_ir.c
index 282b6c9ae05b..282b6c9ae05b 100644
--- a/drivers/net/irda/w83977af_ir.c
+++ b/drivers/staging/irda/drivers/w83977af_ir.c
diff --git a/drivers/net/irda/w83977af_ir.h b/drivers/staging/irda/drivers/w83977af_ir.h
index fefe9b11e200..fefe9b11e200 100644
--- a/drivers/net/irda/w83977af_ir.h
+++ b/drivers/staging/irda/drivers/w83977af_ir.h
diff --git a/drivers/staging/irda/include/net/irda/af_irda.h b/drivers/staging/irda/include/net/irda/af_irda.h
new file mode 100644
index 000000000000..0df574931522
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/af_irda.h
@@ -0,0 +1,87 @@
+/*********************************************************************
+ *                
+ * Filename:      af_irda.h
+ * Version:       1.0
+ * Description:   IrDA sockets declarations
+ * Status:        Stable
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Tue Dec  9 21:13:12 1997
+ * Modified at:   Fri Jan 28 13:16:32 2000
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1998-2000 Dag Brattli, All Rights Reserved.
+ *     Copyright (c) 2000-2002 Jean Tourrilhes <jt@hpl.hp.com>
+ *      
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ *  
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is 
+ *     provided "AS-IS" and at no charge.
+ *     
+ ********************************************************************/
+
+#ifndef AF_IRDA_H
+#define AF_IRDA_H
+
+#include <linux/irda.h>
+#include <net/irda/irda.h>
+#include <net/irda/iriap.h>		/* struct iriap_cb */
+#include <net/irda/irias_object.h>	/* struct ias_value */
+#include <net/irda/irlmp.h>		/* struct lsap_cb */
+#include <net/irda/irttp.h>		/* struct tsap_cb */
+#include <net/irda/discovery.h>		/* struct discovery_t */
+#include <net/sock.h>
+
+/* IrDA Socket */
+struct irda_sock {
+	/* struct sock has to be the first member of irda_sock */
+	struct sock sk;
+	__u32 saddr;          /* my local address */
+	__u32 daddr;          /* peer address */
+
+	struct lsap_cb *lsap; /* LSAP used by Ultra */
+	__u8  pid;            /* Protocol IP (PID) used by Ultra */
+
+	struct tsap_cb *tsap; /* TSAP used by this connection */
+	__u8 dtsap_sel;       /* remote TSAP address */
+	__u8 stsap_sel;       /* local TSAP address */
+	
+	__u32 max_sdu_size_rx;
+	__u32 max_sdu_size_tx;
+	__u32 max_data_size;
+	__u8  max_header_size;
+	struct qos_info qos_tx;
+
+	__u16_host_order mask;           /* Hint bits mask */
+	__u16_host_order hints;          /* Hint bits */
+
+	void *ckey;           /* IrLMP client handle */
+	void *skey;           /* IrLMP service handle */
+
+	struct ias_object *ias_obj;   /* Our service name + lsap in IAS */
+	struct iriap_cb *iriap;	      /* Used to query remote IAS */
+	struct ias_value *ias_result; /* Result of remote IAS query */
+
+	hashbin_t *cachelog;		/* Result of discovery query */
+	__u32 cachedaddr;	/* Result of selective discovery query */
+
+	int nslots;           /* Number of slots to use for discovery */
+
+	int errno;            /* status of the IAS query */
+
+	wait_queue_head_t query_wait;	/* Wait for the answer to a query */
+	struct timer_list watchdog;	/* Timeout for discovery */
+
+	LOCAL_FLOW tx_flow;
+	LOCAL_FLOW rx_flow;
+};
+
+static inline struct irda_sock *irda_sk(struct sock *sk)
+{
+	return (struct irda_sock *)sk;
+}
+
+#endif /* AF_IRDA_H */
diff --git a/drivers/staging/irda/include/net/irda/crc.h b/drivers/staging/irda/include/net/irda/crc.h
new file mode 100644
index 000000000000..f202296df9bb
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/crc.h
@@ -0,0 +1,29 @@
+/*********************************************************************
+ *                
+ * Filename:      crc.h
+ * Version:       
+ * Description:   CRC routines
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Mon Aug  4 20:40:53 1997
+ * Modified at:   Sun May  2 20:25:23 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ ********************************************************************/
+
+#ifndef IRDA_CRC_H
+#define IRDA_CRC_H
+
+#include <linux/types.h>
+#include <linux/crc-ccitt.h>
+
+#define INIT_FCS  0xffff   /* Initial FCS value */
+#define GOOD_FCS  0xf0b8   /* Good final FCS value */
+
+/* Recompute the FCS with one more character appended. */
+#define irda_fcs(fcs, c) crc_ccitt_byte(fcs, c)
+
+/* Recompute the FCS with len bytes appended. */
+#define irda_calc_crc16(fcs, buf, len) crc_ccitt(fcs, buf, len)
+
+#endif
diff --git a/drivers/staging/irda/include/net/irda/discovery.h b/drivers/staging/irda/include/net/irda/discovery.h
new file mode 100644
index 000000000000..63ae32530567
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/discovery.h
@@ -0,0 +1,95 @@
+/*********************************************************************
+ *                
+ * Filename:      discovery.h
+ * Version:       
+ * Description:   
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Tue Apr  6 16:53:53 1999
+ * Modified at:   Tue Oct  5 10:05:10 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1999 Dag Brattli, All Rights Reserved.
+ *     Copyright (c) 2000-2002 Jean Tourrilhes <jt@hpl.hp.com>
+ *     
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ * 
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ * 
+ *     You should have received a copy of the GNU General Public License 
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *     
+ ********************************************************************/
+
+#ifndef DISCOVERY_H
+#define DISCOVERY_H
+
+#include <asm/param.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/irqueue.h>		/* irda_queue_t */
+#include <net/irda/irlap_event.h>	/* LAP_REASON */
+
+#define DISCOVERY_EXPIRE_TIMEOUT (2*sysctl_discovery_timeout*HZ)
+#define DISCOVERY_DEFAULT_SLOTS  0
+
+/*
+ *  This type is used by the protocols that transmit 16 bits words in 
+ *  little endian format. A little endian machine stores MSB of word in
+ *  byte[1] and LSB in byte[0]. A big endian machine stores MSB in byte[0] 
+ *  and LSB in byte[1].
+ *
+ * This structure is used in the code for things that are endian neutral
+ * but that fit in a word so that we can manipulate them efficiently.
+ * By endian neutral, I mean things that are really an array of bytes,
+ * and always used as such, for example the hint bits. Jean II
+ */
+typedef union {
+	__u16 word;
+	__u8  byte[2];
+} __u16_host_order;
+
+/* Types of discovery */
+typedef enum {
+	DISCOVERY_LOG,		/* What's in our discovery log */
+	DISCOVERY_ACTIVE,	/* Doing our own discovery on the medium */
+	DISCOVERY_PASSIVE,	/* Peer doing discovery on the medium */
+	EXPIRY_TIMEOUT,		/* Entry expired due to timeout */
+} DISCOVERY_MODE;
+
+#define NICKNAME_MAX_LEN 21
+
+/* Basic discovery information about a peer */
+typedef struct irda_device_info		discinfo_t;	/* linux/irda.h */
+
+/*
+ * The DISCOVERY structure is used for both discovery requests and responses
+ */
+typedef struct discovery_t {
+	irda_queue_t	q;		/* Must be first! */
+
+	discinfo_t	data;		/* Basic discovery information */
+	int		name_len;	/* Length of nickname */
+
+	LAP_REASON	condition;	/* More info about the discovery */
+	int		gen_addr_bit;	/* Need to generate a new device
+					 * address? */
+	int		nslots;		/* Number of slots to use when
+					 * discovering */
+	unsigned long	timestamp;	/* Last time discovered */
+	unsigned long	firststamp;	/* First time discovered */
+} discovery_t;
+
+void irlmp_add_discovery(hashbin_t *cachelog, discovery_t *discovery);
+void irlmp_add_discovery_log(hashbin_t *cachelog, hashbin_t *log);
+void irlmp_expire_discoveries(hashbin_t *log, __u32 saddr, int force);
+struct irda_device_info *irlmp_copy_discoveries(hashbin_t *log, int *pn,
+						__u16 mask, int old_entries);
+
+#endif
diff --git a/drivers/staging/irda/include/net/irda/ircomm_core.h b/drivers/staging/irda/include/net/irda/ircomm_core.h
new file mode 100644
index 000000000000..2a580ce9edad
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/ircomm_core.h
@@ -0,0 +1,106 @@
+/*********************************************************************
+ *                
+ * Filename:      ircomm_core.h
+ * Version:       
+ * Description:   
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Wed Jun  9 08:58:43 1999
+ * Modified at:   Mon Dec 13 11:52:29 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1999 Dag Brattli, All Rights Reserved.
+ *     
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ * 
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ * 
+ *     You should have received a copy of the GNU General Public License 
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *     
+ ********************************************************************/
+
+#ifndef IRCOMM_CORE_H
+#define IRCOMM_CORE_H
+
+#include <net/irda/irda.h>
+#include <net/irda/irqueue.h>
+#include <net/irda/ircomm_event.h>
+
+#define IRCOMM_MAGIC 0x98347298
+#define IRCOMM_HEADER_SIZE 1
+
+struct ircomm_cb;   /* Forward decl. */
+
+/*
+ * A small call-table, so we don't have to check the service-type whenever
+ * we want to do something
+ */
+typedef struct {
+	int (*data_request)(struct ircomm_cb *, struct sk_buff *, int clen);
+	int (*connect_request)(struct ircomm_cb *, struct sk_buff *, 
+			       struct ircomm_info *);
+	int (*connect_response)(struct ircomm_cb *, struct sk_buff *);
+	int (*disconnect_request)(struct ircomm_cb *, struct sk_buff *, 
+				  struct ircomm_info *);	
+} call_t;
+
+struct ircomm_cb {
+	irda_queue_t queue;
+	magic_t magic;
+
+	notify_t notify;
+	call_t   issue;
+
+	int state;
+	int line;            /* Which TTY line we are using */
+
+	struct tsap_cb *tsap;
+	struct lsap_cb *lsap;
+	
+	__u8 dlsap_sel;      /* Destination LSAP/TSAP selector */
+	__u8 slsap_sel;      /* Source LSAP/TSAP selector */
+
+	__u32 saddr;         /* Source device address (link we are using) */
+	__u32 daddr;         /* Destination device address */
+
+	int max_header_size; /* Header space we must reserve for each frame */
+	int max_data_size;   /* The amount of data we can fill in each frame */
+
+	LOCAL_FLOW flow_status; /* Used by ircomm_lmp */
+	int pkt_count;          /* Number of frames we have sent to IrLAP */
+
+	__u8 service_type;
+};
+
+extern hashbin_t *ircomm;
+
+struct ircomm_cb *ircomm_open(notify_t *notify, __u8 service_type, int line);
+int ircomm_close(struct ircomm_cb *self);
+
+int ircomm_data_request(struct ircomm_cb *self, struct sk_buff *skb);
+void ircomm_data_indication(struct ircomm_cb *self, struct sk_buff *skb);
+void ircomm_process_data(struct ircomm_cb *self, struct sk_buff *skb);
+int ircomm_control_request(struct ircomm_cb *self, struct sk_buff *skb);
+int ircomm_connect_request(struct ircomm_cb *self, __u8 dlsap_sel, 
+			   __u32 saddr, __u32 daddr, struct sk_buff *skb,
+			   __u8 service_type);
+void ircomm_connect_indication(struct ircomm_cb *self, struct sk_buff *skb,
+			       struct ircomm_info *info);
+void ircomm_connect_confirm(struct ircomm_cb *self, struct sk_buff *skb,
+			    struct ircomm_info *info);
+int ircomm_connect_response(struct ircomm_cb *self, struct sk_buff *userdata);
+int ircomm_disconnect_request(struct ircomm_cb *self, struct sk_buff *userdata);
+void ircomm_disconnect_indication(struct ircomm_cb *self, struct sk_buff *skb,
+				  struct ircomm_info *info);
+void ircomm_flow_request(struct ircomm_cb *self, LOCAL_FLOW flow);
+
+#define ircomm_is_connected(self) (self->state == IRCOMM_CONN)
+
+#endif
diff --git a/drivers/staging/irda/include/net/irda/ircomm_event.h b/drivers/staging/irda/include/net/irda/ircomm_event.h
new file mode 100644
index 000000000000..5bbc32998d57
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/ircomm_event.h
@@ -0,0 +1,83 @@
+/*********************************************************************
+ *                
+ * Filename:      ircomm_event.h
+ * Version:       
+ * Description:   
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sun Jun  6 23:51:13 1999
+ * Modified at:   Thu Jun 10 08:36:25 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1999 Dag Brattli, All Rights Reserved.
+ *     
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ * 
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ * 
+ *     You should have received a copy of the GNU General Public License 
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *     
+ ********************************************************************/
+
+#ifndef IRCOMM_EVENT_H
+#define IRCOMM_EVENT_H
+
+#include <net/irda/irmod.h>
+
+typedef enum {
+        IRCOMM_IDLE,
+        IRCOMM_WAITI,
+        IRCOMM_WAITR,
+        IRCOMM_CONN,
+} IRCOMM_STATE;
+
+/* IrCOMM Events */
+typedef enum {
+        IRCOMM_CONNECT_REQUEST,
+        IRCOMM_CONNECT_RESPONSE,
+        IRCOMM_TTP_CONNECT_INDICATION,
+	IRCOMM_LMP_CONNECT_INDICATION,
+        IRCOMM_TTP_CONNECT_CONFIRM,
+	IRCOMM_LMP_CONNECT_CONFIRM,
+
+        IRCOMM_LMP_DISCONNECT_INDICATION,
+	IRCOMM_TTP_DISCONNECT_INDICATION,
+        IRCOMM_DISCONNECT_REQUEST,
+
+        IRCOMM_TTP_DATA_INDICATION,
+	IRCOMM_LMP_DATA_INDICATION,
+        IRCOMM_DATA_REQUEST,
+        IRCOMM_CONTROL_REQUEST,
+        IRCOMM_CONTROL_INDICATION,
+} IRCOMM_EVENT;
+
+/*
+ * Used for passing information through the state-machine
+ */
+struct ircomm_info {
+        __u32     saddr;               /* Source device address */
+        __u32     daddr;               /* Destination device address */
+        __u8      dlsap_sel;
+        LM_REASON reason;              /* Reason for disconnect */
+	__u32     max_data_size;
+	__u32     max_header_size;
+
+	struct qos_info *qos;
+};
+
+extern const char *const ircomm_state[];
+
+struct ircomm_cb;   /* Forward decl. */
+
+int ircomm_do_event(struct ircomm_cb *self, IRCOMM_EVENT event,
+		    struct sk_buff *skb, struct ircomm_info *info);
+void ircomm_next_state(struct ircomm_cb *self, IRCOMM_STATE state);
+
+#endif
diff --git a/drivers/staging/irda/include/net/irda/ircomm_lmp.h b/drivers/staging/irda/include/net/irda/ircomm_lmp.h
new file mode 100644
index 000000000000..5042a5021a04
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/ircomm_lmp.h
@@ -0,0 +1,36 @@
+/*********************************************************************
+ *                
+ * Filename:      ircomm_lmp.h
+ * Version:       
+ * Description:   
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Wed Jun  9 10:06:07 1999
+ * Modified at:   Fri Aug 13 07:32:32 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1999 Dag Brattli, All Rights Reserved.
+ *     
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ * 
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ * 
+ *     You should have received a copy of the GNU General Public License 
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *     
+ ********************************************************************/
+
+#ifndef IRCOMM_LMP_H
+#define IRCOMM_LMP_H
+
+#include <net/irda/ircomm_core.h>
+
+int ircomm_open_lsap(struct ircomm_cb *self);
+
+#endif
diff --git a/drivers/staging/irda/include/net/irda/ircomm_param.h b/drivers/staging/irda/include/net/irda/ircomm_param.h
new file mode 100644
index 000000000000..1f67432321c4
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/ircomm_param.h
@@ -0,0 +1,147 @@
+/*********************************************************************
+ *                
+ * Filename:      ircomm_param.h
+ * Version:       1.0
+ * Description:   Parameter handling for the IrCOMM protocol
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Mon Jun  7 08:47:28 1999
+ * Modified at:   Wed Aug 25 13:46:33 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1999 Dag Brattli, All Rights Reserved.
+ *     
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ * 
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ * 
+ *     You should have received a copy of the GNU General Public License 
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *     
+ ********************************************************************/
+
+#ifndef IRCOMM_PARAMS_H
+#define IRCOMM_PARAMS_H
+
+#include <net/irda/parameters.h>
+
+/* Parameters common to all service types */
+#define IRCOMM_SERVICE_TYPE     0x00
+#define IRCOMM_PORT_TYPE        0x01 /* Only used in LM-IAS */
+#define IRCOMM_PORT_NAME        0x02 /* Only used in LM-IAS */
+
+/* Parameters for both 3 wire and 9 wire */
+#define IRCOMM_DATA_RATE        0x10
+#define IRCOMM_DATA_FORMAT      0x11
+#define IRCOMM_FLOW_CONTROL     0x12
+#define IRCOMM_XON_XOFF         0x13
+#define IRCOMM_ENQ_ACK          0x14
+#define IRCOMM_LINE_STATUS      0x15
+#define IRCOMM_BREAK            0x16
+
+/* Parameters for 9 wire */
+#define IRCOMM_DTE              0x20
+#define IRCOMM_DCE              0x21
+#define IRCOMM_POLL             0x22
+
+/* Service type (details) */
+#define IRCOMM_3_WIRE_RAW       0x01
+#define IRCOMM_3_WIRE           0x02
+#define IRCOMM_9_WIRE           0x04
+#define IRCOMM_CENTRONICS       0x08
+
+/* Port type (details) */
+#define IRCOMM_SERIAL           0x00
+#define IRCOMM_PARALLEL         0x01
+
+/* Data format (details) */
+#define IRCOMM_WSIZE_5          0x00
+#define IRCOMM_WSIZE_6          0x01
+#define IRCOMM_WSIZE_7          0x02
+#define IRCOMM_WSIZE_8          0x03
+
+#define IRCOMM_1_STOP_BIT       0x00
+#define IRCOMM_2_STOP_BIT       0x04 /* 1.5 if char len 5 */
+
+#define IRCOMM_PARITY_DISABLE   0x00
+#define IRCOMM_PARITY_ENABLE    0x08
+
+#define IRCOMM_PARITY_ODD       0x00
+#define IRCOMM_PARITY_EVEN      0x10
+#define IRCOMM_PARITY_MARK      0x20
+#define IRCOMM_PARITY_SPACE     0x30
+
+/* Flow control */
+#define IRCOMM_XON_XOFF_IN      0x01
+#define IRCOMM_XON_XOFF_OUT     0x02
+#define IRCOMM_RTS_CTS_IN       0x04
+#define IRCOMM_RTS_CTS_OUT      0x08
+#define IRCOMM_DSR_DTR_IN       0x10
+#define IRCOMM_DSR_DTR_OUT      0x20
+#define IRCOMM_ENQ_ACK_IN       0x40
+#define IRCOMM_ENQ_ACK_OUT      0x80
+
+/* Line status */
+#define IRCOMM_OVERRUN_ERROR    0x02
+#define IRCOMM_PARITY_ERROR     0x04
+#define IRCOMM_FRAMING_ERROR    0x08
+
+/* DTE (Data terminal equipment) line settings */
+#define IRCOMM_DELTA_DTR        0x01
+#define IRCOMM_DELTA_RTS        0x02
+#define IRCOMM_DTR              0x04
+#define IRCOMM_RTS              0x08
+
+/* DCE (Data communications equipment) line settings */
+#define IRCOMM_DELTA_CTS        0x01  /* Clear to send has changed */
+#define IRCOMM_DELTA_DSR        0x02  /* Data set ready has changed */
+#define IRCOMM_DELTA_RI         0x04  /* Ring indicator has changed */
+#define IRCOMM_DELTA_CD         0x08  /* Carrier detect has changed */
+#define IRCOMM_CTS              0x10  /* Clear to send is high */
+#define IRCOMM_DSR              0x20  /* Data set ready is high */
+#define IRCOMM_RI               0x40  /* Ring indicator is high */
+#define IRCOMM_CD               0x80  /* Carrier detect is high */
+#define IRCOMM_DCE_DELTA_ANY    0x0f
+
+/*
+ * Parameter state
+ */
+struct ircomm_params {
+	/* General control params */
+	__u8  service_type;
+	__u8  port_type;
+	char  port_name[32];
+
+	/* Control params for 3- and 9-wire service type */
+	__u32 data_rate;         /* Data rate in bps */
+	__u8  data_format;
+	__u8  flow_control;
+	char  xonxoff[2];
+	char  enqack[2];
+	__u8  line_status;
+	__u8  _break;
+
+	__u8  null_modem;
+
+	/* Control params for 9-wire service type */
+	__u8 dte;
+	__u8 dce;
+	__u8 poll;
+
+	/* Control params for Centronics service type */
+};
+
+struct ircomm_tty_cb; /* Forward decl. */
+
+int ircomm_param_request(struct ircomm_tty_cb *self, __u8 pi, int flush);
+
+extern pi_param_info_t ircomm_param_info;
+
+#endif /* IRCOMM_PARAMS_H */
+
diff --git a/drivers/staging/irda/include/net/irda/ircomm_ttp.h b/drivers/staging/irda/include/net/irda/ircomm_ttp.h
new file mode 100644
index 000000000000..c5627288bca3
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/ircomm_ttp.h
@@ -0,0 +1,37 @@
+/*********************************************************************
+ *                
+ * Filename:      ircomm_ttp.h
+ * Version:       
+ * Description:   
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Wed Jun  9 10:06:07 1999
+ * Modified at:   Fri Aug 13 07:32:22 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1999 Dag Brattli, All Rights Reserved.
+ *     
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ * 
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ * 
+ *     You should have received a copy of the GNU General Public License 
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *     
+ ********************************************************************/
+
+#ifndef IRCOMM_TTP_H
+#define IRCOMM_TTP_H
+
+#include <net/irda/ircomm_core.h>
+
+int  ircomm_open_tsap(struct ircomm_cb *self);
+
+#endif
+
diff --git a/drivers/staging/irda/include/net/irda/ircomm_tty.h b/drivers/staging/irda/include/net/irda/ircomm_tty.h
new file mode 100644
index 000000000000..8d4f588974bc
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/ircomm_tty.h
@@ -0,0 +1,121 @@
+/*********************************************************************
+ *                
+ * Filename:      ircomm_tty.h
+ * Version:       
+ * Description:   
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sun Jun  6 23:24:22 1999
+ * Modified at:   Fri Jan 28 13:16:57 2000
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1999-2000 Dag Brattli, All Rights Reserved.
+ *     
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ * 
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ * 
+ *     You should have received a copy of the GNU General Public License 
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *     
+ ********************************************************************/
+
+#ifndef IRCOMM_TTY_H
+#define IRCOMM_TTY_H
+
+#include <linux/serial.h>
+#include <linux/termios.h>
+#include <linux/timer.h>
+#include <linux/tty.h>		/* struct tty_struct */
+
+#include <net/irda/irias_object.h>
+#include <net/irda/ircomm_core.h>
+#include <net/irda/ircomm_param.h>
+
+#define IRCOMM_TTY_PORTS 32
+#define IRCOMM_TTY_MAGIC 0x3432
+#define IRCOMM_TTY_MAJOR 161
+#define IRCOMM_TTY_MINOR 0
+
+/* This is used as an initial value to max_header_size before the proper
+ * value is filled in (5 for ttp, 4 for lmp). This allow us to detect
+ * the state of the underlying connection. - Jean II */
+#define IRCOMM_TTY_HDR_UNINITIALISED	16
+/* Same for payload size. See qos.c for the smallest max data size */
+#define IRCOMM_TTY_DATA_UNINITIALISED	(64 - IRCOMM_TTY_HDR_UNINITIALISED)
+
+/*
+ * IrCOMM TTY driver state
+ */
+struct ircomm_tty_cb {
+	irda_queue_t queue;            /* Must be first */
+	struct tty_port port;
+	magic_t magic;
+
+	int state;                /* Connect state */
+
+	struct ircomm_cb *ircomm; /* IrCOMM layer instance */
+
+	struct sk_buff *tx_skb;   /* Transmit buffer */
+	struct sk_buff *ctrl_skb; /* Control data buffer */
+
+	/* Parameters */
+	struct ircomm_params settings;
+
+	__u8 service_type;        /* The service that we support */
+	int client;               /* True if we are a client */
+	LOCAL_FLOW flow;          /* IrTTP flow status */
+
+	int line;
+
+	__u8 dlsap_sel;
+	__u8 slsap_sel;
+
+	__u32 saddr;
+	__u32 daddr;
+
+	__u32 max_data_size;   /* Max data we can transmit in one packet */
+	__u32 max_header_size; /* The amount of header space we must reserve */
+	__u32 tx_data_size;	/* Max data size of current tx_skb */
+
+	struct iriap_cb *iriap; /* Instance used for querying remote IAS */
+	struct ias_object* obj;
+	void *skey;
+	void *ckey;
+
+	struct timer_list watchdog_timer;
+	struct work_struct  tqueue;
+
+	/* Protect concurent access to :
+	 *	o self->ctrl_skb
+	 *	o self->tx_skb
+	 * Maybe other things may gain to be protected as well...
+	 * Jean II */
+	spinlock_t spinlock;
+};
+
+void ircomm_tty_start(struct tty_struct *tty);
+void ircomm_tty_check_modem_status(struct ircomm_tty_cb *self);
+
+int ircomm_tty_tiocmget(struct tty_struct *tty);
+int ircomm_tty_tiocmset(struct tty_struct *tty, unsigned int set,
+			unsigned int clear);
+int ircomm_tty_ioctl(struct tty_struct *tty, unsigned int cmd,
+		     unsigned long arg);
+void ircomm_tty_set_termios(struct tty_struct *tty,
+			    struct ktermios *old_termios);
+
+#endif
+
+
+
+
+
+
+
diff --git a/drivers/staging/irda/include/net/irda/ircomm_tty_attach.h b/drivers/staging/irda/include/net/irda/ircomm_tty_attach.h
new file mode 100644
index 000000000000..20dcbdf258cf
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/ircomm_tty_attach.h
@@ -0,0 +1,92 @@
+/*********************************************************************
+ *                
+ * Filename:      ircomm_tty_attach.h
+ * Version:       
+ * Description:   
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Wed Jun  9 15:55:18 1999
+ * Modified at:   Fri Dec 10 21:04:55 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1999 Dag Brattli, All Rights Reserved.
+ *     
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ * 
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ * 
+ *     You should have received a copy of the GNU General Public License 
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *     
+ ********************************************************************/
+
+#ifndef IRCOMM_TTY_ATTACH_H
+#define IRCOMM_TTY_ATTACH_H
+
+#include <net/irda/ircomm_tty.h>
+
+typedef enum {
+        IRCOMM_TTY_IDLE,
+	IRCOMM_TTY_SEARCH,
+        IRCOMM_TTY_QUERY_PARAMETERS,
+	IRCOMM_TTY_QUERY_LSAP_SEL,
+	IRCOMM_TTY_SETUP,
+        IRCOMM_TTY_READY,
+} IRCOMM_TTY_STATE;
+
+/* IrCOMM TTY Events */
+typedef enum {
+	IRCOMM_TTY_ATTACH_CABLE,
+	IRCOMM_TTY_DETACH_CABLE,
+	IRCOMM_TTY_DATA_REQUEST,
+	IRCOMM_TTY_DATA_INDICATION,
+	IRCOMM_TTY_DISCOVERY_REQUEST,
+	IRCOMM_TTY_DISCOVERY_INDICATION,
+	IRCOMM_TTY_CONNECT_CONFIRM,
+	IRCOMM_TTY_CONNECT_INDICATION,
+	IRCOMM_TTY_DISCONNECT_REQUEST,
+	IRCOMM_TTY_DISCONNECT_INDICATION,
+	IRCOMM_TTY_WD_TIMER_EXPIRED,
+	IRCOMM_TTY_GOT_PARAMETERS,
+	IRCOMM_TTY_GOT_LSAPSEL,
+} IRCOMM_TTY_EVENT;
+
+/* Used for passing information through the state-machine */
+struct ircomm_tty_info {
+        __u32     saddr;               /* Source device address */
+        __u32     daddr;               /* Destination device address */
+        __u8      dlsap_sel;
+};
+
+extern const char *const ircomm_state[];
+extern const char *const ircomm_tty_state[];
+
+int ircomm_tty_do_event(struct ircomm_tty_cb *self, IRCOMM_TTY_EVENT event,
+			struct sk_buff *skb, struct ircomm_tty_info *info);
+
+
+int  ircomm_tty_attach_cable(struct ircomm_tty_cb *self);
+void ircomm_tty_detach_cable(struct ircomm_tty_cb *self);
+void ircomm_tty_connect_confirm(void *instance, void *sap, 
+				struct qos_info *qos, 
+				__u32 max_sdu_size, 
+				__u8 max_header_size, 
+				struct sk_buff *skb);
+void ircomm_tty_disconnect_indication(void *instance, void *sap, 
+				      LM_REASON reason,
+				      struct sk_buff *skb);
+void ircomm_tty_connect_indication(void *instance, void *sap, 
+				   struct qos_info *qos, 
+				   __u32 max_sdu_size,
+				   __u8 max_header_size, 
+				   struct sk_buff *skb);
+int ircomm_tty_send_initial_parameters(struct ircomm_tty_cb *self);
+void ircomm_tty_link_established(struct ircomm_tty_cb *self);
+
+#endif /* IRCOMM_TTY_ATTACH_H */
diff --git a/drivers/staging/irda/include/net/irda/irda.h b/drivers/staging/irda/include/net/irda/irda.h
new file mode 100644
index 000000000000..92c8fb575213
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/irda.h
@@ -0,0 +1,115 @@
+/*********************************************************************
+ *                
+ * Filename:      irda.h
+ * Version:       1.0
+ * Description:   IrDA common include file for kernel internal use
+ * Status:        Stable
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Tue Dec  9 21:13:12 1997
+ * Modified at:   Fri Jan 28 13:16:32 2000
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1998-2000 Dag Brattli, All Rights Reserved.
+ *     Copyright (c) 2000-2002 Jean Tourrilhes <jt@hpl.hp.com>
+ *      
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ *  
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is 
+ *     provided "AS-IS" and at no charge.
+ *     
+ ********************************************************************/
+
+#ifndef NET_IRDA_H
+#define NET_IRDA_H
+
+#include <linux/skbuff.h>		/* struct sk_buff */
+#include <linux/kernel.h>
+#include <linux/if.h>			/* sa_family_t in <linux/irda.h> */
+#include <linux/irda.h>
+
+typedef __u32 magic_t;
+
+#ifndef TRUE
+#define TRUE 1
+#endif
+
+#ifndef FALSE 
+#define FALSE 0
+#endif
+
+/* Hack to do small backoff when setting media busy in IrLAP */
+#ifndef SMALL
+#define SMALL 5
+#endif
+
+#ifndef IRDA_MIN /* Lets not mix this MIN with other header files */
+#define IRDA_MIN(a, b) (((a) < (b)) ? (a) : (b))
+#endif
+
+#ifndef IRDA_ALIGN
+#  define IRDA_ALIGN __attribute__((aligned))
+#endif
+
+#ifdef CONFIG_IRDA_DEBUG
+#define IRDA_ASSERT(expr, func) \
+do { if(!(expr)) { \
+	printk( "Assertion failed! %s:%s:%d %s\n", \
+		__FILE__,__func__,__LINE__,(#expr) ); \
+	func } } while (0)
+#define IRDA_ASSERT_LABEL(label)	label
+#else
+#define IRDA_ASSERT(expr, func) do { (void)(expr); } while (0)
+#define IRDA_ASSERT_LABEL(label)
+#endif /* CONFIG_IRDA_DEBUG */
+
+/*
+ *  Magic numbers used by Linux-IrDA. Random numbers which must be unique to 
+ *  give the best protection
+ */
+
+#define IRTTY_MAGIC        0x2357
+#define LAP_MAGIC          0x1357
+#define LMP_MAGIC          0x4321
+#define LMP_LSAP_MAGIC     0x69333
+#define LMP_LAP_MAGIC      0x3432
+#define IRDA_DEVICE_MAGIC  0x63454
+#define IAS_MAGIC          0x007
+#define TTP_MAGIC          0x241169
+#define TTP_TSAP_MAGIC     0x4345
+#define IROBEX_MAGIC       0x341324
+#define HB_MAGIC           0x64534
+#define IRLAN_MAGIC        0x754
+#define IAS_OBJECT_MAGIC   0x34234
+#define IAS_ATTRIB_MAGIC   0x45232
+#define IRDA_TASK_MAGIC    0x38423
+
+#define IAS_DEVICE_ID 0x0000 /* Defined by IrDA, IrLMP section 4.1 (page 68) */
+#define IAS_PNP_ID    0xd342
+#define IAS_OBEX_ID   0x34323
+#define IAS_IRLAN_ID  0x34234
+#define IAS_IRCOMM_ID 0x2343
+#define IAS_IRLPT_ID  0x9876
+
+struct net_device;
+struct packet_type;
+
+void irda_proc_register(void);
+void irda_proc_unregister(void);
+
+int irda_sysctl_register(void);
+void irda_sysctl_unregister(void);
+
+int irsock_init(void);
+void irsock_cleanup(void);
+
+int irda_nl_register(void);
+void irda_nl_unregister(void);
+
+int irlap_driver_rcv(struct sk_buff *skb, struct net_device *dev,
+		     struct packet_type *ptype, struct net_device *orig_dev);
+
+#endif /* NET_IRDA_H */
diff --git a/drivers/staging/irda/include/net/irda/irda_device.h b/drivers/staging/irda/include/net/irda/irda_device.h
new file mode 100644
index 000000000000..664bf8178412
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/irda_device.h
@@ -0,0 +1,285 @@
+/*********************************************************************
+ *                
+ * Filename:      irda_device.h
+ * Version:       0.9
+ * Description:   Contains various declarations used by the drivers
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Tue Apr 14 12:41:42 1998
+ * Modified at:   Mon Mar 20 09:08:57 2000
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1999-2000 Dag Brattli, All Rights Reserved.
+ *     Copyright (c) 1998 Thomas Davis, <ratbert@radiks.net>,
+ *     Copyright (c) 2000-2002 Jean Tourrilhes <jt@hpl.hp.com>
+ *
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ * 
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ * 
+ *     You should have received a copy of the GNU General Public License 
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *     
+ ********************************************************************/
+
+/*
+ * This header contains all the IrDA definitions a driver really
+ * needs, and therefore the driver should not need to include
+ * any other IrDA headers - Jean II
+ */
+
+#ifndef IRDA_DEVICE_H
+#define IRDA_DEVICE_H
+
+#include <linux/tty.h>
+#include <linux/netdevice.h>
+#include <linux/spinlock.h>
+#include <linux/skbuff.h>		/* struct sk_buff */
+#include <linux/irda.h>
+#include <linux/types.h>
+
+#include <net/pkt_sched.h>
+#include <net/irda/irda.h>
+#include <net/irda/qos.h>		/* struct qos_info */
+#include <net/irda/irqueue.h>		/* irda_queue_t */
+
+/* A few forward declarations (to make compiler happy) */
+struct irlap_cb;
+
+/* Some non-standard interface flags (should not conflict with any in if.h) */
+#define IFF_SIR 	0x0001 /* Supports SIR speeds */
+#define IFF_MIR 	0x0002 /* Supports MIR speeds */
+#define IFF_FIR 	0x0004 /* Supports FIR speeds */
+#define IFF_VFIR        0x0008 /* Supports VFIR speeds */
+#define IFF_PIO   	0x0010 /* Supports PIO transfer of data */
+#define IFF_DMA		0x0020 /* Supports DMA transfer of data */
+#define IFF_SHM         0x0040 /* Supports shared memory data transfers */
+#define IFF_DONGLE      0x0080 /* Interface has a dongle attached */
+#define IFF_AIR         0x0100 /* Supports Advanced IR (AIR) standards */
+
+#define IO_XMIT 0x01
+#define IO_RECV 0x02
+
+typedef enum {
+	IRDA_IRLAP, /* IrDA mode, and deliver to IrLAP */
+	IRDA_RAW,   /* IrDA mode */
+	SHARP_ASK,
+	TV_REMOTE,  /* Also known as Consumer Electronics IR */
+} INFRARED_MODE;
+
+typedef enum {
+	IRDA_TASK_INIT,        /* All tasks are initialized with this state */
+	IRDA_TASK_DONE,        /* Signals that the task is finished */
+	IRDA_TASK_WAIT,
+	IRDA_TASK_WAIT1,
+	IRDA_TASK_WAIT2,
+	IRDA_TASK_WAIT3,
+	IRDA_TASK_CHILD_INIT,  /* Initializing child task */
+	IRDA_TASK_CHILD_WAIT,  /* Waiting for child task to finish */
+	IRDA_TASK_CHILD_DONE   /* Child task is finished */
+} IRDA_TASK_STATE;
+
+struct irda_task;
+typedef int (*IRDA_TASK_CALLBACK) (struct irda_task *task);
+
+struct irda_task {
+	irda_queue_t q;
+	magic_t magic;
+
+	IRDA_TASK_STATE state;
+	IRDA_TASK_CALLBACK function;
+	IRDA_TASK_CALLBACK finished;
+
+	struct irda_task *parent;
+	struct timer_list timer;
+
+	void *instance; /* Instance being called */
+	void *param;    /* Parameter to be used by instance */
+};
+
+/* Dongle info */
+struct dongle_reg;
+typedef struct {
+	struct dongle_reg *issue;     /* Registration info */
+	struct net_device *dev;           /* Device we are attached to */
+	struct irda_task *speed_task; /* Task handling speed change */
+	struct irda_task *reset_task; /* Task handling reset */
+	__u32 speed;                  /* Current speed */
+
+	/* Callbacks to the IrDA device driver */
+	int (*set_mode)(struct net_device *, int mode);
+	int (*read)(struct net_device *dev, __u8 *buf, int len);
+	int (*write)(struct net_device *dev, __u8 *buf, int len);
+	int (*set_dtr_rts)(struct net_device *dev, int dtr, int rts);
+} dongle_t;
+
+/* Dongle registration info */
+struct dongle_reg {
+	irda_queue_t q;         /* Must be first */
+	IRDA_DONGLE type;
+
+	void (*open)(dongle_t *dongle, struct qos_info *qos);
+	void (*close)(dongle_t *dongle);
+	int  (*reset)(struct irda_task *task);
+	int  (*change_speed)(struct irda_task *task);
+	struct module *owner;
+};
+
+/* 
+ * Per-packet information we need to hide inside sk_buff 
+ * (must not exceed 48 bytes, check with struct sk_buff)
+ * The default_qdisc_pad field is a temporary hack.
+ */
+struct irda_skb_cb {
+	unsigned int default_qdisc_pad;
+	magic_t magic;       /* Be sure that we can trust the information */
+	__u32   next_speed;  /* The Speed to be set *after* this frame */
+	__u16   mtt;         /* Minimum turn around time */
+	__u16   xbofs;       /* Number of xbofs required, used by SIR mode */
+	__u16   next_xbofs;  /* Number of xbofs required *after* this frame */
+	void    *context;    /* May be used by drivers */
+	void    (*destructor)(struct sk_buff *skb); /* Used for flow control */
+	__u16   xbofs_delay; /* Number of xbofs used for generating the mtt */
+	__u8    line;        /* Used by IrCOMM in IrLPT mode */
+};
+
+/* Chip specific info */
+typedef struct {
+	int cfg_base;         /* Config register IO base */
+        int sir_base;         /* SIR IO base */
+	int fir_base;         /* FIR IO base */
+	int mem_base;         /* Shared memory base */
+        int sir_ext;          /* Length of SIR iobase */
+	int fir_ext;          /* Length of FIR iobase */
+        int irq, irq2;        /* Interrupts used */
+        int dma, dma2;        /* DMA channel(s) used */
+        int fifo_size;        /* FIFO size */
+        int irqflags;         /* interrupt flags (ie, IRQF_SHARED) */
+	int direction;        /* Link direction, used by some FIR drivers */
+	int enabled;          /* Powered on? */
+	int suspended;        /* Suspended by APM */
+	__u32 speed;          /* Currently used speed */
+	__u32 new_speed;      /* Speed we must change to when Tx is finished */
+	int dongle_id;        /* Dongle or transceiver currently used */
+} chipio_t;
+
+/* IO buffer specific info (inspired by struct sk_buff) */
+typedef struct {
+	int state;            /* Receiving state (transmit state not used) */
+	int in_frame;         /* True if receiving frame */
+
+	__u8 *head;	      /* start of buffer */
+	__u8 *data;	      /* start of data in buffer */
+
+	int len;	      /* current length of data */
+	int truesize;	      /* total allocated size of buffer */
+	__u16 fcs;
+
+	struct sk_buff *skb;	/* ZeroCopy Rx in async_unwrap_char() */
+} iobuff_t;
+
+/* Maximum SIR frame (skb) that we expect to receive *unwrapped*.
+ * Max LAP MTU (I field) is 2048 bytes max (IrLAP 1.1, chapt 6.6.5, p40).
+ * Max LAP header is 2 bytes (for now).
+ * Max CRC is 2 bytes at SIR, 4 bytes at FIR. 
+ * Need 1 byte for skb_reserve() to align IP header for IrLAN.
+ * Add a few extra bytes just to be safe (buffer is power of two anyway)
+ * Jean II */
+#define IRDA_SKB_MAX_MTU	2064
+/* Maximum SIR frame that we expect to send, wrapped (i.e. with XBOFS
+ * and escaped characters on top of above). */
+#define IRDA_SIR_MAX_FRAME	4269
+
+/* The SIR unwrapper async_unwrap_char() will use a Rx-copy-break mechanism
+ * when using the optional ZeroCopy Rx, where only small frames are memcpy
+ * to a smaller skb to save memory. This is the threshold under which copy
+ * will happen (and over which it won't happen).
+ * Some FIR drivers may use this #define as well...
+ * This is the same value as various Ethernet drivers. - Jean II */
+#define IRDA_RX_COPY_THRESHOLD  256
+
+/* Function prototypes */
+int  irda_device_init(void);
+void irda_device_cleanup(void);
+
+/* IrLAP entry points used by the drivers.
+ * We declare them here to avoid the driver pulling a whole bunch stack
+ * headers they don't really need - Jean II */
+struct irlap_cb *irlap_open(struct net_device *dev, struct qos_info *qos,
+			    const char *hw_name);
+void irlap_close(struct irlap_cb *self);
+
+/* Interface to be uses by IrLAP */
+void irda_device_set_media_busy(struct net_device *dev, int status);
+int  irda_device_is_media_busy(struct net_device *dev);
+int  irda_device_is_receiving(struct net_device *dev);
+
+/* Interface for internal use */
+static inline int irda_device_txqueue_empty(const struct net_device *dev)
+{
+	return qdisc_all_tx_empty(dev);
+}
+int  irda_device_set_raw_mode(struct net_device* self, int status);
+struct net_device *alloc_irdadev(int sizeof_priv);
+
+void irda_setup_dma(int channel, dma_addr_t buffer, int count, int mode);
+
+/*
+ * Function irda_get_mtt (skb)
+ *
+ *    Utility function for getting the minimum turnaround time out of 
+ *    the skb, where it has been hidden in the cb field.
+ */
+static inline __u16 irda_get_mtt(const struct sk_buff *skb)
+{
+	const struct irda_skb_cb *cb = (const struct irda_skb_cb *) skb->cb;
+	return (cb->magic == LAP_MAGIC) ? cb->mtt : 10000;
+}
+
+/*
+ * Function irda_get_next_speed (skb)
+ *
+ *    Extract the speed that should be set *after* this frame from the skb
+ *
+ * Note : return -1 for user space frames
+ */
+static inline __u32 irda_get_next_speed(const struct sk_buff *skb)
+{
+	const struct irda_skb_cb *cb = (const struct irda_skb_cb *) skb->cb;
+	return (cb->magic == LAP_MAGIC) ? cb->next_speed : -1;
+}
+
+/*
+ * Function irda_get_next_xbofs (skb)
+ *
+ *    Extract the xbofs that should be set for this frame from the skb
+ *
+ * Note : default to 10 for user space frames
+ */
+static inline __u16 irda_get_xbofs(const struct sk_buff *skb)
+{
+	const struct irda_skb_cb *cb = (const struct irda_skb_cb *) skb->cb;
+	return (cb->magic == LAP_MAGIC) ? cb->xbofs : 10;
+}
+
+/*
+ * Function irda_get_next_xbofs (skb)
+ *
+ *    Extract the xbofs that should be set *after* this frame from the skb
+ *
+ * Note : return -1 for user space frames
+ */
+static inline __u16 irda_get_next_xbofs(const struct sk_buff *skb)
+{
+	const struct irda_skb_cb *cb = (const struct irda_skb_cb *) skb->cb;
+	return (cb->magic == LAP_MAGIC) ? cb->next_xbofs : -1;
+}
+#endif /* IRDA_DEVICE_H */
+
+
diff --git a/drivers/staging/irda/include/net/irda/iriap.h b/drivers/staging/irda/include/net/irda/iriap.h
new file mode 100644
index 000000000000..fcc896491a95
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/iriap.h
@@ -0,0 +1,108 @@
+/*********************************************************************
+ *                
+ * Filename:      iriap.h
+ * Version:       0.5
+ * Description:   Information Access Protocol (IAP)
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Thu Aug 21 00:02:07 1997
+ * Modified at:   Sat Dec 25 16:42:09 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1997-1999 Dag Brattli <dagb@cs.uit.no>, 
+ *     All Rights Reserved.
+ *     
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is 
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#ifndef IRIAP_H
+#define IRIAP_H
+
+#include <linux/types.h>
+#include <linux/skbuff.h>
+
+#include <net/irda/iriap_event.h>
+#include <net/irda/irias_object.h>
+#include <net/irda/irqueue.h>		/* irda_queue_t */
+#include <net/irda/timer.h>		/* struct timer_list */
+
+#define IAP_LST 0x80
+#define IAP_ACK 0x40
+
+#define IAS_SERVER 0
+#define IAS_CLIENT 1
+
+/* IrIAP Op-codes */
+#define GET_INFO_BASE      0x01
+#define GET_OBJECTS        0x02
+#define GET_VALUE          0x03
+#define GET_VALUE_BY_CLASS 0x04
+#define GET_OBJECT_INFO    0x05
+#define GET_ATTRIB_NAMES   0x06
+
+#define IAS_SUCCESS        0
+#define IAS_CLASS_UNKNOWN  1
+#define IAS_ATTRIB_UNKNOWN 2
+#define IAS_DISCONNECT     10
+
+typedef void (*CONFIRM_CALLBACK)(int result, __u16 obj_id, 
+				 struct ias_value *value, void *priv);
+
+struct iriap_cb {
+	irda_queue_t q; /* Must be first */	
+	magic_t magic;  /* Magic cookie */
+
+	int          mode;   /* Client or server */
+
+	__u32        saddr;
+	__u32        daddr;
+	__u8         operation;
+
+	struct sk_buff *request_skb;
+	struct lsap_cb *lsap;
+	__u8 slsap_sel;
+
+	/* Client states */
+	IRIAP_STATE client_state;
+	IRIAP_STATE call_state;
+	
+	/* Server states */
+	IRIAP_STATE server_state;
+	IRIAP_STATE r_connect_state;
+	
+	CONFIRM_CALLBACK confirm;
+	void *priv;                /* Used to identify client */
+
+	__u8 max_header_size;
+	__u32 max_data_size;
+	
+	struct timer_list watchdog_timer;
+};
+
+int  iriap_init(void);
+void iriap_cleanup(void);
+
+struct iriap_cb *iriap_open(__u8 slsap_sel, int mode, void *priv, 
+			    CONFIRM_CALLBACK callback);
+void iriap_close(struct iriap_cb *self);
+
+int iriap_getvaluebyclass_request(struct iriap_cb *self, 
+				  __u32 saddr, __u32 daddr,
+				  char *name, char *attr);
+void iriap_connect_request(struct iriap_cb *self);
+void iriap_send_ack( struct iriap_cb *self);
+void iriap_call_indication(struct iriap_cb *self, struct sk_buff *skb);
+
+void iriap_register_server(void);
+
+#endif
+
+
diff --git a/drivers/staging/irda/include/net/irda/iriap_event.h b/drivers/staging/irda/include/net/irda/iriap_event.h
new file mode 100644
index 000000000000..89747f06d9eb
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/iriap_event.h
@@ -0,0 +1,85 @@
+/*********************************************************************
+ *                
+ * Filename:      iriap_event.h
+ * Version:       
+ * Description:   
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Mon Aug  4 20:40:53 1997
+ * Modified at:   Sun Oct 31 22:02:54 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1998-1999 Dag Brattli <dagb@cs.uit.no>, All Rights Reserved.
+ *     
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is 
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#ifndef IRIAP_FSM_H
+#define IRIAP_FSM_H
+
+/* Forward because of circular include dependecies */
+struct iriap_cb;
+
+/* IrIAP states */
+typedef enum {
+	/* Client */
+	S_DISCONNECT,
+	S_CONNECTING,
+	S_CALL,
+
+	/* S-Call */
+	S_MAKE_CALL,
+	S_CALLING,
+	S_OUTSTANDING,
+	S_REPLYING,
+	S_WAIT_FOR_CALL,
+	S_WAIT_ACTIVE,
+
+	/* Server */
+	R_DISCONNECT,
+	R_CALL,
+	
+	/* R-Connect */
+	R_WAITING,
+	R_WAIT_ACTIVE,
+	R_RECEIVING,
+	R_EXECUTE,
+	R_RETURNING,
+} IRIAP_STATE;
+
+typedef enum {
+	IAP_CALL_REQUEST,
+	IAP_CALL_REQUEST_GVBC,
+	IAP_CALL_RESPONSE,
+	IAP_RECV_F_LST,
+	IAP_LM_DISCONNECT_INDICATION,
+	IAP_LM_CONNECT_INDICATION,
+	IAP_LM_CONNECT_CONFIRM,
+} IRIAP_EVENT;
+
+void iriap_next_client_state   (struct iriap_cb *self, IRIAP_STATE state);
+void iriap_next_call_state     (struct iriap_cb *self, IRIAP_STATE state);
+void iriap_next_server_state   (struct iriap_cb *self, IRIAP_STATE state);
+void iriap_next_r_connect_state(struct iriap_cb *self, IRIAP_STATE state);
+
+
+void iriap_do_client_event(struct iriap_cb *self, IRIAP_EVENT event, 
+			   struct sk_buff *skb);
+void iriap_do_call_event  (struct iriap_cb *self, IRIAP_EVENT event, 
+			   struct sk_buff *skb);
+
+void iriap_do_server_event   (struct iriap_cb *self, IRIAP_EVENT event, 
+			      struct sk_buff *skb);
+void iriap_do_r_connect_event(struct iriap_cb *self, IRIAP_EVENT event, 
+			      struct sk_buff *skb);
+
+#endif /* IRIAP_FSM_H */
+
diff --git a/drivers/staging/irda/include/net/irda/irias_object.h b/drivers/staging/irda/include/net/irda/irias_object.h
new file mode 100644
index 000000000000..83f78081799c
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/irias_object.h
@@ -0,0 +1,108 @@
+/*********************************************************************
+ *                
+ * Filename:      irias_object.h
+ * Version:       
+ * Description:   
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Thu Oct  1 22:49:50 1998
+ * Modified at:   Wed Dec 15 11:20:57 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1998-1999 Dag Brattli, All Rights Reserved.
+ *      
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ *  
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is 
+ *     provided "AS-IS" and at no charge.
+ *     
+ ********************************************************************/
+
+#ifndef LM_IAS_OBJECT_H
+#define LM_IAS_OBJECT_H
+
+#include <net/irda/irda.h>
+#include <net/irda/irqueue.h>
+
+/* LM-IAS Attribute types */
+#define IAS_MISSING 0
+#define IAS_INTEGER 1
+#define IAS_OCT_SEQ 2
+#define IAS_STRING  3
+
+/* Object ownership of attributes (user or kernel) */
+#define IAS_KERNEL_ATTR	0
+#define IAS_USER_ATTR	1
+
+/*
+ *  LM-IAS Object
+ */
+struct ias_object {
+	irda_queue_t q;     /* Must be first! */
+	magic_t magic;
+	
+	char  *name;
+	int   id;
+	hashbin_t *attribs;
+};
+
+/*
+ *  Values used by LM-IAS attributes
+ */
+struct ias_value {
+        __u8    type;    /* Value description */
+	__u8	owner;	/* Managed from user/kernel space */
+	int     charset; /* Only used by string type */
+        int     len;
+	
+	/* Value */
+	union {
+		int integer;
+		char *string;
+		__u8 *oct_seq;
+	} t;
+};
+
+/*
+ *  Attributes used by LM-IAS objects
+ */
+struct ias_attrib {
+	irda_queue_t q; /* Must be first! */
+	int magic;
+
+        char *name;   	         /* Attribute name */
+	struct ias_value *value; /* Attribute value */
+};
+
+struct ias_object *irias_new_object(char *name, int id);
+void irias_insert_object(struct ias_object *obj);
+int  irias_delete_object(struct ias_object *obj);
+int  irias_delete_attrib(struct ias_object *obj, struct ias_attrib *attrib,
+			 int cleanobject);
+void __irias_delete_object(struct ias_object *obj);
+
+void irias_add_integer_attrib(struct ias_object *obj, char *name, int value,
+			      int user);
+void irias_add_string_attrib(struct ias_object *obj, char *name, char *value,
+			     int user);
+void irias_add_octseq_attrib(struct ias_object *obj, char *name, __u8 *octets,
+			     int len, int user);
+int irias_object_change_attribute(char *obj_name, char *attrib_name, 
+				  struct ias_value *new_value);
+struct ias_object *irias_find_object(char *name);
+struct ias_attrib *irias_find_attrib(struct ias_object *obj, char *name);
+
+struct ias_value *irias_new_string_value(char *string);
+struct ias_value *irias_new_integer_value(int integer);
+struct ias_value *irias_new_octseq_value(__u8 *octseq , int len);
+struct ias_value *irias_new_missing_value(void);
+void irias_delete_value(struct ias_value *value);
+
+extern struct ias_value irias_missing;
+extern hashbin_t *irias_objects;
+
+#endif
diff --git a/drivers/staging/irda/include/net/irda/irlan_client.h b/drivers/staging/irda/include/net/irda/irlan_client.h
new file mode 100644
index 000000000000..fa8455eda280
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/irlan_client.h
@@ -0,0 +1,42 @@
+/*********************************************************************
+ *                
+ * Filename:      irlan_client.h
+ * Version:       0.3
+ * Description:   IrDA LAN access layer
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sun Aug 31 20:14:37 1997
+ * Modified at:   Thu Apr 22 14:13:34 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1998 Dag Brattli <dagb@cs.uit.no>, All Rights Reserved.
+ *     
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is 
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#ifndef IRLAN_CLIENT_H
+#define IRLAN_CLIENT_H
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/skbuff.h>
+#include <linux/netdevice.h>
+
+#include <net/irda/irias_object.h>
+#include <net/irda/irlan_event.h>
+
+void irlan_client_discovery_indication(discinfo_t *, DISCOVERY_MODE, void *);
+void irlan_client_wakeup(struct irlan_cb *self, __u32 saddr, __u32 daddr);
+
+void irlan_client_parse_response(struct irlan_cb *self, struct sk_buff *skb);
+void irlan_client_get_value_confirm(int result, __u16 obj_id, 
+				    struct ias_value *value, void *priv);
+#endif
diff --git a/drivers/staging/irda/include/net/irda/irlan_common.h b/drivers/staging/irda/include/net/irda/irlan_common.h
new file mode 100644
index 000000000000..550c2d6ec7ff
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/irlan_common.h
@@ -0,0 +1,230 @@
+/*********************************************************************
+ *                
+ * Filename:      irlan_common.h
+ * Version:       0.8
+ * Description:   IrDA LAN access layer
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sun Aug 31 20:14:37 1997
+ * Modified at:   Sun Oct 31 19:41:24 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1998-1999 Dag Brattli <dagb@cs.uit.no>, 
+ *     All Rights Reserved.
+ *     
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is 
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#ifndef IRLAN_H
+#define IRLAN_H
+
+#include <asm/param.h>  /* for HZ */
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/skbuff.h>
+#include <linux/netdevice.h>
+#include <linux/if_ether.h>
+
+#include <net/irda/irttp.h>
+
+#define IRLAN_MTU        1518
+#define IRLAN_TIMEOUT    10*HZ /* 10 seconds */
+
+/* Command packet types */
+#define CMD_GET_PROVIDER_INFO   0
+#define CMD_GET_MEDIA_CHAR      1
+#define CMD_OPEN_DATA_CHANNEL   2
+#define CMD_CLOSE_DATA_CHAN     3
+#define CMD_RECONNECT_DATA_CHAN 4
+#define CMD_FILTER_OPERATION    5
+
+/* Some responses */
+#define RSP_SUCCESS                 0
+#define RSP_INSUFFICIENT_RESOURCES  1
+#define RSP_INVALID_COMMAND_FORMAT  2
+#define RSP_COMMAND_NOT_SUPPORTED   3
+#define RSP_PARAM_NOT_SUPPORTED     4
+#define RSP_VALUE_NOT_SUPPORTED     5
+#define RSP_NOT_OPEN                6
+#define RSP_AUTHENTICATION_REQUIRED 7
+#define RSP_INVALID_PASSWORD        8
+#define RSP_PROTOCOL_ERROR          9
+#define RSP_ASYNCHRONOUS_ERROR    255
+
+/* Media types */
+#define MEDIA_802_3 1
+#define MEDIA_802_5 2
+
+/* Filter parameters */
+#define DATA_CHAN   1
+#define FILTER_TYPE 2
+#define FILTER_MODE 3
+
+/* Filter types */
+#define IRLAN_DIRECTED   0x01
+#define IRLAN_FUNCTIONAL 0x02
+#define IRLAN_GROUP      0x04
+#define IRLAN_MAC_FRAME  0x08
+#define IRLAN_MULTICAST  0x10
+#define IRLAN_BROADCAST  0x20
+#define IRLAN_IPX_SOCKET 0x40
+
+/* Filter modes */
+#define ALL     1
+#define FILTER  2
+#define NONE    3
+
+/* Filter operations */
+#define GET     1
+#define CLEAR   2
+#define ADD     3
+#define REMOVE  4
+#define DYNAMIC 5
+
+/* Access types */
+#define ACCESS_DIRECT  1
+#define ACCESS_PEER    2
+#define ACCESS_HOSTED  3
+
+#define IRLAN_BYTE   0
+#define IRLAN_SHORT  1
+#define IRLAN_ARRAY  2
+
+/* IrLAN sits on top if IrTTP */
+#define IRLAN_MAX_HEADER (TTP_HEADER+LMP_HEADER)
+/* 1 byte for the command code and 1 byte for the parameter count */
+#define IRLAN_CMD_HEADER 2
+
+#define IRLAN_STRING_PARAMETER_LEN(name, value) (1 + strlen((name)) + 2 \
+						+ strlen ((value)))
+#define IRLAN_BYTE_PARAMETER_LEN(name)          (1 + strlen((name)) + 2 + 1)
+#define IRLAN_SHORT_PARAMETER_LEN(name)         (1 + strlen((name)) + 2 + 2)
+
+/*
+ *  IrLAN client
+ */
+struct irlan_client_cb {
+	int state;
+
+	int open_retries;
+
+	struct tsap_cb *tsap_ctrl;
+	__u32 max_sdu_size;
+	__u8  max_header_size;
+	
+	int access_type;         /* Access type of provider */
+	__u8 reconnect_key[255];
+	__u8 key_len;
+	
+	__u16 recv_arb_val;
+	__u16 max_frame;
+	int filter_type;
+
+	int unicast_open;
+	int broadcast_open;
+
+	int tx_busy;
+	struct sk_buff_head txq; /* Transmit control queue */
+
+	struct iriap_cb *iriap;
+
+	struct timer_list kick_timer;
+};
+
+/*
+ * IrLAN provider
+ */
+struct irlan_provider_cb {
+	int state;
+	
+	struct tsap_cb *tsap_ctrl;
+	__u32 max_sdu_size;
+	__u8  max_header_size;
+
+	/*
+	 *  Store some values here which are used by the provider to parse
+	 *  the filter operations
+	 */
+	int data_chan;
+	int filter_type;
+	int filter_mode;
+	int filter_operation;
+	int filter_entry;
+	int access_type;     /* Access type */
+	__u16 send_arb_val;
+
+	__u8 mac_address[ETH_ALEN]; /* Generated MAC address for peer device */
+};
+
+/*
+ *  IrLAN control block
+ */
+struct irlan_cb {
+	int    magic;
+	struct list_head  dev_list;
+	struct net_device *dev;        /* Ethernet device structure*/
+
+	__u32 saddr;               /* Source device address */
+	__u32 daddr;               /* Destination device address */
+	int disconnect_reason;     /* Why we got disconnected */
+	
+	int media;                 /* Media type */
+	__u8 version[2];           /* IrLAN version */
+	
+	struct tsap_cb *tsap_data; /* Data TSAP */
+
+	int  use_udata;            /* Use Unit Data transfers */
+
+	__u8 stsap_sel_data;       /* Source data TSAP selector */
+	__u8 dtsap_sel_data;       /* Destination data TSAP selector */
+	__u8 dtsap_sel_ctrl;       /* Destination ctrl TSAP selector */
+
+	struct irlan_client_cb   client;   /* Client specific fields */
+	struct irlan_provider_cb provider; /* Provider specific fields */
+
+	__u32 max_sdu_size;
+	__u8  max_header_size;
+	
+	wait_queue_head_t open_wait;
+	struct timer_list watchdog_timer;
+};
+
+void irlan_close(struct irlan_cb *self);
+void irlan_close_tsaps(struct irlan_cb *self);
+
+int  irlan_register_netdev(struct irlan_cb *self);
+void irlan_ias_register(struct irlan_cb *self, __u8 tsap_sel);
+void irlan_start_watchdog_timer(struct irlan_cb *self, int timeout);
+
+void irlan_open_data_tsap(struct irlan_cb *self);
+
+int irlan_run_ctrl_tx_queue(struct irlan_cb *self);
+
+struct irlan_cb *irlan_get_any(void);
+void irlan_get_provider_info(struct irlan_cb *self);
+void irlan_get_media_char(struct irlan_cb *self);
+void irlan_open_data_channel(struct irlan_cb *self);
+void irlan_close_data_channel(struct irlan_cb *self);
+void irlan_set_multicast_filter(struct irlan_cb *self, int status);
+void irlan_set_broadcast_filter(struct irlan_cb *self, int status);
+
+int irlan_insert_byte_param(struct sk_buff *skb, char *param, __u8 value);
+int irlan_insert_short_param(struct sk_buff *skb, char *param, __u16 value);
+int irlan_insert_string_param(struct sk_buff *skb, char *param, char *value);
+int irlan_insert_array_param(struct sk_buff *skb, char *name, __u8 *value, 
+			     __u16 value_len);
+
+int irlan_extract_param(__u8 *buf, char *name, char *value, __u16 *len);
+
+#endif
+
+
diff --git a/drivers/staging/irda/include/net/irda/irlan_eth.h b/drivers/staging/irda/include/net/irda/irlan_eth.h
new file mode 100644
index 000000000000..de5c81691f33
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/irlan_eth.h
@@ -0,0 +1,32 @@
+/*********************************************************************
+ *                
+ * Filename:      irlan_eth.h
+ * Version:       
+ * Description:   
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Thu Oct 15 08:36:58 1998
+ * Modified at:   Fri May 14 23:29:00 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1998-1999 Dag Brattli, All Rights Reserved.
+ *      
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ *  
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is 
+ *     provided "AS-IS" and at no charge.
+ *     
+ ********************************************************************/
+
+#ifndef IRLAN_ETH_H
+#define IRLAN_ETH_H
+
+struct net_device *alloc_irlandev(const char *name);
+int  irlan_eth_receive(void *instance, void *sap, struct sk_buff *skb);
+
+void irlan_eth_flow_indication( void *instance, void *sap, LOCAL_FLOW flow);
+#endif
diff --git a/drivers/staging/irda/include/net/irda/irlan_event.h b/drivers/staging/irda/include/net/irda/irlan_event.h
new file mode 100644
index 000000000000..018b5a77e610
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/irlan_event.h
@@ -0,0 +1,81 @@
+/*********************************************************************
+ *                
+ * Filename:      irlan_event.h
+ * Version:       
+ * Description:   LAN access
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sun Aug 31 20:14:37 1997
+ * Modified at:   Tue Feb  2 09:45:17 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1997 Dag Brattli <dagb@cs.uit.no>, All Rights Reserved.
+ *     
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is 
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#ifndef IRLAN_EVENT_H
+#define IRLAN_EVENT_H
+
+#include <linux/kernel.h>
+#include <linux/skbuff.h>
+
+#include <net/irda/irlan_common.h>
+
+typedef enum {
+	IRLAN_IDLE,
+	IRLAN_QUERY,
+	IRLAN_CONN, 
+	IRLAN_INFO,
+	IRLAN_MEDIA,
+	IRLAN_OPEN,
+	IRLAN_WAIT,
+	IRLAN_ARB, 
+	IRLAN_DATA,
+	IRLAN_CLOSE,
+	IRLAN_SYNC
+} IRLAN_STATE;
+
+typedef enum {
+	IRLAN_DISCOVERY_INDICATION,
+	IRLAN_IAS_PROVIDER_AVAIL,
+	IRLAN_IAS_PROVIDER_NOT_AVAIL,
+	IRLAN_LAP_DISCONNECT,
+	IRLAN_LMP_DISCONNECT,
+	IRLAN_CONNECT_COMPLETE,
+	IRLAN_DATA_INDICATION,
+	IRLAN_DATA_CONNECT_INDICATION,
+	IRLAN_RETRY_CONNECT,
+
+	IRLAN_CONNECT_INDICATION,
+	IRLAN_GET_INFO_CMD,
+	IRLAN_GET_MEDIA_CMD,
+	IRLAN_OPEN_DATA_CMD,
+	IRLAN_FILTER_CONFIG_CMD,
+
+	IRLAN_CHECK_CON_ARB,
+	IRLAN_PROVIDER_SIGNAL,
+
+	IRLAN_WATCHDOG_TIMEOUT,
+} IRLAN_EVENT;
+
+extern const char * const irlan_state[];
+
+void irlan_do_client_event(struct irlan_cb *self, IRLAN_EVENT event, 
+			   struct sk_buff *skb);
+
+void irlan_do_provider_event(struct irlan_cb *self, IRLAN_EVENT event, 
+			     struct sk_buff *skb);
+
+void irlan_next_client_state(struct irlan_cb *self, IRLAN_STATE state);
+void irlan_next_provider_state(struct irlan_cb *self, IRLAN_STATE state);
+
+#endif
diff --git a/drivers/staging/irda/include/net/irda/irlan_filter.h b/drivers/staging/irda/include/net/irda/irlan_filter.h
new file mode 100644
index 000000000000..a5a2539485bd
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/irlan_filter.h
@@ -0,0 +1,35 @@
+/*********************************************************************
+ *                
+ * Filename:      irlan_filter.h
+ * Version:       
+ * Description:   
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Fri Jan 29 15:24:08 1999
+ * Modified at:   Sun Feb  7 23:35:31 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1998 Dag Brattli, All Rights Reserved.
+ *      
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ *  
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is 
+ *     provided "AS-IS" and at no charge.
+ *     
+ ********************************************************************/
+
+#ifndef IRLAN_FILTER_H
+#define IRLAN_FILTER_H
+
+void irlan_check_command_param(struct irlan_cb *self, char *param, 
+			       char *value);
+void irlan_filter_request(struct irlan_cb *self, struct sk_buff *skb);
+#ifdef CONFIG_PROC_FS
+void irlan_print_filter(struct seq_file *seq, int filter_type);
+#endif
+
+#endif /* IRLAN_FILTER_H */
diff --git a/drivers/staging/irda/include/net/irda/irlan_provider.h b/drivers/staging/irda/include/net/irda/irlan_provider.h
new file mode 100644
index 000000000000..92f3b0e1029b
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/irlan_provider.h
@@ -0,0 +1,52 @@
+/*********************************************************************
+ *                
+ * Filename:      irlan_provider.h
+ * Version:       0.1
+ * Description:   IrDA LAN access layer
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sun Aug 31 20:14:37 1997
+ * Modified at:   Sun May  9 12:26:11 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1998-1999 Dag Brattli <dagb@cs.uit.no>, All Rights Reserved.
+ *     
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is 
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#ifndef IRLAN_SERVER_H
+#define IRLAN_SERVER_H
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/skbuff.h>
+#include <linux/netdevice.h>
+
+#include <net/irda/irlan_common.h>
+
+void irlan_provider_ctrl_disconnect_indication(void *instance, void *sap, 
+					       LM_REASON reason, 
+					       struct sk_buff *skb);
+
+
+void irlan_provider_connect_response(struct irlan_cb *, struct tsap_cb *);
+
+int irlan_parse_open_data_cmd(struct irlan_cb *self, struct sk_buff *skb);
+int irlan_provider_parse_command(struct irlan_cb *self, int cmd,
+				 struct sk_buff *skb);
+
+void irlan_provider_send_reply(struct irlan_cb *self, int command, 
+			       int ret_code);
+int irlan_provider_open_ctrl_tsap(struct irlan_cb *self);
+
+#endif
+
+
diff --git a/drivers/staging/irda/include/net/irda/irlap.h b/drivers/staging/irda/include/net/irda/irlap.h
new file mode 100644
index 000000000000..6f23e820618c
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/irlap.h
@@ -0,0 +1,311 @@
+/*********************************************************************
+ *                
+ * Filename:      irlap.h
+ * Version:       0.8
+ * Description:   An IrDA LAP driver for Linux
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Mon Aug  4 20:40:53 1997
+ * Modified at:   Fri Dec 10 13:21:17 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1998-1999 Dag Brattli <dagb@cs.uit.no>, 
+ *     All Rights Reserved.
+ *     Copyright (c) 2000-2002 Jean Tourrilhes <jt@hpl.hp.com>
+ *     
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is 
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#ifndef IRLAP_H
+#define IRLAP_H
+
+#include <linux/types.h>
+#include <linux/skbuff.h>
+#include <linux/netdevice.h>
+#include <linux/timer.h>
+
+#include <net/irda/irqueue.h>		/* irda_queue_t */
+#include <net/irda/qos.h>		/* struct qos_info */
+#include <net/irda/discovery.h>		/* discovery_t */
+#include <net/irda/irlap_event.h>	/* IRLAP_STATE, ... */
+#include <net/irda/irmod.h>		/* struct notify_t */
+
+#define CONFIG_IRDA_DYNAMIC_WINDOW 1
+
+#define LAP_RELIABLE   1
+#define LAP_UNRELIABLE 0
+
+#define LAP_ADDR_HEADER 1  /* IrLAP Address Header */
+#define LAP_CTRL_HEADER 1  /* IrLAP Control Header */
+
+/* May be different when we get VFIR */
+#define LAP_MAX_HEADER (LAP_ADDR_HEADER + LAP_CTRL_HEADER)
+
+/* Each IrDA device gets a random 32 bits IRLAP device address */
+#define LAP_ALEN 4
+
+#define BROADCAST  0xffffffff /* Broadcast device address */
+#define CBROADCAST 0xfe       /* Connection broadcast address */
+#define XID_FORMAT 0x01       /* Discovery XID format */
+
+/* Nobody seems to use this constant. */
+#define LAP_WINDOW_SIZE 8
+/* We keep the LAP queue very small to minimise the amount of buffering.
+ * this improve latency and reduce resource consumption.
+ * This work only because we have synchronous refilling of IrLAP through
+ * the flow control mechanism (via scheduler and IrTTP).
+ * 2 buffers is the minimum we can work with, one that we send while polling
+ * IrTTP, and another to know that we should not send the pf bit.
+ * Jean II */
+#define LAP_HIGH_THRESHOLD     2
+/* Some rare non TTP clients don't implement flow control, and
+ * so don't comply with the above limit (and neither with this one).
+ * For IAP and management, it doesn't matter, because they never transmit much.
+ *.For IrLPT, this should be fixed.
+ * - Jean II */
+#define LAP_MAX_QUEUE 10
+/* Please note that all IrDA management frames (LMP/TTP conn req/disc and
+ * IAS queries) fall in the second category and are sent to LAP even if TTP
+ * is stopped. This means that those frames will wait only a maximum of
+ * two (2) data frames before beeing sent on the "wire", which speed up
+ * new socket setup when the link is saturated.
+ * Same story for two sockets competing for the medium : if one saturates
+ * the LAP, when the other want to transmit it only has to wait for
+ * maximum three (3) packets (2 + one scheduling), which improve performance
+ * of delay sensitive applications.
+ * Jean II */
+
+#define NR_EXPECTED     1
+#define NR_UNEXPECTED   0
+#define NR_INVALID     -1
+
+#define NS_EXPECTED     1
+#define NS_UNEXPECTED   0
+#define NS_INVALID     -1
+
+/*
+ *  Meta information passed within the IrLAP state machine
+ */
+struct irlap_info {
+	__u8 caddr;   /* Connection address */
+	__u8 control; /* Frame type */
+        __u8 cmd;
+
+	__u32 saddr;
+	__u32 daddr;
+	
+	int pf;        /* Poll/final bit set */
+
+	__u8  nr;      /* Sequence number of next frame expected */
+	__u8  ns;      /* Sequence number of frame sent */
+
+	int  S;        /* Number of slots */
+	int  slot;     /* Random chosen slot */
+	int  s;        /* Current slot */
+
+	discovery_t *discovery; /* Discovery information */
+};
+
+/* Main structure of IrLAP */
+struct irlap_cb {
+	irda_queue_t q;     /* Must be first */
+	magic_t magic;
+
+	/* Device we are attached to */
+	struct net_device  *netdev;
+	char		hw_name[2*IFNAMSIZ + 1];
+
+	/* Connection state */
+	volatile IRLAP_STATE state;       /* Current state */
+
+	/* Timers used by IrLAP */
+	struct timer_list query_timer;
+	struct timer_list slot_timer;
+	struct timer_list discovery_timer;
+	struct timer_list final_timer;
+	struct timer_list poll_timer;
+	struct timer_list wd_timer;
+	struct timer_list backoff_timer;
+
+	/* Media busy stuff */
+	struct timer_list media_busy_timer;
+	int media_busy;
+
+	/* Timeouts which will be different with different turn time */
+	int slot_timeout;
+	int poll_timeout;
+	int final_timeout;
+	int wd_timeout;
+
+	struct sk_buff_head txq;  /* Frames to be transmitted */
+	struct sk_buff_head txq_ultra;
+
+ 	__u8    caddr;        /* Connection address */
+	__u32   saddr;        /* Source device address */
+	__u32   daddr;        /* Destination device address */
+
+	int     retry_count;  /* Times tried to establish connection */
+	int     add_wait;     /* True if we are waiting for frame */
+
+	__u8    connect_pending;
+	__u8    disconnect_pending;
+
+	/*  To send a faster RR if tx queue empty */
+#ifdef CONFIG_IRDA_FAST_RR
+	int     fast_RR_timeout;
+	int     fast_RR;      
+#endif /* CONFIG_IRDA_FAST_RR */
+	
+	int N1; /* N1 * F-timer = Negitiated link disconnect warning threshold */
+	int N2; /* N2 * F-timer = Negitiated link disconnect time */
+	int N3; /* Connection retry count */
+
+	int     local_busy;
+	int     remote_busy;
+	int     xmitflag;
+
+	__u8    vs;            /* Next frame to be sent */
+	__u8    vr;            /* Next frame to be received */
+	__u8    va;            /* Last frame acked */
+ 	int     window;        /* Nr of I-frames allowed to send */
+	int     window_size;   /* Current negotiated window size */
+
+#ifdef CONFIG_IRDA_DYNAMIC_WINDOW
+	__u32   line_capacity; /* Number of bytes allowed to send */
+	__u32   bytes_left;    /* Number of bytes still allowed to transmit */
+#endif /* CONFIG_IRDA_DYNAMIC_WINDOW */
+
+	struct sk_buff_head wx_list;
+
+	__u8    ack_required;
+	
+	/* XID parameters */
+ 	__u8    S;           /* Number of slots */
+	__u8    slot;        /* Random chosen slot */
+ 	__u8    s;           /* Current slot */
+	int     frame_sent;  /* Have we sent reply? */
+
+	hashbin_t   *discovery_log;
+ 	discovery_t *discovery_cmd;
+
+	__u32 speed;		/* Link speed */
+
+	struct qos_info  qos_tx;   /* QoS requested by peer */
+	struct qos_info  qos_rx;   /* QoS requested by self */
+	struct qos_info *qos_dev;  /* QoS supported by device */
+
+	notify_t notify; /* Callbacks to IrLMP */
+
+	int    mtt_required;  /* Minimum turnaround time required */
+	int    xbofs_delay;   /* Nr of XBOF's used to MTT */
+	int    bofs_count;    /* Negotiated extra BOFs */
+	int    next_bofs;     /* Negotiated extra BOFs after next frame */
+
+	int    mode;     /* IrLAP mode (primary, secondary or monitor) */
+};
+
+/* 
+ *  Function prototypes 
+ */
+int irlap_init(void);
+void irlap_cleanup(void);
+
+struct irlap_cb *irlap_open(struct net_device *dev, struct qos_info *qos,
+			    const char *hw_name);
+void irlap_close(struct irlap_cb *self);
+
+void irlap_connect_request(struct irlap_cb *self, __u32 daddr, 
+			   struct qos_info *qos, int sniff);
+void irlap_connect_response(struct irlap_cb *self, struct sk_buff *skb);
+void irlap_connect_indication(struct irlap_cb *self, struct sk_buff *skb);
+void irlap_connect_confirm(struct irlap_cb *, struct sk_buff *skb);
+
+void irlap_data_indication(struct irlap_cb *, struct sk_buff *, int unreliable);
+void irlap_data_request(struct irlap_cb *, struct sk_buff *, int unreliable);
+
+#ifdef CONFIG_IRDA_ULTRA
+void irlap_unitdata_request(struct irlap_cb *, struct sk_buff *);
+void irlap_unitdata_indication(struct irlap_cb *, struct sk_buff *);
+#endif /* CONFIG_IRDA_ULTRA */
+
+void irlap_disconnect_request(struct irlap_cb *);
+void irlap_disconnect_indication(struct irlap_cb *, LAP_REASON reason);
+
+void irlap_status_indication(struct irlap_cb *, int quality_of_link);
+
+void irlap_test_request(__u8 *info, int len);
+
+void irlap_discovery_request(struct irlap_cb *, discovery_t *discovery);
+void irlap_discovery_confirm(struct irlap_cb *, hashbin_t *discovery_log);
+void irlap_discovery_indication(struct irlap_cb *, discovery_t *discovery);
+
+void irlap_reset_indication(struct irlap_cb *self);
+void irlap_reset_confirm(void);
+
+void irlap_update_nr_received(struct irlap_cb *, int nr);
+int irlap_validate_nr_received(struct irlap_cb *, int nr);
+int irlap_validate_ns_received(struct irlap_cb *, int ns);
+
+int  irlap_generate_rand_time_slot(int S, int s);
+void irlap_initiate_connection_state(struct irlap_cb *);
+void irlap_flush_all_queues(struct irlap_cb *);
+void irlap_wait_min_turn_around(struct irlap_cb *, struct qos_info *);
+
+void irlap_apply_default_connection_parameters(struct irlap_cb *self);
+void irlap_apply_connection_parameters(struct irlap_cb *self, int now);
+
+#define IRLAP_GET_HEADER_SIZE(self) (LAP_MAX_HEADER)
+#define IRLAP_GET_TX_QUEUE_LEN(self) skb_queue_len(&self->txq)
+
+/* Return TRUE if the node is in primary mode (i.e. master)
+ * - Jean II */
+static inline int irlap_is_primary(struct irlap_cb *self)
+{
+	int ret;
+	switch(self->state) {
+	case LAP_XMIT_P:
+	case LAP_NRM_P:
+		ret = 1;
+		break;
+	case LAP_XMIT_S:
+	case LAP_NRM_S:
+		ret = 0;
+		break;
+	default:
+		ret = -1;
+	}
+	return ret;
+}
+
+/* Clear a pending IrLAP disconnect. - Jean II */
+static inline void irlap_clear_disconnect(struct irlap_cb *self)
+{
+	self->disconnect_pending = FALSE;
+}
+
+/*
+ * Function irlap_next_state (self, state)
+ *
+ *    Switches state and provides debug information
+ *
+ */
+static inline void irlap_next_state(struct irlap_cb *self, IRLAP_STATE state)
+{
+	/*
+	if (!self || self->magic != LAP_MAGIC)
+		return;
+
+		pr_debug("next LAP state = %s\n", irlap_state[state]);
+	*/
+	self->state = state;
+}
+
+#endif
diff --git a/drivers/staging/irda/include/net/irda/irlap_event.h b/drivers/staging/irda/include/net/irda/irlap_event.h
new file mode 100644
index 000000000000..e4325fee1267
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/irlap_event.h
@@ -0,0 +1,129 @@
+/*********************************************************************
+ *                
+ *                
+ * Filename:      irlap_event.h
+ * Version:       0.1
+ * Description:   
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sat Aug 16 00:59:29 1997
+ * Modified at:   Tue Dec 21 11:20:30 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1998-1999 Dag Brattli <dagb@cs.uit.no>, 
+ *     All Rights Reserved.
+ *     Copyright (c) 2000-2002 Jean Tourrilhes <jt@hpl.hp.com>
+ *     
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ * 
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ * 
+ *     You should have received a copy of the GNU General Public License 
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *     
+ ********************************************************************/
+
+#ifndef IRLAP_EVENT_H
+#define IRLAP_EVENT_H
+
+#include <net/irda/irda.h>
+
+/* A few forward declarations (to make compiler happy) */
+struct irlap_cb;
+struct irlap_info;
+
+/* IrLAP States */
+typedef enum {
+	LAP_NDM,         /* Normal disconnected mode */
+	LAP_QUERY,
+	LAP_REPLY,
+	LAP_CONN,        /* Connect indication */
+	LAP_SETUP,       /* Setting up connection */
+	LAP_OFFLINE,     /* A really boring state */
+	LAP_XMIT_P,
+	LAP_PCLOSE,
+	LAP_NRM_P,       /* Normal response mode as primary */
+	LAP_RESET_WAIT,
+	LAP_RESET,
+	LAP_NRM_S,       /* Normal response mode as secondary */
+	LAP_XMIT_S,
+	LAP_SCLOSE,
+	LAP_RESET_CHECK,
+} IRLAP_STATE;
+
+/* IrLAP Events */
+typedef enum {
+	/* Services events */
+	DISCOVERY_REQUEST,
+	CONNECT_REQUEST,
+	CONNECT_RESPONSE,
+	DISCONNECT_REQUEST,
+	DATA_REQUEST,
+	RESET_REQUEST,
+	RESET_RESPONSE,
+
+	/* Send events */
+	SEND_I_CMD,
+	SEND_UI_FRAME,
+
+	/* Receive events */
+	RECV_DISCOVERY_XID_CMD,
+	RECV_DISCOVERY_XID_RSP,
+	RECV_SNRM_CMD,
+	RECV_TEST_CMD,
+	RECV_TEST_RSP,
+	RECV_UA_RSP,
+	RECV_DM_RSP,
+	RECV_RD_RSP,
+	RECV_I_CMD,
+	RECV_I_RSP,
+	RECV_UI_FRAME,
+	RECV_FRMR_RSP,
+	RECV_RR_CMD,
+	RECV_RR_RSP,
+	RECV_RNR_CMD,
+	RECV_RNR_RSP,
+	RECV_REJ_CMD,
+	RECV_REJ_RSP,
+	RECV_SREJ_CMD,
+	RECV_SREJ_RSP,
+	RECV_DISC_CMD,
+
+	/* Timer events */
+	SLOT_TIMER_EXPIRED,
+	QUERY_TIMER_EXPIRED,
+	FINAL_TIMER_EXPIRED,
+	POLL_TIMER_EXPIRED,
+	DISCOVERY_TIMER_EXPIRED,
+	WD_TIMER_EXPIRED,
+	BACKOFF_TIMER_EXPIRED,
+	MEDIA_BUSY_TIMER_EXPIRED,
+} IRLAP_EVENT;
+
+/*
+ * Disconnect reason code
+ */
+typedef enum { /* FIXME check the two first reason codes */
+	LAP_DISC_INDICATION=1, /* Received a disconnect request from peer */
+	LAP_NO_RESPONSE,       /* To many retransmits without response */
+	LAP_RESET_INDICATION,  /* To many retransmits, or invalid nr/ns */
+	LAP_FOUND_NONE,        /* No devices were discovered */
+	LAP_MEDIA_BUSY,
+	LAP_PRIMARY_CONFLICT,
+} LAP_REASON;
+
+extern const char *const irlap_state[];
+
+void irlap_do_event(struct irlap_cb *self, IRLAP_EVENT event, 
+		    struct sk_buff *skb, struct irlap_info *info);
+void irlap_print_event(IRLAP_EVENT event);
+
+int irlap_qos_negotiate(struct irlap_cb *self, struct sk_buff *skb);
+
+#endif
diff --git a/drivers/staging/irda/include/net/irda/irlap_frame.h b/drivers/staging/irda/include/net/irda/irlap_frame.h
new file mode 100644
index 000000000000..cbc12a926e5f
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/irlap_frame.h
@@ -0,0 +1,167 @@
+/*********************************************************************
+ *                
+ * Filename:      irlap_frame.h
+ * Version:       0.9
+ * Description:   IrLAP frame declarations
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Tue Aug 19 10:27:26 1997
+ * Modified at:   Sat Dec 25 21:07:26 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1997-1999 Dag Brattli <dagb@cs.uit.no>,
+ *     All Rights Reserved.
+ *     Copyright (c) 2000-2002 Jean Tourrilhes <jt@hpl.hp.com>
+ *     
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ * 
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ * 
+ *     You should have received a copy of the GNU General Public License 
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *     
+ ********************************************************************/
+
+#ifndef IRLAP_FRAME_H
+#define IRLAP_FRAME_H
+
+#include <linux/skbuff.h>
+
+#include <net/irda/irda.h>
+
+/* A few forward declarations (to make compiler happy) */
+struct irlap_cb;
+struct discovery_t;
+
+/* Frame types and templates */
+#define INVALID   0xff
+
+/* Unnumbered (U) commands */
+#define SNRM_CMD  0x83 /* Set Normal Response Mode */
+#define DISC_CMD  0x43 /* Disconnect */
+#define XID_CMD   0x2f /* Exchange Station Identification */
+#define TEST_CMD  0xe3 /* Test */
+
+/* Unnumbered responses */
+#define RNRM_RSP  0x83 /* Request Normal Response Mode */
+#define UA_RSP    0x63 /* Unnumbered Acknowledgement */
+#define FRMR_RSP  0x87 /* Frame Reject */
+#define DM_RSP    0x0f /* Disconnect Mode */
+#define RD_RSP    0x43 /* Request Disconnection */
+#define XID_RSP   0xaf /* Exchange Station Identification */
+#define TEST_RSP  0xe3 /* Test frame */
+
+/* Supervisory (S) */
+#define RR        0x01 /* Receive Ready */
+#define REJ       0x09 /* Reject */
+#define RNR       0x05 /* Receive Not Ready */
+#define SREJ      0x0d /* Selective Reject */
+
+/* Information (I) */
+#define I_FRAME   0x00 /* Information Format */
+#define UI_FRAME  0x03 /* Unnumbered Information */
+
+#define CMD_FRAME 0x01
+#define RSP_FRAME 0x00
+
+#define PF_BIT    0x10 /* Poll/final bit */
+
+/* Some IrLAP field lengths */
+/*
+ * Only baud rate triplet is 4 bytes (PV can be 2 bytes).
+ * All others params (7) are 3 bytes, so that's 7*3 + 1*4 bytes.
+ */
+#define IRLAP_NEGOCIATION_PARAMS_LEN 25
+#define IRLAP_DISCOVERY_INFO_LEN     32
+
+struct disc_frame {
+	__u8 caddr;          /* Connection address */
+	__u8 control;
+} __packed;
+
+struct xid_frame {
+	__u8  caddr; /* Connection address */
+	__u8  control;
+	__u8  ident; /* Should always be XID_FORMAT */ 
+	__le32 saddr; /* Source device address */
+	__le32 daddr; /* Destination device address */
+	__u8  flags; /* Discovery flags */
+	__u8  slotnr;
+	__u8  version;
+} __packed;
+
+struct test_frame {
+	__u8 caddr;          /* Connection address */
+	__u8 control;
+	__le32 saddr;         /* Source device address */
+	__le32 daddr;         /* Destination device address */
+} __packed;
+
+struct ua_frame {
+	__u8 caddr;
+	__u8 control;
+	__le32 saddr; /* Source device address */
+	__le32 daddr; /* Dest device address */
+} __packed;
+
+struct dm_frame {
+	__u8 caddr;          /* Connection address */
+	__u8 control;
+} __packed;
+
+struct rd_frame {
+	__u8 caddr;          /* Connection address */
+	__u8 control;
+} __packed;
+
+struct rr_frame {
+	__u8 caddr;          /* Connection address */
+	__u8 control;
+} __packed;
+
+struct i_frame {
+	__u8 caddr;
+	__u8 control;
+} __packed;
+
+struct snrm_frame {
+	__u8  caddr;
+	__u8  control;
+	__le32 saddr;
+	__le32 daddr;
+	__u8  ncaddr;
+} __packed;
+
+void irlap_queue_xmit(struct irlap_cb *self, struct sk_buff *skb);
+void irlap_send_discovery_xid_frame(struct irlap_cb *, int S, __u8 s, 
+				    __u8 command,
+				    struct discovery_t *discovery);
+void irlap_send_snrm_frame(struct irlap_cb *, struct qos_info *);
+void irlap_send_test_frame(struct irlap_cb *self, __u8 caddr, __u32 daddr, 
+			   struct sk_buff *cmd);
+void irlap_send_ua_response_frame(struct irlap_cb *, struct qos_info *);
+void irlap_send_dm_frame(struct irlap_cb *self);
+void irlap_send_rd_frame(struct irlap_cb *self);
+void irlap_send_disc_frame(struct irlap_cb *self);
+void irlap_send_rr_frame(struct irlap_cb *self, int command);
+
+void irlap_send_data_primary(struct irlap_cb *, struct sk_buff *);
+void irlap_send_data_primary_poll(struct irlap_cb *, struct sk_buff *);
+void irlap_send_data_secondary(struct irlap_cb *, struct sk_buff *);
+void irlap_send_data_secondary_final(struct irlap_cb *, struct sk_buff *);
+void irlap_resend_rejected_frames(struct irlap_cb *, int command);
+void irlap_resend_rejected_frame(struct irlap_cb *self, int command);
+
+void irlap_send_ui_frame(struct irlap_cb *self, struct sk_buff *skb,
+			 __u8 caddr, int command);
+
+int irlap_insert_qos_negotiation_params(struct irlap_cb *self,
+					struct sk_buff *skb);
+
+#endif
diff --git a/drivers/staging/irda/include/net/irda/irlmp.h b/drivers/staging/irda/include/net/irda/irlmp.h
new file mode 100644
index 000000000000..f132924cc9da
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/irlmp.h
@@ -0,0 +1,295 @@
+/*********************************************************************
+ *                
+ * Filename:      irlmp.h
+ * Version:       0.9
+ * Description:   IrDA Link Management Protocol (LMP) layer
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sun Aug 17 20:54:32 1997
+ * Modified at:   Fri Dec 10 13:23:01 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1998-1999 Dag Brattli <dagb@cs.uit.no>, 
+ *     All Rights Reserved.
+ *     Copyright (c) 2000-2002 Jean Tourrilhes <jt@hpl.hp.com>
+ *     
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is 
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#ifndef IRLMP_H
+#define IRLMP_H
+
+#include <asm/param.h>  /* for HZ */
+
+#include <linux/types.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/qos.h>
+#include <net/irda/irlap.h>		/* LAP_MAX_HEADER, ... */
+#include <net/irda/irlmp_event.h>
+#include <net/irda/irqueue.h>
+#include <net/irda/discovery.h>
+
+/* LSAP-SEL's */
+#define LSAP_MASK     0x7f
+#define LSAP_IAS      0x00
+#define LSAP_ANY      0xff
+#define LSAP_MAX      0x6f /* 0x70-0x7f are reserved */
+#define LSAP_CONNLESS 0x70 /* Connectionless LSAP, mostly used for Ultra */
+
+#define DEV_ADDR_ANY  0xffffffff
+
+#define LMP_HEADER          2    /* Dest LSAP + Source LSAP */
+#define LMP_CONTROL_HEADER  4    /* LMP_HEADER + opcode + parameter */
+#define LMP_PID_HEADER      1    /* Used by Ultra */
+#define LMP_MAX_HEADER      (LMP_CONTROL_HEADER+LAP_MAX_HEADER)
+
+#define LM_MAX_CONNECTIONS  10
+
+#define LM_IDLE_TIMEOUT     2*HZ /* 2 seconds for now */
+
+typedef enum {
+	S_PNP = 0,
+	S_PDA,
+	S_COMPUTER,
+	S_PRINTER,
+	S_MODEM,
+	S_FAX,
+	S_LAN,
+	S_TELEPHONY,
+	S_COMM,
+	S_OBEX,
+	S_ANY,
+	S_END,
+} SERVICE;
+
+/* For selective discovery */
+typedef void (*DISCOVERY_CALLBACK1) (discinfo_t *, DISCOVERY_MODE, void *);
+/* For expiry (the same) */
+typedef void (*DISCOVERY_CALLBACK2) (discinfo_t *, DISCOVERY_MODE, void *);
+
+typedef struct {
+	irda_queue_t queue; /* Must be first */
+
+	__u16_host_order hints; /* Hint bits */
+} irlmp_service_t;
+
+typedef struct {
+	irda_queue_t queue; /* Must be first */
+
+	__u16_host_order hint_mask;
+
+	DISCOVERY_CALLBACK1 disco_callback;	/* Selective discovery */
+	DISCOVERY_CALLBACK2 expir_callback;	/* Selective expiration */
+	void *priv;                /* Used to identify client */
+} irlmp_client_t;
+
+/*
+ *  Information about each logical LSAP connection
+ */
+struct lsap_cb {
+	irda_queue_t queue;      /* Must be first */
+	magic_t magic;
+
+	unsigned long connected;	/* set_bit used on this */
+	int  persistent;
+
+	__u8 slsap_sel;   /* Source (this) LSAP address */
+	__u8 dlsap_sel;   /* Destination LSAP address (if connected) */
+#ifdef CONFIG_IRDA_ULTRA
+	__u8 pid;         /* Used by connectionless LSAP */
+#endif /* CONFIG_IRDA_ULTRA */
+	struct sk_buff *conn_skb; /* Store skb here while connecting */
+
+	struct timer_list watchdog_timer;
+
+	LSAP_STATE      lsap_state;  /* Connection state */
+	notify_t        notify;      /* Indication/Confirm entry points */
+	struct qos_info qos;         /* QoS for this connection */
+
+	struct lap_cb *lap; /* Pointer to LAP connection structure */
+};
+
+/*
+ *  Used for caching the last slsap->dlsap->handle mapping
+ *
+ * We don't need to keep/match the remote address in the cache because
+ * we are associated with a specific LAP (which implies it).
+ * Jean II
+ */
+typedef struct {
+	int valid;
+
+	__u8 slsap_sel;
+	__u8 dlsap_sel;
+	struct lsap_cb *lsap;
+} CACHE_ENTRY;
+
+/*
+ *  Information about each registered IrLAP layer
+ */
+struct lap_cb {
+	irda_queue_t queue; /* Must be first */
+	magic_t magic;
+
+	int reason;    /* LAP disconnect reason */
+
+	IRLMP_STATE lap_state;
+
+	struct irlap_cb *irlap;   /* Instance of IrLAP layer */
+	hashbin_t *lsaps;         /* LSAP associated with this link */
+	struct lsap_cb *flow_next;	/* Next lsap to be polled for Tx */
+
+	__u8  caddr;  /* Connection address */
+ 	__u32 saddr;  /* Source device address */
+ 	__u32 daddr;  /* Destination device address */
+	
+	struct qos_info *qos;  /* LAP QoS for this session */
+	struct timer_list idle_timer;
+	
+#ifdef CONFIG_IRDA_CACHE_LAST_LSAP
+	/* The lsap cache was moved from struct irlmp_cb to here because
+	 * it must be associated with the specific LAP. Also, this
+	 * improves performance. - Jean II */
+	CACHE_ENTRY cache;  /* Caching last slsap->dlsap->handle mapping */
+#endif
+};
+
+/*
+ *  Main structure for IrLMP
+ */
+struct irlmp_cb {
+	magic_t magic;
+
+	__u8 conflict_flag;
+	
+	discovery_t discovery_cmd; /* Discovery command to use by IrLAP */
+	discovery_t discovery_rsp; /* Discovery response to use by IrLAP */
+
+	/* Last lsap picked automatically by irlmp_find_free_slsap() */
+	int	last_lsap_sel;
+
+	struct timer_list discovery_timer;
+
+ 	hashbin_t *links;         /* IrLAP connection table */
+	hashbin_t *unconnected_lsaps;
+ 	hashbin_t *clients;
+	hashbin_t *services;
+
+	hashbin_t *cachelog;	/* Current discovery log */
+
+	int running;
+
+	__u16_host_order hints; /* Hint bits */
+};
+
+/* Prototype declarations */
+int  irlmp_init(void);
+void irlmp_cleanup(void);
+struct lsap_cb *irlmp_open_lsap(__u8 slsap, notify_t *notify, __u8 pid);
+void irlmp_close_lsap( struct lsap_cb *self);
+
+__u16 irlmp_service_to_hint(int service);
+void *irlmp_register_service(__u16 hints);
+int irlmp_unregister_service(void *handle);
+void *irlmp_register_client(__u16 hint_mask, DISCOVERY_CALLBACK1 disco_clb,
+			    DISCOVERY_CALLBACK2 expir_clb, void *priv);
+int irlmp_unregister_client(void *handle);
+int irlmp_update_client(void *handle, __u16 hint_mask, 
+			DISCOVERY_CALLBACK1 disco_clb,
+			DISCOVERY_CALLBACK2 expir_clb, void *priv);
+
+void irlmp_register_link(struct irlap_cb *, __u32 saddr, notify_t *);
+void irlmp_unregister_link(__u32 saddr);
+
+int  irlmp_connect_request(struct lsap_cb *, __u8 dlsap_sel, 
+			   __u32 saddr, __u32 daddr,
+			   struct qos_info *, struct sk_buff *);
+void irlmp_connect_indication(struct lsap_cb *self, struct sk_buff *skb);
+int  irlmp_connect_response(struct lsap_cb *, struct sk_buff *);
+void irlmp_connect_confirm(struct lsap_cb *, struct sk_buff *);
+struct lsap_cb *irlmp_dup(struct lsap_cb *self, void *instance);
+
+void irlmp_disconnect_indication(struct lsap_cb *self, LM_REASON reason, 
+				 struct sk_buff *userdata);
+int  irlmp_disconnect_request(struct lsap_cb *, struct sk_buff *userdata);
+
+void irlmp_discovery_confirm(hashbin_t *discovery_log, DISCOVERY_MODE mode);
+void irlmp_discovery_request(int nslots);
+discinfo_t *irlmp_get_discoveries(int *pn, __u16 mask, int nslots);
+void irlmp_do_expiry(void);
+void irlmp_do_discovery(int nslots);
+discovery_t *irlmp_get_discovery_response(void);
+void irlmp_discovery_expiry(discinfo_t *expiry, int number);
+
+int  irlmp_data_request(struct lsap_cb *, struct sk_buff *);
+void irlmp_data_indication(struct lsap_cb *, struct sk_buff *);
+
+int  irlmp_udata_request(struct lsap_cb *, struct sk_buff *);
+void irlmp_udata_indication(struct lsap_cb *, struct sk_buff *);
+
+#ifdef CONFIG_IRDA_ULTRA
+int  irlmp_connless_data_request(struct lsap_cb *, struct sk_buff *, __u8);
+void irlmp_connless_data_indication(struct lsap_cb *, struct sk_buff *);
+#endif /* CONFIG_IRDA_ULTRA */
+
+void irlmp_status_indication(struct lap_cb *, LINK_STATUS link, LOCK_STATUS lock);
+void irlmp_flow_indication(struct lap_cb *self, LOCAL_FLOW flow);
+
+LM_REASON irlmp_convert_lap_reason(LAP_REASON);
+
+static inline __u32 irlmp_get_saddr(const struct lsap_cb *self)
+{
+	return (self && self->lap) ? self->lap->saddr : 0;
+}
+
+static inline __u32 irlmp_get_daddr(const struct lsap_cb *self)
+{
+	return (self && self->lap) ? self->lap->daddr : 0;
+}
+
+const char *irlmp_reason_str(LM_REASON reason);
+
+extern int sysctl_discovery_timeout;
+extern int sysctl_discovery_slots;
+extern int sysctl_discovery;
+extern int sysctl_lap_keepalive_time;	/* in ms, default is LM_IDLE_TIMEOUT */
+extern struct irlmp_cb *irlmp;
+
+/* Check if LAP queue is full.
+ * Used by IrTTP for low control, see comments in irlap.h - Jean II */
+static inline int irlmp_lap_tx_queue_full(struct lsap_cb *self)
+{
+	if (self == NULL)
+		return 0;
+	if (self->lap == NULL)
+		return 0;
+	if (self->lap->irlap == NULL)
+		return 0;
+
+	return IRLAP_GET_TX_QUEUE_LEN(self->lap->irlap) >= LAP_HIGH_THRESHOLD;
+}
+
+/* After doing a irlmp_dup(), this get one of the two socket back into
+ * a state where it's waiting incoming connections.
+ * Note : this can be used *only* if the socket is not yet connected
+ * (i.e. NO irlmp_connect_response() done on this socket).
+ * - Jean II */
+static inline void irlmp_listen(struct lsap_cb *self)
+{
+	self->dlsap_sel = LSAP_ANY;
+	self->lap = NULL;
+	self->lsap_state = LSAP_DISCONNECTED;
+	/* Started when we received the LM_CONNECT_INDICATION */
+	del_timer(&self->watchdog_timer);
+}
+
+#endif
diff --git a/drivers/staging/irda/include/net/irda/irlmp_event.h b/drivers/staging/irda/include/net/irda/irlmp_event.h
new file mode 100644
index 000000000000..9e4ec17a7449
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/irlmp_event.h
@@ -0,0 +1,98 @@
+/*********************************************************************
+ *                
+ * Filename:      irlmp_event.h
+ * Version:       0.1
+ * Description:   IrDA-LMP event handling
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Mon Aug  4 20:40:53 1997
+ * Modified at:   Thu Jul  8 12:18:54 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1997, 1999 Dag Brattli <dagb@cs.uit.no>, 
+ *     All Rights Reserved.
+ *     Copyright (c) 2000-2002 Jean Tourrilhes <jt@hpl.hp.com>
+ *     
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is 
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#ifndef IRLMP_EVENT_H
+#define IRLMP_EVENT_H
+
+/* A few forward declarations (to make compiler happy) */
+struct irlmp_cb;
+struct lsap_cb;
+struct lap_cb;
+struct discovery_t;
+
+/* LAP states */
+typedef enum {
+	/* IrLAP connection control states */
+	LAP_STANDBY,             /* No LAP connection */
+	LAP_U_CONNECT,           /* Starting LAP connection */
+	LAP_ACTIVE,              /* LAP connection is active */
+} IRLMP_STATE;
+
+/* LSAP connection control states */
+typedef enum {
+	LSAP_DISCONNECTED,        /* No LSAP connection */
+	LSAP_CONNECT,             /* Connect indication from peer */
+	LSAP_CONNECT_PEND,        /* Connect request from service user */
+	LSAP_DATA_TRANSFER_READY, /* LSAP connection established */          
+	LSAP_SETUP,               /* Trying to set up LSAP connection */
+	LSAP_SETUP_PEND,          /* Request to start LAP connection */
+} LSAP_STATE;
+
+typedef enum {
+	/* LSAP events */
+ 	LM_CONNECT_REQUEST,
+ 	LM_CONNECT_CONFIRM,
+	LM_CONNECT_RESPONSE,
+ 	LM_CONNECT_INDICATION, 	
+	
+	LM_DISCONNECT_INDICATION,
+	LM_DISCONNECT_REQUEST,
+
+ 	LM_DATA_REQUEST,
+	LM_UDATA_REQUEST,
+ 	LM_DATA_INDICATION,
+	LM_UDATA_INDICATION,
+
+	LM_WATCHDOG_TIMEOUT,
+
+	/* IrLAP events */
+	LM_LAP_CONNECT_REQUEST,
+ 	LM_LAP_CONNECT_INDICATION, 
+ 	LM_LAP_CONNECT_CONFIRM,
+ 	LM_LAP_DISCONNECT_INDICATION, 
+	LM_LAP_DISCONNECT_REQUEST,
+	LM_LAP_DISCOVERY_REQUEST,
+ 	LM_LAP_DISCOVERY_CONFIRM,
+	LM_LAP_IDLE_TIMEOUT,
+} IRLMP_EVENT;
+
+extern const char *const irlmp_state[];
+extern const char *const irlsap_state[];
+
+void irlmp_watchdog_timer_expired(void *data);
+void irlmp_discovery_timer_expired(void *data);
+void irlmp_idle_timer_expired(void *data);
+
+void irlmp_do_lap_event(struct lap_cb *self, IRLMP_EVENT event, 
+			struct sk_buff *skb);
+int irlmp_do_lsap_event(struct lsap_cb *self, IRLMP_EVENT event, 
+			struct sk_buff *skb);
+
+#endif /* IRLMP_EVENT_H */
+
+
+
+
diff --git a/drivers/staging/irda/include/net/irda/irlmp_frame.h b/drivers/staging/irda/include/net/irda/irlmp_frame.h
new file mode 100644
index 000000000000..1906eb71422e
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/irlmp_frame.h
@@ -0,0 +1,62 @@
+/*********************************************************************
+ *                
+ * Filename:      irlmp_frame.h
+ * Version:       0.9
+ * Description:   
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Tue Aug 19 02:09:59 1997
+ * Modified at:   Fri Dec 10 13:21:53 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1997, 1999 Dag Brattli <dagb@cs.uit.no>, 
+ *     All Rights Reserved.
+ *     
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is 
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#ifndef IRMLP_FRAME_H
+#define IRMLP_FRAME_H
+
+#include <linux/skbuff.h>
+
+#include <net/irda/discovery.h>
+
+/* IrLMP frame opcodes */
+#define CONNECT_CMD    0x01
+#define CONNECT_CNF    0x81
+#define DISCONNECT     0x02
+#define ACCESSMODE_CMD 0x03
+#define ACCESSMODE_CNF 0x83
+
+#define CONTROL_BIT    0x80
+
+void irlmp_send_data_pdu(struct lap_cb *self, __u8 dlsap, __u8 slsap,
+				int expedited, struct sk_buff *skb);
+void irlmp_send_lcf_pdu(struct lap_cb *self, __u8 dlsap, __u8 slsap, 
+			__u8 opcode, struct sk_buff *skb);
+void irlmp_link_data_indication(struct lap_cb *, struct sk_buff *, 
+				int unreliable);
+#ifdef CONFIG_IRDA_ULTRA
+void irlmp_link_unitdata_indication(struct lap_cb *, struct sk_buff *);
+#endif /* CONFIG_IRDA_ULTRA */
+
+void irlmp_link_connect_indication(struct lap_cb *, __u32 saddr, __u32 daddr,
+				   struct qos_info *qos, struct sk_buff *skb);
+void irlmp_link_connect_request(__u32 daddr);
+void irlmp_link_connect_confirm(struct lap_cb *self, struct qos_info *qos, 
+				struct sk_buff *skb);
+void irlmp_link_disconnect_indication(struct lap_cb *, struct irlap_cb *, 
+				      LAP_REASON reason, struct sk_buff *); 
+void irlmp_link_discovery_confirm(struct lap_cb *self, hashbin_t *log);
+void irlmp_link_discovery_indication(struct lap_cb *, discovery_t *discovery);
+
+#endif
diff --git a/drivers/staging/irda/include/net/irda/irmod.h b/drivers/staging/irda/include/net/irda/irmod.h
new file mode 100644
index 000000000000..86f0dbb8ee5d
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/irmod.h
@@ -0,0 +1,109 @@
+/*********************************************************************
+ *                
+ * Filename:      irmod.h
+ * Version:       0.3
+ * Description:   IrDA module and utilities functions
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Mon Dec 15 13:58:52 1997
+ * Modified at:   Fri Jan 28 13:15:24 2000
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1998-2000 Dag Brattli, All Rights Reserved.
+ *     Copyright (c) 2000-2002 Jean Tourrilhes <jt@hpl.hp.com>
+ *      
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ *  
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is 
+ *     provided "AS-IS" and at no charg.
+ *     
+ ********************************************************************/
+
+#ifndef IRMOD_H
+#define IRMOD_H
+
+/* Misc status information */
+typedef enum {
+	STATUS_OK,
+	STATUS_ABORTED,
+	STATUS_NO_ACTIVITY,
+	STATUS_NOISY,
+	STATUS_REMOTE,
+} LINK_STATUS;
+
+typedef enum {
+	LOCK_NO_CHANGE,
+	LOCK_LOCKED,
+	LOCK_UNLOCKED,
+} LOCK_STATUS;
+
+typedef enum { FLOW_STOP, FLOW_START } LOCAL_FLOW;
+
+/*  
+ *  IrLMP disconnect reasons. The order is very important, since they 
+ *  correspond to disconnect reasons sent in IrLMP disconnect frames, so
+ *  please do not touch :-)
+ */
+typedef enum {
+	LM_USER_REQUEST = 1,  /* User request */
+	LM_LAP_DISCONNECT,    /* Unexpected IrLAP disconnect */
+	LM_CONNECT_FAILURE,   /* Failed to establish IrLAP connection */
+	LM_LAP_RESET,         /* IrLAP reset */
+	LM_INIT_DISCONNECT,   /* Link Management initiated disconnect */
+	LM_LSAP_NOTCONN,      /* Data delivered on unconnected LSAP */
+	LM_NON_RESP_CLIENT,   /* Non responsive LM-MUX client */
+	LM_NO_AVAIL_CLIENT,   /* No available LM-MUX client */
+	LM_CONN_HALF_OPEN,    /* Connection is half open */
+	LM_BAD_SOURCE_ADDR,   /* Illegal source address (i.e 0x00) */
+} LM_REASON;
+#define LM_UNKNOWN 0xff       /* Unspecified disconnect reason */
+
+/* A few forward declarations (to make compiler happy) */
+struct qos_info;		/* in <net/irda/qos.h> */
+
+/*
+ *  Notify structure used between transport and link management layers
+ */
+typedef struct {
+	int (*data_indication)(void *priv, void *sap, struct sk_buff *skb);
+	int (*udata_indication)(void *priv, void *sap, struct sk_buff *skb);
+	void (*connect_confirm)(void *instance, void *sap, 
+				struct qos_info *qos, __u32 max_sdu_size,
+				__u8 max_header_size, struct sk_buff *skb);
+	void (*connect_indication)(void *instance, void *sap, 
+				   struct qos_info *qos, __u32 max_sdu_size, 
+				   __u8 max_header_size, struct sk_buff *skb);
+	void (*disconnect_indication)(void *instance, void *sap, 
+				      LM_REASON reason, struct sk_buff *);
+	void (*flow_indication)(void *instance, void *sap, LOCAL_FLOW flow);
+	void (*status_indication)(void *instance,
+				  LINK_STATUS link, LOCK_STATUS lock);
+	void *instance; /* Layer instance pointer */
+	char name[16];  /* Name of layer */
+} notify_t;
+
+#define NOTIFY_MAX_NAME 16
+
+/* Zero the notify structure */
+void irda_notify_init(notify_t *notify);
+
+/* Locking wrapper - Note the inverted logic on irda_lock().
+ * Those function basically return false if the lock is already in the
+ * position you want to set it. - Jean II */
+#define irda_lock(lock)		(! test_and_set_bit(0, (void *) (lock)))
+#define irda_unlock(lock)	(test_and_clear_bit(0, (void *) (lock)))
+
+#endif /* IRMOD_H */
+
+
+
+
+
+
+
+
+
diff --git a/drivers/staging/irda/include/net/irda/irqueue.h b/drivers/staging/irda/include/net/irda/irqueue.h
new file mode 100644
index 000000000000..37f512bd6733
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/irqueue.h
@@ -0,0 +1,96 @@
+/*********************************************************************
+ *                
+ * Filename:      irqueue.h
+ * Version:       0.3
+ * Description:   General queue implementation
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Tue Jun  9 13:26:50 1998
+ * Modified at:   Thu Oct  7 13:25:16 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (C) 1998-1999, Aage Kvalnes <aage@cs.uit.no>
+ *     Copyright (c) 1998, Dag Brattli
+ *     All Rights Reserved.
+ *      
+ *     This code is taken from the Vortex Operating System written by Aage
+ *     Kvalnes and has been ported to Linux and Linux/IR by Dag Brattli
+ *
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ *  
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is 
+ *     provided "AS-IS" and at no charge.
+ *     
+ ********************************************************************/
+
+#include <linux/types.h>
+#include <linux/spinlock.h>
+
+#ifndef IRDA_QUEUE_H
+#define IRDA_QUEUE_H
+
+#define NAME_SIZE      32
+
+/*
+ * Hash types (some flags can be xored)
+ * See comments in irqueue.c for which one to use...
+ */
+#define HB_NOLOCK	0	/* No concurent access prevention */
+#define HB_LOCK		1	/* Prevent concurent write with global lock */
+
+/*
+ * Hash defines
+ */
+#define HASHBIN_SIZE   8
+#define HASHBIN_MASK   0x7
+
+#ifndef IRDA_ALIGN 
+#define IRDA_ALIGN __attribute__((aligned))
+#endif
+
+#define Q_NULL { NULL, NULL, "", 0 }
+
+typedef void (*FREE_FUNC)(void *arg);
+
+struct irda_queue {
+	struct irda_queue *q_next;
+	struct irda_queue *q_prev;
+
+	char   q_name[NAME_SIZE];
+	long   q_hash;			/* Must be able to cast a (void *) */
+};
+typedef struct irda_queue irda_queue_t;
+
+typedef struct hashbin_t {
+	__u32      magic;
+	int        hb_type;
+	int        hb_size;
+	spinlock_t hb_spinlock;		/* HB_LOCK - Can be used by the user */
+
+	irda_queue_t* hb_queue[HASHBIN_SIZE] IRDA_ALIGN;
+
+	irda_queue_t* hb_current;
+} hashbin_t;
+
+hashbin_t *hashbin_new(int type);
+int      hashbin_delete(hashbin_t* hashbin, FREE_FUNC func);
+int      hashbin_clear(hashbin_t* hashbin, FREE_FUNC free_func);
+void     hashbin_insert(hashbin_t* hashbin, irda_queue_t* entry, long hashv, 
+			const char* name);
+void*    hashbin_remove(hashbin_t* hashbin, long hashv, const char* name);
+void*    hashbin_remove_first(hashbin_t *hashbin);
+void*	 hashbin_remove_this( hashbin_t* hashbin, irda_queue_t* entry);
+void*    hashbin_find(hashbin_t* hashbin, long hashv, const char* name);
+void*    hashbin_lock_find(hashbin_t* hashbin, long hashv, const char* name);
+void*    hashbin_find_next(hashbin_t* hashbin, long hashv, const char* name,
+			   void ** pnext);
+irda_queue_t *hashbin_get_first(hashbin_t *hashbin);
+irda_queue_t *hashbin_get_next(hashbin_t *hashbin);
+
+#define HASHBIN_GET_SIZE(hashbin) hashbin->hb_size
+
+#endif
diff --git a/drivers/staging/irda/include/net/irda/irttp.h b/drivers/staging/irda/include/net/irda/irttp.h
new file mode 100644
index 000000000000..98682d4bae8f
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/irttp.h
@@ -0,0 +1,210 @@
+/*********************************************************************
+ *                
+ * Filename:      irttp.h
+ * Version:       1.0
+ * Description:   Tiny Transport Protocol (TTP) definitions
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sun Aug 31 20:14:31 1997
+ * Modified at:   Sun Dec 12 13:09:07 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1998-1999 Dag Brattli <dagb@cs.uit.no>, 
+ *     All Rights Reserved.
+ *     Copyright (c) 2000-2002 Jean Tourrilhes <jt@hpl.hp.com>
+ *     
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is 
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#ifndef IRTTP_H
+#define IRTTP_H
+
+#include <linux/types.h>
+#include <linux/skbuff.h>
+#include <linux/spinlock.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/irlmp.h>		/* struct lsap_cb */
+#include <net/irda/qos.h>		/* struct qos_info */
+#include <net/irda/irqueue.h>
+
+#define TTP_MAX_CONNECTIONS    LM_MAX_CONNECTIONS
+#define TTP_HEADER             1
+#define TTP_MAX_HEADER         (TTP_HEADER + LMP_MAX_HEADER)
+#define TTP_SAR_HEADER         5
+#define TTP_PARAMETERS         0x80
+#define TTP_MORE               0x80
+
+/* Transmission queue sizes */
+/* Worst case scenario, two window of data - Jean II */
+#define TTP_TX_MAX_QUEUE	14
+/* We need to keep at least 5 frames to make sure that we can refill
+ * appropriately the LAP layer. LAP keeps only two buffers, and we need
+ * to have 7 to make a full window - Jean II */
+#define TTP_TX_LOW_THRESHOLD	5
+/* Most clients are synchronous with respect to flow control, so we can
+ * keep a low number of Tx buffers in TTP - Jean II */
+#define TTP_TX_HIGH_THRESHOLD	7
+
+/* Receive queue sizes */
+/* Minimum of credit that the peer should hold.
+ * If the peer has less credits than 9 frames, we will explicitly send
+ * him some credits (through irttp_give_credit() and a specific frame).
+ * Note that when we give credits it's likely that it won't be sent in
+ * this LAP window, but in the next one. So, we make sure that the peer
+ * has something to send while waiting for credits (one LAP window == 7
+ * + 1 frames while he process the credits). - Jean II */
+#define TTP_RX_MIN_CREDIT	8
+/* This is the default maximum number of credits held by the peer, so the
+ * default maximum number of frames he can send us before needing flow
+ * control answer from us (this may be negociated differently at TSAP setup).
+ * We want to minimise the number of times we have to explicitly send some
+ * credit to the peer, hoping we can piggyback it on the return data. In
+ * particular, it doesn't make sense for us to send credit more than once
+ * per LAP window.
+ * Moreover, giving credits has some latency, so we need strictly more than
+ * a LAP window, otherwise we may already have credits in our Tx queue.
+ * But on the other hand, we don't want to keep too many Rx buffer here
+ * before starting to flow control the other end, so make it exactly one
+ * LAP window + 1 + MIN_CREDITS. - Jean II */
+#define TTP_RX_DEFAULT_CREDIT	16
+/* Maximum number of credits we can allow the peer to have, and therefore
+ * maximum Rx queue size.
+ * Note that we try to deliver packets to the higher layer every time we
+ * receive something, so in normal mode the Rx queue will never contains
+ * more than one or two packets. - Jean II */
+#define TTP_RX_MAX_CREDIT	21
+
+/* What clients should use when calling ttp_open_tsap() */
+#define DEFAULT_INITIAL_CREDIT	TTP_RX_DEFAULT_CREDIT
+
+/* Some priorities for disconnect requests */
+#define P_NORMAL    0
+#define P_HIGH      1
+
+#define TTP_SAR_DISABLE 0
+#define TTP_SAR_UNBOUND 0xffffffff
+
+/* Parameters */
+#define TTP_MAX_SDU_SIZE 0x01
+
+/*
+ *  This structure contains all data associated with one instance of a TTP 
+ *  connection.
+ */
+struct tsap_cb {
+	irda_queue_t q;            /* Must be first */
+	magic_t magic;        /* Just in case */
+
+	__u8 stsap_sel;       /* Source TSAP */
+	__u8 dtsap_sel;       /* Destination TSAP */
+
+	struct lsap_cb *lsap; /* Corresponding LSAP to this TSAP */
+
+	__u8 connected;       /* TSAP connected */
+	 
+	__u8 initial_credit;  /* Initial credit to give peer */
+
+        int avail_credit;    /* Available credit to return to peer */
+	int remote_credit;   /* Credit held by peer TTP entity */
+	int send_credit;     /* Credit held by local TTP entity */
+	
+	struct sk_buff_head tx_queue; /* Frames to be transmitted */
+	struct sk_buff_head rx_queue; /* Received frames */
+	struct sk_buff_head rx_fragments;
+	int tx_queue_lock;
+	int rx_queue_lock;
+	spinlock_t lock;
+
+	notify_t notify;       /* Callbacks to client layer */
+
+	struct net_device_stats stats;
+	struct timer_list todo_timer; 
+
+	__u32 max_seg_size;     /* Max data that fit into an IrLAP frame */
+	__u8  max_header_size;
+
+	int   rx_sdu_busy;     /* RxSdu.busy */
+	__u32 rx_sdu_size;     /* Current size of a partially received frame */
+	__u32 rx_max_sdu_size; /* Max receive user data size */
+
+	int tx_sdu_busy;       /* TxSdu.busy */
+	__u32 tx_max_sdu_size; /* Max transmit user data size */
+
+	int close_pend;        /* Close, but disconnect_pend */
+	unsigned long disconnect_pend; /* Disconnect, but still data to send */
+	struct sk_buff *disconnect_skb;
+};
+
+struct irttp_cb {
+	magic_t    magic;	
+	hashbin_t *tsaps;
+};
+
+int  irttp_init(void);
+void irttp_cleanup(void);
+
+struct tsap_cb *irttp_open_tsap(__u8 stsap_sel, int credit, notify_t *notify);
+int irttp_close_tsap(struct tsap_cb *self);
+
+int irttp_data_request(struct tsap_cb *self, struct sk_buff *skb);
+int irttp_udata_request(struct tsap_cb *self, struct sk_buff *skb);
+
+int irttp_connect_request(struct tsap_cb *self, __u8 dtsap_sel, 
+			  __u32 saddr, __u32 daddr,
+			  struct qos_info *qos, __u32 max_sdu_size, 
+			  struct sk_buff *userdata);
+int irttp_connect_response(struct tsap_cb *self, __u32 max_sdu_size, 
+			    struct sk_buff *userdata);
+int irttp_disconnect_request(struct tsap_cb *self, struct sk_buff *skb,
+			     int priority);
+void irttp_flow_request(struct tsap_cb *self, LOCAL_FLOW flow);
+struct tsap_cb *irttp_dup(struct tsap_cb *self, void *instance);
+
+static inline __u32 irttp_get_saddr(struct tsap_cb *self)
+{
+	return irlmp_get_saddr(self->lsap);
+}
+
+static inline __u32 irttp_get_daddr(struct tsap_cb *self)
+{
+	return irlmp_get_daddr(self->lsap);
+}
+
+static inline __u32 irttp_get_max_seg_size(struct tsap_cb *self)
+{
+	return self->max_seg_size;
+}
+
+/* After doing a irttp_dup(), this get one of the two socket back into
+ * a state where it's waiting incoming connections.
+ * Note : this can be used *only* if the socket is not yet connected
+ * (i.e. NO irttp_connect_response() done on this socket).
+ * - Jean II */
+static inline void irttp_listen(struct tsap_cb *self)
+{
+	irlmp_listen(self->lsap);
+	self->dtsap_sel = LSAP_ANY;
+}
+
+/* Return TRUE if the node is in primary mode (i.e. master)
+ * - Jean II */
+static inline int irttp_is_primary(struct tsap_cb *self)
+{
+	if ((self == NULL) ||
+	    (self->lsap == NULL) ||
+	    (self->lsap->lap == NULL) ||
+	    (self->lsap->lap->irlap == NULL))
+		return -2;
+	return irlap_is_primary(self->lsap->lap->irlap);
+}
+
+#endif /* IRTTP_H */
diff --git a/drivers/staging/irda/include/net/irda/parameters.h b/drivers/staging/irda/include/net/irda/parameters.h
new file mode 100644
index 000000000000..2d9cd0007cba
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/parameters.h
@@ -0,0 +1,100 @@
+/*********************************************************************
+ *                
+ * Filename:      parameters.h
+ * Version:       1.0
+ * Description:   A more general way to handle (pi,pl,pv) parameters
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Mon Jun  7 08:47:28 1999
+ * Modified at:   Sun Jan 30 14:05:14 2000
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1999-2000 Dag Brattli, All Rights Reserved.
+ *     
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ * 
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ * 
+ *     You should have received a copy of the GNU General Public License 
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ *     Michel Dänzer <daenzer@debian.org>, 10/2001
+ *     - simplify irda_pv_t to avoid endianness issues
+ *     
+ ********************************************************************/
+
+#ifndef IRDA_PARAMS_H
+#define IRDA_PARAMS_H
+
+/*
+ *  The currently supported types. Beware not to change the sequence since
+ *  it a good reason why the sized integers has a value equal to their size
+ */
+typedef enum {
+	PV_INTEGER,      /* Integer of any (pl) length */
+	PV_INT_8_BITS,   /* Integer of 8 bits in length */
+	PV_INT_16_BITS,  /* Integer of 16 bits in length */
+	PV_STRING,       /* \0 terminated string */
+	PV_INT_32_BITS,  /* Integer of 32 bits in length */
+	PV_OCT_SEQ,      /* Octet sequence */
+	PV_NO_VALUE      /* Does not contain any value (pl=0) */
+} PV_TYPE;
+
+/* Bit 7 of type field */
+#define PV_BIG_ENDIAN    0x80 
+#define PV_LITTLE_ENDIAN 0x00
+#define PV_MASK          0x7f   /* To mask away endian bit */
+
+#define PV_PUT 0
+#define PV_GET 1
+
+typedef union {
+	char   *c;
+	__u32   i;
+	__u32 *ip;
+} irda_pv_t;
+
+typedef struct {
+	__u8 pi;
+	__u8 pl;
+	irda_pv_t pv;
+} irda_param_t;
+
+typedef int (*PI_HANDLER)(void *self, irda_param_t *param, int get);
+typedef int (*PV_HANDLER)(void *self, __u8 *buf, int len, __u8 pi,
+			  PV_TYPE type, PI_HANDLER func);
+
+typedef struct {
+	const PI_HANDLER func;  /* Handler for this parameter identifier */
+	PV_TYPE    type;  /* Data type for this parameter */
+} pi_minor_info_t;
+
+typedef struct {
+	const pi_minor_info_t *pi_minor_call_table;
+	int len;
+} pi_major_info_t;
+
+typedef struct {
+	const pi_major_info_t *tables;
+	int              len;
+	__u8             pi_mask;
+	int              pi_major_offset;
+} pi_param_info_t;
+
+int irda_param_pack(__u8 *buf, char *fmt, ...);
+
+int irda_param_insert(void *self, __u8 pi, __u8 *buf, int len, 
+		      pi_param_info_t *info);
+int irda_param_extract_all(void *self, __u8 *buf, int len, 
+			   pi_param_info_t *info);
+
+#define irda_param_insert_byte(buf,pi,pv) irda_param_pack(buf,"bbb",pi,1,pv)
+
+#endif /* IRDA_PARAMS_H */
+
diff --git a/drivers/staging/irda/include/net/irda/qos.h b/drivers/staging/irda/include/net/irda/qos.h
new file mode 100644
index 000000000000..05a5a249956f
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/qos.h
@@ -0,0 +1,101 @@
+/*********************************************************************
+ *                
+ * Filename:      qos.h
+ * Version:       1.0
+ * Description:   Quality of Service definitions
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Fri Sep 19 23:21:09 1997
+ * Modified at:   Thu Dec  2 13:51:54 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1999 Dag Brattli, All Rights Reserved.
+ *     
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ * 
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ * 
+ *     You should have received a copy of the GNU General Public License 
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *     
+ ********************************************************************/
+
+#ifndef IRDA_QOS_H
+#define IRDA_QOS_H
+
+#include <linux/skbuff.h>
+
+#include <net/irda/parameters.h>
+
+#define PI_BAUD_RATE     0x01
+#define PI_MAX_TURN_TIME 0x82
+#define PI_DATA_SIZE     0x83
+#define PI_WINDOW_SIZE   0x84
+#define PI_ADD_BOFS      0x85
+#define PI_MIN_TURN_TIME 0x86
+#define PI_LINK_DISC     0x08
+
+#define IR_115200_MAX 0x3f
+
+/* Baud rates (first byte) */
+#define IR_2400     0x01
+#define IR_9600     0x02
+#define IR_19200    0x04
+#define IR_38400    0x08
+#define IR_57600    0x10
+#define IR_115200   0x20
+#define IR_576000   0x40
+#define IR_1152000  0x80
+
+/* Baud rates (second byte) */
+#define IR_4000000  0x01
+#define IR_16000000 0x02
+
+/* Quality of Service information */
+typedef struct {
+	__u32 value;
+	__u16 bits; /* LSB is first byte, MSB is second byte */
+} qos_value_t;
+
+struct qos_info {
+	magic_t magic;
+
+	qos_value_t baud_rate;       /* IR_11520O | ... */
+	qos_value_t max_turn_time;
+	qos_value_t data_size;
+	qos_value_t window_size;
+	qos_value_t additional_bofs;
+	qos_value_t min_turn_time;
+	qos_value_t link_disc_time;
+	
+	qos_value_t power;
+};
+
+extern int sysctl_max_baud_rate;
+extern int sysctl_max_inactive_time;
+
+void irda_init_max_qos_capabilies(struct qos_info *qos);
+void irda_qos_compute_intersection(struct qos_info *, struct qos_info *);
+
+__u32 irlap_max_line_capacity(__u32 speed, __u32 max_turn_time);
+
+void irda_qos_bits_to_value(struct qos_info *qos);
+
+/* So simple, how could we not inline those two ?
+ * Note : one byte is 10 bits if you include start and stop bits
+ * Jean II */
+#define irlap_min_turn_time_in_bytes(speed, min_turn_time) (	\
+	speed * min_turn_time / 10000000			\
+)
+#define irlap_xbofs_in_usec(speed, xbofs) (			\
+	xbofs * 10000000 / speed				\
+)
+
+#endif
+
diff --git a/drivers/staging/irda/include/net/irda/timer.h b/drivers/staging/irda/include/net/irda/timer.h
new file mode 100644
index 000000000000..d784f242cf7b
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/timer.h
@@ -0,0 +1,105 @@
+/*********************************************************************
+ *                
+ * Filename:      timer.h
+ * Version:       
+ * Description:   
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sat Aug 16 00:59:29 1997
+ * Modified at:   Thu Oct  7 12:25:24 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1997, 1998-1999 Dag Brattli <dagb@cs.uit.no>, 
+ *     All Rights Reserved.
+ *     Copyright (c) 2000-2002 Jean Tourrilhes <jt@hpl.hp.com>
+ *     
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is 
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#ifndef TIMER_H
+#define TIMER_H
+
+#include <linux/timer.h>
+#include <linux/jiffies.h>
+
+#include <asm/param.h>  /* for HZ */
+
+#include <net/irda/irda.h>
+
+/* A few forward declarations (to make compiler happy) */
+struct irlmp_cb;
+struct irlap_cb;
+struct lsap_cb;
+struct lap_cb;
+
+/* 
+ *  Timeout definitions, some defined in IrLAP 6.13.5 - p. 92
+ */
+#define POLL_TIMEOUT        (450*HZ/1000)    /* Must never exceed 500 ms */
+#define FINAL_TIMEOUT       (500*HZ/1000)    /* Must never exceed 500 ms */
+
+/* 
+ *  Normally twice of p-timer. Note 3, IrLAP 6.3.11.2 - p. 60 suggests
+ *  at least twice duration of the P-timer.
+ */
+#define WD_TIMEOUT          (POLL_TIMEOUT*2)
+
+#define MEDIABUSY_TIMEOUT   (500*HZ/1000)    /* 500 msec */
+#define SMALLBUSY_TIMEOUT   (100*HZ/1000)    /* 100 msec - IrLAP 6.13.4 */
+
+/*
+ *  Slot timer must never exceed 85 ms, and must always be at least 25 ms, 
+ *  suggested to  75-85 msec by IrDA lite. This doesn't work with a lot of
+ *  devices, and other stackes uses a lot more, so it's best we do it as well
+ *  (Note : this is the default value and sysctl overrides it - Jean II)
+ */
+#define SLOT_TIMEOUT            (90*HZ/1000)
+
+/* 
+ *  The latest discovery frame (XID) is longer due to the extra discovery
+ *  information (hints, device name...). This is its extra length.
+ *  We use that when setting the query timeout. Jean II
+ */
+#define XIDEXTRA_TIMEOUT        (34*HZ/1000)  /* 34 msec */
+
+#define WATCHDOG_TIMEOUT        (20*HZ)       /* 20 sec */
+
+typedef void (*TIMER_CALLBACK)(void *);
+
+static inline void irda_start_timer(struct timer_list *ptimer, int timeout, 
+				    void* data, TIMER_CALLBACK callback)
+{
+	ptimer->function = (void (*)(unsigned long)) callback;
+	ptimer->data = (unsigned long) data;
+	
+	/* Set new value for timer (update or add timer).
+	 * We use mod_timer() because it's more efficient and also
+	 * safer with respect to race conditions - Jean II */
+	mod_timer(ptimer, jiffies + timeout);
+}
+
+
+void irlap_start_slot_timer(struct irlap_cb *self, int timeout);
+void irlap_start_query_timer(struct irlap_cb *self, int S, int s);
+void irlap_start_final_timer(struct irlap_cb *self, int timeout);
+void irlap_start_wd_timer(struct irlap_cb *self, int timeout);
+void irlap_start_backoff_timer(struct irlap_cb *self, int timeout);
+
+void irlap_start_mbusy_timer(struct irlap_cb *self, int timeout);
+void irlap_stop_mbusy_timer(struct irlap_cb *);
+
+void irlmp_start_watchdog_timer(struct lsap_cb *, int timeout);
+void irlmp_start_discovery_timer(struct irlmp_cb *, int timeout);
+void irlmp_start_idle_timer(struct lap_cb *, int timeout);
+void irlmp_stop_idle_timer(struct lap_cb *self);
+
+#endif
+
diff --git a/drivers/staging/irda/include/net/irda/wrapper.h b/drivers/staging/irda/include/net/irda/wrapper.h
new file mode 100644
index 000000000000..eef53ebe3d76
--- /dev/null
+++ b/drivers/staging/irda/include/net/irda/wrapper.h
@@ -0,0 +1,58 @@
+/*********************************************************************
+ *                
+ * Filename:      wrapper.h
+ * Version:       1.2
+ * Description:   IrDA SIR async wrapper layer
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Mon Aug  4 20:40:53 1997
+ * Modified at:   Tue Jan 11 12:37:29 2000
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * 
+ *     Copyright (c) 1998-2000 Dag Brattli <dagb@cs.uit.no>, 
+ *     All Rights Reserved.
+ *     
+ *     This program is free software; you can redistribute it and/or 
+ *     modify it under the terms of the GNU General Public License as 
+ *     published by the Free Software Foundation; either version 2 of 
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is 
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#ifndef WRAPPER_H
+#define WRAPPER_H
+
+#include <linux/types.h>
+#include <linux/skbuff.h>
+#include <linux/netdevice.h>
+
+#include <net/irda/irda_device.h>	/* iobuff_t */
+
+#define BOF  0xc0 /* Beginning of frame */
+#define XBOF 0xff
+#define EOF  0xc1 /* End of frame */
+#define CE   0x7d /* Control escape */
+
+#define STA BOF  /* Start flag */
+#define STO EOF  /* End flag */
+
+#define IRDA_TRANS 0x20    /* Asynchronous transparency modifier */       
+
+/* States for receiving a frame in async mode */
+enum {
+	OUTSIDE_FRAME, 
+	BEGIN_FRAME, 
+	LINK_ESCAPE, 
+	INSIDE_FRAME
+};
+
+/* Proto definitions */
+int async_wrap_skb(struct sk_buff *skb, __u8 *tx_buff, int buffsize);
+void async_unwrap_char(struct net_device *dev, struct net_device_stats *stats,
+		       iobuff_t *buf, __u8 byte);
+
+#endif
diff --git a/drivers/staging/irda/net/Kconfig b/drivers/staging/irda/net/Kconfig
new file mode 100644
index 000000000000..6abeae6c666a
--- /dev/null
+++ b/drivers/staging/irda/net/Kconfig
@@ -0,0 +1,96 @@
+#
+# IrDA protocol configuration
+#
+
+menuconfig IRDA
+	depends on NET && !S390
+	tristate "IrDA (infrared) subsystem support"
+	select CRC_CCITT
+	---help---
+	  Say Y here if you want to build support for the IrDA (TM) protocols.
+	  The Infrared Data Associations (tm) specifies standards for wireless
+	  infrared communication and is supported by most laptops and PDA's.
+
+	  To use Linux support for the IrDA (tm) protocols, you will also need
+	  some user-space utilities like irattach.  For more information, see
+	  the file <file:Documentation/networking/irda.txt>.  You also want to
+	  read the IR-HOWTO, available at
+	  <http://www.tldp.org/docs.html#howto>.
+
+	  If you want to exchange bits of data (vCal, vCard) with a PDA, you
+	  will need to install some OBEX application, such as OpenObex :
+	  <http://sourceforge.net/projects/openobex/>
+
+	  To compile this support as a module, choose M here: the module will
+	  be called irda.
+
+comment "IrDA protocols"
+	depends on IRDA
+
+source "drivers/staging/irda/net/irlan/Kconfig"
+
+source "drivers/staging/irda/net/irnet/Kconfig"
+
+source "drivers/staging/irda/net/ircomm/Kconfig"
+
+config IRDA_ULTRA
+	bool "Ultra (connectionless) protocol"
+	depends on IRDA
+	help
+	  Say Y here to support the connectionless Ultra IRDA protocol.
+	  Ultra allows to exchange data over IrDA with really simple devices
+	  (watch, beacon) without the overhead of the IrDA protocol (no handshaking,
+	  no management frames, simple fixed header).
+	  Ultra is available as a special socket : socket(AF_IRDA, SOCK_DGRAM, 1);
+
+comment "IrDA options"
+	depends on IRDA
+
+config IRDA_CACHE_LAST_LSAP
+	bool "Cache last LSAP"
+	depends on IRDA
+	help
+	  Say Y here if you want IrLMP to cache the last LSAP used.  This
+	  makes sense since most frames will be sent/received on the same
+	  connection.  Enabling this option will save a hash-lookup per frame.
+
+	  If unsure, say Y.
+
+config IRDA_FAST_RR
+	bool "Fast RRs (low latency)"
+	depends on IRDA
+	---help---
+	  Say Y here is you want IrLAP to send fast RR (Receive Ready) frames
+	  when acting as a primary station.
+	  Disabling this option will make latency over IrDA very bad. Enabling
+	  this option will make the IrDA stack send more packet than strictly
+	  necessary, thus reduce your battery life (but not that much).
+
+	  Fast RR will make IrLAP send out a RR frame immediately when
+	  receiving a frame if its own transmit queue is currently empty. This
+	  will give a lot of speed improvement when receiving much data since
+	  the secondary station will not have to wait the max. turn around
+	  time (usually 500ms) before it is allowed to transmit the next time.
+	  If the transmit queue of the secondary is also empty, the primary will
+	  start backing-off before sending another RR frame, waiting longer
+	  each time until the back-off reaches the max. turn around time.
+	  This back-off increase in controlled via
+	  /proc/sys/net/irda/fast_poll_increase
+
+	  If unsure, say Y.
+
+config IRDA_DEBUG
+	bool "Debug information"
+	depends on IRDA
+	help
+	  Say Y here if you want the IrDA subsystem to write debug information
+	  to your syslog. You can change the debug level in
+	  /proc/sys/net/irda/debug .
+	  When this option is enabled, the IrDA also perform many extra internal
+	  verifications which will usually prevent the kernel to crash in case of
+	  bugs.
+
+	  If unsure, say Y (since it makes it easier to find the bugs).
+
+source "drivers/staging/irda/drivers/Kconfig"
+
diff --git a/drivers/staging/irda/net/Makefile b/drivers/staging/irda/net/Makefile
new file mode 100644
index 000000000000..bd1a635b88cf
--- /dev/null
+++ b/drivers/staging/irda/net/Makefile
@@ -0,0 +1,17 @@
+#
+# Makefile for the Linux IrDA protocol layer.
+#
+
+subdir-ccflags-y += -I$(srctree)/drivers/staging/irda/include
+
+obj-$(CONFIG_IRDA) += irda.o
+obj-$(CONFIG_IRLAN) += irlan/
+obj-$(CONFIG_IRNET) += irnet/
+obj-$(CONFIG_IRCOMM) += ircomm/
+
+irda-y := iriap.o iriap_event.o irlmp.o irlmp_event.o irlmp_frame.o \
+          irlap.o irlap_event.o irlap_frame.o timer.o qos.o irqueue.o \
+          irttp.o irda_device.o irias_object.o wrapper.o af_irda.o \
+	  discovery.o parameters.o irnetlink.o irmod.o
+irda-$(CONFIG_PROC_FS) += irproc.o
+irda-$(CONFIG_SYSCTL) += irsysctl.o
diff --git a/drivers/staging/irda/net/af_irda.c b/drivers/staging/irda/net/af_irda.c
new file mode 100644
index 000000000000..23fa7c8b09a5
--- /dev/null
+++ b/drivers/staging/irda/net/af_irda.c
@@ -0,0 +1,2695 @@
+/*********************************************************************
+ *
+ * Filename:      af_irda.c
+ * Version:       0.9
+ * Description:   IrDA sockets implementation
+ * Status:        Stable
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sun May 31 10:12:43 1998
+ * Modified at:   Sat Dec 25 21:10:23 1999
+ * Modified by:   Dag Brattli <dag@brattli.net>
+ * Sources:       af_netroom.c, af_ax25.c, af_rose.c, af_x25.c etc.
+ *
+ *     Copyright (c) 1999 Dag Brattli <dagb@cs.uit.no>
+ *     Copyright (c) 1999-2003 Jean Tourrilhes <jt@hpl.hp.com>
+ *     All Rights Reserved.
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ *
+ *     You should have received a copy of the GNU General Public License
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ *     Linux-IrDA now supports four different types of IrDA sockets:
+ *
+ *     o SOCK_STREAM:    TinyTP connections with SAR disabled. The
+ *                       max SDU size is 0 for conn. of this type
+ *     o SOCK_SEQPACKET: TinyTP connections with SAR enabled. TTP may
+ *                       fragment the messages, but will preserve
+ *                       the message boundaries
+ *     o SOCK_DGRAM:     IRDAPROTO_UNITDATA: TinyTP connections with Unitdata
+ *                       (unreliable) transfers
+ *                       IRDAPROTO_ULTRA: Connectionless and unreliable data
+ *
+ ********************************************************************/
+
+#include <linux/capability.h>
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/socket.h>
+#include <linux/sockios.h>
+#include <linux/slab.h>
+#include <linux/sched/signal.h>
+#include <linux/init.h>
+#include <linux/net.h>
+#include <linux/irda.h>
+#include <linux/poll.h>
+
+#include <asm/ioctls.h>		/* TIOCOUTQ, TIOCINQ */
+#include <linux/uaccess.h>
+
+#include <net/sock.h>
+#include <net/tcp_states.h>
+
+#include <net/irda/af_irda.h>
+
+static int irda_create(struct net *net, struct socket *sock, int protocol, int kern);
+
+static const struct proto_ops irda_stream_ops;
+static const struct proto_ops irda_seqpacket_ops;
+static const struct proto_ops irda_dgram_ops;
+
+#ifdef CONFIG_IRDA_ULTRA
+static const struct proto_ops irda_ultra_ops;
+#define ULTRA_MAX_DATA 382
+#endif /* CONFIG_IRDA_ULTRA */
+
+#define IRDA_MAX_HEADER (TTP_MAX_HEADER)
+
+/*
+ * Function irda_data_indication (instance, sap, skb)
+ *
+ *    Received some data from TinyTP. Just queue it on the receive queue
+ *
+ */
+static int irda_data_indication(void *instance, void *sap, struct sk_buff *skb)
+{
+	struct irda_sock *self;
+	struct sock *sk;
+	int err;
+
+	self = instance;
+	sk = instance;
+
+	err = sock_queue_rcv_skb(sk, skb);
+	if (err) {
+		pr_debug("%s(), error: no more mem!\n", __func__);
+		self->rx_flow = FLOW_STOP;
+
+		/* When we return error, TTP will need to requeue the skb */
+		return err;
+	}
+
+	return 0;
+}
+
+/*
+ * Function irda_disconnect_indication (instance, sap, reason, skb)
+ *
+ *    Connection has been closed. Check reason to find out why
+ *
+ */
+static void irda_disconnect_indication(void *instance, void *sap,
+				       LM_REASON reason, struct sk_buff *skb)
+{
+	struct irda_sock *self;
+	struct sock *sk;
+
+	self = instance;
+
+	pr_debug("%s(%p)\n", __func__, self);
+
+	/* Don't care about it, but let's not leak it */
+	if(skb)
+		dev_kfree_skb(skb);
+
+	sk = instance;
+	if (sk == NULL) {
+		pr_debug("%s(%p) : BUG : sk is NULL\n",
+			 __func__, self);
+		return;
+	}
+
+	/* Prevent race conditions with irda_release() and irda_shutdown() */
+	bh_lock_sock(sk);
+	if (!sock_flag(sk, SOCK_DEAD) && sk->sk_state != TCP_CLOSE) {
+		sk->sk_state     = TCP_CLOSE;
+		sk->sk_shutdown |= SEND_SHUTDOWN;
+
+		sk->sk_state_change(sk);
+
+		/* Close our TSAP.
+		 * If we leave it open, IrLMP put it back into the list of
+		 * unconnected LSAPs. The problem is that any incoming request
+		 * can then be matched to this socket (and it will be, because
+		 * it is at the head of the list). This would prevent any
+		 * listening socket waiting on the same TSAP to get those
+		 * requests. Some apps forget to close sockets, or hang to it
+		 * a bit too long, so we may stay in this dead state long
+		 * enough to be noticed...
+		 * Note : all socket function do check sk->sk_state, so we are
+		 * safe...
+		 * Jean II
+		 */
+		if (self->tsap) {
+			irttp_close_tsap(self->tsap);
+			self->tsap = NULL;
+		}
+	}
+	bh_unlock_sock(sk);
+
+	/* Note : once we are there, there is not much you want to do
+	 * with the socket anymore, apart from closing it.
+	 * For example, bind() and connect() won't reset sk->sk_err,
+	 * sk->sk_shutdown and sk->sk_flags to valid values...
+	 * Jean II
+	 */
+}
+
+/*
+ * Function irda_connect_confirm (instance, sap, qos, max_sdu_size, skb)
+ *
+ *    Connections has been confirmed by the remote device
+ *
+ */
+static void irda_connect_confirm(void *instance, void *sap,
+				 struct qos_info *qos,
+				 __u32 max_sdu_size, __u8 max_header_size,
+				 struct sk_buff *skb)
+{
+	struct irda_sock *self;
+	struct sock *sk;
+
+	self = instance;
+
+	pr_debug("%s(%p)\n", __func__, self);
+
+	sk = instance;
+	if (sk == NULL) {
+		dev_kfree_skb(skb);
+		return;
+	}
+
+	dev_kfree_skb(skb);
+	// Should be ??? skb_queue_tail(&sk->sk_receive_queue, skb);
+
+	/* How much header space do we need to reserve */
+	self->max_header_size = max_header_size;
+
+	/* IrTTP max SDU size in transmit direction */
+	self->max_sdu_size_tx = max_sdu_size;
+
+	/* Find out what the largest chunk of data that we can transmit is */
+	switch (sk->sk_type) {
+	case SOCK_STREAM:
+		if (max_sdu_size != 0) {
+			net_err_ratelimited("%s: max_sdu_size must be 0\n",
+					    __func__);
+			return;
+		}
+		self->max_data_size = irttp_get_max_seg_size(self->tsap);
+		break;
+	case SOCK_SEQPACKET:
+		if (max_sdu_size == 0) {
+			net_err_ratelimited("%s: max_sdu_size cannot be 0\n",
+					    __func__);
+			return;
+		}
+		self->max_data_size = max_sdu_size;
+		break;
+	default:
+		self->max_data_size = irttp_get_max_seg_size(self->tsap);
+	}
+
+	pr_debug("%s(), max_data_size=%d\n", __func__,
+		 self->max_data_size);
+
+	memcpy(&self->qos_tx, qos, sizeof(struct qos_info));
+
+	/* We are now connected! */
+	sk->sk_state = TCP_ESTABLISHED;
+	sk->sk_state_change(sk);
+}
+
+/*
+ * Function irda_connect_indication(instance, sap, qos, max_sdu_size, userdata)
+ *
+ *    Incoming connection
+ *
+ */
+static void irda_connect_indication(void *instance, void *sap,
+				    struct qos_info *qos, __u32 max_sdu_size,
+				    __u8 max_header_size, struct sk_buff *skb)
+{
+	struct irda_sock *self;
+	struct sock *sk;
+
+	self = instance;
+
+	pr_debug("%s(%p)\n", __func__, self);
+
+	sk = instance;
+	if (sk == NULL) {
+		dev_kfree_skb(skb);
+		return;
+	}
+
+	/* How much header space do we need to reserve */
+	self->max_header_size = max_header_size;
+
+	/* IrTTP max SDU size in transmit direction */
+	self->max_sdu_size_tx = max_sdu_size;
+
+	/* Find out what the largest chunk of data that we can transmit is */
+	switch (sk->sk_type) {
+	case SOCK_STREAM:
+		if (max_sdu_size != 0) {
+			net_err_ratelimited("%s: max_sdu_size must be 0\n",
+					    __func__);
+			kfree_skb(skb);
+			return;
+		}
+		self->max_data_size = irttp_get_max_seg_size(self->tsap);
+		break;
+	case SOCK_SEQPACKET:
+		if (max_sdu_size == 0) {
+			net_err_ratelimited("%s: max_sdu_size cannot be 0\n",
+					    __func__);
+			kfree_skb(skb);
+			return;
+		}
+		self->max_data_size = max_sdu_size;
+		break;
+	default:
+		self->max_data_size = irttp_get_max_seg_size(self->tsap);
+	}
+
+	pr_debug("%s(), max_data_size=%d\n", __func__,
+		 self->max_data_size);
+
+	memcpy(&self->qos_tx, qos, sizeof(struct qos_info));
+
+	skb_queue_tail(&sk->sk_receive_queue, skb);
+	sk->sk_state_change(sk);
+}
+
+/*
+ * Function irda_connect_response (handle)
+ *
+ *    Accept incoming connection
+ *
+ */
+static void irda_connect_response(struct irda_sock *self)
+{
+	struct sk_buff *skb;
+
+	skb = alloc_skb(TTP_MAX_HEADER + TTP_SAR_HEADER, GFP_KERNEL);
+	if (skb == NULL) {
+		pr_debug("%s() Unable to allocate sk_buff!\n",
+			 __func__);
+		return;
+	}
+
+	/* Reserve space for MUX_CONTROL and LAP header */
+	skb_reserve(skb, IRDA_MAX_HEADER);
+
+	irttp_connect_response(self->tsap, self->max_sdu_size_rx, skb);
+}
+
+/*
+ * Function irda_flow_indication (instance, sap, flow)
+ *
+ *    Used by TinyTP to tell us if it can accept more data or not
+ *
+ */
+static void irda_flow_indication(void *instance, void *sap, LOCAL_FLOW flow)
+{
+	struct irda_sock *self;
+	struct sock *sk;
+
+	self = instance;
+	sk = instance;
+	BUG_ON(sk == NULL);
+
+	switch (flow) {
+	case FLOW_STOP:
+		pr_debug("%s(), IrTTP wants us to slow down\n",
+			 __func__);
+		self->tx_flow = flow;
+		break;
+	case FLOW_START:
+		self->tx_flow = flow;
+		pr_debug("%s(), IrTTP wants us to start again\n",
+			 __func__);
+		wake_up_interruptible(sk_sleep(sk));
+		break;
+	default:
+		pr_debug("%s(), Unknown flow command!\n", __func__);
+		/* Unknown flow command, better stop */
+		self->tx_flow = flow;
+		break;
+	}
+}
+
+/*
+ * Function irda_getvalue_confirm (obj_id, value, priv)
+ *
+ *    Got answer from remote LM-IAS, just pass object to requester...
+ *
+ * Note : duplicate from above, but we need our own version that
+ * doesn't touch the dtsap_sel and save the full value structure...
+ */
+static void irda_getvalue_confirm(int result, __u16 obj_id,
+				  struct ias_value *value, void *priv)
+{
+	struct irda_sock *self;
+
+	self = priv;
+	if (!self) {
+		net_warn_ratelimited("%s: lost myself!\n", __func__);
+		return;
+	}
+
+	pr_debug("%s(%p)\n", __func__, self);
+
+	/* We probably don't need to make any more queries */
+	iriap_close(self->iriap);
+	self->iriap = NULL;
+
+	/* Check if request succeeded */
+	if (result != IAS_SUCCESS) {
+		pr_debug("%s(), IAS query failed! (%d)\n", __func__,
+			 result);
+
+		self->errno = result;	/* We really need it later */
+
+		/* Wake up any processes waiting for result */
+		wake_up_interruptible(&self->query_wait);
+
+		return;
+	}
+
+	/* Pass the object to the caller (so the caller must delete it) */
+	self->ias_result = value;
+	self->errno = 0;
+
+	/* Wake up any processes waiting for result */
+	wake_up_interruptible(&self->query_wait);
+}
+
+/*
+ * Function irda_selective_discovery_indication (discovery)
+ *
+ *    Got a selective discovery indication from IrLMP.
+ *
+ * IrLMP is telling us that this node is new and matching our hint bit
+ * filter. Wake up any process waiting for answer...
+ */
+static void irda_selective_discovery_indication(discinfo_t *discovery,
+						DISCOVERY_MODE mode,
+						void *priv)
+{
+	struct irda_sock *self;
+
+	self = priv;
+	if (!self) {
+		net_warn_ratelimited("%s: lost myself!\n", __func__);
+		return;
+	}
+
+	/* Pass parameter to the caller */
+	self->cachedaddr = discovery->daddr;
+
+	/* Wake up process if its waiting for device to be discovered */
+	wake_up_interruptible(&self->query_wait);
+}
+
+/*
+ * Function irda_discovery_timeout (priv)
+ *
+ *    Timeout in the selective discovery process
+ *
+ * We were waiting for a node to be discovered, but nothing has come up
+ * so far. Wake up the user and tell him that we failed...
+ */
+static void irda_discovery_timeout(u_long priv)
+{
+	struct irda_sock *self;
+
+	self = (struct irda_sock *) priv;
+	BUG_ON(self == NULL);
+
+	/* Nothing for the caller */
+	self->cachelog = NULL;
+	self->cachedaddr = 0;
+	self->errno = -ETIME;
+
+	/* Wake up process if its still waiting... */
+	wake_up_interruptible(&self->query_wait);
+}
+
+/*
+ * Function irda_open_tsap (self)
+ *
+ *    Open local Transport Service Access Point (TSAP)
+ *
+ */
+static int irda_open_tsap(struct irda_sock *self, __u8 tsap_sel, char *name)
+{
+	notify_t notify;
+
+	if (self->tsap) {
+		pr_debug("%s: busy!\n", __func__);
+		return -EBUSY;
+	}
+
+	/* Initialize callbacks to be used by the IrDA stack */
+	irda_notify_init(&notify);
+	notify.connect_confirm       = irda_connect_confirm;
+	notify.connect_indication    = irda_connect_indication;
+	notify.disconnect_indication = irda_disconnect_indication;
+	notify.data_indication       = irda_data_indication;
+	notify.udata_indication	     = irda_data_indication;
+	notify.flow_indication       = irda_flow_indication;
+	notify.instance = self;
+	strncpy(notify.name, name, NOTIFY_MAX_NAME);
+
+	self->tsap = irttp_open_tsap(tsap_sel, DEFAULT_INITIAL_CREDIT,
+				     &notify);
+	if (self->tsap == NULL) {
+		pr_debug("%s(), Unable to allocate TSAP!\n",
+			 __func__);
+		return -ENOMEM;
+	}
+	/* Remember which TSAP selector we actually got */
+	self->stsap_sel = self->tsap->stsap_sel;
+
+	return 0;
+}
+
+/*
+ * Function irda_open_lsap (self)
+ *
+ *    Open local Link Service Access Point (LSAP). Used for opening Ultra
+ *    sockets
+ */
+#ifdef CONFIG_IRDA_ULTRA
+static int irda_open_lsap(struct irda_sock *self, int pid)
+{
+	notify_t notify;
+
+	if (self->lsap) {
+		net_warn_ratelimited("%s(), busy!\n", __func__);
+		return -EBUSY;
+	}
+
+	/* Initialize callbacks to be used by the IrDA stack */
+	irda_notify_init(&notify);
+	notify.udata_indication	= irda_data_indication;
+	notify.instance = self;
+	strncpy(notify.name, "Ultra", NOTIFY_MAX_NAME);
+
+	self->lsap = irlmp_open_lsap(LSAP_CONNLESS, &notify, pid);
+	if (self->lsap == NULL) {
+		pr_debug("%s(), Unable to allocate LSAP!\n", __func__);
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+#endif /* CONFIG_IRDA_ULTRA */
+
+/*
+ * Function irda_find_lsap_sel (self, name)
+ *
+ *    Try to lookup LSAP selector in remote LM-IAS
+ *
+ * Basically, we start a IAP query, and then go to sleep. When the query
+ * return, irda_getvalue_confirm will wake us up, and we can examine the
+ * result of the query...
+ * Note that in some case, the query fail even before we go to sleep,
+ * creating some races...
+ */
+static int irda_find_lsap_sel(struct irda_sock *self, char *name)
+{
+	pr_debug("%s(%p, %s)\n", __func__, self, name);
+
+	if (self->iriap) {
+		net_warn_ratelimited("%s(): busy with a previous query\n",
+				     __func__);
+		return -EBUSY;
+	}
+
+	self->iriap = iriap_open(LSAP_ANY, IAS_CLIENT, self,
+				 irda_getvalue_confirm);
+	if(self->iriap == NULL)
+		return -ENOMEM;
+
+	/* Treat unexpected wakeup as disconnect */
+	self->errno = -EHOSTUNREACH;
+
+	/* Query remote LM-IAS */
+	iriap_getvaluebyclass_request(self->iriap, self->saddr, self->daddr,
+				      name, "IrDA:TinyTP:LsapSel");
+
+	/* Wait for answer, if not yet finished (or failed) */
+	if (wait_event_interruptible(self->query_wait, (self->iriap==NULL)))
+		/* Treat signals as disconnect */
+		return -EHOSTUNREACH;
+
+	/* Check what happened */
+	if (self->errno)
+	{
+		/* Requested object/attribute doesn't exist */
+		if((self->errno == IAS_CLASS_UNKNOWN) ||
+		   (self->errno == IAS_ATTRIB_UNKNOWN))
+			return -EADDRNOTAVAIL;
+		else
+			return -EHOSTUNREACH;
+	}
+
+	/* Get the remote TSAP selector */
+	switch (self->ias_result->type) {
+	case IAS_INTEGER:
+		pr_debug("%s() int=%d\n",
+			 __func__, self->ias_result->t.integer);
+
+		if (self->ias_result->t.integer != -1)
+			self->dtsap_sel = self->ias_result->t.integer;
+		else
+			self->dtsap_sel = 0;
+		break;
+	default:
+		self->dtsap_sel = 0;
+		pr_debug("%s(), bad type!\n", __func__);
+		break;
+	}
+	if (self->ias_result)
+		irias_delete_value(self->ias_result);
+
+	if (self->dtsap_sel)
+		return 0;
+
+	return -EADDRNOTAVAIL;
+}
+
+/*
+ * Function irda_discover_daddr_and_lsap_sel (self, name)
+ *
+ *    This try to find a device with the requested service.
+ *
+ * It basically look into the discovery log. For each address in the list,
+ * it queries the LM-IAS of the device to find if this device offer
+ * the requested service.
+ * If there is more than one node supporting the service, we complain
+ * to the user (it should move devices around).
+ * The, we set both the destination address and the lsap selector to point
+ * on the service on the unique device we have found.
+ *
+ * Note : this function fails if there is more than one device in range,
+ * because IrLMP doesn't disconnect the LAP when the last LSAP is closed.
+ * Moreover, we would need to wait the LAP disconnection...
+ */
+static int irda_discover_daddr_and_lsap_sel(struct irda_sock *self, char *name)
+{
+	discinfo_t *discoveries;	/* Copy of the discovery log */
+	int	number;			/* Number of nodes in the log */
+	int	i;
+	int	err = -ENETUNREACH;
+	__u32	daddr = DEV_ADDR_ANY;	/* Address we found the service on */
+	__u8	dtsap_sel = 0x0;	/* TSAP associated with it */
+
+	pr_debug("%s(), name=%s\n", __func__, name);
+
+	/* Ask lmp for the current discovery log
+	 * Note : we have to use irlmp_get_discoveries(), as opposed
+	 * to play with the cachelog directly, because while we are
+	 * making our ias query, le log might change... */
+	discoveries = irlmp_get_discoveries(&number, self->mask.word,
+					    self->nslots);
+	/* Check if the we got some results */
+	if (discoveries == NULL)
+		return -ENETUNREACH;	/* No nodes discovered */
+
+	/*
+	 * Now, check all discovered devices (if any), and connect
+	 * client only about the services that the client is
+	 * interested in...
+	 */
+	for(i = 0; i < number; i++) {
+		/* Try the address in the log */
+		self->daddr = discoveries[i].daddr;
+		self->saddr = 0x0;
+		pr_debug("%s(), trying daddr = %08x\n",
+			 __func__, self->daddr);
+
+		/* Query remote LM-IAS for this service */
+		err = irda_find_lsap_sel(self, name);
+		switch (err) {
+		case 0:
+			/* We found the requested service */
+			if(daddr != DEV_ADDR_ANY) {
+				pr_debug("%s(), discovered service ''%s'' in two different devices !!!\n",
+					 __func__, name);
+				self->daddr = DEV_ADDR_ANY;
+				kfree(discoveries);
+				return -ENOTUNIQ;
+			}
+			/* First time we found that one, save it ! */
+			daddr = self->daddr;
+			dtsap_sel = self->dtsap_sel;
+			break;
+		case -EADDRNOTAVAIL:
+			/* Requested service simply doesn't exist on this node */
+			break;
+		default:
+			/* Something bad did happen :-( */
+			pr_debug("%s(), unexpected IAS query failure\n",
+				 __func__);
+			self->daddr = DEV_ADDR_ANY;
+			kfree(discoveries);
+			return -EHOSTUNREACH;
+		}
+	}
+	/* Cleanup our copy of the discovery log */
+	kfree(discoveries);
+
+	/* Check out what we found */
+	if(daddr == DEV_ADDR_ANY) {
+		pr_debug("%s(), cannot discover service ''%s'' in any device !!!\n",
+			 __func__, name);
+		self->daddr = DEV_ADDR_ANY;
+		return -EADDRNOTAVAIL;
+	}
+
+	/* Revert back to discovered device & service */
+	self->daddr = daddr;
+	self->saddr = 0x0;
+	self->dtsap_sel = dtsap_sel;
+
+	pr_debug("%s(), discovered requested service ''%s'' at address %08x\n",
+		 __func__, name, self->daddr);
+
+	return 0;
+}
+
+/*
+ * Function irda_getname (sock, uaddr, uaddr_len, peer)
+ *
+ *    Return the our own, or peers socket address (sockaddr_irda)
+ *
+ */
+static int irda_getname(struct socket *sock, struct sockaddr *uaddr,
+			int *uaddr_len, int peer)
+{
+	struct sockaddr_irda saddr;
+	struct sock *sk = sock->sk;
+	struct irda_sock *self = irda_sk(sk);
+
+	memset(&saddr, 0, sizeof(saddr));
+	if (peer) {
+		if (sk->sk_state != TCP_ESTABLISHED)
+			return -ENOTCONN;
+
+		saddr.sir_family = AF_IRDA;
+		saddr.sir_lsap_sel = self->dtsap_sel;
+		saddr.sir_addr = self->daddr;
+	} else {
+		saddr.sir_family = AF_IRDA;
+		saddr.sir_lsap_sel = self->stsap_sel;
+		saddr.sir_addr = self->saddr;
+	}
+
+	pr_debug("%s(), tsap_sel = %#x\n", __func__, saddr.sir_lsap_sel);
+	pr_debug("%s(), addr = %08x\n", __func__, saddr.sir_addr);
+
+	/* uaddr_len come to us uninitialised */
+	*uaddr_len = sizeof (struct sockaddr_irda);
+	memcpy(uaddr, &saddr, *uaddr_len);
+
+	return 0;
+}
+
+/*
+ * Function irda_listen (sock, backlog)
+ *
+ *    Just move to the listen state
+ *
+ */
+static int irda_listen(struct socket *sock, int backlog)
+{
+	struct sock *sk = sock->sk;
+	int err = -EOPNOTSUPP;
+
+	lock_sock(sk);
+
+	if ((sk->sk_type != SOCK_STREAM) && (sk->sk_type != SOCK_SEQPACKET) &&
+	    (sk->sk_type != SOCK_DGRAM))
+		goto out;
+
+	if (sk->sk_state != TCP_LISTEN) {
+		sk->sk_max_ack_backlog = backlog;
+		sk->sk_state           = TCP_LISTEN;
+
+		err = 0;
+	}
+out:
+	release_sock(sk);
+
+	return err;
+}
+
+/*
+ * Function irda_bind (sock, uaddr, addr_len)
+ *
+ *    Used by servers to register their well known TSAP
+ *
+ */
+static int irda_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
+{
+	struct sock *sk = sock->sk;
+	struct sockaddr_irda *addr = (struct sockaddr_irda *) uaddr;
+	struct irda_sock *self = irda_sk(sk);
+	int err;
+
+	pr_debug("%s(%p)\n", __func__, self);
+
+	if (addr_len != sizeof(struct sockaddr_irda))
+		return -EINVAL;
+
+	lock_sock(sk);
+#ifdef CONFIG_IRDA_ULTRA
+	/* Special care for Ultra sockets */
+	if ((sk->sk_type == SOCK_DGRAM) &&
+	    (sk->sk_protocol == IRDAPROTO_ULTRA)) {
+		self->pid = addr->sir_lsap_sel;
+		err = -EOPNOTSUPP;
+		if (self->pid & 0x80) {
+			pr_debug("%s(), extension in PID not supp!\n",
+				 __func__);
+			goto out;
+		}
+		err = irda_open_lsap(self, self->pid);
+		if (err < 0)
+			goto out;
+
+		/* Pretend we are connected */
+		sock->state = SS_CONNECTED;
+		sk->sk_state   = TCP_ESTABLISHED;
+		err = 0;
+
+		goto out;
+	}
+#endif /* CONFIG_IRDA_ULTRA */
+
+	self->ias_obj = irias_new_object(addr->sir_name, jiffies);
+	err = -ENOMEM;
+	if (self->ias_obj == NULL)
+		goto out;
+
+	err = irda_open_tsap(self, addr->sir_lsap_sel, addr->sir_name);
+	if (err < 0) {
+		irias_delete_object(self->ias_obj);
+		self->ias_obj = NULL;
+		goto out;
+	}
+
+	/*  Register with LM-IAS */
+	irias_add_integer_attrib(self->ias_obj, "IrDA:TinyTP:LsapSel",
+				 self->stsap_sel, IAS_KERNEL_ATTR);
+	irias_insert_object(self->ias_obj);
+
+	err = 0;
+out:
+	release_sock(sk);
+	return err;
+}
+
+/*
+ * Function irda_accept (sock, newsock, flags)
+ *
+ *    Wait for incoming connection
+ *
+ */
+static int irda_accept(struct socket *sock, struct socket *newsock, int flags,
+		       bool kern)
+{
+	struct sock *sk = sock->sk;
+	struct irda_sock *new, *self = irda_sk(sk);
+	struct sock *newsk;
+	struct sk_buff *skb = NULL;
+	int err;
+
+	err = irda_create(sock_net(sk), newsock, sk->sk_protocol, kern);
+	if (err)
+		return err;
+
+	err = -EINVAL;
+
+	lock_sock(sk);
+	if (sock->state != SS_UNCONNECTED)
+		goto out;
+
+	err = -EOPNOTSUPP;
+	if ((sk->sk_type != SOCK_STREAM) && (sk->sk_type != SOCK_SEQPACKET) &&
+	    (sk->sk_type != SOCK_DGRAM))
+		goto out;
+
+	err = -EINVAL;
+	if (sk->sk_state != TCP_LISTEN)
+		goto out;
+
+	/*
+	 *	The read queue this time is holding sockets ready to use
+	 *	hooked into the SABM we saved
+	 */
+
+	/*
+	 * We can perform the accept only if there is incoming data
+	 * on the listening socket.
+	 * So, we will block the caller until we receive any data.
+	 * If the caller was waiting on select() or poll() before
+	 * calling us, the data is waiting for us ;-)
+	 * Jean II
+	 */
+	while (1) {
+		skb = skb_dequeue(&sk->sk_receive_queue);
+		if (skb)
+			break;
+
+		/* Non blocking operation */
+		err = -EWOULDBLOCK;
+		if (flags & O_NONBLOCK)
+			goto out;
+
+		err = wait_event_interruptible(*(sk_sleep(sk)),
+					skb_peek(&sk->sk_receive_queue));
+		if (err)
+			goto out;
+	}
+
+	newsk = newsock->sk;
+	err = -EIO;
+	if (newsk == NULL)
+		goto out;
+
+	newsk->sk_state = TCP_ESTABLISHED;
+
+	new = irda_sk(newsk);
+
+	/* Now attach up the new socket */
+	new->tsap = irttp_dup(self->tsap, new);
+	err = -EPERM; /* value does not seem to make sense. -arnd */
+	if (!new->tsap) {
+		pr_debug("%s(), dup failed!\n", __func__);
+		goto out;
+	}
+
+	new->stsap_sel = new->tsap->stsap_sel;
+	new->dtsap_sel = new->tsap->dtsap_sel;
+	new->saddr = irttp_get_saddr(new->tsap);
+	new->daddr = irttp_get_daddr(new->tsap);
+
+	new->max_sdu_size_tx = self->max_sdu_size_tx;
+	new->max_sdu_size_rx = self->max_sdu_size_rx;
+	new->max_data_size   = self->max_data_size;
+	new->max_header_size = self->max_header_size;
+
+	memcpy(&new->qos_tx, &self->qos_tx, sizeof(struct qos_info));
+
+	/* Clean up the original one to keep it in listen state */
+	irttp_listen(self->tsap);
+
+	sk->sk_ack_backlog--;
+
+	newsock->state = SS_CONNECTED;
+
+	irda_connect_response(new);
+	err = 0;
+out:
+	kfree_skb(skb);
+	release_sock(sk);
+	return err;
+}
+
+/*
+ * Function irda_connect (sock, uaddr, addr_len, flags)
+ *
+ *    Connect to a IrDA device
+ *
+ * The main difference with a "standard" connect is that with IrDA we need
+ * to resolve the service name into a TSAP selector (in TCP, port number
+ * doesn't have to be resolved).
+ * Because of this service name resolution, we can offer "auto-connect",
+ * where we connect to a service without specifying a destination address.
+ *
+ * Note : by consulting "errno", the user space caller may learn the cause
+ * of the failure. Most of them are visible in the function, others may come
+ * from subroutines called and are listed here :
+ *	o EBUSY : already processing a connect
+ *	o EHOSTUNREACH : bad addr->sir_addr argument
+ *	o EADDRNOTAVAIL : bad addr->sir_name argument
+ *	o ENOTUNIQ : more than one node has addr->sir_name (auto-connect)
+ *	o ENETUNREACH : no node found on the network (auto-connect)
+ */
+static int irda_connect(struct socket *sock, struct sockaddr *uaddr,
+			int addr_len, int flags)
+{
+	struct sock *sk = sock->sk;
+	struct sockaddr_irda *addr = (struct sockaddr_irda *) uaddr;
+	struct irda_sock *self = irda_sk(sk);
+	int err;
+
+	pr_debug("%s(%p)\n", __func__, self);
+
+	lock_sock(sk);
+	/* Don't allow connect for Ultra sockets */
+	err = -ESOCKTNOSUPPORT;
+	if ((sk->sk_type == SOCK_DGRAM) && (sk->sk_protocol == IRDAPROTO_ULTRA))
+		goto out;
+
+	if (sk->sk_state == TCP_ESTABLISHED && sock->state == SS_CONNECTING) {
+		sock->state = SS_CONNECTED;
+		err = 0;
+		goto out;   /* Connect completed during a ERESTARTSYS event */
+	}
+
+	if (sk->sk_state == TCP_CLOSE && sock->state == SS_CONNECTING) {
+		sock->state = SS_UNCONNECTED;
+		err = -ECONNREFUSED;
+		goto out;
+	}
+
+	err = -EISCONN;      /* No reconnect on a seqpacket socket */
+	if (sk->sk_state == TCP_ESTABLISHED)
+		goto out;
+
+	sk->sk_state   = TCP_CLOSE;
+	sock->state = SS_UNCONNECTED;
+
+	err = -EINVAL;
+	if (addr_len != sizeof(struct sockaddr_irda))
+		goto out;
+
+	/* Check if user supplied any destination device address */
+	if ((!addr->sir_addr) || (addr->sir_addr == DEV_ADDR_ANY)) {
+		/* Try to find one suitable */
+		err = irda_discover_daddr_and_lsap_sel(self, addr->sir_name);
+		if (err) {
+			pr_debug("%s(), auto-connect failed!\n", __func__);
+			goto out;
+		}
+	} else {
+		/* Use the one provided by the user */
+		self->daddr = addr->sir_addr;
+		pr_debug("%s(), daddr = %08x\n", __func__, self->daddr);
+
+		/* If we don't have a valid service name, we assume the
+		 * user want to connect on a specific LSAP. Prevent
+		 * the use of invalid LSAPs (IrLMP 1.1 p10). Jean II */
+		if((addr->sir_name[0] != '\0') ||
+		   (addr->sir_lsap_sel >= 0x70)) {
+			/* Query remote LM-IAS using service name */
+			err = irda_find_lsap_sel(self, addr->sir_name);
+			if (err) {
+				pr_debug("%s(), connect failed!\n", __func__);
+				goto out;
+			}
+		} else {
+			/* Directly connect to the remote LSAP
+			 * specified by the sir_lsap field.
+			 * Please use with caution, in IrDA LSAPs are
+			 * dynamic and there is no "well-known" LSAP. */
+			self->dtsap_sel = addr->sir_lsap_sel;
+		}
+	}
+
+	/* Check if we have opened a local TSAP */
+	if (!self->tsap) {
+		err = irda_open_tsap(self, LSAP_ANY, addr->sir_name);
+		if (err)
+			goto out;
+	}
+
+	/* Move to connecting socket, start sending Connect Requests */
+	sock->state = SS_CONNECTING;
+	sk->sk_state   = TCP_SYN_SENT;
+
+	/* Connect to remote device */
+	err = irttp_connect_request(self->tsap, self->dtsap_sel,
+				    self->saddr, self->daddr, NULL,
+				    self->max_sdu_size_rx, NULL);
+	if (err) {
+		pr_debug("%s(), connect failed!\n", __func__);
+		goto out;
+	}
+
+	/* Now the loop */
+	err = -EINPROGRESS;
+	if (sk->sk_state != TCP_ESTABLISHED && (flags & O_NONBLOCK))
+		goto out;
+
+	err = -ERESTARTSYS;
+	if (wait_event_interruptible(*(sk_sleep(sk)),
+				     (sk->sk_state != TCP_SYN_SENT)))
+		goto out;
+
+	if (sk->sk_state != TCP_ESTABLISHED) {
+		sock->state = SS_UNCONNECTED;
+		err = sock_error(sk);
+		if (!err)
+			err = -ECONNRESET;
+		goto out;
+	}
+
+	sock->state = SS_CONNECTED;
+
+	/* At this point, IrLMP has assigned our source address */
+	self->saddr = irttp_get_saddr(self->tsap);
+	err = 0;
+out:
+	release_sock(sk);
+	return err;
+}
+
+static struct proto irda_proto = {
+	.name	  = "IRDA",
+	.owner	  = THIS_MODULE,
+	.obj_size = sizeof(struct irda_sock),
+};
+
+/*
+ * Function irda_create (sock, protocol)
+ *
+ *    Create IrDA socket
+ *
+ */
+static int irda_create(struct net *net, struct socket *sock, int protocol,
+		       int kern)
+{
+	struct sock *sk;
+	struct irda_sock *self;
+
+	if (protocol < 0 || protocol > SK_PROTOCOL_MAX)
+		return -EINVAL;
+
+	if (net != &init_net)
+		return -EAFNOSUPPORT;
+
+	/* Check for valid socket type */
+	switch (sock->type) {
+	case SOCK_STREAM:     /* For TTP connections with SAR disabled */
+	case SOCK_SEQPACKET:  /* For TTP connections with SAR enabled */
+	case SOCK_DGRAM:      /* For TTP Unitdata or LMP Ultra transfers */
+		break;
+	default:
+		return -ESOCKTNOSUPPORT;
+	}
+
+	/* Allocate networking socket */
+	sk = sk_alloc(net, PF_IRDA, GFP_KERNEL, &irda_proto, kern);
+	if (sk == NULL)
+		return -ENOMEM;
+
+	self = irda_sk(sk);
+	pr_debug("%s() : self is %p\n", __func__, self);
+
+	init_waitqueue_head(&self->query_wait);
+
+	switch (sock->type) {
+	case SOCK_STREAM:
+		sock->ops = &irda_stream_ops;
+		self->max_sdu_size_rx = TTP_SAR_DISABLE;
+		break;
+	case SOCK_SEQPACKET:
+		sock->ops = &irda_seqpacket_ops;
+		self->max_sdu_size_rx = TTP_SAR_UNBOUND;
+		break;
+	case SOCK_DGRAM:
+		switch (protocol) {
+#ifdef CONFIG_IRDA_ULTRA
+		case IRDAPROTO_ULTRA:
+			sock->ops = &irda_ultra_ops;
+			/* Initialise now, because we may send on unbound
+			 * sockets. Jean II */
+			self->max_data_size = ULTRA_MAX_DATA - LMP_PID_HEADER;
+			self->max_header_size = IRDA_MAX_HEADER + LMP_PID_HEADER;
+			break;
+#endif /* CONFIG_IRDA_ULTRA */
+		case IRDAPROTO_UNITDATA:
+			sock->ops = &irda_dgram_ops;
+			/* We let Unitdata conn. be like seqpack conn. */
+			self->max_sdu_size_rx = TTP_SAR_UNBOUND;
+			break;
+		default:
+			sk_free(sk);
+			return -ESOCKTNOSUPPORT;
+		}
+		break;
+	default:
+		sk_free(sk);
+		return -ESOCKTNOSUPPORT;
+	}
+
+	/* Initialise networking socket struct */
+	sock_init_data(sock, sk);	/* Note : set sk->sk_refcnt to 1 */
+	sk->sk_family = PF_IRDA;
+	sk->sk_protocol = protocol;
+
+	/* Register as a client with IrLMP */
+	self->ckey = irlmp_register_client(0, NULL, NULL, NULL);
+	self->mask.word = 0xffff;
+	self->rx_flow = self->tx_flow = FLOW_START;
+	self->nslots = DISCOVERY_DEFAULT_SLOTS;
+	self->daddr = DEV_ADDR_ANY;	/* Until we get connected */
+	self->saddr = 0x0;		/* so IrLMP assign us any link */
+	return 0;
+}
+
+/*
+ * Function irda_destroy_socket (self)
+ *
+ *    Destroy socket
+ *
+ */
+static void irda_destroy_socket(struct irda_sock *self)
+{
+	pr_debug("%s(%p)\n", __func__, self);
+
+	/* Unregister with IrLMP */
+	irlmp_unregister_client(self->ckey);
+	irlmp_unregister_service(self->skey);
+
+	/* Unregister with LM-IAS */
+	if (self->ias_obj) {
+		irias_delete_object(self->ias_obj);
+		self->ias_obj = NULL;
+	}
+
+	if (self->iriap) {
+		iriap_close(self->iriap);
+		self->iriap = NULL;
+	}
+
+	if (self->tsap) {
+		irttp_disconnect_request(self->tsap, NULL, P_NORMAL);
+		irttp_close_tsap(self->tsap);
+		self->tsap = NULL;
+	}
+#ifdef CONFIG_IRDA_ULTRA
+	if (self->lsap) {
+		irlmp_close_lsap(self->lsap);
+		self->lsap = NULL;
+	}
+#endif /* CONFIG_IRDA_ULTRA */
+}
+
+/*
+ * Function irda_release (sock)
+ */
+static int irda_release(struct socket *sock)
+{
+	struct sock *sk = sock->sk;
+
+	if (sk == NULL)
+		return 0;
+
+	lock_sock(sk);
+	sk->sk_state       = TCP_CLOSE;
+	sk->sk_shutdown   |= SEND_SHUTDOWN;
+	sk->sk_state_change(sk);
+
+	/* Destroy IrDA socket */
+	irda_destroy_socket(irda_sk(sk));
+
+	sock_orphan(sk);
+	sock->sk   = NULL;
+	release_sock(sk);
+
+	/* Purge queues (see sock_init_data()) */
+	skb_queue_purge(&sk->sk_receive_queue);
+
+	/* Destroy networking socket if we are the last reference on it,
+	 * i.e. if(sk->sk_refcnt == 0) -> sk_free(sk) */
+	sock_put(sk);
+
+	/* Notes on socket locking and deallocation... - Jean II
+	 * In theory we should put pairs of sock_hold() / sock_put() to
+	 * prevent the socket to be destroyed whenever there is an
+	 * outstanding request or outstanding incoming packet or event.
+	 *
+	 * 1) This may include IAS request, both in connect and getsockopt.
+	 * Unfortunately, the situation is a bit more messy than it looks,
+	 * because we close iriap and kfree(self) above.
+	 *
+	 * 2) This may include selective discovery in getsockopt.
+	 * Same stuff as above, irlmp registration and self are gone.
+	 *
+	 * Probably 1 and 2 may not matter, because it's all triggered
+	 * by a process and the socket layer already prevent the
+	 * socket to go away while a process is holding it, through
+	 * sockfd_put() and fput()...
+	 *
+	 * 3) This may include deferred TSAP closure. In particular,
+	 * we may receive a late irda_disconnect_indication()
+	 * Fortunately, (tsap_cb *)->close_pend should protect us
+	 * from that.
+	 *
+	 * I did some testing on SMP, and it looks solid. And the socket
+	 * memory leak is now gone... - Jean II
+	 */
+
+	return 0;
+}
+
+/*
+ * Function irda_sendmsg (sock, msg, len)
+ *
+ *    Send message down to TinyTP. This function is used for both STREAM and
+ *    SEQPACK services. This is possible since it forces the client to
+ *    fragment the message if necessary
+ */
+static int irda_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
+{
+	struct sock *sk = sock->sk;
+	struct irda_sock *self;
+	struct sk_buff *skb;
+	int err = -EPIPE;
+
+	pr_debug("%s(), len=%zd\n", __func__, len);
+
+	/* Note : socket.c set MSG_EOR on SEQPACKET sockets */
+	if (msg->msg_flags & ~(MSG_DONTWAIT | MSG_EOR | MSG_CMSG_COMPAT |
+			       MSG_NOSIGNAL)) {
+		return -EINVAL;
+	}
+
+	lock_sock(sk);
+
+	if (sk->sk_shutdown & SEND_SHUTDOWN)
+		goto out_err;
+
+	if (sk->sk_state != TCP_ESTABLISHED) {
+		err = -ENOTCONN;
+		goto out;
+	}
+
+	self = irda_sk(sk);
+
+	/* Check if IrTTP is wants us to slow down */
+
+	if (wait_event_interruptible(*(sk_sleep(sk)),
+	    (self->tx_flow != FLOW_STOP  ||  sk->sk_state != TCP_ESTABLISHED))) {
+		err = -ERESTARTSYS;
+		goto out;
+	}
+
+	/* Check if we are still connected */
+	if (sk->sk_state != TCP_ESTABLISHED) {
+		err = -ENOTCONN;
+		goto out;
+	}
+
+	/* Check that we don't send out too big frames */
+	if (len > self->max_data_size) {
+		pr_debug("%s(), Chopping frame from %zd to %d bytes!\n",
+			 __func__, len, self->max_data_size);
+		len = self->max_data_size;
+	}
+
+	skb = sock_alloc_send_skb(sk, len + self->max_header_size + 16,
+				  msg->msg_flags & MSG_DONTWAIT, &err);
+	if (!skb)
+		goto out_err;
+
+	skb_reserve(skb, self->max_header_size + 16);
+	skb_reset_transport_header(skb);
+	skb_put(skb, len);
+	err = memcpy_from_msg(skb_transport_header(skb), msg, len);
+	if (err) {
+		kfree_skb(skb);
+		goto out_err;
+	}
+
+	/*
+	 * Just send the message to TinyTP, and let it deal with possible
+	 * errors. No need to duplicate all that here
+	 */
+	err = irttp_data_request(self->tsap, skb);
+	if (err) {
+		pr_debug("%s(), err=%d\n", __func__, err);
+		goto out_err;
+	}
+
+	release_sock(sk);
+	/* Tell client how much data we actually sent */
+	return len;
+
+out_err:
+	err = sk_stream_error(sk, msg->msg_flags, err);
+out:
+	release_sock(sk);
+	return err;
+
+}
+
+/*
+ * Function irda_recvmsg_dgram (sock, msg, size, flags)
+ *
+ *    Try to receive message and copy it to user. The frame is discarded
+ *    after being read, regardless of how much the user actually read
+ */
+static int irda_recvmsg_dgram(struct socket *sock, struct msghdr *msg,
+			      size_t size, int flags)
+{
+	struct sock *sk = sock->sk;
+	struct irda_sock *self = irda_sk(sk);
+	struct sk_buff *skb;
+	size_t copied;
+	int err;
+
+	skb = skb_recv_datagram(sk, flags & ~MSG_DONTWAIT,
+				flags & MSG_DONTWAIT, &err);
+	if (!skb)
+		return err;
+
+	skb_reset_transport_header(skb);
+	copied = skb->len;
+
+	if (copied > size) {
+		pr_debug("%s(), Received truncated frame (%zd < %zd)!\n",
+			 __func__, copied, size);
+		copied = size;
+		msg->msg_flags |= MSG_TRUNC;
+	}
+	skb_copy_datagram_msg(skb, 0, msg, copied);
+
+	skb_free_datagram(sk, skb);
+
+	/*
+	 *  Check if we have previously stopped IrTTP and we know
+	 *  have more free space in our rx_queue. If so tell IrTTP
+	 *  to start delivering frames again before our rx_queue gets
+	 *  empty
+	 */
+	if (self->rx_flow == FLOW_STOP) {
+		if ((atomic_read(&sk->sk_rmem_alloc) << 2) <= sk->sk_rcvbuf) {
+			pr_debug("%s(), Starting IrTTP\n", __func__);
+			self->rx_flow = FLOW_START;
+			irttp_flow_request(self->tsap, FLOW_START);
+		}
+	}
+
+	return copied;
+}
+
+/*
+ * Function irda_recvmsg_stream (sock, msg, size, flags)
+ */
+static int irda_recvmsg_stream(struct socket *sock, struct msghdr *msg,
+			       size_t size, int flags)
+{
+	struct sock *sk = sock->sk;
+	struct irda_sock *self = irda_sk(sk);
+	int noblock = flags & MSG_DONTWAIT;
+	size_t copied = 0;
+	int target, err;
+	long timeo;
+
+	if ((err = sock_error(sk)) < 0)
+		return err;
+
+	if (sock->flags & __SO_ACCEPTCON)
+		return -EINVAL;
+
+	err =-EOPNOTSUPP;
+	if (flags & MSG_OOB)
+		return -EOPNOTSUPP;
+
+	err = 0;
+	target = sock_rcvlowat(sk, flags & MSG_WAITALL, size);
+	timeo = sock_rcvtimeo(sk, noblock);
+
+	do {
+		int chunk;
+		struct sk_buff *skb = skb_dequeue(&sk->sk_receive_queue);
+
+		if (skb == NULL) {
+			DEFINE_WAIT(wait);
+			err = 0;
+
+			if (copied >= target)
+				break;
+
+			prepare_to_wait_exclusive(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
+
+			/*
+			 *	POSIX 1003.1g mandates this order.
+			 */
+			err = sock_error(sk);
+			if (err)
+				;
+			else if (sk->sk_shutdown & RCV_SHUTDOWN)
+				;
+			else if (noblock)
+				err = -EAGAIN;
+			else if (signal_pending(current))
+				err = sock_intr_errno(timeo);
+			else if (sk->sk_state != TCP_ESTABLISHED)
+				err = -ENOTCONN;
+			else if (skb_peek(&sk->sk_receive_queue) == NULL)
+				/* Wait process until data arrives */
+				schedule();
+
+			finish_wait(sk_sleep(sk), &wait);
+
+			if (err)
+				return err;
+			if (sk->sk_shutdown & RCV_SHUTDOWN)
+				break;
+
+			continue;
+		}
+
+		chunk = min_t(unsigned int, skb->len, size);
+		if (memcpy_to_msg(msg, skb->data, chunk)) {
+			skb_queue_head(&sk->sk_receive_queue, skb);
+			if (copied == 0)
+				copied = -EFAULT;
+			break;
+		}
+		copied += chunk;
+		size -= chunk;
+
+		/* Mark read part of skb as used */
+		if (!(flags & MSG_PEEK)) {
+			skb_pull(skb, chunk);
+
+			/* put the skb back if we didn't use it up.. */
+			if (skb->len) {
+				pr_debug("%s(), back on q!\n",
+					 __func__);
+				skb_queue_head(&sk->sk_receive_queue, skb);
+				break;
+			}
+
+			kfree_skb(skb);
+		} else {
+			pr_debug("%s() questionable!?\n", __func__);
+
+			/* put message back and return */
+			skb_queue_head(&sk->sk_receive_queue, skb);
+			break;
+		}
+	} while (size);
+
+	/*
+	 *  Check if we have previously stopped IrTTP and we know
+	 *  have more free space in our rx_queue. If so tell IrTTP
+	 *  to start delivering frames again before our rx_queue gets
+	 *  empty
+	 */
+	if (self->rx_flow == FLOW_STOP) {
+		if ((atomic_read(&sk->sk_rmem_alloc) << 2) <= sk->sk_rcvbuf) {
+			pr_debug("%s(), Starting IrTTP\n", __func__);
+			self->rx_flow = FLOW_START;
+			irttp_flow_request(self->tsap, FLOW_START);
+		}
+	}
+
+	return copied;
+}
+
+/*
+ * Function irda_sendmsg_dgram (sock, msg, len)
+ *
+ *    Send message down to TinyTP for the unreliable sequenced
+ *    packet service...
+ *
+ */
+static int irda_sendmsg_dgram(struct socket *sock, struct msghdr *msg,
+			      size_t len)
+{
+	struct sock *sk = sock->sk;
+	struct irda_sock *self;
+	struct sk_buff *skb;
+	int err;
+
+	pr_debug("%s(), len=%zd\n", __func__, len);
+
+	if (msg->msg_flags & ~(MSG_DONTWAIT|MSG_CMSG_COMPAT))
+		return -EINVAL;
+
+	lock_sock(sk);
+
+	if (sk->sk_shutdown & SEND_SHUTDOWN) {
+		send_sig(SIGPIPE, current, 0);
+		err = -EPIPE;
+		goto out;
+	}
+
+	err = -ENOTCONN;
+	if (sk->sk_state != TCP_ESTABLISHED)
+		goto out;
+
+	self = irda_sk(sk);
+
+	/*
+	 * Check that we don't send out too big frames. This is an unreliable
+	 * service, so we have no fragmentation and no coalescence
+	 */
+	if (len > self->max_data_size) {
+		pr_debug("%s(), Warning too much data! Chopping frame from %zd to %d bytes!\n",
+			 __func__, len, self->max_data_size);
+		len = self->max_data_size;
+	}
+
+	skb = sock_alloc_send_skb(sk, len + self->max_header_size,
+				  msg->msg_flags & MSG_DONTWAIT, &err);
+	err = -ENOBUFS;
+	if (!skb)
+		goto out;
+
+	skb_reserve(skb, self->max_header_size);
+	skb_reset_transport_header(skb);
+
+	pr_debug("%s(), appending user data\n", __func__);
+	skb_put(skb, len);
+	err = memcpy_from_msg(skb_transport_header(skb), msg, len);
+	if (err) {
+		kfree_skb(skb);
+		goto out;
+	}
+
+	/*
+	 * Just send the message to TinyTP, and let it deal with possible
+	 * errors. No need to duplicate all that here
+	 */
+	err = irttp_udata_request(self->tsap, skb);
+	if (err) {
+		pr_debug("%s(), err=%d\n", __func__, err);
+		goto out;
+	}
+
+	release_sock(sk);
+	return len;
+
+out:
+	release_sock(sk);
+	return err;
+}
+
+/*
+ * Function irda_sendmsg_ultra (sock, msg, len)
+ *
+ *    Send message down to IrLMP for the unreliable Ultra
+ *    packet service...
+ */
+#ifdef CONFIG_IRDA_ULTRA
+static int irda_sendmsg_ultra(struct socket *sock, struct msghdr *msg,
+			      size_t len)
+{
+	struct sock *sk = sock->sk;
+	struct irda_sock *self;
+	__u8 pid = 0;
+	int bound = 0;
+	struct sk_buff *skb;
+	int err;
+
+	pr_debug("%s(), len=%zd\n", __func__, len);
+
+	err = -EINVAL;
+	if (msg->msg_flags & ~(MSG_DONTWAIT|MSG_CMSG_COMPAT))
+		return -EINVAL;
+
+	lock_sock(sk);
+
+	err = -EPIPE;
+	if (sk->sk_shutdown & SEND_SHUTDOWN) {
+		send_sig(SIGPIPE, current, 0);
+		goto out;
+	}
+
+	self = irda_sk(sk);
+
+	/* Check if an address was specified with sendto. Jean II */
+	if (msg->msg_name) {
+		DECLARE_SOCKADDR(struct sockaddr_irda *, addr, msg->msg_name);
+		err = -EINVAL;
+		/* Check address, extract pid. Jean II */
+		if (msg->msg_namelen < sizeof(*addr))
+			goto out;
+		if (addr->sir_family != AF_IRDA)
+			goto out;
+
+		pid = addr->sir_lsap_sel;
+		if (pid & 0x80) {
+			pr_debug("%s(), extension in PID not supp!\n",
+				 __func__);
+			err = -EOPNOTSUPP;
+			goto out;
+		}
+	} else {
+		/* Check that the socket is properly bound to an Ultra
+		 * port. Jean II */
+		if ((self->lsap == NULL) ||
+		    (sk->sk_state != TCP_ESTABLISHED)) {
+			pr_debug("%s(), socket not bound to Ultra PID.\n",
+				 __func__);
+			err = -ENOTCONN;
+			goto out;
+		}
+		/* Use PID from socket */
+		bound = 1;
+	}
+
+	/*
+	 * Check that we don't send out too big frames. This is an unreliable
+	 * service, so we have no fragmentation and no coalescence
+	 */
+	if (len > self->max_data_size) {
+		pr_debug("%s(), Warning too much data! Chopping frame from %zd to %d bytes!\n",
+			 __func__, len, self->max_data_size);
+		len = self->max_data_size;
+	}
+
+	skb = sock_alloc_send_skb(sk, len + self->max_header_size,
+				  msg->msg_flags & MSG_DONTWAIT, &err);
+	err = -ENOBUFS;
+	if (!skb)
+		goto out;
+
+	skb_reserve(skb, self->max_header_size);
+	skb_reset_transport_header(skb);
+
+	pr_debug("%s(), appending user data\n", __func__);
+	skb_put(skb, len);
+	err = memcpy_from_msg(skb_transport_header(skb), msg, len);
+	if (err) {
+		kfree_skb(skb);
+		goto out;
+	}
+
+	err = irlmp_connless_data_request((bound ? self->lsap : NULL),
+					  skb, pid);
+	if (err)
+		pr_debug("%s(), err=%d\n", __func__, err);
+out:
+	release_sock(sk);
+	return err ? : len;
+}
+#endif /* CONFIG_IRDA_ULTRA */
+
+/*
+ * Function irda_shutdown (sk, how)
+ */
+static int irda_shutdown(struct socket *sock, int how)
+{
+	struct sock *sk = sock->sk;
+	struct irda_sock *self = irda_sk(sk);
+
+	pr_debug("%s(%p)\n", __func__, self);
+
+	lock_sock(sk);
+
+	sk->sk_state       = TCP_CLOSE;
+	sk->sk_shutdown   |= SEND_SHUTDOWN;
+	sk->sk_state_change(sk);
+
+	if (self->iriap) {
+		iriap_close(self->iriap);
+		self->iriap = NULL;
+	}
+
+	if (self->tsap) {
+		irttp_disconnect_request(self->tsap, NULL, P_NORMAL);
+		irttp_close_tsap(self->tsap);
+		self->tsap = NULL;
+	}
+
+	/* A few cleanup so the socket look as good as new... */
+	self->rx_flow = self->tx_flow = FLOW_START;	/* needed ??? */
+	self->daddr = DEV_ADDR_ANY;	/* Until we get re-connected */
+	self->saddr = 0x0;		/* so IrLMP assign us any link */
+
+	release_sock(sk);
+
+	return 0;
+}
+
+/*
+ * Function irda_poll (file, sock, wait)
+ */
+static unsigned int irda_poll(struct file * file, struct socket *sock,
+			      poll_table *wait)
+{
+	struct sock *sk = sock->sk;
+	struct irda_sock *self = irda_sk(sk);
+	unsigned int mask;
+
+	poll_wait(file, sk_sleep(sk), wait);
+	mask = 0;
+
+	/* Exceptional events? */
+	if (sk->sk_err)
+		mask |= POLLERR;
+	if (sk->sk_shutdown & RCV_SHUTDOWN) {
+		pr_debug("%s(), POLLHUP\n", __func__);
+		mask |= POLLHUP;
+	}
+
+	/* Readable? */
+	if (!skb_queue_empty(&sk->sk_receive_queue)) {
+		pr_debug("Socket is readable\n");
+		mask |= POLLIN | POLLRDNORM;
+	}
+
+	/* Connection-based need to check for termination and startup */
+	switch (sk->sk_type) {
+	case SOCK_STREAM:
+		if (sk->sk_state == TCP_CLOSE) {
+			pr_debug("%s(), POLLHUP\n", __func__);
+			mask |= POLLHUP;
+		}
+
+		if (sk->sk_state == TCP_ESTABLISHED) {
+			if ((self->tx_flow == FLOW_START) &&
+			    sock_writeable(sk))
+			{
+				mask |= POLLOUT | POLLWRNORM | POLLWRBAND;
+			}
+		}
+		break;
+	case SOCK_SEQPACKET:
+		if ((self->tx_flow == FLOW_START) &&
+		    sock_writeable(sk))
+		{
+			mask |= POLLOUT | POLLWRNORM | POLLWRBAND;
+		}
+		break;
+	case SOCK_DGRAM:
+		if (sock_writeable(sk))
+			mask |= POLLOUT | POLLWRNORM | POLLWRBAND;
+		break;
+	default:
+		break;
+	}
+
+	return mask;
+}
+
+/*
+ * Function irda_ioctl (sock, cmd, arg)
+ */
+static int irda_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
+{
+	struct sock *sk = sock->sk;
+	int err;
+
+	pr_debug("%s(), cmd=%#x\n", __func__, cmd);
+
+	err = -EINVAL;
+	switch (cmd) {
+	case TIOCOUTQ: {
+		long amount;
+
+		amount = sk->sk_sndbuf - sk_wmem_alloc_get(sk);
+		if (amount < 0)
+			amount = 0;
+		err = put_user(amount, (unsigned int __user *)arg);
+		break;
+	}
+
+	case TIOCINQ: {
+		struct sk_buff *skb;
+		long amount = 0L;
+		/* These two are safe on a single CPU system as only user tasks fiddle here */
+		if ((skb = skb_peek(&sk->sk_receive_queue)) != NULL)
+			amount = skb->len;
+		err = put_user(amount, (unsigned int __user *)arg);
+		break;
+	}
+
+	case SIOCGSTAMP:
+		if (sk != NULL)
+			err = sock_get_timestamp(sk, (struct timeval __user *)arg);
+		break;
+
+	case SIOCGIFADDR:
+	case SIOCSIFADDR:
+	case SIOCGIFDSTADDR:
+	case SIOCSIFDSTADDR:
+	case SIOCGIFBRDADDR:
+	case SIOCSIFBRDADDR:
+	case SIOCGIFNETMASK:
+	case SIOCSIFNETMASK:
+	case SIOCGIFMETRIC:
+	case SIOCSIFMETRIC:
+		break;
+	default:
+		pr_debug("%s(), doing device ioctl!\n", __func__);
+		err = -ENOIOCTLCMD;
+	}
+
+	return err;
+}
+
+#ifdef CONFIG_COMPAT
+/*
+ * Function irda_ioctl (sock, cmd, arg)
+ */
+static int irda_compat_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
+{
+	/*
+	 * All IRDA's ioctl are standard ones.
+	 */
+	return -ENOIOCTLCMD;
+}
+#endif
+
+/*
+ * Function irda_setsockopt (sock, level, optname, optval, optlen)
+ *
+ *    Set some options for the socket
+ *
+ */
+static int irda_setsockopt(struct socket *sock, int level, int optname,
+			   char __user *optval, unsigned int optlen)
+{
+	struct sock *sk = sock->sk;
+	struct irda_sock *self = irda_sk(sk);
+	struct irda_ias_set    *ias_opt;
+	struct ias_object      *ias_obj;
+	struct ias_attrib *	ias_attr;	/* Attribute in IAS object */
+	int opt, free_ias = 0, err = 0;
+
+	pr_debug("%s(%p)\n", __func__, self);
+
+	if (level != SOL_IRLMP)
+		return -ENOPROTOOPT;
+
+	lock_sock(sk);
+
+	switch (optname) {
+	case IRLMP_IAS_SET:
+		/* The user want to add an attribute to an existing IAS object
+		 * (in the IAS database) or to create a new object with this
+		 * attribute.
+		 * We first query IAS to know if the object exist, and then
+		 * create the right attribute...
+		 */
+
+		if (optlen != sizeof(struct irda_ias_set)) {
+			err = -EINVAL;
+			goto out;
+		}
+
+		/* Copy query to the driver. */
+		ias_opt = memdup_user(optval, optlen);
+		if (IS_ERR(ias_opt)) {
+			err = PTR_ERR(ias_opt);
+			goto out;
+		}
+
+		/* Find the object we target.
+		 * If the user gives us an empty string, we use the object
+		 * associated with this socket. This will workaround
+		 * duplicated class name - Jean II */
+		if(ias_opt->irda_class_name[0] == '\0') {
+			if(self->ias_obj == NULL) {
+				kfree(ias_opt);
+				err = -EINVAL;
+				goto out;
+			}
+			ias_obj = self->ias_obj;
+		} else
+			ias_obj = irias_find_object(ias_opt->irda_class_name);
+
+		/* Only ROOT can mess with the global IAS database.
+		 * Users can only add attributes to the object associated
+		 * with the socket they own - Jean II */
+		if((!capable(CAP_NET_ADMIN)) &&
+		   ((ias_obj == NULL) || (ias_obj != self->ias_obj))) {
+			kfree(ias_opt);
+			err = -EPERM;
+			goto out;
+		}
+
+		/* If the object doesn't exist, create it */
+		if(ias_obj == (struct ias_object *) NULL) {
+			/* Create a new object */
+			ias_obj = irias_new_object(ias_opt->irda_class_name,
+						   jiffies);
+			if (ias_obj == NULL) {
+				kfree(ias_opt);
+				err = -ENOMEM;
+				goto out;
+			}
+			free_ias = 1;
+		}
+
+		/* Do we have the attribute already ? */
+		if(irias_find_attrib(ias_obj, ias_opt->irda_attrib_name)) {
+			kfree(ias_opt);
+			if (free_ias) {
+				kfree(ias_obj->name);
+				kfree(ias_obj);
+			}
+			err = -EINVAL;
+			goto out;
+		}
+
+		/* Look at the type */
+		switch(ias_opt->irda_attrib_type) {
+		case IAS_INTEGER:
+			/* Add an integer attribute */
+			irias_add_integer_attrib(
+				ias_obj,
+				ias_opt->irda_attrib_name,
+				ias_opt->attribute.irda_attrib_int,
+				IAS_USER_ATTR);
+			break;
+		case IAS_OCT_SEQ:
+			/* Check length */
+			if(ias_opt->attribute.irda_attrib_octet_seq.len >
+			   IAS_MAX_OCTET_STRING) {
+				kfree(ias_opt);
+				if (free_ias) {
+					kfree(ias_obj->name);
+					kfree(ias_obj);
+				}
+
+				err = -EINVAL;
+				goto out;
+			}
+			/* Add an octet sequence attribute */
+			irias_add_octseq_attrib(
+			      ias_obj,
+			      ias_opt->irda_attrib_name,
+			      ias_opt->attribute.irda_attrib_octet_seq.octet_seq,
+			      ias_opt->attribute.irda_attrib_octet_seq.len,
+			      IAS_USER_ATTR);
+			break;
+		case IAS_STRING:
+			/* Should check charset & co */
+			/* Check length */
+			/* The length is encoded in a __u8, and
+			 * IAS_MAX_STRING == 256, so there is no way
+			 * userspace can pass us a string too large.
+			 * Jean II */
+			/* NULL terminate the string (avoid troubles) */
+			ias_opt->attribute.irda_attrib_string.string[ias_opt->attribute.irda_attrib_string.len] = '\0';
+			/* Add a string attribute */
+			irias_add_string_attrib(
+				ias_obj,
+				ias_opt->irda_attrib_name,
+				ias_opt->attribute.irda_attrib_string.string,
+				IAS_USER_ATTR);
+			break;
+		default :
+			kfree(ias_opt);
+			if (free_ias) {
+				kfree(ias_obj->name);
+				kfree(ias_obj);
+			}
+			err = -EINVAL;
+			goto out;
+		}
+		irias_insert_object(ias_obj);
+		kfree(ias_opt);
+		break;
+	case IRLMP_IAS_DEL:
+		/* The user want to delete an object from our local IAS
+		 * database. We just need to query the IAS, check is the
+		 * object is not owned by the kernel and delete it.
+		 */
+
+		if (optlen != sizeof(struct irda_ias_set)) {
+			err = -EINVAL;
+			goto out;
+		}
+
+		/* Copy query to the driver. */
+		ias_opt = memdup_user(optval, optlen);
+		if (IS_ERR(ias_opt)) {
+			err = PTR_ERR(ias_opt);
+			goto out;
+		}
+
+		/* Find the object we target.
+		 * If the user gives us an empty string, we use the object
+		 * associated with this socket. This will workaround
+		 * duplicated class name - Jean II */
+		if(ias_opt->irda_class_name[0] == '\0')
+			ias_obj = self->ias_obj;
+		else
+			ias_obj = irias_find_object(ias_opt->irda_class_name);
+		if(ias_obj == (struct ias_object *) NULL) {
+			kfree(ias_opt);
+			err = -EINVAL;
+			goto out;
+		}
+
+		/* Only ROOT can mess with the global IAS database.
+		 * Users can only del attributes from the object associated
+		 * with the socket they own - Jean II */
+		if((!capable(CAP_NET_ADMIN)) &&
+		   ((ias_obj == NULL) || (ias_obj != self->ias_obj))) {
+			kfree(ias_opt);
+			err = -EPERM;
+			goto out;
+		}
+
+		/* Find the attribute (in the object) we target */
+		ias_attr = irias_find_attrib(ias_obj,
+					     ias_opt->irda_attrib_name);
+		if(ias_attr == (struct ias_attrib *) NULL) {
+			kfree(ias_opt);
+			err = -EINVAL;
+			goto out;
+		}
+
+		/* Check is the user space own the object */
+		if(ias_attr->value->owner != IAS_USER_ATTR) {
+			pr_debug("%s(), attempting to delete a kernel attribute\n",
+				 __func__);
+			kfree(ias_opt);
+			err = -EPERM;
+			goto out;
+		}
+
+		/* Remove the attribute (and maybe the object) */
+		irias_delete_attrib(ias_obj, ias_attr, 1);
+		kfree(ias_opt);
+		break;
+	case IRLMP_MAX_SDU_SIZE:
+		if (optlen < sizeof(int)) {
+			err = -EINVAL;
+			goto out;
+		}
+
+		if (get_user(opt, (int __user *)optval)) {
+			err = -EFAULT;
+			goto out;
+		}
+
+		/* Only possible for a seqpacket service (TTP with SAR) */
+		if (sk->sk_type != SOCK_SEQPACKET) {
+			pr_debug("%s(), setting max_sdu_size = %d\n",
+				 __func__, opt);
+			self->max_sdu_size_rx = opt;
+		} else {
+			net_warn_ratelimited("%s: not allowed to set MAXSDUSIZE for this socket type!\n",
+					     __func__);
+			err = -ENOPROTOOPT;
+			goto out;
+		}
+		break;
+	case IRLMP_HINTS_SET:
+		if (optlen < sizeof(int)) {
+			err = -EINVAL;
+			goto out;
+		}
+
+		/* The input is really a (__u8 hints[2]), easier as an int */
+		if (get_user(opt, (int __user *)optval)) {
+			err = -EFAULT;
+			goto out;
+		}
+
+		/* Unregister any old registration */
+		irlmp_unregister_service(self->skey);
+
+		self->skey = irlmp_register_service((__u16) opt);
+		break;
+	case IRLMP_HINT_MASK_SET:
+		/* As opposed to the previous case which set the hint bits
+		 * that we advertise, this one set the filter we use when
+		 * making a discovery (nodes which don't match any hint
+		 * bit in the mask are not reported).
+		 */
+		if (optlen < sizeof(int)) {
+			err = -EINVAL;
+			goto out;
+		}
+
+		/* The input is really a (__u8 hints[2]), easier as an int */
+		if (get_user(opt, (int __user *)optval)) {
+			err = -EFAULT;
+			goto out;
+		}
+
+		/* Set the new hint mask */
+		self->mask.word = (__u16) opt;
+		/* Mask out extension bits */
+		self->mask.word &= 0x7f7f;
+		/* Check if no bits */
+		if(!self->mask.word)
+			self->mask.word = 0xFFFF;
+
+		break;
+	default:
+		err = -ENOPROTOOPT;
+		break;
+	}
+
+out:
+	release_sock(sk);
+
+	return err;
+}
+
+/*
+ * Function irda_extract_ias_value(ias_opt, ias_value)
+ *
+ *    Translate internal IAS value structure to the user space representation
+ *
+ * The external representation of IAS values, as we exchange them with
+ * user space program is quite different from the internal representation,
+ * as stored in the IAS database (because we need a flat structure for
+ * crossing kernel boundary).
+ * This function transform the former in the latter. We also check
+ * that the value type is valid.
+ */
+static int irda_extract_ias_value(struct irda_ias_set *ias_opt,
+				  struct ias_value *ias_value)
+{
+	/* Look at the type */
+	switch (ias_value->type) {
+	case IAS_INTEGER:
+		/* Copy the integer */
+		ias_opt->attribute.irda_attrib_int = ias_value->t.integer;
+		break;
+	case IAS_OCT_SEQ:
+		/* Set length */
+		ias_opt->attribute.irda_attrib_octet_seq.len = ias_value->len;
+		/* Copy over */
+		memcpy(ias_opt->attribute.irda_attrib_octet_seq.octet_seq,
+		       ias_value->t.oct_seq, ias_value->len);
+		break;
+	case IAS_STRING:
+		/* Set length */
+		ias_opt->attribute.irda_attrib_string.len = ias_value->len;
+		ias_opt->attribute.irda_attrib_string.charset = ias_value->charset;
+		/* Copy over */
+		memcpy(ias_opt->attribute.irda_attrib_string.string,
+		       ias_value->t.string, ias_value->len);
+		/* NULL terminate the string (avoid troubles) */
+		ias_opt->attribute.irda_attrib_string.string[ias_value->len] = '\0';
+		break;
+	case IAS_MISSING:
+	default :
+		return -EINVAL;
+	}
+
+	/* Copy type over */
+	ias_opt->irda_attrib_type = ias_value->type;
+
+	return 0;
+}
+
+/*
+ * Function irda_getsockopt (sock, level, optname, optval, optlen)
+ */
+static int irda_getsockopt(struct socket *sock, int level, int optname,
+			   char __user *optval, int __user *optlen)
+{
+	struct sock *sk = sock->sk;
+	struct irda_sock *self = irda_sk(sk);
+	struct irda_device_list list = { 0 };
+	struct irda_device_info *discoveries;
+	struct irda_ias_set *	ias_opt;	/* IAS get/query params */
+	struct ias_object *	ias_obj;	/* Object in IAS */
+	struct ias_attrib *	ias_attr;	/* Attribute in IAS object */
+	int daddr = DEV_ADDR_ANY;	/* Dest address for IAS queries */
+	int val = 0;
+	int len = 0;
+	int err = 0;
+	int offset, total;
+
+	pr_debug("%s(%p)\n", __func__, self);
+
+	if (level != SOL_IRLMP)
+		return -ENOPROTOOPT;
+
+	if (get_user(len, optlen))
+		return -EFAULT;
+
+	if(len < 0)
+		return -EINVAL;
+
+	lock_sock(sk);
+
+	switch (optname) {
+	case IRLMP_ENUMDEVICES:
+
+		/* Offset to first device entry */
+		offset = sizeof(struct irda_device_list) -
+			sizeof(struct irda_device_info);
+
+		if (len < offset) {
+			err = -EINVAL;
+			goto out;
+		}
+
+		/* Ask lmp for the current discovery log */
+		discoveries = irlmp_get_discoveries(&list.len, self->mask.word,
+						    self->nslots);
+		/* Check if the we got some results */
+		if (discoveries == NULL) {
+			err = -EAGAIN;
+			goto out;		/* Didn't find any devices */
+		}
+
+		/* Write total list length back to client */
+		if (copy_to_user(optval, &list, offset))
+			err = -EFAULT;
+
+		/* Copy the list itself - watch for overflow */
+		if (list.len > 2048) {
+			err = -EINVAL;
+			goto bed;
+		}
+		total = offset + (list.len * sizeof(struct irda_device_info));
+		if (total > len)
+			total = len;
+		if (copy_to_user(optval+offset, discoveries, total - offset))
+			err = -EFAULT;
+
+		/* Write total number of bytes used back to client */
+		if (put_user(total, optlen))
+			err = -EFAULT;
+bed:
+		/* Free up our buffer */
+		kfree(discoveries);
+		break;
+	case IRLMP_MAX_SDU_SIZE:
+		val = self->max_data_size;
+		len = sizeof(int);
+		if (put_user(len, optlen)) {
+			err = -EFAULT;
+			goto out;
+		}
+
+		if (copy_to_user(optval, &val, len)) {
+			err = -EFAULT;
+			goto out;
+		}
+
+		break;
+	case IRLMP_IAS_GET:
+		/* The user want an object from our local IAS database.
+		 * We just need to query the IAS and return the value
+		 * that we found */
+
+		/* Check that the user has allocated the right space for us */
+		if (len != sizeof(struct irda_ias_set)) {
+			err = -EINVAL;
+			goto out;
+		}
+
+		/* Copy query to the driver. */
+		ias_opt = memdup_user(optval, len);
+		if (IS_ERR(ias_opt)) {
+			err = PTR_ERR(ias_opt);
+			goto out;
+		}
+
+		/* Find the object we target.
+		 * If the user gives us an empty string, we use the object
+		 * associated with this socket. This will workaround
+		 * duplicated class name - Jean II */
+		if(ias_opt->irda_class_name[0] == '\0')
+			ias_obj = self->ias_obj;
+		else
+			ias_obj = irias_find_object(ias_opt->irda_class_name);
+		if(ias_obj == (struct ias_object *) NULL) {
+			kfree(ias_opt);
+			err = -EINVAL;
+			goto out;
+		}
+
+		/* Find the attribute (in the object) we target */
+		ias_attr = irias_find_attrib(ias_obj,
+					     ias_opt->irda_attrib_name);
+		if(ias_attr == (struct ias_attrib *) NULL) {
+			kfree(ias_opt);
+			err = -EINVAL;
+			goto out;
+		}
+
+		/* Translate from internal to user structure */
+		err = irda_extract_ias_value(ias_opt, ias_attr->value);
+		if(err) {
+			kfree(ias_opt);
+			goto out;
+		}
+
+		/* Copy reply to the user */
+		if (copy_to_user(optval, ias_opt,
+				 sizeof(struct irda_ias_set))) {
+			kfree(ias_opt);
+			err = -EFAULT;
+			goto out;
+		}
+		/* Note : don't need to put optlen, we checked it */
+		kfree(ias_opt);
+		break;
+	case IRLMP_IAS_QUERY:
+		/* The user want an object from a remote IAS database.
+		 * We need to use IAP to query the remote database and
+		 * then wait for the answer to come back. */
+
+		/* Check that the user has allocated the right space for us */
+		if (len != sizeof(struct irda_ias_set)) {
+			err = -EINVAL;
+			goto out;
+		}
+
+		/* Copy query to the driver. */
+		ias_opt = memdup_user(optval, len);
+		if (IS_ERR(ias_opt)) {
+			err = PTR_ERR(ias_opt);
+			goto out;
+		}
+
+		/* At this point, there are two cases...
+		 * 1) the socket is connected - that's the easy case, we
+		 *	just query the device we are connected to...
+		 * 2) the socket is not connected - the user doesn't want
+		 *	to connect and/or may not have a valid service name
+		 *	(so can't create a fake connection). In this case,
+		 *	we assume that the user pass us a valid destination
+		 *	address in the requesting structure...
+		 */
+		if(self->daddr != DEV_ADDR_ANY) {
+			/* We are connected - reuse known daddr */
+			daddr = self->daddr;
+		} else {
+			/* We are not connected, we must specify a valid
+			 * destination address */
+			daddr = ias_opt->daddr;
+			if((!daddr) || (daddr == DEV_ADDR_ANY)) {
+				kfree(ias_opt);
+				err = -EINVAL;
+				goto out;
+			}
+		}
+
+		/* Check that we can proceed with IAP */
+		if (self->iriap) {
+			net_warn_ratelimited("%s: busy with a previous query\n",
+					     __func__);
+			kfree(ias_opt);
+			err = -EBUSY;
+			goto out;
+		}
+
+		self->iriap = iriap_open(LSAP_ANY, IAS_CLIENT, self,
+					 irda_getvalue_confirm);
+
+		if (self->iriap == NULL) {
+			kfree(ias_opt);
+			err = -ENOMEM;
+			goto out;
+		}
+
+		/* Treat unexpected wakeup as disconnect */
+		self->errno = -EHOSTUNREACH;
+
+		/* Query remote LM-IAS */
+		iriap_getvaluebyclass_request(self->iriap,
+					      self->saddr, daddr,
+					      ias_opt->irda_class_name,
+					      ias_opt->irda_attrib_name);
+
+		/* Wait for answer, if not yet finished (or failed) */
+		if (wait_event_interruptible(self->query_wait,
+					     (self->iriap == NULL))) {
+			/* pending request uses copy of ias_opt-content
+			 * we can free it regardless! */
+			kfree(ias_opt);
+			/* Treat signals as disconnect */
+			err = -EHOSTUNREACH;
+			goto out;
+		}
+
+		/* Check what happened */
+		if (self->errno)
+		{
+			kfree(ias_opt);
+			/* Requested object/attribute doesn't exist */
+			if((self->errno == IAS_CLASS_UNKNOWN) ||
+			   (self->errno == IAS_ATTRIB_UNKNOWN))
+				err = -EADDRNOTAVAIL;
+			else
+				err = -EHOSTUNREACH;
+
+			goto out;
+		}
+
+		/* Translate from internal to user structure */
+		err = irda_extract_ias_value(ias_opt, self->ias_result);
+		if (self->ias_result)
+			irias_delete_value(self->ias_result);
+		if (err) {
+			kfree(ias_opt);
+			goto out;
+		}
+
+		/* Copy reply to the user */
+		if (copy_to_user(optval, ias_opt,
+				 sizeof(struct irda_ias_set))) {
+			kfree(ias_opt);
+			err = -EFAULT;
+			goto out;
+		}
+		/* Note : don't need to put optlen, we checked it */
+		kfree(ias_opt);
+		break;
+	case IRLMP_WAITDEVICE:
+		/* This function is just another way of seeing life ;-)
+		 * IRLMP_ENUMDEVICES assumes that you have a static network,
+		 * and that you just want to pick one of the devices present.
+		 * On the other hand, in here we assume that no device is
+		 * present and that at some point in the future a device will
+		 * come into range. When this device arrive, we just wake
+		 * up the caller, so that he has time to connect to it before
+		 * the device goes away...
+		 * Note : once the node has been discovered for more than a
+		 * few second, it won't trigger this function, unless it
+		 * goes away and come back changes its hint bits (so we
+		 * might call it IRLMP_WAITNEWDEVICE).
+		 */
+
+		/* Check that the user is passing us an int */
+		if (len != sizeof(int)) {
+			err = -EINVAL;
+			goto out;
+		}
+		/* Get timeout in ms (max time we block the caller) */
+		if (get_user(val, (int __user *)optval)) {
+			err = -EFAULT;
+			goto out;
+		}
+
+		/* Tell IrLMP we want to be notified */
+		irlmp_update_client(self->ckey, self->mask.word,
+				    irda_selective_discovery_indication,
+				    NULL, (void *) self);
+
+		/* Do some discovery (and also return cached results) */
+		irlmp_discovery_request(self->nslots);
+
+		/* Wait until a node is discovered */
+		if (!self->cachedaddr) {
+			pr_debug("%s(), nothing discovered yet, going to sleep...\n",
+				 __func__);
+
+			/* Set watchdog timer to expire in <val> ms. */
+			self->errno = 0;
+			setup_timer(&self->watchdog, irda_discovery_timeout,
+					(unsigned long)self);
+			mod_timer(&self->watchdog,
+				  jiffies + msecs_to_jiffies(val));
+
+			/* Wait for IR-LMP to call us back */
+			err = __wait_event_interruptible(self->query_wait,
+			      (self->cachedaddr != 0 || self->errno == -ETIME));
+
+			/* If watchdog is still activated, kill it! */
+			del_timer(&(self->watchdog));
+
+			pr_debug("%s(), ...waking up !\n", __func__);
+
+			if (err != 0)
+				goto out;
+		}
+		else
+			pr_debug("%s(), found immediately !\n",
+				 __func__);
+
+		/* Tell IrLMP that we have been notified */
+		irlmp_update_client(self->ckey, self->mask.word,
+				    NULL, NULL, NULL);
+
+		/* Check if the we got some results */
+		if (!self->cachedaddr) {
+			err = -EAGAIN;		/* Didn't find any devices */
+			goto out;
+		}
+		daddr = self->cachedaddr;
+		/* Cleanup */
+		self->cachedaddr = 0;
+
+		/* We return the daddr of the device that trigger the
+		 * wakeup. As irlmp pass us only the new devices, we
+		 * are sure that it's not an old device.
+		 * If the user want more details, he should query
+		 * the whole discovery log and pick one device...
+		 */
+		if (put_user(daddr, (int __user *)optval)) {
+			err = -EFAULT;
+			goto out;
+		}
+
+		break;
+	default:
+		err = -ENOPROTOOPT;
+	}
+
+out:
+
+	release_sock(sk);
+
+	return err;
+}
+
+static const struct net_proto_family irda_family_ops = {
+	.family = PF_IRDA,
+	.create = irda_create,
+	.owner	= THIS_MODULE,
+};
+
+static const struct proto_ops irda_stream_ops = {
+	.family =	PF_IRDA,
+	.owner =	THIS_MODULE,
+	.release =	irda_release,
+	.bind =		irda_bind,
+	.connect =	irda_connect,
+	.socketpair =	sock_no_socketpair,
+	.accept =	irda_accept,
+	.getname =	irda_getname,
+	.poll =		irda_poll,
+	.ioctl =	irda_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl =	irda_compat_ioctl,
+#endif
+	.listen =	irda_listen,
+	.shutdown =	irda_shutdown,
+	.setsockopt =	irda_setsockopt,
+	.getsockopt =	irda_getsockopt,
+	.sendmsg =	irda_sendmsg,
+	.recvmsg =	irda_recvmsg_stream,
+	.mmap =		sock_no_mmap,
+	.sendpage =	sock_no_sendpage,
+};
+
+static const struct proto_ops irda_seqpacket_ops = {
+	.family =	PF_IRDA,
+	.owner =	THIS_MODULE,
+	.release =	irda_release,
+	.bind =		irda_bind,
+	.connect =	irda_connect,
+	.socketpair =	sock_no_socketpair,
+	.accept =	irda_accept,
+	.getname =	irda_getname,
+	.poll =		datagram_poll,
+	.ioctl =	irda_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl =	irda_compat_ioctl,
+#endif
+	.listen =	irda_listen,
+	.shutdown =	irda_shutdown,
+	.setsockopt =	irda_setsockopt,
+	.getsockopt =	irda_getsockopt,
+	.sendmsg =	irda_sendmsg,
+	.recvmsg =	irda_recvmsg_dgram,
+	.mmap =		sock_no_mmap,
+	.sendpage =	sock_no_sendpage,
+};
+
+static const struct proto_ops irda_dgram_ops = {
+	.family =	PF_IRDA,
+	.owner =	THIS_MODULE,
+	.release =	irda_release,
+	.bind =		irda_bind,
+	.connect =	irda_connect,
+	.socketpair =	sock_no_socketpair,
+	.accept =	irda_accept,
+	.getname =	irda_getname,
+	.poll =		datagram_poll,
+	.ioctl =	irda_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl =	irda_compat_ioctl,
+#endif
+	.listen =	irda_listen,
+	.shutdown =	irda_shutdown,
+	.setsockopt =	irda_setsockopt,
+	.getsockopt =	irda_getsockopt,
+	.sendmsg =	irda_sendmsg_dgram,
+	.recvmsg =	irda_recvmsg_dgram,
+	.mmap =		sock_no_mmap,
+	.sendpage =	sock_no_sendpage,
+};
+
+#ifdef CONFIG_IRDA_ULTRA
+static const struct proto_ops irda_ultra_ops = {
+	.family =	PF_IRDA,
+	.owner =	THIS_MODULE,
+	.release =	irda_release,
+	.bind =		irda_bind,
+	.connect =	sock_no_connect,
+	.socketpair =	sock_no_socketpair,
+	.accept =	sock_no_accept,
+	.getname =	irda_getname,
+	.poll =		datagram_poll,
+	.ioctl =	irda_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl =	irda_compat_ioctl,
+#endif
+	.listen =	sock_no_listen,
+	.shutdown =	irda_shutdown,
+	.setsockopt =	irda_setsockopt,
+	.getsockopt =	irda_getsockopt,
+	.sendmsg =	irda_sendmsg_ultra,
+	.recvmsg =	irda_recvmsg_dgram,
+	.mmap =		sock_no_mmap,
+	.sendpage =	sock_no_sendpage,
+};
+#endif /* CONFIG_IRDA_ULTRA */
+
+/*
+ * Function irsock_init (pro)
+ *
+ *    Initialize IrDA protocol
+ *
+ */
+int __init irsock_init(void)
+{
+	int rc = proto_register(&irda_proto, 0);
+
+	if (rc == 0)
+		rc = sock_register(&irda_family_ops);
+
+	return rc;
+}
+
+/*
+ * Function irsock_cleanup (void)
+ *
+ *    Remove IrDA protocol
+ *
+ */
+void irsock_cleanup(void)
+{
+	sock_unregister(PF_IRDA);
+	proto_unregister(&irda_proto);
+}
diff --git a/drivers/staging/irda/net/discovery.c b/drivers/staging/irda/net/discovery.c
new file mode 100644
index 000000000000..364d70aed068
--- /dev/null
+++ b/drivers/staging/irda/net/discovery.c
@@ -0,0 +1,417 @@
+/*********************************************************************
+ *
+ * Filename:      discovery.c
+ * Version:       0.1
+ * Description:   Routines for handling discoveries at the IrLMP layer
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Tue Apr  6 15:33:50 1999
+ * Modified at:   Sat Oct  9 17:11:31 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * Modified at:   Fri May 28  3:11 CST 1999
+ * Modified by:   Horst von Brand <vonbrand@sleipnir.valparaiso.cl>
+ *
+ *     Copyright (c) 1999 Dag Brattli, All Rights Reserved.
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ *
+ *     You should have received a copy of the GNU General Public License
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ ********************************************************************/
+
+#include <linux/string.h>
+#include <linux/socket.h>
+#include <linux/fs.h>
+#include <linux/seq_file.h>
+#include <linux/slab.h>
+#include <linux/export.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/irlmp.h>
+
+#include <net/irda/discovery.h>
+
+#include <asm/unaligned.h>
+
+/*
+ * Function irlmp_add_discovery (cachelog, discovery)
+ *
+ *    Add a new discovery to the cachelog, and remove any old discoveries
+ *    from the same device
+ *
+ * Note : we try to preserve the time this device was *first* discovered
+ * (as opposed to the time of last discovery used for cleanup). This is
+ * used by clients waiting for discovery events to tell if the device
+ * discovered is "new" or just the same old one. They can't rely there
+ * on a binary flag (new/old), because not all discovery events are
+ * propagated to them, and they might not always listen, so they would
+ * miss some new devices popping up...
+ * Jean II
+ */
+void irlmp_add_discovery(hashbin_t *cachelog, discovery_t *new)
+{
+	discovery_t *discovery, *node;
+	unsigned long flags;
+
+	/* Set time of first discovery if node is new (see below) */
+	new->firststamp = new->timestamp;
+
+	spin_lock_irqsave(&cachelog->hb_spinlock, flags);
+
+	/*
+	 * Remove all discoveries of devices that has previously been
+	 * discovered on the same link with the same name (info), or the
+	 * same daddr. We do this since some devices (mostly PDAs) change
+	 * their device address between every discovery.
+	 */
+	discovery = (discovery_t *) hashbin_get_first(cachelog);
+	while (discovery != NULL ) {
+		node = discovery;
+
+		/* Be sure to stay one item ahead */
+		discovery = (discovery_t *) hashbin_get_next(cachelog);
+
+		if ((node->data.saddr == new->data.saddr) &&
+		    ((node->data.daddr == new->data.daddr) ||
+		     (strcmp(node->data.info, new->data.info) == 0)))
+		{
+			/* This discovery is a previous discovery
+			 * from the same device, so just remove it
+			 */
+			hashbin_remove_this(cachelog, (irda_queue_t *) node);
+			/* Check if hints bits are unchanged */
+			if (get_unaligned((__u16 *)node->data.hints) == get_unaligned((__u16 *)new->data.hints))
+				/* Set time of first discovery for this node */
+				new->firststamp = node->firststamp;
+			kfree(node);
+		}
+	}
+
+	/* Insert the new and updated version */
+	hashbin_insert(cachelog, (irda_queue_t *) new, new->data.daddr, NULL);
+
+	spin_unlock_irqrestore(&cachelog->hb_spinlock, flags);
+}
+
+/*
+ * Function irlmp_add_discovery_log (cachelog, log)
+ *
+ *    Merge a disovery log into the cachelog.
+ *
+ */
+void irlmp_add_discovery_log(hashbin_t *cachelog, hashbin_t *log)
+{
+	discovery_t *discovery;
+
+	/*
+	 *  If log is missing this means that IrLAP was unable to perform the
+	 *  discovery, so restart discovery again with just the half timeout
+	 *  of the normal one.
+	 */
+	/* Well... It means that there was nobody out there - Jean II */
+	if (log == NULL) {
+		/* irlmp_start_discovery_timer(irlmp, 150); */
+		return;
+	}
+
+	/*
+	 * Locking : we are the only owner of this discovery log, so
+	 * no need to lock it.
+	 * We just need to lock the global log in irlmp_add_discovery().
+	 */
+	discovery = (discovery_t *) hashbin_remove_first(log);
+	while (discovery != NULL) {
+		irlmp_add_discovery(cachelog, discovery);
+
+		discovery = (discovery_t *) hashbin_remove_first(log);
+	}
+
+	/* Delete the now empty log */
+	hashbin_delete(log, (FREE_FUNC) kfree);
+}
+
+/*
+ * Function irlmp_expire_discoveries (log, saddr, force)
+ *
+ *    Go through all discoveries and expire all that has stayed too long
+ *
+ * Note : this assume that IrLAP won't change its saddr, which
+ * currently is a valid assumption...
+ */
+void irlmp_expire_discoveries(hashbin_t *log, __u32 saddr, int force)
+{
+	discovery_t *		discovery;
+	discovery_t *		curr;
+	unsigned long		flags;
+	discinfo_t *		buffer = NULL;
+	int			n;		/* Size of the full log */
+	int			i = 0;		/* How many we expired */
+
+	IRDA_ASSERT(log != NULL, return;);
+	spin_lock_irqsave(&log->hb_spinlock, flags);
+
+	discovery = (discovery_t *) hashbin_get_first(log);
+	while (discovery != NULL) {
+		/* Be sure to be one item ahead */
+		curr = discovery;
+		discovery = (discovery_t *) hashbin_get_next(log);
+
+		/* Test if it's time to expire this discovery */
+		if ((curr->data.saddr == saddr) &&
+		    (force ||
+		     ((jiffies - curr->timestamp) > DISCOVERY_EXPIRE_TIMEOUT)))
+		{
+			/* Create buffer as needed.
+			 * As this function get called a lot and most time
+			 * we don't have anything to put in the log (we are
+			 * quite picky), we can save a lot of overhead
+			 * by not calling kmalloc. Jean II */
+			if(buffer == NULL) {
+				/* Create the client specific buffer */
+				n = HASHBIN_GET_SIZE(log);
+				buffer = kmalloc(n * sizeof(struct irda_device_info), GFP_ATOMIC);
+				if (buffer == NULL) {
+					spin_unlock_irqrestore(&log->hb_spinlock, flags);
+					return;
+				}
+
+			}
+
+			/* Copy discovery information */
+			memcpy(&(buffer[i]), &(curr->data),
+			       sizeof(discinfo_t));
+			i++;
+
+			/* Remove it from the log */
+			curr = hashbin_remove_this(log, (irda_queue_t *) curr);
+			kfree(curr);
+		}
+	}
+
+	/* Drop the spinlock before calling the higher layers, as
+	 * we can't guarantee they won't call us back and create a
+	 * deadlock. We will work on our own private data, so we
+	 * don't care to be interrupted. - Jean II */
+	spin_unlock_irqrestore(&log->hb_spinlock, flags);
+
+	if(buffer == NULL)
+		return;
+
+	/* Tell IrLMP and registered clients about it */
+	irlmp_discovery_expiry(buffer, i);
+
+	/* Free up our buffer */
+	kfree(buffer);
+}
+
+#if 0
+/*
+ * Function irlmp_dump_discoveries (log)
+ *
+ *    Print out all discoveries in log
+ *
+ */
+void irlmp_dump_discoveries(hashbin_t *log)
+{
+	discovery_t *discovery;
+
+	IRDA_ASSERT(log != NULL, return;);
+
+	discovery = (discovery_t *) hashbin_get_first(log);
+	while (discovery != NULL) {
+		pr_debug("Discovery:\n");
+		pr_debug("  daddr=%08x\n", discovery->data.daddr);
+		pr_debug("  saddr=%08x\n", discovery->data.saddr);
+		pr_debug("  nickname=%s\n", discovery->data.info);
+
+		discovery = (discovery_t *) hashbin_get_next(log);
+	}
+}
+#endif
+
+/*
+ * Function irlmp_copy_discoveries (log, pn, mask)
+ *
+ *    Copy all discoveries in a buffer
+ *
+ * This function implement a safe way for lmp clients to access the
+ * discovery log. The basic problem is that we don't want the log
+ * to change (add/remove) while the client is reading it. If the
+ * lmp client manipulate directly the hashbin, he is sure to get
+ * into troubles...
+ * The idea is that we copy all the current discovery log in a buffer
+ * which is specific to the client and pass this copy to him. As we
+ * do this operation with the spinlock grabbed, we are safe...
+ * Note : we don't want those clients to grab the spinlock, because
+ * we have no control on how long they will hold it...
+ * Note : we choose to copy the log in "struct irda_device_info" to
+ * save space...
+ * Note : the client must kfree himself() the log...
+ * Jean II
+ */
+struct irda_device_info *irlmp_copy_discoveries(hashbin_t *log, int *pn,
+						__u16 mask, int old_entries)
+{
+	discovery_t *		discovery;
+	unsigned long		flags;
+	discinfo_t *		buffer = NULL;
+	int			j_timeout = (sysctl_discovery_timeout * HZ);
+	int			n;		/* Size of the full log */
+	int			i = 0;		/* How many we picked */
+
+	IRDA_ASSERT(pn != NULL, return NULL;);
+	IRDA_ASSERT(log != NULL, return NULL;);
+
+	/* Save spin lock */
+	spin_lock_irqsave(&log->hb_spinlock, flags);
+
+	discovery = (discovery_t *) hashbin_get_first(log);
+	while (discovery != NULL) {
+		/* Mask out the ones we don't want :
+		 * We want to match the discovery mask, and to get only
+		 * the most recent one (unless we want old ones) */
+		if ((get_unaligned((__u16 *)discovery->data.hints) & mask) &&
+		    ((old_entries) ||
+		     ((jiffies - discovery->firststamp) < j_timeout))) {
+			/* Create buffer as needed.
+			 * As this function get called a lot and most time
+			 * we don't have anything to put in the log (we are
+			 * quite picky), we can save a lot of overhead
+			 * by not calling kmalloc. Jean II */
+			if(buffer == NULL) {
+				/* Create the client specific buffer */
+				n = HASHBIN_GET_SIZE(log);
+				buffer = kmalloc(n * sizeof(struct irda_device_info), GFP_ATOMIC);
+				if (buffer == NULL) {
+					spin_unlock_irqrestore(&log->hb_spinlock, flags);
+					return NULL;
+				}
+
+			}
+
+			/* Copy discovery information */
+			memcpy(&(buffer[i]), &(discovery->data),
+			       sizeof(discinfo_t));
+			i++;
+		}
+		discovery = (discovery_t *) hashbin_get_next(log);
+	}
+
+	spin_unlock_irqrestore(&log->hb_spinlock, flags);
+
+	/* Get the actual number of device in the buffer and return */
+	*pn = i;
+	return buffer;
+}
+
+#ifdef CONFIG_PROC_FS
+static inline discovery_t *discovery_seq_idx(loff_t pos)
+
+{
+	discovery_t *discovery;
+
+	for (discovery = (discovery_t *) hashbin_get_first(irlmp->cachelog);
+	     discovery != NULL;
+	     discovery = (discovery_t *) hashbin_get_next(irlmp->cachelog)) {
+		if (pos-- == 0)
+			break;
+	}
+
+	return discovery;
+}
+
+static void *discovery_seq_start(struct seq_file *seq, loff_t *pos)
+{
+	spin_lock_irq(&irlmp->cachelog->hb_spinlock);
+	return *pos ? discovery_seq_idx(*pos - 1) : SEQ_START_TOKEN;
+}
+
+static void *discovery_seq_next(struct seq_file *seq, void *v, loff_t *pos)
+{
+	++*pos;
+	return (v == SEQ_START_TOKEN)
+		? (void *) hashbin_get_first(irlmp->cachelog)
+		: (void *) hashbin_get_next(irlmp->cachelog);
+}
+
+static void discovery_seq_stop(struct seq_file *seq, void *v)
+{
+	spin_unlock_irq(&irlmp->cachelog->hb_spinlock);
+}
+
+static int discovery_seq_show(struct seq_file *seq, void *v)
+{
+	if (v == SEQ_START_TOKEN)
+		seq_puts(seq, "IrLMP: Discovery log:\n\n");
+	else {
+		const discovery_t *discovery = v;
+
+		seq_printf(seq, "nickname: %s, hint: 0x%02x%02x",
+			   discovery->data.info,
+			   discovery->data.hints[0],
+			   discovery->data.hints[1]);
+#if 0
+		if ( discovery->data.hints[0] & HINT_PNP)
+			seq_puts(seq, "PnP Compatible ");
+		if ( discovery->data.hints[0] & HINT_PDA)
+			seq_puts(seq, "PDA/Palmtop ");
+		if ( discovery->data.hints[0] & HINT_COMPUTER)
+			seq_puts(seq, "Computer ");
+		if ( discovery->data.hints[0] & HINT_PRINTER)
+			seq_puts(seq, "Printer ");
+		if ( discovery->data.hints[0] & HINT_MODEM)
+			seq_puts(seq, "Modem ");
+		if ( discovery->data.hints[0] & HINT_FAX)
+			seq_puts(seq, "Fax ");
+		if ( discovery->data.hints[0] & HINT_LAN)
+			seq_puts(seq, "LAN Access ");
+
+		if ( discovery->data.hints[1] & HINT_TELEPHONY)
+			seq_puts(seq, "Telephony ");
+		if ( discovery->data.hints[1] & HINT_FILE_SERVER)
+			seq_puts(seq, "File Server ");
+		if ( discovery->data.hints[1] & HINT_COMM)
+			seq_puts(seq, "IrCOMM ");
+		if ( discovery->data.hints[1] & HINT_OBEX)
+			seq_puts(seq, "IrOBEX ");
+#endif
+		seq_printf(seq,", saddr: 0x%08x, daddr: 0x%08x\n\n",
+			       discovery->data.saddr,
+			       discovery->data.daddr);
+
+		seq_putc(seq, '\n');
+	}
+	return 0;
+}
+
+static const struct seq_operations discovery_seq_ops = {
+	.start  = discovery_seq_start,
+	.next   = discovery_seq_next,
+	.stop   = discovery_seq_stop,
+	.show   = discovery_seq_show,
+};
+
+static int discovery_seq_open(struct inode *inode, struct file *file)
+{
+	IRDA_ASSERT(irlmp != NULL, return -EINVAL;);
+
+	return seq_open(file, &discovery_seq_ops);
+}
+
+const struct file_operations discovery_seq_fops = {
+	.owner		= THIS_MODULE,
+	.open           = discovery_seq_open,
+	.read           = seq_read,
+	.llseek         = seq_lseek,
+	.release	= seq_release,
+};
+#endif
diff --git a/drivers/staging/irda/net/ircomm/Kconfig b/drivers/staging/irda/net/ircomm/Kconfig
new file mode 100644
index 000000000000..19492c1707b7
--- /dev/null
+++ b/drivers/staging/irda/net/ircomm/Kconfig
@@ -0,0 +1,12 @@
+config IRCOMM
+	tristate "IrCOMM protocol"
+	depends on IRDA && TTY
+	help
+	  Say Y here if you want to build support for the IrCOMM protocol.
+	  To compile it as modules, choose M here: the modules will be
+	  called ircomm and ircomm_tty.
+	  IrCOMM implements serial port emulation, and makes it possible to
+	  use all existing applications that understands TTY's with an
+	  infrared link.  Thus you should be able to use application like PPP,
+	  minicom and others.
+
diff --git a/drivers/staging/irda/net/ircomm/Makefile b/drivers/staging/irda/net/ircomm/Makefile
new file mode 100644
index 000000000000..ab23b5ba7e33
--- /dev/null
+++ b/drivers/staging/irda/net/ircomm/Makefile
@@ -0,0 +1,8 @@
+#
+# Makefile for the Linux IrDA IrCOMM protocol layer.
+#
+
+obj-$(CONFIG_IRCOMM) += ircomm.o ircomm-tty.o
+
+ircomm-y := ircomm_core.o ircomm_event.o ircomm_lmp.o ircomm_ttp.o
+ircomm-tty-y := ircomm_tty.o ircomm_tty_attach.o ircomm_tty_ioctl.o ircomm_param.o
diff --git a/drivers/staging/irda/net/ircomm/ircomm_core.c b/drivers/staging/irda/net/ircomm/ircomm_core.c
new file mode 100644
index 000000000000..3af219545f6d
--- /dev/null
+++ b/drivers/staging/irda/net/ircomm/ircomm_core.c
@@ -0,0 +1,563 @@
+/*********************************************************************
+ *
+ * Filename:      ircomm_core.c
+ * Version:       1.0
+ * Description:   IrCOMM service interface
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sun Jun  6 20:37:34 1999
+ * Modified at:   Tue Dec 21 13:26:41 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1999 Dag Brattli, All Rights Reserved.
+ *     Copyright (c) 2000-2003 Jean Tourrilhes <jt@hpl.hp.com>
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ *
+ *     You should have received a copy of the GNU General Public License
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ ********************************************************************/
+
+#include <linux/module.h>
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/irmod.h>
+#include <net/irda/irlmp.h>
+#include <net/irda/iriap.h>
+#include <net/irda/irttp.h>
+#include <net/irda/irias_object.h>
+
+#include <net/irda/ircomm_event.h>
+#include <net/irda/ircomm_lmp.h>
+#include <net/irda/ircomm_ttp.h>
+#include <net/irda/ircomm_param.h>
+#include <net/irda/ircomm_core.h>
+
+static int __ircomm_close(struct ircomm_cb *self);
+static void ircomm_control_indication(struct ircomm_cb *self,
+				      struct sk_buff *skb, int clen);
+
+#ifdef CONFIG_PROC_FS
+extern struct proc_dir_entry *proc_irda;
+static int ircomm_seq_open(struct inode *, struct file *);
+
+static const struct file_operations ircomm_proc_fops = {
+	.owner		= THIS_MODULE,
+	.open           = ircomm_seq_open,
+	.read           = seq_read,
+	.llseek         = seq_lseek,
+	.release	= seq_release,
+};
+#endif /* CONFIG_PROC_FS */
+
+hashbin_t *ircomm = NULL;
+
+static int __init ircomm_init(void)
+{
+	ircomm = hashbin_new(HB_LOCK);
+	if (ircomm == NULL) {
+		net_err_ratelimited("%s(), can't allocate hashbin!\n",
+				    __func__);
+		return -ENOMEM;
+	}
+
+#ifdef CONFIG_PROC_FS
+	{ struct proc_dir_entry *ent;
+	ent = proc_create("ircomm", 0, proc_irda, &ircomm_proc_fops);
+	if (!ent) {
+		printk(KERN_ERR "ircomm_init: can't create /proc entry!\n");
+		return -ENODEV;
+	}
+	}
+#endif /* CONFIG_PROC_FS */
+
+	net_info_ratelimited("IrCOMM protocol (Dag Brattli)\n");
+
+	return 0;
+}
+
+static void __exit ircomm_cleanup(void)
+{
+	hashbin_delete(ircomm, (FREE_FUNC) __ircomm_close);
+
+#ifdef CONFIG_PROC_FS
+	remove_proc_entry("ircomm", proc_irda);
+#endif /* CONFIG_PROC_FS */
+}
+
+/*
+ * Function ircomm_open (client_notify)
+ *
+ *    Start a new IrCOMM instance
+ *
+ */
+struct ircomm_cb *ircomm_open(notify_t *notify, __u8 service_type, int line)
+{
+	struct ircomm_cb *self = NULL;
+	int ret;
+
+	pr_debug("%s(), service_type=0x%02x\n", __func__ ,
+		 service_type);
+
+	IRDA_ASSERT(ircomm != NULL, return NULL;);
+
+	self = kzalloc(sizeof(struct ircomm_cb), GFP_KERNEL);
+	if (self == NULL)
+		return NULL;
+
+	self->notify = *notify;
+	self->magic = IRCOMM_MAGIC;
+
+	/* Check if we should use IrLMP or IrTTP */
+	if (service_type & IRCOMM_3_WIRE_RAW) {
+		self->flow_status = FLOW_START;
+		ret = ircomm_open_lsap(self);
+	} else
+		ret = ircomm_open_tsap(self);
+
+	if (ret < 0) {
+		kfree(self);
+		return NULL;
+	}
+
+	self->service_type = service_type;
+	self->line = line;
+
+	hashbin_insert(ircomm, (irda_queue_t *) self, line, NULL);
+
+	ircomm_next_state(self, IRCOMM_IDLE);
+
+	return self;
+}
+
+EXPORT_SYMBOL(ircomm_open);
+
+/*
+ * Function ircomm_close_instance (self)
+ *
+ *    Remove IrCOMM instance
+ *
+ */
+static int __ircomm_close(struct ircomm_cb *self)
+{
+	/* Disconnect link if any */
+	ircomm_do_event(self, IRCOMM_DISCONNECT_REQUEST, NULL, NULL);
+
+	/* Remove TSAP */
+	if (self->tsap) {
+		irttp_close_tsap(self->tsap);
+		self->tsap = NULL;
+	}
+
+	/* Remove LSAP */
+	if (self->lsap) {
+		irlmp_close_lsap(self->lsap);
+		self->lsap = NULL;
+	}
+	self->magic = 0;
+
+	kfree(self);
+
+	return 0;
+}
+
+/*
+ * Function ircomm_close (self)
+ *
+ *    Closes and removes the specified IrCOMM instance
+ *
+ */
+int ircomm_close(struct ircomm_cb *self)
+{
+	struct ircomm_cb *entry;
+
+	IRDA_ASSERT(self != NULL, return -EIO;);
+	IRDA_ASSERT(self->magic == IRCOMM_MAGIC, return -EIO;);
+
+	entry = hashbin_remove(ircomm, self->line, NULL);
+
+	IRDA_ASSERT(entry == self, return -1;);
+
+	return __ircomm_close(self);
+}
+
+EXPORT_SYMBOL(ircomm_close);
+
+/*
+ * Function ircomm_connect_request (self, service_type)
+ *
+ *    Impl. of this function is differ from one of the reference. This
+ *    function does discovery as well as sending connect request
+ *
+ */
+int ircomm_connect_request(struct ircomm_cb *self, __u8 dlsap_sel,
+			   __u32 saddr, __u32 daddr, struct sk_buff *skb,
+			   __u8 service_type)
+{
+	struct ircomm_info info;
+	int ret;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_MAGIC, return -1;);
+
+	self->service_type= service_type;
+
+	info.dlsap_sel = dlsap_sel;
+	info.saddr = saddr;
+	info.daddr = daddr;
+
+	ret = ircomm_do_event(self, IRCOMM_CONNECT_REQUEST, skb, &info);
+
+	return ret;
+}
+
+EXPORT_SYMBOL(ircomm_connect_request);
+
+/*
+ * Function ircomm_connect_indication (self, qos, skb)
+ *
+ *    Notify user layer about the incoming connection
+ *
+ */
+void ircomm_connect_indication(struct ircomm_cb *self, struct sk_buff *skb,
+			       struct ircomm_info *info)
+{
+	/*
+	 * If there are any data hiding in the control channel, we must
+	 * deliver it first. The side effect is that the control channel
+	 * will be removed from the skb
+	 */
+	if (self->notify.connect_indication)
+		self->notify.connect_indication(self->notify.instance, self,
+						info->qos, info->max_data_size,
+						info->max_header_size, skb);
+	else {
+		pr_debug("%s(), missing handler\n", __func__);
+	}
+}
+
+/*
+ * Function ircomm_connect_response (self, userdata, max_sdu_size)
+ *
+ *    User accepts connection
+ *
+ */
+int ircomm_connect_response(struct ircomm_cb *self, struct sk_buff *userdata)
+{
+	int ret;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_MAGIC, return -1;);
+
+	ret = ircomm_do_event(self, IRCOMM_CONNECT_RESPONSE, userdata, NULL);
+
+	return ret;
+}
+
+EXPORT_SYMBOL(ircomm_connect_response);
+
+/*
+ * Function connect_confirm (self, skb)
+ *
+ *    Notify user layer that the link is now connected
+ *
+ */
+void ircomm_connect_confirm(struct ircomm_cb *self, struct sk_buff *skb,
+			    struct ircomm_info *info)
+{
+	if (self->notify.connect_confirm )
+		self->notify.connect_confirm(self->notify.instance,
+					     self, info->qos,
+					     info->max_data_size,
+					     info->max_header_size, skb);
+	else {
+		pr_debug("%s(), missing handler\n", __func__);
+	}
+}
+
+/*
+ * Function ircomm_data_request (self, userdata)
+ *
+ *    Send IrCOMM data to peer device
+ *
+ */
+int ircomm_data_request(struct ircomm_cb *self, struct sk_buff *skb)
+{
+	int ret;
+
+	IRDA_ASSERT(self != NULL, return -EFAULT;);
+	IRDA_ASSERT(self->magic == IRCOMM_MAGIC, return -EFAULT;);
+	IRDA_ASSERT(skb != NULL, return -EFAULT;);
+
+	ret = ircomm_do_event(self, IRCOMM_DATA_REQUEST, skb, NULL);
+
+	return ret;
+}
+
+EXPORT_SYMBOL(ircomm_data_request);
+
+/*
+ * Function ircomm_data_indication (self, skb)
+ *
+ *    Data arrived, so deliver it to user
+ *
+ */
+void ircomm_data_indication(struct ircomm_cb *self, struct sk_buff *skb)
+{
+	IRDA_ASSERT(skb->len > 0, return;);
+
+	if (self->notify.data_indication)
+		self->notify.data_indication(self->notify.instance, self, skb);
+	else {
+		pr_debug("%s(), missing handler\n", __func__);
+	}
+}
+
+/*
+ * Function ircomm_process_data (self, skb)
+ *
+ *    Data arrived which may contain control channel data
+ *
+ */
+void ircomm_process_data(struct ircomm_cb *self, struct sk_buff *skb)
+{
+	int clen;
+
+	IRDA_ASSERT(skb->len > 0, return;);
+
+	clen = skb->data[0];
+
+	/*
+	 * Input validation check: a stir4200/mcp2150 combinations sometimes
+	 * results in frames with clen > remaining packet size. These are
+	 * illegal; if we throw away just this frame then it seems to carry on
+	 * fine
+	 */
+	if (unlikely(skb->len < (clen + 1))) {
+		pr_debug("%s() throwing away illegal frame\n",
+			 __func__);
+		return;
+	}
+
+	/*
+	 * If there are any data hiding in the control channel, we must
+	 * deliver it first. The side effect is that the control channel
+	 * will be removed from the skb
+	 */
+	if (clen > 0)
+		ircomm_control_indication(self, skb, clen);
+
+	/* Remove control channel from data channel */
+	skb_pull(skb, clen+1);
+
+	if (skb->len)
+		ircomm_data_indication(self, skb);
+	else {
+		pr_debug("%s(), data was control info only!\n",
+			 __func__);
+	}
+}
+
+/*
+ * Function ircomm_control_request (self, params)
+ *
+ *    Send control data to peer device
+ *
+ */
+int ircomm_control_request(struct ircomm_cb *self, struct sk_buff *skb)
+{
+	int ret;
+
+	IRDA_ASSERT(self != NULL, return -EFAULT;);
+	IRDA_ASSERT(self->magic == IRCOMM_MAGIC, return -EFAULT;);
+	IRDA_ASSERT(skb != NULL, return -EFAULT;);
+
+	ret = ircomm_do_event(self, IRCOMM_CONTROL_REQUEST, skb, NULL);
+
+	return ret;
+}
+
+EXPORT_SYMBOL(ircomm_control_request);
+
+/*
+ * Function ircomm_control_indication (self, skb)
+ *
+ *    Data has arrived on the control channel
+ *
+ */
+static void ircomm_control_indication(struct ircomm_cb *self,
+				      struct sk_buff *skb, int clen)
+{
+	/* Use udata for delivering data on the control channel */
+	if (self->notify.udata_indication) {
+		struct sk_buff *ctrl_skb;
+
+		/* We don't own the skb, so clone it */
+		ctrl_skb = skb_clone(skb, GFP_ATOMIC);
+		if (!ctrl_skb)
+			return;
+
+		/* Remove data channel from control channel */
+		skb_trim(ctrl_skb, clen+1);
+
+		self->notify.udata_indication(self->notify.instance, self,
+					      ctrl_skb);
+
+		/* Drop reference count -
+		 * see ircomm_tty_control_indication(). */
+		dev_kfree_skb(ctrl_skb);
+	} else {
+		pr_debug("%s(), missing handler\n", __func__);
+	}
+}
+
+/*
+ * Function ircomm_disconnect_request (self, userdata, priority)
+ *
+ *    User layer wants to disconnect the IrCOMM connection
+ *
+ */
+int ircomm_disconnect_request(struct ircomm_cb *self, struct sk_buff *userdata)
+{
+	struct ircomm_info info;
+	int ret;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_MAGIC, return -1;);
+
+	ret = ircomm_do_event(self, IRCOMM_DISCONNECT_REQUEST, userdata,
+			      &info);
+	return ret;
+}
+
+EXPORT_SYMBOL(ircomm_disconnect_request);
+
+/*
+ * Function disconnect_indication (self, skb)
+ *
+ *    Tell user that the link has been disconnected
+ *
+ */
+void ircomm_disconnect_indication(struct ircomm_cb *self, struct sk_buff *skb,
+				  struct ircomm_info *info)
+{
+	IRDA_ASSERT(info != NULL, return;);
+
+	if (self->notify.disconnect_indication) {
+		self->notify.disconnect_indication(self->notify.instance, self,
+						   info->reason, skb);
+	} else {
+		pr_debug("%s(), missing handler\n", __func__);
+	}
+}
+
+/*
+ * Function ircomm_flow_request (self, flow)
+ *
+ *
+ *
+ */
+void ircomm_flow_request(struct ircomm_cb *self, LOCAL_FLOW flow)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_MAGIC, return;);
+
+	if (self->service_type == IRCOMM_3_WIRE_RAW)
+		return;
+
+	irttp_flow_request(self->tsap, flow);
+}
+
+EXPORT_SYMBOL(ircomm_flow_request);
+
+#ifdef CONFIG_PROC_FS
+static void *ircomm_seq_start(struct seq_file *seq, loff_t *pos)
+{
+	struct ircomm_cb *self;
+	loff_t off = 0;
+
+	spin_lock_irq(&ircomm->hb_spinlock);
+
+	for (self = (struct ircomm_cb *) hashbin_get_first(ircomm);
+	     self != NULL;
+	     self = (struct ircomm_cb *) hashbin_get_next(ircomm)) {
+		if (off++ == *pos)
+			break;
+
+	}
+	return self;
+}
+
+static void *ircomm_seq_next(struct seq_file *seq, void *v, loff_t *pos)
+{
+	++*pos;
+
+	return (void *) hashbin_get_next(ircomm);
+}
+
+static void ircomm_seq_stop(struct seq_file *seq, void *v)
+{
+	spin_unlock_irq(&ircomm->hb_spinlock);
+}
+
+static int ircomm_seq_show(struct seq_file *seq, void *v)
+{
+	const struct ircomm_cb *self = v;
+
+	IRDA_ASSERT(self->magic == IRCOMM_MAGIC, return -EINVAL; );
+
+	if(self->line < 0x10)
+		seq_printf(seq, "ircomm%d", self->line);
+	else
+		seq_printf(seq, "irlpt%d", self->line - 0x10);
+
+	seq_printf(seq,
+		   " state: %s, slsap_sel: %#02x, dlsap_sel: %#02x, mode:",
+		   ircomm_state[ self->state],
+		   self->slsap_sel, self->dlsap_sel);
+
+	if(self->service_type & IRCOMM_3_WIRE_RAW)
+		seq_printf(seq, " 3-wire-raw");
+	if(self->service_type & IRCOMM_3_WIRE)
+		seq_printf(seq, " 3-wire");
+	if(self->service_type & IRCOMM_9_WIRE)
+		seq_printf(seq, " 9-wire");
+	if(self->service_type & IRCOMM_CENTRONICS)
+		seq_printf(seq, " Centronics");
+	seq_putc(seq, '\n');
+
+	return 0;
+}
+
+static const struct seq_operations ircomm_seq_ops = {
+	.start  = ircomm_seq_start,
+	.next   = ircomm_seq_next,
+	.stop   = ircomm_seq_stop,
+	.show   = ircomm_seq_show,
+};
+
+static int ircomm_seq_open(struct inode *inode, struct file *file)
+{
+	return seq_open(file, &ircomm_seq_ops);
+}
+#endif /* CONFIG_PROC_FS */
+
+MODULE_AUTHOR("Dag Brattli <dag@brattli.net>");
+MODULE_DESCRIPTION("IrCOMM protocol");
+MODULE_LICENSE("GPL");
+
+module_init(ircomm_init);
+module_exit(ircomm_cleanup);
diff --git a/drivers/staging/irda/net/ircomm/ircomm_event.c b/drivers/staging/irda/net/ircomm/ircomm_event.c
new file mode 100644
index 000000000000..b0730ac9f388
--- /dev/null
+++ b/drivers/staging/irda/net/ircomm/ircomm_event.c
@@ -0,0 +1,246 @@
+/*********************************************************************
+ *
+ * Filename:      ircomm_event.c
+ * Version:       1.0
+ * Description:   IrCOMM layer state machine
+ * Status:        Stable
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sun Jun  6 20:33:11 1999
+ * Modified at:   Sun Dec 12 13:44:32 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1999 Dag Brattli, All Rights Reserved.
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ *
+ *     You should have received a copy of the GNU General Public License
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ ********************************************************************/
+
+#include <linux/proc_fs.h>
+#include <linux/init.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/irlmp.h>
+#include <net/irda/iriap.h>
+#include <net/irda/irttp.h>
+#include <net/irda/irias_object.h>
+
+#include <net/irda/ircomm_core.h>
+#include <net/irda/ircomm_event.h>
+
+static int ircomm_state_idle(struct ircomm_cb *self, IRCOMM_EVENT event,
+			     struct sk_buff *skb, struct ircomm_info *info);
+static int ircomm_state_waiti(struct ircomm_cb *self, IRCOMM_EVENT event,
+			      struct sk_buff *skb, struct ircomm_info *info);
+static int ircomm_state_waitr(struct ircomm_cb *self, IRCOMM_EVENT event,
+			      struct sk_buff *skb, struct ircomm_info *info);
+static int ircomm_state_conn(struct ircomm_cb *self, IRCOMM_EVENT event,
+			     struct sk_buff *skb, struct ircomm_info *info);
+
+const char *const ircomm_state[] = {
+	"IRCOMM_IDLE",
+	"IRCOMM_WAITI",
+	"IRCOMM_WAITR",
+	"IRCOMM_CONN",
+};
+
+static const char *const ircomm_event[] __maybe_unused = {
+	"IRCOMM_CONNECT_REQUEST",
+	"IRCOMM_CONNECT_RESPONSE",
+	"IRCOMM_TTP_CONNECT_INDICATION",
+	"IRCOMM_LMP_CONNECT_INDICATION",
+	"IRCOMM_TTP_CONNECT_CONFIRM",
+	"IRCOMM_LMP_CONNECT_CONFIRM",
+
+	"IRCOMM_LMP_DISCONNECT_INDICATION",
+	"IRCOMM_TTP_DISCONNECT_INDICATION",
+	"IRCOMM_DISCONNECT_REQUEST",
+
+	"IRCOMM_TTP_DATA_INDICATION",
+	"IRCOMM_LMP_DATA_INDICATION",
+	"IRCOMM_DATA_REQUEST",
+	"IRCOMM_CONTROL_REQUEST",
+	"IRCOMM_CONTROL_INDICATION",
+};
+
+static int (*state[])(struct ircomm_cb *self, IRCOMM_EVENT event,
+		      struct sk_buff *skb, struct ircomm_info *info) =
+{
+	ircomm_state_idle,
+	ircomm_state_waiti,
+	ircomm_state_waitr,
+	ircomm_state_conn,
+};
+
+/*
+ * Function ircomm_state_idle (self, event, skb)
+ *
+ *    IrCOMM is currently idle
+ *
+ */
+static int ircomm_state_idle(struct ircomm_cb *self, IRCOMM_EVENT event,
+			     struct sk_buff *skb, struct ircomm_info *info)
+{
+	int ret = 0;
+
+	switch (event) {
+	case IRCOMM_CONNECT_REQUEST:
+		ircomm_next_state(self, IRCOMM_WAITI);
+		ret = self->issue.connect_request(self, skb, info);
+		break;
+	case IRCOMM_TTP_CONNECT_INDICATION:
+	case IRCOMM_LMP_CONNECT_INDICATION:
+		ircomm_next_state(self, IRCOMM_WAITR);
+		ircomm_connect_indication(self, skb, info);
+		break;
+	default:
+		pr_debug("%s(), unknown event: %s\n", __func__ ,
+			 ircomm_event[event]);
+		ret = -EINVAL;
+	}
+	return ret;
+}
+
+/*
+ * Function ircomm_state_waiti (self, event, skb)
+ *
+ *    The IrCOMM user has requested an IrCOMM connection to the remote
+ *    device and is awaiting confirmation
+ */
+static int ircomm_state_waiti(struct ircomm_cb *self, IRCOMM_EVENT event,
+			      struct sk_buff *skb, struct ircomm_info *info)
+{
+	int ret = 0;
+
+	switch (event) {
+	case IRCOMM_TTP_CONNECT_CONFIRM:
+	case IRCOMM_LMP_CONNECT_CONFIRM:
+		ircomm_next_state(self, IRCOMM_CONN);
+		ircomm_connect_confirm(self, skb, info);
+		break;
+	case IRCOMM_TTP_DISCONNECT_INDICATION:
+	case IRCOMM_LMP_DISCONNECT_INDICATION:
+		ircomm_next_state(self, IRCOMM_IDLE);
+		ircomm_disconnect_indication(self, skb, info);
+		break;
+	default:
+		pr_debug("%s(), unknown event: %s\n", __func__ ,
+			 ircomm_event[event]);
+		ret = -EINVAL;
+	}
+	return ret;
+}
+
+/*
+ * Function ircomm_state_waitr (self, event, skb)
+ *
+ *    IrCOMM has received an incoming connection request and is awaiting
+ *    response from the user
+ */
+static int ircomm_state_waitr(struct ircomm_cb *self, IRCOMM_EVENT event,
+			      struct sk_buff *skb, struct ircomm_info *info)
+{
+	int ret = 0;
+
+	switch (event) {
+	case IRCOMM_CONNECT_RESPONSE:
+		ircomm_next_state(self, IRCOMM_CONN);
+		ret = self->issue.connect_response(self, skb);
+		break;
+	case IRCOMM_DISCONNECT_REQUEST:
+		ircomm_next_state(self, IRCOMM_IDLE);
+		ret = self->issue.disconnect_request(self, skb, info);
+		break;
+	case IRCOMM_TTP_DISCONNECT_INDICATION:
+	case IRCOMM_LMP_DISCONNECT_INDICATION:
+		ircomm_next_state(self, IRCOMM_IDLE);
+		ircomm_disconnect_indication(self, skb, info);
+		break;
+	default:
+		pr_debug("%s(), unknown event = %s\n", __func__ ,
+			 ircomm_event[event]);
+		ret = -EINVAL;
+	}
+	return ret;
+}
+
+/*
+ * Function ircomm_state_conn (self, event, skb)
+ *
+ *    IrCOMM is connected to the peer IrCOMM device
+ *
+ */
+static int ircomm_state_conn(struct ircomm_cb *self, IRCOMM_EVENT event,
+			     struct sk_buff *skb, struct ircomm_info *info)
+{
+	int ret = 0;
+
+	switch (event) {
+	case IRCOMM_DATA_REQUEST:
+		ret = self->issue.data_request(self, skb, 0);
+		break;
+	case IRCOMM_TTP_DATA_INDICATION:
+		ircomm_process_data(self, skb);
+		break;
+	case IRCOMM_LMP_DATA_INDICATION:
+		ircomm_data_indication(self, skb);
+		break;
+	case IRCOMM_CONTROL_REQUEST:
+		/* Just send a separate frame for now */
+		ret = self->issue.data_request(self, skb, skb->len);
+		break;
+	case IRCOMM_TTP_DISCONNECT_INDICATION:
+	case IRCOMM_LMP_DISCONNECT_INDICATION:
+		ircomm_next_state(self, IRCOMM_IDLE);
+		ircomm_disconnect_indication(self, skb, info);
+		break;
+	case IRCOMM_DISCONNECT_REQUEST:
+		ircomm_next_state(self, IRCOMM_IDLE);
+		ret = self->issue.disconnect_request(self, skb, info);
+		break;
+	default:
+		pr_debug("%s(), unknown event = %s\n", __func__ ,
+			 ircomm_event[event]);
+		ret = -EINVAL;
+	}
+	return ret;
+}
+
+/*
+ * Function ircomm_do_event (self, event, skb)
+ *
+ *    Process event
+ *
+ */
+int ircomm_do_event(struct ircomm_cb *self, IRCOMM_EVENT event,
+		    struct sk_buff *skb, struct ircomm_info *info)
+{
+	pr_debug("%s: state=%s, event=%s\n", __func__ ,
+		 ircomm_state[self->state], ircomm_event[event]);
+
+	return (*state[self->state])(self, event, skb, info);
+}
+
+/*
+ * Function ircomm_next_state (self, state)
+ *
+ *    Switch state
+ *
+ */
+void ircomm_next_state(struct ircomm_cb *self, IRCOMM_STATE state)
+{
+	self->state = state;
+
+	pr_debug("%s: next state=%s, service type=%d\n", __func__ ,
+		 ircomm_state[self->state], self->service_type);
+}
diff --git a/drivers/staging/irda/net/ircomm/ircomm_lmp.c b/drivers/staging/irda/net/ircomm/ircomm_lmp.c
new file mode 100644
index 000000000000..e4cc847bb933
--- /dev/null
+++ b/drivers/staging/irda/net/ircomm/ircomm_lmp.c
@@ -0,0 +1,350 @@
+/*********************************************************************
+ *
+ * Filename:      ircomm_lmp.c
+ * Version:       1.0
+ * Description:   Interface between IrCOMM and IrLMP
+ * Status:        Stable
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sun Jun  6 20:48:27 1999
+ * Modified at:   Sun Dec 12 13:44:17 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * Sources:       Previous IrLPT work by Thomas Davis
+ *
+ *     Copyright (c) 1999 Dag Brattli, All Rights Reserved.
+ *     Copyright (c) 2000-2003 Jean Tourrilhes <jt@hpl.hp.com>
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ *
+ *     You should have received a copy of the GNU General Public License
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ ********************************************************************/
+
+#include <linux/init.h>
+#include <linux/gfp.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/irlmp.h>
+#include <net/irda/iriap.h>
+#include <net/irda/irda_device.h>	/* struct irda_skb_cb */
+
+#include <net/irda/ircomm_event.h>
+#include <net/irda/ircomm_lmp.h>
+
+
+/*
+ * Function ircomm_lmp_connect_request (self, userdata)
+ *
+ *
+ *
+ */
+static int ircomm_lmp_connect_request(struct ircomm_cb *self,
+				      struct sk_buff *userdata,
+				      struct ircomm_info *info)
+{
+	int ret = 0;
+
+	/* Don't forget to refcount it - should be NULL anyway */
+	if(userdata)
+		skb_get(userdata);
+
+	ret = irlmp_connect_request(self->lsap, info->dlsap_sel,
+				    info->saddr, info->daddr, NULL, userdata);
+	return ret;
+}
+
+/*
+ * Function ircomm_lmp_connect_response (self, skb)
+ *
+ *
+ *
+ */
+static int ircomm_lmp_connect_response(struct ircomm_cb *self,
+				       struct sk_buff *userdata)
+{
+	struct sk_buff *tx_skb;
+
+	/* Any userdata supplied? */
+	if (userdata == NULL) {
+		tx_skb = alloc_skb(LMP_MAX_HEADER, GFP_ATOMIC);
+		if (!tx_skb)
+			return -ENOMEM;
+
+		/* Reserve space for MUX and LAP header */
+		skb_reserve(tx_skb, LMP_MAX_HEADER);
+	} else {
+		/*
+		 *  Check that the client has reserved enough space for
+		 *  headers
+		 */
+		IRDA_ASSERT(skb_headroom(userdata) >= LMP_MAX_HEADER,
+			    return -1;);
+
+		/* Don't forget to refcount it - should be NULL anyway */
+		skb_get(userdata);
+		tx_skb = userdata;
+	}
+
+	return irlmp_connect_response(self->lsap, tx_skb);
+}
+
+static int ircomm_lmp_disconnect_request(struct ircomm_cb *self,
+					 struct sk_buff *userdata,
+					 struct ircomm_info *info)
+{
+	struct sk_buff *tx_skb;
+	int ret;
+
+	if (!userdata) {
+		tx_skb = alloc_skb(LMP_MAX_HEADER, GFP_ATOMIC);
+		if (!tx_skb)
+			return -ENOMEM;
+
+		/*  Reserve space for MUX and LAP header */
+		skb_reserve(tx_skb, LMP_MAX_HEADER);
+		userdata = tx_skb;
+	} else {
+		/* Don't forget to refcount it - should be NULL anyway */
+		skb_get(userdata);
+	}
+
+	ret = irlmp_disconnect_request(self->lsap, userdata);
+
+	return ret;
+}
+
+/*
+ * Function ircomm_lmp_flow_control (skb)
+ *
+ *    This function is called when a data frame we have sent to IrLAP has
+ *    been deallocated. We do this to make sure we don't flood IrLAP with
+ *    frames, since we are not using the IrTTP flow control mechanism
+ */
+static void ircomm_lmp_flow_control(struct sk_buff *skb)
+{
+	struct irda_skb_cb *cb;
+	struct ircomm_cb *self;
+	int line;
+
+	IRDA_ASSERT(skb != NULL, return;);
+
+	cb = (struct irda_skb_cb *) skb->cb;
+
+	line = cb->line;
+
+	self = (struct ircomm_cb *) hashbin_lock_find(ircomm, line, NULL);
+	if (!self) {
+		pr_debug("%s(), didn't find myself\n", __func__);
+		return;
+	}
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_MAGIC, return;);
+
+	self->pkt_count--;
+
+	if ((self->pkt_count < 2) && (self->flow_status == FLOW_STOP)) {
+		pr_debug("%s(), asking TTY to start again!\n", __func__);
+		self->flow_status = FLOW_START;
+		if (self->notify.flow_indication)
+			self->notify.flow_indication(self->notify.instance,
+						     self, FLOW_START);
+	}
+}
+
+/*
+ * Function ircomm_lmp_data_request (self, userdata)
+ *
+ *    Send data frame to peer device
+ *
+ */
+static int ircomm_lmp_data_request(struct ircomm_cb *self,
+				   struct sk_buff *skb,
+				   int not_used)
+{
+	struct irda_skb_cb *cb;
+	int ret;
+
+	IRDA_ASSERT(skb != NULL, return -1;);
+
+	cb = (struct irda_skb_cb *) skb->cb;
+
+	cb->line = self->line;
+
+	pr_debug("%s(), sending frame\n", __func__);
+
+	/* Don't forget to refcount it - see ircomm_tty_do_softint() */
+	skb_get(skb);
+
+	skb_orphan(skb);
+	skb->destructor = ircomm_lmp_flow_control;
+
+	if ((self->pkt_count++ > 7) && (self->flow_status == FLOW_START)) {
+		pr_debug("%s(), asking TTY to slow down!\n", __func__);
+		self->flow_status = FLOW_STOP;
+		if (self->notify.flow_indication)
+			self->notify.flow_indication(self->notify.instance,
+						     self, FLOW_STOP);
+	}
+	ret = irlmp_data_request(self->lsap, skb);
+	if (ret) {
+		net_err_ratelimited("%s(), failed\n", __func__);
+		/* irlmp_data_request already free the packet */
+	}
+
+	return ret;
+}
+
+/*
+ * Function ircomm_lmp_data_indication (instance, sap, skb)
+ *
+ *    Incoming data which we must deliver to the state machine, to check
+ *    we are still connected.
+ */
+static int ircomm_lmp_data_indication(void *instance, void *sap,
+				      struct sk_buff *skb)
+{
+	struct ircomm_cb *self = (struct ircomm_cb *) instance;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_MAGIC, return -1;);
+	IRDA_ASSERT(skb != NULL, return -1;);
+
+	ircomm_do_event(self, IRCOMM_LMP_DATA_INDICATION, skb, NULL);
+
+	/* Drop reference count - see ircomm_tty_data_indication(). */
+	dev_kfree_skb(skb);
+
+	return 0;
+}
+
+/*
+ * Function ircomm_lmp_connect_confirm (instance, sap, qos, max_sdu_size,
+ *                                       max_header_size, skb)
+ *
+ *    Connection has been confirmed by peer device
+ *
+ */
+static void ircomm_lmp_connect_confirm(void *instance, void *sap,
+				       struct qos_info *qos,
+				       __u32 max_seg_size,
+				       __u8 max_header_size,
+				       struct sk_buff *skb)
+{
+	struct ircomm_cb *self = (struct ircomm_cb *) instance;
+	struct ircomm_info info;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_MAGIC, return;);
+	IRDA_ASSERT(skb != NULL, return;);
+	IRDA_ASSERT(qos != NULL, return;);
+
+	info.max_data_size = max_seg_size;
+	info.max_header_size = max_header_size;
+	info.qos = qos;
+
+	ircomm_do_event(self, IRCOMM_LMP_CONNECT_CONFIRM, skb, &info);
+
+	/* Drop reference count - see ircomm_tty_connect_confirm(). */
+	dev_kfree_skb(skb);
+}
+
+/*
+ * Function ircomm_lmp_connect_indication (instance, sap, qos, max_sdu_size,
+ *                                         max_header_size, skb)
+ *
+ *    Peer device wants to make a connection with us
+ *
+ */
+static void ircomm_lmp_connect_indication(void *instance, void *sap,
+					  struct qos_info *qos,
+					  __u32 max_seg_size,
+					  __u8 max_header_size,
+					  struct sk_buff *skb)
+{
+	struct ircomm_cb *self = (struct ircomm_cb *)instance;
+	struct ircomm_info info;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_MAGIC, return;);
+	IRDA_ASSERT(skb != NULL, return;);
+	IRDA_ASSERT(qos != NULL, return;);
+
+	info.max_data_size = max_seg_size;
+	info.max_header_size = max_header_size;
+	info.qos = qos;
+
+	ircomm_do_event(self, IRCOMM_LMP_CONNECT_INDICATION, skb, &info);
+
+	/* Drop reference count - see ircomm_tty_connect_indication(). */
+	dev_kfree_skb(skb);
+}
+
+/*
+ * Function ircomm_lmp_disconnect_indication (instance, sap, reason, skb)
+ *
+ *    Peer device has closed the connection, or the link went down for some
+ *    other reason
+ */
+static void ircomm_lmp_disconnect_indication(void *instance, void *sap,
+					     LM_REASON reason,
+					     struct sk_buff *skb)
+{
+	struct ircomm_cb *self = (struct ircomm_cb *) instance;
+	struct ircomm_info info;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_MAGIC, return;);
+
+	info.reason = reason;
+
+	ircomm_do_event(self, IRCOMM_LMP_DISCONNECT_INDICATION, skb, &info);
+
+	/* Drop reference count - see ircomm_tty_disconnect_indication(). */
+	if(skb)
+		dev_kfree_skb(skb);
+}
+/*
+ * Function ircomm_open_lsap (self)
+ *
+ *    Open LSAP. This function will only be used when using "raw" services
+ *
+ */
+int ircomm_open_lsap(struct ircomm_cb *self)
+{
+	notify_t notify;
+
+	/* Register callbacks */
+	irda_notify_init(&notify);
+	notify.data_indication       = ircomm_lmp_data_indication;
+	notify.connect_confirm       = ircomm_lmp_connect_confirm;
+	notify.connect_indication    = ircomm_lmp_connect_indication;
+	notify.disconnect_indication = ircomm_lmp_disconnect_indication;
+	notify.instance = self;
+	strlcpy(notify.name, "IrCOMM", sizeof(notify.name));
+
+	self->lsap = irlmp_open_lsap(LSAP_ANY, &notify, 0);
+	if (!self->lsap) {
+		pr_debug("%sfailed to allocate tsap\n", __func__);
+		return -1;
+	}
+	self->slsap_sel = self->lsap->slsap_sel;
+
+	/*
+	 *  Initialize the call-table for issuing commands
+	 */
+	self->issue.data_request       = ircomm_lmp_data_request;
+	self->issue.connect_request    = ircomm_lmp_connect_request;
+	self->issue.connect_response   = ircomm_lmp_connect_response;
+	self->issue.disconnect_request = ircomm_lmp_disconnect_request;
+
+	return 0;
+}
diff --git a/drivers/staging/irda/net/ircomm/ircomm_param.c b/drivers/staging/irda/net/ircomm/ircomm_param.c
new file mode 100644
index 000000000000..5728e76ca6d5
--- /dev/null
+++ b/drivers/staging/irda/net/ircomm/ircomm_param.c
@@ -0,0 +1,501 @@
+/*********************************************************************
+ *
+ * Filename:      ircomm_param.c
+ * Version:       1.0
+ * Description:   Parameter handling for the IrCOMM protocol
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Mon Jun  7 10:25:11 1999
+ * Modified at:   Sun Jan 30 14:32:03 2000
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1999-2000 Dag Brattli, All Rights Reserved.
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ *
+ *     You should have received a copy of the GNU General Public License
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ ********************************************************************/
+
+#include <linux/gfp.h>
+#include <linux/workqueue.h>
+#include <linux/interrupt.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/parameters.h>
+
+#include <net/irda/ircomm_core.h>
+#include <net/irda/ircomm_tty_attach.h>
+#include <net/irda/ircomm_tty.h>
+
+#include <net/irda/ircomm_param.h>
+
+static int ircomm_param_service_type(void *instance, irda_param_t *param,
+				     int get);
+static int ircomm_param_port_type(void *instance, irda_param_t *param,
+				  int get);
+static int ircomm_param_port_name(void *instance, irda_param_t *param,
+				  int get);
+static int ircomm_param_service_type(void *instance, irda_param_t *param,
+				     int get);
+static int ircomm_param_data_rate(void *instance, irda_param_t *param,
+				  int get);
+static int ircomm_param_data_format(void *instance, irda_param_t *param,
+				    int get);
+static int ircomm_param_flow_control(void *instance, irda_param_t *param,
+				     int get);
+static int ircomm_param_xon_xoff(void *instance, irda_param_t *param, int get);
+static int ircomm_param_enq_ack(void *instance, irda_param_t *param, int get);
+static int ircomm_param_line_status(void *instance, irda_param_t *param,
+				    int get);
+static int ircomm_param_dte(void *instance, irda_param_t *param, int get);
+static int ircomm_param_dce(void *instance, irda_param_t *param, int get);
+static int ircomm_param_poll(void *instance, irda_param_t *param, int get);
+
+static const pi_minor_info_t pi_minor_call_table_common[] = {
+	{ ircomm_param_service_type, PV_INT_8_BITS },
+	{ ircomm_param_port_type,    PV_INT_8_BITS },
+	{ ircomm_param_port_name,    PV_STRING }
+};
+static const pi_minor_info_t pi_minor_call_table_non_raw[] = {
+	{ ircomm_param_data_rate,    PV_INT_32_BITS | PV_BIG_ENDIAN },
+	{ ircomm_param_data_format,  PV_INT_8_BITS },
+	{ ircomm_param_flow_control, PV_INT_8_BITS },
+	{ ircomm_param_xon_xoff,     PV_INT_16_BITS },
+	{ ircomm_param_enq_ack,      PV_INT_16_BITS },
+	{ ircomm_param_line_status,  PV_INT_8_BITS }
+};
+static const pi_minor_info_t pi_minor_call_table_9_wire[] = {
+	{ ircomm_param_dte,          PV_INT_8_BITS },
+	{ ircomm_param_dce,          PV_INT_8_BITS },
+	{ ircomm_param_poll,         PV_NO_VALUE },
+};
+
+static const pi_major_info_t pi_major_call_table[] = {
+	{ pi_minor_call_table_common,  3 },
+	{ pi_minor_call_table_non_raw, 6 },
+	{ pi_minor_call_table_9_wire,  3 }
+/* 	{ pi_minor_call_table_centronics }  */
+};
+
+pi_param_info_t ircomm_param_info = { pi_major_call_table, 3, 0x0f, 4 };
+
+/*
+ * Function ircomm_param_request (self, pi, flush)
+ *
+ *    Queue a parameter for the control channel
+ *
+ */
+int ircomm_param_request(struct ircomm_tty_cb *self, __u8 pi, int flush)
+{
+	unsigned long flags;
+	struct sk_buff *skb;
+	int count;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return -1;);
+
+	/* Make sure we don't send parameters for raw mode */
+	if (self->service_type == IRCOMM_3_WIRE_RAW)
+		return 0;
+
+	spin_lock_irqsave(&self->spinlock, flags);
+
+	skb = self->ctrl_skb;
+	if (!skb) {
+		skb = alloc_skb(256, GFP_ATOMIC);
+		if (!skb) {
+			spin_unlock_irqrestore(&self->spinlock, flags);
+			return -ENOMEM;
+		}
+
+		skb_reserve(skb, self->max_header_size);
+		self->ctrl_skb = skb;
+	}
+	/*
+	 * Inserting is a little bit tricky since we don't know how much
+	 * room we will need. But this should hopefully work OK
+	 */
+	count = irda_param_insert(self, pi, skb_tail_pointer(skb),
+				  skb_tailroom(skb), &ircomm_param_info);
+	if (count < 0) {
+		net_warn_ratelimited("%s(), no room for parameter!\n",
+				     __func__);
+		spin_unlock_irqrestore(&self->spinlock, flags);
+		return -1;
+	}
+	skb_put(skb, count);
+	pr_debug("%s(), skb->len=%d\n", __func__, skb->len);
+
+	spin_unlock_irqrestore(&self->spinlock, flags);
+
+	if (flush) {
+		/* ircomm_tty_do_softint will take care of the rest */
+		schedule_work(&self->tqueue);
+	}
+
+	return count;
+}
+
+/*
+ * Function ircomm_param_service_type (self, buf, len)
+ *
+ *    Handle service type, this function will both be called after the LM-IAS
+ *    query and then the remote device sends its initial parameters
+ *
+ */
+static int ircomm_param_service_type(void *instance, irda_param_t *param,
+				     int get)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) instance;
+	__u8 service_type = (__u8) param->pv.i;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return -1;);
+
+	if (get) {
+		param->pv.i = self->settings.service_type;
+		return 0;
+	}
+
+	/* Find all common service types */
+	service_type &= self->service_type;
+	if (!service_type) {
+		pr_debug("%s(), No common service type to use!\n", __func__);
+		return -1;
+	}
+	pr_debug("%s(), services in common=%02x\n", __func__ ,
+		 service_type);
+
+	/*
+	 * Now choose a preferred service type of those available
+	 */
+	if (service_type & IRCOMM_CENTRONICS)
+		self->settings.service_type = IRCOMM_CENTRONICS;
+	else if (service_type & IRCOMM_9_WIRE)
+		self->settings.service_type = IRCOMM_9_WIRE;
+	else if (service_type & IRCOMM_3_WIRE)
+		self->settings.service_type = IRCOMM_3_WIRE;
+	else if (service_type & IRCOMM_3_WIRE_RAW)
+		self->settings.service_type = IRCOMM_3_WIRE_RAW;
+
+	pr_debug("%s(), resulting service type=0x%02x\n", __func__ ,
+		 self->settings.service_type);
+
+	/*
+	 * Now the line is ready for some communication. Check if we are a
+	 * server, and send over some initial parameters.
+	 * Client do it in ircomm_tty_state_setup().
+	 * Note : we may get called from ircomm_tty_getvalue_confirm(),
+	 * therefore before we even have open any socket. And self->client
+	 * is initialised to TRUE only later. So, we check if the link is
+	 * really initialised. - Jean II
+	 */
+	if ((self->max_header_size != IRCOMM_TTY_HDR_UNINITIALISED) &&
+	    (!self->client) &&
+	    (self->settings.service_type != IRCOMM_3_WIRE_RAW))
+	{
+		/* Init connection */
+		ircomm_tty_send_initial_parameters(self);
+		ircomm_tty_link_established(self);
+	}
+
+	return 0;
+}
+
+/*
+ * Function ircomm_param_port_type (self, param)
+ *
+ *    The port type parameter tells if the devices are serial or parallel.
+ *    Since we only advertise serial service, this parameter should only
+ *    be equal to IRCOMM_SERIAL.
+ */
+static int ircomm_param_port_type(void *instance, irda_param_t *param, int get)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) instance;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return -1;);
+
+	if (get)
+		param->pv.i = IRCOMM_SERIAL;
+	else {
+		self->settings.port_type = (__u8) param->pv.i;
+
+		pr_debug("%s(), port type=%d\n", __func__ ,
+			 self->settings.port_type);
+	}
+	return 0;
+}
+
+/*
+ * Function ircomm_param_port_name (self, param)
+ *
+ *    Exchange port name
+ *
+ */
+static int ircomm_param_port_name(void *instance, irda_param_t *param, int get)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) instance;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return -1;);
+
+	if (get) {
+		pr_debug("%s(), not imp!\n", __func__);
+	} else {
+		pr_debug("%s(), port-name=%s\n", __func__ , param->pv.c);
+		strncpy(self->settings.port_name, param->pv.c, 32);
+	}
+
+	return 0;
+}
+
+/*
+ * Function ircomm_param_data_rate (self, param)
+ *
+ *    Exchange data rate to be used in this settings
+ *
+ */
+static int ircomm_param_data_rate(void *instance, irda_param_t *param, int get)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) instance;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return -1;);
+
+	if (get)
+		param->pv.i = self->settings.data_rate;
+	else
+		self->settings.data_rate = param->pv.i;
+
+	pr_debug("%s(), data rate = %d\n", __func__ , param->pv.i);
+
+	return 0;
+}
+
+/*
+ * Function ircomm_param_data_format (self, param)
+ *
+ *    Exchange data format to be used in this settings
+ *
+ */
+static int ircomm_param_data_format(void *instance, irda_param_t *param,
+				    int get)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) instance;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return -1;);
+
+	if (get)
+		param->pv.i = self->settings.data_format;
+	else
+		self->settings.data_format = (__u8) param->pv.i;
+
+	return 0;
+}
+
+/*
+ * Function ircomm_param_flow_control (self, param)
+ *
+ *    Exchange flow control settings to be used in this settings
+ *
+ */
+static int ircomm_param_flow_control(void *instance, irda_param_t *param,
+				     int get)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) instance;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return -1;);
+
+	if (get)
+		param->pv.i = self->settings.flow_control;
+	else
+		self->settings.flow_control = (__u8) param->pv.i;
+
+	pr_debug("%s(), flow control = 0x%02x\n", __func__ , (__u8)param->pv.i);
+
+	return 0;
+}
+
+/*
+ * Function ircomm_param_xon_xoff (self, param)
+ *
+ *    Exchange XON/XOFF characters
+ *
+ */
+static int ircomm_param_xon_xoff(void *instance, irda_param_t *param, int get)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) instance;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return -1;);
+
+	if (get) {
+		param->pv.i = self->settings.xonxoff[0];
+		param->pv.i |= self->settings.xonxoff[1] << 8;
+	} else {
+		self->settings.xonxoff[0] = (__u16) param->pv.i & 0xff;
+		self->settings.xonxoff[1] = (__u16) param->pv.i >> 8;
+	}
+
+	pr_debug("%s(), XON/XOFF = 0x%02x,0x%02x\n", __func__ ,
+		 param->pv.i & 0xff, param->pv.i >> 8);
+
+	return 0;
+}
+
+/*
+ * Function ircomm_param_enq_ack (self, param)
+ *
+ *    Exchange ENQ/ACK characters
+ *
+ */
+static int ircomm_param_enq_ack(void *instance, irda_param_t *param, int get)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) instance;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return -1;);
+
+	if (get) {
+		param->pv.i = self->settings.enqack[0];
+		param->pv.i |= self->settings.enqack[1] << 8;
+	} else {
+		self->settings.enqack[0] = (__u16) param->pv.i & 0xff;
+		self->settings.enqack[1] = (__u16) param->pv.i >> 8;
+	}
+
+	pr_debug("%s(), ENQ/ACK = 0x%02x,0x%02x\n", __func__ ,
+		 param->pv.i & 0xff, param->pv.i >> 8);
+
+	return 0;
+}
+
+/*
+ * Function ircomm_param_line_status (self, param)
+ *
+ *
+ *
+ */
+static int ircomm_param_line_status(void *instance, irda_param_t *param,
+				    int get)
+{
+	pr_debug("%s(), not impl.\n", __func__);
+
+	return 0;
+}
+
+/*
+ * Function ircomm_param_dte (instance, param)
+ *
+ *    If we get here, there must be some sort of null-modem connection, and
+ *    we are probably working in server mode as well.
+ */
+static int ircomm_param_dte(void *instance, irda_param_t *param, int get)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) instance;
+	__u8 dte;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return -1;);
+
+	if (get)
+		param->pv.i = self->settings.dte;
+	else {
+		dte = (__u8) param->pv.i;
+
+		self->settings.dce = 0;
+
+		if (dte & IRCOMM_DELTA_DTR)
+			self->settings.dce |= (IRCOMM_DELTA_DSR|
+					      IRCOMM_DELTA_RI |
+					      IRCOMM_DELTA_CD);
+		if (dte & IRCOMM_DTR)
+			self->settings.dce |= (IRCOMM_DSR|
+					      IRCOMM_RI |
+					      IRCOMM_CD);
+
+		if (dte & IRCOMM_DELTA_RTS)
+			self->settings.dce |= IRCOMM_DELTA_CTS;
+		if (dte & IRCOMM_RTS)
+			self->settings.dce |= IRCOMM_CTS;
+
+		/* Take appropriate actions */
+		ircomm_tty_check_modem_status(self);
+
+		/* Null modem cable emulator */
+		self->settings.null_modem = TRUE;
+	}
+
+	return 0;
+}
+
+/*
+ * Function ircomm_param_dce (instance, param)
+ *
+ *
+ *
+ */
+static int ircomm_param_dce(void *instance, irda_param_t *param, int get)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) instance;
+	__u8 dce;
+
+	pr_debug("%s(), dce = 0x%02x\n", __func__ , (__u8)param->pv.i);
+
+	dce = (__u8) param->pv.i;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return -1;);
+
+	self->settings.dce = dce;
+
+	/* Check if any of the settings have changed */
+	if (dce & 0x0f) {
+		if (dce & IRCOMM_DELTA_CTS) {
+			pr_debug("%s(), CTS\n", __func__);
+		}
+	}
+
+	ircomm_tty_check_modem_status(self);
+
+	return 0;
+}
+
+/*
+ * Function ircomm_param_poll (instance, param)
+ *
+ *    Called when the peer device is polling for the line settings
+ *
+ */
+static int ircomm_param_poll(void *instance, irda_param_t *param, int get)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) instance;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return -1;);
+
+	/* Poll parameters are always of length 0 (just a signal) */
+	if (!get) {
+		/* Respond with DTE line settings */
+		ircomm_param_request(self, IRCOMM_DTE, TRUE);
+	}
+	return 0;
+}
+
+
+
+
+
diff --git a/drivers/staging/irda/net/ircomm/ircomm_ttp.c b/drivers/staging/irda/net/ircomm/ircomm_ttp.c
new file mode 100644
index 000000000000..4b81e0934770
--- /dev/null
+++ b/drivers/staging/irda/net/ircomm/ircomm_ttp.c
@@ -0,0 +1,350 @@
+/*********************************************************************
+ *
+ * Filename:      ircomm_ttp.c
+ * Version:       1.0
+ * Description:   Interface between IrCOMM and IrTTP
+ * Status:        Stable
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sun Jun  6 20:48:27 1999
+ * Modified at:   Mon Dec 13 11:35:13 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1999 Dag Brattli, All Rights Reserved.
+ *     Copyright (c) 2000-2003 Jean Tourrilhes <jt@hpl.hp.com>
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ *
+ *     You should have received a copy of the GNU General Public License
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ ********************************************************************/
+
+#include <linux/init.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/irlmp.h>
+#include <net/irda/iriap.h>
+#include <net/irda/irttp.h>
+
+#include <net/irda/ircomm_event.h>
+#include <net/irda/ircomm_ttp.h>
+
+static int ircomm_ttp_data_indication(void *instance, void *sap,
+				      struct sk_buff *skb);
+static void ircomm_ttp_connect_confirm(void *instance, void *sap,
+				       struct qos_info *qos,
+				       __u32 max_sdu_size,
+				       __u8 max_header_size,
+				       struct sk_buff *skb);
+static void ircomm_ttp_connect_indication(void *instance, void *sap,
+					  struct qos_info *qos,
+					  __u32 max_sdu_size,
+					  __u8 max_header_size,
+					  struct sk_buff *skb);
+static void ircomm_ttp_flow_indication(void *instance, void *sap,
+				       LOCAL_FLOW cmd);
+static void ircomm_ttp_disconnect_indication(void *instance, void *sap,
+					     LM_REASON reason,
+					     struct sk_buff *skb);
+static int ircomm_ttp_data_request(struct ircomm_cb *self,
+				   struct sk_buff *skb,
+				   int clen);
+static int ircomm_ttp_connect_request(struct ircomm_cb *self,
+				      struct sk_buff *userdata,
+				      struct ircomm_info *info);
+static int ircomm_ttp_connect_response(struct ircomm_cb *self,
+				       struct sk_buff *userdata);
+static int ircomm_ttp_disconnect_request(struct ircomm_cb *self,
+					 struct sk_buff *userdata,
+					 struct ircomm_info *info);
+
+/*
+ * Function ircomm_open_tsap (self)
+ *
+ *
+ *
+ */
+int ircomm_open_tsap(struct ircomm_cb *self)
+{
+	notify_t notify;
+
+	/* Register callbacks */
+	irda_notify_init(&notify);
+	notify.data_indication       = ircomm_ttp_data_indication;
+	notify.connect_confirm       = ircomm_ttp_connect_confirm;
+	notify.connect_indication    = ircomm_ttp_connect_indication;
+	notify.flow_indication       = ircomm_ttp_flow_indication;
+	notify.disconnect_indication = ircomm_ttp_disconnect_indication;
+	notify.instance = self;
+	strlcpy(notify.name, "IrCOMM", sizeof(notify.name));
+
+	self->tsap = irttp_open_tsap(LSAP_ANY, DEFAULT_INITIAL_CREDIT,
+				     &notify);
+	if (!self->tsap) {
+		pr_debug("%sfailed to allocate tsap\n", __func__);
+		return -1;
+	}
+	self->slsap_sel = self->tsap->stsap_sel;
+
+	/*
+	 *  Initialize the call-table for issuing commands
+	 */
+	self->issue.data_request       = ircomm_ttp_data_request;
+	self->issue.connect_request    = ircomm_ttp_connect_request;
+	self->issue.connect_response   = ircomm_ttp_connect_response;
+	self->issue.disconnect_request = ircomm_ttp_disconnect_request;
+
+	return 0;
+}
+
+/*
+ * Function ircomm_ttp_connect_request (self, userdata)
+ *
+ *
+ *
+ */
+static int ircomm_ttp_connect_request(struct ircomm_cb *self,
+				      struct sk_buff *userdata,
+				      struct ircomm_info *info)
+{
+	int ret = 0;
+
+	/* Don't forget to refcount it - should be NULL anyway */
+	if(userdata)
+		skb_get(userdata);
+
+	ret = irttp_connect_request(self->tsap, info->dlsap_sel,
+				    info->saddr, info->daddr, NULL,
+				    TTP_SAR_DISABLE, userdata);
+
+	return ret;
+}
+
+/*
+ * Function ircomm_ttp_connect_response (self, skb)
+ *
+ *
+ *
+ */
+static int ircomm_ttp_connect_response(struct ircomm_cb *self,
+				       struct sk_buff *userdata)
+{
+	int ret;
+
+	/* Don't forget to refcount it - should be NULL anyway */
+	if(userdata)
+		skb_get(userdata);
+
+	ret = irttp_connect_response(self->tsap, TTP_SAR_DISABLE, userdata);
+
+	return ret;
+}
+
+/*
+ * Function ircomm_ttp_data_request (self, userdata)
+ *
+ *    Send IrCOMM data to IrTTP layer. Currently we do not try to combine
+ *    control data with pure data, so they will be sent as separate frames.
+ *    Should not be a big problem though, since control frames are rare. But
+ *    some of them are sent after connection establishment, so this can
+ *    increase the latency a bit.
+ */
+static int ircomm_ttp_data_request(struct ircomm_cb *self,
+				   struct sk_buff *skb,
+				   int clen)
+{
+	int ret;
+
+	IRDA_ASSERT(skb != NULL, return -1;);
+
+	pr_debug("%s(), clen=%d\n", __func__ , clen);
+
+	/*
+	 * Insert clen field, currently we either send data only, or control
+	 * only frames, to make things easier and avoid queueing
+	 */
+	IRDA_ASSERT(skb_headroom(skb) >= IRCOMM_HEADER_SIZE, return -1;);
+
+	/* Don't forget to refcount it - see ircomm_tty_do_softint() */
+	skb_get(skb);
+
+	skb_push(skb, IRCOMM_HEADER_SIZE);
+
+	skb->data[0] = clen;
+
+	ret = irttp_data_request(self->tsap, skb);
+	if (ret) {
+		net_err_ratelimited("%s(), failed\n", __func__);
+		/* irttp_data_request already free the packet */
+	}
+
+	return ret;
+}
+
+/*
+ * Function ircomm_ttp_data_indication (instance, sap, skb)
+ *
+ *    Incoming data
+ *
+ */
+static int ircomm_ttp_data_indication(void *instance, void *sap,
+				      struct sk_buff *skb)
+{
+	struct ircomm_cb *self = (struct ircomm_cb *) instance;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_MAGIC, return -1;);
+	IRDA_ASSERT(skb != NULL, return -1;);
+
+	ircomm_do_event(self, IRCOMM_TTP_DATA_INDICATION, skb, NULL);
+
+	/* Drop reference count - see ircomm_tty_data_indication(). */
+	dev_kfree_skb(skb);
+
+	return 0;
+}
+
+static void ircomm_ttp_connect_confirm(void *instance, void *sap,
+				       struct qos_info *qos,
+				       __u32 max_sdu_size,
+				       __u8 max_header_size,
+				       struct sk_buff *skb)
+{
+	struct ircomm_cb *self = (struct ircomm_cb *) instance;
+	struct ircomm_info info;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_MAGIC, return;);
+	IRDA_ASSERT(skb != NULL, return;);
+	IRDA_ASSERT(qos != NULL, goto out;);
+
+	if (max_sdu_size != TTP_SAR_DISABLE) {
+		net_err_ratelimited("%s(), SAR not allowed for IrCOMM!\n",
+				    __func__);
+		goto out;
+	}
+
+	info.max_data_size = irttp_get_max_seg_size(self->tsap)
+		- IRCOMM_HEADER_SIZE;
+	info.max_header_size = max_header_size + IRCOMM_HEADER_SIZE;
+	info.qos = qos;
+
+	ircomm_do_event(self, IRCOMM_TTP_CONNECT_CONFIRM, skb, &info);
+
+out:
+	/* Drop reference count - see ircomm_tty_connect_confirm(). */
+	dev_kfree_skb(skb);
+}
+
+/*
+ * Function ircomm_ttp_connect_indication (instance, sap, qos, max_sdu_size,
+ *                                         max_header_size, skb)
+ *
+ *
+ *
+ */
+static void ircomm_ttp_connect_indication(void *instance, void *sap,
+					  struct qos_info *qos,
+					  __u32 max_sdu_size,
+					  __u8 max_header_size,
+					  struct sk_buff *skb)
+{
+	struct ircomm_cb *self = (struct ircomm_cb *)instance;
+	struct ircomm_info info;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_MAGIC, return;);
+	IRDA_ASSERT(skb != NULL, return;);
+	IRDA_ASSERT(qos != NULL, goto out;);
+
+	if (max_sdu_size != TTP_SAR_DISABLE) {
+		net_err_ratelimited("%s(), SAR not allowed for IrCOMM!\n",
+				    __func__);
+		goto out;
+	}
+
+	info.max_data_size = irttp_get_max_seg_size(self->tsap)
+		- IRCOMM_HEADER_SIZE;
+	info.max_header_size = max_header_size + IRCOMM_HEADER_SIZE;
+	info.qos = qos;
+
+	ircomm_do_event(self, IRCOMM_TTP_CONNECT_INDICATION, skb, &info);
+
+out:
+	/* Drop reference count - see ircomm_tty_connect_indication(). */
+	dev_kfree_skb(skb);
+}
+
+/*
+ * Function ircomm_ttp_disconnect_request (self, userdata, info)
+ *
+ *
+ *
+ */
+static int ircomm_ttp_disconnect_request(struct ircomm_cb *self,
+					 struct sk_buff *userdata,
+					 struct ircomm_info *info)
+{
+	int ret;
+
+	/* Don't forget to refcount it - should be NULL anyway */
+	if(userdata)
+		skb_get(userdata);
+
+	ret = irttp_disconnect_request(self->tsap, userdata, P_NORMAL);
+
+	return ret;
+}
+
+/*
+ * Function ircomm_ttp_disconnect_indication (instance, sap, reason, skb)
+ *
+ *
+ *
+ */
+static void ircomm_ttp_disconnect_indication(void *instance, void *sap,
+					     LM_REASON reason,
+					     struct sk_buff *skb)
+{
+	struct ircomm_cb *self = (struct ircomm_cb *) instance;
+	struct ircomm_info info;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_MAGIC, return;);
+
+	info.reason = reason;
+
+	ircomm_do_event(self, IRCOMM_TTP_DISCONNECT_INDICATION, skb, &info);
+
+	/* Drop reference count - see ircomm_tty_disconnect_indication(). */
+	if(skb)
+		dev_kfree_skb(skb);
+}
+
+/*
+ * Function ircomm_ttp_flow_indication (instance, sap, cmd)
+ *
+ *    Layer below is telling us to start or stop the flow of data
+ *
+ */
+static void ircomm_ttp_flow_indication(void *instance, void *sap,
+				       LOCAL_FLOW cmd)
+{
+	struct ircomm_cb *self = (struct ircomm_cb *) instance;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_MAGIC, return;);
+
+	if (self->notify.flow_indication)
+		self->notify.flow_indication(self->notify.instance, self, cmd);
+}
+
+
diff --git a/drivers/staging/irda/net/ircomm/ircomm_tty.c b/drivers/staging/irda/net/ircomm/ircomm_tty.c
new file mode 100644
index 000000000000..ec157c3419b5
--- /dev/null
+++ b/drivers/staging/irda/net/ircomm/ircomm_tty.c
@@ -0,0 +1,1329 @@
+/*********************************************************************
+ *
+ * Filename:      ircomm_tty.c
+ * Version:       1.0
+ * Description:   IrCOMM serial TTY driver
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sun Jun  6 21:00:56 1999
+ * Modified at:   Wed Feb 23 00:09:02 2000
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * Sources:       serial.c and previous IrCOMM work by Takahide Higuchi
+ *
+ *     Copyright (c) 1999-2000 Dag Brattli, All Rights Reserved.
+ *     Copyright (c) 2000-2003 Jean Tourrilhes <jt@hpl.hp.com>
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ *
+ *     You should have received a copy of the GNU General Public License
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ ********************************************************************/
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/fs.h>
+#include <linux/slab.h>
+#include <linux/sched/signal.h>
+#include <linux/seq_file.h>
+#include <linux/termios.h>
+#include <linux/tty.h>
+#include <linux/tty_flip.h>
+#include <linux/interrupt.h>
+#include <linux/device.h>		/* for MODULE_ALIAS_CHARDEV_MAJOR */
+
+#include <linux/uaccess.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/irmod.h>
+
+#include <net/irda/ircomm_core.h>
+#include <net/irda/ircomm_param.h>
+#include <net/irda/ircomm_tty_attach.h>
+#include <net/irda/ircomm_tty.h>
+
+static int ircomm_tty_install(struct tty_driver *driver,
+		struct tty_struct *tty);
+static int  ircomm_tty_open(struct tty_struct *tty, struct file *filp);
+static void ircomm_tty_close(struct tty_struct * tty, struct file *filp);
+static int  ircomm_tty_write(struct tty_struct * tty,
+			     const unsigned char *buf, int count);
+static int  ircomm_tty_write_room(struct tty_struct *tty);
+static void ircomm_tty_throttle(struct tty_struct *tty);
+static void ircomm_tty_unthrottle(struct tty_struct *tty);
+static int  ircomm_tty_chars_in_buffer(struct tty_struct *tty);
+static void ircomm_tty_flush_buffer(struct tty_struct *tty);
+static void ircomm_tty_send_xchar(struct tty_struct *tty, char ch);
+static void ircomm_tty_wait_until_sent(struct tty_struct *tty, int timeout);
+static void ircomm_tty_hangup(struct tty_struct *tty);
+static void ircomm_tty_do_softint(struct work_struct *work);
+static void ircomm_tty_shutdown(struct ircomm_tty_cb *self);
+static void ircomm_tty_stop(struct tty_struct *tty);
+
+static int ircomm_tty_data_indication(void *instance, void *sap,
+				      struct sk_buff *skb);
+static int ircomm_tty_control_indication(void *instance, void *sap,
+					 struct sk_buff *skb);
+static void ircomm_tty_flow_indication(void *instance, void *sap,
+				       LOCAL_FLOW cmd);
+#ifdef CONFIG_PROC_FS
+static const struct file_operations ircomm_tty_proc_fops;
+#endif /* CONFIG_PROC_FS */
+static struct tty_driver *driver;
+
+static hashbin_t *ircomm_tty = NULL;
+
+static const struct tty_operations ops = {
+	.install	 = ircomm_tty_install,
+	.open            = ircomm_tty_open,
+	.close           = ircomm_tty_close,
+	.write           = ircomm_tty_write,
+	.write_room      = ircomm_tty_write_room,
+	.chars_in_buffer = ircomm_tty_chars_in_buffer,
+	.flush_buffer    = ircomm_tty_flush_buffer,
+	.ioctl           = ircomm_tty_ioctl,	/* ircomm_tty_ioctl.c */
+	.tiocmget        = ircomm_tty_tiocmget,	/* ircomm_tty_ioctl.c */
+	.tiocmset        = ircomm_tty_tiocmset,	/* ircomm_tty_ioctl.c */
+	.throttle        = ircomm_tty_throttle,
+	.unthrottle      = ircomm_tty_unthrottle,
+	.send_xchar      = ircomm_tty_send_xchar,
+	.set_termios     = ircomm_tty_set_termios,
+	.stop            = ircomm_tty_stop,
+	.start           = ircomm_tty_start,
+	.hangup          = ircomm_tty_hangup,
+	.wait_until_sent = ircomm_tty_wait_until_sent,
+#ifdef CONFIG_PROC_FS
+	.proc_fops       = &ircomm_tty_proc_fops,
+#endif /* CONFIG_PROC_FS */
+};
+
+static void ircomm_port_raise_dtr_rts(struct tty_port *port, int raise)
+{
+	struct ircomm_tty_cb *self = container_of(port, struct ircomm_tty_cb,
+			port);
+	/*
+	 * Here, we use to lock those two guys, but as ircomm_param_request()
+	 * does it itself, I don't see the point (and I see the deadlock).
+	 * Jean II
+	 */
+	if (raise)
+		self->settings.dte |= IRCOMM_RTS | IRCOMM_DTR;
+	else
+		self->settings.dte &= ~(IRCOMM_RTS | IRCOMM_DTR);
+
+	ircomm_param_request(self, IRCOMM_DTE, TRUE);
+}
+
+static int ircomm_port_carrier_raised(struct tty_port *port)
+{
+	struct ircomm_tty_cb *self = container_of(port, struct ircomm_tty_cb,
+			port);
+	return self->settings.dce & IRCOMM_CD;
+}
+
+static const struct tty_port_operations ircomm_port_ops = {
+	.dtr_rts = ircomm_port_raise_dtr_rts,
+	.carrier_raised = ircomm_port_carrier_raised,
+};
+
+/*
+ * Function ircomm_tty_init()
+ *
+ *    Init IrCOMM TTY layer/driver
+ *
+ */
+static int __init ircomm_tty_init(void)
+{
+	driver = alloc_tty_driver(IRCOMM_TTY_PORTS);
+	if (!driver)
+		return -ENOMEM;
+	ircomm_tty = hashbin_new(HB_LOCK);
+	if (ircomm_tty == NULL) {
+		net_err_ratelimited("%s(), can't allocate hashbin!\n",
+				    __func__);
+		put_tty_driver(driver);
+		return -ENOMEM;
+	}
+
+	driver->driver_name     = "ircomm";
+	driver->name            = "ircomm";
+	driver->major           = IRCOMM_TTY_MAJOR;
+	driver->minor_start     = IRCOMM_TTY_MINOR;
+	driver->type            = TTY_DRIVER_TYPE_SERIAL;
+	driver->subtype         = SERIAL_TYPE_NORMAL;
+	driver->init_termios    = tty_std_termios;
+	driver->init_termios.c_cflag = B9600 | CS8 | CREAD | HUPCL | CLOCAL;
+	driver->flags           = TTY_DRIVER_REAL_RAW;
+	tty_set_operations(driver, &ops);
+	if (tty_register_driver(driver)) {
+		net_err_ratelimited("%s(): Couldn't register serial driver\n",
+				    __func__);
+		put_tty_driver(driver);
+		return -1;
+	}
+	return 0;
+}
+
+static void __exit __ircomm_tty_cleanup(struct ircomm_tty_cb *self)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return;);
+
+	ircomm_tty_shutdown(self);
+
+	self->magic = 0;
+	tty_port_destroy(&self->port);
+	kfree(self);
+}
+
+/*
+ * Function ircomm_tty_cleanup ()
+ *
+ *    Remove IrCOMM TTY layer/driver
+ *
+ */
+static void __exit ircomm_tty_cleanup(void)
+{
+	int ret;
+
+	ret = tty_unregister_driver(driver);
+	if (ret) {
+		net_err_ratelimited("%s(), failed to unregister driver\n",
+				    __func__);
+		return;
+	}
+
+	hashbin_delete(ircomm_tty, (FREE_FUNC) __ircomm_tty_cleanup);
+	put_tty_driver(driver);
+}
+
+/*
+ * Function ircomm_startup (self)
+ *
+ *
+ *
+ */
+static int ircomm_tty_startup(struct ircomm_tty_cb *self)
+{
+	notify_t notify;
+	int ret = -ENODEV;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return -1;);
+
+	/* Check if already open */
+	if (tty_port_initialized(&self->port)) {
+		pr_debug("%s(), already open so break out!\n", __func__);
+		return 0;
+	}
+	tty_port_set_initialized(&self->port, 1);
+
+	/* Register with IrCOMM */
+	irda_notify_init(&notify);
+	/* These callbacks we must handle ourselves */
+	notify.data_indication       = ircomm_tty_data_indication;
+	notify.udata_indication      = ircomm_tty_control_indication;
+	notify.flow_indication       = ircomm_tty_flow_indication;
+
+	/* Use the ircomm_tty interface for these ones */
+	notify.disconnect_indication = ircomm_tty_disconnect_indication;
+	notify.connect_confirm       = ircomm_tty_connect_confirm;
+	notify.connect_indication    = ircomm_tty_connect_indication;
+	strlcpy(notify.name, "ircomm_tty", sizeof(notify.name));
+	notify.instance = self;
+
+	if (!self->ircomm) {
+		self->ircomm = ircomm_open(&notify, self->service_type,
+					   self->line);
+	}
+	if (!self->ircomm)
+		goto err;
+
+	self->slsap_sel = self->ircomm->slsap_sel;
+
+	/* Connect IrCOMM link with remote device */
+	ret = ircomm_tty_attach_cable(self);
+	if (ret < 0) {
+		net_err_ratelimited("%s(), error attaching cable!\n", __func__);
+		goto err;
+	}
+
+	return 0;
+err:
+	tty_port_set_initialized(&self->port, 0);
+	return ret;
+}
+
+/*
+ * Function ircomm_block_til_ready (self, filp)
+ *
+ *
+ *
+ */
+static int ircomm_tty_block_til_ready(struct ircomm_tty_cb *self,
+		struct tty_struct *tty, struct file *filp)
+{
+	struct tty_port *port = &self->port;
+	DECLARE_WAITQUEUE(wait, current);
+	int		retval;
+	int		do_clocal = 0;
+	unsigned long	flags;
+
+	/*
+	 * If non-blocking mode is set, or the port is not enabled,
+	 * then make the check up front and then exit.
+	 */
+	if (tty_io_error(tty)) {
+		tty_port_set_active(port, 1);
+		return 0;
+	}
+
+	if (filp->f_flags & O_NONBLOCK) {
+		/* nonblock mode is set */
+		if (C_BAUD(tty))
+			tty_port_raise_dtr_rts(port);
+		tty_port_set_active(port, 1);
+		pr_debug("%s(), O_NONBLOCK requested!\n", __func__);
+		return 0;
+	}
+
+	if (C_CLOCAL(tty)) {
+		pr_debug("%s(), doing CLOCAL!\n", __func__);
+		do_clocal = 1;
+	}
+
+	/* Wait for carrier detect and the line to become
+	 * free (i.e., not in use by the callout).  While we are in
+	 * this loop, port->count is dropped by one, so that
+	 * mgsl_close() knows when to free things.  We restore it upon
+	 * exit, either normal or abnormal.
+	 */
+
+	retval = 0;
+	add_wait_queue(&port->open_wait, &wait);
+
+	pr_debug("%s(%d):block_til_ready before block on %s open_count=%d\n",
+		 __FILE__, __LINE__, tty->driver->name, port->count);
+
+	spin_lock_irqsave(&port->lock, flags);
+	port->count--;
+	port->blocked_open++;
+	spin_unlock_irqrestore(&port->lock, flags);
+
+	while (1) {
+		if (C_BAUD(tty) && tty_port_initialized(port))
+			tty_port_raise_dtr_rts(port);
+
+		set_current_state(TASK_INTERRUPTIBLE);
+
+		if (tty_hung_up_p(filp) || !tty_port_initialized(port)) {
+			retval = (port->flags & ASYNC_HUP_NOTIFY) ?
+					-EAGAIN : -ERESTARTSYS;
+			break;
+		}
+
+		/*
+		 * Check if link is ready now. Even if CLOCAL is
+		 * specified, we cannot return before the IrCOMM link is
+		 * ready
+		 */
+		if ((do_clocal || tty_port_carrier_raised(port)) &&
+		    self->state == IRCOMM_TTY_READY)
+		{
+			break;
+		}
+
+		if (signal_pending(current)) {
+			retval = -ERESTARTSYS;
+			break;
+		}
+
+		pr_debug("%s(%d):block_til_ready blocking on %s open_count=%d\n",
+			 __FILE__, __LINE__, tty->driver->name, port->count);
+
+		schedule();
+	}
+
+	__set_current_state(TASK_RUNNING);
+	remove_wait_queue(&port->open_wait, &wait);
+
+	spin_lock_irqsave(&port->lock, flags);
+	if (!tty_hung_up_p(filp))
+		port->count++;
+	port->blocked_open--;
+	spin_unlock_irqrestore(&port->lock, flags);
+
+	pr_debug("%s(%d):block_til_ready after blocking on %s open_count=%d\n",
+		 __FILE__, __LINE__, tty->driver->name, port->count);
+
+	if (!retval)
+		tty_port_set_active(port, 1);
+
+	return retval;
+}
+
+
+static int ircomm_tty_install(struct tty_driver *driver, struct tty_struct *tty)
+{
+	struct ircomm_tty_cb *self;
+	unsigned int line = tty->index;
+
+	/* Check if instance already exists */
+	self = hashbin_lock_find(ircomm_tty, line, NULL);
+	if (!self) {
+		/* No, so make new instance */
+		self = kzalloc(sizeof(struct ircomm_tty_cb), GFP_KERNEL);
+		if (self == NULL)
+			return -ENOMEM;
+
+		tty_port_init(&self->port);
+		self->port.ops = &ircomm_port_ops;
+		self->magic = IRCOMM_TTY_MAGIC;
+		self->flow = FLOW_STOP;
+
+		self->line = line;
+		INIT_WORK(&self->tqueue, ircomm_tty_do_softint);
+		self->max_header_size = IRCOMM_TTY_HDR_UNINITIALISED;
+		self->max_data_size = IRCOMM_TTY_DATA_UNINITIALISED;
+
+		/* Init some important stuff */
+		init_timer(&self->watchdog_timer);
+		spin_lock_init(&self->spinlock);
+
+		/*
+		 * Force TTY into raw mode by default which is usually what
+		 * we want for IrCOMM and IrLPT. This way applications will
+		 * not have to twiddle with printcap etc.
+		 *
+		 * Note this is completely usafe and doesn't work properly
+		 */
+		tty->termios.c_iflag = 0;
+		tty->termios.c_oflag = 0;
+
+		/* Insert into hash */
+		hashbin_insert(ircomm_tty, (irda_queue_t *) self, line, NULL);
+	}
+
+	tty->driver_data = self;
+
+	return tty_port_install(&self->port, driver, tty);
+}
+
+/*
+ * Function ircomm_tty_open (tty, filp)
+ *
+ *    This routine is called when a particular tty device is opened. This
+ *    routine is mandatory; if this routine is not filled in, the attempted
+ *    open will fail with ENODEV.
+ */
+static int ircomm_tty_open(struct tty_struct *tty, struct file *filp)
+{
+	struct ircomm_tty_cb *self = tty->driver_data;
+	unsigned long	flags;
+	int ret;
+
+	/* ++ is not atomic, so this should be protected - Jean II */
+	spin_lock_irqsave(&self->port.lock, flags);
+	self->port.count++;
+	spin_unlock_irqrestore(&self->port.lock, flags);
+	tty_port_tty_set(&self->port, tty);
+
+	pr_debug("%s(), %s%d, count = %d\n", __func__ , tty->driver->name,
+		 self->line, self->port.count);
+
+	/* Not really used by us, but lets do it anyway */
+	self->port.low_latency = (self->port.flags & ASYNC_LOW_LATENCY) ? 1 : 0;
+
+	/* Check if this is a "normal" ircomm device, or an irlpt device */
+	if (self->line < 0x10) {
+		self->service_type = IRCOMM_3_WIRE | IRCOMM_9_WIRE;
+		self->settings.service_type = IRCOMM_9_WIRE; /* 9 wire as default */
+		/* Jan Kiszka -> add DSR/RI -> Conform to IrCOMM spec */
+		self->settings.dce = IRCOMM_CTS | IRCOMM_CD | IRCOMM_DSR | IRCOMM_RI; /* Default line settings */
+		pr_debug("%s(), IrCOMM device\n", __func__);
+	} else {
+		pr_debug("%s(), IrLPT device\n", __func__);
+		self->service_type = IRCOMM_3_WIRE_RAW;
+		self->settings.service_type = IRCOMM_3_WIRE_RAW; /* Default */
+	}
+
+	ret = ircomm_tty_startup(self);
+	if (ret)
+		return ret;
+
+	ret = ircomm_tty_block_til_ready(self, tty, filp);
+	if (ret) {
+		pr_debug("%s(), returning after block_til_ready with %d\n",
+			 __func__, ret);
+
+		return ret;
+	}
+	return 0;
+}
+
+/*
+ * Function ircomm_tty_close (tty, filp)
+ *
+ *    This routine is called when a particular tty device is closed.
+ *
+ */
+static void ircomm_tty_close(struct tty_struct *tty, struct file *filp)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) tty->driver_data;
+	struct tty_port *port = &self->port;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return;);
+
+	if (tty_port_close_start(port, tty, filp) == 0)
+		return;
+
+	ircomm_tty_shutdown(self);
+
+	tty_driver_flush_buffer(tty);
+
+	tty_port_close_end(port, tty);
+	tty_port_tty_set(port, NULL);
+}
+
+/*
+ * Function ircomm_tty_flush_buffer (tty)
+ *
+ *
+ *
+ */
+static void ircomm_tty_flush_buffer(struct tty_struct *tty)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) tty->driver_data;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return;);
+
+	/*
+	 * Let do_softint() do this to avoid race condition with
+	 * do_softint() ;-)
+	 */
+	schedule_work(&self->tqueue);
+}
+
+/*
+ * Function ircomm_tty_do_softint (work)
+ *
+ *    We use this routine to give the write wakeup to the user at at a
+ *    safe time (as fast as possible after write have completed). This
+ *    can be compared to the Tx interrupt.
+ */
+static void ircomm_tty_do_softint(struct work_struct *work)
+{
+	struct ircomm_tty_cb *self =
+		container_of(work, struct ircomm_tty_cb, tqueue);
+	struct tty_struct *tty;
+	unsigned long flags;
+	struct sk_buff *skb, *ctrl_skb;
+
+	if (!self || self->magic != IRCOMM_TTY_MAGIC)
+		return;
+
+	tty = tty_port_tty_get(&self->port);
+	if (!tty)
+		return;
+
+	/* Unlink control buffer */
+	spin_lock_irqsave(&self->spinlock, flags);
+
+	ctrl_skb = self->ctrl_skb;
+	self->ctrl_skb = NULL;
+
+	spin_unlock_irqrestore(&self->spinlock, flags);
+
+	/* Flush control buffer if any */
+	if(ctrl_skb) {
+		if(self->flow == FLOW_START)
+			ircomm_control_request(self->ircomm, ctrl_skb);
+		/* Drop reference count - see ircomm_ttp_data_request(). */
+		dev_kfree_skb(ctrl_skb);
+	}
+
+	if (tty->hw_stopped)
+		goto put;
+
+	/* Unlink transmit buffer */
+	spin_lock_irqsave(&self->spinlock, flags);
+
+	skb = self->tx_skb;
+	self->tx_skb = NULL;
+
+	spin_unlock_irqrestore(&self->spinlock, flags);
+
+	/* Flush transmit buffer if any */
+	if (skb) {
+		ircomm_tty_do_event(self, IRCOMM_TTY_DATA_REQUEST, skb, NULL);
+		/* Drop reference count - see ircomm_ttp_data_request(). */
+		dev_kfree_skb(skb);
+	}
+
+	/* Check if user (still) wants to be waken up */
+	tty_wakeup(tty);
+put:
+	tty_kref_put(tty);
+}
+
+/*
+ * Function ircomm_tty_write (tty, buf, count)
+ *
+ *    This routine is called by the kernel to write a series of characters
+ *    to the tty device. The characters may come from user space or kernel
+ *    space. This routine will return the number of characters actually
+ *    accepted for writing. This routine is mandatory.
+ */
+static int ircomm_tty_write(struct tty_struct *tty,
+			    const unsigned char *buf, int count)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) tty->driver_data;
+	unsigned long flags;
+	struct sk_buff *skb;
+	int tailroom = 0;
+	int len = 0;
+	int size;
+
+	pr_debug("%s(), count=%d, hw_stopped=%d\n", __func__ , count,
+		 tty->hw_stopped);
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return -1;);
+
+	/* We may receive packets from the TTY even before we have finished
+	 * our setup. Not cool.
+	 * The problem is that we don't know the final header and data size
+	 * to create the proper skb, so any skb we would create would have
+	 * bogus header and data size, so need care.
+	 * We use a bogus header size to safely detect this condition.
+	 * Another problem is that hw_stopped was set to 0 way before it
+	 * should be, so we would drop this skb. It should now be fixed.
+	 * One option is to not accept data until we are properly setup.
+	 * But, I suspect that when it happens, the ppp line discipline
+	 * just "drops" the data, which might screw up connect scripts.
+	 * The second option is to create a "safe skb", with large header
+	 * and small size (see ircomm_tty_open() for values).
+	 * We just need to make sure that when the real values get filled,
+	 * we don't mess up the original "safe skb" (see tx_data_size).
+	 * Jean II */
+	if (self->max_header_size == IRCOMM_TTY_HDR_UNINITIALISED) {
+		pr_debug("%s() : not initialised\n", __func__);
+#ifdef IRCOMM_NO_TX_BEFORE_INIT
+		/* We didn't consume anything, TTY will retry */
+		return 0;
+#endif
+	}
+
+	if (count < 1)
+		return 0;
+
+	/* Protect our manipulation of self->tx_skb and related */
+	spin_lock_irqsave(&self->spinlock, flags);
+
+	/* Fetch current transmit buffer */
+	skb = self->tx_skb;
+
+	/*
+	 * Send out all the data we get, possibly as multiple fragmented
+	 * frames, but this will only happen if the data is larger than the
+	 * max data size. The normal case however is just the opposite, and
+	 * this function may be called multiple times, and will then actually
+	 * defragment the data and send it out as one packet as soon as
+	 * possible, but at a safer point in time
+	 */
+	while (count) {
+		size = count;
+
+		/* Adjust data size to the max data size */
+		if (size > self->max_data_size)
+			size = self->max_data_size;
+
+		/*
+		 * Do we already have a buffer ready for transmit, or do
+		 * we need to allocate a new frame
+		 */
+		if (skb) {
+			/*
+			 * Any room for more data at the end of the current
+			 * transmit buffer? Cannot use skb_tailroom, since
+			 * dev_alloc_skb gives us a larger skb than we
+			 * requested
+			 * Note : use tx_data_size, because max_data_size
+			 * may have changed and we don't want to overwrite
+			 * the skb. - Jean II
+			 */
+			if ((tailroom = (self->tx_data_size - skb->len)) > 0) {
+				/* Adjust data to tailroom */
+				if (size > tailroom)
+					size = tailroom;
+			} else {
+				/*
+				 * Current transmit frame is full, so break
+				 * out, so we can send it as soon as possible
+				 */
+				break;
+			}
+		} else {
+			/* Prepare a full sized frame */
+			skb = alloc_skb(self->max_data_size+
+					self->max_header_size,
+					GFP_ATOMIC);
+			if (!skb) {
+				spin_unlock_irqrestore(&self->spinlock, flags);
+				return -ENOBUFS;
+			}
+			skb_reserve(skb, self->max_header_size);
+			self->tx_skb = skb;
+			/* Remember skb size because max_data_size may
+			 * change later on - Jean II */
+			self->tx_data_size = self->max_data_size;
+		}
+
+		/* Copy data */
+		skb_put_data(skb, buf + len, size);
+
+		count -= size;
+		len += size;
+	}
+
+	spin_unlock_irqrestore(&self->spinlock, flags);
+
+	/*
+	 * Schedule a new thread which will transmit the frame as soon
+	 * as possible, but at a safe point in time. We do this so the
+	 * "user" can give us data multiple times, as PPP does (because of
+	 * its 256 byte tx buffer). We will then defragment and send out
+	 * all this data as one single packet.
+	 */
+	schedule_work(&self->tqueue);
+
+	return len;
+}
+
+/*
+ * Function ircomm_tty_write_room (tty)
+ *
+ *    This routine returns the numbers of characters the tty driver will
+ *    accept for queuing to be written. This number is subject to change as
+ *    output buffers get emptied, or if the output flow control is acted.
+ */
+static int ircomm_tty_write_room(struct tty_struct *tty)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) tty->driver_data;
+	unsigned long flags;
+	int ret;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return -1;);
+
+#ifdef IRCOMM_NO_TX_BEFORE_INIT
+	/* max_header_size tells us if the channel is initialised or not. */
+	if (self->max_header_size == IRCOMM_TTY_HDR_UNINITIALISED)
+		/* Don't bother us yet */
+		return 0;
+#endif
+
+	/* Check if we are allowed to transmit any data.
+	 * hw_stopped is the regular flow control.
+	 * Jean II */
+	if (tty->hw_stopped)
+		ret = 0;
+	else {
+		spin_lock_irqsave(&self->spinlock, flags);
+		if (self->tx_skb)
+			ret = self->tx_data_size - self->tx_skb->len;
+		else
+			ret = self->max_data_size;
+		spin_unlock_irqrestore(&self->spinlock, flags);
+	}
+	pr_debug("%s(), ret=%d\n", __func__ , ret);
+
+	return ret;
+}
+
+/*
+ * Function ircomm_tty_wait_until_sent (tty, timeout)
+ *
+ *    This routine waits until the device has written out all of the
+ *    characters in its transmitter FIFO.
+ */
+static void ircomm_tty_wait_until_sent(struct tty_struct *tty, int timeout)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) tty->driver_data;
+	unsigned long orig_jiffies, poll_time;
+	unsigned long flags;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return;);
+
+	orig_jiffies = jiffies;
+
+	/* Set poll time to 200 ms */
+	poll_time = msecs_to_jiffies(200);
+	if (timeout)
+		poll_time = min_t(unsigned long, timeout, poll_time);
+
+	spin_lock_irqsave(&self->spinlock, flags);
+	while (self->tx_skb && self->tx_skb->len) {
+		spin_unlock_irqrestore(&self->spinlock, flags);
+		schedule_timeout_interruptible(poll_time);
+		spin_lock_irqsave(&self->spinlock, flags);
+		if (signal_pending(current))
+			break;
+		if (timeout && time_after(jiffies, orig_jiffies + timeout))
+			break;
+	}
+	spin_unlock_irqrestore(&self->spinlock, flags);
+	__set_current_state(TASK_RUNNING);
+}
+
+/*
+ * Function ircomm_tty_throttle (tty)
+ *
+ *    This routine notifies the tty driver that input buffers for the line
+ *    discipline are close to full, and it should somehow signal that no
+ *    more characters should be sent to the tty.
+ */
+static void ircomm_tty_throttle(struct tty_struct *tty)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) tty->driver_data;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return;);
+
+	/* Software flow control? */
+	if (I_IXOFF(tty))
+		ircomm_tty_send_xchar(tty, STOP_CHAR(tty));
+
+	/* Hardware flow control? */
+	if (C_CRTSCTS(tty)) {
+		self->settings.dte &= ~IRCOMM_RTS;
+		self->settings.dte |= IRCOMM_DELTA_RTS;
+
+		ircomm_param_request(self, IRCOMM_DTE, TRUE);
+	}
+
+	ircomm_flow_request(self->ircomm, FLOW_STOP);
+}
+
+/*
+ * Function ircomm_tty_unthrottle (tty)
+ *
+ *    This routine notifies the tty drivers that it should signals that
+ *    characters can now be sent to the tty without fear of overrunning the
+ *    input buffers of the line disciplines.
+ */
+static void ircomm_tty_unthrottle(struct tty_struct *tty)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) tty->driver_data;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return;);
+
+	/* Using software flow control? */
+	if (I_IXOFF(tty))
+		ircomm_tty_send_xchar(tty, START_CHAR(tty));
+
+	/* Using hardware flow control? */
+	if (C_CRTSCTS(tty)) {
+		self->settings.dte |= (IRCOMM_RTS|IRCOMM_DELTA_RTS);
+
+		ircomm_param_request(self, IRCOMM_DTE, TRUE);
+		pr_debug("%s(), FLOW_START\n", __func__);
+	}
+	ircomm_flow_request(self->ircomm, FLOW_START);
+}
+
+/*
+ * Function ircomm_tty_chars_in_buffer (tty)
+ *
+ *    Indicates if there are any data in the buffer
+ *
+ */
+static int ircomm_tty_chars_in_buffer(struct tty_struct *tty)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) tty->driver_data;
+	unsigned long flags;
+	int len = 0;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return -1;);
+
+	spin_lock_irqsave(&self->spinlock, flags);
+
+	if (self->tx_skb)
+		len = self->tx_skb->len;
+
+	spin_unlock_irqrestore(&self->spinlock, flags);
+
+	return len;
+}
+
+static void ircomm_tty_shutdown(struct ircomm_tty_cb *self)
+{
+	unsigned long flags;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return;);
+
+	if (!tty_port_initialized(&self->port))
+		return;
+	tty_port_set_initialized(&self->port, 0);
+
+	ircomm_tty_detach_cable(self);
+
+	spin_lock_irqsave(&self->spinlock, flags);
+
+	del_timer(&self->watchdog_timer);
+
+	/* Free parameter buffer */
+	if (self->ctrl_skb) {
+		dev_kfree_skb(self->ctrl_skb);
+		self->ctrl_skb = NULL;
+	}
+
+	/* Free transmit buffer */
+	if (self->tx_skb) {
+		dev_kfree_skb(self->tx_skb);
+		self->tx_skb = NULL;
+	}
+
+	if (self->ircomm) {
+		ircomm_close(self->ircomm);
+		self->ircomm = NULL;
+	}
+
+	spin_unlock_irqrestore(&self->spinlock, flags);
+}
+
+/*
+ * Function ircomm_tty_hangup (tty)
+ *
+ *    This routine notifies the tty driver that it should hangup the tty
+ *    device.
+ *
+ */
+static void ircomm_tty_hangup(struct tty_struct *tty)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) tty->driver_data;
+	struct tty_port *port = &self->port;
+	unsigned long	flags;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return;);
+
+	/* ircomm_tty_flush_buffer(tty); */
+	ircomm_tty_shutdown(self);
+
+	spin_lock_irqsave(&port->lock, flags);
+	if (port->tty) {
+		set_bit(TTY_IO_ERROR, &port->tty->flags);
+		tty_kref_put(port->tty);
+	}
+	port->tty = NULL;
+	port->count = 0;
+	spin_unlock_irqrestore(&port->lock, flags);
+	tty_port_set_active(port, 0);
+
+	wake_up_interruptible(&port->open_wait);
+}
+
+/*
+ * Function ircomm_tty_send_xchar (tty, ch)
+ *
+ *    This routine is used to send a high-priority XON/XOFF character to
+ *    the device.
+ */
+static void ircomm_tty_send_xchar(struct tty_struct *tty, char ch)
+{
+	pr_debug("%s(), not impl\n", __func__);
+}
+
+/*
+ * Function ircomm_tty_start (tty)
+ *
+ *    This routine notifies the tty driver that it resume sending
+ *    characters to the tty device.
+ */
+void ircomm_tty_start(struct tty_struct *tty)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) tty->driver_data;
+
+	ircomm_flow_request(self->ircomm, FLOW_START);
+}
+
+/*
+ * Function ircomm_tty_stop (tty)
+ *
+ *     This routine notifies the tty driver that it should stop outputting
+ *     characters to the tty device.
+ */
+static void ircomm_tty_stop(struct tty_struct *tty)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) tty->driver_data;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return;);
+
+	ircomm_flow_request(self->ircomm, FLOW_STOP);
+}
+
+/*
+ * Function ircomm_check_modem_status (self)
+ *
+ *    Check for any changes in the DCE's line settings. This function should
+ *    be called whenever the dce parameter settings changes, to update the
+ *    flow control settings and other things
+ */
+void ircomm_tty_check_modem_status(struct ircomm_tty_cb *self)
+{
+	struct tty_struct *tty;
+	int status;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return;);
+
+	tty = tty_port_tty_get(&self->port);
+
+	status = self->settings.dce;
+
+	if (status & IRCOMM_DCE_DELTA_ANY) {
+		/*wake_up_interruptible(&self->delta_msr_wait);*/
+	}
+	if (tty_port_check_carrier(&self->port) && (status & IRCOMM_DELTA_CD)) {
+		pr_debug("%s(), ircomm%d CD now %s...\n", __func__ , self->line,
+			 (status & IRCOMM_CD) ? "on" : "off");
+
+		if (status & IRCOMM_CD) {
+			wake_up_interruptible(&self->port.open_wait);
+		} else {
+			pr_debug("%s(), Doing serial hangup..\n", __func__);
+			if (tty)
+				tty_hangup(tty);
+
+			/* Hangup will remote the tty, so better break out */
+			goto put;
+		}
+	}
+	if (tty && tty_port_cts_enabled(&self->port)) {
+		if (tty->hw_stopped) {
+			if (status & IRCOMM_CTS) {
+				pr_debug("%s(), CTS tx start...\n", __func__);
+				tty->hw_stopped = 0;
+
+				/* Wake up processes blocked on open */
+				wake_up_interruptible(&self->port.open_wait);
+
+				schedule_work(&self->tqueue);
+				goto put;
+			}
+		} else {
+			if (!(status & IRCOMM_CTS)) {
+				pr_debug("%s(), CTS tx stop...\n", __func__);
+				tty->hw_stopped = 1;
+			}
+		}
+	}
+put:
+	tty_kref_put(tty);
+}
+
+/*
+ * Function ircomm_tty_data_indication (instance, sap, skb)
+ *
+ *    Handle incoming data, and deliver it to the line discipline
+ *
+ */
+static int ircomm_tty_data_indication(void *instance, void *sap,
+				      struct sk_buff *skb)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) instance;
+	struct tty_struct *tty;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return -1;);
+	IRDA_ASSERT(skb != NULL, return -1;);
+
+	tty = tty_port_tty_get(&self->port);
+	if (!tty) {
+		pr_debug("%s(), no tty!\n", __func__);
+		return 0;
+	}
+
+	/*
+	 * If we receive data when hardware is stopped then something is wrong.
+	 * We try to poll the peers line settings to check if we are up todate.
+	 * Devices like WinCE can do this, and since they don't send any
+	 * params, we can just as well declare the hardware for running.
+	 */
+	if (tty->hw_stopped && (self->flow == FLOW_START)) {
+		pr_debug("%s(), polling for line settings!\n", __func__);
+		ircomm_param_request(self, IRCOMM_POLL, TRUE);
+
+		/* We can just as well declare the hardware for running */
+		ircomm_tty_send_initial_parameters(self);
+		ircomm_tty_link_established(self);
+	}
+	tty_kref_put(tty);
+
+	/*
+	 * Use flip buffer functions since the code may be called from interrupt
+	 * context
+	 */
+	tty_insert_flip_string(&self->port, skb->data, skb->len);
+	tty_flip_buffer_push(&self->port);
+
+	/* No need to kfree_skb - see ircomm_ttp_data_indication() */
+
+	return 0;
+}
+
+/*
+ * Function ircomm_tty_control_indication (instance, sap, skb)
+ *
+ *    Parse all incoming parameters (easy!)
+ *
+ */
+static int ircomm_tty_control_indication(void *instance, void *sap,
+					 struct sk_buff *skb)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) instance;
+	int clen;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return -1;);
+	IRDA_ASSERT(skb != NULL, return -1;);
+
+	clen = skb->data[0];
+
+	irda_param_extract_all(self, skb->data+1, IRDA_MIN(skb->len-1, clen),
+			       &ircomm_param_info);
+
+	/* No need to kfree_skb - see ircomm_control_indication() */
+
+	return 0;
+}
+
+/*
+ * Function ircomm_tty_flow_indication (instance, sap, cmd)
+ *
+ *    This function is called by IrTTP when it wants us to slow down the
+ *    transmission of data. We just mark the hardware as stopped, and wait
+ *    for IrTTP to notify us that things are OK again.
+ */
+static void ircomm_tty_flow_indication(void *instance, void *sap,
+				       LOCAL_FLOW cmd)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) instance;
+	struct tty_struct *tty;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return;);
+
+	tty = tty_port_tty_get(&self->port);
+
+	switch (cmd) {
+	case FLOW_START:
+		pr_debug("%s(), hw start!\n", __func__);
+		if (tty)
+			tty->hw_stopped = 0;
+
+		/* ircomm_tty_do_softint will take care of the rest */
+		schedule_work(&self->tqueue);
+		break;
+	default:  /* If we get here, something is very wrong, better stop */
+	case FLOW_STOP:
+		pr_debug("%s(), hw stopped!\n", __func__);
+		if (tty)
+			tty->hw_stopped = 1;
+		break;
+	}
+
+	tty_kref_put(tty);
+	self->flow = cmd;
+}
+
+#ifdef CONFIG_PROC_FS
+static void ircomm_tty_line_info(struct ircomm_tty_cb *self, struct seq_file *m)
+{
+	struct tty_struct *tty;
+	char sep;
+
+	seq_printf(m, "State: %s\n", ircomm_tty_state[self->state]);
+
+	seq_puts(m, "Service type: ");
+	if (self->service_type & IRCOMM_9_WIRE)
+		seq_puts(m, "9_WIRE");
+	else if (self->service_type & IRCOMM_3_WIRE)
+		seq_puts(m, "3_WIRE");
+	else if (self->service_type & IRCOMM_3_WIRE_RAW)
+		seq_puts(m, "3_WIRE_RAW");
+	else
+		seq_puts(m, "No common service type!\n");
+	seq_putc(m, '\n');
+
+	seq_printf(m, "Port name: %s\n", self->settings.port_name);
+
+	seq_printf(m, "DTE status:");
+	sep = ' ';
+	if (self->settings.dte & IRCOMM_RTS) {
+		seq_printf(m, "%cRTS", sep);
+		sep = '|';
+	}
+	if (self->settings.dte & IRCOMM_DTR) {
+		seq_printf(m, "%cDTR", sep);
+		sep = '|';
+	}
+	seq_putc(m, '\n');
+
+	seq_puts(m, "DCE status:");
+	sep = ' ';
+	if (self->settings.dce & IRCOMM_CTS) {
+		seq_printf(m, "%cCTS", sep);
+		sep = '|';
+	}
+	if (self->settings.dce & IRCOMM_DSR) {
+		seq_printf(m, "%cDSR", sep);
+		sep = '|';
+	}
+	if (self->settings.dce & IRCOMM_CD) {
+		seq_printf(m, "%cCD", sep);
+		sep = '|';
+	}
+	if (self->settings.dce & IRCOMM_RI) {
+		seq_printf(m, "%cRI", sep);
+		sep = '|';
+	}
+	seq_putc(m, '\n');
+
+	seq_puts(m, "Configuration: ");
+	if (!self->settings.null_modem)
+		seq_puts(m, "DTE <-> DCE\n");
+	else
+		seq_puts(m, "DTE <-> DTE (null modem emulation)\n");
+
+	seq_printf(m, "Data rate: %d\n", self->settings.data_rate);
+
+	seq_puts(m, "Flow control:");
+	sep = ' ';
+	if (self->settings.flow_control & IRCOMM_XON_XOFF_IN) {
+		seq_printf(m, "%cXON_XOFF_IN", sep);
+		sep = '|';
+	}
+	if (self->settings.flow_control & IRCOMM_XON_XOFF_OUT) {
+		seq_printf(m, "%cXON_XOFF_OUT", sep);
+		sep = '|';
+	}
+	if (self->settings.flow_control & IRCOMM_RTS_CTS_IN) {
+		seq_printf(m, "%cRTS_CTS_IN", sep);
+		sep = '|';
+	}
+	if (self->settings.flow_control & IRCOMM_RTS_CTS_OUT) {
+		seq_printf(m, "%cRTS_CTS_OUT", sep);
+		sep = '|';
+	}
+	if (self->settings.flow_control & IRCOMM_DSR_DTR_IN) {
+		seq_printf(m, "%cDSR_DTR_IN", sep);
+		sep = '|';
+	}
+	if (self->settings.flow_control & IRCOMM_DSR_DTR_OUT) {
+		seq_printf(m, "%cDSR_DTR_OUT", sep);
+		sep = '|';
+	}
+	if (self->settings.flow_control & IRCOMM_ENQ_ACK_IN) {
+		seq_printf(m, "%cENQ_ACK_IN", sep);
+		sep = '|';
+	}
+	if (self->settings.flow_control & IRCOMM_ENQ_ACK_OUT) {
+		seq_printf(m, "%cENQ_ACK_OUT", sep);
+		sep = '|';
+	}
+	seq_putc(m, '\n');
+
+	seq_puts(m, "Flags:");
+	sep = ' ';
+	if (tty_port_cts_enabled(&self->port)) {
+		seq_printf(m, "%cASYNC_CTS_FLOW", sep);
+		sep = '|';
+	}
+	if (tty_port_check_carrier(&self->port)) {
+		seq_printf(m, "%cASYNC_CHECK_CD", sep);
+		sep = '|';
+	}
+	if (tty_port_initialized(&self->port)) {
+		seq_printf(m, "%cASYNC_INITIALIZED", sep);
+		sep = '|';
+	}
+	if (self->port.flags & ASYNC_LOW_LATENCY) {
+		seq_printf(m, "%cASYNC_LOW_LATENCY", sep);
+		sep = '|';
+	}
+	if (tty_port_active(&self->port)) {
+		seq_printf(m, "%cASYNC_NORMAL_ACTIVE", sep);
+		sep = '|';
+	}
+	seq_putc(m, '\n');
+
+	seq_printf(m, "Role: %s\n", self->client ? "client" : "server");
+	seq_printf(m, "Open count: %d\n", self->port.count);
+	seq_printf(m, "Max data size: %d\n", self->max_data_size);
+	seq_printf(m, "Max header size: %d\n", self->max_header_size);
+
+	tty = tty_port_tty_get(&self->port);
+	if (tty) {
+		seq_printf(m, "Hardware: %s\n",
+			       tty->hw_stopped ? "Stopped" : "Running");
+		tty_kref_put(tty);
+	}
+}
+
+static int ircomm_tty_proc_show(struct seq_file *m, void *v)
+{
+	struct ircomm_tty_cb *self;
+	unsigned long flags;
+
+	spin_lock_irqsave(&ircomm_tty->hb_spinlock, flags);
+
+	self = (struct ircomm_tty_cb *) hashbin_get_first(ircomm_tty);
+	while (self != NULL) {
+		if (self->magic != IRCOMM_TTY_MAGIC)
+			break;
+
+		ircomm_tty_line_info(self, m);
+		self = (struct ircomm_tty_cb *) hashbin_get_next(ircomm_tty);
+	}
+	spin_unlock_irqrestore(&ircomm_tty->hb_spinlock, flags);
+	return 0;
+}
+
+static int ircomm_tty_proc_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, ircomm_tty_proc_show, NULL);
+}
+
+static const struct file_operations ircomm_tty_proc_fops = {
+	.owner		= THIS_MODULE,
+	.open		= ircomm_tty_proc_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+#endif /* CONFIG_PROC_FS */
+
+MODULE_AUTHOR("Dag Brattli <dagb@cs.uit.no>");
+MODULE_DESCRIPTION("IrCOMM serial TTY driver");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS_CHARDEV_MAJOR(IRCOMM_TTY_MAJOR);
+
+module_init(ircomm_tty_init);
+module_exit(ircomm_tty_cleanup);
diff --git a/drivers/staging/irda/net/ircomm/ircomm_tty_attach.c b/drivers/staging/irda/net/ircomm/ircomm_tty_attach.c
new file mode 100644
index 000000000000..0a411019c098
--- /dev/null
+++ b/drivers/staging/irda/net/ircomm/ircomm_tty_attach.c
@@ -0,0 +1,987 @@
+/*********************************************************************
+ *
+ * Filename:      ircomm_tty_attach.c
+ * Version:
+ * Description:   Code for attaching the serial driver to IrCOMM
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sat Jun  5 17:42:00 1999
+ * Modified at:   Tue Jan  4 14:20:49 2000
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1999-2000 Dag Brattli, All Rights Reserved.
+ *     Copyright (c) 2000-2003 Jean Tourrilhes <jt@hpl.hp.com>
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ *
+ *     You should have received a copy of the GNU General Public License
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ ********************************************************************/
+
+#include <linux/init.h>
+#include <linux/sched.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/irlmp.h>
+#include <net/irda/iriap.h>
+#include <net/irda/irttp.h>
+#include <net/irda/irias_object.h>
+#include <net/irda/parameters.h>
+
+#include <net/irda/ircomm_core.h>
+#include <net/irda/ircomm_param.h>
+#include <net/irda/ircomm_event.h>
+
+#include <net/irda/ircomm_tty.h>
+#include <net/irda/ircomm_tty_attach.h>
+
+static void ircomm_tty_ias_register(struct ircomm_tty_cb *self);
+static void ircomm_tty_discovery_indication(discinfo_t *discovery,
+					    DISCOVERY_MODE mode,
+					    void *priv);
+static void ircomm_tty_getvalue_confirm(int result, __u16 obj_id,
+					struct ias_value *value, void *priv);
+static void ircomm_tty_start_watchdog_timer(struct ircomm_tty_cb *self,
+					    int timeout);
+static void ircomm_tty_watchdog_timer_expired(void *data);
+
+static int ircomm_tty_state_idle(struct ircomm_tty_cb *self,
+				 IRCOMM_TTY_EVENT event,
+				 struct sk_buff *skb,
+				 struct ircomm_tty_info *info);
+static int ircomm_tty_state_search(struct ircomm_tty_cb *self,
+				   IRCOMM_TTY_EVENT event,
+				   struct sk_buff *skb,
+				   struct ircomm_tty_info *info);
+static int ircomm_tty_state_query_parameters(struct ircomm_tty_cb *self,
+					     IRCOMM_TTY_EVENT event,
+					     struct sk_buff *skb,
+					     struct ircomm_tty_info *info);
+static int ircomm_tty_state_query_lsap_sel(struct ircomm_tty_cb *self,
+					   IRCOMM_TTY_EVENT event,
+					   struct sk_buff *skb,
+					   struct ircomm_tty_info *info);
+static int ircomm_tty_state_setup(struct ircomm_tty_cb *self,
+				  IRCOMM_TTY_EVENT event,
+				  struct sk_buff *skb,
+				  struct ircomm_tty_info *info);
+static int ircomm_tty_state_ready(struct ircomm_tty_cb *self,
+				  IRCOMM_TTY_EVENT event,
+				  struct sk_buff *skb,
+				  struct ircomm_tty_info *info);
+
+const char *const ircomm_tty_state[] = {
+	"IRCOMM_TTY_IDLE",
+	"IRCOMM_TTY_SEARCH",
+	"IRCOMM_TTY_QUERY_PARAMETERS",
+	"IRCOMM_TTY_QUERY_LSAP_SEL",
+	"IRCOMM_TTY_SETUP",
+	"IRCOMM_TTY_READY",
+	"*** ERROR *** ",
+};
+
+static const char *const ircomm_tty_event[] __maybe_unused = {
+	"IRCOMM_TTY_ATTACH_CABLE",
+	"IRCOMM_TTY_DETACH_CABLE",
+	"IRCOMM_TTY_DATA_REQUEST",
+	"IRCOMM_TTY_DATA_INDICATION",
+	"IRCOMM_TTY_DISCOVERY_REQUEST",
+	"IRCOMM_TTY_DISCOVERY_INDICATION",
+	"IRCOMM_TTY_CONNECT_CONFIRM",
+	"IRCOMM_TTY_CONNECT_INDICATION",
+	"IRCOMM_TTY_DISCONNECT_REQUEST",
+	"IRCOMM_TTY_DISCONNECT_INDICATION",
+	"IRCOMM_TTY_WD_TIMER_EXPIRED",
+	"IRCOMM_TTY_GOT_PARAMETERS",
+	"IRCOMM_TTY_GOT_LSAPSEL",
+	"*** ERROR ****",
+};
+
+static int (*state[])(struct ircomm_tty_cb *self, IRCOMM_TTY_EVENT event,
+		      struct sk_buff *skb, struct ircomm_tty_info *info) =
+{
+	ircomm_tty_state_idle,
+	ircomm_tty_state_search,
+	ircomm_tty_state_query_parameters,
+	ircomm_tty_state_query_lsap_sel,
+	ircomm_tty_state_setup,
+	ircomm_tty_state_ready,
+};
+
+/*
+ * Function ircomm_tty_attach_cable (driver)
+ *
+ *    Try to attach cable (IrCOMM link). This function will only return
+ *    when the link has been connected, or if an error condition occurs.
+ *    If success, the return value is the resulting service type.
+ */
+int ircomm_tty_attach_cable(struct ircomm_tty_cb *self)
+{
+	struct tty_struct *tty;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return -1;);
+
+	/* Check if somebody has already connected to us */
+	if (ircomm_is_connected(self->ircomm)) {
+		pr_debug("%s(), already connected!\n", __func__);
+		return 0;
+	}
+
+	/* Make sure nobody tries to write before the link is up */
+	tty = tty_port_tty_get(&self->port);
+	if (tty) {
+		tty->hw_stopped = 1;
+		tty_kref_put(tty);
+	}
+
+	ircomm_tty_ias_register(self);
+
+	ircomm_tty_do_event(self, IRCOMM_TTY_ATTACH_CABLE, NULL, NULL);
+
+	return 0;
+}
+
+/*
+ * Function ircomm_detach_cable (driver)
+ *
+ *    Detach cable, or cable has been detached by peer
+ *
+ */
+void ircomm_tty_detach_cable(struct ircomm_tty_cb *self)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return;);
+
+	del_timer(&self->watchdog_timer);
+
+	/* Remove discovery handler */
+	if (self->ckey) {
+		irlmp_unregister_client(self->ckey);
+		self->ckey = NULL;
+	}
+	/* Remove IrCOMM hint bits */
+	if (self->skey) {
+		irlmp_unregister_service(self->skey);
+		self->skey = NULL;
+	}
+
+	if (self->iriap) {
+		iriap_close(self->iriap);
+		self->iriap = NULL;
+	}
+
+	/* Remove LM-IAS object */
+	if (self->obj) {
+		irias_delete_object(self->obj);
+		self->obj = NULL;
+	}
+
+	ircomm_tty_do_event(self, IRCOMM_TTY_DETACH_CABLE, NULL, NULL);
+
+	/* Reset some values */
+	self->daddr = self->saddr = 0;
+	self->dlsap_sel = self->slsap_sel = 0;
+
+	memset(&self->settings, 0, sizeof(struct ircomm_params));
+}
+
+/*
+ * Function ircomm_tty_ias_register (self)
+ *
+ *    Register with LM-IAS depending on which service type we are
+ *
+ */
+static void ircomm_tty_ias_register(struct ircomm_tty_cb *self)
+{
+	__u8 oct_seq[6];
+	__u16 hints;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return;);
+
+	/* Compute hint bits based on service */
+	hints = irlmp_service_to_hint(S_COMM);
+	if (self->service_type & IRCOMM_3_WIRE_RAW)
+		hints |= irlmp_service_to_hint(S_PRINTER);
+
+	/* Advertise IrCOMM hint bit in discovery */
+	if (!self->skey)
+		self->skey = irlmp_register_service(hints);
+	/* Set up a discovery handler */
+	if (!self->ckey)
+		self->ckey = irlmp_register_client(hints,
+						   ircomm_tty_discovery_indication,
+						   NULL, (void *) self);
+
+	/* If already done, no need to do it again */
+	if (self->obj)
+		return;
+
+	if (self->service_type & IRCOMM_3_WIRE_RAW) {
+		/* Register IrLPT with LM-IAS */
+		self->obj = irias_new_object("IrLPT", IAS_IRLPT_ID);
+		irias_add_integer_attrib(self->obj, "IrDA:IrLMP:LsapSel",
+					 self->slsap_sel, IAS_KERNEL_ATTR);
+	} else {
+		/* Register IrCOMM with LM-IAS */
+		self->obj = irias_new_object("IrDA:IrCOMM", IAS_IRCOMM_ID);
+		irias_add_integer_attrib(self->obj, "IrDA:TinyTP:LsapSel",
+					 self->slsap_sel, IAS_KERNEL_ATTR);
+
+		/* Code the parameters into the buffer */
+		irda_param_pack(oct_seq, "bbbbbb",
+				IRCOMM_SERVICE_TYPE, 1, self->service_type,
+				IRCOMM_PORT_TYPE,    1, IRCOMM_SERIAL);
+
+		/* Register parameters with LM-IAS */
+		irias_add_octseq_attrib(self->obj, "Parameters", oct_seq, 6,
+					IAS_KERNEL_ATTR);
+	}
+	irias_insert_object(self->obj);
+}
+
+/*
+ * Function ircomm_tty_ias_unregister (self)
+ *
+ *    Remove our IAS object and client hook while connected.
+ *
+ */
+static void ircomm_tty_ias_unregister(struct ircomm_tty_cb *self)
+{
+	/* Remove LM-IAS object now so it is not reused.
+	 * IrCOMM deals very poorly with multiple incoming connections.
+	 * It should looks a lot more like IrNET, and "dup" a server TSAP
+	 * to the application TSAP (based on various rules).
+	 * This is a cheap workaround allowing multiple clients to
+	 * connect to us. It will not always work.
+	 * Each IrCOMM socket has an IAS entry. Incoming connection will
+	 * pick the first one found. So, when we are fully connected,
+	 * we remove our IAS entries so that the next IAS entry is used.
+	 * We do that for *both* client and server, because a server
+	 * can also create client instances.
+	 * Jean II */
+	if (self->obj) {
+		irias_delete_object(self->obj);
+		self->obj = NULL;
+	}
+
+#if 0
+	/* Remove discovery handler.
+	 * While we are connected, we no longer need to receive
+	 * discovery events. This would be the case if there is
+	 * multiple IrLAP interfaces. Jean II */
+	if (self->ckey) {
+		irlmp_unregister_client(self->ckey);
+		self->ckey = NULL;
+	}
+#endif
+}
+
+/*
+ * Function ircomm_send_initial_parameters (self)
+ *
+ *    Send initial parameters to the remote IrCOMM device. These parameters
+ *    must be sent before any data.
+ */
+int ircomm_tty_send_initial_parameters(struct ircomm_tty_cb *self)
+{
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return -1;);
+
+	if (self->service_type & IRCOMM_3_WIRE_RAW)
+		return 0;
+
+	/*
+	 * Set default values, but only if the application for some reason
+	 * haven't set them already
+	 */
+	pr_debug("%s(), data-rate = %d\n", __func__ ,
+		 self->settings.data_rate);
+	if (!self->settings.data_rate)
+		self->settings.data_rate = 9600;
+	pr_debug("%s(), data-format = %d\n", __func__ ,
+		 self->settings.data_format);
+	if (!self->settings.data_format)
+		self->settings.data_format = IRCOMM_WSIZE_8;  /* 8N1 */
+
+	pr_debug("%s(), flow-control = %d\n", __func__ ,
+		 self->settings.flow_control);
+	/*self->settings.flow_control = IRCOMM_RTS_CTS_IN|IRCOMM_RTS_CTS_OUT;*/
+
+	/* Do not set delta values for the initial parameters */
+	self->settings.dte = IRCOMM_DTR | IRCOMM_RTS;
+
+	/* Only send service type parameter when we are the client */
+	if (self->client)
+		ircomm_param_request(self, IRCOMM_SERVICE_TYPE, FALSE);
+	ircomm_param_request(self, IRCOMM_DATA_RATE, FALSE);
+	ircomm_param_request(self, IRCOMM_DATA_FORMAT, FALSE);
+
+	/* For a 3 wire service, we just flush the last parameter and return */
+	if (self->settings.service_type == IRCOMM_3_WIRE) {
+		ircomm_param_request(self, IRCOMM_FLOW_CONTROL, TRUE);
+		return 0;
+	}
+
+	/* Only 9-wire service types continue here */
+	ircomm_param_request(self, IRCOMM_FLOW_CONTROL, FALSE);
+#if 0
+	ircomm_param_request(self, IRCOMM_XON_XOFF, FALSE);
+	ircomm_param_request(self, IRCOMM_ENQ_ACK, FALSE);
+#endif
+	/* Notify peer that we are ready to receive data */
+	ircomm_param_request(self, IRCOMM_DTE, TRUE);
+
+	return 0;
+}
+
+/*
+ * Function ircomm_tty_discovery_indication (discovery)
+ *
+ *    Remote device is discovered, try query the remote IAS to see which
+ *    device it is, and which services it has.
+ *
+ */
+static void ircomm_tty_discovery_indication(discinfo_t *discovery,
+					    DISCOVERY_MODE mode,
+					    void *priv)
+{
+	struct ircomm_tty_cb *self;
+	struct ircomm_tty_info info;
+
+	/* Important note :
+	 * We need to drop all passive discoveries.
+	 * The LSAP management of IrComm is deficient and doesn't deal
+	 * with the case of two instance connecting to each other
+	 * simultaneously (it will deadlock in LMP).
+	 * The proper fix would be to use the same technique as in IrNET,
+	 * to have one server socket and separate instances for the
+	 * connecting/connected socket.
+	 * The workaround is to drop passive discovery, which drastically
+	 * reduce the probability of this happening.
+	 * Jean II */
+	if(mode == DISCOVERY_PASSIVE)
+		return;
+
+	info.daddr = discovery->daddr;
+	info.saddr = discovery->saddr;
+
+	self = priv;
+	ircomm_tty_do_event(self, IRCOMM_TTY_DISCOVERY_INDICATION,
+			    NULL, &info);
+}
+
+/*
+ * Function ircomm_tty_disconnect_indication (instance, sap, reason, skb)
+ *
+ *    Link disconnected
+ *
+ */
+void ircomm_tty_disconnect_indication(void *instance, void *sap,
+				      LM_REASON reason,
+				      struct sk_buff *skb)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) instance;
+	struct tty_struct *tty;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return;);
+
+	tty = tty_port_tty_get(&self->port);
+	if (!tty)
+		return;
+
+	/* This will stop control data transfers */
+	self->flow = FLOW_STOP;
+
+	/* Stop data transfers */
+	tty->hw_stopped = 1;
+
+	ircomm_tty_do_event(self, IRCOMM_TTY_DISCONNECT_INDICATION, NULL,
+			    NULL);
+	tty_kref_put(tty);
+}
+
+/*
+ * Function ircomm_tty_getvalue_confirm (result, obj_id, value, priv)
+ *
+ *    Got result from the IAS query we make
+ *
+ */
+static void ircomm_tty_getvalue_confirm(int result, __u16 obj_id,
+					struct ias_value *value,
+					void *priv)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) priv;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return;);
+
+	/* We probably don't need to make any more queries */
+	iriap_close(self->iriap);
+	self->iriap = NULL;
+
+	/* Check if request succeeded */
+	if (result != IAS_SUCCESS) {
+		pr_debug("%s(), got NULL value!\n", __func__);
+		return;
+	}
+
+	switch (value->type) {
+	case IAS_OCT_SEQ:
+		pr_debug("%s(), got octet sequence\n", __func__);
+
+		irda_param_extract_all(self, value->t.oct_seq, value->len,
+				       &ircomm_param_info);
+
+		ircomm_tty_do_event(self, IRCOMM_TTY_GOT_PARAMETERS, NULL,
+				    NULL);
+		break;
+	case IAS_INTEGER:
+		/* Got LSAP selector */
+		pr_debug("%s(), got lsapsel = %d\n", __func__ ,
+			 value->t.integer);
+
+		if (value->t.integer == -1) {
+			pr_debug("%s(), invalid value!\n", __func__);
+		} else
+			self->dlsap_sel = value->t.integer;
+
+		ircomm_tty_do_event(self, IRCOMM_TTY_GOT_LSAPSEL, NULL, NULL);
+		break;
+	case IAS_MISSING:
+		pr_debug("%s(), got IAS_MISSING\n", __func__);
+		break;
+	default:
+		pr_debug("%s(), got unknown type!\n", __func__);
+		break;
+	}
+	irias_delete_value(value);
+}
+
+/*
+ * Function ircomm_tty_connect_confirm (instance, sap, qos, max_sdu_size, skb)
+ *
+ *    Connection confirmed
+ *
+ */
+void ircomm_tty_connect_confirm(void *instance, void *sap,
+				struct qos_info *qos,
+				__u32 max_data_size,
+				__u8 max_header_size,
+				struct sk_buff *skb)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) instance;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return;);
+
+	self->client = TRUE;
+	self->max_data_size = max_data_size;
+	self->max_header_size = max_header_size;
+	self->flow = FLOW_START;
+
+	ircomm_tty_do_event(self, IRCOMM_TTY_CONNECT_CONFIRM, NULL, NULL);
+
+	/* No need to kfree_skb - see ircomm_ttp_connect_confirm() */
+}
+
+/*
+ * Function ircomm_tty_connect_indication (instance, sap, qos, max_sdu_size,
+ *                                         skb)
+ *
+ *    we are discovered and being requested to connect by remote device !
+ *
+ */
+void ircomm_tty_connect_indication(void *instance, void *sap,
+				   struct qos_info *qos,
+				   __u32 max_data_size,
+				   __u8 max_header_size,
+				   struct sk_buff *skb)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) instance;
+	int clen;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return;);
+
+	self->client = FALSE;
+	self->max_data_size = max_data_size;
+	self->max_header_size = max_header_size;
+	self->flow = FLOW_START;
+
+	clen = skb->data[0];
+	if (clen)
+		irda_param_extract_all(self, skb->data+1,
+				       IRDA_MIN(skb->len, clen),
+				       &ircomm_param_info);
+
+	ircomm_tty_do_event(self, IRCOMM_TTY_CONNECT_INDICATION, NULL, NULL);
+
+	/* No need to kfree_skb - see ircomm_ttp_connect_indication() */
+}
+
+/*
+ * Function ircomm_tty_link_established (self)
+ *
+ *    Called when the IrCOMM link is established
+ *
+ */
+void ircomm_tty_link_established(struct ircomm_tty_cb *self)
+{
+	struct tty_struct *tty;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return;);
+
+	tty = tty_port_tty_get(&self->port);
+	if (!tty)
+		return;
+
+	del_timer(&self->watchdog_timer);
+
+	/*
+	 * IrCOMM link is now up, and if we are not using hardware
+	 * flow-control, then declare the hardware as running. Otherwise we
+	 * will have to wait for the peer device (DCE) to raise the CTS
+	 * line.
+	 */
+	if (tty_port_cts_enabled(&self->port) &&
+			((self->settings.dce & IRCOMM_CTS) == 0)) {
+		pr_debug("%s(), waiting for CTS ...\n", __func__);
+		goto put;
+	} else {
+		pr_debug("%s(), starting hardware!\n", __func__);
+
+		tty->hw_stopped = 0;
+
+		/* Wake up processes blocked on open */
+		wake_up_interruptible(&self->port.open_wait);
+	}
+
+	schedule_work(&self->tqueue);
+put:
+	tty_kref_put(tty);
+}
+
+/*
+ * Function ircomm_tty_start_watchdog_timer (self, timeout)
+ *
+ *    Start the watchdog timer. This timer is used to make sure that any
+ *    connection attempt is successful, and if not, we will retry after
+ *    the timeout
+ */
+static void ircomm_tty_start_watchdog_timer(struct ircomm_tty_cb *self,
+					    int timeout)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return;);
+
+	irda_start_timer(&self->watchdog_timer, timeout, (void *) self,
+			 ircomm_tty_watchdog_timer_expired);
+}
+
+/*
+ * Function ircomm_tty_watchdog_timer_expired (data)
+ *
+ *    Called when the connect procedure have taken to much time.
+ *
+ */
+static void ircomm_tty_watchdog_timer_expired(void *data)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) data;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return;);
+
+	ircomm_tty_do_event(self, IRCOMM_TTY_WD_TIMER_EXPIRED, NULL, NULL);
+}
+
+
+/*
+ * Function ircomm_tty_do_event (self, event, skb)
+ *
+ *    Process event
+ *
+ */
+int ircomm_tty_do_event(struct ircomm_tty_cb *self, IRCOMM_TTY_EVENT event,
+			struct sk_buff *skb, struct ircomm_tty_info *info)
+{
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return -1;);
+
+	pr_debug("%s: state=%s, event=%s\n", __func__ ,
+		 ircomm_tty_state[self->state], ircomm_tty_event[event]);
+
+	return (*state[self->state])(self, event, skb, info);
+}
+
+/*
+ * Function ircomm_tty_next_state (self, state)
+ *
+ *    Switch state
+ *
+ */
+static inline void ircomm_tty_next_state(struct ircomm_tty_cb *self, IRCOMM_TTY_STATE state)
+{
+	/*
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return;);
+
+	pr_debug("%s: next state=%s, service type=%d\n", __func__ ,
+	ircomm_tty_state[self->state], self->service_type);
+	*/
+	self->state = state;
+}
+
+/*
+ * Function ircomm_tty_state_idle (self, event, skb, info)
+ *
+ *    Just hanging around
+ *
+ */
+static int ircomm_tty_state_idle(struct ircomm_tty_cb *self,
+				 IRCOMM_TTY_EVENT event,
+				 struct sk_buff *skb,
+				 struct ircomm_tty_info *info)
+{
+	int ret = 0;
+
+	pr_debug("%s: state=%s, event=%s\n", __func__ ,
+		 ircomm_tty_state[self->state], ircomm_tty_event[event]);
+	switch (event) {
+	case IRCOMM_TTY_ATTACH_CABLE:
+		/* Try to discover any remote devices */
+		ircomm_tty_start_watchdog_timer(self, 3*HZ);
+		ircomm_tty_next_state(self, IRCOMM_TTY_SEARCH);
+
+		irlmp_discovery_request(DISCOVERY_DEFAULT_SLOTS);
+		break;
+	case IRCOMM_TTY_DISCOVERY_INDICATION:
+		self->daddr = info->daddr;
+		self->saddr = info->saddr;
+
+		if (self->iriap) {
+			net_warn_ratelimited("%s(), busy with a previous query\n",
+					     __func__);
+			return -EBUSY;
+		}
+
+		self->iriap = iriap_open(LSAP_ANY, IAS_CLIENT, self,
+					 ircomm_tty_getvalue_confirm);
+
+		iriap_getvaluebyclass_request(self->iriap,
+					      self->saddr, self->daddr,
+					      "IrDA:IrCOMM", "Parameters");
+
+		ircomm_tty_start_watchdog_timer(self, 3*HZ);
+		ircomm_tty_next_state(self, IRCOMM_TTY_QUERY_PARAMETERS);
+		break;
+	case IRCOMM_TTY_CONNECT_INDICATION:
+		del_timer(&self->watchdog_timer);
+
+		/* Accept connection */
+		ircomm_connect_response(self->ircomm, NULL);
+		ircomm_tty_next_state(self, IRCOMM_TTY_READY);
+		break;
+	case IRCOMM_TTY_WD_TIMER_EXPIRED:
+		/* Just stay idle */
+		break;
+	case IRCOMM_TTY_DETACH_CABLE:
+		ircomm_tty_next_state(self, IRCOMM_TTY_IDLE);
+		break;
+	default:
+		pr_debug("%s(), unknown event: %s\n", __func__ ,
+			 ircomm_tty_event[event]);
+		ret = -EINVAL;
+	}
+	return ret;
+}
+
+/*
+ * Function ircomm_tty_state_search (self, event, skb, info)
+ *
+ *    Trying to discover an IrCOMM device
+ *
+ */
+static int ircomm_tty_state_search(struct ircomm_tty_cb *self,
+				   IRCOMM_TTY_EVENT event,
+				   struct sk_buff *skb,
+				   struct ircomm_tty_info *info)
+{
+	int ret = 0;
+
+	pr_debug("%s: state=%s, event=%s\n", __func__ ,
+		 ircomm_tty_state[self->state], ircomm_tty_event[event]);
+
+	switch (event) {
+	case IRCOMM_TTY_DISCOVERY_INDICATION:
+		self->daddr = info->daddr;
+		self->saddr = info->saddr;
+
+		if (self->iriap) {
+			net_warn_ratelimited("%s(), busy with a previous query\n",
+					     __func__);
+			return -EBUSY;
+		}
+
+		self->iriap = iriap_open(LSAP_ANY, IAS_CLIENT, self,
+					 ircomm_tty_getvalue_confirm);
+
+		if (self->service_type == IRCOMM_3_WIRE_RAW) {
+			iriap_getvaluebyclass_request(self->iriap, self->saddr,
+						      self->daddr, "IrLPT",
+						      "IrDA:IrLMP:LsapSel");
+			ircomm_tty_next_state(self, IRCOMM_TTY_QUERY_LSAP_SEL);
+		} else {
+			iriap_getvaluebyclass_request(self->iriap, self->saddr,
+						      self->daddr,
+						      "IrDA:IrCOMM",
+						      "Parameters");
+
+			ircomm_tty_next_state(self, IRCOMM_TTY_QUERY_PARAMETERS);
+		}
+		ircomm_tty_start_watchdog_timer(self, 3*HZ);
+		break;
+	case IRCOMM_TTY_CONNECT_INDICATION:
+		del_timer(&self->watchdog_timer);
+		ircomm_tty_ias_unregister(self);
+
+		/* Accept connection */
+		ircomm_connect_response(self->ircomm, NULL);
+		ircomm_tty_next_state(self, IRCOMM_TTY_READY);
+		break;
+	case IRCOMM_TTY_WD_TIMER_EXPIRED:
+#if 1
+		/* Give up */
+#else
+		/* Try to discover any remote devices */
+		ircomm_tty_start_watchdog_timer(self, 3*HZ);
+		irlmp_discovery_request(DISCOVERY_DEFAULT_SLOTS);
+#endif
+		break;
+	case IRCOMM_TTY_DETACH_CABLE:
+		ircomm_tty_next_state(self, IRCOMM_TTY_IDLE);
+		break;
+	default:
+		pr_debug("%s(), unknown event: %s\n", __func__ ,
+			 ircomm_tty_event[event]);
+		ret = -EINVAL;
+	}
+	return ret;
+}
+
+/*
+ * Function ircomm_tty_state_query (self, event, skb, info)
+ *
+ *    Querying the remote LM-IAS for IrCOMM parameters
+ *
+ */
+static int ircomm_tty_state_query_parameters(struct ircomm_tty_cb *self,
+					     IRCOMM_TTY_EVENT event,
+					     struct sk_buff *skb,
+					     struct ircomm_tty_info *info)
+{
+	int ret = 0;
+
+	pr_debug("%s: state=%s, event=%s\n", __func__ ,
+		 ircomm_tty_state[self->state], ircomm_tty_event[event]);
+
+	switch (event) {
+	case IRCOMM_TTY_GOT_PARAMETERS:
+		if (self->iriap) {
+			net_warn_ratelimited("%s(), busy with a previous query\n",
+					     __func__);
+			return -EBUSY;
+		}
+
+		self->iriap = iriap_open(LSAP_ANY, IAS_CLIENT, self,
+					 ircomm_tty_getvalue_confirm);
+
+		iriap_getvaluebyclass_request(self->iriap, self->saddr,
+					      self->daddr, "IrDA:IrCOMM",
+					      "IrDA:TinyTP:LsapSel");
+
+		ircomm_tty_start_watchdog_timer(self, 3*HZ);
+		ircomm_tty_next_state(self, IRCOMM_TTY_QUERY_LSAP_SEL);
+		break;
+	case IRCOMM_TTY_WD_TIMER_EXPIRED:
+		/* Go back to search mode */
+		ircomm_tty_next_state(self, IRCOMM_TTY_SEARCH);
+		ircomm_tty_start_watchdog_timer(self, 3*HZ);
+		break;
+	case IRCOMM_TTY_CONNECT_INDICATION:
+		del_timer(&self->watchdog_timer);
+		ircomm_tty_ias_unregister(self);
+
+		/* Accept connection */
+		ircomm_connect_response(self->ircomm, NULL);
+		ircomm_tty_next_state(self, IRCOMM_TTY_READY);
+		break;
+	case IRCOMM_TTY_DETACH_CABLE:
+		ircomm_tty_next_state(self, IRCOMM_TTY_IDLE);
+		break;
+	default:
+		pr_debug("%s(), unknown event: %s\n", __func__ ,
+			 ircomm_tty_event[event]);
+		ret = -EINVAL;
+	}
+	return ret;
+}
+
+/*
+ * Function ircomm_tty_state_query_lsap_sel (self, event, skb, info)
+ *
+ *    Query remote LM-IAS for the LSAP selector which we can connect to
+ *
+ */
+static int ircomm_tty_state_query_lsap_sel(struct ircomm_tty_cb *self,
+					   IRCOMM_TTY_EVENT event,
+					   struct sk_buff *skb,
+					   struct ircomm_tty_info *info)
+{
+	int ret = 0;
+
+	pr_debug("%s: state=%s, event=%s\n", __func__ ,
+		 ircomm_tty_state[self->state], ircomm_tty_event[event]);
+
+	switch (event) {
+	case IRCOMM_TTY_GOT_LSAPSEL:
+		/* Connect to remote device */
+		ret = ircomm_connect_request(self->ircomm, self->dlsap_sel,
+					     self->saddr, self->daddr,
+					     NULL, self->service_type);
+		ircomm_tty_start_watchdog_timer(self, 3*HZ);
+		ircomm_tty_next_state(self, IRCOMM_TTY_SETUP);
+		break;
+	case IRCOMM_TTY_WD_TIMER_EXPIRED:
+		/* Go back to search mode */
+		ircomm_tty_next_state(self, IRCOMM_TTY_SEARCH);
+		ircomm_tty_start_watchdog_timer(self, 3*HZ);
+		break;
+	case IRCOMM_TTY_CONNECT_INDICATION:
+		del_timer(&self->watchdog_timer);
+		ircomm_tty_ias_unregister(self);
+
+		/* Accept connection */
+		ircomm_connect_response(self->ircomm, NULL);
+		ircomm_tty_next_state(self, IRCOMM_TTY_READY);
+		break;
+	case IRCOMM_TTY_DETACH_CABLE:
+		ircomm_tty_next_state(self, IRCOMM_TTY_IDLE);
+		break;
+	default:
+		pr_debug("%s(), unknown event: %s\n", __func__ ,
+			 ircomm_tty_event[event]);
+		ret = -EINVAL;
+	}
+	return ret;
+}
+
+/*
+ * Function ircomm_tty_state_setup (self, event, skb, info)
+ *
+ *    Trying to connect
+ *
+ */
+static int ircomm_tty_state_setup(struct ircomm_tty_cb *self,
+				  IRCOMM_TTY_EVENT event,
+				  struct sk_buff *skb,
+				  struct ircomm_tty_info *info)
+{
+	int ret = 0;
+
+	pr_debug("%s: state=%s, event=%s\n", __func__ ,
+		 ircomm_tty_state[self->state], ircomm_tty_event[event]);
+
+	switch (event) {
+	case IRCOMM_TTY_CONNECT_CONFIRM:
+		del_timer(&self->watchdog_timer);
+		ircomm_tty_ias_unregister(self);
+
+		/*
+		 * Send initial parameters. This will also send out queued
+		 * parameters waiting for the connection to come up
+		 */
+		ircomm_tty_send_initial_parameters(self);
+		ircomm_tty_link_established(self);
+		ircomm_tty_next_state(self, IRCOMM_TTY_READY);
+		break;
+	case IRCOMM_TTY_CONNECT_INDICATION:
+		del_timer(&self->watchdog_timer);
+		ircomm_tty_ias_unregister(self);
+
+		/* Accept connection */
+		ircomm_connect_response(self->ircomm, NULL);
+		ircomm_tty_next_state(self, IRCOMM_TTY_READY);
+		break;
+	case IRCOMM_TTY_WD_TIMER_EXPIRED:
+		/* Go back to search mode */
+		ircomm_tty_next_state(self, IRCOMM_TTY_SEARCH);
+		ircomm_tty_start_watchdog_timer(self, 3*HZ);
+		break;
+	case IRCOMM_TTY_DETACH_CABLE:
+		/* ircomm_disconnect_request(self->ircomm, NULL); */
+		ircomm_tty_next_state(self, IRCOMM_TTY_IDLE);
+		break;
+	default:
+		pr_debug("%s(), unknown event: %s\n", __func__ ,
+			 ircomm_tty_event[event]);
+		ret = -EINVAL;
+	}
+	return ret;
+}
+
+/*
+ * Function ircomm_tty_state_ready (self, event, skb, info)
+ *
+ *    IrCOMM is now connected
+ *
+ */
+static int ircomm_tty_state_ready(struct ircomm_tty_cb *self,
+				  IRCOMM_TTY_EVENT event,
+				  struct sk_buff *skb,
+				  struct ircomm_tty_info *info)
+{
+	int ret = 0;
+
+	switch (event) {
+	case IRCOMM_TTY_DATA_REQUEST:
+		ret = ircomm_data_request(self->ircomm, skb);
+		break;
+	case IRCOMM_TTY_DETACH_CABLE:
+		ircomm_disconnect_request(self->ircomm, NULL);
+		ircomm_tty_next_state(self, IRCOMM_TTY_IDLE);
+		break;
+	case IRCOMM_TTY_DISCONNECT_INDICATION:
+		ircomm_tty_ias_register(self);
+		ircomm_tty_next_state(self, IRCOMM_TTY_SEARCH);
+		ircomm_tty_start_watchdog_timer(self, 3*HZ);
+
+		if (tty_port_check_carrier(&self->port)) {
+			/* Drop carrier */
+			self->settings.dce = IRCOMM_DELTA_CD;
+			ircomm_tty_check_modem_status(self);
+		} else {
+			pr_debug("%s(), hanging up!\n", __func__);
+			tty_port_tty_hangup(&self->port, false);
+		}
+		break;
+	default:
+		pr_debug("%s(), unknown event: %s\n", __func__ ,
+			 ircomm_tty_event[event]);
+		ret = -EINVAL;
+	}
+	return ret;
+}
+
diff --git a/drivers/staging/irda/net/ircomm/ircomm_tty_ioctl.c b/drivers/staging/irda/net/ircomm/ircomm_tty_ioctl.c
new file mode 100644
index 000000000000..171c3dee760e
--- /dev/null
+++ b/drivers/staging/irda/net/ircomm/ircomm_tty_ioctl.c
@@ -0,0 +1,291 @@
+/*********************************************************************
+ *
+ * Filename:      ircomm_tty_ioctl.c
+ * Version:
+ * Description:
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Thu Jun 10 14:39:09 1999
+ * Modified at:   Wed Jan  5 14:45:43 2000
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1999-2000 Dag Brattli, All Rights Reserved.
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ *
+ *     You should have received a copy of the GNU General Public License
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ ********************************************************************/
+
+#include <linux/init.h>
+#include <linux/fs.h>
+#include <linux/termios.h>
+#include <linux/tty.h>
+#include <linux/serial.h>
+
+#include <linux/uaccess.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/irmod.h>
+
+#include <net/irda/ircomm_core.h>
+#include <net/irda/ircomm_param.h>
+#include <net/irda/ircomm_tty_attach.h>
+#include <net/irda/ircomm_tty.h>
+
+#define RELEVANT_IFLAG(iflag) (iflag & (IGNBRK|BRKINT|IGNPAR|PARMRK|INPCK))
+
+/*
+ * Function ircomm_tty_change_speed (driver)
+ *
+ *    Change speed of the driver. If the remote device is a DCE, then this
+ *    should make it change the speed of its serial port
+ */
+static void ircomm_tty_change_speed(struct ircomm_tty_cb *self,
+		struct tty_struct *tty)
+{
+	unsigned int cflag, cval;
+	int baud;
+
+	if (!self->ircomm)
+		return;
+
+	cflag = tty->termios.c_cflag;
+
+	/*  byte size and parity */
+	switch (cflag & CSIZE) {
+	case CS5: cval = IRCOMM_WSIZE_5; break;
+	case CS6: cval = IRCOMM_WSIZE_6; break;
+	case CS7: cval = IRCOMM_WSIZE_7; break;
+	case CS8: cval = IRCOMM_WSIZE_8; break;
+	default:  cval = IRCOMM_WSIZE_5; break;
+	}
+	if (cflag & CSTOPB)
+		cval |= IRCOMM_2_STOP_BIT;
+
+	if (cflag & PARENB)
+		cval |= IRCOMM_PARITY_ENABLE;
+	if (!(cflag & PARODD))
+		cval |= IRCOMM_PARITY_EVEN;
+
+	/* Determine divisor based on baud rate */
+	baud = tty_get_baud_rate(tty);
+	if (!baud)
+		baud = 9600;	/* B0 transition handled in rs_set_termios */
+
+	self->settings.data_rate = baud;
+	ircomm_param_request(self, IRCOMM_DATA_RATE, FALSE);
+
+	/* CTS flow control flag and modem status interrupts */
+	tty_port_set_cts_flow(&self->port, cflag & CRTSCTS);
+	if (cflag & CRTSCTS) {
+		self->settings.flow_control |= IRCOMM_RTS_CTS_IN;
+		/* This got me. Bummer. Jean II */
+		if (self->service_type == IRCOMM_3_WIRE_RAW)
+			net_warn_ratelimited("%s(), enabling RTS/CTS on link that doesn't support it (3-wire-raw)\n",
+					     __func__);
+	} else {
+		self->settings.flow_control &= ~IRCOMM_RTS_CTS_IN;
+	}
+	tty_port_set_check_carrier(&self->port, ~cflag & CLOCAL);
+
+	self->settings.data_format = cval;
+
+	ircomm_param_request(self, IRCOMM_DATA_FORMAT, FALSE);
+	ircomm_param_request(self, IRCOMM_FLOW_CONTROL, TRUE);
+}
+
+/*
+ * Function ircomm_tty_set_termios (tty, old_termios)
+ *
+ *    This routine allows the tty driver to be notified when device's
+ *    termios settings have changed.  Note that a well-designed tty driver
+ *    should be prepared to accept the case where old == NULL, and try to
+ *    do something rational.
+ */
+void ircomm_tty_set_termios(struct tty_struct *tty,
+			    struct ktermios *old_termios)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) tty->driver_data;
+	unsigned int cflag = tty->termios.c_cflag;
+
+	if ((cflag == old_termios->c_cflag) &&
+	    (RELEVANT_IFLAG(tty->termios.c_iflag) ==
+	     RELEVANT_IFLAG(old_termios->c_iflag)))
+	{
+		return;
+	}
+
+	ircomm_tty_change_speed(self, tty);
+
+	/* Handle transition to B0 status */
+	if ((old_termios->c_cflag & CBAUD) && !(cflag & CBAUD)) {
+		self->settings.dte &= ~(IRCOMM_DTR|IRCOMM_RTS);
+		ircomm_param_request(self, IRCOMM_DTE, TRUE);
+	}
+
+	/* Handle transition away from B0 status */
+	if (!(old_termios->c_cflag & CBAUD) && (cflag & CBAUD)) {
+		self->settings.dte |= IRCOMM_DTR;
+		if (!C_CRTSCTS(tty) || !tty_throttled(tty))
+			self->settings.dte |= IRCOMM_RTS;
+		ircomm_param_request(self, IRCOMM_DTE, TRUE);
+	}
+
+	/* Handle turning off CRTSCTS */
+	if ((old_termios->c_cflag & CRTSCTS) && !C_CRTSCTS(tty))
+	{
+		tty->hw_stopped = 0;
+		ircomm_tty_start(tty);
+	}
+}
+
+/*
+ * Function ircomm_tty_tiocmget (tty)
+ *
+ *
+ *
+ */
+int ircomm_tty_tiocmget(struct tty_struct *tty)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) tty->driver_data;
+	unsigned int result;
+
+	if (tty_io_error(tty))
+		return -EIO;
+
+	result =  ((self->settings.dte & IRCOMM_RTS) ? TIOCM_RTS : 0)
+		| ((self->settings.dte & IRCOMM_DTR) ? TIOCM_DTR : 0)
+		| ((self->settings.dce & IRCOMM_CD)  ? TIOCM_CAR : 0)
+		| ((self->settings.dce & IRCOMM_RI)  ? TIOCM_RNG : 0)
+		| ((self->settings.dce & IRCOMM_DSR) ? TIOCM_DSR : 0)
+		| ((self->settings.dce & IRCOMM_CTS) ? TIOCM_CTS : 0);
+	return result;
+}
+
+/*
+ * Function ircomm_tty_tiocmset (tty, set, clear)
+ *
+ *
+ *
+ */
+int ircomm_tty_tiocmset(struct tty_struct *tty,
+			unsigned int set, unsigned int clear)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) tty->driver_data;
+
+	if (tty_io_error(tty))
+		return -EIO;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRCOMM_TTY_MAGIC, return -1;);
+
+	if (set & TIOCM_RTS)
+		self->settings.dte |= IRCOMM_RTS;
+	if (set & TIOCM_DTR)
+		self->settings.dte |= IRCOMM_DTR;
+
+	if (clear & TIOCM_RTS)
+		self->settings.dte &= ~IRCOMM_RTS;
+	if (clear & TIOCM_DTR)
+		self->settings.dte &= ~IRCOMM_DTR;
+
+	if ((set|clear) & TIOCM_RTS)
+		self->settings.dte |= IRCOMM_DELTA_RTS;
+	if ((set|clear) & TIOCM_DTR)
+		self->settings.dte |= IRCOMM_DELTA_DTR;
+
+	ircomm_param_request(self, IRCOMM_DTE, TRUE);
+
+	return 0;
+}
+
+/*
+ * Function get_serial_info (driver, retinfo)
+ *
+ *
+ *
+ */
+static int ircomm_tty_get_serial_info(struct ircomm_tty_cb *self,
+				      struct serial_struct __user *retinfo)
+{
+	struct serial_struct info;
+
+	memset(&info, 0, sizeof(info));
+	info.line = self->line;
+	info.flags = self->port.flags;
+	info.baud_base = self->settings.data_rate;
+	info.close_delay = self->port.close_delay;
+	info.closing_wait = self->port.closing_wait;
+
+	/* For compatibility  */
+	info.type = PORT_16550A;
+
+	if (copy_to_user(retinfo, &info, sizeof(*retinfo)))
+		return -EFAULT;
+
+	return 0;
+}
+
+/*
+ * Function set_serial_info (driver, new_info)
+ *
+ *
+ *
+ */
+static int ircomm_tty_set_serial_info(struct ircomm_tty_cb *self,
+				      struct serial_struct __user *new_info)
+{
+	return 0;
+}
+
+/*
+ * Function ircomm_tty_ioctl (tty, cmd, arg)
+ *
+ *
+ *
+ */
+int ircomm_tty_ioctl(struct tty_struct *tty,
+		     unsigned int cmd, unsigned long arg)
+{
+	struct ircomm_tty_cb *self = (struct ircomm_tty_cb *) tty->driver_data;
+	int ret = 0;
+
+	if ((cmd != TIOCGSERIAL) && (cmd != TIOCSSERIAL) &&
+	    (cmd != TIOCSERCONFIG) && (cmd != TIOCSERGSTRUCT) &&
+	    (cmd != TIOCMIWAIT) && (cmd != TIOCGICOUNT)) {
+		if (tty_io_error(tty))
+		    return -EIO;
+	}
+
+	switch (cmd) {
+	case TIOCGSERIAL:
+		ret = ircomm_tty_get_serial_info(self, (struct serial_struct __user *) arg);
+		break;
+	case TIOCSSERIAL:
+		ret = ircomm_tty_set_serial_info(self, (struct serial_struct __user *) arg);
+		break;
+	case TIOCMIWAIT:
+		pr_debug("(), TIOCMIWAIT, not impl!\n");
+		break;
+
+	case TIOCGICOUNT:
+		pr_debug("%s(), TIOCGICOUNT not impl!\n", __func__);
+		return 0;
+	default:
+		ret = -ENOIOCTLCMD;  /* ioctls which we must ignore */
+	}
+	return ret;
+}
+
+
+
diff --git a/drivers/staging/irda/net/irda_device.c b/drivers/staging/irda/net/irda_device.c
new file mode 100644
index 000000000000..890b90d055d5
--- /dev/null
+++ b/drivers/staging/irda/net/irda_device.c
@@ -0,0 +1,316 @@
+/*********************************************************************
+ *
+ * Filename:      irda_device.c
+ * Version:       0.9
+ * Description:   Utility functions used by the device drivers
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sat Oct  9 09:22:27 1999
+ * Modified at:   Sun Jan 23 17:41:24 2000
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1999-2000 Dag Brattli, All Rights Reserved.
+ *     Copyright (c) 2000-2001 Jean Tourrilhes <jt@hpl.hp.com>
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ *
+ *     You should have received a copy of the GNU General Public License
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ ********************************************************************/
+
+#include <linux/string.h>
+#include <linux/proc_fs.h>
+#include <linux/skbuff.h>
+#include <linux/capability.h>
+#include <linux/if.h>
+#include <linux/if_ether.h>
+#include <linux/if_arp.h>
+#include <linux/netdevice.h>
+#include <linux/init.h>
+#include <linux/tty.h>
+#include <linux/kmod.h>
+#include <linux/spinlock.h>
+#include <linux/slab.h>
+#include <linux/export.h>
+
+#include <asm/ioctls.h>
+#include <linux/uaccess.h>
+#include <asm/dma.h>
+#include <asm/io.h>
+
+#include <net/irda/irda_device.h>
+#include <net/irda/irlap.h>
+#include <net/irda/timer.h>
+#include <net/irda/wrapper.h>
+
+static void __irda_task_delete(struct irda_task *task);
+
+static hashbin_t *dongles = NULL;
+static hashbin_t *tasks = NULL;
+
+static void irda_task_timer_expired(void *data);
+
+int __init irda_device_init( void)
+{
+	dongles = hashbin_new(HB_NOLOCK);
+	if (dongles == NULL) {
+		net_warn_ratelimited("IrDA: Can't allocate dongles hashbin!\n");
+		return -ENOMEM;
+	}
+	spin_lock_init(&dongles->hb_spinlock);
+
+	tasks = hashbin_new(HB_LOCK);
+	if (tasks == NULL) {
+		net_warn_ratelimited("IrDA: Can't allocate tasks hashbin!\n");
+		hashbin_delete(dongles, NULL);
+		return -ENOMEM;
+	}
+
+	/* We no longer initialise the driver ourselves here, we let
+	 * the system do it for us... - Jean II */
+
+	return 0;
+}
+
+static void leftover_dongle(void *arg)
+{
+	struct dongle_reg *reg = arg;
+	net_warn_ratelimited("IrDA: Dongle type %x not unregistered\n",
+			     reg->type);
+}
+
+void irda_device_cleanup(void)
+{
+	hashbin_delete(tasks, (FREE_FUNC) __irda_task_delete);
+
+	hashbin_delete(dongles, leftover_dongle);
+}
+
+/*
+ * Function irda_device_set_media_busy (self, status)
+ *
+ *    Called when we have detected that another station is transmitting
+ *    in contention mode.
+ */
+void irda_device_set_media_busy(struct net_device *dev, int status)
+{
+	struct irlap_cb *self;
+
+	pr_debug("%s(%s)\n", __func__, status ? "TRUE" : "FALSE");
+
+	self = (struct irlap_cb *) dev->atalk_ptr;
+
+	/* Some drivers may enable the receive interrupt before calling
+	 * irlap_open(), or they may disable the receive interrupt
+	 * after calling irlap_close().
+	 * The IrDA stack is protected from this in irlap_driver_rcv().
+	 * However, the driver calls directly the wrapper, that calls
+	 * us directly. Make sure we protect ourselves.
+	 * Jean II */
+	if (!self || self->magic != LAP_MAGIC)
+		return;
+
+	if (status) {
+		self->media_busy = TRUE;
+		if (status == SMALL)
+			irlap_start_mbusy_timer(self, SMALLBUSY_TIMEOUT);
+		else
+			irlap_start_mbusy_timer(self, MEDIABUSY_TIMEOUT);
+		pr_debug("Media busy!\n");
+	} else {
+		self->media_busy = FALSE;
+		irlap_stop_mbusy_timer(self);
+	}
+}
+EXPORT_SYMBOL(irda_device_set_media_busy);
+
+
+/*
+ * Function irda_device_is_receiving (dev)
+ *
+ *    Check if the device driver is currently receiving data
+ *
+ */
+int irda_device_is_receiving(struct net_device *dev)
+{
+	struct if_irda_req req;
+	int ret;
+
+	if (!dev->netdev_ops->ndo_do_ioctl) {
+		net_err_ratelimited("%s: do_ioctl not impl. by device driver\n",
+				    __func__);
+		return -1;
+	}
+
+	ret = (dev->netdev_ops->ndo_do_ioctl)(dev, (struct ifreq *) &req,
+					      SIOCGRECEIVING);
+	if (ret < 0)
+		return ret;
+
+	return req.ifr_receiving;
+}
+
+static void __irda_task_delete(struct irda_task *task)
+{
+	del_timer(&task->timer);
+
+	kfree(task);
+}
+
+static void irda_task_delete(struct irda_task *task)
+{
+	/* Unregister task */
+	hashbin_remove(tasks, (long) task, NULL);
+
+	__irda_task_delete(task);
+}
+
+/*
+ * Function irda_task_kick (task)
+ *
+ *    Tries to execute a task possible multiple times until the task is either
+ *    finished, or askes for a timeout. When a task is finished, we do post
+ *    processing, and notify the parent task, that is waiting for this task
+ *    to complete.
+ */
+static int irda_task_kick(struct irda_task *task)
+{
+	int finished = TRUE;
+	int count = 0;
+	int timeout;
+
+	IRDA_ASSERT(task != NULL, return -1;);
+	IRDA_ASSERT(task->magic == IRDA_TASK_MAGIC, return -1;);
+
+	/* Execute task until it's finished, or askes for a timeout */
+	do {
+		timeout = task->function(task);
+		if (count++ > 100) {
+			net_err_ratelimited("%s: error in task handler!\n",
+					    __func__);
+			irda_task_delete(task);
+			return TRUE;
+		}
+	} while ((timeout == 0) && (task->state != IRDA_TASK_DONE));
+
+	if (timeout < 0) {
+		net_err_ratelimited("%s: Error executing task!\n", __func__);
+		irda_task_delete(task);
+		return TRUE;
+	}
+
+	/* Check if we are finished */
+	if (task->state == IRDA_TASK_DONE) {
+		del_timer(&task->timer);
+
+		/* Do post processing */
+		if (task->finished)
+			task->finished(task);
+
+		/* Notify parent */
+		if (task->parent) {
+			/* Check if parent is waiting for us to complete */
+			if (task->parent->state == IRDA_TASK_CHILD_WAIT) {
+				task->parent->state = IRDA_TASK_CHILD_DONE;
+
+				/* Stop timer now that we are here */
+				del_timer(&task->parent->timer);
+
+				/* Kick parent task */
+				irda_task_kick(task->parent);
+			}
+		}
+		irda_task_delete(task);
+	} else if (timeout > 0) {
+		irda_start_timer(&task->timer, timeout, (void *) task,
+				 irda_task_timer_expired);
+		finished = FALSE;
+	} else {
+		pr_debug("%s(), not finished, and no timeout!\n",
+			 __func__);
+		finished = FALSE;
+	}
+
+	return finished;
+}
+
+/*
+ * Function irda_task_timer_expired (data)
+ *
+ *    Task time has expired. We now try to execute task (again), and restart
+ *    the timer if the task has not finished yet
+ */
+static void irda_task_timer_expired(void *data)
+{
+	struct irda_task *task;
+
+	task = data;
+
+	irda_task_kick(task);
+}
+
+/*
+ * Function irda_device_setup (dev)
+ *
+ *    This function should be used by low level device drivers in a similar way
+ *    as ether_setup() is used by normal network device drivers
+ */
+static void irda_device_setup(struct net_device *dev)
+{
+	dev->hard_header_len = 0;
+	dev->addr_len        = LAP_ALEN;
+
+	dev->type            = ARPHRD_IRDA;
+	dev->tx_queue_len    = 8; /* Window size + 1 s-frame */
+
+	memset(dev->broadcast, 0xff, LAP_ALEN);
+
+	dev->mtu = 2048;
+	dev->flags = IFF_NOARP;
+}
+
+/*
+ * Funciton  alloc_irdadev
+ * 	Allocates and sets up an IRDA device in a manner similar to
+ * 	alloc_etherdev.
+ */
+struct net_device *alloc_irdadev(int sizeof_priv)
+{
+	return alloc_netdev(sizeof_priv, "irda%d", NET_NAME_UNKNOWN,
+			    irda_device_setup);
+}
+EXPORT_SYMBOL(alloc_irdadev);
+
+#ifdef CONFIG_ISA_DMA_API
+/*
+ * Function setup_dma (idev, buffer, count, mode)
+ *
+ *    Setup the DMA channel. Commonly used by LPC FIR drivers
+ *
+ */
+void irda_setup_dma(int channel, dma_addr_t buffer, int count, int mode)
+{
+	unsigned long flags;
+
+	flags = claim_dma_lock();
+
+	disable_dma(channel);
+	clear_dma_ff(channel);
+	set_dma_mode(channel, mode);
+	set_dma_addr(channel, buffer);
+	set_dma_count(channel, count);
+	enable_dma(channel);
+
+	release_dma_lock(flags);
+}
+EXPORT_SYMBOL(irda_setup_dma);
+#endif
diff --git a/drivers/staging/irda/net/iriap.c b/drivers/staging/irda/net/iriap.c
new file mode 100644
index 000000000000..1138eaf5c682
--- /dev/null
+++ b/drivers/staging/irda/net/iriap.c
@@ -0,0 +1,1085 @@
+/*********************************************************************
+ *
+ * Filename:      iriap.c
+ * Version:       0.8
+ * Description:   Information Access Protocol (IAP)
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Thu Aug 21 00:02:07 1997
+ * Modified at:   Sat Dec 25 16:42:42 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1998-1999 Dag Brattli <dagb@cs.uit.no>,
+ *     All Rights Reserved.
+ *     Copyright (c) 2000-2003 Jean Tourrilhes <jt@hpl.hp.com>
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/skbuff.h>
+#include <linux/fs.h>
+#include <linux/string.h>
+#include <linux/init.h>
+#include <linux/seq_file.h>
+#include <linux/slab.h>
+
+#include <asm/byteorder.h>
+#include <asm/unaligned.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/irttp.h>
+#include <net/irda/irlmp.h>
+#include <net/irda/irias_object.h>
+#include <net/irda/iriap_event.h>
+#include <net/irda/iriap.h>
+
+/* FIXME: This one should go in irlmp.c */
+static const char *const ias_charset_types[] __maybe_unused = {
+	"CS_ASCII",
+	"CS_ISO_8859_1",
+	"CS_ISO_8859_2",
+	"CS_ISO_8859_3",
+	"CS_ISO_8859_4",
+	"CS_ISO_8859_5",
+	"CS_ISO_8859_6",
+	"CS_ISO_8859_7",
+	"CS_ISO_8859_8",
+	"CS_ISO_8859_9",
+	"CS_UNICODE"
+};
+
+static hashbin_t *iriap = NULL;
+static void *service_handle;
+
+static void __iriap_close(struct iriap_cb *self);
+static int iriap_register_lsap(struct iriap_cb *self, __u8 slsap_sel, int mode);
+static void iriap_disconnect_indication(void *instance, void *sap,
+					LM_REASON reason, struct sk_buff *skb);
+static void iriap_connect_indication(void *instance, void *sap,
+				     struct qos_info *qos, __u32 max_sdu_size,
+				     __u8 max_header_size,
+				     struct sk_buff *skb);
+static void iriap_connect_confirm(void *instance, void *sap,
+				  struct qos_info *qos,
+				  __u32 max_sdu_size, __u8 max_header_size,
+				  struct sk_buff *skb);
+static int iriap_data_indication(void *instance, void *sap,
+				 struct sk_buff *skb);
+
+static void iriap_watchdog_timer_expired(void *data);
+
+static inline void iriap_start_watchdog_timer(struct iriap_cb *self,
+					      int timeout)
+{
+	irda_start_timer(&self->watchdog_timer, timeout, self,
+			 iriap_watchdog_timer_expired);
+}
+
+static struct lock_class_key irias_objects_key;
+
+/*
+ * Function iriap_init (void)
+ *
+ *    Initializes the IrIAP layer, called by the module initialization code
+ *    in irmod.c
+ */
+int __init iriap_init(void)
+{
+	struct ias_object *obj;
+	struct iriap_cb *server;
+	__u8 oct_seq[6];
+	__u16 hints;
+
+	/* Allocate master array */
+	iriap = hashbin_new(HB_LOCK);
+	if (!iriap)
+		return -ENOMEM;
+
+	/* Object repository - defined in irias_object.c */
+	irias_objects = hashbin_new(HB_LOCK);
+	if (!irias_objects) {
+		net_warn_ratelimited("%s: Can't allocate irias_objects hashbin!\n",
+				     __func__);
+		hashbin_delete(iriap, NULL);
+		return -ENOMEM;
+	}
+
+	lockdep_set_class_and_name(&irias_objects->hb_spinlock, &irias_objects_key,
+				   "irias_objects");
+
+	/*
+	 *  Register some default services for IrLMP
+	 */
+	hints  = irlmp_service_to_hint(S_COMPUTER);
+	service_handle = irlmp_register_service(hints);
+
+	/* Register the Device object with LM-IAS */
+	obj = irias_new_object("Device", IAS_DEVICE_ID);
+	irias_add_string_attrib(obj, "DeviceName", "Linux", IAS_KERNEL_ATTR);
+
+	oct_seq[0] = 0x01;  /* Version 1 */
+	oct_seq[1] = 0x00;  /* IAS support bits */
+	oct_seq[2] = 0x00;  /* LM-MUX support bits */
+#ifdef CONFIG_IRDA_ULTRA
+	oct_seq[2] |= 0x04; /* Connectionless Data support */
+#endif
+	irias_add_octseq_attrib(obj, "IrLMPSupport", oct_seq, 3,
+				IAS_KERNEL_ATTR);
+	irias_insert_object(obj);
+
+	/*
+	 *  Register server support with IrLMP so we can accept incoming
+	 *  connections
+	 */
+	server = iriap_open(LSAP_IAS, IAS_SERVER, NULL, NULL);
+	if (!server) {
+		pr_debug("%s(), unable to open server\n", __func__);
+		return -1;
+	}
+	iriap_register_lsap(server, LSAP_IAS, IAS_SERVER);
+
+	return 0;
+}
+
+/*
+ * Function iriap_cleanup (void)
+ *
+ *    Initializes the IrIAP layer, called by the module cleanup code in
+ *    irmod.c
+ */
+void iriap_cleanup(void)
+{
+	irlmp_unregister_service(service_handle);
+
+	hashbin_delete(iriap, (FREE_FUNC) __iriap_close);
+	hashbin_delete(irias_objects, (FREE_FUNC) __irias_delete_object);
+}
+
+/*
+ * Function iriap_open (void)
+ *
+ *    Opens an instance of the IrIAP layer, and registers with IrLMP
+ */
+struct iriap_cb *iriap_open(__u8 slsap_sel, int mode, void *priv,
+			    CONFIRM_CALLBACK callback)
+{
+	struct iriap_cb *self;
+
+	self = kzalloc(sizeof(*self), GFP_ATOMIC);
+	if (!self)
+		return NULL;
+
+	/*
+	 *  Initialize instance
+	 */
+
+	self->magic = IAS_MAGIC;
+	self->mode = mode;
+	if (mode == IAS_CLIENT) {
+		if (iriap_register_lsap(self, slsap_sel, mode)) {
+			kfree(self);
+			return NULL;
+		}
+	}
+
+	self->confirm = callback;
+	self->priv = priv;
+
+	/* iriap_getvaluebyclass_request() will construct packets before
+	 * we connect, so this must have a sane value... Jean II */
+	self->max_header_size = LMP_MAX_HEADER;
+
+	init_timer(&self->watchdog_timer);
+
+	hashbin_insert(iriap, (irda_queue_t *) self, (long) self, NULL);
+
+	/* Initialize state machines */
+	iriap_next_client_state(self, S_DISCONNECT);
+	iriap_next_call_state(self, S_MAKE_CALL);
+	iriap_next_server_state(self, R_DISCONNECT);
+	iriap_next_r_connect_state(self, R_WAITING);
+
+	return self;
+}
+EXPORT_SYMBOL(iriap_open);
+
+/*
+ * Function __iriap_close (self)
+ *
+ *    Removes (deallocates) the IrIAP instance
+ *
+ */
+static void __iriap_close(struct iriap_cb *self)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, return;);
+
+	del_timer(&self->watchdog_timer);
+
+	if (self->request_skb)
+		dev_kfree_skb(self->request_skb);
+
+	self->magic = 0;
+
+	kfree(self);
+}
+
+/*
+ * Function iriap_close (void)
+ *
+ *    Closes IrIAP and deregisters with IrLMP
+ */
+void iriap_close(struct iriap_cb *self)
+{
+	struct iriap_cb *entry;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, return;);
+
+	if (self->lsap) {
+		irlmp_close_lsap(self->lsap);
+		self->lsap = NULL;
+	}
+
+	entry = (struct iriap_cb *) hashbin_remove(iriap, (long) self, NULL);
+	IRDA_ASSERT(entry == self, return;);
+
+	__iriap_close(self);
+}
+EXPORT_SYMBOL(iriap_close);
+
+static int iriap_register_lsap(struct iriap_cb *self, __u8 slsap_sel, int mode)
+{
+	notify_t notify;
+
+	irda_notify_init(&notify);
+	notify.connect_confirm       = iriap_connect_confirm;
+	notify.connect_indication    = iriap_connect_indication;
+	notify.disconnect_indication = iriap_disconnect_indication;
+	notify.data_indication       = iriap_data_indication;
+	notify.instance = self;
+	if (mode == IAS_CLIENT)
+		strcpy(notify.name, "IrIAS cli");
+	else
+		strcpy(notify.name, "IrIAS srv");
+
+	self->lsap = irlmp_open_lsap(slsap_sel, &notify, 0);
+	if (self->lsap == NULL) {
+		net_err_ratelimited("%s: Unable to allocated LSAP!\n",
+				    __func__);
+		return -1;
+	}
+	self->slsap_sel = self->lsap->slsap_sel;
+
+	return 0;
+}
+
+/*
+ * Function iriap_disconnect_indication (handle, reason)
+ *
+ *    Got disconnect, so clean up everything associated with this connection
+ *
+ */
+static void iriap_disconnect_indication(void *instance, void *sap,
+					LM_REASON reason,
+					struct sk_buff *skb)
+{
+	struct iriap_cb *self;
+
+	pr_debug("%s(), reason=%s [%d]\n", __func__,
+		 irlmp_reason_str(reason), reason);
+
+	self = instance;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, return;);
+
+	IRDA_ASSERT(iriap != NULL, return;);
+
+	del_timer(&self->watchdog_timer);
+
+	/* Not needed */
+	if (skb)
+		dev_kfree_skb(skb);
+
+	if (self->mode == IAS_CLIENT) {
+		pr_debug("%s(), disconnect as client\n", __func__);
+
+
+		iriap_do_client_event(self, IAP_LM_DISCONNECT_INDICATION,
+				      NULL);
+		/*
+		 * Inform service user that the request failed by sending
+		 * it a NULL value. Warning, the client might close us, so
+		 * remember no to use self anymore after calling confirm
+		 */
+		if (self->confirm)
+			self->confirm(IAS_DISCONNECT, 0, NULL, self->priv);
+	} else {
+		pr_debug("%s(), disconnect as server\n", __func__);
+		iriap_do_server_event(self, IAP_LM_DISCONNECT_INDICATION,
+				      NULL);
+		iriap_close(self);
+	}
+}
+
+/*
+ * Function iriap_disconnect_request (handle)
+ */
+static void iriap_disconnect_request(struct iriap_cb *self)
+{
+	struct sk_buff *tx_skb;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, return;);
+
+	tx_skb = alloc_skb(LMP_MAX_HEADER, GFP_ATOMIC);
+	if (tx_skb == NULL) {
+		pr_debug("%s(), Could not allocate an sk_buff of length %d\n",
+			 __func__, LMP_MAX_HEADER);
+		return;
+	}
+
+	/*
+	 *  Reserve space for MUX control and LAP header
+	 */
+	skb_reserve(tx_skb, LMP_MAX_HEADER);
+
+	irlmp_disconnect_request(self->lsap, tx_skb);
+}
+
+/*
+ * Function iriap_getvaluebyclass (addr, name, attr)
+ *
+ *    Retrieve all values from attribute in all objects with given class
+ *    name
+ */
+int iriap_getvaluebyclass_request(struct iriap_cb *self,
+				  __u32 saddr, __u32 daddr,
+				  char *name, char *attr)
+{
+	struct sk_buff *tx_skb;
+	int name_len, attr_len, skb_len;
+	__u8 *frame;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, return -1;);
+
+	/* Client must supply the destination device address */
+	if (!daddr)
+		return -1;
+
+	self->daddr = daddr;
+	self->saddr = saddr;
+
+	/*
+	 *  Save operation, so we know what the later indication is about
+	 */
+	self->operation = GET_VALUE_BY_CLASS;
+
+	/* Give ourselves 10 secs to finish this operation */
+	iriap_start_watchdog_timer(self, 10*HZ);
+
+	name_len = strlen(name);	/* Up to IAS_MAX_CLASSNAME = 60 */
+	attr_len = strlen(attr);	/* Up to IAS_MAX_ATTRIBNAME = 60 */
+
+	skb_len = self->max_header_size+2+name_len+1+attr_len+4;
+	tx_skb = alloc_skb(skb_len, GFP_ATOMIC);
+	if (!tx_skb)
+		return -ENOMEM;
+
+	/* Reserve space for MUX and LAP header */
+	skb_reserve(tx_skb, self->max_header_size);
+	skb_put(tx_skb, 3+name_len+attr_len);
+	frame = tx_skb->data;
+
+	/* Build frame */
+	frame[0] = IAP_LST | GET_VALUE_BY_CLASS;
+	frame[1] = name_len;                       /* Insert length of name */
+	memcpy(frame+2, name, name_len);           /* Insert name */
+	frame[2+name_len] = attr_len;              /* Insert length of attr */
+	memcpy(frame+3+name_len, attr, attr_len);  /* Insert attr */
+
+	iriap_do_client_event(self, IAP_CALL_REQUEST_GVBC, tx_skb);
+
+	/* Drop reference count - see state_s_disconnect(). */
+	dev_kfree_skb(tx_skb);
+
+	return 0;
+}
+EXPORT_SYMBOL(iriap_getvaluebyclass_request);
+
+/*
+ * Function iriap_getvaluebyclass_confirm (self, skb)
+ *
+ *    Got result from GetValueByClass command. Parse it and return result
+ *    to service user.
+ *
+ */
+static void iriap_getvaluebyclass_confirm(struct iriap_cb *self,
+					  struct sk_buff *skb)
+{
+	struct ias_value *value;
+	int charset;
+	__u32 value_len;
+	__u32 tmp_cpu32;
+	__u16 obj_id;
+	__u16 len;
+	__u8  type;
+	__u8 *fp;
+	int n;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, return;);
+	IRDA_ASSERT(skb != NULL, return;);
+
+	/* Initialize variables */
+	fp = skb->data;
+	n = 2;
+
+	/* Get length, MSB first */
+	len = get_unaligned_be16(fp + n);
+	n += 2;
+
+	pr_debug("%s(), len=%d\n", __func__, len);
+
+	/* Get object ID, MSB first */
+	obj_id = get_unaligned_be16(fp + n);
+	n += 2;
+
+	type = fp[n++];
+	pr_debug("%s(), Value type = %d\n", __func__, type);
+
+	switch (type) {
+	case IAS_INTEGER:
+		memcpy(&tmp_cpu32, fp+n, 4); n += 4;
+		be32_to_cpus(&tmp_cpu32);
+		value = irias_new_integer_value(tmp_cpu32);
+
+		/*  Legal values restricted to 0x01-0x6f, page 15 irttp */
+		pr_debug("%s(), lsap=%d\n", __func__, value->t.integer);
+		break;
+	case IAS_STRING:
+		charset = fp[n++];
+
+		switch (charset) {
+		case CS_ASCII:
+			break;
+/*		case CS_ISO_8859_1: */
+/*		case CS_ISO_8859_2: */
+/*		case CS_ISO_8859_3: */
+/*		case CS_ISO_8859_4: */
+/*		case CS_ISO_8859_5: */
+/*		case CS_ISO_8859_6: */
+/*		case CS_ISO_8859_7: */
+/*		case CS_ISO_8859_8: */
+/*		case CS_ISO_8859_9: */
+/*		case CS_UNICODE: */
+		default:
+			pr_debug("%s(), charset [%d] %s, not supported\n",
+				 __func__, charset,
+				 charset < ARRAY_SIZE(ias_charset_types) ?
+				 ias_charset_types[charset] :
+				 "(unknown)");
+
+			/* Aborting, close connection! */
+			iriap_disconnect_request(self);
+			return;
+			/* break; */
+		}
+		value_len = fp[n++];
+		pr_debug("%s(), strlen=%d\n", __func__, value_len);
+
+		/* Make sure the string is null-terminated */
+		if (n + value_len < skb->len)
+			fp[n + value_len] = 0x00;
+		pr_debug("Got string %s\n", fp+n);
+
+		/* Will truncate to IAS_MAX_STRING bytes */
+		value = irias_new_string_value(fp+n);
+		break;
+	case IAS_OCT_SEQ:
+		value_len = get_unaligned_be16(fp + n);
+		n += 2;
+
+		/* Will truncate to IAS_MAX_OCTET_STRING bytes */
+		value = irias_new_octseq_value(fp+n, value_len);
+		break;
+	default:
+		value = irias_new_missing_value();
+		break;
+	}
+
+	/* Finished, close connection! */
+	iriap_disconnect_request(self);
+
+	/* Warning, the client might close us, so remember no to use self
+	 * anymore after calling confirm
+	 */
+	if (self->confirm)
+		self->confirm(IAS_SUCCESS, obj_id, value, self->priv);
+	else {
+		pr_debug("%s(), missing handler!\n", __func__);
+		irias_delete_value(value);
+	}
+}
+
+/*
+ * Function iriap_getvaluebyclass_response ()
+ *
+ *    Send answer back to remote LM-IAS
+ *
+ */
+static void iriap_getvaluebyclass_response(struct iriap_cb *self,
+					   __u16 obj_id,
+					   __u8 ret_code,
+					   struct ias_value *value)
+{
+	struct sk_buff *tx_skb;
+	int n;
+	__be32 tmp_be32;
+	__be16 tmp_be16;
+	__u8 *fp;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, return;);
+	IRDA_ASSERT(value != NULL, return;);
+	IRDA_ASSERT(value->len <= 1024, return;);
+
+	/* Initialize variables */
+	n = 0;
+
+	/*
+	 *  We must adjust the size of the response after the length of the
+	 *  value. We add 32 bytes because of the 6 bytes for the frame and
+	 *  max 5 bytes for the value coding.
+	 */
+	tx_skb = alloc_skb(value->len + self->max_header_size + 32,
+			   GFP_ATOMIC);
+	if (!tx_skb)
+		return;
+
+	/* Reserve space for MUX and LAP header */
+	skb_reserve(tx_skb, self->max_header_size);
+	skb_put(tx_skb, 6);
+
+	fp = tx_skb->data;
+
+	/* Build frame */
+	fp[n++] = GET_VALUE_BY_CLASS | IAP_LST;
+	fp[n++] = ret_code;
+
+	/* Insert list length (MSB first) */
+	tmp_be16 = htons(0x0001);
+	memcpy(fp+n, &tmp_be16, 2);  n += 2;
+
+	/* Insert object identifier ( MSB first) */
+	tmp_be16 = cpu_to_be16(obj_id);
+	memcpy(fp+n, &tmp_be16, 2); n += 2;
+
+	switch (value->type) {
+	case IAS_STRING:
+		skb_put(tx_skb, 3 + value->len);
+		fp[n++] = value->type;
+		fp[n++] = 0; /* ASCII */
+		fp[n++] = (__u8) value->len;
+		memcpy(fp+n, value->t.string, value->len); n+=value->len;
+		break;
+	case IAS_INTEGER:
+		skb_put(tx_skb, 5);
+		fp[n++] = value->type;
+
+		tmp_be32 = cpu_to_be32(value->t.integer);
+		memcpy(fp+n, &tmp_be32, 4); n += 4;
+		break;
+	case IAS_OCT_SEQ:
+		skb_put(tx_skb, 3 + value->len);
+		fp[n++] = value->type;
+
+		tmp_be16 = cpu_to_be16(value->len);
+		memcpy(fp+n, &tmp_be16, 2); n += 2;
+		memcpy(fp+n, value->t.oct_seq, value->len); n+=value->len;
+		break;
+	case IAS_MISSING:
+		pr_debug("%s: sending IAS_MISSING\n", __func__);
+		skb_put(tx_skb, 1);
+		fp[n++] = value->type;
+		break;
+	default:
+		pr_debug("%s(), type not implemented!\n", __func__);
+		break;
+	}
+	iriap_do_r_connect_event(self, IAP_CALL_RESPONSE, tx_skb);
+
+	/* Drop reference count - see state_r_execute(). */
+	dev_kfree_skb(tx_skb);
+}
+
+/*
+ * Function iriap_getvaluebyclass_indication (self, skb)
+ *
+ *    getvaluebyclass is requested from peer LM-IAS
+ *
+ */
+static void iriap_getvaluebyclass_indication(struct iriap_cb *self,
+					     struct sk_buff *skb)
+{
+	struct ias_object *obj;
+	struct ias_attrib *attrib;
+	int name_len;
+	int attr_len;
+	char name[IAS_MAX_CLASSNAME + 1];	/* 60 bytes */
+	char attr[IAS_MAX_ATTRIBNAME + 1];	/* 60 bytes */
+	__u8 *fp;
+	int n;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, return;);
+	IRDA_ASSERT(skb != NULL, return;);
+
+	fp = skb->data;
+	n = 1;
+
+	name_len = fp[n++];
+
+	IRDA_ASSERT(name_len < IAS_MAX_CLASSNAME + 1, return;);
+
+	memcpy(name, fp+n, name_len); n+=name_len;
+	name[name_len] = '\0';
+
+	attr_len = fp[n++];
+
+	IRDA_ASSERT(attr_len < IAS_MAX_ATTRIBNAME + 1, return;);
+
+	memcpy(attr, fp+n, attr_len); n+=attr_len;
+	attr[attr_len] = '\0';
+
+	pr_debug("LM-IAS: Looking up %s: %s\n", name, attr);
+	obj = irias_find_object(name);
+
+	if (obj == NULL) {
+		pr_debug("LM-IAS: Object %s not found\n", name);
+		iriap_getvaluebyclass_response(self, 0x1235, IAS_CLASS_UNKNOWN,
+					       &irias_missing);
+		return;
+	}
+	pr_debug("LM-IAS: found %s, id=%d\n", obj->name, obj->id);
+
+	attrib = irias_find_attrib(obj, attr);
+	if (attrib == NULL) {
+		pr_debug("LM-IAS: Attribute %s not found\n", attr);
+		iriap_getvaluebyclass_response(self, obj->id,
+					       IAS_ATTRIB_UNKNOWN,
+					       &irias_missing);
+		return;
+	}
+
+	/* We have a match; send the value.  */
+	iriap_getvaluebyclass_response(self, obj->id, IAS_SUCCESS,
+				       attrib->value);
+}
+
+/*
+ * Function iriap_send_ack (void)
+ *
+ *    Currently not used
+ *
+ */
+void iriap_send_ack(struct iriap_cb *self)
+{
+	struct sk_buff *tx_skb;
+	__u8 *frame;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, return;);
+
+	tx_skb = alloc_skb(LMP_MAX_HEADER + 1, GFP_ATOMIC);
+	if (!tx_skb)
+		return;
+
+	/* Reserve space for MUX and LAP header */
+	skb_reserve(tx_skb, self->max_header_size);
+	skb_put(tx_skb, 1);
+	frame = tx_skb->data;
+
+	/* Build frame */
+	frame[0] = IAP_LST | IAP_ACK | self->operation;
+
+	irlmp_data_request(self->lsap, tx_skb);
+}
+
+void iriap_connect_request(struct iriap_cb *self)
+{
+	int ret;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, return;);
+
+	ret = irlmp_connect_request(self->lsap, LSAP_IAS,
+				    self->saddr, self->daddr,
+				    NULL, NULL);
+	if (ret < 0) {
+		pr_debug("%s(), connect failed!\n", __func__);
+		self->confirm(IAS_DISCONNECT, 0, NULL, self->priv);
+	}
+}
+
+/*
+ * Function iriap_connect_confirm (handle, skb)
+ *
+ *    LSAP connection confirmed!
+ *
+ */
+static void iriap_connect_confirm(void *instance, void *sap,
+				  struct qos_info *qos, __u32 max_seg_size,
+				  __u8 max_header_size,
+				  struct sk_buff *skb)
+{
+	struct iriap_cb *self;
+
+	self = instance;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, return;);
+	IRDA_ASSERT(skb != NULL, return;);
+
+	self->max_data_size = max_seg_size;
+	self->max_header_size = max_header_size;
+
+	del_timer(&self->watchdog_timer);
+
+	iriap_do_client_event(self, IAP_LM_CONNECT_CONFIRM, skb);
+
+	/* Drop reference count - see state_s_make_call(). */
+	dev_kfree_skb(skb);
+}
+
+/*
+ * Function iriap_connect_indication ( handle, skb)
+ *
+ *    Remote LM-IAS is requesting connection
+ *
+ */
+static void iriap_connect_indication(void *instance, void *sap,
+				     struct qos_info *qos, __u32 max_seg_size,
+				     __u8 max_header_size,
+				     struct sk_buff *skb)
+{
+	struct iriap_cb *self, *new;
+
+	self = instance;
+
+	IRDA_ASSERT(skb != NULL, return;);
+	IRDA_ASSERT(self != NULL, goto out;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, goto out;);
+
+	/* Start new server */
+	new = iriap_open(LSAP_IAS, IAS_SERVER, NULL, NULL);
+	if (!new) {
+		pr_debug("%s(), open failed\n", __func__);
+		goto out;
+	}
+
+	/* Now attach up the new "socket" */
+	new->lsap = irlmp_dup(self->lsap, new);
+	if (!new->lsap) {
+		pr_debug("%s(), dup failed!\n", __func__);
+		goto out;
+	}
+
+	new->max_data_size = max_seg_size;
+	new->max_header_size = max_header_size;
+
+	/* Clean up the original one to keep it in listen state */
+	irlmp_listen(self->lsap);
+
+	iriap_do_server_event(new, IAP_LM_CONNECT_INDICATION, skb);
+
+out:
+	/* Drop reference count - see state_r_disconnect(). */
+	dev_kfree_skb(skb);
+}
+
+/*
+ * Function iriap_data_indication (handle, skb)
+ *
+ *    Receives data from connection identified by handle from IrLMP
+ *
+ */
+static int iriap_data_indication(void *instance, void *sap,
+				 struct sk_buff *skb)
+{
+	struct iriap_cb *self;
+	__u8  *frame;
+	__u8  opcode;
+
+	self = instance;
+
+	IRDA_ASSERT(skb != NULL, return 0;);
+	IRDA_ASSERT(self != NULL, goto out;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, goto out;);
+
+	frame = skb->data;
+
+	if (self->mode == IAS_SERVER) {
+		/* Call server */
+		pr_debug("%s(), Calling server!\n", __func__);
+		iriap_do_r_connect_event(self, IAP_RECV_F_LST, skb);
+		goto out;
+	}
+	opcode = frame[0];
+	if (~opcode & IAP_LST) {
+		net_warn_ratelimited("%s:, IrIAS multiframe commands or results is not implemented yet!\n",
+				     __func__);
+		goto out;
+	}
+
+	/* Check for ack frames since they don't contain any data */
+	if (opcode & IAP_ACK) {
+		pr_debug("%s() Got ack frame!\n", __func__);
+		goto out;
+	}
+
+	opcode &= ~IAP_LST; /* Mask away LST bit */
+
+	switch (opcode) {
+	case GET_INFO_BASE:
+		pr_debug("IrLMP GetInfoBaseDetails not implemented!\n");
+		break;
+	case GET_VALUE_BY_CLASS:
+		iriap_do_call_event(self, IAP_RECV_F_LST, NULL);
+
+		switch (frame[1]) {
+		case IAS_SUCCESS:
+			iriap_getvaluebyclass_confirm(self, skb);
+			break;
+		case IAS_CLASS_UNKNOWN:
+			pr_debug("%s(), No such class!\n", __func__);
+			/* Finished, close connection! */
+			iriap_disconnect_request(self);
+
+			/*
+			 * Warning, the client might close us, so remember
+			 * no to use self anymore after calling confirm
+			 */
+			if (self->confirm)
+				self->confirm(IAS_CLASS_UNKNOWN, 0, NULL,
+					      self->priv);
+			break;
+		case IAS_ATTRIB_UNKNOWN:
+			pr_debug("%s(), No such attribute!\n", __func__);
+			/* Finished, close connection! */
+			iriap_disconnect_request(self);
+
+			/*
+			 * Warning, the client might close us, so remember
+			 * no to use self anymore after calling confirm
+			 */
+			if (self->confirm)
+				self->confirm(IAS_ATTRIB_UNKNOWN, 0, NULL,
+					      self->priv);
+			break;
+		}
+		break;
+	default:
+		pr_debug("%s(), Unknown op-code: %02x\n", __func__,
+			 opcode);
+		break;
+	}
+
+out:
+	/* Cleanup - sub-calls will have done skb_get() as needed. */
+	dev_kfree_skb(skb);
+	return 0;
+}
+
+/*
+ * Function iriap_call_indication (self, skb)
+ *
+ *    Received call to server from peer LM-IAS
+ *
+ */
+void iriap_call_indication(struct iriap_cb *self, struct sk_buff *skb)
+{
+	__u8 *fp;
+	__u8 opcode;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, return;);
+	IRDA_ASSERT(skb != NULL, return;);
+
+	fp = skb->data;
+
+	opcode = fp[0];
+	if (~opcode & 0x80) {
+		net_warn_ratelimited("%s: IrIAS multiframe commands or results is not implemented yet!\n",
+				     __func__);
+		return;
+	}
+	opcode &= 0x7f; /* Mask away LST bit */
+
+	switch (opcode) {
+	case GET_INFO_BASE:
+		net_warn_ratelimited("%s: GetInfoBaseDetails not implemented yet!\n",
+				     __func__);
+		break;
+	case GET_VALUE_BY_CLASS:
+		iriap_getvaluebyclass_indication(self, skb);
+		break;
+	}
+	/* skb will be cleaned up in iriap_data_indication */
+}
+
+/*
+ * Function iriap_watchdog_timer_expired (data)
+ *
+ *    Query has taken too long time, so abort
+ *
+ */
+static void iriap_watchdog_timer_expired(void *data)
+{
+	struct iriap_cb *self = (struct iriap_cb *) data;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, return;);
+
+	/* iriap_close(self); */
+}
+
+#ifdef CONFIG_PROC_FS
+
+static const char *const ias_value_types[] = {
+	"IAS_MISSING",
+	"IAS_INTEGER",
+	"IAS_OCT_SEQ",
+	"IAS_STRING"
+};
+
+static inline struct ias_object *irias_seq_idx(loff_t pos)
+{
+	struct ias_object *obj;
+
+	for (obj = (struct ias_object *) hashbin_get_first(irias_objects);
+	     obj; obj = (struct ias_object *) hashbin_get_next(irias_objects)) {
+		if (pos-- == 0)
+			break;
+	}
+
+	return obj;
+}
+
+static void *irias_seq_start(struct seq_file *seq, loff_t *pos)
+{
+	spin_lock_irq(&irias_objects->hb_spinlock);
+
+	return *pos ? irias_seq_idx(*pos - 1) : SEQ_START_TOKEN;
+}
+
+static void *irias_seq_next(struct seq_file *seq, void *v, loff_t *pos)
+{
+	++*pos;
+
+	return (v == SEQ_START_TOKEN)
+		? (void *) hashbin_get_first(irias_objects)
+		: (void *) hashbin_get_next(irias_objects);
+}
+
+static void irias_seq_stop(struct seq_file *seq, void *v)
+{
+	spin_unlock_irq(&irias_objects->hb_spinlock);
+}
+
+static int irias_seq_show(struct seq_file *seq, void *v)
+{
+	if (v == SEQ_START_TOKEN)
+		seq_puts(seq, "LM-IAS Objects:\n");
+	else {
+		struct ias_object *obj = v;
+		struct ias_attrib *attrib;
+
+		IRDA_ASSERT(obj->magic == IAS_OBJECT_MAGIC, return -EINVAL;);
+
+		seq_printf(seq, "name: %s, id=%d\n",
+			   obj->name, obj->id);
+
+		/* Careful for priority inversions here !
+		 * All other uses of attrib spinlock are independent of
+		 * the object spinlock, so we are safe. Jean II */
+		spin_lock(&obj->attribs->hb_spinlock);
+
+		/* List all attributes for this object */
+		for (attrib = (struct ias_attrib *) hashbin_get_first(obj->attribs);
+		     attrib != NULL;
+		     attrib = (struct ias_attrib *) hashbin_get_next(obj->attribs)) {
+
+			IRDA_ASSERT(attrib->magic == IAS_ATTRIB_MAGIC,
+				    goto outloop; );
+
+			seq_printf(seq, " - Attribute name: \"%s\", ",
+				   attrib->name);
+			seq_printf(seq, "value[%s]: ",
+				   ias_value_types[attrib->value->type]);
+
+			switch (attrib->value->type) {
+			case IAS_INTEGER:
+				seq_printf(seq, "%d\n",
+					   attrib->value->t.integer);
+				break;
+			case IAS_STRING:
+				seq_printf(seq, "\"%s\"\n",
+					   attrib->value->t.string);
+				break;
+			case IAS_OCT_SEQ:
+				seq_printf(seq, "octet sequence (%d bytes)\n",
+					   attrib->value->len);
+				break;
+			case IAS_MISSING:
+				seq_puts(seq, "missing\n");
+				break;
+			default:
+				seq_printf(seq, "type %d?\n",
+					   attrib->value->type);
+			}
+			seq_putc(seq, '\n');
+
+		}
+	IRDA_ASSERT_LABEL(outloop:)
+		spin_unlock(&obj->attribs->hb_spinlock);
+	}
+
+	return 0;
+}
+
+static const struct seq_operations irias_seq_ops = {
+	.start  = irias_seq_start,
+	.next   = irias_seq_next,
+	.stop   = irias_seq_stop,
+	.show   = irias_seq_show,
+};
+
+static int irias_seq_open(struct inode *inode, struct file *file)
+{
+	IRDA_ASSERT( irias_objects != NULL, return -EINVAL;);
+
+	return seq_open(file, &irias_seq_ops);
+}
+
+const struct file_operations irias_seq_fops = {
+	.owner		= THIS_MODULE,
+	.open           = irias_seq_open,
+	.read           = seq_read,
+	.llseek         = seq_lseek,
+	.release	= seq_release,
+};
+
+#endif /* PROC_FS */
diff --git a/drivers/staging/irda/net/iriap_event.c b/drivers/staging/irda/net/iriap_event.c
new file mode 100644
index 000000000000..e6098b2e048a
--- /dev/null
+++ b/drivers/staging/irda/net/iriap_event.c
@@ -0,0 +1,496 @@
+/*********************************************************************
+ *
+ * Filename:      iriap_event.c
+ * Version:       0.1
+ * Description:   IAP Finite State Machine
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Thu Aug 21 00:02:07 1997
+ * Modified at:   Wed Mar  1 11:28:34 2000
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1997, 1999-2000 Dag Brattli <dagb@cs.uit.no>,
+ *     All Rights Reserved.
+ *     Copyright (c) 2000-2003 Jean Tourrilhes <jt@hpl.hp.com>
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#include <linux/slab.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/irlmp.h>
+#include <net/irda/iriap.h>
+#include <net/irda/iriap_event.h>
+
+static void state_s_disconnect   (struct iriap_cb *self, IRIAP_EVENT event,
+				  struct sk_buff *skb);
+static void state_s_connecting   (struct iriap_cb *self, IRIAP_EVENT event,
+				  struct sk_buff *skb);
+static void state_s_call         (struct iriap_cb *self, IRIAP_EVENT event,
+				  struct sk_buff *skb);
+
+static void state_s_make_call    (struct iriap_cb *self, IRIAP_EVENT event,
+				  struct sk_buff *skb);
+static void state_s_calling      (struct iriap_cb *self, IRIAP_EVENT event,
+				  struct sk_buff *skb);
+static void state_s_outstanding  (struct iriap_cb *self, IRIAP_EVENT event,
+				  struct sk_buff *skb);
+static void state_s_replying     (struct iriap_cb *self, IRIAP_EVENT event,
+				  struct sk_buff *skb);
+static void state_s_wait_for_call(struct iriap_cb *self, IRIAP_EVENT event,
+				  struct sk_buff *skb);
+static void state_s_wait_active  (struct iriap_cb *self, IRIAP_EVENT event,
+				  struct sk_buff *skb);
+
+static void state_r_disconnect   (struct iriap_cb *self, IRIAP_EVENT event,
+				  struct sk_buff *skb);
+static void state_r_call         (struct iriap_cb *self, IRIAP_EVENT event,
+				  struct sk_buff *skb);
+static void state_r_waiting      (struct iriap_cb *self, IRIAP_EVENT event,
+				  struct sk_buff *skb);
+static void state_r_wait_active  (struct iriap_cb *self, IRIAP_EVENT event,
+				  struct sk_buff *skb);
+static void state_r_receiving    (struct iriap_cb *self, IRIAP_EVENT event,
+				  struct sk_buff *skb);
+static void state_r_execute      (struct iriap_cb *self, IRIAP_EVENT event,
+				  struct sk_buff *skb);
+static void state_r_returning    (struct iriap_cb *self, IRIAP_EVENT event,
+				  struct sk_buff *skb);
+
+static void (*iriap_state[])(struct iriap_cb *self, IRIAP_EVENT event,
+			     struct sk_buff *skb) = {
+	/* Client FSM */
+	state_s_disconnect,
+	state_s_connecting,
+	state_s_call,
+
+	/* S-Call FSM */
+	state_s_make_call,
+	state_s_calling,
+	state_s_outstanding,
+	state_s_replying,
+	state_s_wait_for_call,
+	state_s_wait_active,
+
+	/* Server FSM */
+	state_r_disconnect,
+	state_r_call,
+
+	/* R-Connect FSM */
+	state_r_waiting,
+	state_r_wait_active,
+	state_r_receiving,
+	state_r_execute,
+	state_r_returning,
+};
+
+void iriap_next_client_state(struct iriap_cb *self, IRIAP_STATE state)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, return;);
+
+	self->client_state = state;
+}
+
+void iriap_next_call_state(struct iriap_cb *self, IRIAP_STATE state)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, return;);
+
+	self->call_state = state;
+}
+
+void iriap_next_server_state(struct iriap_cb *self, IRIAP_STATE state)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, return;);
+
+	self->server_state = state;
+}
+
+void iriap_next_r_connect_state(struct iriap_cb *self, IRIAP_STATE state)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, return;);
+
+	self->r_connect_state = state;
+}
+
+void iriap_do_client_event(struct iriap_cb *self, IRIAP_EVENT event,
+			   struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, return;);
+
+	(*iriap_state[ self->client_state]) (self, event, skb);
+}
+
+void iriap_do_call_event(struct iriap_cb *self, IRIAP_EVENT event,
+			 struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, return;);
+
+	(*iriap_state[ self->call_state]) (self, event, skb);
+}
+
+void iriap_do_server_event(struct iriap_cb *self, IRIAP_EVENT event,
+			   struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, return;);
+
+	(*iriap_state[ self->server_state]) (self, event, skb);
+}
+
+void iriap_do_r_connect_event(struct iriap_cb *self, IRIAP_EVENT event,
+			      struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, return;);
+
+	(*iriap_state[ self->r_connect_state]) (self, event, skb);
+}
+
+
+/*
+ * Function state_s_disconnect (event, skb)
+ *
+ *    S-Disconnect, The device has no LSAP connection to a particular
+ *    remote device.
+ */
+static void state_s_disconnect(struct iriap_cb *self, IRIAP_EVENT event,
+			       struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, return;);
+
+	switch (event) {
+	case IAP_CALL_REQUEST_GVBC:
+		iriap_next_client_state(self, S_CONNECTING);
+		IRDA_ASSERT(self->request_skb == NULL, return;);
+		/* Don't forget to refcount it -
+		 * see iriap_getvaluebyclass_request(). */
+		skb_get(skb);
+		self->request_skb = skb;
+		iriap_connect_request(self);
+		break;
+	case IAP_LM_DISCONNECT_INDICATION:
+		break;
+	default:
+		pr_debug("%s(), Unknown event %d\n", __func__, event);
+		break;
+	}
+}
+
+/*
+ * Function state_s_connecting (self, event, skb)
+ *
+ *    S-Connecting
+ *
+ */
+static void state_s_connecting(struct iriap_cb *self, IRIAP_EVENT event,
+			       struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, return;);
+
+	switch (event) {
+	case IAP_LM_CONNECT_CONFIRM:
+		/*
+		 *  Jump to S-Call FSM
+		 */
+		iriap_do_call_event(self, IAP_CALL_REQUEST, skb);
+		/* iriap_call_request(self, 0,0,0); */
+		iriap_next_client_state(self, S_CALL);
+		break;
+	case IAP_LM_DISCONNECT_INDICATION:
+		/* Abort calls */
+		iriap_next_call_state(self, S_MAKE_CALL);
+		iriap_next_client_state(self, S_DISCONNECT);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %d\n", __func__, event);
+		break;
+	}
+}
+
+/*
+ * Function state_s_call (self, event, skb)
+ *
+ *    S-Call, The device can process calls to a specific remote
+ *    device. Whenever the LSAP connection is disconnected, this state
+ *    catches that event and clears up
+ */
+static void state_s_call(struct iriap_cb *self, IRIAP_EVENT event,
+			 struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return;);
+
+	switch (event) {
+	case IAP_LM_DISCONNECT_INDICATION:
+		/* Abort calls */
+		iriap_next_call_state(self, S_MAKE_CALL);
+		iriap_next_client_state(self, S_DISCONNECT);
+		break;
+	default:
+		pr_debug("state_s_call: Unknown event %d\n", event);
+		break;
+	}
+}
+
+/*
+ * Function state_s_make_call (event, skb)
+ *
+ *    S-Make-Call
+ *
+ */
+static void state_s_make_call(struct iriap_cb *self, IRIAP_EVENT event,
+			      struct sk_buff *skb)
+{
+	struct sk_buff *tx_skb;
+
+	IRDA_ASSERT(self != NULL, return;);
+
+	switch (event) {
+	case IAP_CALL_REQUEST:
+		/* Already refcounted - see state_s_disconnect() */
+		tx_skb = self->request_skb;
+		self->request_skb = NULL;
+
+		irlmp_data_request(self->lsap, tx_skb);
+		iriap_next_call_state(self, S_OUTSTANDING);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %d\n", __func__, event);
+		break;
+	}
+}
+
+/*
+ * Function state_s_calling (event, skb)
+ *
+ *    S-Calling
+ *
+ */
+static void state_s_calling(struct iriap_cb *self, IRIAP_EVENT event,
+			    struct sk_buff *skb)
+{
+	pr_debug("%s(), Not implemented\n", __func__);
+}
+
+/*
+ * Function state_s_outstanding (event, skb)
+ *
+ *    S-Outstanding, The device is waiting for a response to a command
+ *
+ */
+static void state_s_outstanding(struct iriap_cb *self, IRIAP_EVENT event,
+				struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return;);
+
+	switch (event) {
+	case IAP_RECV_F_LST:
+		/*iriap_send_ack(self);*/
+		/*LM_Idle_request(idle); */
+
+		iriap_next_call_state(self, S_WAIT_FOR_CALL);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %d\n", __func__, event);
+		break;
+	}
+}
+
+/*
+ * Function state_s_replying (event, skb)
+ *
+ *    S-Replying, The device is collecting a multiple part response
+ */
+static void state_s_replying(struct iriap_cb *self, IRIAP_EVENT event,
+			     struct sk_buff *skb)
+{
+	pr_debug("%s(), Not implemented\n", __func__);
+}
+
+/*
+ * Function state_s_wait_for_call (event, skb)
+ *
+ *    S-Wait-for-Call
+ *
+ */
+static void state_s_wait_for_call(struct iriap_cb *self, IRIAP_EVENT event,
+				  struct sk_buff *skb)
+{
+	pr_debug("%s(), Not implemented\n", __func__);
+}
+
+
+/*
+ * Function state_s_wait_active (event, skb)
+ *
+ *    S-Wait-Active
+ *
+ */
+static void state_s_wait_active(struct iriap_cb *self, IRIAP_EVENT event,
+				struct sk_buff *skb)
+{
+	pr_debug("%s(), Not implemented\n", __func__);
+}
+
+/**************************************************************************
+ *
+ *  Server FSM
+ *
+ **************************************************************************/
+
+/*
+ * Function state_r_disconnect (self, event, skb)
+ *
+ *    LM-IAS server is disconnected (not processing any requests!)
+ *
+ */
+static void state_r_disconnect(struct iriap_cb *self, IRIAP_EVENT event,
+			       struct sk_buff *skb)
+{
+	struct sk_buff *tx_skb;
+
+	switch (event) {
+	case IAP_LM_CONNECT_INDICATION:
+		tx_skb = alloc_skb(LMP_MAX_HEADER, GFP_ATOMIC);
+		if (tx_skb == NULL)
+			return;
+
+		/* Reserve space for MUX_CONTROL and LAP header */
+		skb_reserve(tx_skb, LMP_MAX_HEADER);
+
+		irlmp_connect_response(self->lsap, tx_skb);
+		/*LM_Idle_request(idle); */
+
+		iriap_next_server_state(self, R_CALL);
+
+		/*
+		 *  Jump to R-Connect FSM, we skip R-Waiting since we do not
+		 *  care about LM_Idle_request()!
+		 */
+		iriap_next_r_connect_state(self, R_RECEIVING);
+		break;
+	default:
+		pr_debug("%s(), unknown event %d\n", __func__, event);
+		break;
+	}
+}
+
+/*
+ * Function state_r_call (self, event, skb)
+ */
+static void state_r_call(struct iriap_cb *self, IRIAP_EVENT event,
+			 struct sk_buff *skb)
+{
+	switch (event) {
+	case IAP_LM_DISCONNECT_INDICATION:
+		/* Abort call */
+		iriap_next_server_state(self, R_DISCONNECT);
+		iriap_next_r_connect_state(self, R_WAITING);
+		break;
+	default:
+		pr_debug("%s(), unknown event!\n", __func__);
+		break;
+	}
+}
+
+/*
+ *  R-Connect FSM
+ */
+
+/*
+ * Function state_r_waiting (self, event, skb)
+ */
+static void state_r_waiting(struct iriap_cb *self, IRIAP_EVENT event,
+			    struct sk_buff *skb)
+{
+	pr_debug("%s(), Not implemented\n", __func__);
+}
+
+static void state_r_wait_active(struct iriap_cb *self, IRIAP_EVENT event,
+				struct sk_buff *skb)
+{
+	pr_debug("%s(), Not implemented\n", __func__);
+}
+
+/*
+ * Function state_r_receiving (self, event, skb)
+ *
+ *    We are receiving a command
+ *
+ */
+static void state_r_receiving(struct iriap_cb *self, IRIAP_EVENT event,
+			      struct sk_buff *skb)
+{
+	switch (event) {
+	case IAP_RECV_F_LST:
+		iriap_next_r_connect_state(self, R_EXECUTE);
+
+		iriap_call_indication(self, skb);
+		break;
+	default:
+		pr_debug("%s(), unknown event!\n", __func__);
+		break;
+	}
+}
+
+/*
+ * Function state_r_execute (self, event, skb)
+ *
+ *    The server is processing the request
+ *
+ */
+static void state_r_execute(struct iriap_cb *self, IRIAP_EVENT event,
+			    struct sk_buff *skb)
+{
+	IRDA_ASSERT(skb != NULL, return;);
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IAS_MAGIC, return;);
+
+	switch (event) {
+	case IAP_CALL_RESPONSE:
+		/*
+		 *  Since we don't implement the Waiting state, we return
+		 *  to state Receiving instead, DB.
+		 */
+		iriap_next_r_connect_state(self, R_RECEIVING);
+
+		/* Don't forget to refcount it - see
+		 * iriap_getvaluebyclass_response(). */
+		skb_get(skb);
+
+		irlmp_data_request(self->lsap, skb);
+		break;
+	default:
+		pr_debug("%s(), unknown event!\n", __func__);
+		break;
+	}
+}
+
+static void state_r_returning(struct iriap_cb *self, IRIAP_EVENT event,
+			      struct sk_buff *skb)
+{
+	pr_debug("%s(), event=%d\n", __func__, event);
+
+	switch (event) {
+	case IAP_RECV_F_LST:
+		break;
+	default:
+		break;
+	}
+}
diff --git a/drivers/staging/irda/net/irias_object.c b/drivers/staging/irda/net/irias_object.c
new file mode 100644
index 000000000000..53b86d0e1630
--- /dev/null
+++ b/drivers/staging/irda/net/irias_object.c
@@ -0,0 +1,555 @@
+/*********************************************************************
+ *
+ * Filename:      irias_object.c
+ * Version:       0.3
+ * Description:   IAS object database and functions
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Thu Oct  1 22:50:04 1998
+ * Modified at:   Wed Dec 15 11:23:16 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1998-1999 Dag Brattli, All Rights Reserved.
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/socket.h>
+#include <linux/module.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/irias_object.h>
+
+hashbin_t *irias_objects;
+
+/*
+ *  Used when a missing value needs to be returned
+ */
+struct ias_value irias_missing = { IAS_MISSING, 0, 0, 0, {0}};
+
+
+/*
+ * Function ias_new_object (name, id)
+ *
+ *    Create a new IAS object
+ *
+ */
+struct ias_object *irias_new_object( char *name, int id)
+{
+	struct ias_object *obj;
+
+	obj = kzalloc(sizeof(struct ias_object), GFP_ATOMIC);
+	if (obj == NULL) {
+		net_warn_ratelimited("%s(), Unable to allocate object!\n",
+				     __func__);
+		return NULL;
+	}
+
+	obj->magic = IAS_OBJECT_MAGIC;
+	obj->name = kstrndup(name, IAS_MAX_CLASSNAME, GFP_ATOMIC);
+	if (!obj->name) {
+		net_warn_ratelimited("%s(), Unable to allocate name!\n",
+				     __func__);
+		kfree(obj);
+		return NULL;
+	}
+	obj->id = id;
+
+	/* Locking notes : the attrib spinlock has lower precendence
+	 * than the objects spinlock. Never grap the objects spinlock
+	 * while holding any attrib spinlock (risk of deadlock). Jean II */
+	obj->attribs = hashbin_new(HB_LOCK);
+
+	if (obj->attribs == NULL) {
+		net_warn_ratelimited("%s(), Unable to allocate attribs!\n",
+				     __func__);
+		kfree(obj->name);
+		kfree(obj);
+		return NULL;
+	}
+
+	return obj;
+}
+EXPORT_SYMBOL(irias_new_object);
+
+/*
+ * Function irias_delete_attrib (attrib)
+ *
+ *    Delete given attribute and deallocate all its memory
+ *
+ */
+static void __irias_delete_attrib(struct ias_attrib *attrib)
+{
+	IRDA_ASSERT(attrib != NULL, return;);
+	IRDA_ASSERT(attrib->magic == IAS_ATTRIB_MAGIC, return;);
+
+	kfree(attrib->name);
+
+	irias_delete_value(attrib->value);
+	attrib->magic = ~IAS_ATTRIB_MAGIC;
+
+	kfree(attrib);
+}
+
+void __irias_delete_object(struct ias_object *obj)
+{
+	IRDA_ASSERT(obj != NULL, return;);
+	IRDA_ASSERT(obj->magic == IAS_OBJECT_MAGIC, return;);
+
+	kfree(obj->name);
+
+	hashbin_delete(obj->attribs, (FREE_FUNC) __irias_delete_attrib);
+
+	obj->magic = ~IAS_OBJECT_MAGIC;
+
+	kfree(obj);
+}
+
+/*
+ * Function irias_delete_object (obj)
+ *
+ *    Remove object from hashbin and deallocate all attributes associated with
+ *    with this object and the object itself
+ *
+ */
+int irias_delete_object(struct ias_object *obj)
+{
+	struct ias_object *node;
+
+	IRDA_ASSERT(obj != NULL, return -1;);
+	IRDA_ASSERT(obj->magic == IAS_OBJECT_MAGIC, return -1;);
+
+	/* Remove from list */
+	node = hashbin_remove_this(irias_objects, (irda_queue_t *) obj);
+	if (!node)
+		pr_debug("%s(), object already removed!\n",
+			 __func__);
+
+	/* Destroy */
+	__irias_delete_object(obj);
+
+	return 0;
+}
+EXPORT_SYMBOL(irias_delete_object);
+
+/*
+ * Function irias_delete_attrib (obj)
+ *
+ *    Remove attribute from hashbin and, if it was the last attribute of
+ *    the object, remove the object as well.
+ *
+ */
+int irias_delete_attrib(struct ias_object *obj, struct ias_attrib *attrib,
+			int cleanobject)
+{
+	struct ias_attrib *node;
+
+	IRDA_ASSERT(obj != NULL, return -1;);
+	IRDA_ASSERT(obj->magic == IAS_OBJECT_MAGIC, return -1;);
+	IRDA_ASSERT(attrib != NULL, return -1;);
+
+	/* Remove attribute from object */
+	node = hashbin_remove_this(obj->attribs, (irda_queue_t *) attrib);
+	if (!node)
+		return 0; /* Already removed or non-existent */
+
+	/* Deallocate attribute */
+	__irias_delete_attrib(node);
+
+	/* Check if object has still some attributes, destroy it if none.
+	 * At first glance, this look dangerous, as the kernel reference
+	 * various IAS objects. However, we only use this function on
+	 * user attributes, not kernel attributes, so there is no risk
+	 * of deleting a kernel object this way. Jean II */
+	node = (struct ias_attrib *) hashbin_get_first(obj->attribs);
+	if (cleanobject && !node)
+		irias_delete_object(obj);
+
+	return 0;
+}
+
+/*
+ * Function irias_insert_object (obj)
+ *
+ *    Insert an object into the LM-IAS database
+ *
+ */
+void irias_insert_object(struct ias_object *obj)
+{
+	IRDA_ASSERT(obj != NULL, return;);
+	IRDA_ASSERT(obj->magic == IAS_OBJECT_MAGIC, return;);
+
+	hashbin_insert(irias_objects, (irda_queue_t *) obj, 0, obj->name);
+}
+EXPORT_SYMBOL(irias_insert_object);
+
+/*
+ * Function irias_find_object (name)
+ *
+ *    Find object with given name
+ *
+ */
+struct ias_object *irias_find_object(char *name)
+{
+	IRDA_ASSERT(name != NULL, return NULL;);
+
+	/* Unsafe (locking), object might change */
+	return hashbin_lock_find(irias_objects, 0, name);
+}
+EXPORT_SYMBOL(irias_find_object);
+
+/*
+ * Function irias_find_attrib (obj, name)
+ *
+ *    Find named attribute in object
+ *
+ */
+struct ias_attrib *irias_find_attrib(struct ias_object *obj, char *name)
+{
+	struct ias_attrib *attrib;
+
+	IRDA_ASSERT(obj != NULL, return NULL;);
+	IRDA_ASSERT(obj->magic == IAS_OBJECT_MAGIC, return NULL;);
+	IRDA_ASSERT(name != NULL, return NULL;);
+
+	attrib = hashbin_lock_find(obj->attribs, 0, name);
+	if (attrib == NULL)
+		return NULL;
+
+	/* Unsafe (locking), attrib might change */
+	return attrib;
+}
+
+/*
+ * Function irias_add_attribute (obj, attrib)
+ *
+ *    Add attribute to object
+ *
+ */
+static void irias_add_attrib(struct ias_object *obj, struct ias_attrib *attrib,
+			     int owner)
+{
+	IRDA_ASSERT(obj != NULL, return;);
+	IRDA_ASSERT(obj->magic == IAS_OBJECT_MAGIC, return;);
+
+	IRDA_ASSERT(attrib != NULL, return;);
+	IRDA_ASSERT(attrib->magic == IAS_ATTRIB_MAGIC, return;);
+
+	/* Set if attrib is owned by kernel or user space */
+	attrib->value->owner = owner;
+
+	hashbin_insert(obj->attribs, (irda_queue_t *) attrib, 0, attrib->name);
+}
+
+/*
+ * Function irias_object_change_attribute (obj_name, attrib_name, new_value)
+ *
+ *    Change the value of an objects attribute.
+ *
+ */
+int irias_object_change_attribute(char *obj_name, char *attrib_name,
+				  struct ias_value *new_value)
+{
+	struct ias_object *obj;
+	struct ias_attrib *attrib;
+	unsigned long flags;
+
+	/* Find object */
+	obj = hashbin_lock_find(irias_objects, 0, obj_name);
+	if (obj == NULL) {
+		net_warn_ratelimited("%s: Unable to find object: %s\n",
+				     __func__, obj_name);
+		return -1;
+	}
+
+	/* Slightly unsafe (obj might get removed under us) */
+	spin_lock_irqsave(&obj->attribs->hb_spinlock, flags);
+
+	/* Find attribute */
+	attrib = hashbin_find(obj->attribs, 0, attrib_name);
+	if (attrib == NULL) {
+		net_warn_ratelimited("%s: Unable to find attribute: %s\n",
+				     __func__, attrib_name);
+		spin_unlock_irqrestore(&obj->attribs->hb_spinlock, flags);
+		return -1;
+	}
+
+	if ( attrib->value->type != new_value->type) {
+		pr_debug("%s(), changing value type not allowed!\n",
+			 __func__);
+		spin_unlock_irqrestore(&obj->attribs->hb_spinlock, flags);
+		return -1;
+	}
+
+	/* Delete old value */
+	irias_delete_value(attrib->value);
+
+	/* Insert new value */
+	attrib->value = new_value;
+
+	/* Success */
+	spin_unlock_irqrestore(&obj->attribs->hb_spinlock, flags);
+	return 0;
+}
+EXPORT_SYMBOL(irias_object_change_attribute);
+
+/*
+ * Function irias_object_add_integer_attrib (obj, name, value)
+ *
+ *    Add an integer attribute to an LM-IAS object
+ *
+ */
+void irias_add_integer_attrib(struct ias_object *obj, char *name, int value,
+			      int owner)
+{
+	struct ias_attrib *attrib;
+
+	IRDA_ASSERT(obj != NULL, return;);
+	IRDA_ASSERT(obj->magic == IAS_OBJECT_MAGIC, return;);
+	IRDA_ASSERT(name != NULL, return;);
+
+	attrib = kzalloc(sizeof(struct ias_attrib), GFP_ATOMIC);
+	if (attrib == NULL) {
+		net_warn_ratelimited("%s: Unable to allocate attribute!\n",
+				     __func__);
+		return;
+	}
+
+	attrib->magic = IAS_ATTRIB_MAGIC;
+	attrib->name = kstrndup(name, IAS_MAX_ATTRIBNAME, GFP_ATOMIC);
+
+	/* Insert value */
+	attrib->value = irias_new_integer_value(value);
+	if (!attrib->name || !attrib->value) {
+		net_warn_ratelimited("%s: Unable to allocate attribute!\n",
+				     __func__);
+		if (attrib->value)
+			irias_delete_value(attrib->value);
+		kfree(attrib->name);
+		kfree(attrib);
+		return;
+	}
+
+	irias_add_attrib(obj, attrib, owner);
+}
+EXPORT_SYMBOL(irias_add_integer_attrib);
+
+ /*
+ * Function irias_add_octseq_attrib (obj, name, octet_seq, len)
+ *
+ *    Add a octet sequence attribute to an LM-IAS object
+ *
+ */
+
+void irias_add_octseq_attrib(struct ias_object *obj, char *name, __u8 *octets,
+			     int len, int owner)
+{
+	struct ias_attrib *attrib;
+
+	IRDA_ASSERT(obj != NULL, return;);
+	IRDA_ASSERT(obj->magic == IAS_OBJECT_MAGIC, return;);
+
+	IRDA_ASSERT(name != NULL, return;);
+	IRDA_ASSERT(octets != NULL, return;);
+
+	attrib = kzalloc(sizeof(struct ias_attrib), GFP_ATOMIC);
+	if (attrib == NULL) {
+		net_warn_ratelimited("%s: Unable to allocate attribute!\n",
+				     __func__);
+		return;
+	}
+
+	attrib->magic = IAS_ATTRIB_MAGIC;
+	attrib->name = kstrndup(name, IAS_MAX_ATTRIBNAME, GFP_ATOMIC);
+
+	attrib->value = irias_new_octseq_value( octets, len);
+	if (!attrib->name || !attrib->value) {
+		net_warn_ratelimited("%s: Unable to allocate attribute!\n",
+				     __func__);
+		if (attrib->value)
+			irias_delete_value(attrib->value);
+		kfree(attrib->name);
+		kfree(attrib);
+		return;
+	}
+
+	irias_add_attrib(obj, attrib, owner);
+}
+EXPORT_SYMBOL(irias_add_octseq_attrib);
+
+/*
+ * Function irias_object_add_string_attrib (obj, string)
+ *
+ *    Add a string attribute to an LM-IAS object
+ *
+ */
+void irias_add_string_attrib(struct ias_object *obj, char *name, char *value,
+			     int owner)
+{
+	struct ias_attrib *attrib;
+
+	IRDA_ASSERT(obj != NULL, return;);
+	IRDA_ASSERT(obj->magic == IAS_OBJECT_MAGIC, return;);
+
+	IRDA_ASSERT(name != NULL, return;);
+	IRDA_ASSERT(value != NULL, return;);
+
+	attrib = kzalloc(sizeof( struct ias_attrib), GFP_ATOMIC);
+	if (attrib == NULL) {
+		net_warn_ratelimited("%s: Unable to allocate attribute!\n",
+				     __func__);
+		return;
+	}
+
+	attrib->magic = IAS_ATTRIB_MAGIC;
+	attrib->name = kstrndup(name, IAS_MAX_ATTRIBNAME, GFP_ATOMIC);
+
+	attrib->value = irias_new_string_value(value);
+	if (!attrib->name || !attrib->value) {
+		net_warn_ratelimited("%s: Unable to allocate attribute!\n",
+				     __func__);
+		if (attrib->value)
+			irias_delete_value(attrib->value);
+		kfree(attrib->name);
+		kfree(attrib);
+		return;
+	}
+
+	irias_add_attrib(obj, attrib, owner);
+}
+EXPORT_SYMBOL(irias_add_string_attrib);
+
+/*
+ * Function irias_new_integer_value (integer)
+ *
+ *    Create new IAS integer value
+ *
+ */
+struct ias_value *irias_new_integer_value(int integer)
+{
+	struct ias_value *value;
+
+	value = kzalloc(sizeof(struct ias_value), GFP_ATOMIC);
+	if (value == NULL)
+		return NULL;
+
+	value->type = IAS_INTEGER;
+	value->len = 4;
+	value->t.integer = integer;
+
+	return value;
+}
+EXPORT_SYMBOL(irias_new_integer_value);
+
+/*
+ * Function irias_new_string_value (string)
+ *
+ *    Create new IAS string value
+ *
+ * Per IrLMP 1.1, 4.3.3.2, strings are up to 256 chars - Jean II
+ */
+struct ias_value *irias_new_string_value(char *string)
+{
+	struct ias_value *value;
+
+	value = kzalloc(sizeof(struct ias_value), GFP_ATOMIC);
+	if (value == NULL)
+		return NULL;
+
+	value->type = IAS_STRING;
+	value->charset = CS_ASCII;
+	value->t.string = kstrndup(string, IAS_MAX_STRING, GFP_ATOMIC);
+	if (!value->t.string) {
+		net_warn_ratelimited("%s: Unable to kmalloc!\n", __func__);
+		kfree(value);
+		return NULL;
+	}
+
+	value->len = strlen(value->t.string);
+
+	return value;
+}
+
+/*
+ * Function irias_new_octseq_value (octets, len)
+ *
+ *    Create new IAS octet-sequence value
+ *
+ * Per IrLMP 1.1, 4.3.3.2, octet-sequence are up to 1024 bytes - Jean II
+ */
+struct ias_value *irias_new_octseq_value(__u8 *octseq , int len)
+{
+	struct ias_value *value;
+
+	value = kzalloc(sizeof(struct ias_value), GFP_ATOMIC);
+	if (value == NULL)
+		return NULL;
+
+	value->type = IAS_OCT_SEQ;
+	/* Check length */
+	if(len > IAS_MAX_OCTET_STRING)
+		len = IAS_MAX_OCTET_STRING;
+	value->len = len;
+
+	value->t.oct_seq = kmemdup(octseq, len, GFP_ATOMIC);
+	if (value->t.oct_seq == NULL){
+		net_warn_ratelimited("%s: Unable to kmalloc!\n", __func__);
+		kfree(value);
+		return NULL;
+	}
+	return value;
+}
+
+struct ias_value *irias_new_missing_value(void)
+{
+	struct ias_value *value;
+
+	value = kzalloc(sizeof(struct ias_value), GFP_ATOMIC);
+	if (value == NULL)
+		return NULL;
+
+	value->type = IAS_MISSING;
+
+	return value;
+}
+
+/*
+ * Function irias_delete_value (value)
+ *
+ *    Delete IAS value
+ *
+ */
+void irias_delete_value(struct ias_value *value)
+{
+	IRDA_ASSERT(value != NULL, return;);
+
+	switch (value->type) {
+	case IAS_INTEGER: /* Fallthrough */
+	case IAS_MISSING:
+		/* No need to deallocate */
+		break;
+	case IAS_STRING:
+		/* Deallocate string */
+		kfree(value->t.string);
+		break;
+	case IAS_OCT_SEQ:
+		/* Deallocate byte stream */
+		 kfree(value->t.oct_seq);
+		 break;
+	default:
+		pr_debug("%s(), Unknown value type!\n", __func__);
+		break;
+	}
+	kfree(value);
+}
+EXPORT_SYMBOL(irias_delete_value);
diff --git a/drivers/staging/irda/net/irlan/Kconfig b/drivers/staging/irda/net/irlan/Kconfig
new file mode 100644
index 000000000000..951abc2e3a7f
--- /dev/null
+++ b/drivers/staging/irda/net/irlan/Kconfig
@@ -0,0 +1,14 @@
+config IRLAN
+	tristate "IrLAN protocol"
+	depends on IRDA
+	help
+	  Say Y here if you want to build support for the IrLAN protocol.
+	  To compile it as a module, choose M here: the module will be called
+	  irlan.  IrLAN emulates an Ethernet and makes it possible to put up
+	  a wireless LAN using infrared beams.
+
+	  The IrLAN protocol can be used to talk with infrared access points
+	  like the HP NetbeamIR, or the ESI JetEye NET.  You can also connect
+	  to another Linux machine running the IrLAN protocol for ad-hoc
+	  networking!
+
diff --git a/drivers/staging/irda/net/irlan/Makefile b/drivers/staging/irda/net/irlan/Makefile
new file mode 100644
index 000000000000..94eefbc8e6b9
--- /dev/null
+++ b/drivers/staging/irda/net/irlan/Makefile
@@ -0,0 +1,7 @@
+#
+# Makefile for the Linux IrDA IrLAN protocol layer.
+#
+
+obj-$(CONFIG_IRLAN) += irlan.o
+
+irlan-y := irlan_common.o irlan_eth.o irlan_event.o irlan_client.o irlan_provider.o irlan_filter.o irlan_provider_event.o irlan_client_event.o
diff --git a/drivers/staging/irda/net/irlan/irlan_client.c b/drivers/staging/irda/net/irlan/irlan_client.c
new file mode 100644
index 000000000000..c5837a40c78e
--- /dev/null
+++ b/drivers/staging/irda/net/irlan/irlan_client.c
@@ -0,0 +1,559 @@
+/*********************************************************************
+ *
+ * Filename:      irlan_client.c
+ * Version:       0.9
+ * Description:   IrDA LAN Access Protocol (IrLAN) Client
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sun Aug 31 20:14:37 1997
+ * Modified at:   Tue Dec 14 15:47:02 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * Sources:       skeleton.c by Donald Becker <becker@CESDIS.gsfc.nasa.gov>
+ *                slip.c by Laurence Culhane, <loz@holmes.demon.co.uk>
+ *                          Fred N. van Kempen, <waltje@uwalt.nl.mugnet.org>
+ *
+ *     Copyright (c) 1998-1999 Dag Brattli <dagb@cs.uit.no>,
+ *     All Rights Reserved.
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#include <linux/kernel.h>
+#include <linux/string.h>
+#include <linux/slab.h>
+#include <linux/errno.h>
+#include <linux/init.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/if_arp.h>
+#include <linux/bitops.h>
+#include <net/arp.h>
+
+#include <asm/byteorder.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/irttp.h>
+#include <net/irda/irlmp.h>
+#include <net/irda/irias_object.h>
+#include <net/irda/iriap.h>
+#include <net/irda/timer.h>
+
+#include <net/irda/irlan_common.h>
+#include <net/irda/irlan_event.h>
+#include <net/irda/irlan_eth.h>
+#include <net/irda/irlan_provider.h>
+#include <net/irda/irlan_client.h>
+
+#undef CONFIG_IRLAN_GRATUITOUS_ARP
+
+static void irlan_client_ctrl_disconnect_indication(void *instance, void *sap,
+						    LM_REASON reason,
+						    struct sk_buff *);
+static int irlan_client_ctrl_data_indication(void *instance, void *sap,
+					     struct sk_buff *skb);
+static void irlan_client_ctrl_connect_confirm(void *instance, void *sap,
+					      struct qos_info *qos,
+					      __u32 max_sdu_size,
+					      __u8 max_header_size,
+					      struct sk_buff *);
+static void irlan_check_response_param(struct irlan_cb *self, char *param,
+				       char *value, int val_len);
+static void irlan_client_open_ctrl_tsap(struct irlan_cb *self);
+
+static void irlan_client_kick_timer_expired(void *data)
+{
+	struct irlan_cb *self = (struct irlan_cb *) data;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	/*
+	 * If we are in peer mode, the client may not have got the discovery
+	 * indication it needs to make progress. If the client is still in
+	 * IDLE state, we must kick it to, but only if the provider is not IDLE
+	 */
+	if ((self->provider.access_type == ACCESS_PEER) &&
+	    (self->client.state == IRLAN_IDLE) &&
+	    (self->provider.state != IRLAN_IDLE)) {
+		irlan_client_wakeup(self, self->saddr, self->daddr);
+	}
+}
+
+static void irlan_client_start_kick_timer(struct irlan_cb *self, int timeout)
+{
+	irda_start_timer(&self->client.kick_timer, timeout, (void *) self,
+			 irlan_client_kick_timer_expired);
+}
+
+/*
+ * Function irlan_client_wakeup (self, saddr, daddr)
+ *
+ *    Wake up client
+ *
+ */
+void irlan_client_wakeup(struct irlan_cb *self, __u32 saddr, __u32 daddr)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	/*
+	 * Check if we are already awake, or if we are a provider in direct
+	 * mode (in that case we must leave the client idle
+	 */
+	if ((self->client.state != IRLAN_IDLE) ||
+	    (self->provider.access_type == ACCESS_DIRECT))
+	{
+		pr_debug("%s(), already awake!\n", __func__);
+			return;
+	}
+
+	/* Addresses may have changed! */
+	self->saddr = saddr;
+	self->daddr = daddr;
+
+	if (self->disconnect_reason == LM_USER_REQUEST) {
+		pr_debug("%s(), still stopped by user\n", __func__);
+			return;
+	}
+
+	/* Open TSAPs */
+	irlan_client_open_ctrl_tsap(self);
+	irlan_open_data_tsap(self);
+
+	irlan_do_client_event(self, IRLAN_DISCOVERY_INDICATION, NULL);
+
+	/* Start kick timer */
+	irlan_client_start_kick_timer(self, 2*HZ);
+}
+
+/*
+ * Function irlan_discovery_indication (daddr)
+ *
+ *    Remote device with IrLAN server support discovered
+ *
+ */
+void irlan_client_discovery_indication(discinfo_t *discovery,
+				       DISCOVERY_MODE mode,
+				       void *priv)
+{
+	struct irlan_cb *self;
+	__u32 saddr, daddr;
+
+	IRDA_ASSERT(discovery != NULL, return;);
+
+	/*
+	 * I didn't check it, but I bet that IrLAN suffer from the same
+	 * deficiency as IrComm and doesn't handle two instances
+	 * simultaneously connecting to each other.
+	 * Same workaround, drop passive discoveries.
+	 * Jean II */
+	if(mode == DISCOVERY_PASSIVE)
+		return;
+
+	saddr = discovery->saddr;
+	daddr = discovery->daddr;
+
+	/* Find instance */
+	rcu_read_lock();
+	self = irlan_get_any();
+	if (self) {
+		IRDA_ASSERT(self->magic == IRLAN_MAGIC, goto out;);
+
+		pr_debug("%s(), Found instance (%08x)!\n", __func__ ,
+			 daddr);
+
+		irlan_client_wakeup(self, saddr, daddr);
+	}
+IRDA_ASSERT_LABEL(out:)
+	rcu_read_unlock();
+}
+
+/*
+ * Function irlan_client_data_indication (handle, skb)
+ *
+ *    This function gets the data that is received on the control channel
+ *
+ */
+static int irlan_client_ctrl_data_indication(void *instance, void *sap,
+					     struct sk_buff *skb)
+{
+	struct irlan_cb *self;
+
+	self = instance;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return -1;);
+	IRDA_ASSERT(skb != NULL, return -1;);
+
+	irlan_do_client_event(self, IRLAN_DATA_INDICATION, skb);
+
+	/* Ready for a new command */
+	pr_debug("%s(), clearing tx_busy\n", __func__);
+	self->client.tx_busy = FALSE;
+
+	/* Check if we have some queued commands waiting to be sent */
+	irlan_run_ctrl_tx_queue(self);
+
+	return 0;
+}
+
+static void irlan_client_ctrl_disconnect_indication(void *instance, void *sap,
+						    LM_REASON reason,
+						    struct sk_buff *userdata)
+{
+	struct irlan_cb *self;
+	struct tsap_cb *tsap;
+	struct sk_buff *skb;
+
+	pr_debug("%s(), reason=%d\n", __func__ , reason);
+
+	self = instance;
+	tsap = sap;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+	IRDA_ASSERT(tsap != NULL, return;);
+	IRDA_ASSERT(tsap->magic == TTP_TSAP_MAGIC, return;);
+
+	IRDA_ASSERT(tsap == self->client.tsap_ctrl, return;);
+
+	/* Remove frames queued on the control channel */
+	while ((skb = skb_dequeue(&self->client.txq)) != NULL) {
+		dev_kfree_skb(skb);
+	}
+	self->client.tx_busy = FALSE;
+
+	irlan_do_client_event(self, IRLAN_LMP_DISCONNECT, NULL);
+}
+
+/*
+ * Function irlan_client_open_tsaps (self)
+ *
+ *    Initialize callbacks and open IrTTP TSAPs
+ *
+ */
+static void irlan_client_open_ctrl_tsap(struct irlan_cb *self)
+{
+	struct tsap_cb *tsap;
+	notify_t notify;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	/* Check if already open */
+	if (self->client.tsap_ctrl)
+		return;
+
+	irda_notify_init(&notify);
+
+	/* Set up callbacks */
+	notify.data_indication       = irlan_client_ctrl_data_indication;
+	notify.connect_confirm       = irlan_client_ctrl_connect_confirm;
+	notify.disconnect_indication = irlan_client_ctrl_disconnect_indication;
+	notify.instance = self;
+	strlcpy(notify.name, "IrLAN ctrl (c)", sizeof(notify.name));
+
+	tsap = irttp_open_tsap(LSAP_ANY, DEFAULT_INITIAL_CREDIT, &notify);
+	if (!tsap) {
+		pr_debug("%s(), Got no tsap!\n", __func__);
+		return;
+	}
+	self->client.tsap_ctrl = tsap;
+}
+
+/*
+ * Function irlan_client_connect_confirm (handle, skb)
+ *
+ *    Connection to peer IrLAN laye confirmed
+ *
+ */
+static void irlan_client_ctrl_connect_confirm(void *instance, void *sap,
+					      struct qos_info *qos,
+					      __u32 max_sdu_size,
+					      __u8 max_header_size,
+					      struct sk_buff *skb)
+{
+	struct irlan_cb *self;
+
+	self = instance;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	self->client.max_sdu_size = max_sdu_size;
+	self->client.max_header_size = max_header_size;
+
+	/* TODO: we could set the MTU depending on the max_sdu_size */
+
+	irlan_do_client_event(self, IRLAN_CONNECT_COMPLETE, NULL);
+}
+
+/*
+ * Function print_ret_code (code)
+ *
+ *    Print return code of request to peer IrLAN layer.
+ *
+ */
+static void print_ret_code(__u8 code)
+{
+	switch(code) {
+	case 0:
+		printk(KERN_INFO "Success\n");
+		break;
+	case 1:
+		net_warn_ratelimited("IrLAN: Insufficient resources\n");
+		break;
+	case 2:
+		net_warn_ratelimited("IrLAN: Invalid command format\n");
+		break;
+	case 3:
+		net_warn_ratelimited("IrLAN: Command not supported\n");
+		break;
+	case 4:
+		net_warn_ratelimited("IrLAN: Parameter not supported\n");
+		break;
+	case 5:
+		net_warn_ratelimited("IrLAN: Value not supported\n");
+		break;
+	case 6:
+		net_warn_ratelimited("IrLAN: Not open\n");
+		break;
+	case 7:
+		net_warn_ratelimited("IrLAN: Authentication required\n");
+		break;
+	case 8:
+		net_warn_ratelimited("IrLAN: Invalid password\n");
+		break;
+	case 9:
+		net_warn_ratelimited("IrLAN: Protocol error\n");
+		break;
+	case 255:
+		net_warn_ratelimited("IrLAN: Asynchronous status\n");
+		break;
+	}
+}
+
+/*
+ * Function irlan_client_parse_response (self, skb)
+ *
+ *    Extract all parameters from received buffer, then feed them to
+ *    check_params for parsing
+ */
+void irlan_client_parse_response(struct irlan_cb *self, struct sk_buff *skb)
+{
+	__u8 *frame;
+	__u8 *ptr;
+	int count;
+	int ret;
+	__u16 val_len;
+	int i;
+	char *name;
+	char *value;
+
+	IRDA_ASSERT(skb != NULL, return;);
+
+	pr_debug("%s() skb->len=%d\n", __func__ , (int)skb->len);
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	if (!skb) {
+		net_err_ratelimited("%s(), Got NULL skb!\n", __func__);
+		return;
+	}
+	frame = skb->data;
+
+	/*
+	 *  Check return code and print it if not success
+	 */
+	if (frame[0]) {
+		print_ret_code(frame[0]);
+		return;
+	}
+
+	name = kmalloc(255, GFP_ATOMIC);
+	if (!name)
+		return;
+	value = kmalloc(1016, GFP_ATOMIC);
+	if (!value) {
+		kfree(name);
+		return;
+	}
+
+	/* How many parameters? */
+	count = frame[1];
+
+	pr_debug("%s(), got %d parameters\n", __func__ , count);
+
+	ptr = frame+2;
+
+	/* For all parameters */
+	for (i=0; i<count;i++) {
+		ret = irlan_extract_param(ptr, name, value, &val_len);
+		if (ret < 0) {
+			pr_debug("%s(), IrLAN, Error!\n", __func__);
+			break;
+		}
+		ptr += ret;
+		irlan_check_response_param(self, name, value, val_len);
+	}
+	/* Cleanup */
+	kfree(name);
+	kfree(value);
+}
+
+/*
+ * Function irlan_check_response_param (self, param, value, val_len)
+ *
+ *     Check which parameter is received and update local variables
+ *
+ */
+static void irlan_check_response_param(struct irlan_cb *self, char *param,
+				       char *value, int val_len)
+{
+	__u16 tmp_cpu; /* Temporary value in host order */
+	__u8 *bytes;
+	int i;
+
+	pr_debug("%s(), parm=%s\n", __func__ , param);
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	/* Media type */
+	if (strcmp(param, "MEDIA") == 0) {
+		if (strcmp(value, "802.3") == 0)
+			self->media = MEDIA_802_3;
+		else
+			self->media = MEDIA_802_5;
+		return;
+	}
+	if (strcmp(param, "FILTER_TYPE") == 0) {
+		if (strcmp(value, "DIRECTED") == 0)
+			self->client.filter_type |= IRLAN_DIRECTED;
+		else if (strcmp(value, "FUNCTIONAL") == 0)
+			self->client.filter_type |= IRLAN_FUNCTIONAL;
+		else if (strcmp(value, "GROUP") == 0)
+			self->client.filter_type |= IRLAN_GROUP;
+		else if (strcmp(value, "MAC_FRAME") == 0)
+			self->client.filter_type |= IRLAN_MAC_FRAME;
+		else if (strcmp(value, "MULTICAST") == 0)
+			self->client.filter_type |= IRLAN_MULTICAST;
+		else if (strcmp(value, "BROADCAST") == 0)
+			self->client.filter_type |= IRLAN_BROADCAST;
+		else if (strcmp(value, "IPX_SOCKET") == 0)
+			self->client.filter_type |= IRLAN_IPX_SOCKET;
+
+	}
+	if (strcmp(param, "ACCESS_TYPE") == 0) {
+		if (strcmp(value, "DIRECT") == 0)
+			self->client.access_type = ACCESS_DIRECT;
+		else if (strcmp(value, "PEER") == 0)
+			self->client.access_type = ACCESS_PEER;
+		else if (strcmp(value, "HOSTED") == 0)
+			self->client.access_type = ACCESS_HOSTED;
+		else {
+			pr_debug("%s(), unknown access type!\n", __func__);
+		}
+	}
+	/* IRLAN version */
+	if (strcmp(param, "IRLAN_VER") == 0) {
+		pr_debug("IrLAN version %d.%d\n", (__u8)value[0],
+			 (__u8)value[1]);
+
+		self->version[0] = value[0];
+		self->version[1] = value[1];
+		return;
+	}
+	/* Which remote TSAP to use for data channel */
+	if (strcmp(param, "DATA_CHAN") == 0) {
+		self->dtsap_sel_data = value[0];
+		pr_debug("Data TSAP = %02x\n", self->dtsap_sel_data);
+		return;
+	}
+	if (strcmp(param, "CON_ARB") == 0) {
+		memcpy(&tmp_cpu, value, 2); /* Align value */
+		le16_to_cpus(&tmp_cpu);     /* Convert to host order */
+		self->client.recv_arb_val = tmp_cpu;
+		pr_debug("%s(), receive arb val=%d\n", __func__ ,
+			 self->client.recv_arb_val);
+	}
+	if (strcmp(param, "MAX_FRAME") == 0) {
+		memcpy(&tmp_cpu, value, 2); /* Align value */
+		le16_to_cpus(&tmp_cpu);     /* Convert to host order */
+		self->client.max_frame = tmp_cpu;
+		pr_debug("%s(), max frame=%d\n", __func__ ,
+			 self->client.max_frame);
+	}
+
+	/* RECONNECT_KEY, in case the link goes down! */
+	if (strcmp(param, "RECONNECT_KEY") == 0) {
+		pr_debug("Got reconnect key: ");
+		/* for (i = 0; i < val_len; i++) */
+/* 			printk("%02x", value[i]); */
+		memcpy(self->client.reconnect_key, value, val_len);
+		self->client.key_len = val_len;
+		pr_debug("\n");
+	}
+	/* FILTER_ENTRY, have we got an ethernet address? */
+	if (strcmp(param, "FILTER_ENTRY") == 0) {
+		bytes = value;
+		pr_debug("Ethernet address = %pM\n", bytes);
+		for (i = 0; i < 6; i++)
+			self->dev->dev_addr[i] = bytes[i];
+	}
+}
+
+/*
+ * Function irlan_client_get_value_confirm (obj_id, value)
+ *
+ *    Got results from remote LM-IAS
+ *
+ */
+void irlan_client_get_value_confirm(int result, __u16 obj_id,
+				    struct ias_value *value, void *priv)
+{
+	struct irlan_cb *self;
+
+	IRDA_ASSERT(priv != NULL, return;);
+
+	self = priv;
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	/* We probably don't need to make any more queries */
+	iriap_close(self->client.iriap);
+	self->client.iriap = NULL;
+
+	/* Check if request succeeded */
+	if (result != IAS_SUCCESS) {
+		pr_debug("%s(), got NULL value!\n", __func__);
+		irlan_do_client_event(self, IRLAN_IAS_PROVIDER_NOT_AVAIL,
+				      NULL);
+		return;
+	}
+
+	switch (value->type) {
+	case IAS_INTEGER:
+		self->dtsap_sel_ctrl = value->t.integer;
+
+		if (value->t.integer != -1) {
+			irlan_do_client_event(self, IRLAN_IAS_PROVIDER_AVAIL,
+					      NULL);
+			return;
+		}
+		irias_delete_value(value);
+		break;
+	default:
+		pr_debug("%s(), unknown type!\n", __func__);
+		break;
+	}
+	irlan_do_client_event(self, IRLAN_IAS_PROVIDER_NOT_AVAIL, NULL);
+}
diff --git a/drivers/staging/irda/net/irlan/irlan_client_event.c b/drivers/staging/irda/net/irlan/irlan_client_event.c
new file mode 100644
index 000000000000..cc93fabbbb19
--- /dev/null
+++ b/drivers/staging/irda/net/irlan/irlan_client_event.c
@@ -0,0 +1,511 @@
+/*********************************************************************
+ *
+ * Filename:      irlan_client_event.c
+ * Version:       0.9
+ * Description:   IrLAN client state machine
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sun Aug 31 20:14:37 1997
+ * Modified at:   Sun Dec 26 21:52:24 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1998-1999 Dag Brattli <dagb@cs.uit.no>,
+ *     All Rights Reserved.
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#include <linux/skbuff.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/timer.h>
+#include <net/irda/irmod.h>
+#include <net/irda/iriap.h>
+#include <net/irda/irlmp.h>
+#include <net/irda/irttp.h>
+
+#include <net/irda/irlan_common.h>
+#include <net/irda/irlan_client.h>
+#include <net/irda/irlan_event.h>
+
+static int irlan_client_state_idle (struct irlan_cb *self, IRLAN_EVENT event,
+				    struct sk_buff *skb);
+static int irlan_client_state_query(struct irlan_cb *self, IRLAN_EVENT event,
+				    struct sk_buff *skb);
+static int irlan_client_state_conn (struct irlan_cb *self, IRLAN_EVENT event,
+				    struct sk_buff *skb);
+static int irlan_client_state_info (struct irlan_cb *self, IRLAN_EVENT event,
+				    struct sk_buff *skb);
+static int irlan_client_state_media(struct irlan_cb *self, IRLAN_EVENT event,
+				    struct sk_buff *skb);
+static int irlan_client_state_open (struct irlan_cb *self, IRLAN_EVENT event,
+				    struct sk_buff *skb);
+static int irlan_client_state_wait (struct irlan_cb *self, IRLAN_EVENT event,
+				    struct sk_buff *skb);
+static int irlan_client_state_arb  (struct irlan_cb *self, IRLAN_EVENT event,
+				    struct sk_buff *skb);
+static int irlan_client_state_data (struct irlan_cb *self, IRLAN_EVENT event,
+				    struct sk_buff *skb);
+static int irlan_client_state_close(struct irlan_cb *self, IRLAN_EVENT event,
+				    struct sk_buff *skb);
+static int irlan_client_state_sync (struct irlan_cb *self, IRLAN_EVENT event,
+				    struct sk_buff *skb);
+
+static int (*state[])(struct irlan_cb *, IRLAN_EVENT event, struct sk_buff *) =
+{
+	irlan_client_state_idle,
+	irlan_client_state_query,
+	irlan_client_state_conn,
+	irlan_client_state_info,
+	irlan_client_state_media,
+	irlan_client_state_open,
+	irlan_client_state_wait,
+	irlan_client_state_arb,
+	irlan_client_state_data,
+	irlan_client_state_close,
+	irlan_client_state_sync
+};
+
+void irlan_do_client_event(struct irlan_cb *self, IRLAN_EVENT event,
+			   struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	(*state[ self->client.state]) (self, event, skb);
+}
+
+/*
+ * Function irlan_client_state_idle (event, skb, info)
+ *
+ *    IDLE, We are waiting for an indication that there is a provider
+ *    available.
+ */
+static int irlan_client_state_idle(struct irlan_cb *self, IRLAN_EVENT event,
+				   struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return -1;);
+
+	switch (event) {
+	case IRLAN_DISCOVERY_INDICATION:
+		if (self->client.iriap) {
+			net_warn_ratelimited("%s(), busy with a previous query\n",
+					     __func__);
+			return -EBUSY;
+		}
+
+		self->client.iriap = iriap_open(LSAP_ANY, IAS_CLIENT, self,
+						irlan_client_get_value_confirm);
+		/* Get some values from peer IAS */
+		irlan_next_client_state(self, IRLAN_QUERY);
+		iriap_getvaluebyclass_request(self->client.iriap,
+					      self->saddr, self->daddr,
+					      "IrLAN", "IrDA:TinyTP:LsapSel");
+		break;
+	case IRLAN_WATCHDOG_TIMEOUT:
+		pr_debug("%s(), IRLAN_WATCHDOG_TIMEOUT\n", __func__);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %d\n", __func__ , event);
+		break;
+	}
+	if (skb)
+		dev_kfree_skb(skb);
+
+	return 0;
+}
+
+/*
+ * Function irlan_client_state_query (event, skb, info)
+ *
+ *    QUERY, We have queryed the remote IAS and is ready to connect
+ *    to provider, just waiting for the confirm.
+ *
+ */
+static int irlan_client_state_query(struct irlan_cb *self, IRLAN_EVENT event,
+				    struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return -1;);
+
+	switch(event) {
+	case IRLAN_IAS_PROVIDER_AVAIL:
+		IRDA_ASSERT(self->dtsap_sel_ctrl != 0, return -1;);
+
+		self->client.open_retries = 0;
+
+		irttp_connect_request(self->client.tsap_ctrl,
+				      self->dtsap_sel_ctrl,
+				      self->saddr, self->daddr, NULL,
+				      IRLAN_MTU, NULL);
+		irlan_next_client_state(self, IRLAN_CONN);
+		break;
+	case IRLAN_IAS_PROVIDER_NOT_AVAIL:
+		pr_debug("%s(), IAS_PROVIDER_NOT_AVAIL\n", __func__);
+		irlan_next_client_state(self, IRLAN_IDLE);
+
+		/* Give the client a kick! */
+		if ((self->provider.access_type == ACCESS_PEER) &&
+		    (self->provider.state != IRLAN_IDLE))
+			irlan_client_wakeup(self, self->saddr, self->daddr);
+		break;
+	case IRLAN_LMP_DISCONNECT:
+	case IRLAN_LAP_DISCONNECT:
+		irlan_next_client_state(self, IRLAN_IDLE);
+		break;
+	case IRLAN_WATCHDOG_TIMEOUT:
+		pr_debug("%s(), IRLAN_WATCHDOG_TIMEOUT\n", __func__);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %d\n", __func__ , event);
+		break;
+	}
+	if (skb)
+		dev_kfree_skb(skb);
+
+	return 0;
+}
+
+/*
+ * Function irlan_client_state_conn (event, skb, info)
+ *
+ *    CONN, We have connected to a provider but has not issued any
+ *    commands yet.
+ *
+ */
+static int irlan_client_state_conn(struct irlan_cb *self, IRLAN_EVENT event,
+				   struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return -1;);
+
+	switch (event) {
+	case IRLAN_CONNECT_COMPLETE:
+		/* Send getinfo cmd */
+		irlan_get_provider_info(self);
+		irlan_next_client_state(self, IRLAN_INFO);
+		break;
+	case IRLAN_LMP_DISCONNECT:
+	case IRLAN_LAP_DISCONNECT:
+		irlan_next_client_state(self, IRLAN_IDLE);
+		break;
+	case IRLAN_WATCHDOG_TIMEOUT:
+		pr_debug("%s(), IRLAN_WATCHDOG_TIMEOUT\n", __func__);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %d\n", __func__ , event);
+		break;
+	}
+	if (skb)
+		dev_kfree_skb(skb);
+
+	return 0;
+}
+
+/*
+ * Function irlan_client_state_info (self, event, skb, info)
+ *
+ *    INFO, We have issued a GetInfo command and is awaiting a reply.
+ */
+static int irlan_client_state_info(struct irlan_cb *self, IRLAN_EVENT event,
+				   struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return -1;);
+
+	switch (event) {
+	case IRLAN_DATA_INDICATION:
+		IRDA_ASSERT(skb != NULL, return -1;);
+
+		irlan_client_parse_response(self, skb);
+
+		irlan_next_client_state(self, IRLAN_MEDIA);
+
+		irlan_get_media_char(self);
+		break;
+
+	case IRLAN_LMP_DISCONNECT:
+	case IRLAN_LAP_DISCONNECT:
+		irlan_next_client_state(self, IRLAN_IDLE);
+		break;
+	case IRLAN_WATCHDOG_TIMEOUT:
+		pr_debug("%s(), IRLAN_WATCHDOG_TIMEOUT\n", __func__);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %d\n", __func__ , event);
+		break;
+	}
+	if (skb)
+		dev_kfree_skb(skb);
+
+	return 0;
+}
+
+/*
+ * Function irlan_client_state_media (self, event, skb, info)
+ *
+ *    MEDIA, The irlan_client has issued a GetMedia command and is awaiting a
+ *    reply.
+ *
+ */
+static int irlan_client_state_media(struct irlan_cb *self, IRLAN_EVENT event,
+				    struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return -1;);
+
+	switch(event) {
+	case IRLAN_DATA_INDICATION:
+		irlan_client_parse_response(self, skb);
+		irlan_open_data_channel(self);
+		irlan_next_client_state(self, IRLAN_OPEN);
+		break;
+	case IRLAN_LMP_DISCONNECT:
+	case IRLAN_LAP_DISCONNECT:
+		irlan_next_client_state(self, IRLAN_IDLE);
+		break;
+	case IRLAN_WATCHDOG_TIMEOUT:
+		pr_debug("%s(), IRLAN_WATCHDOG_TIMEOUT\n", __func__);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %d\n", __func__ , event);
+		break;
+	}
+	if (skb)
+		dev_kfree_skb(skb);
+
+	return 0;
+}
+
+/*
+ * Function irlan_client_state_open (self, event, skb, info)
+ *
+ *    OPEN, The irlan_client has issued a OpenData command and is awaiting a
+ *    reply
+ *
+ */
+static int irlan_client_state_open(struct irlan_cb *self, IRLAN_EVENT event,
+				   struct sk_buff *skb)
+{
+	struct qos_info qos;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+
+	switch(event) {
+	case IRLAN_DATA_INDICATION:
+		irlan_client_parse_response(self, skb);
+
+		/*
+		 *  Check if we have got the remote TSAP for data
+		 *  communications
+		 */
+		IRDA_ASSERT(self->dtsap_sel_data != 0, return -1;);
+
+		/* Check which access type we are dealing with */
+		switch (self->client.access_type) {
+		case ACCESS_PEER:
+		    if (self->provider.state == IRLAN_OPEN) {
+
+			    irlan_next_client_state(self, IRLAN_ARB);
+			    irlan_do_client_event(self, IRLAN_CHECK_CON_ARB,
+						  NULL);
+		    } else {
+
+			    irlan_next_client_state(self, IRLAN_WAIT);
+		    }
+		    break;
+		case ACCESS_DIRECT:
+		case ACCESS_HOSTED:
+			qos.link_disc_time.bits = 0x01; /* 3 secs */
+
+			irttp_connect_request(self->tsap_data,
+					      self->dtsap_sel_data,
+					      self->saddr, self->daddr, &qos,
+					      IRLAN_MTU, NULL);
+
+			irlan_next_client_state(self, IRLAN_DATA);
+			break;
+		default:
+			pr_debug("%s(), unknown access type!\n", __func__);
+			break;
+		}
+		break;
+	case IRLAN_LMP_DISCONNECT:
+	case IRLAN_LAP_DISCONNECT:
+		irlan_next_client_state(self, IRLAN_IDLE);
+		break;
+	case IRLAN_WATCHDOG_TIMEOUT:
+		pr_debug("%s(), IRLAN_WATCHDOG_TIMEOUT\n", __func__);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %d\n", __func__ , event);
+		break;
+	}
+
+	if (skb)
+		dev_kfree_skb(skb);
+
+	return 0;
+}
+
+/*
+ * Function irlan_client_state_wait (self, event, skb, info)
+ *
+ *    WAIT, The irlan_client is waiting for the local provider to enter the
+ *    provider OPEN state.
+ *
+ */
+static int irlan_client_state_wait(struct irlan_cb *self, IRLAN_EVENT event,
+				   struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return -1;);
+
+	switch(event) {
+	case IRLAN_PROVIDER_SIGNAL:
+		irlan_next_client_state(self, IRLAN_ARB);
+		irlan_do_client_event(self, IRLAN_CHECK_CON_ARB, NULL);
+		break;
+	case IRLAN_LMP_DISCONNECT:
+	case IRLAN_LAP_DISCONNECT:
+		irlan_next_client_state(self, IRLAN_IDLE);
+		break;
+	case IRLAN_WATCHDOG_TIMEOUT:
+		pr_debug("%s(), IRLAN_WATCHDOG_TIMEOUT\n", __func__);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %d\n", __func__ , event);
+		break;
+	}
+	if (skb)
+		dev_kfree_skb(skb);
+
+	return 0;
+}
+
+static int irlan_client_state_arb(struct irlan_cb *self, IRLAN_EVENT event,
+				  struct sk_buff *skb)
+{
+	struct qos_info qos;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+
+	switch(event) {
+	case IRLAN_CHECK_CON_ARB:
+		if (self->client.recv_arb_val == self->provider.send_arb_val) {
+			irlan_next_client_state(self, IRLAN_CLOSE);
+			irlan_close_data_channel(self);
+		} else if (self->client.recv_arb_val <
+			   self->provider.send_arb_val)
+		{
+			qos.link_disc_time.bits = 0x01; /* 3 secs */
+
+			irlan_next_client_state(self, IRLAN_DATA);
+			irttp_connect_request(self->tsap_data,
+					      self->dtsap_sel_data,
+					      self->saddr, self->daddr, &qos,
+					      IRLAN_MTU, NULL);
+		} else if (self->client.recv_arb_val >
+			   self->provider.send_arb_val)
+		{
+			pr_debug("%s(), lost the battle :-(\n", __func__);
+		}
+		break;
+	case IRLAN_DATA_CONNECT_INDICATION:
+		irlan_next_client_state(self, IRLAN_DATA);
+		break;
+	case IRLAN_LMP_DISCONNECT:
+	case IRLAN_LAP_DISCONNECT:
+		irlan_next_client_state(self, IRLAN_IDLE);
+		break;
+	case IRLAN_WATCHDOG_TIMEOUT:
+		pr_debug("%s(), IRLAN_WATCHDOG_TIMEOUT\n", __func__);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %d\n", __func__ , event);
+		break;
+	}
+	if (skb)
+		dev_kfree_skb(skb);
+
+	return 0;
+}
+
+/*
+ * Function irlan_client_state_data (self, event, skb, info)
+ *
+ *    DATA, The data channel is connected, allowing data transfers between
+ *    the local and remote machines.
+ *
+ */
+static int irlan_client_state_data(struct irlan_cb *self, IRLAN_EVENT event,
+				   struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return -1;);
+
+	switch(event) {
+	case IRLAN_DATA_INDICATION:
+		irlan_client_parse_response(self, skb);
+		break;
+	case IRLAN_LMP_DISCONNECT: /* FALLTHROUGH */
+	case IRLAN_LAP_DISCONNECT:
+		irlan_next_client_state(self, IRLAN_IDLE);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %d\n", __func__ , event);
+		break;
+	}
+	if (skb)
+		dev_kfree_skb(skb);
+
+	return 0;
+}
+
+/*
+ * Function irlan_client_state_close (self, event, skb, info)
+ *
+ *
+ *
+ */
+static int irlan_client_state_close(struct irlan_cb *self, IRLAN_EVENT event,
+				    struct sk_buff *skb)
+{
+	if (skb)
+		dev_kfree_skb(skb);
+
+	return 0;
+}
+
+/*
+ * Function irlan_client_state_sync (self, event, skb, info)
+ *
+ *
+ *
+ */
+static int irlan_client_state_sync(struct irlan_cb *self, IRLAN_EVENT event,
+				   struct sk_buff *skb)
+{
+	if (skb)
+		dev_kfree_skb(skb);
+
+	return 0;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/drivers/staging/irda/net/irlan/irlan_common.c b/drivers/staging/irda/net/irlan/irlan_common.c
new file mode 100644
index 000000000000..481bbc2a4349
--- /dev/null
+++ b/drivers/staging/irda/net/irlan/irlan_common.c
@@ -0,0 +1,1176 @@
+/*********************************************************************
+ *
+ * Filename:      irlan_common.c
+ * Version:       0.9
+ * Description:   IrDA LAN Access Protocol Implementation
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sun Aug 31 20:14:37 1997
+ * Modified at:   Sun Dec 26 21:53:10 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1997, 1999 Dag Brattli <dagb@cs.uit.no>,
+ *     All Rights Reserved.
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#include <linux/module.h>
+
+#include <linux/kernel.h>
+#include <linux/string.h>
+#include <linux/gfp.h>
+#include <linux/init.h>
+#include <linux/errno.h>
+#include <linux/proc_fs.h>
+#include <linux/sched.h>
+#include <linux/seq_file.h>
+#include <linux/random.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/rtnetlink.h>
+#include <linux/moduleparam.h>
+#include <linux/bitops.h>
+
+#include <asm/byteorder.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/irttp.h>
+#include <net/irda/irlmp.h>
+#include <net/irda/iriap.h>
+#include <net/irda/timer.h>
+
+#include <net/irda/irlan_common.h>
+#include <net/irda/irlan_client.h>
+#include <net/irda/irlan_provider.h>
+#include <net/irda/irlan_eth.h>
+#include <net/irda/irlan_filter.h>
+
+
+/* extern char sysctl_devname[]; */
+
+/*
+ *  Master structure
+ */
+static LIST_HEAD(irlans);
+
+static void *ckey;
+static void *skey;
+
+/* Module parameters */
+static bool eth;   /* Use "eth" or "irlan" name for devices */
+static int access = ACCESS_PEER; /* PEER, DIRECT or HOSTED */
+
+#ifdef CONFIG_PROC_FS
+static const char *const irlan_access[] = {
+	"UNKNOWN",
+	"DIRECT",
+	"PEER",
+	"HOSTED"
+};
+
+static const char *const irlan_media[] = {
+	"UNKNOWN",
+	"802.3",
+	"802.5"
+};
+
+extern struct proc_dir_entry *proc_irda;
+
+static int irlan_seq_open(struct inode *inode, struct file *file);
+
+static const struct file_operations irlan_fops = {
+	.owner	 = THIS_MODULE,
+	.open    = irlan_seq_open,
+	.read    = seq_read,
+	.llseek  = seq_lseek,
+	.release = seq_release,
+};
+
+extern struct proc_dir_entry *proc_irda;
+#endif /* CONFIG_PROC_FS */
+
+static struct irlan_cb __init *irlan_open(__u32 saddr, __u32 daddr);
+static void __irlan_close(struct irlan_cb *self);
+static int __irlan_insert_param(struct sk_buff *skb, char *param, int type,
+				__u8 value_byte, __u16 value_short,
+				__u8 *value_array, __u16 value_len);
+static void irlan_open_unicast_addr(struct irlan_cb *self);
+static void irlan_get_unicast_addr(struct irlan_cb *self);
+void irlan_close_tsaps(struct irlan_cb *self);
+
+/*
+ * Function irlan_init (void)
+ *
+ *    Initialize IrLAN layer
+ *
+ */
+static int __init irlan_init(void)
+{
+	struct irlan_cb *new;
+	__u16 hints;
+
+#ifdef CONFIG_PROC_FS
+	{ struct proc_dir_entry *proc;
+	proc = proc_create("irlan", 0, proc_irda, &irlan_fops);
+	if (!proc) {
+		printk(KERN_ERR "irlan_init: can't create /proc entry!\n");
+		return -ENODEV;
+	}
+	}
+#endif /* CONFIG_PROC_FS */
+
+	hints = irlmp_service_to_hint(S_LAN);
+
+	/* Register with IrLMP as a client */
+	ckey = irlmp_register_client(hints, &irlan_client_discovery_indication,
+				     NULL, NULL);
+	if (!ckey)
+		goto err_ckey;
+
+	/* Register with IrLMP as a service */
+	skey = irlmp_register_service(hints);
+	if (!skey)
+		goto err_skey;
+
+	/* Start the master IrLAN instance (the only one for now) */
+	new = irlan_open(DEV_ADDR_ANY, DEV_ADDR_ANY);
+	if (!new)
+		goto err_open;
+
+	/* The master will only open its (listen) control TSAP */
+	irlan_provider_open_ctrl_tsap(new);
+
+	/* Do some fast discovery! */
+	irlmp_discovery_request(DISCOVERY_DEFAULT_SLOTS);
+
+	return 0;
+
+err_open:
+	irlmp_unregister_service(skey);
+err_skey:
+	irlmp_unregister_client(ckey);
+err_ckey:
+#ifdef CONFIG_PROC_FS
+	remove_proc_entry("irlan", proc_irda);
+#endif /* CONFIG_PROC_FS */
+
+	return -ENOMEM;
+}
+
+static void __exit irlan_cleanup(void)
+{
+	struct irlan_cb *self, *next;
+
+	irlmp_unregister_client(ckey);
+	irlmp_unregister_service(skey);
+
+#ifdef CONFIG_PROC_FS
+	remove_proc_entry("irlan", proc_irda);
+#endif /* CONFIG_PROC_FS */
+
+	/* Cleanup any leftover network devices */
+	rtnl_lock();
+	list_for_each_entry_safe(self, next, &irlans, dev_list) {
+		__irlan_close(self);
+	}
+	rtnl_unlock();
+}
+
+/*
+ * Function irlan_open (void)
+ *
+ *    Open new instance of a client/provider, we should only register the
+ *    network device if this instance is ment for a particular client/provider
+ */
+static struct irlan_cb __init *irlan_open(__u32 saddr, __u32 daddr)
+{
+	struct net_device *dev;
+	struct irlan_cb *self;
+
+	/* Create network device with irlan */
+	dev = alloc_irlandev(eth ? "eth%d" : "irlan%d");
+	if (!dev)
+		return NULL;
+
+	self = netdev_priv(dev);
+	self->dev = dev;
+
+	/*
+	 *  Initialize local device structure
+	 */
+	self->magic = IRLAN_MAGIC;
+	self->saddr = saddr;
+	self->daddr = daddr;
+
+	/* Provider access can only be PEER, DIRECT, or HOSTED */
+	self->provider.access_type = access;
+	if (access == ACCESS_DIRECT) {
+		/*
+		 * Since we are emulating an IrLAN sever we will have to
+		 * give ourself an ethernet address!
+		 */
+		dev->dev_addr[0] = 0x40;
+		dev->dev_addr[1] = 0x00;
+		dev->dev_addr[2] = 0x00;
+		dev->dev_addr[3] = 0x00;
+		get_random_bytes(dev->dev_addr+4, 1);
+		get_random_bytes(dev->dev_addr+5, 1);
+	}
+
+	self->media = MEDIA_802_3;
+	self->disconnect_reason = LM_USER_REQUEST;
+	init_timer(&self->watchdog_timer);
+	init_timer(&self->client.kick_timer);
+	init_waitqueue_head(&self->open_wait);
+
+	skb_queue_head_init(&self->client.txq);
+
+	irlan_next_client_state(self, IRLAN_IDLE);
+	irlan_next_provider_state(self, IRLAN_IDLE);
+
+	if (register_netdev(dev)) {
+		pr_debug("%s(), register_netdev() failed!\n",
+			 __func__);
+		self = NULL;
+		free_netdev(dev);
+	} else {
+		rtnl_lock();
+		list_add_rcu(&self->dev_list, &irlans);
+		rtnl_unlock();
+	}
+
+	return self;
+}
+/*
+ * Function __irlan_close (self)
+ *
+ *    This function closes and deallocates the IrLAN client instances. Be
+ *    aware that other functions which calls client_close() must
+ *    remove self from irlans list first.
+ */
+static void __irlan_close(struct irlan_cb *self)
+{
+	ASSERT_RTNL();
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	del_timer_sync(&self->watchdog_timer);
+	del_timer_sync(&self->client.kick_timer);
+
+	/* Close all open connections and remove TSAPs */
+	irlan_close_tsaps(self);
+
+	if (self->client.iriap)
+		iriap_close(self->client.iriap);
+
+	/* Remove frames queued on the control channel */
+	skb_queue_purge(&self->client.txq);
+
+	/* Unregister and free self via destructor */
+	unregister_netdevice(self->dev);
+}
+
+/* Find any instance of irlan, used for client discovery wakeup */
+struct irlan_cb *irlan_get_any(void)
+{
+	struct irlan_cb *self;
+
+	list_for_each_entry_rcu(self, &irlans, dev_list) {
+		return self;
+	}
+	return NULL;
+}
+
+/*
+ * Function irlan_connect_indication (instance, sap, qos, max_sdu_size, skb)
+ *
+ *    Here we receive the connect indication for the data channel
+ *
+ */
+static void irlan_connect_indication(void *instance, void *sap,
+				     struct qos_info *qos,
+				     __u32 max_sdu_size,
+				     __u8 max_header_size,
+				     struct sk_buff *skb)
+{
+	struct irlan_cb *self;
+	struct tsap_cb *tsap;
+
+	self = instance;
+	tsap = sap;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+	IRDA_ASSERT(tsap == self->tsap_data,return;);
+
+	self->max_sdu_size = max_sdu_size;
+	self->max_header_size = max_header_size;
+
+	pr_debug("%s: We are now connected!\n", __func__);
+
+	del_timer(&self->watchdog_timer);
+
+	/* If you want to pass the skb to *both* state machines, you will
+	 * need to skb_clone() it, so that you don't free it twice.
+	 * As the state machines don't need it, git rid of it here...
+	 * Jean II */
+	if (skb)
+		dev_kfree_skb(skb);
+
+	irlan_do_provider_event(self, IRLAN_DATA_CONNECT_INDICATION, NULL);
+	irlan_do_client_event(self, IRLAN_DATA_CONNECT_INDICATION, NULL);
+
+	if (self->provider.access_type == ACCESS_PEER) {
+		/*
+		 * Data channel is open, so we are now allowed to
+		 * configure the remote filter
+		 */
+		irlan_get_unicast_addr(self);
+		irlan_open_unicast_addr(self);
+	}
+	/* Ready to transfer Ethernet frames (at last) */
+	netif_start_queue(self->dev); /* Clear reason */
+}
+
+static void irlan_connect_confirm(void *instance, void *sap,
+				  struct qos_info *qos,
+				  __u32 max_sdu_size,
+				  __u8 max_header_size,
+				  struct sk_buff *skb)
+{
+	struct irlan_cb *self;
+
+	self = instance;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	self->max_sdu_size = max_sdu_size;
+	self->max_header_size = max_header_size;
+
+	/* TODO: we could set the MTU depending on the max_sdu_size */
+
+	pr_debug("%s: We are now connected!\n", __func__);
+	del_timer(&self->watchdog_timer);
+
+	/*
+	 * Data channel is open, so we are now allowed to configure the remote
+	 * filter
+	 */
+	irlan_get_unicast_addr(self);
+	irlan_open_unicast_addr(self);
+
+	/* Open broadcast and multicast filter by default */
+	irlan_set_broadcast_filter(self, TRUE);
+	irlan_set_multicast_filter(self, TRUE);
+
+	/* Ready to transfer Ethernet frames */
+	netif_start_queue(self->dev);
+	self->disconnect_reason = 0; /* Clear reason */
+	wake_up_interruptible(&self->open_wait);
+}
+
+/*
+ * Function irlan_client_disconnect_indication (handle)
+ *
+ *    Callback function for the IrTTP layer. Indicates a disconnection of
+ *    the specified connection (handle)
+ */
+static void irlan_disconnect_indication(void *instance,
+					void *sap, LM_REASON reason,
+					struct sk_buff *userdata)
+{
+	struct irlan_cb *self;
+	struct tsap_cb *tsap;
+
+	pr_debug("%s(), reason=%d\n", __func__ , reason);
+
+	self = instance;
+	tsap = sap;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+	IRDA_ASSERT(tsap != NULL, return;);
+	IRDA_ASSERT(tsap->magic == TTP_TSAP_MAGIC, return;);
+
+	IRDA_ASSERT(tsap == self->tsap_data, return;);
+
+	pr_debug("IrLAN, data channel disconnected by peer!\n");
+
+	/* Save reason so we know if we should try to reconnect or not */
+	self->disconnect_reason = reason;
+
+	switch (reason) {
+	case LM_USER_REQUEST: /* User request */
+		pr_debug("%s(), User requested\n", __func__);
+		break;
+	case LM_LAP_DISCONNECT: /* Unexpected IrLAP disconnect */
+		pr_debug("%s(), Unexpected IrLAP disconnect\n", __func__);
+		break;
+	case LM_CONNECT_FAILURE: /* Failed to establish IrLAP connection */
+		pr_debug("%s(), IrLAP connect failed\n", __func__);
+		break;
+	case LM_LAP_RESET:  /* IrLAP reset */
+		pr_debug("%s(), IrLAP reset\n", __func__);
+		break;
+	case LM_INIT_DISCONNECT:
+		pr_debug("%s(), IrLMP connect failed\n", __func__);
+		break;
+	default:
+		net_err_ratelimited("%s(), Unknown disconnect reason\n",
+				    __func__);
+		break;
+	}
+
+	/* If you want to pass the skb to *both* state machines, you will
+	 * need to skb_clone() it, so that you don't free it twice.
+	 * As the state machines don't need it, git rid of it here...
+	 * Jean II */
+	if (userdata)
+		dev_kfree_skb(userdata);
+
+	irlan_do_client_event(self, IRLAN_LMP_DISCONNECT, NULL);
+	irlan_do_provider_event(self, IRLAN_LMP_DISCONNECT, NULL);
+
+	wake_up_interruptible(&self->open_wait);
+}
+
+void irlan_open_data_tsap(struct irlan_cb *self)
+{
+	struct tsap_cb *tsap;
+	notify_t notify;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	/* Check if already open */
+	if (self->tsap_data)
+		return;
+
+	irda_notify_init(&notify);
+
+	notify.data_indication       = irlan_eth_receive;
+	notify.udata_indication      = irlan_eth_receive;
+	notify.connect_indication    = irlan_connect_indication;
+	notify.connect_confirm       = irlan_connect_confirm;
+	notify.flow_indication       = irlan_eth_flow_indication;
+	notify.disconnect_indication = irlan_disconnect_indication;
+	notify.instance              = self;
+	strlcpy(notify.name, "IrLAN data", sizeof(notify.name));
+
+	tsap = irttp_open_tsap(LSAP_ANY, DEFAULT_INITIAL_CREDIT, &notify);
+	if (!tsap) {
+		pr_debug("%s(), Got no tsap!\n", __func__);
+		return;
+	}
+	self->tsap_data = tsap;
+
+	/*
+	 *  This is the data TSAP selector which we will pass to the client
+	 *  when the client ask for it.
+	 */
+	self->stsap_sel_data = self->tsap_data->stsap_sel;
+}
+
+void irlan_close_tsaps(struct irlan_cb *self)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	/* Disconnect and close all open TSAP connections */
+	if (self->tsap_data) {
+		irttp_disconnect_request(self->tsap_data, NULL, P_NORMAL);
+		irttp_close_tsap(self->tsap_data);
+		self->tsap_data = NULL;
+	}
+	if (self->client.tsap_ctrl) {
+		irttp_disconnect_request(self->client.tsap_ctrl, NULL,
+					 P_NORMAL);
+		irttp_close_tsap(self->client.tsap_ctrl);
+		self->client.tsap_ctrl = NULL;
+	}
+	if (self->provider.tsap_ctrl) {
+		irttp_disconnect_request(self->provider.tsap_ctrl, NULL,
+					 P_NORMAL);
+		irttp_close_tsap(self->provider.tsap_ctrl);
+		self->provider.tsap_ctrl = NULL;
+	}
+	self->disconnect_reason = LM_USER_REQUEST;
+}
+
+/*
+ * Function irlan_ias_register (self, tsap_sel)
+ *
+ *    Register with LM-IAS
+ *
+ */
+void irlan_ias_register(struct irlan_cb *self, __u8 tsap_sel)
+{
+	struct ias_object *obj;
+	struct ias_value *new_value;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	/*
+	 * Check if object has already been registered by a previous provider.
+	 * If that is the case, we just change the value of the attribute
+	 */
+	if (!irias_find_object("IrLAN")) {
+		obj = irias_new_object("IrLAN", IAS_IRLAN_ID);
+		irias_add_integer_attrib(obj, "IrDA:TinyTP:LsapSel", tsap_sel,
+					 IAS_KERNEL_ATTR);
+		irias_insert_object(obj);
+	} else {
+		new_value = irias_new_integer_value(tsap_sel);
+		irias_object_change_attribute("IrLAN", "IrDA:TinyTP:LsapSel",
+					      new_value);
+	}
+
+	/* Register PnP object only if not registered before */
+	if (!irias_find_object("PnP")) {
+		obj = irias_new_object("PnP", IAS_PNP_ID);
+#if 0
+		irias_add_string_attrib(obj, "Name", sysctl_devname,
+					IAS_KERNEL_ATTR);
+#else
+		irias_add_string_attrib(obj, "Name", "Linux", IAS_KERNEL_ATTR);
+#endif
+		irias_add_string_attrib(obj, "DeviceID", "HWP19F0",
+					IAS_KERNEL_ATTR);
+		irias_add_integer_attrib(obj, "CompCnt", 1, IAS_KERNEL_ATTR);
+		if (self->provider.access_type == ACCESS_PEER)
+			irias_add_string_attrib(obj, "Comp#01", "PNP8389",
+						IAS_KERNEL_ATTR);
+		else
+			irias_add_string_attrib(obj, "Comp#01", "PNP8294",
+						IAS_KERNEL_ATTR);
+
+		irias_add_string_attrib(obj, "Manufacturer",
+					"Linux-IrDA Project", IAS_KERNEL_ATTR);
+		irias_insert_object(obj);
+	}
+}
+
+/*
+ * Function irlan_run_ctrl_tx_queue (self)
+ *
+ *    Try to send the next command in the control transmit queue
+ *
+ */
+int irlan_run_ctrl_tx_queue(struct irlan_cb *self)
+{
+	struct sk_buff *skb;
+
+	if (irda_lock(&self->client.tx_busy) == FALSE)
+		return -EBUSY;
+
+	skb = skb_dequeue(&self->client.txq);
+	if (!skb) {
+		self->client.tx_busy = FALSE;
+		return 0;
+	}
+
+	/* Check that it's really possible to send commands */
+	if ((self->client.tsap_ctrl == NULL) ||
+	    (self->client.state == IRLAN_IDLE))
+	{
+		self->client.tx_busy = FALSE;
+		dev_kfree_skb(skb);
+		return -1;
+	}
+	pr_debug("%s(), sending ...\n", __func__);
+
+	return irttp_data_request(self->client.tsap_ctrl, skb);
+}
+
+/*
+ * Function irlan_ctrl_data_request (self, skb)
+ *
+ *    This function makes sure that commands on the control channel is being
+ *    sent in a command/response fashion
+ */
+static void irlan_ctrl_data_request(struct irlan_cb *self, struct sk_buff *skb)
+{
+	/* Queue command */
+	skb_queue_tail(&self->client.txq, skb);
+
+	/* Try to send command */
+	irlan_run_ctrl_tx_queue(self);
+}
+
+/*
+ * Function irlan_get_provider_info (self)
+ *
+ *    Send Get Provider Information command to peer IrLAN layer
+ *
+ */
+void irlan_get_provider_info(struct irlan_cb *self)
+{
+	struct sk_buff *skb;
+	__u8 *frame;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	skb = alloc_skb(IRLAN_MAX_HEADER + IRLAN_CMD_HEADER,
+			GFP_ATOMIC);
+	if (!skb)
+		return;
+
+	/* Reserve space for TTP, LMP, and LAP header */
+	skb_reserve(skb, self->client.max_header_size);
+	skb_put(skb, 2);
+
+	frame = skb->data;
+
+	frame[0] = CMD_GET_PROVIDER_INFO;
+	frame[1] = 0x00;                 /* Zero parameters */
+
+	irlan_ctrl_data_request(self, skb);
+}
+
+/*
+ * Function irlan_open_data_channel (self)
+ *
+ *    Send an Open Data Command to provider
+ *
+ */
+void irlan_open_data_channel(struct irlan_cb *self)
+{
+	struct sk_buff *skb;
+	__u8 *frame;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	skb = alloc_skb(IRLAN_MAX_HEADER + IRLAN_CMD_HEADER +
+			IRLAN_STRING_PARAMETER_LEN("MEDIA", "802.3") +
+			IRLAN_STRING_PARAMETER_LEN("ACCESS_TYPE", "DIRECT"),
+			GFP_ATOMIC);
+	if (!skb)
+		return;
+
+	skb_reserve(skb, self->client.max_header_size);
+	skb_put(skb, 2);
+
+	frame = skb->data;
+
+	/* Build frame */
+	frame[0] = CMD_OPEN_DATA_CHANNEL;
+	frame[1] = 0x02; /* Two parameters */
+
+	irlan_insert_string_param(skb, "MEDIA", "802.3");
+	irlan_insert_string_param(skb, "ACCESS_TYPE", "DIRECT");
+	/* irlan_insert_string_param(skb, "MODE", "UNRELIABLE"); */
+
+/* 	self->use_udata = TRUE; */
+
+	irlan_ctrl_data_request(self, skb);
+}
+
+void irlan_close_data_channel(struct irlan_cb *self)
+{
+	struct sk_buff *skb;
+	__u8 *frame;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	/* Check if the TSAP is still there */
+	if (self->client.tsap_ctrl == NULL)
+		return;
+
+	skb = alloc_skb(IRLAN_MAX_HEADER + IRLAN_CMD_HEADER +
+			IRLAN_BYTE_PARAMETER_LEN("DATA_CHAN"),
+			GFP_ATOMIC);
+	if (!skb)
+		return;
+
+	skb_reserve(skb, self->client.max_header_size);
+	skb_put(skb, 2);
+
+	frame = skb->data;
+
+	/* Build frame */
+	frame[0] = CMD_CLOSE_DATA_CHAN;
+	frame[1] = 0x01; /* One parameter */
+
+	irlan_insert_byte_param(skb, "DATA_CHAN", self->dtsap_sel_data);
+
+	irlan_ctrl_data_request(self, skb);
+}
+
+/*
+ * Function irlan_open_unicast_addr (self)
+ *
+ *    Make IrLAN provider accept ethernet frames addressed to the unicast
+ *    address.
+ *
+ */
+static void irlan_open_unicast_addr(struct irlan_cb *self)
+{
+	struct sk_buff *skb;
+	__u8 *frame;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	skb = alloc_skb(IRLAN_MAX_HEADER + IRLAN_CMD_HEADER +
+			IRLAN_BYTE_PARAMETER_LEN("DATA_CHAN") +
+			IRLAN_STRING_PARAMETER_LEN("FILTER_TYPE", "DIRECTED") +
+			IRLAN_STRING_PARAMETER_LEN("FILTER_MODE", "FILTER"),
+			GFP_ATOMIC);
+	if (!skb)
+		return;
+
+	/* Reserve space for TTP, LMP, and LAP header */
+	skb_reserve(skb, self->max_header_size);
+	skb_put(skb, 2);
+
+	frame = skb->data;
+
+	frame[0] = CMD_FILTER_OPERATION;
+	frame[1] = 0x03;                 /* Three parameters */
+	irlan_insert_byte_param(skb, "DATA_CHAN" , self->dtsap_sel_data);
+	irlan_insert_string_param(skb, "FILTER_TYPE", "DIRECTED");
+	irlan_insert_string_param(skb, "FILTER_MODE", "FILTER");
+
+	irlan_ctrl_data_request(self, skb);
+}
+
+/*
+ * Function irlan_set_broadcast_filter (self, status)
+ *
+ *    Make IrLAN provider accept ethernet frames addressed to the broadcast
+ *    address. Be careful with the use of this one, since there may be a lot
+ *    of broadcast traffic out there. We can still function without this
+ *    one but then _we_ have to initiate all communication with other
+ *    hosts, since ARP request for this host will not be answered.
+ */
+void irlan_set_broadcast_filter(struct irlan_cb *self, int status)
+{
+	struct sk_buff *skb;
+	__u8 *frame;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	skb = alloc_skb(IRLAN_MAX_HEADER + IRLAN_CMD_HEADER +
+			IRLAN_BYTE_PARAMETER_LEN("DATA_CHAN") +
+			IRLAN_STRING_PARAMETER_LEN("FILTER_TYPE", "BROADCAST") +
+			/* We may waste one byte here...*/
+			IRLAN_STRING_PARAMETER_LEN("FILTER_MODE", "FILTER"),
+			GFP_ATOMIC);
+	if (!skb)
+		return;
+
+	/* Reserve space for TTP, LMP, and LAP header */
+	skb_reserve(skb, self->client.max_header_size);
+	skb_put(skb, 2);
+
+	frame = skb->data;
+
+	frame[0] = CMD_FILTER_OPERATION;
+	frame[1] = 0x03;                 /* Three parameters */
+	irlan_insert_byte_param(skb, "DATA_CHAN", self->dtsap_sel_data);
+	irlan_insert_string_param(skb, "FILTER_TYPE", "BROADCAST");
+	if (status)
+		irlan_insert_string_param(skb, "FILTER_MODE", "FILTER");
+	else
+		irlan_insert_string_param(skb, "FILTER_MODE", "NONE");
+
+	irlan_ctrl_data_request(self, skb);
+}
+
+/*
+ * Function irlan_set_multicast_filter (self, status)
+ *
+ *    Make IrLAN provider accept ethernet frames addressed to the multicast
+ *    address.
+ *
+ */
+void irlan_set_multicast_filter(struct irlan_cb *self, int status)
+{
+	struct sk_buff *skb;
+	__u8 *frame;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	skb = alloc_skb(IRLAN_MAX_HEADER + IRLAN_CMD_HEADER +
+			IRLAN_BYTE_PARAMETER_LEN("DATA_CHAN") +
+			IRLAN_STRING_PARAMETER_LEN("FILTER_TYPE", "MULTICAST") +
+			/* We may waste one byte here...*/
+			IRLAN_STRING_PARAMETER_LEN("FILTER_MODE", "NONE"),
+			GFP_ATOMIC);
+	if (!skb)
+		return;
+
+	/* Reserve space for TTP, LMP, and LAP header */
+	skb_reserve(skb, self->client.max_header_size);
+	skb_put(skb, 2);
+
+	frame = skb->data;
+
+	frame[0] = CMD_FILTER_OPERATION;
+	frame[1] = 0x03;                 /* Three parameters */
+	irlan_insert_byte_param(skb, "DATA_CHAN", self->dtsap_sel_data);
+	irlan_insert_string_param(skb, "FILTER_TYPE", "MULTICAST");
+	if (status)
+		irlan_insert_string_param(skb, "FILTER_MODE", "ALL");
+	else
+		irlan_insert_string_param(skb, "FILTER_MODE", "NONE");
+
+	irlan_ctrl_data_request(self, skb);
+}
+
+/*
+ * Function irlan_get_unicast_addr (self)
+ *
+ *    Retrieves the unicast address from the IrLAN provider. This address
+ *    will be inserted into the devices structure, so the ethernet layer
+ *    can construct its packets.
+ *
+ */
+static void irlan_get_unicast_addr(struct irlan_cb *self)
+{
+	struct sk_buff *skb;
+	__u8 *frame;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	skb = alloc_skb(IRLAN_MAX_HEADER + IRLAN_CMD_HEADER +
+			IRLAN_BYTE_PARAMETER_LEN("DATA_CHAN") +
+			IRLAN_STRING_PARAMETER_LEN("FILTER_TYPE", "DIRECTED") +
+			IRLAN_STRING_PARAMETER_LEN("FILTER_OPERATION",
+						   "DYNAMIC"),
+			GFP_ATOMIC);
+	if (!skb)
+		return;
+
+	/* Reserve space for TTP, LMP, and LAP header */
+	skb_reserve(skb, self->client.max_header_size);
+	skb_put(skb, 2);
+
+	frame = skb->data;
+
+	frame[0] = CMD_FILTER_OPERATION;
+	frame[1] = 0x03;                 /* Three parameters */
+	irlan_insert_byte_param(skb, "DATA_CHAN", self->dtsap_sel_data);
+	irlan_insert_string_param(skb, "FILTER_TYPE", "DIRECTED");
+	irlan_insert_string_param(skb, "FILTER_OPERATION", "DYNAMIC");
+
+	irlan_ctrl_data_request(self, skb);
+}
+
+/*
+ * Function irlan_get_media_char (self)
+ *
+ *
+ *
+ */
+void irlan_get_media_char(struct irlan_cb *self)
+{
+	struct sk_buff *skb;
+	__u8 *frame;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	skb = alloc_skb(IRLAN_MAX_HEADER + IRLAN_CMD_HEADER +
+			IRLAN_STRING_PARAMETER_LEN("MEDIA", "802.3"),
+			GFP_ATOMIC);
+
+	if (!skb)
+		return;
+
+	/* Reserve space for TTP, LMP, and LAP header */
+	skb_reserve(skb, self->client.max_header_size);
+	skb_put(skb, 2);
+
+	frame = skb->data;
+
+	/* Build frame */
+	frame[0] = CMD_GET_MEDIA_CHAR;
+	frame[1] = 0x01; /* One parameter */
+
+	irlan_insert_string_param(skb, "MEDIA", "802.3");
+	irlan_ctrl_data_request(self, skb);
+}
+
+/*
+ * Function insert_byte_param (skb, param, value)
+ *
+ *    Insert byte parameter into frame
+ *
+ */
+int irlan_insert_byte_param(struct sk_buff *skb, char *param, __u8 value)
+{
+	return __irlan_insert_param(skb, param, IRLAN_BYTE, value, 0, NULL, 0);
+}
+
+int irlan_insert_short_param(struct sk_buff *skb, char *param, __u16 value)
+{
+	return __irlan_insert_param(skb, param, IRLAN_SHORT, 0, value, NULL, 0);
+}
+
+/*
+ * Function insert_string (skb, param, value)
+ *
+ *    Insert string parameter into frame
+ *
+ */
+int irlan_insert_string_param(struct sk_buff *skb, char *param, char *string)
+{
+	int string_len = strlen(string);
+
+	return __irlan_insert_param(skb, param, IRLAN_ARRAY, 0, 0, string,
+				    string_len);
+}
+
+/*
+ * Function insert_array_param(skb, param, value, len_value)
+ *
+ *    Insert array parameter into frame
+ *
+ */
+int irlan_insert_array_param(struct sk_buff *skb, char *name, __u8 *array,
+			     __u16 array_len)
+{
+	return __irlan_insert_param(skb, name, IRLAN_ARRAY, 0, 0, array,
+				    array_len);
+}
+
+/*
+ * Function insert_param (skb, param, value, byte)
+ *
+ *    Insert parameter at end of buffer, structure of a parameter is:
+ *
+ *    -----------------------------------------------------------------------
+ *    | Name Length[1] | Param Name[1..255] | Val Length[2] | Value[0..1016]|
+ *    -----------------------------------------------------------------------
+ */
+static int __irlan_insert_param(struct sk_buff *skb, char *param, int type,
+				__u8 value_byte, __u16 value_short,
+				__u8 *value_array, __u16 value_len)
+{
+	__u8 *frame;
+	__u8 param_len;
+	__le16 tmp_le; /* Temporary value in little endian format */
+	int n=0;
+
+	if (skb == NULL) {
+		pr_debug("%s(), Got NULL skb\n", __func__);
+		return 0;
+	}
+
+	param_len = strlen(param);
+	switch (type) {
+	case IRLAN_BYTE:
+		value_len = 1;
+		break;
+	case IRLAN_SHORT:
+		value_len = 2;
+		break;
+	case IRLAN_ARRAY:
+		IRDA_ASSERT(value_array != NULL, return 0;);
+		IRDA_ASSERT(value_len > 0, return 0;);
+		break;
+	default:
+		pr_debug("%s(), Unknown parameter type!\n", __func__);
+		return 0;
+	}
+
+	/* Insert at end of sk-buffer */
+	frame = skb_tail_pointer(skb);
+
+	/* Make space for data */
+	if (skb_tailroom(skb) < (param_len+value_len+3)) {
+		pr_debug("%s(), No more space at end of skb\n", __func__);
+		return 0;
+	}
+	skb_put(skb, param_len+value_len+3);
+
+	/* Insert parameter length */
+	frame[n++] = param_len;
+
+	/* Insert parameter */
+	memcpy(frame+n, param, param_len); n += param_len;
+
+	/* Insert value length (2 byte little endian format, LSB first) */
+	tmp_le = cpu_to_le16(value_len);
+	memcpy(frame+n, &tmp_le, 2); n += 2; /* To avoid alignment problems */
+
+	/* Insert value */
+	switch (type) {
+	case IRLAN_BYTE:
+		frame[n++] = value_byte;
+		break;
+	case IRLAN_SHORT:
+		tmp_le = cpu_to_le16(value_short);
+		memcpy(frame+n, &tmp_le, 2); n += 2;
+		break;
+	case IRLAN_ARRAY:
+		memcpy(frame+n, value_array, value_len); n+=value_len;
+		break;
+	default:
+		break;
+	}
+	IRDA_ASSERT(n == (param_len+value_len+3), return 0;);
+
+	return param_len+value_len+3;
+}
+
+/*
+ * Function irlan_extract_param (buf, name, value, len)
+ *
+ *    Extracts a single parameter name/value pair from buffer and updates
+ *    the buffer pointer to point to the next name/value pair.
+ */
+int irlan_extract_param(__u8 *buf, char *name, char *value, __u16 *len)
+{
+	__u8 name_len;
+	__u16 val_len;
+	int n=0;
+
+	/* get length of parameter name (1 byte) */
+	name_len = buf[n++];
+
+	if (name_len > 254) {
+		pr_debug("%s(), name_len > 254\n", __func__);
+		return -RSP_INVALID_COMMAND_FORMAT;
+	}
+
+	/* get parameter name */
+	memcpy(name, buf+n, name_len);
+	name[name_len] = '\0';
+	n+=name_len;
+
+	/*
+	 *  Get length of parameter value (2 bytes in little endian
+	 *  format)
+	 */
+	memcpy(&val_len, buf+n, 2); /* To avoid alignment problems */
+	le16_to_cpus(&val_len); n+=2;
+
+	if (val_len >= 1016) {
+		pr_debug("%s(), parameter length to long\n", __func__);
+		return -RSP_INVALID_COMMAND_FORMAT;
+	}
+	*len = val_len;
+
+	/* get parameter value */
+	memcpy(value, buf+n, val_len);
+	value[val_len] = '\0';
+	n+=val_len;
+
+	pr_debug("Parameter: %s ", name);
+	pr_debug("Value: %s\n", value);
+
+	return n;
+}
+
+#ifdef CONFIG_PROC_FS
+
+/*
+ * Start of reading /proc entries.
+ * Return entry at pos,
+ *	or start_token to indicate print header line
+ *	or NULL if end of file
+ */
+static void *irlan_seq_start(struct seq_file *seq, loff_t *pos)
+{
+	rcu_read_lock();
+	return seq_list_start_head(&irlans, *pos);
+}
+
+/* Return entry after v, and increment pos */
+static void *irlan_seq_next(struct seq_file *seq, void *v, loff_t *pos)
+{
+	return seq_list_next(v, &irlans, pos);
+}
+
+/* End of reading /proc file */
+static void irlan_seq_stop(struct seq_file *seq, void *v)
+{
+	rcu_read_unlock();
+}
+
+
+/*
+ * Show one entry in /proc file.
+ */
+static int irlan_seq_show(struct seq_file *seq, void *v)
+{
+	if (v == &irlans)
+		seq_puts(seq, "IrLAN instances:\n");
+	else {
+		struct irlan_cb *self = list_entry(v, struct irlan_cb, dev_list);
+
+		IRDA_ASSERT(self != NULL, return -1;);
+		IRDA_ASSERT(self->magic == IRLAN_MAGIC, return -1;);
+
+		seq_printf(seq,"ifname: %s,\n",
+			       self->dev->name);
+		seq_printf(seq,"client state: %s, ",
+			       irlan_state[ self->client.state]);
+		seq_printf(seq,"provider state: %s,\n",
+			       irlan_state[ self->provider.state]);
+		seq_printf(seq,"saddr: %#08x, ",
+			       self->saddr);
+		seq_printf(seq,"daddr: %#08x\n",
+			       self->daddr);
+		seq_printf(seq,"version: %d.%d,\n",
+			       self->version[1], self->version[0]);
+		seq_printf(seq,"access type: %s\n",
+			       irlan_access[self->client.access_type]);
+		seq_printf(seq,"media: %s\n",
+			       irlan_media[self->media]);
+
+		seq_printf(seq,"local filter:\n");
+		seq_printf(seq,"remote filter: ");
+		irlan_print_filter(seq, self->client.filter_type);
+		seq_printf(seq,"tx busy: %s\n",
+			       netif_queue_stopped(self->dev) ? "TRUE" : "FALSE");
+
+		seq_putc(seq,'\n');
+	}
+	return 0;
+}
+
+static const struct seq_operations irlan_seq_ops = {
+	.start = irlan_seq_start,
+	.next  = irlan_seq_next,
+	.stop  = irlan_seq_stop,
+	.show  = irlan_seq_show,
+};
+
+static int irlan_seq_open(struct inode *inode, struct file *file)
+{
+	return seq_open(file, &irlan_seq_ops);
+}
+#endif
+
+MODULE_AUTHOR("Dag Brattli <dagb@cs.uit.no>");
+MODULE_DESCRIPTION("The Linux IrDA LAN protocol");
+MODULE_LICENSE("GPL");
+
+module_param(eth, bool, 0);
+MODULE_PARM_DESC(eth, "Name devices ethX (0) or irlanX (1)");
+module_param(access, int, 0);
+MODULE_PARM_DESC(access, "Access type DIRECT=1, PEER=2, HOSTED=3");
+
+module_init(irlan_init);
+module_exit(irlan_cleanup);
+
diff --git a/drivers/staging/irda/net/irlan/irlan_eth.c b/drivers/staging/irda/net/irlan/irlan_eth.c
new file mode 100644
index 000000000000..3be852808a9d
--- /dev/null
+++ b/drivers/staging/irda/net/irlan/irlan_eth.c
@@ -0,0 +1,340 @@
+/*********************************************************************
+ *
+ * Filename:      irlan_eth.c
+ * Version:
+ * Description:
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Thu Oct 15 08:37:58 1998
+ * Modified at:   Tue Mar 21 09:06:41 2000
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * Sources:       skeleton.c by Donald Becker <becker@CESDIS.gsfc.nasa.gov>
+ *                slip.c by Laurence Culhane,   <loz@holmes.demon.co.uk>
+ *                          Fred N. van Kempen, <waltje@uwalt.nl.mugnet.org>
+ *
+ *     Copyright (c) 1998-2000 Dag Brattli, All Rights Reserved.
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/inetdevice.h>
+#include <linux/if_arp.h>
+#include <linux/module.h>
+#include <linux/sched.h>
+#include <net/arp.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/irmod.h>
+#include <net/irda/irlan_common.h>
+#include <net/irda/irlan_client.h>
+#include <net/irda/irlan_event.h>
+#include <net/irda/irlan_eth.h>
+
+static int  irlan_eth_open(struct net_device *dev);
+static int  irlan_eth_close(struct net_device *dev);
+static netdev_tx_t  irlan_eth_xmit(struct sk_buff *skb,
+					 struct net_device *dev);
+static void irlan_eth_set_multicast_list(struct net_device *dev);
+
+static const struct net_device_ops irlan_eth_netdev_ops = {
+	.ndo_open		= irlan_eth_open,
+	.ndo_stop		= irlan_eth_close,
+	.ndo_start_xmit		= irlan_eth_xmit,
+	.ndo_set_rx_mode	= irlan_eth_set_multicast_list,
+	.ndo_validate_addr	= eth_validate_addr,
+};
+
+/*
+ * Function irlan_eth_setup (dev)
+ *
+ *    The network device initialization function.
+ *
+ */
+static void irlan_eth_setup(struct net_device *dev)
+{
+	ether_setup(dev);
+
+	dev->netdev_ops		= &irlan_eth_netdev_ops;
+	dev->needs_free_netdev	= true;
+	dev->min_mtu		= 0;
+	dev->max_mtu		= ETH_MAX_MTU;
+
+	/*
+	 * Lets do all queueing in IrTTP instead of this device driver.
+	 * Queueing here as well can introduce some strange latency
+	 * problems, which we will avoid by setting the queue size to 0.
+	 */
+	/*
+	 * The bugs in IrTTP and IrLAN that created this latency issue
+	 * have now been fixed, and we can propagate flow control properly
+	 * to the network layer. However, this requires a minimal queue of
+	 * packets for the device.
+	 * Without flow control, the Tx Queue is 14 (ttp) + 0 (dev) = 14
+	 * With flow control, the Tx Queue is 7 (ttp) + 4 (dev) = 11
+	 * See irlan_eth_flow_indication()...
+	 * Note : this number was randomly selected and would need to
+	 * be adjusted.
+	 * Jean II */
+	dev->tx_queue_len = 4;
+}
+
+/*
+ * Function alloc_irlandev
+ *
+ *    Allocate network device and control block
+ *
+ */
+struct net_device *alloc_irlandev(const char *name)
+{
+	return alloc_netdev(sizeof(struct irlan_cb), name, NET_NAME_UNKNOWN,
+			    irlan_eth_setup);
+}
+
+/*
+ * Function irlan_eth_open (dev)
+ *
+ *    Network device has been opened by user
+ *
+ */
+static int irlan_eth_open(struct net_device *dev)
+{
+	struct irlan_cb *self = netdev_priv(dev);
+
+	/* Ready to play! */
+	netif_stop_queue(dev); /* Wait until data link is ready */
+
+	/* We are now open, so time to do some work */
+	self->disconnect_reason = 0;
+	irlan_client_wakeup(self, self->saddr, self->daddr);
+
+	/* Make sure we have a hardware address before we return,
+	   so DHCP clients gets happy */
+	return wait_event_interruptible(self->open_wait,
+					!self->tsap_data->connected);
+}
+
+/*
+ * Function irlan_eth_close (dev)
+ *
+ *    Stop the ether network device, his function will usually be called by
+ *    ifconfig down. We should now disconnect the link, We start the
+ *    close timer, so that the instance will be removed if we are unable
+ *    to discover the remote device after the disconnect.
+ */
+static int irlan_eth_close(struct net_device *dev)
+{
+	struct irlan_cb *self = netdev_priv(dev);
+
+	/* Stop device */
+	netif_stop_queue(dev);
+
+	irlan_close_data_channel(self);
+	irlan_close_tsaps(self);
+
+	irlan_do_client_event(self, IRLAN_LMP_DISCONNECT, NULL);
+	irlan_do_provider_event(self, IRLAN_LMP_DISCONNECT, NULL);
+
+	/* Remove frames queued on the control channel */
+	skb_queue_purge(&self->client.txq);
+
+	self->client.tx_busy = 0;
+
+	return 0;
+}
+
+/*
+ * Function irlan_eth_tx (skb)
+ *
+ *    Transmits ethernet frames over IrDA link.
+ *
+ */
+static netdev_tx_t irlan_eth_xmit(struct sk_buff *skb,
+					struct net_device *dev)
+{
+	struct irlan_cb *self = netdev_priv(dev);
+	int ret;
+	unsigned int len;
+
+	/* skb headroom large enough to contain all IrDA-headers? */
+	if ((skb_headroom(skb) < self->max_header_size) || (skb_shared(skb))) {
+		struct sk_buff *new_skb =
+			skb_realloc_headroom(skb, self->max_header_size);
+
+		/*  We have to free the original skb anyway */
+		dev_kfree_skb(skb);
+
+		/* Did the realloc succeed? */
+		if (new_skb == NULL)
+			return NETDEV_TX_OK;
+
+		/* Use the new skb instead */
+		skb = new_skb;
+	}
+
+	netif_trans_update(dev);
+
+	len = skb->len;
+	/* Now queue the packet in the transport layer */
+	if (self->use_udata)
+		ret = irttp_udata_request(self->tsap_data, skb);
+	else
+		ret = irttp_data_request(self->tsap_data, skb);
+
+	if (ret < 0) {
+		/*
+		 * IrTTPs tx queue is full, so we just have to
+		 * drop the frame! You might think that we should
+		 * just return -1 and don't deallocate the frame,
+		 * but that is dangerous since it's possible that
+		 * we have replaced the original skb with a new
+		 * one with larger headroom, and that would really
+		 * confuse do_dev_queue_xmit() in dev.c! I have
+		 * tried :-) DB
+		 */
+		/* irttp_data_request already free the packet */
+		dev->stats.tx_dropped++;
+	} else {
+		dev->stats.tx_packets++;
+		dev->stats.tx_bytes += len;
+	}
+
+	return NETDEV_TX_OK;
+}
+
+/*
+ * Function irlan_eth_receive (handle, skb)
+ *
+ *    This function gets the data that is received on the data channel
+ *
+ */
+int irlan_eth_receive(void *instance, void *sap, struct sk_buff *skb)
+{
+	struct irlan_cb *self = instance;
+	struct net_device *dev = self->dev;
+
+	if (skb == NULL) {
+		dev->stats.rx_dropped++;
+		return 0;
+	}
+	if (skb->len < ETH_HLEN) {
+		pr_debug("%s() : IrLAN frame too short (%d)\n",
+			 __func__, skb->len);
+		dev->stats.rx_dropped++;
+		dev_kfree_skb(skb);
+		return 0;
+	}
+
+	/*
+	 * Adopt this frame! Important to set all these fields since they
+	 * might have been previously set by the low level IrDA network
+	 * device driver
+	 */
+	skb->protocol = eth_type_trans(skb, dev); /* Remove eth header */
+
+	dev->stats.rx_packets++;
+	dev->stats.rx_bytes += skb->len;
+
+	netif_rx(skb);   /* Eat it! */
+
+	return 0;
+}
+
+/*
+ * Function irlan_eth_flow (status)
+ *
+ *    Do flow control between IP/Ethernet and IrLAN/IrTTP. This is done by
+ *    controlling the queue stop/start.
+ *
+ * The IrDA link layer has the advantage to have flow control, and
+ * IrTTP now properly handles that. Flow controlling the higher layers
+ * prevent us to drop Tx packets in here (up to 15% for a TCP socket,
+ * more for UDP socket).
+ * Also, this allow us to reduce the overall transmit queue, which means
+ * less latency in case of mixed traffic.
+ * Jean II
+ */
+void irlan_eth_flow_indication(void *instance, void *sap, LOCAL_FLOW flow)
+{
+	struct irlan_cb *self;
+	struct net_device *dev;
+
+	self = instance;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	dev = self->dev;
+
+	IRDA_ASSERT(dev != NULL, return;);
+
+	pr_debug("%s() : flow %s ; running %d\n", __func__,
+		 flow == FLOW_STOP ? "FLOW_STOP" : "FLOW_START",
+		 netif_running(dev));
+
+	switch (flow) {
+	case FLOW_STOP:
+		/* IrTTP is full, stop higher layers */
+		netif_stop_queue(dev);
+		break;
+	case FLOW_START:
+	default:
+		/* Tell upper layers that its time to transmit frames again */
+		/* Schedule network layer */
+		netif_wake_queue(dev);
+		break;
+	}
+}
+
+/*
+ * Function set_multicast_list (dev)
+ *
+ *    Configure the filtering of the device
+ *
+ */
+#define HW_MAX_ADDRS 4 /* Must query to get it! */
+static void irlan_eth_set_multicast_list(struct net_device *dev)
+{
+	struct irlan_cb *self = netdev_priv(dev);
+
+	/* Check if data channel has been connected yet */
+	if (self->client.state != IRLAN_DATA) {
+		pr_debug("%s(), delaying!\n", __func__);
+		return;
+	}
+
+	if (dev->flags & IFF_PROMISC) {
+		/* Enable promiscuous mode */
+		net_warn_ratelimited("Promiscuous mode not implemented by IrLAN!\n");
+	} else if ((dev->flags & IFF_ALLMULTI) ||
+		 netdev_mc_count(dev) > HW_MAX_ADDRS) {
+		/* Disable promiscuous mode, use normal mode. */
+		pr_debug("%s(), Setting multicast filter\n", __func__);
+		/* hardware_set_filter(NULL); */
+
+		irlan_set_multicast_filter(self, TRUE);
+	} else if (!netdev_mc_empty(dev)) {
+		pr_debug("%s(), Setting multicast filter\n", __func__);
+		/* Walk the address list, and load the filter */
+		/* hardware_set_filter(dev->mc_list); */
+
+		irlan_set_multicast_filter(self, TRUE);
+	} else {
+		pr_debug("%s(), Clearing multicast filter\n", __func__);
+		irlan_set_multicast_filter(self, FALSE);
+	}
+
+	if (dev->flags & IFF_BROADCAST)
+		irlan_set_broadcast_filter(self, TRUE);
+	else
+		irlan_set_broadcast_filter(self, FALSE);
+}
diff --git a/drivers/staging/irda/net/irlan/irlan_event.c b/drivers/staging/irda/net/irlan/irlan_event.c
new file mode 100644
index 000000000000..9a1cc11c16f6
--- /dev/null
+++ b/drivers/staging/irda/net/irlan/irlan_event.c
@@ -0,0 +1,60 @@
+/*********************************************************************
+ *
+ * Filename:      irlan_event.c
+ * Version:
+ * Description:
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Tue Oct 20 09:10:16 1998
+ * Modified at:   Sat Oct 30 12:59:01 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1998-1999 Dag Brattli, All Rights Reserved.
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#include <net/irda/irlan_event.h>
+
+const char * const irlan_state[] = {
+	"IRLAN_IDLE",
+	"IRLAN_QUERY",
+	"IRLAN_CONN",
+	"IRLAN_INFO",
+	"IRLAN_MEDIA",
+	"IRLAN_OPEN",
+	"IRLAN_WAIT",
+	"IRLAN_ARB",
+	"IRLAN_DATA",
+	"IRLAN_CLOSE",
+	"IRLAN_SYNC",
+};
+
+void irlan_next_client_state(struct irlan_cb *self, IRLAN_STATE state)
+{
+	pr_debug("%s(), %s\n", __func__ , irlan_state[state]);
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	self->client.state = state;
+}
+
+void irlan_next_provider_state(struct irlan_cb *self, IRLAN_STATE state)
+{
+	pr_debug("%s(), %s\n", __func__ , irlan_state[state]);
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	self->provider.state = state;
+}
+
diff --git a/drivers/staging/irda/net/irlan/irlan_filter.c b/drivers/staging/irda/net/irlan/irlan_filter.c
new file mode 100644
index 000000000000..e755e90b2f26
--- /dev/null
+++ b/drivers/staging/irda/net/irlan/irlan_filter.c
@@ -0,0 +1,240 @@
+/*********************************************************************
+ *
+ * Filename:      irlan_filter.c
+ * Version:
+ * Description:
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Fri Jan 29 11:16:38 1999
+ * Modified at:   Sat Oct 30 12:58:45 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1998-1999 Dag Brattli, All Rights Reserved.
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#include <linux/skbuff.h>
+#include <linux/random.h>
+#include <linux/seq_file.h>
+
+#include <net/irda/irlan_common.h>
+#include <net/irda/irlan_filter.h>
+
+/*
+ * Function irlan_filter_request (self, skb)
+ *
+ *    Handle filter request from client peer device
+ *
+ */
+void irlan_filter_request(struct irlan_cb *self, struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	if ((self->provider.filter_type == IRLAN_DIRECTED) &&
+	    (self->provider.filter_operation == DYNAMIC))
+	{
+		pr_debug("Giving peer a dynamic Ethernet address\n");
+		self->provider.mac_address[0] = 0x40;
+		self->provider.mac_address[1] = 0x00;
+		self->provider.mac_address[2] = 0x00;
+		self->provider.mac_address[3] = 0x00;
+
+		/* Use arbitration value to generate MAC address */
+		if (self->provider.access_type == ACCESS_PEER) {
+			self->provider.mac_address[4] =
+				self->provider.send_arb_val & 0xff;
+			self->provider.mac_address[5] =
+				(self->provider.send_arb_val >> 8) & 0xff;
+		} else {
+			/* Just generate something for now */
+			get_random_bytes(self->provider.mac_address+4, 1);
+			get_random_bytes(self->provider.mac_address+5, 1);
+		}
+
+		skb->data[0] = 0x00; /* Success */
+		skb->data[1] = 0x03;
+		irlan_insert_string_param(skb, "FILTER_MODE", "NONE");
+		irlan_insert_short_param(skb, "MAX_ENTRY", 0x0001);
+		irlan_insert_array_param(skb, "FILTER_ENTRY",
+					 self->provider.mac_address, 6);
+		return;
+	}
+
+	if ((self->provider.filter_type == IRLAN_DIRECTED) &&
+	    (self->provider.filter_mode == FILTER))
+	{
+		pr_debug("Directed filter on\n");
+		skb->data[0] = 0x00; /* Success */
+		skb->data[1] = 0x00;
+		return;
+	}
+	if ((self->provider.filter_type == IRLAN_DIRECTED) &&
+	    (self->provider.filter_mode == NONE))
+	{
+		pr_debug("Directed filter off\n");
+		skb->data[0] = 0x00; /* Success */
+		skb->data[1] = 0x00;
+		return;
+	}
+
+	if ((self->provider.filter_type == IRLAN_BROADCAST) &&
+	    (self->provider.filter_mode == FILTER))
+	{
+		pr_debug("Broadcast filter on\n");
+		skb->data[0] = 0x00; /* Success */
+		skb->data[1] = 0x00;
+		return;
+	}
+	if ((self->provider.filter_type == IRLAN_BROADCAST) &&
+	    (self->provider.filter_mode == NONE))
+	{
+		pr_debug("Broadcast filter off\n");
+		skb->data[0] = 0x00; /* Success */
+		skb->data[1] = 0x00;
+		return;
+	}
+	if ((self->provider.filter_type == IRLAN_MULTICAST) &&
+	    (self->provider.filter_mode == FILTER))
+	{
+		pr_debug("Multicast filter on\n");
+		skb->data[0] = 0x00; /* Success */
+		skb->data[1] = 0x00;
+		return;
+	}
+	if ((self->provider.filter_type == IRLAN_MULTICAST) &&
+	    (self->provider.filter_mode == NONE))
+	{
+		pr_debug("Multicast filter off\n");
+		skb->data[0] = 0x00; /* Success */
+		skb->data[1] = 0x00;
+		return;
+	}
+	if ((self->provider.filter_type == IRLAN_MULTICAST) &&
+	    (self->provider.filter_operation == GET))
+	{
+		pr_debug("Multicast filter get\n");
+		skb->data[0] = 0x00; /* Success? */
+		skb->data[1] = 0x02;
+		irlan_insert_string_param(skb, "FILTER_MODE", "NONE");
+		irlan_insert_short_param(skb, "MAX_ENTRY", 16);
+		return;
+	}
+	skb->data[0] = 0x00; /* Command not supported */
+	skb->data[1] = 0x00;
+
+	pr_debug("Not implemented!\n");
+}
+
+/*
+ * Function check_request_param (self, param, value)
+ *
+ *    Check parameters in request from peer device
+ *
+ */
+void irlan_check_command_param(struct irlan_cb *self, char *param, char *value)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	pr_debug("%s, %s\n", param, value);
+
+	/*
+	 *  This is experimental!! DB.
+	 */
+	 if (strcmp(param, "MODE") == 0) {
+		self->use_udata = TRUE;
+		return;
+	}
+
+	/*
+	 *  FILTER_TYPE
+	 */
+	if (strcmp(param, "FILTER_TYPE") == 0) {
+		if (strcmp(value, "DIRECTED") == 0) {
+			self->provider.filter_type = IRLAN_DIRECTED;
+			return;
+		}
+		if (strcmp(value, "MULTICAST") == 0) {
+			self->provider.filter_type = IRLAN_MULTICAST;
+			return;
+		}
+		if (strcmp(value, "BROADCAST") == 0) {
+			self->provider.filter_type = IRLAN_BROADCAST;
+			return;
+		}
+	}
+	/*
+	 *  FILTER_MODE
+	 */
+	if (strcmp(param, "FILTER_MODE") == 0) {
+		if (strcmp(value, "ALL") == 0) {
+			self->provider.filter_mode = ALL;
+			return;
+		}
+		if (strcmp(value, "FILTER") == 0) {
+			self->provider.filter_mode = FILTER;
+			return;
+		}
+		if (strcmp(value, "NONE") == 0) {
+			self->provider.filter_mode = FILTER;
+			return;
+		}
+	}
+	/*
+	 *  FILTER_OPERATION
+	 */
+	if (strcmp(param, "FILTER_OPERATION") == 0) {
+		if (strcmp(value, "DYNAMIC") == 0) {
+			self->provider.filter_operation = DYNAMIC;
+			return;
+		}
+		if (strcmp(value, "GET") == 0) {
+			self->provider.filter_operation = GET;
+			return;
+		}
+	}
+}
+
+/*
+ * Function irlan_print_filter (filter_type, buf)
+ *
+ *    Print status of filter. Used by /proc file system
+ *
+ */
+#ifdef CONFIG_PROC_FS
+#define MASK2STR(m,s)	{ .mask = m, .str = s }
+
+void irlan_print_filter(struct seq_file *seq, int filter_type)
+{
+	static struct {
+		int mask;
+		const char *str;
+	} filter_mask2str[] = {
+		MASK2STR(IRLAN_DIRECTED,	"DIRECTED"),
+		MASK2STR(IRLAN_FUNCTIONAL,	"FUNCTIONAL"),
+		MASK2STR(IRLAN_GROUP,		"GROUP"),
+		MASK2STR(IRLAN_MAC_FRAME,	"MAC_FRAME"),
+		MASK2STR(IRLAN_MULTICAST,	"MULTICAST"),
+		MASK2STR(IRLAN_BROADCAST,	"BROADCAST"),
+		MASK2STR(IRLAN_IPX_SOCKET,	"IPX_SOCKET"),
+		MASK2STR(0,			NULL)
+	}, *p;
+
+	for (p = filter_mask2str; p->str; p++) {
+		if (filter_type & p->mask)
+			seq_printf(seq, "%s ", p->str);
+	}
+	seq_putc(seq, '\n');
+}
+#undef MASK2STR
+#endif
diff --git a/drivers/staging/irda/net/irlan/irlan_provider.c b/drivers/staging/irda/net/irlan/irlan_provider.c
new file mode 100644
index 000000000000..15c292cf2644
--- /dev/null
+++ b/drivers/staging/irda/net/irlan/irlan_provider.c
@@ -0,0 +1,408 @@
+/*********************************************************************
+ *
+ * Filename:      irlan_provider.c
+ * Version:       0.9
+ * Description:   IrDA LAN Access Protocol Implementation
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sun Aug 31 20:14:37 1997
+ * Modified at:   Sat Oct 30 12:52:10 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * Sources:       skeleton.c by Donald Becker <becker@CESDIS.gsfc.nasa.gov>
+ *                slip.c by Laurence Culhane,   <loz@holmes.demon.co.uk>
+ *                          Fred N. van Kempen, <waltje@uwalt.nl.mugnet.org>
+ *
+ *     Copyright (c) 1998-1999 Dag Brattli <dagb@cs.uit.no>,
+ *     All Rights Reserved.
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#include <linux/kernel.h>
+#include <linux/string.h>
+#include <linux/errno.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/init.h>
+#include <linux/random.h>
+#include <linux/bitops.h>
+#include <linux/slab.h>
+
+#include <asm/byteorder.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/irttp.h>
+#include <net/irda/irlmp.h>
+#include <net/irda/irias_object.h>
+#include <net/irda/iriap.h>
+#include <net/irda/timer.h>
+
+#include <net/irda/irlan_common.h>
+#include <net/irda/irlan_eth.h>
+#include <net/irda/irlan_event.h>
+#include <net/irda/irlan_provider.h>
+#include <net/irda/irlan_filter.h>
+#include <net/irda/irlan_client.h>
+
+static void irlan_provider_connect_indication(void *instance, void *sap,
+					      struct qos_info *qos,
+					      __u32 max_sdu_size,
+					      __u8 max_header_size,
+					      struct sk_buff *skb);
+
+/*
+ * Function irlan_provider_control_data_indication (handle, skb)
+ *
+ *    This function gets the data that is received on the control channel
+ *
+ */
+static int irlan_provider_data_indication(void *instance, void *sap,
+					  struct sk_buff *skb)
+{
+	struct irlan_cb *self;
+	__u8 code;
+
+	self = instance;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return -1;);
+
+	IRDA_ASSERT(skb != NULL, return -1;);
+
+	code = skb->data[0];
+	switch(code) {
+	case CMD_GET_PROVIDER_INFO:
+		pr_debug("Got GET_PROVIDER_INFO command!\n");
+		irlan_do_provider_event(self, IRLAN_GET_INFO_CMD, skb);
+		break;
+
+	case CMD_GET_MEDIA_CHAR:
+		pr_debug("Got GET_MEDIA_CHAR command!\n");
+		irlan_do_provider_event(self, IRLAN_GET_MEDIA_CMD, skb);
+		break;
+	case CMD_OPEN_DATA_CHANNEL:
+		pr_debug("Got OPEN_DATA_CHANNEL command!\n");
+		irlan_do_provider_event(self, IRLAN_OPEN_DATA_CMD, skb);
+		break;
+	case CMD_FILTER_OPERATION:
+		pr_debug("Got FILTER_OPERATION command!\n");
+		irlan_do_provider_event(self, IRLAN_FILTER_CONFIG_CMD, skb);
+		break;
+	case CMD_RECONNECT_DATA_CHAN:
+		pr_debug("%s(), Got RECONNECT_DATA_CHAN command\n", __func__);
+		pr_debug("%s(), NOT IMPLEMENTED\n", __func__);
+		break;
+	case CMD_CLOSE_DATA_CHAN:
+		pr_debug("Got CLOSE_DATA_CHAN command!\n");
+		pr_debug("%s(), NOT IMPLEMENTED\n", __func__);
+		break;
+	default:
+		pr_debug("%s(), Unknown command!\n", __func__);
+		break;
+	}
+	return 0;
+}
+
+/*
+ * Function irlan_provider_connect_indication (handle, skb, priv)
+ *
+ *    Got connection from peer IrLAN client
+ *
+ */
+static void irlan_provider_connect_indication(void *instance, void *sap,
+					      struct qos_info *qos,
+					      __u32 max_sdu_size,
+					      __u8 max_header_size,
+					      struct sk_buff *skb)
+{
+	struct irlan_cb *self;
+	struct tsap_cb *tsap;
+
+	self = instance;
+	tsap = sap;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	IRDA_ASSERT(tsap == self->provider.tsap_ctrl,return;);
+	IRDA_ASSERT(self->provider.state == IRLAN_IDLE, return;);
+
+	self->provider.max_sdu_size = max_sdu_size;
+	self->provider.max_header_size = max_header_size;
+
+	irlan_do_provider_event(self, IRLAN_CONNECT_INDICATION, NULL);
+
+	/*
+	 * If we are in peer mode, the client may not have got the discovery
+	 * indication it needs to make progress. If the client is still in
+	 * IDLE state, we must kick it.
+	 */
+	if ((self->provider.access_type == ACCESS_PEER) &&
+	    (self->client.state == IRLAN_IDLE))
+	{
+		irlan_client_wakeup(self, self->saddr, self->daddr);
+	}
+}
+
+/*
+ * Function irlan_provider_connect_response (handle)
+ *
+ *    Accept incoming connection
+ *
+ */
+void irlan_provider_connect_response(struct irlan_cb *self,
+				     struct tsap_cb *tsap)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	/* Just accept */
+	irttp_connect_response(tsap, IRLAN_MTU, NULL);
+}
+
+static void irlan_provider_disconnect_indication(void *instance, void *sap,
+						 LM_REASON reason,
+						 struct sk_buff *userdata)
+{
+	struct irlan_cb *self;
+	struct tsap_cb *tsap;
+
+	pr_debug("%s(), reason=%d\n", __func__ , reason);
+
+	self = instance;
+	tsap = sap;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+	IRDA_ASSERT(tsap != NULL, return;);
+	IRDA_ASSERT(tsap->magic == TTP_TSAP_MAGIC, return;);
+
+	IRDA_ASSERT(tsap == self->provider.tsap_ctrl, return;);
+
+	irlan_do_provider_event(self, IRLAN_LMP_DISCONNECT, NULL);
+}
+
+/*
+ * Function irlan_parse_open_data_cmd (self, skb)
+ *
+ *
+ *
+ */
+int irlan_parse_open_data_cmd(struct irlan_cb *self, struct sk_buff *skb)
+{
+	int ret;
+
+	ret = irlan_provider_parse_command(self, CMD_OPEN_DATA_CHANNEL, skb);
+
+	/* Open data channel */
+	irlan_open_data_tsap(self);
+
+	return ret;
+}
+
+/*
+ * Function parse_command (skb)
+ *
+ *    Extract all parameters from received buffer, then feed them to
+ *    check_params for parsing
+ *
+ */
+int irlan_provider_parse_command(struct irlan_cb *self, int cmd,
+				 struct sk_buff *skb)
+{
+	__u8 *frame;
+	__u8 *ptr;
+	int count;
+	__u16 val_len;
+	int i;
+	char *name;
+	char *value;
+	int ret = RSP_SUCCESS;
+
+	IRDA_ASSERT(skb != NULL, return -RSP_PROTOCOL_ERROR;);
+
+	pr_debug("%s(), skb->len=%d\n", __func__ , (int)skb->len);
+
+	IRDA_ASSERT(self != NULL, return -RSP_PROTOCOL_ERROR;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return -RSP_PROTOCOL_ERROR;);
+
+	if (!skb)
+		return -RSP_PROTOCOL_ERROR;
+
+	frame = skb->data;
+
+	name = kmalloc(255, GFP_ATOMIC);
+	if (!name)
+		return -RSP_INSUFFICIENT_RESOURCES;
+	value = kmalloc(1016, GFP_ATOMIC);
+	if (!value) {
+		kfree(name);
+		return -RSP_INSUFFICIENT_RESOURCES;
+	}
+
+	/* How many parameters? */
+	count = frame[1];
+
+	pr_debug("Got %d parameters\n", count);
+
+	ptr = frame+2;
+
+	/* For all parameters */
+	for (i=0; i<count;i++) {
+		ret = irlan_extract_param(ptr, name, value, &val_len);
+		if (ret < 0) {
+			pr_debug("%s(), IrLAN, Error!\n", __func__);
+			break;
+		}
+		ptr+=ret;
+		ret = RSP_SUCCESS;
+		irlan_check_command_param(self, name, value);
+	}
+	/* Cleanup */
+	kfree(name);
+	kfree(value);
+
+	return ret;
+}
+
+/*
+ * Function irlan_provider_send_reply (self, info)
+ *
+ *    Send reply to query to peer IrLAN layer
+ *
+ */
+void irlan_provider_send_reply(struct irlan_cb *self, int command,
+			       int ret_code)
+{
+	struct sk_buff *skb;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return;);
+
+	skb = alloc_skb(IRLAN_MAX_HEADER + IRLAN_CMD_HEADER +
+			/* Bigger param length comes from CMD_GET_MEDIA_CHAR */
+			IRLAN_STRING_PARAMETER_LEN("FILTER_TYPE", "DIRECTED") +
+			IRLAN_STRING_PARAMETER_LEN("FILTER_TYPE", "BROADCAST") +
+			IRLAN_STRING_PARAMETER_LEN("FILTER_TYPE", "MULTICAST") +
+			IRLAN_STRING_PARAMETER_LEN("ACCESS_TYPE", "HOSTED"),
+			GFP_ATOMIC);
+
+	if (!skb)
+		return;
+
+	/* Reserve space for TTP, LMP, and LAP header */
+	skb_reserve(skb, self->provider.max_header_size);
+	skb_put(skb, 2);
+
+	switch (command) {
+	case CMD_GET_PROVIDER_INFO:
+		skb->data[0] = 0x00; /* Success */
+		skb->data[1] = 0x02; /* 2 parameters */
+		switch (self->media) {
+		case MEDIA_802_3:
+			irlan_insert_string_param(skb, "MEDIA", "802.3");
+			break;
+		case MEDIA_802_5:
+			irlan_insert_string_param(skb, "MEDIA", "802.5");
+			break;
+		default:
+			pr_debug("%s(), unknown media type!\n", __func__);
+			break;
+		}
+		irlan_insert_short_param(skb, "IRLAN_VER", 0x0101);
+		break;
+
+	case CMD_GET_MEDIA_CHAR:
+		skb->data[0] = 0x00; /* Success */
+		skb->data[1] = 0x05; /* 5 parameters */
+		irlan_insert_string_param(skb, "FILTER_TYPE", "DIRECTED");
+		irlan_insert_string_param(skb, "FILTER_TYPE", "BROADCAST");
+		irlan_insert_string_param(skb, "FILTER_TYPE", "MULTICAST");
+
+		switch (self->provider.access_type) {
+		case ACCESS_DIRECT:
+			irlan_insert_string_param(skb, "ACCESS_TYPE", "DIRECT");
+			break;
+		case ACCESS_PEER:
+			irlan_insert_string_param(skb, "ACCESS_TYPE", "PEER");
+			break;
+		case ACCESS_HOSTED:
+			irlan_insert_string_param(skb, "ACCESS_TYPE", "HOSTED");
+			break;
+		default:
+			pr_debug("%s(), Unknown access type\n", __func__);
+			break;
+		}
+		irlan_insert_short_param(skb, "MAX_FRAME", 0x05ee);
+		break;
+	case CMD_OPEN_DATA_CHANNEL:
+		skb->data[0] = 0x00; /* Success */
+		if (self->provider.send_arb_val) {
+			skb->data[1] = 0x03; /* 3 parameters */
+			irlan_insert_short_param(skb, "CON_ARB",
+						 self->provider.send_arb_val);
+		} else
+			skb->data[1] = 0x02; /* 2 parameters */
+		irlan_insert_byte_param(skb, "DATA_CHAN", self->stsap_sel_data);
+		irlan_insert_string_param(skb, "RECONNECT_KEY", "LINUX RULES!");
+		break;
+	case CMD_FILTER_OPERATION:
+		irlan_filter_request(self, skb);
+		break;
+	default:
+		pr_debug("%s(), Unknown command!\n", __func__);
+		break;
+	}
+
+	irttp_data_request(self->provider.tsap_ctrl, skb);
+}
+
+/*
+ * Function irlan_provider_register(void)
+ *
+ *    Register provider support so we can accept incoming connections.
+ *
+ */
+int irlan_provider_open_ctrl_tsap(struct irlan_cb *self)
+{
+	struct tsap_cb *tsap;
+	notify_t notify;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return -1;);
+
+	/* Check if already open */
+	if (self->provider.tsap_ctrl)
+		return -1;
+
+	/*
+	 *  First register well known control TSAP
+	 */
+	irda_notify_init(&notify);
+	notify.data_indication       = irlan_provider_data_indication;
+	notify.connect_indication    = irlan_provider_connect_indication;
+	notify.disconnect_indication = irlan_provider_disconnect_indication;
+	notify.instance = self;
+	strlcpy(notify.name, "IrLAN ctrl (p)", sizeof(notify.name));
+
+	tsap = irttp_open_tsap(LSAP_ANY, 1, &notify);
+	if (!tsap) {
+		pr_debug("%s(), Got no tsap!\n", __func__);
+		return -1;
+	}
+	self->provider.tsap_ctrl = tsap;
+
+	/* Register with LM-IAS */
+	irlan_ias_register(self, tsap->stsap_sel);
+
+	return 0;
+}
+
diff --git a/drivers/staging/irda/net/irlan/irlan_provider_event.c b/drivers/staging/irda/net/irlan/irlan_provider_event.c
new file mode 100644
index 000000000000..9c4f7f51d6b5
--- /dev/null
+++ b/drivers/staging/irda/net/irlan/irlan_provider_event.c
@@ -0,0 +1,233 @@
+/*********************************************************************
+ *
+ * Filename:      irlan_provider_event.c
+ * Version:       0.9
+ * Description:   IrLAN provider state machine)
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sun Aug 31 20:14:37 1997
+ * Modified at:   Sat Oct 30 12:52:41 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1998-1999 Dag Brattli <dagb@cs.uit.no>, All Rights Reserved.
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#include <net/irda/irda.h>
+#include <net/irda/iriap.h>
+#include <net/irda/irlmp.h>
+#include <net/irda/irttp.h>
+
+#include <net/irda/irlan_provider.h>
+#include <net/irda/irlan_event.h>
+
+static int irlan_provider_state_idle(struct irlan_cb *self, IRLAN_EVENT event,
+				     struct sk_buff *skb);
+static int irlan_provider_state_info(struct irlan_cb *self, IRLAN_EVENT event,
+				     struct sk_buff *skb);
+static int irlan_provider_state_open(struct irlan_cb *self, IRLAN_EVENT event,
+				     struct sk_buff *skb);
+static int irlan_provider_state_data(struct irlan_cb *self, IRLAN_EVENT event,
+				     struct sk_buff *skb);
+
+static int (*state[])(struct irlan_cb *self, IRLAN_EVENT event,
+		      struct sk_buff *skb) =
+{
+	irlan_provider_state_idle,
+	NULL, /* Query */
+	NULL, /* Info */
+	irlan_provider_state_info,
+	NULL, /* Media */
+	irlan_provider_state_open,
+	NULL, /* Wait */
+	NULL, /* Arb */
+	irlan_provider_state_data,
+	NULL, /* Close */
+	NULL, /* Sync */
+};
+
+void irlan_do_provider_event(struct irlan_cb *self, IRLAN_EVENT event,
+			     struct sk_buff *skb)
+{
+	IRDA_ASSERT(*state[ self->provider.state] != NULL, return;);
+
+	(*state[self->provider.state]) (self, event, skb);
+}
+
+/*
+ * Function irlan_provider_state_idle (event, skb, info)
+ *
+ *    IDLE, We are waiting for an indication that there is a provider
+ *    available.
+ */
+static int irlan_provider_state_idle(struct irlan_cb *self, IRLAN_EVENT event,
+				     struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return -1;);
+
+	switch(event) {
+	case IRLAN_CONNECT_INDICATION:
+	     irlan_provider_connect_response( self, self->provider.tsap_ctrl);
+	     irlan_next_provider_state( self, IRLAN_INFO);
+	     break;
+	default:
+		pr_debug("%s(), Unknown event %d\n", __func__ , event);
+		break;
+	}
+	if (skb)
+		dev_kfree_skb(skb);
+
+	return 0;
+}
+
+/*
+ * Function irlan_provider_state_info (self, event, skb, info)
+ *
+ *    INFO, We have issued a GetInfo command and is awaiting a reply.
+ */
+static int irlan_provider_state_info(struct irlan_cb *self, IRLAN_EVENT event,
+				     struct sk_buff *skb)
+{
+	int ret;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+
+	switch(event) {
+	case IRLAN_GET_INFO_CMD:
+		/* Be sure to use 802.3 in case of peer mode */
+		if (self->provider.access_type == ACCESS_PEER) {
+			self->media = MEDIA_802_3;
+
+			/* Check if client has started yet */
+			if (self->client.state == IRLAN_IDLE) {
+				/* This should get the client going */
+				irlmp_discovery_request(8);
+			}
+		}
+
+		irlan_provider_send_reply(self, CMD_GET_PROVIDER_INFO,
+					  RSP_SUCCESS);
+		/* Keep state */
+		break;
+	case IRLAN_GET_MEDIA_CMD:
+		irlan_provider_send_reply(self, CMD_GET_MEDIA_CHAR,
+					  RSP_SUCCESS);
+		/* Keep state */
+		break;
+	case IRLAN_OPEN_DATA_CMD:
+		ret = irlan_parse_open_data_cmd(self, skb);
+		if (self->provider.access_type == ACCESS_PEER) {
+			/* FIXME: make use of random functions! */
+			self->provider.send_arb_val = (jiffies & 0xffff);
+		}
+		irlan_provider_send_reply(self, CMD_OPEN_DATA_CHANNEL, ret);
+
+		if (ret == RSP_SUCCESS) {
+			irlan_next_provider_state(self, IRLAN_OPEN);
+
+			/* Signal client that we are now open */
+			irlan_do_client_event(self, IRLAN_PROVIDER_SIGNAL, NULL);
+		}
+		break;
+	case IRLAN_LMP_DISCONNECT:  /* FALLTHROUGH */
+	case IRLAN_LAP_DISCONNECT:
+		irlan_next_provider_state(self, IRLAN_IDLE);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %d\n", __func__ , event);
+		break;
+	}
+	if (skb)
+		dev_kfree_skb(skb);
+
+	return 0;
+}
+
+/*
+ * Function irlan_provider_state_open (self, event, skb, info)
+ *
+ *    OPEN, The client has issued a OpenData command and is awaiting a
+ *    reply
+ *
+ */
+static int irlan_provider_state_open(struct irlan_cb *self, IRLAN_EVENT event,
+				     struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return -1;);
+
+	switch(event) {
+	case IRLAN_FILTER_CONFIG_CMD:
+		irlan_provider_parse_command(self, CMD_FILTER_OPERATION, skb);
+		irlan_provider_send_reply(self, CMD_FILTER_OPERATION,
+					  RSP_SUCCESS);
+		/* Keep state */
+		break;
+	case IRLAN_DATA_CONNECT_INDICATION:
+		irlan_next_provider_state(self, IRLAN_DATA);
+		irlan_provider_connect_response(self, self->tsap_data);
+		break;
+	case IRLAN_LMP_DISCONNECT:  /* FALLTHROUGH */
+	case IRLAN_LAP_DISCONNECT:
+		irlan_next_provider_state(self, IRLAN_IDLE);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %d\n", __func__ , event);
+		break;
+	}
+	if (skb)
+		dev_kfree_skb(skb);
+
+	return 0;
+}
+
+/*
+ * Function irlan_provider_state_data (self, event, skb, info)
+ *
+ *    DATA, The data channel is connected, allowing data transfers between
+ *    the local and remote machines.
+ *
+ */
+static int irlan_provider_state_data(struct irlan_cb *self, IRLAN_EVENT event,
+				     struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == IRLAN_MAGIC, return -1;);
+
+	switch(event) {
+	case IRLAN_FILTER_CONFIG_CMD:
+		irlan_provider_parse_command(self, CMD_FILTER_OPERATION, skb);
+		irlan_provider_send_reply(self, CMD_FILTER_OPERATION,
+					  RSP_SUCCESS);
+		break;
+	case IRLAN_LMP_DISCONNECT: /* FALLTHROUGH */
+	case IRLAN_LAP_DISCONNECT:
+		irlan_next_provider_state(self, IRLAN_IDLE);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %d\n", __func__ , event);
+		break;
+	}
+	if (skb)
+		dev_kfree_skb(skb);
+
+	return 0;
+}
+
+
+
+
+
+
+
+
+
+
diff --git a/drivers/staging/irda/net/irlap.c b/drivers/staging/irda/net/irlap.c
new file mode 100644
index 000000000000..1cde711bcab5
--- /dev/null
+++ b/drivers/staging/irda/net/irlap.c
@@ -0,0 +1,1207 @@
+/*********************************************************************
+ *
+ * Filename:      irlap.c
+ * Version:       1.0
+ * Description:   IrLAP implementation for Linux
+ * Status:        Stable
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Mon Aug  4 20:40:53 1997
+ * Modified at:   Tue Dec 14 09:26:44 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1998-1999 Dag Brattli, All Rights Reserved.
+ *     Copyright (c) 2000-2003 Jean Tourrilhes <jt@hpl.hp.com>
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ *
+ *     You should have received a copy of the GNU General Public License
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ ********************************************************************/
+
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/skbuff.h>
+#include <linux/delay.h>
+#include <linux/proc_fs.h>
+#include <linux/init.h>
+#include <linux/random.h>
+#include <linux/module.h>
+#include <linux/seq_file.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/irda_device.h>
+#include <net/irda/irqueue.h>
+#include <net/irda/irlmp.h>
+#include <net/irda/irlmp_frame.h>
+#include <net/irda/irlap_frame.h>
+#include <net/irda/irlap.h>
+#include <net/irda/timer.h>
+#include <net/irda/qos.h>
+
+static hashbin_t *irlap = NULL;
+int sysctl_slot_timeout = SLOT_TIMEOUT * 1000 / HZ;
+
+/* This is the delay of missed pf period before generating an event
+ * to the application. The spec mandate 3 seconds, but in some cases
+ * it's way too long. - Jean II */
+int sysctl_warn_noreply_time = 3;
+
+extern void irlap_queue_xmit(struct irlap_cb *self, struct sk_buff *skb);
+static void __irlap_close(struct irlap_cb *self);
+static void irlap_init_qos_capabilities(struct irlap_cb *self,
+					struct qos_info *qos_user);
+
+static const char *const lap_reasons[] __maybe_unused = {
+	"ERROR, NOT USED",
+	"LAP_DISC_INDICATION",
+	"LAP_NO_RESPONSE",
+	"LAP_RESET_INDICATION",
+	"LAP_FOUND_NONE",
+	"LAP_MEDIA_BUSY",
+	"LAP_PRIMARY_CONFLICT",
+	"ERROR, NOT USED",
+};
+
+int __init irlap_init(void)
+{
+	/* Check if the compiler did its job properly.
+	 * May happen on some ARM configuration, check with Russell King. */
+	IRDA_ASSERT(sizeof(struct xid_frame) == 14, ;);
+	IRDA_ASSERT(sizeof(struct test_frame) == 10, ;);
+	IRDA_ASSERT(sizeof(struct ua_frame) == 10, ;);
+	IRDA_ASSERT(sizeof(struct snrm_frame) == 11, ;);
+
+	/* Allocate master array */
+	irlap = hashbin_new(HB_LOCK);
+	if (irlap == NULL) {
+		net_err_ratelimited("%s: can't allocate irlap hashbin!\n",
+				    __func__);
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+void irlap_cleanup(void)
+{
+	IRDA_ASSERT(irlap != NULL, return;);
+
+	hashbin_delete(irlap, (FREE_FUNC) __irlap_close);
+}
+
+/*
+ * Function irlap_open (driver)
+ *
+ *    Initialize IrLAP layer
+ *
+ */
+struct irlap_cb *irlap_open(struct net_device *dev, struct qos_info *qos,
+			    const char *hw_name)
+{
+	struct irlap_cb *self;
+
+	/* Initialize the irlap structure. */
+	self = kzalloc(sizeof(struct irlap_cb), GFP_KERNEL);
+	if (self == NULL)
+		return NULL;
+
+	self->magic = LAP_MAGIC;
+
+	/* Make a binding between the layers */
+	self->netdev = dev;
+	self->qos_dev = qos;
+	/* Copy hardware name */
+	if(hw_name != NULL) {
+		strlcpy(self->hw_name, hw_name, sizeof(self->hw_name));
+	} else {
+		self->hw_name[0] = '\0';
+	}
+
+	/* FIXME: should we get our own field? */
+	dev->atalk_ptr = self;
+
+	self->state = LAP_OFFLINE;
+
+	/* Initialize transmit queue */
+	skb_queue_head_init(&self->txq);
+	skb_queue_head_init(&self->txq_ultra);
+	skb_queue_head_init(&self->wx_list);
+
+	/* My unique IrLAP device address! */
+	/* We don't want the broadcast address, neither the NULL address
+	 * (most often used to signify "invalid"), and we don't want an
+	 * address already in use (otherwise connect won't be able
+	 * to select the proper link). - Jean II */
+	do {
+		get_random_bytes(&self->saddr, sizeof(self->saddr));
+	} while ((self->saddr == 0x0) || (self->saddr == BROADCAST) ||
+		 (hashbin_lock_find(irlap, self->saddr, NULL)) );
+	/* Copy to the driver */
+	memcpy(dev->dev_addr, &self->saddr, 4);
+
+	init_timer(&self->slot_timer);
+	init_timer(&self->query_timer);
+	init_timer(&self->discovery_timer);
+	init_timer(&self->final_timer);
+	init_timer(&self->poll_timer);
+	init_timer(&self->wd_timer);
+	init_timer(&self->backoff_timer);
+	init_timer(&self->media_busy_timer);
+
+	irlap_apply_default_connection_parameters(self);
+
+	self->N3 = 3; /* # connections attempts to try before giving up */
+
+	self->state = LAP_NDM;
+
+	hashbin_insert(irlap, (irda_queue_t *) self, self->saddr, NULL);
+
+	irlmp_register_link(self, self->saddr, &self->notify);
+
+	return self;
+}
+EXPORT_SYMBOL(irlap_open);
+
+/*
+ * Function __irlap_close (self)
+ *
+ *    Remove IrLAP and all allocated memory. Stop any pending timers.
+ *
+ */
+static void __irlap_close(struct irlap_cb *self)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	/* Stop timers */
+	del_timer(&self->slot_timer);
+	del_timer(&self->query_timer);
+	del_timer(&self->discovery_timer);
+	del_timer(&self->final_timer);
+	del_timer(&self->poll_timer);
+	del_timer(&self->wd_timer);
+	del_timer(&self->backoff_timer);
+	del_timer(&self->media_busy_timer);
+
+	irlap_flush_all_queues(self);
+
+	self->magic = 0;
+
+	kfree(self);
+}
+
+/*
+ * Function irlap_close (self)
+ *
+ *    Remove IrLAP instance
+ *
+ */
+void irlap_close(struct irlap_cb *self)
+{
+	struct irlap_cb *lap;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	/* We used to send a LAP_DISC_INDICATION here, but this was
+	 * racy. This has been move within irlmp_unregister_link()
+	 * itself. Jean II */
+
+	/* Kill the LAP and all LSAPs on top of it */
+	irlmp_unregister_link(self->saddr);
+	self->notify.instance = NULL;
+
+	/* Be sure that we manage to remove ourself from the hash */
+	lap = hashbin_remove(irlap, self->saddr, NULL);
+	if (!lap) {
+		pr_debug("%s(), Didn't find myself!\n", __func__);
+		return;
+	}
+	__irlap_close(lap);
+}
+EXPORT_SYMBOL(irlap_close);
+
+/*
+ * Function irlap_connect_indication (self, skb)
+ *
+ *    Another device is attempting to make a connection
+ *
+ */
+void irlap_connect_indication(struct irlap_cb *self, struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	irlap_init_qos_capabilities(self, NULL); /* No user QoS! */
+
+	irlmp_link_connect_indication(self->notify.instance, self->saddr,
+				      self->daddr, &self->qos_tx, skb);
+}
+
+/*
+ * Function irlap_connect_response (self, skb)
+ *
+ *    Service user has accepted incoming connection
+ *
+ */
+void irlap_connect_response(struct irlap_cb *self, struct sk_buff *userdata)
+{
+	irlap_do_event(self, CONNECT_RESPONSE, userdata, NULL);
+}
+
+/*
+ * Function irlap_connect_request (self, daddr, qos_user, sniff)
+ *
+ *    Request connection with another device, sniffing is not implemented
+ *    yet.
+ *
+ */
+void irlap_connect_request(struct irlap_cb *self, __u32 daddr,
+			   struct qos_info *qos_user, int sniff)
+{
+	pr_debug("%s(), daddr=0x%08x\n", __func__, daddr);
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	self->daddr = daddr;
+
+	/*
+	 *  If the service user specifies QoS values for this connection,
+	 *  then use them
+	 */
+	irlap_init_qos_capabilities(self, qos_user);
+
+	if ((self->state == LAP_NDM) && !self->media_busy)
+		irlap_do_event(self, CONNECT_REQUEST, NULL, NULL);
+	else
+		self->connect_pending = TRUE;
+}
+
+/*
+ * Function irlap_connect_confirm (self, skb)
+ *
+ *    Connection request has been accepted
+ *
+ */
+void irlap_connect_confirm(struct irlap_cb *self, struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	irlmp_link_connect_confirm(self->notify.instance, &self->qos_tx, skb);
+}
+
+/*
+ * Function irlap_data_indication (self, skb)
+ *
+ *    Received data frames from IR-port, so we just pass them up to
+ *    IrLMP for further processing
+ *
+ */
+void irlap_data_indication(struct irlap_cb *self, struct sk_buff *skb,
+			   int unreliable)
+{
+	/* Hide LAP header from IrLMP layer */
+	skb_pull(skb, LAP_ADDR_HEADER+LAP_CTRL_HEADER);
+
+	irlmp_link_data_indication(self->notify.instance, skb, unreliable);
+}
+
+
+/*
+ * Function irlap_data_request (self, skb)
+ *
+ *    Queue data for transmission, must wait until XMIT state
+ *
+ */
+void irlap_data_request(struct irlap_cb *self, struct sk_buff *skb,
+			int unreliable)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	IRDA_ASSERT(skb_headroom(skb) >= (LAP_ADDR_HEADER+LAP_CTRL_HEADER),
+		    return;);
+	skb_push(skb, LAP_ADDR_HEADER+LAP_CTRL_HEADER);
+
+	/*
+	 *  Must set frame format now so that the rest of the code knows
+	 *  if its dealing with an I or an UI frame
+	 */
+	if (unreliable)
+		skb->data[1] = UI_FRAME;
+	else
+		skb->data[1] = I_FRAME;
+
+	/* Don't forget to refcount it - see irlmp_connect_request(). */
+	skb_get(skb);
+
+	/* Add at the end of the queue (keep ordering) - Jean II */
+	skb_queue_tail(&self->txq, skb);
+
+	/*
+	 *  Send event if this frame only if we are in the right state
+	 *  FIXME: udata should be sent first! (skb_queue_head?)
+	 */
+	if ((self->state == LAP_XMIT_P) || (self->state == LAP_XMIT_S)) {
+		/* If we are not already processing the Tx queue, trigger
+		 * transmission immediately - Jean II */
+		if((skb_queue_len(&self->txq) <= 1) && (!self->local_busy))
+			irlap_do_event(self, DATA_REQUEST, skb, NULL);
+		/* Otherwise, the packets will be sent normally at the
+		 * next pf-poll - Jean II */
+	}
+}
+
+/*
+ * Function irlap_unitdata_request (self, skb)
+ *
+ *    Send Ultra data. This is data that must be sent outside any connection
+ *
+ */
+#ifdef CONFIG_IRDA_ULTRA
+void irlap_unitdata_request(struct irlap_cb *self, struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	IRDA_ASSERT(skb_headroom(skb) >= (LAP_ADDR_HEADER+LAP_CTRL_HEADER),
+	       return;);
+	skb_push(skb, LAP_ADDR_HEADER+LAP_CTRL_HEADER);
+
+	skb->data[0] = CBROADCAST;
+	skb->data[1] = UI_FRAME;
+
+	/* Don't need to refcount, see irlmp_connless_data_request() */
+
+	skb_queue_tail(&self->txq_ultra, skb);
+
+	irlap_do_event(self, SEND_UI_FRAME, NULL, NULL);
+}
+#endif /*CONFIG_IRDA_ULTRA */
+
+/*
+ * Function irlap_udata_indication (self, skb)
+ *
+ *    Receive Ultra data. This is data that is received outside any connection
+ *
+ */
+#ifdef CONFIG_IRDA_ULTRA
+void irlap_unitdata_indication(struct irlap_cb *self, struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+	IRDA_ASSERT(skb != NULL, return;);
+
+	/* Hide LAP header from IrLMP layer */
+	skb_pull(skb, LAP_ADDR_HEADER+LAP_CTRL_HEADER);
+
+	irlmp_link_unitdata_indication(self->notify.instance, skb);
+}
+#endif /* CONFIG_IRDA_ULTRA */
+
+/*
+ * Function irlap_disconnect_request (void)
+ *
+ *    Request to disconnect connection by service user
+ */
+void irlap_disconnect_request(struct irlap_cb *self)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	/* Don't disconnect until all data frames are successfully sent */
+	if (!skb_queue_empty(&self->txq)) {
+		self->disconnect_pending = TRUE;
+		return;
+	}
+
+	/* Check if we are in the right state for disconnecting */
+	switch (self->state) {
+	case LAP_XMIT_P:        /* FALLTHROUGH */
+	case LAP_XMIT_S:        /* FALLTHROUGH */
+	case LAP_CONN:          /* FALLTHROUGH */
+	case LAP_RESET_WAIT:    /* FALLTHROUGH */
+	case LAP_RESET_CHECK:
+		irlap_do_event(self, DISCONNECT_REQUEST, NULL, NULL);
+		break;
+	default:
+		pr_debug("%s(), disconnect pending!\n", __func__);
+		self->disconnect_pending = TRUE;
+		break;
+	}
+}
+
+/*
+ * Function irlap_disconnect_indication (void)
+ *
+ *    Disconnect request from other device
+ *
+ */
+void irlap_disconnect_indication(struct irlap_cb *self, LAP_REASON reason)
+{
+	pr_debug("%s(), reason=%s\n", __func__, lap_reasons[reason]);
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	/* Flush queues */
+	irlap_flush_all_queues(self);
+
+	switch (reason) {
+	case LAP_RESET_INDICATION:
+		pr_debug("%s(), Sending reset request!\n", __func__);
+		irlap_do_event(self, RESET_REQUEST, NULL, NULL);
+		break;
+	case LAP_NO_RESPONSE:	   /* FALLTHROUGH */
+	case LAP_DISC_INDICATION:  /* FALLTHROUGH */
+	case LAP_FOUND_NONE:       /* FALLTHROUGH */
+	case LAP_MEDIA_BUSY:
+		irlmp_link_disconnect_indication(self->notify.instance, self,
+						 reason, NULL);
+		break;
+	default:
+		net_err_ratelimited("%s: Unknown reason %d\n",
+				    __func__, reason);
+	}
+}
+
+/*
+ * Function irlap_discovery_request (gen_addr_bit)
+ *
+ *    Start one single discovery operation.
+ *
+ */
+void irlap_discovery_request(struct irlap_cb *self, discovery_t *discovery)
+{
+	struct irlap_info info;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+	IRDA_ASSERT(discovery != NULL, return;);
+
+	pr_debug("%s(), nslots = %d\n", __func__, discovery->nslots);
+
+	IRDA_ASSERT((discovery->nslots == 1) || (discovery->nslots == 6) ||
+		    (discovery->nslots == 8) || (discovery->nslots == 16),
+		    return;);
+
+	/* Discovery is only possible in NDM mode */
+	if (self->state != LAP_NDM) {
+		pr_debug("%s(), discovery only possible in NDM mode\n",
+			 __func__);
+		irlap_discovery_confirm(self, NULL);
+		/* Note : in theory, if we are not in NDM, we could postpone
+		 * the discovery like we do for connection request.
+		 * In practice, it's not worth it. If the media was busy,
+		 * it's likely next time around it won't be busy. If we are
+		 * in REPLY state, we will get passive discovery info & event.
+		 * Jean II */
+		return;
+	}
+
+	/* Check if last discovery request finished in time, or if
+	 * it was aborted due to the media busy flag. */
+	if (self->discovery_log != NULL) {
+		hashbin_delete(self->discovery_log, (FREE_FUNC) kfree);
+		self->discovery_log = NULL;
+	}
+
+	/* All operations will occur at predictable time, no need to lock */
+	self->discovery_log = hashbin_new(HB_NOLOCK);
+
+	if (self->discovery_log == NULL) {
+		net_warn_ratelimited("%s(), Unable to allocate discovery log!\n",
+				     __func__);
+		return;
+	}
+
+	info.S = discovery->nslots; /* Number of slots */
+	info.s = 0; /* Current slot */
+
+	self->discovery_cmd = discovery;
+	info.discovery = discovery;
+
+	/* sysctl_slot_timeout bounds are checked in irsysctl.c - Jean II */
+	self->slot_timeout = msecs_to_jiffies(sysctl_slot_timeout);
+
+	irlap_do_event(self, DISCOVERY_REQUEST, NULL, &info);
+}
+
+/*
+ * Function irlap_discovery_confirm (log)
+ *
+ *    A device has been discovered in front of this station, we
+ *    report directly to LMP.
+ */
+void irlap_discovery_confirm(struct irlap_cb *self, hashbin_t *discovery_log)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	IRDA_ASSERT(self->notify.instance != NULL, return;);
+
+	/*
+	 * Check for successful discovery, since we are then allowed to clear
+	 * the media busy condition (IrLAP 6.13.4 - p.94). This should allow
+	 * us to make connection attempts much faster and easier (i.e. no
+	 * collisions).
+	 * Setting media busy to false will also generate an event allowing
+	 * to process pending events in NDM state machine.
+	 * Note : the spec doesn't define what's a successful discovery is.
+	 * If we want Ultra to work, it's successful even if there is
+	 * nobody discovered - Jean II
+	 */
+	if (discovery_log)
+		irda_device_set_media_busy(self->netdev, FALSE);
+
+	/* Inform IrLMP */
+	irlmp_link_discovery_confirm(self->notify.instance, discovery_log);
+}
+
+/*
+ * Function irlap_discovery_indication (log)
+ *
+ *    Somebody is trying to discover us!
+ *
+ */
+void irlap_discovery_indication(struct irlap_cb *self, discovery_t *discovery)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+	IRDA_ASSERT(discovery != NULL, return;);
+
+	IRDA_ASSERT(self->notify.instance != NULL, return;);
+
+	/* A device is very likely to connect immediately after it performs
+	 * a successful discovery. This means that in our case, we are much
+	 * more likely to receive a connection request over the medium.
+	 * So, we backoff to avoid collisions.
+	 * IrLAP spec 6.13.4 suggest 100ms...
+	 * Note : this little trick actually make a *BIG* difference. If I set
+	 * my Linux box with discovery enabled and one Ultra frame sent every
+	 * second, my Palm has no trouble connecting to it every time !
+	 * Jean II */
+	irda_device_set_media_busy(self->netdev, SMALL);
+
+	irlmp_link_discovery_indication(self->notify.instance, discovery);
+}
+
+/*
+ * Function irlap_status_indication (quality_of_link)
+ */
+void irlap_status_indication(struct irlap_cb *self, int quality_of_link)
+{
+	switch (quality_of_link) {
+	case STATUS_NO_ACTIVITY:
+		net_info_ratelimited("IrLAP, no activity on link!\n");
+		break;
+	case STATUS_NOISY:
+		net_info_ratelimited("IrLAP, noisy link!\n");
+		break;
+	default:
+		break;
+	}
+	irlmp_status_indication(self->notify.instance,
+				quality_of_link, LOCK_NO_CHANGE);
+}
+
+/*
+ * Function irlap_reset_indication (void)
+ */
+void irlap_reset_indication(struct irlap_cb *self)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	if (self->state == LAP_RESET_WAIT)
+		irlap_do_event(self, RESET_REQUEST, NULL, NULL);
+	else
+		irlap_do_event(self, RESET_RESPONSE, NULL, NULL);
+}
+
+/*
+ * Function irlap_reset_confirm (void)
+ */
+void irlap_reset_confirm(void)
+{
+}
+
+/*
+ * Function irlap_generate_rand_time_slot (S, s)
+ *
+ *    Generate a random time slot between s and S-1 where
+ *    S = Number of slots (0 -> S-1)
+ *    s = Current slot
+ */
+int irlap_generate_rand_time_slot(int S, int s)
+{
+	static int rand;
+	int slot;
+
+	IRDA_ASSERT((S - s) > 0, return 0;);
+
+	rand += jiffies;
+	rand ^= (rand << 12);
+	rand ^= (rand >> 20);
+
+	slot = s + rand % (S-s);
+
+	IRDA_ASSERT((slot >= s) || (slot < S), return 0;);
+
+	return slot;
+}
+
+/*
+ * Function irlap_update_nr_received (nr)
+ *
+ *    Remove all acknowledged frames in current window queue. This code is
+ *    not intuitive and you should not try to change it. If you think it
+ *    contains bugs, please mail a patch to the author instead.
+ */
+void irlap_update_nr_received(struct irlap_cb *self, int nr)
+{
+	struct sk_buff *skb = NULL;
+	int count = 0;
+
+	/*
+	 * Remove all the ack-ed frames from the window queue.
+	 */
+
+	/*
+	 *  Optimize for the common case. It is most likely that the receiver
+	 *  will acknowledge all the frames we have sent! So in that case we
+	 *  delete all frames stored in window.
+	 */
+	if (nr == self->vs) {
+		while ((skb = skb_dequeue(&self->wx_list)) != NULL) {
+			dev_kfree_skb(skb);
+		}
+		/* The last acked frame is the next to send minus one */
+		self->va = nr - 1;
+	} else {
+		/* Remove all acknowledged frames in current window */
+		while ((skb_peek(&self->wx_list) != NULL) &&
+		       (((self->va+1) % 8) != nr))
+		{
+			skb = skb_dequeue(&self->wx_list);
+			dev_kfree_skb(skb);
+
+			self->va = (self->va + 1) % 8;
+			count++;
+		}
+	}
+
+	/* Advance window */
+	self->window = self->window_size - skb_queue_len(&self->wx_list);
+}
+
+/*
+ * Function irlap_validate_ns_received (ns)
+ *
+ *    Validate the next to send (ns) field from received frame.
+ */
+int irlap_validate_ns_received(struct irlap_cb *self, int ns)
+{
+	/*  ns as expected?  */
+	if (ns == self->vr)
+		return NS_EXPECTED;
+	/*
+	 *  Stations are allowed to treat invalid NS as unexpected NS
+	 *  IrLAP, Recv ... with-invalid-Ns. p. 84
+	 */
+	return NS_UNEXPECTED;
+
+	/* return NR_INVALID; */
+}
+/*
+ * Function irlap_validate_nr_received (nr)
+ *
+ *    Validate the next to receive (nr) field from received frame.
+ *
+ */
+int irlap_validate_nr_received(struct irlap_cb *self, int nr)
+{
+	/*  nr as expected?  */
+	if (nr == self->vs) {
+		pr_debug("%s(), expected!\n", __func__);
+		return NR_EXPECTED;
+	}
+
+	/*
+	 *  unexpected nr? (but within current window), first we check if the
+	 *  ns numbers of the frames in the current window wrap.
+	 */
+	if (self->va < self->vs) {
+		if ((nr >= self->va) && (nr <= self->vs))
+			return NR_UNEXPECTED;
+	} else {
+		if ((nr >= self->va) || (nr <= self->vs))
+			return NR_UNEXPECTED;
+	}
+
+	/* Invalid nr!  */
+	return NR_INVALID;
+}
+
+/*
+ * Function irlap_initiate_connection_state ()
+ *
+ *    Initialize the connection state parameters
+ *
+ */
+void irlap_initiate_connection_state(struct irlap_cb *self)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	/* Next to send and next to receive */
+	self->vs = self->vr = 0;
+
+	/* Last frame which got acked (0 - 1) % 8 */
+	self->va = 7;
+
+	self->window = 1;
+
+	self->remote_busy = FALSE;
+	self->retry_count = 0;
+}
+
+/*
+ * Function irlap_wait_min_turn_around (self, qos)
+ *
+ *    Wait negotiated minimum turn around time, this function actually sets
+ *    the number of BOS's that must be sent before the next transmitted
+ *    frame in order to delay for the specified amount of time. This is
+ *    done to avoid using timers, and the forbidden udelay!
+ */
+void irlap_wait_min_turn_around(struct irlap_cb *self, struct qos_info *qos)
+{
+	__u32 min_turn_time;
+	__u32 speed;
+
+	/* Get QoS values.  */
+	speed = qos->baud_rate.value;
+	min_turn_time = qos->min_turn_time.value;
+
+	/* No need to calculate XBOFs for speeds over 115200 bps */
+	if (speed > 115200) {
+		self->mtt_required = min_turn_time;
+		return;
+	}
+
+	/*
+	 *  Send additional BOF's for the next frame for the requested
+	 *  min turn time, so now we must calculate how many chars (XBOF's) we
+	 *  must send for the requested time period (min turn time)
+	 */
+	self->xbofs_delay = irlap_min_turn_time_in_bytes(speed, min_turn_time);
+}
+
+/*
+ * Function irlap_flush_all_queues (void)
+ *
+ *    Flush all queues
+ *
+ */
+void irlap_flush_all_queues(struct irlap_cb *self)
+{
+	struct sk_buff* skb;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	/* Free transmission queue */
+	while ((skb = skb_dequeue(&self->txq)) != NULL)
+		dev_kfree_skb(skb);
+
+	while ((skb = skb_dequeue(&self->txq_ultra)) != NULL)
+		dev_kfree_skb(skb);
+
+	/* Free sliding window buffered packets */
+	while ((skb = skb_dequeue(&self->wx_list)) != NULL)
+		dev_kfree_skb(skb);
+}
+
+/*
+ * Function irlap_setspeed (self, speed)
+ *
+ *    Change the speed of the IrDA port
+ *
+ */
+static void irlap_change_speed(struct irlap_cb *self, __u32 speed, int now)
+{
+	struct sk_buff *skb;
+
+	pr_debug("%s(), setting speed to %d\n", __func__, speed);
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	self->speed = speed;
+
+	/* Change speed now, or just piggyback speed on frames */
+	if (now) {
+		/* Send down empty frame to trigger speed change */
+		skb = alloc_skb(0, GFP_ATOMIC);
+		if (skb)
+			irlap_queue_xmit(self, skb);
+	}
+}
+
+/*
+ * Function irlap_init_qos_capabilities (self, qos)
+ *
+ *    Initialize QoS for this IrLAP session, What we do is to compute the
+ *    intersection of the QoS capabilities for the user, driver and for
+ *    IrLAP itself. Normally, IrLAP will not specify any values, but it can
+ *    be used to restrict certain values.
+ */
+static void irlap_init_qos_capabilities(struct irlap_cb *self,
+					struct qos_info *qos_user)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+	IRDA_ASSERT(self->netdev != NULL, return;);
+
+	/* Start out with the maximum QoS support possible */
+	irda_init_max_qos_capabilies(&self->qos_rx);
+
+	/* Apply drivers QoS capabilities */
+	irda_qos_compute_intersection(&self->qos_rx, self->qos_dev);
+
+	/*
+	 *  Check for user supplied QoS parameters. The service user is only
+	 *  allowed to supply these values. We check each parameter since the
+	 *  user may not have set all of them.
+	 */
+	if (qos_user) {
+		pr_debug("%s(), Found user specified QoS!\n", __func__);
+
+		if (qos_user->baud_rate.bits)
+			self->qos_rx.baud_rate.bits &= qos_user->baud_rate.bits;
+
+		if (qos_user->max_turn_time.bits)
+			self->qos_rx.max_turn_time.bits &= qos_user->max_turn_time.bits;
+		if (qos_user->data_size.bits)
+			self->qos_rx.data_size.bits &= qos_user->data_size.bits;
+
+		if (qos_user->link_disc_time.bits)
+			self->qos_rx.link_disc_time.bits &= qos_user->link_disc_time.bits;
+	}
+
+	/* Use 500ms in IrLAP for now */
+	self->qos_rx.max_turn_time.bits &= 0x01;
+
+	/* Set data size */
+	/*self->qos_rx.data_size.bits &= 0x03;*/
+
+	irda_qos_bits_to_value(&self->qos_rx);
+}
+
+/*
+ * Function irlap_apply_default_connection_parameters (void, now)
+ *
+ *    Use the default connection and transmission parameters
+ */
+void irlap_apply_default_connection_parameters(struct irlap_cb *self)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	/* xbofs : Default value in NDM */
+	self->next_bofs   = 12;
+	self->bofs_count  = 12;
+
+	/* NDM Speed is 9600 */
+	irlap_change_speed(self, 9600, TRUE);
+
+	/* Set mbusy when going to NDM state */
+	irda_device_set_media_busy(self->netdev, TRUE);
+
+	/*
+	 * Generate random connection address for this session, which must
+	 * be 7 bits wide and different from 0x00 and 0xfe
+	 */
+	while ((self->caddr == 0x00) || (self->caddr == 0xfe)) {
+		get_random_bytes(&self->caddr, sizeof(self->caddr));
+		self->caddr &= 0xfe;
+	}
+
+	/* Use default values until connection has been negitiated */
+	self->slot_timeout = sysctl_slot_timeout;
+	self->final_timeout = FINAL_TIMEOUT;
+	self->poll_timeout = POLL_TIMEOUT;
+	self->wd_timeout = WD_TIMEOUT;
+
+	/* Set some default values */
+	self->qos_tx.baud_rate.value = 9600;
+	self->qos_rx.baud_rate.value = 9600;
+	self->qos_tx.max_turn_time.value = 0;
+	self->qos_rx.max_turn_time.value = 0;
+	self->qos_tx.min_turn_time.value = 0;
+	self->qos_rx.min_turn_time.value = 0;
+	self->qos_tx.data_size.value = 64;
+	self->qos_rx.data_size.value = 64;
+	self->qos_tx.window_size.value = 1;
+	self->qos_rx.window_size.value = 1;
+	self->qos_tx.additional_bofs.value = 12;
+	self->qos_rx.additional_bofs.value = 12;
+	self->qos_tx.link_disc_time.value = 0;
+	self->qos_rx.link_disc_time.value = 0;
+
+	irlap_flush_all_queues(self);
+
+	self->disconnect_pending = FALSE;
+	self->connect_pending = FALSE;
+}
+
+/*
+ * Function irlap_apply_connection_parameters (qos, now)
+ *
+ *    Initialize IrLAP with the negotiated QoS values
+ *
+ * If 'now' is false, the speed and xbofs will be changed after the next
+ * frame is sent.
+ * If 'now' is true, the speed and xbofs is changed immediately
+ */
+void irlap_apply_connection_parameters(struct irlap_cb *self, int now)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	/* Set the negotiated xbofs value */
+	self->next_bofs   = self->qos_tx.additional_bofs.value;
+	if (now)
+		self->bofs_count = self->next_bofs;
+
+	/* Set the negotiated link speed (may need the new xbofs value) */
+	irlap_change_speed(self, self->qos_tx.baud_rate.value, now);
+
+	self->window_size = self->qos_tx.window_size.value;
+	self->window      = self->qos_tx.window_size.value;
+
+#ifdef CONFIG_IRDA_DYNAMIC_WINDOW
+	/*
+	 *  Calculate how many bytes it is possible to transmit before the
+	 *  link must be turned around
+	 */
+	self->line_capacity =
+		irlap_max_line_capacity(self->qos_tx.baud_rate.value,
+					self->qos_tx.max_turn_time.value);
+	self->bytes_left = self->line_capacity;
+#endif /* CONFIG_IRDA_DYNAMIC_WINDOW */
+
+
+	/*
+	 *  Initialize timeout values, some of the rules are listed on
+	 *  page 92 in IrLAP.
+	 */
+	IRDA_ASSERT(self->qos_tx.max_turn_time.value != 0, return;);
+	IRDA_ASSERT(self->qos_rx.max_turn_time.value != 0, return;);
+	/* The poll timeout applies only to the primary station.
+	 * It defines the maximum time the primary stay in XMIT mode
+	 * before timeout and turning the link around (sending a RR).
+	 * Or, this is how much we can keep the pf bit in primary mode.
+	 * Therefore, it must be lower or equal than our *OWN* max turn around.
+	 * Jean II */
+	self->poll_timeout = msecs_to_jiffies(
+				self->qos_tx.max_turn_time.value);
+	/* The Final timeout applies only to the primary station.
+	 * It defines the maximum time the primary wait (mostly in RECV mode)
+	 * for an answer from the secondary station before polling it again.
+	 * Therefore, it must be greater or equal than our *PARTNER*
+	 * max turn around time - Jean II */
+	self->final_timeout = msecs_to_jiffies(
+				self->qos_rx.max_turn_time.value);
+	/* The Watchdog Bit timeout applies only to the secondary station.
+	 * It defines the maximum time the secondary wait (mostly in RECV mode)
+	 * for poll from the primary station before getting annoyed.
+	 * Therefore, it must be greater or equal than our *PARTNER*
+	 * max turn around time - Jean II */
+	self->wd_timeout = self->final_timeout * 2;
+
+	/*
+	 * N1 and N2 are maximum retry count for *both* the final timer
+	 * and the wd timer (with a factor 2) as defined above.
+	 * After N1 retry of a timer, we give a warning to the user.
+	 * After N2 retry, we consider the link dead and disconnect it.
+	 * Jean II
+	 */
+
+	/*
+	 *  Set N1 to 0 if Link Disconnect/Threshold Time = 3 and set it to
+	 *  3 seconds otherwise. See page 71 in IrLAP for more details.
+	 *  Actually, it's not always 3 seconds, as we allow to set
+	 *  it via sysctl... Max maxtt is 500ms, and N1 need to be multiple
+	 *  of 2, so 1 second is minimum we can allow. - Jean II
+	 */
+	if (self->qos_tx.link_disc_time.value == sysctl_warn_noreply_time)
+		/*
+		 * If we set N1 to 0, it will trigger immediately, which is
+		 * not what we want. What we really want is to disable it,
+		 * Jean II
+		 */
+		self->N1 = -2; /* Disable - Need to be multiple of 2*/
+	else
+		self->N1 = sysctl_warn_noreply_time * 1000 /
+		  self->qos_rx.max_turn_time.value;
+
+	pr_debug("Setting N1 = %d\n", self->N1);
+
+	/* Set N2 to match our own disconnect time */
+	self->N2 = self->qos_tx.link_disc_time.value * 1000 /
+		self->qos_rx.max_turn_time.value;
+	pr_debug("Setting N2 = %d\n", self->N2);
+}
+
+#ifdef CONFIG_PROC_FS
+struct irlap_iter_state {
+	int id;
+};
+
+static void *irlap_seq_start(struct seq_file *seq, loff_t *pos)
+{
+	struct irlap_iter_state *iter = seq->private;
+	struct irlap_cb *self;
+
+	/* Protect our access to the tsap list */
+	spin_lock_irq(&irlap->hb_spinlock);
+	iter->id = 0;
+
+	for (self = (struct irlap_cb *) hashbin_get_first(irlap);
+	     self; self = (struct irlap_cb *) hashbin_get_next(irlap)) {
+		if (iter->id == *pos)
+			break;
+		++iter->id;
+	}
+
+	return self;
+}
+
+static void *irlap_seq_next(struct seq_file *seq, void *v, loff_t *pos)
+{
+	struct irlap_iter_state *iter = seq->private;
+
+	++*pos;
+	++iter->id;
+	return (void *) hashbin_get_next(irlap);
+}
+
+static void irlap_seq_stop(struct seq_file *seq, void *v)
+{
+	spin_unlock_irq(&irlap->hb_spinlock);
+}
+
+static int irlap_seq_show(struct seq_file *seq, void *v)
+{
+	const struct irlap_iter_state *iter = seq->private;
+	const struct irlap_cb *self = v;
+
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return -EINVAL;);
+
+	seq_printf(seq, "irlap%d ", iter->id);
+	seq_printf(seq, "state: %s\n",
+		   irlap_state[self->state]);
+
+	seq_printf(seq, "  device name: %s, ",
+		   (self->netdev) ? self->netdev->name : "bug");
+	seq_printf(seq, "hardware name: %s\n", self->hw_name);
+
+	seq_printf(seq, "  caddr: %#02x, ", self->caddr);
+	seq_printf(seq, "saddr: %#08x, ", self->saddr);
+	seq_printf(seq, "daddr: %#08x\n", self->daddr);
+
+	seq_printf(seq, "  win size: %d, ",
+		   self->window_size);
+	seq_printf(seq, "win: %d, ", self->window);
+#ifdef CONFIG_IRDA_DYNAMIC_WINDOW
+	seq_printf(seq, "line capacity: %d, ",
+		   self->line_capacity);
+	seq_printf(seq, "bytes left: %d\n", self->bytes_left);
+#endif /* CONFIG_IRDA_DYNAMIC_WINDOW */
+	seq_printf(seq, "  tx queue len: %d ",
+		   skb_queue_len(&self->txq));
+	seq_printf(seq, "win queue len: %d ",
+		   skb_queue_len(&self->wx_list));
+	seq_printf(seq, "rbusy: %s", self->remote_busy ?
+		   "TRUE" : "FALSE");
+	seq_printf(seq, " mbusy: %s\n", self->media_busy ?
+		   "TRUE" : "FALSE");
+
+	seq_printf(seq, "  retrans: %d ", self->retry_count);
+	seq_printf(seq, "vs: %d ", self->vs);
+	seq_printf(seq, "vr: %d ", self->vr);
+	seq_printf(seq, "va: %d\n", self->va);
+
+	seq_printf(seq, "  qos\tbps\tmaxtt\tdsize\twinsize\taddbofs\tmintt\tldisc\tcomp\n");
+
+	seq_printf(seq, "  tx\t%d\t",
+		   self->qos_tx.baud_rate.value);
+	seq_printf(seq, "%d\t",
+		   self->qos_tx.max_turn_time.value);
+	seq_printf(seq, "%d\t",
+		   self->qos_tx.data_size.value);
+	seq_printf(seq, "%d\t",
+		   self->qos_tx.window_size.value);
+	seq_printf(seq, "%d\t",
+		   self->qos_tx.additional_bofs.value);
+	seq_printf(seq, "%d\t",
+		   self->qos_tx.min_turn_time.value);
+	seq_printf(seq, "%d\t",
+		   self->qos_tx.link_disc_time.value);
+	seq_printf(seq, "\n");
+
+	seq_printf(seq, "  rx\t%d\t",
+		   self->qos_rx.baud_rate.value);
+	seq_printf(seq, "%d\t",
+		   self->qos_rx.max_turn_time.value);
+	seq_printf(seq, "%d\t",
+		   self->qos_rx.data_size.value);
+	seq_printf(seq, "%d\t",
+		   self->qos_rx.window_size.value);
+	seq_printf(seq, "%d\t",
+		   self->qos_rx.additional_bofs.value);
+	seq_printf(seq, "%d\t",
+		   self->qos_rx.min_turn_time.value);
+	seq_printf(seq, "%d\n",
+		   self->qos_rx.link_disc_time.value);
+
+	return 0;
+}
+
+static const struct seq_operations irlap_seq_ops = {
+	.start  = irlap_seq_start,
+	.next   = irlap_seq_next,
+	.stop   = irlap_seq_stop,
+	.show   = irlap_seq_show,
+};
+
+static int irlap_seq_open(struct inode *inode, struct file *file)
+{
+	if (irlap == NULL)
+		return -EINVAL;
+
+	return seq_open_private(file, &irlap_seq_ops,
+			sizeof(struct irlap_iter_state));
+}
+
+const struct file_operations irlap_seq_fops = {
+	.owner		= THIS_MODULE,
+	.open           = irlap_seq_open,
+	.read           = seq_read,
+	.llseek         = seq_lseek,
+	.release	= seq_release_private,
+};
+
+#endif /* CONFIG_PROC_FS */
diff --git a/drivers/staging/irda/net/irlap_event.c b/drivers/staging/irda/net/irlap_event.c
new file mode 100644
index 000000000000..0e1b4d79f745
--- /dev/null
+++ b/drivers/staging/irda/net/irlap_event.c
@@ -0,0 +1,2316 @@
+/*********************************************************************
+ *
+ * Filename:      irlap_event.c
+ * Version:       0.9
+ * Description:   IrLAP state machine implementation
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dag@brattli.net>
+ * Created at:    Sat Aug 16 00:59:29 1997
+ * Modified at:   Sat Dec 25 21:07:57 1999
+ * Modified by:   Dag Brattli <dag@brattli.net>
+ *
+ *     Copyright (c) 1998-2000 Dag Brattli <dag@brattli.net>,
+ *     Copyright (c) 1998      Thomas Davis <ratbert@radiks.net>
+ *     All Rights Reserved.
+ *     Copyright (c) 2000-2003 Jean Tourrilhes <jt@hpl.hp.com>
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#include <linux/string.h>
+#include <linux/kernel.h>
+#include <linux/delay.h>
+#include <linux/skbuff.h>
+#include <linux/slab.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/irlap_event.h>
+
+#include <net/irda/timer.h>
+#include <net/irda/irlap.h>
+#include <net/irda/irlap_frame.h>
+#include <net/irda/qos.h>
+#include <net/irda/parameters.h>
+#include <net/irda/irlmp.h>		/* irlmp_flow_indication(), ... */
+
+#include <net/irda/irda_device.h>
+
+#ifdef CONFIG_IRDA_FAST_RR
+int sysctl_fast_poll_increase = 50;
+#endif
+
+static int irlap_state_ndm    (struct irlap_cb *self, IRLAP_EVENT event,
+			       struct sk_buff *skb, struct irlap_info *info);
+static int irlap_state_query  (struct irlap_cb *self, IRLAP_EVENT event,
+			       struct sk_buff *skb, struct irlap_info *info);
+static int irlap_state_reply  (struct irlap_cb *self, IRLAP_EVENT event,
+			       struct sk_buff *skb, struct irlap_info *info);
+static int irlap_state_conn   (struct irlap_cb *self, IRLAP_EVENT event,
+			       struct sk_buff *skb, struct irlap_info *info);
+static int irlap_state_setup  (struct irlap_cb *self, IRLAP_EVENT event,
+			       struct sk_buff *skb, struct irlap_info *info);
+static int irlap_state_offline(struct irlap_cb *self, IRLAP_EVENT event,
+			       struct sk_buff *skb, struct irlap_info *info);
+static int irlap_state_xmit_p (struct irlap_cb *self, IRLAP_EVENT event,
+			       struct sk_buff *skb, struct irlap_info *info);
+static int irlap_state_pclose (struct irlap_cb *self, IRLAP_EVENT event,
+			       struct sk_buff *skb, struct irlap_info *info);
+static int irlap_state_nrm_p  (struct irlap_cb *self, IRLAP_EVENT event,
+			       struct sk_buff *skb, struct irlap_info *info);
+static int irlap_state_reset_wait(struct irlap_cb *self, IRLAP_EVENT event,
+				  struct sk_buff *skb, struct irlap_info *info);
+static int irlap_state_reset  (struct irlap_cb *self, IRLAP_EVENT event,
+			       struct sk_buff *skb, struct irlap_info *info);
+static int irlap_state_nrm_s  (struct irlap_cb *self, IRLAP_EVENT event,
+			       struct sk_buff *skb, struct irlap_info *info);
+static int irlap_state_xmit_s (struct irlap_cb *self, IRLAP_EVENT event,
+			       struct sk_buff *skb, struct irlap_info *info);
+static int irlap_state_sclose (struct irlap_cb *self, IRLAP_EVENT event,
+			       struct sk_buff *skb, struct irlap_info *info);
+static int irlap_state_reset_check(struct irlap_cb *, IRLAP_EVENT event,
+				   struct sk_buff *, struct irlap_info *);
+
+static const char *const irlap_event[] __maybe_unused = {
+	"DISCOVERY_REQUEST",
+	"CONNECT_REQUEST",
+	"CONNECT_RESPONSE",
+	"DISCONNECT_REQUEST",
+	"DATA_REQUEST",
+	"RESET_REQUEST",
+	"RESET_RESPONSE",
+	"SEND_I_CMD",
+	"SEND_UI_FRAME",
+	"RECV_DISCOVERY_XID_CMD",
+	"RECV_DISCOVERY_XID_RSP",
+	"RECV_SNRM_CMD",
+	"RECV_TEST_CMD",
+	"RECV_TEST_RSP",
+	"RECV_UA_RSP",
+	"RECV_DM_RSP",
+	"RECV_RD_RSP",
+	"RECV_I_CMD",
+	"RECV_I_RSP",
+	"RECV_UI_FRAME",
+	"RECV_FRMR_RSP",
+	"RECV_RR_CMD",
+	"RECV_RR_RSP",
+	"RECV_RNR_CMD",
+	"RECV_RNR_RSP",
+	"RECV_REJ_CMD",
+	"RECV_REJ_RSP",
+	"RECV_SREJ_CMD",
+	"RECV_SREJ_RSP",
+	"RECV_DISC_CMD",
+	"SLOT_TIMER_EXPIRED",
+	"QUERY_TIMER_EXPIRED",
+	"FINAL_TIMER_EXPIRED",
+	"POLL_TIMER_EXPIRED",
+	"DISCOVERY_TIMER_EXPIRED",
+	"WD_TIMER_EXPIRED",
+	"BACKOFF_TIMER_EXPIRED",
+	"MEDIA_BUSY_TIMER_EXPIRED",
+};
+
+const char *const irlap_state[] = {
+	"LAP_NDM",
+	"LAP_QUERY",
+	"LAP_REPLY",
+	"LAP_CONN",
+	"LAP_SETUP",
+	"LAP_OFFLINE",
+	"LAP_XMIT_P",
+	"LAP_PCLOSE",
+	"LAP_NRM_P",
+	"LAP_RESET_WAIT",
+	"LAP_RESET",
+	"LAP_NRM_S",
+	"LAP_XMIT_S",
+	"LAP_SCLOSE",
+	"LAP_RESET_CHECK",
+};
+
+static int (*state[])(struct irlap_cb *self, IRLAP_EVENT event,
+		      struct sk_buff *skb, struct irlap_info *info) =
+{
+	irlap_state_ndm,
+	irlap_state_query,
+	irlap_state_reply,
+	irlap_state_conn,
+	irlap_state_setup,
+	irlap_state_offline,
+	irlap_state_xmit_p,
+	irlap_state_pclose,
+	irlap_state_nrm_p,
+	irlap_state_reset_wait,
+	irlap_state_reset,
+	irlap_state_nrm_s,
+	irlap_state_xmit_s,
+	irlap_state_sclose,
+	irlap_state_reset_check,
+};
+
+/*
+ * Function irda_poll_timer_expired (data)
+ *
+ *    Poll timer has expired. Normally we must now send a RR frame to the
+ *    remote device
+ */
+static void irlap_poll_timer_expired(void *data)
+{
+	struct irlap_cb *self = (struct irlap_cb *) data;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	irlap_do_event(self, POLL_TIMER_EXPIRED, NULL, NULL);
+}
+
+/*
+ * Calculate and set time before we will have to send back the pf bit
+ * to the peer. Use in primary.
+ * Make sure that state is XMIT_P/XMIT_S when calling this function
+ * (and that nobody messed up with the state). - Jean II
+ */
+static void irlap_start_poll_timer(struct irlap_cb *self, int timeout)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+#ifdef CONFIG_IRDA_FAST_RR
+	/*
+	 * Send out the RR frames faster if our own transmit queue is empty, or
+	 * if the peer is busy. The effect is a much faster conversation
+	 */
+	if (skb_queue_empty(&self->txq) || self->remote_busy) {
+		if (self->fast_RR == TRUE) {
+			/*
+			 *  Assert that the fast poll timer has not reached the
+			 *  normal poll timer yet
+			 */
+			if (self->fast_RR_timeout < timeout) {
+				/*
+				 *  FIXME: this should be a more configurable
+				 *         function
+				 */
+				self->fast_RR_timeout +=
+					(sysctl_fast_poll_increase * HZ/1000);
+
+				/* Use this fast(er) timeout instead */
+				timeout = self->fast_RR_timeout;
+			}
+		} else {
+			self->fast_RR = TRUE;
+
+			/* Start with just 0 ms */
+			self->fast_RR_timeout = 0;
+			timeout = 0;
+		}
+	} else
+		self->fast_RR = FALSE;
+
+	pr_debug("%s(), timeout=%d (%ld)\n", __func__, timeout, jiffies);
+#endif /* CONFIG_IRDA_FAST_RR */
+
+	if (timeout == 0)
+		irlap_do_event(self, POLL_TIMER_EXPIRED, NULL, NULL);
+	else
+		irda_start_timer(&self->poll_timer, timeout, self,
+				 irlap_poll_timer_expired);
+}
+
+/*
+ * Function irlap_do_event (event, skb, info)
+ *
+ *    Rushes through the state machine without any delay. If state == XMIT
+ *    then send queued data frames.
+ */
+void irlap_do_event(struct irlap_cb *self, IRLAP_EVENT event,
+		    struct sk_buff *skb, struct irlap_info *info)
+{
+	int ret;
+
+	if (!self || self->magic != LAP_MAGIC)
+		return;
+
+	pr_debug("%s(), event = %s, state = %s\n", __func__,
+		 irlap_event[event], irlap_state[self->state]);
+
+	ret = (*state[self->state])(self, event, skb, info);
+
+	/*
+	 *  Check if there are any pending events that needs to be executed
+	 */
+	switch (self->state) {
+	case LAP_XMIT_P: /* FALLTHROUGH */
+	case LAP_XMIT_S:
+		/*
+		 * We just received the pf bit and are at the beginning
+		 * of a new LAP transmit window.
+		 * Check if there are any queued data frames, and do not
+		 * try to disconnect link if we send any data frames, since
+		 * that will change the state away form XMIT
+		 */
+		pr_debug("%s() : queue len = %d\n", __func__,
+			 skb_queue_len(&self->txq));
+
+		if (!skb_queue_empty(&self->txq)) {
+			/* Prevent race conditions with irlap_data_request() */
+			self->local_busy = TRUE;
+
+			/* Theory of operation.
+			 * We send frames up to when we fill the window or
+			 * reach line capacity. Those frames will queue up
+			 * in the device queue, and the driver will slowly
+			 * send them.
+			 * After each frame that we send, we poll the higher
+			 * layer for more data. It's the right time to do
+			 * that because the link layer need to perform the mtt
+			 * and then send the first frame, so we can afford
+			 * to send a bit of time in kernel space.
+			 * The explicit flow indication allow to minimise
+			 * buffers (== lower latency), to avoid higher layer
+			 * polling via timers (== less context switches) and
+			 * to implement a crude scheduler - Jean II */
+
+			/* Try to send away all queued data frames */
+			while ((skb = skb_dequeue(&self->txq)) != NULL) {
+				/* Send one frame */
+				ret = (*state[self->state])(self, SEND_I_CMD,
+							    skb, NULL);
+				/* Drop reference count.
+				 * It will be increase as needed in
+				 * irlap_send_data_xxx() */
+				kfree_skb(skb);
+
+				/* Poll the higher layers for one more frame */
+				irlmp_flow_indication(self->notify.instance,
+						      FLOW_START);
+
+				if (ret == -EPROTO)
+					break; /* Try again later! */
+			}
+			/* Finished transmitting */
+			self->local_busy = FALSE;
+		} else if (self->disconnect_pending) {
+			self->disconnect_pending = FALSE;
+
+			ret = (*state[self->state])(self, DISCONNECT_REQUEST,
+						    NULL, NULL);
+		}
+		break;
+/*	case LAP_NDM: */
+/*	case LAP_CONN: */
+/*	case LAP_RESET_WAIT: */
+/*	case LAP_RESET_CHECK: */
+	default:
+		break;
+	}
+}
+
+/*
+ * Function irlap_state_ndm (event, skb, frame)
+ *
+ *    NDM (Normal Disconnected Mode) state
+ *
+ */
+static int irlap_state_ndm(struct irlap_cb *self, IRLAP_EVENT event,
+			   struct sk_buff *skb, struct irlap_info *info)
+{
+	discovery_t *discovery_rsp;
+	int ret = 0;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return -1;);
+
+	switch (event) {
+	case CONNECT_REQUEST:
+		IRDA_ASSERT(self->netdev != NULL, return -1;);
+
+		if (self->media_busy) {
+			/* Note : this will never happen, because we test
+			 * media busy in irlap_connect_request() and
+			 * postpone the event... - Jean II */
+			pr_debug("%s(), CONNECT_REQUEST: media busy!\n",
+				 __func__);
+
+			/* Always switch state before calling upper layers */
+			irlap_next_state(self, LAP_NDM);
+
+			irlap_disconnect_indication(self, LAP_MEDIA_BUSY);
+		} else {
+			irlap_send_snrm_frame(self, &self->qos_rx);
+
+			/* Start Final-bit timer */
+			irlap_start_final_timer(self, self->final_timeout);
+
+			self->retry_count = 0;
+			irlap_next_state(self, LAP_SETUP);
+		}
+		break;
+	case RECV_SNRM_CMD:
+		/* Check if the frame contains and I field */
+		if (info) {
+			self->daddr = info->daddr;
+			self->caddr = info->caddr;
+
+			irlap_next_state(self, LAP_CONN);
+
+			irlap_connect_indication(self, skb);
+		} else {
+			pr_debug("%s(), SNRM frame does not contain an I field!\n",
+				 __func__);
+		}
+		break;
+	case DISCOVERY_REQUEST:
+		IRDA_ASSERT(info != NULL, return -1;);
+
+		if (self->media_busy) {
+			pr_debug("%s(), DISCOVERY_REQUEST: media busy!\n",
+				 __func__);
+			/* irlap->log.condition = MEDIA_BUSY; */
+
+			/* This will make IrLMP try again */
+			irlap_discovery_confirm(self, NULL);
+			/* Note : the discovery log is not cleaned up here,
+			 * it will be done in irlap_discovery_request()
+			 * Jean II */
+			return 0;
+		}
+
+		self->S = info->S;
+		self->s = info->s;
+		irlap_send_discovery_xid_frame(self, info->S, info->s, TRUE,
+					       info->discovery);
+		self->frame_sent = FALSE;
+		self->s++;
+
+		irlap_start_slot_timer(self, self->slot_timeout);
+		irlap_next_state(self, LAP_QUERY);
+		break;
+	case RECV_DISCOVERY_XID_CMD:
+		IRDA_ASSERT(info != NULL, return -1;);
+
+		/* Assert that this is not the final slot */
+		if (info->s <= info->S) {
+			self->slot = irlap_generate_rand_time_slot(info->S,
+								   info->s);
+			if (self->slot == info->s) {
+				discovery_rsp = irlmp_get_discovery_response();
+				discovery_rsp->data.daddr = info->daddr;
+
+				irlap_send_discovery_xid_frame(self, info->S,
+							       self->slot,
+							       FALSE,
+							       discovery_rsp);
+				self->frame_sent = TRUE;
+			} else
+				self->frame_sent = FALSE;
+
+			/*
+			 * Go to reply state until end of discovery to
+			 * inhibit our own transmissions. Set the timer
+			 * to not stay forever there... Jean II
+			 */
+			irlap_start_query_timer(self, info->S, info->s);
+			irlap_next_state(self, LAP_REPLY);
+		} else {
+		/* This is the final slot. How is it possible ?
+		 * This would happen is both discoveries are just slightly
+		 * offset (if they are in sync, all packets are lost).
+		 * Most often, all the discovery requests will be received
+		 * in QUERY state (see my comment there), except for the
+		 * last frame that will come here.
+		 * The big trouble when it happen is that active discovery
+		 * doesn't happen, because nobody answer the discoveries
+		 * frame of the other guy, so the log shows up empty.
+		 * What should we do ?
+		 * Not much. It's too late to answer those discovery frames,
+		 * so we just pass the info to IrLMP who will put it in the
+		 * log (and post an event).
+		 * Another cause would be devices that do discovery much
+		 * slower than us, however the latest fixes should minimise
+		 * those cases...
+		 * Jean II
+		 */
+			pr_debug("%s(), Receiving final discovery request, missed the discovery slots :-(\n",
+				 __func__);
+
+			/* Last discovery request -> in the log */
+			irlap_discovery_indication(self, info->discovery);
+		}
+		break;
+	case MEDIA_BUSY_TIMER_EXPIRED:
+		/* A bunch of events may be postponed because the media is
+		 * busy (usually immediately after we close a connection),
+		 * or while we are doing discovery (state query/reply).
+		 * In all those cases, the media busy flag will be cleared
+		 * when it's OK for us to process those postponed events.
+		 * This event is not mentioned in the state machines in the
+		 * IrLAP spec. It's because they didn't consider Ultra and
+		 * postponing connection request is optional.
+		 * Jean II */
+#ifdef CONFIG_IRDA_ULTRA
+		/* Send any pending Ultra frames if any */
+		if (!skb_queue_empty(&self->txq_ultra)) {
+			/* We don't send the frame, just post an event.
+			 * Also, previously this code was in timer.c...
+			 * Jean II */
+			ret = (*state[self->state])(self, SEND_UI_FRAME,
+						    NULL, NULL);
+		}
+#endif /* CONFIG_IRDA_ULTRA */
+		/* Check if we should try to connect.
+		 * This code was previously in irlap_do_event() */
+		if (self->connect_pending) {
+			self->connect_pending = FALSE;
+
+			/* This one *should* not pend in this state, except
+			 * if a socket try to connect and immediately
+			 * disconnect. - clear - Jean II */
+			if (self->disconnect_pending)
+				irlap_disconnect_indication(self, LAP_DISC_INDICATION);
+			else
+				ret = (*state[self->state])(self,
+							    CONNECT_REQUEST,
+							    NULL, NULL);
+			self->disconnect_pending = FALSE;
+		}
+		/* Note : one way to test if this code works well (including
+		 * media busy and small busy) is to create a user space
+		 * application generating an Ultra packet every 3.05 sec (or
+		 * 2.95 sec) and to see how it interact with discovery.
+		 * It's fairly easy to check that no packet is lost, that the
+		 * packets are postponed during discovery and that after
+		 * discovery indication you have a 100ms "gap".
+		 * As connection request and Ultra are now processed the same
+		 * way, this avoid the tedious job of trying IrLAP connection
+		 * in all those cases...
+		 * Jean II */
+		break;
+#ifdef CONFIG_IRDA_ULTRA
+	case SEND_UI_FRAME:
+	{
+		int i;
+		/* Only allowed to repeat an operation twice */
+		for (i=0; ((i<2) && (self->media_busy == FALSE)); i++) {
+			skb = skb_dequeue(&self->txq_ultra);
+			if (skb)
+				irlap_send_ui_frame(self, skb, CBROADCAST,
+						    CMD_FRAME);
+			else
+				break;
+			/* irlap_send_ui_frame() won't increase skb reference
+			 * count, so no dev_kfree_skb() - Jean II */
+		}
+		if (i == 2) {
+			/* Force us to listen 500 ms again */
+			irda_device_set_media_busy(self->netdev, TRUE);
+		}
+		break;
+	}
+	case RECV_UI_FRAME:
+		/* Only accept broadcast frames in NDM mode */
+		if (info->caddr != CBROADCAST) {
+			pr_debug("%s(), not a broadcast frame!\n",
+				 __func__);
+		} else
+			irlap_unitdata_indication(self, skb);
+		break;
+#endif /* CONFIG_IRDA_ULTRA */
+	case RECV_TEST_CMD:
+		/* Remove test frame header */
+		skb_pull(skb, sizeof(struct test_frame));
+
+		/*
+		 * Send response. This skb will not be sent out again, and
+		 * will only be used to send out the same info as the cmd
+		 */
+		irlap_send_test_frame(self, CBROADCAST, info->daddr, skb);
+		break;
+	case RECV_TEST_RSP:
+		pr_debug("%s() not implemented!\n", __func__);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %s\n", __func__,
+			 irlap_event[event]);
+
+		ret = -1;
+		break;
+	}
+	return ret;
+}
+
+/*
+ * Function irlap_state_query (event, skb, info)
+ *
+ *    QUERY state
+ *
+ */
+static int irlap_state_query(struct irlap_cb *self, IRLAP_EVENT event,
+			     struct sk_buff *skb, struct irlap_info *info)
+{
+	int ret = 0;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return -1;);
+
+	switch (event) {
+	case RECV_DISCOVERY_XID_RSP:
+		IRDA_ASSERT(info != NULL, return -1;);
+		IRDA_ASSERT(info->discovery != NULL, return -1;);
+
+		pr_debug("%s(), daddr=%08x\n", __func__,
+			 info->discovery->data.daddr);
+
+		if (!self->discovery_log) {
+			net_warn_ratelimited("%s: discovery log is gone! maybe the discovery timeout has been set too short?\n",
+					     __func__);
+			break;
+		}
+		hashbin_insert(self->discovery_log,
+			       (irda_queue_t *) info->discovery,
+			       info->discovery->data.daddr, NULL);
+
+		/* Keep state */
+		/* irlap_next_state(self, LAP_QUERY);  */
+
+		break;
+	case RECV_DISCOVERY_XID_CMD:
+		/* Yes, it is possible to receive those frames in this mode.
+		 * Note that most often the last discovery request won't
+		 * occur here but in NDM state (see my comment there).
+		 * What should we do ?
+		 * Not much. We are currently performing our own discovery,
+		 * therefore we can't answer those frames. We don't want
+		 * to change state either. We just pass the info to
+		 * IrLMP who will put it in the log (and post an event).
+		 * Jean II
+		 */
+
+		IRDA_ASSERT(info != NULL, return -1;);
+
+		pr_debug("%s(), Receiving discovery request (s = %d) while performing discovery :-(\n",
+			 __func__, info->s);
+
+		/* Last discovery request ? */
+		if (info->s == 0xff)
+			irlap_discovery_indication(self, info->discovery);
+		break;
+	case SLOT_TIMER_EXPIRED:
+		/*
+		 * Wait a little longer if we detect an incoming frame. This
+		 * is not mentioned in the spec, but is a good thing to do,
+		 * since we want to work even with devices that violate the
+		 * timing requirements.
+		 */
+		if (irda_device_is_receiving(self->netdev) && !self->add_wait) {
+			pr_debug("%s(), device is slow to answer, waiting some more!\n",
+				 __func__);
+			irlap_start_slot_timer(self, msecs_to_jiffies(10));
+			self->add_wait = TRUE;
+			return ret;
+		}
+		self->add_wait = FALSE;
+
+		if (self->s < self->S) {
+			irlap_send_discovery_xid_frame(self, self->S,
+						       self->s, TRUE,
+						       self->discovery_cmd);
+			self->s++;
+			irlap_start_slot_timer(self, self->slot_timeout);
+
+			/* Keep state */
+			irlap_next_state(self, LAP_QUERY);
+		} else {
+			/* This is the final slot! */
+			irlap_send_discovery_xid_frame(self, self->S, 0xff,
+						       TRUE,
+						       self->discovery_cmd);
+
+			/* Always switch state before calling upper layers */
+			irlap_next_state(self, LAP_NDM);
+
+			/*
+			 *  We are now finished with the discovery procedure,
+			 *  so now we must return the results
+			 */
+			irlap_discovery_confirm(self, self->discovery_log);
+
+			/* IrLMP should now have taken care of the log */
+			self->discovery_log = NULL;
+		}
+		break;
+	default:
+		pr_debug("%s(), Unknown event %s\n", __func__,
+			 irlap_event[event]);
+
+		ret = -1;
+		break;
+	}
+	return ret;
+}
+
+/*
+ * Function irlap_state_reply (self, event, skb, info)
+ *
+ *    REPLY, we have received a XID discovery frame from a device and we
+ *    are waiting for the right time slot to send a response XID frame
+ *
+ */
+static int irlap_state_reply(struct irlap_cb *self, IRLAP_EVENT event,
+			     struct sk_buff *skb, struct irlap_info *info)
+{
+	discovery_t *discovery_rsp;
+	int ret=0;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return -1;);
+
+	switch (event) {
+	case QUERY_TIMER_EXPIRED:
+		pr_debug("%s(), QUERY_TIMER_EXPIRED <%ld>\n",
+			 __func__, jiffies);
+		irlap_next_state(self, LAP_NDM);
+		break;
+	case RECV_DISCOVERY_XID_CMD:
+		IRDA_ASSERT(info != NULL, return -1;);
+		/* Last frame? */
+		if (info->s == 0xff) {
+			del_timer(&self->query_timer);
+
+			/* info->log.condition = REMOTE; */
+
+			/* Always switch state before calling upper layers */
+			irlap_next_state(self, LAP_NDM);
+
+			irlap_discovery_indication(self, info->discovery);
+		} else {
+			/* If it's our slot, send our reply */
+			if ((info->s >= self->slot) && (!self->frame_sent)) {
+				discovery_rsp = irlmp_get_discovery_response();
+				discovery_rsp->data.daddr = info->daddr;
+
+				irlap_send_discovery_xid_frame(self, info->S,
+							       self->slot,
+							       FALSE,
+							       discovery_rsp);
+
+				self->frame_sent = TRUE;
+			}
+			/* Readjust our timer to accommodate devices
+			 * doing faster or slower discovery than us...
+			 * Jean II */
+			irlap_start_query_timer(self, info->S, info->s);
+
+			/* Keep state */
+			//irlap_next_state(self, LAP_REPLY);
+		}
+		break;
+	default:
+		pr_debug("%s(), Unknown event %d, %s\n", __func__,
+			 event, irlap_event[event]);
+
+		ret = -1;
+		break;
+	}
+	return ret;
+}
+
+/*
+ * Function irlap_state_conn (event, skb, info)
+ *
+ *    CONN, we have received a SNRM command and is waiting for the upper
+ *    layer to accept or refuse connection
+ *
+ */
+static int irlap_state_conn(struct irlap_cb *self, IRLAP_EVENT event,
+			    struct sk_buff *skb, struct irlap_info *info)
+{
+	int ret = 0;
+
+	pr_debug("%s(), event=%s\n", __func__, irlap_event[event]);
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return -1;);
+
+	switch (event) {
+	case CONNECT_RESPONSE:
+		skb_pull(skb, sizeof(struct snrm_frame));
+
+		IRDA_ASSERT(self->netdev != NULL, return -1;);
+
+		irlap_qos_negotiate(self, skb);
+
+		irlap_initiate_connection_state(self);
+
+		/*
+		 * Applying the parameters now will make sure we change speed
+		 * *after* we have sent the next frame
+		 */
+		irlap_apply_connection_parameters(self, FALSE);
+
+		/*
+		 * Sending this frame will force a speed change after it has
+		 * been sent (i.e. the frame will be sent at 9600).
+		 */
+		irlap_send_ua_response_frame(self, &self->qos_rx);
+
+#if 0
+		/*
+		 * We are allowed to send two frames, but this may increase
+		 * the connect latency, so lets not do it for now.
+		 */
+		/* This is full of good intentions, but doesn't work in
+		 * practice.
+		 * After sending the first UA response, we switch the
+		 * dongle to the negotiated speed, which is usually
+		 * different than 9600 kb/s.
+		 * From there, there is two solutions :
+		 * 1) The other end has received the first UA response :
+		 * it will set up the connection, move to state LAP_NRM_P,
+		 * and will ignore and drop the second UA response.
+		 * Actually, it's even worse : the other side will almost
+		 * immediately send a RR that will likely collide with the
+		 * UA response (depending on negotiated turnaround).
+		 * 2) The other end has not received the first UA response,
+		 * will stay at 9600 and will never see the second UA response.
+		 * Jean II */
+		irlap_send_ua_response_frame(self, &self->qos_rx);
+#endif
+
+		/*
+		 *  The WD-timer could be set to the duration of the P-timer
+		 *  for this case, but it is recommended to use twice the
+		 *  value (note 3 IrLAP p. 60).
+		 */
+		irlap_start_wd_timer(self, self->wd_timeout);
+		irlap_next_state(self, LAP_NRM_S);
+
+		break;
+	case RECV_DISCOVERY_XID_CMD:
+		pr_debug("%s(), event RECV_DISCOVER_XID_CMD!\n",
+			 __func__);
+		irlap_next_state(self, LAP_NDM);
+
+		break;
+	case DISCONNECT_REQUEST:
+		pr_debug("%s(), Disconnect request!\n", __func__);
+		irlap_send_dm_frame(self);
+		irlap_next_state( self, LAP_NDM);
+		irlap_disconnect_indication(self, LAP_DISC_INDICATION);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %d, %s\n", __func__,
+			 event, irlap_event[event]);
+
+		ret = -1;
+		break;
+	}
+
+	return ret;
+}
+
+/*
+ * Function irlap_state_setup (event, skb, frame)
+ *
+ *    SETUP state, The local layer has transmitted a SNRM command frame to
+ *    a remote peer layer and is awaiting a reply .
+ *
+ */
+static int irlap_state_setup(struct irlap_cb *self, IRLAP_EVENT event,
+			     struct sk_buff *skb, struct irlap_info *info)
+{
+	int ret = 0;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return -1;);
+
+	switch (event) {
+	case FINAL_TIMER_EXPIRED:
+		if (self->retry_count < self->N3) {
+/*
+ *  Perform random backoff, Wait a random number of time units, minimum
+ *  duration half the time taken to transmitt a SNRM frame, maximum duration
+ *  1.5 times the time taken to transmit a SNRM frame. So this time should
+ *  between 15 msecs and 45 msecs.
+ */
+			irlap_start_backoff_timer(self, msecs_to_jiffies(20 +
+							(jiffies % 30)));
+		} else {
+			/* Always switch state before calling upper layers */
+			irlap_next_state(self, LAP_NDM);
+
+			irlap_disconnect_indication(self, LAP_FOUND_NONE);
+		}
+		break;
+	case BACKOFF_TIMER_EXPIRED:
+		irlap_send_snrm_frame(self, &self->qos_rx);
+		irlap_start_final_timer(self, self->final_timeout);
+		self->retry_count++;
+		break;
+	case RECV_SNRM_CMD:
+		pr_debug("%s(), SNRM battle!\n", __func__);
+
+		IRDA_ASSERT(skb != NULL, return 0;);
+		IRDA_ASSERT(info != NULL, return 0;);
+
+		/*
+		 *  The device with the largest device address wins the battle
+		 *  (both have sent a SNRM command!)
+		 */
+		if (info &&(info->daddr > self->saddr)) {
+			del_timer(&self->final_timer);
+			irlap_initiate_connection_state(self);
+
+			IRDA_ASSERT(self->netdev != NULL, return -1;);
+
+			skb_pull(skb, sizeof(struct snrm_frame));
+
+			irlap_qos_negotiate(self, skb);
+
+			/* Send UA frame and then change link settings */
+			irlap_apply_connection_parameters(self, FALSE);
+			irlap_send_ua_response_frame(self, &self->qos_rx);
+
+			irlap_next_state(self, LAP_NRM_S);
+			irlap_connect_confirm(self, skb);
+
+			/*
+			 *  The WD-timer could be set to the duration of the
+			 *  P-timer for this case, but it is recommended
+			 *  to use twice the value (note 3 IrLAP p. 60).
+			 */
+			irlap_start_wd_timer(self, self->wd_timeout);
+		} else {
+			/* We just ignore the other device! */
+			irlap_next_state(self, LAP_SETUP);
+		}
+		break;
+	case RECV_UA_RSP:
+		/* Stop F-timer */
+		del_timer(&self->final_timer);
+
+		/* Initiate connection state */
+		irlap_initiate_connection_state(self);
+
+		/* Negotiate connection parameters */
+		IRDA_ASSERT(skb->len > 10, return -1;);
+
+		skb_pull(skb, sizeof(struct ua_frame));
+
+		IRDA_ASSERT(self->netdev != NULL, return -1;);
+
+		irlap_qos_negotiate(self, skb);
+
+		/* Set the new link setting *now* (before the rr frame) */
+		irlap_apply_connection_parameters(self, TRUE);
+		self->retry_count = 0;
+
+		/* Wait for turnaround time to give a chance to the other
+		 * device to be ready to receive us.
+		 * Note : the time to switch speed is typically larger
+		 * than the turnaround time, but as we don't have the other
+		 * side speed switch time, that's our best guess...
+		 * Jean II */
+		irlap_wait_min_turn_around(self, &self->qos_tx);
+
+		/* This frame will actually be sent at the new speed */
+		irlap_send_rr_frame(self, CMD_FRAME);
+
+		/* The timer is set to half the normal timer to quickly
+		 * detect a failure to negotiate the new connection
+		 * parameters. IrLAP 6.11.3.2, note 3.
+		 * Note that currently we don't process this failure
+		 * properly, as we should do a quick disconnect.
+		 * Jean II */
+		irlap_start_final_timer(self, self->final_timeout/2);
+		irlap_next_state(self, LAP_NRM_P);
+
+		irlap_connect_confirm(self, skb);
+		break;
+	case RECV_DM_RSP:     /* FALLTHROUGH */
+	case RECV_DISC_CMD:
+		del_timer(&self->final_timer);
+		irlap_next_state(self, LAP_NDM);
+
+		irlap_disconnect_indication(self, LAP_DISC_INDICATION);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %d, %s\n", __func__,
+			 event, irlap_event[event]);
+
+		ret = -1;
+		break;
+	}
+	return ret;
+}
+
+/*
+ * Function irlap_state_offline (self, event, skb, info)
+ *
+ *    OFFLINE state, not used for now!
+ *
+ */
+static int irlap_state_offline(struct irlap_cb *self, IRLAP_EVENT event,
+			       struct sk_buff *skb, struct irlap_info *info)
+{
+	pr_debug("%s(), Unknown event\n", __func__);
+
+	return -1;
+}
+
+/*
+ * Function irlap_state_xmit_p (self, event, skb, info)
+ *
+ *    XMIT, Only the primary station has right to transmit, and we
+ *    therefore do not expect to receive any transmissions from other
+ *    stations.
+ *
+ */
+static int irlap_state_xmit_p(struct irlap_cb *self, IRLAP_EVENT event,
+			      struct sk_buff *skb, struct irlap_info *info)
+{
+	int ret = 0;
+
+	switch (event) {
+	case SEND_I_CMD:
+		/*
+		 *  Only send frame if send-window > 0.
+		 */
+		if ((self->window > 0) && (!self->remote_busy)) {
+			int nextfit;
+#ifdef CONFIG_IRDA_DYNAMIC_WINDOW
+			struct sk_buff *skb_next;
+
+			/* With DYNAMIC_WINDOW, we keep the window size
+			 * maximum, and adapt on the packets we are sending.
+			 * At 115k, we can send only 2 packets of 2048 bytes
+			 * in a 500 ms turnaround. Without this option, we
+			 * would always limit the window to 2. With this
+			 * option, if we send smaller packets, we can send
+			 * up to 7 of them (always depending on QoS).
+			 * Jean II */
+
+			/* Look at the next skb. This is safe, as we are
+			 * the only consumer of the Tx queue (if we are not,
+			 * we have other problems) - Jean II */
+			skb_next = skb_peek(&self->txq);
+
+			/* Check if a subsequent skb exist and would fit in
+			 * the current window (with respect to turnaround
+			 * time).
+			 * This allow us to properly mark the current packet
+			 * with the pf bit, to avoid falling back on the
+			 * second test below, and avoid waiting the
+			 * end of the window and sending a extra RR.
+			 * Note : (skb_next != NULL) <=> (skb_queue_len() > 0)
+			 * Jean II */
+			nextfit = ((skb_next != NULL) &&
+				   ((skb_next->len + skb->len) <=
+				    self->bytes_left));
+
+			/*
+			 * The current packet may not fit ! Because of test
+			 * above, this should not happen any more !!!
+			 *  Test if we have transmitted more bytes over the
+			 *  link than its possible to do with the current
+			 *  speed and turn-around-time.
+			 */
+			if((!nextfit) && (skb->len > self->bytes_left)) {
+				pr_debug("%s(), Not allowed to transmit more bytes!\n",
+					 __func__);
+				/* Requeue the skb */
+				skb_queue_head(&self->txq, skb_get(skb));
+				/*
+				 *  We should switch state to LAP_NRM_P, but
+				 *  that is not possible since we must be sure
+				 *  that we poll the other side. Since we have
+				 *  used up our time, the poll timer should
+				 *  trigger anyway now, so we just wait for it
+				 *  DB
+				 */
+				/*
+				 * Sorry, but that's not totally true. If
+				 * we send 2000B packets, we may wait another
+				 * 1000B until our turnaround expire. That's
+				 * why we need to be proactive in avoiding
+				 * coming here. - Jean II
+				 */
+				return -EPROTO;
+			}
+
+			/* Subtract space used by this skb */
+			self->bytes_left -= skb->len;
+#else	/* CONFIG_IRDA_DYNAMIC_WINDOW */
+			/* Window has been adjusted for the max packet
+			 * size, so much simpler... - Jean II */
+			nextfit = !skb_queue_empty(&self->txq);
+#endif	/* CONFIG_IRDA_DYNAMIC_WINDOW */
+			/*
+			 *  Send data with poll bit cleared only if window > 1
+			 *  and there is more frames after this one to be sent
+			 */
+			if ((self->window > 1) && (nextfit)) {
+				/* More packet to send in current window */
+				irlap_send_data_primary(self, skb);
+				irlap_next_state(self, LAP_XMIT_P);
+			} else {
+				/* Final packet of window */
+				irlap_send_data_primary_poll(self, skb);
+
+				/*
+				 * Make sure state machine does not try to send
+				 * any more frames
+				 */
+				ret = -EPROTO;
+			}
+#ifdef CONFIG_IRDA_FAST_RR
+			/* Peer may want to reply immediately */
+			self->fast_RR = FALSE;
+#endif /* CONFIG_IRDA_FAST_RR */
+		} else {
+			pr_debug("%s(), Unable to send! remote busy?\n",
+				 __func__);
+			skb_queue_head(&self->txq, skb_get(skb));
+
+			/*
+			 *  The next ret is important, because it tells
+			 *  irlap_next_state _not_ to deliver more frames
+			 */
+			ret = -EPROTO;
+		}
+		break;
+	case POLL_TIMER_EXPIRED:
+		pr_debug("%s(), POLL_TIMER_EXPIRED <%ld>\n",
+			 __func__, jiffies);
+		irlap_send_rr_frame(self, CMD_FRAME);
+		/* Return to NRM properly - Jean II  */
+		self->window = self->window_size;
+#ifdef CONFIG_IRDA_DYNAMIC_WINDOW
+		/* Allowed to transmit a maximum number of bytes again. */
+		self->bytes_left = self->line_capacity;
+#endif /* CONFIG_IRDA_DYNAMIC_WINDOW */
+		irlap_start_final_timer(self, self->final_timeout);
+		irlap_next_state(self, LAP_NRM_P);
+		break;
+	case DISCONNECT_REQUEST:
+		del_timer(&self->poll_timer);
+		irlap_wait_min_turn_around(self, &self->qos_tx);
+		irlap_send_disc_frame(self);
+		irlap_flush_all_queues(self);
+		irlap_start_final_timer(self, self->final_timeout);
+		self->retry_count = 0;
+		irlap_next_state(self, LAP_PCLOSE);
+		break;
+	case DATA_REQUEST:
+		/* Nothing to do, irlap_do_event() will send the packet
+		 * when we return... - Jean II */
+		break;
+	default:
+		pr_debug("%s(), Unknown event %s\n",
+			 __func__, irlap_event[event]);
+
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
+
+/*
+ * Function irlap_state_pclose (event, skb, info)
+ *
+ *    PCLOSE state
+ */
+static int irlap_state_pclose(struct irlap_cb *self, IRLAP_EVENT event,
+			      struct sk_buff *skb, struct irlap_info *info)
+{
+	int ret = 0;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return -1;);
+
+	switch (event) {
+	case RECV_UA_RSP: /* FALLTHROUGH */
+	case RECV_DM_RSP:
+		del_timer(&self->final_timer);
+
+		/* Set new link parameters */
+		irlap_apply_default_connection_parameters(self);
+
+		/* Always switch state before calling upper layers */
+		irlap_next_state(self, LAP_NDM);
+
+		irlap_disconnect_indication(self, LAP_DISC_INDICATION);
+		break;
+	case FINAL_TIMER_EXPIRED:
+		if (self->retry_count < self->N3) {
+			irlap_wait_min_turn_around(self, &self->qos_tx);
+			irlap_send_disc_frame(self);
+			irlap_start_final_timer(self, self->final_timeout);
+			self->retry_count++;
+			/* Keep state */
+		} else {
+			irlap_apply_default_connection_parameters(self);
+
+			/*  Always switch state before calling upper layers */
+			irlap_next_state(self, LAP_NDM);
+
+			irlap_disconnect_indication(self, LAP_NO_RESPONSE);
+		}
+		break;
+	default:
+		pr_debug("%s(), Unknown event %d\n", __func__, event);
+
+		ret = -1;
+		break;
+	}
+	return ret;
+}
+
+/*
+ * Function irlap_state_nrm_p (self, event, skb, info)
+ *
+ *   NRM_P (Normal Response Mode as Primary), The primary station has given
+ *   permissions to a secondary station to transmit IrLAP resonse frames
+ *   (by sending a frame with the P bit set). The primary station will not
+ *   transmit any frames and is expecting to receive frames only from the
+ *   secondary to which transmission permissions has been given.
+ */
+static int irlap_state_nrm_p(struct irlap_cb *self, IRLAP_EVENT event,
+			     struct sk_buff *skb, struct irlap_info *info)
+{
+	int ret = 0;
+	int ns_status;
+	int nr_status;
+
+	switch (event) {
+	case RECV_I_RSP: /* Optimize for the common case */
+		if (unlikely(skb->len <= LAP_ADDR_HEADER + LAP_CTRL_HEADER)) {
+			/*
+			 * Input validation check: a stir4200/mcp2150
+			 * combination sometimes results in an empty i:rsp.
+			 * This makes no sense; we can just ignore the frame
+			 * and send an rr:cmd immediately. This happens before
+			 * changing nr or ns so triggers a retransmit
+			 */
+			irlap_wait_min_turn_around(self, &self->qos_tx);
+			irlap_send_rr_frame(self, CMD_FRAME);
+			/* Keep state */
+			break;
+		}
+		/* FIXME: must check for remote_busy below */
+#ifdef CONFIG_IRDA_FAST_RR
+		/*
+		 *  Reset the fast_RR so we can use the fast RR code with
+		 *  full speed the next time since peer may have more frames
+		 *  to transmitt
+		 */
+		self->fast_RR = FALSE;
+#endif /* CONFIG_IRDA_FAST_RR */
+		IRDA_ASSERT( info != NULL, return -1;);
+
+		ns_status = irlap_validate_ns_received(self, info->ns);
+		nr_status = irlap_validate_nr_received(self, info->nr);
+
+		/*
+		 *  Check for expected I(nformation) frame
+		 */
+		if ((ns_status == NS_EXPECTED) && (nr_status == NR_EXPECTED)) {
+
+			/* Update Vr (next frame for us to receive) */
+			self->vr = (self->vr + 1) % 8;
+
+			/* Update Nr received, cleanup our retry queue */
+			irlap_update_nr_received(self, info->nr);
+
+			/*
+			 *  Got expected NR, so reset the
+			 *  retry_count. This is not done by IrLAP spec,
+			 *  which is strange!
+			 */
+			self->retry_count = 0;
+			self->ack_required = TRUE;
+
+			/*  poll bit cleared?  */
+			if (!info->pf) {
+				/* Keep state, do not move this line */
+				irlap_next_state(self, LAP_NRM_P);
+
+				irlap_data_indication(self, skb, FALSE);
+			} else {
+				/* No longer waiting for pf */
+				del_timer(&self->final_timer);
+
+				irlap_wait_min_turn_around(self, &self->qos_tx);
+
+				/* Call higher layer *before* changing state
+				 * to give them a chance to send data in the
+				 * next LAP frame.
+				 * Jean II */
+				irlap_data_indication(self, skb, FALSE);
+
+				/* XMIT states are the most dangerous state
+				 * to be in, because user requests are
+				 * processed directly and may change state.
+				 * On the other hand, in NDM_P, those
+				 * requests are queued and we will process
+				 * them when we return to irlap_do_event().
+				 * Jean II
+				 */
+				irlap_next_state(self, LAP_XMIT_P);
+
+				/* This is the last frame.
+				 * Make sure it's always called in XMIT state.
+				 * - Jean II */
+				irlap_start_poll_timer(self, self->poll_timeout);
+			}
+			break;
+
+		}
+		/* Unexpected next to send (Ns) */
+		if ((ns_status == NS_UNEXPECTED) && (nr_status == NR_EXPECTED))
+		{
+			if (!info->pf) {
+				irlap_update_nr_received(self, info->nr);
+
+				/*
+				 *  Wait until the last frame before doing
+				 *  anything
+				 */
+
+				/* Keep state */
+				irlap_next_state(self, LAP_NRM_P);
+			} else {
+				pr_debug("%s(), missing or duplicate frame!\n",
+					 __func__);
+
+				/* Update Nr received */
+				irlap_update_nr_received(self, info->nr);
+
+				irlap_wait_min_turn_around(self, &self->qos_tx);
+				irlap_send_rr_frame(self, CMD_FRAME);
+
+				self->ack_required = FALSE;
+
+				irlap_start_final_timer(self, self->final_timeout);
+				irlap_next_state(self, LAP_NRM_P);
+			}
+			break;
+		}
+		/*
+		 *  Unexpected next to receive (Nr)
+		 */
+		if ((ns_status == NS_EXPECTED) && (nr_status == NR_UNEXPECTED))
+		{
+			if (info->pf) {
+				self->vr = (self->vr + 1) % 8;
+
+				/* Update Nr received */
+				irlap_update_nr_received(self, info->nr);
+
+				/* Resend rejected frames */
+				irlap_resend_rejected_frames(self, CMD_FRAME);
+
+				self->ack_required = FALSE;
+
+				/* Make sure we account for the time
+				 * to transmit our frames. See comemnts
+				 * in irlap_send_data_primary_poll().
+				 * Jean II */
+				irlap_start_final_timer(self, 2 * self->final_timeout);
+
+				/* Keep state, do not move this line */
+				irlap_next_state(self, LAP_NRM_P);
+
+				irlap_data_indication(self, skb, FALSE);
+			} else {
+				/*
+				 *  Do not resend frames until the last
+				 *  frame has arrived from the other
+				 *  device. This is not documented in
+				 *  IrLAP!!
+				 */
+				self->vr = (self->vr + 1) % 8;
+
+				/* Update Nr received */
+				irlap_update_nr_received(self, info->nr);
+
+				self->ack_required = FALSE;
+
+				/* Keep state, do not move this line!*/
+				irlap_next_state(self, LAP_NRM_P);
+
+				irlap_data_indication(self, skb, FALSE);
+			}
+			break;
+		}
+		/*
+		 *  Unexpected next to send (Ns) and next to receive (Nr)
+		 *  Not documented by IrLAP!
+		 */
+		if ((ns_status == NS_UNEXPECTED) &&
+		    (nr_status == NR_UNEXPECTED))
+		{
+			pr_debug("%s(), unexpected nr and ns!\n",
+				 __func__);
+			if (info->pf) {
+				/* Resend rejected frames */
+				irlap_resend_rejected_frames(self, CMD_FRAME);
+
+				/* Give peer some time to retransmit!
+				 * But account for our own Tx. */
+				irlap_start_final_timer(self, 2 * self->final_timeout);
+
+				/* Keep state, do not move this line */
+				irlap_next_state(self, LAP_NRM_P);
+			} else {
+				/* Update Nr received */
+				/* irlap_update_nr_received( info->nr); */
+
+				self->ack_required = FALSE;
+			}
+			break;
+		}
+
+		/*
+		 *  Invalid NR or NS
+		 */
+		if ((nr_status == NR_INVALID) || (ns_status == NS_INVALID)) {
+			if (info->pf) {
+				del_timer(&self->final_timer);
+
+				irlap_next_state(self, LAP_RESET_WAIT);
+
+				irlap_disconnect_indication(self, LAP_RESET_INDICATION);
+				self->xmitflag = TRUE;
+			} else {
+				del_timer(&self->final_timer);
+
+				irlap_disconnect_indication(self, LAP_RESET_INDICATION);
+
+				self->xmitflag = FALSE;
+			}
+			break;
+		}
+		pr_debug("%s(), Not implemented!\n", __func__);
+		pr_debug("%s(), event=%s, ns_status=%d, nr_status=%d\n",
+			 __func__, irlap_event[event], ns_status, nr_status);
+		break;
+	case RECV_UI_FRAME:
+		/* Poll bit cleared? */
+		if (!info->pf) {
+			irlap_data_indication(self, skb, TRUE);
+			irlap_next_state(self, LAP_NRM_P);
+		} else {
+			del_timer(&self->final_timer);
+			irlap_data_indication(self, skb, TRUE);
+			irlap_next_state(self, LAP_XMIT_P);
+			pr_debug("%s: RECV_UI_FRAME: next state %s\n",
+				 __func__, irlap_state[self->state]);
+			irlap_start_poll_timer(self, self->poll_timeout);
+		}
+		break;
+	case RECV_RR_RSP:
+		/*
+		 *  If you get a RR, the remote isn't busy anymore,
+		 *  no matter what the NR
+		 */
+		self->remote_busy = FALSE;
+
+		/* Stop final timer */
+		del_timer(&self->final_timer);
+
+		/*
+		 *  Nr as expected?
+		 */
+		ret = irlap_validate_nr_received(self, info->nr);
+		if (ret == NR_EXPECTED) {
+			/* Update Nr received */
+			irlap_update_nr_received(self, info->nr);
+
+			/*
+			 *  Got expected NR, so reset the retry_count. This
+			 *  is not done by the IrLAP standard , which is
+			 *  strange! DB.
+			 */
+			self->retry_count = 0;
+			irlap_wait_min_turn_around(self, &self->qos_tx);
+
+			irlap_next_state(self, LAP_XMIT_P);
+
+			/* Start poll timer */
+			irlap_start_poll_timer(self, self->poll_timeout);
+		} else if (ret == NR_UNEXPECTED) {
+			IRDA_ASSERT(info != NULL, return -1;);
+			/*
+			 *  Unexpected nr!
+			 */
+
+			/* Update Nr received */
+			irlap_update_nr_received(self, info->nr);
+
+			pr_debug("RECV_RR_FRAME: Retrans:%d, nr=%d, va=%d, vs=%d, vr=%d\n",
+				 self->retry_count, info->nr, self->va,
+				 self->vs, self->vr);
+
+			/* Resend rejected frames */
+			irlap_resend_rejected_frames(self, CMD_FRAME);
+			irlap_start_final_timer(self, self->final_timeout * 2);
+
+			irlap_next_state(self, LAP_NRM_P);
+		} else if (ret == NR_INVALID) {
+			pr_debug("%s(), Received RR with invalid nr !\n",
+				 __func__);
+
+			irlap_next_state(self, LAP_RESET_WAIT);
+
+			irlap_disconnect_indication(self, LAP_RESET_INDICATION);
+			self->xmitflag = TRUE;
+		}
+		break;
+	case RECV_RNR_RSP:
+		IRDA_ASSERT(info != NULL, return -1;);
+
+		/* Stop final timer */
+		del_timer(&self->final_timer);
+		self->remote_busy = TRUE;
+
+		/* Update Nr received */
+		irlap_update_nr_received(self, info->nr);
+		irlap_next_state(self, LAP_XMIT_P);
+
+		/* Start poll timer */
+		irlap_start_poll_timer(self, self->poll_timeout);
+		break;
+	case RECV_FRMR_RSP:
+		del_timer(&self->final_timer);
+		self->xmitflag = TRUE;
+		irlap_next_state(self, LAP_RESET_WAIT);
+		irlap_reset_indication(self);
+		break;
+	case FINAL_TIMER_EXPIRED:
+		/*
+		 *  We are allowed to wait for additional 300 ms if
+		 *  final timer expires when we are in the middle
+		 *  of receiving a frame (page 45, IrLAP). Check that
+		 *  we only do this once for each frame.
+		 */
+		if (irda_device_is_receiving(self->netdev) && !self->add_wait) {
+			pr_debug("FINAL_TIMER_EXPIRED when receiving a frame! Waiting a little bit more!\n");
+			irlap_start_final_timer(self, msecs_to_jiffies(300));
+
+			/*
+			 *  Don't allow this to happen one more time in a row,
+			 *  or else we can get a pretty tight loop here if
+			 *  if we only receive half a frame. DB.
+			 */
+			self->add_wait = TRUE;
+			break;
+		}
+		self->add_wait = FALSE;
+
+		/* N2 is the disconnect timer. Until we reach it, we retry */
+		if (self->retry_count < self->N2) {
+			if (skb_peek(&self->wx_list) == NULL) {
+				/* Retry sending the pf bit to the secondary */
+				pr_debug("nrm_p: resending rr");
+				irlap_wait_min_turn_around(self, &self->qos_tx);
+				irlap_send_rr_frame(self, CMD_FRAME);
+			} else {
+				pr_debug("nrm_p: resend frames");
+				irlap_resend_rejected_frames(self, CMD_FRAME);
+			}
+
+			irlap_start_final_timer(self, self->final_timeout);
+			self->retry_count++;
+			pr_debug("irlap_state_nrm_p: FINAL_TIMER_EXPIRED: retry_count=%d\n",
+				 self->retry_count);
+
+			/* Early warning event. I'm using a pretty liberal
+			 * interpretation of the spec and generate an event
+			 * every time the timer is multiple of N1 (and not
+			 * only the first time). This allow application
+			 * to know precisely if connectivity restart...
+			 * Jean II */
+			if((self->retry_count % self->N1) == 0)
+				irlap_status_indication(self,
+							STATUS_NO_ACTIVITY);
+
+			/* Keep state */
+		} else {
+			irlap_apply_default_connection_parameters(self);
+
+			/* Always switch state before calling upper layers */
+			irlap_next_state(self, LAP_NDM);
+			irlap_disconnect_indication(self, LAP_NO_RESPONSE);
+		}
+		break;
+	case RECV_REJ_RSP:
+		irlap_update_nr_received(self, info->nr);
+		if (self->remote_busy) {
+			irlap_wait_min_turn_around(self, &self->qos_tx);
+			irlap_send_rr_frame(self, CMD_FRAME);
+		} else
+			irlap_resend_rejected_frames(self, CMD_FRAME);
+		irlap_start_final_timer(self, 2 * self->final_timeout);
+		break;
+	case RECV_SREJ_RSP:
+		irlap_update_nr_received(self, info->nr);
+		if (self->remote_busy) {
+			irlap_wait_min_turn_around(self, &self->qos_tx);
+			irlap_send_rr_frame(self, CMD_FRAME);
+		} else
+			irlap_resend_rejected_frame(self, CMD_FRAME);
+		irlap_start_final_timer(self, 2 * self->final_timeout);
+		break;
+	case RECV_RD_RSP:
+		pr_debug("%s(), RECV_RD_RSP\n", __func__);
+
+		irlap_flush_all_queues(self);
+		irlap_next_state(self, LAP_XMIT_P);
+		/* Call back the LAP state machine to do a proper disconnect */
+		irlap_disconnect_request(self);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %s\n",
+			 __func__, irlap_event[event]);
+
+		ret = -1;
+		break;
+	}
+	return ret;
+}
+
+/*
+ * Function irlap_state_reset_wait (event, skb, info)
+ *
+ *    We have informed the service user of a reset condition, and is
+ *    awaiting reset of disconnect request.
+ *
+ */
+static int irlap_state_reset_wait(struct irlap_cb *self, IRLAP_EVENT event,
+				  struct sk_buff *skb, struct irlap_info *info)
+{
+	int ret = 0;
+
+	pr_debug("%s(), event = %s\n", __func__, irlap_event[event]);
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return -1;);
+
+	switch (event) {
+	case RESET_REQUEST:
+		if (self->xmitflag) {
+			irlap_wait_min_turn_around(self, &self->qos_tx);
+			irlap_send_snrm_frame(self, NULL);
+			irlap_start_final_timer(self, self->final_timeout);
+			irlap_next_state(self, LAP_RESET);
+		} else {
+			irlap_start_final_timer(self, self->final_timeout);
+			irlap_next_state(self, LAP_RESET);
+		}
+		break;
+	case DISCONNECT_REQUEST:
+		irlap_wait_min_turn_around( self, &self->qos_tx);
+		irlap_send_disc_frame( self);
+		irlap_flush_all_queues( self);
+		irlap_start_final_timer( self, self->final_timeout);
+		self->retry_count = 0;
+		irlap_next_state( self, LAP_PCLOSE);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %s\n", __func__,
+			 irlap_event[event]);
+
+		ret = -1;
+		break;
+	}
+	return ret;
+}
+
+/*
+ * Function irlap_state_reset (self, event, skb, info)
+ *
+ *    We have sent a SNRM reset command to the peer layer, and is awaiting
+ *    reply.
+ *
+ */
+static int irlap_state_reset(struct irlap_cb *self, IRLAP_EVENT event,
+			     struct sk_buff *skb, struct irlap_info *info)
+{
+	int ret = 0;
+
+	pr_debug("%s(), event = %s\n", __func__, irlap_event[event]);
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return -1;);
+
+	switch (event) {
+	case RECV_DISC_CMD:
+		del_timer(&self->final_timer);
+
+		irlap_apply_default_connection_parameters(self);
+
+		/* Always switch state before calling upper layers */
+		irlap_next_state(self, LAP_NDM);
+
+		irlap_disconnect_indication(self, LAP_NO_RESPONSE);
+
+		break;
+	case RECV_UA_RSP:
+		del_timer(&self->final_timer);
+
+		/* Initiate connection state */
+		irlap_initiate_connection_state(self);
+
+		irlap_reset_confirm();
+
+		self->remote_busy = FALSE;
+
+		irlap_next_state(self, LAP_XMIT_P);
+
+		irlap_start_poll_timer(self, self->poll_timeout);
+
+		break;
+	case FINAL_TIMER_EXPIRED:
+		if (self->retry_count < 3) {
+			irlap_wait_min_turn_around(self, &self->qos_tx);
+
+			IRDA_ASSERT(self->netdev != NULL, return -1;);
+			irlap_send_snrm_frame(self, self->qos_dev);
+
+			self->retry_count++; /* Experimental!! */
+
+			irlap_start_final_timer(self, self->final_timeout);
+			irlap_next_state(self, LAP_RESET);
+		} else if (self->retry_count >= self->N3) {
+			irlap_apply_default_connection_parameters(self);
+
+			/* Always switch state before calling upper layers */
+			irlap_next_state(self, LAP_NDM);
+
+			irlap_disconnect_indication(self, LAP_NO_RESPONSE);
+		}
+		break;
+	case RECV_SNRM_CMD:
+		/*
+		 * SNRM frame is not allowed to contain an I-field in this
+		 * state
+		 */
+		if (!info) {
+			pr_debug("%s(), RECV_SNRM_CMD\n", __func__);
+			irlap_initiate_connection_state(self);
+			irlap_wait_min_turn_around(self, &self->qos_tx);
+			irlap_send_ua_response_frame(self, &self->qos_rx);
+			irlap_reset_confirm();
+			irlap_start_wd_timer(self, self->wd_timeout);
+			irlap_next_state(self, LAP_NDM);
+		} else {
+			pr_debug("%s(), SNRM frame contained an I field!\n",
+				 __func__);
+		}
+		break;
+	default:
+		pr_debug("%s(), Unknown event %s\n",
+			 __func__, irlap_event[event]);
+
+		ret = -1;
+		break;
+	}
+	return ret;
+}
+
+/*
+ * Function irlap_state_xmit_s (event, skb, info)
+ *
+ *   XMIT_S, The secondary station has been given the right to transmit,
+ *   and we therefore do not expect to receive any transmissions from other
+ *   stations.
+ */
+static int irlap_state_xmit_s(struct irlap_cb *self, IRLAP_EVENT event,
+			      struct sk_buff *skb, struct irlap_info *info)
+{
+	int ret = 0;
+
+	pr_debug("%s(), event=%s\n", __func__, irlap_event[event]);
+
+	IRDA_ASSERT(self != NULL, return -ENODEV;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return -EBADR;);
+
+	switch (event) {
+	case SEND_I_CMD:
+		/*
+		 *  Send frame only if send window > 0
+		 */
+		if ((self->window > 0) && (!self->remote_busy)) {
+			int nextfit;
+#ifdef CONFIG_IRDA_DYNAMIC_WINDOW
+			struct sk_buff *skb_next;
+
+			/*
+			 * Same deal as in irlap_state_xmit_p(), so see
+			 * the comments at that point.
+			 * We are the secondary, so there are only subtle
+			 * differences. - Jean II
+			 */
+
+			/* Check if a subsequent skb exist and would fit in
+			 * the current window (with respect to turnaround
+			 * time). - Jean II */
+			skb_next = skb_peek(&self->txq);
+			nextfit = ((skb_next != NULL) &&
+				   ((skb_next->len + skb->len) <=
+				    self->bytes_left));
+
+			/*
+			 *  Test if we have transmitted more bytes over the
+			 *  link than its possible to do with the current
+			 *  speed and turn-around-time.
+			 */
+			if((!nextfit) && (skb->len > self->bytes_left)) {
+				pr_debug("%s(), Not allowed to transmit more bytes!\n",
+					 __func__);
+				/* Requeue the skb */
+				skb_queue_head(&self->txq, skb_get(skb));
+
+				/*
+				 *  Switch to NRM_S, this is only possible
+				 *  when we are in secondary mode, since we
+				 *  must be sure that we don't miss any RR
+				 *  frames
+				 */
+				self->window = self->window_size;
+				self->bytes_left = self->line_capacity;
+				irlap_start_wd_timer(self, self->wd_timeout);
+
+				irlap_next_state(self, LAP_NRM_S);
+				/* Slight difference with primary :
+				 * here we would wait for the other side to
+				 * expire the turnaround. - Jean II */
+
+				return -EPROTO; /* Try again later */
+			}
+			/* Subtract space used by this skb */
+			self->bytes_left -= skb->len;
+#else	/* CONFIG_IRDA_DYNAMIC_WINDOW */
+			/* Window has been adjusted for the max packet
+			 * size, so much simpler... - Jean II */
+			nextfit = !skb_queue_empty(&self->txq);
+#endif /* CONFIG_IRDA_DYNAMIC_WINDOW */
+			/*
+			 *  Send data with final bit cleared only if window > 1
+			 *  and there is more frames to be sent
+			 */
+			if ((self->window > 1) && (nextfit)) {
+				irlap_send_data_secondary(self, skb);
+				irlap_next_state(self, LAP_XMIT_S);
+			} else {
+				irlap_send_data_secondary_final(self, skb);
+				irlap_next_state(self, LAP_NRM_S);
+
+				/*
+				 * Make sure state machine does not try to send
+				 * any more frames
+				 */
+				ret = -EPROTO;
+			}
+		} else {
+			pr_debug("%s(), Unable to send!\n", __func__);
+			skb_queue_head(&self->txq, skb_get(skb));
+			ret = -EPROTO;
+		}
+		break;
+	case DISCONNECT_REQUEST:
+		irlap_send_rd_frame(self);
+		irlap_flush_all_queues(self);
+		irlap_start_wd_timer(self, self->wd_timeout);
+		irlap_next_state(self, LAP_SCLOSE);
+		break;
+	case DATA_REQUEST:
+		/* Nothing to do, irlap_do_event() will send the packet
+		 * when we return... - Jean II */
+		break;
+	default:
+		pr_debug("%s(), Unknown event %s\n", __func__,
+			 irlap_event[event]);
+
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
+
+/*
+ * Function irlap_state_nrm_s (event, skb, info)
+ *
+ *    NRM_S (Normal Response Mode as Secondary) state, in this state we are
+ *    expecting to receive frames from the primary station
+ *
+ */
+static int irlap_state_nrm_s(struct irlap_cb *self, IRLAP_EVENT event,
+			     struct sk_buff *skb, struct irlap_info *info)
+{
+	int ns_status;
+	int nr_status;
+	int ret = 0;
+
+	pr_debug("%s(), event=%s\n", __func__, irlap_event[event]);
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return -1;);
+
+	switch (event) {
+	case RECV_I_CMD: /* Optimize for the common case */
+		/* FIXME: must check for remote_busy below */
+		pr_debug("%s(), event=%s nr=%d, vs=%d, ns=%d, vr=%d, pf=%d\n",
+			 __func__, irlap_event[event], info->nr,
+			 self->vs, info->ns, self->vr, info->pf);
+
+		self->retry_count = 0;
+
+		ns_status = irlap_validate_ns_received(self, info->ns);
+		nr_status = irlap_validate_nr_received(self, info->nr);
+		/*
+		 *  Check for expected I(nformation) frame
+		 */
+		if ((ns_status == NS_EXPECTED) && (nr_status == NR_EXPECTED)) {
+
+			/* Update Vr (next frame for us to receive) */
+			self->vr = (self->vr + 1) % 8;
+
+			/* Update Nr received */
+			irlap_update_nr_received(self, info->nr);
+
+			/*
+			 *  poll bit cleared?
+			 */
+			if (!info->pf) {
+
+				self->ack_required = TRUE;
+
+				/*
+				 *  Starting WD-timer here is optional, but
+				 *  not recommended. Note 6 IrLAP p. 83
+				 */
+#if 0
+				irda_start_timer(WD_TIMER, self->wd_timeout);
+#endif
+				/* Keep state, do not move this line */
+				irlap_next_state(self, LAP_NRM_S);
+
+				irlap_data_indication(self, skb, FALSE);
+				break;
+			} else {
+				/*
+				 *  We should wait before sending RR, and
+				 *  also before changing to XMIT_S
+				 *  state. (note 1, IrLAP p. 82)
+				 */
+				irlap_wait_min_turn_around(self, &self->qos_tx);
+
+				/*
+				 * Give higher layers a chance to
+				 * immediately reply with some data before
+				 * we decide if we should send a RR frame
+				 * or not
+				 */
+				irlap_data_indication(self, skb, FALSE);
+
+				/* Any pending data requests?  */
+				if (!skb_queue_empty(&self->txq) &&
+				    (self->window > 0))
+				{
+					self->ack_required = TRUE;
+
+					del_timer(&self->wd_timer);
+
+					irlap_next_state(self, LAP_XMIT_S);
+				} else {
+					irlap_send_rr_frame(self, RSP_FRAME);
+					irlap_start_wd_timer(self,
+							     self->wd_timeout);
+
+					/* Keep the state */
+					irlap_next_state(self, LAP_NRM_S);
+				}
+				break;
+			}
+		}
+		/*
+		 *  Check for Unexpected next to send (Ns)
+		 */
+		if ((ns_status == NS_UNEXPECTED) && (nr_status == NR_EXPECTED))
+		{
+			/* Unexpected next to send, with final bit cleared */
+			if (!info->pf) {
+				irlap_update_nr_received(self, info->nr);
+
+				irlap_start_wd_timer(self, self->wd_timeout);
+			} else {
+				/* Update Nr received */
+				irlap_update_nr_received(self, info->nr);
+
+				irlap_wait_min_turn_around(self, &self->qos_tx);
+				irlap_send_rr_frame(self, RSP_FRAME);
+
+				irlap_start_wd_timer(self, self->wd_timeout);
+			}
+			break;
+		}
+
+		/*
+		 *  Unexpected Next to Receive(NR) ?
+		 */
+		if ((ns_status == NS_EXPECTED) && (nr_status == NR_UNEXPECTED))
+		{
+			if (info->pf) {
+				pr_debug("RECV_I_RSP: frame(s) lost\n");
+
+				self->vr = (self->vr + 1) % 8;
+
+				/* Update Nr received */
+				irlap_update_nr_received(self, info->nr);
+
+				/* Resend rejected frames */
+				irlap_resend_rejected_frames(self, RSP_FRAME);
+
+				/* Keep state, do not move this line */
+				irlap_next_state(self, LAP_NRM_S);
+
+				irlap_data_indication(self, skb, FALSE);
+				irlap_start_wd_timer(self, self->wd_timeout);
+				break;
+			}
+			/*
+			 *  This is not documented in IrLAP!! Unexpected NR
+			 *  with poll bit cleared
+			 */
+			if (!info->pf) {
+				self->vr = (self->vr + 1) % 8;
+
+				/* Update Nr received */
+				irlap_update_nr_received(self, info->nr);
+
+				/* Keep state, do not move this line */
+				irlap_next_state(self, LAP_NRM_S);
+
+				irlap_data_indication(self, skb, FALSE);
+				irlap_start_wd_timer(self, self->wd_timeout);
+			}
+			break;
+		}
+
+		if (ret == NR_INVALID) {
+			pr_debug("NRM_S, NR_INVALID not implemented!\n");
+		}
+		if (ret == NS_INVALID) {
+			pr_debug("NRM_S, NS_INVALID not implemented!\n");
+		}
+		break;
+	case RECV_UI_FRAME:
+		/*
+		 *  poll bit cleared?
+		 */
+		if (!info->pf) {
+			irlap_data_indication(self, skb, TRUE);
+			irlap_next_state(self, LAP_NRM_S); /* Keep state */
+		} else {
+			/*
+			 *  Any pending data requests?
+			 */
+			if (!skb_queue_empty(&self->txq) &&
+			    (self->window > 0) && !self->remote_busy)
+			{
+				irlap_data_indication(self, skb, TRUE);
+
+				del_timer(&self->wd_timer);
+
+				irlap_next_state(self, LAP_XMIT_S);
+			} else {
+				irlap_data_indication(self, skb, TRUE);
+
+				irlap_wait_min_turn_around(self, &self->qos_tx);
+
+				irlap_send_rr_frame(self, RSP_FRAME);
+				self->ack_required = FALSE;
+
+				irlap_start_wd_timer(self, self->wd_timeout);
+
+				/* Keep the state */
+				irlap_next_state(self, LAP_NRM_S);
+			}
+		}
+		break;
+	case RECV_RR_CMD:
+		self->retry_count = 0;
+
+		/*
+		 *  Nr as expected?
+		 */
+		nr_status = irlap_validate_nr_received(self, info->nr);
+		if (nr_status == NR_EXPECTED) {
+			if (!skb_queue_empty(&self->txq) &&
+			    (self->window > 0)) {
+				self->remote_busy = FALSE;
+
+				/* Update Nr received */
+				irlap_update_nr_received(self, info->nr);
+				del_timer(&self->wd_timer);
+
+				irlap_wait_min_turn_around(self, &self->qos_tx);
+				irlap_next_state(self, LAP_XMIT_S);
+			} else {
+				self->remote_busy = FALSE;
+				/* Update Nr received */
+				irlap_update_nr_received(self, info->nr);
+				irlap_wait_min_turn_around(self, &self->qos_tx);
+				irlap_start_wd_timer(self, self->wd_timeout);
+
+				/* Note : if the link is idle (this case),
+				 * we never go in XMIT_S, so we never get a
+				 * chance to process any DISCONNECT_REQUEST.
+				 * Do it now ! - Jean II */
+				if (self->disconnect_pending) {
+					/* Disconnect */
+					irlap_send_rd_frame(self);
+					irlap_flush_all_queues(self);
+
+					irlap_next_state(self, LAP_SCLOSE);
+				} else {
+					/* Just send back pf bit */
+					irlap_send_rr_frame(self, RSP_FRAME);
+
+					irlap_next_state(self, LAP_NRM_S);
+				}
+			}
+		} else if (nr_status == NR_UNEXPECTED) {
+			self->remote_busy = FALSE;
+			irlap_update_nr_received(self, info->nr);
+			irlap_resend_rejected_frames(self, RSP_FRAME);
+
+			irlap_start_wd_timer(self, self->wd_timeout);
+
+			/* Keep state */
+			irlap_next_state(self, LAP_NRM_S);
+		} else {
+			pr_debug("%s(), invalid nr not implemented!\n",
+				 __func__);
+		}
+		break;
+	case RECV_SNRM_CMD:
+		/* SNRM frame is not allowed to contain an I-field */
+		if (!info) {
+			del_timer(&self->wd_timer);
+			pr_debug("%s(), received SNRM cmd\n", __func__);
+			irlap_next_state(self, LAP_RESET_CHECK);
+
+			irlap_reset_indication(self);
+		} else {
+			pr_debug("%s(), SNRM frame contained an I-field!\n",
+				 __func__);
+
+		}
+		break;
+	case RECV_REJ_CMD:
+		irlap_update_nr_received(self, info->nr);
+		if (self->remote_busy) {
+			irlap_wait_min_turn_around(self, &self->qos_tx);
+			irlap_send_rr_frame(self, RSP_FRAME);
+		} else
+			irlap_resend_rejected_frames(self, RSP_FRAME);
+		irlap_start_wd_timer(self, self->wd_timeout);
+		break;
+	case RECV_SREJ_CMD:
+		irlap_update_nr_received(self, info->nr);
+		if (self->remote_busy) {
+			irlap_wait_min_turn_around(self, &self->qos_tx);
+			irlap_send_rr_frame(self, RSP_FRAME);
+		} else
+			irlap_resend_rejected_frame(self, RSP_FRAME);
+		irlap_start_wd_timer(self, self->wd_timeout);
+		break;
+	case WD_TIMER_EXPIRED:
+		/*
+		 *  Wait until retry_count * n matches negotiated threshold/
+		 *  disconnect time (note 2 in IrLAP p. 82)
+		 *
+		 * Similar to irlap_state_nrm_p() -> FINAL_TIMER_EXPIRED
+		 * Note : self->wd_timeout = (self->final_timeout * 2),
+		 *   which explain why we use (self->N2 / 2) here !!!
+		 * Jean II
+		 */
+		pr_debug("%s(), retry_count = %d\n", __func__,
+			 self->retry_count);
+
+		if (self->retry_count < (self->N2 / 2)) {
+			/* No retry, just wait for primary */
+			irlap_start_wd_timer(self, self->wd_timeout);
+			self->retry_count++;
+
+			if((self->retry_count % (self->N1 / 2)) == 0)
+				irlap_status_indication(self,
+							STATUS_NO_ACTIVITY);
+		} else {
+			irlap_apply_default_connection_parameters(self);
+
+			/* Always switch state before calling upper layers */
+			irlap_next_state(self, LAP_NDM);
+			irlap_disconnect_indication(self, LAP_NO_RESPONSE);
+		}
+		break;
+	case RECV_DISC_CMD:
+		/* Always switch state before calling upper layers */
+		irlap_next_state(self, LAP_NDM);
+
+		/* Send disconnect response */
+		irlap_wait_min_turn_around(self, &self->qos_tx);
+		irlap_send_ua_response_frame(self, NULL);
+
+		del_timer(&self->wd_timer);
+		irlap_flush_all_queues(self);
+		/* Set default link parameters */
+		irlap_apply_default_connection_parameters(self);
+
+		irlap_disconnect_indication(self, LAP_DISC_INDICATION);
+		break;
+	case RECV_DISCOVERY_XID_CMD:
+		irlap_wait_min_turn_around(self, &self->qos_tx);
+		irlap_send_rr_frame(self, RSP_FRAME);
+		self->ack_required = TRUE;
+		irlap_start_wd_timer(self, self->wd_timeout);
+		irlap_next_state(self, LAP_NRM_S);
+
+		break;
+	case RECV_TEST_CMD:
+		/* Remove test frame header (only LAP header in NRM) */
+		skb_pull(skb, LAP_ADDR_HEADER + LAP_CTRL_HEADER);
+
+		irlap_wait_min_turn_around(self, &self->qos_tx);
+		irlap_start_wd_timer(self, self->wd_timeout);
+
+		/* Send response (info will be copied) */
+		irlap_send_test_frame(self, self->caddr, info->daddr, skb);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %d, (%s)\n", __func__,
+			 event, irlap_event[event]);
+
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
+
+/*
+ * Function irlap_state_sclose (self, event, skb, info)
+ */
+static int irlap_state_sclose(struct irlap_cb *self, IRLAP_EVENT event,
+			      struct sk_buff *skb, struct irlap_info *info)
+{
+	IRDA_ASSERT(self != NULL, return -ENODEV;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return -EBADR;);
+
+	switch (event) {
+	case RECV_DISC_CMD:
+		/* Always switch state before calling upper layers */
+		irlap_next_state(self, LAP_NDM);
+
+		/* Send disconnect response */
+		irlap_wait_min_turn_around(self, &self->qos_tx);
+		irlap_send_ua_response_frame(self, NULL);
+
+		del_timer(&self->wd_timer);
+		/* Set default link parameters */
+		irlap_apply_default_connection_parameters(self);
+
+		irlap_disconnect_indication(self, LAP_DISC_INDICATION);
+		break;
+	case RECV_DM_RSP:
+		/* IrLAP-1.1 p.82: in SCLOSE, S and I type RSP frames
+		 * shall take us down into default NDM state, like DM_RSP
+		 */
+	case RECV_RR_RSP:
+	case RECV_RNR_RSP:
+	case RECV_REJ_RSP:
+	case RECV_SREJ_RSP:
+	case RECV_I_RSP:
+		/* Always switch state before calling upper layers */
+		irlap_next_state(self, LAP_NDM);
+
+		del_timer(&self->wd_timer);
+		irlap_apply_default_connection_parameters(self);
+
+		irlap_disconnect_indication(self, LAP_DISC_INDICATION);
+		break;
+	case WD_TIMER_EXPIRED:
+		/* Always switch state before calling upper layers */
+		irlap_next_state(self, LAP_NDM);
+
+		irlap_apply_default_connection_parameters(self);
+
+		irlap_disconnect_indication(self, LAP_DISC_INDICATION);
+		break;
+	default:
+		/* IrLAP-1.1 p.82: in SCLOSE, basically any received frame
+		 * with pf=1 shall restart the wd-timer and resend the rd:rsp
+		 */
+		if (info != NULL  &&  info->pf) {
+			del_timer(&self->wd_timer);
+			irlap_wait_min_turn_around(self, &self->qos_tx);
+			irlap_send_rd_frame(self);
+			irlap_start_wd_timer(self, self->wd_timeout);
+			break;		/* stay in SCLOSE */
+		}
+
+		pr_debug("%s(), Unknown event %d, (%s)\n", __func__,
+			 event, irlap_event[event]);
+
+		break;
+	}
+
+	return -1;
+}
+
+static int irlap_state_reset_check( struct irlap_cb *self, IRLAP_EVENT event,
+				   struct sk_buff *skb,
+				   struct irlap_info *info)
+{
+	int ret = 0;
+
+	pr_debug("%s(), event=%s\n", __func__, irlap_event[event]);
+
+	IRDA_ASSERT(self != NULL, return -ENODEV;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return -EBADR;);
+
+	switch (event) {
+	case RESET_RESPONSE:
+		irlap_send_ua_response_frame(self, &self->qos_rx);
+		irlap_initiate_connection_state(self);
+		irlap_start_wd_timer(self, WD_TIMEOUT);
+		irlap_flush_all_queues(self);
+
+		irlap_next_state(self, LAP_NRM_S);
+		break;
+	case DISCONNECT_REQUEST:
+		irlap_wait_min_turn_around(self, &self->qos_tx);
+		irlap_send_rd_frame(self);
+		irlap_start_wd_timer(self, WD_TIMEOUT);
+		irlap_next_state(self, LAP_SCLOSE);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %d, (%s)\n", __func__,
+			 event, irlap_event[event]);
+
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
diff --git a/drivers/staging/irda/net/irlap_frame.c b/drivers/staging/irda/net/irlap_frame.c
new file mode 100644
index 000000000000..debda3de4726
--- /dev/null
+++ b/drivers/staging/irda/net/irlap_frame.c
@@ -0,0 +1,1407 @@
+/*********************************************************************
+ *
+ * Filename:      irlap_frame.c
+ * Version:       1.0
+ * Description:   Build and transmit IrLAP frames
+ * Status:        Stable
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Tue Aug 19 10:27:26 1997
+ * Modified at:   Wed Jan  5 08:59:04 2000
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1998-2000 Dag Brattli <dagb@cs.uit.no>,
+ *     All Rights Reserved.
+ *     Copyright (c) 2000-2003 Jean Tourrilhes <jt@hpl.hp.com>
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#include <linux/skbuff.h>
+#include <linux/if.h>
+#include <linux/if_ether.h>
+#include <linux/netdevice.h>
+#include <linux/irda.h>
+#include <linux/slab.h>
+
+#include <net/pkt_sched.h>
+#include <net/sock.h>
+
+#include <asm/byteorder.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/irda_device.h>
+#include <net/irda/irlap.h>
+#include <net/irda/wrapper.h>
+#include <net/irda/timer.h>
+#include <net/irda/irlap_frame.h>
+#include <net/irda/qos.h>
+
+static void irlap_send_i_frame(struct irlap_cb *self, struct sk_buff *skb,
+			       int command);
+
+/*
+ * Function irlap_insert_info (self, skb)
+ *
+ *    Insert minimum turnaround time and speed information into the skb. We
+ *    need to do this since it's per packet relevant information. Safe to
+ *    have this function inlined since it's only called from one place
+ */
+static inline void irlap_insert_info(struct irlap_cb *self,
+				     struct sk_buff *skb)
+{
+	struct irda_skb_cb *cb = (struct irda_skb_cb *) skb->cb;
+
+	/*
+	 * Insert MTT (min. turn time) and speed into skb, so that the
+	 * device driver knows which settings to use
+	 */
+	cb->magic = LAP_MAGIC;
+	cb->mtt = self->mtt_required;
+	cb->next_speed = self->speed;
+
+	/* Reset */
+	self->mtt_required = 0;
+
+	/*
+	 * Delay equals negotiated BOFs count, plus the number of BOFs to
+	 * force the negotiated minimum turnaround time
+	 */
+	cb->xbofs = self->bofs_count;
+	cb->next_xbofs = self->next_bofs;
+	cb->xbofs_delay = self->xbofs_delay;
+
+	/* Reset XBOF's delay (used only for getting min turn time) */
+	self->xbofs_delay = 0;
+	/* Put the correct xbofs value for the next packet */
+	self->bofs_count = self->next_bofs;
+}
+
+/*
+ * Function irlap_queue_xmit (self, skb)
+ *
+ *    A little wrapper for dev_queue_xmit, so we can insert some common
+ *    code into it.
+ */
+void irlap_queue_xmit(struct irlap_cb *self, struct sk_buff *skb)
+{
+	/* Some common init stuff */
+	skb->dev = self->netdev;
+	skb_reset_mac_header(skb);
+	skb_reset_network_header(skb);
+	skb_reset_transport_header(skb);
+	skb->protocol = htons(ETH_P_IRDA);
+	skb->priority = TC_PRIO_BESTEFFORT;
+
+	irlap_insert_info(self, skb);
+
+	if (unlikely(self->mode & IRDA_MODE_MONITOR)) {
+		pr_debug("%s(): %s is in monitor mode\n", __func__,
+			 self->netdev->name);
+		dev_kfree_skb(skb);
+		return;
+	}
+
+	dev_queue_xmit(skb);
+}
+
+/*
+ * Function irlap_send_snrm_cmd (void)
+ *
+ *    Transmits a connect SNRM command frame
+ */
+void irlap_send_snrm_frame(struct irlap_cb *self, struct qos_info *qos)
+{
+	struct sk_buff *tx_skb;
+	struct snrm_frame *frame;
+	int ret;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	/* Allocate frame */
+	tx_skb = alloc_skb(sizeof(struct snrm_frame) +
+			   IRLAP_NEGOCIATION_PARAMS_LEN,
+			   GFP_ATOMIC);
+	if (!tx_skb)
+		return;
+
+	frame = skb_put(tx_skb, 2);
+
+	/* Insert connection address field */
+	if (qos)
+		frame->caddr = CMD_FRAME | CBROADCAST;
+	else
+		frame->caddr = CMD_FRAME | self->caddr;
+
+	/* Insert control field */
+	frame->control = SNRM_CMD | PF_BIT;
+
+	/*
+	 *  If we are establishing a connection then insert QoS parameters
+	 */
+	if (qos) {
+		skb_put(tx_skb, 9); /* 25 left */
+		frame->saddr = cpu_to_le32(self->saddr);
+		frame->daddr = cpu_to_le32(self->daddr);
+
+		frame->ncaddr = self->caddr;
+
+		ret = irlap_insert_qos_negotiation_params(self, tx_skb);
+		if (ret < 0) {
+			dev_kfree_skb(tx_skb);
+			return;
+		}
+	}
+	irlap_queue_xmit(self, tx_skb);
+}
+
+/*
+ * Function irlap_recv_snrm_cmd (skb, info)
+ *
+ *    Received SNRM (Set Normal Response Mode) command frame
+ *
+ */
+static void irlap_recv_snrm_cmd(struct irlap_cb *self, struct sk_buff *skb,
+				struct irlap_info *info)
+{
+	struct snrm_frame *frame;
+
+	if (pskb_may_pull(skb,sizeof(struct snrm_frame))) {
+		frame = (struct snrm_frame *) skb->data;
+
+		/* Copy the new connection address ignoring the C/R bit */
+		info->caddr = frame->ncaddr & 0xFE;
+
+		/* Check if the new connection address is valid */
+		if ((info->caddr == 0x00) || (info->caddr == 0xfe)) {
+			pr_debug("%s(), invalid connection address!\n",
+				 __func__);
+			return;
+		}
+
+		/* Copy peer device address */
+		info->daddr = le32_to_cpu(frame->saddr);
+		info->saddr = le32_to_cpu(frame->daddr);
+
+		/* Only accept if addressed directly to us */
+		if (info->saddr != self->saddr) {
+			pr_debug("%s(), not addressed to us!\n",
+				 __func__);
+			return;
+		}
+		irlap_do_event(self, RECV_SNRM_CMD, skb, info);
+	} else {
+		/* Signal that this SNRM frame does not contain and I-field */
+		irlap_do_event(self, RECV_SNRM_CMD, skb, NULL);
+	}
+}
+
+/*
+ * Function irlap_send_ua_response_frame (qos)
+ *
+ *    Send UA (Unnumbered Acknowledgement) frame
+ *
+ */
+void irlap_send_ua_response_frame(struct irlap_cb *self, struct qos_info *qos)
+{
+	struct sk_buff *tx_skb;
+	struct ua_frame *frame;
+	int ret;
+
+	pr_debug("%s() <%ld>\n", __func__, jiffies);
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	/* Allocate frame */
+	tx_skb = alloc_skb(sizeof(struct ua_frame) +
+			   IRLAP_NEGOCIATION_PARAMS_LEN,
+			   GFP_ATOMIC);
+	if (!tx_skb)
+		return;
+
+	frame = skb_put(tx_skb, 10);
+
+	/* Build UA response */
+	frame->caddr = self->caddr;
+	frame->control = UA_RSP | PF_BIT;
+
+	frame->saddr = cpu_to_le32(self->saddr);
+	frame->daddr = cpu_to_le32(self->daddr);
+
+	/* Should we send QoS negotiation parameters? */
+	if (qos) {
+		ret = irlap_insert_qos_negotiation_params(self, tx_skb);
+		if (ret < 0) {
+			dev_kfree_skb(tx_skb);
+			return;
+		}
+	}
+
+	irlap_queue_xmit(self, tx_skb);
+}
+
+
+/*
+ * Function irlap_send_dm_frame (void)
+ *
+ *    Send disconnected mode (DM) frame
+ *
+ */
+void irlap_send_dm_frame( struct irlap_cb *self)
+{
+	struct sk_buff *tx_skb = NULL;
+	struct dm_frame *frame;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	tx_skb = alloc_skb(sizeof(struct dm_frame), GFP_ATOMIC);
+	if (!tx_skb)
+		return;
+
+	frame = skb_put(tx_skb, 2);
+
+	if (self->state == LAP_NDM)
+		frame->caddr = CBROADCAST;
+	else
+		frame->caddr = self->caddr;
+
+	frame->control = DM_RSP | PF_BIT;
+
+	irlap_queue_xmit(self, tx_skb);
+}
+
+/*
+ * Function irlap_send_disc_frame (void)
+ *
+ *    Send disconnect (DISC) frame
+ *
+ */
+void irlap_send_disc_frame(struct irlap_cb *self)
+{
+	struct sk_buff *tx_skb = NULL;
+	struct disc_frame *frame;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	tx_skb = alloc_skb(sizeof(struct disc_frame), GFP_ATOMIC);
+	if (!tx_skb)
+		return;
+
+	frame = skb_put(tx_skb, 2);
+
+	frame->caddr = self->caddr | CMD_FRAME;
+	frame->control = DISC_CMD | PF_BIT;
+
+	irlap_queue_xmit(self, tx_skb);
+}
+
+/*
+ * Function irlap_send_discovery_xid_frame (S, s, command)
+ *
+ *    Build and transmit a XID (eXchange station IDentifier) discovery
+ *    frame.
+ */
+void irlap_send_discovery_xid_frame(struct irlap_cb *self, int S, __u8 s,
+				    __u8 command, discovery_t *discovery)
+{
+	struct sk_buff *tx_skb = NULL;
+	struct xid_frame *frame;
+	__u32 bcast = BROADCAST;
+	__u8 *info;
+
+	pr_debug("%s(), s=%d, S=%d, command=%d\n", __func__,
+		 s, S, command);
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+	IRDA_ASSERT(discovery != NULL, return;);
+
+	tx_skb = alloc_skb(sizeof(struct xid_frame) + IRLAP_DISCOVERY_INFO_LEN,
+			   GFP_ATOMIC);
+	if (!tx_skb)
+		return;
+
+	skb_put(tx_skb, 14);
+	frame = (struct xid_frame *) tx_skb->data;
+
+	if (command) {
+		frame->caddr = CBROADCAST | CMD_FRAME;
+		frame->control =  XID_CMD | PF_BIT;
+	} else {
+		frame->caddr = CBROADCAST;
+		frame->control =  XID_RSP | PF_BIT;
+	}
+	frame->ident = XID_FORMAT;
+
+	frame->saddr = cpu_to_le32(self->saddr);
+
+	if (command)
+		frame->daddr = cpu_to_le32(bcast);
+	else
+		frame->daddr = cpu_to_le32(discovery->data.daddr);
+
+	switch (S) {
+	case 1:
+		frame->flags = 0x00;
+		break;
+	case 6:
+		frame->flags = 0x01;
+		break;
+	case 8:
+		frame->flags = 0x02;
+		break;
+	case 16:
+		frame->flags = 0x03;
+		break;
+	default:
+		frame->flags = 0x02;
+		break;
+	}
+
+	frame->slotnr = s;
+	frame->version = 0x00;
+
+	/*
+	 *  Provide info for final slot only in commands, and for all
+	 *  responses. Send the second byte of the hint only if the
+	 *  EXTENSION bit is set in the first byte.
+	 */
+	if (!command || (frame->slotnr == 0xff)) {
+		int len;
+
+		if (discovery->data.hints[0] & HINT_EXTENSION) {
+			info = skb_put(tx_skb, 2);
+			info[0] = discovery->data.hints[0];
+			info[1] = discovery->data.hints[1];
+		} else {
+			info = skb_put(tx_skb, 1);
+			info[0] = discovery->data.hints[0];
+		}
+		info = skb_put(tx_skb, 1);
+		info[0] = discovery->data.charset;
+
+		len = IRDA_MIN(discovery->name_len, skb_tailroom(tx_skb));
+		skb_put_data(tx_skb, discovery->data.info, len);
+	}
+	irlap_queue_xmit(self, tx_skb);
+}
+
+/*
+ * Function irlap_recv_discovery_xid_rsp (skb, info)
+ *
+ *    Received a XID discovery response
+ *
+ */
+static void irlap_recv_discovery_xid_rsp(struct irlap_cb *self,
+					 struct sk_buff *skb,
+					 struct irlap_info *info)
+{
+	struct xid_frame *xid;
+	discovery_t *discovery = NULL;
+	__u8 *discovery_info;
+	char *text;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	if (!pskb_may_pull(skb, sizeof(struct xid_frame))) {
+		net_err_ratelimited("%s: frame too short!\n", __func__);
+		return;
+	}
+
+	xid = (struct xid_frame *) skb->data;
+
+	info->daddr = le32_to_cpu(xid->saddr);
+	info->saddr = le32_to_cpu(xid->daddr);
+
+	/* Make sure frame is addressed to us */
+	if ((info->saddr != self->saddr) && (info->saddr != BROADCAST)) {
+		pr_debug("%s(), frame is not addressed to us!\n",
+			 __func__);
+		return;
+	}
+
+	if ((discovery = kzalloc(sizeof(discovery_t), GFP_ATOMIC)) == NULL) {
+		net_warn_ratelimited("%s: kmalloc failed!\n", __func__);
+		return;
+	}
+
+	discovery->data.daddr = info->daddr;
+	discovery->data.saddr = self->saddr;
+	discovery->timestamp = jiffies;
+
+	pr_debug("%s(), daddr=%08x\n", __func__,
+		 discovery->data.daddr);
+
+	discovery_info = skb_pull(skb, sizeof(struct xid_frame));
+
+	/* Get info returned from peer */
+	discovery->data.hints[0] = discovery_info[0];
+	if (discovery_info[0] & HINT_EXTENSION) {
+		pr_debug("EXTENSION\n");
+		discovery->data.hints[1] = discovery_info[1];
+		discovery->data.charset = discovery_info[2];
+		text = (char *) &discovery_info[3];
+	} else {
+		discovery->data.hints[1] = 0;
+		discovery->data.charset = discovery_info[1];
+		text = (char *) &discovery_info[2];
+	}
+	/*
+	 *  Terminate info string, should be safe since this is where the
+	 *  FCS bytes resides.
+	 */
+	skb->data[skb->len] = '\0';
+	strncpy(discovery->data.info, text, NICKNAME_MAX_LEN);
+	discovery->name_len = strlen(discovery->data.info);
+
+	info->discovery = discovery;
+
+	irlap_do_event(self, RECV_DISCOVERY_XID_RSP, skb, info);
+}
+
+/*
+ * Function irlap_recv_discovery_xid_cmd (skb, info)
+ *
+ *    Received a XID discovery command
+ *
+ */
+static void irlap_recv_discovery_xid_cmd(struct irlap_cb *self,
+					 struct sk_buff *skb,
+					 struct irlap_info *info)
+{
+	struct xid_frame *xid;
+	discovery_t *discovery = NULL;
+	__u8 *discovery_info;
+	char *text;
+
+	if (!pskb_may_pull(skb, sizeof(struct xid_frame))) {
+		net_err_ratelimited("%s: frame too short!\n", __func__);
+		return;
+	}
+
+	xid = (struct xid_frame *) skb->data;
+
+	info->daddr = le32_to_cpu(xid->saddr);
+	info->saddr = le32_to_cpu(xid->daddr);
+
+	/* Make sure frame is addressed to us */
+	if ((info->saddr != self->saddr) && (info->saddr != BROADCAST)) {
+		pr_debug("%s(), frame is not addressed to us!\n",
+			 __func__);
+		return;
+	}
+
+	switch (xid->flags & 0x03) {
+	case 0x00:
+		info->S = 1;
+		break;
+	case 0x01:
+		info->S = 6;
+		break;
+	case 0x02:
+		info->S = 8;
+		break;
+	case 0x03:
+		info->S = 16;
+		break;
+	default:
+		/* Error!! */
+		return;
+	}
+	info->s = xid->slotnr;
+
+	discovery_info = skb_pull(skb, sizeof(struct xid_frame));
+
+	/*
+	 *  Check if last frame
+	 */
+	if (info->s == 0xff) {
+		/* Check if things are sane at this point... */
+		if((discovery_info == NULL) ||
+		   !pskb_may_pull(skb, 3)) {
+			net_err_ratelimited("%s: discovery frame too short!\n",
+					    __func__);
+			return;
+		}
+
+		/*
+		 *  We now have some discovery info to deliver!
+		 */
+		discovery = kzalloc(sizeof(discovery_t), GFP_ATOMIC);
+		if (!discovery)
+			return;
+
+		discovery->data.daddr = info->daddr;
+		discovery->data.saddr = self->saddr;
+		discovery->timestamp = jiffies;
+
+		discovery->data.hints[0] = discovery_info[0];
+		if (discovery_info[0] & HINT_EXTENSION) {
+			discovery->data.hints[1] = discovery_info[1];
+			discovery->data.charset = discovery_info[2];
+			text = (char *) &discovery_info[3];
+		} else {
+			discovery->data.hints[1] = 0;
+			discovery->data.charset = discovery_info[1];
+			text = (char *) &discovery_info[2];
+		}
+		/*
+		 *  Terminate string, should be safe since this is where the
+		 *  FCS bytes resides.
+		 */
+		skb->data[skb->len] = '\0';
+		strncpy(discovery->data.info, text, NICKNAME_MAX_LEN);
+		discovery->name_len = strlen(discovery->data.info);
+
+		info->discovery = discovery;
+	} else
+		info->discovery = NULL;
+
+	irlap_do_event(self, RECV_DISCOVERY_XID_CMD, skb, info);
+}
+
+/*
+ * Function irlap_send_rr_frame (self, command)
+ *
+ *    Build and transmit RR (Receive Ready) frame. Notice that it is currently
+ *    only possible to send RR frames with the poll bit set.
+ */
+void irlap_send_rr_frame(struct irlap_cb *self, int command)
+{
+	struct sk_buff *tx_skb;
+	struct rr_frame *frame;
+
+	tx_skb = alloc_skb(sizeof(struct rr_frame), GFP_ATOMIC);
+	if (!tx_skb)
+		return;
+
+	frame = skb_put(tx_skb, 2);
+
+	frame->caddr = self->caddr;
+	frame->caddr |= (command) ? CMD_FRAME : 0;
+
+	frame->control = RR | PF_BIT | (self->vr << 5);
+
+	irlap_queue_xmit(self, tx_skb);
+}
+
+/*
+ * Function irlap_send_rd_frame (self)
+ *
+ *    Request disconnect. Used by a secondary station to request the
+ *    disconnection of the link.
+ */
+void irlap_send_rd_frame(struct irlap_cb *self)
+{
+	struct sk_buff *tx_skb;
+	struct rd_frame *frame;
+
+	tx_skb = alloc_skb(sizeof(struct rd_frame), GFP_ATOMIC);
+	if (!tx_skb)
+		return;
+
+	frame = skb_put(tx_skb, 2);
+
+	frame->caddr = self->caddr;
+	frame->control = RD_RSP | PF_BIT;
+
+	irlap_queue_xmit(self, tx_skb);
+}
+
+/*
+ * Function irlap_recv_rr_frame (skb, info)
+ *
+ *    Received RR (Receive Ready) frame from peer station, no harm in
+ *    making it inline since its called only from one single place
+ *    (irlap_driver_rcv).
+ */
+static inline void irlap_recv_rr_frame(struct irlap_cb *self,
+				       struct sk_buff *skb,
+				       struct irlap_info *info, int command)
+{
+	info->nr = skb->data[1] >> 5;
+
+	/* Check if this is a command or a response frame */
+	if (command)
+		irlap_do_event(self, RECV_RR_CMD, skb, info);
+	else
+		irlap_do_event(self, RECV_RR_RSP, skb, info);
+}
+
+/*
+ * Function irlap_recv_rnr_frame (self, skb, info)
+ *
+ *    Received RNR (Receive Not Ready) frame from peer station
+ *
+ */
+static void irlap_recv_rnr_frame(struct irlap_cb *self, struct sk_buff *skb,
+				 struct irlap_info *info, int command)
+{
+	info->nr = skb->data[1] >> 5;
+
+	pr_debug("%s(), nr=%d, %ld\n", __func__, info->nr, jiffies);
+
+	if (command)
+		irlap_do_event(self, RECV_RNR_CMD, skb, info);
+	else
+		irlap_do_event(self, RECV_RNR_RSP, skb, info);
+}
+
+static void irlap_recv_rej_frame(struct irlap_cb *self, struct sk_buff *skb,
+				 struct irlap_info *info, int command)
+{
+	info->nr = skb->data[1] >> 5;
+
+	/* Check if this is a command or a response frame */
+	if (command)
+		irlap_do_event(self, RECV_REJ_CMD, skb, info);
+	else
+		irlap_do_event(self, RECV_REJ_RSP, skb, info);
+}
+
+static void irlap_recv_srej_frame(struct irlap_cb *self, struct sk_buff *skb,
+				  struct irlap_info *info, int command)
+{
+	info->nr = skb->data[1] >> 5;
+
+	/* Check if this is a command or a response frame */
+	if (command)
+		irlap_do_event(self, RECV_SREJ_CMD, skb, info);
+	else
+		irlap_do_event(self, RECV_SREJ_RSP, skb, info);
+}
+
+static void irlap_recv_disc_frame(struct irlap_cb *self, struct sk_buff *skb,
+				  struct irlap_info *info, int command)
+{
+	/* Check if this is a command or a response frame */
+	if (command)
+		irlap_do_event(self, RECV_DISC_CMD, skb, info);
+	else
+		irlap_do_event(self, RECV_RD_RSP, skb, info);
+}
+
+/*
+ * Function irlap_recv_ua_frame (skb, frame)
+ *
+ *    Received UA (Unnumbered Acknowledgement) frame
+ *
+ */
+static inline void irlap_recv_ua_frame(struct irlap_cb *self,
+				       struct sk_buff *skb,
+				       struct irlap_info *info)
+{
+	irlap_do_event(self, RECV_UA_RSP, skb, info);
+}
+
+/*
+ * Function irlap_send_data_primary(self, skb)
+ *
+ *    Send I-frames as the primary station but without the poll bit set
+ *
+ */
+void irlap_send_data_primary(struct irlap_cb *self, struct sk_buff *skb)
+{
+	struct sk_buff *tx_skb;
+
+	if (skb->data[1] == I_FRAME) {
+
+		/*
+		 *  Insert frame sequence number (Vs) in control field before
+		 *  inserting into transmit window queue.
+		 */
+		skb->data[1] = I_FRAME | (self->vs << 1);
+
+		/*
+		 *  Insert frame in store, in case of retransmissions
+		 *  Increase skb reference count, see irlap_do_event()
+		 */
+		skb_get(skb);
+		skb_queue_tail(&self->wx_list, skb);
+
+		/* Copy buffer */
+		tx_skb = skb_clone(skb, GFP_ATOMIC);
+		if (tx_skb == NULL) {
+			return;
+		}
+
+		self->vs = (self->vs + 1) % 8;
+		self->ack_required = FALSE;
+		self->window -= 1;
+
+		irlap_send_i_frame( self, tx_skb, CMD_FRAME);
+	} else {
+		pr_debug("%s(), sending unreliable frame\n", __func__);
+		irlap_send_ui_frame(self, skb_get(skb), self->caddr, CMD_FRAME);
+		self->window -= 1;
+	}
+}
+/*
+ * Function irlap_send_data_primary_poll (self, skb)
+ *
+ *    Send I(nformation) frame as primary with poll bit set
+ */
+void irlap_send_data_primary_poll(struct irlap_cb *self, struct sk_buff *skb)
+{
+	struct sk_buff *tx_skb;
+	int transmission_time;
+
+	/* Stop P timer */
+	del_timer(&self->poll_timer);
+
+	/* Is this reliable or unreliable data? */
+	if (skb->data[1] == I_FRAME) {
+
+		/*
+		 *  Insert frame sequence number (Vs) in control field before
+		 *  inserting into transmit window queue.
+		 */
+		skb->data[1] = I_FRAME | (self->vs << 1);
+
+		/*
+		 *  Insert frame in store, in case of retransmissions
+		 *  Increase skb reference count, see irlap_do_event()
+		 */
+		skb_get(skb);
+		skb_queue_tail(&self->wx_list, skb);
+
+		/* Copy buffer */
+		tx_skb = skb_clone(skb, GFP_ATOMIC);
+		if (tx_skb == NULL) {
+			return;
+		}
+
+		/*
+		 *  Set poll bit if necessary. We do this to the copied
+		 *  skb, since retransmitted need to set or clear the poll
+		 *  bit depending on when they are sent.
+		 */
+		tx_skb->data[1] |= PF_BIT;
+
+		self->vs = (self->vs + 1) % 8;
+		self->ack_required = FALSE;
+
+		irlap_next_state(self, LAP_NRM_P);
+		irlap_send_i_frame(self, tx_skb, CMD_FRAME);
+	} else {
+		pr_debug("%s(), sending unreliable frame\n", __func__);
+
+		if (self->ack_required) {
+			irlap_send_ui_frame(self, skb_get(skb), self->caddr, CMD_FRAME);
+			irlap_next_state(self, LAP_NRM_P);
+			irlap_send_rr_frame(self, CMD_FRAME);
+			self->ack_required = FALSE;
+		} else {
+			skb->data[1] |= PF_BIT;
+			irlap_next_state(self, LAP_NRM_P);
+			irlap_send_ui_frame(self, skb_get(skb), self->caddr, CMD_FRAME);
+		}
+	}
+
+	/* How much time we took for transmission of all frames.
+	 * We don't know, so let assume we used the full window. Jean II */
+	transmission_time = self->final_timeout;
+
+	/* Reset parameter so that we can fill next window */
+	self->window = self->window_size;
+
+#ifdef CONFIG_IRDA_DYNAMIC_WINDOW
+	/* Remove what we have not used. Just do a prorata of the
+	 * bytes left in window to window capacity.
+	 * See max_line_capacities[][] in qos.c for details. Jean II */
+	transmission_time -= (self->final_timeout * self->bytes_left
+			      / self->line_capacity);
+	pr_debug("%s() adjusting transmission_time : ft=%d, bl=%d, lc=%d -> tt=%d\n",
+		 __func__, self->final_timeout, self->bytes_left,
+		 self->line_capacity, transmission_time);
+
+	/* We are allowed to transmit a maximum number of bytes again. */
+	self->bytes_left = self->line_capacity;
+#endif /* CONFIG_IRDA_DYNAMIC_WINDOW */
+
+	/*
+	 * The network layer has a intermediate buffer between IrLAP
+	 * and the IrDA driver which can contain 8 frames. So, even
+	 * though IrLAP is currently sending the *last* frame of the
+	 * tx-window, the driver most likely has only just started
+	 * sending the *first* frame of the same tx-window.
+	 * I.e. we are always at the very beginning of or Tx window.
+	 * Now, we are supposed to set the final timer from the end
+	 * of our tx-window to let the other peer reply. So, we need
+	 * to add extra time to compensate for the fact that we
+	 * are really at the start of tx-window, otherwise the final timer
+	 * might expire before he can answer...
+	 * Jean II
+	 */
+	irlap_start_final_timer(self, self->final_timeout + transmission_time);
+
+	/*
+	 * The clever amongst you might ask why we do this adjustement
+	 * only here, and not in all the other cases in irlap_event.c.
+	 * In all those other case, we only send a very short management
+	 * frame (few bytes), so the adjustement would be lost in the
+	 * noise...
+	 * The exception of course is irlap_resend_rejected_frame().
+	 * Jean II */
+}
+
+/*
+ * Function irlap_send_data_secondary_final (self, skb)
+ *
+ *    Send I(nformation) frame as secondary with final bit set
+ *
+ */
+void irlap_send_data_secondary_final(struct irlap_cb *self,
+				     struct sk_buff *skb)
+{
+	struct sk_buff *tx_skb = NULL;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+	IRDA_ASSERT(skb != NULL, return;);
+
+	/* Is this reliable or unreliable data? */
+	if (skb->data[1] == I_FRAME) {
+
+		/*
+		 *  Insert frame sequence number (Vs) in control field before
+		 *  inserting into transmit window queue.
+		 */
+		skb->data[1] = I_FRAME | (self->vs << 1);
+
+		/*
+		 *  Insert frame in store, in case of retransmissions
+		 *  Increase skb reference count, see irlap_do_event()
+		 */
+		skb_get(skb);
+		skb_queue_tail(&self->wx_list, skb);
+
+		tx_skb = skb_clone(skb, GFP_ATOMIC);
+		if (tx_skb == NULL) {
+			return;
+		}
+
+		tx_skb->data[1] |= PF_BIT;
+
+		self->vs = (self->vs + 1) % 8;
+		self->ack_required = FALSE;
+
+		irlap_send_i_frame(self, tx_skb, RSP_FRAME);
+	} else {
+		if (self->ack_required) {
+			irlap_send_ui_frame(self, skb_get(skb), self->caddr, RSP_FRAME);
+			irlap_send_rr_frame(self, RSP_FRAME);
+			self->ack_required = FALSE;
+		} else {
+			skb->data[1] |= PF_BIT;
+			irlap_send_ui_frame(self, skb_get(skb), self->caddr, RSP_FRAME);
+		}
+	}
+
+	self->window = self->window_size;
+#ifdef CONFIG_IRDA_DYNAMIC_WINDOW
+	/* We are allowed to transmit a maximum number of bytes again. */
+	self->bytes_left = self->line_capacity;
+#endif /* CONFIG_IRDA_DYNAMIC_WINDOW */
+
+	irlap_start_wd_timer(self, self->wd_timeout);
+}
+
+/*
+ * Function irlap_send_data_secondary (self, skb)
+ *
+ *    Send I(nformation) frame as secondary without final bit set
+ *
+ */
+void irlap_send_data_secondary(struct irlap_cb *self, struct sk_buff *skb)
+{
+	struct sk_buff *tx_skb = NULL;
+
+	/* Is this reliable or unreliable data? */
+	if (skb->data[1] == I_FRAME) {
+
+		/*
+		 *  Insert frame sequence number (Vs) in control field before
+		 *  inserting into transmit window queue.
+		 */
+		skb->data[1] = I_FRAME | (self->vs << 1);
+
+		/*
+		 *  Insert frame in store, in case of retransmissions
+		 *  Increase skb reference count, see irlap_do_event()
+		 */
+		skb_get(skb);
+		skb_queue_tail(&self->wx_list, skb);
+
+		tx_skb = skb_clone(skb, GFP_ATOMIC);
+		if (tx_skb == NULL) {
+			return;
+		}
+
+		self->vs = (self->vs + 1) % 8;
+		self->ack_required = FALSE;
+		self->window -= 1;
+
+		irlap_send_i_frame(self, tx_skb, RSP_FRAME);
+	} else {
+		irlap_send_ui_frame(self, skb_get(skb), self->caddr, RSP_FRAME);
+		self->window -= 1;
+	}
+}
+
+/*
+ * Function irlap_resend_rejected_frames (nr)
+ *
+ *    Resend frames which has not been acknowledged. Should be safe to
+ *    traverse the list without locking it since this function will only be
+ *    called from interrupt context (BH)
+ */
+void irlap_resend_rejected_frames(struct irlap_cb *self, int command)
+{
+	struct sk_buff *tx_skb;
+	struct sk_buff *skb;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	/*  Resend unacknowledged frame(s) */
+	skb_queue_walk(&self->wx_list, skb) {
+		irlap_wait_min_turn_around(self, &self->qos_tx);
+
+		/* We copy the skb to be retransmitted since we will have to
+		 * modify it. Cloning will confuse packet sniffers
+		 */
+		/* tx_skb = skb_clone( skb, GFP_ATOMIC); */
+		tx_skb = skb_copy(skb, GFP_ATOMIC);
+		if (!tx_skb) {
+			pr_debug("%s(), unable to copy\n", __func__);
+			return;
+		}
+
+		/* Clear old Nr field + poll bit */
+		tx_skb->data[1] &= 0x0f;
+
+		/*
+		 *  Set poll bit on the last frame retransmitted
+		 */
+		if (skb_queue_is_last(&self->wx_list, skb))
+			tx_skb->data[1] |= PF_BIT; /* Set p/f bit */
+		else
+			tx_skb->data[1] &= ~PF_BIT; /* Clear p/f bit */
+
+		irlap_send_i_frame(self, tx_skb, command);
+	}
+#if 0 /* Not yet */
+	/*
+	 *  We can now fill the window with additional data frames
+	 */
+	while (!skb_queue_empty(&self->txq)) {
+
+		pr_debug("%s(), sending additional frames!\n", __func__);
+		if (self->window > 0) {
+			skb = skb_dequeue( &self->txq);
+			IRDA_ASSERT(skb != NULL, return;);
+
+			/*
+			 *  If send window > 1 then send frame with pf
+			 *  bit cleared
+			 */
+			if ((self->window > 1) &&
+			    !skb_queue_empty(&self->txq)) {
+				irlap_send_data_primary(self, skb);
+			} else {
+				irlap_send_data_primary_poll(self, skb);
+			}
+			kfree_skb(skb);
+		}
+	}
+#endif
+}
+
+void irlap_resend_rejected_frame(struct irlap_cb *self, int command)
+{
+	struct sk_buff *tx_skb;
+	struct sk_buff *skb;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	/*  Resend unacknowledged frame(s) */
+	skb = skb_peek(&self->wx_list);
+	if (skb != NULL) {
+		irlap_wait_min_turn_around(self, &self->qos_tx);
+
+		/* We copy the skb to be retransmitted since we will have to
+		 * modify it. Cloning will confuse packet sniffers
+		 */
+		/* tx_skb = skb_clone( skb, GFP_ATOMIC); */
+		tx_skb = skb_copy(skb, GFP_ATOMIC);
+		if (!tx_skb) {
+			pr_debug("%s(), unable to copy\n", __func__);
+			return;
+		}
+
+		/* Clear old Nr field + poll bit */
+		tx_skb->data[1] &= 0x0f;
+
+		/*  Set poll/final bit */
+		tx_skb->data[1] |= PF_BIT; /* Set p/f bit */
+
+		irlap_send_i_frame(self, tx_skb, command);
+	}
+}
+
+/*
+ * Function irlap_send_ui_frame (self, skb, command)
+ *
+ *    Contruct and transmit an Unnumbered Information (UI) frame
+ *
+ */
+void irlap_send_ui_frame(struct irlap_cb *self, struct sk_buff *skb,
+			 __u8 caddr, int command)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+	IRDA_ASSERT(skb != NULL, return;);
+
+	/* Insert connection address */
+	skb->data[0] = caddr | ((command) ? CMD_FRAME : 0);
+
+	irlap_queue_xmit(self, skb);
+}
+
+/*
+ * Function irlap_send_i_frame (skb)
+ *
+ *    Contruct and transmit Information (I) frame
+ */
+static void irlap_send_i_frame(struct irlap_cb *self, struct sk_buff *skb,
+			       int command)
+{
+	/* Insert connection address */
+	skb->data[0] = self->caddr;
+	skb->data[0] |= (command) ? CMD_FRAME : 0;
+
+	/* Insert next to receive (Vr) */
+	skb->data[1] |= (self->vr << 5);  /* insert nr */
+
+	irlap_queue_xmit(self, skb);
+}
+
+/*
+ * Function irlap_recv_i_frame (skb, frame)
+ *
+ *    Receive and parse an I (Information) frame, no harm in making it inline
+ *    since it's called only from one single place (irlap_driver_rcv).
+ */
+static inline void irlap_recv_i_frame(struct irlap_cb *self,
+				      struct sk_buff *skb,
+				      struct irlap_info *info, int command)
+{
+	info->nr = skb->data[1] >> 5;          /* Next to receive */
+	info->pf = skb->data[1] & PF_BIT;      /* Final bit */
+	info->ns = (skb->data[1] >> 1) & 0x07; /* Next to send */
+
+	/* Check if this is a command or a response frame */
+	if (command)
+		irlap_do_event(self, RECV_I_CMD, skb, info);
+	else
+		irlap_do_event(self, RECV_I_RSP, skb, info);
+}
+
+/*
+ * Function irlap_recv_ui_frame (self, skb, info)
+ *
+ *    Receive and parse an Unnumbered Information (UI) frame
+ *
+ */
+static void irlap_recv_ui_frame(struct irlap_cb *self, struct sk_buff *skb,
+				struct irlap_info *info)
+{
+	info->pf = skb->data[1] & PF_BIT;      /* Final bit */
+
+	irlap_do_event(self, RECV_UI_FRAME, skb, info);
+}
+
+/*
+ * Function irlap_recv_frmr_frame (skb, frame)
+ *
+ *    Received Frame Reject response.
+ *
+ */
+static void irlap_recv_frmr_frame(struct irlap_cb *self, struct sk_buff *skb,
+				  struct irlap_info *info)
+{
+	__u8 *frame;
+	int w, x, y, z;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+	IRDA_ASSERT(skb != NULL, return;);
+	IRDA_ASSERT(info != NULL, return;);
+
+	if (!pskb_may_pull(skb, 4)) {
+		net_err_ratelimited("%s: frame too short!\n", __func__);
+		return;
+	}
+
+	frame = skb->data;
+
+	info->nr = frame[2] >> 5;          /* Next to receive */
+	info->pf = frame[2] & PF_BIT;      /* Final bit */
+	info->ns = (frame[2] >> 1) & 0x07; /* Next to send */
+
+	w = frame[3] & 0x01;
+	x = frame[3] & 0x02;
+	y = frame[3] & 0x04;
+	z = frame[3] & 0x08;
+
+	if (w) {
+		pr_debug("Rejected control field is undefined or not implemented\n");
+	}
+	if (x) {
+		pr_debug("Rejected control field was invalid because it contained a non permitted I field\n");
+	}
+	if (y) {
+		pr_debug("Received I field exceeded the maximum negotiated for the existing connection or exceeded the maximum this station supports if no connection exists\n");
+	}
+	if (z) {
+		pr_debug("Rejected control field control field contained an invalid Nr count\n");
+	}
+	irlap_do_event(self, RECV_FRMR_RSP, skb, info);
+}
+
+/*
+ * Function irlap_send_test_frame (self, daddr)
+ *
+ *    Send a test frame response
+ *
+ */
+void irlap_send_test_frame(struct irlap_cb *self, __u8 caddr, __u32 daddr,
+			   struct sk_buff *cmd)
+{
+	struct sk_buff *tx_skb;
+	struct test_frame *frame;
+
+	tx_skb = alloc_skb(cmd->len + sizeof(struct test_frame), GFP_ATOMIC);
+	if (!tx_skb)
+		return;
+
+	/* Broadcast frames must include saddr and daddr fields */
+	if (caddr == CBROADCAST) {
+		frame = skb_put(tx_skb, sizeof(struct test_frame));
+
+		/* Insert the swapped addresses */
+		frame->saddr = cpu_to_le32(self->saddr);
+		frame->daddr = cpu_to_le32(daddr);
+	} else
+		frame = skb_put(tx_skb, LAP_ADDR_HEADER + LAP_CTRL_HEADER);
+
+	frame->caddr = caddr;
+	frame->control = TEST_RSP | PF_BIT;
+
+	/* Copy info */
+	skb_put_data(tx_skb, cmd->data, cmd->len);
+
+	/* Return to sender */
+	irlap_wait_min_turn_around(self, &self->qos_tx);
+	irlap_queue_xmit(self, tx_skb);
+}
+
+/*
+ * Function irlap_recv_test_frame (self, skb)
+ *
+ *    Receive a test frame
+ *
+ */
+static void irlap_recv_test_frame(struct irlap_cb *self, struct sk_buff *skb,
+				  struct irlap_info *info, int command)
+{
+	struct test_frame *frame;
+
+	if (!pskb_may_pull(skb, sizeof(*frame))) {
+		net_err_ratelimited("%s: frame too short!\n", __func__);
+		return;
+	}
+	frame = (struct test_frame *) skb->data;
+
+	/* Broadcast frames must carry saddr and daddr fields */
+	if (info->caddr == CBROADCAST) {
+		if (skb->len < sizeof(struct test_frame)) {
+			pr_debug("%s() test frame too short!\n",
+				 __func__);
+			return;
+		}
+
+		/* Read and swap addresses */
+		info->daddr = le32_to_cpu(frame->saddr);
+		info->saddr = le32_to_cpu(frame->daddr);
+
+		/* Make sure frame is addressed to us */
+		if ((info->saddr != self->saddr) &&
+		    (info->saddr != BROADCAST)) {
+			return;
+		}
+	}
+
+	if (command)
+		irlap_do_event(self, RECV_TEST_CMD, skb, info);
+	else
+		irlap_do_event(self, RECV_TEST_RSP, skb, info);
+}
+
+/*
+ * Function irlap_driver_rcv (skb, netdev, ptype)
+ *
+ *    Called when a frame is received. Dispatches the right receive function
+ *    for processing of the frame.
+ *
+ * Note on skb management :
+ * After calling the higher layers of the IrDA stack, we always
+ * kfree() the skb, which drop the reference count (and potentially
+ * destroy it).
+ * If a higher layer of the stack want to keep the skb around (to put
+ * in a queue or pass it to the higher layer), it will need to use
+ * skb_get() to keep a reference on it. This is usually done at the
+ * LMP level in irlmp.c.
+ * Jean II
+ */
+int irlap_driver_rcv(struct sk_buff *skb, struct net_device *dev,
+		     struct packet_type *ptype, struct net_device *orig_dev)
+{
+	struct irlap_info info;
+	struct irlap_cb *self;
+	int command;
+	__u8 control;
+	int ret = -1;
+
+	if (!net_eq(dev_net(dev), &init_net))
+		goto out;
+
+	/* FIXME: should we get our own field? */
+	self = (struct irlap_cb *) dev->atalk_ptr;
+
+	/* If the net device is down, then IrLAP is gone! */
+	if (!self || self->magic != LAP_MAGIC)
+		goto err;
+
+	/* We are no longer an "old" protocol, so we need to handle
+	 * share and non linear skbs. This should never happen, so
+	 * we don't need to be clever about it. Jean II */
+	if ((skb = skb_share_check(skb, GFP_ATOMIC)) == NULL) {
+		net_err_ratelimited("%s: can't clone shared skb!\n", __func__);
+		goto err;
+	}
+
+	/* Check if frame is large enough for parsing */
+	if (!pskb_may_pull(skb, 2)) {
+		net_err_ratelimited("%s: frame too short!\n", __func__);
+		goto err;
+	}
+
+	command    = skb->data[0] & CMD_FRAME;
+	info.caddr = skb->data[0] & CBROADCAST;
+
+	info.pf      = skb->data[1] &  PF_BIT;
+	info.control = skb->data[1] & ~PF_BIT; /* Mask away poll/final bit */
+
+	control = info.control;
+
+	/*  First we check if this frame has a valid connection address */
+	if ((info.caddr != self->caddr) && (info.caddr != CBROADCAST)) {
+		pr_debug("%s(), wrong connection address!\n",
+			 __func__);
+		goto out;
+	}
+	/*
+	 *  Optimize for the common case and check if the frame is an
+	 *  I(nformation) frame. Only I-frames have bit 0 set to 0
+	 */
+	if (~control & 0x01) {
+		irlap_recv_i_frame(self, skb, &info, command);
+		goto out;
+	}
+	/*
+	 *  We now check is the frame is an S(upervisory) frame. Only
+	 *  S-frames have bit 0 set to 1 and bit 1 set to 0
+	 */
+	if (~control & 0x02) {
+		/*
+		 *  Received S(upervisory) frame, check which frame type it is
+		 *  only the first nibble is of interest
+		 */
+		switch (control & 0x0f) {
+		case RR:
+			irlap_recv_rr_frame(self, skb, &info, command);
+			break;
+		case RNR:
+			irlap_recv_rnr_frame(self, skb, &info, command);
+			break;
+		case REJ:
+			irlap_recv_rej_frame(self, skb, &info, command);
+			break;
+		case SREJ:
+			irlap_recv_srej_frame(self, skb, &info, command);
+			break;
+		default:
+			net_warn_ratelimited("%s: Unknown S-frame %02x received!\n",
+					     __func__, info.control);
+			break;
+		}
+		goto out;
+	}
+	/*
+	 *  This must be a C(ontrol) frame
+	 */
+	switch (control) {
+	case XID_RSP:
+		irlap_recv_discovery_xid_rsp(self, skb, &info);
+		break;
+	case XID_CMD:
+		irlap_recv_discovery_xid_cmd(self, skb, &info);
+		break;
+	case SNRM_CMD:
+		irlap_recv_snrm_cmd(self, skb, &info);
+		break;
+	case DM_RSP:
+		irlap_do_event(self, RECV_DM_RSP, skb, &info);
+		break;
+	case DISC_CMD: /* And RD_RSP since they have the same value */
+		irlap_recv_disc_frame(self, skb, &info, command);
+		break;
+	case TEST_CMD:
+		irlap_recv_test_frame(self, skb, &info, command);
+		break;
+	case UA_RSP:
+		irlap_recv_ua_frame(self, skb, &info);
+		break;
+	case FRMR_RSP:
+		irlap_recv_frmr_frame(self, skb, &info);
+		break;
+	case UI_FRAME:
+		irlap_recv_ui_frame(self, skb, &info);
+		break;
+	default:
+		net_warn_ratelimited("%s: Unknown frame %02x received!\n",
+				     __func__, info.control);
+		break;
+	}
+out:
+	ret = 0;
+err:
+	/* Always drop our reference on the skb */
+	dev_kfree_skb(skb);
+	return ret;
+}
diff --git a/drivers/staging/irda/net/irlmp.c b/drivers/staging/irda/net/irlmp.c
new file mode 100644
index 000000000000..43964594aa12
--- /dev/null
+++ b/drivers/staging/irda/net/irlmp.c
@@ -0,0 +1,1996 @@
+/*********************************************************************
+ *
+ * Filename:      irlmp.c
+ * Version:       1.0
+ * Description:   IrDA Link Management Protocol (LMP) layer
+ * Status:        Stable.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sun Aug 17 20:54:32 1997
+ * Modified at:   Wed Jan  5 11:26:03 2000
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1998-2000 Dag Brattli <dagb@cs.uit.no>,
+ *     All Rights Reserved.
+ *     Copyright (c) 2000-2003 Jean Tourrilhes <jt@hpl.hp.com>
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/skbuff.h>
+#include <linux/types.h>
+#include <linux/proc_fs.h>
+#include <linux/init.h>
+#include <linux/kmod.h>
+#include <linux/random.h>
+#include <linux/seq_file.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/timer.h>
+#include <net/irda/qos.h>
+#include <net/irda/irlap.h>
+#include <net/irda/iriap.h>
+#include <net/irda/irlmp.h>
+#include <net/irda/irlmp_frame.h>
+
+#include <asm/unaligned.h>
+
+static __u8 irlmp_find_free_slsap(void);
+static int irlmp_slsap_inuse(__u8 slsap_sel);
+
+/* Master structure */
+struct irlmp_cb *irlmp = NULL;
+
+/* These can be altered by the sysctl interface */
+int  sysctl_discovery         = 0;
+int  sysctl_discovery_timeout = 3; /* 3 seconds by default */
+int  sysctl_discovery_slots   = 6; /* 6 slots by default */
+int  sysctl_lap_keepalive_time = LM_IDLE_TIMEOUT * 1000 / HZ;
+char sysctl_devname[65];
+
+static const char *irlmp_reasons[] = {
+	"ERROR, NOT USED",
+	"LM_USER_REQUEST",
+	"LM_LAP_DISCONNECT",
+	"LM_CONNECT_FAILURE",
+	"LM_LAP_RESET",
+	"LM_INIT_DISCONNECT",
+	"ERROR, NOT USED",
+	"UNKNOWN",
+};
+
+const char *irlmp_reason_str(LM_REASON reason)
+{
+	reason = min_t(size_t, reason, ARRAY_SIZE(irlmp_reasons) - 1);
+	return irlmp_reasons[reason];
+}
+
+/*
+ * Function irlmp_init (void)
+ *
+ *    Create (allocate) the main IrLMP structure
+ *
+ */
+int __init irlmp_init(void)
+{
+	/* Initialize the irlmp structure. */
+	irlmp = kzalloc( sizeof(struct irlmp_cb), GFP_KERNEL);
+	if (irlmp == NULL)
+		return -ENOMEM;
+
+	irlmp->magic = LMP_MAGIC;
+
+	irlmp->clients = hashbin_new(HB_LOCK);
+	irlmp->services = hashbin_new(HB_LOCK);
+	irlmp->links = hashbin_new(HB_LOCK);
+	irlmp->unconnected_lsaps = hashbin_new(HB_LOCK);
+	irlmp->cachelog = hashbin_new(HB_NOLOCK);
+
+	if ((irlmp->clients == NULL) ||
+	    (irlmp->services == NULL) ||
+	    (irlmp->links == NULL) ||
+	    (irlmp->unconnected_lsaps == NULL) ||
+	    (irlmp->cachelog == NULL)) {
+		return -ENOMEM;
+	}
+
+	spin_lock_init(&irlmp->cachelog->hb_spinlock);
+
+	irlmp->last_lsap_sel = 0x0f; /* Reserved 0x00-0x0f */
+	strcpy(sysctl_devname, "Linux");
+
+	init_timer(&irlmp->discovery_timer);
+
+	/* Do discovery every 3 seconds, conditionally */
+	if (sysctl_discovery)
+		irlmp_start_discovery_timer(irlmp,
+					    sysctl_discovery_timeout*HZ);
+
+	return 0;
+}
+
+/*
+ * Function irlmp_cleanup (void)
+ *
+ *    Remove IrLMP layer
+ *
+ */
+void irlmp_cleanup(void)
+{
+	/* Check for main structure */
+	IRDA_ASSERT(irlmp != NULL, return;);
+	IRDA_ASSERT(irlmp->magic == LMP_MAGIC, return;);
+
+	del_timer(&irlmp->discovery_timer);
+
+	hashbin_delete(irlmp->links, (FREE_FUNC) kfree);
+	hashbin_delete(irlmp->unconnected_lsaps, (FREE_FUNC) kfree);
+	hashbin_delete(irlmp->clients, (FREE_FUNC) kfree);
+	hashbin_delete(irlmp->services, (FREE_FUNC) kfree);
+	hashbin_delete(irlmp->cachelog, (FREE_FUNC) kfree);
+
+	/* De-allocate main structure */
+	kfree(irlmp);
+	irlmp = NULL;
+}
+
+/*
+ * Function irlmp_open_lsap (slsap, notify)
+ *
+ *   Register with IrLMP and create a local LSAP,
+ *   returns handle to LSAP.
+ */
+struct lsap_cb *irlmp_open_lsap(__u8 slsap_sel, notify_t *notify, __u8 pid)
+{
+	struct lsap_cb *self;
+
+	IRDA_ASSERT(notify != NULL, return NULL;);
+	IRDA_ASSERT(irlmp != NULL, return NULL;);
+	IRDA_ASSERT(irlmp->magic == LMP_MAGIC, return NULL;);
+	IRDA_ASSERT(notify->instance != NULL, return NULL;);
+
+	/*  Does the client care which Source LSAP selector it gets?  */
+	if (slsap_sel == LSAP_ANY) {
+		slsap_sel = irlmp_find_free_slsap();
+		if (!slsap_sel)
+			return NULL;
+	} else if (irlmp_slsap_inuse(slsap_sel))
+		return NULL;
+
+	/* Allocate new instance of a LSAP connection */
+	self = kzalloc(sizeof(struct lsap_cb), GFP_ATOMIC);
+	if (self == NULL)
+		return NULL;
+
+	self->magic = LMP_LSAP_MAGIC;
+	self->slsap_sel = slsap_sel;
+
+	/* Fix connectionless LSAP's */
+	if (slsap_sel == LSAP_CONNLESS) {
+#ifdef CONFIG_IRDA_ULTRA
+		self->dlsap_sel = LSAP_CONNLESS;
+		self->pid = pid;
+#endif /* CONFIG_IRDA_ULTRA */
+	} else
+		self->dlsap_sel = LSAP_ANY;
+	/* self->connected = FALSE; -> already NULL via memset() */
+
+	init_timer(&self->watchdog_timer);
+
+	self->notify = *notify;
+
+	self->lsap_state = LSAP_DISCONNECTED;
+
+	/* Insert into queue of unconnected LSAPs */
+	hashbin_insert(irlmp->unconnected_lsaps, (irda_queue_t *) self,
+		       (long) self, NULL);
+
+	return self;
+}
+EXPORT_SYMBOL(irlmp_open_lsap);
+
+/*
+ * Function __irlmp_close_lsap (self)
+ *
+ *    Remove an instance of LSAP
+ */
+static void __irlmp_close_lsap(struct lsap_cb *self)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LMP_LSAP_MAGIC, return;);
+
+	/*
+	 *  Set some of the variables to preset values
+	 */
+	self->magic = 0;
+	del_timer(&self->watchdog_timer); /* Important! */
+
+	if (self->conn_skb)
+		dev_kfree_skb(self->conn_skb);
+
+	kfree(self);
+}
+
+/*
+ * Function irlmp_close_lsap (self)
+ *
+ *    Close and remove LSAP
+ *
+ */
+void irlmp_close_lsap(struct lsap_cb *self)
+{
+	struct lap_cb *lap;
+	struct lsap_cb *lsap = NULL;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LMP_LSAP_MAGIC, return;);
+
+	/*
+	 *  Find out if we should remove this LSAP from a link or from the
+	 *  list of unconnected lsaps (not associated with a link)
+	 */
+	lap = self->lap;
+	if (lap) {
+		IRDA_ASSERT(lap->magic == LMP_LAP_MAGIC, return;);
+		/* We might close a LSAP before it has completed the
+		 * connection setup. In those case, higher layers won't
+		 * send a proper disconnect request. Harmless, except
+		 * that we will forget to close LAP... - Jean II */
+		if(self->lsap_state != LSAP_DISCONNECTED) {
+			self->lsap_state = LSAP_DISCONNECTED;
+			irlmp_do_lap_event(self->lap,
+					   LM_LAP_DISCONNECT_REQUEST, NULL);
+		}
+		/* Now, remove from the link */
+		lsap = hashbin_remove(lap->lsaps, (long) self, NULL);
+#ifdef CONFIG_IRDA_CACHE_LAST_LSAP
+		lap->cache.valid = FALSE;
+#endif
+	}
+	self->lap = NULL;
+	/* Check if we found the LSAP! If not then try the unconnected lsaps */
+	if (!lsap) {
+		lsap = hashbin_remove(irlmp->unconnected_lsaps, (long) self,
+				      NULL);
+	}
+	if (!lsap) {
+		pr_debug("%s(), Looks like somebody has removed me already!\n",
+			 __func__);
+		return;
+	}
+	__irlmp_close_lsap(self);
+}
+EXPORT_SYMBOL(irlmp_close_lsap);
+
+/*
+ * Function irlmp_register_irlap (saddr, notify)
+ *
+ *    Register IrLAP layer with IrLMP. There is possible to have multiple
+ *    instances of the IrLAP layer, each connected to different IrDA ports
+ *
+ */
+void irlmp_register_link(struct irlap_cb *irlap, __u32 saddr, notify_t *notify)
+{
+	struct lap_cb *lap;
+
+	IRDA_ASSERT(irlmp != NULL, return;);
+	IRDA_ASSERT(irlmp->magic == LMP_MAGIC, return;);
+	IRDA_ASSERT(notify != NULL, return;);
+
+	/*
+	 *  Allocate new instance of a LSAP connection
+	 */
+	lap = kzalloc(sizeof(struct lap_cb), GFP_KERNEL);
+	if (lap == NULL)
+		return;
+
+	lap->irlap = irlap;
+	lap->magic = LMP_LAP_MAGIC;
+	lap->saddr = saddr;
+	lap->daddr = DEV_ADDR_ANY;
+#ifdef CONFIG_IRDA_CACHE_LAST_LSAP
+	lap->cache.valid = FALSE;
+#endif
+	lap->lsaps = hashbin_new(HB_LOCK);
+	if (lap->lsaps == NULL) {
+		net_warn_ratelimited("%s(), unable to kmalloc lsaps\n",
+				     __func__);
+		kfree(lap);
+		return;
+	}
+
+	lap->lap_state = LAP_STANDBY;
+
+	init_timer(&lap->idle_timer);
+
+	/*
+	 *  Insert into queue of LMP links
+	 */
+	hashbin_insert(irlmp->links, (irda_queue_t *) lap, lap->saddr, NULL);
+
+	/*
+	 *  We set only this variable so IrLAP can tell us on which link the
+	 *  different events happened on
+	 */
+	irda_notify_init(notify);
+	notify->instance = lap;
+}
+
+/*
+ * Function irlmp_unregister_irlap (saddr)
+ *
+ *    IrLAP layer has been removed!
+ *
+ */
+void irlmp_unregister_link(__u32 saddr)
+{
+	struct lap_cb *link;
+
+	/* We must remove ourselves from the hashbin *first*. This ensure
+	 * that no more LSAPs will be open on this link and no discovery
+	 * will be triggered anymore. Jean II */
+	link = hashbin_remove(irlmp->links, saddr, NULL);
+	if (link) {
+		IRDA_ASSERT(link->magic == LMP_LAP_MAGIC, return;);
+
+		/* Kill all the LSAPs on this link. Jean II */
+		link->reason = LAP_DISC_INDICATION;
+		link->daddr = DEV_ADDR_ANY;
+		irlmp_do_lap_event(link, LM_LAP_DISCONNECT_INDICATION, NULL);
+
+		/* Remove all discoveries discovered at this link */
+		irlmp_expire_discoveries(irlmp->cachelog, link->saddr, TRUE);
+
+		/* Final cleanup */
+		del_timer(&link->idle_timer);
+		link->magic = 0;
+		hashbin_delete(link->lsaps, (FREE_FUNC) __irlmp_close_lsap);
+		kfree(link);
+	}
+}
+
+/*
+ * Function irlmp_connect_request (handle, dlsap, userdata)
+ *
+ *    Connect with a peer LSAP
+ *
+ */
+int irlmp_connect_request(struct lsap_cb *self, __u8 dlsap_sel,
+			  __u32 saddr, __u32 daddr,
+			  struct qos_info *qos, struct sk_buff *userdata)
+{
+	struct sk_buff *tx_skb = userdata;
+	struct lap_cb *lap;
+	struct lsap_cb *lsap;
+	int ret;
+
+	IRDA_ASSERT(self != NULL, return -EBADR;);
+	IRDA_ASSERT(self->magic == LMP_LSAP_MAGIC, return -EBADR;);
+
+	pr_debug("%s(), slsap_sel=%02x, dlsap_sel=%02x, saddr=%08x, daddr=%08x\n",
+		 __func__, self->slsap_sel, dlsap_sel, saddr, daddr);
+
+	if (test_bit(0, &self->connected)) {
+		ret = -EISCONN;
+		goto err;
+	}
+
+	/* Client must supply destination device address */
+	if (!daddr) {
+		ret = -EINVAL;
+		goto err;
+	}
+
+	/* Any userdata? */
+	if (tx_skb == NULL) {
+		tx_skb = alloc_skb(LMP_MAX_HEADER, GFP_ATOMIC);
+		if (!tx_skb)
+			return -ENOMEM;
+
+		skb_reserve(tx_skb, LMP_MAX_HEADER);
+	}
+
+	/* Make room for MUX control header (3 bytes) */
+	IRDA_ASSERT(skb_headroom(tx_skb) >= LMP_CONTROL_HEADER, return -1;);
+	skb_push(tx_skb, LMP_CONTROL_HEADER);
+
+	self->dlsap_sel = dlsap_sel;
+
+	/*
+	 * Find the link to where we should try to connect since there may
+	 * be more than one IrDA port on this machine. If the client has
+	 * passed us the saddr (and already knows which link to use), then
+	 * we use that to find the link, if not then we have to look in the
+	 * discovery log and check if any of the links has discovered a
+	 * device with the given daddr
+	 */
+	if ((!saddr) || (saddr == DEV_ADDR_ANY)) {
+		discovery_t *discovery;
+		unsigned long flags;
+
+		spin_lock_irqsave(&irlmp->cachelog->hb_spinlock, flags);
+		if (daddr != DEV_ADDR_ANY)
+			discovery = hashbin_find(irlmp->cachelog, daddr, NULL);
+		else {
+			pr_debug("%s(), no daddr\n", __func__);
+			discovery = (discovery_t *)
+				hashbin_get_first(irlmp->cachelog);
+		}
+
+		if (discovery) {
+			saddr = discovery->data.saddr;
+			daddr = discovery->data.daddr;
+		}
+		spin_unlock_irqrestore(&irlmp->cachelog->hb_spinlock, flags);
+	}
+	lap = hashbin_lock_find(irlmp->links, saddr, NULL);
+	if (lap == NULL) {
+		pr_debug("%s(), Unable to find a usable link!\n", __func__);
+		ret = -EHOSTUNREACH;
+		goto err;
+	}
+
+	/* Check if LAP is disconnected or already connected */
+	if (lap->daddr == DEV_ADDR_ANY)
+		lap->daddr = daddr;
+	else if (lap->daddr != daddr) {
+		/* Check if some LSAPs are active on this LAP */
+		if (HASHBIN_GET_SIZE(lap->lsaps) == 0) {
+			/* No active connection, but LAP hasn't been
+			 * disconnected yet (waiting for timeout in LAP).
+			 * Maybe we could give LAP a bit of help in this case.
+			 */
+			pr_debug("%s(), sorry, but I'm waiting for LAP to timeout!\n",
+				 __func__);
+			ret = -EAGAIN;
+			goto err;
+		}
+
+		/* LAP is already connected to a different node, and LAP
+		 * can only talk to one node at a time */
+		pr_debug("%s(), sorry, but link is busy!\n", __func__);
+		ret = -EBUSY;
+		goto err;
+	}
+
+	self->lap = lap;
+
+	/*
+	 *  Remove LSAP from list of unconnected LSAPs and insert it into the
+	 *  list of connected LSAPs for the particular link
+	 */
+	lsap = hashbin_remove(irlmp->unconnected_lsaps, (long) self, NULL);
+
+	IRDA_ASSERT(lsap != NULL, return -1;);
+	IRDA_ASSERT(lsap->magic == LMP_LSAP_MAGIC, return -1;);
+	IRDA_ASSERT(lsap->lap != NULL, return -1;);
+	IRDA_ASSERT(lsap->lap->magic == LMP_LAP_MAGIC, return -1;);
+
+	hashbin_insert(self->lap->lsaps, (irda_queue_t *) self, (long) self,
+		       NULL);
+
+	set_bit(0, &self->connected);	/* TRUE */
+
+	/*
+	 *  User supplied qos specifications?
+	 */
+	if (qos)
+		self->qos = *qos;
+
+	irlmp_do_lsap_event(self, LM_CONNECT_REQUEST, tx_skb);
+
+	/* Drop reference count - see irlap_data_request(). */
+	dev_kfree_skb(tx_skb);
+
+	return 0;
+
+err:
+	/* Cleanup */
+	if(tx_skb)
+		dev_kfree_skb(tx_skb);
+	return ret;
+}
+EXPORT_SYMBOL(irlmp_connect_request);
+
+/*
+ * Function irlmp_connect_indication (self)
+ *
+ *    Incoming connection
+ *
+ */
+void irlmp_connect_indication(struct lsap_cb *self, struct sk_buff *skb)
+{
+	int max_seg_size;
+	int lap_header_size;
+	int max_header_size;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LMP_LSAP_MAGIC, return;);
+	IRDA_ASSERT(skb != NULL, return;);
+	IRDA_ASSERT(self->lap != NULL, return;);
+
+	pr_debug("%s(), slsap_sel=%02x, dlsap_sel=%02x\n",
+		 __func__, self->slsap_sel, self->dlsap_sel);
+
+	/* Note : self->lap is set in irlmp_link_data_indication(),
+	 * (case CONNECT_CMD:) because we have no way to set it here.
+	 * Similarly, self->dlsap_sel is usually set in irlmp_find_lsap().
+	 * Jean II */
+
+	self->qos = *self->lap->qos;
+
+	max_seg_size = self->lap->qos->data_size.value-LMP_HEADER;
+	lap_header_size = IRLAP_GET_HEADER_SIZE(self->lap->irlap);
+	max_header_size = LMP_HEADER + lap_header_size;
+
+	/* Hide LMP_CONTROL_HEADER header from layer above */
+	skb_pull(skb, LMP_CONTROL_HEADER);
+
+	if (self->notify.connect_indication) {
+		/* Don't forget to refcount it - see irlap_driver_rcv(). */
+		skb_get(skb);
+		self->notify.connect_indication(self->notify.instance, self,
+						&self->qos, max_seg_size,
+						max_header_size, skb);
+	}
+}
+
+/*
+ * Function irlmp_connect_response (handle, userdata)
+ *
+ *    Service user is accepting connection
+ *
+ */
+int irlmp_connect_response(struct lsap_cb *self, struct sk_buff *userdata)
+{
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == LMP_LSAP_MAGIC, return -1;);
+	IRDA_ASSERT(userdata != NULL, return -1;);
+
+	/* We set the connected bit and move the lsap to the connected list
+	 * in the state machine itself. Jean II */
+
+	pr_debug("%s(), slsap_sel=%02x, dlsap_sel=%02x\n",
+		 __func__, self->slsap_sel, self->dlsap_sel);
+
+	/* Make room for MUX control header (3 bytes) */
+	IRDA_ASSERT(skb_headroom(userdata) >= LMP_CONTROL_HEADER, return -1;);
+	skb_push(userdata, LMP_CONTROL_HEADER);
+
+	irlmp_do_lsap_event(self, LM_CONNECT_RESPONSE, userdata);
+
+	/* Drop reference count - see irlap_data_request(). */
+	dev_kfree_skb(userdata);
+
+	return 0;
+}
+EXPORT_SYMBOL(irlmp_connect_response);
+
+/*
+ * Function irlmp_connect_confirm (handle, skb)
+ *
+ *    LSAP connection confirmed peer device!
+ */
+void irlmp_connect_confirm(struct lsap_cb *self, struct sk_buff *skb)
+{
+	int max_header_size;
+	int lap_header_size;
+	int max_seg_size;
+
+	IRDA_ASSERT(skb != NULL, return;);
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LMP_LSAP_MAGIC, return;);
+	IRDA_ASSERT(self->lap != NULL, return;);
+
+	self->qos = *self->lap->qos;
+
+	max_seg_size    = self->lap->qos->data_size.value-LMP_HEADER;
+	lap_header_size = IRLAP_GET_HEADER_SIZE(self->lap->irlap);
+	max_header_size = LMP_HEADER + lap_header_size;
+
+	pr_debug("%s(), max_header_size=%d\n",
+		 __func__, max_header_size);
+
+	/* Hide LMP_CONTROL_HEADER header from layer above */
+	skb_pull(skb, LMP_CONTROL_HEADER);
+
+	if (self->notify.connect_confirm) {
+		/* Don't forget to refcount it - see irlap_driver_rcv() */
+		skb_get(skb);
+		self->notify.connect_confirm(self->notify.instance, self,
+					     &self->qos, max_seg_size,
+					     max_header_size, skb);
+	}
+}
+
+/*
+ * Function irlmp_dup (orig, instance)
+ *
+ *    Duplicate LSAP, can be used by servers to confirm a connection on a
+ *    new LSAP so it can keep listening on the old one.
+ *
+ */
+struct lsap_cb *irlmp_dup(struct lsap_cb *orig, void *instance)
+{
+	struct lsap_cb *new;
+	unsigned long flags;
+
+	spin_lock_irqsave(&irlmp->unconnected_lsaps->hb_spinlock, flags);
+
+	/* Only allowed to duplicate unconnected LSAP's, and only LSAPs
+	 * that have received a connect indication. Jean II */
+	if ((!hashbin_find(irlmp->unconnected_lsaps, (long) orig, NULL)) ||
+	    (orig->lap == NULL)) {
+		pr_debug("%s(), invalid LSAP (wrong state)\n",
+			 __func__);
+		spin_unlock_irqrestore(&irlmp->unconnected_lsaps->hb_spinlock,
+				       flags);
+		return NULL;
+	}
+
+	/* Allocate a new instance */
+	new = kmemdup(orig, sizeof(*new), GFP_ATOMIC);
+	if (!new)  {
+		pr_debug("%s(), unable to kmalloc\n", __func__);
+		spin_unlock_irqrestore(&irlmp->unconnected_lsaps->hb_spinlock,
+				       flags);
+		return NULL;
+	}
+	/* new->lap = orig->lap; => done in the memcpy() */
+	/* new->slsap_sel = orig->slsap_sel; => done in the memcpy() */
+	new->conn_skb = NULL;
+
+	spin_unlock_irqrestore(&irlmp->unconnected_lsaps->hb_spinlock, flags);
+
+	/* Not everything is the same */
+	new->notify.instance = instance;
+
+	init_timer(&new->watchdog_timer);
+
+	hashbin_insert(irlmp->unconnected_lsaps, (irda_queue_t *) new,
+		       (long) new, NULL);
+
+#ifdef CONFIG_IRDA_CACHE_LAST_LSAP
+	/* Make sure that we invalidate the LSAP cache */
+	new->lap->cache.valid = FALSE;
+#endif /* CONFIG_IRDA_CACHE_LAST_LSAP */
+
+	return new;
+}
+
+/*
+ * Function irlmp_disconnect_request (handle, userdata)
+ *
+ *    The service user is requesting disconnection, this will not remove the
+ *    LSAP, but only mark it as disconnected
+ */
+int irlmp_disconnect_request(struct lsap_cb *self, struct sk_buff *userdata)
+{
+	struct lsap_cb *lsap;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == LMP_LSAP_MAGIC, return -1;);
+	IRDA_ASSERT(userdata != NULL, return -1;);
+
+	/* Already disconnected ?
+	 * There is a race condition between irlmp_disconnect_indication()
+	 * and us that might mess up the hashbins below. This fixes it.
+	 * Jean II */
+	if (! test_and_clear_bit(0, &self->connected)) {
+		pr_debug("%s(), already disconnected!\n", __func__);
+		dev_kfree_skb(userdata);
+		return -1;
+	}
+
+	skb_push(userdata, LMP_CONTROL_HEADER);
+
+	/*
+	 *  Do the event before the other stuff since we must know
+	 *  which lap layer that the frame should be transmitted on
+	 */
+	irlmp_do_lsap_event(self, LM_DISCONNECT_REQUEST, userdata);
+
+	/* Drop reference count - see irlap_data_request(). */
+	dev_kfree_skb(userdata);
+
+	/*
+	 *  Remove LSAP from list of connected LSAPs for the particular link
+	 *  and insert it into the list of unconnected LSAPs
+	 */
+	IRDA_ASSERT(self->lap != NULL, return -1;);
+	IRDA_ASSERT(self->lap->magic == LMP_LAP_MAGIC, return -1;);
+	IRDA_ASSERT(self->lap->lsaps != NULL, return -1;);
+
+	lsap = hashbin_remove(self->lap->lsaps, (long) self, NULL);
+#ifdef CONFIG_IRDA_CACHE_LAST_LSAP
+	self->lap->cache.valid = FALSE;
+#endif
+
+	IRDA_ASSERT(lsap != NULL, return -1;);
+	IRDA_ASSERT(lsap->magic == LMP_LSAP_MAGIC, return -1;);
+	IRDA_ASSERT(lsap == self, return -1;);
+
+	hashbin_insert(irlmp->unconnected_lsaps, (irda_queue_t *) self,
+		       (long) self, NULL);
+
+	/* Reset some values */
+	self->dlsap_sel = LSAP_ANY;
+	self->lap = NULL;
+
+	return 0;
+}
+EXPORT_SYMBOL(irlmp_disconnect_request);
+
+/*
+ * Function irlmp_disconnect_indication (reason, userdata)
+ *
+ *    LSAP is being closed!
+ */
+void irlmp_disconnect_indication(struct lsap_cb *self, LM_REASON reason,
+				 struct sk_buff *skb)
+{
+	struct lsap_cb *lsap;
+
+	pr_debug("%s(), reason=%s [%d]\n", __func__,
+		 irlmp_reason_str(reason), reason);
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LMP_LSAP_MAGIC, return;);
+
+	pr_debug("%s(), slsap_sel=%02x, dlsap_sel=%02x\n",
+		 __func__, self->slsap_sel, self->dlsap_sel);
+
+	/* Already disconnected ?
+	 * There is a race condition between irlmp_disconnect_request()
+	 * and us that might mess up the hashbins below. This fixes it.
+	 * Jean II */
+	if (! test_and_clear_bit(0, &self->connected)) {
+		pr_debug("%s(), already disconnected!\n", __func__);
+		return;
+	}
+
+	/*
+	 *  Remove association between this LSAP and the link it used
+	 */
+	IRDA_ASSERT(self->lap != NULL, return;);
+	IRDA_ASSERT(self->lap->lsaps != NULL, return;);
+
+	lsap = hashbin_remove(self->lap->lsaps, (long) self, NULL);
+#ifdef CONFIG_IRDA_CACHE_LAST_LSAP
+	self->lap->cache.valid = FALSE;
+#endif
+
+	IRDA_ASSERT(lsap != NULL, return;);
+	IRDA_ASSERT(lsap == self, return;);
+	hashbin_insert(irlmp->unconnected_lsaps, (irda_queue_t *) lsap,
+		       (long) lsap, NULL);
+
+	self->dlsap_sel = LSAP_ANY;
+	self->lap = NULL;
+
+	/*
+	 *  Inform service user
+	 */
+	if (self->notify.disconnect_indication) {
+		/* Don't forget to refcount it - see irlap_driver_rcv(). */
+		if(skb)
+			skb_get(skb);
+		self->notify.disconnect_indication(self->notify.instance,
+						   self, reason, skb);
+	} else {
+		pr_debug("%s(), no handler\n", __func__);
+	}
+}
+
+/*
+ * Function irlmp_do_expiry (void)
+ *
+ *    Do a cleanup of the discovery log (remove old entries)
+ *
+ * Note : separate from irlmp_do_discovery() so that we can handle
+ * passive discovery properly.
+ */
+void irlmp_do_expiry(void)
+{
+	struct lap_cb *lap;
+
+	/*
+	 * Expire discovery on all links which are *not* connected.
+	 * On links which are connected, we can't do discovery
+	 * anymore and can't refresh the log, so we freeze the
+	 * discovery log to keep info about the device we are
+	 * connected to.
+	 * This info is mandatory if we want irlmp_connect_request()
+	 * to work properly. - Jean II
+	 */
+	lap = (struct lap_cb *) hashbin_get_first(irlmp->links);
+	while (lap != NULL) {
+		IRDA_ASSERT(lap->magic == LMP_LAP_MAGIC, return;);
+
+		if (lap->lap_state == LAP_STANDBY) {
+			/* Expire discoveries discovered on this link */
+			irlmp_expire_discoveries(irlmp->cachelog, lap->saddr,
+						 FALSE);
+		}
+		lap = (struct lap_cb *) hashbin_get_next(irlmp->links);
+	}
+}
+
+/*
+ * Function irlmp_do_discovery (nslots)
+ *
+ *    Do some discovery on all links
+ *
+ * Note : log expiry is done above.
+ */
+void irlmp_do_discovery(int nslots)
+{
+	struct lap_cb *lap;
+	__u16 *data_hintsp;
+
+	/* Make sure the value is sane */
+	if ((nslots != 1) && (nslots != 6) && (nslots != 8) && (nslots != 16)){
+		net_warn_ratelimited("%s: invalid value for number of slots!\n",
+				     __func__);
+		nslots = sysctl_discovery_slots = 8;
+	}
+
+	/* Construct new discovery info to be used by IrLAP, */
+	data_hintsp = (__u16 *) irlmp->discovery_cmd.data.hints;
+	put_unaligned(irlmp->hints.word, data_hintsp);
+
+	/*
+	 *  Set character set for device name (we use ASCII), and
+	 *  copy device name. Remember to make room for a \0 at the
+	 *  end
+	 */
+	irlmp->discovery_cmd.data.charset = CS_ASCII;
+	strncpy(irlmp->discovery_cmd.data.info, sysctl_devname,
+		NICKNAME_MAX_LEN);
+	irlmp->discovery_cmd.name_len = strlen(irlmp->discovery_cmd.data.info);
+	irlmp->discovery_cmd.nslots = nslots;
+
+	/*
+	 * Try to send discovery packets on all links
+	 */
+	lap = (struct lap_cb *) hashbin_get_first(irlmp->links);
+	while (lap != NULL) {
+		IRDA_ASSERT(lap->magic == LMP_LAP_MAGIC, return;);
+
+		if (lap->lap_state == LAP_STANDBY) {
+			/* Try to discover */
+			irlmp_do_lap_event(lap, LM_LAP_DISCOVERY_REQUEST,
+					   NULL);
+		}
+		lap = (struct lap_cb *) hashbin_get_next(irlmp->links);
+	}
+}
+
+/*
+ * Function irlmp_discovery_request (nslots)
+ *
+ *    Do a discovery of devices in front of the computer
+ *
+ * If the caller has registered a client discovery callback, this
+ * allow him to receive the full content of the discovery log through
+ * this callback (as normally he will receive only new discoveries).
+ */
+void irlmp_discovery_request(int nslots)
+{
+	/* Return current cached discovery log (in full) */
+	irlmp_discovery_confirm(irlmp->cachelog, DISCOVERY_LOG);
+
+	/*
+	 * Start a single discovery operation if discovery is not already
+	 * running
+	 */
+	if (!sysctl_discovery) {
+		/* Check if user wants to override the default */
+		if (nslots == DISCOVERY_DEFAULT_SLOTS)
+			nslots = sysctl_discovery_slots;
+
+		irlmp_do_discovery(nslots);
+		/* Note : we never do expiry here. Expiry will run on the
+		 * discovery timer regardless of the state of sysctl_discovery
+		 * Jean II */
+	}
+}
+EXPORT_SYMBOL(irlmp_discovery_request);
+
+/*
+ * Function irlmp_get_discoveries (pn, mask, slots)
+ *
+ *    Return the current discovery log
+ *
+ * If discovery is not enabled, you should call this function again
+ * after 1 or 2 seconds (i.e. after discovery has been done).
+ */
+struct irda_device_info *irlmp_get_discoveries(int *pn, __u16 mask, int nslots)
+{
+	/* If discovery is not enabled, it's likely that the discovery log
+	 * will be empty. So, we trigger a single discovery, so that next
+	 * time the user call us there might be some results in the log.
+	 * Jean II
+	 */
+	if (!sysctl_discovery) {
+		/* Check if user wants to override the default */
+		if (nslots == DISCOVERY_DEFAULT_SLOTS)
+			nslots = sysctl_discovery_slots;
+
+		/* Start discovery - will complete sometime later */
+		irlmp_do_discovery(nslots);
+		/* Note : we never do expiry here. Expiry will run on the
+		 * discovery timer regardless of the state of sysctl_discovery
+		 * Jean II */
+	}
+
+	/* Return current cached discovery log */
+	return irlmp_copy_discoveries(irlmp->cachelog, pn, mask, TRUE);
+}
+EXPORT_SYMBOL(irlmp_get_discoveries);
+
+/*
+ * Function irlmp_notify_client (log)
+ *
+ *    Notify all about discovered devices
+ *
+ * Clients registered with IrLMP are :
+ *	o IrComm
+ *	o IrLAN
+ *	o Any socket (in any state - ouch, that may be a lot !)
+ * The client may have defined a callback to be notified in case of
+ * partial/selective discovery based on the hints that it passed to IrLMP.
+ */
+static inline void
+irlmp_notify_client(irlmp_client_t *client,
+		    hashbin_t *log, DISCOVERY_MODE mode)
+{
+	discinfo_t *discoveries;	/* Copy of the discovery log */
+	int	number;			/* Number of nodes in the log */
+	int	i;
+
+	/* Check if client wants or not partial/selective log (optimisation) */
+	if (!client->disco_callback)
+		return;
+
+	/*
+	 * Locking notes :
+	 * the old code was manipulating the log directly, which was
+	 * very racy. Now, we use copy_discoveries, that protects
+	 * itself while dumping the log for us.
+	 * The overhead of the copy is compensated by the fact that
+	 * we only pass new discoveries in normal mode and don't
+	 * pass the same old entry every 3s to the caller as we used
+	 * to do (virtual function calling is expensive).
+	 * Jean II
+	 */
+
+	/*
+	 * Now, check all discovered devices (if any), and notify client
+	 * only about the services that the client is interested in
+	 * We also notify only about the new devices unless the caller
+	 * explicitly request a dump of the log. Jean II
+	 */
+	discoveries = irlmp_copy_discoveries(log, &number,
+					     client->hint_mask.word,
+					     (mode == DISCOVERY_LOG));
+	/* Check if the we got some results */
+	if (discoveries == NULL)
+		return;	/* No nodes discovered */
+
+	/* Pass all entries to the listener */
+	for(i = 0; i < number; i++)
+		client->disco_callback(&(discoveries[i]), mode, client->priv);
+
+	/* Free up our buffer */
+	kfree(discoveries);
+}
+
+/*
+ * Function irlmp_discovery_confirm ( self, log)
+ *
+ *    Some device(s) answered to our discovery request! Check to see which
+ *    device it is, and give indication to the client(s)
+ *
+ */
+void irlmp_discovery_confirm(hashbin_t *log, DISCOVERY_MODE mode)
+{
+	irlmp_client_t *client;
+	irlmp_client_t *client_next;
+
+	IRDA_ASSERT(log != NULL, return;);
+
+	if (!(HASHBIN_GET_SIZE(log)))
+		return;
+
+	/* For each client - notify callback may touch client list */
+	client = (irlmp_client_t *) hashbin_get_first(irlmp->clients);
+	while (NULL != hashbin_find_next(irlmp->clients, (long) client, NULL,
+					 (void *) &client_next) ) {
+		/* Check if we should notify client */
+		irlmp_notify_client(client, log, mode);
+
+		client = client_next;
+	}
+}
+
+/*
+ * Function irlmp_discovery_expiry (expiry)
+ *
+ *	This device is no longer been discovered, and therefore it is being
+ *	purged from the discovery log. Inform all clients who have
+ *	registered for this event...
+ *
+ *	Note : called exclusively from discovery.c
+ *	Note : this is no longer called under discovery spinlock, so the
+ *		client can do whatever he wants in the callback.
+ */
+void irlmp_discovery_expiry(discinfo_t *expiries, int number)
+{
+	irlmp_client_t *client;
+	irlmp_client_t *client_next;
+	int		i;
+
+	IRDA_ASSERT(expiries != NULL, return;);
+
+	/* For each client - notify callback may touch client list */
+	client = (irlmp_client_t *) hashbin_get_first(irlmp->clients);
+	while (NULL != hashbin_find_next(irlmp->clients, (long) client, NULL,
+					 (void *) &client_next) ) {
+
+		/* Pass all entries to the listener */
+		for(i = 0; i < number; i++) {
+			/* Check if we should notify client */
+			if ((client->expir_callback) &&
+			    (client->hint_mask.word &
+			     get_unaligned((__u16 *)expiries[i].hints)
+			     & 0x7f7f) )
+				client->expir_callback(&(expiries[i]),
+						       EXPIRY_TIMEOUT,
+						       client->priv);
+		}
+
+		/* Next client */
+		client = client_next;
+	}
+}
+
+/*
+ * Function irlmp_get_discovery_response ()
+ *
+ *    Used by IrLAP to get the discovery info it needs when answering
+ *    discovery requests by other devices.
+ */
+discovery_t *irlmp_get_discovery_response(void)
+{
+	IRDA_ASSERT(irlmp != NULL, return NULL;);
+
+	put_unaligned(irlmp->hints.word, (__u16 *)irlmp->discovery_rsp.data.hints);
+
+	/*
+	 *  Set character set for device name (we use ASCII), and
+	 *  copy device name. Remember to make room for a \0 at the
+	 *  end
+	 */
+	irlmp->discovery_rsp.data.charset = CS_ASCII;
+
+	strncpy(irlmp->discovery_rsp.data.info, sysctl_devname,
+		NICKNAME_MAX_LEN);
+	irlmp->discovery_rsp.name_len = strlen(irlmp->discovery_rsp.data.info);
+
+	return &irlmp->discovery_rsp;
+}
+
+/*
+ * Function irlmp_data_request (self, skb)
+ *
+ *    Send some data to peer device
+ *
+ * Note on skb management :
+ * After calling the lower layers of the IrDA stack, we always
+ * kfree() the skb, which drop the reference count (and potentially
+ * destroy it).
+ * IrLMP and IrLAP may queue the packet, and in those cases will need
+ * to use skb_get() to keep it around.
+ * Jean II
+ */
+int irlmp_data_request(struct lsap_cb *self, struct sk_buff *userdata)
+{
+	int	ret;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == LMP_LSAP_MAGIC, return -1;);
+
+	/* Make room for MUX header */
+	IRDA_ASSERT(skb_headroom(userdata) >= LMP_HEADER, return -1;);
+	skb_push(userdata, LMP_HEADER);
+
+	ret = irlmp_do_lsap_event(self, LM_DATA_REQUEST, userdata);
+
+	/* Drop reference count - see irlap_data_request(). */
+	dev_kfree_skb(userdata);
+
+	return ret;
+}
+EXPORT_SYMBOL(irlmp_data_request);
+
+/*
+ * Function irlmp_data_indication (handle, skb)
+ *
+ *    Got data from LAP layer so pass it up to upper layer
+ *
+ */
+void irlmp_data_indication(struct lsap_cb *self, struct sk_buff *skb)
+{
+	/* Hide LMP header from layer above */
+	skb_pull(skb, LMP_HEADER);
+
+	if (self->notify.data_indication) {
+		/* Don't forget to refcount it - see irlap_driver_rcv(). */
+		skb_get(skb);
+		self->notify.data_indication(self->notify.instance, self, skb);
+	}
+}
+
+/*
+ * Function irlmp_udata_request (self, skb)
+ */
+int irlmp_udata_request(struct lsap_cb *self, struct sk_buff *userdata)
+{
+	int	ret;
+
+	IRDA_ASSERT(userdata != NULL, return -1;);
+
+	/* Make room for MUX header */
+	IRDA_ASSERT(skb_headroom(userdata) >= LMP_HEADER, return -1;);
+	skb_push(userdata, LMP_HEADER);
+
+	ret = irlmp_do_lsap_event(self, LM_UDATA_REQUEST, userdata);
+
+	/* Drop reference count - see irlap_data_request(). */
+	dev_kfree_skb(userdata);
+
+	return ret;
+}
+
+/*
+ * Function irlmp_udata_indication (self, skb)
+ *
+ *    Send unreliable data (but still within the connection)
+ *
+ */
+void irlmp_udata_indication(struct lsap_cb *self, struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LMP_LSAP_MAGIC, return;);
+	IRDA_ASSERT(skb != NULL, return;);
+
+	/* Hide LMP header from layer above */
+	skb_pull(skb, LMP_HEADER);
+
+	if (self->notify.udata_indication) {
+		/* Don't forget to refcount it - see irlap_driver_rcv(). */
+		skb_get(skb);
+		self->notify.udata_indication(self->notify.instance, self,
+					      skb);
+	}
+}
+
+/*
+ * Function irlmp_connless_data_request (self, skb)
+ */
+#ifdef CONFIG_IRDA_ULTRA
+int irlmp_connless_data_request(struct lsap_cb *self, struct sk_buff *userdata,
+				__u8 pid)
+{
+	struct sk_buff *clone_skb;
+	struct lap_cb *lap;
+
+	IRDA_ASSERT(userdata != NULL, return -1;);
+
+	/* Make room for MUX and PID header */
+	IRDA_ASSERT(skb_headroom(userdata) >= LMP_HEADER+LMP_PID_HEADER,
+		    return -1;);
+
+	/* Insert protocol identifier */
+	skb_push(userdata, LMP_PID_HEADER);
+	if(self != NULL)
+	  userdata->data[0] = self->pid;
+	else
+	  userdata->data[0] = pid;
+
+	/* Connectionless sockets must use 0x70 */
+	skb_push(userdata, LMP_HEADER);
+	userdata->data[0] = userdata->data[1] = LSAP_CONNLESS;
+
+	/* Try to send Connectionless  packets out on all links */
+	lap = (struct lap_cb *) hashbin_get_first(irlmp->links);
+	while (lap != NULL) {
+		IRDA_ASSERT(lap->magic == LMP_LAP_MAGIC, return -1;);
+
+		clone_skb = skb_clone(userdata, GFP_ATOMIC);
+		if (!clone_skb) {
+			dev_kfree_skb(userdata);
+			return -ENOMEM;
+		}
+
+		irlap_unitdata_request(lap->irlap, clone_skb);
+		/* irlap_unitdata_request() don't increase refcount,
+		 * so no dev_kfree_skb() - Jean II */
+
+		lap = (struct lap_cb *) hashbin_get_next(irlmp->links);
+	}
+	dev_kfree_skb(userdata);
+
+	return 0;
+}
+#endif /* CONFIG_IRDA_ULTRA */
+
+/*
+ * Function irlmp_connless_data_indication (self, skb)
+ *
+ *    Receive unreliable data outside any connection. Mostly used by Ultra
+ *
+ */
+#ifdef CONFIG_IRDA_ULTRA
+void irlmp_connless_data_indication(struct lsap_cb *self, struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LMP_LSAP_MAGIC, return;);
+	IRDA_ASSERT(skb != NULL, return;);
+
+	/* Hide LMP and PID header from layer above */
+	skb_pull(skb, LMP_HEADER+LMP_PID_HEADER);
+
+	if (self->notify.udata_indication) {
+		/* Don't forget to refcount it - see irlap_driver_rcv(). */
+		skb_get(skb);
+		self->notify.udata_indication(self->notify.instance, self,
+					      skb);
+	}
+}
+#endif /* CONFIG_IRDA_ULTRA */
+
+/*
+ * Propagate status indication from LAP to LSAPs (via LMP)
+ * This don't trigger any change of state in lap_cb, lmp_cb or lsap_cb,
+ * and the event is stateless, therefore we can bypass both state machines
+ * and send the event direct to the LSAP user.
+ * Jean II
+ */
+void irlmp_status_indication(struct lap_cb *self,
+			     LINK_STATUS link, LOCK_STATUS lock)
+{
+	struct lsap_cb *next;
+	struct lsap_cb *curr;
+
+	/* Send status_indication to all LSAPs using this link */
+	curr = (struct lsap_cb *) hashbin_get_first( self->lsaps);
+	while (NULL != hashbin_find_next(self->lsaps, (long) curr, NULL,
+					 (void *) &next) ) {
+		IRDA_ASSERT(curr->magic == LMP_LSAP_MAGIC, return;);
+		/*
+		 *  Inform service user if he has requested it
+		 */
+		if (curr->notify.status_indication != NULL)
+			curr->notify.status_indication(curr->notify.instance,
+						       link, lock);
+		else
+			pr_debug("%s(), no handler\n", __func__);
+
+		curr = next;
+	}
+}
+
+/*
+ * Receive flow control indication from LAP.
+ * LAP want us to send it one more frame. We implement a simple round
+ * robin scheduler between the active sockets so that we get a bit of
+ * fairness. Note that the round robin is far from perfect, but it's
+ * better than nothing.
+ * We then poll the selected socket so that we can do synchronous
+ * refilling of IrLAP (which allow to minimise the number of buffers).
+ * Jean II
+ */
+void irlmp_flow_indication(struct lap_cb *self, LOCAL_FLOW flow)
+{
+	struct lsap_cb *next;
+	struct lsap_cb *curr;
+	int	lsap_todo;
+
+	IRDA_ASSERT(self->magic == LMP_LAP_MAGIC, return;);
+	IRDA_ASSERT(flow == FLOW_START, return;);
+
+	/* Get the number of lsap. That's the only safe way to know
+	 * that we have looped around... - Jean II */
+	lsap_todo = HASHBIN_GET_SIZE(self->lsaps);
+	pr_debug("%s() : %d lsaps to scan\n", __func__, lsap_todo);
+
+	/* Poll lsap in order until the queue is full or until we
+	 * tried them all.
+	 * Most often, the current LSAP will have something to send,
+	 * so we will go through this loop only once. - Jean II */
+	while((lsap_todo--) &&
+	      (IRLAP_GET_TX_QUEUE_LEN(self->irlap) < LAP_HIGH_THRESHOLD)) {
+		/* Try to find the next lsap we should poll. */
+		next = self->flow_next;
+		/* If we have no lsap, restart from first one */
+		if(next == NULL)
+			next = (struct lsap_cb *) hashbin_get_first(self->lsaps);
+		/* Verify current one and find the next one */
+		curr = hashbin_find_next(self->lsaps, (long) next, NULL,
+					 (void *) &self->flow_next);
+		/* Uh-oh... Paranoia */
+		if(curr == NULL)
+			break;
+		pr_debug("%s() : curr is %p, next was %p and is now %p, still %d to go - queue len = %d\n",
+			 __func__, curr, next, self->flow_next, lsap_todo,
+			 IRLAP_GET_TX_QUEUE_LEN(self->irlap));
+
+		/* Inform lsap user that it can send one more packet. */
+		if (curr->notify.flow_indication != NULL)
+			curr->notify.flow_indication(curr->notify.instance,
+						     curr, flow);
+		else
+			pr_debug("%s(), no handler\n", __func__);
+	}
+}
+
+#if 0
+/*
+ * Function irlmp_hint_to_service (hint)
+ *
+ *    Returns a list of all servics contained in the given hint bits. This
+ *    function assumes that the hint bits have the size of two bytes only
+ */
+__u8 *irlmp_hint_to_service(__u8 *hint)
+{
+	__u8 *service;
+	int i = 0;
+
+	/*
+	 * Allocate array to store services in. 16 entries should be safe
+	 * since we currently only support 2 hint bytes
+	 */
+	service = kmalloc(16, GFP_ATOMIC);
+	if (!service)
+		return NULL;
+
+	if (!hint[0]) {
+		pr_debug("<None>\n");
+		kfree(service);
+		return NULL;
+	}
+	if (hint[0] & HINT_PNP)
+		pr_debug("PnP Compatible ");
+	if (hint[0] & HINT_PDA)
+		pr_debug("PDA/Palmtop ");
+	if (hint[0] & HINT_COMPUTER)
+		pr_debug("Computer ");
+	if (hint[0] & HINT_PRINTER) {
+		pr_debug("Printer ");
+		service[i++] = S_PRINTER;
+	}
+	if (hint[0] & HINT_MODEM)
+		pr_debug("Modem ");
+	if (hint[0] & HINT_FAX)
+		pr_debug("Fax ");
+	if (hint[0] & HINT_LAN) {
+		pr_debug("LAN Access ");
+		service[i++] = S_LAN;
+	}
+	/*
+	 *  Test if extension byte exists. This byte will usually be
+	 *  there, but this is not really required by the standard.
+	 *  (IrLMP p. 29)
+	 */
+	if (hint[0] & HINT_EXTENSION) {
+		if (hint[1] & HINT_TELEPHONY) {
+			pr_debug("Telephony ");
+			service[i++] = S_TELEPHONY;
+		}
+		if (hint[1] & HINT_FILE_SERVER)
+			pr_debug("File Server ");
+
+		if (hint[1] & HINT_COMM) {
+			pr_debug("IrCOMM ");
+			service[i++] = S_COMM;
+		}
+		if (hint[1] & HINT_OBEX) {
+			pr_debug("IrOBEX ");
+			service[i++] = S_OBEX;
+		}
+	}
+	pr_debug("\n");
+
+	/* So that client can be notified about any discovery */
+	service[i++] = S_ANY;
+
+	service[i] = S_END;
+
+	return service;
+}
+#endif
+
+static const __u16 service_hint_mapping[S_END][2] = {
+	{ HINT_PNP,		0 },			/* S_PNP */
+	{ HINT_PDA,		0 },			/* S_PDA */
+	{ HINT_COMPUTER,	0 },			/* S_COMPUTER */
+	{ HINT_PRINTER,		0 },			/* S_PRINTER */
+	{ HINT_MODEM,		0 },			/* S_MODEM */
+	{ HINT_FAX,		0 },			/* S_FAX */
+	{ HINT_LAN,		0 },			/* S_LAN */
+	{ HINT_EXTENSION,	HINT_TELEPHONY },	/* S_TELEPHONY */
+	{ HINT_EXTENSION,	HINT_COMM },		/* S_COMM */
+	{ HINT_EXTENSION,	HINT_OBEX },		/* S_OBEX */
+	{ 0xFF,			0xFF },			/* S_ANY */
+};
+
+/*
+ * Function irlmp_service_to_hint (service)
+ *
+ *    Converts a service type, to a hint bit
+ *
+ *    Returns: a 16 bit hint value, with the service bit set
+ */
+__u16 irlmp_service_to_hint(int service)
+{
+	__u16_host_order hint;
+
+	hint.byte[0] = service_hint_mapping[service][0];
+	hint.byte[1] = service_hint_mapping[service][1];
+
+	return hint.word;
+}
+EXPORT_SYMBOL(irlmp_service_to_hint);
+
+/*
+ * Function irlmp_register_service (service)
+ *
+ *    Register local service with IrLMP
+ *
+ */
+void *irlmp_register_service(__u16 hints)
+{
+	irlmp_service_t *service;
+
+	pr_debug("%s(), hints = %04x\n", __func__, hints);
+
+	/* Make a new registration */
+	service = kmalloc(sizeof(irlmp_service_t), GFP_ATOMIC);
+	if (!service)
+		return NULL;
+
+	service->hints.word = hints;
+	hashbin_insert(irlmp->services, (irda_queue_t *) service,
+		       (long) service, NULL);
+
+	irlmp->hints.word |= hints;
+
+	return (void *)service;
+}
+EXPORT_SYMBOL(irlmp_register_service);
+
+/*
+ * Function irlmp_unregister_service (handle)
+ *
+ *    Unregister service with IrLMP.
+ *
+ *    Returns: 0 on success, -1 on error
+ */
+int irlmp_unregister_service(void *handle)
+{
+	irlmp_service_t *service;
+	unsigned long flags;
+
+	if (!handle)
+		return -1;
+
+	/* Caller may call with invalid handle (it's legal) - Jean II */
+	service = hashbin_lock_find(irlmp->services, (long) handle, NULL);
+	if (!service) {
+		pr_debug("%s(), Unknown service!\n", __func__);
+		return -1;
+	}
+
+	hashbin_remove_this(irlmp->services, (irda_queue_t *) service);
+	kfree(service);
+
+	/* Remove old hint bits */
+	irlmp->hints.word = 0;
+
+	/* Refresh current hint bits */
+	spin_lock_irqsave(&irlmp->services->hb_spinlock, flags);
+	service = (irlmp_service_t *) hashbin_get_first(irlmp->services);
+	while (service) {
+		irlmp->hints.word |= service->hints.word;
+
+		service = (irlmp_service_t *)hashbin_get_next(irlmp->services);
+	}
+	spin_unlock_irqrestore(&irlmp->services->hb_spinlock, flags);
+	return 0;
+}
+EXPORT_SYMBOL(irlmp_unregister_service);
+
+/*
+ * Function irlmp_register_client (hint_mask, callback1, callback2)
+ *
+ *    Register a local client with IrLMP
+ *	First callback is selective discovery (based on hints)
+ *	Second callback is for selective discovery expiries
+ *
+ *    Returns: handle > 0 on success, 0 on error
+ */
+void *irlmp_register_client(__u16 hint_mask, DISCOVERY_CALLBACK1 disco_clb,
+			    DISCOVERY_CALLBACK2 expir_clb, void *priv)
+{
+	irlmp_client_t *client;
+
+	IRDA_ASSERT(irlmp != NULL, return NULL;);
+
+	/* Make a new registration */
+	client = kmalloc(sizeof(irlmp_client_t), GFP_ATOMIC);
+	if (!client)
+		return NULL;
+
+	/* Register the details */
+	client->hint_mask.word = hint_mask;
+	client->disco_callback = disco_clb;
+	client->expir_callback = expir_clb;
+	client->priv = priv;
+
+	hashbin_insert(irlmp->clients, (irda_queue_t *) client,
+		       (long) client, NULL);
+
+	return (void *) client;
+}
+EXPORT_SYMBOL(irlmp_register_client);
+
+/*
+ * Function irlmp_update_client (handle, hint_mask, callback1, callback2)
+ *
+ *    Updates specified client (handle) with possibly new hint_mask and
+ *    callback
+ *
+ *    Returns: 0 on success, -1 on error
+ */
+int irlmp_update_client(void *handle, __u16 hint_mask,
+			DISCOVERY_CALLBACK1 disco_clb,
+			DISCOVERY_CALLBACK2 expir_clb, void *priv)
+{
+	irlmp_client_t *client;
+
+	if (!handle)
+		return -1;
+
+	client = hashbin_lock_find(irlmp->clients, (long) handle, NULL);
+	if (!client) {
+		pr_debug("%s(), Unknown client!\n", __func__);
+		return -1;
+	}
+
+	client->hint_mask.word = hint_mask;
+	client->disco_callback = disco_clb;
+	client->expir_callback = expir_clb;
+	client->priv = priv;
+
+	return 0;
+}
+EXPORT_SYMBOL(irlmp_update_client);
+
+/*
+ * Function irlmp_unregister_client (handle)
+ *
+ *    Returns: 0 on success, -1 on error
+ *
+ */
+int irlmp_unregister_client(void *handle)
+{
+	struct irlmp_client *client;
+
+	if (!handle)
+		return -1;
+
+	/* Caller may call with invalid handle (it's legal) - Jean II */
+	client = hashbin_lock_find(irlmp->clients, (long) handle, NULL);
+	if (!client) {
+		pr_debug("%s(), Unknown client!\n", __func__);
+		return -1;
+	}
+
+	pr_debug("%s(), removing client!\n", __func__);
+	hashbin_remove_this(irlmp->clients, (irda_queue_t *) client);
+	kfree(client);
+
+	return 0;
+}
+EXPORT_SYMBOL(irlmp_unregister_client);
+
+/*
+ * Function irlmp_slsap_inuse (slsap)
+ *
+ *    Check if the given source LSAP selector is in use
+ *
+ * This function is clearly not very efficient. On the mitigating side, the
+ * stack make sure that in 99% of the cases, we are called only once
+ * for each socket allocation. We could probably keep a bitmap
+ * of the allocated LSAP, but I'm not sure the complexity is worth it.
+ * Jean II
+ */
+static int irlmp_slsap_inuse(__u8 slsap_sel)
+{
+	struct lsap_cb *self;
+	struct lap_cb *lap;
+	unsigned long flags;
+
+	IRDA_ASSERT(irlmp != NULL, return TRUE;);
+	IRDA_ASSERT(irlmp->magic == LMP_MAGIC, return TRUE;);
+	IRDA_ASSERT(slsap_sel != LSAP_ANY, return TRUE;);
+
+#ifdef CONFIG_IRDA_ULTRA
+	/* Accept all bindings to the connectionless LSAP */
+	if (slsap_sel == LSAP_CONNLESS)
+		return FALSE;
+#endif /* CONFIG_IRDA_ULTRA */
+
+	/* Valid values are between 0 and 127 (0x0-0x6F) */
+	if (slsap_sel > LSAP_MAX)
+		return TRUE;
+
+	/*
+	 *  Check if slsap is already in use. To do this we have to loop over
+	 *  every IrLAP connection and check every LSAP associated with each
+	 *  the connection.
+	 */
+	spin_lock_irqsave_nested(&irlmp->links->hb_spinlock, flags,
+			SINGLE_DEPTH_NESTING);
+	lap = (struct lap_cb *) hashbin_get_first(irlmp->links);
+	while (lap != NULL) {
+		IRDA_ASSERT(lap->magic == LMP_LAP_MAGIC, goto errlap;);
+
+		/* Careful for priority inversions here !
+		 * irlmp->links is never taken while another IrDA
+		 * spinlock is held, so we are safe. Jean II */
+		spin_lock(&lap->lsaps->hb_spinlock);
+
+		/* For this IrLAP, check all the LSAPs */
+		self = (struct lsap_cb *) hashbin_get_first(lap->lsaps);
+		while (self != NULL) {
+			IRDA_ASSERT(self->magic == LMP_LSAP_MAGIC,
+				    goto errlsap;);
+
+			if ((self->slsap_sel == slsap_sel)) {
+				pr_debug("Source LSAP selector=%02x in use\n",
+					 self->slsap_sel);
+				goto errlsap;
+			}
+			self = (struct lsap_cb*) hashbin_get_next(lap->lsaps);
+		}
+		spin_unlock(&lap->lsaps->hb_spinlock);
+
+		/* Next LAP */
+		lap = (struct lap_cb *) hashbin_get_next(irlmp->links);
+	}
+	spin_unlock_irqrestore(&irlmp->links->hb_spinlock, flags);
+
+	/*
+	 * Server sockets are typically waiting for connections and
+	 * therefore reside in the unconnected list. We don't want
+	 * to give out their LSAPs for obvious reasons...
+	 * Jean II
+	 */
+	spin_lock_irqsave(&irlmp->unconnected_lsaps->hb_spinlock, flags);
+
+	self = (struct lsap_cb *) hashbin_get_first(irlmp->unconnected_lsaps);
+	while (self != NULL) {
+		IRDA_ASSERT(self->magic == LMP_LSAP_MAGIC, goto erruncon;);
+		if ((self->slsap_sel == slsap_sel)) {
+			pr_debug("Source LSAP selector=%02x in use (unconnected)\n",
+				 self->slsap_sel);
+			goto erruncon;
+		}
+		self = (struct lsap_cb*) hashbin_get_next(irlmp->unconnected_lsaps);
+	}
+	spin_unlock_irqrestore(&irlmp->unconnected_lsaps->hb_spinlock, flags);
+
+	return FALSE;
+
+	/* Error exit from within one of the two nested loops.
+	 * Make sure we release the right spinlock in the righ order.
+	 * Jean II */
+errlsap:
+	spin_unlock(&lap->lsaps->hb_spinlock);
+IRDA_ASSERT_LABEL(errlap:)
+	spin_unlock_irqrestore(&irlmp->links->hb_spinlock, flags);
+	return TRUE;
+
+	/* Error exit from within the unconnected loop.
+	 * Just one spinlock to release... Jean II */
+erruncon:
+	spin_unlock_irqrestore(&irlmp->unconnected_lsaps->hb_spinlock, flags);
+	return TRUE;
+}
+
+/*
+ * Function irlmp_find_free_slsap ()
+ *
+ *    Find a free source LSAP to use. This function is called if the service
+ *    user has requested a source LSAP equal to LM_ANY
+ */
+static __u8 irlmp_find_free_slsap(void)
+{
+	__u8 lsap_sel;
+	int wrapped = 0;
+
+	IRDA_ASSERT(irlmp != NULL, return -1;);
+	IRDA_ASSERT(irlmp->magic == LMP_MAGIC, return -1;);
+
+	/* Most users don't really care which LSAPs they are given,
+	 * and therefore we automatically give them a free LSAP.
+	 * This function try to find a suitable LSAP, i.e. which is
+	 * not in use and is within the acceptable range. Jean II */
+
+	do {
+		/* Always increment to LSAP number before using it.
+		 * In theory, we could reuse the last LSAP number, as long
+		 * as it is no longer in use. Some IrDA stack do that.
+		 * However, the previous socket may be half closed, i.e.
+		 * we closed it, we think it's no longer in use, but the
+		 * other side did not receive our close and think it's
+		 * active and still send data on it.
+		 * This is similar to what is done with PIDs and TCP ports.
+		 * Also, this reduce the number of calls to irlmp_slsap_inuse()
+		 * which is an expensive function to call.
+		 * Jean II */
+		irlmp->last_lsap_sel++;
+
+		/* Check if we need to wraparound (0x70-0x7f are reserved) */
+		if (irlmp->last_lsap_sel > LSAP_MAX) {
+			/* 0x00-0x10 are also reserved for well know ports */
+			irlmp->last_lsap_sel = 0x10;
+
+			/* Make sure we terminate the loop */
+			if (wrapped++) {
+				net_err_ratelimited("%s: no more free LSAPs !\n",
+						    __func__);
+				return 0;
+			}
+		}
+
+		/* If the LSAP is in use, try the next one.
+		 * Despite the autoincrement, we need to check if the lsap
+		 * is really in use or not, first because LSAP may be
+		 * directly allocated in irlmp_open_lsap(), and also because
+		 * we may wraparound on old sockets. Jean II */
+	} while (irlmp_slsap_inuse(irlmp->last_lsap_sel));
+
+	/* Got it ! */
+	lsap_sel = irlmp->last_lsap_sel;
+	pr_debug("%s(), found free lsap_sel=%02x\n",
+		 __func__, lsap_sel);
+
+	return lsap_sel;
+}
+
+/*
+ * Function irlmp_convert_lap_reason (lap_reason)
+ *
+ *    Converts IrLAP disconnect reason codes to IrLMP disconnect reason
+ *    codes
+ *
+ */
+LM_REASON irlmp_convert_lap_reason( LAP_REASON lap_reason)
+{
+	int reason = LM_LAP_DISCONNECT;
+
+	switch (lap_reason) {
+	case LAP_DISC_INDICATION: /* Received a disconnect request from peer */
+		pr_debug("%s(), LAP_DISC_INDICATION\n", __func__);
+		reason = LM_USER_REQUEST;
+		break;
+	case LAP_NO_RESPONSE:    /* To many retransmits without response */
+		pr_debug("%s(), LAP_NO_RESPONSE\n", __func__);
+		reason = LM_LAP_DISCONNECT;
+		break;
+	case LAP_RESET_INDICATION:
+		pr_debug("%s(), LAP_RESET_INDICATION\n", __func__);
+		reason = LM_LAP_RESET;
+		break;
+	case LAP_FOUND_NONE:
+	case LAP_MEDIA_BUSY:
+	case LAP_PRIMARY_CONFLICT:
+		pr_debug("%s(), LAP_FOUND_NONE, LAP_MEDIA_BUSY or LAP_PRIMARY_CONFLICT\n",
+			 __func__);
+		reason = LM_CONNECT_FAILURE;
+		break;
+	default:
+		pr_debug("%s(), Unknown IrLAP disconnect reason %d!\n",
+			 __func__, lap_reason);
+		reason = LM_LAP_DISCONNECT;
+		break;
+	}
+
+	return reason;
+}
+
+#ifdef CONFIG_PROC_FS
+
+struct irlmp_iter_state {
+	hashbin_t *hashbin;
+};
+
+#define LSAP_START_TOKEN	((void *)1)
+#define LINK_START_TOKEN	((void *)2)
+
+static void *irlmp_seq_hb_idx(struct irlmp_iter_state *iter, loff_t *off)
+{
+	void *element;
+
+	spin_lock_irq(&iter->hashbin->hb_spinlock);
+	for (element = hashbin_get_first(iter->hashbin);
+	     element != NULL;
+	     element = hashbin_get_next(iter->hashbin)) {
+		if (!off || (*off)-- == 0) {
+			/* NB: hashbin left locked */
+			return element;
+		}
+	}
+	spin_unlock_irq(&iter->hashbin->hb_spinlock);
+	iter->hashbin = NULL;
+	return NULL;
+}
+
+
+static void *irlmp_seq_start(struct seq_file *seq, loff_t *pos)
+{
+	struct irlmp_iter_state *iter = seq->private;
+	void *v;
+	loff_t off = *pos;
+
+	iter->hashbin = NULL;
+	if (off-- == 0)
+		return LSAP_START_TOKEN;
+
+	iter->hashbin = irlmp->unconnected_lsaps;
+	v = irlmp_seq_hb_idx(iter, &off);
+	if (v)
+		return v;
+
+	if (off-- == 0)
+		return LINK_START_TOKEN;
+
+	iter->hashbin = irlmp->links;
+	return irlmp_seq_hb_idx(iter, &off);
+}
+
+static void *irlmp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
+{
+	struct irlmp_iter_state *iter = seq->private;
+
+	++*pos;
+
+	if (v == LSAP_START_TOKEN) {		/* start of list of lsaps */
+		iter->hashbin = irlmp->unconnected_lsaps;
+		v = irlmp_seq_hb_idx(iter, NULL);
+		return v ? v : LINK_START_TOKEN;
+	}
+
+	if (v == LINK_START_TOKEN) {		/* start of list of links */
+		iter->hashbin = irlmp->links;
+		return irlmp_seq_hb_idx(iter, NULL);
+	}
+
+	v = hashbin_get_next(iter->hashbin);
+
+	if (v == NULL) {			/* no more in this hash bin */
+		spin_unlock_irq(&iter->hashbin->hb_spinlock);
+
+		if (iter->hashbin == irlmp->unconnected_lsaps)
+			v =  LINK_START_TOKEN;
+
+		iter->hashbin = NULL;
+	}
+	return v;
+}
+
+static void irlmp_seq_stop(struct seq_file *seq, void *v)
+{
+	struct irlmp_iter_state *iter = seq->private;
+
+	if (iter->hashbin)
+		spin_unlock_irq(&iter->hashbin->hb_spinlock);
+}
+
+static int irlmp_seq_show(struct seq_file *seq, void *v)
+{
+	const struct irlmp_iter_state *iter = seq->private;
+	struct lsap_cb *self = v;
+
+	if (v == LSAP_START_TOKEN)
+		seq_puts(seq, "Unconnected LSAPs:\n");
+	else if (v == LINK_START_TOKEN)
+		seq_puts(seq, "\nRegistered Link Layers:\n");
+	else if (iter->hashbin == irlmp->unconnected_lsaps) {
+		self = v;
+		IRDA_ASSERT(self->magic == LMP_LSAP_MAGIC, return -EINVAL; );
+		seq_printf(seq, "lsap state: %s, ",
+			   irlsap_state[ self->lsap_state]);
+		seq_printf(seq,
+			   "slsap_sel: %#02x, dlsap_sel: %#02x, ",
+			   self->slsap_sel, self->dlsap_sel);
+		seq_printf(seq, "(%s)", self->notify.name);
+		seq_printf(seq, "\n");
+	} else if (iter->hashbin == irlmp->links) {
+		struct lap_cb *lap = v;
+
+		seq_printf(seq, "lap state: %s, ",
+			   irlmp_state[lap->lap_state]);
+
+		seq_printf(seq, "saddr: %#08x, daddr: %#08x, ",
+			   lap->saddr, lap->daddr);
+		seq_printf(seq, "num lsaps: %d",
+			   HASHBIN_GET_SIZE(lap->lsaps));
+		seq_printf(seq, "\n");
+
+		/* Careful for priority inversions here !
+		 * All other uses of attrib spinlock are independent of
+		 * the object spinlock, so we are safe. Jean II */
+		spin_lock(&lap->lsaps->hb_spinlock);
+
+		seq_printf(seq, "\n  Connected LSAPs:\n");
+		for (self = (struct lsap_cb *) hashbin_get_first(lap->lsaps);
+		     self != NULL;
+		     self = (struct lsap_cb *)hashbin_get_next(lap->lsaps)) {
+			IRDA_ASSERT(self->magic == LMP_LSAP_MAGIC,
+				    goto outloop;);
+			seq_printf(seq, "  lsap state: %s, ",
+				   irlsap_state[ self->lsap_state]);
+			seq_printf(seq,
+				   "slsap_sel: %#02x, dlsap_sel: %#02x, ",
+				   self->slsap_sel, self->dlsap_sel);
+			seq_printf(seq, "(%s)", self->notify.name);
+			seq_putc(seq, '\n');
+
+		}
+	IRDA_ASSERT_LABEL(outloop:)
+		spin_unlock(&lap->lsaps->hb_spinlock);
+		seq_putc(seq, '\n');
+	} else
+		return -EINVAL;
+
+	return 0;
+}
+
+static const struct seq_operations irlmp_seq_ops = {
+	.start  = irlmp_seq_start,
+	.next   = irlmp_seq_next,
+	.stop   = irlmp_seq_stop,
+	.show   = irlmp_seq_show,
+};
+
+static int irlmp_seq_open(struct inode *inode, struct file *file)
+{
+	IRDA_ASSERT(irlmp != NULL, return -EINVAL;);
+
+	return seq_open_private(file, &irlmp_seq_ops,
+			sizeof(struct irlmp_iter_state));
+}
+
+const struct file_operations irlmp_seq_fops = {
+	.owner		= THIS_MODULE,
+	.open           = irlmp_seq_open,
+	.read           = seq_read,
+	.llseek         = seq_lseek,
+	.release	= seq_release_private,
+};
+
+#endif /* PROC_FS */
diff --git a/drivers/staging/irda/net/irlmp_event.c b/drivers/staging/irda/net/irlmp_event.c
new file mode 100644
index 000000000000..e306cf2c1e04
--- /dev/null
+++ b/drivers/staging/irda/net/irlmp_event.c
@@ -0,0 +1,886 @@
+/*********************************************************************
+ *
+ * Filename:      irlmp_event.c
+ * Version:       0.8
+ * Description:   An IrDA LMP event driver for Linux
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Mon Aug  4 20:40:53 1997
+ * Modified at:   Tue Dec 14 23:04:16 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1998-1999 Dag Brattli <dagb@cs.uit.no>,
+ *     All Rights Reserved.
+ *     Copyright (c) 2000-2003 Jean Tourrilhes <jt@hpl.hp.com>
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#include <linux/kernel.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/timer.h>
+#include <net/irda/irlap.h>
+#include <net/irda/irlmp.h>
+#include <net/irda/irlmp_frame.h>
+#include <net/irda/irlmp_event.h>
+
+const char *const irlmp_state[] = {
+	"LAP_STANDBY",
+	"LAP_U_CONNECT",
+	"LAP_ACTIVE",
+};
+
+const char *const irlsap_state[] = {
+	"LSAP_DISCONNECTED",
+	"LSAP_CONNECT",
+	"LSAP_CONNECT_PEND",
+	"LSAP_DATA_TRANSFER_READY",
+	"LSAP_SETUP",
+	"LSAP_SETUP_PEND",
+};
+
+static const char *const irlmp_event[] __maybe_unused = {
+	"LM_CONNECT_REQUEST",
+	"LM_CONNECT_CONFIRM",
+	"LM_CONNECT_RESPONSE",
+	"LM_CONNECT_INDICATION",
+
+	"LM_DISCONNECT_INDICATION",
+	"LM_DISCONNECT_REQUEST",
+
+	"LM_DATA_REQUEST",
+	"LM_UDATA_REQUEST",
+	"LM_DATA_INDICATION",
+	"LM_UDATA_INDICATION",
+
+	"LM_WATCHDOG_TIMEOUT",
+
+	/* IrLAP events */
+	"LM_LAP_CONNECT_REQUEST",
+	"LM_LAP_CONNECT_INDICATION",
+	"LM_LAP_CONNECT_CONFIRM",
+	"LM_LAP_DISCONNECT_INDICATION",
+	"LM_LAP_DISCONNECT_REQUEST",
+	"LM_LAP_DISCOVERY_REQUEST",
+	"LM_LAP_DISCOVERY_CONFIRM",
+	"LM_LAP_IDLE_TIMEOUT",
+};
+
+/* LAP Connection control proto declarations */
+static void irlmp_state_standby  (struct lap_cb *, IRLMP_EVENT,
+				  struct sk_buff *);
+static void irlmp_state_u_connect(struct lap_cb *, IRLMP_EVENT,
+				  struct sk_buff *);
+static void irlmp_state_active   (struct lap_cb *, IRLMP_EVENT,
+				  struct sk_buff *);
+
+/* LSAP Connection control proto declarations */
+static int irlmp_state_disconnected(struct lsap_cb *, IRLMP_EVENT,
+				    struct sk_buff *);
+static int irlmp_state_connect     (struct lsap_cb *, IRLMP_EVENT,
+				    struct sk_buff *);
+static int irlmp_state_connect_pend(struct lsap_cb *, IRLMP_EVENT,
+				    struct sk_buff *);
+static int irlmp_state_dtr         (struct lsap_cb *, IRLMP_EVENT,
+				    struct sk_buff *);
+static int irlmp_state_setup       (struct lsap_cb *, IRLMP_EVENT,
+				    struct sk_buff *);
+static int irlmp_state_setup_pend  (struct lsap_cb *, IRLMP_EVENT,
+				    struct sk_buff *);
+
+static void (*lap_state[]) (struct lap_cb *, IRLMP_EVENT, struct sk_buff *) =
+{
+	irlmp_state_standby,
+	irlmp_state_u_connect,
+	irlmp_state_active,
+};
+
+static int (*lsap_state[])( struct lsap_cb *, IRLMP_EVENT, struct sk_buff *) =
+{
+	irlmp_state_disconnected,
+	irlmp_state_connect,
+	irlmp_state_connect_pend,
+	irlmp_state_dtr,
+	irlmp_state_setup,
+	irlmp_state_setup_pend
+};
+
+static inline void irlmp_next_lap_state(struct lap_cb *self,
+					IRLMP_STATE state)
+{
+	/*
+	  pr_debug("%s(), LMP LAP = %s\n", __func__, irlmp_state[state]);
+	*/
+	self->lap_state = state;
+}
+
+static inline void irlmp_next_lsap_state(struct lsap_cb *self,
+					 LSAP_STATE state)
+{
+	/*
+	IRDA_ASSERT(self != NULL, return;);
+	pr_debug("%s(), LMP LSAP = %s\n", __func__, irlsap_state[state]);
+	*/
+	self->lsap_state = state;
+}
+
+/* Do connection control events */
+int irlmp_do_lsap_event(struct lsap_cb *self, IRLMP_EVENT event,
+			struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == LMP_LSAP_MAGIC, return -1;);
+
+	pr_debug("%s(), EVENT = %s, STATE = %s\n",
+		 __func__, irlmp_event[event], irlsap_state[self->lsap_state]);
+
+	return (*lsap_state[self->lsap_state]) (self, event, skb);
+}
+
+/*
+ * Function do_lap_event (event, skb, info)
+ *
+ *    Do IrLAP control events
+ *
+ */
+void irlmp_do_lap_event(struct lap_cb *self, IRLMP_EVENT event,
+			struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LMP_LAP_MAGIC, return;);
+
+	pr_debug("%s(), EVENT = %s, STATE = %s\n", __func__,
+		 irlmp_event[event],
+		 irlmp_state[self->lap_state]);
+
+	(*lap_state[self->lap_state]) (self, event, skb);
+}
+
+void irlmp_discovery_timer_expired(void *data)
+{
+	/* We always cleanup the log (active & passive discovery) */
+	irlmp_do_expiry();
+
+	irlmp_do_discovery(sysctl_discovery_slots);
+
+	/* Restart timer */
+	irlmp_start_discovery_timer(irlmp, sysctl_discovery_timeout * HZ);
+}
+
+void irlmp_watchdog_timer_expired(void *data)
+{
+	struct lsap_cb *self = (struct lsap_cb *) data;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LMP_LSAP_MAGIC, return;);
+
+	irlmp_do_lsap_event(self, LM_WATCHDOG_TIMEOUT, NULL);
+}
+
+void irlmp_idle_timer_expired(void *data)
+{
+	struct lap_cb *self = (struct lap_cb *) data;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LMP_LAP_MAGIC, return;);
+
+	irlmp_do_lap_event(self, LM_LAP_IDLE_TIMEOUT, NULL);
+}
+
+/*
+ * Send an event on all LSAPs attached to this LAP.
+ */
+static inline void
+irlmp_do_all_lsap_event(hashbin_t *	lsap_hashbin,
+			IRLMP_EVENT	event)
+{
+	struct lsap_cb *lsap;
+	struct lsap_cb *lsap_next;
+
+	/* Note : this function use the new hashbin_find_next()
+	 * function, instead of the old hashbin_get_next().
+	 * This make sure that we are always pointing one lsap
+	 * ahead, so that if the current lsap is removed as the
+	 * result of sending the event, we don't care.
+	 * Also, as we store the context ourselves, if an enumeration
+	 * of the same lsap hashbin happens as the result of sending the
+	 * event, we don't care.
+	 * The only problem is if the next lsap is removed. In that case,
+	 * hashbin_find_next() will return NULL and we will abort the
+	 * enumeration. - Jean II */
+
+	/* Also : we don't accept any skb in input. We can *NOT* pass
+	 * the same skb to multiple clients safely, we would need to
+	 * skb_clone() it. - Jean II */
+
+	lsap = (struct lsap_cb *) hashbin_get_first(lsap_hashbin);
+
+	while (NULL != hashbin_find_next(lsap_hashbin,
+					 (long) lsap,
+					 NULL,
+					 (void *) &lsap_next) ) {
+		irlmp_do_lsap_event(lsap, event, NULL);
+		lsap = lsap_next;
+	}
+}
+
+/*********************************************************************
+ *
+ *    LAP connection control states
+ *
+ ********************************************************************/
+
+/*
+ * Function irlmp_state_standby (event, skb, info)
+ *
+ *    STANDBY, The IrLAP connection does not exist.
+ *
+ */
+static void irlmp_state_standby(struct lap_cb *self, IRLMP_EVENT event,
+				struct sk_buff *skb)
+{
+	IRDA_ASSERT(self->irlap != NULL, return;);
+
+	switch (event) {
+	case LM_LAP_DISCOVERY_REQUEST:
+		/* irlmp_next_station_state( LMP_DISCOVER); */
+
+		irlap_discovery_request(self->irlap, &irlmp->discovery_cmd);
+		break;
+	case LM_LAP_CONNECT_INDICATION:
+		/*  It's important to switch state first, to avoid IrLMP to
+		 *  think that the link is free since IrLMP may then start
+		 *  discovery before the connection is properly set up. DB.
+		 */
+		irlmp_next_lap_state(self, LAP_ACTIVE);
+
+		/* Just accept connection TODO, this should be fixed */
+		irlap_connect_response(self->irlap, skb);
+		break;
+	case LM_LAP_CONNECT_REQUEST:
+		pr_debug("%s() LS_CONNECT_REQUEST\n", __func__);
+
+		irlmp_next_lap_state(self, LAP_U_CONNECT);
+
+		/* FIXME: need to set users requested QoS */
+		irlap_connect_request(self->irlap, self->daddr, NULL, 0);
+		break;
+	case LM_LAP_DISCONNECT_INDICATION:
+		pr_debug("%s(), Error LM_LAP_DISCONNECT_INDICATION\n",
+			 __func__);
+
+		irlmp_next_lap_state(self, LAP_STANDBY);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %s\n",
+			 __func__, irlmp_event[event]);
+		break;
+	}
+}
+
+/*
+ * Function irlmp_state_u_connect (event, skb, info)
+ *
+ *    U_CONNECT, The layer above has tried to open an LSAP connection but
+ *    since the IrLAP connection does not exist, we must first start an
+ *    IrLAP connection. We are now waiting response from IrLAP.
+ * */
+static void irlmp_state_u_connect(struct lap_cb *self, IRLMP_EVENT event,
+				  struct sk_buff *skb)
+{
+	pr_debug("%s(), event=%s\n", __func__, irlmp_event[event]);
+
+	switch (event) {
+	case LM_LAP_CONNECT_INDICATION:
+		/*  It's important to switch state first, to avoid IrLMP to
+		 *  think that the link is free since IrLMP may then start
+		 *  discovery before the connection is properly set up. DB.
+		 */
+		irlmp_next_lap_state(self, LAP_ACTIVE);
+
+		/* Just accept connection TODO, this should be fixed */
+		irlap_connect_response(self->irlap, skb);
+
+		/* Tell LSAPs that they can start sending data */
+		irlmp_do_all_lsap_event(self->lsaps, LM_LAP_CONNECT_CONFIRM);
+
+		/* Note : by the time we get there (LAP retries and co),
+		 * the lsaps may already have gone. This avoid getting stuck
+		 * forever in LAP_ACTIVE state - Jean II */
+		if (HASHBIN_GET_SIZE(self->lsaps) == 0) {
+			pr_debug("%s() NO LSAPs !\n",  __func__);
+			irlmp_start_idle_timer(self, LM_IDLE_TIMEOUT);
+		}
+		break;
+	case LM_LAP_CONNECT_REQUEST:
+		/* Already trying to connect */
+		break;
+	case LM_LAP_CONNECT_CONFIRM:
+		/* For all lsap_ce E Associated do LS_Connect_confirm */
+		irlmp_next_lap_state(self, LAP_ACTIVE);
+
+		/* Tell LSAPs that they can start sending data */
+		irlmp_do_all_lsap_event(self->lsaps, LM_LAP_CONNECT_CONFIRM);
+
+		/* Note : by the time we get there (LAP retries and co),
+		 * the lsaps may already have gone. This avoid getting stuck
+		 * forever in LAP_ACTIVE state - Jean II */
+		if (HASHBIN_GET_SIZE(self->lsaps) == 0) {
+			pr_debug("%s() NO LSAPs !\n",  __func__);
+			irlmp_start_idle_timer(self, LM_IDLE_TIMEOUT);
+		}
+		break;
+	case LM_LAP_DISCONNECT_INDICATION:
+		pr_debug("%s(), LM_LAP_DISCONNECT_INDICATION\n",  __func__);
+		irlmp_next_lap_state(self, LAP_STANDBY);
+
+		/* Send disconnect event to all LSAPs using this link */
+		irlmp_do_all_lsap_event(self->lsaps,
+					LM_LAP_DISCONNECT_INDICATION);
+		break;
+	case LM_LAP_DISCONNECT_REQUEST:
+		pr_debug("%s(), LM_LAP_DISCONNECT_REQUEST\n",  __func__);
+
+		/* One of the LSAP did timeout or was closed, if it was
+		 * the last one, try to get out of here - Jean II */
+		if (HASHBIN_GET_SIZE(self->lsaps) <= 1) {
+			irlap_disconnect_request(self->irlap);
+		}
+		break;
+	default:
+		pr_debug("%s(), Unknown event %s\n",
+			 __func__, irlmp_event[event]);
+		break;
+	}
+}
+
+/*
+ * Function irlmp_state_active (event, skb, info)
+ *
+ *    ACTIVE, IrLAP connection is active
+ *
+ */
+static void irlmp_state_active(struct lap_cb *self, IRLMP_EVENT event,
+			       struct sk_buff *skb)
+{
+	switch (event) {
+	case LM_LAP_CONNECT_REQUEST:
+		pr_debug("%s(), LS_CONNECT_REQUEST\n", __func__);
+
+		/*
+		 * IrLAP may have a pending disconnect. We tried to close
+		 * IrLAP, but it was postponed because the link was
+		 * busy or we were still sending packets. As we now
+		 * need it, make sure it stays on. Jean II
+		 */
+		irlap_clear_disconnect(self->irlap);
+
+		/*
+		 *  LAP connection already active, just bounce back! Since we
+		 *  don't know which LSAP that tried to do this, we have to
+		 *  notify all LSAPs using this LAP, but that should be safe to
+		 *  do anyway.
+		 */
+		irlmp_do_all_lsap_event(self->lsaps, LM_LAP_CONNECT_CONFIRM);
+
+		/* Needed by connect indication */
+		irlmp_do_all_lsap_event(irlmp->unconnected_lsaps,
+					LM_LAP_CONNECT_CONFIRM);
+		/* Keep state */
+		break;
+	case LM_LAP_DISCONNECT_REQUEST:
+		/*
+		 *  Need to find out if we should close IrLAP or not. If there
+		 *  is only one LSAP connection left on this link, that LSAP
+		 *  must be the one that tries to close IrLAP. It will be
+		 *  removed later and moved to the list of unconnected LSAPs
+		 */
+		if (HASHBIN_GET_SIZE(self->lsaps) > 0) {
+			/* Timer value is checked in irsysctl - Jean II */
+			irlmp_start_idle_timer(self, sysctl_lap_keepalive_time * HZ / 1000);
+		} else {
+			/* No more connections, so close IrLAP */
+
+			/* We don't want to change state just yet, because
+			 * we want to reflect accurately the real state of
+			 * the LAP, not the state we wish it was in,
+			 * so that we don't lose LM_LAP_CONNECT_REQUEST.
+			 * In some cases, IrLAP won't close the LAP
+			 * immediately. For example, it might still be
+			 * retrying packets or waiting for the pf bit.
+			 * As the LAP always send a DISCONNECT_INDICATION
+			 * in PCLOSE or SCLOSE, just change state on that.
+			 * Jean II */
+			irlap_disconnect_request(self->irlap);
+		}
+		break;
+	case LM_LAP_IDLE_TIMEOUT:
+		if (HASHBIN_GET_SIZE(self->lsaps) == 0) {
+			/* Same reasoning as above - keep state */
+			irlap_disconnect_request(self->irlap);
+		}
+		break;
+	case LM_LAP_DISCONNECT_INDICATION:
+		irlmp_next_lap_state(self, LAP_STANDBY);
+
+		/* In some case, at this point our side has already closed
+		 * all lsaps, and we are waiting for the idle_timer to
+		 * expire. If another device reconnect immediately, the
+		 * idle timer will expire in the midle of the connection
+		 * initialisation, screwing up things a lot...
+		 * Therefore, we must stop the timer... */
+		irlmp_stop_idle_timer(self);
+
+		/*
+		 *  Inform all connected LSAP's using this link
+		 */
+		irlmp_do_all_lsap_event(self->lsaps,
+					LM_LAP_DISCONNECT_INDICATION);
+
+		/* Force an expiry of the discovery log.
+		 * Now that the LAP is free, the system may attempt to
+		 * connect to another device. Unfortunately, our entries
+		 * are stale. There is a small window (<3s) before the
+		 * normal discovery will run and where irlmp_connect_request()
+		 * can get the wrong info, so make sure things get
+		 * cleaned *NOW* ;-) - Jean II */
+		irlmp_do_expiry();
+		break;
+	default:
+		pr_debug("%s(), Unknown event %s\n",
+			 __func__, irlmp_event[event]);
+		break;
+	}
+}
+
+/*********************************************************************
+ *
+ *    LSAP connection control states
+ *
+ ********************************************************************/
+
+/*
+ * Function irlmp_state_disconnected (event, skb, info)
+ *
+ *    DISCONNECTED
+ *
+ */
+static int irlmp_state_disconnected(struct lsap_cb *self, IRLMP_EVENT event,
+				    struct sk_buff *skb)
+{
+	int ret = 0;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == LMP_LSAP_MAGIC, return -1;);
+
+	switch (event) {
+#ifdef CONFIG_IRDA_ULTRA
+	case LM_UDATA_INDICATION:
+		/* This is most bizarre. Those packets are  aka unreliable
+		 * connected, aka IrLPT or SOCK_DGRAM/IRDAPROTO_UNITDATA.
+		 * Why do we pass them as Ultra ??? Jean II */
+		irlmp_connless_data_indication(self, skb);
+		break;
+#endif /* CONFIG_IRDA_ULTRA */
+	case LM_CONNECT_REQUEST:
+		pr_debug("%s(), LM_CONNECT_REQUEST\n", __func__);
+
+		if (self->conn_skb) {
+			net_warn_ratelimited("%s: busy with another request!\n",
+					     __func__);
+			return -EBUSY;
+		}
+		/* Don't forget to refcount it (see irlmp_connect_request()) */
+		skb_get(skb);
+		self->conn_skb = skb;
+
+		irlmp_next_lsap_state(self, LSAP_SETUP_PEND);
+
+		/* Start watchdog timer (5 secs for now) */
+		irlmp_start_watchdog_timer(self, 5*HZ);
+
+		irlmp_do_lap_event(self->lap, LM_LAP_CONNECT_REQUEST, NULL);
+		break;
+	case LM_CONNECT_INDICATION:
+		if (self->conn_skb) {
+			net_warn_ratelimited("%s: busy with another request!\n",
+					     __func__);
+			return -EBUSY;
+		}
+		/* Don't forget to refcount it (see irlap_driver_rcv()) */
+		skb_get(skb);
+		self->conn_skb = skb;
+
+		irlmp_next_lsap_state(self, LSAP_CONNECT_PEND);
+
+		/* Start watchdog timer
+		 * This is not mentionned in the spec, but there is a rare
+		 * race condition that can get the socket stuck.
+		 * If we receive this event while our LAP is closing down,
+		 * the LM_LAP_CONNECT_REQUEST get lost and we get stuck in
+		 * CONNECT_PEND state forever.
+		 * The other cause of getting stuck down there is if the
+		 * higher layer never reply to the CONNECT_INDICATION.
+		 * Anyway, it make sense to make sure that we always have
+		 * a backup plan. 1 second is plenty (should be immediate).
+		 * Jean II */
+		irlmp_start_watchdog_timer(self, 1*HZ);
+
+		irlmp_do_lap_event(self->lap, LM_LAP_CONNECT_REQUEST, NULL);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %s on LSAP %#02x\n",
+			 __func__, irlmp_event[event], self->slsap_sel);
+		break;
+	}
+	return ret;
+}
+
+/*
+ * Function irlmp_state_connect (self, event, skb)
+ *
+ *    CONNECT
+ *
+ */
+static int irlmp_state_connect(struct lsap_cb *self, IRLMP_EVENT event,
+				struct sk_buff *skb)
+{
+	struct lsap_cb *lsap;
+	int ret = 0;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == LMP_LSAP_MAGIC, return -1;);
+
+	switch (event) {
+	case LM_CONNECT_RESPONSE:
+		/*
+		 *  Bind this LSAP to the IrLAP link where the connect was
+		 *  received
+		 */
+		lsap = hashbin_remove(irlmp->unconnected_lsaps, (long) self,
+				      NULL);
+
+		IRDA_ASSERT(lsap == self, return -1;);
+		IRDA_ASSERT(self->lap != NULL, return -1;);
+		IRDA_ASSERT(self->lap->lsaps != NULL, return -1;);
+
+		hashbin_insert(self->lap->lsaps, (irda_queue_t *) self,
+			       (long) self, NULL);
+
+		set_bit(0, &self->connected);	/* TRUE */
+
+		irlmp_send_lcf_pdu(self->lap, self->dlsap_sel,
+				   self->slsap_sel, CONNECT_CNF, skb);
+
+		del_timer(&self->watchdog_timer);
+
+		irlmp_next_lsap_state(self, LSAP_DATA_TRANSFER_READY);
+		break;
+	case LM_WATCHDOG_TIMEOUT:
+		/* May happen, who knows...
+		 * Jean II */
+		pr_debug("%s() WATCHDOG_TIMEOUT!\n",  __func__);
+
+		/* Disconnect, get out... - Jean II */
+		self->lap = NULL;
+		self->dlsap_sel = LSAP_ANY;
+		irlmp_next_lsap_state(self, LSAP_DISCONNECTED);
+		break;
+	default:
+		/* LM_LAP_DISCONNECT_INDICATION : Should never happen, we
+		 * are *not* yet bound to the IrLAP link. Jean II */
+		pr_debug("%s(), Unknown event %s on LSAP %#02x\n",
+			 __func__, irlmp_event[event], self->slsap_sel);
+		break;
+	}
+	return ret;
+}
+
+/*
+ * Function irlmp_state_connect_pend (event, skb, info)
+ *
+ *    CONNECT_PEND
+ *
+ */
+static int irlmp_state_connect_pend(struct lsap_cb *self, IRLMP_EVENT event,
+				    struct sk_buff *skb)
+{
+	struct sk_buff *tx_skb;
+	int ret = 0;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == LMP_LSAP_MAGIC, return -1;);
+
+	switch (event) {
+	case LM_CONNECT_REQUEST:
+		/* Keep state */
+		break;
+	case LM_CONNECT_RESPONSE:
+		pr_debug("%s(), LM_CONNECT_RESPONSE, no indication issued yet\n",
+			 __func__);
+		/* Keep state */
+		break;
+	case LM_DISCONNECT_REQUEST:
+		pr_debug("%s(), LM_DISCONNECT_REQUEST, not yet bound to IrLAP connection\n",
+			 __func__);
+		/* Keep state */
+		break;
+	case LM_LAP_CONNECT_CONFIRM:
+		pr_debug("%s(), LS_CONNECT_CONFIRM\n",  __func__);
+		irlmp_next_lsap_state(self, LSAP_CONNECT);
+
+		tx_skb = self->conn_skb;
+		self->conn_skb = NULL;
+
+		irlmp_connect_indication(self, tx_skb);
+		/* Drop reference count - see irlmp_connect_indication(). */
+		dev_kfree_skb(tx_skb);
+		break;
+	case LM_WATCHDOG_TIMEOUT:
+		/* Will happen in some rare cases because of a race condition.
+		 * Just make sure we don't stay there forever...
+		 * Jean II */
+		pr_debug("%s() WATCHDOG_TIMEOUT!\n",  __func__);
+
+		/* Go back to disconnected mode, keep the socket waiting */
+		self->lap = NULL;
+		self->dlsap_sel = LSAP_ANY;
+		if(self->conn_skb)
+			dev_kfree_skb(self->conn_skb);
+		self->conn_skb = NULL;
+		irlmp_next_lsap_state(self, LSAP_DISCONNECTED);
+		break;
+	default:
+		/* LM_LAP_DISCONNECT_INDICATION : Should never happen, we
+		 * are *not* yet bound to the IrLAP link. Jean II */
+		pr_debug("%s(), Unknown event %s on LSAP %#02x\n",
+			 __func__, irlmp_event[event], self->slsap_sel);
+		break;
+	}
+	return ret;
+}
+
+/*
+ * Function irlmp_state_dtr (self, event, skb)
+ *
+ *    DATA_TRANSFER_READY
+ *
+ */
+static int irlmp_state_dtr(struct lsap_cb *self, IRLMP_EVENT event,
+			   struct sk_buff *skb)
+{
+	LM_REASON reason;
+	int ret = 0;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == LMP_LSAP_MAGIC, return -1;);
+	IRDA_ASSERT(self->lap != NULL, return -1;);
+
+	switch (event) {
+	case LM_DATA_REQUEST: /* Optimize for the common case */
+		irlmp_send_data_pdu(self->lap, self->dlsap_sel,
+				    self->slsap_sel, FALSE, skb);
+		break;
+	case LM_DATA_INDICATION: /* Optimize for the common case */
+		irlmp_data_indication(self, skb);
+		break;
+	case LM_UDATA_REQUEST:
+		IRDA_ASSERT(skb != NULL, return -1;);
+		irlmp_send_data_pdu(self->lap, self->dlsap_sel,
+				    self->slsap_sel, TRUE, skb);
+		break;
+	case LM_UDATA_INDICATION:
+		irlmp_udata_indication(self, skb);
+		break;
+	case LM_CONNECT_REQUEST:
+		pr_debug("%s(), LM_CONNECT_REQUEST, error, LSAP already connected\n",
+			 __func__);
+		/* Keep state */
+		break;
+	case LM_CONNECT_RESPONSE:
+		pr_debug("%s(), LM_CONNECT_RESPONSE, error, LSAP already connected\n",
+			 __func__);
+		/* Keep state */
+		break;
+	case LM_DISCONNECT_REQUEST:
+		irlmp_send_lcf_pdu(self->lap, self->dlsap_sel, self->slsap_sel,
+				   DISCONNECT, skb);
+		irlmp_next_lsap_state(self, LSAP_DISCONNECTED);
+		/* Called only from irlmp_disconnect_request(), will
+		 * unbind from LAP over there. Jean II */
+
+		/* Try to close the LAP connection if its still there */
+		if (self->lap) {
+			pr_debug("%s(), trying to close IrLAP\n",
+				 __func__);
+			irlmp_do_lap_event(self->lap,
+					   LM_LAP_DISCONNECT_REQUEST,
+					   NULL);
+		}
+		break;
+	case LM_LAP_DISCONNECT_INDICATION:
+		irlmp_next_lsap_state(self, LSAP_DISCONNECTED);
+
+		reason = irlmp_convert_lap_reason(self->lap->reason);
+
+		irlmp_disconnect_indication(self, reason, NULL);
+		break;
+	case LM_DISCONNECT_INDICATION:
+		irlmp_next_lsap_state(self, LSAP_DISCONNECTED);
+
+		IRDA_ASSERT(self->lap != NULL, return -1;);
+		IRDA_ASSERT(self->lap->magic == LMP_LAP_MAGIC, return -1;);
+
+		IRDA_ASSERT(skb != NULL, return -1;);
+		IRDA_ASSERT(skb->len > 3, return -1;);
+		reason = skb->data[3];
+
+		 /* Try to close the LAP connection */
+		pr_debug("%s(), trying to close IrLAP\n", __func__);
+		irlmp_do_lap_event(self->lap, LM_LAP_DISCONNECT_REQUEST, NULL);
+
+		irlmp_disconnect_indication(self, reason, skb);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %s on LSAP %#02x\n",
+			 __func__, irlmp_event[event], self->slsap_sel);
+		break;
+	}
+	return ret;
+}
+
+/*
+ * Function irlmp_state_setup (event, skb, info)
+ *
+ *    SETUP, Station Control has set up the underlying IrLAP connection.
+ *    An LSAP connection request has been transmitted to the peer
+ *    LSAP-Connection Control FSM and we are awaiting reply.
+ */
+static int irlmp_state_setup(struct lsap_cb *self, IRLMP_EVENT event,
+			     struct sk_buff *skb)
+{
+	LM_REASON reason;
+	int ret = 0;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == LMP_LSAP_MAGIC, return -1;);
+
+	switch (event) {
+	case LM_CONNECT_CONFIRM:
+		irlmp_next_lsap_state(self, LSAP_DATA_TRANSFER_READY);
+
+		del_timer(&self->watchdog_timer);
+
+		irlmp_connect_confirm(self, skb);
+		break;
+	case LM_DISCONNECT_INDICATION:
+		irlmp_next_lsap_state(self, LSAP_DISCONNECTED);
+
+		IRDA_ASSERT(self->lap != NULL, return -1;);
+		IRDA_ASSERT(self->lap->magic == LMP_LAP_MAGIC, return -1;);
+
+		IRDA_ASSERT(skb != NULL, return -1;);
+		IRDA_ASSERT(skb->len > 3, return -1;);
+		reason = skb->data[3];
+
+		 /* Try to close the LAP connection */
+		pr_debug("%s(), trying to close IrLAP\n",  __func__);
+		irlmp_do_lap_event(self->lap, LM_LAP_DISCONNECT_REQUEST, NULL);
+
+		irlmp_disconnect_indication(self, reason, skb);
+		break;
+	case LM_LAP_DISCONNECT_INDICATION:
+		irlmp_next_lsap_state(self, LSAP_DISCONNECTED);
+
+		del_timer(&self->watchdog_timer);
+
+		IRDA_ASSERT(self->lap != NULL, return -1;);
+		IRDA_ASSERT(self->lap->magic == LMP_LAP_MAGIC, return -1;);
+
+		reason = irlmp_convert_lap_reason(self->lap->reason);
+
+		irlmp_disconnect_indication(self, reason, skb);
+		break;
+	case LM_WATCHDOG_TIMEOUT:
+		pr_debug("%s() WATCHDOG_TIMEOUT!\n", __func__);
+
+		IRDA_ASSERT(self->lap != NULL, return -1;);
+		irlmp_do_lap_event(self->lap, LM_LAP_DISCONNECT_REQUEST, NULL);
+		irlmp_next_lsap_state(self, LSAP_DISCONNECTED);
+
+		irlmp_disconnect_indication(self, LM_CONNECT_FAILURE, NULL);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %s on LSAP %#02x\n",
+			 __func__, irlmp_event[event], self->slsap_sel);
+		break;
+	}
+	return ret;
+}
+
+/*
+ * Function irlmp_state_setup_pend (event, skb, info)
+ *
+ *    SETUP_PEND, An LM_CONNECT_REQUEST has been received from the service
+ *    user to set up an LSAP connection. A request has been sent to the
+ *    LAP FSM to set up the underlying IrLAP connection, and we
+ *    are awaiting confirm.
+ */
+static int irlmp_state_setup_pend(struct lsap_cb *self, IRLMP_EVENT event,
+				  struct sk_buff *skb)
+{
+	struct sk_buff *tx_skb;
+	LM_REASON reason;
+	int ret = 0;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(irlmp != NULL, return -1;);
+
+	switch (event) {
+	case LM_LAP_CONNECT_CONFIRM:
+		IRDA_ASSERT(self->conn_skb != NULL, return -1;);
+
+		tx_skb = self->conn_skb;
+		self->conn_skb = NULL;
+
+		irlmp_send_lcf_pdu(self->lap, self->dlsap_sel,
+				   self->slsap_sel, CONNECT_CMD, tx_skb);
+		/* Drop reference count - see irlap_data_request(). */
+		dev_kfree_skb(tx_skb);
+
+		irlmp_next_lsap_state(self, LSAP_SETUP);
+		break;
+	case LM_WATCHDOG_TIMEOUT:
+		pr_debug("%s() : WATCHDOG_TIMEOUT !\n",  __func__);
+
+		IRDA_ASSERT(self->lap != NULL, return -1;);
+		irlmp_do_lap_event(self->lap, LM_LAP_DISCONNECT_REQUEST, NULL);
+		irlmp_next_lsap_state(self, LSAP_DISCONNECTED);
+
+		irlmp_disconnect_indication(self, LM_CONNECT_FAILURE, NULL);
+		break;
+	case LM_LAP_DISCONNECT_INDICATION: /* LS_Disconnect.indication */
+		del_timer( &self->watchdog_timer);
+
+		irlmp_next_lsap_state(self, LSAP_DISCONNECTED);
+
+		reason = irlmp_convert_lap_reason(self->lap->reason);
+
+		irlmp_disconnect_indication(self, reason, NULL);
+		break;
+	default:
+		pr_debug("%s(), Unknown event %s on LSAP %#02x\n",
+			 __func__, irlmp_event[event], self->slsap_sel);
+		break;
+	}
+	return ret;
+}
diff --git a/drivers/staging/irda/net/irlmp_frame.c b/drivers/staging/irda/net/irlmp_frame.c
new file mode 100644
index 000000000000..38b0f994bc7b
--- /dev/null
+++ b/drivers/staging/irda/net/irlmp_frame.c
@@ -0,0 +1,476 @@
+/*********************************************************************
+ *
+ * Filename:      irlmp_frame.c
+ * Version:       0.9
+ * Description:   IrLMP frame implementation
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Tue Aug 19 02:09:59 1997
+ * Modified at:   Mon Dec 13 13:41:12 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1998-1999 Dag Brattli <dagb@cs.uit.no>
+ *     All Rights Reserved.
+ *     Copyright (c) 2000-2003 Jean Tourrilhes <jt@hpl.hp.com>
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#include <linux/skbuff.h>
+#include <linux/kernel.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/irlap.h>
+#include <net/irda/timer.h>
+#include <net/irda/irlmp.h>
+#include <net/irda/irlmp_frame.h>
+#include <net/irda/discovery.h>
+
+static struct lsap_cb *irlmp_find_lsap(struct lap_cb *self, __u8 dlsap,
+				       __u8 slsap, int status, hashbin_t *);
+
+inline void irlmp_send_data_pdu(struct lap_cb *self, __u8 dlsap, __u8 slsap,
+				int expedited, struct sk_buff *skb)
+{
+	skb->data[0] = dlsap;
+	skb->data[1] = slsap;
+
+	if (expedited) {
+		pr_debug("%s(), sending expedited data\n", __func__);
+		irlap_data_request(self->irlap, skb, TRUE);
+	} else
+		irlap_data_request(self->irlap, skb, FALSE);
+}
+
+/*
+ * Function irlmp_send_lcf_pdu (dlsap, slsap, opcode,skb)
+ *
+ *    Send Link Control Frame to IrLAP
+ */
+void irlmp_send_lcf_pdu(struct lap_cb *self, __u8 dlsap, __u8 slsap,
+			__u8 opcode, struct sk_buff *skb)
+{
+	__u8 *frame;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LMP_LAP_MAGIC, return;);
+	IRDA_ASSERT(skb != NULL, return;);
+
+	frame = skb->data;
+
+	frame[0] = dlsap | CONTROL_BIT;
+	frame[1] = slsap;
+
+	frame[2] = opcode;
+
+	if (opcode == DISCONNECT)
+		frame[3] = 0x01; /* Service user request */
+	else
+		frame[3] = 0x00; /* rsvd */
+
+	irlap_data_request(self->irlap, skb, FALSE);
+}
+
+/*
+ * Function irlmp_input (skb)
+ *
+ *    Used by IrLAP to pass received data frames to IrLMP layer
+ *
+ */
+void irlmp_link_data_indication(struct lap_cb *self, struct sk_buff *skb,
+				int unreliable)
+{
+	struct lsap_cb *lsap;
+	__u8   slsap_sel;   /* Source (this) LSAP address */
+	__u8   dlsap_sel;   /* Destination LSAP address */
+	__u8   *fp;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LMP_LAP_MAGIC, return;);
+	IRDA_ASSERT(skb->len > 2, return;);
+
+	fp = skb->data;
+
+	/*
+	 *  The next statements may be confusing, but we do this so that
+	 *  destination LSAP of received frame is source LSAP in our view
+	 */
+	slsap_sel = fp[0] & LSAP_MASK;
+	dlsap_sel = fp[1];
+
+	/*
+	 *  Check if this is an incoming connection, since we must deal with
+	 *  it in a different way than other established connections.
+	 */
+	if ((fp[0] & CONTROL_BIT) && (fp[2] == CONNECT_CMD)) {
+		pr_debug("%s(), incoming connection, source LSAP=%d, dest LSAP=%d\n",
+			 __func__, slsap_sel, dlsap_sel);
+
+		/* Try to find LSAP among the unconnected LSAPs */
+		lsap = irlmp_find_lsap(self, dlsap_sel, slsap_sel, CONNECT_CMD,
+				       irlmp->unconnected_lsaps);
+
+		/* Maybe LSAP was already connected, so try one more time */
+		if (!lsap) {
+			pr_debug("%s(), incoming connection for LSAP already connected\n",
+				 __func__);
+			lsap = irlmp_find_lsap(self, dlsap_sel, slsap_sel, 0,
+					       self->lsaps);
+		}
+	} else
+		lsap = irlmp_find_lsap(self, dlsap_sel, slsap_sel, 0,
+				       self->lsaps);
+
+	if (lsap == NULL) {
+		pr_debug("IrLMP, Sorry, no LSAP for received frame!\n");
+		pr_debug("%s(), slsap_sel = %02x, dlsap_sel = %02x\n",
+			 __func__, slsap_sel, dlsap_sel);
+		if (fp[0] & CONTROL_BIT) {
+			pr_debug("%s(), received control frame %02x\n",
+				 __func__, fp[2]);
+		} else {
+			pr_debug("%s(), received data frame\n", __func__);
+		}
+		return;
+	}
+
+	/*
+	 *  Check if we received a control frame?
+	 */
+	if (fp[0] & CONTROL_BIT) {
+		switch (fp[2]) {
+		case CONNECT_CMD:
+			lsap->lap = self;
+			irlmp_do_lsap_event(lsap, LM_CONNECT_INDICATION, skb);
+			break;
+		case CONNECT_CNF:
+			irlmp_do_lsap_event(lsap, LM_CONNECT_CONFIRM, skb);
+			break;
+		case DISCONNECT:
+			pr_debug("%s(), Disconnect indication!\n",
+				 __func__);
+			irlmp_do_lsap_event(lsap, LM_DISCONNECT_INDICATION,
+					    skb);
+			break;
+		case ACCESSMODE_CMD:
+			pr_debug("Access mode cmd not implemented!\n");
+			break;
+		case ACCESSMODE_CNF:
+			pr_debug("Access mode cnf not implemented!\n");
+			break;
+		default:
+			pr_debug("%s(), Unknown control frame %02x\n",
+				 __func__, fp[2]);
+			break;
+		}
+	} else if (unreliable) {
+		/* Optimize and bypass the state machine if possible */
+		if (lsap->lsap_state == LSAP_DATA_TRANSFER_READY)
+			irlmp_udata_indication(lsap, skb);
+		else
+			irlmp_do_lsap_event(lsap, LM_UDATA_INDICATION, skb);
+	} else {
+		/* Optimize and bypass the state machine if possible */
+		if (lsap->lsap_state == LSAP_DATA_TRANSFER_READY)
+			irlmp_data_indication(lsap, skb);
+		else
+			irlmp_do_lsap_event(lsap, LM_DATA_INDICATION, skb);
+	}
+}
+
+/*
+ * Function irlmp_link_unitdata_indication (self, skb)
+ *
+ *
+ *
+ */
+#ifdef CONFIG_IRDA_ULTRA
+void irlmp_link_unitdata_indication(struct lap_cb *self, struct sk_buff *skb)
+{
+	struct lsap_cb *lsap;
+	__u8   slsap_sel;   /* Source (this) LSAP address */
+	__u8   dlsap_sel;   /* Destination LSAP address */
+	__u8   pid;         /* Protocol identifier */
+	__u8   *fp;
+	unsigned long flags;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LMP_LAP_MAGIC, return;);
+	IRDA_ASSERT(skb->len > 2, return;);
+
+	fp = skb->data;
+
+	/*
+	 *  The next statements may be confusing, but we do this so that
+	 *  destination LSAP of received frame is source LSAP in our view
+	 */
+	slsap_sel = fp[0] & LSAP_MASK;
+	dlsap_sel = fp[1];
+	pid       = fp[2];
+
+	if (pid & 0x80) {
+		pr_debug("%s(), extension in PID not supp!\n",
+			 __func__);
+		return;
+	}
+
+	/* Check if frame is addressed to the connectionless LSAP */
+	if ((slsap_sel != LSAP_CONNLESS) || (dlsap_sel != LSAP_CONNLESS)) {
+		pr_debug("%s(), dropping frame!\n", __func__);
+		return;
+	}
+
+	/* Search the connectionless LSAP */
+	spin_lock_irqsave(&irlmp->unconnected_lsaps->hb_spinlock, flags);
+	lsap = (struct lsap_cb *) hashbin_get_first(irlmp->unconnected_lsaps);
+	while (lsap != NULL) {
+		/*
+		 *  Check if source LSAP and dest LSAP selectors and PID match.
+		 */
+		if ((lsap->slsap_sel == slsap_sel) &&
+		    (lsap->dlsap_sel == dlsap_sel) &&
+		    (lsap->pid == pid))
+		{
+			break;
+		}
+		lsap = (struct lsap_cb *) hashbin_get_next(irlmp->unconnected_lsaps);
+	}
+	spin_unlock_irqrestore(&irlmp->unconnected_lsaps->hb_spinlock, flags);
+
+	if (lsap)
+		irlmp_connless_data_indication(lsap, skb);
+	else {
+		pr_debug("%s(), found no matching LSAP!\n", __func__);
+	}
+}
+#endif /* CONFIG_IRDA_ULTRA */
+
+/*
+ * Function irlmp_link_disconnect_indication (reason, userdata)
+ *
+ *    IrLAP has disconnected
+ *
+ */
+void irlmp_link_disconnect_indication(struct lap_cb *lap,
+				      struct irlap_cb *irlap,
+				      LAP_REASON reason,
+				      struct sk_buff *skb)
+{
+	IRDA_ASSERT(lap != NULL, return;);
+	IRDA_ASSERT(lap->magic == LMP_LAP_MAGIC, return;);
+
+	lap->reason = reason;
+	lap->daddr = DEV_ADDR_ANY;
+
+	/* FIXME: must do something with the skb if any */
+
+	/*
+	 *  Inform station state machine
+	 */
+	irlmp_do_lap_event(lap, LM_LAP_DISCONNECT_INDICATION, NULL);
+}
+
+/*
+ * Function irlmp_link_connect_indication (qos)
+ *
+ *    Incoming LAP connection!
+ *
+ */
+void irlmp_link_connect_indication(struct lap_cb *self, __u32 saddr,
+				   __u32 daddr, struct qos_info *qos,
+				   struct sk_buff *skb)
+{
+	/* Copy QoS settings for this session */
+	self->qos = qos;
+
+	/* Update destination device address */
+	self->daddr = daddr;
+	IRDA_ASSERT(self->saddr == saddr, return;);
+
+	irlmp_do_lap_event(self, LM_LAP_CONNECT_INDICATION, skb);
+}
+
+/*
+ * Function irlmp_link_connect_confirm (qos)
+ *
+ *    LAP connection confirmed!
+ *
+ */
+void irlmp_link_connect_confirm(struct lap_cb *self, struct qos_info *qos,
+				struct sk_buff *skb)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LMP_LAP_MAGIC, return;);
+	IRDA_ASSERT(qos != NULL, return;);
+
+	/* Don't need use the skb for now */
+
+	/* Copy QoS settings for this session */
+	self->qos = qos;
+
+	irlmp_do_lap_event(self, LM_LAP_CONNECT_CONFIRM, NULL);
+}
+
+/*
+ * Function irlmp_link_discovery_indication (self, log)
+ *
+ *    Device is discovering us
+ *
+ * It's not an answer to our own discoveries, just another device trying
+ * to perform discovery, but we don't want to miss the opportunity
+ * to exploit this information, because :
+ *	o We may not actively perform discovery (just passive discovery)
+ *	o This type of discovery is much more reliable. In some cases, it
+ *	  seem that less than 50% of our discoveries get an answer, while
+ *	  we always get ~100% of these.
+ *	o Make faster discovery, statistically divide time of discovery
+ *	  events by 2 (important for the latency aspect and user feel)
+ *	o Even is we do active discovery, the other node might not
+ *	  answer our discoveries (ex: Palm). The Palm will just perform
+ *	  one active discovery and connect directly to us.
+ *
+ * However, when both devices discover each other, they might attempt to
+ * connect to each other following the discovery event, and it would create
+ * collisions on the medium (SNRM battle).
+ * The "fix" for that is to disable all connection requests in IrLAP
+ * for 100ms after a discovery indication by setting the media_busy flag.
+ * Previously, we used to postpone the event which was quite ugly. Now
+ * that IrLAP takes care of this problem, just pass the event up...
+ *
+ * Jean II
+ */
+void irlmp_link_discovery_indication(struct lap_cb *self,
+				     discovery_t *discovery)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LMP_LAP_MAGIC, return;);
+
+	/* Add to main log, cleanup */
+	irlmp_add_discovery(irlmp->cachelog, discovery);
+
+	/* Just handle it the same way as a discovery confirm,
+	 * bypass the LM_LAP state machine (see below) */
+	irlmp_discovery_confirm(irlmp->cachelog, DISCOVERY_PASSIVE);
+}
+
+/*
+ * Function irlmp_link_discovery_confirm (self, log)
+ *
+ *    Called by IrLAP with a list of discoveries after the discovery
+ *    request has been carried out. A NULL log is received if IrLAP
+ *    was unable to carry out the discovery request
+ *
+ */
+void irlmp_link_discovery_confirm(struct lap_cb *self, hashbin_t *log)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LMP_LAP_MAGIC, return;);
+
+	/* Add to main log, cleanup */
+	irlmp_add_discovery_log(irlmp->cachelog, log);
+
+	/* Propagate event to various LSAPs registered for it.
+	 * We bypass the LM_LAP state machine because
+	 *	1) We do it regardless of the LM_LAP state
+	 *	2) It doesn't affect the LM_LAP state
+	 *	3) Faster, slimer, simpler, ...
+	 * Jean II */
+	irlmp_discovery_confirm(irlmp->cachelog, DISCOVERY_ACTIVE);
+}
+
+#ifdef CONFIG_IRDA_CACHE_LAST_LSAP
+static inline void irlmp_update_cache(struct lap_cb *lap,
+				      struct lsap_cb *lsap)
+{
+	/* Prevent concurrent read to get garbage */
+	lap->cache.valid = FALSE;
+	/* Update cache entry */
+	lap->cache.dlsap_sel = lsap->dlsap_sel;
+	lap->cache.slsap_sel = lsap->slsap_sel;
+	lap->cache.lsap = lsap;
+	lap->cache.valid = TRUE;
+}
+#endif
+
+/*
+ * Function irlmp_find_handle (self, dlsap_sel, slsap_sel, status, queue)
+ *
+ *    Find handle associated with destination and source LSAP
+ *
+ * Any IrDA connection (LSAP/TSAP) is uniquely identified by
+ * 3 parameters, the local lsap, the remote lsap and the remote address.
+ * We may initiate multiple connections to the same remote service
+ * (they will have different local lsap), a remote device may initiate
+ * multiple connections to the same local service (they will have
+ * different remote lsap), or multiple devices may connect to the same
+ * service and may use the same remote lsap (and they will have
+ * different remote address).
+ * So, where is the remote address ? Each LAP connection is made with
+ * a single remote device, so imply a specific remote address.
+ * Jean II
+ */
+static struct lsap_cb *irlmp_find_lsap(struct lap_cb *self, __u8 dlsap_sel,
+				       __u8 slsap_sel, int status,
+				       hashbin_t *queue)
+{
+	struct lsap_cb *lsap;
+	unsigned long flags;
+
+	/*
+	 *  Optimize for the common case. We assume that the last frame
+	 *  received is in the same connection as the last one, so check in
+	 *  cache first to avoid the linear search
+	 */
+#ifdef CONFIG_IRDA_CACHE_LAST_LSAP
+	if ((self->cache.valid) &&
+	    (self->cache.slsap_sel == slsap_sel) &&
+	    (self->cache.dlsap_sel == dlsap_sel))
+	{
+		return self->cache.lsap;
+	}
+#endif
+
+	spin_lock_irqsave(&queue->hb_spinlock, flags);
+
+	lsap = (struct lsap_cb *) hashbin_get_first(queue);
+	while (lsap != NULL) {
+		/*
+		 *  If this is an incoming connection, then the destination
+		 *  LSAP selector may have been specified as LM_ANY so that
+		 *  any client can connect. In that case we only need to check
+		 *  if the source LSAP (in our view!) match!
+		 */
+		if ((status == CONNECT_CMD) &&
+		    (lsap->slsap_sel == slsap_sel) &&
+		    (lsap->dlsap_sel == LSAP_ANY)) {
+			/* This is where the dest lsap sel is set on incoming
+			 * lsaps */
+			lsap->dlsap_sel = dlsap_sel;
+			break;
+		}
+		/*
+		 *  Check if source LSAP and dest LSAP selectors match.
+		 */
+		if ((lsap->slsap_sel == slsap_sel) &&
+		    (lsap->dlsap_sel == dlsap_sel))
+			break;
+
+		lsap = (struct lsap_cb *) hashbin_get_next(queue);
+	}
+#ifdef CONFIG_IRDA_CACHE_LAST_LSAP
+	if(lsap)
+		irlmp_update_cache(self, lsap);
+#endif
+	spin_unlock_irqrestore(&queue->hb_spinlock, flags);
+
+	/* Return what we've found or NULL */
+	return lsap;
+}
diff --git a/drivers/staging/irda/net/irmod.c b/drivers/staging/irda/net/irmod.c
new file mode 100644
index 000000000000..4319f4ff66b0
--- /dev/null
+++ b/drivers/staging/irda/net/irmod.c
@@ -0,0 +1,199 @@
+/*********************************************************************
+ *
+ * Filename:      irmod.c
+ * Version:       0.9
+ * Description:   IrDA stack main entry points
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Mon Dec 15 13:55:39 1997
+ * Modified at:   Wed Jan  5 15:12:41 2000
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1997, 1999-2000 Dag Brattli, All Rights Reserved.
+ *     Copyright (c) 2000-2004 Jean Tourrilhes <jt@hpl.hp.com>
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+/*
+ * This file contains the main entry points of the IrDA stack.
+ * They are in this file and not af_irda.c because some developpers
+ * are using the IrDA stack without the socket API (compiling out
+ * af_irda.c).
+ * Jean II
+ */
+
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/irmod.h>		/* notify_t */
+#include <net/irda/irlap.h>		/* irlap_init */
+#include <net/irda/irlmp.h>		/* irlmp_init */
+#include <net/irda/iriap.h>		/* iriap_init */
+#include <net/irda/irttp.h>		/* irttp_init */
+#include <net/irda/irda_device.h>	/* irda_device_init */
+
+/* Packet type handler.
+ * Tell the kernel how IrDA packets should be handled.
+ */
+static struct packet_type irda_packet_type __read_mostly = {
+	.type	= cpu_to_be16(ETH_P_IRDA),
+	.func	= irlap_driver_rcv,	/* Packet type handler irlap_frame.c */
+};
+
+/*
+ * Function irda_notify_init (notify)
+ *
+ *    Used for initializing the notify structure
+ *
+ */
+void irda_notify_init(notify_t *notify)
+{
+	notify->data_indication = NULL;
+	notify->udata_indication = NULL;
+	notify->connect_confirm = NULL;
+	notify->connect_indication = NULL;
+	notify->disconnect_indication = NULL;
+	notify->flow_indication = NULL;
+	notify->status_indication = NULL;
+	notify->instance = NULL;
+	strlcpy(notify->name, "Unknown", sizeof(notify->name));
+}
+EXPORT_SYMBOL(irda_notify_init);
+
+/*
+ * Function irda_init (void)
+ *
+ *  Protocol stack initialisation entry point.
+ *  Initialise the various components of the IrDA stack
+ */
+static int __init irda_init(void)
+{
+	int ret = 0;
+
+	/* Lower layer of the stack */
+	irlmp_init();
+	irlap_init();
+
+	/* Driver/dongle support */
+	irda_device_init();
+
+	/* Higher layers of the stack */
+	iriap_init();
+	irttp_init();
+	ret = irsock_init();
+	if (ret < 0)
+		goto out_err_1;
+
+	/* Add IrDA packet type (Start receiving packets) */
+	dev_add_pack(&irda_packet_type);
+
+	/* External APIs */
+#ifdef CONFIG_PROC_FS
+	irda_proc_register();
+#endif
+#ifdef CONFIG_SYSCTL
+	ret = irda_sysctl_register();
+	if (ret < 0)
+		goto out_err_2;
+#endif
+
+	ret = irda_nl_register();
+	if (ret < 0)
+		goto out_err_3;
+
+	return 0;
+
+ out_err_3:
+#ifdef CONFIG_SYSCTL
+	irda_sysctl_unregister();
+ out_err_2:
+#endif
+#ifdef CONFIG_PROC_FS
+	irda_proc_unregister();
+#endif
+
+	/* Remove IrDA packet type (stop receiving packets) */
+	dev_remove_pack(&irda_packet_type);
+
+	/* Remove higher layers */
+	irsock_cleanup();
+ out_err_1:
+	irttp_cleanup();
+	iriap_cleanup();
+
+	/* Remove lower layers */
+	irda_device_cleanup();
+	irlap_cleanup(); /* Must be done before irlmp_cleanup()! DB */
+
+	/* Remove middle layer */
+	irlmp_cleanup();
+
+
+	return ret;
+}
+
+/*
+ * Function irda_cleanup (void)
+ *
+ *  Protocol stack cleanup/removal entry point.
+ *  Cleanup the various components of the IrDA stack
+ */
+static void __exit irda_cleanup(void)
+{
+	/* Remove External APIs */
+	irda_nl_unregister();
+
+#ifdef CONFIG_SYSCTL
+	irda_sysctl_unregister();
+#endif
+#ifdef CONFIG_PROC_FS
+	irda_proc_unregister();
+#endif
+
+	/* Remove IrDA packet type (stop receiving packets) */
+	dev_remove_pack(&irda_packet_type);
+
+	/* Remove higher layers */
+	irsock_cleanup();
+	irttp_cleanup();
+	iriap_cleanup();
+
+	/* Remove lower layers */
+	irda_device_cleanup();
+	irlap_cleanup(); /* Must be done before irlmp_cleanup()! DB */
+
+	/* Remove middle layer */
+	irlmp_cleanup();
+}
+
+/*
+ * The IrDA stack must be initialised *before* drivers get initialised,
+ * and *before* higher protocols (IrLAN/IrCOMM/IrNET) get initialised,
+ * otherwise bad things will happen (hashbins will be NULL for example).
+ * Those modules are at module_init()/device_initcall() level.
+ *
+ * On the other hand, it needs to be initialised *after* the basic
+ * networking, the /proc/net filesystem and sysctl module. Those are
+ * currently initialised in .../init/main.c (before initcalls).
+ * Also, IrDA drivers needs to be initialised *after* the random number
+ * generator (main stack and higher layer init don't need it anymore).
+ *
+ * Jean II
+ */
+device_initcall(irda_init);
+module_exit(irda_cleanup);
+
+MODULE_AUTHOR("Dag Brattli <dagb@cs.uit.no> & Jean Tourrilhes <jt@hpl.hp.com>");
+MODULE_DESCRIPTION("The Linux IrDA Protocol Stack");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS_NETPROTO(PF_IRDA);
diff --git a/drivers/staging/irda/net/irnet/Kconfig b/drivers/staging/irda/net/irnet/Kconfig
new file mode 100644
index 000000000000..28c557f0fdd2
--- /dev/null
+++ b/drivers/staging/irda/net/irnet/Kconfig
@@ -0,0 +1,13 @@
+config IRNET
+	tristate "IrNET protocol"
+	depends on IRDA && PPP
+	help
+	  Say Y here if you want to build support for the IrNET protocol.
+	  To compile it as a module, choose M here: the module will be
+	  called irnet.  IrNET is a PPP driver, so you will also need a
+	  working PPP subsystem (driver, daemon and config)...
+
+	  IrNET is an alternate way to transfer TCP/IP traffic over IrDA.  It
+	  uses synchronous PPP over a set of point to point IrDA sockets.  You
+	  can use it between Linux machine or with W2k.
+
diff --git a/drivers/staging/irda/net/irnet/Makefile b/drivers/staging/irda/net/irnet/Makefile
new file mode 100644
index 000000000000..61c365c8a2a0
--- /dev/null
+++ b/drivers/staging/irda/net/irnet/Makefile
@@ -0,0 +1,7 @@
+#
+# Makefile for the Linux IrDA IrNET protocol layer.
+#
+
+obj-$(CONFIG_IRNET) += irnet.o
+
+irnet-y := irnet_ppp.o irnet_irda.o
diff --git a/drivers/staging/irda/net/irnet/irnet.h b/drivers/staging/irda/net/irnet/irnet.h
new file mode 100644
index 000000000000..9d451f8ed47a
--- /dev/null
+++ b/drivers/staging/irda/net/irnet/irnet.h
@@ -0,0 +1,522 @@
+/*
+ *	IrNET protocol module : Synchronous PPP over an IrDA socket.
+ *
+ *		Jean II - HPL `00 - <jt@hpl.hp.com>
+ *
+ * This file contains definitions and declarations global to the IrNET module,
+ * all grouped in one place...
+ * This file is a *private* header, so other modules don't want to know
+ * what's in there...
+ *
+ * Note : as most part of the Linux kernel, this module is available
+ * under the GNU General Public License (GPL).
+ */
+
+#ifndef IRNET_H
+#define IRNET_H
+
+/************************** DOCUMENTATION ***************************/
+/*
+ * What is IrNET
+ * -------------
+ * IrNET is a protocol allowing to carry TCP/IP traffic between two
+ * IrDA peers in an efficient fashion. It is a thin layer, passing PPP
+ * packets to IrTTP and vice versa. It uses PPP in synchronous mode,
+ * because IrTTP offer a reliable sequenced packet service (as opposed
+ * to a byte stream). In fact, you could see IrNET as carrying TCP/IP
+ * in a IrDA socket, using PPP to provide the glue.
+ *
+ * The main difference with traditional PPP over IrCOMM is that we
+ * avoid the framing and serial emulation which are a performance
+ * bottleneck. It also allows multipoint communications in a sensible
+ * fashion.
+ *
+ * The main difference with IrLAN is that we use PPP for the link
+ * management, which is more standard, interoperable and flexible than
+ * the IrLAN protocol. For example, PPP adds authentication,
+ * encryption, compression, header compression and automated routing
+ * setup. And, as IrNET let PPP do the hard work, the implementation
+ * is much simpler than IrLAN.
+ *
+ * The Linux implementation
+ * ------------------------
+ * IrNET is written on top of the Linux-IrDA stack, and interface with
+ * the generic Linux PPP driver. Because IrNET depend on recent
+ * changes of the PPP driver interface, IrNET will work only with very
+ * recent kernel (2.3.99-pre6 and up).
+ *
+ * The present implementation offer the following features :
+ *	o simple user interface using pppd
+ *	o efficient implementation (interface directly to PPP and IrTTP)
+ *	o addressing (you can specify the name of the IrNET recipient)
+ *	o multipoint operation (limited by IrLAP specification)
+ *	o information in /proc/net/irda/irnet
+ *	o IrNET events on /dev/irnet (for user space daemon)
+ *	o IrNET daemon (irnetd) to automatically handle incoming requests
+ *	o Windows 2000 compatibility (tested, but need more work)
+ * Currently missing :
+ *	o Lot's of testing (that's your job)
+ *	o Connection retries (may be too hard to do)
+ *	o Check pppd persist mode
+ *	o User space daemon (to automatically handle incoming requests)
+ *
+ * The setup is not currently the most easy, but this should get much
+ * better when everything will get integrated...
+ *
+ * Acknowledgements
+ * ----------------
+ * This module is based on :
+ *	o The PPP driver (ppp_synctty/ppp_generic) by Paul Mackerras
+ *	o The IrLAN protocol (irlan_common/XXX) by Dag Brattli
+ *	o The IrSock interface (af_irda) by Dag Brattli
+ *	o Some other bits from the kernel and my drivers...
+ * Infinite thanks to those brave souls for providing the infrastructure
+ * upon which IrNET is built.
+ *
+ * Thanks to all my colleagues in HP for helping me. In particular,
+ * thanks to Salil Pradhan and Bill Serra for W2k testing...
+ * Thanks to Luiz Magalhaes for irnetd and much testing...
+ *
+ * Thanks to Alan Cox for answering lot's of my stupid questions, and
+ * to Paul Mackerras answering my questions on how to best integrate
+ * IrNET and pppd.
+ *
+ * Jean II
+ *
+ * Note on some implementations choices...
+ * ------------------------------------
+ *	1) Direct interface vs tty/socket
+ * I could have used a tty interface to hook to ppp and use the full
+ * socket API to connect to IrDA. The code would have been easier to
+ * maintain, and maybe the code would have been smaller...
+ * Instead, we hook directly to ppp_generic and to IrTTP, which make
+ * things more complicated...
+ *
+ * The first reason is flexibility : this allow us to create IrNET
+ * instances on demand (no /dev/ircommX crap) and to allow linkname
+ * specification on pppd command line...
+ *
+ * Second reason is speed optimisation. If you look closely at the
+ * transmit and receive paths, you will notice that they are "super lean"
+ * (that's why they look ugly), with no function calls and as little data
+ * copy and modification as I could...
+ *
+ *	2) irnetd in user space
+ * irnetd is implemented in user space, which is necessary to call pppd.
+ * This also give maximum benefits in term of flexibility and customability,
+ * and allow to offer the event channel, useful for other stuff like debug.
+ *
+ * On the other hand, this require a loose coordination between the
+ * present module and irnetd. One critical area is how incoming request
+ * are handled.
+ * When irnet receive an incoming request, it send an event to irnetd and
+ * drop the incoming IrNET socket.
+ * irnetd start a pppd instance, which create a new IrNET socket. This new
+ * socket is then connected in the originating node to the pppd instance.
+ * At this point, in the originating node, the first socket is closed.
+ *
+ * I admit, this is a bit messy and waste some resources. The alternative
+ * is caching incoming socket, and that's also quite messy and waste
+ * resources.
+ * We also make connection time slower. For example, on a 115 kb/s link it
+ * adds 60ms to the connection time (770 ms). However, this is slower than
+ * the time it takes to fire up pppd on my P133...
+ *
+ *
+ * History :
+ * -------
+ *
+ * v1 - 15.5.00 - Jean II
+ *	o Basic IrNET (hook to ppp_generic & IrTTP - incl. multipoint)
+ *	o control channel on /dev/irnet (set name/address)
+ *	o event channel on /dev/irnet (for user space daemon)
+ *
+ * v2 - 5.6.00 - Jean II
+ *	o Enable DROP_NOT_READY to avoid PPP timeouts & other weirdness...
+ *	o Add DISCONNECT_TO event and rename DISCONNECT_FROM.
+ *	o Set official device number alloaction on /dev/irnet
+ *
+ * v3 - 30.8.00 - Jean II
+ *	o Update to latest Linux-IrDA changes :
+ *		- queue_t => irda_queue_t
+ *	o Update to ppp-2.4.0 :
+ *		- move irda_irnet_connect from PPPIOCATTACH to TIOCSETD
+ *	o Add EXPIRE event (depend on new IrDA-Linux patch)
+ *	o Switch from `hashbin_remove' to `hashbin_remove_this' to fix
+ *	  a multilink bug... (depend on new IrDA-Linux patch)
+ *	o fix a self->daddr to self->raddr in irda_irnet_connect to fix
+ *	  another multilink bug (darn !)
+ *	o Remove LINKNAME_IOCTL cruft
+ *
+ * v3b - 31.8.00 - Jean II
+ *	o Dump discovery log at event channel startup
+ *
+ * v4 - 28.9.00 - Jean II
+ *	o Fix interaction between poll/select and dump discovery log
+ *	o Add IRNET_BLOCKED_LINK event (depend on new IrDA-Linux patch)
+ *	o Add IRNET_NOANSWER_FROM event (mostly to help support)
+ *	o Release flow control in disconnect_indication
+ *	o Block packets while connecting (speed up connections)
+ *
+ * v5 - 11.01.01 - Jean II
+ *	o Init self->max_header_size, just in case...
+ *	o Set up ap->chan.hdrlen, to get zero copy on tx side working.
+ *	o avoid tx->ttp->flow->ppp->tx->... loop, by checking flow state
+ *		Thanks to Christian Gennerat for finding this bug !
+ *	---
+ *	o Declare the proper MTU/MRU that we can support
+ *		(but PPP doesn't read the MTU value :-()
+ *	o Declare hashbin HB_NOLOCK instead of HB_LOCAL to avoid
+ *		disabling and enabling irq twice
+ *
+ * v6 - 31.05.01 - Jean II
+ *	o Print source address in Found, Discovery, Expiry & Request events
+ *	o Print requested source address in /proc/net/irnet
+ *	o Change control channel input. Allow multiple commands in one line.
+ *	o Add saddr command to change ap->rsaddr (and use that in IrDA)
+ *	---
+ *	o Make the IrDA connection procedure totally asynchronous.
+ *	  Heavy rewrite of the IAS query code and the whole connection
+ *	  procedure. Now, irnet_connect() no longer need to be called from
+ *	  a process context...
+ *	o Enable IrDA connect retries in ppp_irnet_send(). The good thing
+ *	  is that IrDA connect retries are directly driven by PPP LCP
+ *	  retries (we retry for each LCP packet), so that everything
+ *	  is transparently controlled from pppd lcp-max-configure.
+ *	o Add ttp_connect flag to prevent rentry on the connect procedure
+ *	o Test and fixups to eliminate side effects of retries
+ *
+ * v7 - 22.08.01 - Jean II
+ *	o Cleanup : Change "saddr = 0x0" to "saddr = DEV_ADDR_ANY"
+ *	o Fix bug in BLOCK_WHEN_CONNECT introduced in v6 : due to the
+ *	  asynchronous IAS query, self->tsap is NULL when PPP send the
+ *	  first packet.  This was preventing "connect-delay 0" to work.
+ *	  Change the test in ppp_irnet_send() to self->ttp_connect.
+ *
+ * v8 - 1.11.01 - Jean II
+ *	o Tighten the use of self->ttp_connect and self->ttp_open to
+ *	  prevent various race conditions.
+ *	o Avoid leaking discovery log and skb
+ *	o Replace "self" with "server" in irnet_connect_indication() to
+ *	  better detect cut'n'paste error ;-)
+ *
+ * v9 - 29.11.01 - Jean II
+ *	o Fix event generation in disconnect indication that I broke in v8
+ *	  It was always generation "No-Answer" because I was testing ttp_open
+ *	  just after clearing it. *blush*.
+ *	o Use newly created irttp_listen() to fix potential crash when LAP
+ *	  destroyed before irnet module removed.
+ *
+ * v10 - 4.3.2 - Jean II
+ *	o When receiving a disconnect indication, don't reenable the
+ *	  PPP Tx queue, this will trigger a reconnect. Instead, close
+ *	  the channel, which will kill pppd...
+ *
+ * v11 - 20.3.02 - Jean II
+ *	o Oops ! v10 fix disabled IrNET retries and passive behaviour.
+ *	  Better fix in irnet_disconnect_indication() :
+ *	  - if connected, kill pppd via hangup.
+ *	  - if not connected, reenable ppp Tx, which trigger IrNET retry.
+ *
+ * v12 - 10.4.02 - Jean II
+ *	o Fix race condition in irnet_connect_indication().
+ *	  If the socket was already trying to connect, drop old connection
+ *	  and use new one only if acting as primary. See comments.
+ *
+ * v13 - 30.5.02 - Jean II
+ *	o Update module init code
+ *
+ * v14 - 20.2.03 - Jean II
+ *	o Add discovery hint bits in the control channel.
+ *	o Remove obsolete MOD_INC/DEC_USE_COUNT in favor of .owner
+ *
+ * v15 - 7.4.03 - Jean II
+ *	o Replace spin_lock_irqsave() with spin_lock_bh() so that we can
+ *	  use ppp_unit_number(). It's probably also better overall...
+ *	o Disable call to ppp_unregister_channel(), because we can't do it.
+ */
+
+/***************************** INCLUDES *****************************/
+
+#include <linux/module.h>
+
+#include <linux/kernel.h>
+#include <linux/skbuff.h>
+#include <linux/tty.h>
+#include <linux/proc_fs.h>
+#include <linux/netdevice.h>
+#include <linux/poll.h>
+#include <linux/capability.h>
+#include <linux/ctype.h>	/* isspace() */
+#include <linux/string.h>	/* skip_spaces() */
+#include <linux/uaccess.h>
+#include <linux/init.h>
+
+#include <linux/ppp_defs.h>
+#include <linux/ppp-ioctl.h>
+#include <linux/ppp_channel.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/iriap.h>
+#include <net/irda/irias_object.h>
+#include <net/irda/irlmp.h>
+#include <net/irda/irttp.h>
+#include <net/irda/discovery.h>
+
+/***************************** OPTIONS *****************************/
+/*
+ * Define or undefine to compile or not some optional part of the
+ * IrNET driver...
+ * Note : the present defaults make sense, play with that at your
+ * own risk...
+ */
+/* IrDA side of the business... */
+#define DISCOVERY_NOMASK	/* To enable W2k compatibility... */
+#define ADVERTISE_HINT		/* Advertise IrLAN hint bit */
+#define ALLOW_SIMULT_CONNECT	/* This seem to work, cross fingers... */
+#define DISCOVERY_EVENTS	/* Query the discovery log to post events */
+#define INITIAL_DISCOVERY	/* Dump current discovery log as events */
+#undef STREAM_COMPAT		/* Not needed - potentially messy */
+#undef CONNECT_INDIC_KICK	/* Might mess IrDA, not needed */
+#undef FAIL_SEND_DISCONNECT	/* Might mess IrDA, not needed */
+#undef PASS_CONNECT_PACKETS	/* Not needed ? Safe */
+#undef MISSING_PPP_API		/* Stuff I wish I could do */
+
+/* PPP side of the business */
+#define BLOCK_WHEN_CONNECT	/* Block packets when connecting */
+#define CONNECT_IN_SEND		/* Retry IrDA connection procedure */
+#undef FLUSH_TO_PPP		/* Not sure about this one, let's play safe */
+#undef SECURE_DEVIRNET		/* Bah... */
+
+/****************************** DEBUG ******************************/
+
+/*
+ * This set of flags enable and disable all the various warning,
+ * error and debug message of this driver.
+ * Each section can be enabled and disabled independently
+ */
+/* In the PPP part */
+#define DEBUG_CTRL_TRACE	0	/* Control channel */
+#define DEBUG_CTRL_INFO		0	/* various info */
+#define DEBUG_CTRL_ERROR	1	/* problems */
+#define DEBUG_FS_TRACE		0	/* filesystem callbacks */
+#define DEBUG_FS_INFO		0	/* various info */
+#define DEBUG_FS_ERROR		1	/* problems */
+#define DEBUG_PPP_TRACE		0	/* PPP related functions */
+#define DEBUG_PPP_INFO		0	/* various info */
+#define DEBUG_PPP_ERROR		1	/* problems */
+#define DEBUG_MODULE_TRACE	0	/* module insertion/removal */
+#define DEBUG_MODULE_ERROR	1	/* problems */
+
+/* In the IrDA part */
+#define DEBUG_IRDA_SR_TRACE	0	/* IRDA subroutines */
+#define DEBUG_IRDA_SR_INFO	0	/* various info */
+#define DEBUG_IRDA_SR_ERROR	1	/* problems */
+#define DEBUG_IRDA_SOCK_TRACE	0	/* IRDA main socket functions */
+#define DEBUG_IRDA_SOCK_INFO	0	/* various info */
+#define DEBUG_IRDA_SOCK_ERROR	1	/* problems */
+#define DEBUG_IRDA_SERV_TRACE	0	/* The IrNET server */
+#define DEBUG_IRDA_SERV_INFO	0	/* various info */
+#define DEBUG_IRDA_SERV_ERROR	1	/* problems */
+#define DEBUG_IRDA_TCB_TRACE	0	/* IRDA IrTTP callbacks */
+#define DEBUG_IRDA_CB_INFO	0	/* various info */
+#define DEBUG_IRDA_CB_ERROR	1	/* problems */
+#define DEBUG_IRDA_OCB_TRACE	0	/* IRDA other callbacks */
+#define DEBUG_IRDA_OCB_INFO	0	/* various info */
+#define DEBUG_IRDA_OCB_ERROR	1	/* problems */
+
+#define DEBUG_ASSERT		0	/* Verify all assertions */
+
+/*
+ * These are the macros we are using to actually print the debug
+ * statements. Don't look at it, it's ugly...
+ *
+ * One of the trick is that, as the DEBUG_XXX are constant, the
+ * compiler will optimise away the if() in all cases.
+ */
+/* All error messages (will show up in the normal logs) */
+#define DERROR(dbg, format, args...) \
+	{if(DEBUG_##dbg) \
+		printk(KERN_INFO "irnet: %s(): " format, __func__ , ##args);}
+
+/* Normal debug message (will show up in /var/log/debug) */
+#define DEBUG(dbg, format, args...) \
+	{if(DEBUG_##dbg) \
+		printk(KERN_DEBUG "irnet: %s(): " format, __func__ , ##args);}
+
+/* Entering a function (trace) */
+#define DENTER(dbg, format, args...) \
+	{if(DEBUG_##dbg) \
+		printk(KERN_DEBUG "irnet: -> %s" format, __func__ , ##args);}
+
+/* Entering and exiting a function in one go (trace) */
+#define DPASS(dbg, format, args...) \
+	{if(DEBUG_##dbg) \
+		printk(KERN_DEBUG "irnet: <>%s" format, __func__ , ##args);}
+
+/* Exiting a function (trace) */
+#define DEXIT(dbg, format, args...) \
+	{if(DEBUG_##dbg) \
+		printk(KERN_DEBUG "irnet: <-%s()" format, __func__ , ##args);}
+
+/* Exit a function with debug */
+#define DRETURN(ret, dbg, args...) \
+	{DEXIT(dbg, ": " args);\
+	return ret; }
+
+/* Exit a function on failed condition */
+#define DABORT(cond, ret, dbg, args...) \
+	{if(cond) {\
+		DERROR(dbg, args);\
+		return ret; }}
+
+/* Invalid assertion, print out an error and exit... */
+#define DASSERT(cond, ret, dbg, args...) \
+	{if((DEBUG_ASSERT) && !(cond)) {\
+		DERROR(dbg, "Invalid assertion: " args);\
+		return ret; }}
+
+/************************ CONSTANTS & MACROS ************************/
+
+/* Paranoia */
+#define IRNET_MAGIC	0xB00754
+
+/* Number of control events in the control channel buffer... */
+#define IRNET_MAX_EVENTS	8	/* Should be more than enough... */
+
+/****************************** TYPES ******************************/
+
+/*
+ * This is the main structure where we store all the data pertaining to
+ * one instance of irnet.
+ * Note : in irnet functions, a pointer this structure is usually called
+ * "ap" or "self". If the code is borrowed from the IrDA stack, it tend
+ * to be called "self", and if it is borrowed from the PPP driver it is
+ * "ap". Apart from that, it's exactly the same structure ;-)
+ */
+typedef struct irnet_socket
+{
+  /* ------------------- Instance management ------------------- */
+  /* We manage a linked list of IrNET socket instances */
+  irda_queue_t		q;		/* Must be first - for hasbin */
+  int			magic;		/* Paranoia */
+
+  /* --------------------- FileSystem part --------------------- */
+  /* "pppd" interact directly with us on a /dev/ file */
+  struct file *		file;		/* File descriptor of this instance */
+  /* TTY stuff - to keep "pppd" happy */
+  struct ktermios	termios;	/* Various tty flags */
+  /* Stuff for the control channel */
+  int			event_index;	/* Last read in the event log */
+
+  /* ------------------------- PPP part ------------------------- */
+  /* We interface directly to the ppp_generic driver in the kernel */
+  int			ppp_open;	/* registered with ppp_generic */
+  struct ppp_channel	chan;		/* Interface to generic ppp layer */
+
+  int			mru;		/* Max size of PPP payload */
+  u32			xaccm[8];	/* Asynchronous character map (just */
+  u32			raccm;		/* to please pppd - dummy) */
+  unsigned int		flags;		/* PPP flags (compression, ...) */
+  unsigned int		rbits;		/* Unused receive flags ??? */
+  struct work_struct disconnect_work;   /* Process context disconnection */
+  /* ------------------------ IrTTP part ------------------------ */
+  /* We create a pseudo "socket" over the IrDA tranport */
+  unsigned long		ttp_open;	/* Set when IrTTP is ready */
+  unsigned long		ttp_connect;	/* Set when IrTTP is connecting */
+  struct tsap_cb *	tsap;		/* IrTTP instance (the connection) */
+
+  char			rname[NICKNAME_MAX_LEN + 1];
+					/* IrDA nickname of destination */
+  __u32			rdaddr;		/* Requested peer IrDA address */
+  __u32			rsaddr;		/* Requested local IrDA address */
+  __u32			daddr;		/* actual peer IrDA address */
+  __u32			saddr;		/* my local IrDA address */
+  __u8			dtsap_sel;	/* Remote TSAP selector */
+  __u8			stsap_sel;	/* Local TSAP selector */
+
+  __u32			max_sdu_size_rx;/* Socket parameters used for IrTTP */
+  __u32			max_sdu_size_tx;
+  __u32			max_data_size;
+  __u8			max_header_size;
+  LOCAL_FLOW		tx_flow;	/* State of the Tx path in IrTTP */
+
+  /* ------------------- IrLMP and IrIAS part ------------------- */
+  /* Used for IrDA Discovery and socket name resolution */
+  void *		ckey;		/* IrLMP client handle */
+  __u16			mask;		/* Hint bits mask (filter discov.)*/
+  int			nslots;		/* Number of slots for discovery */
+
+  struct iriap_cb *	iriap;		/* Used to query remote IAS */
+  int			errno;		/* status of the IAS query */
+
+  /* -------------------- Discovery log part -------------------- */
+  /* Used by initial discovery on the control channel
+   * and by irnet_discover_daddr_and_lsap_sel() */
+  struct irda_device_info *discoveries;	/* Copy of the discovery log */
+  int			disco_index;	/* Last read in the discovery log */
+  int			disco_number;	/* Size of the discovery log */
+
+  struct mutex		lock;
+
+} irnet_socket;
+
+/*
+ * This is the various event that we will generate on the control channel
+ */
+typedef enum irnet_event
+{
+  IRNET_DISCOVER,		/* New IrNET node discovered */
+  IRNET_EXPIRE,			/* IrNET node expired */
+  IRNET_CONNECT_TO,		/* IrNET socket has connected to other node */
+  IRNET_CONNECT_FROM,		/* Other node has connected to IrNET socket */
+  IRNET_REQUEST_FROM,		/* Non satisfied connection request */
+  IRNET_NOANSWER_FROM,		/* Failed connection request */
+  IRNET_BLOCKED_LINK,		/* Link (IrLAP) is blocked for > 3s */
+  IRNET_DISCONNECT_FROM,	/* IrNET socket has disconnected */
+  IRNET_DISCONNECT_TO		/* Closing IrNET socket */
+} irnet_event;
+
+/*
+ * This is the storage for an event and its arguments
+ */
+typedef struct irnet_log
+{
+  irnet_event	event;
+  int		unit;
+  __u32		saddr;
+  __u32		daddr;
+  char		name[NICKNAME_MAX_LEN + 1];	/* 21 + 1 */
+  __u16_host_order hints;			/* Discovery hint bits */
+} irnet_log;
+
+/*
+ * This is the storage for all events and related stuff...
+ */
+typedef struct irnet_ctrl_channel
+{
+  irnet_log	log[IRNET_MAX_EVENTS];	/* Event log */
+  int		index;		/* Current index in log */
+  spinlock_t	spinlock;	/* Serialize access to the event log */
+  wait_queue_head_t	rwait;	/* processes blocked on read (or poll) */
+} irnet_ctrl_channel;
+
+/**************************** PROTOTYPES ****************************/
+/*
+ * Global functions of the IrNET module
+ * Note : we list here also functions called from one file to the other.
+ */
+
+/* -------------------------- IRDA PART -------------------------- */
+int irda_irnet_create(irnet_socket *);	/* Initialise an IrNET socket */
+int irda_irnet_connect(irnet_socket *);	/* Try to connect over IrDA */
+void irda_irnet_destroy(irnet_socket *);	/* Teardown an IrNET socket */
+int irda_irnet_init(void);		/* Initialise IrDA part of IrNET */
+void irda_irnet_cleanup(void);		/* Teardown IrDA part of IrNET */
+
+/**************************** VARIABLES ****************************/
+
+/* Control channel stuff - allocated in irnet_irda.h */
+extern struct irnet_ctrl_channel	irnet_events;
+
+#endif /* IRNET_H */
diff --git a/drivers/staging/irda/net/irnet/irnet_irda.c b/drivers/staging/irda/net/irnet/irnet_irda.c
new file mode 100644
index 000000000000..e390bceeb2f8
--- /dev/null
+++ b/drivers/staging/irda/net/irnet/irnet_irda.c
@@ -0,0 +1,1885 @@
+/*
+ *	IrNET protocol module : Synchronous PPP over an IrDA socket.
+ *
+ *		Jean II - HPL `00 - <jt@hpl.hp.com>
+ *
+ * This file implement the IRDA interface of IrNET.
+ * Basically, we sit on top of IrTTP. We set up IrTTP, IrIAS properly,
+ * and exchange frames with IrTTP.
+ */
+
+#include "irnet_irda.h"		/* Private header */
+#include <linux/sched.h>
+#include <linux/seq_file.h>
+#include <linux/slab.h>
+#include <asm/unaligned.h>
+
+/*
+ * PPP disconnect work: we need to make sure we're in
+ * process context when calling ppp_unregister_channel().
+ */
+static void irnet_ppp_disconnect(struct work_struct *work)
+{
+	irnet_socket * self =
+		container_of(work, irnet_socket, disconnect_work);
+
+	if (self == NULL)
+		return;
+	/*
+	 * If we were connected, cleanup & close the PPP
+	 * channel, which will kill pppd (hangup) and the rest.
+	 */
+	if (self->ppp_open && !self->ttp_open && !self->ttp_connect) {
+		ppp_unregister_channel(&self->chan);
+		self->ppp_open = 0;
+	}
+}
+
+/************************* CONTROL CHANNEL *************************/
+/*
+ * When ppp is not active, /dev/irnet act as a control channel.
+ * Writing allow to set up the IrDA destination of the IrNET channel,
+ * and any application may be read events happening on IrNET...
+ */
+
+/*------------------------------------------------------------------*/
+/*
+ * Post an event to the control channel...
+ * Put the event in the log, and then wait all process blocked on read
+ * so they can read the log...
+ */
+static void
+irnet_post_event(irnet_socket *	ap,
+		 irnet_event	event,
+		 __u32		saddr,
+		 __u32		daddr,
+		 char *		name,
+		 __u16		hints)
+{
+  int			index;		/* In the log */
+
+  DENTER(CTRL_TRACE, "(ap=0x%p, event=%d, daddr=%08x, name=``%s'')\n",
+	 ap, event, daddr, name);
+
+  /* Protect this section via spinlock.
+   * Note : as we are the only event producer, we only need to exclude
+   * ourself when touching the log, which is nice and easy.
+   */
+  spin_lock_bh(&irnet_events.spinlock);
+
+  /* Copy the event in the log */
+  index = irnet_events.index;
+  irnet_events.log[index].event = event;
+  irnet_events.log[index].daddr = daddr;
+  irnet_events.log[index].saddr = saddr;
+  /* Try to copy IrDA nickname */
+  if(name)
+    strcpy(irnet_events.log[index].name, name);
+  else
+    irnet_events.log[index].name[0] = '\0';
+  /* Copy hints */
+  irnet_events.log[index].hints.word = hints;
+  /* Try to get ppp unit number */
+  if((ap != (irnet_socket *) NULL) && (ap->ppp_open))
+    irnet_events.log[index].unit = ppp_unit_number(&ap->chan);
+  else
+    irnet_events.log[index].unit = -1;
+
+  /* Increment the index
+   * Note that we increment the index only after the event is written,
+   * to make sure that the readers don't get garbage... */
+  irnet_events.index = (index + 1) % IRNET_MAX_EVENTS;
+
+  DEBUG(CTRL_INFO, "New event index is %d\n", irnet_events.index);
+
+  /* Spin lock end */
+  spin_unlock_bh(&irnet_events.spinlock);
+
+  /* Now : wake up everybody waiting for events... */
+  wake_up_interruptible_all(&irnet_events.rwait);
+
+  DEXIT(CTRL_TRACE, "\n");
+}
+
+/************************* IRDA SUBROUTINES *************************/
+/*
+ * These are a bunch of subroutines called from other functions
+ * down there, mostly common code or to improve readability...
+ *
+ * Note : we duplicate quite heavily some routines of af_irda.c,
+ * because our input structure (self) is quite different
+ * (struct irnet instead of struct irda_sock), which make sharing
+ * the same code impossible (at least, without templates).
+ */
+
+/*------------------------------------------------------------------*/
+/*
+ * Function irda_open_tsap (self)
+ *
+ *    Open local Transport Service Access Point (TSAP)
+ *
+ * Create a IrTTP instance for us and set all the IrTTP callbacks.
+ */
+static inline int
+irnet_open_tsap(irnet_socket *	self)
+{
+  notify_t	notify;		/* Callback structure */
+
+  DENTER(IRDA_SR_TRACE, "(self=0x%p)\n", self);
+
+  DABORT(self->tsap != NULL, -EBUSY, IRDA_SR_ERROR, "Already busy !\n");
+
+  /* Initialize IrTTP callbacks to be used by the IrDA stack */
+  irda_notify_init(&notify);
+  notify.connect_confirm	= irnet_connect_confirm;
+  notify.connect_indication	= irnet_connect_indication;
+  notify.disconnect_indication	= irnet_disconnect_indication;
+  notify.data_indication	= irnet_data_indication;
+  /*notify.udata_indication	= NULL;*/
+  notify.flow_indication	= irnet_flow_indication;
+  notify.status_indication	= irnet_status_indication;
+  notify.instance		= self;
+  strlcpy(notify.name, IRNET_NOTIFY_NAME, sizeof(notify.name));
+
+  /* Open an IrTTP instance */
+  self->tsap = irttp_open_tsap(LSAP_ANY, DEFAULT_INITIAL_CREDIT,
+			       &notify);
+  DABORT(self->tsap == NULL, -ENOMEM,
+	 IRDA_SR_ERROR, "Unable to allocate TSAP !\n");
+
+  /* Remember which TSAP selector we actually got */
+  self->stsap_sel = self->tsap->stsap_sel;
+
+  DEXIT(IRDA_SR_TRACE, " - tsap=0x%p, sel=0x%X\n",
+	self->tsap, self->stsap_sel);
+  return 0;
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Function irnet_ias_to_tsap (self, result, value)
+ *
+ *    Examine an IAS object and extract TSAP
+ *
+ * We do an IAP query to find the TSAP associated with the IrNET service.
+ * When IrIAP pass us the result of the query, this function look at
+ * the return values to check for failures and extract the TSAP if
+ * possible.
+ * Also deallocate value
+ * The failure is in self->errno
+ * Return TSAP or -1
+ */
+static inline __u8
+irnet_ias_to_tsap(irnet_socket *	self,
+		  int			result,
+		  struct ias_value *	value)
+{
+  __u8	dtsap_sel = 0;		/* TSAP we are looking for */
+
+  DENTER(IRDA_SR_TRACE, "(self=0x%p)\n", self);
+
+  /* By default, no error */
+  self->errno = 0;
+
+  /* Check if request succeeded */
+  switch(result)
+    {
+      /* Standard errors : service not available */
+    case IAS_CLASS_UNKNOWN:
+    case IAS_ATTRIB_UNKNOWN:
+      DEBUG(IRDA_SR_INFO, "IAS object doesn't exist ! (%d)\n", result);
+      self->errno = -EADDRNOTAVAIL;
+      break;
+
+      /* Other errors, most likely IrDA stack failure */
+    default :
+      DEBUG(IRDA_SR_INFO, "IAS query failed ! (%d)\n", result);
+      self->errno = -EHOSTUNREACH;
+      break;
+
+      /* Success : we got what we wanted */
+    case IAS_SUCCESS:
+      break;
+    }
+
+  /* Check what was returned to us */
+  if(value != NULL)
+    {
+      /* What type of argument have we got ? */
+      switch(value->type)
+	{
+	case IAS_INTEGER:
+	  DEBUG(IRDA_SR_INFO, "result=%d\n", value->t.integer);
+	  if(value->t.integer != -1)
+	    /* Get the remote TSAP selector */
+	    dtsap_sel = value->t.integer;
+	  else
+	    self->errno = -EADDRNOTAVAIL;
+	  break;
+	default:
+	  self->errno = -EADDRNOTAVAIL;
+	  DERROR(IRDA_SR_ERROR, "bad type ! (0x%X)\n", value->type);
+	  break;
+	}
+
+      /* Cleanup */
+      irias_delete_value(value);
+    }
+  else	/* value == NULL */
+    {
+      /* Nothing returned to us - usually result != SUCCESS */
+      if(!(self->errno))
+	{
+	  DERROR(IRDA_SR_ERROR,
+		 "IrDA bug : result == SUCCESS && value == NULL\n");
+	  self->errno = -EHOSTUNREACH;
+	}
+    }
+  DEXIT(IRDA_SR_TRACE, "\n");
+
+  /* Return the TSAP */
+  return dtsap_sel;
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Function irnet_find_lsap_sel (self)
+ *
+ *    Try to lookup LSAP selector in remote LM-IAS
+ *
+ * Basically, we start a IAP query, and then go to sleep. When the query
+ * return, irnet_getvalue_confirm will wake us up, and we can examine the
+ * result of the query...
+ * Note that in some case, the query fail even before we go to sleep,
+ * creating some races...
+ */
+static inline int
+irnet_find_lsap_sel(irnet_socket *	self)
+{
+  DENTER(IRDA_SR_TRACE, "(self=0x%p)\n", self);
+
+  /* This should not happen */
+  DABORT(self->iriap, -EBUSY, IRDA_SR_ERROR, "busy with a previous query.\n");
+
+  /* Create an IAP instance, will be closed in irnet_getvalue_confirm() */
+  self->iriap = iriap_open(LSAP_ANY, IAS_CLIENT, self,
+			   irnet_getvalue_confirm);
+
+  /* Treat unexpected signals as disconnect */
+  self->errno = -EHOSTUNREACH;
+
+  /* Query remote LM-IAS */
+  iriap_getvaluebyclass_request(self->iriap, self->rsaddr, self->daddr,
+				IRNET_SERVICE_NAME, IRNET_IAS_VALUE);
+
+  /* The above request is non-blocking.
+   * After a while, IrDA will call us back in irnet_getvalue_confirm()
+   * We will then call irnet_ias_to_tsap() and finish the
+   * connection procedure */
+
+  DEXIT(IRDA_SR_TRACE, "\n");
+  return 0;
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Function irnet_connect_tsap (self)
+ *
+ *    Initialise the TTP socket and initiate TTP connection
+ *
+ */
+static inline int
+irnet_connect_tsap(irnet_socket *	self)
+{
+  int		err;
+
+  DENTER(IRDA_SR_TRACE, "(self=0x%p)\n", self);
+
+  /* Open a local TSAP (an IrTTP instance) */
+  err = irnet_open_tsap(self);
+  if(err != 0)
+    {
+      clear_bit(0, &self->ttp_connect);
+      DERROR(IRDA_SR_ERROR, "connect aborted!\n");
+      return err;
+    }
+
+  /* Connect to remote device */
+  err = irttp_connect_request(self->tsap, self->dtsap_sel,
+			      self->rsaddr, self->daddr, NULL,
+			      self->max_sdu_size_rx, NULL);
+  if(err != 0)
+    {
+      clear_bit(0, &self->ttp_connect);
+      DERROR(IRDA_SR_ERROR, "connect aborted!\n");
+      return err;
+    }
+
+  /* The above call is non-blocking.
+   * After a while, the IrDA stack will either call us back in
+   * irnet_connect_confirm() or irnet_disconnect_indication()
+   * See you there ;-) */
+
+  DEXIT(IRDA_SR_TRACE, "\n");
+  return err;
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Function irnet_discover_next_daddr (self)
+ *
+ *    Query the IrNET TSAP of the next device in the log.
+ *
+ * Used in the TSAP discovery procedure.
+ */
+static inline int
+irnet_discover_next_daddr(irnet_socket *	self)
+{
+  /* Close the last instance of IrIAP, and open a new one.
+   * We can't reuse the IrIAP instance in the IrIAP callback */
+  if(self->iriap)
+    {
+      iriap_close(self->iriap);
+      self->iriap = NULL;
+    }
+  /* Create a new IAP instance */
+  self->iriap = iriap_open(LSAP_ANY, IAS_CLIENT, self,
+			   irnet_discovervalue_confirm);
+  if(self->iriap == NULL)
+    return -ENOMEM;
+
+  /* Next discovery - before the call to avoid races */
+  self->disco_index++;
+
+  /* Check if we have one more address to try */
+  if(self->disco_index < self->disco_number)
+    {
+      /* Query remote LM-IAS */
+      iriap_getvaluebyclass_request(self->iriap,
+				    self->discoveries[self->disco_index].saddr,
+				    self->discoveries[self->disco_index].daddr,
+				    IRNET_SERVICE_NAME, IRNET_IAS_VALUE);
+      /* The above request is non-blocking.
+       * After a while, IrDA will call us back in irnet_discovervalue_confirm()
+       * We will then call irnet_ias_to_tsap() and come back here again... */
+      return 0;
+    }
+  else
+    return 1;
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Function irnet_discover_daddr_and_lsap_sel (self)
+ *
+ *    This try to find a device with the requested service.
+ *
+ * Initiate a TSAP discovery procedure.
+ * It basically look into the discovery log. For each address in the list,
+ * it queries the LM-IAS of the device to find if this device offer
+ * the requested service.
+ * If there is more than one node supporting the service, we complain
+ * to the user (it should move devices around).
+ * If we find one node which have the requested TSAP, we connect to it.
+ *
+ * This function just start the whole procedure. It request the discovery
+ * log and submit the first IAS query.
+ * The bulk of the job is handled in irnet_discovervalue_confirm()
+ *
+ * Note : this procedure fails if there is more than one device in range
+ * on the same dongle, because IrLMP doesn't disconnect the LAP when the
+ * last LSAP is closed. Moreover, we would need to wait the LAP
+ * disconnection...
+ */
+static inline int
+irnet_discover_daddr_and_lsap_sel(irnet_socket *	self)
+{
+  int	ret;
+
+  DENTER(IRDA_SR_TRACE, "(self=0x%p)\n", self);
+
+  /* Ask lmp for the current discovery log */
+  self->discoveries = irlmp_get_discoveries(&self->disco_number, self->mask,
+					    DISCOVERY_DEFAULT_SLOTS);
+
+  /* Check if the we got some results */
+  if(self->discoveries == NULL)
+    {
+      self->disco_number = -1;
+      clear_bit(0, &self->ttp_connect);
+      DRETURN(-ENETUNREACH, IRDA_SR_INFO, "No Cachelog...\n");
+    }
+  DEBUG(IRDA_SR_INFO, "Got the log (0x%p), size is %d\n",
+	self->discoveries, self->disco_number);
+
+  /* Start with the first discovery */
+  self->disco_index = -1;
+  self->daddr = DEV_ADDR_ANY;
+
+  /* This will fail if the log is empty - this is non-blocking */
+  ret = irnet_discover_next_daddr(self);
+  if(ret)
+    {
+      /* Close IAP */
+      if(self->iriap)
+	iriap_close(self->iriap);
+      self->iriap = NULL;
+
+      /* Cleanup our copy of the discovery log */
+      kfree(self->discoveries);
+      self->discoveries = NULL;
+
+      clear_bit(0, &self->ttp_connect);
+      DRETURN(-ENETUNREACH, IRDA_SR_INFO, "Cachelog empty...\n");
+    }
+
+  /* Follow me in irnet_discovervalue_confirm() */
+
+  DEXIT(IRDA_SR_TRACE, "\n");
+  return 0;
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Function irnet_dname_to_daddr (self)
+ *
+ *    Convert an IrDA nickname to a valid IrDA address
+ *
+ * It basically look into the discovery log until there is a match.
+ */
+static inline int
+irnet_dname_to_daddr(irnet_socket *	self)
+{
+  struct irda_device_info *discoveries;	/* Copy of the discovery log */
+  int	number;			/* Number of nodes in the log */
+  int	i;
+
+  DENTER(IRDA_SR_TRACE, "(self=0x%p)\n", self);
+
+  /* Ask lmp for the current discovery log */
+  discoveries = irlmp_get_discoveries(&number, 0xffff,
+				      DISCOVERY_DEFAULT_SLOTS);
+  /* Check if the we got some results */
+  if(discoveries == NULL)
+    DRETURN(-ENETUNREACH, IRDA_SR_INFO, "Cachelog empty...\n");
+
+  /*
+   * Now, check all discovered devices (if any), and connect
+   * client only about the services that the client is
+   * interested in...
+   */
+  for(i = 0; i < number; i++)
+    {
+      /* Does the name match ? */
+      if(!strncmp(discoveries[i].info, self->rname, NICKNAME_MAX_LEN))
+	{
+	  /* Yes !!! Get it.. */
+	  self->daddr = discoveries[i].daddr;
+	  DEBUG(IRDA_SR_INFO, "discovered device ``%s'' at address 0x%08x.\n",
+		self->rname, self->daddr);
+	  kfree(discoveries);
+	  DEXIT(IRDA_SR_TRACE, "\n");
+	  return 0;
+	}
+    }
+  /* No luck ! */
+  DEBUG(IRDA_SR_INFO, "cannot discover device ``%s'' !!!\n", self->rname);
+  kfree(discoveries);
+  return -EADDRNOTAVAIL;
+}
+
+
+/************************* SOCKET ROUTINES *************************/
+/*
+ * This are the main operations on IrNET sockets, basically to create
+ * and destroy IrNET sockets. These are called from the PPP part...
+ */
+
+/*------------------------------------------------------------------*/
+/*
+ * Create a IrNET instance : just initialise some parameters...
+ */
+int
+irda_irnet_create(irnet_socket *	self)
+{
+  DENTER(IRDA_SOCK_TRACE, "(self=0x%p)\n", self);
+
+  self->magic = IRNET_MAGIC;	/* Paranoia */
+
+  self->ttp_open = 0;		/* Prevent higher layer from accessing IrTTP */
+  self->ttp_connect = 0;	/* Not connecting yet */
+  self->rname[0] = '\0';	/* May be set via control channel */
+  self->rdaddr = DEV_ADDR_ANY;	/* May be set via control channel */
+  self->rsaddr = DEV_ADDR_ANY;	/* May be set via control channel */
+  self->daddr = DEV_ADDR_ANY;	/* Until we get connected */
+  self->saddr = DEV_ADDR_ANY;	/* Until we get connected */
+  self->max_sdu_size_rx = TTP_SAR_UNBOUND;
+
+  /* Register as a client with IrLMP */
+  self->ckey = irlmp_register_client(0, NULL, NULL, NULL);
+#ifdef DISCOVERY_NOMASK
+  self->mask = 0xffff;		/* For W2k compatibility */
+#else /* DISCOVERY_NOMASK */
+  self->mask = irlmp_service_to_hint(S_LAN);
+#endif /* DISCOVERY_NOMASK */
+  self->tx_flow = FLOW_START;	/* Flow control from IrTTP */
+
+  INIT_WORK(&self->disconnect_work, irnet_ppp_disconnect);
+
+  DEXIT(IRDA_SOCK_TRACE, "\n");
+  return 0;
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Connect to the other side :
+ *	o convert device name to an address
+ *	o find the socket number (dlsap)
+ *	o Establish the connection
+ *
+ * Note : We no longer mimic af_irda. The IAS query for finding the TSAP
+ * is done asynchronously, like the TTP connection. This allow us to
+ * call this function from any context (not only process).
+ * The downside is that following what's happening in there is tricky
+ * because it involve various functions all over the place...
+ */
+int
+irda_irnet_connect(irnet_socket *	self)
+{
+  int		err;
+
+  DENTER(IRDA_SOCK_TRACE, "(self=0x%p)\n", self);
+
+  /* Check if we are already trying to connect.
+   * Because irda_irnet_connect() can be called directly by pppd plus
+   * packet retries in ppp_generic and connect may take time, plus we may
+   * race with irnet_connect_indication(), we need to be careful there... */
+  if(test_and_set_bit(0, &self->ttp_connect))
+    DRETURN(-EBUSY, IRDA_SOCK_INFO, "Already connecting...\n");
+  if((self->iriap != NULL) || (self->tsap != NULL))
+    DERROR(IRDA_SOCK_ERROR, "Socket not cleaned up...\n");
+
+  /* Insert ourselves in the hashbin so that the IrNET server can find us.
+   * Notes : 4th arg is string of 32 char max and must be null terminated
+   *	     When 4th arg is used (string), 3rd arg isn't (int)
+   *	     Can't re-insert (MUST remove first) so check for that... */
+  if((irnet_server.running) && (self->q.q_next == NULL))
+    {
+      spin_lock_bh(&irnet_server.spinlock);
+      hashbin_insert(irnet_server.list, (irda_queue_t *) self, 0, self->rname);
+      spin_unlock_bh(&irnet_server.spinlock);
+      DEBUG(IRDA_SOCK_INFO, "Inserted ``%s'' in hashbin...\n", self->rname);
+    }
+
+  /* If we don't have anything (no address, no name) */
+  if((self->rdaddr == DEV_ADDR_ANY) && (self->rname[0] == '\0'))
+    {
+      /* Try to find a suitable address */
+      if((err = irnet_discover_daddr_and_lsap_sel(self)) != 0)
+	DRETURN(err, IRDA_SOCK_INFO, "auto-connect failed!\n");
+      /* In most cases, the call above is non-blocking */
+    }
+  else
+    {
+      /* If we have only the name (no address), try to get an address */
+      if(self->rdaddr == DEV_ADDR_ANY)
+	{
+	  if((err = irnet_dname_to_daddr(self)) != 0)
+	    DRETURN(err, IRDA_SOCK_INFO, "name connect failed!\n");
+	}
+      else
+	/* Use the requested destination address */
+	self->daddr = self->rdaddr;
+
+      /* Query remote LM-IAS to find LSAP selector */
+      irnet_find_lsap_sel(self);
+      /* The above call is non blocking */
+    }
+
+  /* At this point, we are waiting for the IrDA stack to call us back,
+   * or we have already failed.
+   * We will finish the connection procedure in irnet_connect_tsap().
+   */
+  DEXIT(IRDA_SOCK_TRACE, "\n");
+  return 0;
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Function irda_irnet_destroy(self)
+ *
+ *    Destroy irnet instance
+ *
+ * Note : this need to be called from a process context.
+ */
+void
+irda_irnet_destroy(irnet_socket *	self)
+{
+  DENTER(IRDA_SOCK_TRACE, "(self=0x%p)\n", self);
+  if(self == NULL)
+    return;
+
+  /* Remove ourselves from hashbin (if we are queued in hashbin)
+   * Note : `irnet_server.running' protect us from calls in hashbin_delete() */
+  if((irnet_server.running) && (self->q.q_next != NULL))
+    {
+      struct irnet_socket *	entry;
+      DEBUG(IRDA_SOCK_INFO, "Removing from hash..\n");
+      spin_lock_bh(&irnet_server.spinlock);
+      entry = hashbin_remove_this(irnet_server.list, (irda_queue_t *) self);
+      self->q.q_next = NULL;
+      spin_unlock_bh(&irnet_server.spinlock);
+      DASSERT(entry == self, , IRDA_SOCK_ERROR, "Can't remove from hash.\n");
+    }
+
+  /* If we were connected, post a message */
+  if(test_bit(0, &self->ttp_open))
+    {
+      /* Note : as the disconnect comes from ppp_generic, the unit number
+       * doesn't exist anymore when we post the event, so we need to pass
+       * NULL as the first arg... */
+      irnet_post_event(NULL, IRNET_DISCONNECT_TO,
+		       self->saddr, self->daddr, self->rname, 0);
+    }
+
+  /* Prevent various IrDA callbacks from messing up things
+   * Need to be first */
+  clear_bit(0, &self->ttp_connect);
+
+  /* Prevent higher layer from accessing IrTTP */
+  clear_bit(0, &self->ttp_open);
+
+  /* Unregister with IrLMP */
+  irlmp_unregister_client(self->ckey);
+
+  /* Unregister with LM-IAS */
+  if(self->iriap)
+    {
+      iriap_close(self->iriap);
+      self->iriap = NULL;
+    }
+
+  /* Cleanup eventual discoveries from connection attempt or control channel */
+  if(self->discoveries != NULL)
+    {
+      /* Cleanup our copy of the discovery log */
+      kfree(self->discoveries);
+      self->discoveries = NULL;
+    }
+
+  /* Close our IrTTP connection */
+  if(self->tsap)
+    {
+      DEBUG(IRDA_SOCK_INFO, "Closing our TTP connection.\n");
+      irttp_disconnect_request(self->tsap, NULL, P_NORMAL);
+      irttp_close_tsap(self->tsap);
+      self->tsap = NULL;
+    }
+  self->stsap_sel = 0;
+
+  DEXIT(IRDA_SOCK_TRACE, "\n");
+}
+
+
+/************************** SERVER SOCKET **************************/
+/*
+ * The IrNET service is composed of one server socket and a variable
+ * number of regular IrNET sockets. The server socket is supposed to
+ * handle incoming connections and redirect them to one IrNET sockets.
+ * It's a superset of the regular IrNET socket, but has a very distinct
+ * behaviour...
+ */
+
+/*------------------------------------------------------------------*/
+/*
+ * Function irnet_daddr_to_dname (self)
+ *
+ *    Convert an IrDA address to a IrDA nickname
+ *
+ * It basically look into the discovery log until there is a match.
+ */
+static inline int
+irnet_daddr_to_dname(irnet_socket *	self)
+{
+  struct irda_device_info *discoveries;	/* Copy of the discovery log */
+  int	number;			/* Number of nodes in the log */
+  int	i;
+
+  DENTER(IRDA_SERV_TRACE, "(self=0x%p)\n", self);
+
+  /* Ask lmp for the current discovery log */
+  discoveries = irlmp_get_discoveries(&number, 0xffff,
+				      DISCOVERY_DEFAULT_SLOTS);
+  /* Check if the we got some results */
+  if (discoveries == NULL)
+    DRETURN(-ENETUNREACH, IRDA_SERV_INFO, "Cachelog empty...\n");
+
+  /* Now, check all discovered devices (if any) */
+  for(i = 0; i < number; i++)
+    {
+      /* Does the name match ? */
+      if(discoveries[i].daddr == self->daddr)
+	{
+	  /* Yes !!! Get it.. */
+	  strlcpy(self->rname, discoveries[i].info, sizeof(self->rname));
+	  self->rname[sizeof(self->rname) - 1] = '\0';
+	  DEBUG(IRDA_SERV_INFO, "Device 0x%08x is in fact ``%s''.\n",
+		self->daddr, self->rname);
+	  kfree(discoveries);
+	  DEXIT(IRDA_SERV_TRACE, "\n");
+	  return 0;
+	}
+    }
+  /* No luck ! */
+  DEXIT(IRDA_SERV_INFO, ": cannot discover device 0x%08x !!!\n", self->daddr);
+  kfree(discoveries);
+  return -EADDRNOTAVAIL;
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Function irda_find_socket (self)
+ *
+ *    Find the correct IrNET socket
+ *
+ * Look into the list of IrNET sockets and finds one with the right
+ * properties...
+ */
+static inline irnet_socket *
+irnet_find_socket(irnet_socket *	self)
+{
+  irnet_socket *	new = (irnet_socket *) NULL;
+  int			err;
+
+  DENTER(IRDA_SERV_TRACE, "(self=0x%p)\n", self);
+
+  /* Get the addresses of the requester */
+  self->daddr = irttp_get_daddr(self->tsap);
+  self->saddr = irttp_get_saddr(self->tsap);
+
+  /* Try to get the IrDA nickname of the requester */
+  err = irnet_daddr_to_dname(self);
+
+  /* Protect access to the instance list */
+  spin_lock_bh(&irnet_server.spinlock);
+
+  /* So now, try to get an socket having specifically
+   * requested that nickname */
+  if(err == 0)
+    {
+      new = (irnet_socket *) hashbin_find(irnet_server.list,
+					  0, self->rname);
+      if(new)
+	DEBUG(IRDA_SERV_INFO, "Socket 0x%p matches rname ``%s''.\n",
+	      new, new->rname);
+    }
+
+  /* If no name matches, try to find an socket by the destination address */
+  /* It can be either the requested destination address (set via the
+   * control channel), or the current destination address if the
+   * socket is in the middle of a connection request */
+  if(new == (irnet_socket *) NULL)
+    {
+      new = (irnet_socket *) hashbin_get_first(irnet_server.list);
+      while(new !=(irnet_socket *) NULL)
+	{
+	  /* Does it have the same address ? */
+	  if((new->rdaddr == self->daddr) || (new->daddr == self->daddr))
+	    {
+	      /* Yes !!! Get it.. */
+	      DEBUG(IRDA_SERV_INFO, "Socket 0x%p matches daddr %#08x.\n",
+		    new, self->daddr);
+	      break;
+	    }
+	  new = (irnet_socket *) hashbin_get_next(irnet_server.list);
+	}
+    }
+
+  /* If we don't have any socket, get the first unconnected socket */
+  if(new == (irnet_socket *) NULL)
+    {
+      new = (irnet_socket *) hashbin_get_first(irnet_server.list);
+      while(new !=(irnet_socket *) NULL)
+	{
+	  /* Is it available ? */
+	  if(!(test_bit(0, &new->ttp_open)) && (new->rdaddr == DEV_ADDR_ANY) &&
+	     (new->rname[0] == '\0') && (new->ppp_open))
+	    {
+	      /* Yes !!! Get it.. */
+	      DEBUG(IRDA_SERV_INFO, "Socket 0x%p is free.\n",
+		    new);
+	      break;
+	    }
+	  new = (irnet_socket *) hashbin_get_next(irnet_server.list);
+	}
+    }
+
+  /* Spin lock end */
+  spin_unlock_bh(&irnet_server.spinlock);
+
+  DEXIT(IRDA_SERV_TRACE, " - new = 0x%p\n", new);
+  return new;
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Function irda_connect_socket (self)
+ *
+ *    Connect an incoming connection to the socket
+ *
+ */
+static inline int
+irnet_connect_socket(irnet_socket *	server,
+		     irnet_socket *	new,
+		     struct qos_info *	qos,
+		     __u32		max_sdu_size,
+		     __u8		max_header_size)
+{
+  DENTER(IRDA_SERV_TRACE, "(server=0x%p, new=0x%p)\n",
+	 server, new);
+
+  /* Now attach up the new socket */
+  new->tsap = irttp_dup(server->tsap, new);
+  DABORT(new->tsap == NULL, -1, IRDA_SERV_ERROR, "dup failed!\n");
+
+  /* Set up all the relevant parameters on the new socket */
+  new->stsap_sel = new->tsap->stsap_sel;
+  new->dtsap_sel = new->tsap->dtsap_sel;
+  new->saddr = irttp_get_saddr(new->tsap);
+  new->daddr = irttp_get_daddr(new->tsap);
+
+  new->max_header_size = max_header_size;
+  new->max_sdu_size_tx = max_sdu_size;
+  new->max_data_size   = max_sdu_size;
+#ifdef STREAM_COMPAT
+  /* If we want to receive "stream sockets" */
+  if(max_sdu_size == 0)
+    new->max_data_size = irttp_get_max_seg_size(new->tsap);
+#endif /* STREAM_COMPAT */
+
+  /* Clean up the original one to keep it in listen state */
+  irttp_listen(server->tsap);
+
+  /* Send a connection response on the new socket */
+  irttp_connect_response(new->tsap, new->max_sdu_size_rx, NULL);
+
+  /* Allow PPP to send its junk over the new socket... */
+  set_bit(0, &new->ttp_open);
+
+  /* Not connecting anymore, and clean up last possible remains
+   * of connection attempts on the socket */
+  clear_bit(0, &new->ttp_connect);
+  if(new->iriap)
+    {
+      iriap_close(new->iriap);
+      new->iriap = NULL;
+    }
+  if(new->discoveries != NULL)
+    {
+      kfree(new->discoveries);
+      new->discoveries = NULL;
+    }
+
+#ifdef CONNECT_INDIC_KICK
+  /* As currently we don't block packets in ppp_irnet_send() while passive,
+   * this is not really needed...
+   * Also, not doing it give IrDA a chance to finish the setup properly
+   * before being swamped with packets... */
+  ppp_output_wakeup(&new->chan);
+#endif /* CONNECT_INDIC_KICK */
+
+  /* Notify the control channel */
+  irnet_post_event(new, IRNET_CONNECT_FROM,
+		   new->saddr, new->daddr, server->rname, 0);
+
+  DEXIT(IRDA_SERV_TRACE, "\n");
+  return 0;
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Function irda_disconnect_server (self)
+ *
+ *    Cleanup the server socket when the incoming connection abort
+ *
+ */
+static inline void
+irnet_disconnect_server(irnet_socket *	self,
+			struct sk_buff *skb)
+{
+  DENTER(IRDA_SERV_TRACE, "(self=0x%p)\n", self);
+
+  /* Put the received packet in the black hole */
+  kfree_skb(skb);
+
+#ifdef FAIL_SEND_DISCONNECT
+  /* Tell the other party we don't want to be connected */
+  /* Hum... Is it the right thing to do ? And do we need to send
+   * a connect response before ? It looks ok without this... */
+  irttp_disconnect_request(self->tsap, NULL, P_NORMAL);
+#endif /* FAIL_SEND_DISCONNECT */
+
+  /* Notify the control channel (see irnet_find_socket()) */
+  irnet_post_event(NULL, IRNET_REQUEST_FROM,
+		   self->saddr, self->daddr, self->rname, 0);
+
+  /* Clean up the server to keep it in listen state */
+  irttp_listen(self->tsap);
+
+  DEXIT(IRDA_SERV_TRACE, "\n");
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Function irda_setup_server (self)
+ *
+ *    Create a IrTTP server and set it up...
+ *
+ * Register the IrLAN hint bit, create a IrTTP instance for us,
+ * set all the IrTTP callbacks and create an IrIAS entry...
+ */
+static inline int
+irnet_setup_server(void)
+{
+  __u16		hints;
+
+  DENTER(IRDA_SERV_TRACE, "()\n");
+
+  /* Initialise the regular socket part of the server */
+  irda_irnet_create(&irnet_server.s);
+
+  /* Open a local TSAP (an IrTTP instance) for the server */
+  irnet_open_tsap(&irnet_server.s);
+
+  /* PPP part setup */
+  irnet_server.s.ppp_open = 0;
+  irnet_server.s.chan.private = NULL;
+  irnet_server.s.file = NULL;
+
+  /* Get the hint bit corresponding to IrLAN */
+  /* Note : we overload the IrLAN hint bit. As it is only a "hint", and as
+   * we provide roughly the same functionality as IrLAN, this is ok.
+   * In fact, the situation is similar as JetSend overloading the Obex hint
+   */
+  hints = irlmp_service_to_hint(S_LAN);
+
+#ifdef ADVERTISE_HINT
+  /* Register with IrLMP as a service (advertise our hint bit) */
+  irnet_server.skey = irlmp_register_service(hints);
+#endif /* ADVERTISE_HINT */
+
+  /* Register with LM-IAS (so that people can connect to us) */
+  irnet_server.ias_obj = irias_new_object(IRNET_SERVICE_NAME, jiffies);
+  irias_add_integer_attrib(irnet_server.ias_obj, IRNET_IAS_VALUE,
+			   irnet_server.s.stsap_sel, IAS_KERNEL_ATTR);
+  irias_insert_object(irnet_server.ias_obj);
+
+#ifdef DISCOVERY_EVENTS
+  /* Tell IrLMP we want to be notified of newly discovered nodes */
+  irlmp_update_client(irnet_server.s.ckey, hints,
+		      irnet_discovery_indication, irnet_expiry_indication,
+		      (void *) &irnet_server.s);
+#endif
+
+  DEXIT(IRDA_SERV_TRACE, " - self=0x%p\n", &irnet_server.s);
+  return 0;
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Function irda_destroy_server (self)
+ *
+ *    Destroy the IrTTP server...
+ *
+ * Reverse of the previous function...
+ */
+static inline void
+irnet_destroy_server(void)
+{
+  DENTER(IRDA_SERV_TRACE, "()\n");
+
+#ifdef ADVERTISE_HINT
+  /* Unregister with IrLMP */
+  irlmp_unregister_service(irnet_server.skey);
+#endif /* ADVERTISE_HINT */
+
+  /* Unregister with LM-IAS */
+  if(irnet_server.ias_obj)
+    irias_delete_object(irnet_server.ias_obj);
+
+  /* Cleanup the socket part */
+  irda_irnet_destroy(&irnet_server.s);
+
+  DEXIT(IRDA_SERV_TRACE, "\n");
+}
+
+
+/************************ IRDA-TTP CALLBACKS ************************/
+/*
+ * When we create a IrTTP instance, we pass to it a set of callbacks
+ * that IrTTP will call in case of various events.
+ * We take care of those events here.
+ */
+
+/*------------------------------------------------------------------*/
+/*
+ * Function irnet_data_indication (instance, sap, skb)
+ *
+ *    Received some data from TinyTP. Just queue it on the receive queue
+ *
+ */
+static int
+irnet_data_indication(void *	instance,
+		      void *	sap,
+		      struct sk_buff *skb)
+{
+  irnet_socket *	ap = (irnet_socket *) instance;
+  unsigned char *	p;
+  int			code = 0;
+
+  DENTER(IRDA_TCB_TRACE, "(self/ap=0x%p, skb=0x%p)\n",
+	 ap, skb);
+  DASSERT(skb != NULL, 0, IRDA_CB_ERROR, "skb is NULL !!!\n");
+
+  /* Check is ppp is ready to receive our packet */
+  if(!ap->ppp_open)
+    {
+      DERROR(IRDA_CB_ERROR, "PPP not ready, dropping packet...\n");
+      /* When we return error, TTP will need to requeue the skb and
+       * will stop the sender. IrTTP will stall until we send it a
+       * flow control request... */
+      return -ENOMEM;
+    }
+
+  /* strip address/control field if present */
+  p = skb->data;
+  if((p[0] == PPP_ALLSTATIONS) && (p[1] == PPP_UI))
+    {
+      /* chop off address/control */
+      if(skb->len < 3)
+	goto err_exit;
+      p = skb_pull(skb, 2);
+    }
+
+  /* decompress protocol field if compressed */
+  if(p[0] & 1)
+    {
+      /* protocol is compressed */
+      *(u8 *)skb_push(skb, 1) = 0;
+    }
+  else
+    if(skb->len < 2)
+      goto err_exit;
+
+  /* pass to generic ppp layer */
+  /* Note : how do I know if ppp can accept or not the packet ? This is
+   * essential if I want to manage flow control smoothly... */
+  ppp_input(&ap->chan, skb);
+
+  DEXIT(IRDA_TCB_TRACE, "\n");
+  return 0;
+
+ err_exit:
+  DERROR(IRDA_CB_ERROR, "Packet too small, dropping...\n");
+  kfree_skb(skb);
+  ppp_input_error(&ap->chan, code);
+  return 0;	/* Don't return an error code, only for flow control... */
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Function irnet_disconnect_indication (instance, sap, reason, skb)
+ *
+ *    Connection has been closed. Chech reason to find out why
+ *
+ * Note : there are many cases where we come here :
+ *	o attempted to connect, timeout
+ *	o connected, link is broken, LAP has timeout
+ *	o connected, other side close the link
+ *	o connection request on the server not handled
+ */
+static void
+irnet_disconnect_indication(void *	instance,
+			    void *	sap,
+			    LM_REASON	reason,
+			    struct sk_buff *skb)
+{
+  irnet_socket *	self = (irnet_socket *) instance;
+  int			test_open;
+  int			test_connect;
+
+  DENTER(IRDA_TCB_TRACE, "(self=0x%p)\n", self);
+  DASSERT(self != NULL, , IRDA_CB_ERROR, "Self is NULL !!!\n");
+
+  /* Don't care about it, but let's not leak it */
+  if(skb)
+    dev_kfree_skb(skb);
+
+  /* Prevent higher layer from accessing IrTTP */
+  test_open = test_and_clear_bit(0, &self->ttp_open);
+  /* Not connecting anymore...
+   * (note : TSAP is open, so IAP callbacks are no longer pending...) */
+  test_connect = test_and_clear_bit(0, &self->ttp_connect);
+
+  /* If both self->ttp_open and self->ttp_connect are NULL, it mean that we
+   * have a race condition with irda_irnet_destroy() or
+   * irnet_connect_indication(), so don't mess up tsap...
+   */
+  if(!(test_open || test_connect))
+    {
+      DERROR(IRDA_CB_ERROR, "Race condition detected...\n");
+      return;
+    }
+
+  /* If we were active, notify the control channel */
+  if(test_open)
+    irnet_post_event(self, IRNET_DISCONNECT_FROM,
+		     self->saddr, self->daddr, self->rname, 0);
+  else
+    /* If we were trying to connect, notify the control channel */
+    if((self->tsap) && (self != &irnet_server.s))
+      irnet_post_event(self, IRNET_NOANSWER_FROM,
+		       self->saddr, self->daddr, self->rname, 0);
+
+  /* Close our IrTTP connection, cleanup tsap */
+  if((self->tsap) && (self != &irnet_server.s))
+    {
+      DEBUG(IRDA_CB_INFO, "Closing our TTP connection.\n");
+      irttp_close_tsap(self->tsap);
+      self->tsap = NULL;
+    }
+  /* Cleanup the socket in case we want to reconnect in ppp_output_wakeup() */
+  self->stsap_sel = 0;
+  self->daddr = DEV_ADDR_ANY;
+  self->tx_flow = FLOW_START;
+
+  /* Deal with the ppp instance if it's still alive */
+  if(self->ppp_open)
+    {
+      if(test_open)
+	{
+	  /* ppp_unregister_channel() wants a user context. */
+	  schedule_work(&self->disconnect_work);
+	}
+      else
+	{
+	  /* If we were trying to connect, flush (drain) ppp_generic
+	   * Tx queue (most often we have blocked it), which will
+	   * trigger an other attempt to connect. If we are passive,
+	   * this will empty the Tx queue after last try. */
+	  ppp_output_wakeup(&self->chan);
+	}
+    }
+
+  DEXIT(IRDA_TCB_TRACE, "\n");
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Function irnet_connect_confirm (instance, sap, qos, max_sdu_size, skb)
+ *
+ *    Connections has been confirmed by the remote device
+ *
+ */
+static void
+irnet_connect_confirm(void *	instance,
+		      void *	sap,
+		      struct qos_info *qos,
+		      __u32	max_sdu_size,
+		      __u8	max_header_size,
+		      struct sk_buff *skb)
+{
+  irnet_socket *	self = (irnet_socket *) instance;
+
+  DENTER(IRDA_TCB_TRACE, "(self=0x%p)\n", self);
+
+  /* Check if socket is closing down (via irda_irnet_destroy()) */
+  if(! test_bit(0, &self->ttp_connect))
+    {
+      DERROR(IRDA_CB_ERROR, "Socket no longer connecting. Ouch !\n");
+      return;
+    }
+
+  /* How much header space do we need to reserve */
+  self->max_header_size = max_header_size;
+
+  /* IrTTP max SDU size in transmit direction */
+  self->max_sdu_size_tx = max_sdu_size;
+  self->max_data_size = max_sdu_size;
+#ifdef STREAM_COMPAT
+  if(max_sdu_size == 0)
+    self->max_data_size = irttp_get_max_seg_size(self->tsap);
+#endif /* STREAM_COMPAT */
+
+  /* At this point, IrLMP has assigned our source address */
+  self->saddr = irttp_get_saddr(self->tsap);
+
+  /* Allow higher layer to access IrTTP */
+  set_bit(0, &self->ttp_open);
+  clear_bit(0, &self->ttp_connect);	/* Not racy, IrDA traffic is serial */
+  /* Give a kick in the ass of ppp_generic so that he sends us some data */
+  ppp_output_wakeup(&self->chan);
+
+  /* Check size of received packet */
+  if(skb->len > 0)
+    {
+#ifdef PASS_CONNECT_PACKETS
+      DEBUG(IRDA_CB_INFO, "Passing connect packet to PPP.\n");
+      /* Try to pass it to PPP */
+      irnet_data_indication(instance, sap, skb);
+#else /* PASS_CONNECT_PACKETS */
+      DERROR(IRDA_CB_ERROR, "Dropping non empty packet.\n");
+      kfree_skb(skb);	/* Note : will be optimised with other kfree... */
+#endif /* PASS_CONNECT_PACKETS */
+    }
+  else
+    kfree_skb(skb);
+
+  /* Notify the control channel */
+  irnet_post_event(self, IRNET_CONNECT_TO,
+		   self->saddr, self->daddr, self->rname, 0);
+
+  DEXIT(IRDA_TCB_TRACE, "\n");
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Function irnet_flow_indication (instance, sap, flow)
+ *
+ *    Used by TinyTP to tell us if it can accept more data or not
+ *
+ */
+static void
+irnet_flow_indication(void *	instance,
+		      void *	sap,
+		      LOCAL_FLOW flow)
+{
+  irnet_socket *	self = (irnet_socket *) instance;
+  LOCAL_FLOW		oldflow = self->tx_flow;
+
+  DENTER(IRDA_TCB_TRACE, "(self=0x%p, flow=%d)\n", self, flow);
+
+  /* Update our state */
+  self->tx_flow = flow;
+
+  /* Check what IrTTP want us to do... */
+  switch(flow)
+    {
+    case FLOW_START:
+      DEBUG(IRDA_CB_INFO, "IrTTP wants us to start again\n");
+      /* Check if we really need to wake up PPP */
+      if(oldflow == FLOW_STOP)
+	ppp_output_wakeup(&self->chan);
+      else
+	DEBUG(IRDA_CB_INFO, "But we were already transmitting !!!\n");
+      break;
+    case FLOW_STOP:
+      DEBUG(IRDA_CB_INFO, "IrTTP wants us to slow down\n");
+      break;
+    default:
+      DEBUG(IRDA_CB_INFO, "Unknown flow command!\n");
+      break;
+    }
+
+  DEXIT(IRDA_TCB_TRACE, "\n");
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Function irnet_status_indication (instance, sap, reason, skb)
+ *
+ *    Link (IrLAP) status report.
+ *
+ */
+static void
+irnet_status_indication(void *	instance,
+			LINK_STATUS link,
+			LOCK_STATUS lock)
+{
+  irnet_socket *	self = (irnet_socket *) instance;
+
+  DENTER(IRDA_TCB_TRACE, "(self=0x%p)\n", self);
+  DASSERT(self != NULL, , IRDA_CB_ERROR, "Self is NULL !!!\n");
+
+  /* We can only get this event if we are connected */
+  switch(link)
+    {
+    case STATUS_NO_ACTIVITY:
+      irnet_post_event(self, IRNET_BLOCKED_LINK,
+		       self->saddr, self->daddr, self->rname, 0);
+      break;
+    default:
+      DEBUG(IRDA_CB_INFO, "Unknown status...\n");
+    }
+
+  DEXIT(IRDA_TCB_TRACE, "\n");
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Function irnet_connect_indication(instance, sap, qos, max_sdu_size, userdata)
+ *
+ *    Incoming connection
+ *
+ * In theory, this function is called only on the server socket.
+ * Some other node is attempting to connect to the IrNET service, and has
+ * sent a connection request on our server socket.
+ * We just redirect the connection to the relevant IrNET socket.
+ *
+ * Note : we also make sure that between 2 irnet nodes, there can
+ * exist only one irnet connection.
+ */
+static void
+irnet_connect_indication(void *		instance,
+			 void *		sap,
+			 struct qos_info *qos,
+			 __u32		max_sdu_size,
+			 __u8		max_header_size,
+			 struct sk_buff *skb)
+{
+  irnet_socket *	server = &irnet_server.s;
+  irnet_socket *	new = (irnet_socket *) NULL;
+
+  DENTER(IRDA_TCB_TRACE, "(server=0x%p)\n", server);
+  DASSERT(instance == &irnet_server, , IRDA_CB_ERROR,
+	  "Invalid instance (0x%p) !!!\n", instance);
+  DASSERT(sap == irnet_server.s.tsap, , IRDA_CB_ERROR, "Invalid sap !!!\n");
+
+  /* Try to find the most appropriate IrNET socket */
+  new = irnet_find_socket(server);
+
+  /* After all this hard work, do we have an socket ? */
+  if(new == (irnet_socket *) NULL)
+    {
+      DEXIT(IRDA_CB_INFO, ": No socket waiting for this connection.\n");
+      irnet_disconnect_server(server, skb);
+      return;
+    }
+
+  /* Is the socket already busy ? */
+  if(test_bit(0, &new->ttp_open))
+    {
+      DEXIT(IRDA_CB_INFO, ": Socket already connected.\n");
+      irnet_disconnect_server(server, skb);
+      return;
+    }
+
+  /* The following code is a bit tricky, so need comments ;-)
+   */
+  /* If ttp_connect is set, the socket is trying to connect to the other
+   * end and may have sent a IrTTP connection request and is waiting for
+   * a connection response (that may never come).
+   * Now, the pain is that the socket may have opened a tsap and is
+   * waiting on it, while the other end is trying to connect to it on
+   * another tsap.
+   * Because IrNET can be peer to peer, we need to workaround this.
+   * Furthermore, the way the irnetd script is implemented, the
+   * target will create a second IrNET connection back to the
+   * originator and expect the originator to bind this new connection
+   * to the original PPPD instance.
+   * And of course, if we don't use irnetd, we can have a race when
+   * both side try to connect simultaneously, which could leave both
+   * connections half closed (yuck).
+   * Conclusions :
+   *	1) The "originator" must accept the new connection and get rid
+   *	   of the old one so that irnetd works
+   *	2) One side must deny the new connection to avoid races,
+   *	   but both side must agree on which side it is...
+   * Most often, the originator is primary at the LAP layer.
+   * Jean II
+   */
+  /* Now, let's look at the way I wrote the test...
+   * We need to clear up the ttp_connect flag atomically to prevent
+   * irnet_disconnect_indication() to mess up the tsap we are going to close.
+   * We want to clear the ttp_connect flag only if we close the tsap,
+   * otherwise we will never close it, so we need to check for primary
+   * *before* doing the test on the flag.
+   * And of course, ALLOW_SIMULT_CONNECT can disable this entirely...
+   * Jean II
+   */
+
+  /* Socket already connecting ? On primary ? */
+  if(0
+#ifdef ALLOW_SIMULT_CONNECT
+     || ((irttp_is_primary(server->tsap) == 1) &&	/* primary */
+	 (test_and_clear_bit(0, &new->ttp_connect)))
+#endif /* ALLOW_SIMULT_CONNECT */
+     )
+    {
+      DERROR(IRDA_CB_ERROR, "Socket already connecting, but going to reuse it !\n");
+
+      /* Cleanup the old TSAP if necessary - IrIAP will be cleaned up later */
+      if(new->tsap != NULL)
+	{
+	  /* Close the old connection the new socket was attempting,
+	   * so that we can hook it up to the new connection.
+	   * It's now safe to do it... */
+	  irttp_close_tsap(new->tsap);
+	  new->tsap = NULL;
+	}
+    }
+  else
+    {
+      /* Three options :
+       * 1) socket was not connecting or connected : ttp_connect should be 0.
+       * 2) we don't want to connect the socket because we are secondary or
+       * ALLOW_SIMULT_CONNECT is undefined. ttp_connect should be 1.
+       * 3) we are half way in irnet_disconnect_indication(), and it's a
+       * nice race condition... Fortunately, we can detect that by checking
+       * if tsap is still alive. On the other hand, we can't be in
+       * irda_irnet_destroy() otherwise we would not have found this
+       * socket in the hashbin.
+       * Jean II */
+      if((test_bit(0, &new->ttp_connect)) || (new->tsap != NULL))
+	{
+	  /* Don't mess this socket, somebody else in in charge... */
+	  DERROR(IRDA_CB_ERROR, "Race condition detected, socket in use, abort connect...\n");
+	  irnet_disconnect_server(server, skb);
+	  return;
+	}
+    }
+
+  /* So : at this point, we have a socket, and it is idle. Good ! */
+  irnet_connect_socket(server, new, qos, max_sdu_size, max_header_size);
+
+  /* Check size of received packet */
+  if(skb->len > 0)
+    {
+#ifdef PASS_CONNECT_PACKETS
+      DEBUG(IRDA_CB_INFO, "Passing connect packet to PPP.\n");
+      /* Try to pass it to PPP */
+      irnet_data_indication(new, new->tsap, skb);
+#else /* PASS_CONNECT_PACKETS */
+      DERROR(IRDA_CB_ERROR, "Dropping non empty packet.\n");
+      kfree_skb(skb);	/* Note : will be optimised with other kfree... */
+#endif /* PASS_CONNECT_PACKETS */
+    }
+  else
+    kfree_skb(skb);
+
+  DEXIT(IRDA_TCB_TRACE, "\n");
+}
+
+
+/********************** IRDA-IAS/LMP CALLBACKS **********************/
+/*
+ * These are the callbacks called by other layers of the IrDA stack,
+ * mainly LMP for discovery and IAS for name queries.
+ */
+
+/*------------------------------------------------------------------*/
+/*
+ * Function irnet_getvalue_confirm (result, obj_id, value, priv)
+ *
+ *    Got answer from remote LM-IAS, just connect
+ *
+ * This is the reply to a IAS query we were doing to find the TSAP of
+ * the device we want to connect to.
+ * If we have found a valid TSAP, just initiate the TTP connection
+ * on this TSAP.
+ */
+static void
+irnet_getvalue_confirm(int	result,
+		       __u16	obj_id,
+		       struct ias_value *value,
+		       void *	priv)
+{
+  irnet_socket *	self = (irnet_socket *) priv;
+
+  DENTER(IRDA_OCB_TRACE, "(self=0x%p)\n", self);
+  DASSERT(self != NULL, , IRDA_OCB_ERROR, "Self is NULL !!!\n");
+
+  /* Check if already connected (via irnet_connect_socket())
+   * or socket is closing down (via irda_irnet_destroy()) */
+  if(! test_bit(0, &self->ttp_connect))
+    {
+      DERROR(IRDA_OCB_ERROR, "Socket no longer connecting. Ouch !\n");
+      return;
+    }
+
+  /* We probably don't need to make any more queries */
+  iriap_close(self->iriap);
+  self->iriap = NULL;
+
+  /* Post process the IAS reply */
+  self->dtsap_sel = irnet_ias_to_tsap(self, result, value);
+
+  /* If error, just go out */
+  if(self->errno)
+    {
+      clear_bit(0, &self->ttp_connect);
+      DERROR(IRDA_OCB_ERROR, "IAS connect failed ! (0x%X)\n", self->errno);
+      return;
+    }
+
+  DEBUG(IRDA_OCB_INFO, "daddr = %08x, lsap = %d, starting IrTTP connection\n",
+	self->daddr, self->dtsap_sel);
+
+  /* Start up TTP - non blocking */
+  irnet_connect_tsap(self);
+
+  DEXIT(IRDA_OCB_TRACE, "\n");
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Function irnet_discovervalue_confirm (result, obj_id, value, priv)
+ *
+ *    Handle the TSAP discovery procedure state machine.
+ *    Got answer from remote LM-IAS, try next device
+ *
+ * We are doing a  TSAP discovery procedure, and we got an answer to
+ * a IAS query we were doing to find the TSAP on one of the address
+ * in the discovery log.
+ *
+ * If we have found a valid TSAP for the first time, save it. If it's
+ * not the first time we found one, complain.
+ *
+ * If we have more addresses in the log, just initiate a new query.
+ * Note that those query may fail (see irnet_discover_daddr_and_lsap_sel())
+ *
+ * Otherwise, wrap up the procedure (cleanup), check if we have found
+ * any device and connect to it.
+ */
+static void
+irnet_discovervalue_confirm(int		result,
+			    __u16	obj_id,
+			    struct ias_value *value,
+			    void *	priv)
+{
+  irnet_socket *	self = (irnet_socket *) priv;
+  __u8			dtsap_sel;		/* TSAP we are looking for */
+
+  DENTER(IRDA_OCB_TRACE, "(self=0x%p)\n", self);
+  DASSERT(self != NULL, , IRDA_OCB_ERROR, "Self is NULL !!!\n");
+
+  /* Check if already connected (via irnet_connect_socket())
+   * or socket is closing down (via irda_irnet_destroy()) */
+  if(! test_bit(0, &self->ttp_connect))
+    {
+      DERROR(IRDA_OCB_ERROR, "Socket no longer connecting. Ouch !\n");
+      return;
+    }
+
+  /* Post process the IAS reply */
+  dtsap_sel = irnet_ias_to_tsap(self, result, value);
+
+  /* Have we got something ? */
+  if(self->errno == 0)
+    {
+      /* We found the requested service */
+      if(self->daddr != DEV_ADDR_ANY)
+	{
+	  DERROR(IRDA_OCB_ERROR, "More than one device in range supports IrNET...\n");
+	}
+      else
+	{
+	  /* First time we found that one, save it ! */
+	  self->daddr = self->discoveries[self->disco_index].daddr;
+	  self->dtsap_sel = dtsap_sel;
+	}
+    }
+
+  /* If no failure */
+  if((self->errno == -EADDRNOTAVAIL) || (self->errno == 0))
+    {
+      int	ret;
+
+      /* Search the next node */
+      ret = irnet_discover_next_daddr(self);
+      if(!ret)
+	{
+	  /* In this case, the above request was non-blocking.
+	   * We will return here after a while... */
+	  return;
+	}
+      /* In this case, we have processed the last discovery item */
+    }
+
+  /* No more queries to be done (failure or last one) */
+
+  /* We probably don't need to make any more queries */
+  iriap_close(self->iriap);
+  self->iriap = NULL;
+
+  /* No more items : remove the log and signal termination */
+  DEBUG(IRDA_OCB_INFO, "Cleaning up log (0x%p)\n",
+	self->discoveries);
+  if(self->discoveries != NULL)
+    {
+      /* Cleanup our copy of the discovery log */
+      kfree(self->discoveries);
+      self->discoveries = NULL;
+    }
+  self->disco_number = -1;
+
+  /* Check out what we found */
+  if(self->daddr == DEV_ADDR_ANY)
+    {
+      self->daddr = DEV_ADDR_ANY;
+      clear_bit(0, &self->ttp_connect);
+      DEXIT(IRDA_OCB_TRACE, ": cannot discover IrNET in any device !!!\n");
+      return;
+    }
+
+  /* We have a valid address - just connect */
+
+  DEBUG(IRDA_OCB_INFO, "daddr = %08x, lsap = %d, starting IrTTP connection\n",
+	self->daddr, self->dtsap_sel);
+
+  /* Start up TTP - non blocking */
+  irnet_connect_tsap(self);
+
+  DEXIT(IRDA_OCB_TRACE, "\n");
+}
+
+#ifdef DISCOVERY_EVENTS
+/*------------------------------------------------------------------*/
+/*
+ * Function irnet_discovery_indication (discovery)
+ *
+ *    Got a discovery indication from IrLMP, post an event
+ *
+ * Note : IrLMP take care of matching the hint mask for us, and also
+ * check if it is a "new" node for us...
+ *
+ * As IrLMP filter on the IrLAN hint bit, we get both IrLAN and IrNET
+ * nodes, so it's only at connection time that we will know if the
+ * node support IrNET, IrLAN or both. The other solution is to check
+ * in IAS the PNP ids and service name.
+ * Note : even if a node support IrNET (or IrLAN), it's no guarantee
+ * that we will be able to connect to it, the node might already be
+ * busy...
+ *
+ * One last thing : in some case, this function will trigger duplicate
+ * discovery events. On the other hand, we should catch all
+ * discoveries properly (i.e. not miss one). Filtering duplicate here
+ * is to messy, so we leave that to user space...
+ */
+static void
+irnet_discovery_indication(discinfo_t *		discovery,
+			   DISCOVERY_MODE	mode,
+			   void *		priv)
+{
+  irnet_socket *	self = &irnet_server.s;
+
+  DENTER(IRDA_OCB_TRACE, "(self=0x%p)\n", self);
+  DASSERT(priv == &irnet_server, , IRDA_OCB_ERROR,
+	  "Invalid instance (0x%p) !!!\n", priv);
+
+  DEBUG(IRDA_OCB_INFO, "Discovered new IrNET/IrLAN node %s...\n",
+	discovery->info);
+
+  /* Notify the control channel */
+  irnet_post_event(NULL, IRNET_DISCOVER,
+		   discovery->saddr, discovery->daddr, discovery->info,
+		   get_unaligned((__u16 *)discovery->hints));
+
+  DEXIT(IRDA_OCB_TRACE, "\n");
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Function irnet_expiry_indication (expiry)
+ *
+ *    Got a expiry indication from IrLMP, post an event
+ *
+ * Note : IrLMP take care of matching the hint mask for us, we only
+ * check if it is a "new" node...
+ */
+static void
+irnet_expiry_indication(discinfo_t *	expiry,
+			DISCOVERY_MODE	mode,
+			void *		priv)
+{
+  irnet_socket *	self = &irnet_server.s;
+
+  DENTER(IRDA_OCB_TRACE, "(self=0x%p)\n", self);
+  DASSERT(priv == &irnet_server, , IRDA_OCB_ERROR,
+	  "Invalid instance (0x%p) !!!\n", priv);
+
+  DEBUG(IRDA_OCB_INFO, "IrNET/IrLAN node %s expired...\n",
+	expiry->info);
+
+  /* Notify the control channel */
+  irnet_post_event(NULL, IRNET_EXPIRE,
+		   expiry->saddr, expiry->daddr, expiry->info,
+		   get_unaligned((__u16 *)expiry->hints));
+
+  DEXIT(IRDA_OCB_TRACE, "\n");
+}
+#endif /* DISCOVERY_EVENTS */
+
+
+/*********************** PROC ENTRY CALLBACKS ***********************/
+/*
+ * We create a instance in the /proc filesystem, and here we take care
+ * of that...
+ */
+
+#ifdef CONFIG_PROC_FS
+static int
+irnet_proc_show(struct seq_file *m, void *v)
+{
+  irnet_socket *	self;
+  char *		state;
+  int			i = 0;
+
+  /* Get the IrNET server information... */
+  seq_printf(m, "IrNET server - ");
+  seq_printf(m, "IrDA state: %s, ",
+		 (irnet_server.running ? "running" : "dead"));
+  seq_printf(m, "stsap_sel: %02x, ", irnet_server.s.stsap_sel);
+  seq_printf(m, "dtsap_sel: %02x\n", irnet_server.s.dtsap_sel);
+
+  /* Do we need to continue ? */
+  if(!irnet_server.running)
+    return 0;
+
+  /* Protect access to the instance list */
+  spin_lock_bh(&irnet_server.spinlock);
+
+  /* Get the sockets one by one... */
+  self = (irnet_socket *) hashbin_get_first(irnet_server.list);
+  while(self != NULL)
+    {
+      /* Start printing info about the socket. */
+      seq_printf(m, "\nIrNET socket %d - ", i++);
+
+      /* First, get the requested configuration */
+      seq_printf(m, "Requested IrDA name: \"%s\", ", self->rname);
+      seq_printf(m, "daddr: %08x, ", self->rdaddr);
+      seq_printf(m, "saddr: %08x\n", self->rsaddr);
+
+      /* Second, get all the PPP info */
+      seq_printf(m, "	PPP state: %s",
+		 (self->ppp_open ? "registered" : "unregistered"));
+      if(self->ppp_open)
+	{
+	  seq_printf(m, ", unit: ppp%d",
+			 ppp_unit_number(&self->chan));
+	  seq_printf(m, ", channel: %d",
+			 ppp_channel_index(&self->chan));
+	  seq_printf(m, ", mru: %d",
+			 self->mru);
+	  /* Maybe add self->flags ? Later... */
+	}
+
+      /* Then, get all the IrDA specific info... */
+      if(self->ttp_open)
+	state = "connected";
+      else
+	if(self->tsap != NULL)
+	  state = "connecting";
+	else
+	  if(self->iriap != NULL)
+	    state = "searching";
+	  else
+	    if(self->ttp_connect)
+	      state = "weird";
+	    else
+	      state = "idle";
+      seq_printf(m, "\n	IrDA state: %s, ", state);
+      seq_printf(m, "daddr: %08x, ", self->daddr);
+      seq_printf(m, "stsap_sel: %02x, ", self->stsap_sel);
+      seq_printf(m, "dtsap_sel: %02x\n", self->dtsap_sel);
+
+      /* Next socket, please... */
+      self = (irnet_socket *) hashbin_get_next(irnet_server.list);
+    }
+
+  /* Spin lock end */
+  spin_unlock_bh(&irnet_server.spinlock);
+
+  return 0;
+}
+
+static int irnet_proc_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, irnet_proc_show, NULL);
+}
+
+static const struct file_operations irnet_proc_fops = {
+	.owner		= THIS_MODULE,
+	.open		= irnet_proc_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+#endif /* PROC_FS */
+
+
+/********************** CONFIGURATION/CLEANUP **********************/
+/*
+ * Initialisation and teardown of the IrDA part, called at module
+ * insertion and removal...
+ */
+
+/*------------------------------------------------------------------*/
+/*
+ * Prepare the IrNET layer for operation...
+ */
+int __init
+irda_irnet_init(void)
+{
+  int		err = 0;
+
+  DENTER(MODULE_TRACE, "()\n");
+
+  /* Pure paranoia - should be redundant */
+  memset(&irnet_server, 0, sizeof(struct irnet_root));
+
+  /* Setup start of irnet instance list */
+  irnet_server.list = hashbin_new(HB_NOLOCK);
+  DABORT(irnet_server.list == NULL, -ENOMEM,
+	 MODULE_ERROR, "Can't allocate hashbin!\n");
+  /* Init spinlock for instance list */
+  spin_lock_init(&irnet_server.spinlock);
+
+  /* Initialise control channel */
+  init_waitqueue_head(&irnet_events.rwait);
+  irnet_events.index = 0;
+  /* Init spinlock for event logging */
+  spin_lock_init(&irnet_events.spinlock);
+
+#ifdef CONFIG_PROC_FS
+  /* Add a /proc file for irnet infos */
+  proc_create("irnet", 0, proc_irda, &irnet_proc_fops);
+#endif /* CONFIG_PROC_FS */
+
+  /* Setup the IrNET server */
+  err = irnet_setup_server();
+
+  if(!err)
+    /* We are no longer functional... */
+    irnet_server.running = 1;
+
+  DEXIT(MODULE_TRACE, "\n");
+  return err;
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Cleanup at exit...
+ */
+void __exit
+irda_irnet_cleanup(void)
+{
+  DENTER(MODULE_TRACE, "()\n");
+
+  /* We are no longer there... */
+  irnet_server.running = 0;
+
+#ifdef CONFIG_PROC_FS
+  /* Remove our /proc file */
+  remove_proc_entry("irnet", proc_irda);
+#endif /* CONFIG_PROC_FS */
+
+  /* Remove our IrNET server from existence */
+  irnet_destroy_server();
+
+  /* Remove all instances of IrNET socket still present */
+  hashbin_delete(irnet_server.list, (FREE_FUNC) irda_irnet_destroy);
+
+  DEXIT(MODULE_TRACE, "\n");
+}
diff --git a/drivers/staging/irda/net/irnet/irnet_irda.h b/drivers/staging/irda/net/irnet/irnet_irda.h
new file mode 100644
index 000000000000..3e408952a3f1
--- /dev/null
+++ b/drivers/staging/irda/net/irnet/irnet_irda.h
@@ -0,0 +1,178 @@
+/*
+ *	IrNET protocol module : Synchronous PPP over an IrDA socket.
+ *
+ *		Jean II - HPL `00 - <jt@hpl.hp.com>
+ *
+ * This file contains all definitions and declarations necessary for the
+ * IRDA part of the IrNET module (dealing with IrTTP, IrIAS and co).
+ * This file is a private header, so other modules don't want to know
+ * what's in there...
+ */
+
+#ifndef IRNET_IRDA_H
+#define IRNET_IRDA_H
+
+/***************************** INCLUDES *****************************/
+/* Please add other headers in irnet.h */
+
+#include "irnet.h"		/* Module global include */
+
+/************************ CONSTANTS & MACROS ************************/
+
+/*
+ * Name of the service (socket name) used by IrNET
+ */
+/* IAS object name (or part of it) */
+#define IRNET_SERVICE_NAME	"IrNetv1"
+/* IAS attribute */
+#define IRNET_IAS_VALUE		"IrDA:TinyTP:LsapSel"
+/* LMP notify name for client (only for /proc/net/irda/irlmp) */
+#define IRNET_NOTIFY_NAME	"IrNET socket"
+/* LMP notify name for server (only for /proc/net/irda/irlmp) */
+#define IRNET_NOTIFY_NAME_SERV	"IrNET server"
+
+/****************************** TYPES ******************************/
+
+/*
+ * This is the main structure where we store all the data pertaining to
+ * the IrNET server (listen for connection requests) and the root
+ * of the IrNET socket list
+ */
+typedef struct irnet_root
+{
+  irnet_socket		s;		/* To pretend we are a client... */
+
+  /* Generic stuff */
+  int			magic;		/* Paranoia */
+  int			running;	/* Are we operational ? */
+
+  /* Link list of all IrNET instances opened */
+  hashbin_t *		list;
+  spinlock_t		spinlock;	/* Serialize access to the list */
+  /* Note : the way hashbin has been designed is absolutely not
+   * reentrant, beware... So, we blindly protect all with spinlock */
+
+  /* Handle for the hint bit advertised in IrLMP */
+  void *		skey;
+
+  /* Server socket part */
+  struct ias_object *	ias_obj;	/* Our service name + lsap in IAS */
+
+} irnet_root;
+
+
+/**************************** PROTOTYPES ****************************/
+
+/* ----------------------- CONTROL CHANNEL ----------------------- */
+static void
+	irnet_post_event(irnet_socket *,
+			 irnet_event,
+			 __u32,
+			 __u32,
+			 char *,
+			 __u16);
+/* ----------------------- IRDA SUBROUTINES ----------------------- */
+static inline int
+	irnet_open_tsap(irnet_socket *);
+static inline __u8
+	irnet_ias_to_tsap(irnet_socket *,
+			  int,
+			  struct ias_value *);
+static inline int
+	irnet_find_lsap_sel(irnet_socket *);
+static inline int
+	irnet_connect_tsap(irnet_socket *);
+static inline int
+	irnet_discover_next_daddr(irnet_socket *);
+static inline int
+	irnet_discover_daddr_and_lsap_sel(irnet_socket *);
+static inline int
+	irnet_dname_to_daddr(irnet_socket *);
+/* ------------------------ SERVER SOCKET ------------------------ */
+static inline int
+	irnet_daddr_to_dname(irnet_socket *);
+static inline irnet_socket *
+	irnet_find_socket(irnet_socket *);
+static inline int
+	irnet_connect_socket(irnet_socket *,
+			     irnet_socket *,
+			     struct qos_info *,
+			     __u32,
+			     __u8);
+static inline void
+	irnet_disconnect_server(irnet_socket *,
+				struct sk_buff *);
+static inline int
+	irnet_setup_server(void);
+static inline void
+	irnet_destroy_server(void);
+/* ---------------------- IRDA-TTP CALLBACKS ---------------------- */
+static int
+	irnet_data_indication(void *,		/* instance */
+			      void *,		/* sap */
+			      struct sk_buff *);
+static void
+	irnet_disconnect_indication(void *,
+				    void *,
+				    LM_REASON,
+				    struct sk_buff *);
+static void
+	irnet_connect_confirm(void *,
+			      void *,
+			      struct qos_info *,
+			      __u32,
+			      __u8,
+			      struct sk_buff *);
+static void
+	irnet_flow_indication(void *,
+			      void *,
+			      LOCAL_FLOW);
+static void
+	irnet_status_indication(void *,
+				LINK_STATUS,
+				LOCK_STATUS);
+static void
+	irnet_connect_indication(void *,
+				 void *,
+				 struct qos_info *,
+				 __u32,
+				 __u8,
+				 struct sk_buff *);
+/* -------------------- IRDA-IAS/LMP CALLBACKS -------------------- */
+static void
+	irnet_getvalue_confirm(int,
+			       __u16,
+			       struct ias_value *,
+			       void *);
+static void
+	irnet_discovervalue_confirm(int,
+				    __u16,
+				    struct ias_value *,
+				    void *);
+#ifdef DISCOVERY_EVENTS
+static void
+	irnet_discovery_indication(discinfo_t *,
+				   DISCOVERY_MODE,
+				   void *);
+static void
+	irnet_expiry_indication(discinfo_t *,
+				DISCOVERY_MODE,
+				void *);
+#endif
+
+/**************************** VARIABLES ****************************/
+
+/*
+ * The IrNET server. Listen to connection requests and co...
+ */
+static struct irnet_root	irnet_server;
+
+/* Control channel stuff (note : extern) */
+struct irnet_ctrl_channel	irnet_events;
+
+/* The /proc/net/irda directory, defined elsewhere... */
+#ifdef CONFIG_PROC_FS
+extern struct proc_dir_entry *proc_irda;
+#endif /* CONFIG_PROC_FS */
+
+#endif /* IRNET_IRDA_H */
diff --git a/drivers/staging/irda/net/irnet/irnet_ppp.c b/drivers/staging/irda/net/irnet/irnet_ppp.c
new file mode 100644
index 000000000000..7025dcb853d0
--- /dev/null
+++ b/drivers/staging/irda/net/irnet/irnet_ppp.c
@@ -0,0 +1,1189 @@
+/*
+ *	IrNET protocol module : Synchronous PPP over an IrDA socket.
+ *
+ *		Jean II - HPL `00 - <jt@hpl.hp.com>
+ *
+ * This file implement the PPP interface and /dev/irnet character device.
+ * The PPP interface hook to the ppp_generic module, handle all our
+ *	relationship to the PPP code in the kernel (and by extension to pppd),
+ *	and exchange PPP frames with this module (send/receive).
+ * The /dev/irnet device is used primarily for 2 functions :
+ *	1) as a stub for pppd (the ppp daemon), so that we can appropriately
+ *	generate PPP sessions (we pretend we are a tty).
+ *	2) as a control channel (write commands, read events)
+ */
+
+#include <linux/sched/signal.h>
+#include <linux/slab.h>
+
+#include "irnet_ppp.h"		/* Private header */
+/* Please put other headers in irnet.h - Thanks */
+
+/* Generic PPP callbacks (to call us) */
+static const struct ppp_channel_ops irnet_ppp_ops = {
+	.start_xmit = ppp_irnet_send,
+	.ioctl = ppp_irnet_ioctl
+};
+
+/************************* CONTROL CHANNEL *************************/
+/*
+ * When a pppd instance is not active on /dev/irnet, it acts as a control
+ * channel.
+ * Writing allow to set up the IrDA destination of the IrNET channel,
+ * and any application may be read events happening in IrNET...
+ */
+
+/*------------------------------------------------------------------*/
+/*
+ * Write is used to send a command to configure a IrNET channel
+ * before it is open by pppd. The syntax is : "command argument"
+ * Currently there is only two defined commands :
+ *	o name : set the requested IrDA nickname of the IrNET peer.
+ *	o addr : set the requested IrDA address of the IrNET peer.
+ * Note : the code is crude, but effective...
+ */
+static inline ssize_t
+irnet_ctrl_write(irnet_socket *	ap,
+		 const char __user *buf,
+		 size_t		count)
+{
+  char		command[IRNET_MAX_COMMAND];
+  char *	start;		/* Current command being processed */
+  char *	next;		/* Next command to process */
+  int		length;		/* Length of current command */
+
+  DENTER(CTRL_TRACE, "(ap=0x%p, count=%zd)\n", ap, count);
+
+  /* Check for overflow... */
+  DABORT(count >= IRNET_MAX_COMMAND, -ENOMEM,
+	 CTRL_ERROR, "Too much data !!!\n");
+
+  /* Get the data in the driver */
+  if(copy_from_user(command, buf, count))
+    {
+      DERROR(CTRL_ERROR, "Invalid user space pointer.\n");
+      return -EFAULT;
+    }
+
+  /* Safe terminate the string */
+  command[count] = '\0';
+  DEBUG(CTRL_INFO, "Command line received is ``%s'' (%zd).\n",
+	command, count);
+
+  /* Check every commands in the command line */
+  next = command;
+  while(next != NULL)
+    {
+      /* Look at the next command */
+      start = next;
+
+	/* Scrap whitespaces before the command */
+	start = skip_spaces(start);
+
+      /* ',' is our command separator */
+      next = strchr(start, ',');
+      if(next)
+	{
+	  *next = '\0';			/* Terminate command */
+	  length = next - start;	/* Length */
+	  next++;			/* Skip the '\0' */
+	}
+      else
+	length = strlen(start);
+
+      DEBUG(CTRL_INFO, "Found command ``%s'' (%d).\n", start, length);
+
+      /* Check if we recognised one of the known command
+       * We can't use "switch" with strings, so hack with "continue" */
+
+      /* First command : name -> Requested IrDA nickname */
+      if(!strncmp(start, "name", 4))
+	{
+	  /* Copy the name only if is included and not "any" */
+	  if((length > 5) && (strcmp(start + 5, "any")))
+	    {
+	      /* Strip out trailing whitespaces */
+	      while(isspace(start[length - 1]))
+		length--;
+
+	      DABORT(length < 5 || length > NICKNAME_MAX_LEN + 5,
+		     -EINVAL, CTRL_ERROR, "Invalid nickname.\n");
+
+	      /* Copy the name for later reuse */
+	      memcpy(ap->rname, start + 5, length - 5);
+	      ap->rname[length - 5] = '\0';
+	    }
+	  else
+	    ap->rname[0] = '\0';
+	  DEBUG(CTRL_INFO, "Got rname = ``%s''\n", ap->rname);
+
+	  /* Restart the loop */
+	  continue;
+	}
+
+      /* Second command : addr, daddr -> Requested IrDA destination address
+       * Also process : saddr -> Requested IrDA source address */
+      if((!strncmp(start, "addr", 4)) ||
+	 (!strncmp(start, "daddr", 5)) ||
+	 (!strncmp(start, "saddr", 5)))
+	{
+	  __u32		addr = DEV_ADDR_ANY;
+
+	  /* Copy the address only if is included and not "any" */
+	  if((length > 5) && (strcmp(start + 5, "any")))
+	    {
+	      char *	begp = start + 5;
+	      char *	endp;
+
+	      /* Scrap whitespaces before the command */
+	      begp = skip_spaces(begp);
+
+	      /* Convert argument to a number (last arg is the base) */
+	      addr = simple_strtoul(begp, &endp, 16);
+	      /* Has it worked  ? (endp should be start + length) */
+	      DABORT(endp <= (start + 5), -EINVAL,
+		     CTRL_ERROR, "Invalid address.\n");
+	    }
+	  /* Which type of address ? */
+	  if(start[0] == 's')
+	    {
+	      /* Save it */
+	      ap->rsaddr = addr;
+	      DEBUG(CTRL_INFO, "Got rsaddr = %08x\n", ap->rsaddr);
+	    }
+	  else
+	    {
+	      /* Save it */
+	      ap->rdaddr = addr;
+	      DEBUG(CTRL_INFO, "Got rdaddr = %08x\n", ap->rdaddr);
+	    }
+
+	  /* Restart the loop */
+	  continue;
+	}
+
+      /* Other possible command : connect N (number of retries) */
+
+      /* No command matched -> Failed... */
+      DABORT(1, -EINVAL, CTRL_ERROR, "Not a recognised IrNET command.\n");
+    }
+
+  /* Success : we have parsed all commands successfully */
+  return count;
+}
+
+#ifdef INITIAL_DISCOVERY
+/*------------------------------------------------------------------*/
+/*
+ * Function irnet_get_discovery_log (self)
+ *
+ *    Query the content on the discovery log if not done
+ *
+ * This function query the current content of the discovery log
+ * at the startup of the event channel and save it in the internal struct.
+ */
+static void
+irnet_get_discovery_log(irnet_socket *	ap)
+{
+  __u16		mask = irlmp_service_to_hint(S_LAN);
+
+  /* Ask IrLMP for the current discovery log */
+  ap->discoveries = irlmp_get_discoveries(&ap->disco_number, mask,
+					  DISCOVERY_DEFAULT_SLOTS);
+
+  /* Check if the we got some results */
+  if(ap->discoveries == NULL)
+    ap->disco_number = -1;
+
+  DEBUG(CTRL_INFO, "Got the log (0x%p), size is %d\n",
+	ap->discoveries, ap->disco_number);
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Function irnet_read_discovery_log (self, event)
+ *
+ *    Read the content on the discovery log
+ *
+ * This function dump the current content of the discovery log
+ * at the startup of the event channel.
+ * Return 1 if wrote an event on the control channel...
+ *
+ * State of the ap->disco_XXX variables :
+ * Socket creation :  discoveries = NULL ; disco_index = 0 ; disco_number = 0
+ * While reading :    discoveries = ptr  ; disco_index = X ; disco_number = Y
+ * After reading :    discoveries = NULL ; disco_index = Y ; disco_number = -1
+ */
+static inline int
+irnet_read_discovery_log(irnet_socket *ap, char *event, int buf_size)
+{
+  int		done_event = 0;
+
+  DENTER(CTRL_TRACE, "(ap=0x%p, event=0x%p)\n",
+	 ap, event);
+
+  /* Test if we have some work to do or we have already finished */
+  if(ap->disco_number == -1)
+    {
+      DEBUG(CTRL_INFO, "Already done\n");
+      return 0;
+    }
+
+  /* Test if it's the first time and therefore we need to get the log */
+  if(ap->discoveries == NULL)
+    irnet_get_discovery_log(ap);
+
+  /* Check if we have more item to dump */
+  if(ap->disco_index < ap->disco_number)
+    {
+      /* Write an event */
+      snprintf(event, buf_size,
+	       "Found %08x (%s) behind %08x {hints %02X-%02X}\n",
+	       ap->discoveries[ap->disco_index].daddr,
+	       ap->discoveries[ap->disco_index].info,
+	       ap->discoveries[ap->disco_index].saddr,
+	       ap->discoveries[ap->disco_index].hints[0],
+	       ap->discoveries[ap->disco_index].hints[1]);
+      DEBUG(CTRL_INFO, "Writing discovery %d : %s\n",
+	    ap->disco_index, ap->discoveries[ap->disco_index].info);
+
+      /* We have an event */
+      done_event = 1;
+      /* Next discovery */
+      ap->disco_index++;
+    }
+
+  /* Check if we have done the last item */
+  if(ap->disco_index >= ap->disco_number)
+    {
+      /* No more items : remove the log and signal termination */
+      DEBUG(CTRL_INFO, "Cleaning up log (0x%p)\n",
+	    ap->discoveries);
+      if(ap->discoveries != NULL)
+	{
+	  /* Cleanup our copy of the discovery log */
+	  kfree(ap->discoveries);
+	  ap->discoveries = NULL;
+	}
+      ap->disco_number = -1;
+    }
+
+  return done_event;
+}
+#endif /* INITIAL_DISCOVERY */
+
+/*------------------------------------------------------------------*/
+/*
+ * Read is used to get IrNET events
+ */
+static inline ssize_t
+irnet_ctrl_read(irnet_socket *	ap,
+		struct file *	file,
+		char __user *	buf,
+		size_t		count)
+{
+  DECLARE_WAITQUEUE(wait, current);
+  char		event[75];
+  ssize_t	ret = 0;
+
+  DENTER(CTRL_TRACE, "(ap=0x%p, count=%zd)\n", ap, count);
+
+#ifdef INITIAL_DISCOVERY
+  /* Check if we have read the log */
+  if (irnet_read_discovery_log(ap, event, sizeof(event)))
+    {
+      count = min(strlen(event), count);
+      if (copy_to_user(buf, event, count))
+	{
+	  DERROR(CTRL_ERROR, "Invalid user space pointer.\n");
+	  return -EFAULT;
+	}
+
+      DEXIT(CTRL_TRACE, "\n");
+      return count;
+    }
+#endif /* INITIAL_DISCOVERY */
+
+  /* Put ourselves on the wait queue to be woken up */
+  add_wait_queue(&irnet_events.rwait, &wait);
+  set_current_state(TASK_INTERRUPTIBLE);
+  for(;;)
+    {
+      /* If there is unread events */
+      ret = 0;
+      if(ap->event_index != irnet_events.index)
+	break;
+      ret = -EAGAIN;
+      if(file->f_flags & O_NONBLOCK)
+	break;
+      ret = -ERESTARTSYS;
+      if(signal_pending(current))
+	break;
+      /* Yield and wait to be woken up */
+      schedule();
+    }
+  __set_current_state(TASK_RUNNING);
+  remove_wait_queue(&irnet_events.rwait, &wait);
+
+  /* Did we got it ? */
+  if(ret != 0)
+    {
+      /* No, return the error code */
+      DEXIT(CTRL_TRACE, " - ret %zd\n", ret);
+      return ret;
+    }
+
+  /* Which event is it ? */
+  switch(irnet_events.log[ap->event_index].event)
+    {
+    case IRNET_DISCOVER:
+      snprintf(event, sizeof(event),
+	       "Discovered %08x (%s) behind %08x {hints %02X-%02X}\n",
+	       irnet_events.log[ap->event_index].daddr,
+	       irnet_events.log[ap->event_index].name,
+	       irnet_events.log[ap->event_index].saddr,
+	       irnet_events.log[ap->event_index].hints.byte[0],
+	       irnet_events.log[ap->event_index].hints.byte[1]);
+      break;
+    case IRNET_EXPIRE:
+      snprintf(event, sizeof(event),
+	       "Expired %08x (%s) behind %08x {hints %02X-%02X}\n",
+	       irnet_events.log[ap->event_index].daddr,
+	       irnet_events.log[ap->event_index].name,
+	       irnet_events.log[ap->event_index].saddr,
+	       irnet_events.log[ap->event_index].hints.byte[0],
+	       irnet_events.log[ap->event_index].hints.byte[1]);
+      break;
+    case IRNET_CONNECT_TO:
+      snprintf(event, sizeof(event), "Connected to %08x (%s) on ppp%d\n",
+	       irnet_events.log[ap->event_index].daddr,
+	       irnet_events.log[ap->event_index].name,
+	       irnet_events.log[ap->event_index].unit);
+      break;
+    case IRNET_CONNECT_FROM:
+      snprintf(event, sizeof(event), "Connection from %08x (%s) on ppp%d\n",
+	       irnet_events.log[ap->event_index].daddr,
+	       irnet_events.log[ap->event_index].name,
+	       irnet_events.log[ap->event_index].unit);
+      break;
+    case IRNET_REQUEST_FROM:
+      snprintf(event, sizeof(event), "Request from %08x (%s) behind %08x\n",
+	       irnet_events.log[ap->event_index].daddr,
+	       irnet_events.log[ap->event_index].name,
+	       irnet_events.log[ap->event_index].saddr);
+      break;
+    case IRNET_NOANSWER_FROM:
+      snprintf(event, sizeof(event), "No-answer from %08x (%s) on ppp%d\n",
+	       irnet_events.log[ap->event_index].daddr,
+	       irnet_events.log[ap->event_index].name,
+	       irnet_events.log[ap->event_index].unit);
+      break;
+    case IRNET_BLOCKED_LINK:
+      snprintf(event, sizeof(event), "Blocked link with %08x (%s) on ppp%d\n",
+	       irnet_events.log[ap->event_index].daddr,
+	       irnet_events.log[ap->event_index].name,
+	       irnet_events.log[ap->event_index].unit);
+      break;
+    case IRNET_DISCONNECT_FROM:
+      snprintf(event, sizeof(event), "Disconnection from %08x (%s) on ppp%d\n",
+	       irnet_events.log[ap->event_index].daddr,
+	       irnet_events.log[ap->event_index].name,
+	       irnet_events.log[ap->event_index].unit);
+      break;
+    case IRNET_DISCONNECT_TO:
+      snprintf(event, sizeof(event), "Disconnected to %08x (%s)\n",
+	       irnet_events.log[ap->event_index].daddr,
+	       irnet_events.log[ap->event_index].name);
+      break;
+    default:
+      snprintf(event, sizeof(event), "Bug\n");
+    }
+  /* Increment our event index */
+  ap->event_index = (ap->event_index + 1) % IRNET_MAX_EVENTS;
+
+  DEBUG(CTRL_INFO, "Event is :%s", event);
+
+  count = min(strlen(event), count);
+  if (copy_to_user(buf, event, count))
+    {
+      DERROR(CTRL_ERROR, "Invalid user space pointer.\n");
+      return -EFAULT;
+    }
+
+  DEXIT(CTRL_TRACE, "\n");
+  return count;
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Poll : called when someone do a select on /dev/irnet.
+ * Just check if there are new events...
+ */
+static inline unsigned int
+irnet_ctrl_poll(irnet_socket *	ap,
+		struct file *	file,
+		poll_table *	wait)
+{
+  unsigned int mask;
+
+  DENTER(CTRL_TRACE, "(ap=0x%p)\n", ap);
+
+  poll_wait(file, &irnet_events.rwait, wait);
+  mask = POLLOUT | POLLWRNORM;
+  /* If there is unread events */
+  if(ap->event_index != irnet_events.index)
+    mask |= POLLIN | POLLRDNORM;
+#ifdef INITIAL_DISCOVERY
+  if(ap->disco_number != -1)
+    {
+      /* Test if it's the first time and therefore we need to get the log */
+      if(ap->discoveries == NULL)
+	irnet_get_discovery_log(ap);
+      /* Recheck */
+      if(ap->disco_number != -1)
+	mask |= POLLIN | POLLRDNORM;
+    }
+#endif /* INITIAL_DISCOVERY */
+
+  DEXIT(CTRL_TRACE, " - mask=0x%X\n", mask);
+  return mask;
+}
+
+
+/*********************** FILESYSTEM CALLBACKS ***********************/
+/*
+ * Implement the usual open, read, write functions that will be called
+ * by the file system when some action is performed on /dev/irnet.
+ * Most of those actions will in fact be performed by "pppd" or
+ * the control channel, we just act as a redirector...
+ */
+
+/*------------------------------------------------------------------*/
+/*
+ * Open : when somebody open /dev/irnet
+ * We basically create a new instance of irnet and initialise it.
+ */
+static int
+dev_irnet_open(struct inode *	inode,
+	       struct file *	file)
+{
+  struct irnet_socket *	ap;
+  int			err;
+
+  DENTER(FS_TRACE, "(file=0x%p)\n", file);
+
+#ifdef SECURE_DEVIRNET
+  /* This could (should?) be enforced by the permissions on /dev/irnet. */
+  if(!capable(CAP_NET_ADMIN))
+    return -EPERM;
+#endif /* SECURE_DEVIRNET */
+
+  /* Allocate a private structure for this IrNET instance */
+  ap = kzalloc(sizeof(*ap), GFP_KERNEL);
+  DABORT(ap == NULL, -ENOMEM, FS_ERROR, "Can't allocate struct irnet...\n");
+
+  /* initialize the irnet structure */
+  ap->file = file;
+
+  /* PPP channel setup */
+  ap->ppp_open = 0;
+  ap->chan.private = ap;
+  ap->chan.ops = &irnet_ppp_ops;
+  ap->chan.mtu = (2048 - TTP_MAX_HEADER - 2 - PPP_HDRLEN);
+  ap->chan.hdrlen = 2 + TTP_MAX_HEADER;		/* for A/C + Max IrDA hdr */
+  /* PPP parameters */
+  ap->mru = (2048 - TTP_MAX_HEADER - 2 - PPP_HDRLEN);
+  ap->xaccm[0] = ~0U;
+  ap->xaccm[3] = 0x60000000U;
+  ap->raccm = ~0U;
+
+  /* Setup the IrDA part... */
+  err = irda_irnet_create(ap);
+  if(err)
+    {
+      DERROR(FS_ERROR, "Can't setup IrDA link...\n");
+      kfree(ap);
+
+      return err;
+    }
+
+  /* For the control channel */
+  ap->event_index = irnet_events.index;	/* Cancel all past events */
+
+  mutex_init(&ap->lock);
+
+  /* Put our stuff where we will be able to find it later */
+  file->private_data = ap;
+
+  DEXIT(FS_TRACE, " - ap=0x%p\n", ap);
+
+  return 0;
+}
+
+
+/*------------------------------------------------------------------*/
+/*
+ * Close : when somebody close /dev/irnet
+ * Destroy the instance of /dev/irnet
+ */
+static int
+dev_irnet_close(struct inode *	inode,
+		struct file *	file)
+{
+  irnet_socket *	ap = file->private_data;
+
+  DENTER(FS_TRACE, "(file=0x%p, ap=0x%p)\n",
+	 file, ap);
+  DABORT(ap == NULL, 0, FS_ERROR, "ap is NULL !!!\n");
+
+  /* Detach ourselves */
+  file->private_data = NULL;
+
+  /* Close IrDA stuff */
+  irda_irnet_destroy(ap);
+
+  /* Disconnect from the generic PPP layer if not already done */
+  if(ap->ppp_open)
+    {
+      DERROR(FS_ERROR, "Channel still registered - deregistering !\n");
+      ap->ppp_open = 0;
+      ppp_unregister_channel(&ap->chan);
+    }
+
+  kfree(ap);
+
+  DEXIT(FS_TRACE, "\n");
+  return 0;
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Write does nothing.
+ * (we receive packet from ppp_generic through ppp_irnet_send())
+ */
+static ssize_t
+dev_irnet_write(struct file *	file,
+		const char __user *buf,
+		size_t		count,
+		loff_t *	ppos)
+{
+  irnet_socket *	ap = file->private_data;
+
+  DPASS(FS_TRACE, "(file=0x%p, ap=0x%p, count=%zd)\n",
+	file, ap, count);
+  DABORT(ap == NULL, -ENXIO, FS_ERROR, "ap is NULL !!!\n");
+
+  /* If we are connected to ppp_generic, let it handle the job */
+  if(ap->ppp_open)
+    return -EAGAIN;
+  else
+    return irnet_ctrl_write(ap, buf, count);
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Read doesn't do much either.
+ * (pppd poll us, but ultimately reads through /dev/ppp)
+ */
+static ssize_t
+dev_irnet_read(struct file *	file,
+	       char __user *	buf,
+	       size_t		count,
+	       loff_t *		ppos)
+{
+  irnet_socket *	ap = file->private_data;
+
+  DPASS(FS_TRACE, "(file=0x%p, ap=0x%p, count=%zd)\n",
+	file, ap, count);
+  DABORT(ap == NULL, -ENXIO, FS_ERROR, "ap is NULL !!!\n");
+
+  /* If we are connected to ppp_generic, let it handle the job */
+  if(ap->ppp_open)
+    return -EAGAIN;
+  else
+    return irnet_ctrl_read(ap, file, buf, count);
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Poll : called when someone do a select on /dev/irnet
+ */
+static unsigned int
+dev_irnet_poll(struct file *	file,
+	       poll_table *	wait)
+{
+  irnet_socket *	ap = file->private_data;
+  unsigned int		mask;
+
+  DENTER(FS_TRACE, "(file=0x%p, ap=0x%p)\n",
+	 file, ap);
+
+  mask = POLLOUT | POLLWRNORM;
+  DABORT(ap == NULL, mask, FS_ERROR, "ap is NULL !!!\n");
+
+  /* If we are connected to ppp_generic, let it handle the job */
+  if(!ap->ppp_open)
+    mask |= irnet_ctrl_poll(ap, file, wait);
+
+  DEXIT(FS_TRACE, " - mask=0x%X\n", mask);
+  return mask;
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * IOCtl : Called when someone does some ioctls on /dev/irnet
+ * This is the way pppd configure us and control us while the PPP
+ * instance is active.
+ */
+static long
+dev_irnet_ioctl(
+		struct file *	file,
+		unsigned int	cmd,
+		unsigned long	arg)
+{
+  irnet_socket *	ap = file->private_data;
+  int			err;
+  int			val;
+  void __user *argp = (void __user *)arg;
+
+  DENTER(FS_TRACE, "(file=0x%p, ap=0x%p, cmd=0x%X)\n",
+	 file, ap, cmd);
+
+  /* Basic checks... */
+  DASSERT(ap != NULL, -ENXIO, PPP_ERROR, "ap is NULL...\n");
+#ifdef SECURE_DEVIRNET
+  if(!capable(CAP_NET_ADMIN))
+    return -EPERM;
+#endif /* SECURE_DEVIRNET */
+
+  err = -EFAULT;
+  switch(cmd)
+    {
+      /* Set discipline (should be N_SYNC_PPP or N_TTY) */
+    case TIOCSETD:
+      if(get_user(val, (int __user *)argp))
+	break;
+      if((val == N_SYNC_PPP) || (val == N_PPP))
+	{
+	  DEBUG(FS_INFO, "Entering PPP discipline.\n");
+	  /* PPP channel setup (ap->chan in configured in dev_irnet_open())*/
+	  if (mutex_lock_interruptible(&ap->lock))
+		  return -EINTR;
+
+	  err = ppp_register_channel(&ap->chan);
+	  if(err == 0)
+	    {
+	      /* Our ppp side is active */
+	      ap->ppp_open = 1;
+
+	      DEBUG(FS_INFO, "Trying to establish a connection.\n");
+	      /* Setup the IrDA link now - may fail... */
+	      irda_irnet_connect(ap);
+	    }
+	  else
+	    DERROR(FS_ERROR, "Can't setup PPP channel...\n");
+
+          mutex_unlock(&ap->lock);
+	}
+      else
+	{
+	  /* In theory, should be N_TTY */
+	  DEBUG(FS_INFO, "Exiting PPP discipline.\n");
+	  /* Disconnect from the generic PPP layer */
+	  if (mutex_lock_interruptible(&ap->lock))
+		  return -EINTR;
+
+	  if(ap->ppp_open)
+	    {
+	      ap->ppp_open = 0;
+	      ppp_unregister_channel(&ap->chan);
+	    }
+	  else
+	    DERROR(FS_ERROR, "Channel not registered !\n");
+	  err = 0;
+
+	  mutex_unlock(&ap->lock);
+	}
+      break;
+
+      /* Query PPP channel and unit number */
+    case PPPIOCGCHAN:
+      if (mutex_lock_interruptible(&ap->lock))
+	      return -EINTR;
+
+      if(ap->ppp_open && !put_user(ppp_channel_index(&ap->chan),
+						(int __user *)argp))
+	err = 0;
+
+      mutex_unlock(&ap->lock);
+      break;
+    case PPPIOCGUNIT:
+      if (mutex_lock_interruptible(&ap->lock))
+	      return -EINTR;
+
+      if(ap->ppp_open && !put_user(ppp_unit_number(&ap->chan),
+						(int __user *)argp))
+        err = 0;
+
+      mutex_unlock(&ap->lock);
+      break;
+
+      /* All these ioctls can be passed both directly and from ppp_generic,
+       * so we just deal with them in one place...
+       */
+    case PPPIOCGFLAGS:
+    case PPPIOCSFLAGS:
+    case PPPIOCGASYNCMAP:
+    case PPPIOCSASYNCMAP:
+    case PPPIOCGRASYNCMAP:
+    case PPPIOCSRASYNCMAP:
+    case PPPIOCGXASYNCMAP:
+    case PPPIOCSXASYNCMAP:
+    case PPPIOCGMRU:
+    case PPPIOCSMRU:
+      DEBUG(FS_INFO, "Standard PPP ioctl.\n");
+      if(!capable(CAP_NET_ADMIN))
+	err = -EPERM;
+      else {
+	if (mutex_lock_interruptible(&ap->lock))
+	      return -EINTR;
+
+	err = ppp_irnet_ioctl(&ap->chan, cmd, arg);
+
+	mutex_unlock(&ap->lock);
+      }
+      break;
+
+      /* TTY IOCTLs : Pretend that we are a tty, to keep pppd happy */
+      /* Get termios */
+    case TCGETS:
+      DEBUG(FS_INFO, "Get termios.\n");
+      if (mutex_lock_interruptible(&ap->lock))
+	      return -EINTR;
+
+#ifndef TCGETS2
+      if(!kernel_termios_to_user_termios((struct termios __user *)argp, &ap->termios))
+	err = 0;
+#else
+      if(kernel_termios_to_user_termios_1((struct termios __user *)argp, &ap->termios))
+	err = 0;
+#endif
+
+      mutex_unlock(&ap->lock);
+      break;
+      /* Set termios */
+    case TCSETSF:
+      DEBUG(FS_INFO, "Set termios.\n");
+      if (mutex_lock_interruptible(&ap->lock))
+	      return -EINTR;
+
+#ifndef TCGETS2
+      if(!user_termios_to_kernel_termios(&ap->termios, (struct termios __user *)argp))
+	err = 0;
+#else
+      if(!user_termios_to_kernel_termios_1(&ap->termios, (struct termios __user *)argp))
+	err = 0;
+#endif
+
+      mutex_unlock(&ap->lock);
+      break;
+
+      /* Set DTR/RTS */
+    case TIOCMBIS:
+    case TIOCMBIC:
+      /* Set exclusive/non-exclusive mode */
+    case TIOCEXCL:
+    case TIOCNXCL:
+      DEBUG(FS_INFO, "TTY compatibility.\n");
+      err = 0;
+      break;
+
+    case TCGETA:
+      DEBUG(FS_INFO, "TCGETA\n");
+      break;
+
+    case TCFLSH:
+      DEBUG(FS_INFO, "TCFLSH\n");
+      /* Note : this will flush buffers in PPP, so it *must* be done
+       * We should also worry that we don't accept junk here and that
+       * we get rid of our own buffers */
+#ifdef FLUSH_TO_PPP
+      if (mutex_lock_interruptible(&ap->lock))
+	      return -EINTR;
+      ppp_output_wakeup(&ap->chan);
+      mutex_unlock(&ap->lock);
+#endif /* FLUSH_TO_PPP */
+      err = 0;
+      break;
+
+    case FIONREAD:
+      DEBUG(FS_INFO, "FIONREAD\n");
+      val = 0;
+      if(put_user(val, (int __user *)argp))
+	break;
+      err = 0;
+      break;
+
+    default:
+      DERROR(FS_ERROR, "Unsupported ioctl (0x%X)\n", cmd);
+      err = -ENOTTY;
+    }
+
+  DEXIT(FS_TRACE, " - err = 0x%X\n", err);
+  return err;
+}
+
+/************************** PPP CALLBACKS **************************/
+/*
+ * This are the functions that the generic PPP driver in the kernel
+ * will call to communicate to us.
+ */
+
+/*------------------------------------------------------------------*/
+/*
+ * Prepare the ppp frame for transmission over the IrDA socket.
+ * We make sure that the header space is enough, and we change ppp header
+ * according to flags passed by pppd.
+ * This is not a callback, but just a helper function used in ppp_irnet_send()
+ */
+static inline struct sk_buff *
+irnet_prepare_skb(irnet_socket *	ap,
+		  struct sk_buff *	skb)
+{
+  unsigned char *	data;
+  int			proto;		/* PPP protocol */
+  int			islcp;		/* Protocol == LCP */
+  int			needaddr;	/* Need PPP address */
+
+  DENTER(PPP_TRACE, "(ap=0x%p, skb=0x%p)\n",
+	 ap, skb);
+
+  /* Extract PPP protocol from the frame */
+  data  = skb->data;
+  proto = (data[0] << 8) + data[1];
+
+  /* LCP packets with codes between 1 (configure-request)
+   * and 7 (code-reject) must be sent as though no options
+   * have been negotiated. */
+  islcp = (proto == PPP_LCP) && (1 <= data[2]) && (data[2] <= 7);
+
+  /* compress protocol field if option enabled */
+  if((data[0] == 0) && (ap->flags & SC_COMP_PROT) && (!islcp))
+    skb_pull(skb,1);
+
+  /* Check if we need address/control fields */
+  needaddr = 2*((ap->flags & SC_COMP_AC) == 0 || islcp);
+
+  /* Is the skb headroom large enough to contain all IrDA-headers? */
+  if((skb_headroom(skb) < (ap->max_header_size + needaddr)) ||
+      (skb_shared(skb)))
+    {
+      struct sk_buff *	new_skb;
+
+      DEBUG(PPP_INFO, "Reallocating skb\n");
+
+      /* Create a new skb */
+      new_skb = skb_realloc_headroom(skb, ap->max_header_size + needaddr);
+
+      /* We have to free the original skb anyway */
+      dev_kfree_skb(skb);
+
+      /* Did the realloc succeed ? */
+      DABORT(new_skb == NULL, NULL, PPP_ERROR, "Could not realloc skb\n");
+
+      /* Use the new skb instead */
+      skb = new_skb;
+    }
+
+  /* prepend address/control fields if necessary */
+  if(needaddr)
+    {
+      skb_push(skb, 2);
+      skb->data[0] = PPP_ALLSTATIONS;
+      skb->data[1] = PPP_UI;
+    }
+
+  DEXIT(PPP_TRACE, "\n");
+
+  return skb;
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Send a packet to the peer over the IrTTP connection.
+ * Returns 1 iff the packet was accepted.
+ * Returns 0 iff packet was not consumed.
+ * If the packet was not accepted, we will call ppp_output_wakeup
+ * at some later time to reactivate flow control in ppp_generic.
+ */
+static int
+ppp_irnet_send(struct ppp_channel *	chan,
+	       struct sk_buff *		skb)
+{
+  irnet_socket *	self = (struct irnet_socket *) chan->private;
+  int			ret;
+
+  DENTER(PPP_TRACE, "(channel=0x%p, ap/self=0x%p)\n",
+	 chan, self);
+
+  /* Check if things are somewhat valid... */
+  DASSERT(self != NULL, 0, PPP_ERROR, "Self is NULL !!!\n");
+
+  /* Check if we are connected */
+  if(!(test_bit(0, &self->ttp_open)))
+    {
+#ifdef CONNECT_IN_SEND
+      /* Let's try to connect one more time... */
+      /* Note : we won't be connected after this call, but we should be
+       * ready for next packet... */
+      /* If we are already connecting, this will fail */
+      irda_irnet_connect(self);
+#endif /* CONNECT_IN_SEND */
+
+      DEBUG(PPP_INFO, "IrTTP not ready ! (%ld-%ld)\n",
+	    self->ttp_open, self->ttp_connect);
+
+      /* Note : we can either drop the packet or block the packet.
+       *
+       * Blocking the packet allow us a better connection time,
+       * because by calling ppp_output_wakeup() we can have
+       * ppp_generic resending the LCP request immediately to us,
+       * rather than waiting for one of pppd periodic transmission of
+       * LCP request.
+       *
+       * On the other hand, if we block all packet, all those periodic
+       * transmissions of pppd accumulate in ppp_generic, creating a
+       * backlog of LCP request. When we eventually connect later on,
+       * we have to transmit all this backlog before we can connect
+       * proper (if we don't timeout before).
+       *
+       * The current strategy is as follow :
+       * While we are attempting to connect, we block packets to get
+       * a better connection time.
+       * If we fail to connect, we drain the queue and start dropping packets
+       */
+#ifdef BLOCK_WHEN_CONNECT
+      /* If we are attempting to connect */
+      if(test_bit(0, &self->ttp_connect))
+	{
+	  /* Blocking packet, ppp_generic will retry later */
+	  return 0;
+	}
+#endif /* BLOCK_WHEN_CONNECT */
+
+      /* Dropping packet, pppd will retry later */
+      dev_kfree_skb(skb);
+      return 1;
+    }
+
+  /* Check if the queue can accept any packet, otherwise block */
+  if(self->tx_flow != FLOW_START)
+    DRETURN(0, PPP_INFO, "IrTTP queue full (%d skbs)...\n",
+	    skb_queue_len(&self->tsap->tx_queue));
+
+  /* Prepare ppp frame for transmission */
+  skb = irnet_prepare_skb(self, skb);
+  DABORT(skb == NULL, 1, PPP_ERROR, "Prepare skb for Tx failed.\n");
+
+  /* Send the packet to IrTTP */
+  ret = irttp_data_request(self->tsap, skb);
+  if(ret < 0)
+    {
+      /*
+       * > IrTTPs tx queue is full, so we just have to
+       * > drop the frame! You might think that we should
+       * > just return -1 and don't deallocate the frame,
+       * > but that is dangerous since it's possible that
+       * > we have replaced the original skb with a new
+       * > one with larger headroom, and that would really
+       * > confuse do_dev_queue_xmit() in dev.c! I have
+       * > tried :-) DB
+       * Correction : we verify the flow control above (self->tx_flow),
+       * so we come here only if IrTTP doesn't like the packet (empty,
+       * too large, IrTTP not connected). In those rare cases, it's ok
+       * to drop it, we don't want to see it here again...
+       * Jean II
+       */
+      DERROR(PPP_ERROR, "IrTTP doesn't like this packet !!! (0x%X)\n", ret);
+      /* irttp_data_request already free the packet */
+    }
+
+  DEXIT(PPP_TRACE, "\n");
+  return 1;	/* Packet has been consumed */
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Take care of the ioctls that ppp_generic doesn't want to deal with...
+ * Note : we are also called from dev_irnet_ioctl().
+ */
+static int
+ppp_irnet_ioctl(struct ppp_channel *	chan,
+		unsigned int		cmd,
+		unsigned long		arg)
+{
+  irnet_socket *	ap = (struct irnet_socket *) chan->private;
+  int			err;
+  int			val;
+  u32			accm[8];
+  void __user *argp = (void __user *)arg;
+
+  DENTER(PPP_TRACE, "(channel=0x%p, ap=0x%p, cmd=0x%X)\n",
+	 chan, ap, cmd);
+
+  /* Basic checks... */
+  DASSERT(ap != NULL, -ENXIO, PPP_ERROR, "ap is NULL...\n");
+
+  err = -EFAULT;
+  switch(cmd)
+    {
+      /* PPP flags */
+    case PPPIOCGFLAGS:
+      val = ap->flags | ap->rbits;
+      if(put_user(val, (int __user *) argp))
+	break;
+      err = 0;
+      break;
+    case PPPIOCSFLAGS:
+      if(get_user(val, (int __user *) argp))
+	break;
+      ap->flags = val & ~SC_RCV_BITS;
+      ap->rbits = val & SC_RCV_BITS;
+      err = 0;
+      break;
+
+      /* Async map stuff - all dummy to please pppd */
+    case PPPIOCGASYNCMAP:
+      if(put_user(ap->xaccm[0], (u32 __user *) argp))
+	break;
+      err = 0;
+      break;
+    case PPPIOCSASYNCMAP:
+      if(get_user(ap->xaccm[0], (u32 __user *) argp))
+	break;
+      err = 0;
+      break;
+    case PPPIOCGRASYNCMAP:
+      if(put_user(ap->raccm, (u32 __user *) argp))
+	break;
+      err = 0;
+      break;
+    case PPPIOCSRASYNCMAP:
+      if(get_user(ap->raccm, (u32 __user *) argp))
+	break;
+      err = 0;
+      break;
+    case PPPIOCGXASYNCMAP:
+      if(copy_to_user(argp, ap->xaccm, sizeof(ap->xaccm)))
+	break;
+      err = 0;
+      break;
+    case PPPIOCSXASYNCMAP:
+      if(copy_from_user(accm, argp, sizeof(accm)))
+	break;
+      accm[2] &= ~0x40000000U;		/* can't escape 0x5e */
+      accm[3] |= 0x60000000U;		/* must escape 0x7d, 0x7e */
+      memcpy(ap->xaccm, accm, sizeof(ap->xaccm));
+      err = 0;
+      break;
+
+      /* Max PPP frame size */
+    case PPPIOCGMRU:
+      if(put_user(ap->mru, (int __user *) argp))
+	break;
+      err = 0;
+      break;
+    case PPPIOCSMRU:
+      if(get_user(val, (int __user *) argp))
+	break;
+      if(val < PPP_MRU)
+	val = PPP_MRU;
+      ap->mru = val;
+      err = 0;
+      break;
+
+    default:
+      DEBUG(PPP_INFO, "Unsupported ioctl (0x%X)\n", cmd);
+      err = -ENOIOCTLCMD;
+    }
+
+  DEXIT(PPP_TRACE, " - err = 0x%X\n", err);
+  return err;
+}
+
+/************************** INITIALISATION **************************/
+/*
+ * Module initialisation and all that jazz...
+ */
+
+/*------------------------------------------------------------------*/
+/*
+ * Hook our device callbacks in the filesystem, to connect our code
+ * to /dev/irnet
+ */
+static inline int __init
+ppp_irnet_init(void)
+{
+  int err = 0;
+
+  DENTER(MODULE_TRACE, "()\n");
+
+  /* Allocate ourselves as a minor in the misc range */
+  err = misc_register(&irnet_misc_device);
+
+  DEXIT(MODULE_TRACE, "\n");
+  return err;
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Cleanup at exit...
+ */
+static inline void __exit
+ppp_irnet_cleanup(void)
+{
+  DENTER(MODULE_TRACE, "()\n");
+
+  /* De-allocate /dev/irnet minor in misc range */
+  misc_deregister(&irnet_misc_device);
+
+  DEXIT(MODULE_TRACE, "\n");
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Module main entry point
+ */
+static int __init
+irnet_init(void)
+{
+  int err;
+
+  /* Initialise both parts... */
+  err = irda_irnet_init();
+  if(!err)
+    err = ppp_irnet_init();
+  return err;
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Module exit
+ */
+static void __exit
+irnet_cleanup(void)
+{
+  irda_irnet_cleanup();
+  ppp_irnet_cleanup();
+}
+
+/*------------------------------------------------------------------*/
+/*
+ * Module magic
+ */
+module_init(irnet_init);
+module_exit(irnet_cleanup);
+MODULE_AUTHOR("Jean Tourrilhes <jt@hpl.hp.com>");
+MODULE_DESCRIPTION("IrNET : Synchronous PPP over IrDA");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS_CHARDEV(10, 187);
diff --git a/drivers/staging/irda/net/irnet/irnet_ppp.h b/drivers/staging/irda/net/irnet/irnet_ppp.h
new file mode 100644
index 000000000000..32061442cc8e
--- /dev/null
+++ b/drivers/staging/irda/net/irnet/irnet_ppp.h
@@ -0,0 +1,116 @@
+/*
+ *	IrNET protocol module : Synchronous PPP over an IrDA socket.
+ *
+ *		Jean II - HPL `00 - <jt@hpl.hp.com>
+ *
+ * This file contains all definitions and declarations necessary for the
+ * PPP part of the IrNET module.
+ * This file is a private header, so other modules don't want to know
+ * what's in there...
+ */
+
+#ifndef IRNET_PPP_H
+#define IRNET_PPP_H
+
+/***************************** INCLUDES *****************************/
+
+#include "irnet.h"		/* Module global include */
+#include <linux/miscdevice.h>
+
+/************************ CONSTANTS & MACROS ************************/
+
+/* IrNET control channel stuff */
+#define IRNET_MAX_COMMAND	256	/* Max length of a command line */
+
+/* PPP hardcore stuff */
+
+/* Bits in rbits (PPP flags in irnet struct) */
+#define SC_RCV_BITS	(SC_RCV_B7_1|SC_RCV_B7_0|SC_RCV_ODDP|SC_RCV_EVNP)
+
+/* Bit numbers in busy */
+#define XMIT_BUSY	0
+#define RECV_BUSY	1
+#define XMIT_WAKEUP	2
+#define XMIT_FULL	3
+
+/* Queue management */
+#define PPPSYNC_MAX_RQLEN	32	/* arbitrary */
+
+/****************************** TYPES ******************************/
+
+
+/**************************** PROTOTYPES ****************************/
+
+/* ----------------------- CONTROL CHANNEL ----------------------- */
+static inline ssize_t
+	irnet_ctrl_write(irnet_socket *,
+			 const char *,
+			 size_t);
+static inline ssize_t
+	irnet_ctrl_read(irnet_socket *,
+			struct file *,
+			char *,
+			size_t);
+static inline unsigned int
+	irnet_ctrl_poll(irnet_socket *,
+			struct file *,
+			poll_table *);
+/* ----------------------- CHARACTER DEVICE ----------------------- */
+static int
+	dev_irnet_open(struct inode *,	/* fs callback : open */
+		       struct file *),
+	dev_irnet_close(struct inode *,
+			struct file *);
+static ssize_t
+	dev_irnet_write(struct file *,
+			const char __user *,
+			size_t,
+			loff_t *),
+	dev_irnet_read(struct file *,
+		       char __user *,
+		       size_t,
+		       loff_t *);
+static unsigned int
+	dev_irnet_poll(struct file *,
+		       poll_table *);
+static long
+	dev_irnet_ioctl(struct file *,
+			unsigned int,
+			unsigned long);
+/* ------------------------ PPP INTERFACE ------------------------ */
+static inline struct sk_buff *
+	irnet_prepare_skb(irnet_socket *,
+			  struct sk_buff *);
+static int
+	ppp_irnet_send(struct ppp_channel *,
+		      struct sk_buff *);
+static int
+	ppp_irnet_ioctl(struct ppp_channel *,
+			unsigned int,
+			unsigned long);
+
+/**************************** VARIABLES ****************************/
+
+/* Filesystem callbacks (to call us) */
+static const struct file_operations irnet_device_fops =
+{
+	.owner		= THIS_MODULE,
+	.read		= dev_irnet_read,
+	.write		= dev_irnet_write,
+	.poll		= dev_irnet_poll,
+	.unlocked_ioctl	= dev_irnet_ioctl,
+	.open		= dev_irnet_open,
+	.release	= dev_irnet_close,
+	.llseek		= noop_llseek,
+  /* Also : llseek, readdir, mmap, flush, fsync, fasync, lock, readv, writev */
+};
+
+/* Structure so that the misc major (drivers/char/misc.c) take care of us... */
+static struct miscdevice irnet_misc_device =
+{
+	.minor = IRNET_MINOR,
+	.name = "irnet",
+	.fops = &irnet_device_fops
+};
+
+#endif /* IRNET_PPP_H */
diff --git a/drivers/staging/irda/net/irnetlink.c b/drivers/staging/irda/net/irnetlink.c
new file mode 100644
index 000000000000..7fc340e574cf
--- /dev/null
+++ b/drivers/staging/irda/net/irnetlink.c
@@ -0,0 +1,162 @@
+/*
+ * IrDA netlink layer, for stack configuration.
+ *
+ * Copyright (c) 2007 Samuel Ortiz <samuel@sortiz.org>
+ *
+ * Partly based on the 802.11 nelink implementation
+ * (see net/wireless/nl80211.c) which is:
+ * Copyright 2006 Johannes Berg <johannes@sipsolutions.net>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#include <linux/socket.h>
+#include <linux/irda.h>
+#include <linux/gfp.h>
+#include <net/net_namespace.h>
+#include <net/sock.h>
+#include <net/irda/irda.h>
+#include <net/irda/irlap.h>
+#include <net/genetlink.h>
+
+
+
+static struct genl_family irda_nl_family;
+
+static struct net_device * ifname_to_netdev(struct net *net, struct genl_info *info)
+{
+	char * ifname;
+
+	if (!info->attrs[IRDA_NL_ATTR_IFNAME])
+		return NULL;
+
+	ifname = nla_data(info->attrs[IRDA_NL_ATTR_IFNAME]);
+
+	pr_debug("%s(): Looking for %s\n", __func__, ifname);
+
+	return dev_get_by_name(net, ifname);
+}
+
+static int irda_nl_set_mode(struct sk_buff *skb, struct genl_info *info)
+{
+	struct net_device * dev;
+	struct irlap_cb * irlap;
+	u32 mode;
+
+	if (!info->attrs[IRDA_NL_ATTR_MODE])
+		return -EINVAL;
+
+	mode = nla_get_u32(info->attrs[IRDA_NL_ATTR_MODE]);
+
+	pr_debug("%s(): Switching to mode: %d\n", __func__, mode);
+
+	dev = ifname_to_netdev(&init_net, info);
+	if (!dev)
+		return -ENODEV;
+
+	irlap = (struct irlap_cb *)dev->atalk_ptr;
+	if (!irlap) {
+		dev_put(dev);
+		return -ENODEV;
+	}
+
+	irlap->mode = mode;
+
+	dev_put(dev);
+
+	return 0;
+}
+
+static int irda_nl_get_mode(struct sk_buff *skb, struct genl_info *info)
+{
+	struct net_device * dev;
+	struct irlap_cb * irlap;
+	struct sk_buff *msg;
+	void *hdr;
+	int ret = -ENOBUFS;
+
+	dev = ifname_to_netdev(&init_net, info);
+	if (!dev)
+		return -ENODEV;
+
+	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+	if (!msg) {
+		dev_put(dev);
+		return -ENOMEM;
+	}
+
+	irlap = (struct irlap_cb *)dev->atalk_ptr;
+	if (!irlap) {
+		ret = -ENODEV;
+		goto err_out;
+	}
+
+	hdr = genlmsg_put(msg, info->snd_portid, info->snd_seq,
+			  &irda_nl_family, 0,  IRDA_NL_CMD_GET_MODE);
+	if (hdr == NULL) {
+		ret = -EMSGSIZE;
+		goto err_out;
+	}
+
+	if(nla_put_string(msg, IRDA_NL_ATTR_IFNAME,
+			  dev->name))
+		goto err_out;
+
+	if(nla_put_u32(msg, IRDA_NL_ATTR_MODE, irlap->mode))
+		goto err_out;
+
+	genlmsg_end(msg, hdr);
+
+	return genlmsg_reply(msg, info);
+
+ err_out:
+	nlmsg_free(msg);
+	dev_put(dev);
+
+	return ret;
+}
+
+static const struct nla_policy irda_nl_policy[IRDA_NL_ATTR_MAX + 1] = {
+	[IRDA_NL_ATTR_IFNAME] = { .type = NLA_NUL_STRING,
+				  .len = IFNAMSIZ-1 },
+	[IRDA_NL_ATTR_MODE] = { .type = NLA_U32 },
+};
+
+static const struct genl_ops irda_nl_ops[] = {
+	{
+		.cmd = IRDA_NL_CMD_SET_MODE,
+		.doit = irda_nl_set_mode,
+		.policy = irda_nl_policy,
+		.flags = GENL_ADMIN_PERM,
+	},
+	{
+		.cmd = IRDA_NL_CMD_GET_MODE,
+		.doit = irda_nl_get_mode,
+		.policy = irda_nl_policy,
+		/* can be retrieved by unprivileged users */
+	},
+
+};
+
+static struct genl_family irda_nl_family __ro_after_init = {
+	.name = IRDA_NL_NAME,
+	.hdrsize = 0,
+	.version = IRDA_NL_VERSION,
+	.maxattr = IRDA_NL_CMD_MAX,
+	.module = THIS_MODULE,
+	.ops = irda_nl_ops,
+	.n_ops = ARRAY_SIZE(irda_nl_ops),
+};
+
+int __init irda_nl_register(void)
+{
+	return genl_register_family(&irda_nl_family);
+}
+
+void irda_nl_unregister(void)
+{
+	genl_unregister_family(&irda_nl_family);
+}
diff --git a/drivers/staging/irda/net/irproc.c b/drivers/staging/irda/net/irproc.c
new file mode 100644
index 000000000000..77cfdde9d82f
--- /dev/null
+++ b/drivers/staging/irda/net/irproc.c
@@ -0,0 +1,96 @@
+/*********************************************************************
+ *
+ * Filename:      irproc.c
+ * Version:       1.0
+ * Description:   Various entries in the /proc file system
+ * Status:        Experimental.
+ * Author:        Thomas Davis, <ratbert@radiks.net>
+ * Created at:    Sat Feb 21 21:33:24 1998
+ * Modified at:   Sun Nov 14 08:54:54 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1998-1999, Dag Brattli <dagb@cs.uit.no>
+ *     Copyright (c) 1998, Thomas Davis, <ratbert@radiks.net>,
+ *     All Rights Reserved.
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     I, Thomas Davis, provide no warranty for any of this software.
+ *     This material is provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <net/net_namespace.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/irlap.h>
+#include <net/irda/irlmp.h>
+
+extern const struct file_operations discovery_seq_fops;
+extern const struct file_operations irlap_seq_fops;
+extern const struct file_operations irlmp_seq_fops;
+extern const struct file_operations irttp_seq_fops;
+extern const struct file_operations irias_seq_fops;
+
+struct irda_entry {
+	const char *name;
+	const struct file_operations *fops;
+};
+
+struct proc_dir_entry *proc_irda;
+EXPORT_SYMBOL(proc_irda);
+
+static const struct irda_entry irda_dirs[] = {
+	{"discovery",	&discovery_seq_fops},
+	{"irttp",	&irttp_seq_fops},
+	{"irlmp",	&irlmp_seq_fops},
+	{"irlap",	&irlap_seq_fops},
+	{"irias",	&irias_seq_fops},
+};
+
+/*
+ * Function irda_proc_register (void)
+ *
+ *    Register irda entry in /proc file system
+ *
+ */
+void __init irda_proc_register(void)
+{
+	int i;
+
+	proc_irda = proc_mkdir("irda", init_net.proc_net);
+	if (proc_irda == NULL)
+		return;
+
+	for (i = 0; i < ARRAY_SIZE(irda_dirs); i++)
+		(void) proc_create(irda_dirs[i].name, 0, proc_irda,
+				   irda_dirs[i].fops);
+}
+
+/*
+ * Function irda_proc_unregister (void)
+ *
+ *    Unregister irda entry in /proc file system
+ *
+ */
+void irda_proc_unregister(void)
+{
+	int i;
+
+	if (proc_irda) {
+		for (i=0; i<ARRAY_SIZE(irda_dirs); i++)
+			remove_proc_entry(irda_dirs[i].name, proc_irda);
+
+		remove_proc_entry("irda", init_net.proc_net);
+		proc_irda = NULL;
+	}
+}
+
+
diff --git a/drivers/staging/irda/net/irqueue.c b/drivers/staging/irda/net/irqueue.c
new file mode 100644
index 000000000000..160dc89335e2
--- /dev/null
+++ b/drivers/staging/irda/net/irqueue.c
@@ -0,0 +1,911 @@
+/*********************************************************************
+ *
+ * Filename:      irqueue.c
+ * Version:       0.3
+ * Description:   General queue implementation
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Tue Jun  9 13:29:31 1998
+ * Modified at:   Sun Dec 12 13:48:22 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * Modified at:   Thu Jan  4 14:29:10 CET 2001
+ * Modified by:   Marc Zyngier <mzyngier@freesurf.fr>
+ *
+ *     Copyright (C) 1998-1999, Aage Kvalnes <aage@cs.uit.no>
+ *     Copyright (C) 1998, Dag Brattli,
+ *     All Rights Reserved.
+ *
+ *     This code is taken from the Vortex Operating System written by Aage
+ *     Kvalnes. Aage has agreed that this code can use the GPL licence,
+ *     although he does not use that licence in his own code.
+ *
+ *     This copyright does however _not_ include the ELF hash() function
+ *     which I currently don't know which licence or copyright it
+ *     has. Please inform me if you know.
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+/*
+ * NOTE :
+ * There are various problems with this package :
+ *	o the hash function for ints is pathetic (but could be changed)
+ *	o locking is sometime suspicious (especially during enumeration)
+ *	o most users have only a few elements (== overhead)
+ *	o most users never use search, so don't benefit from hashing
+ * Problem already fixed :
+ *	o not 64 bit compliant (most users do hashv = (int) self)
+ *	o hashbin_remove() is broken => use hashbin_remove_this()
+ * I think most users would be better served by a simple linked list
+ * (like include/linux/list.h) with a global spinlock per list.
+ * Jean II
+ */
+
+/*
+ * Notes on the concurrent access to hashbin and other SMP issues
+ * -------------------------------------------------------------
+ *	Hashbins are very often in the IrDA stack a global repository of
+ * information, and therefore used in a very asynchronous manner following
+ * various events (driver calls, timers, user calls...).
+ *	Therefore, very often it is highly important to consider the
+ * management of concurrent access to the hashbin and how to guarantee the
+ * consistency of the operations on it.
+ *
+ *	First, we need to define the objective of locking :
+ *		1) Protect user data (content pointed by the hashbin)
+ *		2) Protect hashbin structure itself (linked list in each bin)
+ *
+ *			     OLD LOCKING
+ *			     -----------
+ *
+ *	The previous locking strategy, either HB_LOCAL or HB_GLOBAL were
+ * both inadequate in *both* aspect.
+ *		o HB_GLOBAL was using a spinlock for each bin (local locking).
+ *		o HB_LOCAL was disabling irq on *all* CPUs, so use a single
+ *		  global semaphore.
+ *	The problems were :
+ *		A) Global irq disabling is no longer supported by the kernel
+ *		B) No protection for the hashbin struct global data
+ *			o hashbin_delete()
+ *			o hb_current
+ *		C) No protection for user data in some cases
+ *
+ *	A) HB_LOCAL use global irq disabling, so doesn't work on kernel
+ * 2.5.X. Even when it is supported (kernel 2.4.X and earlier), its
+ * performance is not satisfactory on SMP setups. Most hashbins were
+ * HB_LOCAL, so (A) definitely need fixing.
+ *	B) HB_LOCAL could be modified to fix (B). However, because HB_GLOBAL
+ * lock only the individual bins, it will never be able to lock the
+ * global data, so can't do (B).
+ *	C) Some functions return pointer to data that is still in the
+ * hashbin :
+ *		o hashbin_find()
+ *		o hashbin_get_first()
+ *		o hashbin_get_next()
+ *	As the data is still in the hashbin, it may be changed or free'd
+ * while the caller is examinimg the data. In those case, locking can't
+ * be done within the hashbin, but must include use of the data within
+ * the caller.
+ *	The caller can easily do this with HB_LOCAL (just disable irqs).
+ * However, this is impossible with HB_GLOBAL because the caller has no
+ * way to know the proper bin, so don't know which spinlock to use.
+ *
+ *	Quick summary : can no longer use HB_LOCAL, and HB_GLOBAL is
+ * fundamentally broken and will never work.
+ *
+ *			     NEW LOCKING
+ *			     -----------
+ *
+ *	To fix those problems, I've introduce a few changes in the
+ * hashbin locking :
+ *		1) New HB_LOCK scheme
+ *		2) hashbin->hb_spinlock
+ *		3) New hashbin usage policy
+ *
+ * HB_LOCK :
+ * -------
+ *	HB_LOCK is a locking scheme intermediate between the old HB_LOCAL
+ * and HB_GLOBAL. It uses a single spinlock to protect the whole content
+ * of the hashbin. As it is a single spinlock, it can protect the global
+ * data of the hashbin and not only the bins themselves.
+ *	HB_LOCK can only protect some of the hashbin calls, so it only lock
+ * call that can be made 100% safe and leave other call unprotected.
+ *	HB_LOCK in theory is slower than HB_GLOBAL, but as the hashbin
+ * content is always small contention is not high, so it doesn't matter
+ * much. HB_LOCK is probably faster than HB_LOCAL.
+ *
+ * hashbin->hb_spinlock :
+ * --------------------
+ *	The spinlock that HB_LOCK uses is available for caller, so that
+ * the caller can protect unprotected calls (see below).
+ *	If the caller want to do entirely its own locking (HB_NOLOCK), he
+ * can do so and may use safely this spinlock.
+ *	Locking is done like this :
+ *		spin_lock_irqsave(&hashbin->hb_spinlock, flags);
+ *	Releasing the lock :
+ *		spin_unlock_irqrestore(&hashbin->hb_spinlock, flags);
+ *
+ * Safe & Protected calls :
+ * ----------------------
+ *	The following calls are safe or protected via HB_LOCK :
+ *		o hashbin_new()		-> safe
+ *		o hashbin_delete()
+ *		o hashbin_insert()
+ *		o hashbin_remove_first()
+ *		o hashbin_remove()
+ *		o hashbin_remove_this()
+ *		o HASHBIN_GET_SIZE()	-> atomic
+ *
+ *	The following calls only protect the hashbin itself :
+ *		o hashbin_lock_find()
+ *		o hashbin_find_next()
+ *
+ * Unprotected calls :
+ * -----------------
+ *	The following calls need to be protected by the caller :
+ *		o hashbin_find()
+ *		o hashbin_get_first()
+ *		o hashbin_get_next()
+ *
+ * Locking Policy :
+ * --------------
+ *	If the hashbin is used only in a single thread of execution
+ * (explicitly or implicitely), you can use HB_NOLOCK
+ *	If the calling module already provide concurrent access protection,
+ * you may use HB_NOLOCK.
+ *
+ *	In all other cases, you need to use HB_LOCK and lock the hashbin
+ * every time before calling one of the unprotected calls. You also must
+ * use the pointer returned by the unprotected call within the locked
+ * region.
+ *
+ * Extra care for enumeration :
+ * --------------------------
+ *	hashbin_get_first() and hashbin_get_next() use the hashbin to
+ * store the current position, in hb_current.
+ *	As long as the hashbin remains locked, this is safe. If you unlock
+ * the hashbin, the current position may change if anybody else modify
+ * or enumerate the hashbin.
+ *	Summary : do the full enumeration while locked.
+ *
+ *	Alternatively, you may use hashbin_find_next(). But, this will
+ * be slower, is more complex to use and doesn't protect the hashbin
+ * content. So, care is needed here as well.
+ *
+ * Other issues :
+ * ------------
+ *	I believe that we are overdoing it by using spin_lock_irqsave()
+ * and we should use only spin_lock_bh() or similar. But, I don't have
+ * the balls to try it out.
+ *	Don't believe that because hashbin are now (somewhat) SMP safe
+ * that the rest of the code is. Higher layers tend to be safest,
+ * but LAP and LMP would need some serious dedicated love.
+ *
+ * Jean II
+ */
+#include <linux/module.h>
+#include <linux/slab.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/irqueue.h>
+
+/************************ QUEUE SUBROUTINES ************************/
+
+/*
+ * Hashbin
+ */
+#define GET_HASHBIN(x) ( x & HASHBIN_MASK )
+
+/*
+ * Function hash (name)
+ *
+ *    This function hash the input string 'name' using the ELF hash
+ *    function for strings.
+ */
+static __u32 hash( const char* name)
+{
+	__u32 h = 0;
+	__u32 g;
+
+	while(*name) {
+		h = (h<<4) + *name++;
+		if ((g = (h & 0xf0000000)))
+			h ^=g>>24;
+		h &=~g;
+	}
+	return h;
+}
+
+/*
+ * Function enqueue_first (queue, proc)
+ *
+ *    Insert item first in queue.
+ *
+ */
+static void enqueue_first(irda_queue_t **queue, irda_queue_t* element)
+{
+
+	/*
+	 * Check if queue is empty.
+	 */
+	if ( *queue == NULL ) {
+		/*
+		 * Queue is empty.  Insert one element into the queue.
+		 */
+		element->q_next = element->q_prev = *queue = element;
+
+	} else {
+		/*
+		 * Queue is not empty.  Insert element into front of queue.
+		 */
+		element->q_next          = (*queue);
+		(*queue)->q_prev->q_next = element;
+		element->q_prev          = (*queue)->q_prev;
+		(*queue)->q_prev         = element;
+		(*queue)                 = element;
+	}
+}
+
+
+/*
+ * Function dequeue (queue)
+ *
+ *    Remove first entry in queue
+ *
+ */
+static irda_queue_t *dequeue_first(irda_queue_t **queue)
+{
+	irda_queue_t *ret;
+
+	pr_debug("dequeue_first()\n");
+
+	/*
+	 * Set return value
+	 */
+	ret =  *queue;
+
+	if ( *queue == NULL ) {
+		/*
+		 * Queue was empty.
+		 */
+	} else if ( (*queue)->q_next == *queue ) {
+		/*
+		 *  Queue only contained a single element. It will now be
+		 *  empty.
+		 */
+		*queue = NULL;
+	} else {
+		/*
+		 * Queue contained several element.  Remove the first one.
+		 */
+		(*queue)->q_prev->q_next = (*queue)->q_next;
+		(*queue)->q_next->q_prev = (*queue)->q_prev;
+		*queue = (*queue)->q_next;
+	}
+
+	/*
+	 * Return the removed entry (or NULL of queue was empty).
+	 */
+	return ret;
+}
+
+/*
+ * Function dequeue_general (queue, element)
+ *
+ *
+ */
+static irda_queue_t *dequeue_general(irda_queue_t **queue, irda_queue_t* element)
+{
+	irda_queue_t *ret;
+
+	pr_debug("dequeue_general()\n");
+
+	/*
+	 * Set return value
+	 */
+	ret =  *queue;
+
+	if ( *queue == NULL ) {
+		/*
+		 * Queue was empty.
+		 */
+	} else if ( (*queue)->q_next == *queue ) {
+		/*
+		 *  Queue only contained a single element. It will now be
+		 *  empty.
+		 */
+		*queue = NULL;
+
+	} else {
+		/*
+		 *  Remove specific element.
+		 */
+		element->q_prev->q_next = element->q_next;
+		element->q_next->q_prev = element->q_prev;
+		if ( (*queue) == element)
+			(*queue) = element->q_next;
+	}
+
+	/*
+	 * Return the removed entry (or NULL of queue was empty).
+	 */
+	return ret;
+}
+
+/************************ HASHBIN MANAGEMENT ************************/
+
+/*
+ * Function hashbin_create ( type, name )
+ *
+ *    Create hashbin!
+ *
+ */
+hashbin_t *hashbin_new(int type)
+{
+	hashbin_t* hashbin;
+
+	/*
+	 * Allocate new hashbin
+	 */
+	hashbin = kzalloc(sizeof(*hashbin), GFP_ATOMIC);
+	if (!hashbin)
+		return NULL;
+
+	/*
+	 * Initialize structure
+	 */
+	hashbin->hb_type = type;
+	hashbin->magic = HB_MAGIC;
+	//hashbin->hb_current = NULL;
+
+	/* Make sure all spinlock's are unlocked */
+	if ( hashbin->hb_type & HB_LOCK ) {
+		spin_lock_init(&hashbin->hb_spinlock);
+	}
+
+	return hashbin;
+}
+EXPORT_SYMBOL(hashbin_new);
+
+
+/*
+ * Function hashbin_delete (hashbin, free_func)
+ *
+ *    Destroy hashbin, the free_func can be a user supplied special routine
+ *    for deallocating this structure if it's complex. If not the user can
+ *    just supply kfree, which should take care of the job.
+ */
+int hashbin_delete( hashbin_t* hashbin, FREE_FUNC free_func)
+{
+	irda_queue_t* queue;
+	unsigned long flags = 0;
+	int i;
+
+	IRDA_ASSERT(hashbin != NULL, return -1;);
+	IRDA_ASSERT(hashbin->magic == HB_MAGIC, return -1;);
+
+	/* Synchronize */
+	if (hashbin->hb_type & HB_LOCK)
+		spin_lock_irqsave(&hashbin->hb_spinlock, flags);
+
+	/*
+	 *  Free the entries in the hashbin, TODO: use hashbin_clear when
+	 *  it has been shown to work
+	 */
+	for (i = 0; i < HASHBIN_SIZE; i ++ ) {
+		while (1) {
+			queue = dequeue_first((irda_queue_t**) &hashbin->hb_queue[i]);
+
+			if (!queue)
+				break;
+
+			if (free_func) {
+				if (hashbin->hb_type & HB_LOCK)
+					spin_unlock_irqrestore(&hashbin->hb_spinlock, flags);
+				free_func(queue);
+				if (hashbin->hb_type & HB_LOCK)
+					spin_lock_irqsave(&hashbin->hb_spinlock, flags);
+			}
+		}
+	}
+
+	/* Cleanup local data */
+	hashbin->hb_current = NULL;
+	hashbin->magic = ~HB_MAGIC;
+
+	/* Release lock */
+	if (hashbin->hb_type & HB_LOCK)
+		spin_unlock_irqrestore(&hashbin->hb_spinlock, flags);
+
+	/*
+	 *  Free the hashbin structure
+	 */
+	kfree(hashbin);
+
+	return 0;
+}
+EXPORT_SYMBOL(hashbin_delete);
+
+/********************* HASHBIN LIST OPERATIONS *********************/
+
+/*
+ * Function hashbin_insert (hashbin, entry, name)
+ *
+ *    Insert an entry into the hashbin
+ *
+ */
+void hashbin_insert(hashbin_t* hashbin, irda_queue_t* entry, long hashv,
+		    const char* name)
+{
+	unsigned long flags = 0;
+	int bin;
+
+	IRDA_ASSERT( hashbin != NULL, return;);
+	IRDA_ASSERT( hashbin->magic == HB_MAGIC, return;);
+
+	/*
+	 * Locate hashbin
+	 */
+	if ( name )
+		hashv = hash( name );
+	bin = GET_HASHBIN( hashv );
+
+	/* Synchronize */
+	if ( hashbin->hb_type & HB_LOCK ) {
+		spin_lock_irqsave(&hashbin->hb_spinlock, flags);
+	} /* Default is no-lock  */
+
+	/*
+	 * Store name and key
+	 */
+	entry->q_hash = hashv;
+	if ( name )
+		strlcpy( entry->q_name, name, sizeof(entry->q_name));
+
+	/*
+	 * Insert new entry first
+	 */
+	enqueue_first( (irda_queue_t**) &hashbin->hb_queue[ bin ],
+		       entry);
+	hashbin->hb_size++;
+
+	/* Release lock */
+	if ( hashbin->hb_type & HB_LOCK ) {
+		spin_unlock_irqrestore(&hashbin->hb_spinlock, flags);
+	} /* Default is no-lock  */
+}
+EXPORT_SYMBOL(hashbin_insert);
+
+/*
+ *  Function hashbin_remove_first (hashbin)
+ *
+ *    Remove first entry of the hashbin
+ *
+ * Note : this function no longer use hashbin_remove(), but does things
+ * similar to hashbin_remove_this(), so can be considered safe.
+ * Jean II
+ */
+void *hashbin_remove_first( hashbin_t *hashbin)
+{
+	unsigned long flags = 0;
+	irda_queue_t *entry = NULL;
+
+	/* Synchronize */
+	if ( hashbin->hb_type & HB_LOCK ) {
+		spin_lock_irqsave(&hashbin->hb_spinlock, flags);
+	} /* Default is no-lock  */
+
+	entry = hashbin_get_first( hashbin);
+	if ( entry != NULL) {
+		int	bin;
+		long	hashv;
+		/*
+		 * Locate hashbin
+		 */
+		hashv = entry->q_hash;
+		bin = GET_HASHBIN( hashv );
+
+		/*
+		 * Dequeue the entry...
+		 */
+		dequeue_general( (irda_queue_t**) &hashbin->hb_queue[ bin ],
+				 entry);
+		hashbin->hb_size--;
+		entry->q_next = NULL;
+		entry->q_prev = NULL;
+
+		/*
+		 *  Check if this item is the currently selected item, and in
+		 *  that case we must reset hb_current
+		 */
+		if ( entry == hashbin->hb_current)
+			hashbin->hb_current = NULL;
+	}
+
+	/* Release lock */
+	if ( hashbin->hb_type & HB_LOCK ) {
+		spin_unlock_irqrestore(&hashbin->hb_spinlock, flags);
+	} /* Default is no-lock  */
+
+	return entry;
+}
+
+
+/*
+ *  Function hashbin_remove (hashbin, hashv, name)
+ *
+ *    Remove entry with the given name
+ *
+ *  The use of this function is highly discouraged, because the whole
+ *  concept behind hashbin_remove() is broken. In many cases, it's not
+ *  possible to guarantee the unicity of the index (either hashv or name),
+ *  leading to removing the WRONG entry.
+ *  The only simple safe use is :
+ *		hashbin_remove(hasbin, (int) self, NULL);
+ *  In other case, you must think hard to guarantee unicity of the index.
+ *  Jean II
+ */
+void* hashbin_remove( hashbin_t* hashbin, long hashv, const char* name)
+{
+	int bin, found = FALSE;
+	unsigned long flags = 0;
+	irda_queue_t* entry;
+
+	IRDA_ASSERT( hashbin != NULL, return NULL;);
+	IRDA_ASSERT( hashbin->magic == HB_MAGIC, return NULL;);
+
+	/*
+	 * Locate hashbin
+	 */
+	if ( name )
+		hashv = hash( name );
+	bin = GET_HASHBIN( hashv );
+
+	/* Synchronize */
+	if ( hashbin->hb_type & HB_LOCK ) {
+		spin_lock_irqsave(&hashbin->hb_spinlock, flags);
+	} /* Default is no-lock  */
+
+	/*
+	 * Search for entry
+	 */
+	entry = hashbin->hb_queue[ bin ];
+	if ( entry ) {
+		do {
+			/*
+			 * Check for key
+			 */
+			if ( entry->q_hash == hashv ) {
+				/*
+				 * Name compare too?
+				 */
+				if ( name ) {
+					if ( strcmp( entry->q_name, name) == 0)
+					{
+						found = TRUE;
+						break;
+					}
+				} else {
+					found = TRUE;
+					break;
+				}
+			}
+			entry = entry->q_next;
+		} while ( entry != hashbin->hb_queue[ bin ] );
+	}
+
+	/*
+	 * If entry was found, dequeue it
+	 */
+	if ( found ) {
+		dequeue_general( (irda_queue_t**) &hashbin->hb_queue[ bin ],
+				 entry);
+		hashbin->hb_size--;
+
+		/*
+		 *  Check if this item is the currently selected item, and in
+		 *  that case we must reset hb_current
+		 */
+		if ( entry == hashbin->hb_current)
+			hashbin->hb_current = NULL;
+	}
+
+	/* Release lock */
+	if ( hashbin->hb_type & HB_LOCK ) {
+		spin_unlock_irqrestore(&hashbin->hb_spinlock, flags);
+	} /* Default is no-lock  */
+
+
+	/* Return */
+	if ( found )
+		return entry;
+	else
+		return NULL;
+
+}
+EXPORT_SYMBOL(hashbin_remove);
+
+/*
+ *  Function hashbin_remove_this (hashbin, entry)
+ *
+ *    Remove entry with the given name
+ *
+ * In some cases, the user of hashbin can't guarantee the unicity
+ * of either the hashv or name.
+ * In those cases, using the above function is guaranteed to cause troubles,
+ * so we use this one instead...
+ * And by the way, it's also faster, because we skip the search phase ;-)
+ */
+void* hashbin_remove_this( hashbin_t* hashbin, irda_queue_t* entry)
+{
+	unsigned long flags = 0;
+	int	bin;
+	long	hashv;
+
+	IRDA_ASSERT( hashbin != NULL, return NULL;);
+	IRDA_ASSERT( hashbin->magic == HB_MAGIC, return NULL;);
+	IRDA_ASSERT( entry != NULL, return NULL;);
+
+	/* Synchronize */
+	if ( hashbin->hb_type & HB_LOCK ) {
+		spin_lock_irqsave(&hashbin->hb_spinlock, flags);
+	} /* Default is no-lock  */
+
+	/* Check if valid and not already removed... */
+	if((entry->q_next == NULL) || (entry->q_prev == NULL)) {
+		entry = NULL;
+		goto out;
+	}
+
+	/*
+	 * Locate hashbin
+	 */
+	hashv = entry->q_hash;
+	bin = GET_HASHBIN( hashv );
+
+	/*
+	 * Dequeue the entry...
+	 */
+	dequeue_general( (irda_queue_t**) &hashbin->hb_queue[ bin ],
+			 entry);
+	hashbin->hb_size--;
+	entry->q_next = NULL;
+	entry->q_prev = NULL;
+
+	/*
+	 *  Check if this item is the currently selected item, and in
+	 *  that case we must reset hb_current
+	 */
+	if ( entry == hashbin->hb_current)
+		hashbin->hb_current = NULL;
+out:
+	/* Release lock */
+	if ( hashbin->hb_type & HB_LOCK ) {
+		spin_unlock_irqrestore(&hashbin->hb_spinlock, flags);
+	} /* Default is no-lock  */
+
+	return entry;
+}
+EXPORT_SYMBOL(hashbin_remove_this);
+
+/*********************** HASHBIN ENUMERATION ***********************/
+
+/*
+ * Function hashbin_common_find (hashbin, hashv, name)
+ *
+ *    Find item with the given hashv or name
+ *
+ */
+void* hashbin_find( hashbin_t* hashbin, long hashv, const char* name )
+{
+	int bin;
+	irda_queue_t* entry;
+
+	pr_debug("hashbin_find()\n");
+
+	IRDA_ASSERT( hashbin != NULL, return NULL;);
+	IRDA_ASSERT( hashbin->magic == HB_MAGIC, return NULL;);
+
+	/*
+	 * Locate hashbin
+	 */
+	if ( name )
+		hashv = hash( name );
+	bin = GET_HASHBIN( hashv );
+
+	/*
+	 * Search for entry
+	 */
+	entry = hashbin->hb_queue[ bin];
+	if ( entry ) {
+		do {
+			/*
+			 * Check for key
+			 */
+			if ( entry->q_hash == hashv ) {
+				/*
+				 * Name compare too?
+				 */
+				if ( name ) {
+					if ( strcmp( entry->q_name, name ) == 0 ) {
+						return entry;
+					}
+				} else {
+					return entry;
+				}
+			}
+			entry = entry->q_next;
+		} while ( entry != hashbin->hb_queue[ bin ] );
+	}
+
+	return NULL;
+}
+EXPORT_SYMBOL(hashbin_find);
+
+/*
+ * Function hashbin_lock_find (hashbin, hashv, name)
+ *
+ *    Find item with the given hashv or name
+ *
+ * Same, but with spinlock protection...
+ * I call it safe, but it's only safe with respect to the hashbin, not its
+ * content. - Jean II
+ */
+void* hashbin_lock_find( hashbin_t* hashbin, long hashv, const char* name )
+{
+	unsigned long flags = 0;
+	irda_queue_t* entry;
+
+	/* Synchronize */
+	spin_lock_irqsave(&hashbin->hb_spinlock, flags);
+
+	/*
+	 * Search for entry
+	 */
+	entry = hashbin_find(hashbin, hashv, name);
+
+	/* Release lock */
+	spin_unlock_irqrestore(&hashbin->hb_spinlock, flags);
+
+	return entry;
+}
+EXPORT_SYMBOL(hashbin_lock_find);
+
+/*
+ * Function hashbin_find (hashbin, hashv, name, pnext)
+ *
+ *    Find an item with the given hashv or name, and its successor
+ *
+ * This function allow to do concurrent enumerations without the
+ * need to lock over the whole session, because the caller keep the
+ * context of the search. On the other hand, it might fail and return
+ * NULL if the entry is removed. - Jean II
+ */
+void* hashbin_find_next( hashbin_t* hashbin, long hashv, const char* name,
+			 void ** pnext)
+{
+	unsigned long flags = 0;
+	irda_queue_t* entry;
+
+	/* Synchronize */
+	spin_lock_irqsave(&hashbin->hb_spinlock, flags);
+
+	/*
+	 * Search for current entry
+	 * This allow to check if the current item is still in the
+	 * hashbin or has been removed.
+	 */
+	entry = hashbin_find(hashbin, hashv, name);
+
+	/*
+	 * Trick hashbin_get_next() to return what we want
+	 */
+	if(entry) {
+		hashbin->hb_current = entry;
+		*pnext = hashbin_get_next( hashbin );
+	} else
+		*pnext = NULL;
+
+	/* Release lock */
+	spin_unlock_irqrestore(&hashbin->hb_spinlock, flags);
+
+	return entry;
+}
+
+/*
+ * Function hashbin_get_first (hashbin)
+ *
+ *    Get a pointer to first element in hashbin, this function must be
+ *    called before any calls to hashbin_get_next()!
+ *
+ */
+irda_queue_t *hashbin_get_first( hashbin_t* hashbin)
+{
+	irda_queue_t *entry;
+	int i;
+
+	IRDA_ASSERT( hashbin != NULL, return NULL;);
+	IRDA_ASSERT( hashbin->magic == HB_MAGIC, return NULL;);
+
+	if ( hashbin == NULL)
+		return NULL;
+
+	for ( i = 0; i < HASHBIN_SIZE; i ++ ) {
+		entry = hashbin->hb_queue[ i];
+		if ( entry) {
+			hashbin->hb_current = entry;
+			return entry;
+		}
+	}
+	/*
+	 *  Did not find any item in hashbin
+	 */
+	return NULL;
+}
+EXPORT_SYMBOL(hashbin_get_first);
+
+/*
+ * Function hashbin_get_next (hashbin)
+ *
+ *    Get next item in hashbin. A series of hashbin_get_next() calls must
+ *    be started by a call to hashbin_get_first(). The function returns
+ *    NULL when all items have been traversed
+ *
+ * The context of the search is stored within the hashbin, so you must
+ * protect yourself from concurrent enumerations. - Jean II
+ */
+irda_queue_t *hashbin_get_next( hashbin_t *hashbin)
+{
+	irda_queue_t* entry;
+	int bin;
+	int i;
+
+	IRDA_ASSERT( hashbin != NULL, return NULL;);
+	IRDA_ASSERT( hashbin->magic == HB_MAGIC, return NULL;);
+
+	if ( hashbin->hb_current == NULL) {
+		IRDA_ASSERT( hashbin->hb_current != NULL, return NULL;);
+		return NULL;
+	}
+	entry = hashbin->hb_current->q_next;
+	bin = GET_HASHBIN( entry->q_hash);
+
+	/*
+	 *  Make sure that we are not back at the beginning of the queue
+	 *  again
+	 */
+	if ( entry != hashbin->hb_queue[ bin ]) {
+		hashbin->hb_current = entry;
+
+		return entry;
+	}
+
+	/*
+	 *  Check that this is not the last queue in hashbin
+	 */
+	if ( bin >= HASHBIN_SIZE)
+		return NULL;
+
+	/*
+	 *  Move to next queue in hashbin
+	 */
+	bin++;
+	for ( i = bin; i < HASHBIN_SIZE; i++ ) {
+		entry = hashbin->hb_queue[ i];
+		if ( entry) {
+			hashbin->hb_current = entry;
+
+			return entry;
+		}
+	}
+	return NULL;
+}
+EXPORT_SYMBOL(hashbin_get_next);
diff --git a/drivers/staging/irda/net/irsysctl.c b/drivers/staging/irda/net/irsysctl.c
new file mode 100644
index 000000000000..873da5e7d428
--- /dev/null
+++ b/drivers/staging/irda/net/irsysctl.c
@@ -0,0 +1,258 @@
+/*********************************************************************
+ *
+ * Filename:      irsysctl.c
+ * Version:       1.0
+ * Description:   Sysctl interface for IrDA
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sun May 24 22:12:06 1998
+ * Modified at:   Fri Jun  4 02:50:15 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1997, 1999 Dag Brattli, All Rights Reserved.
+ *     Copyright (c) 2000-2001 Jean Tourrilhes <jt@hpl.hp.com>
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#include <linux/mm.h>
+#include <linux/ctype.h>
+#include <linux/sysctl.h>
+#include <linux/init.h>
+
+#include <net/irda/irda.h>		/* irda_debug */
+#include <net/irda/irlmp.h>
+#include <net/irda/timer.h>
+#include <net/irda/irias_object.h>
+
+extern int  sysctl_discovery;
+extern int  sysctl_discovery_slots;
+extern int  sysctl_discovery_timeout;
+extern int  sysctl_slot_timeout;
+extern int  sysctl_fast_poll_increase;
+extern char sysctl_devname[];
+extern int  sysctl_max_baud_rate;
+extern unsigned int sysctl_min_tx_turn_time;
+extern unsigned int sysctl_max_tx_data_size;
+extern unsigned int sysctl_max_tx_window;
+extern int  sysctl_max_noreply_time;
+extern int  sysctl_warn_noreply_time;
+extern int  sysctl_lap_keepalive_time;
+
+extern struct irlmp_cb *irlmp;
+
+/* this is needed for the proc_dointvec_minmax - Jean II */
+static int max_discovery_slots = 16;		/* ??? */
+static int min_discovery_slots = 1;
+/* IrLAP 6.13.2 says 25ms to 10+70ms - allow higher since some devices
+ * seems to require it. (from Dag's comment) */
+static int max_slot_timeout = 160;
+static int min_slot_timeout = 20;
+static int max_max_baud_rate = 16000000;	/* See qos.c - IrLAP spec */
+static int min_max_baud_rate = 2400;
+static int max_min_tx_turn_time = 10000;	/* See qos.c - IrLAP spec */
+static int min_min_tx_turn_time;
+static int max_max_tx_data_size = 2048;		/* See qos.c - IrLAP spec */
+static int min_max_tx_data_size = 64;
+static int max_max_tx_window = 7;		/* See qos.c - IrLAP spec */
+static int min_max_tx_window = 1;
+static int max_max_noreply_time = 40;		/* See qos.c - IrLAP spec */
+static int min_max_noreply_time = 3;
+static int max_warn_noreply_time = 3;		/* 3s == standard */
+static int min_warn_noreply_time = 1;		/* 1s == min WD_TIMER */
+static int max_lap_keepalive_time = 10000;	/* 10s */
+static int min_lap_keepalive_time = 100;	/* 100us */
+/* For other sysctl, I've no idea of the range. Maybe Dag could help
+ * us on that - Jean II */
+
+static int do_devname(struct ctl_table *table, int write,
+		      void __user *buffer, size_t *lenp, loff_t *ppos)
+{
+	int ret;
+
+	ret = proc_dostring(table, write, buffer, lenp, ppos);
+	if (ret == 0 && write) {
+		struct ias_value *val;
+
+		val = irias_new_string_value(sysctl_devname);
+		if (val)
+			irias_object_change_attribute("Device", "DeviceName", val);
+	}
+	return ret;
+}
+
+
+static int do_discovery(struct ctl_table *table, int write,
+                    void __user *buffer, size_t *lenp, loff_t *ppos)
+{
+       int ret;
+
+       ret = proc_dointvec(table, write, buffer, lenp, ppos);
+       if (ret)
+	       return ret;
+
+       if (irlmp == NULL)
+	       return -ENODEV;
+
+       if (sysctl_discovery)
+	       irlmp_start_discovery_timer(irlmp, sysctl_discovery_timeout*HZ);
+       else
+	       del_timer_sync(&irlmp->discovery_timer);
+
+       return ret;
+}
+
+/* One file */
+static struct ctl_table irda_table[] = {
+	{
+		.procname	= "discovery",
+		.data		= &sysctl_discovery,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= do_discovery,
+	},
+	{
+		.procname	= "devname",
+		.data		= sysctl_devname,
+		.maxlen		= 65,
+		.mode		= 0644,
+		.proc_handler	= do_devname,
+	},
+#ifdef CONFIG_IRDA_FAST_RR
+	{
+		.procname	= "fast_poll_increase",
+		.data		= &sysctl_fast_poll_increase,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec
+	},
+#endif
+	{
+		.procname	= "discovery_slots",
+		.data		= &sysctl_discovery_slots,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= &min_discovery_slots,
+		.extra2		= &max_discovery_slots
+	},
+	{
+		.procname	= "discovery_timeout",
+		.data		= &sysctl_discovery_timeout,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec
+	},
+	{
+		.procname	= "slot_timeout",
+		.data		= &sysctl_slot_timeout,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= &min_slot_timeout,
+		.extra2		= &max_slot_timeout
+	},
+	{
+		.procname	= "max_baud_rate",
+		.data		= &sysctl_max_baud_rate,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= &min_max_baud_rate,
+		.extra2		= &max_max_baud_rate
+	},
+	{
+		.procname	= "min_tx_turn_time",
+		.data		= &sysctl_min_tx_turn_time,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= &min_min_tx_turn_time,
+		.extra2		= &max_min_tx_turn_time
+	},
+	{
+		.procname	= "max_tx_data_size",
+		.data		= &sysctl_max_tx_data_size,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= &min_max_tx_data_size,
+		.extra2		= &max_max_tx_data_size
+	},
+	{
+		.procname	= "max_tx_window",
+		.data		= &sysctl_max_tx_window,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= &min_max_tx_window,
+		.extra2		= &max_max_tx_window
+	},
+	{
+		.procname	= "max_noreply_time",
+		.data		= &sysctl_max_noreply_time,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= &min_max_noreply_time,
+		.extra2		= &max_max_noreply_time
+	},
+	{
+		.procname	= "warn_noreply_time",
+		.data		= &sysctl_warn_noreply_time,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= &min_warn_noreply_time,
+		.extra2		= &max_warn_noreply_time
+	},
+	{
+		.procname	= "lap_keepalive_time",
+		.data		= &sysctl_lap_keepalive_time,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= &min_lap_keepalive_time,
+		.extra2		= &max_lap_keepalive_time
+	},
+	{ }
+};
+
+static struct ctl_table_header *irda_table_header;
+
+/*
+ * Function irda_sysctl_register (void)
+ *
+ *    Register our sysctl interface
+ *
+ */
+int __init irda_sysctl_register(void)
+{
+	irda_table_header = register_net_sysctl(&init_net, "net/irda", irda_table);
+	if (!irda_table_header)
+		return -ENOMEM;
+
+	return 0;
+}
+
+/*
+ * Function irda_sysctl_unregister (void)
+ *
+ *    Unregister our sysctl interface
+ *
+ */
+void irda_sysctl_unregister(void)
+{
+	unregister_net_sysctl_table(irda_table_header);
+}
+
+
+
diff --git a/drivers/staging/irda/net/irttp.c b/drivers/staging/irda/net/irttp.c
new file mode 100644
index 000000000000..b6ab41d5b3a3
--- /dev/null
+++ b/drivers/staging/irda/net/irttp.c
@@ -0,0 +1,1891 @@
+/*********************************************************************
+ *
+ * Filename:      irttp.c
+ * Version:       1.2
+ * Description:   Tiny Transport Protocol (TTP) implementation
+ * Status:        Stable
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sun Aug 31 20:14:31 1997
+ * Modified at:   Wed Jan  5 11:31:27 2000
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1998-2000 Dag Brattli <dagb@cs.uit.no>,
+ *     All Rights Reserved.
+ *     Copyright (c) 2000-2003 Jean Tourrilhes <jt@hpl.hp.com>
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#include <linux/skbuff.h>
+#include <linux/init.h>
+#include <linux/fs.h>
+#include <linux/seq_file.h>
+#include <linux/slab.h>
+#include <linux/export.h>
+
+#include <asm/byteorder.h>
+#include <asm/unaligned.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/irlap.h>
+#include <net/irda/irlmp.h>
+#include <net/irda/parameters.h>
+#include <net/irda/irttp.h>
+
+static struct irttp_cb *irttp;
+
+static void __irttp_close_tsap(struct tsap_cb *self);
+
+static int irttp_data_indication(void *instance, void *sap,
+				 struct sk_buff *skb);
+static int irttp_udata_indication(void *instance, void *sap,
+				  struct sk_buff *skb);
+static void irttp_disconnect_indication(void *instance, void *sap,
+					LM_REASON reason, struct sk_buff *);
+static void irttp_connect_indication(void *instance, void *sap,
+				     struct qos_info *qos, __u32 max_sdu_size,
+				     __u8 header_size, struct sk_buff *skb);
+static void irttp_connect_confirm(void *instance, void *sap,
+				  struct qos_info *qos, __u32 max_sdu_size,
+				  __u8 header_size, struct sk_buff *skb);
+static void irttp_run_tx_queue(struct tsap_cb *self);
+static void irttp_run_rx_queue(struct tsap_cb *self);
+
+static void irttp_flush_queues(struct tsap_cb *self);
+static void irttp_fragment_skb(struct tsap_cb *self, struct sk_buff *skb);
+static struct sk_buff *irttp_reassemble_skb(struct tsap_cb *self);
+static void irttp_todo_expired(unsigned long data);
+static int irttp_param_max_sdu_size(void *instance, irda_param_t *param,
+				    int get);
+
+static void irttp_flow_indication(void *instance, void *sap, LOCAL_FLOW flow);
+static void irttp_status_indication(void *instance,
+				    LINK_STATUS link, LOCK_STATUS lock);
+
+/* Information for parsing parameters in IrTTP */
+static const pi_minor_info_t pi_minor_call_table[] = {
+	{ NULL, 0 },                                             /* 0x00 */
+	{ irttp_param_max_sdu_size, PV_INTEGER | PV_BIG_ENDIAN } /* 0x01 */
+};
+static const pi_major_info_t pi_major_call_table[] = {
+	{ pi_minor_call_table, 2 }
+};
+static pi_param_info_t param_info = { pi_major_call_table, 1, 0x0f, 4 };
+
+/************************ GLOBAL PROCEDURES ************************/
+
+/*
+ * Function irttp_init (void)
+ *
+ *    Initialize the IrTTP layer. Called by module initialization code
+ *
+ */
+int __init irttp_init(void)
+{
+	irttp = kzalloc(sizeof(struct irttp_cb), GFP_KERNEL);
+	if (irttp == NULL)
+		return -ENOMEM;
+
+	irttp->magic = TTP_MAGIC;
+
+	irttp->tsaps = hashbin_new(HB_LOCK);
+	if (!irttp->tsaps) {
+		net_err_ratelimited("%s: can't allocate IrTTP hashbin!\n",
+				    __func__);
+		kfree(irttp);
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+/*
+ * Function irttp_cleanup (void)
+ *
+ *    Called by module destruction/cleanup code
+ *
+ */
+void irttp_cleanup(void)
+{
+	/* Check for main structure */
+	IRDA_ASSERT(irttp->magic == TTP_MAGIC, return;);
+
+	/*
+	 *  Delete hashbin and close all TSAP instances in it
+	 */
+	hashbin_delete(irttp->tsaps, (FREE_FUNC) __irttp_close_tsap);
+
+	irttp->magic = 0;
+
+	/* De-allocate main structure */
+	kfree(irttp);
+
+	irttp = NULL;
+}
+
+/*************************** SUBROUTINES ***************************/
+
+/*
+ * Function irttp_start_todo_timer (self, timeout)
+ *
+ *    Start todo timer.
+ *
+ * Made it more effient and unsensitive to race conditions - Jean II
+ */
+static inline void irttp_start_todo_timer(struct tsap_cb *self, int timeout)
+{
+	/* Set new value for timer */
+	mod_timer(&self->todo_timer, jiffies + timeout);
+}
+
+/*
+ * Function irttp_todo_expired (data)
+ *
+ *    Todo timer has expired!
+ *
+ * One of the restriction of the timer is that it is run only on the timer
+ * interrupt which run every 10ms. This mean that even if you set the timer
+ * with a delay of 0, it may take up to 10ms before it's run.
+ * So, to minimise latency and keep cache fresh, we try to avoid using
+ * it as much as possible.
+ * Note : we can't use tasklets, because they can't be asynchronously
+ * killed (need user context), and we can't guarantee that here...
+ * Jean II
+ */
+static void irttp_todo_expired(unsigned long data)
+{
+	struct tsap_cb *self = (struct tsap_cb *) data;
+
+	/* Check that we still exist */
+	if (!self || self->magic != TTP_TSAP_MAGIC)
+		return;
+
+	pr_debug("%s(instance=%p)\n", __func__, self);
+
+	/* Try to make some progress, especially on Tx side - Jean II */
+	irttp_run_rx_queue(self);
+	irttp_run_tx_queue(self);
+
+	/* Check if time for disconnect */
+	if (test_bit(0, &self->disconnect_pend)) {
+		/* Check if it's possible to disconnect yet */
+		if (skb_queue_empty(&self->tx_queue)) {
+			/* Make sure disconnect is not pending anymore */
+			clear_bit(0, &self->disconnect_pend);	/* FALSE */
+
+			/* Note : self->disconnect_skb may be NULL */
+			irttp_disconnect_request(self, self->disconnect_skb,
+						 P_NORMAL);
+			self->disconnect_skb = NULL;
+		} else {
+			/* Try again later */
+			irttp_start_todo_timer(self, HZ/10);
+
+			/* No reason to try and close now */
+			return;
+		}
+	}
+
+	/* Check if it's closing time */
+	if (self->close_pend)
+		/* Finish cleanup */
+		irttp_close_tsap(self);
+}
+
+/*
+ * Function irttp_flush_queues (self)
+ *
+ *     Flushes (removes all frames) in transitt-buffer (tx_list)
+ */
+static void irttp_flush_queues(struct tsap_cb *self)
+{
+	struct sk_buff *skb;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == TTP_TSAP_MAGIC, return;);
+
+	/* Deallocate frames waiting to be sent */
+	while ((skb = skb_dequeue(&self->tx_queue)) != NULL)
+		dev_kfree_skb(skb);
+
+	/* Deallocate received frames */
+	while ((skb = skb_dequeue(&self->rx_queue)) != NULL)
+		dev_kfree_skb(skb);
+
+	/* Deallocate received fragments */
+	while ((skb = skb_dequeue(&self->rx_fragments)) != NULL)
+		dev_kfree_skb(skb);
+}
+
+/*
+ * Function irttp_reassemble (self)
+ *
+ *    Makes a new (continuous) skb of all the fragments in the fragment
+ *    queue
+ *
+ */
+static struct sk_buff *irttp_reassemble_skb(struct tsap_cb *self)
+{
+	struct sk_buff *skb, *frag;
+	int n = 0;  /* Fragment index */
+
+	IRDA_ASSERT(self != NULL, return NULL;);
+	IRDA_ASSERT(self->magic == TTP_TSAP_MAGIC, return NULL;);
+
+	pr_debug("%s(), self->rx_sdu_size=%d\n", __func__,
+		 self->rx_sdu_size);
+
+	skb = dev_alloc_skb(TTP_HEADER + self->rx_sdu_size);
+	if (!skb)
+		return NULL;
+
+	/*
+	 * Need to reserve space for TTP header in case this skb needs to
+	 * be requeued in case delivery failes
+	 */
+	skb_reserve(skb, TTP_HEADER);
+	skb_put(skb, self->rx_sdu_size);
+
+	/*
+	 *  Copy all fragments to a new buffer
+	 */
+	while ((frag = skb_dequeue(&self->rx_fragments)) != NULL) {
+		skb_copy_to_linear_data_offset(skb, n, frag->data, frag->len);
+		n += frag->len;
+
+		dev_kfree_skb(frag);
+	}
+
+	pr_debug("%s(), frame len=%d, rx_sdu_size=%d, rx_max_sdu_size=%d\n",
+		 __func__, n, self->rx_sdu_size, self->rx_max_sdu_size);
+	/* Note : irttp_run_rx_queue() calculate self->rx_sdu_size
+	 * by summing the size of all fragments, so we should always
+	 * have n == self->rx_sdu_size, except in cases where we
+	 * droped the last fragment (when self->rx_sdu_size exceed
+	 * self->rx_max_sdu_size), where n < self->rx_sdu_size.
+	 * Jean II */
+	IRDA_ASSERT(n <= self->rx_sdu_size, n = self->rx_sdu_size;);
+
+	/* Set the new length */
+	skb_trim(skb, n);
+
+	self->rx_sdu_size = 0;
+
+	return skb;
+}
+
+/*
+ * Function irttp_fragment_skb (skb)
+ *
+ *    Fragments a frame and queues all the fragments for transmission
+ *
+ */
+static inline void irttp_fragment_skb(struct tsap_cb *self,
+				      struct sk_buff *skb)
+{
+	struct sk_buff *frag;
+	__u8 *frame;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == TTP_TSAP_MAGIC, return;);
+	IRDA_ASSERT(skb != NULL, return;);
+
+	/*
+	 *  Split frame into a number of segments
+	 */
+	while (skb->len > self->max_seg_size) {
+		pr_debug("%s(), fragmenting ...\n", __func__);
+
+		/* Make new segment */
+		frag = alloc_skb(self->max_seg_size+self->max_header_size,
+				 GFP_ATOMIC);
+		if (!frag)
+			return;
+
+		skb_reserve(frag, self->max_header_size);
+
+		/* Copy data from the original skb into this fragment. */
+		skb_copy_from_linear_data(skb, skb_put(frag, self->max_seg_size),
+			      self->max_seg_size);
+
+		/* Insert TTP header, with the more bit set */
+		frame = skb_push(frag, TTP_HEADER);
+		frame[0] = TTP_MORE;
+
+		/* Hide the copied data from the original skb */
+		skb_pull(skb, self->max_seg_size);
+
+		/* Queue fragment */
+		skb_queue_tail(&self->tx_queue, frag);
+	}
+	/* Queue what is left of the original skb */
+	pr_debug("%s(), queuing last segment\n", __func__);
+
+	frame = skb_push(skb, TTP_HEADER);
+	frame[0] = 0x00; /* Clear more bit */
+
+	/* Queue fragment */
+	skb_queue_tail(&self->tx_queue, skb);
+}
+
+/*
+ * Function irttp_param_max_sdu_size (self, param)
+ *
+ *    Handle the MaxSduSize parameter in the connect frames, this function
+ *    will be called both when this parameter needs to be inserted into, and
+ *    extracted from the connect frames
+ */
+static int irttp_param_max_sdu_size(void *instance, irda_param_t *param,
+				    int get)
+{
+	struct tsap_cb *self;
+
+	self = instance;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == TTP_TSAP_MAGIC, return -1;);
+
+	if (get)
+		param->pv.i = self->tx_max_sdu_size;
+	else
+		self->tx_max_sdu_size = param->pv.i;
+
+	pr_debug("%s(), MaxSduSize=%d\n", __func__, param->pv.i);
+
+	return 0;
+}
+
+/*************************** CLIENT CALLS ***************************/
+/************************** LMP CALLBACKS **************************/
+/* Everything is happily mixed up. Waiting for next clean up - Jean II */
+
+/*
+ * Initialization, that has to be done on new tsap
+ * instance allocation and on duplication
+ */
+static void irttp_init_tsap(struct tsap_cb *tsap)
+{
+	spin_lock_init(&tsap->lock);
+	init_timer(&tsap->todo_timer);
+
+	skb_queue_head_init(&tsap->rx_queue);
+	skb_queue_head_init(&tsap->tx_queue);
+	skb_queue_head_init(&tsap->rx_fragments);
+}
+
+/*
+ * Function irttp_open_tsap (stsap, notify)
+ *
+ *    Create TSAP connection endpoint,
+ */
+struct tsap_cb *irttp_open_tsap(__u8 stsap_sel, int credit, notify_t *notify)
+{
+	struct tsap_cb *self;
+	struct lsap_cb *lsap;
+	notify_t ttp_notify;
+
+	IRDA_ASSERT(irttp->magic == TTP_MAGIC, return NULL;);
+
+	/* The IrLMP spec (IrLMP 1.1 p10) says that we have the right to
+	 * use only 0x01-0x6F. Of course, we can use LSAP_ANY as well.
+	 * JeanII */
+	if ((stsap_sel != LSAP_ANY) &&
+	   ((stsap_sel < 0x01) || (stsap_sel >= 0x70))) {
+		pr_debug("%s(), invalid tsap!\n", __func__);
+		return NULL;
+	}
+
+	self = kzalloc(sizeof(struct tsap_cb), GFP_ATOMIC);
+	if (self == NULL)
+		return NULL;
+
+	/* Initialize internal objects */
+	irttp_init_tsap(self);
+
+	/* Initialise todo timer */
+	self->todo_timer.data     = (unsigned long) self;
+	self->todo_timer.function = &irttp_todo_expired;
+
+	/* Initialize callbacks for IrLMP to use */
+	irda_notify_init(&ttp_notify);
+	ttp_notify.connect_confirm = irttp_connect_confirm;
+	ttp_notify.connect_indication = irttp_connect_indication;
+	ttp_notify.disconnect_indication = irttp_disconnect_indication;
+	ttp_notify.data_indication = irttp_data_indication;
+	ttp_notify.udata_indication = irttp_udata_indication;
+	ttp_notify.flow_indication = irttp_flow_indication;
+	if (notify->status_indication != NULL)
+		ttp_notify.status_indication = irttp_status_indication;
+	ttp_notify.instance = self;
+	strncpy(ttp_notify.name, notify->name, NOTIFY_MAX_NAME);
+
+	self->magic = TTP_TSAP_MAGIC;
+	self->connected = FALSE;
+
+	/*
+	 *  Create LSAP at IrLMP layer
+	 */
+	lsap = irlmp_open_lsap(stsap_sel, &ttp_notify, 0);
+	if (lsap == NULL) {
+		pr_debug("%s: unable to allocate LSAP!!\n", __func__);
+		__irttp_close_tsap(self);
+		return NULL;
+	}
+
+	/*
+	 *  If user specified LSAP_ANY as source TSAP selector, then IrLMP
+	 *  will replace it with whatever source selector which is free, so
+	 *  the stsap_sel we have might not be valid anymore
+	 */
+	self->stsap_sel = lsap->slsap_sel;
+	pr_debug("%s(), stsap_sel=%02x\n", __func__, self->stsap_sel);
+
+	self->notify = *notify;
+	self->lsap = lsap;
+
+	hashbin_insert(irttp->tsaps, (irda_queue_t *) self, (long) self, NULL);
+
+	if (credit > TTP_RX_MAX_CREDIT)
+		self->initial_credit = TTP_RX_MAX_CREDIT;
+	else
+		self->initial_credit = credit;
+
+	return self;
+}
+EXPORT_SYMBOL(irttp_open_tsap);
+
+/*
+ * Function irttp_close (handle)
+ *
+ *    Remove an instance of a TSAP. This function should only deal with the
+ *    deallocation of the TSAP, and resetting of the TSAPs values;
+ *
+ */
+static void __irttp_close_tsap(struct tsap_cb *self)
+{
+	/* First make sure we're connected. */
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == TTP_TSAP_MAGIC, return;);
+
+	irttp_flush_queues(self);
+
+	del_timer(&self->todo_timer);
+
+	/* This one won't be cleaned up if we are disconnect_pend + close_pend
+	 * and we receive a disconnect_indication */
+	if (self->disconnect_skb)
+		dev_kfree_skb(self->disconnect_skb);
+
+	self->connected = FALSE;
+	self->magic = ~TTP_TSAP_MAGIC;
+
+	kfree(self);
+}
+
+/*
+ * Function irttp_close (self)
+ *
+ *    Remove TSAP from list of all TSAPs and then deallocate all resources
+ *    associated with this TSAP
+ *
+ * Note : because we *free* the tsap structure, it is the responsibility
+ * of the caller to make sure we are called only once and to deal with
+ * possible race conditions. - Jean II
+ */
+int irttp_close_tsap(struct tsap_cb *self)
+{
+	struct tsap_cb *tsap;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == TTP_TSAP_MAGIC, return -1;);
+
+	/* Make sure tsap has been disconnected */
+	if (self->connected) {
+		/* Check if disconnect is not pending */
+		if (!test_bit(0, &self->disconnect_pend)) {
+			net_warn_ratelimited("%s: TSAP still connected!\n",
+					     __func__);
+			irttp_disconnect_request(self, NULL, P_NORMAL);
+		}
+		self->close_pend = TRUE;
+		irttp_start_todo_timer(self, HZ/10);
+
+		return 0; /* Will be back! */
+	}
+
+	tsap = hashbin_remove(irttp->tsaps, (long) self, NULL);
+
+	IRDA_ASSERT(tsap == self, return -1;);
+
+	/* Close corresponding LSAP */
+	if (self->lsap) {
+		irlmp_close_lsap(self->lsap);
+		self->lsap = NULL;
+	}
+
+	__irttp_close_tsap(self);
+
+	return 0;
+}
+EXPORT_SYMBOL(irttp_close_tsap);
+
+/*
+ * Function irttp_udata_request (self, skb)
+ *
+ *    Send unreliable data on this TSAP
+ *
+ */
+int irttp_udata_request(struct tsap_cb *self, struct sk_buff *skb)
+{
+	int ret;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == TTP_TSAP_MAGIC, return -1;);
+	IRDA_ASSERT(skb != NULL, return -1;);
+
+	/* Take shortcut on zero byte packets */
+	if (skb->len == 0) {
+		ret = 0;
+		goto err;
+	}
+
+	/* Check that nothing bad happens */
+	if (!self->connected) {
+		net_warn_ratelimited("%s(), Not connected\n", __func__);
+		ret = -ENOTCONN;
+		goto err;
+	}
+
+	if (skb->len > self->max_seg_size) {
+		net_err_ratelimited("%s(), UData is too large for IrLAP!\n",
+				    __func__);
+		ret = -EMSGSIZE;
+		goto err;
+	}
+
+	irlmp_udata_request(self->lsap, skb);
+	self->stats.tx_packets++;
+
+	return 0;
+
+err:
+	dev_kfree_skb(skb);
+	return ret;
+}
+EXPORT_SYMBOL(irttp_udata_request);
+
+
+/*
+ * Function irttp_data_request (handle, skb)
+ *
+ *    Queue frame for transmission. If SAR is enabled, fragement the frame
+ *    and queue the fragments for transmission
+ */
+int irttp_data_request(struct tsap_cb *self, struct sk_buff *skb)
+{
+	__u8 *frame;
+	int ret;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == TTP_TSAP_MAGIC, return -1;);
+	IRDA_ASSERT(skb != NULL, return -1;);
+
+	pr_debug("%s() : queue len = %d\n", __func__,
+		 skb_queue_len(&self->tx_queue));
+
+	/* Take shortcut on zero byte packets */
+	if (skb->len == 0) {
+		ret = 0;
+		goto err;
+	}
+
+	/* Check that nothing bad happens */
+	if (!self->connected) {
+		net_warn_ratelimited("%s: Not connected\n", __func__);
+		ret = -ENOTCONN;
+		goto err;
+	}
+
+	/*
+	 *  Check if SAR is disabled, and the frame is larger than what fits
+	 *  inside an IrLAP frame
+	 */
+	if ((self->tx_max_sdu_size == 0) && (skb->len > self->max_seg_size)) {
+		net_err_ratelimited("%s: SAR disabled, and data is too large for IrLAP!\n",
+				    __func__);
+		ret = -EMSGSIZE;
+		goto err;
+	}
+
+	/*
+	 *  Check if SAR is enabled, and the frame is larger than the
+	 *  TxMaxSduSize
+	 */
+	if ((self->tx_max_sdu_size != 0) &&
+	    (self->tx_max_sdu_size != TTP_SAR_UNBOUND) &&
+	    (skb->len > self->tx_max_sdu_size)) {
+		net_err_ratelimited("%s: SAR enabled, but data is larger than TxMaxSduSize!\n",
+				    __func__);
+		ret = -EMSGSIZE;
+		goto err;
+	}
+	/*
+	 *  Check if transmit queue is full
+	 */
+	if (skb_queue_len(&self->tx_queue) >= TTP_TX_MAX_QUEUE) {
+		/*
+		 *  Give it a chance to empty itself
+		 */
+		irttp_run_tx_queue(self);
+
+		/* Drop packet. This error code should trigger the caller
+		 * to resend the data in the client code - Jean II */
+		ret = -ENOBUFS;
+		goto err;
+	}
+
+	/* Queue frame, or queue frame segments */
+	if ((self->tx_max_sdu_size == 0) || (skb->len < self->max_seg_size)) {
+		/* Queue frame */
+		IRDA_ASSERT(skb_headroom(skb) >= TTP_HEADER, return -1;);
+		frame = skb_push(skb, TTP_HEADER);
+		frame[0] = 0x00; /* Clear more bit */
+
+		skb_queue_tail(&self->tx_queue, skb);
+	} else {
+		/*
+		 *  Fragment the frame, this function will also queue the
+		 *  fragments, we don't care about the fact the transmit
+		 *  queue may be overfilled by all the segments for a little
+		 *  while
+		 */
+		irttp_fragment_skb(self, skb);
+	}
+
+	/* Check if we can accept more data from client */
+	if ((!self->tx_sdu_busy) &&
+	    (skb_queue_len(&self->tx_queue) > TTP_TX_HIGH_THRESHOLD)) {
+		/* Tx queue filling up, so stop client. */
+		if (self->notify.flow_indication) {
+			self->notify.flow_indication(self->notify.instance,
+						     self, FLOW_STOP);
+		}
+		/* self->tx_sdu_busy is the state of the client.
+		 * Update state after notifying client to avoid
+		 * race condition with irttp_flow_indication().
+		 * If the queue empty itself after our test but before
+		 * we set the flag, we will fix ourselves below in
+		 * irttp_run_tx_queue().
+		 * Jean II */
+		self->tx_sdu_busy = TRUE;
+	}
+
+	/* Try to make some progress */
+	irttp_run_tx_queue(self);
+
+	return 0;
+
+err:
+	dev_kfree_skb(skb);
+	return ret;
+}
+EXPORT_SYMBOL(irttp_data_request);
+
+/*
+ * Function irttp_run_tx_queue (self)
+ *
+ *    Transmit packets queued for transmission (if possible)
+ *
+ */
+static void irttp_run_tx_queue(struct tsap_cb *self)
+{
+	struct sk_buff *skb;
+	unsigned long flags;
+	int n;
+
+	pr_debug("%s() : send_credit = %d, queue_len = %d\n",
+		 __func__,
+		 self->send_credit, skb_queue_len(&self->tx_queue));
+
+	/* Get exclusive access to the tx queue, otherwise don't touch it */
+	if (irda_lock(&self->tx_queue_lock) == FALSE)
+		return;
+
+	/* Try to send out frames as long as we have credits
+	 * and as long as LAP is not full. If LAP is full, it will
+	 * poll us through irttp_flow_indication() - Jean II */
+	while ((self->send_credit > 0) &&
+	       (!irlmp_lap_tx_queue_full(self->lsap)) &&
+	       (skb = skb_dequeue(&self->tx_queue))) {
+		/*
+		 *  Since we can transmit and receive frames concurrently,
+		 *  the code below is a critical region and we must assure that
+		 *  nobody messes with the credits while we update them.
+		 */
+		spin_lock_irqsave(&self->lock, flags);
+
+		n = self->avail_credit;
+		self->avail_credit = 0;
+
+		/* Only room for 127 credits in frame */
+		if (n > 127) {
+			self->avail_credit = n-127;
+			n = 127;
+		}
+		self->remote_credit += n;
+		self->send_credit--;
+
+		spin_unlock_irqrestore(&self->lock, flags);
+
+		/*
+		 *  More bit must be set by the data_request() or fragment()
+		 *  functions
+		 */
+		skb->data[0] |= (n & 0x7f);
+
+		/* Detach from socket.
+		 * The current skb has a reference to the socket that sent
+		 * it (skb->sk). When we pass it to IrLMP, the skb will be
+		 * stored in in IrLAP (self->wx_list). When we are within
+		 * IrLAP, we lose the notion of socket, so we should not
+		 * have a reference to a socket. So, we drop it here.
+		 *
+		 * Why does it matter ?
+		 * When the skb is freed (kfree_skb), if it is associated
+		 * with a socket, it release buffer space on the socket
+		 * (through sock_wfree() and sock_def_write_space()).
+		 * If the socket no longer exist, we may crash. Hard.
+		 * When we close a socket, we make sure that associated packets
+		 * in IrTTP are freed. However, we have no way to cancel
+		 * the packet that we have passed to IrLAP. So, if a packet
+		 * remains in IrLAP (retry on the link or else) after we
+		 * close the socket, we are dead !
+		 * Jean II */
+		if (skb->sk != NULL) {
+			/* IrSOCK application, IrOBEX, ... */
+			skb_orphan(skb);
+		}
+			/* IrCOMM over IrTTP, IrLAN, ... */
+
+		/* Pass the skb to IrLMP - done */
+		irlmp_data_request(self->lsap, skb);
+		self->stats.tx_packets++;
+	}
+
+	/* Check if we can accept more frames from client.
+	 * We don't want to wait until the todo timer to do that, and we
+	 * can't use tasklets (grr...), so we are obliged to give control
+	 * to client. That's ok, this test will be true not too often
+	 * (max once per LAP window) and we are called from places
+	 * where we can spend a bit of time doing stuff. - Jean II */
+	if ((self->tx_sdu_busy) &&
+	    (skb_queue_len(&self->tx_queue) < TTP_TX_LOW_THRESHOLD) &&
+	    (!self->close_pend)) {
+		if (self->notify.flow_indication)
+			self->notify.flow_indication(self->notify.instance,
+						     self, FLOW_START);
+
+		/* self->tx_sdu_busy is the state of the client.
+		 * We don't really have a race here, but it's always safer
+		 * to update our state after the client - Jean II */
+		self->tx_sdu_busy = FALSE;
+	}
+
+	/* Reset lock */
+	self->tx_queue_lock = 0;
+}
+
+/*
+ * Function irttp_give_credit (self)
+ *
+ *    Send a dataless flowdata TTP-PDU and give available credit to peer
+ *    TSAP
+ */
+static inline void irttp_give_credit(struct tsap_cb *self)
+{
+	struct sk_buff *tx_skb = NULL;
+	unsigned long flags;
+	int n;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == TTP_TSAP_MAGIC, return;);
+
+	pr_debug("%s() send=%d,avail=%d,remote=%d\n",
+		 __func__,
+		 self->send_credit, self->avail_credit, self->remote_credit);
+
+	/* Give credit to peer */
+	tx_skb = alloc_skb(TTP_MAX_HEADER, GFP_ATOMIC);
+	if (!tx_skb)
+		return;
+
+	/* Reserve space for LMP, and LAP header */
+	skb_reserve(tx_skb, LMP_MAX_HEADER);
+
+	/*
+	 *  Since we can transmit and receive frames concurrently,
+	 *  the code below is a critical region and we must assure that
+	 *  nobody messes with the credits while we update them.
+	 */
+	spin_lock_irqsave(&self->lock, flags);
+
+	n = self->avail_credit;
+	self->avail_credit = 0;
+
+	/* Only space for 127 credits in frame */
+	if (n > 127) {
+		self->avail_credit = n - 127;
+		n = 127;
+	}
+	self->remote_credit += n;
+
+	spin_unlock_irqrestore(&self->lock, flags);
+
+	skb_put(tx_skb, 1);
+	tx_skb->data[0] = (__u8) (n & 0x7f);
+
+	irlmp_data_request(self->lsap, tx_skb);
+	self->stats.tx_packets++;
+}
+
+/*
+ * Function irttp_udata_indication (instance, sap, skb)
+ *
+ *    Received some unit-data (unreliable)
+ *
+ */
+static int irttp_udata_indication(void *instance, void *sap,
+				  struct sk_buff *skb)
+{
+	struct tsap_cb *self;
+	int err;
+
+	self = instance;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == TTP_TSAP_MAGIC, return -1;);
+	IRDA_ASSERT(skb != NULL, return -1;);
+
+	self->stats.rx_packets++;
+
+	/* Just pass data to layer above */
+	if (self->notify.udata_indication) {
+		err = self->notify.udata_indication(self->notify.instance,
+						    self, skb);
+		/* Same comment as in irttp_do_data_indication() */
+		if (!err)
+			return 0;
+	}
+	/* Either no handler, or handler returns an error */
+	dev_kfree_skb(skb);
+
+	return 0;
+}
+
+/*
+ * Function irttp_data_indication (instance, sap, skb)
+ *
+ *    Receive segment from IrLMP.
+ *
+ */
+static int irttp_data_indication(void *instance, void *sap,
+				 struct sk_buff *skb)
+{
+	struct tsap_cb *self;
+	unsigned long flags;
+	int n;
+
+	self = instance;
+
+	n = skb->data[0] & 0x7f;     /* Extract the credits */
+
+	self->stats.rx_packets++;
+
+	/*  Deal with inbound credit
+	 *  Since we can transmit and receive frames concurrently,
+	 *  the code below is a critical region and we must assure that
+	 *  nobody messes with the credits while we update them.
+	 */
+	spin_lock_irqsave(&self->lock, flags);
+	self->send_credit += n;
+	if (skb->len > 1)
+		self->remote_credit--;
+	spin_unlock_irqrestore(&self->lock, flags);
+
+	/*
+	 *  Data or dataless packet? Dataless frames contains only the
+	 *  TTP_HEADER.
+	 */
+	if (skb->len > 1) {
+		/*
+		 *  We don't remove the TTP header, since we must preserve the
+		 *  more bit, so the defragment routing knows what to do
+		 */
+		skb_queue_tail(&self->rx_queue, skb);
+	} else {
+		/* Dataless flowdata TTP-PDU */
+		dev_kfree_skb(skb);
+	}
+
+
+	/* Push data to the higher layer.
+	 * We do it synchronously because running the todo timer for each
+	 * receive packet would be too much overhead and latency.
+	 * By passing control to the higher layer, we run the risk that
+	 * it may take time or grab a lock. Most often, the higher layer
+	 * will only put packet in a queue.
+	 * Anyway, packets are only dripping through the IrDA, so we can
+	 * have time before the next packet.
+	 * Further, we are run from NET_BH, so the worse that can happen is
+	 * us missing the optimal time to send back the PF bit in LAP.
+	 * Jean II */
+	irttp_run_rx_queue(self);
+
+	/* We now give credits to peer in irttp_run_rx_queue().
+	 * We need to send credit *NOW*, otherwise we are going
+	 * to miss the next Tx window. The todo timer may take
+	 * a while before it's run... - Jean II */
+
+	/*
+	 * If the peer device has given us some credits and we didn't have
+	 * anyone from before, then we need to shedule the tx queue.
+	 * We need to do that because our Tx have stopped (so we may not
+	 * get any LAP flow indication) and the user may be stopped as
+	 * well. - Jean II
+	 */
+	if (self->send_credit == n) {
+		/* Restart pushing stuff to LAP */
+		irttp_run_tx_queue(self);
+		/* Note : we don't want to schedule the todo timer
+		 * because it has horrible latency. No tasklets
+		 * because the tasklet API is broken. - Jean II */
+	}
+
+	return 0;
+}
+
+/*
+ * Function irttp_status_indication (self, reason)
+ *
+ *    Status_indication, just pass to the higher layer...
+ *
+ */
+static void irttp_status_indication(void *instance,
+				    LINK_STATUS link, LOCK_STATUS lock)
+{
+	struct tsap_cb *self;
+
+	self = instance;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == TTP_TSAP_MAGIC, return;);
+
+	/* Check if client has already closed the TSAP and gone away */
+	if (self->close_pend)
+		return;
+
+	/*
+	 *  Inform service user if he has requested it
+	 */
+	if (self->notify.status_indication != NULL)
+		self->notify.status_indication(self->notify.instance,
+					       link, lock);
+	else
+		pr_debug("%s(), no handler\n", __func__);
+}
+
+/*
+ * Function irttp_flow_indication (self, reason)
+ *
+ *    Flow_indication : IrLAP tells us to send more data.
+ *
+ */
+static void irttp_flow_indication(void *instance, void *sap, LOCAL_FLOW flow)
+{
+	struct tsap_cb *self;
+
+	self = instance;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == TTP_TSAP_MAGIC, return;);
+
+	pr_debug("%s(instance=%p)\n", __func__, self);
+
+	/* We are "polled" directly from LAP, and the LAP want to fill
+	 * its Tx window. We want to do our best to send it data, so that
+	 * we maximise the window. On the other hand, we want to limit the
+	 * amount of work here so that LAP doesn't hang forever waiting
+	 * for packets. - Jean II */
+
+	/* Try to send some packets. Currently, LAP calls us every time
+	 * there is one free slot, so we will send only one packet.
+	 * This allow the scheduler to do its round robin - Jean II */
+	irttp_run_tx_queue(self);
+
+	/* Note regarding the interraction with higher layer.
+	 * irttp_run_tx_queue() may call the client when its queue
+	 * start to empty, via notify.flow_indication(). Initially.
+	 * I wanted this to happen in a tasklet, to avoid client
+	 * grabbing the CPU, but we can't use tasklets safely. And timer
+	 * is definitely too slow.
+	 * This will happen only once per LAP window, and usually at
+	 * the third packet (unless window is smaller). LAP is still
+	 * doing mtt and sending first packet so it's sort of OK
+	 * to do that. Jean II */
+
+	/* If we need to send disconnect. try to do it now */
+	if (self->disconnect_pend)
+		irttp_start_todo_timer(self, 0);
+}
+
+/*
+ * Function irttp_flow_request (self, command)
+ *
+ *    This function could be used by the upper layers to tell IrTTP to stop
+ *    delivering frames if the receive queues are starting to get full, or
+ *    to tell IrTTP to start delivering frames again.
+ */
+void irttp_flow_request(struct tsap_cb *self, LOCAL_FLOW flow)
+{
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == TTP_TSAP_MAGIC, return;);
+
+	switch (flow) {
+	case FLOW_STOP:
+		pr_debug("%s(), flow stop\n", __func__);
+		self->rx_sdu_busy = TRUE;
+		break;
+	case FLOW_START:
+		pr_debug("%s(), flow start\n", __func__);
+		self->rx_sdu_busy = FALSE;
+
+		/* Client say he can accept more data, try to free our
+		 * queues ASAP - Jean II */
+		irttp_run_rx_queue(self);
+
+		break;
+	default:
+		pr_debug("%s(), Unknown flow command!\n", __func__);
+	}
+}
+EXPORT_SYMBOL(irttp_flow_request);
+
+/*
+ * Function irttp_connect_request (self, dtsap_sel, daddr, qos)
+ *
+ *    Try to connect to remote destination TSAP selector
+ *
+ */
+int irttp_connect_request(struct tsap_cb *self, __u8 dtsap_sel,
+			  __u32 saddr, __u32 daddr,
+			  struct qos_info *qos, __u32 max_sdu_size,
+			  struct sk_buff *userdata)
+{
+	struct sk_buff *tx_skb;
+	__u8 *frame;
+	__u8 n;
+
+	pr_debug("%s(), max_sdu_size=%d\n", __func__, max_sdu_size);
+
+	IRDA_ASSERT(self != NULL, return -EBADR;);
+	IRDA_ASSERT(self->magic == TTP_TSAP_MAGIC, return -EBADR;);
+
+	if (self->connected) {
+		if (userdata)
+			dev_kfree_skb(userdata);
+		return -EISCONN;
+	}
+
+	/* Any userdata supplied? */
+	if (userdata == NULL) {
+		tx_skb = alloc_skb(TTP_MAX_HEADER + TTP_SAR_HEADER,
+				   GFP_ATOMIC);
+		if (!tx_skb)
+			return -ENOMEM;
+
+		/* Reserve space for MUX_CONTROL and LAP header */
+		skb_reserve(tx_skb, TTP_MAX_HEADER + TTP_SAR_HEADER);
+	} else {
+		tx_skb = userdata;
+		/*
+		 *  Check that the client has reserved enough space for
+		 *  headers
+		 */
+		IRDA_ASSERT(skb_headroom(userdata) >= TTP_MAX_HEADER,
+			{ dev_kfree_skb(userdata); return -1; });
+	}
+
+	/* Initialize connection parameters */
+	self->connected = FALSE;
+	self->avail_credit = 0;
+	self->rx_max_sdu_size = max_sdu_size;
+	self->rx_sdu_size = 0;
+	self->rx_sdu_busy = FALSE;
+	self->dtsap_sel = dtsap_sel;
+
+	n = self->initial_credit;
+
+	self->remote_credit = 0;
+	self->send_credit = 0;
+
+	/*
+	 *  Give away max 127 credits for now
+	 */
+	if (n > 127) {
+		self->avail_credit = n - 127;
+		n = 127;
+	}
+
+	self->remote_credit = n;
+
+	/* SAR enabled? */
+	if (max_sdu_size > 0) {
+		IRDA_ASSERT(skb_headroom(tx_skb) >= (TTP_MAX_HEADER + TTP_SAR_HEADER),
+			{ dev_kfree_skb(tx_skb); return -1; });
+
+		/* Insert SAR parameters */
+		frame = skb_push(tx_skb, TTP_HEADER + TTP_SAR_HEADER);
+
+		frame[0] = TTP_PARAMETERS | n;
+		frame[1] = 0x04; /* Length */
+		frame[2] = 0x01; /* MaxSduSize */
+		frame[3] = 0x02; /* Value length */
+
+		put_unaligned(cpu_to_be16((__u16) max_sdu_size),
+			      (__be16 *)(frame+4));
+	} else {
+		/* Insert plain TTP header */
+		frame = skb_push(tx_skb, TTP_HEADER);
+
+		/* Insert initial credit in frame */
+		frame[0] = n & 0x7f;
+	}
+
+	/* Connect with IrLMP. No QoS parameters for now */
+	return irlmp_connect_request(self->lsap, dtsap_sel, saddr, daddr, qos,
+				     tx_skb);
+}
+EXPORT_SYMBOL(irttp_connect_request);
+
+/*
+ * Function irttp_connect_confirm (handle, qos, skb)
+ *
+ *    Service user confirms TSAP connection with peer.
+ *
+ */
+static void irttp_connect_confirm(void *instance, void *sap,
+				  struct qos_info *qos, __u32 max_seg_size,
+				  __u8 max_header_size, struct sk_buff *skb)
+{
+	struct tsap_cb *self;
+	int parameters;
+	int ret;
+	__u8 plen;
+	__u8 n;
+
+	self = instance;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == TTP_TSAP_MAGIC, return;);
+	IRDA_ASSERT(skb != NULL, return;);
+
+	self->max_seg_size = max_seg_size - TTP_HEADER;
+	self->max_header_size = max_header_size + TTP_HEADER;
+
+	/*
+	 *  Check if we have got some QoS parameters back! This should be the
+	 *  negotiated QoS for the link.
+	 */
+	if (qos) {
+		pr_debug("IrTTP, Negotiated BAUD_RATE: %02x\n",
+			 qos->baud_rate.bits);
+		pr_debug("IrTTP, Negotiated BAUD_RATE: %d bps.\n",
+			 qos->baud_rate.value);
+	}
+
+	n = skb->data[0] & 0x7f;
+
+	pr_debug("%s(), Initial send_credit=%d\n", __func__, n);
+
+	self->send_credit = n;
+	self->tx_max_sdu_size = 0;
+	self->connected = TRUE;
+
+	parameters = skb->data[0] & 0x80;
+
+	IRDA_ASSERT(skb->len >= TTP_HEADER, return;);
+	skb_pull(skb, TTP_HEADER);
+
+	if (parameters) {
+		plen = skb->data[0];
+
+		ret = irda_param_extract_all(self, skb->data+1,
+					     IRDA_MIN(skb->len-1, plen),
+					     &param_info);
+
+		/* Any errors in the parameter list? */
+		if (ret < 0) {
+			net_warn_ratelimited("%s: error extracting parameters\n",
+					     __func__);
+			dev_kfree_skb(skb);
+
+			/* Do not accept this connection attempt */
+			return;
+		}
+		/* Remove parameters */
+		skb_pull(skb, IRDA_MIN(skb->len, plen+1));
+	}
+
+	pr_debug("%s() send=%d,avail=%d,remote=%d\n", __func__,
+		 self->send_credit, self->avail_credit, self->remote_credit);
+
+	pr_debug("%s(), MaxSduSize=%d\n", __func__,
+		 self->tx_max_sdu_size);
+
+	if (self->notify.connect_confirm) {
+		self->notify.connect_confirm(self->notify.instance, self, qos,
+					     self->tx_max_sdu_size,
+					     self->max_header_size, skb);
+	} else
+		dev_kfree_skb(skb);
+}
+
+/*
+ * Function irttp_connect_indication (handle, skb)
+ *
+ *    Some other device is connecting to this TSAP
+ *
+ */
+static void irttp_connect_indication(void *instance, void *sap,
+		struct qos_info *qos, __u32 max_seg_size, __u8 max_header_size,
+		struct sk_buff *skb)
+{
+	struct tsap_cb *self;
+	struct lsap_cb *lsap;
+	int parameters;
+	int ret;
+	__u8 plen;
+	__u8 n;
+
+	self = instance;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == TTP_TSAP_MAGIC, return;);
+	IRDA_ASSERT(skb != NULL, return;);
+
+	lsap = sap;
+
+	self->max_seg_size = max_seg_size - TTP_HEADER;
+	self->max_header_size = max_header_size+TTP_HEADER;
+
+	pr_debug("%s(), TSAP sel=%02x\n", __func__, self->stsap_sel);
+
+	/* Need to update dtsap_sel if its equal to LSAP_ANY */
+	self->dtsap_sel = lsap->dlsap_sel;
+
+	n = skb->data[0] & 0x7f;
+
+	self->send_credit = n;
+	self->tx_max_sdu_size = 0;
+
+	parameters = skb->data[0] & 0x80;
+
+	IRDA_ASSERT(skb->len >= TTP_HEADER, return;);
+	skb_pull(skb, TTP_HEADER);
+
+	if (parameters) {
+		plen = skb->data[0];
+
+		ret = irda_param_extract_all(self, skb->data+1,
+					     IRDA_MIN(skb->len-1, plen),
+					     &param_info);
+
+		/* Any errors in the parameter list? */
+		if (ret < 0) {
+			net_warn_ratelimited("%s: error extracting parameters\n",
+					     __func__);
+			dev_kfree_skb(skb);
+
+			/* Do not accept this connection attempt */
+			return;
+		}
+
+		/* Remove parameters */
+		skb_pull(skb, IRDA_MIN(skb->len, plen+1));
+	}
+
+	if (self->notify.connect_indication) {
+		self->notify.connect_indication(self->notify.instance, self,
+						qos, self->tx_max_sdu_size,
+						self->max_header_size, skb);
+	} else
+		dev_kfree_skb(skb);
+}
+
+/*
+ * Function irttp_connect_response (handle, userdata)
+ *
+ *    Service user is accepting the connection, just pass it down to
+ *    IrLMP!
+ *
+ */
+int irttp_connect_response(struct tsap_cb *self, __u32 max_sdu_size,
+			   struct sk_buff *userdata)
+{
+	struct sk_buff *tx_skb;
+	__u8 *frame;
+	int ret;
+	__u8 n;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == TTP_TSAP_MAGIC, return -1;);
+
+	pr_debug("%s(), Source TSAP selector=%02x\n", __func__,
+		 self->stsap_sel);
+
+	/* Any userdata supplied? */
+	if (userdata == NULL) {
+		tx_skb = alloc_skb(TTP_MAX_HEADER + TTP_SAR_HEADER,
+				   GFP_ATOMIC);
+		if (!tx_skb)
+			return -ENOMEM;
+
+		/* Reserve space for MUX_CONTROL and LAP header */
+		skb_reserve(tx_skb, TTP_MAX_HEADER + TTP_SAR_HEADER);
+	} else {
+		tx_skb = userdata;
+		/*
+		 *  Check that the client has reserved enough space for
+		 *  headers
+		 */
+		IRDA_ASSERT(skb_headroom(userdata) >= TTP_MAX_HEADER,
+			{ dev_kfree_skb(userdata); return -1; });
+	}
+
+	self->avail_credit = 0;
+	self->remote_credit = 0;
+	self->rx_max_sdu_size = max_sdu_size;
+	self->rx_sdu_size = 0;
+	self->rx_sdu_busy = FALSE;
+
+	n = self->initial_credit;
+
+	/* Frame has only space for max 127 credits (7 bits) */
+	if (n > 127) {
+		self->avail_credit = n - 127;
+		n = 127;
+	}
+
+	self->remote_credit = n;
+	self->connected = TRUE;
+
+	/* SAR enabled? */
+	if (max_sdu_size > 0) {
+		IRDA_ASSERT(skb_headroom(tx_skb) >= (TTP_MAX_HEADER + TTP_SAR_HEADER),
+			{ dev_kfree_skb(tx_skb); return -1; });
+
+		/* Insert TTP header with SAR parameters */
+		frame = skb_push(tx_skb, TTP_HEADER + TTP_SAR_HEADER);
+
+		frame[0] = TTP_PARAMETERS | n;
+		frame[1] = 0x04; /* Length */
+
+		/* irda_param_insert(self, IRTTP_MAX_SDU_SIZE, frame+1,  */
+/*				  TTP_SAR_HEADER, &param_info) */
+
+		frame[2] = 0x01; /* MaxSduSize */
+		frame[3] = 0x02; /* Value length */
+
+		put_unaligned(cpu_to_be16((__u16) max_sdu_size),
+			      (__be16 *)(frame+4));
+	} else {
+		/* Insert TTP header */
+		frame = skb_push(tx_skb, TTP_HEADER);
+
+		frame[0] = n & 0x7f;
+	}
+
+	ret = irlmp_connect_response(self->lsap, tx_skb);
+
+	return ret;
+}
+EXPORT_SYMBOL(irttp_connect_response);
+
+/*
+ * Function irttp_dup (self, instance)
+ *
+ *    Duplicate TSAP, can be used by servers to confirm a connection on a
+ *    new TSAP so it can keep listening on the old one.
+ */
+struct tsap_cb *irttp_dup(struct tsap_cb *orig, void *instance)
+{
+	struct tsap_cb *new;
+	unsigned long flags;
+
+	/* Protect our access to the old tsap instance */
+	spin_lock_irqsave(&irttp->tsaps->hb_spinlock, flags);
+
+	/* Find the old instance */
+	if (!hashbin_find(irttp->tsaps, (long) orig, NULL)) {
+		pr_debug("%s(), unable to find TSAP\n", __func__);
+		spin_unlock_irqrestore(&irttp->tsaps->hb_spinlock, flags);
+		return NULL;
+	}
+
+	/* Allocate a new instance */
+	new = kmemdup(orig, sizeof(struct tsap_cb), GFP_ATOMIC);
+	if (!new) {
+		pr_debug("%s(), unable to kmalloc\n", __func__);
+		spin_unlock_irqrestore(&irttp->tsaps->hb_spinlock, flags);
+		return NULL;
+	}
+	spin_lock_init(&new->lock);
+
+	/* We don't need the old instance any more */
+	spin_unlock_irqrestore(&irttp->tsaps->hb_spinlock, flags);
+
+	/* Try to dup the LSAP (may fail if we were too slow) */
+	new->lsap = irlmp_dup(orig->lsap, new);
+	if (!new->lsap) {
+		pr_debug("%s(), dup failed!\n", __func__);
+		kfree(new);
+		return NULL;
+	}
+
+	/* Not everything should be copied */
+	new->notify.instance = instance;
+
+	/* Initialize internal objects */
+	irttp_init_tsap(new);
+
+	/* This is locked */
+	hashbin_insert(irttp->tsaps, (irda_queue_t *) new, (long) new, NULL);
+
+	return new;
+}
+EXPORT_SYMBOL(irttp_dup);
+
+/*
+ * Function irttp_disconnect_request (self)
+ *
+ *    Close this connection please! If priority is high, the queued data
+ *    segments, if any, will be deallocated first
+ *
+ */
+int irttp_disconnect_request(struct tsap_cb *self, struct sk_buff *userdata,
+			     int priority)
+{
+	int ret;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == TTP_TSAP_MAGIC, return -1;);
+
+	/* Already disconnected? */
+	if (!self->connected) {
+		pr_debug("%s(), already disconnected!\n", __func__);
+		if (userdata)
+			dev_kfree_skb(userdata);
+		return -1;
+	}
+
+	/* Disconnect already pending ?
+	 * We need to use an atomic operation to prevent reentry. This
+	 * function may be called from various context, like user, timer
+	 * for following a disconnect_indication() (i.e. net_bh).
+	 * Jean II */
+	if (test_and_set_bit(0, &self->disconnect_pend)) {
+		pr_debug("%s(), disconnect already pending\n",
+			 __func__);
+		if (userdata)
+			dev_kfree_skb(userdata);
+
+		/* Try to make some progress */
+		irttp_run_tx_queue(self);
+		return -1;
+	}
+
+	/*
+	 *  Check if there is still data segments in the transmit queue
+	 */
+	if (!skb_queue_empty(&self->tx_queue)) {
+		if (priority == P_HIGH) {
+			/*
+			 *  No need to send the queued data, if we are
+			 *  disconnecting right now since the data will
+			 *  not have any usable connection to be sent on
+			 */
+			pr_debug("%s(): High priority!!()\n", __func__);
+			irttp_flush_queues(self);
+		} else if (priority == P_NORMAL) {
+			/*
+			 *  Must delay disconnect until after all data segments
+			 *  have been sent and the tx_queue is empty
+			 */
+			/* We'll reuse this one later for the disconnect */
+			self->disconnect_skb = userdata;  /* May be NULL */
+
+			irttp_run_tx_queue(self);
+
+			irttp_start_todo_timer(self, HZ/10);
+			return -1;
+		}
+	}
+	/* Note : we don't need to check if self->rx_queue is full and the
+	 * state of self->rx_sdu_busy because the disconnect response will
+	 * be sent at the LMP level (so even if the peer has its Tx queue
+	 * full of data). - Jean II */
+
+	pr_debug("%s(), Disconnecting ...\n", __func__);
+	self->connected = FALSE;
+
+	if (!userdata) {
+		struct sk_buff *tx_skb;
+		tx_skb = alloc_skb(LMP_MAX_HEADER, GFP_ATOMIC);
+		if (!tx_skb)
+			return -ENOMEM;
+
+		/*
+		 *  Reserve space for MUX and LAP header
+		 */
+		skb_reserve(tx_skb, LMP_MAX_HEADER);
+
+		userdata = tx_skb;
+	}
+	ret = irlmp_disconnect_request(self->lsap, userdata);
+
+	/* The disconnect is no longer pending */
+	clear_bit(0, &self->disconnect_pend);	/* FALSE */
+
+	return ret;
+}
+EXPORT_SYMBOL(irttp_disconnect_request);
+
+/*
+ * Function irttp_disconnect_indication (self, reason)
+ *
+ *    Disconnect indication, TSAP disconnected by peer?
+ *
+ */
+static void irttp_disconnect_indication(void *instance, void *sap,
+		LM_REASON reason, struct sk_buff *skb)
+{
+	struct tsap_cb *self;
+
+	self = instance;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == TTP_TSAP_MAGIC, return;);
+
+	/* Prevent higher layer to send more data */
+	self->connected = FALSE;
+
+	/* Check if client has already tried to close the TSAP */
+	if (self->close_pend) {
+		/* In this case, the higher layer is probably gone. Don't
+		 * bother it and clean up the remains - Jean II */
+		if (skb)
+			dev_kfree_skb(skb);
+		irttp_close_tsap(self);
+		return;
+	}
+
+	/* If we are here, we assume that is the higher layer is still
+	 * waiting for the disconnect notification and able to process it,
+	 * even if he tried to disconnect. Otherwise, it would have already
+	 * attempted to close the tsap and self->close_pend would be TRUE.
+	 * Jean II */
+
+	/* No need to notify the client if has already tried to disconnect */
+	if (self->notify.disconnect_indication)
+		self->notify.disconnect_indication(self->notify.instance, self,
+						   reason, skb);
+	else
+		if (skb)
+			dev_kfree_skb(skb);
+}
+
+/*
+ * Function irttp_do_data_indication (self, skb)
+ *
+ *    Try to deliver reassembled skb to layer above, and requeue it if that
+ *    for some reason should fail. We mark rx sdu as busy to apply back
+ *    pressure is necessary.
+ */
+static void irttp_do_data_indication(struct tsap_cb *self, struct sk_buff *skb)
+{
+	int err;
+
+	/* Check if client has already closed the TSAP and gone away */
+	if (self->close_pend) {
+		dev_kfree_skb(skb);
+		return;
+	}
+
+	err = self->notify.data_indication(self->notify.instance, self, skb);
+
+	/* Usually the layer above will notify that it's input queue is
+	 * starting to get filled by using the flow request, but this may
+	 * be difficult, so it can instead just refuse to eat it and just
+	 * give an error back
+	 */
+	if (err) {
+		pr_debug("%s() requeueing skb!\n", __func__);
+
+		/* Make sure we take a break */
+		self->rx_sdu_busy = TRUE;
+
+		/* Need to push the header in again */
+		skb_push(skb, TTP_HEADER);
+		skb->data[0] = 0x00; /* Make sure MORE bit is cleared */
+
+		/* Put skb back on queue */
+		skb_queue_head(&self->rx_queue, skb);
+	}
+}
+
+/*
+ * Function irttp_run_rx_queue (self)
+ *
+ *     Check if we have any frames to be transmitted, or if we have any
+ *     available credit to give away.
+ */
+static void irttp_run_rx_queue(struct tsap_cb *self)
+{
+	struct sk_buff *skb;
+	int more = 0;
+
+	pr_debug("%s() send=%d,avail=%d,remote=%d\n", __func__,
+		 self->send_credit, self->avail_credit, self->remote_credit);
+
+	/* Get exclusive access to the rx queue, otherwise don't touch it */
+	if (irda_lock(&self->rx_queue_lock) == FALSE)
+		return;
+
+	/*
+	 *  Reassemble all frames in receive queue and deliver them
+	 */
+	while (!self->rx_sdu_busy && (skb = skb_dequeue(&self->rx_queue))) {
+		/* This bit will tell us if it's the last fragment or not */
+		more = skb->data[0] & 0x80;
+
+		/* Remove TTP header */
+		skb_pull(skb, TTP_HEADER);
+
+		/* Add the length of the remaining data */
+		self->rx_sdu_size += skb->len;
+
+		/*
+		 * If SAR is disabled, or user has requested no reassembly
+		 * of received fragments then we just deliver them
+		 * immediately. This can be requested by clients that
+		 * implements byte streams without any message boundaries
+		 */
+		if (self->rx_max_sdu_size == TTP_SAR_DISABLE) {
+			irttp_do_data_indication(self, skb);
+			self->rx_sdu_size = 0;
+
+			continue;
+		}
+
+		/* Check if this is a fragment, and not the last fragment */
+		if (more) {
+			/*
+			 *  Queue the fragment if we still are within the
+			 *  limits of the maximum size of the rx_sdu
+			 */
+			if (self->rx_sdu_size <= self->rx_max_sdu_size) {
+				pr_debug("%s(), queueing frag\n",
+					 __func__);
+				skb_queue_tail(&self->rx_fragments, skb);
+			} else {
+				/* Free the part of the SDU that is too big */
+				dev_kfree_skb(skb);
+			}
+			continue;
+		}
+		/*
+		 *  This is the last fragment, so time to reassemble!
+		 */
+		if ((self->rx_sdu_size <= self->rx_max_sdu_size) ||
+		    (self->rx_max_sdu_size == TTP_SAR_UNBOUND)) {
+			/*
+			 * A little optimizing. Only queue the fragment if
+			 * there are other fragments. Since if this is the
+			 * last and only fragment, there is no need to
+			 * reassemble :-)
+			 */
+			if (!skb_queue_empty(&self->rx_fragments)) {
+				skb_queue_tail(&self->rx_fragments,
+					       skb);
+
+				skb = irttp_reassemble_skb(self);
+			}
+
+			/* Now we can deliver the reassembled skb */
+			irttp_do_data_indication(self, skb);
+		} else {
+			pr_debug("%s(), Truncated frame\n", __func__);
+
+			/* Free the part of the SDU that is too big */
+			dev_kfree_skb(skb);
+
+			/* Deliver only the valid but truncated part of SDU */
+			skb = irttp_reassemble_skb(self);
+
+			irttp_do_data_indication(self, skb);
+		}
+		self->rx_sdu_size = 0;
+	}
+
+	/*
+	 * It's not trivial to keep track of how many credits are available
+	 * by incrementing at each packet, because delivery may fail
+	 * (irttp_do_data_indication() may requeue the frame) and because
+	 * we need to take care of fragmentation.
+	 * We want the other side to send up to initial_credit packets.
+	 * We have some frames in our queues, and we have already allowed it
+	 * to send remote_credit.
+	 * No need to spinlock, write is atomic and self correcting...
+	 * Jean II
+	 */
+	self->avail_credit = (self->initial_credit -
+			      (self->remote_credit +
+			       skb_queue_len(&self->rx_queue) +
+			       skb_queue_len(&self->rx_fragments)));
+
+	/* Do we have too much credits to send to peer ? */
+	if ((self->remote_credit <= TTP_RX_MIN_CREDIT) &&
+	    (self->avail_credit > 0)) {
+		/* Send explicit credit frame */
+		irttp_give_credit(self);
+		/* Note : do *NOT* check if tx_queue is non-empty, that
+		 * will produce deadlocks. I repeat : send a credit frame
+		 * even if we have something to send in our Tx queue.
+		 * If we have credits, it means that our Tx queue is blocked.
+		 *
+		 * Let's suppose the peer can't keep up with our Tx. He will
+		 * flow control us by not sending us any credits, and we
+		 * will stop Tx and start accumulating credits here.
+		 * Up to the point where the peer will stop its Tx queue,
+		 * for lack of credits.
+		 * Let's assume the peer application is single threaded.
+		 * It will block on Tx and never consume any Rx buffer.
+		 * Deadlock. Guaranteed. - Jean II
+		 */
+	}
+
+	/* Reset lock */
+	self->rx_queue_lock = 0;
+}
+
+#ifdef CONFIG_PROC_FS
+struct irttp_iter_state {
+	int id;
+};
+
+static void *irttp_seq_start(struct seq_file *seq, loff_t *pos)
+{
+	struct irttp_iter_state *iter = seq->private;
+	struct tsap_cb *self;
+
+	/* Protect our access to the tsap list */
+	spin_lock_irq(&irttp->tsaps->hb_spinlock);
+	iter->id = 0;
+
+	for (self = (struct tsap_cb *) hashbin_get_first(irttp->tsaps);
+	     self != NULL;
+	     self = (struct tsap_cb *) hashbin_get_next(irttp->tsaps)) {
+		if (iter->id == *pos)
+			break;
+		++iter->id;
+	}
+
+	return self;
+}
+
+static void *irttp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
+{
+	struct irttp_iter_state *iter = seq->private;
+
+	++*pos;
+	++iter->id;
+	return (void *) hashbin_get_next(irttp->tsaps);
+}
+
+static void irttp_seq_stop(struct seq_file *seq, void *v)
+{
+	spin_unlock_irq(&irttp->tsaps->hb_spinlock);
+}
+
+static int irttp_seq_show(struct seq_file *seq, void *v)
+{
+	const struct irttp_iter_state *iter = seq->private;
+	const struct tsap_cb *self = v;
+
+	seq_printf(seq, "TSAP %d, ", iter->id);
+	seq_printf(seq, "stsap_sel: %02x, ",
+		   self->stsap_sel);
+	seq_printf(seq, "dtsap_sel: %02x\n",
+		   self->dtsap_sel);
+	seq_printf(seq, "  connected: %s, ",
+		   self->connected ? "TRUE" : "FALSE");
+	seq_printf(seq, "avail credit: %d, ",
+		   self->avail_credit);
+	seq_printf(seq, "remote credit: %d, ",
+		   self->remote_credit);
+	seq_printf(seq, "send credit: %d\n",
+		   self->send_credit);
+	seq_printf(seq, "  tx packets: %lu, ",
+		   self->stats.tx_packets);
+	seq_printf(seq, "rx packets: %lu, ",
+		   self->stats.rx_packets);
+	seq_printf(seq, "tx_queue len: %u ",
+		   skb_queue_len(&self->tx_queue));
+	seq_printf(seq, "rx_queue len: %u\n",
+		   skb_queue_len(&self->rx_queue));
+	seq_printf(seq, "  tx_sdu_busy: %s, ",
+		   self->tx_sdu_busy ? "TRUE" : "FALSE");
+	seq_printf(seq, "rx_sdu_busy: %s\n",
+		   self->rx_sdu_busy ? "TRUE" : "FALSE");
+	seq_printf(seq, "  max_seg_size: %u, ",
+		   self->max_seg_size);
+	seq_printf(seq, "tx_max_sdu_size: %u, ",
+		   self->tx_max_sdu_size);
+	seq_printf(seq, "rx_max_sdu_size: %u\n",
+		   self->rx_max_sdu_size);
+
+	seq_printf(seq, "  Used by (%s)\n\n",
+		   self->notify.name);
+	return 0;
+}
+
+static const struct seq_operations irttp_seq_ops = {
+	.start  = irttp_seq_start,
+	.next   = irttp_seq_next,
+	.stop   = irttp_seq_stop,
+	.show   = irttp_seq_show,
+};
+
+static int irttp_seq_open(struct inode *inode, struct file *file)
+{
+	return seq_open_private(file, &irttp_seq_ops,
+			sizeof(struct irttp_iter_state));
+}
+
+const struct file_operations irttp_seq_fops = {
+	.owner		= THIS_MODULE,
+	.open           = irttp_seq_open,
+	.read           = seq_read,
+	.llseek         = seq_lseek,
+	.release	= seq_release_private,
+};
+
+#endif /* PROC_FS */
diff --git a/drivers/staging/irda/net/parameters.c b/drivers/staging/irda/net/parameters.c
new file mode 100644
index 000000000000..16ce32ffe004
--- /dev/null
+++ b/drivers/staging/irda/net/parameters.c
@@ -0,0 +1,584 @@
+/*********************************************************************
+ *
+ * Filename:      parameters.c
+ * Version:       1.0
+ * Description:   A more general way to handle (pi,pl,pv) parameters
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Mon Jun  7 10:25:11 1999
+ * Modified at:   Sun Jan 30 14:08:39 2000
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1999-2000 Dag Brattli, All Rights Reserved.
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ *
+ *     You should have received a copy of the GNU General Public License
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ ********************************************************************/
+
+#include <linux/types.h>
+#include <linux/module.h>
+
+#include <asm/unaligned.h>
+#include <asm/byteorder.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/parameters.h>
+
+static int irda_extract_integer(void *self, __u8 *buf, int len, __u8 pi,
+				PV_TYPE type, PI_HANDLER func);
+static int irda_extract_string(void *self, __u8 *buf, int len, __u8 pi,
+			       PV_TYPE type, PI_HANDLER func);
+static int irda_extract_octseq(void *self, __u8 *buf, int len, __u8 pi,
+			       PV_TYPE type, PI_HANDLER func);
+static int irda_extract_no_value(void *self, __u8 *buf, int len, __u8 pi,
+				 PV_TYPE type, PI_HANDLER func);
+
+static int irda_insert_integer(void *self, __u8 *buf, int len, __u8 pi,
+			       PV_TYPE type, PI_HANDLER func);
+static int irda_insert_no_value(void *self, __u8 *buf, int len, __u8 pi,
+				PV_TYPE type, PI_HANDLER func);
+
+static int irda_param_unpack(__u8 *buf, char *fmt, ...);
+
+/* Parameter value call table. Must match PV_TYPE */
+static const PV_HANDLER pv_extract_table[] = {
+	irda_extract_integer, /* Handler for any length integers */
+	irda_extract_integer, /* Handler for 8  bits integers */
+	irda_extract_integer, /* Handler for 16 bits integers */
+	irda_extract_string,  /* Handler for strings */
+	irda_extract_integer, /* Handler for 32 bits integers */
+	irda_extract_octseq,  /* Handler for octet sequences */
+	irda_extract_no_value /* Handler for no value parameters */
+};
+
+static const PV_HANDLER pv_insert_table[] = {
+	irda_insert_integer, /* Handler for any length integers */
+	irda_insert_integer, /* Handler for 8  bits integers */
+	irda_insert_integer, /* Handler for 16 bits integers */
+	NULL,                /* Handler for strings */
+	irda_insert_integer, /* Handler for 32 bits integers */
+	NULL,                /* Handler for octet sequences */
+	irda_insert_no_value /* Handler for no value parameters */
+};
+
+/*
+ * Function irda_insert_no_value (self, buf, len, pi, type, func)
+ */
+static int irda_insert_no_value(void *self, __u8 *buf, int len, __u8 pi,
+				PV_TYPE type, PI_HANDLER func)
+{
+	irda_param_t p;
+	int ret;
+
+	p.pi = pi;
+	p.pl = 0;
+
+	/* Call handler for this parameter */
+	ret = (*func)(self, &p, PV_GET);
+
+	/* Extract values anyway, since handler may need them */
+	irda_param_pack(buf, "bb", p.pi, p.pl);
+
+	if (ret < 0)
+		return ret;
+
+	return 2; /* Inserted pl+2 bytes */
+}
+
+/*
+ * Function irda_extract_no_value (self, buf, len, type, func)
+ *
+ *    Extracts a parameter without a pv field (pl=0)
+ *
+ */
+static int irda_extract_no_value(void *self, __u8 *buf, int len, __u8 pi,
+				 PV_TYPE type, PI_HANDLER func)
+{
+	irda_param_t p;
+	int ret;
+
+	/* Extract values anyway, since handler may need them */
+	irda_param_unpack(buf, "bb", &p.pi, &p.pl);
+
+	/* Call handler for this parameter */
+	ret = (*func)(self, &p, PV_PUT);
+
+	if (ret < 0)
+		return ret;
+
+	return 2; /* Extracted pl+2 bytes */
+}
+
+/*
+ * Function irda_insert_integer (self, buf, len, pi, type, func)
+ */
+static int irda_insert_integer(void *self, __u8 *buf, int len, __u8 pi,
+			       PV_TYPE type, PI_HANDLER func)
+{
+	irda_param_t p;
+	int n = 0;
+	int err;
+
+	p.pi = pi;             /* In case handler needs to know */
+	p.pl = type & PV_MASK; /* The integer type codes the length as well */
+	p.pv.i = 0;            /* Clear value */
+
+	/* Call handler for this parameter */
+	err = (*func)(self, &p, PV_GET);
+	if (err < 0)
+		return err;
+
+	/*
+	 * If parameter length is still 0, then (1) this is an any length
+	 * integer, and (2) the handler function does not care which length
+	 * we choose to use, so we pick the one the gives the fewest bytes.
+	 */
+	if (p.pl == 0) {
+		if (p.pv.i < 0xff) {
+			pr_debug("%s(), using 1 byte\n", __func__);
+			p.pl = 1;
+		} else if (p.pv.i < 0xffff) {
+			pr_debug("%s(), using 2 bytes\n", __func__);
+			p.pl = 2;
+		} else {
+			pr_debug("%s(), using 4 bytes\n", __func__);
+			p.pl = 4; /* Default length */
+		}
+	}
+	/* Check if buffer is long enough for insertion */
+	if (len < (2+p.pl)) {
+		net_warn_ratelimited("%s: buffer too short for insertion!\n",
+				     __func__);
+		return -1;
+	}
+	pr_debug("%s(), pi=%#x, pl=%d, pi=%d\n", __func__,
+		 p.pi, p.pl, p.pv.i);
+	switch (p.pl) {
+	case 1:
+		n += irda_param_pack(buf, "bbb", p.pi, p.pl, (__u8) p.pv.i);
+		break;
+	case 2:
+		if (type & PV_BIG_ENDIAN)
+			p.pv.i = cpu_to_be16((__u16) p.pv.i);
+		else
+			p.pv.i = cpu_to_le16((__u16) p.pv.i);
+		n += irda_param_pack(buf, "bbs", p.pi, p.pl, (__u16) p.pv.i);
+		break;
+	case 4:
+		if (type & PV_BIG_ENDIAN)
+			cpu_to_be32s(&p.pv.i);
+		else
+			cpu_to_le32s(&p.pv.i);
+		n += irda_param_pack(buf, "bbi", p.pi, p.pl, p.pv.i);
+
+		break;
+	default:
+		net_warn_ratelimited("%s: length %d not supported\n",
+				     __func__, p.pl);
+		/* Skip parameter */
+		return -1;
+	}
+
+	return p.pl+2; /* Inserted pl+2 bytes */
+}
+
+/*
+ * Function irda_extract integer (self, buf, len, pi, type, func)
+ *
+ *    Extract a possibly variable length integer from buffer, and call
+ *    handler for processing of the parameter
+ */
+static int irda_extract_integer(void *self, __u8 *buf, int len, __u8 pi,
+				PV_TYPE type, PI_HANDLER func)
+{
+	irda_param_t p;
+	int n = 0;
+	int extract_len;	/* Real length we extract */
+	int err;
+
+	p.pi = pi;     /* In case handler needs to know */
+	p.pl = buf[1]; /* Extract length of value */
+	p.pv.i = 0;    /* Clear value */
+	extract_len = p.pl;	/* Default : extract all */
+
+	/* Check if buffer is long enough for parsing */
+	if (len < (2+p.pl)) {
+		net_warn_ratelimited("%s: buffer too short for parsing! Need %d bytes, but len is only %d\n",
+				     __func__, p.pl, len);
+		return -1;
+	}
+
+	/*
+	 * Check that the integer length is what we expect it to be. If the
+	 * handler want a 16 bits integer then a 32 bits is not good enough
+	 * PV_INTEGER means that the handler is flexible.
+	 */
+	if (((type & PV_MASK) != PV_INTEGER) && ((type & PV_MASK) != p.pl)) {
+		net_err_ratelimited("%s: invalid parameter length! Expected %d bytes, but value had %d bytes!\n",
+				    __func__, type & PV_MASK, p.pl);
+
+		/* Most parameters are bit/byte fields or little endian,
+		 * so it's ok to only extract a subset of it (the subset
+		 * that the handler expect). This is necessary, as some
+		 * broken implementations seems to add extra undefined bits.
+		 * If the parameter is shorter than we expect or is big
+		 * endian, we can't play those tricks. Jean II */
+		if((p.pl < (type & PV_MASK)) || (type & PV_BIG_ENDIAN)) {
+			/* Skip parameter */
+			return p.pl+2;
+		} else {
+			/* Extract subset of it, fallthrough */
+			extract_len = type & PV_MASK;
+		}
+	}
+
+
+	switch (extract_len) {
+	case 1:
+		n += irda_param_unpack(buf+2, "b", &p.pv.i);
+		break;
+	case 2:
+		n += irda_param_unpack(buf+2, "s", &p.pv.i);
+		if (type & PV_BIG_ENDIAN)
+			p.pv.i = be16_to_cpu((__u16) p.pv.i);
+		else
+			p.pv.i = le16_to_cpu((__u16) p.pv.i);
+		break;
+	case 4:
+		n += irda_param_unpack(buf+2, "i", &p.pv.i);
+		if (type & PV_BIG_ENDIAN)
+			be32_to_cpus(&p.pv.i);
+		else
+			le32_to_cpus(&p.pv.i);
+		break;
+	default:
+		net_warn_ratelimited("%s: length %d not supported\n",
+				     __func__, p.pl);
+
+		/* Skip parameter */
+		return p.pl+2;
+	}
+
+	pr_debug("%s(), pi=%#x, pl=%d, pi=%d\n", __func__,
+		 p.pi, p.pl, p.pv.i);
+	/* Call handler for this parameter */
+	err = (*func)(self, &p, PV_PUT);
+	if (err < 0)
+		return err;
+
+	return p.pl+2; /* Extracted pl+2 bytes */
+}
+
+/*
+ * Function irda_extract_string (self, buf, len, type, func)
+ */
+static int irda_extract_string(void *self, __u8 *buf, int len, __u8 pi,
+			       PV_TYPE type, PI_HANDLER func)
+{
+	char str[33];
+	irda_param_t p;
+	int err;
+
+	p.pi = pi;     /* In case handler needs to know */
+	p.pl = buf[1]; /* Extract length of value */
+	if (p.pl > 32)
+		p.pl = 32;
+
+	pr_debug("%s(), pi=%#x, pl=%d\n", __func__,
+		 p.pi, p.pl);
+
+	/* Check if buffer is long enough for parsing */
+	if (len < (2+p.pl)) {
+		net_warn_ratelimited("%s: buffer too short for parsing! Need %d bytes, but len is only %d\n",
+				     __func__, p.pl, len);
+		return -1;
+	}
+
+	/* Should be safe to copy string like this since we have already
+	 * checked that the buffer is long enough */
+	strncpy(str, buf+2, p.pl);
+
+	pr_debug("%s(), str=0x%02x 0x%02x\n",
+		 __func__, (__u8)str[0], (__u8)str[1]);
+
+	/* Null terminate string */
+	str[p.pl] = '\0';
+
+	p.pv.c = str; /* Handler will need to take a copy */
+
+	/* Call handler for this parameter */
+	err = (*func)(self, &p, PV_PUT);
+	if (err < 0)
+		return err;
+
+	return p.pl+2; /* Extracted pl+2 bytes */
+}
+
+/*
+ * Function irda_extract_octseq (self, buf, len, type, func)
+ */
+static int irda_extract_octseq(void *self, __u8 *buf, int len, __u8 pi,
+			       PV_TYPE type, PI_HANDLER func)
+{
+	irda_param_t p;
+
+	p.pi = pi;     /* In case handler needs to know */
+	p.pl = buf[1]; /* Extract length of value */
+
+	/* Check if buffer is long enough for parsing */
+	if (len < (2+p.pl)) {
+		net_warn_ratelimited("%s: buffer too short for parsing! Need %d bytes, but len is only %d\n",
+				     __func__, p.pl, len);
+		return -1;
+	}
+
+	pr_debug("%s(), not impl\n", __func__);
+
+	return p.pl+2; /* Extracted pl+2 bytes */
+}
+
+/*
+ * Function irda_param_pack (skb, fmt, ...)
+ *
+ *    Format:
+ *        'i' = 32 bits integer
+ *        's' = string
+ *
+ */
+int irda_param_pack(__u8 *buf, char *fmt, ...)
+{
+	irda_pv_t arg;
+	va_list args;
+	char *p;
+	int n = 0;
+
+	va_start(args, fmt);
+
+	for (p = fmt; *p != '\0'; p++) {
+		switch (*p) {
+		case 'b':  /* 8 bits unsigned byte */
+			buf[n++] = (__u8)va_arg(args, int);
+			break;
+		case 's':  /* 16 bits unsigned short */
+			arg.i = (__u16)va_arg(args, int);
+			put_unaligned((__u16)arg.i, (__u16 *)(buf+n)); n+=2;
+			break;
+		case 'i':  /* 32 bits unsigned integer */
+			arg.i = va_arg(args, __u32);
+			put_unaligned(arg.i, (__u32 *)(buf+n)); n+=4;
+			break;
+#if 0
+		case 'c': /* \0 terminated string */
+			arg.c = va_arg(args, char *);
+			strcpy(buf+n, arg.c);
+			n += strlen(arg.c) + 1;
+			break;
+#endif
+		default:
+			va_end(args);
+			return -1;
+		}
+	}
+	va_end(args);
+
+	return 0;
+}
+EXPORT_SYMBOL(irda_param_pack);
+
+/*
+ * Function irda_param_unpack (skb, fmt, ...)
+ */
+static int irda_param_unpack(__u8 *buf, char *fmt, ...)
+{
+	irda_pv_t arg;
+	va_list args;
+	char *p;
+	int n = 0;
+
+	va_start(args, fmt);
+
+	for (p = fmt; *p != '\0'; p++) {
+		switch (*p) {
+		case 'b':  /* 8 bits byte */
+			arg.ip = va_arg(args, __u32 *);
+			*arg.ip = buf[n++];
+			break;
+		case 's':  /* 16 bits short */
+			arg.ip = va_arg(args, __u32 *);
+			*arg.ip = get_unaligned((__u16 *)(buf+n)); n+=2;
+			break;
+		case 'i':  /* 32 bits unsigned integer */
+			arg.ip = va_arg(args, __u32 *);
+			*arg.ip = get_unaligned((__u32 *)(buf+n)); n+=4;
+			break;
+#if 0
+		case 'c':   /* \0 terminated string */
+			arg.c = va_arg(args, char *);
+			strcpy(arg.c, buf+n);
+			n += strlen(arg.c) + 1;
+			break;
+#endif
+		default:
+			va_end(args);
+			return -1;
+		}
+
+	}
+	va_end(args);
+
+	return 0;
+}
+
+/*
+ * Function irda_param_insert (self, pi, buf, len, info)
+ *
+ *    Insert the specified parameter (pi) into buffer. Returns number of
+ *    bytes inserted
+ */
+int irda_param_insert(void *self, __u8 pi, __u8 *buf, int len,
+		      pi_param_info_t *info)
+{
+	const pi_minor_info_t *pi_minor_info;
+	__u8 pi_minor;
+	__u8 pi_major;
+	int type;
+	int ret = -1;
+	int n = 0;
+
+	IRDA_ASSERT(buf != NULL, return ret;);
+	IRDA_ASSERT(info != NULL, return ret;);
+
+	pi_minor = pi & info->pi_mask;
+	pi_major = pi >> info->pi_major_offset;
+
+	/* Check if the identifier value (pi) is valid */
+	if ((pi_major > info->len-1) ||
+	    (pi_minor > info->tables[pi_major].len-1))
+	{
+		pr_debug("%s(), no handler for parameter=0x%02x\n",
+			 __func__, pi);
+
+		/* Skip this parameter */
+		return -1;
+	}
+
+	/* Lookup the info on how to parse this parameter */
+	pi_minor_info = &info->tables[pi_major].pi_minor_call_table[pi_minor];
+
+	/* Find expected data type for this parameter identifier (pi)*/
+	type = pi_minor_info->type;
+
+	/*  Check if handler has been implemented */
+	if (!pi_minor_info->func) {
+		net_info_ratelimited("%s: no handler for pi=%#x\n",
+				     __func__, pi);
+		/* Skip this parameter */
+		return -1;
+	}
+
+	/* Insert parameter value */
+	ret = (*pv_insert_table[type & PV_MASK])(self, buf+n, len, pi, type,
+						 pi_minor_info->func);
+	return ret;
+}
+EXPORT_SYMBOL(irda_param_insert);
+
+/*
+ * Function irda_param_extract (self, buf, len, info)
+ *
+ *    Parse all parameters. If len is correct, then everything should be
+ *    safe. Returns the number of bytes that was parsed
+ *
+ */
+static int irda_param_extract(void *self, __u8 *buf, int len,
+			      pi_param_info_t *info)
+{
+	const pi_minor_info_t *pi_minor_info;
+	__u8 pi_minor;
+	__u8 pi_major;
+	int type;
+	int ret = -1;
+	int n = 0;
+
+	IRDA_ASSERT(buf != NULL, return ret;);
+	IRDA_ASSERT(info != NULL, return ret;);
+
+	pi_minor = buf[n] & info->pi_mask;
+	pi_major = buf[n] >> info->pi_major_offset;
+
+	/* Check if the identifier value (pi) is valid */
+	if ((pi_major > info->len-1) ||
+	    (pi_minor > info->tables[pi_major].len-1))
+	{
+		pr_debug("%s(), no handler for parameter=0x%02x\n",
+			 __func__, buf[0]);
+
+		/* Skip this parameter */
+		return 2 + buf[n + 1];  /* Continue */
+	}
+
+	/* Lookup the info on how to parse this parameter */
+	pi_minor_info = &info->tables[pi_major].pi_minor_call_table[pi_minor];
+
+	/* Find expected data type for this parameter identifier (pi)*/
+	type = pi_minor_info->type;
+
+	pr_debug("%s(), pi=[%d,%d], type=%d\n", __func__,
+		 pi_major, pi_minor, type);
+
+	/*  Check if handler has been implemented */
+	if (!pi_minor_info->func) {
+		net_info_ratelimited("%s: no handler for pi=%#x\n",
+				     __func__, buf[n]);
+		/* Skip this parameter */
+		return 2 + buf[n + 1]; /* Continue */
+	}
+
+	/* Parse parameter value */
+	ret = (*pv_extract_table[type & PV_MASK])(self, buf+n, len, buf[n],
+						  type, pi_minor_info->func);
+	return ret;
+}
+
+/*
+ * Function irda_param_extract_all (self, buf, len, info)
+ *
+ *    Parse all parameters. If len is correct, then everything should be
+ *    safe. Returns the number of bytes that was parsed
+ *
+ */
+int irda_param_extract_all(void *self, __u8 *buf, int len,
+			   pi_param_info_t *info)
+{
+	int ret = -1;
+	int n = 0;
+
+	IRDA_ASSERT(buf != NULL, return ret;);
+	IRDA_ASSERT(info != NULL, return ret;);
+
+	/*
+	 * Parse all parameters. Each parameter must be at least two bytes
+	 * long or else there is no point in trying to parse it
+	 */
+	while (len > 2) {
+		ret = irda_param_extract(self, buf+n, len, info);
+		if (ret < 0)
+			return ret;
+
+		n += ret;
+		len -= ret;
+	}
+	return n;
+}
+EXPORT_SYMBOL(irda_param_extract_all);
diff --git a/drivers/staging/irda/net/qos.c b/drivers/staging/irda/net/qos.c
new file mode 100644
index 000000000000..25ba8509ad3e
--- /dev/null
+++ b/drivers/staging/irda/net/qos.c
@@ -0,0 +1,771 @@
+/*********************************************************************
+ *
+ * Filename:      qos.c
+ * Version:       1.0
+ * Description:   IrLAP QoS parameter negotiation
+ * Status:        Stable
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Tue Sep  9 00:00:26 1997
+ * Modified at:   Sun Jan 30 14:29:16 2000
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1998-2000 Dag Brattli <dagb@cs.uit.no>,
+ *     All Rights Reserved.
+ *     Copyright (c) 2000-2001 Jean Tourrilhes <jt@hpl.hp.com>
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     This program is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *     GNU General Public License for more details.
+ *
+ *     You should have received a copy of the GNU General Public License
+ *     along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ ********************************************************************/
+
+#include <linux/export.h>
+
+#include <asm/byteorder.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/parameters.h>
+#include <net/irda/qos.h>
+#include <net/irda/irlap.h>
+#include <net/irda/irlap_frame.h>
+
+/*
+ * Maximum values of the baud rate we negotiate with the other end.
+ * Most often, you don't have to change that, because Linux-IrDA will
+ * use the maximum offered by the link layer, which usually works fine.
+ * In some very rare cases, you may want to limit it to lower speeds...
+ */
+int sysctl_max_baud_rate = 16000000;
+/*
+ * Maximum value of the lap disconnect timer we negotiate with the other end.
+ * Most often, the value below represent the best compromise, but some user
+ * may want to keep the LAP alive longer or shorter in case of link failure.
+ * Remember that the threshold time (early warning) is fixed to 3s...
+ */
+int sysctl_max_noreply_time = 12;
+/*
+ * Minimum turn time to be applied before transmitting to the peer.
+ * Nonzero values (usec) are used as lower limit to the per-connection
+ * mtt value which was announced by the other end during negotiation.
+ * Might be helpful if the peer device provides too short mtt.
+ * Default is 10us which means using the unmodified value given by the
+ * peer except if it's 0 (0 is likely a bug in the other stack).
+ */
+unsigned int sysctl_min_tx_turn_time = 10;
+/*
+ * Maximum data size to be used in transmission in payload of LAP frame.
+ * There is a bit of confusion in the IrDA spec :
+ * The LAP spec defines the payload of a LAP frame (I field) to be
+ * 2048 bytes max (IrLAP 1.1, chapt 6.6.5, p40).
+ * On the other hand, the PHY mention frames of 2048 bytes max (IrPHY
+ * 1.2, chapt 5.3.2.1, p41). But, this number includes the LAP header
+ * (2 bytes), and CRC (32 bits at 4 Mb/s). So, for the I field (LAP
+ * payload), that's only 2042 bytes. Oups !
+ * My nsc-ircc hardware has troubles receiving 2048 bytes frames at 4 Mb/s,
+ * so adjust to 2042... I don't know if this bug applies only for 2048
+ * bytes frames or all negotiated frame sizes, but you can use the sysctl
+ * to play with this value anyway.
+ * Jean II */
+unsigned int sysctl_max_tx_data_size = 2042;
+/*
+ * Maximum transmit window, i.e. number of LAP frames between turn-around.
+ * This allow to override what the peer told us. Some peers are buggy and
+ * don't always support what they tell us.
+ * Jean II */
+unsigned int sysctl_max_tx_window = 7;
+
+static int irlap_param_baud_rate(void *instance, irda_param_t *param, int get);
+static int irlap_param_link_disconnect(void *instance, irda_param_t *parm,
+				       int get);
+static int irlap_param_max_turn_time(void *instance, irda_param_t *param,
+				     int get);
+static int irlap_param_data_size(void *instance, irda_param_t *param, int get);
+static int irlap_param_window_size(void *instance, irda_param_t *param,
+				   int get);
+static int irlap_param_additional_bofs(void *instance, irda_param_t *parm,
+				       int get);
+static int irlap_param_min_turn_time(void *instance, irda_param_t *param,
+				     int get);
+
+#ifndef CONFIG_IRDA_DYNAMIC_WINDOW
+static __u32 irlap_requested_line_capacity(struct qos_info *qos);
+#endif
+
+static __u32 min_turn_times[]  = { 10000, 5000, 1000, 500, 100, 50, 10, 0 }; /* us */
+static __u32 baud_rates[]      = { 2400, 9600, 19200, 38400, 57600, 115200, 576000,
+				   1152000, 4000000, 16000000 };           /* bps */
+static __u32 data_sizes[]      = { 64, 128, 256, 512, 1024, 2048 };        /* bytes */
+static __u32 add_bofs[]        = { 48, 24, 12, 5, 3, 2, 1, 0 };            /* bytes */
+static __u32 max_turn_times[]  = { 500, 250, 100, 50 };                    /* ms */
+static __u32 link_disc_times[] = { 3, 8, 12, 16, 20, 25, 30, 40 };         /* secs */
+
+static __u32 max_line_capacities[10][4] = {
+       /* 500 ms     250 ms  100 ms  50 ms (max turn time) */
+	{    100,      0,      0,     0 }, /*     2400 bps */
+	{    400,      0,      0,     0 }, /*     9600 bps */
+	{    800,      0,      0,     0 }, /*    19200 bps */
+	{   1600,      0,      0,     0 }, /*    38400 bps */
+	{   2360,      0,      0,     0 }, /*    57600 bps */
+	{   4800,   2400,    960,   480 }, /*   115200 bps */
+	{  28800,  11520,   5760,  2880 }, /*   576000 bps */
+	{  57600,  28800,  11520,  5760 }, /*  1152000 bps */
+	{ 200000, 100000,  40000, 20000 }, /*  4000000 bps */
+	{ 800000, 400000, 160000, 80000 }, /* 16000000 bps */
+};
+
+static const pi_minor_info_t pi_minor_call_table_type_0[] = {
+	{ NULL, 0 },
+/* 01 */{ irlap_param_baud_rate,       PV_INTEGER | PV_LITTLE_ENDIAN },
+	{ NULL, 0 },
+	{ NULL, 0 },
+	{ NULL, 0 },
+	{ NULL, 0 },
+	{ NULL, 0 },
+	{ NULL, 0 },
+/* 08 */{ irlap_param_link_disconnect, PV_INT_8_BITS }
+};
+
+static const pi_minor_info_t pi_minor_call_table_type_1[] = {
+	{ NULL, 0 },
+	{ NULL, 0 },
+/* 82 */{ irlap_param_max_turn_time,   PV_INT_8_BITS },
+/* 83 */{ irlap_param_data_size,       PV_INT_8_BITS },
+/* 84 */{ irlap_param_window_size,     PV_INT_8_BITS },
+/* 85 */{ irlap_param_additional_bofs, PV_INT_8_BITS },
+/* 86 */{ irlap_param_min_turn_time,   PV_INT_8_BITS },
+};
+
+static const pi_major_info_t pi_major_call_table[] = {
+	{ pi_minor_call_table_type_0, 9 },
+	{ pi_minor_call_table_type_1, 7 },
+};
+
+static pi_param_info_t irlap_param_info = { pi_major_call_table, 2, 0x7f, 7 };
+
+/* ---------------------- LOCAL SUBROUTINES ---------------------- */
+/* Note : we start with a bunch of local subroutines.
+ * As the compiler is "one pass", this is the only way to get them to
+ * inline properly...
+ * Jean II
+ */
+/*
+ * Function value_index (value, array, size)
+ *
+ *    Returns the index to the value in the specified array
+ */
+static inline int value_index(__u32 value, __u32 *array, int size)
+{
+	int i;
+
+	for (i=0; i < size; i++)
+		if (array[i] == value)
+			break;
+	return i;
+}
+
+/*
+ * Function index_value (index, array)
+ *
+ *    Returns value to index in array, easy!
+ *
+ */
+static inline __u32 index_value(int index, __u32 *array)
+{
+	return array[index];
+}
+
+/*
+ * Function msb_index (word)
+ *
+ *    Returns index to most significant bit (MSB) in word
+ *
+ */
+static int msb_index (__u16 word)
+{
+	__u16 msb = 0x8000;
+	int index = 15;   /* Current MSB */
+
+	/* Check for buggy peers.
+	 * Note : there is a small probability that it could be us, but I
+	 * would expect driver authors to catch that pretty early and be
+	 * able to check precisely what's going on. If a end user sees this,
+	 * it's very likely the peer. - Jean II */
+	if (word == 0) {
+		net_warn_ratelimited("%s(), Detected buggy peer, adjust null PV to 0x1!\n",
+				     __func__);
+		/* The only safe choice (we don't know the array size) */
+		word = 0x1;
+	}
+
+	while (msb) {
+		if (word & msb)
+			break;   /* Found it! */
+		msb >>=1;
+		index--;
+	}
+	return index;
+}
+
+/*
+ * Function value_lower_bits (value, array)
+ *
+ *    Returns a bit field marking all possibility lower than value.
+ */
+static inline int value_lower_bits(__u32 value, __u32 *array, int size, __u16 *field)
+{
+	int	i;
+	__u16	mask = 0x1;
+	__u16	result = 0x0;
+
+	for (i=0; i < size; i++) {
+		/* Add the current value to the bit field, shift mask */
+		result |= mask;
+		mask <<= 1;
+		/* Finished ? */
+		if (array[i] >= value)
+			break;
+	}
+	/* Send back a valid index */
+	if(i >= size)
+	  i = size - 1;	/* Last item */
+	*field = result;
+	return i;
+}
+
+/*
+ * Function value_highest_bit (value, array)
+ *
+ *    Returns a bit field marking the highest possibility lower than value.
+ */
+static inline int value_highest_bit(__u32 value, __u32 *array, int size, __u16 *field)
+{
+	int	i;
+	__u16	mask = 0x1;
+	__u16	result = 0x0;
+
+	for (i=0; i < size; i++) {
+		/* Finished ? */
+		if (array[i] <= value)
+			break;
+		/* Shift mask */
+		mask <<= 1;
+	}
+	/* Set the current value to the bit field */
+	result |= mask;
+	/* Send back a valid index */
+	if(i >= size)
+	  i = size - 1;	/* Last item */
+	*field = result;
+	return i;
+}
+
+/* -------------------------- MAIN CALLS -------------------------- */
+
+/*
+ * Function irda_qos_compute_intersection (qos, new)
+ *
+ *    Compute the intersection of the old QoS capabilities with new ones
+ *
+ */
+void irda_qos_compute_intersection(struct qos_info *qos, struct qos_info *new)
+{
+	IRDA_ASSERT(qos != NULL, return;);
+	IRDA_ASSERT(new != NULL, return;);
+
+	/* Apply */
+	qos->baud_rate.bits       &= new->baud_rate.bits;
+	qos->window_size.bits     &= new->window_size.bits;
+	qos->min_turn_time.bits   &= new->min_turn_time.bits;
+	qos->max_turn_time.bits   &= new->max_turn_time.bits;
+	qos->data_size.bits       &= new->data_size.bits;
+	qos->link_disc_time.bits  &= new->link_disc_time.bits;
+	qos->additional_bofs.bits &= new->additional_bofs.bits;
+
+	irda_qos_bits_to_value(qos);
+}
+
+/*
+ * Function irda_init_max_qos_capabilies (qos)
+ *
+ *    The purpose of this function is for layers and drivers to be able to
+ *    set the maximum QoS possible and then "and in" their own limitations
+ *
+ */
+void irda_init_max_qos_capabilies(struct qos_info *qos)
+{
+	int i;
+	/*
+	 *  These are the maximum supported values as specified on pages
+	 *  39-43 in IrLAP
+	 */
+
+	/* Use sysctl to set some configurable values... */
+	/* Set configured max speed */
+	i = value_lower_bits(sysctl_max_baud_rate, baud_rates, 10,
+			     &qos->baud_rate.bits);
+	sysctl_max_baud_rate = index_value(i, baud_rates);
+
+	/* Set configured max disc time */
+	i = value_lower_bits(sysctl_max_noreply_time, link_disc_times, 8,
+			     &qos->link_disc_time.bits);
+	sysctl_max_noreply_time = index_value(i, link_disc_times);
+
+	/* LSB is first byte, MSB is second byte */
+	qos->baud_rate.bits    &= 0x03ff;
+
+	qos->window_size.bits     = 0x7f;
+	qos->min_turn_time.bits   = 0xff;
+	qos->max_turn_time.bits   = 0x0f;
+	qos->data_size.bits       = 0x3f;
+	qos->link_disc_time.bits &= 0xff;
+	qos->additional_bofs.bits = 0xff;
+}
+EXPORT_SYMBOL(irda_init_max_qos_capabilies);
+
+/*
+ * Function irlap_adjust_qos_settings (qos)
+ *
+ *     Adjust QoS settings in case some values are not possible to use because
+ *     of other settings
+ */
+static void irlap_adjust_qos_settings(struct qos_info *qos)
+{
+	__u32 line_capacity;
+	int index;
+
+	/*
+	 * Make sure the mintt is sensible.
+	 * Main culprit : Ericsson T39. - Jean II
+	 */
+	if (sysctl_min_tx_turn_time > qos->min_turn_time.value) {
+		int i;
+
+		net_warn_ratelimited("%s(), Detected buggy peer, adjust mtt to %dus!\n",
+				     __func__, sysctl_min_tx_turn_time);
+
+		/* We don't really need bits, but easier this way */
+		i = value_highest_bit(sysctl_min_tx_turn_time, min_turn_times,
+				      8, &qos->min_turn_time.bits);
+		sysctl_min_tx_turn_time = index_value(i, min_turn_times);
+		qos->min_turn_time.value = sysctl_min_tx_turn_time;
+	}
+
+	/*
+	 * Not allowed to use a max turn time less than 500 ms if the baudrate
+	 * is less than 115200
+	 */
+	if ((qos->baud_rate.value < 115200) &&
+	    (qos->max_turn_time.value < 500))
+	{
+		pr_debug("%s(), adjusting max turn time from %d to 500 ms\n",
+			 __func__, qos->max_turn_time.value);
+		qos->max_turn_time.value = 500;
+	}
+
+	/*
+	 * The data size must be adjusted according to the baud rate and max
+	 * turn time
+	 */
+	index = value_index(qos->data_size.value, data_sizes, 6);
+	line_capacity = irlap_max_line_capacity(qos->baud_rate.value,
+						qos->max_turn_time.value);
+
+#ifdef CONFIG_IRDA_DYNAMIC_WINDOW
+	while ((qos->data_size.value > line_capacity) && (index > 0)) {
+		qos->data_size.value = data_sizes[index--];
+		pr_debug("%s(), reducing data size to %d\n",
+			 __func__, qos->data_size.value);
+	}
+#else /* Use method described in section 6.6.11 of IrLAP */
+	while (irlap_requested_line_capacity(qos) > line_capacity) {
+		IRDA_ASSERT(index != 0, return;);
+
+		/* Must be able to send at least one frame */
+		if (qos->window_size.value > 1) {
+			qos->window_size.value--;
+			pr_debug("%s(), reducing window size to %d\n",
+				 __func__, qos->window_size.value);
+		} else if (index > 1) {
+			qos->data_size.value = data_sizes[index--];
+			pr_debug("%s(), reducing data size to %d\n",
+				 __func__, qos->data_size.value);
+		} else {
+			net_warn_ratelimited("%s(), nothing more we can do!\n",
+					     __func__);
+		}
+	}
+#endif /* CONFIG_IRDA_DYNAMIC_WINDOW */
+	/*
+	 * Fix tx data size according to user limits - Jean II
+	 */
+	if (qos->data_size.value > sysctl_max_tx_data_size)
+		/* Allow non discrete adjustement to avoid losing capacity */
+		qos->data_size.value = sysctl_max_tx_data_size;
+	/*
+	 * Override Tx window if user request it. - Jean II
+	 */
+	if (qos->window_size.value > sysctl_max_tx_window)
+		qos->window_size.value = sysctl_max_tx_window;
+}
+
+/*
+ * Function irlap_negotiate (qos_device, qos_session, skb)
+ *
+ *    Negotiate QoS values, not really that much negotiation :-)
+ *    We just set the QoS capabilities for the peer station
+ *
+ */
+int irlap_qos_negotiate(struct irlap_cb *self, struct sk_buff *skb)
+{
+	int ret;
+
+	ret = irda_param_extract_all(self, skb->data, skb->len,
+				     &irlap_param_info);
+
+	/* Convert the negotiated bits to values */
+	irda_qos_bits_to_value(&self->qos_tx);
+	irda_qos_bits_to_value(&self->qos_rx);
+
+	irlap_adjust_qos_settings(&self->qos_tx);
+
+	pr_debug("Setting BAUD_RATE to %d bps.\n",
+		 self->qos_tx.baud_rate.value);
+	pr_debug("Setting DATA_SIZE to %d bytes\n",
+		 self->qos_tx.data_size.value);
+	pr_debug("Setting WINDOW_SIZE to %d\n",
+		 self->qos_tx.window_size.value);
+	pr_debug("Setting XBOFS to %d\n",
+		 self->qos_tx.additional_bofs.value);
+	pr_debug("Setting MAX_TURN_TIME to %d ms.\n",
+		 self->qos_tx.max_turn_time.value);
+	pr_debug("Setting MIN_TURN_TIME to %d usecs.\n",
+		 self->qos_tx.min_turn_time.value);
+	pr_debug("Setting LINK_DISC to %d secs.\n",
+		 self->qos_tx.link_disc_time.value);
+	return ret;
+}
+
+/*
+ * Function irlap_insert_negotiation_params (qos, fp)
+ *
+ *    Insert QoS negotiaion pararameters into frame
+ *
+ */
+int irlap_insert_qos_negotiation_params(struct irlap_cb *self,
+					struct sk_buff *skb)
+{
+	int ret;
+
+	/* Insert data rate */
+	ret = irda_param_insert(self, PI_BAUD_RATE, skb_tail_pointer(skb),
+				skb_tailroom(skb), &irlap_param_info);
+	if (ret < 0)
+		return ret;
+	skb_put(skb, ret);
+
+	/* Insert max turnaround time */
+	ret = irda_param_insert(self, PI_MAX_TURN_TIME, skb_tail_pointer(skb),
+				skb_tailroom(skb), &irlap_param_info);
+	if (ret < 0)
+		return ret;
+	skb_put(skb, ret);
+
+	/* Insert data size */
+	ret = irda_param_insert(self, PI_DATA_SIZE, skb_tail_pointer(skb),
+				skb_tailroom(skb), &irlap_param_info);
+	if (ret < 0)
+		return ret;
+	skb_put(skb, ret);
+
+	/* Insert window size */
+	ret = irda_param_insert(self, PI_WINDOW_SIZE, skb_tail_pointer(skb),
+				skb_tailroom(skb), &irlap_param_info);
+	if (ret < 0)
+		return ret;
+	skb_put(skb, ret);
+
+	/* Insert additional BOFs */
+	ret = irda_param_insert(self, PI_ADD_BOFS, skb_tail_pointer(skb),
+				skb_tailroom(skb), &irlap_param_info);
+	if (ret < 0)
+		return ret;
+	skb_put(skb, ret);
+
+	/* Insert minimum turnaround time */
+	ret = irda_param_insert(self, PI_MIN_TURN_TIME, skb_tail_pointer(skb),
+				skb_tailroom(skb), &irlap_param_info);
+	if (ret < 0)
+		return ret;
+	skb_put(skb, ret);
+
+	/* Insert link disconnect/threshold time */
+	ret = irda_param_insert(self, PI_LINK_DISC, skb_tail_pointer(skb),
+				skb_tailroom(skb), &irlap_param_info);
+	if (ret < 0)
+		return ret;
+	skb_put(skb, ret);
+
+	return 0;
+}
+
+/*
+ * Function irlap_param_baud_rate (instance, param, get)
+ *
+ *    Negotiate data-rate
+ *
+ */
+static int irlap_param_baud_rate(void *instance, irda_param_t *param, int get)
+{
+	__u16 final;
+
+	struct irlap_cb *self = (struct irlap_cb *) instance;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return -1;);
+
+	if (get) {
+		param->pv.i = self->qos_rx.baud_rate.bits;
+		pr_debug("%s(), baud rate = 0x%02x\n",
+			 __func__, param->pv.i);
+	} else {
+		/*
+		 *  Stations must agree on baud rate, so calculate
+		 *  intersection
+		 */
+		pr_debug("Requested BAUD_RATE: 0x%04x\n", (__u16)param->pv.i);
+		final = (__u16) param->pv.i & self->qos_rx.baud_rate.bits;
+
+		pr_debug("Final BAUD_RATE: 0x%04x\n", final);
+		self->qos_tx.baud_rate.bits = final;
+		self->qos_rx.baud_rate.bits = final;
+	}
+
+	return 0;
+}
+
+/*
+ * Function irlap_param_link_disconnect (instance, param, get)
+ *
+ *    Negotiate link disconnect/threshold time.
+ *
+ */
+static int irlap_param_link_disconnect(void *instance, irda_param_t *param,
+				       int get)
+{
+	__u16 final;
+
+	struct irlap_cb *self = (struct irlap_cb *) instance;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return -1;);
+
+	if (get)
+		param->pv.i = self->qos_rx.link_disc_time.bits;
+	else {
+		/*
+		 *  Stations must agree on link disconnect/threshold
+		 *  time.
+		 */
+		pr_debug("LINK_DISC: %02x\n", (__u8)param->pv.i);
+		final = (__u8) param->pv.i & self->qos_rx.link_disc_time.bits;
+
+		pr_debug("Final LINK_DISC: %02x\n", final);
+		self->qos_tx.link_disc_time.bits = final;
+		self->qos_rx.link_disc_time.bits = final;
+	}
+	return 0;
+}
+
+/*
+ * Function irlap_param_max_turn_time (instance, param, get)
+ *
+ *    Negotiate the maximum turnaround time. This is a type 1 parameter and
+ *    will be negotiated independently for each station
+ *
+ */
+static int irlap_param_max_turn_time(void *instance, irda_param_t *param,
+				     int get)
+{
+	struct irlap_cb *self = (struct irlap_cb *) instance;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return -1;);
+
+	if (get)
+		param->pv.i = self->qos_rx.max_turn_time.bits;
+	else
+		self->qos_tx.max_turn_time.bits = (__u8) param->pv.i;
+
+	return 0;
+}
+
+/*
+ * Function irlap_param_data_size (instance, param, get)
+ *
+ *    Negotiate the data size. This is a type 1 parameter and
+ *    will be negotiated independently for each station
+ *
+ */
+static int irlap_param_data_size(void *instance, irda_param_t *param, int get)
+{
+	struct irlap_cb *self = (struct irlap_cb *) instance;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return -1;);
+
+	if (get)
+		param->pv.i = self->qos_rx.data_size.bits;
+	else
+		self->qos_tx.data_size.bits = (__u8) param->pv.i;
+
+	return 0;
+}
+
+/*
+ * Function irlap_param_window_size (instance, param, get)
+ *
+ *    Negotiate the window size. This is a type 1 parameter and
+ *    will be negotiated independently for each station
+ *
+ */
+static int irlap_param_window_size(void *instance, irda_param_t *param,
+				   int get)
+{
+	struct irlap_cb *self = (struct irlap_cb *) instance;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return -1;);
+
+	if (get)
+		param->pv.i = self->qos_rx.window_size.bits;
+	else
+		self->qos_tx.window_size.bits = (__u8) param->pv.i;
+
+	return 0;
+}
+
+/*
+ * Function irlap_param_additional_bofs (instance, param, get)
+ *
+ *    Negotiate additional BOF characters. This is a type 1 parameter and
+ *    will be negotiated independently for each station.
+ */
+static int irlap_param_additional_bofs(void *instance, irda_param_t *param, int get)
+{
+	struct irlap_cb *self = (struct irlap_cb *) instance;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return -1;);
+
+	if (get)
+		param->pv.i = self->qos_rx.additional_bofs.bits;
+	else
+		self->qos_tx.additional_bofs.bits = (__u8) param->pv.i;
+
+	return 0;
+}
+
+/*
+ * Function irlap_param_min_turn_time (instance, param, get)
+ *
+ *    Negotiate the minimum turn around time. This is a type 1 parameter and
+ *    will be negotiated independently for each station
+ */
+static int irlap_param_min_turn_time(void *instance, irda_param_t *param,
+				     int get)
+{
+	struct irlap_cb *self = (struct irlap_cb *) instance;
+
+	IRDA_ASSERT(self != NULL, return -1;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return -1;);
+
+	if (get)
+		param->pv.i = self->qos_rx.min_turn_time.bits;
+	else
+		self->qos_tx.min_turn_time.bits = (__u8) param->pv.i;
+
+	return 0;
+}
+
+/*
+ * Function irlap_max_line_capacity (speed, max_turn_time, min_turn_time)
+ *
+ *    Calculate the maximum line capacity
+ *
+ */
+__u32 irlap_max_line_capacity(__u32 speed, __u32 max_turn_time)
+{
+	__u32 line_capacity;
+	int i,j;
+
+	pr_debug("%s(), speed=%d, max_turn_time=%d\n",
+		 __func__, speed, max_turn_time);
+
+	i = value_index(speed, baud_rates, 10);
+	j = value_index(max_turn_time, max_turn_times, 4);
+
+	IRDA_ASSERT(((i >=0) && (i <10)), return 0;);
+	IRDA_ASSERT(((j >=0) && (j <4)), return 0;);
+
+	line_capacity = max_line_capacities[i][j];
+
+	pr_debug("%s(), line capacity=%d bytes\n",
+		 __func__, line_capacity);
+
+	return line_capacity;
+}
+
+#ifndef CONFIG_IRDA_DYNAMIC_WINDOW
+static __u32 irlap_requested_line_capacity(struct qos_info *qos)
+{
+	__u32 line_capacity;
+
+	line_capacity = qos->window_size.value *
+		(qos->data_size.value + 6 + qos->additional_bofs.value) +
+		irlap_min_turn_time_in_bytes(qos->baud_rate.value,
+					     qos->min_turn_time.value);
+
+	pr_debug("%s(), requested line capacity=%d\n",
+		 __func__, line_capacity);
+
+	return line_capacity;
+}
+#endif
+
+void irda_qos_bits_to_value(struct qos_info *qos)
+{
+	int index;
+
+	IRDA_ASSERT(qos != NULL, return;);
+
+	index = msb_index(qos->baud_rate.bits);
+	qos->baud_rate.value = baud_rates[index];
+
+	index = msb_index(qos->data_size.bits);
+	qos->data_size.value = data_sizes[index];
+
+	index = msb_index(qos->window_size.bits);
+	qos->window_size.value = index+1;
+
+	index = msb_index(qos->min_turn_time.bits);
+	qos->min_turn_time.value = min_turn_times[index];
+
+	index = msb_index(qos->max_turn_time.bits);
+	qos->max_turn_time.value = max_turn_times[index];
+
+	index = msb_index(qos->link_disc_time.bits);
+	qos->link_disc_time.value = link_disc_times[index];
+
+	index = msb_index(qos->additional_bofs.bits);
+	qos->additional_bofs.value = add_bofs[index];
+}
+EXPORT_SYMBOL(irda_qos_bits_to_value);
diff --git a/drivers/staging/irda/net/timer.c b/drivers/staging/irda/net/timer.c
new file mode 100644
index 000000000000..f2280f73b057
--- /dev/null
+++ b/drivers/staging/irda/net/timer.c
@@ -0,0 +1,231 @@
+/*********************************************************************
+ *
+ * Filename:      timer.c
+ * Version:
+ * Description:
+ * Status:        Experimental.
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Sat Aug 16 00:59:29 1997
+ * Modified at:   Wed Dec  8 12:50:34 1999
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ *
+ *     Copyright (c) 1997, 1999 Dag Brattli <dagb@cs.uit.no>,
+ *     All Rights Reserved.
+ *     Copyright (c) 2000-2002 Jean Tourrilhes <jt@hpl.hp.com>
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#include <linux/delay.h>
+
+#include <net/irda/timer.h>
+#include <net/irda/irda.h>
+#include <net/irda/irda_device.h>
+#include <net/irda/irlap.h>
+#include <net/irda/irlmp.h>
+
+extern int  sysctl_slot_timeout;
+
+static void irlap_slot_timer_expired(void* data);
+static void irlap_query_timer_expired(void* data);
+static void irlap_final_timer_expired(void* data);
+static void irlap_wd_timer_expired(void* data);
+static void irlap_backoff_timer_expired(void* data);
+static void irlap_media_busy_expired(void* data);
+
+void irlap_start_slot_timer(struct irlap_cb *self, int timeout)
+{
+	irda_start_timer(&self->slot_timer, timeout, (void *) self,
+			 irlap_slot_timer_expired);
+}
+
+void irlap_start_query_timer(struct irlap_cb *self, int S, int s)
+{
+	int timeout;
+
+	/* Calculate when the peer discovery should end. Normally, we
+	 * get the end-of-discovery frame, so this is just in case
+	 * we miss it.
+	 * Basically, we multiply the number of remaining slots by our
+	 * slot time, plus add some extra time to properly receive the last
+	 * discovery packet (which is longer due to extra discovery info),
+	 * to avoid messing with for incoming connections requests and
+	 * to accommodate devices that perform discovery slower than us.
+	 * Jean II */
+	timeout = msecs_to_jiffies(sysctl_slot_timeout) * (S - s)
+		   + XIDEXTRA_TIMEOUT + SMALLBUSY_TIMEOUT;
+
+	/* Set or re-set the timer. We reset the timer for each received
+	 * discovery query, which allow us to automatically adjust to
+	 * the speed of the peer discovery (faster or slower). Jean II */
+	irda_start_timer( &self->query_timer, timeout, (void *) self,
+			  irlap_query_timer_expired);
+}
+
+void irlap_start_final_timer(struct irlap_cb *self, int timeout)
+{
+	irda_start_timer(&self->final_timer, timeout, (void *) self,
+			 irlap_final_timer_expired);
+}
+
+void irlap_start_wd_timer(struct irlap_cb *self, int timeout)
+{
+	irda_start_timer(&self->wd_timer, timeout, (void *) self,
+			 irlap_wd_timer_expired);
+}
+
+void irlap_start_backoff_timer(struct irlap_cb *self, int timeout)
+{
+	irda_start_timer(&self->backoff_timer, timeout, (void *) self,
+			 irlap_backoff_timer_expired);
+}
+
+void irlap_start_mbusy_timer(struct irlap_cb *self, int timeout)
+{
+	irda_start_timer(&self->media_busy_timer, timeout,
+			 (void *) self, irlap_media_busy_expired);
+}
+
+void irlap_stop_mbusy_timer(struct irlap_cb *self)
+{
+	/* If timer is activated, kill it! */
+	del_timer(&self->media_busy_timer);
+
+	/* If we are in NDM, there is a bunch of events in LAP that
+	 * that be pending due to the media_busy condition, such as
+	 * CONNECT_REQUEST and SEND_UI_FRAME. If we don't generate
+	 * an event, they will wait forever...
+	 * Jean II */
+	if (self->state == LAP_NDM)
+		irlap_do_event(self, MEDIA_BUSY_TIMER_EXPIRED, NULL, NULL);
+}
+
+void irlmp_start_watchdog_timer(struct lsap_cb *self, int timeout)
+{
+	irda_start_timer(&self->watchdog_timer, timeout, (void *) self,
+			 irlmp_watchdog_timer_expired);
+}
+
+void irlmp_start_discovery_timer(struct irlmp_cb *self, int timeout)
+{
+	irda_start_timer(&self->discovery_timer, timeout, (void *) self,
+			 irlmp_discovery_timer_expired);
+}
+
+void irlmp_start_idle_timer(struct lap_cb *self, int timeout)
+{
+	irda_start_timer(&self->idle_timer, timeout, (void *) self,
+			 irlmp_idle_timer_expired);
+}
+
+void irlmp_stop_idle_timer(struct lap_cb *self)
+{
+	/* If timer is activated, kill it! */
+	del_timer(&self->idle_timer);
+}
+
+/*
+ * Function irlap_slot_timer_expired (data)
+ *
+ *    IrLAP slot timer has expired
+ *
+ */
+static void irlap_slot_timer_expired(void *data)
+{
+	struct irlap_cb *self = (struct irlap_cb *) data;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	irlap_do_event(self, SLOT_TIMER_EXPIRED, NULL, NULL);
+}
+
+/*
+ * Function irlap_query_timer_expired (data)
+ *
+ *    IrLAP query timer has expired
+ *
+ */
+static void irlap_query_timer_expired(void *data)
+{
+	struct irlap_cb *self = (struct irlap_cb *) data;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	irlap_do_event(self, QUERY_TIMER_EXPIRED, NULL, NULL);
+}
+
+/*
+ * Function irda_final_timer_expired (data)
+ *
+ *
+ *
+ */
+static void irlap_final_timer_expired(void *data)
+{
+	struct irlap_cb *self = (struct irlap_cb *) data;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	irlap_do_event(self, FINAL_TIMER_EXPIRED, NULL, NULL);
+}
+
+/*
+ * Function irda_wd_timer_expired (data)
+ *
+ *
+ *
+ */
+static void irlap_wd_timer_expired(void *data)
+{
+	struct irlap_cb *self = (struct irlap_cb *) data;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	irlap_do_event(self, WD_TIMER_EXPIRED, NULL, NULL);
+}
+
+/*
+ * Function irda_backoff_timer_expired (data)
+ *
+ *
+ *
+ */
+static void irlap_backoff_timer_expired(void *data)
+{
+	struct irlap_cb *self = (struct irlap_cb *) data;
+
+	IRDA_ASSERT(self != NULL, return;);
+	IRDA_ASSERT(self->magic == LAP_MAGIC, return;);
+
+	irlap_do_event(self, BACKOFF_TIMER_EXPIRED, NULL, NULL);
+}
+
+
+/*
+ * Function irtty_media_busy_expired (data)
+ *
+ *
+ */
+static void irlap_media_busy_expired(void *data)
+{
+	struct irlap_cb *self = (struct irlap_cb *) data;
+
+	IRDA_ASSERT(self != NULL, return;);
+
+	irda_device_set_media_busy(self->netdev, FALSE);
+	/* Note : the LAP event will be send in irlap_stop_mbusy_timer(),
+	* to catch other cases where the flag is cleared (for example
+	* after a discovery) - Jean II */
+}
diff --git a/drivers/staging/irda/net/wrapper.c b/drivers/staging/irda/net/wrapper.c
new file mode 100644
index 000000000000..40a0f993bf13
--- /dev/null
+++ b/drivers/staging/irda/net/wrapper.c
@@ -0,0 +1,492 @@
+/*********************************************************************
+ *
+ * Filename:      wrapper.c
+ * Version:       1.2
+ * Description:   IrDA SIR async wrapper layer
+ * Status:        Stable
+ * Author:        Dag Brattli <dagb@cs.uit.no>
+ * Created at:    Mon Aug  4 20:40:53 1997
+ * Modified at:   Fri Jan 28 13:21:09 2000
+ * Modified by:   Dag Brattli <dagb@cs.uit.no>
+ * Modified at:   Fri May 28  3:11 CST 1999
+ * Modified by:   Horst von Brand <vonbrand@sleipnir.valparaiso.cl>
+ *
+ *     Copyright (c) 1998-2000 Dag Brattli <dagb@cs.uit.no>,
+ *     All Rights Reserved.
+ *     Copyright (c) 2000-2002 Jean Tourrilhes <jt@hpl.hp.com>
+ *
+ *     This program is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of
+ *     the License, or (at your option) any later version.
+ *
+ *     Neither Dag Brattli nor University of Tromsø admit liability nor
+ *     provide warranty for any of this software. This material is
+ *     provided "AS-IS" and at no charge.
+ *
+ ********************************************************************/
+
+#include <linux/skbuff.h>
+#include <linux/string.h>
+#include <linux/module.h>
+#include <asm/byteorder.h>
+
+#include <net/irda/irda.h>
+#include <net/irda/wrapper.h>
+#include <net/irda/crc.h>
+#include <net/irda/irlap.h>
+#include <net/irda/irlap_frame.h>
+#include <net/irda/irda_device.h>
+
+/************************** FRAME WRAPPING **************************/
+/*
+ * Unwrap and unstuff SIR frames
+ *
+ * Note : at FIR and MIR, HDLC framing is used and usually handled
+ * by the controller, so we come here only for SIR... Jean II
+ */
+
+/*
+ * Function stuff_byte (byte, buf)
+ *
+ *    Byte stuff one single byte and put the result in buffer pointed to by
+ *    buf. The buffer must at all times be able to have two bytes inserted.
+ *
+ * This is in a tight loop, better inline it, so need to be prior to callers.
+ * (2000 bytes on P6 200MHz, non-inlined ~370us, inline ~170us) - Jean II
+ */
+static inline int stuff_byte(__u8 byte, __u8 *buf)
+{
+	switch (byte) {
+	case BOF: /* FALLTHROUGH */
+	case EOF: /* FALLTHROUGH */
+	case CE:
+		/* Insert transparently coded */
+		buf[0] = CE;               /* Send link escape */
+		buf[1] = byte^IRDA_TRANS;    /* Complement bit 5 */
+		return 2;
+		/* break; */
+	default:
+		 /* Non-special value, no transparency required */
+		buf[0] = byte;
+		return 1;
+		/* break; */
+	}
+}
+
+/*
+ * Function async_wrap (skb, *tx_buff, buffsize)
+ *
+ *    Makes a new buffer with wrapping and stuffing, should check that
+ *    we don't get tx buffer overflow.
+ */
+int async_wrap_skb(struct sk_buff *skb, __u8 *tx_buff, int buffsize)
+{
+	struct irda_skb_cb *cb = (struct irda_skb_cb *) skb->cb;
+	int xbofs;
+	int i;
+	int n;
+	union {
+		__u16 value;
+		__u8 bytes[2];
+	} fcs;
+
+	/* Initialize variables */
+	fcs.value = INIT_FCS;
+	n = 0;
+
+	/*
+	 *  Send  XBOF's for required min. turn time and for the negotiated
+	 *  additional XBOFS
+	 */
+
+	if (cb->magic != LAP_MAGIC) {
+		/*
+		 * This will happen for all frames sent from user-space.
+		 * Nothing to worry about, but we set the default number of
+		 * BOF's
+		 */
+		pr_debug("%s(), wrong magic in skb!\n", __func__);
+		xbofs = 10;
+	} else
+		xbofs = cb->xbofs + cb->xbofs_delay;
+
+	pr_debug("%s(), xbofs=%d\n", __func__, xbofs);
+
+	/* Check that we never use more than 115 + 48 xbofs */
+	if (xbofs > 163) {
+		pr_debug("%s(), too many xbofs (%d)\n", __func__,
+			 xbofs);
+		xbofs = 163;
+	}
+
+	memset(tx_buff + n, XBOF, xbofs);
+	n += xbofs;
+
+	/* Start of packet character BOF */
+	tx_buff[n++] = BOF;
+
+	/* Insert frame and calc CRC */
+	for (i=0; i < skb->len; i++) {
+		/*
+		 *  Check for the possibility of tx buffer overflow. We use
+		 *  bufsize-5 since the maximum number of bytes that can be
+		 *  transmitted after this point is 5.
+		 */
+		if(n >= (buffsize-5)) {
+			net_err_ratelimited("%s(), tx buffer overflow (n=%d)\n",
+					    __func__, n);
+			return n;
+		}
+
+		n += stuff_byte(skb->data[i], tx_buff+n);
+		fcs.value = irda_fcs(fcs.value, skb->data[i]);
+	}
+
+	/* Insert CRC in little endian format (LSB first) */
+	fcs.value = ~fcs.value;
+#ifdef __LITTLE_ENDIAN
+	n += stuff_byte(fcs.bytes[0], tx_buff+n);
+	n += stuff_byte(fcs.bytes[1], tx_buff+n);
+#else /* ifdef __BIG_ENDIAN */
+	n += stuff_byte(fcs.bytes[1], tx_buff+n);
+	n += stuff_byte(fcs.bytes[0], tx_buff+n);
+#endif
+	tx_buff[n++] = EOF;
+
+	return n;
+}
+EXPORT_SYMBOL(async_wrap_skb);
+
+/************************* FRAME UNWRAPPING *************************/
+/*
+ * Unwrap and unstuff SIR frames
+ *
+ * Complete rewrite by Jean II :
+ * More inline, faster, more compact, more logical. Jean II
+ * (16 bytes on P6 200MHz, old 5 to 7 us, new 4 to 6 us)
+ * (24 bytes on P6 200MHz, old 9 to 10 us, new 7 to 8 us)
+ * (for reference, 115200 b/s is 1 byte every 69 us)
+ * And reduce wrapper.o by ~900B in the process ;-)
+ *
+ * Then, we have the addition of ZeroCopy, which is optional
+ * (i.e. the driver must initiate it) and improve final processing.
+ * (2005 B frame + EOF on P6 200MHz, without 30 to 50 us, with 10 to 25 us)
+ *
+ * Note : at FIR and MIR, HDLC framing is used and usually handled
+ * by the controller, so we come here only for SIR... Jean II
+ */
+
+/*
+ * We can also choose where we want to do the CRC calculation. We can
+ * do it "inline", as we receive the bytes, or "postponed", when
+ * receiving the End-Of-Frame.
+ * (16 bytes on P6 200MHz, inlined 4 to 6 us, postponed 4 to 5 us)
+ * (24 bytes on P6 200MHz, inlined 7 to 8 us, postponed 5 to 7 us)
+ * With ZeroCopy :
+ * (2005 B frame on P6 200MHz, inlined 10 to 25 us, postponed 140 to 180 us)
+ * Without ZeroCopy :
+ * (2005 B frame on P6 200MHz, inlined 30 to 50 us, postponed 150 to 180 us)
+ * (Note : numbers taken with irq disabled)
+ *
+ * From those numbers, it's not clear which is the best strategy, because
+ * we end up running through a lot of data one way or another (i.e. cache
+ * misses). I personally prefer to avoid the huge latency spike of the
+ * "postponed" solution, because it come just at the time when we have
+ * lot's of protocol processing to do and it will hurt our ability to
+ * reach low link turnaround times... Jean II
+ */
+//#define POSTPONE_RX_CRC
+
+/*
+ * Function async_bump (buf, len, stats)
+ *
+ *    Got a frame, make a copy of it, and pass it up the stack! We can try
+ *    to inline it since it's only called from state_inside_frame
+ */
+static inline void
+async_bump(struct net_device *dev,
+	   struct net_device_stats *stats,
+	   iobuff_t *rx_buff)
+{
+	struct sk_buff *newskb;
+	struct sk_buff *dataskb;
+	int		docopy;
+
+	/* Check if we need to copy the data to a new skb or not.
+	 * If the driver doesn't use ZeroCopy Rx, we have to do it.
+	 * With ZeroCopy Rx, the rx_buff already point to a valid
+	 * skb. But, if the frame is small, it is more efficient to
+	 * copy it to save memory (copy will be fast anyway - that's
+	 * called Rx-copy-break). Jean II */
+	docopy = ((rx_buff->skb == NULL) ||
+		  (rx_buff->len < IRDA_RX_COPY_THRESHOLD));
+
+	/* Allocate a new skb */
+	newskb = dev_alloc_skb(docopy ? rx_buff->len + 1 : rx_buff->truesize);
+	if (!newskb)  {
+		stats->rx_dropped++;
+		/* We could deliver the current skb if doing ZeroCopy Rx,
+		 * but this would stall the Rx path. Better drop the
+		 * packet... Jean II */
+		return;
+	}
+
+	/* Align IP header to 20 bytes (i.e. increase skb->data)
+	 * Note this is only useful with IrLAN, as PPP has a variable
+	 * header size (2 or 1 bytes) - Jean II */
+	skb_reserve(newskb, 1);
+
+	if(docopy) {
+		/* Copy data without CRC (length already checked) */
+		skb_copy_to_linear_data(newskb, rx_buff->data,
+					rx_buff->len - 2);
+		/* Deliver this skb */
+		dataskb = newskb;
+	} else {
+		/* We are using ZeroCopy. Deliver old skb */
+		dataskb = rx_buff->skb;
+		/* And hook the new skb to the rx_buff */
+		rx_buff->skb = newskb;
+		rx_buff->head = newskb->data;	/* NOT newskb->head */
+		//printk(KERN_DEBUG "ZeroCopy : len = %d, dataskb = %p, newskb = %p\n", rx_buff->len, dataskb, newskb);
+	}
+
+	/* Set proper length on skb (without CRC) */
+	skb_put(dataskb, rx_buff->len - 2);
+
+	/* Feed it to IrLAP layer */
+	dataskb->dev = dev;
+	skb_reset_mac_header(dataskb);
+	dataskb->protocol = htons(ETH_P_IRDA);
+
+	netif_rx(dataskb);
+
+	stats->rx_packets++;
+	stats->rx_bytes += rx_buff->len;
+
+	/* Clean up rx_buff (redundant with async_unwrap_bof() ???) */
+	rx_buff->data = rx_buff->head;
+	rx_buff->len = 0;
+}
+
+/*
+ * Function async_unwrap_bof(dev, byte)
+ *
+ *    Handle Beginning Of Frame character received within a frame
+ *
+ */
+static inline void
+async_unwrap_bof(struct net_device *dev,
+		 struct net_device_stats *stats,
+		 iobuff_t *rx_buff, __u8 byte)
+{
+	switch(rx_buff->state) {
+	case LINK_ESCAPE:
+	case INSIDE_FRAME:
+		/* Not supposed to happen, the previous frame is not
+		 * finished - Jean II */
+		pr_debug("%s(), Discarding incomplete frame\n",
+			 __func__);
+		stats->rx_errors++;
+		stats->rx_missed_errors++;
+		irda_device_set_media_busy(dev, TRUE);
+		break;
+
+	case OUTSIDE_FRAME:
+	case BEGIN_FRAME:
+	default:
+		/* We may receive multiple BOF at the start of frame */
+		break;
+	}
+
+	/* Now receiving frame */
+	rx_buff->state = BEGIN_FRAME;
+	rx_buff->in_frame = TRUE;
+
+	/* Time to initialize receive buffer */
+	rx_buff->data = rx_buff->head;
+	rx_buff->len = 0;
+	rx_buff->fcs = INIT_FCS;
+}
+
+/*
+ * Function async_unwrap_eof(dev, byte)
+ *
+ *    Handle End Of Frame character received within a frame
+ *
+ */
+static inline void
+async_unwrap_eof(struct net_device *dev,
+		 struct net_device_stats *stats,
+		 iobuff_t *rx_buff, __u8 byte)
+{
+#ifdef POSTPONE_RX_CRC
+	int	i;
+#endif
+
+	switch(rx_buff->state) {
+	case OUTSIDE_FRAME:
+		/* Probably missed the BOF */
+		stats->rx_errors++;
+		stats->rx_missed_errors++;
+		irda_device_set_media_busy(dev, TRUE);
+		break;
+
+	case BEGIN_FRAME:
+	case LINK_ESCAPE:
+	case INSIDE_FRAME:
+	default:
+		/* Note : in the case of BEGIN_FRAME and LINK_ESCAPE,
+		 * the fcs will most likely not match and generate an
+		 * error, as expected - Jean II */
+		rx_buff->state = OUTSIDE_FRAME;
+		rx_buff->in_frame = FALSE;
+
+#ifdef POSTPONE_RX_CRC
+		/* If we haven't done the CRC as we receive bytes, we
+		 * must do it now... Jean II */
+		for(i = 0; i < rx_buff->len; i++)
+			rx_buff->fcs = irda_fcs(rx_buff->fcs,
+						rx_buff->data[i]);
+#endif
+
+		/* Test FCS and signal success if the frame is good */
+		if (rx_buff->fcs == GOOD_FCS) {
+			/* Deliver frame */
+			async_bump(dev, stats, rx_buff);
+			break;
+		} else {
+			/* Wrong CRC, discard frame!  */
+			irda_device_set_media_busy(dev, TRUE);
+
+			pr_debug("%s(), crc error\n", __func__);
+			stats->rx_errors++;
+			stats->rx_crc_errors++;
+		}
+		break;
+	}
+}
+
+/*
+ * Function async_unwrap_ce(dev, byte)
+ *
+ *    Handle Character Escape character received within a frame
+ *
+ */
+static inline void
+async_unwrap_ce(struct net_device *dev,
+		 struct net_device_stats *stats,
+		 iobuff_t *rx_buff, __u8 byte)
+{
+	switch(rx_buff->state) {
+	case OUTSIDE_FRAME:
+		/* Activate carrier sense */
+		irda_device_set_media_busy(dev, TRUE);
+		break;
+
+	case LINK_ESCAPE:
+		net_warn_ratelimited("%s: state not defined\n", __func__);
+		break;
+
+	case BEGIN_FRAME:
+	case INSIDE_FRAME:
+	default:
+		/* Stuffed byte coming */
+		rx_buff->state = LINK_ESCAPE;
+		break;
+	}
+}
+
+/*
+ * Function async_unwrap_other(dev, byte)
+ *
+ *    Handle other characters received within a frame
+ *
+ */
+static inline void
+async_unwrap_other(struct net_device *dev,
+		   struct net_device_stats *stats,
+		   iobuff_t *rx_buff, __u8 byte)
+{
+	switch(rx_buff->state) {
+		/* This is on the critical path, case are ordered by
+		 * probability (most frequent first) - Jean II */
+	case INSIDE_FRAME:
+		/* Must be the next byte of the frame */
+		if (rx_buff->len < rx_buff->truesize)  {
+			rx_buff->data[rx_buff->len++] = byte;
+#ifndef POSTPONE_RX_CRC
+			rx_buff->fcs = irda_fcs(rx_buff->fcs, byte);
+#endif
+		} else {
+			pr_debug("%s(), Rx buffer overflow, aborting\n",
+				 __func__);
+			rx_buff->state = OUTSIDE_FRAME;
+		}
+		break;
+
+	case LINK_ESCAPE:
+		/*
+		 *  Stuffed char, complement bit 5 of byte
+		 *  following CE, IrLAP p.114
+		 */
+		byte ^= IRDA_TRANS;
+		if (rx_buff->len < rx_buff->truesize)  {
+			rx_buff->data[rx_buff->len++] = byte;
+#ifndef POSTPONE_RX_CRC
+			rx_buff->fcs = irda_fcs(rx_buff->fcs, byte);
+#endif
+			rx_buff->state = INSIDE_FRAME;
+		} else {
+			pr_debug("%s(), Rx buffer overflow, aborting\n",
+				 __func__);
+			rx_buff->state = OUTSIDE_FRAME;
+		}
+		break;
+
+	case OUTSIDE_FRAME:
+		/* Activate carrier sense */
+		if(byte != XBOF)
+			irda_device_set_media_busy(dev, TRUE);
+		break;
+
+	case BEGIN_FRAME:
+	default:
+		rx_buff->data[rx_buff->len++] = byte;
+#ifndef POSTPONE_RX_CRC
+		rx_buff->fcs = irda_fcs(rx_buff->fcs, byte);
+#endif
+		rx_buff->state = INSIDE_FRAME;
+		break;
+	}
+}
+
+/*
+ * Function async_unwrap_char (dev, rx_buff, byte)
+ *
+ *    Parse and de-stuff frame received from the IrDA-port
+ *
+ * This is the main entry point for SIR drivers.
+ */
+void async_unwrap_char(struct net_device *dev,
+		       struct net_device_stats *stats,
+		       iobuff_t *rx_buff, __u8 byte)
+{
+	switch(byte) {
+	case CE:
+		async_unwrap_ce(dev, stats, rx_buff, byte);
+		break;
+	case BOF:
+		async_unwrap_bof(dev, stats, rx_buff, byte);
+		break;
+	case EOF:
+		async_unwrap_eof(dev, stats, rx_buff, byte);
+		break;
+	default:
+		async_unwrap_other(dev, stats, rx_buff, byte);
+		break;
+	}
+}
+EXPORT_SYMBOL(async_unwrap_char);
+
diff --git a/drivers/staging/ks7010/ks7010_sdio.c b/drivers/staging/ks7010/ks7010_sdio.c
index 9b28ee1cfb1e..8cfdff198334 100644
--- a/drivers/staging/ks7010/ks7010_sdio.c
+++ b/drivers/staging/ks7010/ks7010_sdio.c
@@ -834,8 +834,6 @@ static int ks7010_sdio_probe(struct sdio_func *func,
 	unsigned char byte;
 	int ret;
 
-	DPRINTK(5, "ks7010_sdio_probe()\n");
-
 	priv = NULL;
 	netdev = NULL;
 
@@ -1008,8 +1006,6 @@ static void ks7010_sdio_remove(struct sdio_func *func)
 	struct ks_sdio_card *card;
 	struct ks_wlan_private *priv;
 
-	DPRINTK(1, "ks7010_sdio_remove()\n");
-
 	card = sdio_get_drvdata(func);
 
 	if (!card)
diff --git a/drivers/staging/ks7010/ks_hostif.c b/drivers/staging/ks7010/ks_hostif.c
index 24a63161f92c..975dbbb3abd0 100644
--- a/drivers/staging/ks7010/ks_hostif.c
+++ b/drivers/staging/ks7010/ks_hostif.c
@@ -1658,8 +1658,8 @@ void hostif_phy_information_request(struct ks_wlan_private *priv)
 
 static
 void hostif_power_mgmt_request(struct ks_wlan_private *priv,
-				unsigned long mode, unsigned long wake_up,
-				unsigned long receive_dtims)
+			       unsigned long mode, unsigned long wake_up,
+			       unsigned long receive_dtims)
 {
 	struct hostif_power_mgmt_request_t *pp;
 
diff --git a/drivers/staging/ks7010/ks_wlan_net.c b/drivers/staging/ks7010/ks_wlan_net.c
index 8aa12e813bd7..0f9348ba5d84 100644
--- a/drivers/staging/ks7010/ks_wlan_net.c
+++ b/drivers/staging/ks7010/ks_wlan_net.c
@@ -1356,7 +1356,7 @@ static inline char *ks_wlan_translate_scan(struct net_device *dev,
 
 	/* Add mode */
 	iwe.cmd = SIOCGIWMODE;
-	capabilities = le16_to_cpu(ap->capability);
+	capabilities = ap->capability;
 	if (capabilities & (BSS_CAP_ESS | BSS_CAP_IBSS)) {
 		if (capabilities & BSS_CAP_ESS)
 			iwe.u.mode = IW_MODE_INFRA;
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs.h b/drivers/staging/lustre/include/linux/libcfs/libcfs.h
index cc2c0e97bb7e..b48e2f093bcc 100644
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs.h
+++ b/drivers/staging/lustre/include/linux/libcfs/libcfs.h
@@ -33,10 +33,21 @@
 #ifndef __LIBCFS_LIBCFS_H__
 #define __LIBCFS_LIBCFS_H__
 
-#include "linux/libcfs.h"
 #include <linux/gfp.h>
+#include <linux/list.h>
 
-#include "curproc.h"
+#include <uapi/linux/lnet/libcfs_ioctl.h>
+#include <linux/libcfs/linux/libcfs.h>
+#include <linux/libcfs/libcfs_debug.h>
+#include <linux/libcfs/libcfs_private.h>
+#include <linux/libcfs/libcfs_cpu.h>
+#include <linux/libcfs/libcfs_prim.h>
+#include <linux/libcfs/libcfs_time.h>
+#include <linux/libcfs/libcfs_string.h>
+#include <linux/libcfs/libcfs_workitem.h>
+#include <linux/libcfs/libcfs_hash.h>
+#include <linux/libcfs/libcfs_fail.h>
+#include <linux/libcfs/curproc.h>
 
 #define LIBCFS_VERSION "0.7.0"
 
@@ -49,8 +60,6 @@
 #define LERRCHKSUM(hexnum) (((hexnum) & 0xf) ^ ((hexnum) >> 4 & 0xf) ^ \
 			   ((hexnum) >> 8 & 0xf))
 
-#include <linux/list.h>
-
 /* need both kernel and user-land acceptor */
 #define LNET_ACCEPTOR_MIN_RESERVED_PORT    512
 #define LNET_ACCEPTOR_MAX_RESERVED_PORT    1023
@@ -74,17 +83,6 @@ unsigned int cfs_rand(void);
 void cfs_srand(unsigned int seed1, unsigned int seed2);
 void cfs_get_random_bytes(void *buf, int size);
 
-#include "libcfs_debug.h"
-#include "libcfs_cpu.h"
-#include "libcfs_private.h"
-#include "libcfs_ioctl.h"
-#include "libcfs_prim.h"
-#include "libcfs_time.h"
-#include "libcfs_string.h"
-#include "libcfs_workitem.h"
-#include "libcfs_hash.h"
-#include "libcfs_fail.h"
-
 struct libcfs_ioctl_handler {
 	struct list_head item;
 	int (*handle_ioctl)(unsigned int cmd, struct libcfs_ioctl_hdr *hdr);
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs_debug.h b/drivers/staging/lustre/include/linux/libcfs/libcfs_debug.h
index b7bd6e8ab33f..e7c37415a0c7 100644
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs_debug.h
+++ b/drivers/staging/lustre/include/linux/libcfs/libcfs_debug.h
@@ -38,6 +38,8 @@
 #ifndef __LIBCFS_DEBUG_H__
 #define __LIBCFS_DEBUG_H__
 
+#include <uapi/linux/lnet/libcfs_debug.h>
+
 /*
  *  Debugging
  */
@@ -59,108 +61,6 @@ int libcfs_debug_str2mask(int *mask, const char *str, int is_subsys);
 extern unsigned int libcfs_catastrophe;
 extern unsigned int libcfs_panic_on_lbug;
 
-/**
- * Format for debug message headers
- */
-struct ptldebug_header {
-	__u32 ph_len;
-	__u32 ph_flags;
-	__u32 ph_subsys;
-	__u32 ph_mask;
-	__u16 ph_cpu_id;
-	__u16 ph_type;
-	/* time_t overflow in 2106 */
-	__u32 ph_sec;
-	__u64 ph_usec;
-	__u32 ph_stack;
-	__u32 ph_pid;
-	__u32 ph_extern_pid;
-	__u32 ph_line_num;
-} __packed;
-
-#define PH_FLAG_FIRST_RECORD 1
-
-/* Debugging subsystems (32 bits, non-overlapping) */
-#define S_UNDEFINED	0x00000001
-#define S_MDC		0x00000002
-#define S_MDS		0x00000004
-#define S_OSC		0x00000008
-#define S_OST		0x00000010
-#define S_CLASS		0x00000020
-#define S_LOG		0x00000040
-#define S_LLITE		0x00000080
-#define S_RPC		0x00000100
-#define S_MGMT		0x00000200
-#define S_LNET		0x00000400
-#define S_LND		0x00000800 /* ALL LNDs */
-#define S_PINGER	0x00001000
-#define S_FILTER	0x00002000
-/* unused */
-#define S_ECHO		0x00008000
-#define S_LDLM		0x00010000
-#define S_LOV		0x00020000
-#define S_LQUOTA	0x00040000
-#define S_OSD		0x00080000
-#define S_LFSCK		0x00100000
-/* unused */
-/* unused */
-#define S_LMV		0x00800000 /* b_new_cmd */
-/* unused */
-#define S_SEC		0x02000000 /* upcall cache */
-#define S_GSS		0x04000000 /* b_new_cmd */
-/* unused */
-#define S_MGC		0x10000000
-#define S_MGS		0x20000000
-#define S_FID		0x40000000 /* b_new_cmd */
-#define S_FLD		0x80000000 /* b_new_cmd */
-
-#define LIBCFS_DEBUG_SUBSYS_NAMES {					\
-	"undefined", "mdc", "mds", "osc", "ost", "class", "log",	\
-	"llite", "rpc", "mgmt", "lnet", "lnd", "pinger", "filter", "",	\
-	"echo", "ldlm", "lov", "lquota", "osd", "lfsck", "", "", "lmv",	\
-	"", "sec", "gss", "", "mgc", "mgs", "fid", "fld", NULL }
-
-/* Debugging masks (32 bits, non-overlapping) */
-#define D_TRACE		0x00000001 /* ENTRY/EXIT markers */
-#define D_INODE		0x00000002
-#define D_SUPER		0x00000004
-#define D_EXT2		0x00000008 /* anything from ext2_debug */
-#define D_MALLOC	0x00000010 /* print malloc, free information */
-#define D_CACHE		0x00000020 /* cache-related items */
-#define D_INFO		0x00000040 /* general information */
-#define D_IOCTL		0x00000080 /* ioctl related information */
-#define D_NETERROR	0x00000100 /* network errors */
-#define D_NET		0x00000200 /* network communications */
-#define D_WARNING	0x00000400 /* CWARN(...) == CDEBUG (D_WARNING, ...) */
-#define D_BUFFS		0x00000800
-#define D_OTHER		0x00001000
-#define D_DENTRY	0x00002000
-#define D_NETTRACE	0x00004000
-#define D_PAGE		0x00008000 /* bulk page handling */
-#define D_DLMTRACE	0x00010000
-#define D_ERROR		0x00020000 /* CERROR(...) == CDEBUG (D_ERROR, ...) */
-#define D_EMERG		0x00040000 /* CEMERG(...) == CDEBUG (D_EMERG, ...) */
-#define D_HA		0x00080000 /* recovery and failover */
-#define D_RPCTRACE	0x00100000 /* for distributed debugging */
-#define D_VFSTRACE	0x00200000
-#define D_READA		0x00400000 /* read-ahead */
-#define D_MMAP		0x00800000
-#define D_CONFIG	0x01000000
-#define D_CONSOLE	0x02000000
-#define D_QUOTA		0x04000000
-#define D_SEC		0x08000000
-#define D_LFSCK		0x10000000 /* For both OI scrub and LFSCK */
-#define D_HSM		0x20000000
-
-#define LIBCFS_DEBUG_MASKS_NAMES {					\
-	"trace", "inode", "super", "ext2", "malloc", "cache", "info",	\
-	"ioctl", "neterror", "net", "warning", "buffs", "other",	\
-	"dentry", "nettrace", "page", "dlmtrace", "error", "emerg",	\
-	"ha", "rpctrace", "vfstrace", "reada", "mmap", "config",	\
-	"console", "quota", "sec", "lfsck", "hsm", NULL }
-
-#define D_CANTMASK   (D_ERROR | D_EMERG | D_WARNING | D_CONSOLE)
-
 #ifndef DEBUG_SUBSYSTEM
 # define DEBUG_SUBSYSTEM S_UNDEFINED
 #endif
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs_private.h b/drivers/staging/lustre/include/linux/libcfs/libcfs_private.h
index e774c75ecadd..709771d27f89 100644
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs_private.h
+++ b/drivers/staging/lustre/include/linux/libcfs/libcfs_private.h
@@ -154,18 +154,6 @@ do {								    \
 
 /******************************************************************************/
 
-/* htonl hack - either this, or compile with -O2. Stupid byteorder/generic.h */
-#if defined(__GNUC__) && (__GNUC__ >= 2) && !defined(__OPTIMIZE__)
-#define ___htonl(x) __cpu_to_be32(x)
-#define ___htons(x) __cpu_to_be16(x)
-#define ___ntohl(x) __be32_to_cpu(x)
-#define ___ntohs(x) __be16_to_cpu(x)
-#define htonl(x) ___htonl(x)
-#define ntohl(x) ___ntohl(x)
-#define htons(x) ___htons(x)
-#define ntohs(x) ___ntohs(x)
-#endif
-
 void libcfs_debug_dumplog(void);
 int libcfs_debug_init(unsigned long bufsize);
 int libcfs_debug_cleanup(void);
@@ -308,18 +296,4 @@ static inline size_t cfs_round_strlen(char *fset)
 	return cfs_size_round((int)strlen(fset) + 1);
 }
 
-#define LOGL(var, len, ptr)				       \
-do {							    \
-	if (var)						\
-		memcpy((char *)ptr, (const char *)var, len);    \
-	ptr += cfs_size_round(len);			     \
-} while (0)
-
-#define LOGU(var, len, ptr)				       \
-do {							    \
-	if (var)						\
-		memcpy((char *)var, (const char *)ptr, len);    \
-	ptr += cfs_size_round(len);			     \
-} while (0)
-
 #endif
diff --git a/drivers/staging/lustre/include/linux/lnet/api.h b/drivers/staging/lustre/include/linux/lnet/api.h
index f4b6de2ec0ff..9c37f3e4b134 100644
--- a/drivers/staging/lustre/include/linux/lnet/api.h
+++ b/drivers/staging/lustre/include/linux/lnet/api.h
@@ -44,7 +44,7 @@
  * @{
  */
 
-#include "../lnet/types.h"
+#include <uapi/linux/lnet/lnet-types.h>
 
 /** \defgroup lnet_init_fini Initialization and cleanup
  * The LNet must be properly initialized before any LNet calls can be made.
diff --git a/drivers/staging/lustre/include/linux/lnet/lib-lnet.h b/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
index 8ae7423b4543..e0968ab8d95e 100644
--- a/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
+++ b/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
@@ -35,11 +35,13 @@
 #ifndef __LNET_LIB_LNET_H__
 #define __LNET_LIB_LNET_H__
 
-#include "../libcfs/libcfs.h"
-#include "api.h"
-#include "lnet.h"
-#include "lib-types.h"
-#include "lib-dlc.h"
+#include <linux/libcfs/libcfs.h>
+#include <linux/lnet/api.h>
+#include <linux/lnet/lib-types.h>
+#include <uapi/linux/lnet/lnet-dlc.h>
+#include <uapi/linux/lnet/lnet-types.h>
+#include <uapi/linux/lnet/lnetctl.h>
+#include <uapi/linux/lnet/nidstr.h>
 
 extern struct lnet the_lnet;	/* THE network */
 
@@ -453,7 +455,8 @@ extern int portal_rotor;
 int lnet_lib_init(void);
 void lnet_lib_exit(void);
 
-int lnet_notify(struct lnet_ni *ni, lnet_nid_t peer, int alive, unsigned long when);
+int lnet_notify(struct lnet_ni *ni, lnet_nid_t peer, int alive,
+		unsigned long when);
 void lnet_notify_locked(struct lnet_peer *lp, int notifylnd, int alive,
 			unsigned long when);
 int lnet_add_route(__u32 net, __u32 hops, lnet_nid_t gateway_nid,
diff --git a/drivers/staging/lustre/include/linux/lnet/lib-types.h b/drivers/staging/lustre/include/linux/lnet/lib-types.h
index 321752dfe58b..eea3b8e5e406 100644
--- a/drivers/staging/lustre/include/linux/lnet/lib-types.h
+++ b/drivers/staging/lustre/include/linux/lnet/lib-types.h
@@ -40,8 +40,8 @@
 #include <linux/types.h>
 #include <linux/completion.h>
 
-#include "types.h"
-#include "lnetctl.h"
+#include <uapi/linux/lnet/lnet-types.h>
+#include <uapi/linux/lnet/lnetctl.h>
 
 /* Max payload size */
 #define LNET_MAX_PAYLOAD      CONFIG_LNET_MAX_PAYLOAD
@@ -308,9 +308,11 @@ struct lnet_rc_data {
 struct lnet_peer {
 	struct list_head	 lp_hashlist;	/* chain on peer hash */
 	struct list_head	 lp_txq;	/* messages blocking for
-						   tx credits */
+						 * tx credits
+						 */
 	struct list_head	 lp_rtrq;	/* messages blocking for
-						   router credits */
+						 * router credits
+						 */
 	struct list_head	 lp_rtr_list;	/* chain on router list */
 	int			 lp_txcredits;	/* # tx credits available */
 	int			 lp_mintxcredits;  /* low water mark */
@@ -319,23 +321,31 @@ struct lnet_peer {
 	unsigned int		 lp_alive:1;	   /* alive/dead? */
 	unsigned int		 lp_notify:1;	/* notification outstanding? */
 	unsigned int		 lp_notifylnd:1;/* outstanding notification
-						   for LND? */
+						 * for LND?
+						 */
 	unsigned int		 lp_notifying:1; /* some thread is handling
-						    notification */
+						  * notification
+						  */
 	unsigned int		 lp_ping_notsent;/* SEND event outstanding
-						    from ping */
+						  * from ping
+						  */
 	int			 lp_alive_count; /* # times router went
-						    dead<->alive */
-	long			 lp_txqnob;	 /* bytes queued for sending */
+						  * dead<->alive
+						  */
+	long			 lp_txqnob;	 /* ytes queued for sending */
 	unsigned long		 lp_timestamp;	 /* time of last aliveness
-						    news */
+						  * news
+						  */
 	unsigned long		 lp_ping_timestamp;/* time of last ping
-						      attempt */
+						    * attempt
+						    */
 	unsigned long		 lp_ping_deadline; /* != 0 if ping reply
-						      expected */
+						    * expected
+						    */
 	unsigned long		 lp_last_alive;	/* when I was last alive */
 	unsigned long		 lp_last_query;	/* when lp_ni was queried
-						   last time */
+						 * last time
+						 */
 	struct lnet_ni		*lp_ni;		/* interface peer is on */
 	lnet_nid_t		 lp_nid;	/* peer's NID */
 	int			 lp_refcount;	/* # refs */
@@ -386,7 +396,8 @@ struct lnet_route {
 
 struct lnet_remotenet {
 	struct list_head	lrn_list;	/* chain on
-						   ln_remote_nets_hash */
+						 * ln_remote_nets_hash
+						 */
 	struct list_head	lrn_routes;	/* routes to me */
 	__u32			lrn_net;	/* my net number */
 };
@@ -399,14 +410,16 @@ struct lnet_remotenet {
 struct lnet_rtrbufpool {
 	struct list_head	rbp_bufs;	/* my free buffer pool */
 	struct list_head	rbp_msgs;	/* messages blocking
-						   for a buffer */
+						 * for a buffer
+						 */
 	int			rbp_npages;	/* # pages in each buffer */
 	/* requested number of buffers */
 	int			rbp_req_nbuffers;
 	/* # buffers actually allocated */
 	int			rbp_nbuffers;
-	int			rbp_credits;	/* # free buffers /
-						     blocked messages */
+	int			rbp_credits;	/* # free buffers
+						 * blocked messages
+						 */
 	int			rbp_mincredits;	/* low water mark */
 };
 
@@ -442,7 +455,8 @@ enum lnet_match_flags {
 #define LNET_PTL_LAZY		(1 << 0)
 #define LNET_PTL_MATCH_UNIQUE	(1 << 1)	/* unique match, for RDMA */
 #define LNET_PTL_MATCH_WILDCARD	(1 << 2)	/* wildcard match,
-						   request portal */
+						 * request portal
+						 */
 
 /* parameter for matching operations (GET, PUT) */
 struct lnet_match_info {
diff --git a/drivers/staging/lustre/include/linux/lnet/socklnd.h b/drivers/staging/lustre/include/linux/lnet/socklnd.h
index dd5bc0e46560..553fb64b3e80 100644
--- a/drivers/staging/lustre/include/linux/lnet/socklnd.h
+++ b/drivers/staging/lustre/include/linux/lnet/socklnd.h
@@ -34,16 +34,8 @@
 #ifndef __LNET_LNET_SOCKLND_H__
 #define __LNET_LNET_SOCKLND_H__
 
-#include "types.h"
-
-#define SOCKLND_CONN_NONE     (-1)
-#define SOCKLND_CONN_ANY	0
-#define SOCKLND_CONN_CONTROL	1
-#define SOCKLND_CONN_BULK_IN	2
-#define SOCKLND_CONN_BULK_OUT	3
-#define SOCKLND_CONN_NTYPES	4
-
-#define SOCKLND_CONN_ACK	SOCKLND_CONN_BULK_IN
+#include <uapi/linux/lnet/lnet-types.h>
+#include <uapi/linux/lnet/socklnd.h>
 
 struct ksock_hello_msg {
 	__u32		kshm_magic;	/* magic number of socklnd message */
@@ -76,7 +68,8 @@ struct ksock_msg {
 	__u64	ksm_zc_cookies[2];	/* Zero-Copy request/ACK cookie */
 	union {
 		struct ksock_lnet_msg lnetmsg; /* lnet message, it's empty if
-					  * it's NOOP */
+						* it's NOOP
+						*/
 	} WIRE_ATTR ksm_u;
 } WIRE_ATTR;
 
diff --git a/drivers/staging/lustre/include/uapi/linux/lnet/libcfs_debug.h b/drivers/staging/lustre/include/uapi/linux/lnet/libcfs_debug.h
new file mode 100644
index 000000000000..c4d9472b374f
--- /dev/null
+++ b/drivers/staging/lustre/include/uapi/linux/lnet/libcfs_debug.h
@@ -0,0 +1,149 @@
+/*
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see
+ * http://www.gnu.org/licenses/gpl-2.0.html
+ *
+ * GPL HEADER END
+ */
+/*
+ * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Use is subject to license terms.
+ *
+ * Copyright (c) 2012, 2014, Intel Corporation.
+ */
+/*
+ * This file is part of Lustre, http://www.lustre.org/
+ * Lustre is a trademark of Sun Microsystems, Inc.
+ *
+ * libcfs/include/libcfs/libcfs_debug.h
+ *
+ * Debug messages and assertions
+ *
+ */
+
+#ifndef __UAPI_LIBCFS_DEBUG_H__
+#define __UAPI_LIBCFS_DEBUG_H__
+
+/**
+ * Format for debug message headers
+ */
+struct ptldebug_header {
+	__u32 ph_len;
+	__u32 ph_flags;
+	__u32 ph_subsys;
+	__u32 ph_mask;
+	__u16 ph_cpu_id;
+	__u16 ph_type;
+	/* time_t overflow in 2106 */
+	__u32 ph_sec;
+	__u64 ph_usec;
+	__u32 ph_stack;
+	__u32 ph_pid;
+	__u32 ph_extern_pid;
+	__u32 ph_line_num;
+} __attribute__((packed));
+
+#define PH_FLAG_FIRST_RECORD	1
+
+/* Debugging subsystems (32 bits, non-overlapping) */
+#define S_UNDEFINED     0x00000001
+#define S_MDC           0x00000002
+#define S_MDS           0x00000004
+#define S_OSC           0x00000008
+#define S_OST           0x00000010
+#define S_CLASS         0x00000020
+#define S_LOG           0x00000040
+#define S_LLITE         0x00000080
+#define S_RPC           0x00000100
+#define S_MGMT          0x00000200
+#define S_LNET          0x00000400
+#define S_LND           0x00000800 /* ALL LNDs */
+#define S_PINGER        0x00001000
+#define S_FILTER        0x00002000
+#define S_LIBCFS        0x00004000
+#define S_ECHO          0x00008000
+#define S_LDLM          0x00010000
+#define S_LOV           0x00020000
+#define S_LQUOTA        0x00040000
+#define S_OSD           0x00080000
+#define S_LFSCK         0x00100000
+#define S_SNAPSHOT      0x00200000
+/* unused */
+#define S_LMV           0x00800000 /* b_new_cmd */
+/* unused */
+#define S_SEC           0x02000000 /* upcall cache */
+#define S_GSS           0x04000000 /* b_new_cmd */
+/* unused */
+#define S_MGC           0x10000000
+#define S_MGS           0x20000000
+#define S_FID           0x40000000 /* b_new_cmd */
+#define S_FLD           0x80000000 /* b_new_cmd */
+
+#define LIBCFS_DEBUG_SUBSYS_NAMES {					\
+	"undefined", "mdc", "mds", "osc", "ost", "class", "log",	\
+	"llite", "rpc", "mgmt", "lnet", "lnd", "pinger", "filter",	\
+	"libcfs", "echo", "ldlm", "lov", "lquota", "osd", "lfsck",	\
+	"snapshot", "", "lmv", "", "sec", "gss", "", "mgc", "mgs",	\
+	"fid", "fld", NULL }
+
+/* Debugging masks (32 bits, non-overlapping) */
+#define D_TRACE         0x00000001 /* ENTRY/EXIT markers */
+#define D_INODE         0x00000002
+#define D_SUPER         0x00000004
+#define D_EXT2          0x00000008 /* anything from ext2_debug */
+#define D_MALLOC        0x00000010 /* print malloc, free information */
+#define D_CACHE         0x00000020 /* cache-related items */
+#define D_INFO          0x00000040 /* general information */
+#define D_IOCTL         0x00000080 /* ioctl related information */
+#define D_NETERROR      0x00000100 /* network errors */
+#define D_NET           0x00000200 /* network communications */
+#define D_WARNING       0x00000400 /* CWARN(...) == CDEBUG (D_WARNING, ...) */
+#define D_BUFFS         0x00000800
+#define D_OTHER         0x00001000
+#define D_DENTRY        0x00002000
+#define D_NETTRACE      0x00004000
+#define D_PAGE          0x00008000 /* bulk page handling */
+#define D_DLMTRACE      0x00010000
+#define D_ERROR         0x00020000 /* CERROR(...) == CDEBUG (D_ERROR, ...) */
+#define D_EMERG         0x00040000 /* CEMERG(...) == CDEBUG (D_EMERG, ...) */
+#define D_HA            0x00080000 /* recovery and failover */
+#define D_RPCTRACE      0x00100000 /* for distributed debugging */
+#define D_VFSTRACE      0x00200000
+#define D_READA         0x00400000 /* read-ahead */
+#define D_MMAP          0x00800000
+#define D_CONFIG        0x01000000
+#define D_CONSOLE       0x02000000
+#define D_QUOTA         0x04000000
+#define D_SEC           0x08000000
+#define D_LFSCK         0x10000000 /* For both OI scrub and LFSCK */
+#define D_HSM           0x20000000
+#define D_SNAPSHOT      0x40000000 /* snapshot */
+#define D_LAYOUT        0x80000000
+
+#define LIBCFS_DEBUG_MASKS_NAMES {					\
+	"trace", "inode", "super", "ext2", "malloc", "cache", "info",	\
+	"ioctl", "neterror", "net", "warning", "buffs", "other",	\
+	"dentry", "nettrace", "page", "dlmtrace", "error", "emerg",	\
+	"ha", "rpctrace", "vfstrace", "reada", "mmap", "config",	\
+	"console", "quota", "sec", "lfsck", "hsm", "snapshot", "layout",\
+	NULL }
+
+#define D_CANTMASK   (D_ERROR | D_EMERG | D_WARNING | D_CONSOLE)
+
+#define LIBCFS_DEBUG_FILE_PATH_DEFAULT "/tmp/lustre-log"
+
+#endif	/* __UAPI_LIBCFS_DEBUG_H__ */
diff --git a/drivers/staging/lustre/include/linux/libcfs/libcfs_ioctl.h b/drivers/staging/lustre/include/uapi/linux/lnet/libcfs_ioctl.h
index cce6b58e3682..cce6b58e3682 100644
--- a/drivers/staging/lustre/include/linux/libcfs/libcfs_ioctl.h
+++ b/drivers/staging/lustre/include/uapi/linux/lnet/libcfs_ioctl.h
diff --git a/drivers/staging/lustre/include/linux/lnet/lib-dlc.h b/drivers/staging/lustre/include/uapi/linux/lnet/lnet-dlc.h
index dfff17088403..e45d828bfd1b 100644
--- a/drivers/staging/lustre/include/linux/lnet/lib-dlc.h
+++ b/drivers/staging/lustre/include/uapi/linux/lnet/lnet-dlc.h
@@ -29,8 +29,8 @@
 #ifndef LNET_DLC_H
 #define LNET_DLC_H
 
-#include "../libcfs/libcfs_ioctl.h"
-#include "types.h"
+#include <uapi/linux/lnet/libcfs_ioctl.h>
+#include <uapi/linux/lnet/lnet-types.h>
 
 #define MAX_NUM_SHOW_ENTRIES	32
 #define LNET_MAX_STR_LEN	128
diff --git a/drivers/staging/lustre/include/linux/lnet/types.h b/drivers/staging/lustre/include/uapi/linux/lnet/lnet-types.h
index 1be9b7aa7326..1be9b7aa7326 100644
--- a/drivers/staging/lustre/include/linux/lnet/types.h
+++ b/drivers/staging/lustre/include/uapi/linux/lnet/lnet-types.h
diff --git a/drivers/staging/lustre/include/linux/lnet/lnetctl.h b/drivers/staging/lustre/include/uapi/linux/lnet/lnetctl.h
index 39575073b00b..d9da625d70de 100644
--- a/drivers/staging/lustre/include/linux/lnet/lnetctl.h
+++ b/drivers/staging/lustre/include/uapi/linux/lnet/lnetctl.h
@@ -15,7 +15,7 @@
 #ifndef _LNETCTL_H_
 #define _LNETCTL_H_
 
-#include "types.h"
+#include <uapi/linux/lnet/lnet-types.h>
 
 /** \addtogroup lnet_fault_simulation
  * @{
@@ -32,10 +32,10 @@ enum {
 	LNET_CTL_DELAY_LIST,
 };
 
-#define LNET_ACK_BIT		BIT(0)
-#define LNET_PUT_BIT		BIT(1)
-#define LNET_GET_BIT		BIT(2)
-#define LNET_REPLY_BIT		BIT(3)
+#define LNET_ACK_BIT		(1 << 0)
+#define LNET_PUT_BIT		(1 << 1)
+#define LNET_GET_BIT		(1 << 2)
+#define LNET_REPLY_BIT		(1 << 3)
 
 /** ioctl parameter for LNet fault simulation */
 struct lnet_fault_attr {
@@ -131,45 +131,4 @@ struct lnet_fault_stat {
 #define SMFS_DEV_MAJOR 10
 #define SMFS_DEV_MINOR 242
 
-int ptl_initialize(int argc, char **argv);
-int jt_ptl_network(int argc, char **argv);
-int jt_ptl_list_nids(int argc, char **argv);
-int jt_ptl_which_nid(int argc, char **argv);
-int jt_ptl_print_interfaces(int argc, char **argv);
-int jt_ptl_add_interface(int argc, char **argv);
-int jt_ptl_del_interface(int argc, char **argv);
-int jt_ptl_print_peers(int argc, char **argv);
-int jt_ptl_add_peer(int argc, char **argv);
-int jt_ptl_del_peer(int argc, char **argv);
-int jt_ptl_print_connections(int argc, char **argv);
-int jt_ptl_disconnect(int argc, char **argv);
-int jt_ptl_push_connection(int argc, char **argv);
-int jt_ptl_print_active_txs(int argc, char **argv);
-int jt_ptl_ping(int argc, char **argv);
-int jt_ptl_mynid(int argc, char **argv);
-int jt_ptl_add_uuid(int argc, char **argv);
-int jt_ptl_add_uuid_old(int argc, char **argv); /* backwards compatibility  */
-int jt_ptl_close_uuid(int argc, char **argv);
-int jt_ptl_del_uuid(int argc, char **argv);
-int jt_ptl_add_route(int argc, char **argv);
-int jt_ptl_del_route(int argc, char **argv);
-int jt_ptl_notify_router(int argc, char **argv);
-int jt_ptl_print_routes(int argc, char **argv);
-int jt_ptl_fail_nid(int argc, char **argv);
-int jt_ptl_lwt(int argc, char **argv);
-int jt_ptl_testprotocompat(int argc, char **argv);
-int jt_ptl_memhog(int argc, char **argv);
-
-int dbg_initialize(int argc, char **argv);
-int jt_dbg_filter(int argc, char **argv);
-int jt_dbg_show(int argc, char **argv);
-int jt_dbg_list(int argc, char **argv);
-int jt_dbg_debug_kernel(int argc, char **argv);
-int jt_dbg_debug_daemon(int argc, char **argv);
-int jt_dbg_debug_file(int argc, char **argv);
-int jt_dbg_clear_debug_buf(int argc, char **argv);
-int jt_dbg_mark_debug_buf(int argc, char **argv);
-int jt_dbg_modules(int argc, char **argv);
-int jt_dbg_panic(int argc, char **argv);
-
 #endif
diff --git a/drivers/staging/lustre/include/linux/lnet/lnetst.h b/drivers/staging/lustre/include/uapi/linux/lnet/lnetst.h
index ea736f8d5231..a4f9ff01d458 100644
--- a/drivers/staging/lustre/include/linux/lnet/lnetst.h
+++ b/drivers/staging/lustre/include/uapi/linux/lnet/lnetst.h
@@ -54,7 +54,8 @@
 #define LSTIO_GROUP_ADD		0xC10	/* add group */
 #define LSTIO_GROUP_LIST	0xC11	/* list all groups in session */
 #define LSTIO_GROUP_INFO	0xC12	/* query default information of
-					 * specified group */
+					 * specified group
+					 */
 #define LSTIO_GROUP_DEL		0xC13	/* delete group */
 #define LSTIO_NODES_ADD		0xC14	/* add nodes to specified group */
 #define LSTIO_GROUP_UPDATE      0xC15	/* update group */
@@ -102,27 +103,32 @@ struct lstcon_test_ent {
 	int	tse_type;       /* test type */
 	int	tse_loop;       /* loop count */
 	int	tse_concur;     /* concurrency of test */
-};				/*** test summary entry, for
-				 *** list_batch command */
+};				/* test summary entry, for
+				 * list_batch command
+				 */
 
 struct lstcon_batch_ent {
 	int	bae_state;	/* batch status */
 	int	bae_timeout;	/* batch timeout */
 	int	bae_ntest;	/* # of tests in the batch */
-};				/*** batch summary entry, for
-				 *** list_batch command */
+};				/* batch summary entry, for
+				 * list_batch command
+				 */
 
 struct lstcon_test_batch_ent {
 	struct lstcon_ndlist_ent   tbe_cli_nle;	/* client (group) node_list
-						 * entry */
+						 * entry
+						 */
 	struct lstcon_ndlist_ent   tbe_srv_nle;	/* server (group) node_list
-						 * entry */
+						 * entry
+						 */
 	union {
 		struct lstcon_test_ent	tbe_test; /* test entry */
 		struct lstcon_batch_ent tbe_batch;/* batch entry */
 	} u;
-};				/*** test/batch verbose information entry,
-				 *** for list_batch command */
+};				/* test/batch verbose information entry,
+				 * for list_batch command
+				 */
 
 struct lstcon_rpc_ent {
 	struct list_head	rpe_link;	/* link chain */
@@ -138,10 +144,10 @@ struct lstcon_rpc_ent {
 };
 
 struct lstcon_trans_stat {
-	int	trs_rpc_stat[4];	/* RPCs stat (0: total
-						      1: failed
-						      2: finished
-						      4: reserved */
+	int	trs_rpc_stat[4];	/* RPCs stat (0: total 1: failed
+					 * 2: finished
+					 * 4: reserved
+					 */
 	int	trs_rpc_errno;		/* RPC errno */
 	int	trs_fwk_stat[8];	/* framework stat */
 	int	trs_fwk_errno;		/* errno of the first remote error */
@@ -275,22 +281,28 @@ struct lstio_session_end_args {
 struct lstio_debug_args {
 	int			 lstio_dbg_key;		/* IN: session key */
 	int			 lstio_dbg_type;	/* IN: debug
-								session|batch|
-								group|nodes
-								list */
+							 * session|batch|
+							 * group|nodes list
+							 */
 	int			 lstio_dbg_flags;	/* IN: reserved debug
-							       flags */
+							 * flags
+							 */
 	int			 lstio_dbg_timeout;	/* IN: timeout of
-							       debug */
+							 * debug
+							 */
 	int			 lstio_dbg_nmlen;	/* IN: len of name */
 	char __user		*lstio_dbg_namep;	/* IN: name of
-							       group|batch */
+							 * group|batch
+							 */
 	int			 lstio_dbg_count;	/* IN: # of test nodes
-							       to debug */
+							 * to debug
+							 */
 	struct lnet_process_id __user *lstio_dbg_idsp;	/* IN: id of test
-							       nodes */
+							 * nodes
+							 */
 	struct list_head __user	*lstio_dbg_resultp;	/* OUT: list head of
-								result buffer */
+							 * result buffer
+							 */
 };
 
 struct lstio_group_add_args {
@@ -307,7 +319,8 @@ struct lstio_group_del_args {
 
 #define LST_GROUP_CLEAN		1	/* remove inactive nodes in the group */
 #define LST_GROUP_REFRESH	2	/* refresh inactive nodes
-					 * in the group */
+					 * in the group
+					 */
 #define LST_GROUP_RMND		3	/* delete nodes from the group */
 
 struct lstio_group_update_args {
@@ -319,7 +332,8 @@ struct lstio_group_update_args {
 	int			 lstio_grp_count;	/* IN: # of nodes id */
 	struct lnet_process_id __user *lstio_grp_idsp;	/* IN: array of nodes */
 	struct list_head __user	*lstio_grp_resultp;	/* OUT: list head of
-								result buffer */
+							 * result buffer
+							 */
 };
 
 struct lstio_group_nodes_args {
@@ -331,7 +345,8 @@ struct lstio_group_nodes_args {
 	unsigned int __user	*lstio_grp_featp;
 	struct lnet_process_id __user *lstio_grp_idsp;	/* IN: nodes */
 	struct list_head __user	*lstio_grp_resultp;	/* OUT: list head of
-								result buffer */
+							 * result buffer
+							 */
 };
 
 struct lstio_group_list_args {
@@ -345,8 +360,9 @@ struct lstio_group_info_args {
 	int			 lstio_grp_key;		/* IN: session key */
 	int			 lstio_grp_nmlen;	/* IN: name len */
 	char __user		*lstio_grp_namep;	/* IN: name */
-	struct lstcon_ndlist_ent __user *lstio_grp_entp;/* OUT: description of
-							   group */
+	struct lstcon_ndlist_ent __user *lstio_grp_entp;/* OUT: description
+							 * of group
+							 */
 	int __user		*lstio_grp_idxp;	/* IN/OUT: node index */
 	int __user		*lstio_grp_ndentp;	/* IN/OUT: # of nodent */
 	struct lstcon_node_ent __user *lstio_grp_dentsp;/* OUT: nodent array */
@@ -369,34 +385,41 @@ struct lstio_batch_del_args {
 struct lstio_batch_run_args {
 	int			 lstio_bat_key;		/* IN: session key */
 	int			 lstio_bat_timeout;	/* IN: timeout for
-							       the batch */
+							 * the batch
+							 */
 	int			 lstio_bat_nmlen;	/* IN: name length */
 	char __user		*lstio_bat_namep;	/* IN: batch name */
 	struct list_head __user	*lstio_bat_resultp;	/* OUT: list head of
-								result buffer */
+							 * result buffer
+							 */
 };
 
 struct lstio_batch_stop_args {
 	int			 lstio_bat_key;		/* IN: session key */
 	int			 lstio_bat_force;	/* IN: abort unfinished
-							       test RPC */
+							 * test RPC
+							 */
 	int			 lstio_bat_nmlen;	/* IN: name length */
 	char __user		*lstio_bat_namep;	/* IN: batch name */
 	struct list_head __user	*lstio_bat_resultp;	/* OUT: list head of
-								result buffer */
+							 * result buffer
+							 */
 };
 
 struct lstio_batch_query_args {
 	int			 lstio_bat_key;		/* IN: session key */
 	int			 lstio_bat_testidx;	/* IN: test index */
 	int			 lstio_bat_client;	/* IN: we testing
-							       client? */
+							 * client?
+							 */
 	int			 lstio_bat_timeout;	/* IN: timeout for
-							       waiting */
+							 * waiting
+							 */
 	int			 lstio_bat_nmlen;	/* IN: name length */
 	char __user		*lstio_bat_namep;	/* IN: batch name */
 	struct list_head __user	*lstio_bat_resultp;	/* OUT: list head of
-								result buffer */
+							 * result buffer
+							 */
 };
 
 struct lstio_batch_list_args {
@@ -411,7 +434,8 @@ struct lstio_batch_info_args {
 	int			 lstio_bat_nmlen;	/* IN: name length */
 	char __user		*lstio_bat_namep;	/* IN: name */
 	int			 lstio_bat_server;	/* IN: query server
-							       or not */
+							 * or not
+							 */
 	int			 lstio_bat_testidx;	/* IN: test index */
 	struct lstcon_test_batch_ent __user *lstio_bat_entp;/* OUT: batch ent */
 
@@ -424,14 +448,17 @@ struct lstio_batch_info_args {
 struct lstio_stat_args {
 	int			 lstio_sta_key;		/* IN: session key */
 	int			 lstio_sta_timeout;	/* IN: timeout for
-							       stat request */
+							 * stat request
+							 */
 	int			 lstio_sta_nmlen;	/* IN: group name
-							       length */
+							 * length
+							 */
 	char __user		*lstio_sta_namep;	/* IN: group name */
 	int			 lstio_sta_count;	/* IN: # of pid */
 	struct lnet_process_id __user *lstio_sta_idsp;	/* IN: pid */
 	struct list_head __user	*lstio_sta_resultp;	/* OUT: list head of
-								result buffer */
+							 * result buffer
+							 */
 };
 
 enum lst_test_type {
@@ -452,26 +479,32 @@ struct lstio_test_args {
 	int		  lstio_tes_concur;	/* IN: concurrency */
 
 	int		  lstio_tes_dist;	/* IN: node distribution in
-						       destination groups */
+						 * destination groups
+						 */
 	int		  lstio_tes_span;	/* IN: node span in
-						       destination groups */
+						 * destination groups
+						 */
 	int		  lstio_tes_sgrp_nmlen;	/* IN: source group
-						       name length */
+						 * name length
+						 */
 	char __user	 *lstio_tes_sgrp_name;	/* IN: group name */
 	int		  lstio_tes_dgrp_nmlen;	/* IN: destination group
-						       name length */
+						 * name length
+						 */
 	char __user	 *lstio_tes_dgrp_name;	/* IN: group name */
 
 	int		  lstio_tes_param_len;	/* IN: param buffer len */
 	void __user	 *lstio_tes_param;	/* IN: parameter for specified
-						       test:
-						       lstio_bulk_param_t,
-						       lstio_ping_param_t,
-						       ... more */
+						 * test: lstio_bulk_param_t,
+						 * lstio_ping_param_t,
+						 * ... more
+						 */
 	int __user	 *lstio_tes_retp;	/* OUT: private returned
-							value */
+						 * value
+						 */
 	struct list_head __user *lstio_tes_resultp;/* OUT: list head of
-							result buffer */
+						    * result buffer
+						    */
 };
 
 enum lst_brw_type {
diff --git a/drivers/staging/lustre/include/linux/lnet/nidstr.h b/drivers/staging/lustre/include/uapi/linux/lnet/nidstr.h
index ecdd0db04d0a..882074ed6021 100644
--- a/drivers/staging/lustre/include/linux/lnet/nidstr.h
+++ b/drivers/staging/lustre/include/uapi/linux/lnet/nidstr.h
@@ -28,7 +28,7 @@
 #ifndef _LNET_NIDSTRINGS_H
 #define _LNET_NIDSTRINGS_H
 
-#include "types.h"
+#include <uapi/linux/lnet/lnet-types.h>
 
 /**
  *  Lustre Network Driver types.
diff --git a/drivers/staging/lustre/include/linux/lnet/lnet.h b/drivers/staging/lustre/include/uapi/linux/lnet/socklnd.h
index 5d1559a26638..6453e053fa99 100644
--- a/drivers/staging/lustre/include/linux/lnet/lnet.h
+++ b/drivers/staging/lustre/include/uapi/linux/lnet/socklnd.h
@@ -22,23 +22,23 @@
 /*
  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  * Use is subject to license terms.
- *
- * Copyright (c) 2012 - 2015, Intel Corporation.
  */
 /*
  * This file is part of Lustre, http://www.lustre.org/
- * Lustre is a trademark of Seagate, Inc.
+ * Lustre is a trademark of Sun Microsystems, Inc.
+ *
+ * #defines shared between socknal implementation and utilities
  */
+#ifndef __UAPI_LNET_SOCKLND_H__
+#define __UAPI_LNET_SOCKLND_H__
 
-#ifndef __LNET_H__
-#define __LNET_H__
+#define SOCKLND_CONN_NONE     (-1)
+#define SOCKLND_CONN_ANY	0
+#define SOCKLND_CONN_CONTROL	1
+#define SOCKLND_CONN_BULK_IN	2
+#define SOCKLND_CONN_BULK_OUT	3
+#define SOCKLND_CONN_NTYPES	4
 
-/*
- * lnet.h
- *
- * User application interface file
- */
-#include "types.h"
-#include "nidstr.h"
+#define SOCKLND_CONN_ACK	SOCKLND_CONN_BULK_IN
 
 #endif
diff --git a/drivers/staging/lustre/lustre/include/lustre_cfg.h b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_cfg.h
index 8eb394e64b25..11b51d93f64c 100644
--- a/drivers/staging/lustre/lustre/include/lustre_cfg.h
+++ b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_cfg.h
@@ -30,8 +30,12 @@
  * Lustre is a trademark of Sun Microsystems, Inc.
  */
 
-#ifndef _LUSTRE_CFG_H
-#define _LUSTRE_CFG_H
+#ifndef _UAPI_LUSTRE_CFG_H_
+#define _UAPI_LUSTRE_CFG_H_
+
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <uapi/linux/lustre/lustre_user.h>
 
 /** \defgroup cfg cfg
  *
@@ -46,55 +50,68 @@
 #define LUSTRE_CFG_MAX_BUFCOUNT 8
 
 #define LCFG_HDR_SIZE(count) \
-	cfs_size_round(offsetof(struct lustre_cfg, lcfg_buflens[(count)]))
+	__ALIGN_KERNEL(offsetof(struct lustre_cfg, lcfg_buflens[(count)]), 8)
 
 /** If the LCFG_REQUIRED bit is set in a configuration command,
  * then the client is required to understand this parameter
  * in order to mount the filesystem. If it does not understand
  * a REQUIRED command the client mount will fail.
  */
-#define LCFG_REQUIRED	 0x0001000
+#define LCFG_REQUIRED	0x0001000
 
 enum lcfg_command_type {
-	LCFG_ATTACH	     = 0x00cf001, /**< create a new obd instance */
-	LCFG_DETACH	     = 0x00cf002, /**< destroy obd instance */
-	LCFG_SETUP	      = 0x00cf003, /**< call type-specific setup */
-	LCFG_CLEANUP	    = 0x00cf004, /**< call type-specific cleanup */
-	LCFG_ADD_UUID	   = 0x00cf005, /**< add a nid to a niduuid */
-	LCFG_DEL_UUID	   = 0x00cf006, /**< remove a nid from a niduuid */
-	LCFG_MOUNTOPT	   = 0x00cf007, /**< create a profile (mdc, osc) */
-	LCFG_DEL_MOUNTOPT       = 0x00cf008, /**< destroy a profile */
-	LCFG_SET_TIMEOUT	= 0x00cf009, /**< set obd_timeout */
-	LCFG_SET_UPCALL	 = 0x00cf00a, /**< deprecated */
-	LCFG_ADD_CONN	   = 0x00cf00b, /**< add a failover niduuid to an obd */
-	LCFG_DEL_CONN	   = 0x00cf00c, /**< remove a failover niduuid */
-	LCFG_LOV_ADD_OBD	= 0x00cf00d, /**< add an osc to a lov */
-	LCFG_LOV_DEL_OBD	= 0x00cf00e, /**< remove an osc from a lov */
-	LCFG_PARAM	      = 0x00cf00f, /**< set a proc parameter */
-	LCFG_MARKER	     = 0x00cf010, /**< metadata about next cfg rec */
-	LCFG_LOG_START	  = 0x00ce011, /**< mgc only, process a cfg log */
-	LCFG_LOG_END	    = 0x00ce012, /**< stop processing updates */
-	LCFG_LOV_ADD_INA	= 0x00ce013, /**< like LOV_ADD_OBD, inactive */
-	LCFG_ADD_MDC	    = 0x00cf014, /**< add an mdc to a lmv */
-	LCFG_DEL_MDC	    = 0x00cf015, /**< remove an mdc from a lmv */
-	LCFG_SPTLRPC_CONF       = 0x00ce016, /**< security */
-	LCFG_POOL_NEW	   = 0x00ce020, /**< create an ost pool name */
-	LCFG_POOL_ADD	   = 0x00ce021, /**< add an ost to a pool */
-	LCFG_POOL_REM	   = 0x00ce022, /**< remove an ost from a pool */
-	LCFG_POOL_DEL	   = 0x00ce023, /**< destroy an ost pool name */
-	LCFG_SET_LDLM_TIMEOUT   = 0x00ce030, /**< set ldlm_timeout */
-	LCFG_PRE_CLEANUP	= 0x00cf031, /**< call type-specific pre
-					      * cleanup cleanup
-					      */
-	LCFG_SET_PARAM		= 0x00ce032, /**< use set_param syntax to set
-					      * a proc parameters
-					      */
+	LCFG_ATTACH		  = 0x00cf001, /**< create a new obd instance */
+	LCFG_DETACH		  = 0x00cf002, /**< destroy obd instance */
+	LCFG_SETUP		  = 0x00cf003, /**< call type-specific setup */
+	LCFG_CLEANUP		  = 0x00cf004, /**< call type-specific cleanup
+						 */
+	LCFG_ADD_UUID		  = 0x00cf005, /**< add a nid to a niduuid */
+	LCFG_DEL_UUID		  = 0x00cf006, /**< remove a nid from
+						 *  a niduuid
+						 */
+	LCFG_MOUNTOPT		  = 0x00cf007, /**< create a profile
+						 * (mdc, osc)
+						 */
+	LCFG_DEL_MOUNTOPT	  = 0x00cf008, /**< destroy a profile */
+	LCFG_SET_TIMEOUT	  = 0x00cf009, /**< set obd_timeout */
+	LCFG_SET_UPCALL		  = 0x00cf00a, /**< deprecated */
+	LCFG_ADD_CONN		  = 0x00cf00b, /**< add a failover niduuid to
+						 *  an obd
+						 */
+	LCFG_DEL_CONN		  = 0x00cf00c, /**< remove a failover niduuid */
+	LCFG_LOV_ADD_OBD	  = 0x00cf00d, /**< add an osc to a lov */
+	LCFG_LOV_DEL_OBD	  = 0x00cf00e, /**< remove an osc from a lov */
+	LCFG_PARAM		  = 0x00cf00f, /**< set a proc parameter */
+	LCFG_MARKER		  = 0x00cf010, /**< metadata about next
+						 *  cfg rec
+						 */
+	LCFG_LOG_START		  = 0x00ce011, /**< mgc only, process a
+						 *  cfg log
+						 */
+	LCFG_LOG_END		  = 0x00ce012, /**< stop processing updates */
+	LCFG_LOV_ADD_INA	  = 0x00ce013, /**< like LOV_ADD_OBD,
+						 *  inactive
+						 */
+	LCFG_ADD_MDC		  = 0x00cf014, /**< add an mdc to a lmv */
+	LCFG_DEL_MDC		  = 0x00cf015, /**< remove an mdc from a lmv */
+	LCFG_SPTLRPC_CONF	  = 0x00ce016, /**< security */
+	LCFG_POOL_NEW		  = 0x00ce020, /**< create an ost pool name */
+	LCFG_POOL_ADD		  = 0x00ce021, /**< add an ost to a pool */
+	LCFG_POOL_REM		  = 0x00ce022, /**< remove an ost from a pool */
+	LCFG_POOL_DEL		  = 0x00ce023, /**< destroy an ost pool name */
+	LCFG_SET_LDLM_TIMEOUT	  = 0x00ce030, /**< set ldlm_timeout */
+	LCFG_PRE_CLEANUP	  = 0x00cf031, /**< call type-specific pre
+						 * cleanup cleanup
+						 */
+	LCFG_SET_PARAM		  = 0x00ce032, /**< use set_param syntax to set
+						 * a proc parameters
+						 */
 };
 
 struct lustre_cfg_bufs {
-	void    *lcfg_buf[LUSTRE_CFG_MAX_BUFCOUNT];
-	__u32    lcfg_buflen[LUSTRE_CFG_MAX_BUFCOUNT];
-	__u32    lcfg_bufcount;
+	void  *lcfg_buf[LUSTRE_CFG_MAX_BUFCOUNT];
+	__u32 lcfg_buflen[LUSTRE_CFG_MAX_BUFCOUNT];
+	__u32 lcfg_bufcount;
 };
 
 struct lustre_cfg {
@@ -111,40 +128,37 @@ struct lustre_cfg {
 };
 
 enum cfg_record_type {
-	PORTALS_CFG_TYPE = 1,
-	LUSTRE_CFG_TYPE = 123,
+	PORTALS_CFG_TYPE	= 1,
+	LUSTRE_CFG_TYPE		= 123,
 };
 
-#define LUSTRE_CFG_BUFLEN(lcfg, idx)	    \
-	((lcfg)->lcfg_bufcount <= (idx)	 \
-	 ? 0				    \
-	 : (lcfg)->lcfg_buflens[(idx)])
+#define LUSTRE_CFG_BUFLEN(lcfg, idx)					\
+	((lcfg)->lcfg_bufcount <= (idx) ? 0 : (lcfg)->lcfg_buflens[(idx)])
 
 static inline void lustre_cfg_bufs_set(struct lustre_cfg_bufs *bufs,
-				       __u32		   index,
-				       void		   *buf,
-				       __u32		   buflen)
+				       __u32 index, void *buf, __u32 buflen)
 {
 	if (index >= LUSTRE_CFG_MAX_BUFCOUNT)
 		return;
+
 	if (!bufs)
 		return;
 
 	if (bufs->lcfg_bufcount <= index)
 		bufs->lcfg_bufcount = index + 1;
 
-	bufs->lcfg_buf[index]    = buf;
+	bufs->lcfg_buf[index] = buf;
 	bufs->lcfg_buflen[index] = buflen;
 }
 
 static inline void lustre_cfg_bufs_set_string(struct lustre_cfg_bufs *bufs,
-					      __u32 index,
-					      char *str)
+					      __u32 index, char *str)
 {
 	lustre_cfg_bufs_set(bufs, index, str, str ? strlen(str) + 1 : 0);
 }
 
-static inline void lustre_cfg_bufs_reset(struct lustre_cfg_bufs *bufs, char *name)
+static inline void lustre_cfg_bufs_reset(struct lustre_cfg_bufs *bufs,
+					 char *name)
 {
 	memset((bufs), 0, sizeof(*bufs));
 	if (name)
@@ -157,13 +171,16 @@ static inline void *lustre_cfg_buf(struct lustre_cfg *lcfg, __u32 index)
 	size_t offset;
 	__u32 bufcount;
 
+	if (!lcfg)
+		return NULL;
+
 	bufcount = lcfg->lcfg_bufcount;
 	if (index >= bufcount)
 		return NULL;
 
 	offset = LCFG_HDR_SIZE(lcfg->lcfg_bufcount);
 	for (i = 0; i < index; i++)
-		offset += cfs_size_round(lcfg->lcfg_buflens[i]);
+		offset += __ALIGN_KERNEL(lcfg->lcfg_buflens[i], 8);
 	return (char *)lcfg + offset;
 }
 
@@ -179,35 +196,6 @@ static inline void lustre_cfg_bufs_init(struct lustre_cfg_bufs *bufs,
 	}
 }
 
-static inline char *lustre_cfg_string(struct lustre_cfg *lcfg, __u32 index)
-{
-	char *s;
-
-	if (lcfg->lcfg_buflens[index] == 0)
-		return NULL;
-
-	s = lustre_cfg_buf(lcfg, index);
-	if (!s)
-		return NULL;
-
-	/*
-	 * make sure it's NULL terminated, even if this kills a char
-	 * of data.  Try to use the padding first though.
-	 */
-	if (s[lcfg->lcfg_buflens[index] - 1] != '\0') {
-		size_t last = min((size_t)lcfg->lcfg_buflens[index],
-				  cfs_size_round(lcfg->lcfg_buflens[index]) - 1);
-		char lost = s[last];
-
-		s[last] = '\0';
-		if (lost != '\0') {
-			CWARN("Truncated buf %d to '%s' (lost '%c'...)\n",
-			      index, s, lost);
-		}
-	}
-	return s;
-}
-
 static inline __u32 lustre_cfg_len(__u32 bufcount, __u32 *buflens)
 {
 	__u32 i;
@@ -215,24 +203,16 @@ static inline __u32 lustre_cfg_len(__u32 bufcount, __u32 *buflens)
 
 	len = LCFG_HDR_SIZE(bufcount);
 	for (i = 0; i < bufcount; i++)
-		len += cfs_size_round(buflens[i]);
+		len += __ALIGN_KERNEL(buflens[i], 8);
 
-	return cfs_size_round(len);
+	return __ALIGN_KERNEL(len, 8);
 }
 
-#include "obd_support.h"
-
-static inline struct lustre_cfg *lustre_cfg_new(int cmd,
-						struct lustre_cfg_bufs *bufs)
+static inline void lustre_cfg_init(struct lustre_cfg *lcfg, int cmd,
+				   struct lustre_cfg_bufs *bufs)
 {
-	struct lustre_cfg *lcfg;
 	char *ptr;
-	int i;
-
-	lcfg = kzalloc(lustre_cfg_len(bufs->lcfg_bufcount, bufs->lcfg_buflen),
-		       GFP_NOFS);
-	if (!lcfg)
-		return ERR_PTR(-ENOMEM);
+	__u32 i;
 
 	lcfg->lcfg_version = LUSTRE_CFG_VERSION;
 	lcfg->lcfg_command = cmd;
@@ -241,15 +221,11 @@ static inline struct lustre_cfg *lustre_cfg_new(int cmd,
 	ptr = (char *)lcfg + LCFG_HDR_SIZE(lcfg->lcfg_bufcount);
 	for (i = 0; i < lcfg->lcfg_bufcount; i++) {
 		lcfg->lcfg_buflens[i] = bufs->lcfg_buflen[i];
-		LOGL((char *)bufs->lcfg_buf[i], bufs->lcfg_buflen[i], ptr);
+		if (bufs->lcfg_buf[i]) {
+			memcpy(ptr, bufs->lcfg_buf[i], bufs->lcfg_buflen[i]);
+			ptr += __ALIGN_KERNEL(bufs->lcfg_buflen[i], 8);
+		}
 	}
-	return lcfg;
-}
-
-static inline void lustre_cfg_free(struct lustre_cfg *lcfg)
-{
-	kfree(lcfg);
-	return;
 }
 
 static inline int lustre_cfg_sanity_check(void *buf, size_t len)
@@ -280,8 +256,6 @@ static inline int lustre_cfg_sanity_check(void *buf, size_t len)
 	return 0;
 }
 
-#include "lustre/lustre_user.h"
-
 /** @} cfg */
 
-#endif /* _LUSTRE_CFG_H */
+#endif /* _UAPI_LUSTRE_CFG_H_ */
diff --git a/drivers/staging/lustre/include/uapi/linux/lustre/lustre_fid.h b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_fid.h
new file mode 100644
index 000000000000..2e7a8d103777
--- /dev/null
+++ b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_fid.h
@@ -0,0 +1,293 @@
+/*
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see
+ * http://www.gnu.org/licenses/gpl-2.0.html
+ *
+ * GPL HEADER END
+ */
+/*
+ * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Use is subject to license terms.
+ *
+ * Copyright (c) 2011, 2014, Intel Corporation.
+ *
+ * Copyright 2016 Cray Inc, all rights reserved.
+ * Author: Ben Evans.
+ *
+ * all fid manipulation functions go here
+ *
+ * FIDS are globally unique within a Lustre filessytem, and are made up
+ * of three parts: sequence, Object ID, and version.
+ *
+ */
+#ifndef _UAPI_LUSTRE_FID_H_
+#define _UAPI_LUSTRE_FID_H_
+
+#include <uapi/linux/lustre/lustre_idl.h>
+
+/** returns fid object sequence */
+static inline __u64 fid_seq(const struct lu_fid *fid)
+{
+	return fid->f_seq;
+}
+
+/** returns fid object id */
+static inline __u32 fid_oid(const struct lu_fid *fid)
+{
+	return fid->f_oid;
+}
+
+/** returns fid object version */
+static inline __u32 fid_ver(const struct lu_fid *fid)
+{
+	return fid->f_ver;
+}
+
+static inline void fid_zero(struct lu_fid *fid)
+{
+	memset(fid, 0, sizeof(*fid));
+}
+
+static inline __u64 fid_ver_oid(const struct lu_fid *fid)
+{
+	return (__u64)fid_ver(fid) << 32 | fid_oid(fid);
+}
+
+static inline bool fid_seq_is_mdt0(__u64 seq)
+{
+	return seq == FID_SEQ_OST_MDT0;
+}
+
+static inline bool fid_seq_is_mdt(__u64 seq)
+{
+	return seq == FID_SEQ_OST_MDT0 || seq >= FID_SEQ_NORMAL;
+};
+
+static inline bool fid_seq_is_echo(__u64 seq)
+{
+	return seq == FID_SEQ_ECHO;
+}
+
+static inline bool fid_is_echo(const struct lu_fid *fid)
+{
+	return fid_seq_is_echo(fid_seq(fid));
+}
+
+static inline bool fid_seq_is_llog(__u64 seq)
+{
+	return seq == FID_SEQ_LLOG;
+}
+
+static inline bool fid_is_llog(const struct lu_fid *fid)
+{
+	/* file with OID == 0 is not llog but contains last oid */
+	return fid_seq_is_llog(fid_seq(fid)) && fid_oid(fid) > 0;
+}
+
+static inline bool fid_seq_is_rsvd(__u64 seq)
+{
+	return seq > FID_SEQ_OST_MDT0 && seq <= FID_SEQ_RSVD;
+};
+
+static inline bool fid_seq_is_special(__u64 seq)
+{
+	return seq == FID_SEQ_SPECIAL;
+};
+
+static inline bool fid_seq_is_local_file(__u64 seq)
+{
+	return seq == FID_SEQ_LOCAL_FILE ||
+	       seq == FID_SEQ_LOCAL_NAME;
+};
+
+static inline bool fid_seq_is_root(__u64 seq)
+{
+	return seq == FID_SEQ_ROOT;
+}
+
+static inline bool fid_seq_is_dot(__u64 seq)
+{
+	return seq == FID_SEQ_DOT_LUSTRE;
+}
+
+static inline bool fid_seq_is_default(__u64 seq)
+{
+	return seq == FID_SEQ_LOV_DEFAULT;
+}
+
+static inline bool fid_is_mdt0(const struct lu_fid *fid)
+{
+	return fid_seq_is_mdt0(fid_seq(fid));
+}
+
+/**
+ * Check if a fid is igif or not.
+ * \param fid the fid to be tested.
+ * \return true if the fid is an igif; otherwise false.
+ */
+static inline bool fid_seq_is_igif(__u64 seq)
+{
+	return seq >= FID_SEQ_IGIF && seq <= FID_SEQ_IGIF_MAX;
+}
+
+static inline bool fid_is_igif(const struct lu_fid *fid)
+{
+	return fid_seq_is_igif(fid_seq(fid));
+}
+
+/**
+ * Check if a fid is idif or not.
+ * \param fid the fid to be tested.
+ * \return true if the fid is an idif; otherwise false.
+ */
+static inline bool fid_seq_is_idif(__u64 seq)
+{
+	return seq >= FID_SEQ_IDIF && seq <= FID_SEQ_IDIF_MAX;
+}
+
+static inline bool fid_is_idif(const struct lu_fid *fid)
+{
+	return fid_seq_is_idif(fid_seq(fid));
+}
+
+static inline bool fid_is_local_file(const struct lu_fid *fid)
+{
+	return fid_seq_is_local_file(fid_seq(fid));
+}
+
+static inline bool fid_seq_is_norm(__u64 seq)
+{
+	return (seq >= FID_SEQ_NORMAL);
+}
+
+static inline bool fid_is_norm(const struct lu_fid *fid)
+{
+	return fid_seq_is_norm(fid_seq(fid));
+}
+
+/* convert an OST objid into an IDIF FID SEQ number */
+static inline __u64 fid_idif_seq(__u64 id, __u32 ost_idx)
+{
+	return FID_SEQ_IDIF | (ost_idx << 16) | ((id >> 32) & 0xffff);
+}
+
+/* convert a packed IDIF FID into an OST objid */
+static inline __u64 fid_idif_id(__u64 seq, __u32 oid, __u32 ver)
+{
+	return ((__u64)ver << 48) | ((seq & 0xffff) << 32) | oid;
+}
+
+static inline __u32 idif_ost_idx(__u64 seq)
+{
+	return (seq >> 16) & 0xffff;
+}
+
+/* extract ost index from IDIF FID */
+static inline __u32 fid_idif_ost_idx(const struct lu_fid *fid)
+{
+	return idif_ost_idx(fid_seq(fid));
+}
+
+/**
+ * Get inode number from an igif.
+ * \param fid an igif to get inode number from.
+ * \return inode number for the igif.
+ */
+static inline ino_t lu_igif_ino(const struct lu_fid *fid)
+{
+	return fid_seq(fid);
+}
+
+/**
+ * Get inode generation from an igif.
+ * \param fid an igif to get inode generation from.
+ * \return inode generation for the igif.
+ */
+static inline __u32 lu_igif_gen(const struct lu_fid *fid)
+{
+	return fid_oid(fid);
+}
+
+/**
+ * Build igif from the inode number/generation.
+ */
+static inline void lu_igif_build(struct lu_fid *fid, __u32 ino, __u32 gen)
+{
+	fid->f_seq = ino;
+	fid->f_oid = gen;
+	fid->f_ver = 0;
+}
+
+/*
+ * Fids are transmitted across network (in the sender byte-ordering),
+ * and stored on disk in big-endian order.
+ */
+static inline void fid_cpu_to_le(struct lu_fid *dst, const struct lu_fid *src)
+{
+	dst->f_seq = __cpu_to_le64(fid_seq(src));
+	dst->f_oid = __cpu_to_le32(fid_oid(src));
+	dst->f_ver = __cpu_to_le32(fid_ver(src));
+}
+
+static inline void fid_le_to_cpu(struct lu_fid *dst, const struct lu_fid *src)
+{
+	dst->f_seq = __le64_to_cpu(fid_seq(src));
+	dst->f_oid = __le32_to_cpu(fid_oid(src));
+	dst->f_ver = __le32_to_cpu(fid_ver(src));
+}
+
+static inline void fid_cpu_to_be(struct lu_fid *dst, const struct lu_fid *src)
+{
+	dst->f_seq = __cpu_to_be64(fid_seq(src));
+	dst->f_oid = __cpu_to_be32(fid_oid(src));
+	dst->f_ver = __cpu_to_be32(fid_ver(src));
+}
+
+static inline void fid_be_to_cpu(struct lu_fid *dst, const struct lu_fid *src)
+{
+	dst->f_seq = __be64_to_cpu(fid_seq(src));
+	dst->f_oid = __be32_to_cpu(fid_oid(src));
+	dst->f_ver = __be32_to_cpu(fid_ver(src));
+}
+
+static inline bool fid_is_sane(const struct lu_fid *fid)
+{
+	return fid && ((fid_seq(fid) >= FID_SEQ_START && !fid_ver(fid)) ||
+			fid_is_igif(fid) || fid_is_idif(fid) ||
+			fid_seq_is_rsvd(fid_seq(fid)));
+}
+
+static inline bool lu_fid_eq(const struct lu_fid *f0, const struct lu_fid *f1)
+{
+	return !memcmp(f0, f1, sizeof(*f0));
+}
+
+static inline int lu_fid_cmp(const struct lu_fid *f0,
+			     const struct lu_fid *f1)
+{
+	if (fid_seq(f0) != fid_seq(f1))
+		return fid_seq(f0) > fid_seq(f1) ? 1 : -1;
+
+	if (fid_oid(f0) != fid_oid(f1))
+		return fid_oid(f0) > fid_oid(f1) ? 1 : -1;
+
+	if (fid_ver(f0) != fid_ver(f1))
+		return fid_ver(f0) > fid_ver(f1) ? 1 : -1;
+
+	return 0;
+}
+#endif
diff --git a/drivers/staging/lustre/lustre/include/lustre/ll_fiemap.h b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_fiemap.h
index b8ad5559a3b9..f5214dc33c60 100644
--- a/drivers/staging/lustre/lustre/include/lustre/ll_fiemap.h
+++ b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_fiemap.h
@@ -29,8 +29,6 @@
  * This file is part of Lustre, http://www.lustre.org/
  * Lustre is a trademark of Sun Microsystems, Inc.
  *
- * lustre/include/lustre/ll_fiemap.h
- *
  * FIEMAP data structures and flags. This header file will be used until
  * fiemap.h is available in the upstream kernel.
  *
@@ -41,10 +39,8 @@
 #ifndef _LUSTRE_FIEMAP_H
 #define _LUSTRE_FIEMAP_H
 
-#ifndef __KERNEL__
 #include <stddef.h>
-#include <fiemap.h>
-#endif
+#include <linux/fiemap.h>
 
 /* XXX: We use fiemap_extent::fe_reserved[0] */
 #define fe_device	fe_reserved[0]
diff --git a/drivers/staging/lustre/lustre/include/lustre/lustre_idl.h b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_idl.h
index 77995fa47691..aac98dbcf6e3 100644
--- a/drivers/staging/lustre/lustre/include/lustre/lustre_idl.h
+++ b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_idl.h
@@ -29,8 +29,6 @@
  * This file is part of Lustre, http://www.lustre.org/
  * Lustre is a trademark of Sun Microsystems, Inc.
  *
- * lustre/include/lustre/lustre_idl.h
- *
  * Lustre wire protocol definitions.
  */
 
@@ -69,13 +67,13 @@
 #ifndef _LUSTRE_IDL_H_
 #define _LUSTRE_IDL_H_
 
-#include "../../../include/linux/libcfs/libcfs.h"
-#include "../../../include/linux/lnet/types.h"
+#include <asm/byteorder.h>
+#include <linux/types.h>
 
+#include <uapi/linux/lnet/lnet-types.h>
 /* Defn's shared with user-space. */
-#include "lustre_user.h"
-#include "lustre_errno.h"
-#include "../lustre_ver.h"
+#include <uapi/linux/lustre/lustre_user.h>
+#include <uapi/linux/lustre/lustre_ver.h>
 
 /*
  *  GENERAL STUFF
@@ -217,34 +215,6 @@ enum {
 	LUSTRE_FID_INIT_OID  = 1UL
 };
 
-/** returns fid object sequence */
-static inline __u64 fid_seq(const struct lu_fid *fid)
-{
-	return fid->f_seq;
-}
-
-/** returns fid object id */
-static inline __u32 fid_oid(const struct lu_fid *fid)
-{
-	return fid->f_oid;
-}
-
-/** returns fid object version */
-static inline __u32 fid_ver(const struct lu_fid *fid)
-{
-	return fid->f_ver;
-}
-
-static inline void fid_zero(struct lu_fid *fid)
-{
-	memset(fid, 0, sizeof(*fid));
-}
-
-static inline __u64 fid_ver_oid(const struct lu_fid *fid)
-{
-	return ((__u64)fid_ver(fid) << 32 | fid_oid(fid));
-}
-
 /* copytool uses a 32b bitmask field to encode archive-Ids during register
  * with MDT thru kuc.
  * archive num = 0 => all
@@ -313,458 +283,12 @@ enum dot_lustre_oid {
 	FID_OID_DOT_LUSTRE_OBF = 2UL,
 };
 
-static inline bool fid_seq_is_mdt0(__u64 seq)
-{
-	return (seq == FID_SEQ_OST_MDT0);
-}
-
-static inline bool fid_seq_is_mdt(__u64 seq)
-{
-	return seq == FID_SEQ_OST_MDT0 || seq >= FID_SEQ_NORMAL;
+/** OID for FID_SEQ_ROOT */
+enum root_oid {
+	FID_OID_ROOT		= 1UL,
+	FID_OID_ECHO_ROOT	= 2UL,
 };
 
-static inline bool fid_seq_is_echo(__u64 seq)
-{
-	return (seq == FID_SEQ_ECHO);
-}
-
-static inline bool fid_is_echo(const struct lu_fid *fid)
-{
-	return fid_seq_is_echo(fid_seq(fid));
-}
-
-static inline bool fid_seq_is_llog(__u64 seq)
-{
-	return (seq == FID_SEQ_LLOG);
-}
-
-static inline bool fid_is_llog(const struct lu_fid *fid)
-{
-	/* file with OID == 0 is not llog but contains last oid */
-	return fid_seq_is_llog(fid_seq(fid)) && fid_oid(fid) > 0;
-}
-
-static inline bool fid_seq_is_rsvd(__u64 seq)
-{
-	return (seq > FID_SEQ_OST_MDT0 && seq <= FID_SEQ_RSVD);
-};
-
-static inline bool fid_seq_is_special(__u64 seq)
-{
-	return seq == FID_SEQ_SPECIAL;
-};
-
-static inline bool fid_seq_is_local_file(__u64 seq)
-{
-	return seq == FID_SEQ_LOCAL_FILE ||
-	       seq == FID_SEQ_LOCAL_NAME;
-};
-
-static inline bool fid_seq_is_root(__u64 seq)
-{
-	return seq == FID_SEQ_ROOT;
-}
-
-static inline bool fid_seq_is_dot(__u64 seq)
-{
-	return seq == FID_SEQ_DOT_LUSTRE;
-}
-
-static inline bool fid_seq_is_default(__u64 seq)
-{
-	return seq == FID_SEQ_LOV_DEFAULT;
-}
-
-static inline bool fid_is_mdt0(const struct lu_fid *fid)
-{
-	return fid_seq_is_mdt0(fid_seq(fid));
-}
-
-static inline void lu_root_fid(struct lu_fid *fid)
-{
-	fid->f_seq = FID_SEQ_ROOT;
-	fid->f_oid = 1;
-	fid->f_ver = 0;
-}
-
-/**
- * Check if a fid is igif or not.
- * \param fid the fid to be tested.
- * \return true if the fid is a igif; otherwise false.
- */
-static inline bool fid_seq_is_igif(__u64 seq)
-{
-	return seq >= FID_SEQ_IGIF && seq <= FID_SEQ_IGIF_MAX;
-}
-
-static inline bool fid_is_igif(const struct lu_fid *fid)
-{
-	return fid_seq_is_igif(fid_seq(fid));
-}
-
-/**
- * Check if a fid is idif or not.
- * \param fid the fid to be tested.
- * \return true if the fid is a idif; otherwise false.
- */
-static inline bool fid_seq_is_idif(__u64 seq)
-{
-	return seq >= FID_SEQ_IDIF && seq <= FID_SEQ_IDIF_MAX;
-}
-
-static inline bool fid_is_idif(const struct lu_fid *fid)
-{
-	return fid_seq_is_idif(fid_seq(fid));
-}
-
-static inline bool fid_is_local_file(const struct lu_fid *fid)
-{
-	return fid_seq_is_local_file(fid_seq(fid));
-}
-
-static inline bool fid_seq_is_norm(__u64 seq)
-{
-	return (seq >= FID_SEQ_NORMAL);
-}
-
-static inline bool fid_is_norm(const struct lu_fid *fid)
-{
-	return fid_seq_is_norm(fid_seq(fid));
-}
-
-/* convert an OST objid into an IDIF FID SEQ number */
-static inline __u64 fid_idif_seq(__u64 id, __u32 ost_idx)
-{
-	return FID_SEQ_IDIF | (ost_idx << 16) | ((id >> 32) & 0xffff);
-}
-
-/* convert a packed IDIF FID into an OST objid */
-static inline __u64 fid_idif_id(__u64 seq, __u32 oid, __u32 ver)
-{
-	return ((__u64)ver << 48) | ((seq & 0xffff) << 32) | oid;
-}
-
-/* extract ost index from IDIF FID */
-static inline __u32 fid_idif_ost_idx(const struct lu_fid *fid)
-{
-	return (fid_seq(fid) >> 16) & 0xffff;
-}
-
-/* extract OST sequence (group) from a wire ost_id (id/seq) pair */
-static inline __u64 ostid_seq(const struct ost_id *ostid)
-{
-	if (fid_seq_is_mdt0(ostid->oi.oi_seq))
-		return FID_SEQ_OST_MDT0;
-
-	if (unlikely(fid_seq_is_default(ostid->oi.oi_seq)))
-		return FID_SEQ_LOV_DEFAULT;
-
-	if (fid_is_idif(&ostid->oi_fid))
-		return FID_SEQ_OST_MDT0;
-
-	return fid_seq(&ostid->oi_fid);
-}
-
-/* extract OST objid from a wire ost_id (id/seq) pair */
-static inline __u64 ostid_id(const struct ost_id *ostid)
-{
-	if (fid_seq_is_mdt0(ostid->oi.oi_seq))
-		return ostid->oi.oi_id & IDIF_OID_MASK;
-
-	if (unlikely(fid_seq_is_default(ostid->oi.oi_seq)))
-		return ostid->oi.oi_id;
-
-	if (fid_is_idif(&ostid->oi_fid))
-		return fid_idif_id(fid_seq(&ostid->oi_fid),
-				   fid_oid(&ostid->oi_fid), 0);
-
-	return fid_oid(&ostid->oi_fid);
-}
-
-static inline void ostid_set_seq(struct ost_id *oi, __u64 seq)
-{
-	if (fid_seq_is_mdt0(seq) || fid_seq_is_default(seq)) {
-		oi->oi.oi_seq = seq;
-	} else {
-		oi->oi_fid.f_seq = seq;
-		/* Note: if f_oid + f_ver is zero, we need init it
-		 * to be 1, otherwise, ostid_seq will treat this
-		 * as old ostid (oi_seq == 0)
-		 */
-		if (oi->oi_fid.f_oid == 0 && oi->oi_fid.f_ver == 0)
-			oi->oi_fid.f_oid = LUSTRE_FID_INIT_OID;
-	}
-}
-
-static inline void ostid_set_seq_mdt0(struct ost_id *oi)
-{
-	ostid_set_seq(oi, FID_SEQ_OST_MDT0);
-}
-
-static inline void ostid_set_seq_echo(struct ost_id *oi)
-{
-	ostid_set_seq(oi, FID_SEQ_ECHO);
-}
-
-static inline void ostid_set_seq_llog(struct ost_id *oi)
-{
-	ostid_set_seq(oi, FID_SEQ_LLOG);
-}
-
-/**
- * Note: we need check oi_seq to decide where to set oi_id,
- * so oi_seq should always be set ahead of oi_id.
- */
-static inline void ostid_set_id(struct ost_id *oi, __u64 oid)
-{
-	if (fid_seq_is_mdt0(oi->oi.oi_seq)) {
-		if (oid >= IDIF_MAX_OID) {
-			CERROR("Too large OID %#llx to set MDT0 " DOSTID "\n",
-			       oid, POSTID(oi));
-			return;
-		}
-		oi->oi.oi_id = oid;
-	} else if (fid_is_idif(&oi->oi_fid)) {
-		if (oid >= IDIF_MAX_OID) {
-			CERROR("Too large OID %#llx to set IDIF " DOSTID "\n",
-			       oid, POSTID(oi));
-			return;
-		}
-		oi->oi_fid.f_seq = fid_idif_seq(oid,
-						fid_idif_ost_idx(&oi->oi_fid));
-		oi->oi_fid.f_oid = oid;
-		oi->oi_fid.f_ver = oid >> 48;
-	} else {
-		if (oid >= OBIF_MAX_OID) {
-			CERROR("Bad %llu to set " DOSTID "\n", oid, POSTID(oi));
-			return;
-		}
-		oi->oi_fid.f_oid = oid;
-	}
-}
-
-static inline int fid_set_id(struct lu_fid *fid, __u64 oid)
-{
-	if (unlikely(fid_seq_is_igif(fid->f_seq))) {
-		CERROR("bad IGIF, " DFID "\n", PFID(fid));
-		return -EBADF;
-	}
-
-	if (fid_is_idif(fid)) {
-		if (oid >= IDIF_MAX_OID) {
-			CERROR("Too large OID %#llx to set IDIF " DFID "\n",
-			       (unsigned long long)oid, PFID(fid));
-			return -EBADF;
-		}
-		fid->f_seq = fid_idif_seq(oid, fid_idif_ost_idx(fid));
-		fid->f_oid = oid;
-		fid->f_ver = oid >> 48;
-	} else {
-		if (oid >= OBIF_MAX_OID) {
-			CERROR("Too large OID %#llx to set REG " DFID "\n",
-			       (unsigned long long)oid, PFID(fid));
-			return -EBADF;
-		}
-		fid->f_oid = oid;
-	}
-	return 0;
-}
-
-/**
- * Unpack an OST object id/seq (group) into a FID.  This is needed for
- * converting all obdo, lmm, lsm, etc. 64-bit id/seq pairs into proper
- * FIDs.  Note that if an id/seq is already in FID/IDIF format it will
- * be passed through unchanged.  Only legacy OST objects in "group 0"
- * will be mapped into the IDIF namespace so that they can fit into the
- * struct lu_fid fields without loss.  For reference see:
- * http://wiki.old.lustre.org/index.php/Architecture_-_Interoperability_fids_zfs
- */
-static inline int ostid_to_fid(struct lu_fid *fid, struct ost_id *ostid,
-			       __u32 ost_idx)
-{
-	__u64 seq = ostid_seq(ostid);
-
-	if (ost_idx > 0xffff) {
-		CERROR("bad ost_idx, " DOSTID " ost_idx:%u\n", POSTID(ostid),
-		       ost_idx);
-		return -EBADF;
-	}
-
-	if (fid_seq_is_mdt0(seq)) {
-		__u64 oid = ostid_id(ostid);
-
-		/* This is a "legacy" (old 1.x/2.early) OST object in "group 0"
-		 * that we map into the IDIF namespace.  It allows up to 2^48
-		 * objects per OST, as this is the object namespace that has
-		 * been in production for years.  This can handle create rates
-		 * of 1M objects/s/OST for 9 years, or combinations thereof.
-		 */
-		if (oid >= IDIF_MAX_OID) {
-			CERROR("bad MDT0 id, " DOSTID " ost_idx:%u\n",
-			       POSTID(ostid), ost_idx);
-			return -EBADF;
-		}
-		fid->f_seq = fid_idif_seq(oid, ost_idx);
-		/* truncate to 32 bits by assignment */
-		fid->f_oid = oid;
-		/* in theory, not currently used */
-		fid->f_ver = oid >> 48;
-	} else if (likely(!fid_seq_is_default(seq))) {
-	       /* This is either an IDIF object, which identifies objects across
-		* all OSTs, or a regular FID.  The IDIF namespace maps legacy
-		* OST objects into the FID namespace.  In both cases, we just
-		* pass the FID through, no conversion needed.
-		*/
-		if (ostid->oi_fid.f_ver != 0) {
-			CERROR("bad MDT0 id, " DOSTID " ost_idx:%u\n",
-			       POSTID(ostid), ost_idx);
-			return -EBADF;
-		}
-		*fid = ostid->oi_fid;
-	}
-
-	return 0;
-}
-
-/* pack any OST FID into an ostid (id/seq) for the wire/disk */
-static inline int fid_to_ostid(const struct lu_fid *fid, struct ost_id *ostid)
-{
-	if (unlikely(fid_seq_is_igif(fid->f_seq))) {
-		CERROR("bad IGIF, " DFID "\n", PFID(fid));
-		return -EBADF;
-	}
-
-	if (fid_is_idif(fid)) {
-		ostid_set_seq_mdt0(ostid);
-		ostid_set_id(ostid, fid_idif_id(fid_seq(fid), fid_oid(fid),
-						fid_ver(fid)));
-	} else {
-		ostid->oi_fid = *fid;
-	}
-
-	return 0;
-}
-
-/* Check whether the fid is for LAST_ID */
-static inline bool fid_is_last_id(const struct lu_fid *fid)
-{
-	return (fid_oid(fid) == 0);
-}
-
-/**
- * Get inode number from a igif.
- * \param fid a igif to get inode number from.
- * \return inode number for the igif.
- */
-static inline ino_t lu_igif_ino(const struct lu_fid *fid)
-{
-	return fid_seq(fid);
-}
-
-/**
- * Get inode generation from a igif.
- * \param fid a igif to get inode generation from.
- * \return inode generation for the igif.
- */
-static inline __u32 lu_igif_gen(const struct lu_fid *fid)
-{
-	return fid_oid(fid);
-}
-
-/**
- * Build igif from the inode number/generation.
- */
-static inline void lu_igif_build(struct lu_fid *fid, __u32 ino, __u32 gen)
-{
-	fid->f_seq = ino;
-	fid->f_oid = gen;
-	fid->f_ver = 0;
-}
-
-/*
- * Fids are transmitted across network (in the sender byte-ordering),
- * and stored on disk in big-endian order.
- */
-static inline void fid_cpu_to_le(struct lu_fid *dst, const struct lu_fid *src)
-{
-	dst->f_seq = cpu_to_le64(fid_seq(src));
-	dst->f_oid = cpu_to_le32(fid_oid(src));
-	dst->f_ver = cpu_to_le32(fid_ver(src));
-}
-
-static inline void fid_le_to_cpu(struct lu_fid *dst, const struct lu_fid *src)
-{
-	dst->f_seq = le64_to_cpu(fid_seq(src));
-	dst->f_oid = le32_to_cpu(fid_oid(src));
-	dst->f_ver = le32_to_cpu(fid_ver(src));
-}
-
-static inline void fid_cpu_to_be(struct lu_fid *dst, const struct lu_fid *src)
-{
-	dst->f_seq = cpu_to_be64(fid_seq(src));
-	dst->f_oid = cpu_to_be32(fid_oid(src));
-	dst->f_ver = cpu_to_be32(fid_ver(src));
-}
-
-static inline void fid_be_to_cpu(struct lu_fid *dst, const struct lu_fid *src)
-{
-	dst->f_seq = be64_to_cpu(fid_seq(src));
-	dst->f_oid = be32_to_cpu(fid_oid(src));
-	dst->f_ver = be32_to_cpu(fid_ver(src));
-}
-
-static inline bool fid_is_sane(const struct lu_fid *fid)
-{
-	return fid &&
-	       ((fid_seq(fid) >= FID_SEQ_START && fid_ver(fid) == 0) ||
-		fid_is_igif(fid) || fid_is_idif(fid) ||
-		fid_seq_is_rsvd(fid_seq(fid)));
-}
-
-static inline bool lu_fid_eq(const struct lu_fid *f0, const struct lu_fid *f1)
-{
-	return memcmp(f0, f1, sizeof(*f0)) == 0;
-}
-
-#define __diff_normalize(val0, val1)			    \
-({							      \
-	typeof(val0) __val0 = (val0);			   \
-	typeof(val1) __val1 = (val1);			   \
-								\
-	(__val0 == __val1 ? 0 : __val0 > __val1 ? 1 : -1);     \
-})
-
-static inline int lu_fid_cmp(const struct lu_fid *f0,
-			     const struct lu_fid *f1)
-{
-	return
-		__diff_normalize(fid_seq(f0), fid_seq(f1)) ?:
-		__diff_normalize(fid_oid(f0), fid_oid(f1)) ?:
-		__diff_normalize(fid_ver(f0), fid_ver(f1));
-}
-
-static inline void ostid_cpu_to_le(const struct ost_id *src_oi,
-				   struct ost_id *dst_oi)
-{
-	if (fid_seq_is_mdt0(ostid_seq(src_oi))) {
-		dst_oi->oi.oi_id = cpu_to_le64(src_oi->oi.oi_id);
-		dst_oi->oi.oi_seq = cpu_to_le64(src_oi->oi.oi_seq);
-	} else {
-		fid_cpu_to_le(&dst_oi->oi_fid, &src_oi->oi_fid);
-	}
-}
-
-static inline void ostid_le_to_cpu(const struct ost_id *src_oi,
-				   struct ost_id *dst_oi)
-{
-	if (fid_seq_is_mdt0(ostid_seq(src_oi))) {
-		dst_oi->oi.oi_id = le64_to_cpu(src_oi->oi.oi_id);
-		dst_oi->oi.oi_seq = le64_to_cpu(src_oi->oi.oi_seq);
-	} else {
-		fid_le_to_cpu(&dst_oi->oi_fid, &src_oi->oi_fid);
-	}
-}
-
 /** @} lu_fid */
 
 /** \defgroup lu_dir lu_dir
@@ -866,7 +390,7 @@ enum lu_dirpage_flags {
 
 static inline struct lu_dirent *lu_dirent_start(struct lu_dirpage *dp)
 {
-	if (le32_to_cpu(dp->ldp_flags) & LDF_EMPTY)
+	if (__le32_to_cpu(dp->ldp_flags) & LDF_EMPTY)
 		return NULL;
 	else
 		return dp->ldp_entries;
@@ -876,8 +400,8 @@ static inline struct lu_dirent *lu_dirent_next(struct lu_dirent *ent)
 {
 	struct lu_dirent *next;
 
-	if (le16_to_cpu(ent->lde_reclen) != 0)
-		next = ((void *)ent) + le16_to_cpu(ent->lde_reclen);
+	if (__le16_to_cpu(ent->lde_reclen) != 0)
+		next = ((void *)ent) + __le16_to_cpu(ent->lde_reclen);
 	else
 		next = NULL;
 
@@ -1122,7 +646,7 @@ struct ptlrpc_body_v2 {
 #define OBD_CONNECT_AT		    0x1000000ULL /*client uses AT */
 #define OBD_CONNECT_LRU_RESIZE      0x2000000ULL /*LRU resize feature. */
 #define OBD_CONNECT_MDS_MDS	    0x4000000ULL /*MDS-MDS connection */
-#define OBD_CONNECT_REAL	    0x8000000ULL /*real connection */
+#define OBD_CONNECT_REAL	    0x8000000ULL /* obsolete since 2.8 */
 #define OBD_CONNECT_CHANGE_QS      0x10000000ULL /*Not used since 2.4 */
 #define OBD_CONNECT_CKSUM	   0x20000000ULL /*support several cksum algos*/
 #define OBD_CONNECT_FID		   0x40000000ULL /*FID is supported by server */
@@ -1384,71 +908,6 @@ struct lov_mds_md_v1 {	    /* LOV EA mds/wire data (little-endian) */
 	struct lov_ost_data_v1 lmm_objects[0]; /* per-stripe data */
 };
 
-/**
- * Sigh, because pre-2.4 uses
- * struct lov_mds_md_v1 {
- *	........
- *	__u64 lmm_object_id;
- *	__u64 lmm_object_seq;
- *      ......
- *      }
- * to identify the LOV(MDT) object, and lmm_object_seq will
- * be normal_fid, which make it hard to combine these conversion
- * to ostid_to FID. so we will do lmm_oi/fid conversion separately
- *
- * We can tell the lmm_oi by this way,
- * 1.8: lmm_object_id = {inode}, lmm_object_gr = 0
- * 2.1: lmm_object_id = {oid < 128k}, lmm_object_seq = FID_SEQ_NORMAL
- * 2.4: lmm_oi.f_seq = FID_SEQ_NORMAL, lmm_oi.f_oid = {oid < 128k},
- *      lmm_oi.f_ver = 0
- *
- * But currently lmm_oi/lsm_oi does not have any "real" usages,
- * except for printing some information, and the user can always
- * get the real FID from LMA, besides this multiple case check might
- * make swab more complicate. So we will keep using id/seq for lmm_oi.
- */
-
-static inline void fid_to_lmm_oi(const struct lu_fid *fid,
-				 struct ost_id *oi)
-{
-	oi->oi.oi_id = fid_oid(fid);
-	oi->oi.oi_seq = fid_seq(fid);
-}
-
-static inline void lmm_oi_set_seq(struct ost_id *oi, __u64 seq)
-{
-	oi->oi.oi_seq = seq;
-}
-
-static inline void lmm_oi_set_id(struct ost_id *oi, __u64 oid)
-{
-	oi->oi.oi_id = oid;
-}
-
-static inline __u64 lmm_oi_id(const struct ost_id *oi)
-{
-	return oi->oi.oi_id;
-}
-
-static inline __u64 lmm_oi_seq(const struct ost_id *oi)
-{
-	return oi->oi.oi_seq;
-}
-
-static inline void lmm_oi_le_to_cpu(struct ost_id *dst_oi,
-				    const struct ost_id *src_oi)
-{
-	dst_oi->oi.oi_id = le64_to_cpu(src_oi->oi.oi_id);
-	dst_oi->oi.oi_seq = le64_to_cpu(src_oi->oi.oi_seq);
-}
-
-static inline void lmm_oi_cpu_to_le(struct ost_id *dst_oi,
-				    const struct ost_id *src_oi)
-{
-	dst_oi->oi.oi_id = cpu_to_le64(src_oi->oi.oi_id);
-	dst_oi->oi.oi_seq = cpu_to_le64(src_oi->oi.oi_seq);
-}
-
 #define MAX_MD_SIZE							\
 	(sizeof(struct lov_mds_md) + 4 * sizeof(struct lov_ost_data))
 #define MIN_MD_SIZE							\
@@ -2095,8 +1554,8 @@ enum mds_op_bias {
 	MDS_CREATE_VOLATILE	= 1 << 10,
 	MDS_OWNEROVERRIDE	= 1 << 11,
 	MDS_HSM_RELEASE		= 1 << 12,
-	MDS_RENAME_MIGRATE	= BIT(13),
-	MDS_CLOSE_LAYOUT_SWAP   = BIT(14),
+	MDS_RENAME_MIGRATE	= 1 << 13,
+	MDS_CLOSE_LAYOUT_SWAP	= 1 << 14,
 };
 
 /* instance of mdt_reint_rec */
@@ -2130,17 +1589,6 @@ struct mdt_rec_create {
 	__u32	   cr_padding_4;   /* rr_padding_4 */
 };
 
-static inline void set_mrc_cr_flags(struct mdt_rec_create *mrc, __u64 flags)
-{
-	mrc->cr_flags_l = (__u32)(flags & 0xFFFFFFFFUll);
-	mrc->cr_flags_h = (__u32)(flags >> 32);
-}
-
-static inline __u64 get_mrc_cr_flags(struct mdt_rec_create *mrc)
-{
-	return ((__u64)(mrc->cr_flags_l) | ((__u64)mrc->cr_flags_h << 32));
-}
-
 /* instance of mdt_reint_rec */
 struct mdt_rec_link {
 	__u32	   lk_opcode;
@@ -2393,35 +1841,16 @@ static inline ssize_t lmv_mds_md_size(int stripe_count, unsigned int lmm_magic)
 
 static inline int lmv_mds_md_stripe_count_get(const union lmv_mds_md *lmm)
 {
-	switch (le32_to_cpu(lmm->lmv_magic)) {
+	switch (__le32_to_cpu(lmm->lmv_magic)) {
 	case LMV_MAGIC_V1:
-		return le32_to_cpu(lmm->lmv_md_v1.lmv_stripe_count);
+		return __le32_to_cpu(lmm->lmv_md_v1.lmv_stripe_count);
 	case LMV_USER_MAGIC:
-		return le32_to_cpu(lmm->lmv_user_md.lum_stripe_count);
+		return __le32_to_cpu(lmm->lmv_user_md.lum_stripe_count);
 	default:
 		return -EINVAL;
 	}
 }
 
-static inline int lmv_mds_md_stripe_count_set(union lmv_mds_md *lmm,
-					      unsigned int stripe_count)
-{
-	int rc = 0;
-
-	switch (le32_to_cpu(lmm->lmv_magic)) {
-	case LMV_MAGIC_V1:
-		lmm->lmv_md_v1.lmv_stripe_count = cpu_to_le32(stripe_count);
-		break;
-	case LMV_USER_MAGIC:
-		lmm->lmv_user_md.lum_stripe_count = cpu_to_le32(stripe_count);
-		break;
-	default:
-		rc = -EINVAL;
-		break;
-	}
-	return rc;
-}
-
 enum fld_rpc_opc {
 	FLD_QUERY	= 900,
 	FLD_READ	= 901,
@@ -2502,12 +1931,6 @@ struct ldlm_res_id {
 #define PLDLMRES(res)	(res)->lr_name.name[0], (res)->lr_name.name[1], \
 			(res)->lr_name.name[2], (res)->lr_name.name[3]
 
-static inline bool ldlm_res_eq(const struct ldlm_res_id *res0,
-			       const struct ldlm_res_id *res1)
-{
-	return !memcmp(res0, res1, sizeof(*res0));
-}
-
 /* lock types */
 enum ldlm_mode {
 	LCK_MINMODE = 0,
@@ -2540,19 +1963,6 @@ struct ldlm_extent {
 	__u64 gid;
 };
 
-static inline int ldlm_extent_overlap(const struct ldlm_extent *ex1,
-				      const struct ldlm_extent *ex2)
-{
-	return (ex1->start <= ex2->end) && (ex2->start <= ex1->end);
-}
-
-/* check if @ex1 contains @ex2 */
-static inline int ldlm_extent_contain(const struct ldlm_extent *ex1,
-				      const struct ldlm_extent *ex2)
-{
-	return (ex1->start <= ex2->start) && (ex1->end >= ex2->end);
-}
-
 struct ldlm_inodebits {
 	__u64 bits;
 };
@@ -2583,21 +1993,21 @@ union ldlm_gl_desc {
 };
 
 enum ldlm_intent_flags {
-	IT_OPEN		= BIT(0),
-	IT_CREAT	= BIT(1),
-	IT_OPEN_CREAT	= BIT(1) | BIT(0),
-	IT_READDIR	= BIT(2),
-	IT_GETATTR	= BIT(3),
-	IT_LOOKUP	= BIT(4),
-	IT_UNLINK	= BIT(5),
-	IT_TRUNC	= BIT(6),
-	IT_GETXATTR	= BIT(7),
-	IT_EXEC		= BIT(8),
-	IT_PIN		= BIT(9),
-	IT_LAYOUT	= BIT(10),
-	IT_QUOTA_DQACQ	= BIT(11),
-	IT_QUOTA_CONN	= BIT(12),
-	IT_SETXATTR	= BIT(13),
+	IT_OPEN		= 0x00000001,
+	IT_CREAT	= 0x00000002,
+	IT_OPEN_CREAT	= 0x00000003,
+	IT_READDIR	= 0x00000004,
+	IT_GETATTR	= 0x00000008,
+	IT_LOOKUP	= 0x00000010,
+	IT_UNLINK	= 0x00000020,
+	IT_TRUNC	= 0x00000040,
+	IT_GETXATTR	= 0x00000080,
+	IT_EXEC		= 0x00000100,
+	IT_PIN		= 0x00000200,
+	IT_LAYOUT	= 0x00000400,
+	IT_QUOTA_DQACQ	= 0x00000800,
+	IT_QUOTA_CONN	= 0x00001000,
+	IT_SETXATTR	= 0x00002000,
 };
 
 struct ldlm_intent {
@@ -2627,18 +2037,6 @@ struct ldlm_request {
 	struct lustre_handle lock_handle[LDLM_LOCKREQ_HANDLES];
 };
 
-/* If LDLM_ENQUEUE, 1 slot is already occupied, 1 is available.
- * Otherwise, 2 are available.
- */
-#define ldlm_request_bufsize(count, type)				\
-({								      \
-	int _avail = LDLM_LOCKREQ_HANDLES;			      \
-	_avail -= (type == LDLM_ENQUEUE ? LDLM_ENQUEUE_CANCEL_OFF : 0); \
-	sizeof(struct ldlm_request) +				   \
-	(count > _avail ? count - _avail : 0) *			 \
-	sizeof(struct lustre_handle);				   \
-})
-
 struct ldlm_reply {
 	__u32 lock_flags;
 	__u32 lock_padding;     /* also fix lustre_swab_ldlm_reply */
@@ -2942,12 +2340,6 @@ static inline const char *agent_req_status2name(const enum agent_req_status ars)
 	}
 }
 
-static inline bool agent_req_in_final_state(enum agent_req_status ars)
-{
-	return ((ars == ARS_SUCCEED) || (ars == ARS_FAILED) ||
-		(ars == ARS_CANCELED));
-}
-
 struct llog_agent_req_rec {
 	struct llog_rec_hdr	arr_hdr;	/**< record header */
 	__u32			arr_status;	/**< status of the request */
@@ -2983,8 +2375,8 @@ enum llog_flag {
 	LLOG_F_ZAP_WHEN_EMPTY	= 0x1,
 	LLOG_F_IS_CAT		= 0x2,
 	LLOG_F_IS_PLAIN		= 0x4,
-	LLOG_F_EXT_JOBID        = BIT(3),
-	LLOG_F_IS_FIXSIZE	= BIT(4),
+	LLOG_F_EXT_JOBID        = 0x8,
+	LLOG_F_IS_FIXSIZE	= 0x10,
 
 	/*
 	 * Note: Flags covered by LLOG_F_EXT_MASK will be inherited from
@@ -3142,12 +2534,6 @@ struct ll_fiemap_info_key {
 	struct fiemap	lfik_fiemap;
 };
 
-/* Functions for dumping PTLRPC fields */
-void dump_rniobuf(struct niobuf_remote *rnb);
-void dump_ioo(struct obd_ioobj *nb);
-void dump_ost_body(struct ost_body *ob);
-void dump_rcs(__u32 *rc);
-
 /* security opcodes */
 enum sec_cmd {
 	SEC_CTX_INIT	    = 801,
@@ -3217,9 +2603,8 @@ struct link_ea_header {
 	__u32 leh_magic;
 	__u32 leh_reccount;
 	__u64 leh_len;      /* total size */
-	/* future use */
-	__u32 padding1;
-	__u32 padding2;
+	__u32 leh_overflow_time;
+	__u32 leh_padding;
 };
 
 /** Hardlink data is name and parent fid.
diff --git a/drivers/staging/lustre/lustre/include/lustre/lustre_ioctl.h b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_ioctl.h
index eb08df33b2db..9590864e0b50 100644
--- a/drivers/staging/lustre/lustre/include/lustre/lustre_ioctl.h
+++ b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_ioctl.h
@@ -25,22 +25,13 @@
  *
  * Copyright (c) 2011, 2015, Intel Corporation.
  */
-#ifndef LUSTRE_IOCTL_H_
-#define LUSTRE_IOCTL_H_
+#ifndef _UAPI_LUSTRE_IOCTL_H_
+#define _UAPI_LUSTRE_IOCTL_H_
 
+#include <linux/ioctl.h>
+#include <linux/kernel.h>
 #include <linux/types.h>
-#include "../../../include/linux/libcfs/libcfs.h"
-#include "lustre_idl.h"
-
-#ifdef __KERNEL__
-# include <linux/ioctl.h>
-# include <linux/string.h>
-# include "../obd_support.h"
-#else /* __KERNEL__ */
-# include <malloc.h>
-# include <string.h>
-#include <libcfs/util/ioctl.h>
-#endif /* !__KERNEL__ */
+#include <uapi/linux/lustre/lustre_idl.h>
 
 #if !defined(__KERNEL__) && !defined(LUSTRE_UTILS)
 # error This file is for Lustre internal use only.
@@ -65,7 +56,6 @@ enum md_echo_cmd {
 
 #define OBD_IOCTL_VERSION	0x00010004
 #define OBD_DEV_BY_DEVNAME	0xffffd0de
-#define OBD_MAX_IOCTL_BUFFER	CONFIG_LUSTRE_OBD_MAX_IOCTL_BUFFER
 
 struct obd_ioctl_data {
 	__u32		ioc_len;
@@ -122,187 +112,16 @@ struct obd_ioctl_hdr {
 
 static inline __u32 obd_ioctl_packlen(struct obd_ioctl_data *data)
 {
-	__u32 len = cfs_size_round(sizeof(*data));
+	__u32 len = __ALIGN_KERNEL(sizeof(*data), 8);
 
-	len += cfs_size_round(data->ioc_inllen1);
-	len += cfs_size_round(data->ioc_inllen2);
-	len += cfs_size_round(data->ioc_inllen3);
-	len += cfs_size_round(data->ioc_inllen4);
+	len += __ALIGN_KERNEL(data->ioc_inllen1, 8);
+	len += __ALIGN_KERNEL(data->ioc_inllen2, 8);
+	len += __ALIGN_KERNEL(data->ioc_inllen3, 8);
+	len += __ALIGN_KERNEL(data->ioc_inllen4, 8);
 
 	return len;
 }
 
-static inline int obd_ioctl_is_invalid(struct obd_ioctl_data *data)
-{
-	if (data->ioc_len > (1 << 30)) {
-		CERROR("OBD ioctl: ioc_len larger than 1<<30\n");
-		return 1;
-	}
-
-	if (data->ioc_inllen1 > (1 << 30)) {
-		CERROR("OBD ioctl: ioc_inllen1 larger than 1<<30\n");
-		return 1;
-	}
-
-	if (data->ioc_inllen2 > (1 << 30)) {
-		CERROR("OBD ioctl: ioc_inllen2 larger than 1<<30\n");
-		return 1;
-	}
-
-	if (data->ioc_inllen3 > (1 << 30)) {
-		CERROR("OBD ioctl: ioc_inllen3 larger than 1<<30\n");
-		return 1;
-	}
-
-	if (data->ioc_inllen4 > (1 << 30)) {
-		CERROR("OBD ioctl: ioc_inllen4 larger than 1<<30\n");
-		return 1;
-	}
-
-	if (data->ioc_inlbuf1 && !data->ioc_inllen1) {
-		CERROR("OBD ioctl: inlbuf1 pointer but 0 length\n");
-		return 1;
-	}
-
-	if (data->ioc_inlbuf2 && !data->ioc_inllen2) {
-		CERROR("OBD ioctl: inlbuf2 pointer but 0 length\n");
-		return 1;
-	}
-
-	if (data->ioc_inlbuf3 && !data->ioc_inllen3) {
-		CERROR("OBD ioctl: inlbuf3 pointer but 0 length\n");
-		return 1;
-	}
-
-	if (data->ioc_inlbuf4 && !data->ioc_inllen4) {
-		CERROR("OBD ioctl: inlbuf4 pointer but 0 length\n");
-		return 1;
-	}
-
-	if (data->ioc_pbuf1 && !data->ioc_plen1) {
-		CERROR("OBD ioctl: pbuf1 pointer but 0 length\n");
-		return 1;
-	}
-
-	if (data->ioc_pbuf2 && !data->ioc_plen2) {
-		CERROR("OBD ioctl: pbuf2 pointer but 0 length\n");
-		return 1;
-	}
-
-	if (!data->ioc_pbuf1 && data->ioc_plen1) {
-		CERROR("OBD ioctl: plen1 set but NULL pointer\n");
-		return 1;
-	}
-
-	if (!data->ioc_pbuf2 && data->ioc_plen2) {
-		CERROR("OBD ioctl: plen2 set but NULL pointer\n");
-		return 1;
-	}
-
-	if (obd_ioctl_packlen(data) > data->ioc_len) {
-		CERROR("OBD ioctl: packlen exceeds ioc_len (%d > %d)\n",
-		       obd_ioctl_packlen(data), data->ioc_len);
-		return 1;
-	}
-
-	return 0;
-}
-
-#ifdef __KERNEL__
-
-int obd_ioctl_getdata(char **buf, int *len, void __user *arg);
-int obd_ioctl_popdata(void __user *arg, void *data, int len);
-
-static inline void obd_ioctl_freedata(char *buf, size_t len)
-{
-	kvfree(buf);
-}
-
-#else /* __KERNEL__ */
-
-static inline int obd_ioctl_pack(struct obd_ioctl_data *data, char **pbuf,
-				 int max_len)
-{
-	char *ptr;
-	struct obd_ioctl_data *overlay;
-
-	data->ioc_len = obd_ioctl_packlen(data);
-	data->ioc_version = OBD_IOCTL_VERSION;
-
-	if (*pbuf && data->ioc_len > max_len) {
-		fprintf(stderr, "pbuf = %p, ioc_len = %u, max_len = %d\n",
-			*pbuf, data->ioc_len, max_len);
-		return -EINVAL;
-	}
-
-	if (!*pbuf)
-		*pbuf = malloc(data->ioc_len);
-
-	if (!*pbuf)
-		return -ENOMEM;
-
-	overlay = (struct obd_ioctl_data *)*pbuf;
-	memcpy(*pbuf, data, sizeof(*data));
-
-	ptr = overlay->ioc_bulk;
-	if (data->ioc_inlbuf1)
-		LOGL(data->ioc_inlbuf1, data->ioc_inllen1, ptr);
-
-	if (data->ioc_inlbuf2)
-		LOGL(data->ioc_inlbuf2, data->ioc_inllen2, ptr);
-
-	if (data->ioc_inlbuf3)
-		LOGL(data->ioc_inlbuf3, data->ioc_inllen3, ptr);
-
-	if (data->ioc_inlbuf4)
-		LOGL(data->ioc_inlbuf4, data->ioc_inllen4, ptr);
-
-	if (obd_ioctl_is_invalid(overlay)) {
-		fprintf(stderr, "invalid ioctl data: ioc_len = %u, max_len = %d\n",
-			data->ioc_len, max_len);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static inline int
-obd_ioctl_unpack(struct obd_ioctl_data *data, char *pbuf, int max_len)
-{
-	char *ptr;
-	struct obd_ioctl_data *overlay;
-
-	if (!pbuf)
-		return 1;
-
-	overlay = (struct obd_ioctl_data *)pbuf;
-
-	/* Preserve the caller's buffer pointers */
-	overlay->ioc_inlbuf1 = data->ioc_inlbuf1;
-	overlay->ioc_inlbuf2 = data->ioc_inlbuf2;
-	overlay->ioc_inlbuf3 = data->ioc_inlbuf3;
-	overlay->ioc_inlbuf4 = data->ioc_inlbuf4;
-
-	memcpy(data, pbuf, sizeof(*data));
-
-	ptr = overlay->ioc_bulk;
-	if (data->ioc_inlbuf1)
-		LOGU(data->ioc_inlbuf1, data->ioc_inllen1, ptr);
-
-	if (data->ioc_inlbuf2)
-		LOGU(data->ioc_inlbuf2, data->ioc_inllen2, ptr);
-
-	if (data->ioc_inlbuf3)
-		LOGU(data->ioc_inlbuf3, data->ioc_inllen3, ptr);
-
-	if (data->ioc_inlbuf4)
-		LOGU(data->ioc_inlbuf4, data->ioc_inllen4, ptr);
-
-	return 0;
-}
-
-#endif /* !__KERNEL__ */
-
 /*
  * OBD_IOC_DATA_TYPE is only for compatibility reasons with older
  * Linux Lustre user tools. New ioctls should NOT use this macro as
@@ -409,4 +228,4 @@ obd_ioctl_unpack(struct obd_ioctl_data *data, char *pbuf, int max_len)
 
 #define IOC_OSC_SET_ACTIVE	_IOWR('h', 21, void *)
 
-#endif /* LUSTRE_IOCTL_H_ */
+#endif /* _UAPI_LUSTRE_IOCTL_H_ */
diff --git a/drivers/staging/lustre/lustre/include/uapi_kernelcomm.h b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_kernelcomm.h
index 5e998362e44b..94dadbe8e069 100644
--- a/drivers/staging/lustre/lustre/include/uapi_kernelcomm.h
+++ b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_kernelcomm.h
@@ -34,8 +34,8 @@
  * The definitions below are used in the kernel and userspace.
  */
 
-#ifndef __UAPI_KERNELCOMM_H__
-#define __UAPI_KERNELCOMM_H__
+#ifndef __UAPI_LUSTRE_KERNELCOMM_H__
+#define __UAPI_LUSTRE_KERNELCOMM_H__
 
 #include <linux/types.h>
 
@@ -91,4 +91,4 @@ struct lustre_kernelcomm {
 	__u32 lk_flags;
 } __packed;
 
-#endif	/* __UAPI_KERNELCOMM_H__ */
+#endif	/* __UAPI_LUSTRE_KERNELCOMM_H__ */
diff --git a/drivers/staging/lustre/include/uapi/linux/lustre/lustre_ostid.h b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_ostid.h
new file mode 100644
index 000000000000..3343b602219b
--- /dev/null
+++ b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_ostid.h
@@ -0,0 +1,236 @@
+/*
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see
+ * http://www.gnu.org/licenses/gpl-2.0.html
+ *
+ * GPL HEADER END
+ */
+/*
+ * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Use is subject to license terms.
+ *
+ * Copyright (c) 2011, 2014, Intel Corporation.
+ *
+ * Copyright 2015 Cray Inc, all rights reserved.
+ * Author: Ben Evans.
+ *
+ * Define ost_id  associated functions
+ */
+
+#ifndef _UAPI_LUSTRE_OSTID_H_
+#define _UAPI_LUSTRE_OSTID_H_
+
+#include <linux/errno.h>
+#include <uapi/linux/lustre/lustre_fid.h>
+
+static inline __u64 lmm_oi_id(const struct ost_id *oi)
+{
+	return oi->oi.oi_id;
+}
+
+static inline __u64 lmm_oi_seq(const struct ost_id *oi)
+{
+	return oi->oi.oi_seq;
+}
+
+static inline void lmm_oi_set_seq(struct ost_id *oi, __u64 seq)
+{
+	oi->oi.oi_seq = seq;
+}
+
+static inline void lmm_oi_set_id(struct ost_id *oi, __u64 oid)
+{
+	oi->oi.oi_id = oid;
+}
+
+static inline void lmm_oi_le_to_cpu(struct ost_id *dst_oi,
+				    const struct ost_id *src_oi)
+{
+	dst_oi->oi.oi_id = __le64_to_cpu(src_oi->oi.oi_id);
+	dst_oi->oi.oi_seq = __le64_to_cpu(src_oi->oi.oi_seq);
+}
+
+static inline void lmm_oi_cpu_to_le(struct ost_id *dst_oi,
+				    const struct ost_id *src_oi)
+{
+	dst_oi->oi.oi_id = __cpu_to_le64(src_oi->oi.oi_id);
+	dst_oi->oi.oi_seq = __cpu_to_le64(src_oi->oi.oi_seq);
+}
+
+/* extract OST sequence (group) from a wire ost_id (id/seq) pair */
+static inline __u64 ostid_seq(const struct ost_id *ostid)
+{
+	if (fid_seq_is_mdt0(ostid->oi.oi_seq))
+		return FID_SEQ_OST_MDT0;
+
+	if (fid_seq_is_default(ostid->oi.oi_seq))
+		return FID_SEQ_LOV_DEFAULT;
+
+	if (fid_is_idif(&ostid->oi_fid))
+		return FID_SEQ_OST_MDT0;
+
+	return fid_seq(&ostid->oi_fid);
+}
+
+/* extract OST objid from a wire ost_id (id/seq) pair */
+static inline __u64 ostid_id(const struct ost_id *ostid)
+{
+	if (fid_seq_is_mdt0(ostid->oi.oi_seq))
+		return ostid->oi.oi_id & IDIF_OID_MASK;
+
+	if (fid_seq_is_default(ostid->oi.oi_seq))
+		return ostid->oi.oi_id;
+
+	if (fid_is_idif(&ostid->oi_fid))
+		return fid_idif_id(fid_seq(&ostid->oi_fid),
+				   fid_oid(&ostid->oi_fid), 0);
+
+	return fid_oid(&ostid->oi_fid);
+}
+
+static inline void ostid_set_seq(struct ost_id *oi, __u64 seq)
+{
+	if (fid_seq_is_mdt0(seq) || fid_seq_is_default(seq)) {
+		oi->oi.oi_seq = seq;
+	} else {
+		oi->oi_fid.f_seq = seq;
+		/*
+		 * Note: if f_oid + f_ver is zero, we need init it
+		 * to be 1, otherwise, ostid_seq will treat this
+		 * as old ostid (oi_seq == 0)
+		 */
+		if (!oi->oi_fid.f_oid && !oi->oi_fid.f_ver)
+			oi->oi_fid.f_oid = LUSTRE_FID_INIT_OID;
+	}
+}
+
+static inline void ostid_set_seq_mdt0(struct ost_id *oi)
+{
+	ostid_set_seq(oi, FID_SEQ_OST_MDT0);
+}
+
+static inline void ostid_set_seq_echo(struct ost_id *oi)
+{
+	ostid_set_seq(oi, FID_SEQ_ECHO);
+}
+
+static inline void ostid_set_seq_llog(struct ost_id *oi)
+{
+	ostid_set_seq(oi, FID_SEQ_LLOG);
+}
+
+static inline void ostid_cpu_to_le(const struct ost_id *src_oi,
+				   struct ost_id *dst_oi)
+{
+	if (fid_seq_is_mdt0(src_oi->oi.oi_seq)) {
+		dst_oi->oi.oi_id = __cpu_to_le64(src_oi->oi.oi_id);
+		dst_oi->oi.oi_seq = __cpu_to_le64(src_oi->oi.oi_seq);
+	} else {
+		fid_cpu_to_le(&dst_oi->oi_fid, &src_oi->oi_fid);
+	}
+}
+
+static inline void ostid_le_to_cpu(const struct ost_id *src_oi,
+				   struct ost_id *dst_oi)
+{
+	if (fid_seq_is_mdt0(src_oi->oi.oi_seq)) {
+		dst_oi->oi.oi_id = __le64_to_cpu(src_oi->oi.oi_id);
+		dst_oi->oi.oi_seq = __le64_to_cpu(src_oi->oi.oi_seq);
+	} else {
+		fid_le_to_cpu(&dst_oi->oi_fid, &src_oi->oi_fid);
+	}
+}
+
+/**
+ * Sigh, because pre-2.4 uses
+ * struct lov_mds_md_v1 {
+ *	........
+ *	__u64 lmm_object_id;
+ *	__u64 lmm_object_seq;
+ *      ......
+ *      }
+ * to identify the LOV(MDT) object, and lmm_object_seq will
+ * be normal_fid, which make it hard to combine these conversion
+ * to ostid_to FID. so we will do lmm_oi/fid conversion separately
+ *
+ * We can tell the lmm_oi by this way,
+ * 1.8: lmm_object_id = {inode}, lmm_object_gr = 0
+ * 2.1: lmm_object_id = {oid < 128k}, lmm_object_seq = FID_SEQ_NORMAL
+ * 2.4: lmm_oi.f_seq = FID_SEQ_NORMAL, lmm_oi.f_oid = {oid < 128k},
+ *      lmm_oi.f_ver = 0
+ *
+ * But currently lmm_oi/lsm_oi does not have any "real" usages,
+ * except for printing some information, and the user can always
+ * get the real FID from LMA, besides this multiple case check might
+ * make swab more complicate. So we will keep using id/seq for lmm_oi.
+ */
+
+static inline void fid_to_lmm_oi(const struct lu_fid *fid,
+				 struct ost_id *oi)
+{
+	oi->oi.oi_id = fid_oid(fid);
+	oi->oi.oi_seq = fid_seq(fid);
+}
+
+/**
+ * Unpack an OST object id/seq (group) into a FID.  This is needed for
+ * converting all obdo, lmm, lsm, etc. 64-bit id/seq pairs into proper
+ * FIDs.  Note that if an id/seq is already in FID/IDIF format it will
+ * be passed through unchanged.  Only legacy OST objects in "group 0"
+ * will be mapped into the IDIF namespace so that they can fit into the
+ * struct lu_fid fields without loss.
+ */
+static inline int ostid_to_fid(struct lu_fid *fid, const struct ost_id *ostid,
+			       __u32 ost_idx)
+{
+	__u64 seq = ostid_seq(ostid);
+
+	if (ost_idx > 0xffff)
+		return -EBADF;
+
+	if (fid_seq_is_mdt0(seq)) {
+		__u64 oid = ostid_id(ostid);
+
+		/* This is a "legacy" (old 1.x/2.early) OST object in "group 0"
+		 * that we map into the IDIF namespace.  It allows up to 2^48
+		 * objects per OST, as this is the object namespace that has
+		 * been in production for years.  This can handle create rates
+		 * of 1M objects/s/OST for 9 years, or combinations thereof.
+		 */
+		if (oid >= IDIF_MAX_OID)
+			return -EBADF;
+
+		fid->f_seq = fid_idif_seq(oid, ost_idx);
+		/* truncate to 32 bits by assignment */
+		fid->f_oid = oid;
+		/* in theory, not currently used */
+		fid->f_ver = oid >> 48;
+	} else if (!fid_seq_is_default(seq)) {
+		/* This is either an IDIF object, which identifies objects
+		 * across all OSTs, or a regular FID.  The IDIF namespace
+		 * maps legacy OST objects into the FID namespace.  In both
+		 * cases, we just pass the FID through, no conversion needed.
+		 */
+		if (ostid->oi_fid.f_ver)
+			return -EBADF;
+
+		*fid = ostid->oi_fid;
+	}
+
+	return 0;
+}
+#endif /* _UAPI_LUSTRE_OSTID_H_ */
diff --git a/drivers/staging/lustre/include/uapi/linux/lustre/lustre_param.h b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_param.h
new file mode 100644
index 000000000000..1eab2ceca338
--- /dev/null
+++ b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_param.h
@@ -0,0 +1,94 @@
+/*
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see
+ * http://www.gnu.org/licenses/gpl-2.0.html
+ *
+ * GPL HEADER END
+ */
+/*
+ * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Use is subject to license terms.
+ *
+ * Copyright (c) 2011, 2015, Intel Corporation.
+ */
+/*
+ * This file is part of Lustre, http://www.lustre.org/
+ * Lustre is a trademark of Sun Microsystems, Inc.
+ *
+ * User-settable parameter keys
+ *
+ * Author: Nathan Rutman <nathan@clusterfs.com>
+ */
+
+#ifndef _UAPI_LUSTRE_PARAM_H_
+#define _UAPI_LUSTRE_PARAM_H_
+
+/** \defgroup param param
+ *
+ * @{
+ */
+
+/****************** User-settable parameter keys *********************/
+/* e.g.
+ *	tunefs.lustre --param="failover.node=192.168.0.13@tcp0" /dev/sda
+ *	lctl conf_param testfs-OST0000 failover.node=3@elan,192.168.0.3@tcp0
+ *		    ... testfs-MDT0000.lov.stripesize=4M
+ *		    ... testfs-OST0000.ost.client_cache_seconds=15
+ *		    ... testfs.sys.timeout=<secs>
+ *		    ... testfs.llite.max_read_ahead_mb=16
+ */
+
+/* System global or special params not handled in obd's proc
+ * See mgs_write_log_sys()
+ */
+#define PARAM_TIMEOUT		"timeout="	   /* global */
+#define PARAM_LDLM_TIMEOUT	"ldlm_timeout="	   /* global */
+#define PARAM_AT_MIN		"at_min="	   /* global */
+#define PARAM_AT_MAX		"at_max="	   /* global */
+#define PARAM_AT_EXTRA		"at_extra="	   /* global */
+#define PARAM_AT_EARLY_MARGIN	"at_early_margin=" /* global */
+#define PARAM_AT_HISTORY	"at_history="	   /* global */
+#define PARAM_JOBID_VAR		"jobid_var="	   /* global */
+#define PARAM_MGSNODE		"mgsnode="	   /* only at mounttime */
+#define PARAM_FAILNODE		"failover.node="   /* add failover nid */
+#define PARAM_FAILMODE		"failover.mode="   /* initial mount only */
+#define PARAM_ACTIVE		"active="	   /* activate/deactivate */
+#define PARAM_NETWORK		"network="	   /* bind on nid */
+#define PARAM_ID_UPCALL		"identity_upcall=" /* identity upcall */
+
+/* Prefixes for parameters handled by obd's proc methods (XXX_process_config) */
+#define PARAM_OST		"ost."
+#define PARAM_OSD		"osd."
+#define PARAM_OSC		"osc."
+#define PARAM_MDT		"mdt."
+#define PARAM_HSM		"mdt.hsm."
+#define PARAM_MDD		"mdd."
+#define PARAM_MDC		"mdc."
+#define PARAM_LLITE		"llite."
+#define PARAM_LOV		"lov."
+#define PARAM_LOD		"lod."
+#define PARAM_OSP		"osp."
+#define PARAM_SYS		"sys."		/* global */
+#define PARAM_SRPC		"srpc."
+#define PARAM_SRPC_FLVR		"srpc.flavor."
+#define PARAM_SRPC_UDESC	"srpc.udesc.cli2mdt"
+#define PARAM_SEC		"security."
+#define PARAM_QUOTA		"quota."	/* global */
+
+/** @} param */
+
+#endif /* _UAPI_LUSTRE_PARAM_H_ */
diff --git a/drivers/staging/lustre/lustre/include/lustre/lustre_user.h b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_user.h
index edff8dc34430..5e332e3af68a 100644
--- a/drivers/staging/lustre/lustre/include/lustre/lustre_user.h
+++ b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_user.h
@@ -43,6 +43,7 @@
  */
 
 #ifdef __KERNEL__
+# include <linux/fs.h>
 # include <linux/quota.h>
 # include <linux/sched/signal.h>
 # include <linux/string.h> /* snprintf() */
@@ -54,7 +55,7 @@
 # include <sys/quota.h>
 # include <sys/stat.h>
 #endif /* __KERNEL__ */
-#include "ll_fiemap.h"
+#include <uapi/linux/lustre/lustre_fiemap.h>
 
 /*
  * We need to always use 64bit version because the structure
@@ -644,7 +645,7 @@ struct if_quotactl {
 #define SWAP_LAYOUTS_CHECK_DV2		(1 << 1)
 #define SWAP_LAYOUTS_KEEP_MTIME		(1 << 2)
 #define SWAP_LAYOUTS_KEEP_ATIME		(1 << 3)
-#define SWAP_LAYOUTS_CLOSE		BIT(4)
+#define SWAP_LAYOUTS_CLOSE		(1 << 4)
 
 /* Swap XATTR_NAME_HSM as well, only on the MDT so far */
 #define SWAP_LAYOUTS_MDS_HSM		(1 << 31)
@@ -791,15 +792,15 @@ static inline void hsm_set_cl_error(int *flags, int error)
 
 enum changelog_send_flag {
 	/* Not yet implemented */
-	CHANGELOG_FLAG_FOLLOW	= BIT(0),
+	CHANGELOG_FLAG_FOLLOW	= 0x01,
 	/*
 	 * Blocking IO makes sense in case of slow user parsing of the records,
 	 * but it also prevents us from cleaning up if the records are not
 	 * consumed.
 	 */
-	CHANGELOG_FLAG_BLOCK	= BIT(1),
+	CHANGELOG_FLAG_BLOCK	= 0x02,
 	/* Pack jobid into the changelog records if available. */
-	CHANGELOG_FLAG_JOBID	= BIT(2),
+	CHANGELOG_FLAG_JOBID	= 0x04,
 };
 
 #define CR_MAXSIZE cfs_size_round(2 * NAME_MAX + 2 + \
@@ -980,8 +981,8 @@ struct ioc_data_version {
 	__u64 idv_flags;     /* See LL_DV_xxx */
 };
 
-#define LL_DV_RD_FLUSH	BIT(0)	/* Flush dirty pages from clients */
-#define LL_DV_WR_FLUSH	BIT(1)	/* Flush all caching pages from clients */
+#define LL_DV_RD_FLUSH	(1 << 0) /* Flush dirty pages from clients */
+#define LL_DV_WR_FLUSH	(1 << 1) /* Flush all caching pages from clients */
 
 #ifndef offsetof
 # define offsetof(typ, memb)     ((unsigned long)((char *)&(((typ *)0)->memb)))
diff --git a/drivers/staging/lustre/lustre/include/lustre_ver.h b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_ver.h
index 19c9135e2273..19c9135e2273 100644
--- a/drivers/staging/lustre/lustre/include/lustre_ver.h
+++ b/drivers/staging/lustre/include/uapi/linux/lustre/lustre_ver.h
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/Makefile b/drivers/staging/lustre/lnet/klnds/o2iblnd/Makefile
index e0a7aa72b7d5..4affe1d79948 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/Makefile
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/Makefile
@@ -1,2 +1,5 @@
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/include
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/lustre/include
+
 obj-$(CONFIG_LNET_XPRT_IB) += ko2iblnd.o
 ko2iblnd-y := o2iblnd.o o2iblnd_cb.o o2iblnd_modparams.o
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
index 0520f02f670d..64763aacda57 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
@@ -2936,7 +2936,7 @@ failed:
 net_failed:
 	kiblnd_shutdown(ni);
 
-	CDEBUG(D_NET, "kiblnd_startup failed\n");
+	CDEBUG(D_NET, "%s failed\n", __func__);
 	return -ENETDOWN;
 }
 
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
index 16e437b3ad1e..a1e994a1cc84 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
@@ -63,9 +63,8 @@
 
 #define DEBUG_SUBSYSTEM S_LND
 
-#include "../../../include/linux/libcfs/libcfs.h"
-#include "../../../include/linux/lnet/lnet.h"
-#include "../../../include/linux/lnet/lib-lnet.h"
+#include <linux/libcfs/libcfs.h>
+#include <linux/lnet/lib-lnet.h>
 
 #define IBLND_PEER_HASH_SIZE		101	/* # peer lists */
 /* # scheduler loops before reschedule */
diff --git a/drivers/staging/lustre/lnet/klnds/socklnd/Makefile b/drivers/staging/lustre/lnet/klnds/socklnd/Makefile
index c011581d3453..a7da1abfc804 100644
--- a/drivers/staging/lustre/lnet/klnds/socklnd/Makefile
+++ b/drivers/staging/lustre/lnet/klnds/socklnd/Makefile
@@ -1,3 +1,6 @@
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/include
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/lustre/include
+
 obj-$(CONFIG_LNET) += ksocklnd.o
 
 ksocklnd-y := socklnd.o socklnd_cb.o socklnd_proto.o socklnd_modparams.o socklnd_lib.o
diff --git a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.h b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.h
index 5540de65f9a2..e6428c4b7aec 100644
--- a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.h
+++ b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.h
@@ -46,10 +46,9 @@
 #include <net/sock.h>
 #include <net/tcp.h>
 
-#include "../../../include/linux/libcfs/libcfs.h"
-#include "../../../include/linux/lnet/lnet.h"
-#include "../../../include/linux/lnet/lib-lnet.h"
-#include "../../../include/linux/lnet/socklnd.h"
+#include <linux/libcfs/libcfs.h>
+#include <linux/lnet/lib-lnet.h>
+#include <linux/lnet/socklnd.h>
 
 /* assume one thread for each connection type */
 #define SOCKNAL_NSCHEDS		3
@@ -519,17 +518,6 @@ extern struct ksock_proto ksocknal_protocol_v3x;
 #define CPU_MASK_NONE   0UL
 #endif
 
-static inline __u32 ksocknal_csum(__u32 crc, unsigned char const *p, size_t len)
-{
-#if 1
-	return crc32_le(crc, p, len);
-#else
-	while (len-- > 0)
-		crc = ((crc + 0x100) & ~0xff) | ((crc + *p++) & 0xff) ;
-	return crc;
-#endif
-}
-
 static inline int
 ksocknal_route_mask(void)
 {
diff --git a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_lib.c b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_lib.c
index 8a036f4eb8d8..9c328dc6537b 100644
--- a/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_lib.c
+++ b/drivers/staging/lustre/lnet/klnds/socklnd/socklnd_lib.c
@@ -201,9 +201,9 @@ ksocknal_lib_recv_iov(struct ksock_conn *conn)
 			if (fragnob > sum)
 				fragnob = sum;
 
-			conn->ksnc_rx_csum = ksocknal_csum(conn->ksnc_rx_csum,
-							   iov[i].iov_base,
-							   fragnob);
+			conn->ksnc_rx_csum = crc32_le(conn->ksnc_rx_csum,
+						      iov[i].iov_base,
+						      fragnob);
 		}
 		conn->ksnc_msg.ksm_csum = saved_csum;
 	}
@@ -243,8 +243,8 @@ ksocknal_lib_recv_kiov(struct ksock_conn *conn)
 			if (fragnob > sum)
 				fragnob = sum;
 
-			conn->ksnc_rx_csum = ksocknal_csum(conn->ksnc_rx_csum,
-							   base, fragnob);
+			conn->ksnc_rx_csum = crc32_le(conn->ksnc_rx_csum,
+						      base, fragnob);
 
 			kunmap(kiov[i].bv_page);
 		}
@@ -265,22 +265,22 @@ ksocknal_lib_csum_tx(struct ksock_tx *tx)
 
 	tx->tx_msg.ksm_csum = 0;
 
-	csum = ksocknal_csum(~0, tx->tx_iov[0].iov_base,
-			     tx->tx_iov[0].iov_len);
+	csum = crc32_le(~0, tx->tx_iov[0].iov_base,
+			tx->tx_iov[0].iov_len);
 
 	if (tx->tx_kiov) {
 		for (i = 0; i < tx->tx_nkiov; i++) {
 			base = kmap(tx->tx_kiov[i].bv_page) +
 			       tx->tx_kiov[i].bv_offset;
 
-			csum = ksocknal_csum(csum, base, tx->tx_kiov[i].bv_len);
+			csum = crc32_le(csum, base, tx->tx_kiov[i].bv_len);
 
 			kunmap(tx->tx_kiov[i].bv_page);
 		}
 	} else {
 		for (i = 1; i < tx->tx_niov; i++)
-			csum = ksocknal_csum(csum, tx->tx_iov[i].iov_base,
-					     tx->tx_iov[i].iov_len);
+			csum = crc32_le(csum, tx->tx_iov[i].iov_base,
+					tx->tx_iov[i].iov_len);
 	}
 
 	if (*ksocknal_tunables.ksnd_inject_csum_error) {
diff --git a/drivers/staging/lustre/lnet/libcfs/Makefile b/drivers/staging/lustre/lnet/libcfs/Makefile
index 8c8945545375..215fa23827d1 100644
--- a/drivers/staging/lustre/lnet/libcfs/Makefile
+++ b/drivers/staging/lustre/lnet/libcfs/Makefile
@@ -1,3 +1,6 @@
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/include
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/lustre/include
+
 obj-$(CONFIG_LNET) += libcfs.o
 
 libcfs-linux-objs := linux-tracefile.o linux-debug.o
diff --git a/drivers/staging/lustre/lnet/libcfs/debug.c b/drivers/staging/lustre/lnet/libcfs/debug.c
index 49deb448b044..1ab394c1fabc 100644
--- a/drivers/staging/lustre/lnet/libcfs/debug.c
+++ b/drivers/staging/lustre/lnet/libcfs/debug.c
@@ -37,7 +37,7 @@
 
 # define DEBUG_SUBSYSTEM S_LNET
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 #include "tracefile.h"
 
 static char debug_file_name[1024];
diff --git a/drivers/staging/lustre/lnet/libcfs/fail.c b/drivers/staging/lustre/lnet/libcfs/fail.c
index 12dd50ad4efb..24f4701a7a1e 100644
--- a/drivers/staging/lustre/lnet/libcfs/fail.c
+++ b/drivers/staging/lustre/lnet/libcfs/fail.c
@@ -29,7 +29,7 @@
  * Lustre is a trademark of Oracle Corporation, Inc.
  */
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
 unsigned long cfs_fail_loc;
 EXPORT_SYMBOL(cfs_fail_loc);
diff --git a/drivers/staging/lustre/lnet/libcfs/hash.c b/drivers/staging/lustre/lnet/libcfs/hash.c
index 5c2ce2ee6fd9..49a04a2b4ec4 100644
--- a/drivers/staging/lustre/lnet/libcfs/hash.c
+++ b/drivers/staging/lustre/lnet/libcfs/hash.c
@@ -105,7 +105,7 @@
 #include <linux/seq_file.h>
 #include <linux/log2.h>
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
 #if CFS_HASH_DEBUG_LEVEL >= CFS_HASH_DEBUG_1
 static unsigned int warn_on_depth = 8;
@@ -1008,7 +1008,7 @@ cfs_hash_create(char *name, unsigned int cur_bits, unsigned int max_bits,
 	LASSERT(ops->hs_object);
 	LASSERT(ops->hs_keycmp);
 	LASSERT(ops->hs_get);
-	LASSERT(ops->hs_put_locked);
+	LASSERT(ops->hs_put || ops->hs_put_locked);
 
 	if (flags & CFS_HASH_REHASH)
 		flags |= CFS_HASH_COUNTER; /* must have counter */
@@ -1553,19 +1553,20 @@ static int
 cfs_hash_for_each_relax(struct cfs_hash *hs, cfs_hash_for_each_cb_t func,
 			void *data, int start)
 {
+	struct hlist_node *next = NULL;
 	struct hlist_node *hnode;
-	struct hlist_node *tmp;
 	struct cfs_hash_bd bd;
 	u32 version;
 	int count = 0;
 	int stop_on_change;
+	int has_put_locked;
 	int end = -1;
 	int rc = 0;
 	int i;
 
 	stop_on_change = cfs_hash_with_rehash_key(hs) ||
-			 !cfs_hash_with_no_itemref(hs) ||
-			 !hs->hs_ops->hs_put_locked;
+			 !cfs_hash_with_no_itemref(hs);
+	has_put_locked = hs->hs_ops->hs_put_locked != NULL;
 	cfs_hash_lock(hs, 0);
 again:
 	LASSERT(!cfs_hash_is_rehashing(hs));
@@ -1582,38 +1583,52 @@ again:
 		version = cfs_hash_bd_version_get(&bd);
 
 		cfs_hash_bd_for_each_hlist(hs, &bd, hhead) {
-			for (hnode = hhead->first; hnode;) {
+			hnode = hhead->first;
+			if (!hnode)
+				continue;
+			cfs_hash_get(hs, hnode);
+
+			for (; hnode; hnode = next) {
 				cfs_hash_bucket_validate(hs, &bd, hnode);
-				cfs_hash_get(hs, hnode);
+				next = hnode->next;
+				if (next)
+					cfs_hash_get(hs, next);
 				cfs_hash_bd_unlock(hs, &bd, 0);
 				cfs_hash_unlock(hs, 0);
 
 				rc = func(hs, &bd, hnode, data);
-				if (stop_on_change)
+				if (stop_on_change || !has_put_locked)
 					cfs_hash_put(hs, hnode);
 				cond_resched();
 				count++;
 
 				cfs_hash_lock(hs, 0);
 				cfs_hash_bd_lock(hs, &bd, 0);
-				if (!stop_on_change) {
-					tmp = hnode->next;
-					cfs_hash_put_locked(hs, hnode);
-					hnode = tmp;
-				} else { /* bucket changed? */
+				if (stop_on_change) {
 					if (version !=
 					    cfs_hash_bd_version_get(&bd))
-						break;
-					/* safe to continue because no change */
-					hnode = hnode->next;
+						rc = -EINTR;
+				} else if (has_put_locked) {
+					cfs_hash_put_locked(hs, hnode);
 				}
 				if (rc) /* callback wants to break iteration */
 					break;
 			}
-			if (rc) /* callback wants to break iteration */
+			if (next) {
+				if (has_put_locked) {
+					cfs_hash_put_locked(hs, next);
+					next = NULL;
+				}
 				break;
+			} else if (rc) {
+				break;
+			}
 		}
 		cfs_hash_bd_unlock(hs, &bd, 0);
+		if (next && !has_put_locked) {
+			cfs_hash_put(hs, next);
+			next = NULL;
+		}
 		if (rc) /* callback wants to break iteration */
 			break;
 	}
diff --git a/drivers/staging/lustre/lnet/libcfs/libcfs_cpu.c b/drivers/staging/lustre/lnet/libcfs/libcfs_cpu.c
index 55caa19def51..2ddd09a83cd0 100644
--- a/drivers/staging/lustre/lnet/libcfs/libcfs_cpu.c
+++ b/drivers/staging/lustre/lnet/libcfs/libcfs_cpu.c
@@ -30,7 +30,7 @@
 
 #define DEBUG_SUBSYSTEM S_LNET
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
 /** Global CPU partition table */
 struct cfs_cpt_table   *cfs_cpt_table __read_mostly;
diff --git a/drivers/staging/lustre/lnet/libcfs/libcfs_lock.c b/drivers/staging/lustre/lnet/libcfs/libcfs_lock.c
index 1967b97c4afc..77fd3d06cde9 100644
--- a/drivers/staging/lustre/lnet/libcfs/libcfs_lock.c
+++ b/drivers/staging/lustre/lnet/libcfs/libcfs_lock.c
@@ -27,7 +27,7 @@
 
 #define DEBUG_SUBSYSTEM S_LNET
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
 /** destroy cpu-partition lock, see libcfs_private.h for more detail */
 void
diff --git a/drivers/staging/lustre/lnet/libcfs/libcfs_mem.c b/drivers/staging/lustre/lnet/libcfs/libcfs_mem.c
index ef085ba23194..1a0c7cad5983 100644
--- a/drivers/staging/lustre/lnet/libcfs/libcfs_mem.c
+++ b/drivers/staging/lustre/lnet/libcfs/libcfs_mem.c
@@ -28,7 +28,7 @@
 
 #define DEBUG_SUBSYSTEM S_LNET
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
 struct cfs_var_array {
 	unsigned int		va_count;	/* # of buffers */
diff --git a/drivers/staging/lustre/lnet/libcfs/libcfs_string.c b/drivers/staging/lustre/lnet/libcfs/libcfs_string.c
index 02de1ee720fd..333e47febf87 100644
--- a/drivers/staging/lustre/lnet/libcfs/libcfs_string.c
+++ b/drivers/staging/lustre/lnet/libcfs/libcfs_string.c
@@ -36,7 +36,7 @@
  * Author: Nathan Rutman <nathan.rutman@sun.com>
  */
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
 /* Convert a text string to a bitmask */
 int cfs_str2mask(const char *str, const char *(*bit2str)(int bit),
diff --git a/drivers/staging/lustre/lnet/libcfs/linux/linux-cpu.c b/drivers/staging/lustre/lnet/libcfs/linux/linux-cpu.c
index 4d35a371216c..2da051c0d251 100644
--- a/drivers/staging/lustre/lnet/libcfs/linux/linux-cpu.c
+++ b/drivers/staging/lustre/lnet/libcfs/linux/linux-cpu.c
@@ -31,7 +31,7 @@
 
 #include <linux/cpu.h>
 #include <linux/sched.h>
-#include "../../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
 #ifdef CONFIG_SMP
 
diff --git a/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto.c b/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto.c
index 68e34b4a76c9..55663390b608 100644
--- a/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto.c
+++ b/drivers/staging/lustre/lnet/libcfs/linux/linux-crypto.c
@@ -29,8 +29,8 @@
 
 #include <crypto/hash.h>
 #include <linux/scatterlist.h>
-#include "../../../include/linux/libcfs/libcfs.h"
-#include "../../../include/linux/libcfs/libcfs_crypto.h"
+#include <linux/libcfs/libcfs.h>
+#include <linux/libcfs/libcfs_crypto.h>
 #include "linux-crypto.h"
 
 /**
diff --git a/drivers/staging/lustre/lnet/libcfs/linux/linux-curproc.c b/drivers/staging/lustre/lnet/libcfs/linux/linux-curproc.c
index 3e22cad18a8b..528d49794881 100644
--- a/drivers/staging/lustre/lnet/libcfs/linux/linux-curproc.c
+++ b/drivers/staging/lustre/lnet/libcfs/linux/linux-curproc.c
@@ -44,7 +44,7 @@
 
 #define DEBUG_SUBSYSTEM S_LNET
 
-#include "../../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
 /*
  * Implementation of cfs_curproc API (see portals/include/libcfs/curproc.h)
diff --git a/drivers/staging/lustre/lnet/libcfs/linux/linux-debug.c b/drivers/staging/lustre/lnet/libcfs/linux/linux-debug.c
index 7035356e56b3..972677bdf6bc 100644
--- a/drivers/staging/lustre/lnet/libcfs/linux/linux-debug.c
+++ b/drivers/staging/lustre/lnet/libcfs/linux/linux-debug.c
@@ -51,7 +51,7 @@
 
 # define DEBUG_SUBSYSTEM S_LNET
 
-#include "../../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
 #include "../tracefile.h"
 
diff --git a/drivers/staging/lustre/lnet/libcfs/linux/linux-mem.c b/drivers/staging/lustre/lnet/libcfs/linux/linux-mem.c
index 8f638267e704..3f5dec153571 100644
--- a/drivers/staging/lustre/lnet/libcfs/linux/linux-mem.c
+++ b/drivers/staging/lustre/lnet/libcfs/linux/linux-mem.c
@@ -29,7 +29,7 @@
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
 
-#include "../../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
 void *libcfs_kvzalloc(size_t size, gfp_t flags)
 {
diff --git a/drivers/staging/lustre/lnet/libcfs/linux/linux-module.c b/drivers/staging/lustre/lnet/libcfs/linux/linux-module.c
index 075826bd3a2a..435722175cce 100644
--- a/drivers/staging/lustre/lnet/libcfs/linux/linux-module.c
+++ b/drivers/staging/lustre/lnet/libcfs/linux/linux-module.c
@@ -32,7 +32,7 @@
 
 #define DEBUG_SUBSYSTEM S_LNET
 
-#include "../../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
 #define LNET_MINOR 240
 
@@ -134,7 +134,7 @@ int libcfs_ioctl_getdata(struct libcfs_ioctl_hdr **hdr_pp,
 		return -EINVAL;
 	}
 
-	if (hdr.ioc_len < sizeof(struct libcfs_ioctl_data)) {
+	if (hdr.ioc_len < sizeof(hdr)) {
 		CERROR("libcfs ioctl: user buffer too small for ioctl\n");
 		return -EINVAL;
 	}
diff --git a/drivers/staging/lustre/lnet/libcfs/linux/linux-prim.c b/drivers/staging/lustre/lnet/libcfs/linux/linux-prim.c
index bcf9f3dd0310..4e331e71083d 100644
--- a/drivers/staging/lustre/lnet/libcfs/linux/linux-prim.c
+++ b/drivers/staging/lustre/lnet/libcfs/linux/linux-prim.c
@@ -36,7 +36,7 @@
 #include <linux/fs_struct.h>
 #include <linux/sched/signal.h>
 
-#include "../../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
 #if defined(CONFIG_KGDB)
 #include <linux/kgdb.h>
diff --git a/drivers/staging/lustre/lnet/libcfs/linux/linux-tracefile.c b/drivers/staging/lustre/lnet/libcfs/linux/linux-tracefile.c
index a5a94788f11f..16a3ae791bb6 100644
--- a/drivers/staging/lustre/lnet/libcfs/linux/linux-tracefile.c
+++ b/drivers/staging/lustre/lnet/libcfs/linux/linux-tracefile.c
@@ -33,7 +33,7 @@
 #define DEBUG_SUBSYSTEM S_LNET
 #define LUSTRE_TRACEFILE_PRIVATE
 
-#include "../../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 #include "../tracefile.h"
 
 /* percents to share the total debug memory for each type */
diff --git a/drivers/staging/lustre/lnet/libcfs/module.c b/drivers/staging/lustre/lnet/libcfs/module.c
index c388550c2d10..6aed98fc9688 100644
--- a/drivers/staging/lustre/lnet/libcfs/module.c
+++ b/drivers/staging/lustre/lnet/libcfs/module.c
@@ -50,13 +50,12 @@
 
 # define DEBUG_SUBSYSTEM S_LNET
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 #include <asm/div64.h>
 
-#include "../../include/linux/libcfs/libcfs_crypto.h"
-#include "../../include/linux/lnet/lib-lnet.h"
-#include "../../include/linux/lnet/lib-dlc.h"
-#include "../../include/linux/lnet/lnet.h"
+#include <linux/libcfs/libcfs_crypto.h>
+#include <linux/lnet/lib-lnet.h>
+#include <uapi/linux/lnet/lnet-dlc.h>
 #include "tracefile.h"
 
 static struct dentry *lnet_debugfs_root;
diff --git a/drivers/staging/lustre/lnet/libcfs/prng.c b/drivers/staging/lustre/lnet/libcfs/prng.c
index 21d5a3912c5f..963ef4ae93b1 100644
--- a/drivers/staging/lustre/lnet/libcfs/prng.c
+++ b/drivers/staging/lustre/lnet/libcfs/prng.c
@@ -35,7 +35,7 @@
  * algorithm recommended by Marsaglia
  */
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
 /*
  * From: George Marsaglia <geo@stat.fsu.edu>
diff --git a/drivers/staging/lustre/lnet/libcfs/tracefile.c b/drivers/staging/lustre/lnet/libcfs/tracefile.c
index d1aa79bb2017..f916b475e767 100644
--- a/drivers/staging/lustre/lnet/libcfs/tracefile.c
+++ b/drivers/staging/lustre/lnet/libcfs/tracefile.c
@@ -40,7 +40,7 @@
 #define pr_fmt(fmt) "Lustre: " fmt
 #include "tracefile.h"
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
 /* XXX move things up to the top, comment */
 union cfs_trace_data_union (*cfs_trace_data[TCD_MAX_TYPES])[NR_CPUS] __cacheline_aligned;
@@ -731,8 +731,7 @@ int cfs_tracefile_dump_all_pages(char *filename)
 		__LASSERT_TAGE_INVARIANT(tage);
 
 		buf = kmap(tage->page);
-		rc = vfs_write(filp, (__force const char __user *)buf,
-			       tage->used, &filp->f_pos);
+		rc = kernel_write(filp, buf, tage->used, &filp->f_pos);
 		kunmap(tage->page);
 
 		if (rc != (int)tage->used) {
@@ -976,7 +975,6 @@ static int tracefiled(void *arg)
 	struct tracefiled_ctl *tctl = arg;
 	struct cfs_trace_page *tage;
 	struct cfs_trace_page *tmp;
-	mm_segment_t __oldfs;
 	struct file *filp;
 	char *buf;
 	int last_loop = 0;
@@ -1014,8 +1012,6 @@ static int tracefiled(void *arg)
 			__LASSERT(list_empty(&pc.pc_pages));
 			goto end_loop;
 		}
-		__oldfs = get_fs();
-		set_fs(get_ds());
 
 		list_for_each_entry_safe(tage, tmp, &pc.pc_pages, linkage) {
 			static loff_t f_pos;
@@ -1028,8 +1024,7 @@ static int tracefiled(void *arg)
 				f_pos = i_size_read(file_inode(filp));
 
 			buf = kmap(tage->page);
-			rc = vfs_write(filp, (__force const char __user *)buf,
-				       tage->used, &f_pos);
+			rc = kernel_write(filp, buf, tage->used, &f_pos);
 			kunmap(tage->page);
 
 			if (rc != (int)tage->used) {
@@ -1040,7 +1035,6 @@ static int tracefiled(void *arg)
 				break;
 			}
 		}
-		set_fs(__oldfs);
 
 		filp_close(filp, NULL);
 		put_pages_on_daemon_list(&pc);
diff --git a/drivers/staging/lustre/lnet/libcfs/tracefile.h b/drivers/staging/lustre/lnet/libcfs/tracefile.h
index f644cbc5a277..c3547cd4c72c 100644
--- a/drivers/staging/lustre/lnet/libcfs/tracefile.h
+++ b/drivers/staging/lustre/lnet/libcfs/tracefile.h
@@ -33,7 +33,7 @@
 #ifndef __LIBCFS_TRACEFILE_H__
 #define __LIBCFS_TRACEFILE_H__
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
 enum cfs_trace_buf_type {
 	CFS_TCD_TYPE_PROC = 0,
diff --git a/drivers/staging/lustre/lnet/libcfs/workitem.c b/drivers/staging/lustre/lnet/libcfs/workitem.c
index dbc2a9b8dff8..038ed8c52107 100644
--- a/drivers/staging/lustre/lnet/libcfs/workitem.c
+++ b/drivers/staging/lustre/lnet/libcfs/workitem.c
@@ -37,7 +37,7 @@
 
 #define DEBUG_SUBSYSTEM S_LNET
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
 #define CFS_WS_NAME_LEN	 16
 
diff --git a/drivers/staging/lustre/lnet/lnet/Makefile b/drivers/staging/lustre/lnet/lnet/Makefile
index 4c81fa19429a..fd8585cd0ce2 100644
--- a/drivers/staging/lustre/lnet/lnet/Makefile
+++ b/drivers/staging/lustre/lnet/lnet/Makefile
@@ -1,3 +1,6 @@
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/include
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/lustre/include
+
 obj-$(CONFIG_LNET) += lnet.o
 
 lnet-y := api-ni.o config.o nidstrings.o net_fault.o		\
diff --git a/drivers/staging/lustre/lnet/lnet/acceptor.c b/drivers/staging/lustre/lnet/lnet/acceptor.c
index a6f60c3e1184..be2823f8eb02 100644
--- a/drivers/staging/lustre/lnet/lnet/acceptor.c
+++ b/drivers/staging/lustre/lnet/lnet/acceptor.c
@@ -33,7 +33,7 @@
 #define DEBUG_SUBSYSTEM S_LNET
 #include <linux/completion.h>
 #include <net/sock.h>
-#include "../../include/linux/lnet/lib-lnet.h"
+#include <linux/lnet/lib-lnet.h>
 
 static int   accept_port    = 988;
 static int   accept_backlog = 127;
diff --git a/drivers/staging/lustre/lnet/lnet/api-ni.c b/drivers/staging/lustre/lnet/lnet/api-ni.c
index 0b91d1809cb1..ad835035fffa 100644
--- a/drivers/staging/lustre/lnet/lnet/api-ni.c
+++ b/drivers/staging/lustre/lnet/lnet/api-ni.c
@@ -34,8 +34,8 @@
 #include <linux/log2.h>
 #include <linux/ktime.h>
 
-#include "../../include/linux/lnet/lib-lnet.h"
-#include "../../include/linux/lnet/lib-dlc.h"
+#include <linux/lnet/lib-lnet.h>
+#include <uapi/linux/lnet/lnet-dlc.h>
 
 #define D_LNI D_CONSOLE
 
diff --git a/drivers/staging/lustre/lnet/lnet/config.c b/drivers/staging/lustre/lnet/lnet/config.c
index 933988da7fe2..26841a7b6213 100644
--- a/drivers/staging/lustre/lnet/lnet/config.c
+++ b/drivers/staging/lustre/lnet/lnet/config.c
@@ -33,7 +33,7 @@
 #define DEBUG_SUBSYSTEM S_LNET
 #include <linux/nsproxy.h>
 #include <net/net_namespace.h>
-#include "../../include/linux/lnet/lib-lnet.h"
+#include <linux/lnet/lib-lnet.h>
 
 struct lnet_text_buf {	    /* tmp struct for parsing routes */
 	struct list_head ltb_list;	/* stash on lists */
diff --git a/drivers/staging/lustre/lnet/lnet/lib-eq.c b/drivers/staging/lustre/lnet/lnet/lib-eq.c
index 9ebba4ef5f90..6b446a51eeac 100644
--- a/drivers/staging/lustre/lnet/lnet/lib-eq.c
+++ b/drivers/staging/lustre/lnet/lnet/lib-eq.c
@@ -35,7 +35,8 @@
  */
 
 #define DEBUG_SUBSYSTEM S_LNET
-#include "../../include/linux/lnet/lib-lnet.h"
+
+#include <linux/lnet/lib-lnet.h>
 
 /**
  * Create an event queue that has room for \a count number of events.
diff --git a/drivers/staging/lustre/lnet/lnet/lib-md.c b/drivers/staging/lustre/lnet/lnet/lib-md.c
index f08e944f412b..a0aef4b9bce3 100644
--- a/drivers/staging/lustre/lnet/lnet/lib-md.c
+++ b/drivers/staging/lustre/lnet/lnet/lib-md.c
@@ -36,7 +36,7 @@
 
 #define DEBUG_SUBSYSTEM S_LNET
 
-#include "../../include/linux/lnet/lib-lnet.h"
+#include <linux/lnet/lib-lnet.h>
 
 /* must be called with lnet_res_lock held */
 void
diff --git a/drivers/staging/lustre/lnet/lnet/lib-me.c b/drivers/staging/lustre/lnet/lnet/lib-me.c
index e9b3eedca4db..f52a5e8ed386 100644
--- a/drivers/staging/lustre/lnet/lnet/lib-me.c
+++ b/drivers/staging/lustre/lnet/lnet/lib-me.c
@@ -36,7 +36,7 @@
 
 #define DEBUG_SUBSYSTEM S_LNET
 
-#include "../../include/linux/lnet/lib-lnet.h"
+#include <linux/lnet/lib-lnet.h>
 
 /**
  * Create and attach a match entry to the match list of \a portal. The new
diff --git a/drivers/staging/lustre/lnet/lnet/lib-move.c b/drivers/staging/lustre/lnet/lnet/lib-move.c
index 20ebe247071f..bc0779c02d97 100644
--- a/drivers/staging/lustre/lnet/lnet/lib-move.c
+++ b/drivers/staging/lustre/lnet/lnet/lib-move.c
@@ -36,7 +36,7 @@
 
 #define DEBUG_SUBSYSTEM S_LNET
 
-#include "../../include/linux/lnet/lib-lnet.h"
+#include <linux/lnet/lib-lnet.h>
 #include <linux/nsproxy.h>
 #include <net/net_namespace.h>
 
@@ -2034,7 +2034,7 @@ LNetPut(lnet_nid_t self, struct lnet_handle_md mdh, enum lnet_ack_req ack,
 		return -ENOENT;
 	}
 
-	CDEBUG(D_NET, "LNetPut -> %s\n", libcfs_id2str(target));
+	CDEBUG(D_NET, "%s -> %s\n", __func__, libcfs_id2str(target));
 
 	lnet_msg_attach_md(msg, md, 0, 0);
 
@@ -2239,7 +2239,7 @@ LNetGet(lnet_nid_t self, struct lnet_handle_md mdh,
 		return -ENOENT;
 	}
 
-	CDEBUG(D_NET, "LNetGet -> %s\n", libcfs_id2str(target));
+	CDEBUG(D_NET, "%s -> %s\n", __func__, libcfs_id2str(target));
 
 	lnet_msg_attach_md(msg, md, 0, 0);
 
diff --git a/drivers/staging/lustre/lnet/lnet/lib-msg.c b/drivers/staging/lustre/lnet/lnet/lib-msg.c
index 008ac503f27d..d04875e3956f 100644
--- a/drivers/staging/lustre/lnet/lnet/lib-msg.c
+++ b/drivers/staging/lustre/lnet/lnet/lib-msg.c
@@ -36,7 +36,7 @@
 
 #define DEBUG_SUBSYSTEM S_LNET
 
-#include "../../include/linux/lnet/lib-lnet.h"
+#include <linux/lnet/lib-lnet.h>
 
 void
 lnet_build_unlink_event(struct lnet_libmd *md, struct lnet_event *ev)
diff --git a/drivers/staging/lustre/lnet/lnet/lib-ptl.c b/drivers/staging/lustre/lnet/lnet/lib-ptl.c
index 33332724ab94..5946848a7846 100644
--- a/drivers/staging/lustre/lnet/lnet/lib-ptl.c
+++ b/drivers/staging/lustre/lnet/lnet/lib-ptl.c
@@ -31,7 +31,7 @@
 
 #define DEBUG_SUBSYSTEM S_LNET
 
-#include "../../include/linux/lnet/lib-lnet.h"
+#include <linux/lnet/lib-lnet.h>
 
 /* NB: add /proc interfaces in upcoming patches */
 int portal_rotor = LNET_PTL_ROTOR_HASH_RT;
diff --git a/drivers/staging/lustre/lnet/lnet/lib-socket.c b/drivers/staging/lustre/lnet/lnet/lib-socket.c
index 800f4f6c6593..7d0add0c0de3 100644
--- a/drivers/staging/lustre/lnet/lnet/lib-socket.c
+++ b/drivers/staging/lustre/lnet/lnet/lib-socket.c
@@ -40,8 +40,8 @@
 #include <linux/syscalls.h>
 #include <net/sock.h>
 
-#include "../../include/linux/libcfs/libcfs.h"
-#include "../../include/linux/lnet/lib-lnet.h"
+#include <linux/libcfs/libcfs.h>
+#include <linux/lnet/lib-lnet.h>
 
 static int
 kernel_sock_unlocked_ioctl(struct file *filp, int cmd, unsigned long arg)
diff --git a/drivers/staging/lustre/lnet/lnet/lo.c b/drivers/staging/lustre/lnet/lnet/lo.c
index a7504b8edf6d..80c06f4b0c8d 100644
--- a/drivers/staging/lustre/lnet/lnet/lo.c
+++ b/drivers/staging/lustre/lnet/lnet/lo.c
@@ -29,7 +29,8 @@
  */
 
 #define DEBUG_SUBSYSTEM S_LNET
-#include "../../include/linux/lnet/lib-lnet.h"
+
+#include <linux/lnet/lib-lnet.h>
 
 static int
 lolnd_send(struct lnet_ni *ni, void *private, struct lnet_msg *lntmsg)
diff --git a/drivers/staging/lustre/lnet/lnet/module.c b/drivers/staging/lustre/lnet/lnet/module.c
index 4ffbd3e441e8..7d12a7fb36a4 100644
--- a/drivers/staging/lustre/lnet/lnet/module.c
+++ b/drivers/staging/lustre/lnet/lnet/module.c
@@ -31,8 +31,9 @@
  */
 
 #define DEBUG_SUBSYSTEM S_LNET
-#include "../../include/linux/lnet/lib-lnet.h"
-#include "../../include/linux/lnet/lib-dlc.h"
+
+#include <linux/lnet/lib-lnet.h>
+#include <uapi/linux/lnet/lnet-dlc.h>
 
 static int config_on_load;
 module_param(config_on_load, int, 0444);
diff --git a/drivers/staging/lustre/lnet/lnet/net_fault.c b/drivers/staging/lustre/lnet/lnet/net_fault.c
index 18183cbb9859..03f3d18a1a29 100644
--- a/drivers/staging/lustre/lnet/lnet/net_fault.c
+++ b/drivers/staging/lustre/lnet/lnet/net_fault.c
@@ -35,8 +35,8 @@
 
 #define DEBUG_SUBSYSTEM S_LNET
 
-#include "../../include/linux/lnet/lib-lnet.h"
-#include "../../include/linux/lnet/lnetctl.h"
+#include <linux/lnet/lib-lnet.h>
+#include <uapi/linux/lnet/lnetctl.h>
 
 #define LNET_MSG_MASK		(LNET_PUT_BIT | LNET_ACK_BIT | \
 				 LNET_GET_BIT | LNET_REPLY_BIT)
diff --git a/drivers/staging/lustre/lnet/lnet/nidstrings.c b/drivers/staging/lustre/lnet/lnet/nidstrings.c
index 298533d7b04c..7bd1e6f389aa 100644
--- a/drivers/staging/lustre/lnet/lnet/nidstrings.c
+++ b/drivers/staging/lustre/lnet/lnet/nidstrings.c
@@ -36,8 +36,8 @@
 
 #define DEBUG_SUBSYSTEM S_LNET
 
-#include "../../include/linux/libcfs/libcfs.h"
-#include "../../include/linux/lnet/lnet.h"
+#include <linux/libcfs/libcfs.h>
+#include <uapi/linux/lnet/nidstr.h>
 
 /* max value for numeric network address */
 #define MAX_NUMERIC_VALUE 0xffffffff
diff --git a/drivers/staging/lustre/lnet/lnet/peer.c b/drivers/staging/lustre/lnet/lnet/peer.c
index e62b21f3ab4d..4d55df8ff74e 100644
--- a/drivers/staging/lustre/lnet/lnet/peer.c
+++ b/drivers/staging/lustre/lnet/lnet/peer.c
@@ -34,8 +34,8 @@
 
 #define DEBUG_SUBSYSTEM S_LNET
 
-#include "../../include/linux/lnet/lib-lnet.h"
-#include "../../include/linux/lnet/lib-dlc.h"
+#include <linux/lnet/lib-lnet.h>
+#include <uapi/linux/lnet/lnet-dlc.h>
 
 int
 lnet_peer_tables_create(void)
diff --git a/drivers/staging/lustre/lnet/lnet/router.c b/drivers/staging/lustre/lnet/lnet/router.c
index 12dd1043f9fa..3df101bafd9f 100644
--- a/drivers/staging/lustre/lnet/lnet/router.c
+++ b/drivers/staging/lustre/lnet/lnet/router.c
@@ -18,8 +18,9 @@
  */
 
 #define DEBUG_SUBSYSTEM S_LNET
+
 #include <linux/completion.h>
-#include "../../include/linux/lnet/lib-lnet.h"
+#include <linux/lnet/lib-lnet.h>
 
 #define LNET_NRB_TINY_MIN	512	/* min value for each CPT */
 #define LNET_NRB_TINY		(LNET_NRB_TINY_MIN * 4)
diff --git a/drivers/staging/lustre/lnet/lnet/router_proc.c b/drivers/staging/lustre/lnet/lnet/router_proc.c
index 72b80c594108..4a994d113c7d 100644
--- a/drivers/staging/lustre/lnet/lnet/router_proc.c
+++ b/drivers/staging/lustre/lnet/lnet/router_proc.c
@@ -18,8 +18,9 @@
  */
 
 #define DEBUG_SUBSYSTEM S_LNET
-#include "../../include/linux/libcfs/libcfs.h"
-#include "../../include/linux/lnet/lib-lnet.h"
+
+#include <linux/libcfs/libcfs.h>
+#include <linux/lnet/lib-lnet.h>
 
 /*
  * This is really lnet_proc.c. You might need to update sanity test 215
diff --git a/drivers/staging/lustre/lnet/selftest/Makefile b/drivers/staging/lustre/lnet/selftest/Makefile
index c0de6e2d96d0..3ccc8966b566 100644
--- a/drivers/staging/lustre/lnet/selftest/Makefile
+++ b/drivers/staging/lustre/lnet/selftest/Makefile
@@ -1,3 +1,6 @@
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/include
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/lustre/include
+
 obj-$(CONFIG_LNET_SELFTEST) := lnet_selftest.o
 
 lnet_selftest-y := console.o conrpc.o conctl.o framework.o timer.o rpc.o \
diff --git a/drivers/staging/lustre/lnet/selftest/conctl.c b/drivers/staging/lustre/lnet/selftest/conctl.c
index 6ca7192b03b7..9619ecbf8bdf 100644
--- a/drivers/staging/lustre/lnet/selftest/conctl.c
+++ b/drivers/staging/lustre/lnet/selftest/conctl.c
@@ -36,9 +36,9 @@
  * Author: Liang Zhen <liangzhen@clusterfs.com>
  */
 
-#include "../../include/linux/libcfs/libcfs.h"
-#include "../../include/linux/lnet/lib-lnet.h"
-#include "../../include/linux/lnet/lnetst.h"
+#include <linux/libcfs/libcfs.h>
+#include <linux/lnet/lib-lnet.h>
+#include <uapi/linux/lnet/lnetst.h>
 #include "console.h"
 
 static int
@@ -69,8 +69,8 @@ lst_session_new_ioctl(struct lstio_session_new_args *args)
 	rc = lstcon_session_new(name,
 				args->lstio_ses_key,
 				args->lstio_ses_feats,
-				args->lstio_ses_force,
 				args->lstio_ses_timeout,
+				args->lstio_ses_force,
 				args->lstio_ses_idp);
 
 	LIBCFS_FREE(name, args->lstio_ses_nmlen + 1);
diff --git a/drivers/staging/lustre/lnet/selftest/conrpc.c b/drivers/staging/lustre/lnet/selftest/conrpc.c
index da36c55b86d3..196d23c10921 100644
--- a/drivers/staging/lustre/lnet/selftest/conrpc.c
+++ b/drivers/staging/lustre/lnet/selftest/conrpc.c
@@ -36,8 +36,8 @@
  * Author: Liang Zhen <liang@whamcloud.com>
  */
 
-#include "../../include/linux/libcfs/libcfs.h"
-#include "../../include/linux/lnet/lib-lnet.h"
+#include <linux/libcfs/libcfs.h>
+#include <linux/lnet/lib-lnet.h>
 #include "timer.h"
 #include "conrpc.h"
 #include "console.h"
@@ -487,10 +487,9 @@ lstcon_rpc_trans_interpreter(struct lstcon_rpc_trans *trans,
 				   sizeof(struct list_head)))
 			return -EFAULT;
 
-		if (tmp.next == head_up)
-			return 0;
-
 		next = tmp.next;
+		if (next == head_up)
+			return 0;
 
 		ent = list_entry(next, struct lstcon_rpc_ent, rpe_link);
 
diff --git a/drivers/staging/lustre/lnet/selftest/conrpc.h b/drivers/staging/lustre/lnet/selftest/conrpc.h
index 7141d2c902a5..239323679baa 100644
--- a/drivers/staging/lustre/lnet/selftest/conrpc.h
+++ b/drivers/staging/lustre/lnet/selftest/conrpc.h
@@ -39,10 +39,9 @@
 #ifndef __LST_CONRPC_H__
 #define __LST_CONRPC_H__
 
-#include "../../include/linux/libcfs/libcfs.h"
-#include "../../include/linux/lnet/lnet.h"
-#include "../../include/linux/lnet/lib-types.h"
-#include "../../include/linux/lnet/lnetst.h"
+#include <linux/libcfs/libcfs.h>
+#include <linux/lnet/lib-types.h>
+#include <uapi/linux/lnet/lnetst.h>
 #include "rpc.h"
 #include "selftest.h"
 
diff --git a/drivers/staging/lustre/lnet/selftest/console.c b/drivers/staging/lustre/lnet/selftest/console.c
index d62c448ecf8a..289b202c3b36 100644
--- a/drivers/staging/lustre/lnet/selftest/console.c
+++ b/drivers/staging/lustre/lnet/selftest/console.c
@@ -36,8 +36,8 @@
  * Author: Liang Zhen <liangzhen@clusterfs.com>
  */
 
-#include "../../include/linux/libcfs/libcfs.h"
-#include "../../include/linux/lnet/lib-lnet.h"
+#include <linux/libcfs/libcfs.h>
+#include <linux/lnet/lib-lnet.h>
 #include "console.h"
 #include "conrpc.h"
 
diff --git a/drivers/staging/lustre/lnet/selftest/console.h b/drivers/staging/lustre/lnet/selftest/console.h
index e3e11aa52526..143eae9b8d71 100644
--- a/drivers/staging/lustre/lnet/selftest/console.h
+++ b/drivers/staging/lustre/lnet/selftest/console.h
@@ -39,10 +39,9 @@
 #ifndef __LST_CONSOLE_H__
 #define __LST_CONSOLE_H__
 
-#include "../../include/linux/libcfs/libcfs.h"
-#include "../../include/linux/lnet/lnet.h"
-#include "../../include/linux/lnet/lib-types.h"
-#include "../../include/linux/lnet/lnetst.h"
+#include <linux/libcfs/libcfs.h>
+#include <linux/lnet/lib-types.h>
+#include <uapi/linux/lnet/lnetst.h>
 #include "selftest.h"
 #include "conrpc.h"
 
diff --git a/drivers/staging/lustre/lnet/selftest/rpc.h b/drivers/staging/lustre/lnet/selftest/rpc.h
index a765537a79c4..7bb442a8e698 100644
--- a/drivers/staging/lustre/lnet/selftest/rpc.h
+++ b/drivers/staging/lustre/lnet/selftest/rpc.h
@@ -33,7 +33,7 @@
 #ifndef __SELFTEST_RPC_H__
 #define __SELFTEST_RPC_H__
 
-#include "../../include/linux/lnet/lnetst.h"
+#include <uapi/linux/lnet/lnetst.h>
 
 /*
  * LST wired structures
diff --git a/drivers/staging/lustre/lnet/selftest/selftest.h b/drivers/staging/lustre/lnet/selftest/selftest.h
index b614e6f23a70..7adad4302dcf 100644
--- a/drivers/staging/lustre/lnet/selftest/selftest.h
+++ b/drivers/staging/lustre/lnet/selftest/selftest.h
@@ -38,11 +38,10 @@
 
 #define LNET_ONLY
 
-#include "../../include/linux/libcfs/libcfs.h"
-#include "../../include/linux/lnet/lnet.h"
-#include "../../include/linux/lnet/lib-lnet.h"
-#include "../../include/linux/lnet/lib-types.h"
-#include "../../include/linux/lnet/lnetst.h"
+#include <linux/libcfs/libcfs.h>
+#include <linux/lnet/lib-lnet.h>
+#include <linux/lnet/lib-types.h>
+#include <uapi/linux/lnet/lnetst.h>
 
 #include "rpc.h"
 #include "timer.h"
diff --git a/drivers/staging/lustre/lustre/Kconfig b/drivers/staging/lustre/lustre/Kconfig
index 9f5d75f166e7..90d826946c6a 100644
--- a/drivers/staging/lustre/lustre/Kconfig
+++ b/drivers/staging/lustre/lustre/Kconfig
@@ -31,16 +31,6 @@ config LUSTRE_FS
 
 	  See also http://wiki.lustre.org/
 
-config LUSTRE_OBD_MAX_IOCTL_BUFFER
-	int "Lustre obd max ioctl buffer bytes (default 8KB)"
-	depends on LUSTRE_FS
-	default 8192
-	help
-	  This option defines the maximum size of buffer in bytes that user space
-	  applications can pass to Lustre kernel module through ioctl interface.
-
-	  If unsure, use default.
-
 config LUSTRE_DEBUG_EXPENSIVE_CHECK
 	bool "Enable Lustre DEBUG checks"
 	depends on LUSTRE_FS
diff --git a/drivers/staging/lustre/lustre/fid/Makefile b/drivers/staging/lustre/lustre/fid/Makefile
index b7ef314b4b84..77b65b92667d 100644
--- a/drivers/staging/lustre/lustre/fid/Makefile
+++ b/drivers/staging/lustre/lustre/fid/Makefile
@@ -1,2 +1,5 @@
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/include
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/lustre/include/
+
 obj-$(CONFIG_LUSTRE_FS) += fid.o
 fid-y := fid_request.o fid_lib.o lproc_fid.o
diff --git a/drivers/staging/lustre/lustre/fid/fid_internal.h b/drivers/staging/lustre/lustre/fid/fid_internal.h
index 5c53773ecc5a..f48ab9d21428 100644
--- a/drivers/staging/lustre/lustre/fid/fid_internal.h
+++ b/drivers/staging/lustre/lustre/fid/fid_internal.h
@@ -36,8 +36,8 @@
 #ifndef __FID_INTERNAL_H
 #define __FID_INTERNAL_H
 
-#include "../include/lustre/lustre_idl.h"
-#include "../../include/linux/libcfs/libcfs.h"
+#include <uapi/linux/lustre/lustre_idl.h>
+#include <linux/libcfs/libcfs.h>
 
 /* Functions used internally in module. */
 
diff --git a/drivers/staging/lustre/lustre/fid/fid_lib.c b/drivers/staging/lustre/lustre/fid/fid_lib.c
index 9eb405905d1a..c21a5f5b7621 100644
--- a/drivers/staging/lustre/lustre/fid/fid_lib.c
+++ b/drivers/staging/lustre/lustre/fid/fid_lib.c
@@ -39,10 +39,9 @@
 
 #define DEBUG_SUBSYSTEM S_FID
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 #include <linux/module.h>
-#include "../include/lustre/lustre_idl.h"
-#include "../include/lustre_fid.h"
+#include <lustre_fid.h>
 
 /**
  * A cluster-wide range from which fid-sequences are granted to servers and
diff --git a/drivers/staging/lustre/lustre/fid/fid_request.c b/drivers/staging/lustre/lustre/fid/fid_request.c
index 19895faf7626..ba736239243c 100644
--- a/drivers/staging/lustre/lustre/fid/fid_request.c
+++ b/drivers/staging/lustre/lustre/fid/fid_request.c
@@ -38,15 +38,15 @@
 
 #define DEBUG_SUBSYSTEM S_FID
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 #include <linux/module.h>
 
-#include "../include/obd.h"
-#include "../include/obd_class.h"
-#include "../include/obd_support.h"
-#include "../include/lustre_fid.h"
+#include <obd.h>
+#include <obd_class.h>
+#include <obd_support.h>
+#include <lustre_fid.h>
 /* mdc RPC locks */
-#include "../include/lustre_mdc.h"
+#include <lustre_mdc.h>
 #include "fid_internal.h"
 
 static struct dentry *seq_debugfs_dir;
diff --git a/drivers/staging/lustre/lustre/fid/lproc_fid.c b/drivers/staging/lustre/lustre/fid/lproc_fid.c
index 3eed83808545..1a269fbc4b47 100644
--- a/drivers/staging/lustre/lustre/fid/lproc_fid.c
+++ b/drivers/staging/lustre/lustre/fid/lproc_fid.c
@@ -38,14 +38,14 @@
 
 #define DEBUG_SUBSYSTEM S_FID
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 #include <linux/module.h>
 
-#include "../include/obd.h"
-#include "../include/obd_class.h"
-#include "../include/obd_support.h"
-#include "../include/lustre_req_layout.h"
-#include "../include/lustre_fid.h"
+#include <obd.h>
+#include <obd_class.h>
+#include <obd_support.h>
+#include <lustre_req_layout.h>
+#include <lustre_fid.h>
 #include "fid_internal.h"
 
 /* Format: [0x64BIT_INT - 0x64BIT_INT] + 32 bytes just in case */
diff --git a/drivers/staging/lustre/lustre/fld/Makefile b/drivers/staging/lustre/lustre/fld/Makefile
index 646e315d1aa8..426deba8b815 100644
--- a/drivers/staging/lustre/lustre/fld/Makefile
+++ b/drivers/staging/lustre/lustre/fld/Makefile
@@ -1,2 +1,5 @@
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/include
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/lustre/include/
+
 obj-$(CONFIG_LUSTRE_FS) += fld.o
 fld-y := fld_request.o fld_cache.o lproc_fld.o
diff --git a/drivers/staging/lustre/lustre/fld/fld_cache.c b/drivers/staging/lustre/lustre/fld/fld_cache.c
index b852fed0b10f..b723ece02eff 100644
--- a/drivers/staging/lustre/lustre/fld/fld_cache.c
+++ b/drivers/staging/lustre/lustre/fld/fld_cache.c
@@ -39,18 +39,18 @@
 
 #define DEBUG_SUBSYSTEM S_FLD
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 #include <linux/module.h>
 #include <asm/div64.h>
 
-#include "../include/obd.h"
-#include "../include/obd_class.h"
-#include "../include/lustre_ver.h"
-#include "../include/obd_support.h"
-#include "../include/lprocfs_status.h"
+#include <obd.h>
+#include <obd_class.h>
+#include <uapi/linux/lustre/lustre_ver.h>
+#include <obd_support.h>
+#include <lprocfs_status.h>
 
-#include "../include/lustre_req_layout.h"
-#include "../include/lustre_fld.h"
+#include <lustre_req_layout.h>
+#include <lustre_fld.h>
 #include "fld_internal.h"
 
 /**
@@ -348,9 +348,10 @@ static void fld_cache_overlap_handle(struct fld_cache *cache,
 
 		f_curr->fce_range.lsr_end = new_start;
 		fld_cache_entry_add(cache, f_new, &f_curr->fce_list);
-	} else
+	} else {
 		CERROR("NEW range =" DRANGE " curr = " DRANGE "\n",
 		       PRANGE(range), PRANGE(&f_curr->fce_range));
+	}
 }
 
 struct fld_cache_entry
diff --git a/drivers/staging/lustre/lustre/fld/fld_internal.h b/drivers/staging/lustre/lustre/fld/fld_internal.h
index 4a7f0b71c48d..fe6f278a7d9f 100644
--- a/drivers/staging/lustre/lustre/fld/fld_internal.h
+++ b/drivers/staging/lustre/lustre/fld/fld_internal.h
@@ -56,11 +56,11 @@
 #ifndef __FLD_INTERNAL_H
 #define __FLD_INTERNAL_H
 
-#include "../include/lustre/lustre_idl.h"
+#include <uapi/linux/lustre/lustre_idl.h>
 
-#include "../../include/linux/libcfs/libcfs.h"
-#include "../include/lustre_req_layout.h"
-#include "../include/lustre_fld.h"
+#include <linux/libcfs/libcfs.h>
+#include <lustre_req_layout.h>
+#include <lustre_fld.h>
 
 struct fld_stats {
 	__u64   fst_count;
diff --git a/drivers/staging/lustre/lustre/fld/fld_request.c b/drivers/staging/lustre/lustre/fld/fld_request.c
index 4cade7a16800..5b180830eec0 100644
--- a/drivers/staging/lustre/lustre/fld/fld_request.c
+++ b/drivers/staging/lustre/lustre/fld/fld_request.c
@@ -38,19 +38,19 @@
 
 #define DEBUG_SUBSYSTEM S_FLD
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 #include <linux/module.h>
 #include <asm/div64.h>
 
-#include "../include/obd.h"
-#include "../include/obd_class.h"
-#include "../include/lustre_ver.h"
-#include "../include/obd_support.h"
-#include "../include/lprocfs_status.h"
+#include <obd.h>
+#include <obd_class.h>
+#include <uapi/linux/lustre/lustre_ver.h>
+#include <obd_support.h>
+#include <lprocfs_status.h>
 
-#include "../include/lustre_req_layout.h"
-#include "../include/lustre_fld.h"
-#include "../include/lustre_mdc.h"
+#include <lustre_req_layout.h>
+#include <lustre_fld.h>
+#include <lustre_mdc.h>
 #include "fld_internal.h"
 
 static int fld_rrb_hash(struct lu_client_fld *fld, u64 seq)
diff --git a/drivers/staging/lustre/lustre/fld/lproc_fld.c b/drivers/staging/lustre/lustre/fld/lproc_fld.c
index b83d7ebb2d18..6cae803fc8d2 100644
--- a/drivers/staging/lustre/lustre/fld/lproc_fld.c
+++ b/drivers/staging/lustre/lustre/fld/lproc_fld.c
@@ -39,15 +39,15 @@
 
 #define DEBUG_SUBSYSTEM S_FLD
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 #include <linux/module.h>
 
-#include "../include/obd.h"
-#include "../include/obd_class.h"
-#include "../include/obd_support.h"
-#include "../include/lustre_req_layout.h"
-#include "../include/lustre_fld.h"
-#include "../include/lustre_fid.h"
+#include <obd.h>
+#include <obd_class.h>
+#include <obd_support.h>
+#include <lustre_req_layout.h>
+#include <lustre_fld.h>
+#include <lustre_fid.h>
 #include "fld_internal.h"
 
 static int
diff --git a/drivers/staging/lustre/lustre/include/cl_object.h b/drivers/staging/lustre/lustre/include/cl_object.h
index 90a0c501e1ea..9ba184b6017f 100644
--- a/drivers/staging/lustre/lustre/include/cl_object.h
+++ b/drivers/staging/lustre/lustre/include/cl_object.h
@@ -88,8 +88,8 @@
 /*
  * super-class definitions.
  */
-#include "lu_object.h"
-#include "lustre_compat.h"
+#include <lu_object.h>
+#include <lustre_compat.h>
 #include <linux/atomic.h>
 #include <linux/mutex.h>
 #include <linux/radix-tree.h>
@@ -1358,7 +1358,7 @@ struct cl_2queue {
 /** IO types */
 enum cl_io_type {
 	/** read system call */
-	CIT_READ,
+	CIT_READ = 1,
 	/** write system call */
 	CIT_WRITE,
 	/** truncate, utime system calls */
diff --git a/drivers/staging/lustre/lustre/include/interval_tree.h b/drivers/staging/lustre/lustre/include/interval_tree.h
index 0d4f92ec8334..a4d7280e1fa4 100644
--- a/drivers/staging/lustre/lustre/include/interval_tree.h
+++ b/drivers/staging/lustre/lustre/include/interval_tree.h
@@ -111,4 +111,8 @@ enum interval_iter interval_search(struct interval_node *root,
 				   struct interval_node_extent *ex,
 				   interval_callback_t func, void *data);
 
+enum interval_iter interval_iterate_reverse(struct interval_node *root,
+					    interval_callback_t func,
+					    void *data);
+
 #endif
diff --git a/drivers/staging/lustre/lustre/include/llog_swab.h b/drivers/staging/lustre/lustre/include/llog_swab.h
index fd7ffb154ad1..925271db4554 100644
--- a/drivers/staging/lustre/lustre/include/llog_swab.h
+++ b/drivers/staging/lustre/lustre/include/llog_swab.h
@@ -48,7 +48,8 @@
 #ifndef _LLOG_SWAB_H_
 #define _LLOG_SWAB_H_
 
-#include "lustre/lustre_idl.h"
+#include <uapi/linux/lustre/lustre_idl.h>
+
 struct lustre_cfg;
 
 void lustre_swab_lu_fid(struct lu_fid *fid);
diff --git a/drivers/staging/lustre/lustre/include/lprocfs_status.h b/drivers/staging/lustre/lustre/include/lprocfs_status.h
index 915283c04094..98d6b1364c21 100644
--- a/drivers/staging/lustre/lustre/include/lprocfs_status.h
+++ b/drivers/staging/lustre/lustre/include/lprocfs_status.h
@@ -43,9 +43,9 @@
 #include <linux/spinlock.h>
 #include <linux/types.h>
 
-#include "../../include/linux/libcfs/libcfs.h"
-#include "lustre_cfg.h"
-#include "lustre/lustre_idl.h"
+#include <linux/libcfs/libcfs.h>
+#include <uapi/linux/lustre/lustre_cfg.h>
+#include <uapi/linux/lustre/lustre_idl.h>
 
 struct lprocfs_vars {
 	const char		*name;
@@ -59,7 +59,7 @@ struct lprocfs_vars {
 
 struct lprocfs_static_vars {
 	struct lprocfs_vars *obd_vars;
-	struct attribute_group *sysfs_vars;
+	const struct attribute_group *sysfs_vars;
 };
 
 /* if we find more consumers this could be generalized */
@@ -468,7 +468,7 @@ struct dentry *ldebugfs_register(const char *name,
 void ldebugfs_remove(struct dentry **entryp);
 
 int lprocfs_obd_setup(struct obd_device *obd, struct lprocfs_vars *list,
-		      struct attribute_group *attrs);
+		      const struct attribute_group *attrs);
 int lprocfs_obd_cleanup(struct obd_device *obd);
 
 int ldebugfs_seq_create(struct dentry *parent,
diff --git a/drivers/staging/lustre/lustre/include/lu_object.h b/drivers/staging/lustre/lustre/include/lu_object.h
index 2e70602dc2e2..4f213c408cfa 100644
--- a/drivers/staging/lustre/lustre/include/lu_object.h
+++ b/drivers/staging/lustre/lustre/include/lu_object.h
@@ -35,9 +35,9 @@
 
 #include <stdarg.h>
 #include <linux/percpu_counter.h>
-#include "../../include/linux/libcfs/libcfs.h"
-#include "lustre/lustre_idl.h"
-#include "lu_ref.h"
+#include <linux/libcfs/libcfs.h>
+#include <uapi/linux/lustre/lustre_idl.h>
+#include <lu_ref.h>
 
 struct seq_file;
 struct lustre_cfg;
diff --git a/drivers/staging/lustre/lustre/include/lustre_compat.h b/drivers/staging/lustre/lustre/include/lustre_compat.h
index da9ce195c52e..69bfd6a6e0f9 100644
--- a/drivers/staging/lustre/lustre/include/lustre_compat.h
+++ b/drivers/staging/lustre/lustre/include/lustre_compat.h
@@ -37,7 +37,7 @@
 #include <linux/namei.h>
 #include <linux/cred.h>
 
-#include "lustre_patchless_compat.h"
+#include <lustre_patchless_compat.h>
 
 /*
  * set ATTR_BLOCKS to a high value to avoid any risk of collision with other
diff --git a/drivers/staging/lustre/lustre/include/lustre_debug.h b/drivers/staging/lustre/lustre/include/lustre_debug.h
index 93c1bdaf71a4..0be6a534f712 100644
--- a/drivers/staging/lustre/lustre/include/lustre_debug.h
+++ b/drivers/staging/lustre/lustre/include/lustre_debug.h
@@ -38,8 +38,8 @@
  * @{
  */
 
-#include "lustre_net.h"
-#include "obd.h"
+#include <lustre_net.h>
+#include <obd.h>
 
 /* lib/debug.c */
 int dump_req(struct ptlrpc_request *req);
diff --git a/drivers/staging/lustre/lustre/include/lustre_disk.h b/drivers/staging/lustre/lustre/include/lustre_disk.h
index a676bccabd43..2d862b32265b 100644
--- a/drivers/staging/lustre/lustre/include/lustre_disk.h
+++ b/drivers/staging/lustre/lustre/include/lustre_disk.h
@@ -44,9 +44,10 @@
  * @{
  */
 
-#include "../../include/linux/libcfs/libcfs.h"
-#include "../../include/linux/lnet/types.h"
+#include <asm/byteorder.h>
+#include <linux/types.h>
 #include <linux/backing-dev.h>
+#include <linux/libcfs/libcfs.h>
 
 /****************** persistent mount data *********************/
 
@@ -108,14 +109,6 @@ struct lustre_mount_data {
 
 #define lmd_is_client(x) ((x)->lmd_flags & LMD_FLG_CLIENT)
 
-/****************** last_rcvd file *********************/
-
-/** version recovery epoch */
-#define LR_EPOCH_BITS   32
-#define lr_epoch(a) ((a) >> LR_EPOCH_BITS)
-#define LR_EXPIRE_INTERVALS 16 /**< number of intervals to track transno */
-#define ENOENT_VERSION 1 /** 'virtual' version of non-existent object */
-
 /****************** superblock additional info *********************/
 
 struct ll_sb_info;
@@ -141,16 +134,6 @@ struct lustre_sb_info {
 #define     s2lsi_nocast(sb) ((sb)->s_fs_info)
 
 #define     get_profile_name(sb)   (s2lsi(sb)->lsi_lmd->lmd_profile)
-#define	    get_mount_flags(sb)	   (s2lsi(sb)->lsi_lmd->lmd_flags)
-#define	    get_mntdev_name(sb)	   (s2lsi(sb)->lsi_lmd->lmd_dev)
-
-/****************** mount lookup info *********************/
-
-struct lustre_mount_info {
-	char		 *lmi_name;
-	struct super_block   *lmi_sb;
-	struct list_head	    lmi_list_chain;
-};
 
 /****************** prototypes *********************/
 
diff --git a/drivers/staging/lustre/lustre/include/lustre_dlm.h b/drivers/staging/lustre/lustre/include/lustre_dlm.h
index 1e86fb53388a..13c3d2fd31a8 100644
--- a/drivers/staging/lustre/lustre/include/lustre_dlm.h
+++ b/drivers/staging/lustre/lustre/include/lustre_dlm.h
@@ -44,12 +44,12 @@
 #ifndef _LUSTRE_DLM_H__
 #define _LUSTRE_DLM_H__
 
-#include "lustre_lib.h"
-#include "lustre_net.h"
-#include "lustre_import.h"
-#include "lustre_handles.h"
-#include "interval_tree.h"	/* for interval_node{}, ldlm_extent */
-#include "lu_ref.h"
+#include <lustre_lib.h>
+#include <lustre_net.h>
+#include <lustre_import.h>
+#include <lustre_handles.h>
+#include <interval_tree.h>	/* for interval_node{}, ldlm_extent */
+#include <lu_ref.h>
 
 #include "lustre_dlm_flags.h"
 
@@ -1336,5 +1336,18 @@ void ldlm_pool_add(struct ldlm_pool *pl, struct ldlm_lock *lock);
 void ldlm_pool_del(struct ldlm_pool *pl, struct ldlm_lock *lock);
 /** @} */
 
+static inline int ldlm_extent_overlap(const struct ldlm_extent *ex1,
+				      const struct ldlm_extent *ex2)
+{
+	return ex1->start <= ex2->end && ex2->start <= ex1->end;
+}
+
+/* check if @ex1 contains @ex2 */
+static inline int ldlm_extent_contain(const struct ldlm_extent *ex1,
+				      const struct ldlm_extent *ex2)
+{
+	return ex1->start <= ex2->start && ex1->end >= ex2->end;
+}
+
 #endif
 /** @} LDLM */
diff --git a/drivers/staging/lustre/lustre/include/lustre/lustre_errno.h b/drivers/staging/lustre/lustre/include/lustre_errno.h
index 35aefa2cdad1..35aefa2cdad1 100644
--- a/drivers/staging/lustre/lustre/include/lustre/lustre_errno.h
+++ b/drivers/staging/lustre/lustre/include/lustre_errno.h
diff --git a/drivers/staging/lustre/lustre/include/lustre_export.h b/drivers/staging/lustre/lustre/include/lustre_export.h
index 6e7cc4689fb8..3631a69a5c6f 100644
--- a/drivers/staging/lustre/lustre/include/lustre_export.h
+++ b/drivers/staging/lustre/lustre/include/lustre_export.h
@@ -42,9 +42,9 @@
  * @{
  */
 
-#include "lprocfs_status.h"
-#include "lustre/lustre_idl.h"
-#include "lustre_dlm.h"
+#include <lprocfs_status.h>
+#include <uapi/linux/lustre/lustre_idl.h>
+#include <lustre_dlm.h>
 
 enum obd_option {
 	OBD_OPT_FORCE =	 0x0001,
diff --git a/drivers/staging/lustre/lustre/include/lustre_fid.h b/drivers/staging/lustre/lustre/include/lustre_fid.h
index 6dc24a76ddb6..e0f2b8295775 100644
--- a/drivers/staging/lustre/lustre/include/lustre_fid.h
+++ b/drivers/staging/lustre/lustre/include/lustre_fid.h
@@ -41,7 +41,7 @@
  *
  * @{
  *
- * http://wiki.old.lustre.org/index.php/Architecture_-_Interoperability_fids_zfs
+ * http://wiki.lustre.org/index.php/Architecture_-_Interoperability_fids_zfs
  * describes the FID namespace and interoperability requirements for FIDs.
  * The important parts of that document are included here for reference.
  *
@@ -148,9 +148,10 @@
  *  Even so, the MDT and OST resources are also in different LDLM namespaces.
  */
 
-#include "../../include/linux/libcfs/libcfs.h"
-#include "lustre/lustre_idl.h"
-#include "seq_range.h"
+#include <linux/libcfs/libcfs.h>
+#include <uapi/linux/lustre/lustre_fid.h>
+#include <uapi/linux/lustre/lustre_idl.h>
+#include <uapi/linux/lustre/lustre_ostid.h>
 
 struct lu_env;
 struct lu_site;
@@ -288,18 +289,6 @@ static inline int fid_is_quota(const struct lu_fid *fid)
 	       fid_seq(fid) == FID_SEQ_QUOTA_GLB;
 }
 
-static inline int fid_is_namespace_visible(const struct lu_fid *fid)
-{
-	const __u64 seq = fid_seq(fid);
-
-	/* Here, we cannot distinguish whether the normal FID is for OST
-	 * object or not. It is caller's duty to check more if needed.
-	 */
-	return (!fid_is_last_id(fid) &&
-		(fid_seq_is_norm(seq) || fid_seq_is_igif(seq))) ||
-	       fid_is_root(fid) || fid_is_dot_lustre(fid);
-}
-
 static inline int fid_seq_in_fldb(__u64 seq)
 {
 	return fid_seq_is_igif(seq) || fid_seq_is_norm(seq) ||
@@ -506,6 +495,52 @@ static inline int ostid_res_name_eq(const struct ost_id *oi,
 	}
 }
 
+/**
+ * Note: we need check oi_seq to decide where to set oi_id,
+ * so oi_seq should always be set ahead of oi_id.
+ */
+static inline int ostid_set_id(struct ost_id *oi, __u64 oid)
+{
+	if (fid_seq_is_mdt0(oi->oi.oi_seq)) {
+		if (oid >= IDIF_MAX_OID)
+			return -E2BIG;
+		oi->oi.oi_id = oid;
+	} else if (fid_is_idif(&oi->oi_fid)) {
+		if (oid >= IDIF_MAX_OID)
+			return -E2BIG;
+		oi->oi_fid.f_seq = fid_idif_seq(oid,
+						fid_idif_ost_idx(&oi->oi_fid));
+		oi->oi_fid.f_oid = oid;
+		oi->oi_fid.f_ver = oid >> 48;
+	} else {
+		if (oid >= OBIF_MAX_OID)
+			return -E2BIG;
+		oi->oi_fid.f_oid = oid;
+	}
+	return 0;
+}
+
+/* pack any OST FID into an ostid (id/seq) for the wire/disk */
+static inline int fid_to_ostid(const struct lu_fid *fid, struct ost_id *ostid)
+{
+	int rc = 0;
+
+	if (fid_seq_is_igif(fid->f_seq))
+		return -EBADF;
+
+	if (fid_is_idif(fid)) {
+		u64 objid = fid_idif_id(fid_seq(fid), fid_oid(fid),
+					fid_ver(fid));
+
+		ostid_set_seq_mdt0(ostid);
+		rc = ostid_set_id(ostid, objid);
+	} else {
+		ostid->oi_fid = *fid;
+	}
+
+	return rc;
+}
+
 /* The same as osc_build_res_name() */
 static inline void ost_fid_build_resid(const struct lu_fid *fid,
 				       struct ldlm_res_id *resname)
@@ -522,23 +557,6 @@ static inline void ost_fid_build_resid(const struct lu_fid *fid,
 	}
 }
 
-static inline void ost_fid_from_resid(struct lu_fid *fid,
-				      const struct ldlm_res_id *name,
-				      int ost_idx)
-{
-	if (fid_seq_is_mdt0(name->name[LUSTRE_RES_ID_VER_OID_OFF])) {
-		/* old resid */
-		struct ost_id oi;
-
-		ostid_set_seq(&oi, name->name[LUSTRE_RES_ID_VER_OID_OFF]);
-		ostid_set_id(&oi, name->name[LUSTRE_RES_ID_SEQ_OFF]);
-		ostid_to_fid(fid, &oi, ost_idx);
-	} else {
-		/* new resid */
-		fid_extract_from_res_name(fid, name);
-	}
-}
-
 /**
  * Flatten 128-bit FID values into a 64-bit value for use as an inode number.
  * For non-IGIF FIDs this starts just over 2^32, and continues without
diff --git a/drivers/staging/lustre/lustre/include/lustre_fld.h b/drivers/staging/lustre/lustre/include/lustre_fld.h
index 6ef1b03cb986..6125eb0d3395 100644
--- a/drivers/staging/lustre/lustre/include/lustre_fld.h
+++ b/drivers/staging/lustre/lustre/include/lustre_fld.h
@@ -38,8 +38,9 @@
  * @{
  */
 
-#include "lustre/lustre_idl.h"
-#include "../../include/linux/libcfs/libcfs.h"
+#include <uapi/linux/lustre/lustre_idl.h>
+#include <linux/libcfs/libcfs.h>
+#include <seq_range.h>
 
 struct lu_client_fld;
 struct lu_server_fld;
diff --git a/drivers/staging/lustre/lustre/include/lustre_handles.h b/drivers/staging/lustre/lustre/include/lustre_handles.h
index e071bac9df57..d49932628f32 100644
--- a/drivers/staging/lustre/lustre/include/lustre_handles.h
+++ b/drivers/staging/lustre/lustre/include/lustre_handles.h
@@ -44,7 +44,7 @@
 #include <linux/spinlock.h>
 #include <linux/types.h>
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
 struct portals_handle_ops {
 	void (*hop_addref)(void *object);
diff --git a/drivers/staging/lustre/lustre/include/lustre_import.h b/drivers/staging/lustre/lustre/include/lustre_import.h
index f0c931ce1a67..d71d0473a4eb 100644
--- a/drivers/staging/lustre/lustre/include/lustre_import.h
+++ b/drivers/staging/lustre/lustre/include/lustre_import.h
@@ -43,8 +43,8 @@
  * @{
  */
 
-#include "lustre_handles.h"
-#include "lustre/lustre_idl.h"
+#include <lustre_handles.h>
+#include <uapi/linux/lustre/lustre_idl.h>
 
 /**
  * Adaptive Timeout stuff
diff --git a/drivers/staging/lustre/lustre/include/lustre_kernelcomm.h b/drivers/staging/lustre/lustre/include/lustre_kernelcomm.h
index 970610b6de89..f1899a3d7a40 100644
--- a/drivers/staging/lustre/lustre/include/lustre_kernelcomm.h
+++ b/drivers/staging/lustre/lustre/include/lustre_kernelcomm.h
@@ -38,7 +38,7 @@
 #define __LUSTRE_KERNELCOMM_H__
 
 /* For declarations shared with userspace */
-#include "uapi_kernelcomm.h"
+#include <uapi/linux/lustre/lustre_kernelcomm.h>
 
 /* prototype for callback function on kuc groups */
 typedef int (*libcfs_kkuc_cb_t)(void *data, void *cb_arg);
diff --git a/drivers/staging/lustre/lustre/include/lustre_lib.h b/drivers/staging/lustre/lustre/include/lustre_lib.h
index f24970da8323..81b9cbffc050 100644
--- a/drivers/staging/lustre/lustre/include/lustre_lib.h
+++ b/drivers/staging/lustre/lustre/include/lustre_lib.h
@@ -45,18 +45,18 @@
 #include <linux/sched/signal.h>
 #include <linux/signal.h>
 #include <linux/types.h>
-#include "../../include/linux/libcfs/libcfs.h"
-#include "lustre/lustre_idl.h"
-#include "lustre_ver.h"
-#include "lustre_cfg.h"
+#include <linux/libcfs/libcfs.h>
+#include <uapi/linux/lustre/lustre_idl.h>
+#include <uapi/linux/lustre/lustre_ver.h>
+#include <uapi/linux/lustre/lustre_cfg.h>
 
 /* target.c */
 struct ptlrpc_request;
 struct obd_export;
 struct lu_target;
 struct l_wait_info;
-#include "lustre_ha.h"
-#include "lustre_net.h"
+#include <lustre_ha.h>
+#include <lustre_net.h>
 
 #define LI_POISON 0x5a5a5a5a
 #if BITS_PER_LONG > 32
diff --git a/drivers/staging/lustre/lustre/include/lustre_linkea.h b/drivers/staging/lustre/lustre/include/lustre_linkea.h
index 249e8bf4fa22..3ff008fee13d 100644
--- a/drivers/staging/lustre/lustre/include/lustre_linkea.h
+++ b/drivers/staging/lustre/lustre/include/lustre_linkea.h
@@ -26,7 +26,19 @@
  * Author: di wang <di.wang@intel.com>
  */
 
-#define DEFAULT_LINKEA_SIZE	4096
+/* There are several reasons to restrict the linkEA size:
+ *
+ * 1. Under DNE mode, if we do not restrict the linkEA size, and if there
+ *    are too many cross-MDTs hard links to the same object, then it will
+ *    casue the llog overflow.
+ *
+ * 2. Some backend has limited size for EA. For example, if without large
+ *    EA enabled, the ldiskfs will make all EAs to share one (4K) EA block.
+ *
+ * 3. Too many entries in linkEA will seriously affect linkEA performance
+ *    because we only support to locate linkEA entry consecutively.
+ */
+#define MAX_LINKEA_SIZE		4096
 
 struct linkea_data {
 	/**
@@ -43,6 +55,7 @@ struct linkea_data {
 
 int linkea_data_new(struct linkea_data *ldata, struct lu_buf *buf);
 int linkea_init(struct linkea_data *ldata);
+int linkea_init_with_rec(struct linkea_data *ldata);
 void linkea_entry_unpack(const struct link_ea_entry *lee, int *reclen,
 			 struct lu_name *lname, struct lu_fid *pfid);
 int linkea_entry_pack(struct link_ea_entry *lee, const struct lu_name *lname,
diff --git a/drivers/staging/lustre/lustre/include/lustre_lmv.h b/drivers/staging/lustre/lustre/include/lustre_lmv.h
index 5aa3645e64dc..98a82be2037f 100644
--- a/drivers/staging/lustre/lustre/include/lustre_lmv.h
+++ b/drivers/staging/lustre/lustre/include/lustre_lmv.h
@@ -32,7 +32,7 @@
 
 #ifndef _LUSTRE_LMV_H
 #define _LUSTRE_LMV_H
-#include "lustre/lustre_idl.h"
+#include <uapi/linux/lustre/lustre_idl.h>
 
 struct lmv_oinfo {
 	struct lu_fid	lmo_fid;
diff --git a/drivers/staging/lustre/lustre/include/lustre_log.h b/drivers/staging/lustre/lustre/include/lustre_log.h
index 35e37eb1bc2c..24a7777424f6 100644
--- a/drivers/staging/lustre/lustre/include/lustre_log.h
+++ b/drivers/staging/lustre/lustre/include/lustre_log.h
@@ -52,8 +52,8 @@
  * @{
  */
 
-#include "obd_class.h"
-#include "lustre/lustre_idl.h"
+#include <obd_class.h>
+#include <uapi/linux/lustre/lustre_idl.h>
 
 #define LOG_NAME_LIMIT(logname, name)		   \
 	snprintf(logname, sizeof(logname), "LOGS/%s", name)
diff --git a/drivers/staging/lustre/lustre/include/lustre_mdc.h b/drivers/staging/lustre/lustre/include/lustre_mdc.h
index 198ceb0c66f9..c0c44974cb1c 100644
--- a/drivers/staging/lustre/lustre/include/lustre_mdc.h
+++ b/drivers/staging/lustre/lustre/include/lustre_mdc.h
@@ -46,14 +46,13 @@
 
 #include <linux/fs.h>
 #include <linux/dcache.h>
-#include "lustre_intent.h"
-#include "lustre_handles.h"
-#include "../../include/linux/libcfs/libcfs.h"
-#include "obd_class.h"
-#include "lustre/lustre_idl.h"
-#include "lustre_lib.h"
-#include "lustre_dlm.h"
-#include "lustre_export.h"
+#include <lustre_intent.h>
+#include <lustre_handles.h>
+#include <linux/libcfs/libcfs.h>
+#include <obd_class.h>
+#include <lustre_lib.h>
+#include <lustre_dlm.h>
+#include <lustre_export.h>
 
 struct ptlrpc_client;
 struct obd_export;
diff --git a/drivers/staging/lustre/lustre/include/lustre_mds.h b/drivers/staging/lustre/lustre/include/lustre_mds.h
index 23a7e4f78e9a..c424e1239fd5 100644
--- a/drivers/staging/lustre/lustre/include/lustre_mds.h
+++ b/drivers/staging/lustre/lustre/include/lustre_mds.h
@@ -43,12 +43,11 @@
  * @{
  */
 
-#include "lustre_handles.h"
-#include "../../include/linux/libcfs/libcfs.h"
-#include "lustre/lustre_idl.h"
-#include "lustre_lib.h"
-#include "lustre_dlm.h"
-#include "lustre_export.h"
+#include <lustre_handles.h>
+#include <linux/libcfs/libcfs.h>
+#include <lustre_lib.h>
+#include <lustre_dlm.h>
+#include <lustre_export.h>
 
 struct mds_group_info {
 	struct obd_uuid *uuid;
diff --git a/drivers/staging/lustre/lustre/include/lustre_net.h b/drivers/staging/lustre/lustre/include/lustre_net.h
index d61b000dac2c..c6d1646f102a 100644
--- a/drivers/staging/lustre/lustre/include/lustre_net.h
+++ b/drivers/staging/lustre/lustre/include/lustre_net.h
@@ -51,19 +51,20 @@
  */
 
 #include <linux/uio.h>
-#include "../../include/linux/libcfs/libcfs.h"
-#include "../../include/linux/lnet/nidstr.h"
-#include "../../include/linux/lnet/api.h"
-#include "lustre/lustre_idl.h"
-#include "lustre_ha.h"
-#include "lustre_sec.h"
-#include "lustre_import.h"
-#include "lprocfs_status.h"
-#include "lu_object.h"
-#include "lustre_req_layout.h"
-
-#include "obd_support.h"
-#include "lustre_ver.h"
+#include <linux/libcfs/libcfs.h>
+#include <uapi/linux/lnet/nidstr.h>
+#include <linux/lnet/api.h>
+#include <uapi/linux/lustre/lustre_idl.h>
+#include <lustre_errno.h>
+#include <lustre_ha.h>
+#include <lustre_sec.h>
+#include <lustre_import.h>
+#include <lprocfs_status.h>
+#include <lu_object.h>
+#include <lustre_req_layout.h>
+
+#include <obd_support.h>
+#include <uapi/linux/lustre/lustre_ver.h>
 
 /* MD flags we _always_ use */
 #define PTLRPC_MD_OPTIONS  0
@@ -521,7 +522,7 @@ struct lu_env;
 
 struct ldlm_lock;
 
-#include "lustre_nrs.h"
+#include <lustre_nrs.h>
 
 /**
  * Basic request prioritization operations structure.
@@ -558,13 +559,13 @@ struct ptlrpc_cli_req {
 	/** request sent timeval */
 	struct timespec64		 cr_sent_tv;
 	/** time for request really sent out */
-	time_t				 cr_sent_out;
+	time64_t			 cr_sent_out;
 	/** when req reply unlink must finish. */
-	time_t				 cr_reply_deadline;
+	time64_t			 cr_reply_deadline;
 	/** when req bulk unlink must finish. */
-	time_t				 cr_bulk_deadline;
+	time64_t			 cr_bulk_deadline;
 	/** when req unlink must finish. */
-	time_t				 cr_req_deadline;
+	time64_t			 cr_req_deadline;
 	/** Portal to which this request would be sent */
 	short				 cr_req_ptl;
 	/** Portal where to wait for reply and where reply would be sent */
@@ -663,7 +664,7 @@ struct ptlrpc_srv_req {
 	/** history sequence # */
 	__u64				sr_hist_seq;
 	/** the index of service's srv_at_array into which request is linked */
-	time_t				sr_at_index;
+	time64_t			sr_at_index;
 	/** authed uid */
 	uid_t				sr_auth_uid;
 	/** authed uid mapped to */
diff --git a/drivers/staging/lustre/lustre/include/lustre_nrs.h b/drivers/staging/lustre/lustre/include/lustre_nrs.h
index a5028aaa19cd..51f45f7776df 100644
--- a/drivers/staging/lustre/lustre/include/lustre_nrs.h
+++ b/drivers/staging/lustre/lustre/include/lustre_nrs.h
@@ -669,7 +669,7 @@ enum {
 	NRS_RES_MAX
 };
 
-#include "lustre_nrs_fifo.h"
+#include <lustre_nrs_fifo.h>
 
 /**
  * NRS request
diff --git a/drivers/staging/lustre/lustre/include/lustre_obdo.h b/drivers/staging/lustre/lustre/include/lustre_obdo.h
index 1e12f8c0f157..53379f861161 100644
--- a/drivers/staging/lustre/lustre/include/lustre_obdo.h
+++ b/drivers/staging/lustre/lustre/include/lustre_obdo.h
@@ -35,7 +35,7 @@
 #ifndef _LUSTRE_OBDO_H_
 #define _LUSTRE_OBDO_H_
 
-#include "lustre/lustre_idl.h"
+#include <uapi/linux/lustre/lustre_idl.h>
 
 /**
  * Create an obdo to send over the wire
diff --git a/drivers/staging/lustre/lustre/include/lustre_param.h b/drivers/staging/lustre/lustre/include/lustre_param.h
deleted file mode 100644
index 8061a04ee806..000000000000
--- a/drivers/staging/lustre/lustre/include/lustre_param.h
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * GPL HEADER START
- *
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 only,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License version 2 for more details (a copy is included
- * in the LICENSE file that accompanied this code).
- *
- * You should have received a copy of the GNU General Public License
- * version 2 along with this program; If not, see
- * http://www.gnu.org/licenses/gpl-2.0.html
- *
- * GPL HEADER END
- */
-/*
- * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
- * Use is subject to license terms.
- *
- * Copyright (c) 2011, 2015, Intel Corporation.
- */
-/*
- * This file is part of Lustre, http://www.lustre.org/
- * Lustre is a trademark of Sun Microsystems, Inc.
- *
- * lustre/include/lustre_param.h
- *
- * User-settable parameter keys
- *
- * Author: Nathan Rutman <nathan@clusterfs.com>
- */
-
-#ifndef _LUSTRE_PARAM_H
-#define _LUSTRE_PARAM_H
-
-#include "../../include/linux/libcfs/libcfs.h"
-#include "../../include/linux/lnet/types.h"
-
-/** \defgroup param param
- *
- * @{
- */
-
-/* For interoperability */
-struct cfg_interop_param {
-	char *old_param;
-	char *new_param;
-};
-
-/* obd_config.c */
-int class_find_param(char *buf, char *key, char **valp);
-int class_parse_nid(char *buf, lnet_nid_t *nid, char **endh);
-int class_parse_nid_quiet(char *buf, lnet_nid_t *nid, char **endh);
-
-/****************** User-settable parameter keys *********************/
-/* e.g.
-	tunefs.lustre --param="failover.node=192.168.0.13@tcp0" /dev/sda
-	lctl conf_param testfs-OST0000 failover.node=3@elan,192.168.0.3@tcp0
-		    ... testfs-MDT0000.lov.stripesize=4M
-		    ... testfs-OST0000.ost.client_cache_seconds=15
-		    ... testfs.sys.timeout=<secs>
-		    ... testfs.llite.max_read_ahead_mb=16
-*/
-
-/* System global or special params not handled in obd's proc
- * See mgs_write_log_sys()
- */
-#define PARAM_TIMEOUT	      "timeout="	  /* global */
-#define PARAM_LDLM_TIMEOUT	 "ldlm_timeout="     /* global */
-#define PARAM_AT_MIN	       "at_min="	   /* global */
-#define PARAM_AT_MAX	       "at_max="	   /* global */
-#define PARAM_AT_EXTRA	     "at_extra="	 /* global */
-#define PARAM_AT_EARLY_MARGIN      "at_early_margin="  /* global */
-#define PARAM_AT_HISTORY	   "at_history="       /* global */
-#define PARAM_JOBID_VAR		   "jobid_var="	       /* global */
-#define PARAM_MGSNODE	      "mgsnode="	  /* only at mounttime */
-#define PARAM_FAILNODE	     "failover.node="    /* add failover nid */
-#define PARAM_FAILMODE	     "failover.mode="    /* initial mount only */
-#define PARAM_ACTIVE	       "active="	   /* activate/deactivate */
-#define PARAM_NETWORK	      "network="	  /* bind on nid */
-#define PARAM_ID_UPCALL		"identity_upcall="  /* identity upcall */
-
-/* Prefixes for parameters handled by obd's proc methods (XXX_process_config) */
-#define PARAM_OST		  "ost."
-#define PARAM_OSD		"osd."
-#define PARAM_OSC		  "osc."
-#define PARAM_MDT		  "mdt."
-#define PARAM_MDD		  "mdd."
-#define PARAM_MDC		  "mdc."
-#define PARAM_LLITE		"llite."
-#define PARAM_LOV		  "lov."
-#define PARAM_LOD		"lod."
-#define PARAM_OSP		"osp."
-#define PARAM_SYS		  "sys."	      /* global */
-#define PARAM_SRPC		 "srpc."
-#define PARAM_SRPC_FLVR	    "srpc.flavor."
-#define PARAM_SRPC_UDESC	   "srpc.udesc.cli2mdt"
-#define PARAM_SEC		  "security."
-#define PARAM_QUOTA		"quota."	    /* global */
-
-/** @} param */
-
-#endif /* _LUSTRE_PARAM_H */
diff --git a/drivers/staging/lustre/lustre/include/lustre_swab.h b/drivers/staging/lustre/lustre/include/lustre_swab.h
index 26d01c2d6633..765e923c2fc9 100644
--- a/drivers/staging/lustre/lustre/include/lustre_swab.h
+++ b/drivers/staging/lustre/lustre/include/lustre_swab.h
@@ -48,7 +48,7 @@
 #ifndef _LUSTRE_SWAB_H_
 #define _LUSTRE_SWAB_H_
 
-#include "lustre/lustre_idl.h"
+#include <uapi/linux/lustre/lustre_idl.h>
 
 void lustre_swab_ptlrpc_body(struct ptlrpc_body *pb);
 void lustre_swab_connect(struct obd_connect_data *ocd);
@@ -99,4 +99,10 @@ void lustre_swab_swap_layouts(struct mdc_swap_layouts *msl);
 void lustre_swab_close_data(struct close_data *data);
 void lustre_swab_lmv_user_md(struct lmv_user_md *lum);
 
+/* Functions for dumping PTLRPC fields */
+void dump_rniobuf(struct niobuf_remote *rnb);
+void dump_ioo(struct obd_ioobj *nb);
+void dump_ost_body(struct ost_body *ob);
+void dump_rcs(__u32 *rc);
+
 #endif
diff --git a/drivers/staging/lustre/lustre/include/obd.h b/drivers/staging/lustre/lustre/include/obd.h
index 4ce85064f9d0..a986737ec010 100644
--- a/drivers/staging/lustre/lustre/include/obd.h
+++ b/drivers/staging/lustre/lustre/include/obd.h
@@ -35,15 +35,15 @@
 
 #include <linux/spinlock.h>
 
-#include "lustre/lustre_idl.h"
-#include "lustre_lib.h"
-#include "lu_ref.h"
-#include "lustre_export.h"
-#include "lustre_fid.h"
-#include "lustre_fld.h"
-#include "lustre_handles.h"
-#include "lustre_intent.h"
-#include "cl_object.h"
+#include <uapi/linux/lustre/lustre_idl.h>
+#include <lustre_lib.h>
+#include <lu_ref.h>
+#include <lustre_export.h>
+#include <lustre_fid.h>
+#include <lustre_fld.h>
+#include <lustre_handles.h>
+#include <lustre_intent.h>
+#include <cl_object.h>
 
 #define MAX_OBD_DEVICES 8192
 
@@ -404,12 +404,10 @@ struct lmv_tgt_desc {
 };
 
 struct lmv_obd {
-	int			refcount;
 	struct lu_client_fld	lmv_fld;
 	spinlock_t		lmv_lock;
 	struct lmv_desc		desc;
 	struct obd_uuid		cluuid;
-	struct obd_export	*exp;
 
 	struct mutex		lmv_init_mutex;
 	int			connected;
diff --git a/drivers/staging/lustre/lustre/include/obd_cksum.h b/drivers/staging/lustre/lustre/include/obd_cksum.h
index a8a81e662a56..cda3d2808d2f 100644
--- a/drivers/staging/lustre/lustre/include/obd_cksum.h
+++ b/drivers/staging/lustre/lustre/include/obd_cksum.h
@@ -30,9 +30,9 @@
 
 #ifndef __OBD_CKSUM
 #define __OBD_CKSUM
-#include "../../include/linux/libcfs/libcfs.h"
-#include "../../include/linux/libcfs/libcfs_crypto.h"
-#include "lustre/lustre_idl.h"
+#include <linux/libcfs/libcfs.h>
+#include <linux/libcfs/libcfs_crypto.h>
+#include <uapi/linux/lustre/lustre_idl.h>
 
 static inline unsigned char cksum_obd2cfs(enum cksum_type cksum_type)
 {
diff --git a/drivers/staging/lustre/lustre/include/obd_class.h b/drivers/staging/lustre/lustre/include/obd_class.h
index 083a6ff56a05..976005a1e0b2 100644
--- a/drivers/staging/lustre/lustre/include/obd_class.h
+++ b/drivers/staging/lustre/lustre/include/obd_class.h
@@ -32,13 +32,12 @@
 #ifndef __CLASS_OBD_H
 #define __CLASS_OBD_H
 
-#include "obd_support.h"
-#include "lustre_import.h"
-#include "lustre_net.h"
-#include "obd.h"
-#include "lustre_lib.h"
-#include "lustre/lustre_idl.h"
-#include "lprocfs_status.h"
+#include <obd_support.h>
+#include <lustre_import.h>
+#include <lustre_net.h>
+#include <obd.h>
+#include <lustre_lib.h>
+#include <lprocfs_status.h>
 
 #define OBD_STATFS_NODELAY      0x0001  /* requests should be send without delay
 					 * and resends for avoid deadlocks
@@ -46,14 +45,7 @@
 #define OBD_STATFS_FROM_CACHE   0x0002  /* the statfs callback should not update
 					 * obd_osfs_age
 					 */
-#define OBD_STATFS_PTLRPCD      0x0004  /* requests will be sent via ptlrpcd
-					 * instead of a specific set. This
-					 * means that we cannot rely on the set
-					 * interpret routine to be called.
-					 * lov_statfs_fini() must thus be called
-					 * by the request interpret routine
-					 */
-#define OBD_STATFS_FOR_MDT0	0x0008	/* The statfs is only for retrieving
+#define OBD_STATFS_FOR_MDT0	0x0004	/* The statfs is only for retrieving
 					 * information from MDT0.
 					 */
 
@@ -112,10 +104,29 @@ struct llog_handle;
 struct llog_rec_hdr;
 typedef int (*llog_cb_t)(const struct lu_env *, struct llog_handle *,
 			 struct llog_rec_hdr *, void *);
+
 /* obd_config.c */
+char *lustre_cfg_string(struct lustre_cfg *lcfg, u32 index);
 int class_process_config(struct lustre_cfg *lcfg);
 int class_process_proc_param(char *prefix, struct lprocfs_vars *lvars,
 			     struct lustre_cfg *lcfg, void *data);
+
+/* For interoperability */
+struct cfg_interop_param {
+	char *old_param;
+	char *new_param;
+};
+
+int class_find_param(char *buf, char *key, char **valp);
+struct cfg_interop_param *class_find_old_param(const char *param,
+					       struct cfg_interop_param *ptr);
+int class_get_next_param(char **params, char *copy);
+int class_parse_nid(char *buf, lnet_nid_t *nid, char **endh);
+int class_parse_nid_quiet(char *buf, lnet_nid_t *nid, char **endh);
+int class_parse_net(char *buf, u32 *net, char **endh);
+int class_match_nid(char *buf, char *key, lnet_nid_t nid);
+int class_match_net(char *buf, char *key, u32 net);
+
 struct obd_device *class_incref(struct obd_device *obd,
 				const char *scope, const void *source);
 void class_decref(struct obd_device *obd,
@@ -1566,4 +1577,7 @@ struct root_squash_info {
 	struct rw_semaphore	rsi_sem;
 };
 
+/* linux-module.c */
+int obd_ioctl_getdata(char **buf, int *len, void __user *arg);
+
 #endif /* __LINUX_OBD_CLASS_H */
diff --git a/drivers/staging/lustre/lustre/include/obd_support.h b/drivers/staging/lustre/lustre/include/obd_support.h
index 33304041bb63..aea193a882a2 100644
--- a/drivers/staging/lustre/lustre/include/obd_support.h
+++ b/drivers/staging/lustre/lustre/include/obd_support.h
@@ -36,9 +36,9 @@
 #include <linux/slab.h>
 #include <linux/sched/signal.h>
 
-#include "../../include/linux/libcfs/libcfs.h"
-#include "lustre_compat.h"
-#include "lprocfs_status.h"
+#include <linux/libcfs/libcfs.h>
+#include <lustre_compat.h>
+#include <lprocfs_status.h>
 
 /* global variables */
 extern unsigned int obd_debug_peer_on_timeout;
diff --git a/drivers/staging/lustre/lustre/include/seq_range.h b/drivers/staging/lustre/lustre/include/seq_range.h
index 30c4dd66d5c4..d7175485944d 100644
--- a/drivers/staging/lustre/lustre/include/seq_range.h
+++ b/drivers/staging/lustre/lustre/include/seq_range.h
@@ -34,7 +34,7 @@
 #ifndef _SEQ_RANGE_H_
 #define _SEQ_RANGE_H_
 
-#include "lustre/lustre_idl.h"
+#include <uapi/linux/lustre/lustre_idl.h>
 
 /**
  * computes the sequence range type \a range
diff --git a/drivers/staging/lustre/lustre/ldlm/interval_tree.c b/drivers/staging/lustre/lustre/ldlm/interval_tree.c
index e1069021420d..19e285dd2ee1 100644
--- a/drivers/staging/lustre/lustre/ldlm/interval_tree.c
+++ b/drivers/staging/lustre/lustre/ldlm/interval_tree.c
@@ -34,9 +34,9 @@
  * Author: Huang Wei <huangwei@clusterfs.com>
  * Author: Jay Xiong <jinshan.xiong@sun.com>
  */
-#include "../include/lustre_dlm.h"
-#include "../include/obd_support.h"
-#include "../include/interval_tree.h"
+#include <lustre_dlm.h>
+#include <obd_support.h>
+#include <interval_tree.h>
 
 enum {
 	INTERVAL_RED = 0,
@@ -110,6 +110,15 @@ static struct interval_node *interval_first(struct interval_node *node)
 	return node;
 }
 
+static struct interval_node *interval_last(struct interval_node *node)
+{
+	if (!node)
+		return NULL;
+	while (node->in_right)
+		node = node->in_right;
+	return node;
+}
+
 static struct interval_node *interval_next(struct interval_node *node)
 {
 	if (!node)
@@ -121,6 +130,37 @@ static struct interval_node *interval_next(struct interval_node *node)
 	return node->in_parent;
 }
 
+static struct interval_node *interval_prev(struct interval_node *node)
+{
+	if (!node)
+		return NULL;
+
+	if (node->in_left)
+		return interval_last(node->in_left);
+
+	while (node->in_parent && node_is_left_child(node))
+		node = node->in_parent;
+
+	return node->in_parent;
+}
+
+enum interval_iter interval_iterate_reverse(struct interval_node *root,
+					    interval_callback_t func,
+					    void *data)
+{
+	enum interval_iter rc = INTERVAL_ITER_CONT;
+	struct interval_node *node;
+
+	for (node = interval_last(root); node; node = interval_prev(node)) {
+		rc = func(node, data);
+		if (rc == INTERVAL_ITER_STOP)
+			break;
+	}
+
+	return rc;
+}
+EXPORT_SYMBOL(interval_iterate_reverse);
+
 static void __rotate_change_maxhigh(struct interval_node *node,
 				    struct interval_node *rotate)
 {
diff --git a/drivers/staging/lustre/lustre/ldlm/l_lock.c b/drivers/staging/lustre/lustre/ldlm/l_lock.c
index 3845f386f1db..57fd84effdfa 100644
--- a/drivers/staging/lustre/lustre/ldlm/l_lock.c
+++ b/drivers/staging/lustre/lustre/ldlm/l_lock.c
@@ -31,10 +31,10 @@
  */
 
 #define DEBUG_SUBSYSTEM S_LDLM
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
-#include "../include/lustre_dlm.h"
-#include "../include/lustre_lib.h"
+#include <lustre_dlm.h>
+#include <lustre_lib.h>
 
 /**
  * Lock a lock and its resource.
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_extent.c b/drivers/staging/lustre/lustre/ldlm/ldlm_extent.c
index 08f97e2117ed..2cc6dc2b281f 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_extent.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_extent.c
@@ -46,12 +46,12 @@
  */
 
 #define DEBUG_SUBSYSTEM S_LDLM
-#include "../../include/linux/libcfs/libcfs.h"
-#include "../include/lustre_dlm.h"
-#include "../include/obd_support.h"
-#include "../include/obd.h"
-#include "../include/obd_class.h"
-#include "../include/lustre_lib.h"
+#include <linux/libcfs/libcfs.h>
+#include <lustre_dlm.h>
+#include <obd_support.h>
+#include <obd.h>
+#include <obd_class.h>
+#include <lustre_lib.h>
 #include "ldlm_internal.h"
 
 /* When a lock is cancelled by a client, the KMS may undergo change if this
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_flock.c b/drivers/staging/lustre/lustre/ldlm/ldlm_flock.c
index b7f28b39c7b3..cb826e9e840e 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_flock.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_flock.c
@@ -52,10 +52,10 @@
 
 #define DEBUG_SUBSYSTEM S_LDLM
 
-#include "../include/lustre_dlm.h"
-#include "../include/obd_support.h"
-#include "../include/obd_class.h"
-#include "../include/lustre_lib.h"
+#include <lustre_dlm.h>
+#include <obd_support.h>
+#include <obd_class.h>
+#include <lustre_lib.h>
 #include <linux/list.h>
 #include "ldlm_internal.h"
 
@@ -90,8 +90,8 @@ ldlm_flocks_overlap(struct ldlm_lock *lock, struct ldlm_lock *new)
 static inline void
 ldlm_flock_destroy(struct ldlm_lock *lock, enum ldlm_mode mode, __u64 flags)
 {
-	LDLM_DEBUG(lock, "ldlm_flock_destroy(mode: %d, flags: 0x%llx)",
-		   mode, flags);
+	LDLM_DEBUG(lock, "%s(mode: %d, flags: 0x%llx)",
+		   __func__, mode, flags);
 
 	/* Safe to not lock here, since it should be empty anyway */
 	LASSERT(hlist_unhashed(&lock->l_exp_flock_hash));
@@ -596,7 +596,7 @@ granted:
 		default:
 			getlk->fl_type = F_UNLCK;
 		}
-		getlk->fl_pid = (pid_t)lock->l_policy_data.l_flock.pid;
+		getlk->fl_pid = -(pid_t)lock->l_policy_data.l_flock.pid;
 		getlk->fl_start = (loff_t)lock->l_policy_data.l_flock.start;
 		getlk->fl_end = (loff_t)lock->l_policy_data.l_flock.end;
 	} else {
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_inodebits.c b/drivers/staging/lustre/lustre/ldlm/ldlm_inodebits.c
index ae37c3686b1b..fcb6e44bd319 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_inodebits.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_inodebits.c
@@ -49,9 +49,9 @@
 
 #define DEBUG_SUBSYSTEM S_LDLM
 
-#include "../include/lustre_dlm.h"
-#include "../include/obd_support.h"
-#include "../include/lustre_lib.h"
+#include <lustre_dlm.h>
+#include <obd_support.h>
+#include <lustre_lib.h>
 #include "ldlm_internal.h"
 
 void ldlm_ibits_policy_wire_to_local(const union ldlm_wire_policy_data *wpolicy,
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_internal.h b/drivers/staging/lustre/lustre/ldlm/ldlm_internal.h
index ec3b23cd09ec..36808dbe8790 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_internal.h
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_internal.h
@@ -106,9 +106,6 @@ int ldlm_cancel_lru_local(struct ldlm_namespace *ns,
 extern unsigned int ldlm_enqueue_min;
 extern unsigned int ldlm_cancel_unused_locks_before_replay;
 
-/* ldlm_resource.c */
-int ldlm_resource_putref_locked(struct ldlm_resource *res);
-
 /* ldlm_lock.c */
 
 struct ldlm_cb_set_arg {
@@ -336,3 +333,9 @@ void ldlm_flock_policy_wire_to_local(const union ldlm_wire_policy_data *wpolicy,
 				     union ldlm_policy_data *lpolicy);
 void ldlm_flock_policy_local_to_wire(const union ldlm_policy_data *lpolicy,
 				     union ldlm_wire_policy_data *wpolicy);
+
+static inline bool ldlm_res_eq(const struct ldlm_res_id *res0,
+			       const struct ldlm_res_id *res1)
+{
+	return memcmp(res0, res1, sizeof(*res0)) == 0;
+}
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_lib.c b/drivers/staging/lustre/lustre/ldlm/ldlm_lib.c
index 4dc7baee1f28..22600c2a73ea 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_lib.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_lib.c
@@ -39,12 +39,12 @@
 
 #define DEBUG_SUBSYSTEM S_LDLM
 
-#include "../../include/linux/libcfs/libcfs.h"
-#include "../include/obd.h"
-#include "../include/obd_class.h"
-#include "../include/lustre_dlm.h"
-#include "../include/lustre_net.h"
-#include "../include/lustre_sec.h"
+#include <linux/libcfs/libcfs.h>
+#include <obd.h>
+#include <obd_class.h>
+#include <lustre_dlm.h>
+#include <lustre_net.h>
+#include <lustre_sec.h>
 #include "ldlm_internal.h"
 
 /* @priority: If non-zero, move the selected connection to the list head.
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_lock.c b/drivers/staging/lustre/lustre/ldlm/ldlm_lock.c
index ddb46428093f..b5d84f3f6071 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_lock.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_lock.c
@@ -37,10 +37,10 @@
 
 #define DEBUG_SUBSYSTEM S_LDLM
 
-#include "../../include/linux/libcfs/libcfs.h"
-#include "../include/lustre_intent.h"
-#include "../include/lustre_swab.h"
-#include "../include/obd_class.h"
+#include <linux/libcfs/libcfs.h>
+#include <lustre_intent.h>
+#include <lustre_swab.h>
+#include <obd_class.h>
 #include "ldlm_internal.h"
 
 /* lock types */
@@ -1029,11 +1029,11 @@ void ldlm_grant_lock(struct ldlm_lock *lock, struct list_head *work_list)
 	if (work_list && lock->l_completion_ast)
 		ldlm_add_ast_work_item(lock, NULL, work_list);
 
-	if (res->lr_type == LDLM_PLAIN || res->lr_type == LDLM_IBITS)
+	if (res->lr_type == LDLM_PLAIN || res->lr_type == LDLM_IBITS) {
 		ldlm_grant_lock_with_skiplist(lock);
-	else if (res->lr_type == LDLM_EXTENT)
+	} else if (res->lr_type == LDLM_EXTENT) {
 		ldlm_extent_add_lock(res, lock);
-	else if (res->lr_type == LDLM_FLOCK) {
+	} else if (res->lr_type == LDLM_FLOCK) {
 		/*
 		 * We should not add locks to granted list in the following cases:
 		 * - this is an UNLOCK but not a real lock;
@@ -1045,8 +1045,9 @@ void ldlm_grant_lock(struct ldlm_lock *lock, struct list_head *work_list)
 		    ldlm_is_test_lock(lock) || ldlm_is_flock_deadlock(lock))
 			return;
 		ldlm_resource_add_lock(res, &res->lr_granted, lock);
-	} else
+	} else {
 		LBUG();
+	}
 
 	ldlm_pool_add(&ldlm_res_to_ns(res)->ns_pool, lock);
 }
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_lockd.c b/drivers/staging/lustre/lustre/ldlm/ldlm_lockd.c
index fff930fc3cff..e2707336586c 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_lockd.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_lockd.c
@@ -37,9 +37,9 @@
 
 #define DEBUG_SUBSYSTEM S_LDLM
 
-#include "../../include/linux/libcfs/libcfs.h"
-#include "../include/lustre_dlm.h"
-#include "../include/obd_class.h"
+#include <linux/libcfs/libcfs.h>
+#include <lustre_dlm.h>
+#include <obd_class.h>
 #include <linux/list.h>
 #include "ldlm_internal.h"
 
@@ -926,7 +926,7 @@ static struct attribute *ldlm_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group ldlm_attr_group = {
+static const struct attribute_group ldlm_attr_group = {
 	.attrs = ldlm_attrs,
 };
 
@@ -1138,7 +1138,7 @@ int ldlm_init(void)
 void ldlm_exit(void)
 {
 	if (ldlm_refcount)
-		CERROR("ldlm_refcount is %d in ldlm_exit!\n", ldlm_refcount);
+		CERROR("ldlm_refcount is %d in %s!\n", ldlm_refcount, __func__);
 	kmem_cache_destroy(ldlm_resource_slab);
 	/* ldlm_lock_put() use RCU to call ldlm_lock_free, so need call
 	 * synchronize_rcu() to wait a grace period elapsed, so that
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_plain.c b/drivers/staging/lustre/lustre/ldlm/ldlm_plain.c
index 862ea0a1dc97..1ca605fe25ff 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_plain.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_plain.c
@@ -48,9 +48,9 @@
 
 #define DEBUG_SUBSYSTEM S_LDLM
 
-#include "../include/lustre_dlm.h"
-#include "../include/obd_support.h"
-#include "../include/lustre_lib.h"
+#include <lustre_dlm.h>
+#include <obd_support.h>
+#include <lustre_lib.h>
 
 #include "ldlm_internal.h"
 
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_pool.c b/drivers/staging/lustre/lustre/ldlm/ldlm_pool.c
index cf3fc5793377..d77bf0baa84f 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_pool.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_pool.c
@@ -94,10 +94,10 @@
 
 #define DEBUG_SUBSYSTEM S_LDLM
 
-#include "../include/lustre_dlm.h"
-#include "../include/cl_object.h"
-#include "../include/obd_class.h"
-#include "../include/obd_support.h"
+#include <lustre_dlm.h>
+#include <cl_object.h>
+#include <obd_class.h>
+#include <obd_support.h>
 #include "ldlm_internal.h"
 
 /*
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_request.c b/drivers/staging/lustre/lustre/ldlm/ldlm_request.c
index 4028e11249a2..f3bf238d0748 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_request.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_request.c
@@ -57,9 +57,10 @@
 
 #define DEBUG_SUBSYSTEM S_LDLM
 
-#include "../include/lustre_dlm.h"
-#include "../include/obd_class.h"
-#include "../include/obd.h"
+#include <lustre_errno.h>
+#include <lustre_dlm.h>
+#include <obd_class.h>
+#include <obd.h>
 
 #include "ldlm_internal.h"
 
@@ -83,6 +84,33 @@ struct ldlm_async_args {
 	struct lustre_handle lock_handle;
 };
 
+/**
+ * ldlm_request_bufsize
+ *
+ * @count:	number of ldlm handles
+ * @type:	ldlm opcode
+ *
+ * If opcode=LDLM_ENQUEUE, 1 slot is already occupied,
+ * LDLM_LOCKREQ_HANDLE -1 slots are available.
+ * Otherwise, LDLM_LOCKREQ_HANDLE slots are available.
+ *
+ * Return:	size of the request buffer
+ */
+static int ldlm_request_bufsize(int count, int type)
+{
+	int avail = LDLM_LOCKREQ_HANDLES;
+
+	if (type == LDLM_ENQUEUE)
+		avail -= LDLM_ENQUEUE_CANCEL_OFF;
+
+	if (count > avail)
+		avail = (count - avail) * sizeof(struct lustre_handle);
+	else
+		avail = 0;
+
+	return sizeof(struct ldlm_request) + avail;
+}
+
 static int ldlm_expired_completion_wait(void *data)
 {
 	struct lock_wait_data *lwd = data;
@@ -1635,7 +1663,7 @@ int ldlm_cli_cancel_list(struct list_head *cancels, int count,
 
 		if (res < 0) {
 			CDEBUG_LIMIT(res == -ESHUTDOWN ? D_DLMTRACE : D_ERROR,
-				     "ldlm_cli_cancel_list: %d\n", res);
+				     "%s: %d\n", __func__, res);
 			res = count;
 		}
 
diff --git a/drivers/staging/lustre/lustre/ldlm/ldlm_resource.c b/drivers/staging/lustre/lustre/ldlm/ldlm_resource.c
index c9ef247d9be4..c2ddf7312571 100644
--- a/drivers/staging/lustre/lustre/ldlm/ldlm_resource.c
+++ b/drivers/staging/lustre/lustre/ldlm/ldlm_resource.c
@@ -36,9 +36,9 @@
  */
 
 #define DEBUG_SUBSYSTEM S_LDLM
-#include "../include/lustre_dlm.h"
-#include "../include/lustre_fid.h"
-#include "../include/obd_class.h"
+#include <lustre_dlm.h>
+#include <lustre_fid.h>
+#include <obd_class.h>
 #include "ldlm_internal.h"
 
 struct kmem_cache *ldlm_resource_slab, *ldlm_lock_slab;
@@ -223,7 +223,7 @@ static ssize_t lru_size_show(struct kobject *kobj, struct attribute *attr,
 
 	if (ns_connect_lru_resize(ns))
 		nr = &ns->ns_nr_unused;
-	return sprintf(buf, "%u", *nr);
+	return sprintf(buf, "%u\n", *nr);
 }
 
 static ssize_t lru_size_store(struct kobject *kobj, struct attribute *attr,
@@ -318,7 +318,7 @@ static ssize_t lru_max_age_show(struct kobject *kobj, struct attribute *attr,
 	struct ldlm_namespace *ns = container_of(kobj, struct ldlm_namespace,
 						 ns_kobj);
 
-	return sprintf(buf, "%u", ns->ns_max_age);
+	return sprintf(buf, "%u\n", ns->ns_max_age);
 }
 
 static ssize_t lru_max_age_store(struct kobject *kobj, struct attribute *attr,
@@ -536,16 +536,6 @@ static void ldlm_res_hop_get_locked(struct cfs_hash *hs,
 	ldlm_resource_getref(res);
 }
 
-static void ldlm_res_hop_put_locked(struct cfs_hash *hs,
-				    struct hlist_node *hnode)
-{
-	struct ldlm_resource *res;
-
-	res = hlist_entry(hnode, struct ldlm_resource, lr_hash);
-	/* cfs_hash_for_each_nolock is the only chance we call it */
-	ldlm_resource_putref_locked(res);
-}
-
 static void ldlm_res_hop_put(struct cfs_hash *hs, struct hlist_node *hnode)
 {
 	struct ldlm_resource *res;
@@ -561,7 +551,6 @@ static struct cfs_hash_ops ldlm_ns_hash_ops = {
 	.hs_keycpy      = NULL,
 	.hs_object      = ldlm_res_hop_object,
 	.hs_get		= ldlm_res_hop_get_locked,
-	.hs_put_locked  = ldlm_res_hop_put_locked,
 	.hs_put		= ldlm_res_hop_put
 };
 
@@ -572,7 +561,6 @@ static struct cfs_hash_ops ldlm_ns_fid_hash_ops = {
 	.hs_keycpy      = NULL,
 	.hs_object      = ldlm_res_hop_object,
 	.hs_get		= ldlm_res_hop_get_locked,
-	.hs_put_locked  = ldlm_res_hop_put_locked,
 	.hs_put		= ldlm_res_hop_put
 };
 
@@ -1249,37 +1237,6 @@ int ldlm_resource_putref(struct ldlm_resource *res)
 }
 EXPORT_SYMBOL(ldlm_resource_putref);
 
-/* Returns 1 if the resource was freed, 0 if it remains. */
-int ldlm_resource_putref_locked(struct ldlm_resource *res)
-{
-	struct ldlm_namespace *ns = ldlm_res_to_ns(res);
-
-	LASSERT_ATOMIC_GT_LT(&res->lr_refcount, 0, LI_POISON);
-	CDEBUG(D_INFO, "putref res: %p count: %d\n",
-	       res, atomic_read(&res->lr_refcount) - 1);
-
-	if (atomic_dec_and_test(&res->lr_refcount)) {
-		struct cfs_hash_bd bd;
-
-		cfs_hash_bd_get(ldlm_res_to_ns(res)->ns_rs_hash,
-				&res->lr_name, &bd);
-		__ldlm_resource_putref_final(&bd, res);
-		cfs_hash_bd_unlock(ns->ns_rs_hash, &bd, 1);
-		/* NB: ns_rs_hash is created with CFS_HASH_NO_ITEMREF,
-		 * so we should never be here while calling cfs_hash_del,
-		 * cfs_hash_for_each_nolock is the only case we can get
-		 * here, which is safe to release cfs_hash_bd_lock.
-		 */
-		if (ns->ns_lvbo && ns->ns_lvbo->lvbo_free)
-			ns->ns_lvbo->lvbo_free(res);
-		kmem_cache_free(ldlm_resource_slab, res);
-
-		cfs_hash_bd_lock(ns->ns_rs_hash, &bd, 1);
-		return 1;
-	}
-	return 0;
-}
-
 /**
  * Add a lock into a given resource into specified lock list.
  */
diff --git a/drivers/staging/lustre/lustre/llite/Makefile b/drivers/staging/lustre/lustre/llite/Makefile
index 322d4fa63f5d..ef7adef4ccc5 100644
--- a/drivers/staging/lustre/lustre/llite/Makefile
+++ b/drivers/staging/lustre/lustre/llite/Makefile
@@ -1,3 +1,6 @@
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/include
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/lustre/include
+
 obj-$(CONFIG_LUSTRE_FS) += lustre.o
 lustre-y := dcache.o dir.o file.o llite_lib.o llite_nfs.o \
 	    rw.o rw26.o namei.o symlink.o llite_mmap.o range_lock.o \
diff --git a/drivers/staging/lustre/lustre/llite/dcache.c b/drivers/staging/lustre/lustre/llite/dcache.c
index d20425fb8cbe..3670fcaf373f 100644
--- a/drivers/staging/lustre/lustre/llite/dcache.c
+++ b/drivers/staging/lustre/lustre/llite/dcache.c
@@ -36,9 +36,9 @@
 
 #define DEBUG_SUBSYSTEM S_LLITE
 
-#include "../include/obd_support.h"
-#include "../include/lustre/lustre_idl.h"
-#include "../include/lustre_dlm.h"
+#include <obd_support.h>
+#include <uapi/linux/lustre/lustre_idl.h>
+#include <lustre_dlm.h>
 
 #include "llite_internal.h"
 
diff --git a/drivers/staging/lustre/lustre/llite/dir.c b/drivers/staging/lustre/lustre/llite/dir.c
index 03a72c07f57c..1db3e7f345c5 100644
--- a/drivers/staging/lustre/lustre/llite/dir.c
+++ b/drivers/staging/lustre/lustre/llite/dir.c
@@ -44,14 +44,14 @@
 
 #define DEBUG_SUBSYSTEM S_LLITE
 
-#include "../include/obd_support.h"
-#include "../include/obd_class.h"
-#include "../include/lustre/lustre_ioctl.h"
-#include "../include/lustre_lib.h"
-#include "../include/lustre_dlm.h"
-#include "../include/lustre_fid.h"
-#include "../include/lustre_kernelcomm.h"
-#include "../include/lustre_swab.h"
+#include <obd_support.h>
+#include <obd_class.h>
+#include <uapi/linux/lustre/lustre_ioctl.h>
+#include <lustre_lib.h>
+#include <lustre_dlm.h>
+#include <lustre_fid.h>
+#include <lustre_kernelcomm.h>
+#include <lustre_swab.h>
 
 #include "llite_internal.h"
 
@@ -1097,7 +1097,7 @@ static long ll_dir_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 			goto out_free;
 		}
 out_free:
-		obd_ioctl_freedata(buf, len);
+		kvfree(buf);
 		return rc;
 	}
 	case LL_IOC_LMV_SETSTRIPE: {
@@ -1147,7 +1147,7 @@ out_free:
 #endif
 		rc = ll_dir_setdirstripe(inode, lum, filename, mode);
 lmv_out_free:
-		obd_ioctl_freedata(buf, len);
+		kvfree(buf);
 		return rc;
 	}
 	case LL_IOC_LMV_SET_DEFAULT_STRIPE: {
@@ -1626,7 +1626,7 @@ out_quotactl:
 
 		rc = ll_migrate(inode, file, mdtidx, filename, namelen - 1);
 migrate_free:
-		obd_ioctl_freedata(buf, len);
+		kvfree(buf);
 
 		return rc;
 	}
diff --git a/drivers/staging/lustre/lustre/llite/file.c b/drivers/staging/lustre/lustre/llite/file.c
index ab1c85c1ed38..be665454f407 100644
--- a/drivers/staging/lustre/lustre/llite/file.c
+++ b/drivers/staging/lustre/lustre/llite/file.c
@@ -37,16 +37,16 @@
  */
 
 #define DEBUG_SUBSYSTEM S_LLITE
-#include "../include/lustre_dlm.h"
+#include <lustre_dlm.h>
 #include <linux/pagemap.h>
 #include <linux/file.h>
 #include <linux/sched.h>
 #include <linux/mount.h>
-#include "../include/lustre/ll_fiemap.h"
-#include "../include/lustre/lustre_ioctl.h"
-#include "../include/lustre_swab.h"
+#include <uapi/linux/lustre/lustre_fiemap.h>
+#include <uapi/linux/lustre/lustre_ioctl.h>
+#include <lustre_swab.h>
 
-#include "../include/cl_object.h"
+#include <cl_object.h>
 #include "llite_internal.h"
 
 static int
@@ -2364,7 +2364,7 @@ int ll_fsync(struct file *file, loff_t start, loff_t end, int datasync)
 	       PFID(ll_inode2fid(inode)), inode);
 	ll_stats_ops_tally(ll_i2sbi(inode), LPROC_LL_FSYNC, 1);
 
-	rc = filemap_write_and_wait_range(inode->i_mapping, start, end);
+	rc = file_write_and_wait_range(file, start, end);
 	inode_lock(inode);
 
 	/* catch async errors that were recorded back when async writeback
@@ -3035,9 +3035,6 @@ struct posix_acl *ll_get_acl(struct inode *inode, int type)
 	spin_lock(&lli->lli_lock);
 	/* VFS' acl_permission_check->check_acl will release the refcount */
 	acl = posix_acl_dup(lli->lli_posix_acl);
-#ifdef CONFIG_FS_POSIX_ACL
-	forget_cached_acl(inode, type);
-#endif
 	spin_unlock(&lli->lli_lock);
 
 	return acl;
diff --git a/drivers/staging/lustre/lustre/llite/glimpse.c b/drivers/staging/lustre/lustre/llite/glimpse.c
index 0143112e672d..34c2cfecf4b8 100644
--- a/drivers/staging/lustre/lustre/llite/glimpse.c
+++ b/drivers/staging/lustre/lustre/llite/glimpse.c
@@ -36,18 +36,18 @@
  *   Author: Oleg Drokin <oleg.drokin@sun.com>
  */
 
-#include "../../include/linux/libcfs/libcfs.h"
-#include "../include/obd_class.h"
-#include "../include/obd_support.h"
-#include "../include/obd.h"
+#include <linux/libcfs/libcfs.h>
+#include <obd_class.h>
+#include <obd_support.h>
+#include <obd.h>
 
-#include "../include/lustre_dlm.h"
-#include "../include/lustre_mdc.h"
+#include <lustre_dlm.h>
+#include <lustre_mdc.h>
 #include <linux/pagemap.h>
 #include <linux/file.h>
 
-#include "../include/cl_object.h"
-#include "../llite/llite_internal.h"
+#include <cl_object.h>
+#include "llite_internal.h"
 
 static const struct cl_lock_descr whole_file = {
 	.cld_start = 0,
diff --git a/drivers/staging/lustre/lustre/llite/lcommon_cl.c b/drivers/staging/lustre/lustre/llite/lcommon_cl.c
index 96515b839436..d2392e4c6872 100644
--- a/drivers/staging/lustre/lustre/llite/lcommon_cl.c
+++ b/drivers/staging/lustre/lustre/llite/lcommon_cl.c
@@ -37,24 +37,23 @@
 
 #define DEBUG_SUBSYSTEM S_LLITE
 
-#include "../../include/linux/libcfs/libcfs.h"
-# include <linux/fs.h>
-# include <linux/sched.h>
-# include <linux/mm.h>
-# include <linux/quotaops.h>
-# include <linux/highmem.h>
-# include <linux/pagemap.h>
-# include <linux/rbtree.h>
-
-#include "../include/obd.h"
-#include "../include/obd_support.h"
-#include "../include/lustre_fid.h"
-#include "../include/lustre_dlm.h"
-#include "../include/lustre_ver.h"
-#include "../include/lustre_mdc.h"
-#include "../include/cl_object.h"
-
-#include "../llite/llite_internal.h"
+#include <linux/libcfs/libcfs.h>
+#include <linux/fs.h>
+#include <linux/sched.h>
+#include <linux/mm.h>
+#include <linux/quotaops.h>
+#include <linux/highmem.h>
+#include <linux/pagemap.h>
+#include <linux/rbtree.h>
+
+#include <obd.h>
+#include <obd_support.h>
+#include <lustre_fid.h>
+#include <lustre_dlm.h>
+#include <lustre_mdc.h>
+#include <cl_object.h>
+
+#include "llite_internal.h"
 
 /*
  * ccc_ prefix stands for "Common Client Code".
diff --git a/drivers/staging/lustre/lustre/llite/lcommon_misc.c b/drivers/staging/lustre/lustre/llite/lcommon_misc.c
index 7f7f3f1648ef..422f410d95c1 100644
--- a/drivers/staging/lustre/lustre/llite/lcommon_misc.c
+++ b/drivers/staging/lustre/lustre/llite/lcommon_misc.c
@@ -34,10 +34,10 @@
  *
  */
 #define DEBUG_SUBSYSTEM S_LLITE
-#include "../include/obd_class.h"
-#include "../include/obd_support.h"
-#include "../include/obd.h"
-#include "../include/cl_object.h"
+#include <obd_class.h>
+#include <obd_support.h>
+#include <obd.h>
+#include <cl_object.h>
 
 #include "llite_internal.h"
 
diff --git a/drivers/staging/lustre/lustre/llite/llite_internal.h b/drivers/staging/lustre/lustre/llite/llite_internal.h
index cd3311abf999..0287c751e1cd 100644
--- a/drivers/staging/lustre/lustre/llite/llite_internal.h
+++ b/drivers/staging/lustre/lustre/llite/llite_internal.h
@@ -32,18 +32,18 @@
 
 #ifndef LLITE_INTERNAL_H
 #define LLITE_INTERNAL_H
-#include "../include/lustre_debug.h"
-#include "../include/lustre_ver.h"
-#include "../include/lustre_disk.h"	/* for s2sbi */
-#include "../include/lustre_linkea.h"
+#include <lustre_debug.h>
+#include <uapi/linux/lustre/lustre_ver.h>
+#include <lustre_disk.h>	/* for s2sbi */
+#include <lustre_linkea.h>
 
 /* for struct cl_lock_descr and struct cl_io */
-#include "../include/lustre_patchless_compat.h"
-#include "../include/lustre_compat.h"
-#include "../include/cl_object.h"
-#include "../include/lustre_lmv.h"
-#include "../include/lustre_mdc.h"
-#include "../include/lustre_intent.h"
+#include <lustre_patchless_compat.h>
+#include <lustre_compat.h>
+#include <cl_object.h>
+#include <lustre_lmv.h>
+#include <lustre_mdc.h>
+#include <lustre_intent.h>
 #include <linux/compat.h>
 #include <linux/namei.h>
 #include <linux/xattr.h>
diff --git a/drivers/staging/lustre/lustre/llite/llite_lib.c b/drivers/staging/lustre/lustre/llite/llite_lib.c
index 974a05d6c969..25393e3a0fe8 100644
--- a/drivers/staging/lustre/lustre/llite/llite_lib.c
+++ b/drivers/staging/lustre/lustre/llite/llite_lib.c
@@ -41,15 +41,15 @@
 #include <linux/types.h>
 #include <linux/mm.h>
 
-#include "../include/lustre/lustre_ioctl.h"
-#include "../include/lustre_ha.h"
-#include "../include/lustre_dlm.h"
-#include "../include/lprocfs_status.h"
-#include "../include/lustre_disk.h"
-#include "../include/lustre_param.h"
-#include "../include/lustre_log.h"
-#include "../include/cl_object.h"
-#include "../include/obd_cksum.h"
+#include <uapi/linux/lustre/lustre_ioctl.h>
+#include <lustre_ha.h>
+#include <lustre_dlm.h>
+#include <lprocfs_status.h>
+#include <lustre_disk.h>
+#include <uapi/linux/lustre/lustre_param.h>
+#include <lustre_log.h>
+#include <cl_object.h>
+#include <obd_cksum.h>
 #include "llite_internal.h"
 
 struct kmem_cache *ll_file_data_slab;
@@ -210,7 +210,7 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt,
 	data->ocd_ibits_known = MDS_INODELOCK_FULL;
 	data->ocd_version = LUSTRE_VERSION_CODE;
 
-	if (sb->s_flags & MS_RDONLY)
+	if (sb_rdonly(sb))
 		data->ocd_connect_flags |= OBD_CONNECT_RDONLY;
 	if (sbi->ll_flags & LL_SBI_USER_XATTR)
 		data->ocd_connect_flags |= OBD_CONNECT_XATTR;
@@ -222,9 +222,6 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt,
 	else
 		sbi->ll_fop = &ll_file_operations_noflock;
 
-	/* real client */
-	data->ocd_connect_flags |= OBD_CONNECT_REAL;
-
 	/* always ping even if server suppress_pings */
 	if (sbi->ll_flags & LL_SBI_ALWAYS_PING)
 		data->ocd_connect_flags &= ~OBD_CONNECT_PINGLESS;
@@ -1319,6 +1316,7 @@ void ll_clear_inode(struct inode *inode)
 	ll_xattr_cache_destroy(inode);
 
 #ifdef CONFIG_FS_POSIX_ACL
+	forget_all_cached_acls(inode);
 	if (lli->lli_posix_acl) {
 		posix_acl_release(lli->lli_posix_acl);
 		lli->lli_posix_acl = NULL;
@@ -2033,7 +2031,7 @@ int ll_remount_fs(struct super_block *sb, int *flags, char *data)
 	int err;
 	__u32 read_only;
 
-	if ((*flags & MS_RDONLY) != (sb->s_flags & MS_RDONLY)) {
+	if ((bool)(*flags & MS_RDONLY) != sb_rdonly(sb)) {
 		read_only = *flags & MS_RDONLY;
 		err = obd_set_info_async(NULL, sbi->ll_md_exp,
 					 sizeof(KEY_READ_ONLY),
@@ -2233,8 +2231,7 @@ int ll_obd_statfs(struct inode *inode, void __user *arg)
 	if (rc)
 		goto out_statfs;
 out_statfs:
-	if (buf)
-		obd_ioctl_freedata(buf, len);
+	kvfree(buf);
 	return rc;
 }
 
@@ -2543,7 +2540,7 @@ static int ll_linkea_decode(struct linkea_data *ldata, unsigned int linkno,
 	unsigned int idx;
 	int rc;
 
-	rc = linkea_init(ldata);
+	rc = linkea_init_with_rec(ldata);
 	if (rc < 0)
 		return rc;
 
diff --git a/drivers/staging/lustre/lustre/llite/lproc_llite.c b/drivers/staging/lustre/lustre/llite/lproc_llite.c
index aeae6670e262..e3bd2d18eac5 100644
--- a/drivers/staging/lustre/lustre/llite/lproc_llite.c
+++ b/drivers/staging/lustre/lustre/llite/lproc_llite.c
@@ -31,9 +31,9 @@
  */
 #define DEBUG_SUBSYSTEM S_LLITE
 
-#include "../include/lprocfs_status.h"
+#include <lprocfs_status.h>
 #include <linux/seq_file.h>
-#include "../include/obd_support.h"
+#include <obd_support.h>
 
 #include "llite_internal.h"
 #include "vvp_internal.h"
diff --git a/drivers/staging/lustre/lustre/llite/namei.c b/drivers/staging/lustre/lustre/llite/namei.c
index a208a8b02c2c..5cc2b3255207 100644
--- a/drivers/staging/lustre/lustre/llite/namei.c
+++ b/drivers/staging/lustre/lustre/llite/namei.c
@@ -40,10 +40,9 @@
 
 #define DEBUG_SUBSYSTEM S_LLITE
 
-#include "../include/obd_support.h"
-#include "../include/lustre_fid.h"
-#include "../include/lustre_dlm.h"
-#include "../include/lustre_ver.h"
+#include <obd_support.h>
+#include <lustre_fid.h>
+#include <lustre_dlm.h>
 #include "llite_internal.h"
 
 static int ll_create_it(struct inode *dir, struct dentry *dentry,
@@ -490,7 +489,7 @@ static int ll_lookup_it_finish(struct ptlrpc_request *request,
 	*de = alias;
 
 	if (!it_disposition(it, DISP_LOOKUP_NEG)) {
-		/* we have lookup look - unhide dentry */
+		/* We have the "lookup" lock, so unhide dentry */
 		if (bits & MDS_INODELOCK_LOOKUP)
 			d_lustre_revalidate(*de);
 	} else if (!it_disposition(it, DISP_OPEN_CREATE)) {
@@ -562,8 +561,7 @@ static struct dentry *ll_lookup_it(struct inode *parent, struct dentry *dentry,
 		}
 	}
 
-	if (it->it_op & IT_OPEN && it->it_flags & FMODE_WRITE &&
-	    dentry->d_sb->s_flags & MS_RDONLY)
+	if (it->it_op & IT_OPEN && it->it_flags & FMODE_WRITE && sb_rdonly(dentry->d_sb))
 		return ERR_PTR(-EROFS);
 
 	if (it->it_op & IT_CREAT)
diff --git a/drivers/staging/lustre/lustre/llite/range_lock.c b/drivers/staging/lustre/lustre/llite/range_lock.c
index 161391b6fb36..a32598bacdfb 100644
--- a/drivers/staging/lustre/lustre/llite/range_lock.c
+++ b/drivers/staging/lustre/lustre/llite/range_lock.c
@@ -34,7 +34,7 @@
  * Author: Bobi Jam <bobijam.xu@intel.com>
  */
 #include "range_lock.h"
-#include "../include/lustre/lustre_user.h"
+#include <uapi/linux/lustre/lustre_idl.h>
 
 /**
  * Initialize a range lock tree
diff --git a/drivers/staging/lustre/lustre/llite/range_lock.h b/drivers/staging/lustre/lustre/llite/range_lock.h
index 779091ccec4e..1e1519b1e006 100644
--- a/drivers/staging/lustre/lustre/llite/range_lock.h
+++ b/drivers/staging/lustre/lustre/llite/range_lock.h
@@ -36,8 +36,8 @@
 #ifndef _RANGE_LOCK_H
 #define _RANGE_LOCK_H
 
-#include "../../include/linux/libcfs/libcfs.h"
-#include "../include/interval_tree.h"
+#include <linux/libcfs/libcfs.h>
+#include <interval_tree.h>
 
 struct range_lock {
 	struct interval_node	rl_node;
diff --git a/drivers/staging/lustre/lustre/llite/rw.c b/drivers/staging/lustre/lustre/llite/rw.c
index 1bac51f882a7..e72090572bcc 100644
--- a/drivers/staging/lustre/lustre/llite/rw.c
+++ b/drivers/staging/lustre/lustre/llite/rw.c
@@ -51,7 +51,7 @@
 
 #define DEBUG_SUBSYSTEM S_LLITE
 
-#include "../include/obd_cksum.h"
+#include <obd_cksum.h>
 #include "llite_internal.h"
 
 static void ll_ra_stats_inc_sbi(struct ll_sb_info *sbi, enum ra_stat which);
@@ -115,7 +115,7 @@ void ll_ra_count_put(struct ll_sb_info *sbi, unsigned long len)
 
 static void ll_ra_stats_inc_sbi(struct ll_sb_info *sbi, enum ra_stat which)
 {
-	LASSERTF(which >= 0 && which < _NR_RA_STAT, "which: %u\n", which);
+	LASSERTF(which < _NR_RA_STAT, "which: %u\n", which);
 	lprocfs_counter_incr(sbi->ll_ra_stats, which);
 }
 
diff --git a/drivers/staging/lustre/lustre/llite/statahead.c b/drivers/staging/lustre/lustre/llite/statahead.c
index 9bbca018a5fe..ea9d59f07b78 100644
--- a/drivers/staging/lustre/lustre/llite/statahead.c
+++ b/drivers/staging/lustre/lustre/llite/statahead.c
@@ -38,8 +38,8 @@
 
 #define DEBUG_SUBSYSTEM S_LLITE
 
-#include "../include/obd_support.h"
-#include "../include/lustre_dlm.h"
+#include <obd_support.h>
+#include <lustre_dlm.h>
 #include "llite_internal.h"
 
 #define SA_OMITTED_ENTRY_MAX 8ULL
diff --git a/drivers/staging/lustre/lustre/llite/super25.c b/drivers/staging/lustre/lustre/llite/super25.c
index 56f4b10624ce..0da4af81b830 100644
--- a/drivers/staging/lustre/lustre/llite/super25.c
+++ b/drivers/staging/lustre/lustre/llite/super25.c
@@ -34,11 +34,11 @@
 
 #include <linux/module.h>
 #include <linux/types.h>
-#include "../include/lustre_ha.h"
-#include "../include/lustre_dlm.h"
+#include <lustre_ha.h>
+#include <lustre_dlm.h>
 #include <linux/init.h>
 #include <linux/fs.h>
-#include "../include/lprocfs_status.h"
+#include <lprocfs_status.h>
 #include "llite_internal.h"
 
 static struct kmem_cache *ll_inode_cachep;
diff --git a/drivers/staging/lustre/lustre/llite/vvp_dev.c b/drivers/staging/lustre/lustre/llite/vvp_dev.c
index 8e45672b4617..f9d9a161bd4e 100644
--- a/drivers/staging/lustre/lustre/llite/vvp_dev.c
+++ b/drivers/staging/lustre/lustre/llite/vvp_dev.c
@@ -37,7 +37,7 @@
 
 #define DEBUG_SUBSYSTEM S_LLITE
 
-#include "../include/obd.h"
+#include <obd.h>
 #include "llite_internal.h"
 #include "vvp_internal.h"
 
@@ -591,9 +591,10 @@ static void *vvp_pgcache_start(struct seq_file *f, loff_t *pos)
 	env = cl_env_get(&refcheck);
 	if (!IS_ERR(env)) {
 		sbi = f->private;
-		if (sbi->ll_site->ls_obj_hash->hs_cur_bits > 64 - PGC_OBJ_SHIFT)
+		if (sbi->ll_site->ls_obj_hash->hs_cur_bits >
+		    64 - PGC_OBJ_SHIFT) {
 			pos = ERR_PTR(-EFBIG);
-		else {
+		} else {
 			*pos = vvp_pgcache_find(env, &sbi->ll_cl->cd_lu_dev,
 						*pos);
 			if (*pos == ~0ULL)
diff --git a/drivers/staging/lustre/lustre/llite/vvp_internal.h b/drivers/staging/lustre/lustre/llite/vvp_internal.h
index f40fd7f115d1..adce0ff4ae44 100644
--- a/drivers/staging/lustre/lustre/llite/vvp_internal.h
+++ b/drivers/staging/lustre/lustre/llite/vvp_internal.h
@@ -37,8 +37,8 @@
 #ifndef VVP_INTERNAL_H
 #define VVP_INTERNAL_H
 
-#include "../include/lustre/lustre_idl.h"
-#include "../include/cl_object.h"
+#include <uapi/linux/lustre/lustre_idl.h>
+#include <cl_object.h>
 
 enum obd_notify_event;
 struct inode;
diff --git a/drivers/staging/lustre/lustre/llite/vvp_io.c b/drivers/staging/lustre/lustre/llite/vvp_io.c
index c5ba265ef6ad..c83853fa1bb4 100644
--- a/drivers/staging/lustre/lustre/llite/vvp_io.c
+++ b/drivers/staging/lustre/lustre/llite/vvp_io.c
@@ -37,7 +37,7 @@
 
 #define DEBUG_SUBSYSTEM S_LLITE
 
-#include "../include/obd.h"
+#include <obd.h>
 
 #include "llite_internal.h"
 #include "vvp_internal.h"
diff --git a/drivers/staging/lustre/lustre/llite/vvp_lock.c b/drivers/staging/lustre/lustre/llite/vvp_lock.c
index 07eb26cc43f5..e522f7c00617 100644
--- a/drivers/staging/lustre/lustre/llite/vvp_lock.c
+++ b/drivers/staging/lustre/lustre/llite/vvp_lock.c
@@ -36,7 +36,7 @@
 
 #define DEBUG_SUBSYSTEM S_LLITE
 
-#include "../include/obd_support.h"
+#include <obd_support.h>
 
 #include "vvp_internal.h"
 
diff --git a/drivers/staging/lustre/lustre/llite/vvp_object.c b/drivers/staging/lustre/lustre/llite/vvp_object.c
index 9bfd72e514d1..3953750b334e 100644
--- a/drivers/staging/lustre/lustre/llite/vvp_object.c
+++ b/drivers/staging/lustre/lustre/llite/vvp_object.c
@@ -36,9 +36,9 @@
 
 #define DEBUG_SUBSYSTEM S_LLITE
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
-#include "../include/obd.h"
+#include <obd.h>
 
 #include "llite_internal.h"
 #include "vvp_internal.h"
diff --git a/drivers/staging/lustre/lustre/llite/xattr.c b/drivers/staging/lustre/lustre/llite/xattr.c
index bd30abdecfb9..0be55623bac4 100644
--- a/drivers/staging/lustre/lustre/llite/xattr.c
+++ b/drivers/staging/lustre/lustre/llite/xattr.c
@@ -33,13 +33,13 @@
 #include <linux/fs.h>
 #include <linux/sched.h>
 #include <linux/mm.h>
+#include <linux/xattr.h>
 #include <linux/selinux.h>
 
 #define DEBUG_SUBSYSTEM S_LLITE
 
-#include "../include/obd_support.h"
-#include "../include/lustre_dlm.h"
-#include "../include/lustre_ver.h"
+#include <obd_support.h>
+#include <lustre_dlm.h>
 
 #include "llite_internal.h"
 
diff --git a/drivers/staging/lustre/lustre/llite/xattr_cache.c b/drivers/staging/lustre/lustre/llite/xattr_cache.c
index 82cf4211cc0f..80ee3920481a 100644
--- a/drivers/staging/lustre/lustre/llite/xattr_cache.c
+++ b/drivers/staging/lustre/lustre/llite/xattr_cache.c
@@ -12,9 +12,8 @@
 #include <linux/fs.h>
 #include <linux/sched.h>
 #include <linux/mm.h>
-#include "../include/obd_support.h"
-#include "../include/lustre_dlm.h"
-#include "../include/lustre_ver.h"
+#include <obd_support.h>
+#include <lustre_dlm.h>
 #include "llite_internal.h"
 
 /* If we ever have hundreds of extended attributes, we might want to consider
diff --git a/drivers/staging/lustre/lustre/llite/xattr_security.c b/drivers/staging/lustre/lustre/llite/xattr_security.c
index d61d8018001a..391fb25ac31d 100644
--- a/drivers/staging/lustre/lustre/llite/xattr_security.c
+++ b/drivers/staging/lustre/lustre/llite/xattr_security.c
@@ -28,7 +28,10 @@
  * lustre/llite/xattr_security.c
  * Handler for storing security labels as extended attributes.
  */
+
+#include <linux/types.h>
 #include <linux/security.h>
+#include <linux/selinux.h>
 #include <linux/xattr.h>
 #include "llite_internal.h"
 
@@ -48,19 +51,23 @@ static int
 ll_initxattrs(struct inode *inode, const struct xattr *xattr_array,
 	      void *fs_info)
 {
-	const struct xattr_handler *handler;
 	struct dentry *dentry = fs_info;
 	const struct xattr *xattr;
 	int err = 0;
 
-	handler = get_xattr_type(XATTR_SECURITY_PREFIX);
-	if (!handler)
-		return -ENXIO;
-
 	for (xattr = xattr_array; xattr->name; xattr++) {
-		err = handler->set(handler, dentry, inode, xattr->name,
-				   xattr->value, xattr->value_len,
-				   XATTR_CREATE);
+		char *full_name;
+
+		full_name = kasprintf(GFP_KERNEL, "%s%s",
+				      XATTR_SECURITY_PREFIX, xattr->name);
+		if (!full_name) {
+			err = -ENOMEM;
+			break;
+		}
+
+		err = __vfs_setxattr(dentry, inode, full_name, xattr->value,
+				     xattr->value_len, XATTR_CREATE);
+		kfree(full_name);
 		if (err < 0)
 			break;
 	}
diff --git a/drivers/staging/lustre/lustre/lmv/Makefile b/drivers/staging/lustre/lustre/lmv/Makefile
index 1a24299791d7..91c99114aa13 100644
--- a/drivers/staging/lustre/lustre/lmv/Makefile
+++ b/drivers/staging/lustre/lustre/lmv/Makefile
@@ -1,2 +1,5 @@
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/include
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/lustre/include
+
 obj-$(CONFIG_LUSTRE_FS) += lmv.o
 lmv-y := lmv_obd.o lmv_intent.o lmv_fld.o lproc_lmv.o
diff --git a/drivers/staging/lustre/lustre/lmv/lmv_fld.c b/drivers/staging/lustre/lustre/lmv/lmv_fld.c
index 6f8070fb3226..5937468080b8 100644
--- a/drivers/staging/lustre/lustre/lmv/lmv_fld.c
+++ b/drivers/staging/lustre/lustre/lmv/lmv_fld.c
@@ -37,14 +37,13 @@
 #include <asm/div64.h>
 #include <linux/seq_file.h>
 
-#include "../include/obd_support.h"
-#include "../include/lustre/lustre_idl.h"
-#include "../include/lustre_fid.h"
-#include "../include/lustre_lib.h"
-#include "../include/lustre_net.h"
-#include "../include/lustre_dlm.h"
-#include "../include/obd_class.h"
-#include "../include/lprocfs_status.h"
+#include <obd_support.h>
+#include <lustre_fid.h>
+#include <lustre_lib.h>
+#include <lustre_net.h>
+#include <lustre_dlm.h>
+#include <obd_class.h>
+#include <lprocfs_status.h>
 #include "lmv_internal.h"
 
 int lmv_fld_lookup(struct lmv_obd *lmv, const struct lu_fid *fid, u32 *mds)
diff --git a/drivers/staging/lustre/lustre/lmv/lmv_intent.c b/drivers/staging/lustre/lustre/lmv/lmv_intent.c
index f49db6c23217..22c247a7d8ca 100644
--- a/drivers/staging/lustre/lustre/lmv/lmv_intent.c
+++ b/drivers/staging/lustre/lustre/lmv/lmv_intent.c
@@ -37,15 +37,14 @@
 #include <asm/div64.h>
 #include <linux/seq_file.h>
 #include <linux/namei.h>
-#include "../include/lustre_intent.h"
-#include "../include/obd_support.h"
-#include "../include/lustre/lustre_idl.h"
-#include "../include/lustre_lib.h"
-#include "../include/lustre_net.h"
-#include "../include/lustre_dlm.h"
-#include "../include/lustre_mdc.h"
-#include "../include/obd_class.h"
-#include "../include/lprocfs_status.h"
+#include <lustre_intent.h>
+#include <obd_support.h>
+#include <lustre_lib.h>
+#include <lustre_net.h>
+#include <lustre_dlm.h>
+#include <lustre_mdc.h>
+#include <obd_class.h>
+#include <lprocfs_status.h>
 #include "lmv_internal.h"
 
 static int lmv_intent_remote(struct obd_export *exp, struct lookup_intent *it,
@@ -474,7 +473,6 @@ int lmv_intent_lock(struct obd_export *exp, struct md_op_data *op_data,
 		    ldlm_blocking_callback cb_blocking,
 		    __u64 extra_lock_flags)
 {
-	struct obd_device *obd = exp->exp_obd;
 	int		rc;
 
 	LASSERT(fid_is_sane(&op_data->op_fid1));
@@ -484,10 +482,6 @@ int lmv_intent_lock(struct obd_export *exp, struct md_op_data *op_data,
 	       (int)op_data->op_namelen, op_data->op_name,
 	       PFID(&op_data->op_fid1));
 
-	rc = lmv_check_connect(obd);
-	if (rc)
-		return rc;
-
 	if (it->it_op & (IT_LOOKUP | IT_GETATTR | IT_LAYOUT))
 		rc = lmv_intent_lookup(exp, op_data, it, reqp, cb_blocking,
 				       extra_lock_flags);
diff --git a/drivers/staging/lustre/lustre/lmv/lmv_internal.h b/drivers/staging/lustre/lustre/lmv/lmv_internal.h
index 12731a17e263..a0475231dd90 100644
--- a/drivers/staging/lustre/lustre/lmv/lmv_internal.h
+++ b/drivers/staging/lustre/lustre/lmv/lmv_internal.h
@@ -33,17 +33,15 @@
 #ifndef _LMV_INTERNAL_H_
 #define _LMV_INTERNAL_H_
 
-#include "../include/lustre/lustre_idl.h"
-#include "../include/obd.h"
-#include "../include/lustre_lmv.h"
+#include <uapi/linux/lustre/lustre_idl.h>
+#include <obd.h>
+#include <lustre_lmv.h>
 
 #define LMV_MAX_TGT_COUNT 128
 
 #define LL_IT2STR(it)					\
 	((it) ? ldlm_it2str((it)->it_op) : "0")
 
-int lmv_check_connect(struct obd_device *obd);
-
 int lmv_intent_lock(struct obd_export *exp, struct md_op_data *op_data,
 		    struct lookup_intent *it, struct ptlrpc_request **reqp,
 		    ldlm_blocking_callback cb_blocking,
diff --git a/drivers/staging/lustre/lustre/lmv/lmv_obd.c b/drivers/staging/lustre/lustre/lmv/lmv_obd.c
index 64fcaef0bacd..6e16c930a021 100644
--- a/drivers/staging/lustre/lustre/lmv/lmv_obd.c
+++ b/drivers/staging/lustre/lustre/lmv/lmv_obd.c
@@ -41,18 +41,19 @@
 #include <linux/namei.h>
 #include <linux/uaccess.h>
 
-#include "../include/lustre/lustre_idl.h"
-#include "../include/obd_support.h"
-#include "../include/lustre_net.h"
-#include "../include/obd_class.h"
-#include "../include/lustre_lmv.h"
-#include "../include/lprocfs_status.h"
-#include "../include/cl_object.h"
-#include "../include/lustre_fid.h"
-#include "../include/lustre/lustre_ioctl.h"
-#include "../include/lustre_kernelcomm.h"
+#include <obd_support.h>
+#include <lustre_net.h>
+#include <obd_class.h>
+#include <lustre_lmv.h>
+#include <lprocfs_status.h>
+#include <cl_object.h>
+#include <lustre_fid.h>
+#include <uapi/linux/lustre/lustre_ioctl.h>
+#include <lustre_kernelcomm.h>
 #include "lmv_internal.h"
 
+static int lmv_check_connect(struct obd_device *obd);
+
 static void lmv_activate_target(struct lmv_obd *lmv,
 				struct lmv_tgt_desc *tgt,
 				int activate)
@@ -183,59 +184,44 @@ static int lmv_notify(struct obd_device *obd, struct obd_device *watched,
 	return rc;
 }
 
-/**
- * This is fake connect function. Its purpose is to initialize lmv and say
- * caller that everything is okay. Real connection will be performed later.
- */
 static int lmv_connect(const struct lu_env *env,
-		       struct obd_export **exp, struct obd_device *obd,
+		       struct obd_export **pexp, struct obd_device *obd,
 		       struct obd_uuid *cluuid, struct obd_connect_data *data,
 		       void *localdata)
 {
 	struct lmv_obd	*lmv = &obd->u.lmv;
 	struct lustre_handle  conn = { 0 };
+	struct obd_export *exp;
 	int		    rc = 0;
 
-	/*
-	 * We don't want to actually do the underlying connections more than
-	 * once, so keep track.
-	 */
-	lmv->refcount++;
-	if (lmv->refcount > 1) {
-		*exp = NULL;
-		return 0;
-	}
-
 	rc = class_connect(&conn, obd, cluuid);
 	if (rc) {
 		CERROR("class_connection() returned %d\n", rc);
 		return rc;
 	}
 
-	*exp = class_conn2export(&conn);
-	class_export_get(*exp);
+	exp = class_conn2export(&conn);
 
-	lmv->exp = *exp;
 	lmv->connected = 0;
 	lmv->cluuid = *cluuid;
-
-	if (data)
-		lmv->conn_data = *data;
+	lmv->conn_data = *data;
 
 	lmv->lmv_tgts_kobj = kobject_create_and_add("target_obds",
 						    &obd->obd_kobj);
-	/*
-	 * All real clients should perform actual connection right away, because
-	 * it is possible, that LMV will not have opportunity to connect targets
-	 * and MDC stuff will be called directly, for instance while reading
-	 * ../mdc/../kbytesfree procfs file, etc.
-	 */
-	if (data && data->ocd_connect_flags & OBD_CONNECT_REAL)
-		rc = lmv_check_connect(obd);
+	rc = lmv_check_connect(obd);
+	if (rc)
+		goto out_sysfs;
+
+	*pexp = exp;
 
-	if (rc && lmv->lmv_tgts_kobj)
+	return rc;
+
+out_sysfs:
+	if (lmv->lmv_tgts_kobj)
 		kobject_put(lmv->lmv_tgts_kobj);
 
+	class_disconnect(exp);
+
 	return rc;
 }
 
@@ -475,7 +461,7 @@ static int lmv_add_target(struct obd_device *obd, struct obd_uuid *uuidp,
 	return rc;
 }
 
-int lmv_check_connect(struct obd_device *obd)
+static int lmv_check_connect(struct obd_device *obd)
 {
 	struct lmv_obd       *lmv = &obd->u.lmv;
 	struct lmv_tgt_desc  *tgt;
@@ -519,7 +505,6 @@ int lmv_check_connect(struct obd_device *obd)
 			goto out_disc;
 	}
 
-	class_export_put(lmv->exp);
 	lmv->connected = 1;
 	easize = lmv_mds_md_size(lmv->desc.ld_tgt_count, LMV_MAGIC);
 	lmv_init_ea_size(obd->obd_self_export, easize, 0);
@@ -543,7 +528,7 @@ int lmv_check_connect(struct obd_device *obd)
 			}
 		}
 	}
-	class_disconnect(lmv->exp);
+
 	mutex_unlock(&lmv->lmv_init_mutex);
 	return rc;
 }
@@ -598,13 +583,6 @@ static int lmv_disconnect(struct obd_export *exp)
 	if (!lmv->tgts)
 		goto out_local;
 
-	/*
-	 * Only disconnect the underlying layers on the final disconnect.
-	 */
-	lmv->refcount--;
-	if (lmv->refcount != 0)
-		goto out_local;
-
 	for (i = 0; i < lmv->desc.ld_tgt_count; i++) {
 		if (!lmv->tgts[i] || !lmv->tgts[i]->ltd_exp)
 			continue;
@@ -623,8 +601,7 @@ out_local:
 	if (!lmv->connected)
 		class_export_put(exp);
 	rc = class_disconnect(exp);
-	if (lmv->refcount == 0)
-		lmv->connected = 0;
+	lmv->connected = 0;
 	return rc;
 }
 
@@ -657,8 +634,8 @@ repeat_fid2path:
 		char *ptr;
 
 		ori_gf = karg;
-		if (strlen(ori_gf->gf_path) +
-		    strlen(gf->gf_path) > ori_gf->gf_pathlen) {
+		if (strlen(ori_gf->gf_path) + 1 +
+		    strlen(gf->gf_path) + 1 > ori_gf->gf_pathlen) {
 			rc = -EOVERFLOW;
 			goto out_fid2path;
 		}
@@ -1122,7 +1099,8 @@ hsm_req_err:
 			err = obd_iocontrol(cmd, tgt->ltd_exp, len, karg, uarg);
 			if (err) {
 				if (tgt->ltd_active) {
-					CERROR("error: iocontrol MDC %s on MDTidx %d cmd %x: err = %d\n",
+					CERROR("%s: error: iocontrol MDC %s on MDTidx %d cmd %x: err = %d\n",
+					       lmv2obd_dev(lmv)->obd_name,
 					       tgt->ltd_uuid.uuid, i, cmd, err);
 					if (!rc)
 						rc = err;
@@ -1368,10 +1346,6 @@ static int lmv_statfs(const struct lu_env *env, struct obd_export *exp,
 	int		    rc = 0;
 	u32 i;
 
-	rc = lmv_check_connect(obd);
-	if (rc)
-		return rc;
-
 	temp = kzalloc(sizeof(*temp), GFP_NOFS);
 	if (!temp)
 		return -ENOMEM;
@@ -1418,11 +1392,6 @@ static int lmv_getstatus(struct obd_export *exp,
 {
 	struct obd_device    *obd = exp->exp_obd;
 	struct lmv_obd       *lmv = &obd->u.lmv;
-	int		   rc;
-
-	rc = lmv_check_connect(obd);
-	if (rc)
-		return rc;
 
 	return md_getstatus(lmv->tgts[0]->ltd_exp, fid);
 }
@@ -1435,11 +1404,6 @@ static int lmv_getxattr(struct obd_export *exp, const struct lu_fid *fid,
 	struct obd_device      *obd = exp->exp_obd;
 	struct lmv_obd	 *lmv = &obd->u.lmv;
 	struct lmv_tgt_desc    *tgt;
-	int		     rc;
-
-	rc = lmv_check_connect(obd);
-	if (rc)
-		return rc;
 
 	tgt = lmv_find_target(lmv, fid);
 	if (IS_ERR(tgt))
@@ -1458,11 +1422,6 @@ static int lmv_setxattr(struct obd_export *exp, const struct lu_fid *fid,
 	struct obd_device      *obd = exp->exp_obd;
 	struct lmv_obd	 *lmv = &obd->u.lmv;
 	struct lmv_tgt_desc    *tgt;
-	int		     rc;
-
-	rc = lmv_check_connect(obd);
-	if (rc)
-		return rc;
 
 	tgt = lmv_find_target(lmv, fid);
 	if (IS_ERR(tgt))
@@ -1479,11 +1438,6 @@ static int lmv_getattr(struct obd_export *exp, struct md_op_data *op_data,
 	struct obd_device       *obd = exp->exp_obd;
 	struct lmv_obd	  *lmv = &obd->u.lmv;
 	struct lmv_tgt_desc     *tgt;
-	int		      rc;
-
-	rc = lmv_check_connect(obd);
-	if (rc)
-		return rc;
 
 	tgt = lmv_find_target(lmv, &op_data->op_fid1);
 	if (IS_ERR(tgt))
@@ -1502,11 +1456,6 @@ static int lmv_null_inode(struct obd_export *exp, const struct lu_fid *fid)
 	struct obd_device   *obd = exp->exp_obd;
 	struct lmv_obd      *lmv = &obd->u.lmv;
 	u32 i;
-	int		  rc;
-
-	rc = lmv_check_connect(obd);
-	if (rc)
-		return rc;
 
 	CDEBUG(D_INODE, "CBDATA for " DFID "\n", PFID(fid));
 
@@ -1530,11 +1479,6 @@ static int lmv_close(struct obd_export *exp, struct md_op_data *op_data,
 	struct obd_device     *obd = exp->exp_obd;
 	struct lmv_obd	*lmv = &obd->u.lmv;
 	struct lmv_tgt_desc   *tgt;
-	int		    rc;
-
-	rc = lmv_check_connect(obd);
-	if (rc)
-		return rc;
 
 	tgt = lmv_find_target(lmv, &op_data->op_fid1);
 	if (IS_ERR(tgt))
@@ -1661,10 +1605,6 @@ static int lmv_create(struct obd_export *exp, struct md_op_data *op_data,
 	struct lmv_tgt_desc     *tgt;
 	int		      rc;
 
-	rc = lmv_check_connect(obd);
-	if (rc)
-		return rc;
-
 	if (!lmv->desc.ld_active_tgt_count)
 		return -EIO;
 
@@ -1718,11 +1658,6 @@ lmv_enqueue(struct obd_export *exp, struct ldlm_enqueue_info *einfo,
 	struct obd_device	*obd = exp->exp_obd;
 	struct lmv_obd	   *lmv = &obd->u.lmv;
 	struct lmv_tgt_desc      *tgt;
-	int		       rc;
-
-	rc = lmv_check_connect(obd);
-	if (rc)
-		return rc;
 
 	CDEBUG(D_INODE, "ENQUEUE '%s' on " DFID "\n",
 	       LL_IT2STR(it), PFID(&op_data->op_fid1));
@@ -1749,10 +1684,6 @@ lmv_getattr_name(struct obd_export *exp, struct md_op_data *op_data,
 	struct mdt_body	 *body;
 	int		      rc;
 
-	rc = lmv_check_connect(obd);
-	if (rc)
-		return rc;
-
 	tgt = lmv_locate_mds(lmv, op_data, &op_data->op_fid1);
 	if (IS_ERR(tgt))
 		return PTR_ERR(tgt);
@@ -1845,10 +1776,6 @@ static int lmv_link(struct obd_export *exp, struct md_op_data *op_data,
 	struct lmv_tgt_desc     *tgt;
 	int		      rc;
 
-	rc = lmv_check_connect(obd);
-	if (rc)
-		return rc;
-
 	LASSERT(op_data->op_namelen != 0);
 
 	CDEBUG(D_INODE, "LINK " DFID ":%*s to " DFID "\n",
@@ -1907,10 +1834,6 @@ static int lmv_rename(struct obd_export *exp, struct md_op_data *op_data,
 	       (int)newlen, new, PFID(&op_data->op_fid2),
 	       op_data->op_mea2 ? op_data->op_mea2->lsm_md_stripe_count : 0);
 
-	rc = lmv_check_connect(obd);
-	if (rc)
-		return rc;
-
 	op_data->op_fsuid = from_kuid(&init_user_ns, current_fsuid());
 	op_data->op_fsgid = from_kgid(&init_user_ns, current_fsgid());
 	op_data->op_cap = cfs_curproc_cap_pack();
@@ -2063,11 +1986,6 @@ static int lmv_setattr(struct obd_export *exp, struct md_op_data *op_data,
 	struct obd_device       *obd = exp->exp_obd;
 	struct lmv_obd	  *lmv = &obd->u.lmv;
 	struct lmv_tgt_desc     *tgt;
-	int		      rc;
-
-	rc = lmv_check_connect(obd);
-	if (rc)
-		return rc;
 
 	CDEBUG(D_INODE, "SETATTR for " DFID ", valid 0x%x\n",
 	       PFID(&op_data->op_fid1), op_data->op_attr.ia_valid);
@@ -2086,11 +2004,6 @@ static int lmv_sync(struct obd_export *exp, const struct lu_fid *fid,
 	struct obd_device	 *obd = exp->exp_obd;
 	struct lmv_obd	    *lmv = &obd->u.lmv;
 	struct lmv_tgt_desc       *tgt;
-	int			rc;
-
-	rc = lmv_check_connect(obd);
-	if (rc)
-		return rc;
 
 	tgt = lmv_find_target(lmv, fid);
 	if (IS_ERR(tgt))
@@ -2272,7 +2185,6 @@ static int lmv_read_striped_page(struct obd_export *exp,
 {
 	struct inode *master_inode = op_data->op_data;
 	struct lu_fid master_fid = op_data->op_fid1;
-	struct obd_device *obd = exp->exp_obd;
 	__u64 hash_offset = offset;
 	__u32 ldp_flags;
 	struct page *min_ent_page = NULL;
@@ -2286,10 +2198,6 @@ static int lmv_read_striped_page(struct obd_export *exp,
 	void *area;
 	int rc;
 
-	rc = lmv_check_connect(obd);
-	if (rc)
-		return rc;
-
 	/*
 	 * Allocate a page and read entries from all of stripes and fill
 	 * the page by hash order
@@ -2408,11 +2316,6 @@ static int lmv_read_page(struct obd_export *exp, struct md_op_data *op_data,
 	struct obd_device *obd = exp->exp_obd;
 	struct lmv_obd *lmv = &obd->u.lmv;
 	struct lmv_tgt_desc *tgt;
-	int rc;
-
-	rc = lmv_check_connect(obd);
-	if (rc)
-		return rc;
 
 	if (unlikely(lsm))
 		return lmv_read_striped_page(exp, op_data, cb_op, offset, ppage);
@@ -2460,9 +2363,6 @@ static int lmv_unlink(struct obd_export *exp, struct md_op_data *op_data,
 	int stripe_index = 0;
 	int		     rc;
 
-	rc = lmv_check_connect(obd);
-	if (rc)
-		return rc;
 retry_unlink:
 	/* For striped dir, we need to locate the parent as well */
 	if (lsm) {
@@ -2647,10 +2547,6 @@ static int lmv_get_info(const struct lu_env *env, struct obd_export *exp,
 	if (keylen >= strlen("remote_flag") && !strcmp(key, "remote_flag")) {
 		int i;
 
-		rc = lmv_check_connect(obd);
-		if (rc)
-			return rc;
-
 		LASSERT(*vallen == sizeof(__u32));
 		for (i = 0; i < lmv->desc.ld_tgt_count; i++) {
 			struct lmv_tgt_desc *tgt = lmv->tgts[i];
@@ -2669,10 +2565,6 @@ static int lmv_get_info(const struct lu_env *env, struct obd_export *exp,
 	} else if (KEY_IS(KEY_MAX_EASIZE) ||
 		   KEY_IS(KEY_DEFAULT_EASIZE) ||
 		   KEY_IS(KEY_CONN_DATA)) {
-		rc = lmv_check_connect(obd);
-		if (rc)
-			return rc;
-
 		/*
 		 * Forwarding this request to first MDS, it should know LOV
 		 * desc.
@@ -3021,15 +2913,10 @@ static int lmv_intent_getattr_async(struct obd_export *exp,
 	struct lmv_obd	  *lmv = &obd->u.lmv;
 	struct lmv_tgt_desc *ptgt = NULL;
 	struct lmv_tgt_desc *ctgt = NULL;
-	int		      rc;
 
 	if (!fid_is_sane(&op_data->op_fid2))
 		return -EINVAL;
 
-	rc = lmv_check_connect(obd);
-	if (rc)
-		return rc;
-
 	ptgt = lmv_locate_mds(lmv, op_data, &op_data->op_fid1);
 	if (IS_ERR(ptgt))
 		return PTR_ERR(ptgt);
@@ -3056,11 +2943,6 @@ static int lmv_revalidate_lock(struct obd_export *exp, struct lookup_intent *it,
 	struct obd_device       *obd = exp->exp_obd;
 	struct lmv_obd	  *lmv = &obd->u.lmv;
 	struct lmv_tgt_desc     *tgt;
-	int		      rc;
-
-	rc = lmv_check_connect(obd);
-	if (rc)
-		return rc;
 
 	tgt = lmv_find_target(lmv, fid);
 	if (IS_ERR(tgt))
diff --git a/drivers/staging/lustre/lustre/lmv/lproc_lmv.c b/drivers/staging/lustre/lustre/lmv/lproc_lmv.c
index bf25f887062d..f16cfa435f77 100644
--- a/drivers/staging/lustre/lustre/lmv/lproc_lmv.c
+++ b/drivers/staging/lustre/lustre/lmv/lproc_lmv.c
@@ -34,8 +34,8 @@
 
 #include <linux/seq_file.h>
 #include <linux/statfs.h>
-#include "../include/lprocfs_status.h"
-#include "../include/obd_class.h"
+#include <lprocfs_status.h>
+#include <obd_class.h>
 #include "lmv_internal.h"
 
 static ssize_t numobd_show(struct kobject *kobj, struct attribute *attr,
@@ -161,7 +161,7 @@ static struct attribute *lmv_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group lmv_attr_group = {
+static const struct attribute_group lmv_attr_group = {
 	.attrs = lmv_attrs,
 };
 
diff --git a/drivers/staging/lustre/lustre/lov/Makefile b/drivers/staging/lustre/lustre/lov/Makefile
index e4cc0db21014..3abfb4eab3d3 100644
--- a/drivers/staging/lustre/lustre/lov/Makefile
+++ b/drivers/staging/lustre/lustre/lov/Makefile
@@ -1,5 +1,8 @@
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/include
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/lustre/include
+
 obj-$(CONFIG_LUSTRE_FS) += lov.o
 lov-y := lov_obd.o lov_pack.o lov_offset.o lov_merge.o \
 	 lov_request.o lov_ea.o lov_dev.o lov_object.o lov_page.o  \
 	 lov_lock.o lov_io.o lovsub_dev.o lovsub_object.o lovsub_page.o      \
-	 lovsub_lock.o lovsub_io.o lov_pool.o lproc_lov.o
+	 lovsub_lock.o lov_pool.o lproc_lov.o
diff --git a/drivers/staging/lustre/lustre/lov/lov_cl_internal.h b/drivers/staging/lustre/lustre/lov/lov_cl_internal.h
index e889d3a7de9c..89d92b05b48c 100644
--- a/drivers/staging/lustre/lustre/lov/lov_cl_internal.h
+++ b/drivers/staging/lustre/lustre/lov/lov_cl_internal.h
@@ -42,10 +42,10 @@
 #ifndef LOV_CL_INTERNAL_H
 #define LOV_CL_INTERNAL_H
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
-#include "../include/obd.h"
-#include "../include/cl_object.h"
+#include <obd.h>
+#include <cl_object.h>
 #include "lov_internal.h"
 
 /** \defgroup lov lov
@@ -92,35 +92,6 @@ enum lov_device_flags {
  * Upper half.
  */
 
-/**
- * Resources that are used in memory-cleaning path, and whose allocation
- * cannot fail even when memory is tight. They are preallocated in sufficient
- * quantities in lov_device::ld_emerg[], and access to them is serialized
- * lov_device::ld_mutex.
- */
-struct lov_device_emerg {
-	/**
-	 * Page list used to submit IO when memory is in pressure.
-	 */
-	struct cl_page_list emrg_page_list;
-	/**
-	 * sub-io's shared by all threads accessing this device when memory is
-	 * too low to allocate sub-io's dynamically.
-	 */
-	struct cl_io	emrg_subio;
-	/**
-	 * Environments used by sub-io's in
-	 * lov_device_emerg::emrg_subio.
-	 */
-	struct lu_env      *emrg_env;
-	/**
-	 * Refchecks for lov_device_emerg::emrg_env.
-	 *
-	 * \see cl_env_get()
-	 */
-	u16		 emrg_refcheck;
-};
-
 struct lov_device {
 	/*
 	 * XXX Locking of lov-private data is missing.
@@ -131,14 +102,6 @@ struct lov_device {
 	__u32		     ld_target_nr;
 	struct lovsub_device    **ld_target;
 	__u32		     ld_flags;
-
-	/** Emergency resources used in memory-cleansing paths. */
-	struct lov_device_emerg **ld_emrg;
-	/**
-	 * Serializes access to lov_device::ld_emrg in low-memory
-	 * conditions.
-	 */
-	struct mutex		  ld_mutex;
 };
 
 /**
@@ -299,8 +262,6 @@ struct lov_page {
 
 struct lovsub_device {
 	struct cl_device   acid_cl;
-	struct lov_device *acid_super;
-	int		acid_idx;
 	struct cl_device  *acid_next;
 };
 
@@ -312,42 +273,10 @@ struct lovsub_object {
 };
 
 /**
- * A link between a top-lock and a sub-lock. Separate data-structure is
- * necessary, because top-locks and sub-locks are in M:N relationship.
- *
- * \todo This can be optimized for a (by far) most frequent case of a single
- * top-lock per sub-lock.
- */
-struct lov_lock_link {
-	struct lov_lock *lll_super;
-	/** An index within parent lock. */
-	int	      lll_idx;
-	/**
-	 * A linkage into per sub-lock list of all corresponding top-locks,
-	 * hanging off lovsub_lock::lss_parents.
-	 */
-	struct list_head       lll_list;
-};
-
-/**
  * Lock state at lovsub layer.
  */
 struct lovsub_lock {
 	struct cl_lock_slice  lss_cl;
-	/**
-	 * List of top-locks that have given sub-lock as their part. Protected
-	 * by cl_lock::cll_guard mutex.
-	 */
-	struct list_head	    lss_parents;
-	/**
-	 * Top-lock that initiated current operation on this sub-lock. This is
-	 * only set during top-to-bottom lock operations like enqueue, and is
-	 * used to optimize state change notification. Protected by
-	 * cl_lock::cll_guard mutex.
-	 *
-	 * \see lovsub_lock_state_one().
-	 */
-	struct cl_lock       *lss_active;
 };
 
 /**
@@ -356,7 +285,6 @@ struct lovsub_lock {
 struct lov_sublock_env {
 	const struct lu_env *lse_env;
 	struct cl_io	*lse_io;
-	struct lov_io_sub   *lse_sub;
 };
 
 struct lovsub_page {
@@ -366,12 +294,10 @@ struct lovsub_page {
 struct lov_thread_info {
 	struct cl_object_conf   lti_stripe_conf;
 	struct lu_fid	   lti_fid;
-	struct cl_lock_descr    lti_ldescr;
 	struct ost_lvb	  lti_lvb;
 	struct cl_2queue	lti_cl2q;
 	struct cl_page_list     lti_plist;
 	wait_queue_entry_t	  lti_waiter;
-	struct cl_attr          lti_attr;
 };
 
 /**
@@ -385,7 +311,6 @@ struct lov_io_sub {
 	 * \see cl_env_get()
 	 */
 	u16			 sub_refcheck;
-	u16			 sub_reenter;
 	/**
 	 * true, iff cl_io_init() was successfully executed against
 	 * lov_io_sub::sub_io.
@@ -445,7 +370,6 @@ struct lov_io {
 	 */
 	u64	    lis_endpos;
 
-	int		lis_mem_frozen;
 	int		lis_stripe_count;
 	int		lis_active_subios;
 
@@ -485,8 +409,6 @@ extern struct kmem_cache *lov_session_kmem;
 extern struct kmem_cache *lovsub_lock_kmem;
 extern struct kmem_cache *lovsub_object_kmem;
 
-extern struct kmem_cache *lov_lock_link_kmem;
-
 int lov_object_init(const struct lu_env *env, struct lu_object *obj,
 		    const struct lu_object_conf *conf);
 int lovsub_object_init(const struct lu_env *env, struct lu_object *obj,
@@ -508,15 +430,9 @@ int lov_io_init_empty(const struct lu_env *env, struct cl_object *obj,
 		      struct cl_io *io);
 int lov_io_init_released(const struct lu_env *env, struct cl_object *obj,
 			 struct cl_io *io);
-void lov_lock_unlink(const struct lu_env *env, struct lov_lock_link *link,
-		     struct lovsub_lock *sub);
 
 struct lov_io_sub *lov_sub_get(const struct lu_env *env, struct lov_io *lio,
 			       int stripe);
-void lov_sub_put(struct lov_io_sub *sub);
-int lov_sublock_modify(const struct lu_env *env, struct lov_lock *lov,
-		       struct lovsub_lock *sublock,
-		       const struct cl_lock_descr *d, int idx);
 
 int lov_page_init(const struct lu_env *env, struct cl_object *ob,
 		  struct cl_page *page, pgoff_t index);
@@ -533,12 +449,6 @@ struct lu_object *lovsub_object_alloc(const struct lu_env *env,
 				      const struct lu_object_header *hdr,
 				      struct lu_device *dev);
 
-struct lov_lock_link *lov_lock_link_find(const struct lu_env *env,
-					 struct lov_lock *lck,
-					 struct lovsub_lock *sub);
-struct lov_io_sub *lov_page_subio(const struct lu_env *env, struct lov_io *lio,
-				  const struct cl_page_slice *slice);
-
 struct lov_stripe_md *lov_lsm_addref(struct lov_object *lov);
 int lov_page_stripe(const struct cl_page *page);
 
diff --git a/drivers/staging/lustre/lustre/lov/lov_dev.c b/drivers/staging/lustre/lustre/lov/lov_dev.c
index 7301f6e579a1..cea5f9dcd04e 100644
--- a/drivers/staging/lustre/lustre/lov/lov_dev.c
+++ b/drivers/staging/lustre/lustre/lov/lov_dev.c
@@ -37,7 +37,7 @@
 #define DEBUG_SUBSYSTEM S_LOV
 
 /* class_name2obd() */
-#include "../include/obd_class.h"
+#include <obd_class.h>
 
 #include "lov_cl_internal.h"
 #include "lov_internal.h"
@@ -50,11 +50,6 @@ struct kmem_cache *lov_session_kmem;
 struct kmem_cache *lovsub_lock_kmem;
 struct kmem_cache *lovsub_object_kmem;
 
-struct kmem_cache *lov_lock_link_kmem;
-
-/** Lock class of lov_device::ld_mutex. */
-static struct lock_class_key cl_lov_device_mutex_class;
-
 struct lu_kmem_descr lov_caches[] = {
 	{
 		.ckd_cache = &lov_lock_kmem,
@@ -87,11 +82,6 @@ struct lu_kmem_descr lov_caches[] = {
 		.ckd_size  = sizeof(struct lovsub_object)
 	},
 	{
-		.ckd_cache = &lov_lock_link_kmem,
-		.ckd_name  = "lov_lock_link_kmem",
-		.ckd_size  = sizeof(struct lov_lock_link)
-	},
-	{
 		.ckd_cache = NULL
 	}
 };
@@ -204,8 +194,6 @@ static int lov_device_init(const struct lu_env *env, struct lu_device *d,
 			break;
 		}
 		lsd = cl2lovsub_dev(cl);
-		lsd->acid_idx = i;
-		lsd->acid_super = ld;
 		ld->ld_target[i] = lsd;
 	}
 
@@ -217,34 +205,13 @@ static int lov_device_init(const struct lu_env *env, struct lu_device *d,
 	return rc;
 }
 
-static void lov_emerg_free(struct lov_device_emerg **emrg, int nr)
-{
-	int i;
-
-	for (i = 0; i < nr; ++i) {
-		struct lov_device_emerg *em;
-
-		em = emrg[i];
-		if (em) {
-			LASSERT(em->emrg_page_list.pl_nr == 0);
-			if (em->emrg_env)
-				cl_env_put(em->emrg_env, &em->emrg_refcheck);
-			kfree(em);
-		}
-	}
-	kfree(emrg);
-}
-
 static struct lu_device *lov_device_free(const struct lu_env *env,
 					 struct lu_device *d)
 {
 	struct lov_device *ld = lu2lov_dev(d);
-	const int	  nr = ld->ld_target_nr;
 
 	cl_device_fini(lu2cl_dev(d));
 	kfree(ld->ld_target);
-	if (ld->ld_emrg)
-		lov_emerg_free(ld->ld_emrg, nr);
 	kfree(ld);
 	return NULL;
 }
@@ -260,41 +227,6 @@ static void lov_cl_del_target(const struct lu_env *env, struct lu_device *dev,
 	}
 }
 
-static struct lov_device_emerg **lov_emerg_alloc(int nr)
-{
-	struct lov_device_emerg **emerg;
-	int i;
-	int result;
-
-	emerg = kcalloc(nr, sizeof(emerg[0]), GFP_NOFS);
-	if (!emerg)
-		return ERR_PTR(-ENOMEM);
-	for (result = i = 0; i < nr && result == 0; i++) {
-		struct lov_device_emerg *em;
-
-		em = kzalloc(sizeof(*em), GFP_NOFS);
-		if (em) {
-			emerg[i] = em;
-			cl_page_list_init(&em->emrg_page_list);
-			em->emrg_env = cl_env_alloc(&em->emrg_refcheck,
-						    LCT_REMEMBER | LCT_NOREF);
-			if (!IS_ERR(em->emrg_env)) {
-				em->emrg_env->le_ctx.lc_cookie = 0x2;
-			} else {
-				result = PTR_ERR(em->emrg_env);
-				em->emrg_env = NULL;
-			}
-		} else {
-			result = -ENOMEM;
-		}
-	}
-	if (result != 0) {
-		lov_emerg_free(emerg, nr);
-		emerg = ERR_PTR(result);
-	}
-	return emerg;
-}
-
 static int lov_expand_targets(const struct lu_env *env, struct lov_device *dev)
 {
 	int   result;
@@ -306,29 +238,17 @@ static int lov_expand_targets(const struct lu_env *env, struct lov_device *dev)
 	sub_size = dev->ld_target_nr;
 	if (sub_size < tgt_size) {
 		struct lovsub_device    **newd;
-		struct lov_device_emerg **emerg;
 		const size_t	      sz   = sizeof(newd[0]);
 
-		emerg = lov_emerg_alloc(tgt_size);
-		if (IS_ERR(emerg))
-			return PTR_ERR(emerg);
-
 		newd = kcalloc(tgt_size, sz, GFP_NOFS);
 		if (newd) {
-			mutex_lock(&dev->ld_mutex);
 			if (sub_size > 0) {
 				memcpy(newd, dev->ld_target, sub_size * sz);
 				kfree(dev->ld_target);
 			}
 			dev->ld_target    = newd;
 			dev->ld_target_nr = tgt_size;
-
-			if (dev->ld_emrg)
-				lov_emerg_free(dev->ld_emrg, sub_size);
-			dev->ld_emrg = emerg;
-			mutex_unlock(&dev->ld_mutex);
 		} else {
-			lov_emerg_free(emerg, tgt_size);
 			result = -ENOMEM;
 		}
 	}
@@ -362,8 +282,6 @@ static int lov_cl_add_target(const struct lu_env *env, struct lu_device *dev,
 				   tgt->ltd_obd->obd_lu_dev);
 		if (!IS_ERR(cl)) {
 			lsd = cl2lovsub_dev(cl);
-			lsd->acid_idx = index;
-			lsd->acid_super = ld;
 			ld->ld_target[index] = lsd;
 		} else {
 			CERROR("add failed (%d), deleting %s\n", rc,
@@ -428,9 +346,6 @@ static struct lu_device *lov_device_alloc(const struct lu_env *env,
 	d = lov2lu_dev(ld);
 	d->ld_ops	= &lov_lu_ops;
 
-	mutex_init(&ld->ld_mutex);
-	lockdep_set_class(&ld->ld_mutex, &cl_lov_device_mutex_class);
-
 	/* setup the LOV OBD */
 	obd = class_name2obd(lustre_cfg_string(cfg, 0));
 	LASSERT(obd);
diff --git a/drivers/staging/lustre/lustre/lov/lov_ea.c b/drivers/staging/lustre/lustre/lov/lov_ea.c
index ac0bf64c08c1..1124fd5ab32f 100644
--- a/drivers/staging/lustre/lustre/lov/lov_ea.c
+++ b/drivers/staging/lustre/lustre/lov/lov_ea.c
@@ -37,10 +37,10 @@
 #define DEBUG_SUBSYSTEM S_LOV
 
 #include <asm/div64.h>
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
-#include "../include/obd_class.h"
-#include "../include/lustre/lustre_idl.h"
+#include <obd_class.h>
+#include <uapi/linux/lustre/lustre_idl.h>
 
 #include "lov_internal.h"
 
@@ -150,9 +150,10 @@ static int lsm_unpackmd_common(struct lov_obd *lov,
 			       struct lov_mds_md *lmm,
 			       struct lov_ost_data_v1 *objects)
 {
-	loff_t stripe_maxbytes = LLONG_MAX;
+	loff_t min_stripe_maxbytes = 0;
 	unsigned int stripe_count;
 	struct lov_oinfo *loi;
+	loff_t lov_bytes;
 	unsigned int i;
 
 	/*
@@ -168,8 +169,6 @@ static int lsm_unpackmd_common(struct lov_obd *lov,
 	stripe_count = lsm_is_released(lsm) ? 0 : lsm->lsm_stripe_count;
 
 	for (i = 0; i < stripe_count; i++) {
-		loff_t tgt_bytes;
-
 		loi = lsm->lsm_oinfo[i];
 		ostid_le_to_cpu(&objects[i].l_ost_oi, &loi->loi_oi);
 		loi->loi_ost_idx = le32_to_cpu(objects[i].l_ost_idx);
@@ -194,17 +193,21 @@ static int lsm_unpackmd_common(struct lov_obd *lov,
 			continue;
 		}
 
-		tgt_bytes = lov_tgt_maxbytes(lov->lov_tgts[loi->loi_ost_idx]);
-		stripe_maxbytes = min_t(loff_t, stripe_maxbytes, tgt_bytes);
+		lov_bytes = lov_tgt_maxbytes(lov->lov_tgts[loi->loi_ost_idx]);
+		if (min_stripe_maxbytes == 0 || lov_bytes < min_stripe_maxbytes)
+			min_stripe_maxbytes = lov_bytes;
 	}
 
-	if (stripe_maxbytes == LLONG_MAX)
-		stripe_maxbytes = LUSTRE_EXT3_STRIPE_MAXBYTES;
+	if (min_stripe_maxbytes == 0)
+		min_stripe_maxbytes = LUSTRE_EXT3_STRIPE_MAXBYTES;
+
+	stripe_count = lsm->lsm_stripe_count ?: lov->desc.ld_tgt_count;
+	lov_bytes = min_stripe_maxbytes * stripe_count;
 
-	if (!lsm->lsm_stripe_count)
-		lsm->lsm_maxbytes = stripe_maxbytes * lov->desc.ld_tgt_count;
+	if (lov_bytes < min_stripe_maxbytes) /* handle overflow */
+		lsm->lsm_maxbytes = MAX_LFS_FILESIZE;
 	else
-		lsm->lsm_maxbytes = stripe_maxbytes * lsm->lsm_stripe_count;
+		lsm->lsm_maxbytes = lov_bytes;
 
 	return 0;
 }
diff --git a/drivers/staging/lustre/lustre/lov/lov_internal.h b/drivers/staging/lustre/lustre/lov/lov_internal.h
index 774499c74daa..a21f074008af 100644
--- a/drivers/staging/lustre/lustre/lov/lov_internal.h
+++ b/drivers/staging/lustre/lustre/lov/lov_internal.h
@@ -33,8 +33,8 @@
 #ifndef LOV_INTERNAL_H
 #define LOV_INTERNAL_H
 
-#include "../include/obd_class.h"
-#include "../include/lustre/lustre_user.h"
+#include <obd_class.h>
+#include <uapi/linux/lustre/lustre_idl.h>
 
 /*
  * If we are unable to get the maximum object size from the OST in
@@ -161,42 +161,21 @@ struct lov_request {
 	struct list_head	       rq_link;
 
 	int		      rq_idx;	/* index in lov->tgts array */
-	int		      rq_stripe;     /* stripe number */
-	int		      rq_complete;
-	int		      rq_rc;
-
-	u32		      rq_oabufs;
-	u32		      rq_pgaidx;
 };
 
 struct lov_request_set {
 	struct obd_info			*set_oi;
-	atomic_t			set_refcount;
-	struct obd_export		*set_exp;
-	/* XXX: There is @set_exp already, however obd_statfs gets obd_device
-	 * only.
-	 */
 	struct obd_device		*set_obd;
 	int				set_count;
 	atomic_t			set_completes;
 	atomic_t			set_success;
-	atomic_t			set_finish_checked;
 	struct list_head			set_list;
-	wait_queue_head_t			set_waitq;
 };
 
 extern struct kmem_cache *lov_oinfo_slab;
 
 extern struct lu_kmem_descr lov_caches[];
 
-void lov_finish_set(struct lov_request_set *set);
-
-static inline void lov_put_reqset(struct lov_request_set *set)
-{
-	if (atomic_dec_and_test(&set->set_refcount))
-		lov_finish_set(set);
-}
-
 #define lov_uuid2str(lv, index) \
 	(char *)((lv)->lov_tgts[index]->ltd_uuid.uuid)
 
@@ -217,15 +196,9 @@ pgoff_t lov_stripe_pgoff(struct lov_stripe_md *lsm, pgoff_t stripe_index,
 			 int stripe);
 
 /* lov_request.c */
-int lov_prep_getattr_set(struct obd_export *exp, struct obd_info *oinfo,
-			 struct lov_request_set **reqset);
-int lov_fini_getattr_set(struct lov_request_set *set);
 int lov_prep_statfs_set(struct obd_device *obd, struct obd_info *oinfo,
 			struct lov_request_set **reqset);
-int lov_fini_statfs(struct obd_device *obd, struct obd_statfs *osfs,
-		    int success);
 int lov_fini_statfs_set(struct lov_request_set *set);
-int lov_statfs_interpret(struct ptlrpc_request_set *rqset, void *data, int rc);
 
 /* lov_obd.c */
 void lov_stripe_lock(struct lov_stripe_md *md);
diff --git a/drivers/staging/lustre/lustre/lov/lov_io.c b/drivers/staging/lustre/lustre/lov/lov_io.c
index babf39adef85..9e3b150967b4 100644
--- a/drivers/staging/lustre/lustre/lov/lov_io.c
+++ b/drivers/staging/lustre/lustre/lov/lov_io.c
@@ -43,24 +43,12 @@
  *  @{
  */
 
-static inline void lov_sub_enter(struct lov_io_sub *sub)
-{
-	sub->sub_reenter++;
-}
-
-static inline void lov_sub_exit(struct lov_io_sub *sub)
-{
-	sub->sub_reenter--;
-}
-
 static void lov_io_sub_fini(const struct lu_env *env, struct lov_io *lio,
 			    struct lov_io_sub *sub)
 {
 	if (sub->sub_io) {
 		if (sub->sub_io_initialized) {
-			lov_sub_enter(sub);
 			cl_io_fini(sub->sub_env, sub->sub_io);
-			lov_sub_exit(sub);
 			sub->sub_io_initialized = 0;
 			lio->lis_active_subios--;
 		}
@@ -142,13 +130,11 @@ static int lov_io_sub_init(const struct lu_env *env, struct lov_io *lio,
 			   struct lov_io_sub *sub)
 {
 	struct lov_object *lov = lio->lis_object;
-	struct lov_device *ld  = lu2lov_dev(lov2cl(lov)->co_lu.lo_dev);
 	struct cl_io      *sub_io;
 	struct cl_object  *sub_obj;
 	struct cl_io      *io  = lio->lis_cl.cis_io;
-
 	int stripe = sub->sub_stripe;
-	int result;
+	int rc;
 
 	LASSERT(!sub->sub_io);
 	LASSERT(!sub->sub_env);
@@ -157,63 +143,53 @@ static int lov_io_sub_init(const struct lu_env *env, struct lov_io *lio,
 	if (unlikely(!lov_r0(lov)->lo_sub[stripe]))
 		return -EIO;
 
-	result = 0;
 	sub->sub_io_initialized = 0;
 	sub->sub_borrowed = 0;
 
-	if (lio->lis_mem_frozen) {
-		LASSERT(mutex_is_locked(&ld->ld_mutex));
-		sub->sub_io  = &ld->ld_emrg[stripe]->emrg_subio;
-		sub->sub_env = ld->ld_emrg[stripe]->emrg_env;
-		sub->sub_borrowed = 1;
-	} else {
-		sub->sub_env = cl_env_get(&sub->sub_refcheck);
-		if (IS_ERR(sub->sub_env))
-			result = PTR_ERR(sub->sub_env);
+	/* obtain new environment */
+	sub->sub_env = cl_env_get(&sub->sub_refcheck);
+	if (IS_ERR(sub->sub_env)) {
+		rc = PTR_ERR(sub->sub_env);
+		goto fini_lov_io;
+	}
 
-		if (result == 0) {
-			/*
-			 * First sub-io. Use ->lis_single_subio to
-			 * avoid dynamic allocation.
-			 */
-			if (lio->lis_active_subios == 0) {
-				sub->sub_io = &lio->lis_single_subio;
-				lio->lis_single_subio_index = stripe;
-			} else {
-				sub->sub_io = kzalloc(sizeof(*sub->sub_io),
-						      GFP_NOFS);
-				if (!sub->sub_io)
-					result = -ENOMEM;
-			}
+	/*
+	 * First sub-io. Use ->lis_single_subio to
+	 * avoid dynamic allocation.
+	 */
+	if (lio->lis_active_subios == 0) {
+		sub->sub_io = &lio->lis_single_subio;
+		lio->lis_single_subio_index = stripe;
+	} else {
+		sub->sub_io = kzalloc(sizeof(*sub->sub_io),
+				      GFP_NOFS);
+		if (!sub->sub_io) {
+			rc = -ENOMEM;
+			goto fini_lov_io;
 		}
 	}
 
-	if (result == 0) {
-		sub_obj = lovsub2cl(lov_r0(lov)->lo_sub[stripe]);
-		sub_io  = sub->sub_io;
-
-		sub_io->ci_obj    = sub_obj;
-		sub_io->ci_result = 0;
-
-		sub_io->ci_parent  = io;
-		sub_io->ci_lockreq = io->ci_lockreq;
-		sub_io->ci_type    = io->ci_type;
-		sub_io->ci_no_srvlock = io->ci_no_srvlock;
-		sub_io->ci_noatime = io->ci_noatime;
-
-		lov_sub_enter(sub);
-		result = cl_io_sub_init(sub->sub_env, sub_io,
-					io->ci_type, sub_obj);
-		lov_sub_exit(sub);
-		if (result >= 0) {
-			lio->lis_active_subios++;
-			sub->sub_io_initialized = 1;
-			result = 0;
-		}
+	sub_obj = lovsub2cl(lov_r0(lov)->lo_sub[stripe]);
+	sub_io = sub->sub_io;
+
+	sub_io->ci_obj = sub_obj;
+	sub_io->ci_result = 0;
+	sub_io->ci_parent = io;
+	sub_io->ci_lockreq = io->ci_lockreq;
+	sub_io->ci_type = io->ci_type;
+	sub_io->ci_no_srvlock = io->ci_no_srvlock;
+	sub_io->ci_noatime = io->ci_noatime;
+
+	rc = cl_io_sub_init(sub->sub_env, sub_io, io->ci_type, sub_obj);
+	if (rc >= 0) {
+		lio->lis_active_subios++;
+		sub->sub_io_initialized = 1;
+		rc = 0;
 	}
-	if (result != 0)
+fini_lov_io:
+	if (rc)
 		lov_io_sub_fini(env, lio, sub);
-	return result;
+	return rc;
 }
 
 struct lov_io_sub *lov_sub_get(const struct lu_env *env,
@@ -230,16 +206,10 @@ struct lov_io_sub *lov_sub_get(const struct lu_env *env,
 	} else {
 		rc = 0;
 	}
-	if (rc == 0)
-		lov_sub_enter(sub);
-	else
+	if (rc < 0)
 		sub = ERR_PTR(rc);
-	return sub;
-}
 
-void lov_sub_put(struct lov_io_sub *sub)
-{
-	lov_sub_exit(sub);
+	return sub;
 }
 
 /*****************************************************************************
@@ -258,22 +228,6 @@ int lov_page_stripe(const struct cl_page *page)
 	return cl2lov_page(slice)->lps_stripe;
 }
 
-struct lov_io_sub *lov_page_subio(const struct lu_env *env, struct lov_io *lio,
-				  const struct cl_page_slice *slice)
-{
-	struct lov_stripe_md *lsm  = lio->lis_object->lo_lsm;
-	struct cl_page       *page = slice->cpl_page;
-	int stripe;
-
-	LASSERT(lio->lis_cl.cis_io);
-	LASSERT(cl2lov(slice->cpl_obj) == lio->lis_object);
-	LASSERT(lsm);
-	LASSERT(lio->lis_nr_subios > 0);
-
-	stripe = lov_page_stripe(page);
-	return lov_sub_get(env, lio, stripe);
-}
-
 static int lov_io_subio_init(const struct lu_env *env, struct lov_io *lio,
 			     struct cl_io *io)
 {
@@ -431,12 +385,10 @@ static int lov_io_iter_init(const struct lu_env *env,
 
 		lov_io_sub_inherit(sub->sub_io, lio, stripe, start, end);
 		rc = cl_io_iter_init(sub->sub_env, sub->sub_io);
-		if (rc)
+		if (rc) {
 			cl_io_iter_fini(sub->sub_env, sub->sub_io);
-		lov_sub_put(sub);
-		if (rc)
 			break;
-
+		}
 		CDEBUG(D_VFSTRACE, "shrink: %d [%llu, %llu)\n",
 		       stripe, start, end);
 
@@ -488,9 +440,7 @@ static int lov_io_call(const struct lu_env *env, struct lov_io *lio,
 	int rc = 0;
 
 	list_for_each_entry(sub, &lio->lis_active, sub_linkage) {
-		lov_sub_enter(sub);
 		rc = iofunc(sub->sub_env, sub->sub_io);
-		lov_sub_exit(sub);
 		if (rc)
 			break;
 
@@ -610,7 +560,6 @@ static int lov_io_read_ahead(const struct lu_env *env,
 	rc = cl_io_read_ahead(sub->sub_env, sub->sub_io,
 			      cl_index(lovsub2cl(r0->lo_sub[stripe]), suboff),
 			      ra);
-	lov_sub_put(sub);
 
 	CDEBUG(D_READA, DFID " cra_end = %lu, stripes = %d, rc = %d\n",
 	       PFID(lu_object_fid(lov2lu(loo))), ra->cra_end, r0->lo_nr, rc);
@@ -679,7 +628,6 @@ static int lov_io_submit(const struct lu_env *env,
 		LASSERT(sub->sub_io == &lio->lis_single_subio);
 		rc = cl_io_submit_rw(sub->sub_env, sub->sub_io,
 				     crt, queue);
-		lov_sub_put(sub);
 		return rc;
 	}
 
@@ -707,7 +655,6 @@ static int lov_io_submit(const struct lu_env *env,
 		if (!IS_ERR(sub)) {
 			rc = cl_io_submit_rw(sub->sub_env, sub->sub_io,
 					     crt, cl2q);
-			lov_sub_put(sub);
 		} else {
 			rc = PTR_ERR(sub);
 		}
@@ -746,7 +693,6 @@ static int lov_io_commit_async(const struct lu_env *env,
 		LASSERT(sub->sub_io == &lio->lis_single_subio);
 		rc = cl_io_commit_async(sub->sub_env, sub->sub_io, queue,
 					from, to, cb);
-		lov_sub_put(sub);
 		return rc;
 	}
 
@@ -777,7 +723,6 @@ static int lov_io_commit_async(const struct lu_env *env,
 		if (!IS_ERR(sub)) {
 			rc = cl_io_commit_async(sub->sub_env, sub->sub_io,
 						plist, from, stripe_to, cb);
-			lov_sub_put(sub);
 		} else {
 			rc = PTR_ERR(sub);
 			break;
@@ -813,7 +758,6 @@ static int lov_io_fault_start(const struct lu_env *env,
 	if (IS_ERR(sub))
 		return PTR_ERR(sub);
 	sub->sub_io->u.ci_fault.ft_nob = fio->ft_nob;
-	lov_sub_put(sub);
 	return lov_io_start(env, ios);
 }
 
@@ -828,9 +772,7 @@ static void lov_io_fsync_end(const struct lu_env *env,
 	list_for_each_entry(sub, &lio->lis_active, sub_linkage) {
 		struct cl_io *subio = sub->sub_io;
 
-		lov_sub_enter(sub);
 		lov_io_end_wrapper(sub->sub_env, subio);
-		lov_sub_exit(sub);
 
 		if (subio->ci_result == 0)
 			*written += subio->u.ci_fsync.fi_nr_written;
@@ -939,12 +881,6 @@ static const struct cl_io_operations lov_empty_io_ops = {
 	.op = {
 		[CIT_READ] = {
 			.cio_fini       = lov_empty_io_fini,
-#if 0
-			.cio_iter_init  = LOV_EMPTY_IMPOSSIBLE,
-			.cio_lock       = LOV_EMPTY_IMPOSSIBLE,
-			.cio_start      = LOV_EMPTY_IMPOSSIBLE,
-			.cio_end	= LOV_EMPTY_IMPOSSIBLE
-#endif
 		},
 		[CIT_WRITE] = {
 			.cio_fini      = lov_empty_io_fini,
@@ -1047,6 +983,8 @@ int lov_io_init_released(const struct lu_env *env, struct cl_object *obj,
 	switch (io->ci_type) {
 	default:
 		LASSERTF(0, "invalid type %d\n", io->ci_type);
+		result = -EOPNOTSUPP;
+		break;
 	case CIT_MISC:
 	case CIT_FSYNC:
 	case CIT_DATA_VERSION:
diff --git a/drivers/staging/lustre/lustre/lov/lov_lock.c b/drivers/staging/lustre/lustre/lov/lov_lock.c
index 8502128e8248..e12dc5afc14f 100644
--- a/drivers/staging/lustre/lustre/lov/lov_lock.c
+++ b/drivers/staging/lustre/lustre/lov/lov_lock.c
@@ -71,13 +71,11 @@ static struct lov_sublock_env *lov_sublock_env_get(const struct lu_env *env,
 	if (!io || !cl_object_same(io->ci_obj, parent->cll_descr.cld_obj)) {
 		subenv->lse_env = env;
 		subenv->lse_io  = io;
-		subenv->lse_sub = NULL;
 	} else {
 		sub = lov_sub_get(env, lio, lls->sub_stripe);
 		if (!IS_ERR(sub)) {
 			subenv->lse_env = sub->sub_env;
 			subenv->lse_io  = sub->sub_io;
-			subenv->lse_sub = sub;
 		} else {
 			subenv = (void *)sub;
 		}
@@ -85,12 +83,6 @@ static struct lov_sublock_env *lov_sublock_env_get(const struct lu_env *env,
 	return subenv;
 }
 
-static void lov_sublock_env_put(struct lov_sublock_env *subenv)
-{
-	if (subenv && subenv->lse_sub)
-		lov_sub_put(subenv->lse_sub);
-}
-
 static int lov_sublock_init(const struct lu_env *env,
 			    const struct cl_lock *parent,
 			    struct lov_lock_sub *lls)
@@ -102,7 +94,6 @@ static int lov_sublock_init(const struct lu_env *env,
 	if (!IS_ERR(subenv)) {
 		result = cl_lock_init(subenv->lse_env, &lls->sub_lock,
 				      subenv->lse_io);
-		lov_sublock_env_put(subenv);
 	} else {
 		/* error occurs. */
 		result = PTR_ERR(subenv);
@@ -244,7 +235,6 @@ static int lov_lock_enqueue(const struct lu_env *env,
 		}
 		rc = cl_lock_enqueue(subenv->lse_env, subenv->lse_io,
 				     &lls->sub_lock, anchor);
-		lov_sublock_env_put(subenv);
 		if (rc != 0)
 			break;
 
@@ -272,11 +262,10 @@ static void lov_lock_cancel(const struct lu_env *env,
 		subenv = lov_sublock_env_get(env, lock, lls);
 		if (!IS_ERR(subenv)) {
 			cl_lock_cancel(subenv->lse_env, sublock);
-			lov_sublock_env_put(subenv);
 		} else {
 			CL_LOCK_DEBUG(D_ERROR, env, slice->cls_lock,
-				      "lov_lock_cancel fails with %ld.\n",
-				      PTR_ERR(subenv));
+				      "%s fails with %ld.\n",
+				      __func__, PTR_ERR(subenv));
 		}
 	}
 }
diff --git a/drivers/staging/lustre/lustre/lov/lov_merge.c b/drivers/staging/lustre/lustre/lov/lov_merge.c
index 034b4fcb38f5..916336115989 100644
--- a/drivers/staging/lustre/lustre/lov/lov_merge.c
+++ b/drivers/staging/lustre/lustre/lov/lov_merge.c
@@ -32,9 +32,9 @@
 
 #define DEBUG_SUBSYSTEM S_LOV
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
-#include "../include/obd_class.h"
+#include <obd_class.h>
 #include "lov_internal.h"
 
 /** Merge the lock value block(&lvb) attributes and KMS from each of the
diff --git a/drivers/staging/lustre/lustre/lov/lov_obd.c b/drivers/staging/lustre/lustre/lov/lov_obd.c
index 25f15da6e189..fefd3c588681 100644
--- a/drivers/staging/lustre/lustre/lov/lov_obd.c
+++ b/drivers/staging/lustre/lustre/lov/lov_obd.c
@@ -38,22 +38,22 @@
  */
 
 #define DEBUG_SUBSYSTEM S_LOV
-#include "../../include/linux/libcfs/libcfs.h"
-
-#include "../include/lustre/lustre_idl.h"
-#include "../include/lustre/lustre_ioctl.h"
-
-#include "../include/cl_object.h"
-#include "../include/lustre_dlm.h"
-#include "../include/lustre_fid.h"
-#include "../include/lustre_lib.h"
-#include "../include/lustre_mds.h"
-#include "../include/lustre_net.h"
-#include "../include/lustre_param.h"
-#include "../include/lustre_swab.h"
-#include "../include/lprocfs_status.h"
-#include "../include/obd_class.h"
-#include "../include/obd_support.h"
+#include <linux/libcfs/libcfs.h>
+
+#include <uapi/linux/lustre/lustre_idl.h>
+#include <uapi/linux/lustre/lustre_ioctl.h>
+
+#include <cl_object.h>
+#include <lustre_dlm.h>
+#include <lustre_fid.h>
+#include <lustre_lib.h>
+#include <lustre_mds.h>
+#include <lustre_net.h>
+#include <uapi/linux/lustre/lustre_param.h>
+#include <lustre_swab.h>
+#include <lprocfs_status.h>
+#include <obd_class.h>
+#include <obd_support.h>
 
 #include "lov_internal.h"
 
@@ -947,7 +947,8 @@ out:
 	return rc;
 }
 
-int lov_statfs_interpret(struct ptlrpc_request_set *rqset, void *data, int rc)
+static int
+lov_statfs_interpret(struct ptlrpc_request_set *rqset, void *data, int rc)
 {
 	struct lov_request_set *lovset = (struct lov_request_set *)data;
 	int err;
@@ -1086,17 +1087,17 @@ static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 		data = (struct obd_ioctl_data *)buf;
 
 		if (sizeof(*desc) > data->ioc_inllen1) {
-			obd_ioctl_freedata(buf, len);
+			kvfree(buf);
 			return -EINVAL;
 		}
 
 		if (sizeof(uuidp->uuid) * count > data->ioc_inllen2) {
-			obd_ioctl_freedata(buf, len);
+			kvfree(buf);
 			return -EINVAL;
 		}
 
 		if (sizeof(__u32) * count > data->ioc_inllen3) {
-			obd_ioctl_freedata(buf, len);
+			kvfree(buf);
 			return -EINVAL;
 		}
 
@@ -1115,7 +1116,7 @@ static int lov_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 
 		if (copy_to_user(uarg, buf, len))
 			rc = -EFAULT;
-		obd_ioctl_freedata(buf, len);
+		kvfree(buf);
 		break;
 	}
 	case OBD_IOC_QUOTACTL: {
diff --git a/drivers/staging/lustre/lustre/lov/lov_object.c b/drivers/staging/lustre/lustre/lov/lov_object.c
index 14f38268d414..334ecb1bc049 100644
--- a/drivers/staging/lustre/lustre/lov/lov_object.c
+++ b/drivers/staging/lustre/lustre/lov/lov_object.c
@@ -638,7 +638,7 @@ static const struct lov_layout_operations lov_dispatch[] = {
 	enum lov_layout_type		    __llt;		  \
 									\
 	__llt = __obj->lo_type;					 \
-	LASSERT(0 <= __llt && __llt < ARRAY_SIZE(lov_dispatch));	\
+	LASSERT(__llt < ARRAY_SIZE(lov_dispatch));		\
 	lov_dispatch[__llt].op(__VA_ARGS__);			    \
 })
 
@@ -697,7 +697,7 @@ do {								    \
 									\
 	lov_conf_freeze(__obj);						\
 	__llt = __obj->lo_type;					 \
-	LASSERT(0 <= __llt && __llt < ARRAY_SIZE(lov_dispatch));	\
+	LASSERT(__llt < ARRAY_SIZE(lov_dispatch));	\
 	lov_dispatch[__llt].op(__VA_ARGS__);			    \
 	lov_conf_thaw(__obj);						\
 } while (0)
@@ -748,13 +748,13 @@ static int lov_layout_change(const struct lu_env *unused,
 	u16 refcheck;
 	int rc;
 
-	LASSERT(0 <= lov->lo_type && lov->lo_type < ARRAY_SIZE(lov_dispatch));
+	LASSERT(lov->lo_type < ARRAY_SIZE(lov_dispatch));
 
 	env = cl_env_get(&refcheck);
 	if (IS_ERR(env))
 		return PTR_ERR(env);
 
-	LASSERT(0 <= llt && llt < ARRAY_SIZE(lov_dispatch));
+	LASSERT(llt < ARRAY_SIZE(lov_dispatch));
 
 	CDEBUG(D_INODE, DFID " from %s to %s\n",
 	       PFID(lu_object_fid(lov2lu(lov))),
@@ -1003,12 +1003,12 @@ int lov_lock_init(const struct lu_env *env, struct cl_object *obj,
  * \retval last_stripe		return the last stripe of the mapping
  */
 static int fiemap_calc_last_stripe(struct lov_stripe_md *lsm,
-				   loff_t fm_start, loff_t fm_end,
+				   u64 fm_start, u64 fm_end,
 				   int start_stripe, int *stripe_count)
 {
 	int last_stripe;
-	loff_t obd_start;
-	loff_t obd_end;
+	u64 obd_start;
+	u64 obd_end;
 	int i, j;
 
 	if (fm_end - fm_start > lsm->lsm_stripe_size * lsm->lsm_stripe_count) {
@@ -1076,14 +1076,14 @@ static void fiemap_prepare_and_copy_exts(struct fiemap *fiemap,
  * \param fm_end [in]		logical end of mapping
  * \param start_stripe [out]	starting stripe will be returned in this
  */
-static loff_t fiemap_calc_fm_end_offset(struct fiemap *fiemap,
-					struct lov_stripe_md *lsm,
-					loff_t fm_start, loff_t fm_end,
-					int *start_stripe)
+static u64 fiemap_calc_fm_end_offset(struct fiemap *fiemap,
+				     struct lov_stripe_md *lsm,
+				     u64 fm_start, u64 fm_end,
+				     int *start_stripe)
 {
-	loff_t local_end = fiemap->fm_extents[0].fe_logical;
-	loff_t lun_start, lun_end;
-	loff_t fm_end_offset;
+	u64 local_end = fiemap->fm_extents[0].fe_logical;
+	u64 lun_start, lun_end;
+	u64 fm_end_offset;
 	int stripe_no = -1;
 	int i;
 
@@ -1126,6 +1126,190 @@ static loff_t fiemap_calc_fm_end_offset(struct fiemap *fiemap,
 	return fm_end_offset;
 }
 
+struct fiemap_state {
+	struct fiemap	*fs_fm;
+	u64		fs_start;
+	u64		fs_length;
+	u64		fs_end;
+	u64		fs_end_offset;
+	int		fs_cur_extent;
+	int		fs_cnt_need;
+	int		fs_start_stripe;
+	int		fs_last_stripe;
+	bool		fs_device_done;
+	bool		fs_finish;
+	bool		fs_enough;
+};
+
+static int fiemap_for_stripe(const struct lu_env *env, struct cl_object *obj,
+			     struct lov_stripe_md *lsm,
+			     struct fiemap *fiemap, size_t *buflen,
+			     struct ll_fiemap_info_key *fmkey, int stripeno,
+			     struct fiemap_state *fs)
+{
+	struct cl_object *subobj;
+	struct lov_obd *lov = lu2lov_dev(obj->co_lu.lo_dev)->ld_lov;
+	struct fiemap_extent *fm_ext = &fs->fs_fm->fm_extents[0];
+	u64 req_fm_len; /* Stores length of required mapping */
+	u64 len_mapped_single_call;
+	u64 lun_start;
+	u64 lun_end;
+	u64 obd_object_end;
+	unsigned int ext_count;
+	/* EOF for object */
+	bool ost_eof = false;
+	/* done with required mapping for this OST? */
+	bool ost_done = false;
+	int ost_index;
+	int rc = 0;
+
+	fs->fs_device_done = false;
+	/* Find out range of mapping on this stripe */
+	if ((lov_stripe_intersects(lsm, stripeno, fs->fs_start, fs->fs_end,
+				   &lun_start, &obd_object_end)) == 0)
+		return 0;
+
+	if (lov_oinfo_is_dummy(lsm->lsm_oinfo[stripeno]))
+		return -EIO;
+
+	/* If this is a continuation FIEMAP call and we are on
+	 * starting stripe then lun_start needs to be set to
+	 * end_offset */
+	if (fs->fs_end_offset != 0 && stripeno == fs->fs_start_stripe)
+		lun_start = fs->fs_end_offset;
+
+	lun_end = fs->fs_length;
+	if (lun_end != ~0ULL) {
+		/* Handle fs->fs_start + fs->fs_length overflow */
+		if (fs->fs_start + fs->fs_length < fs->fs_start)
+			fs->fs_length = ~0ULL - fs->fs_start;
+		lun_end = lov_size_to_stripe(lsm, fs->fs_start + fs->fs_length,
+					     stripeno);
+	}
+
+	if (lun_start == lun_end)
+		return 0;
+
+	req_fm_len = obd_object_end - lun_start;
+	fs->fs_fm->fm_length = 0;
+	len_mapped_single_call = 0;
+
+	/* find lobsub object */
+	subobj = lov_find_subobj(env, cl2lov(obj), lsm, stripeno);
+	if (IS_ERR(subobj))
+		return PTR_ERR(subobj);
+	/* If the output buffer is very large and the objects have many
+	 * extents we may need to loop on a single OST repeatedly */
+	do {
+		if (fiemap->fm_extent_count > 0) {
+			/* Don't get too many extents. */
+			if (fs->fs_cur_extent + fs->fs_cnt_need >
+			    fiemap->fm_extent_count)
+				fs->fs_cnt_need = fiemap->fm_extent_count -
+						  fs->fs_cur_extent;
+		}
+
+		lun_start += len_mapped_single_call;
+		fs->fs_fm->fm_length = req_fm_len - len_mapped_single_call;
+		req_fm_len = fs->fs_fm->fm_length;
+		fs->fs_fm->fm_extent_count = fs->fs_enough ?
+					     1 : fs->fs_cnt_need;
+		fs->fs_fm->fm_mapped_extents = 0;
+		fs->fs_fm->fm_flags = fiemap->fm_flags;
+
+		ost_index = lsm->lsm_oinfo[stripeno]->loi_ost_idx;
+
+		if (ost_index < 0 || ost_index >= lov->desc.ld_tgt_count) {
+			rc = -EINVAL;
+			goto obj_put;
+		}
+		/* If OST is inactive, return extent with UNKNOWN flag. */
+		if (!lov->lov_tgts[ost_index]->ltd_active) {
+			fs->fs_fm->fm_flags |= FIEMAP_EXTENT_LAST;
+			fs->fs_fm->fm_mapped_extents = 1;
+
+			fm_ext[0].fe_logical = lun_start;
+			fm_ext[0].fe_length = obd_object_end - lun_start;
+			fm_ext[0].fe_flags |= FIEMAP_EXTENT_UNKNOWN;
+
+			goto inactive_tgt;
+		}
+
+		fs->fs_fm->fm_start = lun_start;
+		fs->fs_fm->fm_flags &= ~FIEMAP_FLAG_DEVICE_ORDER;
+		memcpy(&fmkey->lfik_fiemap, fs->fs_fm, sizeof(*fs->fs_fm));
+		*buflen = fiemap_count_to_size(fs->fs_fm->fm_extent_count);
+
+		rc = cl_object_fiemap(env, subobj, fmkey, fs->fs_fm, buflen);
+		if (rc)
+			goto obj_put;
+inactive_tgt:
+		ext_count = fs->fs_fm->fm_mapped_extents;
+		if (ext_count == 0) {
+			ost_done = true;
+			fs->fs_device_done = true;
+			/* If last stripe has hold at the end,
+			 * we need to return */
+			if (stripeno == fs->fs_last_stripe) {
+				fiemap->fm_mapped_extents = 0;
+				fs->fs_finish = true;
+				goto obj_put;
+			}
+			break;
+		} else if (fs->fs_enough) {
+			/*
+			 * We've collected enough extents and there are
+			 * more extents after it.
+			 */
+			fs->fs_finish = true;
+			goto obj_put;
+		}
+
+		/* If we just need num of extents, got to next device */
+		if (fiemap->fm_extent_count == 0) {
+			fs->fs_cur_extent += ext_count;
+			break;
+		}
+
+		/* prepare to copy retrived map extents */
+		len_mapped_single_call = fm_ext[ext_count - 1].fe_logical +
+					 fm_ext[ext_count - 1].fe_length -
+					 lun_start;
+
+		/* Have we finished mapping on this device? */
+		if (req_fm_len <= len_mapped_single_call) {
+			ost_done = true;
+			fs->fs_device_done = true;
+		}
+
+		/* Clear the EXTENT_LAST flag which can be present on
+		 * the last extent */
+		if (fm_ext[ext_count - 1].fe_flags & FIEMAP_EXTENT_LAST)
+			fm_ext[ext_count - 1].fe_flags &= ~FIEMAP_EXTENT_LAST;
+		if (lov_stripe_size(lsm, fm_ext[ext_count - 1].fe_logical +
+					 fm_ext[ext_count - 1].fe_length,
+				    stripeno) >= fmkey->lfik_oa.o_size) {
+			ost_eof = true;
+			fs->fs_device_done = true;
+		}
+
+		fiemap_prepare_and_copy_exts(fiemap, fm_ext, ost_index,
+					     ext_count, fs->fs_cur_extent);
+		fs->fs_cur_extent += ext_count;
+
+		/* Ran out of available extents? */
+		if (fs->fs_cur_extent >= fiemap->fm_extent_count)
+			fs->fs_enough = true;
+	} while (!ost_done && !ost_eof);
+
+	if (stripeno == fs->fs_last_stripe)
+		fs->fs_finish = true;
+obj_put:
+	cl_object_put(env, subobj);
+
+	return rc;
+}
+
 /**
  * Break down the FIEMAP request and send appropriate calls to individual OSTs.
  * This also handles the restarting of FIEMAP calls in case mapping overflows
@@ -1144,31 +1328,13 @@ static int lov_object_fiemap(const struct lu_env *env, struct cl_object *obj,
 			     struct ll_fiemap_info_key *fmkey,
 			     struct fiemap *fiemap, size_t *buflen)
 {
-	struct lov_obd *lov = lu2lov_dev(obj->co_lu.lo_dev)->ld_lov;
 	unsigned int buffer_size = FIEMAP_BUFFER_SIZE;
-	struct fiemap_extent *lcl_fm_ext;
-	struct cl_object *subobj = NULL;
 	struct fiemap *fm_local = NULL;
 	struct lov_stripe_md *lsm;
-	loff_t fm_start;
-	loff_t fm_end;
-	loff_t fm_length;
-	loff_t fm_end_offset;
-	int count_local;
-	int ost_index = 0;
-	int start_stripe;
-	int current_extent = 0;
 	int rc = 0;
-	int last_stripe;
-	int cur_stripe = 0;
-	int cur_stripe_wrap = 0;
+	int cur_stripe;
 	int stripe_count;
-	/* Whether have we collected enough extents */
-	bool enough = false;
-	/* EOF for object */
-	bool ost_eof = false;
-	/* done with required mapping for this OST? */
-	bool ost_done = false;
+	struct fiemap_state fs = { 0 };
 
 	lsm = lov_lsm_addref(cl2lov(obj));
 	if (!lsm)
@@ -1215,28 +1381,37 @@ static int lov_object_fiemap(const struct lu_env *env, struct cl_object *obj,
 		rc = -ENOMEM;
 		goto out;
 	}
-	lcl_fm_ext = &fm_local->fm_extents[0];
-	count_local = fiemap_size_to_count(buffer_size);
+	fs.fs_fm = fm_local;
+	fs.fs_cnt_need = fiemap_size_to_count(buffer_size);
 
-	fm_start = fiemap->fm_start;
-	fm_length = fiemap->fm_length;
+	fs.fs_start = fiemap->fm_start;
+	/* fs_start is beyond the end of the file */
+	if (fs.fs_start > fmkey->lfik_oa.o_size) {
+		rc = -EINVAL;
+		goto out;
+	}
 	/* Calculate start stripe, last stripe and length of mapping */
-	start_stripe = lov_stripe_number(lsm, fm_start);
-	fm_end = (fm_length == ~0ULL) ? fmkey->lfik_oa.o_size :
-					fm_start + fm_length - 1;
-	/* If fm_length != ~0ULL but fm_start_fm_length-1 exceeds file size */
-	if (fm_end > fmkey->lfik_oa.o_size)
-		fm_end = fmkey->lfik_oa.o_size;
-
-	last_stripe = fiemap_calc_last_stripe(lsm, fm_start, fm_end,
-					      start_stripe, &stripe_count);
-	fm_end_offset = fiemap_calc_fm_end_offset(fiemap, lsm, fm_start, fm_end,
-						  &start_stripe);
-	if (fm_end_offset == -EINVAL) {
+	fs.fs_start_stripe = lov_stripe_number(lsm, fs.fs_start);
+	fs.fs_end = (fs.fs_length == ~0ULL) ? fmkey->lfik_oa.o_size :
+					      fs.fs_start + fs.fs_length - 1;
+	/* If fs_length != ~0ULL but fs_start+fs_length-1 exceeds file size */
+	if (fs.fs_end > fmkey->lfik_oa.o_size) {
+		fs.fs_end = fmkey->lfik_oa.o_size;
+		fs.fs_length = fs.fs_end - fs.fs_start;
+	}
+
+	fs.fs_last_stripe = fiemap_calc_last_stripe(lsm, fs.fs_start, fs.fs_end,
+						    fs.fs_start_stripe,
+						    &stripe_count);
+	fs.fs_end_offset = fiemap_calc_fm_end_offset(fiemap, lsm, fs.fs_start,
+						     fs.fs_end,
+						     &fs.fs_start_stripe);
+	if (fs.fs_end_offset == -EINVAL) {
 		rc = -EINVAL;
 		goto out;
 	}
 
+
 	/**
 	 * Requested extent count exceeds the fiemap buffer size, shrink our
 	 * ambition.
@@ -1244,186 +1419,23 @@ static int lov_object_fiemap(const struct lu_env *env, struct cl_object *obj,
 	if (fiemap_count_to_size(fiemap->fm_extent_count) > *buflen)
 		fiemap->fm_extent_count = fiemap_size_to_count(*buflen);
 	if (!fiemap->fm_extent_count)
-		count_local = 0;
+		fs.fs_cnt_need = 0;
+
+	fs.fs_finish = false;
+	fs.fs_enough = false;
+	fs.fs_cur_extent = 0;
 
 	/* Check each stripe */
-	for (cur_stripe = start_stripe; stripe_count > 0;
+	for (cur_stripe = fs.fs_start_stripe; stripe_count > 0;
 	     --stripe_count,
 	     cur_stripe = (cur_stripe + 1) % lsm->lsm_stripe_count) {
-		loff_t req_fm_len; /* Stores length of required mapping */
-		loff_t len_mapped_single_call;
-		loff_t lun_start;
-		loff_t lun_end;
-		loff_t obd_object_end;
-		unsigned int ext_count;
-
-		cur_stripe_wrap = cur_stripe;
-
-		/* Find out range of mapping on this stripe */
-		if (!(lov_stripe_intersects(lsm, cur_stripe, fm_start, fm_end,
-					    &lun_start, &obd_object_end)))
-			continue;
-
-		if (lov_oinfo_is_dummy(lsm->lsm_oinfo[cur_stripe])) {
-			rc = -EIO;
-			goto out;
-		}
-
-		/*
-		 * If this is a continuation FIEMAP call and we are on
-		 * starting stripe then lun_start needs to be set to
-		 * fm_end_offset
-		 */
-		if (fm_end_offset && cur_stripe == start_stripe)
-			lun_start = fm_end_offset;
-
-		if (fm_length != ~0ULL) {
-			/* Handle fm_start + fm_length overflow */
-			if (fm_start + fm_length < fm_start)
-				fm_length = ~0ULL - fm_start;
-			lun_end = lov_size_to_stripe(lsm, fm_start + fm_length,
-						     cur_stripe);
-		} else {
-			lun_end = ~0ULL;
-		}
-
-		if (lun_start == lun_end)
-			continue;
-
-		req_fm_len = obd_object_end - lun_start;
-		fm_local->fm_length = 0;
-		len_mapped_single_call = 0;
-
-		/* find lobsub object */
-		subobj = lov_find_subobj(env, cl2lov(obj), lsm,
-					 cur_stripe);
-		if (IS_ERR(subobj)) {
-			rc = PTR_ERR(subobj);
+		rc = fiemap_for_stripe(env, obj, lsm, fiemap, buflen, fmkey,
+				       cur_stripe, &fs);
+		if (rc < 0)
 			goto out;
-		}
-		/*
-		 * If the output buffer is very large and the objects have many
-		 * extents we may need to loop on a single OST repeatedly
-		 */
-		ost_eof = false;
-		ost_done = false;
-		do {
-			if (fiemap->fm_extent_count > 0) {
-				/* Don't get too many extents. */
-				if (current_extent + count_local >
-				    fiemap->fm_extent_count)
-					count_local = fiemap->fm_extent_count -
-						      current_extent;
-			}
-
-			lun_start += len_mapped_single_call;
-			fm_local->fm_length = req_fm_len -
-					      len_mapped_single_call;
-			req_fm_len = fm_local->fm_length;
-			fm_local->fm_extent_count = enough ? 1 : count_local;
-			fm_local->fm_mapped_extents = 0;
-			fm_local->fm_flags = fiemap->fm_flags;
-
-			ost_index = lsm->lsm_oinfo[cur_stripe]->loi_ost_idx;
-
-			if (ost_index < 0 ||
-			    ost_index >= lov->desc.ld_tgt_count) {
-				rc = -EINVAL;
-				goto obj_put;
-			}
-			/*
-			 * If OST is inactive, return extent with UNKNOWN
-			 * flag.
-			 */
-			if (!lov->lov_tgts[ost_index]->ltd_active) {
-				fm_local->fm_flags |= FIEMAP_EXTENT_LAST;
-				fm_local->fm_mapped_extents = 1;
-
-				lcl_fm_ext[0].fe_logical = lun_start;
-				lcl_fm_ext[0].fe_length = obd_object_end -
-							  lun_start;
-				lcl_fm_ext[0].fe_flags |= FIEMAP_EXTENT_UNKNOWN;
-
-				goto inactive_tgt;
-			}
-
-			fm_local->fm_start = lun_start;
-			fm_local->fm_flags &= ~FIEMAP_FLAG_DEVICE_ORDER;
-			memcpy(&fmkey->lfik_fiemap, fm_local, sizeof(*fm_local));
-			*buflen = fiemap_count_to_size(fm_local->fm_extent_count);
-
-			rc = cl_object_fiemap(env, subobj, fmkey, fm_local,
-					      buflen);
-			if (rc)
-				goto obj_put;
-inactive_tgt:
-			ext_count = fm_local->fm_mapped_extents;
-			if (!ext_count) {
-				ost_done = true;
-				/*
-				 * If last stripe has hold at the end,
-				 * we need to return
-				 */
-				if (cur_stripe_wrap == last_stripe) {
-					fiemap->fm_mapped_extents = 0;
-					goto finish;
-				}
-				break;
-			} else if (enough) {
-				/*
-				 * We've collected enough extents and there are
-				 * more extents after it.
-				 */
-				goto finish;
-			}
-
-			/* If we just need num of extents, got to next device */
-			if (!fiemap->fm_extent_count) {
-				current_extent += ext_count;
-				break;
-			}
-
-			/* prepare to copy retrived map extents */
-			len_mapped_single_call =
-				lcl_fm_ext[ext_count - 1].fe_logical -
-				lun_start + lcl_fm_ext[ext_count - 1].fe_length;
-
-			/* Have we finished mapping on this device? */
-			if (req_fm_len <= len_mapped_single_call)
-				ost_done = true;
-
-			/*
-			 * Clear the EXTENT_LAST flag which can be present on
-			 * the last extent
-			 */
-			if (lcl_fm_ext[ext_count - 1].fe_flags &
-			    FIEMAP_EXTENT_LAST)
-				lcl_fm_ext[ext_count - 1].fe_flags &=
-					~FIEMAP_EXTENT_LAST;
-
-			if (lov_stripe_size(lsm,
-					    lcl_fm_ext[ext_count - 1].fe_logical +
-					    lcl_fm_ext[ext_count - 1].fe_length,
-					    cur_stripe) >= fmkey->lfik_oa.o_size)
-				ost_eof = true;
-
-			fiemap_prepare_and_copy_exts(fiemap, lcl_fm_ext,
-						     ost_index, ext_count,
-						     current_extent);
-			current_extent += ext_count;
-
-			/* Ran out of available extents? */
-			if (current_extent >= fiemap->fm_extent_count)
-				enough = true;
-		} while (!ost_done && !ost_eof);
-
-		cl_object_put(env, subobj);
-		subobj = NULL;
-
-		if (cur_stripe_wrap == last_stripe)
-			goto finish;
+		if (fs.fs_finish)
+			break;
 	} /* for each stripe */
-finish:
 	/*
 	 * Indicate that we are returning device offsets unless file just has
 	 * single stripe
@@ -1438,14 +1450,11 @@ finish:
 	 * Check if we have reached the last stripe and whether mapping for that
 	 * stripe is done.
 	 */
-	if ((cur_stripe_wrap == last_stripe) && (ost_done || ost_eof))
-		fiemap->fm_extents[current_extent - 1].fe_flags |=
+	if ((cur_stripe == fs.fs_last_stripe) && fs.fs_device_done)
+		fiemap->fm_extents[fs.fs_cur_extent - 1].fe_flags |=
 							FIEMAP_EXTENT_LAST;
 skip_last_device_calc:
-	fiemap->fm_mapped_extents = current_extent;
-obj_put:
-	if (subobj)
-		cl_object_put(env, subobj);
+	fiemap->fm_mapped_extents = fs.fs_cur_extent;
 out:
 	kvfree(fm_local);
 	lov_lsm_put(lsm);
diff --git a/drivers/staging/lustre/lustre/lov/lov_offset.c b/drivers/staging/lustre/lustre/lov/lov_offset.c
index ecca74fbff00..899d12c41aab 100644
--- a/drivers/staging/lustre/lustre/lov/lov_offset.c
+++ b/drivers/staging/lustre/lustre/lov/lov_offset.c
@@ -32,9 +32,9 @@
 
 #define DEBUG_SUBSYSTEM S_LOV
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
-#include "../include/obd_class.h"
+#include <obd_class.h>
 
 #include "lov_internal.h"
 
diff --git a/drivers/staging/lustre/lustre/lov/lov_pack.c b/drivers/staging/lustre/lustre/lov/lov_pack.c
index 638b7646ca2c..24fb2a97532b 100644
--- a/drivers/staging/lustre/lustre/lov/lov_pack.c
+++ b/drivers/staging/lustre/lustre/lov/lov_pack.c
@@ -38,14 +38,11 @@
 
 #define DEBUG_SUBSYSTEM S_LOV
 
-#include "../include/lustre/lustre_idl.h"
-#include "../include/lustre/lustre_user.h"
-
-#include "../include/lustre_net.h"
-#include "../include/lustre_swab.h"
-#include "../include/obd.h"
-#include "../include/obd_class.h"
-#include "../include/obd_support.h"
+#include <lustre_net.h>
+#include <lustre_swab.h>
+#include <obd.h>
+#include <obd_class.h>
+#include <obd_support.h>
 
 #include "lov_cl_internal.h"
 #include "lov_internal.h"
diff --git a/drivers/staging/lustre/lustre/lov/lov_page.c b/drivers/staging/lustre/lustre/lov/lov_page.c
index 62ceb6dfdfdf..de43c609cf3d 100644
--- a/drivers/staging/lustre/lustre/lov/lov_page.c
+++ b/drivers/staging/lustre/lustre/lov/lov_page.c
@@ -100,7 +100,6 @@ int lov_page_init_raid0(const struct lu_env *env, struct cl_object *obj,
 				break;
 		}
 	}
-	lov_sub_put(sub);
 
 	return rc;
 }
diff --git a/drivers/staging/lustre/lustre/lov/lov_pool.c b/drivers/staging/lustre/lustre/lov/lov_pool.c
index 39daa17e0736..d774ee2a3675 100644
--- a/drivers/staging/lustre/lustre/lov/lov_pool.c
+++ b/drivers/staging/lustre/lustre/lov/lov_pool.c
@@ -40,9 +40,9 @@
 
 #define DEBUG_SUBSYSTEM S_LOV
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
-#include "../include/obd.h"
+#include <obd.h>
 #include "lov_internal.h"
 
 #define pool_tgt(_p, _i) \
diff --git a/drivers/staging/lustre/lustre/lov/lov_request.c b/drivers/staging/lustre/lustre/lov/lov_request.c
index 3a747913fb4f..9d3b3f3e9f10 100644
--- a/drivers/staging/lustre/lustre/lov/lov_request.c
+++ b/drivers/staging/lustre/lustre/lov/lov_request.c
@@ -32,10 +32,10 @@
 
 #define DEBUG_SUBSYSTEM S_LOV
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
-#include "../include/obd_class.h"
-#include "../include/lustre/lustre_idl.h"
+#include <obd_class.h>
+#include <uapi/linux/lustre/lustre_idl.h>
 #include "lov_internal.h"
 
 static void lov_init_set(struct lov_request_set *set)
@@ -43,13 +43,10 @@ static void lov_init_set(struct lov_request_set *set)
 	set->set_count = 0;
 	atomic_set(&set->set_completes, 0);
 	atomic_set(&set->set_success, 0);
-	atomic_set(&set->set_finish_checked, 0);
 	INIT_LIST_HEAD(&set->set_list);
-	atomic_set(&set->set_refcount, 1);
-	init_waitqueue_head(&set->set_waitq);
 }
 
-void lov_finish_set(struct lov_request_set *set)
+static void lov_finish_set(struct lov_request_set *set)
 {
 	struct list_head *pos, *n;
 
@@ -66,32 +63,12 @@ void lov_finish_set(struct lov_request_set *set)
 	kfree(set);
 }
 
-static int lov_set_finished(struct lov_request_set *set, int idempotent)
-{
-	int completes = atomic_read(&set->set_completes);
-
-	CDEBUG(D_INFO, "check set %d/%d\n", completes, set->set_count);
-
-	if (completes == set->set_count) {
-		if (idempotent)
-			return 1;
-		if (atomic_inc_return(&set->set_finish_checked) == 1)
-			return 1;
-	}
-	return 0;
-}
-
 static void lov_update_set(struct lov_request_set *set,
 			   struct lov_request *req, int rc)
 {
-	req->rq_complete = 1;
-	req->rq_rc = rc;
-
 	atomic_inc(&set->set_completes);
 	if (rc == 0)
 		atomic_inc(&set->set_success);
-
-	wake_up(&set->set_waitq);
 }
 
 static void lov_set_add_req(struct lov_request *req,
@@ -173,8 +150,8 @@ out:
 			(tot) += (add);				 \
 	} while (0)
 
-int lov_fini_statfs(struct obd_device *obd, struct obd_statfs *osfs,
-		    int success)
+static int lov_fini_statfs(struct obd_device *obd, struct obd_statfs *osfs,
+			   int success)
 {
 	if (success) {
 		__u32 expected_stripes = lov_get_stripecnt(&obd->u.lov,
@@ -205,7 +182,9 @@ int lov_fini_statfs_set(struct lov_request_set *set)
 		rc = lov_fini_statfs(set->set_obd, set->set_oi->oi_osfs,
 				     atomic_read(&set->set_success));
 	}
-	lov_put_reqset(set);
+
+	lov_finish_set(set);
+
 	return rc;
 }
 
@@ -307,14 +286,7 @@ static int cb_statfs_update(void *cookie, int rc)
 out_update:
 	lov_update_statfs(osfs, lov_sfs, success);
 	obd_putref(lovobd);
-
 out:
-	if (set->set_oi->oi_flags & OBD_STATFS_PTLRPCD &&
-	    lov_set_finished(set, 0)) {
-		lov_statfs_interpret(NULL, set, set->set_count !=
-				     atomic_read(&set->set_success));
-	}
-
 	return 0;
 }
 
diff --git a/drivers/staging/lustre/lustre/lov/lovsub_dev.c b/drivers/staging/lustre/lustre/lov/lovsub_dev.c
index 5d6536f8a4f7..d4646a0949d2 100644
--- a/drivers/staging/lustre/lustre/lov/lovsub_dev.c
+++ b/drivers/staging/lustre/lustre/lov/lovsub_dev.c
@@ -77,7 +77,6 @@ static struct lu_device *lovsub_device_fini(const struct lu_env *env,
 
 	lsd = lu2lovsub_dev(d);
 	next = cl2lu_dev(lsd->acid_next);
-	lsd->acid_super = NULL;
 	lsd->acid_next = NULL;
 	return next;
 }
diff --git a/drivers/staging/lustre/lustre/lov/lovsub_io.c b/drivers/staging/lustre/lustre/lov/lovsub_io.c
deleted file mode 100644
index 6a9820218a3e..000000000000
--- a/drivers/staging/lustre/lustre/lov/lovsub_io.c
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * GPL HEADER START
- *
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 only,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License version 2 for more details (a copy is included
- * in the LICENSE file that accompanied this code).
- *
- * You should have received a copy of the GNU General Public License
- * version 2 along with this program; If not, see
- * http://www.gnu.org/licenses/gpl-2.0.html
- *
- * GPL HEADER END
- */
-/*
- * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved.
- * Use is subject to license terms.
- */
-/*
- * This file is part of Lustre, http://www.lustre.org/
- * Lustre is a trademark of Sun Microsystems, Inc.
- *
- * Implementation of cl_io for LOVSUB layer.
- *
- *   Author: Nikita Danilov <nikita.danilov@sun.com>
- */
-
-#define DEBUG_SUBSYSTEM S_LOV
-
-#include "lov_cl_internal.h"
-
-/** \addtogroup lov
- *  @{
- */
-
-/*****************************************************************************
- *
- * Lovsub io operations.
- *
- */
-
-/* All trivial */
-
-/** @} lov */
diff --git a/drivers/staging/lustre/lustre/lov/lovsub_lock.c b/drivers/staging/lustre/lustre/lov/lovsub_lock.c
index 38f9b735c241..d29f0bb33980 100644
--- a/drivers/staging/lustre/lustre/lov/lovsub_lock.c
+++ b/drivers/staging/lustre/lustre/lov/lovsub_lock.c
@@ -54,7 +54,6 @@ static void lovsub_lock_fini(const struct lu_env *env,
 	struct lovsub_lock   *lsl;
 
 	lsl = cl2lovsub_lock(slice);
-	LASSERT(list_empty(&lsl->lss_parents));
 	kmem_cache_free(lovsub_lock_kmem, lsl);
 }
 
@@ -70,7 +69,6 @@ int lovsub_lock_init(const struct lu_env *env, struct cl_object *obj,
 
 	lsk = kmem_cache_zalloc(lovsub_lock_kmem, GFP_NOFS);
 	if (lsk) {
-		INIT_LIST_HEAD(&lsk->lss_parents);
 		cl_lock_slice_add(lock, &lsk->lss_cl, obj, &lovsub_lock_ops);
 		result = 0;
 	} else {
diff --git a/drivers/staging/lustre/lustre/lov/lproc_lov.c b/drivers/staging/lustre/lustre/lov/lproc_lov.c
index eb6d30d34e3a..9bb7e9ea0a6a 100644
--- a/drivers/staging/lustre/lustre/lov/lproc_lov.c
+++ b/drivers/staging/lustre/lustre/lov/lproc_lov.c
@@ -32,8 +32,8 @@
 #define DEBUG_SUBSYSTEM S_CLASS
 
 #include <linux/statfs.h>
-#include "../include/lprocfs_status.h"
-#include "../include/obd_class.h"
+#include <lprocfs_status.h>
+#include <obd_class.h>
 #include <linux/seq_file.h>
 #include "lov_internal.h"
 
@@ -279,7 +279,7 @@ static struct attribute *lov_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group lov_attr_group = {
+static const struct attribute_group lov_attr_group = {
 	.attrs = lov_attrs,
 };
 
diff --git a/drivers/staging/lustre/lustre/mdc/Makefile b/drivers/staging/lustre/lustre/mdc/Makefile
index 99ba9ff0d83a..c7bc3351ccb0 100644
--- a/drivers/staging/lustre/lustre/mdc/Makefile
+++ b/drivers/staging/lustre/lustre/mdc/Makefile
@@ -1,2 +1,5 @@
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/include
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/lustre/include
+
 obj-$(CONFIG_LUSTRE_FS) += mdc.o
 mdc-y := mdc_request.o mdc_reint.o mdc_lib.o mdc_locks.o lproc_mdc.o
diff --git a/drivers/staging/lustre/lustre/mdc/lproc_mdc.c b/drivers/staging/lustre/lustre/mdc/lproc_mdc.c
index 9021c465c044..f68513771527 100644
--- a/drivers/staging/lustre/lustre/mdc/lproc_mdc.c
+++ b/drivers/staging/lustre/lustre/mdc/lproc_mdc.c
@@ -32,8 +32,8 @@
 #define DEBUG_SUBSYSTEM S_CLASS
 
 #include <linux/vfs.h>
-#include "../include/obd_class.h"
-#include "../include/lprocfs_status.h"
+#include <obd_class.h>
+#include <lprocfs_status.h>
 #include "mdc_internal.h"
 
 static ssize_t active_show(struct kobject *kobj, struct attribute *attr,
@@ -57,7 +57,7 @@ static ssize_t active_store(struct kobject *kobj, struct attribute *attr,
 	if (rc)
 		return rc;
 
-	if (val < 0 || val > 1)
+	if (val > 1)
 		return -ERANGE;
 
 	/* opposite senses */
@@ -219,7 +219,7 @@ static struct attribute *mdc_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group mdc_attr_group = {
+static const struct attribute_group mdc_attr_group = {
 	.attrs = mdc_attrs,
 };
 
diff --git a/drivers/staging/lustre/lustre/mdc/mdc_internal.h b/drivers/staging/lustre/lustre/mdc/mdc_internal.h
index fecedc8819ed..cbf011501005 100644
--- a/drivers/staging/lustre/lustre/mdc/mdc_internal.h
+++ b/drivers/staging/lustre/lustre/mdc/mdc_internal.h
@@ -33,7 +33,7 @@
 #ifndef _MDC_INTERNAL_H
 #define _MDC_INTERNAL_H
 
-#include "../include/lustre_mdc.h"
+#include <lustre_mdc.h>
 
 void lprocfs_mdc_init_vars(struct lprocfs_static_vars *lvars);
 
diff --git a/drivers/staging/lustre/lustre/mdc/mdc_lib.c b/drivers/staging/lustre/lustre/mdc/mdc_lib.c
index b1853ff7f8b9..ba13f0894e0d 100644
--- a/drivers/staging/lustre/lustre/mdc/mdc_lib.c
+++ b/drivers/staging/lustre/lustre/mdc/mdc_lib.c
@@ -31,10 +31,16 @@
  */
 
 #define DEBUG_SUBSYSTEM S_MDC
-#include "../include/lustre_net.h"
-#include "../include/lustre/lustre_idl.h"
+#include <lustre_net.h>
+#include <uapi/linux/lustre/lustre_idl.h>
 #include "mdc_internal.h"
 
+static void set_mrc_cr_flags(struct mdt_rec_create *mrc, u64 flags)
+{
+	mrc->cr_flags_l = (u32)(flags & 0xFFFFFFFFUll);
+	mrc->cr_flags_h = (u32)(flags >> 32);
+}
+
 static void __mdc_pack_body(struct mdt_body *b, __u32 suppgid)
 {
 	b->mbo_suppgid = suppgid;
diff --git a/drivers/staging/lustre/lustre/mdc/mdc_locks.c b/drivers/staging/lustre/lustre/mdc/mdc_locks.c
index 3eb66cea65db..cbfea3dd0319 100644
--- a/drivers/staging/lustre/lustre/mdc/mdc_locks.c
+++ b/drivers/staging/lustre/lustre/mdc/mdc_locks.c
@@ -32,17 +32,17 @@
 
 #define DEBUG_SUBSYSTEM S_MDC
 
-# include <linux/module.h>
-
-#include "../include/lustre_intent.h"
-#include "../include/obd.h"
-#include "../include/obd_class.h"
-#include "../include/lustre_dlm.h"
-#include "../include/lustre_fid.h"
-#include "../include/lustre_mdc.h"
-#include "../include/lustre_net.h"
-#include "../include/lustre_req_layout.h"
-#include "../include/lustre_swab.h"
+#include <linux/module.h>
+
+#include <lustre_intent.h>
+#include <obd.h>
+#include <obd_class.h>
+#include <lustre_dlm.h>
+#include <lustre_fid.h>
+#include <lustre_mdc.h>
+#include <lustre_net.h>
+#include <lustre_req_layout.h>
+#include <lustre_swab.h>
 
 #include "mdc_internal.h"
 
@@ -1030,7 +1030,7 @@ int mdc_revalidate_lock(struct obd_export *exp, struct lookup_intent *it,
  * If we're performing a creation, that means that unless the creation
  * failed with EEXIST, we should fake up a negative dentry.
  *
- * For everything else, we want to lookup to succeed.
+ * For everything else, we want the lookup to succeed.
  *
  * One additional note: if CREATE or OPEN succeeded, we add an extra
  * reference to the request because we need to keep it around until
@@ -1040,7 +1040,7 @@ int mdc_revalidate_lock(struct obd_export *exp, struct lookup_intent *it,
  * exactly what it_status refers to.
  *
  * If DISP_OPEN_OPEN is set, then it_status refers to the open() call,
- * otherwise if DISP_OPEN_CREATE is set, then it status is the
+ * otherwise if DISP_OPEN_CREATE is set, then it_status is the
  * creation failure mode.  In either case, one of DISP_LOOKUP_NEG or
  * DISP_LOOKUP_POS will be set, indicating whether the child lookup
  * was successful.
diff --git a/drivers/staging/lustre/lustre/mdc/mdc_reint.c b/drivers/staging/lustre/lustre/mdc/mdc_reint.c
index 2287bd46d527..f45c91d1b4ae 100644
--- a/drivers/staging/lustre/lustre/mdc/mdc_reint.c
+++ b/drivers/staging/lustre/lustre/mdc/mdc_reint.c
@@ -35,9 +35,9 @@
 # include <linux/module.h>
 # include <linux/kernel.h>
 
-#include "../include/obd_class.h"
+#include <obd_class.h>
 #include "mdc_internal.h"
-#include "../include/lustre_fid.h"
+#include <lustre_fid.h>
 
 /* mdc_setattr does its own semaphore handling */
 static int mdc_reint(struct ptlrpc_request *request, int level)
diff --git a/drivers/staging/lustre/lustre/mdc/mdc_request.c b/drivers/staging/lustre/lustre/mdc/mdc_request.c
index 1a3fa1bb7f25..6ef8ddec4ab6 100644
--- a/drivers/staging/lustre/lustre/mdc/mdc_request.c
+++ b/drivers/staging/lustre/lustre/mdc/mdc_request.c
@@ -38,18 +38,19 @@
 # include <linux/init.h>
 # include <linux/utsname.h>
 
-#include "../include/cl_object.h"
-#include "../include/llog_swab.h"
-#include "../include/lprocfs_status.h"
-#include "../include/lustre_acl.h"
-#include "../include/lustre_fid.h"
-#include "../include/lustre/lustre_ioctl.h"
-#include "../include/lustre_kernelcomm.h"
-#include "../include/lustre_lmv.h"
-#include "../include/lustre_log.h"
-#include "../include/lustre_param.h"
-#include "../include/lustre_swab.h"
-#include "../include/obd_class.h"
+#include <lustre_errno.h>
+#include <cl_object.h>
+#include <llog_swab.h>
+#include <lprocfs_status.h>
+#include <lustre_acl.h>
+#include <lustre_fid.h>
+#include <uapi/linux/lustre/lustre_ioctl.h>
+#include <lustre_kernelcomm.h>
+#include <lustre_lmv.h>
+#include <lustre_log.h>
+#include <uapi/linux/lustre/lustre_param.h>
+#include <lustre_swab.h>
+#include <obd_class.h>
 
 #include "mdc_internal.h"
 
diff --git a/drivers/staging/lustre/lustre/mgc/Makefile b/drivers/staging/lustre/lustre/mgc/Makefile
index 8ea29a89cf50..8abf108dbcf7 100644
--- a/drivers/staging/lustre/lustre/mgc/Makefile
+++ b/drivers/staging/lustre/lustre/mgc/Makefile
@@ -1,2 +1,5 @@
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/include
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/lustre/include
+
 obj-$(CONFIG_LUSTRE_FS) += mgc.o
 mgc-y := mgc_request.o lproc_mgc.o
diff --git a/drivers/staging/lustre/lustre/mgc/lproc_mgc.c b/drivers/staging/lustre/lustre/mgc/lproc_mgc.c
index 0735220b2a18..2ec2d7f731d3 100644
--- a/drivers/staging/lustre/lustre/mgc/lproc_mgc.c
+++ b/drivers/staging/lustre/lustre/mgc/lproc_mgc.c
@@ -32,8 +32,8 @@
 #define DEBUG_SUBSYSTEM S_CLASS
 
 #include <linux/vfs.h>
-#include "../include/obd_class.h"
-#include "../include/lprocfs_status.h"
+#include <obd_class.h>
+#include <lprocfs_status.h>
 #include "mgc_internal.h"
 
 LPROC_SEQ_FOPS_RO_TYPE(mgc, connect_flags);
diff --git a/drivers/staging/lustre/lustre/mgc/mgc_internal.h b/drivers/staging/lustre/lustre/mgc/mgc_internal.h
index f146f7521c92..7a2f2b7bc6b1 100644
--- a/drivers/staging/lustre/lustre/mgc/mgc_internal.h
+++ b/drivers/staging/lustre/lustre/mgc/mgc_internal.h
@@ -33,12 +33,11 @@
 #ifndef _MGC_INTERNAL_H
 #define _MGC_INTERNAL_H
 
-#include "../../include/linux/libcfs/libcfs.h"
-#include "../include/lustre/lustre_idl.h"
-#include "../include/lustre_lib.h"
-#include "../include/lustre_dlm.h"
-#include "../include/lustre_log.h"
-#include "../include/lustre_export.h"
+#include <linux/libcfs/libcfs.h>
+#include <lustre_lib.h>
+#include <lustre_dlm.h>
+#include <lustre_log.h>
+#include <lustre_export.h>
 
 void lprocfs_mgc_init_vars(struct lprocfs_static_vars *lvars);
 int lprocfs_mgc_rd_ir_state(struct seq_file *m, void *data);
diff --git a/drivers/staging/lustre/lustre/mgc/mgc_request.c b/drivers/staging/lustre/lustre/mgc/mgc_request.c
index eee0b667a33c..3d2b969c90a7 100644
--- a/drivers/staging/lustre/lustre/mgc/mgc_request.c
+++ b/drivers/staging/lustre/lustre/mgc/mgc_request.c
@@ -39,12 +39,12 @@
 
 #include <linux/module.h>
 
-#include "../include/lprocfs_status.h"
-#include "../include/lustre_dlm.h"
-#include "../include/lustre_disk.h"
-#include "../include/lustre_log.h"
-#include "../include/lustre_swab.h"
-#include "../include/obd_class.h"
+#include <lprocfs_status.h>
+#include <lustre_dlm.h>
+#include <lustre_disk.h>
+#include <lustre_log.h>
+#include <lustre_swab.h>
+#include <obd_class.h>
 
 #include "mgc_internal.h"
 
@@ -288,7 +288,7 @@ config_log_add(struct obd_device *obd, char *logname,
 	struct config_llog_data *cld;
 	struct config_llog_data *sptlrpc_cld;
 	struct config_llog_data *params_cld;
-	bool locked = false;
+	struct config_llog_data *recover_cld = NULL;
 	char			seclogname[32];
 	char			*ptr;
 	int			rc;
@@ -333,20 +333,14 @@ config_log_add(struct obd_device *obd, char *logname,
 		goto out_params;
 	}
 
-	cld->cld_sptlrpc = sptlrpc_cld;
-	cld->cld_params = params_cld;
-
 	LASSERT(lsi->lsi_lmd);
 	if (!(lsi->lsi_lmd->lmd_flags & LMD_FLG_NOIR)) {
-		struct config_llog_data *recover_cld;
-
 		ptr = strrchr(seclogname, '-');
 		if (ptr) {
 			*ptr = 0;
 		} else {
 			CERROR("%s: sptlrpc log name not correct, %s: rc = %d\n",
 			       obd->obd_name, seclogname, -EINVAL);
-			config_log_put(cld);
 			rc = -EINVAL;
 			goto out_cld;
 		}
@@ -355,14 +349,10 @@ config_log_add(struct obd_device *obd, char *logname,
 			rc = PTR_ERR(recover_cld);
 			goto out_cld;
 		}
-
-		mutex_lock(&cld->cld_lock);
-		locked = true;
-		cld->cld_recover = recover_cld;
 	}
 
-	if (!locked)
-		mutex_lock(&cld->cld_lock);
+	mutex_lock(&cld->cld_lock);
+	cld->cld_recover = recover_cld;
 	cld->cld_params = params_cld;
 	cld->cld_sptlrpc = sptlrpc_cld;
 	mutex_unlock(&cld->cld_lock);
@@ -1165,6 +1155,7 @@ static int mgc_apply_recover_logs(struct obd_device *mgc,
 		char *cname;
 		char *params;
 		char *uuid;
+		size_t len;
 
 		rc = -EINVAL;
 		if (datalen < sizeof(*entry))
@@ -1293,17 +1284,19 @@ static int mgc_apply_recover_logs(struct obd_device *mgc,
 		lustre_cfg_bufs_set_string(&bufs, 1, params);
 
 		rc = -ENOMEM;
-		lcfg = lustre_cfg_new(LCFG_PARAM, &bufs);
-		if (IS_ERR(lcfg)) {
-			CERROR("mgc: cannot allocate memory\n");
+		len = lustre_cfg_len(bufs.lcfg_bufcount, bufs.lcfg_buflen);
+		lcfg = kzalloc(len, GFP_NOFS);
+		if (!lcfg) {
+			rc = -ENOMEM;
 			break;
 		}
+		lustre_cfg_init(lcfg, LCFG_PARAM, &bufs);
 
 		CDEBUG(D_INFO, "ir apply logs %lld/%lld for %s -> %s\n",
 		       prev_version, max_version, obdname, params);
 
 		rc = class_process_config(lcfg);
-		lustre_cfg_free(lcfg);
+		kfree(lcfg);
 		if (rc)
 			CDEBUG(D_INFO, "process config for %s error %d\n",
 			       obdname, rc);
diff --git a/drivers/staging/lustre/lustre/obdclass/Makefile b/drivers/staging/lustre/lustre/obdclass/Makefile
index af570c0db15b..fa0ad6548ecd 100644
--- a/drivers/staging/lustre/lustre/obdclass/Makefile
+++ b/drivers/staging/lustre/lustre/obdclass/Makefile
@@ -1,3 +1,6 @@
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/include
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/lustre/include
+
 obj-$(CONFIG_LUSTRE_FS) += obdclass.o
 
 obdclass-y := linux/linux-module.o linux/linux-sysctl.o \
diff --git a/drivers/staging/lustre/lustre/obdclass/cl_io.c b/drivers/staging/lustre/lustre/obdclass/cl_io.c
index ee7d67761191..2a70e21ae07f 100644
--- a/drivers/staging/lustre/lustre/obdclass/cl_io.c
+++ b/drivers/staging/lustre/lustre/obdclass/cl_io.c
@@ -37,12 +37,12 @@
 
 #define DEBUG_SUBSYSTEM S_CLASS
 
-#include "../include/obd_class.h"
-#include "../include/obd_support.h"
-#include "../include/lustre_fid.h"
+#include <obd_class.h>
+#include <obd_support.h>
+#include <lustre_fid.h>
 #include <linux/list.h>
 #include <linux/sched.h>
-#include "../include/cl_object.h"
+#include <cl_object.h>
 #include "cl_internal.h"
 
 /*****************************************************************************
diff --git a/drivers/staging/lustre/lustre/obdclass/cl_lock.c b/drivers/staging/lustre/lustre/obdclass/cl_lock.c
index a343e3ab2257..20e64051d2d6 100644
--- a/drivers/staging/lustre/lustre/obdclass/cl_lock.c
+++ b/drivers/staging/lustre/lustre/obdclass/cl_lock.c
@@ -37,11 +37,11 @@
 
 #define DEBUG_SUBSYSTEM S_CLASS
 
-#include "../include/obd_class.h"
-#include "../include/obd_support.h"
-#include "../include/lustre_fid.h"
+#include <obd_class.h>
+#include <obd_support.h>
+#include <lustre_fid.h>
 #include <linux/list.h>
-#include "../include/cl_object.h"
+#include <cl_object.h>
 #include "cl_internal.h"
 
 static void cl_lock_trace0(int level, const struct lu_env *env,
diff --git a/drivers/staging/lustre/lustre/obdclass/cl_object.c b/drivers/staging/lustre/lustre/obdclass/cl_object.c
index 08e55d418537..95c7fa3b532c 100644
--- a/drivers/staging/lustre/lustre/obdclass/cl_object.c
+++ b/drivers/staging/lustre/lustre/obdclass/cl_object.c
@@ -46,15 +46,15 @@
 
 #define DEBUG_SUBSYSTEM S_CLASS
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 /* class_put_type() */
-#include "../include/obd_class.h"
-#include "../include/obd_support.h"
-#include "../include/lustre_fid.h"
+#include <obd_class.h>
+#include <obd_support.h>
+#include <lustre_fid.h>
 #include <linux/list.h>
-#include "../../include/linux/libcfs/libcfs_hash.h"	/* for cfs_hash stuff */
-#include "../include/cl_object.h"
-#include "../include/lu_object.h"
+#include <linux/libcfs/libcfs_hash.h>	/* for cfs_hash stuff */
+#include <cl_object.h>
+#include <lu_object.h>
 #include "cl_internal.h"
 
 static struct kmem_cache *cl_env_kmem;
diff --git a/drivers/staging/lustre/lustre/obdclass/cl_page.c b/drivers/staging/lustre/lustre/obdclass/cl_page.c
index 6b8c41b6f379..3dc084cb93bc 100644
--- a/drivers/staging/lustre/lustre/obdclass/cl_page.c
+++ b/drivers/staging/lustre/lustre/obdclass/cl_page.c
@@ -37,12 +37,12 @@
 
 #define DEBUG_SUBSYSTEM S_CLASS
 
-#include "../../include/linux/libcfs/libcfs.h"
-#include "../include/obd_class.h"
-#include "../include/obd_support.h"
+#include <linux/libcfs/libcfs.h>
+#include <obd_class.h>
+#include <obd_support.h>
 #include <linux/list.h>
 
-#include "../include/cl_object.h"
+#include <cl_object.h>
 #include "cl_internal.h"
 
 static void cl_page_delete0(const struct lu_env *env, struct cl_page *pg);
diff --git a/drivers/staging/lustre/lustre/obdclass/class_obd.c b/drivers/staging/lustre/lustre/obdclass/class_obd.c
index 76e1ee83a723..2df218b010e1 100644
--- a/drivers/staging/lustre/lustre/obdclass/class_obd.c
+++ b/drivers/staging/lustre/lustre/obdclass/class_obd.c
@@ -33,14 +33,14 @@
 #define DEBUG_SUBSYSTEM S_CLASS
 # include <linux/atomic.h>
 
-#include "../include/obd_support.h"
-#include "../include/obd_class.h"
-#include "../../include/linux/lnet/lnetctl.h"
-#include "../include/lustre_debug.h"
-#include "../include/lprocfs_status.h"
+#include <obd_support.h>
+#include <obd_class.h>
+#include <uapi/linux/lnet/lnetctl.h>
+#include <lustre_debug.h>
+#include <lprocfs_status.h>
 #include <linux/list.h>
-#include "../include/cl_object.h"
-#include "../include/lustre/lustre_ioctl.h"
+#include <cl_object.h>
+#include <uapi/linux/lustre/lustre_ioctl.h>
 #include "llog_internal.h"
 
 struct obd_device *obd_devs[MAX_OBD_DEVICES];
@@ -180,7 +180,8 @@ int class_handle_ioctl(unsigned int cmd, unsigned long arg)
 			err = -ENOMEM;
 			goto out;
 		}
-		err = copy_from_user(lcfg, data->ioc_pbuf1, data->ioc_plen1);
+		if (copy_from_user(lcfg, data->ioc_pbuf1, data->ioc_plen1))
+			err = -EFAULT;
 		if (!err)
 			err = lustre_cfg_sanity_check(lcfg, data->ioc_plen1);
 		if (!err)
@@ -206,8 +207,7 @@ int class_handle_ioctl(unsigned int cmd, unsigned long arg)
 		memcpy(data->ioc_bulk, LUSTRE_VERSION_STRING,
 		       strlen(LUSTRE_VERSION_STRING) + 1);
 
-		err = obd_ioctl_popdata((void __user *)arg, data, len);
-		if (err)
+		if (copy_to_user((void __user *)arg, data, len))
 			err = -EFAULT;
 		goto out;
 
@@ -225,9 +225,7 @@ int class_handle_ioctl(unsigned int cmd, unsigned long arg)
 			goto out;
 		}
 
-		err = obd_ioctl_popdata((void __user *)arg, data,
-					sizeof(*data));
-		if (err)
+		if (copy_to_user((void __user *)arg, data, sizeof(*data)))
 			err = -EFAULT;
 		goto out;
 	}
@@ -263,9 +261,8 @@ int class_handle_ioctl(unsigned int cmd, unsigned long arg)
 
 		CDEBUG(D_IOCTL, "device name %s, dev %d\n", data->ioc_inlbuf1,
 		       dev);
-		err = obd_ioctl_popdata((void __user *)arg, data,
-					sizeof(*data));
-		if (err)
+
+		if (copy_to_user((void __user *)arg, data, sizeof(*data)))
 			err = -EFAULT;
 		goto out;
 	}
@@ -304,9 +301,9 @@ int class_handle_ioctl(unsigned int cmd, unsigned long arg)
 			 (int)index, status, obd->obd_type->typ_name,
 			 obd->obd_name, obd->obd_uuid.uuid,
 			 atomic_read(&obd->obd_refcount));
-		err = obd_ioctl_popdata((void __user *)arg, data, len);
 
-		err = 0;
+		if (copy_to_user((void __user *)arg, data, len))
+			err = -EFAULT;
 		goto out;
 	}
 	}
@@ -361,16 +358,14 @@ int class_handle_ioctl(unsigned int cmd, unsigned long arg)
 		if (err)
 			goto out;
 
-		err = obd_ioctl_popdata((void __user *)arg, data, len);
-		if (err)
+		if (copy_to_user((void __user *)arg, data, len))
 			err = -EFAULT;
 		goto out;
 	}
 	}
 
  out:
-	if (buf)
-		obd_ioctl_freedata(buf, len);
+	kvfree(buf);
 	return err;
 } /* class_handle_ioctl */
 
@@ -453,7 +448,7 @@ static int __init obdclass_init(void)
 	obd_zombie_impexp_init();
 
 	err = obd_init_checks();
-	if (err == -EOVERFLOW)
+	if (err)
 		return err;
 
 	class_init_uuidlist();
diff --git a/drivers/staging/lustre/lustre/obdclass/debug.c b/drivers/staging/lustre/lustre/obdclass/debug.c
index 0bd4ad20aba7..7964cad7e780 100644
--- a/drivers/staging/lustre/lustre/obdclass/debug.c
+++ b/drivers/staging/lustre/lustre/obdclass/debug.c
@@ -38,9 +38,9 @@
 
 #include <asm/unaligned.h>
 
-#include "../include/obd_support.h"
-#include "../include/lustre_debug.h"
-#include "../include/lustre_net.h"
+#include <obd_support.h>
+#include <lustre_debug.h>
+#include <lustre_net.h>
 
 #define LPDS sizeof(__u64)
 int block_debug_setup(void *addr, int len, __u64 off, __u64 id)
diff --git a/drivers/staging/lustre/lustre/obdclass/genops.c b/drivers/staging/lustre/lustre/obdclass/genops.c
index fa0d38ddccb2..739bfb9421ca 100644
--- a/drivers/staging/lustre/lustre/obdclass/genops.c
+++ b/drivers/staging/lustre/lustre/obdclass/genops.c
@@ -36,9 +36,9 @@
  */
 
 #define DEBUG_SUBSYSTEM S_CLASS
-#include "../include/obd_class.h"
-#include "../include/lprocfs_status.h"
-#include "../include/lustre_kernelcomm.h"
+#include <obd_class.h>
+#include <lprocfs_status.h>
+#include <lustre_kernelcomm.h>
 
 spinlock_t obd_types_lock;
 
diff --git a/drivers/staging/lustre/lustre/obdclass/kernelcomm.c b/drivers/staging/lustre/lustre/obdclass/kernelcomm.c
index a0f65c470f4d..4f0a42633d5a 100644
--- a/drivers/staging/lustre/lustre/obdclass/kernelcomm.c
+++ b/drivers/staging/lustre/lustre/obdclass/kernelcomm.c
@@ -38,8 +38,8 @@
 #define DEBUG_SUBSYSTEM S_CLASS
 #define D_KUC D_OTHER
 
-#include "../include/obd_support.h"
-#include "../include/lustre_kernelcomm.h"
+#include <obd_support.h>
+#include <lustre_kernelcomm.h>
 
 /**
  * libcfs_kkuc_msg_put - send an message from kernel to userspace
@@ -52,7 +52,6 @@ int libcfs_kkuc_msg_put(struct file *filp, void *payload)
 	struct kuc_hdr *kuch = (struct kuc_hdr *)payload;
 	ssize_t count = kuch->kuc_msglen;
 	loff_t offset = 0;
-	mm_segment_t fs;
 	int rc = -ENXIO;
 
 	if (IS_ERR_OR_NULL(filp))
@@ -63,18 +62,14 @@ int libcfs_kkuc_msg_put(struct file *filp, void *payload)
 		return rc;
 	}
 
-	fs = get_fs();
-	set_fs(KERNEL_DS);
 	while (count > 0) {
-		rc = vfs_write(filp, (void __force __user *)payload,
-			       count, &offset);
+		rc = kernel_write(filp, payload, count, &offset);
 		if (rc < 0)
 			break;
 		count -= rc;
 		payload += rc;
 		rc = 0;
 	}
-	set_fs(fs);
 
 	if (rc < 0)
 		CWARN("message send failed (%d)\n", rc);
diff --git a/drivers/staging/lustre/lustre/obdclass/linkea.c b/drivers/staging/lustre/lustre/obdclass/linkea.c
index 0b1d2f0a422c..9af86d3d56e4 100644
--- a/drivers/staging/lustre/lustre/obdclass/linkea.c
+++ b/drivers/staging/lustre/lustre/obdclass/linkea.c
@@ -26,9 +26,9 @@
  * Author: Di Wang <di.wang@intel.com>
  */
 
-#include "../include/lustre/lustre_idl.h"
-#include "../include/obd.h"
-#include "../include/lustre_linkea.h"
+#include <uapi/linux/lustre/lustre_idl.h>
+#include <obd.h>
+#include <lustre_linkea.h>
 
 int linkea_data_new(struct linkea_data *ldata, struct lu_buf *buf)
 {
@@ -39,6 +39,8 @@ int linkea_data_new(struct linkea_data *ldata, struct lu_buf *buf)
 	ldata->ld_leh->leh_magic = LINK_EA_MAGIC;
 	ldata->ld_leh->leh_len = sizeof(struct link_ea_header);
 	ldata->ld_leh->leh_reccount = 0;
+	ldata->ld_leh->leh_overflow_time = 0;
+	ldata->ld_leh->leh_padding = 0;
 	return 0;
 }
 EXPORT_SYMBOL(linkea_data_new);
@@ -53,11 +55,15 @@ int linkea_init(struct linkea_data *ldata)
 		leh->leh_magic = LINK_EA_MAGIC;
 		leh->leh_reccount = __swab32(leh->leh_reccount);
 		leh->leh_len = __swab64(leh->leh_len);
-		/* entries are swabbed by linkea_entry_unpack */
+		leh->leh_overflow_time = __swab32(leh->leh_overflow_time);
+		leh->leh_padding = __swab32(leh->leh_padding);
+		/* individual entries are swabbed by linkea_entry_unpack() */
 	}
+
 	if (leh->leh_magic != LINK_EA_MAGIC)
 		return -EINVAL;
-	if (leh->leh_reccount == 0)
+
+	if (leh->leh_reccount == 0 && leh->leh_overflow_time == 0)
 		return -ENODATA;
 
 	ldata->ld_leh = leh;
@@ -65,6 +71,18 @@ int linkea_init(struct linkea_data *ldata)
 }
 EXPORT_SYMBOL(linkea_init);
 
+int linkea_init_with_rec(struct linkea_data *ldata)
+{
+	int rc;
+
+	rc = linkea_init(ldata);
+	if (!rc && ldata->ld_leh->leh_reccount == 0)
+		rc = -ENODATA;
+
+	return rc;
+}
+EXPORT_SYMBOL(linkea_init_with_rec);
+
 /**
  * Pack a link_ea_entry.
  * All elements are stored as chars to avoid alignment issues.
@@ -94,6 +112,8 @@ EXPORT_SYMBOL(linkea_entry_pack);
 void linkea_entry_unpack(const struct link_ea_entry *lee, int *reclen,
 			 struct lu_name *lname, struct lu_fid *pfid)
 {
+	LASSERT(lee);
+
 	*reclen = (lee->lee_reclen[0] << 8) | lee->lee_reclen[1];
 	memcpy(pfid, &lee->lee_parent_fid, sizeof(*pfid));
 	fid_be_to_cpu(pfid, pfid);
@@ -110,25 +130,44 @@ EXPORT_SYMBOL(linkea_entry_unpack);
 int linkea_add_buf(struct linkea_data *ldata, const struct lu_name *lname,
 		   const struct lu_fid *pfid)
 {
-	LASSERT(ldata->ld_leh);
+	struct link_ea_header *leh = ldata->ld_leh;
+	int reclen;
+
+	LASSERT(leh);
 
 	if (!lname || !pfid)
 		return -EINVAL;
 
-	ldata->ld_reclen = lname->ln_namelen + sizeof(struct link_ea_entry);
-	if (ldata->ld_leh->leh_len + ldata->ld_reclen >
-	    ldata->ld_buf->lb_len) {
+	reclen = lname->ln_namelen + sizeof(struct link_ea_entry);
+	if (unlikely(leh->leh_len + reclen > MAX_LINKEA_SIZE)) {
+		/*
+		 * Use 32-bits to save the overflow time, although it will
+		 * shrink the ktime_get_real_seconds() returned 64-bits value
+		 * to 32-bits value, it is still quite large and can be used
+		 * for about 140 years. That is enough.
+		 */
+		leh->leh_overflow_time = ktime_get_real_seconds();
+		if (unlikely(leh->leh_overflow_time == 0))
+			leh->leh_overflow_time++;
+
+		CDEBUG(D_INODE, "No enough space to hold linkea entry '" DFID ": %.*s' at %u\n",
+		       PFID(pfid), lname->ln_namelen,
+		       lname->ln_name, leh->leh_overflow_time);
+		return 0;
+	}
+
+	if (leh->leh_len + reclen > ldata->ld_buf->lb_len) {
 		if (lu_buf_check_and_grow(ldata->ld_buf,
-					  ldata->ld_leh->leh_len +
-					  ldata->ld_reclen) < 0)
+					  leh->leh_len + reclen) < 0)
 			return -ENOMEM;
+
+		leh = ldata->ld_leh = ldata->ld_buf->lb_buf;
 	}
 
-	ldata->ld_leh = ldata->ld_buf->lb_buf;
-	ldata->ld_lee = ldata->ld_buf->lb_buf + ldata->ld_leh->leh_len;
+	ldata->ld_lee = ldata->ld_buf->lb_buf + leh->leh_len;
 	ldata->ld_reclen = linkea_entry_pack(ldata->ld_lee, lname, pfid);
-	ldata->ld_leh->leh_len += ldata->ld_reclen;
-	ldata->ld_leh->leh_reccount++;
+	leh->leh_len += ldata->ld_reclen;
+	leh->leh_reccount++;
 	CDEBUG(D_INODE, "New link_ea name '" DFID ":%.*s' is added\n",
 	       PFID(pfid), lname->ln_namelen, lname->ln_name);
 	return 0;
@@ -139,6 +178,7 @@ EXPORT_SYMBOL(linkea_add_buf);
 void linkea_del_buf(struct linkea_data *ldata, const struct lu_name *lname)
 {
 	LASSERT(ldata->ld_leh && ldata->ld_lee);
+	LASSERT(ldata->ld_leh->leh_reccount > 0);
 
 	ldata->ld_leh->leh_reccount--;
 	ldata->ld_leh->leh_len -= ldata->ld_reclen;
@@ -174,8 +214,9 @@ int linkea_links_find(struct linkea_data *ldata, const struct lu_name *lname,
 
 	LASSERT(ldata->ld_leh);
 
-	/* link #0 */
-	ldata->ld_lee = (struct link_ea_entry *)(ldata->ld_leh + 1);
+	/* link #0, if leh_reccount == 0 we skip the loop and return -ENOENT */
+	if (likely(ldata->ld_leh->leh_reccount > 0))
+		ldata->ld_lee = (struct link_ea_entry *)(ldata->ld_leh + 1);
 
 	for (count = 0; count < ldata->ld_leh->leh_reccount; count++) {
 		linkea_entry_unpack(ldata->ld_lee, &ldata->ld_reclen,
diff --git a/drivers/staging/lustre/lustre/obdclass/linux/linux-module.c b/drivers/staging/lustre/lustre/obdclass/linux/linux-module.c
index 9f5e8299d7e4..6df911112731 100644
--- a/drivers/staging/lustre/lustre/obdclass/linux/linux-module.c
+++ b/drivers/staging/lustre/lustre/obdclass/linux/linux-module.c
@@ -60,13 +60,91 @@
 #include <linux/seq_file.h>
 #include <linux/kobject.h>
 
-#include "../../../include/linux/libcfs/libcfs.h"
-#include "../../../include/linux/lnet/lnetctl.h"
-#include "../../include/obd_support.h"
-#include "../../include/obd_class.h"
-#include "../../include/lprocfs_status.h"
-#include "../../include/lustre/lustre_ioctl.h"
-#include "../../include/lustre_ver.h"
+#include <linux/libcfs/libcfs.h>
+#include <uapi/linux/lnet/lnetctl.h>
+#include <obd_support.h>
+#include <obd_class.h>
+#include <lprocfs_status.h>
+#include <uapi/linux/lustre/lustre_ioctl.h>
+#include <uapi/linux/lustre/lustre_ver.h>
+
+#define OBD_MAX_IOCTL_BUFFER	8192
+
+static int obd_ioctl_is_invalid(struct obd_ioctl_data *data)
+{
+	if (data->ioc_len > BIT(30)) {
+		CERROR("OBD ioctl: ioc_len larger than 1<<30\n");
+		return 1;
+	}
+
+	if (data->ioc_inllen1 > BIT(30)) {
+		CERROR("OBD ioctl: ioc_inllen1 larger than 1<<30\n");
+		return 1;
+	}
+
+	if (data->ioc_inllen2 > BIT(30)) {
+		CERROR("OBD ioctl: ioc_inllen2 larger than 1<<30\n");
+		return 1;
+	}
+
+	if (data->ioc_inllen3 > BIT(30)) {
+		CERROR("OBD ioctl: ioc_inllen3 larger than 1<<30\n");
+		return 1;
+	}
+
+	if (data->ioc_inllen4 > BIT(30)) {
+		CERROR("OBD ioctl: ioc_inllen4 larger than 1<<30\n");
+		return 1;
+	}
+
+	if (data->ioc_inlbuf1 && data->ioc_inllen1 == 0) {
+		CERROR("OBD ioctl: inlbuf1 pointer but 0 length\n");
+		return 1;
+	}
+
+	if (data->ioc_inlbuf2 && data->ioc_inllen2 == 0) {
+		CERROR("OBD ioctl: inlbuf2 pointer but 0 length\n");
+		return 1;
+	}
+
+	if (data->ioc_inlbuf3 && data->ioc_inllen3 == 0) {
+		CERROR("OBD ioctl: inlbuf3 pointer but 0 length\n");
+		return 1;
+	}
+
+	if (data->ioc_inlbuf4 && data->ioc_inllen4 == 0) {
+		CERROR("OBD ioctl: inlbuf4 pointer but 0 length\n");
+		return 1;
+	}
+
+	if (data->ioc_pbuf1 && data->ioc_plen1 == 0) {
+		CERROR("OBD ioctl: pbuf1 pointer but 0 length\n");
+		return 1;
+	}
+
+	if (data->ioc_pbuf2 && data->ioc_plen2 == 0) {
+		CERROR("OBD ioctl: pbuf2 pointer but 0 length\n");
+		return 1;
+	}
+
+	if (!data->ioc_pbuf1 && data->ioc_plen1 != 0) {
+		CERROR("OBD ioctl: plen1 set but NULL pointer\n");
+		return 1;
+	}
+
+	if (!data->ioc_pbuf2 && data->ioc_plen2 != 0) {
+		CERROR("OBD ioctl: plen2 set but NULL pointer\n");
+		return 1;
+	}
+
+	if (obd_ioctl_packlen(data) > data->ioc_len) {
+		CERROR("OBD ioctl: packlen exceeds ioc_len (%d > %d)\n",
+		obd_ioctl_packlen(data), data->ioc_len);
+		return 1;
+	}
+
+	return 0;
+}
 
 /* buffer MUST be at least the size of obd_ioctl_hdr */
 int obd_ioctl_getdata(char **buf, int *len, void __user *arg)
@@ -151,14 +229,6 @@ free_buf:
 }
 EXPORT_SYMBOL(obd_ioctl_getdata);
 
-int obd_ioctl_popdata(void __user *arg, void *data, int len)
-{
-	int err;
-
-	err = copy_to_user(arg, data, len) ? -EFAULT : 0;
-	return err;
-}
-
 /*  opening /dev/obd */
 static int obd_class_open(struct inode *inode, struct file *file)
 {
@@ -405,7 +475,7 @@ static const struct file_operations obd_device_list_fops = {
 struct kobject *lustre_kobj;
 EXPORT_SYMBOL_GPL(lustre_kobj);
 
-static struct attribute_group lustre_attr_group = {
+static const struct attribute_group lustre_attr_group = {
 	.attrs = lustre_attrs,
 };
 
diff --git a/drivers/staging/lustre/lustre/obdclass/linux/linux-sysctl.c b/drivers/staging/lustre/lustre/obdclass/linux/linux-sysctl.c
index e6c785afceba..e92cccceefa1 100644
--- a/drivers/staging/lustre/lustre/obdclass/linux/linux-sysctl.c
+++ b/drivers/staging/lustre/lustre/obdclass/linux/linux-sysctl.c
@@ -43,9 +43,9 @@
 
 #define DEBUG_SUBSYSTEM S_CLASS
 
-#include "../../include/obd_support.h"
-#include "../../include/lprocfs_status.h"
-#include "../../include/obd_class.h"
+#include <obd_support.h>
+#include <lprocfs_status.h>
+#include <obd_class.h>
 
 struct static_lustre_uintvalue_attr {
 	struct {
@@ -151,7 +151,7 @@ static struct attribute *lustre_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group lustre_attr_group = {
+static const struct attribute_group lustre_attr_group = {
 	.attrs = lustre_attrs,
 };
 
diff --git a/drivers/staging/lustre/lustre/obdclass/llog.c b/drivers/staging/lustre/lustre/obdclass/llog.c
index 736ea1067c93..98021a2d7238 100644
--- a/drivers/staging/lustre/lustre/obdclass/llog.c
+++ b/drivers/staging/lustre/lustre/obdclass/llog.c
@@ -43,9 +43,9 @@
 
 #define DEBUG_SUBSYSTEM S_LOG
 
-#include "../include/llog_swab.h"
-#include "../include/lustre_log.h"
-#include "../include/obd_class.h"
+#include <llog_swab.h>
+#include <lustre_log.h>
+#include <obd_class.h>
 #include "llog_internal.h"
 
 /*
diff --git a/drivers/staging/lustre/lustre/obdclass/llog_cat.c b/drivers/staging/lustre/lustre/obdclass/llog_cat.c
index 8f1533c127a8..8fa969101650 100644
--- a/drivers/staging/lustre/lustre/obdclass/llog_cat.c
+++ b/drivers/staging/lustre/lustre/obdclass/llog_cat.c
@@ -44,7 +44,7 @@
 
 #define DEBUG_SUBSYSTEM S_LOG
 
-#include "../include/obd_class.h"
+#include <obd_class.h>
 
 #include "llog_internal.h"
 
diff --git a/drivers/staging/lustre/lustre/obdclass/llog_internal.h b/drivers/staging/lustre/lustre/obdclass/llog_internal.h
index 21a93c73756a..8de90bc638b4 100644
--- a/drivers/staging/lustre/lustre/obdclass/llog_internal.h
+++ b/drivers/staging/lustre/lustre/obdclass/llog_internal.h
@@ -33,7 +33,7 @@
 #ifndef __LLOG_INTERNAL_H__
 #define __LLOG_INTERNAL_H__
 
-#include "../include/lustre_log.h"
+#include <lustre_log.h>
 
 struct llog_process_info {
 	struct llog_handle *lpi_loghandle;
diff --git a/drivers/staging/lustre/lustre/obdclass/llog_obd.c b/drivers/staging/lustre/lustre/obdclass/llog_obd.c
index 8574ad401f66..3c42de966077 100644
--- a/drivers/staging/lustre/lustre/obdclass/llog_obd.c
+++ b/drivers/staging/lustre/lustre/obdclass/llog_obd.c
@@ -32,8 +32,8 @@
 
 #define DEBUG_SUBSYSTEM S_LOG
 
-#include "../include/obd_class.h"
-#include "../include/lustre_log.h"
+#include <obd_class.h>
+#include <lustre_log.h>
 #include "llog_internal.h"
 
 /* helper functions for calling the llog obd methods */
diff --git a/drivers/staging/lustre/lustre/obdclass/llog_swab.c b/drivers/staging/lustre/lustre/obdclass/llog_swab.c
index 016046d26010..d2d3114ce008 100644
--- a/drivers/staging/lustre/lustre/obdclass/llog_swab.c
+++ b/drivers/staging/lustre/lustre/obdclass/llog_swab.c
@@ -38,8 +38,8 @@
 
 #define DEBUG_SUBSYSTEM S_LOG
 
-#include "../include/llog_swab.h"
-#include "../include/lustre_log.h"
+#include <llog_swab.h>
+#include <lustre_log.h>
 
 static void print_llogd_body(struct llogd_body *d)
 {
diff --git a/drivers/staging/lustre/lustre/obdclass/lprocfs_counters.c b/drivers/staging/lustre/lustre/obdclass/lprocfs_counters.c
index 13aca5b93c6a..e4829880dc10 100644
--- a/drivers/staging/lustre/lustre/obdclass/lprocfs_counters.c
+++ b/drivers/staging/lustre/lustre/obdclass/lprocfs_counters.c
@@ -38,8 +38,8 @@
  */
 
 #include <linux/module.h>
-#include "../include/lprocfs_status.h"
-#include "../include/obd_support.h"
+#include <lprocfs_status.h>
+#include <obd_support.h>
 
 void lprocfs_counter_add(struct lprocfs_stats *stats, int idx, long amount)
 {
diff --git a/drivers/staging/lustre/lustre/obdclass/lprocfs_status.c b/drivers/staging/lustre/lustre/obdclass/lprocfs_status.c
index bc19f19d38d9..e79485b4bf7f 100644
--- a/drivers/staging/lustre/lustre/obdclass/lprocfs_status.c
+++ b/drivers/staging/lustre/lustre/obdclass/lprocfs_status.c
@@ -36,9 +36,9 @@
 
 #define DEBUG_SUBSYSTEM S_CLASS
 
-#include "../include/obd_class.h"
-#include "../include/lprocfs_status.h"
-#include "../include/lustre/lustre_idl.h"
+#include <obd_class.h>
+#include <lprocfs_status.h>
+#include <uapi/linux/lustre/lustre_idl.h>
 #include <linux/seq_file.h>
 #include <linux/ctype.h>
 
@@ -1031,7 +1031,7 @@ static struct kobj_type obd_ktype = {
 };
 
 int lprocfs_obd_setup(struct obd_device *obd, struct lprocfs_vars *list,
-		      struct attribute_group *attrs)
+		      const struct attribute_group *attrs)
 {
 	int rc = 0;
 
diff --git a/drivers/staging/lustre/lustre/obdclass/lu_object.c b/drivers/staging/lustre/lustre/obdclass/lu_object.c
index bb9d514525ce..09c98184a291 100644
--- a/drivers/staging/lustre/lustre/obdclass/lu_object.c
+++ b/drivers/staging/lustre/lustre/obdclass/lu_object.c
@@ -40,19 +40,19 @@
 
 #define DEBUG_SUBSYSTEM S_CLASS
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
-# include <linux/module.h>
+#include <linux/module.h>
 
 /* hash_long() */
-#include "../../include/linux/libcfs/libcfs_hash.h"
-#include "../include/obd_class.h"
-#include "../include/obd_support.h"
-#include "../include/lustre_disk.h"
-#include "../include/lustre_fid.h"
-#include "../include/lu_object.h"
-#include "../include/cl_object.h"
-#include "../include/lu_ref.h"
+#include <linux/libcfs/libcfs_hash.h>
+#include <obd_class.h>
+#include <obd_support.h>
+#include <lustre_disk.h>
+#include <lustre_fid.h>
+#include <lu_object.h>
+#include <cl_object.h>
+#include <lu_ref.h>
 #include <linux/list.h>
 
 enum {
@@ -1409,9 +1409,9 @@ void lu_context_key_degister(struct lu_context_key *key)
 	 */
 	while (atomic_read(&key->lct_used) > 1) {
 		spin_unlock(&lu_keys_guard);
-		CDEBUG(D_INFO, "lu_context_key_degister: \"%s\" %p, %d\n",
-		       key->lct_owner ? key->lct_owner->name : "", key,
-		       atomic_read(&key->lct_used));
+		CDEBUG(D_INFO, "%s: \"%s\" %p, %d\n",
+		       __func__, key->lct_owner ? key->lct_owner->name : "",
+		       key, atomic_read(&key->lct_used));
 		schedule();
 		spin_lock(&lu_keys_guard);
 	}
@@ -1548,7 +1548,8 @@ void lu_context_key_quiesce(struct lu_context_key *key)
 		 */
 		while (atomic_read(&lu_key_initing_cnt) > 0) {
 			spin_unlock(&lu_keys_guard);
-			CDEBUG(D_INFO, "lu_context_key_quiesce: \"%s\" %p, %d (%d)\n",
+			CDEBUG(D_INFO, "%s: \"%s\" %p, %d (%d)\n",
+			       __func__,
 			       key->lct_owner ? key->lct_owner->name : "",
 			       key, atomic_read(&key->lct_used),
 			atomic_read(&lu_key_initing_cnt));
diff --git a/drivers/staging/lustre/lustre/obdclass/lu_ref.c b/drivers/staging/lustre/lustre/obdclass/lu_ref.c
index e9f6040d19eb..fa690b2bd643 100644
--- a/drivers/staging/lustre/lustre/obdclass/lu_ref.c
+++ b/drivers/staging/lustre/lustre/obdclass/lu_ref.c
@@ -38,9 +38,9 @@
 
 #define DEBUG_SUBSYSTEM S_CLASS
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
-#include "../include/obd.h"
-#include "../include/obd_class.h"
-#include "../include/obd_support.h"
-#include "../include/lu_ref.h"
+#include <obd.h>
+#include <obd_class.h>
+#include <obd_support.h>
+#include <lu_ref.h>
diff --git a/drivers/staging/lustre/lustre/obdclass/lustre_handles.c b/drivers/staging/lustre/lustre/obdclass/lustre_handles.c
index c9445e5ec271..e1273c997b5f 100644
--- a/drivers/staging/lustre/lustre/obdclass/lustre_handles.c
+++ b/drivers/staging/lustre/lustre/obdclass/lustre_handles.c
@@ -36,9 +36,9 @@
 
 #define DEBUG_SUBSYSTEM S_CLASS
 
-#include "../include/obd_support.h"
-#include "../include/lustre_handles.h"
-#include "../include/lustre_lib.h"
+#include <obd_support.h>
+#include <lustre_handles.h>
+#include <lustre_lib.h>
 
 static __u64 handle_base;
 #define HANDLE_INCR 7
diff --git a/drivers/staging/lustre/lustre/obdclass/lustre_peer.c b/drivers/staging/lustre/lustre/obdclass/lustre_peer.c
index ffa740aa861c..2798d35ad318 100644
--- a/drivers/staging/lustre/lustre/obdclass/lustre_peer.c
+++ b/drivers/staging/lustre/lustre/obdclass/lustre_peer.c
@@ -32,13 +32,13 @@
 
 #define DEBUG_SUBSYSTEM S_RPC
 
-#include "../include/obd.h"
-#include "../include/obd_support.h"
-#include "../include/obd_class.h"
-#include "../include/lustre_lib.h"
-#include "../include/lustre_ha.h"
-#include "../include/lustre_net.h"
-#include "../include/lprocfs_status.h"
+#include <obd.h>
+#include <obd_support.h>
+#include <obd_class.h>
+#include <lustre_lib.h>
+#include <lustre_ha.h>
+#include <lustre_net.h>
+#include <lprocfs_status.h>
 
 #define NIDS_MAX	32
 
diff --git a/drivers/staging/lustre/lustre/obdclass/obd_config.c b/drivers/staging/lustre/lustre/obdclass/obd_config.c
index 6a7e7a7d2af1..94a940faca5d 100644
--- a/drivers/staging/lustre/lustre/obdclass/obd_config.c
+++ b/drivers/staging/lustre/lustre/obdclass/obd_config.c
@@ -38,12 +38,12 @@
 
 #include <linux/string.h>
 
-#include "../include/lustre/lustre_ioctl.h"
-#include "../include/llog_swab.h"
-#include "../include/lprocfs_status.h"
-#include "../include/lustre_log.h"
-#include "../include/lustre_param.h"
-#include "../include/obd_class.h"
+#include <uapi/linux/lustre/lustre_ioctl.h>
+#include <llog_swab.h>
+#include <lprocfs_status.h>
+#include <lustre_log.h>
+#include <uapi/linux/lustre/lustre_param.h>
+#include <obd_class.h>
 
 #include "llog_internal.h"
 
@@ -170,6 +170,40 @@ int class_parse_nid_quiet(char *buf, lnet_nid_t *nid, char **endh)
 }
 EXPORT_SYMBOL(class_parse_nid_quiet);
 
+char *lustre_cfg_string(struct lustre_cfg *lcfg, u32 index)
+{
+	char *s;
+
+	if (!lcfg->lcfg_buflens[index])
+		return NULL;
+
+	s = lustre_cfg_buf(lcfg, index);
+	if (!s)
+		return NULL;
+
+	/*
+	 * make sure it's NULL terminated, even if this kills a char
+	 * of data.  Try to use the padding first though.
+	 */
+	if (s[lcfg->lcfg_buflens[index] - 1] != '\0') {
+		size_t last = ALIGN(lcfg->lcfg_buflens[index], 8) - 1;
+		char lost;
+
+		/* Use the smaller value */
+		if (last > lcfg->lcfg_buflens[index])
+			last = lcfg->lcfg_buflens[index];
+
+		lost = s[last];
+		s[last] = '\0';
+		if (lost != '\0') {
+			CWARN("Truncated buf %d to '%s' (lost '%c'...)\n",
+			      index, s, lost);
+		}
+	}
+	return s;
+}
+EXPORT_SYMBOL(lustre_cfg_string);
+
 /********************** class fns **********************/
 
 /**
@@ -1107,7 +1141,8 @@ int class_config_llog_handler(const struct lu_env *env,
 		struct lustre_cfg_bufs bufs;
 		char *inst_name = NULL;
 		int inst_len = 0;
-		int inst = 0, swab = 0;
+		size_t lcfg_len;
+		int swab = 0;
 
 		lcfg = (struct lustre_cfg *)cfg_buf;
 		if (lcfg->lcfg_version == __swab32(LUSTRE_CFG_VERSION)) {
@@ -1198,7 +1233,6 @@ int class_config_llog_handler(const struct lu_env *env,
 
 		if (clli && clli->cfg_instance &&
 		    LUSTRE_CFG_BUFLEN(lcfg, 0) > 0) {
-			inst = 1;
 			inst_len = LUSTRE_CFG_BUFLEN(lcfg, 0) +
 				   sizeof(clli->cfg_instance) * 2 + 4;
 			inst_name = kasprintf(GFP_NOFS, "%s-%p",
@@ -1238,8 +1272,14 @@ int class_config_llog_handler(const struct lu_env *env,
 						   clli->cfg_obdname);
 		}
 
-		lcfg_new = lustre_cfg_new(lcfg->lcfg_command, &bufs);
+		lcfg_len = lustre_cfg_len(bufs.lcfg_bufcount, bufs.lcfg_buflen);
+		lcfg_new = kzalloc(lcfg_len, GFP_NOFS);
+		if (!lcfg_new) {
+			rc = -ENOMEM;
+			goto out;
+		}
 
+		lustre_cfg_init(lcfg_new, lcfg->lcfg_command, &bufs);
 		lcfg_new->lcfg_num   = lcfg->lcfg_num;
 		lcfg_new->lcfg_flags = lcfg->lcfg_flags;
 
@@ -1262,10 +1302,8 @@ int class_config_llog_handler(const struct lu_env *env,
 		lcfg_new->lcfg_nal = 0; /* illegal value for obsolete field */
 
 		rc = class_process_config(lcfg_new);
-		lustre_cfg_free(lcfg_new);
-
-		if (inst)
-			kfree(inst_name);
+		kfree(lcfg_new);
+		kfree(inst_name);
 		break;
 	}
 	default:
@@ -1426,9 +1464,11 @@ int class_manual_cleanup(struct obd_device *obd)
 
 	lustre_cfg_bufs_reset(&bufs, obd->obd_name);
 	lustre_cfg_bufs_set_string(&bufs, 1, flags);
-	lcfg = lustre_cfg_new(LCFG_CLEANUP, &bufs);
-	if (IS_ERR(lcfg))
-		return PTR_ERR(lcfg);
+	lcfg = kzalloc(lustre_cfg_len(bufs.lcfg_bufcount, bufs.lcfg_buflen),
+			GFP_NOFS);
+	if (!lcfg)
+		return -ENOMEM;
+	lustre_cfg_init(lcfg, LCFG_CLEANUP, &bufs);
 
 	rc = class_process_config(lcfg);
 	if (rc) {
@@ -1442,7 +1482,7 @@ int class_manual_cleanup(struct obd_device *obd)
 	if (rc)
 		CERROR("detach failed %d: %s\n", rc, obd->obd_name);
 out:
-	lustre_cfg_free(lcfg);
+	kfree(lcfg);
 	return rc;
 }
 EXPORT_SYMBOL(class_manual_cleanup);
diff --git a/drivers/staging/lustre/lustre/obdclass/obd_mount.c b/drivers/staging/lustre/lustre/obdclass/obd_mount.c
index 8e0d4b1d86dc..1256034b60c1 100644
--- a/drivers/staging/lustre/lustre/obdclass/obd_mount.c
+++ b/drivers/staging/lustre/lustre/obdclass/obd_mount.c
@@ -40,13 +40,13 @@
 #define D_MOUNT (D_SUPER | D_CONFIG/*|D_WARNING */)
 #define PRINT_CMD CDEBUG
 
-#include "../include/obd.h"
-#include "../include/lustre_compat.h"
-#include "../include/obd_class.h"
-#include "../include/lustre/lustre_user.h"
-#include "../include/lustre_log.h"
-#include "../include/lustre_disk.h"
-#include "../include/lustre_param.h"
+#include <obd.h>
+#include <lustre_compat.h>
+#include <obd_class.h>
+#include <uapi/linux/lustre/lustre_idl.h>
+#include <lustre_log.h>
+#include <lustre_disk.h>
+#include <uapi/linux/lustre/lustre_param.h>
 
 static int (*client_fill_super)(struct super_block *sb,
 				struct vfsmount *mnt);
@@ -88,10 +88,17 @@ int lustre_process_log(struct super_block *sb, char *logname,
 	lustre_cfg_bufs_set_string(bufs, 1, logname);
 	lustre_cfg_bufs_set(bufs, 2, cfg, sizeof(*cfg));
 	lustre_cfg_bufs_set(bufs, 3, &sb, sizeof(sb));
-	lcfg = lustre_cfg_new(LCFG_LOG_START, bufs);
-	rc = obd_process_config(mgc, sizeof(*lcfg), lcfg);
-	lustre_cfg_free(lcfg);
+	lcfg = kzalloc(lustre_cfg_len(bufs->lcfg_bufcount, bufs->lcfg_buflen),
+		       GFP_NOFS);
+	if (!lcfg) {
+		rc = -ENOMEM;
+		goto out;
+	}
+	lustre_cfg_init(lcfg, LCFG_LOG_START, bufs);
 
+	rc = obd_process_config(mgc, sizeof(*lcfg), lcfg);
+	kfree(lcfg);
+out:
 	kfree(bufs);
 
 	if (rc == -EINVAL)
@@ -126,9 +133,14 @@ int lustre_end_log(struct super_block *sb, char *logname,
 	lustre_cfg_bufs_set_string(&bufs, 1, logname);
 	if (cfg)
 		lustre_cfg_bufs_set(&bufs, 2, cfg, sizeof(*cfg));
-	lcfg = lustre_cfg_new(LCFG_LOG_END, &bufs);
+	lcfg = kzalloc(lustre_cfg_len(bufs.lcfg_bufcount, bufs.lcfg_buflen),
+		       GFP_NOFS);
+	if (!lcfg)
+		return -ENOMEM;
+	lustre_cfg_init(lcfg, LCFG_LOG_END, &bufs);
+
 	rc = obd_process_config(mgc, sizeof(*lcfg), lcfg);
-	lustre_cfg_free(lcfg);
+	kfree(lcfg);
 	return rc;
 }
 EXPORT_SYMBOL(lustre_end_log);
@@ -158,10 +170,14 @@ static int do_lcfg(char *cfgname, lnet_nid_t nid, int cmd,
 	if (s4)
 		lustre_cfg_bufs_set_string(&bufs, 4, s4);
 
-	lcfg = lustre_cfg_new(cmd, &bufs);
+	lcfg = kzalloc(lustre_cfg_len(bufs.lcfg_bufcount, bufs.lcfg_buflen),
+		       GFP_NOFS);
+	if (!lcfg)
+		return -ENOMEM;
+	lustre_cfg_init(lcfg, cmd, &bufs);
 	lcfg->lcfg_nid = nid;
 	rc = class_process_config(lcfg);
-	lustre_cfg_free(lcfg);
+	kfree(lcfg);
 	return rc;
 }
 
diff --git a/drivers/staging/lustre/lustre/obdclass/obdo.c b/drivers/staging/lustre/lustre/obdclass/obdo.c
index b1dfa1622ae7..7083f8786e9a 100644
--- a/drivers/staging/lustre/lustre/obdclass/obdo.c
+++ b/drivers/staging/lustre/lustre/obdclass/obdo.c
@@ -38,9 +38,9 @@
 
 #define DEBUG_SUBSYSTEM S_CLASS
 
-#include "../include/obd_class.h"
-#include "../include/lustre/lustre_idl.h"
-#include "../include/lustre_obdo.h"
+#include <obd_class.h>
+#include <uapi/linux/lustre/lustre_idl.h>
+#include <lustre_obdo.h>
 
 void obdo_set_parent_fid(struct obdo *dst, const struct lu_fid *parent)
 {
diff --git a/drivers/staging/lustre/lustre/obdclass/statfs_pack.c b/drivers/staging/lustre/lustre/obdclass/statfs_pack.c
index 4bad1fa27d40..89abea26a1f8 100644
--- a/drivers/staging/lustre/lustre/obdclass/statfs_pack.c
+++ b/drivers/staging/lustre/lustre/obdclass/statfs_pack.c
@@ -37,10 +37,10 @@
 #define DEBUG_SUBSYSTEM S_CLASS
 
 #include <linux/statfs.h>
-#include "../include/lustre_export.h"
-#include "../include/lustre_net.h"
-#include "../include/obd_support.h"
-#include "../include/obd_class.h"
+#include <lustre_export.h>
+#include <lustre_net.h>
+#include <obd_support.h>
+#include <obd_class.h>
 
 void statfs_unpack(struct kstatfs *sfs, struct obd_statfs *osfs)
 {
diff --git a/drivers/staging/lustre/lustre/obdclass/uuid.c b/drivers/staging/lustre/lustre/obdclass/uuid.c
index abd9b1ae72cd..9b1872b99f2a 100644
--- a/drivers/staging/lustre/lustre/obdclass/uuid.c
+++ b/drivers/staging/lustre/lustre/obdclass/uuid.c
@@ -34,10 +34,10 @@
 
 #define DEBUG_SUBSYSTEM S_CLASS
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
-#include "../include/obd_support.h"
-#include "../include/obd_class.h"
+#include <obd_support.h>
+#include <obd_class.h>
 
 void class_uuid_unparse(class_uuid_t uu, struct obd_uuid *out)
 {
diff --git a/drivers/staging/lustre/lustre/obdecho/Makefile b/drivers/staging/lustre/lustre/obdecho/Makefile
index a659a37a7e93..6be66fbab872 100644
--- a/drivers/staging/lustre/lustre/obdecho/Makefile
+++ b/drivers/staging/lustre/lustre/obdecho/Makefile
@@ -1,2 +1,5 @@
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/include
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/lustre/include
+
 obj-$(CONFIG_LUSTRE_FS) += obdecho.o
 obdecho-y := echo_client.o
diff --git a/drivers/staging/lustre/lustre/obdecho/echo_client.c b/drivers/staging/lustre/lustre/obdecho/echo_client.c
index 1c4a8fe87dd8..f9808d1cc352 100644
--- a/drivers/staging/lustre/lustre/obdecho/echo_client.c
+++ b/drivers/staging/lustre/lustre/obdecho/echo_client.c
@@ -31,18 +31,18 @@
  */
 
 #define DEBUG_SUBSYSTEM S_ECHO
-#include "../../include/linux/libcfs/libcfs.h"
-
-#include "../include/obd.h"
-#include "../include/obd_support.h"
-#include "../include/obd_class.h"
-#include "../include/lustre_debug.h"
-#include "../include/lprocfs_status.h"
-#include "../include/cl_object.h"
-#include "../include/lustre_fid.h"
-#include "../include/lustre_acl.h"
-#include "../include/lustre/lustre_ioctl.h"
-#include "../include/lustre_net.h"
+#include <linux/libcfs/libcfs.h>
+
+#include <obd.h>
+#include <obd_support.h>
+#include <obd_class.h>
+#include <lustre_debug.h>
+#include <lprocfs_status.h>
+#include <cl_object.h>
+#include <lustre_fid.h>
+#include <lustre_acl.h>
+#include <uapi/linux/lustre/lustre_ioctl.h>
+#include <lustre_net.h>
 
 #include "echo_internal.h"
 
@@ -319,7 +319,7 @@ static void echo_lock_fini(const struct lu_env *env,
 	kmem_cache_free(echo_lock_kmem, ecl);
 }
 
-static struct cl_lock_operations echo_lock_ops = {
+static const struct cl_lock_operations echo_lock_ops = {
 	.clo_fini      = echo_lock_fini,
 };
 
@@ -1102,8 +1102,11 @@ static int echo_create_object(const struct lu_env *env, struct echo_device *ed,
 		return -EINVAL;
 	}
 
-	if (!ostid_id(&oa->o_oi))
-		ostid_set_id(&oa->o_oi, ++last_object_id);
+	if (!ostid_id(&oa->o_oi)) {
+		rc = ostid_set_id(&oa->o_oi, ++last_object_id);
+		if (rc)
+			goto failed;
+	}
 
 	rc = obd_create(env, ec->ec_exp, oa);
 	if (rc != 0) {
diff --git a/drivers/staging/lustre/lustre/osc/Makefile b/drivers/staging/lustre/lustre/osc/Makefile
index 37cdeea9ac49..30dec90e64e8 100644
--- a/drivers/staging/lustre/lustre/osc/Makefile
+++ b/drivers/staging/lustre/lustre/osc/Makefile
@@ -1,3 +1,6 @@
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/include
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/lustre/include
+
 obj-$(CONFIG_LUSTRE_FS) += osc.o
 osc-y := osc_request.o osc_dev.o osc_object.o \
 	 osc_page.o osc_lock.o osc_io.o osc_quota.o osc_cache.o lproc_osc.o
diff --git a/drivers/staging/lustre/lustre/osc/lproc_osc.c b/drivers/staging/lustre/lustre/osc/lproc_osc.c
index 86f252d6adbd..ae13eb055229 100644
--- a/drivers/staging/lustre/lustre/osc/lproc_osc.c
+++ b/drivers/staging/lustre/lustre/osc/lproc_osc.c
@@ -32,9 +32,9 @@
 #define DEBUG_SUBSYSTEM S_CLASS
 
 #include <linux/statfs.h>
-#include "../include/obd_cksum.h"
-#include "../include/obd_class.h"
-#include "../include/lprocfs_status.h"
+#include <obd_cksum.h>
+#include <obd_class.h>
+#include <lprocfs_status.h>
 #include <linux/seq_file.h>
 #include "osc_internal.h"
 
@@ -831,7 +831,7 @@ static struct attribute *osc_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group osc_attr_group = {
+static const struct attribute_group osc_attr_group = {
 	.attrs = osc_attrs,
 };
 
diff --git a/drivers/staging/lustre/lustre/osc/osc_cache.c b/drivers/staging/lustre/lustre/osc/osc_cache.c
index d8a95f8fe1ff..e1207c227b79 100644
--- a/drivers/staging/lustre/lustre/osc/osc_cache.c
+++ b/drivers/staging/lustre/lustre/osc/osc_cache.c
@@ -783,6 +783,7 @@ restart:
 			/* pull ext's start back to cover cur */
 			ext->oe_start = cur->oe_start;
 			ext->oe_grants += chunksize;
+			LASSERT(*grants >= chunksize);
 			*grants -= chunksize;
 
 			found = osc_extent_hold(ext);
@@ -790,6 +791,7 @@ restart:
 			/* rear merge */
 			ext->oe_end = cur->oe_end;
 			ext->oe_grants += chunksize;
+			LASSERT(*grants >= chunksize);
 			*grants -= chunksize;
 
 			/* try to merge with the next one because we just fill
@@ -819,8 +821,8 @@ restart:
 		/* create a new extent */
 		EASSERT(osc_extent_is_overlapped(obj, cur) == 0, cur);
 		cur->oe_grants = chunksize + cli->cl_extent_tax;
+		LASSERT(*grants >= cur->oe_grants);
 		*grants -= cur->oe_grants;
-		LASSERT(*grants >= 0);
 
 		cur->oe_state = OES_CACHE;
 		found = osc_extent_hold(cur);
@@ -849,7 +851,6 @@ restart:
 
 out:
 	osc_extent_put(env, cur);
-	LASSERT(*grants >= 0);
 	return found;
 }
 
@@ -1219,8 +1220,8 @@ static int osc_extent_expand(struct osc_extent *ext, pgoff_t index,
 
 	ext->oe_end = end_index;
 	ext->oe_grants += chunksize;
+	LASSERT(*grants >= chunksize);
 	*grants -= chunksize;
-	LASSERT(*grants >= 0);
 	EASSERTF(osc_extent_is_overlapped(obj, ext) == 0, ext,
 		 "overlapped after expanding for %lu.\n", index);
 
@@ -1887,6 +1888,7 @@ struct extent_rpc_data {
 	unsigned int		erd_page_count;
 	unsigned int		erd_max_pages;
 	unsigned int		erd_max_chunks;
+	unsigned int		erd_max_extents;
 };
 
 static inline unsigned int osc_extent_chunks(const struct osc_extent *ext)
@@ -1915,11 +1917,23 @@ static int try_to_add_extent_for_io(struct client_obd *cli,
 	EASSERT((ext->oe_state == OES_CACHE || ext->oe_state == OES_LOCK_DONE),
 		ext);
 
+	if (!data->erd_max_extents)
+		return 0;
+
 	chunk_count = osc_extent_chunks(ext);
+	EASSERTF(data->erd_page_count != 0 ||
+		 chunk_count <= data->erd_max_chunks, ext,
+		 "The first extent to be fit in a RPC contains %u chunks, which is over the limit %u.\n",
+		 chunk_count, data->erd_max_chunks);
+
 	if (chunk_count > data->erd_max_chunks)
 		return 0;
 
 	data->erd_max_pages = max(ext->oe_mppr, data->erd_max_pages);
+	EASSERTF(data->erd_page_count != 0 ||
+		 ext->oe_nr_pages <= data->erd_max_pages, ext,
+		 "The first extent to be fit in a RPC contains %u pages, which is over the limit %u.\n",
+		 ext->oe_nr_pages, data->erd_max_pages);
 	if (data->erd_page_count + ext->oe_nr_pages > data->erd_max_pages)
 		return 0;
 
@@ -1943,6 +1957,7 @@ static int try_to_add_extent_for_io(struct client_obd *cli,
 		break;
 	}
 
+	data->erd_max_extents--;
 	data->erd_max_chunks -= chunk_count;
 	data->erd_page_count += ext->oe_nr_pages;
 	list_move_tail(&ext->oe_link, data->erd_rpc_list);
@@ -1972,10 +1987,12 @@ static inline unsigned int osc_max_write_chunks(const struct client_obd *cli)
 	 *
 	 * This limitation doesn't apply to ldiskfs, which allows as many
 	 * chunks in one RPC as we want. However, it won't have any benefits
-	 * to have too many discontiguous pages in one RPC. Therefore, it
-	 * can only have 256 chunks at most in one RPC.
+	 * to have too many discontiguous pages in one RPC.
+	 *
+	 * An osc_extent won't cover over a RPC size, so the chunks in an
+	 * osc_extent won't bigger than PTLRPC_MAX_BRW_SIZE >> chunkbits.
 	 */
-	return min(PTLRPC_MAX_BRW_SIZE >> cli->cl_chunkbits, 256);
+	return PTLRPC_MAX_BRW_SIZE >> cli->cl_chunkbits;
 }
 
 /**
@@ -2002,6 +2019,7 @@ static unsigned int get_write_extents(struct osc_object *obj,
 		.erd_page_count = 0,
 		.erd_max_pages = cli->cl_max_pages_per_rpc,
 		.erd_max_chunks = osc_max_write_chunks(cli),
+		.erd_max_extents = 256,
 	};
 
 	LASSERT(osc_object_is_locked(obj));
@@ -2140,6 +2158,7 @@ osc_send_read_rpc(const struct lu_env *env, struct client_obd *cli,
 		.erd_page_count = 0,
 		.erd_max_pages = cli->cl_max_pages_per_rpc,
 		.erd_max_chunks = UINT_MAX,
+		.erd_max_extents = UINT_MAX,
 	};
 	int rc = 0;
 
diff --git a/drivers/staging/lustre/lustre/osc/osc_cl_internal.h b/drivers/staging/lustre/lustre/osc/osc_cl_internal.h
index 270212f4e5cf..35bdbfb8660d 100644
--- a/drivers/staging/lustre/lustre/osc/osc_cl_internal.h
+++ b/drivers/staging/lustre/lustre/osc/osc_cl_internal.h
@@ -42,11 +42,11 @@
 #ifndef OSC_CL_INTERNAL_H
 #define OSC_CL_INTERNAL_H
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
-#include "../include/obd.h"
+#include <obd.h>
 /* osc_build_res_name() */
-#include "../include/cl_object.h"
+#include <cl_object.h>
 #include "osc_internal.h"
 
 /** \defgroup osc osc
diff --git a/drivers/staging/lustre/lustre/osc/osc_dev.c b/drivers/staging/lustre/lustre/osc/osc_dev.c
index c5d62aeaeab5..cf7b8879d7f0 100644
--- a/drivers/staging/lustre/lustre/osc/osc_dev.c
+++ b/drivers/staging/lustre/lustre/osc/osc_dev.c
@@ -37,7 +37,7 @@
 #define DEBUG_SUBSYSTEM S_OSC
 
 /* class_name2obd() */
-#include "../include/obd_class.h"
+#include <obd_class.h>
 
 #include "osc_cl_internal.h"
 
diff --git a/drivers/staging/lustre/lustre/osc/osc_internal.h b/drivers/staging/lustre/lustre/osc/osc_internal.h
index 13a40f6423ff..a536908fb26a 100644
--- a/drivers/staging/lustre/lustre/osc/osc_internal.h
+++ b/drivers/staging/lustre/lustre/osc/osc_internal.h
@@ -99,7 +99,7 @@ void osc_update_next_shrink(struct client_obd *cli);
 /*
  * cl integration.
  */
-#include "../include/cl_object.h"
+#include <cl_object.h>
 
 extern struct ptlrpc_request_set *PTLRPCD_SET;
 
diff --git a/drivers/staging/lustre/lustre/osc/osc_io.c b/drivers/staging/lustre/lustre/osc/osc_io.c
index cbab80092442..f7969e33f28a 100644
--- a/drivers/staging/lustre/lustre/osc/osc_io.c
+++ b/drivers/staging/lustre/lustre/osc/osc_io.c
@@ -37,7 +37,7 @@
 
 #define DEBUG_SUBSYSTEM S_OSC
 
-#include "../include/lustre_obdo.h"
+#include <lustre_obdo.h>
 
 #include "osc_cl_internal.h"
 
diff --git a/drivers/staging/lustre/lustre/osc/osc_lock.c b/drivers/staging/lustre/lustre/osc/osc_lock.c
index 940c10c1d7a1..b4f1f74dead8 100644
--- a/drivers/staging/lustre/lustre/osc/osc_lock.c
+++ b/drivers/staging/lustre/lustre/osc/osc_lock.c
@@ -37,9 +37,9 @@
 
 #define DEBUG_SUBSYSTEM S_OSC
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 /* fid_build_reg_res_name() */
-#include "../include/lustre_fid.h"
+#include <lustre_fid.h>
 
 #include "osc_cl_internal.h"
 
diff --git a/drivers/staging/lustre/lustre/osc/osc_object.c b/drivers/staging/lustre/lustre/osc/osc_object.c
index fa621bda1ffe..945ae6e5a8b1 100644
--- a/drivers/staging/lustre/lustre/osc/osc_object.c
+++ b/drivers/staging/lustre/lustre/osc/osc_object.c
@@ -369,7 +369,14 @@ static void osc_req_attr_set(const struct lu_env *env, struct cl_object *obj,
 		oa->o_valid |= OBD_MD_FLGROUP;
 	}
 	if (flags & OBD_MD_FLID) {
-		ostid_set_id(&oa->o_oi, ostid_id(&oinfo->loi_oi));
+		int rc;
+
+		rc = ostid_set_id(&oa->o_oi, ostid_id(&oinfo->loi_oi));
+		if (rc) {
+			CERROR("Bad %llu to set " DOSTID " : rc %d\n",
+			       (unsigned long long)ostid_id(&oinfo->loi_oi),
+			       POSTID(&oa->o_oi), rc);
+		}
 		oa->o_valid |= OBD_MD_FLID;
 	}
 	if (flags & OBD_MD_FLHANDLE) {
diff --git a/drivers/staging/lustre/lustre/osc/osc_quota.c b/drivers/staging/lustre/lustre/osc/osc_quota.c
index fed4da63ee45..a6118f8ba446 100644
--- a/drivers/staging/lustre/lustre/osc/osc_quota.c
+++ b/drivers/staging/lustre/lustre/osc/osc_quota.c
@@ -23,7 +23,7 @@
  * Code originally extracted from quota directory
  */
 
-#include "../include/obd_class.h"
+#include <obd_class.h>
 #include "osc_internal.h"
 
 static inline struct osc_quota_info *osc_oqi_alloc(u32 id)
diff --git a/drivers/staging/lustre/lustre/osc/osc_request.c b/drivers/staging/lustre/lustre/osc/osc_request.c
index 922d0cbe83dc..4c68c42b2281 100644
--- a/drivers/staging/lustre/lustre/osc/osc_request.c
+++ b/drivers/staging/lustre/lustre/osc/osc_request.c
@@ -32,22 +32,22 @@
 
 #define DEBUG_SUBSYSTEM S_OSC
 
-#include "../../include/linux/libcfs/libcfs.h"
-
-#include "../include/lustre_dlm.h"
-#include "../include/lustre_net.h"
-#include "../include/lustre/lustre_user.h"
-#include "../include/obd_cksum.h"
-
-#include "../include/lustre_ha.h"
-#include "../include/lprocfs_status.h"
-#include "../include/lustre/lustre_ioctl.h"
-#include "../include/lustre_debug.h"
-#include "../include/lustre_obdo.h"
-#include "../include/lustre_param.h"
-#include "../include/lustre_fid.h"
-#include "../include/obd_class.h"
-#include "../include/obd.h"
+#include <linux/libcfs/libcfs.h>
+
+#include <lustre_dlm.h>
+#include <lustre_net.h>
+#include <uapi/linux/lustre/lustre_idl.h>
+#include <obd_cksum.h>
+
+#include <lustre_ha.h>
+#include <lprocfs_status.h>
+#include <uapi/linux/lustre/lustre_ioctl.h>
+#include <lustre_debug.h>
+#include <lustre_obdo.h>
+#include <uapi/linux/lustre/lustre_param.h>
+#include <lustre_fid.h>
+#include <obd_class.h>
+#include <obd.h>
 #include "osc_internal.h"
 #include "osc_cl_internal.h"
 
diff --git a/drivers/staging/lustre/lustre/ptlrpc/Makefile b/drivers/staging/lustre/lustre/ptlrpc/Makefile
index 24bbac19ddd1..a518001cdfe8 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/Makefile
+++ b/drivers/staging/lustre/lustre/ptlrpc/Makefile
@@ -1,3 +1,6 @@
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/include
+subdir-ccflags-y += -I$(srctree)/drivers/staging/lustre/lustre/include
+
 obj-$(CONFIG_LUSTRE_FS) += ptlrpc.o
 LDLM := ../../lustre/ldlm/
 
diff --git a/drivers/staging/lustre/lustre/ptlrpc/client.c b/drivers/staging/lustre/lustre/ptlrpc/client.c
index 1c7779215eed..b1d379a6a70f 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/client.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/client.c
@@ -34,12 +34,12 @@
 
 #define DEBUG_SUBSYSTEM S_RPC
 
-#include "../include/obd_support.h"
-#include "../include/obd_class.h"
-#include "../include/lustre_lib.h"
-#include "../include/lustre_ha.h"
-#include "../include/lustre_import.h"
-#include "../include/lustre_req_layout.h"
+#include <obd_support.h>
+#include <obd_class.h>
+#include <lustre_lib.h>
+#include <lustre_ha.h>
+#include <lustre_import.h>
+#include <lustre_req_layout.h>
 
 #include "ptlrpc_internal.h"
 
@@ -367,9 +367,8 @@ void ptlrpc_at_adj_net_latency(struct ptlrpc_request *req,
 		 */
 		CDEBUG((lustre_msg_get_flags(req->rq_reqmsg) & MSG_RESENT) ?
 		       D_ADAPTTO : D_WARNING,
-		       "Reported service time %u > total measured time " CFS_DURATION_T "\n",
-		       service_time,
-		       (long)(now - req->rq_sent));
+		       "Reported service time %u > total measured time %lld\n",
+		       service_time, now - req->rq_sent);
 		return;
 	}
 
@@ -742,7 +741,7 @@ int ptlrpc_request_bufs_pack(struct ptlrpc_request *request,
 
 	/* Let's setup deadline for req/reply/bulk unlink for opcode. */
 	if (cfs_fail_val == opcode) {
-		time_t *fail_t = NULL, *fail2_t = NULL;
+		time64_t *fail_t = NULL, *fail2_t = NULL;
 
 		if (CFS_FAIL_CHECK(OBD_FAIL_PTLRPC_LONG_BULK_UNLINK)) {
 			fail_t = &request->rq_bulk_deadline;
@@ -3116,13 +3115,20 @@ void ptlrpc_set_bulk_mbits(struct ptlrpc_request *req)
 
 	LASSERT(bd);
 
-	if (!req->rq_resend) {
-		/* this request has a new xid, just use it as bulk matchbits */
-		req->rq_mbits = req->rq_xid;
-
-	} else { /* needs to generate a new matchbits for resend */
+	/*
+	 * Generate new matchbits for all resend requests, including
+	 * resend replay.
+	 */
+	if (req->rq_resend) {
 		u64 old_mbits = req->rq_mbits;
 
+		/*
+		 * First time resend on -EINPROGRESS will generate new xid,
+		 * so we can actually use the rq_xid as rq_mbits in such case,
+		 * however, it's bit hard to distinguish such resend with a
+		 * 'resend for the -EINPROGRESS resend'. To make it simple,
+		 * we opt to generate mbits for all resend cases.
+		 */
 		if ((bd->bd_import->imp_connect_data.ocd_connect_flags &
 		     OBD_CONNECT_BULK_MBITS)) {
 			req->rq_mbits = ptlrpc_next_xid();
@@ -3131,12 +3137,21 @@ void ptlrpc_set_bulk_mbits(struct ptlrpc_request *req)
 			spin_lock(&req->rq_import->imp_lock);
 			list_del_init(&req->rq_unreplied_list);
 			ptlrpc_assign_next_xid_nolock(req);
-			req->rq_mbits = req->rq_xid;
 			spin_unlock(&req->rq_import->imp_lock);
+			req->rq_mbits = req->rq_xid;
 		}
 
 		CDEBUG(D_HA, "resend bulk old x%llu new x%llu\n",
 		       old_mbits, req->rq_mbits);
+	} else if (!(lustre_msg_get_flags(req->rq_reqmsg) & MSG_REPLAY)) {
+		/* Request being sent first time, use xid as matchbits. */
+		req->rq_mbits = req->rq_xid;
+	} else {
+		/*
+		 * Replay request, xid and matchbits have already been
+		 * correctly assigned.
+		 */
+		return;
 	}
 
 	/*
diff --git a/drivers/staging/lustre/lustre/ptlrpc/connection.c b/drivers/staging/lustre/lustre/ptlrpc/connection.c
index 73a2dbbeb7e6..cfdcbcec2779 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/connection.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/connection.c
@@ -31,9 +31,9 @@
  */
 
 #define DEBUG_SUBSYSTEM S_RPC
-#include "../include/obd_support.h"
-#include "../include/obd_class.h"
-#include "../include/lustre_net.h"
+#include <obd_support.h>
+#include <obd_class.h>
+#include <lustre_net.h>
 
 #include "ptlrpc_internal.h"
 
diff --git a/drivers/staging/lustre/lustre/ptlrpc/errno.c b/drivers/staging/lustre/lustre/ptlrpc/errno.c
index 73f8374f190e..cb788364a553 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/errno.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/errno.c
@@ -25,8 +25,8 @@
  * Copyright (c) 2013, Intel Corporation.
  */
 
-#include "../../include/linux/libcfs/libcfs.h"
-#include "../include/lustre/lustre_errno.h"
+#include <linux/libcfs/libcfs.h>
+#include <lustre_errno.h>
 
 /*
  * The two translation tables below must define a one-to-one mapping between
diff --git a/drivers/staging/lustre/lustre/ptlrpc/events.c b/drivers/staging/lustre/lustre/ptlrpc/events.c
index 978bdaca3cdd..62951f19b2ce 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/events.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/events.c
@@ -32,14 +32,14 @@
 
 #define DEBUG_SUBSYSTEM S_RPC
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 # ifdef __mips64__
 #  include <linux/kernel.h>
 # endif
 
-#include "../include/obd_class.h"
-#include "../include/lustre_net.h"
-#include "../include/lustre_sec.h"
+#include <obd_class.h>
+#include <lustre_net.h>
+#include <lustre_sec.h>
 #include "ptlrpc_internal.h"
 
 struct lnet_handle_eq ptlrpc_eq_h;
diff --git a/drivers/staging/lustre/lustre/ptlrpc/import.c b/drivers/staging/lustre/lustre/ptlrpc/import.c
index 52cb1f0c9c94..21f528957b73 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/import.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/import.c
@@ -36,14 +36,14 @@
 
 #define DEBUG_SUBSYSTEM S_RPC
 
-#include "../include/obd_support.h"
-#include "../include/lustre_ha.h"
-#include "../include/lustre_net.h"
-#include "../include/lustre_import.h"
-#include "../include/lustre_export.h"
-#include "../include/obd.h"
-#include "../include/obd_cksum.h"
-#include "../include/obd_class.h"
+#include <obd_support.h>
+#include <lustre_ha.h>
+#include <lustre_net.h>
+#include <lustre_import.h>
+#include <lustre_export.h>
+#include <obd.h>
+#include <obd_cksum.h>
+#include <obd_class.h>
 
 #include "ptlrpc_internal.h"
 
@@ -1026,7 +1026,7 @@ static int ptlrpc_connect_interpret(const struct lu_env *env,
 	/* check that server granted subset of flags we asked for. */
 	if ((ocd->ocd_connect_flags & imp->imp_connect_flags_orig) !=
 	    ocd->ocd_connect_flags) {
-		CERROR("%s: Server didn't granted asked subset of flags: asked=%#llx grranted=%#llx\n",
+		CERROR("%s: Server didn't grant the asked for subset of flags: asked=%#llx granted=%#llx\n",
 		       imp->imp_obd->obd_name, imp->imp_connect_flags_orig,
 		       ocd->ocd_connect_flags);
 		rc = -EPROTO;
diff --git a/drivers/staging/lustre/lustre/ptlrpc/layout.c b/drivers/staging/lustre/lustre/ptlrpc/layout.c
index 5810bbab6585..85854d9a376d 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/layout.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/layout.c
@@ -46,18 +46,18 @@
 
 #include <linux/module.h>
 
-#include "../include/lustre/lustre_idl.h"
+#include <uapi/linux/lustre/lustre_idl.h>
 
-#include "../include/llog_swab.h"
-#include "../include/lustre_debug.h"
-#include "../include/lustre_swab.h"
-#include "../include/lustre_ver.h"
-#include "../include/obd.h"
-#include "../include/obd_support.h"
+#include <llog_swab.h>
+#include <lustre_debug.h>
+#include <lustre_swab.h>
+#include <uapi/linux/lustre/lustre_ver.h>
+#include <obd.h>
+#include <obd_support.h>
 
 /* struct ptlrpc_request, lustre_msg* */
-#include "../include/lustre_req_layout.h"
-#include "../include/lustre_acl.h"
+#include <lustre_req_layout.h>
+#include <lustre_acl.h>
 
 /*
  * RQFs (see below) refer to two struct req_msg_field arrays describing the
diff --git a/drivers/staging/lustre/lustre/ptlrpc/llog_client.c b/drivers/staging/lustre/lustre/ptlrpc/llog_client.c
index 110d9f505787..480c20a6a792 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/llog_client.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/llog_client.c
@@ -38,11 +38,11 @@
 
 #define DEBUG_SUBSYSTEM S_LOG
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
-#include "../include/obd_class.h"
-#include "../include/lustre_log.h"
-#include "../include/lustre_net.h"
+#include <obd_class.h>
+#include <lustre_log.h>
+#include <lustre_net.h>
 #include <linux/list.h>
 
 #define LLOG_CLIENT_ENTRY(ctxt, imp) do {				\
diff --git a/drivers/staging/lustre/lustre/ptlrpc/llog_net.c b/drivers/staging/lustre/lustre/ptlrpc/llog_net.c
index bccdace7e51f..bc5aa7bcdba8 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/llog_net.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/llog_net.c
@@ -42,10 +42,10 @@
 
 #define DEBUG_SUBSYSTEM S_LOG
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
-#include "../include/obd_class.h"
-#include "../include/lustre_log.h"
+#include <obd_class.h>
+#include <lustre_log.h>
 #include <linux/list.h>
 
 int llog_initiator_connect(struct llog_ctxt *ctxt)
diff --git a/drivers/staging/lustre/lustre/ptlrpc/lproc_ptlrpc.c b/drivers/staging/lustre/lustre/ptlrpc/lproc_ptlrpc.c
index f87478180013..1392ae9747bd 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/lproc_ptlrpc.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/lproc_ptlrpc.c
@@ -31,12 +31,12 @@
  */
 #define DEBUG_SUBSYSTEM S_CLASS
 
-#include "../include/obd_support.h"
-#include "../include/obd.h"
-#include "../include/lprocfs_status.h"
-#include "../include/lustre/lustre_idl.h"
-#include "../include/lustre_net.h"
-#include "../include/obd_class.h"
+#include <obd_support.h>
+#include <obd.h>
+#include <lprocfs_status.h>
+#include <uapi/linux/lustre/lustre_idl.h>
+#include <lustre_net.h>
+#include <obd_class.h>
 #include "ptlrpc_internal.h"
 
 static struct ll_rpc_opcode {
@@ -905,11 +905,18 @@ static int ptlrpc_lprocfs_svc_req_history_show(struct seq_file *s, void *iter)
 	rc = ptlrpc_lprocfs_svc_req_history_seek(svcpt, srhi, srhi->srhi_seq);
 
 	if (rc == 0) {
+		struct timespec64 arrival, sent, arrivaldiff;
 		char nidstr[LNET_NIDSTR_SIZE];
 
 		req = srhi->srhi_req;
 
 		libcfs_nid2str_r(req->rq_self, nidstr, sizeof(nidstr));
+		arrival.tv_sec = req->rq_arrival_time.tv_sec;
+		arrival.tv_nsec = req->rq_arrival_time.tv_nsec;
+		sent.tv_sec = req->rq_sent;
+		sent.tv_nsec = 0;
+		arrivaldiff = timespec64_sub(sent, arrival);
+
 		/* Print common req fields.
 		 * CAVEAT EMPTOR: we're racing with the service handler
 		 * here.  The request could contain any old crap, so you
@@ -917,13 +924,15 @@ static int ptlrpc_lprocfs_svc_req_history_show(struct seq_file *s, void *iter)
 		 * parser. Currently I only print stuff here I know is OK
 		 * to look at coz it was set up in request_in_callback()!!!
 		 */
-		seq_printf(s, "%lld:%s:%s:x%llu:%d:%s:%lld:%lds(%+lds) ",
+		seq_printf(s, "%lld:%s:%s:x%llu:%d:%s:%lld.%06lld:%lld.%06llds(%+lld.0s) ",
 			   req->rq_history_seq, nidstr,
 			   libcfs_id2str(req->rq_peer), req->rq_xid,
 			   req->rq_reqlen, ptlrpc_rqphase2str(req),
 			   (s64)req->rq_arrival_time.tv_sec,
-			   (long)(req->rq_sent - req->rq_arrival_time.tv_sec),
-			   (long)(req->rq_sent - req->rq_deadline));
+			   (s64)req->rq_arrival_time.tv_nsec / NSEC_PER_USEC,
+			   (s64)arrivaldiff.tv_sec,
+			   (s64)(arrivaldiff.tv_nsec / NSEC_PER_USEC),
+			   (s64)(req->rq_sent - req->rq_deadline));
 		if (!svc->srv_ops.so_req_printer)
 			seq_putc(s, '\n');
 		else
diff --git a/drivers/staging/lustre/lustre/ptlrpc/niobuf.c b/drivers/staging/lustre/lustre/ptlrpc/niobuf.c
index eddc1927a8d2..12149fb64719 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/niobuf.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/niobuf.c
@@ -31,11 +31,11 @@
  */
 
 #define DEBUG_SUBSYSTEM S_RPC
-#include "../include/obd_support.h"
-#include "../include/lustre_net.h"
-#include "../include/lustre_lib.h"
-#include "../include/obd.h"
-#include "../include/obd_class.h"
+#include <obd_support.h>
+#include <lustre_net.h>
+#include <lustre_lib.h>
+#include <obd.h>
+#include <obd_class.h>
 #include "ptlrpc_internal.h"
 
 /**
diff --git a/drivers/staging/lustre/lustre/ptlrpc/nrs.c b/drivers/staging/lustre/lustre/ptlrpc/nrs.c
index ef19dbe2ea5c..2969d8da270e 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/nrs.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/nrs.c
@@ -36,11 +36,11 @@
  */
 
 #define DEBUG_SUBSYSTEM S_RPC
-#include "../include/obd_support.h"
-#include "../include/obd_class.h"
-#include "../include/lustre_net.h"
-#include "../include/lprocfs_status.h"
-#include "../../include/linux/libcfs/libcfs.h"
+#include <obd_support.h>
+#include <obd_class.h>
+#include <lustre_net.h>
+#include <lprocfs_status.h>
+#include <linux/libcfs/libcfs.h>
 #include "ptlrpc_internal.h"
 
 /**
diff --git a/drivers/staging/lustre/lustre/ptlrpc/nrs_fifo.c b/drivers/staging/lustre/lustre/ptlrpc/nrs_fifo.c
index b123a93242ba..df330e43bfe5 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/nrs_fifo.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/nrs_fifo.c
@@ -43,9 +43,9 @@
  */
 
 #define DEBUG_SUBSYSTEM S_RPC
-#include "../include/obd_support.h"
-#include "../include/obd_class.h"
-#include "../../include/linux/libcfs/libcfs.h"
+#include <obd_support.h>
+#include <obd_class.h>
+#include <linux/libcfs/libcfs.h>
 #include "ptlrpc_internal.h"
 
 /**
diff --git a/drivers/staging/lustre/lustre/ptlrpc/pack_generic.c b/drivers/staging/lustre/lustre/ptlrpc/pack_generic.c
index 55e8696e7d86..aad4ff191d95 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/pack_generic.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/pack_generic.c
@@ -40,16 +40,16 @@
 
 #define DEBUG_SUBSYSTEM S_RPC
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
-#include "../include/lustre/ll_fiemap.h"
+#include <uapi/linux/lustre/lustre_fiemap.h>
 
-#include "../include/llog_swab.h"
-#include "../include/lustre_net.h"
-#include "../include/lustre_swab.h"
-#include "../include/obd_cksum.h"
-#include "../include/obd_support.h"
-#include "../include/obd_class.h"
+#include <llog_swab.h>
+#include <lustre_net.h>
+#include <lustre_swab.h>
+#include <obd_cksum.h>
+#include <obd_support.h>
+#include <obd_class.h>
 
 #include "ptlrpc_internal.h"
 
@@ -186,7 +186,9 @@ void lustre_init_msg_v2(struct lustre_msg_v2 *msg, int count, __u32 *lens,
 	for (i = 0; i < count; i++) {
 		char *tmp = bufs[i];
 
-		LOGL(tmp, lens[i], ptr);
+		if (tmp)
+			memcpy(ptr, tmp, lens[i]);
+		ptr += cfs_size_round(lens[i]);
 	}
 }
 EXPORT_SYMBOL(lustre_init_msg_v2);
diff --git a/drivers/staging/lustre/lustre/ptlrpc/pers.c b/drivers/staging/lustre/lustre/ptlrpc/pers.c
index df4994f406e9..643388b03af7 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/pers.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/pers.c
@@ -32,11 +32,11 @@
 
 #define DEBUG_SUBSYSTEM S_RPC
 
-#include "../include/obd_support.h"
-#include "../include/obd_class.h"
-#include "../include/lustre_lib.h"
-#include "../include/lustre_ha.h"
-#include "../include/lustre_import.h"
+#include <obd_support.h>
+#include <obd_class.h>
+#include <lustre_lib.h>
+#include <lustre_ha.h>
+#include <lustre_import.h>
 
 #include "ptlrpc_internal.h"
 
diff --git a/drivers/staging/lustre/lustre/ptlrpc/pinger.c b/drivers/staging/lustre/lustre/ptlrpc/pinger.c
index 5504fc2363ac..e4de50e18d08 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/pinger.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/pinger.c
@@ -36,8 +36,8 @@
 
 #define DEBUG_SUBSYSTEM S_RPC
 
-#include "../include/obd_support.h"
-#include "../include/obd_class.h"
+#include <obd_support.h>
+#include <obd_class.h>
 #include "ptlrpc_internal.h"
 
 struct mutex pinger_mutex;
diff --git a/drivers/staging/lustre/lustre/ptlrpc/ptlrpc_module.c b/drivers/staging/lustre/lustre/ptlrpc/ptlrpc_module.c
index a70d5843f30e..38e488dd5409 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/ptlrpc_module.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/ptlrpc_module.c
@@ -32,10 +32,10 @@
 
 #define DEBUG_SUBSYSTEM S_RPC
 
-#include "../include/obd_support.h"
-#include "../include/obd_class.h"
-#include "../include/lustre_net.h"
-#include "../include/lustre_req_layout.h"
+#include <obd_support.h>
+#include <obd_class.h>
+#include <lustre_net.h>
+#include <lustre_req_layout.h>
 
 #include "ptlrpc_internal.h"
 
diff --git a/drivers/staging/lustre/lustre/ptlrpc/ptlrpcd.c b/drivers/staging/lustre/lustre/ptlrpc/ptlrpcd.c
index 59b5813bd559..0e476828cf75 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/ptlrpcd.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/ptlrpcd.c
@@ -51,15 +51,15 @@
 
 #define DEBUG_SUBSYSTEM S_RPC
 
-#include "../../include/linux/libcfs/libcfs.h"
-
-#include "../include/lustre_net.h"
-#include "../include/lustre_lib.h"
-#include "../include/lustre_ha.h"
-#include "../include/obd_class.h"	/* for obd_zombie */
-#include "../include/obd_support.h"	/* for OBD_FAIL_CHECK */
-#include "../include/cl_object.h"	/* cl_env_{get,put}() */
-#include "../include/lprocfs_status.h"
+#include <linux/libcfs/libcfs.h>
+
+#include <lustre_net.h>
+#include <lustre_lib.h>
+#include <lustre_ha.h>
+#include <obd_class.h>		/* for obd_zombie */
+#include <obd_support.h>	/* for OBD_FAIL_CHECK */
+#include <cl_object.h>		/* cl_env_{get,put}() */
+#include <lprocfs_status.h>
 
 #include "ptlrpc_internal.h"
 
diff --git a/drivers/staging/lustre/lustre/ptlrpc/recover.c b/drivers/staging/lustre/lustre/ptlrpc/recover.c
index 7b58545c2de4..72a19a379e2f 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/recover.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/recover.c
@@ -35,15 +35,15 @@
  */
 
 #define DEBUG_SUBSYSTEM S_RPC
-#include "../../include/linux/libcfs/libcfs.h"
-
-#include "../include/obd_support.h"
-#include "../include/lustre_ha.h"
-#include "../include/lustre_net.h"
-#include "../include/lustre_import.h"
-#include "../include/lustre_export.h"
-#include "../include/obd.h"
-#include "../include/obd_class.h"
+#include <linux/libcfs/libcfs.h>
+
+#include <obd_support.h>
+#include <lustre_ha.h>
+#include <lustre_net.h>
+#include <lustre_import.h>
+#include <lustre_export.h>
+#include <obd.h>
+#include <obd_class.h>
 #include <linux/list.h>
 
 #include "ptlrpc_internal.h"
diff --git a/drivers/staging/lustre/lustre/ptlrpc/sec.c b/drivers/staging/lustre/lustre/ptlrpc/sec.c
index 366f2ce20f5e..cd7a5391a574 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/sec.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/sec.c
@@ -36,19 +36,19 @@
 
 #define DEBUG_SUBSYSTEM S_SEC
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 #include <linux/crypto.h>
 #include <linux/cred.h>
 #include <linux/key.h>
 #include <linux/sched/task.h>
 
-#include "../include/obd.h"
-#include "../include/obd_class.h"
-#include "../include/obd_support.h"
-#include "../include/lustre_net.h"
-#include "../include/lustre_import.h"
-#include "../include/lustre_dlm.h"
-#include "../include/lustre_sec.h"
+#include <obd.h>
+#include <obd_class.h>
+#include <obd_support.h>
+#include <lustre_net.h>
+#include <lustre_import.h>
+#include <lustre_dlm.h>
+#include <lustre_sec.h>
 
 #include "ptlrpc_internal.h"
 
diff --git a/drivers/staging/lustre/lustre/ptlrpc/sec_bulk.c b/drivers/staging/lustre/lustre/ptlrpc/sec_bulk.c
index 128838a695e0..059294aad172 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/sec_bulk.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/sec_bulk.c
@@ -36,16 +36,16 @@
 
 #define DEBUG_SUBSYSTEM S_SEC
 
-#include "../../include/linux/libcfs/libcfs.h"
-
-#include "../include/obd.h"
-#include "../include/obd_cksum.h"
-#include "../include/obd_class.h"
-#include "../include/obd_support.h"
-#include "../include/lustre_net.h"
-#include "../include/lustre_import.h"
-#include "../include/lustre_dlm.h"
-#include "../include/lustre_sec.h"
+#include <linux/libcfs/libcfs.h>
+
+#include <obd.h>
+#include <obd_cksum.h>
+#include <obd_class.h>
+#include <obd_support.h>
+#include <lustre_net.h>
+#include <lustre_import.h>
+#include <lustre_dlm.h>
+#include <lustre_sec.h>
 
 #include "ptlrpc_internal.h"
 
diff --git a/drivers/staging/lustre/lustre/ptlrpc/sec_config.c b/drivers/staging/lustre/lustre/ptlrpc/sec_config.c
index 2181a85efd49..0f4af66688a3 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/sec_config.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/sec_config.c
@@ -32,15 +32,16 @@
 
 #define DEBUG_SUBSYSTEM S_SEC
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 #include <linux/crypto.h>
 #include <linux/key.h>
 
-#include "../include/obd.h"
-#include "../include/obd_support.h"
-#include "../include/lustre_import.h"
-#include "../include/lustre_param.h"
-#include "../include/lustre_sec.h"
+#include <obd.h>
+#include <obd_class.h>
+#include <obd_support.h>
+#include <lustre_import.h>
+#include <uapi/linux/lustre/lustre_param.h>
+#include <lustre_sec.h>
 
 #include "ptlrpc_internal.h"
 
diff --git a/drivers/staging/lustre/lustre/ptlrpc/sec_gc.c b/drivers/staging/lustre/lustre/ptlrpc/sec_gc.c
index 8ffd000eafac..d10a8053d04f 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/sec_gc.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/sec_gc.c
@@ -36,12 +36,12 @@
 
 #define DEBUG_SUBSYSTEM S_SEC
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 
-#include "../include/obd_support.h"
-#include "../include/obd_class.h"
-#include "../include/lustre_net.h"
-#include "../include/lustre_sec.h"
+#include <obd_support.h>
+#include <obd_class.h>
+#include <lustre_net.h>
+#include <lustre_sec.h>
 
 #include "ptlrpc_internal.h"
 
@@ -66,7 +66,7 @@ void sptlrpc_gc_add_sec(struct ptlrpc_sec *sec)
 	sec->ps_gc_next = ktime_get_real_seconds() + sec->ps_gc_interval;
 
 	spin_lock(&sec_gc_list_lock);
-	list_add_tail(&sec_gc_list, &sec->ps_gc_list);
+	list_add_tail(&sec->ps_gc_list, &sec_gc_list);
 	spin_unlock(&sec_gc_list_lock);
 
 	CDEBUG(D_SEC, "added sec %p(%s)\n", sec, sec->ps_policy->sp_name);
diff --git a/drivers/staging/lustre/lustre/ptlrpc/sec_lproc.c b/drivers/staging/lustre/lustre/ptlrpc/sec_lproc.c
index 07273f577969..7792132eb145 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/sec_lproc.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/sec_lproc.c
@@ -34,16 +34,16 @@
 
 #define DEBUG_SUBSYSTEM S_SEC
 
-#include "../../include/linux/libcfs/libcfs.h"
+#include <linux/libcfs/libcfs.h>
 #include <linux/crypto.h>
 
-#include "../include/obd.h"
-#include "../include/obd_class.h"
-#include "../include/obd_support.h"
-#include "../include/lustre_net.h"
-#include "../include/lustre_import.h"
-#include "../include/lustre_dlm.h"
-#include "../include/lustre_sec.h"
+#include <obd.h>
+#include <obd_class.h>
+#include <obd_support.h>
+#include <lustre_net.h>
+#include <lustre_import.h>
+#include <lustre_dlm.h>
+#include <lustre_sec.h>
 
 #include "ptlrpc_internal.h"
 
diff --git a/drivers/staging/lustre/lustre/ptlrpc/sec_null.c b/drivers/staging/lustre/lustre/ptlrpc/sec_null.c
index 70a61e12bb7b..dc39a54c5e1a 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/sec_null.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/sec_null.c
@@ -36,11 +36,11 @@
 
 #define DEBUG_SUBSYSTEM S_SEC
 
-#include "../include/obd_support.h"
-#include "../include/obd_cksum.h"
-#include "../include/obd_class.h"
-#include "../include/lustre_net.h"
-#include "../include/lustre_sec.h"
+#include <obd_support.h>
+#include <obd_cksum.h>
+#include <obd_class.h>
+#include <lustre_net.h>
+#include <lustre_sec.h>
 
 #include "ptlrpc_internal.h"
 
diff --git a/drivers/staging/lustre/lustre/ptlrpc/sec_plain.c b/drivers/staging/lustre/lustre/ptlrpc/sec_plain.c
index c5e7a2309fce..6aa9b65b1926 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/sec_plain.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/sec_plain.c
@@ -36,11 +36,11 @@
 
 #define DEBUG_SUBSYSTEM S_SEC
 
-#include "../include/obd_support.h"
-#include "../include/obd_cksum.h"
-#include "../include/obd_class.h"
-#include "../include/lustre_net.h"
-#include "../include/lustre_sec.h"
+#include <obd_support.h>
+#include <obd_cksum.h>
+#include <obd_class.h>
+#include <lustre_net.h>
+#include <lustre_sec.h>
 #include "ptlrpc_internal.h"
 
 struct plain_sec {
diff --git a/drivers/staging/lustre/lustre/ptlrpc/service.c b/drivers/staging/lustre/lustre/ptlrpc/service.c
index 759aa6c16e28..155f6a45cc8b 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/service.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/service.c
@@ -31,11 +31,12 @@
  */
 
 #define DEBUG_SUBSYSTEM S_RPC
-#include "../include/obd_support.h"
-#include "../include/obd_class.h"
-#include "../include/lustre_net.h"
-#include "../include/lu_object.h"
-#include "../../include/linux/lnet/types.h"
+
+#include <obd_support.h>
+#include <obd_class.h>
+#include <lustre_net.h>
+#include <lu_object.h>
+#include <uapi/linux/lnet/lnet-types.h>
 #include "ptlrpc_internal.h"
 
 /* The following are visible and mutable through /sys/module/ptlrpc */
@@ -1565,9 +1566,9 @@ ptlrpc_server_handle_req_in(struct ptlrpc_service_part *svcpt,
 
 	/* req_in handling should/must be fast */
 	if (ktime_get_real_seconds() - req->rq_arrival_time.tv_sec > 5)
-		DEBUG_REQ(D_WARNING, req, "Slow req_in handling " CFS_DURATION_T "s",
-			  (long)(ktime_get_real_seconds() -
-				 req->rq_arrival_time.tv_sec));
+		DEBUG_REQ(D_WARNING, req, "Slow req_in handling %llds",
+			  (s64)(ktime_get_real_seconds() -
+				req->rq_arrival_time.tv_sec));
 
 	/* Set rpc server deadline and add it to the timed list */
 	deadline = (lustre_msghdr_get_flags(req->rq_reqmsg) &
@@ -1674,12 +1675,11 @@ ptlrpc_server_handle_request(struct ptlrpc_service_part *svcpt,
 	 * The deadline is increased if we send an early reply.
 	 */
 	if (ktime_get_real_seconds() > request->rq_deadline) {
-		DEBUG_REQ(D_ERROR, request, "Dropping timed-out request from %s: deadline " CFS_DURATION_T ":" CFS_DURATION_T "s ago\n",
+		DEBUG_REQ(D_ERROR, request, "Dropping timed-out request from %s: deadline %lld:%llds ago\n",
 			  libcfs_id2str(request->rq_peer),
-			  (long)(request->rq_deadline -
-				 request->rq_arrival_time.tv_sec),
-			  (long)(ktime_get_real_seconds() -
-				 request->rq_deadline));
+			  request->rq_deadline -
+			  request->rq_arrival_time.tv_sec,
+			  ktime_get_real_seconds() - request->rq_deadline);
 		goto put_conn;
 	}
 
diff --git a/drivers/staging/lustre/lustre/ptlrpc/wiretest.c b/drivers/staging/lustre/lustre/ptlrpc/wiretest.c
index 367f7e24e3da..07b86a1b6550 100644
--- a/drivers/staging/lustre/lustre/ptlrpc/wiretest.c
+++ b/drivers/staging/lustre/lustre/ptlrpc/wiretest.c
@@ -35,10 +35,10 @@
 #include <linux/fs.h>
 #include <linux/posix_acl_xattr.h>
 
-#include "../include/obd_support.h"
-#include "../include/obd_class.h"
-#include "../include/lustre_net.h"
-#include "../include/lustre_disk.h"
+#include <obd_support.h>
+#include <obd_class.h>
+#include <lustre_net.h>
+#include <lustre_disk.h>
 #include "ptlrpc_internal.h"
 
 void lustre_assert_wire_constants(void)
@@ -3820,14 +3820,14 @@ void lustre_assert_wire_constants(void)
 		 (long long)(int)offsetof(struct link_ea_header, leh_len));
 	LASSERTF((int)sizeof(((struct link_ea_header *)0)->leh_len) == 8, "found %lld\n",
 		 (long long)(int)sizeof(((struct link_ea_header *)0)->leh_len));
-	LASSERTF((int)offsetof(struct link_ea_header, padding1) == 16, "found %lld\n",
-		 (long long)(int)offsetof(struct link_ea_header, padding1));
-	LASSERTF((int)sizeof(((struct link_ea_header *)0)->padding1) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct link_ea_header *)0)->padding1));
-	LASSERTF((int)offsetof(struct link_ea_header, padding2) == 20, "found %lld\n",
-		 (long long)(int)offsetof(struct link_ea_header, padding2));
-	LASSERTF((int)sizeof(((struct link_ea_header *)0)->padding2) == 4, "found %lld\n",
-		 (long long)(int)sizeof(((struct link_ea_header *)0)->padding2));
+	LASSERTF((int)offsetof(struct link_ea_header, leh_overflow_time) == 16, "found %lld\n",
+		 (long long)(int)offsetof(struct link_ea_header, leh_overflow_time));
+	LASSERTF((int)sizeof(((struct link_ea_header *)0)->leh_overflow_time) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct link_ea_header *)0)->leh_overflow_time));
+	LASSERTF((int)offsetof(struct link_ea_header, leh_padding) == 20, "found %lld\n",
+		 (long long)(int)offsetof(struct link_ea_header, leh_padding));
+	LASSERTF((int)sizeof(((struct link_ea_header *)0)->leh_padding) == 4, "found %lld\n",
+		 (long long)(int)sizeof(((struct link_ea_header *)0)->leh_padding));
 	BUILD_BUG_ON(LINK_EA_MAGIC != 0x11EAF1DFUL);
 
 	/* Checks for struct link_ea_entry */
diff --git a/drivers/staging/media/atomisp/i2c/ap1302.c b/drivers/staging/media/atomisp/i2c/ap1302.c
index bacffbe962d4..2f772a020c8b 100644
--- a/drivers/staging/media/atomisp/i2c/ap1302.c
+++ b/drivers/staging/media/atomisp/i2c/ap1302.c
@@ -11,11 +11,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301, USA.
- *
  */
 
 #include "../include/linux/atomisp.h"
@@ -1098,7 +1093,7 @@ static const struct v4l2_ctrl_config ctrls[] = {
 	},
 };
 
-static struct v4l2_subdev_sensor_ops ap1302_sensor_ops = {
+static const struct v4l2_subdev_sensor_ops ap1302_sensor_ops = {
 	.g_skip_frames	= ap1302_g_skip_frames,
 };
 
diff --git a/drivers/staging/media/atomisp/i2c/gc0310.c b/drivers/staging/media/atomisp/i2c/gc0310.c
index 350fd7fd5b86..35ed51ffe944 100644
--- a/drivers/staging/media/atomisp/i2c/gc0310.c
+++ b/drivers/staging/media/atomisp/i2c/gc0310.c
@@ -118,9 +118,8 @@ static int gc0310_write_reg(struct i2c_client *client, u16 data_length,
 	/* high byte goes out first */
 	*wreg = (u8)(reg & 0xff);
 
-	if (data_length == GC0310_8BIT) {
+	if (data_length == GC0310_8BIT)
 		data[1] = (u8)(val);
-	}
 
 	ret = gc0310_i2c_write(client, len, data);
 	if (ret)
@@ -1453,7 +1452,7 @@ out_free:
 	return ret;
 }
 
-static struct acpi_device_id gc0310_acpi_match[] = {
+static const struct acpi_device_id gc0310_acpi_match[] = {
 	{"XXGC0310"},
 	{"INT0310"},
 	{},
diff --git a/drivers/staging/media/atomisp/i2c/gc2235.c b/drivers/staging/media/atomisp/i2c/gc2235.c
index 50f431729b6c..e43d31ea9676 100644
--- a/drivers/staging/media/atomisp/i2c/gc2235.c
+++ b/drivers/staging/media/atomisp/i2c/gc2235.c
@@ -480,7 +480,7 @@ static const struct v4l2_ctrl_ops ctrl_ops = {
 	.g_volatile_ctrl = gc2235_g_volatile_ctrl
 };
 
-struct v4l2_ctrl_config gc2235_controls[] = {
+static struct v4l2_ctrl_config gc2235_controls[] = {
 	{
 	 .ops = &ctrl_ops,
 	 .id = V4L2_CID_EXPOSURE_ABSOLUTE,
@@ -1183,7 +1183,7 @@ out_free:
 	return ret;
 }
 
-static struct acpi_device_id gc2235_acpi_match[] = {
+static const struct acpi_device_id gc2235_acpi_match[] = {
 	{ "INT33F8" },
 	{},
 };
diff --git a/drivers/staging/media/atomisp/i2c/gc2235.h b/drivers/staging/media/atomisp/i2c/gc2235.h
index 7c3d994180cc..a8d6aa9c9a5d 100644
--- a/drivers/staging/media/atomisp/i2c/gc2235.h
+++ b/drivers/staging/media/atomisp/i2c/gc2235.h
@@ -530,7 +530,7 @@ static struct gc2235_reg const gc2235_1616_1216_30fps[] = {
 	{ GC2235_TOK_TERM, 0, 0 }
 };
 
-struct gc2235_resolution gc2235_res_preview[] = {
+static struct gc2235_resolution gc2235_res_preview[] = {
 
 	{
 		.desc = "gc2235_1600_900_30fps",
@@ -582,7 +582,7 @@ struct gc2235_resolution gc2235_res_preview[] = {
 };
 #define N_RES_PREVIEW (ARRAY_SIZE(gc2235_res_preview))
 
-struct gc2235_resolution gc2235_res_still[] = {
+static struct gc2235_resolution gc2235_res_still[] = {
 	{
 		.desc = "gc2235_1600_900_30fps",
 		.width = 1600,
@@ -632,7 +632,7 @@ struct gc2235_resolution gc2235_res_still[] = {
 };
 #define N_RES_STILL (ARRAY_SIZE(gc2235_res_still))
 
-struct gc2235_resolution gc2235_res_video[] = {
+static struct gc2235_resolution gc2235_res_video[] = {
 	{
 		.desc = "gc2235_1296_736_30fps",
 		.width = 1296,
diff --git a/drivers/staging/media/atomisp/i2c/imx/ad5816g.c b/drivers/staging/media/atomisp/i2c/imx/ad5816g.c
index d68ebb49f002..558dcdf135d9 100644
--- a/drivers/staging/media/atomisp/i2c/imx/ad5816g.c
+++ b/drivers/staging/media/atomisp/i2c/imx/ad5816g.c
@@ -136,7 +136,7 @@ int ad5816g_vcm_power_down(struct v4l2_subdev *sd)
 }
 
 
-int ad5816g_t_focus_vcm(struct v4l2_subdev *sd, u16 val)
+static int ad5816g_t_focus_vcm(struct v4l2_subdev *sd, u16 val)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
 	u16 data = val & VCM_CODE_MASK;
@@ -214,12 +214,3 @@ int ad5816g_t_vcm_timing(struct v4l2_subdev *sd, s32 value)
 {
 	return 0;
 }
-
-int ad5816g_vcm_init(struct v4l2_subdev *sd)
-{
-	ad5816g_dev.platform_data = camera_get_af_platform_data();
-	return (NULL == ad5816g_dev.platform_data) ? -ENODEV : 0;
-
-}
-
-
diff --git a/drivers/staging/media/atomisp/i2c/imx/drv201.c b/drivers/staging/media/atomisp/i2c/imx/drv201.c
index 915e4019cfeb..6d9d4c968722 100644
--- a/drivers/staging/media/atomisp/i2c/imx/drv201.c
+++ b/drivers/staging/media/atomisp/i2c/imx/drv201.c
@@ -128,7 +128,7 @@ int drv201_vcm_power_down(struct v4l2_subdev *sd)
 }
 
 
-int drv201_t_focus_vcm(struct v4l2_subdev *sd, u16 val)
+static int drv201_t_focus_vcm(struct v4l2_subdev *sd, u16 val)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
 	u16 data = val & VCM_CODE_MASK;
@@ -207,12 +207,3 @@ int drv201_t_vcm_timing(struct v4l2_subdev *sd, s32 value)
 {
 	return 0;
 }
-
-int drv201_vcm_init(struct v4l2_subdev *sd)
-{
-	drv201_dev.platform_data = camera_get_af_platform_data();
-	return (NULL == drv201_dev.platform_data) ? -ENODEV : 0;
-}
-
-
-
diff --git a/drivers/staging/media/atomisp/i2c/imx/dw9714.c b/drivers/staging/media/atomisp/i2c/imx/dw9714.c
index b7dee1b6bb37..6397a7ee0af6 100644
--- a/drivers/staging/media/atomisp/i2c/imx/dw9714.c
+++ b/drivers/staging/media/atomisp/i2c/imx/dw9714.c
@@ -56,7 +56,7 @@ int dw9714_vcm_power_down(struct v4l2_subdev *sd)
 }
 
 
-int dw9714_t_focus_vcm(struct v4l2_subdev *sd, u16 val)
+static int dw9714_t_focus_vcm(struct v4l2_subdev *sd, u16 val)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
 	int ret = -EINVAL;
@@ -221,15 +221,3 @@ int dw9714_t_vcm_timing(struct v4l2_subdev *sd, s32 value)
 
 	return 0;
 }
-
-int dw9714_vcm_init(struct v4l2_subdev *sd)
-{
-
-	/* set VCM to home position and vcm mode to direct*/
-	dw9714_dev.vcm_mode = DW9714_DIRECT;
-	dw9714_dev.vcm_settings.update = false;
-	dw9714_dev.platform_data = camera_get_af_platform_data();
-	return (NULL == dw9714_dev.platform_data) ? -ENODEV : 0;
-
-}
-
diff --git a/drivers/staging/media/atomisp/i2c/imx/dw9718.c b/drivers/staging/media/atomisp/i2c/imx/dw9718.c
index 65a1fcf187d5..c02b9f0a2440 100644
--- a/drivers/staging/media/atomisp/i2c/imx/dw9718.c
+++ b/drivers/staging/media/atomisp/i2c/imx/dw9718.c
@@ -204,11 +204,6 @@ int dw9718_q_focus_status(struct v4l2_subdev *sd, s32 *value)
 	return 0;
 }
 
-int dw9718_t_focus_vcm(struct v4l2_subdev *sd, u16 val)
-{
-	return -EINVAL;
-}
-
 int dw9718_t_focus_rel(struct v4l2_subdev *sd, s32 value)
 {
 	return dw9718_t_focus_abs(sd, dw9718_dev.focus + value);
diff --git a/drivers/staging/media/atomisp/i2c/imx/dw9719.c b/drivers/staging/media/atomisp/i2c/imx/dw9719.c
index eca2d7640030..565237796bb4 100644
--- a/drivers/staging/media/atomisp/i2c/imx/dw9719.c
+++ b/drivers/staging/media/atomisp/i2c/imx/dw9719.c
@@ -161,11 +161,6 @@ int dw9719_q_focus_status(struct v4l2_subdev *sd, s32 *value)
 	return 0;
 }
 
-int dw9719_t_focus_vcm(struct v4l2_subdev *sd, u16 val)
-{
-	return -EINVAL;
-}
-
 int dw9719_t_focus_abs(struct v4l2_subdev *sd, s32 value)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
@@ -201,9 +196,3 @@ int dw9719_t_vcm_timing(struct v4l2_subdev *sd, s32 value)
 {
 	return 0;
 }
-
-int dw9719_vcm_init(struct v4l2_subdev *sd)
-{
-	dw9719_dev.platform_data = camera_get_af_platform_data();
-	return (NULL == dw9719_dev.platform_data) ? -ENODEV : 0;
-}
diff --git a/drivers/staging/media/atomisp/i2c/imx/imx.c b/drivers/staging/media/atomisp/i2c/imx/imx.c
index 408a7b945153..49ab0af87096 100644
--- a/drivers/staging/media/atomisp/i2c/imx/imx.c
+++ b/drivers/staging/media/atomisp/i2c/imx/imx.c
@@ -1084,46 +1084,15 @@ static int imx_g_bin_factor_y(struct v4l2_subdev *sd, s32 *val)
 	return 0;
 }
 
-int imx_vcm_power_up(struct v4l2_subdev *sd)
-{
-	struct imx_device *dev = to_imx_sensor(sd);
-	if (dev->vcm_driver && dev->vcm_driver->power_up)
-		return dev->vcm_driver->power_up(sd);
-	return 0;
-}
-
-int imx_vcm_power_down(struct v4l2_subdev *sd)
-{
-	struct imx_device *dev = to_imx_sensor(sd);
-	if (dev->vcm_driver && dev->vcm_driver->power_down)
-		return dev->vcm_driver->power_down(sd);
-	return 0;
-}
-
-int imx_vcm_init(struct v4l2_subdev *sd)
-{
-	struct imx_device *dev = to_imx_sensor(sd);
-	if (dev->vcm_driver && dev->vcm_driver->init)
-		return dev->vcm_driver->init(sd);
-	return 0;
-}
-
-int imx_t_focus_vcm(struct v4l2_subdev *sd, u16 val)
-{
-	struct imx_device *dev = to_imx_sensor(sd);
-	if (dev->vcm_driver && dev->vcm_driver->t_focus_vcm)
-		return dev->vcm_driver->t_focus_vcm(sd, val);
-	return 0;
-}
-
-int imx_t_focus_abs(struct v4l2_subdev *sd, s32 value)
+static int imx_t_focus_abs(struct v4l2_subdev *sd, s32 value)
 {
 	struct imx_device *dev = to_imx_sensor(sd);
 	if (dev->vcm_driver && dev->vcm_driver->t_focus_abs)
 		return dev->vcm_driver->t_focus_abs(sd, value);
 	return 0;
 }
-int imx_t_focus_rel(struct v4l2_subdev *sd, s32 value)
+
+static int imx_t_focus_rel(struct v4l2_subdev *sd, s32 value)
 {
 	struct imx_device *dev = to_imx_sensor(sd);
 	if (dev->vcm_driver && dev->vcm_driver->t_focus_rel)
@@ -1131,7 +1100,7 @@ int imx_t_focus_rel(struct v4l2_subdev *sd, s32 value)
 	return 0;
 }
 
-int imx_q_focus_status(struct v4l2_subdev *sd, s32 *value)
+static int imx_q_focus_status(struct v4l2_subdev *sd, s32 *value)
 {
 	struct imx_device *dev = to_imx_sensor(sd);
 	if (dev->vcm_driver && dev->vcm_driver->q_focus_status)
@@ -1139,7 +1108,7 @@ int imx_q_focus_status(struct v4l2_subdev *sd, s32 *value)
 	return 0;
 }
 
-int imx_q_focus_abs(struct v4l2_subdev *sd, s32 *value)
+static int imx_q_focus_abs(struct v4l2_subdev *sd, s32 *value)
 {
 	struct imx_device *dev = to_imx_sensor(sd);
 	if (dev->vcm_driver && dev->vcm_driver->q_focus_abs)
@@ -1147,7 +1116,7 @@ int imx_q_focus_abs(struct v4l2_subdev *sd, s32 *value)
 	return 0;
 }
 
-int imx_t_vcm_slew(struct v4l2_subdev *sd, s32 value)
+static int imx_t_vcm_slew(struct v4l2_subdev *sd, s32 value)
 {
 	struct imx_device *dev = to_imx_sensor(sd);
 	if (dev->vcm_driver && dev->vcm_driver->t_vcm_slew)
@@ -1155,7 +1124,7 @@ int imx_t_vcm_slew(struct v4l2_subdev *sd, s32 value)
 	return 0;
 }
 
-int imx_t_vcm_timing(struct v4l2_subdev *sd, s32 value)
+static int imx_t_vcm_timing(struct v4l2_subdev *sd, s32 value)
 {
 	struct imx_device *dev = to_imx_sensor(sd);
 	if (dev->vcm_driver && dev->vcm_driver->t_vcm_timing)
@@ -2105,8 +2074,7 @@ imx_s_parm(struct v4l2_subdev *sd, struct v4l2_streamparm *param)
 	return 0;
 }
 
-int
-imx_g_frame_interval(struct v4l2_subdev *sd,
+static int imx_g_frame_interval(struct v4l2_subdev *sd,
 				struct v4l2_subdev_frame_interval *interval)
 {
 	struct imx_device *dev = to_imx_sensor(sd);
diff --git a/drivers/staging/media/atomisp/i2c/imx/imx.h b/drivers/staging/media/atomisp/i2c/imx/imx.h
index 41b4133ca995..30beb2a0ed93 100644
--- a/drivers/staging/media/atomisp/i2c/imx/imx.h
+++ b/drivers/staging/media/atomisp/i2c/imx/imx.h
@@ -222,8 +222,6 @@
 struct imx_vcm {
 	int (*power_up)(struct v4l2_subdev *sd);
 	int (*power_down)(struct v4l2_subdev *sd);
-	int (*init)(struct v4l2_subdev *sd);
-	int (*t_focus_vcm)(struct v4l2_subdev *sd, u16 val);
 	int (*t_focus_abs)(struct v4l2_subdev *sd, s32 value);
 	int (*t_focus_abs_init)(struct v4l2_subdev *sd);
 	int (*t_focus_rel)(struct v4l2_subdev *sd, s32 value);
@@ -549,9 +547,6 @@ static const struct imx_reg imx219_param_update[] = {
 
 extern int ad5816g_vcm_power_up(struct v4l2_subdev *sd);
 extern int ad5816g_vcm_power_down(struct v4l2_subdev *sd);
-extern int ad5816g_vcm_init(struct v4l2_subdev *sd);
-
-extern int ad5816g_t_focus_vcm(struct v4l2_subdev *sd, u16 val);
 extern int ad5816g_t_focus_abs(struct v4l2_subdev *sd, s32 value);
 extern int ad5816g_t_focus_rel(struct v4l2_subdev *sd, s32 value);
 extern int ad5816g_q_focus_status(struct v4l2_subdev *sd, s32 *value);
@@ -561,9 +556,6 @@ extern int ad5816g_t_vcm_timing(struct v4l2_subdev *sd, s32 value);
 
 extern int drv201_vcm_power_up(struct v4l2_subdev *sd);
 extern int drv201_vcm_power_down(struct v4l2_subdev *sd);
-extern int drv201_vcm_init(struct v4l2_subdev *sd);
-
-extern int drv201_t_focus_vcm(struct v4l2_subdev *sd, u16 val);
 extern int drv201_t_focus_abs(struct v4l2_subdev *sd, s32 value);
 extern int drv201_t_focus_rel(struct v4l2_subdev *sd, s32 value);
 extern int drv201_q_focus_status(struct v4l2_subdev *sd, s32 *value);
@@ -573,9 +565,6 @@ extern int drv201_t_vcm_timing(struct v4l2_subdev *sd, s32 value);
 
 extern int dw9714_vcm_power_up(struct v4l2_subdev *sd);
 extern int dw9714_vcm_power_down(struct v4l2_subdev *sd);
-extern int dw9714_vcm_init(struct v4l2_subdev *sd);
-
-extern int dw9714_t_focus_vcm(struct v4l2_subdev *sd, u16 val);
 extern int dw9714_t_focus_abs(struct v4l2_subdev *sd, s32 value);
 extern int dw9714_t_focus_abs_init(struct v4l2_subdev *sd);
 extern int dw9714_t_focus_rel(struct v4l2_subdev *sd, s32 value);
@@ -586,9 +575,6 @@ extern int dw9714_t_vcm_timing(struct v4l2_subdev *sd, s32 value);
 
 extern int dw9719_vcm_power_up(struct v4l2_subdev *sd);
 extern int dw9719_vcm_power_down(struct v4l2_subdev *sd);
-extern int dw9719_vcm_init(struct v4l2_subdev *sd);
-
-extern int dw9719_t_focus_vcm(struct v4l2_subdev *sd, u16 val);
 extern int dw9719_t_focus_abs(struct v4l2_subdev *sd, s32 value);
 extern int dw9719_t_focus_rel(struct v4l2_subdev *sd, s32 value);
 extern int dw9719_q_focus_status(struct v4l2_subdev *sd, s32 *value);
@@ -598,9 +584,6 @@ extern int dw9719_t_vcm_timing(struct v4l2_subdev *sd, s32 value);
 
 extern int dw9718_vcm_power_up(struct v4l2_subdev *sd);
 extern int dw9718_vcm_power_down(struct v4l2_subdev *sd);
-extern int dw9718_vcm_init(struct v4l2_subdev *sd);
-
-extern int dw9718_t_focus_vcm(struct v4l2_subdev *sd, u16 val);
 extern int dw9718_t_focus_abs(struct v4l2_subdev *sd, s32 value);
 extern int dw9718_t_focus_rel(struct v4l2_subdev *sd, s32 value);
 extern int dw9718_q_focus_status(struct v4l2_subdev *sd, s32 *value);
@@ -615,8 +598,6 @@ struct imx_vcm imx_vcms[] = {
 	[IMX175_MERRFLD] = {
 		.power_up = drv201_vcm_power_up,
 		.power_down = drv201_vcm_power_down,
-		.init = drv201_vcm_init,
-		.t_focus_vcm = drv201_t_focus_vcm,
 		.t_focus_abs = drv201_t_focus_abs,
 		.t_focus_abs_init = NULL,
 		.t_focus_rel = drv201_t_focus_rel,
@@ -628,8 +609,6 @@ struct imx_vcm imx_vcms[] = {
 	[IMX175_VALLEYVIEW] = {
 		.power_up = dw9714_vcm_power_up,
 		.power_down = dw9714_vcm_power_down,
-		.init = dw9714_vcm_init,
-		.t_focus_vcm = dw9714_t_focus_vcm,
 		.t_focus_abs = dw9714_t_focus_abs,
 		.t_focus_abs_init = NULL,
 		.t_focus_rel = dw9714_t_focus_rel,
@@ -641,8 +620,6 @@ struct imx_vcm imx_vcms[] = {
 	[IMX135_SALTBAY] = {
 		.power_up = ad5816g_vcm_power_up,
 		.power_down = ad5816g_vcm_power_down,
-		.init = ad5816g_vcm_init,
-		.t_focus_vcm = ad5816g_t_focus_vcm,
 		.t_focus_abs = ad5816g_t_focus_abs,
 		.t_focus_abs_init = NULL,
 		.t_focus_rel = ad5816g_t_focus_rel,
@@ -654,8 +631,6 @@ struct imx_vcm imx_vcms[] = {
 	[IMX135_VICTORIABAY] = {
 		.power_up = dw9719_vcm_power_up,
 		.power_down = dw9719_vcm_power_down,
-		.init = dw9719_vcm_init,
-		.t_focus_vcm = dw9719_t_focus_vcm,
 		.t_focus_abs = dw9719_t_focus_abs,
 		.t_focus_abs_init = NULL,
 		.t_focus_rel = dw9719_t_focus_rel,
@@ -667,8 +642,6 @@ struct imx_vcm imx_vcms[] = {
 	[IMX134_VALLEYVIEW] = {
 		.power_up = dw9714_vcm_power_up,
 		.power_down = dw9714_vcm_power_down,
-		.init = dw9714_vcm_init,
-		.t_focus_vcm = dw9714_t_focus_vcm,
 		.t_focus_abs = dw9714_t_focus_abs,
 		.t_focus_abs_init = dw9714_t_focus_abs_init,
 		.t_focus_rel = dw9714_t_focus_rel,
@@ -680,8 +653,6 @@ struct imx_vcm imx_vcms[] = {
 	[IMX219_MFV0_PRH] = {
 		.power_up = dw9718_vcm_power_up,
 		.power_down = dw9718_vcm_power_down,
-		.init = dw9718_vcm_init,
-		.t_focus_vcm = dw9718_t_focus_vcm,
 		.t_focus_abs = dw9718_t_focus_abs,
 		.t_focus_abs_init = NULL,
 		.t_focus_rel = dw9718_t_focus_rel,
diff --git a/drivers/staging/media/atomisp/i2c/lm3554.c b/drivers/staging/media/atomisp/i2c/lm3554.c
index 2b170c07aaba..679176f7c542 100644
--- a/drivers/staging/media/atomisp/i2c/lm3554.c
+++ b/drivers/staging/media/atomisp/i2c/lm3554.c
@@ -974,7 +974,7 @@ static const struct dev_pm_ops lm3554_pm_ops = {
 	.resume = lm3554_resume,
 };
 
-static struct acpi_device_id lm3554_acpi_match[] = {
+static const struct acpi_device_id lm3554_acpi_match[] = {
 	{ "INTCF1C" },
 	{},
 };
diff --git a/drivers/staging/media/atomisp/i2c/mt9m114.c b/drivers/staging/media/atomisp/i2c/mt9m114.c
index 3fa915313e53..3c837cb8859c 100644
--- a/drivers/staging/media/atomisp/i2c/mt9m114.c
+++ b/drivers/staging/media/atomisp/i2c/mt9m114.c
@@ -1209,10 +1209,10 @@ static int mt9m114_s_exposure_selection(struct v4l2_subdev *sd,
 		return -EINVAL;
 	}
 
-	clamp_t(int, win_left, 0, 4);
-	clamp_t(int, win_top, 0, 4);
-	clamp_t(int, win_right, 0, 4);
-	clamp_t(int, win_bottom, 0, 4);
+	win_left   = clamp_t(int, win_left, 0, 4);
+	win_top    = clamp_t(int, win_top, 0, 4);
+	win_right  = clamp_t(int, win_right, 0, 4);
+	win_bottom = clamp_t(int, win_bottom, 0, 4);
 
 	ret = mt9m114_write_reg_array(client, mt9m114_exp_average, NO_POLLING);
 	if (ret) {
@@ -1806,7 +1806,7 @@ static const struct v4l2_subdev_video_ops mt9m114_video_ops = {
 	.g_frame_interval = mt9m114_g_frame_interval,
 };
 
-static struct v4l2_subdev_sensor_ops mt9m114_sensor_ops = {
+static const struct v4l2_subdev_sensor_ops mt9m114_sensor_ops = {
 	.g_skip_frames	= mt9m114_g_skip_frames,
 };
 
@@ -1928,7 +1928,7 @@ static int mt9m114_probe(struct i2c_client *client,
 
 MODULE_DEVICE_TABLE(i2c, mt9m114_id);
 
-static struct acpi_device_id mt9m114_acpi_match[] = {
+static const struct acpi_device_id mt9m114_acpi_match[] = {
 	{ "INT33F0" },
 	{ "CRMT1040" },
 	{},
diff --git a/drivers/staging/media/atomisp/i2c/ov2680.c b/drivers/staging/media/atomisp/i2c/ov2680.c
index 3cabfe54c669..51b7d61df0f5 100644
--- a/drivers/staging/media/atomisp/i2c/ov2680.c
+++ b/drivers/staging/media/atomisp/i2c/ov2680.c
@@ -89,7 +89,7 @@ static int ov2680_read_reg(struct i2c_client *client,
 			"read from offset 0x%x error %d", reg, err);
 		return err;
 	}
-	
+
 	*val = 0;
 	/* high byte comes first */
 	if (data_length == OV2680_8BIT)
@@ -285,7 +285,6 @@ static int ov2680_g_fnumber(struct v4l2_subdev *sd, s32 *val)
 
 static int ov2680_g_fnumber_range(struct v4l2_subdev *sd, s32 *val)
 {
-	
 	*val = (OV2680_F_NUMBER_DEFAULT_NUM << 24) |
 		(OV2680_F_NUMBER_DEM << 16) |
 		(OV2680_F_NUMBER_DEFAULT_NUM << 8) | OV2680_F_NUMBER_DEM;
@@ -306,7 +305,7 @@ static int ov2680_g_bin_factor_y(struct v4l2_subdev *sd, s32 *val)
 {
 	struct ov2680_device *dev = to_ov2680_sensor(sd);
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	
+
 	*val = ov2680_res[dev->fmt_idx].bin_factor_y;
 	dev_dbg(&client->dev,  "++++ov2680_g_bin_factor_y\n");
 	return 0;
@@ -399,7 +398,7 @@ static long __ov2680_set_exposure(struct v4l2_subdev *sd, int coarse_itg,
 	struct ov2680_device *dev = to_ov2680_sensor(sd);
 	u16 vts,hts;
 	int ret,exp_val;
-	
+
        dev_dbg(&client->dev, "+++++++__ov2680_set_exposure coarse_itg %d, gain %d, digitgain %d++\n",coarse_itg, gain, digitgain);
 
 	hts = ov2680_res[dev->fmt_idx].pixels_per_line;
@@ -542,7 +541,7 @@ static long ov2680_ioctl(struct v4l2_subdev *sd, unsigned int cmd, void *arg)
 	switch (cmd) {
 	case ATOMISP_IOC_S_EXPOSURE:
 		return ov2680_s_exposure(sd, arg);
-	
+
 	default:
 		return -EINVAL;
 	}
@@ -983,7 +982,7 @@ static int ov2680_s_power(struct v4l2_subdev *sd, int on)
 	if (on == 0){
 		ret = power_down(sd);
 	} else {
-		ret = power_up(sd);	
+		ret = power_up(sd);
 		if (!ret)
 			return ov2680_init(sd);
 	}
@@ -1207,7 +1206,7 @@ static int ov2680_s_stream(struct v4l2_subdev *sd, int enable)
 		dev_dbg(&client->dev, "ov2680_s_stream one \n");
 	else
 		dev_dbg(&client->dev, "ov2680_s_stream off \n");
-	
+
 	ret = ov2680_write_reg(client, OV2680_8BIT, OV2680_SW_STREAM,
 				enable ? OV2680_START_STREAMING :
 				OV2680_STOP_STREAMING);
@@ -1267,7 +1266,7 @@ static int ov2680_s_config(struct v4l2_subdev *sd,
 		dev_err(&client->dev, "ov2680_detect err s_config.\n");
 		goto fail_csi_cfg;
 	}
-	
+
 	/* turn off sensor, after probed */
 	ret = power_down(sd);
 	if (ret) {
@@ -1385,7 +1384,7 @@ static int ov2680_enum_frame_size(struct v4l2_subdev *sd,
 static int ov2680_g_skip_frames(struct v4l2_subdev *sd, u32 *frames)
 {
 	struct ov2680_device *dev = to_ov2680_sensor(sd);
-	
+
 	mutex_lock(&dev->input_lock);
 	*frames = ov2680_res[dev->fmt_idx].skip_frames;
 	mutex_unlock(&dev->input_lock);
@@ -1517,7 +1516,7 @@ out_free:
 	return ret;
 }
 
-static struct acpi_device_id ov2680_acpi_match[] = {
+static const struct acpi_device_id ov2680_acpi_match[] = {
 	{"XXOV2680"},
 	{"OVTI2680"},
 	{},
diff --git a/drivers/staging/media/atomisp/i2c/ov2722.c b/drivers/staging/media/atomisp/i2c/ov2722.c
index b7afadebdf89..10094ac56561 100644
--- a/drivers/staging/media/atomisp/i2c/ov2722.c
+++ b/drivers/staging/media/atomisp/i2c/ov2722.c
@@ -1337,7 +1337,7 @@ out_free:
 
 MODULE_DEVICE_TABLE(i2c, ov2722_id);
 
-static struct acpi_device_id ov2722_acpi_match[] = {
+static const struct acpi_device_id ov2722_acpi_match[] = {
 	{ "INT33FB" },
 	{},
 };
diff --git a/drivers/staging/media/atomisp/i2c/ov5693/ov5693.c b/drivers/staging/media/atomisp/i2c/ov5693/ov5693.c
index d6447398f5ef..123642557aa8 100644
--- a/drivers/staging/media/atomisp/i2c/ov5693/ov5693.c
+++ b/drivers/staging/media/atomisp/i2c/ov5693/ov5693.c
@@ -146,7 +146,7 @@ static int ov5693_read_reg(struct i2c_client *client,
 		return -EINVAL;
 	}
 
-	memset(msg, 0 , sizeof(msg));
+	memset(msg, 0, sizeof(msg));
 
 	msg[0].addr = client->addr;
 	msg[0].flags = 0;
@@ -702,7 +702,7 @@ static long ov5693_s_exposure(struct v4l2_subdev *sd,
 }
 
 static int ov5693_read_otp_reg_array(struct i2c_client *client, u16 size,
-				     u16 addr, u8 * buf)
+				     u16 addr, u8 *buf)
 {
 	u16 index;
 	int ret;
@@ -720,7 +720,7 @@ static int ov5693_read_otp_reg_array(struct i2c_client *client, u16 size,
 	return 0;
 }
 
-static int __ov5693_otp_read(struct v4l2_subdev *sd, u8 * buf)
+static int __ov5693_otp_read(struct v4l2_subdev *sd, u8 *buf)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
 	struct ov5693_device *dev = to_ov5693_sensor(sd);
@@ -913,7 +913,7 @@ err:
 	return ret;
 }
 
-int ad5823_t_focus_vcm(struct v4l2_subdev *sd, u16 val)
+static int ad5823_t_focus_vcm(struct v4l2_subdev *sd, u16 val)
 {
 	struct i2c_client *client = v4l2_get_subdevdata(sd);
 	int ret = -EINVAL;
@@ -2032,7 +2032,7 @@ out_free:
 
 MODULE_DEVICE_TABLE(i2c, ov5693_id);
 
-static struct acpi_device_id ov5693_acpi_match[] = {
+static const struct acpi_device_id ov5693_acpi_match[] = {
 	{"INT33BE"},
 	{},
 };
diff --git a/drivers/staging/media/atomisp/i2c/ov8858.c b/drivers/staging/media/atomisp/i2c/ov8858.c
index 9574bc49113c..43e1638fd674 100644
--- a/drivers/staging/media/atomisp/i2c/ov8858.c
+++ b/drivers/staging/media/atomisp/i2c/ov8858.c
@@ -2189,7 +2189,7 @@ static const struct i2c_device_id ov8858_id[] = {
 
 MODULE_DEVICE_TABLE(i2c, ov8858_id);
 
-static struct acpi_device_id ov8858_acpi_match[] = {
+static const struct acpi_device_id ov8858_acpi_match[] = {
 	{"INT3477"},
 	{},
 };
diff --git a/drivers/staging/media/atomisp/i2c/ov8858.h b/drivers/staging/media/atomisp/i2c/ov8858.h
index d3fde200c013..638d1a803a2b 100644
--- a/drivers/staging/media/atomisp/i2c/ov8858.h
+++ b/drivers/staging/media/atomisp/i2c/ov8858.h
@@ -164,7 +164,6 @@ struct ov8858_vcm {
 	int (*power_up)(struct v4l2_subdev *sd);
 	int (*power_down)(struct v4l2_subdev *sd);
 	int (*init)(struct v4l2_subdev *sd);
-	int (*t_focus_vcm)(struct v4l2_subdev *sd, u16 val);
 	int (*t_focus_abs)(struct v4l2_subdev *sd, s32 value);
 	int (*t_focus_rel)(struct v4l2_subdev *sd, s32 value);
 	int (*q_focus_status)(struct v4l2_subdev *sd, s32 *value);
@@ -312,7 +311,6 @@ static const struct ov8858_reg ov8858_param_update[] = {
 extern int dw9718_vcm_power_up(struct v4l2_subdev *sd);
 extern int dw9718_vcm_power_down(struct v4l2_subdev *sd);
 extern int dw9718_vcm_init(struct v4l2_subdev *sd);
-extern int dw9718_t_focus_vcm(struct v4l2_subdev *sd, u16 val);
 extern int dw9718_t_focus_abs(struct v4l2_subdev *sd, s32 value);
 extern int dw9718_t_focus_rel(struct v4l2_subdev *sd, s32 value);
 extern int dw9718_q_focus_status(struct v4l2_subdev *sd, s32 *value);
@@ -328,7 +326,6 @@ static struct ov8858_vcm ov8858_vcms[] = {
 		.power_up = dw9718_vcm_power_up,
 		.power_down = dw9718_vcm_power_down,
 		.init = dw9718_vcm_init,
-		.t_focus_vcm = dw9718_t_focus_vcm,
 		.t_focus_abs = dw9718_t_focus_abs,
 		.t_focus_rel = dw9718_t_focus_rel,
 		.q_focus_status = dw9718_q_focus_status,
diff --git a/drivers/staging/media/atomisp/i2c/ov8858_btns.h b/drivers/staging/media/atomisp/i2c/ov8858_btns.h
index f9a3cf8fbf1a..7d74a8899fae 100644
--- a/drivers/staging/media/atomisp/i2c/ov8858_btns.h
+++ b/drivers/staging/media/atomisp/i2c/ov8858_btns.h
@@ -164,7 +164,6 @@ struct ov8858_vcm {
 	int (*power_up)(struct v4l2_subdev *sd);
 	int (*power_down)(struct v4l2_subdev *sd);
 	int (*init)(struct v4l2_subdev *sd);
-	int (*t_focus_vcm)(struct v4l2_subdev *sd, u16 val);
 	int (*t_focus_abs)(struct v4l2_subdev *sd, s32 value);
 	int (*t_focus_rel)(struct v4l2_subdev *sd, s32 value);
 	int (*q_focus_status)(struct v4l2_subdev *sd, s32 *value);
@@ -312,7 +311,6 @@ static const struct ov8858_reg ov8858_param_update[] = {
 extern int dw9718_vcm_power_up(struct v4l2_subdev *sd);
 extern int dw9718_vcm_power_down(struct v4l2_subdev *sd);
 extern int dw9718_vcm_init(struct v4l2_subdev *sd);
-extern int dw9718_t_focus_vcm(struct v4l2_subdev *sd, u16 val);
 extern int dw9718_t_focus_abs(struct v4l2_subdev *sd, s32 value);
 extern int dw9718_t_focus_rel(struct v4l2_subdev *sd, s32 value);
 extern int dw9718_q_focus_status(struct v4l2_subdev *sd, s32 *value);
@@ -328,7 +326,6 @@ static struct ov8858_vcm ov8858_vcms[] = {
 		.power_up = dw9718_vcm_power_up,
 		.power_down = dw9718_vcm_power_down,
 		.init = dw9718_vcm_init,
-		.t_focus_vcm = dw9718_t_focus_vcm,
 		.t_focus_abs = dw9718_t_focus_abs,
 		.t_focus_rel = dw9718_t_focus_rel,
 		.q_focus_status = dw9718_q_focus_status,
diff --git a/drivers/staging/media/atomisp/include/linux/atomisp.h b/drivers/staging/media/atomisp/include/linux/atomisp.h
index 35865462ccf9..d67dd658cff9 100644
--- a/drivers/staging/media/atomisp/include/linux/atomisp.h
+++ b/drivers/staging/media/atomisp/include/linux/atomisp.h
@@ -28,12 +28,6 @@
 #include <linux/types.h>
 #include <linux/version.h>
 
-/* struct media_device_info.driver_version */
-#define ATOMISP_CSS_VERSION_MASK	0x00ffffff
-#define ATOMISP_CSS_VERSION_15		KERNEL_VERSION(1, 5, 0)
-#define ATOMISP_CSS_VERSION_20		KERNEL_VERSION(2, 0, 0)
-#define ATOMISP_CSS_VERSION_21		KERNEL_VERSION(2, 1, 0)
-
 /* struct media_device_info.hw_revision */
 #define ATOMISP_HW_REVISION_MASK	0x0000ff00
 #define ATOMISP_HW_REVISION_SHIFT	8
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_cmd.c b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_cmd.c
index 97093baf28ac..f48bf451c1f5 100644
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_cmd.c
+++ b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_cmd.c
@@ -83,48 +83,6 @@ union host {
 };
 
 /*
- * atomisp_kernel_malloc: chooses whether kmalloc() or vmalloc() is preferable.
- *
- * It is also a wrap functions to pass into css framework.
- */
-void *atomisp_kernel_malloc(size_t bytes)
-{
-	/* vmalloc() is preferable if allocating more than 1 page */
-	if (bytes > PAGE_SIZE)
-		return vmalloc(bytes);
-
-	return kmalloc(bytes, GFP_KERNEL);
-}
-
-/*
- * atomisp_kernel_zalloc: chooses whether set 0 to the allocated memory.
- *
- * It is also a wrap functions to pass into css framework.
- */
-void *atomisp_kernel_zalloc(size_t bytes, bool zero_mem)
-{
-	void *ptr = atomisp_kernel_malloc(bytes);
-
-	if (ptr && zero_mem)
-		memset(ptr, 0, bytes);
-
-	return ptr;
-}
-
-/*
- * Free buffer allocated with atomisp_kernel_malloc()/atomisp_kernel_zalloc
- * helper
- */
-void atomisp_kernel_free(void *ptr)
-{
-	/* Verify if buffer was allocated by vmalloc() or kmalloc() */
-	if (is_vmalloc_addr(ptr))
-		vfree(ptr);
-	else
-		kfree(ptr);
-}
-
-/*
  * get sensor:dis71430/ov2720 related info from v4l2_subdev->priv data field.
  * subdev->priv is set in mrst.c
  */
@@ -785,7 +743,7 @@ void atomisp_flush_params_queue(struct atomisp_video_pipe *pipe)
 				   struct atomisp_css_params_with_list, list);
 		list_del(&param->list);
 		atomisp_free_css_parameters(&param->params);
-		atomisp_kernel_free(param);
+		kvfree(param);
 	}
 }
 
@@ -1132,7 +1090,7 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
 				asd->params.dvs_6axis = NULL;
 			atomisp_free_css_parameters(
 				&pipe->frame_params[vb->i]->params);
-			atomisp_kernel_free(pipe->frame_params[vb->i]);
+			kvfree(pipe->frame_params[vb->i]);
 			pipe->frame_params[vb->i] = NULL;
 		}
 
@@ -4329,7 +4287,7 @@ int atomisp_set_parameters(struct video_device *vdev,
 		 * are ready, the parameters will be set to CSS.
 		 * per-frame setting only works for the main output frame.
 		 */
-		param = atomisp_kernel_zalloc(sizeof(*param), true);
+		param = kvzalloc(sizeof(*param), GFP_KERNEL);
 		if (!param) {
 			dev_err(asd->isp->dev, "%s: failed to alloc params buffer\n",
 				__func__);
@@ -4375,7 +4333,7 @@ apply_parameter_failed:
 	if (css_param)
 		atomisp_free_css_parameters(css_param);
 	if (param)
-		atomisp_kernel_free(param);
+		kvfree(param);
 
 	return ret;
 }
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_cmd.h b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_cmd.h
index 8e6d9df7ad1a..31ba4e613d13 100644
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_cmd.h
+++ b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_cmd.h
@@ -78,9 +78,6 @@ static inline void __iomem *atomisp_get_io_virt_addr(unsigned int address)
 	return ret;
 }
 */
-void *atomisp_kernel_malloc(size_t bytes);
-void *atomisp_kernel_zalloc(size_t bytes, bool zero_mem);
-void atomisp_kernel_free(void *ptr);
 
 /*
  * Interrupt functions
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat_css20.c b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat_css20.c
index ad2c610d2ce3..05897b747349 100644
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat_css20.c
+++ b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_compat_css20.c
@@ -1671,12 +1671,12 @@ int atomisp_alloc_metadata_output_buf(struct atomisp_sub_device *asd)
 	/* We allocate the cpu-side buffer used for communication with user
 	 * space */
 	for (i = 0; i < ATOMISP_METADATA_TYPE_NUM; i++) {
-		asd->params.metadata_user[i] = atomisp_kernel_malloc(
+		asd->params.metadata_user[i] = kvmalloc(
 				asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].
-				stream_info.metadata_info.size);
+				stream_info.metadata_info.size, GFP_KERNEL);
 		if (!asd->params.metadata_user[i]) {
 			while (--i >= 0) {
-				atomisp_kernel_free(asd->params.metadata_user[i]);
+				kvfree(asd->params.metadata_user[i]);
 				asd->params.metadata_user[i] = NULL;
 			}
 			return -ENOMEM;
@@ -1692,7 +1692,7 @@ void atomisp_free_metadata_output_buf(struct atomisp_sub_device *asd)
 
 	for (i = 0; i < ATOMISP_METADATA_TYPE_NUM; i++) {
 		if (asd->params.metadata_user[i]) {
-			atomisp_kernel_free(asd->params.metadata_user[i]);
+			kvfree(asd->params.metadata_user[i]);
 			asd->params.metadata_user[i] = NULL;
 		}
 	}
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_fops.c b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_fops.c
index c151c848cf8f..d8cfed358d55 100644
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_fops.c
+++ b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_fops.c
@@ -643,14 +643,14 @@ static void atomisp_buf_release_output(struct videobuf_queue *vq,
 	vb->state = VIDEOBUF_NEEDS_INIT;
 }
 
-static struct videobuf_queue_ops videobuf_qops = {
+static const struct videobuf_queue_ops videobuf_qops = {
 	.buf_setup	= atomisp_buf_setup,
 	.buf_prepare	= atomisp_buf_prepare,
 	.buf_queue	= atomisp_buf_queue,
 	.buf_release	= atomisp_buf_release,
 };
 
-static struct videobuf_queue_ops videobuf_qops_output = {
+static const struct videobuf_queue_ops videobuf_qops_output = {
 	.buf_setup	= atomisp_buf_setup_output,
 	.buf_prepare	= atomisp_buf_prepare_output,
 	.buf_queue	= atomisp_buf_queue_output,
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_internal.h b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_internal.h
index c8e0c4fe3717..7542a72f1d0f 100644
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_internal.h
+++ b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_internal.h
@@ -75,15 +75,6 @@
 #define ATOMISP_PCI_REV_MRFLD_A0_MAX	0
 #define ATOMISP_PCI_REV_BYT_A0_MAX	4
 
-#define ATOMISP_MAJOR		0
-#define ATOMISP_MINOR		5
-#define ATOMISP_PATCHLEVEL	1
-
-#define DRIVER_VERSION_STR	__stringify(ATOMISP_MAJOR) \
-	"." __stringify(ATOMISP_MINOR) "." __stringify(ATOMISP_PATCHLEVEL)
-#define DRIVER_VERSION		KERNEL_VERSION(ATOMISP_MAJOR, \
-	ATOMISP_MINOR, ATOMISP_PATCHLEVEL)
-
 #define ATOM_ISP_STEP_WIDTH	2
 #define ATOM_ISP_STEP_HEIGHT	2
 
@@ -310,10 +301,6 @@ struct atomisp_device {
 
 extern struct device *atomisp_dev;
 
-extern void *atomisp_kernel_malloc(size_t bytes);
-
-extern void atomisp_kernel_free(void *ptr);
-
 #define atomisp_is_wdt_running(a) timer_pending(&(a)->wdt)
 #ifdef ISP2401
 extern void atomisp_wdt_refresh_pipe(struct atomisp_video_pipe *pipe,
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_ioctl.c b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_ioctl.c
index aa0526ebaff1..717647951fb6 100644
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_ioctl.c
+++ b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_ioctl.c
@@ -51,7 +51,6 @@
 static const char *DRIVER = "atomisp";	/* max size 15 */
 static const char *CARD = "ATOM ISP";	/* max size 31 */
 static const char *BUS_INFO = "PCI-3";	/* max size 31 */
-static const u32 VERSION = DRIVER_VERSION;
 
 /*
  * FIXME: ISP should not know beforehand all CIDs supported by sensor.
@@ -562,8 +561,6 @@ static int atomisp_querycap(struct file *file, void *fh,
 	strncpy(cap->card, CARD, sizeof(cap->card) - 1);
 	strncpy(cap->bus_info, BUS_INFO, sizeof(cap->card) - 1);
 
-	cap->version = VERSION;
-
 	cap->device_caps = V4L2_CAP_VIDEO_CAPTURE |
 	    V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_OUTPUT;
 	cap->capabilities = cap->device_caps | V4L2_CAP_DEVICE_CAPS;
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_subdev.c b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_subdev.c
index 3d6bb166927c..744ab6eb42a0 100644
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_subdev.c
+++ b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_subdev.c
@@ -1253,8 +1253,7 @@ int atomisp_create_pads_links(struct atomisp_device *isp)
 {
 	struct atomisp_sub_device *asd;
 	int i, j, ret = 0;
-	isp->num_of_streams = isp->media_dev.driver_version >=
-	    ATOMISP_CSS_VERSION_20 ? 2 : 1;
+	isp->num_of_streams = 2;
 	for (i = 0; i < ATOMISP_CAMERA_NR_PORTS; i++) {
 		for (j = 0; j < isp->num_of_streams; j++) {
 			ret =
@@ -1414,8 +1413,7 @@ int atomisp_subdev_init(struct atomisp_device *isp)
 	 * CSS2.0 running ISP2400 support
 	 * multiple streams
 	 */
-	isp->num_of_streams = isp->media_dev.driver_version >=
-	    ATOMISP_CSS_VERSION_20 ? 2 : 1;
+	isp->num_of_streams = 2;
 	isp->asd = devm_kzalloc(isp->dev, sizeof(struct atomisp_sub_device) *
 			       isp->num_of_streams, GFP_KERNEL);
 	if (!isp->asd)
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_v4l2.c b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_v4l2.c
index a543def739fc..663aa916e3ca 100644
--- a/drivers/staging/media/atomisp/pci/atomisp2/atomisp_v4l2.c
+++ b/drivers/staging/media/atomisp/pci/atomisp2/atomisp_v4l2.c
@@ -1083,27 +1083,23 @@ atomisp_load_firmware(struct atomisp_device *isp)
 	if (skip_fwload)
 		return NULL;
 
-	if (isp->media_dev.driver_version == ATOMISP_CSS_VERSION_21) {
-		if (isp->media_dev.hw_revision ==
-		    ((ATOMISP_HW_REVISION_ISP2401 << ATOMISP_HW_REVISION_SHIFT)
-		     | ATOMISP_HW_STEPPING_A0))
-			fw_path = "shisp_2401a0_v21.bin";
-
-		if (isp->media_dev.hw_revision ==
-		    ((ATOMISP_HW_REVISION_ISP2401_LEGACY << ATOMISP_HW_REVISION_SHIFT)
-		     | ATOMISP_HW_STEPPING_A0))
-			fw_path = "shisp_2401a0_legacy_v21.bin";
-
-		if (isp->media_dev.hw_revision ==
-		    ((ATOMISP_HW_REVISION_ISP2400 << ATOMISP_HW_REVISION_SHIFT)
-		     | ATOMISP_HW_STEPPING_B0))
-			fw_path = "shisp_2400b0_v21.bin";
-	}
+	if (isp->media_dev.hw_revision ==
+	    ((ATOMISP_HW_REVISION_ISP2401 << ATOMISP_HW_REVISION_SHIFT)
+	     | ATOMISP_HW_STEPPING_A0))
+		fw_path = "shisp_2401a0_v21.bin";
+
+	if (isp->media_dev.hw_revision ==
+	    ((ATOMISP_HW_REVISION_ISP2401_LEGACY << ATOMISP_HW_REVISION_SHIFT)
+	     | ATOMISP_HW_STEPPING_A0))
+		fw_path = "shisp_2401a0_legacy_v21.bin";
+
+	if (isp->media_dev.hw_revision ==
+	    ((ATOMISP_HW_REVISION_ISP2400 << ATOMISP_HW_REVISION_SHIFT)
+	     | ATOMISP_HW_STEPPING_B0))
+		fw_path = "shisp_2400b0_v21.bin";
 
 	if (!fw_path) {
-		dev_err(isp->dev,
-			"Unsupported driver_version 0x%x, hw_revision 0x%x\n",
-			isp->media_dev.driver_version,
+		dev_err(isp->dev, "Unsupported hw_revision 0x%x\n",
 			isp->media_dev.hw_revision);
 		return NULL;
 	}
@@ -1251,7 +1247,6 @@ static int atomisp_pci_probe(struct pci_dev *dev,
 	/* This is not a true PCI device on SoC, so the delay is not needed. */
 	isp->pdev->d3_delay = 0;
 
-	isp->media_dev.driver_version = ATOMISP_CSS_VERSION_21;
 	switch (id->device & ATOMISP_PCI_DEVICE_SOC_MASK) {
 	case ATOMISP_PCI_DEVICE_SOC_MRFLD:
 		isp->media_dev.hw_revision =
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/ibuf_ctrl_rmgr.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/ibuf_ctrl_rmgr.c
index 76d9142fd37e..faef97672eac 100644
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/ibuf_ctrl_rmgr.c
+++ b/drivers/staging/media/atomisp/pci/atomisp2/css2400/runtime/isys/src/ibuf_ctrl_rmgr.c
@@ -14,18 +14,18 @@
  */
 #else
 /**
-Support for Intel Camera Imaging ISP subsystem.
-Copyright (c) 2010 - 2015, Intel Corporation.
-
-This program is free software; you can redistribute it and/or modify it
-under the terms and conditions of the GNU General Public License,
-version 2, as published by the Free Software Foundation.
-
-This program is distributed in the hope it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-*/
+ * Support for Intel Camera Imaging ISP subsystem.
+ * Copyright (c) 2010 - 2015, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
 #endif
 
 #include "system_global.h"
@@ -130,8 +130,7 @@ void ia_css_isys_ibuf_rmgr_release(
 
 	for (i = 0; i < ibuf_rsrc.num_allocated; i++) {
 		handle = getHandle(i);
-		if ((handle->start_addr == *start_addr)
-		    && ( true == handle->active)) {
+		if (handle->active && handle->start_addr == *start_addr) {
 			handle->active = false;
 			ibuf_rsrc.num_active--;
 			break;
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css.c
index 471f2be974e2..e882b5596813 100644
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css.c
+++ b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css.c
@@ -1939,6 +1939,7 @@ void *sh_css_calloc(size_t N, size_t size)
 		p = sh_css_malloc(N*size);
 		if (p)
 			memset(p, 0, size);
+		return p;
 	}
 	return NULL;
 }
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_firmware.c b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_firmware.c
index eecd8cf71951..63582161050a 100644
--- a/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_firmware.c
+++ b/drivers/staging/media/atomisp/pci/atomisp2/css2400/sh_css_firmware.c
@@ -131,14 +131,10 @@ sh_css_load_blob_info(const char *fw, const struct ia_css_fw_info *bi, struct ia
 
 	if (bi->type == ia_css_isp_firmware || bi->type == ia_css_sp_firmware) {
 		char *namebuffer;
-		int namelength = (int)strlen(name);
 
-		namebuffer = (char *) kmalloc(namelength + 1, GFP_KERNEL);
-		if (namebuffer == NULL)
+		namebuffer = kstrdup(name, GFP_KERNEL);
+		if (!namebuffer)
 			return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY;
-
-		memcpy(namebuffer, name, namelength + 1);
-
 		bd->name = fw_minibuffer[index].name = namebuffer;
 	} else {
 		bd->name = name;
diff --git a/drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm.c b/drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm.c
index 05eeff58a229..b8aae4ba5a78 100644
--- a/drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm.c
+++ b/drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm.c
@@ -46,14 +46,16 @@ static ia_css_ptr dummy_ptr;
 static bool hmm_initialized;
 struct _hmm_mem_stat hmm_mem_stat;
 
-/* p: private
-   s: shared
-   u: user
-   i: ion */
+/*
+ * p: private
+ * s: shared
+ * u: user
+ * i: ion
+ */
 static const char hmm_bo_type_string[] = "psui";
 
 static ssize_t bo_show(struct device *dev, struct device_attribute *attr,
-			char *buf, struct list_head *bo_list, bool active)
+		       char *buf, struct list_head *bo_list, bool active)
 {
 	ssize_t ret = 0;
 	struct hmm_buffer_object *bo;
@@ -73,10 +75,10 @@ static ssize_t bo_show(struct device *dev, struct device_attribute *attr,
 	spin_lock_irqsave(&bo_device.list_lock, flags);
 	list_for_each_entry(bo, bo_list, list) {
 		if ((active && (bo->status & HMM_BO_ALLOCED)) ||
-			(!active && !(bo->status & HMM_BO_ALLOCED))) {
+		    (!active && !(bo->status & HMM_BO_ALLOCED))) {
 			ret = scnprintf(buf + index1, PAGE_SIZE - index1,
-				"%c %d\n",
-				hmm_bo_type_string[bo->type], bo->pgnr);
+					"%c %d\n",
+					hmm_bo_type_string[bo->type], bo->pgnr);
 
 			total[bo->type] += bo->pgnr;
 			count[bo->type]++;
@@ -89,9 +91,10 @@ static ssize_t bo_show(struct device *dev, struct device_attribute *attr,
 	for (i = 0; i < HMM_BO_LAST; i++) {
 		if (count[i]) {
 			ret = scnprintf(buf + index1 + index2,
-				PAGE_SIZE - index1 - index2,
-				"%ld %c buffer objects: %ld KB\n",
-				count[i], hmm_bo_type_string[i], total[i] * 4);
+					PAGE_SIZE - index1 - index2,
+					"%ld %c buffer objects: %ld KB\n",
+					count[i], hmm_bo_type_string[i],
+					total[i] * 4);
 			if (ret > 0)
 				index2 += ret;
 		}
@@ -101,23 +104,21 @@ static ssize_t bo_show(struct device *dev, struct device_attribute *attr,
 	return index1 + index2 + 1;
 }
 
-static ssize_t active_bo_show(struct device *dev,
-		struct device_attribute *attr,
-		char *buf)
+static ssize_t active_bo_show(struct device *dev, struct device_attribute *attr,
+			      char *buf)
 {
 	return bo_show(dev, attr, buf, &bo_device.entire_bo_list, true);
 }
 
-static ssize_t free_bo_show(struct device *dev,
-		struct device_attribute *attr,
-		char *buf)
+static ssize_t free_bo_show(struct device *dev, struct device_attribute *attr,
+			    char *buf)
 {
 	return bo_show(dev, attr, buf, &bo_device.entire_bo_list, false);
 }
 
 static ssize_t reserved_pool_show(struct device *dev,
-		struct device_attribute *attr,
-		char *buf)
+				  struct device_attribute *attr,
+				  char *buf)
 {
 	ssize_t ret = 0;
 
@@ -129,7 +130,7 @@ static ssize_t reserved_pool_show(struct device *dev,
 
 	spin_lock_irqsave(&pinfo->list_lock, flags);
 	ret = scnprintf(buf, PAGE_SIZE, "%d out of %d pages available\n",
-					pinfo->index, pinfo->pgnr);
+			pinfo->index, pinfo->pgnr);
 	spin_unlock_irqrestore(&pinfo->list_lock, flags);
 
 	if (ret > 0)
@@ -139,8 +140,8 @@ static ssize_t reserved_pool_show(struct device *dev,
 };
 
 static ssize_t dynamic_pool_show(struct device *dev,
-		struct device_attribute *attr,
-		char *buf)
+				 struct device_attribute *attr,
+				 char *buf)
 {
 	ssize_t ret = 0;
 
@@ -152,7 +153,7 @@ static ssize_t dynamic_pool_show(struct device *dev,
 
 	spin_lock_irqsave(&pinfo->list_lock, flags);
 	ret = scnprintf(buf, PAGE_SIZE, "%d (max %d) pages available\n",
-					pinfo->pgnr, pinfo->pool_size);
+			pinfo->pgnr, pinfo->pool_size);
 	spin_unlock_irqrestore(&pinfo->list_lock, flags);
 
 	if (ret > 0)
@@ -200,7 +201,7 @@ int hmm_init(void)
 
 	if (!ret) {
 		ret = sysfs_create_group(&atomisp_dev->kobj,
-				atomisp_attribute_group);
+					 atomisp_attribute_group);
 		if (ret)
 			dev_err(atomisp_dev,
 				"%s Failed to create sysfs\n", __func__);
@@ -213,9 +214,7 @@ void hmm_cleanup(void)
 {
 	sysfs_remove_group(&atomisp_dev->kobj, atomisp_attribute_group);
 
-	/*
-	 * free dummy memory first
-	 */
+	/* free dummy memory first */
 	hmm_free(dummy_ptr);
 	dummy_ptr = 0;
 
@@ -224,36 +223,37 @@ void hmm_cleanup(void)
 }
 
 ia_css_ptr hmm_alloc(size_t bytes, enum hmm_bo_type type,
-		int from_highmem, void *userptr, bool cached)
+		     int from_highmem, void *userptr, bool cached)
 {
 	unsigned int pgnr;
 	struct hmm_buffer_object *bo;
 	int ret;
 
-	/* Check if we are initialized. In the ideal world we wouldn't need
-	   this but we can tackle it once the driver is a lot cleaner */
+	/*
+	 * Check if we are initialized. In the ideal world we wouldn't need
+	 * this but we can tackle it once the driver is a lot cleaner
+	 */
 
 	if (!hmm_initialized)
 		hmm_init();
-	/*Get page number from size*/
+	/* Get page number from size */
 	pgnr = size_to_pgnr_ceil(bytes);
 
-	/*Buffer object structure init*/
+	/* Buffer object structure init */
 	bo = hmm_bo_alloc(&bo_device, pgnr);
 	if (!bo) {
 		dev_err(atomisp_dev, "hmm_bo_create failed.\n");
 		goto create_bo_err;
 	}
 
-	/*Allocate pages for memory*/
+	/* Allocate pages for memory */
 	ret = hmm_bo_alloc_pages(bo, type, from_highmem, userptr, cached);
 	if (ret) {
-		dev_err(atomisp_dev,
-			    "hmm_bo_alloc_pages failed.\n");
+		dev_err(atomisp_dev, "hmm_bo_alloc_pages failed.\n");
 		goto alloc_page_err;
 	}
 
-	/*Combind the virtual address and pages togather*/
+	/* Combind the virtual address and pages togather */
 	ret = hmm_bo_bind(bo);
 	if (ret) {
 		dev_err(atomisp_dev, "hmm_bo_bind failed.\n");
@@ -282,8 +282,8 @@ void hmm_free(ia_css_ptr virt)
 
 	if (!bo) {
 		dev_err(atomisp_dev,
-			    "can not find buffer object start with "
-			    "address 0x%x\n", (unsigned int)virt);
+			"can not find buffer object start with address 0x%x\n",
+			(unsigned int)virt);
 		return;
 	}
 
@@ -298,29 +298,29 @@ static inline int hmm_check_bo(struct hmm_buffer_object *bo, unsigned int ptr)
 {
 	if (!bo) {
 		dev_err(atomisp_dev,
-			    "can not find buffer object contains "
-			    "address 0x%x\n", ptr);
+			"can not find buffer object contains address 0x%x\n",
+			ptr);
 		return -EINVAL;
 	}
 
 	if (!hmm_bo_page_allocated(bo)) {
 		dev_err(atomisp_dev,
-			    "buffer object has no page allocated.\n");
+			"buffer object has no page allocated.\n");
 		return -EINVAL;
 	}
 
 	if (!hmm_bo_allocated(bo)) {
 		dev_err(atomisp_dev,
-			    "buffer object has no virtual address"
-			    " space allocated.\n");
+			"buffer object has no virtual address space allocated.\n");
 		return -EINVAL;
 	}
 
 	return 0;
 }
 
-/*Read function in ISP memory management*/
-static int load_and_flush_by_kmap(ia_css_ptr virt, void *data, unsigned int bytes)
+/* Read function in ISP memory management */
+static int load_and_flush_by_kmap(ia_css_ptr virt, void *data,
+				  unsigned int bytes)
 {
 	struct hmm_buffer_object *bo;
 	unsigned int idx, offset, len;
@@ -362,7 +362,7 @@ static int load_and_flush_by_kmap(ia_css_ptr virt, void *data, unsigned int byte
 	return 0;
 }
 
-/*Read function in ISP memory management*/
+/* Read function in ISP memory management */
 static int load_and_flush(ia_css_ptr virt, void *data, unsigned int bytes)
 {
 	struct hmm_buffer_object *bo;
@@ -397,24 +397,24 @@ static int load_and_flush(ia_css_ptr virt, void *data, unsigned int bytes)
 	return 0;
 }
 
-/*Read function in ISP memory management*/
+/* Read function in ISP memory management */
 int hmm_load(ia_css_ptr virt, void *data, unsigned int bytes)
 {
 	if (!data) {
 		dev_err(atomisp_dev,
-			 "hmm_load NULL argument\n");
+			"hmm_load NULL argument\n");
 		return -EINVAL;
 	}
 	return load_and_flush(virt, data, bytes);
 }
 
-/*Flush hmm data from the data cache*/
+/* Flush hmm data from the data cache */
 int hmm_flush(ia_css_ptr virt, unsigned int bytes)
 {
 	return load_and_flush(virt, NULL, bytes);
 }
 
-/*Write function in ISP memory management*/
+/* Write function in ISP memory management */
 int hmm_store(ia_css_ptr virt, const void *data, unsigned int bytes)
 {
 	struct hmm_buffer_object *bo;
@@ -460,8 +460,8 @@ int hmm_store(ia_css_ptr virt, const void *data, unsigned int bytes)
 
 		if (!des) {
 			dev_err(atomisp_dev,
-				    "kmap buffer object page failed: "
-				    "pg_idx = %d\n", idx);
+				"kmap buffer object page failed: pg_idx = %d\n",
+				idx);
 			return -EINVAL;
 		}
 
@@ -496,7 +496,7 @@ int hmm_store(ia_css_ptr virt, const void *data, unsigned int bytes)
 	return 0;
 }
 
-/*memset function in ISP memory management*/
+/* memset function in ISP memory management */
 int hmm_set(ia_css_ptr virt, int c, unsigned int bytes)
 {
 	struct hmm_buffer_object *bo;
@@ -556,7 +556,7 @@ int hmm_set(ia_css_ptr virt, int c, unsigned int bytes)
 	return 0;
 }
 
-/*Virtual address to physical address convert*/
+/* Virtual address to physical address convert */
 phys_addr_t hmm_virt_to_phys(ia_css_ptr virt)
 {
 	unsigned int idx, offset;
@@ -591,7 +591,7 @@ int hmm_mmap(struct vm_area_struct *vma, ia_css_ptr virt)
 	return hmm_bo_mmap(vma, bo);
 }
 
-/*Map ISP virtual address into IA virtual address*/
+/* Map ISP virtual address into IA virtual address */
 void *hmm_vmap(ia_css_ptr virt, bool cached)
 {
 	struct hmm_buffer_object *bo;
@@ -600,8 +600,8 @@ void *hmm_vmap(ia_css_ptr virt, bool cached)
 	bo = hmm_bo_device_search_in_range(&bo_device, virt);
 	if (!bo) {
 		dev_err(atomisp_dev,
-			    "can not find buffer object contains address 0x%x\n",
-			    virt);
+			"can not find buffer object contains address 0x%x\n",
+			virt);
 		return NULL;
 	}
 
@@ -620,8 +620,8 @@ void hmm_flush_vmap(ia_css_ptr virt)
 	bo = hmm_bo_device_search_in_range(&bo_device, virt);
 	if (!bo) {
 		dev_warn(atomisp_dev,
-			    "can not find buffer object contains address 0x%x\n",
-			    virt);
+			 "can not find buffer object contains address 0x%x\n",
+			 virt);
 		return;
 	}
 
@@ -635,26 +635,25 @@ void hmm_vunmap(ia_css_ptr virt)
 	bo = hmm_bo_device_search_in_range(&bo_device, virt);
 	if (!bo) {
 		dev_warn(atomisp_dev,
-			"can not find buffer object contains address 0x%x\n",
-			virt);
+			 "can not find buffer object contains address 0x%x\n",
+			 virt);
 		return;
 	}
 
-	return hmm_bo_vunmap(bo);
+	hmm_bo_vunmap(bo);
 }
 
-int hmm_pool_register(unsigned int pool_size,
-			enum hmm_pool_type pool_type)
+int hmm_pool_register(unsigned int pool_size, enum hmm_pool_type pool_type)
 {
 	switch (pool_type) {
 	case HMM_POOL_TYPE_RESERVED:
 		reserved_pool.pops = &reserved_pops;
 		return reserved_pool.pops->pool_init(&reserved_pool.pool_info,
-							pool_size);
+						     pool_size);
 	case HMM_POOL_TYPE_DYNAMIC:
 		dynamic_pool.pops = &dynamic_pops;
 		return dynamic_pool.pops->pool_init(&dynamic_pool.pool_info,
-							pool_size);
+						    pool_size);
 	default:
 		dev_err(atomisp_dev, "invalid pool type.\n");
 		return -EINVAL;
@@ -703,10 +702,10 @@ ia_css_ptr hmm_host_vaddr_to_hrt_vaddr(const void *ptr)
 void hmm_show_mem_stat(const char *func, const int line)
 {
 	trace_printk("tol_cnt=%d usr_size=%d res_size=%d res_cnt=%d sys_size=%d  dyc_thr=%d dyc_size=%d.\n",
-			hmm_mem_stat.tol_cnt,
-			hmm_mem_stat.usr_size, hmm_mem_stat.res_size,
-			hmm_mem_stat.res_cnt, hmm_mem_stat.sys_size,
-			hmm_mem_stat.dyc_thr, hmm_mem_stat.dyc_size);
+		     hmm_mem_stat.tol_cnt,
+		     hmm_mem_stat.usr_size, hmm_mem_stat.res_size,
+		     hmm_mem_stat.res_cnt, hmm_mem_stat.sys_size,
+		     hmm_mem_stat.dyc_thr, hmm_mem_stat.dyc_size);
 }
 
 void hmm_init_mem_stat(int res_pgnr, int dyc_en, int dyc_pgnr)
diff --git a/drivers/staging/media/bcm2048/radio-bcm2048.c b/drivers/staging/media/bcm2048/radio-bcm2048.c
index 38f72d069e27..58adaea44eb5 100644
--- a/drivers/staging/media/bcm2048/radio-bcm2048.c
+++ b/drivers/staging/media/bcm2048/radio-bcm2048.c
@@ -48,7 +48,6 @@
 /* driver definitions */
 #define BCM2048_DRIVER_AUTHOR	"Eero Nurkkala <ext-eero.nurkkala@nokia.com>"
 #define BCM2048_DRIVER_NAME	BCM2048_NAME
-#define BCM2048_DRIVER_VERSION	KERNEL_VERSION(0, 0, 1)
 #define BCM2048_DRIVER_CARD	"Broadcom bcm2048 FM Radio Receiver"
 #define BCM2048_DRIVER_DESC	"I2C driver for BCM2048 FM Radio Receiver"
 
@@ -2565,7 +2564,7 @@ static const struct v4l2_ioctl_ops bcm2048_ioctl_ops = {
 /*
  * bcm2048_viddev_template - video device interface
  */
-static struct video_device bcm2048_viddev_template = {
+static const struct video_device bcm2048_viddev_template = {
 	.fops			= &bcm2048_fops,
 	.name			= BCM2048_DRIVER_NAME,
 	.release		= video_device_release_empty,
diff --git a/drivers/staging/media/cxd2099/cxd2099.c b/drivers/staging/media/cxd2099/cxd2099.c
index f28916ea69f1..3e30f4864e2b 100644
--- a/drivers/staging/media/cxd2099/cxd2099.c
+++ b/drivers/staging/media/cxd2099/cxd2099.c
@@ -33,8 +33,9 @@
 
 #include "cxd2099.h"
 
-/* comment this line to deactivate the cxd2099ar buffer mode */
-#define BUFFER_MODE 1
+static int buffermode;
+module_param(buffermode, int, 0444);
+MODULE_PARM_DESC(buffermode, "Enable use of the CXD2099AR buffer mode (default: disabled)");
 
 static int read_data(struct dvb_ca_en50221 *ca, int slot, u8 *ebuf, int ecount);
 
@@ -221,7 +222,6 @@ static int write_reg(struct cxd *ci, u8 reg, u8 val)
 	return write_regm(ci, reg, val, 0xff);
 }
 
-#ifdef BUFFER_MODE
 static int write_block(struct cxd *ci, u8 adr, u8 *data, u16 n)
 {
 	int status = 0;
@@ -248,7 +248,6 @@ static int write_block(struct cxd *ci, u8 adr, u8 *data, u16 n)
 	}
 	return status;
 }
-#endif
 
 static void set_mode(struct cxd *ci, int mode)
 {
@@ -642,8 +641,6 @@ static int read_data(struct dvb_ca_en50221 *ca, int slot, u8 *ebuf, int ecount)
 	return len;
 }
 
-#ifdef BUFFER_MODE
-
 static int write_data(struct dvb_ca_en50221 *ca, int slot, u8 *ebuf, int ecount)
 {
 	struct cxd *ci = ca->data;
@@ -658,7 +655,6 @@ static int write_data(struct dvb_ca_en50221 *ca, int slot, u8 *ebuf, int ecount)
 	mutex_unlock(&ci->lock);
 	return ecount;
 }
-#endif
 
 static struct dvb_ca_en50221 en_templ = {
 	.read_attribute_mem  = read_attribute_mem,
@@ -669,11 +665,8 @@ static struct dvb_ca_en50221 en_templ = {
 	.slot_shutdown       = slot_shutdown,
 	.slot_ts_enable      = slot_ts_enable,
 	.poll_slot_status    = poll_slot_status,
-#ifdef BUFFER_MODE
 	.read_data           = read_data,
 	.write_data          = write_data,
-#endif
-
 };
 
 struct dvb_ca_en50221 *cxd2099_attach(struct cxd2099_cfg *cfg,
@@ -703,6 +696,14 @@ struct dvb_ca_en50221 *cxd2099_attach(struct cxd2099_cfg *cfg,
 	ci->en.data = ci;
 	init(ci);
 	dev_info(&i2c->dev, "Attached CXD2099AR at %02x\n", ci->cfg.adr);
+
+	if (!buffermode) {
+		ci->en.read_data = NULL;
+		ci->en.write_data = NULL;
+	} else {
+		dev_info(&i2c->dev, "Using CXD2099AR buffer mode");
+	}
+
 	return &ci->en;
 }
 EXPORT_SYMBOL(cxd2099_attach);
diff --git a/drivers/staging/media/davinci_vpfe/vpfe_video.c b/drivers/staging/media/davinci_vpfe/vpfe_video.c
index 8b2117ee0f60..155e8c758e4b 100644
--- a/drivers/staging/media/davinci_vpfe/vpfe_video.c
+++ b/drivers/staging/media/davinci_vpfe/vpfe_video.c
@@ -1304,7 +1304,7 @@ static void vpfe_buf_cleanup(struct vb2_buffer *vb)
 		list_del_init(&buf->list);
 }
 
-static struct vb2_ops video_qops = {
+static const struct vb2_ops video_qops = {
 	.queue_setup		= vpfe_buffer_queue_setup,
 	.buf_init		= vpfe_buffer_init,
 	.buf_prepare		= vpfe_buffer_prepare,
diff --git a/drivers/staging/media/imx/Kconfig b/drivers/staging/media/imx/Kconfig
index 7eff50bcea39..2be921cd0d55 100644
--- a/drivers/staging/media/imx/Kconfig
+++ b/drivers/staging/media/imx/Kconfig
@@ -1,6 +1,8 @@
 config VIDEO_IMX_MEDIA
 	tristate "i.MX5/6 V4L2 media core driver"
 	depends on MEDIA_CONTROLLER && VIDEO_V4L2 && ARCH_MXC && IMX_IPUV3_CORE
+	depends on VIDEO_V4L2_SUBDEV_API
+	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_FWNODE
 	---help---
 	  Say yes here to enable support for video4linux media controller
@@ -12,7 +14,6 @@ menu "i.MX5/6 Media Sub devices"
 config VIDEO_IMX_CSI
 	tristate "i.MX5/6 Camera Sensor Interface driver"
 	depends on VIDEO_IMX_MEDIA && VIDEO_DEV && I2C
-	select VIDEOBUF2_DMA_CONTIG
 	default y
 	---help---
 	  A video4linux camera sensor interface driver for i.MX5/6.
diff --git a/drivers/staging/media/imx/imx-ic-prpencvf.c b/drivers/staging/media/imx/imx-ic-prpencvf.c
index ed363fe3b3d0..0790b3d9e255 100644
--- a/drivers/staging/media/imx/imx-ic-prpencvf.c
+++ b/drivers/staging/media/imx/imx-ic-prpencvf.c
@@ -134,19 +134,19 @@ static inline struct prp_priv *sd_to_priv(struct v4l2_subdev *sd)
 
 static void prp_put_ipu_resources(struct prp_priv *priv)
 {
-	if (!IS_ERR_OR_NULL(priv->ic))
+	if (priv->ic)
 		ipu_ic_put(priv->ic);
 	priv->ic = NULL;
 
-	if (!IS_ERR_OR_NULL(priv->out_ch))
+	if (priv->out_ch)
 		ipu_idmac_put(priv->out_ch);
 	priv->out_ch = NULL;
 
-	if (!IS_ERR_OR_NULL(priv->rot_in_ch))
+	if (priv->rot_in_ch)
 		ipu_idmac_put(priv->rot_in_ch);
 	priv->rot_in_ch = NULL;
 
-	if (!IS_ERR_OR_NULL(priv->rot_out_ch))
+	if (priv->rot_out_ch)
 		ipu_idmac_put(priv->rot_out_ch);
 	priv->rot_out_ch = NULL;
 }
@@ -154,43 +154,46 @@ static void prp_put_ipu_resources(struct prp_priv *priv)
 static int prp_get_ipu_resources(struct prp_priv *priv)
 {
 	struct imx_ic_priv *ic_priv = priv->ic_priv;
+	struct ipu_ic *ic;
+	struct ipuv3_channel *out_ch, *rot_in_ch, *rot_out_ch;
 	int ret, task = ic_priv->task_id;
 
 	priv->ipu = priv->md->ipu[ic_priv->ipu_id];
 
-	priv->ic = ipu_ic_get(priv->ipu, task);
-	if (IS_ERR(priv->ic)) {
+	ic = ipu_ic_get(priv->ipu, task);
+	if (IS_ERR(ic)) {
 		v4l2_err(&ic_priv->sd, "failed to get IC\n");
-		ret = PTR_ERR(priv->ic);
+		ret = PTR_ERR(ic);
 		goto out;
 	}
+	priv->ic = ic;
 
-	priv->out_ch = ipu_idmac_get(priv->ipu,
-				     prp_channel[task].out_ch);
-	if (IS_ERR(priv->out_ch)) {
+	out_ch = ipu_idmac_get(priv->ipu, prp_channel[task].out_ch);
+	if (IS_ERR(out_ch)) {
 		v4l2_err(&ic_priv->sd, "could not get IDMAC channel %u\n",
 			 prp_channel[task].out_ch);
-		ret = PTR_ERR(priv->out_ch);
+		ret = PTR_ERR(out_ch);
 		goto out;
 	}
+	priv->out_ch = out_ch;
 
-	priv->rot_in_ch = ipu_idmac_get(priv->ipu,
-					prp_channel[task].rot_in_ch);
-	if (IS_ERR(priv->rot_in_ch)) {
+	rot_in_ch = ipu_idmac_get(priv->ipu, prp_channel[task].rot_in_ch);
+	if (IS_ERR(rot_in_ch)) {
 		v4l2_err(&ic_priv->sd, "could not get IDMAC channel %u\n",
 			 prp_channel[task].rot_in_ch);
-		ret = PTR_ERR(priv->rot_in_ch);
+		ret = PTR_ERR(rot_in_ch);
 		goto out;
 	}
+	priv->rot_in_ch = rot_in_ch;
 
-	priv->rot_out_ch = ipu_idmac_get(priv->ipu,
-					 prp_channel[task].rot_out_ch);
-	if (IS_ERR(priv->rot_out_ch)) {
+	rot_out_ch = ipu_idmac_get(priv->ipu, prp_channel[task].rot_out_ch);
+	if (IS_ERR(rot_out_ch)) {
 		v4l2_err(&ic_priv->sd, "could not get IDMAC channel %u\n",
 			 prp_channel[task].rot_out_ch);
-		ret = PTR_ERR(priv->rot_out_ch);
+		ret = PTR_ERR(rot_out_ch);
 		goto out;
 	}
+	priv->rot_out_ch = rot_out_ch;
 
 	return 0;
 out:
@@ -374,6 +377,17 @@ static int prp_setup_channel(struct prp_priv *priv,
 	image.phys0 = addr0;
 	image.phys1 = addr1;
 
+	if (channel == priv->out_ch || channel == priv->rot_out_ch) {
+		switch (image.pix.pixelformat) {
+		case V4L2_PIX_FMT_YUV420:
+		case V4L2_PIX_FMT_YVU420:
+		case V4L2_PIX_FMT_NV12:
+			/* Skip writing U and V components to odd rows */
+			ipu_cpmem_skip_odd_chroma_rows(channel);
+			break;
+		}
+	}
+
 	ret = ipu_cpmem_set_image(channel, &image);
 	if (ret)
 		return ret;
@@ -1278,9 +1292,8 @@ static int prp_init(struct imx_ic_priv *ic_priv)
 	priv->ic_priv = ic_priv;
 
 	spin_lock_init(&priv->irqlock);
-	init_timer(&priv->eof_timeout_timer);
-	priv->eof_timeout_timer.data = (unsigned long)priv;
-	priv->eof_timeout_timer.function = prp_eof_timeout;
+	setup_timer(&priv->eof_timeout_timer, prp_eof_timeout,
+		    (unsigned long)priv);
 
 	priv->vdev = imx_media_capture_device_init(&ic_priv->sd,
 						   PRPENCVF_SRC_PAD);
diff --git a/drivers/staging/media/imx/imx-media-capture.c b/drivers/staging/media/imx/imx-media-capture.c
index ddab4c249da2..ea145bafb880 100644
--- a/drivers/staging/media/imx/imx-media-capture.c
+++ b/drivers/staging/media/imx/imx-media-capture.c
@@ -62,7 +62,7 @@ struct capture_priv {
 /* In bytes, per queue */
 #define VID_MEM_LIMIT	SZ_64M
 
-static struct vb2_ops capture_qops;
+static const struct vb2_ops capture_qops;
 
 /*
  * Video ioctls follow
@@ -503,7 +503,7 @@ static void capture_stop_streaming(struct vb2_queue *vq)
 	spin_unlock_irqrestore(&priv->q_lock, flags);
 }
 
-static struct vb2_ops capture_qops = {
+static const struct vb2_ops capture_qops = {
 	.queue_setup	 = capture_queue_setup,
 	.buf_init        = capture_buf_init,
 	.buf_prepare	 = capture_buf_prepare,
diff --git a/drivers/staging/media/imx/imx-media-csi.c b/drivers/staging/media/imx/imx-media-csi.c
index a2d26693912e..6d856118c223 100644
--- a/drivers/staging/media/imx/imx-media-csi.c
+++ b/drivers/staging/media/imx/imx-media-csi.c
@@ -122,11 +122,11 @@ static inline struct csi_priv *sd_to_dev(struct v4l2_subdev *sdev)
 
 static void csi_idmac_put_ipu_resources(struct csi_priv *priv)
 {
-	if (!IS_ERR_OR_NULL(priv->idmac_ch))
+	if (priv->idmac_ch)
 		ipu_idmac_put(priv->idmac_ch);
 	priv->idmac_ch = NULL;
 
-	if (!IS_ERR_OR_NULL(priv->smfc))
+	if (priv->smfc)
 		ipu_smfc_put(priv->smfc);
 	priv->smfc = NULL;
 }
@@ -134,23 +134,27 @@ static void csi_idmac_put_ipu_resources(struct csi_priv *priv)
 static int csi_idmac_get_ipu_resources(struct csi_priv *priv)
 {
 	int ch_num, ret;
+	struct ipu_smfc *smfc;
+	struct ipuv3_channel *idmac_ch;
 
 	ch_num = IPUV3_CHANNEL_CSI0 + priv->smfc_id;
 
-	priv->smfc = ipu_smfc_get(priv->ipu, ch_num);
-	if (IS_ERR(priv->smfc)) {
+	smfc = ipu_smfc_get(priv->ipu, ch_num);
+	if (IS_ERR(smfc)) {
 		v4l2_err(&priv->sd, "failed to get SMFC\n");
-		ret = PTR_ERR(priv->smfc);
+		ret = PTR_ERR(smfc);
 		goto out;
 	}
+	priv->smfc = smfc;
 
-	priv->idmac_ch = ipu_idmac_get(priv->ipu, ch_num);
-	if (IS_ERR(priv->idmac_ch)) {
+	idmac_ch = ipu_idmac_get(priv->ipu, ch_num);
+	if (IS_ERR(idmac_ch)) {
 		v4l2_err(&priv->sd, "could not get IDMAC channel %u\n",
 			 ch_num);
-		ret = PTR_ERR(priv->idmac_ch);
+		ret = PTR_ERR(idmac_ch);
 		goto out;
 	}
+	priv->idmac_ch = idmac_ch;
 
 	return 0;
 out:
@@ -357,6 +361,8 @@ static int csi_idmac_setup_channel(struct csi_priv *priv)
 		passthrough = (sensor_ep->bus_type != V4L2_MBUS_CSI2 &&
 			       sensor_ep->bus.parallel.bus_width >= 16);
 		passthrough_bits = 16;
+		/* Skip writing U and V components to odd rows */
+		ipu_cpmem_skip_odd_chroma_rows(priv->idmac_ch);
 		break;
 	case V4L2_PIX_FMT_YUYV:
 	case V4L2_PIX_FMT_UYVY:
@@ -1583,6 +1589,7 @@ static int csi_unsubscribe_event(struct v4l2_subdev *sd, struct v4l2_fh *fh,
 static int csi_registered(struct v4l2_subdev *sd)
 {
 	struct csi_priv *priv = v4l2_get_subdevdata(sd);
+	struct ipu_csi *csi;
 	int i, ret;
 	u32 code;
 
@@ -1590,11 +1597,12 @@ static int csi_registered(struct v4l2_subdev *sd)
 	priv->md = dev_get_drvdata(sd->v4l2_dev->dev);
 
 	/* get handle to IPU CSI */
-	priv->csi = ipu_csi_get(priv->ipu, priv->csi_id);
-	if (IS_ERR(priv->csi)) {
+	csi = ipu_csi_get(priv->ipu, priv->csi_id);
+	if (IS_ERR(csi)) {
 		v4l2_err(&priv->sd, "failed to get CSI%d\n", priv->csi_id);
-		return PTR_ERR(priv->csi);
+		return PTR_ERR(csi);
 	}
+	priv->csi = csi;
 
 	for (i = 0; i < CSI_NUM_PADS; i++) {
 		priv->pad[i].flags = (i == CSI_SINK_PAD) ?
@@ -1663,7 +1671,7 @@ static void csi_unregistered(struct v4l2_subdev *sd)
 	if (priv->fim)
 		imx_media_fim_free(priv->fim);
 
-	if (!IS_ERR_OR_NULL(priv->csi))
+	if (priv->csi)
 		ipu_csi_put(priv->csi);
 }
 
@@ -1731,9 +1739,8 @@ static int imx_csi_probe(struct platform_device *pdev)
 	priv->csi_id = pdata->csi;
 	priv->smfc_id = (priv->csi_id == 0) ? 0 : 2;
 
-	init_timer(&priv->eof_timeout_timer);
-	priv->eof_timeout_timer.data = (unsigned long)priv;
-	priv->eof_timeout_timer.function = csi_idmac_eof_timeout;
+	setup_timer(&priv->eof_timeout_timer, csi_idmac_eof_timeout,
+		    (unsigned long)priv);
 	spin_lock_init(&priv->irqlock);
 
 	v4l2_subdev_init(&priv->sd, &csi_subdev_ops);
diff --git a/drivers/staging/media/imx/imx-media-dev.c b/drivers/staging/media/imx/imx-media-dev.c
index 48cbc7716758..d96f4512224f 100644
--- a/drivers/staging/media/imx/imx-media-dev.c
+++ b/drivers/staging/media/imx/imx-media-dev.c
@@ -87,11 +87,11 @@ imx_media_add_async_subdev(struct imx_media_dev *imxmd,
 	if (pdev)
 		devname = dev_name(&pdev->dev);
 
-	/* return NULL if this subdev already added */
+	/* return -EEXIST if this subdev already added */
 	if (imx_media_find_async_subdev(imxmd, np, devname)) {
 		dev_dbg(imxmd->md.dev, "%s: already added %s\n",
 			__func__, np ? np->name : devname);
-		imxsd = NULL;
+		imxsd = ERR_PTR(-EEXIST);
 		goto out;
 	}
 
diff --git a/drivers/staging/media/imx/imx-media-of.c b/drivers/staging/media/imx/imx-media-of.c
index b026fe66467c..12df09f52490 100644
--- a/drivers/staging/media/imx/imx-media-of.c
+++ b/drivers/staging/media/imx/imx-media-of.c
@@ -100,9 +100,9 @@ static void of_get_remote_pad(struct device_node *epnode,
 	}
 }
 
-static struct imx_media_subdev *
+static int
 of_parse_subdev(struct imx_media_dev *imxmd, struct device_node *sd_np,
-		bool is_csi_port)
+		bool is_csi_port, struct imx_media_subdev **subdev)
 {
 	struct imx_media_subdev *imxsd;
 	int i, num_pads, ret;
@@ -110,13 +110,25 @@ of_parse_subdev(struct imx_media_dev *imxmd, struct device_node *sd_np,
 	if (!of_device_is_available(sd_np)) {
 		dev_dbg(imxmd->md.dev, "%s: %s not enabled\n", __func__,
 			sd_np->name);
-		return NULL;
+		*subdev = NULL;
+		/* unavailable is not an error */
+		return 0;
 	}
 
 	/* register this subdev with async notifier */
 	imxsd = imx_media_add_async_subdev(imxmd, sd_np, NULL);
-	if (IS_ERR_OR_NULL(imxsd))
-		return imxsd;
+	ret = PTR_ERR_OR_ZERO(imxsd);
+	if (ret) {
+		if (ret == -EEXIST) {
+			/* already added, everything is fine */
+			*subdev = NULL;
+			return 0;
+		}
+
+		/* other error, can't continue */
+		return ret;
+	}
+	*subdev = imxsd;
 
 	if (is_csi_port) {
 		/*
@@ -137,10 +149,11 @@ of_parse_subdev(struct imx_media_dev *imxmd, struct device_node *sd_np,
 	} else {
 		num_pads = of_get_port_count(sd_np);
 		if (num_pads != 1) {
+			/* confused, but no reason to give up here */
 			dev_warn(imxmd->md.dev,
 				 "%s: unknown device %s with %d ports\n",
 				 __func__, sd_np->name, num_pads);
-			return NULL;
+			return 0;
 		}
 
 		/*
@@ -151,7 +164,7 @@ of_parse_subdev(struct imx_media_dev *imxmd, struct device_node *sd_np,
 	}
 
 	if (imxsd->num_sink_pads >= num_pads)
-		return ERR_PTR(-EINVAL);
+		return -EINVAL;
 
 	imxsd->num_src_pads = num_pads - imxsd->num_sink_pads;
 
@@ -191,20 +204,15 @@ of_parse_subdev(struct imx_media_dev *imxmd, struct device_node *sd_np,
 
 			ret = of_add_pad_link(imxmd, pad, sd_np, remote_np,
 					      i, remote_pad);
-			if (ret) {
-				imxsd = ERR_PTR(ret);
+			if (ret)
 				break;
-			}
 
 			if (i < imxsd->num_sink_pads) {
 				/* follow sink endpoints upstream */
-				remote_imxsd = of_parse_subdev(imxmd,
-							       remote_np,
-							       false);
-				if (IS_ERR(remote_imxsd)) {
-					imxsd = remote_imxsd;
+				ret = of_parse_subdev(imxmd, remote_np,
+						      false, &remote_imxsd);
+				if (ret)
 					break;
-				}
 			}
 
 			of_node_put(remote_np);
@@ -212,14 +220,14 @@ of_parse_subdev(struct imx_media_dev *imxmd, struct device_node *sd_np,
 
 		if (port != sd_np)
 			of_node_put(port);
-		if (IS_ERR(imxsd)) {
+		if (ret) {
 			of_node_put(remote_np);
 			of_node_put(epnode);
 			break;
 		}
 	}
 
-	return imxsd;
+	return ret;
 }
 
 int imx_media_of_parse(struct imx_media_dev *imxmd,
@@ -236,11 +244,9 @@ int imx_media_of_parse(struct imx_media_dev *imxmd,
 		if (!csi_np)
 			break;
 
-		lcsi = of_parse_subdev(imxmd, csi_np, true);
-		if (IS_ERR(lcsi)) {
-			ret = PTR_ERR(lcsi);
+		ret = of_parse_subdev(imxmd, csi_np, true, &lcsi);
+		if (ret)
 			goto err_put;
-		}
 
 		ret = of_property_read_u32(csi_np, "reg", &csi_id);
 		if (ret) {
diff --git a/drivers/staging/media/imx/imx-media-vdic.c b/drivers/staging/media/imx/imx-media-vdic.c
index 7eabdc4aa79f..433474d58e3e 100644
--- a/drivers/staging/media/imx/imx-media-vdic.c
+++ b/drivers/staging/media/imx/imx-media-vdic.c
@@ -126,15 +126,15 @@ struct vdic_priv {
 
 static void vdic_put_ipu_resources(struct vdic_priv *priv)
 {
-	if (!IS_ERR_OR_NULL(priv->vdi_in_ch_p))
+	if (priv->vdi_in_ch_p)
 		ipu_idmac_put(priv->vdi_in_ch_p);
 	priv->vdi_in_ch_p = NULL;
 
-	if (!IS_ERR_OR_NULL(priv->vdi_in_ch))
+	if (priv->vdi_in_ch)
 		ipu_idmac_put(priv->vdi_in_ch);
 	priv->vdi_in_ch = NULL;
 
-	if (!IS_ERR_OR_NULL(priv->vdi_in_ch_n))
+	if (priv->vdi_in_ch_n)
 		ipu_idmac_put(priv->vdi_in_ch_n);
 	priv->vdi_in_ch_n = NULL;
 
@@ -146,40 +146,43 @@ static void vdic_put_ipu_resources(struct vdic_priv *priv)
 static int vdic_get_ipu_resources(struct vdic_priv *priv)
 {
 	int ret, err_chan;
+	struct ipuv3_channel *ch;
+	struct ipu_vdi *vdi;
 
 	priv->ipu = priv->md->ipu[priv->ipu_id];
 
-	priv->vdi = ipu_vdi_get(priv->ipu);
-	if (IS_ERR(priv->vdi)) {
+	vdi = ipu_vdi_get(priv->ipu);
+	if (IS_ERR(vdi)) {
 		v4l2_err(&priv->sd, "failed to get VDIC\n");
-		ret = PTR_ERR(priv->vdi);
+		ret = PTR_ERR(vdi);
 		goto out;
 	}
+	priv->vdi = vdi;
 
 	if (!priv->csi_direct) {
-		priv->vdi_in_ch_p = ipu_idmac_get(priv->ipu,
-						  IPUV3_CHANNEL_MEM_VDI_PREV);
-		if (IS_ERR(priv->vdi_in_ch_p)) {
+		ch = ipu_idmac_get(priv->ipu, IPUV3_CHANNEL_MEM_VDI_PREV);
+		if (IS_ERR(ch)) {
 			err_chan = IPUV3_CHANNEL_MEM_VDI_PREV;
-			ret = PTR_ERR(priv->vdi_in_ch_p);
+			ret = PTR_ERR(ch);
 			goto out_err_chan;
 		}
+		priv->vdi_in_ch_p = ch;
 
-		priv->vdi_in_ch = ipu_idmac_get(priv->ipu,
-						IPUV3_CHANNEL_MEM_VDI_CUR);
-		if (IS_ERR(priv->vdi_in_ch)) {
+		ch = ipu_idmac_get(priv->ipu, IPUV3_CHANNEL_MEM_VDI_CUR);
+		if (IS_ERR(ch)) {
 			err_chan = IPUV3_CHANNEL_MEM_VDI_CUR;
-			ret = PTR_ERR(priv->vdi_in_ch);
+			ret = PTR_ERR(ch);
 			goto out_err_chan;
 		}
+		priv->vdi_in_ch = ch;
 
-		priv->vdi_in_ch_n = ipu_idmac_get(priv->ipu,
-						  IPUV3_CHANNEL_MEM_VDI_NEXT);
+		ch = ipu_idmac_get(priv->ipu, IPUV3_CHANNEL_MEM_VDI_NEXT);
 		if (IS_ERR(priv->vdi_in_ch_n)) {
 			err_chan = IPUV3_CHANNEL_MEM_VDI_NEXT;
-			ret = PTR_ERR(priv->vdi_in_ch_n);
+			ret = PTR_ERR(ch);
 			goto out_err_chan;
 		}
+		priv->vdi_in_ch_n = ch;
 	}
 
 	return 0;
diff --git a/drivers/staging/media/lirc/lirc_zilog.c b/drivers/staging/media/lirc/lirc_zilog.c
index 015e41bd036e..71af13bd0ebd 100644
--- a/drivers/staging/media/lirc/lirc_zilog.c
+++ b/drivers/staging/media/lirc/lirc_zilog.c
@@ -288,7 +288,7 @@ static void release_ir_tx(struct kref *ref)
 	struct IR_tx *tx = container_of(ref, struct IR_tx, ref);
 	struct IR *ir = tx->ir;
 
-	ir->l.features &= ~LIRC_CAN_SEND_PULSE;
+	ir->l.features &= ~LIRC_CAN_SEND_LIRCCODE;
 	/* Don't put_ir_device(tx->ir) here, so our lock doesn't get freed */
 	ir->tx = NULL;
 	kfree(tx);
@@ -1249,7 +1249,7 @@ static long ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
 		break;
 	case LIRC_GET_REC_MODE:
 		if (!(features & LIRC_CAN_REC_MASK))
-			return -ENOSYS;
+			return -ENOTTY;
 
 		result = put_user(LIRC_REC2MODE
 				  (features & LIRC_CAN_REC_MASK),
@@ -1257,24 +1257,24 @@ static long ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
 		break;
 	case LIRC_SET_REC_MODE:
 		if (!(features & LIRC_CAN_REC_MASK))
-			return -ENOSYS;
+			return -ENOTTY;
 
 		result = get_user(mode, uptr);
 		if (!result && !(LIRC_MODE2REC(mode) & features))
-			result = -EINVAL;
+			result = -ENOTTY;
 		break;
 	case LIRC_GET_SEND_MODE:
 		if (!(features & LIRC_CAN_SEND_MASK))
-			return -ENOSYS;
+			return -ENOTTY;
 
-		result = put_user(LIRC_MODE_PULSE, uptr);
+		result = put_user(LIRC_MODE_LIRCCODE, uptr);
 		break;
 	case LIRC_SET_SEND_MODE:
 		if (!(features & LIRC_CAN_SEND_MASK))
-			return -ENOSYS;
+			return -ENOTTY;
 
 		result = get_user(mode, uptr);
-		if (!result && mode != LIRC_MODE_PULSE)
+		if (!result && mode != LIRC_MODE_LIRCCODE)
 			return -EINVAL;
 		break;
 	default:
@@ -1512,7 +1512,7 @@ static int ir_probe(struct i2c_client *client, const struct i2c_device_id *id)
 		kref_init(&tx->ref);
 		ir->tx = tx;
 
-		ir->l.features |= LIRC_CAN_SEND_PULSE;
+		ir->l.features |= LIRC_CAN_SEND_LIRCCODE;
 		mutex_init(&tx->client_lock);
 		tx->c = client;
 		tx->need_boot = 1;
diff --git a/drivers/staging/media/omap4iss/iss_video.c b/drivers/staging/media/omap4iss/iss_video.c
index 0bac58241a22..9e2f0421a01e 100644
--- a/drivers/staging/media/omap4iss/iss_video.c
+++ b/drivers/staging/media/omap4iss/iss_video.c
@@ -1199,7 +1199,7 @@ static int iss_video_mmap(struct file *file, struct vm_area_struct *vma)
 	return vb2_mmap(&vfh->queue, vma);
 }
 
-static struct v4l2_file_operations iss_video_fops = {
+static const struct v4l2_file_operations iss_video_fops = {
 	.owner = THIS_MODULE,
 	.unlocked_ioctl = video_ioctl2,
 	.open = iss_video_open,
diff --git a/drivers/staging/most/hdm-dim2/dim2_hdm.c b/drivers/staging/most/hdm-dim2/dim2_hdm.c
index 4607d03c577b..df7021c522b3 100644
--- a/drivers/staging/most/hdm-dim2/dim2_hdm.c
+++ b/drivers/staging/most/hdm-dim2/dim2_hdm.c
@@ -761,8 +761,8 @@ static int dim2_probe(struct platform_device *pdev)
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
-		dev_err(&pdev->dev, "failed to get ahb0_int irq\n");
-		return -ENODEV;
+		dev_err(&pdev->dev, "failed to get ahb0_int irq: %d\n", irq);
+		return irq;
 	}
 
 	ret = devm_request_irq(&pdev->dev, irq, dim2_ahb_isr, 0,
@@ -774,8 +774,8 @@ static int dim2_probe(struct platform_device *pdev)
 
 	irq = platform_get_irq(pdev, 1);
 	if (irq < 0) {
-		dev_err(&pdev->dev, "failed to get mlb_int irq\n");
-		return -ENODEV;
+		dev_err(&pdev->dev, "failed to get mlb_int irq: %d\n", irq);
+		return irq;
 	}
 
 	ret = devm_request_irq(&pdev->dev, irq, dim2_mlb_isr, 0,
@@ -894,7 +894,7 @@ static int dim2_remove(struct platform_device *pdev)
 	return 0;
 }
 
-static struct platform_device_id dim2_id[] = {
+static const struct platform_device_id dim2_id[] = {
 	{ "medialb_dim2" },
 	{ }, /* Terminating entry */
 };
diff --git a/drivers/staging/most/hdm-usb/hdm_usb.c b/drivers/staging/most/hdm-usb/hdm_usb.c
index d0f68cb3c173..85775da293fb 100644
--- a/drivers/staging/most/hdm-usb/hdm_usb.c
+++ b/drivers/staging/most/hdm-usb/hdm_usb.c
@@ -832,7 +832,7 @@ static const struct file_operations hdm_usb_fops = {
 /**
  * usb_device_id - ID table for HCD device probing
  */
-static struct usb_device_id usbid[] = {
+static const struct usb_device_id usbid[] = {
 	{ USB_DEVICE(USB_VENDOR_ID_SMSC, USB_DEV_ID_BRDG), },
 	{ USB_DEVICE(USB_VENDOR_ID_SMSC, USB_DEV_ID_OS81118), },
 	{ USB_DEVICE(USB_VENDOR_ID_SMSC, USB_DEV_ID_OS81119), },
@@ -1301,25 +1301,7 @@ static struct usb_driver hdm_usb = {
 	.disconnect = hdm_disconnect,
 };
 
-static int __init hdm_usb_init(void)
-{
-	pr_info("hdm_usb_init()\n");
-	if (usb_register(&hdm_usb)) {
-		pr_err("could not register hdm_usb driver\n");
-		return -EIO;
-	}
-
-	return 0;
-}
-
-static void __exit hdm_usb_exit(void)
-{
-	pr_info("hdm_usb_exit()\n");
-	usb_deregister(&hdm_usb);
-}
-
-module_init(hdm_usb_init);
-module_exit(hdm_usb_exit);
+module_usb_driver(hdm_usb);
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Christian Gromm <christian.gromm@microchip.com>");
 MODULE_DESCRIPTION("HDM_4_USB");
diff --git a/drivers/staging/mt29f_spinand/mt29f_spinand.c b/drivers/staging/mt29f_spinand/mt29f_spinand.c
index a4e3ae8f0c85..13eaf16ecd16 100644
--- a/drivers/staging/mt29f_spinand/mt29f_spinand.c
+++ b/drivers/staging/mt29f_spinand/mt29f_spinand.c
@@ -18,7 +18,7 @@
 #include <linux/delay.h>
 #include <linux/mtd/mtd.h>
 #include <linux/mtd/partitions.h>
-#include <linux/mtd/nand.h>
+#include <linux/mtd/rawnand.h>
 #include <linux/spi/spi.h>
 
 #include "mt29f_spinand.h"
diff --git a/drivers/staging/nvec/nvec.c b/drivers/staging/nvec/nvec.c
index c1feccf8d94a..4ff8f47385da 100644
--- a/drivers/staging/nvec/nvec.c
+++ b/drivers/staging/nvec/nvec.c
@@ -831,7 +831,7 @@ static int tegra_nvec_probe(struct platform_device *pdev)
 		return -ENODEV;
 	}
 
-	nvec->rst = devm_reset_control_get(&pdev->dev, "i2c");
+	nvec->rst = devm_reset_control_get_exclusive(&pdev->dev, "i2c");
 	if (IS_ERR(nvec->rst)) {
 		dev_err(nvec->dev, "failed to get controller reset\n");
 		return PTR_ERR(nvec->rst);
diff --git a/drivers/staging/octeon/ethernet-rx.c b/drivers/staging/octeon/ethernet-rx.c
index 72baedefa0f1..1a44291318ee 100644
--- a/drivers/staging/octeon/ethernet-rx.c
+++ b/drivers/staging/octeon/ethernet-rx.c
@@ -149,6 +149,46 @@ static inline int cvm_oct_check_rcv_error(cvmx_wqe_t *work)
 	return 0;
 }
 
+static void copy_segments_to_skb(cvmx_wqe_t *work, struct sk_buff *skb)
+{
+	int segments = work->word2.s.bufs;
+	union cvmx_buf_ptr segment_ptr = work->packet_ptr;
+	int len = work->word1.len;
+	int segment_size;
+
+	while (segments--) {
+		union cvmx_buf_ptr next_ptr;
+
+		next_ptr = *(union cvmx_buf_ptr *)
+			cvmx_phys_to_ptr(segment_ptr.s.addr - 8);
+
+		/*
+		 * Octeon Errata PKI-100: The segment size is wrong.
+		 *
+		 * Until it is fixed, calculate the segment size based on
+		 * the packet pool buffer size.
+		 * When it is fixed, the following line should be replaced
+		 * with this one:
+		 * int segment_size = segment_ptr.s.size;
+		 */
+		segment_size =
+			CVMX_FPA_PACKET_POOL_SIZE -
+			(segment_ptr.s.addr -
+			 (((segment_ptr.s.addr >> 7) -
+			   segment_ptr.s.back) << 7));
+
+		/* Don't copy more than what is left in the packet */
+		if (segment_size > len)
+			segment_size = len;
+
+		/* Copy the data into the packet */
+		skb_put_data(skb, cvmx_phys_to_ptr(segment_ptr.s.addr),
+			     segment_size);
+		len -= segment_size;
+		segment_ptr = next_ptr;
+	}
+}
+
 static int cvm_oct_poll(struct oct_rx_group *rx_group, int budget)
 {
 	const int	coreid = cvmx_get_core_num();
@@ -290,44 +330,7 @@ static int cvm_oct_poll(struct oct_rx_group *rx_group, int budget)
 				skb_put_data(skb, ptr, work->word1.len);
 				/* No packet buffers to free */
 			} else {
-				int segments = work->word2.s.bufs;
-				union cvmx_buf_ptr segment_ptr =
-				    work->packet_ptr;
-				int len = work->word1.len;
-
-				while (segments--) {
-					union cvmx_buf_ptr next_ptr =
-					    *(union cvmx_buf_ptr *)
-					      cvmx_phys_to_ptr(
-					      segment_ptr.s.addr - 8);
-
-			/*
-			 * Octeon Errata PKI-100: The segment size is
-			 * wrong. Until it is fixed, calculate the
-			 * segment size based on the packet pool
-			 * buffer size. When it is fixed, the
-			 * following line should be replaced with this
-			 * one: int segment_size =
-			 * segment_ptr.s.size;
-			 */
-					int segment_size =
-					    CVMX_FPA_PACKET_POOL_SIZE -
-					    (segment_ptr.s.addr -
-					     (((segment_ptr.s.addr >> 7) -
-					       segment_ptr.s.back) << 7));
-					/*
-					 * Don't copy more than what
-					 * is left in the packet.
-					 */
-					if (segment_size > len)
-						segment_size = len;
-					/* Copy the data into the packet */
-					skb_put_data(skb,
-						     cvmx_phys_to_ptr(segment_ptr.s.addr),
-						     segment_size);
-					len -= segment_size;
-					segment_ptr = next_ptr;
-				}
+				copy_segments_to_skb(work, skb);
 			}
 			packet_not_copied = 0;
 		}
diff --git a/drivers/staging/olpc_dcon/olpc_dcon.c b/drivers/staging/olpc_dcon/olpc_dcon.c
index f7f3a780ec10..82bffd911435 100644
--- a/drivers/staging/olpc_dcon/olpc_dcon.c
+++ b/drivers/staging/olpc_dcon/olpc_dcon.c
@@ -456,8 +456,6 @@ static ssize_t dcon_freeze_store(struct device *dev,
 	if (ret)
 		return ret;
 
-	pr_info("dcon_freeze_store: %lu\n", output);
-
 	switch (output) {
 	case 0:
 		dcon_set_source(dcon, DCON_SOURCE_CPU);
diff --git a/drivers/staging/pi433/Documentation/devicetree/pi433-overlay.dts b/drivers/staging/pi433/Documentation/devicetree/pi433-overlay.dts
new file mode 100644
index 000000000000..004b5027a934
--- /dev/null
+++ b/drivers/staging/pi433/Documentation/devicetree/pi433-overlay.dts
@@ -0,0 +1,53 @@
+// Definitions for Pi433
+/dts-v1/;
+/plugin/;
+
+/ {
+        compatible = "bcm,bcm2835", "bcm,bcm2708", "bcm,bcm2709";
+
+        fragment@0 {
+                target = <&spi0>;
+                __overlay__ {
+                        status = "okay";
+
+                        spidev@0{
+                                status = "disabled";
+                        };
+
+                        spidev@1{
+                                status = "disabled";
+                        };
+                };
+        };
+
+	fragment@1 {
+		target = <&gpio>;
+		__overlay__ {
+			pi433_pins: pi433_pins {
+				brcm,pins = <7 25 24>;
+				brcm,function = <0 0 0>; // in in in
+			};
+		};
+	};
+
+	fragment@2 {
+		target = <&spi0>;
+		__overlay__ {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "okay";
+
+			pi433: pi433@0 {
+				compatible = "Smarthome-Wolf,pi433";
+				reg = <0>;
+				spi-max-frequency = <10000000>;
+				status = "okay";
+
+				pinctrl-0 = <&pi433_pins>;
+				DIO0-gpio = <&gpio 24 0>;
+				DIO1-gpio = <&gpio 25 0>;
+				DIO2-gpio = <&gpio  7 0>;
+			};
+		};
+	};
+};
diff --git a/drivers/staging/pi433/Documentation/devicetree/pi433.txt b/drivers/staging/pi433/Documentation/devicetree/pi433.txt
new file mode 100644
index 000000000000..9ff217fbcbbd
--- /dev/null
+++ b/drivers/staging/pi433/Documentation/devicetree/pi433.txt
@@ -0,0 +1,62 @@
+* Smarthome-Wolf Pi433 - a 433MHz radio module/shield for Raspberry Pi (see www.pi433.de)
+
+Required properties:
+- compatible: must be "Smarthome-Wolf,pi433"
+- reg: chip select of SPI Interface
+- DIOx-gpio must be dedicated to the GPIO, connected with DIOx of the RFM69 module
+
+
+Example:
+
+With the following lines in gpio-section, the gpio pins, connected with pi433 are
+reserved/declared.
+
+&gpio{
+	[...]
+
+	pi433_pins: pi433_pins {
+		brcm,pins = <7 25 24>;
+		brcm,function = <0 0 0>; // in in in
+	};
+
+	[...]
+}
+
+With the following lines in spi section, the device pi433 is declared.
+It consists of the three gpio pins and an spi interface (here chip select 0)
+
+&spi0{
+	[...]
+
+	pi433: pi433@0 {
+		compatible = "Smarthome-Wolf,pi433";
+		reg = <0>; /* CE 0 */
+		#address-cells = <1>;
+		#size-cells = <0>;
+		spi-max-frequency = <10000000>;
+
+		pinctrl-0 = <&pi433_pins>;
+		DIO0-gpio = <&gpio 24 0>;
+		DIO1-gpio = <&gpio 25 0>;
+		DIO2-gpio = <&gpio  7 0>;
+	};
+}
+
+
+
+For Raspbian users only
+=======================
+Since Raspbian supports device tree overlays, you may use and overlay, instead
+of editing your boards device tree.
+For using the overlay, you need to compile the file pi433-overlay.dts you can
+find aside to this documentation.
+The file needs to be compiled - either manually or by integration in your kernel
+source tree. For a manual compile, you may use a command line like the following:
+'linux/scripts/dtc/dtc -@ -I dts -O dtb -o pi433.dtbo pi433-overlay.dts'
+
+For compiling inside of the kernel tree, you need to copy pi433-overlay.dts to
+arch/arm/boot/dts/overlays and you need to add the file to the list of files
+in the Makefile over there. Execute 'make dtbs' in kernel tree root to make the
+kernel make files compile the device tree overlay for you.
+
+
diff --git a/drivers/staging/pi433/Documentation/pi433.txt b/drivers/staging/pi433/Documentation/pi433.txt
new file mode 100644
index 000000000000..38b83b86c334
--- /dev/null
+++ b/drivers/staging/pi433/Documentation/pi433.txt
@@ -0,0 +1,274 @@
+=====
+Pi433
+=====
+
+
+Introduction
+============
+This driver is for controlling pi433, a radio module for the Raspberry Pi
+(www.pi433.de). It supports transmission and reception. It can be opened
+by multiple applications for transmission and reception. While transmit
+jobs were queued and process automatically in the background, the first
+application asking for reception will block out all other applications
+until something gets received terminates the read request.
+The driver supports on the fly reloading of the hardware fifo of the rf
+chip, thus enabling for much longer telegrams then hardware fifo size.
+
+Discription of driver operation
+===============================
+
+a) transmission
+
+Each transmission can take place with a different configuration of the rf
+module. Therfore each application can set its own set of parameters. The driver
+takes care, that each transmission takes place with the parameterset of the
+application, that requests the transmission. To allow the transmission to take
+place in the background, a tx thread is introduced.
+The transfer of data from the main thread to the tx thread is realised by a
+kfifo. With each write request of an application, the passed in data and the
+corresponding parameter set gets written to the kfifo.
+On the other "side" of the kfifo, the tx thread continuously checks, whether the
+kfifo is empty. If not, it gets one set of config and data from the kfifo. If
+there is no receive request or the receiver is still waiting for something in
+the air, the rf module is set to standby, the parameters for transmission gets
+set, the hardware fifo of the rf chip gets preloaded and the transmission gets
+started. Upon hardware fifo threshold interrupt it gets reloaded, thus enabling
+much longer telegrams then hardware fifo size. If the telegram is send and there
+is more data available in the kfifo, the procedure is repeated. If not the
+transmission cycle ends.
+
+b) reception
+
+Since there is only one application allowed to receive data at a time, for
+reception there is only one configuration set.
+As soon as an application sets an request for receiving a telegram, the reception
+configuration set is written to the rf module and it gets set into receiving mode.
+Now the driver is waiting, that a predefined RSSI level (signal strength at the
+receiver) is reached. Until this hasn't happened, the reception can be
+interrupted by the transmission thread at any time to insert a transmission cycle.
+As soon as the predefined RSSI level is meat, a receiving cycle starts. Similar
+as described for the transmission cycle the read out of the hardware fifo is done
+dynamically. Upon each hardware fifo threshold interrupt, a portion of data gets
+read. So also for reception it is possible to receive more data then the hardware
+fifo can hold.
+
+
+Driver API
+==========
+
+The driver is currently implemented as a character device. Therefore it supports
+the calls open, ioctl, read, write and close.
+
+
+params for ioctl
+----------------
+
+There are four options:
+PI433_IOC_RD_TX_CFG - get the transmission parameters from the driver
+PI433_IOC_WR_TX_CFG - set the transmission parameters
+PI433_IOC_RD_RX_CFG - get the receiving parameters from the driver
+PI433_IOC_WR_RX_CFG - set the receiving parameters
+
+The tx configuration is transfered via struct pi433_tx_cfg, the parameterset for transmission.
+It is devided into two sections: rf parameters and packet format.
+
+rf params:
+	frequency
+		frequency used for transmission.
+		Allowed values: 433050000...434790000
+	bit_rate
+		bit rate used for transmission.
+		Allowed values: #####
+	dev_frequency
+		frequency deviation in case of FSK.
+		Allowed values: 600...500000
+	modulation
+		FSK - frequency shift key
+		OOK - On-Off-key
+	modShaping
+		shapingOff	- no shaping
+		shaping1_0	- gauss filter with BT 1 (FSK only)
+		shaping0_5	- gauss filter with BT 0.5 (FSK only)
+		shaping0_3	- gauss filter with BT 0.3 (FSK only)
+		shapingBR	- filter cut off at BR (OOK only)
+		shaping2BR	- filter cut off at 2*BR (OOK only)
+	paRamp (FSK only)
+		ramp3400	- amp ramps up in 3.4ms
+		ramp2000	- amp ramps up in 2.0ms
+		ramp1000	- amp ramps up in 1ms
+		ramp500		- amp ramps up in 500us
+		ramp250		- amp ramps up in 250us
+		ramp125		- amp ramps up in 125us
+		ramp100		- amp ramps up in 100us
+		ramp62		- amp ramps up in 62us
+		ramp50		- amp ramps up in 50us
+		ramp40		- amp ramps up in 40us
+		ramp31		- amp ramps up in 31us
+		ramp25		- amp ramps up in 25us
+		ramp20		- amp ramps up in 20us
+		ramp15		- amp ramps up in 15us
+		ramp12		- amp ramps up in 12us
+		ramp10		- amp ramps up in 10us
+	tx_start_condition
+		fifoLevel	- transmission starts, if fifo is filled to
+				  threshold level
+		fifoNotEmpty	- transmission starts, as soon as there is one
+				  byte in internal fifo
+	repetitions
+		This gives the option, to send a telegram multiple times. Default: 1
+
+packet format:
+	enable_preamble
+		optionOn	- a preamble will be automatically generated
+		optionOff	- no preamble will be generated
+	enable_sync
+		optionOn	- a sync word will be automatically added to
+				  the telegram after preamble
+		optionOff	- no sync word will be added
+		Attention: While possible to generate sync without preamble, the
+		receiver won't be able to detect the sync without preamble.
+	enable_length_byte
+		optionOn	- the length of the telegram will be automatically
+				  added to the telegram. It's part of the payload
+		optionOff	- no length information will be automatically added
+				  to the telegram.
+		Attention: For telegram length over 255 bytes, this option can't be used
+		Attention: should be used in combination with sync, only
+	enable_address_byte
+		optionOn	- the address byte will be automatically added to the
+				  telgram. It's part of the payload
+		optionOff	- the address byte will not be added to the telegram.
+		The address byte can be used for address filtering, so the receiver
+		will only receive telegrams with a given address byte.
+		Attention: should be used in combination with sync, only
+	enable_crc
+		optionOn	- an crc will be automatically calculated over the
+				  payload of the telegram and added to the telegram
+				  after payload.
+		optionOff	- no crc will be calculated
+	preamble_length
+		length of the preamble. Allowed values: 0...65536
+	sync_length
+		length of the sync word. Allowed values: 0...8
+	fixed_message_length
+		length of the payload of the telegram. Will override the length
+		given by the buffer, passed in with the write command. Will be
+		ignored if set to zero.
+	sync_pattern[8]
+		contains up to eight values, that are used as the sync pattern
+		on sync option
+	address_byte
+		one byte, used as address byte on address byte option.
+
+
+The rx configuration is transfered via struct pi433_rx_cfg, the parameterset for receiving. It is devided into two sections: rf parameters and packet format.
+
+rf params:
+	frequency
+		frequency used for transmission.
+		Allowed values: 433050000...434790000
+	bit_rate
+		bit rate used for transmission.
+		Allowed values: #####
+	dev_frequency
+		frequency deviation in case of FSK.
+		Allowed values: 600...500000
+	modulation
+		FSK - frequency shift key
+		OOK - on off key
+	rssi_threshold
+		threshold value for the signal strength on the receiver input.
+		If this value is exeeded, a reception cycle starts
+		Allowed values: 0...255
+	thresholdDecrement
+		in order to adapt to different levels of singnal strength, over
+		time the receiver gets more and more sensitive. This value
+		determs, how fast the sensitivity increases.
+		step_0_5db	- increase in 0,5dB steps
+		step_1_0db	- increase in 1 db steps
+		step_1_5db	- increase in 1,5dB steps
+		step_2_0db	- increase in 2 db steps
+		step_3_0db	- increase in 3 db steps
+		step_4_0db	- increase in 4 db steps
+		step_5_0db	- increase in 5 db steps
+		step_6_0db	- increase in 6 db steps
+	antennaImpedance
+		sets the electrical adoption of the antenna
+		fiftyOhm	- for antennas with an impedance of 50Ohm
+		twohundretOhm	- for antennas with an impedance of 200Ohm
+	lnaGain
+		sets the gain of the low noise amp
+		automatic	- lna gain is determed by an agc
+		max		- lna gain is set to maximum
+		maxMinus6	- lna gain is set to  6db below max
+		maxMinus12	- lna gain is set to 12db below max
+		maxMinus24	- lna gain is set to 24db below max
+		maxMinus36	- lna gain is set to 36db below max
+		maxMinus48	- lna gain is set to 48db below max
+	bw_mantisse
+		sets the bandwidth of the channel filter - part one: mantisse.
+		mantisse16	- mantisse is set to 16
+		mantisse20	- mantisse is set to 20
+		mantisse24	- mantisse is set to 24
+	bw_exponent
+		sets the bandwidth of the channel filter - part two: exponent.
+		Allowd values: 0...7
+	dagc;
+		operation mode of the digital automatic gain control
+		normalMode
+		improve
+		improve4LowModulationIndex
+
+ packet format:
+	enable_sync
+		optionOn  - sync detection is enabled. If configured sync pattern
+			    isn't found, telegram will be internally discarded
+		optionOff - sync detection is disabled.
+	enable_length_byte
+		optionOn   - First byte of payload will be used as length byte,
+			     regardless of the amount of bytes that were requested
+			     by the read request.
+		optionOff  - Number of bytes to be read will be set according to
+			     amount of bytes that were requested by the read request.
+		Attention: should be used in combination with sync, only
+	enable_address_filtering;
+		filteringOff		- no adress filtering will take place
+		nodeAddress		- all telegrams, not matching the node
+					  address will be internally discarded
+		nodeOrBroadcastAddress	- all telegrams, neither matching the
+					  node, nor the broadcast address will
+					  be internally discarded
+		Attention: Sync option must be enabled in order to use this feature
+	enable_crc
+		optionOn	- a crc will be calculated over the payload of
+				  the telegram, that was received. If the
+				  calculated crc doesn't match to two bytes,
+				  that follow the payload, the telegram will be
+				  internally discarded.
+		Attention: This option is only operational, if sync on and fixed length
+		or length byte is used
+	sync_length
+		Gives the length of the payload.
+		Attention: This setting must meet the setting of the transmitter,
+		if sync option is used.
+	fixed_message_length
+		Overrides the telegram length either given by the first byte of
+		payload or by the read request.
+	bytes_to_drop
+		gives the number of bytes, that will be dropped before transfering
+		data to the read buffer
+		This option is only usefull, if all packet helper are switched
+		off and the rf chip is used in raw receiving mode. This may be
+		needed, if a telegram of a third party device should be received,
+		using a protocol not compatible with the packet engine of the rf69 chip.
+	sync_pattern[8]
+		contains up to eight values, that are used as the sync pattern
+		on sync option.
+		This setting must meet the configuration of the transmitting device,
+		if sync option is enabled.
+	node_address
+		one byte, used as node address byte on address byte option.
+	broadcast_address
+		one byte, used as broadcast address byte on address byte option.
+
+
diff --git a/drivers/staging/pi433/Kconfig b/drivers/staging/pi433/Kconfig
new file mode 100644
index 000000000000..87c2ee192cca
--- /dev/null
+++ b/drivers/staging/pi433/Kconfig
@@ -0,0 +1,16 @@
+config PI433
+        tristate "Pi433 - a 433MHz radio module for Raspberry Pi"
+        depends on SPI
+        ---help---
+          This option allows you to enable support for the radio module Pi433.
+
+          Pi433 is a shield that fits onto the GPIO header of a Raspberry Pi
+          or compatible. It extends the Raspberry Pi with the option, to
+          send and receive data in the 433MHz ISM band - for example to
+          communicate between two systems without using ethernet or bluetooth
+          or for control or read sockets, actors, sensors, widely available
+          for low price.
+
+          For details or the option to buy, please visit https://pi433.de/en.html
+
+          If in doubt, say N here, but saying yes most probably won't hurt
diff --git a/drivers/staging/pi433/Makefile b/drivers/staging/pi433/Makefile
new file mode 100644
index 000000000000..417f3e4d12b1
--- /dev/null
+++ b/drivers/staging/pi433/Makefile
@@ -0,0 +1,3 @@
+obj-$(CONFIG_PI433) += pi433.o
+
+pi433-objs := pi433_if.o rf69.o
diff --git a/drivers/staging/pi433/TODO b/drivers/staging/pi433/TODO
new file mode 100644
index 000000000000..63a40bfcc67e
--- /dev/null
+++ b/drivers/staging/pi433/TODO
@@ -0,0 +1,5 @@
+* coding style does not fully comply with the kernel style guide.
+* still TODOs, annotated in the code
+* currently the code introduces new IOCTLs. I'm afraid this is a bad idea.
+  -> Replace this with another interface, hints are welcome!
+* Some missing data (marked with ###) needs to be added in the documentation
diff --git a/drivers/staging/pi433/pi433_if.c b/drivers/staging/pi433/pi433_if.c
new file mode 100644
index 000000000000..93c01680f016
--- /dev/null
+++ b/drivers/staging/pi433/pi433_if.c
@@ -0,0 +1,1338 @@
+/*
+ * userspace interface for pi433 radio module
+ *
+ * Pi433 is a 433MHz radio module for the Raspberry Pi.
+ * It is based on the HopeRf Module RFM69CW. Therefore inside of this
+ * driver, you'll find an abstraction of the rf69 chip.
+ *
+ * If needed, this driver could be extended, to also support other
+ * devices, basing on HopeRfs rf69.
+ *
+ * The driver can also be extended, to support other modules of
+ * HopeRf with a similar interace - e. g. RFM69HCW, RFM12, RFM95, ...
+ *
+ * Copyright (C) 2016 Wolf-Entwicklungen
+ *	Marcus Wolf <linux@wolf-entwicklungen.de>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#undef DEBUG
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/idr.h>
+#include <linux/ioctl.h>
+#include <linux/uaccess.h>
+#include <linux/fs.h>
+#include <linux/device.h>
+#include <linux/cdev.h>
+#include <linux/err.h>
+#include <linux/kfifo.h>
+#include <linux/errno.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/gpio/consumer.h>
+#include <linux/kthread.h>
+#include <linux/wait.h>
+#include <linux/spi/spi.h>
+#ifdef CONFIG_COMPAT
+#include <asm/compat.h>
+#endif
+
+#include "pi433_if.h"
+#include "rf69.h"
+
+
+#define N_PI433_MINORS			(1U << MINORBITS) /*32*/	/* ... up to 256 */
+#define MAX_MSG_SIZE			900	/* min: FIFO_SIZE! */
+#define MSG_FIFO_SIZE			65536   /* 65536 = 2^16  */
+#define NUM_DIO				2
+
+static dev_t pi433_dev;
+static DEFINE_IDR(pi433_idr);
+static DEFINE_MUTEX(minor_lock); /* Protect idr accesses */
+
+static struct class *pi433_class; /* mainly for udev to create /dev/pi433 */
+
+/* tx config is instance specific
+ * so with each open a new tx config struct is needed
+ */
+/* rx config is device specific
+ * so we have just one rx config, ebedded in device struct
+ */
+struct pi433_device {
+	/* device handling related values */
+	dev_t			devt;
+	int			minor;
+	struct device		*dev;
+	struct cdev		*cdev;
+	struct spi_device	*spi;
+	unsigned		users;
+
+	/* irq related values */
+	struct gpio_desc	*gpiod[NUM_DIO];
+	int			irq_num[NUM_DIO];
+	u8			irq_state[NUM_DIO];
+
+	/* tx related values */
+	STRUCT_KFIFO_REC_1(MSG_FIFO_SIZE) tx_fifo;
+	struct mutex		tx_fifo_lock; // TODO: check, whether necessary or obsolete
+	struct task_struct	*tx_task_struct;
+	wait_queue_head_t	tx_wait_queue;
+	u8			free_in_fifo;
+	char			buffer[MAX_MSG_SIZE];
+
+	/* rx related values */
+	struct pi433_rx_cfg	rx_cfg;
+	u8			*rx_buffer;
+	unsigned int		rx_buffer_size;
+	u32			rx_bytes_to_drop;
+	u32			rx_bytes_dropped;
+	unsigned int		rx_position;
+	struct mutex		rx_lock;
+	wait_queue_head_t	rx_wait_queue;
+
+	/* fifo wait queue */
+	struct task_struct	*fifo_task_struct;
+	wait_queue_head_t	fifo_wait_queue;
+
+	/* flags */
+	bool			rx_active;
+	bool			tx_active;
+	bool			interrupt_rx_allowed;
+};
+
+struct pi433_instance {
+	struct pi433_device	*device;
+	struct pi433_tx_cfg	tx_cfg;
+};
+
+/*-------------------------------------------------------------------------*/
+
+/* macro for checked access of registers of radio module */
+#define SET_CHECKED(retval) \
+	if (retval < 0) \
+		return retval;
+
+/*-------------------------------------------------------------------------*/
+
+/* GPIO interrupt handlers */
+static irqreturn_t DIO0_irq_handler(int irq, void *dev_id)
+{
+	struct pi433_device *device = dev_id;
+
+	if      (device->irq_state[DIO0] == DIO_PacketSent)
+	{
+		device->free_in_fifo = FIFO_SIZE;
+		printk("DIO0 irq: Packet sent\n"); // TODO: printk() should include KERN_ facility level
+		wake_up_interruptible(&device->fifo_wait_queue);
+	}
+	else if (device->irq_state[DIO0] == DIO_Rssi_DIO0)
+	{
+		printk("DIO0 irq: RSSI level over threshold\n");
+		wake_up_interruptible(&device->rx_wait_queue);
+	}
+	else if (device->irq_state[DIO0] == DIO_PayloadReady)
+	{
+		printk("DIO0 irq: PayloadReady\n");
+		device->free_in_fifo = 0;
+		wake_up_interruptible(&device->fifo_wait_queue);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t DIO1_irq_handler(int irq, void *dev_id)
+{
+	struct pi433_device *device = dev_id;
+
+	if      (device->irq_state[DIO1] == DIO_FifoNotEmpty_DIO1)
+	{
+		device->free_in_fifo = FIFO_SIZE;
+	}
+	else if (device->irq_state[DIO1] == DIO_FifoLevel)
+	{
+		if (device->rx_active)	device->free_in_fifo = FIFO_THRESHOLD - 1;
+		else			device->free_in_fifo = FIFO_SIZE - FIFO_THRESHOLD - 1;
+	}
+	printk("DIO1 irq: %d bytes free in fifo\n", device->free_in_fifo); // TODO: printk() should include KERN_ facility level
+	wake_up_interruptible(&device->fifo_wait_queue);
+
+	return IRQ_HANDLED;
+}
+
+/*-------------------------------------------------------------------------*/
+
+static int
+rf69_set_rx_cfg(struct pi433_device *dev, struct pi433_rx_cfg *rx_cfg)
+{
+	int ret;
+	int payload_length;
+
+	/* receiver config */
+	SET_CHECKED(rf69_set_frequency	(dev->spi, rx_cfg->frequency));
+	SET_CHECKED(rf69_set_bit_rate	(dev->spi, rx_cfg->bit_rate));
+	SET_CHECKED(rf69_set_modulation	(dev->spi, rx_cfg->modulation));
+	SET_CHECKED(rf69_set_antenna_impedance	 (dev->spi, rx_cfg->antenna_impedance));
+	SET_CHECKED(rf69_set_rssi_threshold	 (dev->spi, rx_cfg->rssi_threshold));
+	SET_CHECKED(rf69_set_ook_threshold_dec	 (dev->spi, rx_cfg->thresholdDecrement));
+	SET_CHECKED(rf69_set_bandwidth 		 (dev->spi, rx_cfg->bw_mantisse, rx_cfg->bw_exponent));
+	SET_CHECKED(rf69_set_bandwidth_during_afc(dev->spi, rx_cfg->bw_mantisse, rx_cfg->bw_exponent));
+	SET_CHECKED(rf69_set_dagc 		 (dev->spi, rx_cfg->dagc));
+
+	dev->rx_bytes_to_drop = rx_cfg->bytes_to_drop;
+
+	/* packet config */
+	/* enable */
+	SET_CHECKED(rf69_set_sync_enable(dev->spi, rx_cfg->enable_sync));
+	if (rx_cfg->enable_sync == optionOn)
+	{
+		SET_CHECKED(rf69_set_fifo_fill_condition(dev->spi, afterSyncInterrupt));
+	}
+	else
+	{
+		SET_CHECKED(rf69_set_fifo_fill_condition(dev->spi, always));
+	}
+	if (rx_cfg->enable_length_byte == optionOn) {
+		ret = rf69_set_packet_format(dev->spi, packetLengthVar);
+		if (ret < 0)
+			return ret;
+	} else {
+		ret = rf69_set_packet_format(dev->spi, packetLengthFix);
+		if (ret < 0)
+			return ret;
+	}
+	SET_CHECKED(rf69_set_adressFiltering(dev->spi, rx_cfg->enable_address_filtering));
+	SET_CHECKED(rf69_set_crc_enable	    (dev->spi, rx_cfg->enable_crc));
+
+	/* lengths */
+	SET_CHECKED(rf69_set_sync_size(dev->spi, rx_cfg->sync_length));
+	if (rx_cfg->enable_length_byte == optionOn)
+	{
+		SET_CHECKED(rf69_set_payload_length(dev->spi, 0xff));
+	}
+	else if (rx_cfg->fixed_message_length != 0)
+	{
+		payload_length = rx_cfg->fixed_message_length;
+		if (rx_cfg->enable_length_byte  == optionOn) payload_length++;
+		if (rx_cfg->enable_address_filtering != filteringOff) payload_length++;
+		SET_CHECKED(rf69_set_payload_length(dev->spi, payload_length));
+	}
+	else
+	{
+		SET_CHECKED(rf69_set_payload_length(dev->spi, 0));
+	}
+
+	/* values */
+	if (rx_cfg->enable_sync == optionOn)
+	{
+		SET_CHECKED(rf69_set_sync_values(dev->spi, rx_cfg->sync_pattern));
+	}
+	if (rx_cfg->enable_address_filtering != filteringOff)
+	{
+		SET_CHECKED(rf69_set_node_address     (dev->spi, rx_cfg->node_address));
+		SET_CHECKED(rf69_set_broadcast_address(dev->spi, rx_cfg->broadcast_address));
+	}
+
+	return 0;
+}
+
+static int
+rf69_set_tx_cfg(struct pi433_device *dev, struct pi433_tx_cfg *tx_cfg)
+{
+	int ret;
+
+	SET_CHECKED(rf69_set_frequency	(dev->spi, tx_cfg->frequency));
+	SET_CHECKED(rf69_set_bit_rate	(dev->spi, tx_cfg->bit_rate));
+	SET_CHECKED(rf69_set_modulation	(dev->spi, tx_cfg->modulation));
+	SET_CHECKED(rf69_set_deviation	(dev->spi, tx_cfg->dev_frequency));
+	SET_CHECKED(rf69_set_pa_ramp	(dev->spi, tx_cfg->pa_ramp));
+	SET_CHECKED(rf69_set_modulation_shaping(dev->spi, tx_cfg->modShaping));
+	SET_CHECKED(rf69_set_tx_start_condition(dev->spi, tx_cfg->tx_start_condition));
+
+	/* packet format enable */
+	if (tx_cfg->enable_preamble == optionOn)
+	{
+		SET_CHECKED(rf69_set_preamble_length(dev->spi, tx_cfg->preamble_length));
+	}
+	else
+	{
+		SET_CHECKED(rf69_set_preamble_length(dev->spi, 0));
+	}
+	SET_CHECKED(rf69_set_sync_enable  (dev->spi, tx_cfg->enable_sync));
+	if (tx_cfg->enable_length_byte == optionOn) {
+		ret = rf69_set_packet_format(dev->spi, packetLengthVar);
+		if (ret < 0)
+			return ret;
+	} else {
+		ret = rf69_set_packet_format(dev->spi, packetLengthFix);
+		if (ret < 0)
+			return ret;
+	}
+	SET_CHECKED(rf69_set_crc_enable	  (dev->spi, tx_cfg->enable_crc));
+
+	/* configure sync, if enabled */
+	if (tx_cfg->enable_sync == optionOn)
+	{
+		SET_CHECKED(rf69_set_sync_size(dev->spi, tx_cfg->sync_length));
+		SET_CHECKED(rf69_set_sync_values(dev->spi, tx_cfg->sync_pattern));
+	}
+
+	return 0;
+}
+
+/*-------------------------------------------------------------------------*/
+
+static int
+pi433_start_rx(struct pi433_device *dev)
+{
+	int retval;
+
+	/* return without action, if no pending read request */
+	if (!dev->rx_active)
+		return 0;
+
+	/* setup for receiving */
+	retval = rf69_set_rx_cfg(dev, &dev->rx_cfg);
+	if (retval) return retval;
+
+	/* setup rssi irq */
+	SET_CHECKED(rf69_set_dio_mapping(dev->spi, DIO0, DIO_Rssi_DIO0));
+	dev->irq_state[DIO0] = DIO_Rssi_DIO0;
+	irq_set_irq_type(dev->irq_num[DIO0], IRQ_TYPE_EDGE_RISING);
+
+	/* setup fifo level interrupt */
+	SET_CHECKED(rf69_set_fifo_threshold(dev->spi, FIFO_SIZE - FIFO_THRESHOLD));
+	SET_CHECKED(rf69_set_dio_mapping(dev->spi, DIO1, DIO_FifoLevel));
+	dev->irq_state[DIO1] = DIO_FifoLevel;
+	irq_set_irq_type(dev->irq_num[DIO1], IRQ_TYPE_EDGE_RISING);
+
+	/* set module to receiving mode */
+	SET_CHECKED(rf69_set_mode(dev->spi, receive));
+
+	return 0;
+}
+
+
+/*-------------------------------------------------------------------------*/
+
+static int
+pi433_receive(void *data)
+{
+	struct pi433_device *dev = data;
+	struct spi_device *spi = dev->spi; /* needed for SET_CHECKED */
+	int bytes_to_read, bytes_total;
+	int retval;
+
+	dev->interrupt_rx_allowed = false;
+
+	/* wait for any tx to finish */
+	dev_dbg(dev->dev,"rx: going to wait for any tx to finish");
+	retval = wait_event_interruptible(dev->rx_wait_queue, !dev->tx_active);
+	if(retval) /* wait was interrupted */
+	{
+		dev->interrupt_rx_allowed = true;
+		wake_up_interruptible(&dev->tx_wait_queue);
+		return retval;
+	}
+
+	/* prepare status vars */
+	dev->free_in_fifo = FIFO_SIZE;
+	dev->rx_position = 0;
+	dev->rx_bytes_dropped = 0;
+
+	/* setup radio module to listen for something "in the air" */
+	retval = pi433_start_rx(dev);
+	if (retval)
+		return retval;
+
+	/* now check RSSI, if low wait for getting high (RSSI interrupt) */
+	while ( !rf69_get_flag(dev->spi, rssiExceededThreshold) )
+	{
+		/* allow tx to interrupt us while waiting for high RSSI */
+		dev->interrupt_rx_allowed = true;
+		wake_up_interruptible(&dev->tx_wait_queue);
+
+		/* wait for RSSI level to become high */
+		dev_dbg(dev->dev, "rx: going to wait for high RSSI level");
+		retval = wait_event_interruptible(dev->rx_wait_queue,
+			                          rf69_get_flag(dev->spi,
+		                                                rssiExceededThreshold));
+		if (retval) goto abort; /* wait was interrupted */
+		dev->interrupt_rx_allowed = false;
+
+		/* cross check for ongoing tx */
+		if (!dev->tx_active) break;
+	}
+
+	/* configure payload ready irq */
+	SET_CHECKED(rf69_set_dio_mapping(spi, DIO0, DIO_PayloadReady));
+	dev->irq_state[DIO0] = DIO_PayloadReady;
+	irq_set_irq_type(dev->irq_num[DIO0], IRQ_TYPE_EDGE_RISING);
+
+	/* fixed or unlimited length? */
+	if (dev->rx_cfg.fixed_message_length != 0)
+	{
+		if (dev->rx_cfg.fixed_message_length > dev->rx_buffer_size)
+		{
+			retval = -1;
+			goto abort;
+		}
+		bytes_total = dev->rx_cfg.fixed_message_length;
+		dev_dbg(dev->dev,"rx: msg len set to %d by fixed length", bytes_total);
+	}
+	else
+	{
+		bytes_total = dev->rx_buffer_size;
+		dev_dbg(dev->dev, "rx: msg len set to %d as requested by read", bytes_total);
+	}
+
+	/* length byte enabled? */
+	if (dev->rx_cfg.enable_length_byte == optionOn)
+	{
+		retval = wait_event_interruptible(dev->fifo_wait_queue,
+						  dev->free_in_fifo < FIFO_SIZE);
+		if (retval) goto abort; /* wait was interrupted */
+
+		rf69_read_fifo(spi, (u8 *)&bytes_total, 1);
+		if (bytes_total > dev->rx_buffer_size)
+		{
+			retval = -1;
+			goto abort;
+		}
+		dev->free_in_fifo++;
+		dev_dbg(dev->dev, "rx: msg len reset to %d due to length byte", bytes_total);
+	}
+
+	/* address byte enabled? */
+	if (dev->rx_cfg.enable_address_filtering != filteringOff)
+	{
+		u8 dummy;
+
+		bytes_total--;
+
+		retval = wait_event_interruptible(dev->fifo_wait_queue,
+						  dev->free_in_fifo < FIFO_SIZE);
+		if (retval) goto abort; /* wait was interrupted */
+
+		rf69_read_fifo(spi, &dummy, 1);
+		dev->free_in_fifo++;
+		dev_dbg(dev->dev, "rx: address byte stripped off");
+	}
+
+	/* get payload */
+	while (dev->rx_position < bytes_total)
+	{
+		if ( !rf69_get_flag(dev->spi, payloadReady) )
+		{
+			retval = wait_event_interruptible(dev->fifo_wait_queue,
+							  dev->free_in_fifo < FIFO_SIZE);
+			if (retval) goto abort; /* wait was interrupted */
+		}
+
+		/* need to drop bytes or acquire? */
+		if (dev->rx_bytes_to_drop > dev->rx_bytes_dropped)
+			bytes_to_read = dev->rx_bytes_to_drop - dev->rx_bytes_dropped;
+		else
+			bytes_to_read = bytes_total - dev->rx_position;
+
+
+		/* access the fifo */
+		if (bytes_to_read > FIFO_SIZE - dev->free_in_fifo)
+			bytes_to_read = FIFO_SIZE - dev->free_in_fifo;
+		retval = rf69_read_fifo(spi,
+					&dev->rx_buffer[dev->rx_position],
+					bytes_to_read);
+		if (retval) goto abort; /* read failed */
+		dev->free_in_fifo += bytes_to_read;
+
+		/* adjust status vars */
+		if (dev->rx_bytes_to_drop > dev->rx_bytes_dropped)
+			dev->rx_bytes_dropped += bytes_to_read;
+		else
+			dev->rx_position += bytes_to_read;
+	}
+
+
+	/* rx done, wait was interrupted or error occured */
+abort:
+	dev->interrupt_rx_allowed = true;
+	SET_CHECKED(rf69_set_mode(dev->spi, standby));
+	wake_up_interruptible(&dev->tx_wait_queue);
+
+	if (retval)
+		return retval;
+	else
+		return bytes_total;
+}
+
+static int
+pi433_tx_thread(void *data)
+{
+	struct pi433_device *device = data;
+	struct spi_device *spi = device->spi; /* needed for SET_CHECKED */
+	struct pi433_tx_cfg tx_cfg;
+	u8     *buffer = device->buffer;
+	size_t size;
+	bool   rx_interrupted = false;
+	int    position, repetitions;
+	int    retval;
+
+	while (1)
+	{
+		/* wait for fifo to be populated or for request to terminate*/
+		dev_dbg(device->dev, "thread: going to wait for new messages");
+		wait_event_interruptible(device->tx_wait_queue,
+					 ( !kfifo_is_empty(&device->tx_fifo) ||
+					    kthread_should_stop() ));
+		if ( kthread_should_stop() )
+			return 0;
+
+		/* get data from fifo in the following order:
+		 * - tx_cfg
+		 * - size of message
+		 * - message
+		 */
+		mutex_lock(&device->tx_fifo_lock);
+
+		retval = kfifo_out(&device->tx_fifo, &tx_cfg, sizeof(tx_cfg));
+		if (retval != sizeof(tx_cfg))
+		{
+			dev_dbg(device->dev, "reading tx_cfg from fifo failed: got %d byte(s), expected %d", retval, (unsigned int)sizeof(tx_cfg) );
+			mutex_unlock(&device->tx_fifo_lock);
+			continue;
+		}
+
+		retval = kfifo_out(&device->tx_fifo, &size, sizeof(size_t));
+		if (retval != sizeof(size_t))
+		{
+			dev_dbg(device->dev, "reading msg size from fifo failed: got %d, expected %d", retval, (unsigned int)sizeof(size_t) );
+			mutex_unlock(&device->tx_fifo_lock);
+			continue;
+		}
+
+		/* use fixed message length, if requested */
+		if (tx_cfg.fixed_message_length != 0)
+			size = tx_cfg.fixed_message_length;
+
+		/* increase size, if len byte is requested */
+		if (tx_cfg.enable_length_byte == optionOn)
+			size++;
+
+		/* increase size, if adr byte is requested */
+		if (tx_cfg.enable_address_byte == optionOn)
+			size++;
+
+		/* prime buffer */
+		memset(buffer, 0, size);
+		position = 0;
+
+		/* add length byte, if requested */
+		if (tx_cfg.enable_length_byte  == optionOn)
+			buffer[position++] = size-1; /* according to spec length byte itself must be excluded from the length calculation */
+
+		/* add adr byte, if requested */
+		if (tx_cfg.enable_address_byte == optionOn)
+			buffer[position++] = tx_cfg.address_byte;
+
+		/* finally get message data from fifo */
+		retval = kfifo_out(&device->tx_fifo, &buffer[position], sizeof(buffer)-position );
+		dev_dbg(device->dev, "read %d message byte(s) from fifo queue.", retval);
+		mutex_unlock(&device->tx_fifo_lock);
+
+		/* if rx is active, we need to interrupt the waiting for
+		 * incoming telegrams, to be able to send something.
+		 * We are only allowed, if currently no reception takes
+		 * place otherwise we need to  wait for the incoming telegram
+		 * to finish
+		 */
+		wait_event_interruptible(device->tx_wait_queue,
+					 !device->rx_active ||
+					  device->interrupt_rx_allowed == true);
+
+		/* prevent race conditions
+		 * irq will be reenabled after tx config is set
+		 */
+		disable_irq(device->irq_num[DIO0]);
+		device->tx_active = true;
+
+		if (device->rx_active && rx_interrupted == false)
+		{
+			/* rx is currently waiting for a telegram;
+			 * we need to set the radio module to standby
+			 */
+			SET_CHECKED(rf69_set_mode(device->spi, standby));
+			rx_interrupted = true;
+		}
+
+		/* clear fifo, set fifo threshold, set payload length */
+		SET_CHECKED(rf69_set_mode(spi, standby)); /* this clears the fifo */
+		SET_CHECKED(rf69_set_fifo_threshold(spi, FIFO_THRESHOLD));
+		if (tx_cfg.enable_length_byte == optionOn)
+		{
+			SET_CHECKED(rf69_set_payload_length(spi, size * tx_cfg.repetitions));
+		}
+		else
+		{
+			SET_CHECKED(rf69_set_payload_length(spi, 0));
+		}
+
+		/* configure the rf chip */
+		rf69_set_tx_cfg(device, &tx_cfg);
+
+		/* enable fifo level interrupt */
+		SET_CHECKED(rf69_set_dio_mapping(spi, DIO1, DIO_FifoLevel));
+		device->irq_state[DIO1] = DIO_FifoLevel;
+		irq_set_irq_type(device->irq_num[DIO1], IRQ_TYPE_EDGE_FALLING);
+
+		/* enable packet sent interrupt */
+		SET_CHECKED(rf69_set_dio_mapping(spi, DIO0, DIO_PacketSent));
+		device->irq_state[DIO0] = DIO_PacketSent;
+		irq_set_irq_type(device->irq_num[DIO0], IRQ_TYPE_EDGE_RISING);
+		enable_irq(device->irq_num[DIO0]); /* was disabled by rx active check */
+
+		/* enable transmission */
+		SET_CHECKED(rf69_set_mode(spi, transmit));
+
+		/* transfer this msg (and repetitions) to chip fifo */
+		device->free_in_fifo = FIFO_SIZE;
+		position = 0;
+		repetitions = tx_cfg.repetitions;
+		while( (repetitions > 0) && (size > position) )
+		{
+			if ( (size - position) > device->free_in_fifo)
+			{	/* msg to big for fifo - take a part */
+				int temp = device->free_in_fifo;
+				device->free_in_fifo = 0;
+				rf69_write_fifo(spi,
+				                &buffer[position],
+				                temp);
+				position +=temp;
+			}
+			else
+			{	/* msg fits into fifo - take all */
+				device->free_in_fifo -= size;
+				repetitions--;
+				rf69_write_fifo(spi,
+						&buffer[position],
+						(size - position) );
+				position = 0; /* reset for next repetition */
+			}
+
+			retval = wait_event_interruptible(device->fifo_wait_queue,
+							  device->free_in_fifo > 0);
+			if (retval) { printk("ABORT\n"); goto abort; }
+		}
+
+		/* we are done. Wait for packet to get sent */
+		dev_dbg(device->dev, "thread: wait for packet to get sent/fifo to be empty");
+		wait_event_interruptible(device->fifo_wait_queue,
+					 device->free_in_fifo == FIFO_SIZE ||
+					 kthread_should_stop() );
+		if ( kthread_should_stop() )	printk("ABORT\n");
+
+
+		/* STOP_TRANSMISSION */
+		dev_dbg(device->dev, "thread: Packet sent. Set mode to stby.");
+		SET_CHECKED(rf69_set_mode(spi, standby));
+
+		/* everything sent? */
+		if ( kfifo_is_empty(&device->tx_fifo) )
+		{
+abort:
+			if (rx_interrupted)
+			{
+				rx_interrupted = false;
+				pi433_start_rx(device);
+			}
+			device->tx_active = false;
+			wake_up_interruptible(&device->rx_wait_queue);
+		}
+	}
+}
+
+/*-------------------------------------------------------------------------*/
+
+static ssize_t
+pi433_read(struct file *filp, char __user *buf, size_t size, loff_t *f_pos)
+{
+	struct pi433_instance	*instance;
+	struct pi433_device	*device;
+	int			bytes_received;
+	ssize_t			retval;
+
+	/* check, whether internal buffer is big enough for requested size */
+	if (size > MAX_MSG_SIZE)
+		return -EMSGSIZE;
+
+	instance = filp->private_data;
+	device = instance->device;
+
+	/* just one read request at a time */
+	mutex_lock(&device->rx_lock);
+	if (device->rx_active)
+	{
+		mutex_unlock(&device->rx_lock);
+		return -EAGAIN;
+	}
+	else
+	{
+		device->rx_active = true;
+		mutex_unlock(&device->rx_lock);
+	}
+
+	/* start receiving */
+	/* will block until something was received*/
+	device->rx_buffer_size = size;
+	bytes_received = pi433_receive(device);
+
+	/* release rx */
+	mutex_lock(&device->rx_lock);
+	device->rx_active = false;
+	mutex_unlock(&device->rx_lock);
+
+	/* if read was successful copy to user space*/
+	if (bytes_received > 0)
+	{
+		retval = copy_to_user(buf, device->rx_buffer, bytes_received);
+		if (retval)
+			return -EFAULT;
+	}
+
+	return bytes_received;
+}
+
+
+static ssize_t
+pi433_write(struct file *filp, const char __user *buf,
+		size_t count, loff_t *f_pos)
+{
+	struct pi433_instance	*instance;
+	struct pi433_device	*device;
+	int                     copied, retval;
+
+	instance = filp->private_data;
+	device = instance->device;
+
+	/* check, whether internal buffer (tx thread) is big enough for requested size */
+	if (count > MAX_MSG_SIZE)
+		return -EMSGSIZE;
+
+	/* write the following sequence into fifo:
+	 * - tx_cfg
+	 * - size of message
+	 * - message
+	 */
+	mutex_lock(&device->tx_fifo_lock);
+	retval = kfifo_in(&device->tx_fifo, &instance->tx_cfg, sizeof(instance->tx_cfg));
+	if ( retval != sizeof(instance->tx_cfg) )
+		goto abort;
+
+	retval = kfifo_in (&device->tx_fifo, &count, sizeof(size_t));
+	if ( retval != sizeof(size_t) )
+		goto abort;
+
+	retval = kfifo_from_user(&device->tx_fifo, buf, count, &copied);
+	if (retval || copied != count)
+		goto abort;
+
+	mutex_unlock(&device->tx_fifo_lock);
+
+	/* start transfer */
+	wake_up_interruptible(&device->tx_wait_queue);
+	dev_dbg(device->dev, "write: generated new msg with %d bytes.", copied);
+
+	return 0;
+
+abort:
+	dev_dbg(device->dev, "write to fifo failed: 0x%x", retval);
+	kfifo_reset(&device->tx_fifo); // TODO: maybe find a solution, not to discard already stored, valid entries
+	mutex_unlock(&device->tx_fifo_lock);
+	return -EAGAIN;
+}
+
+
+static long
+pi433_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+{
+	int			err = 0;
+	int			retval = 0;
+	struct pi433_instance	*instance;
+	struct pi433_device	*device;
+	u32			tmp;
+
+	/* Check type and command number */
+	if (_IOC_TYPE(cmd) != PI433_IOC_MAGIC)
+		return -ENOTTY;
+
+	/* Check access direction once here; don't repeat below.
+	 * IOC_DIR is from the user perspective, while access_ok is
+	 * from the kernel perspective; so they look reversed.
+	 */
+	if (_IOC_DIR(cmd) & _IOC_READ)
+		err = !access_ok(VERIFY_WRITE,
+				 (void __user *)arg,
+				 _IOC_SIZE(cmd));
+
+	if (err == 0 && _IOC_DIR(cmd) & _IOC_WRITE)
+		err = !access_ok(VERIFY_READ,
+				 (void __user *)arg,
+				 _IOC_SIZE(cmd));
+	if (err)
+		return -EFAULT;
+
+	/* TODO? guard against device removal before, or while,
+	 * we issue this ioctl. --> device_get()
+	 */
+	instance = filp->private_data;
+	device = instance->device;
+
+	if (device == NULL)
+		return -ESHUTDOWN;
+
+	switch (cmd) {
+	case PI433_IOC_RD_TX_CFG:
+		tmp = _IOC_SIZE(cmd);
+		if ( (tmp == 0) || ((tmp % sizeof(struct pi433_tx_cfg)) != 0) )
+		{
+			retval = -EINVAL;
+			break;
+		}
+
+		if (__copy_to_user((void __user *)arg,
+				    &instance->tx_cfg,
+				    tmp))
+		{
+			retval = -EFAULT;
+			break;
+		}
+
+		break;
+	case PI433_IOC_WR_TX_CFG:
+		tmp = _IOC_SIZE(cmd);
+		if ( (tmp == 0) || ((tmp % sizeof(struct pi433_tx_cfg)) != 0) )
+		{
+			retval = -EINVAL;
+			break;
+		}
+
+		if (__copy_from_user(&instance->tx_cfg,
+				     (void __user *)arg,
+				     tmp))
+		{
+			retval = -EFAULT;
+			break;
+		}
+
+		break;
+
+	case PI433_IOC_RD_RX_CFG:
+		tmp = _IOC_SIZE(cmd);
+		if ( (tmp == 0) || ((tmp % sizeof(struct pi433_rx_cfg)) != 0) ) {
+			retval = -EINVAL;
+			break;
+		}
+
+		if (__copy_to_user((void __user *)arg,
+				   &device->rx_cfg,
+				   tmp))
+		{
+			retval = -EFAULT;
+			break;
+		}
+
+		break;
+	case PI433_IOC_WR_RX_CFG:
+		tmp = _IOC_SIZE(cmd);
+		mutex_lock(&device->rx_lock);
+
+		/* during pendig read request, change of config not allowed */
+		if (device->rx_active) {
+			retval = -EAGAIN;
+			mutex_unlock(&device->rx_lock);
+			break;
+		}
+
+		if ( (tmp == 0) || ((tmp % sizeof(struct pi433_rx_cfg)) != 0) ) {
+			retval = -EINVAL;
+			mutex_unlock(&device->rx_lock);
+			break;
+		}
+
+		if (__copy_from_user(&device->rx_cfg,
+				     (void __user *)arg,
+				     tmp))
+		{
+			retval = -EFAULT;
+			mutex_unlock(&device->rx_lock);
+			break;
+		}
+
+		mutex_unlock(&device->rx_lock);
+		break;
+	default:
+		retval = -EINVAL;
+	}
+
+	return retval;
+}
+
+#ifdef CONFIG_COMPAT
+static long
+pi433_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+{
+	return pi433_ioctl(filp, cmd, (unsigned long)compat_ptr(arg));
+}
+#else
+#define pi433_compat_ioctl NULL
+#endif /* CONFIG_COMPAT */
+
+/*-------------------------------------------------------------------------*/
+
+static int pi433_open(struct inode *inode, struct file *filp)
+{
+	struct pi433_device	*device;
+	struct pi433_instance	*instance;
+
+	mutex_lock(&minor_lock);
+	device = idr_find(&pi433_idr, iminor(inode));
+	mutex_unlock(&minor_lock);
+	if (!device) {
+		pr_debug("device: minor %d unknown.\n", iminor(inode));
+		return -ENODEV;
+	}
+
+	if (!device->rx_buffer) {
+		device->rx_buffer = kmalloc(MAX_MSG_SIZE, GFP_KERNEL);
+		if (!device->rx_buffer)
+		{
+			dev_dbg(device->dev, "open/ENOMEM\n");
+			return -ENOMEM;
+		}
+	}
+
+	device->users++;
+	instance = kzalloc(sizeof(*instance), GFP_KERNEL);
+	if (!instance)
+	{
+		kfree(device->rx_buffer);
+		device->rx_buffer = NULL;
+		return -ENOMEM;
+	}
+
+	/* setup instance data*/
+	instance->device = device;
+	instance->tx_cfg.bit_rate = 4711;
+	// TODO: fill instance->tx_cfg;
+
+	/* instance data as context */
+	filp->private_data = instance;
+	nonseekable_open(inode, filp);
+
+	return 0;
+}
+
+static int pi433_release(struct inode *inode, struct file *filp)
+{
+	struct pi433_instance	*instance;
+	struct pi433_device	*device;
+
+	instance = filp->private_data;
+	device = instance->device;
+	kfree(instance);
+	filp->private_data = NULL;
+
+	/* last close? */
+	device->users--;
+
+	if (!device->users) {
+		kfree(device->rx_buffer);
+		device->rx_buffer = NULL;
+		if (device->spi == NULL)
+			kfree(device);
+	}
+
+	return 0;
+}
+
+
+/*-------------------------------------------------------------------------*/
+
+static int setup_GPIOs(struct pi433_device *device)
+{
+	char 	name[5];
+	int	retval;
+	int	i;
+	const irq_handler_t DIO_irq_handler[NUM_DIO] = {
+		DIO0_irq_handler,
+		DIO1_irq_handler
+	};
+
+	for (i=0; i<NUM_DIO; i++)
+	{
+		/* "construct" name and get the gpio descriptor */
+		snprintf(name, sizeof(name), "DIO%d", i);
+		device->gpiod[i] = gpiod_get(&device->spi->dev, name, 0 /*GPIOD_IN*/);
+
+		if (device->gpiod[i] == ERR_PTR(-ENOENT))
+		{
+			dev_dbg(&device->spi->dev, "Could not find entry for %s. Ignoring.", name);
+			continue;
+		}
+
+		if (device->gpiod[i] == ERR_PTR(-EBUSY))
+			dev_dbg(&device->spi->dev, "%s is busy.", name);
+
+		if ( IS_ERR(device->gpiod[i]) )
+		{
+			retval = PTR_ERR(device->gpiod[i]);
+			/* release already allocated gpios */
+			for (i--; i>=0; i--)
+			{
+				free_irq(device->irq_num[i], device);
+				gpiod_put(device->gpiod[i]);
+			}
+			return retval;
+		}
+
+
+		/* configure the pin */
+		gpiod_unexport(device->gpiod[i]);
+		retval = gpiod_direction_input(device->gpiod[i]);
+		if (retval) return retval;
+
+
+		/* configure irq */
+		device->irq_num[i] = gpiod_to_irq(device->gpiod[i]);
+		if (device->irq_num[i] < 0)
+		{
+			device->gpiod[i] = ERR_PTR(-EINVAL);//(struct gpio_desc *)device->irq_num[i];
+			return device->irq_num[i];
+		}
+		retval = request_irq(device->irq_num[i],
+				     DIO_irq_handler[i],
+				     0, /* flags */
+				     name,
+				     device);
+
+		if (retval)
+			return retval;
+
+		dev_dbg(&device->spi->dev, "%s succesfully configured", name);
+	}
+
+	return 0;
+}
+
+static void free_GPIOs(struct pi433_device *device)
+{
+	int i;
+
+	for (i=0; i<NUM_DIO; i++)
+	{
+		/* check if gpiod is valid */
+		if ( IS_ERR(device->gpiod[i]) )
+			continue;
+
+		free_irq(device->irq_num[i], device);
+		gpiod_put(device->gpiod[i]);
+	}
+	return;
+}
+
+static int pi433_get_minor(struct pi433_device *device)
+{
+	int retval = -ENOMEM;
+
+	mutex_lock(&minor_lock);
+	retval = idr_alloc(&pi433_idr, device, 0, N_PI433_MINORS, GFP_KERNEL);
+	if (retval >= 0) {
+		device->minor = retval;
+		retval = 0;
+	} else if (retval == -ENOSPC) {
+		dev_err(device->dev, "too many pi433 devices\n");
+		retval = -EINVAL;
+	}
+	mutex_unlock(&minor_lock);
+	return retval;
+}
+
+static void pi433_free_minor(struct pi433_device *dev)
+{
+	mutex_lock(&minor_lock);
+	idr_remove(&pi433_idr, dev->minor);
+	mutex_unlock(&minor_lock);
+}
+/*-------------------------------------------------------------------------*/
+
+static const struct file_operations pi433_fops = {
+	.owner =	THIS_MODULE,
+	/* REVISIT switch to aio primitives, so that userspace
+	 * gets more complete API coverage.  It'll simplify things
+	 * too, except for the locking.
+	 */
+	.write =	pi433_write,
+	.read =		pi433_read,
+	.unlocked_ioctl = pi433_ioctl,
+	.compat_ioctl = pi433_compat_ioctl,
+	.open =		pi433_open,
+	.release =	pi433_release,
+	.llseek =	no_llseek,
+};
+
+/*-------------------------------------------------------------------------*/
+
+static int pi433_probe(struct spi_device *spi)
+{
+	struct pi433_device	*device;
+	int			retval;
+
+	/* setup spi parameters */
+	spi->mode = 0x00;
+	spi->bits_per_word = 8;
+	/* spi->max_speed_hz = 10000000;  1MHz already set by device tree overlay */
+
+	retval = spi_setup(spi);
+	if (retval)
+	{
+		dev_dbg(&spi->dev, "configuration of SPI interface failed!\n");
+		return retval;
+	}
+	else
+	{
+		dev_dbg(&spi->dev,
+			"spi interface setup: mode 0x%2x, %d bits per word, %dhz max speed",
+			spi->mode, spi->bits_per_word, spi->max_speed_hz);
+	}
+
+	/* Ping the chip by reading the version register */
+	retval = spi_w8r8(spi, 0x10);
+	if (retval < 0)
+		return retval;
+
+	switch (retval) {
+	case 0x24:
+		dev_dbg(&spi->dev, "found pi433 (ver. 0x%x)", retval);
+		break;
+	default:
+		dev_dbg(&spi->dev, "unknown chip version: 0x%x", retval);
+		return -ENODEV;
+	}
+
+	/* Allocate driver data */
+	device = kzalloc(sizeof(*device), GFP_KERNEL);
+	if (!device)
+		return -ENOMEM;
+
+	/* Initialize the driver data */
+	device->spi = spi;
+	device->rx_active = false;
+	device->tx_active = false;
+	device->interrupt_rx_allowed = false;
+
+	/* init wait queues */
+	init_waitqueue_head(&device->tx_wait_queue);
+	init_waitqueue_head(&device->rx_wait_queue);
+	init_waitqueue_head(&device->fifo_wait_queue);
+
+	/* init fifo */
+	INIT_KFIFO(device->tx_fifo);
+
+	/* init mutexes and locks */
+	mutex_init(&device->tx_fifo_lock);
+	mutex_init(&device->rx_lock);
+
+	/* setup GPIO (including irq_handler) for the different DIOs */
+	retval = setup_GPIOs(device);
+	if (retval)
+	{
+		dev_dbg(&spi->dev, "setup of GPIOs failed");
+		goto GPIO_failed;
+	}
+
+	/* setup the radio module */
+	SET_CHECKED(rf69_set_mode		(spi, standby));
+	SET_CHECKED(rf69_set_data_mode		(spi, packet));
+	SET_CHECKED(rf69_set_amplifier_0	(spi, optionOn));
+	SET_CHECKED(rf69_set_amplifier_1	(spi, optionOff));
+	SET_CHECKED(rf69_set_amplifier_2	(spi, optionOff));
+	SET_CHECKED(rf69_set_output_power_level	(spi, 13));
+	SET_CHECKED(rf69_set_antenna_impedance	(spi, fiftyOhm));
+
+	/* start tx thread */
+	device->tx_task_struct = kthread_run(pi433_tx_thread,
+					     device,
+					     "pi433_tx_task");
+	if (IS_ERR(device->tx_task_struct))
+	{
+		dev_dbg(device->dev, "start of send thread failed");
+		goto send_thread_failed;
+	}
+
+	/* determ minor number */
+	retval = pi433_get_minor(device);
+	if (retval)
+	{
+		dev_dbg(device->dev, "get of minor number failed");
+		goto minor_failed;
+	}
+
+	/* create device */
+	device->devt = MKDEV(MAJOR(pi433_dev), device->minor);
+	device->dev = device_create(pi433_class,
+				    &spi->dev,
+				    device->devt,
+				    device,
+				    "pi433");
+	if (IS_ERR(device->dev)) {
+		pr_err("pi433: device register failed\n");
+		retval = PTR_ERR(device->dev);
+		goto device_create_failed;
+	}
+	else {
+		dev_dbg(device->dev,
+			"created device for major %d, minor %d\n",
+			MAJOR(pi433_dev),
+			device->minor);
+	}
+
+	/* create cdev */
+	device->cdev = cdev_alloc();
+	device->cdev->owner = THIS_MODULE;
+	cdev_init(device->cdev, &pi433_fops);
+	retval = cdev_add(device->cdev, device->devt, 1);
+	if (retval)
+	{
+		dev_dbg(device->dev, "register of cdev failed");
+		goto cdev_failed;
+	}
+
+	/* spi setup */
+	spi_set_drvdata(spi, device);
+
+	return 0;
+
+cdev_failed:
+	device_destroy(pi433_class, device->devt);
+device_create_failed:
+	pi433_free_minor(device);
+minor_failed:
+	kthread_stop(device->tx_task_struct);
+send_thread_failed:
+	free_GPIOs(device);
+GPIO_failed:
+	kfree(device);
+
+	return retval;
+}
+
+static int pi433_remove(struct spi_device *spi)
+{
+	struct pi433_device	*device = spi_get_drvdata(spi);
+
+	/* free GPIOs */
+	free_GPIOs(device);
+
+	/* make sure ops on existing fds can abort cleanly */
+	device->spi = NULL;
+
+	kthread_stop(device->tx_task_struct);
+
+	device_destroy(pi433_class, device->devt);
+
+	cdev_del(device->cdev);
+
+	pi433_free_minor(device);
+
+	if (device->users == 0)
+		kfree(device);
+
+	return 0;
+}
+
+static const struct of_device_id pi433_dt_ids[] = {
+	{ .compatible = "Smarthome-Wolf,pi433" },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, pi433_dt_ids);
+
+static struct spi_driver pi433_spi_driver = {
+	.driver = {
+		.name =		"pi433",
+		.owner =	THIS_MODULE,
+		.of_match_table = of_match_ptr(pi433_dt_ids),
+	},
+	.probe =	pi433_probe,
+	.remove =	pi433_remove,
+
+	/* NOTE:  suspend/resume methods are not necessary here.
+	 * We don't do anything except pass the requests to/from
+	 * the underlying controller.  The refrigerator handles
+	 * most issues; the controller driver handles the rest.
+	 */
+};
+
+/*-------------------------------------------------------------------------*/
+
+static int __init pi433_init(void)
+{
+	int status;
+
+	/* If MAX_MSG_SIZE is smaller then FIFO_SIZE, the driver won't
+	 * work stable - risk of buffer overflow
+	 */
+	if (MAX_MSG_SIZE < FIFO_SIZE)
+		return -EINVAL;
+
+	/* Claim device numbers.  Then register a class
+	 * that will key udev/mdev to add/remove /dev nodes.  Last, register
+	 * Last, register the driver which manages those device numbers.
+	 */
+	status = alloc_chrdev_region(&pi433_dev, 0 /*firstminor*/, N_PI433_MINORS /*count*/, "pi433" /*name*/);
+	if (status < 0)
+		return status;
+
+	pi433_class = class_create(THIS_MODULE, "pi433");
+	if (IS_ERR(pi433_class))
+	{
+		unregister_chrdev(MAJOR(pi433_dev), pi433_spi_driver.driver.name);
+		return PTR_ERR(pi433_class);
+	}
+
+	status = spi_register_driver(&pi433_spi_driver);
+	if (status < 0)
+	{
+		class_destroy(pi433_class);
+		unregister_chrdev(MAJOR(pi433_dev), pi433_spi_driver.driver.name);
+	}
+
+	return status;
+}
+
+module_init(pi433_init);
+
+static void __exit pi433_exit(void)
+{
+	spi_unregister_driver(&pi433_spi_driver);
+	class_destroy(pi433_class);
+	unregister_chrdev(MAJOR(pi433_dev), pi433_spi_driver.driver.name);
+}
+module_exit(pi433_exit);
+
+MODULE_AUTHOR("Marcus Wolf, <linux@wolf-entwicklungen.de>");
+MODULE_DESCRIPTION("Driver for Pi433");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("spi:pi433");
diff --git a/drivers/staging/pi433/pi433_if.h b/drivers/staging/pi433/pi433_if.h
new file mode 100644
index 000000000000..e6ed3cd9b2e2
--- /dev/null
+++ b/drivers/staging/pi433/pi433_if.h
@@ -0,0 +1,152 @@
+/*
+ * include/linux/TODO
+ *
+ * userspace interface for pi433 radio module
+ *
+ * Pi433 is a 433MHz radio module for the Raspberry Pi.
+ * It is based on the HopeRf Module RFM69CW. Therefore inside of this
+ * driver, you'll find an abstraction of the rf69 chip.
+ *
+ * If needed, this driver could be extended, to also support other
+ * devices, basing on HopeRfs rf69.
+ *
+ * The driver can also be extended, to support other modules of
+ * HopeRf with a similar interace - e. g. RFM69HCW, RFM12, RFM95, ...
+ * Copyright (C) 2016 Wolf-Entwicklungen
+ *	Marcus Wolf <linux@wolf-entwicklungen.de>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef PI433_H
+#define PI433_H
+
+#include <linux/types.h>
+#include "rf69_enum.h"
+
+/*---------------------------------------------------------------------------*/
+
+
+/*---------------------------------------------------------------------------*/
+
+/* IOCTL structs and commands */
+
+/**
+ * struct pi433_tx_config - describes the configuration of the radio module for sending
+ * @frequency:
+ * @bit_rate:
+ * @modulation:
+ * @data_mode:
+ * @preamble_length:
+ * @sync_pattern:
+ * @tx_start_condition:
+ * @payload_length:
+ * @repetitions:
+ *
+ * ATTENTION:
+ * If the contents of 'pi433_tx_config' ever change
+ * incompatibly, then the ioctl number (see define below) must change.
+ *
+ * NOTE: struct layout is the same in 64bit and 32bit userspace.
+ */
+#define PI433_TX_CFG_IOCTL_NR 	0
+struct pi433_tx_cfg
+{
+	__u32			frequency;
+	__u16			bit_rate;
+	__u32			dev_frequency;
+	enum modulation		modulation;
+	enum modShaping		modShaping;
+
+	enum paRamp		pa_ramp;
+
+	enum txStartCondition	tx_start_condition;
+
+	__u16			repetitions;
+
+
+	/* packet format */
+	enum optionOnOff	enable_preamble;
+	enum optionOnOff	enable_sync;
+	enum optionOnOff	enable_length_byte;
+	enum optionOnOff	enable_address_byte;
+	enum optionOnOff	enable_crc;
+
+	__u16			preamble_length;
+	__u8			sync_length;
+	__u8			fixed_message_length;
+
+	__u8			sync_pattern[8];
+	__u8			address_byte;
+};
+
+
+/**
+ * struct pi433_rx_config - describes the configuration of the radio module for sending
+ * @frequency:
+ * @bit_rate:
+ * @modulation:
+ * @data_mode:
+ * @preamble_length:
+ * @sync_pattern:
+ * @tx_start_condition:
+ * @payload_length:
+ * @repetitions:
+ *
+ * ATTENTION:
+ * If the contents of 'pi433_rx_config' ever change
+ * incompatibly, then the ioctl number (see define below) must change
+ *
+ * NOTE: struct layout is the same in 64bit and 32bit userspace.
+ */
+#define PI433_RX_CFG_IOCTL_NR 	1
+struct pi433_rx_cfg {
+	__u32			frequency;
+	__u16			bit_rate;
+	__u32			dev_frequency;
+
+	enum modulation		modulation;
+
+	__u8			rssi_threshold;
+	enum thresholdDecrement	thresholdDecrement;
+	enum antennaImpedance	antenna_impedance;
+	enum lnaGain		lna_gain;
+	enum mantisse		bw_mantisse;	/* normal: 0x50 */
+	__u8			bw_exponent;	/* during AFC: 0x8b */
+	enum dagc		dagc;
+
+
+
+	/* packet format */
+	enum optionOnOff	enable_sync;
+	enum optionOnOff	enable_length_byte;	  /* should be used in combination with sync, only */
+	enum addressFiltering	enable_address_filtering; /* operational with sync, only */
+	enum optionOnOff	enable_crc;		  /* only operational, if sync on and fixed length or length byte is used */
+
+	__u8			sync_length;
+	__u8			fixed_message_length;
+	__u32			bytes_to_drop;
+
+	__u8			sync_pattern[8];
+	__u8			node_address;
+	__u8			broadcast_address;
+};
+
+
+#define PI433_IOC_MAGIC			'r'
+
+#define PI433_IOC_RD_TX_CFG	_IOR(PI433_IOC_MAGIC, PI433_TX_CFG_IOCTL_NR, char[sizeof(struct pi433_tx_cfg)])
+#define PI433_IOC_WR_TX_CFG	_IOW(PI433_IOC_MAGIC, PI433_TX_CFG_IOCTL_NR, char[sizeof(struct pi433_tx_cfg)])
+
+#define PI433_IOC_RD_RX_CFG	_IOR(PI433_IOC_MAGIC, PI433_RX_CFG_IOCTL_NR, char[sizeof(struct pi433_rx_cfg)])
+#define PI433_IOC_WR_RX_CFG	_IOW(PI433_IOC_MAGIC, PI433_RX_CFG_IOCTL_NR, char[sizeof(struct pi433_rx_cfg)])
+
+#endif /* PI433_H */
diff --git a/drivers/staging/pi433/rf69.c b/drivers/staging/pi433/rf69.c
new file mode 100644
index 000000000000..c4b1b218ea38
--- /dev/null
+++ b/drivers/staging/pi433/rf69.c
@@ -0,0 +1,1032 @@
+/*
+ * abstraction of the spi interface of HopeRf rf69 radio module
+ *
+ * Copyright (C) 2016 Wolf-Entwicklungen
+ *	Marcus Wolf <linux@wolf-entwicklungen.de>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+/* enable prosa debug info */
+#undef DEBUG
+/* enable print of values on reg access */
+#undef DEBUG_VALUES
+/* enable print of values on fifo access */
+#undef DEBUG_FIFO_ACCESS
+
+#include <linux/types.h>
+#include <linux/spi/spi.h>
+
+#include "rf69.h"
+#include "rf69_registers.h"
+
+#define F_OSC	  32000000 /* in Hz */
+#define FIFO_SIZE 66 	   /* in byte */
+
+/*-------------------------------------------------------------------------*/
+
+#define READ_REG(x)	rf69_read_reg (spi, x)
+#define WRITE_REG(x,y)	rf69_write_reg(spi, x, y)
+
+/*-------------------------------------------------------------------------*/
+
+int rf69_set_mode(struct spi_device *spi, enum mode mode)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: mode");
+	#endif
+
+	switch (mode) {
+	case transmit:	  return WRITE_REG(REG_OPMODE, (READ_REG(REG_OPMODE) & ~MASK_OPMODE_MODE) | OPMODE_MODE_TRANSMIT);
+	case receive:	  return WRITE_REG(REG_OPMODE, (READ_REG(REG_OPMODE) & ~MASK_OPMODE_MODE) | OPMODE_MODE_RECEIVE);
+	case synthesizer: return WRITE_REG(REG_OPMODE, (READ_REG(REG_OPMODE) & ~MASK_OPMODE_MODE) | OPMODE_MODE_SYNTHESIZER);
+	case standby:	  return WRITE_REG(REG_OPMODE, (READ_REG(REG_OPMODE) & ~MASK_OPMODE_MODE) | OPMODE_MODE_STANDBY);
+	case mode_sleep:  return WRITE_REG(REG_OPMODE, (READ_REG(REG_OPMODE) & ~MASK_OPMODE_MODE) | OPMODE_MODE_SLEEP);
+	default:
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+
+	// we are using packet mode, so this check is not really needed
+	// but waiting for mode ready is necessary when going from sleep because the FIFO may not be immediately available from previous mode
+	//while (_mode == RF69_MODE_SLEEP && (READ_REG(REG_IRQFLAGS1) & RF_IRQFLAGS1_MODEREADY) == 0x00); // Wait for ModeReady
+
+}
+
+int rf69_set_data_mode(struct spi_device *spi, enum dataMode dataMode)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: data mode");
+	#endif
+
+	switch (dataMode) {
+	case packet:		return WRITE_REG(REG_DATAMODUL, (READ_REG(REG_DATAMODUL) & ~MASK_DATAMODUL_MODE) | DATAMODUL_MODE_PACKET);
+	case continuous:	return WRITE_REG(REG_DATAMODUL, (READ_REG(REG_DATAMODUL) & ~MASK_DATAMODUL_MODE) | DATAMODUL_MODE_CONTINUOUS);
+	case continuousNoSync:  return WRITE_REG(REG_DATAMODUL, (READ_REG(REG_DATAMODUL) & ~MASK_DATAMODUL_MODE) | DATAMODUL_MODE_CONTINUOUS_NOSYNC);
+	default:
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+}
+
+int rf69_set_modulation(struct spi_device *spi, enum modulation modulation)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: modulation");
+	#endif
+
+	switch (modulation) {
+	case OOK:   return WRITE_REG(REG_DATAMODUL, (READ_REG(REG_DATAMODUL) & ~MASK_DATAMODUL_MODULATION_TYPE) | DATAMODUL_MODULATION_TYPE_OOK);
+	case FSK:   return WRITE_REG(REG_DATAMODUL, (READ_REG(REG_DATAMODUL) & ~MASK_DATAMODUL_MODULATION_TYPE) | DATAMODUL_MODULATION_TYPE_FSK);
+	default:
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+}
+
+enum modulation rf69_get_modulation(struct spi_device *spi)
+{
+	u8 currentValue;
+
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "get: mode");
+	#endif
+
+	currentValue = READ_REG(REG_DATAMODUL);
+
+	switch (currentValue & MASK_DATAMODUL_MODULATION_TYPE >> 3) { // TODO improvement: change 3 to define
+	case DATAMODUL_MODULATION_TYPE_OOK: return OOK;
+	case DATAMODUL_MODULATION_TYPE_FSK: return FSK;
+	default:			    return undefined;
+	}
+}
+
+int rf69_set_modulation_shaping(struct spi_device *spi, enum modShaping modShaping)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: mod shaping");
+	#endif
+
+	if (rf69_get_modulation(spi) == FSK) {
+		switch (modShaping) {
+		case shapingOff: return WRITE_REG(REG_DATAMODUL, (READ_REG(REG_DATAMODUL) & ~MASK_DATAMODUL_MODULATION_SHAPE) | DATAMODUL_MODULATION_SHAPE_NONE);
+		case shaping1_0: return WRITE_REG(REG_DATAMODUL, (READ_REG(REG_DATAMODUL) & ~MASK_DATAMODUL_MODULATION_SHAPE) | DATAMODUL_MODULATION_SHAPE_1_0);
+		case shaping0_5: return WRITE_REG(REG_DATAMODUL, (READ_REG(REG_DATAMODUL) & ~MASK_DATAMODUL_MODULATION_SHAPE) | DATAMODUL_MODULATION_SHAPE_0_3);
+		case shaping0_3: return WRITE_REG(REG_DATAMODUL, (READ_REG(REG_DATAMODUL) & ~MASK_DATAMODUL_MODULATION_SHAPE) | DATAMODUL_MODULATION_SHAPE_0_5);
+		default:
+			dev_dbg(&spi->dev, "set: illegal input param");
+			return -EINVAL;
+		}
+	} else {
+		switch (modShaping) {
+		case shapingOff: return WRITE_REG(REG_DATAMODUL, (READ_REG(REG_DATAMODUL) & ~MASK_DATAMODUL_MODULATION_SHAPE) | DATAMODUL_MODULATION_SHAPE_NONE);
+		case shapingBR:	 return WRITE_REG(REG_DATAMODUL, (READ_REG(REG_DATAMODUL) & ~MASK_DATAMODUL_MODULATION_SHAPE) | DATAMODUL_MODULATION_SHAPE_BR);
+		case shaping2BR: return WRITE_REG(REG_DATAMODUL, (READ_REG(REG_DATAMODUL) & ~MASK_DATAMODUL_MODULATION_SHAPE) | DATAMODUL_MODULATION_SHAPE_2BR);
+		default:
+			dev_dbg(&spi->dev, "set: illegal input param");
+			return -EINVAL;
+		}
+	}
+}
+
+int rf69_set_bit_rate(struct spi_device *spi, u16 bitRate)
+{
+	int retval;
+	u32 bitRate_min;
+	u32 bitRate_reg;
+	u8 msb;
+	u8 lsb;
+
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: bit rate");
+	#endif
+
+	// check input value
+	bitRate_min = F_OSC / 8388608; // 8388608 = 2^23;
+	if (bitRate < bitRate_min) {
+		dev_dbg(&spi->dev, "setBitRate: illegal input param");
+		return -EINVAL;
+	}
+
+	// calculate reg settings
+	bitRate_reg = (F_OSC / bitRate);
+
+	msb = (bitRate_reg&0xff00)   >>  8;
+	lsb = (bitRate_reg&0xff);
+
+	// transmit to RF 69
+	retval = WRITE_REG(REG_BITRATE_MSB, msb);
+	if (retval)  return retval;
+	retval = WRITE_REG(REG_BITRATE_LSB, lsb);
+	if (retval)  return retval;
+
+	return 0;
+}
+
+int rf69_set_deviation(struct spi_device *spi, u32 deviation)
+{
+	int retval;
+//	u32 f_max; TODO: Abhängigkeit von Bitrate beachten!!
+	u64 f_reg;
+	u64 f_step;
+	u8 msb;
+	u8 lsb;
+	u64 factor = 1000000; // to improve precision of calculation
+
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: deviation");
+	#endif
+
+	if (deviation < 600 || deviation > 500000) { //TODO: Abhängigkeit von Bitrate beachten!!
+		dev_dbg(&spi->dev, "set_deviation: illegal input param");
+		return -EINVAL;
+	}
+
+	// calculat f step
+	f_step = F_OSC * factor;
+	do_div(f_step, 524288); //  524288 = 2^19
+
+	// calculate register settings
+	f_reg = deviation * factor;
+	do_div(f_reg  , f_step);
+
+	msb = (f_reg&0xff00)   >>  8;
+	lsb = (f_reg&0xff);
+
+	// check msb
+	if (msb & ~FDEVMASB_MASK) {
+		dev_dbg(&spi->dev, "set_deviation: err in calc of msb");
+		return -EINVAL;
+	}
+
+	// write to chip
+	retval = WRITE_REG(REG_FDEV_MSB, msb);
+	if (retval)  return retval;
+	retval = WRITE_REG(REG_FDEV_LSB, lsb);
+	if (retval)  return retval;
+
+	return 0;
+}
+
+int rf69_set_frequency(struct spi_device *spi, u32 frequency)
+{
+	int retval;
+	u32 f_max;
+	u64 f_reg;
+	u64 f_step;
+	u8 msb;
+	u8 mid;
+	u8 lsb;
+	u64 factor = 1000000; // to improve precision of calculation
+
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: frequency");
+	#endif
+
+	// calculat f step
+	f_step = F_OSC * factor;
+	do_div(f_step, 524288); //  524288 = 2^19
+
+	// check input value
+	f_max = div_u64(f_step * 8388608, factor);
+	if (frequency > f_max) {
+		dev_dbg(&spi->dev, "setFrequency: illegal input param");
+		return -EINVAL;
+	}
+
+	// calculate reg settings
+	f_reg = frequency * factor;
+	do_div(f_reg  , f_step);
+
+	msb = (f_reg&0xff0000) >> 16;
+	mid = (f_reg&0xff00)   >>  8;
+	lsb = (f_reg&0xff);
+
+	// write to chip
+	retval = WRITE_REG(REG_FRF_MSB, msb);
+	if (retval)  return retval;
+	retval = WRITE_REG(REG_FRF_MID, mid);
+	if (retval)  return retval;
+	retval = WRITE_REG(REG_FRF_LSB, lsb);
+	if (retval)  return retval;
+
+	return 0;
+}
+
+int rf69_set_amplifier_0(struct spi_device *spi, enum optionOnOff optionOnOff)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: amp #0");
+	#endif
+
+	switch(optionOnOff) {
+	case optionOn:	return WRITE_REG(REG_PALEVEL, (READ_REG(REG_PALEVEL) |  MASK_PALEVEL_PA0) );
+	case optionOff:	return WRITE_REG(REG_PALEVEL, (READ_REG(REG_PALEVEL) & ~MASK_PALEVEL_PA0) );
+	default:
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+}
+
+int rf69_set_amplifier_1(struct spi_device *spi, enum optionOnOff optionOnOff)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: amp #1");
+	#endif
+
+	switch(optionOnOff) {
+	case optionOn:	return WRITE_REG(REG_PALEVEL, (READ_REG(REG_PALEVEL) |  MASK_PALEVEL_PA1) );
+	case optionOff: return WRITE_REG(REG_PALEVEL, (READ_REG(REG_PALEVEL) & ~MASK_PALEVEL_PA1) );
+	default:
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+}
+
+int rf69_set_amplifier_2(struct spi_device *spi, enum optionOnOff optionOnOff)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: amp #2");
+	#endif
+
+	switch(optionOnOff) {
+	case optionOn:	return WRITE_REG(REG_PALEVEL, (READ_REG(REG_PALEVEL) |  MASK_PALEVEL_PA2) );
+	case optionOff:	return WRITE_REG(REG_PALEVEL, (READ_REG(REG_PALEVEL) & ~MASK_PALEVEL_PA2) );
+	default:
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+}
+
+int rf69_set_output_power_level(struct spi_device *spi, u8 powerLevel)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: power level");
+	#endif
+
+	powerLevel +=18; // TODO Abhängigkeit von PA0,1,2 setting
+
+	// check input value
+	if (powerLevel > 0x1f) {
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+
+	// write value
+	return WRITE_REG(REG_PALEVEL, (READ_REG(REG_PALEVEL) & ~MASK_PALEVEL_OUTPUT_POWER) | powerLevel);
+}
+
+int rf69_set_pa_ramp(struct spi_device *spi, enum paRamp paRamp)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: pa ramp");
+	#endif
+
+	switch(paRamp) {
+	case ramp3400:	return WRITE_REG(REG_PARAMP, PARAMP_3400);
+	case ramp2000:	return WRITE_REG(REG_PARAMP, PARAMP_2000);
+	case ramp1000:	return WRITE_REG(REG_PARAMP, PARAMP_1000);
+	case ramp500:	return WRITE_REG(REG_PARAMP, PARAMP_500);
+	case ramp250:	return WRITE_REG(REG_PARAMP, PARAMP_250);
+	case ramp125:	return WRITE_REG(REG_PARAMP, PARAMP_125);
+	case ramp100:	return WRITE_REG(REG_PARAMP, PARAMP_100);
+	case ramp62:	return WRITE_REG(REG_PARAMP, PARAMP_62);
+	case ramp50:	return WRITE_REG(REG_PARAMP, PARAMP_50);
+	case ramp40:	return WRITE_REG(REG_PARAMP, PARAMP_40);
+	case ramp31:	return WRITE_REG(REG_PARAMP, PARAMP_31);
+	case ramp25:	return WRITE_REG(REG_PARAMP, PARAMP_25);
+	case ramp20:	return WRITE_REG(REG_PARAMP, PARAMP_20);
+	case ramp15:	return WRITE_REG(REG_PARAMP, PARAMP_15);
+	case ramp12:	return WRITE_REG(REG_PARAMP, PARAMP_12);
+	case ramp10:	return WRITE_REG(REG_PARAMP, PARAMP_10);
+	default:
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+}
+
+int rf69_set_antenna_impedance(struct spi_device *spi, enum antennaImpedance antennaImpedance)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: antenna impedance");
+	#endif
+
+	switch(antennaImpedance) {
+	case fiftyOhm:	    return WRITE_REG(REG_LNA, (READ_REG(REG_LNA) & ~MASK_LNA_ZIN) );
+	case twohundretOhm: return WRITE_REG(REG_LNA, (READ_REG(REG_LNA) |  MASK_LNA_ZIN) );
+	default:
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+}
+
+int rf69_set_lna_gain(struct spi_device *spi, enum lnaGain lnaGain)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: lna gain");
+	#endif
+
+	switch(lnaGain) {
+	case automatic:	 return WRITE_REG(REG_LNA, ( (READ_REG(REG_LNA) & ~MASK_LNA_GAIN) & LNA_GAIN_AUTO) );
+	case max:	 return WRITE_REG(REG_LNA, ( (READ_REG(REG_LNA) & ~MASK_LNA_GAIN) & LNA_GAIN_MAX) );
+	case maxMinus6:  return WRITE_REG(REG_LNA, ( (READ_REG(REG_LNA) & ~MASK_LNA_GAIN) & LNA_GAIN_MAX_MINUS_6) );
+	case maxMinus12: return WRITE_REG(REG_LNA, ( (READ_REG(REG_LNA) & ~MASK_LNA_GAIN) & LNA_GAIN_MAX_MINUS_12) );
+	case maxMinus24: return WRITE_REG(REG_LNA, ( (READ_REG(REG_LNA) & ~MASK_LNA_GAIN) & LNA_GAIN_MAX_MINUS_24) );
+	case maxMinus36: return WRITE_REG(REG_LNA, ( (READ_REG(REG_LNA) & ~MASK_LNA_GAIN) & LNA_GAIN_MAX_MINUS_36) );
+	case maxMinus48: return WRITE_REG(REG_LNA, ( (READ_REG(REG_LNA) & ~MASK_LNA_GAIN) & LNA_GAIN_MAX_MINUS_48) );
+	default:
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+}
+
+enum lnaGain rf69_get_lna_gain(struct spi_device *spi)
+{
+	u8 currentValue;
+
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "get: lna gain");
+	#endif
+
+	currentValue = READ_REG(REG_LNA);
+
+	switch (currentValue & MASK_LNA_CURRENT_GAIN >> 3) { // improvement: change 3 to define
+	case LNA_GAIN_AUTO:	    return automatic;
+	case LNA_GAIN_MAX:	    return max;
+	case LNA_GAIN_MAX_MINUS_6:  return maxMinus6;
+	case LNA_GAIN_MAX_MINUS_12: return maxMinus12;
+	case LNA_GAIN_MAX_MINUS_24: return maxMinus24;
+	case LNA_GAIN_MAX_MINUS_36: return maxMinus36;
+	case LNA_GAIN_MAX_MINUS_48: return maxMinus48;
+	default:		    return undefined;
+	}
+}
+
+int rf69_set_dc_cut_off_frequency_intern(struct spi_device *spi ,u8 reg, enum dccPercent dccPercent)
+{
+	switch (dccPercent) {
+	case dcc16Percent:	return WRITE_REG(reg, ( (READ_REG(reg) & ~MASK_BW_DCC_FREQ) | BW_DCC_16_PERCENT) );
+	case dcc8Percent:	return WRITE_REG(reg, ( (READ_REG(reg) & ~MASK_BW_DCC_FREQ) | BW_DCC_8_PERCENT) );
+	case dcc4Percent:	return WRITE_REG(reg, ( (READ_REG(reg) & ~MASK_BW_DCC_FREQ) | BW_DCC_4_PERCENT) );
+	case dcc2Percent:	return WRITE_REG(reg, ( (READ_REG(reg) & ~MASK_BW_DCC_FREQ) | BW_DCC_2_PERCENT) );
+	case dcc1Percent:	return WRITE_REG(reg, ( (READ_REG(reg) & ~MASK_BW_DCC_FREQ) | BW_DCC_1_PERCENT) );
+	case dcc0_5Percent:	return WRITE_REG(reg, ( (READ_REG(reg) & ~MASK_BW_DCC_FREQ) | BW_DCC_0_5_PERCENT) );
+	case dcc0_25Percent:	return WRITE_REG(reg, ( (READ_REG(reg) & ~MASK_BW_DCC_FREQ) | BW_DCC_0_25_PERCENT) );
+	case dcc0_125Percent:	return WRITE_REG(reg, ( (READ_REG(reg) & ~MASK_BW_DCC_FREQ) | BW_DCC_0_125_PERCENT) );
+	default:
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+}
+
+int rf69_set_dc_cut_off_frequency(struct spi_device *spi, enum dccPercent dccPercent)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: cut off freq");
+	#endif
+
+	return rf69_set_dc_cut_off_frequency_intern(spi, REG_RXBW, dccPercent);
+}
+
+int rf69_set_dc_cut_off_frequency_during_afc(struct spi_device *spi, enum dccPercent dccPercent)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: cut off freq during afc");
+	#endif
+
+	return rf69_set_dc_cut_off_frequency_intern(spi, REG_AFCBW, dccPercent);
+}
+
+static int rf69_set_bandwidth_intern(struct spi_device *spi, u8 reg,
+				     enum mantisse mantisse, u8 exponent)
+{
+	u8 newValue;
+
+	// check value for mantisse and exponent
+	if (exponent > 7) {
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+
+	if ((mantisse != mantisse16) &&
+	    (mantisse != mantisse20) &&
+	    (mantisse != mantisse24)) {
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+
+	// read old value
+	newValue = READ_REG(reg);
+
+	// "delete" mantisse and exponent = just keep the DCC setting
+	newValue = newValue & MASK_BW_DCC_FREQ;
+
+	// add new mantisse
+	switch(mantisse) {
+	case mantisse16: newValue = newValue | BW_MANT_16;	break;
+	case mantisse20: newValue = newValue | BW_MANT_20;	break;
+	case mantisse24: newValue = newValue | BW_MANT_24;	break;
+	}
+
+	// add new exponent
+	newValue = newValue | exponent;
+
+	// write back
+	return WRITE_REG(reg, newValue);
+}
+
+int rf69_set_bandwidth(struct spi_device *spi, enum mantisse mantisse, u8 exponent)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: band width");
+	#endif
+
+	return rf69_set_bandwidth_intern(spi, REG_RXBW, mantisse, exponent);
+}
+
+int rf69_set_bandwidth_during_afc(struct spi_device *spi, enum mantisse mantisse, u8 exponent)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: band width during afc");
+	#endif
+
+	return rf69_set_bandwidth_intern(spi, REG_AFCBW, mantisse, exponent);
+}
+
+int rf69_set_ook_threshold_type(struct spi_device *spi, enum thresholdType thresholdType)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: threshold type");
+	#endif
+
+	switch (thresholdType) {
+	case fixed:	return WRITE_REG(REG_OOKPEAK, ( (READ_REG(REG_OOKPEAK) & ~MASK_OOKPEAK_THRESTYPE) | OOKPEAK_THRESHTYPE_FIXED) );
+	case peak:	return WRITE_REG(REG_OOKPEAK, ( (READ_REG(REG_OOKPEAK) & ~MASK_OOKPEAK_THRESTYPE) | OOKPEAK_THRESHTYPE_PEAK) );
+	case average:	return WRITE_REG(REG_OOKPEAK, ( (READ_REG(REG_OOKPEAK) & ~MASK_OOKPEAK_THRESTYPE) | OOKPEAK_THRESHTYPE_AVERAGE) );
+	default:
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+}
+
+int rf69_set_ook_threshold_step(struct spi_device *spi, enum thresholdStep thresholdStep)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: threshold step");
+	#endif
+
+	switch (thresholdStep) {
+	case step_0_5db: return WRITE_REG(REG_OOKPEAK, ( (READ_REG(REG_OOKPEAK) & ~MASK_OOKPEAK_THRESSTEP) | OOKPEAK_THRESHSTEP_0_5_DB) );
+	case step_1_0db: return WRITE_REG(REG_OOKPEAK, ( (READ_REG(REG_OOKPEAK) & ~MASK_OOKPEAK_THRESSTEP) | OOKPEAK_THRESHSTEP_1_0_DB) );
+	case step_1_5db: return WRITE_REG(REG_OOKPEAK, ( (READ_REG(REG_OOKPEAK) & ~MASK_OOKPEAK_THRESSTEP) | OOKPEAK_THRESHSTEP_1_5_DB) );
+	case step_2_0db: return WRITE_REG(REG_OOKPEAK, ( (READ_REG(REG_OOKPEAK) & ~MASK_OOKPEAK_THRESSTEP) | OOKPEAK_THRESHSTEP_2_0_DB) );
+	case step_3_0db: return WRITE_REG(REG_OOKPEAK, ( (READ_REG(REG_OOKPEAK) & ~MASK_OOKPEAK_THRESSTEP) | OOKPEAK_THRESHSTEP_3_0_DB) );
+	case step_4_0db: return WRITE_REG(REG_OOKPEAK, ( (READ_REG(REG_OOKPEAK) & ~MASK_OOKPEAK_THRESSTEP) | OOKPEAK_THRESHSTEP_4_0_DB) );
+	case step_5_0db: return WRITE_REG(REG_OOKPEAK, ( (READ_REG(REG_OOKPEAK) & ~MASK_OOKPEAK_THRESSTEP) | OOKPEAK_THRESHSTEP_5_0_DB) );
+	case step_6_0db: return WRITE_REG(REG_OOKPEAK, ( (READ_REG(REG_OOKPEAK) & ~MASK_OOKPEAK_THRESSTEP) | OOKPEAK_THRESHSTEP_6_0_DB) );
+	default:
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+}
+
+int rf69_set_ook_threshold_dec(struct spi_device *spi, enum thresholdDecrement thresholdDecrement)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: threshold decrement");
+	#endif
+
+	switch (thresholdDecrement) {
+	case dec_every8th: return WRITE_REG(REG_OOKPEAK, ( (READ_REG(REG_OOKPEAK) & ~MASK_OOKPEAK_THRESDEC) | OOKPEAK_THRESHDEC_EVERY_8TH) );
+	case dec_every4th: return WRITE_REG(REG_OOKPEAK, ( (READ_REG(REG_OOKPEAK) & ~MASK_OOKPEAK_THRESDEC) | OOKPEAK_THRESHDEC_EVERY_4TH) );
+	case dec_every2nd: return WRITE_REG(REG_OOKPEAK, ( (READ_REG(REG_OOKPEAK) & ~MASK_OOKPEAK_THRESDEC) | OOKPEAK_THRESHDEC_EVERY_2ND) );
+	case dec_once:	   return WRITE_REG(REG_OOKPEAK, ( (READ_REG(REG_OOKPEAK) & ~MASK_OOKPEAK_THRESDEC) | OOKPEAK_THRESHDEC_ONCE) );
+	case dec_twice:	   return WRITE_REG(REG_OOKPEAK, ( (READ_REG(REG_OOKPEAK) & ~MASK_OOKPEAK_THRESDEC) | OOKPEAK_THRESHDEC_TWICE) );
+	case dec_4times:   return WRITE_REG(REG_OOKPEAK, ( (READ_REG(REG_OOKPEAK) & ~MASK_OOKPEAK_THRESDEC) | OOKPEAK_THRESHDEC_4_TIMES) );
+	case dec_8times:   return WRITE_REG(REG_OOKPEAK, ( (READ_REG(REG_OOKPEAK) & ~MASK_OOKPEAK_THRESDEC) | OOKPEAK_THRESHDEC_8_TIMES) );
+	case dec_16times:  return WRITE_REG(REG_OOKPEAK, ( (READ_REG(REG_OOKPEAK) & ~MASK_OOKPEAK_THRESDEC) | OOKPEAK_THRESHDEC_16_TIMES) );
+	default:
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+}
+
+int rf69_set_dio_mapping(struct spi_device *spi, u8 DIONumber, u8 value)
+{
+	u8 mask;
+	u8 shift;
+	u8 regaddr;
+	u8 regValue;
+
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: DIO mapping");
+	#endif
+
+	// check DIO number
+	if (DIONumber > 5) {
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+
+	switch (DIONumber) {
+	case 0: mask=MASK_DIO0; shift=SHIFT_DIO0; regaddr=REG_DIOMAPPING1; break;
+	case 1: mask=MASK_DIO1; shift=SHIFT_DIO1; regaddr=REG_DIOMAPPING1; break;
+	case 2: mask=MASK_DIO2; shift=SHIFT_DIO2; regaddr=REG_DIOMAPPING1; break;
+	case 3: mask=MASK_DIO3; shift=SHIFT_DIO3; regaddr=REG_DIOMAPPING1; break;
+	case 4: mask=MASK_DIO4; shift=SHIFT_DIO4; regaddr=REG_DIOMAPPING2; break;
+	case 5: mask=MASK_DIO5; shift=SHIFT_DIO5; regaddr=REG_DIOMAPPING2; break;
+	}
+
+	// read reg
+	regValue=READ_REG(regaddr);
+	// delete old value
+	regValue = regValue & ~mask;
+	// add new value
+	regValue = regValue | value << shift;
+	// write back
+	return WRITE_REG(regaddr,regValue);
+}
+
+bool rf69_get_flag(struct spi_device *spi, enum flag flag)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "get: flag");
+	#endif
+
+	switch(flag) {
+	case modeSwitchCompleted:     return (READ_REG(REG_IRQFLAGS1) & MASK_IRQFLAGS1_MODE_READY);
+	case readyToReceive:	      return (READ_REG(REG_IRQFLAGS1) & MASK_IRQFLAGS1_RX_READY);
+	case readyToSend:	      return (READ_REG(REG_IRQFLAGS1) & MASK_IRQFLAGS1_TX_READY);
+	case pllLocked:		      return (READ_REG(REG_IRQFLAGS1) & MASK_IRQFLAGS1_PLL_LOCK);
+	case rssiExceededThreshold:   return (READ_REG(REG_IRQFLAGS1) & MASK_IRQFLAGS1_RSSI);
+	case timeout:		      return (READ_REG(REG_IRQFLAGS1) & MASK_IRQFLAGS1_TIMEOUT);
+	case automode:		      return (READ_REG(REG_IRQFLAGS1) & MASK_IRQFLAGS1_AUTOMODE);
+	case syncAddressMatch:	      return (READ_REG(REG_IRQFLAGS1) & MASK_IRQFLAGS1_SYNC_ADDRESS_MATCH);
+	case fifoFull:		      return (READ_REG(REG_IRQFLAGS2) & MASK_IRQFLAGS2_FIFO_FULL);
+/*	case fifoNotEmpty:	      return (READ_REG(REG_IRQFLAGS2) & MASK_IRQFLAGS2_FIFO_NOT_EMPTY); */
+	case fifoEmpty:		      return !(READ_REG(REG_IRQFLAGS2) & MASK_IRQFLAGS2_FIFO_NOT_EMPTY);
+	case fifoLevelBelowThreshold: return (READ_REG(REG_IRQFLAGS2) & MASK_IRQFLAGS2_FIFO_LEVEL);
+	case fifoOverrun:	      return (READ_REG(REG_IRQFLAGS2) & MASK_IRQFLAGS2_FIFO_OVERRUN);
+	case packetSent:	      return (READ_REG(REG_IRQFLAGS2) & MASK_IRQFLAGS2_PACKET_SENT);
+	case payloadReady:	      return (READ_REG(REG_IRQFLAGS2) & MASK_IRQFLAGS2_PAYLOAD_READY);
+	case crcOk:		      return (READ_REG(REG_IRQFLAGS2) & MASK_IRQFLAGS2_CRC_OK);
+	case batteryLow:	      return (READ_REG(REG_IRQFLAGS2) & MASK_IRQFLAGS2_LOW_BAT);
+	default:		      return false;
+	}
+}
+
+int rf69_reset_flag(struct spi_device *spi, enum flag flag)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "reset: flag");
+	#endif
+
+	switch(flag) {
+	case rssiExceededThreshold: return WRITE_REG(REG_IRQFLAGS1, MASK_IRQFLAGS1_RSSI);
+	case syncAddressMatch:	    return WRITE_REG(REG_IRQFLAGS1, MASK_IRQFLAGS1_SYNC_ADDRESS_MATCH);
+	case fifoOverrun:	    return WRITE_REG(REG_IRQFLAGS2, MASK_IRQFLAGS2_FIFO_OVERRUN);
+	default:
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+}
+
+int rf69_set_rssi_threshold(struct spi_device *spi, u8 threshold)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: rssi threshold");
+	#endif
+
+	/* no value check needed - u8 exactly matches register size */
+
+	return WRITE_REG(REG_RSSITHRESH, threshold);
+}
+
+int rf69_set_rx_start_timeout(struct spi_device *spi, u8 timeout)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: start timeout");
+	#endif
+
+	/* no value check needed - u8 exactly matches register size */
+
+	return WRITE_REG(REG_RXTIMEOUT1, timeout);
+}
+
+int rf69_set_rssi_timeout(struct spi_device *spi, u8 timeout)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: rssi timeout");
+	#endif
+
+	/* no value check needed - u8 exactly matches register size */
+
+	return WRITE_REG(REG_RXTIMEOUT2, timeout);
+}
+
+int rf69_set_preamble_length(struct spi_device *spi, u16 preambleLength)
+{
+	int retval;
+	u8 msb, lsb;
+
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: preamble length");
+	#endif
+
+	/* no value check needed - u16 exactly matches register size */
+
+	/* calculate reg settings */
+	msb = (preambleLength&0xff00)   >>  8;
+	lsb = (preambleLength&0xff);
+
+	/* transmit to chip */
+	retval = WRITE_REG(REG_PREAMBLE_MSB, msb);
+	if (retval) return retval;
+	retval = WRITE_REG(REG_PREAMBLE_LSB, lsb);
+
+	return retval;
+}
+
+int rf69_set_sync_enable(struct spi_device *spi, enum optionOnOff optionOnOff)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: sync enable");
+	#endif
+
+	switch(optionOnOff) {
+	case optionOn:	return WRITE_REG(REG_SYNC_CONFIG, (READ_REG(REG_SYNC_CONFIG) |  MASK_SYNC_CONFIG_SYNC_ON) );
+	case optionOff:	return WRITE_REG(REG_SYNC_CONFIG, (READ_REG(REG_SYNC_CONFIG) & ~MASK_SYNC_CONFIG_SYNC_ON) );
+	default:
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+}
+
+int rf69_set_fifo_fill_condition(struct spi_device *spi, enum fifoFillCondition fifoFillCondition)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: fifo fill condition");
+	#endif
+
+	switch(fifoFillCondition) {
+	case always:		 return WRITE_REG(REG_SYNC_CONFIG, (READ_REG(REG_SYNC_CONFIG) |  MASK_SYNC_CONFIG_FIFO_FILL_CONDITION) );
+	case afterSyncInterrupt: return WRITE_REG(REG_SYNC_CONFIG, (READ_REG(REG_SYNC_CONFIG) & ~MASK_SYNC_CONFIG_FIFO_FILL_CONDITION) );
+	default:
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+}
+
+int rf69_set_sync_size(struct spi_device *spi, u8 syncSize)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: sync size");
+	#endif
+
+	// check input value
+	if (syncSize > 0x07) {
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+
+	// write value
+	return WRITE_REG(REG_SYNC_CONFIG, (READ_REG(REG_SYNC_CONFIG) & ~MASK_SYNC_CONFIG_SYNC_SIZE) | (syncSize << 3) );
+}
+
+int rf69_set_sync_tolerance(struct spi_device *spi, u8 syncTolerance)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: sync tolerance");
+	#endif
+
+	// check input value
+	if (syncTolerance > 0x07) {
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+
+	// write value
+	return WRITE_REG(REG_SYNC_CONFIG, (READ_REG(REG_SYNC_CONFIG) & ~MASK_SYNC_CONFIG_SYNC_SIZE) | syncTolerance);
+}
+
+int rf69_set_sync_values(struct spi_device *spi, u8 syncValues[8])
+{
+	int retval = 0;
+
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: sync values");
+	#endif
+
+	retval += WRITE_REG(REG_SYNCVALUE1, syncValues[0]);
+	retval += WRITE_REG(REG_SYNCVALUE2, syncValues[1]);
+	retval += WRITE_REG(REG_SYNCVALUE3, syncValues[2]);
+	retval += WRITE_REG(REG_SYNCVALUE4, syncValues[3]);
+	retval += WRITE_REG(REG_SYNCVALUE5, syncValues[4]);
+	retval += WRITE_REG(REG_SYNCVALUE6, syncValues[5]);
+	retval += WRITE_REG(REG_SYNCVALUE7, syncValues[6]);
+	retval += WRITE_REG(REG_SYNCVALUE8, syncValues[7]);
+
+	return retval;
+}
+
+int rf69_set_packet_format(struct spi_device *spi, enum packetFormat packetFormat)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: packet format");
+	#endif
+
+	switch(packetFormat) {
+	case packetLengthVar: return WRITE_REG(REG_PACKETCONFIG1, (READ_REG(REG_PACKETCONFIG1) |  MASK_PACKETCONFIG1_PAKET_FORMAT_VARIABLE) );
+	case packetLengthFix: return WRITE_REG(REG_PACKETCONFIG1, (READ_REG(REG_PACKETCONFIG1) & ~MASK_PACKETCONFIG1_PAKET_FORMAT_VARIABLE) );
+	default:
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+}
+
+int rf69_set_crc_enable(struct spi_device *spi, enum optionOnOff optionOnOff)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: crc enable");
+	#endif
+
+	switch(optionOnOff) {
+	case optionOn:	return WRITE_REG(REG_PACKETCONFIG1, (READ_REG(REG_PACKETCONFIG1) |  MASK_PACKETCONFIG1_CRC_ON) );
+	case optionOff:	return WRITE_REG(REG_PACKETCONFIG1, (READ_REG(REG_PACKETCONFIG1) & ~MASK_PACKETCONFIG1_CRC_ON) );
+	default:
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+}
+
+int rf69_set_adressFiltering(struct spi_device *spi, enum addressFiltering addressFiltering)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: address filtering");
+	#endif
+
+	switch (addressFiltering) {
+	case filteringOff:	     return WRITE_REG(REG_PACKETCONFIG1, ( (READ_REG(REG_PACKETCONFIG1) & ~MASK_PACKETCONFIG1_ADDRESSFILTERING) | PACKETCONFIG1_ADDRESSFILTERING_OFF) );
+	case nodeAddress:	     return WRITE_REG(REG_PACKETCONFIG1, ( (READ_REG(REG_PACKETCONFIG1) & ~MASK_PACKETCONFIG1_ADDRESSFILTERING) | PACKETCONFIG1_ADDRESSFILTERING_NODE) );
+	case nodeOrBroadcastAddress: return WRITE_REG(REG_PACKETCONFIG1, ( (READ_REG(REG_PACKETCONFIG1) & ~MASK_PACKETCONFIG1_ADDRESSFILTERING) | PACKETCONFIG1_ADDRESSFILTERING_NODEBROADCAST) );
+	default:
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+}
+
+int rf69_set_payload_length(struct spi_device *spi, u8 payloadLength)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: payload length");
+	#endif
+
+	return WRITE_REG(REG_PAYLOAD_LENGTH, payloadLength);
+}
+
+u8  rf69_get_payload_length(struct spi_device *spi)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "get: payload length");
+	#endif
+
+	return (u8) READ_REG(REG_PAYLOAD_LENGTH);
+}
+
+int rf69_set_node_address(struct spi_device *spi, u8 nodeAddress)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: node address");
+	#endif
+
+	return WRITE_REG(REG_NODEADRS, nodeAddress);
+}
+
+int rf69_set_broadcast_address(struct spi_device *spi, u8 broadcastAddress)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: broadcast address");
+	#endif
+
+	return WRITE_REG(REG_BROADCASTADRS, broadcastAddress);
+}
+
+int rf69_set_tx_start_condition(struct spi_device *spi, enum txStartCondition txStartCondition)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: start condition");
+	#endif
+
+	switch(txStartCondition) {
+	case fifoLevel:	   return WRITE_REG(REG_FIFO_THRESH, (READ_REG(REG_FIFO_THRESH) & ~MASK_FIFO_THRESH_TXSTART) );
+	case fifoNotEmpty: return WRITE_REG(REG_FIFO_THRESH, (READ_REG(REG_FIFO_THRESH) |  MASK_FIFO_THRESH_TXSTART) );
+	default:
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+}
+
+int rf69_set_fifo_threshold(struct spi_device *spi, u8 threshold)
+{
+	int retval;
+
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: fifo threshold");
+	#endif
+
+	// check input value
+	if (threshold & 0x80) {
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+
+	// write value
+	retval = WRITE_REG(REG_FIFO_THRESH, (READ_REG(REG_FIFO_THRESH) & ~MASK_FIFO_THRESH_VALUE) | threshold);
+	if (retval)
+		return retval;
+
+	// access the fifo to activate new threshold
+	return rf69_read_fifo (spi, (u8*) &retval, 1); // retval used as buffer
+}
+
+int rf69_set_dagc(struct spi_device *spi, enum dagc dagc)
+{
+	#ifdef DEBUG
+		dev_dbg(&spi->dev, "set: dagc");
+	#endif
+
+	switch(dagc) {
+	case normalMode:		 return WRITE_REG(REG_TESTDAGC, DAGC_NORMAL);
+	case improve:			 return WRITE_REG(REG_TESTDAGC, DAGC_IMPROVED_LOWBETA0);
+	case improve4LowModulationIndex: return WRITE_REG(REG_TESTDAGC, DAGC_IMPROVED_LOWBETA1);
+	default:
+		dev_dbg(&spi->dev, "set: illegal input param");
+		return -EINVAL;
+	}
+}
+
+/*-------------------------------------------------------------------------*/
+
+int rf69_read_fifo (struct spi_device *spi, u8 *buffer, unsigned int size)
+{
+	#ifdef DEBUG_FIFO_ACCESS
+		int i;
+	#endif
+	struct spi_transfer transfer;
+	u8 local_buffer[FIFO_SIZE + 1];
+	int retval;
+
+	if (size > FIFO_SIZE) {
+		#ifdef DEBUG
+			dev_dbg(&spi->dev, "read fifo: passed in buffer bigger then internal buffer \n");
+		#endif
+		return -EMSGSIZE;
+	}
+
+	/* prepare a bidirectional transfer */
+	local_buffer[0] = REG_FIFO;
+	memset(&transfer, 0, sizeof(transfer));
+  	transfer.tx_buf = local_buffer;
+  	transfer.rx_buf = local_buffer;
+	transfer.len	= size+1;
+
+	retval = spi_sync_transfer(spi, &transfer, 1);
+
+	#ifdef DEBUG_FIFO_ACCESS
+		for (i=0; i<size; i++)
+			dev_dbg(&spi->dev, "%d - 0x%x\n", i, local_buffer[i+1]);
+	#endif
+
+	memcpy(buffer, &local_buffer[1], size);  // TODO: ohne memcopy wäre schöner
+
+	return retval;
+}
+
+int rf69_write_fifo(struct spi_device *spi, u8 *buffer, unsigned int size)
+{
+	#ifdef DEBUG_FIFO_ACCESS
+		int i;
+	#endif
+	char spi_address = REG_FIFO | WRITE_BIT;
+	u8 local_buffer[FIFO_SIZE + 1];
+
+	if (size > FIFO_SIZE) {
+		#ifdef DEBUG
+			dev_dbg(&spi->dev, "read fifo: passed in buffer bigger then internal buffer \n");
+		#endif
+		return -EMSGSIZE;
+	}
+
+	local_buffer[0] = spi_address;
+	memcpy(&local_buffer[1], buffer, size);  // TODO: ohne memcopy wäre schöner
+
+	#ifdef DEBUG_FIFO_ACCESS
+		for (i=0; i<size; i++)
+			dev_dbg(&spi->dev, "0x%x\n",buffer[i]);
+	#endif
+
+	return spi_write (spi, local_buffer, size + 1);
+}
+
+/*-------------------------------------------------------------------------*/
+
+u8 rf69_read_reg(struct spi_device *spi, u8 addr)
+{
+	int retval;
+
+	retval = spi_w8r8(spi, addr);
+
+	#ifdef DEBUG_VALUES
+		if (retval < 0)
+			/* should never happen, since we already checked,
+			 * that module is connected. Therefore no error
+			 * handling, just an optional error message...
+			 */
+			dev_dbg(&spi->dev, "read 0x%x FAILED\n",
+				addr);
+		else
+			dev_dbg(&spi->dev, "read 0x%x from reg 0x%x\n",
+				retval,
+				addr);
+	#endif
+
+	return retval;
+}
+
+int rf69_write_reg(struct spi_device *spi, u8 addr, u8 value)
+{
+	int retval;
+	char buffer[2];
+
+	buffer[0] = addr | WRITE_BIT;
+	buffer[1] = value;
+
+	retval = spi_write(spi, &buffer, 2);
+
+	#ifdef DEBUG_VALUES
+		if (retval < 0)
+			/* should never happen, since we already checked,
+			 * that module is connected. Therefore no error
+			 * handling, just an optional error message...
+			 */
+			dev_dbg(&spi->dev, "write 0x%x to 0x%x FAILED\n",
+				value,
+				addr);
+		else
+			dev_dbg(&spi->dev, "wrote 0x%x to reg 0x%x\n",
+				value,
+				addr);
+	#endif
+
+	return retval;
+}
+
+
diff --git a/drivers/staging/pi433/rf69.h b/drivers/staging/pi433/rf69.h
new file mode 100644
index 000000000000..5c0c95628f2f
--- /dev/null
+++ b/drivers/staging/pi433/rf69.h
@@ -0,0 +1,82 @@
+/*
+ * hardware abstraction/register access for HopeRf rf69 radio module
+ *
+ * Copyright (C) 2016 Wolf-Entwicklungen
+ *	Marcus Wolf <linux@wolf-entwicklungen.de>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#ifndef RF69_H
+#define RF69_H
+
+#include "rf69_enum.h"
+#include "rf69_registers.h"
+
+#define F_OSC		32000000  /* in Hz */
+#define FREQUENCY	433920000 /* in Hz, modifying this value impacts CE certification */
+#define FIFO_SIZE	66		/* in byte */
+#define FIFO_THRESHOLD	15		/* in byte */
+
+int rf69_set_mode(struct spi_device *spi, enum mode mode);
+int rf69_set_data_mode(struct spi_device *spi, enum dataMode dataMode);
+int rf69_set_modulation(struct spi_device *spi, enum modulation modulation);
+enum modulation rf69_get_modulation(struct spi_device *spi);
+int rf69_set_modulation_shaping(struct spi_device *spi, enum modShaping modShaping);
+int rf69_set_bit_rate(struct spi_device *spi, u16 bitRate);
+int rf69_set_deviation(struct spi_device *spi, u32 deviation);
+int rf69_set_frequency(struct spi_device *spi, u32 frequency);
+int rf69_set_amplifier_0(struct spi_device *spi, enum optionOnOff optionOnOff);
+int rf69_set_amplifier_1(struct spi_device *spi, enum optionOnOff optionOnOff);
+int rf69_set_amplifier_2(struct spi_device *spi, enum optionOnOff optionOnOff);
+int rf69_set_output_power_level(struct spi_device *spi, u8 powerLevel);
+int rf69_set_pa_ramp(struct spi_device *spi, enum paRamp paRamp);
+int rf69_set_antenna_impedance(struct spi_device *spi, enum antennaImpedance antennaImpedance);
+int rf69_set_lna_gain(struct spi_device *spi, enum lnaGain lnaGain);
+enum lnaGain rf69_get_lna_gain(struct spi_device *spi);
+int rf69_set_dc_cut_off_frequency_intern(struct spi_device *spi, u8 reg, enum dccPercent dccPercent);
+int rf69_set_dc_cut_off_frequency(struct spi_device *spi, enum dccPercent dccPercent);
+int rf69_set_dc_cut_off_frequency_during_afc(struct spi_device *spi, enum dccPercent dccPercent);
+int rf69_set_bandwidth(struct spi_device *spi, enum mantisse mantisse, u8 exponent);
+int rf69_set_bandwidth_during_afc(struct spi_device *spi, enum mantisse mantisse, u8 exponent);
+int rf69_set_ook_threshold_type(struct spi_device *spi, enum thresholdType thresholdType);
+int rf69_set_ook_threshold_step(struct spi_device *spi, enum thresholdStep thresholdStep);
+int rf69_set_ook_threshold_dec(struct spi_device *spi, enum thresholdDecrement thresholdDecrement);
+int rf69_set_dio_mapping(struct spi_device *spi, u8 DIONumber, u8 value);
+bool rf69_get_flag(struct spi_device *spi, enum flag flag);
+int rf69_reset_flag(struct spi_device *spi, enum flag flag);
+int rf69_set_rssi_threshold(struct spi_device *spi, u8 threshold);
+int rf69_set_rx_start_timeout(struct spi_device *spi, u8 timeout);
+int rf69_set_rssi_timeout(struct spi_device *spi, u8 timeout);
+int rf69_set_preamble_length(struct spi_device *spi, u16 preambleLength);
+int rf69_set_sync_enable(struct spi_device *spi, enum optionOnOff optionOnOff);
+int rf69_set_fifo_fill_condition(struct spi_device *spi, enum fifoFillCondition fifoFillCondition);
+int rf69_set_sync_size(struct spi_device *spi, u8 sync_size);
+int rf69_set_sync_tolerance(struct spi_device *spi, u8 syncTolerance);
+int rf69_set_sync_values(struct spi_device *spi, u8 syncValues[8]);
+int rf69_set_packet_format(struct spi_device *spi, enum packetFormat packetFormat);
+int rf69_set_crc_enable(struct spi_device *spi, enum optionOnOff optionOnOff);
+int rf69_set_adressFiltering(struct spi_device *spi, enum addressFiltering addressFiltering);
+int rf69_set_payload_length(struct spi_device *spi, u8 payloadLength);
+u8  rf69_get_payload_length(struct spi_device *spi);
+int rf69_set_node_address(struct spi_device *spi, u8 nodeAddress);
+int rf69_set_broadcast_address(struct spi_device *spi, u8 broadcastAddress);
+int rf69_set_tx_start_condition(struct spi_device *spi, enum txStartCondition txStartCondition);
+int rf69_set_fifo_threshold(struct spi_device *spi, u8 threshold);
+int rf69_set_dagc(struct spi_device *spi, enum dagc dagc);
+
+int rf69_read_fifo (struct spi_device *spi, u8 *buffer, unsigned int size);
+int rf69_write_fifo(struct spi_device *spi, u8 *buffer, unsigned int size);
+
+u8  rf69_read_reg (struct spi_device *spi, u8 addr);
+int rf69_write_reg(struct spi_device *spi, u8 addr, u8 value);
+
+
+#endif
diff --git a/drivers/staging/pi433/rf69_enum.h b/drivers/staging/pi433/rf69_enum.h
new file mode 100644
index 000000000000..fbfb59bd3f3d
--- /dev/null
+++ b/drivers/staging/pi433/rf69_enum.h
@@ -0,0 +1,201 @@
+/*
+ * enumerations for HopeRf rf69 radio module
+ *
+ * Copyright (C) 2016 Wolf-Entwicklungen
+ *	Marcus Wolf <linux@wolf-entwicklungen.de>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef RF69_ENUM_H
+#define RF69_ENUM_H
+
+enum optionOnOff
+{
+    optionOff,
+    optionOn
+};
+
+enum mode
+{
+    mode_sleep,
+    standby,
+    synthesizer,
+    transmit,
+    receive
+};
+
+enum dataMode
+{
+    packet,
+    continuous,
+    continuousNoSync
+};
+
+enum modulation
+{
+    OOK,
+    FSK
+};
+
+enum modShaping
+{
+    shapingOff,
+    shaping1_0,
+    shaping0_5,
+    shaping0_3,
+    shapingBR,
+    shaping2BR
+};
+
+enum paRamp
+{
+    ramp3400,
+    ramp2000,
+    ramp1000,
+    ramp500,
+    ramp250,
+    ramp125,
+    ramp100,
+    ramp62,
+    ramp50,
+    ramp40,
+    ramp31,
+    ramp25,
+    ramp20,
+    ramp15,
+    ramp12,
+    ramp10
+};
+
+enum antennaImpedance
+{
+    fiftyOhm,
+    twohundretOhm
+};
+
+enum lnaGain
+{
+    automatic,
+    max,
+    maxMinus6,
+    maxMinus12,
+    maxMinus24,
+    maxMinus36,
+    maxMinus48,
+    undefined
+};
+
+enum dccPercent
+{
+    dcc16Percent,
+    dcc8Percent,
+    dcc4Percent,
+    dcc2Percent,
+    dcc1Percent,
+    dcc0_5Percent,
+    dcc0_25Percent,
+    dcc0_125Percent
+};
+
+enum mantisse
+{
+    mantisse16,
+    mantisse20,
+    mantisse24
+};
+
+enum thresholdType
+{
+    fixed,
+    peak,
+    average
+};
+
+enum thresholdStep
+{
+    step_0_5db,
+    step_1_0db,
+    step_1_5db,
+    step_2_0db,
+    step_3_0db,
+    step_4_0db,
+    step_5_0db,
+    step_6_0db
+};
+
+enum thresholdDecrement
+{
+    dec_every8th,
+    dec_every4th,
+    dec_every2nd,
+    dec_once,
+    dec_twice,
+    dec_4times,
+    dec_8times,
+    dec_16times
+};
+
+enum flag
+{
+    modeSwitchCompleted,
+    readyToReceive,
+    readyToSend,
+    pllLocked,
+    rssiExceededThreshold,
+    timeout,
+    automode,
+    syncAddressMatch,
+    fifoFull,
+//    fifoNotEmpty, collision with next enum; replaced by following enum...
+    fifoEmpty,
+    fifoLevelBelowThreshold,
+    fifoOverrun,
+    packetSent,
+    payloadReady,
+    crcOk,
+    batteryLow
+};
+
+enum fifoFillCondition
+{
+    afterSyncInterrupt,
+    always
+};
+
+enum packetFormat
+{
+    packetLengthFix,
+    packetLengthVar
+};
+
+enum txStartCondition
+{
+    fifoLevel,
+    fifoNotEmpty
+};
+
+enum addressFiltering
+{
+    filteringOff,
+    nodeAddress,
+    nodeOrBroadcastAddress
+};
+
+enum dagc
+{
+    normalMode,
+    improve,
+    improve4LowModulationIndex
+};
+
+
+#endif
diff --git a/drivers/staging/pi433/rf69_registers.h b/drivers/staging/pi433/rf69_registers.h
new file mode 100644
index 000000000000..6335d42142fe
--- /dev/null
+++ b/drivers/staging/pi433/rf69_registers.h
@@ -0,0 +1,489 @@
+/*
+ * register description for HopeRf rf69 radio module
+ *
+ * Copyright (C) 2016 Wolf-Entwicklungen
+ *	Marcus Wolf <linux@wolf-entwicklungen.de>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+/*******************************************/
+/* RF69 register addresses		   */
+/*******************************************/
+#define  REG_FIFO			0x00
+#define  REG_OPMODE			0x01
+#define  REG_DATAMODUL			0x02
+#define  REG_BITRATE_MSB		0x03
+#define  REG_BITRATE_LSB		0x04
+#define  REG_FDEV_MSB			0x05
+#define  REG_FDEV_LSB			0x06
+#define  REG_FRF_MSB			0x07
+#define  REG_FRF_MID			0x08
+#define  REG_FRF_LSB			0x09
+#define  REG_OSC1			0x0A
+#define  REG_AFCCTRL			0x0B
+#define  REG_LOWBAT			0x0C
+#define  REG_LISTEN1			0x0D
+#define  REG_LISTEN2			0x0E
+#define  REG_LISTEN3			0x0F
+#define  REG_VERSION			0x10
+#define  REG_PALEVEL			0x11
+#define  REG_PARAMP			0x12
+#define  REG_OCP			0x13
+#define  REG_AGCREF			0x14 /* not available on RF69 */
+#define  REG_AGCTHRESH1			0x15 /* not available on RF69 */
+#define  REG_AGCTHRESH2			0x16 /* not available on RF69 */
+#define  REG_AGCTHRESH3			0x17 /* not available on RF69 */
+#define  REG_LNA			0x18
+#define  REG_RXBW			0x19
+#define  REG_AFCBW			0x1A
+#define  REG_OOKPEAK			0x1B
+#define  REG_OOKAVG			0x1C
+#define  REG_OOKFIX			0x1D
+#define  REG_AFCFEI			0x1E
+#define  REG_AFCMSB			0x1F
+#define  REG_AFCLSB			0x20
+#define  REG_FEIMSB			0x21
+#define  REG_FEILSB			0x22
+#define  REG_RSSICONFIG			0x23
+#define  REG_RSSIVALUE			0x24
+#define  REG_DIOMAPPING1		0x25
+#define  REG_DIOMAPPING2		0x26
+#define  REG_IRQFLAGS1			0x27
+#define  REG_IRQFLAGS2			0x28
+#define  REG_RSSITHRESH			0x29
+#define  REG_RXTIMEOUT1			0x2A
+#define  REG_RXTIMEOUT2			0x2B
+#define  REG_PREAMBLE_MSB		0x2C
+#define  REG_PREAMBLE_LSB		0x2D
+#define  REG_SYNC_CONFIG		0x2E
+#define  REG_SYNCVALUE1			0x2F
+#define  REG_SYNCVALUE2			0x30
+#define  REG_SYNCVALUE3			0x31
+#define  REG_SYNCVALUE4			0x32
+#define  REG_SYNCVALUE5			0x33
+#define  REG_SYNCVALUE6			0x34
+#define  REG_SYNCVALUE7			0x35
+#define  REG_SYNCVALUE8			0x36
+#define  REG_PACKETCONFIG1		0x37
+#define  REG_PAYLOAD_LENGTH		0x38
+#define  REG_NODEADRS			0x39
+#define  REG_BROADCASTADRS		0x3A
+#define  REG_AUTOMODES			0x3B
+#define  REG_FIFO_THRESH		0x3C
+#define  REG_PACKETCONFIG2		0x3D
+#define  REG_AESKEY1			0x3E
+#define  REG_AESKEY2			0x3F
+#define  REG_AESKEY3			0x40
+#define  REG_AESKEY4			0x41
+#define  REG_AESKEY5			0x42
+#define  REG_AESKEY6			0x43
+#define  REG_AESKEY7			0x44
+#define  REG_AESKEY8			0x45
+#define  REG_AESKEY9			0x46
+#define  REG_AESKEY10			0x47
+#define  REG_AESKEY11			0x48
+#define  REG_AESKEY12			0x49
+#define  REG_AESKEY13			0x4A
+#define  REG_AESKEY14			0x4B
+#define  REG_AESKEY15			0x4C
+#define  REG_AESKEY16			0x4D
+#define  REG_TEMP1			0x4E
+#define  REG_TEMP2			0x4F
+#define  REG_TESTPA1			0x5A /* only present on RFM69HW */
+#define  REG_TESTPA2			0x5C /* only present on RFM69HW */
+#define  REG_TESTDAGC			0x6F
+
+/******************************************************/
+/* RF69/SX1231 bit definition				*/
+/******************************************************/
+/* write bit */
+#define WRITE_BIT				0x80
+
+/* RegOpMode */
+#define  MASK_OPMODE_SEQUENCER_OFF		0x80
+#define  MASK_OPMODE_LISTEN_ON			0x40
+#define  MASK_OPMODE_LISTEN_ABORT		0x20
+#define  MASK_OPMODE_MODE			0x1C
+
+#define  OPMODE_MODE_SLEEP			0x00
+#define  OPMODE_MODE_STANDBY			0x04 /* default */
+#define  OPMODE_MODE_SYNTHESIZER		0x08
+#define  OPMODE_MODE_TRANSMIT			0x0C
+#define  OPMODE_MODE_RECEIVE			0x10
+
+/* RegDataModul */
+#define  MASK_DATAMODUL_MODE			0x06
+#define  MASK_DATAMODUL_MODULATION_TYPE		0x18
+#define  MASK_DATAMODUL_MODULATION_SHAPE	0x03
+
+#define  DATAMODUL_MODE_PACKET			0x00 /* default */
+#define  DATAMODUL_MODE_CONTINUOUS		0x40
+#define  DATAMODUL_MODE_CONTINUOUS_NOSYNC	0x60
+
+#define  DATAMODUL_MODULATION_TYPE_FSK		0x00 /* default */
+#define  DATAMODUL_MODULATION_TYPE_OOK		0x08
+
+#define  DATAMODUL_MODULATION_SHAPE_NONE	0x00 /* default */
+#define  DATAMODUL_MODULATION_SHAPE_1_0		0x01
+#define  DATAMODUL_MODULATION_SHAPE_0_5		0x02
+#define  DATAMODUL_MODULATION_SHAPE_0_3		0x03
+#define  DATAMODUL_MODULATION_SHAPE_BR		0x01
+#define  DATAMODUL_MODULATION_SHAPE_2BR		0x02
+
+/* RegFDevMsb (0x05)*/
+#define FDEVMASB_MASK				0x3f
+
+/*
+ * // RegOsc1
+ * #define  OSC1_RCCAL_START			0x80
+ * #define  OSC1_RCCAL_DONE			0x40
+ *
+ * // RegLowBat
+ * #define  LOWBAT_MONITOR				0x10
+ * #define  LOWBAT_ON				0x08
+ * #define  LOWBAT_OFF				0x00  // Default
+ *
+ * #define  LOWBAT_TRIM_1695			0x00
+ * #define  LOWBAT_TRIM_1764			0x01
+ * #define  LOWBAT_TRIM_1835			0x02  // Default
+ * #define  LOWBAT_TRIM_1905			0x03
+ * #define  LOWBAT_TRIM_1976			0x04
+ * #define  LOWBAT_TRIM_2045			0x05
+ * #define  LOWBAT_TRIM_2116			0x06
+ * #define  LOWBAT_TRIM_2185			0x07
+ *
+ *
+ * // RegListen1
+ * #define  LISTEN1_RESOL_64			0x50
+ * #define  LISTEN1_RESOL_4100			0xA0  // Default
+ * #define  LISTEN1_RESOL_262000			0xF0
+ *
+ * #define  LISTEN1_CRITERIA_RSSI			0x00  // Default
+ * #define  LISTEN1_CRITERIA_RSSIANDSYNC		0x08
+ *
+ * #define  LISTEN1_END_00				0x00
+ * #define  LISTEN1_END_01				0x02  // Default
+ * #define  LISTEN1_END_10				0x04
+ *
+ *
+ * // RegListen2
+ * #define  LISTEN2_COEFIDLE_VALUE			0xF5 // Default
+ *
+ * // RegListen3
+ * #define  LISTEN3_COEFRX_VALUE			0x20 // Default
+ */
+
+// RegPaLevel
+#define  MASK_PALEVEL_PA0			0x80
+#define  MASK_PALEVEL_PA1			0x40
+#define  MASK_PALEVEL_PA2			0x20
+#define  MASK_PALEVEL_OUTPUT_POWER		0x1F
+
+
+
+// RegPaRamp
+#define  PARAMP_3400				0x00
+#define  PARAMP_2000				0x01
+#define  PARAMP_1000				0x02
+#define  PARAMP_500				0x03
+#define  PARAMP_250				0x04
+#define  PARAMP_125				0x05
+#define  PARAMP_100				0x06
+#define  PARAMP_62				0x07
+#define  PARAMP_50				0x08
+#define  PARAMP_40				0x09 /* default */
+#define  PARAMP_31				0x0A
+#define  PARAMP_25				0x0B
+#define  PARAMP_20				0x0C
+#define  PARAMP_15				0x0D
+#define  PARAMP_12				0x0E
+#define  PARAMP_10				0x0F
+
+#define  MASK_PARAMP				0x0F
+
+/*
+ * // RegOcp
+ * #define  OCP_OFF				0x0F
+ * #define  OCP_ON					0x1A  // Default
+ *
+ * #define  OCP_TRIM_45				0x00
+ * #define  OCP_TRIM_50				0x01
+ * #define  OCP_TRIM_55				0x02
+ * #define  OCP_TRIM_60				0x03
+ * #define  OCP_TRIM_65				0x04
+ * #define  OCP_TRIM_70				0x05
+ * #define  OCP_TRIM_75				0x06
+ * #define  OCP_TRIM_80				0x07
+ * #define  OCP_TRIM_85				0x08
+ * #define  OCP_TRIM_90				0x09
+ * #define  OCP_TRIM_95				0x0A
+ * #define  OCP_TRIM_100				0x0B  // Default
+ * #define  OCP_TRIM_105				0x0C
+ * #define  OCP_TRIM_110				0x0D
+ * #define  OCP_TRIM_115				0x0E
+ * #define  OCP_TRIM_120				0x0F
+ */
+
+/* RegLna (0x18) */
+#define  MASK_LNA_ZIN				0x80
+#define  MASK_LNA_CURRENT_GAIN			0x38
+#define  MASK_LNA_GAIN				0x07
+
+#define  LNA_GAIN_AUTO				0x00 /* default */
+#define  LNA_GAIN_MAX				0x01
+#define  LNA_GAIN_MAX_MINUS_6			0x02
+#define  LNA_GAIN_MAX_MINUS_12			0x03
+#define  LNA_GAIN_MAX_MINUS_24			0x04
+#define  LNA_GAIN_MAX_MINUS_36			0x05
+#define  LNA_GAIN_MAX_MINUS_48			0x06
+
+
+/* RegRxBw (0x19) and RegAfcBw (0x1A) */
+#define  MASK_BW_DCC_FREQ			0xE0
+#define  MASK_BW_MANTISSE			0x18
+#define  MASK_BW_EXPONENT			0x07
+
+#define  BW_DCC_16_PERCENT			0x00
+#define  BW_DCC_8_PERCENT			0x20
+#define  BW_DCC_4_PERCENT			0x40 /* default */
+#define  BW_DCC_2_PERCENT			0x60
+#define  BW_DCC_1_PERCENT			0x80
+#define  BW_DCC_0_5_PERCENT			0xA0
+#define  BW_DCC_0_25_PERCENT			0xC0
+#define  BW_DCC_0_125_PERCENT			0xE0
+
+#define  BW_MANT_16				0x00
+#define  BW_MANT_20				0x08
+#define  BW_MANT_24				0x10 /* default */
+
+
+/* RegOokPeak (0x1B) */
+#define  MASK_OOKPEAK_THRESTYPE			0xc0
+#define  MASK_OOKPEAK_THRESSTEP			0x38
+#define  MASK_OOKPEAK_THRESDEC			0x07
+
+#define  OOKPEAK_THRESHTYPE_FIXED		0x00
+#define  OOKPEAK_THRESHTYPE_PEAK		0x40 /* default */
+#define  OOKPEAK_THRESHTYPE_AVERAGE		0x80
+
+#define  OOKPEAK_THRESHSTEP_0_5_DB		0x00 /* default */
+#define  OOKPEAK_THRESHSTEP_1_0_DB		0x08
+#define  OOKPEAK_THRESHSTEP_1_5_DB		0x10
+#define  OOKPEAK_THRESHSTEP_2_0_DB		0x18
+#define  OOKPEAK_THRESHSTEP_3_0_DB		0x20
+#define  OOKPEAK_THRESHSTEP_4_0_DB		0x28
+#define  OOKPEAK_THRESHSTEP_5_0_DB		0x30
+#define  OOKPEAK_THRESHSTEP_6_0_DB		0x38
+
+#define  OOKPEAK_THRESHDEC_ONCE			0x00 /* default */
+#define  OOKPEAK_THRESHDEC_EVERY_2ND		0x01
+#define  OOKPEAK_THRESHDEC_EVERY_4TH		0x02
+#define  OOKPEAK_THRESHDEC_EVERY_8TH		0x03
+#define  OOKPEAK_THRESHDEC_TWICE		0x04
+#define  OOKPEAK_THRESHDEC_4_TIMES		0x05
+#define  OOKPEAK_THRESHDEC_8_TIMES		0x06
+#define  OOKPEAK_THRESHDEC_16_TIMES		0x07
+
+/*
+ * // RegOokAvg
+ * #define  OOKAVG_AVERAGETHRESHFILT_00		0x00
+ * #define  OOKAVG_AVERAGETHRESHFILT_01		0x40
+ * #define  OOKAVG_AVERAGETHRESHFILT_10		0x80  // Default
+ * #define  OOKAVG_AVERAGETHRESHFILT_11		0xC0
+ *
+ *
+ * // RegAfcFei
+ * #define  AFCFEI_FEI_DONE			0x40
+ * #define  AFCFEI_FEI_START			0x20
+ * #define  AFCFEI_AFC_DONE			0x10
+ * #define  AFCFEI_AFCAUTOCLEAR_ON			0x08
+ * #define  AFCFEI_AFCAUTOCLEAR_OFF		0x00  // Default
+ *
+ * #define  AFCFEI_AFCAUTO_ON			0x04
+ * #define  AFCFEI_AFCAUTO_OFF			0x00  // Default
+ *
+ * #define  AFCFEI_AFC_CLEAR			0x02
+ * #define  AFCFEI_AFC_START			0x01
+ *
+ * // RegRssiConfig
+ * #define  RSSI_FASTRX_ON				0x08
+ * #define  RSSI_FASTRX_OFF			0x00  // Default
+ * #define  RSSI_DONE				0x02
+ * #define  RSSI_START				0x01
+ */
+
+/* RegDioMapping1 */
+#define  MASK_DIO0				0xC0
+#define  MASK_DIO1				0x30
+#define  MASK_DIO2				0x0C
+#define  MASK_DIO3				0x03
+#define  SHIFT_DIO0				6
+#define  SHIFT_DIO1				4
+#define  SHIFT_DIO2				2
+#define  SHIFT_DIO3				0
+
+/* RegDioMapping2 */
+#define  MASK_DIO4				0xC0
+#define  MASK_DIO5				0x30
+#define  SHIFT_DIO4				6
+#define  SHIFT_DIO5				4
+
+/* DIO numbers */
+#define  DIO0					0
+#define  DIO1					1
+#define  DIO2					2
+#define  DIO3					3
+#define  DIO4					4
+#define  DIO5					5
+
+/* DIO Mapping values (packet mode) */
+#define  DIO_ModeReady_DIO4			0x00
+#define  DIO_ModeReady_DIO5			0x03
+#define  DIO_ClkOut				0x00
+#define  DIO_Data				0x01
+#define  DIO_TimeOut_DIO1			0x03
+#define  DIO_TimeOut_DIO4			0x00
+#define  DIO_Rssi_DIO0				0x03
+#define  DIO_Rssi_DIO3_4			0x01
+#define  DIO_RxReady				0x02
+#define  DIO_PLLLock				0x03
+#define  DIO_TxReady				0x01
+#define  DIO_FifoFull_DIO1			0x01
+#define  DIO_FifoFull_DIO3			0x00
+#define  DIO_SyncAddress			0x02
+#define  DIO_FifoNotEmpty_DIO1			0x02
+#define  DIO_FifoNotEmpty_FIO2			0x00
+#define  DIO_Automode				0x04
+#define  DIO_FifoLevel				0x00
+#define  DIO_CrcOk				0x00
+#define  DIO_PayloadReady			0x01
+#define  DIO_PacketSent				0x00
+#define  DIO_Dclk				0x00
+
+/* RegDioMapping2 CLK_OUT part */
+#define  MASK_DIOMAPPING2_CLK_OUT		0x07
+
+#define  DIOMAPPING2_CLK_OUT_NO_DIV		0x00
+#define  DIOMAPPING2_CLK_OUT_DIV_2		0x01
+#define  DIOMAPPING2_CLK_OUT_DIV_4		0x02
+#define  DIOMAPPING2_CLK_OUT_DIV_8		0x03
+#define  DIOMAPPING2_CLK_OUT_DIV_16		0x04
+#define  DIOMAPPING2_CLK_OUT_DIV_32		0x05
+#define  DIOMAPPING2_CLK_OUT_RC			0x06
+#define  DIOMAPPING2_CLK_OUT_OFF		0x07 /* default */
+
+/* RegIrqFlags1 */
+#define  MASK_IRQFLAGS1_MODE_READY		0x80
+#define  MASK_IRQFLAGS1_RX_READY		0x40
+#define  MASK_IRQFLAGS1_TX_READY		0x20
+#define  MASK_IRQFLAGS1_PLL_LOCK		0x10
+#define  MASK_IRQFLAGS1_RSSI			0x08
+#define  MASK_IRQFLAGS1_TIMEOUT			0x04
+#define  MASK_IRQFLAGS1_AUTOMODE		0x02
+#define  MASK_IRQFLAGS1_SYNC_ADDRESS_MATCH	0x01
+
+/* RegIrqFlags2 */
+#define  MASK_IRQFLAGS2_FIFO_FULL		0x80
+#define  MASK_IRQFLAGS2_FIFO_NOT_EMPTY		0x40
+#define  MASK_IRQFLAGS2_FIFO_LEVEL		0x20
+#define  MASK_IRQFLAGS2_FIFO_OVERRUN		0x10
+#define  MASK_IRQFLAGS2_PACKET_SENT		0x08
+#define  MASK_IRQFLAGS2_PAYLOAD_READY		0x04
+#define  MASK_IRQFLAGS2_CRC_OK			0x02
+#define  MASK_IRQFLAGS2_LOW_BAT			0x01
+
+/* RegSyncConfig */
+#define  MASK_SYNC_CONFIG_SYNC_ON		0x80 /* default */
+#define  MASK_SYNC_CONFIG_FIFO_FILL_CONDITION	0x40
+#define  MASK_SYNC_CONFIG_SYNC_SIZE		0x38
+#define  MASK_SYNC_CONFIG_SYNC_TOLERANCE	0x07
+
+/* RegPacketConfig1 */
+#define  MASK_PACKETCONFIG1_PAKET_FORMAT_VARIABLE	0x80
+#define  MASK_PACKETCONFIG1_DCFREE			0x60
+#define  MASK_PACKETCONFIG1_CRC_ON			0x10 /* default */
+#define  MASK_PACKETCONFIG1_CRCAUTOCLEAR_OFF		0x08
+#define  MASK_PACKETCONFIG1_ADDRESSFILTERING		0x06
+
+#define  PACKETCONFIG1_DCFREE_OFF			0x00 /* default */
+#define  PACKETCONFIG1_DCFREE_MANCHESTER		0x20
+#define  PACKETCONFIG1_DCFREE_WHITENING			0x40
+#define  PACKETCONFIG1_ADDRESSFILTERING_OFF		0x00 /* default */
+#define  PACKETCONFIG1_ADDRESSFILTERING_NODE		0x02
+#define  PACKETCONFIG1_ADDRESSFILTERING_NODEBROADCAST	0x04
+
+/*
+ * // RegAutoModes
+ * #define  AUTOMODES_ENTER_OFF			0x00  // Default
+ * #define  AUTOMODES_ENTER_FIFONOTEMPTY		0x20
+ * #define  AUTOMODES_ENTER_FIFOLEVEL		0x40
+ * #define  AUTOMODES_ENTER_CRCOK			0x60
+ * #define  AUTOMODES_ENTER_PAYLOADREADY		0x80
+ * #define  AUTOMODES_ENTER_SYNCADRSMATCH		0xA0
+ * #define  AUTOMODES_ENTER_PACKETSENT		0xC0
+ * #define  AUTOMODES_ENTER_FIFOEMPTY		0xE0
+ *
+ * #define  AUTOMODES_EXIT_OFF			0x00  // Default
+ * #define  AUTOMODES_EXIT_FIFOEMPTY		0x04
+ * #define  AUTOMODES_EXIT_FIFOLEVEL		0x08
+ * #define  AUTOMODES_EXIT_CRCOK			0x0C
+ * #define  AUTOMODES_EXIT_PAYLOADREADY		0x10
+ * #define  AUTOMODES_EXIT_SYNCADRSMATCH		0x14
+ * #define  AUTOMODES_EXIT_PACKETSENT		0x18
+ * #define  AUTOMODES_EXIT_RXTIMEOUT		0x1C
+ *
+ * #define  AUTOMODES_INTERMEDIATE_SLEEP		0x00  // Default
+ * #define  AUTOMODES_INTERMEDIATE_STANDBY		0x01
+ * #define  AUTOMODES_INTERMEDIATE_RECEIVER	0x02
+ * #define  AUTOMODES_INTERMEDIATE_TRANSMITTER	0x03
+ *
+ */
+/* RegFifoThresh (0x3c) */
+#define  MASK_FIFO_THRESH_TXSTART		0x80
+#define  MASK_FIFO_THRESH_VALUE			0x7F
+
+/*
+ *
+ * // RegPacketConfig2
+ * #define  PACKET2_RXRESTARTDELAY_1BIT		0x00  // Default
+ * #define  PACKET2_RXRESTARTDELAY_2BITS		0x10
+ * #define  PACKET2_RXRESTARTDELAY_4BITS		0x20
+ * #define  PACKET2_RXRESTARTDELAY_8BITS		0x30
+ * #define  PACKET2_RXRESTARTDELAY_16BITS		0x40
+ * #define  PACKET2_RXRESTARTDELAY_32BITS		0x50
+ * #define  PACKET2_RXRESTARTDELAY_64BITS		0x60
+ * #define  PACKET2_RXRESTARTDELAY_128BITS		0x70
+ * #define  PACKET2_RXRESTARTDELAY_256BITS		0x80
+ * #define  PACKET2_RXRESTARTDELAY_512BITS		0x90
+ * #define  PACKET2_RXRESTARTDELAY_1024BITS	0xA0
+ * #define  PACKET2_RXRESTARTDELAY_2048BITS	0xB0
+ * #define  PACKET2_RXRESTARTDELAY_NONE		0xC0
+ * #define  PACKET2_RXRESTART			0x04
+ *
+ * #define  PACKET2_AUTORXRESTART_ON		0x02  // Default
+ * #define  PACKET2_AUTORXRESTART_OFF		0x00
+ *
+ * #define  PACKET2_AES_ON				0x01
+ * #define  PACKET2_AES_OFF			0x00  // Default
+ *
+ *
+ * // RegTemp1
+ * #define  TEMP1_MEAS_START			0x08
+ * #define  TEMP1_MEAS_RUNNING			0x04
+ * #define  TEMP1_ADCLOWPOWER_ON			0x01  // Default
+ * #define  TEMP1_ADCLOWPOWER_OFF			0x00
+ */
+
+// RegTestDagc (0x6F)
+#define  DAGC_NORMAL				0x00 /* Reset value */
+#define  DAGC_IMPROVED_LOWBETA1			0x20
+#define  DAGC_IMPROVED_LOWBETA0			0x30 /* Recommended val */
diff --git a/drivers/staging/rtl8188eu/core/rtw_ap.c b/drivers/staging/rtl8188eu/core/rtw_ap.c
index 647a922d79d1..32a483769975 100644
--- a/drivers/staging/rtl8188eu/core/rtw_ap.c
+++ b/drivers/staging/rtl8188eu/core/rtw_ap.c
@@ -119,7 +119,7 @@ static void update_BCNTIM(struct adapter *padapter)
 	}
 	*dst_ie++ = _TIM_IE_;
 
-	if ((pstapriv->tim_bitmap&0xff00) && (pstapriv->tim_bitmap&0x00fc))
+	if ((pstapriv->tim_bitmap & 0xff00) && (pstapriv->tim_bitmap & 0x00fc))
 		tim_ielen = 5;
 	else
 		tim_ielen = 4;
@@ -129,7 +129,7 @@ static void update_BCNTIM(struct adapter *padapter)
 	*dst_ie++ = 0;/* DTIM count */
 	*dst_ie++ = 1;/* DTIM period */
 
-	if (pstapriv->tim_bitmap&BIT(0))/* for bc/mc frames */
+	if (pstapriv->tim_bitmap & BIT(0))/* for bc/mc frames */
 		*dst_ie++ = BIT(0);/* bitmap ctrl */
 	else
 		*dst_ie++ = 0;
@@ -583,10 +583,10 @@ static void update_bmc_sta(struct adapter *padapter)
 		{
 			u8 arg = 0;
 
-			arg = psta->mac_id&0x1f;
+			arg = psta->mac_id & 0x1f;
 			arg |= BIT(7);
 			tx_ra_bitmap |= ((raid << 28) & 0xf0000000);
-			DBG_88E("update_bmc_sta, mask = 0x%x, arg = 0x%x\n", tx_ra_bitmap, arg);
+			DBG_88E("%s, mask = 0x%x, arg = 0x%x\n", __func__, tx_ra_bitmap, arg);
 
 			/* bitmap[0:27] = tx_rate_bitmap */
 			/* bitmap[28:31]= Rate Adaptive id */
@@ -1043,7 +1043,7 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 		    (psecuritypriv->wpa2_pairwise_cipher & WPA_CIPHER_CCMP))
 			pht_cap->ampdu_params_info |= (IEEE80211_HT_CAP_AMPDU_DENSITY & (0x07 << 2));
 		else
-			pht_cap->ampdu_params_info |= (IEEE80211_HT_CAP_AMPDU_DENSITY&0x00);
+			pht_cap->ampdu_params_info |= (IEEE80211_HT_CAP_AMPDU_DENSITY & 0x00);
 
 		/* set  Max Rx AMPDU size  to 64K */
 		pht_cap->ampdu_params_info |= (IEEE80211_HT_CAP_AMPDU_FACTOR & 0x03);
@@ -1719,7 +1719,7 @@ int rtw_sta_flush(struct adapter *padapter)
 
 	DBG_88E(FUNC_NDEV_FMT"\n", FUNC_NDEV_ARG(padapter->pnetdev));
 
-	if ((pmlmeinfo->state&0x03) != WIFI_FW_AP_STATE)
+	if ((pmlmeinfo->state & 0x03) != WIFI_FW_AP_STATE)
 		return 0;
 
 	spin_lock_bh(&pstapriv->asoc_list_lock);
@@ -1754,7 +1754,7 @@ void sta_info_update(struct adapter *padapter, struct sta_info *psta)
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 
 	/* update wmm cap. */
-	if (WLAN_STA_WME&flags)
+	if (WLAN_STA_WME & flags)
 		psta->qos_option = 1;
 	else
 		psta->qos_option = 0;
@@ -1763,7 +1763,7 @@ void sta_info_update(struct adapter *padapter, struct sta_info *psta)
 		psta->qos_option = 0;
 
 	/* update 802.11n ht cap. */
-	if (WLAN_STA_HT&flags) {
+	if (WLAN_STA_HT & flags) {
 		psta->htpriv.ht_option = true;
 		psta->qos_option = 1;
 	} else {
diff --git a/drivers/staging/rtl8188eu/core/rtw_cmd.c b/drivers/staging/rtl8188eu/core/rtw_cmd.c
index a69007ef77bf..9461bce883ea 100644
--- a/drivers/staging/rtl8188eu/core/rtw_cmd.c
+++ b/drivers/staging/rtl8188eu/core/rtw_cmd.c
@@ -186,7 +186,7 @@ _next:
 			pcmd->res = H2C_DROPPED;
 		} else {
 			if (pcmd->cmdcode < ARRAY_SIZE(wlancmds)) {
-			    cmd_hdl = wlancmds[pcmd->cmdcode].h2cfuns;
+				cmd_hdl = wlancmds[pcmd->cmdcode].h2cfuns;
 
 				if (cmd_hdl) {
 					ret = cmd_hdl(pcmd->padapter, pcmd->parmbuf);
diff --git a/drivers/staging/rtl8188eu/core/rtw_ioctl_set.c b/drivers/staging/rtl8188eu/core/rtw_ioctl_set.c
index d8d88b5f68e5..767928a2cbb4 100644
--- a/drivers/staging/rtl8188eu/core/rtw_ioctl_set.c
+++ b/drivers/staging/rtl8188eu/core/rtw_ioctl_set.c
@@ -56,7 +56,7 @@ u8 rtw_do_join(struct adapter *padapter)
 			RT_TRACE(_module_rtl871x_ioctl_set_c_, _drv_info_, ("rtw_do_join(): site survey if scanned_queue is empty\n."));
 			/*  submit site_survey_cmd */
 			ret = rtw_sitesurvey_cmd(padapter, &pmlmepriv->assoc_ssid, 1, NULL, 0);
-			if (_SUCCESS != ret) {
+			if (ret != _SUCCESS) {
 				pmlmepriv->to_join = false;
 				RT_TRACE(_module_rtl871x_ioctl_set_c_, _drv_err_, ("rtw_do_join(): site survey return error\n."));
 			}
@@ -110,7 +110,7 @@ u8 rtw_do_join(struct adapter *padapter)
 				if (!pmlmepriv->LinkDetectInfo.bBusyTraffic ||
 				    pmlmepriv->to_roaming > 0) {
 					ret = rtw_sitesurvey_cmd(padapter, &pmlmepriv->assoc_ssid, 1, NULL, 0);
-					if (_SUCCESS != ret) {
+					if (ret != _SUCCESS) {
 						pmlmepriv->to_join = false;
 						RT_TRACE(_module_rtl871x_ioctl_set_c_, _drv_err_, ("do_join(): site survey return error\n."));
 					}
@@ -621,7 +621,7 @@ int rtw_set_country(struct adapter *adapter, const char *country_code)
 
 	DBG_88E("%s country_code:%s\n", __func__, country_code);
 	for (i = 0; i < ARRAY_SIZE(channel_table); i++) {
-		if (0 == strcmp(channel_table[i].name, country_code)) {
+		if (strcmp(channel_table[i].name, country_code) == 0) {
 			channel_plan = channel_table[i].channel_plan;
 			break;
 		}
diff --git a/drivers/staging/rtl8188eu/core/rtw_mlme.c b/drivers/staging/rtl8188eu/core/rtw_mlme.c
index fde306087844..f663e6c41f8a 100644
--- a/drivers/staging/rtl8188eu/core/rtw_mlme.c
+++ b/drivers/staging/rtl8188eu/core/rtw_mlme.c
@@ -1502,7 +1502,7 @@ int rtw_select_and_join_from_scanned_queue(struct mlme_priv *pmlmepriv)
 	pmlmepriv->pscanned = phead->next;
 	while (phead != pmlmepriv->pscanned) {
 		pnetwork = container_of(pmlmepriv->pscanned, struct wlan_network, list);
-		if (pnetwork == NULL) {
+		if (!pnetwork) {
 			RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_, ("%s return _FAIL:(pnetwork==NULL)\n", __func__));
 			ret = _FAIL;
 			goto exit;
@@ -1566,7 +1566,6 @@ int rtw_set_auth(struct adapter *adapter, struct security_priv *psecuritypriv)
 		res = _FAIL;
 		goto exit;
 	}
-	memset(psetauthparm, 0, sizeof(struct setauth_parm));
 	psetauthparm->mode = (unsigned char)psecuritypriv->dot11AuthAlgrthm;
 	pcmd->cmdcode = _SetAuth_CMD_;
 	pcmd->parmbuf = (unsigned char *)psetauthparm;
@@ -1601,8 +1600,6 @@ int rtw_set_key(struct adapter *adapter, struct security_priv *psecuritypriv, in
 		goto err_free_cmd;
 	}
 
-	memset(psetkeyparm, 0, sizeof(struct setkey_parm));
-
 	if (psecuritypriv->dot11AuthAlgrthm == dot11AuthAlgrthm_8021X) {
 		psetkeyparm->algorithm = (unsigned char)psecuritypriv->dot118021XGrpPrivacy;
 		RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_,
@@ -1708,14 +1705,9 @@ static int SecIsInPMKIDList(struct adapter *Adapter, u8 *bssid)
 
 	do {
 		if ((psecuritypriv->PMKIDList[i].bUsed) &&
-		    (!memcmp(psecuritypriv->PMKIDList[i].Bssid, bssid, ETH_ALEN))) {
+		    (!memcmp(psecuritypriv->PMKIDList[i].Bssid, bssid, ETH_ALEN)))
 			break;
-		} else {
-			i++;
-			/* continue; */
-		}
-
-	} while (i < NUM_PMKID_CACHE);
+	} while (++i < NUM_PMKID_CACHE);
 
 	if (i == NUM_PMKID_CACHE)
 		i = -1;/*  Could not find. */
diff --git a/drivers/staging/rtl8188eu/core/rtw_mlme_ext.c b/drivers/staging/rtl8188eu/core/rtw_mlme_ext.c
index 88a3a2b9c144..611c9409bb98 100644
--- a/drivers/staging/rtl8188eu/core/rtw_mlme_ext.c
+++ b/drivers/staging/rtl8188eu/core/rtw_mlme_ext.c
@@ -3961,7 +3961,8 @@ static void init_mlme_ext_priv_value(struct adapter *padapter)
 
 static int has_channel(struct rt_channel_info *channel_set,
 					   u8 chanset_size,
-					   u8 chan) {
+					   u8 chan)
+{
 	int i;
 
 	for (i = 0; i < chanset_size; i++) {
@@ -3973,7 +3974,8 @@ static int has_channel(struct rt_channel_info *channel_set,
 
 static void init_channel_list(struct adapter *padapter, struct rt_channel_info *channel_set,
 							  u8 chanset_size,
-							  struct p2p_channels *channel_list) {
+							struct p2p_channels *channel_list)
+{
 	struct p2p_oper_class_map op_class[] = {
 		{ IEEE80211G,  81,   1,  13,  1, BW20 },
 		{ IEEE80211G,  82,  14,  14,  1, BW20 },
diff --git a/drivers/staging/rtl8188eu/include/rtw_ioctl.h b/drivers/staging/rtl8188eu/include/rtw_ioctl.h
index 0fa78ed2c1ab..4c925e610997 100644
--- a/drivers/staging/rtl8188eu/include/rtw_ioctl.h
+++ b/drivers/staging/rtl8188eu/include/rtw_ioctl.h
@@ -75,7 +75,8 @@ struct oid_par_priv {
 };
 
 #if defined(_RTW_MP_IOCTL_C_)
-static int oid_null_function(struct oid_par_priv *poid_par_priv) {
+static int oid_null_function(struct oid_par_priv *poid_par_priv)
+{
 	return NDIS_STATUS_SUCCESS;
 }
 #endif
diff --git a/drivers/staging/rtl8188eu/os_dep/ioctl_linux.c b/drivers/staging/rtl8188eu/os_dep/ioctl_linux.c
index 763eccd0c7c9..c0664dc80bf2 100644
--- a/drivers/staging/rtl8188eu/os_dep/ioctl_linux.c
+++ b/drivers/staging/rtl8188eu/os_dep/ioctl_linux.c
@@ -2166,8 +2166,6 @@ static int set_group_key(struct adapter *padapter, u8 *key, u8 alg, int keyid)
 		goto exit;
 	}
 
-	memset(psetkeyparm, 0, sizeof(struct setkey_parm));
-
 	psetkeyparm->keyid = (u8)keyid;
 
 	psetkeyparm->algorithm = alg;
diff --git a/drivers/staging/rtl8188eu/os_dep/usb_intf.c b/drivers/staging/rtl8188eu/os_dep/usb_intf.c
index d283341cfe43..32c7225a831e 100644
--- a/drivers/staging/rtl8188eu/os_dep/usb_intf.c
+++ b/drivers/staging/rtl8188eu/os_dep/usb_intf.c
@@ -32,7 +32,7 @@
 #define USB_VENDER_ID_REALTEK		0x0bda
 
 /* DID_USB_v916_20130116 */
-static struct usb_device_id rtw_usb_id_tbl[] = {
+static const struct usb_device_id rtw_usb_id_tbl[] = {
 	/*=== Realtek demoboard ===*/
 	{USB_DEVICE(USB_VENDER_ID_REALTEK, 0x8179)}, /* 8188EUS */
 	{USB_DEVICE(USB_VENDER_ID_REALTEK, 0x0179)}, /* 8188ETV */
@@ -45,6 +45,7 @@ static struct usb_device_id rtw_usb_id_tbl[] = {
 	{USB_DEVICE(0x2001, 0x3311)}, /* DLink GO-USB-N150 REV B1 */
 	{USB_DEVICE(0x2357, 0x010c)}, /* TP-Link TL-WN722N v2 */
 	{USB_DEVICE(0x0df6, 0x0076)}, /* Sitecom N150 v2 */
+	{USB_DEVICE(USB_VENDER_ID_REALTEK, 0xffef)}, /* Rosewill RNX-N150NUB */
 	{}	/* Terminating entry */
 };
 
diff --git a/drivers/staging/rtl8192u/ieee80211/ieee80211_rx.c b/drivers/staging/rtl8192u/ieee80211/ieee80211_rx.c
index a4aedb489e92..cbf8eb4a049d 100644
--- a/drivers/staging/rtl8192u/ieee80211/ieee80211_rx.c
+++ b/drivers/staging/rtl8192u/ieee80211/ieee80211_rx.c
@@ -2385,7 +2385,7 @@ static inline void ieee80211_process_probe_response(
 	struct ieee80211_probe_response *beacon,
 	struct ieee80211_rx_stats *stats)
 {
-	struct ieee80211_network network;
+	struct ieee80211_network *network;
 	struct ieee80211_network *target;
 	struct ieee80211_network *oldest = NULL;
 #ifdef CONFIG_IEEE80211_DEBUG
@@ -2397,7 +2397,10 @@ static inline void ieee80211_process_probe_response(
 	u16 capability;
 	//u8 wmm_info;
 
-	memset(&network, 0, sizeof(struct ieee80211_network));
+	network = kzalloc(sizeof(*network), GFP_ATOMIC);
+	if (!network)
+		goto out;
+
 	capability = le16_to_cpu(beacon->capability);
 	IEEE80211_DEBUG_SCAN(
 		"'%s' (%pM): %c%c%c%c %c%c%c%c-%c%c%c%c %c%c%c%c\n",
@@ -2420,14 +2423,14 @@ static inline void ieee80211_process_probe_response(
 		(capability & (1 << 0x1)) ? '1' : '0',
 		(capability & (1 << 0x0)) ? '1' : '0');
 
-	if (ieee80211_network_init(ieee, beacon, &network, stats)) {
+	if (ieee80211_network_init(ieee, beacon, network, stats)) {
 		IEEE80211_DEBUG_SCAN("Dropped '%s' (%pM) via %s.\n",
 				     escape_essid(info_element->data,
 						  info_element->len),
 				     beacon->header.addr3,
 				     fc == IEEE80211_STYPE_PROBE_RESP ?
 				     "PROBE RESPONSE" : "BEACON");
-		return;
+		goto out;
 	}
 
 	// For Asus EeePc request,
@@ -2437,8 +2440,8 @@ static inline void ieee80211_process_probe_response(
 	//       then wireless adapter should do active scan from ch1~11 and
 	//       passive scan from ch12~14
 
-	if (!IsLegalChannel(ieee, network.channel))
-		return;
+	if (!IsLegalChannel(ieee, network->channel))
+		goto out;
 	if (ieee->bGlobalDomain)
 	{
 		if (fc == IEEE80211_STYPE_PROBE_RESP)
@@ -2446,19 +2449,19 @@ static inline void ieee80211_process_probe_response(
 			// Case 1: Country code
 			if(IS_COUNTRY_IE_VALID(ieee) )
 			{
-				if (!IsLegalChannel(ieee, network.channel)) {
-					printk("GetScanInfo(): For Country code, filter probe response at channel(%d).\n", network.channel);
-					return;
+				if (!IsLegalChannel(ieee, network->channel)) {
+					printk("GetScanInfo(): For Country code, filter probe response at channel(%d).\n", network->channel);
+					goto out;
 				}
 			}
 			// Case 2: No any country code.
 			else
 			{
 				// Filter over channel ch12~14
-				if (network.channel > 11)
+				if (network->channel > 11)
 				{
-					printk("GetScanInfo(): For Global Domain, filter probe response at channel(%d).\n", network.channel);
-					return;
+					printk("GetScanInfo(): For Global Domain, filter probe response at channel(%d).\n", network->channel);
+					goto out;
 				}
 			}
 		}
@@ -2467,19 +2470,19 @@ static inline void ieee80211_process_probe_response(
 			// Case 1: Country code
 			if(IS_COUNTRY_IE_VALID(ieee) )
 			{
-				if (!IsLegalChannel(ieee, network.channel)) {
-					printk("GetScanInfo(): For Country code, filter beacon at channel(%d).\n",network.channel);
-					return;
+				if (!IsLegalChannel(ieee, network->channel)) {
+					printk("GetScanInfo(): For Country code, filter beacon at channel(%d).\n",network->channel);
+					goto out;
 				}
 			}
 			// Case 2: No any country code.
 			else
 			{
 				// Filter over channel ch12~14
-				if (network.channel > 14)
+				if (network->channel > 14)
 				{
-					printk("GetScanInfo(): For Global Domain, filter beacon at channel(%d).\n",network.channel);
-					return;
+					printk("GetScanInfo(): For Global Domain, filter beacon at channel(%d).\n",network->channel);
+					goto out;
 				}
 			}
 		}
@@ -2497,8 +2500,8 @@ static inline void ieee80211_process_probe_response(
 
 	spin_lock_irqsave(&ieee->lock, flags);
 
-	if (is_same_network(&ieee->current_network, &network, ieee)) {
-		update_network(&ieee->current_network, &network);
+	if (is_same_network(&ieee->current_network, network, ieee)) {
+		update_network(&ieee->current_network, network);
 		if ((ieee->current_network.mode == IEEE_N_24G || ieee->current_network.mode == IEEE_G)
 		&& ieee->current_network.berp_info_valid){
 		if(ieee->current_network.erp_value& ERP_UseProtection)
@@ -2512,11 +2515,11 @@ static inline void ieee80211_process_probe_response(
 				ieee->LinkDetectInfo.NumRecvBcnInPeriod++;
 		}
 		else //hidden AP
-			network.flags = (~NETWORK_EMPTY_ESSID & network.flags)|(NETWORK_EMPTY_ESSID & ieee->current_network.flags);
+			network->flags = (~NETWORK_EMPTY_ESSID & network->flags)|(NETWORK_EMPTY_ESSID & ieee->current_network.flags);
 	}
 
 	list_for_each_entry(target, &ieee->network_list, list) {
-		if (is_same_network(target, &network, ieee))
+		if (is_same_network(target, network, ieee))
 			break;
 		if ((oldest == NULL) ||
 		    (target->last_scanned < oldest->last_scanned))
@@ -2545,16 +2548,16 @@ static inline void ieee80211_process_probe_response(
 
 #ifdef CONFIG_IEEE80211_DEBUG
 		IEEE80211_DEBUG_SCAN("Adding '%s' (%pM) via %s.\n",
-				     escape_essid(network.ssid,
-						  network.ssid_len),
-				     network.bssid,
+				     escape_essid(network->ssid,
+						  network->ssid_len),
+				     network->bssid,
 				     fc == IEEE80211_STYPE_PROBE_RESP ?
 				     "PROBE RESPONSE" : "BEACON");
 #endif
-		memcpy(target, &network, sizeof(*target));
+		memcpy(target, network, sizeof(*target));
 		list_add_tail(&target->list, &ieee->network_list);
 		if(ieee->softmac_features & IEEE_SOFTMAC_ASSOCIATE)
-			ieee80211_softmac_new_net(ieee,&network);
+			ieee80211_softmac_new_net(ieee,network);
 	} else {
 		IEEE80211_DEBUG_SCAN("Updating '%s' (%pM) via %s.\n",
 				     escape_essid(target->ssid,
@@ -2570,27 +2573,30 @@ static inline void ieee80211_process_probe_response(
 		renew = !time_after(target->last_scanned + ieee->scan_age, jiffies);
 		//YJ,add,080819,for hidden ap
 		if(is_beacon(beacon->header.frame_ctl) == 0)
-			network.flags = (~NETWORK_EMPTY_ESSID & network.flags)|(NETWORK_EMPTY_ESSID & target->flags);
-		//if(strncmp(network.ssid, "linksys-c",9) == 0)
-		//	printk("====>2 network.ssid=%s FLAG=%d target.ssid=%s FLAG=%d\n", network.ssid, network.flags, target->ssid, target->flags);
-		if(((network.flags & NETWORK_EMPTY_ESSID) == NETWORK_EMPTY_ESSID) \
-		    && (((network.ssid_len > 0) && (strncmp(target->ssid, network.ssid, network.ssid_len)))\
-		    ||((ieee->current_network.ssid_len == network.ssid_len)&&(strncmp(ieee->current_network.ssid, network.ssid, network.ssid_len) == 0)&&(ieee->state == IEEE80211_NOLINK))))
+			network->flags = (~NETWORK_EMPTY_ESSID & network->flags)|(NETWORK_EMPTY_ESSID & target->flags);
+		//if(strncmp(network->ssid, "linksys-c",9) == 0)
+		//	printk("====>2 network->ssid=%s FLAG=%d target.ssid=%s FLAG=%d\n", network->ssid, network->flags, target->ssid, target->flags);
+		if(((network->flags & NETWORK_EMPTY_ESSID) == NETWORK_EMPTY_ESSID) \
+		    && (((network->ssid_len > 0) && (strncmp(target->ssid, network->ssid, network->ssid_len)))\
+		    ||((ieee->current_network.ssid_len == network->ssid_len)&&(strncmp(ieee->current_network.ssid, network->ssid, network->ssid_len) == 0)&&(ieee->state == IEEE80211_NOLINK))))
 			renew = 1;
 		//YJ,add,080819,for hidden ap,end
 
-		update_network(target, &network);
+		update_network(target, network);
 		if(renew && (ieee->softmac_features & IEEE_SOFTMAC_ASSOCIATE))
-			ieee80211_softmac_new_net(ieee,&network);
+			ieee80211_softmac_new_net(ieee,network);
 	}
 
 	spin_unlock_irqrestore(&ieee->lock, flags);
-	if (is_beacon(beacon->header.frame_ctl)&&is_same_network(&ieee->current_network, &network, ieee)&&\
+	if (is_beacon(beacon->header.frame_ctl)&&is_same_network(&ieee->current_network, network, ieee)&&\
 		(ieee->state == IEEE80211_LINKED)) {
 		if (ieee->handle_beacon != NULL) {
 			ieee->handle_beacon(ieee->dev,beacon,&ieee->current_network);
 		}
 	}
+
+out:
+	kfree(network);
 }
 
 void ieee80211_rx_mgt(struct ieee80211_device *ieee,
diff --git a/drivers/staging/rtl8192u/ieee80211/rtl819x_HTProc.c b/drivers/staging/rtl8192u/ieee80211/rtl819x_HTProc.c
index 60720997784b..9248dbcf3370 100644
--- a/drivers/staging/rtl8192u/ieee80211/rtl819x_HTProc.c
+++ b/drivers/staging/rtl8192u/ieee80211/rtl819x_HTProc.c
@@ -176,7 +176,7 @@ void HTDebugHTInfo(u8 *InfoIE, u8 *TitleString)
 	IEEE80211_DEBUG(IEEE80211_DL_HT, "<Log HT Information Element>. Called by %s\n", TitleString);
 
 	IEEE80211_DEBUG(IEEE80211_DL_HT, "\tPrimary channel = %d\n", pHTInfoEle->ControlChl);
-	IEEE80211_DEBUG(IEEE80211_DL_HT, "\tSenondary channel =");
+	IEEE80211_DEBUG(IEEE80211_DL_HT, "\tSecondary channel =");
 	switch (pHTInfoEle->ExtChlOffset)
 	{
 		case 0:
diff --git a/drivers/staging/rtl8192u/r8192U_core.c b/drivers/staging/rtl8192u/r8192U_core.c
index 779ecdbc4e17..46b3f19e0878 100644
--- a/drivers/staging/rtl8192u/r8192U_core.c
+++ b/drivers/staging/rtl8192u/r8192U_core.c
@@ -270,8 +270,7 @@ int write_nic_byte_E(struct net_device *dev, int indx, u8 data)
 	kfree(usbdata);
 
 	if (status < 0) {
-		netdev_err(dev, "write_nic_byte_E TimeOut! status: %d\n",
-			   status);
+		netdev_err(dev, "%s TimeOut! status: %d\n", __func__, status);
 		return status;
 	}
 	return 0;
@@ -321,7 +320,7 @@ int write_nic_byte(struct net_device *dev, int indx, u8 data)
 	kfree(usbdata);
 
 	if (status < 0) {
-		netdev_err(dev, "write_nic_byte TimeOut! status: %d\n", status);
+		netdev_err(dev, "%s TimeOut! status: %d\n", __func__, status);
 		return status;
 	}
 
@@ -348,7 +347,7 @@ int write_nic_word(struct net_device *dev, int indx, u16 data)
 	kfree(usbdata);
 
 	if (status < 0) {
-		netdev_err(dev, "write_nic_word TimeOut! status: %d\n", status);
+		netdev_err(dev, "%s TimeOut! status: %d\n", __func__, status);
 		return status;
 	}
 
@@ -376,8 +375,7 @@ int write_nic_dword(struct net_device *dev, int indx, u32 data)
 
 
 	if (status < 0) {
-		netdev_err(dev, "write_nic_dword TimeOut! status: %d\n",
-			   status);
+		netdev_err(dev, "%s TimeOut! status: %d\n", __func__, status);
 		return status;
 	}
 
@@ -3095,7 +3093,8 @@ static RESET_TYPE TxCheckStuck(struct net_device *dev)
 	if (bCheckFwTxCnt) {
 		if (HalTxCheckStuck819xUsb(dev)) {
 			RT_TRACE(COMP_RESET,
-				 "TxCheckStuck(): Fw indicates no Tx condition!\n");
+				 "%s: Fw indicates no Tx condition!\n",
+				 __func__);
 			return RESET_TYPE_SILENT;
 		}
 	}
@@ -3237,7 +3236,7 @@ static void CamRestoreAllEntry(struct net_device *dev)
 	static u8	CAM_CONST_BROAD[] = {
 		0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
 
-	RT_TRACE(COMP_SEC, "CamRestoreAllEntry:\n");
+	RT_TRACE(COMP_SEC, "%s:\n", __func__);
 
 
 	if ((priv->ieee80211->pairwise_key_type == KEY_TYPE_WEP40) ||
@@ -3835,8 +3834,8 @@ static u8 HwRateToMRate90(bool bIsHT, u8 rate)
 		default:
 			ret_rate = 0xff;
 			RT_TRACE(COMP_RECV,
-				 "HwRateToMRate90(): Non supported Rate [%x], bIsHT = %d!!!\n",
-				 rate, bIsHT);
+				 "%s: Non supported Rate [%x], bIsHT = %d!!!\n",
+				 __func__, rate, bIsHT);
 			break;
 		}
 
@@ -3897,8 +3896,8 @@ static u8 HwRateToMRate90(bool bIsHT, u8 rate)
 		default:
 			ret_rate = 0xff;
 			RT_TRACE(COMP_RECV,
-				 "HwRateToMRate90(): Non supported Rate [%x], bIsHT = %d!!!\n",
-				 rate, bIsHT);
+				 "%s: Non supported Rate [%x], bIsHT = %d!!!\n",
+				 __func__, rate, bIsHT);
 			break;
 		}
 	}
diff --git a/drivers/staging/rtl8192u/r8192U_hw.h b/drivers/staging/rtl8192u/r8192U_hw.h
index 174ccf618d3e..00a123d44207 100644
--- a/drivers/staging/rtl8192u/r8192U_hw.h
+++ b/drivers/staging/rtl8192u/r8192U_hw.h
@@ -20,25 +20,24 @@
 #ifndef R8192_HW
 #define R8192_HW
 
-typedef enum _VERSION_819xU{
+typedef enum _VERSION_819xU {
 	VERSION_819xU_A, // A-cut
 	VERSION_819xU_B, // B-cut
 	VERSION_819xU_C,// C-cut
 } VERSION_819xU, *PVERSION_819xU;
 //added for different RF type
-typedef enum _RT_RF_TYPE_DEF
-{
+typedef enum _RT_RF_TYPE_DEF {
 	RF_1T2R = 0,
 	RF_2T4R,
 
 	RF_819X_MAX_TYPE
-}RT_RF_TYPE_DEF;
+} RT_RF_TYPE_DEF;
 
 
-typedef enum _BaseBand_Config_Type{
+typedef enum _BaseBand_Config_Type {
 	BaseBand_Config_PHY_REG = 0,			//Radio Path A
 	BaseBand_Config_AGC_TAB = 1,			//Radio Path B
-}BaseBand_Config_Type, *PBaseBand_Config_Type;
+} BaseBand_Config_Type, *PBaseBand_Config_Type;
 #define	RTL8187_REQT_READ	0xc0
 #define	RTL8187_REQT_WRITE	0x40
 #define	RTL8187_REQ_GET_REGS	0x05
diff --git a/drivers/staging/rtl8192u/r819xU_cmdpkt.c b/drivers/staging/rtl8192u/r819xU_cmdpkt.c
index 87ab3ba760fc..ae9a4f1ac8fd 100644
--- a/drivers/staging/rtl8192u/r819xU_cmdpkt.c
+++ b/drivers/staging/rtl8192u/r819xU_cmdpkt.c
@@ -294,7 +294,7 @@ static void cmpk_handle_query_config_rx(struct net_device *dev, u8 *pmsg)
 	 * windows OS. So we have to read the content byte by byte or transfer
 	 * endian type before copy the message copy.
 	 */
-	rx_query_cfg.cfg_action		= (pmsg[4] & 0x80000000) >> 31;
+	rx_query_cfg.cfg_action		= (pmsg[4] & 0x80) >> 7;
 	rx_query_cfg.cfg_type		= (pmsg[4] & 0x60) >> 5;
 	rx_query_cfg.cfg_size		= (pmsg[4] & 0x18) >> 3;
 	rx_query_cfg.cfg_page		= (pmsg[6] & 0x0F) >> 0;
diff --git a/drivers/staging/rtl8712/mlme_linux.c b/drivers/staging/rtl8712/mlme_linux.c
index 20372659d15d..a077069d6227 100644
--- a/drivers/staging/rtl8712/mlme_linux.c
+++ b/drivers/staging/rtl8712/mlme_linux.c
@@ -111,8 +111,8 @@ void r8712_os_indicate_disconnect(struct _adapter *adapter)
 		 */
 
 		memcpy(&backupPMKIDList[0],
-			&adapter->securitypriv.PMKIDList[0],
-			sizeof(struct RT_PMKID_LIST) * NUM_PMKID_CACHE);
+		       &adapter->securitypriv.PMKIDList[0],
+		       sizeof(struct RT_PMKID_LIST) * NUM_PMKID_CACHE);
 		backupPMKIDIndex = adapter->securitypriv.PMKIDIndex;
 		backupTKIPCountermeasure =
 			adapter->securitypriv.btkip_countermeasure;
diff --git a/drivers/staging/rtl8712/rtl8712_cmd.c b/drivers/staging/rtl8712/rtl8712_cmd.c
index 5346c657485d..0104aced113e 100644
--- a/drivers/staging/rtl8712/rtl8712_cmd.c
+++ b/drivers/staging/rtl8712/rtl8712_cmd.c
@@ -385,7 +385,7 @@ _next:
 			if (blnPending)
 				wr_sz += 8;   /* Append 8 bytes */
 			r8712_write_mem(padapter, RTL8712_DMA_H2CCMD, wr_sz,
-				       (u8 *)pdesc);
+					(u8 *)pdesc);
 			pcmdpriv->cmd_seq++;
 			if (pcmd->cmdcode == GEN_CMD_CODE(_CreateBss)) {
 				pcmd->res = H2C_SUCCESS;
diff --git a/drivers/staging/rtl8712/rtl8712_efuse.c b/drivers/staging/rtl8712/rtl8712_efuse.c
index 205298e23656..d90213eb5e20 100644
--- a/drivers/staging/rtl8712/rtl8712_efuse.c
+++ b/drivers/staging/rtl8712/rtl8712_efuse.c
@@ -347,7 +347,7 @@ static u8 fix_header(struct _adapter *padapter, u8 header, u16 header_addr)
 				ret = false;
 				if (value == 0xFF) /* write again */
 					efuse_one_byte_write(padapter, addr,
-							pkt.data[i * 2]);
+							     pkt.data[i * 2]);
 			}
 			if (!efuse_one_byte_read(padapter, addr + 1, &value)) {
 				ret = false;
diff --git a/drivers/staging/rtl8712/rtl8712_xmit.c b/drivers/staging/rtl8712/rtl8712_xmit.c
index 7fe626583c8a..42d014007764 100644
--- a/drivers/staging/rtl8712/rtl8712_xmit.c
+++ b/drivers/staging/rtl8712/rtl8712_xmit.c
@@ -640,7 +640,7 @@ int r8712_xmitframe_complete(struct _adapter *padapter,
 	/* 1st frame dequeued */
 	pxmitframe = dequeue_xframe_ex(pxmitpriv, phwxmits, hwentry);
 	/* need to remember the 1st frame */
-	if (pxmitframe != NULL) {
+	if (pxmitframe) {
 
 #ifdef CONFIG_R8712_TX_AGGR
 		/* 1. dequeue 2nd frame
@@ -653,13 +653,13 @@ int r8712_xmitframe_complete(struct _adapter *padapter,
 			r8712_free_xmitbuf(pxmitpriv, pxmitbuf);
 			return false;
 		}
-		if (p2ndxmitframe != NULL)
+		if (p2ndxmitframe)
 			if (p2ndxmitframe->frame_tag != DATA_FRAMETAG) {
 				r8712_free_xmitbuf(pxmitpriv, pxmitbuf);
 				return false;
 			}
 		r8712_xmitframe_aggr_1st(pxmitbuf, pxmitframe);
-		if (p2ndxmitframe != NULL) {
+		if (p2ndxmitframe) {
 			u16 total_length;
 
 			total_length = r8712_xmitframe_aggr_next(
@@ -667,7 +667,7 @@ int r8712_xmitframe_complete(struct _adapter *padapter,
 			do {
 				p2ndxmitframe = dequeue_xframe_ex(
 					pxmitpriv, phwxmits, hwentry);
-				if (p2ndxmitframe != NULL)
+				if (p2ndxmitframe)
 					total_length =
 						r8712_xmitframe_aggr_next(
 							pxmitbuf,
diff --git a/drivers/staging/rtl8712/usb_intf.c b/drivers/staging/rtl8712/usb_intf.c
index 897d4621a5ce..b3e266bd57ab 100644
--- a/drivers/staging/rtl8712/usb_intf.c
+++ b/drivers/staging/rtl8712/usb_intf.c
@@ -47,7 +47,7 @@ static int r871xu_drv_init(struct usb_interface *pusb_intf,
 
 static void r871xu_dev_remove(struct usb_interface *pusb_intf);
 
-static struct usb_device_id rtl871x_usb_id_tbl[] = {
+static const struct usb_device_id rtl871x_usb_id_tbl[] = {
 
 /* RTL8188SU */
 	/* Realtek */
diff --git a/drivers/staging/rtl8723bs/core/rtw_btcoex.c b/drivers/staging/rtl8723bs/core/rtw_btcoex.c
index 3c5cb78b52ea..01f78d1671de 100644
--- a/drivers/staging/rtl8723bs/core/rtw_btcoex.c
+++ b/drivers/staging/rtl8723bs/core/rtw_btcoex.c
@@ -55,7 +55,7 @@ void rtw_btcoex_ConnectNotify(struct adapter *padapter, u8 action)
 
 void rtw_btcoex_MediaStatusNotify(struct adapter *padapter, u8 mediaStatus)
 {
-	if ((RT_MEDIA_CONNECT == mediaStatus)
+	if ((mediaStatus == RT_MEDIA_CONNECT)
 		&& (check_fwstate(&padapter->mlmepriv, WIFI_AP_STATE) == true)) {
 		rtw_hal_set_hwreg(padapter, HW_VAR_DL_RSVD_PAGE, NULL);
 	}
diff --git a/drivers/staging/rtl8723bs/core/rtw_cmd.c b/drivers/staging/rtl8723bs/core/rtw_cmd.c
index 080c81b9aa94..d381827dba3b 100644
--- a/drivers/staging/rtl8723bs/core/rtw_cmd.c
+++ b/drivers/staging/rtl8723bs/core/rtw_cmd.c
@@ -432,7 +432,7 @@ int rtw_cmd_thread(void *context)
 	unsigned long cmd_process_time;
 	u8 (*cmd_hdl)(struct adapter *padapter, u8 *pbuf);
 	void (*pcmd_callback)(struct adapter *dev, struct cmd_obj *pcmd);
-	struct adapter *padapter = (struct adapter *)context;
+	struct adapter *padapter = context;
 	struct cmd_priv *pcmdpriv = &(padapter->cmdpriv);
 	struct drvextra_cmd_parm *extra_parm = NULL;
 
diff --git a/drivers/staging/rtl8723bs/core/rtw_efuse.c b/drivers/staging/rtl8723bs/core/rtw_efuse.c
index 8e29802fc67f..44b92ef5db92 100644
--- a/drivers/staging/rtl8723bs/core/rtw_efuse.c
+++ b/drivers/staging/rtl8723bs/core/rtw_efuse.c
@@ -163,7 +163,7 @@ Efuse_CalculateWordCnts(u8 word_en)
 /* 	Description: */
 /* 		1. Execute E-Fuse read byte operation according as map offset and */
 /* 		    save to E-Fuse table. */
-/* 		2. Refered from SD1 Richard. */
+/* 		2. Referred from SD1 Richard. */
 /*  */
 /* 	Assumption: */
 /* 		1. Boot from E-Fuse and successfully auto-load. */
diff --git a/drivers/staging/rtl8723bs/core/rtw_ioctl_set.c b/drivers/staging/rtl8723bs/core/rtw_ioctl_set.c
index e0793f8d329d..d815a693fa64 100644
--- a/drivers/staging/rtl8723bs/core/rtw_ioctl_set.c
+++ b/drivers/staging/rtl8723bs/core/rtw_ioctl_set.c
@@ -127,7 +127,6 @@ u8 rtw_do_join(struct adapter *padapter)
 
 				pibss = padapter->registrypriv.dev_network.MacAddress;
 
-				memset(&pdev_network->Ssid, 0, sizeof(struct ndis_802_11_ssid));
 				memcpy(&pdev_network->Ssid, &pmlmepriv->assoc_ssid, sizeof(struct ndis_802_11_ssid));
 
 				rtw_update_registrypriv_dev_network(padapter);
diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme.c b/drivers/staging/rtl8723bs/core/rtw_mlme.c
index d5ab12305e59..6b778206a1a3 100644
--- a/drivers/staging/rtl8723bs/core/rtw_mlme.c
+++ b/drivers/staging/rtl8723bs/core/rtw_mlme.c
@@ -922,7 +922,6 @@ void rtw_surveydone_event_callback(struct adapter	*adapter, u8 *pbuf)
 
 					RT_TRACE(_module_rtl871x_mlme_c_, _drv_err_, ("switching to adhoc master\n"));
 
-					memset(&pdev_network->Ssid, 0, sizeof(struct ndis_802_11_ssid));
 					memcpy(&pdev_network->Ssid, &pmlmepriv->assoc_ssid, sizeof(struct ndis_802_11_ssid));
 
 					rtw_update_registrypriv_dev_network(adapter);
@@ -1384,7 +1383,7 @@ static void rtw_joinbss_update_network(struct adapter *padapter, struct wlan_net
 /* define REJOIN */
 void rtw_joinbss_event_prehandle(struct adapter *adapter, u8 *pbuf)
 {
-	static u8 retry = 0;
+	static u8 retry;
 	u8 timer_cancelled;
 	struct sta_info *ptarget_sta = NULL, *pcur_sta = NULL;
 	struct	sta_priv *pstapriv = &adapter->stapriv;
@@ -1774,7 +1773,6 @@ void rtw_stadel_event_callback(struct adapter *adapter, u8 *pbuf)
 
 			memcpy(pdev_network, &tgt_network->network, get_wlan_bssid_ex_sz(&tgt_network->network));
 
-			memset(&pdev_network->Ssid, 0, sizeof(struct ndis_802_11_ssid));
 			memcpy(&pdev_network->Ssid, &pmlmepriv->assoc_ssid, sizeof(struct ndis_802_11_ssid));
 
 			rtw_update_registrypriv_dev_network(adapter);
@@ -2498,8 +2496,7 @@ sint rtw_restruct_sec_ie(struct adapter *adapter, u8 *in_ie, u8 *out_ie, uint in
 	uint	ndisauthmode = psecuritypriv->ndisauthtype;
 
 	RT_TRACE(_module_rtl871x_mlme_c_, _drv_notice_,
-		 ("+rtw_restruct_sec_ie: ndisauthmode =%d ndissecuritytype =%d\n",
-		  ndisauthmode, ndissecuritytype));
+		 ("+rtw_restruct_sec_ie: ndisauthmode =%d\n", ndisauthmode));
 
 	/* copy fixed ie only */
 	memcpy(out_ie, in_ie, 12);
diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c
index 17d881d66910..b6d137f505e1 100644
--- a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c
+++ b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c
@@ -1228,7 +1228,7 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 	}
 
 	pstat = rtw_get_stainfo(pstapriv, GetAddr2Ptr(pframe));
-	if (pstat == (struct sta_info *)NULL) {
+	if (pstat == NULL) {
 		status = _RSON_CLS2_;
 		goto asoc_class2_error;
 	}
@@ -2392,7 +2392,7 @@ s32 dump_mgntframe_and_wait(struct adapter *padapter, struct xmit_frame *pmgntfr
 
 s32 dump_mgntframe_and_wait_ack(struct adapter *padapter, struct xmit_frame *pmgntframe)
 {
-	static u8 seq_no = 0;
+	static u8 seq_no;
 	s32 ret = _FAIL;
 	u32 timeout_ms = 500;/*   500ms */
 	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
diff --git a/drivers/staging/rtl8723bs/core/rtw_odm.c b/drivers/staging/rtl8723bs/core/rtw_odm.c
index 3144e8ec2fa2..edbcaeb9f8c2 100644
--- a/drivers/staging/rtl8723bs/core/rtw_odm.c
+++ b/drivers/staging/rtl8723bs/core/rtw_odm.c
@@ -18,7 +18,7 @@
 #include <rtw_odm.h>
 #include <hal_data.h>
 
-static const char *odm_comp_str[] = {
+static const char * const odm_comp_str[] = {
 	/* BIT0 */"ODM_COMP_DIG",
 	/* BIT1 */"ODM_COMP_RA_MASK",
 	/* BIT2 */"ODM_COMP_DYNAMIC_TXPWR",
@@ -55,7 +55,7 @@ static const char *odm_comp_str[] = {
 
 #define RTW_ODM_COMP_MAX 32
 
-static const char *odm_ability_str[] = {
+static const char * const odm_ability_str[] = {
 	/* BIT0 */"ODM_BB_DIG",
 	/* BIT1 */"ODM_BB_RA_MASK",
 	/* BIT2 */"ODM_BB_DYNAMIC_TXPWR",
@@ -87,7 +87,7 @@ static const char *odm_ability_str[] = {
 
 #define RTW_ODM_ABILITY_MAX 27
 
-static const char *odm_dbg_level_str[] = {
+static const char * const odm_dbg_level_str[] = {
 	NULL,
 	"ODM_DBG_OFF",
 	"ODM_DBG_SERIOUS",
@@ -127,7 +127,8 @@ void rtw_odm_dbg_level_msg(void *sel, struct adapter *adapter)
 	DBG_871X_SEL_NL(sel, "odm.DebugLevel = %u\n", dbg_level);
 	for (i = 0; i < RTW_ODM_DBG_LEVEL_NUM; i++) {
 		if (odm_dbg_level_str[i])
-			DBG_871X_SEL_NL(sel, "%u %s\n", i, odm_dbg_level_str[i]);
+			DBG_871X_SEL_NL(sel, "%u %s\n",
+					i, odm_dbg_level_str[i]);
 	}
 }
 
@@ -161,20 +162,23 @@ void rtw_odm_adaptivity_parm_msg(void *sel, struct adapter *adapter)
 	struct hal_com_data *pHalData = GET_HAL_DATA(adapter);
 	DM_ODM_T *odm = &pHalData->odmpriv;
 
-	DBG_871X_SEL_NL(sel, "%10s %16s %8s %10s %11s %14s\n"
-		, "TH_L2H_ini", "TH_EDCCA_HL_diff", "IGI_Base", "ForceEDCCA", "AdapEn_RSSI", "IGI_LowerBound");
-	DBG_871X_SEL_NL(sel, "0x%-8x %-16d 0x%-6x %-10d %-11u %-14u\n"
-		, (u8)odm->TH_L2H_ini
-		, odm->TH_EDCCA_HL_diff
-		, odm->IGI_Base
-		, odm->ForceEDCCA
-		, odm->AdapEn_RSSI
-		, odm->IGI_LowerBound
+	DBG_871X_SEL_NL(sel, "%10s %16s %8s %10s %11s %14s\n",
+			"TH_L2H_ini", "TH_EDCCA_HL_diff", "IGI_Base",
+			"ForceEDCCA", "AdapEn_RSSI", "IGI_LowerBound");
+	DBG_871X_SEL_NL(sel, "0x%-8x %-16d 0x%-6x %-10d %-11u %-14u\n",
+			(u8)odm->TH_L2H_ini,
+			odm->TH_EDCCA_HL_diff,
+			odm->IGI_Base,
+			odm->ForceEDCCA,
+			odm->AdapEn_RSSI,
+			odm->IGI_LowerBound
 	);
 }
 
-void rtw_odm_adaptivity_parm_set(struct adapter *adapter, s8 TH_L2H_ini, s8 TH_EDCCA_HL_diff,
-	s8 IGI_Base, bool ForceEDCCA, u8 AdapEn_RSSI, u8 IGI_LowerBound)
+void rtw_odm_adaptivity_parm_set(struct adapter *adapter, s8 TH_L2H_ini,
+				 s8 TH_EDCCA_HL_diff, s8 IGI_Base,
+				 bool ForceEDCCA, u8 AdapEn_RSSI,
+				 u8 IGI_LowerBound)
 {
 	struct hal_com_data *pHalData = GET_HAL_DATA(adapter);
 	DM_ODM_T *odm = &pHalData->odmpriv;
@@ -190,8 +194,8 @@ void rtw_odm_adaptivity_parm_set(struct adapter *adapter, s8 TH_L2H_ini, s8 TH_E
 void rtw_odm_get_perpkt_rssi(void *sel, struct adapter *adapter)
 {
 	struct hal_com_data *hal_data = GET_HAL_DATA(adapter);
-	DM_ODM_T *odm = &(hal_data->odmpriv);
+	DM_ODM_T *odm = &hal_data->odmpriv;
 
 	DBG_871X_SEL_NL(sel, "RxRate = %s, RSSI_A = %d(%%), RSSI_B = %d(%%)\n",
-	HDATA_RATE(odm->RxRate), odm->RSSI_A, odm->RSSI_B);
+			HDATA_RATE(odm->RxRate), odm->RSSI_A, odm->RSSI_B);
 }
diff --git a/drivers/staging/rtl8723bs/core/rtw_pwrctrl.c b/drivers/staging/rtl8723bs/core/rtw_pwrctrl.c
index f708dbf5bfd4..aabdaafcbdd3 100644
--- a/drivers/staging/rtl8723bs/core/rtw_pwrctrl.c
+++ b/drivers/staging/rtl8723bs/core/rtw_pwrctrl.c
@@ -210,8 +210,8 @@ void pwr_state_check_handler(RTW_TIMER_HDL_ARGS)
 
 void traffic_check_for_leave_lps(struct adapter *padapter, u8 tx, u32 tx_packets)
 {
-	static unsigned long start_time = 0;
-	static u32 xmit_cnt = 0;
+	static unsigned long start_time;
+	static u32 xmit_cnt;
 	u8 bLeaveLPS = false;
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 
@@ -829,7 +829,7 @@ static void pwr_rpwm_timeout_handler(void *FunctionContext)
 	struct pwrctrl_priv *pwrpriv;
 
 
-	padapter = (struct adapter *)FunctionContext;
+	padapter = FunctionContext;
 	pwrpriv = adapter_to_pwrctl(padapter);
 	DBG_871X("+%s: rpwm = 0x%02X cpwm = 0x%02X\n", __func__, pwrpriv->rpwm, pwrpriv->cpwm);
 
diff --git a/drivers/staging/rtl8723bs/core/rtw_recv.c b/drivers/staging/rtl8723bs/core/rtw_recv.c
index 695a5c958c80..68a6303e2754 100644
--- a/drivers/staging/rtl8723bs/core/rtw_recv.c
+++ b/drivers/staging/rtl8723bs/core/rtw_recv.c
@@ -1005,7 +1005,7 @@ sint ap2sta_data_frame(
 			if (*psta == NULL) {
 
 				/* for AP multicast issue , modify by yiwei */
-				static unsigned long send_issue_deauth_time = 0;
+				static unsigned long send_issue_deauth_time;
 
 				/* DBG_871X("After send deauth , %u ms has elapsed.\n", jiffies_to_msecs(jiffies - send_issue_deauth_time)); */
 
@@ -2360,7 +2360,7 @@ _err_exit:
 
 void rtw_reordering_ctrl_timeout_handler(void *pcontext)
 {
-	struct recv_reorder_ctrl *preorder_ctrl = (struct recv_reorder_ctrl *)pcontext;
+	struct recv_reorder_ctrl *preorder_ctrl = pcontext;
 	struct adapter *padapter = preorder_ctrl->padapter;
 	struct __queue *ppending_recvframe_queue = &preorder_ctrl->pending_recvframe_queue;
 
diff --git a/drivers/staging/rtl8723bs/core/rtw_security.c b/drivers/staging/rtl8723bs/core/rtw_security.c
index e832f16997b7..06a7e4059fbb 100644
--- a/drivers/staging/rtl8723bs/core/rtw_security.c
+++ b/drivers/staging/rtl8723bs/core/rtw_security.c
@@ -162,7 +162,7 @@ static void arcfour_encrypt(
 		dest[i] = src[i] ^ (unsigned char)arcfour_byte(parc4ctx);
 }
 
-static sint bcrc32initialized = 0;
+static sint bcrc32initialized;
 static u32 crc32_table[256];
 
 
@@ -791,9 +791,9 @@ u32 rtw_tkip_decrypt(struct adapter *padapter, u8 *precvframe)
 		stainfo = rtw_get_stainfo(&padapter->stapriv, &prxattrib->ta[0]);
 		if (stainfo != NULL) {
 			if (IS_MCAST(prxattrib->ra)) {
-				static unsigned long start = 0;
-				static u32 no_gkey_bc_cnt = 0;
-				static u32 no_gkey_mc_cnt = 0;
+				static unsigned long start;
+				static u32 no_gkey_bc_cnt;
+				static u32 no_gkey_mc_cnt;
 
 				if (psecuritypriv->binstallGrpkey == false) {
 					res = _FAIL;
@@ -1882,9 +1882,9 @@ u32 rtw_aes_decrypt(struct adapter *padapter, u8 *precvframe)
 			RT_TRACE(_module_rtl871x_security_c_, _drv_err_, ("rtw_aes_decrypt: stainfo!= NULL!!!\n"));
 
 			if (IS_MCAST(prxattrib->ra)) {
-				static unsigned long start = 0;
-				static u32 no_gkey_bc_cnt = 0;
-				static u32 no_gkey_mc_cnt = 0;
+				static unsigned long start;
+				static u32 no_gkey_bc_cnt;
+				static u32 no_gkey_mc_cnt;
 
 				/* DBG_871X("rx bc/mc packets, to perform sw rtw_aes_decrypt\n"); */
 				/* prwskey = psecuritypriv->dot118021XGrpKey[psecuritypriv->dot118021XGrpKeyid].skey; */
diff --git a/drivers/staging/rtl8723bs/core/rtw_xmit.c b/drivers/staging/rtl8723bs/core/rtw_xmit.c
index 8f2c9a6658bf..022f654419e4 100644
--- a/drivers/staging/rtl8723bs/core/rtw_xmit.c
+++ b/drivers/staging/rtl8723bs/core/rtw_xmit.c
@@ -2301,8 +2301,8 @@ static void do_queue_select(struct adapter	*padapter, struct pkt_attrib *pattrib
  */
 s32 rtw_xmit(struct adapter *padapter, _pkt **ppkt)
 {
-	static unsigned long start = 0;
-	static u32 drop_cnt = 0;
+	static unsigned long start;
+	static u32 drop_cnt;
 
 	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
 	struct xmit_frame *pxmitframe = NULL;
@@ -3002,7 +3002,7 @@ int rtw_xmit_thread(void *context)
 
 
 	err = _SUCCESS;
-	padapter = (struct adapter *)context;
+	padapter = context;
 
 	thread_enter("RTW_XMIT_THREAD");
 
diff --git a/drivers/staging/rtl8723bs/hal/hal_btcoex.c b/drivers/staging/rtl8723bs/hal/hal_btcoex.c
index 9e08a4de4895..86fee109e42d 100644
--- a/drivers/staging/rtl8723bs/hal/hal_btcoex.c
+++ b/drivers/staging/rtl8723bs/hal/hal_btcoex.c
@@ -419,10 +419,10 @@ static u8 halbtcoutsrc_Get(void *pBtcContext, u8 getType, void *pOutBuf)
 	padapter = pBtCoexist->Adapter;
 	pHalData = GET_HAL_DATA(padapter);
 	mlmeext = &padapter->mlmeextpriv;
-	pu8 = (u8 *)pOutBuf;
-	pS4Tmp = (s32 *)pOutBuf;
-	pU4Tmp = (u32 *)pOutBuf;
-	pU1Tmp = (u8 *)pOutBuf;
+	pu8 = pOutBuf;
+	pS4Tmp = pOutBuf;
+	pU4Tmp = pOutBuf;
+	pU1Tmp = pOutBuf;
 	ret = true;
 
 	switch (getType) {
@@ -585,9 +585,9 @@ static u8 halbtcoutsrc_Set(void *pBtcContext, u8 setType, void *pInBuf)
 	pBtCoexist = (PBTC_COEXIST)pBtcContext;
 	padapter = pBtCoexist->Adapter;
 	pHalData = GET_HAL_DATA(padapter);
-	pu8 = (u8 *)pInBuf;
-	pU1Tmp = (u8 *)pInBuf;
-	pU4Tmp = (u32 *)pInBuf;
+	pu8 = pInBuf;
+	pU1Tmp = pInBuf;
+	pU4Tmp = pInBuf;
 	ret = true;
 
 	if (!halbtcoutsrc_IsBtCoexistAvailable(pBtCoexist))
diff --git a/drivers/staging/rtl8723bs/hal/hal_com.c b/drivers/staging/rtl8723bs/hal/hal_com.c
index e3a98322f475..3e63b6d9c097 100644
--- a/drivers/staging/rtl8723bs/hal/hal_com.c
+++ b/drivers/staging/rtl8723bs/hal/hal_com.c
@@ -1311,7 +1311,7 @@ void SetHalODMVar(
 	switch (eVariable) {
 	case HAL_ODM_STA_INFO:
 		{
-			struct sta_info *psta = (struct sta_info *)pValue1;
+			struct sta_info *psta = pValue1;
 			if (bSet) {
 				DBG_8192C("### Set STA_(%d) info ###\n", psta->mac_id);
 				ODM_CmnInfoPtrArrayHook(podmpriv, ODM_CMNINFO_STA_STATUS, psta->mac_id, psta);
@@ -1333,7 +1333,7 @@ void SetHalODMVar(
 	#if defined(CONFIG_SIGNAL_DISPLAY_DBM) && defined(CONFIG_BACKGROUND_NOISE_MONITOR)
 	case HAL_ODM_NOISE_MONITOR:
 		{
-			struct noise_info *pinfo = (struct noise_info *)pValue1;
+			struct noise_info *pinfo = pValue1;
 
 			#ifdef DBG_NOISE_MONITOR
 			DBG_8192C("### Noise monitor chan(%d)-bPauseDIG:%d, IGIValue:0x%02x, max_time:%d (ms) ###\n",
diff --git a/drivers/staging/rtl8723bs/hal/odm.c b/drivers/staging/rtl8723bs/hal/odm.c
index 2dbf19971e04..ff43bb26950b 100644
--- a/drivers/staging/rtl8723bs/hal/odm.c
+++ b/drivers/staging/rtl8723bs/hal/odm.c
@@ -592,95 +592,95 @@ void ODM_CmnInfoHook(PDM_ODM_T pDM_Odm, ODM_CMNINFO_E CmnInfo, void *pValue)
 	/*  Dynamic call by reference pointer. */
 	/*  */
 	case ODM_CMNINFO_MAC_PHY_MODE:
-		pDM_Odm->pMacPhyMode = (u8 *)pValue;
+		pDM_Odm->pMacPhyMode = pValue;
 		break;
 
 	case ODM_CMNINFO_TX_UNI:
-		pDM_Odm->pNumTxBytesUnicast = (u64 *)pValue;
+		pDM_Odm->pNumTxBytesUnicast = pValue;
 		break;
 
 	case ODM_CMNINFO_RX_UNI:
-		pDM_Odm->pNumRxBytesUnicast = (u64 *)pValue;
+		pDM_Odm->pNumRxBytesUnicast = pValue;
 		break;
 
 	case ODM_CMNINFO_WM_MODE:
-		pDM_Odm->pwirelessmode = (u8 *)pValue;
+		pDM_Odm->pwirelessmode = pValue;
 		break;
 
 	case ODM_CMNINFO_BAND:
-		pDM_Odm->pBandType = (u8 *)pValue;
+		pDM_Odm->pBandType = pValue;
 		break;
 
 	case ODM_CMNINFO_SEC_CHNL_OFFSET:
-		pDM_Odm->pSecChOffset = (u8 *)pValue;
+		pDM_Odm->pSecChOffset = pValue;
 		break;
 
 	case ODM_CMNINFO_SEC_MODE:
-		pDM_Odm->pSecurity = (u8 *)pValue;
+		pDM_Odm->pSecurity = pValue;
 		break;
 
 	case ODM_CMNINFO_BW:
-		pDM_Odm->pBandWidth = (u8 *)pValue;
+		pDM_Odm->pBandWidth = pValue;
 		break;
 
 	case ODM_CMNINFO_CHNL:
-		pDM_Odm->pChannel = (u8 *)pValue;
+		pDM_Odm->pChannel = pValue;
 		break;
 
 	case ODM_CMNINFO_DMSP_GET_VALUE:
-		pDM_Odm->pbGetValueFromOtherMac = (bool *)pValue;
+		pDM_Odm->pbGetValueFromOtherMac = pValue;
 		break;
 
 	case ODM_CMNINFO_BUDDY_ADAPTOR:
-		pDM_Odm->pBuddyAdapter = (struct adapter **)pValue;
+		pDM_Odm->pBuddyAdapter = pValue;
 		break;
 
 	case ODM_CMNINFO_DMSP_IS_MASTER:
-		pDM_Odm->pbMasterOfDMSP = (bool *)pValue;
+		pDM_Odm->pbMasterOfDMSP = pValue;
 		break;
 
 	case ODM_CMNINFO_SCAN:
-		pDM_Odm->pbScanInProcess = (bool *)pValue;
+		pDM_Odm->pbScanInProcess = pValue;
 		break;
 
 	case ODM_CMNINFO_POWER_SAVING:
-		pDM_Odm->pbPowerSaving = (bool *)pValue;
+		pDM_Odm->pbPowerSaving = pValue;
 		break;
 
 	case ODM_CMNINFO_ONE_PATH_CCA:
-		pDM_Odm->pOnePathCCA = (u8 *)pValue;
+		pDM_Odm->pOnePathCCA = pValue;
 		break;
 
 	case ODM_CMNINFO_DRV_STOP:
-		pDM_Odm->pbDriverStopped =  (bool *)pValue;
+		pDM_Odm->pbDriverStopped =  pValue;
 		break;
 
 	case ODM_CMNINFO_PNP_IN:
-		pDM_Odm->pbDriverIsGoingToPnpSetPowerSleep =  (bool *)pValue;
+		pDM_Odm->pbDriverIsGoingToPnpSetPowerSleep =  pValue;
 		break;
 
 	case ODM_CMNINFO_INIT_ON:
-		pDM_Odm->pinit_adpt_in_progress =  (bool *)pValue;
+		pDM_Odm->pinit_adpt_in_progress =  pValue;
 		break;
 
 	case ODM_CMNINFO_ANT_TEST:
-		pDM_Odm->pAntennaTest =  (u8 *)pValue;
+		pDM_Odm->pAntennaTest =  pValue;
 		break;
 
 	case ODM_CMNINFO_NET_CLOSED:
-		pDM_Odm->pbNet_closed = (bool *)pValue;
+		pDM_Odm->pbNet_closed = pValue;
 		break;
 
 	case ODM_CMNINFO_FORCED_RATE:
-		pDM_Odm->pForcedDataRate = (u16 *)pValue;
+		pDM_Odm->pForcedDataRate = pValue;
 		break;
 
 	case ODM_CMNINFO_FORCED_IGI_LB:
-		pDM_Odm->pu1ForcedIgiLb = (u8 *)pValue;
+		pDM_Odm->pu1ForcedIgiLb = pValue;
 		break;
 
 	case ODM_CMNINFO_MP_MODE:
-		pDM_Odm->mp_mode = (u8 *)pValue;
+		pDM_Odm->mp_mode = pValue;
 		break;
 
 	/* case ODM_CMNINFO_RTSTA_AID: */
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c b/drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c
index 84a89ef74169..1565f2d67ea4 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c
@@ -63,7 +63,7 @@ static int _BlockWrite(struct adapter *padapter, void *buffer, u32 buffSize)
 	u32 blockSize_p3 = 1; /*  Phase #3 : Use 1-byte, the remnant of FW image. */
 	u32 blockCount_p1 = 0, blockCount_p2 = 0, blockCount_p3 = 0;
 	u32 remainSize_p1 = 0, remainSize_p2 = 0;
-	u8 *bufferPtr = (u8 *)buffer;
+	u8 *bufferPtr = buffer;
 	u32 i = 0, offset = 0;
 
 /* 	printk("====>%s %d\n", __func__, __LINE__); */
@@ -163,7 +163,7 @@ static int _WriteFW(struct adapter *padapter, void *buffer, u32 size)
 	int ret = _SUCCESS;
 	u32 pageNums, remainSize;
 	u32 page, offset;
-	u8 *bufferPtr = (u8 *)buffer;
+	u8 *bufferPtr = buffer;
 
 	pageNums = size / MAX_DLFW_PAGE_SIZE;
 	/* RT_ASSERT((pageNums <= 4), ("Page numbers should not greater then 4\n")); */
@@ -643,7 +643,7 @@ static void Hal_GetEfuseDefinition(
 	case TYPE_EFUSE_MAX_SECTION:
 		{
 			u8 *pMax_section;
-			pMax_section = (u8 *)pOut;
+			pMax_section = pOut;
 
 			if (efuseType == EFUSE_WIFI)
 				*pMax_section = EFUSE_MAX_SECTION_8723B;
@@ -655,7 +655,7 @@ static void Hal_GetEfuseDefinition(
 	case TYPE_EFUSE_REAL_CONTENT_LEN:
 		{
 			u16 *pu2Tmp;
-			pu2Tmp = (u16 *)pOut;
+			pu2Tmp = pOut;
 
 			if (efuseType == EFUSE_WIFI)
 				*pu2Tmp = EFUSE_REAL_CONTENT_LEN_8723B;
@@ -667,7 +667,7 @@ static void Hal_GetEfuseDefinition(
 	case TYPE_AVAILABLE_EFUSE_BYTES_BANK:
 		{
 			u16 *pu2Tmp;
-			pu2Tmp = (u16 *)pOut;
+			pu2Tmp = pOut;
 
 			if (efuseType == EFUSE_WIFI)
 				*pu2Tmp = (EFUSE_REAL_CONTENT_LEN_8723B-EFUSE_OOB_PROTECT_BYTES);
@@ -679,7 +679,7 @@ static void Hal_GetEfuseDefinition(
 	case TYPE_AVAILABLE_EFUSE_BYTES_TOTAL:
 		{
 			u16 *pu2Tmp;
-			pu2Tmp = (u16 *)pOut;
+			pu2Tmp = pOut;
 
 			if (efuseType == EFUSE_WIFI)
 				*pu2Tmp = (EFUSE_REAL_CONTENT_LEN_8723B-EFUSE_OOB_PROTECT_BYTES);
@@ -691,7 +691,7 @@ static void Hal_GetEfuseDefinition(
 	case TYPE_EFUSE_MAP_LEN:
 		{
 			u16 *pu2Tmp;
-			pu2Tmp = (u16 *)pOut;
+			pu2Tmp = pOut;
 
 			if (efuseType == EFUSE_WIFI)
 				*pu2Tmp = EFUSE_MAX_MAP_LEN;
@@ -703,7 +703,7 @@ static void Hal_GetEfuseDefinition(
 	case TYPE_EFUSE_PROTECT_BYTES_BANK:
 		{
 			u8 *pu1Tmp;
-			pu1Tmp = (u8 *)pOut;
+			pu1Tmp = pOut;
 
 			if (efuseType == EFUSE_WIFI)
 				*pu1Tmp = EFUSE_OOB_PROTECT_BYTES;
@@ -715,7 +715,7 @@ static void Hal_GetEfuseDefinition(
 	case TYPE_EFUSE_CONTENT_LEN_BANK:
 		{
 			u16 *pu2Tmp;
-			pu2Tmp = (u16 *)pOut;
+			pu2Tmp = pOut;
 
 			if (efuseType == EFUSE_WIFI)
 				*pu2Tmp = EFUSE_REAL_CONTENT_LEN_8723B;
@@ -727,7 +727,7 @@ static void Hal_GetEfuseDefinition(
 	default:
 		{
 			u8 *pu1Tmp;
-			pu1Tmp = (u8 *)pOut;
+			pu1Tmp = pOut;
 			*pu1Tmp = 0;
 		}
 		break;
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723b_rxdesc.c b/drivers/staging/rtl8723bs/hal/rtl8723b_rxdesc.c
index 92e5a0e7aa59..14bfbe3be0ca 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723b_rxdesc.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723b_rxdesc.c
@@ -64,7 +64,7 @@ static void process_link_qual(struct adapter *padapter, union recv_frame *prfram
 
 void rtl8723b_process_phy_info(struct adapter *padapter, void *prframe)
 {
-	union recv_frame *precvframe = (union recv_frame *)prframe;
+	union recv_frame *precvframe = prframe;
 	/*  */
 	/*  Check RSSI */
 	/*  */
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c b/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c
index b002eb446b2c..d9a4567ca721 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c
@@ -190,7 +190,7 @@ static void rtl8723bs_recv_tasklet(void *priv)
 	u8 shift_sz = 0, rx_report_sz = 0;
 
 
-	padapter = (struct adapter *)priv;
+	padapter = priv;
 	pHalData = GET_HAL_DATA(padapter);
 	precvpriv = &padapter->recvpriv;
 
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723bs_xmit.c b/drivers/staging/rtl8723bs/hal/rtl8723bs_xmit.c
index 9bee2e40be32..d0b317077511 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723bs_xmit.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723bs_xmit.c
@@ -490,7 +490,7 @@ int rtl8723bs_xmit_thread(void *context)
 
 
 	ret = _SUCCESS;
-	padapter = (struct adapter *)context;
+	padapter = context;
 	pxmitpriv = &padapter->xmitpriv;
 
 	rtw_sprintf(thread_name, 20, "%s-"ADPT_FMT, thread_name, ADPT_ARG(padapter));
diff --git a/drivers/staging/rtl8723bs/hal/sdio_ops.c b/drivers/staging/rtl8723bs/hal/sdio_ops.c
index 6285b72faa9a..1d1b14dedd35 100644
--- a/drivers/staging/rtl8723bs/hal/sdio_ops.c
+++ b/drivers/staging/rtl8723bs/hal/sdio_ops.c
@@ -1121,7 +1121,7 @@ void sd_int_dpc(struct adapter *padapter)
 			}
 		} else {
 			/* Error handling for malloc fail */
-			if (rtw_cbuf_push(padapter->evtpriv.c2h_queue, (void *)NULL) != _SUCCESS)
+			if (rtw_cbuf_push(padapter->evtpriv.c2h_queue, NULL) != _SUCCESS)
 				DBG_871X("%s rtw_cbuf_push fail\n", __func__);
 			_set_workitem(&padapter->evtpriv.c2h_wk);
 		}
diff --git a/drivers/staging/rtl8723bs/os_dep/ioctl_linux.c b/drivers/staging/rtl8723bs/os_dep/ioctl_linux.c
index 79d8383d4b9b..d5e5f830f2a1 100644
--- a/drivers/staging/rtl8723bs/os_dep/ioctl_linux.c
+++ b/drivers/staging/rtl8723bs/os_dep/ioctl_linux.c
@@ -19,7 +19,7 @@
 #include <rtw_mp.h>
 #include <linux/jiffies.h>
 
-#define RTL_IOCTL_WPA_SUPPLICANT	SIOCIWFIRSTPRIV+30
+#define RTL_IOCTL_WPA_SUPPLICANT	(SIOCIWFIRSTPRIV+30)
 
 #define SCAN_ITEM_SIZE 768
 #define MAX_CUSTOM_LEN 64
@@ -44,8 +44,7 @@ extern u8 key_2char2num(u8 hch, u8 lch);
 static u32 rtw_rates[] = {1000000, 2000000, 5500000, 11000000,
 	6000000, 9000000, 12000000, 18000000, 24000000, 36000000, 48000000, 54000000};
 
-static const char * const iw_operation_mode[] =
-{
+static const char * const iw_operation_mode[] = {
 	"Auto", "Ad-Hoc", "Managed",  "Master", "Repeater", "Secondary", "Monitor"
 };
 
@@ -190,16 +189,12 @@ static char *translate_scan(struct adapter *padapter,
 	start = iwe_stream_add_point(info, start, stop, &iwe, pnetwork->network.Ssid.Ssid);
 
 	/* parsing HT_CAP_IE */
-	if (pnetwork->network.Reserved[0] == 2) /*  Probe Request */
-	{
+	if (pnetwork->network.Reserved[0] == 2) { /*  Probe Request */
 		p = rtw_get_ie(&pnetwork->network.IEs[0], _HT_CAPABILITY_IE_, &ht_ielen, pnetwork->network.IELength);
-	}
-	else
-	{
+	} else {
 		p = rtw_get_ie(&pnetwork->network.IEs[12], _HT_CAPABILITY_IE_, &ht_ielen, pnetwork->network.IELength-12);
 	}
-	if (p && ht_ielen>0)
-	{
+	if (p && ht_ielen>0) {
 		struct rtw_ieee80211_ht_cap *pht_capie;
 		ht_cap = true;
 		pht_capie = (struct rtw_ieee80211_ht_cap *)(p+2);
@@ -210,33 +205,25 @@ static char *translate_scan(struct adapter *padapter,
 
 	/* Add the protocol name */
 	iwe.cmd = SIOCGIWNAME;
-	if ((rtw_is_cckratesonly_included((u8 *)&pnetwork->network.SupportedRates)) == true)
-	{
+	if ((rtw_is_cckratesonly_included((u8 *)&pnetwork->network.SupportedRates)) == true) {
 		if (ht_cap == true)
 			snprintf(iwe.u.name, IFNAMSIZ, "IEEE 802.11bn");
 		else
 		snprintf(iwe.u.name, IFNAMSIZ, "IEEE 802.11b");
-	}
-	else if ((rtw_is_cckrates_included((u8 *)&pnetwork->network.SupportedRates)) == true)
-	{
+	} else if ((rtw_is_cckrates_included((u8 *)&pnetwork->network.SupportedRates)) == true) {
 		if (ht_cap == true)
 			snprintf(iwe.u.name, IFNAMSIZ, "IEEE 802.11bgn");
 		else
 			snprintf(iwe.u.name, IFNAMSIZ, "IEEE 802.11bg");
-	}
-	else
-	{
-		if (pnetwork->network.Configuration.DSConfig > 14)
-		{
+	} else {
+		if (pnetwork->network.Configuration.DSConfig > 14) {
 			if (vht_cap == true)
 				snprintf(iwe.u.name, IFNAMSIZ, "IEEE 802.11AC");
 			else if (ht_cap == true)
 				snprintf(iwe.u.name, IFNAMSIZ, "IEEE 802.11an");
 			else
 				snprintf(iwe.u.name, IFNAMSIZ, "IEEE 802.11a");
-		}
-		else
-		{
+		} else {
 			if (ht_cap == true)
 				snprintf(iwe.u.name, IFNAMSIZ, "IEEE 802.11gn");
 			else
@@ -247,12 +234,9 @@ static char *translate_scan(struct adapter *padapter,
 	start = iwe_stream_add_event(info, start, stop, &iwe, IW_EV_CHAR_LEN);
 
 	  /* Add mode */
-	if (pnetwork->network.Reserved[0] == 2) /*  Probe Request */
-	{
+	if (pnetwork->network.Reserved[0] == 2) { /*  Probe Request */
 		cap = 0;
-	}
-	else
-	{
+	} else {
 		__le16 le_tmp;
 
 	        iwe.cmd = SIOCGIWMODE;
@@ -295,8 +279,7 @@ static char *translate_scan(struct adapter *padapter,
 		return start;
 	p = custom;
 	p += snprintf(p, MAX_CUSTOM_LEN - (p - custom), " Rates (Mb/s): ");
-	while (pnetwork->network.SupportedRates[i]!= 0)
-	{
+	while (pnetwork->network.SupportedRates[i]!= 0) {
 		rate = pnetwork->network.SupportedRates[i]&0x7F;
 		if (rate > max_rate)
 			max_rate = rate;
@@ -307,20 +290,12 @@ static char *translate_scan(struct adapter *padapter,
 
 	if (vht_cap == true) {
 		max_rate = vht_data_rate;
-	}
-	else if (ht_cap == true)
-	{
-		if (mcs_rate&0x8000)/* MCS15 */
-		{
+	} else if (ht_cap == true) {
+		if (mcs_rate&0x8000) { /* MCS15 */
 			max_rate = (bw_40MHz) ? ((short_GI)?300:270):((short_GI)?144:130);
-
-		}
-		else if (mcs_rate&0x0080)/* MCS7 */
-		{
+		} else if (mcs_rate&0x0080) { /* MCS7 */
 			max_rate = (bw_40MHz) ? ((short_GI)?150:135):((short_GI)?72:65);
-		}
-		else/* default MCS7 */
-		{
+		} else { /* default MCS7 */
 			/* DBG_871X("wx_get_scan, mcs_rate_bitmap = 0x%x\n", mcs_rate); */
 			max_rate = (bw_40MHz) ? ((short_GI)?150:135):((short_GI)?72:65);
 		}
@@ -334,8 +309,7 @@ static char *translate_scan(struct adapter *padapter,
 	start = iwe_stream_add_event(info, start, stop, &iwe, IW_EV_PARAM_LEN);
 
 	/* parsing WPA/WPA2 IE */
-	if (pnetwork->network.Reserved[0] != 2) /*  Probe Request */
-	{
+	if (pnetwork->network.Reserved[0] != 2) { /*  Probe Request */
 		u8 *buf;
 		u8 wpa_ie[255], rsn_ie[255];
 		u16 wpa_len = 0, rsn_len = 0;
@@ -351,15 +325,13 @@ static char *translate_scan(struct adapter *padapter,
 		if (wpa_len > 0) {
 			p =buf;
 			p += sprintf(p, "wpa_ie =");
-			for (i = 0; i < wpa_len; i++) {
+			for (i = 0; i < wpa_len; i++)
 				p += sprintf(p, "%02x", wpa_ie[i]);
-			}
 
 			if (wpa_len > 100) {
 				printk("-----------------Len %d----------------\n", wpa_len);
-				for (i = 0; i < wpa_len; i++) {
+				for (i = 0; i < wpa_len; i++)
 					printk("%02x ", wpa_ie[i]);
-				}
 				printk("\n");
 				printk("-----------------Len %d----------------\n", wpa_len);
 			}
@@ -401,21 +373,16 @@ static char *translate_scan(struct adapter *padapter,
 		u8 *ie_ptr = pnetwork->network.IEs + ie_offset;
 		total_ielen = pnetwork->network.IELength - ie_offset;
 
-		if (pnetwork->network.Reserved[0] == 2) /*  Probe Request */
-		{
+		if (pnetwork->network.Reserved[0] == 2) { /*  Probe Request */
 			ie_ptr = pnetwork->network.IEs;
 			total_ielen = pnetwork->network.IELength;
-		}
-		else     /*  Beacon or Probe Respones */
-		{
+		} else {    /*  Beacon or Probe Respones */
 			ie_ptr = pnetwork->network.IEs + _FIXED_IE_LENGTH_;
 			total_ielen = pnetwork->network.IELength - _FIXED_IE_LENGTH_;
 		}
 
-		while (cnt < total_ielen)
-		{
-			if (rtw_is_wps_ie(&ie_ptr[cnt], &wps_ielen) && (wps_ielen>2))
-			{
+		while (cnt < total_ielen) {
+			if (rtw_is_wps_ie(&ie_ptr[cnt], &wps_ielen) && (wps_ielen>2)) {
 				wpsie_ptr = &ie_ptr[cnt];
 				iwe.cmd =IWEVGENIE;
 				iwe.u.data.length = (u16)wps_ielen;
@@ -507,38 +474,27 @@ static int wpa_set_auth_algs(struct net_device *dev, u32 value)
 	struct adapter *padapter = (struct adapter *) rtw_netdev_priv(dev);
 	int ret = 0;
 
-	if ((value & AUTH_ALG_SHARED_KEY) && (value & AUTH_ALG_OPEN_SYSTEM))
-	{
+	if ((value & AUTH_ALG_SHARED_KEY) && (value & AUTH_ALG_OPEN_SYSTEM)) {
 		DBG_871X("wpa_set_auth_algs, AUTH_ALG_SHARED_KEY and  AUTH_ALG_OPEN_SYSTEM [value:0x%x]\n", value);
 		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled;
 		padapter->securitypriv.ndisauthtype = Ndis802_11AuthModeAutoSwitch;
 		padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_Auto;
-	}
-	else if (value & AUTH_ALG_SHARED_KEY)
-	{
+	} else if (value & AUTH_ALG_SHARED_KEY)	{
 		DBG_871X("wpa_set_auth_algs, AUTH_ALG_SHARED_KEY  [value:0x%x]\n", value);
 		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled;
 
 		padapter->securitypriv.ndisauthtype = Ndis802_11AuthModeShared;
 		padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_Shared;
-	}
-	else if (value & AUTH_ALG_OPEN_SYSTEM)
-	{
+	} else if (value & AUTH_ALG_OPEN_SYSTEM) {
 		DBG_871X("wpa_set_auth_algs, AUTH_ALG_OPEN_SYSTEM\n");
 		/* padapter->securitypriv.ndisencryptstatus = Ndis802_11EncryptionDisabled; */
-		if (padapter->securitypriv.ndisauthtype < Ndis802_11AuthModeWPAPSK)
-		{
+		if (padapter->securitypriv.ndisauthtype < Ndis802_11AuthModeWPAPSK) {
 			padapter->securitypriv.ndisauthtype = Ndis802_11AuthModeOpen;
 			padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_Open;
 		}
-
-	}
-	else if (value & AUTH_ALG_LEAP)
-	{
+	} else if (value & AUTH_ALG_LEAP) {
 		DBG_871X("wpa_set_auth_algs, AUTH_ALG_LEAP\n");
-	}
-	else
-	{
+	} else {
 		DBG_871X("wpa_set_auth_algs, error!\n");
 		ret = -EINVAL;
 	}
@@ -559,33 +515,27 @@ static int wpa_set_encryption(struct net_device *dev, struct ieee_param *param,
 	param->u.crypt.err = 0;
 	param->u.crypt.alg[IEEE_CRYPT_ALG_NAME_LEN - 1] = '\0';
 
-	if (param_len < (u32) ((u8 *) param->u.crypt.key - (u8 *) param) + param->u.crypt.key_len)
-	{
+	if (param_len < (u32) ((u8 *) param->u.crypt.key - (u8 *) param) + param->u.crypt.key_len) {
 		ret =  -EINVAL;
 		goto exit;
 	}
 
 	if (param->sta_addr[0] == 0xff && param->sta_addr[1] == 0xff &&
 	    param->sta_addr[2] == 0xff && param->sta_addr[3] == 0xff &&
-	    param->sta_addr[4] == 0xff && param->sta_addr[5] == 0xff)
-	{
-
+	    param->sta_addr[4] == 0xff && param->sta_addr[5] == 0xff) {
 		if (param->u.crypt.idx >= WEP_KEYS ||
 		    param->u.crypt.idx >= BIP_MAX_KEYID) {
 			ret = -EINVAL;
 			goto exit;
 		}
-	}
-	else
-	{
+	} else {
 		{
 			ret = -EINVAL;
 			goto exit;
 		}
 	}
 
-	if (strcmp(param->u.crypt.alg, "WEP") == 0)
-	{
+	if (strcmp(param->u.crypt.alg, "WEP") == 0) {
 		RT_TRACE(_module_rtl871x_ioctl_os_c, _drv_err_, ("wpa_set_encryption, crypt.alg = WEP\n"));
 		DBG_871X("wpa_set_encryption, crypt.alg = WEP\n");
 
@@ -604,8 +554,7 @@ static int wpa_set_encryption(struct net_device *dev, struct ieee_param *param,
 
 		RT_TRACE(_module_rtl871x_ioctl_os_c, _drv_info_, ("(2)wep_key_idx =%d\n", wep_key_idx));
 
-		if (wep_key_len > 0)
-		{
+		if (wep_key_len > 0) {
 			wep_key_len = wep_key_len <= 5 ? 5 : 13;
 			wep_total_len = wep_key_len + FIELD_OFFSET(struct ndis_802_11_wep, KeyMaterial);
 			pwep =(struct ndis_802_11_wep	 *) rtw_malloc(wep_total_len);
@@ -619,13 +568,11 @@ static int wpa_set_encryption(struct net_device *dev, struct ieee_param *param,
 			pwep->KeyLength = wep_key_len;
 			pwep->Length = wep_total_len;
 
-			if (wep_key_len == 13)
-			{
+			if (wep_key_len == 13) {
 				padapter->securitypriv.dot11PrivacyAlgrthm = _WEP104_;
 				padapter->securitypriv.dot118021XGrpPrivacy = _WEP104_;
 			}
-		}
-		else {
+		} else {
 			ret = -EINVAL;
 			goto exit;
 		}
@@ -635,17 +582,12 @@ static int wpa_set_encryption(struct net_device *dev, struct ieee_param *param,
 
 		memcpy(pwep->KeyMaterial,  param->u.crypt.key, pwep->KeyLength);
 
-		if (param->u.crypt.set_tx)
-		{
+		if (param->u.crypt.set_tx) {
 			DBG_871X("wep, set_tx = 1\n");
 
 			if (rtw_set_802_11_add_wep(padapter, pwep) == (u8)_FAIL)
-			{
 				ret = -EOPNOTSUPP ;
-			}
-		}
-		else
-		{
+		} else {
 			DBG_871X("wep, set_tx = 0\n");
 
 			/* don't update "psecuritypriv->dot11PrivacyAlgrthm" and */
@@ -664,35 +606,28 @@ static int wpa_set_encryption(struct net_device *dev, struct ieee_param *param,
 		goto exit;
 	}
 
-	if (padapter->securitypriv.dot11AuthAlgrthm == dot11AuthAlgrthm_8021X) /*  802_1x */
-	{
+	if (padapter->securitypriv.dot11AuthAlgrthm == dot11AuthAlgrthm_8021X) { /*  802_1x */
 		struct sta_info * psta,*pbcmc_sta;
 		struct sta_priv * pstapriv = &padapter->stapriv;
 
-		if (check_fwstate(pmlmepriv, WIFI_STATION_STATE | WIFI_MP_STATE) == true) /* sta mode */
-		{
+		if (check_fwstate(pmlmepriv, WIFI_STATION_STATE | WIFI_MP_STATE) == true) { /* sta mode */
 			psta = rtw_get_stainfo(pstapriv, get_bssid(pmlmepriv));
 			if (psta == NULL) {
 				/* DEBUG_ERR(("Set wpa_set_encryption: Obtain Sta_info fail\n")); */
-			}
-			else
-			{
+			} else {
 				/* Jeff: don't disable ieee8021x_blocked while clearing key */
 				if (strcmp(param->u.crypt.alg, "none") != 0)
 					psta->ieee8021x_blocked = false;
 
 				if ((padapter->securitypriv.ndisencryptstatus == Ndis802_11Encryption2Enabled)||
-						(padapter->securitypriv.ndisencryptstatus ==  Ndis802_11Encryption3Enabled))
-				{
+						(padapter->securitypriv.ndisencryptstatus ==  Ndis802_11Encryption3Enabled)) {
 					psta->dot118021XPrivacy = padapter->securitypriv.dot11PrivacyAlgrthm;
 				}
 
-				if (param->u.crypt.set_tx == 1)/* pairwise key */
-				{
+				if (param->u.crypt.set_tx == 1) { /* pairwise key */
 					memcpy(psta->dot118021x_UncstKey.skey,  param->u.crypt.key, (param->u.crypt.key_len>16 ?16:param->u.crypt.key_len));
 
-					if (strcmp(param->u.crypt.alg, "TKIP") == 0)/* set mic key */
-					{
+					if (strcmp(param->u.crypt.alg, "TKIP") == 0) { /* set mic key */
 						/* DEBUG_ERR(("\nset key length :param->u.crypt.key_len =%d\n", param->u.crypt.key_len)); */
 						memcpy(psta->dot11tkiptxmickey.skey, &(param->u.crypt.key[16]), 8);
 						memcpy(psta->dot11tkiprxmickey.skey, &(param->u.crypt.key[24]), 8);
@@ -705,15 +640,11 @@ static int wpa_set_encryption(struct net_device *dev, struct ieee_param *param,
 					DBG_871X(" ~~~~set sta key:unicastkey\n");
 
 					rtw_setstakey_cmd(padapter, psta, true, true);
-				}
-				else/* group key */
-				{
-					if (strcmp(param->u.crypt.alg, "TKIP") == 0 || strcmp(param->u.crypt.alg, "CCMP") == 0)
-					{
+				} else { /* group key */
+					if (strcmp(param->u.crypt.alg, "TKIP") == 0 || strcmp(param->u.crypt.alg, "CCMP") == 0) {
 						memcpy(padapter->securitypriv.dot118021XGrpKey[param->u.crypt.idx].skey,  param->u.crypt.key, (param->u.crypt.key_len>16 ?16:param->u.crypt.key_len));
 						/* only TKIP group key need to install this */
-						if (param->u.crypt.key_len > 16)
-						{
+						if (param->u.crypt.key_len > 16) {
 							memcpy(padapter->securitypriv.dot118021XGrptxmickey[param->u.crypt.idx].skey,&(param->u.crypt.key[16]), 8);
 							memcpy(padapter->securitypriv.dot118021XGrprxmickey[param->u.crypt.idx].skey,&(param->u.crypt.key[24]), 8);
 						}
@@ -724,9 +655,7 @@ static int wpa_set_encryption(struct net_device *dev, struct ieee_param *param,
 						padapter->securitypriv.dot118021XGrpKeyid = param->u.crypt.idx;
 
 						rtw_set_key(padapter,&padapter->securitypriv, param->u.crypt.idx, 1, true);
-					}
-					else if (strcmp(param->u.crypt.alg, "BIP") == 0)
-					{
+					} else if (strcmp(param->u.crypt.alg, "BIP") == 0) {
 						/* printk("BIP key_len =%d , index =%d @@@@@@@@@@@@@@@@@@\n", param->u.crypt.key_len, param->u.crypt.idx); */
 						/* save the IGTK key, length 16 bytes */
 						memcpy(padapter->securitypriv.dot11wBIPKey[param->u.crypt.idx].skey,  param->u.crypt.key, (param->u.crypt.key_len>16 ?16:param->u.crypt.key_len));
@@ -742,25 +671,20 @@ static int wpa_set_encryption(struct net_device *dev, struct ieee_param *param,
 			}
 
 			pbcmc_sta =rtw_get_bcmc_stainfo(padapter);
-			if (pbcmc_sta == NULL)
-			{
+			if (pbcmc_sta == NULL) {
 				/* DEBUG_ERR(("Set OID_802_11_ADD_KEY: bcmc stainfo is null\n")); */
-			}
-			else
-			{
+			} else {
 				/* Jeff: don't disable ieee8021x_blocked while clearing key */
 				if (strcmp(param->u.crypt.alg, "none") != 0)
 					pbcmc_sta->ieee8021x_blocked = false;
 
 				if ((padapter->securitypriv.ndisencryptstatus == Ndis802_11Encryption2Enabled)||
-						(padapter->securitypriv.ndisencryptstatus ==  Ndis802_11Encryption3Enabled))
-				{
+						(padapter->securitypriv.ndisencryptstatus ==  Ndis802_11Encryption3Enabled)) {
 					pbcmc_sta->dot118021XPrivacy = padapter->securitypriv.dot11PrivacyAlgrthm;
 				}
 			}
-		}
-		else if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE)) /* adhoc mode */
-		{
+		} else if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE)) {
+			/* adhoc mode */
 		}
 	}
 
@@ -785,8 +709,7 @@ static int rtw_set_wpa_ie(struct adapter *padapter, char *pie, unsigned short ie
 			return -EINVAL;
 	}
 
-	if (ielen)
-	{
+	if (ielen) {
 		buf = rtw_zmalloc(ielen);
 		if (buf == NULL) {
 			ret =  -ENOMEM;
@@ -810,31 +733,24 @@ static int rtw_set_wpa_ie(struct adapter *padapter, char *pie, unsigned short ie
 			goto exit;
 		}
 
-		if (rtw_parse_wpa_ie(buf, ielen, &group_cipher, &pairwise_cipher, NULL) == _SUCCESS)
-		{
+		if (rtw_parse_wpa_ie(buf, ielen, &group_cipher, &pairwise_cipher, NULL) == _SUCCESS) {
 			padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
 			padapter->securitypriv.ndisauthtype =Ndis802_11AuthModeWPAPSK;
 			memcpy(padapter->securitypriv.supplicant_ie, &buf[0], ielen);
 		}
 
-		if (rtw_parse_wpa2_ie(buf, ielen, &group_cipher, &pairwise_cipher, NULL) == _SUCCESS)
-		{
+		if (rtw_parse_wpa2_ie(buf, ielen, &group_cipher, &pairwise_cipher, NULL) == _SUCCESS) {
 			padapter->securitypriv.dot11AuthAlgrthm = dot11AuthAlgrthm_8021X;
 			padapter->securitypriv.ndisauthtype =Ndis802_11AuthModeWPA2PSK;
 			memcpy(padapter->securitypriv.supplicant_ie, &buf[0], ielen);
 		}
 
 		if (group_cipher == 0)
-		{
 			group_cipher = WPA_CIPHER_NONE;
-		}
 		if (pairwise_cipher == 0)
-		{
 			pairwise_cipher = WPA_CIPHER_NONE;
-		}
 
-		switch (group_cipher)
-		{
+		switch (group_cipher) {
 			case WPA_CIPHER_NONE:
 				padapter->securitypriv.dot118021XGrpPrivacy = _NO_PRIVACY_;
 				padapter->securitypriv.ndisencryptstatus =Ndis802_11EncryptionDisabled;
@@ -857,8 +773,7 @@ static int rtw_set_wpa_ie(struct adapter *padapter, char *pie, unsigned short ie
 				break;
 		}
 
-		switch (pairwise_cipher)
-		{
+		switch (pairwise_cipher) {
 			case WPA_CIPHER_NONE:
 				padapter->securitypriv.dot11PrivacyAlgrthm = _NO_PRIVACY_;
 				padapter->securitypriv.ndisencryptstatus =Ndis802_11EncryptionDisabled;
@@ -886,12 +801,10 @@ static int rtw_set_wpa_ie(struct adapter *padapter, char *pie, unsigned short ie
 			u16 cnt = 0;
 			u8 eid, wps_oui[4]={0x0, 0x50, 0xf2, 0x04};
 
-			while (cnt < ielen)
-			{
+			while (cnt < ielen) {
 				eid = buf[cnt];
 
-				if ((eid == _VENDOR_SPECIFIC_IE_) && (!memcmp(&buf[cnt+2], wps_oui, 4)))
-				{
+				if ((eid == _VENDOR_SPECIFIC_IE_) && (!memcmp(&buf[cnt+2], wps_oui, 4))) {
 					DBG_871X("SET WPS_IE\n");
 
 					padapter->securitypriv.wps_ie_len = ((buf[cnt+1]+2) < MAX_WPS_IE_LEN) ? (buf[cnt+1]+2):MAX_WPS_IE_LEN;
@@ -947,48 +860,36 @@ static int rtw_wx_get_name(struct net_device *dev,
 		/* parsing HT_CAP_IE */
 		p = rtw_get_ie(&pcur_bss->IEs[12], _HT_CAPABILITY_IE_, &ht_ielen, pcur_bss->IELength-12);
 		if (p && ht_ielen>0)
-		{
 			ht_cap = true;
-		}
 
 		prates = &pcur_bss->SupportedRates;
 
-		if (rtw_is_cckratesonly_included((u8 *)prates) == true)
-		{
+		if (rtw_is_cckratesonly_included((u8 *)prates) == true) {
 			if (ht_cap == true)
 				snprintf(wrqu->name, IFNAMSIZ, "IEEE 802.11bn");
 			else
 				snprintf(wrqu->name, IFNAMSIZ, "IEEE 802.11b");
-		}
-		else if ((rtw_is_cckrates_included((u8 *)prates)) == true)
-		{
+		} else if ((rtw_is_cckrates_included((u8 *)prates)) == true) {
 			if (ht_cap == true)
 				snprintf(wrqu->name, IFNAMSIZ, "IEEE 802.11bgn");
 			else
 				snprintf(wrqu->name, IFNAMSIZ, "IEEE 802.11bg");
-		}
-		else
-		{
-			if (pcur_bss->Configuration.DSConfig > 14)
-			{
+		} else {
+			if (pcur_bss->Configuration.DSConfig > 14) {
 				if (vht_cap == true)
 					snprintf(wrqu->name, IFNAMSIZ, "IEEE 802.11AC");
 				else if (ht_cap == true)
 					snprintf(wrqu->name, IFNAMSIZ, "IEEE 802.11an");
 				else
 					snprintf(wrqu->name, IFNAMSIZ, "IEEE 802.11a");
-			}
-			else
-			{
+			} else {
 				if (ht_cap == true)
 					snprintf(wrqu->name, IFNAMSIZ, "IEEE 802.11gn");
 				else
 					snprintf(wrqu->name, IFNAMSIZ, "IEEE 802.11g");
 			}
 		}
-	}
-	else
-	{
+	} else {
 		/* prates = &padapter->registrypriv.dev_network.SupportedRates; */
 		/* snprintf(wrqu->name, IFNAMSIZ, "IEEE 802.11g"); */
 		snprintf(wrqu->name, IFNAMSIZ, "unassociated");
@@ -1013,15 +914,13 @@ static int rtw_wx_get_freq(struct net_device *dev,
 	struct	mlme_priv *pmlmepriv = &(padapter->mlmepriv);
 	struct wlan_bssid_ex  *pcur_bss = &pmlmepriv->cur_network.network;
 
-	if (check_fwstate(pmlmepriv, _FW_LINKED) == true)
-	{
+	if (check_fwstate(pmlmepriv, _FW_LINKED) == true) {
 		/* wrqu->freq.m = ieee80211_wlan_frequencies[pcur_bss->Configuration.DSConfig-1] * 100000; */
 		wrqu->freq.m = rtw_ch2freq(pcur_bss->Configuration.DSConfig) * 100000;
 		wrqu->freq.e = 1;
 		wrqu->freq.i = pcur_bss->Configuration.DSConfig;
 
-	}
-	else {
+	} else {
 		wrqu->freq.m = rtw_ch2freq(padapter->mlmeextpriv.cur_channel) * 100000;
 		wrqu->freq.e = 1;
 		wrqu->freq.i = padapter->mlmeextpriv.cur_channel;
@@ -1047,8 +946,7 @@ static int rtw_wx_set_mode(struct net_device *dev, struct iw_request_info *a,
 		goto exit;
 	}
 
-	switch (wrqu->mode)
-	{
+	switch (wrqu->mode) {
 		case IW_MODE_AUTO:
 			networkType = Ndis802_11AutoUnknown;
 			DBG_871X("set_mode = IW_MODE_AUTO\n");
@@ -1105,22 +1003,14 @@ static int rtw_wx_get_mode(struct net_device *dev, struct iw_request_info *a,
 
 	RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_, (" rtw_wx_get_mode\n"));
 
-	if (check_fwstate(pmlmepriv, WIFI_STATION_STATE) == true)
-	{
+	if (check_fwstate(pmlmepriv, WIFI_STATION_STATE) == true) {
 		wrqu->mode = IW_MODE_INFRA;
-	}
-	else if  ((check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == true) ||
-		       (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == true))
-
-	{
+	} else if  ((check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == true) ||
+		       (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == true)) {
 		wrqu->mode = IW_MODE_ADHOC;
-	}
-	else if (check_fwstate(pmlmepriv, WIFI_AP_STATE) == true)
-	{
+	} else if (check_fwstate(pmlmepriv, WIFI_AP_STATE) == true) {
 		wrqu->mode = IW_MODE_MASTER;
-	}
-	else
-	{
+	} else {
 		wrqu->mode = IW_MODE_AUTO;
 	}
 	return 0;
@@ -1147,25 +1037,19 @@ static int rtw_wx_set_pmkid(struct net_device *dev,
         */
 
 	memcpy(strIssueBssid, pPMK->bssid.sa_data, ETH_ALEN);
-        if (pPMK->cmd == IW_PMKSA_ADD)
-        {
+        if (pPMK->cmd == IW_PMKSA_ADD) {
                 DBG_871X("[rtw_wx_set_pmkid] IW_PMKSA_ADD!\n");
                 if (!memcmp(strIssueBssid, strZeroMacAddress, ETH_ALEN))
-                {
                     return(intReturn);
-                }
                 else
-                {
                     intReturn = true;
-                }
+
 		blInserted = false;
 
 		/* overwrite PMKID */
-		for (j = 0 ; j<NUM_PMKID_CACHE; j++)
-		{
-			if (!memcmp(psecuritypriv->PMKIDList[j].Bssid, strIssueBssid, ETH_ALEN))
-			{ /*  BSSID is matched, the same AP => rewrite with new PMKID. */
-
+		for (j = 0 ; j<NUM_PMKID_CACHE; j++) {
+			if (!memcmp(psecuritypriv->PMKIDList[j].Bssid, strIssueBssid, ETH_ALEN)) {
+				/*  BSSID is matched, the same AP => rewrite with new PMKID. */
                                 DBG_871X("[rtw_wx_set_pmkid] BSSID exists in the PMKList.\n");
 
 				memcpy(psecuritypriv->PMKIDList[j].PMKID, pPMK->pmkid, IW_PMKID_LEN);
@@ -1176,8 +1060,7 @@ static int rtw_wx_set_pmkid(struct net_device *dev,
 			}
 	        }
 
-	        if (!blInserted)
-                {
+	        if (!blInserted) {
 		    /*  Find a new entry */
                     DBG_871X("[rtw_wx_set_pmkid] Use the new entry index = %d for this PMKID.\n",
                             psecuritypriv->PMKIDIndex);
@@ -1188,27 +1071,20 @@ static int rtw_wx_set_pmkid(struct net_device *dev,
                     psecuritypriv->PMKIDList[ psecuritypriv->PMKIDIndex ].bUsed = true;
 		    psecuritypriv->PMKIDIndex++ ;
 		    if (psecuritypriv->PMKIDIndex == 16)
-                    {
 		        psecuritypriv->PMKIDIndex = 0;
-                    }
 		}
-        }
-        else if (pPMK->cmd == IW_PMKSA_REMOVE)
-        {
+        } else if (pPMK->cmd == IW_PMKSA_REMOVE) {
                 DBG_871X("[rtw_wx_set_pmkid] IW_PMKSA_REMOVE!\n");
                 intReturn = true;
-		for (j = 0 ; j<NUM_PMKID_CACHE; j++)
-		{
-			if (!memcmp(psecuritypriv->PMKIDList[j].Bssid, strIssueBssid, ETH_ALEN))
-			{ /*  BSSID is matched, the same AP => Remove this PMKID information and reset it. */
+		for (j = 0 ; j<NUM_PMKID_CACHE; j++) {
+			if (!memcmp(psecuritypriv->PMKIDList[j].Bssid, strIssueBssid, ETH_ALEN)) {
+				/*  BSSID is matched, the same AP => Remove this PMKID information and reset it. */
                                 memset(psecuritypriv->PMKIDList[ j ].Bssid, 0x00, ETH_ALEN);
                                 psecuritypriv->PMKIDList[ j ].bUsed = false;
 				break;
 			}
 	        }
-        }
-        else if (pPMK->cmd == IW_PMKSA_FLUSH)
-        {
+        } else if (pPMK->cmd == IW_PMKSA_FLUSH) {
             DBG_871X("[rtw_wx_set_pmkid] IW_PMKSA_FLUSH!\n");
             memset(&psecuritypriv->PMKIDList[ 0 ], 0x00, sizeof(RT_PMKID_LIST) * NUM_PMKID_CACHE);
             psecuritypriv->PMKIDIndex = 0;
@@ -1273,9 +1149,8 @@ static int rtw_wx_get_range(struct net_device *dev,
 
 	range->num_bitrates = RATE_COUNT;
 
-	for (i = 0; i < RATE_COUNT && i < IW_MAX_BITRATES; i++) {
+	for (i = 0; i < RATE_COUNT && i < IW_MAX_BITRATES; i++)
 		range->bitrate[i] = rtw_rates[i];
-	}
 
 	range->min_frag = MIN_FRAG_THRESHOLD;
 	range->max_frag = MAX_FRAG_THRESHOLD;
@@ -1288,8 +1163,7 @@ static int rtw_wx_get_range(struct net_device *dev,
 	for (i = 0, val = 0; i < MAX_CHANNEL_NUM; i++) {
 
 		/*  Include only legal frequencies for some countries */
-		if (pmlmeext->channel_set[i].ChannelNum != 0)
-		{
+		if (pmlmeext->channel_set[i].ChannelNum != 0) {
 			range->freq[val].i = pmlmeext->channel_set[i].ChannelNum;
 			range->freq[val].m = rtw_ch2freq(pmlmeext->channel_set[i].ChannelNum) * 100000;
 			range->freq[val].e = 1;
@@ -1349,8 +1223,7 @@ static int rtw_wx_set_wap(struct net_device *dev,
 	enum NDIS_802_11_AUTHENTICATION_MODE	authmode;
 
 	rtw_ps_deny(padapter, PS_DENY_JOIN);
-	if (_FAIL == rtw_pwr_wakeup(padapter))
-	{
+	if (_FAIL == rtw_pwr_wakeup(padapter)) {
 		ret = -1;
 		goto exit;
 	}
@@ -1383,15 +1256,12 @@ static int rtw_wx_set_wap(struct net_device *dev,
 
 		src_bssid = temp->sa_data;
 
-		if ((!memcmp(dst_bssid, src_bssid, ETH_ALEN)))
-		{
-			if (!rtw_set_802_11_infrastructure_mode(padapter, pnetwork->network.InfrastructureMode))
-			{
+		if ((!memcmp(dst_bssid, src_bssid, ETH_ALEN))) {
+			if (!rtw_set_802_11_infrastructure_mode(padapter, pnetwork->network.InfrastructureMode)) {
 				ret = -1;
 				spin_unlock_bh(&queue->lock);
 				goto exit;
 			}
-
 				break;
 		}
 
@@ -1429,13 +1299,9 @@ static int rtw_wx_get_wap(struct net_device *dev,
 
 	if  (((check_fwstate(pmlmepriv, _FW_LINKED)) == true) ||
 			((check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE)) == true) ||
-			((check_fwstate(pmlmepriv, WIFI_AP_STATE)) == true))
-	{
-
+			((check_fwstate(pmlmepriv, WIFI_AP_STATE)) == true)) {
 		memcpy(wrqu->ap_addr.sa_data, pcur_bss->MacAddress, ETH_ALEN);
-	}
-	else
-	{
+	} else {
 		memset(wrqu->ap_addr.sa_data, 0, ETH_ALEN);
 	}
 
@@ -1461,8 +1327,7 @@ static int rtw_wx_set_mlme(struct net_device *dev,
 
 	DBG_871X("%s, cmd =%d, reason =%d\n", __func__, mlme->cmd, reason);
 
-	switch (mlme->cmd)
-	{
+	switch (mlme->cmd) {
 	case IW_MLME_DEAUTH:
 		if (!rtw_set_802_11_disassociate(padapter))
 			ret = -1;
@@ -1493,8 +1358,7 @@ static int rtw_wx_set_scan(struct net_device *dev, struct iw_request_info *a,
 	#endif
 
 	rtw_ps_deny(padapter, PS_DENY_SCAN);
-	if (_FAIL == rtw_pwr_wakeup(padapter))
-	{
+	if (_FAIL == rtw_pwr_wakeup(padapter)) {
 		ret = -1;
 		goto exit;
 	}
@@ -1518,26 +1382,22 @@ static int rtw_wx_set_scan(struct net_device *dev, struct iw_request_info *a,
 	/*  When Busy Traffic, driver do not site survey. So driver return success. */
 	/*  wpa_supplicant will not issue SIOCSIWSCAN cmd again after scan timeout. */
 	/*  modify by thomas 2011-02-22. */
-	if (pmlmepriv->LinkDetectInfo.bBusyTraffic == true)
-	{
+	if (pmlmepriv->LinkDetectInfo.bBusyTraffic == true) {
 		indicate_wx_scan_complete_event(padapter);
 		goto exit;
 	}
 
-	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY|_FW_UNDER_LINKING) == true)
-	{
+	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY|_FW_UNDER_LINKING) == true) {
 		indicate_wx_scan_complete_event(padapter);
 		goto exit;
 	}
 
 	memset(ssid, 0, sizeof(struct ndis_802_11_ssid)*RTW_SSID_SCAN_AMOUNT);
 
-	if (wrqu->data.length == sizeof(struct iw_scan_req))
-	{
+	if (wrqu->data.length == sizeof(struct iw_scan_req)) {
 		struct iw_scan_req *req = (struct iw_scan_req *)extra;
 
-		if (wrqu->data.flags & IW_SCAN_THIS_ESSID)
-		{
+		if (wrqu->data.flags & IW_SCAN_THIS_ESSID) {
 			int len = min((int)req->essid_len, IW_ESSID_MAX_SIZE);
 
 			memcpy(ssid[0].Ssid, req->essid, len);
@@ -1551,17 +1411,12 @@ static int rtw_wx_set_scan(struct net_device *dev, struct iw_request_info *a,
 
 			spin_unlock_bh(&pmlmepriv->lock);
 
-		}
-		else if (req->scan_type == IW_SCAN_TYPE_PASSIVE)
-		{
+		} else if (req->scan_type == IW_SCAN_TYPE_PASSIVE) {
 			DBG_871X("rtw_wx_set_scan, req->scan_type == IW_SCAN_TYPE_PASSIVE\n");
 		}
 
-	}
-	else if (wrqu->data.length >= WEXT_CSCAN_HEADER_SIZE
-		&& !memcmp(extra, WEXT_CSCAN_HEADER, WEXT_CSCAN_HEADER_SIZE)
-	)
-	{
+	} else if (wrqu->data.length >= WEXT_CSCAN_HEADER_SIZE
+		&& !memcmp(extra, WEXT_CSCAN_HEADER, WEXT_CSCAN_HEADER_SIZE)) {
 		int len = wrqu->data.length -WEXT_CSCAN_HEADER_SIZE;
 		char *pos = extra+WEXT_CSCAN_HEADER_SIZE;
 		char section;
@@ -1626,9 +1481,7 @@ static int rtw_wx_set_scan(struct net_device *dev, struct iw_request_info *a,
 		/* jeff: it has still some scan paramater to parse, we only do this now... */
 		_status = rtw_set_802_11_bssid_list_scan(padapter, ssid, RTW_SSID_SCAN_AMOUNT);
 
-	} else
-
-	{
+	} else {
 		_status = rtw_set_802_11_bssid_list_scan(padapter, NULL, 0);
 	}
 
@@ -1666,8 +1519,7 @@ static int rtw_wx_get_scan(struct net_device *dev, struct iw_request_info *a,
 	DBG_871X("DBG_IOCTL %s:%d\n", __func__, __LINE__);
 	#endif
 
-	if (adapter_to_pwrctl(padapter)->brfoffbyhw && padapter->bDriverStopped)
-	{
+	if (adapter_to_pwrctl(padapter)->brfoffbyhw && padapter->bDriverStopped) {
 		ret = -EINVAL;
 		goto exit;
 	}
@@ -1682,8 +1534,7 @@ static int rtw_wx_get_scan(struct net_device *dev, struct iw_request_info *a,
 	phead = get_list_head(queue);
 	plist = get_next(phead);
 
-	while (1)
-	{
+	while (1) {
 		if (phead == plist)
 			break;
 
@@ -1697,9 +1548,8 @@ static int rtw_wx_get_scan(struct net_device *dev, struct iw_request_info *a,
 		/* report network only if the current channel set contains the channel to which this network belongs */
 		if (rtw_ch_set_search_ch(padapter->mlmeextpriv.channel_set, pnetwork->network.Configuration.DSConfig) >= 0
 			&& rtw_mlme_band_check(padapter, pnetwork->network.Configuration.DSConfig) == true
-			&& true == rtw_validate_ssid(&(pnetwork->network.Ssid))
-		)
-		{
+			&& true == rtw_validate_ssid(&(pnetwork->network.Ssid))) {
+
 			ev =translate_scan(padapter, a, pnetwork, ev, stop);
 		}
 
@@ -1750,8 +1600,7 @@ static int rtw_wx_set_essid(struct net_device *dev,
 		 ("+rtw_wx_set_essid: fw_state = 0x%08x\n", get_fwstate(pmlmepriv)));
 
 	rtw_ps_deny(padapter, PS_DENY_JOIN);
-	if (_FAIL == rtw_pwr_wakeup(padapter))
-	{
+	if (_FAIL == rtw_pwr_wakeup(padapter)) {
 		ret = -1;
 		goto exit;
 	}
@@ -1773,8 +1622,7 @@ static int rtw_wx_set_essid(struct net_device *dev,
 
 	authmode = padapter->securitypriv.ndisauthtype;
 	DBG_871X("=>%s\n", __func__);
-	if (wrqu->essid.flags && wrqu->essid.length)
-	{
+	if (wrqu->essid.flags && wrqu->essid.length) {
 		len = (wrqu->essid.length < IW_ESSID_MAX_SIZE) ? wrqu->essid.length : IW_ESSID_MAX_SIZE;
 
 		if (wrqu->essid.length != 33)
@@ -1791,8 +1639,7 @@ static int rtw_wx_set_essid(struct net_device *dev,
 		pmlmepriv->pscanned = get_next(phead);
 
 		while (1) {
-			if (phead == pmlmepriv->pscanned)
-			{
+			if (phead == pmlmepriv->pscanned) {
 			        RT_TRACE(_module_rtl871x_ioctl_os_c, _drv_warning_,
 					 ("rtw_wx_set_essid: scan_q is empty, set ssid to check if scanning again!\n"));
 
@@ -1810,19 +1657,16 @@ static int rtw_wx_set_essid(struct net_device *dev,
 				  pnetwork->network.Ssid.Ssid));
 
 			if ((!memcmp(dst_ssid, src_ssid, ndis_ssid.SsidLength)) &&
-				(pnetwork->network.Ssid.SsidLength ==ndis_ssid.SsidLength))
-			{
+				(pnetwork->network.Ssid.SsidLength ==ndis_ssid.SsidLength)) {
 				RT_TRACE(_module_rtl871x_ioctl_os_c, _drv_info_,
 					 ("rtw_wx_set_essid: find match, set infra mode\n"));
 
-				if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == true)
-				{
+				if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == true) {
 					if (pnetwork->network.InfrastructureMode != pmlmepriv->cur_network.network.InfrastructureMode)
 						continue;
 				}
 
-				if (rtw_set_802_11_infrastructure_mode(padapter, pnetwork->network.InfrastructureMode) == false)
-				{
+				if (rtw_set_802_11_infrastructure_mode(padapter, pnetwork->network.InfrastructureMode) == false) {
 					ret = -1;
 					spin_unlock_bh(&queue->lock);
 					goto exit;
@@ -1867,8 +1711,7 @@ static int rtw_wx_get_essid(struct net_device *dev,
 	RT_TRACE(_module_rtl871x_mlme_c_, _drv_info_, ("rtw_wx_get_essid\n"));
 
 	if ((check_fwstate(pmlmepriv, _FW_LINKED) == true) ||
-	      (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == true))
-	{
+	      (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == true)) {
 		len = pcur_bss->Ssid.SsidLength;
 
 		wrqu->essid.length = len;
@@ -1876,9 +1719,7 @@ static int rtw_wx_get_essid(struct net_device *dev,
 		memcpy(extra, pcur_bss->Ssid.Ssid, len);
 
 		wrqu->essid.flags = 1;
-	}
-	else
-	{
+	} else {
 		ret = -1;
 		goto exit;
 	}
@@ -1952,15 +1793,12 @@ static int rtw_wx_set_rate(struct net_device *dev,
 
 set_rate:
 
-	for (i = 0; i<NumRates; i++)
-	{
-		if (ratevalue ==mpdatarate[i])
-		{
+	for (i = 0; i<NumRates; i++) {
+		if (ratevalue ==mpdatarate[i]) {
 			datarates[i] = mpdatarate[i];
 			if (fixed == 0)
 				break;
-		}
-		else {
+		} else {
 			datarates[i] = 0xff;
 		}
 
@@ -2097,8 +1935,7 @@ static int rtw_wx_set_enc(struct net_device *dev,
 
 	key = erq->flags & IW_ENCODE_INDEX;
 
-	if (erq->flags & IW_ENCODE_DISABLED)
-	{
+	if (erq->flags & IW_ENCODE_DISABLED) {
 		DBG_871X("EncryptionDisabled\n");
 		padapter->securitypriv.ndisencryptstatus = Ndis802_11EncryptionDisabled;
 		padapter->securitypriv.dot11PrivacyAlgrthm = _NO_PRIVACY_;
@@ -2115,17 +1952,14 @@ static int rtw_wx_set_enc(struct net_device *dev,
 			return -EINVAL;
 		key--;
 		keyindex_provided = 1;
-	}
-	else
-	{
+	} else {
 		keyindex_provided = 0;
 		key = padapter->securitypriv.dot11PrivacyKeyIndex;
 		DBG_871X("rtw_wx_set_enc, key =%d\n", key);
 	}
 
 	/* set authentication mode */
-	if (erq->flags & IW_ENCODE_OPEN)
-	{
+	if (erq->flags & IW_ENCODE_OPEN) {
 		DBG_871X("rtw_wx_set_enc():IW_ENCODE_OPEN\n");
 		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled;/* Ndis802_11EncryptionDisabled; */
 
@@ -2135,9 +1969,7 @@ static int rtw_wx_set_enc(struct net_device *dev,
 		padapter->securitypriv.dot118021XGrpPrivacy = _NO_PRIVACY_;
 		authmode = Ndis802_11AuthModeOpen;
 		padapter->securitypriv.ndisauthtype =authmode;
-	}
-	else if (erq->flags & IW_ENCODE_RESTRICTED)
-	{
+	} else if (erq->flags & IW_ENCODE_RESTRICTED) {
 		DBG_871X("rtw_wx_set_enc():IW_ENCODE_RESTRICTED\n");
 		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled;
 
@@ -2147,9 +1979,7 @@ static int rtw_wx_set_enc(struct net_device *dev,
 		padapter->securitypriv.dot118021XGrpPrivacy = _WEP40_;
 		authmode = Ndis802_11AuthModeShared;
 		padapter->securitypriv.ndisauthtype =authmode;
-	}
-	else
-	{
+	} else {
 		DBG_871X("rtw_wx_set_enc():erq->flags = 0x%x\n", erq->flags);
 
 		padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled;/* Ndis802_11EncryptionDisabled; */
@@ -2161,24 +1991,19 @@ static int rtw_wx_set_enc(struct net_device *dev,
 	}
 
 	wep.KeyIndex = key;
-	if (erq->length > 0)
-	{
+	if (erq->length > 0) {
 		wep.KeyLength = erq->length <= 5 ? 5 : 13;
 
 		wep.Length = wep.KeyLength + FIELD_OFFSET(struct ndis_802_11_wep, KeyMaterial);
-	}
-	else
-	{
+	} else {
 		wep.KeyLength = 0 ;
 
-		if (keyindex_provided == 1)/*  set key_id only, no given KeyMaterial(erq->length == 0). */
-		{
+		if (keyindex_provided == 1) { /*  set key_id only, no given KeyMaterial(erq->length == 0). */
 			padapter->securitypriv.dot11PrivacyKeyIndex = key;
 
 			DBG_871X("(keyindex_provided == 1), keyid =%d, key_len =%d\n", key, padapter->securitypriv.dot11DefKeylen[key]);
 
-			switch (padapter->securitypriv.dot11DefKeylen[key])
-			{
+			switch (padapter->securitypriv.dot11DefKeylen[key]) {
 				case 5:
 					padapter->securitypriv.dot11PrivacyAlgrthm = _WEP40_;
 					break;
@@ -2219,14 +2044,12 @@ static int rtw_wx_get_enc(struct net_device *dev,
 	struct iw_point *erq = &(wrqu->encoding);
 	struct	mlme_priv *pmlmepriv = &(padapter->mlmepriv);
 
-	if (check_fwstate(pmlmepriv, _FW_LINKED) != true)
-	{
-		 if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) != true)
-		 {
-		erq->length = 0;
-		erq->flags |= IW_ENCODE_DISABLED;
-		return 0;
-	}
+	if (check_fwstate(pmlmepriv, _FW_LINKED) != true) {
+		 if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) != true) {
+			 erq->length = 0;
+			 erq->flags |= IW_ENCODE_DISABLED;
+			 return 0;
+		 }
 	}
 
 
@@ -2236,8 +2059,7 @@ static int rtw_wx_get_enc(struct net_device *dev,
 		if (key > WEP_KEYS)
 			return -EINVAL;
 		key--;
-	} else
-	{
+	} else {
 		key = padapter->securitypriv.dot11PrivacyKeyIndex;
 	}
 
@@ -2248,8 +2070,7 @@ static int rtw_wx_get_enc(struct net_device *dev,
 	/*       erq->flags |= IW_ENCODE_OPEN; */
 	/*  */
 
-	switch (padapter->securitypriv.ndisencryptstatus)
-	{
+	switch (padapter->securitypriv.ndisencryptstatus) {
 	case Ndis802_11EncryptionNotSupported:
 	case Ndis802_11EncryptionDisabled:
 		erq->length = 0;
@@ -2258,23 +2079,16 @@ static int rtw_wx_get_enc(struct net_device *dev,
 	case Ndis802_11Encryption1Enabled:
 		erq->length = padapter->securitypriv.dot11DefKeylen[key];
 
-		if (erq->length)
-		{
+		if (erq->length) {
 			memcpy(keybuf, padapter->securitypriv.dot11DefKey[key].skey, padapter->securitypriv.dot11DefKeylen[key]);
 
 			erq->flags |= IW_ENCODE_ENABLED;
 
 			if (padapter->securitypriv.ndisauthtype == Ndis802_11AuthModeOpen)
-			{
 				erq->flags |= IW_ENCODE_OPEN;
-			}
 			else if (padapter->securitypriv.ndisauthtype == Ndis802_11AuthModeShared)
-			{
-		erq->flags |= IW_ENCODE_RESTRICTED;
-			}
-		}
-		else
-		{
+				erq->flags |= IW_ENCODE_RESTRICTED;
+		} else {
 			erq->length = 0;
 			erq->flags |= IW_ENCODE_DISABLED;
 		}
@@ -2343,14 +2157,13 @@ static int rtw_wx_set_auth(struct net_device *dev,
 
 	case IW_AUTH_TKIP_COUNTERMEASURES:
         {
-	    if (param->value)
-            {  /*  wpa_supplicant is enabling the tkip countermeasure. */
-               padapter->securitypriv.btkip_countermeasure = true;
-            }
-            else
-            {  /*  wpa_supplicant is disabling the tkip countermeasure. */
-               padapter->securitypriv.btkip_countermeasure = false;
-            }
+		if (param->value) {
+			/*  wpa_supplicant is enabling the tkip countermeasure. */
+			padapter->securitypriv.btkip_countermeasure = true;
+		} else {
+			/*  wpa_supplicant is disabling the tkip countermeasure. */
+			padapter->securitypriv.btkip_countermeasure = false;
+		}
 		break;
         }
 	case IW_AUTH_DROP_UNENCRYPTED:
@@ -2367,8 +2180,7 @@ static int rtw_wx_set_auth(struct net_device *dev,
 			 * be set.
 			 */
 
-			if (padapter->securitypriv.ndisencryptstatus == Ndis802_11Encryption1Enabled)
-			{
+			if (padapter->securitypriv.ndisencryptstatus == Ndis802_11Encryption1Enabled) {
 				break;/* it means init value, or using wep, ndisencryptstatus = Ndis802_11Encryption1Enabled, */
 						/*  then it needn't reset it; */
 			}
@@ -2462,37 +2274,29 @@ static int rtw_wx_set_enc_ext(struct net_device *dev,
 	strncpy((char *)param->u.crypt.alg, alg_name, IEEE_CRYPT_ALG_NAME_LEN);
 
 	if (pext->ext_flags & IW_ENCODE_EXT_SET_TX_KEY)
-	{
 		param->u.crypt.set_tx = 1;
-	}
 
 	/* cliW: WEP does not have group key
 	 * just not checking GROUP key setting
 	 */
 	if ((pext->alg != IW_ENCODE_ALG_WEP) &&
 		((pext->ext_flags & IW_ENCODE_EXT_GROUP_KEY)
-		|| (pext->ext_flags & IW_ENCODE_ALG_AES_CMAC)
-	))
-	{
+		|| (pext->ext_flags & IW_ENCODE_ALG_AES_CMAC)))	{
 		param->u.crypt.set_tx = 0;
 	}
 
 	param->u.crypt.idx = (pencoding->flags&0x00FF) -1 ;
 
 	if (pext->ext_flags & IW_ENCODE_EXT_RX_SEQ_VALID)
-	{
 		memcpy(param->u.crypt.seq, pext->rx_seq, 8);
-	}
 
-	if (pext->key_len)
-	{
+	if (pext->key_len) {
 		param->u.crypt.key_len = pext->key_len;
 		/* memcpy(param + 1, pext + 1, pext->key_len); */
 		memcpy(param->u.crypt.key, pext + 1, pext->key_len);
 	}
 
-	if (pencoding->flags & IW_ENCODE_DISABLED)
-	{
+	if (pencoding->flags & IW_ENCODE_DISABLED) {
 		/* todo: remove key */
 		/* remove = 1; */
 	}
@@ -2514,8 +2318,7 @@ static int rtw_wx_get_nick(struct net_device *dev,
 	 /* struct mlme_priv *pmlmepriv = &(padapter->mlmepriv); */
 	 /* struct security_priv *psecuritypriv = &padapter->securitypriv; */
 
-	if (extra)
-	{
+	if (extra) {
 		wrqu->data.length = 14;
 		wrqu->data.flags = 1;
 		memcpy(extra, "<WIFI@REALTEK>", 14);
@@ -2683,9 +2486,9 @@ static int rtw_wx_set_channel_plan(struct net_device *dev,
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
 	u8 channel_plan_req = (u8) (*((int *)wrqu));
 
-	if (_SUCCESS == rtw_set_chplan_cmd(padapter, channel_plan_req, 1, 1)) {
+	if (_SUCCESS == rtw_set_chplan_cmd(padapter, channel_plan_req, 1, 1))
 		DBG_871X("%s set channel_plan = 0x%02X\n", __func__, channel_plan_req);
-	} else
+	 else
 		return -EPERM;
 
 	return 0;
@@ -2751,14 +2554,12 @@ static int rtw_get_ap_info(struct net_device *dev,
 
 	DBG_871X("+rtw_get_aplist_info\n");
 
-	if ((padapter->bDriverStopped) || (pdata == NULL))
-	{
+	if ((padapter->bDriverStopped) || (pdata == NULL)) {
 		ret = -EINVAL;
 		goto exit;
 	}
 
-	while ((check_fwstate(pmlmepriv, (_FW_UNDER_SURVEY|_FW_UNDER_LINKING))) == true)
-	{
+	while ((check_fwstate(pmlmepriv, (_FW_UNDER_SURVEY|_FW_UNDER_LINKING))) == true) {
 		msleep(30);
 		cnt++;
 		if (cnt > 100)
@@ -2768,16 +2569,12 @@ static int rtw_get_ap_info(struct net_device *dev,
 
 	/* pdata->length = 0;? */
 	pdata->flags = 0;
-	if (pdata->length>=32)
-	{
-		if (copy_from_user(data, pdata->pointer, 32))
-		{
+	if (pdata->length>=32) {
+		if (copy_from_user(data, pdata->pointer, 32)) {
 			ret = -EINVAL;
 			goto exit;
 		}
-	}
-	else
-	{
+	} else {
 		ret = -EINVAL;
 		goto exit;
 	}
@@ -2787,8 +2584,7 @@ static int rtw_get_ap_info(struct net_device *dev,
 	phead = get_list_head(queue);
 	plist = get_next(phead);
 
-	while (1)
-	{
+	while (1) {
 		if (phead == plist)
 			break;
 
@@ -2796,32 +2592,27 @@ static int rtw_get_ap_info(struct net_device *dev,
 		pnetwork = LIST_CONTAINOR(plist, struct wlan_network, list);
 
 		/* if (hwaddr_aton_i(pdata->pointer, bssid)) */
-		if (hwaddr_aton_i(data, bssid))
-		{
+		if (hwaddr_aton_i(data, bssid)) {
 			DBG_871X("Invalid BSSID '%s'.\n", (u8 *)data);
 			spin_unlock_bh(&(pmlmepriv->scanned_queue.lock));
 			return -EINVAL;
 		}
 
 
-		if (!memcmp(bssid, pnetwork->network.MacAddress, ETH_ALEN))/* BSSID match, then check if supporting wpa/wpa2 */
-		{
+		if (!memcmp(bssid, pnetwork->network.MacAddress, ETH_ALEN)) { /* BSSID match, then check if supporting wpa/wpa2 */
 			DBG_871X("BSSID:" MAC_FMT "\n", MAC_ARG(bssid));
 
 			pbuf = rtw_get_wpa_ie(&pnetwork->network.IEs[12], &wpa_ielen, pnetwork->network.IELength-12);
-			if (pbuf && (wpa_ielen>0))
-			{
+			if (pbuf && (wpa_ielen>0)) {
 				pdata->flags = 1;
 				break;
 			}
 
 			pbuf = rtw_get_wpa2_ie(&pnetwork->network.IEs[12], &wpa_ielen, pnetwork->network.IELength-12);
-			if (pbuf && (wpa_ielen>0))
-			{
+			if (pbuf && (wpa_ielen>0)) {
 				pdata->flags = 2;
 				break;
 			}
-
 		}
 
 		plist = get_next(plist);
@@ -2830,10 +2621,8 @@ static int rtw_get_ap_info(struct net_device *dev,
 
 	spin_unlock_bh(&(pmlmepriv->scanned_queue.lock));
 
-	if (pdata->length>=34)
-	{
-		if (copy_to_user((u8 __force __user *)pdata->pointer+32, (u8 *)&pdata->flags, 1))
-		{
+	if (pdata->length>=34) {
+		if (copy_to_user((u8 __force __user *)pdata->pointer+32, (u8 *)&pdata->flags, 1)) {
 			ret = -EINVAL;
 			goto exit;
 		}
@@ -2855,8 +2644,7 @@ static int rtw_set_pid(struct net_device *dev,
 	int *pdata = (int *)wrqu;
 	int selector;
 
-	if ((padapter->bDriverStopped) || (pdata == NULL))
-	{
+	if ((padapter->bDriverStopped) || (pdata == NULL)) {
 		ret = -EINVAL;
 		goto exit;
 	}
@@ -2886,17 +2674,14 @@ static int rtw_wps_start(struct net_device *dev,
 	u32   u32wps_start = 0;
         unsigned int uintRet = 0;
 
-	if ((true == padapter->bDriverStopped) ||(true ==padapter->bSurpriseRemoved) || (NULL == pdata))
-	{
+	if ((true == padapter->bDriverStopped) ||(true ==padapter->bSurpriseRemoved) || (NULL == pdata)) {
 		ret = -EINVAL;
 		goto exit;
 	}
 
 	uintRet = copy_from_user((void*) &u32wps_start, pdata->pointer, 4);
 	if (u32wps_start == 0)
-	{
 		u32wps_start = *extra;
-	}
 
 	DBG_871X("[%s] wps_start = %d\n", __func__, u32wps_start);
 
@@ -2964,18 +2749,15 @@ static int rtw_rereg_nd_name(struct net_device *dev,
 	if (wrqu->data.length > IFNAMSIZ)
 		return -EFAULT;
 
-	if (copy_from_user(new_ifname, wrqu->data.pointer, IFNAMSIZ)) {
+	if (copy_from_user(new_ifname, wrqu->data.pointer, IFNAMSIZ))
 		return -EFAULT;
-	}
 
-	if (0 == strcmp(rereg_priv->old_ifname, new_ifname)) {
+	if (0 == strcmp(rereg_priv->old_ifname, new_ifname))
 		return ret;
-	}
 
 	DBG_871X("%s new_ifname:%s\n", __func__, new_ifname);
-	if (0 != (ret = rtw_change_ifname(padapter, new_ifname))) {
+	if (0 != (ret = rtw_change_ifname(padapter, new_ifname)))
 		goto exit;
-	}
 
 	strncpy(rereg_priv->old_ifname, new_ifname, IFNAMSIZ);
 	rereg_priv->old_ifname[IFNAMSIZ-1] = 0;
@@ -3021,11 +2803,9 @@ static int rtw_dbg_port(struct net_device *dev,
 
 	extra_arg = *(pdata+1);
 
-	switch (major_cmd)
-	{
+	switch (major_cmd) {
 		case 0x70:/* read_reg */
-			switch (minor_cmd)
-			{
+			switch (minor_cmd) {
 				case 1:
 					DBG_871X("rtw_read8(0x%x) = 0x%02x\n", arg, rtw_read8(padapter, arg));
 					break;
@@ -3038,8 +2818,7 @@ static int rtw_dbg_port(struct net_device *dev,
 			}
 			break;
 		case 0x71:/* write_reg */
-			switch (minor_cmd)
-			{
+			switch (minor_cmd) {
 				case 1:
 					rtw_write8(padapter, arg, extra_arg);
 					DBG_871X("rtw_write8(0x%x) = 0x%02x\n", arg, rtw_read8(padapter, arg));
@@ -3070,8 +2849,7 @@ static int rtw_dbg_port(struct net_device *dev,
 			break;
 
 		case 0x76:
-			switch (minor_cmd)
-			{
+			switch (minor_cmd) {
 				case 0x00: /* normal mode, */
 					padapter->recvpriv.is_signal_dbg = 0;
 					break;
@@ -3108,8 +2886,7 @@ static int rtw_dbg_port(struct net_device *dev,
 				, WLAN_REASON_EXPIRATION_CHK);
 			break;
 		case 0x7F:
-			switch (minor_cmd)
-			{
+			switch (minor_cmd) {
 				case 0x0:
 					DBG_871X("fwstate = 0x%x\n", get_fwstate(pmlmepriv));
 					break;
@@ -3137,8 +2914,7 @@ static int rtw_dbg_port(struct net_device *dev,
 					break;
 				case 0x05:
 					psta = rtw_get_stainfo(pstapriv, cur_network->network.MacAddress);
-					if (psta)
-					{
+					if (psta) {
 						int i;
 						struct recv_reorder_ctrl *preorder_ctrl;
 
@@ -3152,18 +2928,13 @@ static int rtw_dbg_port(struct net_device *dev,
 						DBG_871X("ampdu_enable = %d\n", psta->htpriv.ampdu_enable);
 						DBG_871X("agg_enable_bitmap =%x, candidate_tid_bitmap =%x\n", psta->htpriv.agg_enable_bitmap, psta->htpriv.candidate_tid_bitmap);
 
-						for (i = 0;i<16;i++)
-						{
+						for (i = 0;i<16;i++) {
 							preorder_ctrl = &psta->recvreorder_ctrl[i];
 							if (preorder_ctrl->enable)
-							{
 								DBG_871X("tid =%d, indicate_seq =%d\n", i, preorder_ctrl->indicate_seq);
-							}
 						}
 
-					}
-					else
-					{
+					} else {
 						DBG_871X("can't get sta's macaddr, cur_network's macaddr:" MAC_FMT "\n", MAC_ARG(cur_network->network.MacAddress));
 					}
 					break;
@@ -3196,19 +2967,16 @@ static int rtw_dbg_port(struct net_device *dev,
 
 						spin_lock_bh(&pstapriv->sta_hash_lock);
 
-						for (i = 0; i< NUM_STA; i++)
-						{
+						for (i = 0; i< NUM_STA; i++) {
 							phead = &(pstapriv->sta_hash[i]);
 							plist = get_next(phead);
 
-							while (phead != plist)
-							{
+							while (phead != plist) {
 								psta = LIST_CONTAINOR(plist, struct sta_info, hash_list);
 
 								plist = get_next(plist);
 
-								if (extra_arg == psta->aid)
-								{
+								if (extra_arg == psta->aid) {
 									DBG_871X("sta's macaddr:" MAC_FMT "\n", MAC_ARG(psta->hwaddr));
 									DBG_871X("rtsen =%d, cts2slef =%d\n", psta->rtsen, psta->cts2self);
 									DBG_871X("state = 0x%x, aid =%d, macid =%d, raid =%d\n", psta->state, psta->aid, psta->mac_id, psta->raid);
@@ -3226,17 +2994,12 @@ static int rtw_dbg_port(struct net_device *dev,
 
 
 
-									for (j = 0;j<16;j++)
-									{
+									for (j = 0;j<16;j++) {
 										preorder_ctrl = &psta->recvreorder_ctrl[j];
 										if (preorder_ctrl->enable)
-										{
 											DBG_871X("tid =%d, indicate_seq =%d\n", j, preorder_ctrl->indicate_seq);
-										}
 									}
-
 								}
-
 							}
 						}
 
@@ -3255,8 +3018,7 @@ static int rtw_dbg_port(struct net_device *dev,
 					if (arg == 0) {
 						DBG_871X("disable driver ctrl vcs\n");
 						padapter->driver_vcs_en = 0;
-					}
-					else if (arg == 1) {
+					} else if (arg == 1) {
 						DBG_871X("enable driver ctrl vcs = %d\n", extra_arg);
 						padapter->driver_vcs_en = 1;
 
@@ -3272,8 +3034,7 @@ static int rtw_dbg_port(struct net_device *dev,
 							DBG_871X("dump rx packet (%d)\n", extra_arg);
 							/* pHalData->bDumpRxPkt =extra_arg; */
 							rtw_hal_set_def_var(padapter, HAL_DEF_DBG_DUMP_RXPKT, &(extra_arg));
-						}
-						else if (arg == 1) {
+						} else if (arg == 1) {
 							DBG_871X("dump tx packet (%d)\n", extra_arg);
 							rtw_hal_set_def_var(padapter, HAL_DEF_DBG_DUMP_TXPKT, &(extra_arg));
 						}
@@ -3284,8 +3045,7 @@ static int rtw_dbg_port(struct net_device *dev,
 						if (arg == 0) {
 							DBG_871X("disable driver ctrl rx_ampdu_factor\n");
 							padapter->driver_rx_ampdu_factor = 0xFF;
-						}
-						else if (arg == 1) {
+						} else if (arg == 1) {
 
 							DBG_871X("enable driver ctrl rx_ampdu_factor = %d\n", extra_arg);
 
@@ -3310,12 +3070,10 @@ static int rtw_dbg_port(struct net_device *dev,
 					struct registry_priv *pregpriv = &padapter->registrypriv;
 					/*  0: disable, bit(0):enable 2.4g, bit(1):enable 5g, 0x3: enable both 2.4g and 5g */
 					/* default is set to enable 2.4GHZ for IOT issue with bufflao's AP at 5GHZ */
-					if (pregpriv && (extra_arg == 0 || extra_arg == 1|| extra_arg == 2 || extra_arg == 3))
-					{
+					if (pregpriv && (extra_arg == 0 || extra_arg == 1|| extra_arg == 2 || extra_arg == 3)) {
 						pregpriv->rx_stbc = extra_arg;
 						DBG_871X("set rx_stbc =%d\n", pregpriv->rx_stbc);
-					}
-					else
+					} else
 						DBG_871X("get rx_stbc =%d\n", pregpriv->rx_stbc);
 
 				}
@@ -3324,12 +3082,10 @@ static int rtw_dbg_port(struct net_device *dev,
 				{
 					struct registry_priv *pregpriv = &padapter->registrypriv;
 					/*  0: disable, 0x1:enable (but wifi_spec should be 0), 0x2: force enable (don't care wifi_spec) */
-					if (pregpriv && extra_arg < 3)
-					{
+					if (pregpriv && extra_arg < 3) {
 						pregpriv->ampdu_enable = extra_arg;
 						DBG_871X("set ampdu_enable =%d\n", pregpriv->ampdu_enable);
-					}
-					else
+					} else
 						DBG_871X("get ampdu_enable =%d\n", pregpriv->ampdu_enable);
 
 				}
@@ -3343,8 +3099,7 @@ static int rtw_dbg_port(struct net_device *dev,
 				{
 					if (arg == 0xff) {
 						rtw_odm_dbg_comp_msg(RTW_DBGDUMP, padapter);
-					}
-					else {
+					} else {
 						u64 dbg_comp = (u64)extra_arg;
 						rtw_odm_dbg_comp_set(padapter, dbg_comp);
 					}
@@ -3375,8 +3130,7 @@ static int rtw_dbg_port(struct net_device *dev,
 						if (arg == 0) {
 							DBG_871X("driver disable LDPC\n");
 							pregistrypriv->ldpc_cap = 0x00;
-						}
-						else if (arg == 1) {
+						} else if (arg == 1) {
 							DBG_871X("driver set LDPC cap = 0x%x\n", extra_arg);
 							pregistrypriv->ldpc_cap = (u8)(extra_arg&0x33);
 						}
@@ -3391,8 +3145,7 @@ static int rtw_dbg_port(struct net_device *dev,
 						if (arg == 0) {
 							DBG_871X("driver disable STBC\n");
 							pregistrypriv->stbc_cap = 0x00;
-						}
-						else if (arg == 1) {
+						} else if (arg == 1) {
 							DBG_871X("driver set STBC cap = 0x%x\n", extra_arg);
 							pregistrypriv->stbc_cap = (u8)(extra_arg&0x33);
 						}
@@ -3406,8 +3159,7 @@ static int rtw_dbg_port(struct net_device *dev,
 							DBG_871X("disable driver ctrl max_rx_rate, reset to default_rate_set\n");
 							init_mlme_default_rate_set(padapter);
 							pregistrypriv->ht_enable = (u8)rtw_ht_enable;
-						}
-						else if (arg == 1) {
+						} else if (arg == 1) {
 
 							int i;
 							u8 max_rx_rate;
@@ -3416,18 +3168,15 @@ static int rtw_dbg_port(struct net_device *dev,
 
 							max_rx_rate = (u8)extra_arg;
 
-							if (max_rx_rate < 0xc) /*  max_rx_rate < MSC0 -> B or G -> disable HT */
-							{
+							if (max_rx_rate < 0xc) { /*  max_rx_rate < MSC0 -> B or G -> disable HT */
 								pregistrypriv->ht_enable = 0;
-								for (i = 0; i<NumRates; i++)
-								{
+								for (i = 0; i<NumRates; i++) {
 									if (pmlmeext->datarate[i] > max_rx_rate)
 										pmlmeext->datarate[i] = 0xff;
 								}
 
 							}
-							else if (max_rx_rate < 0x1c) /*  mcs0~mcs15 */
-							{
+							else if (max_rx_rate < 0x1c) { /*  mcs0~mcs15 */
 								u32 mcs_bitmap = 0x0;
 
 								for (i = 0; i<((max_rx_rate+1)-0xc); i++)
@@ -3443,8 +3192,7 @@ static int rtw_dbg_port(struct net_device *dev,
 						if (arg == 0) {
 							DBG_871X("disable driver ctrl ampdu density\n");
 							padapter->driver_ampdu_spacing = 0xFF;
-						}
-						else if (arg == 1) {
+						} else if (arg == 1) {
 
 							DBG_871X("enable driver ctrl ampdu density = %d\n", extra_arg);
 
@@ -3531,15 +3279,12 @@ static int rtw_dbg_port(struct net_device *dev,
 					break;
 				case 0xdd:/* registers dump , 0 for mac reg, 1 for bb reg, 2 for rf reg */
 					{
-						if (extra_arg == 0) {
+						if (extra_arg == 0)
 							mac_reg_dump(RTW_DBGDUMP, padapter);
-						}
-						else if (extra_arg == 1) {
+						else if (extra_arg == 1)
 							bb_reg_dump(RTW_DBGDUMP, padapter);
-						}
-						else if (extra_arg ==2) {
+						else if (extra_arg ==2)
 							rf_reg_dump(RTW_DBGDUMP, padapter);
-						}
 					}
 					break;
 
@@ -3557,8 +3302,7 @@ static int rtw_dbg_port(struct net_device *dev,
 							DBG_871X("extra_arg = 4  - disable BT coexistence - BIT(3)\n");
 							DBG_871X("extra_arg = 5  - disable antenna diversity - BIT(4)\n");
 							DBG_871X("extra_arg = 6  - enable all dynamic func\n");
-						}
-						else {
+						} else {
 							/*extra_arg = 0  - disable all dynamic func
 								extra_arg = 1  - disable DIG
 								extra_arg = 2  - disable tx power tracking
@@ -3634,8 +3378,7 @@ static int wpa_set_param(struct net_device *dev, u8 name, u32 value)
 
 		/* ret = ieee80211_wpa_enable(ieee, value); */
 
-		switch ((value)&0xff)
-		{
+		switch ((value)&0xff) {
 		case 1 : /* WPA */
 			padapter->securitypriv.ndisauthtype = Ndis802_11AuthModeWPAPSK; /* WPA_PSK */
 			padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption2Enabled;
@@ -3721,8 +3464,7 @@ static int wpa_mlme(struct net_device *dev, u32 command, u32 reason)
 	int ret = 0;
 	struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev);
 
-	switch (command)
-	{
+	switch (command) {
 		case IEEE_MLME_STA_DEAUTH:
 
 			if (!rtw_set_802_11_disassociate(padapter))
@@ -3759,14 +3501,12 @@ static int wpa_supplicant_ioctl(struct net_device *dev, struct iw_point *p)
 	}
 
 	param = (struct ieee_param *)rtw_malloc(p->length);
-	if (param == NULL)
-	{
+	if (param == NULL) {
 		ret = -ENOMEM;
 		goto out;
 	}
 
-	if (copy_from_user(param, p->pointer, p->length))
-	{
+	if (copy_from_user(param, p->pointer, p->length)) {
 		kfree(param);
 		ret = -EFAULT;
 		goto out;
@@ -3829,35 +3569,28 @@ static int rtw_set_encryption(struct net_device *dev, struct ieee_param *param,
 
 	/* sizeof(struct ieee_param) = 64 bytes; */
 	/* if (param_len !=  (u32) ((u8 *) param->u.crypt.key - (u8 *) param) + param->u.crypt.key_len) */
-	if (param_len !=  sizeof(struct ieee_param) + param->u.crypt.key_len)
-	{
+	if (param_len !=  sizeof(struct ieee_param) + param->u.crypt.key_len) {
 		ret =  -EINVAL;
 		goto exit;
 	}
 
 	if (param->sta_addr[0] == 0xff && param->sta_addr[1] == 0xff &&
 	    param->sta_addr[2] == 0xff && param->sta_addr[3] == 0xff &&
-	    param->sta_addr[4] == 0xff && param->sta_addr[5] == 0xff)
-	{
-		if (param->u.crypt.idx >= WEP_KEYS)
-		{
+	    param->sta_addr[4] == 0xff && param->sta_addr[5] == 0xff) {
+		if (param->u.crypt.idx >= WEP_KEYS) {
 			ret = -EINVAL;
 			goto exit;
 		}
-	}
-	else
-	{
+	} else {
 		psta = rtw_get_stainfo(pstapriv, param->sta_addr);
-		if (!psta)
-		{
+		if (!psta) {
 			/* ret = -EINVAL; */
 			DBG_871X("rtw_set_encryption(), sta has already been removed or never been added\n");
 			goto exit;
 		}
 	}
 
-	if (strcmp(param->u.crypt.alg, "none") == 0 && (psta == NULL))
-	{
+	if (strcmp(param->u.crypt.alg, "none") == 0 && (psta == NULL)) {
 		/* todo:clear default encryption keys */
 
 		psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_Open;
@@ -3871,8 +3604,7 @@ static int rtw_set_encryption(struct net_device *dev, struct ieee_param *param,
 	}
 
 
-	if (strcmp(param->u.crypt.alg, "WEP") == 0 && (psta == NULL))
-	{
+	if (strcmp(param->u.crypt.alg, "WEP") == 0 && (psta == NULL)) {
 		DBG_871X("r871x_set_encryption, crypt.alg = WEP\n");
 
 		wep_key_idx = param->u.crypt.idx;
@@ -3880,15 +3612,13 @@ static int rtw_set_encryption(struct net_device *dev, struct ieee_param *param,
 
 		DBG_871X("r871x_set_encryption, wep_key_idx =%d, len =%d\n", wep_key_idx, wep_key_len);
 
-		if ((wep_key_idx >= WEP_KEYS) || (wep_key_len<= 0))
-		{
+		if ((wep_key_idx >= WEP_KEYS) || (wep_key_len<= 0)) {
 			ret = -EINVAL;
 			goto exit;
 		}
 
 
-		if (wep_key_len > 0)
-		{
+		if (wep_key_len > 0) {
 			wep_key_len = wep_key_len <= 5 ? 5 : 13;
 			wep_total_len = wep_key_len + FIELD_OFFSET(struct ndis_802_11_wep, KeyMaterial);
 			pwep =(struct ndis_802_11_wep *)rtw_malloc(wep_total_len);
@@ -3908,8 +3638,7 @@ static int rtw_set_encryption(struct net_device *dev, struct ieee_param *param,
 
 		memcpy(pwep->KeyMaterial,  param->u.crypt.key, pwep->KeyLength);
 
-		if (param->u.crypt.set_tx)
-		{
+		if (param->u.crypt.set_tx) {
 			DBG_871X("wep, set_tx = 1\n");
 
 			psecuritypriv->dot11AuthAlgrthm = dot11AuthAlgrthm_Auto;
@@ -3917,8 +3646,7 @@ static int rtw_set_encryption(struct net_device *dev, struct ieee_param *param,
 			psecuritypriv->dot11PrivacyAlgrthm = _WEP40_;
 			psecuritypriv->dot118021XGrpPrivacy = _WEP40_;
 
-			if (pwep->KeyLength == 13)
-			{
+			if (pwep->KeyLength == 13) {
 				psecuritypriv->dot11PrivacyAlgrthm = _WEP104_;
 				psecuritypriv->dot118021XGrpPrivacy = _WEP104_;
 			}
@@ -3931,9 +3659,7 @@ static int rtw_set_encryption(struct net_device *dev, struct ieee_param *param,
 			psecuritypriv->dot11DefKeylen[wep_key_idx]=pwep->KeyLength;
 
 			rtw_ap_set_wep_key(padapter, pwep->KeyMaterial, pwep->KeyLength, wep_key_idx, 1);
-		}
-		else
-		{
+		} else {
 			DBG_871X("wep, set_tx = 0\n");
 
 			/* don't update "psecuritypriv->dot11PrivacyAlgrthm" and */
@@ -3951,25 +3677,18 @@ static int rtw_set_encryption(struct net_device *dev, struct ieee_param *param,
 	}
 
 
-	if (!psta && check_fwstate(pmlmepriv, WIFI_AP_STATE)) /*  group key */
-	{
-		if (param->u.crypt.set_tx == 1)
-		{
-			if (strcmp(param->u.crypt.alg, "WEP") == 0)
-			{
+	if (!psta && check_fwstate(pmlmepriv, WIFI_AP_STATE)) { /*  group key */
+		if (param->u.crypt.set_tx == 1) {
+			if (strcmp(param->u.crypt.alg, "WEP") == 0) {
 				DBG_871X("%s, set group_key, WEP\n", __func__);
 
 				memcpy(psecuritypriv->dot118021XGrpKey[param->u.crypt.idx].skey,  param->u.crypt.key, (param->u.crypt.key_len>16 ?16:param->u.crypt.key_len));
 
 				psecuritypriv->dot118021XGrpPrivacy = _WEP40_;
 				if (param->u.crypt.key_len == 13)
-				{
 						psecuritypriv->dot118021XGrpPrivacy = _WEP104_;
-				}
 
-			}
-			else if (strcmp(param->u.crypt.alg, "TKIP") == 0)
-			{
+			} else if (strcmp(param->u.crypt.alg, "TKIP") == 0) {
 				DBG_871X("%s, set group_key, TKIP\n", __func__);
 
 				psecuritypriv->dot118021XGrpPrivacy = _TKIP_;
@@ -3984,16 +3703,13 @@ static int rtw_set_encryption(struct net_device *dev, struct ieee_param *param,
 				psecuritypriv->busetkipkey = true;
 
 			}
-			else if (strcmp(param->u.crypt.alg, "CCMP") == 0)
-			{
+			else if (strcmp(param->u.crypt.alg, "CCMP") == 0) {
 				DBG_871X("%s, set group_key, CCMP\n", __func__);
 
 				psecuritypriv->dot118021XGrpPrivacy = _AES_;
 
 				memcpy(psecuritypriv->dot118021XGrpKey[param->u.crypt.idx].skey,  param->u.crypt.key, (param->u.crypt.key_len>16 ?16:param->u.crypt.key_len));
-			}
-			else
-			{
+			} else {
 				DBG_871X("%s, set group_key, none\n", __func__);
 
 				psecuritypriv->dot118021XGrpPrivacy = _NO_PRIVACY_;
@@ -4008,38 +3724,28 @@ static int rtw_set_encryption(struct net_device *dev, struct ieee_param *param,
 			rtw_ap_set_group_key(padapter, param->u.crypt.key, psecuritypriv->dot118021XGrpPrivacy, param->u.crypt.idx);
 
 			pbcmc_sta =rtw_get_bcmc_stainfo(padapter);
-			if (pbcmc_sta)
-			{
+			if (pbcmc_sta) {
 				pbcmc_sta->ieee8021x_blocked = false;
 				pbcmc_sta->dot118021XPrivacy = psecuritypriv->dot118021XGrpPrivacy;/* rx will use bmc_sta's dot118021XPrivacy */
 			}
-
 		}
 
 		goto exit;
 
 	}
 
-	if (psecuritypriv->dot11AuthAlgrthm == dot11AuthAlgrthm_8021X && psta) /*  psk/802_1x */
-	{
-		if (check_fwstate(pmlmepriv, WIFI_AP_STATE))
-		{
-			if (param->u.crypt.set_tx == 1)
-			{
+	if (psecuritypriv->dot11AuthAlgrthm == dot11AuthAlgrthm_8021X && psta) { /*  psk/802_1x */
+		if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
+			if (param->u.crypt.set_tx == 1)	{
 				memcpy(psta->dot118021x_UncstKey.skey,  param->u.crypt.key, (param->u.crypt.key_len>16 ?16:param->u.crypt.key_len));
 
-				if (strcmp(param->u.crypt.alg, "WEP") == 0)
-				{
+				if (strcmp(param->u.crypt.alg, "WEP") == 0) {
 					DBG_871X("%s, set pairwise key, WEP\n", __func__);
 
 					psta->dot118021XPrivacy = _WEP40_;
 					if (param->u.crypt.key_len == 13)
-					{
 						psta->dot118021XPrivacy = _WEP104_;
-					}
-				}
-				else if (strcmp(param->u.crypt.alg, "TKIP") == 0)
-				{
+				} else if (strcmp(param->u.crypt.alg, "TKIP") == 0) {
 					DBG_871X("%s, set pairwise key, TKIP\n", __func__);
 
 					psta->dot118021XPrivacy = _TKIP_;
@@ -4051,16 +3757,12 @@ static int rtw_set_encryption(struct net_device *dev, struct ieee_param *param,
 
 					psecuritypriv->busetkipkey = true;
 
-				}
-				else if (strcmp(param->u.crypt.alg, "CCMP") == 0)
-				{
+				} else if (strcmp(param->u.crypt.alg, "CCMP") == 0) {
 
 					DBG_871X("%s, set pairwise key, CCMP\n", __func__);
 
 					psta->dot118021XPrivacy = _AES_;
-				}
-				else
-				{
+				} else {
 					DBG_871X("%s, set pairwise key, none\n", __func__);
 
 					psta->dot118021XPrivacy = _NO_PRIVACY_;
@@ -4070,21 +3772,14 @@ static int rtw_set_encryption(struct net_device *dev, struct ieee_param *param,
 
 				psta->ieee8021x_blocked = false;
 
-			}
-			else/* group key??? */
-			{
-				if (strcmp(param->u.crypt.alg, "WEP") == 0)
-				{
+			} else { /* group key??? */
+				if (strcmp(param->u.crypt.alg, "WEP") == 0) {
 					memcpy(psecuritypriv->dot118021XGrpKey[param->u.crypt.idx].skey,  param->u.crypt.key, (param->u.crypt.key_len>16 ?16:param->u.crypt.key_len));
 
 					psecuritypriv->dot118021XGrpPrivacy = _WEP40_;
 					if (param->u.crypt.key_len == 13)
-					{
 						psecuritypriv->dot118021XGrpPrivacy = _WEP104_;
-					}
-				}
-				else if (strcmp(param->u.crypt.alg, "TKIP") == 0)
-				{
+				} else if (strcmp(param->u.crypt.alg, "TKIP") == 0) {
 					psecuritypriv->dot118021XGrpPrivacy = _TKIP_;
 
 					memcpy(psecuritypriv->dot118021XGrpKey[param->u.crypt.idx].skey,  param->u.crypt.key, (param->u.crypt.key_len>16 ?16:param->u.crypt.key_len));
@@ -4096,15 +3791,11 @@ static int rtw_set_encryption(struct net_device *dev, struct ieee_param *param,
 
 					psecuritypriv->busetkipkey = true;
 
-				}
-				else if (strcmp(param->u.crypt.alg, "CCMP") == 0)
-				{
+				} else if (strcmp(param->u.crypt.alg, "CCMP") == 0) {
 					psecuritypriv->dot118021XGrpPrivacy = _AES_;
 
 					memcpy(psecuritypriv->dot118021XGrpKey[param->u.crypt.idx].skey,  param->u.crypt.key, (param->u.crypt.key_len>16 ?16:param->u.crypt.key_len));
-				}
-				else
-				{
+				} else {
 					psecuritypriv->dot118021XGrpPrivacy = _NO_PRIVACY_;
 				}
 
@@ -4117,16 +3808,12 @@ static int rtw_set_encryption(struct net_device *dev, struct ieee_param *param,
 				rtw_ap_set_group_key(padapter, param->u.crypt.key, psecuritypriv->dot118021XGrpPrivacy, param->u.crypt.idx);
 
 				pbcmc_sta =rtw_get_bcmc_stainfo(padapter);
-				if (pbcmc_sta)
-				{
+				if (pbcmc_sta) {
 					pbcmc_sta->ieee8021x_blocked = false;
 					pbcmc_sta->dot118021XPrivacy = psecuritypriv->dot118021XGrpPrivacy;/* rx will use bmc_sta's dot118021XPrivacy */
 				}
-
 			}
-
 		}
-
 	}
 
 exit:
@@ -4196,14 +3883,11 @@ static int rtw_add_sta(struct net_device *dev, struct ieee_param *param)
 	DBG_871X("rtw_add_sta(aid =%d) =" MAC_FMT "\n", param->u.add_sta.aid, MAC_ARG(param->sta_addr));
 
 	if (check_fwstate(pmlmepriv, (_FW_LINKED|WIFI_AP_STATE)) != true)
-	{
 		return -EINVAL;
-	}
 
 	if (param->sta_addr[0] == 0xff && param->sta_addr[1] == 0xff &&
 	    param->sta_addr[2] == 0xff && param->sta_addr[3] == 0xff &&
-	    param->sta_addr[4] == 0xff && param->sta_addr[5] == 0xff)
-	{
+	    param->sta_addr[4] == 0xff && param->sta_addr[5] == 0xff) {
 		return -EINVAL;
 	}
 
@@ -4221,8 +3905,7 @@ static int rtw_add_sta(struct net_device *dev, struct ieee_param *param)
 */
 	/* psta = rtw_alloc_stainfo(pstapriv, param->sta_addr); */
 	psta = rtw_get_stainfo(pstapriv, param->sta_addr);
-	if (psta)
-	{
+	if (psta) {
 		int flags = param->u.add_sta.flags;
 
 		/* DBG_871X("rtw_add_sta(), init sta's variables, psta =%p\n", psta); */
@@ -4242,14 +3925,11 @@ static int rtw_add_sta(struct net_device *dev, struct ieee_param *param)
 			psta->qos_option = 0;
 
 		/* chec 802.11n ht cap. */
-		if (WLAN_STA_HT&flags)
-		{
+		if (WLAN_STA_HT&flags) {
 			psta->htpriv.ht_option = true;
 			psta->qos_option = 1;
 			memcpy((void*)&psta->htpriv.ht_cap, (void*)&param->u.add_sta.ht_cap, sizeof(struct rtw_ieee80211_ht_cap));
-		}
-		else
-		{
+		} else {
 			psta->htpriv.ht_option = false;
 		}
 
@@ -4259,9 +3939,7 @@ static int rtw_add_sta(struct net_device *dev, struct ieee_param *param)
 		update_sta_info_apmode(padapter, psta);
 
 
-	}
-	else
-	{
+	} else {
 		ret = -ENOMEM;
 	}
 
@@ -4280,27 +3958,22 @@ static int rtw_del_sta(struct net_device *dev, struct ieee_param *param)
 	DBG_871X("rtw_del_sta =" MAC_FMT "\n", MAC_ARG(param->sta_addr));
 
 	if (check_fwstate(pmlmepriv, (_FW_LINKED|WIFI_AP_STATE)) != true)
-	{
 		return -EINVAL;
-	}
 
 	if (param->sta_addr[0] == 0xff && param->sta_addr[1] == 0xff &&
 	    param->sta_addr[2] == 0xff && param->sta_addr[3] == 0xff &&
-	    param->sta_addr[4] == 0xff && param->sta_addr[5] == 0xff)
-	{
+	    param->sta_addr[4] == 0xff && param->sta_addr[5] == 0xff) {
 		return -EINVAL;
 	}
 
 	psta = rtw_get_stainfo(pstapriv, param->sta_addr);
-	if (psta)
-	{
+	if (psta) {
 		u8 updated =false;
 
 		/* DBG_871X("free psta =%p, aid =%d\n", psta, psta->aid); */
 
 		spin_lock_bh(&pstapriv->asoc_list_lock);
-		if (list_empty(&psta->asoc_list) ==false)
-		{
+		if (list_empty(&psta->asoc_list) ==false) {
 			list_del_init(&psta->asoc_list);
 			pstapriv->asoc_list_cnt--;
 			updated = ap_free_sta(padapter, psta, true, WLAN_REASON_DEAUTH_LEAVING);
@@ -4312,9 +3985,7 @@ static int rtw_del_sta(struct net_device *dev, struct ieee_param *param)
 
 		psta = NULL;
 
-	}
-	else
-	{
+	} else {
 		DBG_871X("rtw_del_sta(), sta has already been removed or never been added\n");
 
 		/* ret = -1; */
@@ -4338,20 +4009,16 @@ static int rtw_ioctl_get_sta_data(struct net_device *dev, struct ieee_param *par
 	DBG_871X("rtw_ioctl_get_sta_info, sta_addr: " MAC_FMT "\n", MAC_ARG(param_ex->sta_addr));
 
 	if (check_fwstate(pmlmepriv, (_FW_LINKED|WIFI_AP_STATE)) != true)
-	{
 		return -EINVAL;
-	}
 
 	if (param_ex->sta_addr[0] == 0xff && param_ex->sta_addr[1] == 0xff &&
 	    param_ex->sta_addr[2] == 0xff && param_ex->sta_addr[3] == 0xff &&
-	    param_ex->sta_addr[4] == 0xff && param_ex->sta_addr[5] == 0xff)
-	{
+	    param_ex->sta_addr[4] == 0xff && param_ex->sta_addr[5] == 0xff) {
 		return -EINVAL;
 	}
 
 	psta = rtw_get_stainfo(pstapriv, param_ex->sta_addr);
-	if (psta)
-	{
+	if (psta) {
 		psta_data->aid = (u16)psta->aid;
 		psta_data->capability = psta->capability;
 		psta_data->flags = psta->flags;
@@ -4384,9 +4051,7 @@ static int rtw_ioctl_get_sta_data(struct net_device *dev, struct ieee_param *par
 		psta_data->tx_drops = psta->sta_stats.tx_drops;
 
 
-	}
-	else
-	{
+	} else {
 		ret = -1;
 	}
 
@@ -4405,22 +4070,17 @@ static int rtw_get_sta_wpaie(struct net_device *dev, struct ieee_param *param)
 	DBG_871X("rtw_get_sta_wpaie, sta_addr: " MAC_FMT "\n", MAC_ARG(param->sta_addr));
 
 	if (check_fwstate(pmlmepriv, (_FW_LINKED|WIFI_AP_STATE)) != true)
-	{
 		return -EINVAL;
-	}
 
 	if (param->sta_addr[0] == 0xff && param->sta_addr[1] == 0xff &&
 	    param->sta_addr[2] == 0xff && param->sta_addr[3] == 0xff &&
-	    param->sta_addr[4] == 0xff && param->sta_addr[5] == 0xff)
-	{
+	    param->sta_addr[4] == 0xff && param->sta_addr[5] == 0xff) {
 		return -EINVAL;
 	}
 
 	psta = rtw_get_stainfo(pstapriv, param->sta_addr);
-	if (psta)
-	{
-		if ((psta->wpa_ie[0] == WLAN_EID_RSN) || (psta->wpa_ie[0] == WLAN_EID_GENERIC))
-		{
+	if (psta) {
+		if ((psta->wpa_ie[0] == WLAN_EID_RSN) || (psta->wpa_ie[0] == WLAN_EID_GENERIC)) {
 			int wpa_ie_len;
 			int copy_len;
 
@@ -4431,15 +4091,11 @@ static int rtw_get_sta_wpaie(struct net_device *dev, struct ieee_param *param)
 			param->u.wpa_ie.len = copy_len;
 
 			memcpy(param->u.wpa_ie.reserved, psta->wpa_ie, copy_len);
-		}
-		else
-		{
+		} else {
 			/* ret = -1; */
 			DBG_871X("sta's wpa_ie is NONE\n");
 		}
-	}
-	else
-	{
+	} else {
 		ret = -1;
 	}
 
@@ -4464,14 +4120,10 @@ static int rtw_set_wps_beacon(struct net_device *dev, struct ieee_param *param,
 	ie_len = len-12-2;/*  12 = param header, 2:no packed */
 
 
-	if (pmlmepriv->wps_beacon_ie)
-	{
-		kfree(pmlmepriv->wps_beacon_ie);
-		pmlmepriv->wps_beacon_ie = NULL;
-	}
+	kfree(pmlmepriv->wps_beacon_ie);
+	pmlmepriv->wps_beacon_ie = NULL;
 
-	if (ie_len>0)
-	{
+	if (ie_len>0) {
 		pmlmepriv->wps_beacon_ie = rtw_malloc(ie_len);
 		pmlmepriv->wps_beacon_ie_len = ie_len;
 		if (pmlmepriv->wps_beacon_ie == NULL) {
@@ -4484,7 +4136,6 @@ static int rtw_set_wps_beacon(struct net_device *dev, struct ieee_param *param,
 		update_beacon(padapter, _VENDOR_SPECIFIC_IE_, wps_oui, true);
 
 		pmlmeext->bstart_bss = true;
-
 	}
 
 
@@ -4507,14 +4158,10 @@ static int rtw_set_wps_probe_resp(struct net_device *dev, struct ieee_param *par
 	ie_len = len-12-2;/*  12 = param header, 2:no packed */
 
 
-	if (pmlmepriv->wps_probe_resp_ie)
-	{
-		kfree(pmlmepriv->wps_probe_resp_ie);
-		pmlmepriv->wps_probe_resp_ie = NULL;
-	}
+	kfree(pmlmepriv->wps_probe_resp_ie);
+	pmlmepriv->wps_probe_resp_ie = NULL;
 
-	if (ie_len>0)
-	{
+	if (ie_len>0) {
 		pmlmepriv->wps_probe_resp_ie = rtw_malloc(ie_len);
 		pmlmepriv->wps_probe_resp_ie_len = ie_len;
 		if (pmlmepriv->wps_probe_resp_ie == NULL) {
@@ -4544,14 +4191,10 @@ static int rtw_set_wps_assoc_resp(struct net_device *dev, struct ieee_param *par
 	ie_len = len-12-2;/*  12 = param header, 2:no packed */
 
 
-	if (pmlmepriv->wps_assoc_resp_ie)
-	{
-		kfree(pmlmepriv->wps_assoc_resp_ie);
-		pmlmepriv->wps_assoc_resp_ie = NULL;
-	}
+	kfree(pmlmepriv->wps_assoc_resp_ie);
+	pmlmepriv->wps_assoc_resp_ie = NULL;
 
-	if (ie_len>0)
-	{
+	if (ie_len>0) {
 		pmlmepriv->wps_assoc_resp_ie = rtw_malloc(ie_len);
 		pmlmepriv->wps_assoc_resp_ie_len = ie_len;
 		if (pmlmepriv->wps_assoc_resp_ie == NULL) {
@@ -4632,8 +4275,7 @@ static int rtw_ioctl_acl_remove_sta(struct net_device *dev, struct ieee_param *p
 
 	if (param->sta_addr[0] == 0xff && param->sta_addr[1] == 0xff &&
 	    param->sta_addr[2] == 0xff && param->sta_addr[3] == 0xff &&
-	    param->sta_addr[4] == 0xff && param->sta_addr[5] == 0xff)
-	{
+	    param->sta_addr[4] == 0xff && param->sta_addr[5] == 0xff) {
 		return -EINVAL;
 	}
 
@@ -4654,8 +4296,7 @@ static int rtw_ioctl_acl_add_sta(struct net_device *dev, struct ieee_param *para
 
 	if (param->sta_addr[0] == 0xff && param->sta_addr[1] == 0xff &&
 	    param->sta_addr[2] == 0xff && param->sta_addr[3] == 0xff &&
-	    param->sta_addr[4] == 0xff && param->sta_addr[5] == 0xff)
-	{
+	    param->sta_addr[4] == 0xff && param->sta_addr[5] == 0xff) {
 		return -EINVAL;
 	}
 
@@ -4705,14 +4346,12 @@ static int rtw_hostapd_ioctl(struct net_device *dev, struct iw_point *p)
 	}
 
 	param = (struct ieee_param *)rtw_malloc(p->length);
-	if (param == NULL)
-	{
+	if (param == NULL) {
 		ret = -ENOMEM;
 		goto out;
 	}
 
-	if (copy_from_user(param, p->pointer, p->length))
-	{
+	if (copy_from_user(param, p->pointer, p->length)) {
 		kfree(param);
 		ret = -EFAULT;
 		goto out;
@@ -4720,8 +4359,7 @@ static int rtw_hostapd_ioctl(struct net_device *dev, struct iw_point *p)
 
 	/* DBG_871X("%s, cmd =%d\n", __func__, param->cmd); */
 
-	switch (param->cmd)
-	{
+	switch (param->cmd) {
 		case RTL871X_HOSTAPD_FLUSH:
 
 			ret = rtw_hostapd_sta_flush(dev);
@@ -4861,8 +4499,7 @@ static int rtw_wx_set_priv(struct net_device *dev,
 	/* 	  dev->name, ext)); */
 
 	#ifdef DEBUG_RTW_WX_SET_PRIV
-	if (!(ext_dbg = vmalloc(len)))
-	{
+	if (!(ext_dbg = vmalloc(len))) {
 		vfree(ext, len);
 		return -ENOMEM;
 	}
@@ -4871,8 +4508,7 @@ static int rtw_wx_set_priv(struct net_device *dev,
 	#endif
 
 	/* added for wps2.0 @20110524 */
-	if (dwrq->flags == 0x8766 && len > 8)
-	{
+	if (dwrq->flags == 0x8766 && len > 8) {
 		u32 cp_sz;
 		struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
 		u8 *probereq_wpsie = ext;
@@ -4880,12 +4516,10 @@ static int rtw_wx_set_priv(struct net_device *dev,
 		u8 wps_oui[4]={0x0, 0x50, 0xf2, 0x04};
 
 		if ((_VENDOR_SPECIFIC_IE_ == probereq_wpsie[0]) &&
-			(!memcmp(&probereq_wpsie[2], wps_oui, 4)))
-		{
+			(!memcmp(&probereq_wpsie[2], wps_oui, 4))) {
 			cp_sz = probereq_wpsie_len>MAX_WPS_IE_LEN ? MAX_WPS_IE_LEN:probereq_wpsie_len;
 
-			if (pmlmepriv->wps_probe_req_ie)
-			{
+			if (pmlmepriv->wps_probe_req_ie) {
 				pmlmepriv->wps_probe_req_ie_len = 0;
 				kfree(pmlmepriv->wps_probe_req_ie);
 				pmlmepriv->wps_probe_req_ie = NULL;
@@ -4909,8 +4543,7 @@ static int rtw_wx_set_priv(struct net_device *dev,
 	}
 
 	if (len >= WEXT_CSCAN_HEADER_SIZE
-		&& !memcmp(ext, WEXT_CSCAN_HEADER, WEXT_CSCAN_HEADER_SIZE)
-	) {
+		&& !memcmp(ext, WEXT_CSCAN_HEADER, WEXT_CSCAN_HEADER_SIZE)) {
 		ret = rtw_wx_set_scan(dev, info, awrq, ext);
 		goto FREE_EXT;
 	}
@@ -4939,17 +4572,13 @@ static int rtw_pm_set(struct net_device *dev,
 
 	DBG_871X("[%s] extra = %s\n", __func__, extra);
 
-	if (!memcmp(extra, "lps =", 4))
-	{
+	if (!memcmp(extra, "lps =", 4)) {
 		sscanf(extra+4, "%u", &mode);
 		ret = rtw_pm_set_lps(padapter, mode);
-	}
-	else if (!memcmp(extra, "ips =", 4))
-	{
+	} else if (!memcmp(extra, "ips =", 4)) {
 		sscanf(extra+4, "%u", &mode);
 		ret = rtw_pm_set_ips(padapter, mode);
-	}
-	else {
+	} else {
 		ret = -EINVAL;
 	}
 
@@ -5015,8 +4644,7 @@ static int rtw_widi_set_probe_request(struct net_device *dev,
 	struct adapter	*padapter = (struct adapter *)rtw_netdev_priv(dev);
 
 	pbuf = rtw_malloc(sizeof(l2_msg_t));
-	if (pbuf)
-	{
+	if (pbuf) {
 		if (copy_from_user(pbuf, wrqu->data.pointer, wrqu->data.length))
 			ret = -EFAULT;
 		/* memcpy(pbuf, wrqu->data.pointer, wrqu->data.length); */
@@ -5067,17 +4695,12 @@ static int rtw_test(
 	}
 
 	if (strcmp(pch, "bton") == 0)
-	{
 		rtw_btcoex_SetManualControl(padapter, false);
-	}
 
 	if (strcmp(pch, "btoff") == 0)
-	{
 		rtw_btcoex_SetManualControl(padapter, true);
-	}
 
-	if (strcmp(pch, "h2c") == 0)
-	{
+	if (strcmp(pch, "h2c") == 0) {
 		u8 param[8];
 		u8 count = 0;
 		u32 tmp;
@@ -5104,9 +4727,8 @@ static int rtw_test(
 		ret = rtw_hal_fill_h2c_cmd(padapter, param[0], count-1, &param[1]);
 
 		pos = sprintf(extra, "H2C ID = 0x%02x content =", param[0]);
-		for (i = 1; i<count; i++) {
+		for (i = 1; i<count; i++)
 			pos += sprintf(extra+pos, "%02x,", param[i]);
-		}
 		extra[pos] = 0;
 		pos--;
 		pos += sprintf(extra+pos, " %s", ret == _FAIL?"FAIL":"OK");
@@ -5118,8 +4740,7 @@ static int rtw_test(
 	return 0;
 }
 
-static iw_handler rtw_handlers[] =
-{
+static iw_handler rtw_handlers[] = {
 	NULL,					/* SIOCSIWCOMMIT */
 	rtw_wx_get_name,		/* SIOCGIWNAME */
 	dummy,					/* SIOCSIWNWID */
@@ -5293,8 +4914,7 @@ static const struct iw_priv_args rtw_private_args[] = {
 #endif
 };
 
-static iw_handler rtw_private_handler[] =
-{
+static iw_handler rtw_private_handler[] = {
 	rtw_wx_write32,					/* 0x00 */
 	rtw_wx_read32,					/* 0x01 */
 	rtw_drvext_hdl,					/* 0x02 */
@@ -5350,14 +4970,12 @@ static struct iw_statistics *rtw_get_wireless_stats(struct net_device *dev)
 	int tmp_qual = 0;
 	int tmp_noise = 0;
 
-	if (check_fwstate(&padapter->mlmepriv, _FW_LINKED) != true)
-	{
+	if (check_fwstate(&padapter->mlmepriv, _FW_LINKED) != true) {
 		piwstats->qual.qual = 0;
 		piwstats->qual.level = 0;
 		piwstats->qual.noise = 0;
 		/* DBG_871X("No link  level:%d, qual:%d, noise:%d\n", tmp_level, tmp_qual, tmp_noise); */
-	}
-	else {
+	} else {
 		#ifdef CONFIG_SIGNAL_DISPLAY_DBM
 		tmp_level = translate_percentage_to_dbm(padapter->recvpriv.signal_strength);
 		#else
@@ -5409,8 +5027,7 @@ static struct iw_statistics *rtw_get_wireless_stats(struct net_device *dev)
 	return &padapter->iwstats;
 }
 
-struct iw_handler_def rtw_handlers_def =
-{
+struct iw_handler_def rtw_handlers_def = {
 	.standard = rtw_handlers,
 	.num_standard = sizeof(rtw_handlers) / sizeof(iw_handler),
 #if defined(CONFIG_WEXT_PRIV)
@@ -5523,8 +5140,7 @@ static int rtw_ioctl_wext_private(struct net_device *dev, union iwreq_data *wrq_
 	}
 
 	/* Watch out for sub-ioctls ! */
-	if (priv_args[k].cmd < SIOCDEVPRIVATE)
-	{
+	if (priv_args[k].cmd < SIOCDEVPRIVATE) {
 		int j = -1;
 
 		/* Find the matching *real* ioctl */
@@ -5554,12 +5170,10 @@ static int rtw_ioctl_wext_private(struct net_device *dev, union iwreq_data *wrq_
 
 	/* If we have to set some data */
 	if ((priv_args[k].set_args & IW_PRIV_TYPE_MASK) &&
-		(priv_args[k].set_args & IW_PRIV_SIZE_MASK))
-	{
+		(priv_args[k].set_args & IW_PRIV_SIZE_MASK)) {
 		u8 *str;
 
-		switch (priv_args[k].set_args & IW_PRIV_TYPE_MASK)
-		{
+		switch (priv_args[k].set_args & IW_PRIV_TYPE_MASK) {
 			case IW_PRIV_TYPE_BYTE:
 				/* Fetch args */
 				count = 0;
@@ -5597,8 +5211,7 @@ static int rtw_ioctl_wext_private(struct net_device *dev, union iwreq_data *wrq_
 				break;
 
 			case IW_PRIV_TYPE_CHAR:
-				if (len > 0)
-				{
+				if (len > 0) {
 					/* Size of the string to fetch */
 					wdata.data.length = len;
 					if (wdata.data.length > (priv_args[k].set_args & IW_PRIV_SIZE_MASK))
@@ -5606,9 +5219,7 @@ static int rtw_ioctl_wext_private(struct net_device *dev, union iwreq_data *wrq_
 
 					/* Fetch string */
 					memcpy(buffer, ptr, wdata.data.length);
-				}
-				else
-				{
+				} else {
 					wdata.data.length = 1;
 					buffer[0] = '\0';
 				}
@@ -5622,41 +5233,32 @@ static int rtw_ioctl_wext_private(struct net_device *dev, union iwreq_data *wrq_
 		}
 
 		if ((priv_args[k].set_args & IW_PRIV_SIZE_FIXED) &&
-			(wdata.data.length != (priv_args[k].set_args & IW_PRIV_SIZE_MASK)))
-		{
+			(wdata.data.length != (priv_args[k].set_args & IW_PRIV_SIZE_MASK))) {
 			DBG_8192C("%s: The command %s needs exactly %d argument(s)...\n",
 					__func__, cmdname, priv_args[k].set_args & IW_PRIV_SIZE_MASK);
 			err = -EINVAL;
 			goto exit;
 		}
-	}   /* if args to set */
-	else
-	{
+	} else { /* if args to set */
 		wdata.data.length = 0L;
 	}
 
 	/* Those two tests are important. They define how the driver
 	* will have to handle the data */
 	if ((priv_args[k].set_args & IW_PRIV_SIZE_FIXED) &&
-		((get_priv_size(priv_args[k].set_args) + offset) <= IFNAMSIZ))
-	{
+		((get_priv_size(priv_args[k].set_args) + offset) <= IFNAMSIZ)) {
 		/* First case : all SET args fit within wrq */
 		if (offset)
 			wdata.mode = subcmd;
 		memcpy(wdata.name + offset, buffer, IFNAMSIZ - offset);
-	}
-	else
-	{
+	} else {
 		if ((priv_args[k].set_args == 0) &&
 			(priv_args[k].get_args & IW_PRIV_SIZE_FIXED) &&
-			(get_priv_size(priv_args[k].get_args) <= IFNAMSIZ))
-		{
+			(get_priv_size(priv_args[k].get_args) <= IFNAMSIZ)) {
 			/* Second case : no SET args, GET args fit within wrq */
 			if (offset)
 				wdata.mode = subcmd;
-		}
-		else
-		{
+		} else {
 			/* Third case : args won't fit in wrq, or variable number of args */
 			if (copy_to_user(wdata.data.pointer, buffer, buffer_len)) {
 				err = -EFAULT;
@@ -5670,8 +5272,7 @@ static int rtw_ioctl_wext_private(struct net_device *dev, union iwreq_data *wrq_
 	input = NULL;
 
 	extra_size = 0;
-	if (IW_IS_SET(priv_args[k].cmd))
-	{
+	if (IW_IS_SET(priv_args[k].cmd)) {
 		/* Size of set arguments */
 		extra_size = get_priv_size(priv_args[k].set_args);
 
@@ -5701,8 +5302,7 @@ static int rtw_ioctl_wext_private(struct net_device *dev, union iwreq_data *wrq_
 
 	/* If we have to get some data */
 	if ((priv_args[k].get_args & IW_PRIV_TYPE_MASK) &&
-		(priv_args[k].get_args & IW_PRIV_SIZE_MASK))
-	{
+		(priv_args[k].get_args & IW_PRIV_SIZE_MASK)) {
 		int j;
 		int n = 0;	/* number of args */
 		u8 str[20] = {0};
@@ -5720,12 +5320,10 @@ static int rtw_ioctl_wext_private(struct net_device *dev, union iwreq_data *wrq_
 			goto exit;
 		}
 
-		switch (priv_args[k].get_args & IW_PRIV_TYPE_MASK)
-		{
+		switch (priv_args[k].get_args & IW_PRIV_TYPE_MASK) {
 			case IW_PRIV_TYPE_BYTE:
 				/* Display args */
-				for (j = 0; j < n; j++)
-				{
+				for (j = 0; j < n; j++) {
 					sprintf(str, "%d  ", extra[j]);
 					len = strlen(str);
 					output_len = strlen(output);
@@ -5739,8 +5337,7 @@ static int rtw_ioctl_wext_private(struct net_device *dev, union iwreq_data *wrq_
 
 			case IW_PRIV_TYPE_INT:
 				/* Display args */
-				for (j = 0; j < n; j++)
-				{
+				for (j = 0; j < n; j++) {
 					sprintf(str, "%d  ", ((__s32*)extra)[j]);
 					len = strlen(str);
 					output_len = strlen(output);
@@ -5769,9 +5366,7 @@ static int rtw_ioctl_wext_private(struct net_device *dev, union iwreq_data *wrq_
 			err = -EFAULT;
 			goto exit;
 		}
-	}   /* if args to set */
-	else
-	{
+	} else { /* if args to set */
 		wrq_data->data.length = 0;
 	}
 
@@ -5788,8 +5383,7 @@ int rtw_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
 	struct iwreq *wrq = (struct iwreq *)rq;
 	int ret = 0;
 
-	switch (cmd)
-	{
+	switch (cmd) {
 		case RTL_IOCTL_WPA_SUPPLICANT:
 			ret = wpa_supplicant_ioctl(dev, &wrq->u.data);
 			break;
diff --git a/drivers/staging/rtl8723bs/os_dep/mlme_linux.c b/drivers/staging/rtl8723bs/os_dep/mlme_linux.c
index 46315d1a82f7..80ca2d781c5d 100644
--- a/drivers/staging/rtl8723bs/os_dep/mlme_linux.c
+++ b/drivers/staging/rtl8723bs/os_dep/mlme_linux.c
@@ -21,7 +21,7 @@
 
 static void _dynamic_check_timer_handlder (void *FunctionContext)
 {
-	struct adapter *adapter = (struct adapter *)FunctionContext;
+	struct adapter *adapter = FunctionContext;
 
 	rtw_dynamic_check_timer_handlder(adapter);
 
@@ -30,7 +30,7 @@ static void _dynamic_check_timer_handlder (void *FunctionContext)
 
 static void _rtw_set_scan_deny_timer_hdl(void *FunctionContext)
 {
-	struct adapter *adapter = (struct adapter *)FunctionContext;
+	struct adapter *adapter = FunctionContext;
 	rtw_set_scan_deny_timer_hdl(adapter);
 }
 
@@ -91,8 +91,6 @@ void rtw_reset_securitypriv(struct adapter *adapter)
 		/*  Backup the btkip_countermeasure information. */
 		/*  When the countermeasure is trigger, the driver have to disconnect with AP for 60 seconds. */
 
-		memset(&backupPMKIDList[ 0 ], 0x00, sizeof(RT_PMKID_LIST) * NUM_PMKID_CACHE);
-
 		memcpy(&backupPMKIDList[ 0 ], &adapter->securitypriv.PMKIDList[ 0 ], sizeof(RT_PMKID_LIST) * NUM_PMKID_CACHE);
 		backupPMKIDIndex = adapter->securitypriv.PMKIDIndex;
 		backupTKIPCountermeasure = adapter->securitypriv.btkip_countermeasure;
diff --git a/drivers/staging/rtl8723bs/os_dep/osdep_service.c b/drivers/staging/rtl8723bs/os_dep/osdep_service.c
index aa16d1ab955b..a05daf06a870 100644
--- a/drivers/staging/rtl8723bs/os_dep/osdep_service.c
+++ b/drivers/staging/rtl8723bs/os_dep/osdep_service.c
@@ -73,7 +73,7 @@ inline int _rtw_netif_rx(_nic_hdl ndev, struct sk_buff *skb)
 
 void rtw_init_timer(_timer *ptimer, void *padapter, void *pfunc)
 {
-	struct adapter *adapter = (struct adapter *)padapter;
+	struct adapter *adapter = padapter;
 
 	_init_timer(ptimer, adapter->pnetdev, pfunc, adapter);
 }
diff --git a/drivers/staging/rtl8723bs/os_dep/sdio_intf.c b/drivers/staging/rtl8723bs/os_dep/sdio_intf.c
index d2fb489d2e83..943324877707 100644
--- a/drivers/staging/rtl8723bs/os_dep/sdio_intf.c
+++ b/drivers/staging/rtl8723bs/os_dep/sdio_intf.c
@@ -138,7 +138,7 @@ static void sdio_free_irq(struct dvobj_priv *dvobj)
 extern unsigned int oob_irq;
 static irqreturn_t gpio_hostwakeup_irq_thread(int irq, void *data)
 {
-	struct adapter *padapter = (struct adapter *)data;
+	struct adapter *padapter = data;
 	DBG_871X_LEVEL(_drv_always_, "gpio_hostwakeup_irq_thread\n");
 	/* Disable interrupt before calling handler */
 	/* disable_irq_nosync(oob_irq); */
diff --git a/drivers/staging/rtl8723bs/os_dep/sdio_ops_linux.c b/drivers/staging/rtl8723bs/os_dep/sdio_ops_linux.c
index 3aa3e6548fd5..3108a625ada3 100644
--- a/drivers/staging/rtl8723bs/os_dep/sdio_ops_linux.c
+++ b/drivers/staging/rtl8723bs/os_dep/sdio_ops_linux.c
@@ -431,7 +431,7 @@ s32 _sd_read(struct intf_hdl *pintfhdl, u32 addr, u32 cnt, void *pdata)
 	if (unlikely((cnt == 1) || (cnt == 2)))
 	{
 		int i;
-		u8 *pbuf = (u8 *)pdata;
+		u8 *pbuf = pdata;
 
 		for (i = 0; i < cnt; i++)
 		{
@@ -534,7 +534,7 @@ s32 _sd_write(struct intf_hdl *pintfhdl, u32 addr, u32 cnt, void *pdata)
 	if (unlikely((cnt == 1) || (cnt == 2)))
 	{
 		int i;
-		u8 *pbuf = (u8 *)pdata;
+		u8 *pbuf = pdata;
 
 		for (i = 0; i < cnt; i++)
 		{
diff --git a/drivers/staging/rtl8723bs/os_dep/xmit_linux.c b/drivers/staging/rtl8723bs/os_dep/xmit_linux.c
index 76968161f936..f29e110f9bdb 100644
--- a/drivers/staging/rtl8723bs/os_dep/xmit_linux.c
+++ b/drivers/staging/rtl8723bs/os_dep/xmit_linux.c
@@ -37,7 +37,7 @@ uint _rtw_pktfile_read (struct pkt_file *pfile, u8 *rmem, uint rlen)
 	uint	len = 0;
 
 	len =  rtw_remainder_len(pfile);
-	len = (rlen > len)? len: rlen;
+	len = (rlen > len) ? len : rlen;
 
 	if (rmem)
 		skb_copy_bits(pfile->pkt, pfile->buf_len-pfile->pkt_len, rmem, len);
@@ -134,7 +134,7 @@ static void rtw_check_xmit_resource(struct adapter *padapter, _pkt *pkt)
 			netif_stop_subqueue(padapter->pnetdev, queue);
 		}
 	} else {
-		if (pxmitpriv->free_xmitframe_cnt<=4) {
+		if (pxmitpriv->free_xmitframe_cnt <= 4) {
 			if (!netif_tx_queue_stopped(netdev_get_tx_queue(padapter->pnetdev, queue)))
 				netif_stop_subqueue(padapter->pnetdev, queue);
 		}
@@ -150,8 +150,8 @@ static int rtw_mlcst2unicst(struct adapter *padapter, struct sk_buff *skb)
 	struct sta_info *psta = NULL;
 	u8 chk_alive_num = 0;
 	char chk_alive_list[NUM_STA];
-	u8 bc_addr[6]={0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
-	u8 null_addr[6]={0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+	u8 bc_addr[6] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+	u8 null_addr[6] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
 
 	int i;
 	s32	res;
@@ -177,7 +177,7 @@ static int rtw_mlcst2unicst(struct adapter *padapter, struct sk_buff *skb)
 
 	for (i = 0; i < chk_alive_num; i++) {
 		psta = rtw_get_stainfo_by_offset(pstapriv, chk_alive_list[i]);
-		if (!(psta->state &_FW_LINKED))
+		if (!(psta->state & _FW_LINKED))
 		{
 			DBG_COUNTER(padapter->tx_logs.os_tx_m2u_ignore_fw_linked);
 			continue;
diff --git a/drivers/staging/rtlwifi/Kconfig b/drivers/staging/rtlwifi/Kconfig
new file mode 100644
index 000000000000..cb3a29ae764b
--- /dev/null
+++ b/drivers/staging/rtlwifi/Kconfig
@@ -0,0 +1,22 @@
+config R8822BE
+	tristate "Realtek RTL8822BE Wireless Network Adapter"
+	depends on PCI && MAC80211 && m
+	select FW_LOADER
+	---help---
+	This is the staging driver for Realtek RTL8822BE 802.11ac PCIe
+	wireless network adapters.
+
+config RTLHALMAC_ST
+	tristate
+	depends on R8822BE
+	default m
+
+config RTLPHYDM_ST
+	tristate
+	depends on R8822BE
+	default m
+
+config RTLWIFI_DEBUG_ST
+	boolean
+	depends on R8822BE
+	default y
diff --git a/drivers/staging/rtlwifi/Makefile b/drivers/staging/rtlwifi/Makefile
new file mode 100644
index 000000000000..0d738c18b29c
--- /dev/null
+++ b/drivers/staging/rtlwifi/Makefile
@@ -0,0 +1,70 @@
+obj-$(CONFIG_R8822BE) 		+= r8822be.o
+
+r8822be-objs	:=		\
+		base.o		\
+		cam.o		\
+		core.o		\
+		debug.o		\
+		efuse.o		\
+		ps.o		\
+		rc.o		\
+		regd.o		\
+		stats.o		\
+		pci.o		\
+		rtl8822be/fw.o	\
+		rtl8822be/hw.o	\
+		rtl8822be/led.o	\
+		rtl8822be/phy.o	\
+		rtl8822be/sw.o	\
+		rtl8822be/trx.o	\
+		btcoexist/halbtc8822b2ant.o	\
+		btcoexist/halbtc8822b1ant.o	\
+		btcoexist/halbtc8822bwifionly.o	\
+		btcoexist/halbtcoutsrc.o	\
+		btcoexist/rtl_btc.o		\
+		halmac/halmac_api.o	\
+		halmac/halmac_88xx/halmac_api_88xx_usb.o	\
+	        halmac/halmac_88xx/halmac_api_88xx_sdio.o	\
+	        halmac/halmac_88xx/halmac_api_88xx.o	\
+	        halmac/halmac_88xx/halmac_api_88xx_pcie.o	\
+	        halmac/halmac_88xx/halmac_func_88xx.o	\
+	        halmac/halmac_88xx/halmac_8822b/halmac_api_8822b_pcie.o	\
+	        halmac/halmac_88xx/halmac_8822b/halmac_func_8822b.o	\
+	        halmac/halmac_88xx/halmac_8822b/halmac_api_8822b_sdio.o	\
+	        halmac/halmac_88xx/halmac_8822b/halmac_api_8822b.o	\
+	        halmac/halmac_88xx/halmac_8822b/halmac_8822b_phy.o	\
+	        halmac/halmac_88xx/halmac_8822b/halmac_8822b_pwr_seq.o	\
+	        halmac/halmac_88xx/halmac_8822b/halmac_api_8822b_usb.o	\
+	        halmac/rtl_halmac.o					\
+		phydm/phydm_debug.o	\
+		phydm/phydm_antdiv.o\
+		phydm/phydm_interface.o\
+		phydm/phydm_hwconfig.o\
+		phydm/phydm.o\
+		phydm/halphyrf_ce.o\
+		phydm/phydm_edcaturbocheck.o\
+		phydm/phydm_dig.o\
+		phydm/phydm_rainfo.o\
+		phydm/phydm_dynamicbbpowersaving.o\
+		phydm/phydm_powertracking_ce.o\
+		phydm/phydm_dynamictxpower.o\
+		phydm/phydm_adaptivity.o\
+		phydm/phydm_cfotracking.o\
+		phydm/phydm_noisemonitor.o\
+		phydm/phydm_acs.o\
+		phydm/phydm_psd.o\
+		phydm/phydm_adc_sampling.o\
+		phydm/phydm_kfree.o\
+		phydm/phydm_ccx.o		\
+		phydm/rtl8822b/halhwimg8822b_bb.o\
+		phydm/rtl8822b/halhwimg8822b_mac.o\
+		phydm/rtl8822b/halhwimg8822b_rf.o\
+		phydm/rtl8822b/halphyrf_8822b.o\
+		phydm/rtl8822b/phydm_hal_api8822b.o\
+		phydm/rtl8822b/phydm_iqk_8822b.o\
+		phydm/rtl8822b/phydm_regconfig8822b.o\
+		phydm/rtl8822b/phydm_rtl8822b.o	\
+		phydm/rtl_phydm.o
+
+
+obj-$(CONFIG_R8822BE)			+= rtl8822be/
diff --git a/drivers/staging/rtlwifi/TODO b/drivers/staging/rtlwifi/TODO
new file mode 100644
index 000000000000..4a084f2fc5d0
--- /dev/null
+++ b/drivers/staging/rtlwifi/TODO
@@ -0,0 +1,11 @@
+TODO:
+- find and remove code blocks guarded by never set CONFIG_FOO defines
+- convert any remaining unusual variable types
+- find codes that can use %pM and %Nph formatting
+- checkpatch.pl fixes - most of the remaining ones are lines too long. Many
+  of them will require refactoring
+- merge Realtek's bugfixes and new features into the driver
+- address any reviewers comments
+
+Please send any patches to Greg Kroah-Hartman <gregkh@linuxfoundation.org>,
+and Larry Finger <Larry.Finger@lwfinger.net>.
diff --git a/drivers/staging/rtlwifi/base.c b/drivers/staging/rtlwifi/base.c
new file mode 100644
index 000000000000..b88b0e8edd3d
--- /dev/null
+++ b/drivers/staging/rtlwifi/base.c
@@ -0,0 +1,2826 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2012  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#include "wifi.h"
+#include "rc.h"
+#include "base.h"
+#include "efuse.h"
+#include "cam.h"
+#include "ps.h"
+#include "regd.h"
+#include "pci.h"
+#include <linux/ip.h>
+#include <linux/module.h>
+#include <linux/udp.h>
+
+/*
+ *NOTICE!!!: This file will be very big, we should
+ *keep it clear under following roles:
+ *
+ *This file include following parts, so, if you add new
+ *functions into this file, please check which part it
+ *should includes. or check if you should add new part
+ *for this file:
+ *
+ *1) mac80211 init functions
+ *2) tx information functions
+ *3) functions called by core.c
+ *4) wq & timer callback functions
+ *5) frame process functions
+ *6) IOT functions
+ *7) sysfs functions
+ *8) vif functions
+ *9) ...
+ */
+
+/*********************************************************
+ *
+ * mac80211 init functions
+ *
+ *********************************************************/
+static struct ieee80211_channel rtl_channeltable_2g[] = {
+	{.center_freq = 2412, .hw_value = 1,},
+	{.center_freq = 2417, .hw_value = 2,},
+	{.center_freq = 2422, .hw_value = 3,},
+	{.center_freq = 2427, .hw_value = 4,},
+	{.center_freq = 2432, .hw_value = 5,},
+	{.center_freq = 2437, .hw_value = 6,},
+	{.center_freq = 2442, .hw_value = 7,},
+	{.center_freq = 2447, .hw_value = 8,},
+	{.center_freq = 2452, .hw_value = 9,},
+	{.center_freq = 2457, .hw_value = 10,},
+	{.center_freq = 2462, .hw_value = 11,},
+	{.center_freq = 2467, .hw_value = 12,},
+	{.center_freq = 2472, .hw_value = 13,},
+	{.center_freq = 2484, .hw_value = 14,},
+};
+
+static struct ieee80211_channel rtl_channeltable_5g[] = {
+	{.center_freq = 5180, .hw_value = 36,},
+	{.center_freq = 5200, .hw_value = 40,},
+	{.center_freq = 5220, .hw_value = 44,},
+	{.center_freq = 5240, .hw_value = 48,},
+	{.center_freq = 5260, .hw_value = 52,},
+	{.center_freq = 5280, .hw_value = 56,},
+	{.center_freq = 5300, .hw_value = 60,},
+	{.center_freq = 5320, .hw_value = 64,},
+	{.center_freq = 5500, .hw_value = 100,},
+	{.center_freq = 5520, .hw_value = 104,},
+	{.center_freq = 5540, .hw_value = 108,},
+	{.center_freq = 5560, .hw_value = 112,},
+	{.center_freq = 5580, .hw_value = 116,},
+	{.center_freq = 5600, .hw_value = 120,},
+	{.center_freq = 5620, .hw_value = 124,},
+	{.center_freq = 5640, .hw_value = 128,},
+	{.center_freq = 5660, .hw_value = 132,},
+	{.center_freq = 5680, .hw_value = 136,},
+	{.center_freq = 5700, .hw_value = 140,},
+	{.center_freq = 5745, .hw_value = 149,},
+	{.center_freq = 5765, .hw_value = 153,},
+	{.center_freq = 5785, .hw_value = 157,},
+	{.center_freq = 5805, .hw_value = 161,},
+	{.center_freq = 5825, .hw_value = 165,},
+};
+
+static struct ieee80211_rate rtl_ratetable_2g[] = {
+	{.bitrate = 10, .hw_value = 0x00,},
+	{.bitrate = 20, .hw_value = 0x01,},
+	{.bitrate = 55, .hw_value = 0x02,},
+	{.bitrate = 110, .hw_value = 0x03,},
+	{.bitrate = 60, .hw_value = 0x04,},
+	{.bitrate = 90, .hw_value = 0x05,},
+	{.bitrate = 120, .hw_value = 0x06,},
+	{.bitrate = 180, .hw_value = 0x07,},
+	{.bitrate = 240, .hw_value = 0x08,},
+	{.bitrate = 360, .hw_value = 0x09,},
+	{.bitrate = 480, .hw_value = 0x0a,},
+	{.bitrate = 540, .hw_value = 0x0b,},
+};
+
+static struct ieee80211_rate rtl_ratetable_5g[] = {
+	{.bitrate = 60, .hw_value = 0x04,},
+	{.bitrate = 90, .hw_value = 0x05,},
+	{.bitrate = 120, .hw_value = 0x06,},
+	{.bitrate = 180, .hw_value = 0x07,},
+	{.bitrate = 240, .hw_value = 0x08,},
+	{.bitrate = 360, .hw_value = 0x09,},
+	{.bitrate = 480, .hw_value = 0x0a,},
+	{.bitrate = 540, .hw_value = 0x0b,},
+};
+
+static const struct ieee80211_supported_band rtl_band_2ghz = {
+	.band = NL80211_BAND_2GHZ,
+
+	.channels = rtl_channeltable_2g,
+	.n_channels = ARRAY_SIZE(rtl_channeltable_2g),
+
+	.bitrates = rtl_ratetable_2g,
+	.n_bitrates = ARRAY_SIZE(rtl_ratetable_2g),
+
+	.ht_cap = {0},
+};
+
+static struct ieee80211_supported_band rtl_band_5ghz = {
+	.band = NL80211_BAND_5GHZ,
+
+	.channels = rtl_channeltable_5g,
+	.n_channels = ARRAY_SIZE(rtl_channeltable_5g),
+
+	.bitrates = rtl_ratetable_5g,
+	.n_bitrates = ARRAY_SIZE(rtl_ratetable_5g),
+
+	.ht_cap = {0},
+};
+
+static const u8 tid_to_ac[] = {
+	2, /* IEEE80211_AC_BE */
+	3, /* IEEE80211_AC_BK */
+	3, /* IEEE80211_AC_BK */
+	2, /* IEEE80211_AC_BE */
+	1, /* IEEE80211_AC_VI */
+	1, /* IEEE80211_AC_VI */
+	0, /* IEEE80211_AC_VO */
+	0, /* IEEE80211_AC_VO */
+};
+
+u8 rtl_tid_to_ac(u8 tid)
+{
+	return tid_to_ac[tid];
+}
+
+static void _rtl_init_hw_ht_capab(struct ieee80211_hw *hw,
+				  struct ieee80211_sta_ht_cap *ht_cap)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+
+	ht_cap->ht_supported = true;
+	ht_cap->cap = IEEE80211_HT_CAP_SUP_WIDTH_20_40 |
+	    IEEE80211_HT_CAP_SGI_40 |
+	    IEEE80211_HT_CAP_SGI_20 |
+	    IEEE80211_HT_CAP_DSSSCCK40 | IEEE80211_HT_CAP_MAX_AMSDU;
+
+	if (rtlpriv->rtlhal.disable_amsdu_8k)
+		ht_cap->cap &= ~IEEE80211_HT_CAP_MAX_AMSDU;
+
+	/*
+	 *Maximum length of AMPDU that the STA can receive.
+	 *Length = 2 ^ (13 + max_ampdu_length_exp) - 1 (octets)
+	 */
+	ht_cap->ampdu_factor = IEEE80211_HT_MAX_AMPDU_64K;
+
+	/*Minimum MPDU start spacing , */
+	ht_cap->ampdu_density = IEEE80211_HT_MPDU_DENSITY_16;
+
+	ht_cap->mcs.tx_params = IEEE80211_HT_MCS_TX_DEFINED;
+
+	/*hw->wiphy->bands[NL80211_BAND_2GHZ]
+	 *base on ant_num
+	 *rx_mask: RX mask
+	 *if rx_ant = 1 rx_mask[0]= 0xff;==>MCS0-MCS7
+	 *if rx_ant = 2 rx_mask[1]= 0xff;==>MCS8-MCS15
+	 *if rx_ant >= 3 rx_mask[2]= 0xff;
+	 *if BW_40 rx_mask[4]= 0x01;
+	 *highest supported RX rate
+	 */
+	if (rtlpriv->dm.supp_phymode_switch) {
+		pr_info("Support phy mode switch\n");
+
+		ht_cap->mcs.rx_mask[0] = 0xFF;
+		ht_cap->mcs.rx_mask[1] = 0xFF;
+		ht_cap->mcs.rx_mask[4] = 0x01;
+
+		ht_cap->mcs.rx_highest = cpu_to_le16(MAX_BIT_RATE_40MHZ_MCS15);
+	} else {
+		if (get_rf_type(rtlphy) == RF_1T2R ||
+		    get_rf_type(rtlphy) == RF_2T2R) {
+			RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
+				 "1T2R or 2T2R\n");
+			ht_cap->mcs.rx_mask[0] = 0xFF;
+			ht_cap->mcs.rx_mask[1] = 0xFF;
+			ht_cap->mcs.rx_mask[4] = 0x01;
+
+			ht_cap->mcs.rx_highest =
+				 cpu_to_le16(MAX_BIT_RATE_40MHZ_MCS15);
+		} else if (get_rf_type(rtlphy) == RF_1T1R) {
+			RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "1T1R\n");
+
+			ht_cap->mcs.rx_mask[0] = 0xFF;
+			ht_cap->mcs.rx_mask[1] = 0x00;
+			ht_cap->mcs.rx_mask[4] = 0x01;
+
+			ht_cap->mcs.rx_highest =
+				 cpu_to_le16(MAX_BIT_RATE_40MHZ_MCS7);
+		}
+	}
+}
+
+static void _rtl_init_hw_vht_capab(struct ieee80211_hw *hw,
+				   struct ieee80211_sta_vht_cap *vht_cap)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+
+	if (rtlhal->hw_type == HARDWARE_TYPE_RTL8812AE ||
+	    rtlhal->hw_type == HARDWARE_TYPE_RTL8822BE) {
+		u16 mcs_map;
+
+		vht_cap->vht_supported = true;
+		vht_cap->cap =
+			IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454 |
+			IEEE80211_VHT_CAP_SHORT_GI_80 |
+			IEEE80211_VHT_CAP_TXSTBC |
+			IEEE80211_VHT_CAP_RXSTBC_1 |
+			IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE |
+			IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE |
+			IEEE80211_VHT_CAP_HTC_VHT |
+			IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK |
+			IEEE80211_VHT_CAP_RX_ANTENNA_PATTERN |
+			IEEE80211_VHT_CAP_TX_ANTENNA_PATTERN |
+			0;
+
+		mcs_map = IEEE80211_VHT_MCS_SUPPORT_0_9 << 0 |
+			IEEE80211_VHT_MCS_SUPPORT_0_9 << 2 |
+			IEEE80211_VHT_MCS_NOT_SUPPORTED << 4 |
+			IEEE80211_VHT_MCS_NOT_SUPPORTED << 6 |
+			IEEE80211_VHT_MCS_NOT_SUPPORTED << 8 |
+			IEEE80211_VHT_MCS_NOT_SUPPORTED << 10 |
+			IEEE80211_VHT_MCS_NOT_SUPPORTED << 12 |
+			IEEE80211_VHT_MCS_NOT_SUPPORTED << 14;
+
+		vht_cap->vht_mcs.rx_mcs_map = cpu_to_le16(mcs_map);
+		vht_cap->vht_mcs.rx_highest =
+			cpu_to_le16(MAX_BIT_RATE_SHORT_GI_2NSS_80MHZ_MCS9);
+		vht_cap->vht_mcs.tx_mcs_map = cpu_to_le16(mcs_map);
+		vht_cap->vht_mcs.tx_highest =
+			cpu_to_le16(MAX_BIT_RATE_SHORT_GI_2NSS_80MHZ_MCS9);
+	} else if (rtlhal->hw_type == HARDWARE_TYPE_RTL8821AE) {
+		u16 mcs_map;
+
+		vht_cap->vht_supported = true;
+		vht_cap->cap =
+			IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454 |
+			IEEE80211_VHT_CAP_SHORT_GI_80 |
+			IEEE80211_VHT_CAP_TXSTBC |
+			IEEE80211_VHT_CAP_RXSTBC_1 |
+			IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE |
+			IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE |
+			IEEE80211_VHT_CAP_HTC_VHT |
+			IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK |
+			IEEE80211_VHT_CAP_RX_ANTENNA_PATTERN |
+			IEEE80211_VHT_CAP_TX_ANTENNA_PATTERN |
+			0;
+
+		mcs_map = IEEE80211_VHT_MCS_SUPPORT_0_9 << 0 |
+			IEEE80211_VHT_MCS_NOT_SUPPORTED << 2 |
+			IEEE80211_VHT_MCS_NOT_SUPPORTED << 4 |
+			IEEE80211_VHT_MCS_NOT_SUPPORTED << 6 |
+			IEEE80211_VHT_MCS_NOT_SUPPORTED << 8 |
+			IEEE80211_VHT_MCS_NOT_SUPPORTED << 10 |
+			IEEE80211_VHT_MCS_NOT_SUPPORTED << 12 |
+			IEEE80211_VHT_MCS_NOT_SUPPORTED << 14;
+
+		vht_cap->vht_mcs.rx_mcs_map = cpu_to_le16(mcs_map);
+		vht_cap->vht_mcs.rx_highest =
+			cpu_to_le16(MAX_BIT_RATE_SHORT_GI_1NSS_80MHZ_MCS9);
+		vht_cap->vht_mcs.tx_mcs_map = cpu_to_le16(mcs_map);
+		vht_cap->vht_mcs.tx_highest =
+			cpu_to_le16(MAX_BIT_RATE_SHORT_GI_1NSS_80MHZ_MCS9);
+	}
+}
+
+static void _rtl_init_mac80211(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+	struct rtl_mac *rtlmac = rtl_mac(rtl_priv(hw));
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
+	struct ieee80211_supported_band *sband;
+
+	if (rtlhal->macphymode == SINGLEMAC_SINGLEPHY &&
+	    rtlhal->bandset == BAND_ON_BOTH) {
+		/* 1: 2.4 G bands */
+		/* <1> use  mac->bands as mem for hw->wiphy->bands */
+		sband = &rtlmac->bands[NL80211_BAND_2GHZ];
+
+		/* <2> set hw->wiphy->bands[NL80211_BAND_2GHZ]
+		 * to default value(1T1R)
+		 */
+		memcpy(&rtlmac->bands[NL80211_BAND_2GHZ], &rtl_band_2ghz,
+		       sizeof(struct ieee80211_supported_band));
+
+		/* <3> init ht cap base on ant_num */
+		_rtl_init_hw_ht_capab(hw, &sband->ht_cap);
+
+		/* <4> set mac->sband to wiphy->sband */
+		hw->wiphy->bands[NL80211_BAND_2GHZ] = sband;
+
+		/* 2: 5 G bands */
+		/* <1> use  mac->bands as mem for hw->wiphy->bands */
+		sband = &rtlmac->bands[NL80211_BAND_5GHZ];
+
+		/* <2> set hw->wiphy->bands[NL80211_BAND_5GHZ]
+		 * to default value(1T1R)
+		 */
+		memcpy(&rtlmac->bands[NL80211_BAND_5GHZ], &rtl_band_5ghz,
+		       sizeof(struct ieee80211_supported_band));
+
+		/* <3> init ht cap base on ant_num */
+		_rtl_init_hw_ht_capab(hw, &sband->ht_cap);
+
+		_rtl_init_hw_vht_capab(hw, &sband->vht_cap);
+		/* <4> set mac->sband to wiphy->sband */
+		hw->wiphy->bands[NL80211_BAND_5GHZ] = sband;
+	} else {
+		if (rtlhal->current_bandtype == BAND_ON_2_4G) {
+			/* <1> use  mac->bands as mem for hw->wiphy->bands */
+			sband = &rtlmac->bands[NL80211_BAND_2GHZ];
+
+			/* <2> set hw->wiphy->bands[NL80211_BAND_2GHZ]
+			 * to default value(1T1R)
+			 */
+			memcpy(&rtlmac->bands[NL80211_BAND_2GHZ],
+			       &rtl_band_2ghz,
+			       sizeof(struct ieee80211_supported_band));
+
+			/* <3> init ht cap base on ant_num */
+			_rtl_init_hw_ht_capab(hw, &sband->ht_cap);
+
+			/* <4> set mac->sband to wiphy->sband */
+			hw->wiphy->bands[NL80211_BAND_2GHZ] = sband;
+		} else if (rtlhal->current_bandtype == BAND_ON_5G) {
+			/* <1> use  mac->bands as mem for hw->wiphy->bands */
+			sband = &rtlmac->bands[NL80211_BAND_5GHZ];
+
+			/* <2> set hw->wiphy->bands[NL80211_BAND_5GHZ]
+			 * to default value(1T1R)
+			 */
+			memcpy(&rtlmac->bands[NL80211_BAND_5GHZ],
+			       &rtl_band_5ghz,
+			       sizeof(struct ieee80211_supported_band));
+
+			/* <3> init ht cap base on ant_num */
+			_rtl_init_hw_ht_capab(hw, &sband->ht_cap);
+
+			_rtl_init_hw_vht_capab(hw, &sband->vht_cap);
+			/* <4> set mac->sband to wiphy->sband */
+			hw->wiphy->bands[NL80211_BAND_5GHZ] = sband;
+		} else {
+			pr_err("Err BAND %d\n",
+			       rtlhal->current_bandtype);
+		}
+	}
+	/* <5> set hw caps */
+	ieee80211_hw_set(hw, SIGNAL_DBM);
+	ieee80211_hw_set(hw, RX_INCLUDES_FCS);
+	ieee80211_hw_set(hw, AMPDU_AGGREGATION);
+	ieee80211_hw_set(hw, CONNECTION_MONITOR);
+	ieee80211_hw_set(hw, MFP_CAPABLE);
+	ieee80211_hw_set(hw, REPORTS_TX_ACK_STATUS);
+	ieee80211_hw_set(hw, SUPPORTS_TX_FRAG);
+	ieee80211_hw_set(hw, SUPPORT_FAST_XMIT);
+	ieee80211_hw_set(hw, SUPPORTS_AMSDU_IN_AMPDU);
+
+	/* swlps or hwlps has been set in diff chip in init_sw_vars */
+	if (rtlpriv->psc.swctrl_lps) {
+		ieee80211_hw_set(hw, SUPPORTS_PS);
+		ieee80211_hw_set(hw, PS_NULLFUNC_STACK);
+	}
+	if (rtlpriv->psc.fwctrl_lps) {
+		ieee80211_hw_set(hw, SUPPORTS_PS);
+		ieee80211_hw_set(hw, SUPPORTS_DYNAMIC_PS);
+	}
+	hw->wiphy->interface_modes =
+	    BIT(NL80211_IFTYPE_AP) |
+	    BIT(NL80211_IFTYPE_STATION) |
+	    BIT(NL80211_IFTYPE_ADHOC) |
+	    BIT(NL80211_IFTYPE_MESH_POINT) |
+	    BIT(NL80211_IFTYPE_P2P_CLIENT) |
+	    BIT(NL80211_IFTYPE_P2P_GO);
+	hw->wiphy->flags |= WIPHY_FLAG_IBSS_RSN;
+
+	hw->wiphy->flags |= WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL;
+
+	hw->wiphy->rts_threshold = 2347;
+
+	hw->queues = AC_MAX;
+	hw->extra_tx_headroom = RTL_TX_HEADER_SIZE;
+
+	/* TODO: Correct this value for our hw */
+	hw->max_listen_interval = MAX_LISTEN_INTERVAL;
+	hw->max_rate_tries = MAX_RATE_TRIES;
+	/* hw->max_rates = 1; */
+	hw->sta_data_size = sizeof(struct rtl_sta_info);
+
+/* wowlan is not supported by kernel if CONFIG_PM is not defined */
+#ifdef CONFIG_PM
+	if (rtlpriv->psc.wo_wlan_mode) {
+		if (rtlpriv->psc.wo_wlan_mode & WAKE_ON_MAGIC_PACKET)
+			rtlpriv->wowlan.flags = WIPHY_WOWLAN_MAGIC_PKT;
+		if (rtlpriv->psc.wo_wlan_mode & WAKE_ON_PATTERN_MATCH) {
+			rtlpriv->wowlan.n_patterns =
+				MAX_SUPPORT_WOL_PATTERN_NUM;
+			rtlpriv->wowlan.pattern_min_len = MIN_WOL_PATTERN_SIZE;
+			rtlpriv->wowlan.pattern_max_len = MAX_WOL_PATTERN_SIZE;
+		}
+		hw->wiphy->wowlan = &rtlpriv->wowlan;
+	}
+#endif
+
+	/* <6> mac address */
+	if (is_valid_ether_addr(rtlefuse->dev_addr)) {
+		SET_IEEE80211_PERM_ADDR(hw, rtlefuse->dev_addr);
+	} else {
+		u8 rtlmac1[] = { 0x00, 0xe0, 0x4c, 0x81, 0x92, 0x00 };
+
+		get_random_bytes((rtlmac1 + (ETH_ALEN - 1)), 1);
+		SET_IEEE80211_PERM_ADDR(hw, rtlmac1);
+	}
+}
+
+static void _rtl_init_deferred_work(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	/* <1> timer */
+	setup_timer(&rtlpriv->works.watchdog_timer,
+		    rtl_watch_dog_timer_callback, (unsigned long)hw);
+	setup_timer(&rtlpriv->works.dualmac_easyconcurrent_retrytimer,
+		    rtl_easy_concurrent_retrytimer_callback, (unsigned long)hw);
+	/* <2> work queue */
+	rtlpriv->works.hw = hw;
+	rtlpriv->works.rtl_wq = alloc_workqueue("%s", 0, 0, rtlpriv->cfg->name);
+	INIT_DELAYED_WORK(&rtlpriv->works.watchdog_wq,
+			  (void *)rtl_watchdog_wq_callback);
+	INIT_DELAYED_WORK(&rtlpriv->works.ips_nic_off_wq,
+			  (void *)rtl_ips_nic_off_wq_callback);
+	INIT_DELAYED_WORK(&rtlpriv->works.ps_work,
+			  (void *)rtl_swlps_wq_callback);
+	INIT_DELAYED_WORK(&rtlpriv->works.ps_rfon_wq,
+			  (void *)rtl_swlps_rfon_wq_callback);
+	INIT_DELAYED_WORK(&rtlpriv->works.fwevt_wq,
+			  (void *)rtl_fwevt_wq_callback);
+	INIT_DELAYED_WORK(&rtlpriv->works.c2hcmd_wq,
+			  (void *)rtl_c2hcmd_wq_callback);
+}
+
+void rtl_deinit_deferred_work(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	del_timer_sync(&rtlpriv->works.watchdog_timer);
+
+	cancel_delayed_work(&rtlpriv->works.watchdog_wq);
+	cancel_delayed_work(&rtlpriv->works.ips_nic_off_wq);
+	cancel_delayed_work(&rtlpriv->works.ps_work);
+	cancel_delayed_work(&rtlpriv->works.ps_rfon_wq);
+	cancel_delayed_work(&rtlpriv->works.fwevt_wq);
+	cancel_delayed_work(&rtlpriv->works.c2hcmd_wq);
+}
+
+void rtl_init_rfkill(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	bool radio_state;
+	bool blocked;
+	u8 valid = 0;
+
+	/*set init state to on */
+	rtlpriv->rfkill.rfkill_state = true;
+	wiphy_rfkill_set_hw_state(hw->wiphy, 0);
+
+	radio_state = rtlpriv->cfg->ops->radio_onoff_checking(hw, &valid);
+
+	if (valid) {
+		pr_info("rtlwifi: wireless switch is %s\n",
+			rtlpriv->rfkill.rfkill_state ? "on" : "off");
+
+		rtlpriv->rfkill.rfkill_state = radio_state;
+
+		blocked = (rtlpriv->rfkill.rfkill_state == 1) ? 0 : 1;
+		wiphy_rfkill_set_hw_state(hw->wiphy, blocked);
+	}
+
+	wiphy_rfkill_start_polling(hw->wiphy);
+}
+
+void rtl_deinit_rfkill(struct ieee80211_hw *hw)
+{
+	wiphy_rfkill_stop_polling(hw->wiphy);
+}
+
+int rtl_init_core(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *rtlmac = rtl_mac(rtl_priv(hw));
+
+	/* <1> init mac80211 */
+	_rtl_init_mac80211(hw);
+	rtlmac->hw = hw;
+
+	/* <2> rate control register */
+	hw->rate_control_algorithm = "rtl_rc";
+
+	/*
+	 * <3> init CRDA must come after init
+	 * mac80211 hw  in _rtl_init_mac80211.
+	 */
+	if (rtl_regd_init(hw, rtl_reg_notifier)) {
+		pr_err("REGD init failed\n");
+		return 1;
+	}
+
+	/* <4> locks */
+	mutex_init(&rtlpriv->locks.conf_mutex);
+	mutex_init(&rtlpriv->locks.ips_mutex);
+	mutex_init(&rtlpriv->locks.lps_mutex);
+	spin_lock_init(&rtlpriv->locks.irq_th_lock);
+	spin_lock_init(&rtlpriv->locks.h2c_lock);
+	spin_lock_init(&rtlpriv->locks.rf_ps_lock);
+	spin_lock_init(&rtlpriv->locks.rf_lock);
+	spin_lock_init(&rtlpriv->locks.waitq_lock);
+	spin_lock_init(&rtlpriv->locks.entry_list_lock);
+	spin_lock_init(&rtlpriv->locks.c2hcmd_lock);
+	spin_lock_init(&rtlpriv->locks.scan_list_lock);
+	spin_lock_init(&rtlpriv->locks.cck_and_rw_pagea_lock);
+	spin_lock_init(&rtlpriv->locks.fw_ps_lock);
+	spin_lock_init(&rtlpriv->locks.iqk_lock);
+	/* <5> init list */
+	INIT_LIST_HEAD(&rtlpriv->entry_list);
+	INIT_LIST_HEAD(&rtlpriv->c2hcmd_list);
+	INIT_LIST_HEAD(&rtlpriv->scan_list.list);
+
+	rtlmac->link_state = MAC80211_NOLINK;
+
+	/* <6> init deferred work */
+	_rtl_init_deferred_work(hw);
+
+	return 0;
+}
+
+static void rtl_free_entries_from_scan_list(struct ieee80211_hw *hw);
+
+void rtl_deinit_core(struct ieee80211_hw *hw)
+{
+	rtl_c2hcmd_launcher(hw, 0);
+	rtl_free_entries_from_scan_list(hw);
+}
+
+void rtl_init_rx_config(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+
+	rtlpriv->cfg->ops->get_hw_reg(hw, HW_VAR_RCR, (u8 *)&mac->rx_conf);
+}
+
+/*********************************************************
+ *
+ * tx information functions
+ *
+ *********************************************************/
+static void _rtl_qurey_shortpreamble_mode(struct ieee80211_hw *hw,
+					  struct rtl_tcb_desc *tcb_desc,
+					  struct ieee80211_tx_info *info)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 rate_flag = info->control.rates[0].flags;
+
+	tcb_desc->use_shortpreamble = false;
+
+	/* 1M can only use Long Preamble. 11B spec */
+	if (tcb_desc->hw_rate == rtlpriv->cfg->maps[RTL_RC_CCK_RATE1M])
+		return;
+	else if (rate_flag & IEEE80211_TX_RC_USE_SHORT_PREAMBLE)
+		tcb_desc->use_shortpreamble = true;
+}
+
+static void _rtl_query_shortgi(struct ieee80211_hw *hw,
+			       struct ieee80211_sta *sta,
+			       struct rtl_tcb_desc *tcb_desc,
+			       struct ieee80211_tx_info *info)
+{
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	u8 rate_flag = info->control.rates[0].flags;
+	u8 sgi_40 = 0, sgi_20 = 0, bw_40 = 0;
+	u8 sgi_80 = 0, bw_80 = 0;
+
+	tcb_desc->use_shortgi = false;
+
+	if (!sta)
+		return;
+
+	sgi_40 = sta->ht_cap.cap & IEEE80211_HT_CAP_SGI_40;
+	sgi_20 = sta->ht_cap.cap & IEEE80211_HT_CAP_SGI_20;
+	sgi_80 = sta->vht_cap.cap & IEEE80211_VHT_CAP_SHORT_GI_80;
+
+	if ((!sta->ht_cap.ht_supported) && (!sta->vht_cap.vht_supported))
+		return;
+
+	if (!sgi_40 && !sgi_20)
+		return;
+
+	if (mac->opmode == NL80211_IFTYPE_STATION) {
+		bw_40 = mac->bw_40;
+		bw_80 = mac->bw_80;
+	} else if (mac->opmode == NL80211_IFTYPE_AP ||
+		 mac->opmode == NL80211_IFTYPE_ADHOC ||
+		 mac->opmode == NL80211_IFTYPE_MESH_POINT) {
+		bw_40 = sta->ht_cap.cap & IEEE80211_HT_CAP_SUP_WIDTH_20_40;
+		bw_80 = sta->vht_cap.vht_supported;
+	}
+
+	if (bw_80) {
+		if (sgi_80)
+			tcb_desc->use_shortgi = true;
+		else
+			tcb_desc->use_shortgi = false;
+	} else {
+		if (bw_40 && sgi_40)
+			tcb_desc->use_shortgi = true;
+		else if (!bw_40 && sgi_20)
+			tcb_desc->use_shortgi = true;
+	}
+
+	if (!(rate_flag & IEEE80211_TX_RC_SHORT_GI))
+		tcb_desc->use_shortgi = false;
+}
+
+static void _rtl_query_protection_mode(struct ieee80211_hw *hw,
+				       struct rtl_tcb_desc *tcb_desc,
+				       struct ieee80211_tx_info *info)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 rate_flag = info->control.rates[0].flags;
+
+	/* Common Settings */
+	tcb_desc->rts_stbc = false;
+	tcb_desc->cts_enable = false;
+	tcb_desc->rts_sc = 0;
+	tcb_desc->rts_bw = false;
+	tcb_desc->rts_use_shortpreamble = false;
+	tcb_desc->rts_use_shortgi = false;
+
+	if (rate_flag & IEEE80211_TX_RC_USE_CTS_PROTECT) {
+		/* Use CTS-to-SELF in protection mode. */
+		tcb_desc->rts_enable = true;
+		tcb_desc->cts_enable = true;
+		tcb_desc->rts_rate = rtlpriv->cfg->maps[RTL_RC_OFDM_RATE24M];
+	} else if (rate_flag & IEEE80211_TX_RC_USE_RTS_CTS) {
+		/* Use RTS-CTS in protection mode. */
+		tcb_desc->rts_enable = true;
+		tcb_desc->rts_rate = rtlpriv->cfg->maps[RTL_RC_OFDM_RATE24M];
+	}
+}
+
+u8 rtl_mrate_idx_to_arfr_id(
+	struct ieee80211_hw *hw, u8 rate_index,
+	enum wireless_mode wirelessmode)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	u8 ret = 0;
+
+	switch (rate_index) {
+	case RATR_INX_WIRELESS_NGB:
+		if (rtlphy->rf_type == RF_1T1R)
+			ret = RATEID_IDX_BGN_40M_1SS;
+		else
+			ret = RATEID_IDX_BGN_40M_2SS;
+		; break;
+	case RATR_INX_WIRELESS_N:
+	case RATR_INX_WIRELESS_NG:
+		if (rtlphy->rf_type == RF_1T1R)
+			ret = RATEID_IDX_GN_N1SS;
+		else
+			ret = RATEID_IDX_GN_N2SS;
+		; break;
+	case RATR_INX_WIRELESS_NB:
+		if (rtlphy->rf_type == RF_1T1R)
+			ret = RATEID_IDX_BGN_20M_1SS_BN;
+		else
+			ret = RATEID_IDX_BGN_20M_2SS_BN;
+		; break;
+	case RATR_INX_WIRELESS_GB:
+		ret = RATEID_IDX_BG;
+		break;
+	case RATR_INX_WIRELESS_G:
+		ret = RATEID_IDX_G;
+		break;
+	case RATR_INX_WIRELESS_B:
+		ret = RATEID_IDX_B;
+		break;
+	case RATR_INX_WIRELESS_MC:
+		if ((wirelessmode == WIRELESS_MODE_B) ||
+		    (wirelessmode == WIRELESS_MODE_G) ||
+		    (wirelessmode == WIRELESS_MODE_N_24G) ||
+		    (wirelessmode == WIRELESS_MODE_AC_24G))
+			ret = RATEID_IDX_BG;
+		else
+			ret = RATEID_IDX_G;
+		break;
+	case RATR_INX_WIRELESS_AC_5N:
+		if (rtlphy->rf_type == RF_1T1R)
+			ret = RATEID_IDX_VHT_1SS;
+		else
+			ret = RATEID_IDX_VHT_2SS;
+		break;
+	case RATR_INX_WIRELESS_AC_24N:
+		if (rtlphy->current_chan_bw == HT_CHANNEL_WIDTH_80) {
+			if (rtlphy->rf_type == RF_1T1R)
+				ret = RATEID_IDX_VHT_1SS;
+			else
+				ret = RATEID_IDX_VHT_2SS;
+		} else {
+			if (rtlphy->rf_type == RF_1T1R)
+				ret = RATEID_IDX_MIX1;
+			else
+				ret = RATEID_IDX_MIX2;
+		}
+		break;
+	default:
+		ret = RATEID_IDX_BGN_40M_2SS;
+		break;
+	}
+	return ret;
+}
+
+static void _rtl_txrate_selectmode(struct ieee80211_hw *hw,
+				   struct ieee80211_sta *sta,
+				   struct rtl_tcb_desc *tcb_desc)
+{
+#define SET_RATE_ID(rate_id)					\
+	((rtlpriv->cfg->spec_ver & RTL_SPEC_NEW_RATEID) ?	\
+		rtl_mrate_idx_to_arfr_id(hw, rate_id,		\
+			(sta_entry ? sta_entry->wireless_mode :	\
+			 WIRELESS_MODE_G)) :			\
+		rate_id)
+
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct rtl_sta_info *sta_entry = NULL;
+	u8 ratr_index = SET_RATE_ID(RATR_INX_WIRELESS_MC);
+
+	if (sta) {
+		sta_entry = (struct rtl_sta_info *)sta->drv_priv;
+		ratr_index = sta_entry->ratr_index;
+	}
+	if (!tcb_desc->disable_ratefallback || !tcb_desc->use_driver_rate) {
+		if (mac->opmode == NL80211_IFTYPE_STATION) {
+			tcb_desc->ratr_index = 0;
+		} else if (mac->opmode == NL80211_IFTYPE_ADHOC ||
+				mac->opmode == NL80211_IFTYPE_MESH_POINT) {
+			if (tcb_desc->multicast || tcb_desc->broadcast) {
+				tcb_desc->hw_rate =
+				    rtlpriv->cfg->maps[RTL_RC_CCK_RATE2M];
+				tcb_desc->use_driver_rate = 1;
+				tcb_desc->ratr_index =
+					SET_RATE_ID(RATR_INX_WIRELESS_MC);
+			} else {
+				tcb_desc->ratr_index = ratr_index;
+			}
+		} else if (mac->opmode == NL80211_IFTYPE_AP) {
+			tcb_desc->ratr_index = ratr_index;
+		}
+	}
+
+	if (rtlpriv->dm.useramask) {
+		tcb_desc->ratr_index = ratr_index;
+		/* TODO we will differentiate adhoc and station future  */
+		if (mac->opmode == NL80211_IFTYPE_STATION ||
+		    mac->opmode == NL80211_IFTYPE_MESH_POINT) {
+			tcb_desc->mac_id = 0;
+
+			if (sta &&
+			    (rtlpriv->cfg->spec_ver & RTL_SPEC_NEW_RATEID))
+				;	/* use sta_entry->ratr_index */
+			else if (mac->mode == WIRELESS_MODE_AC_5G)
+				tcb_desc->ratr_index =
+					SET_RATE_ID(RATR_INX_WIRELESS_AC_5N);
+			else if (mac->mode == WIRELESS_MODE_AC_24G)
+				tcb_desc->ratr_index =
+					SET_RATE_ID(RATR_INX_WIRELESS_AC_24N);
+			else if (mac->mode == WIRELESS_MODE_N_24G)
+				tcb_desc->ratr_index =
+					SET_RATE_ID(RATR_INX_WIRELESS_NGB);
+			else if (mac->mode == WIRELESS_MODE_N_5G)
+				tcb_desc->ratr_index =
+					SET_RATE_ID(RATR_INX_WIRELESS_NG);
+			else if (mac->mode & WIRELESS_MODE_G)
+				tcb_desc->ratr_index =
+					SET_RATE_ID(RATR_INX_WIRELESS_GB);
+			else if (mac->mode & WIRELESS_MODE_B)
+				tcb_desc->ratr_index =
+					SET_RATE_ID(RATR_INX_WIRELESS_B);
+			else if (mac->mode & WIRELESS_MODE_A)
+				tcb_desc->ratr_index =
+					SET_RATE_ID(RATR_INX_WIRELESS_G);
+
+		} else if (mac->opmode == NL80211_IFTYPE_AP ||
+			mac->opmode == NL80211_IFTYPE_ADHOC) {
+			if (sta) {
+				if (sta->aid > 0)
+					tcb_desc->mac_id = sta->aid + 1;
+				else
+					tcb_desc->mac_id = 1;
+			} else {
+				tcb_desc->mac_id = 0;
+			}
+		}
+	}
+#undef SET_RATE_ID
+}
+
+static void _rtl_query_bandwidth_mode(struct ieee80211_hw *hw,
+				      struct ieee80211_sta *sta,
+				      struct rtl_tcb_desc *tcb_desc)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+
+	tcb_desc->packet_bw = false;
+	if (!sta)
+		return;
+	if (mac->opmode == NL80211_IFTYPE_AP ||
+	    mac->opmode == NL80211_IFTYPE_ADHOC ||
+	    mac->opmode == NL80211_IFTYPE_MESH_POINT) {
+		if (!(sta->ht_cap.ht_supported) ||
+		    !(sta->ht_cap.cap & IEEE80211_HT_CAP_SUP_WIDTH_20_40))
+			return;
+	} else if (mac->opmode == NL80211_IFTYPE_STATION) {
+		if (!mac->bw_40 || !(sta->ht_cap.ht_supported))
+			return;
+	}
+	if (tcb_desc->multicast || tcb_desc->broadcast)
+		return;
+
+	/*use legency rate, shall use 20MHz */
+	if (tcb_desc->hw_rate <= rtlpriv->cfg->maps[RTL_RC_OFDM_RATE54M])
+		return;
+
+	tcb_desc->packet_bw = HT_CHANNEL_WIDTH_20_40;
+
+	if (rtlpriv->rtlhal.hw_type == HARDWARE_TYPE_RTL8812AE ||
+	    rtlpriv->rtlhal.hw_type == HARDWARE_TYPE_RTL8821AE ||
+	    (rtlpriv->cfg->spec_ver & RTL_SPEC_SUPPORT_VHT)) {
+		if (mac->opmode == NL80211_IFTYPE_AP ||
+		    mac->opmode == NL80211_IFTYPE_ADHOC ||
+		    mac->opmode == NL80211_IFTYPE_MESH_POINT) {
+			if (!(sta->vht_cap.vht_supported))
+				return;
+		} else if (mac->opmode == NL80211_IFTYPE_STATION) {
+			if (!mac->bw_80 ||
+			    !(sta->vht_cap.vht_supported))
+				return;
+		}
+		if (tcb_desc->hw_rate <=
+			rtlpriv->cfg->maps[RTL_RC_HT_RATEMCS15])
+			return;
+		tcb_desc->packet_bw = HT_CHANNEL_WIDTH_80;
+	}
+}
+
+static u8 _rtl_get_vht_highest_n_rate(struct ieee80211_hw *hw,
+				      struct ieee80211_sta *sta)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	u8 hw_rate;
+	u16 tx_mcs_map = le16_to_cpu(sta->vht_cap.vht_mcs.tx_mcs_map);
+
+	if ((get_rf_type(rtlphy) == RF_2T2R) &&
+	    (tx_mcs_map & 0x000c) != 0x000c) {
+		if ((tx_mcs_map & 0x000c) >> 2 ==
+			IEEE80211_VHT_MCS_SUPPORT_0_7)
+			hw_rate =
+			rtlpriv->cfg->maps[RTL_RC_VHT_RATE_2SS_MCS7];
+		else if ((tx_mcs_map  & 0x000c) >> 2 ==
+			IEEE80211_VHT_MCS_SUPPORT_0_8)
+			hw_rate =
+			rtlpriv->cfg->maps[RTL_RC_VHT_RATE_2SS_MCS9];
+		else
+			hw_rate =
+			rtlpriv->cfg->maps[RTL_RC_VHT_RATE_2SS_MCS9];
+	} else {
+		if ((tx_mcs_map  & 0x0003) ==
+			IEEE80211_VHT_MCS_SUPPORT_0_7)
+			hw_rate =
+			rtlpriv->cfg->maps[RTL_RC_VHT_RATE_1SS_MCS7];
+		else if ((tx_mcs_map  & 0x0003) ==
+			IEEE80211_VHT_MCS_SUPPORT_0_8)
+			hw_rate =
+			rtlpriv->cfg->maps[RTL_RC_VHT_RATE_1SS_MCS9];
+		else
+			hw_rate =
+			rtlpriv->cfg->maps[RTL_RC_VHT_RATE_1SS_MCS9];
+	}
+
+	return hw_rate;
+}
+
+static u8 _rtl_get_highest_n_rate(struct ieee80211_hw *hw,
+				  struct ieee80211_sta *sta)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	u8 hw_rate;
+
+	if ((get_rf_type(rtlphy) == RF_2T2R) &&
+	    (sta->ht_cap.mcs.rx_mask[1] != 0))
+		hw_rate = rtlpriv->cfg->maps[RTL_RC_HT_RATEMCS15];
+	else
+		hw_rate = rtlpriv->cfg->maps[RTL_RC_HT_RATEMCS7];
+
+	return hw_rate;
+}
+
+/* mac80211's rate_idx is like this:
+ *
+ * 2.4G band:rx_status->band == NL80211_BAND_2GHZ
+ *
+ * B/G rate:
+ * (rx_status->flag & RX_FLAG_HT) = 0,
+ * DESC_RATE1M-->DESC_RATE54M ==> idx is 0-->11,
+ *
+ * N rate:
+ * (rx_status->flag & RX_FLAG_HT) = 1,
+ * DESC_RATEMCS0-->DESC_RATEMCS15 ==> idx is 0-->15
+ *
+ * 5G band:rx_status->band == NL80211_BAND_5GHZ
+ * A rate:
+ * (rx_status->flag & RX_FLAG_HT) = 0,
+ * DESC_RATE6M-->DESC_RATE54M ==> idx is 0-->7,
+ *
+ * N rate:
+ * (rx_status->flag & RX_FLAG_HT) = 1,
+ * DESC_RATEMCS0-->DESC_RATEMCS15 ==> idx is 0-->15
+ *
+ * VHT rates:
+ * DESC_RATEVHT1SS_MCS0-->DESC_RATEVHT1SS_MCS9 ==> idx is 0-->9
+ * DESC_RATEVHT2SS_MCS0-->DESC_RATEVHT2SS_MCS9 ==> idx is 0-->9
+ */
+int rtlwifi_rate_mapping(struct ieee80211_hw *hw, bool isht, bool isvht,
+			 u8 desc_rate)
+{
+	int rate_idx;
+
+	if (isvht) {
+		switch (desc_rate) {
+		case DESC_RATEVHT1SS_MCS0:
+			rate_idx = 0;
+			break;
+		case DESC_RATEVHT1SS_MCS1:
+			rate_idx = 1;
+			break;
+		case DESC_RATEVHT1SS_MCS2:
+			rate_idx = 2;
+			break;
+		case DESC_RATEVHT1SS_MCS3:
+			rate_idx = 3;
+			break;
+		case DESC_RATEVHT1SS_MCS4:
+			rate_idx = 4;
+			break;
+		case DESC_RATEVHT1SS_MCS5:
+			rate_idx = 5;
+			break;
+		case DESC_RATEVHT1SS_MCS6:
+			rate_idx = 6;
+			break;
+		case DESC_RATEVHT1SS_MCS7:
+			rate_idx = 7;
+			break;
+		case DESC_RATEVHT1SS_MCS8:
+			rate_idx = 8;
+			break;
+		case DESC_RATEVHT1SS_MCS9:
+			rate_idx = 9;
+			break;
+		case DESC_RATEVHT2SS_MCS0:
+			rate_idx = 0;
+			break;
+		case DESC_RATEVHT2SS_MCS1:
+			rate_idx = 1;
+			break;
+		case DESC_RATEVHT2SS_MCS2:
+			rate_idx = 2;
+			break;
+		case DESC_RATEVHT2SS_MCS3:
+			rate_idx = 3;
+			break;
+		case DESC_RATEVHT2SS_MCS4:
+			rate_idx = 4;
+			break;
+		case DESC_RATEVHT2SS_MCS5:
+			rate_idx = 5;
+			break;
+		case DESC_RATEVHT2SS_MCS6:
+			rate_idx = 6;
+			break;
+		case DESC_RATEVHT2SS_MCS7:
+			rate_idx = 7;
+			break;
+		case DESC_RATEVHT2SS_MCS8:
+			rate_idx = 8;
+			break;
+		case DESC_RATEVHT2SS_MCS9:
+			rate_idx = 9;
+			break;
+		default:
+			rate_idx = 0;
+			break;
+		}
+		return rate_idx;
+	}
+	if (!isht) {
+		if (hw->conf.chandef.chan->band == NL80211_BAND_2GHZ) {
+			switch (desc_rate) {
+			case DESC_RATE1M:
+				rate_idx = 0;
+				break;
+			case DESC_RATE2M:
+				rate_idx = 1;
+				break;
+			case DESC_RATE5_5M:
+				rate_idx = 2;
+				break;
+			case DESC_RATE11M:
+				rate_idx = 3;
+				break;
+			case DESC_RATE6M:
+				rate_idx = 4;
+				break;
+			case DESC_RATE9M:
+				rate_idx = 5;
+				break;
+			case DESC_RATE12M:
+				rate_idx = 6;
+				break;
+			case DESC_RATE18M:
+				rate_idx = 7;
+				break;
+			case DESC_RATE24M:
+				rate_idx = 8;
+				break;
+			case DESC_RATE36M:
+				rate_idx = 9;
+				break;
+			case DESC_RATE48M:
+				rate_idx = 10;
+				break;
+			case DESC_RATE54M:
+				rate_idx = 11;
+				break;
+			default:
+				rate_idx = 0;
+				break;
+			}
+		} else {
+			switch (desc_rate) {
+			case DESC_RATE6M:
+				rate_idx = 0;
+				break;
+			case DESC_RATE9M:
+				rate_idx = 1;
+				break;
+			case DESC_RATE12M:
+				rate_idx = 2;
+				break;
+			case DESC_RATE18M:
+				rate_idx = 3;
+				break;
+			case DESC_RATE24M:
+				rate_idx = 4;
+				break;
+			case DESC_RATE36M:
+				rate_idx = 5;
+				break;
+			case DESC_RATE48M:
+				rate_idx = 6;
+				break;
+			case DESC_RATE54M:
+				rate_idx = 7;
+				break;
+			default:
+				rate_idx = 0;
+				break;
+			}
+		}
+	} else {
+		switch (desc_rate) {
+		case DESC_RATEMCS0:
+			rate_idx = 0;
+			break;
+		case DESC_RATEMCS1:
+			rate_idx = 1;
+			break;
+		case DESC_RATEMCS2:
+			rate_idx = 2;
+			break;
+		case DESC_RATEMCS3:
+			rate_idx = 3;
+			break;
+		case DESC_RATEMCS4:
+			rate_idx = 4;
+			break;
+		case DESC_RATEMCS5:
+			rate_idx = 5;
+			break;
+		case DESC_RATEMCS6:
+			rate_idx = 6;
+			break;
+		case DESC_RATEMCS7:
+			rate_idx = 7;
+			break;
+		case DESC_RATEMCS8:
+			rate_idx = 8;
+			break;
+		case DESC_RATEMCS9:
+			rate_idx = 9;
+			break;
+		case DESC_RATEMCS10:
+			rate_idx = 10;
+			break;
+		case DESC_RATEMCS11:
+			rate_idx = 11;
+			break;
+		case DESC_RATEMCS12:
+			rate_idx = 12;
+			break;
+		case DESC_RATEMCS13:
+			rate_idx = 13;
+			break;
+		case DESC_RATEMCS14:
+			rate_idx = 14;
+			break;
+		case DESC_RATEMCS15:
+			rate_idx = 15;
+			break;
+		default:
+			rate_idx = 0;
+			break;
+		}
+	}
+	return rate_idx;
+}
+
+static u8 _rtl_get_tx_hw_rate(struct ieee80211_hw *hw,
+			      struct ieee80211_tx_info *info)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct ieee80211_tx_rate *r = &info->status.rates[0];
+	struct ieee80211_rate *txrate;
+	u8 hw_value = 0x0;
+
+	if (r->flags & IEEE80211_TX_RC_MCS) {
+		/* HT MCS0-15 */
+		hw_value = rtlpriv->cfg->maps[RTL_RC_HT_RATEMCS15] - 15 +
+			   r->idx;
+	} else if (r->flags & IEEE80211_TX_RC_VHT_MCS) {
+		/* VHT MCS0-9, NSS */
+		if (ieee80211_rate_get_vht_nss(r) == 2)
+			hw_value = rtlpriv->cfg->maps[RTL_RC_VHT_RATE_2SS_MCS9];
+		else
+			hw_value = rtlpriv->cfg->maps[RTL_RC_VHT_RATE_1SS_MCS9];
+
+		hw_value = hw_value - 9 + ieee80211_rate_get_vht_mcs(r);
+	} else {
+		/* legacy */
+		txrate = ieee80211_get_tx_rate(hw, info);
+
+		if (txrate)
+			hw_value = txrate->hw_value;
+	}
+
+	/* check 5G band */
+	if (rtlpriv->rtlhal.current_bandtype == BAND_ON_5G &&
+	    hw_value < rtlpriv->cfg->maps[RTL_RC_OFDM_RATE6M])
+		hw_value = rtlpriv->cfg->maps[RTL_RC_OFDM_RATE6M];
+
+	return hw_value;
+}
+
+void rtl_get_tcb_desc(struct ieee80211_hw *hw,
+		      struct ieee80211_tx_info *info,
+		      struct ieee80211_sta *sta,
+		      struct sk_buff *skb, struct rtl_tcb_desc *tcb_desc)
+{
+#define SET_RATE_ID(rate_id)					\
+	((rtlpriv->cfg->spec_ver & RTL_SPEC_NEW_RATEID) ?	\
+		rtl_mrate_idx_to_arfr_id(hw, rate_id,		\
+			(sta_entry ? sta_entry->wireless_mode :	\
+			 WIRELESS_MODE_G)) :			\
+		rate_id)
+
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *rtlmac = rtl_mac(rtl_priv(hw));
+	struct ieee80211_hdr *hdr = rtl_get_hdr(skb);
+	struct rtl_sta_info *sta_entry =
+		(sta ? (struct rtl_sta_info *)sta->drv_priv : NULL);
+
+	__le16 fc = rtl_get_fc(skb);
+
+	tcb_desc->hw_rate = _rtl_get_tx_hw_rate(hw, info);
+
+	if (rtl_is_tx_report_skb(hw, skb))
+		tcb_desc->use_spe_rpt = 1;
+
+	if (ieee80211_is_data(fc)) {
+		/*
+		 *we set data rate INX 0
+		 *in rtl_rc.c   if skb is special data or
+		 *mgt which need low data rate.
+		 */
+
+		/*
+		 *So tcb_desc->hw_rate is just used for
+		 *special data and mgt frames
+		 */
+		if (info->control.rates[0].idx == 0 ||
+		    ieee80211_is_nullfunc(fc)) {
+			tcb_desc->use_driver_rate = true;
+			tcb_desc->ratr_index =
+					SET_RATE_ID(RATR_INX_WIRELESS_MC);
+
+			tcb_desc->disable_ratefallback = 1;
+		} else {
+			/* because hw will never use hw_rate
+			 * when tcb_desc->use_driver_rate = false
+			 * so we never set highest N rate here,
+			 * and N rate will all be controlled by FW
+			 * when tcb_desc->use_driver_rate = false
+			 */
+			if (sta && sta->vht_cap.vht_supported) {
+				tcb_desc->hw_rate =
+				_rtl_get_vht_highest_n_rate(hw, sta);
+			} else {
+				if (sta && (sta->ht_cap.ht_supported)) {
+					tcb_desc->hw_rate =
+					    _rtl_get_highest_n_rate(hw, sta);
+				} else {
+					if (rtlmac->mode == WIRELESS_MODE_B) {
+						tcb_desc->hw_rate =
+						    rtlpriv->cfg->maps[RTL_RC_CCK_RATE11M];
+					} else {
+						tcb_desc->hw_rate =
+						    rtlpriv->cfg->maps[RTL_RC_OFDM_RATE54M];
+					}
+				}
+			}
+		}
+
+		if (is_multicast_ether_addr(hdr->addr1))
+			tcb_desc->multicast = 1;
+		else if (is_broadcast_ether_addr(hdr->addr1))
+			tcb_desc->broadcast = 1;
+
+		_rtl_txrate_selectmode(hw, sta, tcb_desc);
+		_rtl_query_bandwidth_mode(hw, sta, tcb_desc);
+		_rtl_qurey_shortpreamble_mode(hw, tcb_desc, info);
+		_rtl_query_shortgi(hw, sta, tcb_desc, info);
+		_rtl_query_protection_mode(hw, tcb_desc, info);
+	} else {
+		tcb_desc->use_driver_rate = true;
+		tcb_desc->ratr_index = SET_RATE_ID(RATR_INX_WIRELESS_MC);
+		tcb_desc->disable_ratefallback = 1;
+		tcb_desc->mac_id = 0;
+		tcb_desc->packet_bw = false;
+	}
+#undef SET_RATE_ID
+}
+
+bool rtl_tx_mgmt_proc(struct ieee80211_hw *hw, struct sk_buff *skb)
+{
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	__le16 fc = rtl_get_fc(skb);
+
+	if (rtlpriv->dm.supp_phymode_switch &&
+	    mac->link_state < MAC80211_LINKED &&
+	    (ieee80211_is_auth(fc) || ieee80211_is_probe_req(fc))) {
+		if (rtlpriv->cfg->ops->chk_switch_dmdp)
+			rtlpriv->cfg->ops->chk_switch_dmdp(hw);
+	}
+	if (ieee80211_is_auth(fc)) {
+		RT_TRACE(rtlpriv, COMP_SEND, DBG_DMESG, "MAC80211_LINKING\n");
+
+		mac->link_state = MAC80211_LINKING;
+		/* Dul mac */
+		rtlpriv->phy.need_iqk = true;
+	}
+	return true;
+}
+
+struct sk_buff *rtl_make_del_ba(struct ieee80211_hw *hw, u8 *sa,
+				u8 *bssid, u16 tid);
+
+static void process_agg_start(struct ieee80211_hw *hw,
+			      struct ieee80211_hdr *hdr, u16 tid)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct ieee80211_rx_status rx_status = { 0 };
+	struct sk_buff *skb_delba = NULL;
+
+	skb_delba = rtl_make_del_ba(hw, hdr->addr2, hdr->addr3, tid);
+	if (skb_delba) {
+		rx_status.freq = hw->conf.chandef.chan->center_freq;
+		rx_status.band = hw->conf.chandef.chan->band;
+		rx_status.flag |= RX_FLAG_DECRYPTED;
+		rx_status.flag |= RX_FLAG_MACTIME_START;
+		rx_status.rate_idx = 0;
+		rx_status.signal = 50 + 10;
+		memcpy(IEEE80211_SKB_RXCB(skb_delba),
+		       &rx_status, sizeof(rx_status));
+		RT_PRINT_DATA(rtlpriv, COMP_INIT, DBG_DMESG,
+			      "fake del\n",
+			      skb_delba->data,
+			      skb_delba->len);
+		ieee80211_rx_irqsafe(hw, skb_delba);
+	}
+}
+
+bool rtl_action_proc(struct ieee80211_hw *hw, struct sk_buff *skb, u8 is_tx)
+{
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct ieee80211_hdr *hdr = rtl_get_hdr(skb);
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	__le16 fc = rtl_get_fc(skb);
+	u8 *act = (u8 *)(((u8 *)skb->data + MAC80211_3ADDR_LEN));
+	u8 category;
+
+	if (!ieee80211_is_action(fc))
+		return true;
+
+	category = *act;
+	act++;
+	switch (category) {
+	case ACT_CAT_BA:
+		switch (*act) {
+		case ACT_ADDBAREQ:
+			if (mac->act_scanning)
+				return false;
+
+			RT_TRACE(rtlpriv, (COMP_SEND | COMP_RECV), DBG_DMESG,
+				 "%s ACT_ADDBAREQ From :%pM\n",
+				 is_tx ? "Tx" : "Rx", hdr->addr2);
+			RT_PRINT_DATA(rtlpriv, COMP_INIT, DBG_DMESG, "req\n",
+				      skb->data, skb->len);
+			if (!is_tx) {
+				struct ieee80211_sta *sta = NULL;
+				struct rtl_sta_info *sta_entry = NULL;
+				struct rtl_tid_data *tid_data;
+				struct ieee80211_mgmt *mgmt = (void *)skb->data;
+				u16 capab = 0, tid = 0;
+
+				rcu_read_lock();
+				sta = rtl_find_sta(hw, hdr->addr3);
+				if (!sta) {
+					RT_TRACE(rtlpriv, COMP_SEND | COMP_RECV,
+						 DBG_DMESG, "sta is NULL\n");
+					rcu_read_unlock();
+					return true;
+				}
+
+				sta_entry =
+					(struct rtl_sta_info *)sta->drv_priv;
+				if (!sta_entry) {
+					rcu_read_unlock();
+					return true;
+				}
+				capab =
+				  le16_to_cpu(mgmt->u.action.u.addba_req.capab);
+				tid = (capab &
+				       IEEE80211_ADDBA_PARAM_TID_MASK) >> 2;
+				if (tid >= MAX_TID_COUNT) {
+					rcu_read_unlock();
+					return true;
+				}
+				tid_data = &sta_entry->tids[tid];
+				if (tid_data->agg.rx_agg_state ==
+				    RTL_RX_AGG_START)
+					process_agg_start(hw, hdr, tid);
+				rcu_read_unlock();
+			}
+			break;
+		case ACT_ADDBARSP:
+			RT_TRACE(rtlpriv, (COMP_SEND | COMP_RECV), DBG_DMESG,
+				 "%s ACT_ADDBARSP From :%pM\n",
+				  is_tx ? "Tx" : "Rx", hdr->addr2);
+			break;
+		case ACT_DELBA:
+			RT_TRACE(rtlpriv, (COMP_SEND | COMP_RECV), DBG_DMESG,
+				 "ACT_ADDBADEL From :%pM\n", hdr->addr2);
+			break;
+		}
+		break;
+	default:
+		break;
+	}
+
+	return true;
+}
+
+static void setup_special_tx(struct rtl_priv *rtlpriv, struct rtl_ps_ctl *ppsc,
+			     int type)
+{
+	struct ieee80211_hw *hw = rtlpriv->hw;
+
+	rtlpriv->ra.is_special_data = true;
+	if (rtlpriv->cfg->ops->get_btc_status())
+		rtlpriv->btcoexist.btc_ops->btc_special_packet_notify(
+					rtlpriv, type);
+	rtl_lps_leave(hw);
+	ppsc->last_delaylps_stamp_jiffies = jiffies;
+}
+
+static const u8 *rtl_skb_ether_type_ptr(struct ieee80211_hw *hw,
+					struct sk_buff *skb, bool is_enc)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 mac_hdr_len = ieee80211_get_hdrlen_from_skb(skb);
+	u8 encrypt_header_len = 0;
+	u8 offset;
+
+	switch (rtlpriv->sec.pairwise_enc_algorithm) {
+	case WEP40_ENCRYPTION:
+	case WEP104_ENCRYPTION:
+		encrypt_header_len = 4;/*WEP_IV_LEN*/
+		break;
+	case TKIP_ENCRYPTION:
+		encrypt_header_len = 8;/*TKIP_IV_LEN*/
+		break;
+	case AESCCMP_ENCRYPTION:
+		encrypt_header_len = 8;/*CCMP_HDR_LEN;*/
+		break;
+	default:
+		break;
+	}
+
+	offset = mac_hdr_len + SNAP_SIZE;
+	if (is_enc)
+		offset += encrypt_header_len;
+
+	return skb->data + offset;
+}
+
+/*should call before software enc*/
+u8 rtl_is_special_data(struct ieee80211_hw *hw, struct sk_buff *skb, u8 is_tx,
+		       bool is_enc)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	__le16 fc = rtl_get_fc(skb);
+	u16 ether_type;
+	const u8 *ether_type_ptr;
+	const struct iphdr *ip;
+
+	if (!ieee80211_is_data(fc))
+		goto end;
+
+	ether_type_ptr = rtl_skb_ether_type_ptr(hw, skb, is_enc);
+	ether_type = be16_to_cpup((__be16 *)ether_type_ptr);
+
+	if (ether_type == ETH_P_IP) {
+		ip = (struct iphdr *)((u8 *)ether_type_ptr +
+		     PROTOC_TYPE_SIZE);
+		if (ip->protocol == IPPROTO_UDP) {
+			struct udphdr *udp = (struct udphdr *)((u8 *)ip +
+							       (ip->ihl << 2));
+			if (((((u8 *)udp)[1] == 68) &&
+			     (((u8 *)udp)[3] == 67)) ||
+			    ((((u8 *)udp)[1] == 67) &&
+			     (((u8 *)udp)[3] == 68))) {
+				/* 68 : UDP BOOTP client
+				 * 67 : UDP BOOTP server
+				 */
+				RT_TRACE(rtlpriv, (COMP_SEND | COMP_RECV),
+					 DBG_DMESG, "dhcp %s !!\n",
+					 (is_tx) ? "Tx" : "Rx");
+
+				if (is_tx)
+					setup_special_tx(rtlpriv, ppsc,
+							 PACKET_DHCP);
+
+				return true;
+			}
+		}
+	} else if (ether_type == ETH_P_ARP) {
+		if (is_tx)
+			setup_special_tx(rtlpriv, ppsc, PACKET_ARP);
+
+		return true;
+	} else if (ether_type == ETH_P_PAE) {
+		/* EAPOL is seen as in-4way */
+		rtlpriv->btcoexist.btc_info.in_4way = true;
+		rtlpriv->btcoexist.btc_info.in_4way_ts = jiffies;
+		rtlpriv->btcoexist.btc_info.in_4way_ts = jiffies;
+
+		RT_TRACE(rtlpriv, (COMP_SEND | COMP_RECV), DBG_DMESG,
+			 "802.1X %s EAPOL pkt!!\n", (is_tx) ? "Tx" : "Rx");
+
+		if (is_tx) {
+			rtlpriv->ra.is_special_data = true;
+			rtl_lps_leave(hw);
+			ppsc->last_delaylps_stamp_jiffies = jiffies;
+
+			setup_special_tx(rtlpriv, ppsc, PACKET_EAPOL);
+		}
+
+		return true;
+	} else if (ether_type == ETH_P_IPV6) {
+		/* TODO: Handle any IPv6 cases that need special handling.
+		 * For now, always return false
+		 */
+		goto end;
+	}
+
+end:
+	rtlpriv->ra.is_special_data = false;
+	return false;
+}
+
+bool rtl_is_tx_report_skb(struct ieee80211_hw *hw, struct sk_buff *skb)
+{
+	u16 ether_type;
+	const u8 *ether_type_ptr;
+
+	ether_type_ptr = rtl_skb_ether_type_ptr(hw, skb, true);
+	ether_type = be16_to_cpup((__be16 *)ether_type_ptr);
+
+	/* EAPOL */
+	if (ether_type == ETH_P_PAE)
+		return true;
+
+	return false;
+}
+
+static u16 rtl_get_tx_report_sn(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_tx_report *tx_report = &rtlpriv->tx_report;
+	u16 sn;
+
+	/*
+	 * SW_DEFINE[11:8] are reserved (driver fills zeros)
+	 * SW_DEFINE[7:2] are used by driver
+	 * SW_DEFINE[1:0] are reserved for firmware (driver fills zeros)
+	 */
+	sn = (atomic_inc_return(&tx_report->sn) & 0x003F) << 2;
+
+	tx_report->last_sent_sn = sn;
+	tx_report->last_sent_time = jiffies;
+
+	RT_TRACE(rtlpriv, COMP_TX_REPORT, DBG_DMESG,
+		 "Send TX-Report sn=0x%X\n", sn);
+
+	return sn;
+}
+
+void rtl_get_tx_report(struct rtl_tcb_desc *ptcb_desc, u8 *pdesc,
+		       struct ieee80211_hw *hw)
+{
+	if (ptcb_desc->use_spe_rpt) {
+		u16 sn = rtl_get_tx_report_sn(hw);
+
+		SET_TX_DESC_SPE_RPT(pdesc, 1);
+		SET_TX_DESC_SW_DEFINE(pdesc, sn);
+	}
+}
+
+void rtl_tx_report_handler(struct ieee80211_hw *hw, u8 *tmp_buf, u8 c2h_cmd_len)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_tx_report *tx_report = &rtlpriv->tx_report;
+	u16 sn;
+	u8 st, retry;
+
+	if (rtlpriv->cfg->spec_ver & RTL_SPEC_NEW_FW_C2H) {
+		sn = tmp_buf[6];
+		st = tmp_buf[7] & 0xC0;
+		retry = tmp_buf[8] & 0x3F;
+	} else {
+		sn = ((tmp_buf[7] & 0x0F) << 8) | tmp_buf[6];
+		st = tmp_buf[0] & 0xC0;
+		retry = tmp_buf[2] & 0x3F;
+	}
+
+	tx_report->last_recv_sn = sn;
+
+	RT_TRACE(rtlpriv, COMP_TX_REPORT, DBG_DMESG,
+		 "Recv TX-Report st=0x%02X sn=0x%X retry=0x%X\n",
+		 st, sn, retry);
+}
+
+bool rtl_check_tx_report_acked(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_tx_report *tx_report = &rtlpriv->tx_report;
+
+	if (tx_report->last_sent_sn == tx_report->last_recv_sn)
+		return true;
+
+	if (time_before(tx_report->last_sent_time + 3 * HZ, jiffies)) {
+		RT_TRACE(rtlpriv, COMP_TX_REPORT, DBG_WARNING,
+			 "Check TX-Report timeout!! s_sn=0x%X r_sn=0x%X\n",
+			 tx_report->last_sent_sn, tx_report->last_recv_sn);
+		return true;	/* 3 sec. (timeout) seen as acked */
+	}
+
+	return false;
+}
+
+void rtl_wait_tx_report_acked(struct ieee80211_hw *hw, u32 wait_ms)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	int i;
+
+	for (i = 0; i < wait_ms; i++) {
+		if (rtl_check_tx_report_acked(hw))
+			break;
+		usleep_range(1000, 2000);
+		RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG,
+			 "Wait 1ms (%d/%d) to disable key.\n", i, wait_ms);
+	}
+}
+
+u32 rtl_get_hal_edca_param(struct ieee80211_hw *hw,
+			   struct ieee80211_vif *vif,
+			   enum wireless_mode wirelessmode,
+			   struct ieee80211_tx_queue_params *param)
+{
+	u32 reg = 0;
+	u8 sifstime = 10;
+	u8 slottime = 20;
+
+	/* AIFS = AIFSN * slot time + SIFS */
+	switch (wirelessmode) {
+	case WIRELESS_MODE_A:
+	case WIRELESS_MODE_N_24G:
+	case WIRELESS_MODE_N_5G:
+	case WIRELESS_MODE_AC_5G:
+	case WIRELESS_MODE_AC_24G:
+		sifstime = 16;
+		slottime = 9;
+		break;
+	case WIRELESS_MODE_G:
+		slottime = (vif->bss_conf.use_short_slot ? 9 : 20);
+		break;
+	default:
+		break;
+	}
+
+	reg |= (param->txop & 0x7FF) << 16;
+	reg |= (fls(param->cw_max) & 0xF) << 12;
+	reg |= (fls(param->cw_min) & 0xF) << 8;
+	reg |= (param->aifs & 0x0F) * slottime + sifstime;
+
+	return reg;
+}
+
+/*********************************************************
+ *
+ * functions called by core.c
+ *
+ *********************************************************/
+int rtl_tx_agg_start(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+		     struct ieee80211_sta *sta, u16 tid, u16 *ssn)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_tid_data *tid_data;
+	struct rtl_sta_info *sta_entry = NULL;
+
+	if (!sta)
+		return -EINVAL;
+
+	if (unlikely(tid >= MAX_TID_COUNT))
+		return -EINVAL;
+
+	sta_entry = (struct rtl_sta_info *)sta->drv_priv;
+	if (!sta_entry)
+		return -ENXIO;
+	tid_data = &sta_entry->tids[tid];
+
+	RT_TRACE(rtlpriv, COMP_SEND, DBG_DMESG,
+		 "on ra = %pM tid = %d seq:%d\n", sta->addr, tid,
+		 tid_data->seq_number);
+
+	*ssn = tid_data->seq_number;
+	tid_data->agg.agg_state = RTL_AGG_START;
+
+	ieee80211_start_tx_ba_cb_irqsafe(vif, sta->addr, tid);
+	return 0;
+}
+
+int rtl_tx_agg_stop(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+		    struct ieee80211_sta *sta, u16 tid)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_tid_data *tid_data;
+	struct rtl_sta_info *sta_entry = NULL;
+
+	if (!sta)
+		return -EINVAL;
+
+	RT_TRACE(rtlpriv, COMP_SEND, DBG_DMESG,
+		 "on ra = %pM tid = %d\n", sta->addr, tid);
+
+	if (unlikely(tid >= MAX_TID_COUNT))
+		return -EINVAL;
+
+	sta_entry = (struct rtl_sta_info *)sta->drv_priv;
+	tid_data = &sta_entry->tids[tid];
+	sta_entry->tids[tid].agg.agg_state = RTL_AGG_STOP;
+
+	ieee80211_stop_tx_ba_cb_irqsafe(vif, sta->addr, tid);
+	return 0;
+}
+
+int rtl_rx_agg_start(struct ieee80211_hw *hw,
+		     struct ieee80211_sta *sta, u16 tid)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_tid_data *tid_data;
+	struct rtl_sta_info *sta_entry = NULL;
+	u8 reject_agg;
+
+	if (!sta)
+		return -EINVAL;
+
+	if (unlikely(tid >= MAX_TID_COUNT))
+		return -EINVAL;
+
+	if (rtlpriv->cfg->ops->get_btc_status()) {
+		rtlpriv->btcoexist.btc_ops->btc_get_ampdu_cfg(rtlpriv,
+							      &reject_agg,
+							      NULL, NULL);
+		if (reject_agg)
+			return -EINVAL;
+	}
+
+	sta_entry = (struct rtl_sta_info *)sta->drv_priv;
+	if (!sta_entry)
+		return -ENXIO;
+	tid_data = &sta_entry->tids[tid];
+
+	RT_TRACE(rtlpriv, COMP_RECV, DBG_DMESG,
+		 "on ra = %pM tid = %d seq:%d\n", sta->addr, tid,
+		 tid_data->seq_number);
+
+	tid_data->agg.rx_agg_state = RTL_RX_AGG_START;
+	return 0;
+}
+
+int rtl_rx_agg_stop(struct ieee80211_hw *hw,
+		    struct ieee80211_sta *sta, u16 tid)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_sta_info *sta_entry = NULL;
+
+	if (!sta)
+		return -EINVAL;
+
+	RT_TRACE(rtlpriv, COMP_SEND, DBG_DMESG,
+		 "on ra = %pM tid = %d\n", sta->addr, tid);
+
+	if (unlikely(tid >= MAX_TID_COUNT))
+		return -EINVAL;
+
+	sta_entry = (struct rtl_sta_info *)sta->drv_priv;
+	sta_entry->tids[tid].agg.rx_agg_state = RTL_RX_AGG_STOP;
+
+	return 0;
+}
+
+int rtl_tx_agg_oper(struct ieee80211_hw *hw,
+		    struct ieee80211_sta *sta, u16 tid)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_sta_info *sta_entry = NULL;
+
+	if (!sta)
+		return -EINVAL;
+
+	RT_TRACE(rtlpriv, COMP_SEND, DBG_DMESG,
+		 "on ra = %pM tid = %d\n", sta->addr, tid);
+
+	if (unlikely(tid >= MAX_TID_COUNT))
+		return -EINVAL;
+
+	sta_entry = (struct rtl_sta_info *)sta->drv_priv;
+	sta_entry->tids[tid].agg.agg_state = RTL_AGG_OPERATIONAL;
+
+	return 0;
+}
+
+void rtl_rx_ampdu_apply(struct rtl_priv *rtlpriv)
+{
+	struct rtl_btc_ops *btc_ops = rtlpriv->btcoexist.btc_ops;
+	u8 reject_agg = 0, ctrl_agg_size = 0, agg_size = 0;
+
+	if (rtlpriv->cfg->ops->get_btc_status())
+		btc_ops->btc_get_ampdu_cfg(rtlpriv, &reject_agg,
+					   &ctrl_agg_size, &agg_size);
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_DMESG,
+		 "Set RX AMPDU: coex - reject=%d, ctrl_agg_size=%d, size=%d",
+		 reject_agg, ctrl_agg_size, agg_size);
+
+	rtlpriv->hw->max_rx_aggregation_subframes =
+		(ctrl_agg_size ? agg_size : IEEE80211_MAX_AMPDU_BUF);
+}
+
+/*********************************************************
+ *
+ * wq & timer callback functions
+ *
+ *********************************************************/
+/* this function is used for roaming */
+void rtl_beacon_statistic(struct ieee80211_hw *hw, struct sk_buff *skb)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
+
+	if (rtlpriv->mac80211.opmode != NL80211_IFTYPE_STATION)
+		return;
+
+	if (rtlpriv->mac80211.link_state < MAC80211_LINKED)
+		return;
+
+	/* check if this really is a beacon */
+	if (!ieee80211_is_beacon(hdr->frame_control) &&
+	    !ieee80211_is_probe_resp(hdr->frame_control))
+		return;
+
+	/* min. beacon length + FCS_LEN */
+	if (skb->len <= 40 + FCS_LEN)
+		return;
+
+	/* and only beacons from the associated BSSID, please */
+	if (!ether_addr_equal(hdr->addr3, rtlpriv->mac80211.bssid))
+		return;
+
+	rtlpriv->link_info.bcn_rx_inperiod++;
+}
+
+static void rtl_free_entries_from_scan_list(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_bssid_entry *entry, *next;
+
+	list_for_each_entry_safe(entry, next, &rtlpriv->scan_list.list, list) {
+		list_del(&entry->list);
+		kfree(entry);
+		rtlpriv->scan_list.num--;
+	}
+}
+
+void rtl_scan_list_expire(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_bssid_entry *entry, *next;
+	unsigned long flags;
+
+	spin_lock_irqsave(&rtlpriv->locks.scan_list_lock, flags);
+
+	list_for_each_entry_safe(entry, next, &rtlpriv->scan_list.list, list) {
+		/* 180 seconds */
+		if (jiffies_to_msecs(jiffies - entry->age) < 180000)
+			continue;
+
+		list_del(&entry->list);
+		rtlpriv->scan_list.num--;
+
+		RT_TRACE(rtlpriv, COMP_SCAN, DBG_LOUD,
+			 "BSSID=%pM is expire in scan list (total=%d)\n",
+			 entry->bssid, rtlpriv->scan_list.num);
+		kfree(entry);
+	}
+
+	spin_unlock_irqrestore(&rtlpriv->locks.scan_list_lock, flags);
+
+	rtlpriv->btcoexist.btc_info.ap_num = rtlpriv->scan_list.num;
+}
+
+void rtl_collect_scan_list(struct ieee80211_hw *hw, struct sk_buff *skb)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	unsigned long flags;
+
+	struct rtl_bssid_entry *entry;
+	bool entry_found = false;
+
+	/* check if it is scanning */
+	if (!mac->act_scanning)
+		return;
+
+	/* check if this really is a beacon */
+	if (!ieee80211_is_beacon(hdr->frame_control) &&
+	    !ieee80211_is_probe_resp(hdr->frame_control))
+		return;
+
+	spin_lock_irqsave(&rtlpriv->locks.scan_list_lock, flags);
+
+	list_for_each_entry(entry, &rtlpriv->scan_list.list, list) {
+		if (memcmp(entry->bssid, hdr->addr3, ETH_ALEN) == 0) {
+			list_del_init(&entry->list);
+			entry_found = true;
+			RT_TRACE(rtlpriv, COMP_SCAN, DBG_LOUD,
+				 "Update BSSID=%pM to scan list (total=%d)\n",
+				 hdr->addr3, rtlpriv->scan_list.num);
+			break;
+		}
+	}
+
+	if (!entry_found) {
+		entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
+
+		if (!entry)
+			goto label_err;
+
+		memcpy(entry->bssid, hdr->addr3, ETH_ALEN);
+		rtlpriv->scan_list.num++;
+
+		RT_TRACE(rtlpriv, COMP_SCAN, DBG_LOUD,
+			 "Add BSSID=%pM to scan list (total=%d)\n",
+			 hdr->addr3, rtlpriv->scan_list.num);
+	}
+
+	entry->age = jiffies;
+
+	list_add_tail(&entry->list, &rtlpriv->scan_list.list);
+
+label_err:
+	spin_unlock_irqrestore(&rtlpriv->locks.scan_list_lock, flags);
+}
+
+void rtl_watchdog_wq_callback(void *data)
+{
+	struct rtl_works *rtlworks = container_of_dwork_rtl(data,
+							    struct rtl_works,
+							    watchdog_wq);
+	struct ieee80211_hw *hw = rtlworks->hw;
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	bool busytraffic = false;
+	bool tx_busy_traffic = false;
+	bool rx_busy_traffic = false;
+	bool higher_busytraffic = false;
+	bool higher_busyrxtraffic = false;
+	u8 idx, tid;
+	u32 rx_cnt_inp4eriod = 0;
+	u32 tx_cnt_inp4eriod = 0;
+	u32 aver_rx_cnt_inperiod = 0;
+	u32 aver_tx_cnt_inperiod = 0;
+	u32 aver_tidtx_inperiod[MAX_TID_COUNT] = {0};
+	u32 tidtx_inp4eriod[MAX_TID_COUNT] = {0};
+
+	if (is_hal_stop(rtlhal))
+		return;
+
+	/* <1> Determine if action frame is allowed */
+	if (mac->link_state > MAC80211_NOLINK) {
+		if (mac->cnt_after_linked < 20)
+			mac->cnt_after_linked++;
+	} else {
+		mac->cnt_after_linked = 0;
+	}
+
+	/* <2> to check if traffic busy, if
+	 * busytraffic we don't change channel
+	 */
+	if (mac->link_state >= MAC80211_LINKED) {
+		/* (1) get aver_rx_cnt_inperiod & aver_tx_cnt_inperiod */
+		for (idx = 0; idx <= 2; idx++) {
+			rtlpriv->link_info.num_rx_in4period[idx] =
+			    rtlpriv->link_info.num_rx_in4period[idx + 1];
+			rtlpriv->link_info.num_tx_in4period[idx] =
+			    rtlpriv->link_info.num_tx_in4period[idx + 1];
+		}
+		rtlpriv->link_info.num_rx_in4period[3] =
+		    rtlpriv->link_info.num_rx_inperiod;
+		rtlpriv->link_info.num_tx_in4period[3] =
+		    rtlpriv->link_info.num_tx_inperiod;
+		for (idx = 0; idx <= 3; idx++) {
+			rx_cnt_inp4eriod +=
+			    rtlpriv->link_info.num_rx_in4period[idx];
+			tx_cnt_inp4eriod +=
+			    rtlpriv->link_info.num_tx_in4period[idx];
+		}
+		aver_rx_cnt_inperiod = rx_cnt_inp4eriod / 4;
+		aver_tx_cnt_inperiod = tx_cnt_inp4eriod / 4;
+
+		/* (2) check traffic busy */
+		if (aver_rx_cnt_inperiod > 100 || aver_tx_cnt_inperiod > 100) {
+			busytraffic = true;
+			if (aver_rx_cnt_inperiod > aver_tx_cnt_inperiod)
+				rx_busy_traffic = true;
+			else
+				tx_busy_traffic = false;
+		}
+
+		/* Higher Tx/Rx data. */
+		if (aver_rx_cnt_inperiod > 4000 ||
+		    aver_tx_cnt_inperiod > 4000) {
+			higher_busytraffic = true;
+
+			/* Extremely high Rx data. */
+			if (aver_rx_cnt_inperiod > 5000)
+				higher_busyrxtraffic = true;
+		}
+
+		/* check every tid's tx traffic */
+		for (tid = 0; tid <= 7; tid++) {
+			for (idx = 0; idx <= 2; idx++)
+				rtlpriv->link_info.tidtx_in4period[tid][idx] =
+					rtlpriv->link_info.tidtx_in4period[tid]
+					[idx + 1];
+			rtlpriv->link_info.tidtx_in4period[tid][3] =
+				rtlpriv->link_info.tidtx_inperiod[tid];
+
+			for (idx = 0; idx <= 3; idx++)
+				tidtx_inp4eriod[tid] +=
+				   rtlpriv->link_info.tidtx_in4period[tid][idx];
+			aver_tidtx_inperiod[tid] = tidtx_inp4eriod[tid] / 4;
+			if (aver_tidtx_inperiod[tid] > 5000)
+				rtlpriv->link_info.higher_busytxtraffic[tid] =
+									true;
+			else
+				rtlpriv->link_info.higher_busytxtraffic[tid] =
+									false;
+		}
+
+		/* PS is controlled by coex. */
+		if (rtlpriv->cfg->ops->get_btc_status() &&
+		    rtlpriv->btcoexist.btc_ops->btc_is_bt_ctrl_lps(rtlpriv))
+			goto label_lps_done;
+
+		if (((rtlpriv->link_info.num_rx_inperiod +
+		      rtlpriv->link_info.num_tx_inperiod) > 8) ||
+		    (rtlpriv->link_info.num_rx_inperiod > 2))
+			rtl_lps_leave(hw);
+		else
+			rtl_lps_enter(hw);
+
+label_lps_done:
+		;
+	}
+
+	rtlpriv->link_info.num_rx_inperiod = 0;
+	rtlpriv->link_info.num_tx_inperiod = 0;
+	for (tid = 0; tid <= 7; tid++)
+		rtlpriv->link_info.tidtx_inperiod[tid] = 0;
+
+	rtlpriv->link_info.busytraffic = busytraffic;
+	rtlpriv->link_info.higher_busytraffic = higher_busytraffic;
+	rtlpriv->link_info.rx_busy_traffic = rx_busy_traffic;
+	rtlpriv->link_info.tx_busy_traffic = tx_busy_traffic;
+	rtlpriv->link_info.higher_busyrxtraffic = higher_busyrxtraffic;
+
+	rtlpriv->stats.txbytesunicast_inperiod =
+		rtlpriv->stats.txbytesunicast -
+		rtlpriv->stats.txbytesunicast_last;
+	rtlpriv->stats.rxbytesunicast_inperiod =
+		rtlpriv->stats.rxbytesunicast -
+		rtlpriv->stats.rxbytesunicast_last;
+	rtlpriv->stats.txbytesunicast_last = rtlpriv->stats.txbytesunicast;
+	rtlpriv->stats.rxbytesunicast_last = rtlpriv->stats.rxbytesunicast;
+
+	rtlpriv->stats.txbytesunicast_inperiod_tp =
+		(u32)(rtlpriv->stats.txbytesunicast_inperiod * 8 / 2 /
+		1024 / 1024);
+	rtlpriv->stats.rxbytesunicast_inperiod_tp =
+		(u32)(rtlpriv->stats.rxbytesunicast_inperiod * 8 / 2 /
+		1024 / 1024);
+
+	/* <3> DM */
+	if (!rtlpriv->cfg->mod_params->disable_watchdog)
+		rtlpriv->cfg->ops->dm_watchdog(hw);
+
+	/* <4> roaming */
+	if (mac->link_state == MAC80211_LINKED &&
+	    mac->opmode == NL80211_IFTYPE_STATION) {
+		if ((rtlpriv->link_info.bcn_rx_inperiod +
+		    rtlpriv->link_info.num_rx_inperiod) == 0) {
+			rtlpriv->link_info.roam_times++;
+			RT_TRACE(rtlpriv, COMP_ERR, DBG_DMESG,
+				 "AP off for %d s\n",
+				(rtlpriv->link_info.roam_times * 2));
+
+			/* if we can't recv beacon for 10s,
+			 * we should reconnect this AP
+			 */
+			if (rtlpriv->link_info.roam_times >= 5) {
+				pr_err("AP off, try to reconnect now\n");
+				rtlpriv->link_info.roam_times = 0;
+				ieee80211_connection_loss(
+					rtlpriv->mac80211.vif);
+			}
+		} else {
+			rtlpriv->link_info.roam_times = 0;
+		}
+	}
+
+	if (rtlpriv->cfg->ops->get_btc_status())
+		rtlpriv->btcoexist.btc_ops->btc_periodical(rtlpriv);
+
+	if (rtlpriv->btcoexist.btc_info.in_4way) {
+		if (time_after(jiffies, rtlpriv->btcoexist.btc_info.in_4way_ts +
+			       msecs_to_jiffies(IN_4WAY_TIMEOUT_TIME)))
+			rtlpriv->btcoexist.btc_info.in_4way = false;
+	}
+
+	rtlpriv->link_info.bcn_rx_inperiod = 0;
+
+	/* <6> scan list */
+	rtl_scan_list_expire(hw);
+}
+
+void rtl_watch_dog_timer_callback(unsigned long data)
+{
+	struct ieee80211_hw *hw = (struct ieee80211_hw *)data;
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	queue_delayed_work(rtlpriv->works.rtl_wq,
+			   &rtlpriv->works.watchdog_wq, 0);
+
+	mod_timer(&rtlpriv->works.watchdog_timer,
+		  jiffies + MSECS(RTL_WATCH_DOG_TIME));
+}
+
+void rtl_fwevt_wq_callback(void *data)
+{
+	struct rtl_works *rtlworks =
+		container_of_dwork_rtl(data, struct rtl_works, fwevt_wq);
+	struct ieee80211_hw *hw = rtlworks->hw;
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	rtlpriv->cfg->ops->c2h_command_handle(hw);
+}
+
+void rtl_c2hcmd_enqueue(struct ieee80211_hw *hw, u8 tag, u8 len, u8 *val)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	unsigned long flags;
+	struct rtl_c2hcmd *c2hcmd;
+
+	c2hcmd = kmalloc(sizeof(*c2hcmd),
+			 in_interrupt() ? GFP_ATOMIC : GFP_KERNEL);
+
+	if (!c2hcmd)
+		goto label_err;
+
+	c2hcmd->val = kmalloc(len,
+			      in_interrupt() ? GFP_ATOMIC : GFP_KERNEL);
+
+	if (!c2hcmd->val)
+		goto label_err2;
+
+	/* fill data */
+	c2hcmd->tag = tag;
+	c2hcmd->len = len;
+	memcpy(c2hcmd->val, val, len);
+
+	/* enqueue */
+	spin_lock_irqsave(&rtlpriv->locks.c2hcmd_lock, flags);
+
+	list_add_tail(&c2hcmd->list, &rtlpriv->c2hcmd_list);
+
+	spin_unlock_irqrestore(&rtlpriv->locks.c2hcmd_lock, flags);
+
+	/* wake up wq */
+	queue_delayed_work(rtlpriv->works.rtl_wq, &rtlpriv->works.c2hcmd_wq, 0);
+
+	return;
+
+label_err2:
+	kfree(c2hcmd);
+
+label_err:
+	RT_TRACE(rtlpriv, COMP_CMD, DBG_WARNING,
+		 "C2H cmd enqueue fail.\n");
+}
+
+void rtl_c2hcmd_launcher(struct ieee80211_hw *hw, int exec)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	unsigned long flags;
+	struct rtl_c2hcmd *c2hcmd;
+	int i;
+
+	for (i = 0; i < 200; i++) {
+		/* dequeue a task */
+		spin_lock_irqsave(&rtlpriv->locks.c2hcmd_lock, flags);
+
+		c2hcmd = list_first_entry_or_null(&rtlpriv->c2hcmd_list,
+						  struct rtl_c2hcmd, list);
+
+		if (c2hcmd)
+			list_del(&c2hcmd->list);
+
+		spin_unlock_irqrestore(&rtlpriv->locks.c2hcmd_lock, flags);
+
+		/* do it */
+		if (!c2hcmd)
+			break;
+
+		if (rtlpriv->cfg->ops->c2h_content_parsing && exec)
+			rtlpriv->cfg->ops->c2h_content_parsing(hw,
+					c2hcmd->tag, c2hcmd->len, c2hcmd->val);
+
+		/* free */
+		kfree(c2hcmd->val);
+
+		kfree(c2hcmd);
+	}
+}
+
+void rtl_c2hcmd_wq_callback(void *data)
+{
+	struct rtl_works *rtlworks = container_of_dwork_rtl(data,
+							    struct rtl_works,
+							    c2hcmd_wq);
+	struct ieee80211_hw *hw = rtlworks->hw;
+
+	rtl_c2hcmd_launcher(hw, 1);
+}
+
+void rtl_easy_concurrent_retrytimer_callback(unsigned long data)
+{
+	struct ieee80211_hw *hw = (struct ieee80211_hw *)data;
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_priv *buddy_priv = rtlpriv->buddy_priv;
+
+	if (!buddy_priv)
+		return;
+
+	rtlpriv->cfg->ops->dualmac_easy_concurrent(hw);
+}
+
+/*********************************************************
+ *
+ * frame process functions
+ *
+ *********************************************************/
+u8 *rtl_find_ie(u8 *data, unsigned int len, u8 ie)
+{
+	struct ieee80211_mgmt *mgmt = (void *)data;
+	u8 *pos, *end;
+
+	pos = (u8 *)mgmt->u.beacon.variable;
+	end = data + len;
+	while (pos < end) {
+		if (pos + 2 + pos[1] > end)
+			return NULL;
+
+		if (pos[0] == ie)
+			return pos;
+
+		pos += 2 + pos[1];
+	}
+	return NULL;
+}
+
+/* when we use 2 rx ants we send IEEE80211_SMPS_OFF */
+/* when we use 1 rx ant we send IEEE80211_SMPS_STATIC */
+static struct sk_buff *rtl_make_smps_action(struct ieee80211_hw *hw,
+					    enum ieee80211_smps_mode smps,
+					    u8 *da, u8 *bssid)
+{
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
+	struct sk_buff *skb;
+	struct ieee80211_mgmt *action_frame;
+
+	/* 27 = header + category + action + smps mode */
+	skb = dev_alloc_skb(27 + hw->extra_tx_headroom);
+	if (!skb)
+		return NULL;
+
+	skb_reserve(skb, hw->extra_tx_headroom);
+	action_frame = skb_put_zero(skb, 27);
+	memcpy(action_frame->da, da, ETH_ALEN);
+	memcpy(action_frame->sa, rtlefuse->dev_addr, ETH_ALEN);
+	memcpy(action_frame->bssid, bssid, ETH_ALEN);
+	action_frame->frame_control = cpu_to_le16(IEEE80211_FTYPE_MGMT |
+						  IEEE80211_STYPE_ACTION);
+	action_frame->u.action.category = WLAN_CATEGORY_HT;
+	action_frame->u.action.u.ht_smps.action = WLAN_HT_ACTION_SMPS;
+	switch (smps) {
+	case IEEE80211_SMPS_AUTOMATIC:/* 0 */
+	case IEEE80211_SMPS_NUM_MODES:/* 4 */
+		WARN_ON(1);
+	/* Here will get a 'MISSING_BREAK' in Coverity Test, just ignore it.
+	 * According to Kernel Code, here is right.
+	 */
+	case IEEE80211_SMPS_OFF:/* 1 */ /*MIMO_PS_NOLIMIT*/
+		action_frame->u.action.u.ht_smps.smps_control =
+				WLAN_HT_SMPS_CONTROL_DISABLED;/* 0 */
+		break;
+	case IEEE80211_SMPS_STATIC:/* 2 */ /*MIMO_PS_STATIC*/
+		action_frame->u.action.u.ht_smps.smps_control =
+				WLAN_HT_SMPS_CONTROL_STATIC;/* 1 */
+		break;
+	case IEEE80211_SMPS_DYNAMIC:/* 3 */ /*MIMO_PS_DYNAMIC*/
+		action_frame->u.action.u.ht_smps.smps_control =
+				WLAN_HT_SMPS_CONTROL_DYNAMIC;/* 3 */
+		break;
+	}
+
+	return skb;
+}
+
+int rtl_send_smps_action(struct ieee80211_hw *hw,
+			 struct ieee80211_sta *sta,
+			 enum ieee80211_smps_mode smps)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	struct sk_buff *skb = NULL;
+	struct rtl_tcb_desc tcb_desc;
+	u8 bssid[ETH_ALEN] = {0};
+
+	memset(&tcb_desc, 0, sizeof(struct rtl_tcb_desc));
+
+	if (rtlpriv->mac80211.act_scanning)
+		goto err_free;
+
+	if (!sta)
+		goto err_free;
+
+	if (unlikely(is_hal_stop(rtlhal) || ppsc->rfpwr_state != ERFON))
+		goto err_free;
+
+	if (!test_bit(RTL_STATUS_INTERFACE_START, &rtlpriv->status))
+		goto err_free;
+
+	if (rtlpriv->mac80211.opmode == NL80211_IFTYPE_AP)
+		memcpy(bssid, rtlpriv->efuse.dev_addr, ETH_ALEN);
+	else
+		memcpy(bssid, rtlpriv->mac80211.bssid, ETH_ALEN);
+
+	skb = rtl_make_smps_action(hw, smps, sta->addr, bssid);
+	/* this is a type = mgmt * stype = action frame */
+	if (skb) {
+		struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+		struct rtl_sta_info *sta_entry =
+			(struct rtl_sta_info *)sta->drv_priv;
+		sta_entry->mimo_ps = smps;
+		/* rtlpriv->cfg->ops->update_rate_tbl(hw, sta, 0, true); */
+
+		info->control.rates[0].idx = 0;
+		info->band = hw->conf.chandef.chan->band;
+		rtlpriv->intf_ops->adapter_tx(hw, sta, skb, &tcb_desc);
+	}
+	return 1;
+
+err_free:
+	return 0;
+}
+
+void rtl_phy_scan_operation_backup(struct ieee80211_hw *hw, u8 operation)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	enum io_type iotype;
+
+	if (!is_hal_stop(rtlhal)) {
+		switch (operation) {
+		case SCAN_OPT_BACKUP:
+			iotype = IO_CMD_PAUSE_DM_BY_SCAN;
+			rtlpriv->cfg->ops->set_hw_reg(hw,
+						      HW_VAR_IO_CMD,
+						      (u8 *)&iotype);
+			break;
+		case SCAN_OPT_RESTORE:
+			iotype = IO_CMD_RESUME_DM_BY_SCAN;
+			rtlpriv->cfg->ops->set_hw_reg(hw,
+						      HW_VAR_IO_CMD,
+						      (u8 *)&iotype);
+			break;
+		default:
+			pr_err("Unknown Scan Backup operation.\n");
+			break;
+		}
+	}
+}
+
+/* because mac80211 have issues when can receive del ba
+ * so here we just make a fake del_ba if we receive a ba_req
+ * but rx_agg was opened to let mac80211 release some ba
+ * related resources, so please this del_ba for tx
+ */
+struct sk_buff *rtl_make_del_ba(struct ieee80211_hw *hw,
+				u8 *sa, u8 *bssid, u16 tid)
+{
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
+	struct sk_buff *skb;
+	struct ieee80211_mgmt *action_frame;
+	u16 params;
+
+	/* 27 = header + category + action + smps mode */
+	skb = dev_alloc_skb(34 + hw->extra_tx_headroom);
+	if (!skb)
+		return NULL;
+
+	skb_reserve(skb, hw->extra_tx_headroom);
+	action_frame = skb_put_zero(skb, 34);
+	memcpy(action_frame->sa, sa, ETH_ALEN);
+	memcpy(action_frame->da, rtlefuse->dev_addr, ETH_ALEN);
+	memcpy(action_frame->bssid, bssid, ETH_ALEN);
+	action_frame->frame_control = cpu_to_le16(IEEE80211_FTYPE_MGMT |
+						  IEEE80211_STYPE_ACTION);
+	action_frame->u.action.category = WLAN_CATEGORY_BACK;
+	action_frame->u.action.u.delba.action_code = WLAN_ACTION_DELBA;
+	params = (u16)(1 << 11);	/* bit 11 initiator */
+	params |= (u16)(tid << 12);	/* bit 15:12 TID number */
+
+	action_frame->u.action.u.delba.params = cpu_to_le16(params);
+	action_frame->u.action.u.delba.reason_code =
+		cpu_to_le16(WLAN_REASON_QSTA_TIMEOUT);
+
+	return skb;
+}
+
+bool rtl_check_beacon_key(struct ieee80211_hw *hw, void *data, unsigned int len)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	struct ieee80211_hdr *hdr = data;
+	struct ieee80211_ht_cap *ht_cap_ie;
+	struct ieee80211_ht_operation *ht_oper_ie = NULL;
+	struct rtl_beacon_keys bcn_key = {};
+	struct rtl_beacon_keys *cur_bcn_key;
+	u8 *ht_cap;
+	u8 ht_cap_len;
+	u8 *ht_oper;
+	u8 ht_oper_len;
+	u8 *ds_param;
+	u8 ds_param_len;
+
+	if (mac->opmode != NL80211_IFTYPE_STATION)
+		return false;
+
+	/* check if this really is a beacon*/
+	if (!ieee80211_is_beacon(hdr->frame_control))
+		return false;
+
+	/* min. beacon length + FCS_LEN */
+	if (len <= 40 + FCS_LEN)
+		return false;
+
+	cur_bcn_key = &mac->cur_beacon_keys;
+
+	if (rtlpriv->mac80211.link_state == MAC80211_NOLINK) {
+		if (cur_bcn_key->valid) {
+			cur_bcn_key->valid = false;
+			RT_TRACE(rtlpriv, COMP_BEACON, DBG_LOUD,
+				 "Reset cur_beacon_keys.valid to false!\n");
+		}
+		return false;
+	}
+
+	/* and only beacons from the associated BSSID, please */
+	if (!ether_addr_equal(hdr->addr3, rtlpriv->mac80211.bssid))
+		return false;
+
+	/***** Parsing DS Param IE ******/
+	ds_param = rtl_find_ie(data, len - FCS_LEN, WLAN_EID_DS_PARAMS);
+
+	if (ds_param && !(ds_param[1] < sizeof(*ds_param))) {
+		ds_param_len = ds_param[1];
+		bcn_key.bcn_channel = ds_param[2];
+	} else {
+		ds_param = NULL;
+	}
+
+	/***** Parsing HT Cap. IE ******/
+	ht_cap = rtl_find_ie(data, len - FCS_LEN, WLAN_EID_HT_CAPABILITY);
+
+	if (ht_cap && !(ht_cap[1] < sizeof(*ht_cap))) {
+		ht_cap_len = ht_cap[1];
+		ht_cap_ie = (struct ieee80211_ht_cap *)&ht_cap[2];
+		bcn_key.ht_cap_info = ht_cap_ie->cap_info;
+	} else  {
+		ht_cap = NULL;
+	}
+
+	/***** Parsing HT Info. IE ******/
+	ht_oper = rtl_find_ie(data, len - FCS_LEN, WLAN_EID_HT_OPERATION);
+
+	if (ht_oper && !(ht_oper[1] < sizeof(*ht_oper))) {
+		ht_oper_len = ht_oper[1];
+		ht_oper_ie = (struct ieee80211_ht_operation *)&ht_oper[2];
+	} else {
+		ht_oper = NULL;
+	}
+
+	/* update bcn_key */
+
+	if (!ds_param && ht_oper && ht_oper_ie)
+		bcn_key.bcn_channel = ht_oper_ie->primary_chan;
+
+	if (ht_oper && ht_oper_ie)
+		bcn_key.ht_info_infos_0_sco = ht_oper_ie->ht_param & 0x03;
+
+	bcn_key.valid = true;
+
+	/* update cur_beacon_keys or compare beacon key */
+	if ((rtlpriv->mac80211.link_state != MAC80211_LINKED) &&
+	    (rtlpriv->mac80211.link_state != MAC80211_LINKED_SCANNING))
+		return true;
+
+	if (!cur_bcn_key->valid) {
+		/* update cur_beacon_keys */
+		memcpy(cur_bcn_key, &bcn_key, sizeof(bcn_key));
+		cur_bcn_key->valid = true;
+
+		RT_TRACE(rtlpriv, COMP_BEACON, DBG_LOUD,
+			 "Beacon key update!ch=%d, ht_cap_info=0x%x, sco=0x%x\n",
+			 cur_bcn_key->bcn_channel,
+			 cur_bcn_key->ht_cap_info,
+			 cur_bcn_key->ht_info_infos_0_sco);
+		return true;
+	}
+
+	/* compare beacon key */
+	if (!memcmp(cur_bcn_key, &bcn_key, sizeof(bcn_key))) {
+		/* same beacon key */
+		mac->new_beacon_cnt = 0;
+		goto chk_exit;
+	}
+
+	if ((cur_bcn_key->bcn_channel == bcn_key.bcn_channel) &&
+	    (cur_bcn_key->ht_cap_info == bcn_key.ht_cap_info)) {
+		/* Beacon HT info IE, secondary channel offset check */
+		/* 40M -> 20M */
+		if (cur_bcn_key->ht_info_infos_0_sco >
+		    bcn_key.ht_info_infos_0_sco) {
+			/* Not a new beacon */
+			RT_TRACE(rtlpriv, COMP_BEACON, DBG_DMESG,
+				 "Beacon BW change! sco:0x%x -> 0x%x\n",
+				 cur_bcn_key->ht_info_infos_0_sco,
+				 bcn_key.ht_info_infos_0_sco);
+
+			cur_bcn_key->ht_info_infos_0_sco =
+					bcn_key.ht_info_infos_0_sco;
+		} else {
+			/* 20M -> 40M */
+			if (rtlphy->max_ht_chan_bw >= HT_CHANNEL_WIDTH_20_40) {
+				/* Not a new beacon */
+				RT_TRACE(rtlpriv, COMP_BEACON, DBG_DMESG,
+					 "Beacon BW change! sco:0x%x -> 0x%x\n",
+					 cur_bcn_key->ht_info_infos_0_sco,
+					 bcn_key.ht_info_infos_0_sco);
+
+				cur_bcn_key->ht_info_infos_0_sco =
+					bcn_key.ht_info_infos_0_sco;
+			} else {
+				mac->new_beacon_cnt++;
+			}
+		}
+	} else {
+		mac->new_beacon_cnt++;
+	}
+
+	if (mac->new_beacon_cnt == 1) {
+		RT_TRACE(rtlpriv, COMP_BEACON, DBG_DMESG,
+			 "Get new beacon.\n");
+		RT_TRACE(rtlpriv, COMP_BEACON, DBG_DMESG,
+			 "Cur : ch=%d, ht_cap=0x%x, sco=0x%x\n",
+			 cur_bcn_key->bcn_channel,
+			 cur_bcn_key->ht_cap_info,
+			 cur_bcn_key->ht_info_infos_0_sco);
+		RT_TRACE(rtlpriv, COMP_BEACON, DBG_DMESG,
+			 "New RX : ch=%d, ht_cap=0x%x, sco=0x%x\n",
+			 bcn_key.bcn_channel,
+			 bcn_key.ht_cap_info,
+			 bcn_key.ht_info_infos_0_sco);
+
+	} else if (mac->new_beacon_cnt > 1) {
+		RT_TRACE(rtlpriv, COMP_BEACON, DBG_DMESG,
+			 "new beacon cnt: %d\n",
+			 mac->new_beacon_cnt);
+	}
+
+	if (mac->new_beacon_cnt > 3) {
+		ieee80211_connection_loss(rtlpriv->mac80211.vif);
+		RT_TRACE(rtlpriv, COMP_BEACON, DBG_DMESG,
+			 "new beacon cnt >3, disconnect !\n");
+	}
+
+chk_exit:
+
+	return true;
+}
+
+/*********************************************************
+ *
+ * IOT functions
+ *
+ *********************************************************/
+static bool rtl_chk_vendor_ouisub(struct ieee80211_hw *hw,
+				  struct octet_string vendor_ie)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	bool matched = false;
+	static u8 athcap_1[] = { 0x00, 0x03, 0x7F };
+	static u8 athcap_2[] = { 0x00, 0x13, 0x74 };
+	static u8 broadcap_1[] = { 0x00, 0x10, 0x18 };
+	static u8 broadcap_2[] = { 0x00, 0x0a, 0xf7 };
+	static u8 broadcap_3[] = { 0x00, 0x05, 0xb5 };
+	static u8 racap[] = { 0x00, 0x0c, 0x43 };
+	static u8 ciscocap[] = { 0x00, 0x40, 0x96 };
+	static u8 marvcap[] = { 0x00, 0x50, 0x43 };
+
+	if (memcmp(vendor_ie.octet, athcap_1, 3) == 0 ||
+	    memcmp(vendor_ie.octet, athcap_2, 3) == 0) {
+		rtlpriv->mac80211.vendor = PEER_ATH;
+		matched = true;
+	} else if (memcmp(vendor_ie.octet, broadcap_1, 3) == 0 ||
+		   memcmp(vendor_ie.octet, broadcap_2, 3) == 0 ||
+		   memcmp(vendor_ie.octet, broadcap_3, 3) == 0) {
+		rtlpriv->mac80211.vendor = PEER_BROAD;
+		matched = true;
+	} else if (memcmp(vendor_ie.octet, racap, 3) == 0) {
+		rtlpriv->mac80211.vendor = PEER_RAL;
+		matched = true;
+	} else if (memcmp(vendor_ie.octet, ciscocap, 3) == 0) {
+		rtlpriv->mac80211.vendor = PEER_CISCO;
+		matched = true;
+	} else if (memcmp(vendor_ie.octet, marvcap, 3) == 0) {
+		rtlpriv->mac80211.vendor = PEER_MARV;
+		matched = true;
+	}
+
+	return matched;
+}
+
+static bool rtl_find_221_ie(struct ieee80211_hw *hw, u8 *data,
+			    unsigned int len)
+{
+	struct ieee80211_mgmt *mgmt = (void *)data;
+	struct octet_string vendor_ie;
+	u8 *pos, *end;
+
+	pos = (u8 *)mgmt->u.beacon.variable;
+	end = data + len;
+	while (pos < end) {
+		if (pos[0] == 221) {
+			vendor_ie.length = pos[1];
+			vendor_ie.octet = &pos[2];
+			if (rtl_chk_vendor_ouisub(hw, vendor_ie))
+				return true;
+		}
+
+		if (pos + 2 + pos[1] > end)
+			return false;
+
+		pos += 2 + pos[1];
+	}
+	return false;
+}
+
+void rtl_recognize_peer(struct ieee80211_hw *hw, u8 *data, unsigned int len)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct ieee80211_hdr *hdr = (void *)data;
+	u32 vendor = PEER_UNKNOWN;
+
+	static u8 ap3_1[3] = { 0x00, 0x14, 0xbf };
+	static u8 ap3_2[3] = { 0x00, 0x1a, 0x70 };
+	static u8 ap3_3[3] = { 0x00, 0x1d, 0x7e };
+	static u8 ap4_1[3] = { 0x00, 0x90, 0xcc };
+	static u8 ap4_2[3] = { 0x00, 0x0e, 0x2e };
+	static u8 ap4_3[3] = { 0x00, 0x18, 0x02 };
+	static u8 ap4_4[3] = { 0x00, 0x17, 0x3f };
+	static u8 ap4_5[3] = { 0x00, 0x1c, 0xdf };
+	static u8 ap5_1[3] = { 0x00, 0x1c, 0xf0 };
+	static u8 ap5_2[3] = { 0x00, 0x21, 0x91 };
+	static u8 ap5_3[3] = { 0x00, 0x24, 0x01 };
+	static u8 ap5_4[3] = { 0x00, 0x15, 0xe9 };
+	static u8 ap5_5[3] = { 0x00, 0x17, 0x9A };
+	static u8 ap5_6[3] = { 0x00, 0x18, 0xE7 };
+	static u8 ap6_1[3] = { 0x00, 0x17, 0x94 };
+	static u8 ap7_1[3] = { 0x00, 0x14, 0xa4 };
+
+	if (mac->opmode != NL80211_IFTYPE_STATION)
+		return;
+
+	if (mac->link_state == MAC80211_NOLINK) {
+		mac->vendor = PEER_UNKNOWN;
+		return;
+	}
+
+	if (mac->cnt_after_linked > 2)
+		return;
+
+	/* check if this really is a beacon */
+	if (!ieee80211_is_beacon(hdr->frame_control))
+		return;
+
+	/* min. beacon length + FCS_LEN */
+	if (len <= 40 + FCS_LEN)
+		return;
+
+	/* and only beacons from the associated BSSID, please */
+	if (!ether_addr_equal_64bits(hdr->addr3, rtlpriv->mac80211.bssid))
+		return;
+
+	if (rtl_find_221_ie(hw, data, len))
+		vendor = mac->vendor;
+
+	if ((memcmp(mac->bssid, ap5_1, 3) == 0) ||
+	    (memcmp(mac->bssid, ap5_2, 3) == 0) ||
+	    (memcmp(mac->bssid, ap5_3, 3) == 0) ||
+	    (memcmp(mac->bssid, ap5_4, 3) == 0) ||
+	    (memcmp(mac->bssid, ap5_5, 3) == 0) ||
+	    (memcmp(mac->bssid, ap5_6, 3) == 0) ||
+	    vendor == PEER_ATH) {
+		vendor = PEER_ATH;
+		RT_TRACE(rtlpriv, COMP_MAC80211, DBG_LOUD, "=>ath find\n");
+	} else if ((memcmp(mac->bssid, ap4_4, 3) == 0) ||
+		(memcmp(mac->bssid, ap4_5, 3) == 0) ||
+		(memcmp(mac->bssid, ap4_1, 3) == 0) ||
+		(memcmp(mac->bssid, ap4_2, 3) == 0) ||
+		(memcmp(mac->bssid, ap4_3, 3) == 0) ||
+		vendor == PEER_RAL) {
+		RT_TRACE(rtlpriv, COMP_MAC80211, DBG_LOUD, "=>ral find\n");
+		vendor = PEER_RAL;
+	} else if (memcmp(mac->bssid, ap6_1, 3) == 0 ||
+		vendor == PEER_CISCO) {
+		vendor = PEER_CISCO;
+		RT_TRACE(rtlpriv, COMP_MAC80211, DBG_LOUD, "=>cisco find\n");
+	} else if ((memcmp(mac->bssid, ap3_1, 3) == 0) ||
+		(memcmp(mac->bssid, ap3_2, 3) == 0) ||
+		(memcmp(mac->bssid, ap3_3, 3) == 0) ||
+		vendor == PEER_BROAD) {
+		RT_TRACE(rtlpriv, COMP_MAC80211, DBG_LOUD, "=>broad find\n");
+		vendor = PEER_BROAD;
+	} else if (memcmp(mac->bssid, ap7_1, 3) == 0 ||
+		vendor == PEER_MARV) {
+		vendor = PEER_MARV;
+		RT_TRACE(rtlpriv, COMP_MAC80211, DBG_LOUD, "=>marv find\n");
+	}
+
+	mac->vendor = vendor;
+}
+
+MODULE_AUTHOR("lizhaoming	<chaoming_li@realsil.com.cn>");
+MODULE_AUTHOR("Realtek WlanFAE	<wlanfae@realtek.com>");
+MODULE_AUTHOR("Larry Finger	<Larry.FInger@lwfinger.net>");
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Realtek 802.11n PCI wireless core");
+
+struct rtl_global_var rtl_global_var = {};
+
+int rtl_core_module_init(void)
+{
+	if (rtl_rate_control_register())
+		pr_err("rtl: Unable to register rtl_rc, use default RC !!\n");
+
+	/* add debugfs */
+	rtl_debugfs_add_topdir();
+
+	/* init some global vars */
+	INIT_LIST_HEAD(&rtl_global_var.glb_priv_list);
+	spin_lock_init(&rtl_global_var.glb_list_lock);
+
+	return 0;
+}
+
+void rtl_core_module_exit(void)
+{
+	/*RC*/
+	rtl_rate_control_unregister();
+
+	/* remove debugfs */
+	rtl_debugfs_remove_topdir();
+}
diff --git a/drivers/staging/rtlwifi/base.h b/drivers/staging/rtlwifi/base.h
new file mode 100644
index 000000000000..1829712dc4e2
--- /dev/null
+++ b/drivers/staging/rtlwifi/base.h
@@ -0,0 +1,186 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2012  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __RTL_BASE_H__
+#define __RTL_BASE_H__
+
+enum ap_peer {
+	PEER_UNKNOWN = 0,
+	PEER_RTL = 1,
+	PEER_RTL_92SE = 2,
+	PEER_BROAD = 3,
+	PEER_RAL = 4,
+	PEER_ATH = 5,
+	PEER_CISCO = 6,
+	PEER_MARV = 7,
+	PEER_AIRGO = 9,
+	PEER_MAX = 10,
+};
+
+#define RTL_DUMMY_OFFSET	0
+#define RTL_DUMMY_UNIT		8
+#define RTL_TX_DUMMY_SIZE	(RTL_DUMMY_OFFSET * RTL_DUMMY_UNIT)
+#define RTL_TX_DESC_SIZE	32
+#define RTL_TX_HEADER_SIZE	(RTL_TX_DESC_SIZE + RTL_TX_DUMMY_SIZE)
+
+#define MAX_BIT_RATE_40MHZ_MCS15	300	/* Mbps */
+#define MAX_BIT_RATE_40MHZ_MCS7		150	/* Mbps */
+
+#define MAX_BIT_RATE_SHORT_GI_2NSS_80MHZ_MCS9	867	/* Mbps */
+#define MAX_BIT_RATE_SHORT_GI_2NSS_80MHZ_MCS7	650	/* Mbps */
+#define MAX_BIT_RATE_LONG_GI_2NSS_80MHZ_MCS9	780	/* Mbps */
+#define MAX_BIT_RATE_LONG_GI_2NSS_80MHZ_MCS7	585	/* Mbps */
+
+#define MAX_BIT_RATE_SHORT_GI_1NSS_80MHZ_MCS9	434	/* Mbps */
+#define MAX_BIT_RATE_SHORT_GI_1NSS_80MHZ_MCS7	325	/* Mbps */
+#define MAX_BIT_RATE_LONG_GI_1NSS_80MHZ_MCS9	390	/* Mbps */
+#define MAX_BIT_RATE_LONG_GI_1NSS_80MHZ_MCS7	293	/* Mbps */
+
+#define FRAME_OFFSET_FRAME_CONTROL	0
+#define FRAME_OFFSET_DURATION		2
+#define FRAME_OFFSET_ADDRESS1		4
+#define FRAME_OFFSET_ADDRESS2		10
+#define FRAME_OFFSET_ADDRESS3		16
+#define FRAME_OFFSET_SEQUENCE		22
+#define FRAME_OFFSET_ADDRESS4		24
+#define MAX_LISTEN_INTERVAL		10
+#define MAX_RATE_TRIES			4
+
+#define SET_80211_HDR_FRAME_CONTROL(_hdr, _val)		\
+	WRITEEF2BYTE(_hdr, _val)
+#define SET_80211_HDR_TYPE_AND_SUBTYPE(_hdr, _val)	\
+	WRITEEF1BYTE(_hdr, _val)
+#define SET_80211_HDR_PWR_MGNT(_hdr, _val)		\
+	SET_BITS_TO_LE_2BYTE(_hdr, 12, 1, _val)
+#define SET_80211_HDR_TO_DS(_hdr, _val)			\
+	SET_BITS_TO_LE_2BYTE(_hdr, 8, 1, _val)
+
+#define SET_80211_PS_POLL_AID(_hdr, _val)		\
+	(*(u16 *)((u8 *)(_hdr) + 2) = _val)
+#define SET_80211_PS_POLL_BSSID(_hdr, _val)		\
+	ether_addr_copy(((u8 *)(_hdr)) + 4, (u8 *)(_val))
+#define SET_80211_PS_POLL_TA(_hdr, _val)		\
+	ether_addr_copy(((u8 *)(_hdr)) + 10, (u8 *)(_val))
+
+#define SET_80211_HDR_DURATION(_hdr, _val)	\
+	(*(u16 *)((u8 *)(_hdr) + FRAME_OFFSET_DURATION) = le16_to_cpu(_val))
+#define SET_80211_HDR_ADDRESS1(_hdr, _val)	\
+	CP_MACADDR((u8 *)(_hdr) + FRAME_OFFSET_ADDRESS1, (u8 *)(_val))
+#define SET_80211_HDR_ADDRESS2(_hdr, _val)	\
+	CP_MACADDR((u8 *)(_hdr) + FRAME_OFFSET_ADDRESS2, (u8 *)(_val))
+#define SET_80211_HDR_ADDRESS3(_hdr, _val)	\
+	CP_MACADDR((u8 *)(_hdr) + FRAME_OFFSET_ADDRESS3, (u8 *)(_val))
+#define SET_80211_HDR_FRAGMENT_SEQUENCE(_hdr, _val)  \
+	WRITEEF2BYTE((u8 *)(_hdr) + FRAME_OFFSET_SEQUENCE, _val)
+
+#define SET_BEACON_PROBE_RSP_TIME_STAMP_LOW(__phdr, __val)	\
+	WRITEEF4BYTE(((u8 *)(__phdr)) + 24, __val)
+#define SET_BEACON_PROBE_RSP_TIME_STAMP_HIGH(__phdr, __val) \
+	WRITEEF4BYTE(((u8 *)(__phdr)) + 28, __val)
+#define SET_BEACON_PROBE_RSP_BEACON_INTERVAL(__phdr, __val) \
+	WRITEEF2BYTE(((u8 *)(__phdr)) + 32, __val)
+#define GET_BEACON_PROBE_RSP_CAPABILITY_INFO(__phdr)		\
+	READEF2BYTE(((u8 *)(__phdr)) + 34)
+#define SET_BEACON_PROBE_RSP_CAPABILITY_INFO(__phdr, __val) \
+	WRITEEF2BYTE(((u8 *)(__phdr)) + 34, __val)
+#define MASK_BEACON_PROBE_RSP_CAPABILITY_INFO(__phdr, __val) \
+	SET_BEACON_PROBE_RSP_CAPABILITY_INFO(__phdr, \
+	(GET_BEACON_PROBE_RSP_CAPABILITY_INFO(__phdr) & (~(__val))))
+
+#define SET_TX_DESC_SPE_RPT(__pdesc, __val)			\
+	SET_BITS_TO_LE_4BYTE((__pdesc) + 8, 19, 1, __val)
+#define SET_TX_DESC_SW_DEFINE(__pdesc, __val)	\
+	SET_BITS_TO_LE_4BYTE((__pdesc) + 24, 0, 12, __val)
+
+int rtl_init_core(struct ieee80211_hw *hw);
+void rtl_deinit_core(struct ieee80211_hw *hw);
+void rtl_init_rx_config(struct ieee80211_hw *hw);
+void rtl_init_rfkill(struct ieee80211_hw *hw);
+void rtl_deinit_rfkill(struct ieee80211_hw *hw);
+
+void rtl_watch_dog_timer_callback(unsigned long data);
+void rtl_deinit_deferred_work(struct ieee80211_hw *hw);
+
+bool rtl_action_proc(struct ieee80211_hw *hw, struct sk_buff *skb, u8 is_tx);
+int rtlwifi_rate_mapping(struct ieee80211_hw *hw, bool isht,
+			 bool isvht, u8 desc_rate);
+bool rtl_tx_mgmt_proc(struct ieee80211_hw *hw, struct sk_buff *skb);
+u8 rtl_is_special_data(struct ieee80211_hw *hw, struct sk_buff *skb, u8 is_tx,
+		       bool is_enc);
+
+bool rtl_is_tx_report_skb(struct ieee80211_hw *hw, struct sk_buff *skb);
+void rtl_get_tx_report(struct rtl_tcb_desc *ptcb_desc, u8 *pdesc,
+		       struct ieee80211_hw *hw);
+void rtl_tx_report_handler(struct ieee80211_hw *hw, u8 *tmp_buf,
+			   u8 c2h_cmd_len);
+bool rtl_check_tx_report_acked(struct ieee80211_hw *hw);
+void rtl_wait_tx_report_acked(struct ieee80211_hw *hw, u32 wait_ms);
+u32 rtl_get_hal_edca_param(struct ieee80211_hw *hw,
+			   struct ieee80211_vif *vif,
+			   enum wireless_mode wirelessmode,
+			   struct ieee80211_tx_queue_params *param);
+
+void rtl_beacon_statistic(struct ieee80211_hw *hw, struct sk_buff *skb);
+void rtl_collect_scan_list(struct ieee80211_hw *hw, struct sk_buff *skb);
+void rtl_scan_list_expire(struct ieee80211_hw *hw);
+int rtl_tx_agg_start(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+		     struct ieee80211_sta *sta, u16 tid, u16 *ssn);
+int rtl_tx_agg_stop(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+		    struct ieee80211_sta *sta, u16 tid);
+int rtl_tx_agg_oper(struct ieee80211_hw *hw,
+		    struct ieee80211_sta *sta, u16 tid);
+int rtl_rx_agg_start(struct ieee80211_hw *hw,
+		     struct ieee80211_sta *sta, u16 tid);
+int rtl_rx_agg_stop(struct ieee80211_hw *hw,
+		    struct ieee80211_sta *sta, u16 tid);
+void rtl_rx_ampdu_apply(struct rtl_priv *rtlpriv);
+void rtl_watchdog_wq_callback(void *data);
+void rtl_fwevt_wq_callback(void *data);
+void rtl_c2hcmd_wq_callback(void *data);
+void rtl_c2hcmd_launcher(struct ieee80211_hw *hw, int exec);
+void rtl_c2hcmd_enqueue(struct ieee80211_hw *hw, u8 tag, u8 len, u8 *val);
+
+u8 rtl_mrate_idx_to_arfr_id(
+	struct ieee80211_hw *hw, u8 rate_index,
+	enum wireless_mode wirelessmode);
+void rtl_get_tcb_desc(struct ieee80211_hw *hw,
+		      struct ieee80211_tx_info *info,
+		      struct ieee80211_sta *sta,
+		      struct sk_buff *skb, struct rtl_tcb_desc *tcb_desc);
+
+int rtl_send_smps_action(struct ieee80211_hw *hw,
+			 struct ieee80211_sta *sta,
+			 enum ieee80211_smps_mode smps);
+u8 *rtl_find_ie(u8 *data, unsigned int len, u8 ie);
+void rtl_recognize_peer(struct ieee80211_hw *hw, u8 *data, unsigned int len);
+u8 rtl_tid_to_ac(u8 tid);
+void rtl_easy_concurrent_retrytimer_callback(unsigned long data);
+extern struct rtl_global_var rtl_global_var;
+void rtl_phy_scan_operation_backup(struct ieee80211_hw *hw, u8 operation);
+bool rtl_check_beacon_key(struct ieee80211_hw *hw, void *data,
+			  unsigned int len);
+int rtl_core_module_init(void);
+void rtl_core_module_exit(void);
+#endif
diff --git a/drivers/staging/rtlwifi/btcoexist/Makefile b/drivers/staging/rtlwifi/btcoexist/Makefile
new file mode 100644
index 000000000000..f600bcc38a15
--- /dev/null
+++ b/drivers/staging/rtlwifi/btcoexist/Makefile
@@ -0,0 +1,8 @@
+btcoexist-objs :=				\
+			halbtc8822b1ant.o	\
+			halbtc8822b2ant.o	\
+			halbtc8822bwifionly.o	\
+			halbtcoutsrc.o		\
+			rtl_btc.o
+
+obj-$(CONFIG_RTLBTCOEXIST) += btcoexist.o
diff --git a/drivers/staging/rtlwifi/btcoexist/halbt_precomp.h b/drivers/staging/rtlwifi/btcoexist/halbt_precomp.h
new file mode 100644
index 000000000000..d78cd9394373
--- /dev/null
+++ b/drivers/staging/rtlwifi/btcoexist/halbt_precomp.h
@@ -0,0 +1,85 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ ******************************************************************************/
+
+#ifndef	__HALBT_PRECOMP_H__
+#define __HALBT_PRECOMP_H__
+/*************************************************************
+ * include files
+ *************************************************************/
+#include "../wifi.h"
+#include "../efuse.h"
+#include "../base.h"
+#include "../regd.h"
+#include "../cam.h"
+#include "../ps.h"
+#include "../pci.h"
+
+#include "halbtcoutsrc.h"
+
+/* Interface type */
+#define RT_PCI_INTERFACE	1
+#define RT_USB_INTERFACE	2
+#define RT_SDIO_INTERFACE	3
+#define DEV_BUS_TYPE		RT_PCI_INTERFACE
+
+#include "halbtc8822b1ant.h"
+#include "halbtc8822b2ant.h"
+#include "halbtc8822bwifionly.h"
+
+#define GETDEFAULTADAPTER(padapter)	padapter
+
+#define BIT0	0x00000001
+#define BIT1	0x00000002
+#define BIT2	0x00000004
+#define BIT3	0x00000008
+#define BIT4	0x00000010
+#define BIT5	0x00000020
+#define BIT6	0x00000040
+#define BIT7	0x00000080
+#define BIT8	0x00000100
+#define BIT9	0x00000200
+#define BIT10	0x00000400
+#define BIT11	0x00000800
+#define BIT12	0x00001000
+#define BIT13	0x00002000
+#define BIT14	0x00004000
+#define BIT15	0x00008000
+#define BIT16	0x00010000
+#define BIT17	0x00020000
+#define BIT18	0x00040000
+#define BIT19	0x00080000
+#define BIT20	0x00100000
+#define BIT21	0x00200000
+#define BIT22	0x00400000
+#define BIT23	0x00800000
+#define BIT24	0x01000000
+#define BIT25	0x02000000
+#define BIT26	0x04000000
+#define BIT27	0x08000000
+#define BIT28	0x10000000
+#define BIT29	0x20000000
+#define BIT30	0x40000000
+#define BIT31	0x80000000
+
+#endif	/* __HALBT_PRECOMP_H__ */
diff --git a/drivers/staging/rtlwifi/btcoexist/halbtc8822b1ant.c b/drivers/staging/rtlwifi/btcoexist/halbtc8822b1ant.c
new file mode 100644
index 000000000000..157395b85405
--- /dev/null
+++ b/drivers/staging/rtlwifi/btcoexist/halbtc8822b1ant.c
@@ -0,0 +1,5244 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+/* ************************************************************
+ * Description:
+ *
+ * This file is for RTL8822B Co-exist mechanism
+ *
+ * History
+ * 2012/11/15 Cosa first check in.
+ *
+ * *************************************************************/
+
+/* ************************************************************
+ * include files
+ * *************************************************************/
+/*only for rf4ce*/
+#include "halbt_precomp.h"
+
+/* ************************************************************
+ * Global variables, these are static variables
+ * *************************************************************/
+static struct coex_dm_8822b_1ant glcoex_dm_8822b_1ant;
+static struct coex_dm_8822b_1ant *coex_dm = &glcoex_dm_8822b_1ant;
+static struct coex_sta_8822b_1ant glcoex_sta_8822b_1ant;
+static struct coex_sta_8822b_1ant *coex_sta = &glcoex_sta_8822b_1ant;
+static struct psdscan_sta_8822b_1ant gl_psd_scan_8822b_1ant;
+static struct psdscan_sta_8822b_1ant *psd_scan = &gl_psd_scan_8822b_1ant;
+static struct rfe_type_8822b_1ant gl_rfe_type_8822b_1ant;
+static struct rfe_type_8822b_1ant *rfe_type = &gl_rfe_type_8822b_1ant;
+
+static const char *const glbt_info_src_8822b_1ant[] = {
+	"BT Info[wifi fw]", "BT Info[bt rsp]", "BT Info[bt auto report]",
+};
+
+static u32 glcoex_ver_date_8822b_1ant = 20170327;
+static u32 glcoex_ver_8822b_1ant = 0x44;
+static u32 glcoex_ver_btdesired_8822b_1ant = 0x42;
+
+/* ************************************************************
+ * local function proto type if needed
+ * ************************************************************
+ * ************************************************************
+ * local function start with halbtc8822b1ant_
+ * *************************************************************/
+
+static u8 halbtc8822b1ant_wifi_rssi_state(struct btc_coexist *btcoexist,
+					  u8 index, u8 level_num,
+					  u8 rssi_thresh, u8 rssi_thresh1)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	s32 wifi_rssi = 0;
+	u8 wifi_rssi_state = coex_sta->pre_wifi_rssi_state[index];
+
+	btcoexist->btc_get(btcoexist, BTC_GET_S4_WIFI_RSSI, &wifi_rssi);
+
+	if (level_num == 2) {
+		if ((coex_sta->pre_wifi_rssi_state[index] ==
+		     BTC_RSSI_STATE_LOW) ||
+		    (coex_sta->pre_wifi_rssi_state[index] ==
+		     BTC_RSSI_STATE_STAY_LOW)) {
+			if (wifi_rssi >=
+			    (rssi_thresh + BTC_RSSI_COEX_THRESH_TOL_8822B_1ANT))
+				wifi_rssi_state = BTC_RSSI_STATE_HIGH;
+			else
+				wifi_rssi_state = BTC_RSSI_STATE_STAY_LOW;
+		} else {
+			if (wifi_rssi < rssi_thresh)
+				wifi_rssi_state = BTC_RSSI_STATE_LOW;
+			else
+				wifi_rssi_state = BTC_RSSI_STATE_STAY_HIGH;
+		}
+	} else if (level_num == 3) {
+		if (rssi_thresh > rssi_thresh1) {
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], wifi RSSI thresh error!!\n");
+			return coex_sta->pre_wifi_rssi_state[index];
+		}
+
+		if ((coex_sta->pre_wifi_rssi_state[index] ==
+		     BTC_RSSI_STATE_LOW) ||
+		    (coex_sta->pre_wifi_rssi_state[index] ==
+		     BTC_RSSI_STATE_STAY_LOW)) {
+			if (wifi_rssi >=
+			    (rssi_thresh + BTC_RSSI_COEX_THRESH_TOL_8822B_1ANT))
+				wifi_rssi_state = BTC_RSSI_STATE_MEDIUM;
+			else
+				wifi_rssi_state = BTC_RSSI_STATE_STAY_LOW;
+		} else if ((coex_sta->pre_wifi_rssi_state[index] ==
+			    BTC_RSSI_STATE_MEDIUM) ||
+			   (coex_sta->pre_wifi_rssi_state[index] ==
+			    BTC_RSSI_STATE_STAY_MEDIUM)) {
+			if (wifi_rssi >= (rssi_thresh1 +
+					  BTC_RSSI_COEX_THRESH_TOL_8822B_1ANT))
+				wifi_rssi_state = BTC_RSSI_STATE_HIGH;
+			else if (wifi_rssi < rssi_thresh)
+				wifi_rssi_state = BTC_RSSI_STATE_LOW;
+			else
+				wifi_rssi_state = BTC_RSSI_STATE_STAY_MEDIUM;
+		} else {
+			if (wifi_rssi < rssi_thresh1)
+				wifi_rssi_state = BTC_RSSI_STATE_MEDIUM;
+			else
+				wifi_rssi_state = BTC_RSSI_STATE_STAY_HIGH;
+		}
+	}
+
+	coex_sta->pre_wifi_rssi_state[index] = wifi_rssi_state;
+
+	return wifi_rssi_state;
+}
+
+static void halbtc8822b1ant_update_ra_mask(struct btc_coexist *btcoexist,
+					   bool force_exec, u32 dis_rate_mask)
+{
+	coex_dm->cur_ra_mask = dis_rate_mask;
+
+	if (force_exec || (coex_dm->pre_ra_mask != coex_dm->cur_ra_mask))
+		btcoexist->btc_set(btcoexist, BTC_SET_ACT_UPDATE_RAMASK,
+				   &coex_dm->cur_ra_mask);
+	coex_dm->pre_ra_mask = coex_dm->cur_ra_mask;
+}
+
+static void
+halbtc8822b1ant_auto_rate_fallback_retry(struct btc_coexist *btcoexist,
+					 bool force_exec, u8 type)
+{
+	bool wifi_under_b_mode = false;
+
+	coex_dm->cur_arfr_type = type;
+
+	if (force_exec || (coex_dm->pre_arfr_type != coex_dm->cur_arfr_type)) {
+		switch (coex_dm->cur_arfr_type) {
+		case 0: /* normal mode */
+			btcoexist->btc_write_4byte(btcoexist, 0x430,
+						   coex_dm->backup_arfr_cnt1);
+			btcoexist->btc_write_4byte(btcoexist, 0x434,
+						   coex_dm->backup_arfr_cnt2);
+			break;
+		case 1:
+			btcoexist->btc_get(btcoexist,
+					   BTC_GET_BL_WIFI_UNDER_B_MODE,
+					   &wifi_under_b_mode);
+			if (wifi_under_b_mode) {
+				btcoexist->btc_write_4byte(btcoexist, 0x430,
+							   0x0);
+				btcoexist->btc_write_4byte(btcoexist, 0x434,
+							   0x01010101);
+			} else {
+				btcoexist->btc_write_4byte(btcoexist, 0x430,
+							   0x0);
+				btcoexist->btc_write_4byte(btcoexist, 0x434,
+							   0x04030201);
+			}
+			break;
+		default:
+			break;
+		}
+	}
+
+	coex_dm->pre_arfr_type = coex_dm->cur_arfr_type;
+}
+
+static void halbtc8822b1ant_retry_limit(struct btc_coexist *btcoexist,
+					bool force_exec, u8 type)
+{
+	coex_dm->cur_retry_limit_type = type;
+
+	if (force_exec ||
+	    (coex_dm->pre_retry_limit_type != coex_dm->cur_retry_limit_type)) {
+		switch (coex_dm->cur_retry_limit_type) {
+		case 0: /* normal mode */
+			btcoexist->btc_write_2byte(btcoexist, 0x42a,
+						   coex_dm->backup_retry_limit);
+			break;
+		case 1: /* retry limit=8 */
+			btcoexist->btc_write_2byte(btcoexist, 0x42a, 0x0808);
+			break;
+		default:
+			break;
+		}
+	}
+
+	coex_dm->pre_retry_limit_type = coex_dm->cur_retry_limit_type;
+}
+
+static void halbtc8822b1ant_ampdu_max_time(struct btc_coexist *btcoexist,
+					   bool force_exec, u8 type)
+{
+	coex_dm->cur_ampdu_time_type = type;
+
+	if (force_exec ||
+	    (coex_dm->pre_ampdu_time_type != coex_dm->cur_ampdu_time_type)) {
+		switch (coex_dm->cur_ampdu_time_type) {
+		case 0: /* normal mode */
+			btcoexist->btc_write_1byte(
+				btcoexist, 0x456,
+				coex_dm->backup_ampdu_max_time);
+			break;
+		case 1: /* AMPDU timw = 0x38 * 32us */
+			btcoexist->btc_write_1byte(btcoexist, 0x456, 0x38);
+			break;
+		default:
+			break;
+		}
+	}
+
+	coex_dm->pre_ampdu_time_type = coex_dm->cur_ampdu_time_type;
+}
+
+static void halbtc8822b1ant_limited_tx(struct btc_coexist *btcoexist,
+				       bool force_exec, u8 ra_mask_type,
+				       u8 arfr_type, u8 retry_limit_type,
+				       u8 ampdu_time_type)
+{
+	switch (ra_mask_type) {
+	case 0: /* normal mode */
+		halbtc8822b1ant_update_ra_mask(btcoexist, force_exec, 0x0);
+		break;
+	case 1: /* disable cck 1/2 */
+		halbtc8822b1ant_update_ra_mask(btcoexist, force_exec,
+					       0x00000003);
+		break;
+	case 2: /* disable cck 1/2/5.5, ofdm 6/9/12/18/24, mcs 0/1/2/3/4 */
+		halbtc8822b1ant_update_ra_mask(btcoexist, force_exec,
+					       0x0001f1f7);
+		break;
+	default:
+		break;
+	}
+
+	halbtc8822b1ant_auto_rate_fallback_retry(btcoexist, force_exec,
+						 arfr_type);
+	halbtc8822b1ant_retry_limit(btcoexist, force_exec, retry_limit_type);
+	halbtc8822b1ant_ampdu_max_time(btcoexist, force_exec, ampdu_time_type);
+}
+
+/*
+ * rx agg size setting :
+ * 1:      true / don't care / don't care
+ * max: false / false / don't care
+ * 7:     false / true / 7
+ */
+
+static void halbtc8822b1ant_limited_rx(struct btc_coexist *btcoexist,
+				       bool force_exec, bool rej_ap_agg_pkt,
+				       bool bt_ctrl_agg_buf_size,
+				       u8 agg_buf_size)
+{
+	bool reject_rx_agg = rej_ap_agg_pkt;
+	bool bt_ctrl_rx_agg_size = bt_ctrl_agg_buf_size;
+	u8 rx_agg_size = agg_buf_size;
+
+	/* ============================================ */
+	/*	Rx Aggregation related setting */
+	/* ============================================ */
+	btcoexist->btc_set(btcoexist, BTC_SET_BL_TO_REJ_AP_AGG_PKT,
+			   &reject_rx_agg);
+	/* decide BT control aggregation buf size or not */
+	btcoexist->btc_set(btcoexist, BTC_SET_BL_BT_CTRL_AGG_SIZE,
+			   &bt_ctrl_rx_agg_size);
+	/* aggregation buf size, only work when BT control Rx aggregation size*/
+	btcoexist->btc_set(btcoexist, BTC_SET_U1_AGG_BUF_SIZE, &rx_agg_size);
+	/* real update aggregation setting */
+	btcoexist->btc_set(btcoexist, BTC_SET_ACT_AGGREGATE_CTRL, NULL);
+}
+
+static void halbtc8822b1ant_query_bt_info(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	u8 h2c_parameter[1] = {0};
+
+	if (coex_sta->bt_disabled) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], No query BT info because BT is disabled!\n");
+		return;
+	}
+
+	h2c_parameter[0] |= BIT(0); /* trigger */
+
+	btcoexist->btc_fill_h2c(btcoexist, 0x61, 1, h2c_parameter);
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], WL query BT info!!\n");
+}
+
+static void halbtc8822b1ant_monitor_bt_ctr(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	u32 reg_hp_txrx, reg_lp_txrx, u32tmp;
+	u32 reg_hp_tx = 0, reg_hp_rx = 0, reg_lp_tx = 0, reg_lp_rx = 0;
+	static u8 num_of_bt_counter_chk, cnt_slave, cnt_autoslot_hang;
+	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
+
+	reg_hp_txrx = 0x770;
+	reg_lp_txrx = 0x774;
+
+	u32tmp = btcoexist->btc_read_4byte(btcoexist, reg_hp_txrx);
+	reg_hp_tx = u32tmp & MASKLWORD;
+	reg_hp_rx = (u32tmp & MASKHWORD) >> 16;
+
+	u32tmp = btcoexist->btc_read_4byte(btcoexist, reg_lp_txrx);
+	reg_lp_tx = u32tmp & MASKLWORD;
+	reg_lp_rx = (u32tmp & MASKHWORD) >> 16;
+
+	coex_sta->high_priority_tx = reg_hp_tx;
+	coex_sta->high_priority_rx = reg_hp_rx;
+	coex_sta->low_priority_tx = reg_lp_tx;
+	coex_sta->low_priority_rx = reg_lp_rx;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], Hi-Pri Rx/Tx: %d/%d, Lo-Pri Rx/Tx: %d/%d\n",
+		 reg_hp_rx, reg_hp_tx, reg_lp_rx, reg_lp_tx);
+
+	/* reset counter */
+	btcoexist->btc_write_1byte(btcoexist, 0x76e, 0xc);
+
+	if ((coex_sta->low_priority_tx > 1150) &&
+	    (!coex_sta->c2h_bt_inquiry_page))
+		coex_sta->pop_event_cnt++;
+
+	if ((coex_sta->low_priority_rx >= 1150) &&
+	    (coex_sta->low_priority_rx >= coex_sta->low_priority_tx) &&
+	    (!coex_sta->under_ips) && (!coex_sta->c2h_bt_inquiry_page) &&
+	    (coex_sta->bt_link_exist)) {
+		if (cnt_slave >= 3) {
+			bt_link_info->slave_role = true;
+			cnt_slave = 3;
+		} else {
+			cnt_slave++;
+		}
+	} else {
+		if (cnt_slave == 0) {
+			bt_link_info->slave_role = false;
+			cnt_slave = 0;
+		} else {
+			cnt_slave--;
+		}
+	}
+
+	if (coex_sta->is_tdma_btautoslot) {
+		if ((coex_sta->low_priority_tx >= 1300) &&
+		    (coex_sta->low_priority_rx <= 150)) {
+			if (cnt_autoslot_hang >= 2) {
+				coex_sta->is_tdma_btautoslot_hang = true;
+				cnt_autoslot_hang = 2;
+			} else {
+				cnt_autoslot_hang++;
+			}
+		} else {
+			if (cnt_autoslot_hang == 0) {
+				coex_sta->is_tdma_btautoslot_hang = false;
+				cnt_autoslot_hang = 0;
+			} else {
+				cnt_autoslot_hang--;
+			}
+		}
+	}
+
+	if (bt_link_info->hid_only) {
+		if (coex_sta->low_priority_rx > 50)
+			coex_sta->is_hid_low_pri_tx_overhead = true;
+		else
+			coex_sta->is_hid_low_pri_tx_overhead = false;
+	}
+
+	if ((coex_sta->high_priority_tx == 0) &&
+	    (coex_sta->high_priority_rx == 0) &&
+	    (coex_sta->low_priority_tx == 0) &&
+	    (coex_sta->low_priority_rx == 0)) {
+		num_of_bt_counter_chk++;
+
+		if (num_of_bt_counter_chk >= 3) {
+			halbtc8822b1ant_query_bt_info(btcoexist);
+			num_of_bt_counter_chk = 0;
+		}
+	}
+}
+
+static void halbtc8822b1ant_monitor_wifi_ctr(struct btc_coexist *btcoexist)
+{
+	s32 wifi_rssi = 0;
+	bool wifi_busy = false, wifi_under_b_mode = false, wifi_scan = false;
+	static u8 cck_lock_counter, wl_noisy_count0, wl_noisy_count1 = 3,
+						     wl_noisy_count2;
+	u32 total_cnt, cck_cnt;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+	btcoexist->btc_get(btcoexist, BTC_GET_S4_WIFI_RSSI, &wifi_rssi);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_B_MODE,
+			   &wifi_under_b_mode);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_SCAN, &wifi_scan);
+
+	coex_sta->crc_ok_cck = btcoexist->btc_phydm_query_phy_counter(
+		btcoexist, "PHYDM_INFO_CRC32_OK_CCK");
+	coex_sta->crc_ok_11g = btcoexist->btc_phydm_query_phy_counter(
+		btcoexist, "PHYDM_INFO_CRC32_OK_LEGACY");
+	coex_sta->crc_ok_11n = btcoexist->btc_phydm_query_phy_counter(
+		btcoexist, "PHYDM_INFO_CRC32_OK_HT");
+	coex_sta->crc_ok_11n_vht = btcoexist->btc_phydm_query_phy_counter(
+		btcoexist, "PHYDM_INFO_CRC32_OK_VHT");
+
+	coex_sta->crc_err_cck = btcoexist->btc_phydm_query_phy_counter(
+		btcoexist, "PHYDM_INFO_CRC32_ERROR_CCK");
+	coex_sta->crc_err_11g = btcoexist->btc_phydm_query_phy_counter(
+		btcoexist, "PHYDM_INFO_CRC32_ERROR_LEGACY");
+	coex_sta->crc_err_11n = btcoexist->btc_phydm_query_phy_counter(
+		btcoexist, "PHYDM_INFO_CRC32_ERROR_HT");
+	coex_sta->crc_err_11n_vht = btcoexist->btc_phydm_query_phy_counter(
+		btcoexist, "PHYDM_INFO_CRC32_ERROR_VHT");
+
+	cck_cnt = coex_sta->crc_ok_cck + coex_sta->crc_err_cck;
+
+	if (cck_cnt > 250) {
+		if (wl_noisy_count2 < 3)
+			wl_noisy_count2++;
+
+		if (wl_noisy_count2 == 3) {
+			wl_noisy_count0 = 0;
+			wl_noisy_count1 = 0;
+		}
+
+	} else if (cck_cnt < 50) {
+		if (wl_noisy_count0 < 3)
+			wl_noisy_count0++;
+
+		if (wl_noisy_count0 == 3) {
+			wl_noisy_count1 = 0;
+			wl_noisy_count2 = 0;
+		}
+
+	} else {
+		if (wl_noisy_count1 < 3)
+			wl_noisy_count1++;
+
+		if (wl_noisy_count1 == 3) {
+			wl_noisy_count0 = 0;
+			wl_noisy_count2 = 0;
+		}
+	}
+
+	if (wl_noisy_count2 == 3)
+		coex_sta->wl_noisy_level = 2;
+	else if (wl_noisy_count1 == 3)
+		coex_sta->wl_noisy_level = 1;
+	else
+		coex_sta->wl_noisy_level = 0;
+
+	if ((wifi_busy) && (wifi_rssi >= 30) && (!wifi_under_b_mode)) {
+		total_cnt = coex_sta->crc_ok_cck + coex_sta->crc_ok_11g +
+			    coex_sta->crc_ok_11n + coex_sta->crc_ok_11n_vht;
+
+		if ((coex_dm->bt_status == BT_8822B_1ANT_BT_STATUS_ACL_BUSY) ||
+		    (coex_dm->bt_status ==
+		     BT_8822B_1ANT_BT_STATUS_ACL_SCO_BUSY) ||
+		    (coex_dm->bt_status == BT_8822B_1ANT_BT_STATUS_SCO_BUSY)) {
+			if (coex_sta->crc_ok_cck >
+			    (total_cnt - coex_sta->crc_ok_cck)) {
+				if (cck_lock_counter < 3)
+					cck_lock_counter++;
+			} else {
+				if (cck_lock_counter > 0)
+					cck_lock_counter--;
+			}
+
+		} else {
+			if (cck_lock_counter > 0)
+				cck_lock_counter--;
+		}
+	} else {
+		if (cck_lock_counter > 0)
+			cck_lock_counter--;
+	}
+
+	if (!coex_sta->pre_ccklock) {
+		if (cck_lock_counter >= 3)
+			coex_sta->cck_lock = true;
+		else
+			coex_sta->cck_lock = false;
+	} else {
+		if (cck_lock_counter == 0)
+			coex_sta->cck_lock = false;
+		else
+			coex_sta->cck_lock = true;
+	}
+
+	if (coex_sta->cck_lock)
+		coex_sta->cck_ever_lock = true;
+
+	coex_sta->pre_ccklock = coex_sta->cck_lock;
+}
+
+static bool
+halbtc8822b1ant_is_wifi_status_changed(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	static bool pre_wifi_busy, pre_under_4way, pre_bt_hs_on,
+		pre_rf4ce_enabled, pre_bt_off, pre_bt_slave,
+		pre_hid_low_pri_tx_overhead, pre_wifi_under_lps,
+		pre_bt_setup_link;
+	static u8 pre_hid_busy_num, pre_wl_noisy_level;
+	bool wifi_busy = false, under_4way = false, bt_hs_on = false,
+	     rf4ce_enabled = false;
+	bool wifi_connected = false;
+	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_CONNECTED,
+			   &wifi_connected);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_HS_OPERATION, &bt_hs_on);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_4_WAY_PROGRESS,
+			   &under_4way);
+
+	if (coex_sta->bt_disabled != pre_bt_off) {
+		pre_bt_off = coex_sta->bt_disabled;
+
+		if (coex_sta->bt_disabled)
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], BT is disabled !!\n");
+		else
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], BT is enabled !!\n");
+
+		coex_sta->bt_coex_supported_feature = 0;
+		coex_sta->bt_coex_supported_version = 0;
+		return true;
+	}
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_RF4CE_CONNECTED,
+			   &rf4ce_enabled);
+
+	if (rf4ce_enabled != pre_rf4ce_enabled) {
+		pre_rf4ce_enabled = rf4ce_enabled;
+
+		if (rf4ce_enabled)
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], rf4ce is enabled !!\n");
+		else
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], rf4ce is disabled !!\n");
+
+		return true;
+	}
+
+	if (wifi_connected) {
+		if (wifi_busy != pre_wifi_busy) {
+			pre_wifi_busy = wifi_busy;
+			return true;
+		}
+		if (under_4way != pre_under_4way) {
+			pre_under_4way = under_4way;
+			return true;
+		}
+		if (bt_hs_on != pre_bt_hs_on) {
+			pre_bt_hs_on = bt_hs_on;
+			return true;
+		}
+		if (coex_sta->wl_noisy_level != pre_wl_noisy_level) {
+			pre_wl_noisy_level = coex_sta->wl_noisy_level;
+			return true;
+		}
+		if (coex_sta->under_lps != pre_wifi_under_lps) {
+			pre_wifi_under_lps = coex_sta->under_lps;
+			if (coex_sta->under_lps)
+				return true;
+		}
+	}
+
+	if (!coex_sta->bt_disabled) {
+		if (coex_sta->hid_busy_num != pre_hid_busy_num) {
+			pre_hid_busy_num = coex_sta->hid_busy_num;
+			return true;
+		}
+
+		if (bt_link_info->slave_role != pre_bt_slave) {
+			pre_bt_slave = bt_link_info->slave_role;
+			return true;
+		}
+
+		if (pre_hid_low_pri_tx_overhead !=
+		    coex_sta->is_hid_low_pri_tx_overhead) {
+			pre_hid_low_pri_tx_overhead =
+				coex_sta->is_hid_low_pri_tx_overhead;
+			return true;
+		}
+
+		if (pre_bt_setup_link != coex_sta->is_setup_link) {
+			pre_bt_setup_link = coex_sta->is_setup_link;
+			return true;
+		}
+	}
+
+	return false;
+}
+
+static void halbtc8822b1ant_update_bt_link_info(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
+	bool bt_hs_on = false;
+	bool bt_busy = false;
+
+	coex_sta->num_of_profile = 0;
+
+	/* set link exist status */
+	if (!(coex_sta->bt_info & BT_INFO_8822B_1ANT_B_CONNECTION)) {
+		coex_sta->bt_link_exist = false;
+		coex_sta->pan_exist = false;
+		coex_sta->a2dp_exist = false;
+		coex_sta->hid_exist = false;
+		coex_sta->sco_exist = false;
+	} else { /* connection exists */
+		coex_sta->bt_link_exist = true;
+		if (coex_sta->bt_info & BT_INFO_8822B_1ANT_B_FTP) {
+			coex_sta->pan_exist = true;
+			coex_sta->num_of_profile++;
+		} else {
+			coex_sta->pan_exist = false;
+		}
+
+		if (coex_sta->bt_info & BT_INFO_8822B_1ANT_B_A2DP) {
+			coex_sta->a2dp_exist = true;
+			coex_sta->num_of_profile++;
+		} else {
+			coex_sta->a2dp_exist = false;
+		}
+
+		if (coex_sta->bt_info & BT_INFO_8822B_1ANT_B_HID) {
+			coex_sta->hid_exist = true;
+			coex_sta->num_of_profile++;
+		} else {
+			coex_sta->hid_exist = false;
+		}
+
+		if (coex_sta->bt_info & BT_INFO_8822B_1ANT_B_SCO_ESCO) {
+			coex_sta->sco_exist = true;
+			coex_sta->num_of_profile++;
+		} else {
+			coex_sta->sco_exist = false;
+		}
+	}
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_HS_OPERATION, &bt_hs_on);
+
+	bt_link_info->bt_link_exist = coex_sta->bt_link_exist;
+	bt_link_info->sco_exist = coex_sta->sco_exist;
+	bt_link_info->a2dp_exist = coex_sta->a2dp_exist;
+	bt_link_info->pan_exist = coex_sta->pan_exist;
+	bt_link_info->hid_exist = coex_sta->hid_exist;
+	bt_link_info->acl_busy = coex_sta->acl_busy;
+
+	/* work around for HS mode. */
+	if (bt_hs_on) {
+		bt_link_info->pan_exist = true;
+		bt_link_info->bt_link_exist = true;
+	}
+
+	/* check if Sco only */
+	if (bt_link_info->sco_exist && !bt_link_info->a2dp_exist &&
+	    !bt_link_info->pan_exist && !bt_link_info->hid_exist)
+		bt_link_info->sco_only = true;
+	else
+		bt_link_info->sco_only = false;
+
+	/* check if A2dp only */
+	if (!bt_link_info->sco_exist && bt_link_info->a2dp_exist &&
+	    !bt_link_info->pan_exist && !bt_link_info->hid_exist)
+		bt_link_info->a2dp_only = true;
+	else
+		bt_link_info->a2dp_only = false;
+
+	/* check if Pan only */
+	if (!bt_link_info->sco_exist && !bt_link_info->a2dp_exist &&
+	    bt_link_info->pan_exist && !bt_link_info->hid_exist)
+		bt_link_info->pan_only = true;
+	else
+		bt_link_info->pan_only = false;
+
+	/* check if Hid only */
+	if (!bt_link_info->sco_exist && !bt_link_info->a2dp_exist &&
+	    !bt_link_info->pan_exist && bt_link_info->hid_exist)
+		bt_link_info->hid_only = true;
+	else
+		bt_link_info->hid_only = false;
+
+	if (coex_sta->bt_info & BT_INFO_8822B_1ANT_B_INQ_PAGE) {
+		coex_dm->bt_status = BT_8822B_1ANT_BT_STATUS_INQ_PAGE;
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], BtInfoNotify(), BT Inq/page!!!\n");
+	} else if (!(coex_sta->bt_info & BT_INFO_8822B_1ANT_B_CONNECTION)) {
+		coex_dm->bt_status = BT_8822B_1ANT_BT_STATUS_NON_CONNECTED_IDLE;
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], BtInfoNotify(), BT Non-Connected idle!!!\n");
+	} else if (coex_sta->bt_info == BT_INFO_8822B_1ANT_B_CONNECTION) {
+		/* connection exists but no busy */
+		coex_dm->bt_status = BT_8822B_1ANT_BT_STATUS_CONNECTED_IDLE;
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], BtInfoNotify(), BT Connected-idle!!!\n");
+	} else if (((coex_sta->bt_info & BT_INFO_8822B_1ANT_B_SCO_ESCO) ||
+		    (coex_sta->bt_info & BT_INFO_8822B_1ANT_B_SCO_BUSY)) &&
+		   (coex_sta->bt_info & BT_INFO_8822B_1ANT_B_ACL_BUSY)) {
+		coex_dm->bt_status = BT_8822B_1ANT_BT_STATUS_ACL_SCO_BUSY;
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], BtInfoNotify(), BT ACL SCO busy!!!\n");
+	} else if ((coex_sta->bt_info & BT_INFO_8822B_1ANT_B_SCO_ESCO) ||
+		   (coex_sta->bt_info & BT_INFO_8822B_1ANT_B_SCO_BUSY)) {
+		coex_dm->bt_status = BT_8822B_1ANT_BT_STATUS_SCO_BUSY;
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], BtInfoNotify(), BT SCO busy!!!\n");
+	} else if (coex_sta->bt_info & BT_INFO_8822B_1ANT_B_ACL_BUSY) {
+		coex_dm->bt_status = BT_8822B_1ANT_BT_STATUS_ACL_BUSY;
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], BtInfoNotify(), BT ACL busy!!!\n");
+	} else {
+		coex_dm->bt_status = BT_8822B_1ANT_BT_STATUS_MAX;
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], BtInfoNotify(), BT Non-Defined state!!!\n");
+	}
+
+	if ((coex_dm->bt_status == BT_8822B_1ANT_BT_STATUS_ACL_BUSY) ||
+	    (coex_dm->bt_status == BT_8822B_1ANT_BT_STATUS_SCO_BUSY) ||
+	    (coex_dm->bt_status == BT_8822B_1ANT_BT_STATUS_ACL_SCO_BUSY))
+		bt_busy = true;
+	else
+		bt_busy = false;
+
+	btcoexist->btc_set(btcoexist, BTC_SET_BL_BT_TRAFFIC_BUSY, &bt_busy);
+}
+
+static void halbtc8822b1ant_update_wifi_ch_info(struct btc_coexist *btcoexist,
+						u8 type)
+{
+	u8 h2c_parameter[3] = {0};
+	u32 wifi_bw;
+	u8 wifi_central_chnl;
+
+	/* only 2.4G we need to inform bt the chnl mask */
+	btcoexist->btc_get(btcoexist, BTC_GET_U1_WIFI_CENTRAL_CHNL,
+			   &wifi_central_chnl);
+	if ((type == BTC_MEDIA_CONNECT) && (wifi_central_chnl <= 14)) {
+		/* enable BT AFH skip WL channel for 8822b
+		 * because BT Rx LO interference
+		 */
+		h2c_parameter[0] = 0x1;
+		h2c_parameter[1] = wifi_central_chnl;
+
+		btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_BW, &wifi_bw);
+
+		if (wifi_bw == BTC_WIFI_BW_HT40)
+			h2c_parameter[2] = 0x30;
+		else
+			h2c_parameter[2] = 0x20;
+	}
+
+	coex_dm->wifi_chnl_info[0] = h2c_parameter[0];
+	coex_dm->wifi_chnl_info[1] = h2c_parameter[1];
+	coex_dm->wifi_chnl_info[2] = h2c_parameter[2];
+
+	btcoexist->btc_fill_h2c(btcoexist, 0x66, 3, h2c_parameter);
+}
+
+static u8 halbtc8822b1ant_action_algorithm(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
+	bool bt_hs_on = false;
+	u8 algorithm = BT_8822B_1ANT_COEX_ALGO_UNDEFINED;
+	u8 num_of_diff_profile = 0;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_HS_OPERATION, &bt_hs_on);
+
+	if (!bt_link_info->bt_link_exist) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], No BT link exists!!!\n");
+		return algorithm;
+	}
+
+	if (bt_link_info->sco_exist)
+		num_of_diff_profile++;
+	if (bt_link_info->hid_exist)
+		num_of_diff_profile++;
+	if (bt_link_info->pan_exist)
+		num_of_diff_profile++;
+	if (bt_link_info->a2dp_exist)
+		num_of_diff_profile++;
+
+	if (num_of_diff_profile == 1) {
+		if (bt_link_info->sco_exist) {
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], BT Profile = SCO only\n");
+			algorithm = BT_8822B_1ANT_COEX_ALGO_SCO;
+		} else {
+			if (bt_link_info->hid_exist) {
+				RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+					 "[BTCoex], BT Profile = HID only\n");
+				algorithm = BT_8822B_1ANT_COEX_ALGO_HID;
+			} else if (bt_link_info->a2dp_exist) {
+				RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+					 "[BTCoex], BT Profile = A2DP only\n");
+				algorithm = BT_8822B_1ANT_COEX_ALGO_A2DP;
+			} else if (bt_link_info->pan_exist) {
+				if (bt_hs_on) {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], BT Profile = PAN(HS) only\n");
+					algorithm =
+						BT_8822B_1ANT_COEX_ALGO_PANHS;
+				} else {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], BT Profile = PAN(EDR) only\n");
+					algorithm =
+						BT_8822B_1ANT_COEX_ALGO_PANEDR;
+				}
+			}
+		}
+	} else if (num_of_diff_profile == 2) {
+		if (bt_link_info->sco_exist) {
+			if (bt_link_info->hid_exist) {
+				RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+					 "[BTCoex], BT Profile = SCO + HID\n");
+				algorithm = BT_8822B_1ANT_COEX_ALGO_HID;
+			} else if (bt_link_info->a2dp_exist) {
+				RT_TRACE(
+					rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+					"[BTCoex], BT Profile = SCO + A2DP ==> SCO\n");
+				algorithm = BT_8822B_1ANT_COEX_ALGO_SCO;
+			} else if (bt_link_info->pan_exist) {
+				if (bt_hs_on) {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], BT Profile = SCO + PAN(HS)\n");
+					algorithm = BT_8822B_1ANT_COEX_ALGO_SCO;
+				} else {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], BT Profile = SCO + PAN(EDR)\n");
+					algorithm =
+					    BT_8822B_1ANT_COEX_ALGO_PANEDR_HID;
+				}
+			}
+		} else {
+			if (bt_link_info->hid_exist &&
+			    bt_link_info->a2dp_exist) {
+				RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+					 "[BTCoex], BT Profile = HID + A2DP\n");
+				algorithm = BT_8822B_1ANT_COEX_ALGO_HID_A2DP;
+			} else if (bt_link_info->hid_exist &&
+				   bt_link_info->pan_exist) {
+				if (bt_hs_on) {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], BT Profile = HID + PAN(HS)\n");
+					algorithm =
+					    BT_8822B_1ANT_COEX_ALGO_HID_A2DP;
+				} else {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], BT Profile = HID + PAN(EDR)\n");
+					algorithm =
+					    BT_8822B_1ANT_COEX_ALGO_PANEDR_HID;
+				}
+			} else if (bt_link_info->pan_exist &&
+				   bt_link_info->a2dp_exist) {
+				if (bt_hs_on) {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], BT Profile = A2DP + PAN(HS)\n");
+					algorithm =
+					    BT_8822B_1ANT_COEX_ALGO_A2DP_PANHS;
+				} else {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], BT Profile = A2DP + PAN(EDR)\n");
+					algorithm =
+					    BT_8822B_1ANT_COEX_ALGO_PANEDR_A2DP;
+				}
+			}
+		}
+	} else if (num_of_diff_profile == 3) {
+		if (bt_link_info->sco_exist) {
+			if (bt_link_info->hid_exist &&
+			    bt_link_info->a2dp_exist) {
+				RT_TRACE(
+					rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+					"[BTCoex], BT Profile = SCO + HID + A2DP ==> HID\n");
+				algorithm = BT_8822B_1ANT_COEX_ALGO_HID;
+			} else if (bt_link_info->hid_exist &&
+				   bt_link_info->pan_exist) {
+				if (bt_hs_on) {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], BT Profile = SCO + HID + PAN(HS)\n");
+					algorithm =
+					    BT_8822B_1ANT_COEX_ALGO_HID_A2DP;
+				} else {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], BT Profile = SCO + HID + PAN(EDR)\n");
+					algorithm =
+					    BT_8822B_1ANT_COEX_ALGO_PANEDR_HID;
+				}
+			} else if (bt_link_info->pan_exist &&
+				   bt_link_info->a2dp_exist) {
+				if (bt_hs_on) {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], BT Profile = SCO + A2DP + PAN(HS)\n");
+					algorithm = BT_8822B_1ANT_COEX_ALGO_SCO;
+				} else {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], BT Profile = SCO + A2DP + PAN(EDR) ==> HID\n");
+					algorithm =
+					    BT_8822B_1ANT_COEX_ALGO_PANEDR_HID;
+				}
+			}
+		} else {
+			if (bt_link_info->hid_exist &&
+			    bt_link_info->pan_exist &&
+			    bt_link_info->a2dp_exist) {
+				if (bt_hs_on) {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], BT Profile = HID + A2DP + PAN(HS)\n");
+					algorithm =
+					    BT_8822B_1ANT_COEX_ALGO_HID_A2DP;
+				} else {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], BT Profile = HID + A2DP + PAN(EDR)\n");
+					algorithm =
+					BT_8822B_1ANT_COEX_ALGO_HID_A2DP_PANEDR;
+				}
+			}
+		}
+	} else if (num_of_diff_profile >= 3) {
+		if (bt_link_info->sco_exist) {
+			if (bt_link_info->hid_exist &&
+			    bt_link_info->pan_exist &&
+			    bt_link_info->a2dp_exist) {
+				if (bt_hs_on) {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], Error!!! BT Profile = SCO + HID + A2DP + PAN(HS)\n");
+
+				} else {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], BT Profile = SCO + HID + A2DP + PAN(EDR)==>PAN(EDR)+HID\n");
+					algorithm =
+					    BT_8822B_1ANT_COEX_ALGO_PANEDR_HID;
+				}
+			}
+		}
+	}
+
+	return algorithm;
+}
+
+static void halbtc8822b1ant_low_penalty_ra(struct btc_coexist *btcoexist,
+					   bool force_exec, bool low_penalty_ra)
+{
+	coex_dm->cur_low_penalty_ra = low_penalty_ra;
+
+	if (!force_exec) {
+		if (coex_dm->pre_low_penalty_ra == coex_dm->cur_low_penalty_ra)
+			return;
+	}
+
+	if (low_penalty_ra)
+		btcoexist->btc_phydm_modify_ra_pcr_threshold(btcoexist, 0, 25);
+	else
+		btcoexist->btc_phydm_modify_ra_pcr_threshold(btcoexist, 0, 0);
+
+	coex_dm->pre_low_penalty_ra = coex_dm->cur_low_penalty_ra;
+}
+
+static void halbtc8822b1ant_write_score_board(struct btc_coexist *btcoexist,
+					      u16 bitpos, bool state)
+{
+	static u16 originalval = 0x8002;
+
+	if (state)
+		originalval = originalval | bitpos;
+	else
+		originalval = originalval & (~bitpos);
+
+	btcoexist->btc_write_2byte(btcoexist, 0xaa, originalval);
+}
+
+static void halbtc8822b1ant_read_score_board(struct btc_coexist *btcoexist,
+					     u16 *score_board_val)
+{
+	*score_board_val =
+		(btcoexist->btc_read_2byte(btcoexist, 0xaa)) & 0x7fff;
+}
+
+static void halbtc8822b1ant_post_state_to_bt(struct btc_coexist *btcoexist,
+					     u16 type, bool state)
+{
+	halbtc8822b1ant_write_score_board(btcoexist, (u16)type, state);
+}
+
+static void
+halbtc8822b1ant_monitor_bt_enable_disable(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	static u32 bt_disable_cnt;
+	bool bt_active = true, bt_disabled = false, wifi_under_5g = false;
+	u16 u16tmp;
+
+	/* This function check if bt is disabled */
+
+	/* Read BT on/off status from scoreboard[1],
+	 * enable this only if BT patch support this feature
+	 */
+	halbtc8822b1ant_read_score_board(btcoexist, &u16tmp);
+
+	bt_active = u16tmp & BIT(1);
+
+	if (bt_active) {
+		bt_disable_cnt = 0;
+		bt_disabled = false;
+		btcoexist->btc_set(btcoexist, BTC_SET_BL_BT_DISABLE,
+				   &bt_disabled);
+	} else {
+		bt_disable_cnt++;
+		if (bt_disable_cnt >= 2) {
+			bt_disabled = true;
+			bt_disable_cnt = 2;
+		}
+
+		btcoexist->btc_set(btcoexist, BTC_SET_BL_BT_DISABLE,
+				   &bt_disabled);
+	}
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_5G, &wifi_under_5g);
+
+	if ((wifi_under_5g) || (bt_disabled))
+		halbtc8822b1ant_low_penalty_ra(btcoexist, NORMAL_EXEC, false);
+	else
+		halbtc8822b1ant_low_penalty_ra(btcoexist, NORMAL_EXEC, true);
+
+	if (coex_sta->bt_disabled != bt_disabled) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], BT is from %s to %s!!\n",
+			 (coex_sta->bt_disabled ? "disabled" : "enabled"),
+			 (bt_disabled ? "disabled" : "enabled"));
+		coex_sta->bt_disabled = bt_disabled;
+	}
+}
+
+static void halbtc8822b1ant_enable_gnt_to_gpio(struct btc_coexist *btcoexist,
+					       bool isenable)
+{
+	static u8 bit_val[5] = {0, 0, 0, 0, 0};
+	static bool state;
+
+	if (!btcoexist->dbg_mode_1ant)
+		return;
+
+	if (state == isenable)
+		return;
+
+	state = isenable;
+
+	if (isenable) {
+		/* enable GNT_WL, GNT_BT to GPIO for debug */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x73, 0x8, 0x1);
+
+		/* store original value */
+		bit_val[0] =
+			(btcoexist->btc_read_1byte(btcoexist, 0x66) & BIT(4)) >>
+			4; /*0x66[4] */
+		bit_val[1] = (btcoexist->btc_read_1byte(btcoexist, 0x67) &
+			      BIT(0)); /*0x66[8] */
+		bit_val[2] =
+			(btcoexist->btc_read_1byte(btcoexist, 0x42) & BIT(3)) >>
+			3; /*0x40[19] */
+		bit_val[3] =
+			(btcoexist->btc_read_1byte(btcoexist, 0x65) & BIT(7)) >>
+			7; /*0x64[15] */
+		bit_val[4] =
+			(btcoexist->btc_read_1byte(btcoexist, 0x72) & BIT(2)) >>
+			2; /*0x70[18] */
+
+		/*  switch GPIO Mux */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x66, BIT(4),
+						   0x0); /*0x66[4] = 0 */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x67, BIT(0),
+						   0x0); /*0x66[8] = 0 */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x42, BIT(3),
+						   0x0); /*0x40[19] = 0 */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x65, BIT(7),
+						   0x0); /*0x64[15] = 0 */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x72, BIT(2),
+						   0x0); /*0x70[18] = 0 */
+
+	} else {
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x73, 0x8, 0x0);
+
+		/*  Restore original value  */
+		/*  switch GPIO Mux */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x66, BIT(4),
+						   bit_val[0]); /*0x66[4] = 0 */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x67, BIT(0),
+						   bit_val[1]); /*0x66[8] = 0 */
+		btcoexist->btc_write_1byte_bitmask(
+			btcoexist, 0x42, BIT(3), bit_val[2]); /*0x40[19] = 0 */
+		btcoexist->btc_write_1byte_bitmask(
+			btcoexist, 0x65, BIT(7), bit_val[3]); /*0x64[15] = 0 */
+		btcoexist->btc_write_1byte_bitmask(
+			btcoexist, 0x72, BIT(2), bit_val[4]); /*0x70[18] = 0 */
+	}
+}
+
+static u32
+halbtc8822b1ant_ltecoex_indirect_read_reg(struct btc_coexist *btcoexist,
+					  u16 reg_addr)
+{
+	u32 delay_count = 0;
+
+	/* wait for ready bit before access 0x1700 */
+	while (1) {
+		if ((btcoexist->btc_read_1byte(btcoexist, 0x1703) & BIT(5)) ==
+		    0) {
+			mdelay(50);
+			delay_count++;
+			if (delay_count >= 10) {
+				delay_count = 0;
+				break;
+			}
+		} else {
+			break;
+		}
+	}
+
+	btcoexist->btc_write_4byte(btcoexist, 0x1700, 0x800F0000 | reg_addr);
+
+	return btcoexist->btc_read_4byte(btcoexist, 0x1708); /* get read data */
+}
+
+static void
+halbtc8822b1ant_ltecoex_indirect_write_reg(struct btc_coexist *btcoexist,
+					   u16 reg_addr, u32 bit_mask,
+					   u32 reg_value)
+{
+	u32 val, i = 0, bitpos = 0, delay_count = 0;
+
+	if (bit_mask == 0x0)
+		return;
+
+	if (bit_mask == 0xffffffff) {
+		/* wait for ready bit before access 0x1700/0x1704 */
+		while (1) {
+			if ((btcoexist->btc_read_1byte(btcoexist, 0x1703) &
+			     BIT(5)) == 0) {
+				mdelay(50);
+				delay_count++;
+				if (delay_count >= 10) {
+					delay_count = 0;
+					break;
+				}
+			} else {
+				break;
+			}
+		}
+
+		btcoexist->btc_write_4byte(btcoexist, 0x1704,
+					   reg_value); /* put write data */
+
+		btcoexist->btc_write_4byte(btcoexist, 0x1700,
+					   0xc00F0000 | reg_addr);
+	} else {
+		for (i = 0; i <= 31; i++) {
+			if (((bit_mask >> i) & 0x1) == 0x1) {
+				bitpos = i;
+				break;
+			}
+		}
+
+		/* read back register value before write */
+		val = halbtc8822b1ant_ltecoex_indirect_read_reg(btcoexist,
+								reg_addr);
+		val = (val & (~bit_mask)) | (reg_value << bitpos);
+
+		/* wait for ready bit before access 0x1700/0x1704 */
+		while (1) {
+			if ((btcoexist->btc_read_1byte(btcoexist, 0x1703) &
+			     BIT(5)) == 0) {
+				mdelay(50);
+				delay_count++;
+				if (delay_count >= 10) {
+					delay_count = 0;
+					break;
+				}
+			} else {
+				break;
+			}
+		}
+
+		btcoexist->btc_write_4byte(btcoexist, 0x1704,
+					   val); /* put write data */
+
+		btcoexist->btc_write_4byte(btcoexist, 0x1700,
+					   0xc00F0000 | reg_addr);
+	}
+}
+
+static void halbtc8822b1ant_ltecoex_enable(struct btc_coexist *btcoexist,
+					   bool enable)
+{
+	u8 val;
+
+	val = (enable) ? 1 : 0;
+	/* 0x38[7] */
+	halbtc8822b1ant_ltecoex_indirect_write_reg(btcoexist, 0x38, 0x80, val);
+}
+
+static void
+halbtc8822b1ant_ltecoex_pathcontrol_owner(struct btc_coexist *btcoexist,
+					  bool wifi_control)
+{
+	u8 val;
+
+	val = (wifi_control) ? 1 : 0;
+	/* 0x70[26] */
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0x73, 0x4, val);
+}
+
+static void halbtc8822b1ant_ltecoex_set_gnt_bt(struct btc_coexist *btcoexist,
+					       u8 control_block,
+					       bool sw_control, u8 state)
+{
+	u32 val = 0, bit_mask;
+
+	state = state & 0x1;
+	/*LTE indirect 0x38=0xccxx (sw : gnt_wl=1,sw gnt_bt=1)
+	 *0x38=0xddxx (sw : gnt_bt=1 , sw gnt_wl=0)
+	 *0x38=0x55xx(hw pta :gnt_wl /gnt_bt )
+	 */
+	val = (sw_control) ? ((state << 1) | 0x1) : 0;
+
+	switch (control_block) {
+	case BT_8822B_1ANT_GNT_BLOCK_RFC_BB:
+	default:
+		bit_mask = 0xc000;
+		halbtc8822b1ant_ltecoex_indirect_write_reg(
+			btcoexist, 0x38, bit_mask, val); /* 0x38[15:14] */
+		bit_mask = 0x0c00;
+		halbtc8822b1ant_ltecoex_indirect_write_reg(
+			btcoexist, 0x38, bit_mask, val); /* 0x38[11:10] */
+		break;
+	case BT_8822B_1ANT_GNT_BLOCK_RFC:
+		bit_mask = 0xc000;
+		halbtc8822b1ant_ltecoex_indirect_write_reg(
+			btcoexist, 0x38, bit_mask, val); /* 0x38[15:14] */
+		break;
+	case BT_8822B_1ANT_GNT_BLOCK_BB:
+		bit_mask = 0x0c00;
+		halbtc8822b1ant_ltecoex_indirect_write_reg(
+			btcoexist, 0x38, bit_mask, val); /* 0x38[11:10] */
+		break;
+	}
+}
+
+static void halbtc8822b1ant_ltecoex_set_gnt_wl(struct btc_coexist *btcoexist,
+					       u8 control_block,
+					       bool sw_control, u8 state)
+{
+	u32 val = 0, bit_mask;
+	/*LTE indirect 0x38=0xccxx (sw : gnt_wl=1,sw gnt_bt=1)
+	 *0x38=0xddxx (sw : gnt_bt=1 , sw gnt_wl=0)
+	 *0x38=0x55xx(hw pta :gnt_wl /gnt_bt )
+	 */
+
+	state = state & 0x1;
+	val = (sw_control) ? ((state << 1) | 0x1) : 0;
+
+	switch (control_block) {
+	case BT_8822B_1ANT_GNT_BLOCK_RFC_BB:
+	default:
+		bit_mask = 0x3000;
+		halbtc8822b1ant_ltecoex_indirect_write_reg(
+			btcoexist, 0x38, bit_mask, val); /* 0x38[13:12] */
+		bit_mask = 0x0300;
+		halbtc8822b1ant_ltecoex_indirect_write_reg(
+			btcoexist, 0x38, bit_mask, val); /* 0x38[9:8] */
+		break;
+	case BT_8822B_1ANT_GNT_BLOCK_RFC:
+		bit_mask = 0x3000;
+		halbtc8822b1ant_ltecoex_indirect_write_reg(
+			btcoexist, 0x38, bit_mask, val); /* 0x38[13:12] */
+		break;
+	case BT_8822B_1ANT_GNT_BLOCK_BB:
+		bit_mask = 0x0300;
+		halbtc8822b1ant_ltecoex_indirect_write_reg(
+			btcoexist, 0x38, bit_mask, val); /* 0x38[9:8] */
+		break;
+	}
+}
+
+static void
+halbtc8822b1ant_ltecoex_set_coex_table(struct btc_coexist *btcoexist,
+				       u8 table_type, u16 table_content)
+{
+	u16 reg_addr = 0x0000;
+
+	switch (table_type) {
+	case BT_8822B_1ANT_CTT_WL_VS_LTE:
+		reg_addr = 0xa0;
+		break;
+	case BT_8822B_1ANT_CTT_BT_VS_LTE:
+		reg_addr = 0xa4;
+		break;
+	}
+
+	if (reg_addr != 0x0000)
+		halbtc8822b1ant_ltecoex_indirect_write_reg(
+			btcoexist, reg_addr, 0xffff,
+			table_content); /* 0xa0[15:0] or 0xa4[15:0] */
+}
+
+static void halbtc8822b1ant_set_wltoggle_coex_table(
+	struct btc_coexist *btcoexist, bool force_exec, u8 interval,
+	u8 val0x6c4_b0, u8 val0x6c4_b1, u8 val0x6c4_b2, u8 val0x6c4_b3)
+{
+	static u8 pre_h2c_parameter[6] = {0};
+	u8 cur_h2c_parameter[6] = {0};
+	u8 i, match_cnt = 0;
+
+	cur_h2c_parameter[0] = 0x7; /* op_code, 0x7= wlan toggle slot*/
+
+	cur_h2c_parameter[1] = interval;
+	cur_h2c_parameter[2] = val0x6c4_b0;
+	cur_h2c_parameter[3] = val0x6c4_b1;
+	cur_h2c_parameter[4] = val0x6c4_b2;
+	cur_h2c_parameter[5] = val0x6c4_b3;
+
+	if (!force_exec) {
+		for (i = 1; i <= 5; i++) {
+			if (cur_h2c_parameter[i] != pre_h2c_parameter[i])
+				break;
+
+			match_cnt++;
+		}
+
+		if (match_cnt == 5)
+			return;
+	}
+
+	for (i = 1; i <= 5; i++)
+		pre_h2c_parameter[i] = cur_h2c_parameter[i];
+
+	btcoexist->btc_fill_h2c(btcoexist, 0x69, 6, cur_h2c_parameter);
+}
+
+static void halbtc8822b1ant_set_coex_table(struct btc_coexist *btcoexist,
+					   u32 val0x6c0, u32 val0x6c4,
+					   u32 val0x6c8, u8 val0x6cc)
+{
+	btcoexist->btc_write_4byte(btcoexist, 0x6c0, val0x6c0);
+
+	btcoexist->btc_write_4byte(btcoexist, 0x6c4, val0x6c4);
+
+	btcoexist->btc_write_4byte(btcoexist, 0x6c8, val0x6c8);
+
+	btcoexist->btc_write_1byte(btcoexist, 0x6cc, val0x6cc);
+}
+
+static void halbtc8822b1ant_coex_table(struct btc_coexist *btcoexist,
+				       bool force_exec, u32 val0x6c0,
+				       u32 val0x6c4, u32 val0x6c8, u8 val0x6cc)
+{
+	coex_dm->cur_val0x6c0 = val0x6c0;
+	coex_dm->cur_val0x6c4 = val0x6c4;
+	coex_dm->cur_val0x6c8 = val0x6c8;
+	coex_dm->cur_val0x6cc = val0x6cc;
+
+	if (!force_exec) {
+		if ((coex_dm->pre_val0x6c0 == coex_dm->cur_val0x6c0) &&
+		    (coex_dm->pre_val0x6c4 == coex_dm->cur_val0x6c4) &&
+		    (coex_dm->pre_val0x6c8 == coex_dm->cur_val0x6c8) &&
+		    (coex_dm->pre_val0x6cc == coex_dm->cur_val0x6cc))
+			return;
+	}
+	halbtc8822b1ant_set_coex_table(btcoexist, val0x6c0, val0x6c4, val0x6c8,
+				       val0x6cc);
+
+	coex_dm->pre_val0x6c0 = coex_dm->cur_val0x6c0;
+	coex_dm->pre_val0x6c4 = coex_dm->cur_val0x6c4;
+	coex_dm->pre_val0x6c8 = coex_dm->cur_val0x6c8;
+	coex_dm->pre_val0x6cc = coex_dm->cur_val0x6cc;
+}
+
+static void halbtc8822b1ant_coex_table_with_type(struct btc_coexist *btcoexist,
+						 bool force_exec, u8 type)
+{
+	u32 break_table;
+	u8 select_table;
+
+	coex_sta->coex_table_type = type;
+
+	if (coex_sta->concurrent_rx_mode_on) {
+		break_table = 0xf0ffffff; /* set WL hi-pri can break BT */
+		select_table = 0x3; /* set Tx response = Hi-Pri
+				     * (ex: Transmitting ACK,BA,CTS)
+				     */
+	} else {
+		break_table = 0xffffff;
+		select_table = 0x3;
+	}
+
+	switch (type) {
+	case 0:
+		halbtc8822b1ant_coex_table(btcoexist, force_exec, 0x55555555,
+					   0x55555555, break_table,
+					   select_table);
+		break;
+	case 1:
+		halbtc8822b1ant_coex_table(btcoexist, force_exec, 0x55555555,
+					   0x5a5a5a5a, break_table,
+					   select_table);
+		break;
+	case 2:
+		halbtc8822b1ant_coex_table(btcoexist, force_exec, 0xaa5a5a5a,
+					   0xaa5a5a5a, break_table,
+					   select_table);
+		break;
+	case 3:
+		halbtc8822b1ant_coex_table(btcoexist, force_exec, 0x55555555,
+					   0xaa5a5a5a, break_table,
+					   select_table);
+		break;
+	case 4:
+		halbtc8822b1ant_coex_table(btcoexist, force_exec, 0xaa555555,
+					   0xaa5a5a5a, break_table,
+					   select_table);
+		break;
+	case 5:
+		halbtc8822b1ant_coex_table(btcoexist, force_exec, 0x5a5a5a5a,
+					   0x5a5a5a5a, break_table,
+					   select_table);
+		break;
+	case 6:
+		halbtc8822b1ant_coex_table(btcoexist, force_exec, 0x55555555,
+					   0xaaaaaaaa, break_table,
+					   select_table);
+		break;
+	case 7:
+		halbtc8822b1ant_coex_table(btcoexist, force_exec, 0xaaaaaaaa,
+					   0xaaaaaaaa, break_table,
+					   select_table);
+		break;
+	case 8:
+		halbtc8822b1ant_coex_table(btcoexist, force_exec, 0xffffffff,
+					   0xffffffff, break_table,
+					   select_table);
+		break;
+	case 9:
+		halbtc8822b1ant_coex_table(btcoexist, force_exec, 0x5a5a5555,
+					   0xaaaa5a5a, break_table,
+					   select_table);
+		break;
+	case 10:
+		halbtc8822b1ant_coex_table(btcoexist, force_exec, 0xaaaa5aaa,
+					   0xaaaa5aaa, break_table,
+					   select_table);
+		break;
+	case 11:
+		halbtc8822b1ant_coex_table(btcoexist, force_exec, 0xaaaaa5aa,
+					   0xaaaaaaaa, break_table,
+					   select_table);
+		break;
+	case 12:
+		halbtc8822b1ant_coex_table(btcoexist, force_exec, 0xaaaaa5aa,
+					   0xaaaaa5aa, break_table,
+					   select_table);
+		break;
+	case 13:
+		halbtc8822b1ant_coex_table(btcoexist, force_exec, 0x55555555,
+					   0xaaaa5a5a, break_table,
+					   select_table);
+		break;
+	case 14:
+		halbtc8822b1ant_coex_table(btcoexist, force_exec, 0x5a5a555a,
+					   0xaaaa5a5a, break_table,
+					   select_table);
+		break;
+	case 15:
+		halbtc8822b1ant_coex_table(btcoexist, force_exec, 0x55555555,
+					   0xaaaa55aa, break_table,
+					   select_table);
+		break;
+	case 16:
+		halbtc8822b1ant_coex_table(btcoexist, force_exec, 0x5a5a555a,
+					   0x5a5a555a, break_table,
+					   select_table);
+		break;
+	case 17:
+		halbtc8822b1ant_coex_table(btcoexist, force_exec, 0xaaaa55aa,
+					   0xaaaa55aa, break_table,
+					   select_table);
+		break;
+	case 18:
+		halbtc8822b1ant_coex_table(btcoexist, force_exec, 0x55555555,
+					   0x5aaa5a5a, break_table,
+					   select_table);
+		break;
+	case 19:
+		halbtc8822b1ant_coex_table(btcoexist, force_exec, 0xa5555555,
+					   0xaaaa5aaa, break_table,
+					   select_table);
+		break;
+	case 20:
+		halbtc8822b1ant_coex_table(btcoexist, force_exec, 0x55555555,
+					   0xaaaa5aaa, break_table,
+					   select_table);
+		break;
+	default:
+		break;
+	}
+}
+
+static void
+halbtc8822b1ant_set_fw_ignore_wlan_act(struct btc_coexist *btcoexist,
+				       bool enable)
+{
+	u8 h2c_parameter[1] = {0};
+
+	if (enable)
+		h2c_parameter[0] |= BIT(0); /* function enable */
+
+	btcoexist->btc_fill_h2c(btcoexist, 0x63, 1, h2c_parameter);
+}
+
+static void halbtc8822b1ant_ignore_wlan_act(struct btc_coexist *btcoexist,
+					    bool force_exec, bool enable)
+{
+	coex_dm->cur_ignore_wlan_act = enable;
+
+	if (!force_exec) {
+		if (coex_dm->pre_ignore_wlan_act ==
+		    coex_dm->cur_ignore_wlan_act) {
+			coex_dm->pre_ignore_wlan_act =
+				coex_dm->cur_ignore_wlan_act;
+			return;
+		}
+	}
+
+	halbtc8822b1ant_set_fw_ignore_wlan_act(btcoexist, enable);
+
+	coex_dm->pre_ignore_wlan_act = coex_dm->cur_ignore_wlan_act;
+}
+
+static void halbtc8822b1ant_set_lps_rpwm(struct btc_coexist *btcoexist,
+					 u8 lps_val, u8 rpwm_val)
+{
+	u8 lps = lps_val;
+	u8 rpwm = rpwm_val;
+
+	btcoexist->btc_set(btcoexist, BTC_SET_U1_LPS_VAL, &lps);
+	btcoexist->btc_set(btcoexist, BTC_SET_U1_RPWM_VAL, &rpwm);
+}
+
+static void halbtc8822b1ant_lps_rpwm(struct btc_coexist *btcoexist,
+				     bool force_exec, u8 lps_val, u8 rpwm_val)
+{
+	coex_dm->cur_lps = lps_val;
+	coex_dm->cur_rpwm = rpwm_val;
+
+	if (!force_exec) {
+		if ((coex_dm->pre_lps == coex_dm->cur_lps) &&
+		    (coex_dm->pre_rpwm == coex_dm->cur_rpwm))
+			return;
+	}
+	halbtc8822b1ant_set_lps_rpwm(btcoexist, lps_val, rpwm_val);
+
+	coex_dm->pre_lps = coex_dm->cur_lps;
+	coex_dm->pre_rpwm = coex_dm->cur_rpwm;
+}
+
+static void halbtc8822b1ant_ps_tdma_check_for_power_save_state(
+	struct btc_coexist *btcoexist, bool new_ps_state)
+{
+	u8 lps_mode = 0x0;
+	u8 h2c_parameter[5] = {0x8, 0, 0, 0, 0};
+
+	btcoexist->btc_get(btcoexist, BTC_GET_U1_LPS_MODE, &lps_mode);
+
+	if (lps_mode) { /* already under LPS state */
+		if (new_ps_state) {
+			/* keep state under LPS, do nothing. */
+		} else {
+			/* will leave LPS state, turn off psTdma first */
+
+			btcoexist->btc_fill_h2c(btcoexist, 0x60, 5,
+						h2c_parameter);
+		}
+	} else { /* NO PS state */
+		if (new_ps_state) {
+			/* will enter LPS state, turn off psTdma first */
+
+			btcoexist->btc_fill_h2c(btcoexist, 0x60, 5,
+						h2c_parameter);
+		} else {
+			/* keep state under NO PS state, do nothing. */
+		}
+	}
+}
+
+static bool halbtc8822b1ant_power_save_state(struct btc_coexist *btcoexist,
+					     u8 ps_type, u8 lps_val,
+					     u8 rpwm_val)
+{
+	bool low_pwr_disable = false, result = true;
+
+	switch (ps_type) {
+	case BTC_PS_WIFI_NATIVE:
+		/* recover to original 32k low power setting */
+		coex_sta->force_lps_ctrl = false;
+		low_pwr_disable = false;
+		btcoexist->btc_set(btcoexist, BTC_SET_ACT_DISABLE_LOW_POWER,
+				   &low_pwr_disable);
+		btcoexist->btc_set(btcoexist, BTC_SET_ACT_NORMAL_LPS, NULL);
+		break;
+	case BTC_PS_LPS_ON:
+
+		coex_sta->force_lps_ctrl = true;
+		halbtc8822b1ant_ps_tdma_check_for_power_save_state(btcoexist,
+								   true);
+		halbtc8822b1ant_lps_rpwm(btcoexist, NORMAL_EXEC, lps_val,
+					 rpwm_val);
+		/* when coex force to enter LPS, do not enter 32k low power. */
+		low_pwr_disable = true;
+		btcoexist->btc_set(btcoexist, BTC_SET_ACT_DISABLE_LOW_POWER,
+				   &low_pwr_disable);
+		/* power save must executed before psTdma. */
+		btcoexist->btc_set(btcoexist, BTC_SET_ACT_ENTER_LPS, NULL);
+
+		break;
+	case BTC_PS_LPS_OFF:
+
+		coex_sta->force_lps_ctrl = true;
+		halbtc8822b1ant_ps_tdma_check_for_power_save_state(btcoexist,
+								   false);
+		result = btcoexist->btc_set(btcoexist, BTC_SET_ACT_LEAVE_LPS,
+					    NULL);
+
+		break;
+	default:
+		break;
+	}
+
+	return result;
+}
+
+static void halbtc8822b1ant_set_fw_pstdma(struct btc_coexist *btcoexist,
+					  u8 byte1, u8 byte2, u8 byte3,
+					  u8 byte4, u8 byte5)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	u8 h2c_parameter[5] = {0};
+	u8 real_byte1 = byte1, real_byte5 = byte5;
+	bool ap_enable = false, result = false;
+	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
+
+	if (byte5 & BIT(2))
+		coex_sta->is_tdma_btautoslot = true;
+	else
+		coex_sta->is_tdma_btautoslot = false;
+
+	/* release bt-auto slot for auto-slot hang is detected!! */
+	if (coex_sta->is_tdma_btautoslot)
+		if ((coex_sta->is_tdma_btautoslot_hang) ||
+		    (bt_link_info->slave_role))
+			byte5 = byte5 & 0xfb;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_AP_MODE_ENABLE,
+			   &ap_enable);
+
+	if ((ap_enable) && (byte1 & BIT(4) && !(byte1 & BIT(5)))) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], %s == FW for 1Ant AP mode\n", __func__);
+
+		real_byte1 &= ~BIT(4);
+		real_byte1 |= BIT(5);
+
+		real_byte5 |= BIT(5);
+		real_byte5 &= ~BIT(6);
+
+		halbtc8822b1ant_power_save_state(btcoexist, BTC_PS_WIFI_NATIVE,
+						 0x0, 0x0);
+
+	} else if (byte1 & BIT(4) && !(byte1 & BIT(5))) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], %s == Force LPS (byte1 = 0x%x)\n",
+			 __func__, byte1);
+		if (!halbtc8822b1ant_power_save_state(btcoexist, BTC_PS_LPS_OFF,
+						      0x50, 0x4))
+			result = true;
+	} else {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], %s == native power save (byte1 = 0x%x)\n",
+			 __func__, byte1);
+		halbtc8822b1ant_power_save_state(btcoexist, BTC_PS_WIFI_NATIVE,
+						 0x0, 0x0);
+	}
+
+	coex_sta->is_set_ps_state_fail = result;
+
+	if (!coex_sta->is_set_ps_state_fail) {
+		h2c_parameter[0] = real_byte1;
+		h2c_parameter[1] = byte2;
+		h2c_parameter[2] = byte3;
+		h2c_parameter[3] = byte4;
+		h2c_parameter[4] = real_byte5;
+
+		coex_dm->ps_tdma_para[0] = real_byte1;
+		coex_dm->ps_tdma_para[1] = byte2;
+		coex_dm->ps_tdma_para[2] = byte3;
+		coex_dm->ps_tdma_para[3] = byte4;
+		coex_dm->ps_tdma_para[4] = real_byte5;
+
+		btcoexist->btc_fill_h2c(btcoexist, 0x60, 5, h2c_parameter);
+
+	} else {
+		coex_sta->cnt_set_ps_state_fail++;
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], %s == Force Leave LPS Fail (cnt = %d)\n",
+			 __func__, coex_sta->cnt_set_ps_state_fail);
+	}
+}
+
+static void halbtc8822b1ant_ps_tdma(struct btc_coexist *btcoexist,
+				    bool force_exec, bool turn_on, u8 type)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
+	bool wifi_busy = false;
+	static u8 ps_tdma_byte4_modify, pre_ps_tdma_byte4_modify;
+	static bool pre_wifi_busy;
+
+	coex_dm->cur_ps_tdma_on = turn_on;
+	coex_dm->cur_ps_tdma = type;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+
+	if (wifi_busy != pre_wifi_busy) {
+		force_exec = true;
+		pre_wifi_busy = wifi_busy;
+	}
+
+	/* 0x778 = 0x1 at wifi slot (no blocking BT Low-Pri pkts) */
+	if (bt_link_info->slave_role)
+		ps_tdma_byte4_modify = 0x1;
+	else
+		ps_tdma_byte4_modify = 0x0;
+
+	if (pre_ps_tdma_byte4_modify != ps_tdma_byte4_modify) {
+		force_exec = true;
+		pre_ps_tdma_byte4_modify = ps_tdma_byte4_modify;
+	}
+
+	if (!force_exec) {
+		if ((coex_dm->pre_ps_tdma_on == coex_dm->cur_ps_tdma_on) &&
+		    (coex_dm->pre_ps_tdma == coex_dm->cur_ps_tdma)) {
+			RT_TRACE(
+				rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				"[BTCoex], Skip TDMA because no change TDMA(%s, %d)\n",
+				(coex_dm->cur_ps_tdma_on ? "on" : "off"),
+				coex_dm->cur_ps_tdma);
+			return;
+		}
+	}
+
+	if (coex_dm->cur_ps_tdma_on) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], ********** TDMA(on, %d) **********\n",
+			 coex_dm->cur_ps_tdma);
+
+		btcoexist->btc_write_1byte_bitmask(
+			btcoexist, 0x550, 0x8, 0x1); /* enable TBTT nterrupt */
+	} else {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], ********** TDMA(off, %d) **********\n",
+			 coex_dm->cur_ps_tdma);
+	}
+
+	if (turn_on) {
+		/* enable TBTT nterrupt */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x550, 0x8, 0x1);
+
+		switch (type) {
+		default:
+			halbtc8822b1ant_set_fw_pstdma(btcoexist, 0x61, 0x35,
+						      0x03, 0x11, 0x11);
+			break;
+		case 1:
+			halbtc8822b1ant_set_fw_pstdma(btcoexist, 0x61, 0x3a,
+						      0x03, 0x11, 0x10);
+			break;
+		case 3:
+			halbtc8822b1ant_set_fw_pstdma(btcoexist, 0x51, 0x30,
+						      0x03, 0x10, 0x50);
+			break;
+		case 4:
+			halbtc8822b1ant_set_fw_pstdma(btcoexist, 0x51, 0x21,
+						      0x03, 0x10, 0x50);
+			break;
+		case 5:
+			halbtc8822b1ant_set_fw_pstdma(btcoexist, 0x61, 0x15,
+						      0x3, 0x11, 0x11);
+			break;
+		case 6:
+			halbtc8822b1ant_set_fw_pstdma(btcoexist, 0x61, 0x20,
+						      0x3, 0x11, 0x10);
+			break;
+		case 7:
+			halbtc8822b1ant_set_fw_pstdma(
+				btcoexist, 0x51, 0x10, 0x03, 0x10,
+				0x54 | ps_tdma_byte4_modify);
+			break;
+		case 8:
+			halbtc8822b1ant_set_fw_pstdma(
+				btcoexist, 0x51, 0x10, 0x03, 0x10,
+				0x14 | ps_tdma_byte4_modify);
+			break;
+		case 11:
+			halbtc8822b1ant_set_fw_pstdma(
+				btcoexist, 0x61, 0x25, 0x03, 0x11,
+				0x10 | ps_tdma_byte4_modify);
+			break;
+		case 12:
+			halbtc8822b1ant_set_fw_pstdma(
+				btcoexist, 0x51, 0x30, 0x03, 0x10,
+				0x50 | ps_tdma_byte4_modify);
+			break;
+		case 13:
+			halbtc8822b1ant_set_fw_pstdma(
+				btcoexist, 0x51, 0x10, 0x07, 0x10,
+				0x54 | ps_tdma_byte4_modify);
+			break;
+		case 14:
+			halbtc8822b1ant_set_fw_pstdma(
+				btcoexist, 0x51, 0x15, 0x03, 0x10,
+				0x50 | ps_tdma_byte4_modify);
+			break;
+		case 15:
+			halbtc8822b1ant_set_fw_pstdma(
+				btcoexist, 0x51, 0x20, 0x03, 0x10,
+				0x10 | ps_tdma_byte4_modify);
+			break;
+		case 17:
+			halbtc8822b1ant_set_fw_pstdma(
+				btcoexist, 0x61, 0x10, 0x03, 0x11,
+				0x14 | ps_tdma_byte4_modify);
+			break;
+		case 18:
+			halbtc8822b1ant_set_fw_pstdma(
+				btcoexist, 0x51, 0x10, 0x03, 0x10,
+				0x50 | ps_tdma_byte4_modify);
+			break;
+
+		case 20:
+			halbtc8822b1ant_set_fw_pstdma(btcoexist, 0x61, 0x30,
+						      0x03, 0x11, 0x10);
+			break;
+		case 22:
+			halbtc8822b1ant_set_fw_pstdma(btcoexist, 0x61, 0x25,
+						      0x03, 0x11, 0x10);
+			break;
+		case 27:
+			halbtc8822b1ant_set_fw_pstdma(btcoexist, 0x61, 0x10,
+						      0x03, 0x11, 0x15);
+			break;
+		case 32:
+			halbtc8822b1ant_set_fw_pstdma(btcoexist, 0x61, 0x35,
+						      0x3, 0x11, 0x11);
+			break;
+		case 33:
+			halbtc8822b1ant_set_fw_pstdma(btcoexist, 0x61, 0x35,
+						      0x03, 0x11, 0x10);
+			break;
+		case 41:
+			halbtc8822b1ant_set_fw_pstdma(btcoexist, 0x51, 0x45,
+						      0x3, 0x11, 0x11);
+			break;
+		case 42:
+			halbtc8822b1ant_set_fw_pstdma(
+				btcoexist, 0x51, 0x1e, 0x3, 0x10,
+				0x14 | ps_tdma_byte4_modify);
+			break;
+		case 43:
+			halbtc8822b1ant_set_fw_pstdma(btcoexist, 0x51, 0x45,
+						      0x3, 0x10, 0x14);
+			break;
+		case 44:
+			halbtc8822b1ant_set_fw_pstdma(btcoexist, 0x51, 0x25,
+						      0x3, 0x10, 0x10);
+			break;
+		case 45:
+			halbtc8822b1ant_set_fw_pstdma(btcoexist, 0x51, 0x29,
+						      0x3, 0x10, 0x10);
+			break;
+		case 46:
+			halbtc8822b1ant_set_fw_pstdma(btcoexist, 0x51, 0x1a,
+						      0x3, 0x10, 0x10);
+			break;
+		case 47:
+			halbtc8822b1ant_set_fw_pstdma(btcoexist, 0x51, 0x32,
+						      0x3, 0x10, 0x10);
+			break;
+		case 48:
+			halbtc8822b1ant_set_fw_pstdma(btcoexist, 0x51, 0x29,
+						      0x3, 0x10, 0x10);
+			break;
+		case 49:
+			halbtc8822b1ant_set_fw_pstdma(btcoexist, 0x55, 0x10,
+						      0x3, 0x10, 0x54);
+			break;
+		case 50:
+			halbtc8822b1ant_set_fw_pstdma(btcoexist, 0x51, 0x4a,
+						      0x3, 0x10, 0x10);
+			break;
+		case 51:
+			halbtc8822b1ant_set_fw_pstdma(btcoexist, 0x61, 0x35,
+						      0x3, 0x10, 0x11);
+			break;
+		}
+	} else {
+		switch (type) {
+		case 0:
+		default: /* Software control, Antenna at BT side */
+			halbtc8822b1ant_set_fw_pstdma(btcoexist, 0x0, 0x0, 0x0,
+						      0x0, 0x0);
+			break;
+		case 8: /* PTA Control */
+			halbtc8822b1ant_set_fw_pstdma(btcoexist, 0x8, 0x0, 0x0,
+						      0x0, 0x0);
+			break;
+		case 9: /* Software control, Antenna at WiFi side */
+			halbtc8822b1ant_set_fw_pstdma(btcoexist, 0x0, 0x0, 0x0,
+						      0x0, 0x0);
+			break;
+		case 10: /* under 5G , 0x778=1*/
+			halbtc8822b1ant_set_fw_pstdma(btcoexist, 0x0, 0x0, 0x0,
+						      0x0, 0x0);
+
+			break;
+		}
+	}
+
+	if (!coex_sta->is_set_ps_state_fail) {
+		/* update pre state */
+		coex_dm->pre_ps_tdma_on = coex_dm->cur_ps_tdma_on;
+		coex_dm->pre_ps_tdma = coex_dm->cur_ps_tdma;
+	}
+}
+
+static void halbtc8822b1ant_sw_mechanism(struct btc_coexist *btcoexist,
+					 bool low_penalty_ra)
+{
+	halbtc8822b1ant_low_penalty_ra(btcoexist, NORMAL_EXEC, low_penalty_ra);
+}
+
+/* rf4 type by efuse, and for ant at main aux inverse use,
+ * because is 2x2, and control types are the same, does not need
+ */
+
+static void halbtc8822b1ant_set_rfe_type(struct btc_coexist *btcoexist)
+{
+	struct btc_board_info *board_info = &btcoexist->board_info;
+
+	/* Ext switch buffer mux */
+	btcoexist->btc_write_1byte(btcoexist, 0x974, 0xff);
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0x1991, 0x3, 0x0);
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcbe, 0x8, 0x0);
+
+	/* the following setup should be got from Efuse in the future */
+	rfe_type->rfe_module_type = board_info->rfe_type;
+
+	rfe_type->ext_ant_switch_ctrl_polarity = 0;
+
+	switch (rfe_type->rfe_module_type) {
+	case 0:
+	default:
+		rfe_type->ext_ant_switch_exist = true;
+		rfe_type->ext_ant_switch_type =
+			BT_8822B_1ANT_EXT_ANT_SWITCH_USE_SPDT;
+		break;
+	case 1:
+		rfe_type->ext_ant_switch_exist = true;
+		rfe_type->ext_ant_switch_type =
+			BT_8822B_1ANT_EXT_ANT_SWITCH_USE_SPDT;
+		break;
+	case 2:
+		rfe_type->ext_ant_switch_exist = true;
+		rfe_type->ext_ant_switch_type =
+			BT_8822B_1ANT_EXT_ANT_SWITCH_USE_SPDT;
+		break;
+	case 3:
+		rfe_type->ext_ant_switch_exist = true;
+		rfe_type->ext_ant_switch_type =
+			BT_8822B_1ANT_EXT_ANT_SWITCH_USE_SPDT;
+		break;
+	case 4:
+		rfe_type->ext_ant_switch_exist = true;
+		rfe_type->ext_ant_switch_type =
+			BT_8822B_1ANT_EXT_ANT_SWITCH_USE_SPDT;
+		break;
+	case 5:
+		rfe_type->ext_ant_switch_exist = true;
+		rfe_type->ext_ant_switch_type =
+			BT_8822B_1ANT_EXT_ANT_SWITCH_USE_SPDT;
+		break;
+	case 6:
+		rfe_type->ext_ant_switch_exist = true;
+		rfe_type->ext_ant_switch_type =
+			BT_8822B_1ANT_EXT_ANT_SWITCH_USE_SPDT;
+		break;
+	case 7:
+		rfe_type->ext_ant_switch_exist = true;
+		rfe_type->ext_ant_switch_type =
+			BT_8822B_1ANT_EXT_ANT_SWITCH_USE_SPDT;
+		break;
+	}
+}
+
+/*anttenna control by bb mac bt antdiv pta to write 0x4c 0xcb4,0xcbd*/
+
+static void halbtc8822b1ant_set_ext_ant_switch(struct btc_coexist *btcoexist,
+					       bool force_exec, u8 ctrl_type,
+					       u8 pos_type)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	bool switch_polatiry_inverse = false;
+	u8 regval_0xcbd = 0, regval_0x64;
+	u32 u32tmp1 = 0, u32tmp2 = 0, u32tmp3 = 0;
+
+	/* Ext switch buffer mux */
+	btcoexist->btc_write_1byte(btcoexist, 0x974, 0xff);
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0x1991, 0x3, 0x0);
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcbe, 0x8, 0x0);
+
+	if (!rfe_type->ext_ant_switch_exist)
+		return;
+
+	coex_dm->cur_ext_ant_switch_status = (ctrl_type << 8) + pos_type;
+
+	if (!force_exec) {
+		if (coex_dm->pre_ext_ant_switch_status ==
+		    coex_dm->cur_ext_ant_switch_status)
+			return;
+	}
+
+	coex_dm->pre_ext_ant_switch_status = coex_dm->cur_ext_ant_switch_status;
+
+	/* swap control polarity if use different switch control polarity*/
+	/* Normal switch polarity for SPDT,
+	 * 0xcbd[1:0] = 2b'01 => Ant to BTG,
+	 * 0xcbd[1:0] = 2b'10 => Ant to WLG
+	 */
+	switch_polatiry_inverse = rfe_type->ext_ant_switch_ctrl_polarity == 1;
+
+	switch (pos_type) {
+	default:
+	case BT_8822B_1ANT_EXT_ANT_SWITCH_TO_BT:
+	case BT_8822B_1ANT_EXT_ANT_SWITCH_TO_NOCARE:
+
+		break;
+	case BT_8822B_1ANT_EXT_ANT_SWITCH_TO_WLG:
+		break;
+	case BT_8822B_1ANT_EXT_ANT_SWITCH_TO_WLA:
+		break;
+	}
+
+	if (rfe_type->ext_ant_switch_type ==
+	    BT_8822B_1ANT_EXT_ANT_SWITCH_USE_SPDT) {
+		switch (ctrl_type) {
+		default:
+		case BT_8822B_1ANT_EXT_ANT_SWITCH_CTRL_BY_BBSW:
+			/*  0x4c[23] = 0 */
+			btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4e,
+							   0x80, 0x0);
+			/* 0x4c[24] = 1 */
+			btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4f,
+							   0x01, 0x1);
+			/* BB SW, DPDT use RFE_ctrl8 and RFE_ctrl9 as ctrl pin*/
+			btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcb4,
+							   0xff, 0x77);
+
+			/* 0xcbd[1:0] = 2b'01 for no switch_polatiry_inverse,
+			 * ANTSWB =1, ANTSW =0
+			 */
+			regval_0xcbd = (!switch_polatiry_inverse ? 0x1 : 0x2);
+			btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcbd,
+							   0x3, regval_0xcbd);
+
+			break;
+		case BT_8822B_1ANT_EXT_ANT_SWITCH_CTRL_BY_PTA:
+			/* 0x4c[23] = 0 */
+			btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4e,
+							   0x80, 0x0);
+			/* 0x4c[24] = 1 */
+			btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4f,
+							   0x01, 0x1);
+			/* PTA,  DPDT use RFE_ctrl8 and RFE_ctrl9 as ctrl pin */
+			btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcb4,
+							   0xff, 0x66);
+
+			/* 0xcbd[1:0] = 2b'10 for no switch_polatiry_inverse,
+			 * ANTSWB =1, ANTSW =0  @ GNT_BT=1
+			 */
+			regval_0xcbd = (!switch_polatiry_inverse ? 0x2 : 0x1);
+			btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcbd,
+							   0x3, regval_0xcbd);
+
+			break;
+		case BT_8822B_1ANT_EXT_ANT_SWITCH_CTRL_BY_ANTDIV:
+			/* 0x4c[23] = 0 */
+			btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4e,
+							   0x80, 0x0);
+			/* 0x4c[24] = 1 */
+			btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4f,
+							   0x01, 0x1);
+			btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcb4,
+							   0xff, 0x88);
+
+			/* no regval_0xcbd setup required, because
+			 * antenna switch control value by antenna diversity
+			 */
+
+			break;
+		case BT_8822B_1ANT_EXT_ANT_SWITCH_CTRL_BY_MAC:
+			/*  0x4c[23] = 1 */
+			btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4e,
+							   0x80, 0x1);
+
+			/* 0x64[0] = 1b'0 for no switch_polatiry_inverse,
+			 * DPDT_SEL_N =1, DPDT_SEL_P =0
+			 */
+			regval_0x64 = (!switch_polatiry_inverse ? 0x0 : 0x1);
+			btcoexist->btc_write_1byte_bitmask(btcoexist, 0x64, 0x1,
+							   regval_0x64);
+			break;
+		case BT_8822B_1ANT_EXT_ANT_SWITCH_CTRL_BY_BT:
+			/* 0x4c[23] = 0 */
+			btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4e,
+							   0x80, 0x0);
+			/* 0x4c[24] = 0 */
+			btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4f,
+							   0x01, 0x0);
+
+			/* no setup required, because antenna switch control
+			 * value by BT vendor 0xac[1:0]
+			 */
+			break;
+		}
+	}
+
+	u32tmp1 = btcoexist->btc_read_4byte(btcoexist, 0xcbc);
+	u32tmp2 = btcoexist->btc_read_4byte(btcoexist, 0x4c);
+	u32tmp3 = btcoexist->btc_read_4byte(btcoexist, 0x64) & 0xff;
+
+	RT_TRACE(
+		rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		"[BTCoex], ********** (After Ext Ant switch setup) 0xcbc = 0x%08x, 0x4c = 0x%08x, 0x64= 0x%02x**********\n",
+		u32tmp1, u32tmp2, u32tmp3);
+}
+
+/* set gnt_wl gnt_bt control by sw high low, or
+ * hwpta while in power on, ini, wlan off, wlan only, wl2g non-currrent,
+ * wl2g current, wl5g
+ */
+
+static void halbtc8822b1ant_set_ant_path(struct btc_coexist *btcoexist,
+					 u8 ant_pos_type, bool force_exec,
+					 u8 phase)
+
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	u8 u8tmp = 0;
+	u32 u32tmp1 = 0;
+	u32 u32tmp2 = 0, u32tmp3 = 0;
+
+	u32tmp1 = halbtc8822b1ant_ltecoex_indirect_read_reg(btcoexist, 0x38);
+
+	/* To avoid indirect access fail  */
+	if (((u32tmp1 & 0xf000) >> 12) != ((u32tmp1 & 0x0f00) >> 8)) {
+		force_exec = true;
+		coex_sta->gnt_error_cnt++;
+
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex],(Before Ant Setup) 0x38= 0x%x\n", u32tmp1);
+	}
+
+	/* Ext switch buffer mux */
+	btcoexist->btc_write_1byte(btcoexist, 0x974, 0xff);
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0x1991, 0x3, 0x0);
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcbe, 0x8, 0x0);
+
+	coex_dm->cur_ant_pos_type = (ant_pos_type << 8) + phase;
+
+	if (!force_exec) {
+		if (coex_dm->cur_ant_pos_type == coex_dm->pre_ant_pos_type)
+			return;
+	}
+
+	coex_dm->pre_ant_pos_type = coex_dm->cur_ant_pos_type;
+
+	if (btcoexist->dbg_mode_1ant) {
+		u32tmp1 = halbtc8822b1ant_ltecoex_indirect_read_reg(btcoexist,
+								    0x38);
+		u32tmp2 = halbtc8822b1ant_ltecoex_indirect_read_reg(btcoexist,
+								    0x54);
+		u32tmp3 = btcoexist->btc_read_4byte(btcoexist, 0xcb4);
+
+		u8tmp = btcoexist->btc_read_1byte(btcoexist, 0x73);
+
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], ********** (Before Ant Setup) 0xcb4 = 0x%x, 0x73 = 0x%x, 0x38= 0x%x, 0x54= 0x%x**********\n",
+			u32tmp3, u8tmp, u32tmp1, u32tmp2);
+	}
+
+	switch (phase) {
+	case BT_8822B_1ANT_PHASE_COEX_INIT:
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], ********** (set_ant_path - 1ANT_PHASE_COEX_INIT) **********\n");
+
+		/* Disable LTE Coex Function in WiFi side
+		 * (this should be on if LTE coex is required)
+		 */
+		halbtc8822b1ant_ltecoex_enable(btcoexist, 0x0);
+
+		/* GNT_WL_LTE always = 1
+		 * (this should be config if LTE coex is required)
+		 */
+		halbtc8822b1ant_ltecoex_set_coex_table(
+			btcoexist, BT_8822B_1ANT_CTT_WL_VS_LTE, 0xffff);
+
+		/* GNT_BT_LTE always = 1
+		 * (this should be config if LTE coex is required)
+		 */
+		halbtc8822b1ant_ltecoex_set_coex_table(
+			btcoexist, BT_8822B_1ANT_CTT_BT_VS_LTE, 0xffff);
+
+		/* set GNT_BT to SW high */
+		halbtc8822b1ant_ltecoex_set_gnt_bt(
+			btcoexist, BT_8822B_1ANT_GNT_BLOCK_RFC_BB,
+			BT_8822B_1ANT_GNT_CTRL_BY_SW,
+			BT_8822B_1ANT_SIG_STA_SET_TO_HIGH);
+
+		/* set GNT_WL to SW low */
+		halbtc8822b1ant_ltecoex_set_gnt_wl(
+			btcoexist, BT_8822B_1ANT_GNT_BLOCK_RFC_BB,
+			BT_8822B_1ANT_GNT_CTRL_BY_SW,
+			BT_8822B_1ANT_SIG_STA_SET_TO_LOW);
+
+		/* set Path control owner to WL at initial step */
+		halbtc8822b1ant_ltecoex_pathcontrol_owner(
+			btcoexist, BT_8822B_1ANT_PCO_WLSIDE);
+
+		coex_sta->run_time_state = false;
+
+		/* Ext switch buffer mux */
+		btcoexist->btc_write_1byte(btcoexist, 0x974, 0xff);
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x1991, 0x3, 0x0);
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcbe, 0x8, 0x0);
+
+		if (ant_pos_type == BTC_ANT_PATH_AUTO)
+			ant_pos_type = BTC_ANT_PATH_BT;
+
+		break;
+	case BT_8822B_1ANT_PHASE_WLANONLY_INIT:
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], ********** (set_ant_path - 1ANT_PHASE_WLANONLY_INIT) **********\n");
+
+		/* Disable LTE Coex Function in WiFi side
+		 * (this should be on if LTE coex is required)
+		 */
+		halbtc8822b1ant_ltecoex_enable(btcoexist, 0x0);
+
+		/* GNT_WL_LTE always = 1
+		 * (this should be config if LTE coex is required)
+		 */
+		halbtc8822b1ant_ltecoex_set_coex_table(
+			btcoexist, BT_8822B_1ANT_CTT_WL_VS_LTE, 0xffff);
+
+		/* GNT_BT_LTE always = 1
+		 * (this should be config if LTE coex is required)
+		 */
+		halbtc8822b1ant_ltecoex_set_coex_table(
+			btcoexist, BT_8822B_1ANT_CTT_BT_VS_LTE, 0xffff);
+
+		/* set GNT_BT to SW Low */
+		halbtc8822b1ant_ltecoex_set_gnt_bt(
+			btcoexist, BT_8822B_1ANT_GNT_BLOCK_RFC_BB,
+			BT_8822B_1ANT_GNT_CTRL_BY_SW,
+			BT_8822B_1ANT_SIG_STA_SET_TO_LOW);
+
+		/* Set GNT_WL to SW high */
+		halbtc8822b1ant_ltecoex_set_gnt_wl(
+			btcoexist, BT_8822B_1ANT_GNT_BLOCK_RFC_BB,
+			BT_8822B_1ANT_GNT_CTRL_BY_SW,
+			BT_8822B_1ANT_SIG_STA_SET_TO_HIGH);
+
+		/* set Path control owner to WL at initial step */
+		halbtc8822b1ant_ltecoex_pathcontrol_owner(
+			btcoexist, BT_8822B_1ANT_PCO_WLSIDE);
+
+		coex_sta->run_time_state = false;
+
+		/* Ext switch buffer mux */
+		btcoexist->btc_write_1byte(btcoexist, 0x974, 0xff);
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x1991, 0x3, 0x0);
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcbe, 0x8, 0x0);
+
+		if (ant_pos_type == BTC_ANT_PATH_AUTO)
+			ant_pos_type = BTC_ANT_PATH_WIFI;
+
+		break;
+	case BT_8822B_1ANT_PHASE_WLAN_OFF:
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], ********** (set_ant_path - 1ANT_PHASE_WLAN_OFF) **********\n");
+
+		/* Disable LTE Coex Function in WiFi side */
+		halbtc8822b1ant_ltecoex_enable(btcoexist, 0x0);
+
+		/* set Path control owner to BT */
+		halbtc8822b1ant_ltecoex_pathcontrol_owner(
+			btcoexist, BT_8822B_1ANT_PCO_BTSIDE);
+
+		/* Set Ext Ant Switch to BT control at wifi off step */
+		halbtc8822b1ant_set_ext_ant_switch(
+			btcoexist, FORCE_EXEC,
+			BT_8822B_1ANT_EXT_ANT_SWITCH_CTRL_BY_BT,
+			BT_8822B_1ANT_EXT_ANT_SWITCH_TO_NOCARE);
+
+		coex_sta->run_time_state = false;
+
+		break;
+	case BT_8822B_1ANT_PHASE_2G_RUNTIME:
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], ********** (set_ant_path - 1ANT_PHASE_2G_RUNTIME) **********\n");
+
+		/* set GNT_BT to PTA */
+		halbtc8822b1ant_ltecoex_set_gnt_bt(
+			btcoexist, BT_8822B_1ANT_GNT_BLOCK_RFC_BB,
+			BT_8822B_1ANT_GNT_CTRL_BY_PTA,
+			BT_8822B_1ANT_SIG_STA_SET_BY_HW);
+
+		/* Set GNT_WL to PTA */
+		halbtc8822b1ant_ltecoex_set_gnt_wl(
+			btcoexist, BT_8822B_1ANT_GNT_BLOCK_RFC_BB,
+			BT_8822B_1ANT_GNT_CTRL_BY_PTA,
+			BT_8822B_1ANT_SIG_STA_SET_BY_HW);
+
+		/* set Path control owner to WL at runtime step */
+		halbtc8822b1ant_ltecoex_pathcontrol_owner(
+			btcoexist, BT_8822B_1ANT_PCO_WLSIDE);
+
+		coex_sta->run_time_state = true;
+
+		if (ant_pos_type == BTC_ANT_PATH_AUTO)
+			ant_pos_type = BTC_ANT_PATH_PTA;
+
+		break;
+	case BT_8822B_1ANT_PHASE_5G_RUNTIME:
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], ********** (set_ant_path - 1ANT_PHASE_5G_RUNTIME) **********\n");
+
+		/* set GNT_BT to SW Hi */
+		halbtc8822b1ant_ltecoex_set_gnt_bt(
+			btcoexist, BT_8822B_1ANT_GNT_BLOCK_RFC_BB,
+			BT_8822B_1ANT_GNT_CTRL_BY_SW,
+			BT_8822B_1ANT_SIG_STA_SET_TO_HIGH);
+
+		/* Set GNT_WL to SW Hi */
+		halbtc8822b1ant_ltecoex_set_gnt_wl(
+			btcoexist, BT_8822B_1ANT_GNT_BLOCK_RFC_BB,
+			BT_8822B_1ANT_GNT_CTRL_BY_SW,
+			BT_8822B_1ANT_SIG_STA_SET_TO_HIGH);
+
+		/* set Path control owner to WL at runtime step */
+		halbtc8822b1ant_ltecoex_pathcontrol_owner(
+			btcoexist, BT_8822B_1ANT_PCO_WLSIDE);
+
+		coex_sta->run_time_state = true;
+
+		if (ant_pos_type == BTC_ANT_PATH_AUTO)
+			ant_pos_type = BTC_ANT_PATH_WIFI5G;
+
+		break;
+	case BT_8822B_1ANT_PHASE_BTMPMODE:
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], ********** (set_ant_path - 1ANT_PHASE_BTMPMODE) **********\n");
+
+		/* Disable LTE Coex Function in WiFi side */
+		halbtc8822b1ant_ltecoex_enable(btcoexist, 0x0);
+
+		/* set GNT_BT to SW Hi */
+		halbtc8822b1ant_ltecoex_set_gnt_bt(
+			btcoexist, BT_8822B_1ANT_GNT_BLOCK_RFC_BB,
+			BT_8822B_1ANT_GNT_CTRL_BY_SW,
+			BT_8822B_1ANT_SIG_STA_SET_TO_HIGH);
+
+		/* Set GNT_WL to SW Lo */
+		halbtc8822b1ant_ltecoex_set_gnt_wl(
+			btcoexist, BT_8822B_1ANT_GNT_BLOCK_RFC_BB,
+			BT_8822B_1ANT_GNT_CTRL_BY_SW,
+			BT_8822B_1ANT_SIG_STA_SET_TO_LOW);
+
+		/* set Path control owner to WL */
+		halbtc8822b1ant_ltecoex_pathcontrol_owner(
+			btcoexist, BT_8822B_1ANT_PCO_WLSIDE);
+
+		coex_sta->run_time_state = false;
+
+		/* Set Ext Ant Switch to BT side at BT MP mode */
+		if (ant_pos_type == BTC_ANT_PATH_AUTO)
+			ant_pos_type = BTC_ANT_PATH_BT;
+
+		break;
+	}
+
+	if (phase != BT_8822B_1ANT_PHASE_WLAN_OFF) {
+		switch (ant_pos_type) {
+		case BTC_ANT_PATH_WIFI:
+			halbtc8822b1ant_set_ext_ant_switch(
+				btcoexist, force_exec,
+				BT_8822B_1ANT_EXT_ANT_SWITCH_CTRL_BY_BBSW,
+				BT_8822B_1ANT_EXT_ANT_SWITCH_TO_WLG);
+			break;
+		case BTC_ANT_PATH_WIFI5G:
+			halbtc8822b1ant_set_ext_ant_switch(
+				btcoexist, force_exec,
+				BT_8822B_1ANT_EXT_ANT_SWITCH_CTRL_BY_BBSW,
+				BT_8822B_1ANT_EXT_ANT_SWITCH_TO_WLA);
+			break;
+		case BTC_ANT_PATH_BT:
+			halbtc8822b1ant_set_ext_ant_switch(
+				btcoexist, force_exec,
+				BT_8822B_1ANT_EXT_ANT_SWITCH_CTRL_BY_BBSW,
+				BT_8822B_1ANT_EXT_ANT_SWITCH_TO_BT);
+			break;
+		default:
+		case BTC_ANT_PATH_PTA:
+			halbtc8822b1ant_set_ext_ant_switch(
+				btcoexist, force_exec,
+				BT_8822B_1ANT_EXT_ANT_SWITCH_CTRL_BY_PTA,
+				BT_8822B_1ANT_EXT_ANT_SWITCH_TO_NOCARE);
+			break;
+		}
+	}
+
+	if (btcoexist->dbg_mode_1ant) {
+		u32tmp1 = halbtc8822b1ant_ltecoex_indirect_read_reg(btcoexist,
+								    0x38);
+		u32tmp2 = halbtc8822b1ant_ltecoex_indirect_read_reg(btcoexist,
+								    0x54);
+		u32tmp3 = btcoexist->btc_read_4byte(btcoexist, 0xcb4);
+
+		u8tmp = btcoexist->btc_read_1byte(btcoexist, 0x73);
+
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], ********** (After Ant Setup) 0xcb4 = 0x%x, 0x73 = 0x%x, 0x38= 0x%x, 0x54= 0x%x**********\n",
+			u32tmp3, u8tmp, u32tmp1, u32tmp2);
+	}
+}
+
+static bool halbtc8822b1ant_is_common_action(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	bool common = false, wifi_connected = false, wifi_busy = false;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_CONNECTED,
+			   &wifi_connected);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+
+	if (!wifi_connected &&
+	    coex_dm->bt_status == BT_8822B_1ANT_BT_STATUS_NON_CONNECTED_IDLE) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], Wifi non connected-idle + BT non connected-idle!!\n");
+
+		/* halbtc8822b1ant_sw_mechanism(btcoexist, false); */
+
+		common = true;
+	} else if (wifi_connected &&
+		   (coex_dm->bt_status ==
+		    BT_8822B_1ANT_BT_STATUS_NON_CONNECTED_IDLE)) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], Wifi connected + BT non connected-idle!!\n");
+
+		/* halbtc8822b1ant_sw_mechanism(btcoexist, false); */
+
+		common = true;
+	} else if (!wifi_connected && (BT_8822B_1ANT_BT_STATUS_CONNECTED_IDLE ==
+				       coex_dm->bt_status)) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], Wifi non connected-idle + BT connected-idle!!\n");
+
+		/* halbtc8822b1ant_sw_mechanism(btcoexist, false); */
+
+		common = true;
+	} else if (wifi_connected && (BT_8822B_1ANT_BT_STATUS_CONNECTED_IDLE ==
+				      coex_dm->bt_status)) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], Wifi connected + BT connected-idle!!\n");
+
+		/* halbtc8822b1ant_sw_mechanism(btcoexist, false); */
+
+		common = true;
+	} else if (!wifi_connected && (BT_8822B_1ANT_BT_STATUS_CONNECTED_IDLE !=
+				       coex_dm->bt_status)) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], Wifi non connected-idle + BT Busy!!\n");
+
+		/* halbtc8822b1ant_sw_mechanism(btcoexist, false); */
+
+		common = true;
+	} else {
+		if (wifi_busy) {
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], Wifi Connected-Busy + BT Busy!!\n");
+		} else {
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], Wifi Connected-Idle + BT Busy!!\n");
+		}
+
+		common = false;
+	}
+
+	return common;
+}
+
+static void halbtc8822b1ant_action_wifi_under5g(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], under 5g start\n");
+	/* for test : s3 bt disappear , fail rate 1/600*/
+	/*set sw gnt wl bt  high*/
+	halbtc8822b1ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO, FORCE_EXEC,
+				     BT_8822B_1ANT_PHASE_5G_RUNTIME);
+
+	halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 8);
+	halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 0);
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcbd, 0x3, 1);
+}
+
+static void halbtc8822b1ant_action_wifi_only(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	bool wifi_under_5g = false, rf4ce_enabled = false;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_5G, &wifi_under_5g);
+	if (wifi_under_5g) {
+		halbtc8822b1ant_action_wifi_under5g(btcoexist);
+
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], ********** (wlan only -- under 5g ) **********\n");
+		return;
+	}
+
+	if (rf4ce_enabled) {
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x45e, 0x8, 0x1);
+
+		halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 50);
+
+		halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 1);
+		return;
+	}
+	halbtc8822b1ant_coex_table_with_type(btcoexist, FORCE_EXEC, 0);
+	halbtc8822b1ant_ps_tdma(btcoexist, FORCE_EXEC, false, 8);
+
+	halbtc8822b1ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO, FORCE_EXEC,
+				     BT_8822B_1ANT_PHASE_2G_RUNTIME);
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], ********** (wlan only -- under 2g ) **********\n");
+}
+
+static void
+halbtc8822b1ant_action_wifi_native_lps(struct btc_coexist *btcoexist)
+{
+	halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 5);
+
+	halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 8);
+}
+
+/* *********************************************
+ *
+ *	Non-Software Coex Mechanism start
+ *
+ * **********************************************/
+
+static void halbtc8822b1ant_action_bt_whck_test(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex],action_bt_whck_test\n");
+
+	halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 8);
+
+	halbtc8822b1ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO, NORMAL_EXEC,
+				     BT_8822B_1ANT_PHASE_2G_RUNTIME);
+
+	halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 0);
+}
+
+static void
+halbtc8822b1ant_action_wifi_multi_port(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex],action_wifi_multi_port\n");
+
+	halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 8);
+
+	halbtc8822b1ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO, NORMAL_EXEC,
+				     BT_8822B_1ANT_PHASE_2G_RUNTIME);
+
+	halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 0);
+}
+
+static void halbtc8822b1ant_action_hs(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD, "[BTCoex], action_hs\n");
+
+	halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 5);
+
+	halbtc8822b1ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO, NORMAL_EXEC,
+				     BT_8822B_1ANT_PHASE_2G_RUNTIME);
+
+	halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 5);
+}
+
+static void halbtc8822b1ant_action_bt_relink(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], run bt multi link function\n");
+
+	if (coex_sta->is_bt_multi_link)
+		return;
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], run bt_re-link function\n");
+
+	halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 8);
+	halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 0);
+}
+
+/*"""bt inquiry"""" + wifi any + bt any*/
+
+static void halbtc8822b1ant_action_bt_inquiry(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
+	bool wifi_connected = false, ap_enable = false, wifi_busy = false,
+	     bt_busy = false, rf4ce_enabled = false;
+
+	bool wifi_scan = false, link = false, roam = false;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], ********** (bt inquiry) **********\n");
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_AP_MODE_ENABLE,
+			   &ap_enable);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_CONNECTED,
+			   &wifi_connected);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+	btcoexist->btc_set(btcoexist, BTC_SET_BL_BT_TRAFFIC_BUSY, &bt_busy);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_SCAN, &wifi_scan);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_LINK, &link);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_ROAM, &roam);
+
+	RT_TRACE(
+		rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		"[BTCoex], ********** scan = %d,  link =%d, roam = %d**********\n",
+		wifi_scan, link, roam);
+
+	if ((link) || (roam) || (coex_sta->wifi_is_high_pri_task)) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], ********** (bt inquiry wifi  connect or scan ) **********\n");
+
+		halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 1);
+
+		halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 6);
+
+	} else if ((wifi_scan) && (coex_sta->bt_create_connection)) {
+		halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 22);
+		halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 6);
+
+	} else if ((!wifi_connected) && (!wifi_scan)) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], ********** (bt inquiry wifi non connect) **********\n");
+
+		halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 8);
+
+		halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 0);
+
+	} else if ((bt_link_info->a2dp_exist) && (bt_link_info->pan_exist)) {
+		halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 22);
+		halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 4);
+	} else if (bt_link_info->a2dp_exist) {
+		halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 32);
+
+		halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 3);
+	} else if (wifi_scan) {
+		halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 20);
+
+		halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 4);
+	} else if (wifi_busy) {
+		/* for BT inquiry/page fail after S4 resume */
+		halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 32);
+		/*aaaa->55aa for bt connect while wl busy*/
+		halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC,
+						     15);
+		if (rf4ce_enabled) {
+			btcoexist->btc_write_1byte_bitmask(btcoexist, 0x45e,
+							   0x8, 0x1);
+
+			halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						50);
+
+			halbtc8822b1ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 0);
+		}
+	} else {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], ********** (bt inquiry wifi connect) **********\n");
+
+		halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 22);
+
+		halbtc8822b1ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO,
+					     NORMAL_EXEC,
+					     BT_8822B_1ANT_PHASE_2G_RUNTIME);
+
+		halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 4);
+	}
+}
+
+static void
+halbtc8822b1ant_action_bt_sco_hid_only_busy(struct btc_coexist *btcoexist)
+{
+	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
+	bool wifi_connected = false, wifi_busy = false;
+	u32 wifi_bw = 1;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_CONNECTED,
+			   &wifi_connected);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_BW, &wifi_bw);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+
+	if (bt_link_info->sco_exist) {
+		halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 5);
+		halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 5);
+	} else {
+		if (coex_sta->is_hid_low_pri_tx_overhead) {
+			halbtc8822b1ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 6);
+			halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						18);
+		} else if (wifi_bw == 0) { /* if 11bg mode */
+
+			if (coex_sta->is_bt_multi_link) {
+				halbtc8822b1ant_coex_table_with_type(
+					btcoexist, NORMAL_EXEC, 11);
+				halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC,
+							true, 11);
+			} else {
+				halbtc8822b1ant_coex_table_with_type(
+					btcoexist, NORMAL_EXEC, 6);
+				halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC,
+							true, 11);
+			}
+		} else {
+			halbtc8822b1ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 6);
+			halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						11);
+		}
+	}
+}
+
+static void
+halbtc8822b1ant_action_wifi_connected_bt_acl_busy(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
+	bool wifi_busy = false, wifi_turbo = false;
+	u32 wifi_bw = 1;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_BW, &wifi_bw);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+	btcoexist->btc_get(btcoexist, BTC_GET_U1_AP_NUM,
+			   &coex_sta->scan_ap_num);
+	RT_TRACE(
+		rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		"############# [BTCoex],  scan_ap_num = %d, wl_noisy_level = %d\n",
+		coex_sta->scan_ap_num, coex_sta->wl_noisy_level);
+
+	if ((wifi_busy) && (coex_sta->wl_noisy_level == 0))
+		wifi_turbo = true;
+
+	if ((coex_sta->bt_relink_downcount != 0) &&
+	    (!bt_link_info->pan_exist) && (wifi_busy)) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"############# [BTCoex],  BT Re-Link + A2DP + WL busy\n");
+
+		halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 8);
+		halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 0);
+
+	} else if ((bt_link_info->a2dp_exist) && (coex_sta->is_bt_a2dp_sink)) {
+		halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 12);
+		halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 6);
+	} else if (bt_link_info->a2dp_only) { /* A2DP		 */
+
+		halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 7);
+
+		if (wifi_turbo)
+			halbtc8822b1ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 19);
+		else
+			halbtc8822b1ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 4);
+	} else if (((bt_link_info->a2dp_exist) && (bt_link_info->pan_exist)) ||
+		   (bt_link_info->hid_exist && bt_link_info->a2dp_exist &&
+		    bt_link_info->pan_exist)) {
+		/* A2DP+PAN(OPP,FTP), HID+A2DP+PAN(OPP,FTP) */
+
+		if (wifi_busy)
+			halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						13);
+		else
+			halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						14);
+
+		if (bt_link_info->hid_exist)
+			halbtc8822b1ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 1);
+		else if (wifi_turbo)
+			halbtc8822b1ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 19);
+		else
+			halbtc8822b1ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 4);
+	} else if (bt_link_info->hid_exist &&
+		   bt_link_info->a2dp_exist) { /* HID+A2DP */
+
+		if (wifi_bw == 0) { /* if 11bg mode */
+			halbtc8822b1ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 1);
+			halbtc8822b1ant_set_wltoggle_coex_table(
+				btcoexist, NORMAL_EXEC, 1, 0xaa, 0x5a, 0xaa,
+				0xaa);
+			halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						49);
+		} else {
+			halbtc8822b1ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 1);
+			halbtc8822b1ant_limited_rx(btcoexist, NORMAL_EXEC,
+						   false, true, 8);
+			halbtc8822b1ant_set_wltoggle_coex_table(
+				btcoexist, NORMAL_EXEC, 1, 0xaa, 0x5a, 0xaa,
+				0xaa);
+			halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						49);
+		}
+		/* PAN(OPP,FTP), HID+PAN(OPP,FTP) */
+
+	} else if ((bt_link_info->pan_only) ||
+		   (bt_link_info->hid_exist && bt_link_info->pan_exist)) {
+		if (!wifi_busy)
+			halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						4);
+		else
+			halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						3);
+
+		if (bt_link_info->hid_exist)
+			halbtc8822b1ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 1);
+		else if (wifi_turbo)
+			halbtc8822b1ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 19);
+		else
+			halbtc8822b1ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 4);
+	} else {
+		/* BT no-profile busy (0x9) */
+		halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 33);
+		halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 4);
+	}
+}
+
+/*wifi not connected + bt action*/
+
+static void
+halbtc8822b1ant_action_wifi_not_connected(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	bool rf4ce_enabled = false;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], ********** (wifi not connect) **********\n");
+
+	/* tdma and coex table */
+	if (rf4ce_enabled) {
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x45e, 0x8, 0x1);
+
+		halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 50);
+
+		halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 1);
+		return;
+	}
+	halbtc8822b1ant_ps_tdma(btcoexist, FORCE_EXEC, false, 8);
+
+	halbtc8822b1ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO, NORMAL_EXEC,
+				     BT_8822B_1ANT_PHASE_2G_RUNTIME);
+
+	halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 0);
+}
+
+/*""""wl not connected scan"""" + bt action*/
+static void
+halbtc8822b1ant_action_wifi_not_connected_scan(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
+	bool bt_hs_on = false;
+	u32 wifi_link_status = 0;
+	u32 num_of_wifi_link = 0;
+	bool bt_ctrl_agg_buf_size = false;
+	u8 agg_buf_size = 5;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], ********** (wifi non connect scan) **********\n");
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_HS_OPERATION, &bt_hs_on);
+	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_LINK_STATUS,
+			   &wifi_link_status);
+
+	num_of_wifi_link = wifi_link_status >> 16;
+
+	if (num_of_wifi_link >= 2) {
+		halbtc8822b1ant_limited_tx(btcoexist, NORMAL_EXEC, 0, 0, 0, 0);
+		halbtc8822b1ant_limited_rx(btcoexist, NORMAL_EXEC, false,
+					   bt_ctrl_agg_buf_size, agg_buf_size);
+
+		if (coex_sta->c2h_bt_inquiry_page) {
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "############# [BTCoex],  BT Is Inquirying\n");
+			halbtc8822b1ant_action_bt_inquiry(btcoexist);
+		} else {
+			halbtc8822b1ant_action_wifi_multi_port(btcoexist);
+		}
+		return;
+	}
+
+	if (coex_sta->c2h_bt_inquiry_page) {
+		halbtc8822b1ant_action_bt_inquiry(btcoexist);
+		return;
+	} else if (bt_hs_on) {
+		halbtc8822b1ant_action_hs(btcoexist);
+		return;
+	}
+
+	/* tdma and coex table */
+	if (coex_dm->bt_status == BT_8822B_1ANT_BT_STATUS_ACL_BUSY) {
+		if (bt_link_info->a2dp_exist) {
+			halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						32);
+			halbtc8822b1ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 1);
+		} else if (bt_link_info->a2dp_exist &&
+			   bt_link_info->pan_exist) {
+			halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						22);
+			halbtc8822b1ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 1);
+		} else {
+			halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						20);
+			halbtc8822b1ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 1);
+		}
+	} else if ((coex_dm->bt_status == BT_8822B_1ANT_BT_STATUS_SCO_BUSY) ||
+		   (BT_8822B_1ANT_BT_STATUS_ACL_SCO_BUSY ==
+		    coex_dm->bt_status)) {
+		halbtc8822b1ant_action_bt_sco_hid_only_busy(btcoexist);
+	} else {
+		halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 8);
+
+		halbtc8822b1ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO,
+					     NORMAL_EXEC,
+					     BT_8822B_1ANT_PHASE_2G_RUNTIME);
+
+		halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 5);
+	}
+}
+
+/*""""wl not connected asso"""" + bt action*/
+
+static void halbtc8822b1ant_action_wifi_not_connected_asso_auth(
+	struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
+	bool bt_hs_on = false;
+	u32 wifi_link_status = 0;
+	u32 num_of_wifi_link = 0;
+	bool bt_ctrl_agg_buf_size = false;
+	u8 agg_buf_size = 5;
+
+	RT_TRACE(
+		rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		"[BTCoex], ********** (wifi non connect asso_auth) **********\n");
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_HS_OPERATION, &bt_hs_on);
+	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_LINK_STATUS,
+			   &wifi_link_status);
+
+	num_of_wifi_link = wifi_link_status >> 16;
+
+	if (num_of_wifi_link >= 2) {
+		halbtc8822b1ant_limited_tx(btcoexist, NORMAL_EXEC, 0, 0, 0, 0);
+		halbtc8822b1ant_limited_rx(btcoexist, NORMAL_EXEC, false,
+					   bt_ctrl_agg_buf_size, agg_buf_size);
+
+		if (coex_sta->c2h_bt_inquiry_page) {
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "############# [BTCoex],  BT Is Inquirying\n");
+			halbtc8822b1ant_action_bt_inquiry(btcoexist);
+		} else {
+			halbtc8822b1ant_action_wifi_multi_port(btcoexist);
+		}
+		return;
+	}
+
+	if (coex_sta->c2h_bt_inquiry_page) {
+		halbtc8822b1ant_action_bt_inquiry(btcoexist);
+		return;
+	} else if (bt_hs_on) {
+		halbtc8822b1ant_action_hs(btcoexist);
+		return;
+	}
+
+	/* tdma and coex table */
+	if ((bt_link_info->sco_exist) || (bt_link_info->hid_exist) ||
+	    (bt_link_info->a2dp_exist)) {
+		halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 32);
+		halbtc8822b1ant_coex_table_with_type(btcoexist, FORCE_EXEC, 4);
+	} else if (bt_link_info->pan_exist) {
+		halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 20);
+		halbtc8822b1ant_coex_table_with_type(btcoexist, FORCE_EXEC, 4);
+	} else {
+		halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 8);
+
+		halbtc8822b1ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO,
+					     NORMAL_EXEC,
+					     BT_8822B_1ANT_PHASE_2G_RUNTIME);
+
+		halbtc8822b1ant_coex_table_with_type(btcoexist, FORCE_EXEC, 2);
+	}
+}
+
+/*""""wl  connected scan"""" + bt action*/
+
+static void
+halbtc8822b1ant_action_wifi_connected_scan(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
+	bool bt_hs_on = false;
+	u32 wifi_link_status = 0;
+	u32 num_of_wifi_link = 0;
+	bool bt_ctrl_agg_buf_size = false;
+	u8 agg_buf_size = 5;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], ********** (wifi connect scan) **********\n");
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_HS_OPERATION, &bt_hs_on);
+	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_LINK_STATUS,
+			   &wifi_link_status);
+
+	num_of_wifi_link = wifi_link_status >> 16;
+
+	if (num_of_wifi_link >= 2) {
+		halbtc8822b1ant_limited_tx(btcoexist, NORMAL_EXEC, 0, 0, 0, 0);
+		halbtc8822b1ant_limited_rx(btcoexist, NORMAL_EXEC, false,
+					   bt_ctrl_agg_buf_size, agg_buf_size);
+
+		if (coex_sta->c2h_bt_inquiry_page) {
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "############# [BTCoex],  BT Is Inquirying\n");
+			halbtc8822b1ant_action_bt_inquiry(btcoexist);
+		} else {
+			halbtc8822b1ant_action_wifi_multi_port(btcoexist);
+		}
+		return;
+	}
+
+	if (coex_sta->c2h_bt_inquiry_page) {
+		halbtc8822b1ant_action_bt_inquiry(btcoexist);
+		return;
+	} else if (bt_hs_on) {
+		halbtc8822b1ant_action_hs(btcoexist);
+		return;
+	}
+
+	/* tdma and coex table */
+	if (coex_dm->bt_status == BT_8822B_1ANT_BT_STATUS_ACL_BUSY) {
+		if (bt_link_info->a2dp_exist) {
+			halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						32);
+			halbtc8822b1ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 1);
+		} else if (bt_link_info->a2dp_exist &&
+			   bt_link_info->pan_exist) {
+			halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						22);
+			halbtc8822b1ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 1);
+		} else {
+			halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						20);
+			halbtc8822b1ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 1);
+		}
+	} else if ((coex_dm->bt_status == BT_8822B_1ANT_BT_STATUS_SCO_BUSY) ||
+		   (BT_8822B_1ANT_BT_STATUS_ACL_SCO_BUSY ==
+		    coex_dm->bt_status)) {
+		halbtc8822b1ant_action_bt_sco_hid_only_busy(btcoexist);
+	} else {
+		halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 8);
+
+		halbtc8822b1ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO,
+					     NORMAL_EXEC,
+					     BT_8822B_1ANT_PHASE_2G_RUNTIME);
+
+		halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 6);
+	}
+}
+
+/*""""wl  connected specific packet"""" + bt action*/
+
+static void halbtc8822b1ant_action_wifi_connected_specific_packet(
+	struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
+	bool bt_hs_on = false;
+	u32 wifi_link_status = 0;
+	u32 num_of_wifi_link = 0;
+	bool bt_ctrl_agg_buf_size = false;
+	u8 agg_buf_size = 5;
+	bool wifi_busy = false;
+
+	RT_TRACE(
+		rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		"[BTCoex], ********** (wifi connect specific packet) **********\n");
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_HS_OPERATION, &bt_hs_on);
+	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_LINK_STATUS,
+			   &wifi_link_status);
+
+	num_of_wifi_link = wifi_link_status >> 16;
+
+	if (num_of_wifi_link >= 2) {
+		halbtc8822b1ant_limited_tx(btcoexist, NORMAL_EXEC, 0, 0, 0, 0);
+		halbtc8822b1ant_limited_rx(btcoexist, NORMAL_EXEC, false,
+					   bt_ctrl_agg_buf_size, agg_buf_size);
+
+		if (coex_sta->c2h_bt_inquiry_page) {
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "############# [BTCoex],  BT Is Inquirying\n");
+			halbtc8822b1ant_action_bt_inquiry(btcoexist);
+		} else {
+			halbtc8822b1ant_action_wifi_multi_port(btcoexist);
+		}
+		return;
+	}
+
+	if (coex_sta->c2h_bt_inquiry_page) {
+		halbtc8822b1ant_action_bt_inquiry(btcoexist);
+		return;
+	} else if (bt_hs_on) {
+		halbtc8822b1ant_action_hs(btcoexist);
+		return;
+	}
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+
+	/* no specific packet process for both WiFi and BT very busy */
+	if ((wifi_busy) &&
+	    ((bt_link_info->pan_exist) || (coex_sta->num_of_profile >= 2)))
+		return;
+
+	/* tdma and coex table */
+	if ((bt_link_info->sco_exist) || (bt_link_info->hid_exist)) {
+		halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 32);
+		halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 5);
+	} else if (bt_link_info->a2dp_exist) {
+		halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 32);
+		/*for a2dp glitch,change from 1 to 15*/
+		halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC,
+						     15);
+	} else if (bt_link_info->pan_exist) {
+		halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 20);
+		halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 1);
+	} else {
+		halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 8);
+
+		halbtc8822b1ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO,
+					     NORMAL_EXEC,
+					     BT_8822B_1ANT_PHASE_2G_RUNTIME);
+
+		halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 5);
+	}
+}
+
+/* wifi connected input point:
+ * to set different ps and tdma case (+bt different case)
+ */
+
+static void halbtc8822b1ant_action_wifi_connected(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	bool wifi_busy = false, rf4ce_enabled = false;
+	bool scan = false, link = false, roam = false;
+	bool under_4way = false, ap_enable = false, wifi_under_5g = false;
+	u8 wifi_rssi_state;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], CoexForWifiConnect()===>\n");
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_5G, &wifi_under_5g);
+
+	if (wifi_under_5g) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], CoexForWifiConnect(), return for wifi is under 5g<===\n");
+
+		halbtc8822b1ant_action_wifi_under5g(btcoexist);
+
+		return;
+	}
+
+	RT_TRACE(
+		rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		"[BTCoex], CoexForWifiConnect(), return for wifi is under 2g<===\n");
+
+	halbtc8822b1ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO, NORMAL_EXEC,
+				     BT_8822B_1ANT_PHASE_2G_RUNTIME);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_4_WAY_PROGRESS,
+			   &under_4way);
+
+	if (under_4way) {
+		halbtc8822b1ant_action_wifi_connected_specific_packet(
+			btcoexist);
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], CoexForWifiConnect(), return for wifi is under 4way<===\n");
+		return;
+	}
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_SCAN, &scan);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_LINK, &link);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_ROAM, &roam);
+	if (scan || link || roam) {
+		if (scan)
+			halbtc8822b1ant_action_wifi_connected_scan(btcoexist);
+		else
+			halbtc8822b1ant_action_wifi_connected_specific_packet(
+				btcoexist);
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], CoexForWifiConnect(), return for wifi is under scan<===\n");
+		return;
+	}
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_AP_MODE_ENABLE,
+			   &ap_enable);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+
+	/* tdma and coex table */
+	if (!wifi_busy) {
+		if (coex_dm->bt_status == BT_8822B_1ANT_BT_STATUS_ACL_BUSY) {
+			halbtc8822b1ant_action_wifi_connected_bt_acl_busy(
+				btcoexist);
+		} else if ((BT_8822B_1ANT_BT_STATUS_SCO_BUSY ==
+			    coex_dm->bt_status) ||
+			   (BT_8822B_1ANT_BT_STATUS_ACL_SCO_BUSY ==
+			    coex_dm->bt_status)) {
+			halbtc8822b1ant_action_bt_sco_hid_only_busy(btcoexist);
+		} else {
+			halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, false,
+						8);
+
+			halbtc8822b1ant_set_ant_path(
+				btcoexist, BTC_ANT_PATH_AUTO, NORMAL_EXEC,
+				BT_8822B_1ANT_PHASE_2G_RUNTIME);
+
+			if ((coex_sta->high_priority_tx) +
+				    (coex_sta->high_priority_rx) <=
+			    60)
+				/*sy modify case16 -> case17*/
+				halbtc8822b1ant_coex_table_with_type(
+					btcoexist, NORMAL_EXEC, 1);
+			else
+				halbtc8822b1ant_coex_table_with_type(
+					btcoexist, NORMAL_EXEC, 1);
+		}
+	} else {
+		if (coex_dm->bt_status == BT_8822B_1ANT_BT_STATUS_ACL_BUSY) {
+			halbtc8822b1ant_action_wifi_connected_bt_acl_busy(
+				btcoexist);
+		} else if ((BT_8822B_1ANT_BT_STATUS_SCO_BUSY ==
+			    coex_dm->bt_status) ||
+			   (BT_8822B_1ANT_BT_STATUS_ACL_SCO_BUSY ==
+			    coex_dm->bt_status)) {
+			halbtc8822b1ant_action_bt_sco_hid_only_busy(btcoexist);
+		} else {
+			if (rf4ce_enabled) {
+				btcoexist->btc_write_1byte_bitmask(
+					btcoexist, 0x45e, 0x8, 0x1);
+
+				halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC,
+							true, 50);
+
+				halbtc8822b1ant_coex_table_with_type(
+					btcoexist, NORMAL_EXEC, 1);
+				return;
+			}
+
+			halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, false,
+						8);
+
+			halbtc8822b1ant_set_ant_path(
+				btcoexist, BTC_ANT_PATH_AUTO, NORMAL_EXEC,
+				BT_8822B_1ANT_PHASE_2G_RUNTIME);
+
+			wifi_rssi_state = halbtc8822b1ant_wifi_rssi_state(
+				btcoexist, 1, 2, 25, 0);
+
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], ********** before  **********\n");
+			if (BT_8822B_1ANT_BT_STATUS_NON_CONNECTED_IDLE ==
+			    coex_dm->bt_status) {
+				if (rf4ce_enabled) {
+					btcoexist->btc_write_1byte_bitmask(
+						btcoexist, 0x45e, 0x8, 0x1);
+
+					halbtc8822b1ant_ps_tdma(btcoexist,
+								NORMAL_EXEC,
+								true, 50);
+
+					halbtc8822b1ant_coex_table_with_type(
+						btcoexist, NORMAL_EXEC, 1);
+					return;
+				}
+
+				halbtc8822b1ant_coex_table_with_type(
+					btcoexist, NORMAL_EXEC, 1);
+			} else {
+				halbtc8822b1ant_coex_table_with_type(
+					btcoexist, NORMAL_EXEC, 1);
+			}
+		}
+	}
+}
+
+static void
+halbtc8822b1ant_run_sw_coexist_mechanism(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	u8 algorithm = 0;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], ********** (run sw coexmech) **********\n");
+	algorithm = halbtc8822b1ant_action_algorithm(btcoexist);
+	coex_dm->cur_algorithm = algorithm;
+
+	if (halbtc8822b1ant_is_common_action(btcoexist)) {
+	} else {
+		switch (coex_dm->cur_algorithm) {
+		case BT_8822B_1ANT_COEX_ALGO_SCO:
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], Action algorithm = SCO.\n");
+			break;
+		case BT_8822B_1ANT_COEX_ALGO_HID:
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], Action algorithm = HID.\n");
+			break;
+		case BT_8822B_1ANT_COEX_ALGO_A2DP:
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], Action algorithm = A2DP.\n");
+			break;
+		case BT_8822B_1ANT_COEX_ALGO_A2DP_PANHS:
+			RT_TRACE(
+				rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				"[BTCoex], Action algorithm = A2DP+PAN(HS).\n");
+			break;
+		case BT_8822B_1ANT_COEX_ALGO_PANEDR:
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], Action algorithm = PAN(EDR).\n");
+			break;
+		case BT_8822B_1ANT_COEX_ALGO_PANHS:
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], Action algorithm = HS mode.\n");
+			break;
+		case BT_8822B_1ANT_COEX_ALGO_PANEDR_A2DP:
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], Action algorithm = PAN+A2DP.\n");
+			break;
+		case BT_8822B_1ANT_COEX_ALGO_PANEDR_HID:
+			RT_TRACE(
+				rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				"[BTCoex], Action algorithm = PAN(EDR)+HID.\n");
+			break;
+		case BT_8822B_1ANT_COEX_ALGO_HID_A2DP_PANEDR:
+			RT_TRACE(
+				rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				"[BTCoex], Action algorithm = HID+A2DP+PAN.\n");
+			break;
+		case BT_8822B_1ANT_COEX_ALGO_HID_A2DP:
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], Action algorithm = HID+A2DP.\n");
+			break;
+		default:
+			RT_TRACE(
+				rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				"[BTCoex], Action algorithm = coexist All Off!!\n");
+			break;
+		}
+		coex_dm->pre_algorithm = coex_dm->cur_algorithm;
+	}
+}
+
+static void halbtc8822b1ant_run_coexist_mechanism(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
+	bool wifi_connected = false, bt_hs_on = false;
+	bool increase_scan_dev_num = false;
+	bool bt_ctrl_agg_buf_size = false;
+	bool miracast_plus_bt = false;
+	u8 agg_buf_size = 5;
+	u32 wifi_link_status = 0;
+	u32 num_of_wifi_link = 0, wifi_bw;
+	u8 iot_peer = BTC_IOT_PEER_UNKNOWN;
+	bool wifi_under_5g = false;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], RunCoexistMechanism()===>\n");
+
+	if (btcoexist->manual_control) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], RunCoexistMechanism(), return for Manual CTRL <===\n");
+		return;
+	}
+
+	if (btcoexist->stop_coex_dm) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], RunCoexistMechanism(), return for Stop Coex DM <===\n");
+		return;
+	}
+
+	if (coex_sta->under_ips) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], wifi is under IPS !!!\n");
+		return;
+	}
+
+	if ((coex_sta->under_lps) &&
+	    (coex_dm->bt_status != BT_8822B_1ANT_BT_STATUS_ACL_BUSY)) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], RunCoexistMechanism(), wifi is under LPS !!!\n");
+		halbtc8822b1ant_action_wifi_native_lps(btcoexist);
+		return;
+	}
+
+	if (!coex_sta->run_time_state) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], return for run_time_state = false !!!\n");
+		return;
+	}
+
+	if (coex_sta->freeze_coexrun_by_btinfo) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], BtInfoNotify(), return for freeze_coexrun_by_btinfo\n");
+		return;
+	}
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_5G, &wifi_under_5g);
+	if (wifi_under_5g) {
+		halbtc8822b1ant_action_wifi_under5g(btcoexist);
+
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], WiFi is under 5G!!!\n");
+		return;
+	}
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], WiFi is under 2G!!!\n");
+
+	halbtc8822b1ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO, NORMAL_EXEC,
+				     BT_8822B_1ANT_PHASE_2G_RUNTIME);
+
+	if (coex_sta->bt_whck_test) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], BT is under WHCK TEST!!!\n");
+		halbtc8822b1ant_action_bt_whck_test(btcoexist);
+		return;
+	}
+
+	if (coex_sta->bt_disabled) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], BT is disabled !!!\n");
+		halbtc8822b1ant_action_wifi_only(btcoexist);
+		return;
+	}
+
+	if (coex_sta->is_setup_link) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], BT is re-link !!!\n");
+		halbtc8822b1ant_action_bt_relink(btcoexist);
+		return;
+	}
+
+	if ((coex_dm->bt_status == BT_8822B_1ANT_BT_STATUS_ACL_BUSY) ||
+	    (coex_dm->bt_status == BT_8822B_1ANT_BT_STATUS_SCO_BUSY) ||
+	    (coex_dm->bt_status == BT_8822B_1ANT_BT_STATUS_ACL_SCO_BUSY))
+		increase_scan_dev_num = true;
+
+	btcoexist->btc_set(btcoexist, BTC_SET_BL_INC_SCAN_DEV_NUM,
+			   &increase_scan_dev_num);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_CONNECTED,
+			   &wifi_connected);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_LINK_STATUS,
+			   &wifi_link_status);
+	num_of_wifi_link = wifi_link_status >> 16;
+
+	if ((num_of_wifi_link >= 2) ||
+	    (wifi_link_status & WIFI_P2P_GO_CONNECTED)) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"############# [BTCoex],  Multi-Port num_of_wifi_link = %d, wifi_link_status = 0x%x\n",
+			num_of_wifi_link, wifi_link_status);
+
+		if (bt_link_info->bt_link_exist) {
+			halbtc8822b1ant_limited_tx(btcoexist, NORMAL_EXEC, 1, 1,
+						   0, 1);
+			miracast_plus_bt = true;
+		} else {
+			halbtc8822b1ant_limited_tx(btcoexist, NORMAL_EXEC, 0, 0,
+						   0, 0);
+			miracast_plus_bt = false;
+		}
+		btcoexist->btc_set(btcoexist, BTC_SET_BL_MIRACAST_PLUS_BT,
+				   &miracast_plus_bt);
+		halbtc8822b1ant_limited_rx(btcoexist, NORMAL_EXEC, false,
+					   bt_ctrl_agg_buf_size, agg_buf_size);
+
+		if ((bt_link_info->a2dp_exist) &&
+		    (coex_sta->c2h_bt_inquiry_page)) {
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "############# [BTCoex],  BT Is Inquirying\n");
+			halbtc8822b1ant_action_bt_inquiry(btcoexist);
+		} else {
+			halbtc8822b1ant_action_wifi_multi_port(btcoexist);
+		}
+
+		return;
+	}
+
+	miracast_plus_bt = false;
+	btcoexist->btc_set(btcoexist, BTC_SET_BL_MIRACAST_PLUS_BT,
+			   &miracast_plus_bt);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_BW, &wifi_bw);
+
+	if ((bt_link_info->bt_link_exist) && (wifi_connected)) {
+		halbtc8822b1ant_limited_tx(btcoexist, NORMAL_EXEC, 1, 1, 0, 1);
+
+		btcoexist->btc_get(btcoexist, BTC_GET_U1_IOT_PEER, &iot_peer);
+
+		if (iot_peer != BTC_IOT_PEER_CISCO) {
+			if (bt_link_info->sco_exist)
+				halbtc8822b1ant_limited_rx(btcoexist,
+							   NORMAL_EXEC, true,
+							   false, 0x5);
+			else
+				halbtc8822b1ant_limited_rx(btcoexist,
+							   NORMAL_EXEC, false,
+							   false, 0x5);
+		} else {
+			if (bt_link_info->sco_exist) {
+				halbtc8822b1ant_limited_rx(btcoexist,
+							   NORMAL_EXEC, true,
+							   false, 0x5);
+			} else {
+				if (wifi_bw == BTC_WIFI_BW_HT40)
+					halbtc8822b1ant_limited_rx(
+						btcoexist, NORMAL_EXEC, false,
+						true, 0x10);
+				else
+					halbtc8822b1ant_limited_rx(
+						btcoexist, NORMAL_EXEC, false,
+						true, 0x8);
+			}
+		}
+
+		halbtc8822b1ant_sw_mechanism(btcoexist, true);
+		halbtc8822b1ant_run_sw_coexist_mechanism(
+			btcoexist); /* just print debug message */
+	} else {
+		halbtc8822b1ant_limited_tx(btcoexist, NORMAL_EXEC, 0, 0, 0, 0);
+
+		halbtc8822b1ant_limited_rx(btcoexist, NORMAL_EXEC, false, false,
+					   0x5);
+
+		halbtc8822b1ant_sw_mechanism(btcoexist, false);
+		halbtc8822b1ant_run_sw_coexist_mechanism(
+			btcoexist); /* just print debug message */
+	}
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_HS_OPERATION, &bt_hs_on);
+	if (coex_sta->c2h_bt_inquiry_page) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "############# [BTCoex],  BT Is Inquirying\n");
+		halbtc8822b1ant_action_bt_inquiry(btcoexist);
+		return;
+	} else if (bt_hs_on) {
+		halbtc8822b1ant_action_hs(btcoexist);
+		return;
+	}
+
+	if (!wifi_connected) {
+		bool scan = false, link = false, roam = false;
+
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], wifi is non connected-idle !!!\n");
+
+		btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_SCAN, &scan);
+		btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_LINK, &link);
+		btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_ROAM, &roam);
+
+		if (scan)
+			halbtc8822b1ant_action_wifi_not_connected_scan(
+				btcoexist);
+		else if (link || roam)
+			halbtc8822b1ant_action_wifi_not_connected_asso_auth(
+				btcoexist);
+		else
+			halbtc8822b1ant_action_wifi_not_connected(btcoexist);
+	} else { /* wifi LPS/Busy */
+		halbtc8822b1ant_action_wifi_connected(btcoexist);
+	}
+}
+
+static void halbtc8822b1ant_init_coex_dm(struct btc_coexist *btcoexist)
+{
+	/* force to reset coex mechanism */
+
+	halbtc8822b1ant_low_penalty_ra(btcoexist, NORMAL_EXEC, false);
+
+	/* sw all off */
+	halbtc8822b1ant_sw_mechanism(btcoexist, false);
+
+	coex_sta->pop_event_cnt = 0;
+}
+
+static void halbtc8822b1ant_init_hw_config(struct btc_coexist *btcoexist,
+					   bool back_up, bool wifi_only)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	u8 u8tmp = 0, i = 0;
+	u32 u32tmp1 = 0, u32tmp2 = 0, u32tmp3 = 0;
+
+	u32tmp3 = btcoexist->btc_read_4byte(btcoexist, 0xcb4);
+	u32tmp1 = halbtc8822b1ant_ltecoex_indirect_read_reg(btcoexist, 0x38);
+	u32tmp2 = halbtc8822b1ant_ltecoex_indirect_read_reg(btcoexist, 0x54);
+
+	RT_TRACE(
+		rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		"[BTCoex], ********** (Before Init HW config) 0xcb4 = 0x%x, 0x38= 0x%x, 0x54= 0x%x**********\n",
+		u32tmp3, u32tmp1, u32tmp2);
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], 1Ant Init HW Config!!\n");
+
+	coex_sta->bt_coex_supported_feature = 0;
+	coex_sta->bt_coex_supported_version = 0;
+	coex_sta->bt_ble_scan_type = 0;
+	coex_sta->bt_ble_scan_para[0] = 0;
+	coex_sta->bt_ble_scan_para[1] = 0;
+	coex_sta->bt_ble_scan_para[2] = 0;
+	coex_sta->bt_reg_vendor_ac = 0xffff;
+	coex_sta->bt_reg_vendor_ae = 0xffff;
+	coex_sta->isolation_btween_wb = BT_8822B_1ANT_DEFAULT_ISOLATION;
+	coex_sta->gnt_error_cnt = 0;
+	coex_sta->bt_relink_downcount = 0;
+	coex_sta->is_set_ps_state_fail = false;
+	coex_sta->cnt_set_ps_state_fail = 0;
+
+	for (i = 0; i <= 9; i++)
+		coex_sta->bt_afh_map[i] = 0;
+
+	/* Setup RF front end type */
+	halbtc8822b1ant_set_rfe_type(btcoexist);
+
+	/* 0xf0[15:12] --> Chip Cut information */
+	coex_sta->cut_version =
+		(btcoexist->btc_read_1byte(btcoexist, 0xf1) & 0xf0) >> 4;
+
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0x550, 0x8,
+					   0x1); /* enable TBTT nterrupt */
+
+	/* BT report packet sample rate	 */
+	/* 0x790[5:0]=0x5 */
+	u8tmp = btcoexist->btc_read_1byte(btcoexist, 0x790);
+	u8tmp &= 0xc0;
+	u8tmp |= 0x5;
+	btcoexist->btc_write_1byte(btcoexist, 0x790, u8tmp);
+
+	/* Enable BT counter statistics */
+	btcoexist->btc_write_1byte(btcoexist, 0x778, 0x1);
+
+	/* Enable PTA (3-wire function form BT side) */
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0x40, 0x20, 0x1);
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0x41, 0x02, 0x1);
+
+	/* Enable PTA (tx/rx signal form WiFi side) */
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4c6, 0x10, 0x1);
+	/*GNT_BT=1 while select both */
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0x763, 0x10, 0x1);
+
+	/* enable GNT_WL */
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4e, 0x40, 0x0);
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0x67, 0x1, 0x0);
+
+	if (btcoexist->btc_read_1byte(btcoexist, 0x80) == 0xc6)
+		halbtc8822b1ant_post_state_to_bt(
+			btcoexist, BT_8822B_1ANT_SCOREBOARD_ONOFF, true);
+
+	/* Antenna config */
+	if (coex_sta->is_rf_state_off) {
+		halbtc8822b1ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO,
+					     FORCE_EXEC,
+					     BT_8822B_1ANT_PHASE_WLAN_OFF);
+
+		btcoexist->stop_coex_dm = true;
+
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], **********  %s (RF Off)**********\n",
+			 __func__);
+	} else if (wifi_only) {
+		coex_sta->concurrent_rx_mode_on = false;
+		halbtc8822b1ant_set_ant_path(btcoexist, BTC_ANT_PATH_WIFI,
+					     FORCE_EXEC,
+					     BT_8822B_1ANT_PHASE_WLANONLY_INIT);
+	} else {
+		coex_sta->concurrent_rx_mode_on = true;
+
+		halbtc8822b1ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO,
+					     FORCE_EXEC,
+					     BT_8822B_1ANT_PHASE_COEX_INIT);
+	}
+
+	/* PTA parameter */
+	halbtc8822b1ant_coex_table_with_type(btcoexist, FORCE_EXEC, 0);
+
+	halbtc8822b1ant_enable_gnt_to_gpio(btcoexist, true);
+}
+
+void ex_btc8822b1ant_power_on_setting(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	struct btc_board_info *board_info = &btcoexist->board_info;
+	u8 u8tmp = 0x0;
+	u16 u16tmp = 0x0;
+
+	RT_TRACE(
+		rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		"xxxxxxxxxxxxxxxx Execute 8822b 1-Ant PowerOn Setting!! xxxxxxxxxxxxxxxx\n");
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "Ant Det Finish = %s, Ant Det Number  = %d\n",
+		 board_info->btdm_ant_det_finish ? "Yes" : "No",
+		 board_info->btdm_ant_num_by_ant_det);
+
+	btcoexist->dbg_mode_1ant = false;
+	btcoexist->stop_coex_dm = true;
+
+	/* enable BB, REG_SYS_FUNC_EN such that we can write 0x948 correctly. */
+	u16tmp = btcoexist->btc_read_2byte(btcoexist, 0x2);
+	btcoexist->btc_write_2byte(btcoexist, 0x2, u16tmp | BIT(0) | BIT(1));
+
+	/* set Path control owner to WiFi */
+	halbtc8822b1ant_ltecoex_pathcontrol_owner(btcoexist,
+						  BT_8822B_1ANT_PCO_WLSIDE);
+
+	/* set GNT_BT to high */
+	halbtc8822b1ant_ltecoex_set_gnt_bt(btcoexist,
+					   BT_8822B_1ANT_GNT_BLOCK_RFC_BB,
+					   BT_8822B_1ANT_GNT_CTRL_BY_SW,
+					   BT_8822B_1ANT_SIG_STA_SET_TO_HIGH);
+	/* Set GNT_WL to low */
+	halbtc8822b1ant_ltecoex_set_gnt_wl(
+		btcoexist, BT_8822B_1ANT_GNT_BLOCK_RFC_BB,
+		BT_8822B_1ANT_GNT_CTRL_BY_SW, BT_8822B_1ANT_SIG_STA_SET_TO_LOW);
+
+	/* set WLAN_ACT = 0 */
+	/* btcoexist->btc_write_1byte(btcoexist, 0x76e, 0x4); */
+
+	/* SD1 Chunchu red x issue */
+	btcoexist->btc_write_1byte(btcoexist, 0xff1a, 0x0);
+
+	halbtc8822b1ant_enable_gnt_to_gpio(btcoexist, true);
+
+	/* */
+	/* S0 or S1 setting and Local register setting
+	 * (By the setting fw can get ant number, S0/S1, ... info)
+	 */
+	/* Local setting bit define */
+	/*	BIT0: "0" for no antenna inverse; "1" for antenna inverse  */
+	/*	BIT1: "0" for internal switch; "1" for external switch */
+	/*	BIT2: "0" for one antenna; "1" for two antenna */
+	/* NOTE: here default all internal switch and 1-antenna ==>
+	 *       BIT1=0 and BIT2=0
+	 */
+
+	u8tmp = 0;
+	board_info->btdm_ant_pos = BTC_ANTENNA_AT_MAIN_PORT;
+
+	if (btcoexist->chip_interface == BTC_INTF_USB)
+		btcoexist->btc_write_local_reg_1byte(btcoexist, 0xfe08, u8tmp);
+	else if (btcoexist->chip_interface == BTC_INTF_SDIO)
+		btcoexist->btc_write_local_reg_1byte(btcoexist, 0x60, u8tmp);
+}
+
+void ex_btc8822b1ant_pre_load_firmware(struct btc_coexist *btcoexist) {}
+
+void ex_btc8822b1ant_init_hw_config(struct btc_coexist *btcoexist,
+				    bool wifi_only)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], ********** (ini hw config) **********\n");
+
+	halbtc8822b1ant_init_hw_config(btcoexist, true, wifi_only);
+	btcoexist->stop_coex_dm = false;
+	btcoexist->auto_report_1ant = true;
+}
+
+void ex_btc8822b1ant_init_coex_dm(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], Coex Mechanism Init!!\n");
+
+	btcoexist->stop_coex_dm = false;
+
+	halbtc8822b1ant_init_coex_dm(btcoexist);
+
+	halbtc8822b1ant_query_bt_info(btcoexist);
+}
+
+void ex_btc8822b1ant_display_coex_info(struct btc_coexist *btcoexist,
+				       struct seq_file *m)
+{
+	struct btc_board_info *board_info = &btcoexist->board_info;
+	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
+
+	u8 u8tmp[4], i, ps_tdma_case = 0;
+	u16 u16tmp[4];
+	u32 u32tmp[4];
+	u32 fa_ofdm, fa_cck, cca_ofdm, cca_cck;
+	u32 fw_ver = 0, bt_patch_ver = 0, bt_coex_ver = 0;
+	static u8 pop_report_in_10s;
+	u32 phyver = 0;
+	bool lte_coex_on = false;
+	static u8 cnt;
+
+	seq_puts(m, "\r\n ============[BT Coexist info]============");
+
+	if (btcoexist->manual_control) {
+		seq_puts(m,
+			 "\r\n ============[Under Manual Control]============");
+		seq_puts(m, "\r\n ==========================================");
+	}
+	if (btcoexist->stop_coex_dm) {
+		seq_puts(m, "\r\n ============[Coex is STOPPED]============");
+		seq_puts(m, "\r\n ==========================================");
+	}
+
+	if (!coex_sta->bt_disabled) {
+		if (coex_sta->bt_coex_supported_feature == 0)
+			btcoexist->btc_get(
+				btcoexist, BTC_GET_U4_SUPPORTED_FEATURE,
+				&coex_sta->bt_coex_supported_feature);
+
+		if ((coex_sta->bt_coex_supported_version == 0) ||
+		    (coex_sta->bt_coex_supported_version == 0xffff))
+			btcoexist->btc_get(
+				btcoexist, BTC_GET_U4_SUPPORTED_VERSION,
+				&coex_sta->bt_coex_supported_version);
+
+		if (coex_sta->bt_reg_vendor_ac == 0xffff)
+			coex_sta->bt_reg_vendor_ac = (u16)(
+				btcoexist->btc_get_bt_reg(btcoexist, 3, 0xac) &
+				0xffff);
+
+		if (coex_sta->bt_reg_vendor_ae == 0xffff)
+			coex_sta->bt_reg_vendor_ae = (u16)(
+				btcoexist->btc_get_bt_reg(btcoexist, 3, 0xae) &
+				0xffff);
+
+		btcoexist->btc_get(btcoexist, BTC_GET_U4_BT_PATCH_VER,
+				   &bt_patch_ver);
+		btcoexist->bt_info.bt_get_fw_ver = bt_patch_ver;
+
+		if (coex_sta->num_of_profile > 0) {
+			cnt++;
+
+			if (cnt >= 3) {
+				btcoexist->btc_get_bt_afh_map_from_bt(
+					btcoexist, 0, &coex_sta->bt_afh_map[0]);
+				cnt = 0;
+			}
+		}
+	}
+
+	if (psd_scan->ant_det_try_count == 0) {
+		seq_printf(
+			m, "\r\n %-35s = %d/ %d/ %s / %d",
+			"Ant PG Num/ Mech/ Pos/ RFE", board_info->pg_ant_num,
+			board_info->btdm_ant_num,
+			(board_info->btdm_ant_pos == BTC_ANTENNA_AT_MAIN_PORT ?
+				 "Main" :
+				 "Aux"),
+			rfe_type->rfe_module_type);
+	} else {
+		seq_printf(
+			m, "\r\n %-35s = %d/ %d/ %s/ %d  (%d/%d/%d)",
+			"Ant PG Num/ Mech(Ant_Det)/ Pos/ RFE",
+			board_info->pg_ant_num,
+			board_info->btdm_ant_num_by_ant_det,
+			(board_info->btdm_ant_pos == BTC_ANTENNA_AT_MAIN_PORT ?
+				 "Main" :
+				 "Aux"),
+			rfe_type->rfe_module_type, psd_scan->ant_det_try_count,
+			psd_scan->ant_det_fail_count, psd_scan->ant_det_result);
+
+		if (board_info->btdm_ant_det_finish) {
+			if (psd_scan->ant_det_result != 12)
+				seq_printf(m, "\r\n %-35s = %s",
+					   "Ant Det PSD Value",
+					   psd_scan->ant_det_peak_val);
+			else
+				seq_printf(m, "\r\n %-35s = %d",
+					   "Ant Det PSD Value",
+					   psd_scan->ant_det_psd_scan_peak_val /
+						   100);
+		}
+	}
+
+	bt_patch_ver = btcoexist->bt_info.bt_get_fw_ver;
+	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_FW_VER, &fw_ver);
+	phyver = btcoexist->btc_get_bt_phydm_version(btcoexist);
+
+	bt_coex_ver = ((coex_sta->bt_coex_supported_version & 0xff00) >> 8);
+
+	seq_printf(
+		m, "\r\n %-35s = %d_%02x/ 0x%02x/ 0x%02x (%s)",
+		"CoexVer WL/  BT_Desired/ BT_Report",
+		glcoex_ver_date_8822b_1ant, glcoex_ver_8822b_1ant,
+		glcoex_ver_btdesired_8822b_1ant, bt_coex_ver,
+		(bt_coex_ver == 0xff ?
+			 "Unknown" :
+			 (coex_sta->bt_disabled ?  "BT-disable" :
+			  (bt_coex_ver >= glcoex_ver_btdesired_8822b_1ant ?
+				   "Match" :
+				   "Mis-Match"))));
+
+	seq_printf(m, "\r\n %-35s = 0x%x/ 0x%x/ v%d/ %c", "W_FW/ B_FW/ Phy/ Kt",
+		   fw_ver, bt_patch_ver, phyver, coex_sta->cut_version + 65);
+
+	seq_printf(m, "\r\n %-35s = %02x %02x %02x ", "AFH Map to BT",
+		   coex_dm->wifi_chnl_info[0], coex_dm->wifi_chnl_info[1],
+		   coex_dm->wifi_chnl_info[2]);
+
+	/* wifi status */
+	seq_printf(m, "\r\n %-35s", "============[Wifi Status]============");
+	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_WIFI_STATUS, m);
+
+	seq_printf(m, "\r\n %-35s", "============[BT Status]============");
+
+	pop_report_in_10s++;
+	seq_printf(
+		m, "\r\n %-35s = [%s/ %d dBm/ %d/ %d] ",
+		"BT [status/ rssi/ retryCnt/ popCnt]",
+		((coex_sta->bt_disabled) ?
+			 ("disabled") :
+			 ((coex_sta->c2h_bt_inquiry_page) ?  ("inquiry/page") :
+			  ((BT_8822B_1ANT_BT_STATUS_NON_CONNECTED_IDLE ==
+			    coex_dm->bt_status) ?
+				   "non-connected idle" :
+				   ((coex_dm->bt_status ==
+				     BT_8822B_1ANT_BT_STATUS_CONNECTED_IDLE) ?
+					    "connected-idle" :
+					    "busy")))),
+		coex_sta->bt_rssi - 100, coex_sta->bt_retry_cnt,
+		coex_sta->pop_event_cnt);
+
+	if (pop_report_in_10s >= 5) {
+		coex_sta->pop_event_cnt = 0;
+		pop_report_in_10s = 0;
+	}
+
+	if (coex_sta->num_of_profile != 0)
+		seq_printf(
+			m, "\r\n %-35s = %s%s%s%s%s", "Profiles",
+			((bt_link_info->a2dp_exist) ?
+				 ((coex_sta->is_bt_a2dp_sink) ? "A2DP sink," :
+								"A2DP,") :
+				 ""),
+			((bt_link_info->sco_exist) ? "HFP," : ""),
+			((bt_link_info->hid_exist) ?
+				 ((coex_sta->hid_busy_num >= 2) ?
+					  "HID(4/18)," :
+					  "HID(2/18),") :
+				 ""),
+			((bt_link_info->pan_exist) ? "PAN," : ""),
+			((coex_sta->voice_over_HOGP) ? "Voice" : ""));
+	else
+		seq_printf(m, "\r\n %-35s = None", "Profiles");
+
+	if (bt_link_info->a2dp_exist) {
+		seq_printf(m, "\r\n %-35s = %s/ %d/ %s",
+			   "A2DP Rate/Bitpool/Auto_Slot",
+			   ((coex_sta->is_A2DP_3M) ? "3M" : "No_3M"),
+			   coex_sta->a2dp_bit_pool,
+			   ((coex_sta->is_autoslot) ? "On" : "Off"));
+	}
+
+	if (bt_link_info->hid_exist) {
+		seq_printf(m, "\r\n %-35s = %d/ %d", "HID PairNum/Forbid_Slot",
+			   coex_sta->hid_pair_cnt, coex_sta->forbidden_slot);
+	}
+
+	seq_printf(m, "\r\n %-35s = %s/ %d/ %s/ 0x%x",
+		   "Role/RoleSwCnt/IgnWlact/Feature",
+		   ((bt_link_info->slave_role) ? "Slave" : "Master"),
+		   coex_sta->cnt_role_switch,
+		   ((coex_dm->cur_ignore_wlan_act) ? "Yes" : "No"),
+		   coex_sta->bt_coex_supported_feature);
+
+	if ((coex_sta->bt_ble_scan_type & 0x7) != 0x0) {
+		seq_printf(m, "\r\n %-35s = 0x%x/ 0x%x/ 0x%x/ 0x%x",
+			   "BLEScan Type/TV/Init/Ble",
+			   coex_sta->bt_ble_scan_type,
+			   (coex_sta->bt_ble_scan_type & 0x1 ?
+				    coex_sta->bt_ble_scan_para[0] :
+				    0x0),
+			   (coex_sta->bt_ble_scan_type & 0x2 ?
+				    coex_sta->bt_ble_scan_para[1] :
+				    0x0),
+			   (coex_sta->bt_ble_scan_type & 0x4 ?
+				    coex_sta->bt_ble_scan_para[2] :
+				    0x0));
+	}
+
+	seq_printf(m, "\r\n %-35s = %d/ %d/ %d/ %d/ %d",
+		   "ReInit/ReLink/IgnWlact/Page/NameReq", coex_sta->cnt_reinit,
+		   coex_sta->cnt_setup_link, coex_sta->cnt_ign_wlan_act,
+		   coex_sta->cnt_page, coex_sta->cnt_remote_name_req);
+
+	halbtc8822b1ant_read_score_board(btcoexist, &u16tmp[0]);
+
+	if ((coex_sta->bt_reg_vendor_ae == 0xffff) ||
+	    (coex_sta->bt_reg_vendor_ac == 0xffff))
+		seq_printf(m, "\r\n %-35s = x/ x/ %04x",
+			   "0xae[4]/0xac[1:0]/Scoreboard", u16tmp[0]);
+	else
+		seq_printf(m, "\r\n %-35s = 0x%x/ 0x%x/ %04x",
+			   "0xae[4]/0xac[1:0]/Scoreboard",
+			   (int)((coex_sta->bt_reg_vendor_ae & BIT(4)) >> 4),
+			   coex_sta->bt_reg_vendor_ac & 0x3, u16tmp[0]);
+
+	if (coex_sta->num_of_profile > 0) {
+		seq_printf(
+			m,
+			"\r\n %-35s = %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x",
+			"AFH MAP", coex_sta->bt_afh_map[0],
+			coex_sta->bt_afh_map[1], coex_sta->bt_afh_map[2],
+			coex_sta->bt_afh_map[3], coex_sta->bt_afh_map[4],
+			coex_sta->bt_afh_map[5], coex_sta->bt_afh_map[6],
+			coex_sta->bt_afh_map[7], coex_sta->bt_afh_map[8],
+			coex_sta->bt_afh_map[9]);
+	}
+
+	for (i = 0; i < BT_INFO_SRC_8822B_1ANT_MAX; i++) {
+		if (coex_sta->bt_info_c2h_cnt[i]) {
+			seq_printf(
+				m,
+				"\r\n %-35s = %02x %02x %02x %02x %02x %02x %02x(%d)",
+				glbt_info_src_8822b_1ant[i],
+				coex_sta->bt_info_c2h[i][0],
+				coex_sta->bt_info_c2h[i][1],
+				coex_sta->bt_info_c2h[i][2],
+				coex_sta->bt_info_c2h[i][3],
+				coex_sta->bt_info_c2h[i][4],
+				coex_sta->bt_info_c2h[i][5],
+				coex_sta->bt_info_c2h[i][6],
+				coex_sta->bt_info_c2h_cnt[i]);
+		}
+	}
+
+	if (btcoexist->manual_control)
+		seq_printf(
+			m, "\r\n %-35s",
+			"============[mechanisms] (before Manual)============");
+	else
+		seq_printf(m, "\r\n %-35s",
+			   "============[Mechanisms]============");
+
+	ps_tdma_case = coex_dm->cur_ps_tdma;
+	seq_printf(m, "\r\n %-35s = %02x %02x %02x %02x %02x (case-%d, %s)",
+		   "TDMA", coex_dm->ps_tdma_para[0], coex_dm->ps_tdma_para[1],
+		   coex_dm->ps_tdma_para[2], coex_dm->ps_tdma_para[3],
+		   coex_dm->ps_tdma_para[4], ps_tdma_case,
+		   (coex_dm->cur_ps_tdma_on ? "TDMA On" : "TDMA Off"));
+
+	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x6c0);
+	u32tmp[1] = btcoexist->btc_read_4byte(btcoexist, 0x6c4);
+	u32tmp[2] = btcoexist->btc_read_4byte(btcoexist, 0x6c8);
+	seq_printf(m, "\r\n %-35s = %d/ 0x%x/ 0x%x/ 0x%x",
+		   "Table/0x6c0/0x6c4/0x6c8", coex_sta->coex_table_type,
+		   u32tmp[0], u32tmp[1], u32tmp[2]);
+
+	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x778);
+	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x6cc);
+	seq_printf(m, "\r\n %-35s = 0x%x/ 0x%x", "0x778/0x6cc", u8tmp[0],
+		   u32tmp[0]);
+
+	seq_printf(m, "\r\n %-35s = %s/ %s/ %s/ %d",
+		   "AntDiv/BtCtrlLPS/LPRA/PsFail",
+		   ((board_info->ant_div_cfg) ? "On" : "Off"),
+		   ((coex_sta->force_lps_ctrl) ? "On" : "Off"),
+		   ((coex_dm->cur_low_penalty_ra) ? "On" : "Off"),
+		   coex_sta->cnt_set_ps_state_fail);
+
+	u32tmp[0] = halbtc8822b1ant_ltecoex_indirect_read_reg(btcoexist, 0x38);
+	lte_coex_on = ((u32tmp[0] & BIT(7)) >> 7) ? true : false;
+
+	if (lte_coex_on) {
+		u32tmp[0] = halbtc8822b1ant_ltecoex_indirect_read_reg(btcoexist,
+								      0xa0);
+		u32tmp[1] = halbtc8822b1ant_ltecoex_indirect_read_reg(btcoexist,
+								      0xa4);
+
+		seq_printf(m, "\r\n %-35s = 0x%x/ 0x%x",
+			   "LTE Coex Table W_L/B_L", u32tmp[0] & 0xffff,
+			   u32tmp[1] & 0xffff);
+
+		u32tmp[0] = halbtc8822b1ant_ltecoex_indirect_read_reg(btcoexist,
+								      0xa8);
+		u32tmp[1] = halbtc8822b1ant_ltecoex_indirect_read_reg(btcoexist,
+								      0xac);
+		u32tmp[2] = halbtc8822b1ant_ltecoex_indirect_read_reg(btcoexist,
+								      0xb0);
+		u32tmp[3] = halbtc8822b1ant_ltecoex_indirect_read_reg(btcoexist,
+								      0xb4);
+
+		seq_printf(m, "\r\n %-35s = 0x%x/ 0x%x/ 0x%x/ 0x%x",
+			   "LTE Break Table W_L/B_L/L_W/L_B",
+			   u32tmp[0] & 0xffff, u32tmp[1] & 0xffff,
+			   u32tmp[2] & 0xffff, u32tmp[3] & 0xffff);
+	}
+
+	/* Hw setting		 */
+	seq_printf(m, "\r\n %-35s", "============[Hw setting]============");
+
+	u32tmp[0] = halbtc8822b1ant_ltecoex_indirect_read_reg(btcoexist, 0x38);
+	u32tmp[1] = halbtc8822b1ant_ltecoex_indirect_read_reg(btcoexist, 0x54);
+	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x73);
+
+	seq_printf(m, "\r\n %-35s = %s/ %s", "LTE Coex/Path Owner",
+		   ((lte_coex_on) ? "On" : "Off"),
+		   ((u8tmp[0] & BIT(2)) ? "WL" : "BT"));
+
+	if (lte_coex_on) {
+		seq_printf(m, "\r\n %-35s = %d/ %d/ %d/ %d",
+			   "LTE 3Wire/OPMode/UART/UARTMode",
+			   (int)((u32tmp[0] & BIT(6)) >> 6),
+			   (int)((u32tmp[0] & (BIT(5) | BIT(4))) >> 4),
+			   (int)((u32tmp[0] & BIT(3)) >> 3),
+			   (int)(u32tmp[0] & (BIT(2) | BIT(1) | BIT(0))));
+
+		seq_printf(m, "\r\n %-35s = %d/ %d", "LTE_Busy/UART_Busy",
+			   (int)((u32tmp[1] & BIT(1)) >> 1),
+			   (int)(u32tmp[1] & BIT(0)));
+	}
+	seq_printf(m, "\r\n %-35s = %s (BB:%s)/ %s (BB:%s)/ %s %d",
+		   "GNT_WL_Ctrl/GNT_BT_Ctrl/Dbg",
+		   ((u32tmp[0] & BIT(12)) ? "SW" : "HW"),
+		   ((u32tmp[0] & BIT(8)) ? "SW" : "HW"),
+		   ((u32tmp[0] & BIT(14)) ? "SW" : "HW"),
+		   ((u32tmp[0] & BIT(10)) ? "SW" : "HW"),
+		   ((u8tmp[0] & BIT(3)) ? "On" : "Off"),
+		   coex_sta->gnt_error_cnt);
+
+	seq_printf(m, "\r\n %-35s = %d/ %d", "GNT_WL/GNT_BT",
+		   (int)((u32tmp[1] & BIT(2)) >> 2),
+		   (int)((u32tmp[1] & BIT(3)) >> 3));
+
+	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0xcb0);
+	u32tmp[1] = btcoexist->btc_read_4byte(btcoexist, 0xcb4);
+	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0xcba);
+
+	seq_printf(m, "\r\n %-35s = 0x%04x/ 0x%04x/ 0x%02x %s",
+		   "0xcb0/0xcb4/0xcb8[23:16]", u32tmp[0], u32tmp[1], u8tmp[0],
+		   ((u8tmp[0] & 0x1) == 0x1 ? "(BTG)" : "(WL_A+G)"));
+
+	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x4c);
+	u8tmp[2] = btcoexist->btc_read_1byte(btcoexist, 0x64);
+	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x4c6);
+	u8tmp[1] = btcoexist->btc_read_1byte(btcoexist, 0x40);
+
+	seq_printf(m, "\r\n %-35s = 0x%x/ 0x%x/ 0x%x/ 0x%x",
+		   "4c[24:23]/64[0]/4c6[4]/40[5]",
+		   (int)((u32tmp[0] & (BIT(24) | BIT(23))) >> 23),
+		   u8tmp[2] & 0x1, (int)((u8tmp[0] & BIT(4)) >> 4),
+		   (int)((u8tmp[1] & BIT(5)) >> 5));
+
+	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x550);
+	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x522);
+	u8tmp[1] = btcoexist->btc_read_1byte(btcoexist, 0x953);
+	u8tmp[2] = btcoexist->btc_read_1byte(btcoexist, 0xc50);
+
+	seq_printf(m, "\r\n %-35s = 0x%x/ 0x%x/ %s/ 0x%x",
+		   "0x550/0x522/4-RxAGC/0xc50", u32tmp[0], u8tmp[0],
+		   (u8tmp[1] & 0x2) ? "On" : "Off", u8tmp[2]);
+
+	fa_ofdm = btcoexist->btc_phydm_query_phy_counter(btcoexist,
+							 "PHYDM_INFO_FA_OFDM");
+	fa_cck = btcoexist->btc_phydm_query_phy_counter(btcoexist,
+							"PHYDM_INFO_FA_CCK");
+	cca_ofdm = btcoexist->btc_phydm_query_phy_counter(
+		btcoexist, "PHYDM_INFO_CCA_OFDM");
+	cca_cck = btcoexist->btc_phydm_query_phy_counter(btcoexist,
+							 "PHYDM_INFO_CCA_CCK");
+
+	seq_printf(m, "\r\n %-35s = 0x%x/ 0x%x/ 0x%x/ 0x%x",
+		   "CCK-CCA/CCK-FA/OFDM-CCA/OFDM-FA", cca_cck, fa_cck, cca_ofdm,
+		   fa_ofdm);
+
+	seq_printf(m, "\r\n %-35s = %d/ %d/ %d/ %d", "CRC_OK CCK/11g/11n/11ac",
+		   coex_sta->crc_ok_cck, coex_sta->crc_ok_11g,
+		   coex_sta->crc_ok_11n, coex_sta->crc_ok_11n_vht);
+
+	seq_printf(m, "\r\n %-35s = %d/ %d/ %d/ %d", "CRC_Err CCK/11g/11n/11ac",
+		   coex_sta->crc_err_cck, coex_sta->crc_err_11g,
+		   coex_sta->crc_err_11n, coex_sta->crc_err_11n_vht);
+
+	seq_printf(m, "\r\n %-35s = %s/ %s/ %s/ %d",
+		   "WlHiPri/ Locking/ Locked/ Noisy",
+		   (coex_sta->wifi_is_high_pri_task ? "Yes" : "No"),
+		   (coex_sta->cck_lock ? "Yes" : "No"),
+		   (coex_sta->cck_ever_lock ? "Yes" : "No"),
+		   coex_sta->wl_noisy_level);
+
+	seq_printf(m, "\r\n %-35s = %d/ %d", "0x770(Hi-pri rx/tx)",
+		   coex_sta->high_priority_rx, coex_sta->high_priority_tx);
+
+	seq_printf(m, "\r\n %-35s = %d/ %d %s", "0x774(Lo-pri rx/tx)",
+		   coex_sta->low_priority_rx, coex_sta->low_priority_tx,
+		   (bt_link_info->slave_role ?
+			    "(Slave!!)" :
+			    (coex_sta->is_tdma_btautoslot_hang ?
+				     "(auto-slot hang!!)" :
+				     "")));
+
+	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_COEX_STATISTICS, m);
+}
+
+void ex_btc8822b1ant_ips_notify(struct btc_coexist *btcoexist, u8 type)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	if (btcoexist->manual_control || btcoexist->stop_coex_dm)
+		return;
+
+	if (type == BTC_IPS_ENTER) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], IPS ENTER notify\n");
+		coex_sta->under_ips = true;
+
+		/* Write WL "Active" in Score-board for LPS off */
+		halbtc8822b1ant_post_state_to_bt(
+			btcoexist, BT_8822B_1ANT_SCOREBOARD_ACTIVE, false);
+
+		halbtc8822b1ant_post_state_to_bt(
+			btcoexist, BT_8822B_1ANT_SCOREBOARD_ONOFF, false);
+
+		halbtc8822b1ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 0);
+
+		halbtc8822b1ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO,
+					     FORCE_EXEC,
+					     BT_8822B_1ANT_PHASE_WLAN_OFF);
+
+		halbtc8822b1ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 0);
+	} else if (type == BTC_IPS_LEAVE) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], IPS LEAVE notify\n");
+		halbtc8822b1ant_post_state_to_bt(
+			btcoexist, BT_8822B_1ANT_SCOREBOARD_ACTIVE, true);
+
+		halbtc8822b1ant_post_state_to_bt(
+			btcoexist, BT_8822B_1ANT_SCOREBOARD_ONOFF, true);
+
+		/*leave IPS : run ini hw config (exclude wifi only)*/
+		halbtc8822b1ant_init_hw_config(btcoexist, false, false);
+		/*sw all off*/
+		halbtc8822b1ant_init_coex_dm(btcoexist);
+		/*leave IPS : Query bt info*/
+		halbtc8822b1ant_query_bt_info(btcoexist);
+
+		coex_sta->under_ips = false;
+	}
+}
+
+void ex_btc8822b1ant_lps_notify(struct btc_coexist *btcoexist, u8 type)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	static bool pre_force_lps_on;
+
+	if (btcoexist->manual_control || btcoexist->stop_coex_dm)
+		return;
+
+	if (type == BTC_LPS_ENABLE) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], LPS ENABLE notify\n");
+		coex_sta->under_lps = true;
+
+		if (coex_sta->force_lps_ctrl) { /* LPS No-32K */
+			/* Write WL "Active" in Score-board for PS-TDMA */
+			pre_force_lps_on = true;
+			halbtc8822b1ant_post_state_to_bt(
+				btcoexist, BT_8822B_1ANT_SCOREBOARD_ACTIVE,
+				true);
+		} else {
+			/* LPS-32K, need check if this h2c 0x71 can work??
+			 * (2015/08/28)
+			 */
+			/* Write WL "Non-Active" in Score-board for Native-PS */
+			pre_force_lps_on = false;
+			halbtc8822b1ant_post_state_to_bt(
+				btcoexist, BT_8822B_1ANT_SCOREBOARD_ACTIVE,
+				false);
+		}
+	} else if (type == BTC_LPS_DISABLE) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], LPS DISABLE notify\n");
+		coex_sta->under_lps = false;
+
+		/* Write WL "Active" in Score-board for LPS off */
+		halbtc8822b1ant_post_state_to_bt(
+			btcoexist, BT_8822B_1ANT_SCOREBOARD_ACTIVE, true);
+
+		if ((!pre_force_lps_on) && (!coex_sta->force_lps_ctrl))
+			halbtc8822b1ant_query_bt_info(btcoexist);
+	}
+}
+
+void ex_btc8822b1ant_scan_notify(struct btc_coexist *btcoexist, u8 type)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	bool wifi_connected = false;
+	bool wifi_under_5g = false;
+
+	if (btcoexist->manual_control || btcoexist->stop_coex_dm)
+		return;
+
+	coex_sta->freeze_coexrun_by_btinfo = false;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_CONNECTED,
+			   &wifi_connected);
+
+	if (wifi_connected)
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], ********** WL connected before SCAN\n");
+	else
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], **********  WL is not connected before SCAN\n");
+
+	halbtc8822b1ant_query_bt_info(btcoexist);
+
+	/*2.4 g 1*/
+	if (type == BTC_SCAN_START) {
+		btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_5G,
+				   &wifi_under_5g);
+		/*5 g 1*/
+
+		if (wifi_under_5g) {
+			RT_TRACE(
+				rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				"[BTCoex], ********** (scan_notify_5g_scan_start) **********\n");
+			halbtc8822b1ant_action_wifi_under5g(btcoexist);
+			return;
+		}
+
+		/* 2.4G.2.3*/
+		coex_sta->wifi_is_high_pri_task = true;
+
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], ********** (scan_notify_2g_scan_start) **********\n");
+
+		if (!wifi_connected) { /* non-connected scan */
+			RT_TRACE(
+				rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				"[BTCoex], ********** wifi is not connected scan **********\n");
+			halbtc8822b1ant_action_wifi_not_connected_scan(
+				btcoexist);
+		} else { /* wifi is connected */
+			RT_TRACE(
+				rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				"[BTCoex], ********** wifi is connected scan **********\n");
+			halbtc8822b1ant_action_wifi_connected_scan(btcoexist);
+		}
+
+		return;
+	}
+
+	if (type == BTC_SCAN_START_2G) {
+		coex_sta->wifi_is_high_pri_task = true;
+
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], ********** (scan_notify_2g_sacn_start_for_switch_band_used) **********\n");
+
+		if (!wifi_connected) { /* non-connected scan */
+			RT_TRACE(
+				rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				"[BTCoex], ********** wifi is not connected **********\n");
+
+			halbtc8822b1ant_action_wifi_not_connected_scan(
+				btcoexist);
+		} else { /* wifi is connected */
+			RT_TRACE(
+				rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				"[BTCoex], ********** wifi is connected **********\n");
+			halbtc8822b1ant_action_wifi_connected_scan(btcoexist);
+		}
+	} else {
+		coex_sta->wifi_is_high_pri_task = false;
+
+		/* 2.4G 5 WL scan finish, then get and update sacn ap numbers */
+		/*5 g 4*/
+		btcoexist->btc_get(btcoexist, BTC_GET_U1_AP_NUM,
+				   &coex_sta->scan_ap_num);
+
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], ********** (scan_finish_notify) **********\n");
+
+		if (!wifi_connected) { /* non-connected scan */
+			halbtc8822b1ant_action_wifi_not_connected(btcoexist);
+		} else {
+			RT_TRACE(
+				rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				"[BTCoex], ********** scan_finish_notify wifi is connected **********\n");
+			halbtc8822b1ant_action_wifi_connected(btcoexist);
+		}
+	}
+}
+
+void ex_btc8822b1ant_scan_notify_without_bt(struct btc_coexist *btcoexist,
+					    u8 type)
+{
+	bool wifi_under_5g = false;
+
+	if (type == BTC_SCAN_START) {
+		btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_5G,
+				   &wifi_under_5g);
+
+		if (wifi_under_5g) {
+			btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcbd,
+							   0x3, 1);
+			return;
+		}
+
+		/* under 2.4G */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcbd, 0x3, 2);
+		return;
+	}
+	if (type == BTC_SCAN_START_2G)
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcbd, 0x3, 2);
+}
+
+void ex_btc8822b1ant_switchband_notify(struct btc_coexist *btcoexist, u8 type)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], ********** (switchband_notify) **********\n");
+
+	if (btcoexist->manual_control || btcoexist->stop_coex_dm)
+		return;
+
+	coex_sta->switch_band_notify_to = type;
+	/*2.4g 4.*/ /*5 g 2*/
+	if (type == BTC_SWITCH_TO_5G) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], ********** (switchband_notify BTC_SWITCH_TO_5G) **********\n");
+
+		halbtc8822b1ant_action_wifi_under5g(btcoexist);
+		return;
+	} else if (type == BTC_SWITCH_TO_24G_NOFORSCAN) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], ********** (switchband_notify BTC_SWITCH_TO_2G (no for scan)) **********\n");
+
+		halbtc8822b1ant_run_coexist_mechanism(btcoexist);
+		/*5 g 3*/
+
+	} else {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], ********** (switchband_notify BTC_SWITCH_TO_2G) **********\n");
+
+		ex_btc8822b1ant_scan_notify(btcoexist, BTC_SCAN_START_2G);
+	}
+	coex_sta->switch_band_notify_to = BTC_NOT_SWITCH;
+}
+
+void ex_btc8822b1ant_switchband_notify_without_bt(struct btc_coexist *btcoexist,
+						  u8 type)
+{
+	bool wifi_under_5g = false;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_5G, &wifi_under_5g);
+
+	if (type == BTC_SWITCH_TO_5G) {
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcbd, 0x3, 1);
+		return;
+	} else if (type == BTC_SWITCH_TO_24G_NOFORSCAN) {
+		if (wifi_under_5g)
+
+			btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcbd,
+							   0x3, 1);
+
+		else
+			btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcbd,
+							   0x3, 2);
+	} else {
+		ex_btc8822b1ant_scan_notify_without_bt(btcoexist,
+						       BTC_SCAN_START_2G);
+	}
+}
+
+void ex_btc8822b1ant_connect_notify(struct btc_coexist *btcoexist, u8 type)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	bool wifi_connected = false;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], ********** (connect notify) **********\n");
+
+	halbtc8822b1ant_post_state_to_bt(btcoexist,
+					 BT_8822B_1ANT_SCOREBOARD_SCAN, true);
+
+	if (btcoexist->manual_control || btcoexist->stop_coex_dm)
+		return;
+
+	if ((type == BTC_ASSOCIATE_5G_START) ||
+	    (type == BTC_ASSOCIATE_5G_FINISH)) {
+		if (type == BTC_ASSOCIATE_5G_START) {
+			RT_TRACE(
+				rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				"[BTCoex], ********** (5G associate start notify) **********\n");
+
+			halbtc8822b1ant_action_wifi_under5g(btcoexist);
+
+		} else if (type == BTC_ASSOCIATE_5G_FINISH) {
+			RT_TRACE(
+				rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				"[BTCoex], ********** (5G associate finish notify) **********\n");
+		}
+
+		return;
+	}
+
+	if (type == BTC_ASSOCIATE_START) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], 2G CONNECT START notify\n");
+
+		coex_sta->wifi_is_high_pri_task = true;
+
+		halbtc8822b1ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO,
+					     FORCE_EXEC,
+					     BT_8822B_1ANT_PHASE_2G_RUNTIME);
+
+		coex_dm->arp_cnt = 0;
+
+		halbtc8822b1ant_action_wifi_not_connected_asso_auth(btcoexist);
+
+		coex_sta->freeze_coexrun_by_btinfo = true;
+
+	} else {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], 2G CONNECT Finish notify\n");
+		coex_sta->wifi_is_high_pri_task = false;
+		coex_sta->freeze_coexrun_by_btinfo = false;
+
+		btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_CONNECTED,
+				   &wifi_connected);
+
+		if (!wifi_connected) /* non-connected scan */
+			halbtc8822b1ant_action_wifi_not_connected(btcoexist);
+		else
+			halbtc8822b1ant_action_wifi_connected(btcoexist);
+	}
+}
+
+void ex_btc8822b1ant_media_status_notify(struct btc_coexist *btcoexist, u8 type)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	bool wifi_under_b_mode = false;
+	bool wifi_under_5g = false;
+
+	if (btcoexist->manual_control || btcoexist->stop_coex_dm)
+		return;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_5G, &wifi_under_5g);
+
+	if (type == BTC_MEDIA_CONNECT) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], 2g media connect notify");
+
+		halbtc8822b1ant_post_state_to_bt(
+			btcoexist, BT_8822B_1ANT_SCOREBOARD_ACTIVE, true);
+
+		if (wifi_under_5g) {
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], 5g media notify\n");
+
+			halbtc8822b1ant_action_wifi_under5g(btcoexist);
+			return;
+		}
+		/* Force antenna setup for no scan result issue */
+		halbtc8822b1ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO,
+					     FORCE_EXEC,
+					     BT_8822B_1ANT_PHASE_2G_RUNTIME);
+
+		btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_B_MODE,
+				   &wifi_under_b_mode);
+
+		/* Set CCK Tx/Rx high Pri except 11b mode */
+		if (wifi_under_b_mode) {
+			RT_TRACE(
+				rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				"[BTCoex], ********** (media status notity under b mode) **********\n");
+			btcoexist->btc_write_1byte(btcoexist, 0x6cd,
+						   0x00); /* CCK Tx */
+			btcoexist->btc_write_1byte(btcoexist, 0x6cf,
+						   0x00); /* CCK Rx */
+		} else {
+			RT_TRACE(
+				rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				"[BTCoex], ********** (media status notity not under b mode) **********\n");
+			btcoexist->btc_write_1byte(btcoexist, 0x6cd,
+						   0x00); /* CCK Tx */
+			btcoexist->btc_write_1byte(btcoexist, 0x6cf,
+						   0x10); /* CCK Rx */
+		}
+
+		coex_dm->backup_arfr_cnt1 =
+			btcoexist->btc_read_4byte(btcoexist, 0x430);
+		coex_dm->backup_arfr_cnt2 =
+			btcoexist->btc_read_4byte(btcoexist, 0x434);
+		coex_dm->backup_retry_limit =
+			btcoexist->btc_read_2byte(btcoexist, 0x42a);
+		coex_dm->backup_ampdu_max_time =
+			btcoexist->btc_read_1byte(btcoexist, 0x456);
+	} else {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], 2g media disconnect notify\n");
+		coex_dm->arp_cnt = 0;
+
+		halbtc8822b1ant_post_state_to_bt(
+			btcoexist, BT_8822B_1ANT_SCOREBOARD_ACTIVE, false);
+
+		btcoexist->btc_write_1byte(btcoexist, 0x6cd, 0x0); /* CCK Tx */
+		btcoexist->btc_write_1byte(btcoexist, 0x6cf, 0x0); /* CCK Rx */
+
+		coex_sta->cck_ever_lock = false;
+	}
+
+	halbtc8822b1ant_update_wifi_ch_info(btcoexist, type);
+}
+
+void ex_btc8822b1ant_specific_packet_notify(struct btc_coexist *btcoexist,
+					    u8 type)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	bool under_4way = false, wifi_under_5g = false;
+
+	if (btcoexist->manual_control || btcoexist->stop_coex_dm)
+		return;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_5G, &wifi_under_5g);
+	if (wifi_under_5g) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], 5g special packet notify\n");
+
+		halbtc8822b1ant_action_wifi_under5g(btcoexist);
+		return;
+	}
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_4_WAY_PROGRESS,
+			   &under_4way);
+
+	if (under_4way) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], specific Packet ---- under_4way!!\n");
+
+		coex_sta->wifi_is_high_pri_task = true;
+		coex_sta->specific_pkt_period_cnt = 2;
+	} else if (type == BTC_PACKET_ARP) {
+		coex_dm->arp_cnt++;
+
+		if (coex_sta->wifi_is_high_pri_task) {
+			RT_TRACE(
+				rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				"[BTCoex], specific Packet ARP notify -cnt = %d\n",
+				coex_dm->arp_cnt);
+		}
+
+	} else {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], specific Packet DHCP or EAPOL notify [Type = %d]\n",
+			type);
+
+		coex_sta->wifi_is_high_pri_task = true;
+		coex_sta->specific_pkt_period_cnt = 2;
+	}
+
+	if (coex_sta->wifi_is_high_pri_task)
+		halbtc8822b1ant_action_wifi_connected_specific_packet(
+			btcoexist);
+}
+
+void ex_btc8822b1ant_bt_info_notify(struct btc_coexist *btcoexist, u8 *tmp_buf,
+				    u8 length)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	u8 i, rsp_source = 0;
+	bool wifi_connected = false;
+	bool wifi_scan = false, wifi_link = false, wifi_roam = false,
+	     wifi_busy = false;
+	static bool is_scoreboard_scan;
+
+	if (psd_scan->is_ant_det_running) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], bt_info_notify return for AntDet is running\n");
+		return;
+	}
+
+	rsp_source = tmp_buf[0] & 0xf;
+	if (rsp_source >= BT_INFO_SRC_8822B_1ANT_MAX)
+		rsp_source = BT_INFO_SRC_8822B_1ANT_WIFI_FW;
+	coex_sta->bt_info_c2h_cnt[rsp_source]++;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], Bt_info[%d], len=%d, data=[", rsp_source, length);
+
+	for (i = 0; i < length; i++) {
+		coex_sta->bt_info_c2h[rsp_source][i] = tmp_buf[i];
+
+		if (i == length - 1) {
+			/* last one */
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "0x%02x]\n", tmp_buf[i]);
+		} else {
+			/* normal */
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD, "0x%02x, ",
+				 tmp_buf[i]);
+		}
+	}
+
+	coex_sta->bt_info = coex_sta->bt_info_c2h[rsp_source][1];
+	coex_sta->bt_info_ext = coex_sta->bt_info_c2h[rsp_source][4];
+	coex_sta->bt_info_ext2 = coex_sta->bt_info_c2h[rsp_source][5];
+
+	if (rsp_source != BT_INFO_SRC_8822B_1ANT_WIFI_FW) {
+		/* if 0xff, it means BT is under WHCK test */
+		coex_sta->bt_whck_test =
+			((coex_sta->bt_info == 0xff) ? true : false);
+
+		coex_sta->bt_create_connection =
+			((coex_sta->bt_info_c2h[rsp_source][2] & 0x80) ? true :
+									 false);
+
+		/* unit: %, value-100 to translate to unit: dBm */
+		coex_sta->bt_rssi =
+			coex_sta->bt_info_c2h[rsp_source][3] * 2 + 10;
+
+		coex_sta->c2h_bt_remote_name_req =
+			((coex_sta->bt_info_c2h[rsp_source][2] & 0x20) ? true :
+									 false);
+
+		coex_sta->is_A2DP_3M =
+			((coex_sta->bt_info_c2h[rsp_source][2] & 0x10) ? true :
+									 false);
+
+		coex_sta->acl_busy =
+			((coex_sta->bt_info_c2h[rsp_source][1] & 0x9) ? true :
+									false);
+
+		coex_sta->voice_over_HOGP =
+			((coex_sta->bt_info_ext & 0x10) ? true : false);
+
+		coex_sta->c2h_bt_inquiry_page =
+			((coex_sta->bt_info & BT_INFO_8822B_1ANT_B_INQ_PAGE) ?
+				 true :
+				 false);
+
+		coex_sta->a2dp_bit_pool =
+			(((coex_sta->bt_info_c2h[rsp_source][1] & 0x49) ==
+			  0x49) ?
+				 (coex_sta->bt_info_c2h[rsp_source][6] & 0x7f) :
+				 0);
+
+		coex_sta->is_bt_a2dp_sink =
+			(coex_sta->bt_info_c2h[rsp_source][6] & 0x80) ? true :
+									false;
+
+		coex_sta->bt_retry_cnt =
+			coex_sta->bt_info_c2h[rsp_source][2] & 0xf;
+
+		coex_sta->is_autoslot = coex_sta->bt_info_ext2 & 0x8;
+
+		coex_sta->forbidden_slot = coex_sta->bt_info_ext2 & 0x7;
+
+		coex_sta->hid_busy_num = (coex_sta->bt_info_ext2 & 0x30) >> 4;
+
+		coex_sta->hid_pair_cnt = (coex_sta->bt_info_ext2 & 0xc0) >> 6;
+		if (coex_sta->bt_retry_cnt >= 1)
+			coex_sta->pop_event_cnt++;
+
+		if (coex_sta->c2h_bt_remote_name_req)
+			coex_sta->cnt_remote_name_req++;
+
+		if (coex_sta->bt_info_ext & BIT(1))
+			coex_sta->cnt_reinit++;
+
+		if (coex_sta->bt_info_ext & BIT(2)) {
+			coex_sta->cnt_setup_link++;
+			coex_sta->is_setup_link = true;
+			coex_sta->bt_relink_downcount = 2;
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], Re-Link start in BT info!!\n");
+		} else {
+			coex_sta->is_setup_link = false;
+			coex_sta->bt_relink_downcount = 0;
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], Re-Link stop in BT info!!\n");
+		}
+
+		if (coex_sta->bt_info_ext & BIT(3))
+			coex_sta->cnt_ign_wlan_act++;
+
+		if (coex_sta->bt_info_ext & BIT(6))
+			coex_sta->cnt_role_switch++;
+
+		if (coex_sta->bt_info_ext & BIT(7))
+			coex_sta->is_bt_multi_link = true;
+		else
+			coex_sta->is_bt_multi_link = false;
+
+		if (coex_sta->bt_create_connection) {
+			coex_sta->cnt_page++;
+
+			btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY,
+					   &wifi_busy);
+
+			btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_SCAN,
+					   &wifi_scan);
+			btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_LINK,
+					   &wifi_link);
+			btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_ROAM,
+					   &wifi_roam);
+
+			if ((wifi_link) || (wifi_roam) || (wifi_scan) ||
+			    (coex_sta->wifi_is_high_pri_task) || (wifi_busy)) {
+				is_scoreboard_scan = true;
+				halbtc8822b1ant_post_state_to_bt(
+					btcoexist,
+					BT_8822B_1ANT_SCOREBOARD_SCAN, true);
+
+			} else {
+				halbtc8822b1ant_post_state_to_bt(
+					btcoexist,
+					BT_8822B_1ANT_SCOREBOARD_SCAN, false);
+			}
+		} else {
+			if (is_scoreboard_scan) {
+				halbtc8822b1ant_post_state_to_bt(
+					btcoexist,
+					BT_8822B_1ANT_SCOREBOARD_SCAN, false);
+				is_scoreboard_scan = false;
+			}
+		}
+
+		/* Here we need to resend some wifi info to BT */
+		/* because bt is reset and loss of the info. */
+
+		if ((!btcoexist->manual_control) &&
+		    (!btcoexist->stop_coex_dm)) {
+			btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_CONNECTED,
+					   &wifi_connected);
+
+			/*  Re-Init */
+			if ((coex_sta->bt_info_ext & BIT(1))) {
+				RT_TRACE(
+					rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+					"[BTCoex], BT ext info bit1 check, send wifi BW&Chnl to BT!!\n");
+				if (wifi_connected)
+					halbtc8822b1ant_update_wifi_ch_info(
+						btcoexist, BTC_MEDIA_CONNECT);
+				else
+					halbtc8822b1ant_update_wifi_ch_info(
+						btcoexist,
+						BTC_MEDIA_DISCONNECT);
+			}
+
+			/*	If Ignore_WLanAct && not SetUp_Link */
+			if ((coex_sta->bt_info_ext & BIT(3)) &&
+			    (!(coex_sta->bt_info_ext & BIT(2)))) {
+				RT_TRACE(
+					rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+					"[BTCoex], BT ext info bit3 check, set BT NOT to ignore Wlan active!!\n");
+				halbtc8822b1ant_ignore_wlan_act(
+					btcoexist, FORCE_EXEC, false);
+			}
+		}
+	}
+
+	if ((coex_sta->bt_info_ext & BIT(5))) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], BT ext info bit4 check, query BLE Scan type!!\n");
+		coex_sta->bt_ble_scan_type =
+			btcoexist->btc_get_ble_scan_type_from_bt(btcoexist);
+
+		if ((coex_sta->bt_ble_scan_type & 0x1) == 0x1)
+			coex_sta->bt_ble_scan_para[0] =
+				btcoexist->btc_get_ble_scan_para_from_bt(
+					btcoexist, 0x1);
+		if ((coex_sta->bt_ble_scan_type & 0x2) == 0x2)
+			coex_sta->bt_ble_scan_para[1] =
+				btcoexist->btc_get_ble_scan_para_from_bt(
+					btcoexist, 0x2);
+		if ((coex_sta->bt_ble_scan_type & 0x4) == 0x4)
+			coex_sta->bt_ble_scan_para[2] =
+				btcoexist->btc_get_ble_scan_para_from_bt(
+					btcoexist, 0x4);
+	}
+
+	halbtc8822b1ant_update_bt_link_info(btcoexist);
+
+	halbtc8822b1ant_run_coexist_mechanism(btcoexist);
+}
+
+void ex_btc8822b1ant_rf_status_notify(struct btc_coexist *btcoexist, u8 type)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], RF Status notify\n");
+
+	if (type == BTC_RF_ON) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], RF is turned ON!!\n");
+		btcoexist->stop_coex_dm = false;
+
+		halbtc8822b1ant_post_state_to_bt(
+			btcoexist, BT_8822B_1ANT_SCOREBOARD_ACTIVE, true);
+		halbtc8822b1ant_post_state_to_bt(
+			btcoexist, BT_8822B_1ANT_SCOREBOARD_ONOFF, true);
+
+	} else if (type == BTC_RF_OFF) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], RF is turned OFF!!\n");
+
+		halbtc8822b1ant_post_state_to_bt(
+			btcoexist, BT_8822B_1ANT_SCOREBOARD_ACTIVE, false);
+		halbtc8822b1ant_post_state_to_bt(
+			btcoexist, BT_8822B_1ANT_SCOREBOARD_ONOFF, false);
+		halbtc8822b1ant_ps_tdma(btcoexist, FORCE_EXEC, false, 0);
+
+		halbtc8822b1ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO,
+					     FORCE_EXEC,
+					     BT_8822B_1ANT_PHASE_WLAN_OFF);
+		/* for test : s3 bt disppear , fail rate 1/600*/
+
+		halbtc8822b1ant_ignore_wlan_act(btcoexist, FORCE_EXEC, true);
+
+		btcoexist->stop_coex_dm = true;
+	}
+}
+
+void ex_btc8822b1ant_halt_notify(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD, "[BTCoex], Halt notify\n");
+
+	halbtc8822b1ant_post_state_to_bt(
+		btcoexist, BT_8822B_1ANT_SCOREBOARD_ACTIVE, false);
+	halbtc8822b1ant_post_state_to_bt(btcoexist,
+					 BT_8822B_1ANT_SCOREBOARD_ONOFF, false);
+
+	halbtc8822b1ant_ps_tdma(btcoexist, FORCE_EXEC, false, 0);
+
+	halbtc8822b1ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO, FORCE_EXEC,
+				     BT_8822B_1ANT_PHASE_WLAN_OFF);
+	/* for test : s3 bt disppear , fail rate 1/600*/
+
+	halbtc8822b1ant_ignore_wlan_act(btcoexist, FORCE_EXEC, true);
+
+	ex_btc8822b1ant_media_status_notify(btcoexist, BTC_MEDIA_DISCONNECT);
+	btcoexist->stop_coex_dm = true;
+}
+
+void ex_btc8822b1ant_pnp_notify(struct btc_coexist *btcoexist, u8 pnp_state)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	bool wifi_under_5g = false;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD, "[BTCoex], Pnp notify\n");
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_5G, &wifi_under_5g);
+
+	if ((pnp_state == BTC_WIFI_PNP_SLEEP) ||
+	    (pnp_state == BTC_WIFI_PNP_SLEEP_KEEP_ANT)) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], Pnp notify to SLEEP\n");
+
+		halbtc8822b1ant_post_state_to_bt(
+			btcoexist, BT_8822B_1ANT_SCOREBOARD_ACTIVE |
+					   BT_8822B_1ANT_SCOREBOARD_ONOFF |
+					   BT_8822B_1ANT_SCOREBOARD_SCAN |
+					   BT_8822B_1ANT_SCOREBOARD_UNDERTEST,
+			false);
+
+		if (pnp_state == BTC_WIFI_PNP_SLEEP_KEEP_ANT) {
+			if (wifi_under_5g)
+				halbtc8822b1ant_set_ant_path(
+					btcoexist, BTC_ANT_PATH_AUTO,
+					FORCE_EXEC,
+					BT_8822B_1ANT_PHASE_5G_RUNTIME);
+			else
+				halbtc8822b1ant_set_ant_path(
+					btcoexist, BTC_ANT_PATH_AUTO,
+					FORCE_EXEC,
+					BT_8822B_1ANT_PHASE_2G_RUNTIME);
+		} else {
+			halbtc8822b1ant_set_ant_path(
+				btcoexist, BTC_ANT_PATH_AUTO, FORCE_EXEC,
+				BT_8822B_1ANT_PHASE_WLAN_OFF);
+		}
+
+		btcoexist->stop_coex_dm = true;
+	} else if (pnp_state == BTC_WIFI_PNP_WAKE_UP) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], Pnp notify to WAKE UP\n");
+		btcoexist->stop_coex_dm = false;
+	}
+}
+
+void ex_btc8822b1ant_coex_dm_reset(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], *****************Coex DM Reset*****************\n");
+
+	halbtc8822b1ant_init_hw_config(btcoexist, false, false);
+	halbtc8822b1ant_init_coex_dm(btcoexist);
+}
+
+void ex_btc8822b1ant_periodical(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	bool bt_relink_finish = false;
+
+	RT_TRACE(
+		rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		"[BTCoex], ==========================Periodical===========================\n");
+
+	if (!btcoexist->auto_report_1ant)
+		halbtc8822b1ant_query_bt_info(btcoexist);
+
+	halbtc8822b1ant_monitor_bt_ctr(btcoexist);
+	halbtc8822b1ant_monitor_wifi_ctr(btcoexist);
+
+	halbtc8822b1ant_monitor_bt_enable_disable(btcoexist);
+
+	if (coex_sta->bt_relink_downcount != 0) {
+		coex_sta->bt_relink_downcount--;
+
+		if (coex_sta->bt_relink_downcount == 0) {
+			coex_sta->is_setup_link = false;
+			bt_relink_finish = true;
+		}
+	}
+
+	/* for 4-way, DHCP, EAPOL packet */
+	if (coex_sta->specific_pkt_period_cnt > 0) {
+		coex_sta->specific_pkt_period_cnt--;
+
+		if ((coex_sta->specific_pkt_period_cnt == 0) &&
+		    (coex_sta->wifi_is_high_pri_task))
+			coex_sta->wifi_is_high_pri_task = false;
+
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], ***************** Hi-Pri Task = %s*****************\n",
+			(coex_sta->wifi_is_high_pri_task ? "Yes" : "No"));
+	}
+
+	if (halbtc8822b1ant_is_wifi_status_changed(btcoexist) ||
+	    (bt_relink_finish) || (coex_sta->is_set_ps_state_fail))
+		halbtc8822b1ant_run_coexist_mechanism(btcoexist);
+}
+
+void ex_btc8822b1ant_antenna_detection(struct btc_coexist *btcoexist,
+				       u32 cent_freq, u32 offset, u32 span,
+				       u32 seconds)
+{
+}
+
+void ex_btc8822b1ant_antenna_isolation(struct btc_coexist *btcoexist,
+				       u32 cent_freq, u32 offset, u32 span,
+				       u32 seconds)
+{
+}
+
+void ex_btc8822b1ant_psd_scan(struct btc_coexist *btcoexist, u32 cent_freq,
+			      u32 offset, u32 span, u32 seconds)
+{
+}
+
+void ex_btc8822b1ant_display_ant_detection(struct btc_coexist *btcoexist) {}
+
+void ex_btc8822b1ant_dbg_control(struct btc_coexist *btcoexist, u8 op_code,
+				 u8 op_len, u8 *pdata)
+{
+}
diff --git a/drivers/staging/rtlwifi/btcoexist/halbtc8822b1ant.h b/drivers/staging/rtlwifi/btcoexist/halbtc8822b1ant.h
new file mode 100644
index 000000000000..583e99dc5cc9
--- /dev/null
+++ b/drivers/staging/rtlwifi/btcoexist/halbtc8822b1ant.h
@@ -0,0 +1,444 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+/* *******************************************
+ * The following is for 8822B 1ANT BT Co-exist definition
+ * ********************************************/
+#define BT_INFO_8822B_1ANT_B_FTP	BIT(7)
+#define BT_INFO_8822B_1ANT_B_A2DP	BIT(6)
+#define BT_INFO_8822B_1ANT_B_HID	BIT(5)
+#define BT_INFO_8822B_1ANT_B_SCO_BUSY	BIT(4)
+#define BT_INFO_8822B_1ANT_B_ACL_BUSY	BIT(3)
+#define BT_INFO_8822B_1ANT_B_INQ_PAGE	BIT(2)
+#define BT_INFO_8822B_1ANT_B_SCO_ESCO	BIT(1)
+#define BT_INFO_8822B_1ANT_B_CONNECTION	BIT(0)
+
+#define BT_INFO_8822B_1ANT_A2DP_BASIC_RATE(_BT_INFO_EXT_)                      \
+	(((_BT_INFO_EXT_ & BIT(0))) ? true : false)
+
+#define BTC_RSSI_COEX_THRESH_TOL_8822B_1ANT	2
+
+#define BT_8822B_1ANT_WIFI_NOISY_THRESH	150 /* max: 255 */
+#define BT_8822B_1ANT_DEFAULT_ISOLATION	15 /*  unit: dB */
+
+/* for Antenna detection */
+#define BT_8822B_1ANT_ANTDET_PSDTHRES_BACKGROUND	50
+#define BT_8822B_1ANT_ANTDET_PSDTHRES_2ANT_BADISOLATION	70
+#define BT_8822B_1ANT_ANTDET_PSDTHRES_2ANT_GOODISOLATION	55
+#define BT_8822B_1ANT_ANTDET_PSDTHRES_1ANT	35
+#define BT_8822B_1ANT_ANTDET_RETRY_INTERVAL                                    \
+	10 /* retry timer if ant det is fail, unit: second */
+#define BT_8822B_1ANT_ANTDET_ENABLE	0
+#define BT_8822B_1ANT_ANTDET_COEXMECHANISMSWITCH_ENABLE	0
+
+#define BT_8822B_1ANT_LTECOEX_INDIRECTREG_ACCESS_TIMEOUT	30000
+
+enum bt_8822b_1ant_signal_state {
+	BT_8822B_1ANT_SIG_STA_SET_TO_LOW	= 0x0,
+	BT_8822B_1ANT_SIG_STA_SET_BY_HW	= 0x0,
+	BT_8822B_1ANT_SIG_STA_SET_TO_HIGH	= 0x1,
+	BT_8822B_1ANT_SIG_STA_MAX
+};
+
+enum bt_8822b_1ant_path_ctrl_owner {
+	BT_8822B_1ANT_PCO_BTSIDE	= 0x0,
+	BT_8822B_1ANT_PCO_WLSIDE	= 0x1,
+	BT_8822B_1ANT_PCO_MAX
+};
+
+enum bt_8822b_1ant_gnt_ctrl_type {
+	BT_8822B_1ANT_GNT_CTRL_BY_PTA	= 0x0,
+	BT_8822B_1ANT_GNT_CTRL_BY_SW	= 0x1,
+	BT_8822B_1ANT_GNT_CTRL_MAX
+};
+
+enum bt_8822b_1ant_gnt_ctrl_block {
+	BT_8822B_1ANT_GNT_BLOCK_RFC_BB	= 0x0,
+	BT_8822B_1ANT_GNT_BLOCK_RFC	= 0x1,
+	BT_8822B_1ANT_GNT_BLOCK_BB	= 0x2,
+	BT_8822B_1ANT_GNT_BLOCK_MAX
+};
+
+enum bt_8822b_1ant_lte_coex_table_type {
+	BT_8822B_1ANT_CTT_WL_VS_LTE	= 0x0,
+	BT_8822B_1ANT_CTT_BT_VS_LTE	= 0x1,
+	BT_8822B_1ANT_CTT_MAX
+};
+
+enum bt_8822b_1ant_lte_break_table_type {
+	BT_8822B_1ANT_LBTT_WL_BREAK_LTE	= 0x0,
+	BT_8822B_1ANT_LBTT_BT_BREAK_LTE	= 0x1,
+	BT_8822B_1ANT_LBTT_LTE_BREAK_WL	= 0x2,
+	BT_8822B_1ANT_LBTT_LTE_BREAK_BT	= 0x3,
+	BT_8822B_1ANT_LBTT_MAX
+};
+
+enum bt_info_src_8822b_1ant {
+	BT_INFO_SRC_8822B_1ANT_WIFI_FW	= 0x0,
+	BT_INFO_SRC_8822B_1ANT_BT_RSP	= 0x1,
+	BT_INFO_SRC_8822B_1ANT_BT_ACTIVE_SEND	= 0x2,
+	BT_INFO_SRC_8822B_1ANT_MAX
+};
+
+enum bt_8822b_1ant_bt_status {
+	BT_8822B_1ANT_BT_STATUS_NON_CONNECTED_IDLE	= 0x0,
+	BT_8822B_1ANT_BT_STATUS_CONNECTED_IDLE	= 0x1,
+	BT_8822B_1ANT_BT_STATUS_INQ_PAGE	= 0x2,
+	BT_8822B_1ANT_BT_STATUS_ACL_BUSY	= 0x3,
+	BT_8822B_1ANT_BT_STATUS_SCO_BUSY	= 0x4,
+	BT_8822B_1ANT_BT_STATUS_ACL_SCO_BUSY	= 0x5,
+	BT_8822B_1ANT_BT_STATUS_MAX
+};
+
+enum bt_8822b_1ant_wifi_status {
+	BT_8822B_1ANT_WIFI_STATUS_NON_CONNECTED_IDLE	= 0x0,
+	BT_8822B_1ANT_WIFI_STATUS_NON_CONNECTED_ASSO_AUTH_SCAN	= 0x1,
+	BT_8822B_1ANT_WIFI_STATUS_CONNECTED_SCAN	= 0x2,
+	BT_8822B_1ANT_WIFI_STATUS_CONNECTED_SPECIFIC_PKT	= 0x3,
+	BT_8822B_1ANT_WIFI_STATUS_CONNECTED_IDLE	= 0x4,
+	BT_8822B_1ANT_WIFI_STATUS_CONNECTED_BUSY	= 0x5,
+	BT_8822B_1ANT_WIFI_STATUS_MAX
+};
+
+enum bt_8822b_1ant_coex_algo {
+	BT_8822B_1ANT_COEX_ALGO_UNDEFINED	= 0x0,
+	BT_8822B_1ANT_COEX_ALGO_SCO	= 0x1,
+	BT_8822B_1ANT_COEX_ALGO_HID	= 0x2,
+	BT_8822B_1ANT_COEX_ALGO_A2DP	= 0x3,
+	BT_8822B_1ANT_COEX_ALGO_A2DP_PANHS	= 0x4,
+	BT_8822B_1ANT_COEX_ALGO_PANEDR	= 0x5,
+	BT_8822B_1ANT_COEX_ALGO_PANHS	= 0x6,
+	BT_8822B_1ANT_COEX_ALGO_PANEDR_A2DP	= 0x7,
+	BT_8822B_1ANT_COEX_ALGO_PANEDR_HID	= 0x8,
+	BT_8822B_1ANT_COEX_ALGO_HID_A2DP_PANEDR	= 0x9,
+	BT_8822B_1ANT_COEX_ALGO_HID_A2DP	= 0xa,
+	BT_8822B_1ANT_COEX_ALGO_MAX	= 0xb,
+};
+
+enum bt_8822b_1ant_ext_ant_switch_type {
+	BT_8822B_1ANT_EXT_ANT_SWITCH_USE_SPDT	= 0x0,
+	BT_8822B_1ANT_EXT_ANT_SWITCH_USE_SP3T	= 0x1,
+	BT_8822B_1ANT_EXT_ANT_SWITCH_MAX
+};
+
+enum bt_8822b_1ant_ext_ant_switch_ctrl_type {
+	BT_8822B_1ANT_EXT_ANT_SWITCH_CTRL_BY_BBSW	= 0x0,
+	BT_8822B_1ANT_EXT_ANT_SWITCH_CTRL_BY_PTA	= 0x1,
+	BT_8822B_1ANT_EXT_ANT_SWITCH_CTRL_BY_ANTDIV	= 0x2,
+	BT_8822B_1ANT_EXT_ANT_SWITCH_CTRL_BY_MAC	= 0x3,
+	BT_8822B_1ANT_EXT_ANT_SWITCH_CTRL_BY_BT	= 0x4,
+	BT_8822B_1ANT_EXT_ANT_SWITCH_CTRL_MAX
+};
+
+enum bt_8822b_1ant_ext_ant_switch_pos_type {
+	BT_8822B_1ANT_EXT_ANT_SWITCH_TO_BT	= 0x0,
+	BT_8822B_1ANT_EXT_ANT_SWITCH_TO_WLG	= 0x1,
+	BT_8822B_1ANT_EXT_ANT_SWITCH_TO_WLA	= 0x2,
+	BT_8822B_1ANT_EXT_ANT_SWITCH_TO_NOCARE	= 0x3,
+	BT_8822B_1ANT_EXT_ANT_SWITCH_TO_MAX
+};
+
+enum bt_8822b_1ant_phase {
+	BT_8822B_1ANT_PHASE_COEX_INIT	= 0x0,
+	BT_8822B_1ANT_PHASE_WLANONLY_INIT	= 0x1,
+	BT_8822B_1ANT_PHASE_WLAN_OFF	= 0x2,
+	BT_8822B_1ANT_PHASE_2G_RUNTIME	= 0x3,
+	BT_8822B_1ANT_PHASE_5G_RUNTIME	= 0x4,
+	BT_8822B_1ANT_PHASE_BTMPMODE	= 0x5,
+	BT_8822B_1ANT_PHASE_MAX
+};
+
+/*ADD SCOREBOARD TO FIX BT LPS 32K ISSUE WHILE WL BUSY*/
+enum bt_8822b_1ant_scoreboard {
+	BT_8822B_1ANT_SCOREBOARD_ACTIVE	= BIT(0),
+	BT_8822B_1ANT_SCOREBOARD_ONOFF	= BIT(1),
+	BT_8822B_1ANT_SCOREBOARD_SCAN	= BIT(2),
+	BT_8822B_1ANT_SCOREBOARD_UNDERTEST	= BIT(3),
+	BT_8822B_1ANT_SCOREBOARD_WLBUSY = BIT(6)
+};
+
+struct coex_dm_8822b_1ant {
+	/* hw setting */
+	u32	pre_ant_pos_type;
+	u32	cur_ant_pos_type;
+	/* fw mechanism */
+	bool	cur_ignore_wlan_act;
+	bool	pre_ignore_wlan_act;
+	u8	pre_ps_tdma;
+	u8	cur_ps_tdma;
+	u8	ps_tdma_para[5];
+	u8	ps_tdma_du_adj_type;
+	bool	auto_tdma_adjust;
+	bool	pre_ps_tdma_on;
+	bool	cur_ps_tdma_on;
+	bool	pre_bt_auto_report;
+	bool	cur_bt_auto_report;
+	u8	pre_lps;
+	u8	cur_lps;
+	u8	pre_rpwm;
+	u8	cur_rpwm;
+
+	/* sw mechanism */
+	bool	pre_low_penalty_ra;
+	bool	cur_low_penalty_ra;
+	u32	pre_val0x6c0;
+	u32	cur_val0x6c0;
+	u32	pre_val0x6c4;
+	u32	cur_val0x6c4;
+	u32	pre_val0x6c8;
+	u32	cur_val0x6c8;
+	u8	pre_val0x6cc;
+	u8	cur_val0x6cc;
+	bool	limited_dig;
+
+	u32	backup_arfr_cnt1; /* Auto Rate Fallback Retry cnt */
+	u32	backup_arfr_cnt2; /* Auto Rate Fallback Retry cnt */
+	u16	backup_retry_limit;
+	u8	backup_ampdu_max_time;
+
+	/* algorithm related */
+	u8	pre_algorithm;
+	u8	cur_algorithm;
+	u8	bt_status;
+	u8	wifi_chnl_info[3];
+
+	u32	pre_ra_mask;
+	u32	cur_ra_mask;
+	u8	pre_arfr_type;
+	u8	cur_arfr_type;
+	u8	pre_retry_limit_type;
+	u8	cur_retry_limit_type;
+	u8	pre_ampdu_time_type;
+	u8	cur_ampdu_time_type;
+	u32	arp_cnt;
+
+	u32	pre_ext_ant_switch_status;
+	u32	cur_ext_ant_switch_status;
+
+	u8	error_condition;
+};
+
+struct coex_sta_8822b_1ant {
+	bool	bt_disabled;
+	bool	bt_link_exist;
+	bool	sco_exist;
+	bool	a2dp_exist;
+	bool	hid_exist;
+	bool	pan_exist;
+	u8	num_of_profile;
+
+	bool	under_lps;
+	bool	under_ips;
+	u32	specific_pkt_period_cnt;
+	u32	high_priority_tx;
+	u32	high_priority_rx;
+	u32	low_priority_tx;
+	u32	low_priority_rx;
+	bool	is_hi_pri_rx_overhead;
+	s8	bt_rssi;
+	u8	pre_bt_rssi_state;
+	u8	pre_wifi_rssi_state[4];
+	u8	bt_info_c2h[BT_INFO_SRC_8822B_1ANT_MAX][10];
+	u32	bt_info_c2h_cnt[BT_INFO_SRC_8822B_1ANT_MAX];
+	bool	bt_whck_test;
+	bool	c2h_bt_inquiry_page;
+	bool	c2h_bt_remote_name_req;
+	bool	c2h_bt_page; /* Add for win8.1 page out issue */
+	bool	wifi_is_high_pri_task; /* Add for win8.1 page out issue */
+
+	u8	bt_info_ext;
+	u8	bt_info_ext2;
+	u32	pop_event_cnt;
+	u8	scan_ap_num;
+	u8	bt_retry_cnt;
+
+	u32	crc_ok_cck;
+	u32	crc_ok_11g;
+	u32	crc_ok_11n;
+	u32	crc_ok_11n_vht;
+
+	u32	crc_err_cck;
+	u32	crc_err_11g;
+	u32	crc_err_11n;
+	u32	crc_err_11n_vht;
+
+	bool	cck_lock;
+	bool	pre_ccklock;
+	bool	cck_ever_lock;
+	u8	coex_table_type;
+
+	bool	force_lps_ctrl;
+
+	bool	concurrent_rx_mode_on;
+
+	u16	score_board;
+	u8	isolation_btween_wb; /* 0~ 50 */
+
+	u8	a2dp_bit_pool;
+	u8	cut_version;
+	bool	acl_busy;
+	bool	bt_create_connection;
+
+	u32	bt_coex_supported_feature;
+	u32	bt_coex_supported_version;
+
+	u8	bt_ble_scan_type;
+	u32	bt_ble_scan_para[3];
+
+	bool	run_time_state;
+	bool	freeze_coexrun_by_btinfo;
+
+	bool	is_A2DP_3M;
+	bool	voice_over_HOGP;
+	u8	bt_info;
+	bool	is_autoslot;
+	u8	forbidden_slot;
+	u8	hid_busy_num;
+	u8	hid_pair_cnt;
+
+	u32	cnt_remote_name_req;
+	u32	cnt_setup_link;
+	u32	cnt_reinit;
+	u32	cnt_ign_wlan_act;
+	u32	cnt_page;
+	u32	cnt_role_switch;
+
+	u16	bt_reg_vendor_ac;
+	u16	bt_reg_vendor_ae;
+
+	bool	is_setup_link;
+	u8	wl_noisy_level;
+	u32	gnt_error_cnt;
+	u8	bt_afh_map[10];
+	u8	bt_relink_downcount;
+	bool	is_tdma_btautoslot;
+	bool	is_tdma_btautoslot_hang;
+
+	u8	switch_band_notify_to;
+	bool	is_rf_state_off;
+
+	bool	is_hid_low_pri_tx_overhead;
+	bool	is_bt_multi_link;
+	bool	is_bt_a2dp_sink;
+	bool	rf4ce_enabled;
+
+	bool	is_set_ps_state_fail;
+	u8	cnt_set_ps_state_fail;
+};
+
+struct rfe_type_8822b_1ant {
+	u8	rfe_module_type;
+	bool	ext_ant_switch_exist;
+	u8	ext_ant_switch_type;
+	/*  iF 0: ANTSW(rfe_sel9)=0, ANTSWB(rfe_sel8)=1 =>  Ant to BT/5G */
+	u8	ext_ant_switch_ctrl_polarity;
+};
+
+#define BT_8822B_1ANT_ANTDET_PSD_POINTS	256 /* MAX:1024 */
+#define BT_8822B_1ANT_ANTDET_PSD_AVGNUM	1 /* MAX:3 */
+#define BT_8822B_1ANT_ANTDET_BUF_LEN	16
+
+struct psdscan_sta_8822b_1ant {
+	u32	ant_det_bt_le_channel; /* BT LE Channel ex:2412 */
+	u32	ant_det_bt_tx_time;
+	u32	ant_det_pre_psdscan_peak_val;
+	bool	ant_det_is_ant_det_available;
+	u32	ant_det_psd_scan_peak_val;
+	bool	ant_det_is_btreply_available;
+	u32	ant_det_psd_scan_peak_freq;
+
+	u8	ant_det_result;
+	u8	ant_det_peak_val[BT_8822B_1ANT_ANTDET_BUF_LEN];
+	u8	ant_det_peak_freq[BT_8822B_1ANT_ANTDET_BUF_LEN];
+	u32	ant_det_try_count;
+	u32	ant_det_fail_count;
+	u32	ant_det_inteval_count;
+	u32	ant_det_thres_offset;
+
+	u32	real_cent_freq;
+	s32	real_offset;
+	u32	real_span;
+
+	u32	psd_band_width; /* unit: Hz */
+	u32	psd_point; /* 128/256/512/1024 */
+	u32	psd_report[1024]; /* unit:dB (20logx), 0~255 */
+	u32	psd_report_max_hold[1024]; /* unit:dB (20logx), 0~255 */
+	u32	psd_start_point;
+	u32	psd_stop_point;
+	u32	psd_max_value_point;
+	u32	psd_max_value;
+	u32	psd_start_base;
+	u32	psd_avg_num; /* 1/8/16/32 */
+	u32	psd_gen_count;
+	bool	is_psd_running;
+	bool	is_psd_show_max_only;
+	bool	is_ant_det_running;
+};
+
+/* *******************************************
+ * The following is interface which will notify coex module.
+ * ********************************************/
+void ex_btc8822b1ant_power_on_setting(struct btc_coexist *btcoexist);
+void ex_btc8822b1ant_pre_load_firmware(struct btc_coexist *btcoexist);
+void ex_btc8822b1ant_init_hw_config(struct btc_coexist *btcoexist,
+				    bool wifi_only);
+void ex_btc8822b1ant_init_coex_dm(struct btc_coexist *btcoexist);
+void ex_btc8822b1ant_ips_notify(struct btc_coexist *btcoexist, u8 type);
+void ex_btc8822b1ant_lps_notify(struct btc_coexist *btcoexist, u8 type);
+void ex_btc8822b1ant_scan_notify(struct btc_coexist *btcoexist, u8 type);
+void ex_btc8822b1ant_scan_notify_without_bt(struct btc_coexist *btcoexist,
+					    u8 type);
+void ex_btc8822b1ant_switchband_notify(struct btc_coexist *btcoexist, u8 type);
+void ex_btc8822b1ant_switchband_notify_without_bt(struct btc_coexist *btcoexist,
+						  u8 type);
+void ex_btc8822b1ant_connect_notify(struct btc_coexist *btcoexist, u8 type);
+void ex_btc8822b1ant_media_status_notify(struct btc_coexist *btcoexist,
+					 u8 type);
+void ex_btc8822b1ant_specific_packet_notify(struct btc_coexist *btcoexist,
+					    u8 type);
+void ex_btc8822b1ant_bt_info_notify(struct btc_coexist *btcoexist, u8 *tmp_buf,
+				    u8 length);
+void ex_btc8822b1ant_rf_status_notify(struct btc_coexist *btcoexist, u8 type);
+void ex_btc8822b1ant_halt_notify(struct btc_coexist *btcoexist);
+void ex_btc8822b1ant_pnp_notify(struct btc_coexist *btcoexist, u8 pnp_state);
+void ex_halbtc8822b1ant_score_board_status_notify(struct btc_coexist *btcoexist,
+						  u8 *tmp_buf, u8 length);
+void ex_btc8822b1ant_coex_dm_reset(struct btc_coexist *btcoexist);
+void ex_btc8822b1ant_periodical(struct btc_coexist *btcoexist);
+void ex_btc8822b1ant_display_coex_info(struct btc_coexist *btcoexist,
+				       struct seq_file *m);
+void ex_btc8822b1ant_antenna_detection(struct btc_coexist *btcoexist,
+				       u32 cent_freq, u32 offset, u32 span,
+				       u32 seconds);
+void ex_btc8822b1ant_antenna_isolation(struct btc_coexist *btcoexist,
+				       u32 cent_freq, u32 offset, u32 span,
+				       u32 seconds);
+
+void ex_btc8822b1ant_psd_scan(struct btc_coexist *btcoexist, u32 cent_freq,
+			      u32 offset, u32 span, u32 seconds);
+void ex_btc8822b1ant_display_ant_detection(struct btc_coexist *btcoexist);
+
+void ex_btc8822b1ant_dbg_control(struct btc_coexist *btcoexist, u8 op_code,
+				 u8 op_len, u8 *pdata);
diff --git a/drivers/staging/rtlwifi/btcoexist/halbtc8822b2ant.c b/drivers/staging/rtlwifi/btcoexist/halbtc8822b2ant.c
new file mode 100644
index 000000000000..ffff5b062672
--- /dev/null
+++ b/drivers/staging/rtlwifi/btcoexist/halbtc8822b2ant.c
@@ -0,0 +1,5225 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+/* ************************************************************
+ * Description:
+ *
+ * This file is for RTL8822B Co-exist mechanism
+ *
+ * History
+ * 2012/11/15 Cosa first check in.
+ *
+ * *************************************************************/
+
+/* ************************************************************
+ * include files
+ * *************************************************************/
+#include "halbt_precomp.h"
+
+/* ************************************************************
+ * Global variables, these are static variables
+ * *************************************************************/
+static struct coex_dm_8822b_2ant glcoex_dm_8822b_2ant;
+static struct coex_dm_8822b_2ant *coex_dm = &glcoex_dm_8822b_2ant;
+static struct coex_sta_8822b_2ant glcoex_sta_8822b_2ant;
+static struct coex_sta_8822b_2ant *coex_sta = &glcoex_sta_8822b_2ant;
+static struct psdscan_sta_8822b_2ant gl_psd_scan_8822b_2ant;
+static struct psdscan_sta_8822b_2ant *psd_scan = &gl_psd_scan_8822b_2ant;
+static struct rfe_type_8822b_2ant gl_rfe_type_8822b_2ant;
+static struct rfe_type_8822b_2ant *rfe_type = &gl_rfe_type_8822b_2ant;
+
+static const char *const glbt_info_src_8822b_2ant[] = {
+	"BT Info[wifi fw]", "BT Info[bt rsp]", "BT Info[bt auto report]",
+};
+
+static u32 glcoex_ver_date_8822b_2ant = 20170327;
+static u32 glcoex_ver_8822b_2ant = 0x44;
+static u32 glcoex_ver_btdesired_8822b_2ant = 0x42;
+
+/* ************************************************************
+ * local function proto type if needed
+ * ************************************************************
+ * ************************************************************
+ * local function start with halbtc8822b2ant_
+ * *************************************************************/
+static u8 halbtc8822b2ant_bt_rssi_state(struct btc_coexist *btcoexist,
+					u8 *ppre_bt_rssi_state, u8 level_num,
+					u8 rssi_thresh, u8 rssi_thresh1)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	s32 bt_rssi = 0;
+	u8 bt_rssi_state = *ppre_bt_rssi_state;
+
+	bt_rssi = coex_sta->bt_rssi;
+
+	if (level_num == 2) {
+		if ((*ppre_bt_rssi_state == BTC_RSSI_STATE_LOW) ||
+		    (*ppre_bt_rssi_state == BTC_RSSI_STATE_STAY_LOW)) {
+			if (bt_rssi >=
+			    (rssi_thresh + BTC_RSSI_COEX_THRESH_TOL_8822B_2ANT))
+				bt_rssi_state = BTC_RSSI_STATE_HIGH;
+			else
+				bt_rssi_state = BTC_RSSI_STATE_STAY_LOW;
+		} else {
+			if (bt_rssi < rssi_thresh)
+				bt_rssi_state = BTC_RSSI_STATE_LOW;
+			else
+				bt_rssi_state = BTC_RSSI_STATE_STAY_HIGH;
+		}
+	} else if (level_num == 3) {
+		if (rssi_thresh > rssi_thresh1) {
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], BT Rssi thresh error!!\n");
+			return *ppre_bt_rssi_state;
+		}
+
+		if ((*ppre_bt_rssi_state == BTC_RSSI_STATE_LOW) ||
+		    (*ppre_bt_rssi_state == BTC_RSSI_STATE_STAY_LOW)) {
+			if (bt_rssi >=
+			    (rssi_thresh + BTC_RSSI_COEX_THRESH_TOL_8822B_2ANT))
+				bt_rssi_state = BTC_RSSI_STATE_MEDIUM;
+			else
+				bt_rssi_state = BTC_RSSI_STATE_STAY_LOW;
+		} else if ((*ppre_bt_rssi_state == BTC_RSSI_STATE_MEDIUM) ||
+			   (*ppre_bt_rssi_state ==
+			    BTC_RSSI_STATE_STAY_MEDIUM)) {
+			if (bt_rssi >= (rssi_thresh1 +
+					BTC_RSSI_COEX_THRESH_TOL_8822B_2ANT))
+				bt_rssi_state = BTC_RSSI_STATE_HIGH;
+			else if (bt_rssi < rssi_thresh)
+				bt_rssi_state = BTC_RSSI_STATE_LOW;
+			else
+				bt_rssi_state = BTC_RSSI_STATE_STAY_MEDIUM;
+		} else {
+			if (bt_rssi < rssi_thresh1)
+				bt_rssi_state = BTC_RSSI_STATE_MEDIUM;
+			else
+				bt_rssi_state = BTC_RSSI_STATE_STAY_HIGH;
+		}
+	}
+
+	*ppre_bt_rssi_state = bt_rssi_state;
+
+	return bt_rssi_state;
+}
+
+static u8 halbtc8822b2ant_wifi_rssi_state(struct btc_coexist *btcoexist,
+					  u8 *pprewifi_rssi_state, u8 level_num,
+					  u8 rssi_thresh, u8 rssi_thresh1)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	s32 wifi_rssi = 0;
+	u8 wifi_rssi_state = *pprewifi_rssi_state;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_S4_WIFI_RSSI, &wifi_rssi);
+
+	if (level_num == 2) {
+		if ((*pprewifi_rssi_state == BTC_RSSI_STATE_LOW) ||
+		    (*pprewifi_rssi_state == BTC_RSSI_STATE_STAY_LOW)) {
+			if (wifi_rssi >=
+			    (rssi_thresh + BTC_RSSI_COEX_THRESH_TOL_8822B_2ANT))
+				wifi_rssi_state = BTC_RSSI_STATE_HIGH;
+			else
+				wifi_rssi_state = BTC_RSSI_STATE_STAY_LOW;
+		} else {
+			if (wifi_rssi < rssi_thresh)
+				wifi_rssi_state = BTC_RSSI_STATE_LOW;
+			else
+				wifi_rssi_state = BTC_RSSI_STATE_STAY_HIGH;
+		}
+	} else if (level_num == 3) {
+		if (rssi_thresh > rssi_thresh1) {
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], wifi RSSI thresh error!!\n");
+			return *pprewifi_rssi_state;
+		}
+
+		if ((*pprewifi_rssi_state == BTC_RSSI_STATE_LOW) ||
+		    (*pprewifi_rssi_state == BTC_RSSI_STATE_STAY_LOW)) {
+			if (wifi_rssi >=
+			    (rssi_thresh + BTC_RSSI_COEX_THRESH_TOL_8822B_2ANT))
+				wifi_rssi_state = BTC_RSSI_STATE_MEDIUM;
+			else
+				wifi_rssi_state = BTC_RSSI_STATE_STAY_LOW;
+		} else if ((*pprewifi_rssi_state == BTC_RSSI_STATE_MEDIUM) ||
+			   (*pprewifi_rssi_state ==
+			    BTC_RSSI_STATE_STAY_MEDIUM)) {
+			if (wifi_rssi >= (rssi_thresh1 +
+					  BTC_RSSI_COEX_THRESH_TOL_8822B_2ANT))
+				wifi_rssi_state = BTC_RSSI_STATE_HIGH;
+			else if (wifi_rssi < rssi_thresh)
+				wifi_rssi_state = BTC_RSSI_STATE_LOW;
+			else
+				wifi_rssi_state = BTC_RSSI_STATE_STAY_MEDIUM;
+		} else {
+			if (wifi_rssi < rssi_thresh1)
+				wifi_rssi_state = BTC_RSSI_STATE_MEDIUM;
+			else
+				wifi_rssi_state = BTC_RSSI_STATE_STAY_HIGH;
+		}
+	}
+
+	*pprewifi_rssi_state = wifi_rssi_state;
+
+	return wifi_rssi_state;
+}
+
+static void halbtc8822b2ant_coex_switch_threshold(struct btc_coexist *btcoexist,
+						  u8 isolation_measuared)
+{
+	s8 interference_wl_tx = 0, interference_bt_tx = 0;
+
+	interference_wl_tx =
+		BT_8822B_2ANT_WIFI_MAX_TX_POWER - isolation_measuared;
+	interference_bt_tx =
+		BT_8822B_2ANT_BT_MAX_TX_POWER - isolation_measuared;
+
+	coex_sta->wifi_coex_thres = BT_8822B_2ANT_WIFI_RSSI_COEXSWITCH_THRES1;
+	coex_sta->wifi_coex_thres2 = BT_8822B_2ANT_WIFI_RSSI_COEXSWITCH_THRES2;
+
+	coex_sta->bt_coex_thres = BT_8822B_2ANT_BT_RSSI_COEXSWITCH_THRES1;
+	coex_sta->bt_coex_thres2 = BT_8822B_2ANT_BT_RSSI_COEXSWITCH_THRES2;
+}
+
+static void halbtc8822b2ant_query_bt_info(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	u8 h2c_parameter[1] = {0};
+
+	if (coex_sta->bt_disabled) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], No query BT info because BT is disabled!\n");
+		return;
+	}
+
+	h2c_parameter[0] |= BIT(0); /* trigger */
+
+	btcoexist->btc_fill_h2c(btcoexist, 0x61, 1, h2c_parameter);
+}
+
+static void halbtc8822b2ant_monitor_bt_ctr(struct btc_coexist *btcoexist)
+{
+	u32 reg_hp_txrx, reg_lp_txrx, u32tmp;
+	u32 reg_hp_tx = 0, reg_hp_rx = 0, reg_lp_tx = 0, reg_lp_rx = 0;
+	static u8 num_of_bt_counter_chk, cnt_slave, cnt_autoslot_hang;
+
+	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
+
+	reg_hp_txrx = 0x770;
+	reg_lp_txrx = 0x774;
+
+	u32tmp = btcoexist->btc_read_4byte(btcoexist, reg_hp_txrx);
+	reg_hp_tx = u32tmp & MASKLWORD;
+	reg_hp_rx = (u32tmp & MASKHWORD) >> 16;
+
+	u32tmp = btcoexist->btc_read_4byte(btcoexist, reg_lp_txrx);
+	reg_lp_tx = u32tmp & MASKLWORD;
+	reg_lp_rx = (u32tmp & MASKHWORD) >> 16;
+
+	coex_sta->high_priority_tx = reg_hp_tx;
+	coex_sta->high_priority_rx = reg_hp_rx;
+	coex_sta->low_priority_tx = reg_lp_tx;
+	coex_sta->low_priority_rx = reg_lp_rx;
+
+	/* reset counter */
+	btcoexist->btc_write_1byte(btcoexist, 0x76e, 0xc);
+
+	if ((coex_sta->low_priority_tx > 1050) &&
+	    (!coex_sta->c2h_bt_inquiry_page))
+		coex_sta->pop_event_cnt++;
+
+	if ((coex_sta->low_priority_rx >= 950) &&
+	    (coex_sta->low_priority_rx >= coex_sta->low_priority_tx) &&
+	    (!coex_sta->under_ips) && (!coex_sta->c2h_bt_inquiry_page) &&
+	    (coex_sta->bt_link_exist)) {
+		if (cnt_slave >= 2) {
+			bt_link_info->slave_role = true;
+			cnt_slave = 2;
+		} else {
+			cnt_slave++;
+		}
+	} else {
+		if (cnt_slave == 0) {
+			bt_link_info->slave_role = false;
+			cnt_slave = 0;
+		} else {
+			cnt_slave--;
+		}
+	}
+
+	if (coex_sta->is_tdma_btautoslot) {
+		if ((coex_sta->low_priority_tx >= 1300) &&
+		    (coex_sta->low_priority_rx <= 150)) {
+			if (cnt_autoslot_hang >= 2) {
+				coex_sta->is_tdma_btautoslot_hang = true;
+				cnt_autoslot_hang = 2;
+			} else {
+				cnt_autoslot_hang++;
+			}
+		} else {
+			if (cnt_autoslot_hang == 0) {
+				coex_sta->is_tdma_btautoslot_hang = false;
+				cnt_autoslot_hang = 0;
+			} else {
+				cnt_autoslot_hang--;
+			}
+		}
+	}
+
+	if (coex_sta->sco_exist) {
+		if ((coex_sta->high_priority_tx >= 400) &&
+		    (coex_sta->high_priority_rx >= 400))
+			coex_sta->is_esco_mode = false;
+		else
+			coex_sta->is_esco_mode = true;
+	}
+
+	if (bt_link_info->hid_only) {
+		if (coex_sta->low_priority_rx > 50)
+			coex_sta->is_hid_low_pri_tx_overhead = true;
+		else
+			coex_sta->is_hid_low_pri_tx_overhead = false;
+	}
+
+	if ((coex_sta->high_priority_tx == 0) &&
+	    (coex_sta->high_priority_rx == 0) &&
+	    (coex_sta->low_priority_tx == 0) &&
+	    (coex_sta->low_priority_rx == 0)) {
+		num_of_bt_counter_chk++;
+		if (num_of_bt_counter_chk >= 3) {
+			halbtc8822b2ant_query_bt_info(btcoexist);
+			num_of_bt_counter_chk = 0;
+		}
+	}
+}
+
+static void halbtc8822b2ant_monitor_wifi_ctr(struct btc_coexist *btcoexist)
+{
+	s32 wifi_rssi = 0;
+	bool wifi_busy = false, wifi_under_b_mode = false, wifi_scan = false;
+	bool bt_idle = false;
+	static u8 cck_lock_counter, wl_noisy_count0, wl_noisy_count1 = 3,
+						     wl_noisy_count2;
+	u32 total_cnt, cck_cnt;
+	u32 cnt_crcok = 0, cnt_crcerr = 0;
+	static u8 cnt;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+	btcoexist->btc_get(btcoexist, BTC_GET_S4_WIFI_RSSI, &wifi_rssi);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_B_MODE,
+			   &wifi_under_b_mode);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_SCAN, &wifi_scan);
+
+	coex_sta->crc_ok_cck = btcoexist->btc_phydm_query_phy_counter(
+		btcoexist, "PHYDM_INFO_CRC32_OK_CCK");
+	coex_sta->crc_ok_11g = btcoexist->btc_phydm_query_phy_counter(
+		btcoexist, "PHYDM_INFO_CRC32_OK_LEGACY");
+	coex_sta->crc_ok_11n = btcoexist->btc_phydm_query_phy_counter(
+		btcoexist, "PHYDM_INFO_CRC32_OK_HT");
+	coex_sta->crc_ok_11n_vht = btcoexist->btc_phydm_query_phy_counter(
+		btcoexist, "PHYDM_INFO_CRC32_OK_VHT");
+
+	coex_sta->crc_err_cck = btcoexist->btc_phydm_query_phy_counter(
+		btcoexist, "PHYDM_INFO_CRC32_ERROR_CCK");
+	coex_sta->crc_err_11g = btcoexist->btc_phydm_query_phy_counter(
+		btcoexist, "PHYDM_INFO_CRC32_ERROR_LEGACY");
+	coex_sta->crc_err_11n = btcoexist->btc_phydm_query_phy_counter(
+		btcoexist, "PHYDM_INFO_CRC32_ERROR_HT");
+	coex_sta->crc_err_11n_vht = btcoexist->btc_phydm_query_phy_counter(
+		btcoexist, "PHYDM_INFO_CRC32_ERROR_VHT");
+
+	cnt_crcok = coex_sta->crc_ok_cck + coex_sta->crc_ok_11g +
+		    coex_sta->crc_ok_11n + coex_sta->crc_ok_11n_vht;
+
+	cnt_crcerr = coex_sta->crc_err_cck + coex_sta->crc_err_11g +
+		     coex_sta->crc_err_11n + coex_sta->crc_err_11n_vht;
+
+	if ((wifi_busy) && (cnt_crcerr != 0)) {
+		coex_sta->now_crc_ratio = cnt_crcok / cnt_crcerr;
+
+		if (cnt == 0)
+			coex_sta->acc_crc_ratio = coex_sta->now_crc_ratio;
+		else
+			coex_sta->acc_crc_ratio =
+				(coex_sta->acc_crc_ratio * 7 +
+				 coex_sta->now_crc_ratio * 3) /
+				10;
+
+		if (cnt >= 10)
+			cnt = 0;
+		else
+			cnt++;
+	}
+
+	cck_cnt = coex_sta->crc_ok_cck + coex_sta->crc_err_cck;
+
+	if ((coex_dm->bt_status ==
+	     BT_8822B_2ANT_BT_STATUS_NON_CONNECTED_IDLE) ||
+	    (coex_dm->bt_status == BT_8822B_2ANT_BT_STATUS_CONNECTED_IDLE) ||
+	    (coex_sta->bt_disabled))
+		bt_idle = true;
+
+	if (cck_cnt > 250) {
+		if (wl_noisy_count2 < 3)
+			wl_noisy_count2++;
+
+		if (wl_noisy_count2 == 3) {
+			wl_noisy_count0 = 0;
+			wl_noisy_count1 = 0;
+		}
+
+	} else if (cck_cnt < 50) {
+		if (wl_noisy_count0 < 3)
+			wl_noisy_count0++;
+
+		if (wl_noisy_count0 == 3) {
+			wl_noisy_count1 = 0;
+			wl_noisy_count2 = 0;
+		}
+
+	} else {
+		if (wl_noisy_count1 < 3)
+			wl_noisy_count1++;
+
+		if (wl_noisy_count1 == 3) {
+			wl_noisy_count0 = 0;
+			wl_noisy_count2 = 0;
+		}
+	}
+
+	if (wl_noisy_count2 == 3)
+		coex_sta->wl_noisy_level = 2;
+	else if (wl_noisy_count1 == 3)
+		coex_sta->wl_noisy_level = 1;
+	else
+		coex_sta->wl_noisy_level = 0;
+
+	if ((wifi_busy) && (wifi_rssi >= 30) && (!wifi_under_b_mode)) {
+		total_cnt = cnt_crcok;
+
+		if ((coex_dm->bt_status == BT_8822B_1ANT_BT_STATUS_ACL_BUSY) ||
+		    (coex_dm->bt_status ==
+		     BT_8822B_1ANT_BT_STATUS_ACL_SCO_BUSY) ||
+		    (coex_dm->bt_status == BT_8822B_1ANT_BT_STATUS_SCO_BUSY)) {
+			if (coex_sta->crc_ok_cck >
+			    (total_cnt - coex_sta->crc_ok_cck)) {
+				if (cck_lock_counter < 3)
+					cck_lock_counter++;
+			} else {
+				if (cck_lock_counter > 0)
+					cck_lock_counter--;
+			}
+
+		} else {
+			if (cck_lock_counter > 0)
+				cck_lock_counter--;
+		}
+	} else {
+		if (cck_lock_counter > 0)
+			cck_lock_counter--;
+	}
+
+	if (!coex_sta->pre_ccklock) {
+		if (cck_lock_counter >= 3)
+			coex_sta->cck_lock = true;
+		else
+			coex_sta->cck_lock = false;
+	} else {
+		if (cck_lock_counter == 0)
+			coex_sta->cck_lock = false;
+		else
+			coex_sta->cck_lock = true;
+	}
+
+	if (coex_sta->cck_lock)
+		coex_sta->cck_ever_lock = true;
+
+	coex_sta->pre_ccklock = coex_sta->cck_lock;
+}
+
+static bool
+halbtc8822b2ant_is_wifibt_status_changed(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	static bool pre_wifi_busy, pre_under_4way, pre_bt_hs_on, pre_bt_off,
+		pre_bt_slave, pre_hid_low_pri_tx_overhead, pre_wifi_under_lps,
+		pre_bt_setup_link;
+	static u8 pre_hid_busy_num, pre_wl_noisy_level;
+	bool wifi_busy = false, under_4way = false, bt_hs_on = false;
+	bool wifi_connected = false;
+	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_CONNECTED,
+			   &wifi_connected);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_HS_OPERATION, &bt_hs_on);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_4_WAY_PROGRESS,
+			   &under_4way);
+
+	if (coex_sta->bt_disabled != pre_bt_off) {
+		pre_bt_off = coex_sta->bt_disabled;
+
+		if (coex_sta->bt_disabled)
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], BT is disabled !!\n");
+		else
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], BT is enabled !!\n");
+
+		coex_sta->bt_coex_supported_feature = 0;
+		coex_sta->bt_coex_supported_version = 0;
+		coex_sta->bt_ble_scan_type = 0;
+		coex_sta->bt_ble_scan_para[0] = 0;
+		coex_sta->bt_ble_scan_para[1] = 0;
+		coex_sta->bt_ble_scan_para[2] = 0;
+		coex_sta->bt_reg_vendor_ac = 0xffff;
+		coex_sta->bt_reg_vendor_ae = 0xffff;
+		return true;
+	}
+
+	if (wifi_connected) {
+		if (wifi_busy != pre_wifi_busy) {
+			pre_wifi_busy = wifi_busy;
+			return true;
+		}
+		if (under_4way != pre_under_4way) {
+			pre_under_4way = under_4way;
+			return true;
+		}
+		if (bt_hs_on != pre_bt_hs_on) {
+			pre_bt_hs_on = bt_hs_on;
+			return true;
+		}
+		if (coex_sta->wl_noisy_level != pre_wl_noisy_level) {
+			pre_wl_noisy_level = coex_sta->wl_noisy_level;
+			return true;
+		}
+		if (coex_sta->under_lps != pre_wifi_under_lps) {
+			pre_wifi_under_lps = coex_sta->under_lps;
+			if (coex_sta->under_lps)
+				return true;
+		}
+	}
+
+	if (!coex_sta->bt_disabled) {
+		if (coex_sta->hid_busy_num != pre_hid_busy_num) {
+			pre_hid_busy_num = coex_sta->hid_busy_num;
+			return true;
+		}
+
+		if (bt_link_info->slave_role != pre_bt_slave) {
+			pre_bt_slave = bt_link_info->slave_role;
+			return true;
+		}
+
+		if (pre_hid_low_pri_tx_overhead !=
+		    coex_sta->is_hid_low_pri_tx_overhead) {
+			pre_hid_low_pri_tx_overhead =
+				coex_sta->is_hid_low_pri_tx_overhead;
+			return true;
+		}
+
+		if (pre_bt_setup_link != coex_sta->is_setup_link) {
+			pre_bt_setup_link = coex_sta->is_setup_link;
+			return true;
+		}
+	}
+
+	return false;
+}
+
+static void halbtc8822b2ant_update_bt_link_info(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
+	bool bt_hs_on = false;
+	bool bt_busy = false;
+
+	coex_sta->num_of_profile = 0;
+
+	/* set link exist status */
+	if (!(coex_sta->bt_info & BT_INFO_8822B_1ANT_B_CONNECTION)) {
+		coex_sta->bt_link_exist = false;
+		coex_sta->pan_exist = false;
+		coex_sta->a2dp_exist = false;
+		coex_sta->hid_exist = false;
+		coex_sta->sco_exist = false;
+	} else { /* connection exists */
+		coex_sta->bt_link_exist = true;
+		if (coex_sta->bt_info & BT_INFO_8822B_1ANT_B_FTP) {
+			coex_sta->pan_exist = true;
+			coex_sta->num_of_profile++;
+		} else {
+			coex_sta->pan_exist = false;
+		}
+
+		if (coex_sta->bt_info & BT_INFO_8822B_1ANT_B_A2DP) {
+			coex_sta->a2dp_exist = true;
+			coex_sta->num_of_profile++;
+		} else {
+			coex_sta->a2dp_exist = false;
+		}
+
+		if (coex_sta->bt_info & BT_INFO_8822B_1ANT_B_HID) {
+			coex_sta->hid_exist = true;
+			coex_sta->num_of_profile++;
+		} else {
+			coex_sta->hid_exist = false;
+		}
+
+		if (coex_sta->bt_info & BT_INFO_8822B_1ANT_B_SCO_ESCO) {
+			coex_sta->sco_exist = true;
+			coex_sta->num_of_profile++;
+		} else {
+			coex_sta->sco_exist = false;
+		}
+	}
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_HS_OPERATION, &bt_hs_on);
+
+	bt_link_info->bt_link_exist = coex_sta->bt_link_exist;
+	bt_link_info->sco_exist = coex_sta->sco_exist;
+	bt_link_info->a2dp_exist = coex_sta->a2dp_exist;
+	bt_link_info->pan_exist = coex_sta->pan_exist;
+	bt_link_info->hid_exist = coex_sta->hid_exist;
+	bt_link_info->acl_busy = coex_sta->acl_busy;
+
+	/* work around for HS mode. */
+	if (bt_hs_on) {
+		bt_link_info->pan_exist = true;
+		bt_link_info->bt_link_exist = true;
+	}
+
+	/* check if Sco only */
+	if (bt_link_info->sco_exist && !bt_link_info->a2dp_exist &&
+	    !bt_link_info->pan_exist && !bt_link_info->hid_exist)
+		bt_link_info->sco_only = true;
+	else
+		bt_link_info->sco_only = false;
+
+	/* check if A2dp only */
+	if (!bt_link_info->sco_exist && bt_link_info->a2dp_exist &&
+	    !bt_link_info->pan_exist && !bt_link_info->hid_exist)
+		bt_link_info->a2dp_only = true;
+	else
+		bt_link_info->a2dp_only = false;
+
+	/* check if Pan only */
+	if (!bt_link_info->sco_exist && !bt_link_info->a2dp_exist &&
+	    bt_link_info->pan_exist && !bt_link_info->hid_exist)
+		bt_link_info->pan_only = true;
+	else
+		bt_link_info->pan_only = false;
+
+	/* check if Hid only */
+	if (!bt_link_info->sco_exist && !bt_link_info->a2dp_exist &&
+	    !bt_link_info->pan_exist && bt_link_info->hid_exist)
+		bt_link_info->hid_only = true;
+	else
+		bt_link_info->hid_only = false;
+
+	if (coex_sta->bt_info & BT_INFO_8822B_2ANT_B_INQ_PAGE) {
+		coex_dm->bt_status = BT_8822B_2ANT_BT_STATUS_INQ_PAGE;
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], BtInfoNotify(), BT Inq/page!!!\n");
+	} else if (!(coex_sta->bt_info & BT_INFO_8822B_2ANT_B_CONNECTION)) {
+		coex_dm->bt_status = BT_8822B_2ANT_BT_STATUS_NON_CONNECTED_IDLE;
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], BtInfoNotify(), BT Non-Connected idle!!!\n");
+	} else if (coex_sta->bt_info == BT_INFO_8822B_2ANT_B_CONNECTION) {
+		/* connection exists but no busy */
+		coex_dm->bt_status = BT_8822B_2ANT_BT_STATUS_CONNECTED_IDLE;
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], BtInfoNotify(), BT Connected-idle!!!\n");
+	} else if (((coex_sta->bt_info & BT_INFO_8822B_2ANT_B_SCO_ESCO) ||
+		    (coex_sta->bt_info & BT_INFO_8822B_2ANT_B_SCO_BUSY)) &&
+		   (coex_sta->bt_info & BT_INFO_8822B_2ANT_B_ACL_BUSY)) {
+		coex_dm->bt_status = BT_8822B_2ANT_BT_STATUS_ACL_SCO_BUSY;
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], BtInfoNotify(), BT ACL SCO busy!!!\n");
+	} else if ((coex_sta->bt_info & BT_INFO_8822B_2ANT_B_SCO_ESCO) ||
+		   (coex_sta->bt_info & BT_INFO_8822B_2ANT_B_SCO_BUSY)) {
+		coex_dm->bt_status = BT_8822B_2ANT_BT_STATUS_SCO_BUSY;
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], BtInfoNotify(), BT SCO busy!!!\n");
+	} else if (coex_sta->bt_info & BT_INFO_8822B_2ANT_B_ACL_BUSY) {
+		coex_dm->bt_status = BT_8822B_2ANT_BT_STATUS_ACL_BUSY;
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], BtInfoNotify(), BT ACL busy!!!\n");
+	} else {
+		coex_dm->bt_status = BT_8822B_2ANT_BT_STATUS_MAX;
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], BtInfoNotify(), BT Non-Defined state!!!\n");
+	}
+
+	if ((coex_dm->bt_status == BT_8822B_2ANT_BT_STATUS_ACL_BUSY) ||
+	    (coex_dm->bt_status == BT_8822B_2ANT_BT_STATUS_SCO_BUSY) ||
+	    (coex_dm->bt_status == BT_8822B_2ANT_BT_STATUS_ACL_SCO_BUSY))
+		bt_busy = true;
+	else
+		bt_busy = false;
+
+	btcoexist->btc_set(btcoexist, BTC_SET_BL_BT_TRAFFIC_BUSY, &bt_busy);
+}
+
+static void halbtc8822b2ant_update_wifi_ch_info(struct btc_coexist *btcoexist,
+						u8 type)
+{
+	u8 h2c_parameter[3] = {0};
+	u32 wifi_bw;
+	u8 wifi_central_chnl;
+	u32 RTL97F_8822B = 0;
+
+	if (RTL97F_8822B)
+		return;
+
+	/* only 2.4G we need to inform bt the chnl mask */
+	btcoexist->btc_get(btcoexist, BTC_GET_U1_WIFI_CENTRAL_CHNL,
+			   &wifi_central_chnl);
+	if ((type == BTC_MEDIA_CONNECT) && (wifi_central_chnl <= 14)) {
+		/* enable BT AFH skip WL channel for 8822b
+		 * because BT Rx LO interference
+		 */
+		h2c_parameter[0] = 0x1;
+		h2c_parameter[1] = wifi_central_chnl;
+		btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_BW, &wifi_bw);
+		if (wifi_bw == BTC_WIFI_BW_HT40)
+			h2c_parameter[2] = 0x30;
+		else
+			h2c_parameter[2] = 0x20;
+	}
+
+	coex_dm->wifi_chnl_info[0] = h2c_parameter[0];
+	coex_dm->wifi_chnl_info[1] = h2c_parameter[1];
+	coex_dm->wifi_chnl_info[2] = h2c_parameter[2];
+
+	btcoexist->btc_fill_h2c(btcoexist, 0x66, 3, h2c_parameter);
+}
+
+static void
+halbtc8822b2ant_set_fw_dac_swing_level(struct btc_coexist *btcoexist,
+				       u8 dac_swing_lvl)
+{
+	u8 h2c_parameter[1] = {0};
+	u32 RTL97F_8822B = 0;
+
+	if (RTL97F_8822B)
+		return;
+
+	/* There are several type of dacswing */
+	/* 0x18/ 0x10/ 0xc/ 0x8/ 0x4/ 0x6 */
+	h2c_parameter[0] = dac_swing_lvl;
+
+	btcoexist->btc_fill_h2c(btcoexist, 0x64, 1, h2c_parameter);
+}
+
+static void halbtc8822b2ant_fw_dac_swing_lvl(struct btc_coexist *btcoexist,
+					     bool force_exec,
+					     u8 fw_dac_swing_lvl)
+{
+	u32 RTL97F_8822B = 0;
+
+	if (RTL97F_8822B)
+		return;
+
+	coex_dm->cur_fw_dac_swing_lvl = fw_dac_swing_lvl;
+
+	if (!force_exec) {
+		if (coex_dm->pre_fw_dac_swing_lvl ==
+		    coex_dm->cur_fw_dac_swing_lvl)
+			return;
+	}
+
+	halbtc8822b2ant_set_fw_dac_swing_level(btcoexist,
+					       coex_dm->cur_fw_dac_swing_lvl);
+
+	coex_dm->pre_fw_dac_swing_lvl = coex_dm->cur_fw_dac_swing_lvl;
+}
+
+static void halbtc8822b2ant_set_fw_dec_bt_pwr(struct btc_coexist *btcoexist,
+					      u8 dec_bt_pwr_lvl)
+{
+	u32 RTL97F_8822B = 0;
+	u8 h2c_parameter[1] = {0};
+
+	if (RTL97F_8822B)
+		return;
+
+	h2c_parameter[0] = dec_bt_pwr_lvl;
+
+	btcoexist->btc_fill_h2c(btcoexist, 0x62, 1, h2c_parameter);
+}
+
+static void halbtc8822b2ant_dec_bt_pwr(struct btc_coexist *btcoexist,
+				       bool force_exec, u8 dec_bt_pwr_lvl)
+{
+	coex_dm->cur_bt_dec_pwr_lvl = dec_bt_pwr_lvl;
+
+	if (!force_exec) {
+		if (coex_dm->pre_bt_dec_pwr_lvl == coex_dm->cur_bt_dec_pwr_lvl)
+			return;
+	}
+	halbtc8822b2ant_set_fw_dec_bt_pwr(btcoexist,
+					  coex_dm->cur_bt_dec_pwr_lvl);
+
+	coex_dm->pre_bt_dec_pwr_lvl = coex_dm->cur_bt_dec_pwr_lvl;
+}
+
+static void halbtc8822b2ant_low_penalty_ra(struct btc_coexist *btcoexist,
+					   bool force_exec, bool low_penalty_ra)
+{
+	coex_dm->cur_low_penalty_ra = low_penalty_ra;
+
+	if (!force_exec) {
+		if (coex_dm->pre_low_penalty_ra == coex_dm->cur_low_penalty_ra)
+			return;
+	}
+
+	if (low_penalty_ra)
+		btcoexist->btc_phydm_modify_ra_pcr_threshold(btcoexist, 0, 50);
+	else
+		btcoexist->btc_phydm_modify_ra_pcr_threshold(btcoexist, 0, 0);
+
+	coex_dm->pre_low_penalty_ra = coex_dm->cur_low_penalty_ra;
+}
+
+static void halbtc8822b2ant_write_score_board(struct btc_coexist *btcoexist,
+					      u16 bitpos, bool state)
+{
+	static u16 originalval = 0x8002;
+
+	if (state)
+		originalval = originalval | bitpos;
+	else
+		originalval = originalval & (~bitpos);
+
+	btcoexist->btc_write_2byte(btcoexist, 0xaa, originalval);
+}
+
+static void halbtc8822b2ant_read_score_board(struct btc_coexist *btcoexist,
+					     u16 *score_board_val)
+{
+	*score_board_val =
+		(btcoexist->btc_read_2byte(btcoexist, 0xaa)) & 0x7fff;
+}
+
+static void halbtc8822b2ant_post_state_to_bt(struct btc_coexist *btcoexist,
+					     u16 type, bool state)
+{
+	halbtc8822b2ant_write_score_board(btcoexist, (u16)type, state);
+}
+
+static void
+halbtc8822b2ant_monitor_bt_enable_disable(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	static u32 bt_disable_cnt;
+	bool bt_active = true, bt_disabled = false, wifi_under_5g = false;
+	u16 u16tmp;
+
+	/* This function check if bt is disabled */
+
+	/* Read BT on/off status from scoreboard[1],
+	 * enable this only if BT patch support this feature
+	 */
+	halbtc8822b2ant_read_score_board(btcoexist, &u16tmp);
+
+	bt_active = u16tmp & BIT(1);
+
+	if (bt_active) {
+		bt_disable_cnt = 0;
+		bt_disabled = false;
+		btcoexist->btc_set(btcoexist, BTC_SET_BL_BT_DISABLE,
+				   &bt_disabled);
+	} else {
+		bt_disable_cnt++;
+		if (bt_disable_cnt >= 10) {
+			bt_disabled = true;
+			bt_disable_cnt = 10;
+		}
+
+		btcoexist->btc_set(btcoexist, BTC_SET_BL_BT_DISABLE,
+				   &bt_disabled);
+	}
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_5G, &wifi_under_5g);
+
+	if ((wifi_under_5g) || (bt_disabled))
+		halbtc8822b2ant_low_penalty_ra(btcoexist, NORMAL_EXEC, false);
+	else
+		halbtc8822b2ant_low_penalty_ra(btcoexist, NORMAL_EXEC, true);
+
+	if (coex_sta->bt_disabled != bt_disabled) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], BT is from %s to %s!!\n",
+			 (coex_sta->bt_disabled ? "disabled" : "enabled"),
+			 (bt_disabled ? "disabled" : "enabled"));
+		coex_sta->bt_disabled = bt_disabled;
+	}
+}
+
+static void halbtc8822b2ant_enable_gnt_to_gpio(struct btc_coexist *btcoexist,
+					       bool isenable)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	static u8 bit_val[5] = {0, 0, 0, 0, 0};
+
+	if (!btcoexist->dbg_mode_2ant)
+		return;
+
+	if (isenable) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], enable_gnt_to_gpio!!\n");
+
+		/* enable GNT_WL, GNT_BT to GPIO for debug */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x73, 0x8, 0x1);
+
+		/* store original value */
+		bit_val[0] =
+			(btcoexist->btc_read_1byte(btcoexist, 0x66) & BIT(4)) >>
+			4; /*0x66[4] */
+		bit_val[1] = (btcoexist->btc_read_1byte(btcoexist, 0x67) &
+			      BIT(0)); /*0x66[8] */
+		bit_val[2] =
+			(btcoexist->btc_read_1byte(btcoexist, 0x42) & BIT(3)) >>
+			3; /*0x40[19] */
+		bit_val[3] =
+			(btcoexist->btc_read_1byte(btcoexist, 0x65) & BIT(7)) >>
+			7; /*0x64[15] */
+		bit_val[4] =
+			(btcoexist->btc_read_1byte(btcoexist, 0x72) & BIT(2)) >>
+			2; /*0x70[18] */
+
+		/*  switch GPIO Mux */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x66, BIT(4),
+						   0x0); /*0x66[4] = 0 */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x67, BIT(0),
+						   0x0); /*0x66[8] = 0 */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x42, BIT(3),
+						   0x0); /*0x40[19] = 0 */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x65, BIT(7),
+						   0x0); /*0x64[15] = 0 */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x72, BIT(2),
+						   0x0); /*0x70[18] = 0 */
+
+	} else {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], disable_gnt_to_gpio!!\n");
+
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x73, 0x8, 0x0);
+
+		/*  Restore original value  */
+		/*  switch GPIO Mux */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x66, BIT(4),
+						   bit_val[0]); /*0x66[4] = 0 */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x67, BIT(0),
+						   bit_val[1]); /*0x66[8] = 0 */
+		btcoexist->btc_write_1byte_bitmask(
+			btcoexist, 0x42, BIT(3), bit_val[2]); /*0x40[19] = 0 */
+		btcoexist->btc_write_1byte_bitmask(
+			btcoexist, 0x65, BIT(7), bit_val[3]); /*0x64[15] = 0 */
+		btcoexist->btc_write_1byte_bitmask(
+			btcoexist, 0x72, BIT(2), bit_val[4]); /*0x70[18] = 0 */
+	}
+}
+
+static u32
+halbtc8822b2ant_ltecoex_indirect_read_reg(struct btc_coexist *btcoexist,
+					  u16 reg_addr)
+{
+	u32 delay_count = 0;
+
+	while (1) {
+		if ((btcoexist->btc_read_1byte(btcoexist, 0x1703) & BIT(5)) ==
+		    0) {
+			mdelay(50);
+			delay_count++;
+			if (delay_count >= 10) {
+				delay_count = 0;
+				break;
+			}
+		} else {
+			break;
+		}
+	}
+
+	/* wait for ready bit before access 0x1700		 */
+	btcoexist->btc_write_4byte(btcoexist, 0x1700, 0x800F0000 | reg_addr);
+
+	return btcoexist->btc_read_4byte(btcoexist, 0x1708); /* get read data */
+}
+
+static void
+halbtc8822b2ant_ltecoex_indirect_write_reg(struct btc_coexist *btcoexist,
+					   u16 reg_addr, u32 bit_mask,
+					   u32 reg_value)
+{
+	u32 val, i = 0, bitpos = 0, delay_count = 0;
+
+	if (bit_mask == 0x0)
+		return;
+	if (bit_mask == 0xffffffff) {
+		/* wait for ready bit before access 0x1700/0x1704 */
+		while (1) {
+			if ((btcoexist->btc_read_1byte(btcoexist, 0x1703) &
+			     BIT(5)) == 0) {
+				mdelay(50);
+				delay_count++;
+				if (delay_count >= 10) {
+					delay_count = 0;
+					break;
+				}
+			} else {
+				break;
+			}
+		}
+
+		btcoexist->btc_write_4byte(btcoexist, 0x1704,
+					   reg_value); /* put write data */
+
+		btcoexist->btc_write_4byte(btcoexist, 0x1700,
+					   0xc00F0000 | reg_addr);
+	} else {
+		for (i = 0; i <= 31; i++) {
+			if (((bit_mask >> i) & 0x1) == 0x1) {
+				bitpos = i;
+				break;
+			}
+		}
+
+		/* read back register value before write */
+		val = halbtc8822b2ant_ltecoex_indirect_read_reg(btcoexist,
+								reg_addr);
+		val = (val & (~bit_mask)) | (reg_value << bitpos);
+
+		/* wait for ready bit before access 0x1700/0x1704 */
+		while (1) {
+			if ((btcoexist->btc_read_1byte(btcoexist, 0x1703) &
+			     BIT(5)) == 0) {
+				mdelay(50);
+				delay_count++;
+				if (delay_count >= 10) {
+					delay_count = 0;
+					break;
+				}
+			} else {
+				break;
+			}
+		}
+
+		btcoexist->btc_write_4byte(btcoexist, 0x1704,
+					   val); /* put write data */
+
+		btcoexist->btc_write_4byte(btcoexist, 0x1700,
+					   0xc00F0000 | reg_addr);
+	}
+}
+
+static void halbtc8822b2ant_ltecoex_enable(struct btc_coexist *btcoexist,
+					   bool enable)
+{
+	u8 val;
+
+	val = (enable) ? 1 : 0;
+	halbtc8822b2ant_ltecoex_indirect_write_reg(btcoexist, 0x38, 0x80,
+						   val); /* 0x38[7] */
+}
+
+static void
+halbtc8822b2ant_ltecoex_pathcontrol_owner(struct btc_coexist *btcoexist,
+					  bool wifi_control)
+{
+	u8 val;
+
+	val = (wifi_control) ? 1 : 0;
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0x73, 0x4,
+					   val); /* 0x70[26] */
+}
+
+static void halbtc8822b2ant_ltecoex_set_gnt_bt(struct btc_coexist *btcoexist,
+					       u8 control_block,
+					       bool sw_control, u8 state)
+{
+	u32 val = 0, bit_mask;
+
+	state = state & 0x1;
+	val = (sw_control) ? ((state << 1) | 0x1) : 0;
+
+	switch (control_block) {
+	case BT_8822B_2ANT_GNT_BLOCK_RFC_BB:
+	default:
+		bit_mask = 0xc000;
+		halbtc8822b2ant_ltecoex_indirect_write_reg(
+			btcoexist, 0x38, bit_mask, val); /* 0x38[15:14] */
+		bit_mask = 0x0c00;
+		halbtc8822b2ant_ltecoex_indirect_write_reg(
+			btcoexist, 0x38, bit_mask, val); /* 0x38[11:10] */
+		break;
+	case BT_8822B_2ANT_GNT_BLOCK_RFC:
+		bit_mask = 0xc000;
+		halbtc8822b2ant_ltecoex_indirect_write_reg(
+			btcoexist, 0x38, bit_mask, val); /* 0x38[15:14] */
+		break;
+	case BT_8822B_2ANT_GNT_BLOCK_BB:
+		bit_mask = 0x0c00;
+		halbtc8822b2ant_ltecoex_indirect_write_reg(
+			btcoexist, 0x38, bit_mask, val); /* 0x38[11:10] */
+		break;
+	}
+}
+
+static void halbtc8822b2ant_ltecoex_set_gnt_wl(struct btc_coexist *btcoexist,
+					       u8 control_block,
+					       bool sw_control, u8 state)
+{
+	u32 val = 0, bit_mask;
+
+	state = state & 0x1;
+	val = (sw_control) ? ((state << 1) | 0x1) : 0;
+
+	switch (control_block) {
+	case BT_8822B_2ANT_GNT_BLOCK_RFC_BB:
+	default:
+		bit_mask = 0x3000;
+		halbtc8822b2ant_ltecoex_indirect_write_reg(
+			btcoexist, 0x38, bit_mask, val); /* 0x38[13:12] */
+		bit_mask = 0x0300;
+		halbtc8822b2ant_ltecoex_indirect_write_reg(
+			btcoexist, 0x38, bit_mask, val); /* 0x38[9:8] */
+		break;
+	case BT_8822B_2ANT_GNT_BLOCK_RFC:
+		bit_mask = 0x3000;
+		halbtc8822b2ant_ltecoex_indirect_write_reg(
+			btcoexist, 0x38, bit_mask, val); /* 0x38[13:12] */
+		break;
+	case BT_8822B_2ANT_GNT_BLOCK_BB:
+		bit_mask = 0x0300;
+		halbtc8822b2ant_ltecoex_indirect_write_reg(
+			btcoexist, 0x38, bit_mask, val); /* 0x38[9:8] */
+		break;
+	}
+}
+
+static void
+halbtc8822b2ant_ltecoex_set_coex_table(struct btc_coexist *btcoexist,
+				       u8 table_type, u16 table_content)
+{
+	u16 reg_addr = 0x0000;
+
+	switch (table_type) {
+	case BT_8822B_2ANT_CTT_WL_VS_LTE:
+		reg_addr = 0xa0;
+		break;
+	case BT_8822B_2ANT_CTT_BT_VS_LTE:
+		reg_addr = 0xa4;
+		break;
+	}
+
+	if (reg_addr != 0x0000)
+		halbtc8822b2ant_ltecoex_indirect_write_reg(
+			btcoexist, reg_addr, 0xffff,
+			table_content); /* 0xa0[15:0] or 0xa4[15:0] */
+}
+
+static void halbtc8822b2ant_set_wltoggle_coex_table(
+	struct btc_coexist *btcoexist, bool force_exec, u8 interval,
+	u8 val0x6c4_b0, u8 val0x6c4_b1, u8 val0x6c4_b2, u8 val0x6c4_b3)
+{
+	static u8 pre_h2c_parameter[6] = {0};
+	u8 cur_h2c_parameter[6] = {0};
+	u8 i, match_cnt = 0;
+
+	cur_h2c_parameter[0] = 0x7; /* op_code, 0x7= wlan toggle slot*/
+
+	cur_h2c_parameter[1] = interval;
+	cur_h2c_parameter[2] = val0x6c4_b0;
+	cur_h2c_parameter[3] = val0x6c4_b1;
+	cur_h2c_parameter[4] = val0x6c4_b2;
+	cur_h2c_parameter[5] = val0x6c4_b3;
+
+	if (!force_exec) {
+		for (i = 1; i <= 5; i++) {
+			if (cur_h2c_parameter[i] != pre_h2c_parameter[i])
+				break;
+
+			match_cnt++;
+		}
+
+		if (match_cnt == 5)
+			return;
+	}
+
+	for (i = 1; i <= 5; i++)
+		pre_h2c_parameter[i] = cur_h2c_parameter[i];
+
+	btcoexist->btc_fill_h2c(btcoexist, 0x69, 6, cur_h2c_parameter);
+}
+
+static void halbtc8822b2ant_set_coex_table(struct btc_coexist *btcoexist,
+					   u32 val0x6c0, u32 val0x6c4,
+					   u32 val0x6c8, u8 val0x6cc)
+{
+	btcoexist->btc_write_4byte(btcoexist, 0x6c0, val0x6c0);
+
+	btcoexist->btc_write_4byte(btcoexist, 0x6c4, val0x6c4);
+
+	btcoexist->btc_write_4byte(btcoexist, 0x6c8, val0x6c8);
+
+	btcoexist->btc_write_1byte(btcoexist, 0x6cc, val0x6cc);
+}
+
+static void halbtc8822b2ant_coex_table(struct btc_coexist *btcoexist,
+				       bool force_exec, u32 val0x6c0,
+				       u32 val0x6c4, u32 val0x6c8, u8 val0x6cc)
+{
+	coex_dm->cur_val0x6c0 = val0x6c0;
+	coex_dm->cur_val0x6c4 = val0x6c4;
+	coex_dm->cur_val0x6c8 = val0x6c8;
+	coex_dm->cur_val0x6cc = val0x6cc;
+
+	if (!force_exec) {
+		if ((coex_dm->pre_val0x6c0 == coex_dm->cur_val0x6c0) &&
+		    (coex_dm->pre_val0x6c4 == coex_dm->cur_val0x6c4) &&
+		    (coex_dm->pre_val0x6c8 == coex_dm->cur_val0x6c8) &&
+		    (coex_dm->pre_val0x6cc == coex_dm->cur_val0x6cc))
+			return;
+	}
+	halbtc8822b2ant_set_coex_table(btcoexist, val0x6c0, val0x6c4, val0x6c8,
+				       val0x6cc);
+
+	coex_dm->pre_val0x6c0 = coex_dm->cur_val0x6c0;
+	coex_dm->pre_val0x6c4 = coex_dm->cur_val0x6c4;
+	coex_dm->pre_val0x6c8 = coex_dm->cur_val0x6c8;
+	coex_dm->pre_val0x6cc = coex_dm->cur_val0x6cc;
+}
+
+static void halbtc8822b2ant_coex_table_with_type(struct btc_coexist *btcoexist,
+						 bool force_exec, u8 type)
+{
+	u32 break_table;
+	u8 select_table;
+
+	coex_sta->coex_table_type = type;
+
+	if (coex_sta->concurrent_rx_mode_on) {
+		break_table = 0xf0ffffff; /* set WL hi-pri can break BT */
+		/* set Tx response = Hi-Pri (ex: Transmitting ACK,BA,CTS) */
+		select_table = 0xb;
+	} else {
+		break_table = 0xffffff;
+		select_table = 0x3;
+	}
+
+	switch (type) {
+	case 0:
+		halbtc8822b2ant_coex_table(btcoexist, force_exec, 0xffffffff,
+					   0xffffffff, break_table,
+					   select_table);
+		break;
+	case 1:
+		halbtc8822b2ant_coex_table(btcoexist, force_exec, 0x55555555,
+					   0x5a5a5a5a, break_table,
+					   select_table);
+		break;
+	case 2:
+		halbtc8822b2ant_coex_table(btcoexist, force_exec, 0x5a5a5a5a,
+					   0x5a5a5a5a, break_table,
+					   select_table);
+		break;
+	case 3:
+		halbtc8822b2ant_coex_table(btcoexist, force_exec, 0x55555555,
+					   0x5a5a5a5a, break_table,
+					   select_table);
+		break;
+	case 4:
+		halbtc8822b2ant_coex_table(btcoexist, force_exec, 0x55555555,
+					   0x5a5a5a5a, break_table,
+					   select_table);
+		break;
+	case 5:
+		halbtc8822b2ant_coex_table(btcoexist, force_exec, 0x55555555,
+					   0x55555555, break_table,
+					   select_table);
+		break;
+	case 6:
+		halbtc8822b2ant_coex_table(btcoexist, force_exec, 0xa5555555,
+					   0xfafafafa, break_table,
+					   select_table);
+		break;
+	case 7:
+		halbtc8822b2ant_coex_table(btcoexist, force_exec, 0xa5555555,
+					   0xaa5a5a5a, break_table,
+					   select_table);
+		break;
+	case 8:
+		halbtc8822b2ant_coex_table(btcoexist, force_exec, 0xa5555555,
+					   0xfafafafa, break_table,
+					   select_table);
+		break;
+	case 9:
+		halbtc8822b2ant_coex_table(btcoexist, force_exec, 0x5a5a5a5a,
+					   0xaaaa5aaa, break_table,
+					   select_table);
+		break;
+	case 10:
+		halbtc8822b2ant_coex_table(btcoexist, force_exec, 0x55555555,
+					   0x5a5a555a, break_table,
+					   select_table);
+		break;
+	default:
+		break;
+	}
+}
+
+static void
+halbtc8822b2ant_set_fw_ignore_wlan_act(struct btc_coexist *btcoexist,
+				       bool enable)
+{
+	u8 h2c_parameter[1] = {0};
+	u32 RTL97F_8822B = 0;
+
+	if (RTL97F_8822B)
+		return;
+
+	if (enable)
+		h2c_parameter[0] |= BIT(0); /* function enable */
+
+	btcoexist->btc_fill_h2c(btcoexist, 0x63, 1, h2c_parameter);
+}
+
+static void halbtc8822b2ant_ignore_wlan_act(struct btc_coexist *btcoexist,
+					    bool force_exec, bool enable)
+{
+	coex_dm->cur_ignore_wlan_act = enable;
+
+	if (!force_exec) {
+		if (coex_dm->pre_ignore_wlan_act ==
+		    coex_dm->cur_ignore_wlan_act)
+			return;
+	}
+	halbtc8822b2ant_set_fw_ignore_wlan_act(btcoexist, enable);
+
+	coex_dm->pre_ignore_wlan_act = coex_dm->cur_ignore_wlan_act;
+}
+
+static void halbtc8822b2ant_set_lps_rpwm(struct btc_coexist *btcoexist,
+					 u8 lps_val, u8 rpwm_val)
+{
+	u8 lps = lps_val;
+	u8 rpwm = rpwm_val;
+
+	btcoexist->btc_set(btcoexist, BTC_SET_U1_LPS_VAL, &lps);
+	btcoexist->btc_set(btcoexist, BTC_SET_U1_RPWM_VAL, &rpwm);
+}
+
+static void halbtc8822b2ant_lps_rpwm(struct btc_coexist *btcoexist,
+				     bool force_exec, u8 lps_val, u8 rpwm_val)
+{
+	coex_dm->cur_lps = lps_val;
+	coex_dm->cur_rpwm = rpwm_val;
+
+	if (!force_exec) {
+		if ((coex_dm->pre_lps == coex_dm->cur_lps) &&
+		    (coex_dm->pre_rpwm == coex_dm->cur_rpwm))
+			return;
+	}
+	halbtc8822b2ant_set_lps_rpwm(btcoexist, lps_val, rpwm_val);
+
+	coex_dm->pre_lps = coex_dm->cur_lps;
+	coex_dm->pre_rpwm = coex_dm->cur_rpwm;
+}
+
+static void halbtc8822b2ant_ps_tdma_check_for_power_save_state(
+	struct btc_coexist *btcoexist, bool new_ps_state)
+{
+	u8 lps_mode = 0x0;
+	u8 h2c_parameter[5] = {0, 0, 0, 0x40, 0};
+	u32 RTL97F_8822B = 0;
+
+	if (RTL97F_8822B)
+		return;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_U1_LPS_MODE, &lps_mode);
+
+	if (lps_mode) { /* already under LPS state */
+		if (new_ps_state) {
+			/* keep state under LPS, do nothing. */
+		} else {
+			/* will leave LPS state, turn off psTdma first */
+			btcoexist->btc_fill_h2c(btcoexist, 0x60, 5,
+						h2c_parameter);
+		}
+	} else { /* NO PS state */
+		if (new_ps_state) {
+			/* will enter LPS state, turn off psTdma first */
+			btcoexist->btc_fill_h2c(btcoexist, 0x60, 5,
+						h2c_parameter);
+		} else {
+			/* keep state under NO PS state, do nothing. */
+		}
+	}
+}
+
+static bool halbtc8822b2ant_power_save_state(struct btc_coexist *btcoexist,
+					     u8 ps_type, u8 lps_val,
+					     u8 rpwm_val)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	bool low_pwr_disable = false, result = true;
+
+	switch (ps_type) {
+	case BTC_PS_WIFI_NATIVE:
+		coex_sta->force_lps_ctrl = false;
+		/* recover to original 32k low power setting */
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], %s == BTC_PS_WIFI_NATIVE\n", __func__);
+
+		low_pwr_disable = false;
+		btcoexist->btc_set(btcoexist, BTC_SET_ACT_DISABLE_LOW_POWER,
+				   &low_pwr_disable);
+		btcoexist->btc_set(btcoexist, BTC_SET_ACT_NORMAL_LPS, NULL);
+		break;
+	case BTC_PS_LPS_ON:
+		coex_sta->force_lps_ctrl = true;
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], %s == BTC_PS_LPS_ON\n", __func__);
+
+		halbtc8822b2ant_ps_tdma_check_for_power_save_state(btcoexist,
+								   true);
+		halbtc8822b2ant_lps_rpwm(btcoexist, NORMAL_EXEC, lps_val,
+					 rpwm_val);
+		/* when coex force to enter LPS, do not enter 32k low power. */
+		low_pwr_disable = true;
+		btcoexist->btc_set(btcoexist, BTC_SET_ACT_DISABLE_LOW_POWER,
+				   &low_pwr_disable);
+		/* power save must executed before psTdma. */
+		btcoexist->btc_set(btcoexist, BTC_SET_ACT_ENTER_LPS, NULL);
+		break;
+	case BTC_PS_LPS_OFF:
+		coex_sta->force_lps_ctrl = true;
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], %s == BTC_PS_LPS_OFF\n", __func__);
+
+		halbtc8822b2ant_ps_tdma_check_for_power_save_state(btcoexist,
+								   false);
+		result = btcoexist->btc_set(btcoexist, BTC_SET_ACT_LEAVE_LPS,
+					    NULL);
+		break;
+	default:
+		break;
+	}
+
+	return result;
+}
+
+static void halbtc8822b2ant_set_fw_pstdma(struct btc_coexist *btcoexist,
+					  u8 byte1, u8 byte2, u8 byte3,
+					  u8 byte4, u8 byte5)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	u8 h2c_parameter[5] = {0};
+	u8 real_byte1 = byte1, real_byte5 = byte5;
+	bool ap_enable = false, result = false;
+	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
+
+	if (byte5 & BIT(2))
+		coex_sta->is_tdma_btautoslot = true;
+	else
+		coex_sta->is_tdma_btautoslot = false;
+
+	/* release bt-auto slot for auto-slot hang is detected!! */
+	if (coex_sta->is_tdma_btautoslot)
+		if ((coex_sta->is_tdma_btautoslot_hang) ||
+		    (bt_link_info->slave_role))
+			byte5 = byte5 & 0xfb;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_AP_MODE_ENABLE,
+			   &ap_enable);
+
+	if ((ap_enable) && (byte1 & BIT(4) && !(byte1 & BIT(5)))) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], %s == FW for AP mode\n", __func__);
+
+		real_byte1 &= ~BIT(4);
+		real_byte1 |= BIT(5);
+
+		real_byte5 |= BIT(5);
+		real_byte5 &= ~BIT(6);
+
+		halbtc8822b2ant_power_save_state(btcoexist, BTC_PS_WIFI_NATIVE,
+						 0x0, 0x0);
+	} else if (byte1 & BIT(4) && !(byte1 & BIT(5))) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], %s == Force LPS (byte1 = 0x%x)\n",
+			 __func__, byte1);
+
+		if (!halbtc8822b2ant_power_save_state(btcoexist, BTC_PS_LPS_OFF,
+						      0x50, 0x4))
+			result = true;
+	} else {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], %s == Native LPS (byte1 = 0x%x)\n",
+			 __func__, byte1);
+
+		halbtc8822b2ant_power_save_state(btcoexist, BTC_PS_WIFI_NATIVE,
+						 0x0, 0x0);
+	}
+
+	coex_sta->is_set_ps_state_fail = result;
+
+	if (!coex_sta->is_set_ps_state_fail) {
+		h2c_parameter[0] = real_byte1;
+		h2c_parameter[1] = byte2;
+		h2c_parameter[2] = byte3;
+		h2c_parameter[3] = byte4;
+		h2c_parameter[4] = real_byte5;
+
+		coex_dm->ps_tdma_para[0] = real_byte1;
+		coex_dm->ps_tdma_para[1] = byte2;
+		coex_dm->ps_tdma_para[2] = byte3;
+		coex_dm->ps_tdma_para[3] = byte4;
+		coex_dm->ps_tdma_para[4] = real_byte5;
+
+		btcoexist->btc_fill_h2c(btcoexist, 0x60, 5, h2c_parameter);
+	} else {
+		coex_sta->cnt_set_ps_state_fail++;
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], %s == Force Leave LPS Fail (cnt = %d)\n",
+			 __func__, coex_sta->cnt_set_ps_state_fail);
+	}
+}
+
+static void halbtc8822b2ant_ps_tdma(struct btc_coexist *btcoexist,
+				    bool force_exec, bool turn_on, u8 type)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	static u8 ps_tdma_byte4_modify, pre_ps_tdma_byte4_modify;
+	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
+
+	coex_dm->cur_ps_tdma_on = turn_on;
+	coex_dm->cur_ps_tdma = type;
+
+	/* 0x778 = 0x1 at wifi slot (no blocking BT Low-Pri pkts) */
+	if (bt_link_info->slave_role)
+		ps_tdma_byte4_modify = 0x1;
+	else
+		ps_tdma_byte4_modify = 0x0;
+
+	if (pre_ps_tdma_byte4_modify != ps_tdma_byte4_modify) {
+		force_exec = true;
+		pre_ps_tdma_byte4_modify = ps_tdma_byte4_modify;
+	}
+
+	if (!force_exec) {
+		if ((coex_dm->pre_ps_tdma_on == coex_dm->cur_ps_tdma_on) &&
+		    (coex_dm->pre_ps_tdma == coex_dm->cur_ps_tdma)) {
+			RT_TRACE(
+				rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				"[BTCoex], Skip TDMA because no change TDMA(%s, %d)\n",
+				(coex_dm->cur_ps_tdma_on ? "on" : "off"),
+				coex_dm->cur_ps_tdma);
+			return;
+		}
+	}
+
+	if (coex_dm->cur_ps_tdma_on) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], ********** TDMA(on, %d) **********\n",
+			 coex_dm->cur_ps_tdma);
+
+		btcoexist->btc_write_1byte_bitmask(
+			btcoexist, 0x550, 0x8, 0x1); /* enable TBTT nterrupt */
+	} else {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], ********** TDMA(off, %d) **********\n",
+			 coex_dm->cur_ps_tdma);
+	}
+
+	if (turn_on) {
+		switch (type) {
+		case 1:
+			halbtc8822b2ant_set_fw_pstdma(
+				btcoexist, 0x61, 0x10, 0x03, 0x91,
+				0x54 | ps_tdma_byte4_modify);
+			break;
+		case 2:
+		default:
+			halbtc8822b2ant_set_fw_pstdma(
+				btcoexist, 0x61, 0x35, 0x03, 0x11,
+				0x11 | ps_tdma_byte4_modify);
+			break;
+		case 3:
+			halbtc8822b2ant_set_fw_pstdma(
+				btcoexist, 0x61, 0x3a, 0x3, 0x91,
+				0x10 | ps_tdma_byte4_modify);
+			break;
+		case 4:
+			halbtc8822b2ant_set_fw_pstdma(
+				btcoexist, 0x61, 0x21, 0x3, 0x91,
+				0x10 | ps_tdma_byte4_modify);
+			break;
+		case 5:
+			halbtc8822b2ant_set_fw_pstdma(
+				btcoexist, 0x61, 0x25, 0x3, 0x91,
+				0x10 | ps_tdma_byte4_modify);
+			break;
+		case 6:
+			halbtc8822b2ant_set_fw_pstdma(
+				btcoexist, 0x61, 0x10, 0x3, 0x91,
+				0x10 | ps_tdma_byte4_modify);
+			break;
+		case 7:
+			halbtc8822b2ant_set_fw_pstdma(
+				btcoexist, 0x61, 0x20, 0x3, 0x91,
+				0x10 | ps_tdma_byte4_modify);
+			break;
+		case 8:
+			halbtc8822b2ant_set_fw_pstdma(btcoexist, 0x61, 0x15,
+						      0x03, 0x11, 0x11);
+			break;
+		case 10:
+			halbtc8822b2ant_set_fw_pstdma(btcoexist, 0x61, 0x30,
+						      0x03, 0x11, 0x10);
+			break;
+		case 11:
+			halbtc8822b2ant_set_fw_pstdma(
+				btcoexist, 0x61, 0x35, 0x03, 0x11,
+				0x10 | ps_tdma_byte4_modify);
+			break;
+		case 12:
+			halbtc8822b2ant_set_fw_pstdma(btcoexist, 0x61, 0x35,
+						      0x03, 0x11, 0x11);
+			break;
+		case 13:
+			halbtc8822b2ant_set_fw_pstdma(
+				btcoexist, 0x61, 0x1c, 0x03, 0x11,
+				0x10 | ps_tdma_byte4_modify);
+			break;
+		case 14:
+			halbtc8822b2ant_set_fw_pstdma(btcoexist, 0x61, 0x20,
+						      0x03, 0x11, 0x11);
+			break;
+		case 15:
+			halbtc8822b2ant_set_fw_pstdma(btcoexist, 0x61, 0x10,
+						      0x03, 0x11, 0x14);
+			break;
+		case 16:
+			halbtc8822b2ant_set_fw_pstdma(btcoexist, 0x61, 0x10,
+						      0x03, 0x11, 0x15);
+			break;
+		case 21:
+			halbtc8822b2ant_set_fw_pstdma(btcoexist, 0x61, 0x30,
+						      0x03, 0x11, 0x10);
+			break;
+		case 22:
+			halbtc8822b2ant_set_fw_pstdma(btcoexist, 0x61, 0x25,
+						      0x03, 0x11, 0x10);
+			break;
+		case 23:
+			halbtc8822b2ant_set_fw_pstdma(btcoexist, 0x61, 0x10,
+						      0x03, 0x11, 0x10);
+			break;
+		case 51:
+			halbtc8822b2ant_set_fw_pstdma(
+				btcoexist, 0x61, 0x10, 0x03, 0x91,
+				0x10 | ps_tdma_byte4_modify);
+			break;
+		case 101:
+			halbtc8822b2ant_set_fw_pstdma(
+				btcoexist, 0x61, 0x25, 0x03, 0x11,
+				0x11 | ps_tdma_byte4_modify);
+			break;
+		case 102:
+			halbtc8822b2ant_set_fw_pstdma(
+				btcoexist, 0x61, 0x35, 0x03, 0x11,
+				0x11 | ps_tdma_byte4_modify);
+			break;
+		case 103:
+			halbtc8822b2ant_set_fw_pstdma(
+				btcoexist, 0x51, 0x3a, 0x3, 0x10,
+				0x50 | ps_tdma_byte4_modify);
+			break;
+		case 104:
+			halbtc8822b2ant_set_fw_pstdma(
+				btcoexist, 0x51, 0x21, 0x3, 0x10,
+				0x50 | ps_tdma_byte4_modify);
+			break;
+		case 105:
+			halbtc8822b2ant_set_fw_pstdma(
+				btcoexist, 0x51, 0x30, 0x3, 0x10,
+				0x50 | ps_tdma_byte4_modify);
+			break;
+		case 106:
+			halbtc8822b2ant_set_fw_pstdma(
+				btcoexist, 0x51, 0x10, 0x3, 0x10,
+				0x50 | ps_tdma_byte4_modify);
+			break;
+		case 107:
+			halbtc8822b2ant_set_fw_pstdma(
+				btcoexist, 0x51, 0x10, 0x7, 0x10,
+				0x54 | ps_tdma_byte4_modify);
+			break;
+		case 108:
+			halbtc8822b2ant_set_fw_pstdma(
+				btcoexist, 0x51, 0x30, 0x3, 0x10,
+				0x50 | ps_tdma_byte4_modify);
+			break;
+		case 109:
+			halbtc8822b2ant_set_fw_pstdma(
+				btcoexist, 0x51, 0x10, 0x03, 0x10,
+				0x54 | ps_tdma_byte4_modify);
+			break;
+		case 110:
+			halbtc8822b2ant_set_fw_pstdma(
+				btcoexist, 0x55, 0x30, 0x03, 0x10,
+				0x50 | ps_tdma_byte4_modify);
+			break;
+		case 111:
+			halbtc8822b2ant_set_fw_pstdma(
+				btcoexist, 0x61, 0x25, 0x03, 0x11,
+				0x11 | ps_tdma_byte4_modify);
+			break;
+		case 151:
+			halbtc8822b2ant_set_fw_pstdma(
+				btcoexist, 0x51, 0x10, 0x03, 0x10,
+				0x50 | ps_tdma_byte4_modify);
+			break;
+		}
+	} else {
+		/* disable PS tdma */
+		switch (type) {
+		case 0:
+			halbtc8822b2ant_set_fw_pstdma(btcoexist, 0x0, 0x0, 0x0,
+						      0x40, 0x0);
+			break;
+		case 1:
+			halbtc8822b2ant_set_fw_pstdma(btcoexist, 0x0, 0x0, 0x0,
+						      0x48, 0x0);
+			break;
+		default:
+			halbtc8822b2ant_set_fw_pstdma(btcoexist, 0x0, 0x0, 0x0,
+						      0x40, 0x0);
+			break;
+		}
+	}
+
+	if (!coex_sta->is_set_ps_state_fail) {
+		/* update pre state */
+		coex_dm->pre_ps_tdma_on = coex_dm->cur_ps_tdma_on;
+		coex_dm->pre_ps_tdma = coex_dm->cur_ps_tdma;
+	}
+}
+
+/*anttenna control by bb mac bt antdiv pta to write 0x4c 0xcb4,0xcbd*/
+static void halbtc8822b2ant_set_ext_ant_switch(struct btc_coexist *btcoexist,
+					       bool force_exec, u8 ctrl_type,
+					       u8 pos_type)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	bool switch_polatiry_inverse = false;
+	u8 regval_0xcbc = 0, regval_0x64;
+	u32 u32tmp1 = 0, u32tmp2 = 0, u32tmp3 = 0;
+
+	if (!rfe_type->ext_ant_switch_exist)
+		return;
+
+	coex_dm->cur_ext_ant_switch_status = (ctrl_type << 8) + pos_type;
+
+	if (!force_exec) {
+		if (coex_dm->pre_ext_ant_switch_status ==
+		    coex_dm->cur_ext_ant_switch_status)
+			return;
+	}
+	coex_dm->pre_ext_ant_switch_status = coex_dm->cur_ext_ant_switch_status;
+
+	/* Ext switch buffer mux */
+	btcoexist->btc_write_1byte(btcoexist, 0x974, 0xff);
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0x1991, 0x3, 0x0);
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcbe, 0x8, 0x0);
+
+	switch (ctrl_type) {
+	default:
+	case BT_8822B_2ANT_EXT_ANT_SWITCH_CTRL_BY_BBSW:
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4e, 0x80,
+						   0x0); /*  0x4c[23] = 0 */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4f, 0x01,
+						   0x1); /* 0x4c[24] = 1 */
+		/* BB SW, DPDT use RFE_ctrl8 and RFE_ctrl9 as conctrol pin */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcb4, 0xff,
+						   0x77);
+
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcbd, 0x03, 01);
+
+		break;
+	case BT_8822B_2ANT_EXT_ANT_SWITCH_CTRL_BY_PTA:
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4e, 0x80,
+						   0x0); /* 0x4c[23] = 0 */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4f, 0x01,
+						   0x1); /* 0x4c[24] = 1 */
+		/* PTA,  DPDT use RFE_ctrl8 and RFE_ctrl9 as conctrol pin */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcb4, 0xff,
+						   0x66);
+
+		/* 0xcb4[29:28] = 2b'10 for no switch_polatiry_inverse,
+		 * DPDT_SEL_N =1, DPDT_SEL_P =0  @ GNT_BT=1
+		 */
+		regval_0xcbc = (!switch_polatiry_inverse ? 0x2 : 0x1);
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcbc, 0x03,
+						   regval_0xcbc);
+
+		break;
+	case BT_8822B_2ANT_EXT_ANT_SWITCH_CTRL_BY_ANTDIV:
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4e, 0x80,
+						   0x0); /* 0x4c[23] = 0 */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4f, 0x01,
+						   0x1); /* 0x4c[24] = 1 */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcb4, 0xff,
+						   0x88);
+		break;
+	case BT_8822B_2ANT_EXT_ANT_SWITCH_CTRL_BY_MAC:
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4e, 0x80,
+						   0x1); /*  0x4c[23] = 1 */
+
+		/* 0x64[0] = 1b'0 for no switch_polatiry_inverse,
+		 * DPDT_SEL_N =1, DPDT_SEL_P =0
+		 */
+		regval_0x64 = (!switch_polatiry_inverse ? 0x0 : 0x1);
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x64, 0x1,
+						   regval_0x64);
+		break;
+	case BT_8822B_2ANT_EXT_ANT_SWITCH_CTRL_BY_BT:
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4e, 0x80,
+						   0x0); /* 0x4c[23] = 0 */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4f, 0x01,
+						   0x0); /* 0x4c[24] = 0 */
+
+		/* no setup required, because  antenna switch control value by
+		 * BT vendor 0x1c[1:0]
+		 */
+		break;
+	}
+
+	/* PAPE, LNA_ON control by BT while WLAN off for current leakage issue*/
+	if (ctrl_type == BT_8822B_2ANT_EXT_ANT_SWITCH_CTRL_BY_BT) {
+		btcoexist->btc_write_1byte_bitmask(
+			btcoexist, 0x67, 0x20, 0x0); /* PAPE   0x64[29] = 0 */
+		btcoexist->btc_write_1byte_bitmask(
+			btcoexist, 0x67, 0x10, 0x0); /* LNA_ON 0x64[28] = 0 */
+	} else {
+		btcoexist->btc_write_1byte_bitmask(
+			btcoexist, 0x67, 0x20, 0x1); /* PAPE   0x64[29] = 1 */
+		btcoexist->btc_write_1byte_bitmask(
+			btcoexist, 0x67, 0x10, 0x1); /* LNA_ON 0x64[28] = 1 */
+	}
+
+	if (btcoexist->dbg_mode_2ant) {
+		u32tmp1 = btcoexist->btc_read_4byte(btcoexist, 0xcb4);
+		u32tmp2 = btcoexist->btc_read_4byte(btcoexist, 0x4c);
+		u32tmp3 = btcoexist->btc_read_4byte(btcoexist, 0x64) & 0xff;
+
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], (After Ext Ant switch setup) 0xcb4 = 0x%08x, 0x4c = 0x%08x, 0x64= 0x%02x\n",
+			u32tmp1, u32tmp2, u32tmp3);
+	}
+}
+
+/* rf4 type by efuse, and for ant at main aux inverse use,
+ * because is 2x2, and control types are the same, does not need
+ */
+static void halbtc8822b2ant_set_rfe_type(struct btc_coexist *btcoexist)
+{
+	struct btc_board_info *board_info = &btcoexist->board_info;
+
+	rfe_type->ext_band_switch_exist = false;
+	rfe_type->ext_band_switch_type =
+		BT_8822B_2ANT_EXT_BAND_SWITCH_USE_SPDT; /* SPDT; */
+	rfe_type->ext_band_switch_ctrl_polarity = 0;
+	/* Ext switch buffer mux */
+	btcoexist->btc_write_1byte(btcoexist, 0x974, 0xff);
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0x1991, 0x3, 0x0);
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcbe, 0x8, 0x0);
+
+	if (rfe_type->ext_band_switch_exist) {
+		/* band switch use RFE_ctrl1 (pin name: PAPE_A) and
+		 * RFE_ctrl3 (pin name: LNAON_A)
+		 */
+
+		/* set RFE_ctrl1 as software control */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcb0, 0xf0, 0x7);
+
+		/* set RFE_ctrl3 as software control */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcb1, 0xf0, 0x7);
+	}
+
+	/* the following setup should be got from Efuse in the future */
+	rfe_type->rfe_module_type = board_info->rfe_type;
+
+	rfe_type->ext_ant_switch_ctrl_polarity = 0;
+
+	switch (rfe_type->rfe_module_type) {
+	case 0:
+	default:
+		rfe_type->ext_ant_switch_exist = true;
+		rfe_type->ext_ant_switch_type =
+			BT_8822B_2ANT_EXT_ANT_SWITCH_USE_SPDT;
+		break;
+	case 1:
+		rfe_type->ext_ant_switch_exist = true;
+		rfe_type->ext_ant_switch_type =
+			BT_8822B_2ANT_EXT_ANT_SWITCH_USE_SPDT;
+		break;
+	case 2:
+		rfe_type->ext_ant_switch_exist = true;
+		rfe_type->ext_ant_switch_type =
+			BT_8822B_2ANT_EXT_ANT_SWITCH_USE_SPDT;
+		break;
+	case 3:
+		rfe_type->ext_ant_switch_exist = true;
+		rfe_type->ext_ant_switch_type =
+			BT_8822B_2ANT_EXT_ANT_SWITCH_USE_SPDT;
+		break;
+	case 4:
+		rfe_type->ext_ant_switch_exist = true;
+		rfe_type->ext_ant_switch_type =
+			BT_8822B_2ANT_EXT_ANT_SWITCH_USE_SPDT;
+		break;
+	case 5:
+		rfe_type->ext_ant_switch_exist = true;
+		rfe_type->ext_ant_switch_type =
+			BT_8822B_2ANT_EXT_ANT_SWITCH_USE_SPDT;
+		break;
+	case 6:
+		rfe_type->ext_ant_switch_exist = true;
+		rfe_type->ext_ant_switch_type =
+			BT_8822B_2ANT_EXT_ANT_SWITCH_USE_SPDT;
+		break;
+	case 7:
+		rfe_type->ext_ant_switch_exist = true;
+		rfe_type->ext_ant_switch_type =
+			BT_8822B_2ANT_EXT_ANT_SWITCH_USE_SPDT;
+		break;
+	}
+}
+
+/* set gnt_wl gnt_bt control by sw high low, or hwpta while in
+ * power on, ini, wlan off, wlan only, wl2g non-currrent, wl2g current, wl5g
+ */
+static void halbtc8822b2ant_set_ant_path(struct btc_coexist *btcoexist,
+					 u8 ant_pos_type, bool force_exec,
+					 u8 phase)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	u8 u8tmp = 0;
+	u32 u32tmp1 = 0;
+	u32 u32tmp2 = 0, u32tmp3 = 0;
+
+	u32tmp1 = halbtc8822b2ant_ltecoex_indirect_read_reg(btcoexist, 0x38);
+
+	/* To avoid indirect access fail  */
+	if (((u32tmp1 & 0xf000) >> 12) != ((u32tmp1 & 0x0f00) >> 8)) {
+		force_exec = true;
+		coex_sta->gnt_error_cnt++;
+	}
+
+	/* Ext switch buffer mux */
+	btcoexist->btc_write_1byte(btcoexist, 0x974, 0xff);
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0x1991, 0x3, 0x0);
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcbe, 0x8, 0x0);
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4e, 0x80,
+					   0x0); /*  0x4c[23] = 0 */
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4f, 0x01,
+					   0x1); /* 0x4c[24] = 1 */
+
+	coex_dm->cur_ant_pos_type = (ant_pos_type << 8) + phase;
+
+	if (!force_exec) {
+		if (coex_dm->cur_ant_pos_type == coex_dm->pre_ant_pos_type)
+			return;
+	}
+
+	coex_dm->pre_ant_pos_type = coex_dm->cur_ant_pos_type;
+
+	if (btcoexist->dbg_mode_2ant) {
+		u32tmp1 = halbtc8822b2ant_ltecoex_indirect_read_reg(btcoexist,
+								    0x38);
+		u32tmp2 = halbtc8822b2ant_ltecoex_indirect_read_reg(btcoexist,
+								    0x54);
+		u8tmp = btcoexist->btc_read_1byte(btcoexist, 0x73);
+
+		u32tmp3 = btcoexist->btc_read_4byte(btcoexist, 0xcb4);
+
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], (Before Ant Setup) 0xcb4 = 0x%x, 0x73 = 0x%x, 0x38= 0x%x, 0x54= 0x%x\n",
+			u32tmp3, u8tmp, u32tmp1, u32tmp2);
+	}
+
+	switch (phase) {
+	case BT_8822B_2ANT_PHASE_COEX_POWERON:
+
+		/* set Path control owner to WL at initial step */
+		halbtc8822b2ant_ltecoex_pathcontrol_owner(
+			btcoexist, BT_8822B_2ANT_PCO_BTSIDE);
+
+		/* set GNT_BT to SW high */
+		halbtc8822b2ant_ltecoex_set_gnt_bt(
+			btcoexist, BT_8822B_2ANT_GNT_BLOCK_RFC_BB,
+			BT_8822B_2ANT_GNT_TYPE_CTRL_BY_SW,
+			BT_8822B_2ANT_SIG_STA_SET_TO_HIGH);
+		/* Set GNT_WL to SW high */
+		halbtc8822b2ant_ltecoex_set_gnt_wl(
+			btcoexist, BT_8822B_2ANT_GNT_BLOCK_RFC_BB,
+			BT_8822B_2ANT_GNT_TYPE_CTRL_BY_SW,
+			BT_8822B_2ANT_SIG_STA_SET_TO_HIGH);
+
+		coex_sta->run_time_state = false;
+
+		break;
+	case BT_8822B_2ANT_PHASE_COEX_INIT:
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4e, 0x80,
+						   0x0); /*  0x4c[23] = 0 */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4f, 0x01,
+						   0x1); /* 0x4c[24] = 1 */
+		/* Disable LTE Coex Function in WiFi side
+		 * (this should be on if LTE coex is required)
+		 */
+		halbtc8822b2ant_ltecoex_enable(btcoexist, 0x0);
+
+		/* GNT_WL_LTE always = 1
+		 * (this should be config if LTE coex is required)
+		 */
+		halbtc8822b2ant_ltecoex_set_coex_table(
+			btcoexist, BT_8822B_2ANT_CTT_WL_VS_LTE, 0xffff);
+
+		/* GNT_BT_LTE always = 1
+		 * (this should be config if LTE coex is required)
+		 */
+		halbtc8822b2ant_ltecoex_set_coex_table(
+			btcoexist, BT_8822B_2ANT_CTT_BT_VS_LTE, 0xffff);
+
+		/* set Path control owner to WL at initial step */
+		halbtc8822b2ant_ltecoex_pathcontrol_owner(
+			btcoexist, BT_8822B_2ANT_PCO_WLSIDE);
+
+		/* set GNT_BT to SW high */
+		halbtc8822b2ant_ltecoex_set_gnt_bt(
+			btcoexist, BT_8822B_2ANT_GNT_BLOCK_RFC_BB,
+			BT_8822B_2ANT_GNT_TYPE_CTRL_BY_SW,
+			BT_8822B_2ANT_SIG_STA_SET_TO_HIGH);
+		/* Set GNT_WL to SW high */
+		halbtc8822b2ant_ltecoex_set_gnt_wl(
+			btcoexist, BT_8822B_2ANT_GNT_BLOCK_RFC_BB,
+			BT_8822B_2ANT_GNT_TYPE_CTRL_BY_SW,
+			BT_8822B_2ANT_SIG_STA_SET_TO_HIGH);
+
+		coex_sta->run_time_state = false;
+
+		break;
+	case BT_8822B_2ANT_PHASE_WLANONLY_INIT:
+		/* Disable LTE Coex Function in WiFi side
+		 * (this should be on if LTE coex is required)
+		 */
+		halbtc8822b2ant_ltecoex_enable(btcoexist, 0x0);
+
+		/* GNT_WL_LTE always = 1
+		 * (this should be config if LTE coex is required)
+		 */
+		halbtc8822b2ant_ltecoex_set_coex_table(
+			btcoexist, BT_8822B_2ANT_CTT_WL_VS_LTE, 0xffff);
+
+		/* GNT_BT_LTE always = 1
+		 * (this should be config if LTE coex is required)
+		 */
+		halbtc8822b2ant_ltecoex_set_coex_table(
+			btcoexist, BT_8822B_2ANT_CTT_BT_VS_LTE, 0xffff);
+
+		/* set Path control owner to WL at initial step */
+		halbtc8822b2ant_ltecoex_pathcontrol_owner(
+			btcoexist, BT_8822B_2ANT_PCO_WLSIDE);
+
+		/* set GNT_BT to SW Low */
+		halbtc8822b2ant_ltecoex_set_gnt_bt(
+			btcoexist, BT_8822B_2ANT_GNT_BLOCK_RFC_BB,
+			BT_8822B_2ANT_GNT_TYPE_CTRL_BY_SW,
+			BT_8822B_2ANT_SIG_STA_SET_TO_LOW);
+		/* Set GNT_WL to SW high */
+		halbtc8822b2ant_ltecoex_set_gnt_wl(
+			btcoexist, BT_8822B_2ANT_GNT_BLOCK_RFC_BB,
+			BT_8822B_2ANT_GNT_TYPE_CTRL_BY_SW,
+			BT_8822B_2ANT_SIG_STA_SET_TO_HIGH);
+
+		coex_sta->run_time_state = false;
+
+		break;
+	case BT_8822B_2ANT_PHASE_WLAN_OFF:
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4e, 0x80,
+						   0x0); /* 0x4c[23] = 0 */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4f, 0x01,
+						   0x0); /* 0x4c[24] = 0 */
+		/* Disable LTE Coex Function in WiFi side */
+		halbtc8822b2ant_ltecoex_enable(btcoexist, 0x0);
+
+		/* set Path control owner to BT */
+		halbtc8822b2ant_ltecoex_pathcontrol_owner(
+			btcoexist, BT_8822B_2ANT_PCO_BTSIDE);
+
+		/* Set Ext Ant Switch to BT control at wifi off step */
+		halbtc8822b2ant_set_ext_ant_switch(
+			btcoexist, FORCE_EXEC,
+			BT_8822B_2ANT_EXT_ANT_SWITCH_CTRL_BY_BT,
+			BT_8822B_2ANT_EXT_ANT_SWITCH_MAIN_TO_NOCARE);
+		coex_sta->run_time_state = false;
+		break;
+	case BT_8822B_2ANT_PHASE_2G_RUNTIME:
+	case BT_8822B_2ANT_PHASE_2G_RUNTIME_CONCURRENT:
+
+		/* set Path control owner to WL at runtime step */
+		halbtc8822b2ant_ltecoex_pathcontrol_owner(
+			btcoexist, BT_8822B_2ANT_PCO_WLSIDE);
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcb4, 0xff,
+						   0x66);
+		if (phase == BT_8822B_2ANT_PHASE_2G_RUNTIME_CONCURRENT) {
+			/* set GNT_BT to PTA */
+			halbtc8822b2ant_ltecoex_set_gnt_bt(
+				btcoexist, BT_8822B_2ANT_GNT_BLOCK_RFC_BB,
+				BT_8822B_2ANT_GNT_TYPE_CTRL_BY_PTA,
+				BT_8822B_2ANT_SIG_STA_SET_BY_HW);
+
+			/* Set GNT_WL to SW High */
+			halbtc8822b2ant_ltecoex_set_gnt_wl(
+				btcoexist, BT_8822B_2ANT_GNT_BLOCK_RFC_BB,
+				BT_8822B_2ANT_GNT_TYPE_CTRL_BY_SW,
+				BT_8822B_2ANT_SIG_STA_SET_TO_HIGH);
+		} else {
+			/* set GNT_BT to PTA */
+			halbtc8822b2ant_ltecoex_set_gnt_bt(
+				btcoexist, BT_8822B_2ANT_GNT_BLOCK_RFC_BB,
+				BT_8822B_2ANT_GNT_TYPE_CTRL_BY_PTA,
+				BT_8822B_2ANT_SIG_STA_SET_BY_HW);
+
+			/* Set GNT_WL to PTA */
+			halbtc8822b2ant_ltecoex_set_gnt_wl(
+				btcoexist, BT_8822B_2ANT_GNT_BLOCK_RFC_BB,
+				BT_8822B_2ANT_GNT_TYPE_CTRL_BY_PTA,
+				BT_8822B_2ANT_SIG_STA_SET_BY_HW);
+		}
+
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], ************* under2g 0xcbd setting =2 *************\n");
+
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcbd, 0x03, 02);
+		break;
+
+	case BT_8822B_2ANT_PHASE_5G_RUNTIME:
+
+		/* set Path control owner to WL at runtime step */
+		halbtc8822b2ant_ltecoex_pathcontrol_owner(
+			btcoexist, BT_8822B_2ANT_PCO_WLSIDE);
+
+		/* set GNT_BT to SW Hi */
+		halbtc8822b2ant_ltecoex_set_gnt_bt(
+			btcoexist, BT_8822B_2ANT_GNT_BLOCK_RFC_BB,
+			BT_8822B_2ANT_GNT_TYPE_CTRL_BY_SW,
+			BT_8822B_2ANT_SIG_STA_SET_TO_HIGH);
+		/* Set GNT_WL to SW Hi */
+		halbtc8822b2ant_ltecoex_set_gnt_wl(
+			btcoexist, BT_8822B_2ANT_GNT_BLOCK_RFC_BB,
+			BT_8822B_2ANT_GNT_TYPE_CTRL_BY_SW,
+			BT_8822B_2ANT_SIG_STA_SET_TO_HIGH);
+		coex_sta->run_time_state = true;
+
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], ************* under5g 0xcbd setting =1 *************\n");
+
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0xcbd, 0x03, 01);
+
+		break;
+	case BT_8822B_2ANT_PHASE_BTMPMODE:
+		/* Disable LTE Coex Function in WiFi side */
+		halbtc8822b2ant_ltecoex_enable(btcoexist, 0x0);
+
+		/* set Path control owner to WL */
+		halbtc8822b2ant_ltecoex_pathcontrol_owner(
+			btcoexist, BT_8822B_2ANT_PCO_WLSIDE);
+
+		/* set GNT_BT to SW Hi */
+		halbtc8822b2ant_ltecoex_set_gnt_bt(
+			btcoexist, BT_8822B_2ANT_GNT_BLOCK_RFC_BB,
+			BT_8822B_2ANT_GNT_TYPE_CTRL_BY_SW,
+			BT_8822B_2ANT_SIG_STA_SET_TO_HIGH);
+
+		/* Set GNT_WL to SW Lo */
+		halbtc8822b2ant_ltecoex_set_gnt_wl(
+			btcoexist, BT_8822B_2ANT_GNT_BLOCK_RFC_BB,
+			BT_8822B_2ANT_GNT_TYPE_CTRL_BY_SW,
+			BT_8822B_2ANT_SIG_STA_SET_TO_LOW);
+
+		coex_sta->run_time_state = false;
+		break;
+	}
+
+	if (btcoexist->dbg_mode_2ant) {
+		u32tmp1 = halbtc8822b2ant_ltecoex_indirect_read_reg(btcoexist,
+								    0x38);
+		u32tmp2 = halbtc8822b2ant_ltecoex_indirect_read_reg(btcoexist,
+								    0x54);
+		u32tmp3 = btcoexist->btc_read_4byte(btcoexist, 0xcb4);
+		u8tmp = btcoexist->btc_read_1byte(btcoexist, 0x73);
+
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], (After Ant-Setup phase---%d) 0xcb4 = 0x%x, 0x73 = 0x%x, 0x38= 0x%x, 0x54= 0x%x\n",
+			phase, u32tmp3, u8tmp, u32tmp1, u32tmp2);
+	}
+}
+
+static u8 halbtc8822b2ant_action_algorithm(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
+	bool bt_hs_on = false;
+	u8 algorithm = BT_8822B_2ANT_COEX_ALGO_UNDEFINED;
+	u8 num_of_diff_profile = 0;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_HS_OPERATION, &bt_hs_on);
+
+	if (!bt_link_info->bt_link_exist) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], No BT link exists!!!\n");
+		return algorithm;
+	}
+
+	if (bt_link_info->sco_exist)
+		num_of_diff_profile++;
+	if (bt_link_info->hid_exist)
+		num_of_diff_profile++;
+	if (bt_link_info->pan_exist)
+		num_of_diff_profile++;
+	if (bt_link_info->a2dp_exist)
+		num_of_diff_profile++;
+
+	if (num_of_diff_profile == 0) {
+		if (bt_link_info->acl_busy) {
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], No-Profile busy\n");
+			algorithm = BT_8822B_2ANT_COEX_ALGO_NOPROFILEBUSY;
+		}
+	} else if ((bt_link_info->a2dp_exist) && (coex_sta->is_bt_a2dp_sink)) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], A2DP Sink\n");
+		algorithm = BT_8822B_2ANT_COEX_ALGO_A2DPSINK;
+	} else if (num_of_diff_profile == 1) {
+		if (bt_link_info->sco_exist) {
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], SCO only\n");
+			algorithm = BT_8822B_2ANT_COEX_ALGO_SCO;
+		} else {
+			if (bt_link_info->hid_exist) {
+				RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+					 "[BTCoex], HID only\n");
+				algorithm = BT_8822B_2ANT_COEX_ALGO_HID;
+			} else if (bt_link_info->a2dp_exist) {
+				RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+					 "[BTCoex], A2DP only\n");
+				algorithm = BT_8822B_2ANT_COEX_ALGO_A2DP;
+			} else if (bt_link_info->pan_exist) {
+				if (bt_hs_on) {
+					RT_TRACE(rtlpriv, COMP_BT_COEXIST,
+						 DBG_LOUD,
+						 "[BTCoex], PAN(HS) only\n");
+					algorithm =
+						BT_8822B_2ANT_COEX_ALGO_PANHS;
+				} else {
+					RT_TRACE(rtlpriv, COMP_BT_COEXIST,
+						 DBG_LOUD,
+						 "[BTCoex], PAN(EDR) only\n");
+					algorithm =
+						BT_8822B_2ANT_COEX_ALGO_PANEDR;
+				}
+			}
+		}
+	} else if (num_of_diff_profile == 2) {
+		if (bt_link_info->sco_exist) {
+			if (bt_link_info->hid_exist) {
+				RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+					 "[BTCoex], SCO + HID\n");
+				algorithm = BT_8822B_2ANT_COEX_ALGO_SCO;
+			} else if (bt_link_info->a2dp_exist) {
+				RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+					 "[BTCoex], SCO + A2DP ==> A2DP\n");
+				algorithm = BT_8822B_2ANT_COEX_ALGO_A2DP;
+			} else if (bt_link_info->pan_exist) {
+				if (bt_hs_on) {
+					RT_TRACE(rtlpriv, COMP_BT_COEXIST,
+						 DBG_LOUD,
+						 "[BTCoex], SCO + PAN(HS)\n");
+					algorithm = BT_8822B_2ANT_COEX_ALGO_SCO;
+				} else {
+					RT_TRACE(rtlpriv, COMP_BT_COEXIST,
+						 DBG_LOUD,
+						 "[BTCoex], SCO + PAN(EDR)\n");
+					algorithm =
+						BT_8822B_2ANT_COEX_ALGO_PANEDR;
+				}
+			}
+		} else {
+			if (bt_link_info->hid_exist &&
+			    bt_link_info->a2dp_exist) {
+				{
+					RT_TRACE(rtlpriv, COMP_BT_COEXIST,
+						 DBG_LOUD,
+						 "[BTCoex], HID + A2DP\n");
+					algorithm =
+					    BT_8822B_2ANT_COEX_ALGO_HID_A2DP;
+				}
+			} else if (bt_link_info->hid_exist &&
+				   bt_link_info->pan_exist) {
+				if (bt_hs_on) {
+					RT_TRACE(rtlpriv, COMP_BT_COEXIST,
+						 DBG_LOUD,
+						 "[BTCoex], HID + PAN(HS)\n");
+					algorithm = BT_8822B_2ANT_COEX_ALGO_HID;
+				} else {
+					RT_TRACE(rtlpriv, COMP_BT_COEXIST,
+						 DBG_LOUD,
+						 "[BTCoex], HID + PAN(EDR)\n");
+					algorithm =
+					    BT_8822B_2ANT_COEX_ALGO_PANEDR_HID;
+				}
+			} else if (bt_link_info->pan_exist &&
+				   bt_link_info->a2dp_exist) {
+				if (bt_hs_on) {
+					RT_TRACE(rtlpriv, COMP_BT_COEXIST,
+						 DBG_LOUD,
+						 "[BTCoex], A2DP + PAN(HS)\n");
+					algorithm =
+					    BT_8822B_2ANT_COEX_ALGO_A2DP_PANHS;
+				} else {
+					RT_TRACE(rtlpriv, COMP_BT_COEXIST,
+						 DBG_LOUD,
+						 "[BTCoex], A2DP + PAN(EDR)\n");
+					algorithm =
+					    BT_8822B_2ANT_COEX_ALGO_PANEDR_A2DP;
+				}
+			}
+		}
+	} else if (num_of_diff_profile == 3) {
+		if (bt_link_info->sco_exist) {
+			if (bt_link_info->hid_exist &&
+			    bt_link_info->a2dp_exist) {
+				RT_TRACE(
+					rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+					"[BTCoex], SCO + HID + A2DP ==> HID + A2DP\n");
+				algorithm = BT_8822B_2ANT_COEX_ALGO_HID_A2DP;
+			} else if (bt_link_info->hid_exist &&
+				   bt_link_info->pan_exist) {
+				if (bt_hs_on) {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], SCO + HID + PAN(HS)\n");
+					algorithm =
+					    BT_8822B_2ANT_COEX_ALGO_PANEDR_HID;
+				} else {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], SCO + HID + PAN(EDR)\n");
+					algorithm =
+					    BT_8822B_2ANT_COEX_ALGO_PANEDR_HID;
+				}
+			} else if (bt_link_info->pan_exist &&
+				   bt_link_info->a2dp_exist) {
+				if (bt_hs_on) {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], SCO + A2DP + PAN(HS)\n");
+					algorithm =
+					    BT_8822B_2ANT_COEX_ALGO_A2DP_PANHS;
+				} else {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], SCO + A2DP + PAN(EDR) ==> HID\n");
+					algorithm =
+					    BT_8822B_2ANT_COEX_ALGO_PANEDR_A2DP;
+				}
+			}
+		} else {
+			if (bt_link_info->hid_exist &&
+			    bt_link_info->pan_exist &&
+			    bt_link_info->a2dp_exist) {
+				if (bt_hs_on) {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], HID + A2DP + PAN(HS)\n");
+					algorithm =
+					BT_8822B_2ANT_COEX_ALGO_HID_A2DP_PANEDR;
+				} else {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], HID + A2DP + PAN(EDR)\n");
+					algorithm =
+					BT_8822B_2ANT_COEX_ALGO_HID_A2DP_PANEDR;
+				}
+			}
+		}
+	} else if (num_of_diff_profile >= 3) {
+		if (bt_link_info->sco_exist) {
+			if (bt_link_info->hid_exist &&
+			    bt_link_info->pan_exist &&
+			    bt_link_info->a2dp_exist) {
+				if (bt_hs_on) {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], Error!!! SCO + HID + A2DP + PAN(HS)\n");
+					algorithm =
+					BT_8822B_2ANT_COEX_ALGO_HID_A2DP_PANEDR;
+				} else {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], SCO + HID + A2DP + PAN(EDR)==>PAN(EDR)+HID\n");
+					algorithm =
+					BT_8822B_2ANT_COEX_ALGO_HID_A2DP_PANEDR;
+				}
+			}
+		}
+	}
+
+	return algorithm;
+}
+
+static void halbtc8822b2ant_action_coex_all_off(struct btc_coexist *btcoexist)
+{
+	halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 5);
+
+	/* fw all off */
+	halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 0);
+
+	halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+}
+
+static void halbtc8822b2ant_action_wifi_under5g(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	/* fw all off */
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], ************* under5g *************\n");
+	halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 0);
+	halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 0);
+
+	halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+
+	halbtc8822b2ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO, FORCE_EXEC,
+				     BT_8822B_2ANT_PHASE_5G_RUNTIME);
+}
+
+static void
+halbtc8822b2ant_action_wifi_native_lps(struct btc_coexist *btcoexist)
+{
+	halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 5);
+
+	halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 0);
+}
+
+static void halbtc8822b2ant_action_bt_inquiry(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	bool wifi_connected = false;
+	bool wifi_scan = false, wifi_link = false, wifi_roam = false;
+	bool wifi_busy = false;
+	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_CONNECTED,
+			   &wifi_connected);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_SCAN, &wifi_scan);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_LINK, &wifi_link);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_ROAM, &wifi_roam);
+
+	if ((coex_sta->bt_create_connection) &&
+	    ((wifi_link) || (wifi_roam) || (wifi_scan) || (wifi_busy) ||
+	     (coex_sta->wifi_is_high_pri_task))) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], Wifi link/roam/Scan/busy/hi-pri-task + BT Inq/Page!!\n");
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 8);
+
+		if ((bt_link_info->a2dp_exist) && (!bt_link_info->pan_exist))
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						15);
+		else
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						11);
+	} else if ((!wifi_connected) && (!wifi_scan)) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], Wifi no-link + no-scan + BT Inq/Page!!\n");
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 5);
+
+		halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 0);
+	} else if (bt_link_info->pan_exist) {
+		halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 22);
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 8);
+
+	} else if (bt_link_info->a2dp_exist) {
+		halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 8);
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC,
+						     10);
+	} else {
+		if ((wifi_link) || (wifi_roam) || (wifi_scan) || (wifi_busy) ||
+		    (coex_sta->wifi_is_high_pri_task))
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						21);
+		else
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						23);
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 8);
+	}
+
+	halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, FORCE_EXEC, 0xd8);
+}
+
+static void
+halbtc8822b2ant_action_wifi_link_process(struct btc_coexist *btcoexist)
+{
+	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
+
+	halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, FORCE_EXEC, 0xd4);
+
+	halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 8);
+
+	if (bt_link_info->pan_exist) {
+		halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 22);
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 8);
+
+	} else if (bt_link_info->a2dp_exist) {
+		halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 16);
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 8);
+	} else {
+		halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 21);
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 8);
+	}
+}
+
+static void
+halbtc8822b2ant_action_wifi_nonconnected(struct btc_coexist *btcoexist)
+{
+	halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 0);
+
+	/* fw all off */
+	halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 0);
+
+	halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+}
+
+static void halbtc8822b2ant_action_bt_relink(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], run bt multi link function\n");
+
+	if (coex_sta->is_bt_multi_link)
+		return;
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], run bt re-link function\n");
+
+	halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 0);
+	halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 5);
+}
+
+static void halbtc8822b2ant_action_bt_idle(struct btc_coexist *btcoexist)
+{
+	bool wifi_busy = false;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+
+	if (!wifi_busy) {
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 8);
+
+		halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 14);
+	} else { /* if wl busy */
+
+		if (BT_8822B_2ANT_BT_STATUS_NON_CONNECTED_IDLE ==
+		    coex_dm->bt_status) {
+			halbtc8822b2ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 0);
+
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false,
+						0);
+		} else {
+			halbtc8822b2ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 8);
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						12);
+		}
+	}
+
+	halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, FORCE_EXEC, 0xd8);
+}
+
+/* SCO only or SCO+PAN(HS) */
+static void halbtc8822b2ant_action_sco(struct btc_coexist *btcoexist)
+{
+	static u8 prewifi_rssi_state = BTC_RSSI_STATE_LOW;
+	static u8 pre_bt_rssi_state = BTC_RSSI_STATE_LOW;
+	u8 wifi_rssi_state, bt_rssi_state;
+
+	static u8 prewifi_rssi_state2 = BTC_RSSI_STATE_LOW;
+	static u8 pre_bt_rssi_state2 = BTC_RSSI_STATE_LOW;
+	u8 wifi_rssi_state2, bt_rssi_state2;
+	bool wifi_busy = false;
+	u32 wifi_bw = 1;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_BW, &wifi_bw);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+
+	wifi_rssi_state = halbtc8822b2ant_wifi_rssi_state(
+		btcoexist, &prewifi_rssi_state, 2, coex_sta->wifi_coex_thres,
+		0);
+
+	wifi_rssi_state2 = halbtc8822b2ant_wifi_rssi_state(
+		btcoexist, &prewifi_rssi_state2, 2, coex_sta->wifi_coex_thres2,
+		0);
+
+	bt_rssi_state = halbtc8822b2ant_bt_rssi_state(
+		btcoexist, &pre_bt_rssi_state, 2, coex_sta->bt_coex_thres, 0);
+
+	bt_rssi_state2 = halbtc8822b2ant_bt_rssi_state(
+		btcoexist, &pre_bt_rssi_state2, 2, coex_sta->bt_coex_thres2, 0);
+
+	if (BTC_RSSI_HIGH(wifi_rssi_state) && BTC_RSSI_HIGH(bt_rssi_state)) {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+		halbtc8822b2ant_dec_bt_pwr(btcoexist, NORMAL_EXEC, 0);
+
+		coex_dm->is_switch_to_1dot5_ant = false;
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 5);
+
+		halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 0);
+	} else {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+		halbtc8822b2ant_dec_bt_pwr(btcoexist, NORMAL_EXEC, 0);
+
+		coex_dm->is_switch_to_1dot5_ant = false;
+
+		if (coex_sta->is_esco_mode)
+			halbtc8822b2ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 1);
+		else /* 2-Ant free run if SCO mode */
+			halbtc8822b2ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 0);
+
+		halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true, 8);
+	}
+}
+
+static void halbtc8822b2ant_action_hid(struct btc_coexist *btcoexist)
+{
+	static u8 prewifi_rssi_state = BTC_RSSI_STATE_LOW;
+	static u8 pre_bt_rssi_state = BTC_RSSI_STATE_LOW;
+	u8 wifi_rssi_state, bt_rssi_state;
+
+	static u8 prewifi_rssi_state2 = BTC_RSSI_STATE_LOW;
+	static u8 pre_bt_rssi_state2 = BTC_RSSI_STATE_LOW;
+	u8 wifi_rssi_state2, bt_rssi_state2;
+	bool wifi_busy = false;
+	u32 wifi_bw = 1;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_BW, &wifi_bw);
+
+	wifi_rssi_state = halbtc8822b2ant_wifi_rssi_state(
+		btcoexist, &prewifi_rssi_state, 2, coex_sta->wifi_coex_thres,
+		0);
+
+	wifi_rssi_state2 = halbtc8822b2ant_wifi_rssi_state(
+		btcoexist, &prewifi_rssi_state2, 2, coex_sta->wifi_coex_thres2,
+		0);
+
+	bt_rssi_state = halbtc8822b2ant_bt_rssi_state(
+		btcoexist, &pre_bt_rssi_state, 2, coex_sta->bt_coex_thres, 0);
+
+	bt_rssi_state2 = halbtc8822b2ant_bt_rssi_state(
+		btcoexist, &pre_bt_rssi_state2, 2, coex_sta->bt_coex_thres2, 0);
+
+	if (BTC_RSSI_HIGH(wifi_rssi_state) && BTC_RSSI_HIGH(bt_rssi_state)) {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+		halbtc8822b2ant_dec_bt_pwr(btcoexist, NORMAL_EXEC, 0);
+
+		coex_dm->is_switch_to_1dot5_ant = false;
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 0);
+
+		halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 0);
+	} else {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+		halbtc8822b2ant_dec_bt_pwr(btcoexist, NORMAL_EXEC, 0);
+
+		coex_dm->is_switch_to_1dot5_ant = false;
+
+		if (coex_sta->is_hid_low_pri_tx_overhead) {
+			halbtc8822b2ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 4);
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						108);
+		} else if (wifi_bw == 0) { /* if 11bg mode */
+
+			halbtc8822b2ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 8);
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						111);
+		} else {
+			halbtc8822b2ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 8);
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						111);
+		}
+	}
+}
+
+static void halbtc8822b2ant_action_a2dpsink(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	static u8 prewifi_rssi_state = BTC_RSSI_STATE_LOW;
+	static u8 pre_bt_rssi_state = BTC_RSSI_STATE_LOW;
+	u8 wifi_rssi_state, bt_rssi_state;
+
+	static u8 prewifi_rssi_state2 = BTC_RSSI_STATE_LOW;
+	static u8 pre_bt_rssi_state2 = BTC_RSSI_STATE_LOW;
+	u8 wifi_rssi_state2, bt_rssi_state2;
+	bool wifi_busy = false, wifi_turbo = false;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+	btcoexist->btc_get(btcoexist, BTC_GET_U1_AP_NUM,
+			   &coex_sta->scan_ap_num);
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "############# [BTCoex],  scan_ap_num = %d, wl_noisy = %d\n",
+		 coex_sta->scan_ap_num, coex_sta->wl_noisy_level);
+
+	if ((wifi_busy) && (coex_sta->wl_noisy_level == 0))
+		wifi_turbo = true;
+
+	wifi_rssi_state = halbtc8822b2ant_wifi_rssi_state(
+		btcoexist, &prewifi_rssi_state, 2, coex_sta->wifi_coex_thres,
+		0);
+
+	wifi_rssi_state2 = halbtc8822b2ant_wifi_rssi_state(
+		btcoexist, &prewifi_rssi_state2, 2, coex_sta->wifi_coex_thres2,
+		0);
+
+	bt_rssi_state = halbtc8822b2ant_bt_rssi_state(
+		btcoexist, &pre_bt_rssi_state, 2, coex_sta->bt_coex_thres, 0);
+
+	bt_rssi_state2 = halbtc8822b2ant_bt_rssi_state(
+		btcoexist, &pre_bt_rssi_state2, 2, coex_sta->bt_coex_thres2, 0);
+
+	if (BTC_RSSI_HIGH(wifi_rssi_state) && BTC_RSSI_HIGH(bt_rssi_state)) {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+		halbtc8822b2ant_dec_bt_pwr(btcoexist, NORMAL_EXEC, 0);
+
+		coex_dm->is_switch_to_1dot5_ant = false;
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 0);
+
+		halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 0);
+	} else if (BTC_RSSI_HIGH(wifi_rssi_state2) &&
+		   BTC_RSSI_HIGH(bt_rssi_state2)) {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xc8);
+		halbtc8822b2ant_dec_bt_pwr(btcoexist, NORMAL_EXEC, 2);
+
+		coex_dm->is_switch_to_1dot5_ant = false;
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 4);
+
+		if (wifi_busy)
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						1);
+		else
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						16);
+	} else {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+		halbtc8822b2ant_dec_bt_pwr(btcoexist, NORMAL_EXEC, 0);
+
+		coex_dm->is_switch_to_1dot5_ant = true;
+
+		if ((coex_sta->bt_relink_downcount != 0) && (wifi_busy)) {
+			RT_TRACE(
+				rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				"############# [BTCoex],  BT Re-Link + A2DP + WL busy\n");
+
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false,
+						0);
+			halbtc8822b2ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 5);
+
+		} else {
+			halbtc8822b2ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 8);
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						105);
+		}
+	}
+}
+
+/* A2DP only / PAN(EDR) only/ A2DP+PAN(HS) */
+static void halbtc8822b2ant_action_a2dp(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	static u8 prewifi_rssi_state = BTC_RSSI_STATE_LOW;
+	static u8 pre_bt_rssi_state = BTC_RSSI_STATE_LOW;
+	u8 wifi_rssi_state, bt_rssi_state;
+
+	static u8 prewifi_rssi_state2 = BTC_RSSI_STATE_LOW;
+	static u8 pre_bt_rssi_state2 = BTC_RSSI_STATE_LOW;
+	u8 wifi_rssi_state2, bt_rssi_state2;
+	bool wifi_busy = false, wifi_turbo = false;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+	btcoexist->btc_get(btcoexist, BTC_GET_U1_AP_NUM,
+			   &coex_sta->scan_ap_num);
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "############# [BTCoex],  scan_ap_num = %d, wl_noisy = %d\n",
+		 coex_sta->scan_ap_num, coex_sta->wl_noisy_level);
+
+	if ((wifi_busy) && (coex_sta->wl_noisy_level == 0))
+		wifi_turbo = true;
+
+	wifi_rssi_state = halbtc8822b2ant_wifi_rssi_state(
+		btcoexist, &prewifi_rssi_state, 2, coex_sta->wifi_coex_thres,
+		0);
+
+	wifi_rssi_state2 = halbtc8822b2ant_wifi_rssi_state(
+		btcoexist, &prewifi_rssi_state2, 2, coex_sta->wifi_coex_thres2,
+		0);
+
+	bt_rssi_state = halbtc8822b2ant_bt_rssi_state(
+		btcoexist, &pre_bt_rssi_state, 2, coex_sta->bt_coex_thres, 0);
+
+	bt_rssi_state2 = halbtc8822b2ant_bt_rssi_state(
+		btcoexist, &pre_bt_rssi_state2, 2, coex_sta->bt_coex_thres2, 0);
+
+	if (BTC_RSSI_HIGH(wifi_rssi_state) && BTC_RSSI_HIGH(bt_rssi_state)) {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+		halbtc8822b2ant_dec_bt_pwr(btcoexist, NORMAL_EXEC, 0);
+
+		coex_dm->is_switch_to_1dot5_ant = false;
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 0);
+
+		halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 0);
+	} else if (BTC_RSSI_HIGH(wifi_rssi_state2) &&
+		   BTC_RSSI_HIGH(bt_rssi_state2)) {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xc8);
+		halbtc8822b2ant_dec_bt_pwr(btcoexist, NORMAL_EXEC, 2);
+
+		coex_dm->is_switch_to_1dot5_ant = false;
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 4);
+
+		if (wifi_busy)
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						1);
+		else
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						16);
+	} else {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+		halbtc8822b2ant_dec_bt_pwr(btcoexist, NORMAL_EXEC, 0);
+
+		coex_dm->is_switch_to_1dot5_ant = true;
+
+		if ((coex_sta->bt_relink_downcount != 0) && (wifi_busy)) {
+			RT_TRACE(
+				rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				"############# [BTCoex],  BT Re-Link + A2DP + WL busy\n");
+
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false,
+						0);
+			halbtc8822b2ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 5);
+
+		} else {
+			if (wifi_turbo)
+				halbtc8822b2ant_coex_table_with_type(
+					btcoexist, NORMAL_EXEC, 10);
+			else
+				halbtc8822b2ant_coex_table_with_type(
+					btcoexist, NORMAL_EXEC, 10);
+
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						109);
+		}
+	}
+}
+
+static void halbtc8822b2ant_action_pan_edr(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	static u8 prewifi_rssi_state = BTC_RSSI_STATE_LOW;
+	static u8 pre_bt_rssi_state = BTC_RSSI_STATE_LOW;
+	u8 wifi_rssi_state, bt_rssi_state;
+
+	static u8 prewifi_rssi_state2 = BTC_RSSI_STATE_LOW;
+	static u8 pre_bt_rssi_state2 = BTC_RSSI_STATE_LOW;
+	u8 wifi_rssi_state2, bt_rssi_state2;
+	bool wifi_busy = false, wifi_turbo = false;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+	btcoexist->btc_get(btcoexist, BTC_GET_U1_AP_NUM,
+			   &coex_sta->scan_ap_num);
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "############# [BTCoex],  scan_ap_num = %d, wl_noisy = %d\n",
+		 coex_sta->scan_ap_num, coex_sta->wl_noisy_level);
+
+	if ((wifi_busy) && (coex_sta->wl_noisy_level == 0))
+		wifi_turbo = true;
+
+	wifi_rssi_state = halbtc8822b2ant_wifi_rssi_state(
+		btcoexist, &prewifi_rssi_state, 2, coex_sta->wifi_coex_thres,
+		0);
+
+	wifi_rssi_state2 = halbtc8822b2ant_wifi_rssi_state(
+		btcoexist, &prewifi_rssi_state2, 2, coex_sta->wifi_coex_thres2,
+		0);
+
+	bt_rssi_state = halbtc8822b2ant_bt_rssi_state(
+		btcoexist, &pre_bt_rssi_state, 2, coex_sta->bt_coex_thres, 0);
+
+	bt_rssi_state2 = halbtc8822b2ant_bt_rssi_state(
+		btcoexist, &pre_bt_rssi_state2, 2, coex_sta->bt_coex_thres2, 0);
+
+	if (BTC_RSSI_HIGH(wifi_rssi_state) && BTC_RSSI_HIGH(bt_rssi_state)) {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+		halbtc8822b2ant_dec_bt_pwr(btcoexist, NORMAL_EXEC, 0);
+
+		coex_dm->is_switch_to_1dot5_ant = false;
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 0);
+
+		halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 0);
+	} else if (BTC_RSSI_HIGH(wifi_rssi_state2) &&
+		   BTC_RSSI_HIGH(bt_rssi_state2)) {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xc8);
+		halbtc8822b2ant_dec_bt_pwr(btcoexist, NORMAL_EXEC, 2);
+
+		coex_dm->is_switch_to_1dot5_ant = false;
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 4);
+
+		if (wifi_busy)
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						3);
+		else
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						4);
+	} else {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+		halbtc8822b2ant_dec_bt_pwr(btcoexist, NORMAL_EXEC, 0);
+
+		coex_dm->is_switch_to_1dot5_ant = true;
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 8);
+
+		if (wifi_busy)
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						103);
+		else
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						104);
+	}
+}
+
+static void halbtc8822b2ant_action_hid_a2dp(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	static u8 prewifi_rssi_state = BTC_RSSI_STATE_LOW;
+	static u8 pre_bt_rssi_state = BTC_RSSI_STATE_LOW;
+	u8 wifi_rssi_state, bt_rssi_state;
+
+	static u8 prewifi_rssi_state2 = BTC_RSSI_STATE_LOW;
+	static u8 pre_bt_rssi_state2 = BTC_RSSI_STATE_LOW;
+	u8 wifi_rssi_state2, bt_rssi_state2;
+	bool wifi_busy = false;
+	u32 wifi_bw = 1;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_BW, &wifi_bw);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+
+	wifi_rssi_state = halbtc8822b2ant_wifi_rssi_state(
+		btcoexist, &prewifi_rssi_state, 2, coex_sta->wifi_coex_thres,
+		0);
+
+	wifi_rssi_state2 = halbtc8822b2ant_wifi_rssi_state(
+		btcoexist, &prewifi_rssi_state2, 2, coex_sta->wifi_coex_thres2,
+		0);
+
+	bt_rssi_state = halbtc8822b2ant_bt_rssi_state(
+		btcoexist, &pre_bt_rssi_state, 2, coex_sta->bt_coex_thres, 0);
+
+	bt_rssi_state2 = halbtc8822b2ant_bt_rssi_state(
+		btcoexist, &pre_bt_rssi_state2, 2, coex_sta->bt_coex_thres2, 0);
+
+	if (BTC_RSSI_HIGH(wifi_rssi_state) && BTC_RSSI_HIGH(bt_rssi_state)) {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+		halbtc8822b2ant_dec_bt_pwr(btcoexist, NORMAL_EXEC, 0);
+
+		coex_dm->is_switch_to_1dot5_ant = false;
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 0);
+		halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 0);
+	} else if (BTC_RSSI_HIGH(wifi_rssi_state2) &&
+		   BTC_RSSI_HIGH(bt_rssi_state2)) {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xc8);
+		halbtc8822b2ant_dec_bt_pwr(btcoexist, NORMAL_EXEC, 2);
+
+		coex_dm->is_switch_to_1dot5_ant = false;
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 4);
+
+		if (wifi_busy)
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						1);
+		else
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						16);
+	} else {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+		halbtc8822b2ant_dec_bt_pwr(btcoexist, NORMAL_EXEC, 0);
+
+		coex_dm->is_switch_to_1dot5_ant = true;
+
+		if ((coex_sta->bt_relink_downcount != 0) && (wifi_busy)) {
+			RT_TRACE(
+				rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				"############# [BTCoex],  BT Re-Link + A2DP + WL busy\n");
+
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false,
+						0);
+			halbtc8822b2ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 5);
+		} else {
+			halbtc8822b2ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 8);
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						109);
+		}
+	}
+}
+
+static void halbtc8822b2ant_action_a2dp_pan_hs(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	static u8 prewifi_rssi_state = BTC_RSSI_STATE_LOW;
+	static u8 pre_bt_rssi_state = BTC_RSSI_STATE_LOW;
+	u8 wifi_rssi_state, bt_rssi_state;
+
+	static u8 prewifi_rssi_state2 = BTC_RSSI_STATE_LOW;
+	static u8 pre_bt_rssi_state2 = BTC_RSSI_STATE_LOW;
+	u8 wifi_rssi_state2, bt_rssi_state2;
+	bool wifi_busy = false, wifi_turbo = false;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+	btcoexist->btc_get(btcoexist, BTC_GET_U1_AP_NUM,
+			   &coex_sta->scan_ap_num);
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "############# [BTCoex],  scan_ap_num = %d, wl_noisy = %d\n",
+		 coex_sta->scan_ap_num, coex_sta->wl_noisy_level);
+
+	if ((wifi_busy) && (coex_sta->wl_noisy_level == 0))
+		wifi_turbo = true;
+
+	wifi_rssi_state = halbtc8822b2ant_wifi_rssi_state(
+		btcoexist, &prewifi_rssi_state, 2, coex_sta->wifi_coex_thres,
+		0);
+
+	wifi_rssi_state2 = halbtc8822b2ant_wifi_rssi_state(
+		btcoexist, &prewifi_rssi_state2, 2, coex_sta->wifi_coex_thres2,
+		0);
+
+	bt_rssi_state = halbtc8822b2ant_bt_rssi_state(
+		btcoexist, &pre_bt_rssi_state, 2, coex_sta->bt_coex_thres, 0);
+
+	bt_rssi_state2 = halbtc8822b2ant_bt_rssi_state(
+		btcoexist, &pre_bt_rssi_state2, 2, coex_sta->bt_coex_thres2, 0);
+
+	if (BTC_RSSI_HIGH(wifi_rssi_state) && BTC_RSSI_HIGH(bt_rssi_state)) {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+		/*halbtc8822b2ant_dec_bt_pwr(btcoexist, NORMAL_EXEC, 0);*/
+
+		coex_dm->is_switch_to_1dot5_ant = false;
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 0);
+
+		halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 0);
+	} else if (BTC_RSSI_HIGH(wifi_rssi_state2) &&
+		   BTC_RSSI_HIGH(bt_rssi_state2)) {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xc8);
+		/*halbtc8822b2ant_dec_bt_pwr(btcoexist, NORMAL_EXEC, 2);*/
+
+		coex_dm->is_switch_to_1dot5_ant = false;
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 4);
+
+		if (wifi_busy) {
+			if ((coex_sta->a2dp_bit_pool > 40) &&
+			    (coex_sta->a2dp_bit_pool < 255))
+				halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC,
+							true, 7);
+			else
+				halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC,
+							true, 5);
+		} else {
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						6);
+		}
+	} else {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+		/*halbtc8822b2ant_dec_bt_pwr(btcoexist, NORMAL_EXEC, 0);*/
+
+		coex_dm->is_switch_to_1dot5_ant = true;
+
+		if (wifi_turbo)
+			halbtc8822b2ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 6);
+		else
+			halbtc8822b2ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 7);
+
+		if (wifi_busy) {
+			if ((coex_sta->a2dp_bit_pool > 40) &&
+			    (coex_sta->a2dp_bit_pool < 255))
+				halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC,
+							true, 107);
+			else
+				halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC,
+							true, 105);
+		} else {
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						106);
+		}
+	}
+}
+
+/* PAN(EDR)+A2DP */
+static void halbtc8822b2ant_action_pan_edr_a2dp(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	static u8 prewifi_rssi_state = BTC_RSSI_STATE_LOW;
+	static u8 pre_bt_rssi_state = BTC_RSSI_STATE_LOW;
+	u8 wifi_rssi_state, bt_rssi_state;
+
+	static u8 prewifi_rssi_state2 = BTC_RSSI_STATE_LOW;
+	static u8 pre_bt_rssi_state2 = BTC_RSSI_STATE_LOW;
+	u8 wifi_rssi_state2, bt_rssi_state2;
+	bool wifi_busy = false, wifi_turbo = false;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+	btcoexist->btc_get(btcoexist, BTC_GET_U1_AP_NUM,
+			   &coex_sta->scan_ap_num);
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "############# [BTCoex],  scan_ap_num = %d, wl_noisy = %d\n",
+		 coex_sta->scan_ap_num, coex_sta->wl_noisy_level);
+
+	if ((wifi_busy) && (coex_sta->wl_noisy_level == 0))
+		wifi_turbo = true;
+
+	wifi_rssi_state = halbtc8822b2ant_wifi_rssi_state(
+		btcoexist, &prewifi_rssi_state, 2, coex_sta->wifi_coex_thres,
+		0);
+
+	wifi_rssi_state2 = halbtc8822b2ant_wifi_rssi_state(
+		btcoexist, &prewifi_rssi_state2, 2, coex_sta->wifi_coex_thres2,
+		0);
+
+	bt_rssi_state = halbtc8822b2ant_bt_rssi_state(
+		btcoexist, &pre_bt_rssi_state, 2, coex_sta->bt_coex_thres, 0);
+
+	bt_rssi_state2 = halbtc8822b2ant_bt_rssi_state(
+		btcoexist, &pre_bt_rssi_state2, 2, coex_sta->bt_coex_thres2, 0);
+
+	if (BTC_RSSI_HIGH(wifi_rssi_state) && BTC_RSSI_HIGH(bt_rssi_state)) {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+		halbtc8822b2ant_dec_bt_pwr(btcoexist, NORMAL_EXEC, 0);
+
+		coex_dm->is_switch_to_1dot5_ant = false;
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 0);
+		halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 0);
+	} else if (BTC_RSSI_HIGH(wifi_rssi_state2) &&
+		   BTC_RSSI_HIGH(bt_rssi_state2)) {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xc8);
+		halbtc8822b2ant_dec_bt_pwr(btcoexist, NORMAL_EXEC, 2);
+
+		coex_dm->is_switch_to_1dot5_ant = false;
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 4);
+
+		if (wifi_busy) {
+			if (((coex_sta->a2dp_bit_pool > 40) &&
+			     (coex_sta->a2dp_bit_pool < 255)) ||
+			    (!coex_sta->is_A2DP_3M))
+				halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC,
+							true, 7);
+			else
+				halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC,
+							true, 5);
+		} else {
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						6);
+		}
+	} else {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+		halbtc8822b2ant_dec_bt_pwr(btcoexist, NORMAL_EXEC, 0);
+
+		coex_dm->is_switch_to_1dot5_ant = true;
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 8);
+		if (wifi_busy)
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						107);
+		else
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						106);
+	}
+}
+
+static void halbtc8822b2ant_action_pan_edr_hid(struct btc_coexist *btcoexist)
+{
+	static u8 prewifi_rssi_state = BTC_RSSI_STATE_LOW;
+	static u8 pre_bt_rssi_state = BTC_RSSI_STATE_LOW;
+	u8 wifi_rssi_state, bt_rssi_state;
+
+	static u8 prewifi_rssi_state2 = BTC_RSSI_STATE_LOW;
+	static u8 pre_bt_rssi_state2 = BTC_RSSI_STATE_LOW;
+	u8 wifi_rssi_state2, bt_rssi_state2;
+	bool wifi_busy = false;
+	u32 wifi_bw = 1;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_BW, &wifi_bw);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+
+	wifi_rssi_state = halbtc8822b2ant_wifi_rssi_state(
+		btcoexist, &prewifi_rssi_state, 2, coex_sta->wifi_coex_thres,
+		0);
+
+	wifi_rssi_state2 = halbtc8822b2ant_wifi_rssi_state(
+		btcoexist, &prewifi_rssi_state2, 2, coex_sta->wifi_coex_thres2,
+		0);
+
+	bt_rssi_state = halbtc8822b2ant_bt_rssi_state(
+		btcoexist, &pre_bt_rssi_state, 2, coex_sta->bt_coex_thres, 0);
+
+	bt_rssi_state2 = halbtc8822b2ant_bt_rssi_state(
+		btcoexist, &pre_bt_rssi_state2, 2, coex_sta->bt_coex_thres2, 0);
+
+	if (BTC_RSSI_HIGH(wifi_rssi_state) && BTC_RSSI_HIGH(bt_rssi_state)) {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+		halbtc8822b2ant_dec_bt_pwr(btcoexist, NORMAL_EXEC, 0);
+
+		coex_dm->is_switch_to_1dot5_ant = false;
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 0);
+		halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 0);
+	} else if (BTC_RSSI_HIGH(wifi_rssi_state2) &&
+		   BTC_RSSI_HIGH(bt_rssi_state2)) {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xc8);
+		halbtc8822b2ant_dec_bt_pwr(btcoexist, NORMAL_EXEC, 2);
+
+		coex_dm->is_switch_to_1dot5_ant = false;
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 4);
+
+		if (wifi_busy)
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						3);
+		else
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						4);
+	} else {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+		halbtc8822b2ant_dec_bt_pwr(btcoexist, NORMAL_EXEC, 0);
+
+		coex_dm->is_switch_to_1dot5_ant = true;
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 8);
+
+		if (wifi_busy)
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						103);
+		else
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						104);
+	}
+}
+
+/* HID+A2DP+PAN(EDR) */
+static void
+halbtc8822b2ant_action_hid_a2dp_pan_edr(struct btc_coexist *btcoexist)
+{
+	static u8 prewifi_rssi_state = BTC_RSSI_STATE_LOW;
+	static u8 pre_bt_rssi_state = BTC_RSSI_STATE_LOW;
+	u8 wifi_rssi_state, bt_rssi_state;
+
+	static u8 prewifi_rssi_state2 = BTC_RSSI_STATE_LOW;
+	static u8 pre_bt_rssi_state2 = BTC_RSSI_STATE_LOW;
+	u8 wifi_rssi_state2, bt_rssi_state2;
+	bool wifi_busy = false;
+	u32 wifi_bw = 1;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_BW, &wifi_bw);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+
+	wifi_rssi_state = halbtc8822b2ant_wifi_rssi_state(
+		btcoexist, &prewifi_rssi_state, 2, coex_sta->wifi_coex_thres,
+		0);
+
+	wifi_rssi_state2 = halbtc8822b2ant_wifi_rssi_state(
+		btcoexist, &prewifi_rssi_state2, 2, coex_sta->wifi_coex_thres2,
+		0);
+
+	bt_rssi_state = halbtc8822b2ant_bt_rssi_state(
+		btcoexist, &pre_bt_rssi_state, 2, coex_sta->bt_coex_thres, 0);
+
+	bt_rssi_state2 = halbtc8822b2ant_bt_rssi_state(
+		btcoexist, &pre_bt_rssi_state2, 2, coex_sta->bt_coex_thres2, 0);
+
+	if (BTC_RSSI_HIGH(wifi_rssi_state) && BTC_RSSI_HIGH(bt_rssi_state)) {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+
+		coex_dm->is_switch_to_1dot5_ant = false;
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 0);
+		halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 0);
+	} else if (BTC_RSSI_HIGH(wifi_rssi_state2) &&
+		   BTC_RSSI_HIGH(bt_rssi_state2)) {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xc8);
+
+		coex_dm->is_switch_to_1dot5_ant = false;
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 4);
+
+		if (wifi_busy) {
+			if (((coex_sta->a2dp_bit_pool > 40) &&
+			     (coex_sta->a2dp_bit_pool < 255)) ||
+			    (!coex_sta->is_A2DP_3M))
+				halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC,
+							true, 7);
+			else
+				halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC,
+							true, 5);
+		} else {
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						6);
+		}
+	} else {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+
+		coex_dm->is_switch_to_1dot5_ant = true;
+
+		if (coex_sta->hid_busy_num >= 2) {
+			halbtc8822b2ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 8);
+
+			if (wifi_bw == 0) {
+				halbtc8822b2ant_set_wltoggle_coex_table(
+					btcoexist, NORMAL_EXEC, 0x1, 0xaa, 0x5a,
+					0xaa, 0xaa);
+			} else {
+				halbtc8822b2ant_set_wltoggle_coex_table(
+					btcoexist, NORMAL_EXEC, 0x2, 0xaa, 0x5a,
+					0xaa, 0xaa);
+			}
+			halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, true,
+						110);
+		} else {
+			halbtc8822b2ant_coex_table_with_type(btcoexist,
+							     NORMAL_EXEC, 1);
+
+			if (wifi_busy) {
+				if ((coex_sta->a2dp_bit_pool > 40) &&
+				    (coex_sta->a2dp_bit_pool < 255))
+					halbtc8822b2ant_ps_tdma(btcoexist,
+								NORMAL_EXEC,
+								true, 107);
+				else
+					halbtc8822b2ant_ps_tdma(btcoexist,
+								NORMAL_EXEC,
+								true, 105);
+			} else {
+				halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC,
+							true, 106);
+			}
+		}
+	}
+}
+
+static void halbtc8822b2ant_action_bt_whck_test(struct btc_coexist *btcoexist)
+{
+	halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+
+	halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 0);
+
+	halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 0);
+}
+
+static void halbtc8822b2ant_action_bt_hs(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	static u8 prewifi_rssi_state = BTC_RSSI_STATE_LOW;
+	static u8 pre_bt_rssi_state = BTC_RSSI_STATE_LOW;
+	u8 wifi_rssi_state, bt_rssi_state;
+
+	static u8 prewifi_rssi_state2 = BTC_RSSI_STATE_LOW;
+	static u8 pre_bt_rssi_state2 = BTC_RSSI_STATE_LOW;
+	u8 wifi_rssi_state2, bt_rssi_state2;
+	bool wifi_busy = false, wifi_turbo = false;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+	btcoexist->btc_get(btcoexist, BTC_GET_U1_AP_NUM,
+			   &coex_sta->scan_ap_num);
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "############# [BTCoex],  scan_ap_num = %d, wl_noisy = %d\n",
+		 coex_sta->scan_ap_num, coex_sta->wl_noisy_level);
+
+	if ((wifi_busy) && (coex_sta->wl_noisy_level == 0))
+		wifi_turbo = true;
+
+	wifi_rssi_state = halbtc8822b2ant_wifi_rssi_state(
+		btcoexist, &prewifi_rssi_state, 2, coex_sta->wifi_coex_thres,
+		0);
+
+	wifi_rssi_state2 = halbtc8822b2ant_wifi_rssi_state(
+		btcoexist, &prewifi_rssi_state2, 2, coex_sta->wifi_coex_thres2,
+		0);
+
+	bt_rssi_state = halbtc8822b2ant_bt_rssi_state(
+		btcoexist, &pre_bt_rssi_state, 2, coex_sta->bt_coex_thres, 0);
+
+	bt_rssi_state2 = halbtc8822b2ant_bt_rssi_state(
+		btcoexist, &pre_bt_rssi_state2, 2, coex_sta->bt_coex_thres2, 0);
+
+	if (BTC_RSSI_HIGH(wifi_rssi_state) && BTC_RSSI_HIGH(bt_rssi_state)) {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+
+		coex_dm->is_switch_to_1dot5_ant = false;
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 0);
+
+		halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 0);
+	} else if (BTC_RSSI_HIGH(wifi_rssi_state2) &&
+		   BTC_RSSI_HIGH(bt_rssi_state2)) {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xc8);
+
+		coex_dm->is_switch_to_1dot5_ant = false;
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 0);
+
+		halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 0);
+
+	} else {
+		halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+
+		coex_dm->is_switch_to_1dot5_ant = true;
+
+		halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 0);
+
+		halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 0);
+	}
+}
+
+static void
+halbtc8822b2ant_action_wifi_multi_port(struct btc_coexist *btcoexist)
+{
+	halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+
+	/* hw all off */
+	halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 0);
+
+	halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 0);
+}
+
+static void halbtc8822b2ant_action_wifi_connected(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	switch (coex_dm->cur_algorithm) {
+	case BT_8822B_2ANT_COEX_ALGO_SCO:
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], Action 2-Ant, algorithm = SCO.\n");
+		halbtc8822b2ant_action_sco(btcoexist);
+		break;
+	case BT_8822B_2ANT_COEX_ALGO_HID:
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], Action 2-Ant, algorithm = HID.\n");
+		halbtc8822b2ant_action_hid(btcoexist);
+		break;
+	case BT_8822B_2ANT_COEX_ALGO_A2DP:
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], Action 2-Ant, algorithm = A2DP.\n");
+		halbtc8822b2ant_action_a2dp(btcoexist);
+		break;
+	case BT_8822B_2ANT_COEX_ALGO_A2DPSINK:
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], Action 2-Ant, algorithm = A2DP Sink.\n");
+		halbtc8822b2ant_action_a2dpsink(btcoexist);
+		break;
+	case BT_8822B_2ANT_COEX_ALGO_A2DP_PANHS:
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], Action 2-Ant, algorithm = A2DP+PAN(HS).\n");
+		halbtc8822b2ant_action_a2dp_pan_hs(btcoexist);
+		break;
+	case BT_8822B_2ANT_COEX_ALGO_PANEDR:
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], Action 2-Ant, algorithm = PAN(EDR).\n");
+		halbtc8822b2ant_action_pan_edr(btcoexist);
+		break;
+	case BT_8822B_2ANT_COEX_ALGO_PANEDR_A2DP:
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], Action 2-Ant, algorithm = PAN+A2DP.\n");
+		halbtc8822b2ant_action_pan_edr_a2dp(btcoexist);
+		break;
+	case BT_8822B_2ANT_COEX_ALGO_PANEDR_HID:
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], Action 2-Ant, algorithm = PAN(EDR)+HID.\n");
+		halbtc8822b2ant_action_pan_edr_hid(btcoexist);
+		break;
+	case BT_8822B_2ANT_COEX_ALGO_HID_A2DP_PANEDR:
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], Action 2-Ant, algorithm = HID+A2DP+PAN.\n");
+		halbtc8822b2ant_action_hid_a2dp_pan_edr(btcoexist);
+		break;
+	case BT_8822B_2ANT_COEX_ALGO_HID_A2DP:
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], Action 2-Ant, algorithm = HID+A2DP.\n");
+		halbtc8822b2ant_action_hid_a2dp(btcoexist);
+		break;
+	case BT_8822B_2ANT_COEX_ALGO_NOPROFILEBUSY:
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], Action 2-Ant, algorithm = No-Profile busy.\n");
+		halbtc8822b2ant_action_bt_idle(btcoexist);
+		break;
+	default:
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], Action 2-Ant, algorithm = coexist All Off!!\n");
+		halbtc8822b2ant_action_coex_all_off(btcoexist);
+		break;
+	}
+
+	coex_dm->pre_algorithm = coex_dm->cur_algorithm;
+}
+
+static void halbtc8822b2ant_run_coexist_mechanism(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	u8 algorithm = 0;
+	u32 num_of_wifi_link = 0;
+	u32 wifi_link_status = 0;
+	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
+	bool miracast_plus_bt = false;
+	bool scan = false, link = false, roam = false, under_4way = false,
+	     wifi_connected = false, wifi_under_5g = false, bt_hs_on = false;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_SCAN, &scan);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_LINK, &link);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_ROAM, &roam);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_4_WAY_PROGRESS,
+			   &under_4way);
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], RunCoexistMechanism()===>\n");
+
+	if (btcoexist->manual_control) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], RunCoexistMechanism(), return for Manual CTRL <===\n");
+		return;
+	}
+
+	if (btcoexist->stop_coex_dm) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], RunCoexistMechanism(), return for Stop Coex DM <===\n");
+		return;
+	}
+
+	if (coex_sta->under_ips) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], wifi is under IPS !!!\n");
+		return;
+	}
+
+	if ((coex_sta->under_lps) &&
+	    (coex_dm->bt_status != BT_8822B_2ANT_BT_STATUS_ACL_BUSY)) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], RunCoexistMechanism(), wifi is under LPS !!!\n");
+		halbtc8822b2ant_action_wifi_native_lps(btcoexist);
+		return;
+	}
+
+	if (!coex_sta->run_time_state) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], return for run_time_state = false !!!\n");
+		return;
+	}
+
+	if (coex_sta->freeze_coexrun_by_btinfo) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], BtInfoNotify(), return for freeze_coexrun_by_btinfo\n");
+		return;
+	}
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_5G, &wifi_under_5g);
+
+	if ((wifi_under_5g) &&
+	    (coex_sta->switch_band_notify_to != BTC_SWITCH_TO_24G) &&
+	    (coex_sta->switch_band_notify_to != BTC_SWITCH_TO_24G_NOFORSCAN)) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], WiFi is under 5G!!!\n");
+
+		halbtc8822b2ant_action_wifi_under5g(btcoexist);
+		return;
+	}
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], WiFi is under 2G!!!\n");
+
+	halbtc8822b2ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO, NORMAL_EXEC,
+				     BT_8822B_2ANT_PHASE_2G_RUNTIME);
+
+	if (coex_sta->bt_whck_test) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], BT is under WHCK TEST!!!\n");
+		halbtc8822b2ant_action_bt_whck_test(btcoexist);
+		return;
+	}
+
+	if (coex_sta->bt_disabled) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], BT is disabled!!!\n");
+		halbtc8822b2ant_action_coex_all_off(btcoexist);
+		return;
+	}
+
+	if (coex_sta->c2h_bt_inquiry_page) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], BT is under inquiry/page scan !!\n");
+		halbtc8822b2ant_action_bt_inquiry(btcoexist);
+		return;
+	}
+
+	if (coex_sta->is_setup_link) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], BT is re-link !!!\n");
+		halbtc8822b2ant_action_bt_relink(btcoexist);
+		return;
+	}
+
+	/* for P2P */
+	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_LINK_STATUS,
+			   &wifi_link_status);
+	num_of_wifi_link = wifi_link_status >> 16;
+
+	if ((num_of_wifi_link >= 2) ||
+	    (wifi_link_status & WIFI_P2P_GO_CONNECTED)) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"############# [BTCoex],  Multi-Port num_of_wifi_link = %d, wifi_link_status = 0x%x\n",
+			num_of_wifi_link, wifi_link_status);
+
+		if (bt_link_info->bt_link_exist)
+			miracast_plus_bt = true;
+		else
+			miracast_plus_bt = false;
+
+		btcoexist->btc_set(btcoexist, BTC_SET_BL_MIRACAST_PLUS_BT,
+				   &miracast_plus_bt);
+
+		if (scan || link || roam || under_4way) {
+			RT_TRACE(
+				rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				"[BTCoex], scan = %d, link = %d, roam = %d 4way = %d!!!\n",
+				scan, link, roam, under_4way);
+
+			RT_TRACE(
+				rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				"[BTCoex], wifi is under linkscan process + Multi-Port !!\n");
+
+			halbtc8822b2ant_action_wifi_link_process(btcoexist);
+		} else {
+			halbtc8822b2ant_action_wifi_multi_port(btcoexist);
+		}
+
+		return;
+	}
+
+	miracast_plus_bt = false;
+	btcoexist->btc_set(btcoexist, BTC_SET_BL_MIRACAST_PLUS_BT,
+			   &miracast_plus_bt);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_HS_OPERATION, &bt_hs_on);
+
+	if (bt_hs_on) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "############# [BTCoex],  BT Is hs\n");
+		halbtc8822b2ant_action_bt_hs(btcoexist);
+		return;
+	}
+
+	if ((BT_8822B_2ANT_BT_STATUS_NON_CONNECTED_IDLE ==
+	     coex_dm->bt_status) ||
+	    (coex_dm->bt_status == BT_8822B_2ANT_BT_STATUS_CONNECTED_IDLE)) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], Action 2-Ant, bt idle!!.\n");
+
+		halbtc8822b2ant_action_bt_idle(btcoexist);
+		return;
+	}
+
+	algorithm = halbtc8822b2ant_action_algorithm(btcoexist);
+	coex_dm->cur_algorithm = algorithm;
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], Algorithm = %d\n", coex_dm->cur_algorithm);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_CONNECTED,
+			   &wifi_connected);
+
+	if (scan || link || roam || under_4way) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], WiFi is under Link Process !!\n");
+		halbtc8822b2ant_action_wifi_link_process(btcoexist);
+	} else if (wifi_connected) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], Action 2-Ant, wifi connected!!.\n");
+		halbtc8822b2ant_action_wifi_connected(btcoexist);
+
+	} else {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], Action 2-Ant, wifi not-connected!!.\n");
+		halbtc8822b2ant_action_wifi_nonconnected(btcoexist);
+	}
+}
+
+static void halbtc8822b2ant_init_coex_dm(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], Coex Mechanism Init!!\n");
+
+	halbtc8822b2ant_low_penalty_ra(btcoexist, NORMAL_EXEC, false);
+
+	halbtc8822b2ant_coex_table_with_type(btcoexist, NORMAL_EXEC, 5);
+
+	/* fw all off */
+	halbtc8822b2ant_ps_tdma(btcoexist, NORMAL_EXEC, false, 0);
+
+	halbtc8822b2ant_fw_dac_swing_lvl(btcoexist, NORMAL_EXEC, 0xd8);
+
+	coex_sta->pop_event_cnt = 0;
+	coex_sta->cnt_remote_name_req = 0;
+	coex_sta->cnt_reinit = 0;
+	coex_sta->cnt_setup_link = 0;
+	coex_sta->cnt_ign_wlan_act = 0;
+	coex_sta->cnt_page = 0;
+	coex_sta->cnt_role_switch = 0;
+	coex_sta->switch_band_notify_to = BTC_NOT_SWITCH;
+
+	halbtc8822b2ant_query_bt_info(btcoexist);
+}
+
+static void halbtc8822b2ant_init_hw_config(struct btc_coexist *btcoexist,
+					   bool wifi_only)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	u32 u32tmp1 = 0, u32tmp2 = 0, u32tmp3 = 0;
+	u32 RTL97F_8822B = 0;
+	u8 i = 0;
+
+	u32tmp3 = btcoexist->btc_read_4byte(btcoexist, 0xcb4);
+	u32tmp1 = halbtc8822b2ant_ltecoex_indirect_read_reg(btcoexist, 0x38);
+	u32tmp2 = halbtc8822b2ant_ltecoex_indirect_read_reg(btcoexist, 0x54);
+
+	if (RTL97F_8822B) {
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x66, 0x04, 0x0);
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x41, 0x02, 0x0);
+
+		/* set GNT_BT to SW high */
+		halbtc8822b2ant_ltecoex_set_gnt_bt(
+			btcoexist, BT_8822B_2ANT_GNT_BLOCK_RFC_BB,
+			BT_8822B_2ANT_GNT_TYPE_CTRL_BY_SW,
+			BT_8822B_2ANT_SIG_STA_SET_TO_HIGH);
+		/* Set GNT_WL to SW high */
+		halbtc8822b2ant_ltecoex_set_gnt_wl(
+			btcoexist, BT_8822B_2ANT_GNT_BLOCK_RFC_BB,
+			BT_8822B_2ANT_GNT_TYPE_CTRL_BY_SW,
+			BT_8822B_2ANT_SIG_STA_SET_TO_HIGH);
+		return;
+	}
+
+	RT_TRACE(
+		rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		"[BTCoex], (Before Init HW config) 0xcb4 = 0x%x, 0x38= 0x%x, 0x54= 0x%x\n",
+		u32tmp3, u32tmp1, u32tmp2);
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], 2Ant Init HW Config!!\n");
+
+	coex_sta->bt_coex_supported_feature = 0;
+	coex_sta->bt_coex_supported_version = 0;
+	coex_sta->bt_ble_scan_type = 0;
+	coex_sta->bt_ble_scan_para[0] = 0;
+	coex_sta->bt_ble_scan_para[1] = 0;
+	coex_sta->bt_ble_scan_para[2] = 0;
+	coex_sta->bt_reg_vendor_ac = 0xffff;
+	coex_sta->bt_reg_vendor_ae = 0xffff;
+	coex_sta->isolation_btween_wb = BT_8822B_2ANT_DEFAULT_ISOLATION;
+	coex_sta->gnt_error_cnt = 0;
+	coex_sta->bt_relink_downcount = 0;
+	coex_sta->is_set_ps_state_fail = false;
+	coex_sta->cnt_set_ps_state_fail = 0;
+
+	for (i = 0; i <= 9; i++)
+		coex_sta->bt_afh_map[i] = 0;
+
+	/* 0xf0[15:12] --> Chip Cut information */
+	coex_sta->cut_version =
+		(btcoexist->btc_read_1byte(btcoexist, 0xf1) & 0xf0) >> 4;
+
+	coex_sta->dis_ver_info_cnt = 0;
+
+	halbtc8822b2ant_coex_switch_threshold(btcoexist,
+					      coex_sta->isolation_btween_wb);
+
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0x550, 0x8,
+					   0x1); /* enable TBTT nterrupt */
+
+	/* BT report packet sample rate	 */
+	btcoexist->btc_write_1byte(btcoexist, 0x790, 0x5);
+
+	/* Init 0x778 = 0x1 for 2-Ant */
+	btcoexist->btc_write_1byte(btcoexist, 0x778, 0x1);
+
+	/* Enable PTA (3-wire function form BT side) */
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0x40, 0x20, 0x1);
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0x41, 0x02, 0x1);
+
+	/* Enable PTA (tx/rx signal form WiFi side) */
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0x4c6, 0x10, 0x1);
+
+	halbtc8822b2ant_enable_gnt_to_gpio(btcoexist, true);
+
+	/*GNT_BT=1 while select both */
+	btcoexist->btc_write_1byte_bitmask(btcoexist, 0x763, 0x10, 0x1);
+
+	/* check if WL firmware download ok */
+	halbtc8822b2ant_post_state_to_bt(btcoexist,
+					 BT_8822B_2ANT_SCOREBOARD_ONOFF, true);
+
+	/* Enable counter statistics */
+	btcoexist->btc_write_1byte(
+		btcoexist, 0x76e,
+		0x4); /* 0x76e[3] =1, WLAN_Act control by PTA */
+
+	halbtc8822b2ant_coex_table_with_type(btcoexist, FORCE_EXEC, 5);
+
+	halbtc8822b2ant_ps_tdma(btcoexist, FORCE_EXEC, false, 0);
+
+	psd_scan->ant_det_is_ant_det_available = true;
+
+	if (coex_sta->is_rf_state_off) {
+		halbtc8822b2ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO,
+					     FORCE_EXEC,
+					     BT_8822B_2ANT_PHASE_WLAN_OFF);
+
+		btcoexist->stop_coex_dm = true;
+
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], **********  %s (RF Off)**********\n",
+			 __func__);
+	} else if (wifi_only) {
+		coex_sta->concurrent_rx_mode_on = false;
+		/* Path config	 */
+		/* Set Antenna Path */
+		halbtc8822b2ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO,
+					     FORCE_EXEC,
+					     BT_8822B_2ANT_PHASE_WLANONLY_INIT);
+
+		btcoexist->stop_coex_dm = true;
+	} else {
+		/* Set BT polluted packet on for Tx rate adaptive not including
+		 * Tx retry break by PTA, 0x45c[19] =1
+		 */
+		btcoexist->btc_write_1byte_bitmask(btcoexist, 0x45e, 0x8, 0x1);
+
+		coex_sta->concurrent_rx_mode_on = true;
+
+		/* RF 0x1[1] = 0->Set GNT_WL_RF_Rx always = 1 for
+		 * con-current Rx, mask Tx only
+		 */
+		btcoexist->btc_set_rf_reg(btcoexist, BTC_RF_A, 0x1, 0x2, 0x0);
+
+		/* Set Antenna Path */
+		halbtc8822b2ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO,
+					     FORCE_EXEC,
+					     BT_8822B_2ANT_PHASE_COEX_INIT);
+
+		btcoexist->stop_coex_dm = false;
+	}
+}
+
+/* ************************************************************
+ * work around function start with wa_halbtc8822b2ant_
+ * ************************************************************
+ * ************************************************************
+ * extern function start with ex_halbtc8822b2ant_
+ * *************************************************************/
+void ex_btc8822b2ant_power_on_setting(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	struct btc_board_info *board_info = &btcoexist->board_info;
+	u8 u8tmp = 0x0;
+	u16 u16tmp = 0x0;
+
+	RT_TRACE(
+		rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		"xxxxxxxxxxxxxxxx Execute 8822b 2-Ant PowerOn Setting xxxxxxxxxxxxxxxx!!\n");
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "Ant Det Finish = %s, Ant Det Number  = %d\n",
+		 (board_info->btdm_ant_det_finish ? "Yes" : "No"),
+		 board_info->btdm_ant_num_by_ant_det);
+
+	btcoexist->dbg_mode_2ant = false;
+	btcoexist->stop_coex_dm = true;
+	psd_scan->ant_det_is_ant_det_available = false;
+
+	/* enable BB, REG_SYS_FUNC_EN such that we can write BB Reg correctly */
+	u16tmp = btcoexist->btc_read_2byte(btcoexist, 0x2);
+	btcoexist->btc_write_2byte(btcoexist, 0x2, u16tmp | BIT(0) | BIT(1));
+
+	/* Local setting bit define */
+	/*	BIT0: "0" for no antenna inverse; "1" for antenna inverse  */
+	/*	BIT1: "0" for internal switch; "1" for external switch */
+	/*	BIT2: "0" for one antenna; "1" for two antenna */
+	/* NOTE: here default all internal switch and 1-antenna ==>
+	 * BIT1=0 and BIT2=0
+	 */
+
+	/* Check efuse 0xc3[6] for Single Antenna Path */
+
+	/* Setup RF front end type */
+	halbtc8822b2ant_set_rfe_type(btcoexist);
+
+	/* Set Antenna Path to BT side */
+	halbtc8822b2ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO, FORCE_EXEC,
+				     BT_8822B_2ANT_PHASE_COEX_POWERON);
+
+	/* Save"single antenna position" info in Local register setting for
+	 * FW reading, because FW may not ready at power on
+	 */
+	if (btcoexist->chip_interface == BTC_INTF_PCI)
+		btcoexist->btc_write_local_reg_1byte(btcoexist, 0x3e0, u8tmp);
+	else if (btcoexist->chip_interface == BTC_INTF_USB)
+		btcoexist->btc_write_local_reg_1byte(btcoexist, 0xfe08, u8tmp);
+	else if (btcoexist->chip_interface == BTC_INTF_SDIO)
+		btcoexist->btc_write_local_reg_1byte(btcoexist, 0x60, u8tmp);
+
+	/* enable GNT_WL/GNT_BT debug signal to GPIO14/15 */
+	halbtc8822b2ant_enable_gnt_to_gpio(btcoexist, true);
+
+	RT_TRACE(
+		rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		"[BTCoex], **********  LTE coex Reg 0x38 (Power-On) = 0x%x**********\n",
+		halbtc8822b2ant_ltecoex_indirect_read_reg(btcoexist, 0x38));
+
+	RT_TRACE(
+		rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		"[BTCoex], **********  MAC Reg 0x70/ BB Reg 0xcb4 (Power-On) = 0x%x / 0x%x\n",
+		btcoexist->btc_read_4byte(btcoexist, 0x70),
+		btcoexist->btc_read_4byte(btcoexist, 0xcb4));
+}
+
+void ex_btc8822b2ant_pre_load_firmware(struct btc_coexist *btcoexist)
+{
+	struct btc_board_info *board_info = &btcoexist->board_info;
+	u8 u8tmp = 0x4; /* Set BIT2 by default since it's 2ant case */
+
+	/* */
+	/* S0 or S1 setting and Local register setting
+	 * (By the setting fw can get ant number, S0/S1, ... info)
+	 */
+	/* Local setting bit define */
+	/*	BIT0: "0" for no antenna inverse; "1" for antenna inverse  */
+	/*	BIT1: "0" for internal switch; "1" for external switch */
+	/*	BIT2: "0" for one antenna; "1" for two antenna */
+	/* NOTE: here default all internal switch and 1-antenna ==>
+	 *       BIT1=0 and BIT2=0
+	 */
+	if (btcoexist->chip_interface == BTC_INTF_USB) {
+		/* fixed at S0 for USB interface */
+		u8tmp |= 0x1; /* antenna inverse */
+		btcoexist->btc_write_local_reg_1byte(btcoexist, 0xfe08, u8tmp);
+	} else {
+		/* for PCIE and SDIO interface, we check efuse 0xc3[6] */
+		if (board_info->single_ant_path == 0) {
+		} else if (board_info->single_ant_path == 1) {
+			/* set to S0 */
+			u8tmp |= 0x1; /* antenna inverse */
+		}
+
+		if (btcoexist->chip_interface == BTC_INTF_PCI)
+			btcoexist->btc_write_local_reg_1byte(btcoexist, 0x3e0,
+							     u8tmp);
+		else if (btcoexist->chip_interface == BTC_INTF_SDIO)
+			btcoexist->btc_write_local_reg_1byte(btcoexist, 0x60,
+							     u8tmp);
+	}
+}
+
+void ex_btc8822b2ant_init_hw_config(struct btc_coexist *btcoexist,
+				    bool wifi_only)
+{
+	halbtc8822b2ant_init_hw_config(btcoexist, wifi_only);
+	btcoexist->auto_report_2ant = true;
+}
+
+void ex_btc8822b2ant_init_coex_dm(struct btc_coexist *btcoexist)
+{
+	halbtc8822b2ant_init_coex_dm(btcoexist);
+}
+
+void ex_btc8822b2ant_display_coex_info(struct btc_coexist *btcoexist,
+				       struct seq_file *m)
+{
+	struct btc_board_info *board_info = &btcoexist->board_info;
+	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
+
+	u8 u8tmp[4], i, ps_tdma_case = 0;
+	u32 u32tmp[4];
+	u16 u16tmp[4];
+	u32 fa_ofdm, fa_cck, cca_ofdm, cca_cck, ratio_ofdm;
+	u32 fw_ver = 0, bt_patch_ver = 0, bt_coex_ver = 0;
+	static u8 pop_report_in_10s;
+	u32 phyver = 0;
+	bool lte_coex_on = false;
+	static u8 cnt;
+
+	seq_puts(m, "\r\n ============[BT Coexist info]============");
+
+	if (btcoexist->manual_control) {
+		seq_puts(m,
+			 "\r\n ============[Under Manual Control]============");
+		seq_puts(m, "\r\n ==========================================");
+	}
+
+	if (!coex_sta->bt_disabled) {
+		if (coex_sta->bt_coex_supported_feature == 0)
+			btcoexist->btc_get(
+				btcoexist, BTC_GET_U4_SUPPORTED_FEATURE,
+				&coex_sta->bt_coex_supported_feature);
+
+		if ((coex_sta->bt_coex_supported_version == 0) ||
+		    (coex_sta->bt_coex_supported_version == 0xffff))
+			btcoexist->btc_get(
+				btcoexist, BTC_GET_U4_SUPPORTED_VERSION,
+				&coex_sta->bt_coex_supported_version);
+
+		if (coex_sta->bt_reg_vendor_ac == 0xffff)
+			coex_sta->bt_reg_vendor_ac = (u16)(
+				btcoexist->btc_get_bt_reg(btcoexist, 3, 0xac) &
+				0xffff);
+
+		if (coex_sta->bt_reg_vendor_ae == 0xffff)
+			coex_sta->bt_reg_vendor_ae = (u16)(
+				btcoexist->btc_get_bt_reg(btcoexist, 3, 0xae) &
+				0xffff);
+
+		btcoexist->btc_get(btcoexist, BTC_GET_U4_BT_PATCH_VER,
+				   &bt_patch_ver);
+		btcoexist->bt_info.bt_get_fw_ver = bt_patch_ver;
+
+		if (coex_sta->num_of_profile > 0) {
+			cnt++;
+
+			if (cnt >= 3) {
+				btcoexist->btc_get_bt_afh_map_from_bt(
+					btcoexist, 0, &coex_sta->bt_afh_map[0]);
+				cnt = 0;
+			}
+		}
+	}
+
+	if (psd_scan->ant_det_try_count == 0) {
+		seq_printf(
+			m, "\r\n %-35s = %d/ %d/ %s / %d",
+			"Ant PG Num/ Mech/ Pos/ RFE", board_info->pg_ant_num,
+			board_info->btdm_ant_num,
+			(board_info->btdm_ant_pos == BTC_ANTENNA_AT_MAIN_PORT ?
+				 "Main" :
+				 "Aux"),
+			rfe_type->rfe_module_type);
+	} else {
+		seq_printf(
+			m, "\r\n %-35s = %d/ %d/ %s/ %d  (%d/%d/%d)",
+			"Ant PG Num/ Mech(Ant_Det)/ Pos/ RFE",
+			board_info->pg_ant_num,
+			board_info->btdm_ant_num_by_ant_det,
+			(board_info->btdm_ant_pos == BTC_ANTENNA_AT_MAIN_PORT ?
+				 "Main" :
+				 "Aux"),
+			rfe_type->rfe_module_type, psd_scan->ant_det_try_count,
+			psd_scan->ant_det_fail_count, psd_scan->ant_det_result);
+
+		if (board_info->btdm_ant_det_finish) {
+			if (psd_scan->ant_det_result != 12)
+				seq_printf(m, "\r\n %-35s = %s",
+					   "Ant Det PSD Value",
+					   psd_scan->ant_det_peak_val);
+			else
+				seq_printf(m, "\r\n %-35s = %d",
+					   "Ant Det PSD Value",
+					   psd_scan->ant_det_psd_scan_peak_val /
+						   100);
+		}
+	}
+
+	bt_patch_ver = btcoexist->bt_info.bt_get_fw_ver;
+	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_FW_VER, &fw_ver);
+	phyver = btcoexist->btc_get_bt_phydm_version(btcoexist);
+
+	bt_coex_ver = (coex_sta->bt_coex_supported_version & 0xff);
+
+	seq_printf(
+		m, "\r\n %-35s = %d_%02x/ 0x%02x/ 0x%02x (%s)",
+		"CoexVer WL/  BT_Desired/ BT_Report",
+		glcoex_ver_date_8822b_2ant, glcoex_ver_8822b_2ant,
+		glcoex_ver_btdesired_8822b_2ant, bt_coex_ver,
+		(bt_coex_ver == 0xff ?
+			 "Unknown" :
+			 (coex_sta->bt_disabled ?  "BT-disable" :
+			  (bt_coex_ver >= glcoex_ver_btdesired_8822b_2ant ?
+					   "Match" :
+					   "Mis-Match"))));
+
+	seq_printf(m, "\r\n %-35s = 0x%x/ 0x%x/ v%d/ %c", "W_FW/ B_FW/ Phy/ Kt",
+		   fw_ver, bt_patch_ver, phyver, coex_sta->cut_version + 65);
+
+	seq_printf(m, "\r\n %-35s = %02x %02x %02x ", "AFH Map to BT",
+		   coex_dm->wifi_chnl_info[0], coex_dm->wifi_chnl_info[1],
+		   coex_dm->wifi_chnl_info[2]);
+
+	seq_printf(m, "\r\n %-35s = %d / %d / %d ",
+		   "Isolation/WL_Thres/BT_Thres", coex_sta->isolation_btween_wb,
+		   coex_sta->wifi_coex_thres, coex_sta->bt_coex_thres);
+
+	/* wifi status */
+	seq_printf(m, "\r\n %-35s", "============[Wifi Status]============");
+	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_WIFI_STATUS, m);
+
+	seq_printf(m, "\r\n %-35s", "============[BT Status]============");
+
+	pop_report_in_10s++;
+	seq_printf(
+		m, "\r\n %-35s = [%s/ %d dBm/ %d/ %d] ",
+		"BT [status/ rssi/ retryCnt/ popCnt]",
+		((coex_sta->bt_disabled) ?
+			 ("disabled") :
+			 ((coex_sta->c2h_bt_inquiry_page) ?  ("inquiry/page") :
+			  ((BT_8822B_2ANT_BT_STATUS_NON_CONNECTED_IDLE ==
+			    coex_dm->bt_status) ?
+				   "non-connected idle" :
+				   ((coex_dm->bt_status ==
+				     BT_8822B_2ANT_BT_STATUS_CONNECTED_IDLE) ?
+					    "connected-idle" :
+					    "busy")))),
+		coex_sta->bt_rssi - 100, coex_sta->bt_retry_cnt,
+		coex_sta->pop_event_cnt);
+
+	if (pop_report_in_10s >= 5) {
+		coex_sta->pop_event_cnt = 0;
+		pop_report_in_10s = 0;
+	}
+
+	if (coex_sta->num_of_profile != 0)
+		seq_printf(
+			m, "\r\n %-35s = %s%s%s%s%s", "Profiles",
+			((bt_link_info->a2dp_exist) ?
+				 ((coex_sta->is_bt_a2dp_sink) ? "A2DP sink," :
+								"A2DP,") :
+				 ""),
+			((bt_link_info->sco_exist) ? "HFP," : ""),
+			((bt_link_info->hid_exist) ?
+				 ((coex_sta->hid_busy_num >= 2) ?
+					  "HID(4/18)," :
+					  "HID(2/18),") :
+				 ""),
+			((bt_link_info->pan_exist) ? "PAN," : ""),
+			((coex_sta->voice_over_HOGP) ? "Voice" : ""));
+	else
+		seq_printf(m, "\r\n %-35s = None", "Profiles");
+
+	if (bt_link_info->a2dp_exist) {
+		seq_printf(m, "\r\n %-35s = %s/ %d/ %s",
+			   "A2DP Rate/Bitpool/Auto_Slot",
+			   ((coex_sta->is_A2DP_3M) ? "3M" : "No_3M"),
+			   coex_sta->a2dp_bit_pool,
+			   ((coex_sta->is_autoslot) ? "On" : "Off"));
+	}
+
+	if (bt_link_info->hid_exist) {
+		seq_printf(m, "\r\n %-35s = %d/ %d", "HID PairNum/Forbid_Slot",
+			   coex_sta->hid_pair_cnt, coex_sta->forbidden_slot);
+	}
+
+	seq_printf(m, "\r\n %-35s = %s/ %d/ %s/ 0x%x",
+		   "Role/RoleSwCnt/IgnWlact/Feature",
+		   ((bt_link_info->slave_role) ? "Slave" : "Master"),
+		   coex_sta->cnt_role_switch,
+		   ((coex_dm->cur_ignore_wlan_act) ? "Yes" : "No"),
+		   coex_sta->bt_coex_supported_feature);
+
+	if ((coex_sta->bt_ble_scan_type & 0x7) != 0x0) {
+		seq_printf(m, "\r\n %-35s = 0x%x/ 0x%x/ 0x%x/ 0x%x",
+			   "BLEScan Type/TV/Init/Ble",
+			   coex_sta->bt_ble_scan_type,
+			   (coex_sta->bt_ble_scan_type & 0x1 ?
+				    coex_sta->bt_ble_scan_para[0] :
+				    0x0),
+			   (coex_sta->bt_ble_scan_type & 0x2 ?
+				    coex_sta->bt_ble_scan_para[1] :
+				    0x0),
+			   (coex_sta->bt_ble_scan_type & 0x4 ?
+				    coex_sta->bt_ble_scan_para[2] :
+				    0x0));
+	}
+
+	seq_printf(m, "\r\n %-35s = %d/ %d/ %d/ %d/ %d",
+		   "ReInit/ReLink/IgnWlact/Page/NameReq", coex_sta->cnt_reinit,
+		   coex_sta->cnt_setup_link, coex_sta->cnt_ign_wlan_act,
+		   coex_sta->cnt_page, coex_sta->cnt_remote_name_req);
+
+	halbtc8822b2ant_read_score_board(btcoexist, &u16tmp[0]);
+
+	if ((coex_sta->bt_reg_vendor_ae == 0xffff) ||
+	    (coex_sta->bt_reg_vendor_ac == 0xffff))
+		seq_printf(m, "\r\n %-35s = x/ x/ %04x",
+			   "0xae[4]/0xac[1:0]/Scoreboard", u16tmp[0]);
+	else
+		seq_printf(m, "\r\n %-35s = 0x%x/ 0x%x/ %04x",
+			   "0xae[4]/0xac[1:0]/Scoreboard",
+			   (int)((coex_sta->bt_reg_vendor_ae & BIT(4)) >> 4),
+			   coex_sta->bt_reg_vendor_ac & 0x3, u16tmp[0]);
+
+	if (coex_sta->num_of_profile > 0) {
+		seq_printf(
+			m,
+			"\r\n %-35s = %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x",
+			"AFH MAP", coex_sta->bt_afh_map[0],
+			coex_sta->bt_afh_map[1], coex_sta->bt_afh_map[2],
+			coex_sta->bt_afh_map[3], coex_sta->bt_afh_map[4],
+			coex_sta->bt_afh_map[5], coex_sta->bt_afh_map[6],
+			coex_sta->bt_afh_map[7], coex_sta->bt_afh_map[8],
+			coex_sta->bt_afh_map[9]);
+	}
+
+	for (i = 0; i < BT_INFO_SRC_8822B_2ANT_MAX; i++) {
+		if (coex_sta->bt_info_c2h_cnt[i]) {
+			seq_printf(
+				m,
+				"\r\n %-35s = %02x %02x %02x %02x %02x %02x %02x(%d)",
+				glbt_info_src_8822b_2ant[i],
+				coex_sta->bt_info_c2h[i][0],
+				coex_sta->bt_info_c2h[i][1],
+				coex_sta->bt_info_c2h[i][2],
+				coex_sta->bt_info_c2h[i][3],
+				coex_sta->bt_info_c2h[i][4],
+				coex_sta->bt_info_c2h[i][5],
+				coex_sta->bt_info_c2h[i][6],
+				coex_sta->bt_info_c2h_cnt[i]);
+		}
+	}
+
+	/* Sw mechanism	 */
+	if (btcoexist->manual_control)
+		seq_printf(
+			m, "\r\n %-35s",
+			"============[mechanism] (before Manual)============");
+	else
+		seq_printf(m, "\r\n %-35s",
+			   "============[Mechanism]============");
+
+	ps_tdma_case = coex_dm->cur_ps_tdma;
+
+	seq_printf(m, "\r\n %-35s = %02x %02x %02x %02x %02x (case-%d, %s, %s)",
+		   "TDMA", coex_dm->ps_tdma_para[0], coex_dm->ps_tdma_para[1],
+		   coex_dm->ps_tdma_para[2], coex_dm->ps_tdma_para[3],
+		   coex_dm->ps_tdma_para[4], ps_tdma_case,
+		   (coex_dm->cur_ps_tdma_on ? "TDMA On" : "TDMA Off"),
+		   (coex_dm->is_switch_to_1dot5_ant ? "1.5Ant" : "2Ant"));
+
+	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x6c0);
+	u32tmp[1] = btcoexist->btc_read_4byte(btcoexist, 0x6c4);
+	u32tmp[2] = btcoexist->btc_read_4byte(btcoexist, 0x6c8);
+	seq_printf(m, "\r\n %-35s = %d/ 0x%x/ 0x%x/ 0x%x",
+		   "Table/0x6c0/0x6c4/0x6c8", coex_sta->coex_table_type,
+		   u32tmp[0], u32tmp[1], u32tmp[2]);
+
+	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x778);
+	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x6cc);
+	seq_printf(m, "\r\n %-35s = 0x%x/ 0x%x", "0x778/0x6cc", u8tmp[0],
+		   u32tmp[0]);
+
+	seq_printf(m, "\r\n %-35s = %s/ %s/ %s/ %d",
+		   "AntDiv/BtCtrlLPS/LPRA/PsFail",
+		   ((board_info->ant_div_cfg) ? "On" : "Off"),
+		   ((coex_sta->force_lps_ctrl) ? "On" : "Off"),
+		   ((coex_dm->cur_low_penalty_ra) ? "On" : "Off"),
+		   coex_sta->cnt_set_ps_state_fail);
+
+	seq_printf(m, "\r\n %-35s = 0x%x/ 0x%x", "WL_DACSwing/ BT_Dec_Pwr",
+		   coex_dm->cur_fw_dac_swing_lvl, coex_dm->cur_bt_dec_pwr_lvl);
+
+	u32tmp[0] = halbtc8822b2ant_ltecoex_indirect_read_reg(btcoexist, 0x38);
+	lte_coex_on = ((u32tmp[0] & BIT(7)) >> 7) ? true : false;
+
+	if (lte_coex_on) {
+		u32tmp[0] = halbtc8822b2ant_ltecoex_indirect_read_reg(btcoexist,
+								      0xa0);
+		u32tmp[1] = halbtc8822b2ant_ltecoex_indirect_read_reg(btcoexist,
+								      0xa4);
+		seq_printf(m, "\r\n %-35s = 0x%x/ 0x%x",
+			   "LTE Coex Table W_L/B_L", u32tmp[0] & 0xffff,
+			   u32tmp[1] & 0xffff);
+
+		u32tmp[0] = halbtc8822b2ant_ltecoex_indirect_read_reg(btcoexist,
+								      0xa8);
+		u32tmp[1] = halbtc8822b2ant_ltecoex_indirect_read_reg(btcoexist,
+								      0xac);
+		u32tmp[2] = halbtc8822b2ant_ltecoex_indirect_read_reg(btcoexist,
+								      0xb0);
+		u32tmp[3] = halbtc8822b2ant_ltecoex_indirect_read_reg(btcoexist,
+								      0xb4);
+		seq_printf(m, "\r\n %-35s = 0x%x/ 0x%x/ 0x%x/ 0x%x",
+			   "LTE Break Table W_L/B_L/L_W/L_B",
+			   u32tmp[0] & 0xffff, u32tmp[1] & 0xffff,
+			   u32tmp[2] & 0xffff, u32tmp[3] & 0xffff);
+	}
+
+	/* Hw setting		 */
+	seq_printf(m, "\r\n %-35s", "============[Hw setting]============");
+
+	u32tmp[0] = halbtc8822b2ant_ltecoex_indirect_read_reg(btcoexist, 0x38);
+	u32tmp[1] = halbtc8822b2ant_ltecoex_indirect_read_reg(btcoexist, 0x54);
+	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x73);
+
+	seq_printf(m, "\r\n %-35s = %s/ %s", "LTE Coex/Path Owner",
+		   ((lte_coex_on) ? "On" : "Off"),
+		   ((u8tmp[0] & BIT(2)) ? "WL" : "BT"));
+
+	if (lte_coex_on) {
+		seq_printf(m, "\r\n %-35s = %d/ %d/ %d/ %d",
+			   "LTE 3Wire/OPMode/UART/UARTMode",
+			   (int)((u32tmp[0] & BIT(6)) >> 6),
+			   (int)((u32tmp[0] & (BIT(5) | BIT(4))) >> 4),
+			   (int)((u32tmp[0] & BIT(3)) >> 3),
+			   (int)(u32tmp[0] & (BIT(2) | BIT(1) | BIT(0))));
+
+		seq_printf(m, "\r\n %-35s = %d/ %d", "LTE_Busy/UART_Busy",
+			   (int)((u32tmp[1] & BIT(1)) >> 1),
+			   (int)(u32tmp[1] & BIT(0)));
+	}
+	seq_printf(m, "\r\n %-35s = %s (BB:%s)/ %s (BB:%s)/ %s %d",
+		   "GNT_WL_Ctrl/GNT_BT_Ctrl/Dbg",
+		   ((u32tmp[0] & BIT(12)) ? "SW" : "HW"),
+		   ((u32tmp[0] & BIT(8)) ? "SW" : "HW"),
+		   ((u32tmp[0] & BIT(14)) ? "SW" : "HW"),
+		   ((u32tmp[0] & BIT(10)) ? "SW" : "HW"),
+		   ((u8tmp[0] & BIT(3)) ? "On" : "Off"),
+		   coex_sta->gnt_error_cnt);
+
+	seq_printf(m, "\r\n %-35s = %d/ %d", "GNT_WL/GNT_BT",
+		   (int)((u32tmp[1] & BIT(2)) >> 2),
+		   (int)((u32tmp[1] & BIT(3)) >> 3));
+
+	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0xcbc);
+	u32tmp[1] = btcoexist->btc_read_4byte(btcoexist, 0xcb4);
+	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0xcba);
+
+	seq_printf(m, "\r\n %-35s = 0x%04x/ 0x%04x/ 0x%02x %s",
+		   "0xcbc/0xcb4/0xcb8[23:16]", u32tmp[0], u32tmp[1], u8tmp[0],
+		   ((u8tmp[0] & 0x1) == 0x1 ? "(BTG)" : "(WL_A+G)"));
+
+	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x4c);
+	u8tmp[2] = btcoexist->btc_read_1byte(btcoexist, 0x64);
+	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x4c6);
+	u8tmp[1] = btcoexist->btc_read_1byte(btcoexist, 0x40);
+
+	seq_printf(m, "\r\n %-35s = 0x%x/ 0x%x/ 0x%x/ 0x%x",
+		   "4c[24:23]/64[0]/4c6[4]/40[5]",
+		   (int)(u32tmp[0] & (BIT(24) | BIT(23))) >> 23, u8tmp[2] & 0x1,
+		   (int)((u8tmp[0] & BIT(4)) >> 4),
+		   (int)((u8tmp[1] & BIT(5)) >> 5));
+
+	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x550);
+	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x522);
+	u8tmp[1] = btcoexist->btc_read_1byte(btcoexist, 0x953);
+	u8tmp[2] = btcoexist->btc_read_1byte(btcoexist, 0xc50);
+
+	seq_printf(m, "\r\n %-35s = 0x%x/ 0x%x/ %s/ 0x%x",
+		   "0x550/0x522/4-RxAGC/0xc50", u32tmp[0], u8tmp[0],
+		   (u8tmp[1] & 0x2) ? "On" : "Off", u8tmp[2]);
+
+	fa_ofdm = btcoexist->btc_phydm_query_phy_counter(btcoexist,
+							 "PHYDM_INFO_FA_OFDM");
+	fa_cck = btcoexist->btc_phydm_query_phy_counter(btcoexist,
+							"PHYDM_INFO_FA_CCK");
+	cca_ofdm = btcoexist->btc_phydm_query_phy_counter(
+		btcoexist, "PHYDM_INFO_CCA_OFDM");
+	cca_cck = btcoexist->btc_phydm_query_phy_counter(btcoexist,
+							 "PHYDM_INFO_CCA_CCK");
+
+	ratio_ofdm = (fa_ofdm == 0) ? 1000 : (cca_ofdm / fa_ofdm);
+
+	seq_printf(m, "\r\n %-35s = 0x%x/ 0x%x/ 0x%x/ 0x%x  (%d)",
+		   "CCK-CCA/CCK-FA/OFDM-CCA/OFDM-FA", cca_cck, fa_cck, cca_ofdm,
+		   fa_ofdm, ratio_ofdm);
+
+	seq_printf(m, "\r\n %-35s = %d/ %d/ %d/ %d", "CRC_OK CCK/11g/11n/11ac",
+		   coex_sta->crc_ok_cck, coex_sta->crc_ok_11g,
+		   coex_sta->crc_ok_11n, coex_sta->crc_ok_11n_vht);
+
+	seq_printf(m, "\r\n %-35s = %d/ %d/ %d/ %d  (%d, %d)",
+		   "CRC_Err CCK/11g/11n/11ac", coex_sta->crc_err_cck,
+		   coex_sta->crc_err_11g, coex_sta->crc_err_11n,
+		   coex_sta->crc_err_11n_vht, coex_sta->now_crc_ratio,
+		   coex_sta->acc_crc_ratio);
+
+	seq_printf(m, "\r\n %-35s = %s/ %s/ %s/ %d",
+		   "WlHiPri/ Locking/ Locked/ Noisy",
+		   (coex_sta->wifi_is_high_pri_task ? "Yes" : "No"),
+		   (coex_sta->cck_lock ? "Yes" : "No"),
+		   (coex_sta->cck_ever_lock ? "Yes" : "No"),
+		   coex_sta->wl_noisy_level);
+
+	seq_printf(m, "\r\n %-35s = %d/ %d", "0x770(Hi-pri rx/tx)",
+		   coex_sta->high_priority_rx, coex_sta->high_priority_tx);
+
+	seq_printf(m, "\r\n %-35s = %d/ %d %s", "0x774(Lo-pri rx/tx)",
+		   coex_sta->low_priority_rx, coex_sta->low_priority_tx,
+		   (bt_link_info->slave_role ?
+			    "(Slave!!)" :
+			    (coex_sta->is_tdma_btautoslot_hang ?
+				     "(auto-slot hang!!)" :
+				     "")));
+
+	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_COEX_STATISTICS, m);
+}
+
+void ex_btc8822b2ant_ips_notify(struct btc_coexist *btcoexist, u8 type)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	if (btcoexist->manual_control || btcoexist->stop_coex_dm)
+		return;
+
+	if (type == BTC_IPS_ENTER) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], IPS ENTER notify\n");
+		coex_sta->under_ips = true;
+		coex_sta->under_lps = false;
+
+		halbtc8822b2ant_post_state_to_bt(
+			btcoexist, BT_8822B_2ANT_SCOREBOARD_ACTIVE, false);
+
+		halbtc8822b2ant_post_state_to_bt(
+			btcoexist, BT_8822B_2ANT_SCOREBOARD_ONOFF, false);
+
+		halbtc8822b2ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO,
+					     FORCE_EXEC,
+					     BT_8822B_2ANT_PHASE_WLAN_OFF);
+
+		halbtc8822b2ant_action_coex_all_off(btcoexist);
+	} else if (type == BTC_IPS_LEAVE) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], IPS LEAVE notify\n");
+		coex_sta->under_ips = false;
+
+		halbtc8822b2ant_post_state_to_bt(
+			btcoexist, BT_8822B_2ANT_SCOREBOARD_ACTIVE, true);
+		halbtc8822b2ant_post_state_to_bt(
+			btcoexist, BT_8822B_2ANT_SCOREBOARD_ONOFF, true);
+		halbtc8822b2ant_init_hw_config(btcoexist, false);
+		halbtc8822b2ant_init_coex_dm(btcoexist);
+		halbtc8822b2ant_query_bt_info(btcoexist);
+	}
+}
+
+void ex_btc8822b2ant_lps_notify(struct btc_coexist *btcoexist, u8 type)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	static bool pre_force_lps_on;
+
+	if (btcoexist->manual_control || btcoexist->stop_coex_dm)
+		return;
+
+	if (type == BTC_LPS_ENABLE) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], LPS ENABLE notify\n");
+		coex_sta->under_lps = true;
+		coex_sta->under_ips = false;
+
+		if (coex_sta->force_lps_ctrl) { /* LPS No-32K */
+			/* Write WL "Active" in Score-board for PS-TDMA */
+			pre_force_lps_on = true;
+			halbtc8822b2ant_post_state_to_bt(
+				btcoexist, BT_8822B_2ANT_SCOREBOARD_ACTIVE,
+				true);
+
+		} else {
+			/* LPS-32K, need check if this h2c 0x71 can work??
+			 * (2015/08/28)
+			 */
+			/* Write WL "Non-Active" in Score-board for Native-PS */
+			pre_force_lps_on = false;
+			halbtc8822b2ant_post_state_to_bt(
+				btcoexist, BT_8822B_2ANT_SCOREBOARD_ACTIVE,
+				false);
+		}
+
+	} else if (type == BTC_LPS_DISABLE) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], LPS DISABLE notify\n");
+		coex_sta->under_lps = false;
+
+		halbtc8822b2ant_post_state_to_bt(
+			btcoexist, BT_8822B_2ANT_SCOREBOARD_ACTIVE, true);
+
+		if ((!pre_force_lps_on) && (!coex_sta->force_lps_ctrl))
+			halbtc8822b2ant_query_bt_info(btcoexist);
+	}
+}
+
+void ex_btc8822b2ant_scan_notify(struct btc_coexist *btcoexist, u8 type)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	bool wifi_connected = false;
+	bool wifi_under_5g = false;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], SCAN notify()\n");
+
+	halbtc8822b2ant_post_state_to_bt(btcoexist,
+					 BT_8822B_2ANT_SCOREBOARD_ACTIVE, true);
+
+	if (btcoexist->manual_control || btcoexist->stop_coex_dm)
+		return;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_CONNECTED,
+			   &wifi_connected);
+
+	/* this can't be removed for RF off_on event, or BT would dis-connect */
+	halbtc8822b2ant_query_bt_info(btcoexist);
+
+	if (type == BTC_SCAN_START) {
+		btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_5G,
+				   &wifi_under_5g);
+
+		if (wifi_under_5g) {
+			RT_TRACE(
+				rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				"[BTCoex], ********** SCAN START notify (5g)\n");
+
+			halbtc8822b2ant_action_wifi_under5g(btcoexist);
+			return;
+		}
+
+		coex_sta->wifi_is_high_pri_task = true;
+
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], ********** SCAN START notify (2g)\n");
+
+		halbtc8822b2ant_run_coexist_mechanism(btcoexist);
+
+		return;
+	}
+
+	if (type == BTC_SCAN_START_2G) {
+		if (!wifi_connected)
+			coex_sta->wifi_is_high_pri_task = true;
+
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], SCAN START notify (2G)\n");
+
+		halbtc8822b2ant_post_state_to_bt(
+			btcoexist, BT_8822B_2ANT_SCOREBOARD_SCAN, true);
+		halbtc8822b2ant_post_state_to_bt(
+			btcoexist, BT_8822B_2ANT_SCOREBOARD_ACTIVE, true);
+
+		halbtc8822b2ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO,
+					     FORCE_EXEC,
+					     BT_8822B_2ANT_PHASE_2G_RUNTIME);
+
+		halbtc8822b2ant_run_coexist_mechanism(btcoexist);
+
+	} else if (type == BTC_SCAN_FINISH) {
+		coex_sta->wifi_is_high_pri_task = false;
+
+		btcoexist->btc_get(btcoexist, BTC_GET_U1_AP_NUM,
+				   &coex_sta->scan_ap_num);
+
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], SCAN FINISH notify  (Scan-AP = %d)\n",
+			 coex_sta->scan_ap_num);
+
+		halbtc8822b2ant_post_state_to_bt(
+			btcoexist, BT_8822B_2ANT_SCOREBOARD_SCAN, false);
+
+		halbtc8822b2ant_run_coexist_mechanism(btcoexist);
+	}
+}
+
+void ex_btc8822b2ant_switchband_notify(struct btc_coexist *btcoexist, u8 type)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	if (btcoexist->manual_control || btcoexist->stop_coex_dm)
+		return;
+	coex_sta->switch_band_notify_to = type;
+
+	if (type == BTC_SWITCH_TO_5G) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], switchband_notify ---  switch to 5G\n");
+
+		halbtc8822b2ant_action_wifi_under5g(btcoexist);
+
+	} else if (type == BTC_SWITCH_TO_24G_NOFORSCAN) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], ********** switchband_notify BTC_SWITCH_TO_2G (no for scan)\n");
+
+		halbtc8822b2ant_run_coexist_mechanism(btcoexist);
+
+	} else {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], switchband_notify ---  switch to 2G\n");
+
+		ex_btc8822b2ant_scan_notify(btcoexist, BTC_SCAN_START_2G);
+	}
+	coex_sta->switch_band_notify_to = BTC_NOT_SWITCH;
+}
+
+void ex_btc8822b2ant_connect_notify(struct btc_coexist *btcoexist, u8 type)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	halbtc8822b2ant_post_state_to_bt(btcoexist,
+					 BT_8822B_2ANT_SCOREBOARD_ACTIVE, true);
+	if (btcoexist->manual_control || btcoexist->stop_coex_dm)
+		return;
+
+	if ((type == BTC_ASSOCIATE_5G_START) ||
+	    (type == BTC_ASSOCIATE_5G_FINISH)) {
+		if (type == BTC_ASSOCIATE_5G_START)
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], connect_notify ---  5G start\n");
+		else
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], connect_notify ---  5G finish\n");
+
+		halbtc8822b2ant_action_wifi_under5g(btcoexist);
+		return;
+	}
+
+	if (type == BTC_ASSOCIATE_START) {
+		coex_sta->wifi_is_high_pri_task = true;
+
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], CONNECT START notify (2G)\n");
+
+		halbtc8822b2ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO,
+					     FORCE_EXEC,
+					     BT_8822B_2ANT_PHASE_2G_RUNTIME);
+
+		halbtc8822b2ant_action_wifi_link_process(btcoexist);
+
+		/* To keep TDMA case during connect process,
+		 * to avoid changed by Btinfo and runcoexmechanism
+		 */
+		coex_sta->freeze_coexrun_by_btinfo = true;
+
+		coex_dm->arp_cnt = 0;
+
+	} else if (type == BTC_ASSOCIATE_FINISH) {
+		coex_sta->wifi_is_high_pri_task = false;
+		coex_sta->freeze_coexrun_by_btinfo = false;
+
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], CONNECT FINISH notify	(2G)\n");
+
+		halbtc8822b2ant_run_coexist_mechanism(btcoexist);
+	}
+}
+
+void ex_btc8822b2ant_media_status_notify(struct btc_coexist *btcoexist, u8 type)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	bool wifi_under_b_mode = false, wifi_under_5g = false;
+
+	if (btcoexist->manual_control || btcoexist->stop_coex_dm)
+		return;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_5G, &wifi_under_5g);
+
+	if (type == BTC_MEDIA_CONNECT) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], MEDIA connect notify\n");
+
+		halbtc8822b2ant_post_state_to_bt(
+			btcoexist, BT_8822B_2ANT_SCOREBOARD_ACTIVE, true);
+
+		if (wifi_under_5g) {
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], WiFi is under 5G!!!\n");
+
+			halbtc8822b2ant_action_wifi_under5g(btcoexist);
+			return;
+		}
+
+		halbtc8822b2ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO,
+					     FORCE_EXEC,
+					     BT_8822B_2ANT_PHASE_2G_RUNTIME);
+
+		btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_B_MODE,
+				   &wifi_under_b_mode);
+
+		/* Set CCK Tx/Rx high Pri except 11b mode */
+		if (wifi_under_b_mode) {
+			btcoexist->btc_write_1byte(btcoexist, 0x6cd,
+						   0x00); /* CCK Tx */
+			btcoexist->btc_write_1byte(btcoexist, 0x6cf,
+						   0x00); /* CCK Rx */
+		} else {
+			btcoexist->btc_write_1byte(btcoexist, 0x6cd,
+						   0x00); /* CCK Tx */
+			btcoexist->btc_write_1byte(btcoexist, 0x6cf,
+						   0x10); /* CCK Rx */
+		}
+
+	} else {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], MEDIA disconnect notify\n");
+
+		btcoexist->btc_write_1byte(btcoexist, 0x6cd, 0x0); /* CCK Tx */
+		btcoexist->btc_write_1byte(btcoexist, 0x6cf, 0x0); /* CCK Rx */
+
+		halbtc8822b2ant_post_state_to_bt(
+			btcoexist, BT_8822B_2ANT_SCOREBOARD_ACTIVE, false);
+	}
+
+	halbtc8822b2ant_update_wifi_ch_info(btcoexist, type);
+}
+
+void ex_btc8822b2ant_specific_packet_notify(struct btc_coexist *btcoexist,
+					    u8 type)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	bool under_4way = false, wifi_under_5g = false;
+
+	if (btcoexist->manual_control || btcoexist->stop_coex_dm)
+		return;
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_5G, &wifi_under_5g);
+
+	if (wifi_under_5g) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], WiFi is under 5G!!!\n");
+
+		halbtc8822b2ant_action_wifi_under5g(btcoexist);
+		return;
+	}
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_4_WAY_PROGRESS,
+			   &under_4way);
+
+	if (under_4way) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], specific Packet ---- under_4way!!\n");
+
+		coex_sta->wifi_is_high_pri_task = true;
+		coex_sta->specific_pkt_period_cnt = 2;
+
+	} else if (type == BTC_PACKET_ARP) {
+		coex_dm->arp_cnt++;
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], specific Packet ARP notify -cnt = %d\n",
+			 coex_dm->arp_cnt);
+
+	} else {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], specific Packet DHCP or EAPOL notify [Type = %d]\n",
+			type);
+
+		coex_sta->wifi_is_high_pri_task = true;
+		coex_sta->specific_pkt_period_cnt = 2;
+	}
+
+	if (coex_sta->wifi_is_high_pri_task)
+		halbtc8822b2ant_run_coexist_mechanism(btcoexist);
+}
+
+void ex_btc8822b2ant_bt_info_notify(struct btc_coexist *btcoexist, u8 *tmp_buf,
+				    u8 length)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	u8 i, rsp_source = 0;
+	bool wifi_connected = false;
+	bool wifi_scan = false, wifi_link = false, wifi_roam = false,
+	     wifi_busy = false;
+	static bool is_scoreboard_scan;
+
+	rsp_source = tmp_buf[0] & 0xf;
+	if (rsp_source >= BT_INFO_SRC_8822B_2ANT_MAX)
+		rsp_source = BT_INFO_SRC_8822B_2ANT_WIFI_FW;
+	coex_sta->bt_info_c2h_cnt[rsp_source]++;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], Bt_info[%d], len=%d, data=[", rsp_source, length);
+
+	for (i = 0; i < length; i++) {
+		coex_sta->bt_info_c2h[rsp_source][i] = tmp_buf[i];
+
+		if (i == length - 1) {
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "0x%02x]\n", tmp_buf[i]);
+		} else {
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD, "0x%02x, ",
+				 tmp_buf[i]);
+		}
+	}
+
+	coex_sta->bt_info = coex_sta->bt_info_c2h[rsp_source][1];
+	coex_sta->bt_info_ext = coex_sta->bt_info_c2h[rsp_source][4];
+	coex_sta->bt_info_ext2 = coex_sta->bt_info_c2h[rsp_source][5];
+
+	if (rsp_source != BT_INFO_SRC_8822B_2ANT_WIFI_FW) {
+		/* if 0xff, it means BT is under WHCK test */
+		coex_sta->bt_whck_test =
+			((coex_sta->bt_info == 0xff) ? true : false);
+
+		coex_sta->bt_create_connection =
+			((coex_sta->bt_info_c2h[rsp_source][2] & 0x80) ? true :
+									 false);
+
+		/* unit: %, value-100 to translate to unit: dBm */
+		coex_sta->bt_rssi =
+			coex_sta->bt_info_c2h[rsp_source][3] * 2 + 10;
+
+		coex_sta->c2h_bt_remote_name_req =
+			((coex_sta->bt_info_c2h[rsp_source][2] & 0x20) ? true :
+									 false);
+
+		coex_sta->is_A2DP_3M =
+			((coex_sta->bt_info_c2h[rsp_source][2] & 0x10) ? true :
+									 false);
+
+		coex_sta->acl_busy =
+			((coex_sta->bt_info_c2h[rsp_source][1] & 0x9) ? true :
+									false);
+
+		coex_sta->voice_over_HOGP =
+			((coex_sta->bt_info_ext & 0x10) ? true : false);
+
+		coex_sta->c2h_bt_inquiry_page =
+			((coex_sta->bt_info & BT_INFO_8822B_2ANT_B_INQ_PAGE) ?
+				 true :
+				 false);
+
+		coex_sta->a2dp_bit_pool =
+			(((coex_sta->bt_info_c2h[rsp_source][1] & 0x49) ==
+			  0x49) ?
+				 (coex_sta->bt_info_c2h[rsp_source][6] & 0x7f) :
+				 0);
+
+		coex_sta->is_bt_a2dp_sink =
+			(coex_sta->bt_info_c2h[rsp_source][6] & 0x80) ? true :
+									false;
+
+		coex_sta->bt_retry_cnt =
+			coex_sta->bt_info_c2h[rsp_source][2] & 0xf;
+
+		coex_sta->is_autoslot = coex_sta->bt_info_ext2 & 0x8;
+
+		coex_sta->forbidden_slot = coex_sta->bt_info_ext2 & 0x7;
+
+		coex_sta->hid_busy_num = (coex_sta->bt_info_ext2 & 0x30) >> 4;
+
+		coex_sta->hid_pair_cnt = (coex_sta->bt_info_ext2 & 0xc0) >> 6;
+
+		if (coex_sta->bt_retry_cnt >= 1)
+			coex_sta->pop_event_cnt++;
+
+		if (coex_sta->c2h_bt_remote_name_req)
+			coex_sta->cnt_remote_name_req++;
+
+		if (coex_sta->bt_info_ext & BIT(1))
+			coex_sta->cnt_reinit++;
+
+		if (coex_sta->bt_info_ext & BIT(2)) {
+			coex_sta->cnt_setup_link++;
+			coex_sta->is_setup_link = true;
+			coex_sta->bt_relink_downcount = 2;
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], Re-Link start in BT info!!\n");
+		} else {
+			coex_sta->is_setup_link = false;
+			coex_sta->bt_relink_downcount = 0;
+			RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+				 "[BTCoex], Re-Link stop in BT info!!\n");
+		}
+
+		if (coex_sta->bt_info_ext & BIT(3))
+			coex_sta->cnt_ign_wlan_act++;
+
+		if (coex_sta->bt_info_ext & BIT(6))
+			coex_sta->cnt_role_switch++;
+
+		if (coex_sta->bt_info_ext & BIT(7))
+			coex_sta->is_bt_multi_link = true;
+		else
+			coex_sta->is_bt_multi_link = false;
+
+		if (coex_sta->bt_create_connection) {
+			coex_sta->cnt_page++;
+
+			btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY,
+					   &wifi_busy);
+
+			btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_SCAN,
+					   &wifi_scan);
+			btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_LINK,
+					   &wifi_link);
+			btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_ROAM,
+					   &wifi_roam);
+
+			if ((wifi_link) || (wifi_roam) || (wifi_scan) ||
+			    (coex_sta->wifi_is_high_pri_task) || (wifi_busy)) {
+				is_scoreboard_scan = true;
+				halbtc8822b2ant_post_state_to_bt(
+					btcoexist,
+					BT_8822B_2ANT_SCOREBOARD_SCAN, true);
+
+			} else {
+				halbtc8822b2ant_post_state_to_bt(
+					btcoexist,
+					BT_8822B_2ANT_SCOREBOARD_SCAN, false);
+			}
+		} else {
+			if (is_scoreboard_scan) {
+				halbtc8822b2ant_post_state_to_bt(
+					btcoexist,
+					BT_8822B_2ANT_SCOREBOARD_SCAN, false);
+				is_scoreboard_scan = false;
+			}
+		}
+
+		/* Here we need to resend some wifi info to BT */
+		/* because bt is reset and loss of the info. */
+
+		if ((!btcoexist->manual_control) &&
+		    (!btcoexist->stop_coex_dm)) {
+			btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_CONNECTED,
+					   &wifi_connected);
+
+			/*  Re-Init */
+			if ((coex_sta->bt_info_ext & BIT(1))) {
+				RT_TRACE(
+					rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+					"[BTCoex], BT ext info bit1 check, send wifi BW&Chnl to BT!!\n");
+				if (wifi_connected)
+					halbtc8822b2ant_update_wifi_ch_info(
+						btcoexist, BTC_MEDIA_CONNECT);
+				else
+					halbtc8822b2ant_update_wifi_ch_info(
+						btcoexist,
+						BTC_MEDIA_DISCONNECT);
+			}
+
+			/*  If Ignore_WLanAct && not SetUp_Link */
+			if ((coex_sta->bt_info_ext & BIT(3)) &&
+			    (!(coex_sta->bt_info_ext & BIT(2))) &&
+			    (!(coex_sta->bt_info_ext & BIT(6)))) {
+				RT_TRACE(
+					rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+					"[BTCoex], BT ext info bit3 check, set BT NOT to ignore Wlan active!!\n");
+				halbtc8822b2ant_ignore_wlan_act(
+					btcoexist, FORCE_EXEC, false);
+			} else {
+				if (coex_sta->bt_info_ext & BIT(2)) {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], BT ignore Wlan active because Re-link!!\n");
+				} else if (coex_sta->bt_info_ext & BIT(6)) {
+					RT_TRACE(
+						rtlpriv, COMP_BT_COEXIST,
+						DBG_LOUD,
+						"[BTCoex], BT ignore Wlan active because Role-Switch!!\n");
+				}
+			}
+		}
+	}
+
+	if ((coex_sta->bt_info_ext & BIT(5))) {
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], BT ext info bit4 check, query BLE Scan type!!\n");
+		coex_sta->bt_ble_scan_type =
+			btcoexist->btc_get_ble_scan_type_from_bt(btcoexist);
+
+		if ((coex_sta->bt_ble_scan_type & 0x1) == 0x1)
+			coex_sta->bt_ble_scan_para[0] =
+				btcoexist->btc_get_ble_scan_para_from_bt(
+					btcoexist, 0x1);
+		if ((coex_sta->bt_ble_scan_type & 0x2) == 0x2)
+			coex_sta->bt_ble_scan_para[1] =
+				btcoexist->btc_get_ble_scan_para_from_bt(
+					btcoexist, 0x2);
+		if ((coex_sta->bt_ble_scan_type & 0x4) == 0x4)
+			coex_sta->bt_ble_scan_para[2] =
+				btcoexist->btc_get_ble_scan_para_from_bt(
+					btcoexist, 0x4);
+	}
+
+	halbtc8822b2ant_update_bt_link_info(btcoexist);
+
+	halbtc8822b2ant_run_coexist_mechanism(btcoexist);
+}
+
+void ex_btc8822b2ant_rf_status_notify(struct btc_coexist *btcoexist, u8 type)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], RF Status notify\n");
+
+	if (type == BTC_RF_ON) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], RF is turned ON!!\n");
+
+		btcoexist->stop_coex_dm = false;
+		coex_sta->is_rf_state_off = false;
+	} else if (type == BTC_RF_OFF) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], RF is turned OFF!!\n");
+
+		halbtc8822b2ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO,
+					     FORCE_EXEC,
+					     BT_8822B_2ANT_PHASE_WLAN_OFF);
+
+		halbtc8822b2ant_action_coex_all_off(btcoexist);
+
+		halbtc8822b2ant_post_state_to_bt(
+			btcoexist, BT_8822B_2ANT_SCOREBOARD_ACTIVE |
+					   BT_8822B_2ANT_SCOREBOARD_ONOFF |
+					   BT_8822B_2ANT_SCOREBOARD_SCAN |
+					   BT_8822B_2ANT_SCOREBOARD_UNDERTEST,
+			false);
+
+		btcoexist->stop_coex_dm = true;
+		coex_sta->is_rf_state_off = true;
+	}
+}
+
+void ex_btc8822b2ant_halt_notify(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD, "[BTCoex], Halt notify\n");
+
+	halbtc8822b2ant_set_ant_path(btcoexist, BTC_ANT_PATH_AUTO, FORCE_EXEC,
+				     BT_8822B_2ANT_PHASE_WLAN_OFF);
+
+	ex_btc8822b2ant_media_status_notify(btcoexist, BTC_MEDIA_DISCONNECT);
+
+	halbtc8822b2ant_post_state_to_bt(
+		btcoexist, BT_8822B_2ANT_SCOREBOARD_ACTIVE, false);
+	halbtc8822b2ant_post_state_to_bt(btcoexist,
+					 BT_8822B_2ANT_SCOREBOARD_ONOFF, false);
+}
+
+void ex_btc8822b2ant_pnp_notify(struct btc_coexist *btcoexist, u8 pnp_state)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	bool wifi_under_5g = false;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD, "[BTCoex], Pnp notify\n");
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_5G, &wifi_under_5g);
+
+	if ((pnp_state == BTC_WIFI_PNP_SLEEP) ||
+	    (pnp_state == BTC_WIFI_PNP_SLEEP_KEEP_ANT)) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], Pnp notify to SLEEP\n");
+
+		/* Sinda 20150819, workaround for driver skip leave IPS/LPS to
+		 * speed up sleep time.
+		 * Driver do not leave IPS/LPS when driver is going to sleep,
+		 * so BTCoexistence think wifi is still under IPS/LPS.
+		 * BT should clear UnderIPS/UnderLPS state to avoid mismatch
+		 * state after wakeup.
+		 */
+		coex_sta->under_ips = false;
+		coex_sta->under_lps = false;
+
+		halbtc8822b2ant_post_state_to_bt(
+			btcoexist, BT_8822B_2ANT_SCOREBOARD_ACTIVE, false);
+		halbtc8822b2ant_post_state_to_bt(
+			btcoexist, BT_8822B_2ANT_SCOREBOARD_ONOFF, false);
+
+		if (pnp_state == BTC_WIFI_PNP_SLEEP_KEEP_ANT) {
+			if (wifi_under_5g)
+				halbtc8822b2ant_set_ant_path(
+					btcoexist, BTC_ANT_PATH_AUTO,
+					FORCE_EXEC,
+					BT_8822B_2ANT_PHASE_5G_RUNTIME);
+			else
+				halbtc8822b2ant_set_ant_path(
+					btcoexist, BTC_ANT_PATH_AUTO,
+					FORCE_EXEC,
+					BT_8822B_2ANT_PHASE_2G_RUNTIME);
+		} else {
+			halbtc8822b2ant_set_ant_path(
+				btcoexist, BTC_ANT_PATH_AUTO, FORCE_EXEC,
+				BT_8822B_2ANT_PHASE_WLAN_OFF);
+		}
+	} else if (pnp_state == BTC_WIFI_PNP_WAKE_UP) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], Pnp notify to WAKE UP\n");
+
+		halbtc8822b2ant_post_state_to_bt(
+			btcoexist, BT_8822B_2ANT_SCOREBOARD_ACTIVE, true);
+		halbtc8822b2ant_post_state_to_bt(
+			btcoexist, BT_8822B_2ANT_SCOREBOARD_ONOFF, true);
+	}
+}
+
+void ex_btc8822b2ant_periodical(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	bool wifi_busy = false;
+	u16 bt_scoreboard_val = 0;
+	bool bt_relink_finish = false;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "[BTCoex], ************* Periodical *************\n");
+
+	if (!btcoexist->auto_report_2ant)
+		halbtc8822b2ant_query_bt_info(btcoexist);
+
+	halbtc8822b2ant_monitor_bt_ctr(btcoexist);
+	halbtc8822b2ant_monitor_wifi_ctr(btcoexist);
+	halbtc8822b2ant_monitor_bt_enable_disable(btcoexist);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+	halbtc8822b2ant_read_score_board(btcoexist, &bt_scoreboard_val);
+
+	if (wifi_busy) {
+		halbtc8822b2ant_post_state_to_bt(
+			btcoexist, BT_8822B_2ANT_SCOREBOARD_UNDERTEST, true);
+		/*for bt lps32 clock offset*/
+		if (bt_scoreboard_val & BIT(6))
+			halbtc8822b2ant_query_bt_info(btcoexist);
+	} else {
+		halbtc8822b2ant_post_state_to_bt(
+			btcoexist, BT_8822B_2ANT_SCOREBOARD_UNDERTEST, false);
+	}
+
+	if (coex_sta->bt_relink_downcount != 0) {
+		coex_sta->bt_relink_downcount--;
+
+		if (coex_sta->bt_relink_downcount == 0) {
+			coex_sta->is_setup_link = false;
+			bt_relink_finish = true;
+		}
+	}
+
+	/* for 4-way, DHCP, EAPOL packet */
+	if (coex_sta->specific_pkt_period_cnt > 0) {
+		coex_sta->specific_pkt_period_cnt--;
+
+		if ((coex_sta->specific_pkt_period_cnt == 0) &&
+		    (coex_sta->wifi_is_high_pri_task))
+			coex_sta->wifi_is_high_pri_task = false;
+
+		RT_TRACE(
+			rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			"[BTCoex], ***************** Hi-Pri Task = %s*****************\n",
+			(coex_sta->wifi_is_high_pri_task ? "Yes" : "No"));
+	}
+
+	if (halbtc8822b2ant_is_wifibt_status_changed(btcoexist) ||
+	    (bt_relink_finish) || (coex_sta->is_set_ps_state_fail))
+		halbtc8822b2ant_run_coexist_mechanism(btcoexist);
+}
+
+void ex_btc8822b2ant_antenna_detection(struct btc_coexist *btcoexist,
+				       u32 cent_freq, u32 offset, u32 span,
+				       u32 seconds)
+{
+}
+
+void ex_btc8822b2ant_display_ant_detection(struct btc_coexist *btcoexist) {}
diff --git a/drivers/staging/rtlwifi/btcoexist/halbtc8822b2ant.h b/drivers/staging/rtlwifi/btcoexist/halbtc8822b2ant.h
new file mode 100644
index 000000000000..212e0c8404fa
--- /dev/null
+++ b/drivers/staging/rtlwifi/btcoexist/halbtc8822b2ant.h
@@ -0,0 +1,498 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+/* *******************************************
+ * The following is for 8822B 2Ant BT Co-exist definition
+ * ********************************************/
+#define BT_INFO_8822B_2ANT_B_FTP	BIT(7)
+#define BT_INFO_8822B_2ANT_B_A2DP	BIT(6)
+#define BT_INFO_8822B_2ANT_B_HID	BIT(5)
+#define BT_INFO_8822B_2ANT_B_SCO_BUSY	BIT(4)
+#define BT_INFO_8822B_2ANT_B_ACL_BUSY	BIT(3)
+#define BT_INFO_8822B_2ANT_B_INQ_PAGE	BIT(2)
+#define BT_INFO_8822B_2ANT_B_SCO_ESCO	BIT(1)
+#define BT_INFO_8822B_2ANT_B_CONNECTION	BIT(0)
+
+#define BTC_RSSI_COEX_THRESH_TOL_8822B_2ANT	2
+
+/* unit: % WiFi RSSI Threshold for 2-Ant free-run/2-Ant TDMA translation.
+ * (default = 42)
+ */
+#define BT_8822B_2ANT_WIFI_RSSI_COEXSWITCH_THRES1	80
+/* unit: % BT RSSI Threshold for 2-Ant free-run/2-Ant TDMA translation.
+ * (default = 46)
+ */
+#define BT_8822B_2ANT_BT_RSSI_COEXSWITCH_THRES1	80
+/* unit: % WiFi RSSI Threshold for 1-Ant TDMA/1-Ant PS-TDMA translation.
+ * (default = 42)
+ */
+#define BT_8822B_2ANT_WIFI_RSSI_COEXSWITCH_THRES2	80
+/* unit: % BT RSSI Threshold for 1-Ant TDMA/1-Ant PS-TDMA translation.
+ * (default = 46)
+ */
+#define BT_8822B_2ANT_BT_RSSI_COEXSWITCH_THRES2	80
+#define BT_8822B_2ANT_DEFAULT_ISOLATION	15 /*  unit: dB */
+#define BT_8822B_2ANT_WIFI_MAX_TX_POWER	15 /*  unit: dBm */
+#define BT_8822B_2ANT_BT_MAX_TX_POWER	3 /*  unit: dBm */
+#define BT_8822B_2ANT_WIFI_SIR_THRES1 -15 /*  unit: dB */
+#define BT_8822B_2ANT_WIFI_SIR_THRES2 -30 /*  unit: dB */
+#define BT_8822B_2ANT_BT_SIR_THRES1 -15 /*  unit: dB */
+#define BT_8822B_2ANT_BT_SIR_THRES2 -30 /*  unit: dB */
+
+/* for Antenna detection */
+#define BT_8822B_2ANT_ANTDET_PSDTHRES_BACKGROUND	50
+#define BT_8822B_2ANT_ANTDET_PSDTHRES_2ANT_BADISOLATION	70
+#define BT_8822B_2ANT_ANTDET_PSDTHRES_2ANT_GOODISOLATION	52
+#define BT_8822B_2ANT_ANTDET_PSDTHRES_1ANT	40
+#define BT_8822B_2ANT_ANTDET_RETRY_INTERVAL                                    \
+	10 /* retry timer if ant det is fail, unit: second */
+#define BT_8822B_2ANT_ANTDET_SWEEPPOINT_DELAY	60000
+#define BT_8822B_2ANT_ANTDET_ENABLE	0
+#define BT_8822B_2ANT_ANTDET_BTTXTIME	100
+#define BT_8822B_2ANT_ANTDET_BTTXCHANNEL	39
+#define BT_8822B_2ANT_ANTDET_PSD_SWWEEPCOUNT	50
+
+#define BT_8822B_2ANT_LTECOEX_INDIRECTREG_ACCESS_TIMEOUT	30000
+
+enum bt_8822b_2ant_signal_state {
+	BT_8822B_2ANT_SIG_STA_SET_TO_LOW	= 0x0,
+	BT_8822B_2ANT_SIG_STA_SET_BY_HW	= 0x0,
+	BT_8822B_2ANT_SIG_STA_SET_TO_HIGH	= 0x1,
+	BT_8822B_2ANT_SIG_STA_MAX
+};
+
+enum bt_8822b_2ant_path_ctrl_owner {
+	BT_8822B_2ANT_PCO_BTSIDE	= 0x0,
+	BT_8822B_2ANT_PCO_WLSIDE	= 0x1,
+	BT_8822B_2ANT_PCO_MAX
+};
+
+enum bt_8822b_2ant_gnt_ctrl_type {
+	BT_8822B_2ANT_GNT_TYPE_CTRL_BY_PTA	= 0x0,
+	BT_8822B_2ANT_GNT_TYPE_CTRL_BY_SW	= 0x1,
+	BT_8822B_2ANT_GNT_TYPE_MAX
+};
+
+enum bt_8822b_2ant_gnt_ctrl_block {
+	BT_8822B_2ANT_GNT_BLOCK_RFC_BB	= 0x0,
+	BT_8822B_2ANT_GNT_BLOCK_RFC	= 0x1,
+	BT_8822B_2ANT_GNT_BLOCK_BB	= 0x2,
+	BT_8822B_2ANT_GNT_BLOCK_MAX
+};
+
+enum bt_8822b_2ant_lte_coex_table_type {
+	BT_8822B_2ANT_CTT_WL_VS_LTE	= 0x0,
+	BT_8822B_2ANT_CTT_BT_VS_LTE	= 0x1,
+	BT_8822B_2ANT_CTT_MAX
+};
+
+enum bt_8822b_2ant_lte_break_table_type {
+	BT_8822B_2ANT_LBTT_WL_BREAK_LTE	= 0x0,
+	BT_8822B_2ANT_LBTT_BT_BREAK_LTE	= 0x1,
+	BT_8822B_2ANT_LBTT_LTE_BREAK_WL	= 0x2,
+	BT_8822B_2ANT_LBTT_LTE_BREAK_BT	= 0x3,
+	BT_8822B_2ANT_LBTT_MAX
+};
+
+enum bt_info_src_8822b_2ant {
+	BT_INFO_SRC_8822B_2ANT_WIFI_FW	= 0x0,
+	BT_INFO_SRC_8822B_2ANT_BT_RSP	= 0x1,
+	BT_INFO_SRC_8822B_2ANT_BT_ACTIVE_SEND	= 0x2,
+	BT_INFO_SRC_8822B_2ANT_MAX
+};
+
+enum bt_8822b_2ant_bt_status {
+	BT_8822B_2ANT_BT_STATUS_NON_CONNECTED_IDLE	= 0x0,
+	BT_8822B_2ANT_BT_STATUS_CONNECTED_IDLE	= 0x1,
+	BT_8822B_2ANT_BT_STATUS_INQ_PAGE	= 0x2,
+	BT_8822B_2ANT_BT_STATUS_ACL_BUSY	= 0x3,
+	BT_8822B_2ANT_BT_STATUS_SCO_BUSY	= 0x4,
+	BT_8822B_2ANT_BT_STATUS_ACL_SCO_BUSY	= 0x5,
+	BT_8822B_2ANT_BT_STATUS_MAX
+};
+
+enum bt_8822b_2ant_coex_algo {
+	BT_8822B_2ANT_COEX_ALGO_UNDEFINED	= 0x0,
+	BT_8822B_2ANT_COEX_ALGO_SCO	= 0x1,
+	BT_8822B_2ANT_COEX_ALGO_HID	= 0x2,
+	BT_8822B_2ANT_COEX_ALGO_A2DP	= 0x3,
+	BT_8822B_2ANT_COEX_ALGO_A2DP_PANHS	= 0x4,
+	BT_8822B_2ANT_COEX_ALGO_PANEDR	= 0x5,
+	BT_8822B_2ANT_COEX_ALGO_PANHS	= 0x6,
+	BT_8822B_2ANT_COEX_ALGO_PANEDR_A2DP	= 0x7,
+	BT_8822B_2ANT_COEX_ALGO_PANEDR_HID	= 0x8,
+	BT_8822B_2ANT_COEX_ALGO_HID_A2DP_PANEDR	= 0x9,
+	BT_8822B_2ANT_COEX_ALGO_HID_A2DP	= 0xa,
+	BT_8822B_2ANT_COEX_ALGO_NOPROFILEBUSY	= 0xb,
+	BT_8822B_2ANT_COEX_ALGO_A2DPSINK	= 0xc,
+	BT_8822B_2ANT_COEX_ALGO_MAX
+};
+
+enum bt_8822b_2ant_ext_ant_switch_type {
+	BT_8822B_2ANT_EXT_ANT_SWITCH_USE_DPDT	= 0x0,
+	BT_8822B_2ANT_EXT_ANT_SWITCH_USE_SPDT	= 0x1,
+	BT_8822B_2ANT_EXT_ANT_SWITCH_NONE	= 0x2,
+	BT_8822B_2ANT_EXT_ANT_SWITCH_MAX
+};
+
+enum bt_8822b_2ant_ext_ant_switch_ctrl_type {
+	BT_8822B_2ANT_EXT_ANT_SWITCH_CTRL_BY_BBSW	= 0x0,
+	BT_8822B_2ANT_EXT_ANT_SWITCH_CTRL_BY_PTA	= 0x1,
+	BT_8822B_2ANT_EXT_ANT_SWITCH_CTRL_BY_ANTDIV	= 0x2,
+	BT_8822B_2ANT_EXT_ANT_SWITCH_CTRL_BY_MAC	= 0x3,
+	BT_8822B_2ANT_EXT_ANT_SWITCH_CTRL_BY_BT	= 0x4,
+	BT_8822B_2ANT_EXT_ANT_SWITCH_CTRL_MAX
+};
+
+enum bt_8822b_2ant_ext_ant_switch_pos_type {
+	BT_8822B_2ANT_EXT_ANT_SWITCH_MAIN_TO_BT	= 0x0,
+	BT_8822B_2ANT_EXT_ANT_SWITCH_MAIN_TO_WLG	= 0x1,
+	BT_8822B_2ANT_EXT_ANT_SWITCH_MAIN_TO_WLA	= 0x2,
+	BT_8822B_2ANT_EXT_ANT_SWITCH_MAIN_TO_NOCARE	= 0x3,
+	BT_8822B_2ANT_EXT_ANT_SWITCH_MAIN_TO_MAX
+};
+
+enum bt_8822b_2ant_ext_band_switch_pos_type {
+	BT_8822B_2ANT_EXT_BAND_SWITCH_TO_WLG	= 0x0,
+	BT_8822B_2ANT_EXT_BAND_SWITCH_TO_WLA	= 0x1,
+	BT_8822B_2ANT_EXT_BAND_SWITCH_TO_MAX
+};
+
+enum bt_8822b_2ant_int_block {
+	BT_8822B_2ANT_INT_BLOCK_SWITCH_TO_WLG_OF_BTG	= 0x0,
+	BT_8822B_2ANT_INT_BLOCK_SWITCH_TO_WLG_OF_WLAG	= 0x1,
+	BT_8822B_2ANT_INT_BLOCK_SWITCH_TO_WLA_OF_WLAG	= 0x2,
+	BT_8822B_2ANT_INT_BLOCK_SWITCH_TO_MAX
+};
+
+enum bt_8822b_2ant_phase {
+	BT_8822B_2ANT_PHASE_COEX_INIT	= 0x0,
+	BT_8822B_2ANT_PHASE_WLANONLY_INIT	= 0x1,
+	BT_8822B_2ANT_PHASE_WLAN_OFF	= 0x2,
+	BT_8822B_2ANT_PHASE_2G_RUNTIME	= 0x3,
+	BT_8822B_2ANT_PHASE_5G_RUNTIME	= 0x4,
+	BT_8822B_2ANT_PHASE_BTMPMODE	= 0x5,
+	BT_8822B_2ANT_PHASE_ANTENNA_DET	= 0x6,
+	BT_8822B_2ANT_PHASE_COEX_POWERON	= 0x7,
+	BT_8822B_2ANT_PHASE_2G_RUNTIME_CONCURRENT	= 0x8,
+	BT_8822B_2ANT_PHASE_MAX
+};
+
+/*ADD SCOREBOARD TO FIX BT LPS 32K ISSUE WHILE WL BUSY*/
+
+enum bt_8822b_2ant_scoreboard {
+	BT_8822B_2ANT_SCOREBOARD_ACTIVE	= BIT(0),
+	BT_8822B_2ANT_SCOREBOARD_ONOFF	= BIT(1),
+	BT_8822B_2ANT_SCOREBOARD_SCAN	= BIT(2),
+	BT_8822B_2ANT_SCOREBOARD_UNDERTEST	= BIT(3),
+	BT_8822B_2ANT_SCOREBOARD_WLBUSY = BIT(6)
+};
+
+struct coex_dm_8822b_2ant {
+	/* hw setting */
+	u32	pre_ant_pos_type;
+	u32	cur_ant_pos_type;
+	/* fw mechanism */
+	u8	pre_bt_dec_pwr_lvl;
+	u8	cur_bt_dec_pwr_lvl;
+	u8	pre_fw_dac_swing_lvl;
+	u8	cur_fw_dac_swing_lvl;
+	bool	cur_ignore_wlan_act;
+	bool	pre_ignore_wlan_act;
+	u8	pre_ps_tdma;
+	u8	cur_ps_tdma;
+	u8	ps_tdma_para[5];
+	u8	ps_tdma_du_adj_type;
+	bool	reset_tdma_adjust;
+	bool	pre_ps_tdma_on;
+	bool	cur_ps_tdma_on;
+	bool	pre_bt_auto_report;
+	bool	cur_bt_auto_report;
+
+	/* sw mechanism */
+	bool	pre_rf_rx_lpf_shrink;
+	bool	cur_rf_rx_lpf_shrink;
+	u32	bt_rf_0x1e_backup;
+	bool	pre_low_penalty_ra;
+	bool	cur_low_penalty_ra;
+	bool	pre_dac_swing_on;
+	u32	pre_dac_swing_lvl;
+	bool	cur_dac_swing_on;
+	u32	cur_dac_swing_lvl;
+	bool	pre_adc_back_off;
+	bool	cur_adc_back_off;
+	bool	pre_agc_table_en;
+	bool	cur_agc_table_en;
+	u32	pre_val0x6c0;
+	u32	cur_val0x6c0;
+	u32	pre_val0x6c4;
+	u32	cur_val0x6c4;
+	u32	pre_val0x6c8;
+	u32	cur_val0x6c8;
+	u8	pre_val0x6cc;
+	u8	cur_val0x6cc;
+	bool	limited_dig;
+
+	/* algorithm related */
+	u8	pre_algorithm;
+	u8	cur_algorithm;
+	u8	bt_status;
+	u8	wifi_chnl_info[3];
+
+	bool	need_recover0x948;
+	u32	backup0x948;
+
+	u8	pre_lps;
+	u8	cur_lps;
+	u8	pre_rpwm;
+	u8	cur_rpwm;
+
+	bool	is_switch_to_1dot5_ant;
+	u8	switch_thres_offset;
+	u32	arp_cnt;
+
+	u32	pre_ext_ant_switch_status;
+	u32	cur_ext_ant_switch_status;
+
+	u8	pre_ext_band_switch_status;
+	u8	cur_ext_band_switch_status;
+
+	u8	pre_int_block_status;
+	u8	cur_int_block_status;
+};
+
+struct coex_sta_8822b_2ant {
+	bool	bt_disabled;
+	bool	bt_link_exist;
+	bool	sco_exist;
+	bool	a2dp_exist;
+	bool	hid_exist;
+	bool	pan_exist;
+
+	bool	under_lps;
+	bool	under_ips;
+	u32	high_priority_tx;
+	u32	high_priority_rx;
+	u32	low_priority_tx;
+	u32	low_priority_rx;
+	bool	is_hi_pri_rx_overhead;
+	u8	bt_rssi;
+	u8	pre_bt_rssi_state;
+	u8	pre_wifi_rssi_state[4];
+	u8	bt_info_c2h[BT_INFO_SRC_8822B_2ANT_MAX][10];
+	u32	bt_info_c2h_cnt[BT_INFO_SRC_8822B_2ANT_MAX];
+	bool	bt_whck_test;
+	bool	c2h_bt_inquiry_page;
+	bool	c2h_bt_remote_name_req;
+
+	u8	bt_info_ext;
+	u8	bt_info_ext2;
+	u32	pop_event_cnt;
+	u8	scan_ap_num;
+	u8	bt_retry_cnt;
+
+	u32	crc_ok_cck;
+	u32	crc_ok_11g;
+	u32	crc_ok_11n;
+	u32	crc_ok_11n_vht;
+
+	u32	crc_err_cck;
+	u32	crc_err_11g;
+	u32	crc_err_11n;
+	u32	crc_err_11n_vht;
+
+	u32	acc_crc_ratio;
+	u32	now_crc_ratio;
+
+	bool	cck_lock;
+	bool	pre_ccklock;
+	bool	cck_ever_lock;
+
+	u8	coex_table_type;
+	bool	force_lps_ctrl;
+
+	u8	dis_ver_info_cnt;
+
+	u8	a2dp_bit_pool;
+	u8	cut_version;
+
+	bool	concurrent_rx_mode_on;
+
+	u16	score_board;
+	u8	isolation_btween_wb; /* 0~ 50 */
+	u8	wifi_coex_thres;
+	u8	bt_coex_thres;
+	u8	wifi_coex_thres2;
+	u8	bt_coex_thres2;
+
+	u8	num_of_profile;
+	bool	acl_busy;
+	bool	bt_create_connection;
+	bool	wifi_is_high_pri_task;
+	u32	specific_pkt_period_cnt;
+	u32	bt_coex_supported_feature;
+	u32	bt_coex_supported_version;
+
+	u8	bt_ble_scan_type;
+	u32	bt_ble_scan_para[3];
+
+	bool	run_time_state;
+	bool	freeze_coexrun_by_btinfo;
+
+	bool	is_A2DP_3M;
+	bool	voice_over_HOGP;
+	u8	bt_info;
+	bool	is_autoslot;
+	u8	forbidden_slot;
+	u8	hid_busy_num;
+	u8	hid_pair_cnt;
+
+	u32	cnt_remote_name_req;
+	u32	cnt_setup_link;
+	u32	cnt_reinit;
+	u32	cnt_ign_wlan_act;
+	u32	cnt_page;
+	u32	cnt_role_switch;
+
+	u16	bt_reg_vendor_ac;
+	u16	bt_reg_vendor_ae;
+
+	bool	is_setup_link;
+	u8	wl_noisy_level;
+	u32	gnt_error_cnt;
+
+	u8	bt_afh_map[10];
+	u8	bt_relink_downcount;
+	bool	is_tdma_btautoslot;
+	bool	is_tdma_btautoslot_hang;
+
+	bool	is_esco_mode;
+	u8	switch_band_notify_to;
+	bool	is_rf_state_off;
+
+	bool	is_hid_low_pri_tx_overhead;
+	bool	is_bt_multi_link;
+	bool	is_bt_a2dp_sink;
+
+	bool	is_set_ps_state_fail;
+	u8	cnt_set_ps_state_fail;
+};
+
+#define BT_8822B_2ANT_EXT_BAND_SWITCH_USE_DPDT	0
+#define BT_8822B_2ANT_EXT_BAND_SWITCH_USE_SPDT	1
+
+struct rfe_type_8822b_2ant {
+	u8	rfe_module_type;
+	bool	ext_ant_switch_exist;
+	u8	ext_ant_switch_type; /* 0:DPDT, 1:SPDT */
+	/*  iF 0: DPDT_P=0, DPDT_N=1 => BTG to Main, WL_A+G to Aux */
+	u8	ext_ant_switch_ctrl_polarity;
+
+	bool	ext_band_switch_exist;
+	u8	ext_band_switch_type; /* 0:DPDT, 1:SPDT */
+	u8	ext_band_switch_ctrl_polarity;
+
+	/*  If true:  WLG at BTG, If false: WLG at WLAG */
+	bool	wlg_locate_at_btg;
+
+	bool	ext_ant_switch_diversity; /* If diversity on */
+};
+
+#define BT_8822B_2ANT_ANTDET_PSD_POINTS	256 /* MAX:1024 */
+#define BT_8822B_2ANT_ANTDET_PSD_AVGNUM	1 /* MAX:3 */
+#define BT_8822B_2ANT_ANTDET_BUF_LEN	16
+
+struct psdscan_sta_8822b_2ant {
+	u32	ant_det_bt_le_channel; /* BT LE Channel ex:2412 */
+	u32	ant_det_bt_tx_time;
+	u32	ant_det_pre_psdscan_peak_val;
+	bool	ant_det_is_ant_det_available;
+	u32	ant_det_psd_scan_peak_val;
+	bool	ant_det_is_btreply_available;
+	u32	ant_det_psd_scan_peak_freq;
+
+	u8	ant_det_result;
+	u8	ant_det_peak_val[BT_8822B_2ANT_ANTDET_BUF_LEN];
+	u8	ant_det_peak_freq[BT_8822B_2ANT_ANTDET_BUF_LEN];
+	u32	ant_det_try_count;
+	u32	ant_det_fail_count;
+	u32	ant_det_inteval_count;
+	u32	ant_det_thres_offset;
+
+	u32	real_cent_freq;
+	s32	real_offset;
+	u32	real_span;
+
+	u32	psd_band_width; /* unit: Hz */
+	u32	psd_point; /* 128/256/512/1024 */
+	u32	psd_report[1024]; /* unit:dB (20logx), 0~255 */
+	u32	psd_report_max_hold[1024]; /* unit:dB (20logx), 0~255 */
+	u32	psd_start_point;
+	u32	psd_stop_point;
+	u32	psd_max_value_point;
+	u32	psd_max_value;
+	u32	psd_max_value2;
+	/* filter loop_max_value that below BT_8822B_1ANT_ANTDET_PSDTHRES_1ANT,
+	 * and average the rest
+	 */
+	u32	psd_avg_value;
+	/*max value in each loop */
+	u32	psd_loop_max_value[BT_8822B_2ANT_ANTDET_PSD_SWWEEPCOUNT];
+	u32	psd_start_base;
+	u32	psd_avg_num; /* 1/8/16/32 */
+	u32	psd_gen_count;
+	bool	is_ant_det_running;
+	bool	is_psd_show_max_only;
+};
+
+/* *******************************************
+ * The following is interface which will notify coex module.
+ * ********************************************/
+void ex_btc8822b2ant_power_on_setting(struct btc_coexist *btcoexist);
+void ex_btc8822b2ant_pre_load_firmware(struct btc_coexist *btcoexist);
+void ex_btc8822b2ant_init_hw_config(struct btc_coexist *btcoexist,
+				    bool wifi_only);
+void ex_btc8822b2ant_init_coex_dm(struct btc_coexist *btcoexist);
+void ex_btc8822b2ant_ips_notify(struct btc_coexist *btcoexist, u8 type);
+void ex_btc8822b2ant_lps_notify(struct btc_coexist *btcoexist, u8 type);
+void ex_btc8822b2ant_scan_notify(struct btc_coexist *btcoexist, u8 type);
+void ex_btc8822b2ant_switchband_notify(struct btc_coexist *btcoexist, u8 type);
+void ex_btc8822b2ant_connect_notify(struct btc_coexist *btcoexist, u8 type);
+void ex_btc8822b2ant_media_status_notify(struct btc_coexist *btcoexist,
+					 u8 type);
+void ex_btc8822b2ant_specific_packet_notify(struct btc_coexist *btcoexist,
+					    u8 type);
+void ex_btc8822b2ant_bt_info_notify(struct btc_coexist *btcoexist, u8 *tmp_buf,
+				    u8 length);
+void ex_btc8822b2ant_rf_status_notify(struct btc_coexist *btcoexist, u8 type);
+void ex_btc8822b2ant_halt_notify(struct btc_coexist *btcoexist);
+void ex_btc8822b2ant_pnp_notify(struct btc_coexist *btcoexist, u8 pnp_state);
+void ex_btc8822b2ant_periodical(struct btc_coexist *btcoexist);
+void ex_btc8822b2ant_display_coex_info(struct btc_coexist *btcoexist,
+				       struct seq_file *m);
+void ex_btc8822b2ant_antenna_detection(struct btc_coexist *btcoexist,
+				       u32 cent_freq, u32 offset, u32 span,
+				       u32 seconds);
+void ex_btc8822b2ant_display_ant_detection(struct btc_coexist *btcoexist);
diff --git a/drivers/staging/rtlwifi/btcoexist/halbtc8822bwifionly.c b/drivers/staging/rtlwifi/btcoexist/halbtc8822bwifionly.c
new file mode 100644
index 000000000000..43d628a71611
--- /dev/null
+++ b/drivers/staging/rtlwifi/btcoexist/halbtc8822bwifionly.c
@@ -0,0 +1,65 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#include "halbt_precomp.h"
+
+void ex_hal8822b_wifi_only_hw_config(struct wifi_only_cfg *wifionlycfg)
+{
+	/*BB control*/
+	halwifionly_phy_set_bb_reg(wifionlycfg, 0x4c, 0x01800000, 0x2);
+	/*SW control*/
+	halwifionly_phy_set_bb_reg(wifionlycfg, 0xcb4, 0xff, 0x77);
+	/*antenna mux switch */
+	halwifionly_phy_set_bb_reg(wifionlycfg, 0x974, 0x300, 0x3);
+
+	halwifionly_phy_set_bb_reg(wifionlycfg, 0x1990, 0x300, 0x0);
+
+	halwifionly_phy_set_bb_reg(wifionlycfg, 0xcbc, 0x80000, 0x0);
+	/*switch to WL side controller and gnt_wl gnt_bt debug signal */
+	halwifionly_phy_set_bb_reg(wifionlycfg, 0x70, 0xff000000, 0x0e);
+	/*gnt_wl=1 , gnt_bt=0*/
+	halwifionly_phy_set_bb_reg(wifionlycfg, 0x1704, 0xffffffff, 0x7700);
+	halwifionly_phy_set_bb_reg(wifionlycfg, 0x1700, 0xffffffff, 0xc00f0038);
+}
+
+void ex_hal8822b_wifi_only_scannotify(struct wifi_only_cfg *wifionlycfg,
+				      u8 is_5g)
+{
+	hal8822b_wifi_only_switch_antenna(wifionlycfg, is_5g);
+}
+
+void ex_hal8822b_wifi_only_switchbandnotify(struct wifi_only_cfg *wifionlycfg,
+					    u8 is_5g)
+{
+	hal8822b_wifi_only_switch_antenna(wifionlycfg, is_5g);
+}
+
+void hal8822b_wifi_only_switch_antenna(struct wifi_only_cfg *wifionlycfg,
+				       u8 is_5g)
+{
+	if (is_5g)
+		halwifionly_phy_set_bb_reg(wifionlycfg, 0xcbc, 0x300, 0x1);
+	else
+		halwifionly_phy_set_bb_reg(wifionlycfg, 0xcbc, 0x300, 0x2);
+}
diff --git a/drivers/staging/rtlwifi/btcoexist/halbtc8822bwifionly.h b/drivers/staging/rtlwifi/btcoexist/halbtc8822bwifionly.h
new file mode 100644
index 000000000000..464774e6e7b4
--- /dev/null
+++ b/drivers/staging/rtlwifi/btcoexist/halbtc8822bwifionly.h
@@ -0,0 +1,35 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef __INC_HAL8822BWIFIONLYHWCFG_H
+#define __INC_HAL8822BWIFIONLYHWCFG_H
+
+void ex_hal8822b_wifi_only_hw_config(struct wifi_only_cfg *wifionlycfg);
+void ex_hal8822b_wifi_only_scannotify(struct wifi_only_cfg *wifionlycfg,
+				      u8 is_5g);
+void ex_hal8822b_wifi_only_switchbandnotify(struct wifi_only_cfg *wifionlycfg,
+					    u8 is_5g);
+void hal8822b_wifi_only_switch_antenna(struct wifi_only_cfg *wifionlycfg,
+				       u8 is_5g);
+#endif
diff --git a/drivers/staging/rtlwifi/btcoexist/halbtcoutsrc.c b/drivers/staging/rtlwifi/btcoexist/halbtcoutsrc.c
new file mode 100644
index 000000000000..52620b72cfa9
--- /dev/null
+++ b/drivers/staging/rtlwifi/btcoexist/halbtcoutsrc.c
@@ -0,0 +1,1881 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2013 Realtek Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ ******************************************************************************/
+
+#include "halbt_precomp.h"
+
+/***************************************************
+ *		Debug related function
+ ***************************************************/
+
+static const char *const gl_btc_wifi_bw_string[] = {
+	"11bg",
+	"HT20",
+	"HT40",
+	"HT80",
+	"HT160"
+};
+
+static const char *const gl_btc_wifi_freq_string[] = {
+	"2.4G",
+	"5G"
+};
+
+static bool halbtc_is_bt_coexist_available(struct btc_coexist *btcoexist)
+{
+	if (!btcoexist->binded || NULL == btcoexist->adapter)
+		return false;
+
+	return true;
+}
+
+static bool halbtc_is_wifi_busy(struct rtl_priv *rtlpriv)
+{
+	if (rtlpriv->link_info.busytraffic)
+		return true;
+	else
+		return false;
+}
+
+static void halbtc_dbg_init(void)
+{
+}
+
+/***************************************************
+ *		helper function
+ ***************************************************/
+static bool is_any_client_connect_to_ap(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	struct rtl_mac *mac = rtl_mac(rtlpriv);
+	struct rtl_sta_info *drv_priv;
+	u8 cnt = 0;
+
+	if (mac->opmode == NL80211_IFTYPE_ADHOC ||
+	    mac->opmode == NL80211_IFTYPE_MESH_POINT ||
+	    mac->opmode == NL80211_IFTYPE_AP) {
+		if (in_interrupt() > 0) {
+			list_for_each_entry(drv_priv, &rtlpriv->entry_list,
+					    list) {
+				cnt++;
+			}
+		} else {
+			spin_lock_bh(&rtlpriv->locks.entry_list_lock);
+			list_for_each_entry(drv_priv, &rtlpriv->entry_list,
+					    list) {
+				cnt++;
+			}
+			spin_unlock_bh(&rtlpriv->locks.entry_list_lock);
+		}
+	}
+	if (cnt > 0)
+		return true;
+	else
+		return false;
+}
+
+static bool halbtc_legacy(struct rtl_priv *adapter)
+{
+	struct rtl_priv *rtlpriv = adapter;
+	struct rtl_mac *mac = rtl_mac(rtlpriv);
+
+	bool is_legacy = false;
+
+	if ((mac->mode == WIRELESS_MODE_B) || (mac->mode == WIRELESS_MODE_G))
+		is_legacy = true;
+
+	return is_legacy;
+}
+
+bool halbtc_is_wifi_uplink(struct rtl_priv *adapter)
+{
+	struct rtl_priv *rtlpriv = adapter;
+
+	if (rtlpriv->link_info.tx_busy_traffic)
+		return true;
+	else
+		return false;
+}
+
+static u32 halbtc_get_wifi_bw(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv =
+		(struct rtl_priv *)btcoexist->adapter;
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	u32 wifi_bw = BTC_WIFI_BW_HT20;
+
+	if (halbtc_legacy(rtlpriv)) {
+		wifi_bw = BTC_WIFI_BW_LEGACY;
+	} else {
+		switch (rtlphy->current_chan_bw) {
+		case HT_CHANNEL_WIDTH_20:
+			wifi_bw = BTC_WIFI_BW_HT20;
+			break;
+		case HT_CHANNEL_WIDTH_20_40:
+			wifi_bw = BTC_WIFI_BW_HT40;
+			break;
+		case HT_CHANNEL_WIDTH_80:
+			wifi_bw = BTC_WIFI_BW_HT80;
+			break;
+		}
+	}
+
+	return wifi_bw;
+}
+
+static u8 halbtc_get_wifi_central_chnl(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	struct rtl_phy	*rtlphy = &rtlpriv->phy;
+	u8 chnl = 1;
+
+	if (rtlphy->current_channel != 0)
+		chnl = rtlphy->current_channel;
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "%s:%d\n", __func__, chnl);
+	return chnl;
+}
+
+static u8 rtl_get_hwpg_single_ant_path(struct rtl_priv *rtlpriv)
+{
+	return rtlpriv->btcoexist.btc_info.single_ant_path;
+}
+
+static u8 rtl_get_hwpg_bt_type(struct rtl_priv *rtlpriv)
+{
+	return rtlpriv->btcoexist.btc_info.bt_type;
+}
+
+static u8 rtl_get_hwpg_ant_num(struct rtl_priv *rtlpriv)
+{
+	u8 num;
+
+	if (rtlpriv->btcoexist.btc_info.ant_num == ANT_X2)
+		num = 2;
+	else
+		num = 1;
+
+	return num;
+}
+
+static u8 rtl_get_hwpg_package_type(struct rtl_priv *rtlpriv)
+{
+	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+
+	return rtlhal->package_type;
+}
+
+static
+u8 rtl_get_hwpg_rfe_type(struct rtl_priv *rtlpriv)
+{
+	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+
+	return rtlhal->rfe_type;
+}
+
+/* ************************************
+ *         Hal helper function
+ * ************************************
+ */
+static
+bool halbtc_is_hw_mailbox_exist(struct btc_coexist *btcoexist)
+{
+	if (IS_HARDWARE_TYPE_8812(btcoexist->adapter))
+		return false;
+	else
+		return true;
+}
+
+static
+bool halbtc_send_bt_mp_operation(struct btc_coexist *btcoexist, u8 op_code,
+				 u8 *cmd, u32 len, unsigned long wait_ms)
+{
+	struct rtl_priv *rtlpriv;
+	const u8 oper_ver = 0;
+	u8 req_num;
+
+	if (!halbtc_is_hw_mailbox_exist(btcoexist))
+		return false;
+
+	if (wait_ms)	/* before h2c to avoid race condition */
+		reinit_completion(&btcoexist->bt_mp_comp);
+
+	rtlpriv = btcoexist->adapter;
+
+	/*
+	 * fill req_num by op_code, and rtl_btc_btmpinfo_notify() use it
+	 * to know message type
+	 */
+	switch (op_code) {
+	case BT_OP_GET_BT_VERSION:
+		req_num = BT_SEQ_GET_BT_VERSION;
+		break;
+	case BT_OP_GET_AFH_MAP_L:
+		req_num = BT_SEQ_GET_AFH_MAP_L;
+		break;
+	case BT_OP_GET_AFH_MAP_M:
+		req_num = BT_SEQ_GET_AFH_MAP_M;
+		break;
+	case BT_OP_GET_AFH_MAP_H:
+		req_num = BT_SEQ_GET_AFH_MAP_H;
+		break;
+	case BT_OP_GET_BT_COEX_SUPPORTED_FEATURE:
+		req_num = BT_SEQ_GET_BT_COEX_SUPPORTED_FEATURE;
+		break;
+	case BT_OP_GET_BT_COEX_SUPPORTED_VERSION:
+		req_num = BT_SEQ_GET_BT_COEX_SUPPORTED_VERSION;
+		break;
+	case BT_OP_GET_BT_ANT_DET_VAL:
+		req_num = BT_SEQ_GET_BT_ANT_DET_VAL;
+		break;
+	case BT_OP_GET_BT_BLE_SCAN_PARA:
+		req_num = BT_SEQ_GET_BT_BLE_SCAN_PARA;
+		break;
+	case BT_OP_GET_BT_BLE_SCAN_TYPE:
+		req_num = BT_SEQ_GET_BT_BLE_SCAN_TYPE;
+		break;
+	case BT_OP_WRITE_REG_ADDR:
+	case BT_OP_WRITE_REG_VALUE:
+	case BT_OP_READ_REG:
+	default:
+		req_num = BT_SEQ_DONT_CARE;
+		break;
+	}
+
+	cmd[0] |= (oper_ver & 0x0f);		/* Set OperVer */
+	cmd[0] |= ((req_num << 4) & 0xf0);	/* Set ReqNum */
+	cmd[1] = op_code;
+	rtlpriv->cfg->ops->fill_h2c_cmd(rtlpriv->mac80211.hw, 0x67, len, cmd);
+
+	/* wait? */
+	if (!wait_ms)
+		return true;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "btmpinfo wait req_num=%d wait=%ld\n", req_num, wait_ms);
+
+	if (in_interrupt())
+		return false;
+
+	if (wait_for_completion_timeout(&btcoexist->bt_mp_comp,
+					msecs_to_jiffies(wait_ms)) == 0) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_DMESG,
+			 "btmpinfo wait (req_num=%d) timeout\n", req_num);
+
+		return false;	/* timeout */
+	}
+
+	return true;
+}
+
+static void halbtc_leave_lps(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv;
+	struct rtl_ps_ctl *ppsc;
+	bool ap_enable = false;
+
+	rtlpriv = btcoexist->adapter;
+	ppsc = rtl_psc(rtlpriv);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_AP_MODE_ENABLE,
+			   &ap_enable);
+
+	if (ap_enable) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_DMESG,
+			 "%s()<--dont leave lps under AP mode\n", __func__);
+		return;
+	}
+
+	btcoexist->bt_info.bt_ctrl_lps = true;
+	btcoexist->bt_info.bt_lps_on = false;
+	rtl_lps_leave(rtlpriv->mac80211.hw);
+}
+
+static void halbtc_enter_lps(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv;
+	struct rtl_ps_ctl *ppsc;
+	bool ap_enable = false;
+
+	rtlpriv = btcoexist->adapter;
+	ppsc = rtl_psc(rtlpriv);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_AP_MODE_ENABLE,
+			   &ap_enable);
+
+	if (ap_enable) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_DMESG,
+			 "%s()<--dont enter lps under AP mode\n", __func__);
+		return;
+	}
+
+	btcoexist->bt_info.bt_ctrl_lps = true;
+	btcoexist->bt_info.bt_lps_on = true;
+	rtl_lps_enter(rtlpriv->mac80211.hw);
+}
+
+static void halbtc_normal_lps(struct btc_coexist *btcoexist)
+{
+	struct rtl_priv *rtlpriv;
+
+	rtlpriv = btcoexist->adapter;
+
+	if (btcoexist->bt_info.bt_ctrl_lps) {
+		btcoexist->bt_info.bt_lps_on = false;
+		rtl_lps_leave(rtlpriv->mac80211.hw);
+		btcoexist->bt_info.bt_ctrl_lps = false;
+	}
+}
+
+static void halbtc_leave_low_power(struct btc_coexist *btcoexist)
+{
+}
+
+static void halbtc_normal_low_power(struct btc_coexist *btcoexist)
+{
+}
+
+static void halbtc_disable_low_power(struct btc_coexist *btcoexist,
+				     bool low_pwr_disable)
+{
+	/* TODO: original/leave 32k low power */
+	btcoexist->bt_info.bt_disable_low_pwr = low_pwr_disable;
+}
+
+static void halbtc_aggregation_check(struct btc_coexist *btcoexist)
+{
+	bool need_to_act = false;
+	static unsigned long pre_time;
+	unsigned long cur_time = 0;
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	/* To void continuous deleteBA=>addBA=>deleteBA=>addBA
+	 * This function is not allowed to continuous called
+	 * It can only be called after 8 seconds
+	 */
+
+	cur_time = jiffies;
+	if (jiffies_to_msecs(cur_time - pre_time) <= 8000) {
+		/* over 8 seconds you can execute this function again. */
+		return;
+	}
+	pre_time = cur_time;
+
+	if (btcoexist->bt_info.reject_agg_pkt) {
+		need_to_act = true;
+		btcoexist->bt_info.pre_reject_agg_pkt =
+			btcoexist->bt_info.reject_agg_pkt;
+	} else {
+		if (btcoexist->bt_info.pre_reject_agg_pkt) {
+			need_to_act = true;
+			btcoexist->bt_info.pre_reject_agg_pkt =
+				btcoexist->bt_info.reject_agg_pkt;
+		}
+
+		if (btcoexist->bt_info.pre_bt_ctrl_agg_buf_size !=
+		    btcoexist->bt_info.bt_ctrl_agg_buf_size) {
+			need_to_act = true;
+			btcoexist->bt_info.pre_bt_ctrl_agg_buf_size =
+				btcoexist->bt_info.bt_ctrl_agg_buf_size;
+		}
+
+		if (btcoexist->bt_info.bt_ctrl_agg_buf_size) {
+			if (btcoexist->bt_info.pre_agg_buf_size !=
+			    btcoexist->bt_info.agg_buf_size) {
+				need_to_act = true;
+			}
+			btcoexist->bt_info.pre_agg_buf_size =
+				btcoexist->bt_info.agg_buf_size;
+		}
+
+		if (need_to_act)
+			rtl_rx_ampdu_apply(rtlpriv);
+	}
+}
+
+static u32 halbtc_get_bt_patch_version(struct btc_coexist *btcoexist)
+{
+	u8 cmd_buffer[4] = {0};
+
+	if (btcoexist->bt_info.bt_real_fw_ver)
+		goto label_done;
+
+	/* cmd_buffer[0] and [1] is filled by halbtc_send_bt_mp_operation() */
+	halbtc_send_bt_mp_operation(btcoexist, BT_OP_GET_BT_VERSION,
+				    cmd_buffer, 4, 200);
+
+label_done:
+	return btcoexist->bt_info.bt_real_fw_ver;
+}
+
+static u32 halbtc_get_bt_coex_supported_feature(void *btc_context)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)btc_context;
+	u8 cmd_buffer[4] = {0};
+
+	if (btcoexist->bt_info.bt_supported_feature)
+		goto label_done;
+
+	/* cmd_buffer[0] and [1] is filled by halbtc_send_bt_mp_operation() */
+	halbtc_send_bt_mp_operation(btcoexist,
+				    BT_OP_GET_BT_COEX_SUPPORTED_FEATURE,
+				    cmd_buffer, 4, 200);
+
+label_done:
+	return btcoexist->bt_info.bt_supported_feature;
+}
+
+static u32 halbtc_get_bt_coex_supported_version(void *btc_context)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)btc_context;
+	u8 cmd_buffer[4] = {0};
+
+	if (btcoexist->bt_info.bt_supported_version)
+		goto label_done;
+
+	/* cmd_buffer[0] and [1] is filled by halbtc_send_bt_mp_operation() */
+	halbtc_send_bt_mp_operation(btcoexist,
+				    BT_OP_GET_BT_COEX_SUPPORTED_VERSION,
+				    cmd_buffer, 4, 200);
+
+label_done:
+	return btcoexist->bt_info.bt_supported_version;
+}
+
+static u32 halbtc_get_wifi_link_status(struct btc_coexist *btcoexist)
+{
+	/* return value:
+	 * [31:16] => connected port number
+	 * [15:0]  => port connected bit define
+	 */
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	struct rtl_mac *mac = rtl_mac(rtlpriv);
+	u32 ret_val = 0;
+	u32 port_connected_status = 0, num_of_connected_port = 0;
+
+	if (mac->opmode == NL80211_IFTYPE_STATION &&
+	    mac->link_state >= MAC80211_LINKED) {
+		port_connected_status |= WIFI_STA_CONNECTED;
+		num_of_connected_port++;
+	}
+	/* AP & ADHOC & MESH */
+	if (is_any_client_connect_to_ap(btcoexist)) {
+		port_connected_status |= WIFI_AP_CONNECTED;
+		num_of_connected_port++;
+	}
+	/* TODO: P2P Connected Status */
+
+	ret_val = (num_of_connected_port << 16) | port_connected_status;
+
+	return ret_val;
+}
+
+static s32 halbtc_get_wifi_rssi(struct rtl_priv *rtlpriv)
+{
+	int undec_sm_pwdb = 0;
+
+	if (rtlpriv->mac80211.link_state >= MAC80211_LINKED)
+		undec_sm_pwdb = rtlpriv->dm.undec_sm_pwdb;
+	else /* associated entry pwdb */
+		undec_sm_pwdb = rtlpriv->dm.undec_sm_pwdb;
+	return undec_sm_pwdb;
+}
+
+static bool halbtc_get(void *void_btcoexist, u8 get_type, void *out_buf)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)void_btcoexist;
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	struct rtl_mac *mac = rtl_mac(rtlpriv);
+	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+	bool *bool_tmp = (bool *)out_buf;
+	int *s32_tmp = (int *)out_buf;
+	u32 *u32_tmp = (u32 *)out_buf;
+	u8 *u8_tmp = (u8 *)out_buf;
+	bool tmp = false;
+	bool ret = true;
+
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return false;
+
+	switch (get_type) {
+	case BTC_GET_BL_HS_OPERATION:
+		*bool_tmp = false;
+		ret = false;
+		break;
+	case BTC_GET_BL_HS_CONNECTING:
+		*bool_tmp = false;
+		ret = false;
+		break;
+	case BTC_GET_BL_WIFI_CONNECTED:
+		if (rtlpriv->mac80211.opmode == NL80211_IFTYPE_STATION &&
+		    rtlpriv->mac80211.link_state >= MAC80211_LINKED)
+			tmp = true;
+		if (is_any_client_connect_to_ap(btcoexist))
+			tmp = true;
+		*bool_tmp = tmp;
+		break;
+	case BTC_GET_BL_WIFI_BUSY:
+		if (halbtc_is_wifi_busy(rtlpriv))
+			*bool_tmp = true;
+		else
+			*bool_tmp = false;
+		break;
+	case BTC_GET_BL_WIFI_SCAN:
+		if (mac->act_scanning)
+			*bool_tmp = true;
+		else
+			*bool_tmp = false;
+		break;
+	case BTC_GET_BL_WIFI_LINK:
+		if (mac->link_state == MAC80211_LINKING)
+			*bool_tmp = true;
+		else
+			*bool_tmp = false;
+		break;
+	case BTC_GET_BL_WIFI_ROAM:
+		if (mac->link_state == MAC80211_LINKING)
+			*bool_tmp = true;
+		else
+			*bool_tmp = false;
+		break;
+	case BTC_GET_BL_WIFI_4_WAY_PROGRESS:
+		*bool_tmp = rtlpriv->btcoexist.btc_info.in_4way;
+		break;
+	case BTC_GET_BL_WIFI_UNDER_5G:
+		if (rtlhal->current_bandtype == BAND_ON_5G)
+			*bool_tmp = true;
+		else
+			*bool_tmp = false;
+		break;
+	case BTC_GET_BL_WIFI_AP_MODE_ENABLE:
+		if (mac->opmode == NL80211_IFTYPE_AP)
+			*bool_tmp = true;
+		else
+			*bool_tmp = false;
+		break;
+	case BTC_GET_BL_WIFI_ENABLE_ENCRYPTION:
+		if (rtlpriv->sec.pairwise_enc_algorithm == NO_ENCRYPTION)
+			*bool_tmp = false;
+		else
+			*bool_tmp = true;
+		break;
+	case BTC_GET_BL_WIFI_UNDER_B_MODE:
+		if (rtlpriv->mac80211.mode == WIRELESS_MODE_B)
+			*bool_tmp = true;
+		else
+			*bool_tmp = false;
+		break;
+	case BTC_GET_BL_EXT_SWITCH:
+		*bool_tmp = false;
+		break;
+	case BTC_GET_BL_WIFI_IS_IN_MP_MODE:
+		*bool_tmp = false;
+		break;
+	case BTC_GET_BL_IS_ASUS_8723B:
+		*bool_tmp = false;
+		break;
+	case BTC_GET_BL_RF4CE_CONNECTED:
+		*bool_tmp = false;
+		break;
+	case BTC_GET_S4_WIFI_RSSI:
+		*s32_tmp = halbtc_get_wifi_rssi(rtlpriv);
+		break;
+	case BTC_GET_S4_HS_RSSI:
+		*s32_tmp = 0;
+		ret = false;
+		break;
+	case BTC_GET_U4_WIFI_BW:
+		*u32_tmp = halbtc_get_wifi_bw(btcoexist);
+		break;
+	case BTC_GET_U4_WIFI_TRAFFIC_DIRECTION:
+		if (halbtc_is_wifi_uplink(rtlpriv))
+			*u32_tmp = BTC_WIFI_TRAFFIC_TX;
+		else
+			*u32_tmp = BTC_WIFI_TRAFFIC_RX;
+		break;
+	case BTC_GET_U4_WIFI_FW_VER:
+		*u32_tmp = (rtlhal->fw_version << 16) | rtlhal->fw_subversion;
+		break;
+	case BTC_GET_U4_WIFI_LINK_STATUS:
+		*u32_tmp = halbtc_get_wifi_link_status(btcoexist);
+		break;
+	case BTC_GET_U4_BT_PATCH_VER:
+		*u32_tmp = halbtc_get_bt_patch_version(btcoexist);
+		break;
+	case BTC_GET_U4_VENDOR:
+		*u32_tmp = BTC_VENDOR_OTHER;
+		break;
+	case BTC_GET_U4_SUPPORTED_VERSION:
+		*u32_tmp = halbtc_get_bt_coex_supported_version(btcoexist);
+		break;
+	case BTC_GET_U4_SUPPORTED_FEATURE:
+		*u32_tmp = halbtc_get_bt_coex_supported_feature(btcoexist);
+		break;
+	case BTC_GET_U4_WIFI_IQK_TOTAL:
+		*u32_tmp = btcoexist->btc_phydm_query_phy_counter(btcoexist,
+								  "IQK_TOTAL");
+		break;
+	case BTC_GET_U4_WIFI_IQK_OK:
+		*u32_tmp = btcoexist->btc_phydm_query_phy_counter(btcoexist,
+								  "IQK_OK");
+		break;
+	case BTC_GET_U4_WIFI_IQK_FAIL:
+		*u32_tmp = btcoexist->btc_phydm_query_phy_counter(btcoexist,
+								  "IQK_FAIL");
+		break;
+	case BTC_GET_U1_WIFI_DOT11_CHNL:
+		*u8_tmp = rtlphy->current_channel;
+		break;
+	case BTC_GET_U1_WIFI_CENTRAL_CHNL:
+		*u8_tmp = halbtc_get_wifi_central_chnl(btcoexist);
+		break;
+	case BTC_GET_U1_WIFI_HS_CHNL:
+		*u8_tmp = 0;
+		ret = false;
+		break;
+	case BTC_GET_U1_AP_NUM:
+		*u8_tmp = rtlpriv->btcoexist.btc_info.ap_num;
+		break;
+	case BTC_GET_U1_ANT_TYPE:
+		*u8_tmp = (u8)BTC_ANT_TYPE_0;
+		break;
+	case BTC_GET_U1_IOT_PEER:
+		*u8_tmp = 0;
+		break;
+
+		/************* 1Ant **************/
+	case BTC_GET_U1_LPS_MODE:
+		*u8_tmp = btcoexist->pwr_mode_val[0];
+		break;
+
+	default:
+		ret = false;
+		break;
+	}
+
+	return ret;
+}
+
+static bool halbtc_set(void *void_btcoexist, u8 set_type, void *in_buf)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)void_btcoexist;
+	bool *bool_tmp = (bool *)in_buf;
+	u8 *u8_tmp = (u8 *)in_buf;
+	u32 *u32_tmp = (u32 *)in_buf;
+	bool ret = true;
+
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return false;
+
+	switch (set_type) {
+	/* set some bool type variables. */
+	case BTC_SET_BL_BT_DISABLE:
+		btcoexist->bt_info.bt_disabled = *bool_tmp;
+		break;
+	case BTC_SET_BL_BT_TRAFFIC_BUSY:
+		btcoexist->bt_info.bt_busy = *bool_tmp;
+		break;
+	case BTC_SET_BL_BT_LIMITED_DIG:
+		btcoexist->bt_info.limited_dig = *bool_tmp;
+		break;
+	case BTC_SET_BL_FORCE_TO_ROAM:
+		btcoexist->bt_info.force_to_roam = *bool_tmp;
+		break;
+	case BTC_SET_BL_TO_REJ_AP_AGG_PKT:
+		btcoexist->bt_info.reject_agg_pkt = *bool_tmp;
+		break;
+	case BTC_SET_BL_BT_CTRL_AGG_SIZE:
+		btcoexist->bt_info.bt_ctrl_agg_buf_size = *bool_tmp;
+		break;
+	case BTC_SET_BL_INC_SCAN_DEV_NUM:
+		btcoexist->bt_info.increase_scan_dev_num = *bool_tmp;
+		break;
+	case BTC_SET_BL_BT_TX_RX_MASK:
+		btcoexist->bt_info.bt_tx_rx_mask = *bool_tmp;
+		break;
+	case BTC_SET_BL_MIRACAST_PLUS_BT:
+		btcoexist->bt_info.miracast_plus_bt = *bool_tmp;
+		break;
+		/* set some u1Byte type variables. */
+	case BTC_SET_U1_RSSI_ADJ_VAL_FOR_AGC_TABLE_ON:
+		btcoexist->bt_info.rssi_adjust_for_agc_table_on = *u8_tmp;
+		break;
+	case BTC_SET_U1_AGG_BUF_SIZE:
+		btcoexist->bt_info.agg_buf_size = *u8_tmp;
+		break;
+
+	/* the following are some action which will be triggered */
+	case BTC_SET_ACT_GET_BT_RSSI:
+		ret = false;
+		break;
+	case BTC_SET_ACT_AGGREGATE_CTRL:
+		halbtc_aggregation_check(btcoexist);
+		break;
+
+	/* 1Ant */
+	case BTC_SET_U1_RSSI_ADJ_VAL_FOR_1ANT_COEX_TYPE:
+		btcoexist->bt_info.rssi_adjust_for_1ant_coex_type = *u8_tmp;
+		break;
+	case BTC_SET_UI_SCAN_SIG_COMPENSATION:
+		break;
+	case BTC_SET_U1_LPS_VAL:
+		btcoexist->bt_info.lps_val = *u8_tmp;
+		break;
+	case BTC_SET_U1_RPWM_VAL:
+		btcoexist->bt_info.rpwm_val = *u8_tmp;
+		break;
+	/* the following are some action which will be triggered  */
+	case BTC_SET_ACT_LEAVE_LPS:
+		halbtc_leave_lps(btcoexist);
+		break;
+	case BTC_SET_ACT_ENTER_LPS:
+		halbtc_enter_lps(btcoexist);
+		break;
+	case BTC_SET_ACT_NORMAL_LPS:
+		halbtc_normal_lps(btcoexist);
+		break;
+	case BTC_SET_ACT_DISABLE_LOW_POWER:
+		halbtc_disable_low_power(btcoexist, *bool_tmp);
+		break;
+	case BTC_SET_ACT_UPDATE_RAMASK:
+		btcoexist->bt_info.ra_mask = *u32_tmp;
+		break;
+	case BTC_SET_ACT_SEND_MIMO_PS:
+		break;
+	case BTC_SET_ACT_CTRL_BT_INFO: /*wait for 8812/8821*/
+		break;
+	case BTC_SET_ACT_CTRL_BT_COEX:
+		break;
+	case BTC_SET_ACT_CTRL_8723B_ANT:
+		break;
+	default:
+		break;
+	}
+
+	return ret;
+}
+
+static void halbtc_display_coex_statistics(struct btc_coexist *btcoexist,
+					   struct seq_file *m)
+{
+}
+
+static void halbtc_display_bt_link_info(struct btc_coexist *btcoexist,
+					struct seq_file *m)
+{
+}
+
+static void halbtc_display_wifi_status(struct btc_coexist *btcoexist,
+				       struct seq_file *m)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	s32 wifi_rssi = 0, bt_hs_rssi = 0;
+	bool scan = false, link = false, roam = false, wifi_busy = false;
+	bool wifi_under_b_mode = false, wifi_under_5g = false;
+	u32 wifi_bw = BTC_WIFI_BW_HT20;
+	u32 wifi_traffic_dir = BTC_WIFI_TRAFFIC_TX;
+	u32 wifi_freq = BTC_FREQ_2_4G;
+	u32 wifi_link_status = 0x0;
+	bool bt_hs_on = false, under_ips = false, under_lps = false;
+	bool low_power = false, dc_mode = false;
+	u8 wifi_chnl = 0, wifi_hs_chnl = 0, fw_ps_state;
+	u8 ap_num = 0;
+
+	wifi_link_status = halbtc_get_wifi_link_status(btcoexist);
+	seq_printf(m, "\n %-35s = %d/ %d/ %d/ %d/ %d",
+		   "STA/vWifi/HS/p2pGo/p2pGc",
+		   ((wifi_link_status & WIFI_STA_CONNECTED) ? 1 : 0),
+		   ((wifi_link_status & WIFI_AP_CONNECTED) ? 1 : 0),
+		   ((wifi_link_status & WIFI_HS_CONNECTED) ? 1 : 0),
+		   ((wifi_link_status & WIFI_P2P_GO_CONNECTED) ? 1 : 0),
+		   ((wifi_link_status & WIFI_P2P_GC_CONNECTED) ? 1 : 0));
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_HS_OPERATION, &bt_hs_on);
+	btcoexist->btc_get(btcoexist, BTC_GET_U1_WIFI_DOT11_CHNL, &wifi_chnl);
+	btcoexist->btc_get(btcoexist, BTC_GET_U1_WIFI_HS_CHNL, &wifi_hs_chnl);
+	seq_printf(m, "\n %-35s = %d / %d(%d)",
+		   "Dot11 channel / HsChnl(High Speed)",
+		   wifi_chnl, wifi_hs_chnl, bt_hs_on);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_S4_WIFI_RSSI, &wifi_rssi);
+	btcoexist->btc_get(btcoexist, BTC_GET_S4_HS_RSSI, &bt_hs_rssi);
+	seq_printf(m, "\n %-35s = %d/ %d",
+		   "Wifi rssi/ HS rssi",
+		   wifi_rssi - 100, bt_hs_rssi - 100);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_SCAN, &scan);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_LINK, &link);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_ROAM, &roam);
+	seq_printf(m, "\n %-35s = %d/ %d/ %d ",
+		   "Wifi link/ roam/ scan",
+		   link, roam, scan);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_5G, &wifi_under_5g);
+	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_BW, &wifi_bw);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_TRAFFIC_DIRECTION,
+			   &wifi_traffic_dir);
+	btcoexist->btc_get(btcoexist, BTC_GET_U1_AP_NUM, &ap_num);
+	wifi_freq = (wifi_under_5g ? BTC_FREQ_5G : BTC_FREQ_2_4G);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_B_MODE,
+			   &wifi_under_b_mode);
+
+	seq_printf(m, "\n %-35s = %s / %s/ %s/ AP=%d ",
+		   "Wifi freq/ bw/ traffic",
+		   gl_btc_wifi_freq_string[wifi_freq],
+		   ((wifi_under_b_mode) ? "11b" :
+		    gl_btc_wifi_bw_string[wifi_bw]),
+		   ((!wifi_busy) ? "idle" : ((BTC_WIFI_TRAFFIC_TX ==
+					      wifi_traffic_dir) ? "uplink" :
+					     "downlink")),
+		   ap_num);
+
+	/* power status	 */
+	dc_mode = true;	/*TODO*/
+	under_ips = rtlpriv->psc.inactive_pwrstate == ERFOFF ? 1 : 0;
+	under_lps = rtlpriv->psc.dot11_psmode == EACTIVE ? 0 : 1;
+	fw_ps_state = 0;
+	low_power = 0; /*TODO*/
+	seq_printf(m, "\n %-35s = %s%s%s%s",
+		   "Power Status",
+		   (dc_mode ? "DC mode" : "AC mode"),
+		   (under_ips ? ", IPS ON" : ""),
+		   (under_lps ? ", LPS ON" : ""),
+		   (low_power ? ", 32k" : ""));
+
+	seq_printf(m,
+		   "\n %-35s = %02x %02x %02x %02x %02x %02x (0x%x/0x%x)",
+		   "Power mode cmd(lps/rpwm)",
+		   btcoexist->pwr_mode_val[0], btcoexist->pwr_mode_val[1],
+		   btcoexist->pwr_mode_val[2], btcoexist->pwr_mode_val[3],
+		   btcoexist->pwr_mode_val[4], btcoexist->pwr_mode_val[5],
+		   btcoexist->bt_info.lps_val,
+		   btcoexist->bt_info.rpwm_val);
+}
+
+/************************************************************
+ *		IO related function
+ ************************************************************/
+static u8 halbtc_read_1byte(void *bt_context, u32 reg_addr)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)bt_context;
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	return	rtl_read_byte(rtlpriv, reg_addr);
+}
+
+static u16 halbtc_read_2byte(void *bt_context, u32 reg_addr)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)bt_context;
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	return	rtl_read_word(rtlpriv, reg_addr);
+}
+
+static u32 halbtc_read_4byte(void *bt_context, u32 reg_addr)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)bt_context;
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	return	rtl_read_dword(rtlpriv, reg_addr);
+}
+
+static void halbtc_write_1byte(void *bt_context, u32 reg_addr, u32 data)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)bt_context;
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	rtl_write_byte(rtlpriv, reg_addr, data);
+}
+
+static void halbtc_bitmask_write_1byte(void *bt_context, u32 reg_addr,
+				       u32 bit_mask, u8 data)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)bt_context;
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	u8 original_value, bit_shift = 0;
+	u8 i;
+
+	if (bit_mask != MASKDWORD) {/*if not "double word" write*/
+		original_value = rtl_read_byte(rtlpriv, reg_addr);
+		for (i = 0; i <= 7; i++) {
+			if ((bit_mask >> i) & 0x1)
+				break;
+		}
+		bit_shift = i;
+		data = (original_value & (~bit_mask)) |
+			((data << bit_shift) & bit_mask);
+	}
+	rtl_write_byte(rtlpriv, reg_addr, data);
+}
+
+static void halbtc_write_2byte(void *bt_context, u32 reg_addr, u16 data)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)bt_context;
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	rtl_write_word(rtlpriv, reg_addr, data);
+}
+
+static void halbtc_write_4byte(void *bt_context, u32 reg_addr, u32 data)
+{
+	struct btc_coexist *btcoexist =
+		(struct btc_coexist *)bt_context;
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	rtl_write_dword(rtlpriv, reg_addr, data);
+}
+
+static void halbtc_write_local_reg_1byte(void *btc_context, u32 reg_addr,
+					 u8 data)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)btc_context;
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	if (btcoexist->chip_interface == BTC_INTF_SDIO)
+		;
+	else if (btcoexist->chip_interface == BTC_INTF_PCI)
+		rtl_write_byte(rtlpriv, reg_addr, data);
+	else if (btcoexist->chip_interface == BTC_INTF_USB)
+		rtl_write_byte(rtlpriv, reg_addr, data);
+}
+
+static void halbtc_set_bbreg(void *bt_context, u32 reg_addr, u32 bit_mask,
+			     u32 data)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)bt_context;
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	rtl_set_bbreg(rtlpriv->mac80211.hw, reg_addr, bit_mask, data);
+}
+
+static u32 halbtc_get_bbreg(void *bt_context, u32 reg_addr, u32 bit_mask)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)bt_context;
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	return rtl_get_bbreg(rtlpriv->mac80211.hw, reg_addr, bit_mask);
+}
+
+static void halbtc_set_rfreg(void *bt_context, u8 rf_path, u32 reg_addr,
+			     u32 bit_mask, u32 data)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)bt_context;
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	rtl_set_rfreg(rtlpriv->mac80211.hw, rf_path, reg_addr, bit_mask, data);
+}
+
+static u32 halbtc_get_rfreg(void *bt_context, u8 rf_path, u32 reg_addr,
+			    u32 bit_mask)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)bt_context;
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	return rtl_get_rfreg(rtlpriv->mac80211.hw, rf_path, reg_addr, bit_mask);
+}
+
+static void halbtc_fill_h2c_cmd(void *bt_context, u8 element_id,
+				u32 cmd_len, u8 *cmd_buf)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)bt_context;
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	rtlpriv->cfg->ops->fill_h2c_cmd(rtlpriv->mac80211.hw, element_id,
+					cmd_len, cmd_buf);
+}
+
+static void halbtc_send_wifi_port_id_cmd(void *bt_context)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)bt_context;
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	u8 cmd_buf[1] = {0};	/* port id [2:0] = 0 */
+
+	rtlpriv->cfg->ops->fill_h2c_cmd(rtlpriv->mac80211.hw, 0x71, 1,
+					cmd_buf);
+}
+
+static void halbtc_set_default_port_id_cmd(void *bt_context)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)bt_context;
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	struct ieee80211_hw *hw = rtlpriv->mac80211.hw;
+
+	if (!rtlpriv->cfg->ops->set_default_port_id_cmd)
+		return;
+
+	rtlpriv->cfg->ops->set_default_port_id_cmd(hw);
+}
+
+static
+void halbtc_set_bt_reg(void *btc_context, u8 reg_type, u32 offset, u32 set_val)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)btc_context;
+	u8 cmd_buffer1[4] = {0};
+	u8 cmd_buffer2[4] = {0};
+
+	/* cmd_buffer[0] and [1] is filled by halbtc_send_bt_mp_operation() */
+	*((__le16 *)&cmd_buffer1[2]) = cpu_to_le16((u16)set_val);
+	if (!halbtc_send_bt_mp_operation(btcoexist, BT_OP_WRITE_REG_VALUE,
+					 cmd_buffer1, 4, 200))
+		return;
+
+	/* cmd_buffer[0] and [1] is filled by halbtc_send_bt_mp_operation() */
+	cmd_buffer2[2] = reg_type;
+	*((u8 *)&cmd_buffer2[3]) = (u8)offset;
+	halbtc_send_bt_mp_operation(btcoexist, BT_OP_WRITE_REG_ADDR,
+				    cmd_buffer2, 4, 200);
+}
+
+static void halbtc_display_dbg_msg(void *bt_context, u8 disp_type,
+				   struct seq_file *m)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)bt_context;
+
+	switch (disp_type) {
+	case BTC_DBG_DISP_COEX_STATISTICS:
+		halbtc_display_coex_statistics(btcoexist, m);
+		break;
+	case BTC_DBG_DISP_BT_LINK_INFO:
+		halbtc_display_bt_link_info(btcoexist, m);
+		break;
+	case BTC_DBG_DISP_WIFI_STATUS:
+		halbtc_display_wifi_status(btcoexist, m);
+		break;
+	default:
+		break;
+	}
+}
+
+static u32 halbtc_get_bt_reg(void *btc_context, u8 reg_type, u32 offset)
+{
+	return 0;
+}
+
+static
+u32 halbtc_get_phydm_version(void *btc_context)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)btc_context;
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+
+	if (rtlpriv->phydm.ops)
+		return rtlpriv->phydm.ops->phydm_get_version(rtlpriv);
+
+	return 0;
+}
+
+static
+void halbtc_phydm_modify_ra_pcr_threshold(void *btc_context,
+					  u8 ra_offset_direction,
+					  u8 ra_threshold_offset)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)btc_context;
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	struct rtl_phydm_ops *phydm_ops = rtlpriv->phydm.ops;
+
+	if (phydm_ops)
+		phydm_ops->phydm_modify_ra_pcr_threshold(rtlpriv,
+							 ra_offset_direction,
+							 ra_threshold_offset);
+}
+
+static
+u32 halbtc_phydm_query_phy_counter(void *btc_context, const char *info_type)
+{
+	/* info_type may be strings below:
+	 * PHYDM_INFO_FA_OFDM, PHYDM_INFO_FA_CCK, PHYDM_INFO_CCA_OFDM,
+	 * PHYDM_INFO_CCA_CCK
+	 * IQK_TOTAL, IQK_OK, IQK_FAIL
+	 */
+
+	struct btc_coexist *btcoexist = (struct btc_coexist *)btc_context;
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	struct rtl_phydm_ops *phydm_ops = rtlpriv->phydm.ops;
+
+	if (phydm_ops)
+		return phydm_ops->phydm_query_counter(rtlpriv, info_type);
+
+	return 0;
+}
+
+static u8 halbtc_get_ant_det_val_from_bt(void *btc_context)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)btc_context;
+	u8 cmd_buffer[4] = {0};
+
+	/* cmd_buffer[0] and [1] is filled by halbtc_send_bt_mp_operation() */
+	halbtc_send_bt_mp_operation(btcoexist, BT_OP_GET_BT_ANT_DET_VAL,
+				    cmd_buffer, 4, 200);
+
+	/* need wait completion to return correct value */
+
+	return btcoexist->bt_info.bt_ant_det_val;
+}
+
+static u8 halbtc_get_ble_scan_type_from_bt(void *btc_context)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)btc_context;
+	u8 cmd_buffer[4] = {0};
+
+	/* cmd_buffer[0] and [1] is filled by halbtc_send_bt_mp_operation() */
+	halbtc_send_bt_mp_operation(btcoexist, BT_OP_GET_BT_BLE_SCAN_TYPE,
+				    cmd_buffer, 4, 200);
+
+	/* need wait completion to return correct value */
+
+	return btcoexist->bt_info.bt_ble_scan_type;
+}
+
+static u32 halbtc_get_ble_scan_para_from_bt(void *btc_context, u8 scan_type)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)btc_context;
+	u8 cmd_buffer[4] = {0};
+
+	/* cmd_buffer[0] and [1] is filled by halbtc_send_bt_mp_operation() */
+	halbtc_send_bt_mp_operation(btcoexist, BT_OP_GET_BT_BLE_SCAN_PARA,
+				    cmd_buffer, 4, 200);
+
+	/* need wait completion to return correct value */
+
+	return btcoexist->bt_info.bt_ble_scan_para;
+}
+
+static bool halbtc_get_bt_afh_map_from_bt(void *btc_context, u8 map_type,
+					  u8 *afh_map)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)btc_context;
+	u8 cmd_buffer[2] = {0};
+	bool ret;
+	u32 *afh_map_l = (u32 *)afh_map;
+	u32 *afh_map_m = (u32 *)(afh_map + 4);
+	u16 *afh_map_h = (u16 *)(afh_map + 8);
+
+	/* cmd_buffer[0] and [1] is filled by halbtc_send_bt_mp_operation() */
+	ret = halbtc_send_bt_mp_operation(btcoexist, BT_OP_GET_AFH_MAP_L,
+					  cmd_buffer, 2, 200);
+	if (!ret)
+		goto exit;
+
+	*afh_map_l = btcoexist->bt_info.afh_map_l;
+
+	/* cmd_buffer[0] and [1] is filled by halbtc_send_bt_mp_operation() */
+	ret = halbtc_send_bt_mp_operation(btcoexist, BT_OP_GET_AFH_MAP_M,
+					  cmd_buffer, 2, 200);
+	if (!ret)
+		goto exit;
+
+	*afh_map_m = btcoexist->bt_info.afh_map_m;
+
+	/* cmd_buffer[0] and [1] is filled by halbtc_send_bt_mp_operation() */
+	ret = halbtc_send_bt_mp_operation(btcoexist, BT_OP_GET_AFH_MAP_H,
+					  cmd_buffer, 2, 200);
+	if (!ret)
+		goto exit;
+
+	*afh_map_h = btcoexist->bt_info.afh_map_h;
+
+exit:
+	return ret;
+}
+
+/*****************************************************************
+ *         Extern functions called by other module
+ *****************************************************************/
+bool exhalbtc_initlize_variables(struct rtl_priv *rtlpriv)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return false;
+
+	halbtc_dbg_init();
+
+	btcoexist->btc_read_1byte = halbtc_read_1byte;
+	btcoexist->btc_write_1byte = halbtc_write_1byte;
+	btcoexist->btc_write_1byte_bitmask = halbtc_bitmask_write_1byte;
+	btcoexist->btc_read_2byte = halbtc_read_2byte;
+	btcoexist->btc_write_2byte = halbtc_write_2byte;
+	btcoexist->btc_read_4byte = halbtc_read_4byte;
+	btcoexist->btc_write_4byte = halbtc_write_4byte;
+	btcoexist->btc_write_local_reg_1byte = halbtc_write_local_reg_1byte;
+
+	btcoexist->btc_set_bb_reg = halbtc_set_bbreg;
+	btcoexist->btc_get_bb_reg = halbtc_get_bbreg;
+
+	btcoexist->btc_set_rf_reg = halbtc_set_rfreg;
+	btcoexist->btc_get_rf_reg = halbtc_get_rfreg;
+
+	btcoexist->btc_fill_h2c = halbtc_fill_h2c_cmd;
+	btcoexist->btc_disp_dbg_msg = halbtc_display_dbg_msg;
+
+	btcoexist->btc_get = halbtc_get;
+	btcoexist->btc_set = halbtc_set;
+	btcoexist->btc_set_bt_reg = halbtc_set_bt_reg;
+	btcoexist->btc_get_bt_reg = halbtc_get_bt_reg;
+
+	btcoexist->bt_info.bt_ctrl_buf_size = false;
+	btcoexist->bt_info.agg_buf_size = 5;
+
+	btcoexist->bt_info.increase_scan_dev_num = false;
+
+	btcoexist->btc_get_bt_coex_supported_feature =
+					halbtc_get_bt_coex_supported_feature;
+	btcoexist->btc_get_bt_coex_supported_version =
+					halbtc_get_bt_coex_supported_version;
+	btcoexist->btc_get_bt_phydm_version = halbtc_get_phydm_version;
+	btcoexist->btc_phydm_modify_ra_pcr_threshold =
+					halbtc_phydm_modify_ra_pcr_threshold;
+	btcoexist->btc_phydm_query_phy_counter = halbtc_phydm_query_phy_counter;
+	btcoexist->btc_get_ant_det_val_from_bt = halbtc_get_ant_det_val_from_bt;
+	btcoexist->btc_get_ble_scan_type_from_bt =
+					halbtc_get_ble_scan_type_from_bt;
+	btcoexist->btc_get_ble_scan_para_from_bt =
+					halbtc_get_ble_scan_para_from_bt;
+	btcoexist->btc_get_bt_afh_map_from_bt =
+					halbtc_get_bt_afh_map_from_bt;
+
+	init_completion(&btcoexist->bt_mp_comp);
+
+	return true;
+}
+
+bool exhalbtc_initlize_variables_wifi_only(struct rtl_priv *rtlpriv)
+{
+	struct wifi_only_cfg *wifionly_cfg = rtl_btc_wifi_only(rtlpriv);
+	struct wifi_only_haldata *wifionly_haldata;
+
+	if (!wifionly_cfg)
+		return false;
+
+	wifionly_cfg->adapter = rtlpriv;
+
+	switch (rtlpriv->rtlhal.interface) {
+	case INTF_PCI:
+		wifionly_cfg->chip_interface = BTC_INTF_PCI;
+		break;
+	case INTF_USB:
+		wifionly_cfg->chip_interface = BTC_INTF_USB;
+		break;
+	default:
+		wifionly_cfg->chip_interface = BTC_INTF_UNKNOWN;
+		break;
+	}
+
+	wifionly_haldata = &wifionly_cfg->haldata_info;
+
+	wifionly_haldata->customer_id = CUSTOMER_NORMAL;
+	wifionly_haldata->efuse_pg_antnum = rtl_get_hwpg_ant_num(rtlpriv);
+	wifionly_haldata->efuse_pg_antpath =
+					rtl_get_hwpg_single_ant_path(rtlpriv);
+	wifionly_haldata->rfe_type = rtl_get_hwpg_rfe_type(rtlpriv);
+	wifionly_haldata->ant_div_cfg = 0;
+
+	return true;
+}
+
+bool exhalbtc_bind_bt_coex_withadapter(void *adapter)
+{
+	struct rtl_priv *rtlpriv = adapter;
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+	u8 ant_num = 2, chip_type, single_ant_path = 0;
+
+	if (!btcoexist)
+		return false;
+
+	if (btcoexist->binded)
+		return false;
+
+	switch (rtlpriv->rtlhal.interface) {
+	case INTF_PCI:
+		btcoexist->chip_interface = BTC_INTF_PCI;
+		break;
+	case INTF_USB:
+		btcoexist->chip_interface = BTC_INTF_USB;
+		break;
+	default:
+		btcoexist->chip_interface = BTC_INTF_UNKNOWN;
+		break;
+	}
+
+	btcoexist->binded = true;
+	btcoexist->statistics.cnt_bind++;
+
+	btcoexist->adapter = adapter;
+
+	btcoexist->stack_info.profile_notified = false;
+
+	btcoexist->bt_info.bt_ctrl_agg_buf_size = false;
+	btcoexist->bt_info.agg_buf_size = 5;
+
+	btcoexist->bt_info.increase_scan_dev_num = false;
+	btcoexist->bt_info.miracast_plus_bt = false;
+
+	chip_type = rtl_get_hwpg_bt_type(rtlpriv);
+	exhalbtc_set_chip_type(btcoexist, chip_type);
+	ant_num = rtl_get_hwpg_ant_num(rtlpriv);
+	exhalbtc_set_ant_num(rtlpriv, BT_COEX_ANT_TYPE_PG, ant_num);
+
+	/* set default antenna position to main  port */
+	btcoexist->board_info.btdm_ant_pos = BTC_ANTENNA_AT_MAIN_PORT;
+
+	single_ant_path = rtl_get_hwpg_single_ant_path(rtlpriv);
+	exhalbtc_set_single_ant_path(btcoexist, single_ant_path);
+
+	if (rtl_get_hwpg_package_type(rtlpriv) == 0)
+		btcoexist->board_info.tfbga_package = false;
+	else if (rtl_get_hwpg_package_type(rtlpriv) == 1)
+		btcoexist->board_info.tfbga_package = false;
+	else
+		btcoexist->board_info.tfbga_package = true;
+
+	if (btcoexist->board_info.tfbga_package)
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], Package Type = TFBGA\n");
+	else
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+			 "[BTCoex], Package Type = Non-TFBGA\n");
+
+	btcoexist->board_info.rfe_type = rtl_get_hwpg_rfe_type(rtlpriv);
+	btcoexist->board_info.ant_div_cfg = 0;
+
+	return true;
+}
+
+void exhalbtc_power_on_setting(struct btc_coexist *btcoexist)
+{
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return;
+
+	btcoexist->statistics.cnt_power_on++;
+
+	if (IS_HARDWARE_TYPE_8822B(btcoexist->adapter)) {
+		if (btcoexist->board_info.btdm_ant_num == 1)
+			ex_btc8822b1ant_power_on_setting(btcoexist);
+		else if (btcoexist->board_info.btdm_ant_num == 2)
+			ex_btc8822b2ant_power_on_setting(btcoexist);
+	}
+}
+
+void exhalbtc_pre_load_firmware(struct btc_coexist *btcoexist)
+{
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return;
+
+	btcoexist->statistics.cnt_pre_load_firmware++;
+
+	if (IS_HARDWARE_TYPE_8822B(btcoexist->adapter)) {
+		if (btcoexist->board_info.btdm_ant_num == 1)
+			ex_btc8822b1ant_pre_load_firmware(btcoexist);
+		else if (btcoexist->board_info.btdm_ant_num == 2)
+			ex_btc8822b2ant_pre_load_firmware(btcoexist);
+	}
+}
+
+void exhalbtc_init_hw_config(struct btc_coexist *btcoexist, bool wifi_only)
+{
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return;
+
+	btcoexist->statistics.cnt_init_hw_config++;
+
+	if (IS_HARDWARE_TYPE_8822B(btcoexist->adapter)) {
+		if (btcoexist->board_info.btdm_ant_num == 1)
+			ex_btc8822b1ant_init_hw_config(btcoexist, wifi_only);
+		else if (btcoexist->board_info.btdm_ant_num == 2)
+			ex_btc8822b2ant_init_hw_config(btcoexist, wifi_only);
+
+		halbtc_set_default_port_id_cmd(btcoexist);
+		halbtc_send_wifi_port_id_cmd(btcoexist);
+	}
+}
+
+void exhalbtc_init_hw_config_wifi_only(struct wifi_only_cfg *wifionly_cfg)
+{
+	if (IS_HARDWARE_TYPE_8822B(wifionly_cfg->adapter))
+		ex_hal8822b_wifi_only_hw_config(wifionly_cfg);
+}
+
+void exhalbtc_init_coex_dm(struct btc_coexist *btcoexist)
+{
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return;
+
+	btcoexist->statistics.cnt_init_coex_dm++;
+
+	if (IS_HARDWARE_TYPE_8822B(btcoexist->adapter)) {
+		if (btcoexist->board_info.btdm_ant_num == 1)
+			ex_btc8822b1ant_init_coex_dm(btcoexist);
+		else if (btcoexist->board_info.btdm_ant_num == 2)
+			ex_btc8822b2ant_init_coex_dm(btcoexist);
+	}
+
+	btcoexist->initilized = true;
+}
+
+void exhalbtc_ips_notify(struct btc_coexist *btcoexist, u8 type)
+{
+	u8 ips_type;
+
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return;
+	btcoexist->statistics.cnt_ips_notify++;
+	if (btcoexist->manual_control)
+		return;
+
+	if (type == ERFOFF)
+		ips_type = BTC_IPS_ENTER;
+	else
+		ips_type = BTC_IPS_LEAVE;
+
+	halbtc_leave_low_power(btcoexist);
+
+	if (IS_HARDWARE_TYPE_8822B(btcoexist->adapter)) {
+		if (btcoexist->board_info.btdm_ant_num == 1)
+			ex_btc8822b1ant_ips_notify(btcoexist, ips_type);
+		else if (btcoexist->board_info.btdm_ant_num == 2)
+			ex_btc8822b2ant_ips_notify(btcoexist, ips_type);
+	}
+
+	halbtc_normal_low_power(btcoexist);
+}
+
+void exhalbtc_lps_notify(struct btc_coexist *btcoexist, u8 type)
+{
+	u8 lps_type;
+
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return;
+	btcoexist->statistics.cnt_lps_notify++;
+	if (btcoexist->manual_control)
+		return;
+
+	if (type == EACTIVE)
+		lps_type = BTC_LPS_DISABLE;
+	else
+		lps_type = BTC_LPS_ENABLE;
+
+	if (IS_HARDWARE_TYPE_8822B(btcoexist->adapter)) {
+		if (btcoexist->board_info.btdm_ant_num == 1)
+			ex_btc8822b1ant_lps_notify(btcoexist, lps_type);
+		else if (btcoexist->board_info.btdm_ant_num == 2)
+			ex_btc8822b2ant_lps_notify(btcoexist, lps_type);
+	}
+}
+
+void exhalbtc_scan_notify(struct btc_coexist *btcoexist, u8 type)
+{
+	u8 scan_type;
+
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return;
+	btcoexist->statistics.cnt_scan_notify++;
+	if (btcoexist->manual_control)
+		return;
+
+	if (type)
+		scan_type = BTC_SCAN_START;
+	else
+		scan_type = BTC_SCAN_FINISH;
+
+	halbtc_leave_low_power(btcoexist);
+
+	if (IS_HARDWARE_TYPE_8822B(btcoexist->adapter)) {
+		if (btcoexist->board_info.btdm_ant_num == 1)
+			ex_btc8822b1ant_scan_notify(btcoexist, scan_type);
+		else if (btcoexist->board_info.btdm_ant_num == 2)
+			ex_btc8822b2ant_scan_notify(btcoexist, scan_type);
+	}
+
+	halbtc_normal_low_power(btcoexist);
+}
+
+void exhalbtc_scan_notify_wifi_only(struct wifi_only_cfg *wifionly_cfg,
+				    u8 is_5g)
+{
+	if (IS_HARDWARE_TYPE_8822B(wifionly_cfg->adapter))
+		ex_hal8822b_wifi_only_scannotify(wifionly_cfg, is_5g);
+}
+
+void exhalbtc_connect_notify(struct btc_coexist *btcoexist, u8 action)
+{
+	u8 asso_type;
+
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return;
+	btcoexist->statistics.cnt_connect_notify++;
+	if (btcoexist->manual_control)
+		return;
+
+	if (action)
+		asso_type = BTC_ASSOCIATE_START;
+	else
+		asso_type = BTC_ASSOCIATE_FINISH;
+
+	halbtc_leave_low_power(btcoexist);
+
+	if (IS_HARDWARE_TYPE_8822B(btcoexist->adapter)) {
+		if (btcoexist->board_info.btdm_ant_num == 1)
+			ex_btc8822b1ant_connect_notify(btcoexist, asso_type);
+		else if (btcoexist->board_info.btdm_ant_num == 2)
+			ex_btc8822b2ant_connect_notify(btcoexist, asso_type);
+	}
+
+	halbtc_normal_low_power(btcoexist);
+}
+
+void exhalbtc_mediastatus_notify(struct btc_coexist *btcoexist,
+				 enum rt_media_status media_status)
+{
+	u8 status;
+
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return;
+	btcoexist->statistics.cnt_media_status_notify++;
+	if (btcoexist->manual_control)
+		return;
+
+	if (media_status == RT_MEDIA_CONNECT)
+		status = BTC_MEDIA_CONNECT;
+	else
+		status = BTC_MEDIA_DISCONNECT;
+
+	halbtc_leave_low_power(btcoexist);
+
+	if (IS_HARDWARE_TYPE_8822B(btcoexist->adapter)) {
+		if (btcoexist->board_info.btdm_ant_num == 1)
+			ex_btc8822b1ant_media_status_notify(btcoexist, status);
+		else if (btcoexist->board_info.btdm_ant_num == 2)
+			ex_btc8822b2ant_media_status_notify(btcoexist, status);
+	}
+
+	halbtc_normal_low_power(btcoexist);
+}
+
+void exhalbtc_special_packet_notify(struct btc_coexist *btcoexist, u8 pkt_type)
+{
+	u8 packet_type;
+
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return;
+	btcoexist->statistics.cnt_special_packet_notify++;
+	if (btcoexist->manual_control)
+		return;
+
+	if (pkt_type == PACKET_DHCP) {
+		packet_type = BTC_PACKET_DHCP;
+	} else if (pkt_type == PACKET_EAPOL) {
+		packet_type = BTC_PACKET_EAPOL;
+	} else if (pkt_type == PACKET_ARP) {
+		packet_type = BTC_PACKET_ARP;
+	} else {
+		packet_type = BTC_PACKET_UNKNOWN;
+		return;
+	}
+
+	halbtc_leave_low_power(btcoexist);
+
+	if (IS_HARDWARE_TYPE_8822B(btcoexist->adapter)) {
+		if (btcoexist->board_info.btdm_ant_num == 1)
+			ex_btc8822b1ant_specific_packet_notify(btcoexist,
+							       packet_type);
+		else if (btcoexist->board_info.btdm_ant_num == 2)
+			ex_btc8822b2ant_specific_packet_notify(btcoexist,
+							       packet_type);
+	}
+
+	halbtc_normal_low_power(btcoexist);
+}
+
+void exhalbtc_bt_info_notify(struct btc_coexist *btcoexist,
+			     u8 *tmp_buf, u8 length)
+{
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return;
+	btcoexist->statistics.cnt_bt_info_notify++;
+
+	halbtc_leave_low_power(btcoexist);
+
+	if (IS_HARDWARE_TYPE_8822B(btcoexist->adapter)) {
+		if (btcoexist->board_info.btdm_ant_num == 1)
+			ex_btc8822b1ant_bt_info_notify(btcoexist, tmp_buf,
+						       length);
+		else if (btcoexist->board_info.btdm_ant_num == 2)
+			ex_btc8822b2ant_bt_info_notify(btcoexist, tmp_buf,
+						       length);
+	}
+
+	halbtc_normal_low_power(btcoexist);
+}
+
+void exhalbtc_rf_status_notify(struct btc_coexist *btcoexist, u8 type)
+{
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return;
+
+	if (IS_HARDWARE_TYPE_8822B(btcoexist->adapter)) {
+		if (btcoexist->board_info.btdm_ant_num == 1)
+			ex_btc8822b1ant_rf_status_notify(btcoexist, type);
+		else if (btcoexist->board_info.btdm_ant_num == 2)
+			ex_btc8822b2ant_rf_status_notify(btcoexist, type);
+	}
+}
+
+void exhalbtc_stack_operation_notify(struct btc_coexist *btcoexist, u8 type)
+{
+	u8 stack_op_type;
+
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return;
+	btcoexist->statistics.cnt_stack_operation_notify++;
+	if (btcoexist->manual_control)
+		return;
+
+	if ((type == HCI_BT_OP_INQUIRY_START) ||
+	    (type == HCI_BT_OP_PAGING_START) ||
+	    (type == HCI_BT_OP_PAIRING_START)) {
+		stack_op_type = BTC_STACK_OP_INQ_PAGE_PAIR_START;
+	} else if ((type == HCI_BT_OP_INQUIRY_FINISH) ||
+		   (type == HCI_BT_OP_PAGING_SUCCESS) ||
+		   (type == HCI_BT_OP_PAGING_UNSUCCESS) ||
+		   (type == HCI_BT_OP_PAIRING_FINISH)) {
+		stack_op_type = BTC_STACK_OP_INQ_PAGE_PAIR_FINISH;
+	} else {
+		stack_op_type = BTC_STACK_OP_NONE;
+	}
+}
+
+void exhalbtc_halt_notify(struct btc_coexist *btcoexist)
+{
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return;
+
+	if (IS_HARDWARE_TYPE_8822B(btcoexist->adapter)) {
+		if (btcoexist->board_info.btdm_ant_num == 1)
+			ex_btc8822b1ant_halt_notify(btcoexist);
+		else if (btcoexist->board_info.btdm_ant_num == 2)
+			ex_btc8822b2ant_halt_notify(btcoexist);
+	}
+
+	btcoexist->binded = false;
+}
+
+void exhalbtc_pnp_notify(struct btc_coexist *btcoexist, u8 pnp_state)
+{
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return;
+
+	/* currently only 1ant we have to do the notification,
+	 * once pnp is notified to sleep state, we have to leave LPS that
+	 * we can sleep normally.
+	 */
+
+	if (IS_HARDWARE_TYPE_8822B(btcoexist->adapter)) {
+		if (btcoexist->board_info.btdm_ant_num == 1)
+			ex_btc8822b1ant_pnp_notify(btcoexist, pnp_state);
+		else if (btcoexist->board_info.btdm_ant_num == 2)
+			ex_btc8822b2ant_pnp_notify(btcoexist, pnp_state);
+	}
+}
+
+void exhalbtc_coex_dm_switch(struct btc_coexist *btcoexist)
+{
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return;
+	btcoexist->statistics.cnt_coex_dm_switch++;
+
+	halbtc_leave_low_power(btcoexist);
+
+	halbtc_normal_low_power(btcoexist);
+}
+
+void exhalbtc_periodical(struct btc_coexist *btcoexist)
+{
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return;
+	btcoexist->statistics.cnt_periodical++;
+
+	halbtc_leave_low_power(btcoexist);
+
+	if (IS_HARDWARE_TYPE_8822B(btcoexist->adapter)) {
+		if (btcoexist->board_info.btdm_ant_num == 1)
+			ex_btc8822b1ant_periodical(btcoexist);
+		else if (btcoexist->board_info.btdm_ant_num == 2)
+			ex_btc8822b2ant_periodical(btcoexist);
+	}
+
+	halbtc_normal_low_power(btcoexist);
+}
+
+void exhalbtc_dbg_control(struct btc_coexist *btcoexist,
+			  u8 code, u8 len, u8 *data)
+{
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return;
+	btcoexist->statistics.cnt_dbg_ctrl++;
+
+	halbtc_leave_low_power(btcoexist);
+
+	halbtc_normal_low_power(btcoexist);
+}
+
+void exhalbtc_antenna_detection(struct btc_coexist *btcoexist, u32 cent_freq,
+				u32 offset, u32 span, u32 seconds)
+{
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return;
+}
+
+void exhalbtc_stack_update_profile_info(void)
+{
+}
+
+void exhalbtc_update_min_bt_rssi(struct btc_coexist *btcoexist, s8 bt_rssi)
+{
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return;
+
+	btcoexist->stack_info.min_bt_rssi = bt_rssi;
+}
+
+void exhalbtc_set_hci_version(struct btc_coexist *btcoexist, u16 hci_version)
+{
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return;
+
+	btcoexist->stack_info.hci_version = hci_version;
+}
+
+void exhalbtc_set_bt_patch_version(struct btc_coexist *btcoexist,
+				   u16 bt_hci_version, u16 bt_patch_version)
+{
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return;
+
+	btcoexist->bt_info.bt_real_fw_ver = bt_patch_version;
+	btcoexist->bt_info.bt_hci_ver = bt_hci_version;
+}
+
+void exhalbtc_set_chip_type(struct btc_coexist *btcoexist, u8 chip_type)
+{
+	switch (chip_type) {
+	default:
+	case BT_2WIRE:
+	case BT_ISSC_3WIRE:
+	case BT_ACCEL:
+	case BT_RTL8756:
+		btcoexist->board_info.bt_chip_type = BTC_CHIP_UNDEF;
+		break;
+	case BT_CSR_BC4:
+		btcoexist->board_info.bt_chip_type = BTC_CHIP_CSR_BC4;
+		break;
+	case BT_CSR_BC8:
+		btcoexist->board_info.bt_chip_type = BTC_CHIP_CSR_BC8;
+		break;
+	case BT_RTL8723A:
+		btcoexist->board_info.bt_chip_type = BTC_CHIP_RTL8723A;
+		break;
+	case BT_RTL8821A:
+		btcoexist->board_info.bt_chip_type = BTC_CHIP_RTL8821;
+		break;
+	case BT_RTL8723B:
+		btcoexist->board_info.bt_chip_type = BTC_CHIP_RTL8723B;
+		break;
+	}
+}
+
+void exhalbtc_set_ant_num(struct rtl_priv *rtlpriv, u8 type, u8 ant_num)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return;
+
+	if (type == BT_COEX_ANT_TYPE_PG) {
+		btcoexist->board_info.pg_ant_num = ant_num;
+		btcoexist->board_info.btdm_ant_num = ant_num;
+	} else if (type == BT_COEX_ANT_TYPE_ANTDIV) {
+		btcoexist->board_info.btdm_ant_num = ant_num;
+	} else if (type == BT_COEX_ANT_TYPE_DETECTED) {
+		btcoexist->board_info.btdm_ant_num = ant_num;
+		if (rtlpriv->cfg->mod_params->ant_sel == 1)
+			btcoexist->board_info.btdm_ant_pos =
+				BTC_ANTENNA_AT_AUX_PORT;
+		else
+			btcoexist->board_info.btdm_ant_pos =
+				BTC_ANTENNA_AT_MAIN_PORT;
+	}
+}
+
+/* Currently used by 8723b only, S0 or S1 */
+void exhalbtc_set_single_ant_path(struct btc_coexist *btcoexist,
+				  u8 single_ant_path)
+{
+	btcoexist->board_info.single_ant_path = single_ant_path;
+}
+
+void exhalbtc_display_bt_coex_info(struct btc_coexist *btcoexist,
+				   struct seq_file *m)
+{
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return;
+
+	halbtc_leave_low_power(btcoexist);
+
+	if (IS_HARDWARE_TYPE_8822B(btcoexist->adapter)) {
+		if (btcoexist->board_info.btdm_ant_num == 1)
+			ex_btc8822b1ant_display_coex_info(btcoexist, m);
+		else if (btcoexist->board_info.btdm_ant_num == 2)
+			ex_btc8822b2ant_display_coex_info(btcoexist, m);
+	}
+
+	halbtc_normal_low_power(btcoexist);
+}
+
+void exhalbtc_switch_band_notify(struct btc_coexist *btcoexist, u8 type)
+{
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return;
+
+	if (btcoexist->manual_control)
+		return;
+
+	halbtc_leave_low_power(btcoexist);
+
+	if (IS_HARDWARE_TYPE_8822B(btcoexist->adapter)) {
+		if (btcoexist->board_info.btdm_ant_num == 1)
+			ex_btc8822b1ant_switchband_notify(btcoexist, type);
+		else if (btcoexist->board_info.btdm_ant_num == 2)
+			ex_btc8822b2ant_switchband_notify(btcoexist, type);
+	}
+
+	halbtc_normal_low_power(btcoexist);
+}
+
+void exhalbtc_switch_band_notify_wifi_only(struct wifi_only_cfg *wifionly_cfg,
+					   u8 is_5g)
+{
+	if (IS_HARDWARE_TYPE_8822B(wifionly_cfg->adapter))
+		ex_hal8822b_wifi_only_switchbandnotify(wifionly_cfg, is_5g);
+}
diff --git a/drivers/staging/rtlwifi/btcoexist/halbtcoutsrc.h b/drivers/staging/rtlwifi/btcoexist/halbtcoutsrc.h
new file mode 100644
index 000000000000..8913983b8ad8
--- /dev/null
+++ b/drivers/staging/rtlwifi/btcoexist/halbtcoutsrc.h
@@ -0,0 +1,802 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2012  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef	__HALBTC_OUT_SRC_H__
+#define __HALBTC_OUT_SRC_H__
+
+#include	"../wifi.h"
+
+#define		BTC_COEX_OFFLOAD			0
+
+#define		NORMAL_EXEC				false
+#define		FORCE_EXEC				true
+
+#define		BTC_RF_OFF				0x0
+#define		BTC_RF_ON				0x1
+
+#define		BTC_RF_A				RF90_PATH_A
+#define		BTC_RF_B				RF90_PATH_B
+#define		BTC_RF_C				RF90_PATH_C
+#define		BTC_RF_D				RF90_PATH_D
+
+#define		BTC_SMSP				SINGLEMAC_SINGLEPHY
+#define		BTC_DMDP				DUALMAC_DUALPHY
+#define		BTC_DMSP				DUALMAC_SINGLEPHY
+#define		BTC_MP_UNKNOWN				0xff
+
+#define		IN
+#define		OUT
+
+#define		BT_TMP_BUF_SIZE				100
+
+#define		BT_COEX_ANT_TYPE_PG			0
+#define		BT_COEX_ANT_TYPE_ANTDIV			1
+#define		BT_COEX_ANT_TYPE_DETECTED		2
+
+#define		BTC_MIMO_PS_STATIC			0
+#define		BTC_MIMO_PS_DYNAMIC			1
+
+#define		BTC_RATE_DISABLE			0
+#define		BTC_RATE_ENABLE				1
+
+/* single Antenna definition */
+#define		BTC_ANT_PATH_WIFI			0
+#define		BTC_ANT_PATH_BT				1
+#define		BTC_ANT_PATH_PTA			2
+#define		BTC_ANT_PATH_WIFI5G			3
+#define		BTC_ANT_PATH_AUTO			4
+/* dual Antenna definition */
+#define		BTC_ANT_WIFI_AT_MAIN			0
+#define		BTC_ANT_WIFI_AT_AUX			1
+#define		BTC_ANT_WIFI_AT_DIVERSITY	2
+/* coupler Antenna definition */
+#define		BTC_ANT_WIFI_AT_CPL_MAIN		0
+#define		BTC_ANT_WIFI_AT_CPL_AUX			1
+
+enum btc_bt_reg_type {
+	BTC_BT_REG_RF		= 0,
+	BTC_BT_REG_MODEM	= 1,
+	BTC_BT_REG_BLUEWIZE	= 2,
+	BTC_BT_REG_VENDOR	= 3,
+	BTC_BT_REG_LE		= 4,
+	BTC_BT_REG_MAX
+};
+
+enum btc_chip_interface {
+	BTC_INTF_UNKNOWN	= 0,
+	BTC_INTF_PCI		= 1,
+	BTC_INTF_USB		= 2,
+	BTC_INTF_SDIO		= 3,
+	BTC_INTF_GSPI		= 4,
+	BTC_INTF_MAX
+};
+
+enum btc_chip_type {
+	BTC_CHIP_UNDEF		= 0,
+	BTC_CHIP_CSR_BC4	= 1,
+	BTC_CHIP_CSR_BC8	= 2,
+	BTC_CHIP_RTL8723A	= 3,
+	BTC_CHIP_RTL8821	= 4,
+	BTC_CHIP_RTL8723B	= 5,
+	BTC_CHIP_MAX
+};
+
+enum btc_msg_type {
+	BTC_MSG_INTERFACE	= 0x0,
+	BTC_MSG_ALGORITHM	= 0x1,
+	BTC_MSG_MAX
+};
+
+/* following is for BTC_MSG_INTERFACE */
+#define		INTF_INIT				BIT0
+#define		INTF_NOTIFY				BIT2
+
+/* following is for BTC_ALGORITHM */
+#define		ALGO_BT_RSSI_STATE			BIT0
+#define		ALGO_WIFI_RSSI_STATE			BIT1
+#define		ALGO_BT_MONITOR				BIT2
+#define		ALGO_TRACE				BIT3
+#define		ALGO_TRACE_FW				BIT4
+#define		ALGO_TRACE_FW_DETAIL			BIT5
+#define		ALGO_TRACE_FW_EXEC			BIT6
+#define		ALGO_TRACE_SW				BIT7
+#define		ALGO_TRACE_SW_DETAIL			BIT8
+#define		ALGO_TRACE_SW_EXEC			BIT9
+
+/* following is for wifi link status */
+#define		WIFI_STA_CONNECTED			BIT0
+#define		WIFI_AP_CONNECTED			BIT1
+#define		WIFI_HS_CONNECTED			BIT2
+#define		WIFI_P2P_GO_CONNECTED			BIT3
+#define		WIFI_P2P_GC_CONNECTED			BIT4
+
+#define	BTC_RSSI_HIGH(_rssi_)	\
+	((_rssi_ == BTC_RSSI_STATE_HIGH ||	\
+	  _rssi_ == BTC_RSSI_STATE_STAY_HIGH) ? true : false)
+#define	BTC_RSSI_MEDIUM(_rssi_)	\
+	((_rssi_ == BTC_RSSI_STATE_MEDIUM ||	\
+	  _rssi_ == BTC_RSSI_STATE_STAY_MEDIUM) ? true : false)
+#define	BTC_RSSI_LOW(_rssi_)	\
+	((_rssi_ == BTC_RSSI_STATE_LOW ||	\
+	  _rssi_ == BTC_RSSI_STATE_STAY_LOW) ? true : false)
+
+enum btc_power_save_type {
+	BTC_PS_WIFI_NATIVE = 0,
+	BTC_PS_LPS_ON = 1,
+	BTC_PS_LPS_OFF = 2,
+	BTC_PS_LPS_MAX
+};
+
+struct btc_board_info {
+	/* The following is some board information */
+	u8 bt_chip_type;
+	u8 pg_ant_num;	/* pg ant number */
+	u8 btdm_ant_num;	/* ant number for btdm */
+	u8 btdm_ant_num_by_ant_det;
+	u8 btdm_ant_pos;
+	u8 single_ant_path; /* current used for 8723b only, 1=>s0,  0=>s1 */
+	bool tfbga_package;
+	bool btdm_ant_det_finish;
+
+	u8 rfe_type;
+	u8 ant_div_cfg;
+};
+
+enum btc_dbg_opcode {
+	BTC_DBG_SET_COEX_NORMAL = 0x0,
+	BTC_DBG_SET_COEX_WIFI_ONLY = 0x1,
+	BTC_DBG_SET_COEX_BT_ONLY = 0x2,
+	BTC_DBG_MAX
+};
+
+enum btc_rssi_state {
+	BTC_RSSI_STATE_HIGH = 0x0,
+	BTC_RSSI_STATE_MEDIUM = 0x1,
+	BTC_RSSI_STATE_LOW = 0x2,
+	BTC_RSSI_STATE_STAY_HIGH = 0x3,
+	BTC_RSSI_STATE_STAY_MEDIUM = 0x4,
+	BTC_RSSI_STATE_STAY_LOW = 0x5,
+	BTC_RSSI_MAX
+};
+
+enum btc_wifi_role {
+	BTC_ROLE_STATION = 0x0,
+	BTC_ROLE_AP = 0x1,
+	BTC_ROLE_IBSS = 0x2,
+	BTC_ROLE_HS_MODE = 0x3,
+	BTC_ROLE_MAX
+};
+
+enum btc_wireless_freq {
+	BTC_FREQ_2_4G = 0x0,
+	BTC_FREQ_5G = 0x1,
+	BTC_FREQ_MAX
+};
+
+enum btc_wifi_bw_mode {
+	BTC_WIFI_BW_LEGACY = 0x0,
+	BTC_WIFI_BW_HT20 = 0x1,
+	BTC_WIFI_BW_HT40 = 0x2,
+	BTC_WIFI_BW_HT80 = 0x3,
+	BTC_WIFI_BW_MAX
+};
+
+enum btc_wifi_traffic_dir {
+	BTC_WIFI_TRAFFIC_TX = 0x0,
+	BTC_WIFI_TRAFFIC_RX = 0x1,
+	BTC_WIFI_TRAFFIC_MAX
+};
+
+enum btc_wifi_pnp {
+	BTC_WIFI_PNP_WAKE_UP = 0x0,
+	BTC_WIFI_PNP_SLEEP = 0x1,
+	BTC_WIFI_PNP_SLEEP_KEEP_ANT = 0x2,
+	BTC_WIFI_PNP_MAX
+};
+
+enum btc_iot_peer {
+	BTC_IOT_PEER_UNKNOWN = 0,
+	BTC_IOT_PEER_REALTEK = 1,
+	BTC_IOT_PEER_REALTEK_92SE = 2,
+	BTC_IOT_PEER_BROADCOM = 3,
+	BTC_IOT_PEER_RALINK = 4,
+	BTC_IOT_PEER_ATHEROS = 5,
+	BTC_IOT_PEER_CISCO = 6,
+	BTC_IOT_PEER_MERU = 7,
+	BTC_IOT_PEER_MARVELL = 8,
+	BTC_IOT_PEER_REALTEK_SOFTAP = 9,
+	BTC_IOT_PEER_SELF_SOFTAP = 10, /* Self is SoftAP */
+	BTC_IOT_PEER_AIRGO = 11,
+	BTC_IOT_PEER_REALTEK_JAGUAR_BCUTAP = 12,
+	BTC_IOT_PEER_REALTEK_JAGUAR_CCUTAP = 13,
+	BTC_IOT_PEER_MAX,
+};
+
+/* for 8723b-d cut large current issue */
+enum bt_wifi_coex_state {
+	BTC_WIFI_STAT_INIT,
+	BTC_WIFI_STAT_IQK,
+	BTC_WIFI_STAT_NORMAL_OFF,
+	BTC_WIFI_STAT_MP_OFF,
+	BTC_WIFI_STAT_NORMAL,
+	BTC_WIFI_STAT_ANT_DIV,
+	BTC_WIFI_STAT_MAX
+};
+
+enum bt_ant_type {
+	BTC_ANT_TYPE_0,
+	BTC_ANT_TYPE_1,
+	BTC_ANT_TYPE_2,
+	BTC_ANT_TYPE_3,
+	BTC_ANT_TYPE_4,
+	BTC_ANT_TYPE_MAX
+};
+
+enum btc_get_type {
+	/* type bool */
+	BTC_GET_BL_HS_OPERATION,
+	BTC_GET_BL_HS_CONNECTING,
+	BTC_GET_BL_WIFI_CONNECTED,
+	BTC_GET_BL_WIFI_BUSY,
+	BTC_GET_BL_WIFI_SCAN,
+	BTC_GET_BL_WIFI_LINK,
+	BTC_GET_BL_WIFI_DHCP,
+	BTC_GET_BL_WIFI_SOFTAP_IDLE,
+	BTC_GET_BL_WIFI_SOFTAP_LINKING,
+	BTC_GET_BL_WIFI_IN_EARLY_SUSPEND,
+	BTC_GET_BL_WIFI_ROAM,
+	BTC_GET_BL_WIFI_4_WAY_PROGRESS,
+	BTC_GET_BL_WIFI_UNDER_5G,
+	BTC_GET_BL_WIFI_AP_MODE_ENABLE,
+	BTC_GET_BL_WIFI_ENABLE_ENCRYPTION,
+	BTC_GET_BL_WIFI_UNDER_B_MODE,
+	BTC_GET_BL_EXT_SWITCH,
+	BTC_GET_BL_WIFI_IS_IN_MP_MODE,
+	BTC_GET_BL_IS_ASUS_8723B,
+	BTC_GET_BL_FW_READY,
+	BTC_GET_BL_RF4CE_CONNECTED,
+
+	/* type s4Byte */
+	BTC_GET_S4_WIFI_RSSI,
+	BTC_GET_S4_HS_RSSI,
+
+	/* type u32 */
+	BTC_GET_U4_WIFI_BW,
+	BTC_GET_U4_WIFI_TRAFFIC_DIRECTION,
+	BTC_GET_U4_WIFI_FW_VER,
+	BTC_GET_U4_WIFI_LINK_STATUS,
+	BTC_GET_U4_BT_PATCH_VER,
+	BTC_GET_U4_VENDOR,
+	BTC_GET_U4_SUPPORTED_VERSION,
+	BTC_GET_U4_SUPPORTED_FEATURE,
+	BTC_GET_U4_WIFI_IQK_TOTAL,
+	BTC_GET_U4_WIFI_IQK_OK,
+	BTC_GET_U4_WIFI_IQK_FAIL,
+
+	/* type u1Byte */
+	BTC_GET_U1_WIFI_DOT11_CHNL,
+	BTC_GET_U1_WIFI_CENTRAL_CHNL,
+	BTC_GET_U1_WIFI_HS_CHNL,
+	BTC_GET_U1_MAC_PHY_MODE,
+	BTC_GET_U1_AP_NUM,
+	BTC_GET_U1_ANT_TYPE,
+	BTC_GET_U1_IOT_PEER,
+
+	/* for 1Ant */
+	BTC_GET_U1_LPS_MODE,
+	BTC_GET_BL_BT_SCO_BUSY,
+
+	/* for test mode */
+	BTC_GET_DRIVER_TEST_CFG,
+	BTC_GET_MAX
+};
+
+enum btc_vendor {
+	BTC_VENDOR_LENOVO,
+	BTC_VENDOR_ASUS,
+	BTC_VENDOR_OTHER
+};
+
+enum btc_set_type {
+	/* type bool */
+	BTC_SET_BL_BT_DISABLE,
+	BTC_SET_BL_BT_ENABLE_DISABLE_CHANGE,
+	BTC_SET_BL_BT_TRAFFIC_BUSY,
+	BTC_SET_BL_BT_LIMITED_DIG,
+	BTC_SET_BL_FORCE_TO_ROAM,
+	BTC_SET_BL_TO_REJ_AP_AGG_PKT,
+	BTC_SET_BL_BT_CTRL_AGG_SIZE,
+	BTC_SET_BL_INC_SCAN_DEV_NUM,
+	BTC_SET_BL_BT_TX_RX_MASK,
+	BTC_SET_BL_MIRACAST_PLUS_BT,
+
+	/* type u1Byte */
+	BTC_SET_U1_RSSI_ADJ_VAL_FOR_AGC_TABLE_ON,
+	BTC_SET_UI_SCAN_SIG_COMPENSATION,
+	BTC_SET_U1_AGG_BUF_SIZE,
+
+	/* type trigger some action */
+	BTC_SET_ACT_GET_BT_RSSI,
+	BTC_SET_ACT_AGGREGATE_CTRL,
+	BTC_SET_ACT_ANTPOSREGRISTRY_CTRL,
+
+	/********* for 1Ant **********/
+	/* type bool */
+	BTC_SET_BL_BT_SCO_BUSY,
+	/* type u1Byte */
+	BTC_SET_U1_RSSI_ADJ_VAL_FOR_1ANT_COEX_TYPE,
+	BTC_SET_U1_LPS_VAL,
+	BTC_SET_U1_RPWM_VAL,
+	BTC_SET_U1_1ANT_LPS,
+	BTC_SET_U1_1ANT_RPWM,
+	/* type trigger some action */
+	BTC_SET_ACT_LEAVE_LPS,
+	BTC_SET_ACT_ENTER_LPS,
+	BTC_SET_ACT_NORMAL_LPS,
+	BTC_SET_ACT_INC_FORCE_EXEC_PWR_CMD_CNT,
+	BTC_SET_ACT_DISABLE_LOW_POWER,
+	BTC_SET_ACT_UPDATE_RAMASK,
+	BTC_SET_ACT_SEND_MIMO_PS,
+	/* BT Coex related */
+	BTC_SET_ACT_CTRL_BT_INFO,
+	BTC_SET_ACT_CTRL_BT_COEX,
+	BTC_SET_ACT_CTRL_8723B_ANT,
+	/***************************/
+	BTC_SET_MAX
+};
+
+enum btc_dbg_disp_type {
+	BTC_DBG_DISP_COEX_STATISTICS = 0x0,
+	BTC_DBG_DISP_BT_LINK_INFO = 0x1,
+	BTC_DBG_DISP_BT_FW_VER = 0x2,
+	BTC_DBG_DISP_FW_PWR_MODE_CMD = 0x3,
+	BTC_DBG_DISP_WIFI_STATUS = 0x04,
+	BTC_DBG_DISP_MAX
+};
+
+enum btc_notify_type_ips {
+	BTC_IPS_LEAVE = 0x0,
+	BTC_IPS_ENTER = 0x1,
+	BTC_IPS_MAX
+};
+
+enum btc_notify_type_lps {
+	BTC_LPS_DISABLE = 0x0,
+	BTC_LPS_ENABLE = 0x1,
+	BTC_LPS_MAX
+};
+
+enum btc_notify_type_scan {
+	BTC_SCAN_FINISH = 0x0,
+	BTC_SCAN_START = 0x1,
+	BTC_SCAN_START_2G = 0x2,
+	BTC_SCAN_MAX
+};
+
+enum btc_notify_type_switchband {
+	BTC_NOT_SWITCH = 0x0,
+	BTC_SWITCH_TO_24G = 0x1,
+	BTC_SWITCH_TO_5G = 0x2,
+	BTC_SWITCH_TO_24G_NOFORSCAN = 0x3,
+	BTC_SWITCH_MAX
+};
+
+enum btc_notify_type_associate {
+	BTC_ASSOCIATE_FINISH = 0x0,
+	BTC_ASSOCIATE_START = 0x1,
+	BTC_ASSOCIATE_5G_FINISH = 0x2,
+	BTC_ASSOCIATE_5G_START = 0x3,
+	BTC_ASSOCIATE_MAX
+};
+
+enum btc_notify_type_media_status {
+	BTC_MEDIA_DISCONNECT = 0x0,
+	BTC_MEDIA_CONNECT = 0x1,
+	BTC_MEDIA_MAX
+};
+
+enum btc_notify_type_special_packet {
+	BTC_PACKET_UNKNOWN = 0x0,
+	BTC_PACKET_DHCP = 0x1,
+	BTC_PACKET_ARP = 0x2,
+	BTC_PACKET_EAPOL = 0x3,
+	BTC_PACKET_MAX
+};
+
+enum hci_ext_bt_operation {
+	HCI_BT_OP_NONE = 0x0,
+	HCI_BT_OP_INQUIRY_START = 0x1,
+	HCI_BT_OP_INQUIRY_FINISH = 0x2,
+	HCI_BT_OP_PAGING_START = 0x3,
+	HCI_BT_OP_PAGING_SUCCESS = 0x4,
+	HCI_BT_OP_PAGING_UNSUCCESS = 0x5,
+	HCI_BT_OP_PAIRING_START = 0x6,
+	HCI_BT_OP_PAIRING_FINISH = 0x7,
+	HCI_BT_OP_BT_DEV_ENABLE = 0x8,
+	HCI_BT_OP_BT_DEV_DISABLE = 0x9,
+	HCI_BT_OP_MAX
+};
+
+enum btc_notify_type_stack_operation {
+	BTC_STACK_OP_NONE = 0x0,
+	BTC_STACK_OP_INQ_PAGE_PAIR_START = 0x1,
+	BTC_STACK_OP_INQ_PAGE_PAIR_FINISH = 0x2,
+	BTC_STACK_OP_MAX
+};
+
+typedef u8 (*bfp_btc_r1)(void *btc_context, u32 reg_addr);
+
+typedef u16 (*bfp_btc_r2)(void *btc_context, u32 reg_addr);
+
+typedef u32 (*bfp_btc_r4)(void *btc_context, u32 reg_addr);
+
+typedef void (*bfp_btc_w1)(void *btc_context, u32 reg_addr, u32 data);
+
+typedef void (*bfp_btc_w1_bit_mak)(void *btc_context, u32 reg_addr,
+				   u32 bit_mask, u8 data1b);
+
+typedef void (*bfp_btc_w2)(void *btc_context, u32 reg_addr, u16 data);
+
+typedef void (*bfp_btc_w4)(void *btc_context, u32 reg_addr, u32 data);
+
+typedef void (*bfp_btc_local_reg_w1)(void *btc_context, u32 reg_addr, u8 data);
+typedef void (*bfp_btc_wr_1byte_bit_mask)(void *btc_context, u32 reg_addr,
+					  u8 bit_mask, u8 data);
+
+typedef void (*bfp_btc_set_bb_reg)(void *btc_context, u32 reg_addr,
+				   u32 bit_mask, u32 data);
+
+typedef u32 (*bfp_btc_get_bb_reg)(void *btc_context, u32 reg_addr,
+				  u32 bit_mask);
+
+typedef void (*bfp_btc_set_rf_reg)(void *btc_context, u8 rf_path, u32 reg_addr,
+				   u32 bit_mask, u32 data);
+
+typedef u32 (*bfp_btc_get_rf_reg)(void *btc_context, u8 rf_path,
+				  u32 reg_addr, u32 bit_mask);
+
+typedef void (*bfp_btc_fill_h2c)(void *btc_context, u8 element_id,
+				 u32 cmd_len, u8 *cmd_buffer);
+
+typedef	bool (*bfp_btc_get)(void *btcoexist, u8 get_type, void *out_buf);
+
+typedef	bool (*bfp_btc_set)(void *btcoexist, u8 set_type, void *in_buf);
+
+typedef u32 (*bfp_btc_get_bt_coex_supported_feature)(void *btcoexist);
+
+typedef u32 (*bfp_btc_get_bt_coex_supported_version)(void *btcoexist);
+
+typedef u32 (*bfp_btc_get_bt_phydm_version)(void *btcoexist);
+
+typedef void (*bfp_btc_phydm_modify_ra_pcr_threshold)(void *btcoexist,
+					  u8 ra_offset_direction,
+					  u8 ra_threshold_offset);
+
+typedef u32 (*bfp_btc_phydm_query_phy_counter)(void *btcoexist,
+					       const char *info_type);
+
+typedef u8 (*bfp_btc_get_ant_det_val_from_bt)(void *btcoexist);
+
+typedef u8 (*bfp_btc_get_ble_scan_type_from_bt)(void *btcoexist);
+
+typedef u32 (*bfp_btc_get_ble_scan_para_from_bt)(void *btcoexist, u8 scan_type);
+
+typedef bool (*bfp_btc_get_bt_afh_map_from_bt)(void *btcoexist, u8 map_type,
+					       u8 *afh_map);
+
+typedef void (*bfp_btc_set_bt_reg)(void *btc_context, u8 reg_type, u32 offset,
+				   u32 value);
+typedef u32 (*bfp_btc_get_bt_reg)(void *btc_context, u8 reg_type, u32 offset);
+
+typedef void (*bfp_btc_disp_dbg_msg)(void *btcoexist, u8 disp_type,
+				     struct seq_file *m);
+
+struct btc_bt_info {
+	bool bt_disabled;
+	u8 rssi_adjust_for_agc_table_on;
+	u8 rssi_adjust_for_1ant_coex_type;
+	bool pre_bt_ctrl_agg_buf_size;
+	bool bt_busy;
+	u8 pre_agg_buf_size;
+	u8 agg_buf_size;
+	bool limited_dig;
+	bool pre_reject_agg_pkt;
+	bool reject_agg_pkt;
+	bool bt_ctrl_buf_size;
+	bool increase_scan_dev_num;
+	bool miracast_plus_bt;
+	bool bt_ctrl_agg_buf_size;
+	bool bt_tx_rx_mask;
+	u16 bt_hci_ver;
+	u16 bt_real_fw_ver;
+	u8 bt_fw_ver;
+	u32 bt_get_fw_ver;
+
+	bool bt_disable_low_pwr;
+
+	/* the following is for 1Ant solution */
+	bool bt_ctrl_lps;
+	bool bt_pwr_save_mode;
+	bool bt_lps_on;
+	bool force_to_roam;
+	u8 force_exec_pwr_cmd_cnt;
+	u8 lps_val;
+	u8 rpwm_val;
+	u32 ra_mask;
+
+	u32 afh_map_l;
+	u32 afh_map_m;
+	u16 afh_map_h;
+	u32 bt_supported_feature;
+	u32 bt_supported_version;
+	u8 bt_ant_det_val;
+	u8 bt_ble_scan_type;
+	u32 bt_ble_scan_para;
+};
+
+struct btc_stack_info {
+	bool profile_notified;
+	u16 hci_version;	/* stack hci version */
+	u8 num_of_link;
+	bool bt_link_exist;
+	bool sco_exist;
+	bool acl_exist;
+	bool a2dp_exist;
+	bool hid_exist;
+	u8 num_of_hid;
+	bool pan_exist;
+	bool unknown_acl_exist;
+	s8 min_bt_rssi;
+};
+
+struct btc_statistics {
+	u32 cnt_bind;
+	u32 cnt_init_hw_config;
+	u32 cnt_init_coex_dm;
+	u32 cnt_ips_notify;
+	u32 cnt_lps_notify;
+	u32 cnt_scan_notify;
+	u32 cnt_connect_notify;
+	u32 cnt_media_status_notify;
+	u32 cnt_special_packet_notify;
+	u32 cnt_bt_info_notify;
+	u32 cnt_periodical;
+	u32 cnt_coex_dm_switch;
+	u32 cnt_stack_operation_notify;
+	u32 cnt_dbg_ctrl;
+	u32 cnt_pre_load_firmware;
+	u32 cnt_power_on;
+};
+
+struct btc_bt_link_info {
+	bool bt_link_exist;
+	bool bt_hi_pri_link_exist;
+	bool sco_exist;
+	bool sco_only;
+	bool a2dp_exist;
+	bool a2dp_only;
+	bool hid_exist;
+	bool hid_only;
+	bool pan_exist;
+	bool pan_only;
+	bool slave_role;
+	bool acl_busy;
+};
+
+enum btc_antenna_pos {
+	BTC_ANTENNA_AT_MAIN_PORT = 0x1,
+	BTC_ANTENNA_AT_AUX_PORT = 0x2,
+};
+
+enum btc_mp_h2c_op_code {
+	BT_OP_GET_BT_VERSION			= 0,
+	BT_OP_WRITE_REG_ADDR			= 12,
+	BT_OP_WRITE_REG_VALUE			= 13,
+	BT_OP_READ_REG				= 17,
+	BT_OP_GET_AFH_MAP_L			= 30,
+	BT_OP_GET_AFH_MAP_M			= 31,
+	BT_OP_GET_AFH_MAP_H			= 32,
+	BT_OP_GET_BT_COEX_SUPPORTED_FEATURE	= 42,
+	BT_OP_GET_BT_COEX_SUPPORTED_VERSION	= 43,
+	BT_OP_GET_BT_ANT_DET_VAL		= 44,
+	BT_OP_GET_BT_BLE_SCAN_PARA		= 45,
+	BT_OP_GET_BT_BLE_SCAN_TYPE		= 46,
+	BT_OP_MAX
+};
+
+enum btc_mp_h2c_req_num {
+	/* 4 bits only */
+	BT_SEQ_DONT_CARE			= 0,
+	BT_SEQ_GET_BT_VERSION			= 0xE,
+	BT_SEQ_GET_AFH_MAP_L			= 0x5,
+	BT_SEQ_GET_AFH_MAP_M			= 0x6,
+	BT_SEQ_GET_AFH_MAP_H			= 0x9,
+	BT_SEQ_GET_BT_COEX_SUPPORTED_FEATURE	= 0x7,
+	BT_SEQ_GET_BT_COEX_SUPPORTED_VERSION	= 0x8,
+	BT_SEQ_GET_BT_ANT_DET_VAL		= 0x2,
+	BT_SEQ_GET_BT_BLE_SCAN_PARA		= 0x3,
+	BT_SEQ_GET_BT_BLE_SCAN_TYPE		= 0x4,
+};
+
+struct btc_coexist {
+	/* make sure only one adapter can bind the data context  */
+	bool binded;
+	/* default adapter */
+	void *adapter;
+	struct btc_board_info board_info;
+	/* some bt info referenced by non-bt module */
+	struct btc_bt_info bt_info;
+	struct btc_stack_info stack_info;
+	enum btc_chip_interface	chip_interface;
+	struct btc_bt_link_info bt_link_info;
+
+	/* boolean variables to replace BT_AUTO_REPORT_ONLY_XXXXY_ZANT
+	 * configuration parameters
+	 */
+	bool auto_report_1ant;
+	bool auto_report_2ant;
+	bool dbg_mode_1ant;
+	bool dbg_mode_2ant;
+	bool initilized;
+	bool stop_coex_dm;
+	bool manual_control;
+	struct btc_statistics statistics;
+	u8 pwr_mode_val[10];
+
+	struct completion bt_mp_comp;
+
+	/* function pointers - io related */
+	bfp_btc_r1 btc_read_1byte;
+	bfp_btc_w1 btc_write_1byte;
+	bfp_btc_w1_bit_mak btc_write_1byte_bitmask;
+	bfp_btc_r2 btc_read_2byte;
+	bfp_btc_w2 btc_write_2byte;
+	bfp_btc_r4 btc_read_4byte;
+	bfp_btc_w4 btc_write_4byte;
+	bfp_btc_local_reg_w1 btc_write_local_reg_1byte;
+
+	bfp_btc_set_bb_reg btc_set_bb_reg;
+	bfp_btc_get_bb_reg btc_get_bb_reg;
+
+	bfp_btc_set_rf_reg btc_set_rf_reg;
+	bfp_btc_get_rf_reg btc_get_rf_reg;
+
+	bfp_btc_fill_h2c btc_fill_h2c;
+
+	bfp_btc_disp_dbg_msg btc_disp_dbg_msg;
+
+	bfp_btc_get btc_get;
+	bfp_btc_set btc_set;
+
+	bfp_btc_set_bt_reg btc_set_bt_reg;
+	bfp_btc_get_bt_reg btc_get_bt_reg;
+
+	bfp_btc_get_bt_coex_supported_feature btc_get_bt_coex_supported_feature;
+	bfp_btc_get_bt_coex_supported_version btc_get_bt_coex_supported_version;
+	bfp_btc_get_bt_phydm_version btc_get_bt_phydm_version;
+	bfp_btc_phydm_modify_ra_pcr_threshold btc_phydm_modify_ra_pcr_threshold;
+	bfp_btc_phydm_query_phy_counter btc_phydm_query_phy_counter;
+	bfp_btc_get_ant_det_val_from_bt btc_get_ant_det_val_from_bt;
+	bfp_btc_get_ble_scan_type_from_bt btc_get_ble_scan_type_from_bt;
+	bfp_btc_get_ble_scan_para_from_bt btc_get_ble_scan_para_from_bt;
+	bfp_btc_get_bt_afh_map_from_bt btc_get_bt_afh_map_from_bt;
+
+};
+
+bool halbtc_is_wifi_uplink(struct rtl_priv *adapter);
+
+#define rtl_btc_coexist(rtlpriv)				\
+	((struct btc_coexist *)((rtlpriv)->btcoexist.btc_context))
+#define rtl_btc_wifi_only(rtlpriv)				\
+	((struct wifi_only_cfg *)((rtlpriv)->btcoexist.wifi_only_context))
+
+struct wifi_only_cfg;
+
+bool exhalbtc_initlize_variables(struct rtl_priv *rtlpriv);
+bool exhalbtc_initlize_variables_wifi_only(struct rtl_priv *rtlpriv);
+bool exhalbtc_bind_bt_coex_withadapter(void *adapter);
+void exhalbtc_power_on_setting(struct btc_coexist *btcoexist);
+void exhalbtc_pre_load_firmware(struct btc_coexist *btcoexist);
+void exhalbtc_init_hw_config(struct btc_coexist *btcoexist, bool wifi_only);
+void exhalbtc_init_hw_config_wifi_only(struct wifi_only_cfg *wifionly_cfg);
+void exhalbtc_init_coex_dm(struct btc_coexist *btcoexist);
+void exhalbtc_ips_notify(struct btc_coexist *btcoexist, u8 type);
+void exhalbtc_lps_notify(struct btc_coexist *btcoexist, u8 type);
+void exhalbtc_scan_notify(struct btc_coexist *btcoexist, u8 type);
+void exhalbtc_scan_notify_wifi_only(struct wifi_only_cfg *wifionly_cfg,
+				    u8 is_5g);
+void exhalbtc_connect_notify(struct btc_coexist *btcoexist, u8 action);
+void exhalbtc_mediastatus_notify(struct btc_coexist *btcoexist,
+				 enum rt_media_status media_status);
+void exhalbtc_special_packet_notify(struct btc_coexist *btcoexist, u8 pkt_type);
+void exhalbtc_bt_info_notify(struct btc_coexist *btcoexist, u8 *tmp_buf,
+			     u8 length);
+void exhalbtc_rf_status_notify(struct btc_coexist *btcoexist, u8 type);
+void exhalbtc_stack_operation_notify(struct btc_coexist *btcoexist, u8 type);
+void exhalbtc_halt_notify(struct btc_coexist *btcoexist);
+void exhalbtc_pnp_notify(struct btc_coexist *btcoexist, u8 pnp_state);
+void exhalbtc_coex_dm_switch(struct btc_coexist *btcoexist);
+void exhalbtc_periodical(struct btc_coexist *btcoexist);
+void exhalbtc_dbg_control(struct btc_coexist *btcoexist, u8 code, u8 len,
+			  u8 *data);
+void exhalbtc_antenna_detection(struct btc_coexist *btcoexist, u32 cent_freq,
+				u32 offset, u32 span, u32 seconds);
+void exhalbtc_stack_update_profile_info(void);
+void exhalbtc_set_hci_version(struct btc_coexist *btcoexist, u16 hci_version);
+void exhalbtc_set_bt_patch_version(struct btc_coexist *btcoexist,
+				   u16 bt_hci_version, u16 bt_patch_version);
+void exhalbtc_update_min_bt_rssi(struct btc_coexist *btcoexist, s8 bt_rssi);
+void exhalbtc_set_bt_exist(struct btc_coexist *btcoexist, bool bt_exist);
+void exhalbtc_set_chip_type(struct btc_coexist *btcoexist, u8 chip_type);
+void exhalbtc_set_ant_num(struct rtl_priv *rtlpriv, u8 type, u8 ant_num);
+void exhalbtc_display_bt_coex_info(struct btc_coexist *btcoexist,
+				   struct seq_file *m);
+void exhalbtc_switch_band_notify(struct btc_coexist *btcoexist, u8 type);
+void exhalbtc_switch_band_notify_wifi_only(struct wifi_only_cfg *wifionly_cfg,
+					   u8 is_5g);
+void exhalbtc_signal_compensation(struct btc_coexist *btcoexist,
+				  u8 *rssi_wifi, u8 *rssi_bt);
+void exhalbtc_lps_leave(struct btc_coexist *btcoexist);
+void exhalbtc_low_wifi_traffic_notify(struct btc_coexist *btcoexist);
+void exhalbtc_set_single_ant_path(struct btc_coexist *btcoexist,
+				  u8 single_ant_path);
+
+/* The following are used by wifi_only case */
+enum wifionly_chip_interface {
+	WIFIONLY_INTF_UNKNOWN	= 0,
+	WIFIONLY_INTF_PCI		= 1,
+	WIFIONLY_INTF_USB		= 2,
+	WIFIONLY_INTF_SDIO		= 3,
+	WIFIONLY_INTF_MAX
+};
+
+enum wifionly_customer_id {
+	CUSTOMER_NORMAL			= 0,
+	CUSTOMER_HP_1			= 1,
+};
+
+struct wifi_only_haldata {
+	u16		customer_id;
+	u8		efuse_pg_antnum;
+	u8		efuse_pg_antpath;
+	u8		rfe_type;
+	u8		ant_div_cfg;
+};
+
+struct wifi_only_cfg {
+	void				*adapter;
+	struct wifi_only_haldata	haldata_info;
+	enum wifionly_chip_interface	chip_interface;
+};
+
+static inline
+void halwifionly_phy_set_bb_reg(struct wifi_only_cfg *wifi_conly_cfg,
+				u32 regaddr, u32 bitmask, u32 data)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)wifi_conly_cfg->adapter;
+
+	rtl_set_bbreg(rtlpriv->hw, regaddr, bitmask, data);
+}
+
+#endif
diff --git a/drivers/staging/rtlwifi/btcoexist/rtl_btc.c b/drivers/staging/rtlwifi/btcoexist/rtl_btc.c
new file mode 100644
index 000000000000..18a4f5b43b5a
--- /dev/null
+++ b/drivers/staging/rtlwifi/btcoexist/rtl_btc.c
@@ -0,0 +1,528 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2013  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#include "../wifi.h"
+#include <linux/vmalloc.h>
+#include <linux/module.h>
+
+#include "rtl_btc.h"
+#include "halbt_precomp.h"
+
+static struct rtl_btc_ops rtl_btc_operation = {
+	.btc_init_variables = rtl_btc_init_variables,
+	.btc_init_variables_wifi_only = rtl_btc_init_variables_wifi_only,
+	.btc_deinit_variables = rtl_btc_deinit_variables,
+	.btc_init_hal_vars = rtl_btc_init_hal_vars,
+	.btc_power_on_setting = rtl_btc_power_on_setting,
+	.btc_init_hw_config = rtl_btc_init_hw_config,
+	.btc_init_hw_config_wifi_only = rtl_btc_init_hw_config_wifi_only,
+	.btc_ips_notify = rtl_btc_ips_notify,
+	.btc_lps_notify = rtl_btc_lps_notify,
+	.btc_scan_notify = rtl_btc_scan_notify,
+	.btc_scan_notify_wifi_only = rtl_btc_scan_notify_wifi_only,
+	.btc_connect_notify = rtl_btc_connect_notify,
+	.btc_mediastatus_notify = rtl_btc_mediastatus_notify,
+	.btc_periodical = rtl_btc_periodical,
+	.btc_halt_notify = rtl_btc_halt_notify,
+	.btc_btinfo_notify = rtl_btc_btinfo_notify,
+	.btc_btmpinfo_notify = rtl_btc_btmpinfo_notify,
+	.btc_is_limited_dig = rtl_btc_is_limited_dig,
+	.btc_is_disable_edca_turbo = rtl_btc_is_disable_edca_turbo,
+	.btc_is_bt_disabled = rtl_btc_is_bt_disabled,
+	.btc_special_packet_notify = rtl_btc_special_packet_notify,
+	.btc_switch_band_notify = rtl_btc_switch_band_notify,
+	.btc_switch_band_notify_wifi_only = rtl_btc_switch_band_notify_wifionly,
+	.btc_record_pwr_mode = rtl_btc_record_pwr_mode,
+	.btc_get_lps_val = rtl_btc_get_lps_val,
+	.btc_get_rpwm_val = rtl_btc_get_rpwm_val,
+	.btc_is_bt_ctrl_lps = rtl_btc_is_bt_ctrl_lps,
+	.btc_is_bt_lps_on = rtl_btc_is_bt_lps_on,
+	.btc_get_ampdu_cfg = rtl_btc_get_ampdu_cfg,
+	.btc_display_bt_coex_info = rtl_btc_display_bt_coex_info,
+};
+
+void rtl_btc_display_bt_coex_info(struct rtl_priv *rtlpriv, struct seq_file *m)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist) {
+		seq_puts(m, "btc_coexist context is NULL!\n");
+		return;
+	}
+
+	exhalbtc_display_bt_coex_info(btcoexist, m);
+}
+
+void rtl_btc_record_pwr_mode(struct rtl_priv *rtlpriv, u8 *buf, u8 len)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+	u8 safe_len;
+
+	if (!btcoexist)
+		return;
+
+	safe_len = sizeof(btcoexist->pwr_mode_val);
+
+	if (safe_len > len)
+		safe_len = len;
+
+	memcpy(btcoexist->pwr_mode_val, buf, safe_len);
+}
+
+u8 rtl_btc_get_lps_val(struct rtl_priv *rtlpriv)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return 0;
+
+	return btcoexist->bt_info.lps_val;
+}
+
+u8 rtl_btc_get_rpwm_val(struct rtl_priv *rtlpriv)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return 0;
+
+	return btcoexist->bt_info.rpwm_val;
+}
+
+bool rtl_btc_is_bt_ctrl_lps(struct rtl_priv *rtlpriv)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return false;
+
+	return btcoexist->bt_info.bt_ctrl_lps;
+}
+
+bool rtl_btc_is_bt_lps_on(struct rtl_priv *rtlpriv)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return false;
+
+	return btcoexist->bt_info.bt_lps_on;
+}
+
+void rtl_btc_get_ampdu_cfg(struct rtl_priv *rtlpriv, u8 *reject_agg,
+			   u8 *ctrl_agg_size, u8 *agg_size)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist) {
+		*reject_agg = false;
+		*ctrl_agg_size = false;
+		return;
+	}
+
+	if (reject_agg)
+		*reject_agg = btcoexist->bt_info.reject_agg_pkt;
+	if (ctrl_agg_size)
+		*ctrl_agg_size = btcoexist->bt_info.bt_ctrl_agg_buf_size;
+	if (agg_size)
+		*agg_size = btcoexist->bt_info.agg_buf_size;
+}
+
+static void rtl_btc_alloc_variable(struct rtl_priv *rtlpriv, bool wifi_only)
+{
+	if (wifi_only)
+		rtlpriv->btcoexist.wifi_only_context =
+			kzalloc(sizeof(struct wifi_only_cfg), GFP_KERNEL);
+	else
+		rtlpriv->btcoexist.btc_context =
+			kzalloc(sizeof(struct btc_coexist), GFP_KERNEL);
+}
+
+static void rtl_btc_free_variable(struct rtl_priv *rtlpriv)
+{
+	kfree(rtlpriv->btcoexist.btc_context);
+	rtlpriv->btcoexist.btc_context = NULL;
+
+	kfree(rtlpriv->btcoexist.wifi_only_context);
+	rtlpriv->btcoexist.wifi_only_context = NULL;
+}
+
+void rtl_btc_init_variables(struct rtl_priv *rtlpriv)
+{
+	rtl_btc_alloc_variable(rtlpriv, false);
+
+	exhalbtc_initlize_variables(rtlpriv);
+	exhalbtc_bind_bt_coex_withadapter(rtlpriv);
+}
+
+void rtl_btc_init_variables_wifi_only(struct rtl_priv *rtlpriv)
+{
+	rtl_btc_alloc_variable(rtlpriv, true);
+
+	exhalbtc_initlize_variables_wifi_only(rtlpriv);
+}
+
+void rtl_btc_deinit_variables(struct rtl_priv *rtlpriv)
+{
+	rtl_btc_free_variable(rtlpriv);
+}
+
+void rtl_btc_init_hal_vars(struct rtl_priv *rtlpriv)
+{
+	/* move ant_num, bt_type and single_ant_path to
+	 * exhalbtc_bind_bt_coex_withadapter()
+	 */
+}
+
+void rtl_btc_power_on_setting(struct rtl_priv *rtlpriv)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return;
+
+	exhalbtc_power_on_setting(btcoexist);
+}
+
+void rtl_btc_init_hw_config(struct rtl_priv *rtlpriv)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	u8 bt_exist;
+
+	bt_exist = rtl_get_hwpg_bt_exist(rtlpriv);
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
+		 "%s, bt_exist is %d\n", __func__, bt_exist);
+
+	if (!btcoexist)
+		return;
+
+	exhalbtc_init_hw_config(btcoexist, !bt_exist);
+	exhalbtc_init_coex_dm(btcoexist);
+}
+
+void rtl_btc_init_hw_config_wifi_only(struct rtl_priv *rtlpriv)
+{
+	struct wifi_only_cfg *wifionly_cfg = rtl_btc_wifi_only(rtlpriv);
+
+	if (!wifionly_cfg)
+		return;
+
+	exhalbtc_init_hw_config_wifi_only(wifionly_cfg);
+}
+
+void rtl_btc_ips_notify(struct rtl_priv *rtlpriv, u8 type)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return;
+
+	exhalbtc_ips_notify(btcoexist, type);
+
+	if (type == ERFON) {
+		/*
+		 * In some situation, it doesn't scan after leaving IPS, and
+		 * this will cause btcoex in wrong state.
+		 */
+		exhalbtc_scan_notify(btcoexist, 1);
+		exhalbtc_scan_notify(btcoexist, 0);
+	}
+}
+
+void rtl_btc_lps_notify(struct rtl_priv *rtlpriv, u8 type)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return;
+
+	exhalbtc_lps_notify(btcoexist, type);
+}
+
+void rtl_btc_scan_notify(struct rtl_priv *rtlpriv, u8 scantype)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return;
+
+	exhalbtc_scan_notify(btcoexist, scantype);
+}
+
+void rtl_btc_scan_notify_wifi_only(struct rtl_priv *rtlpriv, u8 scantype)
+{
+	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+	struct wifi_only_cfg *wifionly_cfg = rtl_btc_wifi_only(rtlpriv);
+	u8 is_5g = (rtlhal->current_bandtype == BAND_ON_5G);
+
+	if (!wifionly_cfg)
+		return;
+
+	exhalbtc_scan_notify_wifi_only(wifionly_cfg, is_5g);
+}
+
+void rtl_btc_connect_notify(struct rtl_priv *rtlpriv, u8 action)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return;
+
+	exhalbtc_connect_notify(btcoexist, action);
+}
+
+void rtl_btc_mediastatus_notify(struct rtl_priv *rtlpriv,
+				enum rt_media_status mstatus)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return;
+
+	exhalbtc_mediastatus_notify(btcoexist, mstatus);
+}
+
+void rtl_btc_periodical(struct rtl_priv *rtlpriv)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return;
+
+	/*rtl_bt_dm_monitor();*/
+	exhalbtc_periodical(btcoexist);
+}
+
+void rtl_btc_halt_notify(struct rtl_priv *rtlpriv)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return;
+
+	exhalbtc_halt_notify(btcoexist);
+}
+
+void rtl_btc_btinfo_notify(struct rtl_priv *rtlpriv, u8 *tmp_buf, u8 length)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return;
+
+	exhalbtc_bt_info_notify(btcoexist, tmp_buf, length);
+}
+
+void rtl_btc_btmpinfo_notify(struct rtl_priv *rtlpriv, u8 *tmp_buf, u8 length)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+	u8 extid, seq, len;
+	u16 bt_real_fw_ver;
+	u8 bt_fw_ver;
+	u8 *data;
+
+	if (!btcoexist)
+		return;
+
+	if ((length < 4) || (!tmp_buf))
+		return;
+
+	extid = tmp_buf[0];
+	/* not response from BT FW then exit*/
+	if (extid != 1) /* C2H_TRIG_BY_BT_FW = 1 */
+		return;
+
+	len = tmp_buf[1] >> 4;
+	seq = tmp_buf[2] >> 4;
+	data = &tmp_buf[3];
+
+	/* BT Firmware version response */
+	switch (seq) {
+	case BT_SEQ_GET_BT_VERSION:
+		bt_real_fw_ver = tmp_buf[3] | (tmp_buf[4] << 8);
+		bt_fw_ver = tmp_buf[5];
+
+		btcoexist->bt_info.bt_real_fw_ver = bt_real_fw_ver;
+		btcoexist->bt_info.bt_fw_ver = bt_fw_ver;
+		break;
+	case BT_SEQ_GET_AFH_MAP_L:
+		btcoexist->bt_info.afh_map_l = le32_to_cpu(*(__le32 *)data);
+		break;
+	case BT_SEQ_GET_AFH_MAP_M:
+		btcoexist->bt_info.afh_map_m = le32_to_cpu(*(__le32 *)data);
+		break;
+	case BT_SEQ_GET_AFH_MAP_H:
+		btcoexist->bt_info.afh_map_h = le16_to_cpu(*(__le16 *)data);
+		break;
+	case BT_SEQ_GET_BT_COEX_SUPPORTED_FEATURE:
+		btcoexist->bt_info.bt_supported_feature = tmp_buf[3] |
+							  (tmp_buf[4] << 8);
+		break;
+	case BT_SEQ_GET_BT_COEX_SUPPORTED_VERSION:
+		btcoexist->bt_info.bt_supported_version = tmp_buf[3] |
+							  (tmp_buf[4] << 8);
+		break;
+	case BT_SEQ_GET_BT_ANT_DET_VAL:
+		btcoexist->bt_info.bt_ant_det_val = tmp_buf[3];
+		break;
+	case BT_SEQ_GET_BT_BLE_SCAN_PARA:
+		btcoexist->bt_info.bt_ble_scan_para = tmp_buf[3] |
+						      (tmp_buf[4] << 8) |
+						      (tmp_buf[5] << 16) |
+						      (tmp_buf[6] << 24);
+		break;
+	case BT_SEQ_GET_BT_BLE_SCAN_TYPE:
+		btcoexist->bt_info.bt_ble_scan_type = tmp_buf[3];
+		break;
+	}
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "btmpinfo complete req_num=%d\n", seq);
+
+	complete(&btcoexist->bt_mp_comp);
+}
+
+bool rtl_btc_is_limited_dig(struct rtl_priv *rtlpriv)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return false;
+
+	return btcoexist->bt_info.limited_dig;
+}
+
+bool rtl_btc_is_disable_edca_turbo(struct rtl_priv *rtlpriv)
+{
+	bool bt_change_edca = false;
+	u32 cur_edca_val;
+	u32 edca_bt_hs_uplink = 0x5ea42b, edca_bt_hs_downlink = 0x5ea42b;
+	u32 edca_hs;
+	u32 edca_addr = 0x504;
+
+	cur_edca_val = rtl_read_dword(rtlpriv, edca_addr);
+	if (halbtc_is_wifi_uplink(rtlpriv)) {
+		if (cur_edca_val != edca_bt_hs_uplink) {
+			edca_hs = edca_bt_hs_uplink;
+			bt_change_edca = true;
+		}
+	} else {
+		if (cur_edca_val != edca_bt_hs_downlink) {
+			edca_hs = edca_bt_hs_downlink;
+			bt_change_edca = true;
+		}
+	}
+
+	if (bt_change_edca)
+		rtl_write_dword(rtlpriv, edca_addr, edca_hs);
+
+	return true;
+}
+
+bool rtl_btc_is_bt_disabled(struct rtl_priv *rtlpriv)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return true;
+
+	/* It seems 'bt_disabled' is never be initialized or set. */
+	if (btcoexist->bt_info.bt_disabled)
+		return true;
+	else
+		return false;
+}
+
+void rtl_btc_special_packet_notify(struct rtl_priv *rtlpriv, u8 pkt_type)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return;
+
+	return exhalbtc_special_packet_notify(btcoexist, pkt_type);
+}
+
+void rtl_btc_switch_band_notify(struct rtl_priv *rtlpriv, u8 band_type,
+				bool scanning)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+	u8 type = BTC_NOT_SWITCH;
+
+	if (!btcoexist)
+		return;
+
+	switch (band_type) {
+	case BAND_ON_2_4G:
+		if (scanning)
+			type = BTC_SWITCH_TO_24G;
+		else
+			type = BTC_SWITCH_TO_24G_NOFORSCAN;
+		break;
+
+	case BAND_ON_5G:
+		type = BTC_SWITCH_TO_5G;
+		break;
+	}
+
+	if (type != BTC_NOT_SWITCH)
+		exhalbtc_switch_band_notify(btcoexist, type);
+}
+
+void rtl_btc_switch_band_notify_wifionly(struct rtl_priv *rtlpriv, u8 band_type,
+					 bool scanning)
+{
+	struct wifi_only_cfg *wifionly_cfg = rtl_btc_wifi_only(rtlpriv);
+	u8 is_5g = (band_type == BAND_ON_5G);
+
+	if (!wifionly_cfg)
+		return;
+
+	exhalbtc_switch_band_notify_wifi_only(wifionly_cfg, is_5g);
+}
+
+struct rtl_btc_ops *rtl_btc_get_ops_pointer(void)
+{
+	return &rtl_btc_operation;
+}
+
+enum rt_media_status mgnt_link_status_query(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	enum rt_media_status    m_status = RT_MEDIA_DISCONNECT;
+
+	u8 bibss = (mac->opmode == NL80211_IFTYPE_ADHOC) ? 1 : 0;
+
+	if (bibss || rtlpriv->mac80211.link_state >= MAC80211_LINKED)
+		m_status = RT_MEDIA_CONNECT;
+
+	return m_status;
+}
+
+u8 rtl_get_hwpg_bt_exist(struct rtl_priv *rtlpriv)
+{
+	return rtlpriv->btcoexist.btc_info.btcoexist;
+}
diff --git a/drivers/staging/rtlwifi/btcoexist/rtl_btc.h b/drivers/staging/rtlwifi/btcoexist/rtl_btc.h
new file mode 100644
index 000000000000..8c996055de71
--- /dev/null
+++ b/drivers/staging/rtlwifi/btcoexist/rtl_btc.h
@@ -0,0 +1,75 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2010  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __RTL_BTC_H__
+#define __RTL_BTC_H__
+
+#include "halbt_precomp.h"
+
+void rtl_btc_init_variables(struct rtl_priv *rtlpriv);
+void rtl_btc_init_variables_wifi_only(struct rtl_priv *rtlpriv);
+void rtl_btc_deinit_variables(struct rtl_priv *rtlpriv);
+void rtl_btc_init_hal_vars(struct rtl_priv *rtlpriv);
+void rtl_btc_power_on_setting(struct rtl_priv *rtlpriv);
+void rtl_btc_init_hw_config(struct rtl_priv *rtlpriv);
+void rtl_btc_init_hw_config_wifi_only(struct rtl_priv *rtlpriv);
+void rtl_btc_ips_notify(struct rtl_priv *rtlpriv, u8 type);
+void rtl_btc_lps_notify(struct rtl_priv *rtlpriv, u8 type);
+void rtl_btc_scan_notify(struct rtl_priv *rtlpriv, u8 scantype);
+void rtl_btc_scan_notify_wifi_only(struct rtl_priv *rtlpriv, u8 scantype);
+void rtl_btc_connect_notify(struct rtl_priv *rtlpriv, u8 action);
+void rtl_btc_mediastatus_notify(struct rtl_priv *rtlpriv,
+				enum rt_media_status mstatus);
+void rtl_btc_periodical(struct rtl_priv *rtlpriv);
+void rtl_btc_halt_notify(struct rtl_priv *rtlpriv);
+void rtl_btc_btinfo_notify(struct rtl_priv *rtlpriv, u8 *tmpbuf, u8 length);
+void rtl_btc_btmpinfo_notify(struct rtl_priv *rtlpriv, u8 *tmp_buf, u8 length);
+bool rtl_btc_is_limited_dig(struct rtl_priv *rtlpriv);
+bool rtl_btc_is_disable_edca_turbo(struct rtl_priv *rtlpriv);
+bool rtl_btc_is_bt_disabled(struct rtl_priv *rtlpriv);
+void rtl_btc_special_packet_notify(struct rtl_priv *rtlpriv, u8 pkt_type);
+void rtl_btc_switch_band_notify(struct rtl_priv *rtlpriv, u8 band_type,
+				bool scanning);
+void rtl_btc_switch_band_notify_wifionly(struct rtl_priv *rtlpriv, u8 band_type,
+					 bool scanning);
+void rtl_btc_display_bt_coex_info(struct rtl_priv *rtlpriv, struct seq_file *m);
+void rtl_btc_record_pwr_mode(struct rtl_priv *rtlpriv, u8 *buf, u8 len);
+u8   rtl_btc_get_lps_val(struct rtl_priv *rtlpriv);
+u8   rtl_btc_get_rpwm_val(struct rtl_priv *rtlpriv);
+bool rtl_btc_is_bt_ctrl_lps(struct rtl_priv *rtlpriv);
+bool rtl_btc_is_bt_lps_on(struct rtl_priv *rtlpriv);
+void rtl_btc_get_ampdu_cfg(struct rtl_priv *rtlpriv, u8 *reject_agg,
+			   u8 *ctrl_agg_size, u8 *agg_size);
+
+struct rtl_btc_ops *rtl_btc_get_ops_pointer(void);
+
+u8 rtl_get_hwpg_bt_exist(struct rtl_priv *rtlpriv);
+u8 rtl_get_hwpg_bt_type(struct rtl_priv *rtlpriv);
+u8 rtl_get_hwpg_ant_num(struct rtl_priv *rtlpriv);
+u8 rtl_get_hwpg_single_ant_path(struct rtl_priv *rtlpriv);
+u8 rtl_get_hwpg_package_type(struct rtl_priv *rtlpriv);
+
+enum rt_media_status mgnt_link_status_query(struct ieee80211_hw *hw);
+
+#endif
diff --git a/drivers/staging/rtlwifi/cam.c b/drivers/staging/rtlwifi/cam.c
new file mode 100644
index 000000000000..9c8c907cb48e
--- /dev/null
+++ b/drivers/staging/rtlwifi/cam.c
@@ -0,0 +1,326 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2012  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#include "wifi.h"
+#include "cam.h"
+#include <linux/export.h>
+
+void rtl_cam_reset_sec_info(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	rtlpriv->sec.use_defaultkey = false;
+	rtlpriv->sec.pairwise_enc_algorithm = NO_ENCRYPTION;
+	rtlpriv->sec.group_enc_algorithm = NO_ENCRYPTION;
+	memset(rtlpriv->sec.key_buf, 0, KEY_BUF_SIZE * MAX_KEY_LEN);
+	memset(rtlpriv->sec.key_len, 0, KEY_BUF_SIZE);
+	rtlpriv->sec.pairwise_key = NULL;
+}
+
+static void rtl_cam_program_entry(struct ieee80211_hw *hw, u32 entry_no,
+				  u8 *mac_addr, u8 *key_cont_128, u16 us_config)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	u32 target_command;
+	u32 target_content = 0;
+	int entry_i;
+
+	RT_PRINT_DATA(rtlpriv, COMP_SEC, DBG_DMESG, "Key content :",
+		      key_cont_128, 16);
+
+	/* 0-1 config + mac, 2-5 fill 128key,6-7 are reserved */
+	for (entry_i = CAM_CONTENT_COUNT - 1; entry_i >= 0; entry_i--) {
+		target_command = entry_i + CAM_CONTENT_COUNT * entry_no;
+		target_command = target_command | BIT(31) | BIT(16);
+
+		if (entry_i == 0) {
+			target_content = (u32)(*(mac_addr + 0)) << 16 |
+					 (u32)(*(mac_addr + 1)) << 24 |
+					 (u32)us_config;
+
+			rtl_write_dword(rtlpriv, rtlpriv->cfg->maps[WCAMI],
+					target_content);
+			rtl_write_dword(rtlpriv, rtlpriv->cfg->maps[RWCAM],
+					target_command);
+
+			RT_TRACE(rtlpriv, COMP_SEC, DBG_LOUD,
+				 "WRITE %x: %x\n",
+				 rtlpriv->cfg->maps[WCAMI], target_content);
+			RT_TRACE(rtlpriv, COMP_SEC, DBG_LOUD,
+				 "The Key ID is %d\n", entry_no);
+			RT_TRACE(rtlpriv, COMP_SEC, DBG_LOUD,
+				 "WRITE %x: %x\n",
+				 rtlpriv->cfg->maps[RWCAM], target_command);
+
+		} else if (entry_i == 1) {
+			target_content = (u32)(*(mac_addr + 5)) << 24 |
+					 (u32)(*(mac_addr + 4)) << 16 |
+					 (u32)(*(mac_addr + 3)) << 8 |
+					 (u32)(*(mac_addr + 2));
+
+			rtl_write_dword(rtlpriv, rtlpriv->cfg->maps[WCAMI],
+					target_content);
+			rtl_write_dword(rtlpriv, rtlpriv->cfg->maps[RWCAM],
+					target_command);
+
+			RT_TRACE(rtlpriv, COMP_SEC, DBG_LOUD,
+				 "WRITE A4: %x\n", target_content);
+			RT_TRACE(rtlpriv, COMP_SEC, DBG_LOUD,
+				 "WRITE A0: %x\n", target_command);
+		} else {
+			target_content =
+			    (u32)(*(key_cont_128 + (entry_i * 4 - 8) + 3)) <<
+			    24 | (u32)(*(key_cont_128 + (entry_i * 4 - 8) + 2))
+			    << 16 |
+			    (u32)(*(key_cont_128 + (entry_i * 4 - 8) + 1)) << 8
+			    | (u32)(*(key_cont_128 + (entry_i * 4 - 8) + 0));
+
+			rtl_write_dword(rtlpriv, rtlpriv->cfg->maps[WCAMI],
+					target_content);
+			rtl_write_dword(rtlpriv, rtlpriv->cfg->maps[RWCAM],
+					target_command);
+
+			RT_TRACE(rtlpriv, COMP_SEC, DBG_LOUD,
+				 "WRITE A4: %x\n", target_content);
+			RT_TRACE(rtlpriv, COMP_SEC, DBG_LOUD,
+				 "WRITE A0: %x\n", target_command);
+		}
+	}
+
+	RT_TRACE(rtlpriv, COMP_SEC, DBG_LOUD,
+		 "after set key, usconfig:%x\n", us_config);
+}
+
+u8 rtl_cam_add_one_entry(struct ieee80211_hw *hw, u8 *mac_addr,
+			 u32 ul_key_id, u32 ul_entry_idx, u32 ul_enc_alg,
+			 u32 ul_default_key, u8 *key_content)
+{
+	u32 us_config;
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG,
+		 "EntryNo:%x, ulKeyId=%x, ulEncAlg=%x, ulUseDK=%x MacAddr %pM\n",
+		 ul_entry_idx, ul_key_id, ul_enc_alg,
+		 ul_default_key, mac_addr);
+
+	if (ul_key_id == TOTAL_CAM_ENTRY) {
+		RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING,
+			 "ulKeyId exceed!\n");
+		return 0;
+	}
+
+	if (ul_default_key == 1)
+		us_config = CFG_VALID | ((u16)(ul_enc_alg) << 2);
+	else
+		us_config = CFG_VALID | ((ul_enc_alg) << 2) | ul_key_id;
+
+	rtl_cam_program_entry(hw, ul_entry_idx, mac_addr,
+			      (u8 *)key_content, us_config);
+
+	RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG, "end\n");
+
+	return 1;
+}
+
+int rtl_cam_delete_one_entry(struct ieee80211_hw *hw,
+			     u8 *mac_addr, u32 ul_key_id)
+{
+	u32 ul_command;
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG, "key_idx:%d\n", ul_key_id);
+
+	ul_command = ul_key_id * CAM_CONTENT_COUNT;
+	ul_command = ul_command | BIT(31) | BIT(16);
+
+	rtl_write_dword(rtlpriv, rtlpriv->cfg->maps[WCAMI], 0);
+	rtl_write_dword(rtlpriv, rtlpriv->cfg->maps[RWCAM], ul_command);
+
+	RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG,
+		 "%s(): WRITE A4: %x\n", __func__, 0);
+	RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG,
+		 "%s(): WRITE A0: %x\n", __func__, ul_command);
+
+	return 0;
+}
+
+void rtl_cam_reset_all_entry(struct ieee80211_hw *hw)
+{
+	u32 ul_command;
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	ul_command = BIT(31) | BIT(30);
+	rtl_write_dword(rtlpriv, rtlpriv->cfg->maps[RWCAM], ul_command);
+}
+
+void rtl_cam_mark_invalid(struct ieee80211_hw *hw, u8 uc_index)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	u32 ul_command;
+	u32 ul_content;
+	u32 ul_enc_algo = rtlpriv->cfg->maps[SEC_CAM_AES];
+
+	switch (rtlpriv->sec.pairwise_enc_algorithm) {
+	case WEP40_ENCRYPTION:
+		ul_enc_algo = rtlpriv->cfg->maps[SEC_CAM_WEP40];
+		break;
+	case WEP104_ENCRYPTION:
+		ul_enc_algo = rtlpriv->cfg->maps[SEC_CAM_WEP104];
+		break;
+	case TKIP_ENCRYPTION:
+		ul_enc_algo = rtlpriv->cfg->maps[SEC_CAM_TKIP];
+		break;
+	case AESCCMP_ENCRYPTION:
+		ul_enc_algo = rtlpriv->cfg->maps[SEC_CAM_AES];
+		break;
+	default:
+		ul_enc_algo = rtlpriv->cfg->maps[SEC_CAM_AES];
+	}
+
+	ul_content = (uc_index & 3) | ((u16)(ul_enc_algo) << 2);
+
+	ul_content |= BIT(15);
+	ul_command = CAM_CONTENT_COUNT * uc_index;
+	ul_command = ul_command | BIT(31) | BIT(16);
+
+	rtl_write_dword(rtlpriv, rtlpriv->cfg->maps[WCAMI], ul_content);
+	rtl_write_dword(rtlpriv, rtlpriv->cfg->maps[RWCAM], ul_command);
+
+	RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG,
+		 "%s(): WRITE A4: %x\n", __func__, ul_content);
+	RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG,
+		 "%s(): WRITE A0: %x\n", __func__, ul_command);
+}
+
+void rtl_cam_empty_entry(struct ieee80211_hw *hw, u8 uc_index)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	u32 ul_command;
+	u32 ul_content;
+	u32 ul_encalgo = rtlpriv->cfg->maps[SEC_CAM_AES];
+	u8 entry_i;
+
+	switch (rtlpriv->sec.pairwise_enc_algorithm) {
+	case WEP40_ENCRYPTION:
+		ul_encalgo = rtlpriv->cfg->maps[SEC_CAM_WEP40];
+		break;
+	case WEP104_ENCRYPTION:
+		ul_encalgo = rtlpriv->cfg->maps[SEC_CAM_WEP104];
+		break;
+	case TKIP_ENCRYPTION:
+		ul_encalgo = rtlpriv->cfg->maps[SEC_CAM_TKIP];
+		break;
+	case AESCCMP_ENCRYPTION:
+		ul_encalgo = rtlpriv->cfg->maps[SEC_CAM_AES];
+		break;
+	default:
+		ul_encalgo = rtlpriv->cfg->maps[SEC_CAM_AES];
+	}
+
+	for (entry_i = 0; entry_i < CAM_CONTENT_COUNT; entry_i++) {
+		if (entry_i == 0) {
+			ul_content =
+			    (uc_index & 0x03) | ((u16)(ul_encalgo) << 2);
+			ul_content |= BIT(15);
+		} else {
+			ul_content = 0;
+		}
+
+		ul_command = CAM_CONTENT_COUNT * uc_index + entry_i;
+		ul_command = ul_command | BIT(31) | BIT(16);
+
+		rtl_write_dword(rtlpriv, rtlpriv->cfg->maps[WCAMI], ul_content);
+		rtl_write_dword(rtlpriv, rtlpriv->cfg->maps[RWCAM], ul_command);
+
+		RT_TRACE(rtlpriv, COMP_SEC, DBG_LOUD,
+			 "%s(): WRITE A4: %x\n", __func__, ul_content);
+		RT_TRACE(rtlpriv, COMP_SEC, DBG_LOUD,
+			 "%s(): WRITE A0: %x\n", __func__, ul_command);
+	}
+}
+
+u8 rtl_cam_get_free_entry(struct ieee80211_hw *hw, u8 *sta_addr)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u32 bitmap = (rtlpriv->sec.hwsec_cam_bitmap) >> 4;
+	u8 entry_idx = 0;
+	u8 i, *addr;
+
+	if (!sta_addr) {
+		pr_err("sta_addr is NULL.\n");
+		return TOTAL_CAM_ENTRY;
+	}
+	/* Does STA already exist? */
+	for (i = 4; i < TOTAL_CAM_ENTRY; i++) {
+		addr = rtlpriv->sec.hwsec_cam_sta_addr[i];
+		if (ether_addr_equal_unaligned(addr, sta_addr))
+			return i;
+	}
+	/* Get a free CAM entry. */
+	for (entry_idx = 4; entry_idx < TOTAL_CAM_ENTRY; entry_idx++) {
+		if ((bitmap & BIT(0)) == 0) {
+			pr_err("-----hwsec_cam_bitmap: 0x%x entry_idx=%d\n",
+			       rtlpriv->sec.hwsec_cam_bitmap, entry_idx);
+			rtlpriv->sec.hwsec_cam_bitmap |= BIT(0) << entry_idx;
+			memcpy(rtlpriv->sec.hwsec_cam_sta_addr[entry_idx],
+			       sta_addr, ETH_ALEN);
+			return entry_idx;
+		}
+		bitmap = bitmap >> 1;
+	}
+	return TOTAL_CAM_ENTRY;
+}
+
+void rtl_cam_del_entry(struct ieee80211_hw *hw, u8 *sta_addr)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u32 bitmap;
+	u8 i, *addr;
+
+	if (!sta_addr) {
+		pr_err("sta_addr is NULL.\n");
+		return;
+	}
+
+	if (is_zero_ether_addr(sta_addr)) {
+		pr_err("sta_addr is %pM\n", sta_addr);
+		return;
+	}
+	/* Does STA already exist? */
+	for (i = 4; i < TOTAL_CAM_ENTRY; i++) {
+		addr = rtlpriv->sec.hwsec_cam_sta_addr[i];
+		bitmap = (rtlpriv->sec.hwsec_cam_bitmap) >> i;
+		if (((bitmap & BIT(0)) == BIT(0)) &&
+		    (ether_addr_equal_unaligned(addr, sta_addr))) {
+			/* Remove from HW Security CAM */
+			eth_zero_addr(rtlpriv->sec.hwsec_cam_sta_addr[i]);
+			rtlpriv->sec.hwsec_cam_bitmap &= ~(BIT(0) << i);
+			RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG,
+				 "&&&&&&&&&del entry %d\n", i);
+		}
+	}
+}
diff --git a/drivers/staging/rtlwifi/cam.h b/drivers/staging/rtlwifi/cam.h
new file mode 100644
index 000000000000..b25729e15b75
--- /dev/null
+++ b/drivers/staging/rtlwifi/cam.h
@@ -0,0 +1,50 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2012  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __RTL_CAM_H_
+#define __RTL_CAM_H_
+
+#define CAM_CONTENT_COUNT				8
+
+#define CFG_VALID					BIT(15)
+
+#define PAIRWISE_KEYIDX					0
+#define CAM_PAIRWISE_KEY_POSITION			4
+
+#define	CAM_CONFIG_NO_USEDK				0
+
+void rtl_cam_reset_all_entry(struct ieee80211_hw *hw);
+u8 rtl_cam_add_one_entry(struct ieee80211_hw *hw, u8 *mac_addr,
+			 u32 ul_key_id, u32 ul_entry_idx, u32 ul_enc_alg,
+			 u32 ul_default_key, u8 *key_content);
+int rtl_cam_delete_one_entry(struct ieee80211_hw *hw, u8 *mac_addr,
+			     u32 ul_key_id);
+void rtl_cam_mark_invalid(struct ieee80211_hw *hw, u8 uc_index);
+void rtl_cam_empty_entry(struct ieee80211_hw *hw, u8 uc_index);
+void rtl_cam_reset_sec_info(struct ieee80211_hw *hw);
+u8 rtl_cam_get_free_entry(struct ieee80211_hw *hw, u8 *sta_addr);
+void rtl_cam_del_entry(struct ieee80211_hw *hw, u8 *sta_addr);
+
+#endif
diff --git a/drivers/staging/rtlwifi/core.c b/drivers/staging/rtlwifi/core.c
new file mode 100644
index 000000000000..d33847d0550d
--- /dev/null
+++ b/drivers/staging/rtlwifi/core.c
@@ -0,0 +1,2046 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2012  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#include "wifi.h"
+#include "core.h"
+#include "cam.h"
+#include "base.h"
+#include "ps.h"
+#include "pwrseqcmd.h"
+
+#include "btcoexist/rtl_btc.h"
+#include <linux/firmware.h>
+#include <linux/export.h>
+#include <net/cfg80211.h>
+
+u8 channel5g[CHANNEL_MAX_NUMBER_5G] = {
+	36, 38, 40, 42, 44, 46, 48,		/* Band 1 */
+	52, 54, 56, 58, 60, 62, 64,		/* Band 2 */
+	100, 102, 104, 106, 108, 110, 112,	/* Band 3 */
+	116, 118, 120, 122, 124, 126, 128,	/* Band 3 */
+	132, 134, 136, 138, 140, 142, 144,	/* Band 3 */
+	149, 151, 153, 155, 157, 159, 161,	/* Band 4 */
+	165, 167, 169, 171, 173, 175, 177	/* Band 4 */
+};
+
+u8 channel5g_80m[CHANNEL_MAX_NUMBER_5G_80M] = {
+	42, 58, 106, 122, 138, 155, 171
+};
+
+void rtl_addr_delay(u32 addr)
+{
+	if (addr == 0xfe)
+		mdelay(50);
+	else if (addr == 0xfd)
+		msleep(5);
+	else if (addr == 0xfc)
+		msleep(1);
+	else if (addr == 0xfb)
+		usleep_range(50, 100);
+	else if (addr == 0xfa)
+		usleep_range(5, 10);
+	else if (addr == 0xf9)
+		usleep_range(1, 2);
+}
+
+void rtl_rfreg_delay(struct ieee80211_hw *hw, enum radio_path rfpath, u32 addr,
+		     u32 mask, u32 data)
+{
+	if (addr >= 0xf9 && addr <= 0xfe) {
+		rtl_addr_delay(addr);
+	} else {
+		rtl_set_rfreg(hw, rfpath, addr, mask, data);
+		udelay(1);
+	}
+}
+
+void rtl_bb_delay(struct ieee80211_hw *hw, u32 addr, u32 data)
+{
+	if (addr >= 0xf9 && addr <= 0xfe) {
+		rtl_addr_delay(addr);
+	} else {
+		rtl_set_bbreg(hw, addr, MASKDWORD, data);
+		udelay(1);
+	}
+}
+
+static void rtl_fw_do_work(const struct firmware *firmware, void *context,
+			   bool is_wow)
+{
+	struct ieee80211_hw *hw = context;
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	int err;
+
+	RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
+		 "Firmware callback routine entered!\n");
+	complete(&rtlpriv->firmware_loading_complete);
+	if (!firmware) {
+		if (rtlpriv->cfg->alt_fw_name) {
+			err = request_firmware(&firmware,
+					       rtlpriv->cfg->alt_fw_name,
+					       rtlpriv->io.dev);
+			pr_info("Loading alternative firmware %s\n",
+				rtlpriv->cfg->alt_fw_name);
+			if (!err)
+				goto found_alt;
+		}
+		pr_err("Selected firmware is not available\n");
+		rtlpriv->max_fw_size = 0;
+		return;
+	}
+found_alt:
+	if (firmware->size > rtlpriv->max_fw_size) {
+		pr_err("Firmware is too big!\n");
+		release_firmware(firmware);
+		return;
+	}
+	if (!is_wow) {
+		memcpy(rtlpriv->rtlhal.pfirmware, firmware->data,
+		       firmware->size);
+		rtlpriv->rtlhal.fwsize = firmware->size;
+	} else {
+		memcpy(rtlpriv->rtlhal.wowlan_firmware, firmware->data,
+		       firmware->size);
+		rtlpriv->rtlhal.wowlan_fwsize = firmware->size;
+	}
+	rtlpriv->rtlhal.fwsize = firmware->size;
+	release_firmware(firmware);
+}
+
+void rtl_fw_cb(const struct firmware *firmware, void *context)
+{
+	rtl_fw_do_work(firmware, context, false);
+}
+
+void rtl_wowlan_fw_cb(const struct firmware *firmware, void *context)
+{
+	rtl_fw_do_work(firmware, context, true);
+}
+
+/*mutex for start & stop is must here. */
+static int rtl_op_start(struct ieee80211_hw *hw)
+{
+	int err = 0;
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+
+	if (!is_hal_stop(rtlhal))
+		return 0;
+	if (!test_bit(RTL_STATUS_INTERFACE_START, &rtlpriv->status))
+		return 0;
+	mutex_lock(&rtlpriv->locks.conf_mutex);
+	err = rtlpriv->intf_ops->adapter_start(hw);
+	if (!err)
+		rtl_watch_dog_timer_callback((unsigned long)hw);
+	mutex_unlock(&rtlpriv->locks.conf_mutex);
+	return err;
+}
+
+static void rtl_op_stop(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	bool support_remote_wakeup = false;
+
+	if (is_hal_stop(rtlhal))
+		return;
+
+	rtlpriv->cfg->ops->get_hw_reg(hw, HAL_DEF_WOWLAN,
+				      (u8 *)(&support_remote_wakeup));
+	/* here is must, because adhoc do stop and start,
+	 * but stop with RFOFF may cause something wrong,
+	 * like adhoc TP
+	 */
+	if (unlikely(ppsc->rfpwr_state == ERFOFF))
+		rtl_ips_nic_on(hw);
+
+	mutex_lock(&rtlpriv->locks.conf_mutex);
+	/* if wowlan supported, DON'T clear connected info */
+	if (!(support_remote_wakeup &&
+	      rtlhal->enter_pnp_sleep)) {
+		mac->link_state = MAC80211_NOLINK;
+		eth_zero_addr(mac->bssid);
+		mac->vendor = PEER_UNKNOWN;
+
+		/* reset sec info */
+		rtl_cam_reset_sec_info(hw);
+
+		rtl_deinit_deferred_work(hw);
+	}
+	rtlpriv->intf_ops->adapter_stop(hw);
+
+	mutex_unlock(&rtlpriv->locks.conf_mutex);
+}
+
+static void rtl_op_tx(struct ieee80211_hw *hw,
+		      struct ieee80211_tx_control *control,
+		      struct sk_buff *skb)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	struct rtl_tcb_desc tcb_desc;
+
+	memset(&tcb_desc, 0, sizeof(struct rtl_tcb_desc));
+
+	if (unlikely(is_hal_stop(rtlhal) || ppsc->rfpwr_state != ERFON))
+		goto err_free;
+
+	if (!test_bit(RTL_STATUS_INTERFACE_START, &rtlpriv->status))
+		goto err_free;
+
+	if (!rtlpriv->intf_ops->waitq_insert(hw, control->sta, skb))
+		rtlpriv->intf_ops->adapter_tx(hw, control->sta, skb, &tcb_desc);
+	return;
+
+err_free:
+	dev_kfree_skb_any(skb);
+}
+
+static int rtl_op_add_interface(struct ieee80211_hw *hw,
+				struct ieee80211_vif *vif)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	int err = 0;
+	u8 retry_limit = 0x30;
+
+	if (mac->vif) {
+		RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING,
+			 "vif has been set!! mac->vif = 0x%p\n", mac->vif);
+		return -EOPNOTSUPP;
+	}
+
+	vif->driver_flags |= IEEE80211_VIF_BEACON_FILTER;
+
+	rtl_ips_nic_on(hw);
+
+	mutex_lock(&rtlpriv->locks.conf_mutex);
+	switch (ieee80211_vif_type_p2p(vif)) {
+	case NL80211_IFTYPE_P2P_CLIENT:
+		mac->p2p = P2P_ROLE_CLIENT;
+		/*fall through*/
+	case NL80211_IFTYPE_STATION:
+		if (mac->beacon_enabled == 1) {
+			RT_TRACE(rtlpriv, COMP_MAC80211, DBG_LOUD,
+				 "NL80211_IFTYPE_STATION\n");
+			mac->beacon_enabled = 0;
+			rtlpriv->cfg->ops->update_interrupt_mask(hw, 0,
+					rtlpriv->cfg->maps[RTL_IBSS_INT_MASKS]);
+		}
+		break;
+	case NL80211_IFTYPE_ADHOC:
+		RT_TRACE(rtlpriv, COMP_MAC80211, DBG_LOUD,
+			 "NL80211_IFTYPE_ADHOC\n");
+
+		mac->link_state = MAC80211_LINKED;
+		rtlpriv->cfg->ops->set_bcn_reg(hw);
+		if (rtlpriv->rtlhal.current_bandtype == BAND_ON_2_4G)
+			mac->basic_rates = 0xfff;
+		else
+			mac->basic_rates = 0xff0;
+		rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_BASIC_RATE,
+				(u8 *)(&mac->basic_rates));
+
+		retry_limit = 0x07;
+		break;
+	case NL80211_IFTYPE_P2P_GO:
+		mac->p2p = P2P_ROLE_GO;
+		/*fall through*/
+	case NL80211_IFTYPE_AP:
+		RT_TRACE(rtlpriv, COMP_MAC80211, DBG_LOUD,
+			 "NL80211_IFTYPE_AP\n");
+
+		mac->link_state = MAC80211_LINKED;
+		rtlpriv->cfg->ops->set_bcn_reg(hw);
+		if (rtlpriv->rtlhal.current_bandtype == BAND_ON_2_4G)
+			mac->basic_rates = 0xfff;
+		else
+			mac->basic_rates = 0xff0;
+		rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_BASIC_RATE,
+					      (u8 *)(&mac->basic_rates));
+
+		retry_limit = 0x07;
+		break;
+	case NL80211_IFTYPE_MESH_POINT:
+		RT_TRACE(rtlpriv, COMP_MAC80211, DBG_LOUD,
+			 "NL80211_IFTYPE_MESH_POINT\n");
+
+		mac->link_state = MAC80211_LINKED;
+		rtlpriv->cfg->ops->set_bcn_reg(hw);
+		if (rtlpriv->rtlhal.current_bandtype == BAND_ON_2_4G)
+			mac->basic_rates = 0xfff;
+		else
+			mac->basic_rates = 0xff0;
+		rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_BASIC_RATE,
+				(u8 *)(&mac->basic_rates));
+
+		retry_limit = 0x07;
+		break;
+	default:
+		pr_err("operation mode %d is not supported!\n",
+		       vif->type);
+		err = -EOPNOTSUPP;
+		goto out;
+	}
+
+	if (mac->p2p) {
+		RT_TRACE(rtlpriv, COMP_MAC80211, DBG_LOUD,
+			 "p2p role %x\n", vif->type);
+		mac->basic_rates = 0xff0;/*disable cck rate for p2p*/
+		rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_BASIC_RATE,
+				(u8 *)(&mac->basic_rates));
+	}
+	mac->vif = vif;
+	mac->opmode = vif->type;
+	rtlpriv->cfg->ops->set_network_type(hw, vif->type);
+	memcpy(mac->mac_addr, vif->addr, ETH_ALEN);
+	rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_ETHER_ADDR, mac->mac_addr);
+
+	mac->retry_long = retry_limit;
+	mac->retry_short = retry_limit;
+	rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_RETRY_LIMIT,
+			(u8 *)(&retry_limit));
+out:
+	mutex_unlock(&rtlpriv->locks.conf_mutex);
+	return err;
+}
+
+static void rtl_op_remove_interface(struct ieee80211_hw *hw,
+				    struct ieee80211_vif *vif)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+
+	mutex_lock(&rtlpriv->locks.conf_mutex);
+
+	/* Free beacon resources */
+	if ((vif->type == NL80211_IFTYPE_AP) ||
+	    (vif->type == NL80211_IFTYPE_ADHOC) ||
+	    (vif->type == NL80211_IFTYPE_MESH_POINT)) {
+		if (mac->beacon_enabled == 1) {
+			mac->beacon_enabled = 0;
+			rtlpriv->cfg->ops->update_interrupt_mask(hw, 0,
+					rtlpriv->cfg->maps[RTL_IBSS_INT_MASKS]);
+		}
+	}
+
+	/*
+	 *Note: We assume NL80211_IFTYPE_UNSPECIFIED as
+	 *NO LINK for our hardware.
+	 */
+	mac->p2p = 0;
+	mac->vif = NULL;
+	mac->link_state = MAC80211_NOLINK;
+	eth_zero_addr(mac->bssid);
+	mac->vendor = PEER_UNKNOWN;
+	mac->opmode = NL80211_IFTYPE_UNSPECIFIED;
+	rtlpriv->cfg->ops->set_network_type(hw, mac->opmode);
+
+	mutex_unlock(&rtlpriv->locks.conf_mutex);
+}
+
+static int rtl_op_change_interface(struct ieee80211_hw *hw,
+				   struct ieee80211_vif *vif,
+				   enum nl80211_iftype new_type, bool p2p)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	int ret;
+
+	rtl_op_remove_interface(hw, vif);
+
+	vif->type = new_type;
+	vif->p2p = p2p;
+	ret = rtl_op_add_interface(hw, vif);
+	RT_TRACE(rtlpriv, COMP_MAC80211, DBG_LOUD,
+		 "p2p  %x\n", p2p);
+	return ret;
+}
+
+#ifdef CONFIG_PM
+static u16 crc16_ccitt(u8 data, u16 crc)
+{
+	u8 shift_in, data_bit, crc_bit11, crc_bit4, crc_bit15;
+	u8 i;
+	u16 result;
+
+	for (i = 0; i < 8; i++) {
+		crc_bit15 = ((crc & BIT(15)) ? 1 : 0);
+		data_bit  = (data & (BIT(0) << i) ? 1 : 0);
+		shift_in = crc_bit15 ^ data_bit;
+
+		result = crc << 1;
+		if (shift_in == 0)
+			result &= (~BIT(0));
+		else
+			result |= BIT(0);
+
+		crc_bit11 = ((crc & BIT(11)) ? 1 : 0) ^ shift_in;
+		if (crc_bit11 == 0)
+			result &= (~BIT(12));
+		else
+			result |= BIT(12);
+
+		crc_bit4 = ((crc & BIT(4)) ? 1 : 0) ^ shift_in;
+		if (crc_bit4 == 0)
+			result &= (~BIT(5));
+		else
+			result |= BIT(5);
+
+		crc = result;
+	}
+
+	return crc;
+}
+
+static u16 _calculate_wol_pattern_crc(u8 *pattern, u16 len)
+{
+	u16 crc = 0xffff;
+	u32 i;
+
+	for (i = 0; i < len; i++)
+		crc = crc16_ccitt(pattern[i], crc);
+
+	crc = ~crc;
+
+	return crc;
+}
+
+static void _rtl_add_wowlan_patterns(struct ieee80211_hw *hw,
+				     struct cfg80211_wowlan *wow)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = &rtlpriv->mac80211;
+	struct cfg80211_pkt_pattern *patterns = wow->patterns;
+	struct rtl_wow_pattern rtl_pattern;
+	const u8 *pattern_os, *mask_os;
+	u8 mask[MAX_WOL_BIT_MASK_SIZE] = {0};
+	u8 content[MAX_WOL_PATTERN_SIZE] = {0};
+	u8 broadcast_addr[6] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+	u8 multicast_addr1[2] = {0x33, 0x33};
+	u8 multicast_addr2[3] = {0x01, 0x00, 0x5e};
+	u8 i, mask_len;
+	u16 j, len;
+
+	for (i = 0; i < wow->n_patterns; i++) {
+		memset(&rtl_pattern, 0, sizeof(struct rtl_wow_pattern));
+		memset(mask, 0, MAX_WOL_BIT_MASK_SIZE);
+		if (patterns[i].pattern_len > MAX_WOL_PATTERN_SIZE) {
+			RT_TRACE(rtlpriv, COMP_POWER, DBG_WARNING,
+				 "Pattern[%d] is too long\n", i);
+			continue;
+		}
+		pattern_os = patterns[i].pattern;
+		mask_len = DIV_ROUND_UP(patterns[i].pattern_len, 8);
+		mask_os = patterns[i].mask;
+		RT_PRINT_DATA(rtlpriv, COMP_POWER, DBG_TRACE,
+			      "pattern content\n", pattern_os,
+			       patterns[i].pattern_len);
+		RT_PRINT_DATA(rtlpriv, COMP_POWER, DBG_TRACE,
+			      "mask content\n", mask_os, mask_len);
+		/* 1. unicast? multicast? or broadcast? */
+		if (memcmp(pattern_os, broadcast_addr, 6) == 0)
+			rtl_pattern.type = BROADCAST_PATTERN;
+		else if (memcmp(pattern_os, multicast_addr1, 2) == 0 ||
+			 memcmp(pattern_os, multicast_addr2, 3) == 0)
+			rtl_pattern.type = MULTICAST_PATTERN;
+		else if  (memcmp(pattern_os, mac->mac_addr, 6) == 0)
+			rtl_pattern.type = UNICAST_PATTERN;
+		else
+			rtl_pattern.type = UNKNOWN_TYPE;
+
+		/* 2. translate mask_from_os to mask_for_hw */
+
+/******************************************************************************
+ * pattern from OS uses 'ethenet frame', like this:
+
+		   |    6   |    6   |   2  |     20    |  Variable  |	4  |
+		   |--------+--------+------+-----------+------------+-----|
+		   |    802.3 Mac Header    | IP Header | TCP Packet | FCS |
+		   |   DA   |   SA   | Type |
+
+ * BUT, packet catched by our HW is in '802.11 frame', begin from LLC,
+
+	|     24 or 30      |    6   |   2  |     20    |  Variable  |  4  |
+	|-------------------+--------+------+-----------+------------+-----|
+	| 802.11 MAC Header |       LLC     | IP Header | TCP Packet | FCS |
+			    | Others | Tpye |
+
+ * Therefore, we need translate mask_from_OS to mask_to_hw.
+ * We should left-shift mask by 6 bits, then set the new bit[0~5] = 0,
+ * because new mask[0~5] means 'SA', but our HW packet begins from LLC,
+ * bit[0~5] corresponds to first 6 Bytes in LLC, they just don't match.
+ ******************************************************************************/
+
+		/* Shift 6 bits */
+		for (j = 0; j < mask_len - 1; j++) {
+			mask[j] = mask_os[j] >> 6;
+			mask[j] |= (mask_os[j + 1] & 0x3F) << 2;
+		}
+		mask[j] = (mask_os[j] >> 6) & 0x3F;
+		/* Set bit 0-5 to zero */
+		mask[0] &= 0xC0;
+
+		RT_PRINT_DATA(rtlpriv, COMP_POWER, DBG_TRACE,
+			      "mask to hw\n", mask, mask_len);
+		for (j = 0; j < (MAX_WOL_BIT_MASK_SIZE + 1) / 4; j++) {
+			rtl_pattern.mask[j] = mask[j * 4];
+			rtl_pattern.mask[j] |= (mask[j * 4 + 1] << 8);
+			rtl_pattern.mask[j] |= (mask[j * 4 + 2] << 16);
+			rtl_pattern.mask[j] |= (mask[j * 4 + 3] << 24);
+		}
+
+		/* To get the wake up pattern from the mask.
+		 * We do not count first 12 bits which means
+		 * DA[6] and SA[6] in the pattern to match HW design.
+		 */
+		len = 0;
+		for (j = 12; j < patterns[i].pattern_len; j++) {
+			if ((mask_os[j / 8] >> (j % 8)) & 0x01) {
+				content[len] = pattern_os[j];
+				len++;
+			}
+		}
+
+		RT_PRINT_DATA(rtlpriv, COMP_POWER, DBG_TRACE,
+			      "pattern to hw\n", content, len);
+		/* 3. calculate crc */
+		rtl_pattern.crc = _calculate_wol_pattern_crc(content, len);
+		RT_TRACE(rtlpriv, COMP_POWER, DBG_TRACE,
+			 "CRC_Remainder = 0x%x\n", rtl_pattern.crc);
+
+		/* 4. write crc & mask_for_hw to hw */
+		rtlpriv->cfg->ops->add_wowlan_pattern(hw, &rtl_pattern, i);
+	}
+	rtl_write_byte(rtlpriv, 0x698, wow->n_patterns);
+}
+
+static int rtl_op_suspend(struct ieee80211_hw *hw,
+			  struct cfg80211_wowlan *wow)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	struct timeval ts;
+
+	RT_TRACE(rtlpriv, COMP_POWER, DBG_DMESG, "\n");
+	if (WARN_ON(!wow))
+		return -EINVAL;
+
+	/* to resolve s4 can not wake up*/
+	do_gettimeofday(&ts);
+	rtlhal->last_suspend_sec = ts.tv_sec;
+
+	if ((ppsc->wo_wlan_mode & WAKE_ON_PATTERN_MATCH) && wow->n_patterns)
+		_rtl_add_wowlan_patterns(hw, wow);
+
+	rtlhal->driver_is_goingto_unload = true;
+	rtlhal->enter_pnp_sleep = true;
+
+	rtl_lps_leave(hw);
+	rtl_op_stop(hw);
+	device_set_wakeup_enable(wiphy_dev(hw->wiphy), true);
+	return 0;
+}
+
+static int rtl_op_resume(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct timeval ts;
+
+	RT_TRACE(rtlpriv, COMP_POWER, DBG_DMESG, "\n");
+	rtlhal->driver_is_goingto_unload = false;
+	rtlhal->enter_pnp_sleep = false;
+	rtlhal->wake_from_pnp_sleep = true;
+
+	/* to resovle s4 can not wake up*/
+	do_gettimeofday(&ts);
+	if (ts.tv_sec - rtlhal->last_suspend_sec < 5)
+		return -1;
+
+	rtl_op_start(hw);
+	device_set_wakeup_enable(wiphy_dev(hw->wiphy), false);
+	ieee80211_resume_disconnect(mac->vif);
+	rtlhal->wake_from_pnp_sleep = false;
+	return 0;
+}
+#endif
+
+static int rtl_op_config(struct ieee80211_hw *hw, u32 changed)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	struct ieee80211_conf *conf = &hw->conf;
+
+	if (mac->skip_scan)
+		return 1;
+
+	mutex_lock(&rtlpriv->locks.conf_mutex);
+	if (changed & IEEE80211_CONF_CHANGE_LISTEN_INTERVAL) {	/* BIT(2)*/
+		RT_TRACE(rtlpriv, COMP_MAC80211, DBG_LOUD,
+			 "IEEE80211_CONF_CHANGE_LISTEN_INTERVAL\n");
+	}
+
+	/*For IPS */
+	if (changed & IEEE80211_CONF_CHANGE_IDLE) {
+		if (hw->conf.flags & IEEE80211_CONF_IDLE)
+			rtl_ips_nic_off(hw);
+		else
+			rtl_ips_nic_on(hw);
+	} else {
+		/*
+		 *although rfoff may not cause by ips, but we will
+		 *check the reason in set_rf_power_state function
+		 */
+		if (unlikely(ppsc->rfpwr_state == ERFOFF))
+			rtl_ips_nic_on(hw);
+	}
+
+	/*For LPS */
+	if ((changed & IEEE80211_CONF_CHANGE_PS) &&
+	    rtlpriv->psc.swctrl_lps && !rtlpriv->psc.fwctrl_lps) {
+		cancel_delayed_work(&rtlpriv->works.ps_work);
+		cancel_delayed_work(&rtlpriv->works.ps_rfon_wq);
+		if (conf->flags & IEEE80211_CONF_PS) {
+			rtlpriv->psc.sw_ps_enabled = true;
+			/* sleep here is must, or we may recv the beacon and
+			 * cause mac80211 into wrong ps state, this will cause
+			 * power save nullfunc send fail, and further cause
+			 * pkt loss, So sleep must quickly but not immediately
+			 * because that will cause nullfunc send by mac80211
+			 * fail, and cause pkt loss, we have tested that 5mA
+			 * works very well
+			 */
+			if (!rtlpriv->psc.multi_buffered)
+				queue_delayed_work(rtlpriv->works.rtl_wq,
+						   &rtlpriv->works.ps_work,
+						   MSECS(5));
+		} else {
+			rtl_swlps_rf_awake(hw);
+			rtlpriv->psc.sw_ps_enabled = false;
+		}
+	}
+
+	if (changed & IEEE80211_CONF_CHANGE_RETRY_LIMITS) {
+		RT_TRACE(rtlpriv, COMP_MAC80211, DBG_LOUD,
+			 "IEEE80211_CONF_CHANGE_RETRY_LIMITS %x\n",
+			 hw->conf.long_frame_max_tx_count);
+		/* brought up everything changes (changed == ~0) indicates first
+		 * open, so use our default value instead of that of wiphy.
+		 */
+		if (changed != ~0) {
+			mac->retry_long = hw->conf.long_frame_max_tx_count;
+			mac->retry_short = hw->conf.long_frame_max_tx_count;
+			rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_RETRY_LIMIT,
+				(u8 *)(&hw->conf.long_frame_max_tx_count));
+		}
+	}
+
+	if (changed & IEEE80211_CONF_CHANGE_CHANNEL &&
+	    !rtlpriv->proximity.proxim_on) {
+		struct ieee80211_channel *channel = hw->conf.chandef.chan;
+		enum nl80211_chan_width width = hw->conf.chandef.width;
+		enum nl80211_channel_type channel_type = NL80211_CHAN_NO_HT;
+		u8 wide_chan = (u8)channel->hw_value;
+
+		/* channel_type is for 20&40M */
+		if (width < NL80211_CHAN_WIDTH_80)
+			channel_type =
+				cfg80211_get_chandef_type(&hw->conf.chandef);
+		if (mac->act_scanning)
+			mac->n_channels++;
+
+		if (rtlpriv->dm.supp_phymode_switch &&
+		    mac->link_state < MAC80211_LINKED &&
+		    !mac->act_scanning) {
+			if (rtlpriv->cfg->ops->chk_switch_dmdp)
+				rtlpriv->cfg->ops->chk_switch_dmdp(hw);
+		}
+
+		/*
+		 *because we should back channel to
+		 *current_network.chan in in scanning,
+		 *So if set_chan == current_network.chan
+		 *we should set it.
+		 *because mac80211 tell us wrong bw40
+		 *info for cisco1253 bw20, so we modify
+		 *it here based on UPPER & LOWER
+		 */
+
+		if (width >= NL80211_CHAN_WIDTH_80) {
+			if (width == NL80211_CHAN_WIDTH_80) {
+				u32 center = hw->conf.chandef.center_freq1;
+				u32 primary =
+				(u32)hw->conf.chandef.chan->center_freq;
+
+				rtlphy->current_chan_bw =
+					HT_CHANNEL_WIDTH_80;
+				mac->bw_80 = true;
+				mac->bw_40 = true;
+				if (center > primary) {
+					mac->cur_80_prime_sc =
+					PRIME_CHNL_OFFSET_LOWER;
+					if (center - primary == 10) {
+						mac->cur_40_prime_sc =
+						PRIME_CHNL_OFFSET_UPPER;
+
+						wide_chan += 2;
+					} else if (center - primary == 30) {
+						mac->cur_40_prime_sc =
+						PRIME_CHNL_OFFSET_LOWER;
+
+						wide_chan += 6;
+					}
+				} else {
+					mac->cur_80_prime_sc =
+					PRIME_CHNL_OFFSET_UPPER;
+					if (primary - center == 10) {
+						mac->cur_40_prime_sc =
+						PRIME_CHNL_OFFSET_LOWER;
+
+						wide_chan -= 2;
+					} else if (primary - center == 30) {
+						mac->cur_40_prime_sc =
+						PRIME_CHNL_OFFSET_UPPER;
+
+						wide_chan -= 6;
+					}
+				}
+			}
+		} else {
+			switch (channel_type) {
+			case NL80211_CHAN_HT20:
+			case NL80211_CHAN_NO_HT:
+					/* SC */
+					mac->cur_40_prime_sc =
+						PRIME_CHNL_OFFSET_DONT_CARE;
+					rtlphy->current_chan_bw =
+						HT_CHANNEL_WIDTH_20;
+					mac->bw_40 = false;
+					mac->bw_80 = false;
+					break;
+			case NL80211_CHAN_HT40MINUS:
+					/* SC */
+					mac->cur_40_prime_sc =
+						PRIME_CHNL_OFFSET_UPPER;
+					rtlphy->current_chan_bw =
+						HT_CHANNEL_WIDTH_20_40;
+					mac->bw_40 = true;
+					mac->bw_80 = false;
+
+					/*wide channel */
+					wide_chan -= 2;
+
+					break;
+			case NL80211_CHAN_HT40PLUS:
+					/* SC */
+					mac->cur_40_prime_sc =
+						PRIME_CHNL_OFFSET_LOWER;
+					rtlphy->current_chan_bw =
+						HT_CHANNEL_WIDTH_20_40;
+					mac->bw_40 = true;
+					mac->bw_80 = false;
+
+					/*wide channel */
+					wide_chan += 2;
+
+					break;
+			default:
+					mac->bw_40 = false;
+					mac->bw_80 = false;
+					pr_err("switch case %#x not processed\n",
+					       channel_type);
+					break;
+			}
+		}
+
+		if (wide_chan <= 0)
+			wide_chan = 1;
+
+		/* In scanning, when before we offchannel we may send a ps=1
+		 * null to AP, and then we may send a ps = 0 null to AP quickly,
+		 * but first null may have caused AP to put lots of packet to
+		 * hw tx buffer. These packets must be tx'd before we go off
+		 * channel so we must delay more time to let AP flush these
+		 * packets before going offchannel, or dis-association or
+		 * delete BA will be caused by AP
+		 */
+		if (rtlpriv->mac80211.offchan_delay) {
+			rtlpriv->mac80211.offchan_delay = false;
+			mdelay(50);
+		}
+
+		rtlphy->current_channel = wide_chan;
+
+		rtlpriv->cfg->ops->switch_channel(hw);
+		rtlpriv->cfg->ops->set_channel_access(hw);
+		rtlpriv->cfg->ops->set_bw_mode(hw, channel_type);
+	}
+
+	mutex_unlock(&rtlpriv->locks.conf_mutex);
+
+	return 0;
+}
+
+static void rtl_op_configure_filter(struct ieee80211_hw *hw,
+				    unsigned int changed_flags,
+				    unsigned int *new_flags, u64 multicast)
+{
+	bool update_rcr = false;
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+
+	*new_flags &= RTL_SUPPORTED_FILTERS;
+	if (changed_flags == 0)
+		return;
+
+	/*TODO: we disable broadcase now, so enable here */
+	if (changed_flags & FIF_ALLMULTI) {
+		if (*new_flags & FIF_ALLMULTI) {
+			mac->rx_conf |= rtlpriv->cfg->maps[MAC_RCR_AM] |
+			    rtlpriv->cfg->maps[MAC_RCR_AB];
+			RT_TRACE(rtlpriv, COMP_MAC80211, DBG_LOUD,
+				 "Enable receive multicast frame\n");
+		} else {
+			mac->rx_conf &= ~(rtlpriv->cfg->maps[MAC_RCR_AM] |
+					  rtlpriv->cfg->maps[MAC_RCR_AB]);
+			RT_TRACE(rtlpriv, COMP_MAC80211, DBG_LOUD,
+				 "Disable receive multicast frame\n");
+		}
+		update_rcr = true;
+	}
+
+	if (changed_flags & FIF_FCSFAIL) {
+		if (*new_flags & FIF_FCSFAIL) {
+			mac->rx_conf |= rtlpriv->cfg->maps[MAC_RCR_ACRC32];
+			RT_TRACE(rtlpriv, COMP_MAC80211, DBG_LOUD,
+				 "Enable receive FCS error frame\n");
+		} else {
+			mac->rx_conf &= ~rtlpriv->cfg->maps[MAC_RCR_ACRC32];
+			RT_TRACE(rtlpriv, COMP_MAC80211, DBG_LOUD,
+				 "Disable receive FCS error frame\n");
+		}
+		if (!update_rcr)
+			update_rcr = true;
+	}
+
+	/* if ssid not set to hw don't check bssid
+	 * here just used for linked scanning, & linked
+	 * and nolink check bssid is set in set network_type
+	 */
+	if ((changed_flags & FIF_BCN_PRBRESP_PROMISC) &&
+	    (mac->link_state >= MAC80211_LINKED)) {
+		if (mac->opmode != NL80211_IFTYPE_AP &&
+		    mac->opmode != NL80211_IFTYPE_MESH_POINT) {
+			if (*new_flags & FIF_BCN_PRBRESP_PROMISC)
+				rtlpriv->cfg->ops->set_chk_bssid(hw, false);
+			else
+				rtlpriv->cfg->ops->set_chk_bssid(hw, true);
+			if (update_rcr)
+				update_rcr = false;
+		}
+	}
+
+	if (changed_flags & FIF_CONTROL) {
+		if (*new_flags & FIF_CONTROL) {
+			mac->rx_conf |= rtlpriv->cfg->maps[MAC_RCR_ACF];
+
+			RT_TRACE(rtlpriv, COMP_MAC80211, DBG_LOUD,
+				 "Enable receive control frame.\n");
+		} else {
+			mac->rx_conf &= ~rtlpriv->cfg->maps[MAC_RCR_ACF];
+			RT_TRACE(rtlpriv, COMP_MAC80211, DBG_LOUD,
+				 "Disable receive control frame.\n");
+		}
+		if (!update_rcr)
+			update_rcr = true;
+	}
+
+	if (changed_flags & FIF_OTHER_BSS) {
+		if (*new_flags & FIF_OTHER_BSS) {
+			mac->rx_conf |= rtlpriv->cfg->maps[MAC_RCR_AAP];
+			RT_TRACE(rtlpriv, COMP_MAC80211, DBG_LOUD,
+				 "Enable receive other BSS's frame.\n");
+		} else {
+			mac->rx_conf &= ~rtlpriv->cfg->maps[MAC_RCR_AAP];
+			RT_TRACE(rtlpriv, COMP_MAC80211, DBG_LOUD,
+				 "Disable receive other BSS's frame.\n");
+		}
+		if (!update_rcr)
+			update_rcr = true;
+	}
+
+	if (update_rcr)
+		rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_RCR,
+					      (u8 *)(&mac->rx_conf));
+}
+
+static int rtl_op_sta_add(struct ieee80211_hw *hw,
+			  struct ieee80211_vif *vif,
+			  struct ieee80211_sta *sta)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct rtl_sta_info *sta_entry;
+
+	if (sta) {
+		sta_entry = (struct rtl_sta_info *)sta->drv_priv;
+		spin_lock_bh(&rtlpriv->locks.entry_list_lock);
+		list_add_tail(&sta_entry->list, &rtlpriv->entry_list);
+		spin_unlock_bh(&rtlpriv->locks.entry_list_lock);
+		if (rtlhal->current_bandtype == BAND_ON_2_4G) {
+			sta_entry->wireless_mode = WIRELESS_MODE_G;
+			if (sta->supp_rates[0] <= 0xf)
+				sta_entry->wireless_mode = WIRELESS_MODE_B;
+			if (sta->ht_cap.ht_supported)
+				sta_entry->wireless_mode = WIRELESS_MODE_N_24G;
+
+			if (vif->type == NL80211_IFTYPE_ADHOC)
+				sta_entry->wireless_mode = WIRELESS_MODE_G;
+		} else if (rtlhal->current_bandtype == BAND_ON_5G) {
+			sta_entry->wireless_mode = WIRELESS_MODE_A;
+			if (sta->ht_cap.ht_supported)
+				sta_entry->wireless_mode = WIRELESS_MODE_N_5G;
+			if (sta->vht_cap.vht_supported)
+				sta_entry->wireless_mode = WIRELESS_MODE_AC_5G;
+
+			if (vif->type == NL80211_IFTYPE_ADHOC)
+				sta_entry->wireless_mode = WIRELESS_MODE_A;
+		}
+		/*disable cck rate for p2p*/
+		if (mac->p2p)
+			sta->supp_rates[0] &= 0xfffffff0;
+
+		if (sta->ht_cap.ht_supported) {
+			if (sta->ht_cap.cap & IEEE80211_HT_CAP_SUP_WIDTH_20_40)
+				rtlphy->max_ht_chan_bw = HT_CHANNEL_WIDTH_20_40;
+			else
+				rtlphy->max_ht_chan_bw = HT_CHANNEL_WIDTH_20;
+		}
+
+		if (sta->vht_cap.vht_supported)
+			rtlphy->max_vht_chan_bw = HT_CHANNEL_WIDTH_80;
+
+		memcpy(sta_entry->mac_addr, sta->addr, ETH_ALEN);
+		RT_TRACE(rtlpriv, COMP_MAC80211, DBG_DMESG,
+			 "Add sta addr is %pM\n", sta->addr);
+		rtlpriv->cfg->ops->update_rate_tbl(hw, sta, 0, true);
+
+		if (rtlpriv->phydm.ops)
+			rtlpriv->phydm.ops->phydm_add_sta(rtlpriv, sta);
+	}
+
+	return 0;
+}
+
+static int rtl_op_sta_remove(struct ieee80211_hw *hw,
+			     struct ieee80211_vif *vif,
+			     struct ieee80211_sta *sta)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_sta_info *sta_entry;
+
+	if (sta) {
+		RT_TRACE(rtlpriv, COMP_MAC80211, DBG_DMESG,
+			 "Remove sta addr is %pM\n", sta->addr);
+
+		if (rtlpriv->phydm.ops)
+			rtlpriv->phydm.ops->phydm_del_sta(rtlpriv, sta);
+
+		sta_entry = (struct rtl_sta_info *)sta->drv_priv;
+		sta_entry->wireless_mode = 0;
+		sta_entry->ratr_index = 0;
+		spin_lock_bh(&rtlpriv->locks.entry_list_lock);
+		list_del(&sta_entry->list);
+		spin_unlock_bh(&rtlpriv->locks.entry_list_lock);
+	}
+	return 0;
+}
+
+static int _rtl_get_hal_qnum(u16 queue)
+{
+	int qnum;
+
+	switch (queue) {
+	case 0:
+		qnum = AC3_VO;
+		break;
+	case 1:
+		qnum = AC2_VI;
+		break;
+	case 2:
+		qnum = AC0_BE;
+		break;
+	case 3:
+		qnum = AC1_BK;
+		break;
+	default:
+		qnum = AC0_BE;
+		break;
+	}
+	return qnum;
+}
+
+static void rtl_op_sta_statistics(struct ieee80211_hw *hw,
+				  struct ieee80211_vif *vif,
+				  struct ieee80211_sta *sta,
+				  struct station_info *sinfo)
+{
+	/* nothing filled by driver, so mac80211 will update all info */
+	sinfo->filled = 0;
+}
+
+static int rtl_op_set_frag_threshold(struct ieee80211_hw *hw, u32 value)
+{
+	return -EOPNOTSUPP;
+}
+
+/*
+ *for mac80211 VO = 0, VI = 1, BE = 2, BK = 3
+ *for rtl819x  BE = 0, BK = 1, VI = 2, VO = 3
+ */
+static int rtl_op_conf_tx(struct ieee80211_hw *hw,
+			  struct ieee80211_vif *vif, u16 queue,
+			  const struct ieee80211_tx_queue_params *param)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	int aci;
+
+	if (queue >= AC_MAX) {
+		RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING,
+			 "queue number %d is incorrect!\n", queue);
+		return -EINVAL;
+	}
+
+	aci = _rtl_get_hal_qnum(queue);
+	mac->ac[aci].aifs = param->aifs;
+	mac->ac[aci].cw_min = cpu_to_le16(param->cw_min);
+	mac->ac[aci].cw_max = cpu_to_le16(param->cw_max);
+	mac->ac[aci].tx_op = cpu_to_le16(param->txop);
+	memcpy(&mac->edca_param[aci], param, sizeof(*param));
+	rtlpriv->cfg->ops->set_qos(hw, aci);
+	return 0;
+}
+
+static void send_beacon_frame(struct ieee80211_hw *hw,
+			      struct ieee80211_vif *vif)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct sk_buff *skb = ieee80211_beacon_get(hw, vif);
+	struct rtl_tcb_desc tcb_desc;
+
+	if (skb) {
+		memset(&tcb_desc, 0, sizeof(struct rtl_tcb_desc));
+		rtlpriv->intf_ops->adapter_tx(hw, NULL, skb, &tcb_desc);
+	}
+}
+
+static void rtl_op_bss_info_changed(struct ieee80211_hw *hw,
+				    struct ieee80211_vif *vif,
+				    struct ieee80211_bss_conf *bss_conf,
+				    u32 changed)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+
+	mutex_lock(&rtlpriv->locks.conf_mutex);
+	if ((vif->type == NL80211_IFTYPE_ADHOC) ||
+	    (vif->type == NL80211_IFTYPE_AP) ||
+	    (vif->type == NL80211_IFTYPE_MESH_POINT)) {
+		if ((changed & BSS_CHANGED_BEACON) ||
+		    (changed & BSS_CHANGED_BEACON_ENABLED &&
+		     bss_conf->enable_beacon)) {
+			if (mac->beacon_enabled == 0) {
+				RT_TRACE(rtlpriv, COMP_MAC80211, DBG_DMESG,
+					 "BSS_CHANGED_BEACON_ENABLED\n");
+
+				/*start hw beacon interrupt. */
+				/*rtlpriv->cfg->ops->set_bcn_reg(hw); */
+				mac->beacon_enabled = 1;
+				rtlpriv->cfg->ops->update_interrupt_mask(hw,
+						rtlpriv->cfg->maps
+						[RTL_IBSS_INT_MASKS], 0);
+
+				if (rtlpriv->cfg->ops->linked_set_reg)
+					rtlpriv->cfg->ops->linked_set_reg(hw);
+				send_beacon_frame(hw, vif);
+			}
+		}
+		if ((changed & BSS_CHANGED_BEACON_ENABLED &&
+		     !bss_conf->enable_beacon)) {
+			if (mac->beacon_enabled == 1) {
+				RT_TRACE(rtlpriv, COMP_MAC80211, DBG_DMESG,
+					 "ADHOC DISABLE BEACON\n");
+
+				mac->beacon_enabled = 0;
+				rtlpriv->cfg->ops->update_interrupt_mask(hw, 0,
+						rtlpriv->cfg->maps
+						[RTL_IBSS_INT_MASKS]);
+			}
+		}
+		if (changed & BSS_CHANGED_BEACON_INT) {
+			RT_TRACE(rtlpriv, COMP_BEACON, DBG_TRACE,
+				 "BSS_CHANGED_BEACON_INT\n");
+			mac->beacon_interval = bss_conf->beacon_int;
+			rtlpriv->cfg->ops->set_bcn_intv(hw);
+		}
+	}
+
+	/*TODO: reference to enum ieee80211_bss_change */
+	if (changed & BSS_CHANGED_ASSOC) {
+		u8 mstatus;
+
+		if (bss_conf->assoc) {
+			struct ieee80211_sta *sta = NULL;
+			u8 keep_alive = 10;
+
+			mstatus = RT_MEDIA_CONNECT;
+			/* we should reset all sec info & cam
+			 * before set cam after linked, we should not
+			 * reset in disassoc, that will cause tkip->wep
+			 * fail because some flag will be wrong
+			 * reset sec info
+			 */
+			rtl_cam_reset_sec_info(hw);
+			/* reset cam to fix wep fail issue
+			 * when change from wpa to wep
+			 */
+			rtl_cam_reset_all_entry(hw);
+
+			mac->link_state = MAC80211_LINKED;
+			mac->cnt_after_linked = 0;
+			mac->assoc_id = bss_conf->aid;
+			memcpy(mac->bssid, bss_conf->bssid, ETH_ALEN);
+
+			if (rtlpriv->cfg->ops->linked_set_reg)
+				rtlpriv->cfg->ops->linked_set_reg(hw);
+
+			rcu_read_lock();
+			sta = ieee80211_find_sta(vif, (u8 *)bss_conf->bssid);
+			if (!sta) {
+				rcu_read_unlock();
+				goto out;
+			}
+			RT_TRACE(rtlpriv, COMP_EASY_CONCURRENT, DBG_LOUD,
+				 "send PS STATIC frame\n");
+			if (rtlpriv->dm.supp_phymode_switch) {
+				if (sta->ht_cap.ht_supported)
+					rtl_send_smps_action(hw, sta,
+							IEEE80211_SMPS_STATIC);
+			}
+
+			if (rtlhal->current_bandtype == BAND_ON_5G) {
+				mac->mode = WIRELESS_MODE_A;
+			} else {
+				if (sta->supp_rates[0] <= 0xf)
+					mac->mode = WIRELESS_MODE_B;
+				else
+					mac->mode = WIRELESS_MODE_G;
+			}
+
+			if (sta->ht_cap.ht_supported) {
+				if (rtlhal->current_bandtype == BAND_ON_2_4G)
+					mac->mode = WIRELESS_MODE_N_24G;
+				else
+					mac->mode = WIRELESS_MODE_N_5G;
+			}
+
+			if (sta->vht_cap.vht_supported) {
+				if (rtlhal->current_bandtype == BAND_ON_5G)
+					mac->mode = WIRELESS_MODE_AC_5G;
+				else
+					mac->mode = WIRELESS_MODE_AC_24G;
+			}
+
+			if (vif->type == NL80211_IFTYPE_STATION)
+				rtlpriv->cfg->ops->update_rate_tbl(hw, sta, 0,
+								   true);
+			rcu_read_unlock();
+
+			/* to avoid AP Disassociation caused by inactivity */
+			rtlpriv->cfg->ops->set_hw_reg(hw,
+						      HW_VAR_KEEP_ALIVE,
+						      (u8 *)(&keep_alive));
+
+			RT_TRACE(rtlpriv, COMP_MAC80211, DBG_DMESG,
+				 "BSS_CHANGED_ASSOC\n");
+		} else {
+			struct cfg80211_bss *bss = NULL;
+
+			mstatus = RT_MEDIA_DISCONNECT;
+
+			if (mac->link_state == MAC80211_LINKED)
+				rtl_lps_leave(hw);
+			if (ppsc->p2p_ps_info.p2p_ps_mode > P2P_PS_NONE)
+				rtl_p2p_ps_cmd(hw, P2P_PS_DISABLE);
+			mac->link_state = MAC80211_NOLINK;
+
+			bss = cfg80211_get_bss(hw->wiphy, NULL,
+					       (u8 *)mac->bssid, NULL, 0,
+					       IEEE80211_BSS_TYPE_ESS,
+					       IEEE80211_PRIVACY_OFF);
+
+			RT_TRACE(rtlpriv, COMP_MAC80211, DBG_DMESG,
+				 "bssid = %x-%x-%x-%x-%x-%x\n",
+				 mac->bssid[0], mac->bssid[1],
+				 mac->bssid[2], mac->bssid[3],
+				 mac->bssid[4], mac->bssid[5]);
+
+			if (bss) {
+				cfg80211_unlink_bss(hw->wiphy, bss);
+				cfg80211_put_bss(hw->wiphy, bss);
+				RT_TRACE(rtlpriv, COMP_MAC80211, DBG_DMESG,
+					"cfg80211_unlink !!\n");
+			}
+
+			eth_zero_addr(mac->bssid);
+			mac->vendor = PEER_UNKNOWN;
+			mac->mode = 0;
+
+			if (rtlpriv->dm.supp_phymode_switch) {
+				if (rtlpriv->cfg->ops->chk_switch_dmdp)
+					rtlpriv->cfg->ops->chk_switch_dmdp(hw);
+			}
+			RT_TRACE(rtlpriv, COMP_MAC80211, DBG_DMESG,
+				 "BSS_CHANGED_UN_ASSOC\n");
+		}
+		rtlpriv->cfg->ops->set_network_type(hw, vif->type);
+		/* For FW LPS:
+		 * To tell firmware we have connected or disconnected
+		 */
+		rtlpriv->cfg->ops->set_hw_reg(hw,
+					      HW_VAR_H2C_FW_JOINBSSRPT,
+					      (u8 *)(&mstatus));
+		ppsc->report_linked = (mstatus == RT_MEDIA_CONNECT) ?
+				      true : false;
+
+		if (rtlpriv->cfg->ops->get_btc_status())
+			rtlpriv->btcoexist.btc_ops->btc_mediastatus_notify(
+							rtlpriv, mstatus);
+	}
+
+	if (changed & BSS_CHANGED_ERP_CTS_PROT) {
+		RT_TRACE(rtlpriv, COMP_MAC80211, DBG_TRACE,
+			 "BSS_CHANGED_ERP_CTS_PROT\n");
+		mac->use_cts_protect = bss_conf->use_cts_prot;
+	}
+
+	if (changed & BSS_CHANGED_ERP_PREAMBLE) {
+		RT_TRACE(rtlpriv, COMP_MAC80211, DBG_LOUD,
+			 "BSS_CHANGED_ERP_PREAMBLE use short preamble:%x\n",
+			  bss_conf->use_short_preamble);
+
+		mac->short_preamble = bss_conf->use_short_preamble;
+		rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_ACK_PREAMBLE,
+					      (u8 *)(&mac->short_preamble));
+	}
+
+	if (changed & BSS_CHANGED_ERP_SLOT) {
+		RT_TRACE(rtlpriv, COMP_MAC80211, DBG_TRACE,
+			 "BSS_CHANGED_ERP_SLOT\n");
+
+		if (bss_conf->use_short_slot)
+			mac->slot_time = RTL_SLOT_TIME_9;
+		else
+			mac->slot_time = RTL_SLOT_TIME_20;
+
+		rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_SLOT_TIME,
+					      (u8 *)(&mac->slot_time));
+	}
+
+	if (changed & BSS_CHANGED_HT) {
+		struct ieee80211_sta *sta = NULL;
+
+		RT_TRACE(rtlpriv, COMP_MAC80211, DBG_TRACE,
+			 "BSS_CHANGED_HT\n");
+
+		rcu_read_lock();
+		sta = ieee80211_find_sta(vif, (u8 *)bss_conf->bssid);
+		if (sta) {
+			if (sta->ht_cap.ampdu_density >
+			    mac->current_ampdu_density)
+				mac->current_ampdu_density =
+				    sta->ht_cap.ampdu_density;
+			if (sta->ht_cap.ampdu_factor <
+			    mac->current_ampdu_factor)
+				mac->current_ampdu_factor =
+				    sta->ht_cap.ampdu_factor;
+
+			if (sta->ht_cap.ht_supported) {
+				if (sta->ht_cap.cap &
+				    IEEE80211_HT_CAP_SUP_WIDTH_20_40)
+					rtlphy->max_ht_chan_bw =
+							HT_CHANNEL_WIDTH_20_40;
+				else
+					rtlphy->max_ht_chan_bw =
+							HT_CHANNEL_WIDTH_20;
+			}
+		}
+		rcu_read_unlock();
+
+		rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_SHORTGI_DENSITY,
+					      (u8 *)(&mac->max_mss_density));
+		rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_AMPDU_FACTOR,
+					      &mac->current_ampdu_factor);
+		rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_AMPDU_MIN_SPACE,
+					      &mac->current_ampdu_density);
+	}
+
+	if (changed & BSS_CHANGED_BANDWIDTH) {
+		struct ieee80211_sta *sta = NULL;
+
+		RT_TRACE(rtlpriv, COMP_MAC80211, DBG_TRACE,
+			 "BSS_CHANGED_BANDWIDTH\n");
+
+		rcu_read_lock();
+		sta = ieee80211_find_sta(vif, (u8 *)bss_conf->bssid);
+
+		if (sta) {
+			if (sta->ht_cap.ht_supported) {
+				if (sta->ht_cap.cap &
+				    IEEE80211_HT_CAP_SUP_WIDTH_20_40)
+					rtlphy->max_ht_chan_bw =
+							HT_CHANNEL_WIDTH_20_40;
+				else
+					rtlphy->max_ht_chan_bw =
+							HT_CHANNEL_WIDTH_20;
+			}
+
+			if (sta->vht_cap.vht_supported)
+				rtlphy->max_vht_chan_bw = HT_CHANNEL_WIDTH_80;
+		}
+		rcu_read_unlock();
+	}
+
+	if (changed & BSS_CHANGED_BSSID) {
+		u32 basic_rates;
+		struct ieee80211_sta *sta = NULL;
+
+		rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_BSSID,
+					      (u8 *)bss_conf->bssid);
+
+		RT_TRACE(rtlpriv, COMP_MAC80211, DBG_DMESG,
+			 "bssid: %pM\n", bss_conf->bssid);
+
+		mac->vendor = PEER_UNKNOWN;
+		memcpy(mac->bssid, bss_conf->bssid, ETH_ALEN);
+
+		rcu_read_lock();
+		sta = ieee80211_find_sta(vif, (u8 *)bss_conf->bssid);
+		if (!sta) {
+			rcu_read_unlock();
+			goto out;
+		}
+
+		if (rtlhal->current_bandtype == BAND_ON_5G) {
+			mac->mode = WIRELESS_MODE_A;
+		} else {
+			if (sta->supp_rates[0] <= 0xf)
+				mac->mode = WIRELESS_MODE_B;
+			else
+				mac->mode = WIRELESS_MODE_G;
+		}
+
+		if (sta->ht_cap.ht_supported) {
+			if (rtlhal->current_bandtype == BAND_ON_2_4G)
+				mac->mode = WIRELESS_MODE_N_24G;
+			else
+				mac->mode = WIRELESS_MODE_N_5G;
+		}
+
+		if (sta->vht_cap.vht_supported) {
+			if (rtlhal->current_bandtype == BAND_ON_5G)
+				mac->mode = WIRELESS_MODE_AC_5G;
+			else
+				mac->mode = WIRELESS_MODE_AC_24G;
+		}
+
+		/* just station need it, because ibss & ap mode will
+		 * set in sta_add, and will be NULL here
+		 */
+		if (vif->type == NL80211_IFTYPE_STATION) {
+			struct rtl_sta_info *sta_entry;
+
+			sta_entry = (struct rtl_sta_info *)sta->drv_priv;
+			sta_entry->wireless_mode = mac->mode;
+		}
+
+		if (sta->ht_cap.ht_supported) {
+			mac->ht_enable = true;
+
+			/* for cisco 1252 bw20 it's wrong
+			 * if (ht_cap & IEEE80211_HT_CAP_SUP_WIDTH_20_40) {
+			 *	mac->bw_40 = true;
+			 * }
+			 */
+		}
+
+		if (sta->vht_cap.vht_supported)
+			mac->vht_enable = true;
+
+		if (changed & BSS_CHANGED_BASIC_RATES) {
+			/* for 5G must << RATE_6M_INDEX = 4,
+			 * because 5G have no cck rate
+			 */
+			if (rtlhal->current_bandtype == BAND_ON_5G)
+				basic_rates = sta->supp_rates[1] << 4;
+			else
+				basic_rates = sta->supp_rates[0];
+
+			mac->basic_rates = basic_rates;
+			rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_BASIC_RATE,
+					(u8 *)(&basic_rates));
+		}
+		rcu_read_unlock();
+	}
+out:
+	mutex_unlock(&rtlpriv->locks.conf_mutex);
+}
+
+static u64 rtl_op_get_tsf(struct ieee80211_hw *hw, struct ieee80211_vif *vif)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u64 tsf;
+
+	rtlpriv->cfg->ops->get_hw_reg(hw, HW_VAR_CORRECT_TSF, (u8 *)(&tsf));
+	return tsf;
+}
+
+static void rtl_op_set_tsf(struct ieee80211_hw *hw,
+			   struct ieee80211_vif *vif, u64 tsf)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	u8 bibss = (mac->opmode == NL80211_IFTYPE_ADHOC) ? 1 : 0;
+
+	mac->tsf = tsf;
+	rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_CORRECT_TSF, (u8 *)(&bibss));
+}
+
+static void rtl_op_reset_tsf(struct ieee80211_hw *hw, struct ieee80211_vif *vif)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 tmp = 0;
+
+	rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_DUAL_TSF_RST, (u8 *)(&tmp));
+}
+
+static void rtl_op_sta_notify(struct ieee80211_hw *hw,
+			      struct ieee80211_vif *vif,
+			      enum sta_notify_cmd cmd,
+			      struct ieee80211_sta *sta)
+{
+	switch (cmd) {
+	case STA_NOTIFY_SLEEP:
+		break;
+	case STA_NOTIFY_AWAKE:
+		break;
+	default:
+		break;
+	}
+}
+
+static int rtl_op_ampdu_action(struct ieee80211_hw *hw,
+			       struct ieee80211_vif *vif,
+			       struct ieee80211_ampdu_params *params)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct ieee80211_sta *sta = params->sta;
+	enum ieee80211_ampdu_mlme_action action = params->action;
+	u16 tid = params->tid;
+	u16 *ssn = &params->ssn;
+
+	switch (action) {
+	case IEEE80211_AMPDU_TX_START:
+		RT_TRACE(rtlpriv, COMP_MAC80211, DBG_TRACE,
+			 "IEEE80211_AMPDU_TX_START: TID:%d\n", tid);
+		return rtl_tx_agg_start(hw, vif, sta, tid, ssn);
+	case IEEE80211_AMPDU_TX_STOP_CONT:
+	case IEEE80211_AMPDU_TX_STOP_FLUSH:
+	case IEEE80211_AMPDU_TX_STOP_FLUSH_CONT:
+		RT_TRACE(rtlpriv, COMP_MAC80211, DBG_TRACE,
+			 "IEEE80211_AMPDU_TX_STOP: TID:%d\n", tid);
+		return rtl_tx_agg_stop(hw, vif, sta, tid);
+	case IEEE80211_AMPDU_TX_OPERATIONAL:
+		RT_TRACE(rtlpriv, COMP_MAC80211, DBG_TRACE,
+			 "IEEE80211_AMPDU_TX_OPERATIONAL:TID:%d\n", tid);
+		rtl_tx_agg_oper(hw, sta, tid);
+		break;
+	case IEEE80211_AMPDU_RX_START:
+		RT_TRACE(rtlpriv, COMP_MAC80211, DBG_TRACE,
+			 "IEEE80211_AMPDU_RX_START:TID:%d\n", tid);
+		return rtl_rx_agg_start(hw, sta, tid);
+	case IEEE80211_AMPDU_RX_STOP:
+		RT_TRACE(rtlpriv, COMP_MAC80211, DBG_TRACE,
+			 "IEEE80211_AMPDU_RX_STOP:TID:%d\n", tid);
+		return rtl_rx_agg_stop(hw, sta, tid);
+	default:
+		pr_err("IEEE80211_AMPDU_ERR!!!!:\n");
+		return -EOPNOTSUPP;
+	}
+	return 0;
+}
+
+static void rtl_op_sw_scan_start(struct ieee80211_hw *hw,
+				 struct ieee80211_vif *vif,
+				 const u8 *mac_addr)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+
+	RT_TRACE(rtlpriv, COMP_MAC80211, DBG_LOUD, "\n");
+	mac->act_scanning = true;
+	if (rtlpriv->link_info.higher_busytraffic) {
+		mac->skip_scan = true;
+		return;
+	}
+
+	if (rtlpriv->phydm.ops)
+		rtlpriv->phydm.ops->phydm_pause_dig(rtlpriv, 1);
+
+	if (rtlpriv->cfg->ops->get_btc_status())
+		rtlpriv->btcoexist.btc_ops->btc_scan_notify(rtlpriv, 1);
+	else if (rtlpriv->btcoexist.btc_ops)
+		rtlpriv->btcoexist.btc_ops->btc_scan_notify_wifi_only(rtlpriv,
+								      1);
+
+	if (rtlpriv->dm.supp_phymode_switch) {
+		if (rtlpriv->cfg->ops->chk_switch_dmdp)
+			rtlpriv->cfg->ops->chk_switch_dmdp(hw);
+	}
+
+	if (mac->link_state == MAC80211_LINKED) {
+		rtl_lps_leave(hw);
+		mac->link_state = MAC80211_LINKED_SCANNING;
+	} else {
+		rtl_ips_nic_on(hw);
+	}
+
+	/* Dul mac */
+	rtlpriv->rtlhal.load_imrandiqk_setting_for2g = false;
+
+	rtlpriv->cfg->ops->led_control(hw, LED_CTL_SITE_SURVEY);
+	rtlpriv->cfg->ops->scan_operation_backup(hw, SCAN_OPT_BACKUP_BAND0);
+}
+
+static void rtl_op_sw_scan_complete(struct ieee80211_hw *hw,
+				    struct ieee80211_vif *vif)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+
+	RT_TRACE(rtlpriv, COMP_MAC80211, DBG_LOUD, "\n");
+	mac->act_scanning = false;
+	mac->skip_scan = false;
+
+	rtlpriv->btcoexist.btc_info.ap_num = rtlpriv->scan_list.num;
+
+	if (rtlpriv->link_info.higher_busytraffic)
+		return;
+
+	/* p2p will use 1/6/11 to scan */
+	if (mac->n_channels == 3)
+		mac->p2p_in_use = true;
+	else
+		mac->p2p_in_use = false;
+	mac->n_channels = 0;
+	/* Dul mac */
+	rtlpriv->rtlhal.load_imrandiqk_setting_for2g = false;
+
+	if (mac->link_state == MAC80211_LINKED_SCANNING) {
+		mac->link_state = MAC80211_LINKED;
+		if (mac->opmode == NL80211_IFTYPE_STATION) {
+			/* fix fwlps issue */
+			rtlpriv->cfg->ops->set_network_type(hw, mac->opmode);
+		}
+	}
+
+	rtlpriv->cfg->ops->scan_operation_backup(hw, SCAN_OPT_RESTORE);
+	if (rtlpriv->cfg->ops->get_btc_status())
+		rtlpriv->btcoexist.btc_ops->btc_scan_notify(rtlpriv, 0);
+	else if (rtlpriv->btcoexist.btc_ops)
+		rtlpriv->btcoexist.btc_ops->btc_scan_notify_wifi_only(rtlpriv,
+								      0);
+
+	if (rtlpriv->phydm.ops)
+		rtlpriv->phydm.ops->phydm_pause_dig(rtlpriv, 0);
+}
+
+static int rtl_op_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
+			  struct ieee80211_vif *vif, struct ieee80211_sta *sta,
+			  struct ieee80211_key_conf *key)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 key_type = NO_ENCRYPTION;
+	u8 key_idx;
+	bool group_key = false;
+	bool wep_only = false;
+	int err = 0;
+	u8 mac_addr[ETH_ALEN];
+	u8 bcast_addr[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
+
+	rtlpriv->btcoexist.btc_info.in_4way = false;
+
+	if (rtlpriv->cfg->mod_params->sw_crypto || rtlpriv->sec.use_sw_sec) {
+		RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING,
+			 "not open hw encryption\n");
+		return -ENOSPC;	/*User disabled HW-crypto */
+	}
+	/* To support IBSS, use sw-crypto for GTK */
+	if (((vif->type == NL80211_IFTYPE_ADHOC) ||
+	     (vif->type == NL80211_IFTYPE_MESH_POINT)) &&
+	    !(key->flags & IEEE80211_KEY_FLAG_PAIRWISE))
+		return -ENOSPC;
+	RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG,
+		 "%s hardware based encryption for keyidx: %d, mac: %pM\n",
+		  cmd == SET_KEY ? "Using" : "Disabling", key->keyidx,
+		  sta ? sta->addr : bcast_addr);
+	rtlpriv->sec.being_setkey = true;
+	rtl_ips_nic_on(hw);
+	mutex_lock(&rtlpriv->locks.conf_mutex);
+	/* <1> get encryption alg */
+
+	switch (key->cipher) {
+	case WLAN_CIPHER_SUITE_WEP40:
+		key_type = WEP40_ENCRYPTION;
+		RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG, "alg:WEP40\n");
+		break;
+	case WLAN_CIPHER_SUITE_WEP104:
+		RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG, "alg:WEP104\n");
+		key_type = WEP104_ENCRYPTION;
+		break;
+	case WLAN_CIPHER_SUITE_TKIP:
+		key_type = TKIP_ENCRYPTION;
+		RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG, "alg:TKIP\n");
+		break;
+	case WLAN_CIPHER_SUITE_CCMP:
+		key_type = AESCCMP_ENCRYPTION;
+		RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG, "alg:CCMP\n");
+		break;
+	case WLAN_CIPHER_SUITE_AES_CMAC:
+		/* HW don't support CMAC encryption,
+		 * use software CMAC encryption
+		 */
+		key_type = AESCMAC_ENCRYPTION;
+		RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG, "alg:CMAC\n");
+		RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG,
+			 "HW don't support CMAC encryption, use software CMAC encryption\n");
+		err = -EOPNOTSUPP;
+		goto out_unlock;
+	default:
+		pr_err("alg_err:%x!!!!:\n", key->cipher);
+		goto out_unlock;
+	}
+	if (key_type == WEP40_ENCRYPTION ||
+	    key_type == WEP104_ENCRYPTION ||
+	    vif->type == NL80211_IFTYPE_ADHOC)
+		rtlpriv->sec.use_defaultkey = true;
+
+	/* <2> get key_idx */
+	key_idx = (u8)(key->keyidx);
+	if (key_idx > 3)
+		goto out_unlock;
+	/* <3> if pairwise key enable_hw_sec */
+	group_key = !(key->flags & IEEE80211_KEY_FLAG_PAIRWISE);
+
+	/* wep always be group key, but there are two conditions:
+	 * 1) wep only: is just for wep enc, in this condition
+	 * rtlpriv->sec.pairwise_enc_algorithm == NO_ENCRYPTION
+	 * will be true & enable_hw_sec will be set when wep
+	 * ke setting.
+	 * 2) wep(group) + AES(pairwise): some AP like cisco
+	 * may use it, in this condition enable_hw_sec will not
+	 * be set when wep key setting.
+	 * we must reset sec_info after lingked before set key,
+	 * or some flag will be wrong
+	 */
+	if (vif->type == NL80211_IFTYPE_AP ||
+	    vif->type == NL80211_IFTYPE_MESH_POINT) {
+		if (!group_key || key_type == WEP40_ENCRYPTION ||
+		    key_type == WEP104_ENCRYPTION) {
+			if (group_key)
+				wep_only = true;
+			rtlpriv->cfg->ops->enable_hw_sec(hw);
+		}
+	} else {
+		if ((!group_key) || (vif->type == NL80211_IFTYPE_ADHOC) ||
+		    rtlpriv->sec.pairwise_enc_algorithm == NO_ENCRYPTION) {
+			if (rtlpriv->sec.pairwise_enc_algorithm ==
+			    NO_ENCRYPTION &&
+			   (key_type == WEP40_ENCRYPTION ||
+			    key_type == WEP104_ENCRYPTION))
+				wep_only = true;
+			rtlpriv->sec.pairwise_enc_algorithm = key_type;
+			RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG,
+				 "set enable_hw_sec, key_type:%x(OPEN:0 WEP40:1 TKIP:2 AES:4 WEP104:5)\n",
+				 key_type);
+			rtlpriv->cfg->ops->enable_hw_sec(hw);
+		}
+	}
+	/* <4> set key based on cmd */
+	switch (cmd) {
+	case SET_KEY:
+		if (wep_only) {
+			RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG,
+				 "set WEP(group/pairwise) key\n");
+			/* Pairwise key with an assigned MAC address. */
+			rtlpriv->sec.pairwise_enc_algorithm = key_type;
+			rtlpriv->sec.group_enc_algorithm = key_type;
+			/*set local buf about wep key. */
+			memcpy(rtlpriv->sec.key_buf[key_idx],
+			       key->key, key->keylen);
+			rtlpriv->sec.key_len[key_idx] = key->keylen;
+			eth_zero_addr(mac_addr);
+		} else if (group_key) {	/* group key */
+			RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG,
+				 "set group key\n");
+			/* group key */
+			rtlpriv->sec.group_enc_algorithm = key_type;
+			/*set local buf about group key. */
+			memcpy(rtlpriv->sec.key_buf[key_idx],
+			       key->key, key->keylen);
+			rtlpriv->sec.key_len[key_idx] = key->keylen;
+			memcpy(mac_addr, bcast_addr, ETH_ALEN);
+		} else {	/* pairwise key */
+			RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG,
+				 "set pairwise key\n");
+			if (!sta) {
+				WARN_ONCE(true,
+					  "rtlwifi: pairwise key without mac_addr\n");
+
+				err = -EOPNOTSUPP;
+				goto out_unlock;
+			}
+			/* Pairwise key with an assigned MAC address. */
+			rtlpriv->sec.pairwise_enc_algorithm = key_type;
+			/*set local buf about pairwise key. */
+			memcpy(rtlpriv->sec.key_buf[PAIRWISE_KEYIDX],
+			       key->key, key->keylen);
+			rtlpriv->sec.key_len[PAIRWISE_KEYIDX] = key->keylen;
+			rtlpriv->sec.pairwise_key =
+			    rtlpriv->sec.key_buf[PAIRWISE_KEYIDX];
+			memcpy(mac_addr, sta->addr, ETH_ALEN);
+		}
+		rtlpriv->cfg->ops->set_key(hw, key_idx, mac_addr,
+					   group_key, key_type, wep_only,
+					   false);
+		/* <5> tell mac80211 do something: */
+		/*must use sw generate IV, or can not work !!!!. */
+		key->flags |= IEEE80211_KEY_FLAG_GENERATE_IV;
+		key->hw_key_idx = key_idx;
+		if (key_type == TKIP_ENCRYPTION)
+			key->flags |= IEEE80211_KEY_FLAG_GENERATE_MMIC;
+		/*use software CCMP encryption for management frames (MFP) */
+		if (key_type == AESCCMP_ENCRYPTION)
+			key->flags |= IEEE80211_KEY_FLAG_SW_MGMT_TX;
+		break;
+	case DISABLE_KEY:
+		RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG,
+			 "disable key delete one entry\n");
+		/*set local buf about wep key. */
+		if (vif->type == NL80211_IFTYPE_AP ||
+		    vif->type == NL80211_IFTYPE_MESH_POINT) {
+			if (sta)
+				rtl_cam_del_entry(hw, sta->addr);
+		}
+		memset(rtlpriv->sec.key_buf[key_idx], 0, key->keylen);
+		rtlpriv->sec.key_len[key_idx] = 0;
+		eth_zero_addr(mac_addr);
+		/*
+		 *mac80211 will delete entries one by one,
+		 *so don't use rtl_cam_reset_all_entry
+		 *or clear all entries here.
+		 */
+		rtl_wait_tx_report_acked(hw, 500); /* wait 500ms for TX ack */
+
+		rtl_cam_delete_one_entry(hw, mac_addr, key_idx);
+		break;
+	default:
+		pr_err("cmd_err:%x!!!!:\n", cmd);
+	}
+out_unlock:
+	mutex_unlock(&rtlpriv->locks.conf_mutex);
+	rtlpriv->sec.being_setkey = false;
+	return err;
+}
+
+static void rtl_op_rfkill_poll(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	bool radio_state;
+	bool blocked;
+	u8 valid = 0;
+
+	if (!test_bit(RTL_STATUS_INTERFACE_START, &rtlpriv->status))
+		return;
+
+	mutex_lock(&rtlpriv->locks.conf_mutex);
+
+	/*if Radio On return true here */
+	radio_state = rtlpriv->cfg->ops->radio_onoff_checking(hw, &valid);
+
+	if (valid) {
+		if (unlikely(radio_state != rtlpriv->rfkill.rfkill_state)) {
+			rtlpriv->rfkill.rfkill_state = radio_state;
+
+			RT_TRACE(rtlpriv, COMP_RF, DBG_DMESG,
+				 "wireless radio switch turned %s\n",
+				  radio_state ? "on" : "off");
+
+			blocked = (rtlpriv->rfkill.rfkill_state == 1) ? 0 : 1;
+			wiphy_rfkill_set_hw_state(hw->wiphy, blocked);
+		}
+	}
+
+	mutex_unlock(&rtlpriv->locks.conf_mutex);
+}
+
+/* this function is called by mac80211 to flush tx buffer
+ * before switch channel or power save, or tx buffer packet
+ * maybe send after offchannel or rf sleep, this may cause
+ * dis-association by AP
+ */
+static void rtl_op_flush(struct ieee80211_hw *hw,
+			 struct ieee80211_vif *vif,
+			 u32 queues,
+			 bool drop)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	if (rtlpriv->intf_ops->flush)
+		rtlpriv->intf_ops->flush(hw, queues, drop);
+}
+
+/*	Description:
+ *		This routine deals with the Power Configuration CMD
+ *		 parsing for RTL8723/RTL8188E Series IC.
+ *	Assumption:
+ *		We should follow specific format that was released from HW SD.
+ */
+bool rtl_hal_pwrseqcmdparsing(struct rtl_priv *rtlpriv, u8 cut_version,
+			      u8 faversion, u8 interface_type,
+			      struct wlan_pwr_cfg pwrcfgcmd[])
+{
+	struct wlan_pwr_cfg cfg_cmd = {0};
+	bool polling_bit = false;
+	u32 ary_idx = 0;
+	u8 value = 0;
+	u32 offset = 0;
+	u32 polling_count = 0;
+	u32 max_polling_cnt = 5000;
+
+	do {
+		cfg_cmd = pwrcfgcmd[ary_idx];
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE,
+			 "%s(): offset(%#x),cut_msk(%#x), famsk(%#x), interface_msk(%#x), base(%#x), cmd(%#x), msk(%#x), value(%#x)\n",
+			 __func__, GET_PWR_CFG_OFFSET(cfg_cmd),
+					    GET_PWR_CFG_CUT_MASK(cfg_cmd),
+			 GET_PWR_CFG_FAB_MASK(cfg_cmd),
+					      GET_PWR_CFG_INTF_MASK(cfg_cmd),
+			 GET_PWR_CFG_BASE(cfg_cmd), GET_PWR_CFG_CMD(cfg_cmd),
+			 GET_PWR_CFG_MASK(cfg_cmd), GET_PWR_CFG_VALUE(cfg_cmd));
+
+		if ((GET_PWR_CFG_FAB_MASK(cfg_cmd) & faversion) &&
+		    (GET_PWR_CFG_CUT_MASK(cfg_cmd) & cut_version) &&
+		    (GET_PWR_CFG_INTF_MASK(cfg_cmd) & interface_type)) {
+			switch (GET_PWR_CFG_CMD(cfg_cmd)) {
+			case PWR_CMD_READ:
+				RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE,
+					 "%s(): PWR_CMD_READ\n", __func__);
+				break;
+			case PWR_CMD_WRITE:
+				RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE,
+					"%s(): PWR_CMD_WRITE\n", __func__);
+				offset = GET_PWR_CFG_OFFSET(cfg_cmd);
+
+				/*Read the value from system register*/
+				value = rtl_read_byte(rtlpriv, offset);
+				value &= (~(GET_PWR_CFG_MASK(cfg_cmd)));
+				value |= (GET_PWR_CFG_VALUE(cfg_cmd) &
+					  GET_PWR_CFG_MASK(cfg_cmd));
+
+				/*Write the value back to system register*/
+				rtl_write_byte(rtlpriv, offset, value);
+				break;
+			case PWR_CMD_POLLING:
+				RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE,
+					 "%s(): PWR_CMD_POLLING\n", __func__);
+				polling_bit = false;
+				offset = GET_PWR_CFG_OFFSET(cfg_cmd);
+
+				do {
+					value = rtl_read_byte(rtlpriv, offset);
+
+					value &= GET_PWR_CFG_MASK(cfg_cmd);
+					if (value ==
+					    (GET_PWR_CFG_VALUE(cfg_cmd) &
+					     GET_PWR_CFG_MASK(cfg_cmd)))
+						polling_bit = true;
+					else
+						udelay(10);
+
+					if (polling_count++ > max_polling_cnt)
+						return false;
+				} while (!polling_bit);
+				break;
+			case PWR_CMD_DELAY:
+				RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE,
+					 "%s(): PWR_CMD_DELAY\n", __func__);
+				if (GET_PWR_CFG_VALUE(cfg_cmd) ==
+				    PWRSEQ_DELAY_US)
+					udelay(GET_PWR_CFG_OFFSET(cfg_cmd));
+				else
+					mdelay(GET_PWR_CFG_OFFSET(cfg_cmd));
+				break;
+			case PWR_CMD_END:
+				RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE,
+					 "%s(): PWR_CMD_END\n", __func__);
+				return true;
+			default:
+				WARN_ONCE(true,
+					  "rtlwifi: %s(): Unknown CMD!!\n", __func__);
+				break;
+			}
+		}
+		ary_idx++;
+	} while (1);
+
+	return true;
+}
+
+bool rtl_cmd_send_packet(struct ieee80211_hw *hw, struct sk_buff *skb)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	struct rtl8192_tx_ring *ring;
+	struct rtl_tx_desc *pdesc;
+	unsigned long flags;
+	struct sk_buff *pskb = NULL;
+
+	ring = &rtlpci->tx_ring[BEACON_QUEUE];
+
+	spin_lock_irqsave(&rtlpriv->locks.irq_th_lock, flags);
+	pskb = __skb_dequeue(&ring->queue);
+	if (pskb)
+		dev_kfree_skb_irq(pskb);
+
+	/*this is wrong, fill_tx_cmddesc needs update*/
+	pdesc = &ring->desc[0];
+
+	rtlpriv->cfg->ops->fill_tx_cmddesc(hw, (u8 *)pdesc, 1, 1, skb);
+
+	__skb_queue_tail(&ring->queue, skb);
+
+	spin_unlock_irqrestore(&rtlpriv->locks.irq_th_lock, flags);
+
+	rtlpriv->cfg->ops->tx_polling(hw, BEACON_QUEUE);
+
+	return true;
+}
+
+const struct ieee80211_ops rtl_ops = {
+	.start = rtl_op_start,
+	.stop = rtl_op_stop,
+	.tx = rtl_op_tx,
+	.add_interface = rtl_op_add_interface,
+	.remove_interface = rtl_op_remove_interface,
+	.change_interface = rtl_op_change_interface,
+#ifdef CONFIG_PM
+	.suspend = rtl_op_suspend,
+	.resume = rtl_op_resume,
+#endif
+	.config = rtl_op_config,
+	.configure_filter = rtl_op_configure_filter,
+	.set_key = rtl_op_set_key,
+	.sta_statistics = rtl_op_sta_statistics,
+	.set_frag_threshold = rtl_op_set_frag_threshold,
+	.conf_tx = rtl_op_conf_tx,
+	.bss_info_changed = rtl_op_bss_info_changed,
+	.get_tsf = rtl_op_get_tsf,
+	.set_tsf = rtl_op_set_tsf,
+	.reset_tsf = rtl_op_reset_tsf,
+	.sta_notify = rtl_op_sta_notify,
+	.ampdu_action = rtl_op_ampdu_action,
+	.sw_scan_start = rtl_op_sw_scan_start,
+	.sw_scan_complete = rtl_op_sw_scan_complete,
+	.rfkill_poll = rtl_op_rfkill_poll,
+	.sta_add = rtl_op_sta_add,
+	.sta_remove = rtl_op_sta_remove,
+	.flush = rtl_op_flush,
+};
+
+bool rtl_btc_status_false(void)
+{
+	return false;
+}
+
+void rtl_dm_diginit(struct ieee80211_hw *hw, u32 cur_igvalue)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct dig_t *dm_digtable = &rtlpriv->dm_digtable;
+
+	dm_digtable->dig_enable_flag = true;
+	dm_digtable->dig_ext_port_stage = DIG_EXT_PORT_STAGE_MAX;
+	dm_digtable->cur_igvalue = cur_igvalue;
+	dm_digtable->pre_igvalue = 0;
+	dm_digtable->cur_sta_cstate = DIG_STA_DISCONNECT;
+	dm_digtable->presta_cstate = DIG_STA_DISCONNECT;
+	dm_digtable->curmultista_cstate = DIG_MULTISTA_DISCONNECT;
+	dm_digtable->rssi_lowthresh = DM_DIG_THRESH_LOW;
+	dm_digtable->rssi_highthresh = DM_DIG_THRESH_HIGH;
+	dm_digtable->fa_lowthresh = DM_FALSEALARM_THRESH_LOW;
+	dm_digtable->fa_highthresh = DM_FALSEALARM_THRESH_HIGH;
+	dm_digtable->rx_gain_max = DM_DIG_MAX;
+	dm_digtable->rx_gain_min = DM_DIG_MIN;
+	dm_digtable->back_val = DM_DIG_BACKOFF_DEFAULT;
+	dm_digtable->back_range_max = DM_DIG_BACKOFF_MAX;
+	dm_digtable->back_range_min = DM_DIG_BACKOFF_MIN;
+	dm_digtable->pre_cck_cca_thres = 0xff;
+	dm_digtable->cur_cck_cca_thres = 0x83;
+	dm_digtable->forbidden_igi = DM_DIG_MIN;
+	dm_digtable->large_fa_hit = 0;
+	dm_digtable->recover_cnt = 0;
+	dm_digtable->dig_min_0 = 0x25;
+	dm_digtable->dig_min_1 = 0x25;
+	dm_digtable->media_connect_0 = false;
+	dm_digtable->media_connect_1 = false;
+	rtlpriv->dm.dm_initialgain_enable = true;
+	dm_digtable->bt30_cur_igi = 0x32;
+	dm_digtable->pre_cck_pd_state = CCK_PD_STAGE_MAX;
+	dm_digtable->cur_cck_pd_state = CCK_PD_STAGE_LOWRSSI;
+}
diff --git a/drivers/staging/rtlwifi/core.h b/drivers/staging/rtlwifi/core.h
new file mode 100644
index 000000000000..782ac2fc4b28
--- /dev/null
+++ b/drivers/staging/rtlwifi/core.h
@@ -0,0 +1,86 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2012  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __RTL_CORE_H__
+#define __RTL_CORE_H__
+
+#define RTL_SUPPORTED_FILTERS		\
+	(FIF_ALLMULTI | FIF_CONTROL | \
+	FIF_OTHER_BSS | \
+	FIF_FCSFAIL | \
+	FIF_BCN_PRBRESP_PROMISC)
+
+#define DM_DIG_THRESH_HIGH		40
+#define DM_DIG_THRESH_LOW		35
+#define DM_FALSEALARM_THRESH_LOW	400
+#define DM_FALSEALARM_THRESH_HIGH	1000
+
+#define DM_DIG_MAX			0x3e
+#define DM_DIG_MIN			0x1e
+#define DM_DIG_MAX_AP			0x32
+#define DM_DIG_BACKOFF_MAX		12
+#define DM_DIG_BACKOFF_MIN		-4
+#define DM_DIG_BACKOFF_DEFAULT		10
+
+enum cck_packet_detection_threshold {
+	CCK_PD_STAGE_LOWRSSI = 0,
+	CCK_PD_STAGE_HIGHRSSI = 1,
+	CCK_FA_STAGE_LOW = 2,
+	CCK_FA_STAGE_HIGH = 3,
+	CCK_PD_STAGE_MAX = 4,
+};
+
+enum dm_dig_ext_port_alg_e {
+	DIG_EXT_PORT_STAGE_0 = 0,
+	DIG_EXT_PORT_STAGE_1 = 1,
+	DIG_EXT_PORT_STAGE_2 = 2,
+	DIG_EXT_PORT_STAGE_3 = 3,
+	DIG_EXT_PORT_STAGE_MAX = 4,
+};
+
+enum dm_dig_connect_e {
+	DIG_STA_DISCONNECT,
+	DIG_STA_CONNECT,
+	DIG_STA_BEFORE_CONNECT,
+	DIG_MULTISTA_DISCONNECT,
+	DIG_MULTISTA_CONNECT,
+	DIG_AP_DISCONNECT,
+	DIG_AP_CONNECT,
+	DIG_AP_ADD_STATION,
+	DIG_CONNECT_MAX
+};
+
+extern const struct ieee80211_ops rtl_ops;
+void rtl_fw_cb(const struct firmware *firmware, void *context);
+void rtl_wowlan_fw_cb(const struct firmware *firmware, void *context);
+void rtl_addr_delay(u32 addr);
+void rtl_rfreg_delay(struct ieee80211_hw *hw, enum radio_path rfpath, u32 addr,
+		     u32 mask, u32 data);
+void rtl_bb_delay(struct ieee80211_hw *hw, u32 addr, u32 data);
+bool rtl_cmd_send_packet(struct ieee80211_hw *hw, struct sk_buff *skb);
+bool rtl_btc_status_false(void);
+void rtl_dm_diginit(struct ieee80211_hw *hw, u32 cur_igval);
+
+#endif
diff --git a/drivers/staging/rtlwifi/debug.c b/drivers/staging/rtlwifi/debug.c
new file mode 100644
index 000000000000..7446d71c41d1
--- /dev/null
+++ b/drivers/staging/rtlwifi/debug.c
@@ -0,0 +1,636 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2012  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *****************************************************************************/
+
+#include "wifi.h"
+#include "cam.h"
+
+#include <linux/moduleparam.h>
+#include <linux/vmalloc.h>
+
+#ifdef CONFIG_RTLWIFI_DEBUG_ST
+void _rtl_dbg_trace(struct rtl_priv *rtlpriv, u64 comp, int level,
+		    const char *fmt, ...)
+{
+	if (unlikely((comp & rtlpriv->cfg->mod_params->debug_mask) &&
+		     (level <= rtlpriv->cfg->mod_params->debug_level))) {
+		struct va_format vaf;
+		va_list args;
+
+		va_start(args, fmt);
+
+		vaf.fmt = fmt;
+		vaf.va = &args;
+
+		pr_info(":<%lx> %pV", in_interrupt(), &vaf);
+
+		va_end(args);
+	}
+}
+
+void _rtl_dbg_print(struct rtl_priv *rtlpriv, u64 comp, int level,
+		    const char *fmt, ...)
+{
+	if (unlikely((comp & rtlpriv->cfg->mod_params->debug_mask) &&
+		     (level <= rtlpriv->cfg->mod_params->debug_level))) {
+		struct va_format vaf;
+		va_list args;
+
+		va_start(args, fmt);
+
+		vaf.fmt = fmt;
+		vaf.va = &args;
+
+		pr_info("%pV", &vaf);
+
+		va_end(args);
+	}
+}
+
+void _rtl_dbg_print_data(struct rtl_priv *rtlpriv, u64 comp, int level,
+			 const char *titlestring,
+			 const void *hexdata, int hexdatalen)
+{
+	if (unlikely(((comp) & rtlpriv->cfg->mod_params->debug_mask) &&
+		     ((level) <= rtlpriv->cfg->mod_params->debug_level))) {
+		pr_info("In process \"%s\" (pid %i): %s\n",
+			current->comm, current->pid, titlestring);
+		print_hex_dump_bytes("", DUMP_PREFIX_NONE,
+				     hexdata, hexdatalen);
+	}
+}
+
+struct rtl_debugfs_priv {
+	struct rtl_priv *rtlpriv;
+	int (*cb_read)(struct seq_file *m, void *v);
+	ssize_t (*cb_write)(struct file *filp, const char __user *buffer,
+			    size_t count, loff_t *loff);
+	u32 cb_data;
+};
+
+static struct dentry *debugfs_topdir;
+
+static int rtl_debug_get_common(struct seq_file *m, void *v)
+{
+	struct rtl_debugfs_priv *debugfs_priv = m->private;
+
+	return debugfs_priv->cb_read(m, v);
+}
+
+static int dl_debug_open_common(struct inode *inode, struct file *file)
+{
+	return single_open(file, rtl_debug_get_common, inode->i_private);
+}
+
+static const struct file_operations file_ops_common = {
+	.open = dl_debug_open_common,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = seq_release,
+};
+
+static int rtl_debug_get_mac_page(struct seq_file *m, void *v)
+{
+	struct rtl_debugfs_priv *debugfs_priv = m->private;
+	struct rtl_priv *rtlpriv = debugfs_priv->rtlpriv;
+	u32 page = debugfs_priv->cb_data;
+	int i, n;
+	int max = 0xff;
+
+	for (n = 0; n <= max; ) {
+		seq_printf(m, "\n%8.8x  ", n + page);
+		for (i = 0; i < 4 && n <= max; i++, n += 4)
+			seq_printf(m, "%8.8x    ",
+				   rtl_read_dword(rtlpriv, (page | n)));
+	}
+	seq_puts(m, "\n");
+	return 0;
+}
+
+#define RTL_DEBUG_IMPL_MAC_SERIES(page, addr)		\
+struct rtl_debugfs_priv rtl_debug_priv_mac_ ##page = {	\
+	.cb_read = rtl_debug_get_mac_page,		\
+	.cb_data = addr,				\
+}
+
+RTL_DEBUG_IMPL_MAC_SERIES(0, 0x0000);
+RTL_DEBUG_IMPL_MAC_SERIES(1, 0x0100);
+RTL_DEBUG_IMPL_MAC_SERIES(2, 0x0200);
+RTL_DEBUG_IMPL_MAC_SERIES(3, 0x0300);
+RTL_DEBUG_IMPL_MAC_SERIES(4, 0x0400);
+RTL_DEBUG_IMPL_MAC_SERIES(5, 0x0500);
+RTL_DEBUG_IMPL_MAC_SERIES(6, 0x0600);
+RTL_DEBUG_IMPL_MAC_SERIES(7, 0x0700);
+RTL_DEBUG_IMPL_MAC_SERIES(10, 0x1000);
+RTL_DEBUG_IMPL_MAC_SERIES(11, 0x1100);
+RTL_DEBUG_IMPL_MAC_SERIES(12, 0x1200);
+RTL_DEBUG_IMPL_MAC_SERIES(13, 0x1300);
+RTL_DEBUG_IMPL_MAC_SERIES(14, 0x1400);
+RTL_DEBUG_IMPL_MAC_SERIES(15, 0x1500);
+RTL_DEBUG_IMPL_MAC_SERIES(16, 0x1600);
+RTL_DEBUG_IMPL_MAC_SERIES(17, 0x1700);
+
+static int rtl_debug_get_bb_page(struct seq_file *m, void *v)
+{
+	struct rtl_debugfs_priv *debugfs_priv = m->private;
+	struct rtl_priv *rtlpriv = debugfs_priv->rtlpriv;
+	struct ieee80211_hw *hw = rtlpriv->hw;
+	u32 page = debugfs_priv->cb_data;
+	int i, n;
+	int max = 0xff;
+
+	for (n = 0; n <= max; ) {
+		seq_printf(m, "\n%8.8x  ", n + page);
+		for (i = 0; i < 4 && n <= max; i++, n += 4)
+			seq_printf(m, "%8.8x    ",
+				   rtl_get_bbreg(hw, (page | n), 0xffffffff));
+	}
+	seq_puts(m, "\n");
+	return 0;
+}
+
+#define RTL_DEBUG_IMPL_BB_SERIES(page, addr)		\
+struct rtl_debugfs_priv rtl_debug_priv_bb_ ##page = {	\
+	.cb_read = rtl_debug_get_bb_page,		\
+	.cb_data = addr,				\
+}
+
+RTL_DEBUG_IMPL_BB_SERIES(8, 0x0800);
+RTL_DEBUG_IMPL_BB_SERIES(9, 0x0900);
+RTL_DEBUG_IMPL_BB_SERIES(a, 0x0a00);
+RTL_DEBUG_IMPL_BB_SERIES(b, 0x0b00);
+RTL_DEBUG_IMPL_BB_SERIES(c, 0x0c00);
+RTL_DEBUG_IMPL_BB_SERIES(d, 0x0d00);
+RTL_DEBUG_IMPL_BB_SERIES(e, 0x0e00);
+RTL_DEBUG_IMPL_BB_SERIES(f, 0x0f00);
+RTL_DEBUG_IMPL_BB_SERIES(18, 0x1800);
+RTL_DEBUG_IMPL_BB_SERIES(19, 0x1900);
+RTL_DEBUG_IMPL_BB_SERIES(1a, 0x1a00);
+RTL_DEBUG_IMPL_BB_SERIES(1b, 0x1b00);
+RTL_DEBUG_IMPL_BB_SERIES(1c, 0x1c00);
+RTL_DEBUG_IMPL_BB_SERIES(1d, 0x1d00);
+RTL_DEBUG_IMPL_BB_SERIES(1e, 0x1e00);
+RTL_DEBUG_IMPL_BB_SERIES(1f, 0x1f00);
+
+static int rtl_debug_get_reg_rf(struct seq_file *m, void *v)
+{
+	struct rtl_debugfs_priv *debugfs_priv = m->private;
+	struct rtl_priv *rtlpriv = debugfs_priv->rtlpriv;
+	struct ieee80211_hw *hw = rtlpriv->hw;
+	enum radio_path rfpath = debugfs_priv->cb_data;
+	int i, n;
+	int max = 0x40;
+
+	if (IS_HARDWARE_TYPE_8822B(rtlpriv))
+		max = 0xff;
+
+	seq_printf(m, "\nPATH(%d)", rfpath);
+
+	for (n = 0; n <= max; ) {
+		seq_printf(m, "\n%8.8x  ", n);
+		for (i = 0; i < 4 && n <= max; n += 1, i++)
+			seq_printf(m, "%8.8x    ",
+				   rtl_get_rfreg(hw, rfpath, n, 0xffffffff));
+	}
+	seq_puts(m, "\n");
+	return 0;
+}
+
+#define RTL_DEBUG_IMPL_RF_SERIES(page, addr)		\
+struct rtl_debugfs_priv rtl_debug_priv_rf_ ##page = {	\
+	.cb_read = rtl_debug_get_reg_rf,		\
+	.cb_data = addr,				\
+}
+
+RTL_DEBUG_IMPL_RF_SERIES(a, RF90_PATH_A);
+RTL_DEBUG_IMPL_RF_SERIES(b, RF90_PATH_B);
+
+static int rtl_debug_get_cam_register(struct seq_file *m, void *v)
+{
+	struct rtl_debugfs_priv *debugfs_priv = m->private;
+	struct rtl_priv *rtlpriv = debugfs_priv->rtlpriv;
+	int start = debugfs_priv->cb_data;
+	u32 target_cmd = 0;
+	u32 target_val = 0;
+	u8 entry_i = 0;
+	u32 ulstatus;
+	int i = 100, j = 0;
+	int end = (start + 11 > TOTAL_CAM_ENTRY ? TOTAL_CAM_ENTRY : start + 11);
+
+	/* This dump the current register page */
+	seq_printf(m,
+		   "\n#################### SECURITY CAM (%d-%d) ##################\n",
+		   start, end - 1);
+
+	for (j = start; j < end; j++) {
+		seq_printf(m, "\nD:  %2x > ", j);
+		for (entry_i = 0; entry_i < CAM_CONTENT_COUNT; entry_i++) {
+			/* polling bit, and No Write enable, and address  */
+			target_cmd = entry_i + CAM_CONTENT_COUNT * j;
+			target_cmd = target_cmd | BIT(31);
+
+			/* Check polling bit is clear */
+			while ((i--) >= 0) {
+				ulstatus = rtl_read_dword(
+						rtlpriv,
+						rtlpriv->cfg->maps[RWCAM]);
+				if (ulstatus & BIT(31))
+					continue;
+				else
+					break;
+			}
+
+			rtl_write_dword(rtlpriv, rtlpriv->cfg->maps[RWCAM],
+					target_cmd);
+			target_val = rtl_read_dword(rtlpriv,
+						    rtlpriv->cfg->maps[RCAMO]);
+			seq_printf(m, "%8.8x ", target_val);
+		}
+	}
+	seq_puts(m, "\n");
+	return 0;
+}
+
+#define RTL_DEBUG_IMPL_CAM_SERIES(page, addr)		\
+struct rtl_debugfs_priv rtl_debug_priv_cam_ ##page = {	\
+	.cb_read = rtl_debug_get_cam_register,		\
+	.cb_data = addr,				\
+}
+
+RTL_DEBUG_IMPL_CAM_SERIES(1, 0);
+RTL_DEBUG_IMPL_CAM_SERIES(2, 11);
+RTL_DEBUG_IMPL_CAM_SERIES(3, 22);
+
+static int rtl_debug_get_btcoex(struct seq_file *m, void *v)
+{
+	struct rtl_debugfs_priv *debugfs_priv = m->private;
+	struct rtl_priv *rtlpriv = debugfs_priv->rtlpriv;
+
+	if (rtlpriv->cfg->ops->get_btc_status())
+		rtlpriv->btcoexist.btc_ops->btc_display_bt_coex_info(rtlpriv,
+								     m);
+
+	seq_puts(m, "\n");
+
+	return 0;
+}
+
+static struct rtl_debugfs_priv rtl_debug_priv_btcoex = {
+	.cb_read = rtl_debug_get_btcoex,
+	.cb_data = 0,
+};
+
+static ssize_t rtl_debugfs_set_write_reg(struct file *filp,
+					 const char __user *buffer,
+					 size_t count, loff_t *loff)
+{
+	struct rtl_debugfs_priv *debugfs_priv = filp->private_data;
+	struct rtl_priv *rtlpriv = debugfs_priv->rtlpriv;
+	char tmp[32 + 1];
+	int tmp_len;
+	u32 addr, val, len;
+	int num;
+
+	if (count < 3)
+		return -EFAULT;
+
+	tmp_len = (count > sizeof(tmp) - 1 ? sizeof(tmp) - 1 : count);
+
+	if (!buffer || copy_from_user(tmp, buffer, tmp_len))
+		return count;
+
+	tmp[tmp_len] = '\0';
+
+	/* write BB/MAC register */
+	num = sscanf(tmp, "%x %x %x", &addr, &val, &len);
+
+	if (num !=  3)
+		return count;
+
+	switch (len) {
+	case 1:
+		rtl_write_byte(rtlpriv, addr, (u8)val);
+		break;
+	case 2:
+		rtl_write_word(rtlpriv, addr, (u16)val);
+		break;
+	case 4:
+		rtl_write_dword(rtlpriv, addr, val);
+		break;
+	default:
+		/*printk("error write length=%d", len);*/
+		break;
+	}
+
+	return count;
+}
+
+static struct rtl_debugfs_priv rtl_debug_priv_write_reg = {
+	.cb_write = rtl_debugfs_set_write_reg,
+};
+
+static ssize_t rtl_debugfs_set_write_h2c(struct file *filp,
+					 const char __user *buffer,
+					 size_t count, loff_t *loff)
+{
+	struct rtl_debugfs_priv *debugfs_priv = filp->private_data;
+	struct rtl_priv *rtlpriv = debugfs_priv->rtlpriv;
+	struct ieee80211_hw *hw = rtlpriv->hw;
+	char tmp[32 + 1];
+	int tmp_len;
+	u8 h2c_len, h2c_data_packed[8];
+	int h2c_data[8];	/* idx 0: cmd */
+	int i;
+
+	if (count < 3)
+		return -EFAULT;
+
+	tmp_len = (count > sizeof(tmp) - 1 ? sizeof(tmp) - 1 : count);
+
+	if (!buffer || copy_from_user(tmp, buffer, tmp_len))
+		return count;
+
+	tmp[tmp_len] = '\0';
+
+	h2c_len = sscanf(tmp, "%X %X %X %X %X %X %X %X",
+			 &h2c_data[0], &h2c_data[1],
+			 &h2c_data[2], &h2c_data[3],
+			 &h2c_data[4], &h2c_data[5],
+			 &h2c_data[6], &h2c_data[7]);
+
+	if (h2c_len <= 0)
+		return count;
+
+	for (i = 0; i < h2c_len; i++)
+		h2c_data_packed[i] = (u8)h2c_data[i];
+
+	rtlpriv->cfg->ops->fill_h2c_cmd(hw, h2c_data_packed[0],
+					h2c_len - 1,
+					&h2c_data_packed[1]);
+
+	return count;
+}
+
+static struct rtl_debugfs_priv rtl_debug_priv_write_h2c = {
+	.cb_write = rtl_debugfs_set_write_h2c,
+};
+
+static ssize_t rtl_debugfs_set_write_rfreg(struct file *filp,
+					   const char __user *buffer,
+					    size_t count, loff_t *loff)
+{
+	struct rtl_debugfs_priv *debugfs_priv = filp->private_data;
+	struct rtl_priv *rtlpriv = debugfs_priv->rtlpriv;
+	struct ieee80211_hw *hw = rtlpriv->hw;
+	char tmp[32 + 1];
+	int tmp_len;
+	int num;
+	int path;
+	u32 addr, bitmask, data;
+
+	if (count < 3)
+		return -EFAULT;
+
+	tmp_len = (count > sizeof(tmp) - 1 ? sizeof(tmp) - 1 : count);
+
+	if (!buffer || copy_from_user(tmp, buffer, tmp_len))
+		return count;
+
+	tmp[tmp_len] = '\0';
+
+	num = sscanf(tmp, "%X %X %X %X",
+		     &path, &addr, &bitmask, &data);
+
+	if (num != 4) {
+		RT_TRACE(rtlpriv, COMP_ERR, DBG_DMESG,
+			 "Format is <path> <addr> <mask> <data>\n");
+		return count;
+	}
+
+	rtl_set_rfreg(hw, path, addr, bitmask, data);
+
+	return count;
+}
+
+static struct rtl_debugfs_priv rtl_debug_priv_write_rfreg = {
+	.cb_write = rtl_debugfs_set_write_rfreg,
+};
+
+static int rtl_debugfs_close(struct inode *inode, struct file *filp)
+{
+	return 0;
+}
+
+static ssize_t rtl_debugfs_common_write(struct file *filp,
+					const char __user *buffer,
+					size_t count, loff_t *loff)
+{
+	struct rtl_debugfs_priv *debugfs_priv = filp->private_data;
+
+	return debugfs_priv->cb_write(filp, buffer, count, loff);
+}
+
+static const struct file_operations file_ops_common_write = {
+	.owner = THIS_MODULE,
+	.write = rtl_debugfs_common_write,
+	.open = simple_open,
+	.release = rtl_debugfs_close,
+};
+
+static ssize_t rtl_debugfs_phydm_cmd(struct file *filp,
+				     const char __user *buffer,
+				     size_t count, loff_t *loff)
+{
+	struct rtl_debugfs_priv *debugfs_priv = filp->private_data;
+	struct rtl_priv *rtlpriv = debugfs_priv->rtlpriv;
+
+	char tmp[64];
+
+	if (!rtlpriv->dbg.msg_buf)
+		return -ENOMEM;
+
+	if (!rtlpriv->phydm.ops)
+		return -EFAULT;
+
+	if (buffer && !copy_from_user(tmp, buffer, sizeof(tmp))) {
+		tmp[count] = '\0';
+
+		rtlpriv->phydm.ops->phydm_debug_cmd(rtlpriv, tmp, count,
+						    rtlpriv->dbg.msg_buf,
+						    80 * 25);
+	}
+
+	return count;
+}
+
+static int rtl_debug_get_phydm_cmd(struct seq_file *m, void *v)
+{
+	struct rtl_debugfs_priv *debugfs_priv = m->private;
+	struct rtl_priv *rtlpriv = debugfs_priv->rtlpriv;
+
+	if (rtlpriv->dbg.msg_buf)
+		seq_puts(m, rtlpriv->dbg.msg_buf);
+
+	return 0;
+}
+
+static int rtl_debugfs_open_rw(struct inode *inode, struct file *filp)
+{
+	if (filp->f_mode & FMODE_READ)
+		single_open(filp, rtl_debug_get_common, inode->i_private);
+	else
+		filp->private_data = inode->i_private;
+
+	return 0;
+}
+
+static int rtl_debugfs_close_rw(struct inode *inode, struct file *filp)
+{
+	if (filp->f_mode == FMODE_READ)
+		seq_release(inode, filp);
+
+	return 0;
+}
+
+static struct rtl_debugfs_priv rtl_debug_priv_phydm_cmd = {
+	.cb_read = rtl_debug_get_phydm_cmd,
+	.cb_write = rtl_debugfs_phydm_cmd,
+	.cb_data = 0,
+};
+
+static const struct file_operations file_ops_common_rw = {
+	.owner = THIS_MODULE,
+	.open = rtl_debugfs_open_rw,
+	.release = rtl_debugfs_close_rw,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.write = rtl_debugfs_common_write,
+};
+
+#define RTL_DEBUGFS_ADD_CORE(name, mode, fopname)			   \
+	do {								   \
+		rtl_debug_priv_ ##name.rtlpriv = rtlpriv;		   \
+		if (!debugfs_create_file(#name, mode,			   \
+					 parent, &rtl_debug_priv_ ##name,  \
+					 &file_ops_ ##fopname))		   \
+			pr_err("Unable to initialize debugfs:%s/%s\n",	   \
+			       rtlpriv->dbg.debugfs_name,		   \
+			       #name);					   \
+	} while (0)
+
+#define RTL_DEBUGFS_ADD(name)						   \
+		RTL_DEBUGFS_ADD_CORE(name, S_IFREG | 0444, common)
+#define RTL_DEBUGFS_ADD_W(name)						   \
+		RTL_DEBUGFS_ADD_CORE(name, S_IFREG | 0222, common_write)
+#define RTL_DEBUGFS_ADD_RW(name)					   \
+		RTL_DEBUGFS_ADD_CORE(name, S_IFREG | 0666, common_rw)
+
+void rtl_debug_add_one(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
+	struct dentry *parent;
+
+	rtlpriv->dbg.msg_buf = vzalloc(80 * 25);
+
+	snprintf(rtlpriv->dbg.debugfs_name, 18, "%pMF", rtlefuse->dev_addr);
+
+	rtlpriv->dbg.debugfs_dir =
+		debugfs_create_dir(rtlpriv->dbg.debugfs_name, debugfs_topdir);
+	if (!rtlpriv->dbg.debugfs_dir) {
+		pr_err("Unable to init debugfs:/%s/%s\n", rtlpriv->cfg->name,
+		       rtlpriv->dbg.debugfs_name);
+		return;
+	}
+
+	parent = rtlpriv->dbg.debugfs_dir;
+
+	RTL_DEBUGFS_ADD(mac_0);
+	RTL_DEBUGFS_ADD(mac_1);
+	RTL_DEBUGFS_ADD(mac_2);
+	RTL_DEBUGFS_ADD(mac_3);
+	RTL_DEBUGFS_ADD(mac_4);
+	RTL_DEBUGFS_ADD(mac_5);
+	RTL_DEBUGFS_ADD(mac_6);
+	RTL_DEBUGFS_ADD(mac_7);
+	RTL_DEBUGFS_ADD(bb_8);
+	RTL_DEBUGFS_ADD(bb_9);
+	RTL_DEBUGFS_ADD(bb_a);
+	RTL_DEBUGFS_ADD(bb_b);
+	RTL_DEBUGFS_ADD(bb_c);
+	RTL_DEBUGFS_ADD(bb_d);
+	RTL_DEBUGFS_ADD(bb_e);
+	RTL_DEBUGFS_ADD(bb_f);
+	RTL_DEBUGFS_ADD(mac_10);
+	RTL_DEBUGFS_ADD(mac_11);
+	RTL_DEBUGFS_ADD(mac_12);
+	RTL_DEBUGFS_ADD(mac_13);
+	RTL_DEBUGFS_ADD(mac_14);
+	RTL_DEBUGFS_ADD(mac_15);
+	RTL_DEBUGFS_ADD(mac_16);
+	RTL_DEBUGFS_ADD(mac_17);
+	RTL_DEBUGFS_ADD(bb_18);
+	RTL_DEBUGFS_ADD(bb_19);
+	RTL_DEBUGFS_ADD(bb_1a);
+	RTL_DEBUGFS_ADD(bb_1b);
+	RTL_DEBUGFS_ADD(bb_1c);
+	RTL_DEBUGFS_ADD(bb_1d);
+	RTL_DEBUGFS_ADD(bb_1e);
+	RTL_DEBUGFS_ADD(bb_1f);
+	RTL_DEBUGFS_ADD(rf_a);
+	RTL_DEBUGFS_ADD(rf_b);
+
+	RTL_DEBUGFS_ADD(cam_1);
+	RTL_DEBUGFS_ADD(cam_2);
+	RTL_DEBUGFS_ADD(cam_3);
+
+	RTL_DEBUGFS_ADD(btcoex);
+
+	RTL_DEBUGFS_ADD_W(write_reg);
+	RTL_DEBUGFS_ADD_W(write_h2c);
+	RTL_DEBUGFS_ADD_W(write_rfreg);
+
+	RTL_DEBUGFS_ADD_RW(phydm_cmd);
+}
+
+void rtl_debug_remove_one(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	debugfs_remove_recursive(rtlpriv->dbg.debugfs_dir);
+	rtlpriv->dbg.debugfs_dir = NULL;
+
+	vfree(rtlpriv->dbg.msg_buf);
+}
+
+void rtl_debugfs_add_topdir(void)
+{
+	debugfs_topdir = debugfs_create_dir("rtlwifi", NULL);
+}
+
+void rtl_debugfs_remove_topdir(void)
+{
+	debugfs_remove_recursive(debugfs_topdir);
+}
+
+#endif
diff --git a/drivers/staging/rtlwifi/debug.h b/drivers/staging/rtlwifi/debug.h
new file mode 100644
index 000000000000..ac942477f629
--- /dev/null
+++ b/drivers/staging/rtlwifi/debug.h
@@ -0,0 +1,234 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2012  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *****************************************************************************/
+
+#ifndef __RTL_DEBUG_H__
+#define __RTL_DEBUG_H__
+
+/*--------------------------------------------------------------
+ *			Debug level
+ *------------------------------------------------------------
+ *
+ *Fatal bug.
+ *For example, Tx/Rx/IO locked up,
+ *memory access violation,
+ *resource allocation failed,
+ *unexpected HW behavior, HW BUG
+ *and so on.
+ */
+/*#define DBG_EMERG			0 */
+
+/*Abnormal, rare, or unexpected cases.
+ *For example, Packet/IO Ctl canceled,
+ *device surprisingly removed and so on.
+ */
+#define	DBG_WARNING			2
+
+/*Normal case driver developer should
+ *open, we can see link status like
+ *assoc/AddBA/DHCP/adapter start and
+ *so on basic and useful infromations.
+ */
+#define DBG_DMESG			3
+
+/*Normal case with useful information
+ *about current SW or HW state.
+ *For example, Tx/Rx descriptor to fill,
+ *Tx/Rx descriptor completed status,
+ *SW protocol state change, dynamic
+ *mechanism state change and so on.
+ */
+#define DBG_LOUD			4
+
+/*Normal case with detail execution
+ *flow or information.
+ */
+#define	DBG_TRACE			5
+
+/*--------------------------------------------------------------
+ *		Define the rt_trace components
+ *--------------------------------------------------------------
+ */
+#define COMP_ERR			BIT(0)
+#define COMP_FW				BIT(1)
+#define COMP_INIT			BIT(2)	/*For init/deinit */
+#define COMP_RECV			BIT(3)	/*For Rx. */
+#define COMP_SEND			BIT(4)	/*For Tx. */
+#define COMP_MLME			BIT(5)	/*For MLME. */
+#define COMP_SCAN			BIT(6)	/*For Scan. */
+#define COMP_INTR			BIT(7)	/*For interrupt Related. */
+#define COMP_LED			BIT(8)	/*For LED. */
+#define COMP_SEC			BIT(9)	/*For sec. */
+#define COMP_BEACON			BIT(10)	/*For beacon. */
+#define COMP_RATE			BIT(11)	/*For rate. */
+#define COMP_RXDESC			BIT(12)	/*For rx desc. */
+#define COMP_DIG			BIT(13)	/*For DIG */
+#define COMP_TXAGC			BIT(14)	/*For Tx power */
+#define COMP_HIPWR			BIT(15)	/*For High Power Mechanism */
+#define COMP_POWER			BIT(16)	/*For lps/ips/aspm. */
+#define COMP_POWER_TRACKING	BIT(17)	/*For TX POWER TRACKING */
+#define COMP_BB_POWERSAVING	BIT(18)
+#define COMP_SWAS			BIT(19)	/*For SW Antenna Switch */
+#define COMP_RF				BIT(20)	/*For RF. */
+#define COMP_TURBO			BIT(21)	/*For EDCA TURBO. */
+#define COMP_RATR			BIT(22)
+#define COMP_CMD			BIT(23)
+#define COMP_EFUSE			BIT(24)
+#define COMP_QOS			BIT(25)
+#define COMP_MAC80211		BIT(26)
+#define COMP_REGD			BIT(27)
+#define COMP_CHAN			BIT(28)
+#define COMP_USB			BIT(29)
+#define COMP_EASY_CONCURRENT	COMP_USB /* reuse of this bit is OK */
+#define COMP_BT_COEXIST			BIT(30)
+#define COMP_IQK			BIT(31)
+#define COMP_TX_REPORT			BIT_ULL(32)
+#define COMP_HALMAC			BIT_ULL(34)
+#define COMP_PHYDM			BIT_ULL(35)
+
+/*--------------------------------------------------------------
+ *		Define the rt_print components
+ *--------------------------------------------------------------
+ */
+/* Define EEPROM and EFUSE  check module bit*/
+#define EEPROM_W			BIT(0)
+#define EFUSE_PG			BIT(1)
+#define EFUSE_READ_ALL			BIT(2)
+
+/* Define init check for module bit*/
+#define	INIT_EEPROM			BIT(0)
+#define	INIT_TXPOWER			BIT(1)
+#define	INIT_IQK			BIT(2)
+#define	INIT_RF				BIT(3)
+
+/* Define PHY-BB/RF/MAC check module bit */
+#define	PHY_BBR				BIT(0)
+#define	PHY_BBW				BIT(1)
+#define	PHY_RFR				BIT(2)
+#define	PHY_RFW				BIT(3)
+#define	PHY_MACR			BIT(4)
+#define	PHY_MACW			BIT(5)
+#define	PHY_ALLR			BIT(6)
+#define	PHY_ALLW			BIT(7)
+#define	PHY_TXPWR			BIT(8)
+#define	PHY_PWRDIFF			BIT(9)
+
+/* Define Dynamic Mechanism check module bit --> FDM */
+#define WA_IOT				BIT(0)
+#define DM_PWDB				BIT(1)
+#define DM_MONITOR			BIT(2)
+#define DM_DIG				BIT(3)
+#define DM_EDCA_TURBO			BIT(4)
+
+#define DM_PWDB				BIT(1)
+
+enum dbgp_flag_e {
+	FQOS = 0,
+	FTX = 1,
+	FRX = 2,
+	FSEC = 3,
+	FMGNT = 4,
+	FMLME = 5,
+	FRESOURCE = 6,
+	FBEACON = 7,
+	FISR = 8,
+	FPHY = 9,
+	FMP = 10,
+	FEEPROM = 11,
+	FPWR = 12,
+	FDM = 13,
+	FDBGCTRL = 14,
+	FC2H = 15,
+	FBT = 16,
+	FINIT = 17,
+	FIOCTL = 18,
+	DBGP_TYPE_MAX
+};
+
+#ifdef CONFIG_RTLWIFI_DEBUG_ST
+
+struct rtl_priv;
+
+__printf(4, 5)
+void _rtl_dbg_trace(struct rtl_priv *rtlpriv, u64 comp, int level,
+		    const char *fmt, ...);
+
+__printf(4, 5)
+void _rtl_dbg_print(struct rtl_priv *rtlpriv, u64 comp, int level,
+		    const char *fmt, ...);
+
+void _rtl_dbg_print_data(struct rtl_priv *rtlpriv, u64 comp, int level,
+			 const char *titlestring,
+			 const void *hexdata, int hexdatalen);
+
+#define RT_TRACE(rtlpriv, comp, level, fmt, ...)			\
+	_rtl_dbg_trace(rtlpriv, comp, level,				\
+		       fmt, ##__VA_ARGS__)
+
+#define RTPRINT(rtlpriv, dbgtype, dbgflag, fmt, ...)			\
+	_rtl_dbg_print(rtlpriv, dbgtype, dbgflag, fmt, ##__VA_ARGS__)
+
+#define RT_PRINT_DATA(rtlpriv, _comp, _level, _titlestring, _hexdata,	\
+		      _hexdatalen)					\
+	_rtl_dbg_print_data(rtlpriv, _comp, _level,			\
+			    _titlestring, _hexdata, _hexdatalen)
+
+#else
+
+struct rtl_priv;
+
+__printf(4, 5)
+static inline void RT_TRACE(struct rtl_priv *rtlpriv,
+			    u64 comp, int level,
+			    const char *fmt, ...)
+{
+}
+
+__printf(4, 5)
+static inline void RTPRINT(struct rtl_priv *rtlpriv,
+			   int dbgtype, int dbgflag,
+			   const char *fmt, ...)
+{
+}
+
+static inline void RT_PRINT_DATA(struct rtl_priv *rtlpriv,
+				 u64 comp, int level,
+				 const char *titlestring,
+				 const void *hexdata, size_t hexdatalen)
+{
+}
+
+#endif
+
+#ifdef CONFIG_RTLWIFI_DEBUG_ST
+void rtl_debug_add_one(struct ieee80211_hw *hw);
+void rtl_debug_remove_one(struct ieee80211_hw *hw);
+void rtl_debugfs_add_topdir(void);
+void rtl_debugfs_remove_topdir(void);
+#else
+#define rtl_debug_add_one(hw)
+#define rtl_debug_remove_one(hw)
+#define rtl_debugfs_add_topdir()
+#define rtl_debugfs_remove_topdir()
+#endif
+#endif
diff --git a/drivers/staging/rtlwifi/efuse.c b/drivers/staging/rtlwifi/efuse.c
new file mode 100644
index 000000000000..6d5e657017c6
--- /dev/null
+++ b/drivers/staging/rtlwifi/efuse.c
@@ -0,0 +1,1342 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2012  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#include "wifi.h"
+#include "efuse.h"
+#include "pci.h"
+#include <linux/export.h>
+
+static const u8 MAX_PGPKT_SIZE = 9;
+static const u8 PGPKT_DATA_SIZE = 8;
+static const int EFUSE_MAX_SIZE = 512;
+
+#define START_ADDRESS		0x1000
+#define REG_MCUFWDL		0x0080
+
+static const struct efuse_map RTL8712_SDIO_EFUSE_TABLE[] = {
+	{0, 0, 0, 2},
+	{0, 1, 0, 2},
+	{0, 2, 0, 2},
+	{1, 0, 0, 1},
+	{1, 0, 1, 1},
+	{1, 1, 0, 1},
+	{1, 1, 1, 3},
+	{1, 3, 0, 17},
+	{3, 3, 1, 48},
+	{10, 0, 0, 6},
+	{10, 3, 0, 1},
+	{10, 3, 1, 1},
+	{11, 0, 0, 28}
+};
+
+static void efuse_shadow_read_1byte(struct ieee80211_hw *hw, u16 offset,
+				    u8 *value);
+static void efuse_shadow_read_2byte(struct ieee80211_hw *hw, u16 offset,
+				    u16 *value);
+static void efuse_shadow_read_4byte(struct ieee80211_hw *hw, u16 offset,
+				    u32 *value);
+static void efuse_shadow_write_1byte(struct ieee80211_hw *hw, u16 offset,
+				     u8 value);
+static void efuse_shadow_write_2byte(struct ieee80211_hw *hw, u16 offset,
+				     u16 value);
+static void efuse_shadow_write_4byte(struct ieee80211_hw *hw, u16 offset,
+				     u32 value);
+static int efuse_one_byte_write(struct ieee80211_hw *hw, u16 addr,
+				u8 data);
+static void efuse_read_all_map(struct ieee80211_hw *hw, u8 *efuse);
+static int efuse_pg_packet_read(struct ieee80211_hw *hw, u8 offset,
+				u8 *data);
+static int efuse_pg_packet_write(struct ieee80211_hw *hw, u8 offset,
+				 u8 word_en, u8 *data);
+static void efuse_word_enable_data_read(u8 word_en, u8 *sourdata,
+					u8 *targetdata);
+static u8 enable_efuse_data_write(struct ieee80211_hw *hw,
+				  u16 efuse_addr, u8 word_en, u8 *data);
+static u16 efuse_get_current_size(struct ieee80211_hw *hw);
+static u8 efuse_calculate_word_cnts(u8 word_en);
+
+void efuse_initialize(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 bytetemp;
+	u8 temp;
+
+	bytetemp = rtl_read_byte(rtlpriv, rtlpriv->cfg->maps[SYS_FUNC_EN] + 1);
+	temp = bytetemp | 0x20;
+	rtl_write_byte(rtlpriv, rtlpriv->cfg->maps[SYS_FUNC_EN] + 1, temp);
+
+	bytetemp = rtl_read_byte(rtlpriv, rtlpriv->cfg->maps[SYS_ISO_CTRL] + 1);
+	temp = bytetemp & 0xFE;
+	rtl_write_byte(rtlpriv, rtlpriv->cfg->maps[SYS_ISO_CTRL] + 1, temp);
+
+	bytetemp = rtl_read_byte(rtlpriv, rtlpriv->cfg->maps[EFUSE_TEST] + 3);
+	temp = bytetemp | 0x80;
+	rtl_write_byte(rtlpriv, rtlpriv->cfg->maps[EFUSE_TEST] + 3, temp);
+
+	rtl_write_byte(rtlpriv, 0x2F8, 0x3);
+
+	rtl_write_byte(rtlpriv, rtlpriv->cfg->maps[EFUSE_CTRL] + 3, 0x72);
+}
+
+u8 efuse_read_1byte(struct ieee80211_hw *hw, u16 address)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 data;
+	u8 bytetemp;
+	u8 temp;
+	u32 k = 0;
+	const u32 efuse_len =
+		rtlpriv->cfg->maps[EFUSE_REAL_CONTENT_SIZE];
+
+	if (address < efuse_len) {
+		temp = address & 0xFF;
+		rtl_write_byte(rtlpriv, rtlpriv->cfg->maps[EFUSE_CTRL] + 1,
+			       temp);
+		bytetemp = rtl_read_byte(rtlpriv,
+					 rtlpriv->cfg->maps[EFUSE_CTRL] + 2);
+		temp = ((address >> 8) & 0x03) | (bytetemp & 0xFC);
+		rtl_write_byte(rtlpriv, rtlpriv->cfg->maps[EFUSE_CTRL] + 2,
+			       temp);
+
+		bytetemp = rtl_read_byte(rtlpriv,
+					 rtlpriv->cfg->maps[EFUSE_CTRL] + 3);
+		temp = bytetemp & 0x7F;
+		rtl_write_byte(rtlpriv, rtlpriv->cfg->maps[EFUSE_CTRL] + 3,
+			       temp);
+
+		bytetemp = rtl_read_byte(rtlpriv,
+					 rtlpriv->cfg->maps[EFUSE_CTRL] + 3);
+		while (!(bytetemp & 0x80)) {
+			bytetemp =
+			   rtl_read_byte(rtlpriv,
+					 rtlpriv->cfg->maps[EFUSE_CTRL] + 3);
+			k++;
+			if (k == 1000) {
+				k = 0;
+				break;
+			}
+		}
+		data = rtl_read_byte(rtlpriv, rtlpriv->cfg->maps[EFUSE_CTRL]);
+		return data;
+	}
+	return 0xFF;
+}
+
+void efuse_write_1byte(struct ieee80211_hw *hw, u16 address, u8 value)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 bytetemp;
+	u8 temp;
+	u32 k = 0;
+	const u32 efuse_len =
+		rtlpriv->cfg->maps[EFUSE_REAL_CONTENT_SIZE];
+
+	RT_TRACE(rtlpriv, COMP_EFUSE, DBG_LOUD, "Addr=%x Data =%x\n",
+		 address, value);
+
+	if (address < efuse_len) {
+		rtl_write_byte(rtlpriv, rtlpriv->cfg->maps[EFUSE_CTRL], value);
+
+		temp = address & 0xFF;
+		rtl_write_byte(rtlpriv, rtlpriv->cfg->maps[EFUSE_CTRL] + 1,
+			       temp);
+		bytetemp = rtl_read_byte(rtlpriv,
+					 rtlpriv->cfg->maps[EFUSE_CTRL] + 2);
+
+		temp = ((address >> 8) & 0x03) | (bytetemp & 0xFC);
+		rtl_write_byte(rtlpriv,
+			       rtlpriv->cfg->maps[EFUSE_CTRL] + 2, temp);
+
+		bytetemp = rtl_read_byte(rtlpriv,
+					 rtlpriv->cfg->maps[EFUSE_CTRL] + 3);
+		temp = bytetemp | 0x80;
+		rtl_write_byte(rtlpriv,
+			       rtlpriv->cfg->maps[EFUSE_CTRL] + 3, temp);
+
+		bytetemp = rtl_read_byte(rtlpriv,
+					 rtlpriv->cfg->maps[EFUSE_CTRL] + 3);
+
+		while (bytetemp & 0x80) {
+			bytetemp =
+			    rtl_read_byte(rtlpriv,
+					  rtlpriv->cfg->maps[EFUSE_CTRL] + 3);
+			k++;
+			if (k == 100) {
+				k = 0;
+				break;
+			}
+		}
+	}
+}
+
+void read_efuse_byte(struct ieee80211_hw *hw, u16 _offset, u8 *pbuf)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u32 value32;
+	u8 readbyte;
+	u16 retry;
+
+	rtl_write_byte(rtlpriv, rtlpriv->cfg->maps[EFUSE_CTRL] + 1,
+		       (_offset & 0xff));
+	readbyte = rtl_read_byte(rtlpriv, rtlpriv->cfg->maps[EFUSE_CTRL] + 2);
+	rtl_write_byte(rtlpriv, rtlpriv->cfg->maps[EFUSE_CTRL] + 2,
+		       ((_offset >> 8) & 0x03) | (readbyte & 0xfc));
+
+	readbyte = rtl_read_byte(rtlpriv, rtlpriv->cfg->maps[EFUSE_CTRL] + 3);
+	rtl_write_byte(rtlpriv, rtlpriv->cfg->maps[EFUSE_CTRL] + 3,
+		       (readbyte & 0x7f));
+
+	retry = 0;
+	value32 = rtl_read_dword(rtlpriv, rtlpriv->cfg->maps[EFUSE_CTRL]);
+	while (!(((value32 >> 24) & 0xff) & 0x80) && (retry < 10000)) {
+		value32 = rtl_read_dword(rtlpriv,
+					 rtlpriv->cfg->maps[EFUSE_CTRL]);
+		retry++;
+	}
+
+	udelay(50);
+	value32 = rtl_read_dword(rtlpriv, rtlpriv->cfg->maps[EFUSE_CTRL]);
+
+	*pbuf = (u8)(value32 & 0xff);
+}
+
+void read_efuse(struct ieee80211_hw *hw, u16 _offset, u16 _size_byte, u8 *pbuf)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
+	u8 *efuse_tbl;
+	u8 rtemp8[1];
+	u16 efuse_addr = 0;
+	u8 offset, wren;
+	u8 u1temp = 0;
+	u16 i;
+	u16 j;
+	const u16 efuse_max_section =
+		rtlpriv->cfg->maps[EFUSE_MAX_SECTION_MAP];
+	const u32 efuse_len =
+		rtlpriv->cfg->maps[EFUSE_REAL_CONTENT_SIZE];
+	u16 **efuse_word;
+	u16 efuse_utilized = 0;
+	u8 efuse_usage;
+
+	if ((_offset + _size_byte) > rtlpriv->cfg->maps[EFUSE_HWSET_MAX_SIZE]) {
+		RT_TRACE(rtlpriv, COMP_EFUSE, DBG_LOUD,
+			 "%s(): Invalid offset(%#x) with read bytes(%#x)!!\n",
+			 __func__, _offset, _size_byte);
+		return;
+	}
+
+	/* allocate memory for efuse_tbl and efuse_word */
+	efuse_tbl = kzalloc(rtlpriv->cfg->maps[EFUSE_HWSET_MAX_SIZE] *
+			    sizeof(u8), GFP_ATOMIC);
+	if (!efuse_tbl)
+		return;
+	efuse_word = kzalloc(EFUSE_MAX_WORD_UNIT * sizeof(u16 *), GFP_ATOMIC);
+	if (!efuse_word)
+		goto out;
+	for (i = 0; i < EFUSE_MAX_WORD_UNIT; i++) {
+		efuse_word[i] = kzalloc(efuse_max_section * sizeof(u16),
+					GFP_ATOMIC);
+		if (!efuse_word[i])
+			goto done;
+	}
+
+	for (i = 0; i < efuse_max_section; i++)
+		for (j = 0; j < EFUSE_MAX_WORD_UNIT; j++)
+			efuse_word[j][i] = 0xFFFF;
+
+	read_efuse_byte(hw, efuse_addr, rtemp8);
+	if (*rtemp8 != 0xFF) {
+		efuse_utilized++;
+		RTPRINT(rtlpriv, FEEPROM, EFUSE_READ_ALL,
+			"Addr=%d\n", efuse_addr);
+		efuse_addr++;
+	}
+
+	while ((*rtemp8 != 0xFF) && (efuse_addr < efuse_len)) {
+		/*  Check PG header for section num.  */
+		if ((*rtemp8 & 0x1F) == 0x0F) {/* extended header */
+			u1temp = ((*rtemp8 & 0xE0) >> 5);
+			read_efuse_byte(hw, efuse_addr, rtemp8);
+
+			if ((*rtemp8 & 0x0F) == 0x0F) {
+				efuse_addr++;
+				read_efuse_byte(hw, efuse_addr, rtemp8);
+
+				if (*rtemp8 != 0xFF &&
+				    (efuse_addr < efuse_len)) {
+					efuse_addr++;
+				}
+				continue;
+			} else {
+				offset = ((*rtemp8 & 0xF0) >> 1) | u1temp;
+				wren = (*rtemp8 & 0x0F);
+				efuse_addr++;
+			}
+		} else {
+			offset = ((*rtemp8 >> 4) & 0x0f);
+			wren = (*rtemp8 & 0x0f);
+		}
+
+		if (offset < efuse_max_section) {
+			RTPRINT(rtlpriv, FEEPROM, EFUSE_READ_ALL,
+				"offset-%d Worden=%x\n", offset, wren);
+
+			for (i = 0; i < EFUSE_MAX_WORD_UNIT; i++) {
+				if (!(wren & 0x01)) {
+					RTPRINT(rtlpriv, FEEPROM,
+						EFUSE_READ_ALL,
+						"Addr=%d\n", efuse_addr);
+
+					read_efuse_byte(hw, efuse_addr, rtemp8);
+					efuse_addr++;
+					efuse_utilized++;
+					efuse_word[i][offset] =
+							 (*rtemp8 & 0xff);
+
+					if (efuse_addr >= efuse_len)
+						break;
+
+					RTPRINT(rtlpriv, FEEPROM,
+						EFUSE_READ_ALL,
+						"Addr=%d\n", efuse_addr);
+
+					read_efuse_byte(hw, efuse_addr, rtemp8);
+					efuse_addr++;
+					efuse_utilized++;
+					efuse_word[i][offset] |=
+					    (((u16)*rtemp8 << 8) & 0xff00);
+
+					if (efuse_addr >= efuse_len)
+						break;
+				}
+
+				wren >>= 1;
+			}
+		}
+
+		RTPRINT(rtlpriv, FEEPROM, EFUSE_READ_ALL,
+			"Addr=%d\n", efuse_addr);
+		read_efuse_byte(hw, efuse_addr, rtemp8);
+		if (*rtemp8 != 0xFF && (efuse_addr < efuse_len)) {
+			efuse_utilized++;
+			efuse_addr++;
+		}
+	}
+
+	for (i = 0; i < efuse_max_section; i++) {
+		for (j = 0; j < EFUSE_MAX_WORD_UNIT; j++) {
+			efuse_tbl[(i * 8) + (j * 2)] =
+			    (efuse_word[j][i] & 0xff);
+			efuse_tbl[(i * 8) + ((j * 2) + 1)] =
+			    ((efuse_word[j][i] >> 8) & 0xff);
+		}
+	}
+
+	for (i = 0; i < _size_byte; i++)
+		pbuf[i] = efuse_tbl[_offset + i];
+
+	rtlefuse->efuse_usedbytes = efuse_utilized;
+	efuse_usage = (u8)((efuse_utilized * 100) / efuse_len);
+	rtlefuse->efuse_usedpercentage = efuse_usage;
+	rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_EFUSE_BYTES,
+				      (u8 *)&efuse_utilized);
+	rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_EFUSE_USAGE,
+				      &efuse_usage);
+done:
+	for (i = 0; i < EFUSE_MAX_WORD_UNIT; i++)
+		kfree(efuse_word[i]);
+	kfree(efuse_word);
+out:
+	kfree(efuse_tbl);
+}
+
+bool efuse_shadow_update_chk(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
+	u8 section_idx, i, base;
+	u16 words_need = 0, hdr_num = 0, totalbytes, efuse_used;
+	bool wordchanged, result = true;
+
+	for (section_idx = 0; section_idx < 16; section_idx++) {
+		base = section_idx * 8;
+		wordchanged = false;
+
+		for (i = 0; i < 8; i = i + 2) {
+			if ((rtlefuse->efuse_map[EFUSE_INIT_MAP][base + i] !=
+			     rtlefuse->efuse_map[EFUSE_MODIFY_MAP][base + i]) ||
+			    (rtlefuse->efuse_map[EFUSE_INIT_MAP][base + i + 1] !=
+			     rtlefuse->efuse_map[EFUSE_MODIFY_MAP][base + i +
+								   1])) {
+				words_need++;
+				wordchanged = true;
+			}
+		}
+
+		if (wordchanged)
+			hdr_num++;
+	}
+
+	totalbytes = hdr_num + words_need * 2;
+	efuse_used = rtlefuse->efuse_usedbytes;
+
+	if ((totalbytes + efuse_used) >=
+	    (EFUSE_MAX_SIZE - rtlpriv->cfg->maps[EFUSE_OOB_PROTECT_BYTES_LEN]))
+		result = false;
+
+	RT_TRACE(rtlpriv, COMP_EFUSE, DBG_LOUD,
+		 "%s(): totalbytes(%#x), hdr_num(%#x), words_need(%#x), efuse_used(%d)\n",
+		 __func__, totalbytes, hdr_num, words_need, efuse_used);
+
+	return result;
+}
+
+void efuse_shadow_read(struct ieee80211_hw *hw, u8 type,
+		       u16 offset, u32 *value)
+{
+	if (type == 1)
+		efuse_shadow_read_1byte(hw, offset, (u8 *)value);
+	else if (type == 2)
+		efuse_shadow_read_2byte(hw, offset, (u16 *)value);
+	else if (type == 4)
+		efuse_shadow_read_4byte(hw, offset, value);
+}
+
+void efuse_shadow_write(struct ieee80211_hw *hw, u8 type, u16 offset,
+			u32 value)
+{
+	if (type == 1)
+		efuse_shadow_write_1byte(hw, offset, (u8)value);
+	else if (type == 2)
+		efuse_shadow_write_2byte(hw, offset, (u16)value);
+	else if (type == 4)
+		efuse_shadow_write_4byte(hw, offset, value);
+}
+
+bool efuse_shadow_update(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
+	u16 i, offset, base;
+	u8 word_en = 0x0F;
+	u8 first_pg = false;
+
+	RT_TRACE(rtlpriv, COMP_EFUSE, DBG_LOUD, "\n");
+
+	if (!efuse_shadow_update_chk(hw)) {
+		efuse_read_all_map(hw, &rtlefuse->efuse_map[EFUSE_INIT_MAP][0]);
+		memcpy(&rtlefuse->efuse_map[EFUSE_MODIFY_MAP][0],
+		       &rtlefuse->efuse_map[EFUSE_INIT_MAP][0],
+		       rtlpriv->cfg->maps[EFUSE_HWSET_MAX_SIZE]);
+
+		RT_TRACE(rtlpriv, COMP_EFUSE, DBG_LOUD,
+			 "efuse out of capacity!!\n");
+		return false;
+	}
+	efuse_power_switch(hw, true, true);
+
+	for (offset = 0; offset < 16; offset++) {
+		word_en = 0x0F;
+		base = offset * 8;
+
+		for (i = 0; i < 8; i++) {
+			if (first_pg) {
+				word_en &= ~(BIT(i / 2));
+
+				rtlefuse->efuse_map[EFUSE_INIT_MAP][base + i] =
+				    rtlefuse->efuse_map[EFUSE_MODIFY_MAP][base + i];
+			} else {
+				if (rtlefuse->efuse_map[EFUSE_INIT_MAP][base + i] !=
+				    rtlefuse->efuse_map[EFUSE_MODIFY_MAP][base + i]) {
+					word_en &= ~(BIT(i / 2));
+
+					rtlefuse->efuse_map[EFUSE_INIT_MAP][base + i] =
+					    rtlefuse->efuse_map[EFUSE_MODIFY_MAP][base + i];
+				}
+			}
+		}
+		if (word_en != 0x0F) {
+			u8 tmpdata[8];
+
+			memcpy(tmpdata,
+			       &rtlefuse->efuse_map[EFUSE_MODIFY_MAP][base],
+			       8);
+			RT_PRINT_DATA(rtlpriv, COMP_INIT, DBG_LOUD,
+				      "U-efuse\n", tmpdata, 8);
+
+			if (!efuse_pg_packet_write(hw, (u8)offset, word_en,
+						   tmpdata)) {
+				RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING,
+					 "PG section(%#x) fail!!\n", offset);
+				break;
+			}
+		}
+	}
+
+	efuse_power_switch(hw, true, false);
+	efuse_read_all_map(hw, &rtlefuse->efuse_map[EFUSE_INIT_MAP][0]);
+
+	memcpy(&rtlefuse->efuse_map[EFUSE_MODIFY_MAP][0],
+	       &rtlefuse->efuse_map[EFUSE_INIT_MAP][0],
+	       rtlpriv->cfg->maps[EFUSE_HWSET_MAX_SIZE]);
+
+	RT_TRACE(rtlpriv, COMP_EFUSE, DBG_LOUD, "\n");
+	return true;
+}
+
+void rtl_efuse_shadow_map_update(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
+
+	if (rtlefuse->autoload_failflag)
+		memset((&rtlefuse->efuse_map[EFUSE_INIT_MAP][0]),
+		       0xFF, rtlpriv->cfg->maps[EFUSE_HWSET_MAX_SIZE]);
+	else
+		efuse_read_all_map(hw, &rtlefuse->efuse_map[EFUSE_INIT_MAP][0]);
+
+	memcpy(&rtlefuse->efuse_map[EFUSE_MODIFY_MAP][0],
+	       &rtlefuse->efuse_map[EFUSE_INIT_MAP][0],
+	       rtlpriv->cfg->maps[EFUSE_HWSET_MAX_SIZE]);
+}
+
+void efuse_force_write_vendor_id(struct ieee80211_hw *hw)
+{
+	u8 tmpdata[8] = { 0xFF, 0xFF, 0xEC, 0x10, 0xFF, 0xFF, 0xFF, 0xFF };
+
+	efuse_power_switch(hw, true, true);
+
+	efuse_pg_packet_write(hw, 1, 0xD, tmpdata);
+
+	efuse_power_switch(hw, true, false);
+}
+
+void efuse_re_pg_section(struct ieee80211_hw *hw, u8 section_idx)
+{
+}
+
+static void efuse_shadow_read_1byte(struct ieee80211_hw *hw,
+				    u16 offset, u8 *value)
+{
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
+	*value = rtlefuse->efuse_map[EFUSE_MODIFY_MAP][offset];
+}
+
+static void efuse_shadow_read_2byte(struct ieee80211_hw *hw,
+				    u16 offset, u16 *value)
+{
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
+
+	*value = rtlefuse->efuse_map[EFUSE_MODIFY_MAP][offset];
+	*value |= rtlefuse->efuse_map[EFUSE_MODIFY_MAP][offset + 1] << 8;
+}
+
+static void efuse_shadow_read_4byte(struct ieee80211_hw *hw,
+				    u16 offset, u32 *value)
+{
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
+
+	*value = rtlefuse->efuse_map[EFUSE_MODIFY_MAP][offset];
+	*value |= rtlefuse->efuse_map[EFUSE_MODIFY_MAP][offset + 1] << 8;
+	*value |= rtlefuse->efuse_map[EFUSE_MODIFY_MAP][offset + 2] << 16;
+	*value |= rtlefuse->efuse_map[EFUSE_MODIFY_MAP][offset + 3] << 24;
+}
+
+static void efuse_shadow_write_1byte(struct ieee80211_hw *hw,
+				     u16 offset, u8 value)
+{
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
+
+	rtlefuse->efuse_map[EFUSE_MODIFY_MAP][offset] = value;
+}
+
+static void efuse_shadow_write_2byte(struct ieee80211_hw *hw,
+				     u16 offset, u16 value)
+{
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
+
+	rtlefuse->efuse_map[EFUSE_MODIFY_MAP][offset] = value & 0x00FF;
+	rtlefuse->efuse_map[EFUSE_MODIFY_MAP][offset + 1] = value >> 8;
+}
+
+static void efuse_shadow_write_4byte(struct ieee80211_hw *hw,
+				     u16 offset, u32 value)
+{
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
+
+	rtlefuse->efuse_map[EFUSE_MODIFY_MAP][offset] =
+	    (u8)(value & 0x000000FF);
+	rtlefuse->efuse_map[EFUSE_MODIFY_MAP][offset + 1] =
+	    (u8)((value >> 8) & 0x0000FF);
+	rtlefuse->efuse_map[EFUSE_MODIFY_MAP][offset + 2] =
+	    (u8)((value >> 16) & 0x00FF);
+	rtlefuse->efuse_map[EFUSE_MODIFY_MAP][offset + 3] =
+	    (u8)((value >> 24) & 0xFF);
+}
+
+int efuse_one_byte_read(struct ieee80211_hw *hw, u16 addr, u8 *data)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 tmpidx = 0;
+	int result;
+
+	rtl_write_byte(rtlpriv, rtlpriv->cfg->maps[EFUSE_CTRL] + 1,
+		       (u8)(addr & 0xff));
+	rtl_write_byte(rtlpriv, rtlpriv->cfg->maps[EFUSE_CTRL] + 2,
+		       ((u8)((addr >> 8) & 0x03)) |
+		       (rtl_read_byte(rtlpriv,
+				      rtlpriv->cfg->maps[EFUSE_CTRL] + 2) &
+			0xFC));
+
+	rtl_write_byte(rtlpriv, rtlpriv->cfg->maps[EFUSE_CTRL] + 3, 0x72);
+
+	while (!(0x80 & rtl_read_byte(rtlpriv,
+				      rtlpriv->cfg->maps[EFUSE_CTRL] + 3)) &&
+	       (tmpidx < 100)) {
+		tmpidx++;
+	}
+
+	if (tmpidx < 100) {
+		*data = rtl_read_byte(rtlpriv, rtlpriv->cfg->maps[EFUSE_CTRL]);
+		result = true;
+	} else {
+		*data = 0xff;
+		result = false;
+	}
+	return result;
+}
+
+static int efuse_one_byte_write(struct ieee80211_hw *hw, u16 addr, u8 data)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 tmpidx = 0;
+
+	RT_TRACE(rtlpriv, COMP_EFUSE, DBG_LOUD,
+		 "Addr = %x Data=%x\n", addr, data);
+
+	rtl_write_byte(rtlpriv,
+		       rtlpriv->cfg->maps[EFUSE_CTRL] + 1, (u8)(addr & 0xff));
+	rtl_write_byte(rtlpriv, rtlpriv->cfg->maps[EFUSE_CTRL] + 2,
+		       (rtl_read_byte(rtlpriv,
+			 rtlpriv->cfg->maps[EFUSE_CTRL] +
+			 2) & 0xFC) | (u8)((addr >> 8) & 0x03));
+
+	rtl_write_byte(rtlpriv, rtlpriv->cfg->maps[EFUSE_CTRL], data);
+	rtl_write_byte(rtlpriv, rtlpriv->cfg->maps[EFUSE_CTRL] + 3, 0xF2);
+
+	while ((0x80 &
+		rtl_read_byte(rtlpriv, rtlpriv->cfg->maps[EFUSE_CTRL] + 3)) &&
+	       (tmpidx < 100)) {
+		tmpidx++;
+	}
+
+	if (tmpidx < 100)
+		return true;
+	return false;
+}
+
+static void efuse_read_all_map(struct ieee80211_hw *hw, u8 *efuse)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	efuse_power_switch(hw, false, true);
+	read_efuse(hw, 0, rtlpriv->cfg->maps[EFUSE_HWSET_MAX_SIZE], efuse);
+	efuse_power_switch(hw, false, false);
+}
+
+static void efuse_read_data_case1(struct ieee80211_hw *hw, u16 *efuse_addr,
+				  u8 efuse_data, u8 offset, u8 *tmpdata,
+				  u8 *readstate)
+{
+	bool dataempty = true;
+	u8 hoffset;
+	u8 tmpidx;
+	u8 hworden;
+	u8 word_cnts;
+
+	hoffset = (efuse_data >> 4) & 0x0F;
+	hworden = efuse_data & 0x0F;
+	word_cnts = efuse_calculate_word_cnts(hworden);
+
+	if (hoffset == offset) {
+		for (tmpidx = 0; tmpidx < word_cnts * 2; tmpidx++) {
+			if (efuse_one_byte_read(hw, *efuse_addr + 1 + tmpidx,
+						&efuse_data)) {
+				tmpdata[tmpidx] = efuse_data;
+				if (efuse_data != 0xff)
+					dataempty = false;
+			}
+		}
+
+		if (!dataempty) {
+			*readstate = PG_STATE_DATA;
+		} else {
+			*efuse_addr = *efuse_addr + (word_cnts * 2) + 1;
+			*readstate = PG_STATE_HEADER;
+		}
+
+	} else {
+		*efuse_addr = *efuse_addr + (word_cnts * 2) + 1;
+		*readstate = PG_STATE_HEADER;
+	}
+}
+
+static int efuse_pg_packet_read(struct ieee80211_hw *hw, u8 offset, u8 *data)
+{
+	u8 readstate = PG_STATE_HEADER;
+
+	bool continual = true;
+
+	u8 efuse_data, word_cnts = 0;
+	u16 efuse_addr = 0;
+	u8 tmpdata[8];
+
+	if (!data)
+		return false;
+	if (offset > 15)
+		return false;
+
+	memset(data, 0xff, PGPKT_DATA_SIZE * sizeof(u8));
+	memset(tmpdata, 0xff, PGPKT_DATA_SIZE * sizeof(u8));
+
+	while (continual && (efuse_addr < EFUSE_MAX_SIZE)) {
+		if (readstate & PG_STATE_HEADER) {
+			if (efuse_one_byte_read(hw, efuse_addr, &efuse_data) &&
+			    (efuse_data != 0xFF))
+				efuse_read_data_case1(hw, &efuse_addr,
+						      efuse_data, offset,
+						      tmpdata, &readstate);
+			else
+				continual = false;
+		} else if (readstate & PG_STATE_DATA) {
+			efuse_word_enable_data_read(0, tmpdata, data);
+			efuse_addr = efuse_addr + (word_cnts * 2) + 1;
+			readstate = PG_STATE_HEADER;
+		}
+	}
+
+	if ((data[0] == 0xff) && (data[1] == 0xff) &&
+	    (data[2] == 0xff) && (data[3] == 0xff) &&
+	    (data[4] == 0xff) && (data[5] == 0xff) &&
+	    (data[6] == 0xff) && (data[7] == 0xff))
+		return false;
+	return true;
+}
+
+static void efuse_write_data_case1(struct ieee80211_hw *hw, u16 *efuse_addr,
+				   u8 efuse_data, u8 offset,
+				   int *continual, u8 *write_state,
+				   struct pgpkt_struct *target_pkt,
+				   int *repeat_times, int *result, u8 word_en)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct pgpkt_struct tmp_pkt;
+	int dataempty = true;
+	u8 originaldata[8 * sizeof(u8)];
+	u8 badworden = 0x0F;
+	u8 match_word_en, tmp_word_en;
+	u8 tmpindex;
+	u8 tmp_header = efuse_data;
+	u8 tmp_word_cnts;
+
+	tmp_pkt.offset = (tmp_header >> 4) & 0x0F;
+	tmp_pkt.word_en = tmp_header & 0x0F;
+	tmp_word_cnts = efuse_calculate_word_cnts(tmp_pkt.word_en);
+
+	if (tmp_pkt.offset != target_pkt->offset) {
+		*efuse_addr = *efuse_addr + (tmp_word_cnts * 2) + 1;
+		*write_state = PG_STATE_HEADER;
+	} else {
+		for (tmpindex = 0; tmpindex < (tmp_word_cnts * 2); tmpindex++) {
+			if (efuse_one_byte_read(hw,
+						(*efuse_addr + 1 + tmpindex),
+						&efuse_data) &&
+			    (efuse_data != 0xFF))
+				dataempty = false;
+		}
+
+		if (!dataempty) {
+			*efuse_addr = *efuse_addr + (tmp_word_cnts * 2) + 1;
+			*write_state = PG_STATE_HEADER;
+		} else {
+			match_word_en = 0x0F;
+			if (!((target_pkt->word_en & BIT(0)) |
+			    (tmp_pkt.word_en & BIT(0))))
+				match_word_en &= (~BIT(0));
+
+			if (!((target_pkt->word_en & BIT(1)) |
+			    (tmp_pkt.word_en & BIT(1))))
+				match_word_en &= (~BIT(1));
+
+			if (!((target_pkt->word_en & BIT(2)) |
+			    (tmp_pkt.word_en & BIT(2))))
+				match_word_en &= (~BIT(2));
+
+			if (!((target_pkt->word_en & BIT(3)) |
+			    (tmp_pkt.word_en & BIT(3))))
+				match_word_en &= (~BIT(3));
+
+			if ((match_word_en & 0x0F) != 0x0F) {
+				badworden =
+				  enable_efuse_data_write(hw,
+							  *efuse_addr + 1,
+							  tmp_pkt.word_en,
+							  target_pkt->data);
+
+				if (0x0F != (badworden & 0x0F))	{
+					u8 reorg_offset = offset;
+					u8 reorg_worden = badworden;
+
+					efuse_pg_packet_write(hw, reorg_offset,
+							      reorg_worden,
+							      originaldata);
+				}
+
+				tmp_word_en = 0x0F;
+				if ((target_pkt->word_en & BIT(0)) ^
+				    (match_word_en & BIT(0)))
+					tmp_word_en &= (~BIT(0));
+
+				if ((target_pkt->word_en & BIT(1)) ^
+				    (match_word_en & BIT(1)))
+					tmp_word_en &= (~BIT(1));
+
+				if ((target_pkt->word_en & BIT(2)) ^
+				    (match_word_en & BIT(2)))
+					tmp_word_en &= (~BIT(2));
+
+				if ((target_pkt->word_en & BIT(3)) ^
+				    (match_word_en & BIT(3)))
+					tmp_word_en &= (~BIT(3));
+
+				if ((tmp_word_en & 0x0F) != 0x0F) {
+					*efuse_addr =
+					    efuse_get_current_size(hw);
+					target_pkt->offset = offset;
+					target_pkt->word_en = tmp_word_en;
+				} else {
+					*continual = false;
+				}
+				*write_state = PG_STATE_HEADER;
+				*repeat_times += 1;
+				if (*repeat_times > EFUSE_REPEAT_THRESHOLD_) {
+					*continual = false;
+					*result = false;
+				}
+			} else {
+				*efuse_addr += (2 * tmp_word_cnts) + 1;
+				target_pkt->offset = offset;
+				target_pkt->word_en = word_en;
+				*write_state = PG_STATE_HEADER;
+			}
+		}
+	}
+	RTPRINT(rtlpriv, FEEPROM, EFUSE_PG, "efuse PG_STATE_HEADER-1\n");
+}
+
+static void efuse_write_data_case2(struct ieee80211_hw *hw, u16 *efuse_addr,
+				   int *continual, u8 *write_state,
+				   struct pgpkt_struct target_pkt,
+				   int *repeat_times, int *result)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct pgpkt_struct tmp_pkt;
+	u8 pg_header;
+	u8 tmp_header;
+	u8 originaldata[8 * sizeof(u8)];
+	u8 tmp_word_cnts;
+	u8 badworden = 0x0F;
+
+	pg_header = ((target_pkt.offset << 4) & 0xf0) | target_pkt.word_en;
+	efuse_one_byte_write(hw, *efuse_addr, pg_header);
+	efuse_one_byte_read(hw, *efuse_addr, &tmp_header);
+
+	if (tmp_header == pg_header) {
+		*write_state = PG_STATE_DATA;
+	} else if (tmp_header == 0xFF) {
+		*write_state = PG_STATE_HEADER;
+		*repeat_times += 1;
+		if (*repeat_times > EFUSE_REPEAT_THRESHOLD_) {
+			*continual = false;
+			*result = false;
+		}
+	} else {
+		tmp_pkt.offset = (tmp_header >> 4) & 0x0F;
+		tmp_pkt.word_en = tmp_header & 0x0F;
+
+		tmp_word_cnts = efuse_calculate_word_cnts(tmp_pkt.word_en);
+
+		memset(originaldata, 0xff,  8 * sizeof(u8));
+
+		if (efuse_pg_packet_read(hw, tmp_pkt.offset, originaldata)) {
+			badworden = enable_efuse_data_write(hw,
+							    *efuse_addr + 1,
+							    tmp_pkt.word_en,
+							    originaldata);
+
+			if (0x0F != (badworden & 0x0F)) {
+				u8 reorg_offset = tmp_pkt.offset;
+				u8 reorg_worden = badworden;
+
+				efuse_pg_packet_write(hw, reorg_offset,
+						      reorg_worden,
+						      originaldata);
+				*efuse_addr = efuse_get_current_size(hw);
+			} else {
+				*efuse_addr = *efuse_addr +
+					      (tmp_word_cnts * 2) + 1;
+			}
+		} else {
+			*efuse_addr = *efuse_addr + (tmp_word_cnts * 2) + 1;
+		}
+
+		*write_state = PG_STATE_HEADER;
+		*repeat_times += 1;
+		if (*repeat_times > EFUSE_REPEAT_THRESHOLD_) {
+			*continual = false;
+			*result = false;
+		}
+
+		RTPRINT(rtlpriv, FEEPROM, EFUSE_PG,
+			"efuse PG_STATE_HEADER-2\n");
+	}
+}
+
+static int efuse_pg_packet_write(struct ieee80211_hw *hw,
+				 u8 offset, u8 word_en, u8 *data)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct pgpkt_struct target_pkt;
+	u8 write_state = PG_STATE_HEADER;
+	int continual = true, dataempty = true, result = true;
+	u16 efuse_addr = 0;
+	u8 efuse_data;
+	u8 target_word_cnts = 0;
+	u8 badworden = 0x0F;
+	static int repeat_times;
+
+	if (efuse_get_current_size(hw) >= (EFUSE_MAX_SIZE -
+		rtlpriv->cfg->maps[EFUSE_OOB_PROTECT_BYTES_LEN])) {
+		RTPRINT(rtlpriv, FEEPROM, EFUSE_PG,
+			"%s error\n", __func__);
+		return false;
+	}
+
+	target_pkt.offset = offset;
+	target_pkt.word_en = word_en;
+
+	memset(target_pkt.data, 0xFF,  8 * sizeof(u8));
+
+	efuse_word_enable_data_read(word_en, data, target_pkt.data);
+	target_word_cnts = efuse_calculate_word_cnts(target_pkt.word_en);
+
+	RTPRINT(rtlpriv, FEEPROM, EFUSE_PG, "efuse Power ON\n");
+
+	while (continual && (efuse_addr < (EFUSE_MAX_SIZE -
+	       rtlpriv->cfg->maps[EFUSE_OOB_PROTECT_BYTES_LEN]))) {
+		if (write_state == PG_STATE_HEADER) {
+			dataempty = true;
+			badworden = 0x0F;
+			RTPRINT(rtlpriv, FEEPROM, EFUSE_PG,
+				"efuse PG_STATE_HEADER\n");
+
+			if (efuse_one_byte_read(hw, efuse_addr, &efuse_data) &&
+			    (efuse_data != 0xFF))
+				efuse_write_data_case1(hw, &efuse_addr,
+						       efuse_data, offset,
+						       &continual,
+						       &write_state,
+						       &target_pkt,
+						       &repeat_times, &result,
+						       word_en);
+			else
+				efuse_write_data_case2(hw, &efuse_addr,
+						       &continual,
+						       &write_state,
+						       target_pkt,
+						       &repeat_times,
+						       &result);
+
+		} else if (write_state == PG_STATE_DATA) {
+			RTPRINT(rtlpriv, FEEPROM, EFUSE_PG,
+				"efuse PG_STATE_DATA\n");
+			badworden = 0x0f;
+			badworden =
+			    enable_efuse_data_write(hw, efuse_addr + 1,
+						    target_pkt.word_en,
+						    target_pkt.data);
+
+			if ((badworden & 0x0F) == 0x0F) {
+				continual = false;
+			} else {
+				efuse_addr =
+				    efuse_addr + (2 * target_word_cnts) + 1;
+
+				target_pkt.offset = offset;
+				target_pkt.word_en = badworden;
+				target_word_cnts =
+				    efuse_calculate_word_cnts(target_pkt.word_en);
+				write_state = PG_STATE_HEADER;
+				repeat_times++;
+				if (repeat_times > EFUSE_REPEAT_THRESHOLD_) {
+					continual = false;
+					result = false;
+				}
+				RTPRINT(rtlpriv, FEEPROM, EFUSE_PG,
+					"efuse PG_STATE_HEADER-3\n");
+			}
+		}
+	}
+
+	if (efuse_addr >= (EFUSE_MAX_SIZE -
+		rtlpriv->cfg->maps[EFUSE_OOB_PROTECT_BYTES_LEN])) {
+		RT_TRACE(rtlpriv, COMP_EFUSE, DBG_LOUD,
+			 "efuse_addr(%#x) Out of size!!\n", efuse_addr);
+	}
+
+	return true;
+}
+
+static void efuse_word_enable_data_read(u8 word_en, u8 *sourdata,
+					u8 *targetdata)
+{
+	if (!(word_en & BIT(0))) {
+		targetdata[0] = sourdata[0];
+		targetdata[1] = sourdata[1];
+	}
+
+	if (!(word_en & BIT(1))) {
+		targetdata[2] = sourdata[2];
+		targetdata[3] = sourdata[3];
+	}
+
+	if (!(word_en & BIT(2))) {
+		targetdata[4] = sourdata[4];
+		targetdata[5] = sourdata[5];
+	}
+
+	if (!(word_en & BIT(3))) {
+		targetdata[6] = sourdata[6];
+		targetdata[7] = sourdata[7];
+	}
+}
+
+static u8 enable_efuse_data_write(struct ieee80211_hw *hw,
+				  u16 efuse_addr, u8 word_en, u8 *data)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u16 tmpaddr;
+	u16 start_addr = efuse_addr;
+	u8 badworden = 0x0F;
+	u8 tmpdata[8];
+
+	memset(tmpdata, 0xff, PGPKT_DATA_SIZE);
+	RT_TRACE(rtlpriv, COMP_EFUSE, DBG_LOUD,
+		 "word_en = %x efuse_addr=%x\n", word_en, efuse_addr);
+
+	if (!(word_en & BIT(0))) {
+		tmpaddr = start_addr;
+		efuse_one_byte_write(hw, start_addr++, data[0]);
+		efuse_one_byte_write(hw, start_addr++, data[1]);
+
+		efuse_one_byte_read(hw, tmpaddr, &tmpdata[0]);
+		efuse_one_byte_read(hw, tmpaddr + 1, &tmpdata[1]);
+		if ((data[0] != tmpdata[0]) || (data[1] != tmpdata[1]))
+			badworden &= (~BIT(0));
+	}
+
+	if (!(word_en & BIT(1))) {
+		tmpaddr = start_addr;
+		efuse_one_byte_write(hw, start_addr++, data[2]);
+		efuse_one_byte_write(hw, start_addr++, data[3]);
+
+		efuse_one_byte_read(hw, tmpaddr, &tmpdata[2]);
+		efuse_one_byte_read(hw, tmpaddr + 1, &tmpdata[3]);
+		if ((data[2] != tmpdata[2]) || (data[3] != tmpdata[3]))
+			badworden &= (~BIT(1));
+	}
+
+	if (!(word_en & BIT(2))) {
+		tmpaddr = start_addr;
+		efuse_one_byte_write(hw, start_addr++, data[4]);
+		efuse_one_byte_write(hw, start_addr++, data[5]);
+
+		efuse_one_byte_read(hw, tmpaddr, &tmpdata[4]);
+		efuse_one_byte_read(hw, tmpaddr + 1, &tmpdata[5]);
+		if ((data[4] != tmpdata[4]) || (data[5] != tmpdata[5]))
+			badworden &= (~BIT(2));
+	}
+
+	if (!(word_en & BIT(3))) {
+		tmpaddr = start_addr;
+		efuse_one_byte_write(hw, start_addr++, data[6]);
+		efuse_one_byte_write(hw, start_addr++, data[7]);
+
+		efuse_one_byte_read(hw, tmpaddr, &tmpdata[6]);
+		efuse_one_byte_read(hw, tmpaddr + 1, &tmpdata[7]);
+		if ((data[6] != tmpdata[6]) || (data[7] != tmpdata[7]))
+			badworden &= (~BIT(3));
+	}
+
+	return badworden;
+}
+
+void efuse_power_switch(struct ieee80211_hw *hw, u8 write, u8 pwrstate)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	u8 tempval;
+	u16 tmpv16;
+
+	if (pwrstate && (rtlhal->hw_type != HARDWARE_TYPE_RTL8192SE)) {
+		if (rtlhal->hw_type != HARDWARE_TYPE_RTL8192CE &&
+		    rtlhal->hw_type != HARDWARE_TYPE_RTL8192DE) {
+			rtl_write_byte(rtlpriv,
+				       rtlpriv->cfg->maps[EFUSE_ACCESS], 0x69);
+		} else {
+			tmpv16 =
+			  rtl_read_word(rtlpriv,
+					rtlpriv->cfg->maps[SYS_ISO_CTRL]);
+			if (!(tmpv16 & rtlpriv->cfg->maps[EFUSE_PWC_EV12V])) {
+				tmpv16 |= rtlpriv->cfg->maps[EFUSE_PWC_EV12V];
+				rtl_write_word(rtlpriv,
+					       rtlpriv->cfg->maps[SYS_ISO_CTRL],
+					       tmpv16);
+			}
+		}
+		tmpv16 = rtl_read_word(rtlpriv,
+				       rtlpriv->cfg->maps[SYS_FUNC_EN]);
+		if (!(tmpv16 & rtlpriv->cfg->maps[EFUSE_FEN_ELDR])) {
+			tmpv16 |= rtlpriv->cfg->maps[EFUSE_FEN_ELDR];
+			rtl_write_word(rtlpriv,
+				       rtlpriv->cfg->maps[SYS_FUNC_EN], tmpv16);
+		}
+
+		tmpv16 = rtl_read_word(rtlpriv, rtlpriv->cfg->maps[SYS_CLK]);
+		if ((!(tmpv16 & rtlpriv->cfg->maps[EFUSE_LOADER_CLK_EN])) ||
+		    (!(tmpv16 & rtlpriv->cfg->maps[EFUSE_ANA8M]))) {
+			tmpv16 |= (rtlpriv->cfg->maps[EFUSE_LOADER_CLK_EN] |
+				   rtlpriv->cfg->maps[EFUSE_ANA8M]);
+			rtl_write_word(rtlpriv,
+				       rtlpriv->cfg->maps[SYS_CLK], tmpv16);
+		}
+	}
+
+	if (pwrstate) {
+		if (write) {
+			tempval = rtl_read_byte(rtlpriv,
+						rtlpriv->cfg->maps[EFUSE_TEST] +
+						3);
+
+			if (rtlhal->hw_type == HARDWARE_TYPE_RTL8812AE) {
+				tempval &= ~(BIT(3) | BIT(4) | BIT(5) | BIT(6));
+				tempval |= (VOLTAGE_V25 << 3);
+			} else if (rtlhal->hw_type != HARDWARE_TYPE_RTL8192SE) {
+				tempval &= 0x0F;
+				tempval |= (VOLTAGE_V25 << 4);
+			}
+
+			rtl_write_byte(rtlpriv,
+				       rtlpriv->cfg->maps[EFUSE_TEST] + 3,
+				       (tempval | 0x80));
+		}
+
+		if (rtlhal->hw_type == HARDWARE_TYPE_RTL8192SE) {
+			rtl_write_byte(rtlpriv, rtlpriv->cfg->maps[EFUSE_CLK],
+				       0x03);
+		}
+	} else {
+		if (rtlhal->hw_type != HARDWARE_TYPE_RTL8192CE &&
+		    rtlhal->hw_type != HARDWARE_TYPE_RTL8192DE)
+			rtl_write_byte(rtlpriv,
+				       rtlpriv->cfg->maps[EFUSE_ACCESS], 0);
+
+		if (write) {
+			tempval = rtl_read_byte(rtlpriv,
+						rtlpriv->cfg->maps[EFUSE_TEST] +
+						3);
+			rtl_write_byte(rtlpriv,
+				       rtlpriv->cfg->maps[EFUSE_TEST] + 3,
+				       (tempval & 0x7F));
+		}
+
+		if (rtlhal->hw_type == HARDWARE_TYPE_RTL8192SE) {
+			rtl_write_byte(rtlpriv, rtlpriv->cfg->maps[EFUSE_CLK],
+				       0x02);
+		}
+	}
+}
+
+static u16 efuse_get_current_size(struct ieee80211_hw *hw)
+{
+	int continual = true;
+	u16 efuse_addr = 0;
+	u8 hoffset, hworden;
+	u8 efuse_data, word_cnts;
+
+	while (continual && efuse_one_byte_read(hw, efuse_addr, &efuse_data) &&
+	       (efuse_addr < EFUSE_MAX_SIZE)) {
+		if (efuse_data != 0xFF) {
+			hoffset = (efuse_data >> 4) & 0x0F;
+			hworden = efuse_data & 0x0F;
+			word_cnts = efuse_calculate_word_cnts(hworden);
+			efuse_addr = efuse_addr + (word_cnts * 2) + 1;
+		} else {
+			continual = false;
+		}
+	}
+
+	return efuse_addr;
+}
+
+static u8 efuse_calculate_word_cnts(u8 word_en)
+{
+	u8 word_cnts = 0;
+
+	if (!(word_en & BIT(0)))
+		word_cnts++;
+	if (!(word_en & BIT(1)))
+		word_cnts++;
+	if (!(word_en & BIT(2)))
+		word_cnts++;
+	if (!(word_en & BIT(3)))
+		word_cnts++;
+	return word_cnts;
+}
+
+int rtl_get_hwinfo(struct ieee80211_hw *hw, struct rtl_priv *rtlpriv,
+		   int max_size, u8 *hwinfo, int *params)
+{
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
+	struct rtl_pci_priv *rtlpcipriv = rtl_pcipriv(hw);
+	struct device *dev = &rtlpcipriv->dev.pdev->dev;
+	u16 eeprom_id;
+	u16 i, usvalue;
+
+	switch (rtlefuse->epromtype) {
+	case EEPROM_BOOT_EFUSE:
+		rtl_efuse_shadow_map_update(hw);
+		break;
+
+	case EEPROM_93C46:
+		pr_err("RTL8XXX did not boot from eeprom, check it !!\n");
+		return 1;
+
+	default:
+		dev_warn(dev, "no efuse data\n");
+		return 1;
+	}
+
+	memcpy(hwinfo, &rtlefuse->efuse_map[EFUSE_INIT_MAP][0], max_size);
+
+	RT_PRINT_DATA(rtlpriv, COMP_INIT, DBG_DMESG, "MAP",
+		      hwinfo, max_size);
+
+	eeprom_id = *((u16 *)&hwinfo[0]);
+	if (eeprom_id != params[0]) {
+		RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING,
+			 "EEPROM ID(%#x) is invalid!!\n", eeprom_id);
+		rtlefuse->autoload_failflag = true;
+	} else {
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD, "Autoload OK\n");
+		rtlefuse->autoload_failflag = false;
+	}
+
+	if (rtlefuse->autoload_failflag)
+		return 1;
+
+	rtlefuse->eeprom_vid = *(u16 *)&hwinfo[params[1]];
+	rtlefuse->eeprom_did = *(u16 *)&hwinfo[params[2]];
+	rtlefuse->eeprom_svid = *(u16 *)&hwinfo[params[3]];
+	rtlefuse->eeprom_smid = *(u16 *)&hwinfo[params[4]];
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+		 "EEPROMId = 0x%4x\n", eeprom_id);
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+		 "EEPROM VID = 0x%4x\n", rtlefuse->eeprom_vid);
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+		 "EEPROM DID = 0x%4x\n", rtlefuse->eeprom_did);
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+		 "EEPROM SVID = 0x%4x\n", rtlefuse->eeprom_svid);
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+		 "EEPROM SMID = 0x%4x\n", rtlefuse->eeprom_smid);
+
+	for (i = 0; i < 6; i += 2) {
+		usvalue = *(u16 *)&hwinfo[params[5] + i];
+		*((u16 *)(&rtlefuse->dev_addr[i])) = usvalue;
+	}
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "%pM\n", rtlefuse->dev_addr);
+
+	rtlefuse->eeprom_channelplan = *&hwinfo[params[6]];
+	rtlefuse->eeprom_version = *(u16 *)&hwinfo[params[7]];
+	rtlefuse->txpwr_fromeprom = true;
+	rtlefuse->eeprom_oemid = *&hwinfo[params[8]];
+
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+		 "EEPROM Customer ID: 0x%2x\n", rtlefuse->eeprom_oemid);
+
+	/* set channel plan to world wide 13 */
+	rtlefuse->channel_plan = params[9];
+
+	return 0;
+}
+
+void rtl_fw_block_write(struct ieee80211_hw *hw, const u8 *buffer, u32 size)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 *pu4byteptr = (u8 *)buffer;
+	u32 i;
+
+	for (i = 0; i < size; i++)
+		rtl_write_byte(rtlpriv, (START_ADDRESS + i), *(pu4byteptr + i));
+}
+
+void rtl_fw_page_write(struct ieee80211_hw *hw, u32 page, const u8 *buffer,
+		       u32 size)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 value8;
+	u8 u8page = (u8)(page & 0x07);
+
+	value8 = (rtl_read_byte(rtlpriv, REG_MCUFWDL + 2) & 0xF8) | u8page;
+
+	rtl_write_byte(rtlpriv, (REG_MCUFWDL + 2), value8);
+	rtl_fw_block_write(hw, buffer, size);
+}
+
+void rtl_fill_dummy(u8 *pfwbuf, u32 *pfwlen)
+{
+	u32 fwlen = *pfwlen;
+	u8 remain = (u8)(fwlen % 4);
+
+	remain = (remain == 0) ? 0 : (4 - remain);
+
+	while (remain > 0) {
+		pfwbuf[fwlen] = 0;
+		fwlen++;
+		remain--;
+	}
+
+	*pfwlen = fwlen;
+}
diff --git a/drivers/staging/rtlwifi/efuse.h b/drivers/staging/rtlwifi/efuse.h
new file mode 100644
index 000000000000..0a23305b0b7a
--- /dev/null
+++ b/drivers/staging/rtlwifi/efuse.h
@@ -0,0 +1,120 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2012  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __RTL_EFUSE_H_
+#define __RTL_EFUSE_H_
+
+#define EFUSE_IC_ID_OFFSET		506
+
+#define EFUSE_MAX_WORD_UNIT		4
+
+#define EFUSE_INIT_MAP			0
+#define EFUSE_MODIFY_MAP		1
+
+#define PG_STATE_HEADER			0x01
+#define PG_STATE_WORD_0			0x02
+#define PG_STATE_WORD_1			0x04
+#define PG_STATE_WORD_2			0x08
+#define PG_STATE_WORD_3			0x10
+#define PG_STATE_DATA			0x20
+
+#define EFUSE_REPEAT_THRESHOLD_		3
+#define EFUSE_ERROE_HANDLE		1
+
+struct efuse_map {
+	u8 offset;
+	u8 word_start;
+	u8 byte_start;
+	u8 byte_cnts;
+};
+
+struct pgpkt_struct {
+	u8 offset;
+	u8 word_en;
+	u8 data[8];
+};
+
+enum efuse_data_item {
+	EFUSE_CHIP_ID = 0,
+	EFUSE_LDO_SETTING,
+	EFUSE_CLK_SETTING,
+	EFUSE_SDIO_SETTING,
+	EFUSE_CCCR,
+	EFUSE_SDIO_MODE,
+	EFUSE_OCR,
+	EFUSE_F0CIS,
+	EFUSE_F1CIS,
+	EFUSE_MAC_ADDR,
+	EFUSE_EEPROM_VER,
+	EFUSE_CHAN_PLAN,
+	EFUSE_TXPW_TAB
+};
+
+enum {
+	VOLTAGE_V25 = 0x03,
+	LDOE25_SHIFT = 28,
+};
+
+struct efuse_priv {
+	u8 id[2];
+	u8 ldo_setting[2];
+	u8 clk_setting[2];
+	u8 cccr;
+	u8 sdio_mode;
+	u8 ocr[3];
+	u8 cis0[17];
+	u8 cis1[48];
+	u8 mac_addr[6];
+	u8 eeprom_verno;
+	u8 channel_plan;
+	u8 tx_power_b[14];
+	u8 tx_power_g[14];
+};
+
+void read_efuse_byte(struct ieee80211_hw *hw, u16 _offset, u8 *pbuf);
+void efuse_initialize(struct ieee80211_hw *hw);
+u8 efuse_read_1byte(struct ieee80211_hw *hw, u16 address);
+int efuse_one_byte_read(struct ieee80211_hw *hw, u16 addr, u8 *data);
+void efuse_write_1byte(struct ieee80211_hw *hw, u16 address, u8 value);
+void read_efuse(struct ieee80211_hw *hw, u16 _offset,
+		u16 _size_byte, u8 *pbuf);
+void efuse_shadow_read(struct ieee80211_hw *hw, u8 type,
+		       u16 offset, u32 *value);
+void efuse_shadow_write(struct ieee80211_hw *hw, u8 type,
+			u16 offset, u32 value);
+bool efuse_shadow_update(struct ieee80211_hw *hw);
+bool efuse_shadow_update_chk(struct ieee80211_hw *hw);
+void rtl_efuse_shadow_map_update(struct ieee80211_hw *hw);
+void efuse_force_write_vendor_id(struct ieee80211_hw *hw);
+void efuse_re_pg_section(struct ieee80211_hw *hw, u8 section_idx);
+void efuse_power_switch(struct ieee80211_hw *hw, u8 write, u8 pwrstate);
+int rtl_get_hwinfo(struct ieee80211_hw *hw, struct rtl_priv *rtlpriv,
+		   int max_size, u8 *hwinfo, int *params);
+void rtl_fill_dummy(u8 *pfwbuf, u32 *pfwlen);
+void rtl_fw_page_write(struct ieee80211_hw *hw, u32 page, const u8 *buffer,
+		       u32 size);
+void rtl_fw_block_write(struct ieee80211_hw *hw, const u8 *buffer, u32 size);
+
+#endif
diff --git a/drivers/staging/rtlwifi/halmac/halmac_2_platform.h b/drivers/staging/rtlwifi/halmac/halmac_2_platform.h
new file mode 100644
index 000000000000..26e60cb873eb
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_2_platform.h
@@ -0,0 +1,52 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HALMAC_2_PLATFORM_H_
+#define _HALMAC_2_PLATFORM_H_
+
+#include "../wifi.h"
+#include <asm/byteorder.h>
+
+#define HALMAC_PLATFORM_LITTLE_ENDIAN 1
+#define HALMAC_PLATFORM_BIG_ENDIAN 0
+
+/* Note : Named HALMAC_PLATFORM_LITTLE_ENDIAN / HALMAC_PLATFORM_BIG_ENDIAN
+ * is not mandatory. But Little endian must be '1'. Big endian must be '0'
+ */
+#if defined(__LITTLE_ENDIAN)
+#define HALMAC_SYSTEM_ENDIAN HALMAC_PLATFORM_LITTLE_ENDIAN
+#elif defined(__BIG_ENDIAN)
+#define HALMAC_SYSTEM_ENDIAN HALMAC_PLATFORM_BIG_ENDIAN
+#else
+#error
+#endif
+
+/* define the Platform SDIO Bus CLK */
+#define PLATFORM_SD_CLK 50000000 /*50MHz*/
+
+/* define the Rx FIFO expanding mode packet size unit for 8821C and 8822B */
+/* Should be 8 Byte alignment */
+#define HALMAC_RX_FIFO_EXPANDING_MODE_PKT_SIZE 16 /*Bytes*/
+
+#endif /* _HALMAC_2_PLATFORM_H_ */
diff --git a/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_8822b_cfg.h b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_8822b_cfg.h
new file mode 100644
index 000000000000..04e44aed9b45
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_8822b_cfg.h
@@ -0,0 +1,132 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HALMAC_8822B_CFG_H_
+#define _HALMAC_8822B_CFG_H_
+
+#include "halmac_8822b_pwr_seq.h"
+#include "halmac_api_8822b.h"
+#include "halmac_api_8822b_usb.h"
+#include "halmac_api_8822b_sdio.h"
+#include "halmac_api_8822b_pcie.h"
+#include "../../halmac_bit2.h"
+#include "../../halmac_reg2.h"
+#include "../../halmac_api.h"
+
+#define HALMAC_TX_FIFO_SIZE_8822B 262144 /* 256k */
+#define HALMAC_TX_FIFO_SIZE_LA_8822B 131072 /* 128k */
+#define HALMAC_RX_FIFO_SIZE_8822B 24576 /* 24k */
+#define HALMAC_TX_PAGE_SIZE_8822B 128 /* PageSize 128Byte */
+#define HALMAC_TX_ALIGN_SIZE_8822B 8
+#define HALMAC_TX_PAGE_SIZE_2_POWER_8822B 7 /* 128 = 2^7 */
+#define HALMAC_SECURITY_CAM_ENTRY_NUM_8822B 64 /* CAM Entry size */
+#define HALMAC_TX_AGG_ALIGNMENT_SIZE_8822B 8
+#define HALMAC_TX_DESC_SIZE_8822B 48
+#define HALMAC_RX_DESC_SIZE_8822B 24
+#define HALMAC_RX_DESC_DUMMY_SIZE_MAX_8822B 120
+#define HALMAC_C2H_PKT_BUF_8822B 256
+#define HALMAC_RX_FIFO_EXPANDING_MODE_PKT_SIZE_MAX_8822B 80 /* align 8 Byte*/
+#define HALMAC_RX_FIFO_EXPANDING_UNIT_8822B                                    \
+	(HALMAC_RX_DESC_SIZE_8822B + HALMAC_RX_DESC_DUMMY_SIZE_MAX_8822B +     \
+	 HALMAC_RX_FIFO_EXPANDING_MODE_PKT_SIZE) /* align 8 Byte*/
+#define HALMAC_RX_FIFO_EXPANDING_UNIT_MAX_8822B                                \
+	(HALMAC_RX_DESC_SIZE_8822B + HALMAC_RX_DESC_DUMMY_SIZE_MAX_8822B +     \
+	 HALMAC_RX_FIFO_EXPANDING_MODE_PKT_SIZE_MAX_8822B) /* align 8 Byte*/
+
+#define HALMAC_TX_FIFO_SIZE_EX_1_BLK_8822B 196608 /* 192k */
+#define HALMAC_RX_FIFO_SIZE_EX_1_BLK_8822B                                     \
+	((((HALMAC_RX_FIFO_EXPANDING_UNIT_8822B << 8) - 1) >> 10)              \
+	 << 10) /* < 56k*/
+#define HALMAC_RX_FIFO_SIZE_EX_1_BLK_MAX_8822B                                 \
+	((((HALMAC_RX_FIFO_EXPANDING_UNIT_MAX_8822B << 8) - 1) >> 10)          \
+	 << 10) /* 55k*/
+#define HALMAC_TX_FIFO_SIZE_EX_2_BLK_8822B 131072 /* 128k */
+#define HALMAC_RX_FIFO_SIZE_EX_2_BLK_8822B 155648 /* 152k */
+#define HALMAC_TX_FIFO_SIZE_EX_3_BLK_8822B 65536 /* 64k */
+#define HALMAC_RX_FIFO_SIZE_EX_3_BLK_8822B 221184 /* 216k */
+
+/* TXFIFO LAYOUT
+ * HIGH_QUEUE
+ * NORMAL_QUEUE
+ * LOW_QUEUE
+ * EXTRA_QUEUE
+ * PUBLIC_QUEUE -- decided after all other queue are defined
+ * GAP_QUEUE -- Used to separate AC queue and Rsvd page
+ *
+ * RSVD_DRIVER -- Driver used rsvd page area
+ * RSVD_H2C_EXTRAINFO -- Extra Information for h2c
+ * RSVD_H2C_QUEUE -- h2c queue in rsvd page
+ * RSVD_CPU_INSTRUCTION -- extend fw code
+ * RSVD_FW_TXBUFF -- fw used this area to send packet
+ *
+ * Symbol: HALMAC_MODE_QUEUE_UNIT_CHIP, ex: HALMAC_LB_2BULKOUT_FWCMD_PGNUM_8822B
+ */
+#define HALMAC_EXTRA_INFO_BUFF_SIZE_FULL_FIFO_8822B                            \
+	16384 /*16K, only used in init case*/
+
+#define HALMAC_RSVD_DRV_PGNUM_8822B 16 /*2048*/
+#define HALMAC_RSVD_H2C_EXTRAINFO_PGNUM_8822B 32 /*4096*/
+#define HALMAC_RSVD_H2C_QUEUE_PGNUM_8822B 8 /*1024*/
+#define HALMAC_RSVD_CPU_INSTRUCTION_PGNUM_8822B 0 /*0*/
+#define HALMAC_RSVD_FW_TXBUFF_PGNUM_8822B 4 /*512*/
+
+#define HALMAC_EFUSE_SIZE_8822B 1024 /* 0x400 */
+#define HALMAC_BT_EFUSE_SIZE_8822B 128 /* 0x80 */
+#define HALMAC_EEPROM_SIZE_8822B 0x300
+#define HALMAC_CR_TRX_ENABLE_8822B                                             \
+	(BIT_HCI_TXDMA_EN | BIT_HCI_RXDMA_EN | BIT_TXDMA_EN | BIT_RXDMA_EN |   \
+	 BIT_PROTOCOL_EN | BIT_SCHEDULE_EN | BIT_MACTXEN | BIT_MACRXEN)
+
+#define HALMAC_BLK_DESC_NUM_8822B 0x3 /* Only for USB */
+
+/* AMPDU max time (unit : 32us) */
+#define HALMAC_AMPDU_MAX_TIME_8822B 0x70
+
+/* Protect mode control */
+#define HALMAC_PROT_RTS_LEN_TH_8822B 0xFF
+#define HALMAC_PROT_RTS_TX_TIME_TH_8822B 0x08
+#define HALMAC_PROT_MAX_AGG_PKT_LIMIT_8822B 0x20
+#define HALMAC_PROT_RTS_MAX_AGG_PKT_LIMIT_8822B 0x20
+
+/* Fast EDCA setting */
+#define HALMAC_FAST_EDCA_VO_TH_8822B 0x06
+#define HALMAC_FAST_EDCA_VI_TH_8822B 0x06
+#define HALMAC_FAST_EDCA_BE_TH_8822B 0x06
+#define HALMAC_FAST_EDCA_BK_TH_8822B 0x06
+
+/* BAR setting */
+#define HALMAC_BAR_RETRY_LIMIT_8822B 0x01
+#define HALMAC_RA_TRY_RATE_AGG_LIMIT_8822B 0x08
+
+enum halmac_normal_rxagg_th_to_8822b {
+	HALMAC_NORMAL_RXAGG_THRESHOLD_8822B = 0xFF,
+	HALMAC_NORMAL_RXAGG_TIMEOUT_8822B = 0x01,
+};
+
+enum halmac_loopback_rxagg_th_to_8822b {
+	HALMAC_LOOPBACK_RXAGG_THRESHOLD_8822B = 0xFF,
+	HALMAC_LOOPBACK_RXAGG_TIMEOUT_8822B = 0x01,
+};
+
+#endif
diff --git a/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_8822b_phy.c b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_8822b_phy.c
new file mode 100644
index 000000000000..b2a5aed75dca
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_8822b_phy.c
@@ -0,0 +1,106 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#include "../halmac_88xx_cfg.h"
+#include "halmac_8822b_cfg.h"
+
+/**
+ * ============ip sel item list============
+ * HALMAC_IP_SEL_INTF_PHY
+ *	USB2 : usb2 phy, 1byte value
+ *	USB3 : usb3 phy, 2byte value
+ *	PCIE1 : pcie gen1 mdio, 2byte value
+ *	PCIE2 : pcie gen2 mdio, 2byte value
+ * HALMAC_IP_SEL_MAC
+ *	USB2, USB3, PCIE1, PCIE2 : mac ip, 1byte value
+ * HALMAC_IP_SEL_PCIE_DBI
+ *	USB2 USB3 : none
+ *	PCIE1, PCIE2 : pcie dbi, 1byte value
+ */
+
+struct halmac_intf_phy_para_ HALMAC_RTL8822B_USB2_PHY[] = {
+	/* {offset, value, ip sel, cut mask, platform mask} */
+	{0xFFFF, 0x00, HALMAC_IP_SEL_INTF_PHY, HALMAC_INTF_PHY_CUT_ALL,
+	 HALMAC_INTF_PHY_PLATFORM_ALL},
+};
+
+struct halmac_intf_phy_para_ HALMAC_RTL8822B_USB3_PHY[] = {
+	/* {offset, value, ip sel, cut mask, platform mask} */
+	{0x0001, 0xA841, HALMAC_IP_SEL_INTF_PHY, HALMAC_INTF_PHY_CUT_D,
+	 HALMAC_INTF_PHY_PLATFORM_ALL},
+	{0xFFFF, 0x0000, HALMAC_IP_SEL_INTF_PHY, HALMAC_INTF_PHY_CUT_ALL,
+	 HALMAC_INTF_PHY_PLATFORM_ALL},
+};
+
+struct halmac_intf_phy_para_ HALMAC_RTL8822B_PCIE_PHY_GEN1[] = {
+	/* {offset, value, ip sel, cut mask, platform mask} */
+	{0x0001, 0xA841, HALMAC_IP_SEL_INTF_PHY, HALMAC_INTF_PHY_CUT_C,
+	 HALMAC_INTF_PHY_PLATFORM_ALL},
+	{0x0002, 0x60C6, HALMAC_IP_SEL_INTF_PHY, HALMAC_INTF_PHY_CUT_C,
+	 HALMAC_INTF_PHY_PLATFORM_ALL},
+	{0x0008, 0x3596, HALMAC_IP_SEL_INTF_PHY, HALMAC_INTF_PHY_CUT_C,
+	 HALMAC_INTF_PHY_PLATFORM_ALL},
+	{0x0009, 0x321C, HALMAC_IP_SEL_INTF_PHY, HALMAC_INTF_PHY_CUT_C,
+	 HALMAC_INTF_PHY_PLATFORM_ALL},
+	{0x000A, 0x9623, HALMAC_IP_SEL_INTF_PHY, HALMAC_INTF_PHY_CUT_C,
+	 HALMAC_INTF_PHY_PLATFORM_ALL},
+	{0x0020, 0x94FF, HALMAC_IP_SEL_INTF_PHY, HALMAC_INTF_PHY_CUT_C,
+	 HALMAC_INTF_PHY_PLATFORM_ALL},
+	{0x0021, 0xFFCF, HALMAC_IP_SEL_INTF_PHY, HALMAC_INTF_PHY_CUT_C,
+	 HALMAC_INTF_PHY_PLATFORM_ALL},
+	{0x0026, 0xC006, HALMAC_IP_SEL_INTF_PHY, HALMAC_INTF_PHY_CUT_C,
+	 HALMAC_INTF_PHY_PLATFORM_ALL},
+	{0x0029, 0xFF0E, HALMAC_IP_SEL_INTF_PHY, HALMAC_INTF_PHY_CUT_C,
+	 HALMAC_INTF_PHY_PLATFORM_ALL},
+	{0x002A, 0x1840, HALMAC_IP_SEL_INTF_PHY, HALMAC_INTF_PHY_CUT_C,
+	 HALMAC_INTF_PHY_PLATFORM_ALL},
+	{0xFFFF, 0x0000, HALMAC_IP_SEL_INTF_PHY, HALMAC_INTF_PHY_CUT_ALL,
+	 HALMAC_INTF_PHY_PLATFORM_ALL},
+};
+
+struct halmac_intf_phy_para_ HALMAC_RTL8822B_PCIE_PHY_GEN2[] = {
+	/* {offset, value, ip sel, cut mask, platform mask} */
+	{0x0001, 0xA841, HALMAC_IP_SEL_INTF_PHY, HALMAC_INTF_PHY_CUT_C,
+	 HALMAC_INTF_PHY_PLATFORM_ALL},
+	{0x0002, 0x60C6, HALMAC_IP_SEL_INTF_PHY, HALMAC_INTF_PHY_CUT_C,
+	 HALMAC_INTF_PHY_PLATFORM_ALL},
+	{0x0008, 0x3597, HALMAC_IP_SEL_INTF_PHY, HALMAC_INTF_PHY_CUT_C,
+	 HALMAC_INTF_PHY_PLATFORM_ALL},
+	{0x0009, 0x321C, HALMAC_IP_SEL_INTF_PHY, HALMAC_INTF_PHY_CUT_C,
+	 HALMAC_INTF_PHY_PLATFORM_ALL},
+	{0x000A, 0x9623, HALMAC_IP_SEL_INTF_PHY, HALMAC_INTF_PHY_CUT_C,
+	 HALMAC_INTF_PHY_PLATFORM_ALL},
+	{0x0020, 0x94FF, HALMAC_IP_SEL_INTF_PHY, HALMAC_INTF_PHY_CUT_C,
+	 HALMAC_INTF_PHY_PLATFORM_ALL},
+	{0x0021, 0xFFCF, HALMAC_IP_SEL_INTF_PHY, HALMAC_INTF_PHY_CUT_C,
+	 HALMAC_INTF_PHY_PLATFORM_ALL},
+	{0x0026, 0xC006, HALMAC_IP_SEL_INTF_PHY, HALMAC_INTF_PHY_CUT_C,
+	 HALMAC_INTF_PHY_PLATFORM_ALL},
+	{0x0029, 0xFF0E, HALMAC_IP_SEL_INTF_PHY, HALMAC_INTF_PHY_CUT_C,
+	 HALMAC_INTF_PHY_PLATFORM_ALL},
+	{0x002A, 0x3040, HALMAC_IP_SEL_INTF_PHY, HALMAC_INTF_PHY_CUT_C,
+	 HALMAC_INTF_PHY_PLATFORM_ALL},
+	{0xFFFF, 0x0000, HALMAC_IP_SEL_INTF_PHY, HALMAC_INTF_PHY_CUT_ALL,
+	 HALMAC_INTF_PHY_PLATFORM_ALL},
+};
diff --git a/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_8822b_pwr_seq.c b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_8822b_pwr_seq.c
new file mode 100644
index 000000000000..0edd1f5a04a8
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_8822b_pwr_seq.c
@@ -0,0 +1,563 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#include "../halmac_88xx_cfg.h"
+#include "halmac_8822b_cfg.h"
+
+static struct halmac_wl_pwr_cfg_ HALMAC_RTL8822B_TRANS_CARDEMU_TO_ACT[] = {
+	/* { offset, cut_msk, fab_msk|interface_msk, base|cmd, msk, value } */
+	{0x0012, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(1), 0}, /*SWR OCP = SWR OCP = 010 1382.40*/
+	{0x0012, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(0), BIT(0)}, /*SWR OCP = 010 1382.40 */
+	{0x0020, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_USB_MSK | HALMAC_PWR_INTF_SDIO_MSK,
+	 HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE, BIT(0),
+	 BIT(0)}, /*0x20[0] = 1b'1 enable LDOA12 MACRO block for all interface*/
+	{0x0001, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_USB_MSK | HALMAC_PWR_INTF_SDIO_MSK,
+	 HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_DELAY, 1,
+	 HALMAC_PWRSEQ_DELAY_MS}, /*Delay 1ms*/
+	{0x0000, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_USB_MSK | HALMAC_PWR_INTF_SDIO_MSK,
+	 HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE, BIT(5),
+	 0}, /*0x00[5] = 1b'0 release analog Ips to digital ,1:isolation*/
+	{0x0005, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 (BIT(4) | BIT(3) | BIT(2)),
+	 0}, /* disable SW LPS 0x04[10]=0 and WLSUS_EN 0x04[12:11]=0*/
+	{0x0075, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_PCI_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(0), BIT(0)}, /* Disable USB suspend */
+	{0x0006, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_POLLING, BIT(1),
+	 BIT(1)}, /* wait till 0x04[17] = 1    power ready*/
+	{0x0075, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_PCI_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(0), 0}, /* Enable USB suspend */
+	{0xFF1A, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_USB_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0}, /*0xFF1A = 0 to release resume signals*/
+	{0x0006, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(0), BIT(0)}, /* release WLON reset  0x04[16]=1*/
+	{0x0005, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(7), 0}, /* disable HWPDN 0x04[15]=0*/
+	{0x0005, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 (BIT(4) | BIT(3)), 0}, /* disable WL suspend*/
+	{0x0005, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(0), BIT(0)}, /* polling until return 0*/
+	{0x0005, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_POLLING, BIT(0), 0},
+	{0x0020, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(3), BIT(3)}, /*Enable XTAL_CLK*/
+	{0x10A8, HALMAC_PWR_CUT_C_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0}, /*NFC pad enabled*/
+	{0x10A9, HALMAC_PWR_CUT_C_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0xef}, /*NFC pad enabled*/
+	{0x10AA, HALMAC_PWR_CUT_C_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0x0c}, /*NFC pad enabled*/
+	{0x0068, HALMAC_PWR_CUT_C_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_WRITE, BIT(4), BIT(4)}, /*SDIO pad power down disabled*/
+	{0x0029, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0xF9}, /*PLL seting*/
+	{0x0024, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(2), 0}, /*Improve TX EVM of CH13 and some 5G channles */
+	{0x0074, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_PCI_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(5), BIT(5)}, /*PCIE WAKE# enabled*/
+	{0xFFFF, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, 0, HALMAC_PWR_CMD_END, 0, 0},
+};
+
+static struct halmac_wl_pwr_cfg_ HALMAC_RTL8822B_TRANS_ACT_TO_CARDEMU[] = {
+	/* { offset, cut_msk, fab_msk|interface_msk, base|cmd, msk, value } */
+	{0x0003, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_WRITE, BIT(2), 0}, /*0x02[10] = 0 Disable MCU Core*/
+	{0x0093, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(3), 0}, /*LPS option 0x93[3]=0 , SWR PFM*/
+	{0x001F, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0}, /*0x1F[7:0] = 0 turn off RF*/
+	{0x00EF, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0}, /*0xEF[7:0] = 0 turn off RF*/
+	{0xFF1A, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_USB_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0x30}, /*0xFF1A = 0x30 to block resume signals*/
+	{0x0049, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(1), 0}, /*Enable rising edge triggering interrupt*/
+	{0x0006, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(0), BIT(0)}, /* release WLON reset  0x04[16]=1*/
+	{0x0002, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(1), 0}, /* Whole BB is reset */
+	{0x0005, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(1), BIT(1)}, /*0x04[9] = 1 turn off MAC by HW state machine*/
+	{0x0005, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_POLLING, BIT(1),
+	 0}, /*wait till 0x04[9] = 0 polling until return 0 to disable*/
+	{0x0020, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(3), 0}, /* XTAL_CLK gated*/
+	{0x0000, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_USB_MSK | HALMAC_PWR_INTF_SDIO_MSK,
+	 HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE, BIT(5),
+	 BIT(5)}, /*0x00[5] = 1b'1 analog Ips to digital ,1:isolation*/
+	{0xFFFF, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, 0, HALMAC_PWR_CMD_END, 0, 0},
+};
+
+static struct halmac_wl_pwr_cfg_ HALMAC_RTL8822B_TRANS_CARDEMU_TO_SUS[] = {
+	/* { offset, cut_msk, fab_msk|interface_msk, base|cmd, msk, value } */
+	{0x0005, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_PCI_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(4) | BIT(3),
+	 (BIT(4) | BIT(3))}, /*0x04[12:11] = 2b'11 enable WL suspend for PCIe*/
+	{0x0005, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_USB_MSK | HALMAC_PWR_INTF_SDIO_MSK,
+	 HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE, BIT(3) | BIT(4),
+	 BIT(3)}, /*0x04[12:11] = 2b'01 enable WL suspend*/
+	{0x0007, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_WRITE, 0xFF,
+	 0x20}, /*0x07[7:0] = 0x20 SDIO SOP option to disable BG/MB/ACK/SWR*/
+	{0x0005, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_PCI_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(3) | BIT(4),
+	 BIT(3) | BIT(4)}, /*0x04[12:11] = 2b'11 enable WL suspend for PCIe*/
+	{0x0086, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_SDIO,
+	 HALMAC_PWR_CMD_WRITE, BIT(0),
+	 BIT(0)}, /*Set SDIO suspend local register*/
+	{0x0086, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_SDIO,
+	 HALMAC_PWR_CMD_POLLING, BIT(1), 0}, /*wait power state to suspend*/
+	{0xFFFF, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, 0, HALMAC_PWR_CMD_END, 0, 0},
+};
+
+static struct halmac_wl_pwr_cfg_ HALMAC_RTL8822B_TRANS_SUS_TO_CARDEMU[] = {
+	/* { offset, cut_msk, fab_msk|interface_msk, base|cmd, msk, value } */
+	{0x0005, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(3) | BIT(7), 0}, /*clear suspend enable and power down enable*/
+	{0x0086, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_SDIO,
+	 HALMAC_PWR_CMD_WRITE, BIT(0), 0}, /*Set SDIO suspend local register*/
+	{0x0086, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_SDIO,
+	 HALMAC_PWR_CMD_POLLING, BIT(1),
+	 BIT(1)}, /*wait power state to suspend*/
+	{0x0005, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(3) | BIT(4), 0}, /*0x04[12:11] = 2b'01enable WL suspend*/
+	{0xFFFF, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, 0, HALMAC_PWR_CMD_END, 0, 0},
+};
+
+static struct halmac_wl_pwr_cfg_ HALMAC_RTL8822B_TRANS_CARDEMU_TO_CARDDIS[] = {
+	/* { offset, cut_msk, fab_msk|interface_msk, base|cmd, msk, value } */
+	{0x0005, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_WRITE, BIT(7),
+	 BIT(7)}, /*suspend enable and power down enable*/
+	{0x0007, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_USB_MSK | HALMAC_PWR_INTF_SDIO_MSK,
+	 HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE, 0xFF,
+	 0x20}, /*0x07=0x20 , SOP option to disable BG/MB*/
+	{0x0067, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(5), 0}, /*0x67[5]=0 , BIT_PAPE_WLBT_SEL*/
+	{0x0005, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_USB_MSK | HALMAC_PWR_INTF_SDIO_MSK,
+	 HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE, BIT(3) | BIT(4),
+	 BIT(3)}, /*0x04[12:11] = 2b'01 enable WL suspend*/
+	{0x0005, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_PCI_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(2), BIT(2)}, /*0x04[10] = 1, enable SW LPS*/
+	{0x004A, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_USB_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(0), 0}, /*0x48[16] = 0 to disable GPIO9 as EXT WAKEUP*/
+	{0x0067, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_WRITE, BIT(5),
+	 0}, /* 0: BT PAPE control ; 1: WL BB LNAON control*/
+	{0x0067, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_WRITE, BIT(4),
+	 0}, /* 0: BT GPIO[11:10] control  ; 1: WL BB LNAON control*/
+	{0x004F, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_WRITE, BIT(0), 0}, /* 0: BT Control*/
+	{0x0067, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_WRITE, BIT(1),
+	 0}, /* turn off BT_3DD_SYNC_B and BT_GPIO[18] */
+	{0x0046, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_WRITE, BIT(6), BIT(6)}, /* GPIO[6] : Output mode*/
+	{0x0067, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_WRITE, BIT(2), 0}, /* turn off BT_GPIO[16] */
+	{0x0046, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_WRITE, BIT(7), BIT(7)}, /* GPIO[7] : Output mode*/
+	{0x0062, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_WRITE, BIT(4), BIT(4)}, /* GPIO[12] : Output mode */
+	{0x0086, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_SDIO,
+	 HALMAC_PWR_CMD_WRITE, BIT(0),
+	 BIT(0)}, /*Set SDIO suspend local register*/
+	{0x0086, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_SDIO,
+	 HALMAC_PWR_CMD_POLLING, BIT(1), 0}, /*wait power state to suspend*/
+	{0x0090, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_USB_MSK | HALMAC_PWR_INTF_PCI_MSK,
+	 HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE, BIT(1),
+	 0}, /*0x90[1]=0 , disable 32k clock*/
+	{0x0044, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_SDIO,
+	 HALMAC_PWR_CMD_WRITE, 0xFF,
+	 0}, /*0x90[1]=0 , disable 32k clock by indirect access*/
+	{0x0040, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_SDIO,
+	 HALMAC_PWR_CMD_WRITE, 0xFF,
+	 0x90}, /*0x90[1]=0 , disable 32k clock by indirect access*/
+	{0x0041, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_SDIO,
+	 HALMAC_PWR_CMD_WRITE, 0xFF,
+	 0x00}, /*0x90[1]=0 , disable 32k clock by indirect access*/
+	{0x0042, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_SDIO,
+	 HALMAC_PWR_CMD_WRITE, 0xFF,
+	 0x04}, /*0x90[1]=0 , disable 32k clock by indirect access*/
+	{0x0081, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(7), 0}, /*0x80[15]clean fw init ready bit*/
+	{0xFFFF, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, 0, HALMAC_PWR_CMD_END, 0, 0},
+};
+
+static struct halmac_wl_pwr_cfg_ HALMAC_RTL8822B_TRANS_CARDDIS_TO_CARDEMU[] = {
+	/* { offset, cut_msk, fab_msk|interface_msk, base|cmd, msk, value } */
+	{0x0086, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_SDIO,
+	 HALMAC_PWR_CMD_WRITE, BIT(0), 0}, /*Set SDIO suspend local register*/
+	{0x0086, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_SDIO,
+	 HALMAC_PWR_CMD_POLLING, BIT(1),
+	 BIT(1)}, /*wait power state to suspend*/
+	{0x004A, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_USB_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(0), 0}, /*0x48[16] = 0 to disable GPIO9 as EXT WAKEUP*/
+	{0x0005, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(3) | BIT(4) | BIT(7),
+	 0}, /*clear suspend enable and power down enable*/
+	{0x0301, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_PCI_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0},
+	{0xFFFF, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, 0, HALMAC_PWR_CMD_END, 0, 0},
+};
+
+static struct halmac_wl_pwr_cfg_ HALMAC_RTL8822B_TRANS_ACT_TO_LPS[] = {
+	/* { offset, cut_msk, fab_msk|interface_msk, base|cmd, msk, value } */
+	{0x0101, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(2), BIT(2)}, /*Enable 32k calibration and thermal meter*/
+	{0x0199, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(3), BIT(3)}, /*Register write data of 32K calibration*/
+	{0x019B, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(7), BIT(7)}, /*Enable 32k calibration reg write*/
+	{0x1138, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(0) | BIT(1), BIT(0) | BIT(1)}, /*set RPWM IMR*/
+	{0x0194, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(0), BIT(0)}, /* enable 32K CLK*/
+	{0x0093, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0x42}, /* LPS Option MAC OFF enable*/
+	{0x0092, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0x20}, /* LPS Option  Enable memory to deep sleep mode*/
+	{0x0090, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(1), BIT(1)}, /* enable reg use 32K CLK*/
+	{0x0301, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_PCI_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0xFF}, /*PCIe DMA stop*/
+	{0x0522, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0xFF}, /*Tx Pause*/
+	{0x05F8, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_POLLING, 0xFF,
+	 0}, /*Should be zero if no packet is transmitting*/
+	{0x05F9, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_POLLING, 0xFF,
+	 0}, /*Should be zero if no packet is transmitting*/
+	{0x05FA, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_POLLING, 0xFF,
+	 0}, /*Should be zero if no packet is transmitting*/
+	{0x05FB, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_POLLING, 0xFF,
+	 0}, /*Should be zero if no packet is transmitting*/
+	{0x0002, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(0), 0}, /*CCK and OFDM are disabled,and clock are gated*/
+	{0x0002, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_DELAY,
+	 0, HALMAC_PWRSEQ_DELAY_US}, /*Delay 1us*/
+	{0x0002, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(1), 0}, /*Whole BB is reset*/
+	{0x0100, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0x3F}, /*Reset MAC TRX*/
+	{0x0101, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(1), 0}, /*check if removed later*/
+	{0x0553, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(5), BIT(5)}, /*Respond TxOK to scheduler*/
+	{0x0008, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(4), BIT(4)}, /* switch TSF clock to 32K*/
+	{0x0109, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_POLLING, BIT(7),
+	 BIT(7)}, /*Polling 0x109[7]=0  TSF in 40M*/
+	{0x0090, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(0), BIT(0)}, /* enable WL_LPS_EN*/
+	{0xFFFF, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, 0, HALMAC_PWR_CMD_END, 0, 0},
+};
+
+static struct halmac_wl_pwr_cfg_ HALMAC_RTL8822B_TRANS_ACT_TO_DEEP_LPS[] = {
+	/* { offset, cut_msk, fab_msk|interface_msk, base|cmd, msk, value } */
+	{0x0101, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(2), BIT(2)}, /*Enable 32k calibration and thermal meter*/
+	{0x0199, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(3), BIT(3)}, /*Register write data of 32K calibration*/
+	{0x019B, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(7), BIT(7)}, /*Enable 32k calibration reg write*/
+	{0x1138, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(0) | BIT(1), BIT(0) | BIT(1)}, /*set RPWM IMR*/
+	{0x0194, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(0), BIT(0)}, /* enable 32K CLK*/
+	{0x0093, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0x40}, /* LPS Option MAC OFF enable*/
+	{0x0092, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0x20}, /* LPS Option  Enable memory to deep sleep mode*/
+	{0x0090, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(1), BIT(1)}, /* enable reg use 32K CLK*/
+	{0x0301, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_PCI_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0xFF}, /*PCIe DMA stop*/
+	{0x0522, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0xFF}, /*Tx Pause*/
+	{0x05F8, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_POLLING, 0xFF,
+	 0}, /*Should be zero if no packet is transmitting*/
+	{0x05F9, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_POLLING, 0xFF,
+	 0}, /*Should be zero if no packet is transmitting*/
+	{0x05FA, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_POLLING, 0xFF,
+	 0}, /*Should be zero if no packet is transmitting*/
+	{0x05FB, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_POLLING, 0xFF,
+	 0}, /*Should be zero if no packet is transmitting*/
+	{0x0002, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(0), 0}, /*CCK and OFDM are disabled,and clock are gated*/
+	{0x0002, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_DELAY,
+	 0, HALMAC_PWRSEQ_DELAY_US}, /*Delay 1us*/
+	{0x0002, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(1), 0}, /*Whole BB is reset*/
+	{0x0100, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0x3F}, /*Reset MAC TRX*/
+	{0x0101, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(1), 0}, /*check if removed later*/
+	{0x0553, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(5), BIT(5)}, /*Respond TxOK to scheduler*/
+	{0x0008, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(4), BIT(4)}, /* switch TSF clock to 32K*/
+	{0x0109, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_POLLING, BIT(7),
+	 BIT(7)}, /*Polling 0x109[7]=1  TSF in 32K*/
+	{0x0090, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(0), BIT(0)}, /* enable WL_LPS_EN*/
+	{0xFFFF, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, 0, HALMAC_PWR_CMD_END, 0, 0},
+};
+
+static struct halmac_wl_pwr_cfg_ HALMAC_RTL8822B_TRANS_LPS_TO_ACT[] = {
+	/* { offset, cut_msk, fab_msk|interface_msk, base|cmd, msk, value } */
+	{0x0080, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_SDIO,
+	 HALMAC_PWR_CMD_WRITE, BIT(7), BIT(7)}, /*SDIO RPWM*/
+	{0x0002, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_DELAY,
+	 0, HALMAC_PWRSEQ_DELAY_MS}, /*Delay*/
+	{0x0080, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_SDIO_MSK, HALMAC_PWR_BASEADDR_SDIO,
+	 HALMAC_PWR_CMD_WRITE, BIT(7), 0}, /*SDIO RPWM*/
+	{0xFE58, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_USB_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0x84}, /*USB RPWM*/
+	{0x0361, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_PCI_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0x84}, /*PCIe RPWM*/
+	{0x0002, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_DELAY,
+	 0, HALMAC_PWRSEQ_DELAY_MS}, /*Delay*/
+	{0x0008, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(4), 0}, /* switch TSF to 40M*/
+	{0x0109, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC,
+	 HALMAC_PWR_CMD_POLLING, BIT(7), 0}, /*Polling 0x109[7]=0  TSF in 40M*/
+	{0x0101, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(1), BIT(1)},
+	{0x0100, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0xFF}, /*nable WMAC TRX*/
+	{0x0002, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(1) | BIT(0), BIT(1) | BIT(0)}, /*nable BB macro*/
+	{0x0522, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0},
+	{0x113C, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0x03}, /*clear RPWM INT*/
+	{0x0124, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0xFF}, /*clear FW INT*/
+	{0x0125, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0xFF}, /*clear FW INT*/
+	{0x0126, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0xFF}, /*clear FW INT*/
+	{0x0127, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 0xFF, 0xFF}, /*clear FW INT*/
+	{0x0090, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(1), 0}, /* disable reg use 32K CLK*/
+	{0x0101, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, HALMAC_PWR_BASEADDR_MAC, HALMAC_PWR_CMD_WRITE,
+	 BIT(2), 0}, /*disable 32k calibration and thermal meter*/
+	{0xFFFF, HALMAC_PWR_CUT_ALL_MSK, HALMAC_PWR_FAB_ALL_MSK,
+	 HALMAC_PWR_INTF_ALL_MSK, 0, HALMAC_PWR_CMD_END, 0, 0},
+};
+
+/* Card Enable Array */
+struct halmac_wl_pwr_cfg_ *halmac_8822b_card_enable_flow[] = {
+	HALMAC_RTL8822B_TRANS_CARDDIS_TO_CARDEMU,
+	HALMAC_RTL8822B_TRANS_CARDEMU_TO_ACT, NULL};
+
+/* Card Disable Array */
+struct halmac_wl_pwr_cfg_ *halmac_8822b_card_disable_flow[] = {
+	HALMAC_RTL8822B_TRANS_ACT_TO_CARDEMU,
+	HALMAC_RTL8822B_TRANS_CARDEMU_TO_CARDDIS, NULL};
+
+/* Suspend Array */
+struct halmac_wl_pwr_cfg_ *halmac_8822b_suspend_flow[] = {
+	HALMAC_RTL8822B_TRANS_ACT_TO_CARDEMU,
+	HALMAC_RTL8822B_TRANS_CARDEMU_TO_SUS, NULL};
+
+/* Resume Array */
+struct halmac_wl_pwr_cfg_ *halmac_8822b_resume_flow[] = {
+	HALMAC_RTL8822B_TRANS_SUS_TO_CARDEMU,
+	HALMAC_RTL8822B_TRANS_CARDEMU_TO_ACT, NULL};
+
+/* HWPDN Array - HW behavior */
+struct halmac_wl_pwr_cfg_ *halmac_8822b_hwpdn_flow[] = {NULL};
+
+/* Enter LPS - FW behavior */
+struct halmac_wl_pwr_cfg_ *halmac_8822b_enter_lps_flow[] = {
+	HALMAC_RTL8822B_TRANS_ACT_TO_LPS, NULL};
+
+/* Enter Deep LPS - FW behavior */
+struct halmac_wl_pwr_cfg_ *halmac_8822b_enter_deep_lps_flow[] = {
+	HALMAC_RTL8822B_TRANS_ACT_TO_DEEP_LPS, NULL};
+
+/* Leave LPS -FW behavior */
+struct halmac_wl_pwr_cfg_ *halmac_8822b_leave_lps_flow[] = {
+	HALMAC_RTL8822B_TRANS_LPS_TO_ACT, NULL};
diff --git a/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_8822b_pwr_seq.h b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_8822b_pwr_seq.h
new file mode 100644
index 000000000000..79a6072ef2ef
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_8822b_pwr_seq.h
@@ -0,0 +1,40 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef HALMAC_POWER_SEQUENCE_8822B
+#define HALMAC_POWER_SEQUENCE_8822B
+
+#include "../../halmac_pwr_seq_cmd.h"
+
+#define HALMAC_8822B_PWR_SEQ_VER "V17"
+extern struct halmac_wl_pwr_cfg_ *halmac_8822b_card_disable_flow[];
+extern struct halmac_wl_pwr_cfg_ *halmac_8822b_card_enable_flow[];
+extern struct halmac_wl_pwr_cfg_ *halmac_8822b_suspend_flow[];
+extern struct halmac_wl_pwr_cfg_ *halmac_8822b_resume_flow[];
+extern struct halmac_wl_pwr_cfg_ *halmac_8822b_hwpdn_flow[];
+extern struct halmac_wl_pwr_cfg_ *halmac_8822b_enter_lps_flow[];
+extern struct halmac_wl_pwr_cfg_ *halmac_8822b_enter_deep_lps_flow[];
+extern struct halmac_wl_pwr_cfg_ *halmac_8822b_leave_lps_flow[];
+
+#endif
diff --git a/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_api_8822b.c b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_api_8822b.c
new file mode 100644
index 000000000000..6b729fe4c096
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_api_8822b.c
@@ -0,0 +1,343 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#include "halmac_8822b_cfg.h"
+#include "halmac_func_8822b.h"
+#include "../halmac_func_88xx.h"
+
+/**
+ * halmac_mount_api_8822b() - attach functions to function pointer
+ * @halmac_adapter
+ *
+ * SD1 internal use
+ *
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ */
+enum halmac_ret_status
+halmac_mount_api_8822b(struct halmac_adapter *halmac_adapter)
+{
+	struct halmac_api *halmac_api =
+		(struct halmac_api *)halmac_adapter->halmac_api;
+
+	halmac_adapter->chip_id = HALMAC_CHIP_ID_8822B;
+	halmac_adapter->hw_config_info.efuse_size = HALMAC_EFUSE_SIZE_8822B;
+	halmac_adapter->hw_config_info.eeprom_size = HALMAC_EEPROM_SIZE_8822B;
+	halmac_adapter->hw_config_info.bt_efuse_size =
+		HALMAC_BT_EFUSE_SIZE_8822B;
+	halmac_adapter->hw_config_info.cam_entry_num =
+		HALMAC_SECURITY_CAM_ENTRY_NUM_8822B;
+	halmac_adapter->hw_config_info.txdesc_size = HALMAC_TX_DESC_SIZE_8822B;
+	halmac_adapter->hw_config_info.rxdesc_size = HALMAC_RX_DESC_SIZE_8822B;
+	halmac_adapter->hw_config_info.tx_fifo_size = HALMAC_TX_FIFO_SIZE_8822B;
+	halmac_adapter->hw_config_info.rx_fifo_size = HALMAC_RX_FIFO_SIZE_8822B;
+	halmac_adapter->hw_config_info.page_size = HALMAC_TX_PAGE_SIZE_8822B;
+	halmac_adapter->hw_config_info.tx_align_size =
+		HALMAC_TX_ALIGN_SIZE_8822B;
+	halmac_adapter->hw_config_info.page_size_2_power =
+		HALMAC_TX_PAGE_SIZE_2_POWER_8822B;
+
+	halmac_adapter->txff_allocation.rsvd_drv_pg_num =
+		HALMAC_RSVD_DRV_PGNUM_8822B;
+
+	halmac_api->halmac_init_trx_cfg = halmac_init_trx_cfg_8822b;
+	halmac_api->halmac_init_protocol_cfg = halmac_init_protocol_cfg_8822b;
+	halmac_api->halmac_init_h2c = halmac_init_h2c_8822b;
+
+	if (halmac_adapter->halmac_interface == HALMAC_INTERFACE_SDIO) {
+		halmac_api->halmac_tx_allowed_sdio =
+			halmac_tx_allowed_sdio_88xx;
+		halmac_api->halmac_cfg_tx_agg_align =
+			halmac_cfg_tx_agg_align_sdio_not_support_88xx;
+		halmac_api->halmac_mac_power_switch =
+			halmac_mac_power_switch_8822b_sdio;
+		halmac_api->halmac_phy_cfg = halmac_phy_cfg_8822b_sdio;
+		halmac_api->halmac_interface_integration_tuning =
+			halmac_interface_integration_tuning_8822b_sdio;
+	} else if (halmac_adapter->halmac_interface == HALMAC_INTERFACE_USB) {
+		halmac_api->halmac_mac_power_switch =
+			halmac_mac_power_switch_8822b_usb;
+		halmac_api->halmac_cfg_tx_agg_align =
+			halmac_cfg_tx_agg_align_usb_not_support_88xx;
+		halmac_api->halmac_phy_cfg = halmac_phy_cfg_8822b_usb;
+		halmac_api->halmac_interface_integration_tuning =
+			halmac_interface_integration_tuning_8822b_usb;
+	} else if (halmac_adapter->halmac_interface == HALMAC_INTERFACE_PCIE) {
+		halmac_api->halmac_mac_power_switch =
+			halmac_mac_power_switch_8822b_pcie;
+		halmac_api->halmac_cfg_tx_agg_align =
+			halmac_cfg_tx_agg_align_pcie_not_support_88xx;
+		halmac_api->halmac_pcie_switch = halmac_pcie_switch_8822b;
+		halmac_api->halmac_phy_cfg = halmac_phy_cfg_8822b_pcie;
+		halmac_api->halmac_interface_integration_tuning =
+			halmac_interface_integration_tuning_8822b_pcie;
+	} else {
+		halmac_api->halmac_pcie_switch = halmac_pcie_switch_8822b_nc;
+	}
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_init_trx_cfg_8822b() - config trx dma register
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_trx_mode : trx mode selection
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_init_trx_cfg_8822b(struct halmac_adapter *halmac_adapter,
+			  enum halmac_trx_mode halmac_trx_mode)
+{
+	u8 value8;
+	u32 value32;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_INIT_TRX_CFG);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+	halmac_adapter->trx_mode = halmac_trx_mode;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"halmac_init_trx_cfg ==========>halmac_trx_mode = %d\n",
+			halmac_trx_mode);
+
+	status = halmac_txdma_queue_mapping_8822b(halmac_adapter,
+						  halmac_trx_mode);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"halmac_txdma_queue_mapping fail!\n");
+		return status;
+	}
+
+	value8 = 0;
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_CR, value8);
+	value8 = HALMAC_CR_TRX_ENABLE_8822B;
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_CR, value8);
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_H2CQ_CSR, BIT(31));
+
+	status = halmac_priority_queue_config_8822b(halmac_adapter,
+						    halmac_trx_mode);
+	if (halmac_adapter->txff_allocation.rx_fifo_expanding_mode !=
+	    HALMAC_RX_FIFO_EXPANDING_MODE_DISABLE)
+		HALMAC_REG_WRITE_8(halmac_adapter, REG_RX_DRVINFO_SZ, 0xF);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"halmac_txdma_queue_mapping fail!\n");
+		return status;
+	}
+
+	/* Config H2C packet buffer */
+	value32 = HALMAC_REG_READ_32(halmac_adapter, REG_H2C_HEAD);
+	value32 = (value32 & 0xFFFC0000) |
+		  (halmac_adapter->txff_allocation.rsvd_h2c_queue_pg_bndy
+		   << HALMAC_TX_PAGE_SIZE_2_POWER_8822B);
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_H2C_HEAD, value32);
+
+	value32 = HALMAC_REG_READ_32(halmac_adapter, REG_H2C_READ_ADDR);
+	value32 = (value32 & 0xFFFC0000) |
+		  (halmac_adapter->txff_allocation.rsvd_h2c_queue_pg_bndy
+		   << HALMAC_TX_PAGE_SIZE_2_POWER_8822B);
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_H2C_READ_ADDR, value32);
+
+	value32 = HALMAC_REG_READ_32(halmac_adapter, REG_H2C_TAIL);
+	value32 = (value32 & 0xFFFC0000) |
+		  ((halmac_adapter->txff_allocation.rsvd_h2c_queue_pg_bndy
+		    << HALMAC_TX_PAGE_SIZE_2_POWER_8822B) +
+		   (HALMAC_RSVD_H2C_QUEUE_PGNUM_8822B
+		    << HALMAC_TX_PAGE_SIZE_2_POWER_8822B));
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_H2C_TAIL, value32);
+
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_H2C_INFO);
+	value8 = (u8)((value8 & 0xFC) | 0x01);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_H2C_INFO, value8);
+
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_H2C_INFO);
+	value8 = (u8)((value8 & 0xFB) | 0x04);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_H2C_INFO, value8);
+
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_TXDMA_OFFSET_CHK + 1);
+	value8 = (u8)((value8 & 0x7f) | 0x80);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_TXDMA_OFFSET_CHK + 1, value8);
+
+	halmac_adapter->h2c_buff_size = HALMAC_RSVD_H2C_QUEUE_PGNUM_8822B
+					<< HALMAC_TX_PAGE_SIZE_2_POWER_8822B;
+	halmac_get_h2c_buff_free_space_88xx(halmac_adapter);
+
+	if (halmac_adapter->h2c_buff_size !=
+	    halmac_adapter->h2c_buf_free_space) {
+		pr_err("get h2c free space error!\n");
+		return HALMAC_RET_GET_H2C_SPACE_ERR;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"halmac_init_trx_cfg <==========\n");
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_init_protocol_cfg_8822b() - config protocol register
+ * @halmac_adapter : the adapter of halmac
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_init_protocol_cfg_8822b(struct halmac_adapter *halmac_adapter)
+{
+	u32 value32;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_INIT_PROTOCOL_CFG);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"[TRACE]%s ==========>\n", __func__);
+
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_AMPDU_MAX_TIME_V1,
+			   HALMAC_AMPDU_MAX_TIME_8822B);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_TX_HANG_CTRL, BIT_EN_EOF_V1);
+
+	value32 = HALMAC_PROT_RTS_LEN_TH_8822B |
+		  (HALMAC_PROT_RTS_TX_TIME_TH_8822B << 8) |
+		  (HALMAC_PROT_MAX_AGG_PKT_LIMIT_8822B << 16) |
+		  (HALMAC_PROT_RTS_MAX_AGG_PKT_LIMIT_8822B << 24);
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_PROT_MODE_CTRL, value32);
+
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_BAR_MODE_CTRL + 2,
+			    HALMAC_BAR_RETRY_LIMIT_8822B |
+				    HALMAC_RA_TRY_RATE_AGG_LIMIT_8822B << 8);
+
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_FAST_EDCA_VOVI_SETTING,
+			   HALMAC_FAST_EDCA_VO_TH_8822B);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_FAST_EDCA_VOVI_SETTING + 2,
+			   HALMAC_FAST_EDCA_VI_TH_8822B);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_FAST_EDCA_BEBK_SETTING,
+			   HALMAC_FAST_EDCA_BE_TH_8822B);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_FAST_EDCA_BEBK_SETTING + 2,
+			   HALMAC_FAST_EDCA_BK_TH_8822B);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"[TRACE]%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_init_h2c_8822b() - config h2c packet buffer
+ * @halmac_adapter : the adapter of halmac
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_init_h2c_8822b(struct halmac_adapter *halmac_adapter)
+{
+	u8 value8;
+	u32 value32;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	value8 = 0;
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_CR, value8);
+	value8 = HALMAC_CR_TRX_ENABLE_8822B;
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_CR, value8);
+
+	value32 = HALMAC_REG_READ_32(halmac_adapter, REG_H2C_HEAD);
+	value32 = (value32 & 0xFFFC0000) |
+		  (halmac_adapter->txff_allocation.rsvd_h2c_queue_pg_bndy
+		   << HALMAC_TX_PAGE_SIZE_2_POWER_8822B);
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_H2C_HEAD, value32);
+
+	value32 = HALMAC_REG_READ_32(halmac_adapter, REG_H2C_READ_ADDR);
+	value32 = (value32 & 0xFFFC0000) |
+		  (halmac_adapter->txff_allocation.rsvd_h2c_queue_pg_bndy
+		   << HALMAC_TX_PAGE_SIZE_2_POWER_8822B);
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_H2C_READ_ADDR, value32);
+
+	value32 = HALMAC_REG_READ_32(halmac_adapter, REG_H2C_TAIL);
+	value32 = (value32 & 0xFFFC0000) |
+		  ((halmac_adapter->txff_allocation.rsvd_h2c_queue_pg_bndy
+		    << HALMAC_TX_PAGE_SIZE_2_POWER_8822B) +
+		   (HALMAC_RSVD_H2C_QUEUE_PGNUM_8822B
+		    << HALMAC_TX_PAGE_SIZE_2_POWER_8822B));
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_H2C_TAIL, value32);
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_H2C_INFO);
+	value8 = (u8)((value8 & 0xFC) | 0x01);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_H2C_INFO, value8);
+
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_H2C_INFO);
+	value8 = (u8)((value8 & 0xFB) | 0x04);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_H2C_INFO, value8);
+
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_TXDMA_OFFSET_CHK + 1);
+	value8 = (u8)((value8 & 0x7f) | 0x80);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_TXDMA_OFFSET_CHK + 1, value8);
+
+	halmac_adapter->h2c_buff_size = HALMAC_RSVD_H2C_QUEUE_PGNUM_8822B
+					<< HALMAC_TX_PAGE_SIZE_2_POWER_8822B;
+	halmac_get_h2c_buff_free_space_88xx(halmac_adapter);
+
+	if (halmac_adapter->h2c_buff_size !=
+	    halmac_adapter->h2c_buf_free_space) {
+		pr_err("get h2c free space error!\n");
+		return HALMAC_RET_GET_H2C_SPACE_ERR;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"h2c free space : %d\n",
+			halmac_adapter->h2c_buf_free_space);
+
+	return HALMAC_RET_SUCCESS;
+}
diff --git a/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_api_8822b.h b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_api_8822b.h
new file mode 100644
index 000000000000..cf21e3d25607
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_api_8822b.h
@@ -0,0 +1,44 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HALMAC_API_8822B_H_
+#define _HALMAC_API_8822B_H_
+
+#include "../../halmac_2_platform.h"
+#include "../../halmac_type.h"
+
+enum halmac_ret_status
+halmac_mount_api_8822b(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_init_trx_cfg_8822b(struct halmac_adapter *halmac_adapter,
+			  enum halmac_trx_mode halmac_trx_mode);
+
+enum halmac_ret_status
+halmac_init_protocol_cfg_8822b(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_init_h2c_8822b(struct halmac_adapter *halmac_adapter);
+
+#endif /* _HALMAC_API_8822B_H_ */
diff --git a/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_api_8822b_pcie.c b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_api_8822b_pcie.c
new file mode 100644
index 000000000000..e25e2b0ebb4c
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_api_8822b_pcie.c
@@ -0,0 +1,323 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#include "../halmac_88xx_cfg.h"
+#include "../halmac_api_88xx_pcie.h"
+#include "halmac_8822b_cfg.h"
+
+/**
+ * halmac_mac_power_switch_8822b_pcie() - switch mac power
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_power : power state
+ * Author : KaiYuan Chang
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_mac_power_switch_8822b_pcie(struct halmac_adapter *halmac_adapter,
+				   enum halmac_mac_power halmac_power)
+{
+	u8 interface_mask;
+	u8 value8;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_MAC_POWER_SWITCH);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(
+		driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+		"halmac_mac_power_switch_88xx_pcie halmac_power =  %x ==========>\n",
+		halmac_power);
+	interface_mask = HALMAC_PWR_INTF_PCI_MSK;
+
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_CR);
+	if (value8 == 0xEA)
+		halmac_adapter->halmac_state.mac_power = HALMAC_MAC_POWER_OFF;
+	else
+		halmac_adapter->halmac_state.mac_power = HALMAC_MAC_POWER_ON;
+
+	/* Check if power switch is needed */
+	if (halmac_power == HALMAC_MAC_POWER_ON &&
+	    halmac_adapter->halmac_state.mac_power == HALMAC_MAC_POWER_ON) {
+		HALMAC_RT_TRACE(
+			driver_adapter, HALMAC_MSG_PWR, DBG_WARNING,
+			"halmac_mac_power_switch power state unchange!\n");
+		return HALMAC_RET_PWR_UNCHANGE;
+	}
+
+	if (halmac_power == HALMAC_MAC_POWER_OFF) {
+		if (halmac_pwr_seq_parser_88xx(
+			    halmac_adapter, HALMAC_PWR_CUT_ALL_MSK,
+			    HALMAC_PWR_FAB_TSMC_MSK, interface_mask,
+			    halmac_8822b_card_disable_flow) !=
+		    HALMAC_RET_SUCCESS) {
+			pr_err("Handle power off cmd error\n");
+			return HALMAC_RET_POWER_OFF_FAIL;
+		}
+
+		halmac_adapter->halmac_state.mac_power = HALMAC_MAC_POWER_OFF;
+		halmac_adapter->halmac_state.ps_state =
+			HALMAC_PS_STATE_UNDEFINE;
+		halmac_adapter->halmac_state.dlfw_state = HALMAC_DLFW_NONE;
+		halmac_init_adapter_dynamic_para_88xx(halmac_adapter);
+	} else {
+		if (halmac_pwr_seq_parser_88xx(
+			    halmac_adapter, HALMAC_PWR_CUT_ALL_MSK,
+			    HALMAC_PWR_FAB_TSMC_MSK, interface_mask,
+			    halmac_8822b_card_enable_flow) !=
+		    HALMAC_RET_SUCCESS) {
+			pr_err("Handle power on cmd error\n");
+			return HALMAC_RET_POWER_ON_FAIL;
+		}
+
+		halmac_adapter->halmac_state.mac_power = HALMAC_MAC_POWER_ON;
+		halmac_adapter->halmac_state.ps_state = HALMAC_PS_STATE_ACT;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"halmac_mac_power_switch_88xx_pcie <==========\n");
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_pcie_switch_8822b() - pcie gen1/gen2 switch
+ * @halmac_adapter : the adapter of halmac
+ * @pcie_cfg : gen1/gen2 selection
+ * Author : KaiYuan Chang
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_pcie_switch_8822b(struct halmac_adapter *halmac_adapter,
+			 enum halmac_pcie_cfg pcie_cfg)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	u8 current_link_speed = 0;
+	u32 count = 0;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_PCIE_SWITCH);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	/* Link Control 2 Register[3:0] Target Link Speed
+	 * Defined encodings are:
+	 * 0001b Target Link 2.5 GT/s
+	 * 0010b Target Link 5.0 GT/s
+	 * 0100b Target Link 8.0 GT/s
+	 */
+
+	if (pcie_cfg == HALMAC_PCIE_GEN1) {
+		/* cfg 0xA0[3:0]=4'b0001 */
+		halmac_dbi_write8_88xx(
+			halmac_adapter, LINK_CTRL2_REG_OFFSET,
+			(halmac_dbi_read8_88xx(halmac_adapter,
+					       LINK_CTRL2_REG_OFFSET) &
+			 0xF0) | BIT(0));
+
+		/* cfg 0x80C[17]=1 //PCIe DesignWave */
+		halmac_dbi_write32_88xx(
+			halmac_adapter, GEN2_CTRL_OFFSET,
+			halmac_dbi_read32_88xx(halmac_adapter,
+					       GEN2_CTRL_OFFSET) |
+				BIT(17));
+
+		/* check link speed if GEN1 */
+		/* cfg 0x82[3:0]=4'b0001 */
+		current_link_speed =
+			halmac_dbi_read8_88xx(halmac_adapter,
+					      LINK_STATUS_REG_OFFSET) &
+			0x0F;
+		count = 2000;
+
+		while (current_link_speed != GEN1_SPEED && count != 0) {
+			usleep_range(50, 60);
+			current_link_speed =
+				halmac_dbi_read8_88xx(halmac_adapter,
+						      LINK_STATUS_REG_OFFSET) &
+				0x0F;
+			count--;
+		}
+
+		if (current_link_speed != GEN1_SPEED) {
+			pr_err("Speed change to GEN1 fail !\n");
+			return HALMAC_RET_FAIL;
+		}
+
+	} else if (pcie_cfg == HALMAC_PCIE_GEN2) {
+		/* cfg 0xA0[3:0]=4'b0010 */
+		halmac_dbi_write8_88xx(
+			halmac_adapter, LINK_CTRL2_REG_OFFSET,
+			(halmac_dbi_read8_88xx(halmac_adapter,
+					       LINK_CTRL2_REG_OFFSET) &
+			 0xF0) | BIT(1));
+
+		/* cfg 0x80C[17]=1 //PCIe DesignWave */
+		halmac_dbi_write32_88xx(
+			halmac_adapter, GEN2_CTRL_OFFSET,
+			halmac_dbi_read32_88xx(halmac_adapter,
+					       GEN2_CTRL_OFFSET) |
+				BIT(17));
+
+		/* check link speed if GEN2 */
+		/* cfg 0x82[3:0]=4'b0010 */
+		current_link_speed =
+			halmac_dbi_read8_88xx(halmac_adapter,
+					      LINK_STATUS_REG_OFFSET) &
+			0x0F;
+		count = 2000;
+
+		while (current_link_speed != GEN2_SPEED && count != 0) {
+			usleep_range(50, 60);
+			current_link_speed =
+				halmac_dbi_read8_88xx(halmac_adapter,
+						      LINK_STATUS_REG_OFFSET) &
+				0x0F;
+			count--;
+		}
+
+		if (current_link_speed != GEN2_SPEED) {
+			pr_err("Speed change to GEN1 fail !\n");
+			return HALMAC_RET_FAIL;
+		}
+
+	} else {
+		pr_err("Error Speed !\n");
+		return HALMAC_RET_FAIL;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_pcie_switch_8822b_nc(struct halmac_adapter *halmac_adapter,
+			    enum halmac_pcie_cfg pcie_cfg)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_PCIE_SWITCH);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_phy_cfg_8822b_pcie() - phy config
+ * @halmac_adapter : the adapter of halmac
+ * Author : KaiYuan Chang
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_phy_cfg_8822b_pcie(struct halmac_adapter *halmac_adapter,
+			  enum halmac_intf_phy_platform platform)
+{
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_PHY_CFG);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"halmac_phy_cfg ==========>\n");
+
+	status = halmac_parse_intf_phy_88xx(halmac_adapter,
+					    HALMAC_RTL8822B_PCIE_PHY_GEN1,
+					    platform, HAL_INTF_PHY_PCIE_GEN1);
+
+	if (status != HALMAC_RET_SUCCESS)
+		return status;
+
+	status = halmac_parse_intf_phy_88xx(halmac_adapter,
+					    HALMAC_RTL8822B_PCIE_PHY_GEN2,
+					    platform, HAL_INTF_PHY_PCIE_GEN2);
+
+	if (status != HALMAC_RET_SUCCESS)
+		return status;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"halmac_phy_cfg <==========\n");
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_interface_integration_tuning_8822b_pcie() - pcie interface fine tuning
+ * @halmac_adapter : the adapter of halmac
+ * Author : Rick Liu
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status halmac_interface_integration_tuning_8822b_pcie(
+	struct halmac_adapter *halmac_adapter)
+{
+	return HALMAC_RET_SUCCESS;
+}
diff --git a/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_api_8822b_pcie.h b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_api_8822b_pcie.h
new file mode 100644
index 000000000000..c68ea0039703
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_api_8822b_pcie.h
@@ -0,0 +1,53 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HALMAC_API_8822B_PCIE_H_
+#define _HALMAC_API_8822B_PCIE_H_
+
+#include "../../halmac_2_platform.h"
+#include "../../halmac_type.h"
+
+extern struct halmac_intf_phy_para_ HALMAC_RTL8822B_PCIE_PHY_GEN1[];
+extern struct halmac_intf_phy_para_ HALMAC_RTL8822B_PCIE_PHY_GEN2[];
+
+enum halmac_ret_status
+halmac_mac_power_switch_8822b_pcie(struct halmac_adapter *halmac_adapter,
+				   enum halmac_mac_power halmac_power);
+
+enum halmac_ret_status
+halmac_pcie_switch_8822b(struct halmac_adapter *halmac_adapter,
+			 enum halmac_pcie_cfg pcie_cfg);
+
+enum halmac_ret_status
+halmac_pcie_switch_8822b_nc(struct halmac_adapter *halmac_adapter,
+			    enum halmac_pcie_cfg pcie_cfg);
+
+enum halmac_ret_status
+halmac_phy_cfg_8822b_pcie(struct halmac_adapter *halmac_adapter,
+			  enum halmac_intf_phy_platform platform);
+
+enum halmac_ret_status halmac_interface_integration_tuning_8822b_pcie(
+	struct halmac_adapter *halmac_adapter);
+
+#endif /* _HALMAC_API_8822B_PCIE_H_ */
diff --git a/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_api_8822b_sdio.c b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_api_8822b_sdio.c
new file mode 100644
index 000000000000..4d708d841bad
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_api_8822b_sdio.c
@@ -0,0 +1,184 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#include "halmac_8822b_cfg.h"
+
+/**
+ * halmac_mac_power_switch_8822b_sdio() - switch mac power
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_power : power state
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_mac_power_switch_8822b_sdio(struct halmac_adapter *halmac_adapter,
+				   enum halmac_mac_power halmac_power)
+{
+	u8 interface_mask;
+	u8 value8;
+	u8 rpwm;
+	u32 imr_backup;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(
+		driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+		"[TRACE]halmac_mac_power_switch_88xx_sdio==========>\n");
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"[TRACE]halmac_power = %x ==========>\n", halmac_power);
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"[TRACE]8822B pwr seq ver = %s\n",
+			HALMAC_8822B_PWR_SEQ_VER);
+
+	interface_mask = HALMAC_PWR_INTF_SDIO_MSK;
+
+	halmac_adapter->rpwm_record =
+		HALMAC_REG_READ_8(halmac_adapter, REG_SDIO_HRPWM1);
+
+	/* Check FW still exist or not */
+	if (HALMAC_REG_READ_16(halmac_adapter, REG_MCUFW_CTRL) == 0xC078) {
+		/* Leave 32K */
+		rpwm = (u8)((halmac_adapter->rpwm_record ^ BIT(7)) & 0x80);
+		HALMAC_REG_WRITE_8(halmac_adapter, REG_SDIO_HRPWM1, rpwm);
+	}
+
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_CR);
+	if (value8 == 0xEA)
+		halmac_adapter->halmac_state.mac_power = HALMAC_MAC_POWER_OFF;
+	else
+		halmac_adapter->halmac_state.mac_power = HALMAC_MAC_POWER_ON;
+
+	/*Check if power switch is needed*/
+	if (halmac_power == HALMAC_MAC_POWER_ON &&
+	    halmac_adapter->halmac_state.mac_power == HALMAC_MAC_POWER_ON) {
+		HALMAC_RT_TRACE(
+			driver_adapter, HALMAC_MSG_PWR, DBG_WARNING,
+			"[WARN]halmac_mac_power_switch power state unchange!\n");
+		return HALMAC_RET_PWR_UNCHANGE;
+	}
+
+	imr_backup = HALMAC_REG_READ_32(halmac_adapter, REG_SDIO_HIMR);
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_SDIO_HIMR, 0);
+
+	if (halmac_power == HALMAC_MAC_POWER_OFF) {
+		if (halmac_pwr_seq_parser_88xx(
+			    halmac_adapter, HALMAC_PWR_CUT_ALL_MSK,
+			    HALMAC_PWR_FAB_TSMC_MSK, interface_mask,
+			    halmac_8822b_card_disable_flow) !=
+		    HALMAC_RET_SUCCESS) {
+			pr_err("[ERR]Handle power off cmd error\n");
+			HALMAC_REG_WRITE_32(halmac_adapter, REG_SDIO_HIMR,
+					    imr_backup);
+			return HALMAC_RET_POWER_OFF_FAIL;
+		}
+
+		halmac_adapter->halmac_state.mac_power = HALMAC_MAC_POWER_OFF;
+		halmac_adapter->halmac_state.ps_state =
+			HALMAC_PS_STATE_UNDEFINE;
+		halmac_adapter->halmac_state.dlfw_state = HALMAC_DLFW_NONE;
+		halmac_init_adapter_dynamic_para_88xx(halmac_adapter);
+	} else {
+		if (halmac_pwr_seq_parser_88xx(
+			    halmac_adapter, HALMAC_PWR_CUT_ALL_MSK,
+			    HALMAC_PWR_FAB_TSMC_MSK, interface_mask,
+			    halmac_8822b_card_enable_flow) !=
+		    HALMAC_RET_SUCCESS) {
+			pr_err("[ERR]Handle power on cmd error\n");
+			HALMAC_REG_WRITE_32(halmac_adapter, REG_SDIO_HIMR,
+					    imr_backup);
+			return HALMAC_RET_POWER_ON_FAIL;
+		}
+
+		halmac_adapter->halmac_state.mac_power = HALMAC_MAC_POWER_ON;
+		halmac_adapter->halmac_state.ps_state = HALMAC_PS_STATE_ACT;
+	}
+
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_SDIO_HIMR, imr_backup);
+
+	HALMAC_RT_TRACE(
+		driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+		"[TRACE]halmac_mac_power_switch_88xx_sdio <==========\n");
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_phy_cfg_8822b_sdio() - phy config
+ * @halmac_adapter : the adapter of halmac
+ * Author : KaiYuan Chang
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_phy_cfg_8822b_sdio(struct halmac_adapter *halmac_adapter,
+			  enum halmac_intf_phy_platform platform)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_PHY_CFG);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"halmac_phy_cfg ==========>\n");
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"sdio no phy\n");
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"halmac_phy_cfg <==========\n");
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_interface_integration_tuning_8822b_sdio() - sdio interface fine tuning
+ * @halmac_adapter : the adapter of halmac
+ * Author : Ivan
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status halmac_interface_integration_tuning_8822b_sdio(
+	struct halmac_adapter *halmac_adapter)
+{
+	return HALMAC_RET_SUCCESS;
+}
diff --git a/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_api_8822b_sdio.h b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_api_8822b_sdio.h
new file mode 100644
index 000000000000..07ffb3baf7c0
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_api_8822b_sdio.h
@@ -0,0 +1,42 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HALMAC_API_8822B_SDIO_H_
+#define _HALMAC_API_8822B_SDIO_H_
+
+#include "../../halmac_2_platform.h"
+#include "../../halmac_type.h"
+
+enum halmac_ret_status
+halmac_mac_power_switch_8822b_sdio(struct halmac_adapter *halmac_adapter,
+				   enum halmac_mac_power halmac_power);
+
+enum halmac_ret_status
+halmac_phy_cfg_8822b_sdio(struct halmac_adapter *halmac_adapter,
+			  enum halmac_intf_phy_platform platform);
+
+enum halmac_ret_status halmac_interface_integration_tuning_8822b_sdio(
+	struct halmac_adapter *halmac_adapter);
+
+#endif /* _HALMAC_API_8822B_SDIO_H_ */
diff --git a/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_api_8822b_usb.c b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_api_8822b_usb.c
new file mode 100644
index 000000000000..5f27eb172430
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_api_8822b_usb.c
@@ -0,0 +1,185 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#include "../halmac_88xx_cfg.h"
+#include "halmac_8822b_cfg.h"
+
+/**
+ * halmac_mac_power_switch_8822b_usb() - switch mac power
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_power : power state
+ * Author : KaiYuan Chang
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_mac_power_switch_8822b_usb(struct halmac_adapter *halmac_adapter,
+				  enum halmac_mac_power halmac_power)
+{
+	u8 interface_mask;
+	u8 value8;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_MAC_POWER_SWITCH);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(
+		driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+		"halmac_mac_power_switch_88xx_usb halmac_power = %x ==========>\n",
+		halmac_power);
+
+	interface_mask = HALMAC_PWR_INTF_USB_MSK;
+
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_CR);
+	if (value8 == 0xEA) {
+		halmac_adapter->halmac_state.mac_power = HALMAC_MAC_POWER_OFF;
+	} else {
+		if (BIT(0) ==
+		    (HALMAC_REG_READ_8(halmac_adapter, REG_SYS_STATUS1 + 1) &
+		     BIT(0)))
+			halmac_adapter->halmac_state.mac_power =
+				HALMAC_MAC_POWER_OFF;
+		else
+			halmac_adapter->halmac_state.mac_power =
+				HALMAC_MAC_POWER_ON;
+	}
+
+	/*Check if power switch is needed*/
+	if (halmac_power == HALMAC_MAC_POWER_ON &&
+	    halmac_adapter->halmac_state.mac_power == HALMAC_MAC_POWER_ON) {
+		HALMAC_RT_TRACE(
+			driver_adapter, HALMAC_MSG_PWR, DBG_WARNING,
+			"halmac_mac_power_switch power state unchange!\n");
+		return HALMAC_RET_PWR_UNCHANGE;
+	}
+	if (halmac_power == HALMAC_MAC_POWER_OFF) {
+		if (halmac_pwr_seq_parser_88xx(
+			    halmac_adapter, HALMAC_PWR_CUT_ALL_MSK,
+			    HALMAC_PWR_FAB_TSMC_MSK, interface_mask,
+			    halmac_8822b_card_disable_flow) !=
+		    HALMAC_RET_SUCCESS) {
+			pr_err("Handle power off cmd error\n");
+			return HALMAC_RET_POWER_OFF_FAIL;
+		}
+
+		halmac_adapter->halmac_state.mac_power = HALMAC_MAC_POWER_OFF;
+		halmac_adapter->halmac_state.ps_state =
+			HALMAC_PS_STATE_UNDEFINE;
+		halmac_adapter->halmac_state.dlfw_state = HALMAC_DLFW_NONE;
+		halmac_init_adapter_dynamic_para_88xx(halmac_adapter);
+	} else {
+		if (halmac_pwr_seq_parser_88xx(
+			    halmac_adapter, HALMAC_PWR_CUT_ALL_MSK,
+			    HALMAC_PWR_FAB_TSMC_MSK, interface_mask,
+			    halmac_8822b_card_enable_flow) !=
+		    HALMAC_RET_SUCCESS) {
+			pr_err("Handle power on cmd error\n");
+			return HALMAC_RET_POWER_ON_FAIL;
+		}
+
+		HALMAC_REG_WRITE_8(
+			halmac_adapter, REG_SYS_STATUS1 + 1,
+			HALMAC_REG_READ_8(halmac_adapter, REG_SYS_STATUS1 + 1) &
+				~(BIT(0)));
+
+		halmac_adapter->halmac_state.mac_power = HALMAC_MAC_POWER_ON;
+		halmac_adapter->halmac_state.ps_state = HALMAC_PS_STATE_ACT;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"halmac_mac_power_switch_88xx_usb <==========\n");
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_phy_cfg_8822b_usb() - phy config
+ * @halmac_adapter : the adapter of halmac
+ * Author : KaiYuan Chang
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_phy_cfg_8822b_usb(struct halmac_adapter *halmac_adapter,
+			 enum halmac_intf_phy_platform platform)
+{
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_PHY_CFG);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"halmac_phy_cfg ==========>\n");
+
+	status = halmac_parse_intf_phy_88xx(halmac_adapter,
+					    HALMAC_RTL8822B_USB2_PHY, platform,
+					    HAL_INTF_PHY_USB2);
+
+	if (status != HALMAC_RET_SUCCESS)
+		return status;
+
+	status = halmac_parse_intf_phy_88xx(halmac_adapter,
+					    HALMAC_RTL8822B_USB3_PHY, platform,
+					    HAL_INTF_PHY_USB3);
+
+	if (status != HALMAC_RET_SUCCESS)
+		return status;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"halmac_phy_cfg <==========\n");
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_interface_integration_tuning_8822b_usb() - usb interface fine tuning
+ * @halmac_adapter : the adapter of halmac
+ * Author : Ivan
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status halmac_interface_integration_tuning_8822b_usb(
+	struct halmac_adapter *halmac_adapter)
+{
+	return HALMAC_RET_SUCCESS;
+}
diff --git a/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_api_8822b_usb.h b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_api_8822b_usb.h
new file mode 100644
index 000000000000..3a99fd5675e0
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_api_8822b_usb.h
@@ -0,0 +1,45 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HALMAC_API_8822B_USB_H_
+#define _HALMAC_API_8822B_USB_H_
+
+extern struct halmac_intf_phy_para_ HALMAC_RTL8822B_USB2_PHY[];
+extern struct halmac_intf_phy_para_ HALMAC_RTL8822B_USB3_PHY[];
+
+#include "../../halmac_2_platform.h"
+#include "../../halmac_type.h"
+
+enum halmac_ret_status
+halmac_mac_power_switch_8822b_usb(struct halmac_adapter *halmac_adapter,
+				  enum halmac_mac_power halmac_power);
+
+enum halmac_ret_status
+halmac_phy_cfg_8822b_usb(struct halmac_adapter *halmac_adapter,
+			 enum halmac_intf_phy_platform platform);
+
+enum halmac_ret_status halmac_interface_integration_tuning_8822b_usb(
+	struct halmac_adapter *halmac_adapter);
+
+#endif /* _HALMAC_API_8822B_USB_H_ */
diff --git a/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_func_8822b.c b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_func_8822b.c
new file mode 100644
index 000000000000..5f1dff8d9e3b
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_func_8822b.c
@@ -0,0 +1,414 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#include "halmac_8822b_cfg.h"
+#include "halmac_func_8822b.h"
+
+/*SDIO RQPN Mapping*/
+static struct halmac_rqpn_ HALMAC_RQPN_SDIO_8822B[] = {
+	/* { mode, vo_map, vi_map, be_map, bk_map, mg_map, hi_map } */
+	{HALMAC_TRX_MODE_NORMAL, HALMAC_MAP2_NQ, HALMAC_MAP2_NQ, HALMAC_MAP2_LQ,
+	 HALMAC_MAP2_LQ, HALMAC_MAP2_EXQ, HALMAC_MAP2_HQ},
+	{HALMAC_TRX_MODE_TRXSHARE, HALMAC_MAP2_NQ, HALMAC_MAP2_NQ,
+	 HALMAC_MAP2_LQ, HALMAC_MAP2_LQ, HALMAC_MAP2_EXQ, HALMAC_MAP2_HQ},
+	{HALMAC_TRX_MODE_WMM, HALMAC_MAP2_HQ, HALMAC_MAP2_NQ, HALMAC_MAP2_LQ,
+	 HALMAC_MAP2_NQ, HALMAC_MAP2_EXQ, HALMAC_MAP2_HQ},
+	{HALMAC_TRX_MODE_P2P, HALMAC_MAP2_NQ, HALMAC_MAP2_NQ, HALMAC_MAP2_LQ,
+	 HALMAC_MAP2_LQ, HALMAC_MAP2_EXQ, HALMAC_MAP2_HQ},
+	{HALMAC_TRX_MODE_LOOPBACK, HALMAC_MAP2_NQ, HALMAC_MAP2_NQ,
+	 HALMAC_MAP2_LQ, HALMAC_MAP2_LQ, HALMAC_MAP2_EXQ, HALMAC_MAP2_HQ},
+	{HALMAC_TRX_MODE_DELAY_LOOPBACK, HALMAC_MAP2_NQ, HALMAC_MAP2_NQ,
+	 HALMAC_MAP2_LQ, HALMAC_MAP2_LQ, HALMAC_MAP2_EXQ, HALMAC_MAP2_HQ},
+};
+
+/*PCIE RQPN Mapping*/
+static struct halmac_rqpn_ HALMAC_RQPN_PCIE_8822B[] = {
+	/* { mode, vo_map, vi_map, be_map, bk_map, mg_map, hi_map } */
+	{HALMAC_TRX_MODE_NORMAL, HALMAC_MAP2_NQ, HALMAC_MAP2_NQ, HALMAC_MAP2_LQ,
+	 HALMAC_MAP2_LQ, HALMAC_MAP2_EXQ, HALMAC_MAP2_HQ},
+	{HALMAC_TRX_MODE_TRXSHARE, HALMAC_MAP2_NQ, HALMAC_MAP2_NQ,
+	 HALMAC_MAP2_LQ, HALMAC_MAP2_LQ, HALMAC_MAP2_EXQ, HALMAC_MAP2_HQ},
+	{HALMAC_TRX_MODE_WMM, HALMAC_MAP2_HQ, HALMAC_MAP2_NQ, HALMAC_MAP2_LQ,
+	 HALMAC_MAP2_NQ, HALMAC_MAP2_EXQ, HALMAC_MAP2_HQ},
+	{HALMAC_TRX_MODE_P2P, HALMAC_MAP2_NQ, HALMAC_MAP2_NQ, HALMAC_MAP2_LQ,
+	 HALMAC_MAP2_LQ, HALMAC_MAP2_EXQ, HALMAC_MAP2_HQ},
+	{HALMAC_TRX_MODE_LOOPBACK, HALMAC_MAP2_NQ, HALMAC_MAP2_NQ,
+	 HALMAC_MAP2_LQ, HALMAC_MAP2_LQ, HALMAC_MAP2_EXQ, HALMAC_MAP2_HQ},
+	{HALMAC_TRX_MODE_DELAY_LOOPBACK, HALMAC_MAP2_NQ, HALMAC_MAP2_NQ,
+	 HALMAC_MAP2_LQ, HALMAC_MAP2_LQ, HALMAC_MAP2_EXQ, HALMAC_MAP2_HQ},
+};
+
+/*USB 2 Bulkout RQPN Mapping*/
+static struct halmac_rqpn_ HALMAC_RQPN_2BULKOUT_8822B[] = {
+	/* { mode, vo_map, vi_map, be_map, bk_map, mg_map, hi_map } */
+	{HALMAC_TRX_MODE_NORMAL, HALMAC_MAP2_NQ, HALMAC_MAP2_NQ, HALMAC_MAP2_NQ,
+	 HALMAC_MAP2_HQ, HALMAC_MAP2_HQ, HALMAC_MAP2_HQ},
+	{HALMAC_TRX_MODE_TRXSHARE, HALMAC_MAP2_NQ, HALMAC_MAP2_NQ,
+	 HALMAC_MAP2_NQ, HALMAC_MAP2_HQ, HALMAC_MAP2_HQ, HALMAC_MAP2_HQ},
+	{HALMAC_TRX_MODE_WMM, HALMAC_MAP2_NQ, HALMAC_MAP2_NQ, HALMAC_MAP2_NQ,
+	 HALMAC_MAP2_HQ, HALMAC_MAP2_HQ, HALMAC_MAP2_HQ},
+	{HALMAC_TRX_MODE_P2P, HALMAC_MAP2_HQ, HALMAC_MAP2_HQ, HALMAC_MAP2_HQ,
+	 HALMAC_MAP2_NQ, HALMAC_MAP2_HQ, HALMAC_MAP2_HQ},
+	{HALMAC_TRX_MODE_LOOPBACK, HALMAC_MAP2_HQ, HALMAC_MAP2_HQ,
+	 HALMAC_MAP2_HQ, HALMAC_MAP2_NQ, HALMAC_MAP2_HQ, HALMAC_MAP2_HQ},
+	{HALMAC_TRX_MODE_DELAY_LOOPBACK, HALMAC_MAP2_HQ, HALMAC_MAP2_HQ,
+	 HALMAC_MAP2_HQ, HALMAC_MAP2_NQ, HALMAC_MAP2_HQ, HALMAC_MAP2_HQ},
+};
+
+/*USB 3 Bulkout RQPN Mapping*/
+static struct halmac_rqpn_ HALMAC_RQPN_3BULKOUT_8822B[] = {
+	/* { mode, vo_map, vi_map, be_map, bk_map, mg_map, hi_map } */
+	{HALMAC_TRX_MODE_NORMAL, HALMAC_MAP2_NQ, HALMAC_MAP2_NQ, HALMAC_MAP2_LQ,
+	 HALMAC_MAP2_LQ, HALMAC_MAP2_HQ, HALMAC_MAP2_HQ},
+	{HALMAC_TRX_MODE_TRXSHARE, HALMAC_MAP2_NQ, HALMAC_MAP2_NQ,
+	 HALMAC_MAP2_LQ, HALMAC_MAP2_LQ, HALMAC_MAP2_HQ, HALMAC_MAP2_HQ},
+	{HALMAC_TRX_MODE_WMM, HALMAC_MAP2_HQ, HALMAC_MAP2_NQ, HALMAC_MAP2_LQ,
+	 HALMAC_MAP2_NQ, HALMAC_MAP2_HQ, HALMAC_MAP2_HQ},
+	{HALMAC_TRX_MODE_P2P, HALMAC_MAP2_HQ, HALMAC_MAP2_HQ, HALMAC_MAP2_LQ,
+	 HALMAC_MAP2_NQ, HALMAC_MAP2_HQ, HALMAC_MAP2_HQ},
+	{HALMAC_TRX_MODE_LOOPBACK, HALMAC_MAP2_HQ, HALMAC_MAP2_HQ,
+	 HALMAC_MAP2_LQ, HALMAC_MAP2_NQ, HALMAC_MAP2_HQ, HALMAC_MAP2_HQ},
+	{HALMAC_TRX_MODE_DELAY_LOOPBACK, HALMAC_MAP2_HQ, HALMAC_MAP2_HQ,
+	 HALMAC_MAP2_LQ, HALMAC_MAP2_NQ, HALMAC_MAP2_HQ, HALMAC_MAP2_HQ},
+};
+
+/*USB 4 Bulkout RQPN Mapping*/
+static struct halmac_rqpn_ HALMAC_RQPN_4BULKOUT_8822B[] = {
+	/* { mode, vo_map, vi_map, be_map, bk_map, mg_map, hi_map } */
+	{HALMAC_TRX_MODE_NORMAL, HALMAC_MAP2_NQ, HALMAC_MAP2_NQ, HALMAC_MAP2_LQ,
+	 HALMAC_MAP2_LQ, HALMAC_MAP2_EXQ, HALMAC_MAP2_HQ},
+	{HALMAC_TRX_MODE_TRXSHARE, HALMAC_MAP2_NQ, HALMAC_MAP2_NQ,
+	 HALMAC_MAP2_LQ, HALMAC_MAP2_LQ, HALMAC_MAP2_EXQ, HALMAC_MAP2_HQ},
+	{HALMAC_TRX_MODE_WMM, HALMAC_MAP2_HQ, HALMAC_MAP2_NQ, HALMAC_MAP2_LQ,
+	 HALMAC_MAP2_NQ, HALMAC_MAP2_EXQ, HALMAC_MAP2_HQ},
+	{HALMAC_TRX_MODE_P2P, HALMAC_MAP2_NQ, HALMAC_MAP2_NQ, HALMAC_MAP2_LQ,
+	 HALMAC_MAP2_LQ, HALMAC_MAP2_EXQ, HALMAC_MAP2_HQ},
+	{HALMAC_TRX_MODE_LOOPBACK, HALMAC_MAP2_NQ, HALMAC_MAP2_NQ,
+	 HALMAC_MAP2_LQ, HALMAC_MAP2_LQ, HALMAC_MAP2_EXQ, HALMAC_MAP2_HQ},
+	{HALMAC_TRX_MODE_DELAY_LOOPBACK, HALMAC_MAP2_NQ, HALMAC_MAP2_NQ,
+	 HALMAC_MAP2_LQ, HALMAC_MAP2_LQ, HALMAC_MAP2_EXQ, HALMAC_MAP2_HQ},
+};
+
+/*SDIO Page Number*/
+static struct halmac_pg_num_ HALMAC_PG_NUM_SDIO_8822B[] = {
+	/* { mode, hq_num, nq_num, lq_num, exq_num, gap_num} */
+	{HALMAC_TRX_MODE_NORMAL, 64, 64, 64, 64, 1},
+	{HALMAC_TRX_MODE_TRXSHARE, 32, 32, 32, 32, 1},
+	{HALMAC_TRX_MODE_WMM, 64, 64, 64, 64, 1},
+	{HALMAC_TRX_MODE_P2P, 64, 64, 64, 64, 1},
+	{HALMAC_TRX_MODE_LOOPBACK, 64, 64, 64, 64, 640},
+	{HALMAC_TRX_MODE_DELAY_LOOPBACK, 64, 64, 64, 64, 640},
+};
+
+/*PCIE Page Number*/
+static struct halmac_pg_num_ HALMAC_PG_NUM_PCIE_8822B[] = {
+	/* { mode, hq_num, nq_num, lq_num, exq_num, gap_num} */
+	{HALMAC_TRX_MODE_NORMAL, 64, 64, 64, 64, 1},
+	{HALMAC_TRX_MODE_TRXSHARE, 64, 64, 64, 64, 1},
+	{HALMAC_TRX_MODE_WMM, 64, 64, 64, 64, 1},
+	{HALMAC_TRX_MODE_P2P, 64, 64, 64, 64, 1},
+	{HALMAC_TRX_MODE_LOOPBACK, 64, 64, 64, 64, 640},
+	{HALMAC_TRX_MODE_DELAY_LOOPBACK, 64, 64, 64, 64, 640},
+};
+
+/*USB 2 Bulkout Page Number*/
+static struct halmac_pg_num_ HALMAC_PG_NUM_2BULKOUT_8822B[] = {
+	/* { mode, hq_num, nq_num, lq_num, exq_num, gap_num} */
+	{HALMAC_TRX_MODE_NORMAL, 64, 64, 0, 0, 1},
+	{HALMAC_TRX_MODE_TRXSHARE, 64, 64, 0, 0, 1},
+	{HALMAC_TRX_MODE_WMM, 64, 64, 0, 0, 1},
+	{HALMAC_TRX_MODE_P2P, 64, 64, 0, 0, 1},
+	{HALMAC_TRX_MODE_LOOPBACK, 64, 64, 0, 0, 1024},
+	{HALMAC_TRX_MODE_DELAY_LOOPBACK, 64, 64, 0, 0, 1024},
+};
+
+/*USB 3 Bulkout Page Number*/
+static struct halmac_pg_num_ HALMAC_PG_NUM_3BULKOUT_8822B[] = {
+	/* { mode, hq_num, nq_num, lq_num, exq_num, gap_num} */
+	{HALMAC_TRX_MODE_NORMAL, 64, 64, 64, 0, 1},
+	{HALMAC_TRX_MODE_TRXSHARE, 64, 64, 64, 0, 1},
+	{HALMAC_TRX_MODE_WMM, 64, 64, 64, 0, 1},
+	{HALMAC_TRX_MODE_P2P, 64, 64, 64, 0, 1},
+	{HALMAC_TRX_MODE_LOOPBACK, 64, 64, 64, 0, 1024},
+	{HALMAC_TRX_MODE_DELAY_LOOPBACK, 64, 64, 64, 0, 1024},
+};
+
+/*USB 4 Bulkout Page Number*/
+static struct halmac_pg_num_ HALMAC_PG_NUM_4BULKOUT_8822B[] = {
+	/* { mode, hq_num, nq_num, lq_num, exq_num, gap_num} */
+	{HALMAC_TRX_MODE_NORMAL, 64, 64, 64, 64, 1},
+	{HALMAC_TRX_MODE_TRXSHARE, 64, 64, 64, 64, 1},
+	{HALMAC_TRX_MODE_WMM, 64, 64, 64, 64, 1},
+	{HALMAC_TRX_MODE_P2P, 64, 64, 64, 64, 1},
+	{HALMAC_TRX_MODE_LOOPBACK, 64, 64, 64, 64, 640},
+	{HALMAC_TRX_MODE_DELAY_LOOPBACK, 64, 64, 64, 64, 640},
+};
+
+enum halmac_ret_status
+halmac_txdma_queue_mapping_8822b(struct halmac_adapter *halmac_adapter,
+				 enum halmac_trx_mode halmac_trx_mode)
+{
+	u16 value16;
+	void *driver_adapter = NULL;
+	struct halmac_rqpn_ *curr_rqpn_sel = NULL;
+	enum halmac_ret_status status;
+	struct halmac_api *halmac_api;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	if (halmac_adapter->halmac_interface == HALMAC_INTERFACE_SDIO) {
+		curr_rqpn_sel = HALMAC_RQPN_SDIO_8822B;
+	} else if (halmac_adapter->halmac_interface == HALMAC_INTERFACE_PCIE) {
+		curr_rqpn_sel = HALMAC_RQPN_PCIE_8822B;
+	} else if (halmac_adapter->halmac_interface == HALMAC_INTERFACE_USB) {
+		if (halmac_adapter->halmac_bulkout_num == 2) {
+			curr_rqpn_sel = HALMAC_RQPN_2BULKOUT_8822B;
+		} else if (halmac_adapter->halmac_bulkout_num == 3) {
+			curr_rqpn_sel = HALMAC_RQPN_3BULKOUT_8822B;
+		} else if (halmac_adapter->halmac_bulkout_num == 4) {
+			curr_rqpn_sel = HALMAC_RQPN_4BULKOUT_8822B;
+		} else {
+			pr_err("[ERR]interface not support\n");
+			return HALMAC_RET_NOT_SUPPORT;
+		}
+	} else {
+		return HALMAC_RET_NOT_SUPPORT;
+	}
+
+	status = halmac_rqpn_parser_88xx(halmac_adapter, halmac_trx_mode,
+					 curr_rqpn_sel);
+	if (status != HALMAC_RET_SUCCESS)
+		return status;
+
+	value16 = 0;
+	value16 |= BIT_TXDMA_HIQ_MAP(
+		halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_HI]);
+	value16 |= BIT_TXDMA_MGQ_MAP(
+		halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_MG]);
+	value16 |= BIT_TXDMA_BKQ_MAP(
+		halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_BK]);
+	value16 |= BIT_TXDMA_BEQ_MAP(
+		halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_BE]);
+	value16 |= BIT_TXDMA_VIQ_MAP(
+		halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_VI]);
+	value16 |= BIT_TXDMA_VOQ_MAP(
+		halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_VO]);
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_TXDMA_PQ_MAP, value16);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_priority_queue_config_8822b(struct halmac_adapter *halmac_adapter,
+				   enum halmac_trx_mode halmac_trx_mode)
+{
+	u8 transfer_mode = 0;
+	u8 value8;
+	u32 counter;
+	enum halmac_ret_status status;
+	struct halmac_pg_num_ *curr_pg_num = NULL;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	if (halmac_adapter->txff_allocation.la_mode == HALMAC_LA_MODE_DISABLE) {
+		if (halmac_adapter->txff_allocation.rx_fifo_expanding_mode ==
+		    HALMAC_RX_FIFO_EXPANDING_MODE_DISABLE) {
+			halmac_adapter->txff_allocation.tx_fifo_pg_num =
+				HALMAC_TX_FIFO_SIZE_8822B >>
+				HALMAC_TX_PAGE_SIZE_2_POWER_8822B;
+		} else if (halmac_adapter->txff_allocation
+				   .rx_fifo_expanding_mode ==
+			   HALMAC_RX_FIFO_EXPANDING_MODE_1_BLOCK) {
+			halmac_adapter->txff_allocation.tx_fifo_pg_num =
+				HALMAC_TX_FIFO_SIZE_EX_1_BLK_8822B >>
+				HALMAC_TX_PAGE_SIZE_2_POWER_8822B;
+			halmac_adapter->hw_config_info.tx_fifo_size =
+				HALMAC_TX_FIFO_SIZE_EX_1_BLK_8822B;
+			if (HALMAC_RX_FIFO_SIZE_EX_1_BLK_8822B <=
+			    HALMAC_RX_FIFO_SIZE_EX_1_BLK_MAX_8822B)
+				halmac_adapter->hw_config_info.rx_fifo_size =
+					HALMAC_RX_FIFO_SIZE_EX_1_BLK_8822B;
+			else
+				halmac_adapter->hw_config_info.rx_fifo_size =
+					HALMAC_RX_FIFO_SIZE_EX_1_BLK_MAX_8822B;
+		} else {
+			halmac_adapter->txff_allocation.tx_fifo_pg_num =
+				HALMAC_TX_FIFO_SIZE_8822B >>
+				HALMAC_TX_PAGE_SIZE_2_POWER_8822B;
+			pr_err("[ERR]rx_fifo_expanding_mode = %d not support\n",
+			       halmac_adapter->txff_allocation
+				       .rx_fifo_expanding_mode);
+		}
+	} else {
+		halmac_adapter->txff_allocation.tx_fifo_pg_num =
+			HALMAC_TX_FIFO_SIZE_LA_8822B >>
+			HALMAC_TX_PAGE_SIZE_2_POWER_8822B;
+	}
+	halmac_adapter->txff_allocation.rsvd_pg_num =
+		(halmac_adapter->txff_allocation.rsvd_drv_pg_num +
+		 HALMAC_RSVD_H2C_EXTRAINFO_PGNUM_8822B +
+		 HALMAC_RSVD_H2C_QUEUE_PGNUM_8822B +
+		 HALMAC_RSVD_CPU_INSTRUCTION_PGNUM_8822B +
+		 HALMAC_RSVD_FW_TXBUFF_PGNUM_8822B);
+	if (halmac_adapter->txff_allocation.rsvd_pg_num >
+	    halmac_adapter->txff_allocation.tx_fifo_pg_num)
+		return HALMAC_RET_CFG_TXFIFO_PAGE_FAIL;
+
+	halmac_adapter->txff_allocation.ac_q_pg_num =
+		halmac_adapter->txff_allocation.tx_fifo_pg_num -
+		halmac_adapter->txff_allocation.rsvd_pg_num;
+	halmac_adapter->txff_allocation.rsvd_pg_bndy =
+		halmac_adapter->txff_allocation.tx_fifo_pg_num -
+		halmac_adapter->txff_allocation.rsvd_pg_num;
+	halmac_adapter->txff_allocation.rsvd_fw_txbuff_pg_bndy =
+		halmac_adapter->txff_allocation.tx_fifo_pg_num -
+		HALMAC_RSVD_FW_TXBUFF_PGNUM_8822B;
+	halmac_adapter->txff_allocation.rsvd_cpu_instr_pg_bndy =
+		halmac_adapter->txff_allocation.rsvd_fw_txbuff_pg_bndy -
+		HALMAC_RSVD_CPU_INSTRUCTION_PGNUM_8822B;
+	halmac_adapter->txff_allocation.rsvd_h2c_queue_pg_bndy =
+		halmac_adapter->txff_allocation.rsvd_cpu_instr_pg_bndy -
+		HALMAC_RSVD_H2C_QUEUE_PGNUM_8822B;
+	halmac_adapter->txff_allocation.rsvd_h2c_extra_info_pg_bndy =
+		halmac_adapter->txff_allocation.rsvd_h2c_queue_pg_bndy -
+		HALMAC_RSVD_H2C_EXTRAINFO_PGNUM_8822B;
+	halmac_adapter->txff_allocation.rsvd_drv_pg_bndy =
+		halmac_adapter->txff_allocation.rsvd_h2c_extra_info_pg_bndy -
+		halmac_adapter->txff_allocation.rsvd_drv_pg_num;
+
+	if (halmac_adapter->halmac_interface == HALMAC_INTERFACE_SDIO) {
+		curr_pg_num = HALMAC_PG_NUM_SDIO_8822B;
+	} else if (halmac_adapter->halmac_interface == HALMAC_INTERFACE_PCIE) {
+		curr_pg_num = HALMAC_PG_NUM_PCIE_8822B;
+	} else if (halmac_adapter->halmac_interface == HALMAC_INTERFACE_USB) {
+		if (halmac_adapter->halmac_bulkout_num == 2) {
+			curr_pg_num = HALMAC_PG_NUM_2BULKOUT_8822B;
+		} else if (halmac_adapter->halmac_bulkout_num == 3) {
+			curr_pg_num = HALMAC_PG_NUM_3BULKOUT_8822B;
+		} else if (halmac_adapter->halmac_bulkout_num == 4) {
+			curr_pg_num = HALMAC_PG_NUM_4BULKOUT_8822B;
+		} else {
+			pr_err("[ERR]interface not support\n");
+			return HALMAC_RET_NOT_SUPPORT;
+		}
+	} else {
+		return HALMAC_RET_NOT_SUPPORT;
+	}
+
+	status = halmac_pg_num_parser_88xx(halmac_adapter, halmac_trx_mode,
+					   curr_pg_num);
+	if (status != HALMAC_RET_SUCCESS)
+		return status;
+
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_FIFOPAGE_INFO_1,
+			    halmac_adapter->txff_allocation.high_queue_pg_num);
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_FIFOPAGE_INFO_2,
+			    halmac_adapter->txff_allocation.low_queue_pg_num);
+	HALMAC_REG_WRITE_16(
+		halmac_adapter, REG_FIFOPAGE_INFO_3,
+		halmac_adapter->txff_allocation.normal_queue_pg_num);
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_FIFOPAGE_INFO_4,
+			    halmac_adapter->txff_allocation.extra_queue_pg_num);
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_FIFOPAGE_INFO_5,
+			    halmac_adapter->txff_allocation.pub_queue_pg_num);
+
+	halmac_adapter->sdio_free_space.high_queue_number =
+		halmac_adapter->txff_allocation.high_queue_pg_num;
+	halmac_adapter->sdio_free_space.normal_queue_number =
+		halmac_adapter->txff_allocation.normal_queue_pg_num;
+	halmac_adapter->sdio_free_space.low_queue_number =
+		halmac_adapter->txff_allocation.low_queue_pg_num;
+	halmac_adapter->sdio_free_space.public_queue_number =
+		halmac_adapter->txff_allocation.pub_queue_pg_num;
+	halmac_adapter->sdio_free_space.extra_queue_number =
+		halmac_adapter->txff_allocation.extra_queue_pg_num;
+
+	HALMAC_REG_WRITE_32(
+		halmac_adapter, REG_RQPN_CTRL_2,
+		HALMAC_REG_READ_32(halmac_adapter, REG_RQPN_CTRL_2) | BIT(31));
+
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_FIFOPAGE_CTRL_2,
+			    (u16)(halmac_adapter->txff_allocation.rsvd_pg_bndy &
+				  BIT_MASK_BCN_HEAD_1_V1));
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_BCNQ_BDNY_V1,
+			    (u16)(halmac_adapter->txff_allocation.rsvd_pg_bndy &
+				  BIT_MASK_BCNQ_PGBNDY_V1));
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_FIFOPAGE_CTRL_2 + 2,
+			    (u16)(halmac_adapter->txff_allocation.rsvd_pg_bndy &
+				  BIT_MASK_BCN_HEAD_1_V1));
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_BCNQ1_BDNY_V1,
+			    (u16)(halmac_adapter->txff_allocation.rsvd_pg_bndy &
+				  BIT_MASK_BCNQ_PGBNDY_V1));
+
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_RXFF_BNDY,
+			    halmac_adapter->hw_config_info.rx_fifo_size -
+				    HALMAC_C2H_PKT_BUF_8822B - 1);
+
+	if (halmac_adapter->halmac_interface == HALMAC_INTERFACE_USB) {
+		value8 = (u8)(
+			HALMAC_REG_READ_8(halmac_adapter, REG_AUTO_LLT_V1) &
+			~(BIT_MASK_BLK_DESC_NUM << BIT_SHIFT_BLK_DESC_NUM));
+		value8 = (u8)(value8 | (HALMAC_BLK_DESC_NUM_8822B
+					<< BIT_SHIFT_BLK_DESC_NUM));
+		HALMAC_REG_WRITE_8(halmac_adapter, REG_AUTO_LLT_V1, value8);
+
+		HALMAC_REG_WRITE_8(halmac_adapter, REG_AUTO_LLT_V1 + 3,
+				   HALMAC_BLK_DESC_NUM_8822B);
+		HALMAC_REG_WRITE_8(halmac_adapter, REG_TXDMA_OFFSET_CHK + 1,
+				   HALMAC_REG_READ_8(halmac_adapter,
+						     REG_TXDMA_OFFSET_CHK + 1) |
+					   BIT(1));
+	}
+
+	HALMAC_REG_WRITE_8(
+		halmac_adapter, REG_AUTO_LLT_V1,
+		(u8)(HALMAC_REG_READ_8(halmac_adapter, REG_AUTO_LLT_V1) |
+		     BIT_AUTO_INIT_LLT_V1));
+	counter = 1000;
+	while (HALMAC_REG_READ_8(halmac_adapter, REG_AUTO_LLT_V1) &
+	       BIT_AUTO_INIT_LLT_V1) {
+		counter--;
+		if (counter == 0)
+			return HALMAC_RET_INIT_LLT_FAIL;
+	}
+
+	if (halmac_trx_mode == HALMAC_TRX_MODE_DELAY_LOOPBACK) {
+		transfer_mode = HALMAC_TRNSFER_LOOPBACK_DELAY;
+		HALMAC_REG_WRITE_16(
+			halmac_adapter, REG_WMAC_LBK_BUF_HD_V1,
+			(u16)halmac_adapter->txff_allocation.rsvd_pg_bndy);
+	} else if (halmac_trx_mode == HALMAC_TRX_MODE_LOOPBACK) {
+		transfer_mode = HALMAC_TRNSFER_LOOPBACK_DIRECT;
+	} else {
+		transfer_mode = HALMAC_TRNSFER_NORMAL;
+	}
+
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_CR + 3, (u8)transfer_mode);
+
+	return HALMAC_RET_SUCCESS;
+}
diff --git a/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_func_8822b.h b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_func_8822b.h
new file mode 100644
index 000000000000..5ac2b15477c0
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_8822b/halmac_func_8822b.h
@@ -0,0 +1,38 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HALMAC_FUNC_8822B_H_
+#define _HALMAC_FUNC_8822B_H_
+
+#include "../../halmac_type.h"
+
+enum halmac_ret_status
+halmac_txdma_queue_mapping_8822b(struct halmac_adapter *halmac_adapter,
+				 enum halmac_trx_mode halmac_trx_mode);
+
+enum halmac_ret_status
+halmac_priority_queue_config_8822b(struct halmac_adapter *halmac_adapter,
+				   enum halmac_trx_mode halmac_trx_mode);
+
+#endif /* _HALMAC_FUNC_8822B_H_ */
diff --git a/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_88xx_cfg.h b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_88xx_cfg.h
new file mode 100644
index 000000000000..ea1206744902
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_88xx_cfg.h
@@ -0,0 +1,171 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HALMAC_88XX_CFG_H_
+#define _HALMAC_88XX_CFG_H_
+
+#include "../halmac_2_platform.h"
+#include "../halmac_type.h"
+#include "../halmac_api.h"
+#include "../halmac_bit2.h"
+#include "../halmac_reg2.h"
+#include "../halmac_pwr_seq_cmd.h"
+#include "halmac_func_88xx.h"
+#include "halmac_api_88xx.h"
+#include "halmac_api_88xx_usb.h"
+#include "halmac_api_88xx_pcie.h"
+#include "halmac_api_88xx_sdio.h"
+
+#define HALMAC_SVN_VER_88XX "13359M"
+
+#define HALMAC_MAJOR_VER_88XX 0x0001 /* major version, ver_1 for async_api */
+/* For halmac_api num change or prototype change, increment prototype version.
+ * Otherwise, increase minor version
+ */
+#define HALMAC_PROTOTYPE_VER_88XX 0x0003 /* prototype version */
+#define HALMAC_MINOR_VER_88XX 0x0005 /* minor version */
+#define HALMAC_PATCH_VER_88XX 0x0000 /* patch version */
+
+#define HALMAC_C2H_DATA_OFFSET_88XX 10
+#define HALMAC_RX_AGG_ALIGNMENT_SIZE_88XX 8
+#define HALMAC_TX_AGG_ALIGNMENT_SIZE_88XX 8
+#define HALMAC_TX_AGG_BUFF_SIZE_88XX 32768
+
+#define HALMAC_EXTRA_INFO_BUFF_SIZE_88XX 4096 /*4K*/
+#define HALMAC_EXTRA_INFO_BUFF_SIZE_FULL_FIFO_88XX 16384 /*16K*/
+#define HALMAC_FW_OFFLOAD_CMD_SIZE_88XX                                        \
+	12 /*Fw config parameter cmd size, each 12 byte*/
+
+#define HALMAC_H2C_CMD_ORIGINAL_SIZE_88XX 8
+#define HALMAC_H2C_CMD_SIZE_UNIT_88XX 32 /* Only support 32 byte packet now */
+
+#define HALMAC_NLO_INFO_SIZE_88XX 1024
+
+/* Download FW */
+#define HALMAC_FW_SIZE_MAX_88XX 0x40000
+#define HALMAC_FWHDR_SIZE_88XX 64
+#define HALMAC_FW_CHKSUM_DUMMY_SIZE_88XX 8
+#define HALMAC_FW_MAX_DL_SIZE_88XX 0x2000 /* need power of 2 */
+/* Max dlfw size can not over 31K, because SDIO HW restriction */
+#define HALMAC_FW_CFG_MAX_DL_SIZE_MAX_88XX 0x7C00
+
+#define DLFW_RESTORE_REG_NUM_88XX 9
+#define ID_INFORM_DLEMEM_RDY 0x80
+
+/* FW header information */
+#define HALMAC_FWHDR_OFFSET_VERSION_88XX 4
+#define HALMAC_FWHDR_OFFSET_SUBVERSION_88XX 6
+#define HALMAC_FWHDR_OFFSET_SUBINDEX_88XX 7
+#define HALMAC_FWHDR_OFFSET_MEM_USAGE_88XX 24
+#define HALMAC_FWHDR_OFFSET_H2C_FORMAT_VER_88XX 28
+#define HALMAC_FWHDR_OFFSET_DMEM_ADDR_88XX 32
+#define HALMAC_FWHDR_OFFSET_DMEM_SIZE_88XX 36
+#define HALMAC_FWHDR_OFFSET_IRAM_SIZE_88XX 48
+#define HALMAC_FWHDR_OFFSET_ERAM_SIZE_88XX 52
+#define HALMAC_FWHDR_OFFSET_EMEM_ADDR_88XX 56
+#define HALMAC_FWHDR_OFFSET_IRAM_ADDR_88XX 60
+
+/* HW memory address */
+#define HALMAC_OCPBASE_TXBUF_88XX 0x18780000
+#define HALMAC_OCPBASE_DMEM_88XX 0x00200000
+#define HALMAC_OCPBASE_IMEM_88XX 0x00000000
+
+/* define the SDIO Bus CLK threshold, for avoiding CMD53 fails that
+ * result from SDIO CLK sync to ana_clk fail
+ */
+#define HALMAC_SD_CLK_THRESHOLD_88XX 150000000 /* 150MHz */
+
+/* MAC clock */
+#define HALMAC_MAC_CLOCK_88XX 80 /* 80M */
+
+/* H2C/C2H*/
+#define HALMAC_H2C_CMD_SIZE_88XX 32
+#define HALMAC_H2C_CMD_HDR_SIZE_88XX 8
+
+#define HALMAC_PROTECTED_EFUSE_SIZE_88XX 0x60
+
+/* Function enable */
+#define HALMAC_FUNCTION_ENABLE_88XX 0xDC
+
+/* FIFO size & packet size */
+/* #define HALMAC_WOWLAN_PATTERN_SIZE	256 */
+
+/* CFEND rate */
+#define HALMAC_BASIC_CFEND_RATE_88XX 0x5
+#define HALMAC_STBC_CFEND_RATE_88XX 0xF
+
+/* Response rate */
+#define HALMAC_RESPONSE_RATE_BITMAP_ALL_88XX 0xFFFFF
+#define HALMAC_RESPONSE_RATE_88XX HALMAC_RESPONSE_RATE_BITMAP_ALL_88XX
+
+/* Spec SIFS */
+#define HALMAC_SIFS_CCK_PTCL_88XX 16
+#define HALMAC_SIFS_OFDM_PTCL_88XX 16
+
+/* Retry limit */
+#define HALMAC_LONG_RETRY_LIMIT_88XX 8
+#define HALMAC_SHORT_RETRY_LIMIT_88XX 7
+
+/* Slot, SIFS, PIFS time */
+#define HALMAC_SLOT_TIME_88XX 0x05
+#define HALMAC_PIFS_TIME_88XX 0x19
+#define HALMAC_SIFS_CCK_CTX_88XX 0xA
+#define HALMAC_SIFS_OFDM_CTX_88XX 0xA
+#define HALMAC_SIFS_CCK_TRX_88XX 0x10
+#define HALMAC_SIFS_OFDM_TRX_88XX 0x10
+
+/* TXOP limit */
+#define HALMAC_VO_TXOP_LIMIT_88XX 0x186
+#define HALMAC_VI_TXOP_LIMIT_88XX 0x3BC
+
+/* NAV */
+#define HALMAC_RDG_NAV_88XX 0x05
+#define HALMAC_TXOP_NAV_88XX 0x1B
+
+/* TSF */
+#define HALMAC_CCK_RX_TSF_88XX 0x30
+#define HALMAC_OFDM_RX_TSF_88XX 0x30
+
+/* Send beacon related */
+#define HALMAC_TBTT_PROHIBIT_88XX 0x04
+#define HALMAC_TBTT_HOLD_TIME_88XX 0x064
+#define HALMAC_DRIVER_EARLY_INT_88XX 0x04
+#define HALMAC_BEACON_DMA_TIM_88XX 0x02
+
+/* RX filter */
+#define HALMAC_RX_FILTER0_RECIVE_ALL_88XX 0xFFFFFFF
+#define HALMAC_RX_FILTER0_88XX HALMAC_RX_FILTER0_RECIVE_ALL_88XX
+#define HALMAC_RX_FILTER_RECIVE_ALL_88XX 0xFFFF
+#define HALMAC_RX_FILTER_88XX HALMAC_RX_FILTER_RECIVE_ALL_88XX
+
+/* RCR */
+#define HALMAC_RCR_CONFIG_88XX 0xE400631E
+
+/* Security config */
+#define HALMAC_SECURITY_CONFIG_88XX 0x01CC
+
+/* CCK rate ACK timeout */
+#define HALMAC_ACK_TO_CCK_88XX 0x40
+
+#endif
diff --git a/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_api_88xx.c b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_api_88xx.c
new file mode 100644
index 000000000000..5f84526cb5b5
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_api_88xx.c
@@ -0,0 +1,5979 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#include "halmac_88xx_cfg.h"
+
+/**
+ * halmac_init_adapter_para_88xx() - int halmac adapter
+ * @halmac_adapter
+ *
+ * SD1 internal use
+ *
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : void
+ */
+void halmac_init_adapter_para_88xx(struct halmac_adapter *halmac_adapter)
+{
+	halmac_adapter->api_record.array_wptr = 0;
+	halmac_adapter->hal_adapter_backup = halmac_adapter;
+	halmac_adapter->hal_efuse_map = (u8 *)NULL;
+	halmac_adapter->hal_efuse_map_valid = false;
+	halmac_adapter->efuse_end = 0;
+	halmac_adapter->hal_mac_addr[0].address_l_h.address_low = 0;
+	halmac_adapter->hal_mac_addr[0].address_l_h.address_high = 0;
+	halmac_adapter->hal_mac_addr[1].address_l_h.address_low = 0;
+	halmac_adapter->hal_mac_addr[1].address_l_h.address_high = 0;
+	halmac_adapter->hal_bss_addr[0].address_l_h.address_low = 0;
+	halmac_adapter->hal_bss_addr[0].address_l_h.address_high = 0;
+	halmac_adapter->hal_bss_addr[1].address_l_h.address_low = 0;
+	halmac_adapter->hal_bss_addr[1].address_l_h.address_high = 0;
+
+	halmac_adapter->low_clk = false;
+	halmac_adapter->max_download_size = HALMAC_FW_MAX_DL_SIZE_88XX;
+
+	/* Init LPS Option */
+	halmac_adapter->fwlps_option.mode = 0x01; /*0:Active 1:LPS 2:WMMPS*/
+	halmac_adapter->fwlps_option.awake_interval = 1;
+	halmac_adapter->fwlps_option.enter_32K = 1;
+	halmac_adapter->fwlps_option.clk_request = 0;
+	halmac_adapter->fwlps_option.rlbm = 0;
+	halmac_adapter->fwlps_option.smart_ps = 0;
+	halmac_adapter->fwlps_option.awake_interval = 1;
+	halmac_adapter->fwlps_option.all_queue_uapsd = 0;
+	halmac_adapter->fwlps_option.pwr_state = 0;
+	halmac_adapter->fwlps_option.low_pwr_rx_beacon = 0;
+	halmac_adapter->fwlps_option.ant_auto_switch = 0;
+	halmac_adapter->fwlps_option.ps_allow_bt_high_priority = 0;
+	halmac_adapter->fwlps_option.protect_bcn = 0;
+	halmac_adapter->fwlps_option.silence_period = 0;
+	halmac_adapter->fwlps_option.fast_bt_connect = 0;
+	halmac_adapter->fwlps_option.two_antenna_en = 0;
+	halmac_adapter->fwlps_option.adopt_user_setting = 1;
+	halmac_adapter->fwlps_option.drv_bcn_early_shift = 0;
+
+	halmac_adapter->config_para_info.cfg_para_buf = NULL;
+	halmac_adapter->config_para_info.para_buf_w = NULL;
+	halmac_adapter->config_para_info.para_num = 0;
+	halmac_adapter->config_para_info.full_fifo_mode = false;
+	halmac_adapter->config_para_info.para_buf_size = 0;
+	halmac_adapter->config_para_info.avai_para_buf_size = 0;
+	halmac_adapter->config_para_info.offset_accumulation = 0;
+	halmac_adapter->config_para_info.value_accumulation = 0;
+	halmac_adapter->config_para_info.datapack_segment = 0;
+
+	halmac_adapter->ch_sw_info.ch_info_buf = NULL;
+	halmac_adapter->ch_sw_info.ch_info_buf_w = NULL;
+	halmac_adapter->ch_sw_info.extra_info_en = 0;
+	halmac_adapter->ch_sw_info.buf_size = 0;
+	halmac_adapter->ch_sw_info.avai_buf_size = 0;
+	halmac_adapter->ch_sw_info.total_size = 0;
+	halmac_adapter->ch_sw_info.ch_num = 0;
+
+	halmac_adapter->drv_info_size = 0;
+
+	memset(halmac_adapter->api_record.api_array, HALMAC_API_STUFF,
+	       sizeof(halmac_adapter->api_record.api_array));
+
+	halmac_adapter->txff_allocation.tx_fifo_pg_num = 0;
+	halmac_adapter->txff_allocation.ac_q_pg_num = 0;
+	halmac_adapter->txff_allocation.rsvd_pg_bndy = 0;
+	halmac_adapter->txff_allocation.rsvd_drv_pg_bndy = 0;
+	halmac_adapter->txff_allocation.rsvd_h2c_extra_info_pg_bndy = 0;
+	halmac_adapter->txff_allocation.rsvd_h2c_queue_pg_bndy = 0;
+	halmac_adapter->txff_allocation.rsvd_cpu_instr_pg_bndy = 0;
+	halmac_adapter->txff_allocation.rsvd_fw_txbuff_pg_bndy = 0;
+	halmac_adapter->txff_allocation.pub_queue_pg_num = 0;
+	halmac_adapter->txff_allocation.high_queue_pg_num = 0;
+	halmac_adapter->txff_allocation.low_queue_pg_num = 0;
+	halmac_adapter->txff_allocation.normal_queue_pg_num = 0;
+	halmac_adapter->txff_allocation.extra_queue_pg_num = 0;
+
+	halmac_adapter->txff_allocation.la_mode = HALMAC_LA_MODE_DISABLE;
+	halmac_adapter->txff_allocation.rx_fifo_expanding_mode =
+		HALMAC_RX_FIFO_EXPANDING_MODE_DISABLE;
+
+	halmac_init_adapter_dynamic_para_88xx(halmac_adapter);
+	halmac_init_state_machine_88xx(halmac_adapter);
+}
+
+/**
+ * halmac_init_adapter_dynamic_para_88xx() - int halmac adapter
+ * @halmac_adapter
+ *
+ * SD1 internal use
+ *
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : void
+ */
+void halmac_init_adapter_dynamic_para_88xx(
+	struct halmac_adapter *halmac_adapter)
+{
+	halmac_adapter->h2c_packet_seq = 0;
+	halmac_adapter->h2c_buf_free_space = 0;
+	halmac_adapter->gen_info_valid = false;
+}
+
+/**
+ * halmac_init_state_machine_88xx() - init halmac software state machine
+ * @halmac_adapter
+ *
+ * SD1 internal use.
+ *
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : void
+ */
+void halmac_init_state_machine_88xx(struct halmac_adapter *halmac_adapter)
+{
+	struct halmac_state *state = &halmac_adapter->halmac_state;
+
+	halmac_init_offload_feature_state_machine_88xx(halmac_adapter);
+
+	state->api_state = HALMAC_API_STATE_INIT;
+
+	state->dlfw_state = HALMAC_DLFW_NONE;
+	state->mac_power = HALMAC_MAC_POWER_OFF;
+	state->ps_state = HALMAC_PS_STATE_UNDEFINE;
+}
+
+/**
+ * halmac_mount_api_88xx() - attach functions to function pointer
+ * @halmac_adapter
+ *
+ * SD1 internal use
+ *
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ */
+enum halmac_ret_status
+halmac_mount_api_88xx(struct halmac_adapter *halmac_adapter)
+{
+	void *driver_adapter = halmac_adapter->driver_adapter;
+	struct halmac_api *halmac_api = (struct halmac_api *)NULL;
+
+	halmac_adapter->halmac_api =
+		kzalloc(sizeof(struct halmac_api), GFP_KERNEL);
+	if (!halmac_adapter->halmac_api)
+		return HALMAC_RET_MALLOC_FAIL;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			HALMAC_SVN_VER_88XX "\n");
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"HALMAC_MAJOR_VER_88XX = %x\n", HALMAC_MAJOR_VER_88XX);
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"HALMAC_PROTOTYPE_88XX = %x\n",
+			HALMAC_PROTOTYPE_VER_88XX);
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"HALMAC_MINOR_VER_88XX = %x\n", HALMAC_MINOR_VER_88XX);
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"HALMAC_PATCH_VER_88XX = %x\n", HALMAC_PATCH_VER_88XX);
+
+	/* Mount function pointer */
+	halmac_api->halmac_download_firmware = halmac_download_firmware_88xx;
+	halmac_api->halmac_free_download_firmware =
+		halmac_free_download_firmware_88xx;
+	halmac_api->halmac_get_fw_version = halmac_get_fw_version_88xx;
+	halmac_api->halmac_cfg_mac_addr = halmac_cfg_mac_addr_88xx;
+	halmac_api->halmac_cfg_bssid = halmac_cfg_bssid_88xx;
+	halmac_api->halmac_cfg_multicast_addr = halmac_cfg_multicast_addr_88xx;
+	halmac_api->halmac_pre_init_system_cfg =
+		halmac_pre_init_system_cfg_88xx;
+	halmac_api->halmac_init_system_cfg = halmac_init_system_cfg_88xx;
+	halmac_api->halmac_init_edca_cfg = halmac_init_edca_cfg_88xx;
+	halmac_api->halmac_cfg_operation_mode = halmac_cfg_operation_mode_88xx;
+	halmac_api->halmac_cfg_ch_bw = halmac_cfg_ch_bw_88xx;
+	halmac_api->halmac_cfg_bw = halmac_cfg_bw_88xx;
+	halmac_api->halmac_init_wmac_cfg = halmac_init_wmac_cfg_88xx;
+	halmac_api->halmac_init_mac_cfg = halmac_init_mac_cfg_88xx;
+	halmac_api->halmac_init_sdio_cfg = halmac_init_sdio_cfg_88xx;
+	halmac_api->halmac_init_usb_cfg = halmac_init_usb_cfg_88xx;
+	halmac_api->halmac_init_pcie_cfg = halmac_init_pcie_cfg_88xx;
+	halmac_api->halmac_deinit_sdio_cfg = halmac_deinit_sdio_cfg_88xx;
+	halmac_api->halmac_deinit_usb_cfg = halmac_deinit_usb_cfg_88xx;
+	halmac_api->halmac_deinit_pcie_cfg = halmac_deinit_pcie_cfg_88xx;
+	halmac_api->halmac_dump_efuse_map = halmac_dump_efuse_map_88xx;
+	halmac_api->halmac_dump_efuse_map_bt = halmac_dump_efuse_map_bt_88xx;
+	halmac_api->halmac_write_efuse_bt = halmac_write_efuse_bt_88xx;
+	halmac_api->halmac_dump_logical_efuse_map =
+		halmac_dump_logical_efuse_map_88xx;
+	halmac_api->halmac_pg_efuse_by_map = halmac_pg_efuse_by_map_88xx;
+	halmac_api->halmac_get_efuse_size = halmac_get_efuse_size_88xx;
+	halmac_api->halmac_get_efuse_available_size =
+		halmac_get_efuse_available_size_88xx;
+	halmac_api->halmac_get_c2h_info = halmac_get_c2h_info_88xx;
+
+	halmac_api->halmac_get_logical_efuse_size =
+		halmac_get_logical_efuse_size_88xx;
+
+	halmac_api->halmac_write_logical_efuse =
+		halmac_write_logical_efuse_88xx;
+	halmac_api->halmac_read_logical_efuse = halmac_read_logical_efuse_88xx;
+
+	halmac_api->halmac_cfg_fwlps_option = halmac_cfg_fwlps_option_88xx;
+	halmac_api->halmac_cfg_fwips_option = halmac_cfg_fwips_option_88xx;
+	halmac_api->halmac_enter_wowlan = halmac_enter_wowlan_88xx;
+	halmac_api->halmac_leave_wowlan = halmac_leave_wowlan_88xx;
+	halmac_api->halmac_enter_ps = halmac_enter_ps_88xx;
+	halmac_api->halmac_leave_ps = halmac_leave_ps_88xx;
+	halmac_api->halmac_h2c_lb = halmac_h2c_lb_88xx;
+	halmac_api->halmac_debug = halmac_debug_88xx;
+	halmac_api->halmac_cfg_parameter = halmac_cfg_parameter_88xx;
+	halmac_api->halmac_update_datapack = halmac_update_datapack_88xx;
+	halmac_api->halmac_run_datapack = halmac_run_datapack_88xx;
+	halmac_api->halmac_cfg_drv_info = halmac_cfg_drv_info_88xx;
+	halmac_api->halmac_send_bt_coex = halmac_send_bt_coex_88xx;
+	halmac_api->halmac_verify_platform_api =
+		halmac_verify_platform_api_88xx;
+	halmac_api->halmac_update_packet = halmac_update_packet_88xx;
+	halmac_api->halmac_bcn_ie_filter = halmac_bcn_ie_filter_88xx;
+	halmac_api->halmac_cfg_txbf = halmac_cfg_txbf_88xx;
+	halmac_api->halmac_cfg_mumimo = halmac_cfg_mumimo_88xx;
+	halmac_api->halmac_cfg_sounding = halmac_cfg_sounding_88xx;
+	halmac_api->halmac_del_sounding = halmac_del_sounding_88xx;
+	halmac_api->halmac_su_bfer_entry_init = halmac_su_bfer_entry_init_88xx;
+	halmac_api->halmac_su_bfee_entry_init = halmac_su_bfee_entry_init_88xx;
+	halmac_api->halmac_mu_bfer_entry_init = halmac_mu_bfer_entry_init_88xx;
+	halmac_api->halmac_mu_bfee_entry_init = halmac_mu_bfee_entry_init_88xx;
+	halmac_api->halmac_su_bfer_entry_del = halmac_su_bfer_entry_del_88xx;
+	halmac_api->halmac_su_bfee_entry_del = halmac_su_bfee_entry_del_88xx;
+	halmac_api->halmac_mu_bfer_entry_del = halmac_mu_bfer_entry_del_88xx;
+	halmac_api->halmac_mu_bfee_entry_del = halmac_mu_bfee_entry_del_88xx;
+
+	halmac_api->halmac_add_ch_info = halmac_add_ch_info_88xx;
+	halmac_api->halmac_add_extra_ch_info = halmac_add_extra_ch_info_88xx;
+	halmac_api->halmac_ctrl_ch_switch = halmac_ctrl_ch_switch_88xx;
+	halmac_api->halmac_p2pps = halmac_p2pps_88xx;
+	halmac_api->halmac_clear_ch_info = halmac_clear_ch_info_88xx;
+	halmac_api->halmac_send_general_info = halmac_send_general_info_88xx;
+
+	halmac_api->halmac_start_iqk = halmac_start_iqk_88xx;
+	halmac_api->halmac_ctrl_pwr_tracking = halmac_ctrl_pwr_tracking_88xx;
+	halmac_api->halmac_psd = halmac_psd_88xx;
+	halmac_api->halmac_cfg_la_mode = halmac_cfg_la_mode_88xx;
+	halmac_api->halmac_cfg_rx_fifo_expanding_mode =
+		halmac_cfg_rx_fifo_expanding_mode_88xx;
+
+	halmac_api->halmac_config_security = halmac_config_security_88xx;
+	halmac_api->halmac_get_used_cam_entry_num =
+		halmac_get_used_cam_entry_num_88xx;
+	halmac_api->halmac_read_cam_entry = halmac_read_cam_entry_88xx;
+	halmac_api->halmac_write_cam = halmac_write_cam_88xx;
+	halmac_api->halmac_clear_cam_entry = halmac_clear_cam_entry_88xx;
+
+	halmac_api->halmac_get_hw_value = halmac_get_hw_value_88xx;
+	halmac_api->halmac_set_hw_value = halmac_set_hw_value_88xx;
+
+	halmac_api->halmac_cfg_drv_rsvd_pg_num =
+		halmac_cfg_drv_rsvd_pg_num_88xx;
+	halmac_api->halmac_get_chip_version = halmac_get_chip_version_88xx;
+
+	halmac_api->halmac_query_status = halmac_query_status_88xx;
+	halmac_api->halmac_reset_feature = halmac_reset_feature_88xx;
+	halmac_api->halmac_check_fw_status = halmac_check_fw_status_88xx;
+	halmac_api->halmac_dump_fw_dmem = halmac_dump_fw_dmem_88xx;
+	halmac_api->halmac_cfg_max_dl_size = halmac_cfg_max_dl_size_88xx;
+
+	halmac_api->halmac_dump_fifo = halmac_dump_fifo_88xx;
+	halmac_api->halmac_get_fifo_size = halmac_get_fifo_size_88xx;
+
+	halmac_api->halmac_chk_txdesc = halmac_chk_txdesc_88xx;
+	halmac_api->halmac_dl_drv_rsvd_page = halmac_dl_drv_rsvd_page_88xx;
+	halmac_api->halmac_cfg_csi_rate = halmac_cfg_csi_rate_88xx;
+
+	halmac_api->halmac_sdio_cmd53_4byte = halmac_sdio_cmd53_4byte_88xx;
+	halmac_api->halmac_txfifo_is_empty = halmac_txfifo_is_empty_88xx;
+
+	if (halmac_adapter->halmac_interface == HALMAC_INTERFACE_SDIO) {
+		halmac_api->halmac_cfg_rx_aggregation =
+			halmac_cfg_rx_aggregation_88xx_sdio;
+		halmac_api->halmac_init_interface_cfg =
+			halmac_init_sdio_cfg_88xx;
+		halmac_api->halmac_deinit_interface_cfg =
+			halmac_deinit_sdio_cfg_88xx;
+		halmac_api->halmac_reg_read_8 = halmac_reg_read_8_sdio_88xx;
+		halmac_api->halmac_reg_write_8 = halmac_reg_write_8_sdio_88xx;
+		halmac_api->halmac_reg_read_16 = halmac_reg_read_16_sdio_88xx;
+		halmac_api->halmac_reg_write_16 = halmac_reg_write_16_sdio_88xx;
+		halmac_api->halmac_reg_read_32 = halmac_reg_read_32_sdio_88xx;
+		halmac_api->halmac_reg_write_32 = halmac_reg_write_32_sdio_88xx;
+		halmac_api->halmac_reg_read_indirect_32 =
+			halmac_reg_read_indirect_32_sdio_88xx;
+		halmac_api->halmac_reg_sdio_cmd53_read_n =
+			halmac_reg_read_nbyte_sdio_88xx;
+	} else if (halmac_adapter->halmac_interface == HALMAC_INTERFACE_USB) {
+		halmac_api->halmac_cfg_rx_aggregation =
+			halmac_cfg_rx_aggregation_88xx_usb;
+		halmac_api->halmac_init_interface_cfg =
+			halmac_init_usb_cfg_88xx;
+		halmac_api->halmac_deinit_interface_cfg =
+			halmac_deinit_usb_cfg_88xx;
+		halmac_api->halmac_reg_read_8 = halmac_reg_read_8_usb_88xx;
+		halmac_api->halmac_reg_write_8 = halmac_reg_write_8_usb_88xx;
+		halmac_api->halmac_reg_read_16 = halmac_reg_read_16_usb_88xx;
+		halmac_api->halmac_reg_write_16 = halmac_reg_write_16_usb_88xx;
+		halmac_api->halmac_reg_read_32 = halmac_reg_read_32_usb_88xx;
+		halmac_api->halmac_reg_write_32 = halmac_reg_write_32_usb_88xx;
+	} else if (halmac_adapter->halmac_interface == HALMAC_INTERFACE_PCIE) {
+		halmac_api->halmac_cfg_rx_aggregation =
+			halmac_cfg_rx_aggregation_88xx_pcie;
+		halmac_api->halmac_init_interface_cfg =
+			halmac_init_pcie_cfg_88xx;
+		halmac_api->halmac_deinit_interface_cfg =
+			halmac_deinit_pcie_cfg_88xx;
+		halmac_api->halmac_reg_read_8 = halmac_reg_read_8_pcie_88xx;
+		halmac_api->halmac_reg_write_8 = halmac_reg_write_8_pcie_88xx;
+		halmac_api->halmac_reg_read_16 = halmac_reg_read_16_pcie_88xx;
+		halmac_api->halmac_reg_write_16 = halmac_reg_write_16_pcie_88xx;
+		halmac_api->halmac_reg_read_32 = halmac_reg_read_32_pcie_88xx;
+		halmac_api->halmac_reg_write_32 = halmac_reg_write_32_pcie_88xx;
+	} else {
+		pr_err("Set halmac io function Error!!\n");
+	}
+
+	halmac_api->halmac_set_bulkout_num = halmac_set_bulkout_num_88xx;
+	halmac_api->halmac_get_sdio_tx_addr = halmac_get_sdio_tx_addr_88xx;
+	halmac_api->halmac_get_usb_bulkout_id = halmac_get_usb_bulkout_id_88xx;
+	halmac_api->halmac_timer_2s = halmac_timer_2s_88xx;
+	halmac_api->halmac_fill_txdesc_checksum =
+		halmac_fill_txdesc_check_sum_88xx;
+
+	if (halmac_adapter->chip_id == HALMAC_CHIP_ID_8822B) {
+		/*mount 8822b function and data*/
+		halmac_mount_api_8822b(halmac_adapter);
+
+	} else if (halmac_adapter->chip_id == HALMAC_CHIP_ID_8821C) {
+	} else if (halmac_adapter->chip_id == HALMAC_CHIP_ID_8814B) {
+	} else if (halmac_adapter->chip_id == HALMAC_CHIP_ID_8197F) {
+	} else {
+		pr_err("Chip ID undefine!!\n");
+		return HALMAC_RET_CHIP_NOT_SUPPORT;
+	}
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_download_firmware_88xx() - download Firmware
+ * @halmac_adapter : the adapter of halmac
+ * @hamacl_fw : firmware bin
+ * @halmac_fw_size : firmware size
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_download_firmware_88xx(struct halmac_adapter *halmac_adapter,
+			      u8 *hamacl_fw, u32 halmac_fw_size)
+{
+	u8 value8;
+	u8 *file_ptr;
+	u32 dest;
+	u16 value16;
+	u32 restore_index = 0;
+	u32 halmac_h2c_ver = 0, fw_h2c_ver = 0;
+	u32 iram_pkt_size, dmem_pkt_size, eram_pkt_size = 0;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	struct halmac_restore_info restore_info[DLFW_RESTORE_REG_NUM_88XX];
+	u32 temp;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_DOWNLOAD_FIRMWARE);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s start!!\n", __func__);
+
+	if (halmac_fw_size > HALMAC_FW_SIZE_MAX_88XX ||
+	    halmac_fw_size < HALMAC_FWHDR_SIZE_88XX) {
+		pr_err("FW size error!\n");
+		return HALMAC_RET_FW_SIZE_ERR;
+	}
+
+	fw_h2c_ver = le32_to_cpu(
+		*((__le32 *)
+		  (hamacl_fw + HALMAC_FWHDR_OFFSET_H2C_FORMAT_VER_88XX)));
+	halmac_h2c_ver = H2C_FORMAT_VERSION;
+	HALMAC_RT_TRACE(
+		driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+		"halmac h2c/c2h format = %x, fw h2c/c2h format = %x!!\n",
+		halmac_h2c_ver, fw_h2c_ver);
+	if (fw_h2c_ver != halmac_h2c_ver)
+		HALMAC_RT_TRACE(
+			driver_adapter, HALMAC_MSG_INIT, DBG_WARNING,
+			"[WARN]H2C/C2H version between HALMAC and FW is compatible!!\n");
+
+	halmac_adapter->halmac_state.dlfw_state = HALMAC_DLFW_NONE;
+
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_SYS_FUNC_EN + 1);
+	value8 = (u8)(value8 & ~(BIT(2)));
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_SYS_FUNC_EN + 1,
+			   value8); /* Disable CPU reset */
+
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_RSV_CTRL + 1);
+	value8 = (u8)(value8 & ~(BIT(0)));
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_RSV_CTRL + 1, value8);
+
+	restore_info[restore_index].length = 1;
+	restore_info[restore_index].mac_register = REG_TXDMA_PQ_MAP + 1;
+	restore_info[restore_index].value =
+		HALMAC_REG_READ_8(halmac_adapter, REG_TXDMA_PQ_MAP + 1);
+	restore_index++;
+	value8 = HALMAC_DMA_MAPPING_HIGH << 6;
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_TXDMA_PQ_MAP + 1,
+			   value8); /* set HIQ to hi priority */
+
+	/* DLFW only use HIQ, map HIQ to hi priority */
+	halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_HI] =
+		HALMAC_DMA_MAPPING_HIGH;
+	restore_info[restore_index].length = 1;
+	restore_info[restore_index].mac_register = REG_CR;
+	restore_info[restore_index].value =
+		HALMAC_REG_READ_8(halmac_adapter, REG_CR);
+	restore_index++;
+	restore_info[restore_index].length = 4;
+	restore_info[restore_index].mac_register = REG_H2CQ_CSR;
+	restore_info[restore_index].value = BIT(31);
+	restore_index++;
+	value8 = BIT_HCI_TXDMA_EN | BIT_TXDMA_EN;
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_CR, value8);
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_H2CQ_CSR, BIT(31));
+
+	/* Config hi priority queue and public priority queue page number
+	 * (only for DLFW)
+	 */
+	restore_info[restore_index].length = 2;
+	restore_info[restore_index].mac_register = REG_FIFOPAGE_INFO_1;
+	restore_info[restore_index].value =
+		HALMAC_REG_READ_16(halmac_adapter, REG_FIFOPAGE_INFO_1);
+	restore_index++;
+	restore_info[restore_index].length = 4;
+	restore_info[restore_index].mac_register = REG_RQPN_CTRL_2;
+	restore_info[restore_index].value =
+		HALMAC_REG_READ_32(halmac_adapter, REG_RQPN_CTRL_2) | BIT(31);
+	restore_index++;
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_FIFOPAGE_INFO_1, 0x200);
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_RQPN_CTRL_2,
+			    restore_info[restore_index - 1].value);
+
+	if (halmac_adapter->halmac_interface == HALMAC_INTERFACE_SDIO) {
+		HALMAC_REG_READ_32(halmac_adapter, REG_SDIO_FREE_TXPG);
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_SDIO_TX_CTRL,
+				    0x00000000);
+	}
+
+	halmac_adapter->fw_version.version = le16_to_cpu(
+		*((__le16 *)(hamacl_fw + HALMAC_FWHDR_OFFSET_VERSION_88XX)));
+	halmac_adapter->fw_version.sub_version =
+		*(hamacl_fw + HALMAC_FWHDR_OFFSET_SUBVERSION_88XX);
+	halmac_adapter->fw_version.sub_index =
+		*(hamacl_fw + HALMAC_FWHDR_OFFSET_SUBINDEX_88XX);
+	halmac_adapter->fw_version.h2c_version = (u16)fw_h2c_ver;
+
+	dmem_pkt_size = le32_to_cpu(*((__le32 *)(hamacl_fw +
+				      HALMAC_FWHDR_OFFSET_DMEM_SIZE_88XX)));
+	iram_pkt_size = le32_to_cpu(*((__le32 *)(hamacl_fw +
+				      HALMAC_FWHDR_OFFSET_IRAM_SIZE_88XX)));
+	if (((*(hamacl_fw + HALMAC_FWHDR_OFFSET_MEM_USAGE_88XX)) & BIT(4)) != 0)
+		eram_pkt_size =
+		     le32_to_cpu(*((__le32 *)(hamacl_fw +
+				   HALMAC_FWHDR_OFFSET_ERAM_SIZE_88XX)));
+
+	dmem_pkt_size += HALMAC_FW_CHKSUM_DUMMY_SIZE_88XX;
+	iram_pkt_size += HALMAC_FW_CHKSUM_DUMMY_SIZE_88XX;
+	if (eram_pkt_size != 0)
+		eram_pkt_size += HALMAC_FW_CHKSUM_DUMMY_SIZE_88XX;
+
+	if (halmac_fw_size != (HALMAC_FWHDR_SIZE_88XX + dmem_pkt_size +
+			       iram_pkt_size + eram_pkt_size)) {
+		pr_err("FW size mismatch the real fw size!\n");
+		goto DLFW_FAIL;
+	}
+
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_CR + 1);
+	restore_info[restore_index].length = 1;
+	restore_info[restore_index].mac_register = REG_CR + 1;
+	restore_info[restore_index].value = value8;
+	restore_index++;
+	value8 = (u8)(value8 | BIT(0));
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_CR + 1,
+			   value8); /* Enable SW TX beacon */
+
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_BCN_CTRL);
+	restore_info[restore_index].length = 1;
+	restore_info[restore_index].mac_register = REG_BCN_CTRL;
+	restore_info[restore_index].value = value8;
+	restore_index++;
+	value8 = (u8)((value8 & (~BIT(3))) | BIT(4));
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_BCN_CTRL,
+			   value8); /* Disable beacon related functions */
+
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_FWHW_TXQ_CTRL + 2);
+	restore_info[restore_index].length = 1;
+	restore_info[restore_index].mac_register = REG_FWHW_TXQ_CTRL + 2;
+	restore_info[restore_index].value = value8;
+	restore_index++;
+	value8 = (u8)(value8 & ~(BIT(6)));
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_FWHW_TXQ_CTRL + 2,
+			   value8); /* Disable ptcl tx bcnq */
+
+	restore_info[restore_index].length = 2;
+	restore_info[restore_index].mac_register = REG_FIFOPAGE_CTRL_2;
+	restore_info[restore_index].value =
+		HALMAC_REG_READ_16(halmac_adapter, REG_FIFOPAGE_CTRL_2) |
+		BIT(15);
+	restore_index++;
+	value16 = 0x8000;
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_FIFOPAGE_CTRL_2,
+			    value16); /* Set beacon header to  0 */
+
+	value16 = (u16)(HALMAC_REG_READ_16(halmac_adapter, REG_MCUFW_CTRL) &
+			0x3800);
+	value16 |= BIT(0);
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_MCUFW_CTRL,
+			    value16); /* MCU/FW setting */
+
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_CPU_DMEM_CON + 2);
+	value8 &= ~(BIT(0));
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_CPU_DMEM_CON + 2, value8);
+	value8 |= BIT(0);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_CPU_DMEM_CON + 2, value8);
+
+	/* Download to DMEM */
+	file_ptr = hamacl_fw + HALMAC_FWHDR_SIZE_88XX;
+	temp = le32_to_cpu(*((__le32 *)(hamacl_fw +
+			   HALMAC_FWHDR_OFFSET_DMEM_ADDR_88XX))) &
+			   ~(BIT(31));
+	if (halmac_dlfw_to_mem_88xx(halmac_adapter, file_ptr, temp,
+				    dmem_pkt_size) != HALMAC_RET_SUCCESS)
+		goto DLFW_END;
+
+	/* Download to IMEM */
+	file_ptr = hamacl_fw + HALMAC_FWHDR_SIZE_88XX + dmem_pkt_size;
+	temp = le32_to_cpu(*((__le32 *)(hamacl_fw +
+			   HALMAC_FWHDR_OFFSET_IRAM_ADDR_88XX))) &
+			   ~(BIT(31));
+	if (halmac_dlfw_to_mem_88xx(halmac_adapter, file_ptr, temp,
+				    iram_pkt_size) != HALMAC_RET_SUCCESS)
+		goto DLFW_END;
+
+	/* Download to EMEM */
+	if (eram_pkt_size != 0) {
+		file_ptr = hamacl_fw + HALMAC_FWHDR_SIZE_88XX + dmem_pkt_size +
+			   iram_pkt_size;
+		dest = le32_to_cpu((*((__le32 *)(hamacl_fw +
+				    HALMAC_FWHDR_OFFSET_EMEM_ADDR_88XX)))) &
+				   ~(BIT(31));
+		if (halmac_dlfw_to_mem_88xx(halmac_adapter, file_ptr, dest,
+					    eram_pkt_size) !=
+		    HALMAC_RET_SUCCESS)
+			goto DLFW_END;
+	}
+
+	halmac_init_offload_feature_state_machine_88xx(halmac_adapter);
+DLFW_END:
+
+	halmac_restore_mac_register_88xx(halmac_adapter, restore_info,
+					 DLFW_RESTORE_REG_NUM_88XX);
+
+	if (halmac_dlfw_end_flow_88xx(halmac_adapter) != HALMAC_RET_SUCCESS)
+		goto DLFW_FAIL;
+
+	halmac_adapter->halmac_state.dlfw_state = HALMAC_DLFW_DONE;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+
+DLFW_FAIL:
+
+	/* Disable FWDL_EN */
+	HALMAC_REG_WRITE_8(
+		halmac_adapter, REG_MCUFW_CTRL,
+		(u8)(HALMAC_REG_READ_8(halmac_adapter, REG_MCUFW_CTRL) &
+		     ~(BIT(0))));
+
+	return HALMAC_RET_DLFW_FAIL;
+}
+
+/**
+ * halmac_free_download_firmware_88xx() - download specific memory firmware
+ * @halmac_adapter
+ * @dlfw_mem : memory selection
+ * @hamacl_fw : firmware bin
+ * @halmac_fw_size : firmware size
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ */
+enum halmac_ret_status
+halmac_free_download_firmware_88xx(struct halmac_adapter *halmac_adapter,
+				   enum halmac_dlfw_mem dlfw_mem, u8 *hamacl_fw,
+				   u32 halmac_fw_size)
+{
+	u8 tx_pause_backup;
+	u8 *file_ptr;
+	u32 dest;
+	u16 bcn_head_backup;
+	u32 iram_pkt_size, dmem_pkt_size, eram_pkt_size = 0;
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_DLFW_FAIL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	if (halmac_fw_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_NO_DLFW;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"[TRACE]%s ==========>\n", __func__);
+
+	if (halmac_fw_size > HALMAC_FW_SIZE_MAX_88XX ||
+	    halmac_fw_size < HALMAC_FWHDR_SIZE_88XX) {
+		pr_err("[ERR]FW size error!\n");
+		return HALMAC_RET_FW_SIZE_ERR;
+	}
+
+	dmem_pkt_size =
+	    le32_to_cpu(*(__le32 *)(hamacl_fw +
+				    HALMAC_FWHDR_OFFSET_DMEM_SIZE_88XX));
+	iram_pkt_size =
+	    le32_to_cpu(*(__le32 *)(hamacl_fw +
+				    HALMAC_FWHDR_OFFSET_IRAM_SIZE_88XX));
+	if (((*(hamacl_fw + HALMAC_FWHDR_OFFSET_MEM_USAGE_88XX)) & BIT(4)) != 0)
+		eram_pkt_size =
+		  le32_to_cpu(*(__le32 *)(hamacl_fw +
+					  HALMAC_FWHDR_OFFSET_ERAM_SIZE_88XX));
+
+	dmem_pkt_size += HALMAC_FW_CHKSUM_DUMMY_SIZE_88XX;
+	iram_pkt_size += HALMAC_FW_CHKSUM_DUMMY_SIZE_88XX;
+	if (eram_pkt_size != 0)
+		eram_pkt_size += HALMAC_FW_CHKSUM_DUMMY_SIZE_88XX;
+
+	if (halmac_fw_size != (HALMAC_FWHDR_SIZE_88XX + dmem_pkt_size +
+			       iram_pkt_size + eram_pkt_size)) {
+		pr_err("[ERR]FW size mismatch the real fw size!\n");
+		return HALMAC_RET_DLFW_FAIL;
+	}
+
+	tx_pause_backup = HALMAC_REG_READ_8(halmac_adapter, REG_TXPAUSE);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_TXPAUSE,
+			   tx_pause_backup | BIT(7));
+
+	bcn_head_backup =
+		HALMAC_REG_READ_16(halmac_adapter, REG_FIFOPAGE_CTRL_2) |
+		BIT(15);
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_FIFOPAGE_CTRL_2, 0x8000);
+
+	if (eram_pkt_size != 0) {
+		file_ptr = hamacl_fw + HALMAC_FWHDR_SIZE_88XX + dmem_pkt_size +
+			   iram_pkt_size;
+		dest = le32_to_cpu(*((__le32 *)(hamacl_fw +
+				   HALMAC_FWHDR_OFFSET_EMEM_ADDR_88XX))) &
+				   ~(BIT(31));
+		status = halmac_dlfw_to_mem_88xx(halmac_adapter, file_ptr, dest,
+						 eram_pkt_size);
+		if (status != HALMAC_RET_SUCCESS)
+			goto DL_FREE_FW_END;
+	}
+
+	status = halmac_free_dl_fw_end_flow_88xx(halmac_adapter);
+
+DL_FREE_FW_END:
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_TXPAUSE, tx_pause_backup);
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_FIFOPAGE_CTRL_2,
+			    bcn_head_backup);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"[TRACE]%s <==========\n", __func__);
+
+	return status;
+}
+
+/**
+ * halmac_get_fw_version_88xx() - get FW version
+ * @halmac_adapter : the adapter of halmac
+ * @fw_version : fw version info
+ * Author : Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_get_fw_version_88xx(struct halmac_adapter *halmac_adapter,
+			   struct halmac_fw_version *fw_version)
+{
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_adapter->halmac_state.dlfw_state == 0)
+		return HALMAC_RET_DLFW_FAIL;
+
+	fw_version->version = halmac_adapter->fw_version.version;
+	fw_version->sub_version = halmac_adapter->fw_version.sub_version;
+	fw_version->sub_index = halmac_adapter->fw_version.sub_index;
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_cfg_mac_addr_88xx() - config mac address
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_port :0 for port0, 1 for port1, 2 for port2, 3 for port3, 4 for port4
+ * @hal_address : mac address
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_cfg_mac_addr_88xx(struct halmac_adapter *halmac_adapter, u8 halmac_port,
+			 union halmac_wlan_addr *hal_address)
+{
+	u16 mac_address_H;
+	u32 mac_address_L;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"[TRACE]%s ==========>\n", __func__);
+
+	if (halmac_port >= HALMAC_PORTIDMAX) {
+		pr_err("[ERR]port index > 5\n");
+		return HALMAC_RET_PORT_NOT_SUPPORT;
+	}
+
+	mac_address_L = le32_to_cpu(hal_address->address_l_h.le_address_low);
+	mac_address_H = le16_to_cpu(hal_address->address_l_h.le_address_high);
+
+	halmac_adapter->hal_mac_addr[halmac_port].address_l_h.address_low =
+		mac_address_L;
+	halmac_adapter->hal_mac_addr[halmac_port].address_l_h.address_high =
+		mac_address_H;
+
+	switch (halmac_port) {
+	case HALMAC_PORTID0:
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_MACID, mac_address_L);
+		HALMAC_REG_WRITE_16(halmac_adapter, REG_MACID + 4,
+				    mac_address_H);
+		break;
+
+	case HALMAC_PORTID1:
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_MACID1, mac_address_L);
+		HALMAC_REG_WRITE_16(halmac_adapter, REG_MACID1 + 4,
+				    mac_address_H);
+		break;
+
+	case HALMAC_PORTID2:
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_MACID2, mac_address_L);
+		HALMAC_REG_WRITE_16(halmac_adapter, REG_MACID2 + 4,
+				    mac_address_H);
+		break;
+
+	case HALMAC_PORTID3:
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_MACID3, mac_address_L);
+		HALMAC_REG_WRITE_16(halmac_adapter, REG_MACID3 + 4,
+				    mac_address_H);
+		break;
+
+	case HALMAC_PORTID4:
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_MACID4, mac_address_L);
+		HALMAC_REG_WRITE_16(halmac_adapter, REG_MACID4 + 4,
+				    mac_address_H);
+		break;
+
+	default:
+
+		break;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"[TRACE]%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_cfg_bssid_88xx() - config BSSID
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_port :0 for port0, 1 for port1, 2 for port2, 3 for port3, 4 for port4
+ * @hal_address : bssid
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_cfg_bssid_88xx(struct halmac_adapter *halmac_adapter, u8 halmac_port,
+		      union halmac_wlan_addr *hal_address)
+{
+	u16 bssid_address_H;
+	u32 bssid_address_L;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"[TRACE]%s ==========>\n", __func__);
+
+	if (halmac_port >= HALMAC_PORTIDMAX) {
+		pr_err("[ERR]port index > 5\n");
+		return HALMAC_RET_PORT_NOT_SUPPORT;
+	}
+
+	bssid_address_L = le32_to_cpu(hal_address->address_l_h.le_address_low);
+	bssid_address_H = le16_to_cpu(hal_address->address_l_h.le_address_high);
+
+	halmac_adapter->hal_bss_addr[halmac_port].address_l_h.address_low =
+		bssid_address_L;
+	halmac_adapter->hal_bss_addr[halmac_port].address_l_h.address_high =
+		bssid_address_H;
+
+	switch (halmac_port) {
+	case HALMAC_PORTID0:
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_BSSID, bssid_address_L);
+		HALMAC_REG_WRITE_16(halmac_adapter, REG_BSSID + 4,
+				    bssid_address_H);
+		break;
+
+	case HALMAC_PORTID1:
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_BSSID1,
+				    bssid_address_L);
+		HALMAC_REG_WRITE_16(halmac_adapter, REG_BSSID1 + 4,
+				    bssid_address_H);
+		break;
+
+	case HALMAC_PORTID2:
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_BSSID2,
+				    bssid_address_L);
+		HALMAC_REG_WRITE_16(halmac_adapter, REG_BSSID2 + 4,
+				    bssid_address_H);
+		break;
+
+	case HALMAC_PORTID3:
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_BSSID3,
+				    bssid_address_L);
+		HALMAC_REG_WRITE_16(halmac_adapter, REG_BSSID3 + 4,
+				    bssid_address_H);
+		break;
+
+	case HALMAC_PORTID4:
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_BSSID4,
+				    bssid_address_L);
+		HALMAC_REG_WRITE_16(halmac_adapter, REG_BSSID4 + 4,
+				    bssid_address_H);
+		break;
+
+	default:
+
+		break;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"[TRACE]%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_cfg_multicast_addr_88xx() - config multicast address
+ * @halmac_adapter : the adapter of halmac
+ * @hal_address : multicast address
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_cfg_multicast_addr_88xx(struct halmac_adapter *halmac_adapter,
+			       union halmac_wlan_addr *hal_address)
+{
+	u16 address_H;
+	u32 address_L;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter,
+				  HALMAC_API_CFG_MULTICAST_ADDR);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	address_L = le32_to_cpu(hal_address->address_l_h.le_address_low);
+	address_H = le16_to_cpu(hal_address->address_l_h.le_address_high);
+
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_MAR, address_L);
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_MAR + 4, address_H);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_pre_init_system_cfg_88xx() - pre-init system config
+ * @halmac_adapter : the adapter of halmac
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_pre_init_system_cfg_88xx(struct halmac_adapter *halmac_adapter)
+{
+	u32 value32, counter;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	bool enable_bb;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter,
+				  HALMAC_API_PRE_INIT_SYSTEM_CFG);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"halmac_pre_init_system_cfg ==========>\n");
+
+	if (halmac_adapter->halmac_interface == HALMAC_INTERFACE_SDIO) {
+		HALMAC_REG_WRITE_8(
+			halmac_adapter, REG_SDIO_HSUS_CTRL,
+			HALMAC_REG_READ_8(halmac_adapter, REG_SDIO_HSUS_CTRL) &
+				~(BIT(0)));
+		counter = 10000;
+		while (!(HALMAC_REG_READ_8(halmac_adapter, REG_SDIO_HSUS_CTRL) &
+			 0x02)) {
+			counter--;
+			if (counter == 0)
+				return HALMAC_RET_SDIO_LEAVE_SUSPEND_FAIL;
+		}
+	} else if (halmac_adapter->halmac_interface == HALMAC_INTERFACE_USB) {
+		if (HALMAC_REG_READ_8(halmac_adapter, REG_SYS_CFG2 + 3) ==
+		    0x20) /* usb3.0 */
+			HALMAC_REG_WRITE_8(
+				halmac_adapter, 0xFE5B,
+				HALMAC_REG_READ_8(halmac_adapter, 0xFE5B) |
+					BIT(4));
+	}
+
+	/* Config PIN Mux */
+	value32 = HALMAC_REG_READ_32(halmac_adapter, REG_PAD_CTRL1);
+	value32 = value32 & (~(BIT(28) | BIT(29)));
+	value32 = value32 | BIT(28) | BIT(29);
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_PAD_CTRL1, value32);
+
+	value32 = HALMAC_REG_READ_32(halmac_adapter, REG_LED_CFG);
+	value32 = value32 & (~(BIT(25) | BIT(26)));
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_LED_CFG, value32);
+
+	value32 = HALMAC_REG_READ_32(halmac_adapter, REG_GPIO_MUXCFG);
+	value32 = value32 & (~(BIT(2)));
+	value32 = value32 | BIT(2);
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_GPIO_MUXCFG, value32);
+
+	enable_bb = false;
+	halmac_set_hw_value_88xx(halmac_adapter, HALMAC_HW_EN_BB_RF,
+				 &enable_bb);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"halmac_pre_init_system_cfg <==========\n");
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_init_system_cfg_88xx() -  init system config
+ * @halmac_adapter : the adapter of halmac
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_init_system_cfg_88xx(struct halmac_adapter *halmac_adapter)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_INIT_SYSTEM_CFG);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"halmac_init_system_cfg ==========>\n");
+
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_SYS_FUNC_EN + 1,
+			   HALMAC_FUNCTION_ENABLE_88XX);
+	HALMAC_REG_WRITE_32(
+		halmac_adapter, REG_SYS_SDIO_CTRL,
+		(u32)(HALMAC_REG_READ_32(halmac_adapter, REG_SYS_SDIO_CTRL) |
+		      BIT_LTE_MUX_CTRL_PATH));
+	HALMAC_REG_WRITE_32(
+		halmac_adapter, REG_CPU_DMEM_CON,
+		(u32)(HALMAC_REG_READ_32(halmac_adapter, REG_CPU_DMEM_CON) |
+		      BIT_WL_PLATFORM_RST));
+
+	/* halmac_api->halmac_init_h2c(halmac_adapter); */
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"halmac_init_system_cfg <==========\n");
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_init_edca_cfg_88xx() - init EDCA config
+ * @halmac_adapter : the adapter of halmac
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_init_edca_cfg_88xx(struct halmac_adapter *halmac_adapter)
+{
+	u8 value8;
+	u32 value32;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_INIT_EDCA_CFG);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	/* Clear TX pause */
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_TXPAUSE, 0x0000);
+
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_SLOT, HALMAC_SLOT_TIME_88XX);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_PIFS, HALMAC_PIFS_TIME_88XX);
+	value32 = HALMAC_SIFS_CCK_CTX_88XX |
+		  (HALMAC_SIFS_OFDM_CTX_88XX << BIT_SHIFT_SIFS_OFDM_CTX) |
+		  (HALMAC_SIFS_CCK_TRX_88XX << BIT_SHIFT_SIFS_CCK_TRX) |
+		  (HALMAC_SIFS_OFDM_TRX_88XX << BIT_SHIFT_SIFS_OFDM_TRX);
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_SIFS, value32);
+
+	HALMAC_REG_WRITE_32(
+		halmac_adapter, REG_EDCA_VO_PARAM,
+		HALMAC_REG_READ_32(halmac_adapter, REG_EDCA_VO_PARAM) & 0xFFFF);
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_EDCA_VO_PARAM + 2,
+			    HALMAC_VO_TXOP_LIMIT_88XX);
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_EDCA_VI_PARAM + 2,
+			    HALMAC_VI_TXOP_LIMIT_88XX);
+
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_RD_NAV_NXT,
+			    HALMAC_RDG_NAV_88XX | (HALMAC_TXOP_NAV_88XX << 16));
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_RXTSF_OFFSET_CCK,
+			    HALMAC_CCK_RX_TSF_88XX |
+				    (HALMAC_OFDM_RX_TSF_88XX) << 8);
+
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_RD_CTRL + 1);
+	value8 |=
+		(BIT_VOQ_RD_INIT_EN | BIT_VIQ_RD_INIT_EN | BIT_BEQ_RD_INIT_EN);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_RD_CTRL + 1, value8);
+
+	/* Set beacon cotnrol - enable TSF and other related functions */
+	HALMAC_REG_WRITE_8(
+		halmac_adapter, REG_BCN_CTRL,
+		(u8)(HALMAC_REG_READ_8(halmac_adapter, REG_BCN_CTRL) |
+		     BIT_EN_BCN_FUNCTION));
+
+	/* Set send beacon related registers */
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_TBTT_PROHIBIT,
+			    HALMAC_TBTT_PROHIBIT_88XX |
+				    (HALMAC_TBTT_HOLD_TIME_88XX
+				     << BIT_SHIFT_TBTT_HOLD_TIME_AP));
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_DRVERLYINT,
+			   HALMAC_DRIVER_EARLY_INT_88XX);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_BCNDMATIM,
+			   HALMAC_BEACON_DMA_TIM_88XX);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_init_wmac_cfg_88xx() - init wmac config
+ * @halmac_adapter : the adapter of halmac
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_init_wmac_cfg_88xx(struct halmac_adapter *halmac_adapter)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_INIT_WMAC_CFG);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_RXFLTMAP0,
+			    HALMAC_RX_FILTER0_88XX);
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_RXFLTMAP,
+			    HALMAC_RX_FILTER_88XX);
+
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_RCR, HALMAC_RCR_CONFIG_88XX);
+
+	HALMAC_REG_WRITE_8(
+		halmac_adapter, REG_TCR + 1,
+		(u8)(HALMAC_REG_READ_8(halmac_adapter, REG_TCR + 1) | 0x30));
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_TCR + 2, 0x30);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_TCR + 1, 0x00);
+
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_WMAC_OPTION_FUNCTION + 8,
+			    0x30810041);
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_WMAC_OPTION_FUNCTION + 4,
+			    0x50802080);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_init_mac_cfg_88xx() - config page1~page7 register
+ * @halmac_adapter : the adapter of halmac
+ * @mode : trx mode
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_init_mac_cfg_88xx(struct halmac_adapter *halmac_adapter,
+			 enum halmac_trx_mode mode)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_INIT_MAC_CFG);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>mode = %d\n", __func__,
+			mode);
+
+	status = halmac_api->halmac_init_trx_cfg(halmac_adapter, mode);
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_init_trx_cfg error = %x\n", status);
+		return status;
+	}
+	status = halmac_api->halmac_init_protocol_cfg(halmac_adapter);
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_init_protocol_cfg_88xx error = %x\n", status);
+		return status;
+	}
+
+	status = halmac_init_edca_cfg_88xx(halmac_adapter);
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_init_edca_cfg_88xx error = %x\n", status);
+		return status;
+	}
+
+	status = halmac_init_wmac_cfg_88xx(halmac_adapter);
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_init_wmac_cfg_88xx error = %x\n", status);
+		return status;
+	}
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return status;
+}
+
+/**
+ * halmac_cfg_operation_mode_88xx() - config operation mode
+ * @halmac_adapter : the adapter of halmac
+ * @wireless_mode : 802.11 standard(b/g/n/ac)
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_cfg_operation_mode_88xx(struct halmac_adapter *halmac_adapter,
+			       enum halmac_wireless_mode wireless_mode)
+{
+	void *driver_adapter = NULL;
+	enum halmac_wireless_mode wireless_mode_local =
+		HALMAC_WIRELESS_MODE_UNDEFINE;
+
+	wireless_mode_local = wireless_mode;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter,
+				  HALMAC_API_CFG_OPERATION_MODE);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(
+		driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+		"%s ==========>wireless_mode = %d\n", __func__,
+		wireless_mode);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_cfg_ch_bw_88xx() - config channel & bandwidth
+ * @halmac_adapter : the adapter of halmac
+ * @channel : WLAN channel, support 2.4G & 5G
+ * @pri_ch_idx : primary channel index, idx1, idx2, idx3, idx4
+ * @bw : band width, 20, 40, 80, 160, 5 ,10
+ * Author : KaiYuan Chang
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_cfg_ch_bw_88xx(struct halmac_adapter *halmac_adapter, u8 channel,
+		      enum halmac_pri_ch_idx pri_ch_idx, enum halmac_bw bw)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_CFG_CH_BW);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>ch = %d, idx=%d, bw=%d\n", __func__,
+			channel, pri_ch_idx, bw);
+
+	halmac_cfg_pri_ch_idx_88xx(halmac_adapter, pri_ch_idx);
+
+	halmac_cfg_bw_88xx(halmac_adapter, bw);
+
+	halmac_cfg_ch_88xx(halmac_adapter, channel);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status halmac_cfg_ch_88xx(struct halmac_adapter *halmac_adapter,
+					  u8 channel)
+{
+	u8 value8;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_CFG_CH_BW);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>ch = %d\n", __func__, channel);
+
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_CCK_CHECK);
+	value8 = value8 & (~(BIT(7)));
+
+	if (channel > 35)
+		value8 = value8 | BIT(7);
+
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_CCK_CHECK, value8);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_cfg_pri_ch_idx_88xx(struct halmac_adapter *halmac_adapter,
+			   enum halmac_pri_ch_idx pri_ch_idx)
+{
+	u8 txsc_40 = 0, txsc_20 = 0;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_CFG_CH_BW);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========> idx=%d\n", __func__,
+			pri_ch_idx);
+
+	txsc_20 = pri_ch_idx;
+	if (txsc_20 == HALMAC_CH_IDX_1 || txsc_20 == HALMAC_CH_IDX_3)
+		txsc_40 = 9;
+	else
+		txsc_40 = 10;
+
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_DATA_SC,
+			   BIT_TXSC_20M(txsc_20) | BIT_TXSC_40M(txsc_40));
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_cfg_bw_88xx() - config bandwidth
+ * @halmac_adapter : the adapter of halmac
+ * @bw : band width, 20, 40, 80, 160, 5 ,10
+ * Author : KaiYuan Chang
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status halmac_cfg_bw_88xx(struct halmac_adapter *halmac_adapter,
+					  enum halmac_bw bw)
+{
+	u32 value32;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_CFG_BW);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>bw=%d\n", __func__, bw);
+
+	/* RF mode */
+	value32 = HALMAC_REG_READ_32(halmac_adapter, REG_WMAC_TRXPTCL_CTL);
+	value32 = value32 & (~(BIT(7) | BIT(8)));
+
+	switch (bw) {
+	case HALMAC_BW_80:
+		value32 = value32 | BIT(7);
+		break;
+	case HALMAC_BW_40:
+		value32 = value32 | BIT(8);
+		break;
+	case HALMAC_BW_20:
+	case HALMAC_BW_10:
+	case HALMAC_BW_5:
+		break;
+	default:
+		pr_err("%s switch case not support\n", __func__);
+		break;
+	}
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_WMAC_TRXPTCL_CTL, value32);
+
+	/* MAC CLK */
+	value32 = HALMAC_REG_READ_32(halmac_adapter, REG_AFE_CTRL1);
+	value32 = (value32 & (~(BIT(20) | BIT(21)))) |
+		  (HALMAC_MAC_CLOCK_HW_DEF_80M << BIT_SHIFT_MAC_CLK_SEL);
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_AFE_CTRL1, value32);
+
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_USTIME_TSF,
+			   HALMAC_MAC_CLOCK_88XX);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_USTIME_EDCA,
+			   HALMAC_MAC_CLOCK_88XX);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_dump_efuse_map_88xx() - dump "physical" efuse map
+ * @halmac_adapter : the adapter of halmac
+ * @cfg : dump efuse method
+ * Author : Ivan Lin/KaiYuan Chang
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_dump_efuse_map_88xx(struct halmac_adapter *halmac_adapter,
+			   enum halmac_efuse_read_cfg cfg)
+{
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+	enum halmac_cmd_process_status *process_status =
+		&halmac_adapter->halmac_state.efuse_state_set.process_status;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_DUMP_EFUSE_MAP);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>cfg=%d\n", __func__, cfg);
+
+	if (*process_status == HALMAC_CMD_PROCESS_SENDING) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"Wait event(dump efuse)...\n");
+		return HALMAC_RET_BUSY_STATE;
+	}
+
+	if (halmac_query_efuse_curr_state_88xx(halmac_adapter) !=
+	    HALMAC_EFUSE_CMD_CONSTRUCT_IDLE) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"Not idle state(dump efuse)...\n");
+		return HALMAC_RET_ERROR_STATE;
+	}
+
+	if (halmac_adapter->halmac_state.mac_power == HALMAC_MAC_POWER_OFF)
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_EFUSE, DBG_WARNING,
+				"[WARN]Dump efuse in suspend mode\n");
+
+	*process_status = HALMAC_CMD_PROCESS_IDLE;
+	halmac_adapter->event_trigger.physical_efuse_map = 1;
+
+	status = halmac_func_switch_efuse_bank_88xx(halmac_adapter,
+						    HALMAC_EFUSE_BANK_WIFI);
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_func_switch_efuse_bank error = %x\n", status);
+		return status;
+	}
+
+	status = halmac_dump_efuse_88xx(halmac_adapter, cfg);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_read_efuse error = %x\n", status);
+		return status;
+	}
+
+	if (halmac_adapter->hal_efuse_map_valid) {
+		*process_status = HALMAC_CMD_PROCESS_DONE;
+
+		PLATFORM_EVENT_INDICATION(
+			driver_adapter, HALMAC_FEATURE_DUMP_PHYSICAL_EFUSE,
+			*process_status, halmac_adapter->hal_efuse_map,
+			halmac_adapter->hw_config_info.efuse_size);
+		halmac_adapter->event_trigger.physical_efuse_map = 0;
+	}
+
+	if (halmac_transition_efuse_state_88xx(
+		    halmac_adapter, HALMAC_EFUSE_CMD_CONSTRUCT_IDLE) !=
+	    HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ERROR_STATE;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_EFUSE, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_dump_efuse_map_bt_88xx() - dump "BT physical" efuse map
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_efuse_bank : bt efuse bank
+ * @bt_efuse_map_size : bt efuse map size. get from halmac_get_efuse_size API
+ * @bt_efuse_map : bt efuse map
+ * Author : Soar / Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_dump_efuse_map_bt_88xx(struct halmac_adapter *halmac_adapter,
+			      enum halmac_efuse_bank halmac_efuse_bank,
+			      u32 bt_efuse_map_size, u8 *bt_efuse_map)
+{
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+	enum halmac_cmd_process_status *process_status =
+		&halmac_adapter->halmac_state.efuse_state_set.process_status;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_DUMP_EFUSE_MAP_BT);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	if (halmac_adapter->hw_config_info.bt_efuse_size != bt_efuse_map_size)
+		return HALMAC_RET_EFUSE_SIZE_INCORRECT;
+
+	if ((halmac_efuse_bank >= HALMAC_EFUSE_BANK_MAX) ||
+	    halmac_efuse_bank == HALMAC_EFUSE_BANK_WIFI) {
+		pr_err("Undefined BT bank\n");
+		return HALMAC_RET_EFUSE_BANK_INCORRECT;
+	}
+
+	if (*process_status == HALMAC_CMD_PROCESS_SENDING) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"Wait event(dump efuse)...\n");
+		return HALMAC_RET_BUSY_STATE;
+	}
+
+	if (halmac_query_efuse_curr_state_88xx(halmac_adapter) !=
+	    HALMAC_EFUSE_CMD_CONSTRUCT_IDLE) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"Not idle state(dump efuse)...\n");
+		return HALMAC_RET_ERROR_STATE;
+	}
+
+	status = halmac_func_switch_efuse_bank_88xx(halmac_adapter,
+						    halmac_efuse_bank);
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_func_switch_efuse_bank error = %x\n", status);
+		return status;
+	}
+
+	status = halmac_read_hw_efuse_88xx(halmac_adapter, 0, bt_efuse_map_size,
+					   bt_efuse_map);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_read_hw_efuse_88xx error = %x\n", status);
+		return status;
+	}
+
+	if (halmac_transition_efuse_state_88xx(
+		    halmac_adapter, HALMAC_EFUSE_CMD_CONSTRUCT_IDLE) !=
+	    HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ERROR_STATE;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_EFUSE, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_write_efuse_bt_88xx() - write "BT physical" efuse offset
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_offset : offset
+ * @halmac_value : Write value
+ * @bt_efuse_map : bt efuse map
+ * Author : Soar
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_write_efuse_bt_88xx(struct halmac_adapter *halmac_adapter,
+			   u32 halmac_offset, u8 halmac_value,
+			   enum halmac_efuse_bank halmac_efuse_bank)
+{
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	enum halmac_cmd_process_status *process_status =
+		&halmac_adapter->halmac_state.efuse_state_set.process_status;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_WRITE_EFUSE_BT);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_EFUSE, DBG_DMESG,
+			"%s ==========>\n", __func__);
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_EFUSE, DBG_DMESG,
+			"offset : %X value : %X Bank : %X\n", halmac_offset,
+			halmac_value, halmac_efuse_bank);
+
+	if (*process_status == HALMAC_CMD_PROCESS_SENDING) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"Wait/Rcvd event(dump efuse)...\n");
+		return HALMAC_RET_BUSY_STATE;
+	}
+
+	if (halmac_query_efuse_curr_state_88xx(halmac_adapter) !=
+	    HALMAC_EFUSE_CMD_CONSTRUCT_IDLE) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"Not idle state(dump efuse)...\n");
+		return HALMAC_RET_ERROR_STATE;
+	}
+
+	if (halmac_offset >= halmac_adapter->hw_config_info.efuse_size) {
+		pr_err("Offset is too large\n");
+		return HALMAC_RET_EFUSE_SIZE_INCORRECT;
+	}
+
+	if (halmac_efuse_bank > HALMAC_EFUSE_BANK_MAX ||
+	    halmac_efuse_bank == HALMAC_EFUSE_BANK_WIFI) {
+		pr_err("Undefined BT bank\n");
+		return HALMAC_RET_EFUSE_BANK_INCORRECT;
+	}
+
+	status = halmac_func_switch_efuse_bank_88xx(halmac_adapter,
+						    halmac_efuse_bank);
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_func_switch_efuse_bank error = %x\n", status);
+		return status;
+	}
+
+	status = halmac_func_write_efuse_88xx(halmac_adapter, halmac_offset,
+					      halmac_value);
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_func_write_efuse error = %x\n", status);
+		return status;
+	}
+
+	if (halmac_transition_efuse_state_88xx(
+		    halmac_adapter, HALMAC_EFUSE_CMD_CONSTRUCT_IDLE) !=
+	    HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ERROR_STATE;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_EFUSE, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_get_efuse_available_size_88xx() - get efuse available size
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_size : physical efuse available size
+ * Author : Soar
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_get_efuse_available_size_88xx(struct halmac_adapter *halmac_adapter,
+				     u32 *halmac_size)
+{
+	enum halmac_ret_status status;
+	void *driver_adapter = NULL;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_EFUSE, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	status = halmac_dump_logical_efuse_map_88xx(halmac_adapter,
+						    HALMAC_EFUSE_R_DRV);
+
+	if (status != HALMAC_RET_SUCCESS)
+		return status;
+
+	*halmac_size = halmac_adapter->hw_config_info.efuse_size -
+		       HALMAC_PROTECTED_EFUSE_SIZE_88XX -
+		       halmac_adapter->efuse_end;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_EFUSE, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_get_efuse_size_88xx() - get "physical" efuse size
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_size : physical efuse size
+ * Author : Ivan Lin/KaiYuan Chang
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_get_efuse_size_88xx(struct halmac_adapter *halmac_adapter,
+			   u32 *halmac_size)
+{
+	void *driver_adapter = NULL;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_GET_EFUSE_SIZE);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_EFUSE, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	*halmac_size = halmac_adapter->hw_config_info.efuse_size;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_EFUSE, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_get_logical_efuse_size_88xx() - get "logical" efuse size
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_size : logical efuse size
+ * Author : Ivan Lin/KaiYuan Chang
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_get_logical_efuse_size_88xx(struct halmac_adapter *halmac_adapter,
+				   u32 *halmac_size)
+{
+	void *driver_adapter = NULL;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter,
+				  HALMAC_API_GET_LOGICAL_EFUSE_SIZE);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_EFUSE, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	*halmac_size = halmac_adapter->hw_config_info.eeprom_size;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_EFUSE, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_dump_logical_efuse_map_88xx() - dump "logical" efuse map
+ * @halmac_adapter : the adapter of halmac
+ * @cfg : dump efuse method
+ * Author : Soar
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_dump_logical_efuse_map_88xx(struct halmac_adapter *halmac_adapter,
+				   enum halmac_efuse_read_cfg cfg)
+{
+	u8 *eeprom_map = NULL;
+	u32 eeprom_size = halmac_adapter->hw_config_info.eeprom_size;
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+	enum halmac_cmd_process_status *process_status =
+		&halmac_adapter->halmac_state.efuse_state_set.process_status;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter,
+				  HALMAC_API_DUMP_LOGICAL_EFUSE_MAP);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_EFUSE, DBG_DMESG,
+			"%s ==========>cfg = %d\n", __func__, cfg);
+
+	if (*process_status == HALMAC_CMD_PROCESS_SENDING) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"Wait/Rcvd event(dump efuse)...\n");
+		return HALMAC_RET_BUSY_STATE;
+	}
+
+	if (halmac_query_efuse_curr_state_88xx(halmac_adapter) !=
+	    HALMAC_EFUSE_CMD_CONSTRUCT_IDLE) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"Not idle state(dump efuse)...\n");
+		return HALMAC_RET_ERROR_STATE;
+	}
+
+	if (halmac_adapter->halmac_state.mac_power == HALMAC_MAC_POWER_OFF)
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_EFUSE, DBG_WARNING,
+				"[WARN]Dump logical efuse in suspend mode\n");
+
+	*process_status = HALMAC_CMD_PROCESS_IDLE;
+	halmac_adapter->event_trigger.logical_efuse_map = 1;
+
+	status = halmac_func_switch_efuse_bank_88xx(halmac_adapter,
+						    HALMAC_EFUSE_BANK_WIFI);
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_func_switch_efuse_bank error = %x\n", status);
+		return status;
+	}
+
+	status = halmac_dump_efuse_88xx(halmac_adapter, cfg);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_eeprom_parser_88xx error = %x\n", status);
+		return status;
+	}
+
+	if (halmac_adapter->hal_efuse_map_valid) {
+		*process_status = HALMAC_CMD_PROCESS_DONE;
+
+		eeprom_map = kzalloc(eeprom_size, GFP_KERNEL);
+		if (!eeprom_map) {
+			/* out of memory */
+			return HALMAC_RET_MALLOC_FAIL;
+		}
+		memset(eeprom_map, 0xFF, eeprom_size);
+
+		if (halmac_eeprom_parser_88xx(halmac_adapter,
+					      halmac_adapter->hal_efuse_map,
+					      eeprom_map) != HALMAC_RET_SUCCESS) {
+			kfree(eeprom_map);
+			return HALMAC_RET_EEPROM_PARSING_FAIL;
+		}
+
+		PLATFORM_EVENT_INDICATION(
+			driver_adapter, HALMAC_FEATURE_DUMP_LOGICAL_EFUSE,
+			*process_status, eeprom_map, eeprom_size);
+		halmac_adapter->event_trigger.logical_efuse_map = 0;
+
+		kfree(eeprom_map);
+	}
+
+	if (halmac_transition_efuse_state_88xx(
+		    halmac_adapter, HALMAC_EFUSE_CMD_CONSTRUCT_IDLE) !=
+	    HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ERROR_STATE;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_EFUSE, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_read_logical_efuse_88xx() - read logical efuse map 1 byte
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_offset : offset
+ * @value : 1 byte efuse value
+ * Author : Soar
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_read_logical_efuse_88xx(struct halmac_adapter *halmac_adapter,
+			       u32 halmac_offset, u8 *value)
+{
+	u8 *eeprom_map = NULL;
+	u32 eeprom_size = halmac_adapter->hw_config_info.eeprom_size;
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	enum halmac_cmd_process_status *process_status =
+		&halmac_adapter->halmac_state.efuse_state_set.process_status;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter,
+				  HALMAC_API_READ_LOGICAL_EFUSE);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_EFUSE, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	if (halmac_offset >= eeprom_size) {
+		pr_err("Offset is too large\n");
+		return HALMAC_RET_EFUSE_SIZE_INCORRECT;
+	}
+
+	if (*process_status == HALMAC_CMD_PROCESS_SENDING) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"Wait/Rcvd event(dump efuse)...\n");
+		return HALMAC_RET_BUSY_STATE;
+	}
+	if (halmac_query_efuse_curr_state_88xx(halmac_adapter) !=
+	    HALMAC_EFUSE_CMD_CONSTRUCT_IDLE) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"Not idle state(dump efuse)...\n");
+		return HALMAC_RET_ERROR_STATE;
+	}
+
+	status = halmac_func_switch_efuse_bank_88xx(halmac_adapter,
+						    HALMAC_EFUSE_BANK_WIFI);
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_func_switch_efuse_bank error = %x\n", status);
+		return status;
+	}
+
+	eeprom_map = kzalloc(eeprom_size, GFP_KERNEL);
+	if (!eeprom_map) {
+		/* out of memory */
+		return HALMAC_RET_MALLOC_FAIL;
+	}
+	memset(eeprom_map, 0xFF, eeprom_size);
+
+	status = halmac_read_logical_efuse_map_88xx(halmac_adapter, eeprom_map);
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_read_logical_efuse_map error = %x\n", status);
+		kfree(eeprom_map);
+		return status;
+	}
+
+	*value = *(eeprom_map + halmac_offset);
+
+	if (halmac_transition_efuse_state_88xx(
+		    halmac_adapter, HALMAC_EFUSE_CMD_CONSTRUCT_IDLE) !=
+	    HALMAC_RET_SUCCESS) {
+		kfree(eeprom_map);
+		return HALMAC_RET_ERROR_STATE;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_EFUSE, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	kfree(eeprom_map);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_write_logical_efuse_88xx() - write "logical" efuse offset
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_offset : offset
+ * @halmac_value : value
+ * Author : Soar
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_write_logical_efuse_88xx(struct halmac_adapter *halmac_adapter,
+				u32 halmac_offset, u8 halmac_value)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	enum halmac_cmd_process_status *process_status =
+		&halmac_adapter->halmac_state.efuse_state_set.process_status;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter,
+				  HALMAC_API_WRITE_LOGICAL_EFUSE);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_EFUSE, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	if (halmac_offset >= halmac_adapter->hw_config_info.eeprom_size) {
+		pr_err("Offset is too large\n");
+		return HALMAC_RET_EFUSE_SIZE_INCORRECT;
+	}
+
+	if (*process_status == HALMAC_CMD_PROCESS_SENDING) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"Wait/Rcvd event(dump efuse)...\n");
+		return HALMAC_RET_BUSY_STATE;
+	}
+
+	if (halmac_query_efuse_curr_state_88xx(halmac_adapter) !=
+	    HALMAC_EFUSE_CMD_CONSTRUCT_IDLE) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"Not idle state(dump efuse)...\n");
+		return HALMAC_RET_ERROR_STATE;
+	}
+
+	status = halmac_func_switch_efuse_bank_88xx(halmac_adapter,
+						    HALMAC_EFUSE_BANK_WIFI);
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_func_switch_efuse_bank error = %x\n", status);
+		return status;
+	}
+
+	status = halmac_func_write_logical_efuse_88xx(
+		halmac_adapter, halmac_offset, halmac_value);
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_write_logical_efuse error = %x\n", status);
+		return status;
+	}
+
+	if (halmac_transition_efuse_state_88xx(
+		    halmac_adapter, HALMAC_EFUSE_CMD_CONSTRUCT_IDLE) !=
+	    HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ERROR_STATE;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_EFUSE, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_pg_efuse_by_map_88xx() - pg logical efuse by map
+ * @halmac_adapter : the adapter of halmac
+ * @pg_efuse_info : efuse map information
+ * @cfg : dump efuse method
+ * Author : Soar
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_pg_efuse_by_map_88xx(struct halmac_adapter *halmac_adapter,
+			    struct halmac_pg_efuse_info *pg_efuse_info,
+			    enum halmac_efuse_read_cfg cfg)
+{
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	enum halmac_cmd_process_status *process_status =
+		&halmac_adapter->halmac_state.efuse_state_set.process_status;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_PG_EFUSE_BY_MAP);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_EFUSE, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	if (pg_efuse_info->efuse_map_size !=
+	    halmac_adapter->hw_config_info.eeprom_size) {
+		pr_err("efuse_map_size is incorrect, should be %d bytes\n",
+		       halmac_adapter->hw_config_info.eeprom_size);
+		return HALMAC_RET_EFUSE_SIZE_INCORRECT;
+	}
+
+	if ((pg_efuse_info->efuse_map_size & 0xF) > 0) {
+		pr_err("efuse_map_size should be multiple of 16\n");
+		return HALMAC_RET_EFUSE_SIZE_INCORRECT;
+	}
+
+	if (pg_efuse_info->efuse_mask_size !=
+	    pg_efuse_info->efuse_map_size >> 4) {
+		pr_err("efuse_mask_size is incorrect, should be %d bytes\n",
+		       pg_efuse_info->efuse_map_size >> 4);
+		return HALMAC_RET_EFUSE_SIZE_INCORRECT;
+	}
+
+	if (!pg_efuse_info->efuse_map) {
+		pr_err("efuse_map is NULL\n");
+		return HALMAC_RET_NULL_POINTER;
+	}
+
+	if (!pg_efuse_info->efuse_mask) {
+		pr_err("efuse_mask is NULL\n");
+		return HALMAC_RET_NULL_POINTER;
+	}
+
+	if (*process_status == HALMAC_CMD_PROCESS_SENDING) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"Wait/Rcvd event(dump efuse)...\n");
+		return HALMAC_RET_BUSY_STATE;
+	}
+
+	if (halmac_query_efuse_curr_state_88xx(halmac_adapter) !=
+	    HALMAC_EFUSE_CMD_CONSTRUCT_IDLE) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"Not idle state(dump efuse)...\n");
+		return HALMAC_RET_ERROR_STATE;
+	}
+
+	status = halmac_func_switch_efuse_bank_88xx(halmac_adapter,
+						    HALMAC_EFUSE_BANK_WIFI);
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_func_switch_efuse_bank error = %x\n", status);
+		return status;
+	}
+
+	status = halmac_func_pg_efuse_by_map_88xx(halmac_adapter, pg_efuse_info,
+						  cfg);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_pg_efuse_by_map error = %x\n", status);
+		return status;
+	}
+
+	if (halmac_transition_efuse_state_88xx(
+		    halmac_adapter, HALMAC_EFUSE_CMD_CONSTRUCT_IDLE) !=
+	    HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ERROR_STATE;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_EFUSE, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_get_c2h_info_88xx() - process halmac C2H packet
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_buf : RX Packet pointer
+ * @halmac_size : RX Packet size
+ * Author : KaiYuan Chang/Ivan Lin
+ *
+ * Used to process c2h packet info from RX path. After receiving the packet,
+ * user need to call this api and pass the packet pointer.
+ *
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_get_c2h_info_88xx(struct halmac_adapter *halmac_adapter, u8 *halmac_buf,
+			 u32 halmac_size)
+{
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_GET_C2H_INFO);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	/* Check if it is C2H packet */
+	if (GET_RX_DESC_C2H(halmac_buf)) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+				"C2H packet, start parsing!\n");
+
+		status = halmac_parse_c2h_packet_88xx(halmac_adapter,
+						      halmac_buf, halmac_size);
+
+		if (status != HALMAC_RET_SUCCESS) {
+			pr_err("halmac_parse_c2h_packet_88xx error = %x\n",
+			       status);
+			return status;
+		}
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_cfg_fwlps_option_88xx(struct halmac_adapter *halmac_adapter,
+			     struct halmac_fwlps_option *lps_option)
+{
+	void *driver_adapter = NULL;
+	struct halmac_fwlps_option *hal_fwlps_option;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_CFG_FWLPS_OPTION);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	hal_fwlps_option = &halmac_adapter->fwlps_option;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	hal_fwlps_option->mode = lps_option->mode;
+	hal_fwlps_option->clk_request = lps_option->clk_request;
+	hal_fwlps_option->rlbm = lps_option->rlbm;
+	hal_fwlps_option->smart_ps = lps_option->smart_ps;
+	hal_fwlps_option->awake_interval = lps_option->awake_interval;
+	hal_fwlps_option->all_queue_uapsd = lps_option->all_queue_uapsd;
+	hal_fwlps_option->pwr_state = lps_option->pwr_state;
+	hal_fwlps_option->low_pwr_rx_beacon = lps_option->low_pwr_rx_beacon;
+	hal_fwlps_option->ant_auto_switch = lps_option->ant_auto_switch;
+	hal_fwlps_option->ps_allow_bt_high_priority =
+		lps_option->ps_allow_bt_high_priority;
+	hal_fwlps_option->protect_bcn = lps_option->protect_bcn;
+	hal_fwlps_option->silence_period = lps_option->silence_period;
+	hal_fwlps_option->fast_bt_connect = lps_option->fast_bt_connect;
+	hal_fwlps_option->two_antenna_en = lps_option->two_antenna_en;
+	hal_fwlps_option->adopt_user_setting = lps_option->adopt_user_setting;
+	hal_fwlps_option->drv_bcn_early_shift = lps_option->drv_bcn_early_shift;
+	hal_fwlps_option->enter_32K = lps_option->enter_32K;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_cfg_fwips_option_88xx(struct halmac_adapter *halmac_adapter,
+			     struct halmac_fwips_option *ips_option)
+{
+	void *driver_adapter = NULL;
+	struct halmac_fwips_option *ips_option_local;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_CFG_FWIPS_OPTION);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	ips_option_local = ips_option;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_enter_wowlan_88xx(struct halmac_adapter *halmac_adapter,
+			 struct halmac_wowlan_option *wowlan_option)
+{
+	void *driver_adapter = NULL;
+	struct halmac_wowlan_option *wowlan_option_local;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_ENTER_WOWLAN);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	wowlan_option_local = wowlan_option;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_leave_wowlan_88xx(struct halmac_adapter *halmac_adapter)
+{
+	void *driver_adapter = NULL;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_LEAVE_WOWLAN);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_enter_ps_88xx(struct halmac_adapter *halmac_adapter,
+		     enum halmac_ps_state ps_state)
+{
+	u8 rpwm;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	if (halmac_fw_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_NO_DLFW;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_ENTER_PS);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	if (ps_state == halmac_adapter->halmac_state.ps_state) {
+		pr_err("power state is already in PS State!!\n");
+		return HALMAC_RET_SUCCESS;
+	}
+
+	if (ps_state == HALMAC_PS_STATE_LPS) {
+		status = halmac_send_h2c_set_pwr_mode_88xx(
+			halmac_adapter, &halmac_adapter->fwlps_option);
+		if (status != HALMAC_RET_SUCCESS) {
+			pr_err("halmac_send_h2c_set_pwr_mode_88xx error = %x!!\n",
+			       status);
+			return status;
+		}
+	} else if (ps_state == HALMAC_PS_STATE_IPS) {
+	}
+
+	halmac_adapter->halmac_state.ps_state = ps_state;
+
+	/* Enter 32K */
+	if (halmac_adapter->halmac_interface == HALMAC_INTERFACE_SDIO) {
+		if (halmac_adapter->fwlps_option.enter_32K) {
+			rpwm = (u8)(((halmac_adapter->rpwm_record ^ (BIT(7))) |
+				     (BIT(0))) &
+				    0x81);
+			HALMAC_REG_WRITE_8(halmac_adapter, REG_SDIO_HRPWM1,
+					   rpwm);
+			halmac_adapter->low_clk = true;
+		}
+	} else if (halmac_adapter->halmac_interface == HALMAC_INTERFACE_USB) {
+		if (halmac_adapter->fwlps_option.enter_32K) {
+			rpwm = (u8)(((halmac_adapter->rpwm_record ^ (BIT(7))) |
+				     (BIT(0))) &
+				    0x81);
+			HALMAC_REG_WRITE_8(halmac_adapter, 0xFE58, rpwm);
+			halmac_adapter->low_clk = true;
+		}
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_leave_ps_88xx(struct halmac_adapter *halmac_adapter)
+{
+	u8 rpwm, cpwm;
+	u32 counter;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	struct halmac_fwlps_option fw_lps_option;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	if (halmac_fw_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_NO_DLFW;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_LEAVE_PS);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	if (halmac_adapter->halmac_state.ps_state == HALMAC_PS_STATE_ACT) {
+		pr_err("power state is already in active!!\n");
+		return HALMAC_RET_SUCCESS;
+	}
+
+	if (halmac_adapter->low_clk) {
+		cpwm = HALMAC_REG_READ_8(halmac_adapter, REG_SDIO_HRPWM1);
+		rpwm = (u8)(
+			((halmac_adapter->rpwm_record ^ (BIT(7))) | (BIT(6))) &
+			0xC0);
+		HALMAC_REG_WRITE_8(halmac_adapter, REG_SDIO_HRPWM1, rpwm);
+
+		cpwm = (u8)((cpwm ^ BIT(7)) & BIT(7));
+		counter = 100;
+		while (cpwm !=
+		       (HALMAC_REG_READ_8(halmac_adapter, REG_SDIO_HRPWM1) &
+			BIT(7))) {
+			usleep_range(50, 60);
+			counter--;
+			if (counter == 0)
+				return HALMAC_RET_CHANGE_PS_FAIL;
+		}
+		halmac_adapter->low_clk = false;
+	}
+
+	memcpy(&fw_lps_option, &halmac_adapter->fwlps_option,
+	       sizeof(struct halmac_fwlps_option));
+	fw_lps_option.mode = 0;
+
+	status = halmac_send_h2c_set_pwr_mode_88xx(halmac_adapter,
+						   &fw_lps_option);
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_send_h2c_set_pwr_mode_88xx error!!=%x\n",
+		       status);
+		return status;
+	}
+
+	halmac_adapter->halmac_state.ps_state = HALMAC_PS_STATE_ACT;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * (debug API)halmac_h2c_lb_88xx() - send h2c loopback packet
+ * @halmac_adapter : the adapter of halmac
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status halmac_h2c_lb_88xx(struct halmac_adapter *halmac_adapter)
+{
+	void *driver_adapter = NULL;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_H2C_LB);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_debug_88xx() - dump information for debugging
+ * @halmac_adapter : the adapter of halmac
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status halmac_debug_88xx(struct halmac_adapter *halmac_adapter)
+{
+	u8 temp8 = 0;
+	u32 i = 0, temp32 = 0;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_DEBUG);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	if (halmac_adapter->halmac_interface == HALMAC_INTERFACE_SDIO) {
+		/* Dump CCCR, it needs new platform api */
+
+		/*Dump SDIO Local Register, use CMD52*/
+		for (i = 0x10250000; i < 0x102500ff; i++) {
+			temp8 = PLATFORM_SDIO_CMD52_READ(halmac_adapter, i);
+			HALMAC_RT_TRACE(
+				driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"halmac_debug: sdio[%x]=%x\n", i, temp8);
+		}
+
+		/*Dump MAC Register*/
+		for (i = 0x0000; i < 0x17ff; i++) {
+			temp8 = PLATFORM_SDIO_CMD52_READ(halmac_adapter, i);
+			HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT,
+					DBG_DMESG, "halmac_debug: mac[%x]=%x\n",
+					i, temp8);
+		}
+
+		/*Check RX Fifo status*/
+		i = REG_RXFF_PTR_V1;
+		temp8 = PLATFORM_SDIO_CMD52_READ(halmac_adapter, i);
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"halmac_debug: mac[%x]=%x\n", i, temp8);
+		i = REG_RXFF_WTR_V1;
+		temp8 = PLATFORM_SDIO_CMD52_READ(halmac_adapter, i);
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"halmac_debug: mac[%x]=%x\n", i, temp8);
+		i = REG_RXFF_PTR_V1;
+		temp8 = PLATFORM_SDIO_CMD52_READ(halmac_adapter, i);
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"halmac_debug: mac[%x]=%x\n", i, temp8);
+		i = REG_RXFF_WTR_V1;
+		temp8 = PLATFORM_SDIO_CMD52_READ(halmac_adapter, i);
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"halmac_debug: mac[%x]=%x\n", i, temp8);
+	} else {
+		/*Dump MAC Register*/
+		for (i = 0x0000; i < 0x17fc; i += 4) {
+			temp32 = HALMAC_REG_READ_32(halmac_adapter, i);
+			HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT,
+					DBG_DMESG, "halmac_debug: mac[%x]=%x\n",
+					i, temp32);
+		}
+
+		/*Check RX Fifo status*/
+		i = REG_RXFF_PTR_V1;
+		temp32 = HALMAC_REG_READ_32(halmac_adapter, i);
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"halmac_debug: mac[%x]=%x\n", i, temp32);
+		i = REG_RXFF_WTR_V1;
+		temp32 = HALMAC_REG_READ_32(halmac_adapter, i);
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"halmac_debug: mac[%x]=%x\n", i, temp32);
+		i = REG_RXFF_PTR_V1;
+		temp32 = HALMAC_REG_READ_32(halmac_adapter, i);
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"halmac_debug: mac[%x]=%x\n", i, temp32);
+		i = REG_RXFF_WTR_V1;
+		temp32 = HALMAC_REG_READ_32(halmac_adapter, i);
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"halmac_debug: mac[%x]=%x\n", i, temp32);
+	}
+
+	/*	TODO: Add check register code, including MAC CLK, CPU CLK */
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_cfg_parameter_88xx() - config parameter by FW
+ * @halmac_adapter : the adapter of halmac
+ * @para_info : cmd id, content
+ * @full_fifo : parameter information
+ *
+ * If msk_en = true, the format of array is {reg_info, mask, value}.
+ * If msk_en =_FAUSE, the format of array is {reg_info, value}
+ * The format of reg_info is
+ * reg_info[31]=rf_reg, 0: MAC_BB reg, 1: RF reg
+ * reg_info[27:24]=rf_path, 0: path_A, 1: path_B
+ * if rf_reg=0(MAC_BB reg), rf_path is meaningless.
+ * ref_info[15:0]=offset
+ *
+ * Example: msk_en = false
+ * {0x8100000a, 0x00001122}
+ * =>Set RF register, path_B, offset 0xA to 0x00001122
+ * {0x00000824, 0x11224433}
+ * =>Set MAC_BB register, offset 0x800 to 0x11224433
+ *
+ * Note : full fifo mode only for init flow
+ *
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_cfg_parameter_88xx(struct halmac_adapter *halmac_adapter,
+			  struct halmac_phy_parameter_info *para_info,
+			  u8 full_fifo)
+{
+	void *driver_adapter = NULL;
+	enum halmac_ret_status ret_status = HALMAC_RET_SUCCESS;
+	enum halmac_cmd_process_status *process_status =
+		&halmac_adapter->halmac_state.cfg_para_state_set.process_status;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	if (halmac_fw_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_NO_DLFW;
+
+	if (halmac_adapter->fw_version.h2c_version < 4)
+		return HALMAC_RET_FW_NO_SUPPORT;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_CFG_PARAMETER);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	if (halmac_adapter->halmac_state.dlfw_state == HALMAC_DLFW_NONE) {
+		pr_err("%s Fail due to DLFW NONE!!\n", __func__);
+		return HALMAC_RET_DLFW_FAIL;
+	}
+
+	if (*process_status == HALMAC_CMD_PROCESS_SENDING) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"Wait event(cfg para)...\n");
+		return HALMAC_RET_BUSY_STATE;
+	}
+
+	if (halmac_query_cfg_para_curr_state_88xx(halmac_adapter) !=
+		    HALMAC_CFG_PARA_CMD_CONSTRUCT_IDLE &&
+	    halmac_query_cfg_para_curr_state_88xx(halmac_adapter) !=
+		    HALMAC_CFG_PARA_CMD_CONSTRUCT_CONSTRUCTING) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"Not idle state(cfg para)...\n");
+		return HALMAC_RET_BUSY_STATE;
+	}
+
+	*process_status = HALMAC_CMD_PROCESS_IDLE;
+
+	ret_status = halmac_send_h2c_phy_parameter_88xx(halmac_adapter,
+							para_info, full_fifo);
+
+	if (ret_status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_send_h2c_phy_parameter_88xx Fail!! = %x\n",
+		       ret_status);
+		return ret_status;
+	}
+
+	return ret_status;
+}
+
+/**
+ * halmac_update_packet_88xx() - send specific packet to FW
+ * @halmac_adapter : the adapter of halmac
+ * @pkt_id : packet id, to know the purpose of this packet
+ * @pkt : packet
+ * @pkt_size : packet size
+ *
+ * Note : TX_DESC is not included in the pkt
+ *
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_update_packet_88xx(struct halmac_adapter *halmac_adapter,
+			  enum halmac_packet_id pkt_id, u8 *pkt, u32 pkt_size)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+	enum halmac_cmd_process_status *process_status =
+		&halmac_adapter->halmac_state.update_packet_set.process_status;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	if (halmac_fw_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_NO_DLFW;
+
+	if (halmac_adapter->fw_version.h2c_version < 4)
+		return HALMAC_RET_FW_NO_SUPPORT;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_UPDATE_PACKET);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	if (*process_status == HALMAC_CMD_PROCESS_SENDING) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"Wait event(update_packet)...\n");
+		return HALMAC_RET_BUSY_STATE;
+	}
+
+	*process_status = HALMAC_CMD_PROCESS_SENDING;
+
+	status = halmac_send_h2c_update_packet_88xx(halmac_adapter, pkt_id, pkt,
+						    pkt_size);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_send_h2c_update_packet_88xx packet = %x,  fail = %x!!\n",
+		       pkt_id, status);
+		return status;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_bcn_ie_filter_88xx(struct halmac_adapter *halmac_adapter,
+			  struct halmac_bcn_ie_info *bcn_ie_info)
+{
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	if (halmac_fw_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_NO_DLFW;
+
+	if (halmac_adapter->fw_version.h2c_version < 4)
+		return HALMAC_RET_FW_NO_SUPPORT;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_BCN_IE_FILTER);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	status = halmac_send_h2c_update_bcn_parse_info_88xx(halmac_adapter,
+							    bcn_ie_info);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_send_h2c_update_bcn_parse_info_88xx fail = %x\n",
+		       status);
+		return status;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_update_datapack_88xx(struct halmac_adapter *halmac_adapter,
+			    enum halmac_data_type halmac_data_type,
+			    struct halmac_phy_parameter_info *para_info)
+{
+	void *driver_adapter = NULL;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	if (halmac_fw_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_NO_DLFW;
+
+	if (halmac_adapter->fw_version.h2c_version < 4)
+		return HALMAC_RET_FW_NO_SUPPORT;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"[TRACE]%s ==========>\n", __func__);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"[TRACE]%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_run_datapack_88xx(struct halmac_adapter *halmac_adapter,
+			 enum halmac_data_type halmac_data_type)
+{
+	void *driver_adapter = NULL;
+	enum halmac_ret_status ret_status = HALMAC_RET_SUCCESS;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	if (halmac_fw_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_NO_DLFW;
+
+	if (halmac_adapter->fw_version.h2c_version < 4)
+		return HALMAC_RET_FW_NO_SUPPORT;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_RUN_DATAPACK);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	ret_status = halmac_send_h2c_run_datapack_88xx(halmac_adapter,
+						       halmac_data_type);
+
+	if (ret_status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_send_h2c_run_datapack_88xx Fail, datatype = %x, status = %x!!\n",
+		       halmac_data_type, ret_status);
+		return ret_status;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"halmac_update_datapack_88xx <==========\n");
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_cfg_drv_info_88xx() - config driver info
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_drv_info : driver information selection
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_cfg_drv_info_88xx(struct halmac_adapter *halmac_adapter,
+			 enum halmac_drv_info halmac_drv_info)
+{
+	u8 drv_info_size = 0;
+	u8 phy_status_en = 0;
+	u8 sniffer_en = 0;
+	u8 plcp_hdr_en = 0;
+	u32 value32;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_CFG_DRV_INFO);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"halmac_cfg_drv_info = %d\n", halmac_drv_info);
+
+	switch (halmac_drv_info) {
+	case HALMAC_DRV_INFO_NONE:
+		drv_info_size = 0;
+		phy_status_en = 0;
+		sniffer_en = 0;
+		plcp_hdr_en = 0;
+		break;
+	case HALMAC_DRV_INFO_PHY_STATUS:
+		drv_info_size = 4;
+		phy_status_en = 1;
+		sniffer_en = 0;
+		plcp_hdr_en = 0;
+		break;
+	case HALMAC_DRV_INFO_PHY_SNIFFER:
+		drv_info_size = 5; /* phy status 4byte, sniffer info 1byte */
+		phy_status_en = 1;
+		sniffer_en = 1;
+		plcp_hdr_en = 0;
+		break;
+	case HALMAC_DRV_INFO_PHY_PLCP:
+		drv_info_size = 6; /* phy status 4byte, plcp header 2byte */
+		phy_status_en = 1;
+		sniffer_en = 0;
+		plcp_hdr_en = 1;
+		break;
+	default:
+		status = HALMAC_RET_SW_CASE_NOT_SUPPORT;
+		pr_err("%s error = %x\n", __func__, status);
+		return status;
+	}
+
+	if (halmac_adapter->txff_allocation.rx_fifo_expanding_mode !=
+	    HALMAC_RX_FIFO_EXPANDING_MODE_DISABLE)
+		drv_info_size = 0xF;
+
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_RX_DRVINFO_SZ, drv_info_size);
+
+	halmac_adapter->drv_info_size = drv_info_size;
+
+	value32 = HALMAC_REG_READ_32(halmac_adapter, REG_RCR);
+	value32 = (value32 & (~BIT_APP_PHYSTS));
+	if (phy_status_en == 1)
+		value32 = value32 | BIT_APP_PHYSTS;
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_RCR, value32);
+
+	value32 = HALMAC_REG_READ_32(halmac_adapter,
+				     REG_WMAC_OPTION_FUNCTION + 4);
+	value32 = (value32 & (~(BIT(8) | BIT(9))));
+	if (sniffer_en == 1)
+		value32 = value32 | BIT(9);
+	if (plcp_hdr_en == 1)
+		value32 = value32 | BIT(8);
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_WMAC_OPTION_FUNCTION + 4,
+			    value32);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_send_bt_coex_88xx(struct halmac_adapter *halmac_adapter, u8 *bt_buf,
+			 u32 bt_size, u8 ack)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	enum halmac_ret_status ret_status = HALMAC_RET_SUCCESS;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	if (halmac_fw_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_NO_DLFW;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_SEND_BT_COEX);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	ret_status = halmac_send_bt_coex_cmd_88xx(halmac_adapter, bt_buf,
+						  bt_size, ack);
+
+	if (ret_status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_send_bt_coex_cmd_88xx Fail = %x!!\n",
+		       ret_status);
+		return ret_status;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * (debug API)halmac_verify_platform_api_88xx() - verify platform api
+ * @halmac_adapter : the adapter of halmac
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_verify_platform_api_88xx(struct halmac_adapter *halmac_adapter)
+{
+	void *driver_adapter = NULL;
+	enum halmac_ret_status ret_status = HALMAC_RET_SUCCESS;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter,
+				  HALMAC_API_VERIFY_PLATFORM_API);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	ret_status = halmac_verify_io_88xx(halmac_adapter);
+
+	if (ret_status != HALMAC_RET_SUCCESS)
+		return ret_status;
+
+	if (halmac_adapter->txff_allocation.la_mode != HALMAC_LA_MODE_FULL)
+		ret_status = halmac_verify_send_rsvd_page_88xx(halmac_adapter);
+
+	if (ret_status != HALMAC_RET_SUCCESS)
+		return ret_status;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return ret_status;
+}
+
+enum halmac_ret_status
+halmac_send_original_h2c_88xx(struct halmac_adapter *halmac_adapter,
+			      u8 *original_h2c, u16 *seq, u8 ack)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	if (halmac_fw_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_NO_DLFW;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_SEND_ORIGINAL_H2C);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	status = halmac_func_send_original_h2c_88xx(halmac_adapter,
+						    original_h2c, seq, ack);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_send_original_h2c FAIL = %x!!\n", status);
+		return status;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_timer_2s_88xx(struct halmac_adapter *halmac_adapter)
+{
+	void *driver_adapter = NULL;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_fill_txdesc_check_sum_88xx() -  fill in tx desc check sum
+ * @halmac_adapter : the adapter of halmac
+ * @cur_desc : tx desc packet
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_fill_txdesc_check_sum_88xx(struct halmac_adapter *halmac_adapter,
+				  u8 *cur_desc)
+{
+	u16 chk_result = 0;
+	u16 *data = (u16 *)NULL;
+	u32 i;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter,
+				  HALMAC_API_FILL_TXDESC_CHECKSUM);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	if (!cur_desc) {
+		pr_err("%s NULL PTR", __func__);
+		return HALMAC_RET_NULL_POINTER;
+	}
+
+	SET_TX_DESC_TXDESC_CHECKSUM(cur_desc, 0x0000);
+
+	data = (u16 *)(cur_desc);
+
+	/* HW clculates only 32byte */
+	for (i = 0; i < 8; i++)
+		chk_result ^= (*(data + 2 * i) ^ *(data + (2 * i + 1)));
+
+	SET_TX_DESC_TXDESC_CHECKSUM(cur_desc, chk_result);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_dump_fifo_88xx() - dump fifo data
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_fifo_sel : FIFO selection
+ * @halmac_start_addr : start address of selected FIFO
+ * @halmac_fifo_dump_size : dump size of selected FIFO
+ * @fifo_map : FIFO data
+ *
+ * Note : before dump fifo, user need to call halmac_get_fifo_size to
+ * get fifo size. Then input this size to halmac_dump_fifo.
+ *
+ * Author : Ivan Lin/KaiYuan Chang
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_dump_fifo_88xx(struct halmac_adapter *halmac_adapter,
+		      enum hal_fifo_sel halmac_fifo_sel, u32 halmac_start_addr,
+		      u32 halmac_fifo_dump_size, u8 *fifo_map)
+{
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_DUMP_FIFO);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	if (halmac_fifo_sel == HAL_FIFO_SEL_TX &&
+	    (halmac_start_addr + halmac_fifo_dump_size) >
+		    halmac_adapter->hw_config_info.tx_fifo_size) {
+		pr_err("TX fifo dump size is too large\n");
+		return HALMAC_RET_DUMP_FIFOSIZE_INCORRECT;
+	}
+
+	if (halmac_fifo_sel == HAL_FIFO_SEL_RX &&
+	    (halmac_start_addr + halmac_fifo_dump_size) >
+		    halmac_adapter->hw_config_info.rx_fifo_size) {
+		pr_err("RX fifo dump size is too large\n");
+		return HALMAC_RET_DUMP_FIFOSIZE_INCORRECT;
+	}
+
+	if ((halmac_fifo_dump_size & (4 - 1)) != 0) {
+		pr_err("halmac_fifo_dump_size shall 4byte align\n");
+		return HALMAC_RET_DUMP_FIFOSIZE_INCORRECT;
+	}
+
+	if (!fifo_map) {
+		pr_err("fifo_map address is NULL\n");
+		return HALMAC_RET_NULL_POINTER;
+	}
+
+	status = halmac_buffer_read_88xx(halmac_adapter, halmac_start_addr,
+					 halmac_fifo_dump_size, halmac_fifo_sel,
+					 fifo_map);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_buffer_read_88xx error = %x\n", status);
+		return status;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_get_fifo_size_88xx() - get fifo size
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_fifo_sel : FIFO selection
+ * Author : Ivan Lin/KaiYuan Chang
+ * Return : u32
+ * More details of status code can be found in prototype document
+ */
+u32 halmac_get_fifo_size_88xx(struct halmac_adapter *halmac_adapter,
+			      enum hal_fifo_sel halmac_fifo_sel)
+{
+	u32 fifo_size = 0;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_GET_FIFO_SIZE);
+
+	if (halmac_fifo_sel == HAL_FIFO_SEL_TX)
+		fifo_size = halmac_adapter->hw_config_info.tx_fifo_size;
+	else if (halmac_fifo_sel == HAL_FIFO_SEL_RX)
+		fifo_size = halmac_adapter->hw_config_info.rx_fifo_size;
+	else if (halmac_fifo_sel == HAL_FIFO_SEL_RSVD_PAGE)
+		fifo_size =
+			((halmac_adapter->hw_config_info.tx_fifo_size >> 7) -
+			 halmac_adapter->txff_allocation.rsvd_pg_bndy)
+			<< 7;
+	else if (halmac_fifo_sel == HAL_FIFO_SEL_REPORT)
+		fifo_size = 65536;
+	else if (halmac_fifo_sel == HAL_FIFO_SEL_LLT)
+		fifo_size = 65536;
+
+	return fifo_size;
+}
+
+/**
+ * halmac_cfg_txbf_88xx() - enable/disable specific user's txbf
+ * @halmac_adapter : the adapter of halmac
+ * @userid : su bfee userid = 0 or 1 to apply TXBF
+ * @bw : the sounding bandwidth
+ * @txbf_en : 0: disable TXBF, 1: enable TXBF
+ * Author : chunchu
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_cfg_txbf_88xx(struct halmac_adapter *halmac_adapter, u8 userid,
+		     enum halmac_bw bw, u8 txbf_en)
+{
+	u16 temp42C = 0;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_CFG_TXBF);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	if (txbf_en) {
+		switch (bw) {
+		case HALMAC_BW_80:
+			temp42C |= BIT_R_TXBF0_80M;
+		case HALMAC_BW_40:
+			temp42C |= BIT_R_TXBF0_40M;
+		case HALMAC_BW_20:
+			temp42C |= BIT_R_TXBF0_20M;
+			break;
+		default:
+			pr_err("%s invalid TXBF BW setting 0x%x of userid %d\n",
+			       __func__, bw, userid);
+			return HALMAC_RET_INVALID_SOUNDING_SETTING;
+		}
+	}
+
+	switch (userid) {
+	case 0:
+		temp42C |=
+			HALMAC_REG_READ_16(halmac_adapter, REG_TXBF_CTRL) &
+			~(BIT_R_TXBF0_20M | BIT_R_TXBF0_40M | BIT_R_TXBF0_80M);
+		HALMAC_REG_WRITE_16(halmac_adapter, REG_TXBF_CTRL, temp42C);
+		break;
+	case 1:
+		temp42C |=
+			HALMAC_REG_READ_16(halmac_adapter, REG_TXBF_CTRL + 2) &
+			~(BIT_R_TXBF0_20M | BIT_R_TXBF0_40M | BIT_R_TXBF0_80M);
+		HALMAC_REG_WRITE_16(halmac_adapter, REG_TXBF_CTRL + 2, temp42C);
+		break;
+	default:
+		pr_err("%s invalid userid %d\n", __func__, userid);
+		return HALMAC_RET_INVALID_SOUNDING_SETTING;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_SND, DBG_DMESG,
+			"%s, txbf_en = %x <==========\n", __func__,
+			txbf_en);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_cfg_mumimo_88xx() -config mumimo
+ * @halmac_adapter : the adapter of halmac
+ * @cfgmu : parameters to configure MU PPDU Tx/Rx
+ * Author : chunchu
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_cfg_mumimo_88xx(struct halmac_adapter *halmac_adapter,
+		       struct halmac_cfg_mumimo_para *cfgmu)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	u8 i, idx, id0, id1, gid, mu_tab_sel;
+	u8 mu_tab_valid = 0;
+	u32 gid_valid[6] = {0};
+	u8 temp14C0 = 0;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_CFG_MUMIMO);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	if (cfgmu->role == HAL_BFEE) {
+		/*config MU BFEE*/
+		temp14C0 = HALMAC_REG_READ_8(halmac_adapter, REG_MU_TX_CTL) &
+			   ~BIT_MASK_R_MU_TABLE_VALID;
+		/*enable MU table 0 and 1, disable MU TX*/
+		HALMAC_REG_WRITE_8(halmac_adapter, REG_MU_TX_CTL,
+				   (temp14C0 | BIT(0) | BIT(1)) & ~(BIT(7)));
+
+		/*config GID valid table and user position table*/
+		mu_tab_sel =
+			HALMAC_REG_READ_8(halmac_adapter, REG_MU_TX_CTL + 1) &
+			~(BIT(0) | BIT(1) | BIT(2));
+		for (i = 0; i < 2; i++) {
+			HALMAC_REG_WRITE_8(halmac_adapter, REG_MU_TX_CTL + 1,
+					   mu_tab_sel | i);
+			HALMAC_REG_WRITE_32(halmac_adapter, REG_MU_STA_GID_VLD,
+					    cfgmu->given_gid_tab[i]);
+			HALMAC_REG_WRITE_32(halmac_adapter,
+					    REG_MU_STA_USER_POS_INFO,
+					    cfgmu->given_user_pos[i * 2]);
+			HALMAC_REG_WRITE_32(halmac_adapter,
+					    REG_MU_STA_USER_POS_INFO + 4,
+					    cfgmu->given_user_pos[i * 2 + 1]);
+		}
+	} else {
+		/*config MU BFER*/
+		if (!cfgmu->mu_tx_en) {
+			HALMAC_REG_WRITE_8(halmac_adapter, REG_MU_TX_CTL,
+					   HALMAC_REG_READ_8(halmac_adapter,
+							     REG_MU_TX_CTL) &
+						   ~(BIT(7)));
+			HALMAC_RT_TRACE(
+				driver_adapter, HALMAC_MSG_SND, DBG_DMESG,
+				"%s disable mu tx <==========\n", __func__);
+			return HALMAC_RET_SUCCESS;
+		}
+
+		/*Transform BB grouping bitmap[14:0] to MAC GID_valid table*/
+		for (idx = 0; idx < 15; idx++) {
+			if (idx < 5) {
+				/*group_bitmap bit0~4, MU_STA0 with MUSTA1~5*/
+				id0 = 0;
+				id1 = (u8)(idx + 1);
+			} else if (idx < 9) {
+				/*group_bitmap bit5~8, MU_STA1 with MUSTA2~5*/
+				id0 = 1;
+				id1 = (u8)(idx - 3);
+			} else if (idx < 12) {
+				/*group_bitmap bit9~11, MU_STA2 with MUSTA3~5*/
+				id0 = 2;
+				id1 = (u8)(idx - 6);
+			} else if (idx < 14) {
+				/*group_bitmap bit12~13, MU_STA3 with MUSTA4~5*/
+				id0 = 3;
+				id1 = (u8)(idx - 8);
+			} else {
+				/*group_bitmap bit14, MU_STA4 with MUSTA5*/
+				id0 = 4;
+				id1 = (u8)(idx - 9);
+			}
+			if (cfgmu->grouping_bitmap & BIT(idx)) {
+				/*Pair 1*/
+				gid = (idx << 1) + 1;
+				gid_valid[id0] |= (BIT(gid));
+				gid_valid[id1] |= (BIT(gid));
+				/*Pair 2*/
+				gid += 1;
+				gid_valid[id0] |= (BIT(gid));
+				gid_valid[id1] |= (BIT(gid));
+			} else {
+				/*Pair 1*/
+				gid = (idx << 1) + 1;
+				gid_valid[id0] &= ~(BIT(gid));
+				gid_valid[id1] &= ~(BIT(gid));
+				/*Pair 2*/
+				gid += 1;
+				gid_valid[id0] &= ~(BIT(gid));
+				gid_valid[id1] &= ~(BIT(gid));
+			}
+		}
+
+		/*set MU STA GID valid TABLE*/
+		mu_tab_sel =
+			HALMAC_REG_READ_8(halmac_adapter, REG_MU_TX_CTL + 1) &
+			~(BIT(0) | BIT(1) | BIT(2));
+		for (idx = 0; idx < 6; idx++) {
+			HALMAC_REG_WRITE_8(halmac_adapter, REG_MU_TX_CTL + 1,
+					   idx | mu_tab_sel);
+			HALMAC_REG_WRITE_32(halmac_adapter, REG_MU_STA_GID_VLD,
+					    gid_valid[idx]);
+		}
+
+		/*To validate the sounding successful MU STA and enable MU TX*/
+		for (i = 0; i < 6; i++) {
+			if (cfgmu->sounding_sts[i])
+				mu_tab_valid |= BIT(i);
+		}
+		HALMAC_REG_WRITE_8(halmac_adapter, REG_MU_TX_CTL,
+				   mu_tab_valid | BIT(7));
+	}
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_SND, DBG_DMESG,
+			"%s <==========\n", __func__);
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_cfg_sounding_88xx() - configure general sounding
+ * @halmac_adapter : the adapter of halmac
+ * @role : driver's role, BFer or BFee
+ * @datarate : set ndpa tx rate if driver is BFer, or set csi response rate
+ *             if driver is BFee
+ * Author : chunchu
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_cfg_sounding_88xx(struct halmac_adapter *halmac_adapter,
+			 enum halmac_snd_role role,
+			 enum halmac_data_rate datarate)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_CFG_SOUNDING);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	switch (role) {
+	case HAL_BFER:
+		HALMAC_REG_WRITE_32(
+			halmac_adapter, REG_TXBF_CTRL,
+			HALMAC_REG_READ_32(halmac_adapter, REG_TXBF_CTRL) |
+				BIT_R_ENABLE_NDPA | BIT_USE_NDPA_PARAMETER |
+				BIT_R_EN_NDPA_INT | BIT_DIS_NDP_BFEN);
+		HALMAC_REG_WRITE_8(halmac_adapter, REG_NDPA_RATE, datarate);
+		HALMAC_REG_WRITE_8(
+			halmac_adapter, REG_NDPA_OPT_CTRL,
+			HALMAC_REG_READ_8(halmac_adapter, REG_NDPA_OPT_CTRL) &
+				(~(BIT(0) | BIT(1))));
+		/*service file length 2 bytes; fix non-STA1 csi start offset */
+		HALMAC_REG_WRITE_8(halmac_adapter, REG_SND_PTCL_CTRL + 1,
+				   0x2 | BIT(7));
+		HALMAC_REG_WRITE_8(halmac_adapter, REG_SND_PTCL_CTRL + 2, 0x2);
+		break;
+	case HAL_BFEE:
+		HALMAC_REG_WRITE_8(halmac_adapter, REG_SND_PTCL_CTRL, 0xDB);
+		HALMAC_REG_WRITE_8(halmac_adapter, REG_SND_PTCL_CTRL + 3, 0x50);
+		/*use ndpa rx rate to decide csi rate*/
+		HALMAC_REG_WRITE_8(halmac_adapter, REG_BBPSF_CTRL + 3,
+				   HALMAC_OFDM54 | BIT(6));
+		HALMAC_REG_WRITE_16(
+			halmac_adapter, REG_RRSR,
+			HALMAC_REG_READ_16(halmac_adapter, REG_RRSR) |
+				BIT(datarate));
+		/*RXFF do not accept BF Rpt Poll, avoid CSI crc error*/
+		HALMAC_REG_WRITE_8(
+			halmac_adapter, REG_RXFLTMAP1,
+			HALMAC_REG_READ_8(halmac_adapter, REG_RXFLTMAP1) &
+				(~(BIT(4))));
+		/*FWFF do not accept BF Rpt Poll, avoid CSI crc error*/
+		HALMAC_REG_WRITE_8(
+			halmac_adapter, REG_RXFLTMAP4,
+			HALMAC_REG_READ_8(halmac_adapter, REG_RXFLTMAP4) &
+				(~(BIT(4))));
+		break;
+	default:
+		pr_err("%s invalid role\n", __func__);
+		return HALMAC_RET_INVALID_SOUNDING_SETTING;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_SND, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_del_sounding_88xx() - reset general sounding
+ * @halmac_adapter : the adapter of halmac
+ * @role : driver's role, BFer or BFee
+ * Author : chunchu
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_del_sounding_88xx(struct halmac_adapter *halmac_adapter,
+			 enum halmac_snd_role role)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_DEL_SOUNDING);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	switch (role) {
+	case HAL_BFER:
+		HALMAC_REG_WRITE_8(halmac_adapter, REG_TXBF_CTRL + 3, 0);
+		break;
+	case HAL_BFEE:
+		HALMAC_REG_WRITE_8(halmac_adapter, REG_SND_PTCL_CTRL, 0);
+		break;
+	default:
+		pr_err("%s invalid role\n", __func__);
+		return HALMAC_RET_INVALID_SOUNDING_SETTING;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_SND, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_su_bfee_entry_init_88xx() - config SU beamformee's registers
+ * @halmac_adapter : the adapter of halmac
+ * @userid : SU bfee userid = 0 or 1 to be added
+ * @paid : partial AID of this bfee
+ * Author : chunchu
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_su_bfee_entry_init_88xx(struct halmac_adapter *halmac_adapter, u8 userid,
+			       u16 paid)
+{
+	u16 temp42C = 0;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter,
+				  HALMAC_API_SU_BFEE_ENTRY_INIT);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	switch (userid) {
+	case 0:
+		temp42C = HALMAC_REG_READ_16(halmac_adapter, REG_TXBF_CTRL) &
+			  ~(BIT_MASK_R_TXBF0_AID | BIT_R_TXBF0_20M |
+			    BIT_R_TXBF0_40M | BIT_R_TXBF0_80M);
+		HALMAC_REG_WRITE_16(halmac_adapter, REG_TXBF_CTRL,
+				    temp42C | paid);
+		HALMAC_REG_WRITE_16(halmac_adapter, REG_ASSOCIATED_BFMEE_SEL,
+				    paid);
+		break;
+	case 1:
+		temp42C =
+			HALMAC_REG_READ_16(halmac_adapter, REG_TXBF_CTRL + 2) &
+			~(BIT_MASK_R_TXBF1_AID | BIT_R_TXBF0_20M |
+			  BIT_R_TXBF0_40M | BIT_R_TXBF0_80M);
+		HALMAC_REG_WRITE_16(halmac_adapter, REG_TXBF_CTRL + 2,
+				    temp42C | paid);
+		HALMAC_REG_WRITE_16(halmac_adapter,
+				    REG_ASSOCIATED_BFMEE_SEL + 2,
+				    paid | BIT(9));
+		break;
+	default:
+		pr_err("%s invalid userid %d\n", __func__,
+		       userid);
+		return HALMAC_RET_INVALID_SOUNDING_SETTING;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_SND, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_su_bfee_entry_init_88xx() - config SU beamformer's registers
+ * @halmac_adapter : the adapter of halmac
+ * @su_bfer_init : parameters to configure SU BFER entry
+ * Author : chunchu
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_su_bfer_entry_init_88xx(struct halmac_adapter *halmac_adapter,
+			       struct halmac_su_bfer_init_para *su_bfer_init)
+{
+	u16 mac_address_H;
+	u32 mac_address_L;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter,
+				  HALMAC_API_SU_BFER_ENTRY_INIT);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	/* mac_address_L = bfer_address.address_l_h.address_low; */
+	/* mac_address_H = bfer_address.address_l_h.address_high; */
+
+	mac_address_L = le32_to_cpu(
+		su_bfer_init->bfer_address.address_l_h.le_address_low);
+	mac_address_H = le16_to_cpu(
+		su_bfer_init->bfer_address.address_l_h.le_address_high);
+
+	switch (su_bfer_init->userid) {
+	case 0:
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_ASSOCIATED_BFMER0_INFO,
+				    mac_address_L);
+		HALMAC_REG_WRITE_16(halmac_adapter,
+				    REG_ASSOCIATED_BFMER0_INFO + 4,
+				    mac_address_H);
+		HALMAC_REG_WRITE_16(halmac_adapter,
+				    REG_ASSOCIATED_BFMER0_INFO + 6,
+				    su_bfer_init->paid);
+		HALMAC_REG_WRITE_16(halmac_adapter, REG_TX_CSI_RPT_PARAM_BW20,
+				    su_bfer_init->csi_para);
+		break;
+	case 1:
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_ASSOCIATED_BFMER1_INFO,
+				    mac_address_L);
+		HALMAC_REG_WRITE_16(halmac_adapter,
+				    REG_ASSOCIATED_BFMER1_INFO + 4,
+				    mac_address_H);
+		HALMAC_REG_WRITE_16(halmac_adapter,
+				    REG_ASSOCIATED_BFMER1_INFO + 6,
+				    su_bfer_init->paid);
+		HALMAC_REG_WRITE_16(halmac_adapter,
+				    REG_TX_CSI_RPT_PARAM_BW20 + 2,
+				    su_bfer_init->csi_para);
+		break;
+	default:
+		pr_err("%s invalid userid %d\n", __func__,
+		       su_bfer_init->userid);
+		return HALMAC_RET_INVALID_SOUNDING_SETTING;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_SND, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_mu_bfee_entry_init_88xx() - config MU beamformee's registers
+ * @halmac_adapter : the adapter of halmac
+ * @mu_bfee_init : parameters to configure MU BFEE entry
+ * Author : chunchu
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_mu_bfee_entry_init_88xx(struct halmac_adapter *halmac_adapter,
+			       struct halmac_mu_bfee_init_para *mu_bfee_init)
+{
+	u16 temp168X = 0, temp14C0;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter,
+				  HALMAC_API_MU_BFEE_ENTRY_INIT);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	temp168X |= mu_bfee_init->paid | BIT(9);
+	HALMAC_REG_WRITE_16(halmac_adapter, (0x1680 + mu_bfee_init->userid * 2),
+			    temp168X);
+
+	temp14C0 = HALMAC_REG_READ_16(halmac_adapter, REG_MU_TX_CTL) &
+		   ~(BIT(8) | BIT(9) | BIT(10));
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_MU_TX_CTL,
+			    temp14C0 | ((mu_bfee_init->userid - 2) << 8));
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_MU_STA_GID_VLD, 0);
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_MU_STA_USER_POS_INFO,
+			    mu_bfee_init->user_position_l);
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_MU_STA_USER_POS_INFO + 4,
+			    mu_bfee_init->user_position_h);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_SND, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_mu_bfer_entry_init_88xx() - config MU beamformer's registers
+ * @halmac_adapter : the adapter of halmac
+ * @mu_bfer_init : parameters to configure MU BFER entry
+ * Author : chunchu
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_mu_bfer_entry_init_88xx(struct halmac_adapter *halmac_adapter,
+			       struct halmac_mu_bfer_init_para *mu_bfer_init)
+{
+	u16 temp1680 = 0;
+	u16 mac_address_H;
+	u32 mac_address_L;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter,
+				  HALMAC_API_MU_BFER_ENTRY_INIT);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	mac_address_L =
+	    le32_to_cpu(mu_bfer_init->bfer_address.address_l_h.le_address_low);
+	mac_address_H =
+	    le16_to_cpu(mu_bfer_init->bfer_address.address_l_h.le_address_high);
+
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_ASSOCIATED_BFMER0_INFO,
+			    mac_address_L);
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_ASSOCIATED_BFMER0_INFO + 4,
+			    mac_address_H);
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_ASSOCIATED_BFMER0_INFO + 6,
+			    mu_bfer_init->paid);
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_TX_CSI_RPT_PARAM_BW20,
+			    mu_bfer_init->csi_para);
+
+	temp1680 = HALMAC_REG_READ_16(halmac_adapter, 0x1680) & 0xC000;
+	temp1680 |= mu_bfer_init->my_aid | (mu_bfer_init->csi_length_sel << 12);
+	HALMAC_REG_WRITE_16(halmac_adapter, 0x1680, temp1680);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_SND, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_su_bfee_entry_del_88xx() - reset SU beamformee's registers
+ * @halmac_adapter : the adapter of halmac
+ * @userid : the SU BFee userid to be deleted
+ * Author : chunchu
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_su_bfee_entry_del_88xx(struct halmac_adapter *halmac_adapter, u8 userid)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_SU_BFEE_ENTRY_DEL);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	switch (userid) {
+	case 0:
+		HALMAC_REG_WRITE_16(
+			halmac_adapter, REG_TXBF_CTRL,
+			HALMAC_REG_READ_16(halmac_adapter, REG_TXBF_CTRL) &
+				~(BIT_MASK_R_TXBF0_AID | BIT_R_TXBF0_20M |
+				  BIT_R_TXBF0_40M | BIT_R_TXBF0_80M));
+		HALMAC_REG_WRITE_16(halmac_adapter, REG_ASSOCIATED_BFMEE_SEL,
+				    0);
+		break;
+	case 1:
+		HALMAC_REG_WRITE_16(
+			halmac_adapter, REG_TXBF_CTRL + 2,
+			HALMAC_REG_READ_16(halmac_adapter, REG_TXBF_CTRL + 2) &
+				~(BIT_MASK_R_TXBF1_AID | BIT_R_TXBF0_20M |
+				  BIT_R_TXBF0_40M | BIT_R_TXBF0_80M));
+		HALMAC_REG_WRITE_16(halmac_adapter,
+				    REG_ASSOCIATED_BFMEE_SEL + 2, 0);
+		break;
+	default:
+		pr_err("%s invalid userid %d\n", __func__,
+		       userid);
+		return HALMAC_RET_INVALID_SOUNDING_SETTING;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_SND, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_su_bfee_entry_del_88xx() - reset SU beamformer's registers
+ * @halmac_adapter : the adapter of halmac
+ * @userid : the SU BFer userid to be deleted
+ * Author : chunchu
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_su_bfer_entry_del_88xx(struct halmac_adapter *halmac_adapter, u8 userid)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_SU_BFER_ENTRY_DEL);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	switch (userid) {
+	case 0:
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_ASSOCIATED_BFMER0_INFO,
+				    0);
+		HALMAC_REG_WRITE_32(halmac_adapter,
+				    REG_ASSOCIATED_BFMER0_INFO + 4, 0);
+		break;
+	case 1:
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_ASSOCIATED_BFMER1_INFO,
+				    0);
+		HALMAC_REG_WRITE_32(halmac_adapter,
+				    REG_ASSOCIATED_BFMER1_INFO + 4, 0);
+		break;
+	default:
+		pr_err("%s invalid userid %d\n", __func__,
+		       userid);
+		return HALMAC_RET_INVALID_SOUNDING_SETTING;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_SND, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_mu_bfee_entry_del_88xx() - reset MU beamformee's registers
+ * @halmac_adapter : the adapter of halmac
+ * @userid : the MU STA userid to be deleted
+ * Author : chunchu
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_mu_bfee_entry_del_88xx(struct halmac_adapter *halmac_adapter, u8 userid)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_MU_BFEE_ENTRY_DEL);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_REG_WRITE_16(halmac_adapter, 0x1680 + userid * 2, 0);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_MU_TX_CTL,
+			   HALMAC_REG_READ_8(halmac_adapter, REG_MU_TX_CTL) &
+				   ~(BIT(userid - 2)));
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_SND, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_mu_bfer_entry_del_88xx() -reset MU beamformer's registers
+ * @halmac_adapter : the adapter of halmac
+ * Author : chunchu
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_mu_bfer_entry_del_88xx(struct halmac_adapter *halmac_adapter)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_MU_BFER_ENTRY_DEL);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_ASSOCIATED_BFMER0_INFO, 0);
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_ASSOCIATED_BFMER0_INFO + 4, 0);
+	HALMAC_REG_WRITE_16(halmac_adapter, 0x1680, 0);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_MU_TX_CTL, 0);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_SND, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_add_ch_info_88xx() -add channel information
+ * @halmac_adapter : the adapter of halmac
+ * @ch_info : channel information
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_add_ch_info_88xx(struct halmac_adapter *halmac_adapter,
+			struct halmac_ch_info *ch_info)
+{
+	void *driver_adapter = NULL;
+	struct halmac_cs_info *ch_sw_info;
+	enum halmac_scan_cmd_construct_state state_scan;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	ch_sw_info = &halmac_adapter->ch_sw_info;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"[TRACE]%s ==========>\n", __func__);
+
+	if (halmac_adapter->halmac_state.dlfw_state != HALMAC_GEN_INFO_SENT) {
+		pr_err("[ERR]%s: gen_info is not send to FW!!!!\n", __func__);
+		return HALMAC_RET_GEN_INFO_NOT_SENT;
+	}
+
+	state_scan = halmac_query_scan_curr_state_88xx(halmac_adapter);
+	if (state_scan != HALMAC_SCAN_CMD_CONSTRUCT_BUFFER_CLEARED &&
+	    state_scan != HALMAC_SCAN_CMD_CONSTRUCT_CONSTRUCTING) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_WARNING,
+				"[WARN]Scan machine fail(add ch info)...\n");
+		return HALMAC_RET_ERROR_STATE;
+	}
+
+	if (!ch_sw_info->ch_info_buf) {
+		ch_sw_info->ch_info_buf =
+			kzalloc(HALMAC_EXTRA_INFO_BUFF_SIZE_88XX, GFP_KERNEL);
+		if (!ch_sw_info->ch_info_buf)
+			return HALMAC_RET_NULL_POINTER;
+		ch_sw_info->ch_info_buf_w = ch_sw_info->ch_info_buf;
+		ch_sw_info->buf_size = HALMAC_EXTRA_INFO_BUFF_SIZE_88XX;
+		ch_sw_info->avai_buf_size = HALMAC_EXTRA_INFO_BUFF_SIZE_88XX;
+		ch_sw_info->total_size = 0;
+		ch_sw_info->extra_info_en = 0;
+		ch_sw_info->ch_num = 0;
+	}
+
+	if (ch_sw_info->extra_info_en == 1) {
+		pr_err("[ERR]%s: construct sequence wrong!!\n", __func__);
+		return HALMAC_RET_CH_SW_SEQ_WRONG;
+	}
+
+	if (ch_sw_info->avai_buf_size < 4) {
+		pr_err("[ERR]%s: no available buffer!!\n", __func__);
+		return HALMAC_RET_CH_SW_NO_BUF;
+	}
+
+	if (halmac_transition_scan_state_88xx(
+		    halmac_adapter, HALMAC_SCAN_CMD_CONSTRUCT_CONSTRUCTING) !=
+	    HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ERROR_STATE;
+
+	CHANNEL_INFO_SET_CHANNEL(ch_sw_info->ch_info_buf_w, ch_info->channel);
+	CHANNEL_INFO_SET_PRI_CH_IDX(ch_sw_info->ch_info_buf_w,
+				    ch_info->pri_ch_idx);
+	CHANNEL_INFO_SET_BANDWIDTH(ch_sw_info->ch_info_buf_w, ch_info->bw);
+	CHANNEL_INFO_SET_TIMEOUT(ch_sw_info->ch_info_buf_w, ch_info->timeout);
+	CHANNEL_INFO_SET_ACTION_ID(ch_sw_info->ch_info_buf_w,
+				   ch_info->action_id);
+	CHANNEL_INFO_SET_CH_EXTRA_INFO(ch_sw_info->ch_info_buf_w,
+				       ch_info->extra_info);
+
+	ch_sw_info->avai_buf_size = ch_sw_info->avai_buf_size - 4;
+	ch_sw_info->total_size = ch_sw_info->total_size + 4;
+	ch_sw_info->ch_num++;
+	ch_sw_info->extra_info_en = ch_info->extra_info;
+	ch_sw_info->ch_info_buf_w = ch_sw_info->ch_info_buf_w + 4;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"[TRACE]%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_add_extra_ch_info_88xx() -add extra channel information
+ * @halmac_adapter : the adapter of halmac
+ * @ch_extra_info : extra channel information
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_add_extra_ch_info_88xx(struct halmac_adapter *halmac_adapter,
+			      struct halmac_ch_extra_info *ch_extra_info)
+{
+	void *driver_adapter = NULL;
+	struct halmac_cs_info *ch_sw_info;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_ADD_EXTRA_CH_INFO);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	ch_sw_info = &halmac_adapter->ch_sw_info;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	if (!ch_sw_info->ch_info_buf) {
+		pr_err("%s: NULL==ch_sw_info->ch_info_buf!!\n", __func__);
+		return HALMAC_RET_CH_SW_SEQ_WRONG;
+	}
+
+	if (ch_sw_info->extra_info_en == 0) {
+		pr_err("%s: construct sequence wrong!!\n", __func__);
+		return HALMAC_RET_CH_SW_SEQ_WRONG;
+	}
+
+	if (ch_sw_info->avai_buf_size <
+	    (u32)(ch_extra_info->extra_info_size + 2)) {
+		/* +2: ch_extra_info_id, ch_extra_info, ch_extra_info_size
+		 * are totally 2Byte
+		 */
+		pr_err("%s: no available buffer!!\n", __func__);
+		return HALMAC_RET_CH_SW_NO_BUF;
+	}
+
+	if (halmac_query_scan_curr_state_88xx(halmac_adapter) !=
+	    HALMAC_SCAN_CMD_CONSTRUCT_CONSTRUCTING) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"Scan machine fail(add extra ch info)...\n");
+		return HALMAC_RET_ERROR_STATE;
+	}
+
+	if (halmac_transition_scan_state_88xx(
+		    halmac_adapter, HALMAC_SCAN_CMD_CONSTRUCT_CONSTRUCTING) !=
+	    HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ERROR_STATE;
+
+	CH_EXTRA_INFO_SET_CH_EXTRA_INFO_ID(ch_sw_info->ch_info_buf_w,
+					   ch_extra_info->extra_action_id);
+	CH_EXTRA_INFO_SET_CH_EXTRA_INFO(ch_sw_info->ch_info_buf_w,
+					ch_extra_info->extra_info);
+	CH_EXTRA_INFO_SET_CH_EXTRA_INFO_SIZE(ch_sw_info->ch_info_buf_w,
+					     ch_extra_info->extra_info_size);
+	memcpy(ch_sw_info->ch_info_buf_w + 2, ch_extra_info->extra_info_data,
+	       ch_extra_info->extra_info_size);
+
+	ch_sw_info->avai_buf_size = ch_sw_info->avai_buf_size -
+				    (2 + ch_extra_info->extra_info_size);
+	ch_sw_info->total_size =
+		ch_sw_info->total_size + (2 + ch_extra_info->extra_info_size);
+	ch_sw_info->extra_info_en = ch_extra_info->extra_info;
+	ch_sw_info->ch_info_buf_w = ch_sw_info->ch_info_buf_w +
+				    (2 + ch_extra_info->extra_info_size);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_ctrl_ch_switch_88xx() -send channel switch cmd
+ * @halmac_adapter : the adapter of halmac
+ * @cs_option : channel switch config
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_ctrl_ch_switch_88xx(struct halmac_adapter *halmac_adapter,
+			   struct halmac_ch_switch_option *cs_option)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+	enum halmac_scan_cmd_construct_state state_scan;
+	enum halmac_cmd_process_status *process_status =
+		&halmac_adapter->halmac_state.scan_state_set.process_status;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	if (halmac_fw_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_NO_DLFW;
+
+	if (halmac_adapter->fw_version.h2c_version < 4)
+		return HALMAC_RET_FW_NO_SUPPORT;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_CTRL_CH_SWITCH);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s  cs_option->switch_en = %d==========>\n", __func__,
+			cs_option->switch_en);
+
+	if (!cs_option->switch_en)
+		*process_status = HALMAC_CMD_PROCESS_IDLE;
+
+	if (*process_status == HALMAC_CMD_PROCESS_SENDING ||
+	    *process_status == HALMAC_CMD_PROCESS_RCVD) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"Wait event(ctrl ch switch)...\n");
+		return HALMAC_RET_BUSY_STATE;
+	}
+
+	state_scan = halmac_query_scan_curr_state_88xx(halmac_adapter);
+	if (cs_option->switch_en) {
+		if (state_scan != HALMAC_SCAN_CMD_CONSTRUCT_CONSTRUCTING) {
+			HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C,
+					DBG_DMESG,
+					"%s(on)  invalid in state %x\n",
+					__func__, state_scan);
+			return HALMAC_RET_ERROR_STATE;
+		}
+	} else {
+		if (state_scan != HALMAC_SCAN_CMD_CONSTRUCT_BUFFER_CLEARED) {
+			HALMAC_RT_TRACE(
+				driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+				"%s(off)  invalid in state %x\n", __func__,
+				state_scan);
+			return HALMAC_RET_ERROR_STATE;
+		}
+	}
+
+	status = halmac_func_ctrl_ch_switch_88xx(halmac_adapter, cs_option);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_ctrl_ch_switch FAIL = %x!!\n", status);
+		return status;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_clear_ch_info_88xx() -clear channel information
+ * @halmac_adapter : the adapter of halmac
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_clear_ch_info_88xx(struct halmac_adapter *halmac_adapter)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_CLEAR_CH_INFO);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	if (halmac_query_scan_curr_state_88xx(halmac_adapter) ==
+	    HALMAC_SCAN_CMD_CONSTRUCT_H2C_SENT) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"Scan machine fail(clear ch info)...\n");
+		return HALMAC_RET_ERROR_STATE;
+	}
+
+	if (halmac_transition_scan_state_88xx(
+		    halmac_adapter, HALMAC_SCAN_CMD_CONSTRUCT_BUFFER_CLEARED) !=
+	    HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ERROR_STATE;
+
+	kfree(halmac_adapter->ch_sw_info.ch_info_buf);
+	halmac_adapter->ch_sw_info.ch_info_buf = NULL;
+	halmac_adapter->ch_sw_info.ch_info_buf_w = NULL;
+	halmac_adapter->ch_sw_info.extra_info_en = 0;
+	halmac_adapter->ch_sw_info.buf_size = 0;
+	halmac_adapter->ch_sw_info.avai_buf_size = 0;
+	halmac_adapter->ch_sw_info.total_size = 0;
+	halmac_adapter->ch_sw_info.ch_num = 0;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status halmac_p2pps_88xx(struct halmac_adapter *halmac_adapter,
+					 struct halmac_p2pps *p2p_ps)
+{
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	if (halmac_fw_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_NO_DLFW;
+
+	if (halmac_adapter->fw_version.h2c_version < 6)
+		return HALMAC_RET_FW_NO_SUPPORT;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	status = halmac_func_p2pps_88xx(halmac_adapter, p2p_ps);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("[ERR]halmac_p2pps FAIL = %x!!\n", status);
+		return status;
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_func_p2pps_88xx(struct halmac_adapter *halmac_adapter,
+		       struct halmac_p2pps *p2p_ps)
+{
+	u8 h2c_buff[HALMAC_H2C_CMD_SIZE_88XX] = {0};
+	u16 h2c_seq_mum = 0;
+	void *driver_adapter = halmac_adapter->driver_adapter;
+	struct halmac_api *halmac_api;
+	struct halmac_h2c_header_info h2c_header_info;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"[TRACE]halmac_p2pps !!\n");
+
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	P2PPS_SET_OFFLOAD_EN(h2c_buff, p2p_ps->offload_en);
+	P2PPS_SET_ROLE(h2c_buff, p2p_ps->role);
+	P2PPS_SET_CTWINDOW_EN(h2c_buff, p2p_ps->ctwindow_en);
+	P2PPS_SET_NOA_EN(h2c_buff, p2p_ps->noa_en);
+	P2PPS_SET_NOA_SEL(h2c_buff, p2p_ps->noa_sel);
+	P2PPS_SET_ALLSTASLEEP(h2c_buff, p2p_ps->all_sta_sleep);
+	P2PPS_SET_DISCOVERY(h2c_buff, p2p_ps->discovery);
+	P2PPS_SET_P2P_PORT_ID(h2c_buff, p2p_ps->p2p_port_id);
+	P2PPS_SET_P2P_GROUP(h2c_buff, p2p_ps->p2p_group);
+	P2PPS_SET_P2P_MACID(h2c_buff, p2p_ps->p2p_macid);
+
+	P2PPS_SET_CTWINDOW_LENGTH(h2c_buff, p2p_ps->ctwindow_length);
+
+	P2PPS_SET_NOA_DURATION_PARA(h2c_buff, p2p_ps->noa_duration_para);
+	P2PPS_SET_NOA_INTERVAL_PARA(h2c_buff, p2p_ps->noa_interval_para);
+	P2PPS_SET_NOA_START_TIME_PARA(h2c_buff, p2p_ps->noa_start_time_para);
+	P2PPS_SET_NOA_COUNT_PARA(h2c_buff, p2p_ps->noa_count_para);
+
+	h2c_header_info.sub_cmd_id = SUB_CMD_ID_P2PPS;
+	h2c_header_info.content_size = 24;
+	h2c_header_info.ack = false;
+	halmac_set_fw_offload_h2c_header_88xx(halmac_adapter, h2c_buff,
+					      &h2c_header_info, &h2c_seq_mum);
+
+	status = halmac_send_h2c_pkt_88xx(halmac_adapter, h2c_buff,
+					  HALMAC_H2C_CMD_SIZE_88XX, false);
+
+	if (status != HALMAC_RET_SUCCESS)
+		pr_err("[ERR]halmac_send_h2c_p2pps_88xx Fail = %x!!\n", status);
+
+	return status;
+}
+
+/**
+ * halmac_send_general_info_88xx() -send general information to FW
+ * @halmac_adapter : the adapter of halmac
+ * @general_info : general information
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_send_general_info_88xx(struct halmac_adapter *halmac_adapter,
+			      struct halmac_general_info *general_info)
+{
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	if (halmac_fw_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_NO_DLFW;
+
+	if (halmac_adapter->fw_version.h2c_version < 4)
+		return HALMAC_RET_FW_NO_SUPPORT;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_SEND_GENERAL_INFO);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	if (halmac_adapter->halmac_state.dlfw_state == HALMAC_DLFW_NONE) {
+		pr_err("%s Fail due to DLFW NONE!!\n", __func__);
+		return HALMAC_RET_DLFW_FAIL;
+	}
+
+	status = halmac_func_send_general_info_88xx(halmac_adapter,
+						    general_info);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_send_general_info error = %x\n", status);
+		return status;
+	}
+
+	if (halmac_adapter->halmac_state.dlfw_state == HALMAC_DLFW_DONE)
+		halmac_adapter->halmac_state.dlfw_state = HALMAC_GEN_INFO_SENT;
+
+	halmac_adapter->gen_info_valid = true;
+	memcpy(&halmac_adapter->general_info, general_info,
+	       sizeof(struct halmac_general_info));
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_start_iqk_88xx() -trigger FW IQK
+ * @halmac_adapter : the adapter of halmac
+ * @iqk_para : IQK parameter
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_start_iqk_88xx(struct halmac_adapter *halmac_adapter,
+		      struct halmac_iqk_para_ *iqk_para)
+{
+	u8 h2c_buff[HALMAC_H2C_CMD_SIZE_88XX] = {0};
+	u16 h2c_seq_num = 0;
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+	struct halmac_h2c_header_info h2c_header_info;
+	enum halmac_cmd_process_status *process_status =
+		&halmac_adapter->halmac_state.iqk_set.process_status;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	if (halmac_fw_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_NO_DLFW;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_START_IQK);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	if (*process_status == HALMAC_CMD_PROCESS_SENDING) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"Wait event(iqk)...\n");
+		return HALMAC_RET_BUSY_STATE;
+	}
+
+	*process_status = HALMAC_CMD_PROCESS_SENDING;
+
+	IQK_SET_CLEAR(h2c_buff, iqk_para->clear);
+	IQK_SET_SEGMENT_IQK(h2c_buff, iqk_para->segment_iqk);
+
+	h2c_header_info.sub_cmd_id = SUB_CMD_ID_IQK;
+	h2c_header_info.content_size = 1;
+	h2c_header_info.ack = true;
+	halmac_set_fw_offload_h2c_header_88xx(halmac_adapter, h2c_buff,
+					      &h2c_header_info, &h2c_seq_num);
+
+	halmac_adapter->halmac_state.iqk_set.seq_num = h2c_seq_num;
+
+	status = halmac_send_h2c_pkt_88xx(halmac_adapter, h2c_buff,
+					  HALMAC_H2C_CMD_SIZE_88XX, true);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_send_h2c_pkt_88xx Fail = %x!!\n", status);
+		return status;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_ctrl_pwr_tracking_88xx() -trigger FW power tracking
+ * @halmac_adapter : the adapter of halmac
+ * @pwr_tracking_opt : power tracking option
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status halmac_ctrl_pwr_tracking_88xx(
+	struct halmac_adapter *halmac_adapter,
+	struct halmac_pwr_tracking_option *pwr_tracking_opt)
+{
+	u8 h2c_buff[HALMAC_H2C_CMD_SIZE_88XX] = {0};
+	u16 h2c_seq_mum = 0;
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+	struct halmac_h2c_header_info h2c_header_info;
+	enum halmac_cmd_process_status *process_status =
+		&halmac_adapter->halmac_state.power_tracking_set.process_status;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	if (halmac_fw_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_NO_DLFW;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_CTRL_PWR_TRACKING);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"halmac_start_iqk_88xx ==========>\n");
+
+	if (*process_status == HALMAC_CMD_PROCESS_SENDING) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"Wait event(pwr tracking)...\n");
+		return HALMAC_RET_BUSY_STATE;
+	}
+
+	*process_status = HALMAC_CMD_PROCESS_SENDING;
+
+	POWER_TRACKING_SET_TYPE(h2c_buff, pwr_tracking_opt->type);
+	POWER_TRACKING_SET_BBSWING_INDEX(h2c_buff,
+					 pwr_tracking_opt->bbswing_index);
+	POWER_TRACKING_SET_ENABLE_A(
+		h2c_buff,
+		pwr_tracking_opt->pwr_tracking_para[HALMAC_RF_PATH_A].enable);
+	POWER_TRACKING_SET_TX_PWR_INDEX_A(
+		h2c_buff, pwr_tracking_opt->pwr_tracking_para[HALMAC_RF_PATH_A]
+				  .tx_pwr_index);
+	POWER_TRACKING_SET_PWR_TRACKING_OFFSET_VALUE_A(
+		h2c_buff, pwr_tracking_opt->pwr_tracking_para[HALMAC_RF_PATH_A]
+				  .pwr_tracking_offset_value);
+	POWER_TRACKING_SET_TSSI_VALUE_A(
+		h2c_buff, pwr_tracking_opt->pwr_tracking_para[HALMAC_RF_PATH_A]
+				  .tssi_value);
+	POWER_TRACKING_SET_ENABLE_B(
+		h2c_buff,
+		pwr_tracking_opt->pwr_tracking_para[HALMAC_RF_PATH_B].enable);
+	POWER_TRACKING_SET_TX_PWR_INDEX_B(
+		h2c_buff, pwr_tracking_opt->pwr_tracking_para[HALMAC_RF_PATH_B]
+				  .tx_pwr_index);
+	POWER_TRACKING_SET_PWR_TRACKING_OFFSET_VALUE_B(
+		h2c_buff, pwr_tracking_opt->pwr_tracking_para[HALMAC_RF_PATH_B]
+				  .pwr_tracking_offset_value);
+	POWER_TRACKING_SET_TSSI_VALUE_B(
+		h2c_buff, pwr_tracking_opt->pwr_tracking_para[HALMAC_RF_PATH_B]
+				  .tssi_value);
+	POWER_TRACKING_SET_ENABLE_C(
+		h2c_buff,
+		pwr_tracking_opt->pwr_tracking_para[HALMAC_RF_PATH_C].enable);
+	POWER_TRACKING_SET_TX_PWR_INDEX_C(
+		h2c_buff, pwr_tracking_opt->pwr_tracking_para[HALMAC_RF_PATH_C]
+				  .tx_pwr_index);
+	POWER_TRACKING_SET_PWR_TRACKING_OFFSET_VALUE_C(
+		h2c_buff, pwr_tracking_opt->pwr_tracking_para[HALMAC_RF_PATH_C]
+				  .pwr_tracking_offset_value);
+	POWER_TRACKING_SET_TSSI_VALUE_C(
+		h2c_buff, pwr_tracking_opt->pwr_tracking_para[HALMAC_RF_PATH_C]
+				  .tssi_value);
+	POWER_TRACKING_SET_ENABLE_D(
+		h2c_buff,
+		pwr_tracking_opt->pwr_tracking_para[HALMAC_RF_PATH_D].enable);
+	POWER_TRACKING_SET_TX_PWR_INDEX_D(
+		h2c_buff, pwr_tracking_opt->pwr_tracking_para[HALMAC_RF_PATH_D]
+				  .tx_pwr_index);
+	POWER_TRACKING_SET_PWR_TRACKING_OFFSET_VALUE_D(
+		h2c_buff, pwr_tracking_opt->pwr_tracking_para[HALMAC_RF_PATH_D]
+				  .pwr_tracking_offset_value);
+	POWER_TRACKING_SET_TSSI_VALUE_D(
+		h2c_buff, pwr_tracking_opt->pwr_tracking_para[HALMAC_RF_PATH_D]
+				  .tssi_value);
+
+	h2c_header_info.sub_cmd_id = SUB_CMD_ID_POWER_TRACKING;
+	h2c_header_info.content_size = 20;
+	h2c_header_info.ack = true;
+	halmac_set_fw_offload_h2c_header_88xx(halmac_adapter, h2c_buff,
+					      &h2c_header_info, &h2c_seq_mum);
+
+	halmac_adapter->halmac_state.power_tracking_set.seq_num = h2c_seq_mum;
+
+	status = halmac_send_h2c_pkt_88xx(halmac_adapter, h2c_buff,
+					  HALMAC_H2C_CMD_SIZE_88XX, true);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_send_h2c_pkt_88xx Fail = %x!!\n", status);
+		return status;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"halmac_start_iqk_88xx <==========\n");
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_query_status_88xx() -query the offload feature status
+ * @halmac_adapter : the adapter of halmac
+ * @feature_id : feature_id
+ * @process_status : feature_status
+ * @data : data buffer
+ * @size : data size
+ *
+ * Note :
+ * If user wants to know the data size, use can allocate zero
+ * size buffer first. If this size less than the data size, halmac
+ * will return  HALMAC_RET_BUFFER_TOO_SMALL. User need to
+ * re-allocate data buffer with correct data size.
+ *
+ * Author : Ivan Lin/KaiYuan Chang
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_query_status_88xx(struct halmac_adapter *halmac_adapter,
+			 enum halmac_feature_id feature_id,
+			 enum halmac_cmd_process_status *process_status,
+			 u8 *data, u32 *size)
+{
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_QUERY_STATE);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	if (!process_status) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+				"null pointer!!\n");
+		return HALMAC_RET_NULL_POINTER;
+	}
+
+	switch (feature_id) {
+	case HALMAC_FEATURE_CFG_PARA:
+		status = halmac_query_cfg_para_status_88xx(
+			halmac_adapter, process_status, data, size);
+		break;
+	case HALMAC_FEATURE_DUMP_PHYSICAL_EFUSE:
+		status = halmac_query_dump_physical_efuse_status_88xx(
+			halmac_adapter, process_status, data, size);
+		break;
+	case HALMAC_FEATURE_DUMP_LOGICAL_EFUSE:
+		status = halmac_query_dump_logical_efuse_status_88xx(
+			halmac_adapter, process_status, data, size);
+		break;
+	case HALMAC_FEATURE_CHANNEL_SWITCH:
+		status = halmac_query_channel_switch_status_88xx(
+			halmac_adapter, process_status, data, size);
+		break;
+	case HALMAC_FEATURE_UPDATE_PACKET:
+		status = halmac_query_update_packet_status_88xx(
+			halmac_adapter, process_status, data, size);
+		break;
+	case HALMAC_FEATURE_IQK:
+		status = halmac_query_iqk_status_88xx(
+			halmac_adapter, process_status, data, size);
+		break;
+	case HALMAC_FEATURE_POWER_TRACKING:
+		status = halmac_query_power_tracking_status_88xx(
+			halmac_adapter, process_status, data, size);
+		break;
+	case HALMAC_FEATURE_PSD:
+		status = halmac_query_psd_status_88xx(
+			halmac_adapter, process_status, data, size);
+		break;
+	default:
+		pr_err("%s invalid feature id %d\n", __func__,
+		       feature_id);
+		return HALMAC_RET_INVALID_FEATURE_ID;
+	}
+
+	return status;
+}
+
+/**
+ * halmac_reset_feature_88xx() -reset async api cmd status
+ * @halmac_adapter : the adapter of halmac
+ * @feature_id : feature_id
+ * Author : Ivan Lin/KaiYuan Chang
+ * Return : enum halmac_ret_status.
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_reset_feature_88xx(struct halmac_adapter *halmac_adapter,
+			  enum halmac_feature_id feature_id)
+{
+	void *driver_adapter = NULL;
+	struct halmac_state *state = &halmac_adapter->halmac_state;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_RESET_FEATURE);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	switch (feature_id) {
+	case HALMAC_FEATURE_CFG_PARA:
+		state->cfg_para_state_set.process_status =
+			HALMAC_CMD_PROCESS_IDLE;
+		state->cfg_para_state_set.cfg_para_cmd_construct_state =
+			HALMAC_CFG_PARA_CMD_CONSTRUCT_IDLE;
+		break;
+	case HALMAC_FEATURE_DUMP_PHYSICAL_EFUSE:
+	case HALMAC_FEATURE_DUMP_LOGICAL_EFUSE:
+		state->efuse_state_set.process_status = HALMAC_CMD_PROCESS_IDLE;
+		state->efuse_state_set.efuse_cmd_construct_state =
+			HALMAC_EFUSE_CMD_CONSTRUCT_IDLE;
+		break;
+	case HALMAC_FEATURE_CHANNEL_SWITCH:
+		state->scan_state_set.process_status = HALMAC_CMD_PROCESS_IDLE;
+		state->scan_state_set.scan_cmd_construct_state =
+			HALMAC_SCAN_CMD_CONSTRUCT_IDLE;
+		break;
+	case HALMAC_FEATURE_UPDATE_PACKET:
+		state->update_packet_set.process_status =
+			HALMAC_CMD_PROCESS_IDLE;
+		break;
+	case HALMAC_FEATURE_ALL:
+		state->cfg_para_state_set.process_status =
+			HALMAC_CMD_PROCESS_IDLE;
+		state->cfg_para_state_set.cfg_para_cmd_construct_state =
+			HALMAC_CFG_PARA_CMD_CONSTRUCT_IDLE;
+		state->efuse_state_set.process_status = HALMAC_CMD_PROCESS_IDLE;
+		state->efuse_state_set.efuse_cmd_construct_state =
+			HALMAC_EFUSE_CMD_CONSTRUCT_IDLE;
+		state->scan_state_set.process_status = HALMAC_CMD_PROCESS_IDLE;
+		state->scan_state_set.scan_cmd_construct_state =
+			HALMAC_SCAN_CMD_CONSTRUCT_IDLE;
+		state->update_packet_set.process_status =
+			HALMAC_CMD_PROCESS_IDLE;
+		break;
+	default:
+		pr_err("%s invalid feature id %d\n", __func__,
+		       feature_id);
+		return HALMAC_RET_INVALID_FEATURE_ID;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_check_fw_status_88xx() -check fw status
+ * @halmac_adapter : the adapter of halmac
+ * @fw_status : fw status
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_check_fw_status_88xx(struct halmac_adapter *halmac_adapter,
+			    bool *fw_status)
+{
+	u32 value32 = 0, value32_backup = 0, i = 0;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_CHECK_FW_STATUS);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	value32 = PLATFORM_REG_READ_32(driver_adapter, REG_FW_DBG6);
+
+	if (value32 != 0) {
+		pr_err("halmac_check_fw_status REG_FW_DBG6 !=0\n");
+		*fw_status = false;
+		return status;
+	}
+
+	value32_backup = PLATFORM_REG_READ_32(driver_adapter, REG_FW_DBG7);
+
+	for (i = 0; i <= 10; i++) {
+		value32 = PLATFORM_REG_READ_32(driver_adapter, REG_FW_DBG7);
+		if (value32_backup != value32)
+			break;
+
+		if (i == 10) {
+			pr_err("halmac_check_fw_status Polling FW PC fail\n");
+			*fw_status = false;
+			return status;
+		}
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return status;
+}
+
+enum halmac_ret_status
+halmac_dump_fw_dmem_88xx(struct halmac_adapter *halmac_adapter, u8 *dmem,
+			 u32 *size)
+{
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_DUMP_FW_DMEM);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return status;
+}
+
+/**
+ * halmac_cfg_max_dl_size_88xx() - config max download FW size
+ * @halmac_adapter : the adapter of halmac
+ * @size : max download fw size
+ *
+ * Halmac uses this setting to set max packet size for
+ * download FW.
+ * If user has not called this API, halmac use default
+ * setting for download FW
+ * Note1 : size need multiple of 2
+ * Note2 : max size is 31K
+ *
+ * Author : Ivan Lin/KaiYuan Chang
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_cfg_max_dl_size_88xx(struct halmac_adapter *halmac_adapter, u32 size)
+{
+	void *driver_adapter = NULL;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_CFG_MAX_DL_SIZE);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_FW, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	if (size > HALMAC_FW_CFG_MAX_DL_SIZE_MAX_88XX) {
+		pr_err("size > HALMAC_FW_CFG_MAX_DL_SIZE_MAX!\n");
+		return HALMAC_RET_CFG_DLFW_SIZE_FAIL;
+	}
+
+	if ((size & (2 - 1)) != 0) {
+		pr_err("size is not power of 2!\n");
+		return HALMAC_RET_CFG_DLFW_SIZE_FAIL;
+	}
+
+	halmac_adapter->max_download_size = size;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_FW, DBG_DMESG,
+			"Cfg max size is : %X\n", size);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_FW, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_psd_88xx() - trigger fw psd
+ * @halmac_adapter : the adapter of halmac
+ * @start_psd : start PSD
+ * @end_psd : end PSD
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status halmac_psd_88xx(struct halmac_adapter *halmac_adapter,
+				       u16 start_psd, u16 end_psd)
+{
+	u8 h2c_buff[HALMAC_H2C_CMD_SIZE_88XX] = {0};
+	u16 h2c_seq_mum = 0;
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+	struct halmac_h2c_header_info h2c_header_info;
+	enum halmac_cmd_process_status *process_status =
+		&halmac_adapter->halmac_state.psd_set.process_status;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	if (halmac_fw_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_NO_DLFW;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_PSD);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	if (*process_status == HALMAC_CMD_PROCESS_SENDING) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"Wait event(psd)...\n");
+		return HALMAC_RET_BUSY_STATE;
+	}
+
+	kfree(halmac_adapter->halmac_state.psd_set.data);
+	halmac_adapter->halmac_state.psd_set.data = (u8 *)NULL;
+
+	halmac_adapter->halmac_state.psd_set.data_size = 0;
+	halmac_adapter->halmac_state.psd_set.segment_size = 0;
+
+	*process_status = HALMAC_CMD_PROCESS_SENDING;
+
+	PSD_SET_START_PSD(h2c_buff, start_psd);
+	PSD_SET_END_PSD(h2c_buff, end_psd);
+
+	h2c_header_info.sub_cmd_id = SUB_CMD_ID_PSD;
+	h2c_header_info.content_size = 4;
+	h2c_header_info.ack = true;
+	halmac_set_fw_offload_h2c_header_88xx(halmac_adapter, h2c_buff,
+					      &h2c_header_info, &h2c_seq_mum);
+
+	status = halmac_send_h2c_pkt_88xx(halmac_adapter, h2c_buff,
+					  HALMAC_H2C_CMD_SIZE_88XX, true);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_send_h2c_pkt_88xx Fail = %x!!\n", status);
+		return status;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_cfg_la_mode_88xx() - config la mode
+ * @halmac_adapter : the adapter of halmac
+ * @la_mode :
+ *	disable : no TXFF space reserved for LA debug
+ *	partial : partial TXFF space is reserved for LA debug
+ *	full : all TXFF space is reserved for LA debug
+ * Author : KaiYuan Chang
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_cfg_la_mode_88xx(struct halmac_adapter *halmac_adapter,
+			enum halmac_la_mode la_mode)
+{
+	void *driver_adapter = NULL;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_CFG_LA_MODE);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s ==========>la_mode = %d\n", __func__,
+			la_mode);
+
+	halmac_adapter->txff_allocation.la_mode = la_mode;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_cfg_rx_fifo_expanding_mode_88xx() - rx fifo expanding
+ * @halmac_adapter : the adapter of halmac
+ * @la_mode :
+ *	disable : normal mode
+ *	1 block : Rx FIFO + 1 FIFO block; Tx fifo - 1 FIFO block
+ *	2 block : Rx FIFO + 2 FIFO block; Tx fifo - 2 FIFO block
+ *	3 block : Rx FIFO + 3 FIFO block; Tx fifo - 3 FIFO block
+ * Author : Soar
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status halmac_cfg_rx_fifo_expanding_mode_88xx(
+	struct halmac_adapter *halmac_adapter,
+	enum halmac_rx_fifo_expanding_mode rx_fifo_expanding_mode)
+{
+	void *driver_adapter = NULL;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter,
+				  HALMAC_API_CFG_RX_FIFO_EXPANDING_MODE);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(
+		driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+		"%s ==========>rx_fifo_expanding_mode = %d\n", __func__,
+		rx_fifo_expanding_mode);
+
+	halmac_adapter->txff_allocation.rx_fifo_expanding_mode =
+		rx_fifo_expanding_mode;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_config_security_88xx(struct halmac_adapter *halmac_adapter,
+			    struct halmac_security_setting *sec_setting)
+{
+	struct halmac_api *halmac_api;
+	void *driver_adapter = NULL;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_COMMON, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_CR,
+			    (u16)(HALMAC_REG_READ_16(halmac_adapter, REG_CR) |
+				  BIT_MAC_SEC_EN));
+
+	if (sec_setting->tx_encryption == 1)
+		HALMAC_REG_WRITE_8(
+			halmac_adapter, REG_SECCFG,
+			HALMAC_REG_READ_8(halmac_adapter, REG_SECCFG) | BIT(2));
+	else
+		HALMAC_REG_WRITE_8(
+			halmac_adapter, REG_SECCFG,
+			HALMAC_REG_READ_8(halmac_adapter, REG_SECCFG) &
+				~(BIT(2)));
+
+	if (sec_setting->rx_decryption == 1)
+		HALMAC_REG_WRITE_8(
+			halmac_adapter, REG_SECCFG,
+			HALMAC_REG_READ_8(halmac_adapter, REG_SECCFG) | BIT(3));
+	else
+		HALMAC_REG_WRITE_8(
+			halmac_adapter, REG_SECCFG,
+			HALMAC_REG_READ_8(halmac_adapter, REG_SECCFG) &
+				~(BIT(3)));
+
+	if (sec_setting->bip_enable == 1) {
+		if (halmac_adapter->chip_id == HALMAC_CHIP_ID_8822B)
+			return HALMAC_RET_BIP_NO_SUPPORT;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_COMMON, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+u8 halmac_get_used_cam_entry_num_88xx(struct halmac_adapter *halmac_adapter,
+				      enum hal_security_type sec_type)
+{
+	u8 entry_num;
+	void *driver_adapter = NULL;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_COMMON, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	switch (sec_type) {
+	case HAL_SECURITY_TYPE_WEP40:
+	case HAL_SECURITY_TYPE_WEP104:
+	case HAL_SECURITY_TYPE_TKIP:
+	case HAL_SECURITY_TYPE_AES128:
+	case HAL_SECURITY_TYPE_GCMP128:
+	case HAL_SECURITY_TYPE_GCMSMS4:
+	case HAL_SECURITY_TYPE_BIP:
+		entry_num = 1;
+		break;
+	case HAL_SECURITY_TYPE_WAPI:
+	case HAL_SECURITY_TYPE_AES256:
+	case HAL_SECURITY_TYPE_GCMP256:
+		entry_num = 2;
+		break;
+	default:
+		entry_num = 0;
+		break;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_COMMON, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return entry_num;
+}
+
+enum halmac_ret_status
+halmac_write_cam_88xx(struct halmac_adapter *halmac_adapter, u32 entry_index,
+		      struct halmac_cam_entry_info *cam_entry_info)
+{
+	u32 i;
+	u32 command = 0x80010000;
+	struct halmac_api *halmac_api;
+	void *driver_adapter = NULL;
+	struct halmac_cam_entry_format *cam_entry_format = NULL;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_COMMON, DBG_DMESG,
+			"[TRACE]%s ==========>\n", __func__);
+
+	if (entry_index >= halmac_adapter->hw_config_info.cam_entry_num)
+		return HALMAC_RET_ENTRY_INDEX_ERROR;
+
+	if (cam_entry_info->key_id > 3)
+		return HALMAC_RET_FAIL;
+
+	cam_entry_format = kzalloc(sizeof(*cam_entry_format), GFP_KERNEL);
+	if (!cam_entry_format)
+		return HALMAC_RET_NULL_POINTER;
+
+	cam_entry_format->key_id = cam_entry_info->key_id;
+	cam_entry_format->valid = cam_entry_info->valid;
+	memcpy(cam_entry_format->mac_address, cam_entry_info->mac_address, 6);
+	memcpy(cam_entry_format->key, cam_entry_info->key, 16);
+
+	switch (cam_entry_info->security_type) {
+	case HAL_SECURITY_TYPE_NONE:
+		cam_entry_format->type = 0;
+		break;
+	case HAL_SECURITY_TYPE_WEP40:
+		cam_entry_format->type = 1;
+		break;
+	case HAL_SECURITY_TYPE_WEP104:
+		cam_entry_format->type = 5;
+		break;
+	case HAL_SECURITY_TYPE_TKIP:
+		cam_entry_format->type = 2;
+		break;
+	case HAL_SECURITY_TYPE_AES128:
+		cam_entry_format->type = 4;
+		break;
+	case HAL_SECURITY_TYPE_WAPI:
+		cam_entry_format->type = 6;
+		break;
+	case HAL_SECURITY_TYPE_AES256:
+		cam_entry_format->type = 4;
+		cam_entry_format->ext_sectype = 1;
+		break;
+	case HAL_SECURITY_TYPE_GCMP128:
+		cam_entry_format->type = 7;
+		break;
+	case HAL_SECURITY_TYPE_GCMP256:
+	case HAL_SECURITY_TYPE_GCMSMS4:
+		cam_entry_format->type = 7;
+		cam_entry_format->ext_sectype = 1;
+		break;
+	case HAL_SECURITY_TYPE_BIP:
+		cam_entry_format->type = cam_entry_info->unicast == 1 ? 4 : 0;
+		cam_entry_format->mgnt = 1;
+		cam_entry_format->grp = cam_entry_info->unicast == 1 ? 0 : 1;
+		break;
+	default:
+		kfree(cam_entry_format);
+		return HALMAC_RET_FAIL;
+	}
+
+	for (i = 0; i < 8; i++) {
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_CAMWRITE,
+				    *((u32 *)cam_entry_format + i));
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_CAMCMD,
+				    command | ((entry_index << 3) + i));
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_COMMON, DBG_DMESG,
+				"[TRACE]1 - CAM entry format : %X\n",
+				*((u32 *)cam_entry_format + i));
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_COMMON, DBG_DMESG,
+				"[TRACE]1 - REG_CAMCMD : %X\n",
+				command | ((entry_index << 3) + i));
+	}
+
+	if (cam_entry_info->security_type == HAL_SECURITY_TYPE_WAPI ||
+	    cam_entry_info->security_type == HAL_SECURITY_TYPE_AES256 ||
+	    cam_entry_info->security_type == HAL_SECURITY_TYPE_GCMP256 ||
+	    cam_entry_info->security_type == HAL_SECURITY_TYPE_GCMSMS4) {
+		cam_entry_format->mic = 1;
+		memcpy(cam_entry_format->key, cam_entry_info->key_ext, 16);
+
+		for (i = 0; i < 8; i++) {
+			HALMAC_REG_WRITE_32(halmac_adapter, REG_CAMWRITE,
+					    *((u32 *)cam_entry_format + i));
+			HALMAC_REG_WRITE_32(
+				halmac_adapter, REG_CAMCMD,
+				command | (((entry_index + 1) << 3) + i));
+			HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_COMMON,
+					DBG_DMESG,
+					"[TRACE]2 - CAM entry format : %X\n",
+					*((u32 *)cam_entry_format + i));
+			HALMAC_RT_TRACE(
+				driver_adapter, HALMAC_MSG_COMMON, DBG_DMESG,
+				"[TRACE]2 - REG_CAMCMD : %X\n",
+				command | (((entry_index + 1) << 3) + i));
+		}
+	}
+
+	kfree(cam_entry_format);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_COMMON, DBG_DMESG,
+			"[TRACE]%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_read_cam_entry_88xx(struct halmac_adapter *halmac_adapter,
+			   u32 entry_index,
+			   struct halmac_cam_entry_format *content)
+{
+	u32 i;
+	u32 command = 0x80000000;
+	struct halmac_api *halmac_api;
+	void *driver_adapter = NULL;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_COMMON, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	if (entry_index >= halmac_adapter->hw_config_info.cam_entry_num)
+		return HALMAC_RET_ENTRY_INDEX_ERROR;
+
+	for (i = 0; i < 8; i++) {
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_CAMCMD,
+				    command | ((entry_index << 3) + i));
+		*((u32 *)content + i) =
+			HALMAC_REG_READ_32(halmac_adapter, REG_CAMREAD);
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_COMMON, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_clear_cam_entry_88xx(struct halmac_adapter *halmac_adapter,
+			    u32 entry_index)
+{
+	u32 i;
+	u32 command = 0x80010000;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	struct halmac_cam_entry_format *cam_entry_format;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"[TRACE]halmac_clear_security_cam_88xx ==========>\n");
+
+	if (entry_index >= halmac_adapter->hw_config_info.cam_entry_num)
+		return HALMAC_RET_ENTRY_INDEX_ERROR;
+
+	cam_entry_format = kzalloc(sizeof(*cam_entry_format), GFP_KERNEL);
+	if (!cam_entry_format)
+		return HALMAC_RET_NULL_POINTER;
+
+	for (i = 0; i < 8; i++) {
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_CAMWRITE,
+				    *((u32 *)cam_entry_format + i));
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_CAMCMD,
+				    command | ((entry_index << 3) + i));
+	}
+
+	kfree(cam_entry_format);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"[TRACE]halmac_clear_security_cam_88xx <==========\n");
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_get_hw_value_88xx() -get hw config value
+ * @halmac_adapter : the adapter of halmac
+ * @hw_id : hw id for driver to query
+ * @pvalue : hw value, reference table to get data type
+ * Author : KaiYuan Chang / Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_get_hw_value_88xx(struct halmac_adapter *halmac_adapter,
+			 enum halmac_hw_id hw_id, void *pvalue)
+{
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_GET_HW_VALUE);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	if (!pvalue) {
+		pr_err("%s (!pvalue)==========>\n", __func__);
+		return HALMAC_RET_NULL_POINTER;
+	}
+
+	switch (hw_id) {
+	case HALMAC_HW_RQPN_MAPPING:
+		((struct halmac_rqpn_map *)pvalue)->dma_map_vo =
+			halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_VO];
+		((struct halmac_rqpn_map *)pvalue)->dma_map_vi =
+			halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_VI];
+		((struct halmac_rqpn_map *)pvalue)->dma_map_be =
+			halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_BE];
+		((struct halmac_rqpn_map *)pvalue)->dma_map_bk =
+			halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_BK];
+		((struct halmac_rqpn_map *)pvalue)->dma_map_mg =
+			halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_MG];
+		((struct halmac_rqpn_map *)pvalue)->dma_map_hi =
+			halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_HI];
+		break;
+	case HALMAC_HW_EFUSE_SIZE:
+		*(u32 *)pvalue = halmac_adapter->hw_config_info.efuse_size;
+		break;
+	case HALMAC_HW_EEPROM_SIZE:
+		*(u32 *)pvalue = halmac_adapter->hw_config_info.eeprom_size;
+		break;
+	case HALMAC_HW_BT_BANK_EFUSE_SIZE:
+		*(u32 *)pvalue = halmac_adapter->hw_config_info.bt_efuse_size;
+		break;
+	case HALMAC_HW_BT_BANK1_EFUSE_SIZE:
+	case HALMAC_HW_BT_BANK2_EFUSE_SIZE:
+		*(u32 *)pvalue = 0;
+		break;
+	case HALMAC_HW_TXFIFO_SIZE:
+		*(u32 *)pvalue = halmac_adapter->hw_config_info.tx_fifo_size;
+		break;
+	case HALMAC_HW_RSVD_PG_BNDY:
+		*(u16 *)pvalue =
+			halmac_adapter->txff_allocation.rsvd_drv_pg_bndy;
+		break;
+	case HALMAC_HW_CAM_ENTRY_NUM:
+		*(u8 *)pvalue = halmac_adapter->hw_config_info.cam_entry_num;
+		break;
+	case HALMAC_HW_WLAN_EFUSE_AVAILABLE_SIZE: /*Remove later*/
+		status = halmac_dump_logical_efuse_map_88xx(halmac_adapter,
+							    HALMAC_EFUSE_R_DRV);
+		if (status != HALMAC_RET_SUCCESS)
+			return status;
+		*(u32 *)pvalue = halmac_adapter->hw_config_info.efuse_size -
+				 HALMAC_PROTECTED_EFUSE_SIZE_88XX -
+				 halmac_adapter->efuse_end;
+		break;
+	case HALMAC_HW_IC_VERSION:
+		*(u8 *)pvalue = halmac_adapter->chip_version;
+		break;
+	case HALMAC_HW_PAGE_SIZE:
+		*(u32 *)pvalue = halmac_adapter->hw_config_info.page_size;
+		break;
+	case HALMAC_HW_TX_AGG_ALIGN_SIZE:
+		*(u16 *)pvalue = halmac_adapter->hw_config_info.tx_align_size;
+		break;
+	case HALMAC_HW_RX_AGG_ALIGN_SIZE:
+		*(u8 *)pvalue = 8;
+		break;
+	case HALMAC_HW_DRV_INFO_SIZE:
+		*(u8 *)pvalue = halmac_adapter->drv_info_size;
+		break;
+	case HALMAC_HW_TXFF_ALLOCATION:
+		memcpy(pvalue, &halmac_adapter->txff_allocation,
+		       sizeof(struct halmac_txff_allocation));
+		break;
+	case HALMAC_HW_TX_DESC_SIZE:
+		*(u32 *)pvalue = halmac_adapter->hw_config_info.txdesc_size;
+		break;
+	case HALMAC_HW_RX_DESC_SIZE:
+		*(u32 *)pvalue = halmac_adapter->hw_config_info.rxdesc_size;
+		break;
+	default:
+		return HALMAC_RET_PARA_NOT_SUPPORT;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_set_hw_value_88xx() -set hw config value
+ * @halmac_adapter : the adapter of halmac
+ * @hw_id : hw id for driver to config
+ * @pvalue : hw value, reference table to get data type
+ * Author : KaiYuan Chang / Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_set_hw_value_88xx(struct halmac_adapter *halmac_adapter,
+			 enum halmac_hw_id hw_id, void *pvalue)
+{
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_GET_HW_VALUE);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	if (!pvalue) {
+		pr_err("%s (!pvalue)==========>\n", __func__);
+		return HALMAC_RET_NULL_POINTER;
+	}
+
+	switch (hw_id) {
+	case HALMAC_HW_USB_MODE:
+		status = halmac_set_usb_mode_88xx(
+			halmac_adapter, *(enum halmac_usb_mode *)pvalue);
+		if (status != HALMAC_RET_SUCCESS)
+			return status;
+		break;
+	case HALMAC_HW_SEQ_EN:
+		break;
+	case HALMAC_HW_BANDWIDTH:
+		halmac_cfg_bw_88xx(halmac_adapter, *(enum halmac_bw *)pvalue);
+		break;
+	case HALMAC_HW_CHANNEL:
+		halmac_cfg_ch_88xx(halmac_adapter, *(u8 *)pvalue);
+		break;
+	case HALMAC_HW_PRI_CHANNEL_IDX:
+		halmac_cfg_pri_ch_idx_88xx(halmac_adapter,
+					   *(enum halmac_pri_ch_idx *)pvalue);
+		break;
+	case HALMAC_HW_EN_BB_RF:
+		halmac_enable_bb_rf_88xx(halmac_adapter, *(u8 *)pvalue);
+		break;
+	case HALMAC_HW_SDIO_TX_PAGE_THRESHOLD:
+		halmac_config_sdio_tx_page_threshold_88xx(
+			halmac_adapter,
+			(struct halmac_tx_page_threshold_info *)pvalue);
+		break;
+	case HALMAC_HW_AMPDU_CONFIG:
+		halmac_config_ampdu_88xx(halmac_adapter,
+					 (struct halmac_ampdu_config *)pvalue);
+		break;
+	default:
+		return HALMAC_RET_PARA_NOT_SUPPORT;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_cfg_drv_rsvd_pg_num_88xx() -config reserved page number for driver
+ * @halmac_adapter : the adapter of halmac
+ * @pg_num : page number
+ * Author : KaiYuan Chang
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_cfg_drv_rsvd_pg_num_88xx(struct halmac_adapter *halmac_adapter,
+				enum halmac_drv_rsvd_pg_num pg_num)
+{
+	void *driver_adapter = NULL;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter,
+				  HALMAC_API_CFG_DRV_RSVD_PG_NUM);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s ==========>pg_num = %d\n", __func__,
+			pg_num);
+
+	switch (pg_num) {
+	case HALMAC_RSVD_PG_NUM16:
+		halmac_adapter->txff_allocation.rsvd_drv_pg_num = 16;
+		break;
+	case HALMAC_RSVD_PG_NUM24:
+		halmac_adapter->txff_allocation.rsvd_drv_pg_num = 24;
+		break;
+	case HALMAC_RSVD_PG_NUM32:
+		halmac_adapter->txff_allocation.rsvd_drv_pg_num = 32;
+		break;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_get_chip_version_88xx(struct halmac_adapter *halmac_adapter,
+			     struct halmac_ver *version)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s ==========>\n", __func__);
+	version->major_ver = (u8)HALMAC_MAJOR_VER_88XX;
+	version->prototype_ver = (u8)HALMAC_PROTOTYPE_VER_88XX;
+	version->minor_ver = (u8)HALMAC_MINOR_VER_88XX;
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_chk_txdesc_88xx() -check if the tx packet format is incorrect
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_buf : tx Packet buffer, tx desc is included
+ * @halmac_size : tx packet size
+ * Author : KaiYuan Chang
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_chk_txdesc_88xx(struct halmac_adapter *halmac_adapter, u8 *halmac_buf,
+		       u32 halmac_size)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	if (GET_TX_DESC_BMC(halmac_buf))
+		if (GET_TX_DESC_AGG_EN(halmac_buf))
+			pr_err("TxDesc: Agg should not be set when BMC\n");
+
+	if (halmac_size < (GET_TX_DESC_TXPKTSIZE(halmac_buf) +
+			   GET_TX_DESC_OFFSET(halmac_buf)))
+		pr_err("TxDesc: PktSize too small\n");
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_dl_drv_rsvd_page_88xx() - download packet to rsvd page
+ * @halmac_adapter : the adapter of halmac
+ * @pg_offset : page offset of driver's rsvd page
+ * @halmac_buf : data to be downloaded, tx_desc is not included
+ * @halmac_size : data size to be downloaded
+ * Author : KaiYuan Chang
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_dl_drv_rsvd_page_88xx(struct halmac_adapter *halmac_adapter,
+			     u8 pg_offset, u8 *halmac_buf, u32 halmac_size)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	enum halmac_ret_status ret_status;
+	u16 drv_pg_bndy = 0;
+	u32 dl_pg_num = 0;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_DL_DRV_RSVD_PG);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	/*check boundary and size valid*/
+	dl_pg_num = halmac_size / halmac_adapter->hw_config_info.page_size +
+		    ((halmac_size &
+		      (halmac_adapter->hw_config_info.page_size - 1)) ?
+			     1 :
+			     0);
+	if (pg_offset + dl_pg_num >
+	    halmac_adapter->txff_allocation.rsvd_drv_pg_num) {
+		pr_err("[ERROR] driver download offset or size error ==========>\n");
+		return HALMAC_RET_DRV_DL_ERR;
+	}
+
+	/*update to target download boundary*/
+	drv_pg_bndy =
+		halmac_adapter->txff_allocation.rsvd_drv_pg_bndy + pg_offset;
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_FIFOPAGE_CTRL_2,
+			    (u16)(drv_pg_bndy & BIT_MASK_BCN_HEAD_1_V1));
+
+	ret_status = halmac_download_rsvd_page_88xx(halmac_adapter, halmac_buf,
+						    halmac_size);
+
+	/*restore to original bundary*/
+	if (ret_status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_download_rsvd_page_88xx Fail = %x!!\n",
+		       ret_status);
+		HALMAC_REG_WRITE_16(
+			halmac_adapter, REG_FIFOPAGE_CTRL_2,
+			(u16)(halmac_adapter->txff_allocation.rsvd_pg_bndy &
+			      BIT_MASK_BCN_HEAD_1_V1));
+		return ret_status;
+	}
+
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_FIFOPAGE_CTRL_2,
+			    (u16)(halmac_adapter->txff_allocation.rsvd_pg_bndy &
+				  BIT_MASK_BCN_HEAD_1_V1));
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s < ==========\n", __func__);
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_cfg_csi_rate_88xx() - config CSI frame Tx rate
+ * @halmac_adapter : the adapter of halmac
+ * @rssi : rssi in decimal value
+ * @current_rate : current CSI frame rate
+ * @fixrate_en : enable to fix CSI frame in VHT rate, otherwise legacy OFDM rate
+ * @new_rate : API returns the final CSI frame rate
+ * Author : chunchu
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_cfg_csi_rate_88xx(struct halmac_adapter *halmac_adapter, u8 rssi,
+			 u8 current_rate, u8 fixrate_en, u8 *new_rate)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	u32 temp_csi_setting;
+	u16 current_rrsr;
+	enum halmac_ret_status ret_status;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_CFG_CSI_RATE);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_SND, DBG_DMESG,
+			"<%s ==========>\n", __func__);
+
+	temp_csi_setting = HALMAC_REG_READ_32(halmac_adapter, REG_BBPSF_CTRL) &
+			   ~(BIT_MASK_WMAC_CSI_RATE << BIT_SHIFT_WMAC_CSI_RATE);
+
+	current_rrsr = HALMAC_REG_READ_16(halmac_adapter, REG_RRSR);
+
+	if (rssi >= 40) {
+		if (current_rate != HALMAC_OFDM54) {
+			HALMAC_REG_WRITE_16(halmac_adapter, REG_RRSR,
+					    current_rrsr | BIT(HALMAC_OFDM54));
+			HALMAC_REG_WRITE_32(
+				halmac_adapter, REG_BBPSF_CTRL,
+				temp_csi_setting |
+					BIT_WMAC_CSI_RATE(HALMAC_OFDM54));
+		}
+		*new_rate = HALMAC_OFDM54;
+		ret_status = HALMAC_RET_SUCCESS;
+	} else {
+		if (current_rate != HALMAC_OFDM24) {
+			HALMAC_REG_WRITE_16(halmac_adapter, REG_RRSR,
+					    current_rrsr &
+						    ~(BIT(HALMAC_OFDM54)));
+			HALMAC_REG_WRITE_32(
+				halmac_adapter, REG_BBPSF_CTRL,
+				temp_csi_setting |
+					BIT_WMAC_CSI_RATE(HALMAC_OFDM24));
+		}
+		*new_rate = HALMAC_OFDM24;
+		ret_status = HALMAC_RET_SUCCESS;
+	}
+
+	return ret_status;
+}
+
+/**
+ * halmac_sdio_cmd53_4byte_88xx() - cmd53 only for 4byte len register IO
+ * @halmac_adapter : the adapter of halmac
+ * @enable : 1->CMD53 only use in 4byte reg, 0 : No limitation
+ * Author : Ivan Lin/KaiYuan Chang
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_sdio_cmd53_4byte_88xx(struct halmac_adapter *halmac_adapter,
+			     enum halmac_sdio_cmd53_4byte_mode cmd53_4byte_mode)
+{
+	halmac_adapter->sdio_cmd53_4byte = cmd53_4byte_mode;
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_txfifo_is_empty_88xx() -check if txfifo is empty
+ * @halmac_adapter : the adapter of halmac
+ * Author : Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_txfifo_is_empty_88xx(struct halmac_adapter *halmac_adapter, u32 chk_num)
+{
+	u32 counter;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_COMMON, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	counter = (chk_num <= 10) ? 10 : chk_num;
+	do {
+		if (HALMAC_REG_READ_8(halmac_adapter, REG_TXPKT_EMPTY) != 0xFF)
+			return HALMAC_RET_TXFIFO_NO_EMPTY;
+
+		if ((HALMAC_REG_READ_8(halmac_adapter, REG_TXPKT_EMPTY + 1) &
+		     0x07) != 0x07)
+			return HALMAC_RET_TXFIFO_NO_EMPTY;
+		counter--;
+
+	} while (counter != 0);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_COMMON, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
diff --git a/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_api_88xx.h b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_api_88xx.h
new file mode 100644
index 000000000000..5debd1ff3abd
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_api_88xx.h
@@ -0,0 +1,396 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HALMAC_API_88XX_H_
+#define _HALMAC_API_88XX_H_
+
+#include "../halmac_2_platform.h"
+#include "../halmac_type.h"
+
+void halmac_init_state_machine_88xx(struct halmac_adapter *halmac_adapter);
+
+void halmac_init_adapter_para_88xx(struct halmac_adapter *halmac_adapter);
+
+void halmac_init_adapter_dynamic_para_88xx(
+	struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_mount_api_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_download_firmware_88xx(struct halmac_adapter *halmac_adapter,
+			      u8 *hamacl_fw, u32 halmac_fw_size);
+
+enum halmac_ret_status
+halmac_free_download_firmware_88xx(struct halmac_adapter *halmac_adapter,
+				   enum halmac_dlfw_mem dlfw_mem, u8 *hamacl_fw,
+				   u32 halmac_fw_size);
+
+enum halmac_ret_status
+halmac_get_fw_version_88xx(struct halmac_adapter *halmac_adapter,
+			   struct halmac_fw_version *fw_version);
+
+enum halmac_ret_status
+halmac_cfg_mac_addr_88xx(struct halmac_adapter *halmac_adapter, u8 halmac_port,
+			 union halmac_wlan_addr *hal_address);
+
+enum halmac_ret_status
+halmac_cfg_bssid_88xx(struct halmac_adapter *halmac_adapter, u8 halmac_port,
+		      union halmac_wlan_addr *hal_address);
+
+enum halmac_ret_status
+halmac_cfg_multicast_addr_88xx(struct halmac_adapter *halmac_adapter,
+			       union halmac_wlan_addr *hal_address);
+
+enum halmac_ret_status
+halmac_pre_init_system_cfg_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_init_system_cfg_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_cfg_rx_aggregation_88xx(struct halmac_adapter *halmac_adapter,
+			       struct halmac_rxagg_cfg halmac_rxagg_cfg);
+
+enum halmac_ret_status
+halmac_init_edca_cfg_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_cfg_operation_mode_88xx(struct halmac_adapter *halmac_adapter,
+			       enum halmac_wireless_mode wireless_mode);
+
+enum halmac_ret_status
+halmac_cfg_ch_bw_88xx(struct halmac_adapter *halmac_adapter, u8 channel,
+		      enum halmac_pri_ch_idx pri_ch_idx, enum halmac_bw bw);
+
+enum halmac_ret_status halmac_cfg_ch_88xx(struct halmac_adapter *halmac_adapter,
+					  u8 channel);
+
+enum halmac_ret_status
+halmac_cfg_pri_ch_idx_88xx(struct halmac_adapter *halmac_adapter,
+			   enum halmac_pri_ch_idx pri_ch_idx);
+
+enum halmac_ret_status halmac_cfg_bw_88xx(struct halmac_adapter *halmac_adapter,
+					  enum halmac_bw bw);
+
+enum halmac_ret_status
+halmac_init_wmac_cfg_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_init_mac_cfg_88xx(struct halmac_adapter *halmac_adapter,
+			 enum halmac_trx_mode mode);
+
+enum halmac_ret_status
+halmac_dump_efuse_map_88xx(struct halmac_adapter *halmac_adapter,
+			   enum halmac_efuse_read_cfg cfg);
+
+enum halmac_ret_status
+halmac_dump_efuse_map_bt_88xx(struct halmac_adapter *halmac_adapter,
+			      enum halmac_efuse_bank halmac_efuse_bank,
+			      u32 bt_efuse_map_size, u8 *bt_efuse_map);
+
+enum halmac_ret_status
+halmac_write_efuse_bt_88xx(struct halmac_adapter *halmac_adapter,
+			   u32 halmac_offset, u8 halmac_value,
+			   enum halmac_efuse_bank halmac_efuse_bank);
+
+enum halmac_ret_status
+halmac_pg_efuse_by_map_88xx(struct halmac_adapter *halmac_adapter,
+			    struct halmac_pg_efuse_info *pg_efuse_info,
+			    enum halmac_efuse_read_cfg cfg);
+
+enum halmac_ret_status
+halmac_get_efuse_size_88xx(struct halmac_adapter *halmac_adapter,
+			   u32 *halmac_size);
+
+enum halmac_ret_status
+halmac_get_efuse_available_size_88xx(struct halmac_adapter *halmac_adapter,
+				     u32 *halmac_size);
+
+enum halmac_ret_status
+halmac_get_c2h_info_88xx(struct halmac_adapter *halmac_adapter, u8 *halmac_buf,
+			 u32 halmac_size);
+
+enum halmac_ret_status
+halmac_get_logical_efuse_size_88xx(struct halmac_adapter *halmac_adapter,
+				   u32 *halmac_size);
+
+enum halmac_ret_status
+halmac_dump_logical_efuse_map_88xx(struct halmac_adapter *halmac_adapter,
+				   enum halmac_efuse_read_cfg cfg);
+
+enum halmac_ret_status
+halmac_write_logical_efuse_88xx(struct halmac_adapter *halmac_adapter,
+				u32 halmac_offset, u8 halmac_value);
+
+enum halmac_ret_status
+halmac_read_logical_efuse_88xx(struct halmac_adapter *halmac_adapter,
+			       u32 halmac_offset, u8 *value);
+
+enum halmac_ret_status
+halmac_cfg_fwlps_option_88xx(struct halmac_adapter *halmac_adapter,
+			     struct halmac_fwlps_option *lps_option);
+
+enum halmac_ret_status
+halmac_cfg_fwips_option_88xx(struct halmac_adapter *halmac_adapter,
+			     struct halmac_fwips_option *ips_option);
+
+enum halmac_ret_status
+halmac_enter_wowlan_88xx(struct halmac_adapter *halmac_adapter,
+			 struct halmac_wowlan_option *wowlan_option);
+
+enum halmac_ret_status
+halmac_leave_wowlan_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_enter_ps_88xx(struct halmac_adapter *halmac_adapter,
+		     enum halmac_ps_state ps_state);
+
+enum halmac_ret_status
+halmac_leave_ps_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_h2c_lb_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status halmac_debug_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_cfg_parameter_88xx(struct halmac_adapter *halmac_adapter,
+			  struct halmac_phy_parameter_info *para_info,
+			  u8 full_fifo);
+
+enum halmac_ret_status
+halmac_update_packet_88xx(struct halmac_adapter *halmac_adapter,
+			  enum halmac_packet_id pkt_id, u8 *pkt, u32 pkt_size);
+
+enum halmac_ret_status
+halmac_bcn_ie_filter_88xx(struct halmac_adapter *halmac_adapter,
+			  struct halmac_bcn_ie_info *bcn_ie_info);
+
+enum halmac_ret_status
+halmac_send_original_h2c_88xx(struct halmac_adapter *halmac_adapter,
+			      u8 *original_h2c, u16 *seq, u8 ack);
+
+enum halmac_ret_status
+halmac_update_datapack_88xx(struct halmac_adapter *halmac_adapter,
+			    enum halmac_data_type halmac_data_type,
+			    struct halmac_phy_parameter_info *para_info);
+
+enum halmac_ret_status
+halmac_run_datapack_88xx(struct halmac_adapter *halmac_adapter,
+			 enum halmac_data_type halmac_data_type);
+
+enum halmac_ret_status
+halmac_cfg_drv_info_88xx(struct halmac_adapter *halmac_adapter,
+			 enum halmac_drv_info halmac_drv_info);
+
+enum halmac_ret_status
+halmac_send_bt_coex_88xx(struct halmac_adapter *halmac_adapter, u8 *bt_buf,
+			 u32 bt_size, u8 ack);
+
+enum halmac_ret_status
+halmac_verify_platform_api_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_timer_2s_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_fill_txdesc_check_sum_88xx(struct halmac_adapter *halmac_adapter,
+				  u8 *cur_desc);
+
+enum halmac_ret_status
+halmac_dump_fifo_88xx(struct halmac_adapter *halmac_adapter,
+		      enum hal_fifo_sel halmac_fifo_sel, u32 halmac_start_addr,
+		      u32 halmac_fifo_dump_size, u8 *fifo_map);
+
+u32 halmac_get_fifo_size_88xx(struct halmac_adapter *halmac_adapter,
+			      enum hal_fifo_sel halmac_fifo_sel);
+
+enum halmac_ret_status
+halmac_cfg_txbf_88xx(struct halmac_adapter *halmac_adapter, u8 userid,
+		     enum halmac_bw bw, u8 txbf_en);
+
+enum halmac_ret_status
+halmac_cfg_mumimo_88xx(struct halmac_adapter *halmac_adapter,
+		       struct halmac_cfg_mumimo_para *cfgmu);
+
+enum halmac_ret_status
+halmac_cfg_sounding_88xx(struct halmac_adapter *halmac_adapter,
+			 enum halmac_snd_role role,
+			 enum halmac_data_rate datarate);
+
+enum halmac_ret_status
+halmac_del_sounding_88xx(struct halmac_adapter *halmac_adapter,
+			 enum halmac_snd_role role);
+
+enum halmac_ret_status
+halmac_su_bfee_entry_init_88xx(struct halmac_adapter *halmac_adapter, u8 userid,
+			       u16 paid);
+
+enum halmac_ret_status
+halmac_su_bfer_entry_init_88xx(struct halmac_adapter *halmac_adapter,
+			       struct halmac_su_bfer_init_para *su_bfer_init);
+
+enum halmac_ret_status
+halmac_mu_bfee_entry_init_88xx(struct halmac_adapter *halmac_adapter,
+			       struct halmac_mu_bfee_init_para *mu_bfee_init);
+
+enum halmac_ret_status
+halmac_mu_bfer_entry_init_88xx(struct halmac_adapter *halmac_adapter,
+			       struct halmac_mu_bfer_init_para *mu_bfer_init);
+
+enum halmac_ret_status
+halmac_su_bfee_entry_del_88xx(struct halmac_adapter *halmac_adapter, u8 userid);
+
+enum halmac_ret_status
+halmac_su_bfer_entry_del_88xx(struct halmac_adapter *halmac_adapter, u8 userid);
+
+enum halmac_ret_status
+halmac_mu_bfee_entry_del_88xx(struct halmac_adapter *halmac_adapter, u8 userid);
+
+enum halmac_ret_status
+halmac_mu_bfer_entry_del_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_add_ch_info_88xx(struct halmac_adapter *halmac_adapter,
+			struct halmac_ch_info *ch_info);
+
+enum halmac_ret_status
+halmac_add_extra_ch_info_88xx(struct halmac_adapter *halmac_adapter,
+			      struct halmac_ch_extra_info *ch_extra_info);
+
+enum halmac_ret_status
+halmac_ctrl_ch_switch_88xx(struct halmac_adapter *halmac_adapter,
+			   struct halmac_ch_switch_option *cs_option);
+
+enum halmac_ret_status halmac_p2pps_88xx(struct halmac_adapter *halmac_adapter,
+					 struct halmac_p2pps *p2p_ps);
+
+enum halmac_ret_status
+halmac_func_p2pps_88xx(struct halmac_adapter *halmac_adapter,
+		       struct halmac_p2pps *p2p_ps);
+
+enum halmac_ret_status
+halmac_clear_ch_info_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_send_general_info_88xx(struct halmac_adapter *halmac_adapter,
+			      struct halmac_general_info *general_info);
+
+enum halmac_ret_status
+halmac_start_iqk_88xx(struct halmac_adapter *halmac_adapter,
+		      struct halmac_iqk_para_ *iqk_para);
+
+enum halmac_ret_status halmac_ctrl_pwr_tracking_88xx(
+	struct halmac_adapter *halmac_adapter,
+	struct halmac_pwr_tracking_option *pwr_tracking_opt);
+
+enum halmac_ret_status
+halmac_query_status_88xx(struct halmac_adapter *halmac_adapter,
+			 enum halmac_feature_id feature_id,
+			 enum halmac_cmd_process_status *process_status,
+			 u8 *data, u32 *size);
+
+enum halmac_ret_status
+halmac_reset_feature_88xx(struct halmac_adapter *halmac_adapter,
+			  enum halmac_feature_id feature_id);
+
+enum halmac_ret_status
+halmac_check_fw_status_88xx(struct halmac_adapter *halmac_adapter,
+			    bool *fw_status);
+
+enum halmac_ret_status
+halmac_dump_fw_dmem_88xx(struct halmac_adapter *halmac_adapter, u8 *dmem,
+			 u32 *size);
+
+enum halmac_ret_status
+halmac_cfg_max_dl_size_88xx(struct halmac_adapter *halmac_adapter, u32 size);
+
+enum halmac_ret_status halmac_psd_88xx(struct halmac_adapter *halmac_adapter,
+				       u16 start_psd, u16 end_psd);
+
+enum halmac_ret_status
+halmac_cfg_la_mode_88xx(struct halmac_adapter *halmac_adapter,
+			enum halmac_la_mode la_mode);
+
+enum halmac_ret_status halmac_cfg_rx_fifo_expanding_mode_88xx(
+	struct halmac_adapter *halmac_adapter,
+	enum halmac_rx_fifo_expanding_mode rx_fifo_expanding_mode);
+
+enum halmac_ret_status
+halmac_config_security_88xx(struct halmac_adapter *halmac_adapter,
+			    struct halmac_security_setting *sec_setting);
+
+u8 halmac_get_used_cam_entry_num_88xx(struct halmac_adapter *halmac_adapter,
+				      enum hal_security_type sec_type);
+
+enum halmac_ret_status
+halmac_write_cam_88xx(struct halmac_adapter *halmac_adapter, u32 entry_index,
+		      struct halmac_cam_entry_info *cam_entry_info);
+
+enum halmac_ret_status
+halmac_read_cam_entry_88xx(struct halmac_adapter *halmac_adapter,
+			   u32 entry_index,
+			   struct halmac_cam_entry_format *content);
+
+enum halmac_ret_status
+halmac_clear_cam_entry_88xx(struct halmac_adapter *halmac_adapter,
+			    u32 entry_index);
+
+enum halmac_ret_status
+halmac_get_hw_value_88xx(struct halmac_adapter *halmac_adapter,
+			 enum halmac_hw_id hw_id, void *pvalue);
+
+enum halmac_ret_status
+halmac_set_hw_value_88xx(struct halmac_adapter *halmac_adapter,
+			 enum halmac_hw_id hw_id, void *pvalue);
+
+enum halmac_ret_status
+halmac_cfg_drv_rsvd_pg_num_88xx(struct halmac_adapter *halmac_adapter,
+				enum halmac_drv_rsvd_pg_num pg_num);
+
+enum halmac_ret_status
+halmac_get_chip_version_88xx(struct halmac_adapter *halmac_adapter,
+			     struct halmac_ver *version);
+
+enum halmac_ret_status
+halmac_chk_txdesc_88xx(struct halmac_adapter *halmac_adapter, u8 *halmac_buf,
+		       u32 halmac_size);
+
+enum halmac_ret_status
+halmac_dl_drv_rsvd_page_88xx(struct halmac_adapter *halmac_adapter,
+			     u8 pg_offset, u8 *halmac_buf, u32 halmac_size);
+
+enum halmac_ret_status
+halmac_cfg_csi_rate_88xx(struct halmac_adapter *halmac_adapter, u8 rssi,
+			 u8 current_rate, u8 fixrate_en, u8 *new_rate);
+
+enum halmac_ret_status halmac_sdio_cmd53_4byte_88xx(
+	struct halmac_adapter *halmac_adapter,
+	enum halmac_sdio_cmd53_4byte_mode cmd53_4byte_mode);
+
+enum halmac_ret_status
+halmac_txfifo_is_empty_88xx(struct halmac_adapter *halmac_adapter, u32 chk_num);
+
+#endif /* _HALMAC_API_H_ */
diff --git a/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_api_88xx_pcie.c b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_api_88xx_pcie.c
new file mode 100644
index 000000000000..fa97cac34742
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_api_88xx_pcie.c
@@ -0,0 +1,329 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#include "halmac_88xx_cfg.h"
+
+/**
+ * halmac_init_pcie_cfg_88xx() -  init PCIe
+ * @halmac_adapter : the adapter of halmac
+ * Author : KaiYuan Chang
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_init_pcie_cfg_88xx(struct halmac_adapter *halmac_adapter)
+{
+	void *driver_adapter = NULL;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_INIT_PCIE_CFG);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_deinit_pcie_cfg_88xx() - deinit PCIE
+ * @halmac_adapter : the adapter of halmac
+ * Author : KaiYuan Chang
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_deinit_pcie_cfg_88xx(struct halmac_adapter *halmac_adapter)
+{
+	void *driver_adapter = NULL;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_DEINIT_PCIE_CFG);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_cfg_rx_aggregation_88xx_pcie() - config rx aggregation
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_rx_agg_mode
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_cfg_rx_aggregation_88xx_pcie(struct halmac_adapter *halmac_adapter,
+				    struct halmac_rxagg_cfg *phalmac_rxagg_cfg)
+{
+	void *driver_adapter = NULL;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter,
+				  HALMAC_API_CFG_RX_AGGREGATION);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_reg_read_8_pcie_88xx() - read 1byte register
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_offset : register offset
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+u8 halmac_reg_read_8_pcie_88xx(struct halmac_adapter *halmac_adapter,
+			       u32 halmac_offset)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	return PLATFORM_REG_READ_8(driver_adapter, halmac_offset);
+}
+
+/**
+ * halmac_reg_write_8_pcie_88xx() - write 1byte register
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_offset : register offset
+ * @halmac_data : register value
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_reg_write_8_pcie_88xx(struct halmac_adapter *halmac_adapter,
+			     u32 halmac_offset, u8 halmac_data)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	PLATFORM_REG_WRITE_8(driver_adapter, halmac_offset, halmac_data);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_reg_read_16_pcie_88xx() - read 2byte register
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_offset : register offset
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+u16 halmac_reg_read_16_pcie_88xx(struct halmac_adapter *halmac_adapter,
+				 u32 halmac_offset)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	return PLATFORM_REG_READ_16(driver_adapter, halmac_offset);
+}
+
+/**
+ * halmac_reg_write_16_pcie_88xx() - write 2byte register
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_offset : register offset
+ * @halmac_data : register value
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_reg_write_16_pcie_88xx(struct halmac_adapter *halmac_adapter,
+			      u32 halmac_offset, u16 halmac_data)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	PLATFORM_REG_WRITE_16(driver_adapter, halmac_offset, halmac_data);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_reg_read_32_pcie_88xx() - read 4byte register
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_offset : register offset
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+u32 halmac_reg_read_32_pcie_88xx(struct halmac_adapter *halmac_adapter,
+				 u32 halmac_offset)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	return PLATFORM_REG_READ_32(driver_adapter, halmac_offset);
+}
+
+/**
+ * halmac_reg_write_32_pcie_88xx() - write 4byte register
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_offset : register offset
+ * @halmac_data : register value
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_reg_write_32_pcie_88xx(struct halmac_adapter *halmac_adapter,
+			      u32 halmac_offset, u32 halmac_data)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	PLATFORM_REG_WRITE_32(driver_adapter, halmac_offset, halmac_data);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_cfg_tx_agg_align_pcie_88xx() -config sdio bus tx agg alignment
+ * @halmac_adapter : the adapter of halmac
+ * @enable : function enable(1)/disable(0)
+ * @align_size : sdio bus tx agg alignment size (2^n, n = 3~11)
+ * Author : Soar Tu
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status halmac_cfg_tx_agg_align_pcie_not_support_88xx(
+	struct halmac_adapter *halmac_adapter, u8 enable, u16 align_size)
+{
+	struct halmac_api *halmac_api;
+	void *driver_adapter = NULL;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_CFG_TX_AGG_ALIGN);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(
+		driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+		"%s ==========>\n", __func__);
+
+	HALMAC_RT_TRACE(
+		driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+		"%s not support\n", __func__);
+	HALMAC_RT_TRACE(
+		driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+		"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
diff --git a/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_api_88xx_pcie.h b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_api_88xx_pcie.h
new file mode 100644
index 000000000000..34969fc5c03e
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_api_88xx_pcie.h
@@ -0,0 +1,71 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HALMAC_API_88XX_PCIE_H_
+#define _HALMAC_API_88XX_PCIE_H_
+
+#include "../halmac_2_platform.h"
+#include "../halmac_type.h"
+
+#define LINK_CTRL2_REG_OFFSET 0xA0
+#define GEN2_CTRL_OFFSET 0x80C
+#define LINK_STATUS_REG_OFFSET 0x82
+#define GEN1_SPEED 0x01
+#define GEN2_SPEED 0x02
+
+enum halmac_ret_status
+halmac_init_pcie_cfg_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_deinit_pcie_cfg_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_cfg_rx_aggregation_88xx_pcie(struct halmac_adapter *halmac_adapter,
+				    struct halmac_rxagg_cfg *phalmac_rxagg_cfg);
+
+u8 halmac_reg_read_8_pcie_88xx(struct halmac_adapter *halmac_adapter,
+			       u32 halmac_offset);
+
+enum halmac_ret_status
+halmac_reg_write_8_pcie_88xx(struct halmac_adapter *halmac_adapter,
+			     u32 halmac_offset, u8 halmac_data);
+
+u16 halmac_reg_read_16_pcie_88xx(struct halmac_adapter *halmac_adapter,
+				 u32 halmac_offset);
+
+enum halmac_ret_status
+halmac_reg_write_16_pcie_88xx(struct halmac_adapter *halmac_adapter,
+			      u32 halmac_offset, u16 halmac_data);
+
+u32 halmac_reg_read_32_pcie_88xx(struct halmac_adapter *halmac_adapter,
+				 u32 halmac_offset);
+
+enum halmac_ret_status
+halmac_reg_write_32_pcie_88xx(struct halmac_adapter *halmac_adapter,
+			      u32 halmac_offset, u32 halmac_data);
+
+enum halmac_ret_status halmac_cfg_tx_agg_align_pcie_not_support_88xx(
+	struct halmac_adapter *halmac_adapter, u8 enable, u16 align_size);
+
+#endif /* _HALMAC_API_88XX_PCIE_H_ */
diff --git a/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_api_88xx_sdio.c b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_api_88xx_sdio.c
new file mode 100644
index 000000000000..69b26a5a3cf3
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_api_88xx_sdio.c
@@ -0,0 +1,974 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#include "halmac_88xx_cfg.h"
+
+/**
+ * halmac_init_sdio_cfg_88xx() - init SDIO
+ * @halmac_adapter : the adapter of halmac
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_init_sdio_cfg_88xx(struct halmac_adapter *halmac_adapter)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_INIT_SDIO_CFG);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	HALMAC_REG_READ_32(halmac_adapter, REG_SDIO_FREE_TXPG);
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_SDIO_TX_CTRL, 0x00000000);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_deinit_sdio_cfg_88xx() - deinit SDIO
+ * @halmac_adapter : the adapter of halmac
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_deinit_sdio_cfg_88xx(struct halmac_adapter *halmac_adapter)
+{
+	void *driver_adapter = NULL;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_DEINIT_SDIO_CFG);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_cfg_rx_aggregation_88xx_sdio() - config rx aggregation
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_rx_agg_mode
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_cfg_rx_aggregation_88xx_sdio(struct halmac_adapter *halmac_adapter,
+				    struct halmac_rxagg_cfg *phalmac_rxagg_cfg)
+{
+	u8 value8;
+	u8 size = 0, timeout = 0, agg_enable = 0;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter,
+				  HALMAC_API_CFG_RX_AGGREGATION);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	agg_enable = HALMAC_REG_READ_8(halmac_adapter, REG_TXDMA_PQ_MAP);
+
+	switch (phalmac_rxagg_cfg->mode) {
+	case HALMAC_RX_AGG_MODE_NONE:
+		agg_enable &= ~(BIT_RXDMA_AGG_EN);
+		break;
+	case HALMAC_RX_AGG_MODE_DMA:
+	case HALMAC_RX_AGG_MODE_USB:
+		agg_enable |= BIT_RXDMA_AGG_EN;
+		break;
+	default:
+		pr_err("halmac_cfg_rx_aggregation_88xx_usb switch case not support\n");
+		agg_enable &= ~BIT_RXDMA_AGG_EN;
+		break;
+	}
+
+	if (!phalmac_rxagg_cfg->threshold.drv_define) {
+		size = 0xFF;
+		timeout = 0x01;
+	} else {
+		size = phalmac_rxagg_cfg->threshold.size;
+		timeout = phalmac_rxagg_cfg->threshold.timeout;
+	}
+
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_TXDMA_PQ_MAP, agg_enable);
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_RXDMA_AGG_PG_TH,
+			    (u16)(size | (timeout << BIT_SHIFT_DMA_AGG_TO)));
+
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_RXDMA_MODE);
+	if ((agg_enable & BIT_RXDMA_AGG_EN) != 0)
+		HALMAC_REG_WRITE_8(halmac_adapter, REG_RXDMA_MODE,
+				   value8 | BIT_DMA_MODE);
+	else
+		HALMAC_REG_WRITE_8(halmac_adapter, REG_RXDMA_MODE,
+				   value8 & ~(BIT_DMA_MODE));
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_reg_read_8_sdio_88xx() - read 1byte register
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_offset : register offset
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+u8 halmac_reg_read_8_sdio_88xx(struct halmac_adapter *halmac_adapter,
+			       u32 halmac_offset)
+{
+	u8 value8;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	if ((halmac_offset & 0xFFFF0000) == 0)
+		halmac_offset |= WLAN_IOREG_OFFSET;
+
+	status = halmac_convert_to_sdio_bus_offset_88xx(halmac_adapter,
+							&halmac_offset);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("%s error = %x\n", __func__, status);
+		return status;
+	}
+
+	value8 = PLATFORM_SDIO_CMD52_READ(driver_adapter, halmac_offset);
+
+	return value8;
+}
+
+/**
+ * halmac_reg_write_8_sdio_88xx() - write 1byte register
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_offset : register offset
+ * @halmac_data : register value
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_reg_write_8_sdio_88xx(struct halmac_adapter *halmac_adapter,
+			     u32 halmac_offset, u8 halmac_data)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	if ((halmac_offset & 0xFFFF0000) == 0)
+		halmac_offset |= WLAN_IOREG_OFFSET;
+
+	status = halmac_convert_to_sdio_bus_offset_88xx(halmac_adapter,
+							&halmac_offset);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("%s error = %x\n", __func__, status);
+		return status;
+	}
+
+	PLATFORM_SDIO_CMD52_WRITE(driver_adapter, halmac_offset, halmac_data);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_reg_read_16_sdio_88xx() - read 2byte register
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_offset : register offset
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+u16 halmac_reg_read_16_sdio_88xx(struct halmac_adapter *halmac_adapter,
+				 u32 halmac_offset)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	union {
+		u16 word;
+		u8 byte[2];
+		__le16 le_word;
+	} value16 = {0x0000};
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	if ((halmac_offset & 0xFFFF0000) == 0)
+		halmac_offset |= WLAN_IOREG_OFFSET;
+
+	status = halmac_convert_to_sdio_bus_offset_88xx(halmac_adapter,
+							&halmac_offset);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("%s error = %x\n", __func__, status);
+		return status;
+	}
+
+	if (halmac_adapter->halmac_state.mac_power == HALMAC_MAC_POWER_OFF ||
+	    (halmac_offset & (2 - 1)) != 0 ||
+	    halmac_adapter->sdio_cmd53_4byte ==
+		    HALMAC_SDIO_CMD53_4BYTE_MODE_RW ||
+	    halmac_adapter->sdio_cmd53_4byte ==
+		    HALMAC_SDIO_CMD53_4BYTE_MODE_R) {
+		value16.byte[0] =
+			PLATFORM_SDIO_CMD52_READ(driver_adapter, halmac_offset);
+		value16.byte[1] = PLATFORM_SDIO_CMD52_READ(driver_adapter,
+							   halmac_offset + 1);
+		value16.word = le16_to_cpu(value16.le_word);
+	} else {
+#if (PLATFORM_SD_CLK > HALMAC_SD_CLK_THRESHOLD_88XX)
+		if ((halmac_offset & 0xffffef00) == 0x00000000) {
+			value16.byte[0] = PLATFORM_SDIO_CMD52_READ(
+				driver_adapter, halmac_offset);
+			value16.byte[1] = PLATFORM_SDIO_CMD52_READ(
+				driver_adapter, halmac_offset + 1);
+			value16.word = le16_to_cpu(value16.word);
+		} else {
+			value16.word = PLATFORM_SDIO_CMD53_READ_16(
+				driver_adapter, halmac_offset);
+		}
+#else
+		value16.word = PLATFORM_SDIO_CMD53_READ_16(driver_adapter,
+							   halmac_offset);
+#endif
+	}
+
+	return value16.word;
+}
+
+/**
+ * halmac_reg_write_16_sdio_88xx() - write 2byte register
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_offset : register offset
+ * @halmac_data : register value
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_reg_write_16_sdio_88xx(struct halmac_adapter *halmac_adapter,
+			      u32 halmac_offset, u16 halmac_data)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	if ((halmac_offset & 0xFFFF0000) == 0)
+		halmac_offset |= WLAN_IOREG_OFFSET;
+
+	status = halmac_convert_to_sdio_bus_offset_88xx(halmac_adapter,
+							&halmac_offset);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("%s error = %x\n", __func__, status);
+		return status;
+	}
+
+	if (halmac_adapter->halmac_state.mac_power == HALMAC_MAC_POWER_OFF ||
+	    (halmac_offset & (2 - 1)) != 0 ||
+	    halmac_adapter->sdio_cmd53_4byte ==
+		    HALMAC_SDIO_CMD53_4BYTE_MODE_RW ||
+	    halmac_adapter->sdio_cmd53_4byte ==
+		    HALMAC_SDIO_CMD53_4BYTE_MODE_W) {
+		PLATFORM_SDIO_CMD52_WRITE(driver_adapter, halmac_offset,
+					  (u8)(halmac_data & 0xFF));
+		PLATFORM_SDIO_CMD52_WRITE(driver_adapter, halmac_offset + 1,
+					  (u8)((halmac_data & 0xFF00) >> 8));
+	} else {
+		PLATFORM_SDIO_CMD53_WRITE_16(driver_adapter, halmac_offset,
+					     halmac_data);
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_reg_read_32_sdio_88xx() - read 4byte register
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_offset : register offset
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+u32 halmac_reg_read_32_sdio_88xx(struct halmac_adapter *halmac_adapter,
+				 u32 halmac_offset)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+	u32 halmac_offset_old = 0;
+
+	union {
+		u32 dword;
+		u8 byte[4];
+		__le32 le_dword;
+	} value32 = {0x00000000};
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	halmac_offset_old = halmac_offset;
+
+	if ((halmac_offset & 0xFFFF0000) == 0)
+		halmac_offset |= WLAN_IOREG_OFFSET;
+
+	status = halmac_convert_to_sdio_bus_offset_88xx(halmac_adapter,
+							&halmac_offset);
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("%s error = %x\n", __func__, status);
+		return status;
+	}
+
+	if (halmac_adapter->halmac_state.mac_power == HALMAC_MAC_POWER_OFF ||
+	    (halmac_offset & (4 - 1)) != 0) {
+		value32.byte[0] =
+			PLATFORM_SDIO_CMD52_READ(driver_adapter, halmac_offset);
+		value32.byte[1] = PLATFORM_SDIO_CMD52_READ(driver_adapter,
+							   halmac_offset + 1);
+		value32.byte[2] = PLATFORM_SDIO_CMD52_READ(driver_adapter,
+							   halmac_offset + 2);
+		value32.byte[3] = PLATFORM_SDIO_CMD52_READ(driver_adapter,
+							   halmac_offset + 3);
+		value32.dword = le32_to_cpu(value32.le_dword);
+	} else {
+#if (PLATFORM_SD_CLK > HALMAC_SD_CLK_THRESHOLD_88XX)
+		if ((halmac_offset_old & 0xffffef00) == 0x00000000) {
+			value32.byte[0] = PLATFORM_SDIO_CMD52_READ(
+				driver_adapter, halmac_offset);
+			value32.byte[1] = PLATFORM_SDIO_CMD52_READ(
+				driver_adapter, halmac_offset + 1);
+			value32.byte[2] = PLATFORM_SDIO_CMD52_READ(
+				driver_adapter, halmac_offset + 2);
+			value32.byte[3] = PLATFORM_SDIO_CMD52_READ(
+				driver_adapter, halmac_offset + 3);
+			value32.dword = le32_to_cpu(value32.dword);
+		} else {
+			value32.dword = PLATFORM_SDIO_CMD53_READ_32(
+				driver_adapter, halmac_offset);
+		}
+#else
+		value32.dword = PLATFORM_SDIO_CMD53_READ_32(driver_adapter,
+							    halmac_offset);
+#endif
+	}
+
+	return value32.dword;
+}
+
+/**
+ * halmac_reg_write_32_sdio_88xx() - write 4byte register
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_offset : register offset
+ * @halmac_data : register value
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_reg_write_32_sdio_88xx(struct halmac_adapter *halmac_adapter,
+			      u32 halmac_offset, u32 halmac_data)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	if ((halmac_offset & 0xFFFF0000) == 0)
+		halmac_offset |= WLAN_IOREG_OFFSET;
+
+	status = halmac_convert_to_sdio_bus_offset_88xx(halmac_adapter,
+							&halmac_offset);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("%s error = %x\n", __func__, status);
+		return status;
+	}
+
+	if (halmac_adapter->halmac_state.mac_power == HALMAC_MAC_POWER_OFF ||
+	    (halmac_offset & (4 - 1)) != 0) {
+		PLATFORM_SDIO_CMD52_WRITE(driver_adapter, halmac_offset,
+					  (u8)(halmac_data & 0xFF));
+		PLATFORM_SDIO_CMD52_WRITE(driver_adapter, halmac_offset + 1,
+					  (u8)((halmac_data & 0xFF00) >> 8));
+		PLATFORM_SDIO_CMD52_WRITE(driver_adapter, halmac_offset + 2,
+					  (u8)((halmac_data & 0xFF0000) >> 16));
+		PLATFORM_SDIO_CMD52_WRITE(
+			driver_adapter, halmac_offset + 3,
+			(u8)((halmac_data & 0xFF000000) >> 24));
+	} else {
+		PLATFORM_SDIO_CMD53_WRITE_32(driver_adapter, halmac_offset,
+					     halmac_data);
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_reg_read_nbyte_sdio_88xx() - read n byte register
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_offset : register offset
+ * @halmac_size : register value size
+ * @halmac_data : register value
+ * Author : Soar
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+u8 halmac_reg_read_nbyte_sdio_88xx(struct halmac_adapter *halmac_adapter,
+				   u32 halmac_offset, u32 halmac_size,
+				   u8 *halmac_data)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	if ((halmac_offset & 0xFFFF0000) == 0) {
+		pr_err("halmac_offset error = 0x%x\n", halmac_offset);
+		return HALMAC_RET_FAIL;
+	}
+
+	status = halmac_convert_to_sdio_bus_offset_88xx(halmac_adapter,
+							&halmac_offset);
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("%s error = %x\n", __func__, status);
+		return status;
+	}
+
+	if (halmac_adapter->halmac_state.mac_power == HALMAC_MAC_POWER_OFF) {
+		pr_err("halmac_state error = 0x%x\n",
+		       halmac_adapter->halmac_state.mac_power);
+		return HALMAC_RET_FAIL;
+	}
+
+	PLATFORM_SDIO_CMD53_READ_N(driver_adapter, halmac_offset, halmac_size,
+				   halmac_data);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_get_sdio_tx_addr_sdio_88xx() - get CMD53 addr for the TX packet
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_buf : tx packet, include txdesc
+ * @halmac_size : tx packet size
+ * @pcmd53_addr : cmd53 addr value
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_get_sdio_tx_addr_88xx(struct halmac_adapter *halmac_adapter,
+			     u8 *halmac_buf, u32 halmac_size, u32 *pcmd53_addr)
+{
+	u32 four_byte_len;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	enum halmac_queue_select queue_sel;
+	enum halmac_dma_mapping dma_mapping;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_GET_SDIO_TX_ADDR);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	if (!halmac_buf) {
+		pr_err("halmac_buf is NULL!!\n");
+		return HALMAC_RET_DATA_BUF_NULL;
+	}
+
+	if (halmac_size == 0) {
+		pr_err("halmac_size is 0!!\n");
+		return HALMAC_RET_DATA_SIZE_INCORRECT;
+	}
+
+	queue_sel = (enum halmac_queue_select)GET_TX_DESC_QSEL(halmac_buf);
+
+	switch (queue_sel) {
+	case HALMAC_QUEUE_SELECT_VO:
+	case HALMAC_QUEUE_SELECT_VO_V2:
+		dma_mapping =
+			halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_VO];
+		break;
+	case HALMAC_QUEUE_SELECT_VI:
+	case HALMAC_QUEUE_SELECT_VI_V2:
+		dma_mapping =
+			halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_VI];
+		break;
+	case HALMAC_QUEUE_SELECT_BE:
+	case HALMAC_QUEUE_SELECT_BE_V2:
+		dma_mapping =
+			halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_BE];
+		break;
+	case HALMAC_QUEUE_SELECT_BK:
+	case HALMAC_QUEUE_SELECT_BK_V2:
+		dma_mapping =
+			halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_BK];
+		break;
+	case HALMAC_QUEUE_SELECT_MGNT:
+		dma_mapping =
+			halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_MG];
+		break;
+	case HALMAC_QUEUE_SELECT_HIGH:
+	case HALMAC_QUEUE_SELECT_BCN:
+	case HALMAC_QUEUE_SELECT_CMD:
+		dma_mapping =
+			halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_HI];
+		break;
+	default:
+		pr_err("Qsel is out of range\n");
+		return HALMAC_RET_QSEL_INCORRECT;
+	}
+
+	four_byte_len = (halmac_size >> 2) + ((halmac_size & (4 - 1)) ? 1 : 0);
+
+	switch (dma_mapping) {
+	case HALMAC_DMA_MAPPING_HIGH:
+		*pcmd53_addr = HALMAC_SDIO_CMD_ADDR_TXFF_HIGH;
+		break;
+	case HALMAC_DMA_MAPPING_NORMAL:
+		*pcmd53_addr = HALMAC_SDIO_CMD_ADDR_TXFF_NORMAL;
+		break;
+	case HALMAC_DMA_MAPPING_LOW:
+		*pcmd53_addr = HALMAC_SDIO_CMD_ADDR_TXFF_LOW;
+		break;
+	case HALMAC_DMA_MAPPING_EXTRA:
+		*pcmd53_addr = HALMAC_SDIO_CMD_ADDR_TXFF_EXTRA;
+		break;
+	default:
+		pr_err("DmaMapping is out of range\n");
+		return HALMAC_RET_DMA_MAP_INCORRECT;
+	}
+
+	*pcmd53_addr = (*pcmd53_addr << 13) |
+		       (four_byte_len & HALMAC_SDIO_4BYTE_LEN_MASK);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_cfg_tx_agg_align_sdio_88xx() -config sdio bus tx agg alignment
+ * @halmac_adapter : the adapter of halmac
+ * @enable : function enable(1)/disable(0)
+ * @align_size : sdio bus tx agg alignment size (2^n, n = 3~11)
+ * Author : Soar Tu
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_cfg_tx_agg_align_sdio_88xx(struct halmac_adapter *halmac_adapter,
+				  u8 enable, u16 align_size)
+{
+	struct halmac_api *halmac_api;
+	void *driver_adapter = NULL;
+	u8 i, align_size_ok = 0;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_CFG_TX_AGG_ALIGN);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	if ((align_size & 0xF000) != 0) {
+		pr_err("Align size is out of range\n");
+		return HALMAC_RET_FAIL;
+	}
+
+	for (i = 3; i <= 11; i++) {
+		if (align_size == 1 << i) {
+			align_size_ok = 1;
+			break;
+		}
+	}
+	if (align_size_ok == 0) {
+		pr_err("Align size is not 2^3 ~ 2^11\n");
+		return HALMAC_RET_FAIL;
+	}
+
+	/*Keep sdio tx agg alignment size for driver query*/
+	halmac_adapter->hw_config_info.tx_align_size = align_size;
+
+	if (enable)
+		HALMAC_REG_WRITE_16(halmac_adapter, REG_RQPN_CTRL_2,
+				    0x8000 | align_size);
+	else
+		HALMAC_REG_WRITE_16(halmac_adapter, REG_RQPN_CTRL_2,
+				    align_size);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status halmac_cfg_tx_agg_align_sdio_not_support_88xx(
+	struct halmac_adapter *halmac_adapter, u8 enable, u16 align_size)
+{
+	struct halmac_api *halmac_api;
+	void *driver_adapter = NULL;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_CFG_TX_AGG_ALIGN);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(
+		driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+		"%s ==========>\n", __func__);
+
+	HALMAC_RT_TRACE(
+		driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+		"%s not support\n", __func__);
+	HALMAC_RT_TRACE(
+		driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+		"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_tx_allowed_sdio_88xx() - check tx status
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_buf : tx packet, include txdesc
+ * @halmac_size : tx packet size, include txdesc
+ * Author : Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_tx_allowed_sdio_88xx(struct halmac_adapter *halmac_adapter,
+			    u8 *halmac_buf, u32 halmac_size)
+{
+	u8 *curr_packet;
+	u16 *curr_free_space;
+	u32 i, counter;
+	u32 tx_agg_num, packet_size = 0;
+	u32 tx_required_page_num, total_required_page_num = 0;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+	void *driver_adapter = NULL;
+	enum halmac_dma_mapping dma_mapping;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_TX_ALLOWED_SDIO);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	tx_agg_num = GET_TX_DESC_DMA_TXAGG_NUM(halmac_buf);
+	curr_packet = halmac_buf;
+
+	tx_agg_num = tx_agg_num == 0 ? 1 : tx_agg_num;
+
+	switch ((enum halmac_queue_select)GET_TX_DESC_QSEL(curr_packet)) {
+	case HALMAC_QUEUE_SELECT_VO:
+	case HALMAC_QUEUE_SELECT_VO_V2:
+		dma_mapping =
+			halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_VO];
+		break;
+	case HALMAC_QUEUE_SELECT_VI:
+	case HALMAC_QUEUE_SELECT_VI_V2:
+		dma_mapping =
+			halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_VI];
+		break;
+	case HALMAC_QUEUE_SELECT_BE:
+	case HALMAC_QUEUE_SELECT_BE_V2:
+		dma_mapping =
+			halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_BE];
+		break;
+	case HALMAC_QUEUE_SELECT_BK:
+	case HALMAC_QUEUE_SELECT_BK_V2:
+		dma_mapping =
+			halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_BK];
+		break;
+	case HALMAC_QUEUE_SELECT_MGNT:
+		dma_mapping =
+			halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_MG];
+		break;
+	case HALMAC_QUEUE_SELECT_HIGH:
+		dma_mapping =
+			halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_HI];
+		break;
+	case HALMAC_QUEUE_SELECT_BCN:
+	case HALMAC_QUEUE_SELECT_CMD:
+		return HALMAC_RET_SUCCESS;
+	default:
+		pr_err("Qsel is out of range\n");
+		return HALMAC_RET_QSEL_INCORRECT;
+	}
+
+	switch (dma_mapping) {
+	case HALMAC_DMA_MAPPING_HIGH:
+		curr_free_space =
+			&halmac_adapter->sdio_free_space.high_queue_number;
+		break;
+	case HALMAC_DMA_MAPPING_NORMAL:
+		curr_free_space =
+			&halmac_adapter->sdio_free_space.normal_queue_number;
+		break;
+	case HALMAC_DMA_MAPPING_LOW:
+		curr_free_space =
+			&halmac_adapter->sdio_free_space.low_queue_number;
+		break;
+	case HALMAC_DMA_MAPPING_EXTRA:
+		curr_free_space =
+			&halmac_adapter->sdio_free_space.extra_queue_number;
+		break;
+	default:
+		pr_err("DmaMapping is out of range\n");
+		return HALMAC_RET_DMA_MAP_INCORRECT;
+	}
+
+	for (i = 0; i < tx_agg_num; i++) {
+		packet_size = GET_TX_DESC_TXPKTSIZE(curr_packet) +
+			      GET_TX_DESC_OFFSET(curr_packet) +
+			      (GET_TX_DESC_PKT_OFFSET(curr_packet) << 3);
+		tx_required_page_num =
+			(packet_size >>
+			 halmac_adapter->hw_config_info.page_size_2_power) +
+			((packet_size &
+			  (halmac_adapter->hw_config_info.page_size - 1)) ?
+				 1 :
+				 0);
+		total_required_page_num += tx_required_page_num;
+
+		packet_size = HALMAC_ALIGN(packet_size, 8);
+
+		curr_packet += packet_size;
+	}
+
+	counter = 10;
+	do {
+		if ((u32)(*curr_free_space +
+			  halmac_adapter->sdio_free_space.public_queue_number) >
+		    total_required_page_num) {
+			if (*curr_free_space >= total_required_page_num) {
+				*curr_free_space -=
+					(u16)total_required_page_num;
+			} else {
+				halmac_adapter->sdio_free_space
+					.public_queue_number -=
+					(u16)(total_required_page_num -
+					      *curr_free_space);
+				*curr_free_space = 0;
+			}
+
+			status = halmac_check_oqt_88xx(halmac_adapter,
+						       tx_agg_num, halmac_buf);
+
+			if (status != HALMAC_RET_SUCCESS)
+				return status;
+
+			break;
+		}
+
+		halmac_update_sdio_free_page_88xx(halmac_adapter);
+
+		counter--;
+		if (counter == 0)
+			return HALMAC_RET_FREE_SPACE_NOT_ENOUGH;
+	} while (1);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_reg_read_indirect_32_sdio_88xx() - read MAC reg by SDIO reg
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_offset : register offset
+ * Author : Soar
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+u32 halmac_reg_read_indirect_32_sdio_88xx(struct halmac_adapter *halmac_adapter,
+					  u32 halmac_offset)
+{
+	u8 rtemp;
+	u32 counter = 1000;
+	void *driver_adapter = NULL;
+
+	union {
+		u32 dword;
+		u8 byte[4];
+	} value32 = {0x00000000};
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	PLATFORM_SDIO_CMD53_WRITE_32(
+		driver_adapter,
+		(HALMAC_SDIO_CMD_ADDR_SDIO_REG << 13) |
+			(REG_SDIO_INDIRECT_REG_CFG & HALMAC_SDIO_LOCAL_MSK),
+		halmac_offset | BIT(19) | BIT(17));
+
+	do {
+		rtemp = PLATFORM_SDIO_CMD52_READ(
+			driver_adapter,
+			(HALMAC_SDIO_CMD_ADDR_SDIO_REG << 13) |
+				((REG_SDIO_INDIRECT_REG_CFG + 2) &
+				 HALMAC_SDIO_LOCAL_MSK));
+		counter--;
+	} while ((rtemp & BIT(4)) != 0 && counter > 0);
+
+	value32.dword = PLATFORM_SDIO_CMD53_READ_32(
+		driver_adapter,
+		(HALMAC_SDIO_CMD_ADDR_SDIO_REG << 13) |
+			(REG_SDIO_INDIRECT_REG_DATA & HALMAC_SDIO_LOCAL_MSK));
+
+	return value32.dword;
+}
diff --git a/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_api_88xx_sdio.h b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_api_88xx_sdio.h
new file mode 100644
index 000000000000..ee441eee24d6
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_api_88xx_sdio.h
@@ -0,0 +1,84 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HALMAC_API_88XX_SDIO_H_
+#define _HALMAC_API_88XX_SDIO_H_
+
+#include "../halmac_2_platform.h"
+#include "../halmac_type.h"
+
+enum halmac_ret_status
+halmac_init_sdio_cfg_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_deinit_sdio_cfg_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_cfg_rx_aggregation_88xx_sdio(struct halmac_adapter *halmac_adapter,
+				    struct halmac_rxagg_cfg *phalmac_rxagg_cfg);
+
+u8 halmac_reg_read_8_sdio_88xx(struct halmac_adapter *halmac_adapter,
+			       u32 halmac_offset);
+
+enum halmac_ret_status
+halmac_reg_write_8_sdio_88xx(struct halmac_adapter *halmac_adapter,
+			     u32 halmac_offset, u8 halmac_data);
+
+u16 halmac_reg_read_16_sdio_88xx(struct halmac_adapter *halmac_adapter,
+				 u32 halmac_offset);
+
+enum halmac_ret_status
+halmac_reg_write_16_sdio_88xx(struct halmac_adapter *halmac_adapter,
+			      u32 halmac_offset, u16 halmac_data);
+
+u32 halmac_reg_read_32_sdio_88xx(struct halmac_adapter *halmac_adapter,
+				 u32 halmac_offset);
+
+enum halmac_ret_status
+halmac_reg_write_32_sdio_88xx(struct halmac_adapter *halmac_adapter,
+			      u32 halmac_offset, u32 halmac_data);
+
+enum halmac_ret_status
+halmac_get_sdio_tx_addr_88xx(struct halmac_adapter *halmac_adapter,
+			     u8 *halmac_buf, u32 halmac_size, u32 *pcmd53_addr);
+
+enum halmac_ret_status
+halmac_cfg_tx_agg_align_sdio_88xx(struct halmac_adapter *halmac_adapter,
+				  u8 enable, u16 align_size);
+
+enum halmac_ret_status halmac_cfg_tx_agg_align_sdio_not_support_88xx(
+	struct halmac_adapter *halmac_adapter, u8 enable, u16 align_size);
+
+enum halmac_ret_status
+halmac_tx_allowed_sdio_88xx(struct halmac_adapter *halmac_adapter,
+			    u8 *halmac_buf, u32 halmac_size);
+
+u32 halmac_reg_read_indirect_32_sdio_88xx(struct halmac_adapter *halmac_adapter,
+					  u32 halmac_offset);
+
+u8 halmac_reg_read_nbyte_sdio_88xx(struct halmac_adapter *halmac_adapter,
+				   u32 halmac_offset, u32 halmac_size,
+				   u8 *halmac_data);
+
+#endif /* _HALMAC_API_88XX_SDIO_H_ */
diff --git a/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_api_88xx_usb.c b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_api_88xx_usb.c
new file mode 100644
index 000000000000..17d7c3cc62ec
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_api_88xx_usb.c
@@ -0,0 +1,554 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#include "halmac_88xx_cfg.h"
+
+/**
+ * halmac_init_usb_cfg_88xx() - init USB
+ * @halmac_adapter : the adapter of halmac
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_init_usb_cfg_88xx(struct halmac_adapter *halmac_adapter)
+{
+	void *driver_adapter = NULL;
+	u8 value8 = 0;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_INIT_USB_CFG);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	value8 |= (BIT_DMA_MODE |
+		   (0x3 << BIT_SHIFT_BURST_CNT)); /* burst number = 4 */
+
+	if (PLATFORM_REG_READ_8(driver_adapter, REG_SYS_CFG2 + 3) ==
+	    0x20) { /* usb3.0 */
+		value8 |= (HALMAC_USB_BURST_SIZE_3_0 << BIT_SHIFT_BURST_SIZE);
+	} else {
+		if ((PLATFORM_REG_READ_8(driver_adapter, REG_USB_USBSTAT) &
+		     0x3) == 0x1) /* usb2.0 */
+			value8 |= HALMAC_USB_BURST_SIZE_2_0_HSPEED
+				  << BIT_SHIFT_BURST_SIZE;
+		else /* usb1.1 */
+			value8 |= HALMAC_USB_BURST_SIZE_2_0_FSPEED
+				  << BIT_SHIFT_BURST_SIZE;
+	}
+
+	PLATFORM_REG_WRITE_8(driver_adapter, REG_RXDMA_MODE, value8);
+	PLATFORM_REG_WRITE_16(
+		driver_adapter, REG_TXDMA_OFFSET_CHK,
+		PLATFORM_REG_READ_16(driver_adapter, REG_TXDMA_OFFSET_CHK) |
+			BIT_DROP_DATA_EN);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_deinit_usb_cfg_88xx() - deinit USB
+ * @halmac_adapter : the adapter of halmac
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_deinit_usb_cfg_88xx(struct halmac_adapter *halmac_adapter)
+{
+	void *driver_adapter = NULL;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_DEINIT_USB_CFG);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_cfg_rx_aggregation_88xx_usb() - config rx aggregation
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_rx_agg_mode
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_cfg_rx_aggregation_88xx_usb(struct halmac_adapter *halmac_adapter,
+				   struct halmac_rxagg_cfg *phalmac_rxagg_cfg)
+{
+	u8 dma_usb_agg;
+	u8 size = 0, timeout = 0, agg_enable = 0;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter,
+				  HALMAC_API_CFG_RX_AGGREGATION);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	dma_usb_agg =
+		HALMAC_REG_READ_8(halmac_adapter, REG_RXDMA_AGG_PG_TH + 3);
+	agg_enable = HALMAC_REG_READ_8(halmac_adapter, REG_TXDMA_PQ_MAP);
+
+	switch (phalmac_rxagg_cfg->mode) {
+	case HALMAC_RX_AGG_MODE_NONE:
+		agg_enable &= ~BIT_RXDMA_AGG_EN;
+		break;
+	case HALMAC_RX_AGG_MODE_DMA:
+		agg_enable |= BIT_RXDMA_AGG_EN;
+		dma_usb_agg |= BIT(7);
+		break;
+
+	case HALMAC_RX_AGG_MODE_USB:
+		agg_enable |= BIT_RXDMA_AGG_EN;
+		dma_usb_agg &= ~BIT(7);
+		break;
+	default:
+		pr_err("%s switch case not support\n", __func__);
+		agg_enable &= ~BIT_RXDMA_AGG_EN;
+		break;
+	}
+
+	if (!phalmac_rxagg_cfg->threshold.drv_define) {
+		if (PLATFORM_REG_READ_8(driver_adapter, REG_SYS_CFG2 + 3) ==
+		    0x20) {
+			/* usb3.0 */
+			size = 0x5;
+			timeout = 0xA;
+		} else {
+			/* usb2.0 */
+			size = 0x5;
+			timeout = 0x20;
+		}
+	} else {
+		size = phalmac_rxagg_cfg->threshold.size;
+		timeout = phalmac_rxagg_cfg->threshold.timeout;
+	}
+
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_TXDMA_PQ_MAP, agg_enable);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_RXDMA_AGG_PG_TH + 3,
+			   dma_usb_agg);
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_RXDMA_AGG_PG_TH,
+			    (u16)(size | (timeout << BIT_SHIFT_DMA_AGG_TO)));
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_reg_read_8_usb_88xx() - read 1byte register
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_offset : register offset
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+u8 halmac_reg_read_8_usb_88xx(struct halmac_adapter *halmac_adapter,
+			      u32 halmac_offset)
+{
+	u8 value8;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	value8 = PLATFORM_REG_READ_8(driver_adapter, halmac_offset);
+
+	return value8;
+}
+
+/**
+ * halmac_reg_write_8_usb_88xx() - write 1byte register
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_offset : register offset
+ * @halmac_data : register value
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_reg_write_8_usb_88xx(struct halmac_adapter *halmac_adapter,
+			    u32 halmac_offset, u8 halmac_data)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	PLATFORM_REG_WRITE_8(driver_adapter, halmac_offset, halmac_data);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_reg_read_16_usb_88xx() - read 2byte register
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_offset : register offset
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+u16 halmac_reg_read_16_usb_88xx(struct halmac_adapter *halmac_adapter,
+				u32 halmac_offset)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	union {
+		u16 word;
+		u8 byte[2];
+	} value16 = {0x0000};
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	value16.word = PLATFORM_REG_READ_16(driver_adapter, halmac_offset);
+
+	return value16.word;
+}
+
+/**
+ * halmac_reg_write_16_usb_88xx() - write 2byte register
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_offset : register offset
+ * @halmac_data : register value
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_reg_write_16_usb_88xx(struct halmac_adapter *halmac_adapter,
+			     u32 halmac_offset, u16 halmac_data)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	PLATFORM_REG_WRITE_16(driver_adapter, halmac_offset, halmac_data);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_reg_read_32_usb_88xx() - read 4byte register
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_offset : register offset
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+u32 halmac_reg_read_32_usb_88xx(struct halmac_adapter *halmac_adapter,
+				u32 halmac_offset)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	union {
+		u32 dword;
+		u8 byte[4];
+	} value32 = {0x00000000};
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	value32.dword = PLATFORM_REG_READ_32(driver_adapter, halmac_offset);
+
+	return value32.dword;
+}
+
+/**
+ * halmac_reg_write_32_usb_88xx() - write 4byte register
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_offset : register offset
+ * @halmac_data : register value
+ * Author : KaiYuan Chang/Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_reg_write_32_usb_88xx(struct halmac_adapter *halmac_adapter,
+			     u32 halmac_offset, u32 halmac_data)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	PLATFORM_REG_WRITE_32(driver_adapter, halmac_offset, halmac_data);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_set_bulkout_num_usb_88xx() - inform bulk-out num
+ * @halmac_adapter : the adapter of halmac
+ * @bulkout_num : usb bulk-out number
+ * Author : KaiYuan Chang
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_set_bulkout_num_88xx(struct halmac_adapter *halmac_adapter,
+			    u8 bulkout_num)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_SET_BULKOUT_NUM);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	halmac_adapter->halmac_bulkout_num = bulkout_num;
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_get_usb_bulkout_id_usb_88xx() - get bulk out id for the TX packet
+ * @halmac_adapter : the adapter of halmac
+ * @halmac_buf : tx packet, include txdesc
+ * @halmac_size : tx packet size
+ * @bulkout_id : usb bulk-out id
+ * Author : KaiYuan Chang
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_get_usb_bulkout_id_88xx(struct halmac_adapter *halmac_adapter,
+			       u8 *halmac_buf, u32 halmac_size, u8 *bulkout_id)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	enum halmac_queue_select queue_sel;
+	enum halmac_dma_mapping dma_mapping;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter,
+				  HALMAC_API_GET_USB_BULKOUT_ID);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	if (!halmac_buf) {
+		pr_err("halmac_buf is NULL!!\n");
+		return HALMAC_RET_DATA_BUF_NULL;
+	}
+
+	if (halmac_size == 0) {
+		pr_err("halmac_size is 0!!\n");
+		return HALMAC_RET_DATA_SIZE_INCORRECT;
+	}
+
+	queue_sel = (enum halmac_queue_select)GET_TX_DESC_QSEL(halmac_buf);
+
+	switch (queue_sel) {
+	case HALMAC_QUEUE_SELECT_VO:
+	case HALMAC_QUEUE_SELECT_VO_V2:
+		dma_mapping =
+			halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_VO];
+		break;
+	case HALMAC_QUEUE_SELECT_VI:
+	case HALMAC_QUEUE_SELECT_VI_V2:
+		dma_mapping =
+			halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_VI];
+		break;
+	case HALMAC_QUEUE_SELECT_BE:
+	case HALMAC_QUEUE_SELECT_BE_V2:
+		dma_mapping =
+			halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_BE];
+		break;
+	case HALMAC_QUEUE_SELECT_BK:
+	case HALMAC_QUEUE_SELECT_BK_V2:
+		dma_mapping =
+			halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_BK];
+		break;
+	case HALMAC_QUEUE_SELECT_MGNT:
+		dma_mapping =
+			halmac_adapter->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_MG];
+		break;
+	case HALMAC_QUEUE_SELECT_HIGH:
+	case HALMAC_QUEUE_SELECT_BCN:
+	case HALMAC_QUEUE_SELECT_CMD:
+		dma_mapping = HALMAC_DMA_MAPPING_HIGH;
+		break;
+	default:
+		pr_err("Qsel is out of range\n");
+		return HALMAC_RET_QSEL_INCORRECT;
+	}
+
+	switch (dma_mapping) {
+	case HALMAC_DMA_MAPPING_HIGH:
+		*bulkout_id = 0;
+		break;
+	case HALMAC_DMA_MAPPING_NORMAL:
+		*bulkout_id = 1;
+		break;
+	case HALMAC_DMA_MAPPING_LOW:
+		*bulkout_id = 2;
+		break;
+	case HALMAC_DMA_MAPPING_EXTRA:
+		*bulkout_id = 3;
+		break;
+	default:
+		pr_err("DmaMapping is out of range\n");
+		return HALMAC_RET_DMA_MAP_INCORRECT;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_cfg_tx_agg_align_usb_88xx() -config sdio bus tx agg alignment
+ * @halmac_adapter : the adapter of halmac
+ * @enable : function enable(1)/disable(0)
+ * @align_size : sdio bus tx agg alignment size (2^n, n = 3~11)
+ * Author : Soar Tu
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status halmac_cfg_tx_agg_align_usb_not_support_88xx(
+	struct halmac_adapter *halmac_adapter, u8 enable, u16 align_size)
+{
+	struct halmac_api *halmac_api;
+	void *driver_adapter = NULL;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	if (halmac_api_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_API_INVALID;
+
+	halmac_api_record_id_88xx(halmac_adapter, HALMAC_API_CFG_TX_AGG_ALIGN);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(
+		driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+		"%s ==========>\n", __func__);
+
+	HALMAC_RT_TRACE(
+		driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+		"%s not support\n", __func__);
+	HALMAC_RT_TRACE(
+		driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+		"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
diff --git a/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_api_88xx_usb.h b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_api_88xx_usb.h
new file mode 100644
index 000000000000..a3d2a6abd91b
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_api_88xx_usb.h
@@ -0,0 +1,73 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HALMAC_API_88XX_USB_H_
+#define _HALMAC_API_88XX_USB_H_
+
+#include "../halmac_2_platform.h"
+#include "../halmac_type.h"
+
+enum halmac_ret_status
+halmac_init_usb_cfg_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_deinit_usb_cfg_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_cfg_rx_aggregation_88xx_usb(struct halmac_adapter *halmac_adapter,
+				   struct halmac_rxagg_cfg *phalmac_rxagg_cfg);
+
+u8 halmac_reg_read_8_usb_88xx(struct halmac_adapter *halmac_adapter,
+			      u32 halmac_offset);
+
+enum halmac_ret_status
+halmac_reg_write_8_usb_88xx(struct halmac_adapter *halmac_adapter,
+			    u32 halmac_offset, u8 halmac_data);
+
+u16 halmac_reg_read_16_usb_88xx(struct halmac_adapter *halmac_adapter,
+				u32 halmac_offset);
+
+enum halmac_ret_status
+halmac_reg_write_16_usb_88xx(struct halmac_adapter *halmac_adapter,
+			     u32 halmac_offset, u16 halmac_data);
+
+u32 halmac_reg_read_32_usb_88xx(struct halmac_adapter *halmac_adapter,
+				u32 halmac_offset);
+
+enum halmac_ret_status
+halmac_reg_write_32_usb_88xx(struct halmac_adapter *halmac_adapter,
+			     u32 halmac_offset, u32 halmac_data);
+
+enum halmac_ret_status
+halmac_set_bulkout_num_88xx(struct halmac_adapter *halmac_adapter,
+			    u8 bulkout_num);
+
+enum halmac_ret_status
+halmac_get_usb_bulkout_id_88xx(struct halmac_adapter *halmac_adapter,
+			       u8 *halmac_buf, u32 halmac_size, u8 *bulkout_id);
+
+enum halmac_ret_status halmac_cfg_tx_agg_align_usb_not_support_88xx(
+	struct halmac_adapter *halmac_adapter, u8 enable, u16 align_size);
+
+#endif /* _HALMAC_API_88XX_USB_H_ */
diff --git a/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_func_88xx.c b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_func_88xx.c
new file mode 100644
index 000000000000..f33024e4d853
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_func_88xx.c
@@ -0,0 +1,4494 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#include "halmac_88xx_cfg.h"
+
+static enum halmac_ret_status
+halmac_dump_efuse_fw_88xx(struct halmac_adapter *halmac_adapter);
+
+static enum halmac_ret_status
+halmac_dump_efuse_drv_88xx(struct halmac_adapter *halmac_adapter);
+
+static enum halmac_ret_status
+halmac_update_eeprom_mask_88xx(struct halmac_adapter *halmac_adapter,
+			       struct halmac_pg_efuse_info *pg_efuse_info,
+			       u8 *eeprom_mask_updated);
+
+static enum halmac_ret_status
+halmac_check_efuse_enough_88xx(struct halmac_adapter *halmac_adapter,
+			       struct halmac_pg_efuse_info *pg_efuse_info,
+			       u8 *eeprom_mask_updated);
+
+static enum halmac_ret_status
+halmac_program_efuse_88xx(struct halmac_adapter *halmac_adapter,
+			  struct halmac_pg_efuse_info *pg_efuse_info,
+			  u8 *eeprom_mask_updated);
+
+static enum halmac_ret_status
+halmac_pwr_sub_seq_parer_88xx(struct halmac_adapter *halmac_adapter, u8 cut,
+			      u8 fab, u8 intf,
+			      struct halmac_wl_pwr_cfg_ *pwr_sub_seq_cfg);
+
+static enum halmac_ret_status
+halmac_parse_c2h_debug_88xx(struct halmac_adapter *halmac_adapter, u8 *c2h_buf,
+			    u32 c2h_size);
+
+static enum halmac_ret_status
+halmac_parse_scan_status_rpt_88xx(struct halmac_adapter *halmac_adapter,
+				  u8 *c2h_buf, u32 c2h_size);
+
+static enum halmac_ret_status
+halmac_parse_psd_data_88xx(struct halmac_adapter *halmac_adapter, u8 *c2h_buf,
+			   u32 c2h_size);
+
+static enum halmac_ret_status
+halmac_parse_efuse_data_88xx(struct halmac_adapter *halmac_adapter, u8 *c2h_buf,
+			     u32 c2h_size);
+
+static enum halmac_ret_status
+halmac_parse_h2c_ack_88xx(struct halmac_adapter *halmac_adapter, u8 *c2h_buf,
+			  u32 c2h_size);
+
+static enum halmac_ret_status
+halmac_enqueue_para_buff_88xx(struct halmac_adapter *halmac_adapter,
+			      struct halmac_phy_parameter_info *para_info,
+			      u8 *curr_buff_wptr, bool *end_cmd);
+
+static enum halmac_ret_status
+halmac_parse_h2c_ack_phy_efuse_88xx(struct halmac_adapter *halmac_adapter,
+				    u8 *c2h_buf, u32 c2h_size);
+
+static enum halmac_ret_status
+halmac_parse_h2c_ack_cfg_para_88xx(struct halmac_adapter *halmac_adapter,
+				   u8 *c2h_buf, u32 c2h_size);
+
+static enum halmac_ret_status
+halmac_gen_cfg_para_h2c_88xx(struct halmac_adapter *halmac_adapter,
+			     u8 *h2c_buff);
+
+static enum halmac_ret_status
+halmac_parse_h2c_ack_update_packet_88xx(struct halmac_adapter *halmac_adapter,
+					u8 *c2h_buf, u32 c2h_size);
+
+static enum halmac_ret_status
+halmac_parse_h2c_ack_update_datapack_88xx(struct halmac_adapter *halmac_adapter,
+					  u8 *c2h_buf, u32 c2h_size);
+
+static enum halmac_ret_status
+halmac_parse_h2c_ack_run_datapack_88xx(struct halmac_adapter *halmac_adapter,
+				       u8 *c2h_buf, u32 c2h_size);
+
+static enum halmac_ret_status
+halmac_parse_h2c_ack_channel_switch_88xx(struct halmac_adapter *halmac_adapter,
+					 u8 *c2h_buf, u32 c2h_size);
+
+static enum halmac_ret_status
+halmac_parse_h2c_ack_iqk_88xx(struct halmac_adapter *halmac_adapter,
+			      u8 *c2h_buf, u32 c2h_size);
+
+static enum halmac_ret_status
+halmac_parse_h2c_ack_power_tracking_88xx(struct halmac_adapter *halmac_adapter,
+					 u8 *c2h_buf, u32 c2h_size);
+
+void halmac_init_offload_feature_state_machine_88xx(
+	struct halmac_adapter *halmac_adapter)
+{
+	struct halmac_state *state = &halmac_adapter->halmac_state;
+
+	state->efuse_state_set.efuse_cmd_construct_state =
+		HALMAC_EFUSE_CMD_CONSTRUCT_IDLE;
+	state->efuse_state_set.process_status = HALMAC_CMD_PROCESS_IDLE;
+	state->efuse_state_set.seq_num = halmac_adapter->h2c_packet_seq;
+
+	state->cfg_para_state_set.cfg_para_cmd_construct_state =
+		HALMAC_CFG_PARA_CMD_CONSTRUCT_IDLE;
+	state->cfg_para_state_set.process_status = HALMAC_CMD_PROCESS_IDLE;
+	state->cfg_para_state_set.seq_num = halmac_adapter->h2c_packet_seq;
+
+	state->scan_state_set.scan_cmd_construct_state =
+		HALMAC_SCAN_CMD_CONSTRUCT_IDLE;
+	state->scan_state_set.process_status = HALMAC_CMD_PROCESS_IDLE;
+	state->scan_state_set.seq_num = halmac_adapter->h2c_packet_seq;
+
+	state->update_packet_set.process_status = HALMAC_CMD_PROCESS_IDLE;
+	state->update_packet_set.seq_num = halmac_adapter->h2c_packet_seq;
+
+	state->iqk_set.process_status = HALMAC_CMD_PROCESS_IDLE;
+	state->iqk_set.seq_num = halmac_adapter->h2c_packet_seq;
+
+	state->power_tracking_set.process_status = HALMAC_CMD_PROCESS_IDLE;
+	state->power_tracking_set.seq_num = halmac_adapter->h2c_packet_seq;
+
+	state->psd_set.process_status = HALMAC_CMD_PROCESS_IDLE;
+	state->psd_set.seq_num = halmac_adapter->h2c_packet_seq;
+	state->psd_set.data_size = 0;
+	state->psd_set.segment_size = 0;
+	state->psd_set.data = NULL;
+}
+
+enum halmac_ret_status
+halmac_dump_efuse_88xx(struct halmac_adapter *halmac_adapter,
+		       enum halmac_efuse_read_cfg cfg)
+{
+	u32 chk_h2c_init;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api =
+		(struct halmac_api *)halmac_adapter->halmac_api;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+	enum halmac_cmd_process_status *process_status =
+		&halmac_adapter->halmac_state.efuse_state_set.process_status;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	*process_status = HALMAC_CMD_PROCESS_SENDING;
+
+	if (halmac_transition_efuse_state_88xx(
+		    halmac_adapter, HALMAC_EFUSE_CMD_CONSTRUCT_H2C_SENT) !=
+	    HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ERROR_STATE;
+
+	if (cfg == HALMAC_EFUSE_R_AUTO) {
+		chk_h2c_init = HALMAC_REG_READ_32(halmac_adapter,
+						  REG_H2C_PKT_READADDR);
+		if (halmac_adapter->halmac_state.dlfw_state ==
+			    HALMAC_DLFW_NONE ||
+		    chk_h2c_init == 0)
+			status = halmac_dump_efuse_drv_88xx(halmac_adapter);
+		else
+			status = halmac_dump_efuse_fw_88xx(halmac_adapter);
+	} else if (cfg == HALMAC_EFUSE_R_FW) {
+		status = halmac_dump_efuse_fw_88xx(halmac_adapter);
+	} else {
+		status = halmac_dump_efuse_drv_88xx(halmac_adapter);
+	}
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_read_efuse error = %x\n", status);
+		return status;
+	}
+
+	return status;
+}
+
+enum halmac_ret_status
+halmac_func_read_efuse_88xx(struct halmac_adapter *halmac_adapter, u32 offset,
+			    u32 size, u8 *efuse_map)
+{
+	void *driver_adapter = NULL;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	if (!efuse_map) {
+		pr_err("Malloc for dump efuse map error\n");
+		return HALMAC_RET_NULL_POINTER;
+	}
+
+	if (halmac_adapter->hal_efuse_map_valid)
+		memcpy(efuse_map, halmac_adapter->hal_efuse_map + offset, size);
+	else if (halmac_read_hw_efuse_88xx(halmac_adapter, offset, size,
+					   efuse_map) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_EFUSE_R_FAIL;
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_read_hw_efuse_88xx(struct halmac_adapter *halmac_adapter, u32 offset,
+			  u32 size, u8 *efuse_map)
+{
+	u8 value8;
+	u32 value32;
+	u32 address;
+	u32 tmp32, counter;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	/* Read efuse no need 2.5V LDO */
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_LDO_EFUSE_CTRL + 3);
+	if (value8 & BIT(7))
+		HALMAC_REG_WRITE_8(halmac_adapter, REG_LDO_EFUSE_CTRL + 3,
+				   (u8)(value8 & ~(BIT(7))));
+
+	value32 = HALMAC_REG_READ_32(halmac_adapter, REG_EFUSE_CTRL);
+
+	for (address = offset; address < offset + size; address++) {
+		value32 = value32 &
+			  ~((BIT_MASK_EF_DATA) |
+			    (BIT_MASK_EF_ADDR << BIT_SHIFT_EF_ADDR));
+		value32 = value32 |
+			  ((address & BIT_MASK_EF_ADDR) << BIT_SHIFT_EF_ADDR);
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_EFUSE_CTRL,
+				    value32 & (~BIT_EF_FLAG));
+
+		counter = 1000000;
+		do {
+			udelay(1);
+			tmp32 = HALMAC_REG_READ_32(halmac_adapter,
+						   REG_EFUSE_CTRL);
+			counter--;
+			if (counter == 0) {
+				pr_err("HALMAC_RET_EFUSE_R_FAIL\n");
+				return HALMAC_RET_EFUSE_R_FAIL;
+			}
+		} while ((tmp32 & BIT_EF_FLAG) == 0);
+
+		*(efuse_map + address - offset) =
+			(u8)(tmp32 & BIT_MASK_EF_DATA);
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+static enum halmac_ret_status
+halmac_dump_efuse_drv_88xx(struct halmac_adapter *halmac_adapter)
+{
+	u8 *efuse_map = NULL;
+	u32 efuse_size;
+	void *driver_adapter = NULL;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	efuse_size = halmac_adapter->hw_config_info.efuse_size;
+
+	if (!halmac_adapter->hal_efuse_map) {
+		halmac_adapter->hal_efuse_map = kzalloc(efuse_size, GFP_KERNEL);
+		if (!halmac_adapter->hal_efuse_map) {
+			pr_err("[ERR]halmac allocate efuse map Fail!!\n");
+			return HALMAC_RET_MALLOC_FAIL;
+		}
+	}
+
+	efuse_map = kzalloc(efuse_size, GFP_KERNEL);
+	if (!efuse_map) {
+		/* out of memory */
+		return HALMAC_RET_MALLOC_FAIL;
+	}
+
+	if (halmac_read_hw_efuse_88xx(halmac_adapter, 0, efuse_size,
+				      efuse_map) != HALMAC_RET_SUCCESS) {
+		kfree(efuse_map);
+		return HALMAC_RET_EFUSE_R_FAIL;
+	}
+
+	spin_lock(&halmac_adapter->efuse_lock);
+	memcpy(halmac_adapter->hal_efuse_map, efuse_map, efuse_size);
+	halmac_adapter->hal_efuse_map_valid = true;
+	spin_unlock(&halmac_adapter->efuse_lock);
+
+	kfree(efuse_map);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+static enum halmac_ret_status
+halmac_dump_efuse_fw_88xx(struct halmac_adapter *halmac_adapter)
+{
+	u8 h2c_buff[HALMAC_H2C_CMD_SIZE_88XX] = {0};
+	u16 h2c_seq_mum = 0;
+	void *driver_adapter = NULL;
+	struct halmac_h2c_header_info h2c_header_info;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	h2c_header_info.sub_cmd_id = SUB_CMD_ID_DUMP_PHYSICAL_EFUSE;
+	h2c_header_info.content_size = 0;
+	h2c_header_info.ack = true;
+	halmac_set_fw_offload_h2c_header_88xx(halmac_adapter, h2c_buff,
+					      &h2c_header_info, &h2c_seq_mum);
+	halmac_adapter->halmac_state.efuse_state_set.seq_num = h2c_seq_mum;
+
+	if (!halmac_adapter->hal_efuse_map) {
+		halmac_adapter->hal_efuse_map = kzalloc(
+			halmac_adapter->hw_config_info.efuse_size, GFP_KERNEL);
+		if (!halmac_adapter->hal_efuse_map) {
+			/* out of memory */
+			return HALMAC_RET_MALLOC_FAIL;
+		}
+	}
+
+	if (!halmac_adapter->hal_efuse_map_valid) {
+		status = halmac_send_h2c_pkt_88xx(halmac_adapter, h2c_buff,
+						  HALMAC_H2C_CMD_SIZE_88XX,
+						  true);
+		if (status != HALMAC_RET_SUCCESS) {
+			pr_err("halmac_read_efuse_fw Fail = %x!!\n", status);
+			return status;
+		}
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_func_write_efuse_88xx(struct halmac_adapter *halmac_adapter, u32 offset,
+			     u8 value)
+{
+	const u8 wite_protect_code = 0x69;
+	u32 value32, tmp32, counter;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	spin_lock(&halmac_adapter->efuse_lock);
+	halmac_adapter->hal_efuse_map_valid = false;
+	spin_unlock(&halmac_adapter->efuse_lock);
+
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_PMC_DBG_CTRL2 + 3,
+			   wite_protect_code);
+
+	/* Enable 2.5V LDO */
+	HALMAC_REG_WRITE_8(
+		halmac_adapter, REG_LDO_EFUSE_CTRL + 3,
+		(u8)(HALMAC_REG_READ_8(halmac_adapter, REG_LDO_EFUSE_CTRL + 3) |
+		     BIT(7)));
+
+	value32 = HALMAC_REG_READ_32(halmac_adapter, REG_EFUSE_CTRL);
+	value32 =
+		value32 &
+		~((BIT_MASK_EF_DATA) | (BIT_MASK_EF_ADDR << BIT_SHIFT_EF_ADDR));
+	value32 = value32 | ((offset & BIT_MASK_EF_ADDR) << BIT_SHIFT_EF_ADDR) |
+		  (value & BIT_MASK_EF_DATA);
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_EFUSE_CTRL,
+			    value32 | BIT_EF_FLAG);
+
+	counter = 1000000;
+	do {
+		udelay(1);
+		tmp32 = HALMAC_REG_READ_32(halmac_adapter, REG_EFUSE_CTRL);
+		counter--;
+		if (counter == 0) {
+			pr_err("halmac_write_efuse Fail !!\n");
+			return HALMAC_RET_EFUSE_W_FAIL;
+		}
+	} while ((tmp32 & BIT_EF_FLAG) == BIT_EF_FLAG);
+
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_PMC_DBG_CTRL2 + 3, 0x00);
+
+	/* Disable 2.5V LDO */
+	HALMAC_REG_WRITE_8(
+		halmac_adapter, REG_LDO_EFUSE_CTRL + 3,
+		(u8)(HALMAC_REG_READ_8(halmac_adapter, REG_LDO_EFUSE_CTRL + 3) &
+		     ~(BIT(7))));
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_func_switch_efuse_bank_88xx(struct halmac_adapter *halmac_adapter,
+				   enum halmac_efuse_bank efuse_bank)
+{
+	u8 reg_value;
+	struct halmac_api *halmac_api;
+
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	if (halmac_transition_efuse_state_88xx(
+		    halmac_adapter, HALMAC_EFUSE_CMD_CONSTRUCT_BUSY) !=
+	    HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ERROR_STATE;
+
+	reg_value = HALMAC_REG_READ_8(halmac_adapter, REG_LDO_EFUSE_CTRL + 1);
+
+	if (efuse_bank == (reg_value & (BIT(0) | BIT(1))))
+		return HALMAC_RET_SUCCESS;
+
+	reg_value &= ~(BIT(0) | BIT(1));
+	reg_value |= efuse_bank;
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_LDO_EFUSE_CTRL + 1, reg_value);
+
+	if ((HALMAC_REG_READ_8(halmac_adapter, REG_LDO_EFUSE_CTRL + 1) &
+	     (BIT(0) | BIT(1))) != efuse_bank)
+		return HALMAC_RET_SWITCH_EFUSE_BANK_FAIL;
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_eeprom_parser_88xx(struct halmac_adapter *halmac_adapter,
+			  u8 *physical_efuse_map, u8 *logical_efuse_map)
+{
+	u8 j;
+	u8 value8;
+	u8 block_index;
+	u8 valid_word_enable, word_enable;
+	u8 efuse_read_header, efuse_read_header2 = 0;
+	u32 eeprom_index;
+	u32 efuse_index = 0;
+	u32 eeprom_size = halmac_adapter->hw_config_info.eeprom_size;
+	void *driver_adapter = NULL;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	memset(logical_efuse_map, 0xFF, eeprom_size);
+
+	do {
+		value8 = *(physical_efuse_map + efuse_index);
+		efuse_read_header = value8;
+
+		if ((efuse_read_header & 0x1f) == 0x0f) {
+			efuse_index++;
+			value8 = *(physical_efuse_map + efuse_index);
+			efuse_read_header2 = value8;
+			block_index = ((efuse_read_header2 & 0xF0) >> 1) |
+				      ((efuse_read_header >> 5) & 0x07);
+			word_enable = efuse_read_header2 & 0x0F;
+		} else {
+			block_index = (efuse_read_header & 0xF0) >> 4;
+			word_enable = efuse_read_header & 0x0F;
+		}
+
+		if (efuse_read_header == 0xff)
+			break;
+
+		efuse_index++;
+
+		if (efuse_index >= halmac_adapter->hw_config_info.efuse_size -
+					   HALMAC_PROTECTED_EFUSE_SIZE_88XX - 1)
+			return HALMAC_RET_EEPROM_PARSING_FAIL;
+
+		for (j = 0; j < 4; j++) {
+			valid_word_enable =
+				(u8)((~(word_enable >> j)) & BIT(0));
+			if (valid_word_enable != 1)
+				continue;
+
+			eeprom_index = (block_index << 3) + (j << 1);
+
+			if ((eeprom_index + 1) > eeprom_size) {
+				pr_err("Error: EEPROM addr exceeds eeprom_size:0x%X, at eFuse 0x%X\n",
+				       eeprom_size, efuse_index - 1);
+				if ((efuse_read_header & 0x1f) == 0x0f)
+					pr_err("Error: EEPROM header: 0x%X, 0x%X,\n",
+					       efuse_read_header,
+					       efuse_read_header2);
+				else
+					pr_err("Error: EEPROM header: 0x%X,\n",
+					       efuse_read_header);
+
+				return HALMAC_RET_EEPROM_PARSING_FAIL;
+			}
+
+			value8 = *(physical_efuse_map + efuse_index);
+			*(logical_efuse_map + eeprom_index) = value8;
+
+			eeprom_index++;
+			efuse_index++;
+
+			if (efuse_index >
+			    halmac_adapter->hw_config_info.efuse_size -
+				    HALMAC_PROTECTED_EFUSE_SIZE_88XX - 1)
+				return HALMAC_RET_EEPROM_PARSING_FAIL;
+
+			value8 = *(physical_efuse_map + efuse_index);
+			*(logical_efuse_map + eeprom_index) = value8;
+
+			efuse_index++;
+
+			if (efuse_index >
+			    halmac_adapter->hw_config_info.efuse_size -
+				    HALMAC_PROTECTED_EFUSE_SIZE_88XX)
+				return HALMAC_RET_EEPROM_PARSING_FAIL;
+		}
+	} while (1);
+
+	halmac_adapter->efuse_end = efuse_index;
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_read_logical_efuse_map_88xx(struct halmac_adapter *halmac_adapter,
+				   u8 *map)
+{
+	u8 *efuse_map = NULL;
+	u32 efuse_size;
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	efuse_size = halmac_adapter->hw_config_info.efuse_size;
+
+	if (!halmac_adapter->hal_efuse_map_valid) {
+		efuse_map = kzalloc(efuse_size, GFP_KERNEL);
+		if (!efuse_map) {
+			pr_err("[ERR]halmac allocate local efuse map Fail!!\n");
+			return HALMAC_RET_MALLOC_FAIL;
+		}
+
+		status = halmac_func_read_efuse_88xx(halmac_adapter, 0,
+						     efuse_size, efuse_map);
+		if (status != HALMAC_RET_SUCCESS) {
+			pr_err("[ERR]halmac_read_efuse error = %x\n", status);
+			kfree(efuse_map);
+			return status;
+		}
+
+		if (!halmac_adapter->hal_efuse_map) {
+			halmac_adapter->hal_efuse_map =
+				kzalloc(efuse_size, GFP_KERNEL);
+			if (!halmac_adapter->hal_efuse_map) {
+				pr_err("[ERR]halmac allocate efuse map Fail!!\n");
+				kfree(efuse_map);
+				return HALMAC_RET_MALLOC_FAIL;
+			}
+		}
+
+		spin_lock(&halmac_adapter->efuse_lock);
+		memcpy(halmac_adapter->hal_efuse_map, efuse_map, efuse_size);
+		halmac_adapter->hal_efuse_map_valid = true;
+		spin_unlock(&halmac_adapter->efuse_lock);
+
+		kfree(efuse_map);
+	}
+
+	if (halmac_eeprom_parser_88xx(halmac_adapter,
+				      halmac_adapter->hal_efuse_map,
+				      map) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_EEPROM_PARSING_FAIL;
+
+	return status;
+}
+
+enum halmac_ret_status
+halmac_func_write_logical_efuse_88xx(struct halmac_adapter *halmac_adapter,
+				     u32 offset, u8 value)
+{
+	u8 pg_efuse_byte1, pg_efuse_byte2;
+	u8 pg_block, pg_block_index;
+	u8 pg_efuse_header, pg_efuse_header2;
+	u8 *eeprom_map = NULL;
+	u32 eeprom_size = halmac_adapter->hw_config_info.eeprom_size;
+	u32 efuse_end, pg_efuse_num;
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	eeprom_map = kzalloc(eeprom_size, GFP_KERNEL);
+	if (!eeprom_map) {
+		/* out of memory */
+		return HALMAC_RET_MALLOC_FAIL;
+	}
+	memset(eeprom_map, 0xFF, eeprom_size);
+
+	status = halmac_read_logical_efuse_map_88xx(halmac_adapter, eeprom_map);
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("[ERR]halmac_read_logical_efuse_map_88xx error = %x\n",
+		       status);
+		kfree(eeprom_map);
+		return status;
+	}
+
+	if (*(eeprom_map + offset) != value) {
+		efuse_end = halmac_adapter->efuse_end;
+		pg_block = (u8)(offset >> 3);
+		pg_block_index = (u8)((offset & (8 - 1)) >> 1);
+
+		if (offset > 0x7f) {
+			pg_efuse_header =
+				(((pg_block & 0x07) << 5) & 0xE0) | 0x0F;
+			pg_efuse_header2 =
+				(u8)(((pg_block & 0x78) << 1) +
+				     ((0x1 << pg_block_index) ^ 0x0F));
+		} else {
+			pg_efuse_header =
+				(u8)((pg_block << 4) +
+				     ((0x01 << pg_block_index) ^ 0x0F));
+		}
+
+		if ((offset & 1) == 0) {
+			pg_efuse_byte1 = value;
+			pg_efuse_byte2 = *(eeprom_map + offset + 1);
+		} else {
+			pg_efuse_byte1 = *(eeprom_map + offset - 1);
+			pg_efuse_byte2 = value;
+		}
+
+		if (offset > 0x7f) {
+			pg_efuse_num = 4;
+			if (halmac_adapter->hw_config_info.efuse_size <=
+			    (pg_efuse_num + HALMAC_PROTECTED_EFUSE_SIZE_88XX +
+			     halmac_adapter->efuse_end)) {
+				kfree(eeprom_map);
+				return HALMAC_RET_EFUSE_NOT_ENOUGH;
+			}
+			halmac_func_write_efuse_88xx(halmac_adapter, efuse_end,
+						     pg_efuse_header);
+			halmac_func_write_efuse_88xx(halmac_adapter,
+						     efuse_end + 1,
+						     pg_efuse_header2);
+			halmac_func_write_efuse_88xx(
+				halmac_adapter, efuse_end + 2, pg_efuse_byte1);
+			status = halmac_func_write_efuse_88xx(
+				halmac_adapter, efuse_end + 3, pg_efuse_byte2);
+		} else {
+			pg_efuse_num = 3;
+			if (halmac_adapter->hw_config_info.efuse_size <=
+			    (pg_efuse_num + HALMAC_PROTECTED_EFUSE_SIZE_88XX +
+			     halmac_adapter->efuse_end)) {
+				kfree(eeprom_map);
+				return HALMAC_RET_EFUSE_NOT_ENOUGH;
+			}
+			halmac_func_write_efuse_88xx(halmac_adapter, efuse_end,
+						     pg_efuse_header);
+			halmac_func_write_efuse_88xx(
+				halmac_adapter, efuse_end + 1, pg_efuse_byte1);
+			status = halmac_func_write_efuse_88xx(
+				halmac_adapter, efuse_end + 2, pg_efuse_byte2);
+		}
+
+		if (status != HALMAC_RET_SUCCESS) {
+			pr_err("[ERR]halmac_write_logical_efuse error = %x\n",
+			       status);
+			kfree(eeprom_map);
+			return status;
+		}
+	}
+
+	kfree(eeprom_map);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_func_pg_efuse_by_map_88xx(struct halmac_adapter *halmac_adapter,
+				 struct halmac_pg_efuse_info *pg_efuse_info,
+				 enum halmac_efuse_read_cfg cfg)
+{
+	u8 *eeprom_mask_updated = NULL;
+	u32 eeprom_mask_size = halmac_adapter->hw_config_info.eeprom_size >> 4;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	eeprom_mask_updated = kzalloc(eeprom_mask_size, GFP_KERNEL);
+	if (!eeprom_mask_updated) {
+		/* out of memory */
+		return HALMAC_RET_MALLOC_FAIL;
+	}
+
+	status = halmac_update_eeprom_mask_88xx(halmac_adapter, pg_efuse_info,
+						eeprom_mask_updated);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("[ERR]halmac_update_eeprom_mask_88xx error = %x\n",
+		       status);
+		kfree(eeprom_mask_updated);
+		return status;
+	}
+
+	status = halmac_check_efuse_enough_88xx(halmac_adapter, pg_efuse_info,
+						eeprom_mask_updated);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("[ERR]halmac_check_efuse_enough_88xx error = %x\n",
+		       status);
+		kfree(eeprom_mask_updated);
+		return status;
+	}
+
+	status = halmac_program_efuse_88xx(halmac_adapter, pg_efuse_info,
+					   eeprom_mask_updated);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("[ERR]halmac_program_efuse_88xx error = %x\n", status);
+		kfree(eeprom_mask_updated);
+		return status;
+	}
+
+	kfree(eeprom_mask_updated);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+static enum halmac_ret_status
+halmac_update_eeprom_mask_88xx(struct halmac_adapter *halmac_adapter,
+			       struct halmac_pg_efuse_info *pg_efuse_info,
+			       u8 *eeprom_mask_updated)
+{
+	u8 *eeprom_map = NULL;
+	u32 eeprom_size = halmac_adapter->hw_config_info.eeprom_size;
+	u8 *eeprom_map_pg, *eeprom_mask;
+	u16 i, j;
+	u16 map_byte_offset, mask_byte_offset;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	void *driver_adapter = NULL;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	eeprom_map = kzalloc(eeprom_size, GFP_KERNEL);
+	if (!eeprom_map) {
+		/* out of memory */
+		return HALMAC_RET_MALLOC_FAIL;
+	}
+	memset(eeprom_map, 0xFF, eeprom_size);
+
+	memset(eeprom_mask_updated, 0x00, pg_efuse_info->efuse_mask_size);
+
+	status = halmac_read_logical_efuse_map_88xx(halmac_adapter, eeprom_map);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		kfree(eeprom_map);
+		return status;
+	}
+
+	eeprom_map_pg = pg_efuse_info->efuse_map;
+	eeprom_mask = pg_efuse_info->efuse_mask;
+
+	for (i = 0; i < pg_efuse_info->efuse_mask_size; i++)
+		*(eeprom_mask_updated + i) = *(eeprom_mask + i);
+
+	for (i = 0; i < pg_efuse_info->efuse_map_size; i = i + 16) {
+		for (j = 0; j < 16; j = j + 2) {
+			map_byte_offset = i + j;
+			mask_byte_offset = i >> 4;
+			if (*(eeprom_map_pg + map_byte_offset) ==
+			    *(eeprom_map + map_byte_offset)) {
+				if (*(eeprom_map_pg + map_byte_offset + 1) ==
+				    *(eeprom_map + map_byte_offset + 1)) {
+					switch (j) {
+					case 0:
+						*(eeprom_mask_updated +
+						  mask_byte_offset) =
+							*(eeprom_mask_updated +
+							  mask_byte_offset) &
+							(BIT(4) ^ 0xFF);
+						break;
+					case 2:
+						*(eeprom_mask_updated +
+						  mask_byte_offset) =
+							*(eeprom_mask_updated +
+							  mask_byte_offset) &
+							(BIT(5) ^ 0xFF);
+						break;
+					case 4:
+						*(eeprom_mask_updated +
+						  mask_byte_offset) =
+							*(eeprom_mask_updated +
+							  mask_byte_offset) &
+							(BIT(6) ^ 0xFF);
+						break;
+					case 6:
+						*(eeprom_mask_updated +
+						  mask_byte_offset) =
+							*(eeprom_mask_updated +
+							  mask_byte_offset) &
+							(BIT(7) ^ 0xFF);
+						break;
+					case 8:
+						*(eeprom_mask_updated +
+						  mask_byte_offset) =
+							*(eeprom_mask_updated +
+							  mask_byte_offset) &
+							(BIT(0) ^ 0xFF);
+						break;
+					case 10:
+						*(eeprom_mask_updated +
+						  mask_byte_offset) =
+							*(eeprom_mask_updated +
+							  mask_byte_offset) &
+							(BIT(1) ^ 0xFF);
+						break;
+					case 12:
+						*(eeprom_mask_updated +
+						  mask_byte_offset) =
+							*(eeprom_mask_updated +
+							  mask_byte_offset) &
+							(BIT(2) ^ 0xFF);
+						break;
+					case 14:
+						*(eeprom_mask_updated +
+						  mask_byte_offset) =
+							*(eeprom_mask_updated +
+							  mask_byte_offset) &
+							(BIT(3) ^ 0xFF);
+						break;
+					default:
+						break;
+					}
+				}
+			}
+		}
+	}
+
+	kfree(eeprom_map);
+
+	return status;
+}
+
+static enum halmac_ret_status
+halmac_check_efuse_enough_88xx(struct halmac_adapter *halmac_adapter,
+			       struct halmac_pg_efuse_info *pg_efuse_info,
+			       u8 *eeprom_mask_updated)
+{
+	u8 pre_word_enb, word_enb;
+	u8 pg_efuse_header, pg_efuse_header2;
+	u8 pg_block;
+	u16 i, j;
+	u32 efuse_end;
+	u32 tmp_eeprom_offset, pg_efuse_num = 0;
+
+	efuse_end = halmac_adapter->efuse_end;
+
+	for (i = 0; i < pg_efuse_info->efuse_map_size; i = i + 8) {
+		tmp_eeprom_offset = i;
+
+		if ((tmp_eeprom_offset & 7) > 0) {
+			pre_word_enb =
+				(*(eeprom_mask_updated + (i >> 4)) & 0x0F);
+			word_enb = pre_word_enb ^ 0x0F;
+		} else {
+			pre_word_enb = (*(eeprom_mask_updated + (i >> 4)) >> 4);
+			word_enb = pre_word_enb ^ 0x0F;
+		}
+
+		pg_block = (u8)(tmp_eeprom_offset >> 3);
+
+		if (pre_word_enb > 0) {
+			if (tmp_eeprom_offset > 0x7f) {
+				pg_efuse_header =
+					(((pg_block & 0x07) << 5) & 0xE0) |
+					0x0F;
+				pg_efuse_header2 = (u8)(
+					((pg_block & 0x78) << 1) + word_enb);
+			} else {
+				pg_efuse_header =
+					(u8)((pg_block << 4) + word_enb);
+			}
+
+			if (tmp_eeprom_offset > 0x7f) {
+				pg_efuse_num++;
+				pg_efuse_num++;
+				efuse_end = efuse_end + 2;
+				for (j = 0; j < 4; j++) {
+					if (((pre_word_enb >> j) & 0x1) > 0) {
+						pg_efuse_num++;
+						pg_efuse_num++;
+						efuse_end = efuse_end + 2;
+					}
+				}
+			} else {
+				pg_efuse_num++;
+				efuse_end = efuse_end + 1;
+				for (j = 0; j < 4; j++) {
+					if (((pre_word_enb >> j) & 0x1) > 0) {
+						pg_efuse_num++;
+						pg_efuse_num++;
+						efuse_end = efuse_end + 2;
+					}
+				}
+			}
+		}
+	}
+
+	if (halmac_adapter->hw_config_info.efuse_size <=
+	    (pg_efuse_num + HALMAC_PROTECTED_EFUSE_SIZE_88XX +
+	     halmac_adapter->efuse_end))
+		return HALMAC_RET_EFUSE_NOT_ENOUGH;
+
+	return HALMAC_RET_SUCCESS;
+}
+
+static enum halmac_ret_status
+halmac_program_efuse_88xx(struct halmac_adapter *halmac_adapter,
+			  struct halmac_pg_efuse_info *pg_efuse_info,
+			  u8 *eeprom_mask_updated)
+{
+	u8 pre_word_enb, word_enb;
+	u8 pg_efuse_header, pg_efuse_header2;
+	u8 pg_block;
+	u16 i, j;
+	u32 efuse_end;
+	u32 tmp_eeprom_offset;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	efuse_end = halmac_adapter->efuse_end;
+
+	for (i = 0; i < pg_efuse_info->efuse_map_size; i = i + 8) {
+		tmp_eeprom_offset = i;
+
+		if (((tmp_eeprom_offset >> 3) & 1) > 0) {
+			pre_word_enb =
+				(*(eeprom_mask_updated + (i >> 4)) & 0x0F);
+			word_enb = pre_word_enb ^ 0x0F;
+		} else {
+			pre_word_enb = (*(eeprom_mask_updated + (i >> 4)) >> 4);
+			word_enb = pre_word_enb ^ 0x0F;
+		}
+
+		pg_block = (u8)(tmp_eeprom_offset >> 3);
+
+		if (pre_word_enb <= 0)
+			continue;
+
+		if (tmp_eeprom_offset > 0x7f) {
+			pg_efuse_header =
+				(((pg_block & 0x07) << 5) & 0xE0) | 0x0F;
+			pg_efuse_header2 =
+				(u8)(((pg_block & 0x78) << 1) + word_enb);
+		} else {
+			pg_efuse_header = (u8)((pg_block << 4) + word_enb);
+		}
+
+		if (tmp_eeprom_offset > 0x7f) {
+			halmac_func_write_efuse_88xx(halmac_adapter, efuse_end,
+						     pg_efuse_header);
+			status = halmac_func_write_efuse_88xx(halmac_adapter,
+							      efuse_end + 1,
+							      pg_efuse_header2);
+			efuse_end = efuse_end + 2;
+			for (j = 0; j < 4; j++) {
+				if (((pre_word_enb >> j) & 0x1) > 0) {
+					halmac_func_write_efuse_88xx(
+						halmac_adapter, efuse_end,
+						*(pg_efuse_info->efuse_map +
+						  tmp_eeprom_offset +
+						  (j << 1)));
+					status = halmac_func_write_efuse_88xx(
+						halmac_adapter, efuse_end + 1,
+						*(pg_efuse_info->efuse_map +
+						  tmp_eeprom_offset + (j << 1) +
+						  1));
+					efuse_end = efuse_end + 2;
+				}
+			}
+		} else {
+			status = halmac_func_write_efuse_88xx(
+				halmac_adapter, efuse_end, pg_efuse_header);
+			efuse_end = efuse_end + 1;
+			for (j = 0; j < 4; j++) {
+				if (((pre_word_enb >> j) & 0x1) > 0) {
+					halmac_func_write_efuse_88xx(
+						halmac_adapter, efuse_end,
+						*(pg_efuse_info->efuse_map +
+						  tmp_eeprom_offset +
+						  (j << 1)));
+					status = halmac_func_write_efuse_88xx(
+						halmac_adapter, efuse_end + 1,
+						*(pg_efuse_info->efuse_map +
+						  tmp_eeprom_offset + (j << 1) +
+						  1));
+					efuse_end = efuse_end + 2;
+				}
+			}
+		}
+	}
+
+	return status;
+}
+
+enum halmac_ret_status
+halmac_dlfw_to_mem_88xx(struct halmac_adapter *halmac_adapter, u8 *ram_code,
+			u32 dest, u32 code_size)
+{
+	u8 *code_ptr;
+	u8 first_part;
+	u32 mem_offset;
+	u32 pkt_size_tmp, send_pkt_size;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	code_ptr = ram_code;
+	mem_offset = 0;
+	first_part = 1;
+	pkt_size_tmp = code_size;
+
+	HALMAC_REG_WRITE_32(
+		halmac_adapter, REG_DDMA_CH0CTRL,
+		HALMAC_REG_READ_32(halmac_adapter, REG_DDMA_CH0CTRL) |
+			BIT_DDMACH0_RESET_CHKSUM_STS);
+
+	while (pkt_size_tmp != 0) {
+		if (pkt_size_tmp >= halmac_adapter->max_download_size)
+			send_pkt_size = halmac_adapter->max_download_size;
+		else
+			send_pkt_size = pkt_size_tmp;
+
+		if (halmac_send_fwpkt_88xx(
+			    halmac_adapter, code_ptr + mem_offset,
+			    send_pkt_size) != HALMAC_RET_SUCCESS) {
+			pr_err("halmac_send_fwpkt_88xx fail!!");
+			return HALMAC_RET_DLFW_FAIL;
+		}
+
+		if (halmac_iddma_dlfw_88xx(
+			    halmac_adapter,
+			    HALMAC_OCPBASE_TXBUF_88XX +
+				    halmac_adapter->hw_config_info.txdesc_size,
+			    dest + mem_offset, send_pkt_size,
+			    first_part) != HALMAC_RET_SUCCESS) {
+			pr_err("halmac_iddma_dlfw_88xx fail!!");
+			return HALMAC_RET_DLFW_FAIL;
+		}
+
+		first_part = 0;
+		mem_offset += send_pkt_size;
+		pkt_size_tmp -= send_pkt_size;
+	}
+
+	if (halmac_check_fw_chksum_88xx(halmac_adapter, dest) !=
+	    HALMAC_RET_SUCCESS) {
+		pr_err("halmac_check_fw_chksum_88xx fail!!");
+		return HALMAC_RET_DLFW_FAIL;
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_send_fwpkt_88xx(struct halmac_adapter *halmac_adapter, u8 *ram_code,
+		       u32 code_size)
+{
+	if (halmac_download_rsvd_page_88xx(halmac_adapter, ram_code,
+					   code_size) != HALMAC_RET_SUCCESS) {
+		pr_err("PLATFORM_SEND_RSVD_PAGE 0 error!!\n");
+		return HALMAC_RET_DL_RSVD_PAGE_FAIL;
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_iddma_dlfw_88xx(struct halmac_adapter *halmac_adapter, u32 source,
+		       u32 dest, u32 length, u8 first)
+{
+	u32 counter;
+	u32 ch0_control = (u32)(BIT_DDMACH0_CHKSUM_EN | BIT_DDMACH0_OWN);
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	counter = HALMC_DDMA_POLLING_COUNT;
+	while (HALMAC_REG_READ_32(halmac_adapter, REG_DDMA_CH0CTRL) &
+	       BIT_DDMACH0_OWN) {
+		counter--;
+		if (counter == 0) {
+			pr_err("%s error-1!!\n", __func__);
+			return HALMAC_RET_DDMA_FAIL;
+		}
+	}
+
+	ch0_control |= (length & BIT_MASK_DDMACH0_DLEN);
+	if (first == 0)
+		ch0_control |= BIT_DDMACH0_CHKSUM_CONT;
+
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_DDMA_CH0SA, source);
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_DDMA_CH0DA, dest);
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_DDMA_CH0CTRL, ch0_control);
+
+	counter = HALMC_DDMA_POLLING_COUNT;
+	while (HALMAC_REG_READ_32(halmac_adapter, REG_DDMA_CH0CTRL) &
+	       BIT_DDMACH0_OWN) {
+		counter--;
+		if (counter == 0) {
+			pr_err("%s error-2!!\n", __func__);
+			return HALMAC_RET_DDMA_FAIL;
+		}
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_check_fw_chksum_88xx(struct halmac_adapter *halmac_adapter,
+			    u32 memory_address)
+{
+	u8 mcu_fw_ctrl;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	enum halmac_ret_status status;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	mcu_fw_ctrl = HALMAC_REG_READ_8(halmac_adapter, REG_MCUFW_CTRL);
+
+	if (HALMAC_REG_READ_32(halmac_adapter, REG_DDMA_CH0CTRL) &
+	    BIT_DDMACH0_CHKSUM_STS) {
+		if (memory_address < HALMAC_OCPBASE_DMEM_88XX) {
+			mcu_fw_ctrl |= BIT_IMEM_DW_OK;
+			HALMAC_REG_WRITE_8(
+				halmac_adapter, REG_MCUFW_CTRL,
+				(u8)(mcu_fw_ctrl & ~(BIT_IMEM_CHKSUM_OK)));
+		} else {
+			mcu_fw_ctrl |= BIT_DMEM_DW_OK;
+			HALMAC_REG_WRITE_8(
+				halmac_adapter, REG_MCUFW_CTRL,
+				(u8)(mcu_fw_ctrl & ~(BIT_DMEM_CHKSUM_OK)));
+		}
+
+		pr_err("%s error!!\n", __func__);
+
+		status = HALMAC_RET_FW_CHECKSUM_FAIL;
+	} else {
+		if (memory_address < HALMAC_OCPBASE_DMEM_88XX) {
+			mcu_fw_ctrl |= BIT_IMEM_DW_OK;
+			HALMAC_REG_WRITE_8(
+				halmac_adapter, REG_MCUFW_CTRL,
+				(u8)(mcu_fw_ctrl | BIT_IMEM_CHKSUM_OK));
+		} else {
+			mcu_fw_ctrl |= BIT_DMEM_DW_OK;
+			HALMAC_REG_WRITE_8(
+				halmac_adapter, REG_MCUFW_CTRL,
+				(u8)(mcu_fw_ctrl | BIT_DMEM_CHKSUM_OK));
+		}
+
+		status = HALMAC_RET_SUCCESS;
+	}
+
+	return status;
+}
+
+enum halmac_ret_status
+halmac_dlfw_end_flow_88xx(struct halmac_adapter *halmac_adapter)
+{
+	u8 value8;
+	u32 counter;
+	void *driver_adapter = halmac_adapter->driver_adapter;
+	struct halmac_api *halmac_api =
+		(struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_TXDMA_STATUS, BIT(2));
+
+	/* Check IMEM & DMEM checksum is OK or not */
+	if ((HALMAC_REG_READ_8(halmac_adapter, REG_MCUFW_CTRL) & 0x50) == 0x50)
+		HALMAC_REG_WRITE_16(halmac_adapter, REG_MCUFW_CTRL,
+				    (u16)(HALMAC_REG_READ_16(halmac_adapter,
+							     REG_MCUFW_CTRL) |
+					  BIT_FW_DW_RDY));
+	else
+		return HALMAC_RET_DLFW_FAIL;
+
+	HALMAC_REG_WRITE_8(
+		halmac_adapter, REG_MCUFW_CTRL,
+		(u8)(HALMAC_REG_READ_8(halmac_adapter, REG_MCUFW_CTRL) &
+		     ~(BIT(0))));
+
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_RSV_CTRL + 1);
+	value8 = (u8)(value8 | BIT(0));
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_RSV_CTRL + 1, value8);
+
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_SYS_FUNC_EN + 1);
+	value8 = (u8)(value8 | BIT(2));
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_SYS_FUNC_EN + 1,
+			   value8); /* Release MCU reset */
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"Download Finish, Reset CPU\n");
+
+	counter = 10000;
+	while (HALMAC_REG_READ_16(halmac_adapter, REG_MCUFW_CTRL) != 0xC078) {
+		if (counter == 0) {
+			pr_err("Check 0x80 = 0xC078 fail\n");
+			if ((HALMAC_REG_READ_32(halmac_adapter, REG_FW_DBG7) &
+			     0xFFFFFF00) == 0xFAAAAA00)
+				pr_err("Key fail\n");
+			return HALMAC_RET_DLFW_FAIL;
+		}
+		counter--;
+		usleep_range(50, 60);
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"Check 0x80 = 0xC078 counter = %d\n", counter);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_free_dl_fw_end_flow_88xx(struct halmac_adapter *halmac_adapter)
+{
+	u32 counter;
+	struct halmac_api *halmac_api =
+		(struct halmac_api *)halmac_adapter->halmac_api;
+
+	counter = 100;
+	while (HALMAC_REG_READ_8(halmac_adapter, REG_HMETFR + 3) != 0) {
+		counter--;
+		if (counter == 0) {
+			pr_err("[ERR]0x1CF != 0\n");
+			return HALMAC_RET_DLFW_FAIL;
+		}
+		usleep_range(50, 60);
+	}
+
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_HMETFR + 3,
+			   ID_INFORM_DLEMEM_RDY);
+
+	counter = 10000;
+	while (HALMAC_REG_READ_8(halmac_adapter, REG_C2HEVT_3 + 3) !=
+	       ID_INFORM_DLEMEM_RDY) {
+		counter--;
+		if (counter == 0) {
+			pr_err("[ERR]0x1AF != 0x80\n");
+			return HALMAC_RET_DLFW_FAIL;
+		}
+		usleep_range(50, 60);
+	}
+
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_C2HEVT_3 + 3, 0);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_pwr_seq_parser_88xx(struct halmac_adapter *halmac_adapter, u8 cut,
+			   u8 fab, u8 intf,
+			   struct halmac_wl_pwr_cfg_ **pp_pwr_seq_cfg)
+{
+	u32 seq_idx = 0;
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+	struct halmac_wl_pwr_cfg_ *seq_cmd;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	do {
+		seq_cmd = pp_pwr_seq_cfg[seq_idx];
+
+		if (!seq_cmd)
+			break;
+
+		status = halmac_pwr_sub_seq_parer_88xx(halmac_adapter, cut, fab,
+						       intf, seq_cmd);
+		if (status != HALMAC_RET_SUCCESS) {
+			pr_err("[Err]pwr sub seq parser fail, status = 0x%X!\n",
+			       status);
+			return status;
+		}
+
+		seq_idx++;
+	} while (1);
+
+	return status;
+}
+
+static enum halmac_ret_status
+halmac_pwr_sub_seq_parer_do_cmd_88xx(struct halmac_adapter *halmac_adapter,
+				     struct halmac_wl_pwr_cfg_ *sub_seq_cmd,
+				     bool *reti)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	u8 value, flag;
+	u8 polling_bit;
+	u32 polling_count;
+	static u32 poll_to_static;
+	u32 offset;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+	*reti = true;
+
+	switch (sub_seq_cmd->cmd) {
+	case HALMAC_PWR_CMD_WRITE:
+		if (sub_seq_cmd->base == HALMAC_PWR_BASEADDR_SDIO)
+			offset = sub_seq_cmd->offset | SDIO_LOCAL_OFFSET;
+		else
+			offset = sub_seq_cmd->offset;
+
+		value = HALMAC_REG_READ_8(halmac_adapter, offset);
+		value = (u8)(value & (u8)(~(sub_seq_cmd->msk)));
+		value = (u8)(value |
+			     (u8)(sub_seq_cmd->value & sub_seq_cmd->msk));
+
+		HALMAC_REG_WRITE_8(halmac_adapter, offset, value);
+		break;
+	case HALMAC_PWR_CMD_POLLING:
+		polling_bit = 0;
+		polling_count = HALMAC_POLLING_READY_TIMEOUT_COUNT;
+		flag = 0;
+
+		if (sub_seq_cmd->base == HALMAC_PWR_BASEADDR_SDIO)
+			offset = sub_seq_cmd->offset | SDIO_LOCAL_OFFSET;
+		else
+			offset = sub_seq_cmd->offset;
+
+		do {
+			polling_count--;
+			value = HALMAC_REG_READ_8(halmac_adapter, offset);
+			value = (u8)(value & sub_seq_cmd->msk);
+
+			if (value == (sub_seq_cmd->value & sub_seq_cmd->msk)) {
+				polling_bit = 1;
+				continue;
+			}
+
+			if (polling_count != 0) {
+				usleep_range(50, 60);
+				continue;
+			}
+
+			if (halmac_adapter->halmac_interface ==
+				    HALMAC_INTERFACE_PCIE &&
+			    flag == 0) {
+				/* For PCIE + USB package poll power bit
+				 * timeout issue
+				 */
+				poll_to_static++;
+				HALMAC_RT_TRACE(
+					driver_adapter, HALMAC_MSG_PWR,
+					DBG_WARNING,
+					"[WARN]PCIE polling timeout : %d!!\n",
+					poll_to_static);
+				HALMAC_REG_WRITE_8(
+					halmac_adapter, REG_SYS_PW_CTRL,
+					HALMAC_REG_READ_8(halmac_adapter,
+							  REG_SYS_PW_CTRL) |
+						BIT(3));
+				HALMAC_REG_WRITE_8(
+					halmac_adapter, REG_SYS_PW_CTRL,
+					HALMAC_REG_READ_8(halmac_adapter,
+							  REG_SYS_PW_CTRL) &
+						~BIT(3));
+				polling_bit = 0;
+				polling_count =
+					HALMAC_POLLING_READY_TIMEOUT_COUNT;
+				flag = 1;
+			} else {
+				pr_err("[ERR]Pwr cmd polling timeout!!\n");
+				pr_err("[ERR]Pwr cmd offset : %X!!\n",
+				       sub_seq_cmd->offset);
+				pr_err("[ERR]Pwr cmd value : %X!!\n",
+				       sub_seq_cmd->value);
+				pr_err("[ERR]Pwr cmd msk : %X!!\n",
+				       sub_seq_cmd->msk);
+				pr_err("[ERR]Read offset = %X value = %X!!\n",
+				       offset, value);
+				return HALMAC_RET_PWRSEQ_POLLING_FAIL;
+			}
+		} while (!polling_bit);
+		break;
+	case HALMAC_PWR_CMD_DELAY:
+		if (sub_seq_cmd->value == HALMAC_PWRSEQ_DELAY_US)
+			udelay(sub_seq_cmd->offset);
+		else
+			usleep_range(1000 * sub_seq_cmd->offset,
+				     1000 * sub_seq_cmd->offset + 100);
+
+		break;
+	case HALMAC_PWR_CMD_READ:
+		break;
+	case HALMAC_PWR_CMD_END:
+		return HALMAC_RET_SUCCESS;
+	default:
+		return HALMAC_RET_PWRSEQ_CMD_INCORRECT;
+	}
+
+	*reti = false;
+
+	return HALMAC_RET_SUCCESS;
+}
+
+static enum halmac_ret_status
+halmac_pwr_sub_seq_parer_88xx(struct halmac_adapter *halmac_adapter, u8 cut,
+			      u8 fab, u8 intf,
+			      struct halmac_wl_pwr_cfg_ *pwr_sub_seq_cfg)
+{
+	struct halmac_wl_pwr_cfg_ *sub_seq_cmd;
+	bool reti;
+	enum halmac_ret_status status;
+
+	for (sub_seq_cmd = pwr_sub_seq_cfg;; sub_seq_cmd++) {
+		if ((sub_seq_cmd->interface_msk & intf) &&
+		    (sub_seq_cmd->fab_msk & fab) &&
+		    (sub_seq_cmd->cut_msk & cut)) {
+			status = halmac_pwr_sub_seq_parer_do_cmd_88xx(
+				halmac_adapter, sub_seq_cmd, &reti);
+
+			if (reti)
+				return status;
+		}
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_get_h2c_buff_free_space_88xx(struct halmac_adapter *halmac_adapter)
+{
+	u32 hw_wptr, fw_rptr;
+	struct halmac_api *halmac_api =
+		(struct halmac_api *)halmac_adapter->halmac_api;
+
+	hw_wptr = HALMAC_REG_READ_32(halmac_adapter, REG_H2C_PKT_WRITEADDR) &
+		  BIT_MASK_H2C_WR_ADDR;
+	fw_rptr = HALMAC_REG_READ_32(halmac_adapter, REG_H2C_PKT_READADDR) &
+		  BIT_MASK_H2C_READ_ADDR;
+
+	if (hw_wptr >= fw_rptr)
+		halmac_adapter->h2c_buf_free_space =
+			halmac_adapter->h2c_buff_size - (hw_wptr - fw_rptr);
+	else
+		halmac_adapter->h2c_buf_free_space = fw_rptr - hw_wptr;
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_send_h2c_pkt_88xx(struct halmac_adapter *halmac_adapter, u8 *hal_h2c_cmd,
+			 u32 size, bool ack)
+{
+	u32 counter = 100;
+	void *driver_adapter = halmac_adapter->driver_adapter;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	while (halmac_adapter->h2c_buf_free_space <=
+	       HALMAC_H2C_CMD_SIZE_UNIT_88XX) {
+		halmac_get_h2c_buff_free_space_88xx(halmac_adapter);
+		counter--;
+		if (counter == 0) {
+			pr_err("h2c free space is not enough!!\n");
+			return HALMAC_RET_H2C_SPACE_FULL;
+		}
+	}
+
+	/* Send TxDesc + H2C_CMD */
+	if (!PLATFORM_SEND_H2C_PKT(driver_adapter, hal_h2c_cmd, size)) {
+		pr_err("Send H2C_CMD pkt error!!\n");
+		return HALMAC_RET_SEND_H2C_FAIL;
+	}
+
+	halmac_adapter->h2c_buf_free_space -= HALMAC_H2C_CMD_SIZE_UNIT_88XX;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"H2C free space : %d\n",
+			halmac_adapter->h2c_buf_free_space);
+
+	return status;
+}
+
+enum halmac_ret_status
+halmac_download_rsvd_page_88xx(struct halmac_adapter *halmac_adapter,
+			       u8 *hal_buf, u32 size)
+{
+	u8 restore[3];
+	u8 value8;
+	u32 counter;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	if (size == 0) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+				"Rsvd page packet size is zero!!\n");
+		return HALMAC_RET_ZERO_LEN_RSVD_PACKET;
+	}
+
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_FIFOPAGE_CTRL_2 + 1);
+	value8 = (u8)(value8 | BIT(7));
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_FIFOPAGE_CTRL_2 + 1, value8);
+
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_CR + 1);
+	restore[0] = value8;
+	value8 = (u8)(value8 | BIT(0));
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_CR + 1, value8);
+
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_BCN_CTRL);
+	restore[1] = value8;
+	value8 = (u8)((value8 & ~(BIT(3))) | BIT(4));
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_BCN_CTRL, value8);
+
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_FWHW_TXQ_CTRL + 2);
+	restore[2] = value8;
+	value8 = (u8)(value8 & ~(BIT(6)));
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_FWHW_TXQ_CTRL + 2, value8);
+
+	if (!PLATFORM_SEND_RSVD_PAGE(driver_adapter, hal_buf, size)) {
+		pr_err("PLATFORM_SEND_RSVD_PAGE 1 error!!\n");
+		status = HALMAC_RET_DL_RSVD_PAGE_FAIL;
+	}
+
+	/* Check Bcn_Valid_Bit */
+	counter = 1000;
+	while (!(HALMAC_REG_READ_8(halmac_adapter, REG_FIFOPAGE_CTRL_2 + 1) &
+		 BIT(7))) {
+		udelay(10);
+		counter--;
+		if (counter == 0) {
+			pr_err("Polling Bcn_Valid_Fail error!!\n");
+			status = HALMAC_RET_POLLING_BCN_VALID_FAIL;
+			break;
+		}
+	}
+
+	value8 = HALMAC_REG_READ_8(halmac_adapter, REG_FIFOPAGE_CTRL_2 + 1);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_FIFOPAGE_CTRL_2 + 1,
+			   (value8 | BIT(7)));
+
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_FWHW_TXQ_CTRL + 2, restore[2]);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_BCN_CTRL, restore[1]);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_CR + 1, restore[0]);
+
+	return status;
+}
+
+enum halmac_ret_status
+halmac_set_h2c_header_88xx(struct halmac_adapter *halmac_adapter,
+			   u8 *hal_h2c_hdr, u16 *seq, bool ack)
+{
+	void *driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s!!\n", __func__);
+
+	H2C_CMD_HEADER_SET_CATEGORY(hal_h2c_hdr, 0x00);
+	H2C_CMD_HEADER_SET_TOTAL_LEN(hal_h2c_hdr, 16);
+
+	spin_lock(&halmac_adapter->h2c_seq_lock);
+	H2C_CMD_HEADER_SET_SEQ_NUM(hal_h2c_hdr, halmac_adapter->h2c_packet_seq);
+	*seq = halmac_adapter->h2c_packet_seq;
+	halmac_adapter->h2c_packet_seq++;
+	spin_unlock(&halmac_adapter->h2c_seq_lock);
+
+	if (ack)
+		H2C_CMD_HEADER_SET_ACK(hal_h2c_hdr, 1);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status halmac_set_fw_offload_h2c_header_88xx(
+	struct halmac_adapter *halmac_adapter, u8 *hal_h2c_hdr,
+	struct halmac_h2c_header_info *h2c_header_info, u16 *seq_num)
+{
+	void *driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s!!\n", __func__);
+
+	FW_OFFLOAD_H2C_SET_TOTAL_LEN(hal_h2c_hdr,
+				     8 + h2c_header_info->content_size);
+	FW_OFFLOAD_H2C_SET_SUB_CMD_ID(hal_h2c_hdr, h2c_header_info->sub_cmd_id);
+
+	FW_OFFLOAD_H2C_SET_CATEGORY(hal_h2c_hdr, 0x01);
+	FW_OFFLOAD_H2C_SET_CMD_ID(hal_h2c_hdr, 0xFF);
+
+	spin_lock(&halmac_adapter->h2c_seq_lock);
+	FW_OFFLOAD_H2C_SET_SEQ_NUM(hal_h2c_hdr, halmac_adapter->h2c_packet_seq);
+	*seq_num = halmac_adapter->h2c_packet_seq;
+	halmac_adapter->h2c_packet_seq++;
+	spin_unlock(&halmac_adapter->h2c_seq_lock);
+
+	if (h2c_header_info->ack)
+		FW_OFFLOAD_H2C_SET_ACK(hal_h2c_hdr, 1);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_send_h2c_set_pwr_mode_88xx(struct halmac_adapter *halmac_adapter,
+				  struct halmac_fwlps_option *hal_fw_lps_opt)
+{
+	u8 h2c_buff[HALMAC_H2C_CMD_SIZE_88XX];
+	u8 *h2c_header, *h2c_cmd;
+	u16 seq = 0;
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s!!\n", __func__);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	h2c_header = h2c_buff;
+	h2c_cmd = h2c_header + HALMAC_H2C_CMD_HDR_SIZE_88XX;
+
+	memset(h2c_buff, 0x00, HALMAC_H2C_CMD_SIZE_88XX);
+
+	SET_PWR_MODE_SET_CMD_ID(h2c_cmd, CMD_ID_SET_PWR_MODE);
+	SET_PWR_MODE_SET_CLASS(h2c_cmd, CLASS_SET_PWR_MODE);
+	SET_PWR_MODE_SET_MODE(h2c_cmd, hal_fw_lps_opt->mode);
+	SET_PWR_MODE_SET_CLK_REQUEST(h2c_cmd, hal_fw_lps_opt->clk_request);
+	SET_PWR_MODE_SET_RLBM(h2c_cmd, hal_fw_lps_opt->rlbm);
+	SET_PWR_MODE_SET_SMART_PS(h2c_cmd, hal_fw_lps_opt->smart_ps);
+	SET_PWR_MODE_SET_AWAKE_INTERVAL(h2c_cmd,
+					hal_fw_lps_opt->awake_interval);
+	SET_PWR_MODE_SET_B_ALL_QUEUE_UAPSD(h2c_cmd,
+					   hal_fw_lps_opt->all_queue_uapsd);
+	SET_PWR_MODE_SET_PWR_STATE(h2c_cmd, hal_fw_lps_opt->pwr_state);
+	SET_PWR_MODE_SET_ANT_AUTO_SWITCH(h2c_cmd,
+					 hal_fw_lps_opt->ant_auto_switch);
+	SET_PWR_MODE_SET_PS_ALLOW_BT_HIGH_PRIORITY(
+		h2c_cmd, hal_fw_lps_opt->ps_allow_bt_high_priority);
+	SET_PWR_MODE_SET_PROTECT_BCN(h2c_cmd, hal_fw_lps_opt->protect_bcn);
+	SET_PWR_MODE_SET_SILENCE_PERIOD(h2c_cmd,
+					hal_fw_lps_opt->silence_period);
+	SET_PWR_MODE_SET_FAST_BT_CONNECT(h2c_cmd,
+					 hal_fw_lps_opt->fast_bt_connect);
+	SET_PWR_MODE_SET_TWO_ANTENNA_EN(h2c_cmd,
+					hal_fw_lps_opt->two_antenna_en);
+	SET_PWR_MODE_SET_ADOPT_USER_SETTING(h2c_cmd,
+					    hal_fw_lps_opt->adopt_user_setting);
+	SET_PWR_MODE_SET_DRV_BCN_EARLY_SHIFT(
+		h2c_cmd, hal_fw_lps_opt->drv_bcn_early_shift);
+
+	halmac_set_h2c_header_88xx(halmac_adapter, h2c_header, &seq, true);
+
+	status = halmac_send_h2c_pkt_88xx(halmac_adapter, h2c_buff,
+					  HALMAC_H2C_CMD_SIZE_88XX, true);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("%s Fail = %x!!\n", __func__, status);
+		return status;
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_func_send_original_h2c_88xx(struct halmac_adapter *halmac_adapter,
+				   u8 *original_h2c, u16 *seq, u8 ack)
+{
+	u8 H2c_buff[HALMAC_H2C_CMD_SIZE_88XX] = {0};
+	u8 *h2c_header, *h2c_cmd;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"halmac_send_original_h2c ==========>\n");
+
+	h2c_header = H2c_buff;
+	h2c_cmd = h2c_header + HALMAC_H2C_CMD_HDR_SIZE_88XX;
+	memcpy(h2c_cmd, original_h2c, 8); /* Original H2C 8 byte */
+
+	halmac_set_h2c_header_88xx(halmac_adapter, h2c_header, seq, ack);
+
+	status = halmac_send_h2c_pkt_88xx(halmac_adapter, H2c_buff,
+					  HALMAC_H2C_CMD_SIZE_88XX, ack);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_send_original_h2c Fail = %x!!\n", status);
+		return status;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"halmac_send_original_h2c <==========\n");
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_media_status_rpt_88xx(struct halmac_adapter *halmac_adapter, u8 op_mode,
+			     u8 mac_id_ind, u8 mac_id, u8 mac_id_end)
+{
+	u8 H2c_buff[HALMAC_H2C_CMD_SIZE_88XX] = {0};
+	u8 *h2c_header, *h2c_cmd;
+	u16 seq = 0;
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"halmac_send_h2c_set_pwr_mode_88xx!!\n");
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	h2c_header = H2c_buff;
+	h2c_cmd = h2c_header + HALMAC_H2C_CMD_HDR_SIZE_88XX;
+
+	memset(H2c_buff, 0x00, HALMAC_H2C_CMD_SIZE_88XX);
+
+	MEDIA_STATUS_RPT_SET_CMD_ID(h2c_cmd, CMD_ID_MEDIA_STATUS_RPT);
+	MEDIA_STATUS_RPT_SET_CLASS(h2c_cmd, CLASS_MEDIA_STATUS_RPT);
+	MEDIA_STATUS_RPT_SET_OP_MODE(h2c_cmd, op_mode);
+	MEDIA_STATUS_RPT_SET_MACID_IN(h2c_cmd, mac_id_ind);
+	MEDIA_STATUS_RPT_SET_MACID(h2c_cmd, mac_id);
+	MEDIA_STATUS_RPT_SET_MACID_END(h2c_cmd, mac_id_end);
+
+	halmac_set_h2c_header_88xx(halmac_adapter, h2c_header, &seq, true);
+
+	status = halmac_send_h2c_pkt_88xx(halmac_adapter, H2c_buff,
+					  HALMAC_H2C_CMD_SIZE_88XX, true);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("%s Fail = %x!!\n", __func__, status);
+		return status;
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_send_h2c_update_packet_88xx(struct halmac_adapter *halmac_adapter,
+				   enum halmac_packet_id pkt_id, u8 *pkt,
+				   u32 pkt_size)
+{
+	u8 h2c_buff[HALMAC_H2C_CMD_SIZE_88XX] = {0};
+	u16 h2c_seq_mum = 0;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	struct halmac_h2c_header_info h2c_header_info;
+	enum halmac_ret_status ret_status = HALMAC_RET_SUCCESS;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_FIFOPAGE_CTRL_2,
+			    (u16)(halmac_adapter->txff_allocation
+					  .rsvd_h2c_extra_info_pg_bndy &
+				  BIT_MASK_BCN_HEAD_1_V1));
+
+	ret_status =
+		halmac_download_rsvd_page_88xx(halmac_adapter, pkt, pkt_size);
+
+	if (ret_status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_download_rsvd_page_88xx Fail = %x!!\n",
+		       ret_status);
+		HALMAC_REG_WRITE_16(
+			halmac_adapter, REG_FIFOPAGE_CTRL_2,
+			(u16)(halmac_adapter->txff_allocation.rsvd_pg_bndy &
+			      BIT_MASK_BCN_HEAD_1_V1));
+		return ret_status;
+	}
+
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_FIFOPAGE_CTRL_2,
+			    (u16)(halmac_adapter->txff_allocation.rsvd_pg_bndy &
+				  BIT_MASK_BCN_HEAD_1_V1));
+
+	UPDATE_PACKET_SET_SIZE(
+		h2c_buff,
+		pkt_size + halmac_adapter->hw_config_info.txdesc_size);
+	UPDATE_PACKET_SET_PACKET_ID(h2c_buff, pkt_id);
+	UPDATE_PACKET_SET_PACKET_LOC(
+		h2c_buff,
+		halmac_adapter->txff_allocation.rsvd_h2c_extra_info_pg_bndy -
+			halmac_adapter->txff_allocation.rsvd_pg_bndy);
+
+	h2c_header_info.sub_cmd_id = SUB_CMD_ID_UPDATE_PACKET;
+	h2c_header_info.content_size = 8;
+	h2c_header_info.ack = true;
+	halmac_set_fw_offload_h2c_header_88xx(halmac_adapter, h2c_buff,
+					      &h2c_header_info, &h2c_seq_mum);
+	halmac_adapter->halmac_state.update_packet_set.seq_num = h2c_seq_mum;
+
+	ret_status = halmac_send_h2c_pkt_88xx(halmac_adapter, h2c_buff,
+					      HALMAC_H2C_CMD_SIZE_88XX, true);
+
+	if (ret_status != HALMAC_RET_SUCCESS) {
+		pr_err("%s Fail = %x!!\n", __func__, ret_status);
+		return ret_status;
+	}
+
+	return ret_status;
+}
+
+enum halmac_ret_status
+halmac_send_h2c_phy_parameter_88xx(struct halmac_adapter *halmac_adapter,
+				   struct halmac_phy_parameter_info *para_info,
+				   bool full_fifo)
+{
+	bool drv_trigger_send = false;
+	u8 h2c_buff[HALMAC_H2C_CMD_SIZE_88XX] = {0};
+	u16 h2c_seq_mum = 0;
+	u32 info_size = 0;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	struct halmac_h2c_header_info h2c_header_info;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+	struct halmac_config_para_info *config_para_info;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+	config_para_info = &halmac_adapter->config_para_info;
+
+	if (!config_para_info->cfg_para_buf) {
+		if (full_fifo)
+			config_para_info->para_buf_size =
+				HALMAC_EXTRA_INFO_BUFF_SIZE_FULL_FIFO_88XX;
+		else
+			config_para_info->para_buf_size =
+				HALMAC_EXTRA_INFO_BUFF_SIZE_88XX;
+
+		config_para_info->cfg_para_buf =
+			kzalloc(config_para_info->para_buf_size, GFP_KERNEL);
+
+		if (config_para_info->cfg_para_buf) {
+			memset(config_para_info->cfg_para_buf, 0x00,
+			       config_para_info->para_buf_size);
+			config_para_info->full_fifo_mode = full_fifo;
+			config_para_info->para_buf_w =
+				config_para_info->cfg_para_buf;
+			config_para_info->para_num = 0;
+			config_para_info->avai_para_buf_size =
+				config_para_info->para_buf_size;
+			config_para_info->value_accumulation = 0;
+			config_para_info->offset_accumulation = 0;
+		} else {
+			HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C,
+					DBG_DMESG,
+					"Allocate cfg_para_buf fail!!\n");
+			return HALMAC_RET_MALLOC_FAIL;
+		}
+	}
+
+	if (halmac_transition_cfg_para_state_88xx(
+		    halmac_adapter,
+		    HALMAC_CFG_PARA_CMD_CONSTRUCT_CONSTRUCTING) !=
+	    HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ERROR_STATE;
+
+	halmac_enqueue_para_buff_88xx(halmac_adapter, para_info,
+				      config_para_info->para_buf_w,
+				      &drv_trigger_send);
+
+	if (para_info->cmd_id != HALMAC_PARAMETER_CMD_END) {
+		config_para_info->para_num++;
+		config_para_info->para_buf_w += HALMAC_FW_OFFLOAD_CMD_SIZE_88XX;
+		config_para_info->avai_para_buf_size =
+			config_para_info->avai_para_buf_size -
+			HALMAC_FW_OFFLOAD_CMD_SIZE_88XX;
+	}
+
+	if ((config_para_info->avai_para_buf_size -
+	     halmac_adapter->hw_config_info.txdesc_size) >
+		    HALMAC_FW_OFFLOAD_CMD_SIZE_88XX &&
+	    !drv_trigger_send)
+		return HALMAC_RET_SUCCESS;
+
+	if (config_para_info->para_num == 0) {
+		kfree(config_para_info->cfg_para_buf);
+		config_para_info->cfg_para_buf = NULL;
+		config_para_info->para_buf_w = NULL;
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_WARNING,
+				"no cfg parameter element!!\n");
+
+		if (halmac_transition_cfg_para_state_88xx(
+			    halmac_adapter,
+			    HALMAC_CFG_PARA_CMD_CONSTRUCT_IDLE) !=
+		    HALMAC_RET_SUCCESS)
+			return HALMAC_RET_ERROR_STATE;
+
+		return HALMAC_RET_SUCCESS;
+	}
+
+	if (halmac_transition_cfg_para_state_88xx(
+		    halmac_adapter, HALMAC_CFG_PARA_CMD_CONSTRUCT_H2C_SENT) !=
+	    HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ERROR_STATE;
+
+	halmac_adapter->halmac_state.cfg_para_state_set.process_status =
+		HALMAC_CMD_PROCESS_SENDING;
+
+	if (config_para_info->full_fifo_mode)
+		HALMAC_REG_WRITE_16(halmac_adapter, REG_FIFOPAGE_CTRL_2, 0);
+	else
+		HALMAC_REG_WRITE_16(halmac_adapter, REG_FIFOPAGE_CTRL_2,
+				    (u16)(halmac_adapter->txff_allocation
+						  .rsvd_h2c_extra_info_pg_bndy &
+					  BIT_MASK_BCN_HEAD_1_V1));
+
+	info_size =
+		config_para_info->para_num * HALMAC_FW_OFFLOAD_CMD_SIZE_88XX;
+
+	status = halmac_download_rsvd_page_88xx(
+		halmac_adapter, (u8 *)config_para_info->cfg_para_buf,
+		info_size);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_download_rsvd_page_88xx Fail!!\n");
+	} else {
+		halmac_gen_cfg_para_h2c_88xx(halmac_adapter, h2c_buff);
+
+		h2c_header_info.sub_cmd_id = SUB_CMD_ID_CFG_PARAMETER;
+		h2c_header_info.content_size = 4;
+		h2c_header_info.ack = true;
+		halmac_set_fw_offload_h2c_header_88xx(halmac_adapter, h2c_buff,
+						      &h2c_header_info,
+						      &h2c_seq_mum);
+
+		halmac_adapter->halmac_state.cfg_para_state_set.seq_num =
+			h2c_seq_mum;
+
+		status = halmac_send_h2c_pkt_88xx(halmac_adapter, h2c_buff,
+						  HALMAC_H2C_CMD_SIZE_88XX,
+						  true);
+
+		if (status != HALMAC_RET_SUCCESS)
+			pr_err("halmac_send_h2c_pkt_88xx Fail!!\n");
+
+		HALMAC_RT_TRACE(
+			driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"config parameter time = %d\n",
+			HALMAC_REG_READ_32(halmac_adapter, REG_FW_DBG6));
+	}
+
+	kfree(config_para_info->cfg_para_buf);
+	config_para_info->cfg_para_buf = NULL;
+	config_para_info->para_buf_w = NULL;
+
+	/* Restore bcn head */
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_FIFOPAGE_CTRL_2,
+			    (u16)(halmac_adapter->txff_allocation.rsvd_pg_bndy &
+				  BIT_MASK_BCN_HEAD_1_V1));
+
+	if (halmac_transition_cfg_para_state_88xx(
+		    halmac_adapter, HALMAC_CFG_PARA_CMD_CONSTRUCT_IDLE) !=
+	    HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ERROR_STATE;
+
+	if (!drv_trigger_send) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+				"Buffer full trigger sending H2C!!\n");
+		return HALMAC_RET_PARA_SENDING;
+	}
+
+	return status;
+}
+
+static enum halmac_ret_status
+halmac_enqueue_para_buff_88xx(struct halmac_adapter *halmac_adapter,
+			      struct halmac_phy_parameter_info *para_info,
+			      u8 *curr_buff_wptr, bool *end_cmd)
+{
+	struct halmac_config_para_info *config_para_info =
+		&halmac_adapter->config_para_info;
+
+	*end_cmd = false;
+
+	PHY_PARAMETER_INFO_SET_LENGTH(curr_buff_wptr,
+				      HALMAC_FW_OFFLOAD_CMD_SIZE_88XX);
+	PHY_PARAMETER_INFO_SET_IO_CMD(curr_buff_wptr, para_info->cmd_id);
+
+	switch (para_info->cmd_id) {
+	case HALMAC_PARAMETER_CMD_BB_W8:
+	case HALMAC_PARAMETER_CMD_BB_W16:
+	case HALMAC_PARAMETER_CMD_BB_W32:
+	case HALMAC_PARAMETER_CMD_MAC_W8:
+	case HALMAC_PARAMETER_CMD_MAC_W16:
+	case HALMAC_PARAMETER_CMD_MAC_W32:
+		PHY_PARAMETER_INFO_SET_IO_ADDR(
+			curr_buff_wptr, para_info->content.MAC_REG_W.offset);
+		PHY_PARAMETER_INFO_SET_DATA(curr_buff_wptr,
+					    para_info->content.MAC_REG_W.value);
+		PHY_PARAMETER_INFO_SET_MASK(curr_buff_wptr,
+					    para_info->content.MAC_REG_W.msk);
+		PHY_PARAMETER_INFO_SET_MSK_EN(
+			curr_buff_wptr, para_info->content.MAC_REG_W.msk_en);
+		config_para_info->value_accumulation +=
+			para_info->content.MAC_REG_W.value;
+		config_para_info->offset_accumulation +=
+			para_info->content.MAC_REG_W.offset;
+		break;
+	case HALMAC_PARAMETER_CMD_RF_W:
+		/*In rf register, the address is only 1 byte*/
+		PHY_PARAMETER_INFO_SET_RF_ADDR(
+			curr_buff_wptr, para_info->content.RF_REG_W.offset);
+		PHY_PARAMETER_INFO_SET_RF_PATH(
+			curr_buff_wptr, para_info->content.RF_REG_W.rf_path);
+		PHY_PARAMETER_INFO_SET_DATA(curr_buff_wptr,
+					    para_info->content.RF_REG_W.value);
+		PHY_PARAMETER_INFO_SET_MASK(curr_buff_wptr,
+					    para_info->content.RF_REG_W.msk);
+		PHY_PARAMETER_INFO_SET_MSK_EN(
+			curr_buff_wptr, para_info->content.RF_REG_W.msk_en);
+		config_para_info->value_accumulation +=
+			para_info->content.RF_REG_W.value;
+		config_para_info->offset_accumulation +=
+			(para_info->content.RF_REG_W.offset +
+			 (para_info->content.RF_REG_W.rf_path << 8));
+		break;
+	case HALMAC_PARAMETER_CMD_DELAY_US:
+	case HALMAC_PARAMETER_CMD_DELAY_MS:
+		PHY_PARAMETER_INFO_SET_DELAY_VALUE(
+			curr_buff_wptr,
+			para_info->content.DELAY_TIME.delay_time);
+		break;
+	case HALMAC_PARAMETER_CMD_END:
+		*end_cmd = true;
+		break;
+	default:
+		pr_err(" halmac_send_h2c_phy_parameter_88xx illegal cmd_id!!\n");
+		break;
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+static enum halmac_ret_status
+halmac_gen_cfg_para_h2c_88xx(struct halmac_adapter *halmac_adapter,
+			     u8 *h2c_buff)
+{
+	struct halmac_config_para_info *config_para_info =
+		&halmac_adapter->config_para_info;
+
+	CFG_PARAMETER_SET_NUM(h2c_buff, config_para_info->para_num);
+
+	if (config_para_info->full_fifo_mode) {
+		CFG_PARAMETER_SET_INIT_CASE(h2c_buff, 0x1);
+		CFG_PARAMETER_SET_PHY_PARAMETER_LOC(h2c_buff, 0);
+	} else {
+		CFG_PARAMETER_SET_INIT_CASE(h2c_buff, 0x0);
+		CFG_PARAMETER_SET_PHY_PARAMETER_LOC(
+			h2c_buff,
+			halmac_adapter->txff_allocation
+					.rsvd_h2c_extra_info_pg_bndy -
+				halmac_adapter->txff_allocation.rsvd_pg_bndy);
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_send_h2c_run_datapack_88xx(struct halmac_adapter *halmac_adapter,
+				  enum halmac_data_type halmac_data_type)
+{
+	u8 h2c_buff[HALMAC_H2C_CMD_SIZE_88XX] = {0};
+	u16 h2c_seq_mum = 0;
+	void *driver_adapter = NULL;
+	struct halmac_h2c_header_info h2c_header_info;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s!!\n", __func__);
+
+	RUN_DATAPACK_SET_DATAPACK_ID(h2c_buff, halmac_data_type);
+
+	h2c_header_info.sub_cmd_id = SUB_CMD_ID_RUN_DATAPACK;
+	h2c_header_info.content_size = 4;
+	h2c_header_info.ack = true;
+	halmac_set_fw_offload_h2c_header_88xx(halmac_adapter, h2c_buff,
+					      &h2c_header_info, &h2c_seq_mum);
+
+	status = halmac_send_h2c_pkt_88xx(halmac_adapter, h2c_buff,
+					  HALMAC_H2C_CMD_SIZE_88XX, true);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_send_h2c_pkt_88xx Fail = %x!!\n", status);
+		return status;
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_send_bt_coex_cmd_88xx(struct halmac_adapter *halmac_adapter, u8 *bt_buf,
+			     u32 bt_size, u8 ack)
+{
+	u8 h2c_buff[HALMAC_H2C_CMD_SIZE_88XX] = {0};
+	u16 h2c_seq_mum = 0;
+	void *driver_adapter = NULL;
+	struct halmac_h2c_header_info h2c_header_info;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s!!\n", __func__);
+
+	memcpy(h2c_buff + 8, bt_buf, bt_size);
+
+	h2c_header_info.sub_cmd_id = SUB_CMD_ID_BT_COEX;
+	h2c_header_info.content_size = (u16)bt_size;
+	h2c_header_info.ack = ack;
+	halmac_set_fw_offload_h2c_header_88xx(halmac_adapter, h2c_buff,
+					      &h2c_header_info, &h2c_seq_mum);
+
+	status = halmac_send_h2c_pkt_88xx(halmac_adapter, h2c_buff,
+					  HALMAC_H2C_CMD_SIZE_88XX, ack);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_send_h2c_pkt_88xx Fail = %x!!\n", status);
+		return status;
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_func_ctrl_ch_switch_88xx(struct halmac_adapter *halmac_adapter,
+				struct halmac_ch_switch_option *cs_option)
+{
+	u8 h2c_buff[HALMAC_H2C_CMD_SIZE_88XX] = {0};
+	u16 h2c_seq_mum = 0;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	struct halmac_h2c_header_info h2c_header_info;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+	enum halmac_cmd_process_status *process_status =
+		&halmac_adapter->halmac_state.scan_state_set.process_status;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"halmac_ctrl_ch_switch!!\n");
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	if (halmac_transition_scan_state_88xx(
+		    halmac_adapter, HALMAC_SCAN_CMD_CONSTRUCT_H2C_SENT) !=
+	    HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ERROR_STATE;
+
+	*process_status = HALMAC_CMD_PROCESS_SENDING;
+
+	if (cs_option->switch_en != 0) {
+		HALMAC_REG_WRITE_16(halmac_adapter, REG_FIFOPAGE_CTRL_2,
+				    (u16)(halmac_adapter->txff_allocation
+						  .rsvd_h2c_extra_info_pg_bndy &
+					  BIT_MASK_BCN_HEAD_1_V1));
+
+		status = halmac_download_rsvd_page_88xx(
+			halmac_adapter, halmac_adapter->ch_sw_info.ch_info_buf,
+			halmac_adapter->ch_sw_info.total_size);
+
+		if (status != HALMAC_RET_SUCCESS) {
+			pr_err("halmac_download_rsvd_page_88xx Fail = %x!!\n",
+			       status);
+			HALMAC_REG_WRITE_16(
+				halmac_adapter, REG_FIFOPAGE_CTRL_2,
+				(u16)(halmac_adapter->txff_allocation
+					      .rsvd_pg_bndy &
+				      BIT_MASK_BCN_HEAD_1_V1));
+			return status;
+		}
+
+		HALMAC_REG_WRITE_16(
+			halmac_adapter, REG_FIFOPAGE_CTRL_2,
+			(u16)(halmac_adapter->txff_allocation.rsvd_pg_bndy &
+			      BIT_MASK_BCN_HEAD_1_V1));
+	}
+
+	CHANNEL_SWITCH_SET_SWITCH_START(h2c_buff, cs_option->switch_en);
+	CHANNEL_SWITCH_SET_CHANNEL_NUM(h2c_buff,
+				       halmac_adapter->ch_sw_info.ch_num);
+	CHANNEL_SWITCH_SET_CHANNEL_INFO_LOC(
+		h2c_buff,
+		halmac_adapter->txff_allocation.rsvd_h2c_extra_info_pg_bndy -
+			halmac_adapter->txff_allocation.rsvd_pg_bndy);
+	CHANNEL_SWITCH_SET_DEST_CH_EN(h2c_buff, cs_option->dest_ch_en);
+	CHANNEL_SWITCH_SET_DEST_CH(h2c_buff, cs_option->dest_ch);
+	CHANNEL_SWITCH_SET_PRI_CH_IDX(h2c_buff, cs_option->dest_pri_ch_idx);
+	CHANNEL_SWITCH_SET_ABSOLUTE_TIME(h2c_buff, cs_option->absolute_time_en);
+	CHANNEL_SWITCH_SET_TSF_LOW(h2c_buff, cs_option->tsf_low);
+	CHANNEL_SWITCH_SET_PERIODIC_OPTION(h2c_buff,
+					   cs_option->periodic_option);
+	CHANNEL_SWITCH_SET_NORMAL_CYCLE(h2c_buff, cs_option->normal_cycle);
+	CHANNEL_SWITCH_SET_NORMAL_PERIOD(h2c_buff, cs_option->normal_period);
+	CHANNEL_SWITCH_SET_SLOW_PERIOD(h2c_buff, cs_option->phase_2_period);
+	CHANNEL_SWITCH_SET_CHANNEL_INFO_SIZE(
+		h2c_buff, halmac_adapter->ch_sw_info.total_size);
+
+	h2c_header_info.sub_cmd_id = SUB_CMD_ID_CHANNEL_SWITCH;
+	h2c_header_info.content_size = 20;
+	h2c_header_info.ack = true;
+	halmac_set_fw_offload_h2c_header_88xx(halmac_adapter, h2c_buff,
+					      &h2c_header_info, &h2c_seq_mum);
+	halmac_adapter->halmac_state.scan_state_set.seq_num = h2c_seq_mum;
+
+	status = halmac_send_h2c_pkt_88xx(halmac_adapter, h2c_buff,
+					  HALMAC_H2C_CMD_SIZE_88XX, true);
+
+	if (status != HALMAC_RET_SUCCESS)
+		pr_err("halmac_send_h2c_pkt_88xx Fail = %x!!\n", status);
+
+	kfree(halmac_adapter->ch_sw_info.ch_info_buf);
+	halmac_adapter->ch_sw_info.ch_info_buf = NULL;
+	halmac_adapter->ch_sw_info.ch_info_buf_w = NULL;
+	halmac_adapter->ch_sw_info.extra_info_en = 0;
+	halmac_adapter->ch_sw_info.buf_size = 0;
+	halmac_adapter->ch_sw_info.avai_buf_size = 0;
+	halmac_adapter->ch_sw_info.total_size = 0;
+	halmac_adapter->ch_sw_info.ch_num = 0;
+
+	if (halmac_transition_scan_state_88xx(halmac_adapter,
+					      HALMAC_SCAN_CMD_CONSTRUCT_IDLE) !=
+	    HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ERROR_STATE;
+
+	return status;
+}
+
+enum halmac_ret_status
+halmac_func_send_general_info_88xx(struct halmac_adapter *halmac_adapter,
+				   struct halmac_general_info *general_info)
+{
+	u8 h2c_buff[HALMAC_H2C_CMD_SIZE_88XX] = {0};
+	u16 h2c_seq_mum = 0;
+	void *driver_adapter = NULL;
+	struct halmac_h2c_header_info h2c_header_info;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"halmac_send_general_info!!\n");
+
+	GENERAL_INFO_SET_REF_TYPE(h2c_buff, general_info->rfe_type);
+	GENERAL_INFO_SET_RF_TYPE(h2c_buff, general_info->rf_type);
+	GENERAL_INFO_SET_FW_TX_BOUNDARY(
+		h2c_buff,
+		halmac_adapter->txff_allocation.rsvd_fw_txbuff_pg_bndy -
+			halmac_adapter->txff_allocation.rsvd_pg_bndy);
+
+	h2c_header_info.sub_cmd_id = SUB_CMD_ID_GENERAL_INFO;
+	h2c_header_info.content_size = 4;
+	h2c_header_info.ack = false;
+	halmac_set_fw_offload_h2c_header_88xx(halmac_adapter, h2c_buff,
+					      &h2c_header_info, &h2c_seq_mum);
+
+	status = halmac_send_h2c_pkt_88xx(halmac_adapter, h2c_buff,
+					  HALMAC_H2C_CMD_SIZE_88XX, true);
+
+	if (status != HALMAC_RET_SUCCESS)
+		pr_err("halmac_send_h2c_pkt_88xx Fail = %x!!\n", status);
+
+	return status;
+}
+
+enum halmac_ret_status halmac_send_h2c_update_bcn_parse_info_88xx(
+	struct halmac_adapter *halmac_adapter,
+	struct halmac_bcn_ie_info *bcn_ie_info)
+{
+	u8 h2c_buff[HALMAC_H2C_CMD_SIZE_88XX] = {0};
+	u16 h2c_seq_mum = 0;
+	void *driver_adapter = NULL;
+	struct halmac_h2c_header_info h2c_header_info;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s!!\n", __func__);
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	UPDATE_BEACON_PARSING_INFO_SET_FUNC_EN(h2c_buff, bcn_ie_info->func_en);
+	UPDATE_BEACON_PARSING_INFO_SET_SIZE_TH(h2c_buff, bcn_ie_info->size_th);
+	UPDATE_BEACON_PARSING_INFO_SET_TIMEOUT(h2c_buff, bcn_ie_info->timeout);
+
+	UPDATE_BEACON_PARSING_INFO_SET_IE_ID_BMP_0(
+		h2c_buff, (u32)(bcn_ie_info->ie_bmp[0]));
+	UPDATE_BEACON_PARSING_INFO_SET_IE_ID_BMP_1(
+		h2c_buff, (u32)(bcn_ie_info->ie_bmp[1]));
+	UPDATE_BEACON_PARSING_INFO_SET_IE_ID_BMP_2(
+		h2c_buff, (u32)(bcn_ie_info->ie_bmp[2]));
+	UPDATE_BEACON_PARSING_INFO_SET_IE_ID_BMP_3(
+		h2c_buff, (u32)(bcn_ie_info->ie_bmp[3]));
+	UPDATE_BEACON_PARSING_INFO_SET_IE_ID_BMP_4(
+		h2c_buff, (u32)(bcn_ie_info->ie_bmp[4]));
+
+	h2c_header_info.sub_cmd_id = SUB_CMD_ID_UPDATE_BEACON_PARSING_INFO;
+	h2c_header_info.content_size = 24;
+	h2c_header_info.ack = true;
+	halmac_set_fw_offload_h2c_header_88xx(halmac_adapter, h2c_buff,
+					      &h2c_header_info, &h2c_seq_mum);
+
+	status = halmac_send_h2c_pkt_88xx(halmac_adapter, h2c_buff,
+					  HALMAC_H2C_CMD_SIZE_88XX, true);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_send_h2c_pkt_88xx Fail =%x !!\n", status);
+		return status;
+	}
+
+	return status;
+}
+
+enum halmac_ret_status
+halmac_send_h2c_ps_tuning_para_88xx(struct halmac_adapter *halmac_adapter)
+{
+	u8 h2c_buff[HALMAC_H2C_CMD_SIZE_88XX] = {0};
+	u8 *h2c_header, *h2c_cmd;
+	u16 seq = 0;
+	void *driver_adapter = NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"%s!!\n", __func__);
+
+	h2c_header = h2c_buff;
+	h2c_cmd = h2c_header + HALMAC_H2C_CMD_HDR_SIZE_88XX;
+
+	halmac_set_h2c_header_88xx(halmac_adapter, h2c_header, &seq, false);
+
+	status = halmac_send_h2c_pkt_88xx(halmac_adapter, h2c_buff,
+					  HALMAC_H2C_CMD_SIZE_88XX, false);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_send_h2c_pkt_88xx Fail = %x!!\n", status);
+		return status;
+	}
+
+	return status;
+}
+
+enum halmac_ret_status
+halmac_parse_c2h_packet_88xx(struct halmac_adapter *halmac_adapter,
+			     u8 *halmac_buf, u32 halmac_size)
+{
+	u8 c2h_cmd, c2h_sub_cmd_id;
+	u8 *c2h_buf = halmac_buf + halmac_adapter->hw_config_info.rxdesc_size;
+	u32 c2h_size = halmac_size - halmac_adapter->hw_config_info.rxdesc_size;
+	void *driver_adapter = halmac_adapter->driver_adapter;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	c2h_cmd = (u8)C2H_HDR_GET_CMD_ID(c2h_buf);
+
+	/* FW offload C2H cmd is 0xFF */
+	if (c2h_cmd != 0xFF) {
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+				"C2H_PKT not for FwOffloadC2HFormat!!\n");
+		return HALMAC_RET_C2H_NOT_HANDLED;
+	}
+
+	/* Get C2H sub cmd ID */
+	c2h_sub_cmd_id = (u8)C2H_HDR_GET_C2H_SUB_CMD_ID(c2h_buf);
+
+	switch (c2h_sub_cmd_id) {
+	case C2H_SUB_CMD_ID_C2H_DBG:
+		status = halmac_parse_c2h_debug_88xx(halmac_adapter, c2h_buf,
+						     c2h_size);
+		break;
+	case C2H_SUB_CMD_ID_H2C_ACK_HDR:
+		status = halmac_parse_h2c_ack_88xx(halmac_adapter, c2h_buf,
+						   c2h_size);
+		break;
+	case C2H_SUB_CMD_ID_BT_COEX_INFO:
+		status = HALMAC_RET_C2H_NOT_HANDLED;
+		break;
+	case C2H_SUB_CMD_ID_SCAN_STATUS_RPT:
+		status = halmac_parse_scan_status_rpt_88xx(halmac_adapter,
+							   c2h_buf, c2h_size);
+		break;
+	case C2H_SUB_CMD_ID_PSD_DATA:
+		status = halmac_parse_psd_data_88xx(halmac_adapter, c2h_buf,
+						    c2h_size);
+		break;
+
+	case C2H_SUB_CMD_ID_EFUSE_DATA:
+		status = halmac_parse_efuse_data_88xx(halmac_adapter, c2h_buf,
+						      c2h_size);
+		break;
+	default:
+		pr_err("c2h_sub_cmd_id switch case out of boundary!!\n");
+		pr_err("[ERR]c2h pkt : %.8X %.8X!!\n", *(u32 *)c2h_buf,
+		       *(u32 *)(c2h_buf + 4));
+		status = HALMAC_RET_C2H_NOT_HANDLED;
+		break;
+	}
+
+	return status;
+}
+
+static enum halmac_ret_status
+halmac_parse_c2h_debug_88xx(struct halmac_adapter *halmac_adapter, u8 *c2h_buf,
+			    u32 c2h_size)
+{
+	void *driver_adapter = NULL;
+	u8 *c2h_buf_local = (u8 *)NULL;
+	u32 c2h_size_local = 0;
+	u8 dbg_content_length = 0;
+	u8 dbg_seq_num = 0;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	c2h_buf_local = c2h_buf;
+	c2h_size_local = c2h_size;
+
+	dbg_content_length = (u8)C2H_HDR_GET_LEN((u8 *)c2h_buf_local);
+
+	if (dbg_content_length > C2H_DBG_CONTENT_MAX_LENGTH)
+		return HALMAC_RET_SUCCESS;
+
+	*(c2h_buf_local + C2H_DBG_HEADER_LENGTH + dbg_content_length - 2) =
+		'\n';
+	dbg_seq_num = (u8)(*(c2h_buf_local + C2H_DBG_HEADER_LENGTH));
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"[RTKFW, SEQ=%d]: %s", dbg_seq_num,
+			(char *)(c2h_buf_local + C2H_DBG_HEADER_LENGTH + 1));
+
+	return HALMAC_RET_SUCCESS;
+}
+
+static enum halmac_ret_status
+halmac_parse_scan_status_rpt_88xx(struct halmac_adapter *halmac_adapter,
+				  u8 *c2h_buf, u32 c2h_size)
+{
+	u8 h2c_return_code;
+	void *driver_adapter = halmac_adapter->driver_adapter;
+	enum halmac_cmd_process_status process_status;
+
+	h2c_return_code = (u8)SCAN_STATUS_RPT_GET_H2C_RETURN_CODE(c2h_buf);
+	process_status = (enum halmac_h2c_return_code)h2c_return_code ==
+					 HALMAC_H2C_RETURN_SUCCESS ?
+				 HALMAC_CMD_PROCESS_DONE :
+				 HALMAC_CMD_PROCESS_ERROR;
+
+	PLATFORM_EVENT_INDICATION(driver_adapter, HALMAC_FEATURE_CHANNEL_SWITCH,
+				  process_status, NULL, 0);
+
+	halmac_adapter->halmac_state.scan_state_set.process_status =
+		process_status;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"[TRACE]scan status : %X\n", process_status);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+static enum halmac_ret_status
+halmac_parse_psd_data_88xx(struct halmac_adapter *halmac_adapter, u8 *c2h_buf,
+			   u32 c2h_size)
+{
+	u8 segment_id = 0, segment_size = 0, h2c_seq = 0;
+	u16 total_size;
+	void *driver_adapter = halmac_adapter->driver_adapter;
+	enum halmac_cmd_process_status process_status;
+	struct halmac_psd_state_set *psd_set =
+		&halmac_adapter->halmac_state.psd_set;
+
+	h2c_seq = (u8)PSD_DATA_GET_H2C_SEQ(c2h_buf);
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"[TRACE]Seq num : h2c -> %d c2h -> %d\n",
+			psd_set->seq_num, h2c_seq);
+	if (h2c_seq != psd_set->seq_num) {
+		pr_err("[ERR]Seq num mismatch : h2c -> %d c2h -> %d\n",
+		       psd_set->seq_num, h2c_seq);
+		return HALMAC_RET_SUCCESS;
+	}
+
+	if (psd_set->process_status != HALMAC_CMD_PROCESS_SENDING) {
+		pr_err("[ERR]Not in HALMAC_CMD_PROCESS_SENDING\n");
+		return HALMAC_RET_SUCCESS;
+	}
+
+	total_size = (u16)PSD_DATA_GET_TOTAL_SIZE(c2h_buf);
+	segment_id = (u8)PSD_DATA_GET_SEGMENT_ID(c2h_buf);
+	segment_size = (u8)PSD_DATA_GET_SEGMENT_SIZE(c2h_buf);
+	psd_set->data_size = total_size;
+
+	if (!psd_set->data)
+		psd_set->data = kzalloc(psd_set->data_size, GFP_KERNEL);
+
+	if (segment_id == 0)
+		psd_set->segment_size = segment_size;
+
+	memcpy(psd_set->data + segment_id * psd_set->segment_size,
+	       c2h_buf + HALMAC_C2H_DATA_OFFSET_88XX, segment_size);
+
+	if (!PSD_DATA_GET_END_SEGMENT(c2h_buf))
+		return HALMAC_RET_SUCCESS;
+
+	process_status = HALMAC_CMD_PROCESS_DONE;
+	psd_set->process_status = process_status;
+
+	PLATFORM_EVENT_INDICATION(driver_adapter, HALMAC_FEATURE_PSD,
+				  process_status, psd_set->data,
+				  psd_set->data_size);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+static enum halmac_ret_status
+halmac_parse_efuse_data_88xx(struct halmac_adapter *halmac_adapter, u8 *c2h_buf,
+			     u32 c2h_size)
+{
+	u8 segment_id = 0, segment_size = 0, h2c_seq = 0;
+	u8 *eeprom_map = NULL;
+	u32 eeprom_size = halmac_adapter->hw_config_info.eeprom_size;
+	u8 h2c_return_code = 0;
+	void *driver_adapter = halmac_adapter->driver_adapter;
+	enum halmac_cmd_process_status process_status;
+
+	h2c_seq = (u8)EFUSE_DATA_GET_H2C_SEQ(c2h_buf);
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"[TRACE]Seq num : h2c -> %d c2h -> %d\n",
+			halmac_adapter->halmac_state.efuse_state_set.seq_num,
+			h2c_seq);
+	if (h2c_seq != halmac_adapter->halmac_state.efuse_state_set.seq_num) {
+		pr_err("[ERR]Seq num mismatch : h2c -> %d c2h -> %d\n",
+		       halmac_adapter->halmac_state.efuse_state_set.seq_num,
+		       h2c_seq);
+		return HALMAC_RET_SUCCESS;
+	}
+
+	if (halmac_adapter->halmac_state.efuse_state_set.process_status !=
+	    HALMAC_CMD_PROCESS_SENDING) {
+		pr_err("[ERR]Not in HALMAC_CMD_PROCESS_SENDING\n");
+		return HALMAC_RET_SUCCESS;
+	}
+
+	segment_id = (u8)EFUSE_DATA_GET_SEGMENT_ID(c2h_buf);
+	segment_size = (u8)EFUSE_DATA_GET_SEGMENT_SIZE(c2h_buf);
+	if (segment_id == 0)
+		halmac_adapter->efuse_segment_size = segment_size;
+
+	eeprom_map = kzalloc(eeprom_size, GFP_KERNEL);
+	if (!eeprom_map) {
+		/* out of memory */
+		return HALMAC_RET_MALLOC_FAIL;
+	}
+	memset(eeprom_map, 0xFF, eeprom_size);
+
+	spin_lock(&halmac_adapter->efuse_lock);
+	memcpy(halmac_adapter->hal_efuse_map +
+		       segment_id * halmac_adapter->efuse_segment_size,
+	       c2h_buf + HALMAC_C2H_DATA_OFFSET_88XX, segment_size);
+	spin_unlock(&halmac_adapter->efuse_lock);
+
+	if (!EFUSE_DATA_GET_END_SEGMENT(c2h_buf)) {
+		kfree(eeprom_map);
+		return HALMAC_RET_SUCCESS;
+	}
+
+	h2c_return_code =
+		halmac_adapter->halmac_state.efuse_state_set.fw_return_code;
+
+	if ((enum halmac_h2c_return_code)h2c_return_code ==
+	    HALMAC_H2C_RETURN_SUCCESS) {
+		process_status = HALMAC_CMD_PROCESS_DONE;
+		halmac_adapter->halmac_state.efuse_state_set.process_status =
+			process_status;
+
+		spin_lock(&halmac_adapter->efuse_lock);
+		halmac_adapter->hal_efuse_map_valid = true;
+		spin_unlock(&halmac_adapter->efuse_lock);
+
+		if (halmac_adapter->event_trigger.physical_efuse_map == 1) {
+			PLATFORM_EVENT_INDICATION(
+				driver_adapter,
+				HALMAC_FEATURE_DUMP_PHYSICAL_EFUSE,
+				process_status, halmac_adapter->hal_efuse_map,
+				halmac_adapter->hw_config_info.efuse_size);
+			halmac_adapter->event_trigger.physical_efuse_map = 0;
+		}
+
+		if (halmac_adapter->event_trigger.logical_efuse_map == 1) {
+			if (halmac_eeprom_parser_88xx(
+				    halmac_adapter,
+				    halmac_adapter->hal_efuse_map,
+				    eeprom_map) != HALMAC_RET_SUCCESS) {
+				kfree(eeprom_map);
+				return HALMAC_RET_EEPROM_PARSING_FAIL;
+			}
+			PLATFORM_EVENT_INDICATION(
+				driver_adapter,
+				HALMAC_FEATURE_DUMP_LOGICAL_EFUSE,
+				process_status, eeprom_map, eeprom_size);
+			halmac_adapter->event_trigger.logical_efuse_map = 0;
+		}
+	} else {
+		process_status = HALMAC_CMD_PROCESS_ERROR;
+		halmac_adapter->halmac_state.efuse_state_set.process_status =
+			process_status;
+
+		if (halmac_adapter->event_trigger.physical_efuse_map == 1) {
+			PLATFORM_EVENT_INDICATION(
+				driver_adapter,
+				HALMAC_FEATURE_DUMP_PHYSICAL_EFUSE,
+				process_status,
+				&halmac_adapter->halmac_state.efuse_state_set
+					 .fw_return_code,
+				1);
+			halmac_adapter->event_trigger.physical_efuse_map = 0;
+		}
+
+		if (halmac_adapter->event_trigger.logical_efuse_map == 1) {
+			if (halmac_eeprom_parser_88xx(
+				    halmac_adapter,
+				    halmac_adapter->hal_efuse_map,
+				    eeprom_map) != HALMAC_RET_SUCCESS) {
+				kfree(eeprom_map);
+				return HALMAC_RET_EEPROM_PARSING_FAIL;
+			}
+			PLATFORM_EVENT_INDICATION(
+				driver_adapter,
+				HALMAC_FEATURE_DUMP_LOGICAL_EFUSE,
+				process_status,
+				&halmac_adapter->halmac_state.efuse_state_set
+					 .fw_return_code,
+				1);
+			halmac_adapter->event_trigger.logical_efuse_map = 0;
+		}
+	}
+
+	kfree(eeprom_map);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+static enum halmac_ret_status
+halmac_parse_h2c_ack_88xx(struct halmac_adapter *halmac_adapter, u8 *c2h_buf,
+			  u32 c2h_size)
+{
+	u8 h2c_cmd_id, h2c_sub_cmd_id;
+	u8 h2c_return_code;
+	void *driver_adapter = halmac_adapter->driver_adapter;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"Ack for C2H!!\n");
+
+	h2c_return_code = (u8)H2C_ACK_HDR_GET_H2C_RETURN_CODE(c2h_buf);
+	if ((enum halmac_h2c_return_code)h2c_return_code !=
+	    HALMAC_H2C_RETURN_SUCCESS)
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+				"C2H_PKT Status Error!! Status = %d\n",
+				h2c_return_code);
+
+	h2c_cmd_id = (u8)H2C_ACK_HDR_GET_H2C_CMD_ID(c2h_buf);
+
+	if (h2c_cmd_id != 0xFF) {
+		pr_err("original h2c ack is not handled!!\n");
+		status = HALMAC_RET_C2H_NOT_HANDLED;
+	} else {
+		h2c_sub_cmd_id = (u8)H2C_ACK_HDR_GET_H2C_SUB_CMD_ID(c2h_buf);
+
+		switch (h2c_sub_cmd_id) {
+		case H2C_SUB_CMD_ID_DUMP_PHYSICAL_EFUSE_ACK:
+			status = halmac_parse_h2c_ack_phy_efuse_88xx(
+				halmac_adapter, c2h_buf, c2h_size);
+			break;
+		case H2C_SUB_CMD_ID_CFG_PARAMETER_ACK:
+			status = halmac_parse_h2c_ack_cfg_para_88xx(
+				halmac_adapter, c2h_buf, c2h_size);
+			break;
+		case H2C_SUB_CMD_ID_UPDATE_PACKET_ACK:
+			status = halmac_parse_h2c_ack_update_packet_88xx(
+				halmac_adapter, c2h_buf, c2h_size);
+			break;
+		case H2C_SUB_CMD_ID_UPDATE_DATAPACK_ACK:
+			status = halmac_parse_h2c_ack_update_datapack_88xx(
+				halmac_adapter, c2h_buf, c2h_size);
+			break;
+		case H2C_SUB_CMD_ID_RUN_DATAPACK_ACK:
+			status = halmac_parse_h2c_ack_run_datapack_88xx(
+				halmac_adapter, c2h_buf, c2h_size);
+			break;
+		case H2C_SUB_CMD_ID_CHANNEL_SWITCH_ACK:
+			status = halmac_parse_h2c_ack_channel_switch_88xx(
+				halmac_adapter, c2h_buf, c2h_size);
+			break;
+		case H2C_SUB_CMD_ID_IQK_ACK:
+			status = halmac_parse_h2c_ack_iqk_88xx(
+				halmac_adapter, c2h_buf, c2h_size);
+			break;
+		case H2C_SUB_CMD_ID_POWER_TRACKING_ACK:
+			status = halmac_parse_h2c_ack_power_tracking_88xx(
+				halmac_adapter, c2h_buf, c2h_size);
+			break;
+		case H2C_SUB_CMD_ID_PSD_ACK:
+			break;
+		default:
+			pr_err("h2c_sub_cmd_id switch case out of boundary!!\n");
+			status = HALMAC_RET_C2H_NOT_HANDLED;
+			break;
+		}
+	}
+
+	return status;
+}
+
+static enum halmac_ret_status
+halmac_parse_h2c_ack_phy_efuse_88xx(struct halmac_adapter *halmac_adapter,
+				    u8 *c2h_buf, u32 c2h_size)
+{
+	u8 h2c_seq = 0;
+	u8 h2c_return_code;
+	void *driver_adapter = halmac_adapter->driver_adapter;
+
+	h2c_seq = (u8)H2C_ACK_HDR_GET_H2C_SEQ(c2h_buf);
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"[TRACE]Seq num : h2c -> %d c2h -> %d\n",
+			halmac_adapter->halmac_state.efuse_state_set.seq_num,
+			h2c_seq);
+	if (h2c_seq != halmac_adapter->halmac_state.efuse_state_set.seq_num) {
+		pr_err("[ERR]Seq num mismatch : h2c -> %d c2h -> %d\n",
+		       halmac_adapter->halmac_state.efuse_state_set.seq_num,
+		       h2c_seq);
+		return HALMAC_RET_SUCCESS;
+	}
+
+	if (halmac_adapter->halmac_state.efuse_state_set.process_status !=
+	    HALMAC_CMD_PROCESS_SENDING) {
+		pr_err("[ERR]Not in HALMAC_CMD_PROCESS_SENDING\n");
+		return HALMAC_RET_SUCCESS;
+	}
+
+	h2c_return_code = (u8)H2C_ACK_HDR_GET_H2C_RETURN_CODE(c2h_buf);
+	halmac_adapter->halmac_state.efuse_state_set.fw_return_code =
+		h2c_return_code;
+
+	return HALMAC_RET_SUCCESS;
+}
+
+static enum halmac_ret_status
+halmac_parse_h2c_ack_cfg_para_88xx(struct halmac_adapter *halmac_adapter,
+				   u8 *c2h_buf, u32 c2h_size)
+{
+	u8 h2c_seq = 0;
+	u8 h2c_return_code;
+	u32 offset_accu = 0, value_accu = 0;
+	void *driver_adapter = halmac_adapter->driver_adapter;
+	enum halmac_cmd_process_status process_status =
+		HALMAC_CMD_PROCESS_UNDEFINE;
+
+	h2c_seq = (u8)H2C_ACK_HDR_GET_H2C_SEQ(c2h_buf);
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"Seq num : h2c -> %d c2h -> %d\n",
+			halmac_adapter->halmac_state.cfg_para_state_set.seq_num,
+			h2c_seq);
+	if (h2c_seq !=
+	    halmac_adapter->halmac_state.cfg_para_state_set.seq_num) {
+		pr_err("Seq num mismatch : h2c -> %d c2h -> %d\n",
+		       halmac_adapter->halmac_state.cfg_para_state_set.seq_num,
+		       h2c_seq);
+		return HALMAC_RET_SUCCESS;
+	}
+
+	if (halmac_adapter->halmac_state.cfg_para_state_set.process_status !=
+	    HALMAC_CMD_PROCESS_SENDING) {
+		pr_err("Not in HALMAC_CMD_PROCESS_SENDING\n");
+		return HALMAC_RET_SUCCESS;
+	}
+
+	h2c_return_code = (u8)H2C_ACK_HDR_GET_H2C_RETURN_CODE(c2h_buf);
+	halmac_adapter->halmac_state.cfg_para_state_set.fw_return_code =
+		h2c_return_code;
+	offset_accu = CFG_PARAMETER_ACK_GET_OFFSET_ACCUMULATION(c2h_buf);
+	value_accu = CFG_PARAMETER_ACK_GET_VALUE_ACCUMULATION(c2h_buf);
+
+	if ((offset_accu !=
+	     halmac_adapter->config_para_info.offset_accumulation) ||
+	    (value_accu !=
+	     halmac_adapter->config_para_info.value_accumulation)) {
+		pr_err("[C2H]offset_accu : %x, value_accu : %x!!\n",
+		       offset_accu, value_accu);
+		pr_err("[Adapter]offset_accu : %x, value_accu : %x!!\n",
+		       halmac_adapter->config_para_info.offset_accumulation,
+		       halmac_adapter->config_para_info.value_accumulation);
+		process_status = HALMAC_CMD_PROCESS_ERROR;
+	}
+
+	if ((enum halmac_h2c_return_code)h2c_return_code ==
+		    HALMAC_H2C_RETURN_SUCCESS &&
+	    process_status != HALMAC_CMD_PROCESS_ERROR) {
+		process_status = HALMAC_CMD_PROCESS_DONE;
+		halmac_adapter->halmac_state.cfg_para_state_set.process_status =
+			process_status;
+		PLATFORM_EVENT_INDICATION(driver_adapter,
+					  HALMAC_FEATURE_CFG_PARA,
+					  process_status, NULL, 0);
+	} else {
+		process_status = HALMAC_CMD_PROCESS_ERROR;
+		halmac_adapter->halmac_state.cfg_para_state_set.process_status =
+			process_status;
+		PLATFORM_EVENT_INDICATION(
+			driver_adapter, HALMAC_FEATURE_CFG_PARA, process_status,
+			&halmac_adapter->halmac_state.cfg_para_state_set
+				 .fw_return_code,
+			1);
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+static enum halmac_ret_status
+halmac_parse_h2c_ack_update_packet_88xx(struct halmac_adapter *halmac_adapter,
+					u8 *c2h_buf, u32 c2h_size)
+{
+	u8 h2c_seq = 0;
+	u8 h2c_return_code;
+	void *driver_adapter = halmac_adapter->driver_adapter;
+	enum halmac_cmd_process_status process_status;
+
+	h2c_seq = (u8)H2C_ACK_HDR_GET_H2C_SEQ(c2h_buf);
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"[TRACE]Seq num : h2c -> %d c2h -> %d\n",
+			halmac_adapter->halmac_state.update_packet_set.seq_num,
+			h2c_seq);
+	if (h2c_seq != halmac_adapter->halmac_state.update_packet_set.seq_num) {
+		pr_err("[ERR]Seq num mismatch : h2c -> %d c2h -> %d\n",
+		       halmac_adapter->halmac_state.update_packet_set.seq_num,
+		       h2c_seq);
+		return HALMAC_RET_SUCCESS;
+	}
+
+	if (halmac_adapter->halmac_state.update_packet_set.process_status !=
+	    HALMAC_CMD_PROCESS_SENDING) {
+		pr_err("[ERR]Not in HALMAC_CMD_PROCESS_SENDING\n");
+		return HALMAC_RET_SUCCESS;
+	}
+
+	h2c_return_code = (u8)H2C_ACK_HDR_GET_H2C_RETURN_CODE(c2h_buf);
+	halmac_adapter->halmac_state.update_packet_set.fw_return_code =
+		h2c_return_code;
+
+	if ((enum halmac_h2c_return_code)h2c_return_code ==
+	    HALMAC_H2C_RETURN_SUCCESS) {
+		process_status = HALMAC_CMD_PROCESS_DONE;
+		halmac_adapter->halmac_state.update_packet_set.process_status =
+			process_status;
+		PLATFORM_EVENT_INDICATION(driver_adapter,
+					  HALMAC_FEATURE_UPDATE_PACKET,
+					  process_status, NULL, 0);
+	} else {
+		process_status = HALMAC_CMD_PROCESS_ERROR;
+		halmac_adapter->halmac_state.update_packet_set.process_status =
+			process_status;
+		PLATFORM_EVENT_INDICATION(
+			driver_adapter, HALMAC_FEATURE_UPDATE_PACKET,
+			process_status,
+			&halmac_adapter->halmac_state.update_packet_set
+				 .fw_return_code,
+			1);
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+static enum halmac_ret_status
+halmac_parse_h2c_ack_update_datapack_88xx(struct halmac_adapter *halmac_adapter,
+					  u8 *c2h_buf, u32 c2h_size)
+{
+	void *driver_adapter = halmac_adapter->driver_adapter;
+	enum halmac_cmd_process_status process_status =
+		HALMAC_CMD_PROCESS_UNDEFINE;
+
+	PLATFORM_EVENT_INDICATION(driver_adapter,
+				  HALMAC_FEATURE_UPDATE_DATAPACK,
+				  process_status, NULL, 0);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+static enum halmac_ret_status
+halmac_parse_h2c_ack_run_datapack_88xx(struct halmac_adapter *halmac_adapter,
+				       u8 *c2h_buf, u32 c2h_size)
+{
+	void *driver_adapter = halmac_adapter->driver_adapter;
+	enum halmac_cmd_process_status process_status =
+		HALMAC_CMD_PROCESS_UNDEFINE;
+
+	PLATFORM_EVENT_INDICATION(driver_adapter, HALMAC_FEATURE_RUN_DATAPACK,
+				  process_status, NULL, 0);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+static enum halmac_ret_status
+halmac_parse_h2c_ack_channel_switch_88xx(struct halmac_adapter *halmac_adapter,
+					 u8 *c2h_buf, u32 c2h_size)
+{
+	u8 h2c_seq = 0;
+	u8 h2c_return_code;
+	void *driver_adapter = halmac_adapter->driver_adapter;
+	enum halmac_cmd_process_status process_status;
+
+	h2c_seq = (u8)H2C_ACK_HDR_GET_H2C_SEQ(c2h_buf);
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"[TRACE]Seq num : h2c -> %d c2h -> %d\n",
+			halmac_adapter->halmac_state.scan_state_set.seq_num,
+			h2c_seq);
+	if (h2c_seq != halmac_adapter->halmac_state.scan_state_set.seq_num) {
+		pr_err("[ERR]Seq num misactch : h2c -> %d c2h -> %d\n",
+		       halmac_adapter->halmac_state.scan_state_set.seq_num,
+		       h2c_seq);
+		return HALMAC_RET_SUCCESS;
+	}
+
+	if (halmac_adapter->halmac_state.scan_state_set.process_status !=
+	    HALMAC_CMD_PROCESS_SENDING) {
+		pr_err("[ERR]Not in HALMAC_CMD_PROCESS_SENDING\n");
+		return HALMAC_RET_SUCCESS;
+	}
+
+	h2c_return_code = (u8)H2C_ACK_HDR_GET_H2C_RETURN_CODE(c2h_buf);
+	halmac_adapter->halmac_state.scan_state_set.fw_return_code =
+		h2c_return_code;
+
+	if ((enum halmac_h2c_return_code)h2c_return_code ==
+	    HALMAC_H2C_RETURN_SUCCESS) {
+		process_status = HALMAC_CMD_PROCESS_RCVD;
+		halmac_adapter->halmac_state.scan_state_set.process_status =
+			process_status;
+		PLATFORM_EVENT_INDICATION(driver_adapter,
+					  HALMAC_FEATURE_CHANNEL_SWITCH,
+					  process_status, NULL, 0);
+	} else {
+		process_status = HALMAC_CMD_PROCESS_ERROR;
+		halmac_adapter->halmac_state.scan_state_set.process_status =
+			process_status;
+		PLATFORM_EVENT_INDICATION(
+			driver_adapter, HALMAC_FEATURE_CHANNEL_SWITCH,
+			process_status, &halmac_adapter->halmac_state
+						 .scan_state_set.fw_return_code,
+			1);
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+static enum halmac_ret_status
+halmac_parse_h2c_ack_iqk_88xx(struct halmac_adapter *halmac_adapter,
+			      u8 *c2h_buf, u32 c2h_size)
+{
+	u8 h2c_seq = 0;
+	u8 h2c_return_code;
+	void *driver_adapter = halmac_adapter->driver_adapter;
+	enum halmac_cmd_process_status process_status;
+
+	h2c_seq = (u8)H2C_ACK_HDR_GET_H2C_SEQ(c2h_buf);
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"[TRACE]Seq num : h2c -> %d c2h -> %d\n",
+			halmac_adapter->halmac_state.iqk_set.seq_num, h2c_seq);
+	if (h2c_seq != halmac_adapter->halmac_state.iqk_set.seq_num) {
+		pr_err("[ERR]Seq num misactch : h2c -> %d c2h -> %d\n",
+		       halmac_adapter->halmac_state.iqk_set.seq_num, h2c_seq);
+		return HALMAC_RET_SUCCESS;
+	}
+
+	if (halmac_adapter->halmac_state.iqk_set.process_status !=
+	    HALMAC_CMD_PROCESS_SENDING) {
+		pr_err("[ERR]Not in HALMAC_CMD_PROCESS_SENDING\n");
+		return HALMAC_RET_SUCCESS;
+	}
+
+	h2c_return_code = (u8)H2C_ACK_HDR_GET_H2C_RETURN_CODE(c2h_buf);
+	halmac_adapter->halmac_state.iqk_set.fw_return_code = h2c_return_code;
+
+	if ((enum halmac_h2c_return_code)h2c_return_code ==
+	    HALMAC_H2C_RETURN_SUCCESS) {
+		process_status = HALMAC_CMD_PROCESS_DONE;
+		halmac_adapter->halmac_state.iqk_set.process_status =
+			process_status;
+		PLATFORM_EVENT_INDICATION(driver_adapter, HALMAC_FEATURE_IQK,
+					  process_status, NULL, 0);
+	} else {
+		process_status = HALMAC_CMD_PROCESS_ERROR;
+		halmac_adapter->halmac_state.iqk_set.process_status =
+			process_status;
+		PLATFORM_EVENT_INDICATION(
+			driver_adapter, HALMAC_FEATURE_IQK, process_status,
+			&halmac_adapter->halmac_state.iqk_set.fw_return_code,
+			1);
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+static enum halmac_ret_status
+halmac_parse_h2c_ack_power_tracking_88xx(struct halmac_adapter *halmac_adapter,
+					 u8 *c2h_buf, u32 c2h_size)
+{
+	u8 h2c_seq = 0;
+	u8 h2c_return_code;
+	void *driver_adapter = halmac_adapter->driver_adapter;
+	enum halmac_cmd_process_status process_status;
+
+	h2c_seq = (u8)H2C_ACK_HDR_GET_H2C_SEQ(c2h_buf);
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_H2C, DBG_DMESG,
+			"[TRACE]Seq num : h2c -> %d c2h -> %d\n",
+			halmac_adapter->halmac_state.power_tracking_set.seq_num,
+			h2c_seq);
+	if (h2c_seq !=
+	    halmac_adapter->halmac_state.power_tracking_set.seq_num) {
+		pr_err("[ERR]Seq num mismatch : h2c -> %d c2h -> %d\n",
+		       halmac_adapter->halmac_state.power_tracking_set.seq_num,
+		       h2c_seq);
+		return HALMAC_RET_SUCCESS;
+	}
+
+	if (halmac_adapter->halmac_state.power_tracking_set.process_status !=
+	    HALMAC_CMD_PROCESS_SENDING) {
+		pr_err("[ERR]Not in HALMAC_CMD_PROCESS_SENDING\n");
+		return HALMAC_RET_SUCCESS;
+	}
+
+	h2c_return_code = (u8)H2C_ACK_HDR_GET_H2C_RETURN_CODE(c2h_buf);
+	halmac_adapter->halmac_state.power_tracking_set.fw_return_code =
+		h2c_return_code;
+
+	if ((enum halmac_h2c_return_code)h2c_return_code ==
+	    HALMAC_H2C_RETURN_SUCCESS) {
+		process_status = HALMAC_CMD_PROCESS_DONE;
+		halmac_adapter->halmac_state.power_tracking_set.process_status =
+			process_status;
+		PLATFORM_EVENT_INDICATION(driver_adapter,
+					  HALMAC_FEATURE_POWER_TRACKING,
+					  process_status, NULL, 0);
+	} else {
+		process_status = HALMAC_CMD_PROCESS_ERROR;
+		halmac_adapter->halmac_state.power_tracking_set.process_status =
+			process_status;
+		PLATFORM_EVENT_INDICATION(
+			driver_adapter, HALMAC_FEATURE_POWER_TRACKING,
+			process_status,
+			&halmac_adapter->halmac_state.power_tracking_set
+				 .fw_return_code,
+			1);
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_convert_to_sdio_bus_offset_88xx(struct halmac_adapter *halmac_adapter,
+				       u32 *halmac_offset)
+{
+	void *driver_adapter = NULL;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	switch ((*halmac_offset) & 0xFFFF0000) {
+	case WLAN_IOREG_OFFSET:
+		*halmac_offset = (HALMAC_SDIO_CMD_ADDR_MAC_REG << 13) |
+				 (*halmac_offset & HALMAC_WLAN_MAC_REG_MSK);
+		break;
+	case SDIO_LOCAL_OFFSET:
+		*halmac_offset = (HALMAC_SDIO_CMD_ADDR_SDIO_REG << 13) |
+				 (*halmac_offset & HALMAC_SDIO_LOCAL_MSK);
+		break;
+	default:
+		*halmac_offset = 0xFFFFFFFF;
+		pr_err("Unknown base address!!\n");
+		return HALMAC_RET_CONVERT_SDIO_OFFSET_FAIL;
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_update_sdio_free_page_88xx(struct halmac_adapter *halmac_adapter)
+{
+	u32 free_page = 0, free_page2 = 0, free_page3 = 0;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	struct halmac_sdio_free_space *sdio_free_space;
+	u8 data[12] = {0};
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	sdio_free_space = &halmac_adapter->sdio_free_space;
+	/*need to use HALMAC_REG_READ_N, 20160316, Soar*/
+	HALMAC_REG_SDIO_CMD53_READ_N(halmac_adapter, REG_SDIO_FREE_TXPG, 12,
+				     data);
+	free_page =
+		data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24);
+	free_page2 =
+		data[4] | (data[5] << 8) | (data[6] << 16) | (data[7] << 24);
+	free_page3 =
+		data[8] | (data[9] << 8) | (data[10] << 16) | (data[11] << 24);
+
+	sdio_free_space->high_queue_number =
+		(u16)BIT_GET_HIQ_FREEPG_V1(free_page);
+	sdio_free_space->normal_queue_number =
+		(u16)BIT_GET_MID_FREEPG_V1(free_page);
+	sdio_free_space->low_queue_number =
+		(u16)BIT_GET_LOW_FREEPG_V1(free_page2);
+	sdio_free_space->public_queue_number =
+		(u16)BIT_GET_PUB_FREEPG_V1(free_page2);
+	sdio_free_space->extra_queue_number =
+		(u16)BIT_GET_EXQ_FREEPG_V1(free_page3);
+	sdio_free_space->ac_oqt_number = (u8)((free_page3 >> 16) & 0xFF);
+	sdio_free_space->non_ac_oqt_number = (u8)((free_page3 >> 24) & 0xFF);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_update_oqt_free_space_88xx(struct halmac_adapter *halmac_adapter)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	struct halmac_sdio_free_space *sdio_free_space;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	sdio_free_space = &halmac_adapter->sdio_free_space;
+
+	sdio_free_space->ac_oqt_number = HALMAC_REG_READ_8(
+		halmac_adapter, REG_SDIO_OQT_FREE_TXPG_V1 + 2);
+	sdio_free_space->ac_empty =
+		HALMAC_REG_READ_8(halmac_adapter, REG_TXPKT_EMPTY);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s <==========\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_efuse_cmd_construct_state
+halmac_query_efuse_curr_state_88xx(struct halmac_adapter *halmac_adapter)
+{
+	return halmac_adapter->halmac_state.efuse_state_set
+		.efuse_cmd_construct_state;
+}
+
+enum halmac_ret_status halmac_transition_efuse_state_88xx(
+	struct halmac_adapter *halmac_adapter,
+	enum halmac_efuse_cmd_construct_state dest_state)
+{
+	struct halmac_efuse_state_set *efuse_state =
+		&halmac_adapter->halmac_state.efuse_state_set;
+
+	if (efuse_state->efuse_cmd_construct_state !=
+		    HALMAC_EFUSE_CMD_CONSTRUCT_IDLE &&
+	    efuse_state->efuse_cmd_construct_state !=
+		    HALMAC_EFUSE_CMD_CONSTRUCT_BUSY &&
+	    efuse_state->efuse_cmd_construct_state !=
+		    HALMAC_EFUSE_CMD_CONSTRUCT_H2C_SENT)
+		return HALMAC_RET_ERROR_STATE;
+
+	if (efuse_state->efuse_cmd_construct_state == dest_state)
+		return HALMAC_RET_ERROR_STATE;
+
+	if (dest_state == HALMAC_EFUSE_CMD_CONSTRUCT_BUSY) {
+		if (efuse_state->efuse_cmd_construct_state ==
+		    HALMAC_EFUSE_CMD_CONSTRUCT_H2C_SENT)
+			return HALMAC_RET_ERROR_STATE;
+	} else if (dest_state == HALMAC_EFUSE_CMD_CONSTRUCT_H2C_SENT) {
+		if (efuse_state->efuse_cmd_construct_state ==
+		    HALMAC_EFUSE_CMD_CONSTRUCT_IDLE)
+			return HALMAC_RET_ERROR_STATE;
+	}
+
+	efuse_state->efuse_cmd_construct_state = dest_state;
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_cfg_para_cmd_construct_state
+halmac_query_cfg_para_curr_state_88xx(struct halmac_adapter *halmac_adapter)
+{
+	return halmac_adapter->halmac_state.cfg_para_state_set
+		.cfg_para_cmd_construct_state;
+}
+
+enum halmac_ret_status halmac_transition_cfg_para_state_88xx(
+	struct halmac_adapter *halmac_adapter,
+	enum halmac_cfg_para_cmd_construct_state dest_state)
+{
+	struct halmac_cfg_para_state_set *cfg_para =
+		&halmac_adapter->halmac_state.cfg_para_state_set;
+
+	if (cfg_para->cfg_para_cmd_construct_state !=
+		    HALMAC_CFG_PARA_CMD_CONSTRUCT_IDLE &&
+	    cfg_para->cfg_para_cmd_construct_state !=
+		    HALMAC_CFG_PARA_CMD_CONSTRUCT_CONSTRUCTING &&
+	    cfg_para->cfg_para_cmd_construct_state !=
+		    HALMAC_CFG_PARA_CMD_CONSTRUCT_H2C_SENT)
+		return HALMAC_RET_ERROR_STATE;
+
+	if (dest_state == HALMAC_CFG_PARA_CMD_CONSTRUCT_IDLE) {
+		if (cfg_para->cfg_para_cmd_construct_state ==
+		    HALMAC_CFG_PARA_CMD_CONSTRUCT_CONSTRUCTING)
+			return HALMAC_RET_ERROR_STATE;
+	} else if (dest_state == HALMAC_CFG_PARA_CMD_CONSTRUCT_CONSTRUCTING) {
+		if (cfg_para->cfg_para_cmd_construct_state ==
+		    HALMAC_CFG_PARA_CMD_CONSTRUCT_H2C_SENT)
+			return HALMAC_RET_ERROR_STATE;
+	} else if (dest_state == HALMAC_CFG_PARA_CMD_CONSTRUCT_H2C_SENT) {
+		if (cfg_para->cfg_para_cmd_construct_state ==
+			    HALMAC_CFG_PARA_CMD_CONSTRUCT_IDLE ||
+		    cfg_para->cfg_para_cmd_construct_state ==
+			    HALMAC_CFG_PARA_CMD_CONSTRUCT_H2C_SENT)
+			return HALMAC_RET_ERROR_STATE;
+	}
+
+	cfg_para->cfg_para_cmd_construct_state = dest_state;
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_scan_cmd_construct_state
+halmac_query_scan_curr_state_88xx(struct halmac_adapter *halmac_adapter)
+{
+	return halmac_adapter->halmac_state.scan_state_set
+		.scan_cmd_construct_state;
+}
+
+enum halmac_ret_status halmac_transition_scan_state_88xx(
+	struct halmac_adapter *halmac_adapter,
+	enum halmac_scan_cmd_construct_state dest_state)
+{
+	struct halmac_scan_state_set *scan =
+		&halmac_adapter->halmac_state.scan_state_set;
+
+	if (scan->scan_cmd_construct_state > HALMAC_SCAN_CMD_CONSTRUCT_H2C_SENT)
+		return HALMAC_RET_ERROR_STATE;
+
+	if (dest_state == HALMAC_SCAN_CMD_CONSTRUCT_IDLE) {
+		if (scan->scan_cmd_construct_state ==
+			    HALMAC_SCAN_CMD_CONSTRUCT_BUFFER_CLEARED ||
+		    scan->scan_cmd_construct_state ==
+			    HALMAC_SCAN_CMD_CONSTRUCT_CONSTRUCTING)
+			return HALMAC_RET_ERROR_STATE;
+	} else if (dest_state == HALMAC_SCAN_CMD_CONSTRUCT_BUFFER_CLEARED) {
+		if (scan->scan_cmd_construct_state ==
+		    HALMAC_SCAN_CMD_CONSTRUCT_H2C_SENT)
+			return HALMAC_RET_ERROR_STATE;
+	} else if (dest_state == HALMAC_SCAN_CMD_CONSTRUCT_CONSTRUCTING) {
+		if (scan->scan_cmd_construct_state ==
+			    HALMAC_SCAN_CMD_CONSTRUCT_IDLE ||
+		    scan->scan_cmd_construct_state ==
+			    HALMAC_SCAN_CMD_CONSTRUCT_H2C_SENT)
+			return HALMAC_RET_ERROR_STATE;
+	} else if (dest_state == HALMAC_SCAN_CMD_CONSTRUCT_H2C_SENT) {
+		if (scan->scan_cmd_construct_state !=
+			    HALMAC_SCAN_CMD_CONSTRUCT_CONSTRUCTING &&
+		    scan->scan_cmd_construct_state !=
+			    HALMAC_SCAN_CMD_CONSTRUCT_BUFFER_CLEARED)
+			return HALMAC_RET_ERROR_STATE;
+	}
+
+	scan->scan_cmd_construct_state = dest_state;
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status halmac_query_cfg_para_status_88xx(
+	struct halmac_adapter *halmac_adapter,
+	enum halmac_cmd_process_status *process_status, u8 *data, u32 *size)
+{
+	struct halmac_cfg_para_state_set *cfg_para_state_set =
+		&halmac_adapter->halmac_state.cfg_para_state_set;
+
+	*process_status = cfg_para_state_set->process_status;
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status halmac_query_dump_physical_efuse_status_88xx(
+	struct halmac_adapter *halmac_adapter,
+	enum halmac_cmd_process_status *process_status, u8 *data, u32 *size)
+{
+	void *driver_adapter = NULL;
+	struct halmac_efuse_state_set *efuse_state_set =
+		&halmac_adapter->halmac_state.efuse_state_set;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	*process_status = efuse_state_set->process_status;
+
+	if (!data)
+		return HALMAC_RET_NULL_POINTER;
+
+	if (!size)
+		return HALMAC_RET_NULL_POINTER;
+
+	if (*process_status == HALMAC_CMD_PROCESS_DONE) {
+		if (*size < halmac_adapter->hw_config_info.efuse_size) {
+			*size = halmac_adapter->hw_config_info.efuse_size;
+			return HALMAC_RET_BUFFER_TOO_SMALL;
+		}
+
+		*size = halmac_adapter->hw_config_info.efuse_size;
+		memcpy(data, halmac_adapter->hal_efuse_map, *size);
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status halmac_query_dump_logical_efuse_status_88xx(
+	struct halmac_adapter *halmac_adapter,
+	enum halmac_cmd_process_status *process_status, u8 *data, u32 *size)
+{
+	u8 *eeprom_map = NULL;
+	u32 eeprom_size = halmac_adapter->hw_config_info.eeprom_size;
+	void *driver_adapter = NULL;
+	struct halmac_efuse_state_set *efuse_state_set =
+		&halmac_adapter->halmac_state.efuse_state_set;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	*process_status = efuse_state_set->process_status;
+
+	if (!data)
+		return HALMAC_RET_NULL_POINTER;
+
+	if (!size)
+		return HALMAC_RET_NULL_POINTER;
+
+	if (*process_status == HALMAC_CMD_PROCESS_DONE) {
+		if (*size < eeprom_size) {
+			*size = eeprom_size;
+			return HALMAC_RET_BUFFER_TOO_SMALL;
+		}
+
+		*size = eeprom_size;
+
+		eeprom_map = kzalloc(eeprom_size, GFP_KERNEL);
+		if (!eeprom_map) {
+			/* out of memory */
+			return HALMAC_RET_MALLOC_FAIL;
+		}
+		memset(eeprom_map, 0xFF, eeprom_size);
+
+		if (halmac_eeprom_parser_88xx(
+			    halmac_adapter, halmac_adapter->hal_efuse_map,
+			    eeprom_map) != HALMAC_RET_SUCCESS) {
+			kfree(eeprom_map);
+			return HALMAC_RET_EEPROM_PARSING_FAIL;
+		}
+
+		memcpy(data, eeprom_map, *size);
+
+		kfree(eeprom_map);
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status halmac_query_channel_switch_status_88xx(
+	struct halmac_adapter *halmac_adapter,
+	enum halmac_cmd_process_status *process_status, u8 *data, u32 *size)
+{
+	struct halmac_scan_state_set *scan_state_set =
+		&halmac_adapter->halmac_state.scan_state_set;
+
+	*process_status = scan_state_set->process_status;
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status halmac_query_update_packet_status_88xx(
+	struct halmac_adapter *halmac_adapter,
+	enum halmac_cmd_process_status *process_status, u8 *data, u32 *size)
+{
+	struct halmac_update_packet_state_set *update_packet_set =
+		&halmac_adapter->halmac_state.update_packet_set;
+
+	*process_status = update_packet_set->process_status;
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_query_iqk_status_88xx(struct halmac_adapter *halmac_adapter,
+			     enum halmac_cmd_process_status *process_status,
+			     u8 *data, u32 *size)
+{
+	struct halmac_iqk_state_set *iqk_set =
+		&halmac_adapter->halmac_state.iqk_set;
+
+	*process_status = iqk_set->process_status;
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status halmac_query_power_tracking_status_88xx(
+	struct halmac_adapter *halmac_adapter,
+	enum halmac_cmd_process_status *process_status, u8 *data, u32 *size)
+{
+	struct halmac_power_tracking_state_set *power_tracking_state_set =
+		&halmac_adapter->halmac_state.power_tracking_set;
+	;
+
+	*process_status = power_tracking_state_set->process_status;
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_query_psd_status_88xx(struct halmac_adapter *halmac_adapter,
+			     enum halmac_cmd_process_status *process_status,
+			     u8 *data, u32 *size)
+{
+	void *driver_adapter = NULL;
+	struct halmac_psd_state_set *psd_set =
+		&halmac_adapter->halmac_state.psd_set;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	*process_status = psd_set->process_status;
+
+	if (!data)
+		return HALMAC_RET_NULL_POINTER;
+
+	if (!size)
+		return HALMAC_RET_NULL_POINTER;
+
+	if (*process_status == HALMAC_CMD_PROCESS_DONE) {
+		if (*size < psd_set->data_size) {
+			*size = psd_set->data_size;
+			return HALMAC_RET_BUFFER_TOO_SMALL;
+		}
+
+		*size = psd_set->data_size;
+		memcpy(data, psd_set->data, *size);
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_verify_io_88xx(struct halmac_adapter *halmac_adapter)
+{
+	u8 value8, wvalue8;
+	u32 value32, value32_2, wvalue32;
+	u32 halmac_offset;
+	void *driver_adapter = NULL;
+	enum halmac_ret_status ret_status = HALMAC_RET_SUCCESS;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	if (halmac_adapter->halmac_interface == HALMAC_INTERFACE_SDIO) {
+		halmac_offset = REG_PAGE5_DUMMY;
+		if ((halmac_offset & 0xFFFF0000) == 0)
+			halmac_offset |= WLAN_IOREG_OFFSET;
+
+		ret_status = halmac_convert_to_sdio_bus_offset_88xx(
+			halmac_adapter, &halmac_offset);
+
+		/* Verify CMD52 R/W */
+		wvalue8 = 0xab;
+		PLATFORM_SDIO_CMD52_WRITE(driver_adapter, halmac_offset,
+					  wvalue8);
+
+		value8 =
+			PLATFORM_SDIO_CMD52_READ(driver_adapter, halmac_offset);
+
+		if (value8 != wvalue8) {
+			pr_err("cmd52 r/w fail write = %X read = %X\n", wvalue8,
+			       value8);
+			ret_status = HALMAC_RET_PLATFORM_API_INCORRECT;
+		} else {
+			HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT,
+					DBG_DMESG, "cmd52 r/w ok\n");
+		}
+
+		/* Verify CMD53 R/W */
+		PLATFORM_SDIO_CMD52_WRITE(driver_adapter, halmac_offset, 0xaa);
+		PLATFORM_SDIO_CMD52_WRITE(driver_adapter, halmac_offset + 1,
+					  0xbb);
+		PLATFORM_SDIO_CMD52_WRITE(driver_adapter, halmac_offset + 2,
+					  0xcc);
+		PLATFORM_SDIO_CMD52_WRITE(driver_adapter, halmac_offset + 3,
+					  0xdd);
+
+		value32 = PLATFORM_SDIO_CMD53_READ_32(driver_adapter,
+						      halmac_offset);
+
+		if (value32 != 0xddccbbaa) {
+			pr_err("cmd53 r fail : read = %X\n", value32);
+			ret_status = HALMAC_RET_PLATFORM_API_INCORRECT;
+		} else {
+			HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT,
+					DBG_DMESG, "cmd53 r ok\n");
+		}
+
+		wvalue32 = 0x11223344;
+		PLATFORM_SDIO_CMD53_WRITE_32(driver_adapter, halmac_offset,
+					     wvalue32);
+
+		value32 = PLATFORM_SDIO_CMD53_READ_32(driver_adapter,
+						      halmac_offset);
+
+		if (value32 != wvalue32) {
+			pr_err("cmd53 w fail\n");
+			ret_status = HALMAC_RET_PLATFORM_API_INCORRECT;
+		} else {
+			HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT,
+					DBG_DMESG, "cmd53 w ok\n");
+		}
+
+		value32 = PLATFORM_SDIO_CMD53_READ_32(
+			driver_adapter,
+			halmac_offset + 2); /* value32 should be 0x33441122 */
+
+		wvalue32 = 0x11225566;
+		PLATFORM_SDIO_CMD53_WRITE_32(driver_adapter, halmac_offset,
+					     wvalue32);
+
+		value32_2 = PLATFORM_SDIO_CMD53_READ_32(
+			driver_adapter,
+			halmac_offset + 2); /* value32 should be 0x55661122 */
+		if (value32_2 == value32) {
+			pr_err("cmd52 is used for HAL_SDIO_CMD53_READ_32\n");
+			ret_status = HALMAC_RET_PLATFORM_API_INCORRECT;
+		} else {
+			HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT,
+					DBG_DMESG, "cmd53 is correctly used\n");
+		}
+	} else {
+		wvalue32 = 0x77665511;
+		PLATFORM_REG_WRITE_32(driver_adapter, REG_PAGE5_DUMMY,
+				      wvalue32);
+
+		value32 = PLATFORM_REG_READ_32(driver_adapter, REG_PAGE5_DUMMY);
+		if (value32 != wvalue32) {
+			pr_err("reg rw\n");
+			ret_status = HALMAC_RET_PLATFORM_API_INCORRECT;
+		} else {
+			HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT,
+					DBG_DMESG, "reg rw ok\n");
+		}
+	}
+
+	return ret_status;
+}
+
+enum halmac_ret_status
+halmac_verify_send_rsvd_page_88xx(struct halmac_adapter *halmac_adapter)
+{
+	u8 *rsvd_buf = NULL;
+	u8 *rsvd_page = NULL;
+	u32 i;
+	u32 h2c_pkt_verify_size = 64, h2c_pkt_verify_payload = 0xab;
+	void *driver_adapter = NULL;
+	enum halmac_ret_status ret_status = HALMAC_RET_SUCCESS;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	rsvd_buf = kzalloc(h2c_pkt_verify_size, GFP_KERNEL);
+
+	if (!rsvd_buf) {
+		/*pr_err("[ERR]rsvd buffer malloc fail!!\n");*/
+		return HALMAC_RET_MALLOC_FAIL;
+	}
+
+	memset(rsvd_buf, (u8)h2c_pkt_verify_payload, h2c_pkt_verify_size);
+
+	ret_status = halmac_download_rsvd_page_88xx(halmac_adapter, rsvd_buf,
+						    h2c_pkt_verify_size);
+
+	if (ret_status != HALMAC_RET_SUCCESS) {
+		kfree(rsvd_buf);
+		return ret_status;
+	}
+
+	rsvd_page = kzalloc(h2c_pkt_verify_size +
+				    halmac_adapter->hw_config_info.txdesc_size,
+			    GFP_KERNEL);
+
+	if (!rsvd_page) {
+		pr_err("[ERR]rsvd page malloc fail!!\n");
+		kfree(rsvd_buf);
+		return HALMAC_RET_MALLOC_FAIL;
+	}
+
+	ret_status = halmac_dump_fifo_88xx(
+		halmac_adapter, HAL_FIFO_SEL_RSVD_PAGE, 0,
+		h2c_pkt_verify_size +
+			halmac_adapter->hw_config_info.txdesc_size,
+		rsvd_page);
+
+	if (ret_status != HALMAC_RET_SUCCESS) {
+		kfree(rsvd_buf);
+		kfree(rsvd_page);
+		return ret_status;
+	}
+
+	for (i = 0; i < h2c_pkt_verify_size; i++) {
+		if (*(rsvd_buf + i) !=
+		    *(rsvd_page +
+		      (i + halmac_adapter->hw_config_info.txdesc_size))) {
+			pr_err("[ERR]Compare RSVD page Fail\n");
+			ret_status = HALMAC_RET_PLATFORM_API_INCORRECT;
+		}
+	}
+
+	kfree(rsvd_buf);
+	kfree(rsvd_page);
+
+	return ret_status;
+}
+
+void halmac_power_save_cb_88xx(void *cb_data)
+{
+	void *driver_adapter = NULL;
+	struct halmac_adapter *halmac_adapter = (struct halmac_adapter *)NULL;
+
+	halmac_adapter = (struct halmac_adapter *)cb_data;
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_PWR, DBG_DMESG,
+			"%s\n", __func__);
+}
+
+enum halmac_ret_status
+halmac_buffer_read_88xx(struct halmac_adapter *halmac_adapter, u32 offset,
+			u32 size, enum hal_fifo_sel halmac_fifo_sel,
+			u8 *fifo_map)
+{
+	u32 start_page, value_read;
+	u32 i, counter = 0, residue;
+	struct halmac_api *halmac_api;
+
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	if (halmac_fifo_sel == HAL_FIFO_SEL_RSVD_PAGE)
+		offset = offset +
+			 (halmac_adapter->txff_allocation.rsvd_pg_bndy << 7);
+
+	start_page = offset >> 12;
+	residue = offset & (4096 - 1);
+
+	if (halmac_fifo_sel == HAL_FIFO_SEL_TX ||
+	    halmac_fifo_sel == HAL_FIFO_SEL_RSVD_PAGE)
+		start_page += 0x780;
+	else if (halmac_fifo_sel == HAL_FIFO_SEL_RX)
+		start_page += 0x700;
+	else if (halmac_fifo_sel == HAL_FIFO_SEL_REPORT)
+		start_page += 0x660;
+	else if (halmac_fifo_sel == HAL_FIFO_SEL_LLT)
+		start_page += 0x650;
+	else
+		return HALMAC_RET_NOT_SUPPORT;
+
+	value_read = HALMAC_REG_READ_16(halmac_adapter, REG_PKTBUF_DBG_CTRL);
+
+	do {
+		HALMAC_REG_WRITE_16(halmac_adapter, REG_PKTBUF_DBG_CTRL,
+				    (u16)(start_page | (value_read & 0xF000)));
+
+		for (i = 0x8000 + residue; i <= 0x8FFF; i += 4) {
+			*(u32 *)(fifo_map + counter) =
+				HALMAC_REG_READ_32(halmac_adapter, i);
+			*(u32 *)(fifo_map + counter) =
+				le32_to_cpu(*(__le32 *)(fifo_map + counter));
+			counter += 4;
+			if (size == counter)
+				goto HALMAC_BUF_READ_OK;
+		}
+
+		residue = 0;
+		start_page++;
+	} while (1);
+
+HALMAC_BUF_READ_OK:
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_PKTBUF_DBG_CTRL,
+			    (u16)value_read);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+void halmac_restore_mac_register_88xx(struct halmac_adapter *halmac_adapter,
+				      struct halmac_restore_info *restore_info,
+				      u32 restore_num)
+{
+	u8 value_length;
+	u32 i;
+	u32 mac_register;
+	u32 mac_value;
+	struct halmac_api *halmac_api;
+	struct halmac_restore_info *curr_restore_info = restore_info;
+
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	for (i = 0; i < restore_num; i++) {
+		mac_register = curr_restore_info->mac_register;
+		mac_value = curr_restore_info->value;
+		value_length = curr_restore_info->length;
+
+		if (value_length == 1)
+			HALMAC_REG_WRITE_8(halmac_adapter, mac_register,
+					   (u8)mac_value);
+		else if (value_length == 2)
+			HALMAC_REG_WRITE_16(halmac_adapter, mac_register,
+					    (u16)mac_value);
+		else if (value_length == 4)
+			HALMAC_REG_WRITE_32(halmac_adapter, mac_register,
+					    mac_value);
+
+		curr_restore_info++;
+	}
+}
+
+void halmac_api_record_id_88xx(struct halmac_adapter *halmac_adapter,
+			       enum halmac_api_id api_id)
+{
+}
+
+enum halmac_ret_status
+halmac_set_usb_mode_88xx(struct halmac_adapter *halmac_adapter,
+			 enum halmac_usb_mode usb_mode)
+{
+	u32 usb_temp;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	enum halmac_usb_mode current_usb_mode;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	current_usb_mode =
+		HALMAC_REG_READ_8(halmac_adapter, REG_SYS_CFG2 + 3) == 0x20 ?
+			HALMAC_USB_MODE_U3 :
+			HALMAC_USB_MODE_U2;
+
+	/*check if HW supports usb2_usb3 swtich*/
+	usb_temp = HALMAC_REG_READ_32(halmac_adapter, REG_PAD_CTRL2);
+	if (!BIT_GET_USB23_SW_MODE_V1(usb_temp) &&
+	    !(usb_temp & BIT_USB3_USB2_TRANSITION)) {
+		pr_err("HALMAC_HW_USB_MODE usb mode HW unsupport\n");
+		return HALMAC_RET_USB2_3_SWITCH_UNSUPPORT;
+	}
+
+	if (usb_mode == current_usb_mode) {
+		pr_err("HALMAC_HW_USB_MODE usb mode unchange\n");
+		return HALMAC_RET_USB_MODE_UNCHANGE;
+	}
+
+	usb_temp &= ~(BIT_USB23_SW_MODE_V1(0x3));
+
+	if (usb_mode == HALMAC_USB_MODE_U2) {
+		/* usb3 to usb2 */
+		HALMAC_REG_WRITE_32(
+			halmac_adapter, REG_PAD_CTRL2,
+			usb_temp | BIT_USB23_SW_MODE_V1(HALMAC_USB_MODE_U2) |
+				BIT_RSM_EN_V1);
+	} else {
+		/* usb2 to usb3 */
+		HALMAC_REG_WRITE_32(
+			halmac_adapter, REG_PAD_CTRL2,
+			usb_temp | BIT_USB23_SW_MODE_V1(HALMAC_USB_MODE_U3) |
+				BIT_RSM_EN_V1);
+	}
+
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_PAD_CTRL2 + 1,
+			   4); /* set counter down timer 4x64 ms */
+	HALMAC_REG_WRITE_16(
+		halmac_adapter, REG_SYS_PW_CTRL,
+		HALMAC_REG_READ_16(halmac_adapter, REG_SYS_PW_CTRL) |
+			BIT_APFM_OFFMAC);
+	usleep_range(1000, 1100);
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_PAD_CTRL2,
+			    HALMAC_REG_READ_32(halmac_adapter, REG_PAD_CTRL2) |
+				    BIT_NO_PDN_CHIPOFF_V1);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+void halmac_enable_bb_rf_88xx(struct halmac_adapter *halmac_adapter, u8 enable)
+{
+	u8 value8;
+	u32 value32;
+	struct halmac_api *halmac_api;
+
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	if (enable == 1) {
+		value8 = HALMAC_REG_READ_8(halmac_adapter, REG_SYS_FUNC_EN);
+		value8 = value8 | BIT(0) | BIT(1);
+		HALMAC_REG_WRITE_8(halmac_adapter, REG_SYS_FUNC_EN, value8);
+
+		value8 = HALMAC_REG_READ_8(halmac_adapter, REG_RF_CTRL);
+		value8 = value8 | BIT(0) | BIT(1) | BIT(2);
+		HALMAC_REG_WRITE_8(halmac_adapter, REG_RF_CTRL, value8);
+
+		value32 = HALMAC_REG_READ_32(halmac_adapter, REG_WLRF1);
+		value32 = value32 | BIT(24) | BIT(25) | BIT(26);
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_WLRF1, value32);
+	} else {
+		value8 = HALMAC_REG_READ_8(halmac_adapter, REG_SYS_FUNC_EN);
+		value8 = value8 & (~(BIT(0) | BIT(1)));
+		HALMAC_REG_WRITE_8(halmac_adapter, REG_SYS_FUNC_EN, value8);
+
+		value8 = HALMAC_REG_READ_8(halmac_adapter, REG_RF_CTRL);
+		value8 = value8 & (~(BIT(0) | BIT(1) | BIT(2)));
+		HALMAC_REG_WRITE_8(halmac_adapter, REG_RF_CTRL, value8);
+
+		value32 = HALMAC_REG_READ_32(halmac_adapter, REG_WLRF1);
+		value32 = value32 & (~(BIT(24) | BIT(25) | BIT(26)));
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_WLRF1, value32);
+	}
+}
+
+void halmac_config_sdio_tx_page_threshold_88xx(
+	struct halmac_adapter *halmac_adapter,
+	struct halmac_tx_page_threshold_info *threshold_info)
+{
+	struct halmac_api *halmac_api;
+
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	switch (threshold_info->dma_queue_sel) {
+	case HALMAC_MAP2_HQ:
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_TQPNT1,
+				    threshold_info->threshold);
+		break;
+	case HALMAC_MAP2_NQ:
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_TQPNT2,
+				    threshold_info->threshold);
+		break;
+	case HALMAC_MAP2_LQ:
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_TQPNT3,
+				    threshold_info->threshold);
+		break;
+	case HALMAC_MAP2_EXQ:
+		HALMAC_REG_WRITE_32(halmac_adapter, REG_TQPNT4,
+				    threshold_info->threshold);
+		break;
+	default:
+		break;
+	}
+}
+
+void halmac_config_ampdu_88xx(struct halmac_adapter *halmac_adapter,
+			      struct halmac_ampdu_config *ampdu_config)
+{
+	struct halmac_api *halmac_api;
+
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_PROT_MODE_CTRL + 2,
+			   ampdu_config->max_agg_num);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_PROT_MODE_CTRL + 3,
+			   ampdu_config->max_agg_num);
+};
+
+enum halmac_ret_status
+halmac_check_oqt_88xx(struct halmac_adapter *halmac_adapter, u32 tx_agg_num,
+		      u8 *halmac_buf)
+{
+	u32 counter = 10;
+
+	/*S0, S1 are not allowed to use, 0x4E4[0] should be 0. Soar 20160323*/
+	/*no need to check non_ac_oqt_number. HI and MGQ blocked will cause
+	 *protocal issue before H_OQT being full
+	 */
+	switch ((enum halmac_queue_select)GET_TX_DESC_QSEL(halmac_buf)) {
+	case HALMAC_QUEUE_SELECT_VO:
+	case HALMAC_QUEUE_SELECT_VO_V2:
+	case HALMAC_QUEUE_SELECT_VI:
+	case HALMAC_QUEUE_SELECT_VI_V2:
+	case HALMAC_QUEUE_SELECT_BE:
+	case HALMAC_QUEUE_SELECT_BE_V2:
+	case HALMAC_QUEUE_SELECT_BK:
+	case HALMAC_QUEUE_SELECT_BK_V2:
+		counter = 10;
+		do {
+			if (halmac_adapter->sdio_free_space.ac_empty > 0) {
+				halmac_adapter->sdio_free_space.ac_empty -= 1;
+				break;
+			}
+
+			if (halmac_adapter->sdio_free_space.ac_oqt_number >=
+			    tx_agg_num) {
+				halmac_adapter->sdio_free_space.ac_oqt_number -=
+					(u8)tx_agg_num;
+				break;
+			}
+
+			halmac_update_oqt_free_space_88xx(halmac_adapter);
+
+			counter--;
+			if (counter == 0)
+				return HALMAC_RET_OQT_NOT_ENOUGH;
+		} while (1);
+		break;
+	default:
+		break;
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_rqpn_parser_88xx(struct halmac_adapter *halmac_adapter,
+			enum halmac_trx_mode halmac_trx_mode,
+			struct halmac_rqpn_ *rqpn_table)
+{
+	u8 search_flag;
+	u32 i;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	search_flag = 0;
+	for (i = 0; i < HALMAC_TRX_MODE_MAX; i++) {
+		if (halmac_trx_mode == rqpn_table[i].mode) {
+			halmac_adapter
+				->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_VO] =
+				rqpn_table[i].dma_map_vo;
+			halmac_adapter
+				->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_VI] =
+				rqpn_table[i].dma_map_vi;
+			halmac_adapter
+				->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_BE] =
+				rqpn_table[i].dma_map_be;
+			halmac_adapter
+				->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_BK] =
+				rqpn_table[i].dma_map_bk;
+			halmac_adapter
+				->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_MG] =
+				rqpn_table[i].dma_map_mg;
+			halmac_adapter
+				->halmac_ptcl_queue[HALMAC_PTCL_QUEUE_HI] =
+				rqpn_table[i].dma_map_hi;
+			search_flag = 1;
+			HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT,
+					DBG_DMESG, "%s done\n", __func__);
+			break;
+		}
+	}
+
+	if (search_flag == 0) {
+		pr_err("HALMAC_RET_TRX_MODE_NOT_SUPPORT 1 switch case not support\n");
+		return HALMAC_RET_TRX_MODE_NOT_SUPPORT;
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_pg_num_parser_88xx(struct halmac_adapter *halmac_adapter,
+			  enum halmac_trx_mode halmac_trx_mode,
+			  struct halmac_pg_num_ *pg_num_table)
+{
+	u8 search_flag;
+	u16 HPQ_num = 0, lpq_nnum = 0, NPQ_num = 0, GAPQ_num = 0;
+	u16 EXPQ_num = 0, PUBQ_num = 0;
+	u32 i = 0;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	search_flag = 0;
+	for (i = 0; i < HALMAC_TRX_MODE_MAX; i++) {
+		if (halmac_trx_mode == pg_num_table[i].mode) {
+			HPQ_num = pg_num_table[i].hq_num;
+			lpq_nnum = pg_num_table[i].lq_num;
+			NPQ_num = pg_num_table[i].nq_num;
+			EXPQ_num = pg_num_table[i].exq_num;
+			GAPQ_num = pg_num_table[i].gap_num;
+			PUBQ_num = halmac_adapter->txff_allocation.ac_q_pg_num -
+				   HPQ_num - lpq_nnum - NPQ_num - EXPQ_num -
+				   GAPQ_num;
+			search_flag = 1;
+			HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT,
+					DBG_DMESG, "%s done\n", __func__);
+			break;
+		}
+	}
+
+	if (search_flag == 0) {
+		pr_err("HALMAC_RET_TRX_MODE_NOT_SUPPORT 1 switch case not support\n");
+		return HALMAC_RET_TRX_MODE_NOT_SUPPORT;
+	}
+
+	if (halmac_adapter->txff_allocation.ac_q_pg_num <
+	    HPQ_num + lpq_nnum + NPQ_num + EXPQ_num + GAPQ_num)
+		return HALMAC_RET_CFG_TXFIFO_PAGE_FAIL;
+
+	halmac_adapter->txff_allocation.high_queue_pg_num = HPQ_num;
+	halmac_adapter->txff_allocation.low_queue_pg_num = lpq_nnum;
+	halmac_adapter->txff_allocation.normal_queue_pg_num = NPQ_num;
+	halmac_adapter->txff_allocation.extra_queue_pg_num = EXPQ_num;
+	halmac_adapter->txff_allocation.pub_queue_pg_num = PUBQ_num;
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_parse_intf_phy_88xx(struct halmac_adapter *halmac_adapter,
+			   struct halmac_intf_phy_para_ *intf_phy_para,
+			   enum halmac_intf_phy_platform platform,
+			   enum hal_intf_phy intf_phy)
+{
+	u16 value;
+	u16 curr_cut;
+	u16 offset;
+	u16 ip_sel;
+	struct halmac_intf_phy_para_ *curr_phy_para;
+	struct halmac_api *halmac_api;
+	void *driver_adapter = NULL;
+	u8 result = HALMAC_RET_SUCCESS;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	switch (halmac_adapter->chip_version) {
+	case HALMAC_CHIP_VER_A_CUT:
+		curr_cut = (u16)HALMAC_INTF_PHY_CUT_A;
+		break;
+	case HALMAC_CHIP_VER_B_CUT:
+		curr_cut = (u16)HALMAC_INTF_PHY_CUT_B;
+		break;
+	case HALMAC_CHIP_VER_C_CUT:
+		curr_cut = (u16)HALMAC_INTF_PHY_CUT_C;
+		break;
+	case HALMAC_CHIP_VER_D_CUT:
+		curr_cut = (u16)HALMAC_INTF_PHY_CUT_D;
+		break;
+	case HALMAC_CHIP_VER_E_CUT:
+		curr_cut = (u16)HALMAC_INTF_PHY_CUT_E;
+		break;
+	case HALMAC_CHIP_VER_F_CUT:
+		curr_cut = (u16)HALMAC_INTF_PHY_CUT_F;
+		break;
+	case HALMAC_CHIP_VER_TEST:
+		curr_cut = (u16)HALMAC_INTF_PHY_CUT_TESTCHIP;
+		break;
+	default:
+		return HALMAC_RET_FAIL;
+	}
+
+	for (curr_phy_para = intf_phy_para;; curr_phy_para++) {
+		if (!(curr_phy_para->cut & curr_cut) ||
+		    !(curr_phy_para->plaform & (u16)platform))
+			continue;
+
+		offset = curr_phy_para->offset;
+		value = curr_phy_para->value;
+		ip_sel = curr_phy_para->ip_sel;
+
+		if (offset == 0xFFFF)
+			break;
+
+		if (ip_sel == HALMAC_IP_SEL_MAC) {
+			HALMAC_REG_WRITE_8(halmac_adapter, (u32)offset,
+					   (u8)value);
+		} else if (intf_phy == HAL_INTF_PHY_USB2) {
+			result = halmac_usbphy_write_88xx(halmac_adapter,
+							  (u8)offset, value,
+							  HAL_INTF_PHY_USB2);
+
+			if (result != HALMAC_RET_SUCCESS)
+				pr_err("[ERR]Write USB2PHY fail!\n");
+
+		} else if (intf_phy == HAL_INTF_PHY_USB3) {
+			result = halmac_usbphy_write_88xx(halmac_adapter,
+							  (u8)offset, value,
+							  HAL_INTF_PHY_USB3);
+
+			if (result != HALMAC_RET_SUCCESS)
+				pr_err("[ERR]Write USB3PHY fail!\n");
+
+		} else if (intf_phy == HAL_INTF_PHY_PCIE_GEN1) {
+			if (ip_sel == HALMAC_IP_SEL_INTF_PHY)
+				result = halmac_mdio_write_88xx(
+					halmac_adapter, (u8)offset, value,
+					HAL_INTF_PHY_PCIE_GEN1);
+			else
+				result = halmac_dbi_write8_88xx(
+					halmac_adapter, offset, (u8)value);
+
+			if (result != HALMAC_RET_SUCCESS)
+				pr_err("[ERR]MDIO write GEN1 fail!\n");
+
+		} else if (intf_phy == HAL_INTF_PHY_PCIE_GEN2) {
+			if (ip_sel == HALMAC_IP_SEL_INTF_PHY)
+				result = halmac_mdio_write_88xx(
+					halmac_adapter, (u8)offset, value,
+					HAL_INTF_PHY_PCIE_GEN2);
+			else
+				result = halmac_dbi_write8_88xx(
+					halmac_adapter, offset, (u8)value);
+
+			if (result != HALMAC_RET_SUCCESS)
+				pr_err("[ERR]MDIO write GEN2 fail!\n");
+		} else {
+			pr_err("[ERR]Parse intf phy cfg error!\n");
+		}
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+enum halmac_ret_status
+halmac_dbi_write32_88xx(struct halmac_adapter *halmac_adapter, u16 addr,
+			u32 data)
+{
+	u8 tmp_u1b = 0;
+	u32 count = 0;
+	u16 write_addr = 0;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_REG_WRITE_32(halmac_adapter, REG_DBI_WDATA_V1, data);
+
+	write_addr = ((addr & 0x0ffc) | (0x000F << 12));
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_DBI_FLAG_V1, write_addr);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_DBI, DBG_DMESG,
+			"WriteAddr = %x\n", write_addr);
+
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_DBI_FLAG_V1 + 2, 0x01);
+	tmp_u1b = HALMAC_REG_READ_8(halmac_adapter, REG_DBI_FLAG_V1 + 2);
+
+	count = 20;
+	while (tmp_u1b && count != 0) {
+		udelay(10);
+		tmp_u1b =
+			HALMAC_REG_READ_8(halmac_adapter, REG_DBI_FLAG_V1 + 2);
+		count--;
+	}
+
+	if (tmp_u1b) {
+		pr_err("DBI write fail!\n");
+		return HALMAC_RET_FAIL;
+	} else {
+		return HALMAC_RET_SUCCESS;
+	}
+}
+
+u32 halmac_dbi_read32_88xx(struct halmac_adapter *halmac_adapter, u16 addr)
+{
+	u16 read_addr = addr & 0x0ffc;
+	u8 tmp_u1b = 0;
+	u32 count = 0;
+	u32 ret = 0;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_DBI_FLAG_V1, read_addr);
+
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_DBI_FLAG_V1 + 2, 0x2);
+	tmp_u1b = HALMAC_REG_READ_8(halmac_adapter, REG_DBI_FLAG_V1 + 2);
+
+	count = 20;
+	while (tmp_u1b && count != 0) {
+		udelay(10);
+		tmp_u1b =
+			HALMAC_REG_READ_8(halmac_adapter, REG_DBI_FLAG_V1 + 2);
+		count--;
+	}
+
+	if (tmp_u1b) {
+		ret = 0xFFFF;
+		pr_err("DBI read fail!\n");
+	} else {
+		ret = HALMAC_REG_READ_32(halmac_adapter, REG_DBI_RDATA_V1);
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_DBI, DBG_DMESG,
+				"Read Value = %x\n", ret);
+	}
+
+	return ret;
+}
+
+enum halmac_ret_status
+halmac_dbi_write8_88xx(struct halmac_adapter *halmac_adapter, u16 addr, u8 data)
+{
+	u8 tmp_u1b = 0;
+	u32 count = 0;
+	u16 write_addr = 0;
+	u16 remainder = addr & (4 - 1);
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_DBI_WDATA_V1 + remainder, data);
+
+	write_addr = ((addr & 0x0ffc) | (BIT(0) << (remainder + 12)));
+
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_DBI_FLAG_V1, write_addr);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_DBI, DBG_DMESG,
+			"WriteAddr = %x\n", write_addr);
+
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_DBI_FLAG_V1 + 2, 0x01);
+
+	tmp_u1b = HALMAC_REG_READ_8(halmac_adapter, REG_DBI_FLAG_V1 + 2);
+
+	count = 20;
+	while (tmp_u1b && count != 0) {
+		udelay(10);
+		tmp_u1b =
+			HALMAC_REG_READ_8(halmac_adapter, REG_DBI_FLAG_V1 + 2);
+		count--;
+	}
+
+	if (tmp_u1b) {
+		pr_err("DBI write fail!\n");
+		return HALMAC_RET_FAIL;
+	} else {
+		return HALMAC_RET_SUCCESS;
+	}
+}
+
+u8 halmac_dbi_read8_88xx(struct halmac_adapter *halmac_adapter, u16 addr)
+{
+	u16 read_addr = addr & 0x0ffc;
+	u8 tmp_u1b = 0;
+	u32 count = 0;
+	u8 ret = 0;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_DBI_FLAG_V1, read_addr);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_DBI_FLAG_V1 + 2, 0x2);
+
+	tmp_u1b = HALMAC_REG_READ_8(halmac_adapter, REG_DBI_FLAG_V1 + 2);
+
+	count = 20;
+	while (tmp_u1b && count != 0) {
+		udelay(10);
+		tmp_u1b =
+			HALMAC_REG_READ_8(halmac_adapter, REG_DBI_FLAG_V1 + 2);
+		count--;
+	}
+
+	if (tmp_u1b) {
+		ret = 0xFF;
+		pr_err("DBI read fail!\n");
+	} else {
+		ret = HALMAC_REG_READ_8(halmac_adapter,
+					REG_DBI_RDATA_V1 + (addr & (4 - 1)));
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_DBI, DBG_DMESG,
+				"Read Value = %x\n", ret);
+	}
+
+	return ret;
+}
+
+enum halmac_ret_status
+halmac_mdio_write_88xx(struct halmac_adapter *halmac_adapter, u8 addr, u16 data,
+		       u8 speed)
+{
+	u8 tmp_u1b = 0;
+	u32 count = 0;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	u8 real_addr = 0;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_REG_WRITE_16(halmac_adapter, REG_MDIO_V1, data);
+
+	/* address : 5bit */
+	real_addr = (addr & 0x1F);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_PCIE_MIX_CFG, real_addr);
+
+	if (speed == HAL_INTF_PHY_PCIE_GEN1) {
+		/* GEN1 page 0 */
+		if (addr < 0x20) {
+			/* select MDIO PHY Addr : reg 0x3F8[28:24]=5'b00 */
+			HALMAC_REG_WRITE_8(halmac_adapter, REG_PCIE_MIX_CFG + 3,
+					   0x00);
+
+			/* GEN1 page 1 */
+		} else {
+			/* select MDIO PHY Addr : reg 0x3F8[28:24]=5'b01 */
+			HALMAC_REG_WRITE_8(halmac_adapter, REG_PCIE_MIX_CFG + 3,
+					   0x01);
+		}
+
+	} else if (speed == HAL_INTF_PHY_PCIE_GEN2) {
+		/* GEN2 page 0 */
+		if (addr < 0x20) {
+			/* select MDIO PHY Addr : reg 0x3F8[28:24]=5'b10 */
+			HALMAC_REG_WRITE_8(halmac_adapter, REG_PCIE_MIX_CFG + 3,
+					   0x02);
+
+			/* GEN2 page 1 */
+		} else {
+			/* select MDIO PHY Addr : reg 0x3F8[28:24]=5'b11 */
+			HALMAC_REG_WRITE_8(halmac_adapter, REG_PCIE_MIX_CFG + 3,
+					   0x03);
+		}
+	} else {
+		pr_err("Error Speed !\n");
+	}
+
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_PCIE_MIX_CFG,
+			   HALMAC_REG_READ_8(halmac_adapter, REG_PCIE_MIX_CFG) |
+				   BIT_MDIO_WFLAG_V1);
+
+	tmp_u1b = HALMAC_REG_READ_8(halmac_adapter, REG_PCIE_MIX_CFG) &
+		  BIT_MDIO_WFLAG_V1;
+	count = 20;
+
+	while (tmp_u1b && count != 0) {
+		udelay(10);
+		tmp_u1b = HALMAC_REG_READ_8(halmac_adapter, REG_PCIE_MIX_CFG) &
+			  BIT_MDIO_WFLAG_V1;
+		count--;
+	}
+
+	if (tmp_u1b) {
+		pr_err("MDIO write fail!\n");
+		return HALMAC_RET_FAIL;
+	} else {
+		return HALMAC_RET_SUCCESS;
+	}
+}
+
+u16 halmac_mdio_read_88xx(struct halmac_adapter *halmac_adapter, u8 addr,
+			  u8 speed
+
+			  )
+{
+	u16 ret = 0;
+	u8 tmp_u1b = 0;
+	u32 count = 0;
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	u8 real_addr = 0;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	/* address : 5bit */
+	real_addr = (addr & 0x1F);
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_PCIE_MIX_CFG, real_addr);
+
+	if (speed == HAL_INTF_PHY_PCIE_GEN1) {
+		/* GEN1 page 0 */
+		if (addr < 0x20) {
+			/* select MDIO PHY Addr : reg 0x3F8[28:24]=5'b00 */
+			HALMAC_REG_WRITE_8(halmac_adapter, REG_PCIE_MIX_CFG + 3,
+					   0x00);
+
+			/* GEN1 page 1 */
+		} else {
+			/* select MDIO PHY Addr : reg 0x3F8[28:24]=5'b01 */
+			HALMAC_REG_WRITE_8(halmac_adapter, REG_PCIE_MIX_CFG + 3,
+					   0x01);
+		}
+
+	} else if (speed == HAL_INTF_PHY_PCIE_GEN2) {
+		/* GEN2 page 0 */
+		if (addr < 0x20) {
+			/* select MDIO PHY Addr : reg 0x3F8[28:24]=5'b10 */
+			HALMAC_REG_WRITE_8(halmac_adapter, REG_PCIE_MIX_CFG + 3,
+					   0x02);
+
+			/* GEN2 page 1 */
+		} else {
+			/* select MDIO PHY Addr : reg 0x3F8[28:24]=5'b11 */
+			HALMAC_REG_WRITE_8(halmac_adapter, REG_PCIE_MIX_CFG + 3,
+					   0x03);
+		}
+	} else {
+		pr_err("Error Speed !\n");
+	}
+
+	HALMAC_REG_WRITE_8(halmac_adapter, REG_PCIE_MIX_CFG,
+			   HALMAC_REG_READ_8(halmac_adapter, REG_PCIE_MIX_CFG) |
+				   BIT_MDIO_RFLAG_V1);
+
+	tmp_u1b = HALMAC_REG_READ_8(halmac_adapter, REG_PCIE_MIX_CFG) &
+		  BIT_MDIO_RFLAG_V1;
+	count = 20;
+
+	while (tmp_u1b && count != 0) {
+		udelay(10);
+		tmp_u1b = HALMAC_REG_READ_8(halmac_adapter, REG_PCIE_MIX_CFG) &
+			  BIT_MDIO_RFLAG_V1;
+		count--;
+	}
+
+	if (tmp_u1b) {
+		ret = 0xFFFF;
+		pr_err("MDIO read fail!\n");
+
+	} else {
+		ret = HALMAC_REG_READ_16(halmac_adapter, REG_MDIO_V1 + 2);
+		HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_MDIO, DBG_DMESG,
+				"Read Value = %x\n", ret);
+	}
+
+	return ret;
+}
+
+enum halmac_ret_status
+halmac_usbphy_write_88xx(struct halmac_adapter *halmac_adapter, u8 addr,
+			 u16 data, u8 speed)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	if (speed == HAL_INTF_PHY_USB3) {
+		HALMAC_REG_WRITE_8(halmac_adapter, 0xff0d, (u8)data);
+		HALMAC_REG_WRITE_8(halmac_adapter, 0xff0e, (u8)(data >> 8));
+		HALMAC_REG_WRITE_8(halmac_adapter, 0xff0c, addr | BIT(7));
+	} else if (speed == HAL_INTF_PHY_USB2) {
+		HALMAC_REG_WRITE_8(halmac_adapter, 0xfe41, (u8)data);
+		HALMAC_REG_WRITE_8(halmac_adapter, 0xfe40, addr);
+		HALMAC_REG_WRITE_8(halmac_adapter, 0xfe42, 0x81);
+	} else {
+		pr_err("[ERR]Error USB Speed !\n");
+		return HALMAC_RET_NOT_SUPPORT;
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+u16 halmac_usbphy_read_88xx(struct halmac_adapter *halmac_adapter, u8 addr,
+			    u8 speed)
+{
+	void *driver_adapter = NULL;
+	struct halmac_api *halmac_api;
+	u16 value = 0;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	if (speed == HAL_INTF_PHY_USB3) {
+		HALMAC_REG_WRITE_8(halmac_adapter, 0xff0c, addr | BIT(6));
+		value = (u16)(HALMAC_REG_READ_32(halmac_adapter, 0xff0c) >> 8);
+	} else if (speed == HAL_INTF_PHY_USB2) {
+		if ((addr >= 0xE0) /*&& (addr <= 0xFF)*/)
+			addr -= 0x20;
+		if ((addr >= 0xC0) && (addr <= 0xDF)) {
+			HALMAC_REG_WRITE_8(halmac_adapter, 0xfe40, addr);
+			HALMAC_REG_WRITE_8(halmac_adapter, 0xfe42, 0x81);
+			value = HALMAC_REG_READ_8(halmac_adapter, 0xfe43);
+		} else {
+			pr_err("[ERR]Error USB2PHY offset!\n");
+			return HALMAC_RET_NOT_SUPPORT;
+		}
+	} else {
+		pr_err("[ERR]Error USB Speed !\n");
+		return HALMAC_RET_NOT_SUPPORT;
+	}
+
+	return value;
+}
diff --git a/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_func_88xx.h b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_func_88xx.h
new file mode 100644
index 000000000000..1b59301d1158
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_88xx/halmac_func_88xx.h
@@ -0,0 +1,321 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HALMAC_FUNC_88XX_H_
+#define _HALMAC_FUNC_88XX_H_
+
+#include "../halmac_type.h"
+
+void halmac_init_offload_feature_state_machine_88xx(
+	struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_send_h2c_pkt_88xx(struct halmac_adapter *halmac_adapter, u8 *hal_buff,
+			 u32 size, bool ack);
+
+enum halmac_ret_status
+halmac_download_rsvd_page_88xx(struct halmac_adapter *halmac_adapter,
+			       u8 *hal_buf, u32 size);
+
+enum halmac_ret_status
+halmac_set_h2c_header_88xx(struct halmac_adapter *halmac_adapter,
+			   u8 *hal_h2c_hdr, u16 *seq, bool ack);
+
+enum halmac_ret_status halmac_set_fw_offload_h2c_header_88xx(
+	struct halmac_adapter *halmac_adapter, u8 *hal_h2c_hdr,
+	struct halmac_h2c_header_info *h2c_header_info, u16 *seq_num);
+
+enum halmac_ret_status
+halmac_dump_efuse_88xx(struct halmac_adapter *halmac_adapter,
+		       enum halmac_efuse_read_cfg cfg);
+
+enum halmac_ret_status
+halmac_func_read_efuse_88xx(struct halmac_adapter *halmac_adapter, u32 offset,
+			    u32 size, u8 *efuse_map);
+
+enum halmac_ret_status
+halmac_func_write_efuse_88xx(struct halmac_adapter *halmac_adapter, u32 offset,
+			     u8 value);
+
+enum halmac_ret_status
+halmac_func_switch_efuse_bank_88xx(struct halmac_adapter *halmac_adapter,
+				   enum halmac_efuse_bank efuse_bank);
+
+enum halmac_ret_status
+halmac_read_logical_efuse_map_88xx(struct halmac_adapter *halmac_adapter,
+				   u8 *map);
+
+enum halmac_ret_status
+halmac_func_write_logical_efuse_88xx(struct halmac_adapter *halmac_adapter,
+				     u32 offset, u8 value);
+
+enum halmac_ret_status
+halmac_func_pg_efuse_by_map_88xx(struct halmac_adapter *halmac_adapter,
+				 struct halmac_pg_efuse_info *pg_efuse_info,
+				 enum halmac_efuse_read_cfg cfg);
+
+enum halmac_ret_status
+halmac_eeprom_parser_88xx(struct halmac_adapter *halmac_adapter,
+			  u8 *physical_efuse_map, u8 *logical_efuse_map);
+
+enum halmac_ret_status
+halmac_read_hw_efuse_88xx(struct halmac_adapter *halmac_adapter, u32 offset,
+			  u32 size, u8 *efuse_map);
+
+enum halmac_ret_status
+halmac_dlfw_to_mem_88xx(struct halmac_adapter *halmac_adapter, u8 *ram_code,
+			u32 dest, u32 code_size);
+
+enum halmac_ret_status
+halmac_send_fwpkt_88xx(struct halmac_adapter *halmac_adapter, u8 *ram_code,
+		       u32 code_size);
+
+enum halmac_ret_status
+halmac_iddma_dlfw_88xx(struct halmac_adapter *halmac_adapter, u32 source,
+		       u32 dest, u32 length, u8 first);
+
+enum halmac_ret_status
+halmac_check_fw_chksum_88xx(struct halmac_adapter *halmac_adapter,
+			    u32 memory_address);
+
+enum halmac_ret_status
+halmac_dlfw_end_flow_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_free_dl_fw_end_flow_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_pwr_seq_parser_88xx(struct halmac_adapter *halmac_adapter, u8 cut,
+			   u8 fab, u8 intf,
+			   struct halmac_wl_pwr_cfg_ **pp_pwr_seq_cfg
+
+			   );
+
+enum halmac_ret_status
+halmac_get_h2c_buff_free_space_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_send_h2c_set_pwr_mode_88xx(struct halmac_adapter *halmac_adapter,
+				  struct halmac_fwlps_option *hal_fw_lps_opt);
+
+enum halmac_ret_status
+halmac_func_send_original_h2c_88xx(struct halmac_adapter *halmac_adapter,
+				   u8 *original_h2c, u16 *seq, u8 ack);
+
+enum halmac_ret_status
+halmac_media_status_rpt_88xx(struct halmac_adapter *halmac_adapter, u8 op_mode,
+			     u8 mac_id_ind, u8 mac_id, u8 mac_id_end);
+
+enum halmac_ret_status halmac_send_h2c_update_datapack_88xx(
+	struct halmac_adapter *halmac_adapter,
+	enum halmac_data_type halmac_data_type,
+	struct halmac_phy_parameter_info *para_info);
+
+enum halmac_ret_status
+halmac_send_h2c_run_datapack_88xx(struct halmac_adapter *halmac_adapter,
+				  enum halmac_data_type halmac_data_type);
+
+enum halmac_ret_status
+halmac_send_bt_coex_cmd_88xx(struct halmac_adapter *halmac_adapter, u8 *bt_buf,
+			     u32 bt_size, u8 ack);
+
+enum halmac_ret_status
+halmac_func_ctrl_ch_switch_88xx(struct halmac_adapter *halmac_adapter,
+				struct halmac_ch_switch_option *cs_option);
+
+enum halmac_ret_status
+halmac_func_send_general_info_88xx(struct halmac_adapter *halmac_adapter,
+				   struct halmac_general_info *general_info);
+
+enum halmac_ret_status
+halmac_send_h2c_ps_tuning_para_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_parse_c2h_packet_88xx(struct halmac_adapter *halmac_adapter,
+			     u8 *halmac_buf, u32 halmac_size);
+
+enum halmac_ret_status
+halmac_send_h2c_update_packet_88xx(struct halmac_adapter *halmac_adapter,
+				   enum halmac_packet_id pkt_id, u8 *pkt,
+				   u32 pkt_size);
+
+enum halmac_ret_status
+halmac_send_h2c_phy_parameter_88xx(struct halmac_adapter *halmac_adapter,
+				   struct halmac_phy_parameter_info *para_info,
+				   bool full_fifo);
+
+enum halmac_ret_status
+halmac_dump_physical_efuse_fw_88xx(struct halmac_adapter *halmac_adapter,
+				   u32 offset, u32 size, u8 *efuse_map);
+
+enum halmac_ret_status halmac_send_h2c_update_bcn_parse_info_88xx(
+	struct halmac_adapter *halmac_adapter,
+	struct halmac_bcn_ie_info *bcn_ie_info);
+
+enum halmac_ret_status
+halmac_convert_to_sdio_bus_offset_88xx(struct halmac_adapter *halmac_adapter,
+				       u32 *halmac_offset);
+
+enum halmac_ret_status
+halmac_update_sdio_free_page_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_update_oqt_free_space_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_efuse_cmd_construct_state
+halmac_query_efuse_curr_state_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status halmac_transition_efuse_state_88xx(
+	struct halmac_adapter *halmac_adapter,
+	enum halmac_efuse_cmd_construct_state dest_state);
+
+enum halmac_cfg_para_cmd_construct_state
+halmac_query_cfg_para_curr_state_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status halmac_transition_cfg_para_state_88xx(
+	struct halmac_adapter *halmac_adapter,
+	enum halmac_cfg_para_cmd_construct_state dest_state);
+
+enum halmac_scan_cmd_construct_state
+halmac_query_scan_curr_state_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status halmac_transition_scan_state_88xx(
+	struct halmac_adapter *halmac_adapter,
+	enum halmac_scan_cmd_construct_state dest_state);
+
+enum halmac_ret_status halmac_query_cfg_para_status_88xx(
+	struct halmac_adapter *halmac_adapter,
+	enum halmac_cmd_process_status *process_status, u8 *data, u32 *size);
+
+enum halmac_ret_status halmac_query_dump_physical_efuse_status_88xx(
+	struct halmac_adapter *halmac_adapter,
+	enum halmac_cmd_process_status *process_status, u8 *data, u32 *size);
+
+enum halmac_ret_status halmac_query_dump_logical_efuse_status_88xx(
+	struct halmac_adapter *halmac_adapter,
+	enum halmac_cmd_process_status *process_status, u8 *data, u32 *size);
+
+enum halmac_ret_status halmac_query_channel_switch_status_88xx(
+	struct halmac_adapter *halmac_adapter,
+	enum halmac_cmd_process_status *process_status, u8 *data, u32 *size);
+
+enum halmac_ret_status halmac_query_update_packet_status_88xx(
+	struct halmac_adapter *halmac_adapter,
+	enum halmac_cmd_process_status *process_status, u8 *data, u32 *size);
+
+enum halmac_ret_status
+halmac_query_iqk_status_88xx(struct halmac_adapter *halmac_adapter,
+			     enum halmac_cmd_process_status *process_status,
+			     u8 *data, u32 *size);
+
+enum halmac_ret_status halmac_query_power_tracking_status_88xx(
+	struct halmac_adapter *halmac_adapter,
+	enum halmac_cmd_process_status *process_status, u8 *data, u32 *size);
+
+enum halmac_ret_status
+halmac_query_psd_status_88xx(struct halmac_adapter *halmac_adapter,
+			     enum halmac_cmd_process_status *process_status,
+			     u8 *data, u32 *size);
+
+enum halmac_ret_status
+halmac_verify_io_88xx(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status
+halmac_verify_send_rsvd_page_88xx(struct halmac_adapter *halmac_adapter);
+
+void halmac_power_save_cb_88xx(void *cb_data);
+
+enum halmac_ret_status
+halmac_buffer_read_88xx(struct halmac_adapter *halmac_adapter, u32 offset,
+			u32 size, enum hal_fifo_sel halmac_fifo_sel,
+			u8 *fifo_map);
+
+void halmac_restore_mac_register_88xx(struct halmac_adapter *halmac_adapter,
+				      struct halmac_restore_info *restore_info,
+				      u32 restore_num);
+
+void halmac_api_record_id_88xx(struct halmac_adapter *halmac_adapter,
+			       enum halmac_api_id api_id);
+
+enum halmac_ret_status
+halmac_set_usb_mode_88xx(struct halmac_adapter *halmac_adapter,
+			 enum halmac_usb_mode usb_mode);
+
+void halmac_enable_bb_rf_88xx(struct halmac_adapter *halmac_adapter, u8 enable);
+
+void halmac_config_sdio_tx_page_threshold_88xx(
+	struct halmac_adapter *halmac_adapter,
+	struct halmac_tx_page_threshold_info *threshold_info);
+
+enum halmac_ret_status
+halmac_rqpn_parser_88xx(struct halmac_adapter *halmac_adapter,
+			enum halmac_trx_mode halmac_trx_mode,
+			struct halmac_rqpn_ *pwr_seq_cfg);
+
+enum halmac_ret_status
+halmac_check_oqt_88xx(struct halmac_adapter *halmac_adapter, u32 tx_agg_num,
+		      u8 *halmac_buf);
+
+enum halmac_ret_status
+halmac_pg_num_parser_88xx(struct halmac_adapter *halmac_adapter,
+			  enum halmac_trx_mode halmac_trx_mode,
+			  struct halmac_pg_num_ *pg_num_table);
+
+enum halmac_ret_status
+halmac_parse_intf_phy_88xx(struct halmac_adapter *halmac_adapter,
+			   struct halmac_intf_phy_para_ *intf_phy_para,
+			   enum halmac_intf_phy_platform platform,
+			   enum hal_intf_phy intf_phy);
+
+enum halmac_ret_status
+halmac_dbi_write32_88xx(struct halmac_adapter *halmac_adapter, u16 addr,
+			u32 data);
+
+u32 halmac_dbi_read32_88xx(struct halmac_adapter *halmac_adapter, u16 addr);
+
+enum halmac_ret_status
+halmac_dbi_write8_88xx(struct halmac_adapter *halmac_adapter, u16 addr,
+		       u8 data);
+
+u8 halmac_dbi_read8_88xx(struct halmac_adapter *halmac_adapter, u16 addr);
+
+u16 halmac_mdio_read_88xx(struct halmac_adapter *halmac_adapter, u8 addr,
+			  u8 speed
+
+			  );
+
+enum halmac_ret_status
+halmac_mdio_write_88xx(struct halmac_adapter *halmac_adapter, u8 addr, u16 data,
+		       u8 speed);
+
+void halmac_config_ampdu_88xx(struct halmac_adapter *halmac_adapter,
+			      struct halmac_ampdu_config *ampdu_config);
+
+enum halmac_ret_status
+halmac_usbphy_write_88xx(struct halmac_adapter *halmac_adapter, u8 addr,
+			 u16 data, u8 speed);
+
+u16 halmac_usbphy_read_88xx(struct halmac_adapter *halmac_adapter, u8 addr,
+			    u8 speed);
+#endif /* _HALMAC_FUNC_88XX_H_ */
diff --git a/drivers/staging/rtlwifi/halmac/halmac_api.c b/drivers/staging/rtlwifi/halmac/halmac_api.c
new file mode 100644
index 000000000000..0886a4611da0
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_api.c
@@ -0,0 +1,426 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#include "halmac_2_platform.h"
+#include "halmac_type.h"
+#include "halmac_88xx/halmac_api_88xx.h"
+#include "halmac_88xx/halmac_88xx_cfg.h"
+
+#include "halmac_88xx/halmac_8822b/halmac_8822b_cfg.h"
+
+static enum halmac_ret_status
+halmac_check_platform_api(void *driver_adapter,
+			  enum halmac_interface halmac_interface,
+			  struct halmac_platform_api *halmac_platform_api)
+{
+	void *adapter_local = NULL;
+
+	adapter_local = driver_adapter;
+
+	if (!halmac_platform_api)
+		return HALMAC_RET_PLATFORM_API_NULL;
+
+	if (halmac_interface == HALMAC_INTERFACE_SDIO) {
+		if (!halmac_platform_api->SDIO_CMD52_READ) {
+			pr_err("(!halmac_platform_api->SDIO_CMD52_READ)\n");
+			return HALMAC_RET_PLATFORM_API_NULL;
+		}
+		if (!halmac_platform_api->SDIO_CMD53_READ_8) {
+			pr_err("(!halmac_platform_api->SDIO_CMD53_READ_8)\n");
+			return HALMAC_RET_PLATFORM_API_NULL;
+		}
+		if (!halmac_platform_api->SDIO_CMD53_READ_16) {
+			pr_err("(!halmac_platform_api->SDIO_CMD53_READ_16)\n");
+			return HALMAC_RET_PLATFORM_API_NULL;
+		}
+		if (!halmac_platform_api->SDIO_CMD53_READ_32) {
+			pr_err("(!halmac_platform_api->SDIO_CMD53_READ_32)\n");
+			return HALMAC_RET_PLATFORM_API_NULL;
+		}
+		if (!halmac_platform_api->SDIO_CMD53_READ_N) {
+			pr_err("(!halmac_platform_api->SDIO_CMD53_READ_N)\n");
+			return HALMAC_RET_PLATFORM_API_NULL;
+		}
+		if (!halmac_platform_api->SDIO_CMD52_WRITE) {
+			pr_err("(!halmac_platform_api->SDIO_CMD52_WRITE)\n");
+			return HALMAC_RET_PLATFORM_API_NULL;
+		}
+		if (!halmac_platform_api->SDIO_CMD53_WRITE_8) {
+			pr_err("(!halmac_platform_api->SDIO_CMD53_WRITE_8)\n");
+			return HALMAC_RET_PLATFORM_API_NULL;
+		}
+		if (!halmac_platform_api->SDIO_CMD53_WRITE_16) {
+			pr_err("(!halmac_platform_api->SDIO_CMD53_WRITE_16)\n");
+			return HALMAC_RET_PLATFORM_API_NULL;
+		}
+		if (!halmac_platform_api->SDIO_CMD53_WRITE_32) {
+			pr_err("(!halmac_platform_api->SDIO_CMD53_WRITE_32)\n");
+			return HALMAC_RET_PLATFORM_API_NULL;
+		}
+	}
+
+	if (halmac_interface == HALMAC_INTERFACE_USB ||
+	    halmac_interface == HALMAC_INTERFACE_PCIE) {
+		if (!halmac_platform_api->REG_READ_8) {
+			pr_err("(!halmac_platform_api->REG_READ_8)\n");
+			return HALMAC_RET_PLATFORM_API_NULL;
+		}
+		if (!halmac_platform_api->REG_READ_16) {
+			pr_err("(!halmac_platform_api->REG_READ_16)\n");
+			return HALMAC_RET_PLATFORM_API_NULL;
+		}
+		if (!halmac_platform_api->REG_READ_32) {
+			pr_err("(!halmac_platform_api->REG_READ_32)\n");
+			return HALMAC_RET_PLATFORM_API_NULL;
+		}
+		if (!halmac_platform_api->REG_WRITE_8) {
+			pr_err("(!halmac_platform_api->REG_WRITE_8)\n");
+			return HALMAC_RET_PLATFORM_API_NULL;
+		}
+		if (!halmac_platform_api->REG_WRITE_16) {
+			pr_err("(!halmac_platform_api->REG_WRITE_16)\n");
+			return HALMAC_RET_PLATFORM_API_NULL;
+		}
+		if (!halmac_platform_api->REG_WRITE_32) {
+			pr_err("(!halmac_platform_api->REG_WRITE_32)\n");
+			return HALMAC_RET_PLATFORM_API_NULL;
+		}
+	}
+
+	if (!halmac_platform_api->EVENT_INDICATION) {
+		pr_err("(!halmac_platform_api->EVENT_INDICATION)\n");
+		return HALMAC_RET_PLATFORM_API_NULL;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+static enum halmac_ret_status
+halmac_convert_to_sdio_bus_offset(u32 *halmac_offset)
+{
+	switch ((*halmac_offset) & 0xFFFF0000) {
+	case WLAN_IOREG_OFFSET:
+		*halmac_offset = (HALMAC_SDIO_CMD_ADDR_MAC_REG << 13) |
+				 (*halmac_offset & HALMAC_WLAN_MAC_REG_MSK);
+		break;
+	case SDIO_LOCAL_OFFSET:
+		*halmac_offset = (HALMAC_SDIO_CMD_ADDR_SDIO_REG << 13) |
+				 (*halmac_offset & HALMAC_SDIO_LOCAL_MSK);
+		break;
+	default:
+		*halmac_offset = 0xFFFFFFFF;
+		return HALMAC_RET_CONVERT_SDIO_OFFSET_FAIL;
+	}
+
+	return HALMAC_RET_SUCCESS;
+}
+
+static u8
+platform_reg_read_8_sdio(void *driver_adapter,
+			 struct halmac_platform_api *halmac_platform_api,
+			 u32 offset)
+{
+	u8 value8;
+	u32 halmac_offset = offset;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	if ((halmac_offset & 0xFFFF0000) == 0)
+		halmac_offset |= WLAN_IOREG_OFFSET;
+
+	status = halmac_convert_to_sdio_bus_offset(&halmac_offset);
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("%s error = %x\n", __func__, status);
+		return status;
+	}
+
+	value8 = halmac_platform_api->SDIO_CMD52_READ(driver_adapter,
+						      halmac_offset);
+
+	return value8;
+}
+
+static enum halmac_ret_status
+platform_reg_write_8_sdio(void *driver_adapter,
+			  struct halmac_platform_api *halmac_platform_api,
+			  u32 offset, u8 data)
+{
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+	u32 halmac_offset = offset;
+
+	if ((halmac_offset & 0xFFFF0000) == 0)
+		halmac_offset |= WLAN_IOREG_OFFSET;
+
+	status = halmac_convert_to_sdio_bus_offset(&halmac_offset);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		pr_err("halmac_reg_write_8_sdio_88xx error = %x\n", status);
+		return status;
+	}
+	halmac_platform_api->SDIO_CMD52_WRITE(driver_adapter, halmac_offset,
+					      data);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+static enum halmac_ret_status
+halmac_get_chip_info(void *driver_adapter,
+		     struct halmac_platform_api *halmac_platform_api,
+		     enum halmac_interface halmac_interface,
+		     struct halmac_adapter *halmac_adapter)
+{
+	struct halmac_api *halmac_api = (struct halmac_api *)NULL;
+	u8 chip_id, chip_version;
+	u32 polling_count;
+
+	halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	/* Get Chip_id and Chip_version */
+	if (halmac_adapter->halmac_interface == HALMAC_INTERFACE_SDIO) {
+		platform_reg_write_8_sdio(
+			driver_adapter, halmac_platform_api, REG_SDIO_HSUS_CTRL,
+			platform_reg_read_8_sdio(driver_adapter,
+						 halmac_platform_api,
+						 REG_SDIO_HSUS_CTRL) &
+				~(BIT(0)));
+
+		polling_count = 10000;
+		while (!(platform_reg_read_8_sdio(driver_adapter,
+						  halmac_platform_api,
+						  REG_SDIO_HSUS_CTRL) &
+			 0x02)) {
+			polling_count--;
+			if (polling_count == 0)
+				return HALMAC_RET_SDIO_LEAVE_SUSPEND_FAIL;
+		}
+
+		chip_id = platform_reg_read_8_sdio(
+			driver_adapter, halmac_platform_api, REG_SYS_CFG2);
+		chip_version = platform_reg_read_8_sdio(driver_adapter,
+							halmac_platform_api,
+							REG_SYS_CFG1 + 1) >>
+			       4;
+	} else {
+		chip_id = halmac_platform_api->REG_READ_8(driver_adapter,
+							  REG_SYS_CFG2);
+		chip_version = halmac_platform_api->REG_READ_8(
+				       driver_adapter, REG_SYS_CFG1 + 1) >>
+			       4;
+	}
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"[TRACE]Chip id : 0x%X\n", chip_id);
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"[TRACE]Chip version : 0x%X\n", chip_version);
+
+	halmac_adapter->chip_version = (enum halmac_chip_ver)chip_version;
+
+	if (chip_id == HALMAC_CHIP_ID_HW_DEF_8822B)
+		halmac_adapter->chip_id = HALMAC_CHIP_ID_8822B;
+	else if (chip_id == HALMAC_CHIP_ID_HW_DEF_8821C)
+		halmac_adapter->chip_id = HALMAC_CHIP_ID_8821C;
+	else if (chip_id == HALMAC_CHIP_ID_HW_DEF_8814B)
+		halmac_adapter->chip_id = HALMAC_CHIP_ID_8814B;
+	else if (chip_id == HALMAC_CHIP_ID_HW_DEF_8197F)
+		halmac_adapter->chip_id = HALMAC_CHIP_ID_8197F;
+	else
+		halmac_adapter->chip_id = HALMAC_CHIP_ID_UNDEFINE;
+
+	if (halmac_adapter->chip_id == HALMAC_CHIP_ID_UNDEFINE)
+		return HALMAC_RET_CHIP_NOT_SUPPORT;
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_init_adapter() - init halmac_adapter
+ * @driver_adapter : the adapter of caller
+ * @halmac_platform_api : the platform APIs which is used in halmac APIs
+ * @halmac_interface : bus interface
+ * @pp_halmac_adapter : the adapter of halmac
+ * @pp_halmac_api : the function pointer of APIs, caller shall call APIs by
+ *                 function pointer
+ * Author : KaiYuan Chang / Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_init_adapter(void *driver_adapter,
+		    struct halmac_platform_api *halmac_platform_api,
+		    enum halmac_interface halmac_interface,
+		    struct halmac_adapter **pp_halmac_adapter,
+		    struct halmac_api **pp_halmac_api)
+{
+	struct halmac_adapter *halmac_adapter = (struct halmac_adapter *)NULL;
+	enum halmac_ret_status status = HALMAC_RET_SUCCESS;
+
+	union {
+		u32 i;
+		u8 x[4];
+	} ENDIAN_CHECK = {0x01000000};
+
+	status = halmac_check_platform_api(driver_adapter, halmac_interface,
+					   halmac_platform_api);
+	if (status != HALMAC_RET_SUCCESS)
+		return status;
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			HALMAC_SVN_VER "\n");
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"HALMAC_MAJOR_VER = %x\n", HALMAC_MAJOR_VER);
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"HALMAC_PROTOTYPE_VER = %x\n", HALMAC_PROTOTYPE_VER);
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"HALMAC_MINOR_VER = %x\n", HALMAC_MINOR_VER);
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"HALMAC_PATCH_VER = %x\n", HALMAC_PATCH_VER);
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"halmac_init_adapter_88xx ==========>\n");
+
+	/* Check endian setting - Little endian : 1, Big endian : 0*/
+	if (ENDIAN_CHECK.x[0] == HALMAC_SYSTEM_ENDIAN) {
+		pr_err("Endian setting Err!!\n");
+		return HALMAC_RET_ENDIAN_ERR;
+	}
+
+	halmac_adapter = kzalloc(sizeof(*halmac_adapter), GFP_KERNEL);
+	if (!halmac_adapter) {
+		/* out of memory */
+		return HALMAC_RET_MALLOC_FAIL;
+	}
+
+	/* return halmac adapter address to caller */
+	*pp_halmac_adapter = halmac_adapter;
+
+	/* Record caller info */
+	halmac_adapter->halmac_platform_api = halmac_platform_api;
+	halmac_adapter->driver_adapter = driver_adapter;
+	halmac_interface = halmac_interface == HALMAC_INTERFACE_AXI ?
+				   HALMAC_INTERFACE_PCIE :
+				   halmac_interface;
+	halmac_adapter->halmac_interface = halmac_interface;
+
+	spin_lock_init(&halmac_adapter->efuse_lock);
+	spin_lock_init(&halmac_adapter->h2c_seq_lock);
+
+	/*Get Chip*/
+	if (halmac_get_chip_info(driver_adapter, halmac_platform_api,
+				 halmac_interface,
+				 halmac_adapter) != HALMAC_RET_SUCCESS) {
+		pr_err("HALMAC_RET_CHIP_NOT_SUPPORT\n");
+		return HALMAC_RET_CHIP_NOT_SUPPORT;
+	}
+
+	/* Assign function pointer to halmac API */
+	halmac_init_adapter_para_88xx(halmac_adapter);
+	status = halmac_mount_api_88xx(halmac_adapter);
+
+	/* Return halmac API function pointer */
+	*pp_halmac_api = (struct halmac_api *)halmac_adapter->halmac_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"halmac_init_adapter_88xx <==========\n");
+
+	return status;
+}
+
+/**
+ * halmac_halt_api() - stop halmac_api action
+ * @halmac_adapter : the adapter of halmac
+ * Author : Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status halmac_halt_api(struct halmac_adapter *halmac_adapter)
+{
+	void *driver_adapter = NULL;
+	struct halmac_platform_api *halmac_platform_api =
+		(struct halmac_platform_api *)NULL;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+	halmac_platform_api = halmac_adapter->halmac_platform_api;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+	halmac_adapter->halmac_state.api_state = HALMAC_API_STATE_HALT;
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"%s ==========>\n", __func__);
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_deinit_adapter() - deinit halmac adapter
+ * @halmac_adapter : the adapter of halmac
+ * Author : KaiYuan Chang / Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status
+halmac_deinit_adapter(struct halmac_adapter *halmac_adapter)
+{
+	void *driver_adapter = NULL;
+
+	if (halmac_adapter_validate(halmac_adapter) != HALMAC_RET_SUCCESS)
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	driver_adapter = halmac_adapter->driver_adapter;
+
+	HALMAC_RT_TRACE(driver_adapter, HALMAC_MSG_INIT, DBG_DMESG,
+			"[TRACE]halmac_deinit_adapter_88xx ==========>\n");
+
+	kfree(halmac_adapter->hal_efuse_map);
+	halmac_adapter->hal_efuse_map = (u8 *)NULL;
+
+	kfree(halmac_adapter->halmac_state.psd_set.data);
+	halmac_adapter->halmac_state.psd_set.data = (u8 *)NULL;
+
+	kfree(halmac_adapter->halmac_api);
+	halmac_adapter->halmac_api = NULL;
+
+	halmac_adapter->hal_adapter_backup = NULL;
+	kfree(halmac_adapter);
+
+	return HALMAC_RET_SUCCESS;
+}
+
+/**
+ * halmac_get_version() - get HALMAC version
+ * @version : return version of major, prototype and minor information
+ * Author : KaiYuan Chang / Ivan Lin
+ * Return : enum halmac_ret_status
+ * More details of status code can be found in prototype document
+ */
+enum halmac_ret_status halmac_get_version(struct halmac_ver *version)
+{
+	version->major_ver = (u8)HALMAC_MAJOR_VER;
+	version->prototype_ver = (u8)HALMAC_PROTOTYPE_VER;
+	version->minor_ver = (u8)HALMAC_MINOR_VER;
+
+	return HALMAC_RET_SUCCESS;
+}
diff --git a/drivers/staging/rtlwifi/halmac/halmac_api.h b/drivers/staging/rtlwifi/halmac/halmac_api.h
new file mode 100644
index 000000000000..917a64601053
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_api.h
@@ -0,0 +1,82 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HALMAC_API_H_
+#define _HALMAC_API_H_
+
+#define HALMAC_SVN_VER "13348M"
+
+#define HALMAC_MAJOR_VER 0x0001 /* major version, ver_1 for async_api */
+/* For halmac_api num change or prototype change, increment prototype version.
+ * Otherwise, increase minor version
+ */
+#define HALMAC_PROTOTYPE_VER 0x0003 /* prototype version */
+#define HALMAC_MINOR_VER 0x0005 /* minor version */
+#define HALMAC_PATCH_VER 0x0000 /* patch version */
+
+#include "halmac_2_platform.h"
+#include "halmac_type.h"
+
+#include "halmac_usb_reg.h"
+#include "halmac_sdio_reg.h"
+#include "halmac_pcie_reg.h"
+
+#include "halmac_bit2.h"
+#include "halmac_reg2.h"
+
+#include "halmac_tx_desc_nic.h"
+#include "halmac_rx_desc_nic.h"
+#include "halmac_tx_bd_nic.h"
+#include "halmac_rx_bd_nic.h"
+#include "halmac_fw_offload_c2h_nic.h"
+#include "halmac_fw_offload_h2c_nic.h"
+#include "halmac_h2c_extra_info_nic.h"
+#include "halmac_original_c2h_nic.h"
+#include "halmac_original_h2c_nic.h"
+
+#include "halmac_tx_desc_chip.h"
+#include "halmac_rx_desc_chip.h"
+#include "halmac_tx_bd_chip.h"
+#include "halmac_rx_bd_chip.h"
+#include "halmac_88xx/halmac_88xx_cfg.h"
+
+#include "halmac_88xx/halmac_8822b/halmac_8822b_cfg.h"
+#include "halmac_reg_8822b.h"
+#include "halmac_bit_8822b.h"
+
+enum halmac_ret_status
+halmac_init_adapter(void *driver_adapter,
+		    struct halmac_platform_api *halmac_platform_api,
+		    enum halmac_interface halmac_interface,
+		    struct halmac_adapter **pp_halmac_adapter,
+		    struct halmac_api **pp_halmac_api);
+
+enum halmac_ret_status
+halmac_deinit_adapter(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status halmac_halt_api(struct halmac_adapter *halmac_adapter);
+
+enum halmac_ret_status halmac_get_version(struct halmac_ver *version);
+
+#endif
diff --git a/drivers/staging/rtlwifi/halmac/halmac_bit2.h b/drivers/staging/rtlwifi/halmac/halmac_bit2.h
new file mode 100644
index 000000000000..1c7fe5d7df64
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_bit2.h
@@ -0,0 +1,13407 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef __RTL_WLAN_BITDEF_H__
+#define __RTL_WLAN_BITDEF_H__
+
+/*-------------------------Modification Log-----------------------------------
+ *	Base on MAC_Register.doc SVN391
+ *-------------------------Modification Log-----------------------------------
+ */
+
+/*--------------------------Include File--------------------------------------*/
+/*--------------------------Include File--------------------------------------*/
+
+/* 3 ============Programming guide Start===================== */
+/*
+ *	1. For all bit define, it should be prefixed by "BIT_"
+ *	2. For all bit mask, it should be prefixed by "BIT_MASK_"
+ *	3. For all bit shift, it should be prefixed by "BIT_SHIFT_"
+ *	4. For other case, prefix is not needed
+ *
+ * Example:
+ * #define BIT_SHIFT_MAX_TXDMA		16
+ * #define BIT_MASK_MAX_TXDMA		0x7
+ * #define BIT_MAX_TXDMA(x)		\
+ *			(((x) & BIT_MASK_MAX_TXDMA) << BIT_SHIFT_MAX_TXDMA)
+ * #define BIT_GET_MAX_TXDMA(x)		\
+ *			(((x) >> BIT_SHIFT_MAX_TXDMA) & BIT_MASK_MAX_TXDMA)
+ *
+ */
+/* 3 ============Programming guide End===================== */
+
+#define CPU_OPT_WIDTH 0x1F
+
+#define BIT_SHIFT_WATCH_DOG_RECORD_V1 10
+#define BIT_MASK_WATCH_DOG_RECORD_V1 0x3fff
+#define BIT_WATCH_DOG_RECORD_V1(x)                                             \
+	(((x) & BIT_MASK_WATCH_DOG_RECORD_V1) << BIT_SHIFT_WATCH_DOG_RECORD_V1)
+#define BIT_GET_WATCH_DOG_RECORD_V1(x)                                         \
+	(((x) >> BIT_SHIFT_WATCH_DOG_RECORD_V1) & BIT_MASK_WATCH_DOG_RECORD_V1)
+
+#define BIT_R_IO_TIMEOUT_FLAG_V1 BIT(9)
+
+#define BIT_ISO_MD2PP BIT(0)
+
+#define BIT_SHIFT_R_WMAC_IPV6_MYIPAD 0
+#define BIT_MASK_R_WMAC_IPV6_MYIPAD 0xffffffffffffffffffffffffffffffffL
+#define BIT_R_WMAC_IPV6_MYIPAD(x)                                              \
+	(((x) & BIT_MASK_R_WMAC_IPV6_MYIPAD) << BIT_SHIFT_R_WMAC_IPV6_MYIPAD)
+#define BIT_GET_R_WMAC_IPV6_MYIPAD(x)                                          \
+	(((x) >> BIT_SHIFT_R_WMAC_IPV6_MYIPAD) & BIT_MASK_R_WMAC_IPV6_MYIPAD)
+
+/* 2 REG_SDIO_TX_CTRL			(Offset 0x10250000) */
+
+#define BIT_SHIFT_SDIO_INT_TIMEOUT 16
+#define BIT_MASK_SDIO_INT_TIMEOUT 0xffff
+#define BIT_SDIO_INT_TIMEOUT(x)                                                \
+	(((x) & BIT_MASK_SDIO_INT_TIMEOUT) << BIT_SHIFT_SDIO_INT_TIMEOUT)
+#define BIT_GET_SDIO_INT_TIMEOUT(x)                                            \
+	(((x) >> BIT_SHIFT_SDIO_INT_TIMEOUT) & BIT_MASK_SDIO_INT_TIMEOUT)
+
+/* 2 REG_SYS_ISO_CTRL			(Offset 0x0000) */
+
+#define BIT_PWC_EV12V BIT(15)
+
+/* 2 REG_SDIO_TX_CTRL			(Offset 0x10250000) */
+
+#define BIT_IO_ERR_STATUS BIT(15)
+
+/* 2 REG_SYS_ISO_CTRL			(Offset 0x0000) */
+
+#define BIT_PWC_EV25V BIT(14)
+
+/* 2 REG_SYS_ISO_CTRL			(Offset 0x0000) */
+
+#define BIT_PA33V_EN BIT(13)
+#define BIT_PA12V_EN BIT(12)
+
+/* 2 REG_SYS_ISO_CTRL			(Offset 0x0000) */
+
+#define BIT_UA33V_EN BIT(11)
+#define BIT_UA12V_EN BIT(10)
+
+/* 2 REG_SYS_ISO_CTRL			(Offset 0x0000) */
+
+#define BIT_ISO_RFDIO BIT(9)
+
+/* 2 REG_SDIO_TX_CTRL			(Offset 0x10250000) */
+
+#define BIT_REPLY_ERRCRC_IN_DATA BIT(9)
+
+/* 2 REG_SYS_ISO_CTRL			(Offset 0x0000) */
+
+#define BIT_ISO_EB2CORE BIT(8)
+
+/* 2 REG_SDIO_TX_CTRL			(Offset 0x10250000) */
+
+#define BIT_EN_CMD53_OVERLAP BIT(8)
+
+/* 2 REG_SYS_ISO_CTRL			(Offset 0x0000) */
+
+#define BIT_ISO_DIOE BIT(7)
+
+/* 2 REG_SDIO_TX_CTRL			(Offset 0x10250000) */
+
+#define BIT_REPLY_ERR_IN_R5 BIT(7)
+
+/* 2 REG_SYS_ISO_CTRL			(Offset 0x0000) */
+
+#define BIT_ISO_WLPON2PP BIT(6)
+
+/* 2 REG_SDIO_TX_CTRL			(Offset 0x10250000) */
+
+#define BIT_R18A_EN BIT(6)
+
+/* 2 REG_SYS_ISO_CTRL			(Offset 0x0000) */
+
+#define BIT_ISO_IP2MAC_WA2PP BIT(5)
+
+/* 2 REG_SDIO_TX_CTRL			(Offset 0x10250000) */
+
+#define BIT_INIT_CMD_EN BIT(5)
+
+/* 2 REG_SYS_ISO_CTRL			(Offset 0x0000) */
+
+#define BIT_ISO_PD2CORE BIT(4)
+
+/* 2 REG_SYS_ISO_CTRL			(Offset 0x0000) */
+
+#define BIT_ISO_PA2PCIE BIT(3)
+
+/* 2 REG_SYS_ISO_CTRL			(Offset 0x0000) */
+
+#define BIT_ISO_UD2CORE BIT(2)
+
+/* 2 REG_SDIO_TX_CTRL			(Offset 0x10250000) */
+
+#define BIT_EN_RXDMA_MASK_INT BIT(2)
+
+/* 2 REG_SYS_ISO_CTRL			(Offset 0x0000) */
+
+#define BIT_ISO_UA2USB BIT(1)
+
+/* 2 REG_SDIO_TX_CTRL			(Offset 0x10250000) */
+
+#define BIT_EN_MASK_TIMER BIT(1)
+
+/* 2 REG_SYS_ISO_CTRL			(Offset 0x0000) */
+
+#define BIT_ISO_WD2PP BIT(0)
+
+/* 2 REG_SDIO_TX_CTRL			(Offset 0x10250000) */
+
+#define BIT_CMD_ERR_STOP_INT_EN BIT(0)
+
+/* 2 REG_SYS_FUNC_EN				(Offset 0x0002) */
+
+#define BIT_FEN_MREGEN BIT(15)
+#define BIT_FEN_HWPDN BIT(14)
+
+/* 2 REG_SYS_FUNC_EN				(Offset 0x0002) */
+
+#define BIT_EN_25_1 BIT(13)
+
+/* 2 REG_SYS_FUNC_EN				(Offset 0x0002) */
+
+#define BIT_FEN_ELDR BIT(12)
+#define BIT_FEN_DCORE BIT(11)
+#define BIT_FEN_CPUEN BIT(10)
+#define BIT_FEN_DIOE BIT(9)
+#define BIT_FEN_PCIED BIT(8)
+#define BIT_FEN_PPLL BIT(7)
+#define BIT_FEN_PCIEA BIT(6)
+#define BIT_FEN_DIO_PCIE BIT(5)
+#define BIT_FEN_USBD BIT(4)
+#define BIT_FEN_UPLL BIT(3)
+#define BIT_FEN_USBA BIT(2)
+
+/* 2 REG_SYS_FUNC_EN				(Offset 0x0002) */
+
+#define BIT_FEN_BB_GLB_RSTN BIT(1)
+#define BIT_FEN_BBRSTB BIT(0)
+
+/* 2 REG_SYS_PW_CTRL				(Offset 0x0004) */
+
+#define BIT_SOP_EABM BIT(31)
+
+/* 2 REG_SYS_PW_CTRL				(Offset 0x0004) */
+
+#define BIT_SOP_ACKF BIT(30)
+#define BIT_SOP_ERCK BIT(29)
+
+/* 2 REG_SYS_PW_CTRL				(Offset 0x0004) */
+
+#define BIT_SOP_ESWR BIT(28)
+
+/* 2 REG_SYS_PW_CTRL				(Offset 0x0004) */
+
+#define BIT_SOP_PWMM BIT(27)
+
+/* 2 REG_SYS_PW_CTRL				(Offset 0x0004) */
+
+#define BIT_SOP_EECK BIT(26)
+
+/* 2 REG_SYS_PW_CTRL				(Offset 0x0004) */
+
+#define BIT_SOP_EXTL BIT(24)
+
+/* 2 REG_SYS_PW_CTRL				(Offset 0x0004) */
+
+#define BIT_SYM_OP_RING_12M BIT(22)
+
+/* 2 REG_SYS_PW_CTRL				(Offset 0x0004) */
+
+#define BIT_ROP_SWPR BIT(21)
+
+/* 2 REG_SYS_PW_CTRL				(Offset 0x0004) */
+
+#define BIT_DIS_HW_LPLDM BIT(20)
+
+/* 2 REG_SYS_PW_CTRL				(Offset 0x0004) */
+
+#define BIT_OPT_SWRST_WLMCU BIT(19)
+#define BIT_RDY_SYSPWR BIT(17)
+
+/* 2 REG_SYS_PW_CTRL				(Offset 0x0004) */
+
+#define BIT_EN_WLON BIT(16)
+
+/* 2 REG_SYS_PW_CTRL				(Offset 0x0004) */
+
+#define BIT_APDM_HPDN BIT(15)
+
+/* 2 REG_SYS_PW_CTRL				(Offset 0x0004) */
+
+#define BIT_AFSM_PCIE_SUS_EN BIT(12)
+
+/* 2 REG_SYS_PW_CTRL				(Offset 0x0004) */
+
+#define BIT_AFSM_WLSUS_EN BIT(11)
+
+/* 2 REG_SYS_PW_CTRL				(Offset 0x0004) */
+
+#define BIT_APFM_SWLPS BIT(10)
+
+/* 2 REG_SYS_PW_CTRL				(Offset 0x0004) */
+
+#define BIT_APFM_OFFMAC BIT(9)
+#define BIT_APFN_ONMAC BIT(8)
+
+/* 2 REG_SYS_PW_CTRL				(Offset 0x0004) */
+
+#define BIT_CHIP_PDN_EN BIT(7)
+
+/* 2 REG_SYS_PW_CTRL				(Offset 0x0004) */
+
+#define BIT_RDY_MACDIS BIT(6)
+
+/* 2 REG_SYS_PW_CTRL				(Offset 0x0004) */
+
+#define BIT_RING_CLK_12M_EN BIT(4)
+
+/* 2 REG_SYS_PW_CTRL				(Offset 0x0004) */
+
+#define BIT_PFM_WOWL BIT(3)
+
+/* 2 REG_SYS_PW_CTRL				(Offset 0x0004) */
+
+#define BIT_PFM_LDKP BIT(2)
+
+/* 2 REG_SYS_PW_CTRL				(Offset 0x0004) */
+
+#define BIT_WL_HCI_ALD BIT(1)
+
+/* 2 REG_SYS_PW_CTRL				(Offset 0x0004) */
+
+#define BIT_PFM_LDALL BIT(0)
+
+/* 2 REG_SYS_CLK_CTRL			(Offset 0x0008) */
+
+#define BIT_LDO_DUMMY BIT(15)
+
+/* 2 REG_SYS_CLK_CTRL			(Offset 0x0008) */
+
+#define BIT_CPU_CLK_EN BIT(14)
+
+/* 2 REG_SYS_CLK_CTRL			(Offset 0x0008) */
+
+#define BIT_SYMREG_CLK_EN BIT(13)
+
+/* 2 REG_SYS_CLK_CTRL			(Offset 0x0008) */
+
+#define BIT_HCI_CLK_EN BIT(12)
+
+/* 2 REG_SYS_CLK_CTRL			(Offset 0x0008) */
+
+#define BIT_MAC_CLK_EN BIT(11)
+#define BIT_SEC_CLK_EN BIT(10)
+
+/* 2 REG_SYS_CLK_CTRL			(Offset 0x0008) */
+
+#define BIT_PHY_SSC_RSTB BIT(9)
+#define BIT_EXT_32K_EN BIT(8)
+
+/* 2 REG_SYS_CLK_CTRL			(Offset 0x0008) */
+
+#define BIT_WL_CLK_TEST BIT(7)
+#define BIT_OP_SPS_PWM_EN BIT(6)
+
+/* 2 REG_SYS_CLK_CTRL			(Offset 0x0008) */
+
+#define BIT_LOADER_CLK_EN BIT(5)
+#define BIT_MACSLP BIT(4)
+#define BIT_WAKEPAD_EN BIT(3)
+#define BIT_ROMD16V_EN BIT(2)
+
+/* 2 REG_SYS_CLK_CTRL			(Offset 0x0008) */
+
+#define BIT_CKANA12M_EN BIT(1)
+
+/* 2 REG_SYS_CLK_CTRL			(Offset 0x0008) */
+
+#define BIT_CNTD16V_EN BIT(0)
+
+/* 2 REG_SYS_EEPROM_CTRL			(Offset 0x000A) */
+
+#define BIT_SHIFT_VPDIDX 8
+#define BIT_MASK_VPDIDX 0xff
+#define BIT_VPDIDX(x) (((x) & BIT_MASK_VPDIDX) << BIT_SHIFT_VPDIDX)
+#define BIT_GET_VPDIDX(x) (((x) >> BIT_SHIFT_VPDIDX) & BIT_MASK_VPDIDX)
+
+#define BIT_SHIFT_EEM1_0 6
+#define BIT_MASK_EEM1_0 0x3
+#define BIT_EEM1_0(x) (((x) & BIT_MASK_EEM1_0) << BIT_SHIFT_EEM1_0)
+#define BIT_GET_EEM1_0(x) (((x) >> BIT_SHIFT_EEM1_0) & BIT_MASK_EEM1_0)
+
+#define BIT_AUTOLOAD_SUS BIT(5)
+
+/* 2 REG_SYS_EEPROM_CTRL			(Offset 0x000A) */
+
+#define BIT_EERPOMSEL BIT(4)
+
+/* 2 REG_SYS_EEPROM_CTRL			(Offset 0x000A) */
+
+#define BIT_EECS_V1 BIT(3)
+#define BIT_EESK_V1 BIT(2)
+#define BIT_EEDI_V1 BIT(1)
+#define BIT_EEDO_V1 BIT(0)
+
+/* 2 REG_EE_VPD				(Offset 0x000C) */
+
+#define BIT_SHIFT_VPD_DATA 0
+#define BIT_MASK_VPD_DATA 0xffffffffL
+#define BIT_VPD_DATA(x) (((x) & BIT_MASK_VPD_DATA) << BIT_SHIFT_VPD_DATA)
+#define BIT_GET_VPD_DATA(x) (((x) >> BIT_SHIFT_VPD_DATA) & BIT_MASK_VPD_DATA)
+
+/* 2 REG_SYS_SWR_CTRL1			(Offset 0x0010) */
+
+#define BIT_C2_L_BIT0 BIT(31)
+
+/* 2 REG_SYS_SWR_CTRL1			(Offset 0x0010) */
+
+#define BIT_SHIFT_C1_L 29
+#define BIT_MASK_C1_L 0x3
+#define BIT_C1_L(x) (((x) & BIT_MASK_C1_L) << BIT_SHIFT_C1_L)
+#define BIT_GET_C1_L(x) (((x) >> BIT_SHIFT_C1_L) & BIT_MASK_C1_L)
+
+/* 2 REG_SYS_SWR_CTRL1			(Offset 0x0010) */
+
+#define BIT_SHIFT_REG_FREQ_L 25
+#define BIT_MASK_REG_FREQ_L 0x7
+#define BIT_REG_FREQ_L(x) (((x) & BIT_MASK_REG_FREQ_L) << BIT_SHIFT_REG_FREQ_L)
+#define BIT_GET_REG_FREQ_L(x)                                                  \
+	(((x) >> BIT_SHIFT_REG_FREQ_L) & BIT_MASK_REG_FREQ_L)
+
+#define BIT_REG_EN_DUTY BIT(24)
+
+/* 2 REG_SYS_SWR_CTRL1			(Offset 0x0010) */
+
+#define BIT_SHIFT_REG_MODE 22
+#define BIT_MASK_REG_MODE 0x3
+#define BIT_REG_MODE(x) (((x) & BIT_MASK_REG_MODE) << BIT_SHIFT_REG_MODE)
+#define BIT_GET_REG_MODE(x) (((x) >> BIT_SHIFT_REG_MODE) & BIT_MASK_REG_MODE)
+
+/* 2 REG_SYS_SWR_CTRL1			(Offset 0x0010) */
+
+#define BIT_REG_EN_SP BIT(21)
+#define BIT_REG_AUTO_L BIT(20)
+
+/* 2 REG_SYS_SWR_CTRL1			(Offset 0x0010) */
+
+#define BIT_SW18_SELD_BIT0 BIT(19)
+
+/* 2 REG_SYS_SWR_CTRL1			(Offset 0x0010) */
+
+#define BIT_SW18_POWOCP BIT(18)
+
+/* 2 REG_SYS_SWR_CTRL1			(Offset 0x0010) */
+
+#define BIT_SHIFT_OCP_L1 15
+#define BIT_MASK_OCP_L1 0x7
+#define BIT_OCP_L1(x) (((x) & BIT_MASK_OCP_L1) << BIT_SHIFT_OCP_L1)
+#define BIT_GET_OCP_L1(x) (((x) >> BIT_SHIFT_OCP_L1) & BIT_MASK_OCP_L1)
+
+/* 2 REG_SYS_SWR_CTRL1			(Offset 0x0010) */
+
+#define BIT_SHIFT_CF_L 13
+#define BIT_MASK_CF_L 0x3
+#define BIT_CF_L(x) (((x) & BIT_MASK_CF_L) << BIT_SHIFT_CF_L)
+#define BIT_GET_CF_L(x) (((x) >> BIT_SHIFT_CF_L) & BIT_MASK_CF_L)
+
+/* 2 REG_SYS_SWR_CTRL1			(Offset 0x0010) */
+
+#define BIT_SW18_FPWM BIT(11)
+
+/* 2 REG_SYS_SWR_CTRL1			(Offset 0x0010) */
+
+#define BIT_SW18_SWEN BIT(9)
+#define BIT_SW18_LDEN BIT(8)
+#define BIT_MAC_ID_EN BIT(7)
+
+/* 2 REG_SYS_SWR_CTRL1			(Offset 0x0010) */
+
+#define BIT_AFE_BGEN BIT(0)
+
+/* 2 REG_SYS_SWR_CTRL2			(Offset 0x0014) */
+
+#define BIT_POW_ZCD_L BIT(31)
+
+/* 2 REG_SDIO_HIMR				(Offset 0x10250014) */
+
+#define BIT_SDIO_CRCERR_MSK BIT(31)
+
+/* 2 REG_SYS_SWR_CTRL2			(Offset 0x0014) */
+
+#define BIT_AUTOZCD_L BIT(30)
+#define BIT_SDIO_HSISR3_IND_MSK BIT(30)
+#define BIT_SDIO_HSISR2_IND_MSK BIT(29)
+
+/* 2 REG_SYS_SWR_CTRL2			(Offset 0x0014) */
+
+#define BIT_SHIFT_REG_DELAY 28
+#define BIT_MASK_REG_DELAY 0x3
+#define BIT_REG_DELAY(x) (((x) & BIT_MASK_REG_DELAY) << BIT_SHIFT_REG_DELAY)
+#define BIT_GET_REG_DELAY(x) (((x) >> BIT_SHIFT_REG_DELAY) & BIT_MASK_REG_DELAY)
+
+/* 2 REG_SDIO_HIMR				(Offset 0x10250014) */
+
+#define BIT_SDIO_HEISR_IND_MSK BIT(28)
+
+/* 2 REG_SDIO_HIMR				(Offset 0x10250014) */
+
+#define BIT_SDIO_CTWEND_MSK BIT(27)
+#define BIT_SDIO_ATIMEND_E_MSK BIT(26)
+
+/* 2 REG_SDIO_HIMR				(Offset 0x10250014) */
+
+#define BIT_SDIIO_ATIMEND_MSK BIT(25)
+
+/* 2 REG_SDIO_HIMR				(Offset 0x10250014) */
+
+#define BIT_SDIO_OCPINT_MSK BIT(24)
+
+/* 2 REG_SYS_SWR_CTRL2			(Offset 0x0014) */
+
+#define BIT_SHIFT_V15ADJ_L1_V1 24
+#define BIT_MASK_V15ADJ_L1_V1 0x7
+#define BIT_V15ADJ_L1_V1(x)                                                    \
+	(((x) & BIT_MASK_V15ADJ_L1_V1) << BIT_SHIFT_V15ADJ_L1_V1)
+#define BIT_GET_V15ADJ_L1_V1(x)                                                \
+	(((x) >> BIT_SHIFT_V15ADJ_L1_V1) & BIT_MASK_V15ADJ_L1_V1)
+
+/* 2 REG_SDIO_HIMR				(Offset 0x10250014) */
+
+#define BIT_SDIO_PSTIMEOUT_MSK BIT(23)
+
+/* 2 REG_SDIO_HIMR				(Offset 0x10250014) */
+
+#define BIT_SDIO_GTINT4_MSK BIT(22)
+
+/* 2 REG_SDIO_HIMR				(Offset 0x10250014) */
+
+#define BIT_SDIO_GTINT3_MSK BIT(21)
+
+/* 2 REG_SDIO_HIMR				(Offset 0x10250014) */
+
+#define BIT_SDIO_HSISR_IND_MSK BIT(20)
+
+/* 2 REG_SYS_SWR_CTRL2			(Offset 0x0014) */
+
+#define BIT_SHIFT_VOL_L1_V1 20
+#define BIT_MASK_VOL_L1_V1 0xf
+#define BIT_VOL_L1_V1(x) (((x) & BIT_MASK_VOL_L1_V1) << BIT_SHIFT_VOL_L1_V1)
+#define BIT_GET_VOL_L1_V1(x) (((x) >> BIT_SHIFT_VOL_L1_V1) & BIT_MASK_VOL_L1_V1)
+
+/* 2 REG_SDIO_HIMR				(Offset 0x10250014) */
+
+#define BIT_SDIO_CPWM2_MSK BIT(19)
+
+/* 2 REG_SDIO_HIMR				(Offset 0x10250014) */
+
+#define BIT_SDIO_CPWM1_MSK BIT(18)
+
+/* 2 REG_SDIO_HIMR				(Offset 0x10250014) */
+
+#define BIT_SDIO_C2HCMD_INT_MSK BIT(17)
+
+/* 2 REG_SYS_SWR_CTRL2			(Offset 0x0014) */
+
+#define BIT_SHIFT_IN_L1_V1 17
+#define BIT_MASK_IN_L1_V1 0x7
+#define BIT_IN_L1_V1(x) (((x) & BIT_MASK_IN_L1_V1) << BIT_SHIFT_IN_L1_V1)
+#define BIT_GET_IN_L1_V1(x) (((x) >> BIT_SHIFT_IN_L1_V1) & BIT_MASK_IN_L1_V1)
+
+/* 2 REG_SDIO_HIMR				(Offset 0x10250014) */
+
+#define BIT_SDIO_BCNERLY_INT_MSK BIT(16)
+
+/* 2 REG_SYS_SWR_CTRL2			(Offset 0x0014) */
+
+#define BIT_SHIFT_TBOX_L1 15
+#define BIT_MASK_TBOX_L1 0x3
+#define BIT_TBOX_L1(x) (((x) & BIT_MASK_TBOX_L1) << BIT_SHIFT_TBOX_L1)
+#define BIT_GET_TBOX_L1(x) (((x) >> BIT_SHIFT_TBOX_L1) & BIT_MASK_TBOX_L1)
+
+/* 2 REG_SYS_SWR_CTRL2			(Offset 0x0014) */
+
+#define BIT_SW18_SEL BIT(13)
+
+/* 2 REG_SYS_SWR_CTRL2			(Offset 0x0014) */
+
+#define BIT_SW18_SD BIT(10)
+
+/* 2 REG_SDIO_HIMR				(Offset 0x10250014) */
+
+#define BIT_SDIO_TXBCNERR_MSK BIT(7)
+
+/* 2 REG_SYS_SWR_CTRL2			(Offset 0x0014) */
+
+#define BIT_SHIFT_R3_L 7
+#define BIT_MASK_R3_L 0x3
+#define BIT_R3_L(x) (((x) & BIT_MASK_R3_L) << BIT_SHIFT_R3_L)
+#define BIT_GET_R3_L(x) (((x) >> BIT_SHIFT_R3_L) & BIT_MASK_R3_L)
+
+/* 2 REG_SDIO_HIMR				(Offset 0x10250014) */
+
+#define BIT_SDIO_TXBCNOK_MSK BIT(6)
+
+/* 2 REG_SYS_SWR_CTRL2			(Offset 0x0014) */
+
+#define BIT_SHIFT_SW18_R2 5
+#define BIT_MASK_SW18_R2 0x3
+#define BIT_SW18_R2(x) (((x) & BIT_MASK_SW18_R2) << BIT_SHIFT_SW18_R2)
+#define BIT_GET_SW18_R2(x) (((x) >> BIT_SHIFT_SW18_R2) & BIT_MASK_SW18_R2)
+
+/* 2 REG_SDIO_HIMR				(Offset 0x10250014) */
+
+#define BIT_SDIO_RXFOVW_MSK BIT(5)
+#define BIT_SDIO_TXFOVW_MSK BIT(4)
+
+/* 2 REG_SYS_SWR_CTRL2			(Offset 0x0014) */
+
+#define BIT_SHIFT_SW18_R1 3
+#define BIT_MASK_SW18_R1 0x3
+#define BIT_SW18_R1(x) (((x) & BIT_MASK_SW18_R1) << BIT_SHIFT_SW18_R1)
+#define BIT_GET_SW18_R1(x) (((x) >> BIT_SHIFT_SW18_R1) & BIT_MASK_SW18_R1)
+
+/* 2 REG_SDIO_HIMR				(Offset 0x10250014) */
+
+#define BIT_SDIO_RXERR_MSK BIT(3)
+#define BIT_SDIO_TXERR_MSK BIT(2)
+
+/* 2 REG_SDIO_HIMR				(Offset 0x10250014) */
+
+#define BIT_SDIO_AVAL_MSK BIT(1)
+
+/* 2 REG_SYS_SWR_CTRL2			(Offset 0x0014) */
+
+#define BIT_SHIFT_C3_L_C3 1
+#define BIT_MASK_C3_L_C3 0x3
+#define BIT_C3_L_C3(x) (((x) & BIT_MASK_C3_L_C3) << BIT_SHIFT_C3_L_C3)
+#define BIT_GET_C3_L_C3(x) (((x) >> BIT_SHIFT_C3_L_C3) & BIT_MASK_C3_L_C3)
+
+/* 2 REG_SDIO_HIMR				(Offset 0x10250014) */
+
+#define BIT_RX_REQUEST_MSK BIT(0)
+
+/* 2 REG_SYS_SWR_CTRL2			(Offset 0x0014) */
+
+#define BIT_C2_L_BIT1 BIT(0)
+
+/* 2 REG_SYS_SWR_CTRL3			(Offset 0x0018) */
+
+#define BIT_SPS18_OCP_DIS BIT(31)
+
+/* 2 REG_SDIO_HISR				(Offset 0x10250018) */
+
+#define BIT_SDIO_CRCERR BIT(31)
+
+/* 2 REG_SDIO_HISR				(Offset 0x10250018) */
+
+#define BIT_SDIO_HSISR3_IND BIT(30)
+#define BIT_SDIO_HSISR2_IND BIT(29)
+#define BIT_SDIO_HEISR_IND BIT(28)
+
+/* 2 REG_SDIO_HISR				(Offset 0x10250018) */
+
+#define BIT_SDIO_CTWEND BIT(27)
+#define BIT_SDIO_ATIMEND_E BIT(26)
+#define BIT_SDIO_ATIMEND BIT(25)
+#define BIT_SDIO_OCPINT BIT(24)
+#define BIT_SDIO_PSTIMEOUT BIT(23)
+#define BIT_SDIO_GTINT4 BIT(22)
+#define BIT_SDIO_GTINT3 BIT(21)
+#define BIT_SDIO_HSISR_IND BIT(20)
+#define BIT_SDIO_CPWM2 BIT(19)
+#define BIT_SDIO_CPWM1 BIT(18)
+#define BIT_SDIO_C2HCMD_INT BIT(17)
+
+/* 2 REG_SYS_SWR_CTRL3			(Offset 0x0018) */
+
+#define BIT_SHIFT_SPS18_OCP_TH 16
+#define BIT_MASK_SPS18_OCP_TH 0x7fff
+#define BIT_SPS18_OCP_TH(x)                                                    \
+	(((x) & BIT_MASK_SPS18_OCP_TH) << BIT_SHIFT_SPS18_OCP_TH)
+#define BIT_GET_SPS18_OCP_TH(x)                                                \
+	(((x) >> BIT_SHIFT_SPS18_OCP_TH) & BIT_MASK_SPS18_OCP_TH)
+
+/* 2 REG_SDIO_HISR				(Offset 0x10250018) */
+
+#define BIT_SDIO_BCNERLY_INT BIT(16)
+#define BIT_SDIO_TXBCNERR BIT(7)
+#define BIT_SDIO_TXBCNOK BIT(6)
+#define BIT_SDIO_RXFOVW BIT(5)
+#define BIT_SDIO_TXFOVW BIT(4)
+#define BIT_SDIO_RXERR BIT(3)
+#define BIT_SDIO_TXERR BIT(2)
+#define BIT_SDIO_AVAL BIT(1)
+
+/* 2 REG_SYS_SWR_CTRL3			(Offset 0x0018) */
+
+#define BIT_SHIFT_OCP_WINDOW 0
+#define BIT_MASK_OCP_WINDOW 0xffff
+#define BIT_OCP_WINDOW(x) (((x) & BIT_MASK_OCP_WINDOW) << BIT_SHIFT_OCP_WINDOW)
+#define BIT_GET_OCP_WINDOW(x)                                                  \
+	(((x) >> BIT_SHIFT_OCP_WINDOW) & BIT_MASK_OCP_WINDOW)
+
+/* 2 REG_SDIO_HISR				(Offset 0x10250018) */
+
+#define BIT_RX_REQUEST BIT(0)
+
+/* 2 REG_RSV_CTRL				(Offset 0x001C) */
+
+#define BIT_HREG_DBG BIT(23)
+
+/* 2 REG_RSV_CTRL				(Offset 0x001C) */
+
+#define BIT_WLMCUIOIF BIT(8)
+
+/* 2 REG_RSV_CTRL				(Offset 0x001C) */
+
+#define BIT_LOCK_ALL_EN BIT(7)
+
+/* 2 REG_RSV_CTRL				(Offset 0x001C) */
+
+#define BIT_R_DIS_PRST BIT(6)
+
+/* 2 REG_RSV_CTRL				(Offset 0x001C) */
+
+#define BIT_WLOCK_1C_B6 BIT(5)
+
+/* 2 REG_RSV_CTRL				(Offset 0x001C) */
+
+#define BIT_WLOCK_40 BIT(4)
+#define BIT_WLOCK_08 BIT(3)
+#define BIT_WLOCK_04 BIT(2)
+#define BIT_WLOCK_00 BIT(1)
+#define BIT_WLOCK_ALL BIT(0)
+
+/* 2 REG_SDIO_RX_REQ_LEN			(Offset 0x1025001C) */
+
+#define BIT_SHIFT_RX_REQ_LEN_V1 0
+#define BIT_MASK_RX_REQ_LEN_V1 0x3ffff
+#define BIT_RX_REQ_LEN_V1(x)                                                   \
+	(((x) & BIT_MASK_RX_REQ_LEN_V1) << BIT_SHIFT_RX_REQ_LEN_V1)
+#define BIT_GET_RX_REQ_LEN_V1(x)                                               \
+	(((x) >> BIT_SHIFT_RX_REQ_LEN_V1) & BIT_MASK_RX_REQ_LEN_V1)
+
+/* 2 REG_RF_CTRL				(Offset 0x001F) */
+
+#define BIT_RF_SDMRSTB BIT(2)
+
+/* 2 REG_RF_CTRL				(Offset 0x001F) */
+
+#define BIT_RF_RSTB BIT(1)
+
+/* 2 REG_RF_CTRL				(Offset 0x001F) */
+
+#define BIT_RF_EN BIT(0)
+
+/* 2 REG_SDIO_FREE_TXPG_SEQ_V1		(Offset 0x1025001F) */
+
+#define BIT_SHIFT_FREE_TXPG_SEQ 0
+#define BIT_MASK_FREE_TXPG_SEQ 0xff
+#define BIT_FREE_TXPG_SEQ(x)                                                   \
+	(((x) & BIT_MASK_FREE_TXPG_SEQ) << BIT_SHIFT_FREE_TXPG_SEQ)
+#define BIT_GET_FREE_TXPG_SEQ(x)                                               \
+	(((x) >> BIT_SHIFT_FREE_TXPG_SEQ) & BIT_MASK_FREE_TXPG_SEQ)
+
+/* 2 REG_AFE_LDO_CTRL			(Offset 0x0020) */
+
+#define BIT_SHIFT_LPLDH12_RSV 29
+#define BIT_MASK_LPLDH12_RSV 0x7
+#define BIT_LPLDH12_RSV(x)                                                     \
+	(((x) & BIT_MASK_LPLDH12_RSV) << BIT_SHIFT_LPLDH12_RSV)
+#define BIT_GET_LPLDH12_RSV(x)                                                 \
+	(((x) >> BIT_SHIFT_LPLDH12_RSV) & BIT_MASK_LPLDH12_RSV)
+
+/* 2 REG_AFE_LDO_CTRL			(Offset 0x0020) */
+
+#define BIT_LPLDH12_SLP BIT(28)
+
+#define BIT_SHIFT_LPLDH12_VADJ 24
+#define BIT_MASK_LPLDH12_VADJ 0xf
+#define BIT_LPLDH12_VADJ(x)                                                    \
+	(((x) & BIT_MASK_LPLDH12_VADJ) << BIT_SHIFT_LPLDH12_VADJ)
+#define BIT_GET_LPLDH12_VADJ(x)                                                \
+	(((x) >> BIT_SHIFT_LPLDH12_VADJ) & BIT_MASK_LPLDH12_VADJ)
+
+/* 2 REG_AFE_LDO_CTRL			(Offset 0x0020) */
+
+#define BIT_LDH12_EN BIT(16)
+
+/* 2 REG_SDIO_FREE_TXPG			(Offset 0x10250020) */
+
+#define BIT_SHIFT_MID_FREEPG_V1 16
+#define BIT_MASK_MID_FREEPG_V1 0xfff
+#define BIT_MID_FREEPG_V1(x)                                                   \
+	(((x) & BIT_MASK_MID_FREEPG_V1) << BIT_SHIFT_MID_FREEPG_V1)
+#define BIT_GET_MID_FREEPG_V1(x)                                               \
+	(((x) >> BIT_SHIFT_MID_FREEPG_V1) & BIT_MASK_MID_FREEPG_V1)
+
+/* 2 REG_AFE_LDO_CTRL			(Offset 0x0020) */
+
+#define BIT_WLBBOFF_BIG_PWC_EN BIT(14)
+#define BIT_WLBBOFF_SMALL_PWC_EN BIT(13)
+#define BIT_WLMACOFF_BIG_PWC_EN BIT(12)
+#define BIT_WLPON_PWC_EN BIT(11)
+
+/* 2 REG_AFE_LDO_CTRL			(Offset 0x0020) */
+
+#define BIT_POW_REGU_P1 BIT(10)
+
+/* 2 REG_AFE_LDO_CTRL			(Offset 0x0020) */
+
+#define BIT_LDOV12W_EN BIT(8)
+
+/* 2 REG_AFE_LDO_CTRL			(Offset 0x0020) */
+
+#define BIT_EX_XTAL_DRV_DIGI BIT(7)
+#define BIT_EX_XTAL_DRV_USB BIT(6)
+#define BIT_EX_XTAL_DRV_AFE BIT(5)
+
+/* 2 REG_AFE_LDO_CTRL			(Offset 0x0020) */
+
+#define BIT_EX_XTAL_DRV_RF2 BIT(4)
+
+/* 2 REG_AFE_LDO_CTRL			(Offset 0x0020) */
+
+#define BIT_EX_XTAL_DRV_RF1 BIT(3)
+#define BIT_POW_REGU_P0 BIT(2)
+
+/* 2 REG_AFE_LDO_CTRL			(Offset 0x0020) */
+
+#define BIT_POW_PLL_LDO BIT(0)
+
+/* 2 REG_SDIO_FREE_TXPG			(Offset 0x10250020) */
+
+#define BIT_SHIFT_HIQ_FREEPG_V1 0
+#define BIT_MASK_HIQ_FREEPG_V1 0xfff
+#define BIT_HIQ_FREEPG_V1(x)                                                   \
+	(((x) & BIT_MASK_HIQ_FREEPG_V1) << BIT_SHIFT_HIQ_FREEPG_V1)
+#define BIT_GET_HIQ_FREEPG_V1(x)                                               \
+	(((x) >> BIT_SHIFT_HIQ_FREEPG_V1) & BIT_MASK_HIQ_FREEPG_V1)
+
+/* 2 REG_AFE_CTRL1				(Offset 0x0024) */
+
+#define BIT_AGPIO_GPE BIT(31)
+
+/* 2 REG_AFE_CTRL1				(Offset 0x0024) */
+
+#define BIT_SHIFT_XTAL_CAP_XI 25
+#define BIT_MASK_XTAL_CAP_XI 0x3f
+#define BIT_XTAL_CAP_XI(x)                                                     \
+	(((x) & BIT_MASK_XTAL_CAP_XI) << BIT_SHIFT_XTAL_CAP_XI)
+#define BIT_GET_XTAL_CAP_XI(x)                                                 \
+	(((x) >> BIT_SHIFT_XTAL_CAP_XI) & BIT_MASK_XTAL_CAP_XI)
+
+/* 2 REG_AFE_CTRL1				(Offset 0x0024) */
+
+#define BIT_SHIFT_XTAL_DRV_DIGI 23
+#define BIT_MASK_XTAL_DRV_DIGI 0x3
+#define BIT_XTAL_DRV_DIGI(x)                                                   \
+	(((x) & BIT_MASK_XTAL_DRV_DIGI) << BIT_SHIFT_XTAL_DRV_DIGI)
+#define BIT_GET_XTAL_DRV_DIGI(x)                                               \
+	(((x) >> BIT_SHIFT_XTAL_DRV_DIGI) & BIT_MASK_XTAL_DRV_DIGI)
+
+#define BIT_XTAL_DRV_USB_BIT1 BIT(22)
+
+/* 2 REG_AFE_CTRL1				(Offset 0x0024) */
+
+#define BIT_SHIFT_MAC_CLK_SEL 20
+#define BIT_MASK_MAC_CLK_SEL 0x3
+#define BIT_MAC_CLK_SEL(x)                                                     \
+	(((x) & BIT_MASK_MAC_CLK_SEL) << BIT_SHIFT_MAC_CLK_SEL)
+#define BIT_GET_MAC_CLK_SEL(x)                                                 \
+	(((x) >> BIT_SHIFT_MAC_CLK_SEL) & BIT_MASK_MAC_CLK_SEL)
+
+/* 2 REG_AFE_CTRL1				(Offset 0x0024) */
+
+#define BIT_XTAL_DRV_USB_BIT0 BIT(19)
+
+/* 2 REG_AFE_CTRL1				(Offset 0x0024) */
+
+#define BIT_SHIFT_XTAL_DRV_AFE 17
+#define BIT_MASK_XTAL_DRV_AFE 0x3
+#define BIT_XTAL_DRV_AFE(x)                                                    \
+	(((x) & BIT_MASK_XTAL_DRV_AFE) << BIT_SHIFT_XTAL_DRV_AFE)
+#define BIT_GET_XTAL_DRV_AFE(x)                                                \
+	(((x) >> BIT_SHIFT_XTAL_DRV_AFE) & BIT_MASK_XTAL_DRV_AFE)
+
+/* 2 REG_SDIO_FREE_TXPG2			(Offset 0x10250024) */
+
+#define BIT_SHIFT_PUB_FREEPG_V1 16
+#define BIT_MASK_PUB_FREEPG_V1 0xfff
+#define BIT_PUB_FREEPG_V1(x)                                                   \
+	(((x) & BIT_MASK_PUB_FREEPG_V1) << BIT_SHIFT_PUB_FREEPG_V1)
+#define BIT_GET_PUB_FREEPG_V1(x)                                               \
+	(((x) >> BIT_SHIFT_PUB_FREEPG_V1) & BIT_MASK_PUB_FREEPG_V1)
+
+/* 2 REG_AFE_CTRL1				(Offset 0x0024) */
+
+#define BIT_SHIFT_XTAL_DRV_RF2 15
+#define BIT_MASK_XTAL_DRV_RF2 0x3
+#define BIT_XTAL_DRV_RF2(x)                                                    \
+	(((x) & BIT_MASK_XTAL_DRV_RF2) << BIT_SHIFT_XTAL_DRV_RF2)
+#define BIT_GET_XTAL_DRV_RF2(x)                                                \
+	(((x) >> BIT_SHIFT_XTAL_DRV_RF2) & BIT_MASK_XTAL_DRV_RF2)
+
+/* 2 REG_AFE_CTRL1				(Offset 0x0024) */
+
+#define BIT_SHIFT_XTAL_DRV_RF1 13
+#define BIT_MASK_XTAL_DRV_RF1 0x3
+#define BIT_XTAL_DRV_RF1(x)                                                    \
+	(((x) & BIT_MASK_XTAL_DRV_RF1) << BIT_SHIFT_XTAL_DRV_RF1)
+#define BIT_GET_XTAL_DRV_RF1(x)                                                \
+	(((x) >> BIT_SHIFT_XTAL_DRV_RF1) & BIT_MASK_XTAL_DRV_RF1)
+
+/* 2 REG_AFE_CTRL1				(Offset 0x0024) */
+
+#define BIT_XTAL_DELAY_DIGI BIT(12)
+
+/* 2 REG_AFE_CTRL1				(Offset 0x0024) */
+
+#define BIT_XTAL_DELAY_USB BIT(11)
+#define BIT_XTAL_DELAY_AFE BIT(10)
+
+/* 2 REG_AFE_CTRL1				(Offset 0x0024) */
+
+#define BIT_SHIFT_XTAL_LDO_VREF 7
+#define BIT_MASK_XTAL_LDO_VREF 0x7
+#define BIT_XTAL_LDO_VREF(x)                                                   \
+	(((x) & BIT_MASK_XTAL_LDO_VREF) << BIT_SHIFT_XTAL_LDO_VREF)
+#define BIT_GET_XTAL_LDO_VREF(x)                                               \
+	(((x) >> BIT_SHIFT_XTAL_LDO_VREF) & BIT_MASK_XTAL_LDO_VREF)
+
+/* 2 REG_AFE_CTRL1				(Offset 0x0024) */
+
+#define BIT_XTAL_XQSEL_RF BIT(6)
+#define BIT_XTAL_XQSEL BIT(5)
+
+/* 2 REG_AFE_CTRL1				(Offset 0x0024) */
+
+#define BIT_SHIFT_XTAL_GMN_V2 3
+#define BIT_MASK_XTAL_GMN_V2 0x3
+#define BIT_XTAL_GMN_V2(x)                                                     \
+	(((x) & BIT_MASK_XTAL_GMN_V2) << BIT_SHIFT_XTAL_GMN_V2)
+#define BIT_GET_XTAL_GMN_V2(x)                                                 \
+	(((x) >> BIT_SHIFT_XTAL_GMN_V2) & BIT_MASK_XTAL_GMN_V2)
+
+/* 2 REG_AFE_CTRL1				(Offset 0x0024) */
+
+#define BIT_SHIFT_XTAL_GMP_V2 1
+#define BIT_MASK_XTAL_GMP_V2 0x3
+#define BIT_XTAL_GMP_V2(x)                                                     \
+	(((x) & BIT_MASK_XTAL_GMP_V2) << BIT_SHIFT_XTAL_GMP_V2)
+#define BIT_GET_XTAL_GMP_V2(x)                                                 \
+	(((x) >> BIT_SHIFT_XTAL_GMP_V2) & BIT_MASK_XTAL_GMP_V2)
+
+/* 2 REG_AFE_CTRL1				(Offset 0x0024) */
+
+#define BIT_XTAL_EN BIT(0)
+
+/* 2 REG_SDIO_FREE_TXPG2			(Offset 0x10250024) */
+
+#define BIT_SHIFT_LOW_FREEPG_V1 0
+#define BIT_MASK_LOW_FREEPG_V1 0xfff
+#define BIT_LOW_FREEPG_V1(x)                                                   \
+	(((x) & BIT_MASK_LOW_FREEPG_V1) << BIT_SHIFT_LOW_FREEPG_V1)
+#define BIT_GET_LOW_FREEPG_V1(x)                                               \
+	(((x) >> BIT_SHIFT_LOW_FREEPG_V1) & BIT_MASK_LOW_FREEPG_V1)
+
+/* 2 REG_AFE_CTRL2				(Offset 0x0028) */
+
+#define BIT_SHIFT_REG_C3_V4 30
+#define BIT_MASK_REG_C3_V4 0x3
+#define BIT_REG_C3_V4(x) (((x) & BIT_MASK_REG_C3_V4) << BIT_SHIFT_REG_C3_V4)
+#define BIT_GET_REG_C3_V4(x) (((x) >> BIT_SHIFT_REG_C3_V4) & BIT_MASK_REG_C3_V4)
+
+#define BIT_REG_CP_BIT1 BIT(29)
+
+/* 2 REG_AFE_CTRL2				(Offset 0x0028) */
+
+#define BIT_SHIFT_REG_RS_V4 26
+#define BIT_MASK_REG_RS_V4 0x7
+#define BIT_REG_RS_V4(x) (((x) & BIT_MASK_REG_RS_V4) << BIT_SHIFT_REG_RS_V4)
+#define BIT_GET_REG_RS_V4(x) (((x) >> BIT_SHIFT_REG_RS_V4) & BIT_MASK_REG_RS_V4)
+
+/* 2 REG_SDIO_OQT_FREE_TXPG_V1		(Offset 0x10250028) */
+
+#define BIT_SHIFT_NOAC_OQT_FREEPG_V1 24
+#define BIT_MASK_NOAC_OQT_FREEPG_V1 0xff
+#define BIT_NOAC_OQT_FREEPG_V1(x)                                              \
+	(((x) & BIT_MASK_NOAC_OQT_FREEPG_V1) << BIT_SHIFT_NOAC_OQT_FREEPG_V1)
+#define BIT_GET_NOAC_OQT_FREEPG_V1(x)                                          \
+	(((x) >> BIT_SHIFT_NOAC_OQT_FREEPG_V1) & BIT_MASK_NOAC_OQT_FREEPG_V1)
+
+/* 2 REG_AFE_CTRL2				(Offset 0x0028) */
+
+#define BIT_SHIFT_REG__CS 24
+#define BIT_MASK_REG__CS 0x3
+#define BIT_REG__CS(x) (((x) & BIT_MASK_REG__CS) << BIT_SHIFT_REG__CS)
+#define BIT_GET_REG__CS(x) (((x) >> BIT_SHIFT_REG__CS) & BIT_MASK_REG__CS)
+
+/* 2 REG_AFE_CTRL2				(Offset 0x0028) */
+
+#define BIT_SHIFT_REG_CP_OFFSET 21
+#define BIT_MASK_REG_CP_OFFSET 0x7
+#define BIT_REG_CP_OFFSET(x)                                                   \
+	(((x) & BIT_MASK_REG_CP_OFFSET) << BIT_SHIFT_REG_CP_OFFSET)
+#define BIT_GET_REG_CP_OFFSET(x)                                               \
+	(((x) >> BIT_SHIFT_REG_CP_OFFSET) & BIT_MASK_REG_CP_OFFSET)
+
+/* 2 REG_AFE_CTRL2				(Offset 0x0028) */
+
+#define BIT_SHIFT_CP_BIAS 18
+#define BIT_MASK_CP_BIAS 0x7
+#define BIT_CP_BIAS(x) (((x) & BIT_MASK_CP_BIAS) << BIT_SHIFT_CP_BIAS)
+#define BIT_GET_CP_BIAS(x) (((x) >> BIT_SHIFT_CP_BIAS) & BIT_MASK_CP_BIAS)
+
+/* 2 REG_AFE_CTRL2				(Offset 0x0028) */
+
+#define BIT_REG_IDOUBLE_V2 BIT(17)
+
+/* 2 REG_AFE_CTRL2				(Offset 0x0028) */
+
+#define BIT_EN_SYN BIT(16)
+
+#define BIT_SHIFT_AC_OQT_FREEPG_V1 16
+#define BIT_MASK_AC_OQT_FREEPG_V1 0xff
+#define BIT_AC_OQT_FREEPG_V1(x)                                                \
+	(((x) & BIT_MASK_AC_OQT_FREEPG_V1) << BIT_SHIFT_AC_OQT_FREEPG_V1)
+#define BIT_GET_AC_OQT_FREEPG_V1(x)                                            \
+	(((x) >> BIT_SHIFT_AC_OQT_FREEPG_V1) & BIT_MASK_AC_OQT_FREEPG_V1)
+
+/* 2 REG_AFE_CTRL2				(Offset 0x0028) */
+
+#define BIT_SHIFT_MCCO 14
+#define BIT_MASK_MCCO 0x3
+#define BIT_MCCO(x) (((x) & BIT_MASK_MCCO) << BIT_SHIFT_MCCO)
+#define BIT_GET_MCCO(x) (((x) >> BIT_SHIFT_MCCO) & BIT_MASK_MCCO)
+
+/* 2 REG_AFE_CTRL2				(Offset 0x0028) */
+
+#define BIT_SHIFT_REG_LDO_SEL 12
+#define BIT_MASK_REG_LDO_SEL 0x3
+#define BIT_REG_LDO_SEL(x)                                                     \
+	(((x) & BIT_MASK_REG_LDO_SEL) << BIT_SHIFT_REG_LDO_SEL)
+#define BIT_GET_REG_LDO_SEL(x)                                                 \
+	(((x) >> BIT_SHIFT_REG_LDO_SEL) & BIT_MASK_REG_LDO_SEL)
+
+#define BIT_REG_KVCO_V2 BIT(10)
+
+/* 2 REG_AFE_CTRL2				(Offset 0x0028) */
+
+#define BIT_AGPIO_GPO BIT(9)
+
+/* 2 REG_AFE_CTRL2				(Offset 0x0028) */
+
+#define BIT_SHIFT_AGPIO_DRV 7
+#define BIT_MASK_AGPIO_DRV 0x3
+#define BIT_AGPIO_DRV(x) (((x) & BIT_MASK_AGPIO_DRV) << BIT_SHIFT_AGPIO_DRV)
+#define BIT_GET_AGPIO_DRV(x) (((x) >> BIT_SHIFT_AGPIO_DRV) & BIT_MASK_AGPIO_DRV)
+
+/* 2 REG_AFE_CTRL2				(Offset 0x0028) */
+
+#define BIT_SHIFT_XTAL_CAP_XO 1
+#define BIT_MASK_XTAL_CAP_XO 0x3f
+#define BIT_XTAL_CAP_XO(x)                                                     \
+	(((x) & BIT_MASK_XTAL_CAP_XO) << BIT_SHIFT_XTAL_CAP_XO)
+#define BIT_GET_XTAL_CAP_XO(x)                                                 \
+	(((x) >> BIT_SHIFT_XTAL_CAP_XO) & BIT_MASK_XTAL_CAP_XO)
+
+/* 2 REG_AFE_CTRL2				(Offset 0x0028) */
+
+#define BIT_POW_PLL BIT(0)
+
+/* 2 REG_SDIO_OQT_FREE_TXPG_V1		(Offset 0x10250028) */
+
+#define BIT_SHIFT_EXQ_FREEPG_V1 0
+#define BIT_MASK_EXQ_FREEPG_V1 0xfff
+#define BIT_EXQ_FREEPG_V1(x)                                                   \
+	(((x) & BIT_MASK_EXQ_FREEPG_V1) << BIT_SHIFT_EXQ_FREEPG_V1)
+#define BIT_GET_EXQ_FREEPG_V1(x)                                               \
+	(((x) >> BIT_SHIFT_EXQ_FREEPG_V1) & BIT_MASK_EXQ_FREEPG_V1)
+
+/* 2 REG_AFE_CTRL3				(Offset 0x002C) */
+
+#define BIT_SHIFT_PS 7
+#define BIT_MASK_PS 0x7
+#define BIT_PS(x) (((x) & BIT_MASK_PS) << BIT_SHIFT_PS)
+#define BIT_GET_PS(x) (((x) >> BIT_SHIFT_PS) & BIT_MASK_PS)
+
+/* 2 REG_AFE_CTRL3				(Offset 0x002C) */
+
+#define BIT_PSEN BIT(6)
+#define BIT_DOGENB BIT(5)
+
+/* 2 REG_AFE_CTRL3				(Offset 0x002C) */
+
+#define BIT_REG_MBIAS BIT(4)
+
+/* 2 REG_AFE_CTRL3				(Offset 0x002C) */
+
+#define BIT_SHIFT_REG_R3_V4 1
+#define BIT_MASK_REG_R3_V4 0x7
+#define BIT_REG_R3_V4(x) (((x) & BIT_MASK_REG_R3_V4) << BIT_SHIFT_REG_R3_V4)
+#define BIT_GET_REG_R3_V4(x) (((x) >> BIT_SHIFT_REG_R3_V4) & BIT_MASK_REG_R3_V4)
+
+/* 2 REG_AFE_CTRL3				(Offset 0x002C) */
+
+#define BIT_REG_CP_BIT0 BIT(0)
+
+/* 2 REG_EFUSE_CTRL				(Offset 0x0030) */
+
+#define BIT_EF_FLAG BIT(31)
+
+#define BIT_SHIFT_EF_PGPD 28
+#define BIT_MASK_EF_PGPD 0x7
+#define BIT_EF_PGPD(x) (((x) & BIT_MASK_EF_PGPD) << BIT_SHIFT_EF_PGPD)
+#define BIT_GET_EF_PGPD(x) (((x) >> BIT_SHIFT_EF_PGPD) & BIT_MASK_EF_PGPD)
+
+#define BIT_SHIFT_EF_RDT 24
+#define BIT_MASK_EF_RDT 0xf
+#define BIT_EF_RDT(x) (((x) & BIT_MASK_EF_RDT) << BIT_SHIFT_EF_RDT)
+#define BIT_GET_EF_RDT(x) (((x) >> BIT_SHIFT_EF_RDT) & BIT_MASK_EF_RDT)
+
+#define BIT_SHIFT_EF_PGTS 20
+#define BIT_MASK_EF_PGTS 0xf
+#define BIT_EF_PGTS(x) (((x) & BIT_MASK_EF_PGTS) << BIT_SHIFT_EF_PGTS)
+#define BIT_GET_EF_PGTS(x) (((x) >> BIT_SHIFT_EF_PGTS) & BIT_MASK_EF_PGTS)
+
+/* 2 REG_EFUSE_CTRL				(Offset 0x0030) */
+
+#define BIT_EF_PDWN BIT(19)
+
+/* 2 REG_EFUSE_CTRL				(Offset 0x0030) */
+
+#define BIT_EF_ALDEN BIT(18)
+
+/* 2 REG_SDIO_HTSFR_INFO			(Offset 0x10250030) */
+
+#define BIT_SHIFT_HTSFR1 16
+#define BIT_MASK_HTSFR1 0xffff
+#define BIT_HTSFR1(x) (((x) & BIT_MASK_HTSFR1) << BIT_SHIFT_HTSFR1)
+#define BIT_GET_HTSFR1(x) (((x) >> BIT_SHIFT_HTSFR1) & BIT_MASK_HTSFR1)
+
+/* 2 REG_EFUSE_CTRL				(Offset 0x0030) */
+
+#define BIT_SHIFT_EF_ADDR 8
+#define BIT_MASK_EF_ADDR 0x3ff
+#define BIT_EF_ADDR(x) (((x) & BIT_MASK_EF_ADDR) << BIT_SHIFT_EF_ADDR)
+#define BIT_GET_EF_ADDR(x) (((x) >> BIT_SHIFT_EF_ADDR) & BIT_MASK_EF_ADDR)
+
+#define BIT_SHIFT_EF_DATA 0
+#define BIT_MASK_EF_DATA 0xff
+#define BIT_EF_DATA(x) (((x) & BIT_MASK_EF_DATA) << BIT_SHIFT_EF_DATA)
+#define BIT_GET_EF_DATA(x) (((x) >> BIT_SHIFT_EF_DATA) & BIT_MASK_EF_DATA)
+
+/* 2 REG_SDIO_HTSFR_INFO			(Offset 0x10250030) */
+
+#define BIT_SHIFT_HTSFR0 0
+#define BIT_MASK_HTSFR0 0xffff
+#define BIT_HTSFR0(x) (((x) & BIT_MASK_HTSFR0) << BIT_SHIFT_HTSFR0)
+#define BIT_GET_HTSFR0(x) (((x) >> BIT_SHIFT_HTSFR0) & BIT_MASK_HTSFR0)
+
+/* 2 REG_LDO_EFUSE_CTRL			(Offset 0x0034) */
+
+#define BIT_LDOE25_EN BIT(31)
+
+/* 2 REG_LDO_EFUSE_CTRL			(Offset 0x0034) */
+
+#define BIT_SHIFT_LDOE25_V12ADJ_L 27
+#define BIT_MASK_LDOE25_V12ADJ_L 0xf
+#define BIT_LDOE25_V12ADJ_L(x)                                                 \
+	(((x) & BIT_MASK_LDOE25_V12ADJ_L) << BIT_SHIFT_LDOE25_V12ADJ_L)
+#define BIT_GET_LDOE25_V12ADJ_L(x)                                             \
+	(((x) >> BIT_SHIFT_LDOE25_V12ADJ_L) & BIT_MASK_LDOE25_V12ADJ_L)
+
+/* 2 REG_LDO_EFUSE_CTRL			(Offset 0x0034) */
+
+#define BIT_EF_CRES_SEL BIT(26)
+
+/* 2 REG_LDO_EFUSE_CTRL			(Offset 0x0034) */
+
+#define BIT_SHIFT_EF_SCAN_START_V1 16
+#define BIT_MASK_EF_SCAN_START_V1 0x3ff
+#define BIT_EF_SCAN_START_V1(x)                                                \
+	(((x) & BIT_MASK_EF_SCAN_START_V1) << BIT_SHIFT_EF_SCAN_START_V1)
+#define BIT_GET_EF_SCAN_START_V1(x)                                            \
+	(((x) >> BIT_SHIFT_EF_SCAN_START_V1) & BIT_MASK_EF_SCAN_START_V1)
+
+/* 2 REG_LDO_EFUSE_CTRL			(Offset 0x0034) */
+
+#define BIT_SHIFT_EF_SCAN_END 12
+#define BIT_MASK_EF_SCAN_END 0xf
+#define BIT_EF_SCAN_END(x)                                                     \
+	(((x) & BIT_MASK_EF_SCAN_END) << BIT_SHIFT_EF_SCAN_END)
+#define BIT_GET_EF_SCAN_END(x)                                                 \
+	(((x) >> BIT_SHIFT_EF_SCAN_END) & BIT_MASK_EF_SCAN_END)
+
+/* 2 REG_LDO_EFUSE_CTRL			(Offset 0x0034) */
+
+#define BIT_EF_PD_DIS BIT(11)
+
+/* 2 REG_LDO_EFUSE_CTRL			(Offset 0x0034) */
+
+#define BIT_SHIFT_EF_CELL_SEL 8
+#define BIT_MASK_EF_CELL_SEL 0x3
+#define BIT_EF_CELL_SEL(x)                                                     \
+	(((x) & BIT_MASK_EF_CELL_SEL) << BIT_SHIFT_EF_CELL_SEL)
+#define BIT_GET_EF_CELL_SEL(x)                                                 \
+	(((x) >> BIT_SHIFT_EF_CELL_SEL) & BIT_MASK_EF_CELL_SEL)
+
+/* 2 REG_LDO_EFUSE_CTRL			(Offset 0x0034) */
+
+#define BIT_EF_TRPT BIT(7)
+
+#define BIT_SHIFT_EF_TTHD 0
+#define BIT_MASK_EF_TTHD 0x7f
+#define BIT_EF_TTHD(x) (((x) & BIT_MASK_EF_TTHD) << BIT_SHIFT_EF_TTHD)
+#define BIT_GET_EF_TTHD(x) (((x) >> BIT_SHIFT_EF_TTHD) & BIT_MASK_EF_TTHD)
+
+/* 2 REG_PWR_OPTION_CTRL			(Offset 0x0038) */
+
+#define BIT_SHIFT_DBG_SEL_V1 16
+#define BIT_MASK_DBG_SEL_V1 0xff
+#define BIT_DBG_SEL_V1(x) (((x) & BIT_MASK_DBG_SEL_V1) << BIT_SHIFT_DBG_SEL_V1)
+#define BIT_GET_DBG_SEL_V1(x)                                                  \
+	(((x) >> BIT_SHIFT_DBG_SEL_V1) & BIT_MASK_DBG_SEL_V1)
+
+/* 2 REG_PWR_OPTION_CTRL			(Offset 0x0038) */
+
+#define BIT_SHIFT_DBG_SEL_BYTE 14
+#define BIT_MASK_DBG_SEL_BYTE 0x3
+#define BIT_DBG_SEL_BYTE(x)                                                    \
+	(((x) & BIT_MASK_DBG_SEL_BYTE) << BIT_SHIFT_DBG_SEL_BYTE)
+#define BIT_GET_DBG_SEL_BYTE(x)                                                \
+	(((x) >> BIT_SHIFT_DBG_SEL_BYTE) & BIT_MASK_DBG_SEL_BYTE)
+
+/* 2 REG_PWR_OPTION_CTRL			(Offset 0x0038) */
+
+#define BIT_SHIFT_STD_L1_V1 12
+#define BIT_MASK_STD_L1_V1 0x3
+#define BIT_STD_L1_V1(x) (((x) & BIT_MASK_STD_L1_V1) << BIT_SHIFT_STD_L1_V1)
+#define BIT_GET_STD_L1_V1(x) (((x) >> BIT_SHIFT_STD_L1_V1) & BIT_MASK_STD_L1_V1)
+
+/* 2 REG_PWR_OPTION_CTRL			(Offset 0x0038) */
+
+#define BIT_SYSON_DBG_PAD_E2 BIT(11)
+
+/* 2 REG_PWR_OPTION_CTRL			(Offset 0x0038) */
+
+#define BIT_SYSON_LED_PAD_E2 BIT(10)
+
+/* 2 REG_PWR_OPTION_CTRL			(Offset 0x0038) */
+
+#define BIT_SYSON_GPEE_PAD_E2 BIT(9)
+
+/* 2 REG_PWR_OPTION_CTRL			(Offset 0x0038) */
+
+#define BIT_SYSON_PCI_PAD_E2 BIT(8)
+
+#define BIT_SHIFT_MATCH_CNT 8
+#define BIT_MASK_MATCH_CNT 0xff
+#define BIT_MATCH_CNT(x) (((x) & BIT_MASK_MATCH_CNT) << BIT_SHIFT_MATCH_CNT)
+#define BIT_GET_MATCH_CNT(x) (((x) >> BIT_SHIFT_MATCH_CNT) & BIT_MASK_MATCH_CNT)
+
+/* 2 REG_PWR_OPTION_CTRL			(Offset 0x0038) */
+
+#define BIT_AUTO_SW_LDO_VOL_EN BIT(7)
+
+/* 2 REG_PWR_OPTION_CTRL			(Offset 0x0038) */
+
+#define BIT_SHIFT_SYSON_SPS0WWV_WT 4
+#define BIT_MASK_SYSON_SPS0WWV_WT 0x3
+#define BIT_SYSON_SPS0WWV_WT(x)                                                \
+	(((x) & BIT_MASK_SYSON_SPS0WWV_WT) << BIT_SHIFT_SYSON_SPS0WWV_WT)
+#define BIT_GET_SYSON_SPS0WWV_WT(x)                                            \
+	(((x) >> BIT_SHIFT_SYSON_SPS0WWV_WT) & BIT_MASK_SYSON_SPS0WWV_WT)
+
+/* 2 REG_PWR_OPTION_CTRL			(Offset 0x0038) */
+
+#define BIT_SHIFT_SYSON_SPS0LDO_WT 2
+#define BIT_MASK_SYSON_SPS0LDO_WT 0x3
+#define BIT_SYSON_SPS0LDO_WT(x)                                                \
+	(((x) & BIT_MASK_SYSON_SPS0LDO_WT) << BIT_SHIFT_SYSON_SPS0LDO_WT)
+#define BIT_GET_SYSON_SPS0LDO_WT(x)                                            \
+	(((x) >> BIT_SHIFT_SYSON_SPS0LDO_WT) & BIT_MASK_SYSON_SPS0LDO_WT)
+
+/* 2 REG_PWR_OPTION_CTRL			(Offset 0x0038) */
+
+#define BIT_SHIFT_SYSON_RCLK_SCALE 0
+#define BIT_MASK_SYSON_RCLK_SCALE 0x3
+#define BIT_SYSON_RCLK_SCALE(x)                                                \
+	(((x) & BIT_MASK_SYSON_RCLK_SCALE) << BIT_SHIFT_SYSON_RCLK_SCALE)
+#define BIT_GET_SYSON_RCLK_SCALE(x)                                            \
+	(((x) >> BIT_SHIFT_SYSON_RCLK_SCALE) & BIT_MASK_SYSON_RCLK_SCALE)
+
+/* 2 REG_SDIO_HCPWM1_V2			(Offset 0x10250038) */
+
+#define BIT_SYS_CLK BIT(0)
+
+/* 2 REG_CAL_TIMER				(Offset 0x003C) */
+
+#define BIT_SHIFT_CAL_SCAL 0
+#define BIT_MASK_CAL_SCAL 0xff
+#define BIT_CAL_SCAL(x) (((x) & BIT_MASK_CAL_SCAL) << BIT_SHIFT_CAL_SCAL)
+#define BIT_GET_CAL_SCAL(x) (((x) >> BIT_SHIFT_CAL_SCAL) & BIT_MASK_CAL_SCAL)
+
+/* 2 REG_ACLK_MON				(Offset 0x003E) */
+
+#define BIT_SHIFT_RCLK_MON 5
+#define BIT_MASK_RCLK_MON 0x7ff
+#define BIT_RCLK_MON(x) (((x) & BIT_MASK_RCLK_MON) << BIT_SHIFT_RCLK_MON)
+#define BIT_GET_RCLK_MON(x) (((x) >> BIT_SHIFT_RCLK_MON) & BIT_MASK_RCLK_MON)
+
+#define BIT_CAL_EN BIT(4)
+
+#define BIT_SHIFT_DPSTU 2
+#define BIT_MASK_DPSTU 0x3
+#define BIT_DPSTU(x) (((x) & BIT_MASK_DPSTU) << BIT_SHIFT_DPSTU)
+#define BIT_GET_DPSTU(x) (((x) >> BIT_SHIFT_DPSTU) & BIT_MASK_DPSTU)
+
+#define BIT_SUS_16X BIT(1)
+
+/* 2 REG_SDIO_INDIRECT_REG_CFG		(Offset 0x10250040) */
+
+#define BIT_INDIRECT_REG_RDY BIT(20)
+
+/* 2 REG_GPIO_MUXCFG				(Offset 0x0040) */
+
+#define BIT_FSPI_EN BIT(19)
+
+/* 2 REG_SDIO_INDIRECT_REG_CFG		(Offset 0x10250040) */
+
+#define BIT_INDIRECT_REG_R BIT(19)
+
+/* 2 REG_GPIO_MUXCFG				(Offset 0x0040) */
+
+#define BIT_WL_RTS_EXT_32K_SEL BIT(18)
+
+/* 2 REG_SDIO_INDIRECT_REG_CFG		(Offset 0x10250040) */
+
+#define BIT_INDIRECT_REG_W BIT(18)
+
+/* 2 REG_GPIO_MUXCFG				(Offset 0x0040) */
+
+#define BIT_WLGP_SPI_EN BIT(16)
+
+/* 2 REG_SDIO_INDIRECT_REG_CFG		(Offset 0x10250040) */
+
+#define BIT_SHIFT_INDIRECT_REG_SIZE 16
+#define BIT_MASK_INDIRECT_REG_SIZE 0x3
+#define BIT_INDIRECT_REG_SIZE(x)                                               \
+	(((x) & BIT_MASK_INDIRECT_REG_SIZE) << BIT_SHIFT_INDIRECT_REG_SIZE)
+#define BIT_GET_INDIRECT_REG_SIZE(x)                                           \
+	(((x) >> BIT_SHIFT_INDIRECT_REG_SIZE) & BIT_MASK_INDIRECT_REG_SIZE)
+
+/* 2 REG_GPIO_MUXCFG				(Offset 0x0040) */
+
+#define BIT_SIC_LBK BIT(15)
+#define BIT_ENHTP BIT(14)
+
+/* 2 REG_GPIO_MUXCFG				(Offset 0x0040) */
+
+#define BIT_ENSIC BIT(12)
+#define BIT_SIC_SWRST BIT(11)
+
+/* 2 REG_GPIO_MUXCFG				(Offset 0x0040) */
+
+#define BIT_PO_WIFI_PTA_PINS BIT(10)
+
+/* 2 REG_GPIO_MUXCFG				(Offset 0x0040) */
+
+#define BIT_PO_BT_PTA_PINS BIT(9)
+
+/* 2 REG_GPIO_MUXCFG				(Offset 0x0040) */
+
+#define BIT_ENUART BIT(8)
+
+#define BIT_SHIFT_BTMODE 6
+#define BIT_MASK_BTMODE 0x3
+#define BIT_BTMODE(x) (((x) & BIT_MASK_BTMODE) << BIT_SHIFT_BTMODE)
+#define BIT_GET_BTMODE(x) (((x) >> BIT_SHIFT_BTMODE) & BIT_MASK_BTMODE)
+
+#define BIT_ENBT BIT(5)
+#define BIT_EROM_EN BIT(4)
+
+/* 2 REG_GPIO_MUXCFG				(Offset 0x0040) */
+
+#define BIT_WLRFE_6_7_EN BIT(3)
+
+/* 2 REG_GPIO_MUXCFG				(Offset 0x0040) */
+
+#define BIT_WLRFE_4_5_EN BIT(2)
+
+/* 2 REG_GPIO_MUXCFG				(Offset 0x0040) */
+
+#define BIT_SHIFT_GPIOSEL 0
+#define BIT_MASK_GPIOSEL 0x3
+#define BIT_GPIOSEL(x) (((x) & BIT_MASK_GPIOSEL) << BIT_SHIFT_GPIOSEL)
+#define BIT_GET_GPIOSEL(x) (((x) >> BIT_SHIFT_GPIOSEL) & BIT_MASK_GPIOSEL)
+
+/* 2 REG_SDIO_INDIRECT_REG_CFG		(Offset 0x10250040) */
+
+#define BIT_SHIFT_INDIRECT_REG_ADDR 0
+#define BIT_MASK_INDIRECT_REG_ADDR 0xffff
+#define BIT_INDIRECT_REG_ADDR(x)                                               \
+	(((x) & BIT_MASK_INDIRECT_REG_ADDR) << BIT_SHIFT_INDIRECT_REG_ADDR)
+#define BIT_GET_INDIRECT_REG_ADDR(x)                                           \
+	(((x) >> BIT_SHIFT_INDIRECT_REG_ADDR) & BIT_MASK_INDIRECT_REG_ADDR)
+
+/* 2 REG_GPIO_PIN_CTRL			(Offset 0x0044) */
+
+#define BIT_SHIFT_GPIO_MOD_7_TO_0 24
+#define BIT_MASK_GPIO_MOD_7_TO_0 0xff
+#define BIT_GPIO_MOD_7_TO_0(x)                                                 \
+	(((x) & BIT_MASK_GPIO_MOD_7_TO_0) << BIT_SHIFT_GPIO_MOD_7_TO_0)
+#define BIT_GET_GPIO_MOD_7_TO_0(x)                                             \
+	(((x) >> BIT_SHIFT_GPIO_MOD_7_TO_0) & BIT_MASK_GPIO_MOD_7_TO_0)
+
+#define BIT_SHIFT_GPIO_IO_SEL_7_TO_0 16
+#define BIT_MASK_GPIO_IO_SEL_7_TO_0 0xff
+#define BIT_GPIO_IO_SEL_7_TO_0(x)                                              \
+	(((x) & BIT_MASK_GPIO_IO_SEL_7_TO_0) << BIT_SHIFT_GPIO_IO_SEL_7_TO_0)
+#define BIT_GET_GPIO_IO_SEL_7_TO_0(x)                                          \
+	(((x) >> BIT_SHIFT_GPIO_IO_SEL_7_TO_0) & BIT_MASK_GPIO_IO_SEL_7_TO_0)
+
+#define BIT_SHIFT_GPIO_OUT_7_TO_0 8
+#define BIT_MASK_GPIO_OUT_7_TO_0 0xff
+#define BIT_GPIO_OUT_7_TO_0(x)                                                 \
+	(((x) & BIT_MASK_GPIO_OUT_7_TO_0) << BIT_SHIFT_GPIO_OUT_7_TO_0)
+#define BIT_GET_GPIO_OUT_7_TO_0(x)                                             \
+	(((x) >> BIT_SHIFT_GPIO_OUT_7_TO_0) & BIT_MASK_GPIO_OUT_7_TO_0)
+
+#define BIT_SHIFT_GPIO_IN_7_TO_0 0
+#define BIT_MASK_GPIO_IN_7_TO_0 0xff
+#define BIT_GPIO_IN_7_TO_0(x)                                                  \
+	(((x) & BIT_MASK_GPIO_IN_7_TO_0) << BIT_SHIFT_GPIO_IN_7_TO_0)
+#define BIT_GET_GPIO_IN_7_TO_0(x)                                              \
+	(((x) >> BIT_SHIFT_GPIO_IN_7_TO_0) & BIT_MASK_GPIO_IN_7_TO_0)
+
+/* 2 REG_SDIO_INDIRECT_REG_DATA		(Offset 0x10250044) */
+
+#define BIT_SHIFT_INDIRECT_REG_DATA 0
+#define BIT_MASK_INDIRECT_REG_DATA 0xffffffffL
+#define BIT_INDIRECT_REG_DATA(x)                                               \
+	(((x) & BIT_MASK_INDIRECT_REG_DATA) << BIT_SHIFT_INDIRECT_REG_DATA)
+#define BIT_GET_INDIRECT_REG_DATA(x)                                           \
+	(((x) >> BIT_SHIFT_INDIRECT_REG_DATA) & BIT_MASK_INDIRECT_REG_DATA)
+
+/* 2 REG_GPIO_INTM				(Offset 0x0048) */
+
+#define BIT_SHIFT_MUXDBG_SEL 30
+#define BIT_MASK_MUXDBG_SEL 0x3
+#define BIT_MUXDBG_SEL(x) (((x) & BIT_MASK_MUXDBG_SEL) << BIT_SHIFT_MUXDBG_SEL)
+#define BIT_GET_MUXDBG_SEL(x)                                                  \
+	(((x) >> BIT_SHIFT_MUXDBG_SEL) & BIT_MASK_MUXDBG_SEL)
+
+/* 2 REG_GPIO_INTM				(Offset 0x0048) */
+
+#define BIT_EXTWOL_SEL BIT(17)
+
+/* 2 REG_GPIO_INTM				(Offset 0x0048) */
+
+#define BIT_EXTWOL_EN BIT(16)
+
+/* 2 REG_GPIO_INTM				(Offset 0x0048) */
+
+#define BIT_GPIOF_INT_MD BIT(15)
+#define BIT_GPIOE_INT_MD BIT(14)
+#define BIT_GPIOD_INT_MD BIT(13)
+#define BIT_GPIOC_INT_MD BIT(12)
+#define BIT_GPIOB_INT_MD BIT(11)
+#define BIT_GPIOA_INT_MD BIT(10)
+#define BIT_GPIO9_INT_MD BIT(9)
+#define BIT_GPIO8_INT_MD BIT(8)
+#define BIT_GPIO7_INT_MD BIT(7)
+#define BIT_GPIO6_INT_MD BIT(6)
+#define BIT_GPIO5_INT_MD BIT(5)
+#define BIT_GPIO4_INT_MD BIT(4)
+#define BIT_GPIO3_INT_MD BIT(3)
+#define BIT_GPIO2_INT_MD BIT(2)
+#define BIT_GPIO1_INT_MD BIT(1)
+#define BIT_GPIO0_INT_MD BIT(0)
+
+/* 2 REG_LED_CFG				(Offset 0x004C) */
+
+#define BIT_GPIO3_WL_CTRL_EN BIT(27)
+
+/* 2 REG_LED_CFG				(Offset 0x004C) */
+
+#define BIT_LNAON_SEL_EN BIT(26)
+
+/* 2 REG_LED_CFG				(Offset 0x004C) */
+
+#define BIT_PAPE_SEL_EN BIT(25)
+
+/* 2 REG_LED_CFG				(Offset 0x004C) */
+
+#define BIT_DPDT_WLBT_SEL BIT(24)
+
+/* 2 REG_LED_CFG				(Offset 0x004C) */
+
+#define BIT_DPDT_SEL_EN BIT(23)
+
+/* 2 REG_LED_CFG				(Offset 0x004C) */
+
+#define BIT_GPIO13_14_WL_CTRL_EN BIT(22)
+
+/* 2 REG_LED_CFG				(Offset 0x004C) */
+
+#define BIT_LED2DIS BIT(21)
+
+/* 2 REG_LED_CFG				(Offset 0x004C) */
+
+#define BIT_LED2PL BIT(20)
+#define BIT_LED2SV BIT(19)
+
+#define BIT_SHIFT_LED2CM 16
+#define BIT_MASK_LED2CM 0x7
+#define BIT_LED2CM(x) (((x) & BIT_MASK_LED2CM) << BIT_SHIFT_LED2CM)
+#define BIT_GET_LED2CM(x) (((x) >> BIT_SHIFT_LED2CM) & BIT_MASK_LED2CM)
+
+#define BIT_LED1DIS BIT(15)
+#define BIT_LED1PL BIT(12)
+#define BIT_LED1SV BIT(11)
+
+#define BIT_SHIFT_LED1CM 8
+#define BIT_MASK_LED1CM 0x7
+#define BIT_LED1CM(x) (((x) & BIT_MASK_LED1CM) << BIT_SHIFT_LED1CM)
+#define BIT_GET_LED1CM(x) (((x) >> BIT_SHIFT_LED1CM) & BIT_MASK_LED1CM)
+
+#define BIT_LED0DIS BIT(7)
+
+/* 2 REG_LED_CFG				(Offset 0x004C) */
+
+#define BIT_SHIFT_AFE_LDO_SWR_CHECK 5
+#define BIT_MASK_AFE_LDO_SWR_CHECK 0x3
+#define BIT_AFE_LDO_SWR_CHECK(x)                                               \
+	(((x) & BIT_MASK_AFE_LDO_SWR_CHECK) << BIT_SHIFT_AFE_LDO_SWR_CHECK)
+#define BIT_GET_AFE_LDO_SWR_CHECK(x)                                           \
+	(((x) >> BIT_SHIFT_AFE_LDO_SWR_CHECK) & BIT_MASK_AFE_LDO_SWR_CHECK)
+
+/* 2 REG_LED_CFG				(Offset 0x004C) */
+
+#define BIT_LED0PL BIT(4)
+#define BIT_LED0SV BIT(3)
+
+#define BIT_SHIFT_LED0CM 0
+#define BIT_MASK_LED0CM 0x7
+#define BIT_LED0CM(x) (((x) & BIT_MASK_LED0CM) << BIT_SHIFT_LED0CM)
+#define BIT_GET_LED0CM(x) (((x) >> BIT_SHIFT_LED0CM) & BIT_MASK_LED0CM)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_FS_PDNINT_EN BIT(31)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_NFC_INT_PAD_EN BIT(30)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_FS_SPS_OCP_INT_EN BIT(29)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_FS_PWMERR_INT_EN BIT(28)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_FS_GPIOF_INT_EN BIT(27)
+#define BIT_FS_GPIOE_INT_EN BIT(26)
+#define BIT_FS_GPIOD_INT_EN BIT(25)
+#define BIT_FS_GPIOC_INT_EN BIT(24)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_FS_GPIOB_INT_EN BIT(23)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_FS_GPIOA_INT_EN BIT(22)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_FS_GPIO9_INT_EN BIT(21)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_FS_GPIO8_INT_EN BIT(20)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_FS_GPIO7_INT_EN BIT(19)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_FS_GPIO6_INT_EN BIT(18)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_FS_GPIO5_INT_EN BIT(17)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_FS_GPIO4_INT_EN BIT(16)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_FS_GPIO3_INT_EN BIT(15)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_FS_GPIO2_INT_EN BIT(14)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_FS_GPIO1_INT_EN BIT(13)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_FS_GPIO0_INT_EN BIT(12)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_FS_HCI_SUS_EN BIT(11)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_FS_HCI_RES_EN BIT(10)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_FS_HCI_RESET_EN BIT(9)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_FS_BTON_STS_UPDATE_MSK_EN BIT(7)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_ACT2RECOVERY_INT_EN_V1 BIT(6)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_GEN1GEN2_SWITCH BIT(5)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_HCI_TXDMA_REQ_HIMR BIT(4)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_FS_32K_LEAVE_SETTING_MAK BIT(3)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_FS_32K_ENTER_SETTING_MAK BIT(2)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_FS_USB_LPMRSM_MSK BIT(1)
+
+/* 2 REG_FSIMR				(Offset 0x0050) */
+
+#define BIT_FS_USB_LPMINT_MSK BIT(0)
+
+/* 2 REG_FSISR				(Offset 0x0054) */
+
+#define BIT_FS_PDNINT BIT(31)
+
+/* 2 REG_FSISR				(Offset 0x0054) */
+
+#define BIT_FS_SPS_OCP_INT BIT(29)
+
+/* 2 REG_FSISR				(Offset 0x0054) */
+
+#define BIT_FS_PWMERR_INT BIT(28)
+
+/* 2 REG_FSISR				(Offset 0x0054) */
+
+#define BIT_FS_GPIOF_INT BIT(27)
+#define BIT_FS_GPIOE_INT BIT(26)
+#define BIT_FS_GPIOD_INT BIT(25)
+#define BIT_FS_GPIOC_INT BIT(24)
+
+/* 2 REG_FSISR				(Offset 0x0054) */
+
+#define BIT_FS_GPIOB_INT BIT(23)
+
+/* 2 REG_FSISR				(Offset 0x0054) */
+
+#define BIT_FS_GPIOA_INT BIT(22)
+
+/* 2 REG_FSISR				(Offset 0x0054) */
+
+#define BIT_FS_GPIO9_INT BIT(21)
+
+/* 2 REG_FSISR				(Offset 0x0054) */
+
+#define BIT_FS_GPIO8_INT BIT(20)
+
+/* 2 REG_FSISR				(Offset 0x0054) */
+
+#define BIT_FS_GPIO7_INT BIT(19)
+
+/* 2 REG_FSISR				(Offset 0x0054) */
+
+#define BIT_FS_GPIO6_INT BIT(18)
+
+/* 2 REG_FSISR				(Offset 0x0054) */
+
+#define BIT_FS_GPIO5_INT BIT(17)
+
+/* 2 REG_FSISR				(Offset 0x0054) */
+
+#define BIT_FS_GPIO4_INT BIT(16)
+
+/* 2 REG_FSISR				(Offset 0x0054) */
+
+#define BIT_FS_GPIO3_INT BIT(15)
+
+/* 2 REG_FSISR				(Offset 0x0054) */
+
+#define BIT_FS_GPIO2_INT BIT(14)
+
+/* 2 REG_FSISR				(Offset 0x0054) */
+
+#define BIT_FS_GPIO1_INT BIT(13)
+
+/* 2 REG_FSISR				(Offset 0x0054) */
+
+#define BIT_FS_GPIO0_INT BIT(12)
+
+/* 2 REG_FSISR				(Offset 0x0054) */
+
+#define BIT_FS_HCI_SUS_INT BIT(11)
+
+/* 2 REG_FSISR				(Offset 0x0054) */
+
+#define BIT_FS_HCI_RES_INT BIT(10)
+
+/* 2 REG_FSISR				(Offset 0x0054) */
+
+#define BIT_FS_HCI_RESET_INT BIT(9)
+
+/* 2 REG_FSISR				(Offset 0x0054) */
+
+#define BIT_ACT2RECOVERY BIT(6)
+
+/* 2 REG_FSISR				(Offset 0x0054) */
+
+#define BIT_HCI_TXDMA_REQ_HISR BIT(4)
+
+/* 2 REG_FSISR				(Offset 0x0054) */
+
+#define BIT_FS_32K_LEAVE_SETTING_INT BIT(3)
+
+/* 2 REG_FSISR				(Offset 0x0054) */
+
+#define BIT_FS_32K_ENTER_SETTING_INT BIT(2)
+
+/* 2 REG_FSISR				(Offset 0x0054) */
+
+#define BIT_FS_USB_LPMRSM_INT BIT(1)
+
+/* 2 REG_FSISR				(Offset 0x0054) */
+
+#define BIT_FS_USB_LPMINT_INT BIT(0)
+
+/* 2 REG_HSIMR				(Offset 0x0058) */
+
+#define BIT_GPIOF_INT_EN BIT(31)
+#define BIT_GPIOE_INT_EN BIT(30)
+#define BIT_GPIOD_INT_EN BIT(29)
+#define BIT_GPIOC_INT_EN BIT(28)
+#define BIT_GPIOB_INT_EN BIT(27)
+#define BIT_GPIOA_INT_EN BIT(26)
+#define BIT_GPIO9_INT_EN BIT(25)
+#define BIT_GPIO8_INT_EN BIT(24)
+#define BIT_GPIO7_INT_EN BIT(23)
+#define BIT_GPIO6_INT_EN BIT(22)
+#define BIT_GPIO5_INT_EN BIT(21)
+#define BIT_GPIO4_INT_EN BIT(20)
+#define BIT_GPIO3_INT_EN BIT(19)
+
+/* 2 REG_HSIMR				(Offset 0x0058) */
+
+#define BIT_GPIO1_INT_EN BIT(17)
+#define BIT_GPIO0_INT_EN BIT(16)
+
+/* 2 REG_HSIMR				(Offset 0x0058) */
+
+#define BIT_GPIO2_INT_EN_V1 BIT(16)
+
+/* 2 REG_HSIMR				(Offset 0x0058) */
+
+#define BIT_PDNINT_EN BIT(7)
+
+/* 2 REG_HSIMR				(Offset 0x0058) */
+
+#define BIT_RON_INT_EN BIT(6)
+
+/* 2 REG_HSIMR				(Offset 0x0058) */
+
+#define BIT_SPS_OCP_INT_EN BIT(5)
+
+/* 2 REG_HSIMR				(Offset 0x0058) */
+
+#define BIT_GPIO15_0_INT_EN BIT(0)
+
+/* 2 REG_HSISR				(Offset 0x005C) */
+
+#define BIT_GPIOF_INT BIT(31)
+#define BIT_GPIOE_INT BIT(30)
+#define BIT_GPIOD_INT BIT(29)
+#define BIT_GPIOC_INT BIT(28)
+#define BIT_GPIOB_INT BIT(27)
+#define BIT_GPIOA_INT BIT(26)
+#define BIT_GPIO9_INT BIT(25)
+#define BIT_GPIO8_INT BIT(24)
+#define BIT_GPIO7_INT BIT(23)
+
+/* 2 REG_HSISR				(Offset 0x005C) */
+
+#define BIT_GPIO6_INT BIT(22)
+#define BIT_GPIO5_INT BIT(21)
+#define BIT_GPIO4_INT BIT(20)
+#define BIT_GPIO3_INT BIT(19)
+
+/* 2 REG_HSISR				(Offset 0x005C) */
+
+#define BIT_GPIO1_INT BIT(17)
+#define BIT_GPIO0_INT BIT(16)
+
+/* 2 REG_HSISR				(Offset 0x005C) */
+
+#define BIT_GPIO2_INT_V1 BIT(16)
+
+/* 2 REG_HSISR				(Offset 0x005C) */
+
+#define BIT_PDNINT BIT(7)
+
+/* 2 REG_HSISR				(Offset 0x005C) */
+
+#define BIT_RON_INT BIT(6)
+
+/* 2 REG_HSISR				(Offset 0x005C) */
+
+#define BIT_SPS_OCP_INT BIT(5)
+
+/* 2 REG_HSISR				(Offset 0x005C) */
+
+#define BIT_GPIO15_0_INT BIT(0)
+#define BIT_MCUFWDL_EN BIT(0)
+
+/* 2 REG_GPIO_EXT_CTRL			(Offset 0x0060) */
+
+#define BIT_SHIFT_GPIO_MOD_15_TO_8 24
+#define BIT_MASK_GPIO_MOD_15_TO_8 0xff
+#define BIT_GPIO_MOD_15_TO_8(x)                                                \
+	(((x) & BIT_MASK_GPIO_MOD_15_TO_8) << BIT_SHIFT_GPIO_MOD_15_TO_8)
+#define BIT_GET_GPIO_MOD_15_TO_8(x)                                            \
+	(((x) >> BIT_SHIFT_GPIO_MOD_15_TO_8) & BIT_MASK_GPIO_MOD_15_TO_8)
+
+#define BIT_SHIFT_GPIO_IO_SEL_15_TO_8 16
+#define BIT_MASK_GPIO_IO_SEL_15_TO_8 0xff
+#define BIT_GPIO_IO_SEL_15_TO_8(x)                                             \
+	(((x) & BIT_MASK_GPIO_IO_SEL_15_TO_8) << BIT_SHIFT_GPIO_IO_SEL_15_TO_8)
+#define BIT_GET_GPIO_IO_SEL_15_TO_8(x)                                         \
+	(((x) >> BIT_SHIFT_GPIO_IO_SEL_15_TO_8) & BIT_MASK_GPIO_IO_SEL_15_TO_8)
+
+#define BIT_SHIFT_GPIO_OUT_15_TO_8 8
+#define BIT_MASK_GPIO_OUT_15_TO_8 0xff
+#define BIT_GPIO_OUT_15_TO_8(x)                                                \
+	(((x) & BIT_MASK_GPIO_OUT_15_TO_8) << BIT_SHIFT_GPIO_OUT_15_TO_8)
+#define BIT_GET_GPIO_OUT_15_TO_8(x)                                            \
+	(((x) >> BIT_SHIFT_GPIO_OUT_15_TO_8) & BIT_MASK_GPIO_OUT_15_TO_8)
+
+#define BIT_SHIFT_GPIO_IN_15_TO_8 0
+#define BIT_MASK_GPIO_IN_15_TO_8 0xff
+#define BIT_GPIO_IN_15_TO_8(x)                                                 \
+	(((x) & BIT_MASK_GPIO_IN_15_TO_8) << BIT_SHIFT_GPIO_IN_15_TO_8)
+#define BIT_GET_GPIO_IN_15_TO_8(x)                                             \
+	(((x) >> BIT_SHIFT_GPIO_IN_15_TO_8) & BIT_MASK_GPIO_IN_15_TO_8)
+
+/* 2 REG_SDIO_H2C				(Offset 0x10250060) */
+
+#define BIT_SHIFT_SDIO_H2C_MSG 0
+#define BIT_MASK_SDIO_H2C_MSG 0xffffffffL
+#define BIT_SDIO_H2C_MSG(x)                                                    \
+	(((x) & BIT_MASK_SDIO_H2C_MSG) << BIT_SHIFT_SDIO_H2C_MSG)
+#define BIT_GET_SDIO_H2C_MSG(x)                                                \
+	(((x) >> BIT_SHIFT_SDIO_H2C_MSG) & BIT_MASK_SDIO_H2C_MSG)
+
+/* 2 REG_PAD_CTRL1				(Offset 0x0064) */
+
+#define BIT_PAPE_WLBT_SEL BIT(29)
+#define BIT_LNAON_WLBT_SEL BIT(28)
+
+/* 2 REG_PAD_CTRL1				(Offset 0x0064) */
+
+#define BIT_BTGP_GPG3_FEN BIT(26)
+#define BIT_BTGP_GPG2_FEN BIT(25)
+
+/* 2 REG_PAD_CTRL1				(Offset 0x0064) */
+
+#define BIT_BTGP_JTAG_EN BIT(24)
+
+/* 2 REG_PAD_CTRL1				(Offset 0x0064) */
+
+#define BIT_XTAL_CLK_EXTARNAL_EN BIT(23)
+
+/* 2 REG_PAD_CTRL1				(Offset 0x0064) */
+
+#define BIT_BTGP_UART0_EN BIT(22)
+
+/* 2 REG_PAD_CTRL1				(Offset 0x0064) */
+
+#define BIT_BTGP_UART1_EN BIT(21)
+
+/* 2 REG_PAD_CTRL1				(Offset 0x0064) */
+
+#define BIT_BTGP_SPI_EN BIT(20)
+
+/* 2 REG_PAD_CTRL1				(Offset 0x0064) */
+
+#define BIT_BTGP_GPIO_E2 BIT(19)
+
+/* 2 REG_PAD_CTRL1				(Offset 0x0064) */
+
+#define BIT_BTGP_GPIO_EN BIT(18)
+
+/* 2 REG_PAD_CTRL1				(Offset 0x0064) */
+
+#define BIT_SHIFT_BTGP_GPIO_SL 16
+#define BIT_MASK_BTGP_GPIO_SL 0x3
+#define BIT_BTGP_GPIO_SL(x)                                                    \
+	(((x) & BIT_MASK_BTGP_GPIO_SL) << BIT_SHIFT_BTGP_GPIO_SL)
+#define BIT_GET_BTGP_GPIO_SL(x)                                                \
+	(((x) >> BIT_SHIFT_BTGP_GPIO_SL) & BIT_MASK_BTGP_GPIO_SL)
+
+/* 2 REG_PAD_CTRL1				(Offset 0x0064) */
+
+#define BIT_PAD_SDIO_SR BIT(14)
+
+/* 2 REG_PAD_CTRL1				(Offset 0x0064) */
+
+#define BIT_GPIO14_OUTPUT_PL BIT(13)
+
+/* 2 REG_PAD_CTRL1				(Offset 0x0064) */
+
+#define BIT_HOST_WAKE_PAD_PULL_EN BIT(12)
+
+/* 2 REG_PAD_CTRL1				(Offset 0x0064) */
+
+#define BIT_HOST_WAKE_PAD_SL BIT(11)
+
+/* 2 REG_PAD_CTRL1				(Offset 0x0064) */
+
+#define BIT_PAD_LNAON_SR BIT(10)
+
+/* 2 REG_PAD_CTRL1				(Offset 0x0064) */
+
+#define BIT_PAD_LNAON_E2 BIT(9)
+
+/* 2 REG_PAD_CTRL1				(Offset 0x0064) */
+
+#define BIT_SW_LNAON_G_SEL_DATA BIT(8)
+
+/* 2 REG_PAD_CTRL1				(Offset 0x0064) */
+
+#define BIT_SW_LNAON_A_SEL_DATA BIT(7)
+
+/* 2 REG_PAD_CTRL1				(Offset 0x0064) */
+
+#define BIT_PAD_PAPE_SR BIT(6)
+
+/* 2 REG_PAD_CTRL1				(Offset 0x0064) */
+
+#define BIT_PAD_PAPE_E2 BIT(5)
+
+/* 2 REG_PAD_CTRL1				(Offset 0x0064) */
+
+#define BIT_SW_PAPE_G_SEL_DATA BIT(4)
+
+/* 2 REG_PAD_CTRL1				(Offset 0x0064) */
+
+#define BIT_SW_PAPE_A_SEL_DATA BIT(3)
+
+/* 2 REG_PAD_CTRL1				(Offset 0x0064) */
+
+#define BIT_PAD_DPDT_SR BIT(2)
+
+/* 2 REG_PAD_CTRL1				(Offset 0x0064) */
+
+#define BIT_PAD_DPDT_PAD_E2 BIT(1)
+
+/* 2 REG_PAD_CTRL1				(Offset 0x0064) */
+
+#define BIT_SW_DPDT_SEL_DATA BIT(0)
+
+/* 2 REG_SDIO_C2H				(Offset 0x10250064) */
+
+#define BIT_SHIFT_SDIO_C2H_MSG 0
+#define BIT_MASK_SDIO_C2H_MSG 0xffffffffL
+#define BIT_SDIO_C2H_MSG(x)                                                    \
+	(((x) & BIT_MASK_SDIO_C2H_MSG) << BIT_SHIFT_SDIO_C2H_MSG)
+#define BIT_GET_SDIO_C2H_MSG(x)                                                \
+	(((x) >> BIT_SHIFT_SDIO_C2H_MSG) & BIT_MASK_SDIO_C2H_MSG)
+
+/* 2 REG_WL_BT_PWR_CTRL			(Offset 0x0068) */
+
+#define BIT_ISO_BD2PP BIT(31)
+#define BIT_LDOV12B_EN BIT(30)
+#define BIT_CKEN_BTGPS BIT(29)
+#define BIT_FEN_BTGPS BIT(28)
+
+/* 2 REG_WL_BT_PWR_CTRL			(Offset 0x0068) */
+
+#define BIT_MULRW BIT(27)
+
+/* 2 REG_WL_BT_PWR_CTRL			(Offset 0x0068) */
+
+#define BIT_BTCPU_BOOTSEL BIT(27)
+#define BIT_SPI_SPEEDUP BIT(26)
+
+/* 2 REG_WL_BT_PWR_CTRL			(Offset 0x0068) */
+
+#define BIT_DEVWAKE_PAD_TYPE_SEL BIT(24)
+#define BIT_CLKREQ_PAD_TYPE_SEL BIT(23)
+
+/* 2 REG_WL_BT_PWR_CTRL			(Offset 0x0068) */
+
+#define BIT_EN_CPL_TIMEOUT_PS BIT(22)
+
+/* 2 REG_WL_BT_PWR_CTRL			(Offset 0x0068) */
+
+#define BIT_ISO_BTPON2PP BIT(22)
+
+/* 2 REG_WL_BT_PWR_CTRL			(Offset 0x0068) */
+
+#define BIT_REG_TXDMA_FAIL_PS BIT(21)
+
+/* 2 REG_WL_BT_PWR_CTRL			(Offset 0x0068) */
+
+#define BIT_EN_HWENTR_L1 BIT(19)
+
+/* 2 REG_WL_BT_PWR_CTRL			(Offset 0x0068) */
+
+#define BIT_BT_HWROF_EN BIT(19)
+
+/* 2 REG_WL_BT_PWR_CTRL			(Offset 0x0068) */
+
+#define BIT_EN_ADV_CLKGATE BIT(18)
+
+/* 2 REG_WL_BT_PWR_CTRL			(Offset 0x0068) */
+
+#define BIT_BT_FUNC_EN BIT(18)
+
+/* 2 REG_WL_BT_PWR_CTRL			(Offset 0x0068) */
+
+#define BIT_BT_HWPDN_SL BIT(17)
+
+/* 2 REG_WL_BT_PWR_CTRL			(Offset 0x0068) */
+
+#define BIT_BT_DISN_EN BIT(16)
+
+/* 2 REG_WL_BT_PWR_CTRL			(Offset 0x0068) */
+
+#define BIT_BT_PDN_PULL_EN BIT(15)
+
+/* 2 REG_WL_BT_PWR_CTRL			(Offset 0x0068) */
+
+#define BIT_WL_PDN_PULL_EN BIT(14)
+#define BIT_EXTERNAL_REQUEST_PL BIT(13)
+
+/* 2 REG_WL_BT_PWR_CTRL			(Offset 0x0068) */
+
+#define BIT_GPIO0_2_3_PULL_LOW_EN BIT(12)
+
+/* 2 REG_WL_BT_PWR_CTRL			(Offset 0x0068) */
+
+#define BIT_ISO_BA2PP BIT(11)
+#define BIT_BT_AFE_LDO_EN BIT(10)
+
+/* 2 REG_WL_BT_PWR_CTRL			(Offset 0x0068) */
+
+#define BIT_BT_AFE_PLL_EN BIT(9)
+
+/* 2 REG_WL_BT_PWR_CTRL			(Offset 0x0068) */
+
+#define BIT_BT_DIG_CLK_EN BIT(8)
+
+/* 2 REG_WL_BT_PWR_CTRL			(Offset 0x0068) */
+
+#define BIT_WL_DRV_EXIST_IDX BIT(5)
+
+/* 2 REG_WL_BT_PWR_CTRL			(Offset 0x0068) */
+
+#define BIT_DOP_EHPAD BIT(4)
+
+/* 2 REG_WL_BT_PWR_CTRL			(Offset 0x0068) */
+
+#define BIT_WL_HWROF_EN BIT(3)
+
+/* 2 REG_WL_BT_PWR_CTRL			(Offset 0x0068) */
+
+#define BIT_WL_FUNC_EN BIT(2)
+
+/* 2 REG_WL_BT_PWR_CTRL			(Offset 0x0068) */
+
+#define BIT_WL_HWPDN_SL BIT(1)
+#define BIT_WL_HWPDN_EN BIT(0)
+
+/* 2 REG_SDM_DEBUG				(Offset 0x006C) */
+
+#define BIT_SHIFT_WLCLK_PHASE 0
+#define BIT_MASK_WLCLK_PHASE 0x1f
+#define BIT_WLCLK_PHASE(x)                                                     \
+	(((x) & BIT_MASK_WLCLK_PHASE) << BIT_SHIFT_WLCLK_PHASE)
+#define BIT_GET_WLCLK_PHASE(x)                                                 \
+	(((x) >> BIT_SHIFT_WLCLK_PHASE) & BIT_MASK_WLCLK_PHASE)
+
+/* 2 REG_SYS_SDIO_CTRL			(Offset 0x0070) */
+
+#define BIT_DBG_GNT_WL_BT BIT(27)
+
+/* 2 REG_SYS_SDIO_CTRL			(Offset 0x0070) */
+
+#define BIT_LTE_MUX_CTRL_PATH BIT(26)
+
+/* 2 REG_SYS_SDIO_CTRL			(Offset 0x0070) */
+
+#define BIT_LTE_COEX_UART BIT(25)
+
+/* 2 REG_SYS_SDIO_CTRL			(Offset 0x0070) */
+
+#define BIT_3W_LTE_WL_GPIO BIT(24)
+
+/* 2 REG_SYS_SDIO_CTRL			(Offset 0x0070) */
+
+#define BIT_SDIO_INT_POLARITY BIT(19)
+#define BIT_SDIO_INT BIT(18)
+
+/* 2 REG_SYS_SDIO_CTRL			(Offset 0x0070) */
+
+#define BIT_SDIO_OFF_EN BIT(17)
+
+/* 2 REG_SYS_SDIO_CTRL			(Offset 0x0070) */
+
+#define BIT_SDIO_ON_EN BIT(16)
+
+/* 2 REG_SYS_SDIO_CTRL			(Offset 0x0070) */
+
+#define BIT_PCIE_WAIT_TIMEOUT_EVENT BIT(10)
+#define BIT_PCIE_WAIT_TIME BIT(9)
+
+/* 2 REG_SYS_SDIO_CTRL			(Offset 0x0070) */
+
+#define BIT_MPCIE_REFCLK_XTAL_SEL BIT(8)
+
+/* 2 REG_HCI_OPT_CTRL			(Offset 0x0074) */
+
+#define BIT_SHIFT_TSFT_SEL 29
+#define BIT_MASK_TSFT_SEL 0x7
+#define BIT_TSFT_SEL(x) (((x) & BIT_MASK_TSFT_SEL) << BIT_SHIFT_TSFT_SEL)
+#define BIT_GET_TSFT_SEL(x) (((x) >> BIT_SHIFT_TSFT_SEL) & BIT_MASK_TSFT_SEL)
+
+/* 2 REG_HCI_OPT_CTRL			(Offset 0x0074) */
+
+#define BIT_SHIFT_RPWM 24
+#define BIT_MASK_RPWM 0xff
+#define BIT_RPWM(x) (((x) & BIT_MASK_RPWM) << BIT_SHIFT_RPWM)
+#define BIT_GET_RPWM(x) (((x) >> BIT_SHIFT_RPWM) & BIT_MASK_RPWM)
+
+#define BIT_ROM_DLEN BIT(19)
+
+#define BIT_SHIFT_ROM_PGE 16
+#define BIT_MASK_ROM_PGE 0x7
+#define BIT_ROM_PGE(x) (((x) & BIT_MASK_ROM_PGE) << BIT_SHIFT_ROM_PGE)
+#define BIT_GET_ROM_PGE(x) (((x) >> BIT_SHIFT_ROM_PGE) & BIT_MASK_ROM_PGE)
+
+/* 2 REG_HCI_OPT_CTRL			(Offset 0x0074) */
+
+#define BIT_USB_HOST_PWR_OFF_EN BIT(12)
+
+/* 2 REG_HCI_OPT_CTRL			(Offset 0x0074) */
+
+#define BIT_SYM_LPS_BLOCK_EN BIT(11)
+
+/* 2 REG_HCI_OPT_CTRL			(Offset 0x0074) */
+
+#define BIT_USB_LPM_ACT_EN BIT(10)
+
+/* 2 REG_HCI_OPT_CTRL			(Offset 0x0074) */
+
+#define BIT_USB_LPM_NY BIT(9)
+
+/* 2 REG_HCI_OPT_CTRL			(Offset 0x0074) */
+
+#define BIT_USB_SUS_DIS BIT(8)
+
+/* 2 REG_HCI_OPT_CTRL			(Offset 0x0074) */
+
+#define BIT_SHIFT_SDIO_PAD_E 5
+#define BIT_MASK_SDIO_PAD_E 0x7
+#define BIT_SDIO_PAD_E(x) (((x) & BIT_MASK_SDIO_PAD_E) << BIT_SHIFT_SDIO_PAD_E)
+#define BIT_GET_SDIO_PAD_E(x)                                                  \
+	(((x) >> BIT_SHIFT_SDIO_PAD_E) & BIT_MASK_SDIO_PAD_E)
+
+/* 2 REG_HCI_OPT_CTRL			(Offset 0x0074) */
+
+#define BIT_USB_LPPLL_EN BIT(4)
+
+/* 2 REG_HCI_OPT_CTRL			(Offset 0x0074) */
+
+#define BIT_ROP_SW15 BIT(2)
+
+/* 2 REG_HCI_OPT_CTRL			(Offset 0x0074) */
+
+#define BIT_PCI_CKRDY_OPT BIT(1)
+
+/* 2 REG_HCI_OPT_CTRL			(Offset 0x0074) */
+
+#define BIT_PCI_VAUX_EN BIT(0)
+
+/* 2 REG_LDO_SWR_CTRL			(Offset 0x007C) */
+
+#define BIT_ZCD_HW_AUTO_EN BIT(27)
+#define BIT_ZCD_REGSEL BIT(26)
+
+/* 2 REG_LDO_SWR_CTRL			(Offset 0x007C) */
+
+#define BIT_SHIFT_AUTO_ZCD_IN_CODE 21
+#define BIT_MASK_AUTO_ZCD_IN_CODE 0x1f
+#define BIT_AUTO_ZCD_IN_CODE(x)                                                \
+	(((x) & BIT_MASK_AUTO_ZCD_IN_CODE) << BIT_SHIFT_AUTO_ZCD_IN_CODE)
+#define BIT_GET_AUTO_ZCD_IN_CODE(x)                                            \
+	(((x) >> BIT_SHIFT_AUTO_ZCD_IN_CODE) & BIT_MASK_AUTO_ZCD_IN_CODE)
+
+/* 2 REG_LDO_SWR_CTRL			(Offset 0x007C) */
+
+#define BIT_SHIFT_ZCD_CODE_IN_L 16
+#define BIT_MASK_ZCD_CODE_IN_L 0x1f
+#define BIT_ZCD_CODE_IN_L(x)                                                   \
+	(((x) & BIT_MASK_ZCD_CODE_IN_L) << BIT_SHIFT_ZCD_CODE_IN_L)
+#define BIT_GET_ZCD_CODE_IN_L(x)                                               \
+	(((x) >> BIT_SHIFT_ZCD_CODE_IN_L) & BIT_MASK_ZCD_CODE_IN_L)
+
+/* 2 REG_LDO_SWR_CTRL			(Offset 0x007C) */
+
+#define BIT_SHIFT_LDO_HV5_DUMMY 14
+#define BIT_MASK_LDO_HV5_DUMMY 0x3
+#define BIT_LDO_HV5_DUMMY(x)                                                   \
+	(((x) & BIT_MASK_LDO_HV5_DUMMY) << BIT_SHIFT_LDO_HV5_DUMMY)
+#define BIT_GET_LDO_HV5_DUMMY(x)                                               \
+	(((x) >> BIT_SHIFT_LDO_HV5_DUMMY) & BIT_MASK_LDO_HV5_DUMMY)
+
+/* 2 REG_LDO_SWR_CTRL			(Offset 0x007C) */
+
+#define BIT_SHIFT_REG_VTUNE33_BIT0_TO_BIT1 12
+#define BIT_MASK_REG_VTUNE33_BIT0_TO_BIT1 0x3
+#define BIT_REG_VTUNE33_BIT0_TO_BIT1(x)                                        \
+	(((x) & BIT_MASK_REG_VTUNE33_BIT0_TO_BIT1)                             \
+	 << BIT_SHIFT_REG_VTUNE33_BIT0_TO_BIT1)
+#define BIT_GET_REG_VTUNE33_BIT0_TO_BIT1(x)                                    \
+	(((x) >> BIT_SHIFT_REG_VTUNE33_BIT0_TO_BIT1) &                         \
+	 BIT_MASK_REG_VTUNE33_BIT0_TO_BIT1)
+
+/* 2 REG_LDO_SWR_CTRL			(Offset 0x007C) */
+
+#define BIT_SHIFT_REG_STANDBY33_BIT0_TO_BIT1 10
+#define BIT_MASK_REG_STANDBY33_BIT0_TO_BIT1 0x3
+#define BIT_REG_STANDBY33_BIT0_TO_BIT1(x)                                      \
+	(((x) & BIT_MASK_REG_STANDBY33_BIT0_TO_BIT1)                           \
+	 << BIT_SHIFT_REG_STANDBY33_BIT0_TO_BIT1)
+#define BIT_GET_REG_STANDBY33_BIT0_TO_BIT1(x)                                  \
+	(((x) >> BIT_SHIFT_REG_STANDBY33_BIT0_TO_BIT1) &                       \
+	 BIT_MASK_REG_STANDBY33_BIT0_TO_BIT1)
+
+/* 2 REG_LDO_SWR_CTRL			(Offset 0x007C) */
+
+#define BIT_SHIFT_REG_LOAD33_BIT0_TO_BIT1 8
+#define BIT_MASK_REG_LOAD33_BIT0_TO_BIT1 0x3
+#define BIT_REG_LOAD33_BIT0_TO_BIT1(x)                                         \
+	(((x) & BIT_MASK_REG_LOAD33_BIT0_TO_BIT1)                              \
+	 << BIT_SHIFT_REG_LOAD33_BIT0_TO_BIT1)
+#define BIT_GET_REG_LOAD33_BIT0_TO_BIT1(x)                                     \
+	(((x) >> BIT_SHIFT_REG_LOAD33_BIT0_TO_BIT1) &                          \
+	 BIT_MASK_REG_LOAD33_BIT0_TO_BIT1)
+
+/* 2 REG_LDO_SWR_CTRL			(Offset 0x007C) */
+
+#define BIT_REG_BYPASS_L BIT(7)
+
+/* 2 REG_LDO_SWR_CTRL			(Offset 0x007C) */
+
+#define BIT_REG_LDOF_L BIT(6)
+
+/* 2 REG_LDO_SWR_CTRL			(Offset 0x007C) */
+
+#define BIT_REG_TYPE_L_V1 BIT(5)
+
+/* 2 REG_LDO_SWR_CTRL			(Offset 0x007C) */
+
+#define BIT_ARENB_L BIT(3)
+
+/* 2 REG_LDO_SWR_CTRL			(Offset 0x007C) */
+
+#define BIT_SHIFT_CFC_L 1
+#define BIT_MASK_CFC_L 0x3
+#define BIT_CFC_L(x) (((x) & BIT_MASK_CFC_L) << BIT_SHIFT_CFC_L)
+#define BIT_GET_CFC_L(x) (((x) >> BIT_SHIFT_CFC_L) & BIT_MASK_CFC_L)
+
+/* 2 REG_LDO_SWR_CTRL			(Offset 0x007C) */
+
+#define BIT_REG_OCPS_L_V1 BIT(0)
+
+/* 2 REG_MCUFW_CTRL				(Offset 0x0080) */
+
+#define BIT_ANA_PORT_EN BIT(22)
+#define BIT_MAC_PORT_EN BIT(21)
+#define BIT_BOOT_FSPI_EN BIT(20)
+#define BIT_FW_INIT_RDY BIT(15)
+#define BIT_FW_DW_RDY BIT(14)
+
+/* 2 REG_MCUFW_CTRL				(Offset 0x0080) */
+
+#define BIT_SHIFT_CPU_CLK_SEL 12
+#define BIT_MASK_CPU_CLK_SEL 0x3
+#define BIT_CPU_CLK_SEL(x)                                                     \
+	(((x) & BIT_MASK_CPU_CLK_SEL) << BIT_SHIFT_CPU_CLK_SEL)
+#define BIT_GET_CPU_CLK_SEL(x)                                                 \
+	(((x) >> BIT_SHIFT_CPU_CLK_SEL) & BIT_MASK_CPU_CLK_SEL)
+
+/* 2 REG_MCUFW_CTRL				(Offset 0x0080) */
+
+#define BIT_CCLK_CHG_MASK BIT(11)
+
+/* 2 REG_MCUFW_CTRL				(Offset 0x0080) */
+
+#define BIT_EMEM__TXBUF_CHKSUM_OK BIT(10)
+
+/* 2 REG_MCUFW_CTRL				(Offset 0x0080) */
+
+#define BIT_EMEM_TXBUF_DW_RDY BIT(9)
+
+/* 2 REG_MCUFW_CTRL				(Offset 0x0080) */
+
+#define BIT_EMEM_CHKSUM_OK BIT(8)
+#define BIT_EMEM_DW_OK BIT(7)
+#define BIT_TOGGLING BIT(7)
+#define BIT_DMEM_CHKSUM_OK BIT(6)
+#define BIT_ACK BIT(6)
+
+/* 2 REG_MCUFW_CTRL				(Offset 0x0080) */
+
+#define BIT_DMEM_DW_OK BIT(5)
+
+/* 2 REG_MCUFW_CTRL				(Offset 0x0080) */
+
+#define BIT_IMEM_CHKSUM_OK BIT(4)
+
+/* 2 REG_MCUFW_CTRL				(Offset 0x0080) */
+
+#define BIT_IMEM_DW_OK BIT(3)
+
+/* 2 REG_MCUFW_CTRL				(Offset 0x0080) */
+
+#define BIT_IMEM_BOOT_LOAD_CHKSUM_OK BIT(2)
+
+/* 2 REG_MCUFW_CTRL				(Offset 0x0080) */
+
+#define BIT_IMEM_BOOT_LOAD_DW_OK BIT(1)
+
+/* 2 REG_SDIO_HRPWM1				(Offset 0x10250080) */
+
+#define BIT_32K_PERMISSION BIT(0)
+
+/* 2 REG_MCU_TST_CFG				(Offset 0x0084) */
+
+#define BIT_SHIFT_LBKTST 0
+#define BIT_MASK_LBKTST 0xffff
+#define BIT_LBKTST(x) (((x) & BIT_MASK_LBKTST) << BIT_SHIFT_LBKTST)
+#define BIT_GET_LBKTST(x) (((x) >> BIT_SHIFT_LBKTST) & BIT_MASK_LBKTST)
+
+/* 2 REG_SDIO_BUS_CTRL			(Offset 0x10250085) */
+
+#define BIT_PAD_CLK_XHGE_EN BIT(3)
+#define BIT_INTER_CLK_EN BIT(2)
+#define BIT_EN_RPT_TXCRC BIT(1)
+#define BIT_DIS_RXDMA_STS BIT(0)
+
+/* 2 REG_SDIO_HSUS_CTRL			(Offset 0x10250086) */
+
+#define BIT_INTR_CTRL BIT(4)
+#define BIT_SDIO_VOLTAGE BIT(3)
+#define BIT_BYPASS_INIT BIT(2)
+
+/* 2 REG_SDIO_HSUS_CTRL			(Offset 0x10250086) */
+
+#define BIT_HCI_RESUME_RDY BIT(1)
+#define BIT_HCI_SUS_REQ BIT(0)
+
+/* 2 REG_HMEBOX_E0_E1			(Offset 0x0088) */
+
+#define BIT_SHIFT_HOST_MSG_E1 16
+#define BIT_MASK_HOST_MSG_E1 0xffff
+#define BIT_HOST_MSG_E1(x)                                                     \
+	(((x) & BIT_MASK_HOST_MSG_E1) << BIT_SHIFT_HOST_MSG_E1)
+#define BIT_GET_HOST_MSG_E1(x)                                                 \
+	(((x) >> BIT_SHIFT_HOST_MSG_E1) & BIT_MASK_HOST_MSG_E1)
+
+#define BIT_SHIFT_HOST_MSG_E0 0
+#define BIT_MASK_HOST_MSG_E0 0xffff
+#define BIT_HOST_MSG_E0(x)                                                     \
+	(((x) & BIT_MASK_HOST_MSG_E0) << BIT_SHIFT_HOST_MSG_E0)
+#define BIT_GET_HOST_MSG_E0(x)                                                 \
+	(((x) >> BIT_SHIFT_HOST_MSG_E0) & BIT_MASK_HOST_MSG_E0)
+
+/* 2 REG_SDIO_RESPONSE_TIMER			(Offset 0x10250088) */
+
+#define BIT_SHIFT_CMDIN_2RESP_TIMER 0
+#define BIT_MASK_CMDIN_2RESP_TIMER 0xffff
+#define BIT_CMDIN_2RESP_TIMER(x)                                               \
+	(((x) & BIT_MASK_CMDIN_2RESP_TIMER) << BIT_SHIFT_CMDIN_2RESP_TIMER)
+#define BIT_GET_CMDIN_2RESP_TIMER(x)                                           \
+	(((x) >> BIT_SHIFT_CMDIN_2RESP_TIMER) & BIT_MASK_CMDIN_2RESP_TIMER)
+
+/* 2 REG_SDIO_CMD_CRC			(Offset 0x1025008A) */
+
+#define BIT_SHIFT_SDIO_CMD_CRC_V1 0
+#define BIT_MASK_SDIO_CMD_CRC_V1 0xff
+#define BIT_SDIO_CMD_CRC_V1(x)                                                 \
+	(((x) & BIT_MASK_SDIO_CMD_CRC_V1) << BIT_SHIFT_SDIO_CMD_CRC_V1)
+#define BIT_GET_SDIO_CMD_CRC_V1(x)                                             \
+	(((x) >> BIT_SHIFT_SDIO_CMD_CRC_V1) & BIT_MASK_SDIO_CMD_CRC_V1)
+
+/* 2 REG_HMEBOX_E2_E3			(Offset 0x008C) */
+
+#define BIT_SHIFT_HOST_MSG_E3 16
+#define BIT_MASK_HOST_MSG_E3 0xffff
+#define BIT_HOST_MSG_E3(x)                                                     \
+	(((x) & BIT_MASK_HOST_MSG_E3) << BIT_SHIFT_HOST_MSG_E3)
+#define BIT_GET_HOST_MSG_E3(x)                                                 \
+	(((x) >> BIT_SHIFT_HOST_MSG_E3) & BIT_MASK_HOST_MSG_E3)
+
+#define BIT_SHIFT_HOST_MSG_E2 0
+#define BIT_MASK_HOST_MSG_E2 0xffff
+#define BIT_HOST_MSG_E2(x)                                                     \
+	(((x) & BIT_MASK_HOST_MSG_E2) << BIT_SHIFT_HOST_MSG_E2)
+#define BIT_GET_HOST_MSG_E2(x)                                                 \
+	(((x) >> BIT_SHIFT_HOST_MSG_E2) & BIT_MASK_HOST_MSG_E2)
+
+/* 2 REG_WLLPS_CTRL				(Offset 0x0090) */
+
+#define BIT_WLLPSOP_EABM BIT(31)
+
+/* 2 REG_WLLPS_CTRL				(Offset 0x0090) */
+
+#define BIT_WLLPSOP_ACKF BIT(30)
+
+/* 2 REG_WLLPS_CTRL				(Offset 0x0090) */
+
+#define BIT_WLLPSOP_DLDM BIT(29)
+
+/* 2 REG_WLLPS_CTRL				(Offset 0x0090) */
+
+#define BIT_WLLPSOP_ESWR BIT(28)
+
+/* 2 REG_WLLPS_CTRL				(Offset 0x0090) */
+
+#define BIT_WLLPSOP_PWMM BIT(27)
+#define BIT_WLLPSOP_EECK BIT(26)
+
+/* 2 REG_WLLPS_CTRL				(Offset 0x0090) */
+
+#define BIT_WLLPSOP_WLMACOFF BIT(25)
+
+/* 2 REG_WLLPS_CTRL				(Offset 0x0090) */
+
+#define BIT_WLLPSOP_EXTAL BIT(24)
+
+/* 2 REG_WLLPS_CTRL				(Offset 0x0090) */
+
+#define BIT_WL_SYNPON_VOLTSPDN BIT(23)
+
+/* 2 REG_WLLPS_CTRL				(Offset 0x0090) */
+
+#define BIT_WLLPSOP_WLBBOFF BIT(22)
+
+/* 2 REG_WLLPS_CTRL				(Offset 0x0090) */
+
+#define BIT_WLLPSOP_WLMEM_DS BIT(21)
+
+/* 2 REG_WLLPS_CTRL				(Offset 0x0090) */
+
+#define BIT_SHIFT_LPLDH12_VADJ_STEP_DN 12
+#define BIT_MASK_LPLDH12_VADJ_STEP_DN 0xf
+#define BIT_LPLDH12_VADJ_STEP_DN(x)                                            \
+	(((x) & BIT_MASK_LPLDH12_VADJ_STEP_DN)                                 \
+	 << BIT_SHIFT_LPLDH12_VADJ_STEP_DN)
+#define BIT_GET_LPLDH12_VADJ_STEP_DN(x)                                        \
+	(((x) >> BIT_SHIFT_LPLDH12_VADJ_STEP_DN) &                             \
+	 BIT_MASK_LPLDH12_VADJ_STEP_DN)
+
+/* 2 REG_WLLPS_CTRL				(Offset 0x0090) */
+
+#define BIT_SHIFT_V15ADJ_L1_STEP_DN 8
+#define BIT_MASK_V15ADJ_L1_STEP_DN 0x7
+#define BIT_V15ADJ_L1_STEP_DN(x)                                               \
+	(((x) & BIT_MASK_V15ADJ_L1_STEP_DN) << BIT_SHIFT_V15ADJ_L1_STEP_DN)
+#define BIT_GET_V15ADJ_L1_STEP_DN(x)                                           \
+	(((x) >> BIT_SHIFT_V15ADJ_L1_STEP_DN) & BIT_MASK_V15ADJ_L1_STEP_DN)
+
+#define BIT_REGU_32K_CLK_EN BIT(1)
+#define BIT_DRV_WLAN_INT_CLR BIT(1)
+
+/* 2 REG_WLLPS_CTRL				(Offset 0x0090) */
+
+#define BIT_WL_LPS_EN BIT(0)
+
+/* 2 REG_SDIO_HSISR				(Offset 0x10250090) */
+
+#define BIT_DRV_WLAN_INT BIT(0)
+
+/* 2 REG_SDIO_HSIMR				(Offset 0x10250091) */
+
+#define BIT_HISR_MASK BIT(0)
+
+/* 2 REG_AFE_CTRL5				(Offset 0x0094) */
+
+#define BIT_BB_DBG_SEL_AFE_SDM_BIT0 BIT(31)
+
+/* 2 REG_AFE_CTRL5				(Offset 0x0094) */
+
+#define BIT_ORDER_SDM BIT(30)
+#define BIT_RFE_SEL_SDM BIT(29)
+
+#define BIT_SHIFT_REF_SEL 25
+#define BIT_MASK_REF_SEL 0xf
+#define BIT_REF_SEL(x) (((x) & BIT_MASK_REF_SEL) << BIT_SHIFT_REF_SEL)
+#define BIT_GET_REF_SEL(x) (((x) >> BIT_SHIFT_REF_SEL) & BIT_MASK_REF_SEL)
+
+/* 2 REG_AFE_CTRL5				(Offset 0x0094) */
+
+#define BIT_SHIFT_F0F_SDM 12
+#define BIT_MASK_F0F_SDM 0x1fff
+#define BIT_F0F_SDM(x) (((x) & BIT_MASK_F0F_SDM) << BIT_SHIFT_F0F_SDM)
+#define BIT_GET_F0F_SDM(x) (((x) >> BIT_SHIFT_F0F_SDM) & BIT_MASK_F0F_SDM)
+
+/* 2 REG_AFE_CTRL5				(Offset 0x0094) */
+
+#define BIT_SHIFT_F0N_SDM 9
+#define BIT_MASK_F0N_SDM 0x7
+#define BIT_F0N_SDM(x) (((x) & BIT_MASK_F0N_SDM) << BIT_SHIFT_F0N_SDM)
+#define BIT_GET_F0N_SDM(x) (((x) >> BIT_SHIFT_F0N_SDM) & BIT_MASK_F0N_SDM)
+
+/* 2 REG_AFE_CTRL5				(Offset 0x0094) */
+
+#define BIT_SHIFT_DIVN_SDM 3
+#define BIT_MASK_DIVN_SDM 0x3f
+#define BIT_DIVN_SDM(x) (((x) & BIT_MASK_DIVN_SDM) << BIT_SHIFT_DIVN_SDM)
+#define BIT_GET_DIVN_SDM(x) (((x) >> BIT_SHIFT_DIVN_SDM) & BIT_MASK_DIVN_SDM)
+
+/* 2 REG_GPIO_DEBOUNCE_CTRL			(Offset 0x0098) */
+
+#define BIT_WLGP_DBC1EN BIT(15)
+
+#define BIT_SHIFT_WLGP_DBC1 8
+#define BIT_MASK_WLGP_DBC1 0xf
+#define BIT_WLGP_DBC1(x) (((x) & BIT_MASK_WLGP_DBC1) << BIT_SHIFT_WLGP_DBC1)
+#define BIT_GET_WLGP_DBC1(x) (((x) >> BIT_SHIFT_WLGP_DBC1) & BIT_MASK_WLGP_DBC1)
+
+#define BIT_WLGP_DBC0EN BIT(7)
+
+#define BIT_SHIFT_WLGP_DBC0 0
+#define BIT_MASK_WLGP_DBC0 0xf
+#define BIT_WLGP_DBC0(x) (((x) & BIT_MASK_WLGP_DBC0) << BIT_SHIFT_WLGP_DBC0)
+#define BIT_GET_WLGP_DBC0(x) (((x) >> BIT_SHIFT_WLGP_DBC0) & BIT_MASK_WLGP_DBC0)
+
+/* 2 REG_RPWM2				(Offset 0x009C) */
+
+#define BIT_SHIFT_RPWM2 16
+#define BIT_MASK_RPWM2 0xffff
+#define BIT_RPWM2(x) (((x) & BIT_MASK_RPWM2) << BIT_SHIFT_RPWM2)
+#define BIT_GET_RPWM2(x) (((x) >> BIT_SHIFT_RPWM2) & BIT_MASK_RPWM2)
+
+/* 2 REG_SYSON_FSM_MON			(Offset 0x00A0) */
+
+#define BIT_SHIFT_FSM_MON_SEL 24
+#define BIT_MASK_FSM_MON_SEL 0x7
+#define BIT_FSM_MON_SEL(x)                                                     \
+	(((x) & BIT_MASK_FSM_MON_SEL) << BIT_SHIFT_FSM_MON_SEL)
+#define BIT_GET_FSM_MON_SEL(x)                                                 \
+	(((x) >> BIT_SHIFT_FSM_MON_SEL) & BIT_MASK_FSM_MON_SEL)
+
+#define BIT_DOP_ELDO BIT(23)
+#define BIT_FSM_MON_UPD BIT(15)
+
+#define BIT_SHIFT_FSM_PAR 0
+#define BIT_MASK_FSM_PAR 0x7fff
+#define BIT_FSM_PAR(x) (((x) & BIT_MASK_FSM_PAR) << BIT_SHIFT_FSM_PAR)
+#define BIT_GET_FSM_PAR(x) (((x) >> BIT_SHIFT_FSM_PAR) & BIT_MASK_FSM_PAR)
+
+/* 2 REG_AFE_CTRL6				(Offset 0x00A4) */
+
+#define BIT_SHIFT_BB_DBG_SEL_AFE_SDM_BIT3_1 0
+#define BIT_MASK_BB_DBG_SEL_AFE_SDM_BIT3_1 0x7
+#define BIT_BB_DBG_SEL_AFE_SDM_BIT3_1(x)                                       \
+	(((x) & BIT_MASK_BB_DBG_SEL_AFE_SDM_BIT3_1)                            \
+	 << BIT_SHIFT_BB_DBG_SEL_AFE_SDM_BIT3_1)
+#define BIT_GET_BB_DBG_SEL_AFE_SDM_BIT3_1(x)                                   \
+	(((x) >> BIT_SHIFT_BB_DBG_SEL_AFE_SDM_BIT3_1) &                        \
+	 BIT_MASK_BB_DBG_SEL_AFE_SDM_BIT3_1)
+
+/* 2 REG_PMC_DBG_CTRL1			(Offset 0x00A8) */
+
+#define BIT_BT_INT_EN BIT(31)
+
+#define BIT_SHIFT_RD_WR_WIFI_BT_INFO 16
+#define BIT_MASK_RD_WR_WIFI_BT_INFO 0x7fff
+#define BIT_RD_WR_WIFI_BT_INFO(x)                                              \
+	(((x) & BIT_MASK_RD_WR_WIFI_BT_INFO) << BIT_SHIFT_RD_WR_WIFI_BT_INFO)
+#define BIT_GET_RD_WR_WIFI_BT_INFO(x)                                          \
+	(((x) >> BIT_SHIFT_RD_WR_WIFI_BT_INFO) & BIT_MASK_RD_WR_WIFI_BT_INFO)
+
+/* 2 REG_PMC_DBG_CTRL1			(Offset 0x00A8) */
+
+#define BIT_PMC_WR_OVF BIT(8)
+
+#define BIT_SHIFT_WLPMC_ERRINT 0
+#define BIT_MASK_WLPMC_ERRINT 0xff
+#define BIT_WLPMC_ERRINT(x)                                                    \
+	(((x) & BIT_MASK_WLPMC_ERRINT) << BIT_SHIFT_WLPMC_ERRINT)
+#define BIT_GET_WLPMC_ERRINT(x)                                                \
+	(((x) >> BIT_SHIFT_WLPMC_ERRINT) & BIT_MASK_WLPMC_ERRINT)
+
+/* 2 REG_AFE_CTRL7				(Offset 0x00AC) */
+
+#define BIT_SHIFT_SEL_V 30
+#define BIT_MASK_SEL_V 0x3
+#define BIT_SEL_V(x) (((x) & BIT_MASK_SEL_V) << BIT_SHIFT_SEL_V)
+#define BIT_GET_SEL_V(x) (((x) >> BIT_SHIFT_SEL_V) & BIT_MASK_SEL_V)
+
+/* 2 REG_AFE_CTRL7				(Offset 0x00AC) */
+
+#define BIT_TXFIFO_TH_INT BIT(30)
+
+/* 2 REG_AFE_CTRL7				(Offset 0x00AC) */
+
+#define BIT_SEL_LDO_PC BIT(29)
+
+/* 2 REG_AFE_CTRL7				(Offset 0x00AC) */
+
+#define BIT_SHIFT_CK_MON_SEL 26
+#define BIT_MASK_CK_MON_SEL 0x7
+#define BIT_CK_MON_SEL(x) (((x) & BIT_MASK_CK_MON_SEL) << BIT_SHIFT_CK_MON_SEL)
+#define BIT_GET_CK_MON_SEL(x)                                                  \
+	(((x) >> BIT_SHIFT_CK_MON_SEL) & BIT_MASK_CK_MON_SEL)
+
+/* 2 REG_AFE_CTRL7				(Offset 0x00AC) */
+
+#define BIT_CK_MON_EN BIT(25)
+#define BIT_FREF_EDGE BIT(24)
+#define BIT_CK320M_EN BIT(23)
+#define BIT_CK_5M_EN BIT(22)
+#define BIT_TESTEN BIT(21)
+
+/* 2 REG_HIMR0				(Offset 0x00B0) */
+
+#define BIT_TIMEOUT_INTERRUPT2_MASK BIT(31)
+#define BIT_TIMEOUT_INTERRUTP1_MASK BIT(30)
+#define BIT_PSTIMEOUT_MSK BIT(29)
+#define BIT_GTINT4_MSK BIT(28)
+#define BIT_GTINT3_MSK BIT(27)
+#define BIT_TXBCN0ERR_MSK BIT(26)
+#define BIT_TXBCN0OK_MSK BIT(25)
+#define BIT_TSF_BIT32_TOGGLE_MSK BIT(24)
+#define BIT_BCNDMAINT0_MSK BIT(20)
+#define BIT_BCNDERR0_MSK BIT(16)
+#define BIT_HSISR_IND_ON_INT_MSK BIT(15)
+
+/* 2 REG_HIMR0				(Offset 0x00B0) */
+
+#define BIT_BCNDMAINT_E_MSK BIT(14)
+
+/* 2 REG_HIMR0				(Offset 0x00B0) */
+
+#define BIT_CTWEND_MSK BIT(12)
+#define BIT_HISR1_IND_MSK BIT(11)
+
+/* 2 REG_HIMR0				(Offset 0x00B0) */
+
+#define BIT_C2HCMD_MSK BIT(10)
+#define BIT_CPWM2_MSK BIT(9)
+#define BIT_CPWM_MSK BIT(8)
+#define BIT_HIGHDOK_MSK BIT(7)
+#define BIT_MGTDOK_MSK BIT(6)
+#define BIT_BKDOK_MSK BIT(5)
+#define BIT_BEDOK_MSK BIT(4)
+#define BIT_VIDOK_MSK BIT(3)
+#define BIT_VODOK_MSK BIT(2)
+#define BIT_RDU_MSK BIT(1)
+#define BIT_RXOK_MSK BIT(0)
+
+/* 2 REG_HISR0				(Offset 0x00B4) */
+
+#define BIT_TIMEOUT_INTERRUPT2 BIT(31)
+
+/* 2 REG_HISR0				(Offset 0x00B4) */
+
+#define BIT_TIMEOUT_INTERRUTP1 BIT(30)
+
+/* 2 REG_HISR0				(Offset 0x00B4) */
+
+#define BIT_PSTIMEOUT BIT(29)
+#define BIT_GTINT4 BIT(28)
+#define BIT_GTINT3 BIT(27)
+#define BIT_TXBCN0ERR BIT(26)
+#define BIT_TXBCN0OK BIT(25)
+#define BIT_TSF_BIT32_TOGGLE BIT(24)
+#define BIT_BCNDMAINT0 BIT(20)
+#define BIT_BCNDERR0 BIT(16)
+#define BIT_HSISR_IND_ON_INT BIT(15)
+
+/* 2 REG_HISR0				(Offset 0x00B4) */
+
+#define BIT_BCNDMAINT_E BIT(14)
+
+/* 2 REG_HISR0				(Offset 0x00B4) */
+
+#define BIT_CTWEND BIT(12)
+
+/* 2 REG_HISR0				(Offset 0x00B4) */
+
+#define BIT_HISR1_IND_INT BIT(11)
+#define BIT_C2HCMD BIT(10)
+#define BIT_CPWM2 BIT(9)
+#define BIT_CPWM BIT(8)
+#define BIT_HIGHDOK BIT(7)
+#define BIT_MGTDOK BIT(6)
+#define BIT_BKDOK BIT(5)
+#define BIT_BEDOK BIT(4)
+#define BIT_VIDOK BIT(3)
+#define BIT_VODOK BIT(2)
+#define BIT_RDU BIT(1)
+#define BIT_RXOK BIT(0)
+
+/* 2 REG_HIMR1				(Offset 0x00B8) */
+
+#define BIT_BTON_STS_UPDATE_MASK BIT(29)
+
+/* 2 REG_HIMR1				(Offset 0x00B8) */
+
+#define BIT_MCU_ERR_MASK BIT(28)
+
+/* 2 REG_HIMR1				(Offset 0x00B8) */
+
+#define BIT_BCNDMAINT7__MSK BIT(27)
+
+/* 2 REG_HIMR1				(Offset 0x00B8) */
+
+#define BIT_BCNDMAINT6__MSK BIT(26)
+
+/* 2 REG_HIMR1				(Offset 0x00B8) */
+
+#define BIT_BCNDMAINT5__MSK BIT(25)
+
+/* 2 REG_HIMR1				(Offset 0x00B8) */
+
+#define BIT_BCNDMAINT4__MSK BIT(24)
+
+/* 2 REG_HIMR1				(Offset 0x00B8) */
+
+#define BIT_BCNDMAINT3_MSK BIT(23)
+#define BIT_BCNDMAINT2_MSK BIT(22)
+#define BIT_BCNDMAINT1_MSK BIT(21)
+#define BIT_BCNDERR7_MSK BIT(20)
+#define BIT_BCNDERR6_MSK BIT(19)
+#define BIT_BCNDERR5_MSK BIT(18)
+#define BIT_BCNDERR4_MSK BIT(17)
+#define BIT_BCNDERR3_MSK BIT(16)
+#define BIT_BCNDERR2_MSK BIT(15)
+#define BIT_BCNDERR1_MSK BIT(14)
+
+/* 2 REG_HIMR1				(Offset 0x00B8) */
+
+#define BIT_ATIMEND_E_MSK BIT(13)
+
+/* 2 REG_HIMR1				(Offset 0x00B8) */
+
+#define BIT_ATIMEND__MSK BIT(12)
+
+/* 2 REG_HIMR1				(Offset 0x00B8) */
+
+#define BIT_TXERR_MSK BIT(11)
+#define BIT_RXERR_MSK BIT(10)
+#define BIT_TXFOVW_MSK BIT(9)
+#define BIT_FOVW_MSK BIT(8)
+
+/* 2 REG_HIMR1				(Offset 0x00B8) */
+
+#define BIT_CPU_MGQ_TXDONE_MSK BIT(5)
+#define BIT_PS_TIMER_C_MSK BIT(4)
+#define BIT_PS_TIMER_B_MSK BIT(3)
+#define BIT_PS_TIMER_A_MSK BIT(2)
+#define BIT_CPUMGQ_TX_TIMER_MSK BIT(1)
+
+/* 2 REG_HISR1				(Offset 0x00BC) */
+
+#define BIT_BTON_STS_UPDATE_INT BIT(29)
+
+/* 2 REG_HISR1				(Offset 0x00BC) */
+
+#define BIT_MCU_ERR BIT(28)
+
+/* 2 REG_HISR1				(Offset 0x00BC) */
+
+#define BIT_BCNDMAINT7 BIT(27)
+#define BIT_BCNDMAINT6 BIT(26)
+#define BIT_BCNDMAINT5 BIT(25)
+#define BIT_BCNDMAINT4 BIT(24)
+#define BIT_BCNDMAINT3 BIT(23)
+#define BIT_BCNDMAINT2 BIT(22)
+#define BIT_BCNDMAINT1 BIT(21)
+#define BIT_BCNDERR7 BIT(20)
+#define BIT_BCNDERR6 BIT(19)
+#define BIT_BCNDERR5 BIT(18)
+#define BIT_BCNDERR4 BIT(17)
+#define BIT_BCNDERR3 BIT(16)
+#define BIT_BCNDERR2 BIT(15)
+#define BIT_BCNDERR1 BIT(14)
+
+/* 2 REG_HISR1				(Offset 0x00BC) */
+
+#define BIT_ATIMEND_E BIT(13)
+
+/* 2 REG_HISR1				(Offset 0x00BC) */
+
+#define BIT_ATIMEND BIT(12)
+#define BIT_TXERR_INT BIT(11)
+#define BIT_RXERR_INT BIT(10)
+#define BIT_TXFOVW BIT(9)
+#define BIT_FOVW BIT(8)
+
+/* 2 REG_HISR1				(Offset 0x00BC) */
+
+#define BIT_CPU_MGQ_TXDONE BIT(5)
+#define BIT_PS_TIMER_C BIT(4)
+#define BIT_PS_TIMER_B BIT(3)
+#define BIT_PS_TIMER_A BIT(2)
+#define BIT_CPUMGQ_TX_TIMER BIT(1)
+
+/* 2 REG_SDIO_ERR_RPT			(Offset 0x102500C0) */
+
+#define BIT_HR_FF_OVF BIT(6)
+#define BIT_HR_FF_UDN BIT(5)
+#define BIT_TXDMA_BUSY_ERR BIT(4)
+#define BIT_TXDMA_VLD_ERR BIT(3)
+#define BIT_QSEL_UNKNOWN_ERR BIT(2)
+#define BIT_QSEL_MIS_ERR BIT(1)
+
+/* 2 REG_DBG_PORT_SEL			(Offset 0x00C0) */
+
+#define BIT_SHIFT_DEBUG_ST 0
+#define BIT_MASK_DEBUG_ST 0xffffffffL
+#define BIT_DEBUG_ST(x) (((x) & BIT_MASK_DEBUG_ST) << BIT_SHIFT_DEBUG_ST)
+#define BIT_GET_DEBUG_ST(x) (((x) >> BIT_SHIFT_DEBUG_ST) & BIT_MASK_DEBUG_ST)
+
+/* 2 REG_SDIO_ERR_RPT			(Offset 0x102500C0) */
+
+#define BIT_SDIO_OVERRD_ERR BIT(0)
+
+/* 2 REG_SDIO_CMD_ERRCNT			(Offset 0x102500C1) */
+
+#define BIT_SHIFT_CMD_CRC_ERR_CNT 0
+#define BIT_MASK_CMD_CRC_ERR_CNT 0xff
+#define BIT_CMD_CRC_ERR_CNT(x)                                                 \
+	(((x) & BIT_MASK_CMD_CRC_ERR_CNT) << BIT_SHIFT_CMD_CRC_ERR_CNT)
+#define BIT_GET_CMD_CRC_ERR_CNT(x)                                             \
+	(((x) >> BIT_SHIFT_CMD_CRC_ERR_CNT) & BIT_MASK_CMD_CRC_ERR_CNT)
+
+/* 2 REG_SDIO_DATA_ERRCNT			(Offset 0x102500C2) */
+
+#define BIT_SHIFT_DATA_CRC_ERR_CNT 0
+#define BIT_MASK_DATA_CRC_ERR_CNT 0xff
+#define BIT_DATA_CRC_ERR_CNT(x)                                                \
+	(((x) & BIT_MASK_DATA_CRC_ERR_CNT) << BIT_SHIFT_DATA_CRC_ERR_CNT)
+#define BIT_GET_DATA_CRC_ERR_CNT(x)                                            \
+	(((x) >> BIT_SHIFT_DATA_CRC_ERR_CNT) & BIT_MASK_DATA_CRC_ERR_CNT)
+
+/* 2 REG_PAD_CTRL2				(Offset 0x00C4) */
+
+#define BIT_USB3_USB2_TRANSITION BIT(20)
+
+/* 2 REG_PAD_CTRL2				(Offset 0x00C4) */
+
+#define BIT_SHIFT_USB23_SW_MODE_V1 18
+#define BIT_MASK_USB23_SW_MODE_V1 0x3
+#define BIT_USB23_SW_MODE_V1(x)                                                \
+	(((x) & BIT_MASK_USB23_SW_MODE_V1) << BIT_SHIFT_USB23_SW_MODE_V1)
+#define BIT_GET_USB23_SW_MODE_V1(x)                                            \
+	(((x) >> BIT_SHIFT_USB23_SW_MODE_V1) & BIT_MASK_USB23_SW_MODE_V1)
+
+/* 2 REG_PAD_CTRL2				(Offset 0x00C4) */
+
+#define BIT_NO_PDN_CHIPOFF_V1 BIT(17)
+
+/* 2 REG_PAD_CTRL2				(Offset 0x00C4) */
+
+#define BIT_RSM_EN_V1 BIT(16)
+
+/* 2 REG_PAD_CTRL2				(Offset 0x00C4) */
+
+#define BIT_LD_B12V_EN BIT(7)
+
+/* 2 REG_PAD_CTRL2				(Offset 0x00C4) */
+
+#define BIT_EECS_IOSEL_V1 BIT(6)
+
+/* 2 REG_PAD_CTRL2				(Offset 0x00C4) */
+
+#define BIT_EECS_DATA_O_V1 BIT(5)
+
+/* 2 REG_PAD_CTRL2				(Offset 0x00C4) */
+
+#define BIT_EECS_DATA_I_V1 BIT(4)
+
+/* 2 REG_PAD_CTRL2				(Offset 0x00C4) */
+
+#define BIT_EESK_IOSEL_V1 BIT(2)
+
+/* 2 REG_PAD_CTRL2				(Offset 0x00C4) */
+
+#define BIT_EESK_DATA_O_V1 BIT(1)
+
+/* 2 REG_PAD_CTRL2				(Offset 0x00C4) */
+
+#define BIT_EESK_DATA_I_V1 BIT(0)
+
+/* 2 REG_SDIO_CMD_ERR_CONTENT		(Offset 0x102500C4) */
+
+#define BIT_SHIFT_SDIO_CMD_ERR_CONTENT 0
+#define BIT_MASK_SDIO_CMD_ERR_CONTENT 0xffffffffffL
+#define BIT_SDIO_CMD_ERR_CONTENT(x)                                            \
+	(((x) & BIT_MASK_SDIO_CMD_ERR_CONTENT)                                 \
+	 << BIT_SHIFT_SDIO_CMD_ERR_CONTENT)
+#define BIT_GET_SDIO_CMD_ERR_CONTENT(x)                                        \
+	(((x) >> BIT_SHIFT_SDIO_CMD_ERR_CONTENT) &                             \
+	 BIT_MASK_SDIO_CMD_ERR_CONTENT)
+
+/* 2 REG_SDIO_CRC_ERR_IDX			(Offset 0x102500C9) */
+
+#define BIT_D3_CRC_ERR BIT(4)
+#define BIT_D2_CRC_ERR BIT(3)
+#define BIT_D1_CRC_ERR BIT(2)
+#define BIT_D0_CRC_ERR BIT(1)
+#define BIT_CMD_CRC_ERR BIT(0)
+
+/* 2 REG_SDIO_DATA_CRC			(Offset 0x102500CA) */
+
+#define BIT_SHIFT_SDIO_DATA_CRC 0
+#define BIT_MASK_SDIO_DATA_CRC 0xff
+#define BIT_SDIO_DATA_CRC(x)                                                   \
+	(((x) & BIT_MASK_SDIO_DATA_CRC) << BIT_SHIFT_SDIO_DATA_CRC)
+#define BIT_GET_SDIO_DATA_CRC(x)                                               \
+	(((x) >> BIT_SHIFT_SDIO_DATA_CRC) & BIT_MASK_SDIO_DATA_CRC)
+
+/* 2 REG_SDIO_DATA_REPLY_TIME		(Offset 0x102500CB) */
+
+#define BIT_SHIFT_SDIO_DATA_REPLY_TIME 0
+#define BIT_MASK_SDIO_DATA_REPLY_TIME 0x7
+#define BIT_SDIO_DATA_REPLY_TIME(x)                                            \
+	(((x) & BIT_MASK_SDIO_DATA_REPLY_TIME)                                 \
+	 << BIT_SHIFT_SDIO_DATA_REPLY_TIME)
+#define BIT_GET_SDIO_DATA_REPLY_TIME(x)                                        \
+	(((x) >> BIT_SHIFT_SDIO_DATA_REPLY_TIME) &                             \
+	 BIT_MASK_SDIO_DATA_REPLY_TIME)
+
+/* 2 REG_PMC_DBG_CTRL2			(Offset 0x00CC) */
+
+#define BIT_SHIFT_EFUSE_BURN_GNT 24
+#define BIT_MASK_EFUSE_BURN_GNT 0xff
+#define BIT_EFUSE_BURN_GNT(x)                                                  \
+	(((x) & BIT_MASK_EFUSE_BURN_GNT) << BIT_SHIFT_EFUSE_BURN_GNT)
+#define BIT_GET_EFUSE_BURN_GNT(x)                                              \
+	(((x) >> BIT_SHIFT_EFUSE_BURN_GNT) & BIT_MASK_EFUSE_BURN_GNT)
+
+/* 2 REG_PMC_DBG_CTRL2			(Offset 0x00CC) */
+
+#define BIT_STOP_WL_PMC BIT(9)
+#define BIT_STOP_SYM_PMC BIT(8)
+
+/* 2 REG_PMC_DBG_CTRL2			(Offset 0x00CC) */
+
+#define BIT_REG_RST_WLPMC BIT(5)
+#define BIT_REG_RST_PD12N BIT(4)
+#define BIT_SYSON_DIS_WLREG_WRMSK BIT(3)
+#define BIT_SYSON_DIS_PMCREG_WRMSK BIT(2)
+
+#define BIT_SHIFT_SYSON_REG_ARB 0
+#define BIT_MASK_SYSON_REG_ARB 0x3
+#define BIT_SYSON_REG_ARB(x)                                                   \
+	(((x) & BIT_MASK_SYSON_REG_ARB) << BIT_SHIFT_SYSON_REG_ARB)
+#define BIT_GET_SYSON_REG_ARB(x)                                               \
+	(((x) >> BIT_SHIFT_SYSON_REG_ARB) & BIT_MASK_SYSON_REG_ARB)
+
+/* 2 REG_BIST_CTRL				(Offset 0x00D0) */
+
+#define BIT_BIST_USB_DIS BIT(27)
+
+/* 2 REG_BIST_CTRL				(Offset 0x00D0) */
+
+#define BIT_BIST_PCI_DIS BIT(26)
+
+/* 2 REG_BIST_CTRL				(Offset 0x00D0) */
+
+#define BIT_BIST_BT_DIS BIT(25)
+
+/* 2 REG_BIST_CTRL				(Offset 0x00D0) */
+
+#define BIT_BIST_WL_DIS BIT(24)
+
+/* 2 REG_BIST_CTRL				(Offset 0x00D0) */
+
+#define BIT_SHIFT_BIST_RPT_SEL 16
+#define BIT_MASK_BIST_RPT_SEL 0xf
+#define BIT_BIST_RPT_SEL(x)                                                    \
+	(((x) & BIT_MASK_BIST_RPT_SEL) << BIT_SHIFT_BIST_RPT_SEL)
+#define BIT_GET_BIST_RPT_SEL(x)                                                \
+	(((x) >> BIT_SHIFT_BIST_RPT_SEL) & BIT_MASK_BIST_RPT_SEL)
+
+/* 2 REG_BIST_CTRL				(Offset 0x00D0) */
+
+#define BIT_BIST_RESUME_PS BIT(4)
+
+/* 2 REG_BIST_CTRL				(Offset 0x00D0) */
+
+#define BIT_BIST_RESUME BIT(3)
+#define BIT_BIST_NORMAL BIT(2)
+
+/* 2 REG_BIST_CTRL				(Offset 0x00D0) */
+
+#define BIT_BIST_RSTN BIT(1)
+#define BIT_BIST_CLK_EN BIT(0)
+
+/* 2 REG_BIST_RPT				(Offset 0x00D4) */
+
+#define BIT_SHIFT_MBIST_REPORT 0
+#define BIT_MASK_MBIST_REPORT 0xffffffffL
+#define BIT_MBIST_REPORT(x)                                                    \
+	(((x) & BIT_MASK_MBIST_REPORT) << BIT_SHIFT_MBIST_REPORT)
+#define BIT_GET_MBIST_REPORT(x)                                                \
+	(((x) >> BIT_SHIFT_MBIST_REPORT) & BIT_MASK_MBIST_REPORT)
+
+/* 2 REG_MEM_CTRL				(Offset 0x00D8) */
+
+#define BIT_UMEM_RME BIT(31)
+
+/* 2 REG_MEM_CTRL				(Offset 0x00D8) */
+
+#define BIT_SHIFT_BT_SPRAM 28
+#define BIT_MASK_BT_SPRAM 0x3
+#define BIT_BT_SPRAM(x) (((x) & BIT_MASK_BT_SPRAM) << BIT_SHIFT_BT_SPRAM)
+#define BIT_GET_BT_SPRAM(x) (((x) >> BIT_SHIFT_BT_SPRAM) & BIT_MASK_BT_SPRAM)
+
+/* 2 REG_MEM_CTRL				(Offset 0x00D8) */
+
+#define BIT_SHIFT_BT_ROM 24
+#define BIT_MASK_BT_ROM 0xf
+#define BIT_BT_ROM(x) (((x) & BIT_MASK_BT_ROM) << BIT_SHIFT_BT_ROM)
+#define BIT_GET_BT_ROM(x) (((x) >> BIT_SHIFT_BT_ROM) & BIT_MASK_BT_ROM)
+
+#define BIT_SHIFT_PCI_DPRAM 10
+#define BIT_MASK_PCI_DPRAM 0x3
+#define BIT_PCI_DPRAM(x) (((x) & BIT_MASK_PCI_DPRAM) << BIT_SHIFT_PCI_DPRAM)
+#define BIT_GET_PCI_DPRAM(x) (((x) >> BIT_SHIFT_PCI_DPRAM) & BIT_MASK_PCI_DPRAM)
+
+/* 2 REG_MEM_CTRL				(Offset 0x00D8) */
+
+#define BIT_SHIFT_PCI_SPRAM 8
+#define BIT_MASK_PCI_SPRAM 0x3
+#define BIT_PCI_SPRAM(x) (((x) & BIT_MASK_PCI_SPRAM) << BIT_SHIFT_PCI_SPRAM)
+#define BIT_GET_PCI_SPRAM(x) (((x) >> BIT_SHIFT_PCI_SPRAM) & BIT_MASK_PCI_SPRAM)
+
+#define BIT_SHIFT_USB_SPRAM 6
+#define BIT_MASK_USB_SPRAM 0x3
+#define BIT_USB_SPRAM(x) (((x) & BIT_MASK_USB_SPRAM) << BIT_SHIFT_USB_SPRAM)
+#define BIT_GET_USB_SPRAM(x) (((x) >> BIT_SHIFT_USB_SPRAM) & BIT_MASK_USB_SPRAM)
+
+/* 2 REG_MEM_CTRL				(Offset 0x00D8) */
+
+#define BIT_SHIFT_USB_SPRF 4
+#define BIT_MASK_USB_SPRF 0x3
+#define BIT_USB_SPRF(x) (((x) & BIT_MASK_USB_SPRF) << BIT_SHIFT_USB_SPRF)
+#define BIT_GET_USB_SPRF(x) (((x) >> BIT_SHIFT_USB_SPRF) & BIT_MASK_USB_SPRF)
+
+/* 2 REG_MEM_CTRL				(Offset 0x00D8) */
+
+#define BIT_SHIFT_MCU_ROM 0
+#define BIT_MASK_MCU_ROM 0xf
+#define BIT_MCU_ROM(x) (((x) & BIT_MASK_MCU_ROM) << BIT_SHIFT_MCU_ROM)
+#define BIT_GET_MCU_ROM(x) (((x) >> BIT_SHIFT_MCU_ROM) & BIT_MASK_MCU_ROM)
+
+/* 2 REG_AFE_CTRL8				(Offset 0x00DC) */
+
+#define BIT_SYN_AGPIO BIT(20)
+
+/* 2 REG_AFE_CTRL8				(Offset 0x00DC) */
+
+#define BIT_XTAL_LP BIT(4)
+#define BIT_XTAL_GM_SEP BIT(3)
+
+/* 2 REG_AFE_CTRL8				(Offset 0x00DC) */
+
+#define BIT_SHIFT_XTAL_SEL_TOK 0
+#define BIT_MASK_XTAL_SEL_TOK 0x7
+#define BIT_XTAL_SEL_TOK(x)                                                    \
+	(((x) & BIT_MASK_XTAL_SEL_TOK) << BIT_SHIFT_XTAL_SEL_TOK)
+#define BIT_GET_XTAL_SEL_TOK(x)                                                \
+	(((x) >> BIT_SHIFT_XTAL_SEL_TOK) & BIT_MASK_XTAL_SEL_TOK)
+
+/* 2 REG_USB_SIE_INTF			(Offset 0x00E0) */
+
+#define BIT_RD_SEL BIT(31)
+
+/* 2 REG_USB_SIE_INTF			(Offset 0x00E0) */
+
+#define BIT_USB_SIE_INTF_WE_V1 BIT(30)
+#define BIT_USB_SIE_INTF_BYIOREG_V1 BIT(29)
+#define BIT_USB_SIE_SELECT BIT(28)
+
+/* 2 REG_USB_SIE_INTF			(Offset 0x00E0) */
+
+#define BIT_SHIFT_USB_SIE_INTF_ADDR_V1 16
+#define BIT_MASK_USB_SIE_INTF_ADDR_V1 0x1ff
+#define BIT_USB_SIE_INTF_ADDR_V1(x)                                            \
+	(((x) & BIT_MASK_USB_SIE_INTF_ADDR_V1)                                 \
+	 << BIT_SHIFT_USB_SIE_INTF_ADDR_V1)
+#define BIT_GET_USB_SIE_INTF_ADDR_V1(x)                                        \
+	(((x) >> BIT_SHIFT_USB_SIE_INTF_ADDR_V1) &                             \
+	 BIT_MASK_USB_SIE_INTF_ADDR_V1)
+
+/* 2 REG_USB_SIE_INTF			(Offset 0x00E0) */
+
+#define BIT_SHIFT_USB_SIE_INTF_RD 8
+#define BIT_MASK_USB_SIE_INTF_RD 0xff
+#define BIT_USB_SIE_INTF_RD(x)                                                 \
+	(((x) & BIT_MASK_USB_SIE_INTF_RD) << BIT_SHIFT_USB_SIE_INTF_RD)
+#define BIT_GET_USB_SIE_INTF_RD(x)                                             \
+	(((x) >> BIT_SHIFT_USB_SIE_INTF_RD) & BIT_MASK_USB_SIE_INTF_RD)
+
+#define BIT_SHIFT_USB_SIE_INTF_WD 0
+#define BIT_MASK_USB_SIE_INTF_WD 0xff
+#define BIT_USB_SIE_INTF_WD(x)                                                 \
+	(((x) & BIT_MASK_USB_SIE_INTF_WD) << BIT_SHIFT_USB_SIE_INTF_WD)
+#define BIT_GET_USB_SIE_INTF_WD(x)                                             \
+	(((x) >> BIT_SHIFT_USB_SIE_INTF_WD) & BIT_MASK_USB_SIE_INTF_WD)
+
+/* 2 REG_PCIE_MIO_INTF			(Offset 0x00E4) */
+
+#define BIT_PCIE_MIO_BYIOREG BIT(13)
+#define BIT_PCIE_MIO_RE BIT(12)
+
+#define BIT_SHIFT_PCIE_MIO_WE 8
+#define BIT_MASK_PCIE_MIO_WE 0xf
+#define BIT_PCIE_MIO_WE(x)                                                     \
+	(((x) & BIT_MASK_PCIE_MIO_WE) << BIT_SHIFT_PCIE_MIO_WE)
+#define BIT_GET_PCIE_MIO_WE(x)                                                 \
+	(((x) >> BIT_SHIFT_PCIE_MIO_WE) & BIT_MASK_PCIE_MIO_WE)
+
+#define BIT_SHIFT_PCIE_MIO_ADDR 0
+#define BIT_MASK_PCIE_MIO_ADDR 0xff
+#define BIT_PCIE_MIO_ADDR(x)                                                   \
+	(((x) & BIT_MASK_PCIE_MIO_ADDR) << BIT_SHIFT_PCIE_MIO_ADDR)
+#define BIT_GET_PCIE_MIO_ADDR(x)                                               \
+	(((x) >> BIT_SHIFT_PCIE_MIO_ADDR) & BIT_MASK_PCIE_MIO_ADDR)
+
+/* 2 REG_PCIE_MIO_INTD			(Offset 0x00E8) */
+
+#define BIT_SHIFT_PCIE_MIO_DATA 0
+#define BIT_MASK_PCIE_MIO_DATA 0xffffffffL
+#define BIT_PCIE_MIO_DATA(x)                                                   \
+	(((x) & BIT_MASK_PCIE_MIO_DATA) << BIT_SHIFT_PCIE_MIO_DATA)
+#define BIT_GET_PCIE_MIO_DATA(x)                                               \
+	(((x) >> BIT_SHIFT_PCIE_MIO_DATA) & BIT_MASK_PCIE_MIO_DATA)
+
+/* 2 REG_WLRF1				(Offset 0x00EC) */
+
+#define BIT_SHIFT_WLRF1_CTRL 24
+#define BIT_MASK_WLRF1_CTRL 0xff
+#define BIT_WLRF1_CTRL(x) (((x) & BIT_MASK_WLRF1_CTRL) << BIT_SHIFT_WLRF1_CTRL)
+#define BIT_GET_WLRF1_CTRL(x)                                                  \
+	(((x) >> BIT_SHIFT_WLRF1_CTRL) & BIT_MASK_WLRF1_CTRL)
+
+/* 2 REG_SYS_CFG1				(Offset 0x00F0) */
+
+#define BIT_SHIFT_TRP_ICFG 28
+#define BIT_MASK_TRP_ICFG 0xf
+#define BIT_TRP_ICFG(x) (((x) & BIT_MASK_TRP_ICFG) << BIT_SHIFT_TRP_ICFG)
+#define BIT_GET_TRP_ICFG(x) (((x) >> BIT_SHIFT_TRP_ICFG) & BIT_MASK_TRP_ICFG)
+
+/* 2 REG_SYS_CFG1				(Offset 0x00F0) */
+
+#define BIT_RF_TYPE_ID BIT(27)
+#define BIT_BD_HCI_SEL BIT(26)
+
+/* 2 REG_SYS_CFG1				(Offset 0x00F0) */
+
+#define BIT_BD_PKG_SEL BIT(25)
+
+/* 2 REG_SYS_CFG1				(Offset 0x00F0) */
+
+#define BIT_SPSLDO_SEL BIT(24)
+
+/* 2 REG_SYS_CFG1				(Offset 0x00F0) */
+
+#define BIT_RTL_ID BIT(23)
+#define BIT_PAD_HWPD_IDN BIT(22)
+
+/* 2 REG_SYS_CFG1				(Offset 0x00F0) */
+
+#define BIT_TESTMODE BIT(20)
+
+/* 2 REG_SYS_CFG1				(Offset 0x00F0) */
+
+#define BIT_SHIFT_VENDOR_ID 16
+#define BIT_MASK_VENDOR_ID 0xf
+#define BIT_VENDOR_ID(x) (((x) & BIT_MASK_VENDOR_ID) << BIT_SHIFT_VENDOR_ID)
+#define BIT_GET_VENDOR_ID(x) (((x) >> BIT_SHIFT_VENDOR_ID) & BIT_MASK_VENDOR_ID)
+
+/* 2 REG_SYS_CFG1				(Offset 0x00F0) */
+
+#define BIT_SHIFT_CHIP_VER 12
+#define BIT_MASK_CHIP_VER 0xf
+#define BIT_CHIP_VER(x) (((x) & BIT_MASK_CHIP_VER) << BIT_SHIFT_CHIP_VER)
+#define BIT_GET_CHIP_VER(x) (((x) >> BIT_SHIFT_CHIP_VER) & BIT_MASK_CHIP_VER)
+
+/* 2 REG_SYS_CFG1				(Offset 0x00F0) */
+
+#define BIT_BD_MAC3 BIT(11)
+
+/* 2 REG_SYS_CFG1				(Offset 0x00F0) */
+
+#define BIT_BD_MAC1 BIT(10)
+#define BIT_BD_MAC2 BIT(9)
+#define BIT_SIC_IDLE BIT(8)
+#define BIT_SW_OFFLOAD_EN BIT(7)
+
+/* 2 REG_SYS_CFG1				(Offset 0x00F0) */
+
+#define BIT_OCP_SHUTDN BIT(6)
+
+/* 2 REG_SYS_CFG1				(Offset 0x00F0) */
+
+#define BIT_V15_VLD BIT(5)
+
+/* 2 REG_SYS_CFG1				(Offset 0x00F0) */
+
+#define BIT_PCIRSTB BIT(4)
+
+/* 2 REG_SYS_CFG1				(Offset 0x00F0) */
+
+#define BIT_PCLK_VLD BIT(3)
+
+/* 2 REG_SYS_CFG1				(Offset 0x00F0) */
+
+#define BIT_UCLK_VLD BIT(2)
+
+/* 2 REG_SYS_CFG1				(Offset 0x00F0) */
+
+#define BIT_ACLK_VLD BIT(1)
+
+/* 2 REG_SYS_CFG1				(Offset 0x00F0) */
+
+#define BIT_XCLK_VLD BIT(0)
+
+/* 2 REG_SYS_STATUS1				(Offset 0x00F4) */
+
+#define BIT_SHIFT_RF_RL_ID 28
+#define BIT_MASK_RF_RL_ID 0xf
+#define BIT_RF_RL_ID(x) (((x) & BIT_MASK_RF_RL_ID) << BIT_SHIFT_RF_RL_ID)
+#define BIT_GET_RF_RL_ID(x) (((x) >> BIT_SHIFT_RF_RL_ID) & BIT_MASK_RF_RL_ID)
+
+/* 2 REG_SYS_STATUS1				(Offset 0x00F4) */
+
+#define BIT_HPHY_ICFG BIT(19)
+
+/* 2 REG_SYS_STATUS1				(Offset 0x00F4) */
+
+#define BIT_SHIFT_SEL_0XC0 16
+#define BIT_MASK_SEL_0XC0 0x3
+#define BIT_SEL_0XC0(x) (((x) & BIT_MASK_SEL_0XC0) << BIT_SHIFT_SEL_0XC0)
+#define BIT_GET_SEL_0XC0(x) (((x) >> BIT_SHIFT_SEL_0XC0) & BIT_MASK_SEL_0XC0)
+
+/* 2 REG_SYS_STATUS1				(Offset 0x00F4) */
+
+#define BIT_SHIFT_HCI_SEL_V3 12
+#define BIT_MASK_HCI_SEL_V3 0x7
+#define BIT_HCI_SEL_V3(x) (((x) & BIT_MASK_HCI_SEL_V3) << BIT_SHIFT_HCI_SEL_V3)
+#define BIT_GET_HCI_SEL_V3(x)                                                  \
+	(((x) >> BIT_SHIFT_HCI_SEL_V3) & BIT_MASK_HCI_SEL_V3)
+
+/* 2 REG_SYS_STATUS1				(Offset 0x00F4) */
+
+#define BIT_USB_OPERATION_MODE BIT(10)
+#define BIT_BT_PDN BIT(9)
+#define BIT_AUTO_WLPON BIT(8)
+
+/* 2 REG_SYS_STATUS1				(Offset 0x00F4) */
+
+#define BIT_WL_MODE BIT(7)
+
+/* 2 REG_SYS_STATUS1				(Offset 0x00F4) */
+
+#define BIT_PKG_SEL_HCI BIT(6)
+
+/* 2 REG_SYS_STATUS1				(Offset 0x00F4) */
+
+#define BIT_SHIFT_PAD_HCI_SEL_V1 3
+#define BIT_MASK_PAD_HCI_SEL_V1 0x7
+#define BIT_PAD_HCI_SEL_V1(x)                                                  \
+	(((x) & BIT_MASK_PAD_HCI_SEL_V1) << BIT_SHIFT_PAD_HCI_SEL_V1)
+#define BIT_GET_PAD_HCI_SEL_V1(x)                                              \
+	(((x) >> BIT_SHIFT_PAD_HCI_SEL_V1) & BIT_MASK_PAD_HCI_SEL_V1)
+
+/* 2 REG_SYS_STATUS1				(Offset 0x00F4) */
+
+#define BIT_SHIFT_EFS_HCI_SEL_V1 0
+#define BIT_MASK_EFS_HCI_SEL_V1 0x7
+#define BIT_EFS_HCI_SEL_V1(x)                                                  \
+	(((x) & BIT_MASK_EFS_HCI_SEL_V1) << BIT_SHIFT_EFS_HCI_SEL_V1)
+#define BIT_GET_EFS_HCI_SEL_V1(x)                                              \
+	(((x) >> BIT_SHIFT_EFS_HCI_SEL_V1) & BIT_MASK_EFS_HCI_SEL_V1)
+
+/* 2 REG_SYS_STATUS2				(Offset 0x00F8) */
+
+#define BIT_SIO_ALDN BIT(19)
+#define BIT_USB_ALDN BIT(18)
+#define BIT_PCI_ALDN BIT(17)
+#define BIT_SYS_ALDN BIT(16)
+
+#define BIT_SHIFT_EPVID1 8
+#define BIT_MASK_EPVID1 0xff
+#define BIT_EPVID1(x) (((x) & BIT_MASK_EPVID1) << BIT_SHIFT_EPVID1)
+#define BIT_GET_EPVID1(x) (((x) >> BIT_SHIFT_EPVID1) & BIT_MASK_EPVID1)
+
+#define BIT_SHIFT_EPVID0 0
+#define BIT_MASK_EPVID0 0xff
+#define BIT_EPVID0(x) (((x) & BIT_MASK_EPVID0) << BIT_SHIFT_EPVID0)
+#define BIT_GET_EPVID0(x) (((x) >> BIT_SHIFT_EPVID0) & BIT_MASK_EPVID0)
+
+/* 2 REG_SYS_CFG2				(Offset 0x00FC) */
+
+#define BIT_HCI_SEL_EMBEDDED BIT(8)
+
+/* 2 REG_SYS_CFG2				(Offset 0x00FC) */
+
+#define BIT_SHIFT_HW_ID 0
+#define BIT_MASK_HW_ID 0xff
+#define BIT_HW_ID(x) (((x) & BIT_MASK_HW_ID) << BIT_SHIFT_HW_ID)
+#define BIT_GET_HW_ID(x) (((x) >> BIT_SHIFT_HW_ID) & BIT_MASK_HW_ID)
+
+/* 2 REG_CR					(Offset 0x0100) */
+
+#define BIT_SHIFT_LBMODE 24
+#define BIT_MASK_LBMODE 0x1f
+#define BIT_LBMODE(x) (((x) & BIT_MASK_LBMODE) << BIT_SHIFT_LBMODE)
+#define BIT_GET_LBMODE(x) (((x) >> BIT_SHIFT_LBMODE) & BIT_MASK_LBMODE)
+
+#define BIT_SHIFT_NETYPE1 18
+#define BIT_MASK_NETYPE1 0x3
+#define BIT_NETYPE1(x) (((x) & BIT_MASK_NETYPE1) << BIT_SHIFT_NETYPE1)
+#define BIT_GET_NETYPE1(x) (((x) >> BIT_SHIFT_NETYPE1) & BIT_MASK_NETYPE1)
+
+#define BIT_SHIFT_NETYPE0 16
+#define BIT_MASK_NETYPE0 0x3
+#define BIT_NETYPE0(x) (((x) & BIT_MASK_NETYPE0) << BIT_SHIFT_NETYPE0)
+#define BIT_GET_NETYPE0(x) (((x) >> BIT_SHIFT_NETYPE0) & BIT_MASK_NETYPE0)
+
+/* 2 REG_CR					(Offset 0x0100) */
+
+#define BIT_I2C_MAILBOX_EN BIT(12)
+#define BIT_SHCUT_EN BIT(11)
+
+/* 2 REG_CR					(Offset 0x0100) */
+
+#define BIT_32K_CAL_TMR_EN BIT(10)
+#define BIT_MAC_SEC_EN BIT(9)
+#define BIT_ENSWBCN BIT(8)
+#define BIT_MACRXEN BIT(7)
+#define BIT_MACTXEN BIT(6)
+#define BIT_SCHEDULE_EN BIT(5)
+#define BIT_PROTOCOL_EN BIT(4)
+#define BIT_RXDMA_EN BIT(3)
+#define BIT_TXDMA_EN BIT(2)
+#define BIT_HCI_RXDMA_EN BIT(1)
+#define BIT_HCI_TXDMA_EN BIT(0)
+
+/* 2 REG_PKT_BUFF_ACCESS_CTRL		(Offset 0x0106) */
+
+#define BIT_SHIFT_PKT_BUFF_ACCESS_CTRL 0
+#define BIT_MASK_PKT_BUFF_ACCESS_CTRL 0xff
+#define BIT_PKT_BUFF_ACCESS_CTRL(x)                                            \
+	(((x) & BIT_MASK_PKT_BUFF_ACCESS_CTRL)                                 \
+	 << BIT_SHIFT_PKT_BUFF_ACCESS_CTRL)
+#define BIT_GET_PKT_BUFF_ACCESS_CTRL(x)                                        \
+	(((x) >> BIT_SHIFT_PKT_BUFF_ACCESS_CTRL) &                             \
+	 BIT_MASK_PKT_BUFF_ACCESS_CTRL)
+
+/* 2 REG_TSF_CLK_STATE			(Offset 0x0108) */
+
+#define BIT_TSF_CLK_STABLE BIT(15)
+
+#define BIT_SHIFT_I2C_M_BUS_GNT_FW 4
+#define BIT_MASK_I2C_M_BUS_GNT_FW 0x7
+#define BIT_I2C_M_BUS_GNT_FW(x)                                                \
+	(((x) & BIT_MASK_I2C_M_BUS_GNT_FW) << BIT_SHIFT_I2C_M_BUS_GNT_FW)
+#define BIT_GET_I2C_M_BUS_GNT_FW(x)                                            \
+	(((x) >> BIT_SHIFT_I2C_M_BUS_GNT_FW) & BIT_MASK_I2C_M_BUS_GNT_FW)
+
+#define BIT_I2C_M_GNT_FW BIT(3)
+
+#define BIT_SHIFT_I2C_M_SPEED 1
+#define BIT_MASK_I2C_M_SPEED 0x3
+#define BIT_I2C_M_SPEED(x)                                                     \
+	(((x) & BIT_MASK_I2C_M_SPEED) << BIT_SHIFT_I2C_M_SPEED)
+#define BIT_GET_I2C_M_SPEED(x)                                                 \
+	(((x) >> BIT_SHIFT_I2C_M_SPEED) & BIT_MASK_I2C_M_SPEED)
+
+#define BIT_I2C_M_UNLOCK BIT(0)
+
+/* 2 REG_TXDMA_PQ_MAP			(Offset 0x010C) */
+
+#define BIT_SHIFT_TXDMA_HIQ_MAP 14
+#define BIT_MASK_TXDMA_HIQ_MAP 0x3
+#define BIT_TXDMA_HIQ_MAP(x)                                                   \
+	(((x) & BIT_MASK_TXDMA_HIQ_MAP) << BIT_SHIFT_TXDMA_HIQ_MAP)
+#define BIT_GET_TXDMA_HIQ_MAP(x)                                               \
+	(((x) >> BIT_SHIFT_TXDMA_HIQ_MAP) & BIT_MASK_TXDMA_HIQ_MAP)
+
+#define BIT_SHIFT_TXDMA_MGQ_MAP 12
+#define BIT_MASK_TXDMA_MGQ_MAP 0x3
+#define BIT_TXDMA_MGQ_MAP(x)                                                   \
+	(((x) & BIT_MASK_TXDMA_MGQ_MAP) << BIT_SHIFT_TXDMA_MGQ_MAP)
+#define BIT_GET_TXDMA_MGQ_MAP(x)                                               \
+	(((x) >> BIT_SHIFT_TXDMA_MGQ_MAP) & BIT_MASK_TXDMA_MGQ_MAP)
+
+#define BIT_SHIFT_TXDMA_BKQ_MAP 10
+#define BIT_MASK_TXDMA_BKQ_MAP 0x3
+#define BIT_TXDMA_BKQ_MAP(x)                                                   \
+	(((x) & BIT_MASK_TXDMA_BKQ_MAP) << BIT_SHIFT_TXDMA_BKQ_MAP)
+#define BIT_GET_TXDMA_BKQ_MAP(x)                                               \
+	(((x) >> BIT_SHIFT_TXDMA_BKQ_MAP) & BIT_MASK_TXDMA_BKQ_MAP)
+
+#define BIT_SHIFT_TXDMA_BEQ_MAP 8
+#define BIT_MASK_TXDMA_BEQ_MAP 0x3
+#define BIT_TXDMA_BEQ_MAP(x)                                                   \
+	(((x) & BIT_MASK_TXDMA_BEQ_MAP) << BIT_SHIFT_TXDMA_BEQ_MAP)
+#define BIT_GET_TXDMA_BEQ_MAP(x)                                               \
+	(((x) >> BIT_SHIFT_TXDMA_BEQ_MAP) & BIT_MASK_TXDMA_BEQ_MAP)
+
+#define BIT_SHIFT_TXDMA_VIQ_MAP 6
+#define BIT_MASK_TXDMA_VIQ_MAP 0x3
+#define BIT_TXDMA_VIQ_MAP(x)                                                   \
+	(((x) & BIT_MASK_TXDMA_VIQ_MAP) << BIT_SHIFT_TXDMA_VIQ_MAP)
+#define BIT_GET_TXDMA_VIQ_MAP(x)                                               \
+	(((x) >> BIT_SHIFT_TXDMA_VIQ_MAP) & BIT_MASK_TXDMA_VIQ_MAP)
+
+#define BIT_SHIFT_TXDMA_VOQ_MAP 4
+#define BIT_MASK_TXDMA_VOQ_MAP 0x3
+#define BIT_TXDMA_VOQ_MAP(x)                                                   \
+	(((x) & BIT_MASK_TXDMA_VOQ_MAP) << BIT_SHIFT_TXDMA_VOQ_MAP)
+#define BIT_GET_TXDMA_VOQ_MAP(x)                                               \
+	(((x) >> BIT_SHIFT_TXDMA_VOQ_MAP) & BIT_MASK_TXDMA_VOQ_MAP)
+
+#define BIT_RXDMA_AGG_EN BIT(2)
+#define BIT_RXSHFT_EN BIT(1)
+#define BIT_RXDMA_ARBBW_EN BIT(0)
+
+/* 2 REG_TRXFF_BNDY				(Offset 0x0114) */
+
+#define BIT_SHIFT_RXFFOVFL_RSV_V2 8
+#define BIT_MASK_RXFFOVFL_RSV_V2 0xf
+#define BIT_RXFFOVFL_RSV_V2(x)                                                 \
+	(((x) & BIT_MASK_RXFFOVFL_RSV_V2) << BIT_SHIFT_RXFFOVFL_RSV_V2)
+#define BIT_GET_RXFFOVFL_RSV_V2(x)                                             \
+	(((x) >> BIT_SHIFT_RXFFOVFL_RSV_V2) & BIT_MASK_RXFFOVFL_RSV_V2)
+
+/* 2 REG_TRXFF_BNDY				(Offset 0x0114) */
+
+#define BIT_SHIFT_TXPKTBUF_PGBNDY 0
+#define BIT_MASK_TXPKTBUF_PGBNDY 0xff
+#define BIT_TXPKTBUF_PGBNDY(x)                                                 \
+	(((x) & BIT_MASK_TXPKTBUF_PGBNDY) << BIT_SHIFT_TXPKTBUF_PGBNDY)
+#define BIT_GET_TXPKTBUF_PGBNDY(x)                                             \
+	(((x) >> BIT_SHIFT_TXPKTBUF_PGBNDY) & BIT_MASK_TXPKTBUF_PGBNDY)
+
+/* 2 REG_TRXFF_BNDY				(Offset 0x0114) */
+
+#define BIT_SHIFT_RXFF0_BNDY_V2 0
+#define BIT_MASK_RXFF0_BNDY_V2 0x3ffff
+#define BIT_RXFF0_BNDY_V2(x)                                                   \
+	(((x) & BIT_MASK_RXFF0_BNDY_V2) << BIT_SHIFT_RXFF0_BNDY_V2)
+#define BIT_GET_RXFF0_BNDY_V2(x)                                               \
+	(((x) >> BIT_SHIFT_RXFF0_BNDY_V2) & BIT_MASK_RXFF0_BNDY_V2)
+
+#define BIT_SHIFT_RXFF0_RDPTR_V2 0
+#define BIT_MASK_RXFF0_RDPTR_V2 0x3ffff
+#define BIT_RXFF0_RDPTR_V2(x)                                                  \
+	(((x) & BIT_MASK_RXFF0_RDPTR_V2) << BIT_SHIFT_RXFF0_RDPTR_V2)
+#define BIT_GET_RXFF0_RDPTR_V2(x)                                              \
+	(((x) >> BIT_SHIFT_RXFF0_RDPTR_V2) & BIT_MASK_RXFF0_RDPTR_V2)
+
+#define BIT_SHIFT_RXFF0_WTPTR_V2 0
+#define BIT_MASK_RXFF0_WTPTR_V2 0x3ffff
+#define BIT_RXFF0_WTPTR_V2(x)                                                  \
+	(((x) & BIT_MASK_RXFF0_WTPTR_V2) << BIT_SHIFT_RXFF0_WTPTR_V2)
+#define BIT_GET_RXFF0_WTPTR_V2(x)                                              \
+	(((x) >> BIT_SHIFT_RXFF0_WTPTR_V2) & BIT_MASK_RXFF0_WTPTR_V2)
+
+/* 2 REG_PTA_I2C_MBOX			(Offset 0x0118) */
+
+#define BIT_SHIFT_I2C_M_STATUS 8
+#define BIT_MASK_I2C_M_STATUS 0xf
+#define BIT_I2C_M_STATUS(x)                                                    \
+	(((x) & BIT_MASK_I2C_M_STATUS) << BIT_SHIFT_I2C_M_STATUS)
+#define BIT_GET_I2C_M_STATUS(x)                                                \
+	(((x) >> BIT_SHIFT_I2C_M_STATUS) & BIT_MASK_I2C_M_STATUS)
+
+/* 2 REG_FE1IMR				(Offset 0x0120) */
+
+#define BIT_FS_RXDMA2_DONE_INT_EN BIT(28)
+#define BIT_FS_RXDONE3_INT_EN BIT(27)
+#define BIT_FS_RXDONE2_INT_EN BIT(26)
+#define BIT_FS_RX_BCN_P4_INT_EN BIT(25)
+#define BIT_FS_RX_BCN_P3_INT_EN BIT(24)
+#define BIT_FS_RX_BCN_P2_INT_EN BIT(23)
+#define BIT_FS_RX_BCN_P1_INT_EN BIT(22)
+#define BIT_FS_RX_BCN_P0_INT_EN BIT(21)
+#define BIT_FS_RX_UMD0_INT_EN BIT(20)
+#define BIT_FS_RX_UMD1_INT_EN BIT(19)
+#define BIT_FS_RX_BMD0_INT_EN BIT(18)
+#define BIT_FS_RX_BMD1_INT_EN BIT(17)
+#define BIT_FS_RXDONE_INT_EN BIT(16)
+#define BIT_FS_WWLAN_INT_EN BIT(15)
+#define BIT_FS_SOUND_DONE_INT_EN BIT(14)
+#define BIT_FS_LP_STBY_INT_EN BIT(13)
+#define BIT_FS_TRL_MTR_INT_EN BIT(12)
+#define BIT_FS_BF1_PRETO_INT_EN BIT(11)
+#define BIT_FS_BF0_PRETO_INT_EN BIT(10)
+#define BIT_FS_PTCL_RELEASE_MACID_INT_EN BIT(9)
+
+/* 2 REG_FE1IMR				(Offset 0x0120) */
+
+#define BIT_FS_LTE_COEX_EN BIT(6)
+
+/* 2 REG_FE1IMR				(Offset 0x0120) */
+
+#define BIT_FS_WLACTOFF_INT_EN BIT(5)
+#define BIT_FS_WLACTON_INT_EN BIT(4)
+#define BIT_FS_BTCMD_INT_EN BIT(3)
+
+/* 2 REG_FE1IMR				(Offset 0x0120) */
+
+#define BIT_FS_REG_MAILBOX_TO_I2C_INT_EN BIT(2)
+
+/* 2 REG_FE1IMR				(Offset 0x0120) */
+
+#define BIT_FS_TRPC_TO_INT_EN_V1 BIT(1)
+
+/* 2 REG_FE1IMR				(Offset 0x0120) */
+
+#define BIT_FS_RPC_O_T_INT_EN_V1 BIT(0)
+
+/* 2 REG_FE1ISR				(Offset 0x0124) */
+
+#define BIT_FS_RXDMA2_DONE_INT BIT(28)
+#define BIT_FS_RXDONE3_INT BIT(27)
+#define BIT_FS_RXDONE2_INT BIT(26)
+#define BIT_FS_RX_BCN_P4_INT BIT(25)
+#define BIT_FS_RX_BCN_P3_INT BIT(24)
+#define BIT_FS_RX_BCN_P2_INT BIT(23)
+#define BIT_FS_RX_BCN_P1_INT BIT(22)
+#define BIT_FS_RX_BCN_P0_INT BIT(21)
+#define BIT_FS_RX_UMD0_INT BIT(20)
+#define BIT_FS_RX_UMD1_INT BIT(19)
+#define BIT_FS_RX_BMD0_INT BIT(18)
+#define BIT_FS_RX_BMD1_INT BIT(17)
+#define BIT_FS_RXDONE_INT BIT(16)
+#define BIT_FS_WWLAN_INT BIT(15)
+#define BIT_FS_SOUND_DONE_INT BIT(14)
+#define BIT_FS_LP_STBY_INT BIT(13)
+#define BIT_FS_TRL_MTR_INT BIT(12)
+#define BIT_FS_BF1_PRETO_INT BIT(11)
+#define BIT_FS_BF0_PRETO_INT BIT(10)
+#define BIT_FS_PTCL_RELEASE_MACID_INT BIT(9)
+
+/* 2 REG_FE1ISR				(Offset 0x0124) */
+
+#define BIT_FS_LTE_COEX_INT BIT(6)
+
+/* 2 REG_FE1ISR				(Offset 0x0124) */
+
+#define BIT_FS_WLACTOFF_INT BIT(5)
+#define BIT_FS_WLACTON_INT BIT(4)
+#define BIT_FS_BCN_RX_INT_INT BIT(3)
+
+/* 2 REG_FE1ISR				(Offset 0x0124) */
+
+#define BIT_FS_MAILBOX_TO_I2C_INT BIT(2)
+
+/* 2 REG_FE1ISR				(Offset 0x0124) */
+
+#define BIT_FS_TRPC_TO_INT BIT(1)
+
+/* 2 REG_FE1ISR				(Offset 0x0124) */
+
+#define BIT_FS_RPC_O_T_INT BIT(0)
+
+/* 2 REG_CPWM				(Offset 0x012C) */
+
+#define BIT_CPWM_TOGGLING BIT(31)
+
+#define BIT_SHIFT_CPWM_MOD 24
+#define BIT_MASK_CPWM_MOD 0x7f
+#define BIT_CPWM_MOD(x) (((x) & BIT_MASK_CPWM_MOD) << BIT_SHIFT_CPWM_MOD)
+#define BIT_GET_CPWM_MOD(x) (((x) >> BIT_SHIFT_CPWM_MOD) & BIT_MASK_CPWM_MOD)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_TXBCNOK_MB7_INT_EN BIT(31)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_TXBCNOK_MB6_INT_EN BIT(30)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_TXBCNOK_MB5_INT_EN BIT(29)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_TXBCNOK_MB4_INT_EN BIT(28)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_TXBCNOK_MB3_INT_EN BIT(27)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_TXBCNOK_MB2_INT_EN BIT(26)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_TXBCNOK_MB1_INT_EN BIT(25)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_TXBCNOK_MB0_INT_EN BIT(24)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_TXBCNERR_MB7_INT_EN BIT(23)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_TXBCNERR_MB6_INT_EN BIT(22)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_TXBCNERR_MB5_INT_EN BIT(21)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_TXBCNERR_MB4_INT_EN BIT(20)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_TXBCNERR_MB3_INT_EN BIT(19)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_TXBCNERR_MB2_INT_EN BIT(18)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_TXBCNERR_MB1_INT_EN BIT(17)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_TXBCNERR_MB0_INT_EN BIT(16)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_CPU_MGQ_TXDONE_INT_EN BIT(15)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_SIFS_OVERSPEC_INT_EN BIT(14)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_MGNTQ_RPTR_RELEASE_INT_EN BIT(13)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_MGNTQFF_TO_INT_EN BIT(12)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_DDMA1_LP_INT_EN BIT(11)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_DDMA1_HP_INT_EN BIT(10)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_DDMA0_LP_INT_EN BIT(9)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_DDMA0_HP_INT_EN BIT(8)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_TRXRPT_INT_EN BIT(7)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_C2H_W_READY_INT_EN BIT(6)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_HRCV_INT_EN BIT(5)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_H2CCMD_INT_EN BIT(4)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_TXPKTIN_INT_EN BIT(3)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_ERRORHDL_INT_EN BIT(2)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_TXCCX_INT_EN BIT(1)
+
+/* 2 REG_FWIMR				(Offset 0x0130) */
+
+#define BIT_FS_TXCLOSE_INT_EN BIT(0)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_TXBCNOK_MB7_INT BIT(31)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_TXBCNOK_MB6_INT BIT(30)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_TXBCNOK_MB5_INT BIT(29)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_TXBCNOK_MB4_INT BIT(28)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_TXBCNOK_MB3_INT BIT(27)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_TXBCNOK_MB2_INT BIT(26)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_TXBCNOK_MB1_INT BIT(25)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_TXBCNOK_MB0_INT BIT(24)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_TXBCNERR_MB7_INT BIT(23)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_TXBCNERR_MB6_INT BIT(22)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_TXBCNERR_MB5_INT BIT(21)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_TXBCNERR_MB4_INT BIT(20)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_TXBCNERR_MB3_INT BIT(19)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_TXBCNERR_MB2_INT BIT(18)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_TXBCNERR_MB1_INT BIT(17)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_TXBCNERR_MB0_INT BIT(16)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_CPU_MGQ_TXDONE_INT BIT(15)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_SIFS_OVERSPEC_INT BIT(14)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_MGNTQ_RPTR_RELEASE_INT BIT(13)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_MGNTQFF_TO_INT BIT(12)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_DDMA1_LP_INT BIT(11)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_DDMA1_HP_INT BIT(10)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_DDMA0_LP_INT BIT(9)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_DDMA0_HP_INT BIT(8)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_TRXRPT_INT BIT(7)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_C2H_W_READY_INT BIT(6)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_HRCV_INT BIT(5)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_H2CCMD_INT BIT(4)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_TXPKTIN_INT BIT(3)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_ERRORHDL_INT BIT(2)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_TXCCX_INT BIT(1)
+
+/* 2 REG_FWISR				(Offset 0x0134) */
+
+#define BIT_FS_TXCLOSE_INT BIT(0)
+
+/* 2 REG_FTIMR				(Offset 0x0138) */
+
+#define BIT_PS_TIMER_C_EARLY_INT_EN BIT(23)
+
+/* 2 REG_FTIMR				(Offset 0x0138) */
+
+#define BIT_PS_TIMER_B_EARLY_INT_EN BIT(22)
+
+/* 2 REG_FTIMR				(Offset 0x0138) */
+
+#define BIT_PS_TIMER_A_EARLY_INT_EN BIT(21)
+
+/* 2 REG_FTIMR				(Offset 0x0138) */
+
+#define BIT_CPUMGQ_TX_TIMER_EARLY_INT_EN BIT(20)
+
+/* 2 REG_FTIMR				(Offset 0x0138) */
+
+#define BIT_PS_TIMER_C_INT_EN BIT(19)
+
+/* 2 REG_FTIMR				(Offset 0x0138) */
+
+#define BIT_PS_TIMER_B_INT_EN BIT(18)
+
+/* 2 REG_FTIMR				(Offset 0x0138) */
+
+#define BIT_PS_TIMER_A_INT_EN BIT(17)
+
+/* 2 REG_FTIMR				(Offset 0x0138) */
+
+#define BIT_CPUMGQ_TX_TIMER_INT_EN BIT(16)
+
+/* 2 REG_FTIMR				(Offset 0x0138) */
+
+#define BIT_FS_PS_TIMEOUT2_EN BIT(15)
+
+/* 2 REG_FTIMR				(Offset 0x0138) */
+
+#define BIT_FS_PS_TIMEOUT1_EN BIT(14)
+
+/* 2 REG_FTIMR				(Offset 0x0138) */
+
+#define BIT_FS_PS_TIMEOUT0_EN BIT(13)
+
+/* 2 REG_FTIMR				(Offset 0x0138) */
+
+#define BIT_FS_GTINT8_EN BIT(8)
+
+/* 2 REG_FTIMR				(Offset 0x0138) */
+
+#define BIT_FS_GTINT7_EN BIT(7)
+
+/* 2 REG_FTIMR				(Offset 0x0138) */
+
+#define BIT_FS_GTINT6_EN BIT(6)
+
+/* 2 REG_FTIMR				(Offset 0x0138) */
+
+#define BIT_FS_GTINT5_EN BIT(5)
+
+/* 2 REG_FTIMR				(Offset 0x0138) */
+
+#define BIT_FS_GTINT4_EN BIT(4)
+
+/* 2 REG_FTIMR				(Offset 0x0138) */
+
+#define BIT_FS_GTINT3_EN BIT(3)
+
+/* 2 REG_FTIMR				(Offset 0x0138) */
+
+#define BIT_FS_GTINT2_EN BIT(2)
+
+/* 2 REG_FTIMR				(Offset 0x0138) */
+
+#define BIT_FS_GTINT1_EN BIT(1)
+
+/* 2 REG_FTIMR				(Offset 0x0138) */
+
+#define BIT_FS_GTINT0_EN BIT(0)
+
+/* 2 REG_FTISR				(Offset 0x013C) */
+
+#define BIT_PS_TIMER_C_EARLY__INT BIT(23)
+
+/* 2 REG_FTISR				(Offset 0x013C) */
+
+#define BIT_PS_TIMER_B_EARLY__INT BIT(22)
+
+/* 2 REG_FTISR				(Offset 0x013C) */
+
+#define BIT_PS_TIMER_A_EARLY__INT BIT(21)
+
+/* 2 REG_FTISR				(Offset 0x013C) */
+
+#define BIT_CPUMGQ_TX_TIMER_EARLY_INT BIT(20)
+
+/* 2 REG_FTISR				(Offset 0x013C) */
+
+#define BIT_PS_TIMER_C_INT BIT(19)
+
+/* 2 REG_FTISR				(Offset 0x013C) */
+
+#define BIT_PS_TIMER_B_INT BIT(18)
+
+/* 2 REG_FTISR				(Offset 0x013C) */
+
+#define BIT_PS_TIMER_A_INT BIT(17)
+
+/* 2 REG_FTISR				(Offset 0x013C) */
+
+#define BIT_CPUMGQ_TX_TIMER_INT BIT(16)
+
+/* 2 REG_FTISR				(Offset 0x013C) */
+
+#define BIT_FS_PS_TIMEOUT2_INT BIT(15)
+
+/* 2 REG_FTISR				(Offset 0x013C) */
+
+#define BIT_FS_PS_TIMEOUT1_INT BIT(14)
+
+/* 2 REG_FTISR				(Offset 0x013C) */
+
+#define BIT_FS_PS_TIMEOUT0_INT BIT(13)
+
+/* 2 REG_FTISR				(Offset 0x013C) */
+
+#define BIT_FS_GTINT8_INT BIT(8)
+
+/* 2 REG_FTISR				(Offset 0x013C) */
+
+#define BIT_FS_GTINT7_INT BIT(7)
+
+/* 2 REG_FTISR				(Offset 0x013C) */
+
+#define BIT_FS_GTINT6_INT BIT(6)
+
+/* 2 REG_FTISR				(Offset 0x013C) */
+
+#define BIT_FS_GTINT5_INT BIT(5)
+
+/* 2 REG_FTISR				(Offset 0x013C) */
+
+#define BIT_FS_GTINT4_INT BIT(4)
+
+/* 2 REG_FTISR				(Offset 0x013C) */
+
+#define BIT_FS_GTINT3_INT BIT(3)
+
+/* 2 REG_FTISR				(Offset 0x013C) */
+
+#define BIT_FS_GTINT2_INT BIT(2)
+
+/* 2 REG_FTISR				(Offset 0x013C) */
+
+#define BIT_FS_GTINT1_INT BIT(1)
+
+/* 2 REG_FTISR				(Offset 0x013C) */
+
+#define BIT_FS_GTINT0_INT BIT(0)
+
+/* 2 REG_PKTBUF_DBG_CTRL			(Offset 0x0140) */
+
+#define BIT_SHIFT_PKTBUF_WRITE_EN 24
+#define BIT_MASK_PKTBUF_WRITE_EN 0xff
+#define BIT_PKTBUF_WRITE_EN(x)                                                 \
+	(((x) & BIT_MASK_PKTBUF_WRITE_EN) << BIT_SHIFT_PKTBUF_WRITE_EN)
+#define BIT_GET_PKTBUF_WRITE_EN(x)                                             \
+	(((x) >> BIT_SHIFT_PKTBUF_WRITE_EN) & BIT_MASK_PKTBUF_WRITE_EN)
+
+/* 2 REG_PKTBUF_DBG_CTRL			(Offset 0x0140) */
+
+#define BIT_TXRPTBUF_DBG BIT(23)
+
+/* 2 REG_PKTBUF_DBG_CTRL			(Offset 0x0140) */
+
+#define BIT_TXPKTBUF_DBG_V2 BIT(20)
+
+/* 2 REG_PKTBUF_DBG_CTRL			(Offset 0x0140) */
+
+#define BIT_RXPKTBUF_DBG BIT(16)
+
+/* 2 REG_PKTBUF_DBG_CTRL			(Offset 0x0140) */
+
+#define BIT_SHIFT_PKTBUF_DBG_ADDR 0
+#define BIT_MASK_PKTBUF_DBG_ADDR 0x1fff
+#define BIT_PKTBUF_DBG_ADDR(x)                                                 \
+	(((x) & BIT_MASK_PKTBUF_DBG_ADDR) << BIT_SHIFT_PKTBUF_DBG_ADDR)
+#define BIT_GET_PKTBUF_DBG_ADDR(x)                                             \
+	(((x) >> BIT_SHIFT_PKTBUF_DBG_ADDR) & BIT_MASK_PKTBUF_DBG_ADDR)
+
+/* 2 REG_PKTBUF_DBG_DATA_L			(Offset 0x0144) */
+
+#define BIT_SHIFT_PKTBUF_DBG_DATA_L 0
+#define BIT_MASK_PKTBUF_DBG_DATA_L 0xffffffffL
+#define BIT_PKTBUF_DBG_DATA_L(x)                                               \
+	(((x) & BIT_MASK_PKTBUF_DBG_DATA_L) << BIT_SHIFT_PKTBUF_DBG_DATA_L)
+#define BIT_GET_PKTBUF_DBG_DATA_L(x)                                           \
+	(((x) >> BIT_SHIFT_PKTBUF_DBG_DATA_L) & BIT_MASK_PKTBUF_DBG_DATA_L)
+
+/* 2 REG_PKTBUF_DBG_DATA_H			(Offset 0x0148) */
+
+#define BIT_SHIFT_PKTBUF_DBG_DATA_H 0
+#define BIT_MASK_PKTBUF_DBG_DATA_H 0xffffffffL
+#define BIT_PKTBUF_DBG_DATA_H(x)                                               \
+	(((x) & BIT_MASK_PKTBUF_DBG_DATA_H) << BIT_SHIFT_PKTBUF_DBG_DATA_H)
+#define BIT_GET_PKTBUF_DBG_DATA_H(x)                                           \
+	(((x) >> BIT_SHIFT_PKTBUF_DBG_DATA_H) & BIT_MASK_PKTBUF_DBG_DATA_H)
+
+/* 2 REG_CPWM2				(Offset 0x014C) */
+
+#define BIT_SHIFT_L0S_TO_RCVY_NUM 16
+#define BIT_MASK_L0S_TO_RCVY_NUM 0xff
+#define BIT_L0S_TO_RCVY_NUM(x)                                                 \
+	(((x) & BIT_MASK_L0S_TO_RCVY_NUM) << BIT_SHIFT_L0S_TO_RCVY_NUM)
+#define BIT_GET_L0S_TO_RCVY_NUM(x)                                             \
+	(((x) >> BIT_SHIFT_L0S_TO_RCVY_NUM) & BIT_MASK_L0S_TO_RCVY_NUM)
+
+#define BIT_CPWM2_TOGGLING BIT(15)
+
+#define BIT_SHIFT_CPWM2_MOD 0
+#define BIT_MASK_CPWM2_MOD 0x7fff
+#define BIT_CPWM2_MOD(x) (((x) & BIT_MASK_CPWM2_MOD) << BIT_SHIFT_CPWM2_MOD)
+#define BIT_GET_CPWM2_MOD(x) (((x) >> BIT_SHIFT_CPWM2_MOD) & BIT_MASK_CPWM2_MOD)
+
+/* 2 REG_TC0_CTRL				(Offset 0x0150) */
+
+#define BIT_TC0INT_EN BIT(26)
+#define BIT_TC0MODE BIT(25)
+#define BIT_TC0EN BIT(24)
+
+#define BIT_SHIFT_TC0DATA 0
+#define BIT_MASK_TC0DATA 0xffffff
+#define BIT_TC0DATA(x) (((x) & BIT_MASK_TC0DATA) << BIT_SHIFT_TC0DATA)
+#define BIT_GET_TC0DATA(x) (((x) >> BIT_SHIFT_TC0DATA) & BIT_MASK_TC0DATA)
+
+/* 2 REG_TC1_CTRL				(Offset 0x0154) */
+
+#define BIT_TC1INT_EN BIT(26)
+#define BIT_TC1MODE BIT(25)
+#define BIT_TC1EN BIT(24)
+
+#define BIT_SHIFT_TC1DATA 0
+#define BIT_MASK_TC1DATA 0xffffff
+#define BIT_TC1DATA(x) (((x) & BIT_MASK_TC1DATA) << BIT_SHIFT_TC1DATA)
+#define BIT_GET_TC1DATA(x) (((x) >> BIT_SHIFT_TC1DATA) & BIT_MASK_TC1DATA)
+
+/* 2 REG_TC2_CTRL				(Offset 0x0158) */
+
+#define BIT_TC2INT_EN BIT(26)
+#define BIT_TC2MODE BIT(25)
+#define BIT_TC2EN BIT(24)
+
+#define BIT_SHIFT_TC2DATA 0
+#define BIT_MASK_TC2DATA 0xffffff
+#define BIT_TC2DATA(x) (((x) & BIT_MASK_TC2DATA) << BIT_SHIFT_TC2DATA)
+#define BIT_GET_TC2DATA(x) (((x) >> BIT_SHIFT_TC2DATA) & BIT_MASK_TC2DATA)
+
+/* 2 REG_TC3_CTRL				(Offset 0x015C) */
+
+#define BIT_TC3INT_EN BIT(26)
+#define BIT_TC3MODE BIT(25)
+#define BIT_TC3EN BIT(24)
+
+#define BIT_SHIFT_TC3DATA 0
+#define BIT_MASK_TC3DATA 0xffffff
+#define BIT_TC3DATA(x) (((x) & BIT_MASK_TC3DATA) << BIT_SHIFT_TC3DATA)
+#define BIT_GET_TC3DATA(x) (((x) >> BIT_SHIFT_TC3DATA) & BIT_MASK_TC3DATA)
+
+/* 2 REG_TC4_CTRL				(Offset 0x0160) */
+
+#define BIT_TC4INT_EN BIT(26)
+#define BIT_TC4MODE BIT(25)
+#define BIT_TC4EN BIT(24)
+
+#define BIT_SHIFT_TC4DATA 0
+#define BIT_MASK_TC4DATA 0xffffff
+#define BIT_TC4DATA(x) (((x) & BIT_MASK_TC4DATA) << BIT_SHIFT_TC4DATA)
+#define BIT_GET_TC4DATA(x) (((x) >> BIT_SHIFT_TC4DATA) & BIT_MASK_TC4DATA)
+
+/* 2 REG_TCUNIT_BASE				(Offset 0x0164) */
+
+#define BIT_SHIFT_TCUNIT_BASE 0
+#define BIT_MASK_TCUNIT_BASE 0x3fff
+#define BIT_TCUNIT_BASE(x)                                                     \
+	(((x) & BIT_MASK_TCUNIT_BASE) << BIT_SHIFT_TCUNIT_BASE)
+#define BIT_GET_TCUNIT_BASE(x)                                                 \
+	(((x) >> BIT_SHIFT_TCUNIT_BASE) & BIT_MASK_TCUNIT_BASE)
+
+/* 2 REG_TC5_CTRL				(Offset 0x0168) */
+
+#define BIT_TC5INT_EN BIT(26)
+
+/* 2 REG_TC5_CTRL				(Offset 0x0168) */
+
+#define BIT_TC5MODE BIT(25)
+#define BIT_TC5EN BIT(24)
+
+#define BIT_SHIFT_TC5DATA 0
+#define BIT_MASK_TC5DATA 0xffffff
+#define BIT_TC5DATA(x) (((x) & BIT_MASK_TC5DATA) << BIT_SHIFT_TC5DATA)
+#define BIT_GET_TC5DATA(x) (((x) >> BIT_SHIFT_TC5DATA) & BIT_MASK_TC5DATA)
+
+/* 2 REG_TC6_CTRL				(Offset 0x016C) */
+
+#define BIT_TC6INT_EN BIT(26)
+
+/* 2 REG_TC6_CTRL				(Offset 0x016C) */
+
+#define BIT_TC6MODE BIT(25)
+#define BIT_TC6EN BIT(24)
+
+#define BIT_SHIFT_TC6DATA 0
+#define BIT_MASK_TC6DATA 0xffffff
+#define BIT_TC6DATA(x) (((x) & BIT_MASK_TC6DATA) << BIT_SHIFT_TC6DATA)
+#define BIT_GET_TC6DATA(x) (((x) >> BIT_SHIFT_TC6DATA) & BIT_MASK_TC6DATA)
+
+/* 2 REG_MBIST_FAIL				(Offset 0x0170) */
+
+#define BIT_SHIFT_8051_MBIST_FAIL 26
+#define BIT_MASK_8051_MBIST_FAIL 0x7
+#define BIT_8051_MBIST_FAIL(x)                                                 \
+	(((x) & BIT_MASK_8051_MBIST_FAIL) << BIT_SHIFT_8051_MBIST_FAIL)
+#define BIT_GET_8051_MBIST_FAIL(x)                                             \
+	(((x) >> BIT_SHIFT_8051_MBIST_FAIL) & BIT_MASK_8051_MBIST_FAIL)
+
+#define BIT_SHIFT_USB_MBIST_FAIL 24
+#define BIT_MASK_USB_MBIST_FAIL 0x3
+#define BIT_USB_MBIST_FAIL(x)                                                  \
+	(((x) & BIT_MASK_USB_MBIST_FAIL) << BIT_SHIFT_USB_MBIST_FAIL)
+#define BIT_GET_USB_MBIST_FAIL(x)                                              \
+	(((x) >> BIT_SHIFT_USB_MBIST_FAIL) & BIT_MASK_USB_MBIST_FAIL)
+
+#define BIT_SHIFT_PCIE_MBIST_FAIL 16
+#define BIT_MASK_PCIE_MBIST_FAIL 0x3f
+#define BIT_PCIE_MBIST_FAIL(x)                                                 \
+	(((x) & BIT_MASK_PCIE_MBIST_FAIL) << BIT_SHIFT_PCIE_MBIST_FAIL)
+#define BIT_GET_PCIE_MBIST_FAIL(x)                                             \
+	(((x) >> BIT_SHIFT_PCIE_MBIST_FAIL) & BIT_MASK_PCIE_MBIST_FAIL)
+
+/* 2 REG_MBIST_FAIL				(Offset 0x0170) */
+
+#define BIT_SHIFT_MAC_MBIST_FAIL 0
+#define BIT_MASK_MAC_MBIST_FAIL 0xfff
+#define BIT_MAC_MBIST_FAIL(x)                                                  \
+	(((x) & BIT_MASK_MAC_MBIST_FAIL) << BIT_SHIFT_MAC_MBIST_FAIL)
+#define BIT_GET_MAC_MBIST_FAIL(x)                                              \
+	(((x) >> BIT_SHIFT_MAC_MBIST_FAIL) & BIT_MASK_MAC_MBIST_FAIL)
+
+/* 2 REG_MBIST_START_PAUSE			(Offset 0x0174) */
+
+#define BIT_SHIFT_8051_MBIST_START_PAUSE 26
+#define BIT_MASK_8051_MBIST_START_PAUSE 0x7
+#define BIT_8051_MBIST_START_PAUSE(x)                                          \
+	(((x) & BIT_MASK_8051_MBIST_START_PAUSE)                               \
+	 << BIT_SHIFT_8051_MBIST_START_PAUSE)
+#define BIT_GET_8051_MBIST_START_PAUSE(x)                                      \
+	(((x) >> BIT_SHIFT_8051_MBIST_START_PAUSE) &                           \
+	 BIT_MASK_8051_MBIST_START_PAUSE)
+
+#define BIT_SHIFT_USB_MBIST_START_PAUSE 24
+#define BIT_MASK_USB_MBIST_START_PAUSE 0x3
+#define BIT_USB_MBIST_START_PAUSE(x)                                           \
+	(((x) & BIT_MASK_USB_MBIST_START_PAUSE)                                \
+	 << BIT_SHIFT_USB_MBIST_START_PAUSE)
+#define BIT_GET_USB_MBIST_START_PAUSE(x)                                       \
+	(((x) >> BIT_SHIFT_USB_MBIST_START_PAUSE) &                            \
+	 BIT_MASK_USB_MBIST_START_PAUSE)
+
+#define BIT_SHIFT_PCIE_MBIST_START_PAUSE 16
+#define BIT_MASK_PCIE_MBIST_START_PAUSE 0x3f
+#define BIT_PCIE_MBIST_START_PAUSE(x)                                          \
+	(((x) & BIT_MASK_PCIE_MBIST_START_PAUSE)                               \
+	 << BIT_SHIFT_PCIE_MBIST_START_PAUSE)
+#define BIT_GET_PCIE_MBIST_START_PAUSE(x)                                      \
+	(((x) >> BIT_SHIFT_PCIE_MBIST_START_PAUSE) &                           \
+	 BIT_MASK_PCIE_MBIST_START_PAUSE)
+
+/* 2 REG_MBIST_START_PAUSE			(Offset 0x0174) */
+
+#define BIT_SHIFT_MAC_MBIST_START_PAUSE 0
+#define BIT_MASK_MAC_MBIST_START_PAUSE 0xfff
+#define BIT_MAC_MBIST_START_PAUSE(x)                                           \
+	(((x) & BIT_MASK_MAC_MBIST_START_PAUSE)                                \
+	 << BIT_SHIFT_MAC_MBIST_START_PAUSE)
+#define BIT_GET_MAC_MBIST_START_PAUSE(x)                                       \
+	(((x) >> BIT_SHIFT_MAC_MBIST_START_PAUSE) &                            \
+	 BIT_MASK_MAC_MBIST_START_PAUSE)
+
+/* 2 REG_MBIST_DONE				(Offset 0x0178) */
+
+#define BIT_SHIFT_8051_MBIST_DONE 26
+#define BIT_MASK_8051_MBIST_DONE 0x7
+#define BIT_8051_MBIST_DONE(x)                                                 \
+	(((x) & BIT_MASK_8051_MBIST_DONE) << BIT_SHIFT_8051_MBIST_DONE)
+#define BIT_GET_8051_MBIST_DONE(x)                                             \
+	(((x) >> BIT_SHIFT_8051_MBIST_DONE) & BIT_MASK_8051_MBIST_DONE)
+
+#define BIT_SHIFT_USB_MBIST_DONE 24
+#define BIT_MASK_USB_MBIST_DONE 0x3
+#define BIT_USB_MBIST_DONE(x)                                                  \
+	(((x) & BIT_MASK_USB_MBIST_DONE) << BIT_SHIFT_USB_MBIST_DONE)
+#define BIT_GET_USB_MBIST_DONE(x)                                              \
+	(((x) >> BIT_SHIFT_USB_MBIST_DONE) & BIT_MASK_USB_MBIST_DONE)
+
+#define BIT_SHIFT_PCIE_MBIST_DONE 16
+#define BIT_MASK_PCIE_MBIST_DONE 0x3f
+#define BIT_PCIE_MBIST_DONE(x)                                                 \
+	(((x) & BIT_MASK_PCIE_MBIST_DONE) << BIT_SHIFT_PCIE_MBIST_DONE)
+#define BIT_GET_PCIE_MBIST_DONE(x)                                             \
+	(((x) >> BIT_SHIFT_PCIE_MBIST_DONE) & BIT_MASK_PCIE_MBIST_DONE)
+
+/* 2 REG_MBIST_DONE				(Offset 0x0178) */
+
+#define BIT_SHIFT_MAC_MBIST_DONE 0
+#define BIT_MASK_MAC_MBIST_DONE 0xfff
+#define BIT_MAC_MBIST_DONE(x)                                                  \
+	(((x) & BIT_MASK_MAC_MBIST_DONE) << BIT_SHIFT_MAC_MBIST_DONE)
+#define BIT_GET_MAC_MBIST_DONE(x)                                              \
+	(((x) >> BIT_SHIFT_MAC_MBIST_DONE) & BIT_MASK_MAC_MBIST_DONE)
+
+/* 2 REG_MBIST_FAIL_NRML			(Offset 0x017C) */
+
+#define BIT_SHIFT_MBIST_FAIL_NRML 0
+#define BIT_MASK_MBIST_FAIL_NRML 0xffffffffL
+#define BIT_MBIST_FAIL_NRML(x)                                                 \
+	(((x) & BIT_MASK_MBIST_FAIL_NRML) << BIT_SHIFT_MBIST_FAIL_NRML)
+#define BIT_GET_MBIST_FAIL_NRML(x)                                             \
+	(((x) >> BIT_SHIFT_MBIST_FAIL_NRML) & BIT_MASK_MBIST_FAIL_NRML)
+
+/* 2 REG_AES_DECRPT_DATA			(Offset 0x0180) */
+
+#define BIT_SHIFT_IPS_CFG_ADDR 0
+#define BIT_MASK_IPS_CFG_ADDR 0xff
+#define BIT_IPS_CFG_ADDR(x)                                                    \
+	(((x) & BIT_MASK_IPS_CFG_ADDR) << BIT_SHIFT_IPS_CFG_ADDR)
+#define BIT_GET_IPS_CFG_ADDR(x)                                                \
+	(((x) >> BIT_SHIFT_IPS_CFG_ADDR) & BIT_MASK_IPS_CFG_ADDR)
+
+/* 2 REG_AES_DECRPT_CFG			(Offset 0x0184) */
+
+#define BIT_SHIFT_IPS_CFG_DATA 0
+#define BIT_MASK_IPS_CFG_DATA 0xffffffffL
+#define BIT_IPS_CFG_DATA(x)                                                    \
+	(((x) & BIT_MASK_IPS_CFG_DATA) << BIT_SHIFT_IPS_CFG_DATA)
+#define BIT_GET_IPS_CFG_DATA(x)                                                \
+	(((x) >> BIT_SHIFT_IPS_CFG_DATA) & BIT_MASK_IPS_CFG_DATA)
+
+/* 2 REG_TMETER				(Offset 0x0190) */
+
+#define BIT_TEMP_VALID BIT(31)
+
+#define BIT_SHIFT_TEMP_VALUE 24
+#define BIT_MASK_TEMP_VALUE 0x3f
+#define BIT_TEMP_VALUE(x) (((x) & BIT_MASK_TEMP_VALUE) << BIT_SHIFT_TEMP_VALUE)
+#define BIT_GET_TEMP_VALUE(x)                                                  \
+	(((x) >> BIT_SHIFT_TEMP_VALUE) & BIT_MASK_TEMP_VALUE)
+
+#define BIT_SHIFT_REG_TMETER_TIMER 8
+#define BIT_MASK_REG_TMETER_TIMER 0xfff
+#define BIT_REG_TMETER_TIMER(x)                                                \
+	(((x) & BIT_MASK_REG_TMETER_TIMER) << BIT_SHIFT_REG_TMETER_TIMER)
+#define BIT_GET_REG_TMETER_TIMER(x)                                            \
+	(((x) >> BIT_SHIFT_REG_TMETER_TIMER) & BIT_MASK_REG_TMETER_TIMER)
+
+#define BIT_SHIFT_REG_TEMP_DELTA 2
+#define BIT_MASK_REG_TEMP_DELTA 0x3f
+#define BIT_REG_TEMP_DELTA(x)                                                  \
+	(((x) & BIT_MASK_REG_TEMP_DELTA) << BIT_SHIFT_REG_TEMP_DELTA)
+#define BIT_GET_REG_TEMP_DELTA(x)                                              \
+	(((x) >> BIT_SHIFT_REG_TEMP_DELTA) & BIT_MASK_REG_TEMP_DELTA)
+
+#define BIT_REG_TMETER_EN BIT(0)
+
+/* 2 REG_OSC_32K_CTRL			(Offset 0x0194) */
+
+#define BIT_SHIFT_OSC_32K_CLKGEN_0 16
+#define BIT_MASK_OSC_32K_CLKGEN_0 0xffff
+#define BIT_OSC_32K_CLKGEN_0(x)                                                \
+	(((x) & BIT_MASK_OSC_32K_CLKGEN_0) << BIT_SHIFT_OSC_32K_CLKGEN_0)
+#define BIT_GET_OSC_32K_CLKGEN_0(x)                                            \
+	(((x) >> BIT_SHIFT_OSC_32K_CLKGEN_0) & BIT_MASK_OSC_32K_CLKGEN_0)
+
+/* 2 REG_OSC_32K_CTRL			(Offset 0x0194) */
+
+#define BIT_SHIFT_OSC_32K_RES_COMP 4
+#define BIT_MASK_OSC_32K_RES_COMP 0x3
+#define BIT_OSC_32K_RES_COMP(x)                                                \
+	(((x) & BIT_MASK_OSC_32K_RES_COMP) << BIT_SHIFT_OSC_32K_RES_COMP)
+#define BIT_GET_OSC_32K_RES_COMP(x)                                            \
+	(((x) >> BIT_SHIFT_OSC_32K_RES_COMP) & BIT_MASK_OSC_32K_RES_COMP)
+
+#define BIT_OSC_32K_OUT_SEL BIT(3)
+
+/* 2 REG_OSC_32K_CTRL			(Offset 0x0194) */
+
+#define BIT_ISO_WL_2_OSC_32K BIT(1)
+
+/* 2 REG_OSC_32K_CTRL			(Offset 0x0194) */
+
+#define BIT_POW_CKGEN BIT(0)
+
+/* 2 REG_32K_CAL_REG1			(Offset 0x0198) */
+
+#define BIT_CAL_32K_REG_WR BIT(31)
+#define BIT_CAL_32K_DBG_SEL BIT(22)
+
+#define BIT_SHIFT_CAL_32K_REG_ADDR 16
+#define BIT_MASK_CAL_32K_REG_ADDR 0x3f
+#define BIT_CAL_32K_REG_ADDR(x)                                                \
+	(((x) & BIT_MASK_CAL_32K_REG_ADDR) << BIT_SHIFT_CAL_32K_REG_ADDR)
+#define BIT_GET_CAL_32K_REG_ADDR(x)                                            \
+	(((x) >> BIT_SHIFT_CAL_32K_REG_ADDR) & BIT_MASK_CAL_32K_REG_ADDR)
+
+/* 2 REG_32K_CAL_REG1			(Offset 0x0198) */
+
+#define BIT_SHIFT_CAL_32K_REG_DATA 0
+#define BIT_MASK_CAL_32K_REG_DATA 0xffff
+#define BIT_CAL_32K_REG_DATA(x)                                                \
+	(((x) & BIT_MASK_CAL_32K_REG_DATA) << BIT_SHIFT_CAL_32K_REG_DATA)
+#define BIT_GET_CAL_32K_REG_DATA(x)                                            \
+	(((x) >> BIT_SHIFT_CAL_32K_REG_DATA) & BIT_MASK_CAL_32K_REG_DATA)
+
+/* 2 REG_C2HEVT				(Offset 0x01A0) */
+
+#define BIT_SHIFT_C2HEVT_MSG 0
+#define BIT_MASK_C2HEVT_MSG 0xffffffffffffffffffffffffffffffffL
+#define BIT_C2HEVT_MSG(x) (((x) & BIT_MASK_C2HEVT_MSG) << BIT_SHIFT_C2HEVT_MSG)
+#define BIT_GET_C2HEVT_MSG(x)                                                  \
+	(((x) >> BIT_SHIFT_C2HEVT_MSG) & BIT_MASK_C2HEVT_MSG)
+
+/* 2 REG_SW_DEFINED_PAGE1			(Offset 0x01B8) */
+
+#define BIT_SHIFT_SW_DEFINED_PAGE1 0
+#define BIT_MASK_SW_DEFINED_PAGE1 0xffffffffffffffffL
+#define BIT_SW_DEFINED_PAGE1(x)                                                \
+	(((x) & BIT_MASK_SW_DEFINED_PAGE1) << BIT_SHIFT_SW_DEFINED_PAGE1)
+#define BIT_GET_SW_DEFINED_PAGE1(x)                                            \
+	(((x) >> BIT_SHIFT_SW_DEFINED_PAGE1) & BIT_MASK_SW_DEFINED_PAGE1)
+
+/* 2 REG_MCUTST_I				(Offset 0x01C0) */
+
+#define BIT_SHIFT_MCUDMSG_I 0
+#define BIT_MASK_MCUDMSG_I 0xffffffffL
+#define BIT_MCUDMSG_I(x) (((x) & BIT_MASK_MCUDMSG_I) << BIT_SHIFT_MCUDMSG_I)
+#define BIT_GET_MCUDMSG_I(x) (((x) >> BIT_SHIFT_MCUDMSG_I) & BIT_MASK_MCUDMSG_I)
+
+/* 2 REG_MCUTST_II				(Offset 0x01C4) */
+
+#define BIT_SHIFT_MCUDMSG_II 0
+#define BIT_MASK_MCUDMSG_II 0xffffffffL
+#define BIT_MCUDMSG_II(x) (((x) & BIT_MASK_MCUDMSG_II) << BIT_SHIFT_MCUDMSG_II)
+#define BIT_GET_MCUDMSG_II(x)                                                  \
+	(((x) >> BIT_SHIFT_MCUDMSG_II) & BIT_MASK_MCUDMSG_II)
+
+/* 2 REG_FMETHR				(Offset 0x01C8) */
+
+#define BIT_FMSG_INT BIT(31)
+
+#define BIT_SHIFT_FW_MSG 0
+#define BIT_MASK_FW_MSG 0xffffffffL
+#define BIT_FW_MSG(x) (((x) & BIT_MASK_FW_MSG) << BIT_SHIFT_FW_MSG)
+#define BIT_GET_FW_MSG(x) (((x) >> BIT_SHIFT_FW_MSG) & BIT_MASK_FW_MSG)
+
+/* 2 REG_HMETFR				(Offset 0x01CC) */
+
+#define BIT_SHIFT_HRCV_MSG 24
+#define BIT_MASK_HRCV_MSG 0xff
+#define BIT_HRCV_MSG(x) (((x) & BIT_MASK_HRCV_MSG) << BIT_SHIFT_HRCV_MSG)
+#define BIT_GET_HRCV_MSG(x) (((x) >> BIT_SHIFT_HRCV_MSG) & BIT_MASK_HRCV_MSG)
+
+#define BIT_INT_BOX3 BIT(3)
+#define BIT_INT_BOX2 BIT(2)
+#define BIT_INT_BOX1 BIT(1)
+#define BIT_INT_BOX0 BIT(0)
+
+/* 2 REG_HMEBOX0				(Offset 0x01D0) */
+
+#define BIT_SHIFT_HOST_MSG_0 0
+#define BIT_MASK_HOST_MSG_0 0xffffffffL
+#define BIT_HOST_MSG_0(x) (((x) & BIT_MASK_HOST_MSG_0) << BIT_SHIFT_HOST_MSG_0)
+#define BIT_GET_HOST_MSG_0(x)                                                  \
+	(((x) >> BIT_SHIFT_HOST_MSG_0) & BIT_MASK_HOST_MSG_0)
+
+/* 2 REG_HMEBOX1				(Offset 0x01D4) */
+
+#define BIT_SHIFT_HOST_MSG_1 0
+#define BIT_MASK_HOST_MSG_1 0xffffffffL
+#define BIT_HOST_MSG_1(x) (((x) & BIT_MASK_HOST_MSG_1) << BIT_SHIFT_HOST_MSG_1)
+#define BIT_GET_HOST_MSG_1(x)                                                  \
+	(((x) >> BIT_SHIFT_HOST_MSG_1) & BIT_MASK_HOST_MSG_1)
+
+/* 2 REG_HMEBOX2				(Offset 0x01D8) */
+
+#define BIT_SHIFT_HOST_MSG_2 0
+#define BIT_MASK_HOST_MSG_2 0xffffffffL
+#define BIT_HOST_MSG_2(x) (((x) & BIT_MASK_HOST_MSG_2) << BIT_SHIFT_HOST_MSG_2)
+#define BIT_GET_HOST_MSG_2(x)                                                  \
+	(((x) >> BIT_SHIFT_HOST_MSG_2) & BIT_MASK_HOST_MSG_2)
+
+/* 2 REG_HMEBOX3				(Offset 0x01DC) */
+
+#define BIT_SHIFT_HOST_MSG_3 0
+#define BIT_MASK_HOST_MSG_3 0xffffffffL
+#define BIT_HOST_MSG_3(x) (((x) & BIT_MASK_HOST_MSG_3) << BIT_SHIFT_HOST_MSG_3)
+#define BIT_GET_HOST_MSG_3(x)                                                  \
+	(((x) >> BIT_SHIFT_HOST_MSG_3) & BIT_MASK_HOST_MSG_3)
+
+/* 2 REG_LLT_INIT				(Offset 0x01E0) */
+
+#define BIT_SHIFT_LLTE_RWM 30
+#define BIT_MASK_LLTE_RWM 0x3
+#define BIT_LLTE_RWM(x) (((x) & BIT_MASK_LLTE_RWM) << BIT_SHIFT_LLTE_RWM)
+#define BIT_GET_LLTE_RWM(x) (((x) >> BIT_SHIFT_LLTE_RWM) & BIT_MASK_LLTE_RWM)
+
+/* 2 REG_LLT_INIT				(Offset 0x01E0) */
+
+#define BIT_SHIFT_LLTINI_PDATA_V1 16
+#define BIT_MASK_LLTINI_PDATA_V1 0xfff
+#define BIT_LLTINI_PDATA_V1(x)                                                 \
+	(((x) & BIT_MASK_LLTINI_PDATA_V1) << BIT_SHIFT_LLTINI_PDATA_V1)
+#define BIT_GET_LLTINI_PDATA_V1(x)                                             \
+	(((x) >> BIT_SHIFT_LLTINI_PDATA_V1) & BIT_MASK_LLTINI_PDATA_V1)
+
+/* 2 REG_LLT_INIT				(Offset 0x01E0) */
+
+#define BIT_SHIFT_LLTINI_HDATA_V1 0
+#define BIT_MASK_LLTINI_HDATA_V1 0xfff
+#define BIT_LLTINI_HDATA_V1(x)                                                 \
+	(((x) & BIT_MASK_LLTINI_HDATA_V1) << BIT_SHIFT_LLTINI_HDATA_V1)
+#define BIT_GET_LLTINI_HDATA_V1(x)                                             \
+	(((x) >> BIT_SHIFT_LLTINI_HDATA_V1) & BIT_MASK_LLTINI_HDATA_V1)
+
+/* 2 REG_LLT_INIT_ADDR			(Offset 0x01E4) */
+
+#define BIT_SHIFT_LLTINI_ADDR_V1 0
+#define BIT_MASK_LLTINI_ADDR_V1 0xfff
+#define BIT_LLTINI_ADDR_V1(x)                                                  \
+	(((x) & BIT_MASK_LLTINI_ADDR_V1) << BIT_SHIFT_LLTINI_ADDR_V1)
+#define BIT_GET_LLTINI_ADDR_V1(x)                                              \
+	(((x) >> BIT_SHIFT_LLTINI_ADDR_V1) & BIT_MASK_LLTINI_ADDR_V1)
+
+/* 2 REG_BB_ACCESS_CTRL			(Offset 0x01E8) */
+
+#define BIT_SHIFT_BB_WRITE_READ 30
+#define BIT_MASK_BB_WRITE_READ 0x3
+#define BIT_BB_WRITE_READ(x)                                                   \
+	(((x) & BIT_MASK_BB_WRITE_READ) << BIT_SHIFT_BB_WRITE_READ)
+#define BIT_GET_BB_WRITE_READ(x)                                               \
+	(((x) >> BIT_SHIFT_BB_WRITE_READ) & BIT_MASK_BB_WRITE_READ)
+
+/* 2 REG_BB_ACCESS_CTRL			(Offset 0x01E8) */
+
+#define BIT_SHIFT_BB_WRITE_EN 12
+#define BIT_MASK_BB_WRITE_EN 0xf
+#define BIT_BB_WRITE_EN(x)                                                     \
+	(((x) & BIT_MASK_BB_WRITE_EN) << BIT_SHIFT_BB_WRITE_EN)
+#define BIT_GET_BB_WRITE_EN(x)                                                 \
+	(((x) >> BIT_SHIFT_BB_WRITE_EN) & BIT_MASK_BB_WRITE_EN)
+
+#define BIT_SHIFT_BB_ADDR 2
+#define BIT_MASK_BB_ADDR 0x1ff
+#define BIT_BB_ADDR(x) (((x) & BIT_MASK_BB_ADDR) << BIT_SHIFT_BB_ADDR)
+#define BIT_GET_BB_ADDR(x) (((x) >> BIT_SHIFT_BB_ADDR) & BIT_MASK_BB_ADDR)
+
+/* 2 REG_BB_ACCESS_CTRL			(Offset 0x01E8) */
+
+#define BIT_BB_ERRACC BIT(0)
+
+/* 2 REG_BB_ACCESS_DATA			(Offset 0x01EC) */
+
+#define BIT_SHIFT_BB_DATA 0
+#define BIT_MASK_BB_DATA 0xffffffffL
+#define BIT_BB_DATA(x) (((x) & BIT_MASK_BB_DATA) << BIT_SHIFT_BB_DATA)
+#define BIT_GET_BB_DATA(x) (((x) >> BIT_SHIFT_BB_DATA) & BIT_MASK_BB_DATA)
+
+/* 2 REG_HMEBOX_E0				(Offset 0x01F0) */
+
+#define BIT_SHIFT_HMEBOX_E0 0
+#define BIT_MASK_HMEBOX_E0 0xffffffffL
+#define BIT_HMEBOX_E0(x) (((x) & BIT_MASK_HMEBOX_E0) << BIT_SHIFT_HMEBOX_E0)
+#define BIT_GET_HMEBOX_E0(x) (((x) >> BIT_SHIFT_HMEBOX_E0) & BIT_MASK_HMEBOX_E0)
+
+/* 2 REG_HMEBOX_E1				(Offset 0x01F4) */
+
+#define BIT_SHIFT_HMEBOX_E1 0
+#define BIT_MASK_HMEBOX_E1 0xffffffffL
+#define BIT_HMEBOX_E1(x) (((x) & BIT_MASK_HMEBOX_E1) << BIT_SHIFT_HMEBOX_E1)
+#define BIT_GET_HMEBOX_E1(x) (((x) >> BIT_SHIFT_HMEBOX_E1) & BIT_MASK_HMEBOX_E1)
+
+/* 2 REG_HMEBOX_E2				(Offset 0x01F8) */
+
+#define BIT_SHIFT_HMEBOX_E2 0
+#define BIT_MASK_HMEBOX_E2 0xffffffffL
+#define BIT_HMEBOX_E2(x) (((x) & BIT_MASK_HMEBOX_E2) << BIT_SHIFT_HMEBOX_E2)
+#define BIT_GET_HMEBOX_E2(x) (((x) >> BIT_SHIFT_HMEBOX_E2) & BIT_MASK_HMEBOX_E2)
+
+/* 2 REG_HMEBOX_E3				(Offset 0x01FC) */
+
+#define BIT_LD_RQPN BIT(31)
+
+#define BIT_SHIFT_HMEBOX_E3 0
+#define BIT_MASK_HMEBOX_E3 0xffffffffL
+#define BIT_HMEBOX_E3(x) (((x) & BIT_MASK_HMEBOX_E3) << BIT_SHIFT_HMEBOX_E3)
+#define BIT_GET_HMEBOX_E3(x) (((x) >> BIT_SHIFT_HMEBOX_E3) & BIT_MASK_HMEBOX_E3)
+
+/* 2 REG_FIFOPAGE_CTRL_1			(Offset 0x0200) */
+
+#define BIT_SHIFT_TX_OQT_HE_FREE_SPACE_V1 16
+#define BIT_MASK_TX_OQT_HE_FREE_SPACE_V1 0xff
+#define BIT_TX_OQT_HE_FREE_SPACE_V1(x)                                         \
+	(((x) & BIT_MASK_TX_OQT_HE_FREE_SPACE_V1)                              \
+	 << BIT_SHIFT_TX_OQT_HE_FREE_SPACE_V1)
+#define BIT_GET_TX_OQT_HE_FREE_SPACE_V1(x)                                     \
+	(((x) >> BIT_SHIFT_TX_OQT_HE_FREE_SPACE_V1) &                          \
+	 BIT_MASK_TX_OQT_HE_FREE_SPACE_V1)
+
+/* 2 REG_FIFOPAGE_CTRL_1			(Offset 0x0200) */
+
+#define BIT_SHIFT_TX_OQT_NL_FREE_SPACE_V1 0
+#define BIT_MASK_TX_OQT_NL_FREE_SPACE_V1 0xff
+#define BIT_TX_OQT_NL_FREE_SPACE_V1(x)                                         \
+	(((x) & BIT_MASK_TX_OQT_NL_FREE_SPACE_V1)                              \
+	 << BIT_SHIFT_TX_OQT_NL_FREE_SPACE_V1)
+#define BIT_GET_TX_OQT_NL_FREE_SPACE_V1(x)                                     \
+	(((x) >> BIT_SHIFT_TX_OQT_NL_FREE_SPACE_V1) &                          \
+	 BIT_MASK_TX_OQT_NL_FREE_SPACE_V1)
+
+/* 2 REG_FIFOPAGE_CTRL_2			(Offset 0x0204) */
+
+#define BIT_BCN_VALID_1_V1 BIT(31)
+
+/* 2 REG_FIFOPAGE_CTRL_2			(Offset 0x0204) */
+
+#define BIT_SHIFT_BCN_HEAD_1_V1 16
+#define BIT_MASK_BCN_HEAD_1_V1 0xfff
+#define BIT_BCN_HEAD_1_V1(x)                                                   \
+	(((x) & BIT_MASK_BCN_HEAD_1_V1) << BIT_SHIFT_BCN_HEAD_1_V1)
+#define BIT_GET_BCN_HEAD_1_V1(x)                                               \
+	(((x) >> BIT_SHIFT_BCN_HEAD_1_V1) & BIT_MASK_BCN_HEAD_1_V1)
+
+#define BIT_BCN_VALID_V1 BIT(15)
+
+/* 2 REG_FIFOPAGE_CTRL_2			(Offset 0x0204) */
+
+#define BIT_SHIFT_BCN_HEAD_V1 0
+#define BIT_MASK_BCN_HEAD_V1 0xfff
+#define BIT_BCN_HEAD_V1(x)                                                     \
+	(((x) & BIT_MASK_BCN_HEAD_V1) << BIT_SHIFT_BCN_HEAD_V1)
+#define BIT_GET_BCN_HEAD_V1(x)                                                 \
+	(((x) >> BIT_SHIFT_BCN_HEAD_V1) & BIT_MASK_BCN_HEAD_V1)
+
+/* 2 REG_AUTO_LLT_V1				(Offset 0x0208) */
+
+#define BIT_SHIFT_MAX_TX_PKT_FOR_USB_AND_SDIO_V1 24
+#define BIT_MASK_MAX_TX_PKT_FOR_USB_AND_SDIO_V1 0xff
+#define BIT_MAX_TX_PKT_FOR_USB_AND_SDIO_V1(x)                                  \
+	(((x) & BIT_MASK_MAX_TX_PKT_FOR_USB_AND_SDIO_V1)                       \
+	 << BIT_SHIFT_MAX_TX_PKT_FOR_USB_AND_SDIO_V1)
+#define BIT_GET_MAX_TX_PKT_FOR_USB_AND_SDIO_V1(x)                              \
+	(((x) >> BIT_SHIFT_MAX_TX_PKT_FOR_USB_AND_SDIO_V1) &                   \
+	 BIT_MASK_MAX_TX_PKT_FOR_USB_AND_SDIO_V1)
+
+/* 2 REG_AUTO_LLT_V1				(Offset 0x0208) */
+
+#define BIT_SHIFT_LLT_FREE_PAGE_V1 8
+#define BIT_MASK_LLT_FREE_PAGE_V1 0xffff
+#define BIT_LLT_FREE_PAGE_V1(x)                                                \
+	(((x) & BIT_MASK_LLT_FREE_PAGE_V1) << BIT_SHIFT_LLT_FREE_PAGE_V1)
+#define BIT_GET_LLT_FREE_PAGE_V1(x)                                            \
+	(((x) >> BIT_SHIFT_LLT_FREE_PAGE_V1) & BIT_MASK_LLT_FREE_PAGE_V1)
+
+/* 2 REG_DWBCN0_CTRL				(Offset 0x0208) */
+
+#define BIT_SHIFT_BLK_DESC_NUM 4
+#define BIT_MASK_BLK_DESC_NUM 0xf
+#define BIT_BLK_DESC_NUM(x)                                                    \
+	(((x) & BIT_MASK_BLK_DESC_NUM) << BIT_SHIFT_BLK_DESC_NUM)
+#define BIT_GET_BLK_DESC_NUM(x)                                                \
+	(((x) >> BIT_SHIFT_BLK_DESC_NUM) & BIT_MASK_BLK_DESC_NUM)
+
+/* 2 REG_AUTO_LLT_V1				(Offset 0x0208) */
+
+#define BIT_R_BCN_HEAD_SEL BIT(3)
+#define BIT_R_EN_BCN_SW_HEAD_SEL BIT(2)
+#define BIT_LLT_DBG_SEL BIT(1)
+#define BIT_AUTO_INIT_LLT_V1 BIT(0)
+
+/* 2 REG_TXDMA_OFFSET_CHK			(Offset 0x020C) */
+
+#define BIT_EM_CHKSUM_FIN BIT(31)
+#define BIT_EMN_PCIE_DMA_MOD BIT(30)
+
+/* 2 REG_TXDMA_OFFSET_CHK			(Offset 0x020C) */
+
+#define BIT_EN_TXQUE_CLR BIT(29)
+#define BIT_EN_PCIE_FIFO_MODE BIT(28)
+
+/* 2 REG_TXDMA_OFFSET_CHK			(Offset 0x020C) */
+
+#define BIT_SHIFT_PG_UNDER_TH_V1 16
+#define BIT_MASK_PG_UNDER_TH_V1 0xfff
+#define BIT_PG_UNDER_TH_V1(x)                                                  \
+	(((x) & BIT_MASK_PG_UNDER_TH_V1) << BIT_SHIFT_PG_UNDER_TH_V1)
+#define BIT_GET_PG_UNDER_TH_V1(x)                                              \
+	(((x) >> BIT_SHIFT_PG_UNDER_TH_V1) & BIT_MASK_PG_UNDER_TH_V1)
+
+/* 2 REG_TXDMA_OFFSET_CHK			(Offset 0x020C) */
+
+#define BIT_RESTORE_H2C_ADDRESS BIT(15)
+
+/* 2 REG_TXDMA_OFFSET_CHK			(Offset 0x020C) */
+
+#define BIT_SDIO_TXDESC_CHKSUM_EN BIT(13)
+#define BIT_RST_RDPTR BIT(12)
+#define BIT_RST_WRPTR BIT(11)
+#define BIT_CHK_PG_TH_EN BIT(10)
+#define BIT_DROP_DATA_EN BIT(9)
+#define BIT_CHECK_OFFSET_EN BIT(8)
+
+#define BIT_SHIFT_CHECK_OFFSET 0
+#define BIT_MASK_CHECK_OFFSET 0xff
+#define BIT_CHECK_OFFSET(x)                                                    \
+	(((x) & BIT_MASK_CHECK_OFFSET) << BIT_SHIFT_CHECK_OFFSET)
+#define BIT_GET_CHECK_OFFSET(x)                                                \
+	(((x) >> BIT_SHIFT_CHECK_OFFSET) & BIT_MASK_CHECK_OFFSET)
+
+/* 2 REG_TXDMA_STATUS			(Offset 0x0210) */
+
+#define BIT_HI_OQT_UDN BIT(17)
+#define BIT_HI_OQT_OVF BIT(16)
+#define BIT_PAYLOAD_CHKSUM_ERR BIT(15)
+#define BIT_PAYLOAD_UDN BIT(14)
+#define BIT_PAYLOAD_OVF BIT(13)
+#define BIT_DSC_CHKSUM_FAIL BIT(12)
+#define BIT_UNKNOWN_QSEL BIT(11)
+#define BIT_EP_QSEL_DIFF BIT(10)
+#define BIT_TX_OFFS_UNMATCH BIT(9)
+#define BIT_TXOQT_UDN BIT(8)
+#define BIT_TXOQT_OVF BIT(7)
+#define BIT_TXDMA_SFF_UDN BIT(6)
+#define BIT_TXDMA_SFF_OVF BIT(5)
+#define BIT_LLT_NULL_PG BIT(4)
+#define BIT_PAGE_UDN BIT(3)
+#define BIT_PAGE_OVF BIT(2)
+#define BIT_TXFF_PG_UDN BIT(1)
+#define BIT_TXFF_PG_OVF BIT(0)
+
+/* 2 REG_TQPNT1				(Offset 0x0218) */
+
+#define BIT_SHIFT_HPQ_HIGH_TH_V1 16
+#define BIT_MASK_HPQ_HIGH_TH_V1 0xfff
+#define BIT_HPQ_HIGH_TH_V1(x)                                                  \
+	(((x) & BIT_MASK_HPQ_HIGH_TH_V1) << BIT_SHIFT_HPQ_HIGH_TH_V1)
+#define BIT_GET_HPQ_HIGH_TH_V1(x)                                              \
+	(((x) >> BIT_SHIFT_HPQ_HIGH_TH_V1) & BIT_MASK_HPQ_HIGH_TH_V1)
+
+/* 2 REG_TQPNT1				(Offset 0x0218) */
+
+#define BIT_SHIFT_HPQ_LOW_TH_V1 0
+#define BIT_MASK_HPQ_LOW_TH_V1 0xfff
+#define BIT_HPQ_LOW_TH_V1(x)                                                   \
+	(((x) & BIT_MASK_HPQ_LOW_TH_V1) << BIT_SHIFT_HPQ_LOW_TH_V1)
+#define BIT_GET_HPQ_LOW_TH_V1(x)                                               \
+	(((x) >> BIT_SHIFT_HPQ_LOW_TH_V1) & BIT_MASK_HPQ_LOW_TH_V1)
+
+/* 2 REG_TQPNT2				(Offset 0x021C) */
+
+#define BIT_SHIFT_NPQ_HIGH_TH_V1 16
+#define BIT_MASK_NPQ_HIGH_TH_V1 0xfff
+#define BIT_NPQ_HIGH_TH_V1(x)                                                  \
+	(((x) & BIT_MASK_NPQ_HIGH_TH_V1) << BIT_SHIFT_NPQ_HIGH_TH_V1)
+#define BIT_GET_NPQ_HIGH_TH_V1(x)                                              \
+	(((x) >> BIT_SHIFT_NPQ_HIGH_TH_V1) & BIT_MASK_NPQ_HIGH_TH_V1)
+
+/* 2 REG_TQPNT2				(Offset 0x021C) */
+
+#define BIT_SHIFT_NPQ_LOW_TH_V1 0
+#define BIT_MASK_NPQ_LOW_TH_V1 0xfff
+#define BIT_NPQ_LOW_TH_V1(x)                                                   \
+	(((x) & BIT_MASK_NPQ_LOW_TH_V1) << BIT_SHIFT_NPQ_LOW_TH_V1)
+#define BIT_GET_NPQ_LOW_TH_V1(x)                                               \
+	(((x) >> BIT_SHIFT_NPQ_LOW_TH_V1) & BIT_MASK_NPQ_LOW_TH_V1)
+
+/* 2 REG_TQPNT3				(Offset 0x0220) */
+
+#define BIT_SHIFT_LPQ_HIGH_TH_V1 16
+#define BIT_MASK_LPQ_HIGH_TH_V1 0xfff
+#define BIT_LPQ_HIGH_TH_V1(x)                                                  \
+	(((x) & BIT_MASK_LPQ_HIGH_TH_V1) << BIT_SHIFT_LPQ_HIGH_TH_V1)
+#define BIT_GET_LPQ_HIGH_TH_V1(x)                                              \
+	(((x) >> BIT_SHIFT_LPQ_HIGH_TH_V1) & BIT_MASK_LPQ_HIGH_TH_V1)
+
+/* 2 REG_TQPNT3				(Offset 0x0220) */
+
+#define BIT_SHIFT_LPQ_LOW_TH_V1 0
+#define BIT_MASK_LPQ_LOW_TH_V1 0xfff
+#define BIT_LPQ_LOW_TH_V1(x)                                                   \
+	(((x) & BIT_MASK_LPQ_LOW_TH_V1) << BIT_SHIFT_LPQ_LOW_TH_V1)
+#define BIT_GET_LPQ_LOW_TH_V1(x)                                               \
+	(((x) >> BIT_SHIFT_LPQ_LOW_TH_V1) & BIT_MASK_LPQ_LOW_TH_V1)
+
+/* 2 REG_TQPNT4				(Offset 0x0224) */
+
+#define BIT_SHIFT_EXQ_HIGH_TH_V1 16
+#define BIT_MASK_EXQ_HIGH_TH_V1 0xfff
+#define BIT_EXQ_HIGH_TH_V1(x)                                                  \
+	(((x) & BIT_MASK_EXQ_HIGH_TH_V1) << BIT_SHIFT_EXQ_HIGH_TH_V1)
+#define BIT_GET_EXQ_HIGH_TH_V1(x)                                              \
+	(((x) >> BIT_SHIFT_EXQ_HIGH_TH_V1) & BIT_MASK_EXQ_HIGH_TH_V1)
+
+/* 2 REG_TQPNT4				(Offset 0x0224) */
+
+#define BIT_SHIFT_EXQ_LOW_TH_V1 0
+#define BIT_MASK_EXQ_LOW_TH_V1 0xfff
+#define BIT_EXQ_LOW_TH_V1(x)                                                   \
+	(((x) & BIT_MASK_EXQ_LOW_TH_V1) << BIT_SHIFT_EXQ_LOW_TH_V1)
+#define BIT_GET_EXQ_LOW_TH_V1(x)                                               \
+	(((x) >> BIT_SHIFT_EXQ_LOW_TH_V1) & BIT_MASK_EXQ_LOW_TH_V1)
+
+/* 2 REG_RQPN_CTRL_1				(Offset 0x0228) */
+
+#define BIT_SHIFT_TXPKTNUM_H 16
+#define BIT_MASK_TXPKTNUM_H 0xffff
+#define BIT_TXPKTNUM_H(x) (((x) & BIT_MASK_TXPKTNUM_H) << BIT_SHIFT_TXPKTNUM_H)
+#define BIT_GET_TXPKTNUM_H(x)                                                  \
+	(((x) >> BIT_SHIFT_TXPKTNUM_H) & BIT_MASK_TXPKTNUM_H)
+
+/* 2 REG_RQPN_CTRL_1				(Offset 0x0228) */
+
+#define BIT_SHIFT_TXPKTNUM_V2 0
+#define BIT_MASK_TXPKTNUM_V2 0xffff
+#define BIT_TXPKTNUM_V2(x)                                                     \
+	(((x) & BIT_MASK_TXPKTNUM_V2) << BIT_SHIFT_TXPKTNUM_V2)
+#define BIT_GET_TXPKTNUM_V2(x)                                                 \
+	(((x) >> BIT_SHIFT_TXPKTNUM_V2) & BIT_MASK_TXPKTNUM_V2)
+
+/* 2 REG_RQPN_CTRL_2				(Offset 0x022C) */
+
+#define BIT_EXQ_PUBLIC_DIS_V1 BIT(19)
+#define BIT_NPQ_PUBLIC_DIS_V1 BIT(18)
+#define BIT_LPQ_PUBLIC_DIS_V1 BIT(17)
+#define BIT_HPQ_PUBLIC_DIS_V1 BIT(16)
+
+/* 2 REG_FIFOPAGE_INFO_1			(Offset 0x0230) */
+
+#define BIT_SHIFT_HPQ_AVAL_PG_V1 16
+#define BIT_MASK_HPQ_AVAL_PG_V1 0xfff
+#define BIT_HPQ_AVAL_PG_V1(x)                                                  \
+	(((x) & BIT_MASK_HPQ_AVAL_PG_V1) << BIT_SHIFT_HPQ_AVAL_PG_V1)
+#define BIT_GET_HPQ_AVAL_PG_V1(x)                                              \
+	(((x) >> BIT_SHIFT_HPQ_AVAL_PG_V1) & BIT_MASK_HPQ_AVAL_PG_V1)
+
+#define BIT_SHIFT_HPQ_V1 0
+#define BIT_MASK_HPQ_V1 0xfff
+#define BIT_HPQ_V1(x) (((x) & BIT_MASK_HPQ_V1) << BIT_SHIFT_HPQ_V1)
+#define BIT_GET_HPQ_V1(x) (((x) >> BIT_SHIFT_HPQ_V1) & BIT_MASK_HPQ_V1)
+
+/* 2 REG_FIFOPAGE_INFO_2			(Offset 0x0234) */
+
+#define BIT_SHIFT_LPQ_AVAL_PG_V1 16
+#define BIT_MASK_LPQ_AVAL_PG_V1 0xfff
+#define BIT_LPQ_AVAL_PG_V1(x)                                                  \
+	(((x) & BIT_MASK_LPQ_AVAL_PG_V1) << BIT_SHIFT_LPQ_AVAL_PG_V1)
+#define BIT_GET_LPQ_AVAL_PG_V1(x)                                              \
+	(((x) >> BIT_SHIFT_LPQ_AVAL_PG_V1) & BIT_MASK_LPQ_AVAL_PG_V1)
+
+#define BIT_SHIFT_LPQ_V1 0
+#define BIT_MASK_LPQ_V1 0xfff
+#define BIT_LPQ_V1(x) (((x) & BIT_MASK_LPQ_V1) << BIT_SHIFT_LPQ_V1)
+#define BIT_GET_LPQ_V1(x) (((x) >> BIT_SHIFT_LPQ_V1) & BIT_MASK_LPQ_V1)
+
+/* 2 REG_FIFOPAGE_INFO_3			(Offset 0x0238) */
+
+#define BIT_SHIFT_NPQ_AVAL_PG_V1 16
+#define BIT_MASK_NPQ_AVAL_PG_V1 0xfff
+#define BIT_NPQ_AVAL_PG_V1(x)                                                  \
+	(((x) & BIT_MASK_NPQ_AVAL_PG_V1) << BIT_SHIFT_NPQ_AVAL_PG_V1)
+#define BIT_GET_NPQ_AVAL_PG_V1(x)                                              \
+	(((x) >> BIT_SHIFT_NPQ_AVAL_PG_V1) & BIT_MASK_NPQ_AVAL_PG_V1)
+
+/* 2 REG_FIFOPAGE_INFO_3			(Offset 0x0238) */
+
+#define BIT_SHIFT_NPQ_V1 0
+#define BIT_MASK_NPQ_V1 0xfff
+#define BIT_NPQ_V1(x) (((x) & BIT_MASK_NPQ_V1) << BIT_SHIFT_NPQ_V1)
+#define BIT_GET_NPQ_V1(x) (((x) >> BIT_SHIFT_NPQ_V1) & BIT_MASK_NPQ_V1)
+
+/* 2 REG_FIFOPAGE_INFO_4			(Offset 0x023C) */
+
+#define BIT_SHIFT_EXQ_AVAL_PG_V1 16
+#define BIT_MASK_EXQ_AVAL_PG_V1 0xfff
+#define BIT_EXQ_AVAL_PG_V1(x)                                                  \
+	(((x) & BIT_MASK_EXQ_AVAL_PG_V1) << BIT_SHIFT_EXQ_AVAL_PG_V1)
+#define BIT_GET_EXQ_AVAL_PG_V1(x)                                              \
+	(((x) >> BIT_SHIFT_EXQ_AVAL_PG_V1) & BIT_MASK_EXQ_AVAL_PG_V1)
+
+#define BIT_SHIFT_EXQ_V1 0
+#define BIT_MASK_EXQ_V1 0xfff
+#define BIT_EXQ_V1(x) (((x) & BIT_MASK_EXQ_V1) << BIT_SHIFT_EXQ_V1)
+#define BIT_GET_EXQ_V1(x) (((x) >> BIT_SHIFT_EXQ_V1) & BIT_MASK_EXQ_V1)
+
+/* 2 REG_FIFOPAGE_INFO_5			(Offset 0x0240) */
+
+#define BIT_SHIFT_PUBQ_AVAL_PG_V1 16
+#define BIT_MASK_PUBQ_AVAL_PG_V1 0xfff
+#define BIT_PUBQ_AVAL_PG_V1(x)                                                 \
+	(((x) & BIT_MASK_PUBQ_AVAL_PG_V1) << BIT_SHIFT_PUBQ_AVAL_PG_V1)
+#define BIT_GET_PUBQ_AVAL_PG_V1(x)                                             \
+	(((x) >> BIT_SHIFT_PUBQ_AVAL_PG_V1) & BIT_MASK_PUBQ_AVAL_PG_V1)
+
+#define BIT_SHIFT_PUBQ_V1 0
+#define BIT_MASK_PUBQ_V1 0xfff
+#define BIT_PUBQ_V1(x) (((x) & BIT_MASK_PUBQ_V1) << BIT_SHIFT_PUBQ_V1)
+#define BIT_GET_PUBQ_V1(x) (((x) >> BIT_SHIFT_PUBQ_V1) & BIT_MASK_PUBQ_V1)
+
+/* 2 REG_H2C_HEAD				(Offset 0x0244) */
+
+#define BIT_SHIFT_H2C_HEAD 0
+#define BIT_MASK_H2C_HEAD 0x3ffff
+#define BIT_H2C_HEAD(x) (((x) & BIT_MASK_H2C_HEAD) << BIT_SHIFT_H2C_HEAD)
+#define BIT_GET_H2C_HEAD(x) (((x) >> BIT_SHIFT_H2C_HEAD) & BIT_MASK_H2C_HEAD)
+
+/* 2 REG_H2C_TAIL				(Offset 0x0248) */
+
+#define BIT_SHIFT_H2C_TAIL 0
+#define BIT_MASK_H2C_TAIL 0x3ffff
+#define BIT_H2C_TAIL(x) (((x) & BIT_MASK_H2C_TAIL) << BIT_SHIFT_H2C_TAIL)
+#define BIT_GET_H2C_TAIL(x) (((x) >> BIT_SHIFT_H2C_TAIL) & BIT_MASK_H2C_TAIL)
+
+/* 2 REG_H2C_READ_ADDR			(Offset 0x024C) */
+
+#define BIT_SHIFT_H2C_READ_ADDR 0
+#define BIT_MASK_H2C_READ_ADDR 0x3ffff
+#define BIT_H2C_READ_ADDR(x)                                                   \
+	(((x) & BIT_MASK_H2C_READ_ADDR) << BIT_SHIFT_H2C_READ_ADDR)
+#define BIT_GET_H2C_READ_ADDR(x)                                               \
+	(((x) >> BIT_SHIFT_H2C_READ_ADDR) & BIT_MASK_H2C_READ_ADDR)
+
+/* 2 REG_H2C_WR_ADDR				(Offset 0x0250) */
+
+#define BIT_SHIFT_H2C_WR_ADDR 0
+#define BIT_MASK_H2C_WR_ADDR 0x3ffff
+#define BIT_H2C_WR_ADDR(x)                                                     \
+	(((x) & BIT_MASK_H2C_WR_ADDR) << BIT_SHIFT_H2C_WR_ADDR)
+#define BIT_GET_H2C_WR_ADDR(x)                                                 \
+	(((x) >> BIT_SHIFT_H2C_WR_ADDR) & BIT_MASK_H2C_WR_ADDR)
+
+/* 2 REG_H2C_INFO				(Offset 0x0254) */
+
+#define BIT_H2C_SPACE_VLD BIT(3)
+#define BIT_H2C_WR_ADDR_RST BIT(2)
+
+#define BIT_SHIFT_H2C_LEN_SEL 0
+#define BIT_MASK_H2C_LEN_SEL 0x3
+#define BIT_H2C_LEN_SEL(x)                                                     \
+	(((x) & BIT_MASK_H2C_LEN_SEL) << BIT_SHIFT_H2C_LEN_SEL)
+#define BIT_GET_H2C_LEN_SEL(x)                                                 \
+	(((x) >> BIT_SHIFT_H2C_LEN_SEL) & BIT_MASK_H2C_LEN_SEL)
+
+/* 2 REG_RXDMA_AGG_PG_TH			(Offset 0x0280) */
+
+#define BIT_SHIFT_RXDMA_AGG_OLD_MOD 24
+#define BIT_MASK_RXDMA_AGG_OLD_MOD 0xff
+#define BIT_RXDMA_AGG_OLD_MOD(x)                                               \
+	(((x) & BIT_MASK_RXDMA_AGG_OLD_MOD) << BIT_SHIFT_RXDMA_AGG_OLD_MOD)
+#define BIT_GET_RXDMA_AGG_OLD_MOD(x)                                           \
+	(((x) >> BIT_SHIFT_RXDMA_AGG_OLD_MOD) & BIT_MASK_RXDMA_AGG_OLD_MOD)
+
+/* 2 REG_RXDMA_AGG_PG_TH			(Offset 0x0280) */
+
+#define BIT_SHIFT_PKT_NUM_WOL 16
+#define BIT_MASK_PKT_NUM_WOL 0xff
+#define BIT_PKT_NUM_WOL(x)                                                     \
+	(((x) & BIT_MASK_PKT_NUM_WOL) << BIT_SHIFT_PKT_NUM_WOL)
+#define BIT_GET_PKT_NUM_WOL(x)                                                 \
+	(((x) >> BIT_SHIFT_PKT_NUM_WOL) & BIT_MASK_PKT_NUM_WOL)
+
+/* 2 REG_RXDMA_AGG_PG_TH			(Offset 0x0280) */
+
+#define BIT_SHIFT_DMA_AGG_TO 8
+#define BIT_MASK_DMA_AGG_TO 0xf
+#define BIT_DMA_AGG_TO(x) (((x) & BIT_MASK_DMA_AGG_TO) << BIT_SHIFT_DMA_AGG_TO)
+#define BIT_GET_DMA_AGG_TO(x)                                                  \
+	(((x) >> BIT_SHIFT_DMA_AGG_TO) & BIT_MASK_DMA_AGG_TO)
+
+/* 2 REG_RXDMA_AGG_PG_TH			(Offset 0x0280) */
+
+#define BIT_SHIFT_RXDMA_AGG_PG_TH_V1 0
+#define BIT_MASK_RXDMA_AGG_PG_TH_V1 0xf
+#define BIT_RXDMA_AGG_PG_TH_V1(x)                                              \
+	(((x) & BIT_MASK_RXDMA_AGG_PG_TH_V1) << BIT_SHIFT_RXDMA_AGG_PG_TH_V1)
+#define BIT_GET_RXDMA_AGG_PG_TH_V1(x)                                          \
+	(((x) >> BIT_SHIFT_RXDMA_AGG_PG_TH_V1) & BIT_MASK_RXDMA_AGG_PG_TH_V1)
+
+/* 2 REG_RXPKT_NUM				(Offset 0x0284) */
+
+#define BIT_SHIFT_RXPKT_NUM 24
+#define BIT_MASK_RXPKT_NUM 0xff
+#define BIT_RXPKT_NUM(x) (((x) & BIT_MASK_RXPKT_NUM) << BIT_SHIFT_RXPKT_NUM)
+#define BIT_GET_RXPKT_NUM(x) (((x) >> BIT_SHIFT_RXPKT_NUM) & BIT_MASK_RXPKT_NUM)
+
+/* 2 REG_RXPKT_NUM				(Offset 0x0284) */
+
+#define BIT_SHIFT_FW_UPD_RDPTR19_TO_16 20
+#define BIT_MASK_FW_UPD_RDPTR19_TO_16 0xf
+#define BIT_FW_UPD_RDPTR19_TO_16(x)                                            \
+	(((x) & BIT_MASK_FW_UPD_RDPTR19_TO_16)                                 \
+	 << BIT_SHIFT_FW_UPD_RDPTR19_TO_16)
+#define BIT_GET_FW_UPD_RDPTR19_TO_16(x)                                        \
+	(((x) >> BIT_SHIFT_FW_UPD_RDPTR19_TO_16) &                             \
+	 BIT_MASK_FW_UPD_RDPTR19_TO_16)
+
+/* 2 REG_RXPKT_NUM				(Offset 0x0284) */
+
+#define BIT_RXDMA_REQ BIT(19)
+#define BIT_RW_RELEASE_EN BIT(18)
+#define BIT_RXDMA_IDLE BIT(17)
+#define BIT_RXPKT_RELEASE_POLL BIT(16)
+
+#define BIT_SHIFT_FW_UPD_RDPTR 0
+#define BIT_MASK_FW_UPD_RDPTR 0xffff
+#define BIT_FW_UPD_RDPTR(x)                                                    \
+	(((x) & BIT_MASK_FW_UPD_RDPTR) << BIT_SHIFT_FW_UPD_RDPTR)
+#define BIT_GET_FW_UPD_RDPTR(x)                                                \
+	(((x) >> BIT_SHIFT_FW_UPD_RDPTR) & BIT_MASK_FW_UPD_RDPTR)
+
+/* 2 REG_RXDMA_STATUS			(Offset 0x0288) */
+
+#define BIT_C2H_PKT_OVF BIT(7)
+
+/* 2 REG_RXDMA_STATUS			(Offset 0x0288) */
+
+#define BIT_AGG_CONFGI_ISSUE BIT(6)
+
+/* 2 REG_RXDMA_STATUS			(Offset 0x0288) */
+
+#define BIT_FW_POLL_ISSUE BIT(5)
+#define BIT_RX_DATA_UDN BIT(4)
+#define BIT_RX_SFF_UDN BIT(3)
+#define BIT_RX_SFF_OVF BIT(2)
+
+/* 2 REG_RXDMA_STATUS			(Offset 0x0288) */
+
+#define BIT_RXPKT_OVF BIT(0)
+
+/* 2 REG_RXDMA_DPR				(Offset 0x028C) */
+
+#define BIT_SHIFT_RDE_DEBUG 0
+#define BIT_MASK_RDE_DEBUG 0xffffffffL
+#define BIT_RDE_DEBUG(x) (((x) & BIT_MASK_RDE_DEBUG) << BIT_SHIFT_RDE_DEBUG)
+#define BIT_GET_RDE_DEBUG(x) (((x) >> BIT_SHIFT_RDE_DEBUG) & BIT_MASK_RDE_DEBUG)
+
+/* 2 REG_RXDMA_MODE				(Offset 0x0290) */
+
+#define BIT_SHIFT_PKTNUM_TH_V2 24
+#define BIT_MASK_PKTNUM_TH_V2 0x1f
+#define BIT_PKTNUM_TH_V2(x)                                                    \
+	(((x) & BIT_MASK_PKTNUM_TH_V2) << BIT_SHIFT_PKTNUM_TH_V2)
+#define BIT_GET_PKTNUM_TH_V2(x)                                                \
+	(((x) >> BIT_SHIFT_PKTNUM_TH_V2) & BIT_MASK_PKTNUM_TH_V2)
+
+#define BIT_TXBA_BREAK_USBAGG BIT(23)
+
+#define BIT_SHIFT_PKTLEN_PARA 16
+#define BIT_MASK_PKTLEN_PARA 0x7
+#define BIT_PKTLEN_PARA(x)                                                     \
+	(((x) & BIT_MASK_PKTLEN_PARA) << BIT_SHIFT_PKTLEN_PARA)
+#define BIT_GET_PKTLEN_PARA(x)                                                 \
+	(((x) >> BIT_SHIFT_PKTLEN_PARA) & BIT_MASK_PKTLEN_PARA)
+
+/* 2 REG_RXDMA_MODE				(Offset 0x0290) */
+
+#define BIT_SHIFT_BURST_SIZE 4
+#define BIT_MASK_BURST_SIZE 0x3
+#define BIT_BURST_SIZE(x) (((x) & BIT_MASK_BURST_SIZE) << BIT_SHIFT_BURST_SIZE)
+#define BIT_GET_BURST_SIZE(x)                                                  \
+	(((x) >> BIT_SHIFT_BURST_SIZE) & BIT_MASK_BURST_SIZE)
+
+#define BIT_SHIFT_BURST_CNT 2
+#define BIT_MASK_BURST_CNT 0x3
+#define BIT_BURST_CNT(x) (((x) & BIT_MASK_BURST_CNT) << BIT_SHIFT_BURST_CNT)
+#define BIT_GET_BURST_CNT(x) (((x) >> BIT_SHIFT_BURST_CNT) & BIT_MASK_BURST_CNT)
+
+/* 2 REG_RXDMA_MODE				(Offset 0x0290) */
+
+#define BIT_DMA_MODE BIT(1)
+
+/* 2 REG_C2H_PKT				(Offset 0x0294) */
+
+#define BIT_SHIFT_R_C2H_STR_ADDR_16_TO_19 24
+#define BIT_MASK_R_C2H_STR_ADDR_16_TO_19 0xf
+#define BIT_R_C2H_STR_ADDR_16_TO_19(x)                                         \
+	(((x) & BIT_MASK_R_C2H_STR_ADDR_16_TO_19)                              \
+	 << BIT_SHIFT_R_C2H_STR_ADDR_16_TO_19)
+#define BIT_GET_R_C2H_STR_ADDR_16_TO_19(x)                                     \
+	(((x) >> BIT_SHIFT_R_C2H_STR_ADDR_16_TO_19) &                          \
+	 BIT_MASK_R_C2H_STR_ADDR_16_TO_19)
+
+#define BIT_SHIFT_MDIO_PHY_ADDR 24
+#define BIT_MASK_MDIO_PHY_ADDR 0x1f
+#define BIT_MDIO_PHY_ADDR(x)                                                   \
+	(((x) & BIT_MASK_MDIO_PHY_ADDR) << BIT_SHIFT_MDIO_PHY_ADDR)
+#define BIT_GET_MDIO_PHY_ADDR(x)                                               \
+	(((x) >> BIT_SHIFT_MDIO_PHY_ADDR) & BIT_MASK_MDIO_PHY_ADDR)
+
+/* 2 REG_C2H_PKT				(Offset 0x0294) */
+
+#define BIT_R_C2H_PKT_REQ BIT(16)
+#define BIT_RX_CLOSE_EN BIT(15)
+#define BIT_STOP_BCNQ BIT(14)
+#define BIT_STOP_MGQ BIT(13)
+#define BIT_STOP_VOQ BIT(12)
+#define BIT_STOP_VIQ BIT(11)
+#define BIT_STOP_BEQ BIT(10)
+#define BIT_STOP_BKQ BIT(9)
+#define BIT_STOP_RXQ BIT(8)
+#define BIT_STOP_HI7Q BIT(7)
+#define BIT_STOP_HI6Q BIT(6)
+#define BIT_STOP_HI5Q BIT(5)
+#define BIT_STOP_HI4Q BIT(4)
+#define BIT_STOP_HI3Q BIT(3)
+#define BIT_STOP_HI2Q BIT(2)
+#define BIT_STOP_HI1Q BIT(1)
+
+#define BIT_SHIFT_R_C2H_STR_ADDR 0
+#define BIT_MASK_R_C2H_STR_ADDR 0xffff
+#define BIT_R_C2H_STR_ADDR(x)                                                  \
+	(((x) & BIT_MASK_R_C2H_STR_ADDR) << BIT_SHIFT_R_C2H_STR_ADDR)
+#define BIT_GET_R_C2H_STR_ADDR(x)                                              \
+	(((x) >> BIT_SHIFT_R_C2H_STR_ADDR) & BIT_MASK_R_C2H_STR_ADDR)
+
+#define BIT_STOP_HI0Q BIT(0)
+
+/* 2 REG_FWFF_C2H				(Offset 0x0298) */
+
+#define BIT_SHIFT_C2H_DMA_ADDR 0
+#define BIT_MASK_C2H_DMA_ADDR 0x3ffff
+#define BIT_C2H_DMA_ADDR(x)                                                    \
+	(((x) & BIT_MASK_C2H_DMA_ADDR) << BIT_SHIFT_C2H_DMA_ADDR)
+#define BIT_GET_C2H_DMA_ADDR(x)                                                \
+	(((x) >> BIT_SHIFT_C2H_DMA_ADDR) & BIT_MASK_C2H_DMA_ADDR)
+
+/* 2 REG_FWFF_CTRL				(Offset 0x029C) */
+
+#define BIT_FWFF_DMAPKT_REQ BIT(31)
+
+#define BIT_SHIFT_FWFF_DMA_PKT_NUM 16
+#define BIT_MASK_FWFF_DMA_PKT_NUM 0xff
+#define BIT_FWFF_DMA_PKT_NUM(x)                                                \
+	(((x) & BIT_MASK_FWFF_DMA_PKT_NUM) << BIT_SHIFT_FWFF_DMA_PKT_NUM)
+#define BIT_GET_FWFF_DMA_PKT_NUM(x)                                            \
+	(((x) >> BIT_SHIFT_FWFF_DMA_PKT_NUM) & BIT_MASK_FWFF_DMA_PKT_NUM)
+
+#define BIT_SHIFT_FWFF_STR_ADDR 0
+#define BIT_MASK_FWFF_STR_ADDR 0xffff
+#define BIT_FWFF_STR_ADDR(x)                                                   \
+	(((x) & BIT_MASK_FWFF_STR_ADDR) << BIT_SHIFT_FWFF_STR_ADDR)
+#define BIT_GET_FWFF_STR_ADDR(x)                                               \
+	(((x) >> BIT_SHIFT_FWFF_STR_ADDR) & BIT_MASK_FWFF_STR_ADDR)
+
+/* 2 REG_FWFF_PKT_INFO			(Offset 0x02A0) */
+
+#define BIT_SHIFT_FWFF_PKT_QUEUED 16
+#define BIT_MASK_FWFF_PKT_QUEUED 0xff
+#define BIT_FWFF_PKT_QUEUED(x)                                                 \
+	(((x) & BIT_MASK_FWFF_PKT_QUEUED) << BIT_SHIFT_FWFF_PKT_QUEUED)
+#define BIT_GET_FWFF_PKT_QUEUED(x)                                             \
+	(((x) >> BIT_SHIFT_FWFF_PKT_QUEUED) & BIT_MASK_FWFF_PKT_QUEUED)
+
+/* 2 REG_FWFF_PKT_INFO			(Offset 0x02A0) */
+
+#define BIT_SHIFT_FWFF_PKT_STR_ADDR 0
+#define BIT_MASK_FWFF_PKT_STR_ADDR 0xffff
+#define BIT_FWFF_PKT_STR_ADDR(x)                                               \
+	(((x) & BIT_MASK_FWFF_PKT_STR_ADDR) << BIT_SHIFT_FWFF_PKT_STR_ADDR)
+#define BIT_GET_FWFF_PKT_STR_ADDR(x)                                           \
+	(((x) >> BIT_SHIFT_FWFF_PKT_STR_ADDR) & BIT_MASK_FWFF_PKT_STR_ADDR)
+
+/* 2 REG_PCIE_CTRL				(Offset 0x0300) */
+
+#define BIT_PCIEIO_PERSTB_SEL BIT(31)
+
+/* 2 REG_PCIE_CTRL				(Offset 0x0300) */
+
+#define BIT_SHIFT_PCIE_MAX_RXDMA 28
+#define BIT_MASK_PCIE_MAX_RXDMA 0x7
+#define BIT_PCIE_MAX_RXDMA(x)                                                  \
+	(((x) & BIT_MASK_PCIE_MAX_RXDMA) << BIT_SHIFT_PCIE_MAX_RXDMA)
+#define BIT_GET_PCIE_MAX_RXDMA(x)                                              \
+	(((x) >> BIT_SHIFT_PCIE_MAX_RXDMA) & BIT_MASK_PCIE_MAX_RXDMA)
+
+/* 2 REG_PCIE_CTRL				(Offset 0x0300) */
+
+#define BIT_SHIFT_PCIE_MAX_TXDMA 24
+#define BIT_MASK_PCIE_MAX_TXDMA 0x7
+#define BIT_PCIE_MAX_TXDMA(x)                                                  \
+	(((x) & BIT_MASK_PCIE_MAX_TXDMA) << BIT_SHIFT_PCIE_MAX_TXDMA)
+#define BIT_GET_PCIE_MAX_TXDMA(x)                                              \
+	(((x) >> BIT_SHIFT_PCIE_MAX_TXDMA) & BIT_MASK_PCIE_MAX_TXDMA)
+
+/* 2 REG_PCIE_CTRL				(Offset 0x0300) */
+
+#define BIT_PCIE_RST_TRXDMA_INTF BIT(20)
+
+/* 2 REG_PCIE_CTRL				(Offset 0x0300) */
+
+#define BIT_PCIE_EN_SWENT_L23 BIT(17)
+
+/* 2 REG_PCIE_CTRL				(Offset 0x0300) */
+
+#define BIT_PCIE_EN_HWEXT_L1 BIT(16)
+
+/* 2 REG_INT_MIG				(Offset 0x0304) */
+
+#define BIT_SHIFT_TXTTIMER_MATCH_NUM 28
+#define BIT_MASK_TXTTIMER_MATCH_NUM 0xf
+#define BIT_TXTTIMER_MATCH_NUM(x)                                              \
+	(((x) & BIT_MASK_TXTTIMER_MATCH_NUM) << BIT_SHIFT_TXTTIMER_MATCH_NUM)
+#define BIT_GET_TXTTIMER_MATCH_NUM(x)                                          \
+	(((x) >> BIT_SHIFT_TXTTIMER_MATCH_NUM) & BIT_MASK_TXTTIMER_MATCH_NUM)
+
+#define BIT_SHIFT_TXPKT_NUM_MATCH 24
+#define BIT_MASK_TXPKT_NUM_MATCH 0xf
+#define BIT_TXPKT_NUM_MATCH(x)                                                 \
+	(((x) & BIT_MASK_TXPKT_NUM_MATCH) << BIT_SHIFT_TXPKT_NUM_MATCH)
+#define BIT_GET_TXPKT_NUM_MATCH(x)                                             \
+	(((x) >> BIT_SHIFT_TXPKT_NUM_MATCH) & BIT_MASK_TXPKT_NUM_MATCH)
+
+#define BIT_SHIFT_RXTTIMER_MATCH_NUM 20
+#define BIT_MASK_RXTTIMER_MATCH_NUM 0xf
+#define BIT_RXTTIMER_MATCH_NUM(x)                                              \
+	(((x) & BIT_MASK_RXTTIMER_MATCH_NUM) << BIT_SHIFT_RXTTIMER_MATCH_NUM)
+#define BIT_GET_RXTTIMER_MATCH_NUM(x)                                          \
+	(((x) >> BIT_SHIFT_RXTTIMER_MATCH_NUM) & BIT_MASK_RXTTIMER_MATCH_NUM)
+
+#define BIT_SHIFT_RXPKT_NUM_MATCH 16
+#define BIT_MASK_RXPKT_NUM_MATCH 0xf
+#define BIT_RXPKT_NUM_MATCH(x)                                                 \
+	(((x) & BIT_MASK_RXPKT_NUM_MATCH) << BIT_SHIFT_RXPKT_NUM_MATCH)
+#define BIT_GET_RXPKT_NUM_MATCH(x)                                             \
+	(((x) >> BIT_SHIFT_RXPKT_NUM_MATCH) & BIT_MASK_RXPKT_NUM_MATCH)
+
+#define BIT_SHIFT_MIGRATE_TIMER 0
+#define BIT_MASK_MIGRATE_TIMER 0xffff
+#define BIT_MIGRATE_TIMER(x)                                                   \
+	(((x) & BIT_MASK_MIGRATE_TIMER) << BIT_SHIFT_MIGRATE_TIMER)
+#define BIT_GET_MIGRATE_TIMER(x)                                               \
+	(((x) >> BIT_SHIFT_MIGRATE_TIMER) & BIT_MASK_MIGRATE_TIMER)
+
+/* 2 REG_BCNQ_TXBD_DESA			(Offset 0x0308) */
+
+#define BIT_SHIFT_BCNQ_TXBD_DESA 0
+#define BIT_MASK_BCNQ_TXBD_DESA 0xffffffffffffffffL
+#define BIT_BCNQ_TXBD_DESA(x)                                                  \
+	(((x) & BIT_MASK_BCNQ_TXBD_DESA) << BIT_SHIFT_BCNQ_TXBD_DESA)
+#define BIT_GET_BCNQ_TXBD_DESA(x)                                              \
+	(((x) >> BIT_SHIFT_BCNQ_TXBD_DESA) & BIT_MASK_BCNQ_TXBD_DESA)
+
+/* 2 REG_MGQ_TXBD_DESA			(Offset 0x0310) */
+
+#define BIT_SHIFT_MGQ_TXBD_DESA 0
+#define BIT_MASK_MGQ_TXBD_DESA 0xffffffffffffffffL
+#define BIT_MGQ_TXBD_DESA(x)                                                   \
+	(((x) & BIT_MASK_MGQ_TXBD_DESA) << BIT_SHIFT_MGQ_TXBD_DESA)
+#define BIT_GET_MGQ_TXBD_DESA(x)                                               \
+	(((x) >> BIT_SHIFT_MGQ_TXBD_DESA) & BIT_MASK_MGQ_TXBD_DESA)
+
+/* 2 REG_VOQ_TXBD_DESA			(Offset 0x0318) */
+
+#define BIT_SHIFT_VOQ_TXBD_DESA 0
+#define BIT_MASK_VOQ_TXBD_DESA 0xffffffffffffffffL
+#define BIT_VOQ_TXBD_DESA(x)                                                   \
+	(((x) & BIT_MASK_VOQ_TXBD_DESA) << BIT_SHIFT_VOQ_TXBD_DESA)
+#define BIT_GET_VOQ_TXBD_DESA(x)                                               \
+	(((x) >> BIT_SHIFT_VOQ_TXBD_DESA) & BIT_MASK_VOQ_TXBD_DESA)
+
+/* 2 REG_VIQ_TXBD_DESA			(Offset 0x0320) */
+
+#define BIT_SHIFT_VIQ_TXBD_DESA 0
+#define BIT_MASK_VIQ_TXBD_DESA 0xffffffffffffffffL
+#define BIT_VIQ_TXBD_DESA(x)                                                   \
+	(((x) & BIT_MASK_VIQ_TXBD_DESA) << BIT_SHIFT_VIQ_TXBD_DESA)
+#define BIT_GET_VIQ_TXBD_DESA(x)                                               \
+	(((x) >> BIT_SHIFT_VIQ_TXBD_DESA) & BIT_MASK_VIQ_TXBD_DESA)
+
+/* 2 REG_BEQ_TXBD_DESA			(Offset 0x0328) */
+
+#define BIT_SHIFT_BEQ_TXBD_DESA 0
+#define BIT_MASK_BEQ_TXBD_DESA 0xffffffffffffffffL
+#define BIT_BEQ_TXBD_DESA(x)                                                   \
+	(((x) & BIT_MASK_BEQ_TXBD_DESA) << BIT_SHIFT_BEQ_TXBD_DESA)
+#define BIT_GET_BEQ_TXBD_DESA(x)                                               \
+	(((x) >> BIT_SHIFT_BEQ_TXBD_DESA) & BIT_MASK_BEQ_TXBD_DESA)
+
+/* 2 REG_BKQ_TXBD_DESA			(Offset 0x0330) */
+
+#define BIT_SHIFT_BKQ_TXBD_DESA 0
+#define BIT_MASK_BKQ_TXBD_DESA 0xffffffffffffffffL
+#define BIT_BKQ_TXBD_DESA(x)                                                   \
+	(((x) & BIT_MASK_BKQ_TXBD_DESA) << BIT_SHIFT_BKQ_TXBD_DESA)
+#define BIT_GET_BKQ_TXBD_DESA(x)                                               \
+	(((x) >> BIT_SHIFT_BKQ_TXBD_DESA) & BIT_MASK_BKQ_TXBD_DESA)
+
+/* 2 REG_RXQ_RXBD_DESA			(Offset 0x0338) */
+
+#define BIT_SHIFT_RXQ_RXBD_DESA 0
+#define BIT_MASK_RXQ_RXBD_DESA 0xffffffffffffffffL
+#define BIT_RXQ_RXBD_DESA(x)                                                   \
+	(((x) & BIT_MASK_RXQ_RXBD_DESA) << BIT_SHIFT_RXQ_RXBD_DESA)
+#define BIT_GET_RXQ_RXBD_DESA(x)                                               \
+	(((x) >> BIT_SHIFT_RXQ_RXBD_DESA) & BIT_MASK_RXQ_RXBD_DESA)
+
+/* 2 REG_HI0Q_TXBD_DESA			(Offset 0x0340) */
+
+#define BIT_SHIFT_HI0Q_TXBD_DESA 0
+#define BIT_MASK_HI0Q_TXBD_DESA 0xffffffffffffffffL
+#define BIT_HI0Q_TXBD_DESA(x)                                                  \
+	(((x) & BIT_MASK_HI0Q_TXBD_DESA) << BIT_SHIFT_HI0Q_TXBD_DESA)
+#define BIT_GET_HI0Q_TXBD_DESA(x)                                              \
+	(((x) >> BIT_SHIFT_HI0Q_TXBD_DESA) & BIT_MASK_HI0Q_TXBD_DESA)
+
+/* 2 REG_HI1Q_TXBD_DESA			(Offset 0x0348) */
+
+#define BIT_SHIFT_HI1Q_TXBD_DESA 0
+#define BIT_MASK_HI1Q_TXBD_DESA 0xffffffffffffffffL
+#define BIT_HI1Q_TXBD_DESA(x)                                                  \
+	(((x) & BIT_MASK_HI1Q_TXBD_DESA) << BIT_SHIFT_HI1Q_TXBD_DESA)
+#define BIT_GET_HI1Q_TXBD_DESA(x)                                              \
+	(((x) >> BIT_SHIFT_HI1Q_TXBD_DESA) & BIT_MASK_HI1Q_TXBD_DESA)
+
+/* 2 REG_HI2Q_TXBD_DESA			(Offset 0x0350) */
+
+#define BIT_SHIFT_HI2Q_TXBD_DESA 0
+#define BIT_MASK_HI2Q_TXBD_DESA 0xffffffffffffffffL
+#define BIT_HI2Q_TXBD_DESA(x)                                                  \
+	(((x) & BIT_MASK_HI2Q_TXBD_DESA) << BIT_SHIFT_HI2Q_TXBD_DESA)
+#define BIT_GET_HI2Q_TXBD_DESA(x)                                              \
+	(((x) >> BIT_SHIFT_HI2Q_TXBD_DESA) & BIT_MASK_HI2Q_TXBD_DESA)
+
+/* 2 REG_HI3Q_TXBD_DESA			(Offset 0x0358) */
+
+#define BIT_SHIFT_HI3Q_TXBD_DESA 0
+#define BIT_MASK_HI3Q_TXBD_DESA 0xffffffffffffffffL
+#define BIT_HI3Q_TXBD_DESA(x)                                                  \
+	(((x) & BIT_MASK_HI3Q_TXBD_DESA) << BIT_SHIFT_HI3Q_TXBD_DESA)
+#define BIT_GET_HI3Q_TXBD_DESA(x)                                              \
+	(((x) >> BIT_SHIFT_HI3Q_TXBD_DESA) & BIT_MASK_HI3Q_TXBD_DESA)
+
+/* 2 REG_HI4Q_TXBD_DESA			(Offset 0x0360) */
+
+#define BIT_SHIFT_HI4Q_TXBD_DESA 0
+#define BIT_MASK_HI4Q_TXBD_DESA 0xffffffffffffffffL
+#define BIT_HI4Q_TXBD_DESA(x)                                                  \
+	(((x) & BIT_MASK_HI4Q_TXBD_DESA) << BIT_SHIFT_HI4Q_TXBD_DESA)
+#define BIT_GET_HI4Q_TXBD_DESA(x)                                              \
+	(((x) >> BIT_SHIFT_HI4Q_TXBD_DESA) & BIT_MASK_HI4Q_TXBD_DESA)
+
+/* 2 REG_HI5Q_TXBD_DESA			(Offset 0x0368) */
+
+#define BIT_SHIFT_HI5Q_TXBD_DESA 0
+#define BIT_MASK_HI5Q_TXBD_DESA 0xffffffffffffffffL
+#define BIT_HI5Q_TXBD_DESA(x)                                                  \
+	(((x) & BIT_MASK_HI5Q_TXBD_DESA) << BIT_SHIFT_HI5Q_TXBD_DESA)
+#define BIT_GET_HI5Q_TXBD_DESA(x)                                              \
+	(((x) >> BIT_SHIFT_HI5Q_TXBD_DESA) & BIT_MASK_HI5Q_TXBD_DESA)
+
+/* 2 REG_HI6Q_TXBD_DESA			(Offset 0x0370) */
+
+#define BIT_SHIFT_HI6Q_TXBD_DESA 0
+#define BIT_MASK_HI6Q_TXBD_DESA 0xffffffffffffffffL
+#define BIT_HI6Q_TXBD_DESA(x)                                                  \
+	(((x) & BIT_MASK_HI6Q_TXBD_DESA) << BIT_SHIFT_HI6Q_TXBD_DESA)
+#define BIT_GET_HI6Q_TXBD_DESA(x)                                              \
+	(((x) >> BIT_SHIFT_HI6Q_TXBD_DESA) & BIT_MASK_HI6Q_TXBD_DESA)
+
+/* 2 REG_HI7Q_TXBD_DESA			(Offset 0x0378) */
+
+#define BIT_SHIFT_HI7Q_TXBD_DESA 0
+#define BIT_MASK_HI7Q_TXBD_DESA 0xffffffffffffffffL
+#define BIT_HI7Q_TXBD_DESA(x)                                                  \
+	(((x) & BIT_MASK_HI7Q_TXBD_DESA) << BIT_SHIFT_HI7Q_TXBD_DESA)
+#define BIT_GET_HI7Q_TXBD_DESA(x)                                              \
+	(((x) >> BIT_SHIFT_HI7Q_TXBD_DESA) & BIT_MASK_HI7Q_TXBD_DESA)
+
+/* 2 REG_MGQ_TXBD_NUM			(Offset 0x0380) */
+
+#define BIT_PCIE_MGQ_FLAG BIT(14)
+
+/* 2 REG_MGQ_TXBD_NUM			(Offset 0x0380) */
+
+#define BIT_SHIFT_MGQ_DESC_MODE 12
+#define BIT_MASK_MGQ_DESC_MODE 0x3
+#define BIT_MGQ_DESC_MODE(x)                                                   \
+	(((x) & BIT_MASK_MGQ_DESC_MODE) << BIT_SHIFT_MGQ_DESC_MODE)
+#define BIT_GET_MGQ_DESC_MODE(x)                                               \
+	(((x) >> BIT_SHIFT_MGQ_DESC_MODE) & BIT_MASK_MGQ_DESC_MODE)
+
+#define BIT_SHIFT_MGQ_DESC_NUM 0
+#define BIT_MASK_MGQ_DESC_NUM 0xfff
+#define BIT_MGQ_DESC_NUM(x)                                                    \
+	(((x) & BIT_MASK_MGQ_DESC_NUM) << BIT_SHIFT_MGQ_DESC_NUM)
+#define BIT_GET_MGQ_DESC_NUM(x)                                                \
+	(((x) >> BIT_SHIFT_MGQ_DESC_NUM) & BIT_MASK_MGQ_DESC_NUM)
+
+/* 2 REG_RX_RXBD_NUM				(Offset 0x0382) */
+
+#define BIT_SYS_32_64 BIT(15)
+
+#define BIT_SHIFT_BCNQ_DESC_MODE 13
+#define BIT_MASK_BCNQ_DESC_MODE 0x3
+#define BIT_BCNQ_DESC_MODE(x)                                                  \
+	(((x) & BIT_MASK_BCNQ_DESC_MODE) << BIT_SHIFT_BCNQ_DESC_MODE)
+#define BIT_GET_BCNQ_DESC_MODE(x)                                              \
+	(((x) >> BIT_SHIFT_BCNQ_DESC_MODE) & BIT_MASK_BCNQ_DESC_MODE)
+
+/* 2 REG_RX_RXBD_NUM				(Offset 0x0382) */
+
+#define BIT_PCIE_BCNQ_FLAG BIT(12)
+
+/* 2 REG_RX_RXBD_NUM				(Offset 0x0382) */
+
+#define BIT_SHIFT_RXQ_DESC_NUM 0
+#define BIT_MASK_RXQ_DESC_NUM 0xfff
+#define BIT_RXQ_DESC_NUM(x)                                                    \
+	(((x) & BIT_MASK_RXQ_DESC_NUM) << BIT_SHIFT_RXQ_DESC_NUM)
+#define BIT_GET_RXQ_DESC_NUM(x)                                                \
+	(((x) >> BIT_SHIFT_RXQ_DESC_NUM) & BIT_MASK_RXQ_DESC_NUM)
+
+/* 2 REG_VOQ_TXBD_NUM			(Offset 0x0384) */
+
+#define BIT_PCIE_VOQ_FLAG BIT(14)
+
+/* 2 REG_VOQ_TXBD_NUM			(Offset 0x0384) */
+
+#define BIT_SHIFT_VOQ_DESC_MODE 12
+#define BIT_MASK_VOQ_DESC_MODE 0x3
+#define BIT_VOQ_DESC_MODE(x)                                                   \
+	(((x) & BIT_MASK_VOQ_DESC_MODE) << BIT_SHIFT_VOQ_DESC_MODE)
+#define BIT_GET_VOQ_DESC_MODE(x)                                               \
+	(((x) >> BIT_SHIFT_VOQ_DESC_MODE) & BIT_MASK_VOQ_DESC_MODE)
+
+#define BIT_SHIFT_VOQ_DESC_NUM 0
+#define BIT_MASK_VOQ_DESC_NUM 0xfff
+#define BIT_VOQ_DESC_NUM(x)                                                    \
+	(((x) & BIT_MASK_VOQ_DESC_NUM) << BIT_SHIFT_VOQ_DESC_NUM)
+#define BIT_GET_VOQ_DESC_NUM(x)                                                \
+	(((x) >> BIT_SHIFT_VOQ_DESC_NUM) & BIT_MASK_VOQ_DESC_NUM)
+
+/* 2 REG_VIQ_TXBD_NUM			(Offset 0x0386) */
+
+#define BIT_PCIE_VIQ_FLAG BIT(14)
+
+/* 2 REG_VIQ_TXBD_NUM			(Offset 0x0386) */
+
+#define BIT_SHIFT_VIQ_DESC_MODE 12
+#define BIT_MASK_VIQ_DESC_MODE 0x3
+#define BIT_VIQ_DESC_MODE(x)                                                   \
+	(((x) & BIT_MASK_VIQ_DESC_MODE) << BIT_SHIFT_VIQ_DESC_MODE)
+#define BIT_GET_VIQ_DESC_MODE(x)                                               \
+	(((x) >> BIT_SHIFT_VIQ_DESC_MODE) & BIT_MASK_VIQ_DESC_MODE)
+
+#define BIT_SHIFT_VIQ_DESC_NUM 0
+#define BIT_MASK_VIQ_DESC_NUM 0xfff
+#define BIT_VIQ_DESC_NUM(x)                                                    \
+	(((x) & BIT_MASK_VIQ_DESC_NUM) << BIT_SHIFT_VIQ_DESC_NUM)
+#define BIT_GET_VIQ_DESC_NUM(x)                                                \
+	(((x) >> BIT_SHIFT_VIQ_DESC_NUM) & BIT_MASK_VIQ_DESC_NUM)
+
+/* 2 REG_BEQ_TXBD_NUM			(Offset 0x0388) */
+
+#define BIT_PCIE_BEQ_FLAG BIT(14)
+
+/* 2 REG_BEQ_TXBD_NUM			(Offset 0x0388) */
+
+#define BIT_SHIFT_BEQ_DESC_MODE 12
+#define BIT_MASK_BEQ_DESC_MODE 0x3
+#define BIT_BEQ_DESC_MODE(x)                                                   \
+	(((x) & BIT_MASK_BEQ_DESC_MODE) << BIT_SHIFT_BEQ_DESC_MODE)
+#define BIT_GET_BEQ_DESC_MODE(x)                                               \
+	(((x) >> BIT_SHIFT_BEQ_DESC_MODE) & BIT_MASK_BEQ_DESC_MODE)
+
+#define BIT_SHIFT_BEQ_DESC_NUM 0
+#define BIT_MASK_BEQ_DESC_NUM 0xfff
+#define BIT_BEQ_DESC_NUM(x)                                                    \
+	(((x) & BIT_MASK_BEQ_DESC_NUM) << BIT_SHIFT_BEQ_DESC_NUM)
+#define BIT_GET_BEQ_DESC_NUM(x)                                                \
+	(((x) >> BIT_SHIFT_BEQ_DESC_NUM) & BIT_MASK_BEQ_DESC_NUM)
+
+/* 2 REG_BKQ_TXBD_NUM			(Offset 0x038A) */
+
+#define BIT_PCIE_BKQ_FLAG BIT(14)
+
+/* 2 REG_BKQ_TXBD_NUM			(Offset 0x038A) */
+
+#define BIT_SHIFT_BKQ_DESC_MODE 12
+#define BIT_MASK_BKQ_DESC_MODE 0x3
+#define BIT_BKQ_DESC_MODE(x)                                                   \
+	(((x) & BIT_MASK_BKQ_DESC_MODE) << BIT_SHIFT_BKQ_DESC_MODE)
+#define BIT_GET_BKQ_DESC_MODE(x)                                               \
+	(((x) >> BIT_SHIFT_BKQ_DESC_MODE) & BIT_MASK_BKQ_DESC_MODE)
+
+#define BIT_SHIFT_BKQ_DESC_NUM 0
+#define BIT_MASK_BKQ_DESC_NUM 0xfff
+#define BIT_BKQ_DESC_NUM(x)                                                    \
+	(((x) & BIT_MASK_BKQ_DESC_NUM) << BIT_SHIFT_BKQ_DESC_NUM)
+#define BIT_GET_BKQ_DESC_NUM(x)                                                \
+	(((x) >> BIT_SHIFT_BKQ_DESC_NUM) & BIT_MASK_BKQ_DESC_NUM)
+
+/* 2 REG_HI0Q_TXBD_NUM			(Offset 0x038C) */
+
+#define BIT_HI0Q_FLAG BIT(14)
+
+#define BIT_SHIFT_HI0Q_DESC_MODE 12
+#define BIT_MASK_HI0Q_DESC_MODE 0x3
+#define BIT_HI0Q_DESC_MODE(x)                                                  \
+	(((x) & BIT_MASK_HI0Q_DESC_MODE) << BIT_SHIFT_HI0Q_DESC_MODE)
+#define BIT_GET_HI0Q_DESC_MODE(x)                                              \
+	(((x) >> BIT_SHIFT_HI0Q_DESC_MODE) & BIT_MASK_HI0Q_DESC_MODE)
+
+#define BIT_SHIFT_HI0Q_DESC_NUM 0
+#define BIT_MASK_HI0Q_DESC_NUM 0xfff
+#define BIT_HI0Q_DESC_NUM(x)                                                   \
+	(((x) & BIT_MASK_HI0Q_DESC_NUM) << BIT_SHIFT_HI0Q_DESC_NUM)
+#define BIT_GET_HI0Q_DESC_NUM(x)                                               \
+	(((x) >> BIT_SHIFT_HI0Q_DESC_NUM) & BIT_MASK_HI0Q_DESC_NUM)
+
+/* 2 REG_HI1Q_TXBD_NUM			(Offset 0x038E) */
+
+#define BIT_HI1Q_FLAG BIT(14)
+
+#define BIT_SHIFT_HI1Q_DESC_MODE 12
+#define BIT_MASK_HI1Q_DESC_MODE 0x3
+#define BIT_HI1Q_DESC_MODE(x)                                                  \
+	(((x) & BIT_MASK_HI1Q_DESC_MODE) << BIT_SHIFT_HI1Q_DESC_MODE)
+#define BIT_GET_HI1Q_DESC_MODE(x)                                              \
+	(((x) >> BIT_SHIFT_HI1Q_DESC_MODE) & BIT_MASK_HI1Q_DESC_MODE)
+
+#define BIT_SHIFT_HI1Q_DESC_NUM 0
+#define BIT_MASK_HI1Q_DESC_NUM 0xfff
+#define BIT_HI1Q_DESC_NUM(x)                                                   \
+	(((x) & BIT_MASK_HI1Q_DESC_NUM) << BIT_SHIFT_HI1Q_DESC_NUM)
+#define BIT_GET_HI1Q_DESC_NUM(x)                                               \
+	(((x) >> BIT_SHIFT_HI1Q_DESC_NUM) & BIT_MASK_HI1Q_DESC_NUM)
+
+/* 2 REG_HI2Q_TXBD_NUM			(Offset 0x0390) */
+
+#define BIT_HI2Q_FLAG BIT(14)
+
+#define BIT_SHIFT_HI2Q_DESC_MODE 12
+#define BIT_MASK_HI2Q_DESC_MODE 0x3
+#define BIT_HI2Q_DESC_MODE(x)                                                  \
+	(((x) & BIT_MASK_HI2Q_DESC_MODE) << BIT_SHIFT_HI2Q_DESC_MODE)
+#define BIT_GET_HI2Q_DESC_MODE(x)                                              \
+	(((x) >> BIT_SHIFT_HI2Q_DESC_MODE) & BIT_MASK_HI2Q_DESC_MODE)
+
+#define BIT_SHIFT_HI2Q_DESC_NUM 0
+#define BIT_MASK_HI2Q_DESC_NUM 0xfff
+#define BIT_HI2Q_DESC_NUM(x)                                                   \
+	(((x) & BIT_MASK_HI2Q_DESC_NUM) << BIT_SHIFT_HI2Q_DESC_NUM)
+#define BIT_GET_HI2Q_DESC_NUM(x)                                               \
+	(((x) >> BIT_SHIFT_HI2Q_DESC_NUM) & BIT_MASK_HI2Q_DESC_NUM)
+
+/* 2 REG_HI3Q_TXBD_NUM			(Offset 0x0392) */
+
+#define BIT_HI3Q_FLAG BIT(14)
+
+#define BIT_SHIFT_HI3Q_DESC_MODE 12
+#define BIT_MASK_HI3Q_DESC_MODE 0x3
+#define BIT_HI3Q_DESC_MODE(x)                                                  \
+	(((x) & BIT_MASK_HI3Q_DESC_MODE) << BIT_SHIFT_HI3Q_DESC_MODE)
+#define BIT_GET_HI3Q_DESC_MODE(x)                                              \
+	(((x) >> BIT_SHIFT_HI3Q_DESC_MODE) & BIT_MASK_HI3Q_DESC_MODE)
+
+#define BIT_SHIFT_HI3Q_DESC_NUM 0
+#define BIT_MASK_HI3Q_DESC_NUM 0xfff
+#define BIT_HI3Q_DESC_NUM(x)                                                   \
+	(((x) & BIT_MASK_HI3Q_DESC_NUM) << BIT_SHIFT_HI3Q_DESC_NUM)
+#define BIT_GET_HI3Q_DESC_NUM(x)                                               \
+	(((x) >> BIT_SHIFT_HI3Q_DESC_NUM) & BIT_MASK_HI3Q_DESC_NUM)
+
+/* 2 REG_HI4Q_TXBD_NUM			(Offset 0x0394) */
+
+#define BIT_HI4Q_FLAG BIT(14)
+
+#define BIT_SHIFT_HI4Q_DESC_MODE 12
+#define BIT_MASK_HI4Q_DESC_MODE 0x3
+#define BIT_HI4Q_DESC_MODE(x)                                                  \
+	(((x) & BIT_MASK_HI4Q_DESC_MODE) << BIT_SHIFT_HI4Q_DESC_MODE)
+#define BIT_GET_HI4Q_DESC_MODE(x)                                              \
+	(((x) >> BIT_SHIFT_HI4Q_DESC_MODE) & BIT_MASK_HI4Q_DESC_MODE)
+
+#define BIT_SHIFT_HI4Q_DESC_NUM 0
+#define BIT_MASK_HI4Q_DESC_NUM 0xfff
+#define BIT_HI4Q_DESC_NUM(x)                                                   \
+	(((x) & BIT_MASK_HI4Q_DESC_NUM) << BIT_SHIFT_HI4Q_DESC_NUM)
+#define BIT_GET_HI4Q_DESC_NUM(x)                                               \
+	(((x) >> BIT_SHIFT_HI4Q_DESC_NUM) & BIT_MASK_HI4Q_DESC_NUM)
+
+/* 2 REG_HI5Q_TXBD_NUM			(Offset 0x0396) */
+
+#define BIT_HI5Q_FLAG BIT(14)
+
+#define BIT_SHIFT_HI5Q_DESC_MODE 12
+#define BIT_MASK_HI5Q_DESC_MODE 0x3
+#define BIT_HI5Q_DESC_MODE(x)                                                  \
+	(((x) & BIT_MASK_HI5Q_DESC_MODE) << BIT_SHIFT_HI5Q_DESC_MODE)
+#define BIT_GET_HI5Q_DESC_MODE(x)                                              \
+	(((x) >> BIT_SHIFT_HI5Q_DESC_MODE) & BIT_MASK_HI5Q_DESC_MODE)
+
+#define BIT_SHIFT_HI5Q_DESC_NUM 0
+#define BIT_MASK_HI5Q_DESC_NUM 0xfff
+#define BIT_HI5Q_DESC_NUM(x)                                                   \
+	(((x) & BIT_MASK_HI5Q_DESC_NUM) << BIT_SHIFT_HI5Q_DESC_NUM)
+#define BIT_GET_HI5Q_DESC_NUM(x)                                               \
+	(((x) >> BIT_SHIFT_HI5Q_DESC_NUM) & BIT_MASK_HI5Q_DESC_NUM)
+
+/* 2 REG_HI6Q_TXBD_NUM			(Offset 0x0398) */
+
+#define BIT_HI6Q_FLAG BIT(14)
+
+#define BIT_SHIFT_HI6Q_DESC_MODE 12
+#define BIT_MASK_HI6Q_DESC_MODE 0x3
+#define BIT_HI6Q_DESC_MODE(x)                                                  \
+	(((x) & BIT_MASK_HI6Q_DESC_MODE) << BIT_SHIFT_HI6Q_DESC_MODE)
+#define BIT_GET_HI6Q_DESC_MODE(x)                                              \
+	(((x) >> BIT_SHIFT_HI6Q_DESC_MODE) & BIT_MASK_HI6Q_DESC_MODE)
+
+#define BIT_SHIFT_HI6Q_DESC_NUM 0
+#define BIT_MASK_HI6Q_DESC_NUM 0xfff
+#define BIT_HI6Q_DESC_NUM(x)                                                   \
+	(((x) & BIT_MASK_HI6Q_DESC_NUM) << BIT_SHIFT_HI6Q_DESC_NUM)
+#define BIT_GET_HI6Q_DESC_NUM(x)                                               \
+	(((x) >> BIT_SHIFT_HI6Q_DESC_NUM) & BIT_MASK_HI6Q_DESC_NUM)
+
+/* 2 REG_HI7Q_TXBD_NUM			(Offset 0x039A) */
+
+#define BIT_HI7Q_FLAG BIT(14)
+
+#define BIT_SHIFT_HI7Q_DESC_MODE 12
+#define BIT_MASK_HI7Q_DESC_MODE 0x3
+#define BIT_HI7Q_DESC_MODE(x)                                                  \
+	(((x) & BIT_MASK_HI7Q_DESC_MODE) << BIT_SHIFT_HI7Q_DESC_MODE)
+#define BIT_GET_HI7Q_DESC_MODE(x)                                              \
+	(((x) >> BIT_SHIFT_HI7Q_DESC_MODE) & BIT_MASK_HI7Q_DESC_MODE)
+
+#define BIT_SHIFT_HI7Q_DESC_NUM 0
+#define BIT_MASK_HI7Q_DESC_NUM 0xfff
+#define BIT_HI7Q_DESC_NUM(x)                                                   \
+	(((x) & BIT_MASK_HI7Q_DESC_NUM) << BIT_SHIFT_HI7Q_DESC_NUM)
+#define BIT_GET_HI7Q_DESC_NUM(x)                                               \
+	(((x) >> BIT_SHIFT_HI7Q_DESC_NUM) & BIT_MASK_HI7Q_DESC_NUM)
+
+/* 2 REG_BD_RWPTR_CLR			(Offset 0x039C) */
+
+#define BIT_CLR_HI7Q_HW_IDX BIT(29)
+#define BIT_CLR_HI6Q_HW_IDX BIT(28)
+#define BIT_CLR_HI5Q_HW_IDX BIT(27)
+#define BIT_CLR_HI4Q_HW_IDX BIT(26)
+#define BIT_CLR_HI3Q_HW_IDX BIT(25)
+#define BIT_CLR_HI2Q_HW_IDX BIT(24)
+#define BIT_CLR_HI1Q_HW_IDX BIT(23)
+
+/* 2 REG_BD_RWPTR_CLR			(Offset 0x039C) */
+
+#define BIT_CLR_HI0Q_HW_IDX BIT(22)
+
+/* 2 REG_BD_RWPTR_CLR			(Offset 0x039C) */
+
+#define BIT_CLR_BKQ_HW_IDX BIT(21)
+
+/* 2 REG_BD_RWPTR_CLR			(Offset 0x039C) */
+
+#define BIT_CLR_BEQ_HW_IDX BIT(20)
+
+/* 2 REG_BD_RWPTR_CLR			(Offset 0x039C) */
+
+#define BIT_CLR_VIQ_HW_IDX BIT(19)
+
+/* 2 REG_BD_RWPTR_CLR			(Offset 0x039C) */
+
+#define BIT_CLR_VOQ_HW_IDX BIT(18)
+
+/* 2 REG_BD_RWPTR_CLR			(Offset 0x039C) */
+
+#define BIT_CLR_MGQ_HW_IDX BIT(17)
+
+/* 2 REG_TSFTIMER_HCI			(Offset 0x039C) */
+
+#define BIT_SHIFT_TSFT2_HCI 16
+#define BIT_MASK_TSFT2_HCI 0xffff
+#define BIT_TSFT2_HCI(x) (((x) & BIT_MASK_TSFT2_HCI) << BIT_SHIFT_TSFT2_HCI)
+#define BIT_GET_TSFT2_HCI(x) (((x) >> BIT_SHIFT_TSFT2_HCI) & BIT_MASK_TSFT2_HCI)
+
+#define BIT_CLR_RXQ_HW_IDX BIT(16)
+
+/* 2 REG_BD_RWPTR_CLR			(Offset 0x039C) */
+
+#define BIT_CLR_HI7Q_HOST_IDX BIT(13)
+
+/* 2 REG_BD_RWPTR_CLR			(Offset 0x039C) */
+
+#define BIT_CLR_HI6Q_HOST_IDX BIT(12)
+
+/* 2 REG_BD_RWPTR_CLR			(Offset 0x039C) */
+
+#define BIT_CLR_HI5Q_HOST_IDX BIT(11)
+
+/* 2 REG_BD_RWPTR_CLR			(Offset 0x039C) */
+
+#define BIT_CLR_HI4Q_HOST_IDX BIT(10)
+
+/* 2 REG_BD_RWPTR_CLR			(Offset 0x039C) */
+
+#define BIT_CLR_HI3Q_HOST_IDX BIT(9)
+
+/* 2 REG_BD_RWPTR_CLR			(Offset 0x039C) */
+
+#define BIT_CLR_HI2Q_HOST_IDX BIT(8)
+
+/* 2 REG_BD_RWPTR_CLR			(Offset 0x039C) */
+
+#define BIT_CLR_HI1Q_HOST_IDX BIT(7)
+#define BIT_CLR_HI0Q_HOST_IDX BIT(6)
+
+/* 2 REG_BD_RWPTR_CLR			(Offset 0x039C) */
+
+#define BIT_CLR_BKQ_HOST_IDX BIT(5)
+
+/* 2 REG_BD_RWPTR_CLR			(Offset 0x039C) */
+
+#define BIT_CLR_BEQ_HOST_IDX BIT(4)
+
+/* 2 REG_BD_RWPTR_CLR			(Offset 0x039C) */
+
+#define BIT_CLR_VIQ_HOST_IDX BIT(3)
+
+/* 2 REG_BD_RWPTR_CLR			(Offset 0x039C) */
+
+#define BIT_CLR_VOQ_HOST_IDX BIT(2)
+
+/* 2 REG_BD_RWPTR_CLR			(Offset 0x039C) */
+
+#define BIT_CLR_MGQ_HOST_IDX BIT(1)
+
+/* 2 REG_TSFTIMER_HCI			(Offset 0x039C) */
+
+#define BIT_SHIFT_TSFT1_HCI 0
+#define BIT_MASK_TSFT1_HCI 0xffff
+#define BIT_TSFT1_HCI(x) (((x) & BIT_MASK_TSFT1_HCI) << BIT_SHIFT_TSFT1_HCI)
+#define BIT_GET_TSFT1_HCI(x) (((x) >> BIT_SHIFT_TSFT1_HCI) & BIT_MASK_TSFT1_HCI)
+
+#define BIT_CLR_RXQ_HOST_IDX BIT(0)
+
+/* 2 REG_VOQ_TXBD_IDX			(Offset 0x03A0) */
+
+#define BIT_SHIFT_VOQ_HW_IDX 16
+#define BIT_MASK_VOQ_HW_IDX 0xfff
+#define BIT_VOQ_HW_IDX(x) (((x) & BIT_MASK_VOQ_HW_IDX) << BIT_SHIFT_VOQ_HW_IDX)
+#define BIT_GET_VOQ_HW_IDX(x)                                                  \
+	(((x) >> BIT_SHIFT_VOQ_HW_IDX) & BIT_MASK_VOQ_HW_IDX)
+
+#define BIT_SHIFT_VOQ_HOST_IDX 0
+#define BIT_MASK_VOQ_HOST_IDX 0xfff
+#define BIT_VOQ_HOST_IDX(x)                                                    \
+	(((x) & BIT_MASK_VOQ_HOST_IDX) << BIT_SHIFT_VOQ_HOST_IDX)
+#define BIT_GET_VOQ_HOST_IDX(x)                                                \
+	(((x) >> BIT_SHIFT_VOQ_HOST_IDX) & BIT_MASK_VOQ_HOST_IDX)
+
+/* 2 REG_VIQ_TXBD_IDX			(Offset 0x03A4) */
+
+#define BIT_SHIFT_VIQ_HW_IDX 16
+#define BIT_MASK_VIQ_HW_IDX 0xfff
+#define BIT_VIQ_HW_IDX(x) (((x) & BIT_MASK_VIQ_HW_IDX) << BIT_SHIFT_VIQ_HW_IDX)
+#define BIT_GET_VIQ_HW_IDX(x)                                                  \
+	(((x) >> BIT_SHIFT_VIQ_HW_IDX) & BIT_MASK_VIQ_HW_IDX)
+
+#define BIT_SHIFT_VIQ_HOST_IDX 0
+#define BIT_MASK_VIQ_HOST_IDX 0xfff
+#define BIT_VIQ_HOST_IDX(x)                                                    \
+	(((x) & BIT_MASK_VIQ_HOST_IDX) << BIT_SHIFT_VIQ_HOST_IDX)
+#define BIT_GET_VIQ_HOST_IDX(x)                                                \
+	(((x) >> BIT_SHIFT_VIQ_HOST_IDX) & BIT_MASK_VIQ_HOST_IDX)
+
+/* 2 REG_BEQ_TXBD_IDX			(Offset 0x03A8) */
+
+#define BIT_SHIFT_BEQ_HW_IDX 16
+#define BIT_MASK_BEQ_HW_IDX 0xfff
+#define BIT_BEQ_HW_IDX(x) (((x) & BIT_MASK_BEQ_HW_IDX) << BIT_SHIFT_BEQ_HW_IDX)
+#define BIT_GET_BEQ_HW_IDX(x)                                                  \
+	(((x) >> BIT_SHIFT_BEQ_HW_IDX) & BIT_MASK_BEQ_HW_IDX)
+
+#define BIT_SHIFT_BEQ_HOST_IDX 0
+#define BIT_MASK_BEQ_HOST_IDX 0xfff
+#define BIT_BEQ_HOST_IDX(x)                                                    \
+	(((x) & BIT_MASK_BEQ_HOST_IDX) << BIT_SHIFT_BEQ_HOST_IDX)
+#define BIT_GET_BEQ_HOST_IDX(x)                                                \
+	(((x) >> BIT_SHIFT_BEQ_HOST_IDX) & BIT_MASK_BEQ_HOST_IDX)
+
+/* 2 REG_BKQ_TXBD_IDX			(Offset 0x03AC) */
+
+#define BIT_SHIFT_BKQ_HW_IDX 16
+#define BIT_MASK_BKQ_HW_IDX 0xfff
+#define BIT_BKQ_HW_IDX(x) (((x) & BIT_MASK_BKQ_HW_IDX) << BIT_SHIFT_BKQ_HW_IDX)
+#define BIT_GET_BKQ_HW_IDX(x)                                                  \
+	(((x) >> BIT_SHIFT_BKQ_HW_IDX) & BIT_MASK_BKQ_HW_IDX)
+
+#define BIT_SHIFT_BKQ_HOST_IDX 0
+#define BIT_MASK_BKQ_HOST_IDX 0xfff
+#define BIT_BKQ_HOST_IDX(x)                                                    \
+	(((x) & BIT_MASK_BKQ_HOST_IDX) << BIT_SHIFT_BKQ_HOST_IDX)
+#define BIT_GET_BKQ_HOST_IDX(x)                                                \
+	(((x) >> BIT_SHIFT_BKQ_HOST_IDX) & BIT_MASK_BKQ_HOST_IDX)
+
+/* 2 REG_MGQ_TXBD_IDX			(Offset 0x03B0) */
+
+#define BIT_SHIFT_MGQ_HW_IDX 16
+#define BIT_MASK_MGQ_HW_IDX 0xfff
+#define BIT_MGQ_HW_IDX(x) (((x) & BIT_MASK_MGQ_HW_IDX) << BIT_SHIFT_MGQ_HW_IDX)
+#define BIT_GET_MGQ_HW_IDX(x)                                                  \
+	(((x) >> BIT_SHIFT_MGQ_HW_IDX) & BIT_MASK_MGQ_HW_IDX)
+
+#define BIT_SHIFT_MGQ_HOST_IDX 0
+#define BIT_MASK_MGQ_HOST_IDX 0xfff
+#define BIT_MGQ_HOST_IDX(x)                                                    \
+	(((x) & BIT_MASK_MGQ_HOST_IDX) << BIT_SHIFT_MGQ_HOST_IDX)
+#define BIT_GET_MGQ_HOST_IDX(x)                                                \
+	(((x) >> BIT_SHIFT_MGQ_HOST_IDX) & BIT_MASK_MGQ_HOST_IDX)
+
+/* 2 REG_RXQ_RXBD_IDX			(Offset 0x03B4) */
+
+#define BIT_SHIFT_RXQ_HW_IDX 16
+#define BIT_MASK_RXQ_HW_IDX 0xfff
+#define BIT_RXQ_HW_IDX(x) (((x) & BIT_MASK_RXQ_HW_IDX) << BIT_SHIFT_RXQ_HW_IDX)
+#define BIT_GET_RXQ_HW_IDX(x)                                                  \
+	(((x) >> BIT_SHIFT_RXQ_HW_IDX) & BIT_MASK_RXQ_HW_IDX)
+
+#define BIT_SHIFT_RXQ_HOST_IDX 0
+#define BIT_MASK_RXQ_HOST_IDX 0xfff
+#define BIT_RXQ_HOST_IDX(x)                                                    \
+	(((x) & BIT_MASK_RXQ_HOST_IDX) << BIT_SHIFT_RXQ_HOST_IDX)
+#define BIT_GET_RXQ_HOST_IDX(x)                                                \
+	(((x) >> BIT_SHIFT_RXQ_HOST_IDX) & BIT_MASK_RXQ_HOST_IDX)
+
+/* 2 REG_HI0Q_TXBD_IDX			(Offset 0x03B8) */
+
+#define BIT_SHIFT_HI0Q_HW_IDX 16
+#define BIT_MASK_HI0Q_HW_IDX 0xfff
+#define BIT_HI0Q_HW_IDX(x)                                                     \
+	(((x) & BIT_MASK_HI0Q_HW_IDX) << BIT_SHIFT_HI0Q_HW_IDX)
+#define BIT_GET_HI0Q_HW_IDX(x)                                                 \
+	(((x) >> BIT_SHIFT_HI0Q_HW_IDX) & BIT_MASK_HI0Q_HW_IDX)
+
+#define BIT_SHIFT_HI0Q_HOST_IDX 0
+#define BIT_MASK_HI0Q_HOST_IDX 0xfff
+#define BIT_HI0Q_HOST_IDX(x)                                                   \
+	(((x) & BIT_MASK_HI0Q_HOST_IDX) << BIT_SHIFT_HI0Q_HOST_IDX)
+#define BIT_GET_HI0Q_HOST_IDX(x)                                               \
+	(((x) >> BIT_SHIFT_HI0Q_HOST_IDX) & BIT_MASK_HI0Q_HOST_IDX)
+
+/* 2 REG_HI1Q_TXBD_IDX			(Offset 0x03BC) */
+
+#define BIT_SHIFT_HI1Q_HW_IDX 16
+#define BIT_MASK_HI1Q_HW_IDX 0xfff
+#define BIT_HI1Q_HW_IDX(x)                                                     \
+	(((x) & BIT_MASK_HI1Q_HW_IDX) << BIT_SHIFT_HI1Q_HW_IDX)
+#define BIT_GET_HI1Q_HW_IDX(x)                                                 \
+	(((x) >> BIT_SHIFT_HI1Q_HW_IDX) & BIT_MASK_HI1Q_HW_IDX)
+
+#define BIT_SHIFT_HI1Q_HOST_IDX 0
+#define BIT_MASK_HI1Q_HOST_IDX 0xfff
+#define BIT_HI1Q_HOST_IDX(x)                                                   \
+	(((x) & BIT_MASK_HI1Q_HOST_IDX) << BIT_SHIFT_HI1Q_HOST_IDX)
+#define BIT_GET_HI1Q_HOST_IDX(x)                                               \
+	(((x) >> BIT_SHIFT_HI1Q_HOST_IDX) & BIT_MASK_HI1Q_HOST_IDX)
+
+/* 2 REG_HI2Q_TXBD_IDX			(Offset 0x03C0) */
+
+#define BIT_SHIFT_HI2Q_HW_IDX 16
+#define BIT_MASK_HI2Q_HW_IDX 0xfff
+#define BIT_HI2Q_HW_IDX(x)                                                     \
+	(((x) & BIT_MASK_HI2Q_HW_IDX) << BIT_SHIFT_HI2Q_HW_IDX)
+#define BIT_GET_HI2Q_HW_IDX(x)                                                 \
+	(((x) >> BIT_SHIFT_HI2Q_HW_IDX) & BIT_MASK_HI2Q_HW_IDX)
+
+#define BIT_SHIFT_HI2Q_HOST_IDX 0
+#define BIT_MASK_HI2Q_HOST_IDX 0xfff
+#define BIT_HI2Q_HOST_IDX(x)                                                   \
+	(((x) & BIT_MASK_HI2Q_HOST_IDX) << BIT_SHIFT_HI2Q_HOST_IDX)
+#define BIT_GET_HI2Q_HOST_IDX(x)                                               \
+	(((x) >> BIT_SHIFT_HI2Q_HOST_IDX) & BIT_MASK_HI2Q_HOST_IDX)
+
+/* 2 REG_HI3Q_TXBD_IDX			(Offset 0x03C4) */
+
+#define BIT_SHIFT_HI3Q_HW_IDX 16
+#define BIT_MASK_HI3Q_HW_IDX 0xfff
+#define BIT_HI3Q_HW_IDX(x)                                                     \
+	(((x) & BIT_MASK_HI3Q_HW_IDX) << BIT_SHIFT_HI3Q_HW_IDX)
+#define BIT_GET_HI3Q_HW_IDX(x)                                                 \
+	(((x) >> BIT_SHIFT_HI3Q_HW_IDX) & BIT_MASK_HI3Q_HW_IDX)
+
+#define BIT_SHIFT_HI3Q_HOST_IDX 0
+#define BIT_MASK_HI3Q_HOST_IDX 0xfff
+#define BIT_HI3Q_HOST_IDX(x)                                                   \
+	(((x) & BIT_MASK_HI3Q_HOST_IDX) << BIT_SHIFT_HI3Q_HOST_IDX)
+#define BIT_GET_HI3Q_HOST_IDX(x)                                               \
+	(((x) >> BIT_SHIFT_HI3Q_HOST_IDX) & BIT_MASK_HI3Q_HOST_IDX)
+
+/* 2 REG_HI4Q_TXBD_IDX			(Offset 0x03C8) */
+
+#define BIT_SHIFT_HI4Q_HW_IDX 16
+#define BIT_MASK_HI4Q_HW_IDX 0xfff
+#define BIT_HI4Q_HW_IDX(x)                                                     \
+	(((x) & BIT_MASK_HI4Q_HW_IDX) << BIT_SHIFT_HI4Q_HW_IDX)
+#define BIT_GET_HI4Q_HW_IDX(x)                                                 \
+	(((x) >> BIT_SHIFT_HI4Q_HW_IDX) & BIT_MASK_HI4Q_HW_IDX)
+
+#define BIT_SHIFT_HI4Q_HOST_IDX 0
+#define BIT_MASK_HI4Q_HOST_IDX 0xfff
+#define BIT_HI4Q_HOST_IDX(x)                                                   \
+	(((x) & BIT_MASK_HI4Q_HOST_IDX) << BIT_SHIFT_HI4Q_HOST_IDX)
+#define BIT_GET_HI4Q_HOST_IDX(x)                                               \
+	(((x) >> BIT_SHIFT_HI4Q_HOST_IDX) & BIT_MASK_HI4Q_HOST_IDX)
+
+/* 2 REG_HI5Q_TXBD_IDX			(Offset 0x03CC) */
+
+#define BIT_SHIFT_HI5Q_HW_IDX 16
+#define BIT_MASK_HI5Q_HW_IDX 0xfff
+#define BIT_HI5Q_HW_IDX(x)                                                     \
+	(((x) & BIT_MASK_HI5Q_HW_IDX) << BIT_SHIFT_HI5Q_HW_IDX)
+#define BIT_GET_HI5Q_HW_IDX(x)                                                 \
+	(((x) >> BIT_SHIFT_HI5Q_HW_IDX) & BIT_MASK_HI5Q_HW_IDX)
+
+#define BIT_SHIFT_HI5Q_HOST_IDX 0
+#define BIT_MASK_HI5Q_HOST_IDX 0xfff
+#define BIT_HI5Q_HOST_IDX(x)                                                   \
+	(((x) & BIT_MASK_HI5Q_HOST_IDX) << BIT_SHIFT_HI5Q_HOST_IDX)
+#define BIT_GET_HI5Q_HOST_IDX(x)                                               \
+	(((x) >> BIT_SHIFT_HI5Q_HOST_IDX) & BIT_MASK_HI5Q_HOST_IDX)
+
+/* 2 REG_HI6Q_TXBD_IDX			(Offset 0x03D0) */
+
+#define BIT_SHIFT_HI6Q_HW_IDX 16
+#define BIT_MASK_HI6Q_HW_IDX 0xfff
+#define BIT_HI6Q_HW_IDX(x)                                                     \
+	(((x) & BIT_MASK_HI6Q_HW_IDX) << BIT_SHIFT_HI6Q_HW_IDX)
+#define BIT_GET_HI6Q_HW_IDX(x)                                                 \
+	(((x) >> BIT_SHIFT_HI6Q_HW_IDX) & BIT_MASK_HI6Q_HW_IDX)
+
+#define BIT_SHIFT_HI6Q_HOST_IDX 0
+#define BIT_MASK_HI6Q_HOST_IDX 0xfff
+#define BIT_HI6Q_HOST_IDX(x)                                                   \
+	(((x) & BIT_MASK_HI6Q_HOST_IDX) << BIT_SHIFT_HI6Q_HOST_IDX)
+#define BIT_GET_HI6Q_HOST_IDX(x)                                               \
+	(((x) >> BIT_SHIFT_HI6Q_HOST_IDX) & BIT_MASK_HI6Q_HOST_IDX)
+
+/* 2 REG_HI7Q_TXBD_IDX			(Offset 0x03D4) */
+
+#define BIT_SHIFT_HI7Q_HW_IDX 16
+#define BIT_MASK_HI7Q_HW_IDX 0xfff
+#define BIT_HI7Q_HW_IDX(x)                                                     \
+	(((x) & BIT_MASK_HI7Q_HW_IDX) << BIT_SHIFT_HI7Q_HW_IDX)
+#define BIT_GET_HI7Q_HW_IDX(x)                                                 \
+	(((x) >> BIT_SHIFT_HI7Q_HW_IDX) & BIT_MASK_HI7Q_HW_IDX)
+
+#define BIT_SHIFT_HI7Q_HOST_IDX 0
+#define BIT_MASK_HI7Q_HOST_IDX 0xfff
+#define BIT_HI7Q_HOST_IDX(x)                                                   \
+	(((x) & BIT_MASK_HI7Q_HOST_IDX) << BIT_SHIFT_HI7Q_HOST_IDX)
+#define BIT_GET_HI7Q_HOST_IDX(x)                                               \
+	(((x) >> BIT_SHIFT_HI7Q_HOST_IDX) & BIT_MASK_HI7Q_HOST_IDX)
+
+/* 2 REG_DBG_SEL_V1				(Offset 0x03D8) */
+
+#define BIT_DIS_TXDMA_PRE BIT(7)
+#define BIT_DIS_RXDMA_PRE BIT(6)
+#define BIT_TXFLAG_EXIT_L1_EN BIT(2)
+
+#define BIT_SHIFT_DBG_SEL 0
+#define BIT_MASK_DBG_SEL 0xff
+#define BIT_DBG_SEL(x) (((x) & BIT_MASK_DBG_SEL) << BIT_SHIFT_DBG_SEL)
+#define BIT_GET_DBG_SEL(x) (((x) >> BIT_SHIFT_DBG_SEL) & BIT_MASK_DBG_SEL)
+
+/* 2 REG_PCIE_HRPWM1_V1			(Offset 0x03D9) */
+
+#define BIT_SHIFT_PCIE_HRPWM 0
+#define BIT_MASK_PCIE_HRPWM 0xff
+#define BIT_PCIE_HRPWM(x) (((x) & BIT_MASK_PCIE_HRPWM) << BIT_SHIFT_PCIE_HRPWM)
+#define BIT_GET_PCIE_HRPWM(x)                                                  \
+	(((x) >> BIT_SHIFT_PCIE_HRPWM) & BIT_MASK_PCIE_HRPWM)
+
+/* 2 REG_PCIE_HCPWM1_V1			(Offset 0x03DA) */
+
+#define BIT_SHIFT_PCIE_HCPWM 0
+#define BIT_MASK_PCIE_HCPWM 0xff
+#define BIT_PCIE_HCPWM(x) (((x) & BIT_MASK_PCIE_HCPWM) << BIT_SHIFT_PCIE_HCPWM)
+#define BIT_GET_PCIE_HCPWM(x)                                                  \
+	(((x) >> BIT_SHIFT_PCIE_HCPWM) & BIT_MASK_PCIE_HCPWM)
+
+/* 2 REG_PCIE_CTRL2				(Offset 0x03DB) */
+
+#define BIT_SHIFT_HPS_CLKR_PCIE 4
+#define BIT_MASK_HPS_CLKR_PCIE 0x3
+#define BIT_HPS_CLKR_PCIE(x)                                                   \
+	(((x) & BIT_MASK_HPS_CLKR_PCIE) << BIT_SHIFT_HPS_CLKR_PCIE)
+#define BIT_GET_HPS_CLKR_PCIE(x)                                               \
+	(((x) >> BIT_SHIFT_HPS_CLKR_PCIE) & BIT_MASK_HPS_CLKR_PCIE)
+
+/* 2 REG_PCIE_CTRL2				(Offset 0x03DB) */
+
+#define BIT_PCIE_INT BIT(3)
+
+/* 2 REG_PCIE_CTRL2				(Offset 0x03DB) */
+
+#define BIT_EN_RXDMA_ALIGN BIT(1)
+#define BIT_EN_TXDMA_ALIGN BIT(0)
+
+/* 2 REG_PCIE_HRPWM2_V1			(Offset 0x03DC) */
+
+#define BIT_SHIFT_PCIE_HRPWM2 0
+#define BIT_MASK_PCIE_HRPWM2 0xffff
+#define BIT_PCIE_HRPWM2(x)                                                     \
+	(((x) & BIT_MASK_PCIE_HRPWM2) << BIT_SHIFT_PCIE_HRPWM2)
+#define BIT_GET_PCIE_HRPWM2(x)                                                 \
+	(((x) >> BIT_SHIFT_PCIE_HRPWM2) & BIT_MASK_PCIE_HRPWM2)
+
+/* 2 REG_PCIE_HCPWM2_V1			(Offset 0x03DE) */
+
+#define BIT_SHIFT_PCIE_HCPWM2 0
+#define BIT_MASK_PCIE_HCPWM2 0xffff
+#define BIT_PCIE_HCPWM2(x)                                                     \
+	(((x) & BIT_MASK_PCIE_HCPWM2) << BIT_SHIFT_PCIE_HCPWM2)
+#define BIT_GET_PCIE_HCPWM2(x)                                                 \
+	(((x) >> BIT_SHIFT_PCIE_HCPWM2) & BIT_MASK_PCIE_HCPWM2)
+
+/* 2 REG_PCIE_H2C_MSG_V1			(Offset 0x03E0) */
+
+#define BIT_SHIFT_DRV2FW_INFO 0
+#define BIT_MASK_DRV2FW_INFO 0xffffffffL
+#define BIT_DRV2FW_INFO(x)                                                     \
+	(((x) & BIT_MASK_DRV2FW_INFO) << BIT_SHIFT_DRV2FW_INFO)
+#define BIT_GET_DRV2FW_INFO(x)                                                 \
+	(((x) >> BIT_SHIFT_DRV2FW_INFO) & BIT_MASK_DRV2FW_INFO)
+
+/* 2 REG_PCIE_C2H_MSG_V1			(Offset 0x03E4) */
+
+#define BIT_SHIFT_HCI_PCIE_C2H_MSG 0
+#define BIT_MASK_HCI_PCIE_C2H_MSG 0xffffffffL
+#define BIT_HCI_PCIE_C2H_MSG(x)                                                \
+	(((x) & BIT_MASK_HCI_PCIE_C2H_MSG) << BIT_SHIFT_HCI_PCIE_C2H_MSG)
+#define BIT_GET_HCI_PCIE_C2H_MSG(x)                                            \
+	(((x) >> BIT_SHIFT_HCI_PCIE_C2H_MSG) & BIT_MASK_HCI_PCIE_C2H_MSG)
+
+/* 2 REG_DBI_WDATA_V1			(Offset 0x03E8) */
+
+#define BIT_SHIFT_DBI_WDATA 0
+#define BIT_MASK_DBI_WDATA 0xffffffffL
+#define BIT_DBI_WDATA(x) (((x) & BIT_MASK_DBI_WDATA) << BIT_SHIFT_DBI_WDATA)
+#define BIT_GET_DBI_WDATA(x) (((x) >> BIT_SHIFT_DBI_WDATA) & BIT_MASK_DBI_WDATA)
+
+/* 2 REG_DBI_RDATA_V1			(Offset 0x03EC) */
+
+#define BIT_SHIFT_DBI_RDATA 0
+#define BIT_MASK_DBI_RDATA 0xffffffffL
+#define BIT_DBI_RDATA(x) (((x) & BIT_MASK_DBI_RDATA) << BIT_SHIFT_DBI_RDATA)
+#define BIT_GET_DBI_RDATA(x) (((x) >> BIT_SHIFT_DBI_RDATA) & BIT_MASK_DBI_RDATA)
+
+/* 2 REG_DBI_FLAG_V1				(Offset 0x03F0) */
+
+#define BIT_EN_STUCK_DBG BIT(26)
+#define BIT_RX_STUCK BIT(25)
+#define BIT_TX_STUCK BIT(24)
+#define BIT_DBI_RFLAG BIT(17)
+#define BIT_DBI_WFLAG BIT(16)
+
+#define BIT_SHIFT_DBI_WREN 12
+#define BIT_MASK_DBI_WREN 0xf
+#define BIT_DBI_WREN(x) (((x) & BIT_MASK_DBI_WREN) << BIT_SHIFT_DBI_WREN)
+#define BIT_GET_DBI_WREN(x) (((x) >> BIT_SHIFT_DBI_WREN) & BIT_MASK_DBI_WREN)
+
+#define BIT_SHIFT_DBI_ADDR 0
+#define BIT_MASK_DBI_ADDR 0xfff
+#define BIT_DBI_ADDR(x) (((x) & BIT_MASK_DBI_ADDR) << BIT_SHIFT_DBI_ADDR)
+#define BIT_GET_DBI_ADDR(x) (((x) >> BIT_SHIFT_DBI_ADDR) & BIT_MASK_DBI_ADDR)
+
+/* 2 REG_MDIO_V1				(Offset 0x03F4) */
+
+#define BIT_SHIFT_MDIO_RDATA 16
+#define BIT_MASK_MDIO_RDATA 0xffff
+#define BIT_MDIO_RDATA(x) (((x) & BIT_MASK_MDIO_RDATA) << BIT_SHIFT_MDIO_RDATA)
+#define BIT_GET_MDIO_RDATA(x)                                                  \
+	(((x) >> BIT_SHIFT_MDIO_RDATA) & BIT_MASK_MDIO_RDATA)
+
+#define BIT_SHIFT_MDIO_WDATA 0
+#define BIT_MASK_MDIO_WDATA 0xffff
+#define BIT_MDIO_WDATA(x) (((x) & BIT_MASK_MDIO_WDATA) << BIT_SHIFT_MDIO_WDATA)
+#define BIT_GET_MDIO_WDATA(x)                                                  \
+	(((x) >> BIT_SHIFT_MDIO_WDATA) & BIT_MASK_MDIO_WDATA)
+
+/* 2 REG_PCIE_MIX_CFG			(Offset 0x03F8) */
+
+#define BIT_EN_WATCH_DOG BIT(8)
+
+/* 2 REG_PCIE_MIX_CFG			(Offset 0x03F8) */
+
+#define BIT_SHIFT_MDIO_REG_ADDR_V1 0
+#define BIT_MASK_MDIO_REG_ADDR_V1 0x1f
+#define BIT_MDIO_REG_ADDR_V1(x)                                                \
+	(((x) & BIT_MASK_MDIO_REG_ADDR_V1) << BIT_SHIFT_MDIO_REG_ADDR_V1)
+#define BIT_GET_MDIO_REG_ADDR_V1(x)                                            \
+	(((x) >> BIT_SHIFT_MDIO_REG_ADDR_V1) & BIT_MASK_MDIO_REG_ADDR_V1)
+
+/* 2 REG_HCI_MIX_CFG				(Offset 0x03FC) */
+
+#define BIT_HOST_GEN2_SUPPORT BIT(20)
+
+#define BIT_SHIFT_TXDMA_ERR_FLAG 16
+#define BIT_MASK_TXDMA_ERR_FLAG 0xf
+#define BIT_TXDMA_ERR_FLAG(x)                                                  \
+	(((x) & BIT_MASK_TXDMA_ERR_FLAG) << BIT_SHIFT_TXDMA_ERR_FLAG)
+#define BIT_GET_TXDMA_ERR_FLAG(x)                                              \
+	(((x) >> BIT_SHIFT_TXDMA_ERR_FLAG) & BIT_MASK_TXDMA_ERR_FLAG)
+
+#define BIT_SHIFT_EARLY_MODE_SEL 12
+#define BIT_MASK_EARLY_MODE_SEL 0xf
+#define BIT_EARLY_MODE_SEL(x)                                                  \
+	(((x) & BIT_MASK_EARLY_MODE_SEL) << BIT_SHIFT_EARLY_MODE_SEL)
+#define BIT_GET_EARLY_MODE_SEL(x)                                              \
+	(((x) >> BIT_SHIFT_EARLY_MODE_SEL) & BIT_MASK_EARLY_MODE_SEL)
+
+#define BIT_EPHY_RX50_EN BIT(11)
+
+#define BIT_SHIFT_MSI_TIMEOUT_ID_V1 8
+#define BIT_MASK_MSI_TIMEOUT_ID_V1 0x7
+#define BIT_MSI_TIMEOUT_ID_V1(x)                                               \
+	(((x) & BIT_MASK_MSI_TIMEOUT_ID_V1) << BIT_SHIFT_MSI_TIMEOUT_ID_V1)
+#define BIT_GET_MSI_TIMEOUT_ID_V1(x)                                           \
+	(((x) >> BIT_SHIFT_MSI_TIMEOUT_ID_V1) & BIT_MASK_MSI_TIMEOUT_ID_V1)
+
+#define BIT_RADDR_RD BIT(7)
+#define BIT_EN_MUL_TAG BIT(6)
+#define BIT_EN_EARLY_MODE BIT(5)
+#define BIT_L0S_LINK_OFF BIT(4)
+#define BIT_ACT_LINK_OFF BIT(3)
+
+/* 2 REG_HCI_MIX_CFG				(Offset 0x03FC) */
+
+#define BIT_EN_SLOW_MAC_TX BIT(2)
+#define BIT_EN_SLOW_MAC_RX BIT(1)
+
+/* 2 REG_Q0_INFO				(Offset 0x0400) */
+
+#define BIT_SHIFT_QUEUEMACID_Q0_V1 25
+#define BIT_MASK_QUEUEMACID_Q0_V1 0x7f
+#define BIT_QUEUEMACID_Q0_V1(x)                                                \
+	(((x) & BIT_MASK_QUEUEMACID_Q0_V1) << BIT_SHIFT_QUEUEMACID_Q0_V1)
+#define BIT_GET_QUEUEMACID_Q0_V1(x)                                            \
+	(((x) >> BIT_SHIFT_QUEUEMACID_Q0_V1) & BIT_MASK_QUEUEMACID_Q0_V1)
+
+#define BIT_SHIFT_QUEUEAC_Q0_V1 23
+#define BIT_MASK_QUEUEAC_Q0_V1 0x3
+#define BIT_QUEUEAC_Q0_V1(x)                                                   \
+	(((x) & BIT_MASK_QUEUEAC_Q0_V1) << BIT_SHIFT_QUEUEAC_Q0_V1)
+#define BIT_GET_QUEUEAC_Q0_V1(x)                                               \
+	(((x) >> BIT_SHIFT_QUEUEAC_Q0_V1) & BIT_MASK_QUEUEAC_Q0_V1)
+
+/* 2 REG_Q0_INFO				(Offset 0x0400) */
+
+#define BIT_TIDEMPTY_Q0_V1 BIT(22)
+
+/* 2 REG_Q0_INFO				(Offset 0x0400) */
+
+#define BIT_SHIFT_TAIL_PKT_Q0_V2 11
+#define BIT_MASK_TAIL_PKT_Q0_V2 0x7ff
+#define BIT_TAIL_PKT_Q0_V2(x)                                                  \
+	(((x) & BIT_MASK_TAIL_PKT_Q0_V2) << BIT_SHIFT_TAIL_PKT_Q0_V2)
+#define BIT_GET_TAIL_PKT_Q0_V2(x)                                              \
+	(((x) >> BIT_SHIFT_TAIL_PKT_Q0_V2) & BIT_MASK_TAIL_PKT_Q0_V2)
+
+/* 2 REG_Q0_INFO				(Offset 0x0400) */
+
+#define BIT_SHIFT_HEAD_PKT_Q0_V1 0
+#define BIT_MASK_HEAD_PKT_Q0_V1 0x7ff
+#define BIT_HEAD_PKT_Q0_V1(x)                                                  \
+	(((x) & BIT_MASK_HEAD_PKT_Q0_V1) << BIT_SHIFT_HEAD_PKT_Q0_V1)
+#define BIT_GET_HEAD_PKT_Q0_V1(x)                                              \
+	(((x) >> BIT_SHIFT_HEAD_PKT_Q0_V1) & BIT_MASK_HEAD_PKT_Q0_V1)
+
+/* 2 REG_Q1_INFO				(Offset 0x0404) */
+
+#define BIT_SHIFT_QUEUEMACID_Q1_V1 25
+#define BIT_MASK_QUEUEMACID_Q1_V1 0x7f
+#define BIT_QUEUEMACID_Q1_V1(x)                                                \
+	(((x) & BIT_MASK_QUEUEMACID_Q1_V1) << BIT_SHIFT_QUEUEMACID_Q1_V1)
+#define BIT_GET_QUEUEMACID_Q1_V1(x)                                            \
+	(((x) >> BIT_SHIFT_QUEUEMACID_Q1_V1) & BIT_MASK_QUEUEMACID_Q1_V1)
+
+#define BIT_SHIFT_QUEUEAC_Q1_V1 23
+#define BIT_MASK_QUEUEAC_Q1_V1 0x3
+#define BIT_QUEUEAC_Q1_V1(x)                                                   \
+	(((x) & BIT_MASK_QUEUEAC_Q1_V1) << BIT_SHIFT_QUEUEAC_Q1_V1)
+#define BIT_GET_QUEUEAC_Q1_V1(x)                                               \
+	(((x) >> BIT_SHIFT_QUEUEAC_Q1_V1) & BIT_MASK_QUEUEAC_Q1_V1)
+
+/* 2 REG_Q1_INFO				(Offset 0x0404) */
+
+#define BIT_TIDEMPTY_Q1_V1 BIT(22)
+
+/* 2 REG_Q1_INFO				(Offset 0x0404) */
+
+#define BIT_SHIFT_TAIL_PKT_Q1_V2 11
+#define BIT_MASK_TAIL_PKT_Q1_V2 0x7ff
+#define BIT_TAIL_PKT_Q1_V2(x)                                                  \
+	(((x) & BIT_MASK_TAIL_PKT_Q1_V2) << BIT_SHIFT_TAIL_PKT_Q1_V2)
+#define BIT_GET_TAIL_PKT_Q1_V2(x)                                              \
+	(((x) >> BIT_SHIFT_TAIL_PKT_Q1_V2) & BIT_MASK_TAIL_PKT_Q1_V2)
+
+/* 2 REG_Q1_INFO				(Offset 0x0404) */
+
+#define BIT_SHIFT_HEAD_PKT_Q1_V1 0
+#define BIT_MASK_HEAD_PKT_Q1_V1 0x7ff
+#define BIT_HEAD_PKT_Q1_V1(x)                                                  \
+	(((x) & BIT_MASK_HEAD_PKT_Q1_V1) << BIT_SHIFT_HEAD_PKT_Q1_V1)
+#define BIT_GET_HEAD_PKT_Q1_V1(x)                                              \
+	(((x) >> BIT_SHIFT_HEAD_PKT_Q1_V1) & BIT_MASK_HEAD_PKT_Q1_V1)
+
+/* 2 REG_Q2_INFO				(Offset 0x0408) */
+
+#define BIT_SHIFT_QUEUEMACID_Q2_V1 25
+#define BIT_MASK_QUEUEMACID_Q2_V1 0x7f
+#define BIT_QUEUEMACID_Q2_V1(x)                                                \
+	(((x) & BIT_MASK_QUEUEMACID_Q2_V1) << BIT_SHIFT_QUEUEMACID_Q2_V1)
+#define BIT_GET_QUEUEMACID_Q2_V1(x)                                            \
+	(((x) >> BIT_SHIFT_QUEUEMACID_Q2_V1) & BIT_MASK_QUEUEMACID_Q2_V1)
+
+#define BIT_SHIFT_QUEUEAC_Q2_V1 23
+#define BIT_MASK_QUEUEAC_Q2_V1 0x3
+#define BIT_QUEUEAC_Q2_V1(x)                                                   \
+	(((x) & BIT_MASK_QUEUEAC_Q2_V1) << BIT_SHIFT_QUEUEAC_Q2_V1)
+#define BIT_GET_QUEUEAC_Q2_V1(x)                                               \
+	(((x) >> BIT_SHIFT_QUEUEAC_Q2_V1) & BIT_MASK_QUEUEAC_Q2_V1)
+
+/* 2 REG_Q2_INFO				(Offset 0x0408) */
+
+#define BIT_TIDEMPTY_Q2_V1 BIT(22)
+
+/* 2 REG_Q2_INFO				(Offset 0x0408) */
+
+#define BIT_SHIFT_TAIL_PKT_Q2_V2 11
+#define BIT_MASK_TAIL_PKT_Q2_V2 0x7ff
+#define BIT_TAIL_PKT_Q2_V2(x)                                                  \
+	(((x) & BIT_MASK_TAIL_PKT_Q2_V2) << BIT_SHIFT_TAIL_PKT_Q2_V2)
+#define BIT_GET_TAIL_PKT_Q2_V2(x)                                              \
+	(((x) >> BIT_SHIFT_TAIL_PKT_Q2_V2) & BIT_MASK_TAIL_PKT_Q2_V2)
+
+/* 2 REG_Q2_INFO				(Offset 0x0408) */
+
+#define BIT_SHIFT_HEAD_PKT_Q2_V1 0
+#define BIT_MASK_HEAD_PKT_Q2_V1 0x7ff
+#define BIT_HEAD_PKT_Q2_V1(x)                                                  \
+	(((x) & BIT_MASK_HEAD_PKT_Q2_V1) << BIT_SHIFT_HEAD_PKT_Q2_V1)
+#define BIT_GET_HEAD_PKT_Q2_V1(x)                                              \
+	(((x) >> BIT_SHIFT_HEAD_PKT_Q2_V1) & BIT_MASK_HEAD_PKT_Q2_V1)
+
+/* 2 REG_Q3_INFO				(Offset 0x040C) */
+
+#define BIT_SHIFT_QUEUEMACID_Q3_V1 25
+#define BIT_MASK_QUEUEMACID_Q3_V1 0x7f
+#define BIT_QUEUEMACID_Q3_V1(x)                                                \
+	(((x) & BIT_MASK_QUEUEMACID_Q3_V1) << BIT_SHIFT_QUEUEMACID_Q3_V1)
+#define BIT_GET_QUEUEMACID_Q3_V1(x)                                            \
+	(((x) >> BIT_SHIFT_QUEUEMACID_Q3_V1) & BIT_MASK_QUEUEMACID_Q3_V1)
+
+#define BIT_SHIFT_QUEUEAC_Q3_V1 23
+#define BIT_MASK_QUEUEAC_Q3_V1 0x3
+#define BIT_QUEUEAC_Q3_V1(x)                                                   \
+	(((x) & BIT_MASK_QUEUEAC_Q3_V1) << BIT_SHIFT_QUEUEAC_Q3_V1)
+#define BIT_GET_QUEUEAC_Q3_V1(x)                                               \
+	(((x) >> BIT_SHIFT_QUEUEAC_Q3_V1) & BIT_MASK_QUEUEAC_Q3_V1)
+
+/* 2 REG_Q3_INFO				(Offset 0x040C) */
+
+#define BIT_TIDEMPTY_Q3_V1 BIT(22)
+
+/* 2 REG_Q3_INFO				(Offset 0x040C) */
+
+#define BIT_SHIFT_TAIL_PKT_Q3_V2 11
+#define BIT_MASK_TAIL_PKT_Q3_V2 0x7ff
+#define BIT_TAIL_PKT_Q3_V2(x)                                                  \
+	(((x) & BIT_MASK_TAIL_PKT_Q3_V2) << BIT_SHIFT_TAIL_PKT_Q3_V2)
+#define BIT_GET_TAIL_PKT_Q3_V2(x)                                              \
+	(((x) >> BIT_SHIFT_TAIL_PKT_Q3_V2) & BIT_MASK_TAIL_PKT_Q3_V2)
+
+/* 2 REG_Q3_INFO				(Offset 0x040C) */
+
+#define BIT_SHIFT_HEAD_PKT_Q3_V1 0
+#define BIT_MASK_HEAD_PKT_Q3_V1 0x7ff
+#define BIT_HEAD_PKT_Q3_V1(x)                                                  \
+	(((x) & BIT_MASK_HEAD_PKT_Q3_V1) << BIT_SHIFT_HEAD_PKT_Q3_V1)
+#define BIT_GET_HEAD_PKT_Q3_V1(x)                                              \
+	(((x) >> BIT_SHIFT_HEAD_PKT_Q3_V1) & BIT_MASK_HEAD_PKT_Q3_V1)
+
+/* 2 REG_MGQ_INFO				(Offset 0x0410) */
+
+#define BIT_SHIFT_QUEUEMACID_MGQ_V1 25
+#define BIT_MASK_QUEUEMACID_MGQ_V1 0x7f
+#define BIT_QUEUEMACID_MGQ_V1(x)                                               \
+	(((x) & BIT_MASK_QUEUEMACID_MGQ_V1) << BIT_SHIFT_QUEUEMACID_MGQ_V1)
+#define BIT_GET_QUEUEMACID_MGQ_V1(x)                                           \
+	(((x) >> BIT_SHIFT_QUEUEMACID_MGQ_V1) & BIT_MASK_QUEUEMACID_MGQ_V1)
+
+#define BIT_SHIFT_QUEUEAC_MGQ_V1 23
+#define BIT_MASK_QUEUEAC_MGQ_V1 0x3
+#define BIT_QUEUEAC_MGQ_V1(x)                                                  \
+	(((x) & BIT_MASK_QUEUEAC_MGQ_V1) << BIT_SHIFT_QUEUEAC_MGQ_V1)
+#define BIT_GET_QUEUEAC_MGQ_V1(x)                                              \
+	(((x) >> BIT_SHIFT_QUEUEAC_MGQ_V1) & BIT_MASK_QUEUEAC_MGQ_V1)
+
+/* 2 REG_MGQ_INFO				(Offset 0x0410) */
+
+#define BIT_TIDEMPTY_MGQ_V1 BIT(22)
+
+/* 2 REG_MGQ_INFO				(Offset 0x0410) */
+
+#define BIT_SHIFT_TAIL_PKT_MGQ_V2 11
+#define BIT_MASK_TAIL_PKT_MGQ_V2 0x7ff
+#define BIT_TAIL_PKT_MGQ_V2(x)                                                 \
+	(((x) & BIT_MASK_TAIL_PKT_MGQ_V2) << BIT_SHIFT_TAIL_PKT_MGQ_V2)
+#define BIT_GET_TAIL_PKT_MGQ_V2(x)                                             \
+	(((x) >> BIT_SHIFT_TAIL_PKT_MGQ_V2) & BIT_MASK_TAIL_PKT_MGQ_V2)
+
+/* 2 REG_MGQ_INFO				(Offset 0x0410) */
+
+#define BIT_SHIFT_HEAD_PKT_MGQ_V1 0
+#define BIT_MASK_HEAD_PKT_MGQ_V1 0x7ff
+#define BIT_HEAD_PKT_MGQ_V1(x)                                                 \
+	(((x) & BIT_MASK_HEAD_PKT_MGQ_V1) << BIT_SHIFT_HEAD_PKT_MGQ_V1)
+#define BIT_GET_HEAD_PKT_MGQ_V1(x)                                             \
+	(((x) >> BIT_SHIFT_HEAD_PKT_MGQ_V1) & BIT_MASK_HEAD_PKT_MGQ_V1)
+
+/* 2 REG_HIQ_INFO				(Offset 0x0414) */
+
+#define BIT_SHIFT_QUEUEMACID_HIQ_V1 25
+#define BIT_MASK_QUEUEMACID_HIQ_V1 0x7f
+#define BIT_QUEUEMACID_HIQ_V1(x)                                               \
+	(((x) & BIT_MASK_QUEUEMACID_HIQ_V1) << BIT_SHIFT_QUEUEMACID_HIQ_V1)
+#define BIT_GET_QUEUEMACID_HIQ_V1(x)                                           \
+	(((x) >> BIT_SHIFT_QUEUEMACID_HIQ_V1) & BIT_MASK_QUEUEMACID_HIQ_V1)
+
+#define BIT_SHIFT_QUEUEAC_HIQ_V1 23
+#define BIT_MASK_QUEUEAC_HIQ_V1 0x3
+#define BIT_QUEUEAC_HIQ_V1(x)                                                  \
+	(((x) & BIT_MASK_QUEUEAC_HIQ_V1) << BIT_SHIFT_QUEUEAC_HIQ_V1)
+#define BIT_GET_QUEUEAC_HIQ_V1(x)                                              \
+	(((x) >> BIT_SHIFT_QUEUEAC_HIQ_V1) & BIT_MASK_QUEUEAC_HIQ_V1)
+
+/* 2 REG_HIQ_INFO				(Offset 0x0414) */
+
+#define BIT_TIDEMPTY_HIQ_V1 BIT(22)
+
+/* 2 REG_HIQ_INFO				(Offset 0x0414) */
+
+#define BIT_SHIFT_TAIL_PKT_HIQ_V2 11
+#define BIT_MASK_TAIL_PKT_HIQ_V2 0x7ff
+#define BIT_TAIL_PKT_HIQ_V2(x)                                                 \
+	(((x) & BIT_MASK_TAIL_PKT_HIQ_V2) << BIT_SHIFT_TAIL_PKT_HIQ_V2)
+#define BIT_GET_TAIL_PKT_HIQ_V2(x)                                             \
+	(((x) >> BIT_SHIFT_TAIL_PKT_HIQ_V2) & BIT_MASK_TAIL_PKT_HIQ_V2)
+
+/* 2 REG_HIQ_INFO				(Offset 0x0414) */
+
+#define BIT_SHIFT_HEAD_PKT_HIQ_V1 0
+#define BIT_MASK_HEAD_PKT_HIQ_V1 0x7ff
+#define BIT_HEAD_PKT_HIQ_V1(x)                                                 \
+	(((x) & BIT_MASK_HEAD_PKT_HIQ_V1) << BIT_SHIFT_HEAD_PKT_HIQ_V1)
+#define BIT_GET_HEAD_PKT_HIQ_V1(x)                                             \
+	(((x) >> BIT_SHIFT_HEAD_PKT_HIQ_V1) & BIT_MASK_HEAD_PKT_HIQ_V1)
+
+/* 2 REG_BCNQ_INFO				(Offset 0x0418) */
+
+#define BIT_SHIFT_BCNQ_HEAD_PG_V1 0
+#define BIT_MASK_BCNQ_HEAD_PG_V1 0xfff
+#define BIT_BCNQ_HEAD_PG_V1(x)                                                 \
+	(((x) & BIT_MASK_BCNQ_HEAD_PG_V1) << BIT_SHIFT_BCNQ_HEAD_PG_V1)
+#define BIT_GET_BCNQ_HEAD_PG_V1(x)                                             \
+	(((x) >> BIT_SHIFT_BCNQ_HEAD_PG_V1) & BIT_MASK_BCNQ_HEAD_PG_V1)
+
+/* 2 REG_TXPKT_EMPTY				(Offset 0x041A) */
+
+#define BIT_BCNQ_EMPTY BIT(11)
+#define BIT_HQQ_EMPTY BIT(10)
+#define BIT_MQQ_EMPTY BIT(9)
+#define BIT_MGQ_CPU_EMPTY BIT(8)
+#define BIT_AC7Q_EMPTY BIT(7)
+#define BIT_AC6Q_EMPTY BIT(6)
+#define BIT_AC5Q_EMPTY BIT(5)
+#define BIT_AC4Q_EMPTY BIT(4)
+#define BIT_AC3Q_EMPTY BIT(3)
+#define BIT_AC2Q_EMPTY BIT(2)
+#define BIT_AC1Q_EMPTY BIT(1)
+#define BIT_AC0Q_EMPTY BIT(0)
+
+/* 2 REG_CPU_MGQ_INFO			(Offset 0x041C) */
+
+#define BIT_BCN1_POLL BIT(30)
+
+/* 2 REG_CPU_MGQ_INFO			(Offset 0x041C) */
+
+#define BIT_CPUMGT_POLL BIT(29)
+#define BIT_BCN_POLL BIT(28)
+
+/* 2 REG_CPU_MGQ_INFO			(Offset 0x041C) */
+
+#define BIT_CPUMGQ_FW_NUM_V1 BIT(12)
+
+/* 2 REG_CPU_MGQ_INFO			(Offset 0x041C) */
+
+#define BIT_SHIFT_FW_FREE_TAIL_V1 0
+#define BIT_MASK_FW_FREE_TAIL_V1 0xfff
+#define BIT_FW_FREE_TAIL_V1(x)                                                 \
+	(((x) & BIT_MASK_FW_FREE_TAIL_V1) << BIT_SHIFT_FW_FREE_TAIL_V1)
+#define BIT_GET_FW_FREE_TAIL_V1(x)                                             \
+	(((x) >> BIT_SHIFT_FW_FREE_TAIL_V1) & BIT_MASK_FW_FREE_TAIL_V1)
+
+/* 2 REG_FWHW_TXQ_CTRL			(Offset 0x0420) */
+
+#define BIT_RTS_LIMIT_IN_OFDM BIT(23)
+#define BIT_EN_BCNQ_DL BIT(22)
+#define BIT_EN_RD_RESP_NAV_BK BIT(21)
+#define BIT_EN_WR_FREE_TAIL BIT(20)
+
+#define BIT_SHIFT_EN_QUEUE_RPT 8
+#define BIT_MASK_EN_QUEUE_RPT 0xff
+#define BIT_EN_QUEUE_RPT(x)                                                    \
+	(((x) & BIT_MASK_EN_QUEUE_RPT) << BIT_SHIFT_EN_QUEUE_RPT)
+#define BIT_GET_EN_QUEUE_RPT(x)                                                \
+	(((x) >> BIT_SHIFT_EN_QUEUE_RPT) & BIT_MASK_EN_QUEUE_RPT)
+
+#define BIT_EN_RTY_BK BIT(7)
+#define BIT_EN_USE_INI_RAT BIT(6)
+#define BIT_EN_RTS_NAV_BK BIT(5)
+#define BIT_DIS_SSN_CHECK BIT(4)
+#define BIT_MACID_MATCH_RTS BIT(3)
+
+/* 2 REG_FWHW_TXQ_CTRL			(Offset 0x0420) */
+
+#define BIT_EN_BCN_TRXRPT_V1 BIT(2)
+
+/* 2 REG_FWHW_TXQ_CTRL			(Offset 0x0420) */
+
+#define BIT_EN_FTMACKRPT BIT(1)
+
+/* 2 REG_FWHW_TXQ_CTRL			(Offset 0x0420) */
+
+#define BIT_EN_FTMRPT BIT(0)
+
+/* 2 REG_DATAFB_SEL				(Offset 0x0423) */
+
+#define BIT__R_EN_RTY_BK_COD BIT(2)
+
+/* 2 REG_DATAFB_SEL				(Offset 0x0423) */
+
+#define BIT_SHIFT__R_DATA_FALLBACK_SEL 0
+#define BIT_MASK__R_DATA_FALLBACK_SEL 0x3
+#define BIT__R_DATA_FALLBACK_SEL(x)                                            \
+	(((x) & BIT_MASK__R_DATA_FALLBACK_SEL)                                 \
+	 << BIT_SHIFT__R_DATA_FALLBACK_SEL)
+#define BIT_GET__R_DATA_FALLBACK_SEL(x)                                        \
+	(((x) >> BIT_SHIFT__R_DATA_FALLBACK_SEL) &                             \
+	 BIT_MASK__R_DATA_FALLBACK_SEL)
+
+/* 2 REG_BCNQ_BDNY_V1			(Offset 0x0424) */
+
+#define BIT_SHIFT_BCNQ_PGBNDY_V1 0
+#define BIT_MASK_BCNQ_PGBNDY_V1 0xfff
+#define BIT_BCNQ_PGBNDY_V1(x)                                                  \
+	(((x) & BIT_MASK_BCNQ_PGBNDY_V1) << BIT_SHIFT_BCNQ_PGBNDY_V1)
+#define BIT_GET_BCNQ_PGBNDY_V1(x)                                              \
+	(((x) >> BIT_SHIFT_BCNQ_PGBNDY_V1) & BIT_MASK_BCNQ_PGBNDY_V1)
+
+/* 2 REG_LIFETIME_EN				(Offset 0x0426) */
+
+#define BIT_BT_INT_CPU BIT(7)
+#define BIT_BT_INT_PTA BIT(6)
+
+/* 2 REG_LIFETIME_EN				(Offset 0x0426) */
+
+#define BIT_EN_CTRL_RTYBIT BIT(4)
+
+/* 2 REG_LIFETIME_EN				(Offset 0x0426) */
+
+#define BIT_LIFETIME_BK_EN BIT(3)
+#define BIT_LIFETIME_BE_EN BIT(2)
+#define BIT_LIFETIME_VI_EN BIT(1)
+#define BIT_LIFETIME_VO_EN BIT(0)
+
+/* 2 REG_SPEC_SIFS				(Offset 0x0428) */
+
+#define BIT_SHIFT_SPEC_SIFS_OFDM_PTCL 8
+#define BIT_MASK_SPEC_SIFS_OFDM_PTCL 0xff
+#define BIT_SPEC_SIFS_OFDM_PTCL(x)                                             \
+	(((x) & BIT_MASK_SPEC_SIFS_OFDM_PTCL) << BIT_SHIFT_SPEC_SIFS_OFDM_PTCL)
+#define BIT_GET_SPEC_SIFS_OFDM_PTCL(x)                                         \
+	(((x) >> BIT_SHIFT_SPEC_SIFS_OFDM_PTCL) & BIT_MASK_SPEC_SIFS_OFDM_PTCL)
+
+#define BIT_SHIFT_SPEC_SIFS_CCK_PTCL 0
+#define BIT_MASK_SPEC_SIFS_CCK_PTCL 0xff
+#define BIT_SPEC_SIFS_CCK_PTCL(x)                                              \
+	(((x) & BIT_MASK_SPEC_SIFS_CCK_PTCL) << BIT_SHIFT_SPEC_SIFS_CCK_PTCL)
+#define BIT_GET_SPEC_SIFS_CCK_PTCL(x)                                          \
+	(((x) >> BIT_SHIFT_SPEC_SIFS_CCK_PTCL) & BIT_MASK_SPEC_SIFS_CCK_PTCL)
+
+/* 2 REG_RETRY_LIMIT				(Offset 0x042A) */
+
+#define BIT_SHIFT_SRL 8
+#define BIT_MASK_SRL 0x3f
+#define BIT_SRL(x) (((x) & BIT_MASK_SRL) << BIT_SHIFT_SRL)
+#define BIT_GET_SRL(x) (((x) >> BIT_SHIFT_SRL) & BIT_MASK_SRL)
+
+#define BIT_SHIFT_LRL 0
+#define BIT_MASK_LRL 0x3f
+#define BIT_LRL(x) (((x) & BIT_MASK_LRL) << BIT_SHIFT_LRL)
+#define BIT_GET_LRL(x) (((x) >> BIT_SHIFT_LRL) & BIT_MASK_LRL)
+
+/* 2 REG_TXBF_CTRL				(Offset 0x042C) */
+
+#define BIT_R_ENABLE_NDPA BIT(31)
+#define BIT_USE_NDPA_PARAMETER BIT(30)
+#define BIT_R_PROP_TXBF BIT(29)
+#define BIT_R_EN_NDPA_INT BIT(28)
+#define BIT_R_TXBF1_80M BIT(27)
+#define BIT_R_TXBF1_40M BIT(26)
+#define BIT_R_TXBF1_20M BIT(25)
+
+#define BIT_SHIFT_R_TXBF1_AID 16
+#define BIT_MASK_R_TXBF1_AID 0x1ff
+#define BIT_R_TXBF1_AID(x)                                                     \
+	(((x) & BIT_MASK_R_TXBF1_AID) << BIT_SHIFT_R_TXBF1_AID)
+#define BIT_GET_R_TXBF1_AID(x)                                                 \
+	(((x) >> BIT_SHIFT_R_TXBF1_AID) & BIT_MASK_R_TXBF1_AID)
+
+/* 2 REG_TXBF_CTRL				(Offset 0x042C) */
+
+#define BIT_DIS_NDP_BFEN BIT(15)
+
+/* 2 REG_TXBF_CTRL				(Offset 0x042C) */
+
+#define BIT_R_TXBCN_NOBLOCK_NDP BIT(14)
+
+/* 2 REG_TXBF_CTRL				(Offset 0x042C) */
+
+#define BIT_R_TXBF0_80M BIT(11)
+#define BIT_R_TXBF0_40M BIT(10)
+#define BIT_R_TXBF0_20M BIT(9)
+
+#define BIT_SHIFT_R_TXBF0_AID 0
+#define BIT_MASK_R_TXBF0_AID 0x1ff
+#define BIT_R_TXBF0_AID(x)                                                     \
+	(((x) & BIT_MASK_R_TXBF0_AID) << BIT_SHIFT_R_TXBF0_AID)
+#define BIT_GET_R_TXBF0_AID(x)                                                 \
+	(((x) >> BIT_SHIFT_R_TXBF0_AID) & BIT_MASK_R_TXBF0_AID)
+
+/* 2 REG_DARFRC				(Offset 0x0430) */
+
+#define BIT_SHIFT_DARF_RC8 (56 & CPU_OPT_WIDTH)
+#define BIT_MASK_DARF_RC8 0x1f
+#define BIT_DARF_RC8(x) (((x) & BIT_MASK_DARF_RC8) << BIT_SHIFT_DARF_RC8)
+#define BIT_GET_DARF_RC8(x) (((x) >> BIT_SHIFT_DARF_RC8) & BIT_MASK_DARF_RC8)
+
+#define BIT_SHIFT_DARF_RC7 (48 & CPU_OPT_WIDTH)
+#define BIT_MASK_DARF_RC7 0x1f
+#define BIT_DARF_RC7(x) (((x) & BIT_MASK_DARF_RC7) << BIT_SHIFT_DARF_RC7)
+#define BIT_GET_DARF_RC7(x) (((x) >> BIT_SHIFT_DARF_RC7) & BIT_MASK_DARF_RC7)
+
+#define BIT_SHIFT_DARF_RC6 (40 & CPU_OPT_WIDTH)
+#define BIT_MASK_DARF_RC6 0x1f
+#define BIT_DARF_RC6(x) (((x) & BIT_MASK_DARF_RC6) << BIT_SHIFT_DARF_RC6)
+#define BIT_GET_DARF_RC6(x) (((x) >> BIT_SHIFT_DARF_RC6) & BIT_MASK_DARF_RC6)
+
+#define BIT_SHIFT_DARF_RC5 (32 & CPU_OPT_WIDTH)
+#define BIT_MASK_DARF_RC5 0x1f
+#define BIT_DARF_RC5(x) (((x) & BIT_MASK_DARF_RC5) << BIT_SHIFT_DARF_RC5)
+#define BIT_GET_DARF_RC5(x) (((x) >> BIT_SHIFT_DARF_RC5) & BIT_MASK_DARF_RC5)
+
+#define BIT_SHIFT_DARF_RC4 24
+#define BIT_MASK_DARF_RC4 0x1f
+#define BIT_DARF_RC4(x) (((x) & BIT_MASK_DARF_RC4) << BIT_SHIFT_DARF_RC4)
+#define BIT_GET_DARF_RC4(x) (((x) >> BIT_SHIFT_DARF_RC4) & BIT_MASK_DARF_RC4)
+
+#define BIT_SHIFT_DARF_RC3 16
+#define BIT_MASK_DARF_RC3 0x1f
+#define BIT_DARF_RC3(x) (((x) & BIT_MASK_DARF_RC3) << BIT_SHIFT_DARF_RC3)
+#define BIT_GET_DARF_RC3(x) (((x) >> BIT_SHIFT_DARF_RC3) & BIT_MASK_DARF_RC3)
+
+#define BIT_SHIFT_DARF_RC2 8
+#define BIT_MASK_DARF_RC2 0x1f
+#define BIT_DARF_RC2(x) (((x) & BIT_MASK_DARF_RC2) << BIT_SHIFT_DARF_RC2)
+#define BIT_GET_DARF_RC2(x) (((x) >> BIT_SHIFT_DARF_RC2) & BIT_MASK_DARF_RC2)
+
+#define BIT_SHIFT_DARF_RC1 0
+#define BIT_MASK_DARF_RC1 0x1f
+#define BIT_DARF_RC1(x) (((x) & BIT_MASK_DARF_RC1) << BIT_SHIFT_DARF_RC1)
+#define BIT_GET_DARF_RC1(x) (((x) >> BIT_SHIFT_DARF_RC1) & BIT_MASK_DARF_RC1)
+
+/* 2 REG_RARFRC				(Offset 0x0438) */
+
+#define BIT_SHIFT_RARF_RC8 (56 & CPU_OPT_WIDTH)
+#define BIT_MASK_RARF_RC8 0x1f
+#define BIT_RARF_RC8(x) (((x) & BIT_MASK_RARF_RC8) << BIT_SHIFT_RARF_RC8)
+#define BIT_GET_RARF_RC8(x) (((x) >> BIT_SHIFT_RARF_RC8) & BIT_MASK_RARF_RC8)
+
+#define BIT_SHIFT_RARF_RC7 (48 & CPU_OPT_WIDTH)
+#define BIT_MASK_RARF_RC7 0x1f
+#define BIT_RARF_RC7(x) (((x) & BIT_MASK_RARF_RC7) << BIT_SHIFT_RARF_RC7)
+#define BIT_GET_RARF_RC7(x) (((x) >> BIT_SHIFT_RARF_RC7) & BIT_MASK_RARF_RC7)
+
+#define BIT_SHIFT_RARF_RC6 (40 & CPU_OPT_WIDTH)
+#define BIT_MASK_RARF_RC6 0x1f
+#define BIT_RARF_RC6(x) (((x) & BIT_MASK_RARF_RC6) << BIT_SHIFT_RARF_RC6)
+#define BIT_GET_RARF_RC6(x) (((x) >> BIT_SHIFT_RARF_RC6) & BIT_MASK_RARF_RC6)
+
+#define BIT_SHIFT_RARF_RC5 (32 & CPU_OPT_WIDTH)
+#define BIT_MASK_RARF_RC5 0x1f
+#define BIT_RARF_RC5(x) (((x) & BIT_MASK_RARF_RC5) << BIT_SHIFT_RARF_RC5)
+#define BIT_GET_RARF_RC5(x) (((x) >> BIT_SHIFT_RARF_RC5) & BIT_MASK_RARF_RC5)
+
+#define BIT_SHIFT_RARF_RC4 24
+#define BIT_MASK_RARF_RC4 0x1f
+#define BIT_RARF_RC4(x) (((x) & BIT_MASK_RARF_RC4) << BIT_SHIFT_RARF_RC4)
+#define BIT_GET_RARF_RC4(x) (((x) >> BIT_SHIFT_RARF_RC4) & BIT_MASK_RARF_RC4)
+
+#define BIT_SHIFT_RARF_RC3 16
+#define BIT_MASK_RARF_RC3 0x1f
+#define BIT_RARF_RC3(x) (((x) & BIT_MASK_RARF_RC3) << BIT_SHIFT_RARF_RC3)
+#define BIT_GET_RARF_RC3(x) (((x) >> BIT_SHIFT_RARF_RC3) & BIT_MASK_RARF_RC3)
+
+#define BIT_SHIFT_RARF_RC2 8
+#define BIT_MASK_RARF_RC2 0x1f
+#define BIT_RARF_RC2(x) (((x) & BIT_MASK_RARF_RC2) << BIT_SHIFT_RARF_RC2)
+#define BIT_GET_RARF_RC2(x) (((x) >> BIT_SHIFT_RARF_RC2) & BIT_MASK_RARF_RC2)
+
+#define BIT_SHIFT_RARF_RC1 0
+#define BIT_MASK_RARF_RC1 0x1f
+#define BIT_RARF_RC1(x) (((x) & BIT_MASK_RARF_RC1) << BIT_SHIFT_RARF_RC1)
+#define BIT_GET_RARF_RC1(x) (((x) >> BIT_SHIFT_RARF_RC1) & BIT_MASK_RARF_RC1)
+
+/* 2 REG_RRSR				(Offset 0x0440) */
+
+#define BIT_SHIFT_RRSR_RSC 21
+#define BIT_MASK_RRSR_RSC 0x3
+#define BIT_RRSR_RSC(x) (((x) & BIT_MASK_RRSR_RSC) << BIT_SHIFT_RRSR_RSC)
+#define BIT_GET_RRSR_RSC(x) (((x) >> BIT_SHIFT_RRSR_RSC) & BIT_MASK_RRSR_RSC)
+
+#define BIT_RRSR_BW BIT(20)
+
+#define BIT_SHIFT_RRSC_BITMAP 0
+#define BIT_MASK_RRSC_BITMAP 0xfffff
+#define BIT_RRSC_BITMAP(x)                                                     \
+	(((x) & BIT_MASK_RRSC_BITMAP) << BIT_SHIFT_RRSC_BITMAP)
+#define BIT_GET_RRSC_BITMAP(x)                                                 \
+	(((x) >> BIT_SHIFT_RRSC_BITMAP) & BIT_MASK_RRSC_BITMAP)
+
+/* 2 REG_ARFR0				(Offset 0x0444) */
+
+#define BIT_SHIFT_ARFR0_V1 0
+#define BIT_MASK_ARFR0_V1 0xffffffffffffffffL
+#define BIT_ARFR0_V1(x) (((x) & BIT_MASK_ARFR0_V1) << BIT_SHIFT_ARFR0_V1)
+#define BIT_GET_ARFR0_V1(x) (((x) >> BIT_SHIFT_ARFR0_V1) & BIT_MASK_ARFR0_V1)
+
+/* 2 REG_ARFR1_V1				(Offset 0x044C) */
+
+#define BIT_SHIFT_ARFR1_V1 0
+#define BIT_MASK_ARFR1_V1 0xffffffffffffffffL
+#define BIT_ARFR1_V1(x) (((x) & BIT_MASK_ARFR1_V1) << BIT_SHIFT_ARFR1_V1)
+#define BIT_GET_ARFR1_V1(x) (((x) >> BIT_SHIFT_ARFR1_V1) & BIT_MASK_ARFR1_V1)
+
+/* 2 REG_CCK_CHECK				(Offset 0x0454) */
+
+#define BIT_CHECK_CCK_EN BIT(7)
+#define BIT_EN_BCN_PKT_REL BIT(6)
+#define BIT_BCN_PORT_SEL BIT(5)
+#define BIT_MOREDATA_BYPASS BIT(4)
+#define BIT_EN_CLR_CMD_REL_BCN_PKT BIT(3)
+
+/* 2 REG_CCK_CHECK				(Offset 0x0454) */
+
+#define BIT_R_EN_SET_MOREDATA BIT(2)
+#define BIT__R_DIS_CLEAR_MACID_RELEASE BIT(1)
+#define BIT__R_MACID_RELEASE_EN BIT(0)
+
+/* 2 REG_AMPDU_MAX_TIME			(Offset 0x0456) */
+
+#define BIT_SHIFT_AMPDU_MAX_TIME 0
+#define BIT_MASK_AMPDU_MAX_TIME 0xff
+#define BIT_AMPDU_MAX_TIME(x)                                                  \
+	(((x) & BIT_MASK_AMPDU_MAX_TIME) << BIT_SHIFT_AMPDU_MAX_TIME)
+#define BIT_GET_AMPDU_MAX_TIME(x)                                              \
+	(((x) >> BIT_SHIFT_AMPDU_MAX_TIME) & BIT_MASK_AMPDU_MAX_TIME)
+
+/* 2 REG_BCNQ1_BDNY_V1			(Offset 0x0456) */
+
+#define BIT_SHIFT_BCNQ1_PGBNDY_V1 0
+#define BIT_MASK_BCNQ1_PGBNDY_V1 0xfff
+#define BIT_BCNQ1_PGBNDY_V1(x)                                                 \
+	(((x) & BIT_MASK_BCNQ1_PGBNDY_V1) << BIT_SHIFT_BCNQ1_PGBNDY_V1)
+#define BIT_GET_BCNQ1_PGBNDY_V1(x)                                             \
+	(((x) >> BIT_SHIFT_BCNQ1_PGBNDY_V1) & BIT_MASK_BCNQ1_PGBNDY_V1)
+
+/* 2 REG_AMPDU_MAX_LENGTH			(Offset 0x0458) */
+
+#define BIT_SHIFT_AMPDU_MAX_LENGTH 0
+#define BIT_MASK_AMPDU_MAX_LENGTH 0xffffffffL
+#define BIT_AMPDU_MAX_LENGTH(x)                                                \
+	(((x) & BIT_MASK_AMPDU_MAX_LENGTH) << BIT_SHIFT_AMPDU_MAX_LENGTH)
+#define BIT_GET_AMPDU_MAX_LENGTH(x)                                            \
+	(((x) >> BIT_SHIFT_AMPDU_MAX_LENGTH) & BIT_MASK_AMPDU_MAX_LENGTH)
+
+/* 2 REG_ACQ_STOP				(Offset 0x045C) */
+
+#define BIT_AC7Q_STOP BIT(7)
+#define BIT_AC6Q_STOP BIT(6)
+#define BIT_AC5Q_STOP BIT(5)
+#define BIT_AC4Q_STOP BIT(4)
+#define BIT_AC3Q_STOP BIT(3)
+#define BIT_AC2Q_STOP BIT(2)
+#define BIT_AC1Q_STOP BIT(1)
+#define BIT_AC0Q_STOP BIT(0)
+
+/* 2 REG_NDPA_RATE				(Offset 0x045D) */
+
+#define BIT_SHIFT_R_NDPA_RATE_V1 0
+#define BIT_MASK_R_NDPA_RATE_V1 0xff
+#define BIT_R_NDPA_RATE_V1(x)                                                  \
+	(((x) & BIT_MASK_R_NDPA_RATE_V1) << BIT_SHIFT_R_NDPA_RATE_V1)
+#define BIT_GET_R_NDPA_RATE_V1(x)                                              \
+	(((x) >> BIT_SHIFT_R_NDPA_RATE_V1) & BIT_MASK_R_NDPA_RATE_V1)
+
+/* 2 REG_TX_HANG_CTRL			(Offset 0x045E) */
+
+#define BIT_R_EN_GNT_BT_AWAKE BIT(3)
+
+/* 2 REG_TX_HANG_CTRL			(Offset 0x045E) */
+
+#define BIT_EN_EOF_V1 BIT(2)
+
+/* 2 REG_TX_HANG_CTRL			(Offset 0x045E) */
+
+#define BIT_DIS_OQT_BLOCK BIT(1)
+#define BIT_SEARCH_QUEUE_EN BIT(0)
+
+/* 2 REG_NDPA_OPT_CTRL			(Offset 0x045F) */
+
+#define BIT_R_DIS_MACID_RELEASE_RTY BIT(5)
+
+/* 2 REG_NDPA_OPT_CTRL			(Offset 0x045F) */
+
+#define BIT_SHIFT_BW_SIGTA 3
+#define BIT_MASK_BW_SIGTA 0x3
+#define BIT_BW_SIGTA(x) (((x) & BIT_MASK_BW_SIGTA) << BIT_SHIFT_BW_SIGTA)
+#define BIT_GET_BW_SIGTA(x) (((x) >> BIT_SHIFT_BW_SIGTA) & BIT_MASK_BW_SIGTA)
+
+/* 2 REG_NDPA_OPT_CTRL			(Offset 0x045F) */
+
+#define BIT_EN_BAR_SIGTA BIT(2)
+
+/* 2 REG_NDPA_OPT_CTRL			(Offset 0x045F) */
+
+#define BIT_SHIFT_R_NDPA_BW 0
+#define BIT_MASK_R_NDPA_BW 0x3
+#define BIT_R_NDPA_BW(x) (((x) & BIT_MASK_R_NDPA_BW) << BIT_SHIFT_R_NDPA_BW)
+#define BIT_GET_R_NDPA_BW(x) (((x) >> BIT_SHIFT_R_NDPA_BW) & BIT_MASK_R_NDPA_BW)
+
+/* 2 REG_RD_RESP_PKT_TH			(Offset 0x0463) */
+
+#define BIT_SHIFT_RD_RESP_PKT_TH_V1 0
+#define BIT_MASK_RD_RESP_PKT_TH_V1 0x3f
+#define BIT_RD_RESP_PKT_TH_V1(x)                                               \
+	(((x) & BIT_MASK_RD_RESP_PKT_TH_V1) << BIT_SHIFT_RD_RESP_PKT_TH_V1)
+#define BIT_GET_RD_RESP_PKT_TH_V1(x)                                           \
+	(((x) >> BIT_SHIFT_RD_RESP_PKT_TH_V1) & BIT_MASK_RD_RESP_PKT_TH_V1)
+
+/* 2 REG_CMDQ_INFO				(Offset 0x0464) */
+
+#define BIT_SHIFT_QUEUEMACID_CMDQ_V1 25
+#define BIT_MASK_QUEUEMACID_CMDQ_V1 0x7f
+#define BIT_QUEUEMACID_CMDQ_V1(x)                                              \
+	(((x) & BIT_MASK_QUEUEMACID_CMDQ_V1) << BIT_SHIFT_QUEUEMACID_CMDQ_V1)
+#define BIT_GET_QUEUEMACID_CMDQ_V1(x)                                          \
+	(((x) >> BIT_SHIFT_QUEUEMACID_CMDQ_V1) & BIT_MASK_QUEUEMACID_CMDQ_V1)
+
+/* 2 REG_CMDQ_INFO				(Offset 0x0464) */
+
+#define BIT_SHIFT_QUEUEAC_CMDQ_V1 23
+#define BIT_MASK_QUEUEAC_CMDQ_V1 0x3
+#define BIT_QUEUEAC_CMDQ_V1(x)                                                 \
+	(((x) & BIT_MASK_QUEUEAC_CMDQ_V1) << BIT_SHIFT_QUEUEAC_CMDQ_V1)
+#define BIT_GET_QUEUEAC_CMDQ_V1(x)                                             \
+	(((x) >> BIT_SHIFT_QUEUEAC_CMDQ_V1) & BIT_MASK_QUEUEAC_CMDQ_V1)
+
+/* 2 REG_CMDQ_INFO				(Offset 0x0464) */
+
+#define BIT_TIDEMPTY_CMDQ_V1 BIT(22)
+
+/* 2 REG_CMDQ_INFO				(Offset 0x0464) */
+
+#define BIT_SHIFT_TAIL_PKT_CMDQ_V2 11
+#define BIT_MASK_TAIL_PKT_CMDQ_V2 0x7ff
+#define BIT_TAIL_PKT_CMDQ_V2(x)                                                \
+	(((x) & BIT_MASK_TAIL_PKT_CMDQ_V2) << BIT_SHIFT_TAIL_PKT_CMDQ_V2)
+#define BIT_GET_TAIL_PKT_CMDQ_V2(x)                                            \
+	(((x) >> BIT_SHIFT_TAIL_PKT_CMDQ_V2) & BIT_MASK_TAIL_PKT_CMDQ_V2)
+
+/* 2 REG_CMDQ_INFO				(Offset 0x0464) */
+
+#define BIT_SHIFT_HEAD_PKT_CMDQ_V1 0
+#define BIT_MASK_HEAD_PKT_CMDQ_V1 0x7ff
+#define BIT_HEAD_PKT_CMDQ_V1(x)                                                \
+	(((x) & BIT_MASK_HEAD_PKT_CMDQ_V1) << BIT_SHIFT_HEAD_PKT_CMDQ_V1)
+#define BIT_GET_HEAD_PKT_CMDQ_V1(x)                                            \
+	(((x) >> BIT_SHIFT_HEAD_PKT_CMDQ_V1) & BIT_MASK_HEAD_PKT_CMDQ_V1)
+
+/* 2 REG_Q4_INFO				(Offset 0x0468) */
+
+#define BIT_SHIFT_QUEUEMACID_Q4_V1 25
+#define BIT_MASK_QUEUEMACID_Q4_V1 0x7f
+#define BIT_QUEUEMACID_Q4_V1(x)                                                \
+	(((x) & BIT_MASK_QUEUEMACID_Q4_V1) << BIT_SHIFT_QUEUEMACID_Q4_V1)
+#define BIT_GET_QUEUEMACID_Q4_V1(x)                                            \
+	(((x) >> BIT_SHIFT_QUEUEMACID_Q4_V1) & BIT_MASK_QUEUEMACID_Q4_V1)
+
+#define BIT_SHIFT_QUEUEAC_Q4_V1 23
+#define BIT_MASK_QUEUEAC_Q4_V1 0x3
+#define BIT_QUEUEAC_Q4_V1(x)                                                   \
+	(((x) & BIT_MASK_QUEUEAC_Q4_V1) << BIT_SHIFT_QUEUEAC_Q4_V1)
+#define BIT_GET_QUEUEAC_Q4_V1(x)                                               \
+	(((x) >> BIT_SHIFT_QUEUEAC_Q4_V1) & BIT_MASK_QUEUEAC_Q4_V1)
+
+/* 2 REG_Q4_INFO				(Offset 0x0468) */
+
+#define BIT_TIDEMPTY_Q4_V1 BIT(22)
+
+/* 2 REG_Q4_INFO				(Offset 0x0468) */
+
+#define BIT_SHIFT_TAIL_PKT_Q4_V2 11
+#define BIT_MASK_TAIL_PKT_Q4_V2 0x7ff
+#define BIT_TAIL_PKT_Q4_V2(x)                                                  \
+	(((x) & BIT_MASK_TAIL_PKT_Q4_V2) << BIT_SHIFT_TAIL_PKT_Q4_V2)
+#define BIT_GET_TAIL_PKT_Q4_V2(x)                                              \
+	(((x) >> BIT_SHIFT_TAIL_PKT_Q4_V2) & BIT_MASK_TAIL_PKT_Q4_V2)
+
+/* 2 REG_Q4_INFO				(Offset 0x0468) */
+
+#define BIT_SHIFT_HEAD_PKT_Q4_V1 0
+#define BIT_MASK_HEAD_PKT_Q4_V1 0x7ff
+#define BIT_HEAD_PKT_Q4_V1(x)                                                  \
+	(((x) & BIT_MASK_HEAD_PKT_Q4_V1) << BIT_SHIFT_HEAD_PKT_Q4_V1)
+#define BIT_GET_HEAD_PKT_Q4_V1(x)                                              \
+	(((x) >> BIT_SHIFT_HEAD_PKT_Q4_V1) & BIT_MASK_HEAD_PKT_Q4_V1)
+
+/* 2 REG_Q5_INFO				(Offset 0x046C) */
+
+#define BIT_SHIFT_QUEUEMACID_Q5_V1 25
+#define BIT_MASK_QUEUEMACID_Q5_V1 0x7f
+#define BIT_QUEUEMACID_Q5_V1(x)                                                \
+	(((x) & BIT_MASK_QUEUEMACID_Q5_V1) << BIT_SHIFT_QUEUEMACID_Q5_V1)
+#define BIT_GET_QUEUEMACID_Q5_V1(x)                                            \
+	(((x) >> BIT_SHIFT_QUEUEMACID_Q5_V1) & BIT_MASK_QUEUEMACID_Q5_V1)
+
+#define BIT_SHIFT_QUEUEAC_Q5_V1 23
+#define BIT_MASK_QUEUEAC_Q5_V1 0x3
+#define BIT_QUEUEAC_Q5_V1(x)                                                   \
+	(((x) & BIT_MASK_QUEUEAC_Q5_V1) << BIT_SHIFT_QUEUEAC_Q5_V1)
+#define BIT_GET_QUEUEAC_Q5_V1(x)                                               \
+	(((x) >> BIT_SHIFT_QUEUEAC_Q5_V1) & BIT_MASK_QUEUEAC_Q5_V1)
+
+/* 2 REG_Q5_INFO				(Offset 0x046C) */
+
+#define BIT_TIDEMPTY_Q5_V1 BIT(22)
+
+/* 2 REG_Q5_INFO				(Offset 0x046C) */
+
+#define BIT_SHIFT_TAIL_PKT_Q5_V2 11
+#define BIT_MASK_TAIL_PKT_Q5_V2 0x7ff
+#define BIT_TAIL_PKT_Q5_V2(x)                                                  \
+	(((x) & BIT_MASK_TAIL_PKT_Q5_V2) << BIT_SHIFT_TAIL_PKT_Q5_V2)
+#define BIT_GET_TAIL_PKT_Q5_V2(x)                                              \
+	(((x) >> BIT_SHIFT_TAIL_PKT_Q5_V2) & BIT_MASK_TAIL_PKT_Q5_V2)
+
+/* 2 REG_Q5_INFO				(Offset 0x046C) */
+
+#define BIT_SHIFT_HEAD_PKT_Q5_V1 0
+#define BIT_MASK_HEAD_PKT_Q5_V1 0x7ff
+#define BIT_HEAD_PKT_Q5_V1(x)                                                  \
+	(((x) & BIT_MASK_HEAD_PKT_Q5_V1) << BIT_SHIFT_HEAD_PKT_Q5_V1)
+#define BIT_GET_HEAD_PKT_Q5_V1(x)                                              \
+	(((x) >> BIT_SHIFT_HEAD_PKT_Q5_V1) & BIT_MASK_HEAD_PKT_Q5_V1)
+
+/* 2 REG_Q6_INFO				(Offset 0x0470) */
+
+#define BIT_SHIFT_QUEUEMACID_Q6_V1 25
+#define BIT_MASK_QUEUEMACID_Q6_V1 0x7f
+#define BIT_QUEUEMACID_Q6_V1(x)                                                \
+	(((x) & BIT_MASK_QUEUEMACID_Q6_V1) << BIT_SHIFT_QUEUEMACID_Q6_V1)
+#define BIT_GET_QUEUEMACID_Q6_V1(x)                                            \
+	(((x) >> BIT_SHIFT_QUEUEMACID_Q6_V1) & BIT_MASK_QUEUEMACID_Q6_V1)
+
+#define BIT_SHIFT_QUEUEAC_Q6_V1 23
+#define BIT_MASK_QUEUEAC_Q6_V1 0x3
+#define BIT_QUEUEAC_Q6_V1(x)                                                   \
+	(((x) & BIT_MASK_QUEUEAC_Q6_V1) << BIT_SHIFT_QUEUEAC_Q6_V1)
+#define BIT_GET_QUEUEAC_Q6_V1(x)                                               \
+	(((x) >> BIT_SHIFT_QUEUEAC_Q6_V1) & BIT_MASK_QUEUEAC_Q6_V1)
+
+/* 2 REG_Q6_INFO				(Offset 0x0470) */
+
+#define BIT_TIDEMPTY_Q6_V1 BIT(22)
+
+/* 2 REG_Q6_INFO				(Offset 0x0470) */
+
+#define BIT_SHIFT_TAIL_PKT_Q6_V2 11
+#define BIT_MASK_TAIL_PKT_Q6_V2 0x7ff
+#define BIT_TAIL_PKT_Q6_V2(x)                                                  \
+	(((x) & BIT_MASK_TAIL_PKT_Q6_V2) << BIT_SHIFT_TAIL_PKT_Q6_V2)
+#define BIT_GET_TAIL_PKT_Q6_V2(x)                                              \
+	(((x) >> BIT_SHIFT_TAIL_PKT_Q6_V2) & BIT_MASK_TAIL_PKT_Q6_V2)
+
+/* 2 REG_Q6_INFO				(Offset 0x0470) */
+
+#define BIT_SHIFT_HEAD_PKT_Q6_V1 0
+#define BIT_MASK_HEAD_PKT_Q6_V1 0x7ff
+#define BIT_HEAD_PKT_Q6_V1(x)                                                  \
+	(((x) & BIT_MASK_HEAD_PKT_Q6_V1) << BIT_SHIFT_HEAD_PKT_Q6_V1)
+#define BIT_GET_HEAD_PKT_Q6_V1(x)                                              \
+	(((x) >> BIT_SHIFT_HEAD_PKT_Q6_V1) & BIT_MASK_HEAD_PKT_Q6_V1)
+
+/* 2 REG_Q7_INFO				(Offset 0x0474) */
+
+#define BIT_SHIFT_QUEUEMACID_Q7_V1 25
+#define BIT_MASK_QUEUEMACID_Q7_V1 0x7f
+#define BIT_QUEUEMACID_Q7_V1(x)                                                \
+	(((x) & BIT_MASK_QUEUEMACID_Q7_V1) << BIT_SHIFT_QUEUEMACID_Q7_V1)
+#define BIT_GET_QUEUEMACID_Q7_V1(x)                                            \
+	(((x) >> BIT_SHIFT_QUEUEMACID_Q7_V1) & BIT_MASK_QUEUEMACID_Q7_V1)
+
+#define BIT_SHIFT_QUEUEAC_Q7_V1 23
+#define BIT_MASK_QUEUEAC_Q7_V1 0x3
+#define BIT_QUEUEAC_Q7_V1(x)                                                   \
+	(((x) & BIT_MASK_QUEUEAC_Q7_V1) << BIT_SHIFT_QUEUEAC_Q7_V1)
+#define BIT_GET_QUEUEAC_Q7_V1(x)                                               \
+	(((x) >> BIT_SHIFT_QUEUEAC_Q7_V1) & BIT_MASK_QUEUEAC_Q7_V1)
+
+/* 2 REG_Q7_INFO				(Offset 0x0474) */
+
+#define BIT_TIDEMPTY_Q7_V1 BIT(22)
+
+/* 2 REG_Q7_INFO				(Offset 0x0474) */
+
+#define BIT_SHIFT_TAIL_PKT_Q7_V2 11
+#define BIT_MASK_TAIL_PKT_Q7_V2 0x7ff
+#define BIT_TAIL_PKT_Q7_V2(x)                                                  \
+	(((x) & BIT_MASK_TAIL_PKT_Q7_V2) << BIT_SHIFT_TAIL_PKT_Q7_V2)
+#define BIT_GET_TAIL_PKT_Q7_V2(x)                                              \
+	(((x) >> BIT_SHIFT_TAIL_PKT_Q7_V2) & BIT_MASK_TAIL_PKT_Q7_V2)
+
+/* 2 REG_Q7_INFO				(Offset 0x0474) */
+
+#define BIT_SHIFT_HEAD_PKT_Q7_V1 0
+#define BIT_MASK_HEAD_PKT_Q7_V1 0x7ff
+#define BIT_HEAD_PKT_Q7_V1(x)                                                  \
+	(((x) & BIT_MASK_HEAD_PKT_Q7_V1) << BIT_SHIFT_HEAD_PKT_Q7_V1)
+#define BIT_GET_HEAD_PKT_Q7_V1(x)                                              \
+	(((x) >> BIT_SHIFT_HEAD_PKT_Q7_V1) & BIT_MASK_HEAD_PKT_Q7_V1)
+
+/* 2 REG_WMAC_LBK_BUF_HD_V1			(Offset 0x0478) */
+
+#define BIT_SHIFT_WMAC_LBK_BUF_HEAD_V1 0
+#define BIT_MASK_WMAC_LBK_BUF_HEAD_V1 0xfff
+#define BIT_WMAC_LBK_BUF_HEAD_V1(x)                                            \
+	(((x) & BIT_MASK_WMAC_LBK_BUF_HEAD_V1)                                 \
+	 << BIT_SHIFT_WMAC_LBK_BUF_HEAD_V1)
+#define BIT_GET_WMAC_LBK_BUF_HEAD_V1(x)                                        \
+	(((x) >> BIT_SHIFT_WMAC_LBK_BUF_HEAD_V1) &                             \
+	 BIT_MASK_WMAC_LBK_BUF_HEAD_V1)
+
+/* 2 REG_MGQ_BDNY_V1				(Offset 0x047A) */
+
+#define BIT_SHIFT_MGQ_PGBNDY_V1 0
+#define BIT_MASK_MGQ_PGBNDY_V1 0xfff
+#define BIT_MGQ_PGBNDY_V1(x)                                                   \
+	(((x) & BIT_MASK_MGQ_PGBNDY_V1) << BIT_SHIFT_MGQ_PGBNDY_V1)
+#define BIT_GET_MGQ_PGBNDY_V1(x)                                               \
+	(((x) >> BIT_SHIFT_MGQ_PGBNDY_V1) & BIT_MASK_MGQ_PGBNDY_V1)
+
+/* 2 REG_TXRPT_CTRL				(Offset 0x047C) */
+
+#define BIT_SHIFT_TRXRPT_TIMER_TH 24
+#define BIT_MASK_TRXRPT_TIMER_TH 0xff
+#define BIT_TRXRPT_TIMER_TH(x)                                                 \
+	(((x) & BIT_MASK_TRXRPT_TIMER_TH) << BIT_SHIFT_TRXRPT_TIMER_TH)
+#define BIT_GET_TRXRPT_TIMER_TH(x)                                             \
+	(((x) >> BIT_SHIFT_TRXRPT_TIMER_TH) & BIT_MASK_TRXRPT_TIMER_TH)
+
+/* 2 REG_TXRPT_CTRL				(Offset 0x047C) */
+
+#define BIT_SHIFT_TRXRPT_LEN_TH 16
+#define BIT_MASK_TRXRPT_LEN_TH 0xff
+#define BIT_TRXRPT_LEN_TH(x)                                                   \
+	(((x) & BIT_MASK_TRXRPT_LEN_TH) << BIT_SHIFT_TRXRPT_LEN_TH)
+#define BIT_GET_TRXRPT_LEN_TH(x)                                               \
+	(((x) >> BIT_SHIFT_TRXRPT_LEN_TH) & BIT_MASK_TRXRPT_LEN_TH)
+
+/* 2 REG_TXRPT_CTRL				(Offset 0x047C) */
+
+#define BIT_SHIFT_TRXRPT_READ_PTR 8
+#define BIT_MASK_TRXRPT_READ_PTR 0xff
+#define BIT_TRXRPT_READ_PTR(x)                                                 \
+	(((x) & BIT_MASK_TRXRPT_READ_PTR) << BIT_SHIFT_TRXRPT_READ_PTR)
+#define BIT_GET_TRXRPT_READ_PTR(x)                                             \
+	(((x) >> BIT_SHIFT_TRXRPT_READ_PTR) & BIT_MASK_TRXRPT_READ_PTR)
+
+/* 2 REG_TXRPT_CTRL				(Offset 0x047C) */
+
+#define BIT_SHIFT_TRXRPT_WRITE_PTR 0
+#define BIT_MASK_TRXRPT_WRITE_PTR 0xff
+#define BIT_TRXRPT_WRITE_PTR(x)                                                \
+	(((x) & BIT_MASK_TRXRPT_WRITE_PTR) << BIT_SHIFT_TRXRPT_WRITE_PTR)
+#define BIT_GET_TRXRPT_WRITE_PTR(x)                                            \
+	(((x) >> BIT_SHIFT_TRXRPT_WRITE_PTR) & BIT_MASK_TRXRPT_WRITE_PTR)
+
+/* 2 REG_INIRTS_RATE_SEL			(Offset 0x0480) */
+
+#define BIT_LEAG_RTS_BW_DUP BIT(5)
+
+/* 2 REG_BASIC_CFEND_RATE			(Offset 0x0481) */
+
+#define BIT_SHIFT_BASIC_CFEND_RATE 0
+#define BIT_MASK_BASIC_CFEND_RATE 0x1f
+#define BIT_BASIC_CFEND_RATE(x)                                                \
+	(((x) & BIT_MASK_BASIC_CFEND_RATE) << BIT_SHIFT_BASIC_CFEND_RATE)
+#define BIT_GET_BASIC_CFEND_RATE(x)                                            \
+	(((x) >> BIT_SHIFT_BASIC_CFEND_RATE) & BIT_MASK_BASIC_CFEND_RATE)
+
+/* 2 REG_STBC_CFEND_RATE			(Offset 0x0482) */
+
+#define BIT_SHIFT_STBC_CFEND_RATE 0
+#define BIT_MASK_STBC_CFEND_RATE 0x1f
+#define BIT_STBC_CFEND_RATE(x)                                                 \
+	(((x) & BIT_MASK_STBC_CFEND_RATE) << BIT_SHIFT_STBC_CFEND_RATE)
+#define BIT_GET_STBC_CFEND_RATE(x)                                             \
+	(((x) >> BIT_SHIFT_STBC_CFEND_RATE) & BIT_MASK_STBC_CFEND_RATE)
+
+/* 2 REG_DATA_SC				(Offset 0x0483) */
+
+#define BIT_SHIFT_TXSC_40M 4
+#define BIT_MASK_TXSC_40M 0xf
+#define BIT_TXSC_40M(x) (((x) & BIT_MASK_TXSC_40M) << BIT_SHIFT_TXSC_40M)
+#define BIT_GET_TXSC_40M(x) (((x) >> BIT_SHIFT_TXSC_40M) & BIT_MASK_TXSC_40M)
+
+#define BIT_SHIFT_TXSC_20M 0
+#define BIT_MASK_TXSC_20M 0xf
+#define BIT_TXSC_20M(x) (((x) & BIT_MASK_TXSC_20M) << BIT_SHIFT_TXSC_20M)
+#define BIT_GET_TXSC_20M(x) (((x) >> BIT_SHIFT_TXSC_20M) & BIT_MASK_TXSC_20M)
+
+/* 2 REG_MACID_SLEEP3			(Offset 0x0484) */
+
+#define BIT_SHIFT_MACID127_96_PKTSLEEP 0
+#define BIT_MASK_MACID127_96_PKTSLEEP 0xffffffffL
+#define BIT_MACID127_96_PKTSLEEP(x)                                            \
+	(((x) & BIT_MASK_MACID127_96_PKTSLEEP)                                 \
+	 << BIT_SHIFT_MACID127_96_PKTSLEEP)
+#define BIT_GET_MACID127_96_PKTSLEEP(x)                                        \
+	(((x) >> BIT_SHIFT_MACID127_96_PKTSLEEP) &                             \
+	 BIT_MASK_MACID127_96_PKTSLEEP)
+
+/* 2 REG_MACID_SLEEP1			(Offset 0x0488) */
+
+#define BIT_SHIFT_MACID63_32_PKTSLEEP 0
+#define BIT_MASK_MACID63_32_PKTSLEEP 0xffffffffL
+#define BIT_MACID63_32_PKTSLEEP(x)                                             \
+	(((x) & BIT_MASK_MACID63_32_PKTSLEEP) << BIT_SHIFT_MACID63_32_PKTSLEEP)
+#define BIT_GET_MACID63_32_PKTSLEEP(x)                                         \
+	(((x) >> BIT_SHIFT_MACID63_32_PKTSLEEP) & BIT_MASK_MACID63_32_PKTSLEEP)
+
+/* 2 REG_ARFR2_V1				(Offset 0x048C) */
+
+#define BIT_SHIFT_ARFR2_V1 0
+#define BIT_MASK_ARFR2_V1 0xffffffffffffffffL
+#define BIT_ARFR2_V1(x) (((x) & BIT_MASK_ARFR2_V1) << BIT_SHIFT_ARFR2_V1)
+#define BIT_GET_ARFR2_V1(x) (((x) >> BIT_SHIFT_ARFR2_V1) & BIT_MASK_ARFR2_V1)
+
+/* 2 REG_ARFR3_V1				(Offset 0x0494) */
+
+#define BIT_SHIFT_ARFR3_V1 0
+#define BIT_MASK_ARFR3_V1 0xffffffffffffffffL
+#define BIT_ARFR3_V1(x) (((x) & BIT_MASK_ARFR3_V1) << BIT_SHIFT_ARFR3_V1)
+#define BIT_GET_ARFR3_V1(x) (((x) >> BIT_SHIFT_ARFR3_V1) & BIT_MASK_ARFR3_V1)
+
+/* 2 REG_ARFR4				(Offset 0x049C) */
+
+#define BIT_SHIFT_ARFR4 0
+#define BIT_MASK_ARFR4 0xffffffffffffffffL
+#define BIT_ARFR4(x) (((x) & BIT_MASK_ARFR4) << BIT_SHIFT_ARFR4)
+#define BIT_GET_ARFR4(x) (((x) >> BIT_SHIFT_ARFR4) & BIT_MASK_ARFR4)
+
+/* 2 REG_ARFR5				(Offset 0x04A4) */
+
+#define BIT_SHIFT_ARFR5 0
+#define BIT_MASK_ARFR5 0xffffffffffffffffL
+#define BIT_ARFR5(x) (((x) & BIT_MASK_ARFR5) << BIT_SHIFT_ARFR5)
+#define BIT_GET_ARFR5(x) (((x) >> BIT_SHIFT_ARFR5) & BIT_MASK_ARFR5)
+
+/* 2 REG_TXRPT_START_OFFSET			(Offset 0x04AC) */
+
+#define BIT_SHIFT_MACID_MURATE_OFFSET 24
+#define BIT_MASK_MACID_MURATE_OFFSET 0xff
+#define BIT_MACID_MURATE_OFFSET(x)                                             \
+	(((x) & BIT_MASK_MACID_MURATE_OFFSET) << BIT_SHIFT_MACID_MURATE_OFFSET)
+#define BIT_GET_MACID_MURATE_OFFSET(x)                                         \
+	(((x) >> BIT_SHIFT_MACID_MURATE_OFFSET) & BIT_MASK_MACID_MURATE_OFFSET)
+
+/* 2 REG_TXRPT_START_OFFSET			(Offset 0x04AC) */
+
+#define BIT_RPTFIFO_SIZE_OPT BIT(16)
+
+/* 2 REG_TXRPT_START_OFFSET			(Offset 0x04AC) */
+
+#define BIT_SHIFT_MACID_CTRL_OFFSET 8
+#define BIT_MASK_MACID_CTRL_OFFSET 0xff
+#define BIT_MACID_CTRL_OFFSET(x)                                               \
+	(((x) & BIT_MASK_MACID_CTRL_OFFSET) << BIT_SHIFT_MACID_CTRL_OFFSET)
+#define BIT_GET_MACID_CTRL_OFFSET(x)                                           \
+	(((x) >> BIT_SHIFT_MACID_CTRL_OFFSET) & BIT_MASK_MACID_CTRL_OFFSET)
+
+/* 2 REG_TXRPT_START_OFFSET			(Offset 0x04AC) */
+
+#define BIT_SHIFT_AMPDU_TXRPT_OFFSET 0
+#define BIT_MASK_AMPDU_TXRPT_OFFSET 0xff
+#define BIT_AMPDU_TXRPT_OFFSET(x)                                              \
+	(((x) & BIT_MASK_AMPDU_TXRPT_OFFSET) << BIT_SHIFT_AMPDU_TXRPT_OFFSET)
+#define BIT_GET_AMPDU_TXRPT_OFFSET(x)                                          \
+	(((x) >> BIT_SHIFT_AMPDU_TXRPT_OFFSET) & BIT_MASK_AMPDU_TXRPT_OFFSET)
+
+/* 2 REG_POWER_STAGE1			(Offset 0x04B4) */
+
+#define BIT_PTA_WL_PRI_MASK_CPU_MGQ BIT(31)
+#define BIT_PTA_WL_PRI_MASK_BCNQ BIT(30)
+#define BIT_PTA_WL_PRI_MASK_HIQ BIT(29)
+#define BIT_PTA_WL_PRI_MASK_MGQ BIT(28)
+#define BIT_PTA_WL_PRI_MASK_BK BIT(27)
+#define BIT_PTA_WL_PRI_MASK_BE BIT(26)
+#define BIT_PTA_WL_PRI_MASK_VI BIT(25)
+#define BIT_PTA_WL_PRI_MASK_VO BIT(24)
+
+/* 2 REG_POWER_STAGE1			(Offset 0x04B4) */
+
+#define BIT_SHIFT_POWER_STAGE1 0
+#define BIT_MASK_POWER_STAGE1 0xffffff
+#define BIT_POWER_STAGE1(x)                                                    \
+	(((x) & BIT_MASK_POWER_STAGE1) << BIT_SHIFT_POWER_STAGE1)
+#define BIT_GET_POWER_STAGE1(x)                                                \
+	(((x) >> BIT_SHIFT_POWER_STAGE1) & BIT_MASK_POWER_STAGE1)
+
+/* 2 REG_POWER_STAGE2			(Offset 0x04B8) */
+
+#define BIT__R_CTRL_PKT_POW_ADJ BIT(24)
+
+/* 2 REG_POWER_STAGE2			(Offset 0x04B8) */
+
+#define BIT_SHIFT_POWER_STAGE2 0
+#define BIT_MASK_POWER_STAGE2 0xffffff
+#define BIT_POWER_STAGE2(x)                                                    \
+	(((x) & BIT_MASK_POWER_STAGE2) << BIT_SHIFT_POWER_STAGE2)
+#define BIT_GET_POWER_STAGE2(x)                                                \
+	(((x) >> BIT_SHIFT_POWER_STAGE2) & BIT_MASK_POWER_STAGE2)
+
+/* 2 REG_SW_AMPDU_BURST_MODE_CTRL		(Offset 0x04BC) */
+
+#define BIT_SHIFT_PAD_NUM_THRES 24
+#define BIT_MASK_PAD_NUM_THRES 0x3f
+#define BIT_PAD_NUM_THRES(x)                                                   \
+	(((x) & BIT_MASK_PAD_NUM_THRES) << BIT_SHIFT_PAD_NUM_THRES)
+#define BIT_GET_PAD_NUM_THRES(x)                                               \
+	(((x) >> BIT_SHIFT_PAD_NUM_THRES) & BIT_MASK_PAD_NUM_THRES)
+
+/* 2 REG_SW_AMPDU_BURST_MODE_CTRL		(Offset 0x04BC) */
+
+#define BIT_R_DMA_THIS_QUEUE_BK BIT(23)
+#define BIT_R_DMA_THIS_QUEUE_BE BIT(22)
+#define BIT_R_DMA_THIS_QUEUE_VI BIT(21)
+#define BIT_R_DMA_THIS_QUEUE_VO BIT(20)
+
+#define BIT_SHIFT_R_TOTAL_LEN_TH 8
+#define BIT_MASK_R_TOTAL_LEN_TH 0xfff
+#define BIT_R_TOTAL_LEN_TH(x)                                                  \
+	(((x) & BIT_MASK_R_TOTAL_LEN_TH) << BIT_SHIFT_R_TOTAL_LEN_TH)
+#define BIT_GET_R_TOTAL_LEN_TH(x)                                              \
+	(((x) >> BIT_SHIFT_R_TOTAL_LEN_TH) & BIT_MASK_R_TOTAL_LEN_TH)
+
+/* 2 REG_SW_AMPDU_BURST_MODE_CTRL		(Offset 0x04BC) */
+
+#define BIT_EN_NEW_EARLY BIT(7)
+
+/* 2 REG_SW_AMPDU_BURST_MODE_CTRL		(Offset 0x04BC) */
+
+#define BIT_PRE_TX_CMD BIT(6)
+
+#define BIT_SHIFT_NUM_SCL_EN 4
+#define BIT_MASK_NUM_SCL_EN 0x3
+#define BIT_NUM_SCL_EN(x) (((x) & BIT_MASK_NUM_SCL_EN) << BIT_SHIFT_NUM_SCL_EN)
+#define BIT_GET_NUM_SCL_EN(x)                                                  \
+	(((x) >> BIT_SHIFT_NUM_SCL_EN) & BIT_MASK_NUM_SCL_EN)
+
+#define BIT_BK_EN BIT(3)
+#define BIT_BE_EN BIT(2)
+#define BIT_VI_EN BIT(1)
+#define BIT_VO_EN BIT(0)
+
+/* 2 REG_PKT_LIFE_TIME			(Offset 0x04C0) */
+
+#define BIT_SHIFT_PKT_LIFTIME_BEBK 16
+#define BIT_MASK_PKT_LIFTIME_BEBK 0xffff
+#define BIT_PKT_LIFTIME_BEBK(x)                                                \
+	(((x) & BIT_MASK_PKT_LIFTIME_BEBK) << BIT_SHIFT_PKT_LIFTIME_BEBK)
+#define BIT_GET_PKT_LIFTIME_BEBK(x)                                            \
+	(((x) >> BIT_SHIFT_PKT_LIFTIME_BEBK) & BIT_MASK_PKT_LIFTIME_BEBK)
+
+#define BIT_SHIFT_PKT_LIFTIME_VOVI 0
+#define BIT_MASK_PKT_LIFTIME_VOVI 0xffff
+#define BIT_PKT_LIFTIME_VOVI(x)                                                \
+	(((x) & BIT_MASK_PKT_LIFTIME_VOVI) << BIT_SHIFT_PKT_LIFTIME_VOVI)
+#define BIT_GET_PKT_LIFTIME_VOVI(x)                                            \
+	(((x) >> BIT_SHIFT_PKT_LIFTIME_VOVI) & BIT_MASK_PKT_LIFTIME_VOVI)
+
+/* 2 REG_STBC_SETTING			(Offset 0x04C4) */
+
+#define BIT_SHIFT_CDEND_TXTIME_L 4
+#define BIT_MASK_CDEND_TXTIME_L 0xf
+#define BIT_CDEND_TXTIME_L(x)                                                  \
+	(((x) & BIT_MASK_CDEND_TXTIME_L) << BIT_SHIFT_CDEND_TXTIME_L)
+#define BIT_GET_CDEND_TXTIME_L(x)                                              \
+	(((x) >> BIT_SHIFT_CDEND_TXTIME_L) & BIT_MASK_CDEND_TXTIME_L)
+
+#define BIT_SHIFT_NESS 2
+#define BIT_MASK_NESS 0x3
+#define BIT_NESS(x) (((x) & BIT_MASK_NESS) << BIT_SHIFT_NESS)
+#define BIT_GET_NESS(x) (((x) >> BIT_SHIFT_NESS) & BIT_MASK_NESS)
+
+#define BIT_SHIFT_STBC_CFEND 0
+#define BIT_MASK_STBC_CFEND 0x3
+#define BIT_STBC_CFEND(x) (((x) & BIT_MASK_STBC_CFEND) << BIT_SHIFT_STBC_CFEND)
+#define BIT_GET_STBC_CFEND(x)                                                  \
+	(((x) >> BIT_SHIFT_STBC_CFEND) & BIT_MASK_STBC_CFEND)
+
+/* 2 REG_STBC_SETTING2			(Offset 0x04C5) */
+
+#define BIT_SHIFT_CDEND_TXTIME_H 0
+#define BIT_MASK_CDEND_TXTIME_H 0x1f
+#define BIT_CDEND_TXTIME_H(x)                                                  \
+	(((x) & BIT_MASK_CDEND_TXTIME_H) << BIT_SHIFT_CDEND_TXTIME_H)
+#define BIT_GET_CDEND_TXTIME_H(x)                                              \
+	(((x) >> BIT_SHIFT_CDEND_TXTIME_H) & BIT_MASK_CDEND_TXTIME_H)
+
+/* 2 REG_QUEUE_CTRL				(Offset 0x04C6) */
+
+#define BIT_PTA_EDCCA_EN BIT(5)
+#define BIT_PTA_WL_TX_EN BIT(4)
+
+/* 2 REG_QUEUE_CTRL				(Offset 0x04C6) */
+
+#define BIT_R_USE_DATA_BW BIT(3)
+#define BIT_TRI_PKT_INT_MODE1 BIT(2)
+#define BIT_TRI_PKT_INT_MODE0 BIT(1)
+#define BIT_ACQ_MODE_SEL BIT(0)
+
+/* 2 REG_SINGLE_AMPDU_CTRL			(Offset 0x04C7) */
+
+#define BIT_EN_SINGLE_APMDU BIT(7)
+
+/* 2 REG_PROT_MODE_CTRL			(Offset 0x04C8) */
+
+#define BIT_SHIFT_RTS_MAX_AGG_NUM 24
+#define BIT_MASK_RTS_MAX_AGG_NUM 0x3f
+#define BIT_RTS_MAX_AGG_NUM(x)                                                 \
+	(((x) & BIT_MASK_RTS_MAX_AGG_NUM) << BIT_SHIFT_RTS_MAX_AGG_NUM)
+#define BIT_GET_RTS_MAX_AGG_NUM(x)                                             \
+	(((x) >> BIT_SHIFT_RTS_MAX_AGG_NUM) & BIT_MASK_RTS_MAX_AGG_NUM)
+
+#define BIT_SHIFT_MAX_AGG_NUM 16
+#define BIT_MASK_MAX_AGG_NUM 0x3f
+#define BIT_MAX_AGG_NUM(x)                                                     \
+	(((x) & BIT_MASK_MAX_AGG_NUM) << BIT_SHIFT_MAX_AGG_NUM)
+#define BIT_GET_MAX_AGG_NUM(x)                                                 \
+	(((x) >> BIT_SHIFT_MAX_AGG_NUM) & BIT_MASK_MAX_AGG_NUM)
+
+#define BIT_SHIFT_RTS_TXTIME_TH 8
+#define BIT_MASK_RTS_TXTIME_TH 0xff
+#define BIT_RTS_TXTIME_TH(x)                                                   \
+	(((x) & BIT_MASK_RTS_TXTIME_TH) << BIT_SHIFT_RTS_TXTIME_TH)
+#define BIT_GET_RTS_TXTIME_TH(x)                                               \
+	(((x) >> BIT_SHIFT_RTS_TXTIME_TH) & BIT_MASK_RTS_TXTIME_TH)
+
+#define BIT_SHIFT_RTS_LEN_TH 0
+#define BIT_MASK_RTS_LEN_TH 0xff
+#define BIT_RTS_LEN_TH(x) (((x) & BIT_MASK_RTS_LEN_TH) << BIT_SHIFT_RTS_LEN_TH)
+#define BIT_GET_RTS_LEN_TH(x)                                                  \
+	(((x) >> BIT_SHIFT_RTS_LEN_TH) & BIT_MASK_RTS_LEN_TH)
+
+/* 2 REG_BAR_MODE_CTRL			(Offset 0x04CC) */
+
+#define BIT_SHIFT_BAR_RTY_LMT 16
+#define BIT_MASK_BAR_RTY_LMT 0x3
+#define BIT_BAR_RTY_LMT(x)                                                     \
+	(((x) & BIT_MASK_BAR_RTY_LMT) << BIT_SHIFT_BAR_RTY_LMT)
+#define BIT_GET_BAR_RTY_LMT(x)                                                 \
+	(((x) >> BIT_SHIFT_BAR_RTY_LMT) & BIT_MASK_BAR_RTY_LMT)
+
+#define BIT_SHIFT_BAR_PKT_TXTIME_TH 8
+#define BIT_MASK_BAR_PKT_TXTIME_TH 0xff
+#define BIT_BAR_PKT_TXTIME_TH(x)                                               \
+	(((x) & BIT_MASK_BAR_PKT_TXTIME_TH) << BIT_SHIFT_BAR_PKT_TXTIME_TH)
+#define BIT_GET_BAR_PKT_TXTIME_TH(x)                                           \
+	(((x) >> BIT_SHIFT_BAR_PKT_TXTIME_TH) & BIT_MASK_BAR_PKT_TXTIME_TH)
+
+#define BIT_BAR_EN_V1 BIT(6)
+
+#define BIT_SHIFT_BAR_PKTNUM_TH_V1 0
+#define BIT_MASK_BAR_PKTNUM_TH_V1 0x3f
+#define BIT_BAR_PKTNUM_TH_V1(x)                                                \
+	(((x) & BIT_MASK_BAR_PKTNUM_TH_V1) << BIT_SHIFT_BAR_PKTNUM_TH_V1)
+#define BIT_GET_BAR_PKTNUM_TH_V1(x)                                            \
+	(((x) >> BIT_SHIFT_BAR_PKTNUM_TH_V1) & BIT_MASK_BAR_PKTNUM_TH_V1)
+
+/* 2 REG_RA_TRY_RATE_AGG_LMT			(Offset 0x04CF) */
+
+#define BIT_SHIFT_RA_TRY_RATE_AGG_LMT_V1 0
+#define BIT_MASK_RA_TRY_RATE_AGG_LMT_V1 0x3f
+#define BIT_RA_TRY_RATE_AGG_LMT_V1(x)                                          \
+	(((x) & BIT_MASK_RA_TRY_RATE_AGG_LMT_V1)                               \
+	 << BIT_SHIFT_RA_TRY_RATE_AGG_LMT_V1)
+#define BIT_GET_RA_TRY_RATE_AGG_LMT_V1(x)                                      \
+	(((x) >> BIT_SHIFT_RA_TRY_RATE_AGG_LMT_V1) &                           \
+	 BIT_MASK_RA_TRY_RATE_AGG_LMT_V1)
+
+/* 2 REG_MACID_SLEEP2			(Offset 0x04D0) */
+
+#define BIT_SHIFT_MACID95_64PKTSLEEP 0
+#define BIT_MASK_MACID95_64PKTSLEEP 0xffffffffL
+#define BIT_MACID95_64PKTSLEEP(x)                                              \
+	(((x) & BIT_MASK_MACID95_64PKTSLEEP) << BIT_SHIFT_MACID95_64PKTSLEEP)
+#define BIT_GET_MACID95_64PKTSLEEP(x)                                          \
+	(((x) >> BIT_SHIFT_MACID95_64PKTSLEEP) & BIT_MASK_MACID95_64PKTSLEEP)
+
+/* 2 REG_MACID_SLEEP				(Offset 0x04D4) */
+
+#define BIT_SHIFT_MACID31_0_PKTSLEEP 0
+#define BIT_MASK_MACID31_0_PKTSLEEP 0xffffffffL
+#define BIT_MACID31_0_PKTSLEEP(x)                                              \
+	(((x) & BIT_MASK_MACID31_0_PKTSLEEP) << BIT_SHIFT_MACID31_0_PKTSLEEP)
+#define BIT_GET_MACID31_0_PKTSLEEP(x)                                          \
+	(((x) >> BIT_SHIFT_MACID31_0_PKTSLEEP) & BIT_MASK_MACID31_0_PKTSLEEP)
+
+/* 2 REG_HW_SEQ0				(Offset 0x04D8) */
+
+#define BIT_SHIFT_HW_SSN_SEQ0 0
+#define BIT_MASK_HW_SSN_SEQ0 0xfff
+#define BIT_HW_SSN_SEQ0(x)                                                     \
+	(((x) & BIT_MASK_HW_SSN_SEQ0) << BIT_SHIFT_HW_SSN_SEQ0)
+#define BIT_GET_HW_SSN_SEQ0(x)                                                 \
+	(((x) >> BIT_SHIFT_HW_SSN_SEQ0) & BIT_MASK_HW_SSN_SEQ0)
+
+/* 2 REG_HW_SEQ1				(Offset 0x04DA) */
+
+#define BIT_SHIFT_HW_SSN_SEQ1 0
+#define BIT_MASK_HW_SSN_SEQ1 0xfff
+#define BIT_HW_SSN_SEQ1(x)                                                     \
+	(((x) & BIT_MASK_HW_SSN_SEQ1) << BIT_SHIFT_HW_SSN_SEQ1)
+#define BIT_GET_HW_SSN_SEQ1(x)                                                 \
+	(((x) >> BIT_SHIFT_HW_SSN_SEQ1) & BIT_MASK_HW_SSN_SEQ1)
+
+/* 2 REG_HW_SEQ2				(Offset 0x04DC) */
+
+#define BIT_SHIFT_HW_SSN_SEQ2 0
+#define BIT_MASK_HW_SSN_SEQ2 0xfff
+#define BIT_HW_SSN_SEQ2(x)                                                     \
+	(((x) & BIT_MASK_HW_SSN_SEQ2) << BIT_SHIFT_HW_SSN_SEQ2)
+#define BIT_GET_HW_SSN_SEQ2(x)                                                 \
+	(((x) >> BIT_SHIFT_HW_SSN_SEQ2) & BIT_MASK_HW_SSN_SEQ2)
+
+/* 2 REG_HW_SEQ3				(Offset 0x04DE) */
+
+#define BIT_SHIFT_HW_SSN_SEQ3 0
+#define BIT_MASK_HW_SSN_SEQ3 0xfff
+#define BIT_HW_SSN_SEQ3(x)                                                     \
+	(((x) & BIT_MASK_HW_SSN_SEQ3) << BIT_SHIFT_HW_SSN_SEQ3)
+#define BIT_GET_HW_SSN_SEQ3(x)                                                 \
+	(((x) >> BIT_SHIFT_HW_SSN_SEQ3) & BIT_MASK_HW_SSN_SEQ3)
+
+/* 2 REG_NULL_PKT_STATUS_V1			(Offset 0x04E0) */
+
+#define BIT_SHIFT_PTCL_TOTAL_PG_V2 2
+#define BIT_MASK_PTCL_TOTAL_PG_V2 0x3fff
+#define BIT_PTCL_TOTAL_PG_V2(x)                                                \
+	(((x) & BIT_MASK_PTCL_TOTAL_PG_V2) << BIT_SHIFT_PTCL_TOTAL_PG_V2)
+#define BIT_GET_PTCL_TOTAL_PG_V2(x)                                            \
+	(((x) >> BIT_SHIFT_PTCL_TOTAL_PG_V2) & BIT_MASK_PTCL_TOTAL_PG_V2)
+
+/* 2 REG_NULL_PKT_STATUS			(Offset 0x04E0) */
+
+#define BIT_TX_NULL_1 BIT(1)
+#define BIT_TX_NULL_0 BIT(0)
+
+/* 2 REG_PTCL_ERR_STATUS			(Offset 0x04E2) */
+
+#define BIT_PTCL_RATE_TABLE_INVALID BIT(7)
+#define BIT_FTM_T2R_ERROR BIT(6)
+
+/* 2 REG_PTCL_ERR_STATUS			(Offset 0x04E2) */
+
+#define BIT_PTCL_ERR0 BIT(5)
+#define BIT_PTCL_ERR1 BIT(4)
+#define BIT_PTCL_ERR2 BIT(3)
+#define BIT_PTCL_ERR3 BIT(2)
+#define BIT_PTCL_ERR4 BIT(1)
+#define BIT_PTCL_ERR5 BIT(0)
+
+/* 2 REG_NULL_PKT_STATUS_EXTEND		(Offset 0x04E3) */
+
+#define BIT_CLI3_TX_NULL_1 BIT(7)
+#define BIT_CLI3_TX_NULL_0 BIT(6)
+#define BIT_CLI2_TX_NULL_1 BIT(5)
+#define BIT_CLI2_TX_NULL_0 BIT(4)
+#define BIT_CLI1_TX_NULL_1 BIT(3)
+#define BIT_CLI1_TX_NULL_0 BIT(2)
+#define BIT_CLI0_TX_NULL_1 BIT(1)
+
+/* 2 REG_NULL_PKT_STATUS_EXTEND		(Offset 0x04E3) */
+
+#define BIT_CLI0_TX_NULL_0 BIT(0)
+
+/* 2 REG_VIDEO_ENHANCEMENT_FUN		(Offset 0x04E4) */
+
+#define BIT_VIDEO_JUST_DROP BIT(1)
+#define BIT_VIDEO_ENHANCEMENT_FUN_EN BIT(0)
+
+/* 2 REG_BT_POLLUTE_PKT_CNT			(Offset 0x04E8) */
+
+#define BIT_SHIFT_BT_POLLUTE_PKT_CNT 0
+#define BIT_MASK_BT_POLLUTE_PKT_CNT 0xffff
+#define BIT_BT_POLLUTE_PKT_CNT(x)                                              \
+	(((x) & BIT_MASK_BT_POLLUTE_PKT_CNT) << BIT_SHIFT_BT_POLLUTE_PKT_CNT)
+#define BIT_GET_BT_POLLUTE_PKT_CNT(x)                                          \
+	(((x) >> BIT_SHIFT_BT_POLLUTE_PKT_CNT) & BIT_MASK_BT_POLLUTE_PKT_CNT)
+
+/* 2 REG_PTCL_DBG				(Offset 0x04EC) */
+
+#define BIT_SHIFT_PTCL_DBG 0
+#define BIT_MASK_PTCL_DBG 0xffffffffL
+#define BIT_PTCL_DBG(x) (((x) & BIT_MASK_PTCL_DBG) << BIT_SHIFT_PTCL_DBG)
+#define BIT_GET_PTCL_DBG(x) (((x) >> BIT_SHIFT_PTCL_DBG) & BIT_MASK_PTCL_DBG)
+
+/* 2 REG_CPUMGQ_TIMER_CTRL2			(Offset 0x04F4) */
+
+#define BIT_QUEUE_MACID_AC_NOT_THE_SAME BIT(31)
+
+#define BIT_SHIFT_GTAB_ID 28
+#define BIT_MASK_GTAB_ID 0x7
+#define BIT_GTAB_ID(x) (((x) & BIT_MASK_GTAB_ID) << BIT_SHIFT_GTAB_ID)
+#define BIT_GET_GTAB_ID(x) (((x) >> BIT_SHIFT_GTAB_ID) & BIT_MASK_GTAB_ID)
+
+#define BIT_SHIFT_TRI_HEAD_ADDR 16
+#define BIT_MASK_TRI_HEAD_ADDR 0xfff
+#define BIT_TRI_HEAD_ADDR(x)                                                   \
+	(((x) & BIT_MASK_TRI_HEAD_ADDR) << BIT_SHIFT_TRI_HEAD_ADDR)
+#define BIT_GET_TRI_HEAD_ADDR(x)                                               \
+	(((x) >> BIT_SHIFT_TRI_HEAD_ADDR) & BIT_MASK_TRI_HEAD_ADDR)
+
+#define BIT_QUEUE_MACID_AC_NOT_THE_SAME_V1 BIT(15)
+
+#define BIT_SHIFT_GTAB_ID_V1 12
+#define BIT_MASK_GTAB_ID_V1 0x7
+#define BIT_GTAB_ID_V1(x) (((x) & BIT_MASK_GTAB_ID_V1) << BIT_SHIFT_GTAB_ID_V1)
+#define BIT_GET_GTAB_ID_V1(x)                                                  \
+	(((x) >> BIT_SHIFT_GTAB_ID_V1) & BIT_MASK_GTAB_ID_V1)
+
+#define BIT_DROP_TH_EN BIT(8)
+
+#define BIT_SHIFT_DROP_TH 0
+#define BIT_MASK_DROP_TH 0xff
+#define BIT_DROP_TH(x) (((x) & BIT_MASK_DROP_TH) << BIT_SHIFT_DROP_TH)
+#define BIT_GET_DROP_TH(x) (((x) >> BIT_SHIFT_DROP_TH) & BIT_MASK_DROP_TH)
+
+/* 2 REG_DUMMY_PAGE4_V1			(Offset 0x04FC) */
+
+#define BIT_BCN_EN_EXTHWSEQ BIT(1)
+#define BIT_BCN_EN_HWSEQ BIT(0)
+
+/* 2 REG_MOREDATA				(Offset 0x04FE) */
+
+#define BIT_MOREDATA_CTRL2_EN_V1 BIT(3)
+#define BIT_MOREDATA_CTRL1_EN_V1 BIT(2)
+#define BIT_PKTIN_MOREDATA_REPLACE_ENABLE_V1 BIT(0)
+
+/* 2 REG_EDCA_VO_PARAM			(Offset 0x0500) */
+
+#define BIT_SHIFT_TXOPLIMIT 16
+#define BIT_MASK_TXOPLIMIT 0x7ff
+#define BIT_TXOPLIMIT(x) (((x) & BIT_MASK_TXOPLIMIT) << BIT_SHIFT_TXOPLIMIT)
+#define BIT_GET_TXOPLIMIT(x) (((x) >> BIT_SHIFT_TXOPLIMIT) & BIT_MASK_TXOPLIMIT)
+
+#define BIT_SHIFT_CW 8
+#define BIT_MASK_CW 0xff
+#define BIT_CW(x) (((x) & BIT_MASK_CW) << BIT_SHIFT_CW)
+#define BIT_GET_CW(x) (((x) >> BIT_SHIFT_CW) & BIT_MASK_CW)
+
+#define BIT_SHIFT_AIFS 0
+#define BIT_MASK_AIFS 0xff
+#define BIT_AIFS(x) (((x) & BIT_MASK_AIFS) << BIT_SHIFT_AIFS)
+#define BIT_GET_AIFS(x) (((x) >> BIT_SHIFT_AIFS) & BIT_MASK_AIFS)
+
+/* 2 REG_BCNTCFG				(Offset 0x0510) */
+
+#define BIT_SHIFT_BCNCW_MAX 12
+#define BIT_MASK_BCNCW_MAX 0xf
+#define BIT_BCNCW_MAX(x) (((x) & BIT_MASK_BCNCW_MAX) << BIT_SHIFT_BCNCW_MAX)
+#define BIT_GET_BCNCW_MAX(x) (((x) >> BIT_SHIFT_BCNCW_MAX) & BIT_MASK_BCNCW_MAX)
+
+#define BIT_SHIFT_BCNCW_MIN 8
+#define BIT_MASK_BCNCW_MIN 0xf
+#define BIT_BCNCW_MIN(x) (((x) & BIT_MASK_BCNCW_MIN) << BIT_SHIFT_BCNCW_MIN)
+#define BIT_GET_BCNCW_MIN(x) (((x) >> BIT_SHIFT_BCNCW_MIN) & BIT_MASK_BCNCW_MIN)
+
+#define BIT_SHIFT_BCNIFS 0
+#define BIT_MASK_BCNIFS 0xff
+#define BIT_BCNIFS(x) (((x) & BIT_MASK_BCNIFS) << BIT_SHIFT_BCNIFS)
+#define BIT_GET_BCNIFS(x) (((x) >> BIT_SHIFT_BCNIFS) & BIT_MASK_BCNIFS)
+
+/* 2 REG_PIFS				(Offset 0x0512) */
+
+#define BIT_SHIFT_PIFS 0
+#define BIT_MASK_PIFS 0xff
+#define BIT_PIFS(x) (((x) & BIT_MASK_PIFS) << BIT_SHIFT_PIFS)
+#define BIT_GET_PIFS(x) (((x) >> BIT_SHIFT_PIFS) & BIT_MASK_PIFS)
+
+/* 2 REG_RDG_PIFS				(Offset 0x0513) */
+
+#define BIT_SHIFT_RDG_PIFS 0
+#define BIT_MASK_RDG_PIFS 0xff
+#define BIT_RDG_PIFS(x) (((x) & BIT_MASK_RDG_PIFS) << BIT_SHIFT_RDG_PIFS)
+#define BIT_GET_RDG_PIFS(x) (((x) >> BIT_SHIFT_RDG_PIFS) & BIT_MASK_RDG_PIFS)
+
+/* 2 REG_SIFS				(Offset 0x0514) */
+
+#define BIT_SHIFT_SIFS_OFDM_TRX 24
+#define BIT_MASK_SIFS_OFDM_TRX 0xff
+#define BIT_SIFS_OFDM_TRX(x)                                                   \
+	(((x) & BIT_MASK_SIFS_OFDM_TRX) << BIT_SHIFT_SIFS_OFDM_TRX)
+#define BIT_GET_SIFS_OFDM_TRX(x)                                               \
+	(((x) >> BIT_SHIFT_SIFS_OFDM_TRX) & BIT_MASK_SIFS_OFDM_TRX)
+
+#define BIT_SHIFT_SIFS_CCK_TRX 16
+#define BIT_MASK_SIFS_CCK_TRX 0xff
+#define BIT_SIFS_CCK_TRX(x)                                                    \
+	(((x) & BIT_MASK_SIFS_CCK_TRX) << BIT_SHIFT_SIFS_CCK_TRX)
+#define BIT_GET_SIFS_CCK_TRX(x)                                                \
+	(((x) >> BIT_SHIFT_SIFS_CCK_TRX) & BIT_MASK_SIFS_CCK_TRX)
+
+#define BIT_SHIFT_SIFS_OFDM_CTX 8
+#define BIT_MASK_SIFS_OFDM_CTX 0xff
+#define BIT_SIFS_OFDM_CTX(x)                                                   \
+	(((x) & BIT_MASK_SIFS_OFDM_CTX) << BIT_SHIFT_SIFS_OFDM_CTX)
+#define BIT_GET_SIFS_OFDM_CTX(x)                                               \
+	(((x) >> BIT_SHIFT_SIFS_OFDM_CTX) & BIT_MASK_SIFS_OFDM_CTX)
+
+#define BIT_SHIFT_SIFS_CCK_CTX 0
+#define BIT_MASK_SIFS_CCK_CTX 0xff
+#define BIT_SIFS_CCK_CTX(x)                                                    \
+	(((x) & BIT_MASK_SIFS_CCK_CTX) << BIT_SHIFT_SIFS_CCK_CTX)
+#define BIT_GET_SIFS_CCK_CTX(x)                                                \
+	(((x) >> BIT_SHIFT_SIFS_CCK_CTX) & BIT_MASK_SIFS_CCK_CTX)
+
+/* 2 REG_TSFTR_SYN_OFFSET			(Offset 0x0518) */
+
+#define BIT_SHIFT_TSFTR_SNC_OFFSET 0
+#define BIT_MASK_TSFTR_SNC_OFFSET 0xffff
+#define BIT_TSFTR_SNC_OFFSET(x)                                                \
+	(((x) & BIT_MASK_TSFTR_SNC_OFFSET) << BIT_SHIFT_TSFTR_SNC_OFFSET)
+#define BIT_GET_TSFTR_SNC_OFFSET(x)                                            \
+	(((x) >> BIT_SHIFT_TSFTR_SNC_OFFSET) & BIT_MASK_TSFTR_SNC_OFFSET)
+
+/* 2 REG_AGGR_BREAK_TIME			(Offset 0x051A) */
+
+#define BIT_SHIFT_AGGR_BK_TIME 0
+#define BIT_MASK_AGGR_BK_TIME 0xff
+#define BIT_AGGR_BK_TIME(x)                                                    \
+	(((x) & BIT_MASK_AGGR_BK_TIME) << BIT_SHIFT_AGGR_BK_TIME)
+#define BIT_GET_AGGR_BK_TIME(x)                                                \
+	(((x) >> BIT_SHIFT_AGGR_BK_TIME) & BIT_MASK_AGGR_BK_TIME)
+
+/* 2 REG_SLOT				(Offset 0x051B) */
+
+#define BIT_SHIFT_SLOT 0
+#define BIT_MASK_SLOT 0xff
+#define BIT_SLOT(x) (((x) & BIT_MASK_SLOT) << BIT_SHIFT_SLOT)
+#define BIT_GET_SLOT(x) (((x) >> BIT_SHIFT_SLOT) & BIT_MASK_SLOT)
+
+/* 2 REG_TX_PTCL_CTRL			(Offset 0x0520) */
+
+#define BIT_DIS_EDCCA BIT(15)
+#define BIT_DIS_CCA BIT(14)
+#define BIT_LSIG_TXOP_TXCMD_NAV BIT(13)
+#define BIT_SIFS_BK_EN BIT(12)
+
+#define BIT_SHIFT_TXQ_NAV_MSK 8
+#define BIT_MASK_TXQ_NAV_MSK 0xf
+#define BIT_TXQ_NAV_MSK(x)                                                     \
+	(((x) & BIT_MASK_TXQ_NAV_MSK) << BIT_SHIFT_TXQ_NAV_MSK)
+#define BIT_GET_TXQ_NAV_MSK(x)                                                 \
+	(((x) >> BIT_SHIFT_TXQ_NAV_MSK) & BIT_MASK_TXQ_NAV_MSK)
+
+#define BIT_DIS_CW BIT(7)
+#define BIT_NAV_END_TXOP BIT(6)
+#define BIT_RDG_END_TXOP BIT(5)
+#define BIT_AC_INBCN_HOLD BIT(4)
+#define BIT_MGTQ_TXOP_EN BIT(3)
+#define BIT_MGTQ_RTSMF_EN BIT(2)
+#define BIT_HIQ_RTSMF_EN BIT(1)
+#define BIT_BCN_RTSMF_EN BIT(0)
+
+/* 2 REG_TXPAUSE				(Offset 0x0522) */
+
+#define BIT_STOP_BCN_HI_MGT BIT(7)
+#define BIT_MAC_STOPBCNQ BIT(6)
+#define BIT_MAC_STOPHIQ BIT(5)
+#define BIT_MAC_STOPMGQ BIT(4)
+#define BIT_MAC_STOPBK BIT(3)
+#define BIT_MAC_STOPBE BIT(2)
+#define BIT_MAC_STOPVI BIT(1)
+#define BIT_MAC_STOPVO BIT(0)
+
+/* 2 REG_DIS_TXREQ_CLR			(Offset 0x0523) */
+
+#define BIT_DIS_BT_CCA BIT(7)
+
+/* 2 REG_DIS_TXREQ_CLR			(Offset 0x0523) */
+
+#define BIT_DIS_TXREQ_CLR_HI BIT(5)
+#define BIT_DIS_TXREQ_CLR_MGQ BIT(4)
+#define BIT_DIS_TXREQ_CLR_VO BIT(3)
+#define BIT_DIS_TXREQ_CLR_VI BIT(2)
+#define BIT_DIS_TXREQ_CLR_BE BIT(1)
+#define BIT_DIS_TXREQ_CLR_BK BIT(0)
+
+/* 2 REG_RD_CTRL				(Offset 0x0524) */
+
+#define BIT_EN_CLR_TXREQ_INCCA BIT(15)
+#define BIT_DIS_TX_OVER_BCNQ BIT(14)
+
+/* 2 REG_RD_CTRL				(Offset 0x0524) */
+
+#define BIT_EN_BCNERR_INCCCA BIT(13)
+
+/* 2 REG_RD_CTRL				(Offset 0x0524) */
+
+#define BIT_EDCCA_MSK_CNTDOWN_EN BIT(11)
+#define BIT_DIS_TXOP_CFE BIT(10)
+#define BIT_DIS_LSIG_CFE BIT(9)
+#define BIT_DIS_STBC_CFE BIT(8)
+#define BIT_BKQ_RD_INIT_EN BIT(7)
+#define BIT_BEQ_RD_INIT_EN BIT(6)
+#define BIT_VIQ_RD_INIT_EN BIT(5)
+#define BIT_VOQ_RD_INIT_EN BIT(4)
+#define BIT_BKQ_RD_RESP_EN BIT(3)
+#define BIT_BEQ_RD_RESP_EN BIT(2)
+#define BIT_VIQ_RD_RESP_EN BIT(1)
+#define BIT_VOQ_RD_RESP_EN BIT(0)
+
+/* 2 REG_MBSSID_CTRL				(Offset 0x0526) */
+
+#define BIT_MBID_BCNQ7_EN BIT(7)
+#define BIT_MBID_BCNQ6_EN BIT(6)
+#define BIT_MBID_BCNQ5_EN BIT(5)
+#define BIT_MBID_BCNQ4_EN BIT(4)
+#define BIT_MBID_BCNQ3_EN BIT(3)
+#define BIT_MBID_BCNQ2_EN BIT(2)
+#define BIT_MBID_BCNQ1_EN BIT(1)
+#define BIT_MBID_BCNQ0_EN BIT(0)
+
+/* 2 REG_P2PPS_CTRL				(Offset 0x0527) */
+
+#define BIT_P2P_CTW_ALLSTASLEEP BIT(7)
+#define BIT_P2P_OFF_DISTX_EN BIT(6)
+#define BIT_PWR_MGT_EN BIT(5)
+
+/* 2 REG_P2PPS_CTRL				(Offset 0x0527) */
+
+#define BIT_P2P_NOA1_EN BIT(2)
+#define BIT_P2P_NOA0_EN BIT(1)
+
+/* 2 REG_PKT_LIFETIME_CTRL			(Offset 0x0528) */
+
+#define BIT_EN_P2P_CTWND1 BIT(23)
+
+/* 2 REG_PKT_LIFETIME_CTRL			(Offset 0x0528) */
+
+#define BIT_EN_BKF_CLR_TXREQ BIT(22)
+#define BIT_EN_TSFBIT32_RST_P2P BIT(21)
+#define BIT_EN_BCN_TX_BTCCA BIT(20)
+#define BIT_DIS_PKT_TX_ATIM BIT(19)
+#define BIT_DIS_BCN_DIS_CTN BIT(18)
+#define BIT_EN_NAVEND_RST_TXOP BIT(17)
+#define BIT_EN_FILTER_CCA BIT(16)
+
+#define BIT_SHIFT_CCA_FILTER_THRS 8
+#define BIT_MASK_CCA_FILTER_THRS 0xff
+#define BIT_CCA_FILTER_THRS(x)                                                 \
+	(((x) & BIT_MASK_CCA_FILTER_THRS) << BIT_SHIFT_CCA_FILTER_THRS)
+#define BIT_GET_CCA_FILTER_THRS(x)                                             \
+	(((x) >> BIT_SHIFT_CCA_FILTER_THRS) & BIT_MASK_CCA_FILTER_THRS)
+
+#define BIT_SHIFT_EDCCA_THRS 0
+#define BIT_MASK_EDCCA_THRS 0xff
+#define BIT_EDCCA_THRS(x) (((x) & BIT_MASK_EDCCA_THRS) << BIT_SHIFT_EDCCA_THRS)
+#define BIT_GET_EDCCA_THRS(x)                                                  \
+	(((x) >> BIT_SHIFT_EDCCA_THRS) & BIT_MASK_EDCCA_THRS)
+
+/* 2 REG_P2PPS_SPEC_STATE			(Offset 0x052B) */
+
+#define BIT_SPEC_POWER_STATE BIT(7)
+#define BIT_SPEC_CTWINDOW_ON BIT(6)
+#define BIT_SPEC_BEACON_AREA_ON BIT(5)
+#define BIT_SPEC_CTWIN_EARLY_DISTX BIT(4)
+#define BIT_SPEC_NOA1_OFF_PERIOD BIT(3)
+#define BIT_SPEC_FORCE_DOZE1 BIT(2)
+#define BIT_SPEC_NOA0_OFF_PERIOD BIT(1)
+#define BIT_SPEC_FORCE_DOZE0 BIT(0)
+
+/* 2 REG_QUEUE_INCOL_THR			(Offset 0x0538) */
+
+#define BIT_SHIFT_BK_QUEUE_THR 24
+#define BIT_MASK_BK_QUEUE_THR 0xff
+#define BIT_BK_QUEUE_THR(x)                                                    \
+	(((x) & BIT_MASK_BK_QUEUE_THR) << BIT_SHIFT_BK_QUEUE_THR)
+#define BIT_GET_BK_QUEUE_THR(x)                                                \
+	(((x) >> BIT_SHIFT_BK_QUEUE_THR) & BIT_MASK_BK_QUEUE_THR)
+
+#define BIT_SHIFT_BE_QUEUE_THR 16
+#define BIT_MASK_BE_QUEUE_THR 0xff
+#define BIT_BE_QUEUE_THR(x)                                                    \
+	(((x) & BIT_MASK_BE_QUEUE_THR) << BIT_SHIFT_BE_QUEUE_THR)
+#define BIT_GET_BE_QUEUE_THR(x)                                                \
+	(((x) >> BIT_SHIFT_BE_QUEUE_THR) & BIT_MASK_BE_QUEUE_THR)
+
+#define BIT_SHIFT_VI_QUEUE_THR 8
+#define BIT_MASK_VI_QUEUE_THR 0xff
+#define BIT_VI_QUEUE_THR(x)                                                    \
+	(((x) & BIT_MASK_VI_QUEUE_THR) << BIT_SHIFT_VI_QUEUE_THR)
+#define BIT_GET_VI_QUEUE_THR(x)                                                \
+	(((x) >> BIT_SHIFT_VI_QUEUE_THR) & BIT_MASK_VI_QUEUE_THR)
+
+#define BIT_SHIFT_VO_QUEUE_THR 0
+#define BIT_MASK_VO_QUEUE_THR 0xff
+#define BIT_VO_QUEUE_THR(x)                                                    \
+	(((x) & BIT_MASK_VO_QUEUE_THR) << BIT_SHIFT_VO_QUEUE_THR)
+#define BIT_GET_VO_QUEUE_THR(x)                                                \
+	(((x) >> BIT_SHIFT_VO_QUEUE_THR) & BIT_MASK_VO_QUEUE_THR)
+
+/* 2 REG_QUEUE_INCOL_EN			(Offset 0x053C) */
+
+#define BIT_QUEUE_INCOL_EN BIT(16)
+
+/* 2 REG_QUEUE_INCOL_EN			(Offset 0x053C) */
+
+#define BIT_SHIFT_BE_TRIGGER_NUM 12
+#define BIT_MASK_BE_TRIGGER_NUM 0xf
+#define BIT_BE_TRIGGER_NUM(x)                                                  \
+	(((x) & BIT_MASK_BE_TRIGGER_NUM) << BIT_SHIFT_BE_TRIGGER_NUM)
+#define BIT_GET_BE_TRIGGER_NUM(x)                                              \
+	(((x) >> BIT_SHIFT_BE_TRIGGER_NUM) & BIT_MASK_BE_TRIGGER_NUM)
+
+/* 2 REG_QUEUE_INCOL_EN			(Offset 0x053C) */
+
+#define BIT_SHIFT_BK_TRIGGER_NUM 8
+#define BIT_MASK_BK_TRIGGER_NUM 0xf
+#define BIT_BK_TRIGGER_NUM(x)                                                  \
+	(((x) & BIT_MASK_BK_TRIGGER_NUM) << BIT_SHIFT_BK_TRIGGER_NUM)
+#define BIT_GET_BK_TRIGGER_NUM(x)                                              \
+	(((x) >> BIT_SHIFT_BK_TRIGGER_NUM) & BIT_MASK_BK_TRIGGER_NUM)
+
+/* 2 REG_QUEUE_INCOL_EN			(Offset 0x053C) */
+
+#define BIT_SHIFT_VI_TRIGGER_NUM 4
+#define BIT_MASK_VI_TRIGGER_NUM 0xf
+#define BIT_VI_TRIGGER_NUM(x)                                                  \
+	(((x) & BIT_MASK_VI_TRIGGER_NUM) << BIT_SHIFT_VI_TRIGGER_NUM)
+#define BIT_GET_VI_TRIGGER_NUM(x)                                              \
+	(((x) >> BIT_SHIFT_VI_TRIGGER_NUM) & BIT_MASK_VI_TRIGGER_NUM)
+
+#define BIT_SHIFT_VO_TRIGGER_NUM 0
+#define BIT_MASK_VO_TRIGGER_NUM 0xf
+#define BIT_VO_TRIGGER_NUM(x)                                                  \
+	(((x) & BIT_MASK_VO_TRIGGER_NUM) << BIT_SHIFT_VO_TRIGGER_NUM)
+#define BIT_GET_VO_TRIGGER_NUM(x)                                              \
+	(((x) >> BIT_SHIFT_VO_TRIGGER_NUM) & BIT_MASK_VO_TRIGGER_NUM)
+
+/* 2 REG_TBTT_PROHIBIT			(Offset 0x0540) */
+
+#define BIT_SHIFT_TBTT_HOLD_TIME_AP 8
+#define BIT_MASK_TBTT_HOLD_TIME_AP 0xfff
+#define BIT_TBTT_HOLD_TIME_AP(x)                                               \
+	(((x) & BIT_MASK_TBTT_HOLD_TIME_AP) << BIT_SHIFT_TBTT_HOLD_TIME_AP)
+#define BIT_GET_TBTT_HOLD_TIME_AP(x)                                           \
+	(((x) >> BIT_SHIFT_TBTT_HOLD_TIME_AP) & BIT_MASK_TBTT_HOLD_TIME_AP)
+
+/* 2 REG_TBTT_PROHIBIT			(Offset 0x0540) */
+
+#define BIT_SHIFT_TBTT_PROHIBIT_SETUP 0
+#define BIT_MASK_TBTT_PROHIBIT_SETUP 0xf
+#define BIT_TBTT_PROHIBIT_SETUP(x)                                             \
+	(((x) & BIT_MASK_TBTT_PROHIBIT_SETUP) << BIT_SHIFT_TBTT_PROHIBIT_SETUP)
+#define BIT_GET_TBTT_PROHIBIT_SETUP(x)                                         \
+	(((x) >> BIT_SHIFT_TBTT_PROHIBIT_SETUP) & BIT_MASK_TBTT_PROHIBIT_SETUP)
+
+/* 2 REG_P2PPS_STATE				(Offset 0x0543) */
+
+#define BIT_POWER_STATE BIT(7)
+#define BIT_CTWINDOW_ON BIT(6)
+#define BIT_BEACON_AREA_ON BIT(5)
+#define BIT_CTWIN_EARLY_DISTX BIT(4)
+#define BIT_NOA1_OFF_PERIOD BIT(3)
+#define BIT_FORCE_DOZE1 BIT(2)
+#define BIT_NOA0_OFF_PERIOD BIT(1)
+#define BIT_FORCE_DOZE0 BIT(0)
+
+/* 2 REG_RD_NAV_NXT				(Offset 0x0544) */
+
+#define BIT_SHIFT_RD_NAV_PROT_NXT 0
+#define BIT_MASK_RD_NAV_PROT_NXT 0xffff
+#define BIT_RD_NAV_PROT_NXT(x)                                                 \
+	(((x) & BIT_MASK_RD_NAV_PROT_NXT) << BIT_SHIFT_RD_NAV_PROT_NXT)
+#define BIT_GET_RD_NAV_PROT_NXT(x)                                             \
+	(((x) >> BIT_SHIFT_RD_NAV_PROT_NXT) & BIT_MASK_RD_NAV_PROT_NXT)
+
+/* 2 REG_NAV_PROT_LEN			(Offset 0x0546) */
+
+#define BIT_SHIFT_NAV_PROT_LEN 0
+#define BIT_MASK_NAV_PROT_LEN 0xffff
+#define BIT_NAV_PROT_LEN(x)                                                    \
+	(((x) & BIT_MASK_NAV_PROT_LEN) << BIT_SHIFT_NAV_PROT_LEN)
+#define BIT_GET_NAV_PROT_LEN(x)                                                \
+	(((x) >> BIT_SHIFT_NAV_PROT_LEN) & BIT_MASK_NAV_PROT_LEN)
+
+/* 2 REG_BCN_CTRL				(Offset 0x0550) */
+
+#define BIT_DIS_RX_BSSID_FIT BIT(6)
+
+/* 2 REG_BCN_CTRL				(Offset 0x0550) */
+
+#define BIT_P0_EN_TXBCN_RPT BIT(5)
+
+/* 2 REG_BCN_CTRL				(Offset 0x0550) */
+
+#define BIT_DIS_TSF_UDT BIT(4)
+#define BIT_EN_BCN_FUNCTION BIT(3)
+
+/* 2 REG_BCN_CTRL				(Offset 0x0550) */
+
+#define BIT_P0_EN_RXBCN_RPT BIT(2)
+
+/* 2 REG_BCN_CTRL				(Offset 0x0550) */
+
+#define BIT_EN_P2P_CTWINDOW BIT(1)
+#define BIT_EN_P2P_BCNQ_AREA BIT(0)
+
+/* 2 REG_BCN_CTRL_CLINT0			(Offset 0x0551) */
+
+#define BIT_CLI0_DIS_RX_BSSID_FIT BIT(6)
+
+/* 2 REG_BCN_CTRL_CLINT0			(Offset 0x0551) */
+
+#define BIT_CLI0_DIS_TSF_UDT BIT(4)
+
+/* 2 REG_BCN_CTRL_CLINT0			(Offset 0x0551) */
+
+#define BIT_CLI0_EN_BCN_FUNCTION BIT(3)
+
+/* 2 REG_BCN_CTRL_CLINT0			(Offset 0x0551) */
+
+#define BIT_CLI0_EN_RXBCN_RPT BIT(2)
+
+/* 2 REG_BCN_CTRL_CLINT0			(Offset 0x0551) */
+
+#define BIT_CLI0_ENP2P_CTWINDOW BIT(1)
+#define BIT_CLI0_ENP2P_BCNQ_AREA BIT(0)
+
+/* 2 REG_MBID_NUM				(Offset 0x0552) */
+
+#define BIT_EN_PRE_DL_BEACON BIT(3)
+
+#define BIT_SHIFT_MBID_BCN_NUM 0
+#define BIT_MASK_MBID_BCN_NUM 0x7
+#define BIT_MBID_BCN_NUM(x)                                                    \
+	(((x) & BIT_MASK_MBID_BCN_NUM) << BIT_SHIFT_MBID_BCN_NUM)
+#define BIT_GET_MBID_BCN_NUM(x)                                                \
+	(((x) >> BIT_SHIFT_MBID_BCN_NUM) & BIT_MASK_MBID_BCN_NUM)
+
+/* 2 REG_DUAL_TSF_RST			(Offset 0x0553) */
+
+#define BIT_FREECNT_RST BIT(5)
+
+/* 2 REG_DUAL_TSF_RST			(Offset 0x0553) */
+
+#define BIT_TSFTR_CLI3_RST BIT(4)
+
+/* 2 REG_DUAL_TSF_RST			(Offset 0x0553) */
+
+#define BIT_TSFTR_CLI2_RST BIT(3)
+
+/* 2 REG_DUAL_TSF_RST			(Offset 0x0553) */
+
+#define BIT_TSFTR_CLI1_RST BIT(2)
+
+/* 2 REG_DUAL_TSF_RST			(Offset 0x0553) */
+
+#define BIT_TSFTR_CLI0_RST BIT(1)
+
+/* 2 REG_DUAL_TSF_RST			(Offset 0x0553) */
+
+#define BIT_TSFTR_RST BIT(0)
+
+/* 2 REG_MBSSID_BCN_SPACE			(Offset 0x0554) */
+
+#define BIT_SHIFT_BCN_TIMER_SEL_FWRD 28
+#define BIT_MASK_BCN_TIMER_SEL_FWRD 0x7
+#define BIT_BCN_TIMER_SEL_FWRD(x)                                              \
+	(((x) & BIT_MASK_BCN_TIMER_SEL_FWRD) << BIT_SHIFT_BCN_TIMER_SEL_FWRD)
+#define BIT_GET_BCN_TIMER_SEL_FWRD(x)                                          \
+	(((x) >> BIT_SHIFT_BCN_TIMER_SEL_FWRD) & BIT_MASK_BCN_TIMER_SEL_FWRD)
+
+/* 2 REG_MBSSID_BCN_SPACE			(Offset 0x0554) */
+
+#define BIT_SHIFT_BCN_SPACE_CLINT0 16
+#define BIT_MASK_BCN_SPACE_CLINT0 0xfff
+#define BIT_BCN_SPACE_CLINT0(x)                                                \
+	(((x) & BIT_MASK_BCN_SPACE_CLINT0) << BIT_SHIFT_BCN_SPACE_CLINT0)
+#define BIT_GET_BCN_SPACE_CLINT0(x)                                            \
+	(((x) >> BIT_SHIFT_BCN_SPACE_CLINT0) & BIT_MASK_BCN_SPACE_CLINT0)
+
+/* 2 REG_MBSSID_BCN_SPACE			(Offset 0x0554) */
+
+#define BIT_SHIFT_BCN_SPACE0 0
+#define BIT_MASK_BCN_SPACE0 0xffff
+#define BIT_BCN_SPACE0(x) (((x) & BIT_MASK_BCN_SPACE0) << BIT_SHIFT_BCN_SPACE0)
+#define BIT_GET_BCN_SPACE0(x)                                                  \
+	(((x) >> BIT_SHIFT_BCN_SPACE0) & BIT_MASK_BCN_SPACE0)
+
+/* 2 REG_DRVERLYINT				(Offset 0x0558) */
+
+#define BIT_SHIFT_DRVERLYITV 0
+#define BIT_MASK_DRVERLYITV 0xff
+#define BIT_DRVERLYITV(x) (((x) & BIT_MASK_DRVERLYITV) << BIT_SHIFT_DRVERLYITV)
+#define BIT_GET_DRVERLYITV(x)                                                  \
+	(((x) >> BIT_SHIFT_DRVERLYITV) & BIT_MASK_DRVERLYITV)
+
+/* 2 REG_BCNDMATIM				(Offset 0x0559) */
+
+#define BIT_SHIFT_BCNDMATIM 0
+#define BIT_MASK_BCNDMATIM 0xff
+#define BIT_BCNDMATIM(x) (((x) & BIT_MASK_BCNDMATIM) << BIT_SHIFT_BCNDMATIM)
+#define BIT_GET_BCNDMATIM(x) (((x) >> BIT_SHIFT_BCNDMATIM) & BIT_MASK_BCNDMATIM)
+
+/* 2 REG_ATIMWND				(Offset 0x055A) */
+
+#define BIT_SHIFT_ATIMWND0 0
+#define BIT_MASK_ATIMWND0 0xffff
+#define BIT_ATIMWND0(x) (((x) & BIT_MASK_ATIMWND0) << BIT_SHIFT_ATIMWND0)
+#define BIT_GET_ATIMWND0(x) (((x) >> BIT_SHIFT_ATIMWND0) & BIT_MASK_ATIMWND0)
+
+/* 2 REG_USTIME_TSF				(Offset 0x055C) */
+
+#define BIT_SHIFT_USTIME_TSF_V1 0
+#define BIT_MASK_USTIME_TSF_V1 0xff
+#define BIT_USTIME_TSF_V1(x)                                                   \
+	(((x) & BIT_MASK_USTIME_TSF_V1) << BIT_SHIFT_USTIME_TSF_V1)
+#define BIT_GET_USTIME_TSF_V1(x)                                               \
+	(((x) >> BIT_SHIFT_USTIME_TSF_V1) & BIT_MASK_USTIME_TSF_V1)
+
+/* 2 REG_BCN_MAX_ERR				(Offset 0x055D) */
+
+#define BIT_SHIFT_BCN_MAX_ERR 0
+#define BIT_MASK_BCN_MAX_ERR 0xff
+#define BIT_BCN_MAX_ERR(x)                                                     \
+	(((x) & BIT_MASK_BCN_MAX_ERR) << BIT_SHIFT_BCN_MAX_ERR)
+#define BIT_GET_BCN_MAX_ERR(x)                                                 \
+	(((x) >> BIT_SHIFT_BCN_MAX_ERR) & BIT_MASK_BCN_MAX_ERR)
+
+/* 2 REG_RXTSF_OFFSET_CCK			(Offset 0x055E) */
+
+#define BIT_SHIFT_CCK_RXTSF_OFFSET 0
+#define BIT_MASK_CCK_RXTSF_OFFSET 0xff
+#define BIT_CCK_RXTSF_OFFSET(x)                                                \
+	(((x) & BIT_MASK_CCK_RXTSF_OFFSET) << BIT_SHIFT_CCK_RXTSF_OFFSET)
+#define BIT_GET_CCK_RXTSF_OFFSET(x)                                            \
+	(((x) >> BIT_SHIFT_CCK_RXTSF_OFFSET) & BIT_MASK_CCK_RXTSF_OFFSET)
+
+/* 2 REG_RXTSF_OFFSET_OFDM			(Offset 0x055F) */
+
+#define BIT_SHIFT_OFDM_RXTSF_OFFSET 0
+#define BIT_MASK_OFDM_RXTSF_OFFSET 0xff
+#define BIT_OFDM_RXTSF_OFFSET(x)                                               \
+	(((x) & BIT_MASK_OFDM_RXTSF_OFFSET) << BIT_SHIFT_OFDM_RXTSF_OFFSET)
+#define BIT_GET_OFDM_RXTSF_OFFSET(x)                                           \
+	(((x) >> BIT_SHIFT_OFDM_RXTSF_OFFSET) & BIT_MASK_OFDM_RXTSF_OFFSET)
+
+/* 2 REG_TSFTR				(Offset 0x0560) */
+
+#define BIT_SHIFT_TSF_TIMER 0
+#define BIT_MASK_TSF_TIMER 0xffffffffffffffffL
+#define BIT_TSF_TIMER(x) (((x) & BIT_MASK_TSF_TIMER) << BIT_SHIFT_TSF_TIMER)
+#define BIT_GET_TSF_TIMER(x) (((x) >> BIT_SHIFT_TSF_TIMER) & BIT_MASK_TSF_TIMER)
+
+/* 2 REG_FREERUN_CNT				(Offset 0x0568) */
+
+#define BIT_SHIFT_FREERUN_CNT 0
+#define BIT_MASK_FREERUN_CNT 0xffffffffffffffffL
+#define BIT_FREERUN_CNT(x)                                                     \
+	(((x) & BIT_MASK_FREERUN_CNT) << BIT_SHIFT_FREERUN_CNT)
+#define BIT_GET_FREERUN_CNT(x)                                                 \
+	(((x) >> BIT_SHIFT_FREERUN_CNT) & BIT_MASK_FREERUN_CNT)
+
+/* 2 REG_ATIMWND1_V1				(Offset 0x0570) */
+
+#define BIT_SHIFT_ATIMWND1_V1 0
+#define BIT_MASK_ATIMWND1_V1 0xff
+#define BIT_ATIMWND1_V1(x)                                                     \
+	(((x) & BIT_MASK_ATIMWND1_V1) << BIT_SHIFT_ATIMWND1_V1)
+#define BIT_GET_ATIMWND1_V1(x)                                                 \
+	(((x) >> BIT_SHIFT_ATIMWND1_V1) & BIT_MASK_ATIMWND1_V1)
+
+/* 2 REG_TBTT_PROHIBIT_INFRA			(Offset 0x0571) */
+
+#define BIT_SHIFT_TBTT_PROHIBIT_INFRA 0
+#define BIT_MASK_TBTT_PROHIBIT_INFRA 0xff
+#define BIT_TBTT_PROHIBIT_INFRA(x)                                             \
+	(((x) & BIT_MASK_TBTT_PROHIBIT_INFRA) << BIT_SHIFT_TBTT_PROHIBIT_INFRA)
+#define BIT_GET_TBTT_PROHIBIT_INFRA(x)                                         \
+	(((x) >> BIT_SHIFT_TBTT_PROHIBIT_INFRA) & BIT_MASK_TBTT_PROHIBIT_INFRA)
+
+/* 2 REG_CTWND				(Offset 0x0572) */
+
+#define BIT_SHIFT_CTWND 0
+#define BIT_MASK_CTWND 0xff
+#define BIT_CTWND(x) (((x) & BIT_MASK_CTWND) << BIT_SHIFT_CTWND)
+#define BIT_GET_CTWND(x) (((x) >> BIT_SHIFT_CTWND) & BIT_MASK_CTWND)
+
+/* 2 REG_BCNIVLCUNT				(Offset 0x0573) */
+
+#define BIT_SHIFT_BCNIVLCUNT 0
+#define BIT_MASK_BCNIVLCUNT 0x7f
+#define BIT_BCNIVLCUNT(x) (((x) & BIT_MASK_BCNIVLCUNT) << BIT_SHIFT_BCNIVLCUNT)
+#define BIT_GET_BCNIVLCUNT(x)                                                  \
+	(((x) >> BIT_SHIFT_BCNIVLCUNT) & BIT_MASK_BCNIVLCUNT)
+
+/* 2 REG_BCNDROPCTRL				(Offset 0x0574) */
+
+#define BIT_BEACON_DROP_EN BIT(7)
+
+#define BIT_SHIFT_BEACON_DROP_IVL 0
+#define BIT_MASK_BEACON_DROP_IVL 0x7f
+#define BIT_BEACON_DROP_IVL(x)                                                 \
+	(((x) & BIT_MASK_BEACON_DROP_IVL) << BIT_SHIFT_BEACON_DROP_IVL)
+#define BIT_GET_BEACON_DROP_IVL(x)                                             \
+	(((x) >> BIT_SHIFT_BEACON_DROP_IVL) & BIT_MASK_BEACON_DROP_IVL)
+
+/* 2 REG_HGQ_TIMEOUT_PERIOD			(Offset 0x0575) */
+
+#define BIT_SHIFT_HGQ_TIMEOUT_PERIOD 0
+#define BIT_MASK_HGQ_TIMEOUT_PERIOD 0xff
+#define BIT_HGQ_TIMEOUT_PERIOD(x)                                              \
+	(((x) & BIT_MASK_HGQ_TIMEOUT_PERIOD) << BIT_SHIFT_HGQ_TIMEOUT_PERIOD)
+#define BIT_GET_HGQ_TIMEOUT_PERIOD(x)                                          \
+	(((x) >> BIT_SHIFT_HGQ_TIMEOUT_PERIOD) & BIT_MASK_HGQ_TIMEOUT_PERIOD)
+
+/* 2 REG_TXCMD_TIMEOUT_PERIOD		(Offset 0x0576) */
+
+#define BIT_SHIFT_TXCMD_TIMEOUT_PERIOD 0
+#define BIT_MASK_TXCMD_TIMEOUT_PERIOD 0xff
+#define BIT_TXCMD_TIMEOUT_PERIOD(x)                                            \
+	(((x) & BIT_MASK_TXCMD_TIMEOUT_PERIOD)                                 \
+	 << BIT_SHIFT_TXCMD_TIMEOUT_PERIOD)
+#define BIT_GET_TXCMD_TIMEOUT_PERIOD(x)                                        \
+	(((x) >> BIT_SHIFT_TXCMD_TIMEOUT_PERIOD) &                             \
+	 BIT_MASK_TXCMD_TIMEOUT_PERIOD)
+
+/* 2 REG_MISC_CTRL				(Offset 0x0577) */
+
+#define BIT_DIS_TRX_CAL_BCN BIT(5)
+#define BIT_DIS_TX_CAL_TBTT BIT(4)
+#define BIT_EN_FREECNT BIT(3)
+#define BIT_BCN_AGGRESSION BIT(2)
+
+#define BIT_SHIFT_DIS_SECONDARY_CCA 0
+#define BIT_MASK_DIS_SECONDARY_CCA 0x3
+#define BIT_DIS_SECONDARY_CCA(x)                                               \
+	(((x) & BIT_MASK_DIS_SECONDARY_CCA) << BIT_SHIFT_DIS_SECONDARY_CCA)
+#define BIT_GET_DIS_SECONDARY_CCA(x)                                           \
+	(((x) >> BIT_SHIFT_DIS_SECONDARY_CCA) & BIT_MASK_DIS_SECONDARY_CCA)
+
+/* 2 REG_BCN_CTRL_CLINT1			(Offset 0x0578) */
+
+#define BIT_CLI1_DIS_RX_BSSID_FIT BIT(6)
+#define BIT_CLI1_DIS_TSF_UDT BIT(4)
+#define BIT_CLI1_EN_BCN_FUNCTION BIT(3)
+
+/* 2 REG_BCN_CTRL_CLINT1			(Offset 0x0578) */
+
+#define BIT_CLI1_EN_RXBCN_RPT BIT(2)
+
+/* 2 REG_BCN_CTRL_CLINT1			(Offset 0x0578) */
+
+#define BIT_CLI1_ENP2P_CTWINDOW BIT(1)
+#define BIT_CLI1_ENP2P_BCNQ_AREA BIT(0)
+
+/* 2 REG_BCN_CTRL_CLINT2			(Offset 0x0579) */
+
+#define BIT_CLI2_DIS_RX_BSSID_FIT BIT(6)
+#define BIT_CLI2_DIS_TSF_UDT BIT(4)
+#define BIT_CLI2_EN_BCN_FUNCTION BIT(3)
+
+/* 2 REG_BCN_CTRL_CLINT2			(Offset 0x0579) */
+
+#define BIT_CLI2_EN_RXBCN_RPT BIT(2)
+
+/* 2 REG_BCN_CTRL_CLINT2			(Offset 0x0579) */
+
+#define BIT_CLI2_ENP2P_CTWINDOW BIT(1)
+#define BIT_CLI2_ENP2P_BCNQ_AREA BIT(0)
+
+/* 2 REG_BCN_CTRL_CLINT3			(Offset 0x057A) */
+
+#define BIT_CLI3_DIS_RX_BSSID_FIT BIT(6)
+#define BIT_CLI3_DIS_TSF_UDT BIT(4)
+#define BIT_CLI3_EN_BCN_FUNCTION BIT(3)
+
+/* 2 REG_BCN_CTRL_CLINT3			(Offset 0x057A) */
+
+#define BIT_CLI3_EN_RXBCN_RPT BIT(2)
+
+/* 2 REG_BCN_CTRL_CLINT3			(Offset 0x057A) */
+
+#define BIT_CLI3_ENP2P_CTWINDOW BIT(1)
+#define BIT_CLI3_ENP2P_BCNQ_AREA BIT(0)
+
+/* 2 REG_EXTEND_CTRL				(Offset 0x057B) */
+
+#define BIT_EN_TSFBIT32_RST_P2P2 BIT(5)
+#define BIT_EN_TSFBIT32_RST_P2P1 BIT(4)
+
+#define BIT_SHIFT_PORT_SEL 0
+#define BIT_MASK_PORT_SEL 0x7
+#define BIT_PORT_SEL(x) (((x) & BIT_MASK_PORT_SEL) << BIT_SHIFT_PORT_SEL)
+#define BIT_GET_PORT_SEL(x) (((x) >> BIT_SHIFT_PORT_SEL) & BIT_MASK_PORT_SEL)
+
+/* 2 REG_P2PPS1_SPEC_STATE			(Offset 0x057C) */
+
+#define BIT_P2P1_SPEC_POWER_STATE BIT(7)
+#define BIT_P2P1_SPEC_CTWINDOW_ON BIT(6)
+#define BIT_P2P1_SPEC_BCN_AREA_ON BIT(5)
+#define BIT_P2P1_SPEC_CTWIN_EARLY_DISTX BIT(4)
+#define BIT_P2P1_SPEC_NOA1_OFF_PERIOD BIT(3)
+#define BIT_P2P1_SPEC_FORCE_DOZE1 BIT(2)
+#define BIT_P2P1_SPEC_NOA0_OFF_PERIOD BIT(1)
+#define BIT_P2P1_SPEC_FORCE_DOZE0 BIT(0)
+
+/* 2 REG_P2PPS1_STATE			(Offset 0x057D) */
+
+#define BIT_P2P1_POWER_STATE BIT(7)
+#define BIT_P2P1_CTWINDOW_ON BIT(6)
+#define BIT_P2P1_BEACON_AREA_ON BIT(5)
+#define BIT_P2P1_CTWIN_EARLY_DISTX BIT(4)
+#define BIT_P2P1_NOA1_OFF_PERIOD BIT(3)
+#define BIT_P2P1_FORCE_DOZE1 BIT(2)
+#define BIT_P2P1_NOA0_OFF_PERIOD BIT(1)
+#define BIT_P2P1_FORCE_DOZE0 BIT(0)
+
+/* 2 REG_P2PPS2_SPEC_STATE			(Offset 0x057E) */
+
+#define BIT_P2P2_SPEC_POWER_STATE BIT(7)
+#define BIT_P2P2_SPEC_CTWINDOW_ON BIT(6)
+#define BIT_P2P2_SPEC_BCN_AREA_ON BIT(5)
+#define BIT_P2P2_SPEC_CTWIN_EARLY_DISTX BIT(4)
+#define BIT_P2P2_SPEC_NOA1_OFF_PERIOD BIT(3)
+#define BIT_P2P2_SPEC_FORCE_DOZE1 BIT(2)
+#define BIT_P2P2_SPEC_NOA0_OFF_PERIOD BIT(1)
+#define BIT_P2P2_SPEC_FORCE_DOZE0 BIT(0)
+
+/* 2 REG_P2PPS2_STATE			(Offset 0x057F) */
+
+#define BIT_P2P2_POWER_STATE BIT(7)
+#define BIT_P2P2_CTWINDOW_ON BIT(6)
+#define BIT_P2P2_BEACON_AREA_ON BIT(5)
+#define BIT_P2P2_CTWIN_EARLY_DISTX BIT(4)
+#define BIT_P2P2_NOA1_OFF_PERIOD BIT(3)
+#define BIT_P2P2_FORCE_DOZE1 BIT(2)
+#define BIT_P2P2_NOA0_OFF_PERIOD BIT(1)
+#define BIT_P2P2_FORCE_DOZE0 BIT(0)
+
+/* 2 REG_PS_TIMER0				(Offset 0x0580) */
+
+#define BIT_SHIFT_PSTIMER0_INT 5
+#define BIT_MASK_PSTIMER0_INT 0x7ffffff
+#define BIT_PSTIMER0_INT(x)                                                    \
+	(((x) & BIT_MASK_PSTIMER0_INT) << BIT_SHIFT_PSTIMER0_INT)
+#define BIT_GET_PSTIMER0_INT(x)                                                \
+	(((x) >> BIT_SHIFT_PSTIMER0_INT) & BIT_MASK_PSTIMER0_INT)
+
+/* 2 REG_PS_TIMER1				(Offset 0x0584) */
+
+#define BIT_SHIFT_PSTIMER1_INT 5
+#define BIT_MASK_PSTIMER1_INT 0x7ffffff
+#define BIT_PSTIMER1_INT(x)                                                    \
+	(((x) & BIT_MASK_PSTIMER1_INT) << BIT_SHIFT_PSTIMER1_INT)
+#define BIT_GET_PSTIMER1_INT(x)                                                \
+	(((x) >> BIT_SHIFT_PSTIMER1_INT) & BIT_MASK_PSTIMER1_INT)
+
+/* 2 REG_PS_TIMER2				(Offset 0x0588) */
+
+#define BIT_SHIFT_PSTIMER2_INT 5
+#define BIT_MASK_PSTIMER2_INT 0x7ffffff
+#define BIT_PSTIMER2_INT(x)                                                    \
+	(((x) & BIT_MASK_PSTIMER2_INT) << BIT_SHIFT_PSTIMER2_INT)
+#define BIT_GET_PSTIMER2_INT(x)                                                \
+	(((x) >> BIT_SHIFT_PSTIMER2_INT) & BIT_MASK_PSTIMER2_INT)
+
+/* 2 REG_TBTT_CTN_AREA			(Offset 0x058C) */
+
+#define BIT_SHIFT_TBTT_CTN_AREA 0
+#define BIT_MASK_TBTT_CTN_AREA 0xff
+#define BIT_TBTT_CTN_AREA(x)                                                   \
+	(((x) & BIT_MASK_TBTT_CTN_AREA) << BIT_SHIFT_TBTT_CTN_AREA)
+#define BIT_GET_TBTT_CTN_AREA(x)                                               \
+	(((x) >> BIT_SHIFT_TBTT_CTN_AREA) & BIT_MASK_TBTT_CTN_AREA)
+
+/* 2 REG_FORCE_BCN_IFS			(Offset 0x058E) */
+
+#define BIT_SHIFT_FORCE_BCN_IFS 0
+#define BIT_MASK_FORCE_BCN_IFS 0xff
+#define BIT_FORCE_BCN_IFS(x)                                                   \
+	(((x) & BIT_MASK_FORCE_BCN_IFS) << BIT_SHIFT_FORCE_BCN_IFS)
+#define BIT_GET_FORCE_BCN_IFS(x)                                               \
+	(((x) >> BIT_SHIFT_FORCE_BCN_IFS) & BIT_MASK_FORCE_BCN_IFS)
+
+/* 2 REG_TXOP_MIN				(Offset 0x0590) */
+
+#define BIT_SHIFT_TXOP_MIN 0
+#define BIT_MASK_TXOP_MIN 0x3fff
+#define BIT_TXOP_MIN(x) (((x) & BIT_MASK_TXOP_MIN) << BIT_SHIFT_TXOP_MIN)
+#define BIT_GET_TXOP_MIN(x) (((x) >> BIT_SHIFT_TXOP_MIN) & BIT_MASK_TXOP_MIN)
+
+/* 2 REG_PRE_BKF_TIME			(Offset 0x0592) */
+
+#define BIT_SHIFT_PRE_BKF_TIME 0
+#define BIT_MASK_PRE_BKF_TIME 0xff
+#define BIT_PRE_BKF_TIME(x)                                                    \
+	(((x) & BIT_MASK_PRE_BKF_TIME) << BIT_SHIFT_PRE_BKF_TIME)
+#define BIT_GET_PRE_BKF_TIME(x)                                                \
+	(((x) >> BIT_SHIFT_PRE_BKF_TIME) & BIT_MASK_PRE_BKF_TIME)
+
+/* 2 REG_CROSS_TXOP_CTRL			(Offset 0x0593) */
+
+#define BIT_DTIM_BYPASS BIT(2)
+#define BIT_RTS_NAV_TXOP BIT(1)
+#define BIT_NOT_CROSS_TXOP BIT(0)
+
+/* 2 REG_ATIMWND2				(Offset 0x05A0) */
+
+#define BIT_SHIFT_ATIMWND2 0
+#define BIT_MASK_ATIMWND2 0xff
+#define BIT_ATIMWND2(x) (((x) & BIT_MASK_ATIMWND2) << BIT_SHIFT_ATIMWND2)
+#define BIT_GET_ATIMWND2(x) (((x) >> BIT_SHIFT_ATIMWND2) & BIT_MASK_ATIMWND2)
+
+/* 2 REG_ATIMWND3				(Offset 0x05A1) */
+
+#define BIT_SHIFT_ATIMWND3 0
+#define BIT_MASK_ATIMWND3 0xff
+#define BIT_ATIMWND3(x) (((x) & BIT_MASK_ATIMWND3) << BIT_SHIFT_ATIMWND3)
+#define BIT_GET_ATIMWND3(x) (((x) >> BIT_SHIFT_ATIMWND3) & BIT_MASK_ATIMWND3)
+
+/* 2 REG_ATIMWND4				(Offset 0x05A2) */
+
+#define BIT_SHIFT_ATIMWND4 0
+#define BIT_MASK_ATIMWND4 0xff
+#define BIT_ATIMWND4(x) (((x) & BIT_MASK_ATIMWND4) << BIT_SHIFT_ATIMWND4)
+#define BIT_GET_ATIMWND4(x) (((x) >> BIT_SHIFT_ATIMWND4) & BIT_MASK_ATIMWND4)
+
+/* 2 REG_ATIMWND5				(Offset 0x05A3) */
+
+#define BIT_SHIFT_ATIMWND5 0
+#define BIT_MASK_ATIMWND5 0xff
+#define BIT_ATIMWND5(x) (((x) & BIT_MASK_ATIMWND5) << BIT_SHIFT_ATIMWND5)
+#define BIT_GET_ATIMWND5(x) (((x) >> BIT_SHIFT_ATIMWND5) & BIT_MASK_ATIMWND5)
+
+/* 2 REG_ATIMWND6				(Offset 0x05A4) */
+
+#define BIT_SHIFT_ATIMWND6 0
+#define BIT_MASK_ATIMWND6 0xff
+#define BIT_ATIMWND6(x) (((x) & BIT_MASK_ATIMWND6) << BIT_SHIFT_ATIMWND6)
+#define BIT_GET_ATIMWND6(x) (((x) >> BIT_SHIFT_ATIMWND6) & BIT_MASK_ATIMWND6)
+
+/* 2 REG_ATIMWND7				(Offset 0x05A5) */
+
+#define BIT_SHIFT_ATIMWND7 0
+#define BIT_MASK_ATIMWND7 0xff
+#define BIT_ATIMWND7(x) (((x) & BIT_MASK_ATIMWND7) << BIT_SHIFT_ATIMWND7)
+#define BIT_GET_ATIMWND7(x) (((x) >> BIT_SHIFT_ATIMWND7) & BIT_MASK_ATIMWND7)
+
+/* 2 REG_ATIMUGT				(Offset 0x05A6) */
+
+#define BIT_SHIFT_ATIM_URGENT 0
+#define BIT_MASK_ATIM_URGENT 0xff
+#define BIT_ATIM_URGENT(x)                                                     \
+	(((x) & BIT_MASK_ATIM_URGENT) << BIT_SHIFT_ATIM_URGENT)
+#define BIT_GET_ATIM_URGENT(x)                                                 \
+	(((x) >> BIT_SHIFT_ATIM_URGENT) & BIT_MASK_ATIM_URGENT)
+
+/* 2 REG_HIQ_NO_LMT_EN			(Offset 0x05A7) */
+
+#define BIT_HIQ_NO_LMT_EN_VAP7 BIT(7)
+#define BIT_HIQ_NO_LMT_EN_VAP6 BIT(6)
+#define BIT_HIQ_NO_LMT_EN_VAP5 BIT(5)
+#define BIT_HIQ_NO_LMT_EN_VAP4 BIT(4)
+#define BIT_HIQ_NO_LMT_EN_VAP3 BIT(3)
+#define BIT_HIQ_NO_LMT_EN_VAP2 BIT(2)
+#define BIT_HIQ_NO_LMT_EN_VAP1 BIT(1)
+#define BIT_HIQ_NO_LMT_EN_ROOT BIT(0)
+
+/* 2 REG_DTIM_COUNTER_ROOT			(Offset 0x05A8) */
+
+#define BIT_SHIFT_DTIM_COUNT_ROOT 0
+#define BIT_MASK_DTIM_COUNT_ROOT 0xff
+#define BIT_DTIM_COUNT_ROOT(x)                                                 \
+	(((x) & BIT_MASK_DTIM_COUNT_ROOT) << BIT_SHIFT_DTIM_COUNT_ROOT)
+#define BIT_GET_DTIM_COUNT_ROOT(x)                                             \
+	(((x) >> BIT_SHIFT_DTIM_COUNT_ROOT) & BIT_MASK_DTIM_COUNT_ROOT)
+
+/* 2 REG_DTIM_COUNTER_VAP1			(Offset 0x05A9) */
+
+#define BIT_SHIFT_DTIM_COUNT_VAP1 0
+#define BIT_MASK_DTIM_COUNT_VAP1 0xff
+#define BIT_DTIM_COUNT_VAP1(x)                                                 \
+	(((x) & BIT_MASK_DTIM_COUNT_VAP1) << BIT_SHIFT_DTIM_COUNT_VAP1)
+#define BIT_GET_DTIM_COUNT_VAP1(x)                                             \
+	(((x) >> BIT_SHIFT_DTIM_COUNT_VAP1) & BIT_MASK_DTIM_COUNT_VAP1)
+
+/* 2 REG_DTIM_COUNTER_VAP2			(Offset 0x05AA) */
+
+#define BIT_SHIFT_DTIM_COUNT_VAP2 0
+#define BIT_MASK_DTIM_COUNT_VAP2 0xff
+#define BIT_DTIM_COUNT_VAP2(x)                                                 \
+	(((x) & BIT_MASK_DTIM_COUNT_VAP2) << BIT_SHIFT_DTIM_COUNT_VAP2)
+#define BIT_GET_DTIM_COUNT_VAP2(x)                                             \
+	(((x) >> BIT_SHIFT_DTIM_COUNT_VAP2) & BIT_MASK_DTIM_COUNT_VAP2)
+
+/* 2 REG_DTIM_COUNTER_VAP3			(Offset 0x05AB) */
+
+#define BIT_SHIFT_DTIM_COUNT_VAP3 0
+#define BIT_MASK_DTIM_COUNT_VAP3 0xff
+#define BIT_DTIM_COUNT_VAP3(x)                                                 \
+	(((x) & BIT_MASK_DTIM_COUNT_VAP3) << BIT_SHIFT_DTIM_COUNT_VAP3)
+#define BIT_GET_DTIM_COUNT_VAP3(x)                                             \
+	(((x) >> BIT_SHIFT_DTIM_COUNT_VAP3) & BIT_MASK_DTIM_COUNT_VAP3)
+
+/* 2 REG_DTIM_COUNTER_VAP4			(Offset 0x05AC) */
+
+#define BIT_SHIFT_DTIM_COUNT_VAP4 0
+#define BIT_MASK_DTIM_COUNT_VAP4 0xff
+#define BIT_DTIM_COUNT_VAP4(x)                                                 \
+	(((x) & BIT_MASK_DTIM_COUNT_VAP4) << BIT_SHIFT_DTIM_COUNT_VAP4)
+#define BIT_GET_DTIM_COUNT_VAP4(x)                                             \
+	(((x) >> BIT_SHIFT_DTIM_COUNT_VAP4) & BIT_MASK_DTIM_COUNT_VAP4)
+
+/* 2 REG_DTIM_COUNTER_VAP5			(Offset 0x05AD) */
+
+#define BIT_SHIFT_DTIM_COUNT_VAP5 0
+#define BIT_MASK_DTIM_COUNT_VAP5 0xff
+#define BIT_DTIM_COUNT_VAP5(x)                                                 \
+	(((x) & BIT_MASK_DTIM_COUNT_VAP5) << BIT_SHIFT_DTIM_COUNT_VAP5)
+#define BIT_GET_DTIM_COUNT_VAP5(x)                                             \
+	(((x) >> BIT_SHIFT_DTIM_COUNT_VAP5) & BIT_MASK_DTIM_COUNT_VAP5)
+
+/* 2 REG_DTIM_COUNTER_VAP6			(Offset 0x05AE) */
+
+#define BIT_SHIFT_DTIM_COUNT_VAP6 0
+#define BIT_MASK_DTIM_COUNT_VAP6 0xff
+#define BIT_DTIM_COUNT_VAP6(x)                                                 \
+	(((x) & BIT_MASK_DTIM_COUNT_VAP6) << BIT_SHIFT_DTIM_COUNT_VAP6)
+#define BIT_GET_DTIM_COUNT_VAP6(x)                                             \
+	(((x) >> BIT_SHIFT_DTIM_COUNT_VAP6) & BIT_MASK_DTIM_COUNT_VAP6)
+
+/* 2 REG_DTIM_COUNTER_VAP7			(Offset 0x05AF) */
+
+#define BIT_SHIFT_DTIM_COUNT_VAP7 0
+#define BIT_MASK_DTIM_COUNT_VAP7 0xff
+#define BIT_DTIM_COUNT_VAP7(x)                                                 \
+	(((x) & BIT_MASK_DTIM_COUNT_VAP7) << BIT_SHIFT_DTIM_COUNT_VAP7)
+#define BIT_GET_DTIM_COUNT_VAP7(x)                                             \
+	(((x) >> BIT_SHIFT_DTIM_COUNT_VAP7) & BIT_MASK_DTIM_COUNT_VAP7)
+
+/* 2 REG_DIS_ATIM				(Offset 0x05B0) */
+
+#define BIT_DIS_ATIM_VAP7 BIT(7)
+#define BIT_DIS_ATIM_VAP6 BIT(6)
+#define BIT_DIS_ATIM_VAP5 BIT(5)
+#define BIT_DIS_ATIM_VAP4 BIT(4)
+#define BIT_DIS_ATIM_VAP3 BIT(3)
+#define BIT_DIS_ATIM_VAP2 BIT(2)
+#define BIT_DIS_ATIM_VAP1 BIT(1)
+#define BIT_DIS_ATIM_ROOT BIT(0)
+
+/* 2 REG_EARLY_128US				(Offset 0x05B1) */
+
+#define BIT_SHIFT_TSFT_SEL_TIMER1 3
+#define BIT_MASK_TSFT_SEL_TIMER1 0x7
+#define BIT_TSFT_SEL_TIMER1(x)                                                 \
+	(((x) & BIT_MASK_TSFT_SEL_TIMER1) << BIT_SHIFT_TSFT_SEL_TIMER1)
+#define BIT_GET_TSFT_SEL_TIMER1(x)                                             \
+	(((x) >> BIT_SHIFT_TSFT_SEL_TIMER1) & BIT_MASK_TSFT_SEL_TIMER1)
+
+#define BIT_SHIFT_EARLY_128US 0
+#define BIT_MASK_EARLY_128US 0x7
+#define BIT_EARLY_128US(x)                                                     \
+	(((x) & BIT_MASK_EARLY_128US) << BIT_SHIFT_EARLY_128US)
+#define BIT_GET_EARLY_128US(x)                                                 \
+	(((x) >> BIT_SHIFT_EARLY_128US) & BIT_MASK_EARLY_128US)
+
+/* 2 REG_P2PPS1_CTRL				(Offset 0x05B2) */
+
+#define BIT_P2P1_CTW_ALLSTASLEEP BIT(7)
+#define BIT_P2P1_OFF_DISTX_EN BIT(6)
+#define BIT_P2P1_PWR_MGT_EN BIT(5)
+#define BIT_P2P1_NOA1_EN BIT(2)
+#define BIT_P2P1_NOA0_EN BIT(1)
+
+/* 2 REG_P2PPS2_CTRL				(Offset 0x05B3) */
+
+#define BIT_P2P2_CTW_ALLSTASLEEP BIT(7)
+#define BIT_P2P2_OFF_DISTX_EN BIT(6)
+#define BIT_P2P2_PWR_MGT_EN BIT(5)
+#define BIT_P2P2_NOA1_EN BIT(2)
+#define BIT_P2P2_NOA0_EN BIT(1)
+
+/* 2 REG_TIMER0_SRC_SEL			(Offset 0x05B4) */
+
+#define BIT_SHIFT_SYNC_CLI_SEL 4
+#define BIT_MASK_SYNC_CLI_SEL 0x7
+#define BIT_SYNC_CLI_SEL(x)                                                    \
+	(((x) & BIT_MASK_SYNC_CLI_SEL) << BIT_SHIFT_SYNC_CLI_SEL)
+#define BIT_GET_SYNC_CLI_SEL(x)                                                \
+	(((x) >> BIT_SHIFT_SYNC_CLI_SEL) & BIT_MASK_SYNC_CLI_SEL)
+
+#define BIT_SHIFT_TSFT_SEL_TIMER0 0
+#define BIT_MASK_TSFT_SEL_TIMER0 0x7
+#define BIT_TSFT_SEL_TIMER0(x)                                                 \
+	(((x) & BIT_MASK_TSFT_SEL_TIMER0) << BIT_SHIFT_TSFT_SEL_TIMER0)
+#define BIT_GET_TSFT_SEL_TIMER0(x)                                             \
+	(((x) >> BIT_SHIFT_TSFT_SEL_TIMER0) & BIT_MASK_TSFT_SEL_TIMER0)
+
+/* 2 REG_NOA_UNIT_SEL			(Offset 0x05B5) */
+
+#define BIT_SHIFT_NOA_UNIT2_SEL 8
+#define BIT_MASK_NOA_UNIT2_SEL 0x7
+#define BIT_NOA_UNIT2_SEL(x)                                                   \
+	(((x) & BIT_MASK_NOA_UNIT2_SEL) << BIT_SHIFT_NOA_UNIT2_SEL)
+#define BIT_GET_NOA_UNIT2_SEL(x)                                               \
+	(((x) >> BIT_SHIFT_NOA_UNIT2_SEL) & BIT_MASK_NOA_UNIT2_SEL)
+
+#define BIT_SHIFT_NOA_UNIT1_SEL 4
+#define BIT_MASK_NOA_UNIT1_SEL 0x7
+#define BIT_NOA_UNIT1_SEL(x)                                                   \
+	(((x) & BIT_MASK_NOA_UNIT1_SEL) << BIT_SHIFT_NOA_UNIT1_SEL)
+#define BIT_GET_NOA_UNIT1_SEL(x)                                               \
+	(((x) >> BIT_SHIFT_NOA_UNIT1_SEL) & BIT_MASK_NOA_UNIT1_SEL)
+
+#define BIT_SHIFT_NOA_UNIT0_SEL 0
+#define BIT_MASK_NOA_UNIT0_SEL 0x7
+#define BIT_NOA_UNIT0_SEL(x)                                                   \
+	(((x) & BIT_MASK_NOA_UNIT0_SEL) << BIT_SHIFT_NOA_UNIT0_SEL)
+#define BIT_GET_NOA_UNIT0_SEL(x)                                               \
+	(((x) >> BIT_SHIFT_NOA_UNIT0_SEL) & BIT_MASK_NOA_UNIT0_SEL)
+
+/* 2 REG_P2POFF_DIS_TXTIME			(Offset 0x05B7) */
+
+#define BIT_SHIFT_P2POFF_DIS_TXTIME 0
+#define BIT_MASK_P2POFF_DIS_TXTIME 0xff
+#define BIT_P2POFF_DIS_TXTIME(x)                                               \
+	(((x) & BIT_MASK_P2POFF_DIS_TXTIME) << BIT_SHIFT_P2POFF_DIS_TXTIME)
+#define BIT_GET_P2POFF_DIS_TXTIME(x)                                           \
+	(((x) >> BIT_SHIFT_P2POFF_DIS_TXTIME) & BIT_MASK_P2POFF_DIS_TXTIME)
+
+/* 2 REG_MBSSID_BCN_SPACE2			(Offset 0x05B8) */
+
+#define BIT_SHIFT_BCN_SPACE_CLINT2 16
+#define BIT_MASK_BCN_SPACE_CLINT2 0xfff
+#define BIT_BCN_SPACE_CLINT2(x)                                                \
+	(((x) & BIT_MASK_BCN_SPACE_CLINT2) << BIT_SHIFT_BCN_SPACE_CLINT2)
+#define BIT_GET_BCN_SPACE_CLINT2(x)                                            \
+	(((x) >> BIT_SHIFT_BCN_SPACE_CLINT2) & BIT_MASK_BCN_SPACE_CLINT2)
+
+#define BIT_SHIFT_BCN_SPACE_CLINT1 0
+#define BIT_MASK_BCN_SPACE_CLINT1 0xfff
+#define BIT_BCN_SPACE_CLINT1(x)                                                \
+	(((x) & BIT_MASK_BCN_SPACE_CLINT1) << BIT_SHIFT_BCN_SPACE_CLINT1)
+#define BIT_GET_BCN_SPACE_CLINT1(x)                                            \
+	(((x) >> BIT_SHIFT_BCN_SPACE_CLINT1) & BIT_MASK_BCN_SPACE_CLINT1)
+
+/* 2 REG_MBSSID_BCN_SPACE3			(Offset 0x05BC) */
+
+#define BIT_SHIFT_SUB_BCN_SPACE 16
+#define BIT_MASK_SUB_BCN_SPACE 0xff
+#define BIT_SUB_BCN_SPACE(x)                                                   \
+	(((x) & BIT_MASK_SUB_BCN_SPACE) << BIT_SHIFT_SUB_BCN_SPACE)
+#define BIT_GET_SUB_BCN_SPACE(x)                                               \
+	(((x) >> BIT_SHIFT_SUB_BCN_SPACE) & BIT_MASK_SUB_BCN_SPACE)
+
+/* 2 REG_MBSSID_BCN_SPACE3			(Offset 0x05BC) */
+
+#define BIT_SHIFT_BCN_SPACE_CLINT3 0
+#define BIT_MASK_BCN_SPACE_CLINT3 0xfff
+#define BIT_BCN_SPACE_CLINT3(x)                                                \
+	(((x) & BIT_MASK_BCN_SPACE_CLINT3) << BIT_SHIFT_BCN_SPACE_CLINT3)
+#define BIT_GET_BCN_SPACE_CLINT3(x)                                            \
+	(((x) >> BIT_SHIFT_BCN_SPACE_CLINT3) & BIT_MASK_BCN_SPACE_CLINT3)
+
+/* 2 REG_ACMHWCTRL				(Offset 0x05C0) */
+
+#define BIT_BEQ_ACM_STATUS BIT(7)
+#define BIT_VIQ_ACM_STATUS BIT(6)
+#define BIT_VOQ_ACM_STATUS BIT(5)
+#define BIT_BEQ_ACM_EN BIT(3)
+#define BIT_VIQ_ACM_EN BIT(2)
+#define BIT_VOQ_ACM_EN BIT(1)
+#define BIT_ACMHWEN BIT(0)
+
+/* 2 REG_ACMRSTCTRL				(Offset 0x05C1) */
+
+#define BIT_BE_ACM_RESET_USED_TIME BIT(2)
+#define BIT_VI_ACM_RESET_USED_TIME BIT(1)
+#define BIT_VO_ACM_RESET_USED_TIME BIT(0)
+
+/* 2 REG_ACMAVG				(Offset 0x05C2) */
+
+#define BIT_SHIFT_AVGPERIOD 0
+#define BIT_MASK_AVGPERIOD 0xffff
+#define BIT_AVGPERIOD(x) (((x) & BIT_MASK_AVGPERIOD) << BIT_SHIFT_AVGPERIOD)
+#define BIT_GET_AVGPERIOD(x) (((x) >> BIT_SHIFT_AVGPERIOD) & BIT_MASK_AVGPERIOD)
+
+/* 2 REG_VO_ADMTIME				(Offset 0x05C4) */
+
+#define BIT_SHIFT_VO_ADMITTED_TIME 0
+#define BIT_MASK_VO_ADMITTED_TIME 0xffff
+#define BIT_VO_ADMITTED_TIME(x)                                                \
+	(((x) & BIT_MASK_VO_ADMITTED_TIME) << BIT_SHIFT_VO_ADMITTED_TIME)
+#define BIT_GET_VO_ADMITTED_TIME(x)                                            \
+	(((x) >> BIT_SHIFT_VO_ADMITTED_TIME) & BIT_MASK_VO_ADMITTED_TIME)
+
+/* 2 REG_VI_ADMTIME				(Offset 0x05C6) */
+
+#define BIT_SHIFT_VI_ADMITTED_TIME 0
+#define BIT_MASK_VI_ADMITTED_TIME 0xffff
+#define BIT_VI_ADMITTED_TIME(x)                                                \
+	(((x) & BIT_MASK_VI_ADMITTED_TIME) << BIT_SHIFT_VI_ADMITTED_TIME)
+#define BIT_GET_VI_ADMITTED_TIME(x)                                            \
+	(((x) >> BIT_SHIFT_VI_ADMITTED_TIME) & BIT_MASK_VI_ADMITTED_TIME)
+
+/* 2 REG_BE_ADMTIME				(Offset 0x05C8) */
+
+#define BIT_SHIFT_BE_ADMITTED_TIME 0
+#define BIT_MASK_BE_ADMITTED_TIME 0xffff
+#define BIT_BE_ADMITTED_TIME(x)                                                \
+	(((x) & BIT_MASK_BE_ADMITTED_TIME) << BIT_SHIFT_BE_ADMITTED_TIME)
+#define BIT_GET_BE_ADMITTED_TIME(x)                                            \
+	(((x) >> BIT_SHIFT_BE_ADMITTED_TIME) & BIT_MASK_BE_ADMITTED_TIME)
+
+/* 2 REG_EDCA_RANDOM_GEN			(Offset 0x05CC) */
+
+#define BIT_SHIFT_RANDOM_GEN 0
+#define BIT_MASK_RANDOM_GEN 0xffffff
+#define BIT_RANDOM_GEN(x) (((x) & BIT_MASK_RANDOM_GEN) << BIT_SHIFT_RANDOM_GEN)
+#define BIT_GET_RANDOM_GEN(x)                                                  \
+	(((x) >> BIT_SHIFT_RANDOM_GEN) & BIT_MASK_RANDOM_GEN)
+
+/* 2 REG_TXCMD_NOA_SEL			(Offset 0x05CF) */
+
+#define BIT_SHIFT_NOA_SEL 4
+#define BIT_MASK_NOA_SEL 0x7
+#define BIT_NOA_SEL(x) (((x) & BIT_MASK_NOA_SEL) << BIT_SHIFT_NOA_SEL)
+#define BIT_GET_NOA_SEL(x) (((x) >> BIT_SHIFT_NOA_SEL) & BIT_MASK_NOA_SEL)
+
+/* 2 REG_TXCMD_NOA_SEL			(Offset 0x05CF) */
+
+#define BIT_SHIFT_TXCMD_SEG_SEL 0
+#define BIT_MASK_TXCMD_SEG_SEL 0xf
+#define BIT_TXCMD_SEG_SEL(x)                                                   \
+	(((x) & BIT_MASK_TXCMD_SEG_SEL) << BIT_SHIFT_TXCMD_SEG_SEL)
+#define BIT_GET_TXCMD_SEG_SEL(x)                                               \
+	(((x) >> BIT_SHIFT_TXCMD_SEG_SEL) & BIT_MASK_TXCMD_SEG_SEL)
+
+/* 2 REG_NOA_PARAM				(Offset 0x05E0) */
+
+#define BIT_SHIFT_NOA_COUNT (96 & CPU_OPT_WIDTH)
+#define BIT_MASK_NOA_COUNT 0xff
+#define BIT_NOA_COUNT(x) (((x) & BIT_MASK_NOA_COUNT) << BIT_SHIFT_NOA_COUNT)
+#define BIT_GET_NOA_COUNT(x) (((x) >> BIT_SHIFT_NOA_COUNT) & BIT_MASK_NOA_COUNT)
+
+#define BIT_SHIFT_NOA_START_TIME (64 & CPU_OPT_WIDTH)
+#define BIT_MASK_NOA_START_TIME 0xffffffffL
+#define BIT_NOA_START_TIME(x)                                                  \
+	(((x) & BIT_MASK_NOA_START_TIME) << BIT_SHIFT_NOA_START_TIME)
+#define BIT_GET_NOA_START_TIME(x)                                              \
+	(((x) >> BIT_SHIFT_NOA_START_TIME) & BIT_MASK_NOA_START_TIME)
+
+#define BIT_SHIFT_NOA_INTERVAL (32 & CPU_OPT_WIDTH)
+#define BIT_MASK_NOA_INTERVAL 0xffffffffL
+#define BIT_NOA_INTERVAL(x)                                                    \
+	(((x) & BIT_MASK_NOA_INTERVAL) << BIT_SHIFT_NOA_INTERVAL)
+#define BIT_GET_NOA_INTERVAL(x)                                                \
+	(((x) >> BIT_SHIFT_NOA_INTERVAL) & BIT_MASK_NOA_INTERVAL)
+
+#define BIT_SHIFT_NOA_DURATION 0
+#define BIT_MASK_NOA_DURATION 0xffffffffL
+#define BIT_NOA_DURATION(x)                                                    \
+	(((x) & BIT_MASK_NOA_DURATION) << BIT_SHIFT_NOA_DURATION)
+#define BIT_GET_NOA_DURATION(x)                                                \
+	(((x) >> BIT_SHIFT_NOA_DURATION) & BIT_MASK_NOA_DURATION)
+
+/* 2 REG_P2P_RST				(Offset 0x05F0) */
+
+#define BIT_P2P2_PWR_RST1 BIT(5)
+#define BIT_P2P2_PWR_RST0 BIT(4)
+#define BIT_P2P1_PWR_RST1 BIT(3)
+#define BIT_P2P1_PWR_RST0 BIT(2)
+#define BIT_P2P_PWR_RST1_V1 BIT(1)
+#define BIT_P2P_PWR_RST0_V1 BIT(0)
+
+/* 2 REG_SCHEDULER_RST			(Offset 0x05F1) */
+
+#define BIT_SYNC_CLI BIT(1)
+#define BIT_SCHEDULER_RST_V1 BIT(0)
+
+/* 2 REG_SCH_TXCMD				(Offset 0x05F8) */
+
+#define BIT_SHIFT_SCH_TXCMD 0
+#define BIT_MASK_SCH_TXCMD 0xffffffffL
+#define BIT_SCH_TXCMD(x) (((x) & BIT_MASK_SCH_TXCMD) << BIT_SHIFT_SCH_TXCMD)
+#define BIT_GET_SCH_TXCMD(x) (((x) >> BIT_SHIFT_SCH_TXCMD) & BIT_MASK_SCH_TXCMD)
+
+/* 2 REG_WMAC_CR				(Offset 0x0600) */
+
+#define BIT_IC_MACPHY_M BIT(0)
+
+/* 2 REG_WMAC_FWPKT_CR			(Offset 0x0601) */
+
+#define BIT_FWEN BIT(7)
+
+/* 2 REG_WMAC_FWPKT_CR			(Offset 0x0601) */
+
+#define BIT_PHYSTS_PKT_CTRL BIT(6)
+
+/* 2 REG_WMAC_FWPKT_CR			(Offset 0x0601) */
+
+#define BIT_APPHDR_MIDSRCH_FAIL BIT(4)
+#define BIT_FWPARSING_EN BIT(3)
+
+#define BIT_SHIFT_APPEND_MHDR_LEN 0
+#define BIT_MASK_APPEND_MHDR_LEN 0x7
+#define BIT_APPEND_MHDR_LEN(x)                                                 \
+	(((x) & BIT_MASK_APPEND_MHDR_LEN) << BIT_SHIFT_APPEND_MHDR_LEN)
+#define BIT_GET_APPEND_MHDR_LEN(x)                                             \
+	(((x) >> BIT_SHIFT_APPEND_MHDR_LEN) & BIT_MASK_APPEND_MHDR_LEN)
+
+/* 2 REG_TCR					(Offset 0x0604) */
+
+#define BIT_WMAC_EN_RTS_ADDR BIT(31)
+#define BIT_WMAC_DISABLE_CCK BIT(30)
+#define BIT_WMAC_RAW_LEN BIT(29)
+#define BIT_WMAC_NOTX_IN_RXNDP BIT(28)
+#define BIT_WMAC_EN_EOF BIT(27)
+#define BIT_WMAC_BF_SEL BIT(26)
+#define BIT_WMAC_ANTMODE_SEL BIT(25)
+
+/* 2 REG_TCR					(Offset 0x0604) */
+
+#define BIT_WMAC_TCRPWRMGT_HWCTL BIT(24)
+
+/* 2 REG_TCR					(Offset 0x0604) */
+
+#define BIT_WMAC_SMOOTH_VAL BIT(23)
+
+/* 2 REG_TCR					(Offset 0x0604) */
+
+#define BIT_FETCH_MPDU_AFTER_WSEC_RDY BIT(20)
+
+/* 2 REG_TCR					(Offset 0x0604) */
+
+#define BIT_WMAC_TCR_EN_20MST BIT(19)
+#define BIT_WMAC_DIS_SIGTA BIT(18)
+#define BIT_WMAC_DIS_A2B0 BIT(17)
+#define BIT_WMAC_MSK_SIGBCRC BIT(16)
+
+/* 2 REG_TCR					(Offset 0x0604) */
+
+#define BIT_WMAC_TCR_ERRSTEN_3 BIT(15)
+#define BIT_WMAC_TCR_ERRSTEN_2 BIT(14)
+#define BIT_WMAC_TCR_ERRSTEN_1 BIT(13)
+#define BIT_WMAC_TCR_ERRSTEN_0 BIT(12)
+#define BIT_WMAC_TCR_TXSK_PERPKT BIT(11)
+#define BIT_ICV BIT(10)
+#define BIT_CFEND_FORMAT BIT(9)
+#define BIT_CRC BIT(8)
+#define BIT_PWRBIT_OW_EN BIT(7)
+#define BIT_PWR_ST BIT(6)
+#define BIT_WMAC_TCR_UPD_TIMIE BIT(5)
+#define BIT_WMAC_TCR_UPD_HGQMD BIT(4)
+
+/* 2 REG_TCR					(Offset 0x0604) */
+
+#define BIT_VHTSIGA1_TXPS BIT(3)
+
+/* 2 REG_TCR					(Offset 0x0604) */
+
+#define BIT_PAD_SEL BIT(2)
+#define BIT_DIS_GCLK BIT(1)
+
+/* 2 REG_RCR					(Offset 0x0608) */
+
+#define BIT_APP_FCS BIT(31)
+#define BIT_APP_MIC BIT(30)
+#define BIT_APP_ICV BIT(29)
+#define BIT_APP_PHYSTS BIT(28)
+#define BIT_APP_BASSN BIT(27)
+
+/* 2 REG_RCR					(Offset 0x0608) */
+
+#define BIT_VHT_DACK BIT(26)
+
+/* 2 REG_RCR					(Offset 0x0608) */
+
+#define BIT_TCPOFLD_EN BIT(25)
+#define BIT_ENMBID BIT(24)
+#define BIT_LSIGEN BIT(23)
+#define BIT_MFBEN BIT(22)
+#define BIT_DISCHKPPDLLEN BIT(21)
+#define BIT_PKTCTL_DLEN BIT(20)
+#define BIT_TIM_PARSER_EN BIT(18)
+#define BIT_BC_MD_EN BIT(17)
+#define BIT_UC_MD_EN BIT(16)
+#define BIT_RXSK_PERPKT BIT(15)
+#define BIT_HTC_LOC_CTRL BIT(14)
+
+/* 2 REG_RCR					(Offset 0x0608) */
+
+#define BIT_RPFM_CAM_ENABLE BIT(12)
+
+/* 2 REG_RCR					(Offset 0x0608) */
+
+#define BIT_TA_BCN BIT(11)
+
+/* 2 REG_RCR					(Offset 0x0608) */
+
+#define BIT_DISDECMYPKT BIT(10)
+#define BIT_AICV BIT(9)
+#define BIT_ACRC32 BIT(8)
+#define BIT_CBSSID_BCN BIT(7)
+#define BIT_CBSSID_DATA BIT(6)
+#define BIT_APWRMGT BIT(5)
+#define BIT_ADD3 BIT(4)
+#define BIT_AB BIT(3)
+#define BIT_AM BIT(2)
+#define BIT_APM BIT(1)
+#define BIT_AAP BIT(0)
+
+/* 2 REG_RX_PKT_LIMIT			(Offset 0x060C) */
+
+#define BIT_SHIFT_RXPKTLMT 0
+#define BIT_MASK_RXPKTLMT 0x3f
+#define BIT_RXPKTLMT(x) (((x) & BIT_MASK_RXPKTLMT) << BIT_SHIFT_RXPKTLMT)
+#define BIT_GET_RXPKTLMT(x) (((x) >> BIT_SHIFT_RXPKTLMT) & BIT_MASK_RXPKTLMT)
+
+/* 2 REG_RX_DLK_TIME				(Offset 0x060D) */
+
+#define BIT_SHIFT_RX_DLK_TIME 0
+#define BIT_MASK_RX_DLK_TIME 0xff
+#define BIT_RX_DLK_TIME(x)                                                     \
+	(((x) & BIT_MASK_RX_DLK_TIME) << BIT_SHIFT_RX_DLK_TIME)
+#define BIT_GET_RX_DLK_TIME(x)                                                 \
+	(((x) >> BIT_SHIFT_RX_DLK_TIME) & BIT_MASK_RX_DLK_TIME)
+
+/* 2 REG_RX_DRVINFO_SZ			(Offset 0x060F) */
+
+#define BIT_DATA_RPFM15EN BIT(15)
+#define BIT_DATA_RPFM14EN BIT(14)
+#define BIT_DATA_RPFM13EN BIT(13)
+#define BIT_DATA_RPFM12EN BIT(12)
+#define BIT_DATA_RPFM11EN BIT(11)
+#define BIT_DATA_RPFM10EN BIT(10)
+#define BIT_DATA_RPFM9EN BIT(9)
+#define BIT_DATA_RPFM8EN BIT(8)
+
+/* 2 REG_RX_DRVINFO_SZ			(Offset 0x060F) */
+
+#define BIT_PHYSTS_PER_PKT_MODE BIT(7)
+#define BIT_DATA_RPFM7EN BIT(7)
+
+/* 2 REG_RX_DRVINFO_SZ			(Offset 0x060F) */
+
+#define BIT_DATA_RPFM6EN BIT(6)
+
+/* 2 REG_RX_DRVINFO_SZ			(Offset 0x060F) */
+
+#define BIT_DATA_RPFM5EN BIT(5)
+#define BIT_DATA_RPFM4EN BIT(4)
+#define BIT_DATA_RPFM3EN BIT(3)
+#define BIT_DATA_RPFM2EN BIT(2)
+#define BIT_DATA_RPFM1EN BIT(1)
+
+/* 2 REG_RX_DRVINFO_SZ			(Offset 0x060F) */
+
+#define BIT_SHIFT_DRVINFO_SZ_V1 0
+#define BIT_MASK_DRVINFO_SZ_V1 0xf
+#define BIT_DRVINFO_SZ_V1(x)                                                   \
+	(((x) & BIT_MASK_DRVINFO_SZ_V1) << BIT_SHIFT_DRVINFO_SZ_V1)
+#define BIT_GET_DRVINFO_SZ_V1(x)                                               \
+	(((x) >> BIT_SHIFT_DRVINFO_SZ_V1) & BIT_MASK_DRVINFO_SZ_V1)
+
+/* 2 REG_RX_DRVINFO_SZ			(Offset 0x060F) */
+
+#define BIT_DATA_RPFM0EN BIT(0)
+
+/* 2 REG_MACID				(Offset 0x0610) */
+
+#define BIT_SHIFT_MACID 0
+#define BIT_MASK_MACID 0xffffffffffffL
+#define BIT_MACID(x) (((x) & BIT_MASK_MACID) << BIT_SHIFT_MACID)
+#define BIT_GET_MACID(x) (((x) >> BIT_SHIFT_MACID) & BIT_MASK_MACID)
+
+/* 2 REG_BSSID				(Offset 0x0618) */
+
+#define BIT_SHIFT_BSSID 0
+#define BIT_MASK_BSSID 0xffffffffffffL
+#define BIT_BSSID(x) (((x) & BIT_MASK_BSSID) << BIT_SHIFT_BSSID)
+#define BIT_GET_BSSID(x) (((x) >> BIT_SHIFT_BSSID) & BIT_MASK_BSSID)
+
+/* 2 REG_MAR					(Offset 0x0620) */
+
+#define BIT_SHIFT_MAR 0
+#define BIT_MASK_MAR 0xffffffffffffffffL
+#define BIT_MAR(x) (((x) & BIT_MASK_MAR) << BIT_SHIFT_MAR)
+#define BIT_GET_MAR(x) (((x) >> BIT_SHIFT_MAR) & BIT_MASK_MAR)
+
+/* 2 REG_MBIDCAMCFG_1			(Offset 0x0628) */
+
+#define BIT_SHIFT_MBIDCAM_RWDATA_L 0
+#define BIT_MASK_MBIDCAM_RWDATA_L 0xffffffffL
+#define BIT_MBIDCAM_RWDATA_L(x)                                                \
+	(((x) & BIT_MASK_MBIDCAM_RWDATA_L) << BIT_SHIFT_MBIDCAM_RWDATA_L)
+#define BIT_GET_MBIDCAM_RWDATA_L(x)                                            \
+	(((x) >> BIT_SHIFT_MBIDCAM_RWDATA_L) & BIT_MASK_MBIDCAM_RWDATA_L)
+
+/* 2 REG_MBIDCAMCFG_2			(Offset 0x062C) */
+
+#define BIT_MBIDCAM_POLL BIT(31)
+#define BIT_MBIDCAM_WT_EN BIT(30)
+
+#define BIT_SHIFT_MBIDCAM_ADDR 24
+#define BIT_MASK_MBIDCAM_ADDR 0x1f
+#define BIT_MBIDCAM_ADDR(x)                                                    \
+	(((x) & BIT_MASK_MBIDCAM_ADDR) << BIT_SHIFT_MBIDCAM_ADDR)
+#define BIT_GET_MBIDCAM_ADDR(x)                                                \
+	(((x) >> BIT_SHIFT_MBIDCAM_ADDR) & BIT_MASK_MBIDCAM_ADDR)
+
+#define BIT_MBIDCAM_VALID BIT(23)
+#define BIT_LSIC_TXOP_EN BIT(17)
+
+/* 2 REG_MBIDCAMCFG_2			(Offset 0x062C) */
+
+#define BIT_CTS_EN BIT(16)
+
+/* 2 REG_MBIDCAMCFG_2			(Offset 0x062C) */
+
+#define BIT_SHIFT_MBIDCAM_RWDATA_H 0
+#define BIT_MASK_MBIDCAM_RWDATA_H 0xffff
+#define BIT_MBIDCAM_RWDATA_H(x)                                                \
+	(((x) & BIT_MASK_MBIDCAM_RWDATA_H) << BIT_SHIFT_MBIDCAM_RWDATA_H)
+#define BIT_GET_MBIDCAM_RWDATA_H(x)                                            \
+	(((x) >> BIT_SHIFT_MBIDCAM_RWDATA_H) & BIT_MASK_MBIDCAM_RWDATA_H)
+
+/* 2 REG_WMAC_TCR_TSFT_OFS			(Offset 0x0630) */
+
+#define BIT_SHIFT_WMAC_TCR_TSFT_OFS 0
+#define BIT_MASK_WMAC_TCR_TSFT_OFS 0xffff
+#define BIT_WMAC_TCR_TSFT_OFS(x)                                               \
+	(((x) & BIT_MASK_WMAC_TCR_TSFT_OFS) << BIT_SHIFT_WMAC_TCR_TSFT_OFS)
+#define BIT_GET_WMAC_TCR_TSFT_OFS(x)                                           \
+	(((x) >> BIT_SHIFT_WMAC_TCR_TSFT_OFS) & BIT_MASK_WMAC_TCR_TSFT_OFS)
+
+/* 2 REG_UDF_THSD				(Offset 0x0632) */
+
+#define BIT_SHIFT_UDF_THSD 0
+#define BIT_MASK_UDF_THSD 0xff
+#define BIT_UDF_THSD(x) (((x) & BIT_MASK_UDF_THSD) << BIT_SHIFT_UDF_THSD)
+#define BIT_GET_UDF_THSD(x) (((x) >> BIT_SHIFT_UDF_THSD) & BIT_MASK_UDF_THSD)
+
+/* 2 REG_ZLD_NUM				(Offset 0x0633) */
+
+#define BIT_SHIFT_ZLD_NUM 0
+#define BIT_MASK_ZLD_NUM 0xff
+#define BIT_ZLD_NUM(x) (((x) & BIT_MASK_ZLD_NUM) << BIT_SHIFT_ZLD_NUM)
+#define BIT_GET_ZLD_NUM(x) (((x) >> BIT_SHIFT_ZLD_NUM) & BIT_MASK_ZLD_NUM)
+
+/* 2 REG_STMP_THSD				(Offset 0x0634) */
+
+#define BIT_SHIFT_STMP_THSD 0
+#define BIT_MASK_STMP_THSD 0xff
+#define BIT_STMP_THSD(x) (((x) & BIT_MASK_STMP_THSD) << BIT_SHIFT_STMP_THSD)
+#define BIT_GET_STMP_THSD(x) (((x) >> BIT_SHIFT_STMP_THSD) & BIT_MASK_STMP_THSD)
+
+/* 2 REG_WMAC_TXTIMEOUT			(Offset 0x0635) */
+
+#define BIT_SHIFT_WMAC_TXTIMEOUT 0
+#define BIT_MASK_WMAC_TXTIMEOUT 0xff
+#define BIT_WMAC_TXTIMEOUT(x)                                                  \
+	(((x) & BIT_MASK_WMAC_TXTIMEOUT) << BIT_SHIFT_WMAC_TXTIMEOUT)
+#define BIT_GET_WMAC_TXTIMEOUT(x)                                              \
+	(((x) >> BIT_SHIFT_WMAC_TXTIMEOUT) & BIT_MASK_WMAC_TXTIMEOUT)
+
+/* 2 REG_MCU_TEST_2_V1			(Offset 0x0636) */
+
+#define BIT_SHIFT_MCU_RSVD_2_V1 0
+#define BIT_MASK_MCU_RSVD_2_V1 0xffff
+#define BIT_MCU_RSVD_2_V1(x)                                                   \
+	(((x) & BIT_MASK_MCU_RSVD_2_V1) << BIT_SHIFT_MCU_RSVD_2_V1)
+#define BIT_GET_MCU_RSVD_2_V1(x)                                               \
+	(((x) >> BIT_SHIFT_MCU_RSVD_2_V1) & BIT_MASK_MCU_RSVD_2_V1)
+
+/* 2 REG_USTIME_EDCA				(Offset 0x0638) */
+
+#define BIT_SHIFT_USTIME_EDCA_V1 0
+#define BIT_MASK_USTIME_EDCA_V1 0x1ff
+#define BIT_USTIME_EDCA_V1(x)                                                  \
+	(((x) & BIT_MASK_USTIME_EDCA_V1) << BIT_SHIFT_USTIME_EDCA_V1)
+#define BIT_GET_USTIME_EDCA_V1(x)                                              \
+	(((x) >> BIT_SHIFT_USTIME_EDCA_V1) & BIT_MASK_USTIME_EDCA_V1)
+
+/* 2 REG_MAC_SPEC_SIFS			(Offset 0x063A) */
+
+#define BIT_SHIFT_SPEC_SIFS_OFDM 8
+#define BIT_MASK_SPEC_SIFS_OFDM 0xff
+#define BIT_SPEC_SIFS_OFDM(x)                                                  \
+	(((x) & BIT_MASK_SPEC_SIFS_OFDM) << BIT_SHIFT_SPEC_SIFS_OFDM)
+#define BIT_GET_SPEC_SIFS_OFDM(x)                                              \
+	(((x) >> BIT_SHIFT_SPEC_SIFS_OFDM) & BIT_MASK_SPEC_SIFS_OFDM)
+
+#define BIT_SHIFT_SPEC_SIFS_CCK 0
+#define BIT_MASK_SPEC_SIFS_CCK 0xff
+#define BIT_SPEC_SIFS_CCK(x)                                                   \
+	(((x) & BIT_MASK_SPEC_SIFS_CCK) << BIT_SHIFT_SPEC_SIFS_CCK)
+#define BIT_GET_SPEC_SIFS_CCK(x)                                               \
+	(((x) >> BIT_SHIFT_SPEC_SIFS_CCK) & BIT_MASK_SPEC_SIFS_CCK)
+
+/* 2 REG_RESP_SIFS_CCK			(Offset 0x063C) */
+
+#define BIT_SHIFT_SIFS_R2T_CCK 8
+#define BIT_MASK_SIFS_R2T_CCK 0xff
+#define BIT_SIFS_R2T_CCK(x)                                                    \
+	(((x) & BIT_MASK_SIFS_R2T_CCK) << BIT_SHIFT_SIFS_R2T_CCK)
+#define BIT_GET_SIFS_R2T_CCK(x)                                                \
+	(((x) >> BIT_SHIFT_SIFS_R2T_CCK) & BIT_MASK_SIFS_R2T_CCK)
+
+#define BIT_SHIFT_SIFS_T2T_CCK 0
+#define BIT_MASK_SIFS_T2T_CCK 0xff
+#define BIT_SIFS_T2T_CCK(x)                                                    \
+	(((x) & BIT_MASK_SIFS_T2T_CCK) << BIT_SHIFT_SIFS_T2T_CCK)
+#define BIT_GET_SIFS_T2T_CCK(x)                                                \
+	(((x) >> BIT_SHIFT_SIFS_T2T_CCK) & BIT_MASK_SIFS_T2T_CCK)
+
+/* 2 REG_RESP_SIFS_OFDM			(Offset 0x063E) */
+
+#define BIT_SHIFT_SIFS_R2T_OFDM 8
+#define BIT_MASK_SIFS_R2T_OFDM 0xff
+#define BIT_SIFS_R2T_OFDM(x)                                                   \
+	(((x) & BIT_MASK_SIFS_R2T_OFDM) << BIT_SHIFT_SIFS_R2T_OFDM)
+#define BIT_GET_SIFS_R2T_OFDM(x)                                               \
+	(((x) >> BIT_SHIFT_SIFS_R2T_OFDM) & BIT_MASK_SIFS_R2T_OFDM)
+
+#define BIT_SHIFT_SIFS_T2T_OFDM 0
+#define BIT_MASK_SIFS_T2T_OFDM 0xff
+#define BIT_SIFS_T2T_OFDM(x)                                                   \
+	(((x) & BIT_MASK_SIFS_T2T_OFDM) << BIT_SHIFT_SIFS_T2T_OFDM)
+#define BIT_GET_SIFS_T2T_OFDM(x)                                               \
+	(((x) >> BIT_SHIFT_SIFS_T2T_OFDM) & BIT_MASK_SIFS_T2T_OFDM)
+
+/* 2 REG_ACKTO				(Offset 0x0640) */
+
+#define BIT_SHIFT_ACKTO 0
+#define BIT_MASK_ACKTO 0xff
+#define BIT_ACKTO(x) (((x) & BIT_MASK_ACKTO) << BIT_SHIFT_ACKTO)
+#define BIT_GET_ACKTO(x) (((x) >> BIT_SHIFT_ACKTO) & BIT_MASK_ACKTO)
+
+/* 2 REG_CTS2TO				(Offset 0x0641) */
+
+#define BIT_SHIFT_CTS2TO 0
+#define BIT_MASK_CTS2TO 0xff
+#define BIT_CTS2TO(x) (((x) & BIT_MASK_CTS2TO) << BIT_SHIFT_CTS2TO)
+#define BIT_GET_CTS2TO(x) (((x) >> BIT_SHIFT_CTS2TO) & BIT_MASK_CTS2TO)
+
+/* 2 REG_EIFS				(Offset 0x0642) */
+
+#define BIT_SHIFT_EIFS 0
+#define BIT_MASK_EIFS 0xffff
+#define BIT_EIFS(x) (((x) & BIT_MASK_EIFS) << BIT_SHIFT_EIFS)
+#define BIT_GET_EIFS(x) (((x) >> BIT_SHIFT_EIFS) & BIT_MASK_EIFS)
+
+/* 2 REG_NAV_CTRL				(Offset 0x0650) */
+
+#define BIT_SHIFT_NAV_UPPER 16
+#define BIT_MASK_NAV_UPPER 0xff
+#define BIT_NAV_UPPER(x) (((x) & BIT_MASK_NAV_UPPER) << BIT_SHIFT_NAV_UPPER)
+#define BIT_GET_NAV_UPPER(x) (((x) >> BIT_SHIFT_NAV_UPPER) & BIT_MASK_NAV_UPPER)
+
+#define BIT_SHIFT_RXMYRTS_NAV 8
+#define BIT_MASK_RXMYRTS_NAV 0xf
+#define BIT_RXMYRTS_NAV(x)                                                     \
+	(((x) & BIT_MASK_RXMYRTS_NAV) << BIT_SHIFT_RXMYRTS_NAV)
+#define BIT_GET_RXMYRTS_NAV(x)                                                 \
+	(((x) >> BIT_SHIFT_RXMYRTS_NAV) & BIT_MASK_RXMYRTS_NAV)
+
+#define BIT_SHIFT_RTSRST 0
+#define BIT_MASK_RTSRST 0xff
+#define BIT_RTSRST(x) (((x) & BIT_MASK_RTSRST) << BIT_SHIFT_RTSRST)
+#define BIT_GET_RTSRST(x) (((x) >> BIT_SHIFT_RTSRST) & BIT_MASK_RTSRST)
+
+/* 2 REG_BACAMCMD				(Offset 0x0654) */
+
+#define BIT_BACAM_POLL BIT(31)
+#define BIT_BACAM_RST BIT(17)
+#define BIT_BACAM_RW BIT(16)
+
+#define BIT_SHIFT_TXSBM 14
+#define BIT_MASK_TXSBM 0x3
+#define BIT_TXSBM(x) (((x) & BIT_MASK_TXSBM) << BIT_SHIFT_TXSBM)
+#define BIT_GET_TXSBM(x) (((x) >> BIT_SHIFT_TXSBM) & BIT_MASK_TXSBM)
+
+#define BIT_SHIFT_BACAM_ADDR 0
+#define BIT_MASK_BACAM_ADDR 0x3f
+#define BIT_BACAM_ADDR(x) (((x) & BIT_MASK_BACAM_ADDR) << BIT_SHIFT_BACAM_ADDR)
+#define BIT_GET_BACAM_ADDR(x)                                                  \
+	(((x) >> BIT_SHIFT_BACAM_ADDR) & BIT_MASK_BACAM_ADDR)
+
+/* 2 REG_BACAMCONTENT			(Offset 0x0658) */
+
+#define BIT_SHIFT_BA_CONTENT_H (32 & CPU_OPT_WIDTH)
+#define BIT_MASK_BA_CONTENT_H 0xffffffffL
+#define BIT_BA_CONTENT_H(x)                                                    \
+	(((x) & BIT_MASK_BA_CONTENT_H) << BIT_SHIFT_BA_CONTENT_H)
+#define BIT_GET_BA_CONTENT_H(x)                                                \
+	(((x) >> BIT_SHIFT_BA_CONTENT_H) & BIT_MASK_BA_CONTENT_H)
+
+#define BIT_SHIFT_BA_CONTENT_L 0
+#define BIT_MASK_BA_CONTENT_L 0xffffffffL
+#define BIT_BA_CONTENT_L(x)                                                    \
+	(((x) & BIT_MASK_BA_CONTENT_L) << BIT_SHIFT_BA_CONTENT_L)
+#define BIT_GET_BA_CONTENT_L(x)                                                \
+	(((x) >> BIT_SHIFT_BA_CONTENT_L) & BIT_MASK_BA_CONTENT_L)
+
+/* 2 REG_LBDLY				(Offset 0x0660) */
+
+#define BIT_SHIFT_LBDLY 0
+#define BIT_MASK_LBDLY 0x1f
+#define BIT_LBDLY(x) (((x) & BIT_MASK_LBDLY) << BIT_SHIFT_LBDLY)
+#define BIT_GET_LBDLY(x) (((x) >> BIT_SHIFT_LBDLY) & BIT_MASK_LBDLY)
+
+/* 2 REG_WMAC_BACAM_RPMEN			(Offset 0x0661) */
+
+#define BIT_SHIFT_BITMAP_SSNBK_COUNTER 2
+#define BIT_MASK_BITMAP_SSNBK_COUNTER 0x3f
+#define BIT_BITMAP_SSNBK_COUNTER(x)                                            \
+	(((x) & BIT_MASK_BITMAP_SSNBK_COUNTER)                                 \
+	 << BIT_SHIFT_BITMAP_SSNBK_COUNTER)
+#define BIT_GET_BITMAP_SSNBK_COUNTER(x)                                        \
+	(((x) >> BIT_SHIFT_BITMAP_SSNBK_COUNTER) &                             \
+	 BIT_MASK_BITMAP_SSNBK_COUNTER)
+
+#define BIT_BITMAP_EN BIT(1)
+
+/* 2 REG_WMAC_BACAM_RPMEN			(Offset 0x0661) */
+
+#define BIT_WMAC_BACAM_RPMEN BIT(0)
+
+/* 2 REG_TX_RX				(Offset 0x0662) */
+
+#define BIT_SHIFT_RXPKT_TYPE 2
+#define BIT_MASK_RXPKT_TYPE 0x3f
+#define BIT_RXPKT_TYPE(x) (((x) & BIT_MASK_RXPKT_TYPE) << BIT_SHIFT_RXPKT_TYPE)
+#define BIT_GET_RXPKT_TYPE(x)                                                  \
+	(((x) >> BIT_SHIFT_RXPKT_TYPE) & BIT_MASK_RXPKT_TYPE)
+
+#define BIT_TXACT_IND BIT(1)
+#define BIT_RXACT_IND BIT(0)
+
+/* 2 REG_WMAC_BITMAP_CTL			(Offset 0x0663) */
+
+#define BIT_BITMAP_VO BIT(7)
+#define BIT_BITMAP_VI BIT(6)
+#define BIT_BITMAP_BE BIT(5)
+#define BIT_BITMAP_BK BIT(4)
+
+#define BIT_SHIFT_BITMAP_CONDITION 2
+#define BIT_MASK_BITMAP_CONDITION 0x3
+#define BIT_BITMAP_CONDITION(x)                                                \
+	(((x) & BIT_MASK_BITMAP_CONDITION) << BIT_SHIFT_BITMAP_CONDITION)
+#define BIT_GET_BITMAP_CONDITION(x)                                            \
+	(((x) >> BIT_SHIFT_BITMAP_CONDITION) & BIT_MASK_BITMAP_CONDITION)
+
+#define BIT_BITMAP_SSNBK_COUNTER_CLR BIT(1)
+#define BIT_BITMAP_FORCE BIT(0)
+
+/* 2 REG_RXERR_RPT				(Offset 0x0664) */
+
+#define BIT_SHIFT_RXERR_RPT_SEL_V1_3_0 28
+#define BIT_MASK_RXERR_RPT_SEL_V1_3_0 0xf
+#define BIT_RXERR_RPT_SEL_V1_3_0(x)                                            \
+	(((x) & BIT_MASK_RXERR_RPT_SEL_V1_3_0)                                 \
+	 << BIT_SHIFT_RXERR_RPT_SEL_V1_3_0)
+#define BIT_GET_RXERR_RPT_SEL_V1_3_0(x)                                        \
+	(((x) >> BIT_SHIFT_RXERR_RPT_SEL_V1_3_0) &                             \
+	 BIT_MASK_RXERR_RPT_SEL_V1_3_0)
+
+/* 2 REG_RXERR_RPT				(Offset 0x0664) */
+
+#define BIT_RXERR_RPT_RST BIT(27)
+
+/* 2 REG_RXERR_RPT				(Offset 0x0664) */
+
+#define BIT_RXERR_RPT_SEL_V1_4 BIT(26)
+
+/* 2 REG_RXERR_RPT				(Offset 0x0664) */
+
+#define BIT_W1S BIT(23)
+
+/* 2 REG_RXERR_RPT				(Offset 0x0664) */
+
+#define BIT_UD_SELECT_BSSID BIT(22)
+
+/* 2 REG_RXERR_RPT				(Offset 0x0664) */
+
+#define BIT_SHIFT_UD_SUB_TYPE 18
+#define BIT_MASK_UD_SUB_TYPE 0xf
+#define BIT_UD_SUB_TYPE(x)                                                     \
+	(((x) & BIT_MASK_UD_SUB_TYPE) << BIT_SHIFT_UD_SUB_TYPE)
+#define BIT_GET_UD_SUB_TYPE(x)                                                 \
+	(((x) >> BIT_SHIFT_UD_SUB_TYPE) & BIT_MASK_UD_SUB_TYPE)
+
+#define BIT_SHIFT_UD_TYPE 16
+#define BIT_MASK_UD_TYPE 0x3
+#define BIT_UD_TYPE(x) (((x) & BIT_MASK_UD_TYPE) << BIT_SHIFT_UD_TYPE)
+#define BIT_GET_UD_TYPE(x) (((x) >> BIT_SHIFT_UD_TYPE) & BIT_MASK_UD_TYPE)
+
+#define BIT_SHIFT_RPT_COUNTER 0
+#define BIT_MASK_RPT_COUNTER 0xffff
+#define BIT_RPT_COUNTER(x)                                                     \
+	(((x) & BIT_MASK_RPT_COUNTER) << BIT_SHIFT_RPT_COUNTER)
+#define BIT_GET_RPT_COUNTER(x)                                                 \
+	(((x) >> BIT_SHIFT_RPT_COUNTER) & BIT_MASK_RPT_COUNTER)
+
+/* 2 REG_WMAC_TRXPTCL_CTL			(Offset 0x0668) */
+
+#define BIT_SHIFT_ACKBA_TYPSEL (60 & CPU_OPT_WIDTH)
+#define BIT_MASK_ACKBA_TYPSEL 0xf
+#define BIT_ACKBA_TYPSEL(x)                                                    \
+	(((x) & BIT_MASK_ACKBA_TYPSEL) << BIT_SHIFT_ACKBA_TYPSEL)
+#define BIT_GET_ACKBA_TYPSEL(x)                                                \
+	(((x) >> BIT_SHIFT_ACKBA_TYPSEL) & BIT_MASK_ACKBA_TYPSEL)
+
+#define BIT_SHIFT_ACKBA_ACKPCHK (56 & CPU_OPT_WIDTH)
+#define BIT_MASK_ACKBA_ACKPCHK 0xf
+#define BIT_ACKBA_ACKPCHK(x)                                                   \
+	(((x) & BIT_MASK_ACKBA_ACKPCHK) << BIT_SHIFT_ACKBA_ACKPCHK)
+#define BIT_GET_ACKBA_ACKPCHK(x)                                               \
+	(((x) >> BIT_SHIFT_ACKBA_ACKPCHK) & BIT_MASK_ACKBA_ACKPCHK)
+
+#define BIT_SHIFT_ACKBAR_TYPESEL (48 & CPU_OPT_WIDTH)
+#define BIT_MASK_ACKBAR_TYPESEL 0xff
+#define BIT_ACKBAR_TYPESEL(x)                                                  \
+	(((x) & BIT_MASK_ACKBAR_TYPESEL) << BIT_SHIFT_ACKBAR_TYPESEL)
+#define BIT_GET_ACKBAR_TYPESEL(x)                                              \
+	(((x) >> BIT_SHIFT_ACKBAR_TYPESEL) & BIT_MASK_ACKBAR_TYPESEL)
+
+#define BIT_SHIFT_ACKBAR_ACKPCHK (44 & CPU_OPT_WIDTH)
+#define BIT_MASK_ACKBAR_ACKPCHK 0xf
+#define BIT_ACKBAR_ACKPCHK(x)                                                  \
+	(((x) & BIT_MASK_ACKBAR_ACKPCHK) << BIT_SHIFT_ACKBAR_ACKPCHK)
+#define BIT_GET_ACKBAR_ACKPCHK(x)                                              \
+	(((x) >> BIT_SHIFT_ACKBAR_ACKPCHK) & BIT_MASK_ACKBAR_ACKPCHK)
+
+/* 2 REG_WMAC_TRXPTCL_CTL			(Offset 0x0668) */
+
+#define BIT_RXBA_IGNOREA2 BIT(42)
+#define BIT_EN_SAVE_ALL_TXOPADDR BIT(41)
+#define BIT_EN_TXCTS_TO_TXOPOWNER_INRXNAV BIT(40)
+
+/* 2 REG_WMAC_TRXPTCL_CTL			(Offset 0x0668) */
+
+#define BIT_DIS_TXBA_AMPDUFCSERR BIT(39)
+#define BIT_DIS_TXBA_RXBARINFULL BIT(38)
+#define BIT_DIS_TXCFE_INFULL BIT(37)
+#define BIT_DIS_TXCTS_INFULL BIT(36)
+#define BIT_EN_TXACKBA_IN_TX_RDG BIT(35)
+#define BIT_EN_TXACKBA_IN_TXOP BIT(34)
+#define BIT_EN_TXCTS_IN_RXNAV BIT(33)
+#define BIT_EN_TXCTS_INTXOP BIT(32)
+#define BIT_BLK_EDCA_BBSLP BIT(31)
+#define BIT_BLK_EDCA_BBSBY BIT(30)
+#define BIT_ACKTO_BLOCK_SCH_EN BIT(27)
+#define BIT_EIFS_BLOCK_SCH_EN BIT(26)
+#define BIT_PLCPCHK_RST_EIFS BIT(25)
+#define BIT_CCA_RST_EIFS BIT(24)
+#define BIT_DIS_UPD_MYRXPKTNAV BIT(23)
+#define BIT_EARLY_TXBA BIT(22)
+
+#define BIT_SHIFT_RESP_CHNBUSY 20
+#define BIT_MASK_RESP_CHNBUSY 0x3
+#define BIT_RESP_CHNBUSY(x)                                                    \
+	(((x) & BIT_MASK_RESP_CHNBUSY) << BIT_SHIFT_RESP_CHNBUSY)
+#define BIT_GET_RESP_CHNBUSY(x)                                                \
+	(((x) >> BIT_SHIFT_RESP_CHNBUSY) & BIT_MASK_RESP_CHNBUSY)
+
+#define BIT_RESP_DCTS_EN BIT(19)
+#define BIT_RESP_DCFE_EN BIT(18)
+#define BIT_RESP_SPLCPEN BIT(17)
+#define BIT_RESP_SGIEN BIT(16)
+
+/* 2 REG_WMAC_TRXPTCL_CTL			(Offset 0x0668) */
+
+#define BIT_RESP_LDPC_EN BIT(15)
+#define BIT_DIS_RESP_ACKINCCA BIT(14)
+#define BIT_DIS_RESP_CTSINCCA BIT(13)
+
+/* 2 REG_WMAC_TRXPTCL_CTL			(Offset 0x0668) */
+
+#define BIT_SHIFT_R_WMAC_SECOND_CCA_TIMER 10
+#define BIT_MASK_R_WMAC_SECOND_CCA_TIMER 0x7
+#define BIT_R_WMAC_SECOND_CCA_TIMER(x)                                         \
+	(((x) & BIT_MASK_R_WMAC_SECOND_CCA_TIMER)                              \
+	 << BIT_SHIFT_R_WMAC_SECOND_CCA_TIMER)
+#define BIT_GET_R_WMAC_SECOND_CCA_TIMER(x)                                     \
+	(((x) >> BIT_SHIFT_R_WMAC_SECOND_CCA_TIMER) &                          \
+	 BIT_MASK_R_WMAC_SECOND_CCA_TIMER)
+
+/* 2 REG_WMAC_TRXPTCL_CTL			(Offset 0x0668) */
+
+#define BIT_SHIFT_RFMOD 7
+#define BIT_MASK_RFMOD 0x3
+#define BIT_RFMOD(x) (((x) & BIT_MASK_RFMOD) << BIT_SHIFT_RFMOD)
+#define BIT_GET_RFMOD(x) (((x) >> BIT_SHIFT_RFMOD) & BIT_MASK_RFMOD)
+
+/* 2 REG_WMAC_TRXPTCL_CTL			(Offset 0x0668) */
+
+#define BIT_SHIFT_RESP_CTS_DYNBW_SEL 5
+#define BIT_MASK_RESP_CTS_DYNBW_SEL 0x3
+#define BIT_RESP_CTS_DYNBW_SEL(x)                                              \
+	(((x) & BIT_MASK_RESP_CTS_DYNBW_SEL) << BIT_SHIFT_RESP_CTS_DYNBW_SEL)
+#define BIT_GET_RESP_CTS_DYNBW_SEL(x)                                          \
+	(((x) >> BIT_SHIFT_RESP_CTS_DYNBW_SEL) & BIT_MASK_RESP_CTS_DYNBW_SEL)
+
+/* 2 REG_WMAC_TRXPTCL_CTL			(Offset 0x0668) */
+
+#define BIT_DLY_TX_WAIT_RXANTSEL BIT(4)
+
+/* 2 REG_WMAC_TRXPTCL_CTL			(Offset 0x0668) */
+
+#define BIT_TXRESP_BY_RXANTSEL BIT(3)
+
+/* 2 REG_WMAC_TRXPTCL_CTL			(Offset 0x0668) */
+
+#define BIT_SHIFT_ORIG_DCTS_CHK 0
+#define BIT_MASK_ORIG_DCTS_CHK 0x3
+#define BIT_ORIG_DCTS_CHK(x)                                                   \
+	(((x) & BIT_MASK_ORIG_DCTS_CHK) << BIT_SHIFT_ORIG_DCTS_CHK)
+#define BIT_GET_ORIG_DCTS_CHK(x)                                               \
+	(((x) >> BIT_SHIFT_ORIG_DCTS_CHK) & BIT_MASK_ORIG_DCTS_CHK)
+
+/* 2 REG_CAMCMD				(Offset 0x0670) */
+
+#define BIT_SECCAM_POLLING BIT(31)
+#define BIT_SECCAM_CLR BIT(30)
+#define BIT_MFBCAM_CLR BIT(29)
+
+/* 2 REG_CAMCMD				(Offset 0x0670) */
+
+#define BIT_SECCAM_WE BIT(16)
+
+/* 2 REG_CAMCMD				(Offset 0x0670) */
+
+#define BIT_SHIFT_SECCAM_ADDR_V2 0
+#define BIT_MASK_SECCAM_ADDR_V2 0x3ff
+#define BIT_SECCAM_ADDR_V2(x)                                                  \
+	(((x) & BIT_MASK_SECCAM_ADDR_V2) << BIT_SHIFT_SECCAM_ADDR_V2)
+#define BIT_GET_SECCAM_ADDR_V2(x)                                              \
+	(((x) >> BIT_SHIFT_SECCAM_ADDR_V2) & BIT_MASK_SECCAM_ADDR_V2)
+
+/* 2 REG_CAMWRITE				(Offset 0x0674) */
+
+#define BIT_SHIFT_CAMW_DATA 0
+#define BIT_MASK_CAMW_DATA 0xffffffffL
+#define BIT_CAMW_DATA(x) (((x) & BIT_MASK_CAMW_DATA) << BIT_SHIFT_CAMW_DATA)
+#define BIT_GET_CAMW_DATA(x) (((x) >> BIT_SHIFT_CAMW_DATA) & BIT_MASK_CAMW_DATA)
+
+/* 2 REG_CAMREAD				(Offset 0x0678) */
+
+#define BIT_SHIFT_CAMR_DATA 0
+#define BIT_MASK_CAMR_DATA 0xffffffffL
+#define BIT_CAMR_DATA(x) (((x) & BIT_MASK_CAMR_DATA) << BIT_SHIFT_CAMR_DATA)
+#define BIT_GET_CAMR_DATA(x) (((x) >> BIT_SHIFT_CAMR_DATA) & BIT_MASK_CAMR_DATA)
+
+/* 2 REG_CAMDBG				(Offset 0x067C) */
+
+#define BIT_SECCAM_INFO BIT(31)
+#define BIT_SEC_KEYFOUND BIT(15)
+
+#define BIT_SHIFT_CAMDBG_SEC_TYPE 12
+#define BIT_MASK_CAMDBG_SEC_TYPE 0x7
+#define BIT_CAMDBG_SEC_TYPE(x)                                                 \
+	(((x) & BIT_MASK_CAMDBG_SEC_TYPE) << BIT_SHIFT_CAMDBG_SEC_TYPE)
+#define BIT_GET_CAMDBG_SEC_TYPE(x)                                             \
+	(((x) >> BIT_SHIFT_CAMDBG_SEC_TYPE) & BIT_MASK_CAMDBG_SEC_TYPE)
+
+/* 2 REG_CAMDBG				(Offset 0x067C) */
+
+#define BIT_CAMDBG_EXT_SECTYPE BIT(11)
+
+/* 2 REG_CAMDBG				(Offset 0x067C) */
+
+#define BIT_SHIFT_CAMDBG_MIC_KEY_IDX 5
+#define BIT_MASK_CAMDBG_MIC_KEY_IDX 0x1f
+#define BIT_CAMDBG_MIC_KEY_IDX(x)                                              \
+	(((x) & BIT_MASK_CAMDBG_MIC_KEY_IDX) << BIT_SHIFT_CAMDBG_MIC_KEY_IDX)
+#define BIT_GET_CAMDBG_MIC_KEY_IDX(x)                                          \
+	(((x) >> BIT_SHIFT_CAMDBG_MIC_KEY_IDX) & BIT_MASK_CAMDBG_MIC_KEY_IDX)
+
+#define BIT_SHIFT_CAMDBG_SEC_KEY_IDX 0
+#define BIT_MASK_CAMDBG_SEC_KEY_IDX 0x1f
+#define BIT_CAMDBG_SEC_KEY_IDX(x)                                              \
+	(((x) & BIT_MASK_CAMDBG_SEC_KEY_IDX) << BIT_SHIFT_CAMDBG_SEC_KEY_IDX)
+#define BIT_GET_CAMDBG_SEC_KEY_IDX(x)                                          \
+	(((x) >> BIT_SHIFT_CAMDBG_SEC_KEY_IDX) & BIT_MASK_CAMDBG_SEC_KEY_IDX)
+
+/* 2 REG_SECCFG				(Offset 0x0680) */
+
+#define BIT_DIS_GCLK_WAPI BIT(15)
+#define BIT_DIS_GCLK_AES BIT(14)
+#define BIT_DIS_GCLK_TKIP BIT(13)
+
+/* 2 REG_SECCFG				(Offset 0x0680) */
+
+#define BIT_AES_SEL_QC_1 BIT(12)
+#define BIT_AES_SEL_QC_0 BIT(11)
+
+/* 2 REG_SECCFG				(Offset 0x0680) */
+
+#define BIT_CHK_BMC BIT(9)
+
+/* 2 REG_SECCFG				(Offset 0x0680) */
+
+#define BIT_CHK_KEYID BIT(8)
+#define BIT_RXBCUSEDK BIT(7)
+#define BIT_TXBCUSEDK BIT(6)
+#define BIT_NOSKMC BIT(5)
+#define BIT_SKBYA2 BIT(4)
+#define BIT_RXDEC BIT(3)
+#define BIT_TXENC BIT(2)
+#define BIT_RXUHUSEDK BIT(1)
+#define BIT_TXUHUSEDK BIT(0)
+
+/* 2 REG_RXFILTER_CATEGORY_1			(Offset 0x0682) */
+
+#define BIT_SHIFT_RXFILTER_CATEGORY_1 0
+#define BIT_MASK_RXFILTER_CATEGORY_1 0xff
+#define BIT_RXFILTER_CATEGORY_1(x)                                             \
+	(((x) & BIT_MASK_RXFILTER_CATEGORY_1) << BIT_SHIFT_RXFILTER_CATEGORY_1)
+#define BIT_GET_RXFILTER_CATEGORY_1(x)                                         \
+	(((x) >> BIT_SHIFT_RXFILTER_CATEGORY_1) & BIT_MASK_RXFILTER_CATEGORY_1)
+
+/* 2 REG_RXFILTER_ACTION_1			(Offset 0x0683) */
+
+#define BIT_SHIFT_RXFILTER_ACTION_1 0
+#define BIT_MASK_RXFILTER_ACTION_1 0xff
+#define BIT_RXFILTER_ACTION_1(x)                                               \
+	(((x) & BIT_MASK_RXFILTER_ACTION_1) << BIT_SHIFT_RXFILTER_ACTION_1)
+#define BIT_GET_RXFILTER_ACTION_1(x)                                           \
+	(((x) >> BIT_SHIFT_RXFILTER_ACTION_1) & BIT_MASK_RXFILTER_ACTION_1)
+
+/* 2 REG_RXFILTER_CATEGORY_2			(Offset 0x0684) */
+
+#define BIT_SHIFT_RXFILTER_CATEGORY_2 0
+#define BIT_MASK_RXFILTER_CATEGORY_2 0xff
+#define BIT_RXFILTER_CATEGORY_2(x)                                             \
+	(((x) & BIT_MASK_RXFILTER_CATEGORY_2) << BIT_SHIFT_RXFILTER_CATEGORY_2)
+#define BIT_GET_RXFILTER_CATEGORY_2(x)                                         \
+	(((x) >> BIT_SHIFT_RXFILTER_CATEGORY_2) & BIT_MASK_RXFILTER_CATEGORY_2)
+
+/* 2 REG_RXFILTER_ACTION_2			(Offset 0x0685) */
+
+#define BIT_SHIFT_RXFILTER_ACTION_2 0
+#define BIT_MASK_RXFILTER_ACTION_2 0xff
+#define BIT_RXFILTER_ACTION_2(x)                                               \
+	(((x) & BIT_MASK_RXFILTER_ACTION_2) << BIT_SHIFT_RXFILTER_ACTION_2)
+#define BIT_GET_RXFILTER_ACTION_2(x)                                           \
+	(((x) >> BIT_SHIFT_RXFILTER_ACTION_2) & BIT_MASK_RXFILTER_ACTION_2)
+
+/* 2 REG_RXFILTER_CATEGORY_3			(Offset 0x0686) */
+
+#define BIT_SHIFT_RXFILTER_CATEGORY_3 0
+#define BIT_MASK_RXFILTER_CATEGORY_3 0xff
+#define BIT_RXFILTER_CATEGORY_3(x)                                             \
+	(((x) & BIT_MASK_RXFILTER_CATEGORY_3) << BIT_SHIFT_RXFILTER_CATEGORY_3)
+#define BIT_GET_RXFILTER_CATEGORY_3(x)                                         \
+	(((x) >> BIT_SHIFT_RXFILTER_CATEGORY_3) & BIT_MASK_RXFILTER_CATEGORY_3)
+
+/* 2 REG_RXFILTER_ACTION_3			(Offset 0x0687) */
+
+#define BIT_SHIFT_RXFILTER_ACTION_3 0
+#define BIT_MASK_RXFILTER_ACTION_3 0xff
+#define BIT_RXFILTER_ACTION_3(x)                                               \
+	(((x) & BIT_MASK_RXFILTER_ACTION_3) << BIT_SHIFT_RXFILTER_ACTION_3)
+#define BIT_GET_RXFILTER_ACTION_3(x)                                           \
+	(((x) >> BIT_SHIFT_RXFILTER_ACTION_3) & BIT_MASK_RXFILTER_ACTION_3)
+
+/* 2 REG_RXFLTMAP3				(Offset 0x0688) */
+
+#define BIT_MGTFLT15EN_FW BIT(15)
+#define BIT_MGTFLT14EN_FW BIT(14)
+#define BIT_MGTFLT13EN_FW BIT(13)
+#define BIT_MGTFLT12EN_FW BIT(12)
+#define BIT_MGTFLT11EN_FW BIT(11)
+#define BIT_MGTFLT10EN_FW BIT(10)
+#define BIT_MGTFLT9EN_FW BIT(9)
+#define BIT_MGTFLT8EN_FW BIT(8)
+#define BIT_MGTFLT7EN_FW BIT(7)
+#define BIT_MGTFLT6EN_FW BIT(6)
+#define BIT_MGTFLT5EN_FW BIT(5)
+#define BIT_MGTFLT4EN_FW BIT(4)
+#define BIT_MGTFLT3EN_FW BIT(3)
+#define BIT_MGTFLT2EN_FW BIT(2)
+#define BIT_MGTFLT1EN_FW BIT(1)
+#define BIT_MGTFLT0EN_FW BIT(0)
+
+/* 2 REG_RXFLTMAP4				(Offset 0x068A) */
+
+#define BIT_CTRLFLT15EN_FW BIT(15)
+#define BIT_CTRLFLT14EN_FW BIT(14)
+#define BIT_CTRLFLT13EN_FW BIT(13)
+#define BIT_CTRLFLT12EN_FW BIT(12)
+#define BIT_CTRLFLT11EN_FW BIT(11)
+#define BIT_CTRLFLT10EN_FW BIT(10)
+#define BIT_CTRLFLT9EN_FW BIT(9)
+#define BIT_CTRLFLT8EN_FW BIT(8)
+#define BIT_CTRLFLT7EN_FW BIT(7)
+#define BIT_CTRLFLT6EN_FW BIT(6)
+#define BIT_CTRLFLT5EN_FW BIT(5)
+#define BIT_CTRLFLT4EN_FW BIT(4)
+#define BIT_CTRLFLT3EN_FW BIT(3)
+#define BIT_CTRLFLT2EN_FW BIT(2)
+#define BIT_CTRLFLT1EN_FW BIT(1)
+#define BIT_CTRLFLT0EN_FW BIT(0)
+
+/* 2 REG_RXFLTMAP5				(Offset 0x068C) */
+
+#define BIT_DATAFLT15EN_FW BIT(15)
+#define BIT_DATAFLT14EN_FW BIT(14)
+#define BIT_DATAFLT13EN_FW BIT(13)
+#define BIT_DATAFLT12EN_FW BIT(12)
+#define BIT_DATAFLT11EN_FW BIT(11)
+#define BIT_DATAFLT10EN_FW BIT(10)
+#define BIT_DATAFLT9EN_FW BIT(9)
+#define BIT_DATAFLT8EN_FW BIT(8)
+#define BIT_DATAFLT7EN_FW BIT(7)
+#define BIT_DATAFLT6EN_FW BIT(6)
+#define BIT_DATAFLT5EN_FW BIT(5)
+#define BIT_DATAFLT4EN_FW BIT(4)
+#define BIT_DATAFLT3EN_FW BIT(3)
+#define BIT_DATAFLT2EN_FW BIT(2)
+#define BIT_DATAFLT1EN_FW BIT(1)
+#define BIT_DATAFLT0EN_FW BIT(0)
+
+/* 2 REG_RXFLTMAP6				(Offset 0x068E) */
+
+#define BIT_ACTIONFLT15EN_FW BIT(15)
+#define BIT_ACTIONFLT14EN_FW BIT(14)
+#define BIT_ACTIONFLT13EN_FW BIT(13)
+#define BIT_ACTIONFLT12EN_FW BIT(12)
+#define BIT_ACTIONFLT11EN_FW BIT(11)
+#define BIT_ACTIONFLT10EN_FW BIT(10)
+#define BIT_ACTIONFLT9EN_FW BIT(9)
+#define BIT_ACTIONFLT8EN_FW BIT(8)
+#define BIT_ACTIONFLT7EN_FW BIT(7)
+#define BIT_ACTIONFLT6EN_FW BIT(6)
+#define BIT_ACTIONFLT5EN_FW BIT(5)
+#define BIT_ACTIONFLT4EN_FW BIT(4)
+#define BIT_ACTIONFLT3EN_FW BIT(3)
+#define BIT_ACTIONFLT2EN_FW BIT(2)
+#define BIT_ACTIONFLT1EN_FW BIT(1)
+#define BIT_ACTIONFLT0EN_FW BIT(0)
+
+/* 2 REG_WOW_CTRL				(Offset 0x0690) */
+
+#define BIT_SHIFT_PSF_BSSIDSEL_B2B1 6
+#define BIT_MASK_PSF_BSSIDSEL_B2B1 0x3
+#define BIT_PSF_BSSIDSEL_B2B1(x)                                               \
+	(((x) & BIT_MASK_PSF_BSSIDSEL_B2B1) << BIT_SHIFT_PSF_BSSIDSEL_B2B1)
+#define BIT_GET_PSF_BSSIDSEL_B2B1(x)                                           \
+	(((x) >> BIT_SHIFT_PSF_BSSIDSEL_B2B1) & BIT_MASK_PSF_BSSIDSEL_B2B1)
+
+/* 2 REG_WOW_CTRL				(Offset 0x0690) */
+
+#define BIT_WOWHCI BIT(5)
+
+/* 2 REG_WOW_CTRL				(Offset 0x0690) */
+
+#define BIT_PSF_BSSIDSEL_B0 BIT(4)
+
+/* 2 REG_WOW_CTRL				(Offset 0x0690) */
+
+#define BIT_UWF BIT(3)
+#define BIT_MAGIC BIT(2)
+#define BIT_WOWEN BIT(1)
+#define BIT_FORCE_WAKEUP BIT(0)
+
+/* 2 REG_NAN_RX_TSF_FILTER			(Offset 0x0691) */
+
+#define BIT_CHK_TSF_TA BIT(2)
+#define BIT_CHK_TSF_CBSSID BIT(1)
+#define BIT_CHK_TSF_EN BIT(0)
+
+/* 2 REG_PS_RX_INFO				(Offset 0x0692) */
+
+#define BIT_SHIFT_PORTSEL__PS_RX_INFO 5
+#define BIT_MASK_PORTSEL__PS_RX_INFO 0x7
+#define BIT_PORTSEL__PS_RX_INFO(x)                                             \
+	(((x) & BIT_MASK_PORTSEL__PS_RX_INFO) << BIT_SHIFT_PORTSEL__PS_RX_INFO)
+#define BIT_GET_PORTSEL__PS_RX_INFO(x)                                         \
+	(((x) >> BIT_SHIFT_PORTSEL__PS_RX_INFO) & BIT_MASK_PORTSEL__PS_RX_INFO)
+
+/* 2 REG_PS_RX_INFO				(Offset 0x0692) */
+
+#define BIT_RXCTRLIN0 BIT(4)
+#define BIT_RXMGTIN0 BIT(3)
+#define BIT_RXDATAIN2 BIT(2)
+#define BIT_RXDATAIN1 BIT(1)
+#define BIT_RXDATAIN0 BIT(0)
+
+/* 2 REG_WMMPS_UAPSD_TID			(Offset 0x0693) */
+
+#define BIT_WMMPS_UAPSD_TID7 BIT(7)
+#define BIT_WMMPS_UAPSD_TID6 BIT(6)
+#define BIT_WMMPS_UAPSD_TID5 BIT(5)
+#define BIT_WMMPS_UAPSD_TID4 BIT(4)
+#define BIT_WMMPS_UAPSD_TID3 BIT(3)
+#define BIT_WMMPS_UAPSD_TID2 BIT(2)
+#define BIT_WMMPS_UAPSD_TID1 BIT(1)
+#define BIT_WMMPS_UAPSD_TID0 BIT(0)
+
+/* 2 REG_LPNAV_CTRL				(Offset 0x0694) */
+
+#define BIT_LPNAV_EN BIT(31)
+
+#define BIT_SHIFT_LPNAV_EARLY 16
+#define BIT_MASK_LPNAV_EARLY 0x7fff
+#define BIT_LPNAV_EARLY(x)                                                     \
+	(((x) & BIT_MASK_LPNAV_EARLY) << BIT_SHIFT_LPNAV_EARLY)
+#define BIT_GET_LPNAV_EARLY(x)                                                 \
+	(((x) >> BIT_SHIFT_LPNAV_EARLY) & BIT_MASK_LPNAV_EARLY)
+
+#define BIT_SHIFT_LPNAV_TH 0
+#define BIT_MASK_LPNAV_TH 0xffff
+#define BIT_LPNAV_TH(x) (((x) & BIT_MASK_LPNAV_TH) << BIT_SHIFT_LPNAV_TH)
+#define BIT_GET_LPNAV_TH(x) (((x) >> BIT_SHIFT_LPNAV_TH) & BIT_MASK_LPNAV_TH)
+
+/* 2 REG_WKFMCAM_CMD				(Offset 0x0698) */
+
+#define BIT_WKFCAM_POLLING_V1 BIT(31)
+#define BIT_WKFCAM_CLR_V1 BIT(30)
+
+/* 2 REG_WKFMCAM_CMD				(Offset 0x0698) */
+
+#define BIT_WKFCAM_WE BIT(16)
+
+/* 2 REG_WKFMCAM_CMD				(Offset 0x0698) */
+
+#define BIT_SHIFT_WKFCAM_ADDR_V2 8
+#define BIT_MASK_WKFCAM_ADDR_V2 0xff
+#define BIT_WKFCAM_ADDR_V2(x)                                                  \
+	(((x) & BIT_MASK_WKFCAM_ADDR_V2) << BIT_SHIFT_WKFCAM_ADDR_V2)
+#define BIT_GET_WKFCAM_ADDR_V2(x)                                              \
+	(((x) >> BIT_SHIFT_WKFCAM_ADDR_V2) & BIT_MASK_WKFCAM_ADDR_V2)
+
+#define BIT_SHIFT_WKFCAM_CAM_NUM_V1 0
+#define BIT_MASK_WKFCAM_CAM_NUM_V1 0xff
+#define BIT_WKFCAM_CAM_NUM_V1(x)                                               \
+	(((x) & BIT_MASK_WKFCAM_CAM_NUM_V1) << BIT_SHIFT_WKFCAM_CAM_NUM_V1)
+#define BIT_GET_WKFCAM_CAM_NUM_V1(x)                                           \
+	(((x) >> BIT_SHIFT_WKFCAM_CAM_NUM_V1) & BIT_MASK_WKFCAM_CAM_NUM_V1)
+
+/* 2 REG_WKFMCAM_RWD				(Offset 0x069C) */
+
+#define BIT_SHIFT_WKFMCAM_RWD 0
+#define BIT_MASK_WKFMCAM_RWD 0xffffffffL
+#define BIT_WKFMCAM_RWD(x)                                                     \
+	(((x) & BIT_MASK_WKFMCAM_RWD) << BIT_SHIFT_WKFMCAM_RWD)
+#define BIT_GET_WKFMCAM_RWD(x)                                                 \
+	(((x) >> BIT_SHIFT_WKFMCAM_RWD) & BIT_MASK_WKFMCAM_RWD)
+
+/* 2 REG_RXFLTMAP0				(Offset 0x06A0) */
+
+#define BIT_MGTFLT15EN BIT(15)
+#define BIT_MGTFLT14EN BIT(14)
+
+/* 2 REG_RXFLTMAP0				(Offset 0x06A0) */
+
+#define BIT_MGTFLT13EN BIT(13)
+#define BIT_MGTFLT12EN BIT(12)
+#define BIT_MGTFLT11EN BIT(11)
+#define BIT_MGTFLT10EN BIT(10)
+#define BIT_MGTFLT9EN BIT(9)
+#define BIT_MGTFLT8EN BIT(8)
+
+/* 2 REG_RXFLTMAP0				(Offset 0x06A0) */
+
+#define BIT_MGTFLT7EN BIT(7)
+#define BIT_MGTFLT6EN BIT(6)
+
+/* 2 REG_RXFLTMAP0				(Offset 0x06A0) */
+
+#define BIT_MGTFLT5EN BIT(5)
+#define BIT_MGTFLT4EN BIT(4)
+#define BIT_MGTFLT3EN BIT(3)
+#define BIT_MGTFLT2EN BIT(2)
+#define BIT_MGTFLT1EN BIT(1)
+#define BIT_MGTFLT0EN BIT(0)
+
+/* 2 REG_RXFLTMAP1				(Offset 0x06A2) */
+
+#define BIT_CTRLFLT15EN BIT(15)
+#define BIT_CTRLFLT14EN BIT(14)
+#define BIT_CTRLFLT13EN BIT(13)
+#define BIT_CTRLFLT12EN BIT(12)
+#define BIT_CTRLFLT11EN BIT(11)
+#define BIT_CTRLFLT10EN BIT(10)
+#define BIT_CTRLFLT9EN BIT(9)
+#define BIT_CTRLFLT8EN BIT(8)
+#define BIT_CTRLFLT7EN BIT(7)
+#define BIT_CTRLFLT6EN BIT(6)
+
+/* 2 REG_RXFLTMAP1				(Offset 0x06A2) */
+
+#define BIT_CTRLFLT5EN BIT(5)
+#define BIT_CTRLFLT4EN BIT(4)
+#define BIT_CTRLFLT3EN BIT(3)
+#define BIT_CTRLFLT2EN BIT(2)
+#define BIT_CTRLFLT1EN BIT(1)
+#define BIT_CTRLFLT0EN BIT(0)
+
+/* 2 REG_RXFLTMAP				(Offset 0x06A4) */
+
+#define BIT_DATAFLT15EN BIT(15)
+#define BIT_DATAFLT14EN BIT(14)
+#define BIT_DATAFLT13EN BIT(13)
+#define BIT_DATAFLT12EN BIT(12)
+#define BIT_DATAFLT11EN BIT(11)
+#define BIT_DATAFLT10EN BIT(10)
+#define BIT_DATAFLT9EN BIT(9)
+#define BIT_DATAFLT8EN BIT(8)
+#define BIT_DATAFLT7EN BIT(7)
+#define BIT_DATAFLT6EN BIT(6)
+#define BIT_DATAFLT5EN BIT(5)
+#define BIT_DATAFLT4EN BIT(4)
+#define BIT_DATAFLT3EN BIT(3)
+#define BIT_DATAFLT2EN BIT(2)
+#define BIT_DATAFLT1EN BIT(1)
+#define BIT_DATAFLT0EN BIT(0)
+
+/* 2 REG_BCN_PSR_RPT				(Offset 0x06A8) */
+
+#define BIT_SHIFT_DTIM_CNT 24
+#define BIT_MASK_DTIM_CNT 0xff
+#define BIT_DTIM_CNT(x) (((x) & BIT_MASK_DTIM_CNT) << BIT_SHIFT_DTIM_CNT)
+#define BIT_GET_DTIM_CNT(x) (((x) >> BIT_SHIFT_DTIM_CNT) & BIT_MASK_DTIM_CNT)
+
+#define BIT_SHIFT_DTIM_PERIOD 16
+#define BIT_MASK_DTIM_PERIOD 0xff
+#define BIT_DTIM_PERIOD(x)                                                     \
+	(((x) & BIT_MASK_DTIM_PERIOD) << BIT_SHIFT_DTIM_PERIOD)
+#define BIT_GET_DTIM_PERIOD(x)                                                 \
+	(((x) >> BIT_SHIFT_DTIM_PERIOD) & BIT_MASK_DTIM_PERIOD)
+
+#define BIT_DTIM BIT(15)
+#define BIT_TIM BIT(14)
+
+#define BIT_SHIFT_PS_AID_0 0
+#define BIT_MASK_PS_AID_0 0x7ff
+#define BIT_PS_AID_0(x) (((x) & BIT_MASK_PS_AID_0) << BIT_SHIFT_PS_AID_0)
+#define BIT_GET_PS_AID_0(x) (((x) >> BIT_SHIFT_PS_AID_0) & BIT_MASK_PS_AID_0)
+
+/* 2 REG_FLC_RPC				(Offset 0x06AC) */
+
+#define BIT_SHIFT_FLC_RPC 0
+#define BIT_MASK_FLC_RPC 0xff
+#define BIT_FLC_RPC(x) (((x) & BIT_MASK_FLC_RPC) << BIT_SHIFT_FLC_RPC)
+#define BIT_GET_FLC_RPC(x) (((x) >> BIT_SHIFT_FLC_RPC) & BIT_MASK_FLC_RPC)
+
+/* 2 REG_FLC_RPCT				(Offset 0x06AD) */
+
+#define BIT_SHIFT_FLC_RPCT 0
+#define BIT_MASK_FLC_RPCT 0xff
+#define BIT_FLC_RPCT(x) (((x) & BIT_MASK_FLC_RPCT) << BIT_SHIFT_FLC_RPCT)
+#define BIT_GET_FLC_RPCT(x) (((x) >> BIT_SHIFT_FLC_RPCT) & BIT_MASK_FLC_RPCT)
+
+/* 2 REG_FLC_PTS				(Offset 0x06AE) */
+
+#define BIT_CMF BIT(2)
+#define BIT_CCF BIT(1)
+#define BIT_CDF BIT(0)
+
+/* 2 REG_FLC_TRPC				(Offset 0x06AF) */
+
+#define BIT_FLC_RPCT_V1 BIT(7)
+#define BIT_MODE BIT(6)
+
+#define BIT_SHIFT_TRPCD 0
+#define BIT_MASK_TRPCD 0x3f
+#define BIT_TRPCD(x) (((x) & BIT_MASK_TRPCD) << BIT_SHIFT_TRPCD)
+#define BIT_GET_TRPCD(x) (((x) >> BIT_SHIFT_TRPCD) & BIT_MASK_TRPCD)
+
+/* 2 REG_RXPKTMON_CTRL			(Offset 0x06B0) */
+
+#define BIT_SHIFT_RXBKQPKT_SEQ 20
+#define BIT_MASK_RXBKQPKT_SEQ 0xf
+#define BIT_RXBKQPKT_SEQ(x)                                                    \
+	(((x) & BIT_MASK_RXBKQPKT_SEQ) << BIT_SHIFT_RXBKQPKT_SEQ)
+#define BIT_GET_RXBKQPKT_SEQ(x)                                                \
+	(((x) >> BIT_SHIFT_RXBKQPKT_SEQ) & BIT_MASK_RXBKQPKT_SEQ)
+
+#define BIT_SHIFT_RXBEQPKT_SEQ 16
+#define BIT_MASK_RXBEQPKT_SEQ 0xf
+#define BIT_RXBEQPKT_SEQ(x)                                                    \
+	(((x) & BIT_MASK_RXBEQPKT_SEQ) << BIT_SHIFT_RXBEQPKT_SEQ)
+#define BIT_GET_RXBEQPKT_SEQ(x)                                                \
+	(((x) >> BIT_SHIFT_RXBEQPKT_SEQ) & BIT_MASK_RXBEQPKT_SEQ)
+
+#define BIT_SHIFT_RXVIQPKT_SEQ 12
+#define BIT_MASK_RXVIQPKT_SEQ 0xf
+#define BIT_RXVIQPKT_SEQ(x)                                                    \
+	(((x) & BIT_MASK_RXVIQPKT_SEQ) << BIT_SHIFT_RXVIQPKT_SEQ)
+#define BIT_GET_RXVIQPKT_SEQ(x)                                                \
+	(((x) >> BIT_SHIFT_RXVIQPKT_SEQ) & BIT_MASK_RXVIQPKT_SEQ)
+
+#define BIT_SHIFT_RXVOQPKT_SEQ 8
+#define BIT_MASK_RXVOQPKT_SEQ 0xf
+#define BIT_RXVOQPKT_SEQ(x)                                                    \
+	(((x) & BIT_MASK_RXVOQPKT_SEQ) << BIT_SHIFT_RXVOQPKT_SEQ)
+#define BIT_GET_RXVOQPKT_SEQ(x)                                                \
+	(((x) >> BIT_SHIFT_RXVOQPKT_SEQ) & BIT_MASK_RXVOQPKT_SEQ)
+
+#define BIT_RXBKQPKT_ERR BIT(7)
+#define BIT_RXBEQPKT_ERR BIT(6)
+#define BIT_RXVIQPKT_ERR BIT(5)
+#define BIT_RXVOQPKT_ERR BIT(4)
+#define BIT_RXDMA_MON_EN BIT(2)
+#define BIT_RXPKT_MON_RST BIT(1)
+#define BIT_RXPKT_MON_EN BIT(0)
+
+/* 2 REG_STATE_MON				(Offset 0x06B4) */
+
+#define BIT_SHIFT_STATE_SEL 24
+#define BIT_MASK_STATE_SEL 0x1f
+#define BIT_STATE_SEL(x) (((x) & BIT_MASK_STATE_SEL) << BIT_SHIFT_STATE_SEL)
+#define BIT_GET_STATE_SEL(x) (((x) >> BIT_SHIFT_STATE_SEL) & BIT_MASK_STATE_SEL)
+
+#define BIT_SHIFT_STATE_INFO 8
+#define BIT_MASK_STATE_INFO 0xff
+#define BIT_STATE_INFO(x) (((x) & BIT_MASK_STATE_INFO) << BIT_SHIFT_STATE_INFO)
+#define BIT_GET_STATE_INFO(x)                                                  \
+	(((x) >> BIT_SHIFT_STATE_INFO) & BIT_MASK_STATE_INFO)
+
+#define BIT_UPD_NXT_STATE BIT(7)
+
+/* 2 REG_STATE_MON				(Offset 0x06B4) */
+
+#define BIT_SHIFT_CUR_STATE 0
+#define BIT_MASK_CUR_STATE 0x7f
+#define BIT_CUR_STATE(x) (((x) & BIT_MASK_CUR_STATE) << BIT_SHIFT_CUR_STATE)
+#define BIT_GET_CUR_STATE(x) (((x) >> BIT_SHIFT_CUR_STATE) & BIT_MASK_CUR_STATE)
+
+/* 2 REG_ERROR_MON				(Offset 0x06B8) */
+
+#define BIT_MACRX_ERR_1 BIT(17)
+#define BIT_MACRX_ERR_0 BIT(16)
+#define BIT_MACTX_ERR_3 BIT(3)
+#define BIT_MACTX_ERR_2 BIT(2)
+#define BIT_MACTX_ERR_1 BIT(1)
+#define BIT_MACTX_ERR_0 BIT(0)
+
+/* 2 REG_SEARCH_MACID			(Offset 0x06BC) */
+
+#define BIT_EN_TXRPTBUF_CLK BIT(31)
+
+#define BIT_SHIFT_INFO_INDEX_OFFSET 16
+#define BIT_MASK_INFO_INDEX_OFFSET 0x1fff
+#define BIT_INFO_INDEX_OFFSET(x)                                               \
+	(((x) & BIT_MASK_INFO_INDEX_OFFSET) << BIT_SHIFT_INFO_INDEX_OFFSET)
+#define BIT_GET_INFO_INDEX_OFFSET(x)                                           \
+	(((x) >> BIT_SHIFT_INFO_INDEX_OFFSET) & BIT_MASK_INFO_INDEX_OFFSET)
+
+/* 2 REG_SEARCH_MACID			(Offset 0x06BC) */
+
+#define BIT_WMAC_SRCH_FIFOFULL BIT(15)
+
+/* 2 REG_SEARCH_MACID			(Offset 0x06BC) */
+
+#define BIT_DIS_INFOSRCH BIT(14)
+#define BIT_DISABLE_B0 BIT(13)
+
+#define BIT_SHIFT_INFO_ADDR_OFFSET 0
+#define BIT_MASK_INFO_ADDR_OFFSET 0x1fff
+#define BIT_INFO_ADDR_OFFSET(x)                                                \
+	(((x) & BIT_MASK_INFO_ADDR_OFFSET) << BIT_SHIFT_INFO_ADDR_OFFSET)
+#define BIT_GET_INFO_ADDR_OFFSET(x)                                            \
+	(((x) >> BIT_SHIFT_INFO_ADDR_OFFSET) & BIT_MASK_INFO_ADDR_OFFSET)
+
+/* 2 REG_BT_COEX_TABLE			(Offset 0x06C0) */
+
+#define BIT_PRI_MASK_RX_RESP BIT(126)
+#define BIT_PRI_MASK_RXOFDM BIT(125)
+#define BIT_PRI_MASK_RXCCK BIT(124)
+
+#define BIT_SHIFT_PRI_MASK_TXAC (117 & CPU_OPT_WIDTH)
+#define BIT_MASK_PRI_MASK_TXAC 0x7f
+#define BIT_PRI_MASK_TXAC(x)                                                   \
+	(((x) & BIT_MASK_PRI_MASK_TXAC) << BIT_SHIFT_PRI_MASK_TXAC)
+#define BIT_GET_PRI_MASK_TXAC(x)                                               \
+	(((x) >> BIT_SHIFT_PRI_MASK_TXAC) & BIT_MASK_PRI_MASK_TXAC)
+
+#define BIT_SHIFT_PRI_MASK_NAV (109 & CPU_OPT_WIDTH)
+#define BIT_MASK_PRI_MASK_NAV 0xff
+#define BIT_PRI_MASK_NAV(x)                                                    \
+	(((x) & BIT_MASK_PRI_MASK_NAV) << BIT_SHIFT_PRI_MASK_NAV)
+#define BIT_GET_PRI_MASK_NAV(x)                                                \
+	(((x) >> BIT_SHIFT_PRI_MASK_NAV) & BIT_MASK_PRI_MASK_NAV)
+
+#define BIT_PRI_MASK_CCK BIT(108)
+#define BIT_PRI_MASK_OFDM BIT(107)
+#define BIT_PRI_MASK_RTY BIT(106)
+
+#define BIT_SHIFT_PRI_MASK_NUM (102 & CPU_OPT_WIDTH)
+#define BIT_MASK_PRI_MASK_NUM 0xf
+#define BIT_PRI_MASK_NUM(x)                                                    \
+	(((x) & BIT_MASK_PRI_MASK_NUM) << BIT_SHIFT_PRI_MASK_NUM)
+#define BIT_GET_PRI_MASK_NUM(x)                                                \
+	(((x) >> BIT_SHIFT_PRI_MASK_NUM) & BIT_MASK_PRI_MASK_NUM)
+
+#define BIT_SHIFT_PRI_MASK_TYPE (98 & CPU_OPT_WIDTH)
+#define BIT_MASK_PRI_MASK_TYPE 0xf
+#define BIT_PRI_MASK_TYPE(x)                                                   \
+	(((x) & BIT_MASK_PRI_MASK_TYPE) << BIT_SHIFT_PRI_MASK_TYPE)
+#define BIT_GET_PRI_MASK_TYPE(x)                                               \
+	(((x) >> BIT_SHIFT_PRI_MASK_TYPE) & BIT_MASK_PRI_MASK_TYPE)
+
+#define BIT_OOB BIT(97)
+#define BIT_ANT_SEL BIT(96)
+
+#define BIT_SHIFT_BREAK_TABLE_2 (80 & CPU_OPT_WIDTH)
+#define BIT_MASK_BREAK_TABLE_2 0xffff
+#define BIT_BREAK_TABLE_2(x)                                                   \
+	(((x) & BIT_MASK_BREAK_TABLE_2) << BIT_SHIFT_BREAK_TABLE_2)
+#define BIT_GET_BREAK_TABLE_2(x)                                               \
+	(((x) >> BIT_SHIFT_BREAK_TABLE_2) & BIT_MASK_BREAK_TABLE_2)
+
+#define BIT_SHIFT_BREAK_TABLE_1 (64 & CPU_OPT_WIDTH)
+#define BIT_MASK_BREAK_TABLE_1 0xffff
+#define BIT_BREAK_TABLE_1(x)                                                   \
+	(((x) & BIT_MASK_BREAK_TABLE_1) << BIT_SHIFT_BREAK_TABLE_1)
+#define BIT_GET_BREAK_TABLE_1(x)                                               \
+	(((x) >> BIT_SHIFT_BREAK_TABLE_1) & BIT_MASK_BREAK_TABLE_1)
+
+#define BIT_SHIFT_COEX_TABLE_2 (32 & CPU_OPT_WIDTH)
+#define BIT_MASK_COEX_TABLE_2 0xffffffffL
+#define BIT_COEX_TABLE_2(x)                                                    \
+	(((x) & BIT_MASK_COEX_TABLE_2) << BIT_SHIFT_COEX_TABLE_2)
+#define BIT_GET_COEX_TABLE_2(x)                                                \
+	(((x) >> BIT_SHIFT_COEX_TABLE_2) & BIT_MASK_COEX_TABLE_2)
+
+#define BIT_SHIFT_COEX_TABLE_1 0
+#define BIT_MASK_COEX_TABLE_1 0xffffffffL
+#define BIT_COEX_TABLE_1(x)                                                    \
+	(((x) & BIT_MASK_COEX_TABLE_1) << BIT_SHIFT_COEX_TABLE_1)
+#define BIT_GET_COEX_TABLE_1(x)                                                \
+	(((x) >> BIT_SHIFT_COEX_TABLE_1) & BIT_MASK_COEX_TABLE_1)
+
+/* 2 REG_RXCMD_0				(Offset 0x06D0) */
+
+#define BIT_RXCMD_EN BIT(31)
+
+#define BIT_SHIFT_RXCMD_INFO 0
+#define BIT_MASK_RXCMD_INFO 0x7fffffffL
+#define BIT_RXCMD_INFO(x) (((x) & BIT_MASK_RXCMD_INFO) << BIT_SHIFT_RXCMD_INFO)
+#define BIT_GET_RXCMD_INFO(x)                                                  \
+	(((x) >> BIT_SHIFT_RXCMD_INFO) & BIT_MASK_RXCMD_INFO)
+
+/* 2 REG_RXCMD_1				(Offset 0x06D4) */
+
+#define BIT_SHIFT_RXCMD_PRD 0
+#define BIT_MASK_RXCMD_PRD 0xffff
+#define BIT_RXCMD_PRD(x) (((x) & BIT_MASK_RXCMD_PRD) << BIT_SHIFT_RXCMD_PRD)
+#define BIT_GET_RXCMD_PRD(x) (((x) >> BIT_SHIFT_RXCMD_PRD) & BIT_MASK_RXCMD_PRD)
+
+/* 2 REG_WMAC_RESP_TXINFO			(Offset 0x06D8) */
+
+#define BIT_SHIFT_WMAC_RESP_MFB 25
+#define BIT_MASK_WMAC_RESP_MFB 0x7f
+#define BIT_WMAC_RESP_MFB(x)                                                   \
+	(((x) & BIT_MASK_WMAC_RESP_MFB) << BIT_SHIFT_WMAC_RESP_MFB)
+#define BIT_GET_WMAC_RESP_MFB(x)                                               \
+	(((x) >> BIT_SHIFT_WMAC_RESP_MFB) & BIT_MASK_WMAC_RESP_MFB)
+
+#define BIT_SHIFT_WMAC_ANTINF_SEL 23
+#define BIT_MASK_WMAC_ANTINF_SEL 0x3
+#define BIT_WMAC_ANTINF_SEL(x)                                                 \
+	(((x) & BIT_MASK_WMAC_ANTINF_SEL) << BIT_SHIFT_WMAC_ANTINF_SEL)
+#define BIT_GET_WMAC_ANTINF_SEL(x)                                             \
+	(((x) >> BIT_SHIFT_WMAC_ANTINF_SEL) & BIT_MASK_WMAC_ANTINF_SEL)
+
+#define BIT_SHIFT_WMAC_ANTSEL_SEL 21
+#define BIT_MASK_WMAC_ANTSEL_SEL 0x3
+#define BIT_WMAC_ANTSEL_SEL(x)                                                 \
+	(((x) & BIT_MASK_WMAC_ANTSEL_SEL) << BIT_SHIFT_WMAC_ANTSEL_SEL)
+#define BIT_GET_WMAC_ANTSEL_SEL(x)                                             \
+	(((x) >> BIT_SHIFT_WMAC_ANTSEL_SEL) & BIT_MASK_WMAC_ANTSEL_SEL)
+
+/* 2 REG_WMAC_RESP_TXINFO			(Offset 0x06D8) */
+
+#define BIT_SHIFT_R_WMAC_RESP_TXPOWER 18
+#define BIT_MASK_R_WMAC_RESP_TXPOWER 0x7
+#define BIT_R_WMAC_RESP_TXPOWER(x)                                             \
+	(((x) & BIT_MASK_R_WMAC_RESP_TXPOWER) << BIT_SHIFT_R_WMAC_RESP_TXPOWER)
+#define BIT_GET_R_WMAC_RESP_TXPOWER(x)                                         \
+	(((x) >> BIT_SHIFT_R_WMAC_RESP_TXPOWER) & BIT_MASK_R_WMAC_RESP_TXPOWER)
+
+/* 2 REG_WMAC_RESP_TXINFO			(Offset 0x06D8) */
+
+#define BIT_SHIFT_WMAC_RESP_TXANT 0
+#define BIT_MASK_WMAC_RESP_TXANT 0x3ffff
+#define BIT_WMAC_RESP_TXANT(x)                                                 \
+	(((x) & BIT_MASK_WMAC_RESP_TXANT) << BIT_SHIFT_WMAC_RESP_TXANT)
+#define BIT_GET_WMAC_RESP_TXANT(x)                                             \
+	(((x) >> BIT_SHIFT_WMAC_RESP_TXANT) & BIT_MASK_WMAC_RESP_TXANT)
+
+/* 2 REG_BBPSF_CTRL				(Offset 0x06DC) */
+
+#define BIT_CTL_IDLE_CLR_CSI_RPT BIT(31)
+
+/* 2 REG_BBPSF_CTRL				(Offset 0x06DC) */
+
+#define BIT_WMAC_USE_NDPARATE BIT(30)
+
+#define BIT_SHIFT_WMAC_CSI_RATE 24
+#define BIT_MASK_WMAC_CSI_RATE 0x3f
+#define BIT_WMAC_CSI_RATE(x)                                                   \
+	(((x) & BIT_MASK_WMAC_CSI_RATE) << BIT_SHIFT_WMAC_CSI_RATE)
+#define BIT_GET_WMAC_CSI_RATE(x)                                               \
+	(((x) >> BIT_SHIFT_WMAC_CSI_RATE) & BIT_MASK_WMAC_CSI_RATE)
+
+#define BIT_SHIFT_WMAC_RESP_TXRATE 16
+#define BIT_MASK_WMAC_RESP_TXRATE 0xff
+#define BIT_WMAC_RESP_TXRATE(x)                                                \
+	(((x) & BIT_MASK_WMAC_RESP_TXRATE) << BIT_SHIFT_WMAC_RESP_TXRATE)
+#define BIT_GET_WMAC_RESP_TXRATE(x)                                            \
+	(((x) >> BIT_SHIFT_WMAC_RESP_TXRATE) & BIT_MASK_WMAC_RESP_TXRATE)
+
+/* 2 REG_BBPSF_CTRL				(Offset 0x06DC) */
+
+#define BIT_BBPSF_MPDUCHKEN BIT(5)
+
+/* 2 REG_BBPSF_CTRL				(Offset 0x06DC) */
+
+#define BIT_BBPSF_MHCHKEN BIT(4)
+#define BIT_BBPSF_ERRCHKEN BIT(3)
+
+#define BIT_SHIFT_BBPSF_ERRTHR 0
+#define BIT_MASK_BBPSF_ERRTHR 0x7
+#define BIT_BBPSF_ERRTHR(x)                                                    \
+	(((x) & BIT_MASK_BBPSF_ERRTHR) << BIT_SHIFT_BBPSF_ERRTHR)
+#define BIT_GET_BBPSF_ERRTHR(x)                                                \
+	(((x) >> BIT_SHIFT_BBPSF_ERRTHR) & BIT_MASK_BBPSF_ERRTHR)
+
+/* 2 REG_P2P_RX_BCN_NOA			(Offset 0x06E0) */
+
+#define BIT_NOA_PARSER_EN BIT(15)
+
+/* 2 REG_P2P_RX_BCN_NOA			(Offset 0x06E0) */
+
+#define BIT_BSSID_SEL BIT(14)
+
+/* 2 REG_P2P_RX_BCN_NOA			(Offset 0x06E0) */
+
+#define BIT_SHIFT_P2P_OUI_TYPE 0
+#define BIT_MASK_P2P_OUI_TYPE 0xff
+#define BIT_P2P_OUI_TYPE(x)                                                    \
+	(((x) & BIT_MASK_P2P_OUI_TYPE) << BIT_SHIFT_P2P_OUI_TYPE)
+#define BIT_GET_P2P_OUI_TYPE(x)                                                \
+	(((x) >> BIT_SHIFT_P2P_OUI_TYPE) & BIT_MASK_P2P_OUI_TYPE)
+
+/* 2 REG_ASSOCIATED_BFMER0_INFO		(Offset 0x06E4) */
+
+#define BIT_SHIFT_R_WMAC_TXCSI_AID0 (48 & CPU_OPT_WIDTH)
+#define BIT_MASK_R_WMAC_TXCSI_AID0 0x1ff
+#define BIT_R_WMAC_TXCSI_AID0(x)                                               \
+	(((x) & BIT_MASK_R_WMAC_TXCSI_AID0) << BIT_SHIFT_R_WMAC_TXCSI_AID0)
+#define BIT_GET_R_WMAC_TXCSI_AID0(x)                                           \
+	(((x) >> BIT_SHIFT_R_WMAC_TXCSI_AID0) & BIT_MASK_R_WMAC_TXCSI_AID0)
+
+#define BIT_SHIFT_R_WMAC_SOUNDING_RXADD_R0 0
+#define BIT_MASK_R_WMAC_SOUNDING_RXADD_R0 0xffffffffffffL
+#define BIT_R_WMAC_SOUNDING_RXADD_R0(x)                                        \
+	(((x) & BIT_MASK_R_WMAC_SOUNDING_RXADD_R0)                             \
+	 << BIT_SHIFT_R_WMAC_SOUNDING_RXADD_R0)
+#define BIT_GET_R_WMAC_SOUNDING_RXADD_R0(x)                                    \
+	(((x) >> BIT_SHIFT_R_WMAC_SOUNDING_RXADD_R0) &                         \
+	 BIT_MASK_R_WMAC_SOUNDING_RXADD_R0)
+
+/* 2 REG_ASSOCIATED_BFMER1_INFO		(Offset 0x06EC) */
+
+#define BIT_SHIFT_R_WMAC_TXCSI_AID1 (48 & CPU_OPT_WIDTH)
+#define BIT_MASK_R_WMAC_TXCSI_AID1 0x1ff
+#define BIT_R_WMAC_TXCSI_AID1(x)                                               \
+	(((x) & BIT_MASK_R_WMAC_TXCSI_AID1) << BIT_SHIFT_R_WMAC_TXCSI_AID1)
+#define BIT_GET_R_WMAC_TXCSI_AID1(x)                                           \
+	(((x) >> BIT_SHIFT_R_WMAC_TXCSI_AID1) & BIT_MASK_R_WMAC_TXCSI_AID1)
+
+#define BIT_SHIFT_R_WMAC_SOUNDING_RXADD_R1 0
+#define BIT_MASK_R_WMAC_SOUNDING_RXADD_R1 0xffffffffffffL
+#define BIT_R_WMAC_SOUNDING_RXADD_R1(x)                                        \
+	(((x) & BIT_MASK_R_WMAC_SOUNDING_RXADD_R1)                             \
+	 << BIT_SHIFT_R_WMAC_SOUNDING_RXADD_R1)
+#define BIT_GET_R_WMAC_SOUNDING_RXADD_R1(x)                                    \
+	(((x) >> BIT_SHIFT_R_WMAC_SOUNDING_RXADD_R1) &                         \
+	 BIT_MASK_R_WMAC_SOUNDING_RXADD_R1)
+
+/* 2 REG_TX_CSI_RPT_PARAM_BW20		(Offset 0x06F4) */
+
+#define BIT_SHIFT_R_WMAC_BFINFO_20M_1 16
+#define BIT_MASK_R_WMAC_BFINFO_20M_1 0xfff
+#define BIT_R_WMAC_BFINFO_20M_1(x)                                             \
+	(((x) & BIT_MASK_R_WMAC_BFINFO_20M_1) << BIT_SHIFT_R_WMAC_BFINFO_20M_1)
+#define BIT_GET_R_WMAC_BFINFO_20M_1(x)                                         \
+	(((x) >> BIT_SHIFT_R_WMAC_BFINFO_20M_1) & BIT_MASK_R_WMAC_BFINFO_20M_1)
+
+#define BIT_SHIFT_R_WMAC_BFINFO_20M_0 0
+#define BIT_MASK_R_WMAC_BFINFO_20M_0 0xfff
+#define BIT_R_WMAC_BFINFO_20M_0(x)                                             \
+	(((x) & BIT_MASK_R_WMAC_BFINFO_20M_0) << BIT_SHIFT_R_WMAC_BFINFO_20M_0)
+#define BIT_GET_R_WMAC_BFINFO_20M_0(x)                                         \
+	(((x) >> BIT_SHIFT_R_WMAC_BFINFO_20M_0) & BIT_MASK_R_WMAC_BFINFO_20M_0)
+
+/* 2 REG_TX_CSI_RPT_PARAM_BW40		(Offset 0x06F8) */
+
+#define BIT_SHIFT_WMAC_RESP_ANTCD 0
+#define BIT_MASK_WMAC_RESP_ANTCD 0xf
+#define BIT_WMAC_RESP_ANTCD(x)                                                 \
+	(((x) & BIT_MASK_WMAC_RESP_ANTCD) << BIT_SHIFT_WMAC_RESP_ANTCD)
+#define BIT_GET_WMAC_RESP_ANTCD(x)                                             \
+	(((x) >> BIT_SHIFT_WMAC_RESP_ANTCD) & BIT_MASK_WMAC_RESP_ANTCD)
+
+/* 2 REG_MACID1				(Offset 0x0700) */
+
+#define BIT_SHIFT_MACID1 0
+#define BIT_MASK_MACID1 0xffffffffffffL
+#define BIT_MACID1(x) (((x) & BIT_MASK_MACID1) << BIT_SHIFT_MACID1)
+#define BIT_GET_MACID1(x) (((x) >> BIT_SHIFT_MACID1) & BIT_MASK_MACID1)
+
+/* 2 REG_BSSID1				(Offset 0x0708) */
+
+#define BIT_SHIFT_BSSID1 0
+#define BIT_MASK_BSSID1 0xffffffffffffL
+#define BIT_BSSID1(x) (((x) & BIT_MASK_BSSID1) << BIT_SHIFT_BSSID1)
+#define BIT_GET_BSSID1(x) (((x) >> BIT_SHIFT_BSSID1) & BIT_MASK_BSSID1)
+
+/* 2 REG_BCN_PSR_RPT1			(Offset 0x0710) */
+
+#define BIT_SHIFT_DTIM_CNT1 24
+#define BIT_MASK_DTIM_CNT1 0xff
+#define BIT_DTIM_CNT1(x) (((x) & BIT_MASK_DTIM_CNT1) << BIT_SHIFT_DTIM_CNT1)
+#define BIT_GET_DTIM_CNT1(x) (((x) >> BIT_SHIFT_DTIM_CNT1) & BIT_MASK_DTIM_CNT1)
+
+#define BIT_SHIFT_DTIM_PERIOD1 16
+#define BIT_MASK_DTIM_PERIOD1 0xff
+#define BIT_DTIM_PERIOD1(x)                                                    \
+	(((x) & BIT_MASK_DTIM_PERIOD1) << BIT_SHIFT_DTIM_PERIOD1)
+#define BIT_GET_DTIM_PERIOD1(x)                                                \
+	(((x) >> BIT_SHIFT_DTIM_PERIOD1) & BIT_MASK_DTIM_PERIOD1)
+
+#define BIT_DTIM1 BIT(15)
+#define BIT_TIM1 BIT(14)
+
+#define BIT_SHIFT_PS_AID_1 0
+#define BIT_MASK_PS_AID_1 0x7ff
+#define BIT_PS_AID_1(x) (((x) & BIT_MASK_PS_AID_1) << BIT_SHIFT_PS_AID_1)
+#define BIT_GET_PS_AID_1(x) (((x) >> BIT_SHIFT_PS_AID_1) & BIT_MASK_PS_AID_1)
+
+/* 2 REG_ASSOCIATED_BFMEE_SEL		(Offset 0x0714) */
+
+#define BIT_TXUSER_ID1 BIT(25)
+
+#define BIT_SHIFT_AID1 16
+#define BIT_MASK_AID1 0x1ff
+#define BIT_AID1(x) (((x) & BIT_MASK_AID1) << BIT_SHIFT_AID1)
+#define BIT_GET_AID1(x) (((x) >> BIT_SHIFT_AID1) & BIT_MASK_AID1)
+
+#define BIT_TXUSER_ID0 BIT(9)
+
+#define BIT_SHIFT_AID0 0
+#define BIT_MASK_AID0 0x1ff
+#define BIT_AID0(x) (((x) & BIT_MASK_AID0) << BIT_SHIFT_AID0)
+#define BIT_GET_AID0(x) (((x) >> BIT_SHIFT_AID0) & BIT_MASK_AID0)
+
+/* 2 REG_SND_PTCL_CTRL			(Offset 0x0718) */
+
+#define BIT_SHIFT_NDP_RX_STANDBY_TIMER 24
+#define BIT_MASK_NDP_RX_STANDBY_TIMER 0xff
+#define BIT_NDP_RX_STANDBY_TIMER(x)                                            \
+	(((x) & BIT_MASK_NDP_RX_STANDBY_TIMER)                                 \
+	 << BIT_SHIFT_NDP_RX_STANDBY_TIMER)
+#define BIT_GET_NDP_RX_STANDBY_TIMER(x)                                        \
+	(((x) >> BIT_SHIFT_NDP_RX_STANDBY_TIMER) &                             \
+	 BIT_MASK_NDP_RX_STANDBY_TIMER)
+
+#define BIT_SHIFT_CSI_RPT_OFFSET_HT 16
+#define BIT_MASK_CSI_RPT_OFFSET_HT 0xff
+#define BIT_CSI_RPT_OFFSET_HT(x)                                               \
+	(((x) & BIT_MASK_CSI_RPT_OFFSET_HT) << BIT_SHIFT_CSI_RPT_OFFSET_HT)
+#define BIT_GET_CSI_RPT_OFFSET_HT(x)                                           \
+	(((x) >> BIT_SHIFT_CSI_RPT_OFFSET_HT) & BIT_MASK_CSI_RPT_OFFSET_HT)
+
+/* 2 REG_SND_PTCL_CTRL			(Offset 0x0718) */
+
+#define BIT_SHIFT_R_WMAC_VHT_CATEGORY 8
+#define BIT_MASK_R_WMAC_VHT_CATEGORY 0xff
+#define BIT_R_WMAC_VHT_CATEGORY(x)                                             \
+	(((x) & BIT_MASK_R_WMAC_VHT_CATEGORY) << BIT_SHIFT_R_WMAC_VHT_CATEGORY)
+#define BIT_GET_R_WMAC_VHT_CATEGORY(x)                                         \
+	(((x) >> BIT_SHIFT_R_WMAC_VHT_CATEGORY) & BIT_MASK_R_WMAC_VHT_CATEGORY)
+
+/* 2 REG_SND_PTCL_CTRL			(Offset 0x0718) */
+
+#define BIT_R_WMAC_USE_NSTS BIT(7)
+#define BIT_R_DISABLE_CHECK_VHTSIGB_CRC BIT(6)
+#define BIT_R_DISABLE_CHECK_VHTSIGA_CRC BIT(5)
+#define BIT_R_WMAC_BFPARAM_SEL BIT(4)
+#define BIT_R_WMAC_CSISEQ_SEL BIT(3)
+#define BIT_R_WMAC_CSI_WITHHTC_EN BIT(2)
+#define BIT_R_WMAC_HT_NDPA_EN BIT(1)
+#define BIT_R_WMAC_VHT_NDPA_EN BIT(0)
+
+/* 2 REG_NS_ARP_CTRL				(Offset 0x0720) */
+
+#define BIT_R_WMAC_NSARP_RSPEN BIT(15)
+#define BIT_R_WMAC_NSARP_RARP BIT(9)
+#define BIT_R_WMAC_NSARP_RIPV6 BIT(8)
+
+#define BIT_SHIFT_R_WMAC_NSARP_MODEN 6
+#define BIT_MASK_R_WMAC_NSARP_MODEN 0x3
+#define BIT_R_WMAC_NSARP_MODEN(x)                                              \
+	(((x) & BIT_MASK_R_WMAC_NSARP_MODEN) << BIT_SHIFT_R_WMAC_NSARP_MODEN)
+#define BIT_GET_R_WMAC_NSARP_MODEN(x)                                          \
+	(((x) >> BIT_SHIFT_R_WMAC_NSARP_MODEN) & BIT_MASK_R_WMAC_NSARP_MODEN)
+
+#define BIT_SHIFT_R_WMAC_NSARP_RSPFTP 4
+#define BIT_MASK_R_WMAC_NSARP_RSPFTP 0x3
+#define BIT_R_WMAC_NSARP_RSPFTP(x)                                             \
+	(((x) & BIT_MASK_R_WMAC_NSARP_RSPFTP) << BIT_SHIFT_R_WMAC_NSARP_RSPFTP)
+#define BIT_GET_R_WMAC_NSARP_RSPFTP(x)                                         \
+	(((x) >> BIT_SHIFT_R_WMAC_NSARP_RSPFTP) & BIT_MASK_R_WMAC_NSARP_RSPFTP)
+
+#define BIT_SHIFT_R_WMAC_NSARP_RSPSEC 0
+#define BIT_MASK_R_WMAC_NSARP_RSPSEC 0xf
+#define BIT_R_WMAC_NSARP_RSPSEC(x)                                             \
+	(((x) & BIT_MASK_R_WMAC_NSARP_RSPSEC) << BIT_SHIFT_R_WMAC_NSARP_RSPSEC)
+#define BIT_GET_R_WMAC_NSARP_RSPSEC(x)                                         \
+	(((x) >> BIT_SHIFT_R_WMAC_NSARP_RSPSEC) & BIT_MASK_R_WMAC_NSARP_RSPSEC)
+
+/* 2 REG_NS_ARP_INFO				(Offset 0x0724) */
+
+#define BIT_REQ_IS_MCNS BIT(23)
+#define BIT_REQ_IS_UCNS BIT(22)
+#define BIT_REQ_IS_USNS BIT(21)
+#define BIT_REQ_IS_ARP BIT(20)
+#define BIT_EXPRSP_MH_WITHQC BIT(19)
+
+#define BIT_SHIFT_EXPRSP_SECTYPE 16
+#define BIT_MASK_EXPRSP_SECTYPE 0x7
+#define BIT_EXPRSP_SECTYPE(x)                                                  \
+	(((x) & BIT_MASK_EXPRSP_SECTYPE) << BIT_SHIFT_EXPRSP_SECTYPE)
+#define BIT_GET_EXPRSP_SECTYPE(x)                                              \
+	(((x) >> BIT_SHIFT_EXPRSP_SECTYPE) & BIT_MASK_EXPRSP_SECTYPE)
+
+#define BIT_SHIFT_EXPRSP_CHKSM_7_TO_0 8
+#define BIT_MASK_EXPRSP_CHKSM_7_TO_0 0xff
+#define BIT_EXPRSP_CHKSM_7_TO_0(x)                                             \
+	(((x) & BIT_MASK_EXPRSP_CHKSM_7_TO_0) << BIT_SHIFT_EXPRSP_CHKSM_7_TO_0)
+#define BIT_GET_EXPRSP_CHKSM_7_TO_0(x)                                         \
+	(((x) >> BIT_SHIFT_EXPRSP_CHKSM_7_TO_0) & BIT_MASK_EXPRSP_CHKSM_7_TO_0)
+
+#define BIT_SHIFT_EXPRSP_CHKSM_15_TO_8 0
+#define BIT_MASK_EXPRSP_CHKSM_15_TO_8 0xff
+#define BIT_EXPRSP_CHKSM_15_TO_8(x)                                            \
+	(((x) & BIT_MASK_EXPRSP_CHKSM_15_TO_8)                                 \
+	 << BIT_SHIFT_EXPRSP_CHKSM_15_TO_8)
+#define BIT_GET_EXPRSP_CHKSM_15_TO_8(x)                                        \
+	(((x) >> BIT_SHIFT_EXPRSP_CHKSM_15_TO_8) &                             \
+	 BIT_MASK_EXPRSP_CHKSM_15_TO_8)
+
+/* 2 REG_BEAMFORMING_INFO_NSARP_V1		(Offset 0x0728) */
+
+#define BIT_SHIFT_WMAC_ARPIP 0
+#define BIT_MASK_WMAC_ARPIP 0xffffffffL
+#define BIT_WMAC_ARPIP(x) (((x) & BIT_MASK_WMAC_ARPIP) << BIT_SHIFT_WMAC_ARPIP)
+#define BIT_GET_WMAC_ARPIP(x)                                                  \
+	(((x) >> BIT_SHIFT_WMAC_ARPIP) & BIT_MASK_WMAC_ARPIP)
+
+/* 2 REG_BEAMFORMING_INFO_NSARP		(Offset 0x072C) */
+
+#define BIT_SHIFT_BEAMFORMING_INFO 0
+#define BIT_MASK_BEAMFORMING_INFO 0xffffffffL
+#define BIT_BEAMFORMING_INFO(x)                                                \
+	(((x) & BIT_MASK_BEAMFORMING_INFO) << BIT_SHIFT_BEAMFORMING_INFO)
+#define BIT_GET_BEAMFORMING_INFO(x)                                            \
+	(((x) >> BIT_SHIFT_BEAMFORMING_INFO) & BIT_MASK_BEAMFORMING_INFO)
+
+/* 2 REG_WMAC_RTX_CTX_SUBTYPE_CFG		(Offset 0x0750) */
+
+#define BIT_SHIFT_R_WMAC_CTX_SUBTYPE 4
+#define BIT_MASK_R_WMAC_CTX_SUBTYPE 0xf
+#define BIT_R_WMAC_CTX_SUBTYPE(x)                                              \
+	(((x) & BIT_MASK_R_WMAC_CTX_SUBTYPE) << BIT_SHIFT_R_WMAC_CTX_SUBTYPE)
+#define BIT_GET_R_WMAC_CTX_SUBTYPE(x)                                          \
+	(((x) >> BIT_SHIFT_R_WMAC_CTX_SUBTYPE) & BIT_MASK_R_WMAC_CTX_SUBTYPE)
+
+#define BIT_SHIFT_R_WMAC_RTX_SUBTYPE 0
+#define BIT_MASK_R_WMAC_RTX_SUBTYPE 0xf
+#define BIT_R_WMAC_RTX_SUBTYPE(x)                                              \
+	(((x) & BIT_MASK_R_WMAC_RTX_SUBTYPE) << BIT_SHIFT_R_WMAC_RTX_SUBTYPE)
+#define BIT_GET_R_WMAC_RTX_SUBTYPE(x)                                          \
+	(((x) >> BIT_SHIFT_R_WMAC_RTX_SUBTYPE) & BIT_MASK_R_WMAC_RTX_SUBTYPE)
+
+/* 2 REG_BT_COEX_V2				(Offset 0x0762) */
+
+#define BIT_GNT_BT_POLARITY BIT(12)
+#define BIT_GNT_BT_BYPASS_PRIORITY BIT(8)
+
+#define BIT_SHIFT_TIMER 0
+#define BIT_MASK_TIMER 0xff
+#define BIT_TIMER(x) (((x) & BIT_MASK_TIMER) << BIT_SHIFT_TIMER)
+#define BIT_GET_TIMER(x) (((x) >> BIT_SHIFT_TIMER) & BIT_MASK_TIMER)
+
+/* 2 REG_BT_COEX				(Offset 0x0764) */
+
+#define BIT_R_GNT_BT_RFC_SW BIT(12)
+#define BIT_R_GNT_BT_RFC_SW_EN BIT(11)
+#define BIT_R_GNT_BT_BB_SW BIT(10)
+#define BIT_R_GNT_BT_BB_SW_EN BIT(9)
+#define BIT_R_BT_CNT_THREN BIT(8)
+
+#define BIT_SHIFT_R_BT_CNT_THR 0
+#define BIT_MASK_R_BT_CNT_THR 0xff
+#define BIT_R_BT_CNT_THR(x)                                                    \
+	(((x) & BIT_MASK_R_BT_CNT_THR) << BIT_SHIFT_R_BT_CNT_THR)
+#define BIT_GET_R_BT_CNT_THR(x)                                                \
+	(((x) >> BIT_SHIFT_R_BT_CNT_THR) & BIT_MASK_R_BT_CNT_THR)
+
+/* 2 REG_WLAN_ACT_MASK_CTRL			(Offset 0x0768) */
+
+#define BIT_WLRX_TER_BY_CTL BIT(43)
+#define BIT_WLRX_TER_BY_AD BIT(42)
+#define BIT_ANT_DIVERSITY_SEL BIT(41)
+#define BIT_ANTSEL_FOR_BT_CTRL_EN BIT(40)
+#define BIT_WLACT_LOW_GNTWL_EN BIT(34)
+#define BIT_WLACT_HIGH_GNTBT_EN BIT(33)
+
+/* 2 REG_WLAN_ACT_MASK_CTRL			(Offset 0x0768) */
+
+#define BIT_NAV_UPPER_V1 BIT(32)
+
+/* 2 REG_WLAN_ACT_MASK_CTRL			(Offset 0x0768) */
+
+#define BIT_SHIFT_RXMYRTS_NAV_V1 8
+#define BIT_MASK_RXMYRTS_NAV_V1 0xff
+#define BIT_RXMYRTS_NAV_V1(x)                                                  \
+	(((x) & BIT_MASK_RXMYRTS_NAV_V1) << BIT_SHIFT_RXMYRTS_NAV_V1)
+#define BIT_GET_RXMYRTS_NAV_V1(x)                                              \
+	(((x) >> BIT_SHIFT_RXMYRTS_NAV_V1) & BIT_MASK_RXMYRTS_NAV_V1)
+
+#define BIT_SHIFT_RTSRST_V1 0
+#define BIT_MASK_RTSRST_V1 0xff
+#define BIT_RTSRST_V1(x) (((x) & BIT_MASK_RTSRST_V1) << BIT_SHIFT_RTSRST_V1)
+#define BIT_GET_RTSRST_V1(x) (((x) >> BIT_SHIFT_RTSRST_V1) & BIT_MASK_RTSRST_V1)
+
+/* 2 REG_BT_COEX_ENHANCED_INTR_CTRL		(Offset 0x076E) */
+
+#define BIT_SHIFT_BT_STAT_DELAY 12
+#define BIT_MASK_BT_STAT_DELAY 0xf
+#define BIT_BT_STAT_DELAY(x)                                                   \
+	(((x) & BIT_MASK_BT_STAT_DELAY) << BIT_SHIFT_BT_STAT_DELAY)
+#define BIT_GET_BT_STAT_DELAY(x)                                               \
+	(((x) >> BIT_SHIFT_BT_STAT_DELAY) & BIT_MASK_BT_STAT_DELAY)
+
+#define BIT_SHIFT_BT_TRX_INIT_DETECT 8
+#define BIT_MASK_BT_TRX_INIT_DETECT 0xf
+#define BIT_BT_TRX_INIT_DETECT(x)                                              \
+	(((x) & BIT_MASK_BT_TRX_INIT_DETECT) << BIT_SHIFT_BT_TRX_INIT_DETECT)
+#define BIT_GET_BT_TRX_INIT_DETECT(x)                                          \
+	(((x) >> BIT_SHIFT_BT_TRX_INIT_DETECT) & BIT_MASK_BT_TRX_INIT_DETECT)
+
+#define BIT_SHIFT_BT_PRI_DETECT_TO 4
+#define BIT_MASK_BT_PRI_DETECT_TO 0xf
+#define BIT_BT_PRI_DETECT_TO(x)                                                \
+	(((x) & BIT_MASK_BT_PRI_DETECT_TO) << BIT_SHIFT_BT_PRI_DETECT_TO)
+#define BIT_GET_BT_PRI_DETECT_TO(x)                                            \
+	(((x) >> BIT_SHIFT_BT_PRI_DETECT_TO) & BIT_MASK_BT_PRI_DETECT_TO)
+
+#define BIT_R_GRANTALL_WLMASK BIT(3)
+#define BIT_STATIS_BT_EN BIT(2)
+#define BIT_WL_ACT_MASK_ENABLE BIT(1)
+#define BIT_ENHANCED_BT BIT(0)
+
+/* 2 REG_BT_ACT_STATISTICS			(Offset 0x0770) */
+
+#define BIT_SHIFT_STATIS_BT_LO_RX (48 & CPU_OPT_WIDTH)
+#define BIT_MASK_STATIS_BT_LO_RX 0xffff
+#define BIT_STATIS_BT_LO_RX(x)                                                 \
+	(((x) & BIT_MASK_STATIS_BT_LO_RX) << BIT_SHIFT_STATIS_BT_LO_RX)
+#define BIT_GET_STATIS_BT_LO_RX(x)                                             \
+	(((x) >> BIT_SHIFT_STATIS_BT_LO_RX) & BIT_MASK_STATIS_BT_LO_RX)
+
+#define BIT_SHIFT_STATIS_BT_LO_TX (32 & CPU_OPT_WIDTH)
+#define BIT_MASK_STATIS_BT_LO_TX 0xffff
+#define BIT_STATIS_BT_LO_TX(x)                                                 \
+	(((x) & BIT_MASK_STATIS_BT_LO_TX) << BIT_SHIFT_STATIS_BT_LO_TX)
+#define BIT_GET_STATIS_BT_LO_TX(x)                                             \
+	(((x) >> BIT_SHIFT_STATIS_BT_LO_TX) & BIT_MASK_STATIS_BT_LO_TX)
+
+/* 2 REG_BT_ACT_STATISTICS			(Offset 0x0770) */
+
+#define BIT_SHIFT_STATIS_BT_HI_RX 16
+#define BIT_MASK_STATIS_BT_HI_RX 0xffff
+#define BIT_STATIS_BT_HI_RX(x)                                                 \
+	(((x) & BIT_MASK_STATIS_BT_HI_RX) << BIT_SHIFT_STATIS_BT_HI_RX)
+#define BIT_GET_STATIS_BT_HI_RX(x)                                             \
+	(((x) >> BIT_SHIFT_STATIS_BT_HI_RX) & BIT_MASK_STATIS_BT_HI_RX)
+
+#define BIT_SHIFT_STATIS_BT_HI_TX 0
+#define BIT_MASK_STATIS_BT_HI_TX 0xffff
+#define BIT_STATIS_BT_HI_TX(x)                                                 \
+	(((x) & BIT_MASK_STATIS_BT_HI_TX) << BIT_SHIFT_STATIS_BT_HI_TX)
+#define BIT_GET_STATIS_BT_HI_TX(x)                                             \
+	(((x) >> BIT_SHIFT_STATIS_BT_HI_TX) & BIT_MASK_STATIS_BT_HI_TX)
+
+/* 2 REG_BT_STATISTICS_CONTROL_REGISTER	(Offset 0x0778) */
+
+#define BIT_SHIFT_R_BT_CMD_RPT 16
+#define BIT_MASK_R_BT_CMD_RPT 0xffff
+#define BIT_R_BT_CMD_RPT(x)                                                    \
+	(((x) & BIT_MASK_R_BT_CMD_RPT) << BIT_SHIFT_R_BT_CMD_RPT)
+#define BIT_GET_R_BT_CMD_RPT(x)                                                \
+	(((x) >> BIT_SHIFT_R_BT_CMD_RPT) & BIT_MASK_R_BT_CMD_RPT)
+
+#define BIT_SHIFT_R_RPT_FROM_BT 8
+#define BIT_MASK_R_RPT_FROM_BT 0xff
+#define BIT_R_RPT_FROM_BT(x)                                                   \
+	(((x) & BIT_MASK_R_RPT_FROM_BT) << BIT_SHIFT_R_RPT_FROM_BT)
+#define BIT_GET_R_RPT_FROM_BT(x)                                               \
+	(((x) >> BIT_SHIFT_R_RPT_FROM_BT) & BIT_MASK_R_RPT_FROM_BT)
+
+#define BIT_SHIFT_BT_HID_ISR_SET 6
+#define BIT_MASK_BT_HID_ISR_SET 0x3
+#define BIT_BT_HID_ISR_SET(x)                                                  \
+	(((x) & BIT_MASK_BT_HID_ISR_SET) << BIT_SHIFT_BT_HID_ISR_SET)
+#define BIT_GET_BT_HID_ISR_SET(x)                                              \
+	(((x) >> BIT_SHIFT_BT_HID_ISR_SET) & BIT_MASK_BT_HID_ISR_SET)
+
+#define BIT_TDMA_BT_START_NOTIFY BIT(5)
+#define BIT_ENABLE_TDMA_FW_MODE BIT(4)
+#define BIT_ENABLE_PTA_TDMA_MODE BIT(3)
+#define BIT_ENABLE_COEXIST_TAB_IN_TDMA BIT(2)
+#define BIT_GPIO2_GPIO3_EXANGE_OR_NO_BT_CCA BIT(1)
+#define BIT_RTK_BT_ENABLE BIT(0)
+
+/* 2 REG_BT_STATUS_REPORT_REGISTER		(Offset 0x077C) */
+
+#define BIT_SHIFT_BT_PROFILE 24
+#define BIT_MASK_BT_PROFILE 0xff
+#define BIT_BT_PROFILE(x) (((x) & BIT_MASK_BT_PROFILE) << BIT_SHIFT_BT_PROFILE)
+#define BIT_GET_BT_PROFILE(x)                                                  \
+	(((x) >> BIT_SHIFT_BT_PROFILE) & BIT_MASK_BT_PROFILE)
+
+#define BIT_SHIFT_BT_POWER 16
+#define BIT_MASK_BT_POWER 0xff
+#define BIT_BT_POWER(x) (((x) & BIT_MASK_BT_POWER) << BIT_SHIFT_BT_POWER)
+#define BIT_GET_BT_POWER(x) (((x) >> BIT_SHIFT_BT_POWER) & BIT_MASK_BT_POWER)
+
+#define BIT_SHIFT_BT_PREDECT_STATUS 8
+#define BIT_MASK_BT_PREDECT_STATUS 0xff
+#define BIT_BT_PREDECT_STATUS(x)                                               \
+	(((x) & BIT_MASK_BT_PREDECT_STATUS) << BIT_SHIFT_BT_PREDECT_STATUS)
+#define BIT_GET_BT_PREDECT_STATUS(x)                                           \
+	(((x) >> BIT_SHIFT_BT_PREDECT_STATUS) & BIT_MASK_BT_PREDECT_STATUS)
+
+#define BIT_SHIFT_BT_CMD_INFO 0
+#define BIT_MASK_BT_CMD_INFO 0xff
+#define BIT_BT_CMD_INFO(x)                                                     \
+	(((x) & BIT_MASK_BT_CMD_INFO) << BIT_SHIFT_BT_CMD_INFO)
+#define BIT_GET_BT_CMD_INFO(x)                                                 \
+	(((x) >> BIT_SHIFT_BT_CMD_INFO) & BIT_MASK_BT_CMD_INFO)
+
+/* 2 REG_BT_INTERRUPT_CONTROL_REGISTER	(Offset 0x0780) */
+
+#define BIT_EN_MAC_NULL_PKT_NOTIFY BIT(31)
+#define BIT_EN_WLAN_RPT_AND_BT_QUERY BIT(30)
+#define BIT_EN_BT_STSTUS_RPT BIT(29)
+#define BIT_EN_BT_POWER BIT(28)
+#define BIT_EN_BT_CHANNEL BIT(27)
+#define BIT_EN_BT_SLOT_CHANGE BIT(26)
+#define BIT_EN_BT_PROFILE_OR_HID BIT(25)
+#define BIT_WLAN_RPT_NOTIFY BIT(24)
+
+#define BIT_SHIFT_WLAN_RPT_DATA 16
+#define BIT_MASK_WLAN_RPT_DATA 0xff
+#define BIT_WLAN_RPT_DATA(x)                                                   \
+	(((x) & BIT_MASK_WLAN_RPT_DATA) << BIT_SHIFT_WLAN_RPT_DATA)
+#define BIT_GET_WLAN_RPT_DATA(x)                                               \
+	(((x) >> BIT_SHIFT_WLAN_RPT_DATA) & BIT_MASK_WLAN_RPT_DATA)
+
+#define BIT_SHIFT_CMD_ID 8
+#define BIT_MASK_CMD_ID 0xff
+#define BIT_CMD_ID(x) (((x) & BIT_MASK_CMD_ID) << BIT_SHIFT_CMD_ID)
+#define BIT_GET_CMD_ID(x) (((x) >> BIT_SHIFT_CMD_ID) & BIT_MASK_CMD_ID)
+
+#define BIT_SHIFT_BT_DATA 0
+#define BIT_MASK_BT_DATA 0xff
+#define BIT_BT_DATA(x) (((x) & BIT_MASK_BT_DATA) << BIT_SHIFT_BT_DATA)
+#define BIT_GET_BT_DATA(x) (((x) >> BIT_SHIFT_BT_DATA) & BIT_MASK_BT_DATA)
+
+/* 2 REG_WLAN_REPORT_TIME_OUT_CONTROL_REGISTER (Offset 0x0784) */
+
+#define BIT_SHIFT_WLAN_RPT_TO 0
+#define BIT_MASK_WLAN_RPT_TO 0xff
+#define BIT_WLAN_RPT_TO(x)                                                     \
+	(((x) & BIT_MASK_WLAN_RPT_TO) << BIT_SHIFT_WLAN_RPT_TO)
+#define BIT_GET_WLAN_RPT_TO(x)                                                 \
+	(((x) >> BIT_SHIFT_WLAN_RPT_TO) & BIT_MASK_WLAN_RPT_TO)
+
+/* 2 REG_BT_ISOLATION_TABLE_REGISTER_REGISTER (Offset 0x0785) */
+
+#define BIT_SHIFT_ISOLATION_CHK 1
+#define BIT_MASK_ISOLATION_CHK 0x7fffffffffffffffffffL
+#define BIT_ISOLATION_CHK(x)                                                   \
+	(((x) & BIT_MASK_ISOLATION_CHK) << BIT_SHIFT_ISOLATION_CHK)
+#define BIT_GET_ISOLATION_CHK(x)                                               \
+	(((x) >> BIT_SHIFT_ISOLATION_CHK) & BIT_MASK_ISOLATION_CHK)
+
+/* 2 REG_BT_ISOLATION_TABLE_REGISTER_REGISTER (Offset 0x0785) */
+
+#define BIT_ISOLATION_EN BIT(0)
+
+/* 2 REG_BT_INTERRUPT_STATUS_REGISTER	(Offset 0x078F) */
+
+#define BIT_BT_HID_ISR BIT(7)
+#define BIT_BT_QUERY_ISR BIT(6)
+#define BIT_MAC_NULL_PKT_NOTIFY_ISR BIT(5)
+#define BIT_WLAN_RPT_ISR BIT(4)
+#define BIT_BT_POWER_ISR BIT(3)
+#define BIT_BT_CHANNEL_ISR BIT(2)
+#define BIT_BT_SLOT_CHANGE_ISR BIT(1)
+#define BIT_BT_PROFILE_ISR BIT(0)
+
+/* 2 REG_BT_TDMA_TIME_REGISTER		(Offset 0x0790) */
+
+#define BIT_SHIFT_BT_TIME 6
+#define BIT_MASK_BT_TIME 0x3ffffff
+#define BIT_BT_TIME(x) (((x) & BIT_MASK_BT_TIME) << BIT_SHIFT_BT_TIME)
+#define BIT_GET_BT_TIME(x) (((x) >> BIT_SHIFT_BT_TIME) & BIT_MASK_BT_TIME)
+
+#define BIT_SHIFT_BT_RPT_SAMPLE_RATE 0
+#define BIT_MASK_BT_RPT_SAMPLE_RATE 0x3f
+#define BIT_BT_RPT_SAMPLE_RATE(x)                                              \
+	(((x) & BIT_MASK_BT_RPT_SAMPLE_RATE) << BIT_SHIFT_BT_RPT_SAMPLE_RATE)
+#define BIT_GET_BT_RPT_SAMPLE_RATE(x)                                          \
+	(((x) >> BIT_SHIFT_BT_RPT_SAMPLE_RATE) & BIT_MASK_BT_RPT_SAMPLE_RATE)
+
+/* 2 REG_BT_ACT_REGISTER			(Offset 0x0794) */
+
+#define BIT_SHIFT_BT_EISR_EN 16
+#define BIT_MASK_BT_EISR_EN 0xff
+#define BIT_BT_EISR_EN(x) (((x) & BIT_MASK_BT_EISR_EN) << BIT_SHIFT_BT_EISR_EN)
+#define BIT_GET_BT_EISR_EN(x)                                                  \
+	(((x) >> BIT_SHIFT_BT_EISR_EN) & BIT_MASK_BT_EISR_EN)
+
+#define BIT_BT_ACT_FALLING_ISR BIT(10)
+#define BIT_BT_ACT_RISING_ISR BIT(9)
+#define BIT_TDMA_TO_ISR BIT(8)
+
+#define BIT_SHIFT_BT_CH 0
+#define BIT_MASK_BT_CH 0xff
+#define BIT_BT_CH(x) (((x) & BIT_MASK_BT_CH) << BIT_SHIFT_BT_CH)
+#define BIT_GET_BT_CH(x) (((x) >> BIT_SHIFT_BT_CH) & BIT_MASK_BT_CH)
+
+/* 2 REG_OBFF_CTRL_BASIC			(Offset 0x0798) */
+
+#define BIT_OBFF_EN_V1 BIT(31)
+
+#define BIT_SHIFT_OBFF_STATE_V1 28
+#define BIT_MASK_OBFF_STATE_V1 0x3
+#define BIT_OBFF_STATE_V1(x)                                                   \
+	(((x) & BIT_MASK_OBFF_STATE_V1) << BIT_SHIFT_OBFF_STATE_V1)
+#define BIT_GET_OBFF_STATE_V1(x)                                               \
+	(((x) >> BIT_SHIFT_OBFF_STATE_V1) & BIT_MASK_OBFF_STATE_V1)
+
+#define BIT_OBFF_ACT_RXDMA_EN BIT(27)
+#define BIT_OBFF_BLOCK_INT_EN BIT(26)
+#define BIT_OBFF_AUTOACT_EN BIT(25)
+#define BIT_OBFF_AUTOIDLE_EN BIT(24)
+
+#define BIT_SHIFT_WAKE_MAX_PLS 20
+#define BIT_MASK_WAKE_MAX_PLS 0x7
+#define BIT_WAKE_MAX_PLS(x)                                                    \
+	(((x) & BIT_MASK_WAKE_MAX_PLS) << BIT_SHIFT_WAKE_MAX_PLS)
+#define BIT_GET_WAKE_MAX_PLS(x)                                                \
+	(((x) >> BIT_SHIFT_WAKE_MAX_PLS) & BIT_MASK_WAKE_MAX_PLS)
+
+#define BIT_SHIFT_WAKE_MIN_PLS 16
+#define BIT_MASK_WAKE_MIN_PLS 0x7
+#define BIT_WAKE_MIN_PLS(x)                                                    \
+	(((x) & BIT_MASK_WAKE_MIN_PLS) << BIT_SHIFT_WAKE_MIN_PLS)
+#define BIT_GET_WAKE_MIN_PLS(x)                                                \
+	(((x) >> BIT_SHIFT_WAKE_MIN_PLS) & BIT_MASK_WAKE_MIN_PLS)
+
+#define BIT_SHIFT_WAKE_MAX_F2F 12
+#define BIT_MASK_WAKE_MAX_F2F 0x7
+#define BIT_WAKE_MAX_F2F(x)                                                    \
+	(((x) & BIT_MASK_WAKE_MAX_F2F) << BIT_SHIFT_WAKE_MAX_F2F)
+#define BIT_GET_WAKE_MAX_F2F(x)                                                \
+	(((x) >> BIT_SHIFT_WAKE_MAX_F2F) & BIT_MASK_WAKE_MAX_F2F)
+
+#define BIT_SHIFT_WAKE_MIN_F2F 8
+#define BIT_MASK_WAKE_MIN_F2F 0x7
+#define BIT_WAKE_MIN_F2F(x)                                                    \
+	(((x) & BIT_MASK_WAKE_MIN_F2F) << BIT_SHIFT_WAKE_MIN_F2F)
+#define BIT_GET_WAKE_MIN_F2F(x)                                                \
+	(((x) >> BIT_SHIFT_WAKE_MIN_F2F) & BIT_MASK_WAKE_MIN_F2F)
+
+#define BIT_APP_CPU_ACT_V1 BIT(3)
+#define BIT_APP_OBFF_V1 BIT(2)
+#define BIT_APP_IDLE_V1 BIT(1)
+#define BIT_APP_INIT_V1 BIT(0)
+
+/* 2 REG_OBFF_CTRL2_TIMER			(Offset 0x079C) */
+
+#define BIT_SHIFT_RX_HIGH_TIMER_IDX 24
+#define BIT_MASK_RX_HIGH_TIMER_IDX 0x7
+#define BIT_RX_HIGH_TIMER_IDX(x)                                               \
+	(((x) & BIT_MASK_RX_HIGH_TIMER_IDX) << BIT_SHIFT_RX_HIGH_TIMER_IDX)
+#define BIT_GET_RX_HIGH_TIMER_IDX(x)                                           \
+	(((x) >> BIT_SHIFT_RX_HIGH_TIMER_IDX) & BIT_MASK_RX_HIGH_TIMER_IDX)
+
+#define BIT_SHIFT_RX_MED_TIMER_IDX 16
+#define BIT_MASK_RX_MED_TIMER_IDX 0x7
+#define BIT_RX_MED_TIMER_IDX(x)                                                \
+	(((x) & BIT_MASK_RX_MED_TIMER_IDX) << BIT_SHIFT_RX_MED_TIMER_IDX)
+#define BIT_GET_RX_MED_TIMER_IDX(x)                                            \
+	(((x) >> BIT_SHIFT_RX_MED_TIMER_IDX) & BIT_MASK_RX_MED_TIMER_IDX)
+
+#define BIT_SHIFT_RX_LOW_TIMER_IDX 8
+#define BIT_MASK_RX_LOW_TIMER_IDX 0x7
+#define BIT_RX_LOW_TIMER_IDX(x)                                                \
+	(((x) & BIT_MASK_RX_LOW_TIMER_IDX) << BIT_SHIFT_RX_LOW_TIMER_IDX)
+#define BIT_GET_RX_LOW_TIMER_IDX(x)                                            \
+	(((x) >> BIT_SHIFT_RX_LOW_TIMER_IDX) & BIT_MASK_RX_LOW_TIMER_IDX)
+
+#define BIT_SHIFT_OBFF_INT_TIMER_IDX 0
+#define BIT_MASK_OBFF_INT_TIMER_IDX 0x7
+#define BIT_OBFF_INT_TIMER_IDX(x)                                              \
+	(((x) & BIT_MASK_OBFF_INT_TIMER_IDX) << BIT_SHIFT_OBFF_INT_TIMER_IDX)
+#define BIT_GET_OBFF_INT_TIMER_IDX(x)                                          \
+	(((x) >> BIT_SHIFT_OBFF_INT_TIMER_IDX) & BIT_MASK_OBFF_INT_TIMER_IDX)
+
+/* 2 REG_LTR_CTRL_BASIC			(Offset 0x07A0) */
+
+#define BIT_LTR_EN_V1 BIT(31)
+#define BIT_LTR_HW_EN_V1 BIT(30)
+#define BIT_LRT_ACT_CTS_EN BIT(29)
+#define BIT_LTR_ACT_RXPKT_EN BIT(28)
+#define BIT_LTR_ACT_RXDMA_EN BIT(27)
+#define BIT_LTR_IDLE_NO_SNOOP BIT(26)
+#define BIT_SPDUP_MGTPKT BIT(25)
+#define BIT_RX_AGG_EN BIT(24)
+#define BIT_APP_LTR_ACT BIT(23)
+#define BIT_APP_LTR_IDLE BIT(22)
+
+#define BIT_SHIFT_HIGH_RATE_TRIG_SEL 20
+#define BIT_MASK_HIGH_RATE_TRIG_SEL 0x3
+#define BIT_HIGH_RATE_TRIG_SEL(x)                                              \
+	(((x) & BIT_MASK_HIGH_RATE_TRIG_SEL) << BIT_SHIFT_HIGH_RATE_TRIG_SEL)
+#define BIT_GET_HIGH_RATE_TRIG_SEL(x)                                          \
+	(((x) >> BIT_SHIFT_HIGH_RATE_TRIG_SEL) & BIT_MASK_HIGH_RATE_TRIG_SEL)
+
+#define BIT_SHIFT_MED_RATE_TRIG_SEL 18
+#define BIT_MASK_MED_RATE_TRIG_SEL 0x3
+#define BIT_MED_RATE_TRIG_SEL(x)                                               \
+	(((x) & BIT_MASK_MED_RATE_TRIG_SEL) << BIT_SHIFT_MED_RATE_TRIG_SEL)
+#define BIT_GET_MED_RATE_TRIG_SEL(x)                                           \
+	(((x) >> BIT_SHIFT_MED_RATE_TRIG_SEL) & BIT_MASK_MED_RATE_TRIG_SEL)
+
+#define BIT_SHIFT_LOW_RATE_TRIG_SEL 16
+#define BIT_MASK_LOW_RATE_TRIG_SEL 0x3
+#define BIT_LOW_RATE_TRIG_SEL(x)                                               \
+	(((x) & BIT_MASK_LOW_RATE_TRIG_SEL) << BIT_SHIFT_LOW_RATE_TRIG_SEL)
+#define BIT_GET_LOW_RATE_TRIG_SEL(x)                                           \
+	(((x) >> BIT_SHIFT_LOW_RATE_TRIG_SEL) & BIT_MASK_LOW_RATE_TRIG_SEL)
+
+#define BIT_SHIFT_HIGH_RATE_BD_IDX 8
+#define BIT_MASK_HIGH_RATE_BD_IDX 0x7f
+#define BIT_HIGH_RATE_BD_IDX(x)                                                \
+	(((x) & BIT_MASK_HIGH_RATE_BD_IDX) << BIT_SHIFT_HIGH_RATE_BD_IDX)
+#define BIT_GET_HIGH_RATE_BD_IDX(x)                                            \
+	(((x) >> BIT_SHIFT_HIGH_RATE_BD_IDX) & BIT_MASK_HIGH_RATE_BD_IDX)
+
+#define BIT_SHIFT_LOW_RATE_BD_IDX 0
+#define BIT_MASK_LOW_RATE_BD_IDX 0x7f
+#define BIT_LOW_RATE_BD_IDX(x)                                                 \
+	(((x) & BIT_MASK_LOW_RATE_BD_IDX) << BIT_SHIFT_LOW_RATE_BD_IDX)
+#define BIT_GET_LOW_RATE_BD_IDX(x)                                             \
+	(((x) >> BIT_SHIFT_LOW_RATE_BD_IDX) & BIT_MASK_LOW_RATE_BD_IDX)
+
+/* 2 REG_LTR_CTRL2_TIMER_THRESHOLD		(Offset 0x07A4) */
+
+#define BIT_SHIFT_RX_EMPTY_TIMER_IDX 24
+#define BIT_MASK_RX_EMPTY_TIMER_IDX 0x7
+#define BIT_RX_EMPTY_TIMER_IDX(x)                                              \
+	(((x) & BIT_MASK_RX_EMPTY_TIMER_IDX) << BIT_SHIFT_RX_EMPTY_TIMER_IDX)
+#define BIT_GET_RX_EMPTY_TIMER_IDX(x)                                          \
+	(((x) >> BIT_SHIFT_RX_EMPTY_TIMER_IDX) & BIT_MASK_RX_EMPTY_TIMER_IDX)
+
+#define BIT_SHIFT_RX_AFULL_TH_IDX 20
+#define BIT_MASK_RX_AFULL_TH_IDX 0x7
+#define BIT_RX_AFULL_TH_IDX(x)                                                 \
+	(((x) & BIT_MASK_RX_AFULL_TH_IDX) << BIT_SHIFT_RX_AFULL_TH_IDX)
+#define BIT_GET_RX_AFULL_TH_IDX(x)                                             \
+	(((x) >> BIT_SHIFT_RX_AFULL_TH_IDX) & BIT_MASK_RX_AFULL_TH_IDX)
+
+#define BIT_SHIFT_RX_HIGH_TH_IDX 16
+#define BIT_MASK_RX_HIGH_TH_IDX 0x7
+#define BIT_RX_HIGH_TH_IDX(x)                                                  \
+	(((x) & BIT_MASK_RX_HIGH_TH_IDX) << BIT_SHIFT_RX_HIGH_TH_IDX)
+#define BIT_GET_RX_HIGH_TH_IDX(x)                                              \
+	(((x) >> BIT_SHIFT_RX_HIGH_TH_IDX) & BIT_MASK_RX_HIGH_TH_IDX)
+
+#define BIT_SHIFT_RX_MED_TH_IDX 12
+#define BIT_MASK_RX_MED_TH_IDX 0x7
+#define BIT_RX_MED_TH_IDX(x)                                                   \
+	(((x) & BIT_MASK_RX_MED_TH_IDX) << BIT_SHIFT_RX_MED_TH_IDX)
+#define BIT_GET_RX_MED_TH_IDX(x)                                               \
+	(((x) >> BIT_SHIFT_RX_MED_TH_IDX) & BIT_MASK_RX_MED_TH_IDX)
+
+#define BIT_SHIFT_RX_LOW_TH_IDX 8
+#define BIT_MASK_RX_LOW_TH_IDX 0x7
+#define BIT_RX_LOW_TH_IDX(x)                                                   \
+	(((x) & BIT_MASK_RX_LOW_TH_IDX) << BIT_SHIFT_RX_LOW_TH_IDX)
+#define BIT_GET_RX_LOW_TH_IDX(x)                                               \
+	(((x) >> BIT_SHIFT_RX_LOW_TH_IDX) & BIT_MASK_RX_LOW_TH_IDX)
+
+#define BIT_SHIFT_LTR_SPACE_IDX 4
+#define BIT_MASK_LTR_SPACE_IDX 0x3
+#define BIT_LTR_SPACE_IDX(x)                                                   \
+	(((x) & BIT_MASK_LTR_SPACE_IDX) << BIT_SHIFT_LTR_SPACE_IDX)
+#define BIT_GET_LTR_SPACE_IDX(x)                                               \
+	(((x) >> BIT_SHIFT_LTR_SPACE_IDX) & BIT_MASK_LTR_SPACE_IDX)
+
+#define BIT_SHIFT_LTR_IDLE_TIMER_IDX 0
+#define BIT_MASK_LTR_IDLE_TIMER_IDX 0x7
+#define BIT_LTR_IDLE_TIMER_IDX(x)                                              \
+	(((x) & BIT_MASK_LTR_IDLE_TIMER_IDX) << BIT_SHIFT_LTR_IDLE_TIMER_IDX)
+#define BIT_GET_LTR_IDLE_TIMER_IDX(x)                                          \
+	(((x) >> BIT_SHIFT_LTR_IDLE_TIMER_IDX) & BIT_MASK_LTR_IDLE_TIMER_IDX)
+
+/* 2 REG_LTR_IDLE_LATENCY_V1			(Offset 0x07A8) */
+
+#define BIT_SHIFT_LTR_IDLE_L 0
+#define BIT_MASK_LTR_IDLE_L 0xffffffffL
+#define BIT_LTR_IDLE_L(x) (((x) & BIT_MASK_LTR_IDLE_L) << BIT_SHIFT_LTR_IDLE_L)
+#define BIT_GET_LTR_IDLE_L(x)                                                  \
+	(((x) >> BIT_SHIFT_LTR_IDLE_L) & BIT_MASK_LTR_IDLE_L)
+
+/* 2 REG_LTR_ACTIVE_LATENCY_V1		(Offset 0x07AC) */
+
+#define BIT_SHIFT_LTR_ACT_L 0
+#define BIT_MASK_LTR_ACT_L 0xffffffffL
+#define BIT_LTR_ACT_L(x) (((x) & BIT_MASK_LTR_ACT_L) << BIT_SHIFT_LTR_ACT_L)
+#define BIT_GET_LTR_ACT_L(x) (((x) >> BIT_SHIFT_LTR_ACT_L) & BIT_MASK_LTR_ACT_L)
+
+/* 2 REG_ANTENNA_TRAINING_CONTROL_REGISTER	(Offset 0x07B0) */
+
+#define BIT_APPEND_MACID_IN_RESP_EN BIT(50)
+#define BIT_ADDR2_MATCH_EN BIT(49)
+#define BIT_ANTTRN_EN BIT(48)
+
+#define BIT_SHIFT_TRAIN_STA_ADDR 0
+#define BIT_MASK_TRAIN_STA_ADDR 0xffffffffffffL
+#define BIT_TRAIN_STA_ADDR(x)                                                  \
+	(((x) & BIT_MASK_TRAIN_STA_ADDR) << BIT_SHIFT_TRAIN_STA_ADDR)
+#define BIT_GET_TRAIN_STA_ADDR(x)                                              \
+	(((x) >> BIT_SHIFT_TRAIN_STA_ADDR) & BIT_MASK_TRAIN_STA_ADDR)
+
+/* 2 REG_WMAC_PKTCNT_RWD			(Offset 0x07B8) */
+
+#define BIT_SHIFT_PKTCNT_BSSIDMAP 4
+#define BIT_MASK_PKTCNT_BSSIDMAP 0xf
+#define BIT_PKTCNT_BSSIDMAP(x)                                                 \
+	(((x) & BIT_MASK_PKTCNT_BSSIDMAP) << BIT_SHIFT_PKTCNT_BSSIDMAP)
+#define BIT_GET_PKTCNT_BSSIDMAP(x)                                             \
+	(((x) >> BIT_SHIFT_PKTCNT_BSSIDMAP) & BIT_MASK_PKTCNT_BSSIDMAP)
+
+#define BIT_PKTCNT_CNTRST BIT(1)
+#define BIT_PKTCNT_CNTEN BIT(0)
+
+/* 2 REG_WMAC_PKTCNT_CTRL			(Offset 0x07BC) */
+
+#define BIT_WMAC_PKTCNT_TRST BIT(9)
+#define BIT_WMAC_PKTCNT_FEN BIT(8)
+
+#define BIT_SHIFT_WMAC_PKTCNT_CFGAD 0
+#define BIT_MASK_WMAC_PKTCNT_CFGAD 0xff
+#define BIT_WMAC_PKTCNT_CFGAD(x)                                               \
+	(((x) & BIT_MASK_WMAC_PKTCNT_CFGAD) << BIT_SHIFT_WMAC_PKTCNT_CFGAD)
+#define BIT_GET_WMAC_PKTCNT_CFGAD(x)                                           \
+	(((x) >> BIT_SHIFT_WMAC_PKTCNT_CFGAD) & BIT_MASK_WMAC_PKTCNT_CFGAD)
+
+/* 2 REG_IQ_DUMP				(Offset 0x07C0) */
+
+#define BIT_SHIFT_R_WMAC_MATCH_REF_MAC (64 & CPU_OPT_WIDTH)
+#define BIT_MASK_R_WMAC_MATCH_REF_MAC 0xffffffffL
+#define BIT_R_WMAC_MATCH_REF_MAC(x)                                            \
+	(((x) & BIT_MASK_R_WMAC_MATCH_REF_MAC)                                 \
+	 << BIT_SHIFT_R_WMAC_MATCH_REF_MAC)
+#define BIT_GET_R_WMAC_MATCH_REF_MAC(x)                                        \
+	(((x) >> BIT_SHIFT_R_WMAC_MATCH_REF_MAC) &                             \
+	 BIT_MASK_R_WMAC_MATCH_REF_MAC)
+
+#define BIT_SHIFT_R_WMAC_RX_FIL_LEN (64 & CPU_OPT_WIDTH)
+#define BIT_MASK_R_WMAC_RX_FIL_LEN 0xffff
+#define BIT_R_WMAC_RX_FIL_LEN(x)                                               \
+	(((x) & BIT_MASK_R_WMAC_RX_FIL_LEN) << BIT_SHIFT_R_WMAC_RX_FIL_LEN)
+#define BIT_GET_R_WMAC_RX_FIL_LEN(x)                                           \
+	(((x) >> BIT_SHIFT_R_WMAC_RX_FIL_LEN) & BIT_MASK_R_WMAC_RX_FIL_LEN)
+
+#define BIT_SHIFT_R_WMAC_RXFIFO_FULL_TH (56 & CPU_OPT_WIDTH)
+#define BIT_MASK_R_WMAC_RXFIFO_FULL_TH 0xff
+#define BIT_R_WMAC_RXFIFO_FULL_TH(x)                                           \
+	(((x) & BIT_MASK_R_WMAC_RXFIFO_FULL_TH)                                \
+	 << BIT_SHIFT_R_WMAC_RXFIFO_FULL_TH)
+#define BIT_GET_R_WMAC_RXFIFO_FULL_TH(x)                                       \
+	(((x) >> BIT_SHIFT_R_WMAC_RXFIFO_FULL_TH) &                            \
+	 BIT_MASK_R_WMAC_RXFIFO_FULL_TH)
+
+#define BIT_R_WMAC_SRCH_TXRPT_TYPE BIT(51)
+#define BIT_R_WMAC_NDP_RST BIT(50)
+#define BIT_R_WMAC_POWINT_EN BIT(49)
+#define BIT_R_WMAC_SRCH_TXRPT_PERPKT BIT(48)
+#define BIT_R_WMAC_SRCH_TXRPT_MID BIT(47)
+#define BIT_R_WMAC_PFIN_TOEN BIT(46)
+#define BIT_R_WMAC_FIL_SECERR BIT(45)
+#define BIT_R_WMAC_FIL_CTLPKTLEN BIT(44)
+#define BIT_R_WMAC_FIL_FCTYPE BIT(43)
+#define BIT_R_WMAC_FIL_FCPROVER BIT(42)
+#define BIT_R_WMAC_PHYSTS_SNIF BIT(41)
+#define BIT_R_WMAC_PHYSTS_PLCP BIT(40)
+#define BIT_R_MAC_TCR_VBONF_RD BIT(39)
+#define BIT_R_WMAC_TCR_MPAR_NDP BIT(38)
+#define BIT_R_WMAC_NDP_FILTER BIT(37)
+#define BIT_R_WMAC_RXLEN_SEL BIT(36)
+#define BIT_R_WMAC_RXLEN_SEL1 BIT(35)
+#define BIT_R_OFDM_FILTER BIT(34)
+#define BIT_R_WMAC_CHK_OFDM_LEN BIT(33)
+
+#define BIT_SHIFT_R_WMAC_MASK_LA_MAC (32 & CPU_OPT_WIDTH)
+#define BIT_MASK_R_WMAC_MASK_LA_MAC 0xffffffffL
+#define BIT_R_WMAC_MASK_LA_MAC(x)                                              \
+	(((x) & BIT_MASK_R_WMAC_MASK_LA_MAC) << BIT_SHIFT_R_WMAC_MASK_LA_MAC)
+#define BIT_GET_R_WMAC_MASK_LA_MAC(x)                                          \
+	(((x) >> BIT_SHIFT_R_WMAC_MASK_LA_MAC) & BIT_MASK_R_WMAC_MASK_LA_MAC)
+
+#define BIT_R_WMAC_CHK_CCK_LEN BIT(32)
+
+/* 2 REG_IQ_DUMP				(Offset 0x07C0) */
+
+#define BIT_SHIFT_R_OFDM_LEN 26
+#define BIT_MASK_R_OFDM_LEN 0x3f
+#define BIT_R_OFDM_LEN(x) (((x) & BIT_MASK_R_OFDM_LEN) << BIT_SHIFT_R_OFDM_LEN)
+#define BIT_GET_R_OFDM_LEN(x)                                                  \
+	(((x) >> BIT_SHIFT_R_OFDM_LEN) & BIT_MASK_R_OFDM_LEN)
+
+#define BIT_SHIFT_DUMP_OK_ADDR 15
+#define BIT_MASK_DUMP_OK_ADDR 0x1ffff
+#define BIT_DUMP_OK_ADDR(x)                                                    \
+	(((x) & BIT_MASK_DUMP_OK_ADDR) << BIT_SHIFT_DUMP_OK_ADDR)
+#define BIT_GET_DUMP_OK_ADDR(x)                                                \
+	(((x) >> BIT_SHIFT_DUMP_OK_ADDR) & BIT_MASK_DUMP_OK_ADDR)
+
+#define BIT_SHIFT_R_TRIG_TIME_SEL 8
+#define BIT_MASK_R_TRIG_TIME_SEL 0x7f
+#define BIT_R_TRIG_TIME_SEL(x)                                                 \
+	(((x) & BIT_MASK_R_TRIG_TIME_SEL) << BIT_SHIFT_R_TRIG_TIME_SEL)
+#define BIT_GET_R_TRIG_TIME_SEL(x)                                             \
+	(((x) >> BIT_SHIFT_R_TRIG_TIME_SEL) & BIT_MASK_R_TRIG_TIME_SEL)
+
+#define BIT_SHIFT_R_MAC_TRIG_SEL 6
+#define BIT_MASK_R_MAC_TRIG_SEL 0x3
+#define BIT_R_MAC_TRIG_SEL(x)                                                  \
+	(((x) & BIT_MASK_R_MAC_TRIG_SEL) << BIT_SHIFT_R_MAC_TRIG_SEL)
+#define BIT_GET_R_MAC_TRIG_SEL(x)                                              \
+	(((x) >> BIT_SHIFT_R_MAC_TRIG_SEL) & BIT_MASK_R_MAC_TRIG_SEL)
+
+#define BIT_MAC_TRIG_REG BIT(5)
+
+#define BIT_SHIFT_R_LEVEL_PULSE_SEL 3
+#define BIT_MASK_R_LEVEL_PULSE_SEL 0x3
+#define BIT_R_LEVEL_PULSE_SEL(x)                                               \
+	(((x) & BIT_MASK_R_LEVEL_PULSE_SEL) << BIT_SHIFT_R_LEVEL_PULSE_SEL)
+#define BIT_GET_R_LEVEL_PULSE_SEL(x)                                           \
+	(((x) >> BIT_SHIFT_R_LEVEL_PULSE_SEL) & BIT_MASK_R_LEVEL_PULSE_SEL)
+
+#define BIT_EN_LA_MAC BIT(2)
+#define BIT_R_EN_IQDUMP BIT(1)
+#define BIT_R_IQDATA_DUMP BIT(0)
+
+#define BIT_SHIFT_R_CCK_LEN 0
+#define BIT_MASK_R_CCK_LEN 0xffff
+#define BIT_R_CCK_LEN(x) (((x) & BIT_MASK_R_CCK_LEN) << BIT_SHIFT_R_CCK_LEN)
+#define BIT_GET_R_CCK_LEN(x) (((x) >> BIT_SHIFT_R_CCK_LEN) & BIT_MASK_R_CCK_LEN)
+
+/* 2 REG_WMAC_FTM_CTL			(Offset 0x07CC) */
+
+#define BIT_RXFTM_TXACK_SC BIT(6)
+#define BIT_RXFTM_TXACK_BW BIT(5)
+#define BIT_RXFTM_EN BIT(3)
+#define BIT_RXFTMREQ_BYDRV BIT(2)
+#define BIT_RXFTMREQ_EN BIT(1)
+#define BIT_FTM_EN BIT(0)
+
+/* 2 REG_RX_FILTER_FUNCTION			(Offset 0x07DA) */
+
+#define BIT_R_WMAC_MHRDDY_LATCH BIT(14)
+
+/* 2 REG_RX_FILTER_FUNCTION			(Offset 0x07DA) */
+
+#define BIT_R_WMAC_MHRDDY_CLR BIT(13)
+
+/* 2 REG_RX_FILTER_FUNCTION			(Offset 0x07DA) */
+
+#define BIT_R_RXPKTCTL_FSM_BASED_MPDURDY1 BIT(12)
+
+/* 2 REG_RX_FILTER_FUNCTION			(Offset 0x07DA) */
+
+#define BIT_WMAC_DIS_VHT_PLCP_CHK_MU BIT(11)
+
+/* 2 REG_RX_FILTER_FUNCTION			(Offset 0x07DA) */
+
+#define BIT_R_CHK_DELIMIT_LEN BIT(10)
+#define BIT_R_REAPTER_ADDR_MATCH BIT(9)
+#define BIT_R_RXPKTCTL_FSM_BASED_MPDURDY BIT(8)
+#define BIT_R_LATCH_MACHRDY BIT(7)
+#define BIT_R_WMAC_RXFIL_REND BIT(6)
+#define BIT_R_WMAC_MPDURDY_CLR BIT(5)
+#define BIT_R_WMAC_CLRRXSEC BIT(4)
+#define BIT_R_WMAC_RXFIL_RDEL BIT(3)
+#define BIT_R_WMAC_RXFIL_FCSE BIT(2)
+#define BIT_R_WMAC_RXFIL_MESH_DEL BIT(1)
+#define BIT_R_WMAC_RXFIL_MASKM BIT(0)
+
+/* 2 REG_NDP_SIG				(Offset 0x07E0) */
+
+#define BIT_SHIFT_R_WMAC_TXNDP_SIGB 0
+#define BIT_MASK_R_WMAC_TXNDP_SIGB 0x1fffff
+#define BIT_R_WMAC_TXNDP_SIGB(x)                                               \
+	(((x) & BIT_MASK_R_WMAC_TXNDP_SIGB) << BIT_SHIFT_R_WMAC_TXNDP_SIGB)
+#define BIT_GET_R_WMAC_TXNDP_SIGB(x)                                           \
+	(((x) >> BIT_SHIFT_R_WMAC_TXNDP_SIGB) & BIT_MASK_R_WMAC_TXNDP_SIGB)
+
+/* 2 REG_TXCMD_INFO_FOR_RSP_PKT		(Offset 0x07E4) */
+
+#define BIT_SHIFT_R_MAC_DEBUG (32 & CPU_OPT_WIDTH)
+#define BIT_MASK_R_MAC_DEBUG 0xffffffffL
+#define BIT_R_MAC_DEBUG(x)                                                     \
+	(((x) & BIT_MASK_R_MAC_DEBUG) << BIT_SHIFT_R_MAC_DEBUG)
+#define BIT_GET_R_MAC_DEBUG(x)                                                 \
+	(((x) >> BIT_SHIFT_R_MAC_DEBUG) & BIT_MASK_R_MAC_DEBUG)
+
+/* 2 REG_TXCMD_INFO_FOR_RSP_PKT		(Offset 0x07E4) */
+
+#define BIT_SHIFT_R_MAC_DBG_SHIFT 8
+#define BIT_MASK_R_MAC_DBG_SHIFT 0x7
+#define BIT_R_MAC_DBG_SHIFT(x)                                                 \
+	(((x) & BIT_MASK_R_MAC_DBG_SHIFT) << BIT_SHIFT_R_MAC_DBG_SHIFT)
+#define BIT_GET_R_MAC_DBG_SHIFT(x)                                             \
+	(((x) >> BIT_SHIFT_R_MAC_DBG_SHIFT) & BIT_MASK_R_MAC_DBG_SHIFT)
+
+#define BIT_SHIFT_R_MAC_DBG_SEL 0
+#define BIT_MASK_R_MAC_DBG_SEL 0x3
+#define BIT_R_MAC_DBG_SEL(x)                                                   \
+	(((x) & BIT_MASK_R_MAC_DBG_SEL) << BIT_SHIFT_R_MAC_DBG_SEL)
+#define BIT_GET_R_MAC_DBG_SEL(x)                                               \
+	(((x) >> BIT_SHIFT_R_MAC_DBG_SEL) & BIT_MASK_R_MAC_DBG_SEL)
+
+/* 2 REG_SYS_CFG3				(Offset 0x1000) */
+
+#define BIT_PWC_MA33V BIT(15)
+
+/* 2 REG_SYS_CFG3				(Offset 0x1000) */
+
+#define BIT_PWC_MA12V BIT(14)
+#define BIT_PWC_MD12V BIT(13)
+#define BIT_PWC_PD12V BIT(12)
+#define BIT_PWC_UD12V BIT(11)
+#define BIT_ISO_MA2MD BIT(1)
+
+/* 2 REG_SYS_CFG5				(Offset 0x1070) */
+
+#define BIT_LPS_STATUS BIT(3)
+#define BIT_HCI_TXDMA_BUSY BIT(2)
+#define BIT_HCI_TXDMA_ALLOW BIT(1)
+#define BIT_FW_CTRL_HCI_TXDMA_EN BIT(0)
+
+/* 2 REG_CPU_DMEM_CON			(Offset 0x1080) */
+
+#define BIT_WDT_OPT_IOWRAPPER BIT(19)
+
+/* 2 REG_CPU_DMEM_CON			(Offset 0x1080) */
+
+#define BIT_ANA_PORT_IDLE BIT(18)
+#define BIT_MAC_PORT_IDLE BIT(17)
+#define BIT_WL_PLATFORM_RST BIT(16)
+#define BIT_WL_SECURITY_CLK BIT(15)
+
+/* 2 REG_CPU_DMEM_CON			(Offset 0x1080) */
+
+#define BIT_SHIFT_CPU_DMEM_CON 0
+#define BIT_MASK_CPU_DMEM_CON 0xff
+#define BIT_CPU_DMEM_CON(x)                                                    \
+	(((x) & BIT_MASK_CPU_DMEM_CON) << BIT_SHIFT_CPU_DMEM_CON)
+#define BIT_GET_CPU_DMEM_CON(x)                                                \
+	(((x) >> BIT_SHIFT_CPU_DMEM_CON) & BIT_MASK_CPU_DMEM_CON)
+
+/* 2 REG_BOOT_REASON				(Offset 0x1088) */
+
+#define BIT_SHIFT_BOOT_REASON 0
+#define BIT_MASK_BOOT_REASON 0x7
+#define BIT_BOOT_REASON(x)                                                     \
+	(((x) & BIT_MASK_BOOT_REASON) << BIT_SHIFT_BOOT_REASON)
+#define BIT_GET_BOOT_REASON(x)                                                 \
+	(((x) >> BIT_SHIFT_BOOT_REASON) & BIT_MASK_BOOT_REASON)
+
+/* 2 REG_NFCPAD_CTRL				(Offset 0x10A8) */
+
+#define BIT_PAD_SHUTDW BIT(18)
+#define BIT_SYSON_NFC_PAD BIT(17)
+#define BIT_NFC_INT_PAD_CTRL BIT(16)
+#define BIT_NFC_RFDIS_PAD_CTRL BIT(15)
+#define BIT_NFC_CLK_PAD_CTRL BIT(14)
+#define BIT_NFC_DATA_PAD_CTRL BIT(13)
+#define BIT_NFC_PAD_PULL_CTRL BIT(12)
+
+#define BIT_SHIFT_NFCPAD_IO_SEL 8
+#define BIT_MASK_NFCPAD_IO_SEL 0xf
+#define BIT_NFCPAD_IO_SEL(x)                                                   \
+	(((x) & BIT_MASK_NFCPAD_IO_SEL) << BIT_SHIFT_NFCPAD_IO_SEL)
+#define BIT_GET_NFCPAD_IO_SEL(x)                                               \
+	(((x) >> BIT_SHIFT_NFCPAD_IO_SEL) & BIT_MASK_NFCPAD_IO_SEL)
+
+#define BIT_SHIFT_NFCPAD_OUT 4
+#define BIT_MASK_NFCPAD_OUT 0xf
+#define BIT_NFCPAD_OUT(x) (((x) & BIT_MASK_NFCPAD_OUT) << BIT_SHIFT_NFCPAD_OUT)
+#define BIT_GET_NFCPAD_OUT(x)                                                  \
+	(((x) >> BIT_SHIFT_NFCPAD_OUT) & BIT_MASK_NFCPAD_OUT)
+
+#define BIT_SHIFT_NFCPAD_IN 0
+#define BIT_MASK_NFCPAD_IN 0xf
+#define BIT_NFCPAD_IN(x) (((x) & BIT_MASK_NFCPAD_IN) << BIT_SHIFT_NFCPAD_IN)
+#define BIT_GET_NFCPAD_IN(x) (((x) >> BIT_SHIFT_NFCPAD_IN) & BIT_MASK_NFCPAD_IN)
+
+/* 2 REG_HIMR2				(Offset 0x10B0) */
+
+#define BIT_BCNDMAINT_P4_MSK BIT(31)
+#define BIT_BCNDMAINT_P3_MSK BIT(30)
+#define BIT_BCNDMAINT_P2_MSK BIT(29)
+#define BIT_BCNDMAINT_P1_MSK BIT(28)
+#define BIT_ATIMEND7_MSK BIT(22)
+#define BIT_ATIMEND6_MSK BIT(21)
+#define BIT_ATIMEND5_MSK BIT(20)
+#define BIT_ATIMEND4_MSK BIT(19)
+#define BIT_ATIMEND3_MSK BIT(18)
+#define BIT_ATIMEND2_MSK BIT(17)
+#define BIT_ATIMEND1_MSK BIT(16)
+#define BIT_TXBCN7OK_MSK BIT(14)
+#define BIT_TXBCN6OK_MSK BIT(13)
+#define BIT_TXBCN5OK_MSK BIT(12)
+#define BIT_TXBCN4OK_MSK BIT(11)
+#define BIT_TXBCN3OK_MSK BIT(10)
+#define BIT_TXBCN2OK_MSK BIT(9)
+#define BIT_TXBCN1OK_MSK_V1 BIT(8)
+#define BIT_TXBCN7ERR_MSK BIT(6)
+#define BIT_TXBCN6ERR_MSK BIT(5)
+#define BIT_TXBCN5ERR_MSK BIT(4)
+#define BIT_TXBCN4ERR_MSK BIT(3)
+#define BIT_TXBCN3ERR_MSK BIT(2)
+#define BIT_TXBCN2ERR_MSK BIT(1)
+#define BIT_TXBCN1ERR_MSK_V1 BIT(0)
+
+/* 2 REG_HISR2				(Offset 0x10B4) */
+
+#define BIT_BCNDMAINT_P4 BIT(31)
+#define BIT_BCNDMAINT_P3 BIT(30)
+#define BIT_BCNDMAINT_P2 BIT(29)
+#define BIT_BCNDMAINT_P1 BIT(28)
+#define BIT_ATIMEND7 BIT(22)
+#define BIT_ATIMEND6 BIT(21)
+#define BIT_ATIMEND5 BIT(20)
+#define BIT_ATIMEND4 BIT(19)
+#define BIT_ATIMEND3 BIT(18)
+#define BIT_ATIMEND2 BIT(17)
+#define BIT_ATIMEND1 BIT(16)
+#define BIT_TXBCN7OK BIT(14)
+#define BIT_TXBCN6OK BIT(13)
+#define BIT_TXBCN5OK BIT(12)
+#define BIT_TXBCN4OK BIT(11)
+#define BIT_TXBCN3OK BIT(10)
+#define BIT_TXBCN2OK BIT(9)
+#define BIT_TXBCN1OK BIT(8)
+#define BIT_TXBCN7ERR BIT(6)
+#define BIT_TXBCN6ERR BIT(5)
+#define BIT_TXBCN5ERR BIT(4)
+#define BIT_TXBCN4ERR BIT(3)
+#define BIT_TXBCN3ERR BIT(2)
+#define BIT_TXBCN2ERR BIT(1)
+#define BIT_TXBCN1ERR BIT(0)
+
+/* 2 REG_HIMR3				(Offset 0x10B8) */
+
+#define BIT_WDT_PLATFORM_INT_MSK BIT(18)
+#define BIT_WDT_CPU_INT_MSK BIT(17)
+
+/* 2 REG_HIMR3				(Offset 0x10B8) */
+
+#define BIT_SETH2CDOK_MASK BIT(16)
+#define BIT_H2C_CMD_FULL_MASK BIT(15)
+#define BIT_PWR_INT_127_MASK BIT(14)
+#define BIT_TXSHORTCUT_TXDESUPDATEOK_MASK BIT(13)
+#define BIT_TXSHORTCUT_BKUPDATEOK_MASK BIT(12)
+#define BIT_TXSHORTCUT_BEUPDATEOK_MASK BIT(11)
+#define BIT_TXSHORTCUT_VIUPDATEOK_MAS BIT(10)
+#define BIT_TXSHORTCUT_VOUPDATEOK_MASK BIT(9)
+#define BIT_PWR_INT_127_MASK_V1 BIT(8)
+#define BIT_PWR_INT_126TO96_MASK BIT(7)
+#define BIT_PWR_INT_95TO64_MASK BIT(6)
+#define BIT_PWR_INT_63TO32_MASK BIT(5)
+#define BIT_PWR_INT_31TO0_MASK BIT(4)
+#define BIT_DDMA0_LP_INT_MSK BIT(1)
+#define BIT_DDMA0_HP_INT_MSK BIT(0)
+
+/* 2 REG_HISR3				(Offset 0x10BC) */
+
+#define BIT_WDT_PLATFORM_INT BIT(18)
+#define BIT_WDT_CPU_INT BIT(17)
+
+/* 2 REG_HISR3				(Offset 0x10BC) */
+
+#define BIT_SETH2CDOK BIT(16)
+#define BIT_H2C_CMD_FULL BIT(15)
+#define BIT_PWR_INT_127 BIT(14)
+#define BIT_TXSHORTCUT_TXDESUPDATEOK BIT(13)
+#define BIT_TXSHORTCUT_BKUPDATEOK BIT(12)
+#define BIT_TXSHORTCUT_BEUPDATEOK BIT(11)
+#define BIT_TXSHORTCUT_VIUPDATEOK BIT(10)
+#define BIT_TXSHORTCUT_VOUPDATEOK BIT(9)
+#define BIT_PWR_INT_127_V1 BIT(8)
+#define BIT_PWR_INT_126TO96 BIT(7)
+#define BIT_PWR_INT_95TO64 BIT(6)
+#define BIT_PWR_INT_63TO32 BIT(5)
+#define BIT_PWR_INT_31TO0 BIT(4)
+#define BIT_DDMA0_LP_INT BIT(1)
+#define BIT_DDMA0_HP_INT BIT(0)
+
+/* 2 REG_SW_MDIO				(Offset 0x10C0) */
+
+#define BIT_DIS_TIMEOUT_IO BIT(24)
+
+/* 2 REG_SW_FLUSH				(Offset 0x10C4) */
+
+#define BIT_FLUSH_HOLDN_EN BIT(25)
+#define BIT_FLUSH_WR_EN BIT(24)
+#define BIT_SW_FLASH_CONTROL BIT(23)
+#define BIT_SW_FLASH_WEN_E BIT(19)
+#define BIT_SW_FLASH_HOLDN_E BIT(18)
+#define BIT_SW_FLASH_SO_E BIT(17)
+#define BIT_SW_FLASH_SI_E BIT(16)
+#define BIT_SW_FLASH_SK_O BIT(13)
+#define BIT_SW_FLASH_CEN_O BIT(12)
+#define BIT_SW_FLASH_WEN_O BIT(11)
+#define BIT_SW_FLASH_HOLDN_O BIT(10)
+#define BIT_SW_FLASH_SO_O BIT(9)
+#define BIT_SW_FLASH_SI_O BIT(8)
+#define BIT_SW_FLASH_WEN_I BIT(3)
+#define BIT_SW_FLASH_HOLDN_I BIT(2)
+#define BIT_SW_FLASH_SO_I BIT(1)
+#define BIT_SW_FLASH_SI_I BIT(0)
+
+/* 2 REG_H2C_PKT_READADDR			(Offset 0x10D0) */
+
+#define BIT_SHIFT_H2C_PKT_READADDR 0
+#define BIT_MASK_H2C_PKT_READADDR 0x3ffff
+#define BIT_H2C_PKT_READADDR(x)                                                \
+	(((x) & BIT_MASK_H2C_PKT_READADDR) << BIT_SHIFT_H2C_PKT_READADDR)
+#define BIT_GET_H2C_PKT_READADDR(x)                                            \
+	(((x) >> BIT_SHIFT_H2C_PKT_READADDR) & BIT_MASK_H2C_PKT_READADDR)
+
+/* 2 REG_H2C_PKT_WRITEADDR			(Offset 0x10D4) */
+
+#define BIT_SHIFT_H2C_PKT_WRITEADDR 0
+#define BIT_MASK_H2C_PKT_WRITEADDR 0x3ffff
+#define BIT_H2C_PKT_WRITEADDR(x)                                               \
+	(((x) & BIT_MASK_H2C_PKT_WRITEADDR) << BIT_SHIFT_H2C_PKT_WRITEADDR)
+#define BIT_GET_H2C_PKT_WRITEADDR(x)                                           \
+	(((x) >> BIT_SHIFT_H2C_PKT_WRITEADDR) & BIT_MASK_H2C_PKT_WRITEADDR)
+
+/* 2 REG_MEM_PWR_CRTL			(Offset 0x10D8) */
+
+#define BIT_MEM_BB_SD BIT(17)
+#define BIT_MEM_BB_DS BIT(16)
+#define BIT_MEM_BT_DS BIT(10)
+#define BIT_MEM_SDIO_LS BIT(9)
+#define BIT_MEM_SDIO_DS BIT(8)
+#define BIT_MEM_USB_LS BIT(7)
+#define BIT_MEM_USB_DS BIT(6)
+#define BIT_MEM_PCI_LS BIT(5)
+#define BIT_MEM_PCI_DS BIT(4)
+#define BIT_MEM_WLMAC_LS BIT(3)
+#define BIT_MEM_WLMAC_DS BIT(2)
+#define BIT_MEM_WLMCU_LS BIT(1)
+
+/* 2 REG_MEM_PWR_CRTL			(Offset 0x10D8) */
+
+#define BIT_MEM_WLMCU_DS BIT(0)
+
+/* 2 REG_FW_DBG0				(Offset 0x10E0) */
+
+#define BIT_SHIFT_FW_DBG0 0
+#define BIT_MASK_FW_DBG0 0xffffffffL
+#define BIT_FW_DBG0(x) (((x) & BIT_MASK_FW_DBG0) << BIT_SHIFT_FW_DBG0)
+#define BIT_GET_FW_DBG0(x) (((x) >> BIT_SHIFT_FW_DBG0) & BIT_MASK_FW_DBG0)
+
+/* 2 REG_FW_DBG1				(Offset 0x10E4) */
+
+#define BIT_SHIFT_FW_DBG1 0
+#define BIT_MASK_FW_DBG1 0xffffffffL
+#define BIT_FW_DBG1(x) (((x) & BIT_MASK_FW_DBG1) << BIT_SHIFT_FW_DBG1)
+#define BIT_GET_FW_DBG1(x) (((x) >> BIT_SHIFT_FW_DBG1) & BIT_MASK_FW_DBG1)
+
+/* 2 REG_FW_DBG2				(Offset 0x10E8) */
+
+#define BIT_SHIFT_FW_DBG2 0
+#define BIT_MASK_FW_DBG2 0xffffffffL
+#define BIT_FW_DBG2(x) (((x) & BIT_MASK_FW_DBG2) << BIT_SHIFT_FW_DBG2)
+#define BIT_GET_FW_DBG2(x) (((x) >> BIT_SHIFT_FW_DBG2) & BIT_MASK_FW_DBG2)
+
+/* 2 REG_FW_DBG3				(Offset 0x10EC) */
+
+#define BIT_SHIFT_FW_DBG3 0
+#define BIT_MASK_FW_DBG3 0xffffffffL
+#define BIT_FW_DBG3(x) (((x) & BIT_MASK_FW_DBG3) << BIT_SHIFT_FW_DBG3)
+#define BIT_GET_FW_DBG3(x) (((x) >> BIT_SHIFT_FW_DBG3) & BIT_MASK_FW_DBG3)
+
+/* 2 REG_FW_DBG4				(Offset 0x10F0) */
+
+#define BIT_SHIFT_FW_DBG4 0
+#define BIT_MASK_FW_DBG4 0xffffffffL
+#define BIT_FW_DBG4(x) (((x) & BIT_MASK_FW_DBG4) << BIT_SHIFT_FW_DBG4)
+#define BIT_GET_FW_DBG4(x) (((x) >> BIT_SHIFT_FW_DBG4) & BIT_MASK_FW_DBG4)
+
+/* 2 REG_FW_DBG5				(Offset 0x10F4) */
+
+#define BIT_SHIFT_FW_DBG5 0
+#define BIT_MASK_FW_DBG5 0xffffffffL
+#define BIT_FW_DBG5(x) (((x) & BIT_MASK_FW_DBG5) << BIT_SHIFT_FW_DBG5)
+#define BIT_GET_FW_DBG5(x) (((x) >> BIT_SHIFT_FW_DBG5) & BIT_MASK_FW_DBG5)
+
+/* 2 REG_FW_DBG6				(Offset 0x10F8) */
+
+#define BIT_SHIFT_FW_DBG6 0
+#define BIT_MASK_FW_DBG6 0xffffffffL
+#define BIT_FW_DBG6(x) (((x) & BIT_MASK_FW_DBG6) << BIT_SHIFT_FW_DBG6)
+#define BIT_GET_FW_DBG6(x) (((x) >> BIT_SHIFT_FW_DBG6) & BIT_MASK_FW_DBG6)
+
+/* 2 REG_FW_DBG7				(Offset 0x10FC) */
+
+#define BIT_SHIFT_FW_DBG7 0
+#define BIT_MASK_FW_DBG7 0xffffffffL
+#define BIT_FW_DBG7(x) (((x) & BIT_MASK_FW_DBG7) << BIT_SHIFT_FW_DBG7)
+#define BIT_GET_FW_DBG7(x) (((x) >> BIT_SHIFT_FW_DBG7) & BIT_MASK_FW_DBG7)
+
+/* 2 REG_CR_EXT				(Offset 0x1100) */
+
+#define BIT_SHIFT_PHY_REQ_DELAY 24
+#define BIT_MASK_PHY_REQ_DELAY 0xf
+#define BIT_PHY_REQ_DELAY(x)                                                   \
+	(((x) & BIT_MASK_PHY_REQ_DELAY) << BIT_SHIFT_PHY_REQ_DELAY)
+#define BIT_GET_PHY_REQ_DELAY(x)                                               \
+	(((x) >> BIT_SHIFT_PHY_REQ_DELAY) & BIT_MASK_PHY_REQ_DELAY)
+
+#define BIT_SPD_DOWN BIT(16)
+
+#define BIT_SHIFT_NETYPE4 4
+#define BIT_MASK_NETYPE4 0x3
+#define BIT_NETYPE4(x) (((x) & BIT_MASK_NETYPE4) << BIT_SHIFT_NETYPE4)
+#define BIT_GET_NETYPE4(x) (((x) >> BIT_SHIFT_NETYPE4) & BIT_MASK_NETYPE4)
+
+#define BIT_SHIFT_NETYPE3 2
+#define BIT_MASK_NETYPE3 0x3
+#define BIT_NETYPE3(x) (((x) & BIT_MASK_NETYPE3) << BIT_SHIFT_NETYPE3)
+#define BIT_GET_NETYPE3(x) (((x) >> BIT_SHIFT_NETYPE3) & BIT_MASK_NETYPE3)
+
+#define BIT_SHIFT_NETYPE2 0
+#define BIT_MASK_NETYPE2 0x3
+#define BIT_NETYPE2(x) (((x) & BIT_MASK_NETYPE2) << BIT_SHIFT_NETYPE2)
+#define BIT_GET_NETYPE2(x) (((x) >> BIT_SHIFT_NETYPE2) & BIT_MASK_NETYPE2)
+
+/* 2 REG_FWFF				(Offset 0x1114) */
+
+#define BIT_SHIFT_PKTNUM_TH_V1 24
+#define BIT_MASK_PKTNUM_TH_V1 0xff
+#define BIT_PKTNUM_TH_V1(x)                                                    \
+	(((x) & BIT_MASK_PKTNUM_TH_V1) << BIT_SHIFT_PKTNUM_TH_V1)
+#define BIT_GET_PKTNUM_TH_V1(x)                                                \
+	(((x) >> BIT_SHIFT_PKTNUM_TH_V1) & BIT_MASK_PKTNUM_TH_V1)
+
+/* 2 REG_FWFF				(Offset 0x1114) */
+
+#define BIT_SHIFT_TIMER_TH 16
+#define BIT_MASK_TIMER_TH 0xff
+#define BIT_TIMER_TH(x) (((x) & BIT_MASK_TIMER_TH) << BIT_SHIFT_TIMER_TH)
+#define BIT_GET_TIMER_TH(x) (((x) >> BIT_SHIFT_TIMER_TH) & BIT_MASK_TIMER_TH)
+
+/* 2 REG_FWFF				(Offset 0x1114) */
+
+#define BIT_SHIFT_RXPKT1ENADDR 0
+#define BIT_MASK_RXPKT1ENADDR 0xffff
+#define BIT_RXPKT1ENADDR(x)                                                    \
+	(((x) & BIT_MASK_RXPKT1ENADDR) << BIT_SHIFT_RXPKT1ENADDR)
+#define BIT_GET_RXPKT1ENADDR(x)                                                \
+	(((x) >> BIT_SHIFT_RXPKT1ENADDR) & BIT_MASK_RXPKT1ENADDR)
+
+/* 2 REG_FE2IMR				(Offset 0x1120) */
+
+#define BIT__FE4ISR__IND_MSK BIT(29)
+
+/* 2 REG_FE2IMR				(Offset 0x1120) */
+
+#define BIT_FS_TXSC_DESC_DONE_INT_EN BIT(28)
+#define BIT_FS_TXSC_BKDONE_INT_EN BIT(27)
+#define BIT_FS_TXSC_BEDONE_INT_EN BIT(26)
+#define BIT_FS_TXSC_VIDONE_INT_EN BIT(25)
+#define BIT_FS_TXSC_VODONE_INT_EN BIT(24)
+
+/* 2 REG_FE2IMR				(Offset 0x1120) */
+
+#define BIT_FS_ATIM_MB7_INT_EN BIT(23)
+#define BIT_FS_ATIM_MB6_INT_EN BIT(22)
+#define BIT_FS_ATIM_MB5_INT_EN BIT(21)
+#define BIT_FS_ATIM_MB4_INT_EN BIT(20)
+#define BIT_FS_ATIM_MB3_INT_EN BIT(19)
+#define BIT_FS_ATIM_MB2_INT_EN BIT(18)
+#define BIT_FS_ATIM_MB1_INT_EN BIT(17)
+#define BIT_FS_ATIM_MB0_INT_EN BIT(16)
+#define BIT_FS_TBTT4INT_EN BIT(11)
+#define BIT_FS_TBTT3INT_EN BIT(10)
+#define BIT_FS_TBTT2INT_EN BIT(9)
+#define BIT_FS_TBTT1INT_EN BIT(8)
+#define BIT_FS_TBTT0_MB7INT_EN BIT(7)
+#define BIT_FS_TBTT0_MB6INT_EN BIT(6)
+#define BIT_FS_TBTT0_MB5INT_EN BIT(5)
+#define BIT_FS_TBTT0_MB4INT_EN BIT(4)
+#define BIT_FS_TBTT0_MB3INT_EN BIT(3)
+#define BIT_FS_TBTT0_MB2INT_EN BIT(2)
+#define BIT_FS_TBTT0_MB1INT_EN BIT(1)
+#define BIT_FS_TBTT0_INT_EN BIT(0)
+
+/* 2 REG_FE2ISR				(Offset 0x1124) */
+
+#define BIT__FE4ISR__IND_INT BIT(29)
+
+/* 2 REG_FE2ISR				(Offset 0x1124) */
+
+#define BIT_FS_TXSC_DESC_DONE_INT BIT(28)
+#define BIT_FS_TXSC_BKDONE_INT BIT(27)
+#define BIT_FS_TXSC_BEDONE_INT BIT(26)
+#define BIT_FS_TXSC_VIDONE_INT BIT(25)
+#define BIT_FS_TXSC_VODONE_INT BIT(24)
+
+/* 2 REG_FE2ISR				(Offset 0x1124) */
+
+#define BIT_FS_ATIM_MB7_INT BIT(23)
+#define BIT_FS_ATIM_MB6_INT BIT(22)
+#define BIT_FS_ATIM_MB5_INT BIT(21)
+#define BIT_FS_ATIM_MB4_INT BIT(20)
+#define BIT_FS_ATIM_MB3_INT BIT(19)
+#define BIT_FS_ATIM_MB2_INT BIT(18)
+#define BIT_FS_ATIM_MB1_INT BIT(17)
+#define BIT_FS_ATIM_MB0_INT BIT(16)
+#define BIT_FS_TBTT4INT BIT(11)
+#define BIT_FS_TBTT3INT BIT(10)
+#define BIT_FS_TBTT2INT BIT(9)
+#define BIT_FS_TBTT1INT BIT(8)
+#define BIT_FS_TBTT0_MB7INT BIT(7)
+#define BIT_FS_TBTT0_MB6INT BIT(6)
+#define BIT_FS_TBTT0_MB5INT BIT(5)
+#define BIT_FS_TBTT0_MB4INT BIT(4)
+#define BIT_FS_TBTT0_MB3INT BIT(3)
+#define BIT_FS_TBTT0_MB2INT BIT(2)
+#define BIT_FS_TBTT0_MB1INT BIT(1)
+#define BIT_FS_TBTT0_INT BIT(0)
+
+/* 2 REG_FE3IMR				(Offset 0x1128) */
+
+#define BIT_FS_CLI3_MTI_BCNIVLEAR_INT__EN BIT(31)
+
+/* 2 REG_FE3IMR				(Offset 0x1128) */
+
+#define BIT_FS_CLI2_MTI_BCNIVLEAR_INT__EN BIT(30)
+
+/* 2 REG_FE3IMR				(Offset 0x1128) */
+
+#define BIT_FS_CLI1_MTI_BCNIVLEAR_INT__EN BIT(29)
+
+/* 2 REG_FE3IMR				(Offset 0x1128) */
+
+#define BIT_FS_CLI0_MTI_BCNIVLEAR_INT__EN BIT(28)
+
+/* 2 REG_FE3IMR				(Offset 0x1128) */
+
+#define BIT_FS_BCNDMA4_INT_EN BIT(27)
+#define BIT_FS_BCNDMA3_INT_EN BIT(26)
+#define BIT_FS_BCNDMA2_INT_EN BIT(25)
+#define BIT_FS_BCNDMA1_INT_EN BIT(24)
+#define BIT_FS_BCNDMA0_MB7_INT_EN BIT(23)
+#define BIT_FS_BCNDMA0_MB6_INT_EN BIT(22)
+#define BIT_FS_BCNDMA0_MB5_INT_EN BIT(21)
+#define BIT_FS_BCNDMA0_MB4_INT_EN BIT(20)
+#define BIT_FS_BCNDMA0_MB3_INT_EN BIT(19)
+#define BIT_FS_BCNDMA0_MB2_INT_EN BIT(18)
+#define BIT_FS_BCNDMA0_MB1_INT_EN BIT(17)
+#define BIT_FS_BCNDMA0_INT_EN BIT(16)
+#define BIT_FS_MTI_BCNIVLEAR_INT__EN BIT(15)
+#define BIT_FS_BCNERLY4_INT_EN BIT(11)
+#define BIT_FS_BCNERLY3_INT_EN BIT(10)
+#define BIT_FS_BCNERLY2_INT_EN BIT(9)
+#define BIT_FS_BCNERLY1_INT_EN BIT(8)
+#define BIT_FS_BCNERLY0_MB7INT_EN BIT(7)
+#define BIT_FS_BCNERLY0_MB6INT_EN BIT(6)
+#define BIT_FS_BCNERLY0_MB5INT_EN BIT(5)
+#define BIT_FS_BCNERLY0_MB4INT_EN BIT(4)
+#define BIT_FS_BCNERLY0_MB3INT_EN BIT(3)
+#define BIT_FS_BCNERLY0_MB2INT_EN BIT(2)
+#define BIT_FS_BCNERLY0_MB1INT_EN BIT(1)
+#define BIT_FS_BCNERLY0_INT_EN BIT(0)
+
+/* 2 REG_FE3ISR				(Offset 0x112C) */
+
+#define BIT_FS_CLI3_MTI_BCNIVLEAR_INT BIT(31)
+
+/* 2 REG_FE3ISR				(Offset 0x112C) */
+
+#define BIT_FS_CLI2_MTI_BCNIVLEAR_INT BIT(30)
+
+/* 2 REG_FE3ISR				(Offset 0x112C) */
+
+#define BIT_FS_CLI1_MTI_BCNIVLEAR_INT BIT(29)
+
+/* 2 REG_FE3ISR				(Offset 0x112C) */
+
+#define BIT_FS_CLI0_MTI_BCNIVLEAR_INT BIT(28)
+
+/* 2 REG_FE3ISR				(Offset 0x112C) */
+
+#define BIT_FS_BCNDMA4_INT BIT(27)
+#define BIT_FS_BCNDMA3_INT BIT(26)
+#define BIT_FS_BCNDMA2_INT BIT(25)
+#define BIT_FS_BCNDMA1_INT BIT(24)
+#define BIT_FS_BCNDMA0_MB7_INT BIT(23)
+#define BIT_FS_BCNDMA0_MB6_INT BIT(22)
+#define BIT_FS_BCNDMA0_MB5_INT BIT(21)
+#define BIT_FS_BCNDMA0_MB4_INT BIT(20)
+#define BIT_FS_BCNDMA0_MB3_INT BIT(19)
+#define BIT_FS_BCNDMA0_MB2_INT BIT(18)
+#define BIT_FS_BCNDMA0_MB1_INT BIT(17)
+#define BIT_FS_BCNDMA0_INT BIT(16)
+#define BIT_FS_MTI_BCNIVLEAR_INT BIT(15)
+#define BIT_FS_BCNERLY4_INT BIT(11)
+#define BIT_FS_BCNERLY3_INT BIT(10)
+#define BIT_FS_BCNERLY2_INT BIT(9)
+#define BIT_FS_BCNERLY1_INT BIT(8)
+#define BIT_FS_BCNERLY0_MB7INT BIT(7)
+#define BIT_FS_BCNERLY0_MB6INT BIT(6)
+#define BIT_FS_BCNERLY0_MB5INT BIT(5)
+#define BIT_FS_BCNERLY0_MB4INT BIT(4)
+#define BIT_FS_BCNERLY0_MB3INT BIT(3)
+#define BIT_FS_BCNERLY0_MB2INT BIT(2)
+#define BIT_FS_BCNERLY0_MB1INT BIT(1)
+#define BIT_FS_BCNERLY0_INT BIT(0)
+
+/* 2 REG_FE4IMR				(Offset 0x1130) */
+
+#define BIT_FS_CLI3_TXPKTIN_INT_EN BIT(19)
+
+/* 2 REG_FE4IMR				(Offset 0x1130) */
+
+#define BIT_FS_CLI2_TXPKTIN_INT_EN BIT(18)
+
+/* 2 REG_FE4IMR				(Offset 0x1130) */
+
+#define BIT_FS_CLI1_TXPKTIN_INT_EN BIT(17)
+
+/* 2 REG_FE4IMR				(Offset 0x1130) */
+
+#define BIT_FS_CLI0_TXPKTIN_INT_EN BIT(16)
+
+/* 2 REG_FE4IMR				(Offset 0x1130) */
+
+#define BIT_FS_CLI3_RX_UMD0_INT_EN BIT(15)
+
+/* 2 REG_FE4IMR				(Offset 0x1130) */
+
+#define BIT_FS_CLI3_RX_UMD1_INT_EN BIT(14)
+
+/* 2 REG_FE4IMR				(Offset 0x1130) */
+
+#define BIT_FS_CLI3_RX_BMD0_INT_EN BIT(13)
+
+/* 2 REG_FE4IMR				(Offset 0x1130) */
+
+#define BIT_FS_CLI3_RX_BMD1_INT_EN BIT(12)
+
+/* 2 REG_FE4IMR				(Offset 0x1130) */
+
+#define BIT_FS_CLI2_RX_UMD0_INT_EN BIT(11)
+
+/* 2 REG_FE4IMR				(Offset 0x1130) */
+
+#define BIT_FS_CLI2_RX_UMD1_INT_EN BIT(10)
+
+/* 2 REG_FE4IMR				(Offset 0x1130) */
+
+#define BIT_FS_CLI2_RX_BMD0_INT_EN BIT(9)
+
+/* 2 REG_FE4IMR				(Offset 0x1130) */
+
+#define BIT_FS_CLI2_RX_BMD1_INT_EN BIT(8)
+
+/* 2 REG_FE4IMR				(Offset 0x1130) */
+
+#define BIT_FS_CLI1_RX_UMD0_INT_EN BIT(7)
+
+/* 2 REG_FE4IMR				(Offset 0x1130) */
+
+#define BIT_FS_CLI1_RX_UMD1_INT_EN BIT(6)
+
+/* 2 REG_FE4IMR				(Offset 0x1130) */
+
+#define BIT_FS_CLI1_RX_BMD0_INT_EN BIT(5)
+
+/* 2 REG_FE4IMR				(Offset 0x1130) */
+
+#define BIT_FS_CLI1_RX_BMD1_INT_EN BIT(4)
+
+/* 2 REG_FE4IMR				(Offset 0x1130) */
+
+#define BIT_FS_CLI0_RX_UMD0_INT_EN BIT(3)
+
+/* 2 REG_FE4IMR				(Offset 0x1130) */
+
+#define BIT_FS_CLI0_RX_UMD1_INT_EN BIT(2)
+
+/* 2 REG_FE4IMR				(Offset 0x1130) */
+
+#define BIT_FS_CLI0_RX_BMD0_INT_EN BIT(1)
+
+/* 2 REG_FE4IMR				(Offset 0x1130) */
+
+#define BIT_FS_CLI0_RX_BMD1_INT_EN BIT(0)
+
+/* 2 REG_FE4ISR				(Offset 0x1134) */
+
+#define BIT_FS_CLI3_TXPKTIN_INT BIT(19)
+
+/* 2 REG_FE4ISR				(Offset 0x1134) */
+
+#define BIT_FS_CLI2_TXPKTIN_INT BIT(18)
+
+/* 2 REG_FE4ISR				(Offset 0x1134) */
+
+#define BIT_FS_CLI1_TXPKTIN_INT BIT(17)
+
+/* 2 REG_FE4ISR				(Offset 0x1134) */
+
+#define BIT_FS_CLI0_TXPKTIN_INT BIT(16)
+
+/* 2 REG_FE4ISR				(Offset 0x1134) */
+
+#define BIT_FS_CLI3_RX_UMD0_INT BIT(15)
+
+/* 2 REG_FE4ISR				(Offset 0x1134) */
+
+#define BIT_FS_CLI3_RX_UMD1_INT BIT(14)
+
+/* 2 REG_FE4ISR				(Offset 0x1134) */
+
+#define BIT_FS_CLI3_RX_BMD0_INT BIT(13)
+
+/* 2 REG_FE4ISR				(Offset 0x1134) */
+
+#define BIT_FS_CLI3_RX_BMD1_INT BIT(12)
+
+/* 2 REG_FE4ISR				(Offset 0x1134) */
+
+#define BIT_FS_CLI2_RX_UMD0_INT BIT(11)
+
+/* 2 REG_FE4ISR				(Offset 0x1134) */
+
+#define BIT_FS_CLI2_RX_UMD1_INT BIT(10)
+
+/* 2 REG_FE4ISR				(Offset 0x1134) */
+
+#define BIT_FS_CLI2_RX_BMD0_INT BIT(9)
+
+/* 2 REG_FE4ISR				(Offset 0x1134) */
+
+#define BIT_FS_CLI2_RX_BMD1_INT BIT(8)
+
+/* 2 REG_FE4ISR				(Offset 0x1134) */
+
+#define BIT_FS_CLI1_RX_UMD0_INT BIT(7)
+
+/* 2 REG_FE4ISR				(Offset 0x1134) */
+
+#define BIT_FS_CLI1_RX_UMD1_INT BIT(6)
+
+/* 2 REG_FE4ISR				(Offset 0x1134) */
+
+#define BIT_FS_CLI1_RX_BMD0_INT BIT(5)
+
+/* 2 REG_FE4ISR				(Offset 0x1134) */
+
+#define BIT_FS_CLI1_RX_BMD1_INT BIT(4)
+
+/* 2 REG_FE4ISR				(Offset 0x1134) */
+
+#define BIT_FS_CLI0_RX_UMD0_INT BIT(3)
+
+/* 2 REG_FE4ISR				(Offset 0x1134) */
+
+#define BIT_FS_CLI0_RX_UMD1_INT BIT(2)
+
+/* 2 REG_FE4ISR				(Offset 0x1134) */
+
+#define BIT_FS_CLI0_RX_BMD0_INT BIT(1)
+
+/* 2 REG_FE4ISR				(Offset 0x1134) */
+
+#define BIT_FS_CLI0_RX_BMD1_INT BIT(0)
+
+/* 2 REG_FT1IMR				(Offset 0x1138) */
+
+#define BIT__FT2ISR__IND_MSK BIT(30)
+#define BIT_FTM_PTT_INT_EN BIT(29)
+#define BIT_RXFTMREQ_INT_EN BIT(28)
+#define BIT_RXFTM_INT_EN BIT(27)
+#define BIT_TXFTM_INT_EN BIT(26)
+
+/* 2 REG_FT1IMR				(Offset 0x1138) */
+
+#define BIT_FS_H2C_CMD_OK_INT_EN BIT(25)
+#define BIT_FS_H2C_CMD_FULL_INT_EN BIT(24)
+
+/* 2 REG_FT1IMR				(Offset 0x1138) */
+
+#define BIT_FS_MACID_PWRCHANGE5_INT_EN BIT(23)
+#define BIT_FS_MACID_PWRCHANGE4_INT_EN BIT(22)
+#define BIT_FS_MACID_PWRCHANGE3_INT_EN BIT(21)
+#define BIT_FS_MACID_PWRCHANGE2_INT_EN BIT(20)
+#define BIT_FS_MACID_PWRCHANGE1_INT_EN BIT(19)
+#define BIT_FS_MACID_PWRCHANGE0_INT_EN BIT(18)
+#define BIT_FS_CTWEND2_INT_EN BIT(17)
+#define BIT_FS_CTWEND1_INT_EN BIT(16)
+#define BIT_FS_CTWEND0_INT_EN BIT(15)
+#define BIT_FS_TX_NULL1_INT_EN BIT(14)
+#define BIT_FS_TX_NULL0_INT_EN BIT(13)
+#define BIT_FS_TSF_BIT32_TOGGLE_EN BIT(12)
+#define BIT_FS_P2P_RFON2_INT_EN BIT(11)
+#define BIT_FS_P2P_RFOFF2_INT_EN BIT(10)
+#define BIT_FS_P2P_RFON1_INT_EN BIT(9)
+#define BIT_FS_P2P_RFOFF1_INT_EN BIT(8)
+#define BIT_FS_P2P_RFON0_INT_EN BIT(7)
+#define BIT_FS_P2P_RFOFF0_INT_EN BIT(6)
+#define BIT_FS_RX_UAPSDMD1_EN BIT(5)
+#define BIT_FS_RX_UAPSDMD0_EN BIT(4)
+#define BIT_FS_TRIGGER_PKT_EN BIT(3)
+#define BIT_FS_EOSP_INT_EN BIT(2)
+#define BIT_FS_RPWM2_INT_EN BIT(1)
+#define BIT_FS_RPWM_INT_EN BIT(0)
+
+/* 2 REG_FT1ISR				(Offset 0x113C) */
+
+#define BIT__FT2ISR__IND_INT BIT(30)
+#define BIT_FTM_PTT_INT BIT(29)
+#define BIT_RXFTMREQ_INT BIT(28)
+#define BIT_RXFTM_INT BIT(27)
+#define BIT_TXFTM_INT BIT(26)
+
+/* 2 REG_FT1ISR				(Offset 0x113C) */
+
+#define BIT_FS_H2C_CMD_OK_INT BIT(25)
+#define BIT_FS_H2C_CMD_FULL_INT BIT(24)
+
+/* 2 REG_FT1ISR				(Offset 0x113C) */
+
+#define BIT_FS_MACID_PWRCHANGE5_INT BIT(23)
+#define BIT_FS_MACID_PWRCHANGE4_INT BIT(22)
+#define BIT_FS_MACID_PWRCHANGE3_INT BIT(21)
+#define BIT_FS_MACID_PWRCHANGE2_INT BIT(20)
+#define BIT_FS_MACID_PWRCHANGE1_INT BIT(19)
+#define BIT_FS_MACID_PWRCHANGE0_INT BIT(18)
+#define BIT_FS_CTWEND2_INT BIT(17)
+#define BIT_FS_CTWEND1_INT BIT(16)
+#define BIT_FS_CTWEND0_INT BIT(15)
+#define BIT_FS_TX_NULL1_INT BIT(14)
+#define BIT_FS_TX_NULL0_INT BIT(13)
+#define BIT_FS_TSF_BIT32_TOGGLE_INT BIT(12)
+#define BIT_FS_P2P_RFON2_INT BIT(11)
+#define BIT_FS_P2P_RFOFF2_INT BIT(10)
+#define BIT_FS_P2P_RFON1_INT BIT(9)
+#define BIT_FS_P2P_RFOFF1_INT BIT(8)
+#define BIT_FS_P2P_RFON0_INT BIT(7)
+#define BIT_FS_P2P_RFOFF0_INT BIT(6)
+#define BIT_FS_RX_UAPSDMD1_INT BIT(5)
+#define BIT_FS_RX_UAPSDMD0_INT BIT(4)
+#define BIT_FS_TRIGGER_PKT_INT BIT(3)
+#define BIT_FS_EOSP_INT BIT(2)
+#define BIT_FS_RPWM2_INT BIT(1)
+#define BIT_FS_RPWM_INT BIT(0)
+
+/* 2 REG_SPWR0				(Offset 0x1140) */
+
+#define BIT_SHIFT_MID_31TO0 0
+#define BIT_MASK_MID_31TO0 0xffffffffL
+#define BIT_MID_31TO0(x) (((x) & BIT_MASK_MID_31TO0) << BIT_SHIFT_MID_31TO0)
+#define BIT_GET_MID_31TO0(x) (((x) >> BIT_SHIFT_MID_31TO0) & BIT_MASK_MID_31TO0)
+
+/* 2 REG_SPWR1				(Offset 0x1144) */
+
+#define BIT_SHIFT_MID_63TO32 0
+#define BIT_MASK_MID_63TO32 0xffffffffL
+#define BIT_MID_63TO32(x) (((x) & BIT_MASK_MID_63TO32) << BIT_SHIFT_MID_63TO32)
+#define BIT_GET_MID_63TO32(x)                                                  \
+	(((x) >> BIT_SHIFT_MID_63TO32) & BIT_MASK_MID_63TO32)
+
+/* 2 REG_SPWR2				(Offset 0x1148) */
+
+#define BIT_SHIFT_MID_95O64 0
+#define BIT_MASK_MID_95O64 0xffffffffL
+#define BIT_MID_95O64(x) (((x) & BIT_MASK_MID_95O64) << BIT_SHIFT_MID_95O64)
+#define BIT_GET_MID_95O64(x) (((x) >> BIT_SHIFT_MID_95O64) & BIT_MASK_MID_95O64)
+
+/* 2 REG_SPWR3				(Offset 0x114C) */
+
+#define BIT_SHIFT_MID_127TO96 0
+#define BIT_MASK_MID_127TO96 0xffffffffL
+#define BIT_MID_127TO96(x)                                                     \
+	(((x) & BIT_MASK_MID_127TO96) << BIT_SHIFT_MID_127TO96)
+#define BIT_GET_MID_127TO96(x)                                                 \
+	(((x) >> BIT_SHIFT_MID_127TO96) & BIT_MASK_MID_127TO96)
+
+/* 2 REG_POWSEQ				(Offset 0x1150) */
+
+#define BIT_SHIFT_SEQNUM_MID 16
+#define BIT_MASK_SEQNUM_MID 0xffff
+#define BIT_SEQNUM_MID(x) (((x) & BIT_MASK_SEQNUM_MID) << BIT_SHIFT_SEQNUM_MID)
+#define BIT_GET_SEQNUM_MID(x)                                                  \
+	(((x) >> BIT_SHIFT_SEQNUM_MID) & BIT_MASK_SEQNUM_MID)
+
+#define BIT_SHIFT_REF_MID 0
+#define BIT_MASK_REF_MID 0x7f
+#define BIT_REF_MID(x) (((x) & BIT_MASK_REF_MID) << BIT_SHIFT_REF_MID)
+#define BIT_GET_REF_MID(x) (((x) >> BIT_SHIFT_REF_MID) & BIT_MASK_REF_MID)
+
+/* 2 REG_TC7_CTRL_V1				(Offset 0x1158) */
+
+#define BIT_TC7INT_EN BIT(26)
+#define BIT_TC7MODE BIT(25)
+#define BIT_TC7EN BIT(24)
+
+#define BIT_SHIFT_TC7DATA 0
+#define BIT_MASK_TC7DATA 0xffffff
+#define BIT_TC7DATA(x) (((x) & BIT_MASK_TC7DATA) << BIT_SHIFT_TC7DATA)
+#define BIT_GET_TC7DATA(x) (((x) >> BIT_SHIFT_TC7DATA) & BIT_MASK_TC7DATA)
+
+/* 2 REG_TC8_CTRL_V1				(Offset 0x115C) */
+
+#define BIT_TC8INT_EN BIT(26)
+#define BIT_TC8MODE BIT(25)
+#define BIT_TC8EN BIT(24)
+
+#define BIT_SHIFT_TC8DATA 0
+#define BIT_MASK_TC8DATA 0xffffff
+#define BIT_TC8DATA(x) (((x) & BIT_MASK_TC8DATA) << BIT_SHIFT_TC8DATA)
+#define BIT_GET_TC8DATA(x) (((x) >> BIT_SHIFT_TC8DATA) & BIT_MASK_TC8DATA)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_CLI3_RX_UAPSDMD1_EN BIT(31)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_CLI3_RX_UAPSDMD0_EN BIT(30)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_CLI3_TRIGGER_PKT_EN BIT(29)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_CLI3_EOSP_INT_EN BIT(28)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_CLI2_RX_UAPSDMD1_EN BIT(27)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_CLI2_RX_UAPSDMD0_EN BIT(26)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_CLI2_TRIGGER_PKT_EN BIT(25)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_CLI2_EOSP_INT_EN BIT(24)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_CLI1_RX_UAPSDMD1_EN BIT(23)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_CLI1_RX_UAPSDMD0_EN BIT(22)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_CLI1_TRIGGER_PKT_EN BIT(21)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_CLI1_EOSP_INT_EN BIT(20)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_CLI0_RX_UAPSDMD1_EN BIT(19)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_CLI0_RX_UAPSDMD0_EN BIT(18)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_CLI0_TRIGGER_PKT_EN BIT(17)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_CLI0_EOSP_INT_EN BIT(16)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_TSF_BIT32_TOGGLE_P2P2_EN BIT(9)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_TSF_BIT32_TOGGLE_P2P1_EN BIT(8)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_CLI3_TX_NULL1_INT_EN BIT(7)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_CLI3_TX_NULL0_INT_EN BIT(6)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_CLI2_TX_NULL1_INT_EN BIT(5)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_CLI2_TX_NULL0_INT_EN BIT(4)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_CLI1_TX_NULL1_INT_EN BIT(3)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_CLI1_TX_NULL0_INT_EN BIT(2)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_CLI0_TX_NULL1_INT_EN BIT(1)
+
+/* 2 REG_FT2IMR				(Offset 0x11E0) */
+
+#define BIT_FS_CLI0_TX_NULL0_INT_EN BIT(0)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_CLI3_RX_UAPSDMD1_INT BIT(31)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_CLI3_RX_UAPSDMD0_INT BIT(30)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_CLI3_TRIGGER_PKT_INT BIT(29)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_CLI3_EOSP_INT BIT(28)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_CLI2_RX_UAPSDMD1_INT BIT(27)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_CLI2_RX_UAPSDMD0_INT BIT(26)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_CLI2_TRIGGER_PKT_INT BIT(25)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_CLI2_EOSP_INT BIT(24)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_CLI1_RX_UAPSDMD1_INT BIT(23)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_CLI1_RX_UAPSDMD0_INT BIT(22)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_CLI1_TRIGGER_PKT_INT BIT(21)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_CLI1_EOSP_INT BIT(20)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_CLI0_RX_UAPSDMD1_INT BIT(19)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_CLI0_RX_UAPSDMD0_INT BIT(18)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_CLI0_TRIGGER_PKT_INT BIT(17)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_CLI0_EOSP_INT BIT(16)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_TSF_BIT32_TOGGLE_P2P2_INT BIT(9)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_TSF_BIT32_TOGGLE_P2P1_INT BIT(8)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_CLI3_TX_NULL1_INT BIT(7)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_CLI3_TX_NULL0_INT BIT(6)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_CLI2_TX_NULL1_INT BIT(5)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_CLI2_TX_NULL0_INT BIT(4)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_CLI1_TX_NULL1_INT BIT(3)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_CLI1_TX_NULL0_INT BIT(2)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_CLI0_TX_NULL1_INT BIT(1)
+
+/* 2 REG_FT2ISR				(Offset 0x11E4) */
+
+#define BIT_FS_CLI0_TX_NULL0_INT BIT(0)
+
+/* 2 REG_MSG2				(Offset 0x11F0) */
+
+#define BIT_SHIFT_FW_MSG2 0
+#define BIT_MASK_FW_MSG2 0xffffffffL
+#define BIT_FW_MSG2(x) (((x) & BIT_MASK_FW_MSG2) << BIT_SHIFT_FW_MSG2)
+#define BIT_GET_FW_MSG2(x) (((x) >> BIT_SHIFT_FW_MSG2) & BIT_MASK_FW_MSG2)
+
+/* 2 REG_MSG3				(Offset 0x11F4) */
+
+#define BIT_SHIFT_FW_MSG3 0
+#define BIT_MASK_FW_MSG3 0xffffffffL
+#define BIT_FW_MSG3(x) (((x) & BIT_MASK_FW_MSG3) << BIT_SHIFT_FW_MSG3)
+#define BIT_GET_FW_MSG3(x) (((x) >> BIT_SHIFT_FW_MSG3) & BIT_MASK_FW_MSG3)
+
+/* 2 REG_MSG4				(Offset 0x11F8) */
+
+#define BIT_SHIFT_FW_MSG4 0
+#define BIT_MASK_FW_MSG4 0xffffffffL
+#define BIT_FW_MSG4(x) (((x) & BIT_MASK_FW_MSG4) << BIT_SHIFT_FW_MSG4)
+#define BIT_GET_FW_MSG4(x) (((x) >> BIT_SHIFT_FW_MSG4) & BIT_MASK_FW_MSG4)
+
+/* 2 REG_MSG5				(Offset 0x11FC) */
+
+#define BIT_SHIFT_FW_MSG5 0
+#define BIT_MASK_FW_MSG5 0xffffffffL
+#define BIT_FW_MSG5(x) (((x) & BIT_MASK_FW_MSG5) << BIT_SHIFT_FW_MSG5)
+#define BIT_GET_FW_MSG5(x) (((x) >> BIT_SHIFT_FW_MSG5) & BIT_MASK_FW_MSG5)
+
+/* 2 REG_DDMA_CH0SA				(Offset 0x1200) */
+
+#define BIT_SHIFT_DDMACH0_SA 0
+#define BIT_MASK_DDMACH0_SA 0xffffffffL
+#define BIT_DDMACH0_SA(x) (((x) & BIT_MASK_DDMACH0_SA) << BIT_SHIFT_DDMACH0_SA)
+#define BIT_GET_DDMACH0_SA(x)                                                  \
+	(((x) >> BIT_SHIFT_DDMACH0_SA) & BIT_MASK_DDMACH0_SA)
+
+/* 2 REG_DDMA_CH0DA				(Offset 0x1204) */
+
+#define BIT_SHIFT_DDMACH0_DA 0
+#define BIT_MASK_DDMACH0_DA 0xffffffffL
+#define BIT_DDMACH0_DA(x) (((x) & BIT_MASK_DDMACH0_DA) << BIT_SHIFT_DDMACH0_DA)
+#define BIT_GET_DDMACH0_DA(x)                                                  \
+	(((x) >> BIT_SHIFT_DDMACH0_DA) & BIT_MASK_DDMACH0_DA)
+
+/* 2 REG_DDMA_CH0CTRL			(Offset 0x1208) */
+
+#define BIT_DDMACH0_OWN BIT(31)
+#define BIT_DDMACH0_CHKSUM_EN BIT(29)
+#define BIT_DDMACH0_DA_W_DISABLE BIT(28)
+#define BIT_DDMACH0_CHKSUM_STS BIT(27)
+#define BIT_DDMACH0_DDMA_MODE BIT(26)
+#define BIT_DDMACH0_RESET_CHKSUM_STS BIT(25)
+#define BIT_DDMACH0_CHKSUM_CONT BIT(24)
+
+#define BIT_SHIFT_DDMACH0_DLEN 0
+#define BIT_MASK_DDMACH0_DLEN 0x3ffff
+#define BIT_DDMACH0_DLEN(x)                                                    \
+	(((x) & BIT_MASK_DDMACH0_DLEN) << BIT_SHIFT_DDMACH0_DLEN)
+#define BIT_GET_DDMACH0_DLEN(x)                                                \
+	(((x) >> BIT_SHIFT_DDMACH0_DLEN) & BIT_MASK_DDMACH0_DLEN)
+
+/* 2 REG_DDMA_CH1SA				(Offset 0x1210) */
+
+#define BIT_SHIFT_DDMACH1_SA 0
+#define BIT_MASK_DDMACH1_SA 0xffffffffL
+#define BIT_DDMACH1_SA(x) (((x) & BIT_MASK_DDMACH1_SA) << BIT_SHIFT_DDMACH1_SA)
+#define BIT_GET_DDMACH1_SA(x)                                                  \
+	(((x) >> BIT_SHIFT_DDMACH1_SA) & BIT_MASK_DDMACH1_SA)
+
+/* 2 REG_DDMA_CH1DA				(Offset 0x1214) */
+
+#define BIT_SHIFT_DDMACH1_DA 0
+#define BIT_MASK_DDMACH1_DA 0xffffffffL
+#define BIT_DDMACH1_DA(x) (((x) & BIT_MASK_DDMACH1_DA) << BIT_SHIFT_DDMACH1_DA)
+#define BIT_GET_DDMACH1_DA(x)                                                  \
+	(((x) >> BIT_SHIFT_DDMACH1_DA) & BIT_MASK_DDMACH1_DA)
+
+/* 2 REG_DDMA_CH1CTRL			(Offset 0x1218) */
+
+#define BIT_DDMACH1_OWN BIT(31)
+#define BIT_DDMACH1_CHKSUM_EN BIT(29)
+#define BIT_DDMACH1_DA_W_DISABLE BIT(28)
+#define BIT_DDMACH1_CHKSUM_STS BIT(27)
+#define BIT_DDMACH1_DDMA_MODE BIT(26)
+#define BIT_DDMACH1_RESET_CHKSUM_STS BIT(25)
+#define BIT_DDMACH1_CHKSUM_CONT BIT(24)
+
+#define BIT_SHIFT_DDMACH1_DLEN 0
+#define BIT_MASK_DDMACH1_DLEN 0x3ffff
+#define BIT_DDMACH1_DLEN(x)                                                    \
+	(((x) & BIT_MASK_DDMACH1_DLEN) << BIT_SHIFT_DDMACH1_DLEN)
+#define BIT_GET_DDMACH1_DLEN(x)                                                \
+	(((x) >> BIT_SHIFT_DDMACH1_DLEN) & BIT_MASK_DDMACH1_DLEN)
+
+/* 2 REG_DDMA_CH2SA				(Offset 0x1220) */
+
+#define BIT_SHIFT_DDMACH2_SA 0
+#define BIT_MASK_DDMACH2_SA 0xffffffffL
+#define BIT_DDMACH2_SA(x) (((x) & BIT_MASK_DDMACH2_SA) << BIT_SHIFT_DDMACH2_SA)
+#define BIT_GET_DDMACH2_SA(x)                                                  \
+	(((x) >> BIT_SHIFT_DDMACH2_SA) & BIT_MASK_DDMACH2_SA)
+
+/* 2 REG_DDMA_CH2DA				(Offset 0x1224) */
+
+#define BIT_SHIFT_DDMACH2_DA 0
+#define BIT_MASK_DDMACH2_DA 0xffffffffL
+#define BIT_DDMACH2_DA(x) (((x) & BIT_MASK_DDMACH2_DA) << BIT_SHIFT_DDMACH2_DA)
+#define BIT_GET_DDMACH2_DA(x)                                                  \
+	(((x) >> BIT_SHIFT_DDMACH2_DA) & BIT_MASK_DDMACH2_DA)
+
+/* 2 REG_DDMA_CH2CTRL			(Offset 0x1228) */
+
+#define BIT_DDMACH2_OWN BIT(31)
+#define BIT_DDMACH2_CHKSUM_EN BIT(29)
+#define BIT_DDMACH2_DA_W_DISABLE BIT(28)
+#define BIT_DDMACH2_CHKSUM_STS BIT(27)
+#define BIT_DDMACH2_DDMA_MODE BIT(26)
+#define BIT_DDMACH2_RESET_CHKSUM_STS BIT(25)
+#define BIT_DDMACH2_CHKSUM_CONT BIT(24)
+
+#define BIT_SHIFT_DDMACH2_DLEN 0
+#define BIT_MASK_DDMACH2_DLEN 0x3ffff
+#define BIT_DDMACH2_DLEN(x)                                                    \
+	(((x) & BIT_MASK_DDMACH2_DLEN) << BIT_SHIFT_DDMACH2_DLEN)
+#define BIT_GET_DDMACH2_DLEN(x)                                                \
+	(((x) >> BIT_SHIFT_DDMACH2_DLEN) & BIT_MASK_DDMACH2_DLEN)
+
+/* 2 REG_DDMA_CH3SA				(Offset 0x1230) */
+
+#define BIT_SHIFT_DDMACH3_SA 0
+#define BIT_MASK_DDMACH3_SA 0xffffffffL
+#define BIT_DDMACH3_SA(x) (((x) & BIT_MASK_DDMACH3_SA) << BIT_SHIFT_DDMACH3_SA)
+#define BIT_GET_DDMACH3_SA(x)                                                  \
+	(((x) >> BIT_SHIFT_DDMACH3_SA) & BIT_MASK_DDMACH3_SA)
+
+/* 2 REG_DDMA_CH3DA				(Offset 0x1234) */
+
+#define BIT_SHIFT_DDMACH3_DA 0
+#define BIT_MASK_DDMACH3_DA 0xffffffffL
+#define BIT_DDMACH3_DA(x) (((x) & BIT_MASK_DDMACH3_DA) << BIT_SHIFT_DDMACH3_DA)
+#define BIT_GET_DDMACH3_DA(x)                                                  \
+	(((x) >> BIT_SHIFT_DDMACH3_DA) & BIT_MASK_DDMACH3_DA)
+
+/* 2 REG_DDMA_CH3CTRL			(Offset 0x1238) */
+
+#define BIT_DDMACH3_OWN BIT(31)
+#define BIT_DDMACH3_CHKSUM_EN BIT(29)
+#define BIT_DDMACH3_DA_W_DISABLE BIT(28)
+#define BIT_DDMACH3_CHKSUM_STS BIT(27)
+#define BIT_DDMACH3_DDMA_MODE BIT(26)
+#define BIT_DDMACH3_RESET_CHKSUM_STS BIT(25)
+#define BIT_DDMACH3_CHKSUM_CONT BIT(24)
+
+#define BIT_SHIFT_DDMACH3_DLEN 0
+#define BIT_MASK_DDMACH3_DLEN 0x3ffff
+#define BIT_DDMACH3_DLEN(x)                                                    \
+	(((x) & BIT_MASK_DDMACH3_DLEN) << BIT_SHIFT_DDMACH3_DLEN)
+#define BIT_GET_DDMACH3_DLEN(x)                                                \
+	(((x) >> BIT_SHIFT_DDMACH3_DLEN) & BIT_MASK_DDMACH3_DLEN)
+
+/* 2 REG_DDMA_CH4SA				(Offset 0x1240) */
+
+#define BIT_SHIFT_DDMACH4_SA 0
+#define BIT_MASK_DDMACH4_SA 0xffffffffL
+#define BIT_DDMACH4_SA(x) (((x) & BIT_MASK_DDMACH4_SA) << BIT_SHIFT_DDMACH4_SA)
+#define BIT_GET_DDMACH4_SA(x)                                                  \
+	(((x) >> BIT_SHIFT_DDMACH4_SA) & BIT_MASK_DDMACH4_SA)
+
+/* 2 REG_DDMA_CH4DA				(Offset 0x1244) */
+
+#define BIT_SHIFT_DDMACH4_DA 0
+#define BIT_MASK_DDMACH4_DA 0xffffffffL
+#define BIT_DDMACH4_DA(x) (((x) & BIT_MASK_DDMACH4_DA) << BIT_SHIFT_DDMACH4_DA)
+#define BIT_GET_DDMACH4_DA(x)                                                  \
+	(((x) >> BIT_SHIFT_DDMACH4_DA) & BIT_MASK_DDMACH4_DA)
+
+/* 2 REG_DDMA_CH4CTRL			(Offset 0x1248) */
+
+#define BIT_DDMACH4_OWN BIT(31)
+#define BIT_DDMACH4_CHKSUM_EN BIT(29)
+#define BIT_DDMACH4_DA_W_DISABLE BIT(28)
+#define BIT_DDMACH4_CHKSUM_STS BIT(27)
+#define BIT_DDMACH4_DDMA_MODE BIT(26)
+#define BIT_DDMACH4_RESET_CHKSUM_STS BIT(25)
+#define BIT_DDMACH4_CHKSUM_CONT BIT(24)
+
+#define BIT_SHIFT_DDMACH4_DLEN 0
+#define BIT_MASK_DDMACH4_DLEN 0x3ffff
+#define BIT_DDMACH4_DLEN(x)                                                    \
+	(((x) & BIT_MASK_DDMACH4_DLEN) << BIT_SHIFT_DDMACH4_DLEN)
+#define BIT_GET_DDMACH4_DLEN(x)                                                \
+	(((x) >> BIT_SHIFT_DDMACH4_DLEN) & BIT_MASK_DDMACH4_DLEN)
+
+/* 2 REG_DDMA_CH5SA				(Offset 0x1250) */
+
+#define BIT_SHIFT_DDMACH5_SA 0
+#define BIT_MASK_DDMACH5_SA 0xffffffffL
+#define BIT_DDMACH5_SA(x) (((x) & BIT_MASK_DDMACH5_SA) << BIT_SHIFT_DDMACH5_SA)
+#define BIT_GET_DDMACH5_SA(x)                                                  \
+	(((x) >> BIT_SHIFT_DDMACH5_SA) & BIT_MASK_DDMACH5_SA)
+
+/* 2 REG_DDMA_CH5DA				(Offset 0x1254) */
+
+#define BIT_DDMACH5_OWN BIT(31)
+#define BIT_DDMACH5_CHKSUM_EN BIT(29)
+#define BIT_DDMACH5_DA_W_DISABLE BIT(28)
+#define BIT_DDMACH5_CHKSUM_STS BIT(27)
+#define BIT_DDMACH5_DDMA_MODE BIT(26)
+#define BIT_DDMACH5_RESET_CHKSUM_STS BIT(25)
+#define BIT_DDMACH5_CHKSUM_CONT BIT(24)
+
+#define BIT_SHIFT_DDMACH5_DA 0
+#define BIT_MASK_DDMACH5_DA 0xffffffffL
+#define BIT_DDMACH5_DA(x) (((x) & BIT_MASK_DDMACH5_DA) << BIT_SHIFT_DDMACH5_DA)
+#define BIT_GET_DDMACH5_DA(x)                                                  \
+	(((x) >> BIT_SHIFT_DDMACH5_DA) & BIT_MASK_DDMACH5_DA)
+
+#define BIT_SHIFT_DDMACH5_DLEN 0
+#define BIT_MASK_DDMACH5_DLEN 0x3ffff
+#define BIT_DDMACH5_DLEN(x)                                                    \
+	(((x) & BIT_MASK_DDMACH5_DLEN) << BIT_SHIFT_DDMACH5_DLEN)
+#define BIT_GET_DDMACH5_DLEN(x)                                                \
+	(((x) >> BIT_SHIFT_DDMACH5_DLEN) & BIT_MASK_DDMACH5_DLEN)
+
+/* 2 REG_DDMA_INT_MSK			(Offset 0x12E0) */
+
+#define BIT_DDMACH5_MSK BIT(5)
+#define BIT_DDMACH4_MSK BIT(4)
+#define BIT_DDMACH3_MSK BIT(3)
+#define BIT_DDMACH2_MSK BIT(2)
+#define BIT_DDMACH1_MSK BIT(1)
+#define BIT_DDMACH0_MSK BIT(0)
+
+/* 2 REG_DDMA_CHSTATUS			(Offset 0x12E8) */
+
+#define BIT_DDMACH5_BUSY BIT(5)
+#define BIT_DDMACH4_BUSY BIT(4)
+#define BIT_DDMACH3_BUSY BIT(3)
+#define BIT_DDMACH2_BUSY BIT(2)
+#define BIT_DDMACH1_BUSY BIT(1)
+#define BIT_DDMACH0_BUSY BIT(0)
+
+/* 2 REG_DDMA_CHKSUM				(Offset 0x12F0) */
+
+#define BIT_SHIFT_IDDMA0_CHKSUM 0
+#define BIT_MASK_IDDMA0_CHKSUM 0xffff
+#define BIT_IDDMA0_CHKSUM(x)                                                   \
+	(((x) & BIT_MASK_IDDMA0_CHKSUM) << BIT_SHIFT_IDDMA0_CHKSUM)
+#define BIT_GET_IDDMA0_CHKSUM(x)                                               \
+	(((x) >> BIT_SHIFT_IDDMA0_CHKSUM) & BIT_MASK_IDDMA0_CHKSUM)
+
+/* 2 REG_DDMA_MONITOR			(Offset 0x12FC) */
+
+#define BIT_IDDMA0_PERMU_UNDERFLOW BIT(14)
+#define BIT_IDDMA0_FIFO_UNDERFLOW BIT(13)
+#define BIT_IDDMA0_FIFO_OVERFLOW BIT(12)
+#define BIT_ECRC_EN_V1 BIT(7)
+#define BIT_MDIO_RFLAG_V1 BIT(6)
+#define BIT_CH5_ERR BIT(5)
+#define BIT_MDIO_WFLAG_V1 BIT(5)
+#define BIT_CH4_ERR BIT(4)
+#define BIT_CH3_ERR BIT(3)
+#define BIT_CH2_ERR BIT(2)
+#define BIT_CH1_ERR BIT(1)
+#define BIT_CH0_ERR BIT(0)
+
+/* 2 REG_STC_INT_CS				(Offset 0x1300) */
+
+#define BIT_STC_INT_EN BIT(31)
+
+#define BIT_SHIFT_STC_INT_FLAG 16
+#define BIT_MASK_STC_INT_FLAG 0xff
+#define BIT_STC_INT_FLAG(x)                                                    \
+	(((x) & BIT_MASK_STC_INT_FLAG) << BIT_SHIFT_STC_INT_FLAG)
+#define BIT_GET_STC_INT_FLAG(x)                                                \
+	(((x) >> BIT_SHIFT_STC_INT_FLAG) & BIT_MASK_STC_INT_FLAG)
+
+#define BIT_SHIFT_STC_INT_IDX 8
+#define BIT_MASK_STC_INT_IDX 0x7
+#define BIT_STC_INT_IDX(x)                                                     \
+	(((x) & BIT_MASK_STC_INT_IDX) << BIT_SHIFT_STC_INT_IDX)
+#define BIT_GET_STC_INT_IDX(x)                                                 \
+	(((x) >> BIT_SHIFT_STC_INT_IDX) & BIT_MASK_STC_INT_IDX)
+
+#define BIT_SHIFT_STC_INT_REALTIME_CS 0
+#define BIT_MASK_STC_INT_REALTIME_CS 0x3f
+#define BIT_STC_INT_REALTIME_CS(x)                                             \
+	(((x) & BIT_MASK_STC_INT_REALTIME_CS) << BIT_SHIFT_STC_INT_REALTIME_CS)
+#define BIT_GET_STC_INT_REALTIME_CS(x)                                         \
+	(((x) >> BIT_SHIFT_STC_INT_REALTIME_CS) & BIT_MASK_STC_INT_REALTIME_CS)
+
+/* 2 REG_ST_INT_CFG				(Offset 0x1304) */
+
+#define BIT_STC_INT_GRP_EN BIT(31)
+
+#define BIT_SHIFT_STC_INT_EXPECT_LS 8
+#define BIT_MASK_STC_INT_EXPECT_LS 0x3f
+#define BIT_STC_INT_EXPECT_LS(x)                                               \
+	(((x) & BIT_MASK_STC_INT_EXPECT_LS) << BIT_SHIFT_STC_INT_EXPECT_LS)
+#define BIT_GET_STC_INT_EXPECT_LS(x)                                           \
+	(((x) >> BIT_SHIFT_STC_INT_EXPECT_LS) & BIT_MASK_STC_INT_EXPECT_LS)
+
+#define BIT_SHIFT_STC_INT_EXPECT_CS 0
+#define BIT_MASK_STC_INT_EXPECT_CS 0x3f
+#define BIT_STC_INT_EXPECT_CS(x)                                               \
+	(((x) & BIT_MASK_STC_INT_EXPECT_CS) << BIT_SHIFT_STC_INT_EXPECT_CS)
+#define BIT_GET_STC_INT_EXPECT_CS(x)                                           \
+	(((x) >> BIT_SHIFT_STC_INT_EXPECT_CS) & BIT_MASK_STC_INT_EXPECT_CS)
+
+/* 2 REG_CMU_DLY_CTRL			(Offset 0x1310) */
+
+#define BIT_CMU_DLY_EN BIT(31)
+#define BIT_CMU_DLY_MODE BIT(30)
+
+#define BIT_SHIFT_CMU_DLY_PRE_DIV 0
+#define BIT_MASK_CMU_DLY_PRE_DIV 0xff
+#define BIT_CMU_DLY_PRE_DIV(x)                                                 \
+	(((x) & BIT_MASK_CMU_DLY_PRE_DIV) << BIT_SHIFT_CMU_DLY_PRE_DIV)
+#define BIT_GET_CMU_DLY_PRE_DIV(x)                                             \
+	(((x) >> BIT_SHIFT_CMU_DLY_PRE_DIV) & BIT_MASK_CMU_DLY_PRE_DIV)
+
+/* 2 REG_CMU_DLY_CFG				(Offset 0x1314) */
+
+#define BIT_SHIFT_CMU_DLY_LTR_A2I 24
+#define BIT_MASK_CMU_DLY_LTR_A2I 0xff
+#define BIT_CMU_DLY_LTR_A2I(x)                                                 \
+	(((x) & BIT_MASK_CMU_DLY_LTR_A2I) << BIT_SHIFT_CMU_DLY_LTR_A2I)
+#define BIT_GET_CMU_DLY_LTR_A2I(x)                                             \
+	(((x) >> BIT_SHIFT_CMU_DLY_LTR_A2I) & BIT_MASK_CMU_DLY_LTR_A2I)
+
+#define BIT_SHIFT_CMU_DLY_LTR_I2A 16
+#define BIT_MASK_CMU_DLY_LTR_I2A 0xff
+#define BIT_CMU_DLY_LTR_I2A(x)                                                 \
+	(((x) & BIT_MASK_CMU_DLY_LTR_I2A) << BIT_SHIFT_CMU_DLY_LTR_I2A)
+#define BIT_GET_CMU_DLY_LTR_I2A(x)                                             \
+	(((x) >> BIT_SHIFT_CMU_DLY_LTR_I2A) & BIT_MASK_CMU_DLY_LTR_I2A)
+
+#define BIT_SHIFT_CMU_DLY_LTR_IDLE 8
+#define BIT_MASK_CMU_DLY_LTR_IDLE 0xff
+#define BIT_CMU_DLY_LTR_IDLE(x)                                                \
+	(((x) & BIT_MASK_CMU_DLY_LTR_IDLE) << BIT_SHIFT_CMU_DLY_LTR_IDLE)
+#define BIT_GET_CMU_DLY_LTR_IDLE(x)                                            \
+	(((x) >> BIT_SHIFT_CMU_DLY_LTR_IDLE) & BIT_MASK_CMU_DLY_LTR_IDLE)
+
+#define BIT_SHIFT_CMU_DLY_LTR_ACT 0
+#define BIT_MASK_CMU_DLY_LTR_ACT 0xff
+#define BIT_CMU_DLY_LTR_ACT(x)                                                 \
+	(((x) & BIT_MASK_CMU_DLY_LTR_ACT) << BIT_SHIFT_CMU_DLY_LTR_ACT)
+#define BIT_GET_CMU_DLY_LTR_ACT(x)                                             \
+	(((x) >> BIT_SHIFT_CMU_DLY_LTR_ACT) & BIT_MASK_CMU_DLY_LTR_ACT)
+
+/* 2 REG_H2CQ_TXBD_DESA			(Offset 0x1320) */
+
+#define BIT_SHIFT_H2CQ_TXBD_DESA 0
+#define BIT_MASK_H2CQ_TXBD_DESA 0xffffffffffffffffL
+#define BIT_H2CQ_TXBD_DESA(x)                                                  \
+	(((x) & BIT_MASK_H2CQ_TXBD_DESA) << BIT_SHIFT_H2CQ_TXBD_DESA)
+#define BIT_GET_H2CQ_TXBD_DESA(x)                                              \
+	(((x) >> BIT_SHIFT_H2CQ_TXBD_DESA) & BIT_MASK_H2CQ_TXBD_DESA)
+
+/* 2 REG_H2CQ_TXBD_NUM			(Offset 0x1328) */
+
+#define BIT_PCIE_H2CQ_FLAG BIT(14)
+
+/* 2 REG_H2CQ_TXBD_NUM			(Offset 0x1328) */
+
+#define BIT_SHIFT_H2CQ_DESC_MODE 12
+#define BIT_MASK_H2CQ_DESC_MODE 0x3
+#define BIT_H2CQ_DESC_MODE(x)                                                  \
+	(((x) & BIT_MASK_H2CQ_DESC_MODE) << BIT_SHIFT_H2CQ_DESC_MODE)
+#define BIT_GET_H2CQ_DESC_MODE(x)                                              \
+	(((x) >> BIT_SHIFT_H2CQ_DESC_MODE) & BIT_MASK_H2CQ_DESC_MODE)
+
+#define BIT_SHIFT_H2CQ_DESC_NUM 0
+#define BIT_MASK_H2CQ_DESC_NUM 0xfff
+#define BIT_H2CQ_DESC_NUM(x)                                                   \
+	(((x) & BIT_MASK_H2CQ_DESC_NUM) << BIT_SHIFT_H2CQ_DESC_NUM)
+#define BIT_GET_H2CQ_DESC_NUM(x)                                               \
+	(((x) >> BIT_SHIFT_H2CQ_DESC_NUM) & BIT_MASK_H2CQ_DESC_NUM)
+
+/* 2 REG_H2CQ_TXBD_IDX			(Offset 0x132C) */
+
+#define BIT_SHIFT_H2CQ_HW_IDX 16
+#define BIT_MASK_H2CQ_HW_IDX 0xfff
+#define BIT_H2CQ_HW_IDX(x)                                                     \
+	(((x) & BIT_MASK_H2CQ_HW_IDX) << BIT_SHIFT_H2CQ_HW_IDX)
+#define BIT_GET_H2CQ_HW_IDX(x)                                                 \
+	(((x) >> BIT_SHIFT_H2CQ_HW_IDX) & BIT_MASK_H2CQ_HW_IDX)
+
+#define BIT_SHIFT_H2CQ_HOST_IDX 0
+#define BIT_MASK_H2CQ_HOST_IDX 0xfff
+#define BIT_H2CQ_HOST_IDX(x)                                                   \
+	(((x) & BIT_MASK_H2CQ_HOST_IDX) << BIT_SHIFT_H2CQ_HOST_IDX)
+#define BIT_GET_H2CQ_HOST_IDX(x)                                               \
+	(((x) >> BIT_SHIFT_H2CQ_HOST_IDX) & BIT_MASK_H2CQ_HOST_IDX)
+
+/* 2 REG_H2CQ_CSR				(Offset 0x1330) */
+
+#define BIT_H2CQ_FULL BIT(31)
+#define BIT_CLR_H2CQ_HOST_IDX BIT(16)
+#define BIT_CLR_H2CQ_HW_IDX BIT(8)
+
+/* 2 REG_CHANGE_PCIE_SPEED			(Offset 0x1350) */
+
+#define BIT_CHANGE_PCIE_SPEED BIT(18)
+
+/* 2 REG_CHANGE_PCIE_SPEED			(Offset 0x1350) */
+
+#define BIT_SHIFT_GEN1_GEN2 16
+#define BIT_MASK_GEN1_GEN2 0x3
+#define BIT_GEN1_GEN2(x) (((x) & BIT_MASK_GEN1_GEN2) << BIT_SHIFT_GEN1_GEN2)
+#define BIT_GET_GEN1_GEN2(x) (((x) >> BIT_SHIFT_GEN1_GEN2) & BIT_MASK_GEN1_GEN2)
+
+/* 2 REG_CHANGE_PCIE_SPEED			(Offset 0x1350) */
+
+#define BIT_SHIFT_AUTO_HANG_RELEASE 0
+#define BIT_MASK_AUTO_HANG_RELEASE 0x7
+#define BIT_AUTO_HANG_RELEASE(x)                                               \
+	(((x) & BIT_MASK_AUTO_HANG_RELEASE) << BIT_SHIFT_AUTO_HANG_RELEASE)
+#define BIT_GET_AUTO_HANG_RELEASE(x)                                           \
+	(((x) >> BIT_SHIFT_AUTO_HANG_RELEASE) & BIT_MASK_AUTO_HANG_RELEASE)
+
+/* 2 REG_OLD_DEHANG				(Offset 0x13F4) */
+
+#define BIT_OLD_DEHANG BIT(1)
+
+/* 2 REG_Q0_Q1_INFO				(Offset 0x1400) */
+
+#define BIT_SHIFT_AC1_PKT_INFO 16
+#define BIT_MASK_AC1_PKT_INFO 0xfff
+#define BIT_AC1_PKT_INFO(x)                                                    \
+	(((x) & BIT_MASK_AC1_PKT_INFO) << BIT_SHIFT_AC1_PKT_INFO)
+#define BIT_GET_AC1_PKT_INFO(x)                                                \
+	(((x) >> BIT_SHIFT_AC1_PKT_INFO) & BIT_MASK_AC1_PKT_INFO)
+
+#define BIT_SHIFT_AC0_PKT_INFO 0
+#define BIT_MASK_AC0_PKT_INFO 0xfff
+#define BIT_AC0_PKT_INFO(x)                                                    \
+	(((x) & BIT_MASK_AC0_PKT_INFO) << BIT_SHIFT_AC0_PKT_INFO)
+#define BIT_GET_AC0_PKT_INFO(x)                                                \
+	(((x) >> BIT_SHIFT_AC0_PKT_INFO) & BIT_MASK_AC0_PKT_INFO)
+
+/* 2 REG_Q2_Q3_INFO				(Offset 0x1404) */
+
+#define BIT_SHIFT_AC3_PKT_INFO 16
+#define BIT_MASK_AC3_PKT_INFO 0xfff
+#define BIT_AC3_PKT_INFO(x)                                                    \
+	(((x) & BIT_MASK_AC3_PKT_INFO) << BIT_SHIFT_AC3_PKT_INFO)
+#define BIT_GET_AC3_PKT_INFO(x)                                                \
+	(((x) >> BIT_SHIFT_AC3_PKT_INFO) & BIT_MASK_AC3_PKT_INFO)
+
+#define BIT_SHIFT_AC2_PKT_INFO 0
+#define BIT_MASK_AC2_PKT_INFO 0xfff
+#define BIT_AC2_PKT_INFO(x)                                                    \
+	(((x) & BIT_MASK_AC2_PKT_INFO) << BIT_SHIFT_AC2_PKT_INFO)
+#define BIT_GET_AC2_PKT_INFO(x)                                                \
+	(((x) >> BIT_SHIFT_AC2_PKT_INFO) & BIT_MASK_AC2_PKT_INFO)
+
+/* 2 REG_Q4_Q5_INFO				(Offset 0x1408) */
+
+#define BIT_SHIFT_AC5_PKT_INFO 16
+#define BIT_MASK_AC5_PKT_INFO 0xfff
+#define BIT_AC5_PKT_INFO(x)                                                    \
+	(((x) & BIT_MASK_AC5_PKT_INFO) << BIT_SHIFT_AC5_PKT_INFO)
+#define BIT_GET_AC5_PKT_INFO(x)                                                \
+	(((x) >> BIT_SHIFT_AC5_PKT_INFO) & BIT_MASK_AC5_PKT_INFO)
+
+#define BIT_SHIFT_AC4_PKT_INFO 0
+#define BIT_MASK_AC4_PKT_INFO 0xfff
+#define BIT_AC4_PKT_INFO(x)                                                    \
+	(((x) & BIT_MASK_AC4_PKT_INFO) << BIT_SHIFT_AC4_PKT_INFO)
+#define BIT_GET_AC4_PKT_INFO(x)                                                \
+	(((x) >> BIT_SHIFT_AC4_PKT_INFO) & BIT_MASK_AC4_PKT_INFO)
+
+/* 2 REG_Q6_Q7_INFO				(Offset 0x140C) */
+
+#define BIT_SHIFT_AC7_PKT_INFO 16
+#define BIT_MASK_AC7_PKT_INFO 0xfff
+#define BIT_AC7_PKT_INFO(x)                                                    \
+	(((x) & BIT_MASK_AC7_PKT_INFO) << BIT_SHIFT_AC7_PKT_INFO)
+#define BIT_GET_AC7_PKT_INFO(x)                                                \
+	(((x) >> BIT_SHIFT_AC7_PKT_INFO) & BIT_MASK_AC7_PKT_INFO)
+
+#define BIT_SHIFT_AC6_PKT_INFO 0
+#define BIT_MASK_AC6_PKT_INFO 0xfff
+#define BIT_AC6_PKT_INFO(x)                                                    \
+	(((x) & BIT_MASK_AC6_PKT_INFO) << BIT_SHIFT_AC6_PKT_INFO)
+#define BIT_GET_AC6_PKT_INFO(x)                                                \
+	(((x) >> BIT_SHIFT_AC6_PKT_INFO) & BIT_MASK_AC6_PKT_INFO)
+
+/* 2 REG_MGQ_HIQ_INFO			(Offset 0x1410) */
+
+#define BIT_SHIFT_HIQ_PKT_INFO 16
+#define BIT_MASK_HIQ_PKT_INFO 0xfff
+#define BIT_HIQ_PKT_INFO(x)                                                    \
+	(((x) & BIT_MASK_HIQ_PKT_INFO) << BIT_SHIFT_HIQ_PKT_INFO)
+#define BIT_GET_HIQ_PKT_INFO(x)                                                \
+	(((x) >> BIT_SHIFT_HIQ_PKT_INFO) & BIT_MASK_HIQ_PKT_INFO)
+
+#define BIT_SHIFT_MGQ_PKT_INFO 0
+#define BIT_MASK_MGQ_PKT_INFO 0xfff
+#define BIT_MGQ_PKT_INFO(x)                                                    \
+	(((x) & BIT_MASK_MGQ_PKT_INFO) << BIT_SHIFT_MGQ_PKT_INFO)
+#define BIT_GET_MGQ_PKT_INFO(x)                                                \
+	(((x) >> BIT_SHIFT_MGQ_PKT_INFO) & BIT_MASK_MGQ_PKT_INFO)
+
+/* 2 REG_CMDQ_BCNQ_INFO			(Offset 0x1414) */
+
+#define BIT_SHIFT_CMDQ_PKT_INFO 16
+#define BIT_MASK_CMDQ_PKT_INFO 0xfff
+#define BIT_CMDQ_PKT_INFO(x)                                                   \
+	(((x) & BIT_MASK_CMDQ_PKT_INFO) << BIT_SHIFT_CMDQ_PKT_INFO)
+#define BIT_GET_CMDQ_PKT_INFO(x)                                               \
+	(((x) >> BIT_SHIFT_CMDQ_PKT_INFO) & BIT_MASK_CMDQ_PKT_INFO)
+
+/* 2 REG_CMDQ_BCNQ_INFO			(Offset 0x1414) */
+
+#define BIT_SHIFT_BCNQ_PKT_INFO 0
+#define BIT_MASK_BCNQ_PKT_INFO 0xfff
+#define BIT_BCNQ_PKT_INFO(x)                                                   \
+	(((x) & BIT_MASK_BCNQ_PKT_INFO) << BIT_SHIFT_BCNQ_PKT_INFO)
+#define BIT_GET_BCNQ_PKT_INFO(x)                                               \
+	(((x) >> BIT_SHIFT_BCNQ_PKT_INFO) & BIT_MASK_BCNQ_PKT_INFO)
+
+/* 2 REG_USEREG_SETTING			(Offset 0x1420) */
+
+#define BIT_NDPA_USEREG BIT(21)
+
+#define BIT_SHIFT_RETRY_USEREG 19
+#define BIT_MASK_RETRY_USEREG 0x3
+#define BIT_RETRY_USEREG(x)                                                    \
+	(((x) & BIT_MASK_RETRY_USEREG) << BIT_SHIFT_RETRY_USEREG)
+#define BIT_GET_RETRY_USEREG(x)                                                \
+	(((x) >> BIT_SHIFT_RETRY_USEREG) & BIT_MASK_RETRY_USEREG)
+
+#define BIT_SHIFT_TRYPKT_USEREG 17
+#define BIT_MASK_TRYPKT_USEREG 0x3
+#define BIT_TRYPKT_USEREG(x)                                                   \
+	(((x) & BIT_MASK_TRYPKT_USEREG) << BIT_SHIFT_TRYPKT_USEREG)
+#define BIT_GET_TRYPKT_USEREG(x)                                               \
+	(((x) >> BIT_SHIFT_TRYPKT_USEREG) & BIT_MASK_TRYPKT_USEREG)
+
+#define BIT_CTLPKT_USEREG BIT(16)
+
+/* 2 REG_AESIV_SETTING			(Offset 0x1424) */
+
+#define BIT_SHIFT_AESIV_OFFSET 0
+#define BIT_MASK_AESIV_OFFSET 0xfff
+#define BIT_AESIV_OFFSET(x)                                                    \
+	(((x) & BIT_MASK_AESIV_OFFSET) << BIT_SHIFT_AESIV_OFFSET)
+#define BIT_GET_AESIV_OFFSET(x)                                                \
+	(((x) >> BIT_SHIFT_AESIV_OFFSET) & BIT_MASK_AESIV_OFFSET)
+
+/* 2 REG_BF0_TIME_SETTING			(Offset 0x1428) */
+
+#define BIT_BF0_TIMER_SET BIT(31)
+#define BIT_BF0_TIMER_CLR BIT(30)
+#define BIT_BF0_UPDATE_EN BIT(29)
+#define BIT_BF0_TIMER_EN BIT(28)
+
+#define BIT_SHIFT_BF0_PRETIME_OVER 16
+#define BIT_MASK_BF0_PRETIME_OVER 0xfff
+#define BIT_BF0_PRETIME_OVER(x)                                                \
+	(((x) & BIT_MASK_BF0_PRETIME_OVER) << BIT_SHIFT_BF0_PRETIME_OVER)
+#define BIT_GET_BF0_PRETIME_OVER(x)                                            \
+	(((x) >> BIT_SHIFT_BF0_PRETIME_OVER) & BIT_MASK_BF0_PRETIME_OVER)
+
+#define BIT_SHIFT_BF0_LIFETIME 0
+#define BIT_MASK_BF0_LIFETIME 0xffff
+#define BIT_BF0_LIFETIME(x)                                                    \
+	(((x) & BIT_MASK_BF0_LIFETIME) << BIT_SHIFT_BF0_LIFETIME)
+#define BIT_GET_BF0_LIFETIME(x)                                                \
+	(((x) >> BIT_SHIFT_BF0_LIFETIME) & BIT_MASK_BF0_LIFETIME)
+
+/* 2 REG_BF1_TIME_SETTING			(Offset 0x142C) */
+
+#define BIT_BF1_TIMER_SET BIT(31)
+#define BIT_BF1_TIMER_CLR BIT(30)
+#define BIT_BF1_UPDATE_EN BIT(29)
+#define BIT_BF1_TIMER_EN BIT(28)
+
+#define BIT_SHIFT_BF1_PRETIME_OVER 16
+#define BIT_MASK_BF1_PRETIME_OVER 0xfff
+#define BIT_BF1_PRETIME_OVER(x)                                                \
+	(((x) & BIT_MASK_BF1_PRETIME_OVER) << BIT_SHIFT_BF1_PRETIME_OVER)
+#define BIT_GET_BF1_PRETIME_OVER(x)                                            \
+	(((x) >> BIT_SHIFT_BF1_PRETIME_OVER) & BIT_MASK_BF1_PRETIME_OVER)
+
+#define BIT_SHIFT_BF1_LIFETIME 0
+#define BIT_MASK_BF1_LIFETIME 0xffff
+#define BIT_BF1_LIFETIME(x)                                                    \
+	(((x) & BIT_MASK_BF1_LIFETIME) << BIT_SHIFT_BF1_LIFETIME)
+#define BIT_GET_BF1_LIFETIME(x)                                                \
+	(((x) >> BIT_SHIFT_BF1_LIFETIME) & BIT_MASK_BF1_LIFETIME)
+
+/* 2 REG_BF_TIMEOUT_EN			(Offset 0x1430) */
+
+#define BIT_EN_VHT_LDPC BIT(9)
+#define BIT_EN_HT_LDPC BIT(8)
+#define BIT_BF1_TIMEOUT_EN BIT(1)
+#define BIT_BF0_TIMEOUT_EN BIT(0)
+
+/* 2 REG_MACID_RELEASE0			(Offset 0x1434) */
+
+#define BIT_SHIFT_MACID31_0_RELEASE 0
+#define BIT_MASK_MACID31_0_RELEASE 0xffffffffL
+#define BIT_MACID31_0_RELEASE(x)                                               \
+	(((x) & BIT_MASK_MACID31_0_RELEASE) << BIT_SHIFT_MACID31_0_RELEASE)
+#define BIT_GET_MACID31_0_RELEASE(x)                                           \
+	(((x) >> BIT_SHIFT_MACID31_0_RELEASE) & BIT_MASK_MACID31_0_RELEASE)
+
+/* 2 REG_MACID_RELEASE1			(Offset 0x1438) */
+
+#define BIT_SHIFT_MACID63_32_RELEASE 0
+#define BIT_MASK_MACID63_32_RELEASE 0xffffffffL
+#define BIT_MACID63_32_RELEASE(x)                                              \
+	(((x) & BIT_MASK_MACID63_32_RELEASE) << BIT_SHIFT_MACID63_32_RELEASE)
+#define BIT_GET_MACID63_32_RELEASE(x)                                          \
+	(((x) >> BIT_SHIFT_MACID63_32_RELEASE) & BIT_MASK_MACID63_32_RELEASE)
+
+/* 2 REG_MACID_RELEASE2			(Offset 0x143C) */
+
+#define BIT_SHIFT_MACID95_64_RELEASE 0
+#define BIT_MASK_MACID95_64_RELEASE 0xffffffffL
+#define BIT_MACID95_64_RELEASE(x)                                              \
+	(((x) & BIT_MASK_MACID95_64_RELEASE) << BIT_SHIFT_MACID95_64_RELEASE)
+#define BIT_GET_MACID95_64_RELEASE(x)                                          \
+	(((x) >> BIT_SHIFT_MACID95_64_RELEASE) & BIT_MASK_MACID95_64_RELEASE)
+
+/* 2 REG_MACID_RELEASE3			(Offset 0x1440) */
+
+#define BIT_SHIFT_MACID127_96_RELEASE 0
+#define BIT_MASK_MACID127_96_RELEASE 0xffffffffL
+#define BIT_MACID127_96_RELEASE(x)                                             \
+	(((x) & BIT_MASK_MACID127_96_RELEASE) << BIT_SHIFT_MACID127_96_RELEASE)
+#define BIT_GET_MACID127_96_RELEASE(x)                                         \
+	(((x) >> BIT_SHIFT_MACID127_96_RELEASE) & BIT_MASK_MACID127_96_RELEASE)
+
+/* 2 REG_MACID_RELEASE_SETTING		(Offset 0x1444) */
+
+#define BIT_MACID_VALUE BIT(7)
+
+#define BIT_SHIFT_MACID_OFFSET 0
+#define BIT_MASK_MACID_OFFSET 0x7f
+#define BIT_MACID_OFFSET(x)                                                    \
+	(((x) & BIT_MASK_MACID_OFFSET) << BIT_SHIFT_MACID_OFFSET)
+#define BIT_GET_MACID_OFFSET(x)                                                \
+	(((x) >> BIT_SHIFT_MACID_OFFSET) & BIT_MASK_MACID_OFFSET)
+
+/* 2 REG_FAST_EDCA_VOVI_SETTING		(Offset 0x1448) */
+
+#define BIT_SHIFT_VI_FAST_EDCA_TO 24
+#define BIT_MASK_VI_FAST_EDCA_TO 0xff
+#define BIT_VI_FAST_EDCA_TO(x)                                                 \
+	(((x) & BIT_MASK_VI_FAST_EDCA_TO) << BIT_SHIFT_VI_FAST_EDCA_TO)
+#define BIT_GET_VI_FAST_EDCA_TO(x)                                             \
+	(((x) >> BIT_SHIFT_VI_FAST_EDCA_TO) & BIT_MASK_VI_FAST_EDCA_TO)
+
+#define BIT_VI_THRESHOLD_SEL BIT(23)
+
+#define BIT_SHIFT_VI_FAST_EDCA_PKT_TH 16
+#define BIT_MASK_VI_FAST_EDCA_PKT_TH 0x7f
+#define BIT_VI_FAST_EDCA_PKT_TH(x)                                             \
+	(((x) & BIT_MASK_VI_FAST_EDCA_PKT_TH) << BIT_SHIFT_VI_FAST_EDCA_PKT_TH)
+#define BIT_GET_VI_FAST_EDCA_PKT_TH(x)                                         \
+	(((x) >> BIT_SHIFT_VI_FAST_EDCA_PKT_TH) & BIT_MASK_VI_FAST_EDCA_PKT_TH)
+
+#define BIT_SHIFT_VO_FAST_EDCA_TO 8
+#define BIT_MASK_VO_FAST_EDCA_TO 0xff
+#define BIT_VO_FAST_EDCA_TO(x)                                                 \
+	(((x) & BIT_MASK_VO_FAST_EDCA_TO) << BIT_SHIFT_VO_FAST_EDCA_TO)
+#define BIT_GET_VO_FAST_EDCA_TO(x)                                             \
+	(((x) >> BIT_SHIFT_VO_FAST_EDCA_TO) & BIT_MASK_VO_FAST_EDCA_TO)
+
+#define BIT_VO_THRESHOLD_SEL BIT(7)
+
+#define BIT_SHIFT_VO_FAST_EDCA_PKT_TH 0
+#define BIT_MASK_VO_FAST_EDCA_PKT_TH 0x7f
+#define BIT_VO_FAST_EDCA_PKT_TH(x)                                             \
+	(((x) & BIT_MASK_VO_FAST_EDCA_PKT_TH) << BIT_SHIFT_VO_FAST_EDCA_PKT_TH)
+#define BIT_GET_VO_FAST_EDCA_PKT_TH(x)                                         \
+	(((x) >> BIT_SHIFT_VO_FAST_EDCA_PKT_TH) & BIT_MASK_VO_FAST_EDCA_PKT_TH)
+
+/* 2 REG_FAST_EDCA_BEBK_SETTING		(Offset 0x144C) */
+
+#define BIT_SHIFT_BK_FAST_EDCA_TO 24
+#define BIT_MASK_BK_FAST_EDCA_TO 0xff
+#define BIT_BK_FAST_EDCA_TO(x)                                                 \
+	(((x) & BIT_MASK_BK_FAST_EDCA_TO) << BIT_SHIFT_BK_FAST_EDCA_TO)
+#define BIT_GET_BK_FAST_EDCA_TO(x)                                             \
+	(((x) >> BIT_SHIFT_BK_FAST_EDCA_TO) & BIT_MASK_BK_FAST_EDCA_TO)
+
+#define BIT_BK_THRESHOLD_SEL BIT(23)
+
+#define BIT_SHIFT_BK_FAST_EDCA_PKT_TH 16
+#define BIT_MASK_BK_FAST_EDCA_PKT_TH 0x7f
+#define BIT_BK_FAST_EDCA_PKT_TH(x)                                             \
+	(((x) & BIT_MASK_BK_FAST_EDCA_PKT_TH) << BIT_SHIFT_BK_FAST_EDCA_PKT_TH)
+#define BIT_GET_BK_FAST_EDCA_PKT_TH(x)                                         \
+	(((x) >> BIT_SHIFT_BK_FAST_EDCA_PKT_TH) & BIT_MASK_BK_FAST_EDCA_PKT_TH)
+
+#define BIT_SHIFT_BE_FAST_EDCA_TO 8
+#define BIT_MASK_BE_FAST_EDCA_TO 0xff
+#define BIT_BE_FAST_EDCA_TO(x)                                                 \
+	(((x) & BIT_MASK_BE_FAST_EDCA_TO) << BIT_SHIFT_BE_FAST_EDCA_TO)
+#define BIT_GET_BE_FAST_EDCA_TO(x)                                             \
+	(((x) >> BIT_SHIFT_BE_FAST_EDCA_TO) & BIT_MASK_BE_FAST_EDCA_TO)
+
+#define BIT_BE_THRESHOLD_SEL BIT(7)
+
+#define BIT_SHIFT_BE_FAST_EDCA_PKT_TH 0
+#define BIT_MASK_BE_FAST_EDCA_PKT_TH 0x7f
+#define BIT_BE_FAST_EDCA_PKT_TH(x)                                             \
+	(((x) & BIT_MASK_BE_FAST_EDCA_PKT_TH) << BIT_SHIFT_BE_FAST_EDCA_PKT_TH)
+#define BIT_GET_BE_FAST_EDCA_PKT_TH(x)                                         \
+	(((x) >> BIT_SHIFT_BE_FAST_EDCA_PKT_TH) & BIT_MASK_BE_FAST_EDCA_PKT_TH)
+
+/* 2 REG_MACID_DROP0				(Offset 0x1450) */
+
+#define BIT_SHIFT_MACID31_0_DROP 0
+#define BIT_MASK_MACID31_0_DROP 0xffffffffL
+#define BIT_MACID31_0_DROP(x)                                                  \
+	(((x) & BIT_MASK_MACID31_0_DROP) << BIT_SHIFT_MACID31_0_DROP)
+#define BIT_GET_MACID31_0_DROP(x)                                              \
+	(((x) >> BIT_SHIFT_MACID31_0_DROP) & BIT_MASK_MACID31_0_DROP)
+
+/* 2 REG_MACID_DROP1				(Offset 0x1454) */
+
+#define BIT_SHIFT_MACID63_32_DROP 0
+#define BIT_MASK_MACID63_32_DROP 0xffffffffL
+#define BIT_MACID63_32_DROP(x)                                                 \
+	(((x) & BIT_MASK_MACID63_32_DROP) << BIT_SHIFT_MACID63_32_DROP)
+#define BIT_GET_MACID63_32_DROP(x)                                             \
+	(((x) >> BIT_SHIFT_MACID63_32_DROP) & BIT_MASK_MACID63_32_DROP)
+
+/* 2 REG_MACID_DROP2				(Offset 0x1458) */
+
+#define BIT_SHIFT_MACID95_64_DROP 0
+#define BIT_MASK_MACID95_64_DROP 0xffffffffL
+#define BIT_MACID95_64_DROP(x)                                                 \
+	(((x) & BIT_MASK_MACID95_64_DROP) << BIT_SHIFT_MACID95_64_DROP)
+#define BIT_GET_MACID95_64_DROP(x)                                             \
+	(((x) >> BIT_SHIFT_MACID95_64_DROP) & BIT_MASK_MACID95_64_DROP)
+
+/* 2 REG_MACID_DROP3				(Offset 0x145C) */
+
+#define BIT_SHIFT_MACID127_96_DROP 0
+#define BIT_MASK_MACID127_96_DROP 0xffffffffL
+#define BIT_MACID127_96_DROP(x)                                                \
+	(((x) & BIT_MASK_MACID127_96_DROP) << BIT_SHIFT_MACID127_96_DROP)
+#define BIT_GET_MACID127_96_DROP(x)                                            \
+	(((x) >> BIT_SHIFT_MACID127_96_DROP) & BIT_MASK_MACID127_96_DROP)
+
+/* 2 REG_R_MACID_RELEASE_SUCCESS_0		(Offset 0x1460) */
+
+#define BIT_SHIFT_R_MACID_RELEASE_SUCCESS_0 0
+#define BIT_MASK_R_MACID_RELEASE_SUCCESS_0 0xffffffffL
+#define BIT_R_MACID_RELEASE_SUCCESS_0(x)                                       \
+	(((x) & BIT_MASK_R_MACID_RELEASE_SUCCESS_0)                            \
+	 << BIT_SHIFT_R_MACID_RELEASE_SUCCESS_0)
+#define BIT_GET_R_MACID_RELEASE_SUCCESS_0(x)                                   \
+	(((x) >> BIT_SHIFT_R_MACID_RELEASE_SUCCESS_0) &                        \
+	 BIT_MASK_R_MACID_RELEASE_SUCCESS_0)
+
+/* 2 REG_R_MACID_RELEASE_SUCCESS_1		(Offset 0x1464) */
+
+#define BIT_SHIFT_R_MACID_RELEASE_SUCCESS_1 0
+#define BIT_MASK_R_MACID_RELEASE_SUCCESS_1 0xffffffffL
+#define BIT_R_MACID_RELEASE_SUCCESS_1(x)                                       \
+	(((x) & BIT_MASK_R_MACID_RELEASE_SUCCESS_1)                            \
+	 << BIT_SHIFT_R_MACID_RELEASE_SUCCESS_1)
+#define BIT_GET_R_MACID_RELEASE_SUCCESS_1(x)                                   \
+	(((x) >> BIT_SHIFT_R_MACID_RELEASE_SUCCESS_1) &                        \
+	 BIT_MASK_R_MACID_RELEASE_SUCCESS_1)
+
+/* 2 REG_R_MACID_RELEASE_SUCCESS_2		(Offset 0x1468) */
+
+#define BIT_SHIFT_R_MACID_RELEASE_SUCCESS_2 0
+#define BIT_MASK_R_MACID_RELEASE_SUCCESS_2 0xffffffffL
+#define BIT_R_MACID_RELEASE_SUCCESS_2(x)                                       \
+	(((x) & BIT_MASK_R_MACID_RELEASE_SUCCESS_2)                            \
+	 << BIT_SHIFT_R_MACID_RELEASE_SUCCESS_2)
+#define BIT_GET_R_MACID_RELEASE_SUCCESS_2(x)                                   \
+	(((x) >> BIT_SHIFT_R_MACID_RELEASE_SUCCESS_2) &                        \
+	 BIT_MASK_R_MACID_RELEASE_SUCCESS_2)
+
+/* 2 REG_R_MACID_RELEASE_SUCCESS_3		(Offset 0x146C) */
+
+#define BIT_SHIFT_R_MACID_RELEASE_SUCCESS_3 0
+#define BIT_MASK_R_MACID_RELEASE_SUCCESS_3 0xffffffffL
+#define BIT_R_MACID_RELEASE_SUCCESS_3(x)                                       \
+	(((x) & BIT_MASK_R_MACID_RELEASE_SUCCESS_3)                            \
+	 << BIT_SHIFT_R_MACID_RELEASE_SUCCESS_3)
+#define BIT_GET_R_MACID_RELEASE_SUCCESS_3(x)                                   \
+	(((x) >> BIT_SHIFT_R_MACID_RELEASE_SUCCESS_3) &                        \
+	 BIT_MASK_R_MACID_RELEASE_SUCCESS_3)
+
+/* 2 REG_MGG_FIFO_CRTL			(Offset 0x1470) */
+
+#define BIT_R_MGG_FIFO_EN BIT(31)
+
+#define BIT_SHIFT_R_MGG_FIFO_PG_SIZE 28
+#define BIT_MASK_R_MGG_FIFO_PG_SIZE 0x7
+#define BIT_R_MGG_FIFO_PG_SIZE(x)                                              \
+	(((x) & BIT_MASK_R_MGG_FIFO_PG_SIZE) << BIT_SHIFT_R_MGG_FIFO_PG_SIZE)
+#define BIT_GET_R_MGG_FIFO_PG_SIZE(x)                                          \
+	(((x) >> BIT_SHIFT_R_MGG_FIFO_PG_SIZE) & BIT_MASK_R_MGG_FIFO_PG_SIZE)
+
+#define BIT_SHIFT_R_MGG_FIFO_START_PG 16
+#define BIT_MASK_R_MGG_FIFO_START_PG 0xfff
+#define BIT_R_MGG_FIFO_START_PG(x)                                             \
+	(((x) & BIT_MASK_R_MGG_FIFO_START_PG) << BIT_SHIFT_R_MGG_FIFO_START_PG)
+#define BIT_GET_R_MGG_FIFO_START_PG(x)                                         \
+	(((x) >> BIT_SHIFT_R_MGG_FIFO_START_PG) & BIT_MASK_R_MGG_FIFO_START_PG)
+
+#define BIT_SHIFT_R_MGG_FIFO_SIZE 14
+#define BIT_MASK_R_MGG_FIFO_SIZE 0x3
+#define BIT_R_MGG_FIFO_SIZE(x)                                                 \
+	(((x) & BIT_MASK_R_MGG_FIFO_SIZE) << BIT_SHIFT_R_MGG_FIFO_SIZE)
+#define BIT_GET_R_MGG_FIFO_SIZE(x)                                             \
+	(((x) >> BIT_SHIFT_R_MGG_FIFO_SIZE) & BIT_MASK_R_MGG_FIFO_SIZE)
+
+#define BIT_R_MGG_FIFO_PAUSE BIT(13)
+
+#define BIT_SHIFT_R_MGG_FIFO_RPTR 8
+#define BIT_MASK_R_MGG_FIFO_RPTR 0x1f
+#define BIT_R_MGG_FIFO_RPTR(x)                                                 \
+	(((x) & BIT_MASK_R_MGG_FIFO_RPTR) << BIT_SHIFT_R_MGG_FIFO_RPTR)
+#define BIT_GET_R_MGG_FIFO_RPTR(x)                                             \
+	(((x) >> BIT_SHIFT_R_MGG_FIFO_RPTR) & BIT_MASK_R_MGG_FIFO_RPTR)
+
+#define BIT_R_MGG_FIFO_OV BIT(7)
+#define BIT_R_MGG_FIFO_WPTR_ERROR BIT(6)
+#define BIT_R_EN_CPU_LIFETIME BIT(5)
+
+#define BIT_SHIFT_R_MGG_FIFO_WPTR 0
+#define BIT_MASK_R_MGG_FIFO_WPTR 0x1f
+#define BIT_R_MGG_FIFO_WPTR(x)                                                 \
+	(((x) & BIT_MASK_R_MGG_FIFO_WPTR) << BIT_SHIFT_R_MGG_FIFO_WPTR)
+#define BIT_GET_R_MGG_FIFO_WPTR(x)                                             \
+	(((x) >> BIT_SHIFT_R_MGG_FIFO_WPTR) & BIT_MASK_R_MGG_FIFO_WPTR)
+
+/* 2 REG_MGG_FIFO_INT			(Offset 0x1474) */
+
+#define BIT_SHIFT_R_MGG_FIFO_INT_FLAG 16
+#define BIT_MASK_R_MGG_FIFO_INT_FLAG 0xffff
+#define BIT_R_MGG_FIFO_INT_FLAG(x)                                             \
+	(((x) & BIT_MASK_R_MGG_FIFO_INT_FLAG) << BIT_SHIFT_R_MGG_FIFO_INT_FLAG)
+#define BIT_GET_R_MGG_FIFO_INT_FLAG(x)                                         \
+	(((x) >> BIT_SHIFT_R_MGG_FIFO_INT_FLAG) & BIT_MASK_R_MGG_FIFO_INT_FLAG)
+
+#define BIT_SHIFT_R_MGG_FIFO_INT_MASK 0
+#define BIT_MASK_R_MGG_FIFO_INT_MASK 0xffff
+#define BIT_R_MGG_FIFO_INT_MASK(x)                                             \
+	(((x) & BIT_MASK_R_MGG_FIFO_INT_MASK) << BIT_SHIFT_R_MGG_FIFO_INT_MASK)
+#define BIT_GET_R_MGG_FIFO_INT_MASK(x)                                         \
+	(((x) >> BIT_SHIFT_R_MGG_FIFO_INT_MASK) & BIT_MASK_R_MGG_FIFO_INT_MASK)
+
+/* 2 REG_MGG_FIFO_LIFETIME			(Offset 0x1478) */
+
+#define BIT_SHIFT_R_MGG_FIFO_LIFETIME 16
+#define BIT_MASK_R_MGG_FIFO_LIFETIME 0xffff
+#define BIT_R_MGG_FIFO_LIFETIME(x)                                             \
+	(((x) & BIT_MASK_R_MGG_FIFO_LIFETIME) << BIT_SHIFT_R_MGG_FIFO_LIFETIME)
+#define BIT_GET_R_MGG_FIFO_LIFETIME(x)                                         \
+	(((x) >> BIT_SHIFT_R_MGG_FIFO_LIFETIME) & BIT_MASK_R_MGG_FIFO_LIFETIME)
+
+#define BIT_SHIFT_R_MGG_FIFO_VALID_MAP 0
+#define BIT_MASK_R_MGG_FIFO_VALID_MAP 0xffff
+#define BIT_R_MGG_FIFO_VALID_MAP(x)                                            \
+	(((x) & BIT_MASK_R_MGG_FIFO_VALID_MAP)                                 \
+	 << BIT_SHIFT_R_MGG_FIFO_VALID_MAP)
+#define BIT_GET_R_MGG_FIFO_VALID_MAP(x)                                        \
+	(((x) >> BIT_SHIFT_R_MGG_FIFO_VALID_MAP) &                             \
+	 BIT_MASK_R_MGG_FIFO_VALID_MAP)
+
+/* 2 REG_R_MACID_RELEASE_SUCCESS_CLEAR_OFFSET (Offset 0x147C) */
+
+#define BIT_SHIFT_R_MACID_RELEASE_SUCCESS_CLEAR_OFFSET 0
+#define BIT_MASK_R_MACID_RELEASE_SUCCESS_CLEAR_OFFSET 0x7f
+#define BIT_R_MACID_RELEASE_SUCCESS_CLEAR_OFFSET(x)                            \
+	(((x) & BIT_MASK_R_MACID_RELEASE_SUCCESS_CLEAR_OFFSET)                 \
+	 << BIT_SHIFT_R_MACID_RELEASE_SUCCESS_CLEAR_OFFSET)
+#define BIT_GET_R_MACID_RELEASE_SUCCESS_CLEAR_OFFSET(x)                        \
+	(((x) >> BIT_SHIFT_R_MACID_RELEASE_SUCCESS_CLEAR_OFFSET) &             \
+	 BIT_MASK_R_MACID_RELEASE_SUCCESS_CLEAR_OFFSET)
+
+#define BIT_SHIFT_P2PON_DIS_TXTIME 0
+#define BIT_MASK_P2PON_DIS_TXTIME 0xff
+#define BIT_P2PON_DIS_TXTIME(x)                                                \
+	(((x) & BIT_MASK_P2PON_DIS_TXTIME) << BIT_SHIFT_P2PON_DIS_TXTIME)
+#define BIT_GET_P2PON_DIS_TXTIME(x)                                            \
+	(((x) >> BIT_SHIFT_P2PON_DIS_TXTIME) & BIT_MASK_P2PON_DIS_TXTIME)
+
+/* 2 REG_MACID_SHCUT_OFFSET			(Offset 0x1480) */
+
+#define BIT_SHIFT_MACID_SHCUT_OFFSET_V1 0
+#define BIT_MASK_MACID_SHCUT_OFFSET_V1 0xff
+#define BIT_MACID_SHCUT_OFFSET_V1(x)                                           \
+	(((x) & BIT_MASK_MACID_SHCUT_OFFSET_V1)                                \
+	 << BIT_SHIFT_MACID_SHCUT_OFFSET_V1)
+#define BIT_GET_MACID_SHCUT_OFFSET_V1(x)                                       \
+	(((x) >> BIT_SHIFT_MACID_SHCUT_OFFSET_V1) &                            \
+	 BIT_MASK_MACID_SHCUT_OFFSET_V1)
+
+/* 2 REG_MU_TX_CTL				(Offset 0x14C0) */
+
+#define BIT_R_EN_REVERS_GTAB BIT(6)
+
+#define BIT_SHIFT_R_MU_TABLE_VALID 0
+#define BIT_MASK_R_MU_TABLE_VALID 0x3f
+#define BIT_R_MU_TABLE_VALID(x)                                                \
+	(((x) & BIT_MASK_R_MU_TABLE_VALID) << BIT_SHIFT_R_MU_TABLE_VALID)
+#define BIT_GET_R_MU_TABLE_VALID(x)                                            \
+	(((x) >> BIT_SHIFT_R_MU_TABLE_VALID) & BIT_MASK_R_MU_TABLE_VALID)
+
+#define BIT_SHIFT_R_MU_STA_GTAB_VALID 0
+#define BIT_MASK_R_MU_STA_GTAB_VALID 0xffffffffL
+#define BIT_R_MU_STA_GTAB_VALID(x)                                             \
+	(((x) & BIT_MASK_R_MU_STA_GTAB_VALID) << BIT_SHIFT_R_MU_STA_GTAB_VALID)
+#define BIT_GET_R_MU_STA_GTAB_VALID(x)                                         \
+	(((x) >> BIT_SHIFT_R_MU_STA_GTAB_VALID) & BIT_MASK_R_MU_STA_GTAB_VALID)
+
+#define BIT_SHIFT_R_MU_STA_GTAB_POSITION 0
+#define BIT_MASK_R_MU_STA_GTAB_POSITION 0xffffffffffffffffL
+#define BIT_R_MU_STA_GTAB_POSITION(x)                                          \
+	(((x) & BIT_MASK_R_MU_STA_GTAB_POSITION)                               \
+	 << BIT_SHIFT_R_MU_STA_GTAB_POSITION)
+#define BIT_GET_R_MU_STA_GTAB_POSITION(x)                                      \
+	(((x) >> BIT_SHIFT_R_MU_STA_GTAB_POSITION) &                           \
+	 BIT_MASK_R_MU_STA_GTAB_POSITION)
+
+/* 2 REG_MU_TRX_DBG_CNT			(Offset 0x14D0) */
+
+#define BIT_MU_DNGCNT_RST BIT(20)
+
+#define BIT_SHIFT_MU_DBGCNT_SEL 16
+#define BIT_MASK_MU_DBGCNT_SEL 0xf
+#define BIT_MU_DBGCNT_SEL(x)                                                   \
+	(((x) & BIT_MASK_MU_DBGCNT_SEL) << BIT_SHIFT_MU_DBGCNT_SEL)
+#define BIT_GET_MU_DBGCNT_SEL(x)                                               \
+	(((x) >> BIT_SHIFT_MU_DBGCNT_SEL) & BIT_MASK_MU_DBGCNT_SEL)
+
+#define BIT_SHIFT_MU_DNGCNT 0
+#define BIT_MASK_MU_DNGCNT 0xffff
+#define BIT_MU_DNGCNT(x) (((x) & BIT_MASK_MU_DNGCNT) << BIT_SHIFT_MU_DNGCNT)
+#define BIT_GET_MU_DNGCNT(x) (((x) >> BIT_SHIFT_MU_DNGCNT) & BIT_MASK_MU_DNGCNT)
+
+/* 2 REG_CPUMGQ_TX_TIMER			(Offset 0x1500) */
+
+#define BIT_SHIFT_CPUMGQ_TX_TIMER_V1 0
+#define BIT_MASK_CPUMGQ_TX_TIMER_V1 0xffffffffL
+#define BIT_CPUMGQ_TX_TIMER_V1(x)                                              \
+	(((x) & BIT_MASK_CPUMGQ_TX_TIMER_V1) << BIT_SHIFT_CPUMGQ_TX_TIMER_V1)
+#define BIT_GET_CPUMGQ_TX_TIMER_V1(x)                                          \
+	(((x) >> BIT_SHIFT_CPUMGQ_TX_TIMER_V1) & BIT_MASK_CPUMGQ_TX_TIMER_V1)
+
+/* 2 REG_PS_TIMER_A				(Offset 0x1504) */
+
+#define BIT_SHIFT_PS_TIMER_A_V1 0
+#define BIT_MASK_PS_TIMER_A_V1 0xffffffffL
+#define BIT_PS_TIMER_A_V1(x)                                                   \
+	(((x) & BIT_MASK_PS_TIMER_A_V1) << BIT_SHIFT_PS_TIMER_A_V1)
+#define BIT_GET_PS_TIMER_A_V1(x)                                               \
+	(((x) >> BIT_SHIFT_PS_TIMER_A_V1) & BIT_MASK_PS_TIMER_A_V1)
+
+/* 2 REG_PS_TIMER_B				(Offset 0x1508) */
+
+#define BIT_SHIFT_PS_TIMER_B_V1 0
+#define BIT_MASK_PS_TIMER_B_V1 0xffffffffL
+#define BIT_PS_TIMER_B_V1(x)                                                   \
+	(((x) & BIT_MASK_PS_TIMER_B_V1) << BIT_SHIFT_PS_TIMER_B_V1)
+#define BIT_GET_PS_TIMER_B_V1(x)                                               \
+	(((x) >> BIT_SHIFT_PS_TIMER_B_V1) & BIT_MASK_PS_TIMER_B_V1)
+
+/* 2 REG_PS_TIMER_C				(Offset 0x150C) */
+
+#define BIT_SHIFT_PS_TIMER_C_V1 0
+#define BIT_MASK_PS_TIMER_C_V1 0xffffffffL
+#define BIT_PS_TIMER_C_V1(x)                                                   \
+	(((x) & BIT_MASK_PS_TIMER_C_V1) << BIT_SHIFT_PS_TIMER_C_V1)
+#define BIT_GET_PS_TIMER_C_V1(x)                                               \
+	(((x) >> BIT_SHIFT_PS_TIMER_C_V1) & BIT_MASK_PS_TIMER_C_V1)
+
+/* 2 REG_PS_TIMER_ABC_CPUMGQ_TIMER_CRTL	(Offset 0x1510) */
+
+#define BIT_CPUMGQ_TIMER_EN BIT(31)
+#define BIT_CPUMGQ_TX_EN BIT(28)
+
+#define BIT_SHIFT_CPUMGQ_TIMER_TSF_SEL 24
+#define BIT_MASK_CPUMGQ_TIMER_TSF_SEL 0x7
+#define BIT_CPUMGQ_TIMER_TSF_SEL(x)                                            \
+	(((x) & BIT_MASK_CPUMGQ_TIMER_TSF_SEL)                                 \
+	 << BIT_SHIFT_CPUMGQ_TIMER_TSF_SEL)
+#define BIT_GET_CPUMGQ_TIMER_TSF_SEL(x)                                        \
+	(((x) >> BIT_SHIFT_CPUMGQ_TIMER_TSF_SEL) &                             \
+	 BIT_MASK_CPUMGQ_TIMER_TSF_SEL)
+
+#define BIT_PS_TIMER_C_EN BIT(23)
+
+#define BIT_SHIFT_PS_TIMER_C_TSF_SEL 16
+#define BIT_MASK_PS_TIMER_C_TSF_SEL 0x7
+#define BIT_PS_TIMER_C_TSF_SEL(x)                                              \
+	(((x) & BIT_MASK_PS_TIMER_C_TSF_SEL) << BIT_SHIFT_PS_TIMER_C_TSF_SEL)
+#define BIT_GET_PS_TIMER_C_TSF_SEL(x)                                          \
+	(((x) >> BIT_SHIFT_PS_TIMER_C_TSF_SEL) & BIT_MASK_PS_TIMER_C_TSF_SEL)
+
+#define BIT_PS_TIMER_B_EN BIT(15)
+
+#define BIT_SHIFT_PS_TIMER_B_TSF_SEL 8
+#define BIT_MASK_PS_TIMER_B_TSF_SEL 0x7
+#define BIT_PS_TIMER_B_TSF_SEL(x)                                              \
+	(((x) & BIT_MASK_PS_TIMER_B_TSF_SEL) << BIT_SHIFT_PS_TIMER_B_TSF_SEL)
+#define BIT_GET_PS_TIMER_B_TSF_SEL(x)                                          \
+	(((x) >> BIT_SHIFT_PS_TIMER_B_TSF_SEL) & BIT_MASK_PS_TIMER_B_TSF_SEL)
+
+#define BIT_PS_TIMER_A_EN BIT(7)
+
+#define BIT_SHIFT_PS_TIMER_A_TSF_SEL 0
+#define BIT_MASK_PS_TIMER_A_TSF_SEL 0x7
+#define BIT_PS_TIMER_A_TSF_SEL(x)                                              \
+	(((x) & BIT_MASK_PS_TIMER_A_TSF_SEL) << BIT_SHIFT_PS_TIMER_A_TSF_SEL)
+#define BIT_GET_PS_TIMER_A_TSF_SEL(x)                                          \
+	(((x) >> BIT_SHIFT_PS_TIMER_A_TSF_SEL) & BIT_MASK_PS_TIMER_A_TSF_SEL)
+
+/* 2 REG_CPUMGQ_TX_TIMER_EARLY		(Offset 0x1514) */
+
+#define BIT_SHIFT_CPUMGQ_TX_TIMER_EARLY 0
+#define BIT_MASK_CPUMGQ_TX_TIMER_EARLY 0xff
+#define BIT_CPUMGQ_TX_TIMER_EARLY(x)                                           \
+	(((x) & BIT_MASK_CPUMGQ_TX_TIMER_EARLY)                                \
+	 << BIT_SHIFT_CPUMGQ_TX_TIMER_EARLY)
+#define BIT_GET_CPUMGQ_TX_TIMER_EARLY(x)                                       \
+	(((x) >> BIT_SHIFT_CPUMGQ_TX_TIMER_EARLY) &                            \
+	 BIT_MASK_CPUMGQ_TX_TIMER_EARLY)
+
+/* 2 REG_PS_TIMER_A_EARLY			(Offset 0x1515) */
+
+#define BIT_SHIFT_PS_TIMER_A_EARLY 0
+#define BIT_MASK_PS_TIMER_A_EARLY 0xff
+#define BIT_PS_TIMER_A_EARLY(x)                                                \
+	(((x) & BIT_MASK_PS_TIMER_A_EARLY) << BIT_SHIFT_PS_TIMER_A_EARLY)
+#define BIT_GET_PS_TIMER_A_EARLY(x)                                            \
+	(((x) >> BIT_SHIFT_PS_TIMER_A_EARLY) & BIT_MASK_PS_TIMER_A_EARLY)
+
+/* 2 REG_PS_TIMER_B_EARLY			(Offset 0x1516) */
+
+#define BIT_SHIFT_PS_TIMER_B_EARLY 0
+#define BIT_MASK_PS_TIMER_B_EARLY 0xff
+#define BIT_PS_TIMER_B_EARLY(x)                                                \
+	(((x) & BIT_MASK_PS_TIMER_B_EARLY) << BIT_SHIFT_PS_TIMER_B_EARLY)
+#define BIT_GET_PS_TIMER_B_EARLY(x)                                            \
+	(((x) >> BIT_SHIFT_PS_TIMER_B_EARLY) & BIT_MASK_PS_TIMER_B_EARLY)
+
+/* 2 REG_PS_TIMER_C_EARLY			(Offset 0x1517) */
+
+#define BIT_SHIFT_PS_TIMER_C_EARLY 0
+#define BIT_MASK_PS_TIMER_C_EARLY 0xff
+#define BIT_PS_TIMER_C_EARLY(x)                                                \
+	(((x) & BIT_MASK_PS_TIMER_C_EARLY) << BIT_SHIFT_PS_TIMER_C_EARLY)
+#define BIT_GET_PS_TIMER_C_EARLY(x)                                            \
+	(((x) >> BIT_SHIFT_PS_TIMER_C_EARLY) & BIT_MASK_PS_TIMER_C_EARLY)
+
+/* 2 REG_BCN_PSR_RPT2			(Offset 0x1600) */
+
+#define BIT_SHIFT_DTIM_CNT2 24
+#define BIT_MASK_DTIM_CNT2 0xff
+#define BIT_DTIM_CNT2(x) (((x) & BIT_MASK_DTIM_CNT2) << BIT_SHIFT_DTIM_CNT2)
+#define BIT_GET_DTIM_CNT2(x) (((x) >> BIT_SHIFT_DTIM_CNT2) & BIT_MASK_DTIM_CNT2)
+
+#define BIT_SHIFT_DTIM_PERIOD2 16
+#define BIT_MASK_DTIM_PERIOD2 0xff
+#define BIT_DTIM_PERIOD2(x)                                                    \
+	(((x) & BIT_MASK_DTIM_PERIOD2) << BIT_SHIFT_DTIM_PERIOD2)
+#define BIT_GET_DTIM_PERIOD2(x)                                                \
+	(((x) >> BIT_SHIFT_DTIM_PERIOD2) & BIT_MASK_DTIM_PERIOD2)
+
+#define BIT_DTIM2 BIT(15)
+#define BIT_TIM2 BIT(14)
+
+#define BIT_SHIFT_PS_AID_2 0
+#define BIT_MASK_PS_AID_2 0x7ff
+#define BIT_PS_AID_2(x) (((x) & BIT_MASK_PS_AID_2) << BIT_SHIFT_PS_AID_2)
+#define BIT_GET_PS_AID_2(x) (((x) >> BIT_SHIFT_PS_AID_2) & BIT_MASK_PS_AID_2)
+
+/* 2 REG_BCN_PSR_RPT3			(Offset 0x1604) */
+
+#define BIT_SHIFT_DTIM_CNT3 24
+#define BIT_MASK_DTIM_CNT3 0xff
+#define BIT_DTIM_CNT3(x) (((x) & BIT_MASK_DTIM_CNT3) << BIT_SHIFT_DTIM_CNT3)
+#define BIT_GET_DTIM_CNT3(x) (((x) >> BIT_SHIFT_DTIM_CNT3) & BIT_MASK_DTIM_CNT3)
+
+#define BIT_SHIFT_DTIM_PERIOD3 16
+#define BIT_MASK_DTIM_PERIOD3 0xff
+#define BIT_DTIM_PERIOD3(x)                                                    \
+	(((x) & BIT_MASK_DTIM_PERIOD3) << BIT_SHIFT_DTIM_PERIOD3)
+#define BIT_GET_DTIM_PERIOD3(x)                                                \
+	(((x) >> BIT_SHIFT_DTIM_PERIOD3) & BIT_MASK_DTIM_PERIOD3)
+
+#define BIT_DTIM3 BIT(15)
+#define BIT_TIM3 BIT(14)
+
+#define BIT_SHIFT_PS_AID_3 0
+#define BIT_MASK_PS_AID_3 0x7ff
+#define BIT_PS_AID_3(x) (((x) & BIT_MASK_PS_AID_3) << BIT_SHIFT_PS_AID_3)
+#define BIT_GET_PS_AID_3(x) (((x) >> BIT_SHIFT_PS_AID_3) & BIT_MASK_PS_AID_3)
+
+/* 2 REG_BCN_PSR_RPT4			(Offset 0x1608) */
+
+#define BIT_SHIFT_DTIM_CNT4 24
+#define BIT_MASK_DTIM_CNT4 0xff
+#define BIT_DTIM_CNT4(x) (((x) & BIT_MASK_DTIM_CNT4) << BIT_SHIFT_DTIM_CNT4)
+#define BIT_GET_DTIM_CNT4(x) (((x) >> BIT_SHIFT_DTIM_CNT4) & BIT_MASK_DTIM_CNT4)
+
+#define BIT_SHIFT_DTIM_PERIOD4 16
+#define BIT_MASK_DTIM_PERIOD4 0xff
+#define BIT_DTIM_PERIOD4(x)                                                    \
+	(((x) & BIT_MASK_DTIM_PERIOD4) << BIT_SHIFT_DTIM_PERIOD4)
+#define BIT_GET_DTIM_PERIOD4(x)                                                \
+	(((x) >> BIT_SHIFT_DTIM_PERIOD4) & BIT_MASK_DTIM_PERIOD4)
+
+#define BIT_DTIM4 BIT(15)
+#define BIT_TIM4 BIT(14)
+
+#define BIT_SHIFT_PS_AID_4 0
+#define BIT_MASK_PS_AID_4 0x7ff
+#define BIT_PS_AID_4(x) (((x) & BIT_MASK_PS_AID_4) << BIT_SHIFT_PS_AID_4)
+#define BIT_GET_PS_AID_4(x) (((x) >> BIT_SHIFT_PS_AID_4) & BIT_MASK_PS_AID_4)
+
+/* 2 REG_A1_ADDR_MASK			(Offset 0x160C) */
+
+#define BIT_SHIFT_A1_ADDR_MASK 0
+#define BIT_MASK_A1_ADDR_MASK 0xffffffffL
+#define BIT_A1_ADDR_MASK(x)                                                    \
+	(((x) & BIT_MASK_A1_ADDR_MASK) << BIT_SHIFT_A1_ADDR_MASK)
+#define BIT_GET_A1_ADDR_MASK(x)                                                \
+	(((x) >> BIT_SHIFT_A1_ADDR_MASK) & BIT_MASK_A1_ADDR_MASK)
+
+/* 2 REG_MACID2				(Offset 0x1620) */
+
+#define BIT_SHIFT_MACID2 0
+#define BIT_MASK_MACID2 0xffffffffffffL
+#define BIT_MACID2(x) (((x) & BIT_MASK_MACID2) << BIT_SHIFT_MACID2)
+#define BIT_GET_MACID2(x) (((x) >> BIT_SHIFT_MACID2) & BIT_MASK_MACID2)
+
+/* 2 REG_BSSID2				(Offset 0x1628) */
+
+#define BIT_SHIFT_BSSID2 0
+#define BIT_MASK_BSSID2 0xffffffffffffL
+#define BIT_BSSID2(x) (((x) & BIT_MASK_BSSID2) << BIT_SHIFT_BSSID2)
+#define BIT_GET_BSSID2(x) (((x) >> BIT_SHIFT_BSSID2) & BIT_MASK_BSSID2)
+
+/* 2 REG_MACID3				(Offset 0x1630) */
+
+#define BIT_SHIFT_MACID3 0
+#define BIT_MASK_MACID3 0xffffffffffffL
+#define BIT_MACID3(x) (((x) & BIT_MASK_MACID3) << BIT_SHIFT_MACID3)
+#define BIT_GET_MACID3(x) (((x) >> BIT_SHIFT_MACID3) & BIT_MASK_MACID3)
+
+/* 2 REG_BSSID3				(Offset 0x1638) */
+
+#define BIT_SHIFT_BSSID3 0
+#define BIT_MASK_BSSID3 0xffffffffffffL
+#define BIT_BSSID3(x) (((x) & BIT_MASK_BSSID3) << BIT_SHIFT_BSSID3)
+#define BIT_GET_BSSID3(x) (((x) >> BIT_SHIFT_BSSID3) & BIT_MASK_BSSID3)
+
+/* 2 REG_MACID4				(Offset 0x1640) */
+
+#define BIT_SHIFT_MACID4 0
+#define BIT_MASK_MACID4 0xffffffffffffL
+#define BIT_MACID4(x) (((x) & BIT_MASK_MACID4) << BIT_SHIFT_MACID4)
+#define BIT_GET_MACID4(x) (((x) >> BIT_SHIFT_MACID4) & BIT_MASK_MACID4)
+
+/* 2 REG_BSSID4				(Offset 0x1648) */
+
+#define BIT_SHIFT_BSSID4 0
+#define BIT_MASK_BSSID4 0xffffffffffffL
+#define BIT_BSSID4(x) (((x) & BIT_MASK_BSSID4) << BIT_SHIFT_BSSID4)
+#define BIT_GET_BSSID4(x) (((x) >> BIT_SHIFT_BSSID4) & BIT_MASK_BSSID4)
+
+/* 2 REG_PWRBIT_SETTING			(Offset 0x1660) */
+
+#define BIT_CLI3_PWRBIT_OW_EN BIT(7)
+#define BIT_CLI3_PWR_ST BIT(6)
+#define BIT_CLI2_PWRBIT_OW_EN BIT(5)
+#define BIT_CLI2_PWR_ST BIT(4)
+#define BIT_CLI1_PWRBIT_OW_EN BIT(3)
+#define BIT_CLI1_PWR_ST BIT(2)
+#define BIT_CLI0_PWRBIT_OW_EN BIT(1)
+#define BIT_CLI0_PWR_ST BIT(0)
+
+/* 2 REG_WMAC_MU_BF_OPTION			(Offset 0x167C) */
+
+#define BIT_WMAC_RESP_NONSTA1_DIS BIT(7)
+
+/* 2 REG_WMAC_MU_BF_OPTION			(Offset 0x167C) */
+
+#define BIT_BIT_WMAC_TXMU_ACKPOLICY_EN BIT(6)
+
+/* 2 REG_WMAC_MU_BF_OPTION			(Offset 0x167C) */
+
+#define BIT_SHIFT_WMAC_TXMU_ACKPOLICY 4
+#define BIT_MASK_WMAC_TXMU_ACKPOLICY 0x3
+#define BIT_WMAC_TXMU_ACKPOLICY(x)                                             \
+	(((x) & BIT_MASK_WMAC_TXMU_ACKPOLICY) << BIT_SHIFT_WMAC_TXMU_ACKPOLICY)
+#define BIT_GET_WMAC_TXMU_ACKPOLICY(x)                                         \
+	(((x) >> BIT_SHIFT_WMAC_TXMU_ACKPOLICY) & BIT_MASK_WMAC_TXMU_ACKPOLICY)
+
+#define BIT_SHIFT_WMAC_MU_BFEE_PORT_SEL 1
+#define BIT_MASK_WMAC_MU_BFEE_PORT_SEL 0x7
+#define BIT_WMAC_MU_BFEE_PORT_SEL(x)                                           \
+	(((x) & BIT_MASK_WMAC_MU_BFEE_PORT_SEL)                                \
+	 << BIT_SHIFT_WMAC_MU_BFEE_PORT_SEL)
+#define BIT_GET_WMAC_MU_BFEE_PORT_SEL(x)                                       \
+	(((x) >> BIT_SHIFT_WMAC_MU_BFEE_PORT_SEL) &                            \
+	 BIT_MASK_WMAC_MU_BFEE_PORT_SEL)
+
+#define BIT_WMAC_MU_BFEE_DIS BIT(0)
+
+/* 2 REG_WMAC_PAUSE_BB_CLR_TH		(Offset 0x167D) */
+
+#define BIT_SHIFT_WMAC_PAUSE_BB_CLR_TH 0
+#define BIT_MASK_WMAC_PAUSE_BB_CLR_TH 0xff
+#define BIT_WMAC_PAUSE_BB_CLR_TH(x)                                            \
+	(((x) & BIT_MASK_WMAC_PAUSE_BB_CLR_TH)                                 \
+	 << BIT_SHIFT_WMAC_PAUSE_BB_CLR_TH)
+#define BIT_GET_WMAC_PAUSE_BB_CLR_TH(x)                                        \
+	(((x) >> BIT_SHIFT_WMAC_PAUSE_BB_CLR_TH) &                             \
+	 BIT_MASK_WMAC_PAUSE_BB_CLR_TH)
+
+/* 2 REG_WMAC_MU_ARB				(Offset 0x167E) */
+
+#define BIT_WMAC_ARB_HW_ADAPT_EN BIT(7)
+#define BIT_WMAC_ARB_SW_EN BIT(6)
+
+#define BIT_SHIFT_WMAC_ARB_SW_STATE 0
+#define BIT_MASK_WMAC_ARB_SW_STATE 0x3f
+#define BIT_WMAC_ARB_SW_STATE(x)                                               \
+	(((x) & BIT_MASK_WMAC_ARB_SW_STATE) << BIT_SHIFT_WMAC_ARB_SW_STATE)
+#define BIT_GET_WMAC_ARB_SW_STATE(x)                                           \
+	(((x) >> BIT_SHIFT_WMAC_ARB_SW_STATE) & BIT_MASK_WMAC_ARB_SW_STATE)
+
+/* 2 REG_WMAC_MU_OPTION			(Offset 0x167F) */
+
+#define BIT_SHIFT_WMAC_MU_DBGSEL 5
+#define BIT_MASK_WMAC_MU_DBGSEL 0x3
+#define BIT_WMAC_MU_DBGSEL(x)                                                  \
+	(((x) & BIT_MASK_WMAC_MU_DBGSEL) << BIT_SHIFT_WMAC_MU_DBGSEL)
+#define BIT_GET_WMAC_MU_DBGSEL(x)                                              \
+	(((x) >> BIT_SHIFT_WMAC_MU_DBGSEL) & BIT_MASK_WMAC_MU_DBGSEL)
+
+#define BIT_SHIFT_WMAC_MU_CPRD_TIMEOUT 0
+#define BIT_MASK_WMAC_MU_CPRD_TIMEOUT 0x1f
+#define BIT_WMAC_MU_CPRD_TIMEOUT(x)                                            \
+	(((x) & BIT_MASK_WMAC_MU_CPRD_TIMEOUT)                                 \
+	 << BIT_SHIFT_WMAC_MU_CPRD_TIMEOUT)
+#define BIT_GET_WMAC_MU_CPRD_TIMEOUT(x)                                        \
+	(((x) >> BIT_SHIFT_WMAC_MU_CPRD_TIMEOUT) &                             \
+	 BIT_MASK_WMAC_MU_CPRD_TIMEOUT)
+
+/* 2 REG_WMAC_MU_BF_CTL			(Offset 0x1680) */
+
+#define BIT_WMAC_INVLD_BFPRT_CHK BIT(15)
+#define BIT_WMAC_RETXBFRPTSEQ_UPD BIT(14)
+
+#define BIT_SHIFT_WMAC_MU_BFRPTSEG_SEL 12
+#define BIT_MASK_WMAC_MU_BFRPTSEG_SEL 0x3
+#define BIT_WMAC_MU_BFRPTSEG_SEL(x)                                            \
+	(((x) & BIT_MASK_WMAC_MU_BFRPTSEG_SEL)                                 \
+	 << BIT_SHIFT_WMAC_MU_BFRPTSEG_SEL)
+#define BIT_GET_WMAC_MU_BFRPTSEG_SEL(x)                                        \
+	(((x) >> BIT_SHIFT_WMAC_MU_BFRPTSEG_SEL) &                             \
+	 BIT_MASK_WMAC_MU_BFRPTSEG_SEL)
+
+#define BIT_SHIFT_WMAC_MU_BF_MYAID 0
+#define BIT_MASK_WMAC_MU_BF_MYAID 0xfff
+#define BIT_WMAC_MU_BF_MYAID(x)                                                \
+	(((x) & BIT_MASK_WMAC_MU_BF_MYAID) << BIT_SHIFT_WMAC_MU_BF_MYAID)
+#define BIT_GET_WMAC_MU_BF_MYAID(x)                                            \
+	(((x) >> BIT_SHIFT_WMAC_MU_BF_MYAID) & BIT_MASK_WMAC_MU_BF_MYAID)
+
+#define BIT_SHIFT_BFRPT_PARA 0
+#define BIT_MASK_BFRPT_PARA 0xfff
+#define BIT_BFRPT_PARA(x) (((x) & BIT_MASK_BFRPT_PARA) << BIT_SHIFT_BFRPT_PARA)
+#define BIT_GET_BFRPT_PARA(x)                                                  \
+	(((x) >> BIT_SHIFT_BFRPT_PARA) & BIT_MASK_BFRPT_PARA)
+
+/* 2 REG_WMAC_MU_BFRPT_PARA			(Offset 0x1682) */
+
+#define BIT_SHIFT_BIT_BFRPT_PARA_USERID_SEL 12
+#define BIT_MASK_BIT_BFRPT_PARA_USERID_SEL 0x7
+#define BIT_BIT_BFRPT_PARA_USERID_SEL(x)                                       \
+	(((x) & BIT_MASK_BIT_BFRPT_PARA_USERID_SEL)                            \
+	 << BIT_SHIFT_BIT_BFRPT_PARA_USERID_SEL)
+#define BIT_GET_BIT_BFRPT_PARA_USERID_SEL(x)                                   \
+	(((x) >> BIT_SHIFT_BIT_BFRPT_PARA_USERID_SEL) &                        \
+	 BIT_MASK_BIT_BFRPT_PARA_USERID_SEL)
+
+/* 2 REG_WMAC_ASSOCIATED_MU_BFMEE2		(Offset 0x1684) */
+
+#define BIT_STATUS_BFEE2 BIT(10)
+#define BIT_WMAC_MU_BFEE2_EN BIT(9)
+
+#define BIT_SHIFT_WMAC_MU_BFEE2_AID 0
+#define BIT_MASK_WMAC_MU_BFEE2_AID 0x1ff
+#define BIT_WMAC_MU_BFEE2_AID(x)                                               \
+	(((x) & BIT_MASK_WMAC_MU_BFEE2_AID) << BIT_SHIFT_WMAC_MU_BFEE2_AID)
+#define BIT_GET_WMAC_MU_BFEE2_AID(x)                                           \
+	(((x) >> BIT_SHIFT_WMAC_MU_BFEE2_AID) & BIT_MASK_WMAC_MU_BFEE2_AID)
+
+/* 2 REG_WMAC_ASSOCIATED_MU_BFMEE3		(Offset 0x1686) */
+
+#define BIT_STATUS_BFEE3 BIT(10)
+#define BIT_WMAC_MU_BFEE3_EN BIT(9)
+
+#define BIT_SHIFT_WMAC_MU_BFEE3_AID 0
+#define BIT_MASK_WMAC_MU_BFEE3_AID 0x1ff
+#define BIT_WMAC_MU_BFEE3_AID(x)                                               \
+	(((x) & BIT_MASK_WMAC_MU_BFEE3_AID) << BIT_SHIFT_WMAC_MU_BFEE3_AID)
+#define BIT_GET_WMAC_MU_BFEE3_AID(x)                                           \
+	(((x) >> BIT_SHIFT_WMAC_MU_BFEE3_AID) & BIT_MASK_WMAC_MU_BFEE3_AID)
+
+/* 2 REG_WMAC_ASSOCIATED_MU_BFMEE4		(Offset 0x1688) */
+
+#define BIT_STATUS_BFEE4 BIT(10)
+#define BIT_WMAC_MU_BFEE4_EN BIT(9)
+
+#define BIT_SHIFT_WMAC_MU_BFEE4_AID 0
+#define BIT_MASK_WMAC_MU_BFEE4_AID 0x1ff
+#define BIT_WMAC_MU_BFEE4_AID(x)                                               \
+	(((x) & BIT_MASK_WMAC_MU_BFEE4_AID) << BIT_SHIFT_WMAC_MU_BFEE4_AID)
+#define BIT_GET_WMAC_MU_BFEE4_AID(x)                                           \
+	(((x) >> BIT_SHIFT_WMAC_MU_BFEE4_AID) & BIT_MASK_WMAC_MU_BFEE4_AID)
+
+/* 2 REG_WMAC_ASSOCIATED_MU_BFMEE5		(Offset 0x168A) */
+
+#define BIT_R_WMAC_RX_SYNCFIFO_SYNC BIT(55)
+#define BIT_R_WMAC_RXRST_DLY BIT(54)
+#define BIT_R_WMAC_SRCH_TXRPT_REF_DROP BIT(53)
+#define BIT_R_WMAC_SRCH_TXRPT_UA1 BIT(52)
+#define BIT_STATUS_BFEE5 BIT(10)
+
+/* 2 REG_WMAC_ASSOCIATED_MU_BFMEE5		(Offset 0x168A) */
+
+#define BIT_WMAC_MU_BFEE5_EN BIT(9)
+
+#define BIT_SHIFT_WMAC_MU_BFEE5_AID 0
+#define BIT_MASK_WMAC_MU_BFEE5_AID 0x1ff
+#define BIT_WMAC_MU_BFEE5_AID(x)                                               \
+	(((x) & BIT_MASK_WMAC_MU_BFEE5_AID) << BIT_SHIFT_WMAC_MU_BFEE5_AID)
+#define BIT_GET_WMAC_MU_BFEE5_AID(x)                                           \
+	(((x) >> BIT_SHIFT_WMAC_MU_BFEE5_AID) & BIT_MASK_WMAC_MU_BFEE5_AID)
+
+/* 2 REG_WMAC_ASSOCIATED_MU_BFMEE6		(Offset 0x168C) */
+
+#define BIT_STATUS_BFEE6 BIT(10)
+#define BIT_WMAC_MU_BFEE6_EN BIT(9)
+
+#define BIT_SHIFT_WMAC_MU_BFEE6_AID 0
+#define BIT_MASK_WMAC_MU_BFEE6_AID 0x1ff
+#define BIT_WMAC_MU_BFEE6_AID(x)                                               \
+	(((x) & BIT_MASK_WMAC_MU_BFEE6_AID) << BIT_SHIFT_WMAC_MU_BFEE6_AID)
+#define BIT_GET_WMAC_MU_BFEE6_AID(x)                                           \
+	(((x) >> BIT_SHIFT_WMAC_MU_BFEE6_AID) & BIT_MASK_WMAC_MU_BFEE6_AID)
+
+/* 2 REG_WMAC_ASSOCIATED_MU_BFMEE7		(Offset 0x168E) */
+
+#define BIT_BIT_STATUS_BFEE4 BIT(10)
+#define BIT_WMAC_MU_BFEE7_EN BIT(9)
+
+#define BIT_SHIFT_WMAC_MU_BFEE7_AID 0
+#define BIT_MASK_WMAC_MU_BFEE7_AID 0x1ff
+#define BIT_WMAC_MU_BFEE7_AID(x)                                               \
+	(((x) & BIT_MASK_WMAC_MU_BFEE7_AID) << BIT_SHIFT_WMAC_MU_BFEE7_AID)
+#define BIT_GET_WMAC_MU_BFEE7_AID(x)                                           \
+	(((x) >> BIT_SHIFT_WMAC_MU_BFEE7_AID) & BIT_MASK_WMAC_MU_BFEE7_AID)
+
+/* 2 REG_WMAC_BB_STOP_RX_COUNTER		(Offset 0x1690) */
+
+#define BIT_RST_ALL_COUNTER BIT(31)
+
+#define BIT_SHIFT_ABORT_RX_VBON_COUNTER 16
+#define BIT_MASK_ABORT_RX_VBON_COUNTER 0xff
+#define BIT_ABORT_RX_VBON_COUNTER(x)                                           \
+	(((x) & BIT_MASK_ABORT_RX_VBON_COUNTER)                                \
+	 << BIT_SHIFT_ABORT_RX_VBON_COUNTER)
+#define BIT_GET_ABORT_RX_VBON_COUNTER(x)                                       \
+	(((x) >> BIT_SHIFT_ABORT_RX_VBON_COUNTER) &                            \
+	 BIT_MASK_ABORT_RX_VBON_COUNTER)
+
+#define BIT_SHIFT_ABORT_RX_RDRDY_COUNTER 8
+#define BIT_MASK_ABORT_RX_RDRDY_COUNTER 0xff
+#define BIT_ABORT_RX_RDRDY_COUNTER(x)                                          \
+	(((x) & BIT_MASK_ABORT_RX_RDRDY_COUNTER)                               \
+	 << BIT_SHIFT_ABORT_RX_RDRDY_COUNTER)
+#define BIT_GET_ABORT_RX_RDRDY_COUNTER(x)                                      \
+	(((x) >> BIT_SHIFT_ABORT_RX_RDRDY_COUNTER) &                           \
+	 BIT_MASK_ABORT_RX_RDRDY_COUNTER)
+
+#define BIT_SHIFT_VBON_EARLY_FALLING_COUNTER 0
+#define BIT_MASK_VBON_EARLY_FALLING_COUNTER 0xff
+#define BIT_VBON_EARLY_FALLING_COUNTER(x)                                      \
+	(((x) & BIT_MASK_VBON_EARLY_FALLING_COUNTER)                           \
+	 << BIT_SHIFT_VBON_EARLY_FALLING_COUNTER)
+#define BIT_GET_VBON_EARLY_FALLING_COUNTER(x)                                  \
+	(((x) >> BIT_SHIFT_VBON_EARLY_FALLING_COUNTER) &                       \
+	 BIT_MASK_VBON_EARLY_FALLING_COUNTER)
+
+/* 2 REG_WMAC_PLCP_MONITOR			(Offset 0x1694) */
+
+#define BIT_WMAC_PLCP_TRX_SEL BIT(31)
+
+#define BIT_SHIFT_WMAC_PLCP_RDSIG_SEL 28
+#define BIT_MASK_WMAC_PLCP_RDSIG_SEL 0x7
+#define BIT_WMAC_PLCP_RDSIG_SEL(x)                                             \
+	(((x) & BIT_MASK_WMAC_PLCP_RDSIG_SEL) << BIT_SHIFT_WMAC_PLCP_RDSIG_SEL)
+#define BIT_GET_WMAC_PLCP_RDSIG_SEL(x)                                         \
+	(((x) >> BIT_SHIFT_WMAC_PLCP_RDSIG_SEL) & BIT_MASK_WMAC_PLCP_RDSIG_SEL)
+
+#define BIT_SHIFT_WMAC_RATE_IDX 24
+#define BIT_MASK_WMAC_RATE_IDX 0xf
+#define BIT_WMAC_RATE_IDX(x)                                                   \
+	(((x) & BIT_MASK_WMAC_RATE_IDX) << BIT_SHIFT_WMAC_RATE_IDX)
+#define BIT_GET_WMAC_RATE_IDX(x)                                               \
+	(((x) >> BIT_SHIFT_WMAC_RATE_IDX) & BIT_MASK_WMAC_RATE_IDX)
+
+#define BIT_SHIFT_WMAC_PLCP_RDSIG 0
+#define BIT_MASK_WMAC_PLCP_RDSIG 0xffffff
+#define BIT_WMAC_PLCP_RDSIG(x)                                                 \
+	(((x) & BIT_MASK_WMAC_PLCP_RDSIG) << BIT_SHIFT_WMAC_PLCP_RDSIG)
+#define BIT_GET_WMAC_PLCP_RDSIG(x)                                             \
+	(((x) >> BIT_SHIFT_WMAC_PLCP_RDSIG) & BIT_MASK_WMAC_PLCP_RDSIG)
+
+/* 2 REG_WMAC_PLCP_MONITOR_MUTX		(Offset 0x1698) */
+
+#define BIT_WMAC_MUTX_IDX BIT(24)
+
+/* 2 REG_TRANSMIT_ADDRSS_0			(Offset 0x16A0) */
+
+#define BIT_SHIFT_TA0 0
+#define BIT_MASK_TA0 0xffffffffffffL
+#define BIT_TA0(x) (((x) & BIT_MASK_TA0) << BIT_SHIFT_TA0)
+#define BIT_GET_TA0(x) (((x) >> BIT_SHIFT_TA0) & BIT_MASK_TA0)
+
+/* 2 REG_TRANSMIT_ADDRSS_1			(Offset 0x16A8) */
+
+#define BIT_SHIFT_TA1 0
+#define BIT_MASK_TA1 0xffffffffffffL
+#define BIT_TA1(x) (((x) & BIT_MASK_TA1) << BIT_SHIFT_TA1)
+#define BIT_GET_TA1(x) (((x) >> BIT_SHIFT_TA1) & BIT_MASK_TA1)
+
+/* 2 REG_TRANSMIT_ADDRSS_2			(Offset 0x16B0) */
+
+#define BIT_SHIFT_TA2 0
+#define BIT_MASK_TA2 0xffffffffffffL
+#define BIT_TA2(x) (((x) & BIT_MASK_TA2) << BIT_SHIFT_TA2)
+#define BIT_GET_TA2(x) (((x) >> BIT_SHIFT_TA2) & BIT_MASK_TA2)
+
+/* 2 REG_TRANSMIT_ADDRSS_3			(Offset 0x16B8) */
+
+#define BIT_SHIFT_TA3 0
+#define BIT_MASK_TA3 0xffffffffffffL
+#define BIT_TA3(x) (((x) & BIT_MASK_TA3) << BIT_SHIFT_TA3)
+#define BIT_GET_TA3(x) (((x) >> BIT_SHIFT_TA3) & BIT_MASK_TA3)
+
+/* 2 REG_TRANSMIT_ADDRSS_4			(Offset 0x16C0) */
+
+#define BIT_SHIFT_TA4 0
+#define BIT_MASK_TA4 0xffffffffffffL
+#define BIT_TA4(x) (((x) & BIT_MASK_TA4) << BIT_SHIFT_TA4)
+#define BIT_GET_TA4(x) (((x) >> BIT_SHIFT_TA4) & BIT_MASK_TA4)
+
+/* 2 REG_WL2LTECOEX_INDIRECT_ACCESS_CTRL_V1 (Offset 0x1700) */
+
+#define BIT_LTECOEX_ACCESS_START_V1 BIT(31)
+#define BIT_LTECOEX_WRITE_MODE_V1 BIT(30)
+#define BIT_LTECOEX_READY_BIT_V1 BIT(29)
+
+#define BIT_SHIFT_WRITE_BYTE_EN_V1 16
+#define BIT_MASK_WRITE_BYTE_EN_V1 0xf
+#define BIT_WRITE_BYTE_EN_V1(x)                                                \
+	(((x) & BIT_MASK_WRITE_BYTE_EN_V1) << BIT_SHIFT_WRITE_BYTE_EN_V1)
+#define BIT_GET_WRITE_BYTE_EN_V1(x)                                            \
+	(((x) >> BIT_SHIFT_WRITE_BYTE_EN_V1) & BIT_MASK_WRITE_BYTE_EN_V1)
+
+#define BIT_SHIFT_LTECOEX_REG_ADDR_V1 0
+#define BIT_MASK_LTECOEX_REG_ADDR_V1 0xffff
+#define BIT_LTECOEX_REG_ADDR_V1(x)                                             \
+	(((x) & BIT_MASK_LTECOEX_REG_ADDR_V1) << BIT_SHIFT_LTECOEX_REG_ADDR_V1)
+#define BIT_GET_LTECOEX_REG_ADDR_V1(x)                                         \
+	(((x) >> BIT_SHIFT_LTECOEX_REG_ADDR_V1) & BIT_MASK_LTECOEX_REG_ADDR_V1)
+
+/* 2 REG_WL2LTECOEX_INDIRECT_ACCESS_WRITE_DATA_V1 (Offset 0x1704) */
+
+#define BIT_SHIFT_LTECOEX_W_DATA_V1 0
+#define BIT_MASK_LTECOEX_W_DATA_V1 0xffffffffL
+#define BIT_LTECOEX_W_DATA_V1(x)                                               \
+	(((x) & BIT_MASK_LTECOEX_W_DATA_V1) << BIT_SHIFT_LTECOEX_W_DATA_V1)
+#define BIT_GET_LTECOEX_W_DATA_V1(x)                                           \
+	(((x) >> BIT_SHIFT_LTECOEX_W_DATA_V1) & BIT_MASK_LTECOEX_W_DATA_V1)
+
+/* 2 REG_WL2LTECOEX_INDIRECT_ACCESS_READ_DATA_V1 (Offset 0x1708) */
+
+#define BIT_SHIFT_LTECOEX_R_DATA_V1 0
+#define BIT_MASK_LTECOEX_R_DATA_V1 0xffffffffL
+#define BIT_LTECOEX_R_DATA_V1(x)                                               \
+	(((x) & BIT_MASK_LTECOEX_R_DATA_V1) << BIT_SHIFT_LTECOEX_R_DATA_V1)
+#define BIT_GET_LTECOEX_R_DATA_V1(x)                                           \
+	(((x) >> BIT_SHIFT_LTECOEX_R_DATA_V1) & BIT_MASK_LTECOEX_R_DATA_V1)
+
+#endif /* __RTL_WLAN_BITDEF_H__ */
diff --git a/drivers/staging/rtlwifi/halmac/halmac_bit_8822b.h b/drivers/staging/rtlwifi/halmac/halmac_bit_8822b.h
new file mode 100644
index 000000000000..7d02553f229e
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_bit_8822b.h
@@ -0,0 +1,12103 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef __INC_HALMAC_BIT_8822B_H
+#define __INC_HALMAC_BIT_8822B_H
+
+#define CPU_OPT_WIDTH 0x1F
+
+/* 2 REG_NOT_VALID_8822B */
+
+/* 2 REG_SYS_ISO_CTRL_8822B */
+#define BIT_PWC_EV12V_8822B BIT(15)
+#define BIT_PWC_EV25V_8822B BIT(14)
+#define BIT_PA33V_EN_8822B BIT(13)
+#define BIT_PA12V_EN_8822B BIT(12)
+#define BIT_UA33V_EN_8822B BIT(11)
+#define BIT_UA12V_EN_8822B BIT(10)
+#define BIT_ISO_RFDIO_8822B BIT(9)
+#define BIT_ISO_EB2CORE_8822B BIT(8)
+#define BIT_ISO_DIOE_8822B BIT(7)
+#define BIT_ISO_WLPON2PP_8822B BIT(6)
+#define BIT_ISO_IP2MAC_WA2PP_8822B BIT(5)
+#define BIT_ISO_PD2CORE_8822B BIT(4)
+#define BIT_ISO_PA2PCIE_8822B BIT(3)
+#define BIT_ISO_UD2CORE_8822B BIT(2)
+#define BIT_ISO_UA2USB_8822B BIT(1)
+#define BIT_ISO_WD2PP_8822B BIT(0)
+
+/* 2 REG_SYS_FUNC_EN_8822B */
+#define BIT_FEN_MREGEN_8822B BIT(15)
+#define BIT_FEN_HWPDN_8822B BIT(14)
+#define BIT_EN_25_1_8822B BIT(13)
+#define BIT_FEN_ELDR_8822B BIT(12)
+#define BIT_FEN_DCORE_8822B BIT(11)
+#define BIT_FEN_CPUEN_8822B BIT(10)
+#define BIT_FEN_DIOE_8822B BIT(9)
+#define BIT_FEN_PCIED_8822B BIT(8)
+#define BIT_FEN_PPLL_8822B BIT(7)
+#define BIT_FEN_PCIEA_8822B BIT(6)
+#define BIT_FEN_DIO_PCIE_8822B BIT(5)
+#define BIT_FEN_USBD_8822B BIT(4)
+#define BIT_FEN_UPLL_8822B BIT(3)
+#define BIT_FEN_USBA_8822B BIT(2)
+#define BIT_FEN_BB_GLB_RSTN_8822B BIT(1)
+#define BIT_FEN_BBRSTB_8822B BIT(0)
+
+/* 2 REG_SYS_PW_CTRL_8822B */
+#define BIT_SOP_EABM_8822B BIT(31)
+#define BIT_SOP_ACKF_8822B BIT(30)
+#define BIT_SOP_ERCK_8822B BIT(29)
+#define BIT_SOP_ESWR_8822B BIT(28)
+#define BIT_SOP_PWMM_8822B BIT(27)
+#define BIT_SOP_EECK_8822B BIT(26)
+#define BIT_SOP_EXTL_8822B BIT(24)
+#define BIT_SYM_OP_RING_12M_8822B BIT(22)
+#define BIT_ROP_SWPR_8822B BIT(21)
+#define BIT_DIS_HW_LPLDM_8822B BIT(20)
+#define BIT_OPT_SWRST_WLMCU_8822B BIT(19)
+#define BIT_RDY_SYSPWR_8822B BIT(17)
+#define BIT_EN_WLON_8822B BIT(16)
+#define BIT_APDM_HPDN_8822B BIT(15)
+#define BIT_AFSM_PCIE_SUS_EN_8822B BIT(12)
+#define BIT_AFSM_WLSUS_EN_8822B BIT(11)
+#define BIT_APFM_SWLPS_8822B BIT(10)
+#define BIT_APFM_OFFMAC_8822B BIT(9)
+#define BIT_APFN_ONMAC_8822B BIT(8)
+#define BIT_CHIP_PDN_EN_8822B BIT(7)
+#define BIT_RDY_MACDIS_8822B BIT(6)
+#define BIT_RING_CLK_12M_EN_8822B BIT(4)
+#define BIT_PFM_WOWL_8822B BIT(3)
+#define BIT_PFM_LDKP_8822B BIT(2)
+#define BIT_WL_HCI_ALD_8822B BIT(1)
+#define BIT_PFM_LDALL_8822B BIT(0)
+
+/* 2 REG_SYS_CLK_CTRL_8822B */
+#define BIT_LDO_DUMMY_8822B BIT(15)
+#define BIT_CPU_CLK_EN_8822B BIT(14)
+#define BIT_SYMREG_CLK_EN_8822B BIT(13)
+#define BIT_HCI_CLK_EN_8822B BIT(12)
+#define BIT_MAC_CLK_EN_8822B BIT(11)
+#define BIT_SEC_CLK_EN_8822B BIT(10)
+#define BIT_PHY_SSC_RSTB_8822B BIT(9)
+#define BIT_EXT_32K_EN_8822B BIT(8)
+#define BIT_WL_CLK_TEST_8822B BIT(7)
+#define BIT_OP_SPS_PWM_EN_8822B BIT(6)
+#define BIT_LOADER_CLK_EN_8822B BIT(5)
+#define BIT_MACSLP_8822B BIT(4)
+#define BIT_WAKEPAD_EN_8822B BIT(3)
+#define BIT_ROMD16V_EN_8822B BIT(2)
+#define BIT_CKANA12M_EN_8822B BIT(1)
+#define BIT_CNTD16V_EN_8822B BIT(0)
+
+/* 2 REG_SYS_EEPROM_CTRL_8822B */
+
+#define BIT_SHIFT_VPDIDX_8822B 8
+#define BIT_MASK_VPDIDX_8822B 0xff
+#define BIT_VPDIDX_8822B(x)                                                    \
+	(((x) & BIT_MASK_VPDIDX_8822B) << BIT_SHIFT_VPDIDX_8822B)
+#define BIT_GET_VPDIDX_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_VPDIDX_8822B) & BIT_MASK_VPDIDX_8822B)
+
+#define BIT_SHIFT_EEM1_0_8822B 6
+#define BIT_MASK_EEM1_0_8822B 0x3
+#define BIT_EEM1_0_8822B(x)                                                    \
+	(((x) & BIT_MASK_EEM1_0_8822B) << BIT_SHIFT_EEM1_0_8822B)
+#define BIT_GET_EEM1_0_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_EEM1_0_8822B) & BIT_MASK_EEM1_0_8822B)
+
+#define BIT_AUTOLOAD_SUS_8822B BIT(5)
+#define BIT_EERPOMSEL_8822B BIT(4)
+#define BIT_EECS_V1_8822B BIT(3)
+#define BIT_EESK_V1_8822B BIT(2)
+#define BIT_EEDI_V1_8822B BIT(1)
+#define BIT_EEDO_V1_8822B BIT(0)
+
+/* 2 REG_EE_VPD_8822B */
+
+#define BIT_SHIFT_VPD_DATA_8822B 0
+#define BIT_MASK_VPD_DATA_8822B 0xffffffffL
+#define BIT_VPD_DATA_8822B(x)                                                  \
+	(((x) & BIT_MASK_VPD_DATA_8822B) << BIT_SHIFT_VPD_DATA_8822B)
+#define BIT_GET_VPD_DATA_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_VPD_DATA_8822B) & BIT_MASK_VPD_DATA_8822B)
+
+/* 2 REG_SYS_SWR_CTRL1_8822B */
+#define BIT_C2_L_BIT0_8822B BIT(31)
+
+#define BIT_SHIFT_C1_L_8822B 29
+#define BIT_MASK_C1_L_8822B 0x3
+#define BIT_C1_L_8822B(x) (((x) & BIT_MASK_C1_L_8822B) << BIT_SHIFT_C1_L_8822B)
+#define BIT_GET_C1_L_8822B(x)                                                  \
+	(((x) >> BIT_SHIFT_C1_L_8822B) & BIT_MASK_C1_L_8822B)
+
+#define BIT_SHIFT_REG_FREQ_L_8822B 25
+#define BIT_MASK_REG_FREQ_L_8822B 0x7
+#define BIT_REG_FREQ_L_8822B(x)                                                \
+	(((x) & BIT_MASK_REG_FREQ_L_8822B) << BIT_SHIFT_REG_FREQ_L_8822B)
+#define BIT_GET_REG_FREQ_L_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_REG_FREQ_L_8822B) & BIT_MASK_REG_FREQ_L_8822B)
+
+#define BIT_REG_EN_DUTY_8822B BIT(24)
+
+#define BIT_SHIFT_REG_MODE_8822B 22
+#define BIT_MASK_REG_MODE_8822B 0x3
+#define BIT_REG_MODE_8822B(x)                                                  \
+	(((x) & BIT_MASK_REG_MODE_8822B) << BIT_SHIFT_REG_MODE_8822B)
+#define BIT_GET_REG_MODE_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_REG_MODE_8822B) & BIT_MASK_REG_MODE_8822B)
+
+#define BIT_REG_EN_SP_8822B BIT(21)
+#define BIT_REG_AUTO_L_8822B BIT(20)
+#define BIT_SW18_SELD_BIT0_8822B BIT(19)
+#define BIT_SW18_POWOCP_8822B BIT(18)
+
+#define BIT_SHIFT_OCP_L1_8822B 15
+#define BIT_MASK_OCP_L1_8822B 0x7
+#define BIT_OCP_L1_8822B(x)                                                    \
+	(((x) & BIT_MASK_OCP_L1_8822B) << BIT_SHIFT_OCP_L1_8822B)
+#define BIT_GET_OCP_L1_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_OCP_L1_8822B) & BIT_MASK_OCP_L1_8822B)
+
+#define BIT_SHIFT_CF_L_8822B 13
+#define BIT_MASK_CF_L_8822B 0x3
+#define BIT_CF_L_8822B(x) (((x) & BIT_MASK_CF_L_8822B) << BIT_SHIFT_CF_L_8822B)
+#define BIT_GET_CF_L_8822B(x)                                                  \
+	(((x) >> BIT_SHIFT_CF_L_8822B) & BIT_MASK_CF_L_8822B)
+
+#define BIT_SW18_FPWM_8822B BIT(11)
+#define BIT_SW18_SWEN_8822B BIT(9)
+#define BIT_SW18_LDEN_8822B BIT(8)
+#define BIT_MAC_ID_EN_8822B BIT(7)
+#define BIT_AFE_BGEN_8822B BIT(0)
+
+/* 2 REG_SYS_SWR_CTRL2_8822B */
+#define BIT_POW_ZCD_L_8822B BIT(31)
+#define BIT_AUTOZCD_L_8822B BIT(30)
+
+#define BIT_SHIFT_REG_DELAY_8822B 28
+#define BIT_MASK_REG_DELAY_8822B 0x3
+#define BIT_REG_DELAY_8822B(x)                                                 \
+	(((x) & BIT_MASK_REG_DELAY_8822B) << BIT_SHIFT_REG_DELAY_8822B)
+#define BIT_GET_REG_DELAY_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_REG_DELAY_8822B) & BIT_MASK_REG_DELAY_8822B)
+
+#define BIT_SHIFT_V15ADJ_L1_V1_8822B 24
+#define BIT_MASK_V15ADJ_L1_V1_8822B 0x7
+#define BIT_V15ADJ_L1_V1_8822B(x)                                              \
+	(((x) & BIT_MASK_V15ADJ_L1_V1_8822B) << BIT_SHIFT_V15ADJ_L1_V1_8822B)
+#define BIT_GET_V15ADJ_L1_V1_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_V15ADJ_L1_V1_8822B) & BIT_MASK_V15ADJ_L1_V1_8822B)
+
+#define BIT_SHIFT_VOL_L1_V1_8822B 20
+#define BIT_MASK_VOL_L1_V1_8822B 0xf
+#define BIT_VOL_L1_V1_8822B(x)                                                 \
+	(((x) & BIT_MASK_VOL_L1_V1_8822B) << BIT_SHIFT_VOL_L1_V1_8822B)
+#define BIT_GET_VOL_L1_V1_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_VOL_L1_V1_8822B) & BIT_MASK_VOL_L1_V1_8822B)
+
+#define BIT_SHIFT_IN_L1_V1_8822B 17
+#define BIT_MASK_IN_L1_V1_8822B 0x7
+#define BIT_IN_L1_V1_8822B(x)                                                  \
+	(((x) & BIT_MASK_IN_L1_V1_8822B) << BIT_SHIFT_IN_L1_V1_8822B)
+#define BIT_GET_IN_L1_V1_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_IN_L1_V1_8822B) & BIT_MASK_IN_L1_V1_8822B)
+
+#define BIT_SHIFT_TBOX_L1_8822B 15
+#define BIT_MASK_TBOX_L1_8822B 0x3
+#define BIT_TBOX_L1_8822B(x)                                                   \
+	(((x) & BIT_MASK_TBOX_L1_8822B) << BIT_SHIFT_TBOX_L1_8822B)
+#define BIT_GET_TBOX_L1_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_TBOX_L1_8822B) & BIT_MASK_TBOX_L1_8822B)
+
+#define BIT_SW18_SEL_8822B BIT(13)
+
+/* 2 REG_NOT_VALID_8822B */
+#define BIT_SW18_SD_8822B BIT(10)
+
+#define BIT_SHIFT_R3_L_8822B 7
+#define BIT_MASK_R3_L_8822B 0x3
+#define BIT_R3_L_8822B(x) (((x) & BIT_MASK_R3_L_8822B) << BIT_SHIFT_R3_L_8822B)
+#define BIT_GET_R3_L_8822B(x)                                                  \
+	(((x) >> BIT_SHIFT_R3_L_8822B) & BIT_MASK_R3_L_8822B)
+
+#define BIT_SHIFT_SW18_R2_8822B 5
+#define BIT_MASK_SW18_R2_8822B 0x3
+#define BIT_SW18_R2_8822B(x)                                                   \
+	(((x) & BIT_MASK_SW18_R2_8822B) << BIT_SHIFT_SW18_R2_8822B)
+#define BIT_GET_SW18_R2_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_SW18_R2_8822B) & BIT_MASK_SW18_R2_8822B)
+
+#define BIT_SHIFT_SW18_R1_8822B 3
+#define BIT_MASK_SW18_R1_8822B 0x3
+#define BIT_SW18_R1_8822B(x)                                                   \
+	(((x) & BIT_MASK_SW18_R1_8822B) << BIT_SHIFT_SW18_R1_8822B)
+#define BIT_GET_SW18_R1_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_SW18_R1_8822B) & BIT_MASK_SW18_R1_8822B)
+
+#define BIT_SHIFT_C3_L_C3_8822B 1
+#define BIT_MASK_C3_L_C3_8822B 0x3
+#define BIT_C3_L_C3_8822B(x)                                                   \
+	(((x) & BIT_MASK_C3_L_C3_8822B) << BIT_SHIFT_C3_L_C3_8822B)
+#define BIT_GET_C3_L_C3_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_C3_L_C3_8822B) & BIT_MASK_C3_L_C3_8822B)
+
+#define BIT_C2_L_BIT1_8822B BIT(0)
+
+/* 2 REG_SYS_SWR_CTRL3_8822B */
+#define BIT_SPS18_OCP_DIS_8822B BIT(31)
+
+#define BIT_SHIFT_SPS18_OCP_TH_8822B 16
+#define BIT_MASK_SPS18_OCP_TH_8822B 0x7fff
+#define BIT_SPS18_OCP_TH_8822B(x)                                              \
+	(((x) & BIT_MASK_SPS18_OCP_TH_8822B) << BIT_SHIFT_SPS18_OCP_TH_8822B)
+#define BIT_GET_SPS18_OCP_TH_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_SPS18_OCP_TH_8822B) & BIT_MASK_SPS18_OCP_TH_8822B)
+
+#define BIT_SHIFT_OCP_WINDOW_8822B 0
+#define BIT_MASK_OCP_WINDOW_8822B 0xffff
+#define BIT_OCP_WINDOW_8822B(x)                                                \
+	(((x) & BIT_MASK_OCP_WINDOW_8822B) << BIT_SHIFT_OCP_WINDOW_8822B)
+#define BIT_GET_OCP_WINDOW_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_OCP_WINDOW_8822B) & BIT_MASK_OCP_WINDOW_8822B)
+
+/* 2 REG_RSV_CTRL_8822B */
+#define BIT_HREG_DBG_8822B BIT(23)
+#define BIT_WLMCUIOIF_8822B BIT(8)
+#define BIT_LOCK_ALL_EN_8822B BIT(7)
+#define BIT_R_DIS_PRST_8822B BIT(6)
+#define BIT_WLOCK_1C_B6_8822B BIT(5)
+#define BIT_WLOCK_40_8822B BIT(4)
+#define BIT_WLOCK_08_8822B BIT(3)
+#define BIT_WLOCK_04_8822B BIT(2)
+#define BIT_WLOCK_00_8822B BIT(1)
+#define BIT_WLOCK_ALL_8822B BIT(0)
+
+/* 2 REG_RF_CTRL_8822B */
+#define BIT_RF_SDMRSTB_8822B BIT(2)
+#define BIT_RF_RSTB_8822B BIT(1)
+#define BIT_RF_EN_8822B BIT(0)
+
+/* 2 REG_AFE_LDO_CTRL_8822B */
+
+#define BIT_SHIFT_LPLDH12_RSV_8822B 29
+#define BIT_MASK_LPLDH12_RSV_8822B 0x7
+#define BIT_LPLDH12_RSV_8822B(x)                                               \
+	(((x) & BIT_MASK_LPLDH12_RSV_8822B) << BIT_SHIFT_LPLDH12_RSV_8822B)
+#define BIT_GET_LPLDH12_RSV_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_LPLDH12_RSV_8822B) & BIT_MASK_LPLDH12_RSV_8822B)
+
+#define BIT_LPLDH12_SLP_8822B BIT(28)
+
+#define BIT_SHIFT_LPLDH12_VADJ_8822B 24
+#define BIT_MASK_LPLDH12_VADJ_8822B 0xf
+#define BIT_LPLDH12_VADJ_8822B(x)                                              \
+	(((x) & BIT_MASK_LPLDH12_VADJ_8822B) << BIT_SHIFT_LPLDH12_VADJ_8822B)
+#define BIT_GET_LPLDH12_VADJ_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_LPLDH12_VADJ_8822B) & BIT_MASK_LPLDH12_VADJ_8822B)
+
+#define BIT_LDH12_EN_8822B BIT(16)
+#define BIT_WLBBOFF_BIG_PWC_EN_8822B BIT(14)
+#define BIT_WLBBOFF_SMALL_PWC_EN_8822B BIT(13)
+#define BIT_WLMACOFF_BIG_PWC_EN_8822B BIT(12)
+#define BIT_WLPON_PWC_EN_8822B BIT(11)
+#define BIT_POW_REGU_P1_8822B BIT(10)
+#define BIT_LDOV12W_EN_8822B BIT(8)
+#define BIT_EX_XTAL_DRV_DIGI_8822B BIT(7)
+#define BIT_EX_XTAL_DRV_USB_8822B BIT(6)
+#define BIT_EX_XTAL_DRV_AFE_8822B BIT(5)
+#define BIT_EX_XTAL_DRV_RF2_8822B BIT(4)
+#define BIT_EX_XTAL_DRV_RF1_8822B BIT(3)
+#define BIT_POW_REGU_P0_8822B BIT(2)
+
+/* 2 REG_NOT_VALID_8822B */
+#define BIT_POW_PLL_LDO_8822B BIT(0)
+
+/* 2 REG_AFE_CTRL1_8822B */
+#define BIT_AGPIO_GPE_8822B BIT(31)
+
+#define BIT_SHIFT_XTAL_CAP_XI_8822B 25
+#define BIT_MASK_XTAL_CAP_XI_8822B 0x3f
+#define BIT_XTAL_CAP_XI_8822B(x)                                               \
+	(((x) & BIT_MASK_XTAL_CAP_XI_8822B) << BIT_SHIFT_XTAL_CAP_XI_8822B)
+#define BIT_GET_XTAL_CAP_XI_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_XTAL_CAP_XI_8822B) & BIT_MASK_XTAL_CAP_XI_8822B)
+
+#define BIT_SHIFT_XTAL_DRV_DIGI_8822B 23
+#define BIT_MASK_XTAL_DRV_DIGI_8822B 0x3
+#define BIT_XTAL_DRV_DIGI_8822B(x)                                             \
+	(((x) & BIT_MASK_XTAL_DRV_DIGI_8822B) << BIT_SHIFT_XTAL_DRV_DIGI_8822B)
+#define BIT_GET_XTAL_DRV_DIGI_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_XTAL_DRV_DIGI_8822B) & BIT_MASK_XTAL_DRV_DIGI_8822B)
+
+#define BIT_XTAL_DRV_USB_BIT1_8822B BIT(22)
+
+#define BIT_SHIFT_MAC_CLK_SEL_8822B 20
+#define BIT_MASK_MAC_CLK_SEL_8822B 0x3
+#define BIT_MAC_CLK_SEL_8822B(x)                                               \
+	(((x) & BIT_MASK_MAC_CLK_SEL_8822B) << BIT_SHIFT_MAC_CLK_SEL_8822B)
+#define BIT_GET_MAC_CLK_SEL_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_MAC_CLK_SEL_8822B) & BIT_MASK_MAC_CLK_SEL_8822B)
+
+#define BIT_XTAL_DRV_USB_BIT0_8822B BIT(19)
+
+#define BIT_SHIFT_XTAL_DRV_AFE_8822B 17
+#define BIT_MASK_XTAL_DRV_AFE_8822B 0x3
+#define BIT_XTAL_DRV_AFE_8822B(x)                                              \
+	(((x) & BIT_MASK_XTAL_DRV_AFE_8822B) << BIT_SHIFT_XTAL_DRV_AFE_8822B)
+#define BIT_GET_XTAL_DRV_AFE_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_XTAL_DRV_AFE_8822B) & BIT_MASK_XTAL_DRV_AFE_8822B)
+
+#define BIT_SHIFT_XTAL_DRV_RF2_8822B 15
+#define BIT_MASK_XTAL_DRV_RF2_8822B 0x3
+#define BIT_XTAL_DRV_RF2_8822B(x)                                              \
+	(((x) & BIT_MASK_XTAL_DRV_RF2_8822B) << BIT_SHIFT_XTAL_DRV_RF2_8822B)
+#define BIT_GET_XTAL_DRV_RF2_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_XTAL_DRV_RF2_8822B) & BIT_MASK_XTAL_DRV_RF2_8822B)
+
+#define BIT_SHIFT_XTAL_DRV_RF1_8822B 13
+#define BIT_MASK_XTAL_DRV_RF1_8822B 0x3
+#define BIT_XTAL_DRV_RF1_8822B(x)                                              \
+	(((x) & BIT_MASK_XTAL_DRV_RF1_8822B) << BIT_SHIFT_XTAL_DRV_RF1_8822B)
+#define BIT_GET_XTAL_DRV_RF1_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_XTAL_DRV_RF1_8822B) & BIT_MASK_XTAL_DRV_RF1_8822B)
+
+#define BIT_XTAL_DELAY_DIGI_8822B BIT(12)
+#define BIT_XTAL_DELAY_USB_8822B BIT(11)
+#define BIT_XTAL_DELAY_AFE_8822B BIT(10)
+
+#define BIT_SHIFT_XTAL_LDO_VREF_8822B 7
+#define BIT_MASK_XTAL_LDO_VREF_8822B 0x7
+#define BIT_XTAL_LDO_VREF_8822B(x)                                             \
+	(((x) & BIT_MASK_XTAL_LDO_VREF_8822B) << BIT_SHIFT_XTAL_LDO_VREF_8822B)
+#define BIT_GET_XTAL_LDO_VREF_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_XTAL_LDO_VREF_8822B) & BIT_MASK_XTAL_LDO_VREF_8822B)
+
+#define BIT_XTAL_XQSEL_RF_8822B BIT(6)
+#define BIT_XTAL_XQSEL_8822B BIT(5)
+
+#define BIT_SHIFT_XTAL_GMN_V2_8822B 3
+#define BIT_MASK_XTAL_GMN_V2_8822B 0x3
+#define BIT_XTAL_GMN_V2_8822B(x)                                               \
+	(((x) & BIT_MASK_XTAL_GMN_V2_8822B) << BIT_SHIFT_XTAL_GMN_V2_8822B)
+#define BIT_GET_XTAL_GMN_V2_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_XTAL_GMN_V2_8822B) & BIT_MASK_XTAL_GMN_V2_8822B)
+
+#define BIT_SHIFT_XTAL_GMP_V2_8822B 1
+#define BIT_MASK_XTAL_GMP_V2_8822B 0x3
+#define BIT_XTAL_GMP_V2_8822B(x)                                               \
+	(((x) & BIT_MASK_XTAL_GMP_V2_8822B) << BIT_SHIFT_XTAL_GMP_V2_8822B)
+#define BIT_GET_XTAL_GMP_V2_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_XTAL_GMP_V2_8822B) & BIT_MASK_XTAL_GMP_V2_8822B)
+
+#define BIT_XTAL_EN_8822B BIT(0)
+
+/* 2 REG_AFE_CTRL2_8822B */
+
+#define BIT_SHIFT_REG_C3_V4_8822B 30
+#define BIT_MASK_REG_C3_V4_8822B 0x3
+#define BIT_REG_C3_V4_8822B(x)                                                 \
+	(((x) & BIT_MASK_REG_C3_V4_8822B) << BIT_SHIFT_REG_C3_V4_8822B)
+#define BIT_GET_REG_C3_V4_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_REG_C3_V4_8822B) & BIT_MASK_REG_C3_V4_8822B)
+
+#define BIT_REG_CP_BIT1_8822B BIT(29)
+
+#define BIT_SHIFT_REG_RS_V4_8822B 26
+#define BIT_MASK_REG_RS_V4_8822B 0x7
+#define BIT_REG_RS_V4_8822B(x)                                                 \
+	(((x) & BIT_MASK_REG_RS_V4_8822B) << BIT_SHIFT_REG_RS_V4_8822B)
+#define BIT_GET_REG_RS_V4_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_REG_RS_V4_8822B) & BIT_MASK_REG_RS_V4_8822B)
+
+#define BIT_SHIFT_REG__CS_8822B 24
+#define BIT_MASK_REG__CS_8822B 0x3
+#define BIT_REG__CS_8822B(x)                                                   \
+	(((x) & BIT_MASK_REG__CS_8822B) << BIT_SHIFT_REG__CS_8822B)
+#define BIT_GET_REG__CS_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_REG__CS_8822B) & BIT_MASK_REG__CS_8822B)
+
+#define BIT_SHIFT_REG_CP_OFFSET_8822B 21
+#define BIT_MASK_REG_CP_OFFSET_8822B 0x7
+#define BIT_REG_CP_OFFSET_8822B(x)                                             \
+	(((x) & BIT_MASK_REG_CP_OFFSET_8822B) << BIT_SHIFT_REG_CP_OFFSET_8822B)
+#define BIT_GET_REG_CP_OFFSET_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_REG_CP_OFFSET_8822B) & BIT_MASK_REG_CP_OFFSET_8822B)
+
+#define BIT_SHIFT_CP_BIAS_8822B 18
+#define BIT_MASK_CP_BIAS_8822B 0x7
+#define BIT_CP_BIAS_8822B(x)                                                   \
+	(((x) & BIT_MASK_CP_BIAS_8822B) << BIT_SHIFT_CP_BIAS_8822B)
+#define BIT_GET_CP_BIAS_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_CP_BIAS_8822B) & BIT_MASK_CP_BIAS_8822B)
+
+#define BIT_REG_IDOUBLE_V2_8822B BIT(17)
+#define BIT_EN_SYN_8822B BIT(16)
+
+#define BIT_SHIFT_MCCO_8822B 14
+#define BIT_MASK_MCCO_8822B 0x3
+#define BIT_MCCO_8822B(x) (((x) & BIT_MASK_MCCO_8822B) << BIT_SHIFT_MCCO_8822B)
+#define BIT_GET_MCCO_8822B(x)                                                  \
+	(((x) >> BIT_SHIFT_MCCO_8822B) & BIT_MASK_MCCO_8822B)
+
+#define BIT_SHIFT_REG_LDO_SEL_8822B 12
+#define BIT_MASK_REG_LDO_SEL_8822B 0x3
+#define BIT_REG_LDO_SEL_8822B(x)                                               \
+	(((x) & BIT_MASK_REG_LDO_SEL_8822B) << BIT_SHIFT_REG_LDO_SEL_8822B)
+#define BIT_GET_REG_LDO_SEL_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_REG_LDO_SEL_8822B) & BIT_MASK_REG_LDO_SEL_8822B)
+
+#define BIT_REG_KVCO_V2_8822B BIT(10)
+#define BIT_AGPIO_GPO_8822B BIT(9)
+
+#define BIT_SHIFT_AGPIO_DRV_8822B 7
+#define BIT_MASK_AGPIO_DRV_8822B 0x3
+#define BIT_AGPIO_DRV_8822B(x)                                                 \
+	(((x) & BIT_MASK_AGPIO_DRV_8822B) << BIT_SHIFT_AGPIO_DRV_8822B)
+#define BIT_GET_AGPIO_DRV_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_AGPIO_DRV_8822B) & BIT_MASK_AGPIO_DRV_8822B)
+
+#define BIT_SHIFT_XTAL_CAP_XO_8822B 1
+#define BIT_MASK_XTAL_CAP_XO_8822B 0x3f
+#define BIT_XTAL_CAP_XO_8822B(x)                                               \
+	(((x) & BIT_MASK_XTAL_CAP_XO_8822B) << BIT_SHIFT_XTAL_CAP_XO_8822B)
+#define BIT_GET_XTAL_CAP_XO_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_XTAL_CAP_XO_8822B) & BIT_MASK_XTAL_CAP_XO_8822B)
+
+#define BIT_POW_PLL_8822B BIT(0)
+
+/* 2 REG_AFE_CTRL3_8822B */
+
+#define BIT_SHIFT_PS_8822B 7
+#define BIT_MASK_PS_8822B 0x7
+#define BIT_PS_8822B(x) (((x) & BIT_MASK_PS_8822B) << BIT_SHIFT_PS_8822B)
+#define BIT_GET_PS_8822B(x) (((x) >> BIT_SHIFT_PS_8822B) & BIT_MASK_PS_8822B)
+
+#define BIT_PSEN_8822B BIT(6)
+#define BIT_DOGENB_8822B BIT(5)
+#define BIT_REG_MBIAS_8822B BIT(4)
+
+#define BIT_SHIFT_REG_R3_V4_8822B 1
+#define BIT_MASK_REG_R3_V4_8822B 0x7
+#define BIT_REG_R3_V4_8822B(x)                                                 \
+	(((x) & BIT_MASK_REG_R3_V4_8822B) << BIT_SHIFT_REG_R3_V4_8822B)
+#define BIT_GET_REG_R3_V4_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_REG_R3_V4_8822B) & BIT_MASK_REG_R3_V4_8822B)
+
+#define BIT_REG_CP_BIT0_8822B BIT(0)
+
+/* 2 REG_EFUSE_CTRL_8822B */
+#define BIT_EF_FLAG_8822B BIT(31)
+
+#define BIT_SHIFT_EF_PGPD_8822B 28
+#define BIT_MASK_EF_PGPD_8822B 0x7
+#define BIT_EF_PGPD_8822B(x)                                                   \
+	(((x) & BIT_MASK_EF_PGPD_8822B) << BIT_SHIFT_EF_PGPD_8822B)
+#define BIT_GET_EF_PGPD_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_EF_PGPD_8822B) & BIT_MASK_EF_PGPD_8822B)
+
+#define BIT_SHIFT_EF_RDT_8822B 24
+#define BIT_MASK_EF_RDT_8822B 0xf
+#define BIT_EF_RDT_8822B(x)                                                    \
+	(((x) & BIT_MASK_EF_RDT_8822B) << BIT_SHIFT_EF_RDT_8822B)
+#define BIT_GET_EF_RDT_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_EF_RDT_8822B) & BIT_MASK_EF_RDT_8822B)
+
+#define BIT_SHIFT_EF_PGTS_8822B 20
+#define BIT_MASK_EF_PGTS_8822B 0xf
+#define BIT_EF_PGTS_8822B(x)                                                   \
+	(((x) & BIT_MASK_EF_PGTS_8822B) << BIT_SHIFT_EF_PGTS_8822B)
+#define BIT_GET_EF_PGTS_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_EF_PGTS_8822B) & BIT_MASK_EF_PGTS_8822B)
+
+#define BIT_EF_PDWN_8822B BIT(19)
+#define BIT_EF_ALDEN_8822B BIT(18)
+
+#define BIT_SHIFT_EF_ADDR_8822B 8
+#define BIT_MASK_EF_ADDR_8822B 0x3ff
+#define BIT_EF_ADDR_8822B(x)                                                   \
+	(((x) & BIT_MASK_EF_ADDR_8822B) << BIT_SHIFT_EF_ADDR_8822B)
+#define BIT_GET_EF_ADDR_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_EF_ADDR_8822B) & BIT_MASK_EF_ADDR_8822B)
+
+#define BIT_SHIFT_EF_DATA_8822B 0
+#define BIT_MASK_EF_DATA_8822B 0xff
+#define BIT_EF_DATA_8822B(x)                                                   \
+	(((x) & BIT_MASK_EF_DATA_8822B) << BIT_SHIFT_EF_DATA_8822B)
+#define BIT_GET_EF_DATA_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_EF_DATA_8822B) & BIT_MASK_EF_DATA_8822B)
+
+/* 2 REG_LDO_EFUSE_CTRL_8822B */
+#define BIT_LDOE25_EN_8822B BIT(31)
+
+#define BIT_SHIFT_LDOE25_V12ADJ_L_8822B 27
+#define BIT_MASK_LDOE25_V12ADJ_L_8822B 0xf
+#define BIT_LDOE25_V12ADJ_L_8822B(x)                                           \
+	(((x) & BIT_MASK_LDOE25_V12ADJ_L_8822B)                                \
+	 << BIT_SHIFT_LDOE25_V12ADJ_L_8822B)
+#define BIT_GET_LDOE25_V12ADJ_L_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_LDOE25_V12ADJ_L_8822B) &                            \
+	 BIT_MASK_LDOE25_V12ADJ_L_8822B)
+
+#define BIT_EF_CRES_SEL_8822B BIT(26)
+
+#define BIT_SHIFT_EF_SCAN_START_V1_8822B 16
+#define BIT_MASK_EF_SCAN_START_V1_8822B 0x3ff
+#define BIT_EF_SCAN_START_V1_8822B(x)                                          \
+	(((x) & BIT_MASK_EF_SCAN_START_V1_8822B)                               \
+	 << BIT_SHIFT_EF_SCAN_START_V1_8822B)
+#define BIT_GET_EF_SCAN_START_V1_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_EF_SCAN_START_V1_8822B) &                           \
+	 BIT_MASK_EF_SCAN_START_V1_8822B)
+
+#define BIT_SHIFT_EF_SCAN_END_8822B 12
+#define BIT_MASK_EF_SCAN_END_8822B 0xf
+#define BIT_EF_SCAN_END_8822B(x)                                               \
+	(((x) & BIT_MASK_EF_SCAN_END_8822B) << BIT_SHIFT_EF_SCAN_END_8822B)
+#define BIT_GET_EF_SCAN_END_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_EF_SCAN_END_8822B) & BIT_MASK_EF_SCAN_END_8822B)
+
+#define BIT_EF_PD_DIS_8822B BIT(11)
+
+#define BIT_SHIFT_EF_CELL_SEL_8822B 8
+#define BIT_MASK_EF_CELL_SEL_8822B 0x3
+#define BIT_EF_CELL_SEL_8822B(x)                                               \
+	(((x) & BIT_MASK_EF_CELL_SEL_8822B) << BIT_SHIFT_EF_CELL_SEL_8822B)
+#define BIT_GET_EF_CELL_SEL_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_EF_CELL_SEL_8822B) & BIT_MASK_EF_CELL_SEL_8822B)
+
+#define BIT_EF_TRPT_8822B BIT(7)
+
+#define BIT_SHIFT_EF_TTHD_8822B 0
+#define BIT_MASK_EF_TTHD_8822B 0x7f
+#define BIT_EF_TTHD_8822B(x)                                                   \
+	(((x) & BIT_MASK_EF_TTHD_8822B) << BIT_SHIFT_EF_TTHD_8822B)
+#define BIT_GET_EF_TTHD_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_EF_TTHD_8822B) & BIT_MASK_EF_TTHD_8822B)
+
+/* 2 REG_PWR_OPTION_CTRL_8822B */
+
+#define BIT_SHIFT_DBG_SEL_V1_8822B 16
+#define BIT_MASK_DBG_SEL_V1_8822B 0xff
+#define BIT_DBG_SEL_V1_8822B(x)                                                \
+	(((x) & BIT_MASK_DBG_SEL_V1_8822B) << BIT_SHIFT_DBG_SEL_V1_8822B)
+#define BIT_GET_DBG_SEL_V1_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_DBG_SEL_V1_8822B) & BIT_MASK_DBG_SEL_V1_8822B)
+
+#define BIT_SHIFT_DBG_SEL_BYTE_8822B 14
+#define BIT_MASK_DBG_SEL_BYTE_8822B 0x3
+#define BIT_DBG_SEL_BYTE_8822B(x)                                              \
+	(((x) & BIT_MASK_DBG_SEL_BYTE_8822B) << BIT_SHIFT_DBG_SEL_BYTE_8822B)
+#define BIT_GET_DBG_SEL_BYTE_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_DBG_SEL_BYTE_8822B) & BIT_MASK_DBG_SEL_BYTE_8822B)
+
+#define BIT_SHIFT_STD_L1_V1_8822B 12
+#define BIT_MASK_STD_L1_V1_8822B 0x3
+#define BIT_STD_L1_V1_8822B(x)                                                 \
+	(((x) & BIT_MASK_STD_L1_V1_8822B) << BIT_SHIFT_STD_L1_V1_8822B)
+#define BIT_GET_STD_L1_V1_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_STD_L1_V1_8822B) & BIT_MASK_STD_L1_V1_8822B)
+
+#define BIT_SYSON_DBG_PAD_E2_8822B BIT(11)
+#define BIT_SYSON_LED_PAD_E2_8822B BIT(10)
+#define BIT_SYSON_GPEE_PAD_E2_8822B BIT(9)
+#define BIT_SYSON_PCI_PAD_E2_8822B BIT(8)
+#define BIT_AUTO_SW_LDO_VOL_EN_8822B BIT(7)
+
+#define BIT_SHIFT_SYSON_SPS0WWV_WT_8822B 4
+#define BIT_MASK_SYSON_SPS0WWV_WT_8822B 0x3
+#define BIT_SYSON_SPS0WWV_WT_8822B(x)                                          \
+	(((x) & BIT_MASK_SYSON_SPS0WWV_WT_8822B)                               \
+	 << BIT_SHIFT_SYSON_SPS0WWV_WT_8822B)
+#define BIT_GET_SYSON_SPS0WWV_WT_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_SYSON_SPS0WWV_WT_8822B) &                           \
+	 BIT_MASK_SYSON_SPS0WWV_WT_8822B)
+
+#define BIT_SHIFT_SYSON_SPS0LDO_WT_8822B 2
+#define BIT_MASK_SYSON_SPS0LDO_WT_8822B 0x3
+#define BIT_SYSON_SPS0LDO_WT_8822B(x)                                          \
+	(((x) & BIT_MASK_SYSON_SPS0LDO_WT_8822B)                               \
+	 << BIT_SHIFT_SYSON_SPS0LDO_WT_8822B)
+#define BIT_GET_SYSON_SPS0LDO_WT_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_SYSON_SPS0LDO_WT_8822B) &                           \
+	 BIT_MASK_SYSON_SPS0LDO_WT_8822B)
+
+#define BIT_SHIFT_SYSON_RCLK_SCALE_8822B 0
+#define BIT_MASK_SYSON_RCLK_SCALE_8822B 0x3
+#define BIT_SYSON_RCLK_SCALE_8822B(x)                                          \
+	(((x) & BIT_MASK_SYSON_RCLK_SCALE_8822B)                               \
+	 << BIT_SHIFT_SYSON_RCLK_SCALE_8822B)
+#define BIT_GET_SYSON_RCLK_SCALE_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_SYSON_RCLK_SCALE_8822B) &                           \
+	 BIT_MASK_SYSON_RCLK_SCALE_8822B)
+
+/* 2 REG_CAL_TIMER_8822B */
+
+#define BIT_SHIFT_MATCH_CNT_8822B 8
+#define BIT_MASK_MATCH_CNT_8822B 0xff
+#define BIT_MATCH_CNT_8822B(x)                                                 \
+	(((x) & BIT_MASK_MATCH_CNT_8822B) << BIT_SHIFT_MATCH_CNT_8822B)
+#define BIT_GET_MATCH_CNT_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_MATCH_CNT_8822B) & BIT_MASK_MATCH_CNT_8822B)
+
+#define BIT_SHIFT_CAL_SCAL_8822B 0
+#define BIT_MASK_CAL_SCAL_8822B 0xff
+#define BIT_CAL_SCAL_8822B(x)                                                  \
+	(((x) & BIT_MASK_CAL_SCAL_8822B) << BIT_SHIFT_CAL_SCAL_8822B)
+#define BIT_GET_CAL_SCAL_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_CAL_SCAL_8822B) & BIT_MASK_CAL_SCAL_8822B)
+
+/* 2 REG_ACLK_MON_8822B */
+
+#define BIT_SHIFT_RCLK_MON_8822B 5
+#define BIT_MASK_RCLK_MON_8822B 0x7ff
+#define BIT_RCLK_MON_8822B(x)                                                  \
+	(((x) & BIT_MASK_RCLK_MON_8822B) << BIT_SHIFT_RCLK_MON_8822B)
+#define BIT_GET_RCLK_MON_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_RCLK_MON_8822B) & BIT_MASK_RCLK_MON_8822B)
+
+#define BIT_CAL_EN_8822B BIT(4)
+
+#define BIT_SHIFT_DPSTU_8822B 2
+#define BIT_MASK_DPSTU_8822B 0x3
+#define BIT_DPSTU_8822B(x)                                                     \
+	(((x) & BIT_MASK_DPSTU_8822B) << BIT_SHIFT_DPSTU_8822B)
+#define BIT_GET_DPSTU_8822B(x)                                                 \
+	(((x) >> BIT_SHIFT_DPSTU_8822B) & BIT_MASK_DPSTU_8822B)
+
+#define BIT_SUS_16X_8822B BIT(1)
+
+/* 2 REG_GPIO_MUXCFG_8822B */
+#define BIT_FSPI_EN_8822B BIT(19)
+#define BIT_WL_RTS_EXT_32K_SEL_8822B BIT(18)
+#define BIT_WLGP_SPI_EN_8822B BIT(16)
+#define BIT_SIC_LBK_8822B BIT(15)
+#define BIT_ENHTP_8822B BIT(14)
+#define BIT_ENSIC_8822B BIT(12)
+#define BIT_SIC_SWRST_8822B BIT(11)
+#define BIT_PO_WIFI_PTA_PINS_8822B BIT(10)
+#define BIT_PO_BT_PTA_PINS_8822B BIT(9)
+#define BIT_ENUART_8822B BIT(8)
+
+#define BIT_SHIFT_BTMODE_8822B 6
+#define BIT_MASK_BTMODE_8822B 0x3
+#define BIT_BTMODE_8822B(x)                                                    \
+	(((x) & BIT_MASK_BTMODE_8822B) << BIT_SHIFT_BTMODE_8822B)
+#define BIT_GET_BTMODE_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_BTMODE_8822B) & BIT_MASK_BTMODE_8822B)
+
+#define BIT_ENBT_8822B BIT(5)
+#define BIT_EROM_EN_8822B BIT(4)
+#define BIT_WLRFE_6_7_EN_8822B BIT(3)
+#define BIT_WLRFE_4_5_EN_8822B BIT(2)
+
+#define BIT_SHIFT_GPIOSEL_8822B 0
+#define BIT_MASK_GPIOSEL_8822B 0x3
+#define BIT_GPIOSEL_8822B(x)                                                   \
+	(((x) & BIT_MASK_GPIOSEL_8822B) << BIT_SHIFT_GPIOSEL_8822B)
+#define BIT_GET_GPIOSEL_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_GPIOSEL_8822B) & BIT_MASK_GPIOSEL_8822B)
+
+/* 2 REG_GPIO_PIN_CTRL_8822B */
+
+#define BIT_SHIFT_GPIO_MOD_7_TO_0_8822B 24
+#define BIT_MASK_GPIO_MOD_7_TO_0_8822B 0xff
+#define BIT_GPIO_MOD_7_TO_0_8822B(x)                                           \
+	(((x) & BIT_MASK_GPIO_MOD_7_TO_0_8822B)                                \
+	 << BIT_SHIFT_GPIO_MOD_7_TO_0_8822B)
+#define BIT_GET_GPIO_MOD_7_TO_0_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_GPIO_MOD_7_TO_0_8822B) &                            \
+	 BIT_MASK_GPIO_MOD_7_TO_0_8822B)
+
+#define BIT_SHIFT_GPIO_IO_SEL_7_TO_0_8822B 16
+#define BIT_MASK_GPIO_IO_SEL_7_TO_0_8822B 0xff
+#define BIT_GPIO_IO_SEL_7_TO_0_8822B(x)                                        \
+	(((x) & BIT_MASK_GPIO_IO_SEL_7_TO_0_8822B)                             \
+	 << BIT_SHIFT_GPIO_IO_SEL_7_TO_0_8822B)
+#define BIT_GET_GPIO_IO_SEL_7_TO_0_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_GPIO_IO_SEL_7_TO_0_8822B) &                         \
+	 BIT_MASK_GPIO_IO_SEL_7_TO_0_8822B)
+
+#define BIT_SHIFT_GPIO_OUT_7_TO_0_8822B 8
+#define BIT_MASK_GPIO_OUT_7_TO_0_8822B 0xff
+#define BIT_GPIO_OUT_7_TO_0_8822B(x)                                           \
+	(((x) & BIT_MASK_GPIO_OUT_7_TO_0_8822B)                                \
+	 << BIT_SHIFT_GPIO_OUT_7_TO_0_8822B)
+#define BIT_GET_GPIO_OUT_7_TO_0_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_GPIO_OUT_7_TO_0_8822B) &                            \
+	 BIT_MASK_GPIO_OUT_7_TO_0_8822B)
+
+#define BIT_SHIFT_GPIO_IN_7_TO_0_8822B 0
+#define BIT_MASK_GPIO_IN_7_TO_0_8822B 0xff
+#define BIT_GPIO_IN_7_TO_0_8822B(x)                                            \
+	(((x) & BIT_MASK_GPIO_IN_7_TO_0_8822B)                                 \
+	 << BIT_SHIFT_GPIO_IN_7_TO_0_8822B)
+#define BIT_GET_GPIO_IN_7_TO_0_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_GPIO_IN_7_TO_0_8822B) &                             \
+	 BIT_MASK_GPIO_IN_7_TO_0_8822B)
+
+/* 2 REG_GPIO_INTM_8822B */
+
+#define BIT_SHIFT_MUXDBG_SEL_8822B 30
+#define BIT_MASK_MUXDBG_SEL_8822B 0x3
+#define BIT_MUXDBG_SEL_8822B(x)                                                \
+	(((x) & BIT_MASK_MUXDBG_SEL_8822B) << BIT_SHIFT_MUXDBG_SEL_8822B)
+#define BIT_GET_MUXDBG_SEL_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_MUXDBG_SEL_8822B) & BIT_MASK_MUXDBG_SEL_8822B)
+
+#define BIT_EXTWOL_SEL_8822B BIT(17)
+#define BIT_EXTWOL_EN_8822B BIT(16)
+#define BIT_GPIOF_INT_MD_8822B BIT(15)
+#define BIT_GPIOE_INT_MD_8822B BIT(14)
+#define BIT_GPIOD_INT_MD_8822B BIT(13)
+#define BIT_GPIOF_INT_MD_8822B BIT(15)
+#define BIT_GPIOE_INT_MD_8822B BIT(14)
+#define BIT_GPIOD_INT_MD_8822B BIT(13)
+#define BIT_GPIOC_INT_MD_8822B BIT(12)
+#define BIT_GPIOB_INT_MD_8822B BIT(11)
+#define BIT_GPIOA_INT_MD_8822B BIT(10)
+#define BIT_GPIO9_INT_MD_8822B BIT(9)
+#define BIT_GPIO8_INT_MD_8822B BIT(8)
+#define BIT_GPIO7_INT_MD_8822B BIT(7)
+#define BIT_GPIO6_INT_MD_8822B BIT(6)
+#define BIT_GPIO5_INT_MD_8822B BIT(5)
+#define BIT_GPIO4_INT_MD_8822B BIT(4)
+#define BIT_GPIO3_INT_MD_8822B BIT(3)
+#define BIT_GPIO2_INT_MD_8822B BIT(2)
+#define BIT_GPIO1_INT_MD_8822B BIT(1)
+#define BIT_GPIO0_INT_MD_8822B BIT(0)
+
+/* 2 REG_LED_CFG_8822B */
+#define BIT_GPIO3_WL_CTRL_EN_8822B BIT(27)
+#define BIT_LNAON_SEL_EN_8822B BIT(26)
+#define BIT_PAPE_SEL_EN_8822B BIT(25)
+#define BIT_DPDT_WLBT_SEL_8822B BIT(24)
+#define BIT_DPDT_SEL_EN_8822B BIT(23)
+#define BIT_GPIO13_14_WL_CTRL_EN_8822B BIT(22)
+#define BIT_GPIO13_14_WL_CTRL_EN_8822B BIT(22)
+#define BIT_LED2DIS_8822B BIT(21)
+#define BIT_LED2PL_8822B BIT(20)
+#define BIT_LED2SV_8822B BIT(19)
+
+#define BIT_SHIFT_LED2CM_8822B 16
+#define BIT_MASK_LED2CM_8822B 0x7
+#define BIT_LED2CM_8822B(x)                                                    \
+	(((x) & BIT_MASK_LED2CM_8822B) << BIT_SHIFT_LED2CM_8822B)
+#define BIT_GET_LED2CM_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_LED2CM_8822B) & BIT_MASK_LED2CM_8822B)
+
+#define BIT_LED1DIS_8822B BIT(15)
+#define BIT_LED1PL_8822B BIT(12)
+#define BIT_LED1SV_8822B BIT(11)
+
+#define BIT_SHIFT_LED1CM_8822B 8
+#define BIT_MASK_LED1CM_8822B 0x7
+#define BIT_LED1CM_8822B(x)                                                    \
+	(((x) & BIT_MASK_LED1CM_8822B) << BIT_SHIFT_LED1CM_8822B)
+#define BIT_GET_LED1CM_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_LED1CM_8822B) & BIT_MASK_LED1CM_8822B)
+
+#define BIT_LED0DIS_8822B BIT(7)
+
+#define BIT_SHIFT_AFE_LDO_SWR_CHECK_8822B 5
+#define BIT_MASK_AFE_LDO_SWR_CHECK_8822B 0x3
+#define BIT_AFE_LDO_SWR_CHECK_8822B(x)                                         \
+	(((x) & BIT_MASK_AFE_LDO_SWR_CHECK_8822B)                              \
+	 << BIT_SHIFT_AFE_LDO_SWR_CHECK_8822B)
+#define BIT_GET_AFE_LDO_SWR_CHECK_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_AFE_LDO_SWR_CHECK_8822B) &                          \
+	 BIT_MASK_AFE_LDO_SWR_CHECK_8822B)
+
+#define BIT_LED0PL_8822B BIT(4)
+#define BIT_LED0SV_8822B BIT(3)
+
+#define BIT_SHIFT_LED0CM_8822B 0
+#define BIT_MASK_LED0CM_8822B 0x7
+#define BIT_LED0CM_8822B(x)                                                    \
+	(((x) & BIT_MASK_LED0CM_8822B) << BIT_SHIFT_LED0CM_8822B)
+#define BIT_GET_LED0CM_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_LED0CM_8822B) & BIT_MASK_LED0CM_8822B)
+
+/* 2 REG_FSIMR_8822B */
+#define BIT_FS_PDNINT_EN_8822B BIT(31)
+#define BIT_NFC_INT_PAD_EN_8822B BIT(30)
+#define BIT_FS_SPS_OCP_INT_EN_8822B BIT(29)
+#define BIT_FS_PWMERR_INT_EN_8822B BIT(28)
+#define BIT_FS_GPIOF_INT_EN_8822B BIT(27)
+#define BIT_FS_GPIOE_INT_EN_8822B BIT(26)
+#define BIT_FS_GPIOD_INT_EN_8822B BIT(25)
+#define BIT_FS_GPIOC_INT_EN_8822B BIT(24)
+#define BIT_FS_GPIOB_INT_EN_8822B BIT(23)
+#define BIT_FS_GPIOA_INT_EN_8822B BIT(22)
+#define BIT_FS_GPIO9_INT_EN_8822B BIT(21)
+#define BIT_FS_GPIO8_INT_EN_8822B BIT(20)
+#define BIT_FS_GPIO7_INT_EN_8822B BIT(19)
+#define BIT_FS_GPIO6_INT_EN_8822B BIT(18)
+#define BIT_FS_GPIO5_INT_EN_8822B BIT(17)
+#define BIT_FS_GPIO4_INT_EN_8822B BIT(16)
+#define BIT_FS_GPIO3_INT_EN_8822B BIT(15)
+#define BIT_FS_GPIO2_INT_EN_8822B BIT(14)
+#define BIT_FS_GPIO1_INT_EN_8822B BIT(13)
+#define BIT_FS_GPIO0_INT_EN_8822B BIT(12)
+#define BIT_FS_HCI_SUS_EN_8822B BIT(11)
+#define BIT_FS_HCI_RES_EN_8822B BIT(10)
+#define BIT_FS_HCI_RESET_EN_8822B BIT(9)
+#define BIT_FS_BTON_STS_UPDATE_MSK_EN_8822B BIT(7)
+#define BIT_ACT2RECOVERY_INT_EN_V1_8822B BIT(6)
+#define BIT_GEN1GEN2_SWITCH_8822B BIT(5)
+#define BIT_HCI_TXDMA_REQ_HIMR_8822B BIT(4)
+#define BIT_FS_32K_LEAVE_SETTING_MAK_8822B BIT(3)
+#define BIT_FS_32K_ENTER_SETTING_MAK_8822B BIT(2)
+#define BIT_FS_USB_LPMRSM_MSK_8822B BIT(1)
+#define BIT_FS_USB_LPMINT_MSK_8822B BIT(0)
+
+/* 2 REG_FSISR_8822B */
+#define BIT_FS_PDNINT_8822B BIT(31)
+#define BIT_FS_SPS_OCP_INT_8822B BIT(29)
+#define BIT_FS_PWMERR_INT_8822B BIT(28)
+#define BIT_FS_GPIOF_INT_8822B BIT(27)
+#define BIT_FS_GPIOE_INT_8822B BIT(26)
+#define BIT_FS_GPIOD_INT_8822B BIT(25)
+#define BIT_FS_GPIOC_INT_8822B BIT(24)
+#define BIT_FS_GPIOB_INT_8822B BIT(23)
+#define BIT_FS_GPIOA_INT_8822B BIT(22)
+#define BIT_FS_GPIO9_INT_8822B BIT(21)
+#define BIT_FS_GPIO8_INT_8822B BIT(20)
+#define BIT_FS_GPIO7_INT_8822B BIT(19)
+#define BIT_FS_GPIO6_INT_8822B BIT(18)
+#define BIT_FS_GPIO5_INT_8822B BIT(17)
+#define BIT_FS_GPIO4_INT_8822B BIT(16)
+#define BIT_FS_GPIO3_INT_8822B BIT(15)
+#define BIT_FS_GPIO2_INT_8822B BIT(14)
+#define BIT_FS_GPIO1_INT_8822B BIT(13)
+#define BIT_FS_GPIO0_INT_8822B BIT(12)
+#define BIT_FS_HCI_SUS_INT_8822B BIT(11)
+#define BIT_FS_HCI_RES_INT_8822B BIT(10)
+#define BIT_FS_HCI_RESET_INT_8822B BIT(9)
+#define BIT_ACT2RECOVERY_8822B BIT(6)
+#define BIT_GEN1GEN2_SWITCH_8822B BIT(5)
+#define BIT_HCI_TXDMA_REQ_HISR_8822B BIT(4)
+#define BIT_FS_32K_LEAVE_SETTING_INT_8822B BIT(3)
+#define BIT_FS_32K_ENTER_SETTING_INT_8822B BIT(2)
+#define BIT_FS_USB_LPMRSM_INT_8822B BIT(1)
+#define BIT_FS_USB_LPMINT_INT_8822B BIT(0)
+
+/* 2 REG_HSIMR_8822B */
+#define BIT_GPIOF_INT_EN_8822B BIT(31)
+#define BIT_GPIOE_INT_EN_8822B BIT(30)
+#define BIT_GPIOD_INT_EN_8822B BIT(29)
+#define BIT_GPIOC_INT_EN_8822B BIT(28)
+#define BIT_GPIOB_INT_EN_8822B BIT(27)
+#define BIT_GPIOA_INT_EN_8822B BIT(26)
+#define BIT_GPIO9_INT_EN_8822B BIT(25)
+#define BIT_GPIO8_INT_EN_8822B BIT(24)
+#define BIT_GPIO7_INT_EN_8822B BIT(23)
+#define BIT_GPIO6_INT_EN_8822B BIT(22)
+#define BIT_GPIO5_INT_EN_8822B BIT(21)
+#define BIT_GPIO4_INT_EN_8822B BIT(20)
+#define BIT_GPIO3_INT_EN_8822B BIT(19)
+#define BIT_GPIO2_INT_EN_V1_8822B BIT(16)
+#define BIT_GPIO1_INT_EN_8822B BIT(17)
+#define BIT_GPIO0_INT_EN_8822B BIT(16)
+#define BIT_PDNINT_EN_8822B BIT(7)
+#define BIT_RON_INT_EN_8822B BIT(6)
+#define BIT_SPS_OCP_INT_EN_8822B BIT(5)
+#define BIT_GPIO15_0_INT_EN_8822B BIT(0)
+
+/* 2 REG_HSISR_8822B */
+#define BIT_GPIOF_INT_8822B BIT(31)
+#define BIT_GPIOE_INT_8822B BIT(30)
+#define BIT_GPIOD_INT_8822B BIT(29)
+#define BIT_GPIOC_INT_8822B BIT(28)
+#define BIT_GPIOB_INT_8822B BIT(27)
+#define BIT_GPIOA_INT_8822B BIT(26)
+#define BIT_GPIO9_INT_8822B BIT(25)
+#define BIT_GPIO8_INT_8822B BIT(24)
+#define BIT_GPIO7_INT_8822B BIT(23)
+#define BIT_GPIO6_INT_8822B BIT(22)
+#define BIT_GPIO5_INT_8822B BIT(21)
+#define BIT_GPIO4_INT_8822B BIT(20)
+#define BIT_GPIO3_INT_8822B BIT(19)
+#define BIT_GPIO2_INT_V1_8822B BIT(16)
+#define BIT_GPIO1_INT_8822B BIT(17)
+#define BIT_GPIO0_INT_8822B BIT(16)
+#define BIT_PDNINT_8822B BIT(7)
+#define BIT_RON_INT_8822B BIT(6)
+#define BIT_SPS_OCP_INT_8822B BIT(5)
+#define BIT_GPIO15_0_INT_8822B BIT(0)
+
+/* 2 REG_GPIO_EXT_CTRL_8822B */
+
+#define BIT_SHIFT_GPIO_MOD_15_TO_8_8822B 24
+#define BIT_MASK_GPIO_MOD_15_TO_8_8822B 0xff
+#define BIT_GPIO_MOD_15_TO_8_8822B(x)                                          \
+	(((x) & BIT_MASK_GPIO_MOD_15_TO_8_8822B)                               \
+	 << BIT_SHIFT_GPIO_MOD_15_TO_8_8822B)
+#define BIT_GET_GPIO_MOD_15_TO_8_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_GPIO_MOD_15_TO_8_8822B) &                           \
+	 BIT_MASK_GPIO_MOD_15_TO_8_8822B)
+
+#define BIT_SHIFT_GPIO_IO_SEL_15_TO_8_8822B 16
+#define BIT_MASK_GPIO_IO_SEL_15_TO_8_8822B 0xff
+#define BIT_GPIO_IO_SEL_15_TO_8_8822B(x)                                       \
+	(((x) & BIT_MASK_GPIO_IO_SEL_15_TO_8_8822B)                            \
+	 << BIT_SHIFT_GPIO_IO_SEL_15_TO_8_8822B)
+#define BIT_GET_GPIO_IO_SEL_15_TO_8_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_GPIO_IO_SEL_15_TO_8_8822B) &                        \
+	 BIT_MASK_GPIO_IO_SEL_15_TO_8_8822B)
+
+#define BIT_SHIFT_GPIO_OUT_15_TO_8_8822B 8
+#define BIT_MASK_GPIO_OUT_15_TO_8_8822B 0xff
+#define BIT_GPIO_OUT_15_TO_8_8822B(x)                                          \
+	(((x) & BIT_MASK_GPIO_OUT_15_TO_8_8822B)                               \
+	 << BIT_SHIFT_GPIO_OUT_15_TO_8_8822B)
+#define BIT_GET_GPIO_OUT_15_TO_8_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_GPIO_OUT_15_TO_8_8822B) &                           \
+	 BIT_MASK_GPIO_OUT_15_TO_8_8822B)
+
+#define BIT_SHIFT_GPIO_IN_15_TO_8_8822B 0
+#define BIT_MASK_GPIO_IN_15_TO_8_8822B 0xff
+#define BIT_GPIO_IN_15_TO_8_8822B(x)                                           \
+	(((x) & BIT_MASK_GPIO_IN_15_TO_8_8822B)                                \
+	 << BIT_SHIFT_GPIO_IN_15_TO_8_8822B)
+#define BIT_GET_GPIO_IN_15_TO_8_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_GPIO_IN_15_TO_8_8822B) &                            \
+	 BIT_MASK_GPIO_IN_15_TO_8_8822B)
+
+/* 2 REG_PAD_CTRL1_8822B */
+#define BIT_PAPE_WLBT_SEL_8822B BIT(29)
+#define BIT_LNAON_WLBT_SEL_8822B BIT(28)
+#define BIT_BTGP_GPG3_FEN_8822B BIT(26)
+#define BIT_BTGP_GPG2_FEN_8822B BIT(25)
+#define BIT_BTGP_JTAG_EN_8822B BIT(24)
+#define BIT_XTAL_CLK_EXTARNAL_EN_8822B BIT(23)
+#define BIT_BTGP_UART0_EN_8822B BIT(22)
+#define BIT_BTGP_UART1_EN_8822B BIT(21)
+#define BIT_BTGP_SPI_EN_8822B BIT(20)
+#define BIT_BTGP_GPIO_E2_8822B BIT(19)
+#define BIT_BTGP_GPIO_EN_8822B BIT(18)
+
+#define BIT_SHIFT_BTGP_GPIO_SL_8822B 16
+#define BIT_MASK_BTGP_GPIO_SL_8822B 0x3
+#define BIT_BTGP_GPIO_SL_8822B(x)                                              \
+	(((x) & BIT_MASK_BTGP_GPIO_SL_8822B) << BIT_SHIFT_BTGP_GPIO_SL_8822B)
+#define BIT_GET_BTGP_GPIO_SL_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_BTGP_GPIO_SL_8822B) & BIT_MASK_BTGP_GPIO_SL_8822B)
+
+#define BIT_PAD_SDIO_SR_8822B BIT(14)
+#define BIT_GPIO14_OUTPUT_PL_8822B BIT(13)
+#define BIT_HOST_WAKE_PAD_PULL_EN_8822B BIT(12)
+#define BIT_HOST_WAKE_PAD_SL_8822B BIT(11)
+#define BIT_PAD_LNAON_SR_8822B BIT(10)
+#define BIT_PAD_LNAON_E2_8822B BIT(9)
+#define BIT_SW_LNAON_G_SEL_DATA_8822B BIT(8)
+#define BIT_SW_LNAON_A_SEL_DATA_8822B BIT(7)
+#define BIT_PAD_PAPE_SR_8822B BIT(6)
+#define BIT_PAD_PAPE_E2_8822B BIT(5)
+#define BIT_SW_PAPE_G_SEL_DATA_8822B BIT(4)
+#define BIT_SW_PAPE_A_SEL_DATA_8822B BIT(3)
+#define BIT_PAD_DPDT_SR_8822B BIT(2)
+#define BIT_PAD_DPDT_PAD_E2_8822B BIT(1)
+#define BIT_SW_DPDT_SEL_DATA_8822B BIT(0)
+
+/* 2 REG_WL_BT_PWR_CTRL_8822B */
+#define BIT_ISO_BD2PP_8822B BIT(31)
+#define BIT_LDOV12B_EN_8822B BIT(30)
+#define BIT_CKEN_BTGPS_8822B BIT(29)
+#define BIT_FEN_BTGPS_8822B BIT(28)
+#define BIT_BTCPU_BOOTSEL_8822B BIT(27)
+#define BIT_SPI_SPEEDUP_8822B BIT(26)
+#define BIT_DEVWAKE_PAD_TYPE_SEL_8822B BIT(24)
+#define BIT_CLKREQ_PAD_TYPE_SEL_8822B BIT(23)
+#define BIT_ISO_BTPON2PP_8822B BIT(22)
+#define BIT_BT_HWROF_EN_8822B BIT(19)
+#define BIT_BT_FUNC_EN_8822B BIT(18)
+#define BIT_BT_HWPDN_SL_8822B BIT(17)
+#define BIT_BT_DISN_EN_8822B BIT(16)
+#define BIT_BT_PDN_PULL_EN_8822B BIT(15)
+#define BIT_WL_PDN_PULL_EN_8822B BIT(14)
+#define BIT_EXTERNAL_REQUEST_PL_8822B BIT(13)
+#define BIT_GPIO0_2_3_PULL_LOW_EN_8822B BIT(12)
+#define BIT_ISO_BA2PP_8822B BIT(11)
+#define BIT_BT_AFE_LDO_EN_8822B BIT(10)
+#define BIT_BT_AFE_PLL_EN_8822B BIT(9)
+#define BIT_BT_DIG_CLK_EN_8822B BIT(8)
+#define BIT_WL_DRV_EXIST_IDX_8822B BIT(5)
+#define BIT_DOP_EHPAD_8822B BIT(4)
+#define BIT_WL_HWROF_EN_8822B BIT(3)
+#define BIT_WL_FUNC_EN_8822B BIT(2)
+#define BIT_WL_HWPDN_SL_8822B BIT(1)
+#define BIT_WL_HWPDN_EN_8822B BIT(0)
+
+/* 2 REG_SDM_DEBUG_8822B */
+
+#define BIT_SHIFT_WLCLK_PHASE_8822B 0
+#define BIT_MASK_WLCLK_PHASE_8822B 0x1f
+#define BIT_WLCLK_PHASE_8822B(x)                                               \
+	(((x) & BIT_MASK_WLCLK_PHASE_8822B) << BIT_SHIFT_WLCLK_PHASE_8822B)
+#define BIT_GET_WLCLK_PHASE_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_WLCLK_PHASE_8822B) & BIT_MASK_WLCLK_PHASE_8822B)
+
+/* 2 REG_SYS_SDIO_CTRL_8822B */
+#define BIT_DBG_GNT_WL_BT_8822B BIT(27)
+#define BIT_LTE_MUX_CTRL_PATH_8822B BIT(26)
+#define BIT_LTE_COEX_UART_8822B BIT(25)
+#define BIT_3W_LTE_WL_GPIO_8822B BIT(24)
+#define BIT_SDIO_INT_POLARITY_8822B BIT(19)
+#define BIT_SDIO_INT_8822B BIT(18)
+#define BIT_SDIO_OFF_EN_8822B BIT(17)
+#define BIT_SDIO_ON_EN_8822B BIT(16)
+#define BIT_PCIE_WAIT_TIMEOUT_EVENT_8822B BIT(10)
+#define BIT_PCIE_WAIT_TIME_8822B BIT(9)
+#define BIT_MPCIE_REFCLK_XTAL_SEL_8822B BIT(8)
+
+/* 2 REG_HCI_OPT_CTRL_8822B */
+
+#define BIT_SHIFT_TSFT_SEL_8822B 29
+#define BIT_MASK_TSFT_SEL_8822B 0x7
+#define BIT_TSFT_SEL_8822B(x)                                                  \
+	(((x) & BIT_MASK_TSFT_SEL_8822B) << BIT_SHIFT_TSFT_SEL_8822B)
+#define BIT_GET_TSFT_SEL_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_TSFT_SEL_8822B) & BIT_MASK_TSFT_SEL_8822B)
+
+#define BIT_USB_HOST_PWR_OFF_EN_8822B BIT(12)
+#define BIT_SYM_LPS_BLOCK_EN_8822B BIT(11)
+#define BIT_USB_LPM_ACT_EN_8822B BIT(10)
+#define BIT_USB_LPM_NY_8822B BIT(9)
+#define BIT_USB_SUS_DIS_8822B BIT(8)
+
+#define BIT_SHIFT_SDIO_PAD_E_8822B 5
+#define BIT_MASK_SDIO_PAD_E_8822B 0x7
+#define BIT_SDIO_PAD_E_8822B(x)                                                \
+	(((x) & BIT_MASK_SDIO_PAD_E_8822B) << BIT_SHIFT_SDIO_PAD_E_8822B)
+#define BIT_GET_SDIO_PAD_E_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_SDIO_PAD_E_8822B) & BIT_MASK_SDIO_PAD_E_8822B)
+
+#define BIT_USB_LPPLL_EN_8822B BIT(4)
+#define BIT_ROP_SW15_8822B BIT(2)
+#define BIT_PCI_CKRDY_OPT_8822B BIT(1)
+#define BIT_PCI_VAUX_EN_8822B BIT(0)
+
+/* 2 REG_AFE_CTRL4_8822B */
+
+/* 2 REG_LDO_SWR_CTRL_8822B */
+#define BIT_ZCD_HW_AUTO_EN_8822B BIT(27)
+#define BIT_ZCD_REGSEL_8822B BIT(26)
+
+#define BIT_SHIFT_AUTO_ZCD_IN_CODE_8822B 21
+#define BIT_MASK_AUTO_ZCD_IN_CODE_8822B 0x1f
+#define BIT_AUTO_ZCD_IN_CODE_8822B(x)                                          \
+	(((x) & BIT_MASK_AUTO_ZCD_IN_CODE_8822B)                               \
+	 << BIT_SHIFT_AUTO_ZCD_IN_CODE_8822B)
+#define BIT_GET_AUTO_ZCD_IN_CODE_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_AUTO_ZCD_IN_CODE_8822B) &                           \
+	 BIT_MASK_AUTO_ZCD_IN_CODE_8822B)
+
+#define BIT_SHIFT_ZCD_CODE_IN_L_8822B 16
+#define BIT_MASK_ZCD_CODE_IN_L_8822B 0x1f
+#define BIT_ZCD_CODE_IN_L_8822B(x)                                             \
+	(((x) & BIT_MASK_ZCD_CODE_IN_L_8822B) << BIT_SHIFT_ZCD_CODE_IN_L_8822B)
+#define BIT_GET_ZCD_CODE_IN_L_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_ZCD_CODE_IN_L_8822B) & BIT_MASK_ZCD_CODE_IN_L_8822B)
+
+#define BIT_SHIFT_LDO_HV5_DUMMY_8822B 14
+#define BIT_MASK_LDO_HV5_DUMMY_8822B 0x3
+#define BIT_LDO_HV5_DUMMY_8822B(x)                                             \
+	(((x) & BIT_MASK_LDO_HV5_DUMMY_8822B) << BIT_SHIFT_LDO_HV5_DUMMY_8822B)
+#define BIT_GET_LDO_HV5_DUMMY_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_LDO_HV5_DUMMY_8822B) & BIT_MASK_LDO_HV5_DUMMY_8822B)
+
+#define BIT_SHIFT_REG_VTUNE33_BIT0_TO_BIT1_8822B 12
+#define BIT_MASK_REG_VTUNE33_BIT0_TO_BIT1_8822B 0x3
+#define BIT_REG_VTUNE33_BIT0_TO_BIT1_8822B(x)                                  \
+	(((x) & BIT_MASK_REG_VTUNE33_BIT0_TO_BIT1_8822B)                       \
+	 << BIT_SHIFT_REG_VTUNE33_BIT0_TO_BIT1_8822B)
+#define BIT_GET_REG_VTUNE33_BIT0_TO_BIT1_8822B(x)                              \
+	(((x) >> BIT_SHIFT_REG_VTUNE33_BIT0_TO_BIT1_8822B) &                   \
+	 BIT_MASK_REG_VTUNE33_BIT0_TO_BIT1_8822B)
+
+#define BIT_SHIFT_REG_STANDBY33_BIT0_TO_BIT1_8822B 10
+#define BIT_MASK_REG_STANDBY33_BIT0_TO_BIT1_8822B 0x3
+#define BIT_REG_STANDBY33_BIT0_TO_BIT1_8822B(x)                                \
+	(((x) & BIT_MASK_REG_STANDBY33_BIT0_TO_BIT1_8822B)                     \
+	 << BIT_SHIFT_REG_STANDBY33_BIT0_TO_BIT1_8822B)
+#define BIT_GET_REG_STANDBY33_BIT0_TO_BIT1_8822B(x)                            \
+	(((x) >> BIT_SHIFT_REG_STANDBY33_BIT0_TO_BIT1_8822B) &                 \
+	 BIT_MASK_REG_STANDBY33_BIT0_TO_BIT1_8822B)
+
+#define BIT_SHIFT_REG_LOAD33_BIT0_TO_BIT1_8822B 8
+#define BIT_MASK_REG_LOAD33_BIT0_TO_BIT1_8822B 0x3
+#define BIT_REG_LOAD33_BIT0_TO_BIT1_8822B(x)                                   \
+	(((x) & BIT_MASK_REG_LOAD33_BIT0_TO_BIT1_8822B)                        \
+	 << BIT_SHIFT_REG_LOAD33_BIT0_TO_BIT1_8822B)
+#define BIT_GET_REG_LOAD33_BIT0_TO_BIT1_8822B(x)                               \
+	(((x) >> BIT_SHIFT_REG_LOAD33_BIT0_TO_BIT1_8822B) &                    \
+	 BIT_MASK_REG_LOAD33_BIT0_TO_BIT1_8822B)
+
+#define BIT_REG_BYPASS_L_8822B BIT(7)
+#define BIT_REG_LDOF_L_8822B BIT(6)
+#define BIT_REG_TYPE_L_V1_8822B BIT(5)
+#define BIT_ARENB_L_8822B BIT(3)
+
+#define BIT_SHIFT_CFC_L_8822B 1
+#define BIT_MASK_CFC_L_8822B 0x3
+#define BIT_CFC_L_8822B(x)                                                     \
+	(((x) & BIT_MASK_CFC_L_8822B) << BIT_SHIFT_CFC_L_8822B)
+#define BIT_GET_CFC_L_8822B(x)                                                 \
+	(((x) >> BIT_SHIFT_CFC_L_8822B) & BIT_MASK_CFC_L_8822B)
+
+#define BIT_REG_OCPS_L_V1_8822B BIT(0)
+
+/* 2 REG_MCUFW_CTRL_8822B */
+
+#define BIT_SHIFT_RPWM_8822B 24
+#define BIT_MASK_RPWM_8822B 0xff
+#define BIT_RPWM_8822B(x) (((x) & BIT_MASK_RPWM_8822B) << BIT_SHIFT_RPWM_8822B)
+#define BIT_GET_RPWM_8822B(x)                                                  \
+	(((x) >> BIT_SHIFT_RPWM_8822B) & BIT_MASK_RPWM_8822B)
+
+#define BIT_ANA_PORT_EN_8822B BIT(22)
+#define BIT_MAC_PORT_EN_8822B BIT(21)
+#define BIT_BOOT_FSPI_EN_8822B BIT(20)
+#define BIT_ROM_DLEN_8822B BIT(19)
+
+#define BIT_SHIFT_ROM_PGE_8822B 16
+#define BIT_MASK_ROM_PGE_8822B 0x7
+#define BIT_ROM_PGE_8822B(x)                                                   \
+	(((x) & BIT_MASK_ROM_PGE_8822B) << BIT_SHIFT_ROM_PGE_8822B)
+#define BIT_GET_ROM_PGE_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_ROM_PGE_8822B) & BIT_MASK_ROM_PGE_8822B)
+
+#define BIT_FW_INIT_RDY_8822B BIT(15)
+#define BIT_FW_DW_RDY_8822B BIT(14)
+
+#define BIT_SHIFT_CPU_CLK_SEL_8822B 12
+#define BIT_MASK_CPU_CLK_SEL_8822B 0x3
+#define BIT_CPU_CLK_SEL_8822B(x)                                               \
+	(((x) & BIT_MASK_CPU_CLK_SEL_8822B) << BIT_SHIFT_CPU_CLK_SEL_8822B)
+#define BIT_GET_CPU_CLK_SEL_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_CPU_CLK_SEL_8822B) & BIT_MASK_CPU_CLK_SEL_8822B)
+
+#define BIT_CCLK_CHG_MASK_8822B BIT(11)
+#define BIT_EMEM__TXBUF_CHKSUM_OK_8822B BIT(10)
+#define BIT_EMEM_TXBUF_DW_RDY_8822B BIT(9)
+#define BIT_EMEM_CHKSUM_OK_8822B BIT(8)
+#define BIT_EMEM_DW_OK_8822B BIT(7)
+#define BIT_DMEM_CHKSUM_OK_8822B BIT(6)
+#define BIT_DMEM_DW_OK_8822B BIT(5)
+#define BIT_IMEM_CHKSUM_OK_8822B BIT(4)
+#define BIT_IMEM_DW_OK_8822B BIT(3)
+#define BIT_IMEM_BOOT_LOAD_CHKSUM_OK_8822B BIT(2)
+#define BIT_IMEM_BOOT_LOAD_DW_OK_8822B BIT(1)
+#define BIT_MCUFWDL_EN_8822B BIT(0)
+
+/* 2 REG_MCU_TST_CFG_8822B */
+
+#define BIT_SHIFT_LBKTST_8822B 0
+#define BIT_MASK_LBKTST_8822B 0xffff
+#define BIT_LBKTST_8822B(x)                                                    \
+	(((x) & BIT_MASK_LBKTST_8822B) << BIT_SHIFT_LBKTST_8822B)
+#define BIT_GET_LBKTST_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_LBKTST_8822B) & BIT_MASK_LBKTST_8822B)
+
+/* 2 REG_HMEBOX_E0_E1_8822B */
+
+#define BIT_SHIFT_HOST_MSG_E1_8822B 16
+#define BIT_MASK_HOST_MSG_E1_8822B 0xffff
+#define BIT_HOST_MSG_E1_8822B(x)                                               \
+	(((x) & BIT_MASK_HOST_MSG_E1_8822B) << BIT_SHIFT_HOST_MSG_E1_8822B)
+#define BIT_GET_HOST_MSG_E1_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_HOST_MSG_E1_8822B) & BIT_MASK_HOST_MSG_E1_8822B)
+
+#define BIT_SHIFT_HOST_MSG_E0_8822B 0
+#define BIT_MASK_HOST_MSG_E0_8822B 0xffff
+#define BIT_HOST_MSG_E0_8822B(x)                                               \
+	(((x) & BIT_MASK_HOST_MSG_E0_8822B) << BIT_SHIFT_HOST_MSG_E0_8822B)
+#define BIT_GET_HOST_MSG_E0_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_HOST_MSG_E0_8822B) & BIT_MASK_HOST_MSG_E0_8822B)
+
+/* 2 REG_HMEBOX_E2_E3_8822B */
+
+#define BIT_SHIFT_HOST_MSG_E3_8822B 16
+#define BIT_MASK_HOST_MSG_E3_8822B 0xffff
+#define BIT_HOST_MSG_E3_8822B(x)                                               \
+	(((x) & BIT_MASK_HOST_MSG_E3_8822B) << BIT_SHIFT_HOST_MSG_E3_8822B)
+#define BIT_GET_HOST_MSG_E3_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_HOST_MSG_E3_8822B) & BIT_MASK_HOST_MSG_E3_8822B)
+
+#define BIT_SHIFT_HOST_MSG_E2_8822B 0
+#define BIT_MASK_HOST_MSG_E2_8822B 0xffff
+#define BIT_HOST_MSG_E2_8822B(x)                                               \
+	(((x) & BIT_MASK_HOST_MSG_E2_8822B) << BIT_SHIFT_HOST_MSG_E2_8822B)
+#define BIT_GET_HOST_MSG_E2_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_HOST_MSG_E2_8822B) & BIT_MASK_HOST_MSG_E2_8822B)
+
+/* 2 REG_WLLPS_CTRL_8822B */
+#define BIT_WLLPSOP_EABM_8822B BIT(31)
+#define BIT_WLLPSOP_ACKF_8822B BIT(30)
+#define BIT_WLLPSOP_DLDM_8822B BIT(29)
+#define BIT_WLLPSOP_ESWR_8822B BIT(28)
+#define BIT_WLLPSOP_PWMM_8822B BIT(27)
+#define BIT_WLLPSOP_EECK_8822B BIT(26)
+#define BIT_WLLPSOP_WLMACOFF_8822B BIT(25)
+#define BIT_WLLPSOP_EXTAL_8822B BIT(24)
+#define BIT_WL_SYNPON_VOLTSPDN_8822B BIT(23)
+#define BIT_WLLPSOP_WLBBOFF_8822B BIT(22)
+#define BIT_WLLPSOP_WLMEM_DS_8822B BIT(21)
+
+#define BIT_SHIFT_LPLDH12_VADJ_STEP_DN_8822B 12
+#define BIT_MASK_LPLDH12_VADJ_STEP_DN_8822B 0xf
+#define BIT_LPLDH12_VADJ_STEP_DN_8822B(x)                                      \
+	(((x) & BIT_MASK_LPLDH12_VADJ_STEP_DN_8822B)                           \
+	 << BIT_SHIFT_LPLDH12_VADJ_STEP_DN_8822B)
+#define BIT_GET_LPLDH12_VADJ_STEP_DN_8822B(x)                                  \
+	(((x) >> BIT_SHIFT_LPLDH12_VADJ_STEP_DN_8822B) &                       \
+	 BIT_MASK_LPLDH12_VADJ_STEP_DN_8822B)
+
+#define BIT_SHIFT_V15ADJ_L1_STEP_DN_8822B 8
+#define BIT_MASK_V15ADJ_L1_STEP_DN_8822B 0x7
+#define BIT_V15ADJ_L1_STEP_DN_8822B(x)                                         \
+	(((x) & BIT_MASK_V15ADJ_L1_STEP_DN_8822B)                              \
+	 << BIT_SHIFT_V15ADJ_L1_STEP_DN_8822B)
+#define BIT_GET_V15ADJ_L1_STEP_DN_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_V15ADJ_L1_STEP_DN_8822B) &                          \
+	 BIT_MASK_V15ADJ_L1_STEP_DN_8822B)
+
+#define BIT_REGU_32K_CLK_EN_8822B BIT(1)
+#define BIT_WL_LPS_EN_8822B BIT(0)
+
+/* 2 REG_AFE_CTRL5_8822B */
+#define BIT_BB_DBG_SEL_AFE_SDM_BIT0_8822B BIT(31)
+#define BIT_ORDER_SDM_8822B BIT(30)
+#define BIT_RFE_SEL_SDM_8822B BIT(29)
+
+#define BIT_SHIFT_REF_SEL_8822B 25
+#define BIT_MASK_REF_SEL_8822B 0xf
+#define BIT_REF_SEL_8822B(x)                                                   \
+	(((x) & BIT_MASK_REF_SEL_8822B) << BIT_SHIFT_REF_SEL_8822B)
+#define BIT_GET_REF_SEL_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_REF_SEL_8822B) & BIT_MASK_REF_SEL_8822B)
+
+#define BIT_SHIFT_F0F_SDM_8822B 12
+#define BIT_MASK_F0F_SDM_8822B 0x1fff
+#define BIT_F0F_SDM_8822B(x)                                                   \
+	(((x) & BIT_MASK_F0F_SDM_8822B) << BIT_SHIFT_F0F_SDM_8822B)
+#define BIT_GET_F0F_SDM_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_F0F_SDM_8822B) & BIT_MASK_F0F_SDM_8822B)
+
+#define BIT_SHIFT_F0N_SDM_8822B 9
+#define BIT_MASK_F0N_SDM_8822B 0x7
+#define BIT_F0N_SDM_8822B(x)                                                   \
+	(((x) & BIT_MASK_F0N_SDM_8822B) << BIT_SHIFT_F0N_SDM_8822B)
+#define BIT_GET_F0N_SDM_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_F0N_SDM_8822B) & BIT_MASK_F0N_SDM_8822B)
+
+#define BIT_SHIFT_DIVN_SDM_8822B 3
+#define BIT_MASK_DIVN_SDM_8822B 0x3f
+#define BIT_DIVN_SDM_8822B(x)                                                  \
+	(((x) & BIT_MASK_DIVN_SDM_8822B) << BIT_SHIFT_DIVN_SDM_8822B)
+#define BIT_GET_DIVN_SDM_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_DIVN_SDM_8822B) & BIT_MASK_DIVN_SDM_8822B)
+
+/* 2 REG_GPIO_DEBOUNCE_CTRL_8822B */
+#define BIT_WLGP_DBC1EN_8822B BIT(15)
+
+#define BIT_SHIFT_WLGP_DBC1_8822B 8
+#define BIT_MASK_WLGP_DBC1_8822B 0xf
+#define BIT_WLGP_DBC1_8822B(x)                                                 \
+	(((x) & BIT_MASK_WLGP_DBC1_8822B) << BIT_SHIFT_WLGP_DBC1_8822B)
+#define BIT_GET_WLGP_DBC1_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_WLGP_DBC1_8822B) & BIT_MASK_WLGP_DBC1_8822B)
+
+#define BIT_WLGP_DBC0EN_8822B BIT(7)
+
+#define BIT_SHIFT_WLGP_DBC0_8822B 0
+#define BIT_MASK_WLGP_DBC0_8822B 0xf
+#define BIT_WLGP_DBC0_8822B(x)                                                 \
+	(((x) & BIT_MASK_WLGP_DBC0_8822B) << BIT_SHIFT_WLGP_DBC0_8822B)
+#define BIT_GET_WLGP_DBC0_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_WLGP_DBC0_8822B) & BIT_MASK_WLGP_DBC0_8822B)
+
+/* 2 REG_RPWM2_8822B */
+
+#define BIT_SHIFT_RPWM2_8822B 16
+#define BIT_MASK_RPWM2_8822B 0xffff
+#define BIT_RPWM2_8822B(x)                                                     \
+	(((x) & BIT_MASK_RPWM2_8822B) << BIT_SHIFT_RPWM2_8822B)
+#define BIT_GET_RPWM2_8822B(x)                                                 \
+	(((x) >> BIT_SHIFT_RPWM2_8822B) & BIT_MASK_RPWM2_8822B)
+
+/* 2 REG_SYSON_FSM_MON_8822B */
+
+#define BIT_SHIFT_FSM_MON_SEL_8822B 24
+#define BIT_MASK_FSM_MON_SEL_8822B 0x7
+#define BIT_FSM_MON_SEL_8822B(x)                                               \
+	(((x) & BIT_MASK_FSM_MON_SEL_8822B) << BIT_SHIFT_FSM_MON_SEL_8822B)
+#define BIT_GET_FSM_MON_SEL_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_FSM_MON_SEL_8822B) & BIT_MASK_FSM_MON_SEL_8822B)
+
+#define BIT_DOP_ELDO_8822B BIT(23)
+#define BIT_FSM_MON_UPD_8822B BIT(15)
+
+#define BIT_SHIFT_FSM_PAR_8822B 0
+#define BIT_MASK_FSM_PAR_8822B 0x7fff
+#define BIT_FSM_PAR_8822B(x)                                                   \
+	(((x) & BIT_MASK_FSM_PAR_8822B) << BIT_SHIFT_FSM_PAR_8822B)
+#define BIT_GET_FSM_PAR_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_FSM_PAR_8822B) & BIT_MASK_FSM_PAR_8822B)
+
+/* 2 REG_AFE_CTRL6_8822B */
+
+#define BIT_SHIFT_BB_DBG_SEL_AFE_SDM_BIT3_1_8822B 0
+#define BIT_MASK_BB_DBG_SEL_AFE_SDM_BIT3_1_8822B 0x7
+#define BIT_BB_DBG_SEL_AFE_SDM_BIT3_1_8822B(x)                                 \
+	(((x) & BIT_MASK_BB_DBG_SEL_AFE_SDM_BIT3_1_8822B)                      \
+	 << BIT_SHIFT_BB_DBG_SEL_AFE_SDM_BIT3_1_8822B)
+#define BIT_GET_BB_DBG_SEL_AFE_SDM_BIT3_1_8822B(x)                             \
+	(((x) >> BIT_SHIFT_BB_DBG_SEL_AFE_SDM_BIT3_1_8822B) &                  \
+	 BIT_MASK_BB_DBG_SEL_AFE_SDM_BIT3_1_8822B)
+
+/* 2 REG_PMC_DBG_CTRL1_8822B */
+#define BIT_BT_INT_EN_8822B BIT(31)
+
+#define BIT_SHIFT_RD_WR_WIFI_BT_INFO_8822B 16
+#define BIT_MASK_RD_WR_WIFI_BT_INFO_8822B 0x7fff
+#define BIT_RD_WR_WIFI_BT_INFO_8822B(x)                                        \
+	(((x) & BIT_MASK_RD_WR_WIFI_BT_INFO_8822B)                             \
+	 << BIT_SHIFT_RD_WR_WIFI_BT_INFO_8822B)
+#define BIT_GET_RD_WR_WIFI_BT_INFO_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_RD_WR_WIFI_BT_INFO_8822B) &                         \
+	 BIT_MASK_RD_WR_WIFI_BT_INFO_8822B)
+
+#define BIT_PMC_WR_OVF_8822B BIT(8)
+
+#define BIT_SHIFT_WLPMC_ERRINT_8822B 0
+#define BIT_MASK_WLPMC_ERRINT_8822B 0xff
+#define BIT_WLPMC_ERRINT_8822B(x)                                              \
+	(((x) & BIT_MASK_WLPMC_ERRINT_8822B) << BIT_SHIFT_WLPMC_ERRINT_8822B)
+#define BIT_GET_WLPMC_ERRINT_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_WLPMC_ERRINT_8822B) & BIT_MASK_WLPMC_ERRINT_8822B)
+
+/* 2 REG_AFE_CTRL7_8822B */
+
+#define BIT_SHIFT_SEL_V_8822B 30
+#define BIT_MASK_SEL_V_8822B 0x3
+#define BIT_SEL_V_8822B(x)                                                     \
+	(((x) & BIT_MASK_SEL_V_8822B) << BIT_SHIFT_SEL_V_8822B)
+#define BIT_GET_SEL_V_8822B(x)                                                 \
+	(((x) >> BIT_SHIFT_SEL_V_8822B) & BIT_MASK_SEL_V_8822B)
+
+#define BIT_SEL_LDO_PC_8822B BIT(29)
+
+#define BIT_SHIFT_CK_MON_SEL_8822B 26
+#define BIT_MASK_CK_MON_SEL_8822B 0x7
+#define BIT_CK_MON_SEL_8822B(x)                                                \
+	(((x) & BIT_MASK_CK_MON_SEL_8822B) << BIT_SHIFT_CK_MON_SEL_8822B)
+#define BIT_GET_CK_MON_SEL_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_CK_MON_SEL_8822B) & BIT_MASK_CK_MON_SEL_8822B)
+
+#define BIT_CK_MON_EN_8822B BIT(25)
+#define BIT_FREF_EDGE_8822B BIT(24)
+#define BIT_CK320M_EN_8822B BIT(23)
+#define BIT_CK_5M_EN_8822B BIT(22)
+#define BIT_TESTEN_8822B BIT(21)
+
+/* 2 REG_HIMR0_8822B */
+#define BIT_TIMEOUT_INTERRUPT2_MASK_8822B BIT(31)
+#define BIT_TIMEOUT_INTERRUTP1_MASK_8822B BIT(30)
+#define BIT_PSTIMEOUT_MSK_8822B BIT(29)
+#define BIT_GTINT4_MSK_8822B BIT(28)
+#define BIT_GTINT3_MSK_8822B BIT(27)
+#define BIT_TXBCN0ERR_MSK_8822B BIT(26)
+#define BIT_TXBCN0OK_MSK_8822B BIT(25)
+#define BIT_TSF_BIT32_TOGGLE_MSK_8822B BIT(24)
+#define BIT_BCNDMAINT0_MSK_8822B BIT(20)
+#define BIT_BCNDERR0_MSK_8822B BIT(16)
+#define BIT_HSISR_IND_ON_INT_MSK_8822B BIT(15)
+#define BIT_BCNDMAINT_E_MSK_8822B BIT(14)
+#define BIT_CTWEND_MSK_8822B BIT(12)
+#define BIT_HISR1_IND_MSK_8822B BIT(11)
+#define BIT_C2HCMD_MSK_8822B BIT(10)
+#define BIT_CPWM2_MSK_8822B BIT(9)
+#define BIT_CPWM_MSK_8822B BIT(8)
+#define BIT_HIGHDOK_MSK_8822B BIT(7)
+#define BIT_MGTDOK_MSK_8822B BIT(6)
+#define BIT_BKDOK_MSK_8822B BIT(5)
+#define BIT_BEDOK_MSK_8822B BIT(4)
+#define BIT_VIDOK_MSK_8822B BIT(3)
+#define BIT_VODOK_MSK_8822B BIT(2)
+#define BIT_RDU_MSK_8822B BIT(1)
+#define BIT_RXOK_MSK_8822B BIT(0)
+
+/* 2 REG_HISR0_8822B */
+#define BIT_TIMEOUT_INTERRUPT2_8822B BIT(31)
+#define BIT_TIMEOUT_INTERRUTP1_8822B BIT(30)
+#define BIT_PSTIMEOUT_8822B BIT(29)
+#define BIT_GTINT4_8822B BIT(28)
+#define BIT_GTINT3_8822B BIT(27)
+#define BIT_TXBCN0ERR_8822B BIT(26)
+#define BIT_TXBCN0OK_8822B BIT(25)
+#define BIT_TSF_BIT32_TOGGLE_8822B BIT(24)
+#define BIT_BCNDMAINT0_8822B BIT(20)
+#define BIT_BCNDERR0_8822B BIT(16)
+#define BIT_HSISR_IND_ON_INT_8822B BIT(15)
+#define BIT_BCNDMAINT_E_8822B BIT(14)
+#define BIT_CTWEND_8822B BIT(12)
+#define BIT_HISR1_IND_INT_8822B BIT(11)
+#define BIT_C2HCMD_8822B BIT(10)
+#define BIT_CPWM2_8822B BIT(9)
+#define BIT_CPWM_8822B BIT(8)
+#define BIT_HIGHDOK_8822B BIT(7)
+#define BIT_MGTDOK_8822B BIT(6)
+#define BIT_BKDOK_8822B BIT(5)
+#define BIT_BEDOK_8822B BIT(4)
+#define BIT_VIDOK_8822B BIT(3)
+#define BIT_VODOK_8822B BIT(2)
+#define BIT_RDU_8822B BIT(1)
+#define BIT_RXOK_8822B BIT(0)
+
+/* 2 REG_HIMR1_8822B */
+#define BIT_TXFIFO_TH_INT_8822B BIT(30)
+#define BIT_BTON_STS_UPDATE_MASK_8822B BIT(29)
+#define BIT_MCU_ERR_MASK_8822B BIT(28)
+#define BIT_BCNDMAINT7__MSK_8822B BIT(27)
+#define BIT_BCNDMAINT6__MSK_8822B BIT(26)
+#define BIT_BCNDMAINT5__MSK_8822B BIT(25)
+#define BIT_BCNDMAINT4__MSK_8822B BIT(24)
+#define BIT_BCNDMAINT3_MSK_8822B BIT(23)
+#define BIT_BCNDMAINT2_MSK_8822B BIT(22)
+#define BIT_BCNDMAINT1_MSK_8822B BIT(21)
+#define BIT_BCNDERR7_MSK_8822B BIT(20)
+#define BIT_BCNDERR6_MSK_8822B BIT(19)
+#define BIT_BCNDERR5_MSK_8822B BIT(18)
+#define BIT_BCNDERR4_MSK_8822B BIT(17)
+#define BIT_BCNDERR3_MSK_8822B BIT(16)
+#define BIT_BCNDERR2_MSK_8822B BIT(15)
+#define BIT_BCNDERR1_MSK_8822B BIT(14)
+#define BIT_ATIMEND_E_MSK_8822B BIT(13)
+#define BIT_ATIMEND__MSK_8822B BIT(12)
+#define BIT_TXERR_MSK_8822B BIT(11)
+#define BIT_RXERR_MSK_8822B BIT(10)
+#define BIT_TXFOVW_MSK_8822B BIT(9)
+#define BIT_FOVW_MSK_8822B BIT(8)
+#define BIT_CPU_MGQ_TXDONE_MSK_8822B BIT(5)
+#define BIT_PS_TIMER_C_MSK_8822B BIT(4)
+#define BIT_PS_TIMER_B_MSK_8822B BIT(3)
+#define BIT_PS_TIMER_A_MSK_8822B BIT(2)
+#define BIT_CPUMGQ_TX_TIMER_MSK_8822B BIT(1)
+
+/* 2 REG_HISR1_8822B */
+#define BIT_TXFIFO_TH_INT_8822B BIT(30)
+#define BIT_BTON_STS_UPDATE_INT_8822B BIT(29)
+#define BIT_MCU_ERR_8822B BIT(28)
+#define BIT_BCNDMAINT7_8822B BIT(27)
+#define BIT_BCNDMAINT6_8822B BIT(26)
+#define BIT_BCNDMAINT5_8822B BIT(25)
+#define BIT_BCNDMAINT4_8822B BIT(24)
+#define BIT_BCNDMAINT3_8822B BIT(23)
+#define BIT_BCNDMAINT2_8822B BIT(22)
+#define BIT_BCNDMAINT1_8822B BIT(21)
+#define BIT_BCNDERR7_8822B BIT(20)
+#define BIT_BCNDERR6_8822B BIT(19)
+#define BIT_BCNDERR5_8822B BIT(18)
+#define BIT_BCNDERR4_8822B BIT(17)
+#define BIT_BCNDERR3_8822B BIT(16)
+#define BIT_BCNDERR2_8822B BIT(15)
+#define BIT_BCNDERR1_8822B BIT(14)
+#define BIT_ATIMEND_E_8822B BIT(13)
+#define BIT_ATIMEND_8822B BIT(12)
+#define BIT_TXERR_INT_8822B BIT(11)
+#define BIT_RXERR_INT_8822B BIT(10)
+#define BIT_TXFOVW_8822B BIT(9)
+#define BIT_FOVW_8822B BIT(8)
+#define BIT_CPU_MGQ_TXDONE_8822B BIT(5)
+#define BIT_PS_TIMER_C_8822B BIT(4)
+#define BIT_PS_TIMER_B_8822B BIT(3)
+#define BIT_PS_TIMER_A_8822B BIT(2)
+#define BIT_CPUMGQ_TX_TIMER_8822B BIT(1)
+
+/* 2 REG_DBG_PORT_SEL_8822B */
+
+#define BIT_SHIFT_DEBUG_ST_8822B 0
+#define BIT_MASK_DEBUG_ST_8822B 0xffffffffL
+#define BIT_DEBUG_ST_8822B(x)                                                  \
+	(((x) & BIT_MASK_DEBUG_ST_8822B) << BIT_SHIFT_DEBUG_ST_8822B)
+#define BIT_GET_DEBUG_ST_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_DEBUG_ST_8822B) & BIT_MASK_DEBUG_ST_8822B)
+
+/* 2 REG_PAD_CTRL2_8822B */
+#define BIT_USB3_USB2_TRANSITION_8822B BIT(20)
+
+#define BIT_SHIFT_USB23_SW_MODE_V1_8822B 18
+#define BIT_MASK_USB23_SW_MODE_V1_8822B 0x3
+#define BIT_USB23_SW_MODE_V1_8822B(x)                                          \
+	(((x) & BIT_MASK_USB23_SW_MODE_V1_8822B)                               \
+	 << BIT_SHIFT_USB23_SW_MODE_V1_8822B)
+#define BIT_GET_USB23_SW_MODE_V1_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_USB23_SW_MODE_V1_8822B) &                           \
+	 BIT_MASK_USB23_SW_MODE_V1_8822B)
+
+#define BIT_NO_PDN_CHIPOFF_V1_8822B BIT(17)
+#define BIT_RSM_EN_V1_8822B BIT(16)
+
+#define BIT_SHIFT_MATCH_CNT_8822B 8
+#define BIT_MASK_MATCH_CNT_8822B 0xff
+#define BIT_MATCH_CNT_8822B(x)                                                 \
+	(((x) & BIT_MASK_MATCH_CNT_8822B) << BIT_SHIFT_MATCH_CNT_8822B)
+#define BIT_GET_MATCH_CNT_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_MATCH_CNT_8822B) & BIT_MASK_MATCH_CNT_8822B)
+
+#define BIT_LD_B12V_EN_8822B BIT(7)
+#define BIT_EECS_IOSEL_V1_8822B BIT(6)
+#define BIT_EECS_DATA_O_V1_8822B BIT(5)
+#define BIT_EECS_DATA_I_V1_8822B BIT(4)
+#define BIT_EESK_IOSEL_V1_8822B BIT(2)
+#define BIT_EESK_DATA_O_V1_8822B BIT(1)
+#define BIT_EESK_DATA_I_V1_8822B BIT(0)
+
+/* 2 REG_NOT_VALID_8822B */
+
+/* 2 REG_PMC_DBG_CTRL2_8822B */
+
+#define BIT_SHIFT_EFUSE_BURN_GNT_8822B 24
+#define BIT_MASK_EFUSE_BURN_GNT_8822B 0xff
+#define BIT_EFUSE_BURN_GNT_8822B(x)                                            \
+	(((x) & BIT_MASK_EFUSE_BURN_GNT_8822B)                                 \
+	 << BIT_SHIFT_EFUSE_BURN_GNT_8822B)
+#define BIT_GET_EFUSE_BURN_GNT_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_EFUSE_BURN_GNT_8822B) &                             \
+	 BIT_MASK_EFUSE_BURN_GNT_8822B)
+
+#define BIT_STOP_WL_PMC_8822B BIT(9)
+#define BIT_STOP_SYM_PMC_8822B BIT(8)
+#define BIT_REG_RST_WLPMC_8822B BIT(5)
+#define BIT_REG_RST_PD12N_8822B BIT(4)
+#define BIT_SYSON_DIS_WLREG_WRMSK_8822B BIT(3)
+#define BIT_SYSON_DIS_PMCREG_WRMSK_8822B BIT(2)
+
+#define BIT_SHIFT_SYSON_REG_ARB_8822B 0
+#define BIT_MASK_SYSON_REG_ARB_8822B 0x3
+#define BIT_SYSON_REG_ARB_8822B(x)                                             \
+	(((x) & BIT_MASK_SYSON_REG_ARB_8822B) << BIT_SHIFT_SYSON_REG_ARB_8822B)
+#define BIT_GET_SYSON_REG_ARB_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_SYSON_REG_ARB_8822B) & BIT_MASK_SYSON_REG_ARB_8822B)
+
+/* 2 REG_BIST_CTRL_8822B */
+#define BIT_BIST_USB_DIS_8822B BIT(27)
+#define BIT_BIST_PCI_DIS_8822B BIT(26)
+#define BIT_BIST_BT_DIS_8822B BIT(25)
+#define BIT_BIST_WL_DIS_8822B BIT(24)
+
+#define BIT_SHIFT_BIST_RPT_SEL_8822B 16
+#define BIT_MASK_BIST_RPT_SEL_8822B 0xf
+#define BIT_BIST_RPT_SEL_8822B(x)                                              \
+	(((x) & BIT_MASK_BIST_RPT_SEL_8822B) << BIT_SHIFT_BIST_RPT_SEL_8822B)
+#define BIT_GET_BIST_RPT_SEL_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_BIST_RPT_SEL_8822B) & BIT_MASK_BIST_RPT_SEL_8822B)
+
+#define BIT_BIST_RESUME_PS_8822B BIT(4)
+#define BIT_BIST_RESUME_8822B BIT(3)
+#define BIT_BIST_NORMAL_8822B BIT(2)
+#define BIT_BIST_RSTN_8822B BIT(1)
+#define BIT_BIST_CLK_EN_8822B BIT(0)
+
+/* 2 REG_BIST_RPT_8822B */
+
+#define BIT_SHIFT_MBIST_REPORT_8822B 0
+#define BIT_MASK_MBIST_REPORT_8822B 0xffffffffL
+#define BIT_MBIST_REPORT_8822B(x)                                              \
+	(((x) & BIT_MASK_MBIST_REPORT_8822B) << BIT_SHIFT_MBIST_REPORT_8822B)
+#define BIT_GET_MBIST_REPORT_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_MBIST_REPORT_8822B) & BIT_MASK_MBIST_REPORT_8822B)
+
+/* 2 REG_MEM_CTRL_8822B */
+#define BIT_UMEM_RME_8822B BIT(31)
+
+#define BIT_SHIFT_BT_SPRAM_8822B 28
+#define BIT_MASK_BT_SPRAM_8822B 0x3
+#define BIT_BT_SPRAM_8822B(x)                                                  \
+	(((x) & BIT_MASK_BT_SPRAM_8822B) << BIT_SHIFT_BT_SPRAM_8822B)
+#define BIT_GET_BT_SPRAM_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_BT_SPRAM_8822B) & BIT_MASK_BT_SPRAM_8822B)
+
+#define BIT_SHIFT_BT_ROM_8822B 24
+#define BIT_MASK_BT_ROM_8822B 0xf
+#define BIT_BT_ROM_8822B(x)                                                    \
+	(((x) & BIT_MASK_BT_ROM_8822B) << BIT_SHIFT_BT_ROM_8822B)
+#define BIT_GET_BT_ROM_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_BT_ROM_8822B) & BIT_MASK_BT_ROM_8822B)
+
+#define BIT_SHIFT_PCI_DPRAM_8822B 10
+#define BIT_MASK_PCI_DPRAM_8822B 0x3
+#define BIT_PCI_DPRAM_8822B(x)                                                 \
+	(((x) & BIT_MASK_PCI_DPRAM_8822B) << BIT_SHIFT_PCI_DPRAM_8822B)
+#define BIT_GET_PCI_DPRAM_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_PCI_DPRAM_8822B) & BIT_MASK_PCI_DPRAM_8822B)
+
+#define BIT_SHIFT_PCI_SPRAM_8822B 8
+#define BIT_MASK_PCI_SPRAM_8822B 0x3
+#define BIT_PCI_SPRAM_8822B(x)                                                 \
+	(((x) & BIT_MASK_PCI_SPRAM_8822B) << BIT_SHIFT_PCI_SPRAM_8822B)
+#define BIT_GET_PCI_SPRAM_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_PCI_SPRAM_8822B) & BIT_MASK_PCI_SPRAM_8822B)
+
+#define BIT_SHIFT_USB_SPRAM_8822B 6
+#define BIT_MASK_USB_SPRAM_8822B 0x3
+#define BIT_USB_SPRAM_8822B(x)                                                 \
+	(((x) & BIT_MASK_USB_SPRAM_8822B) << BIT_SHIFT_USB_SPRAM_8822B)
+#define BIT_GET_USB_SPRAM_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_USB_SPRAM_8822B) & BIT_MASK_USB_SPRAM_8822B)
+
+#define BIT_SHIFT_USB_SPRF_8822B 4
+#define BIT_MASK_USB_SPRF_8822B 0x3
+#define BIT_USB_SPRF_8822B(x)                                                  \
+	(((x) & BIT_MASK_USB_SPRF_8822B) << BIT_SHIFT_USB_SPRF_8822B)
+#define BIT_GET_USB_SPRF_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_USB_SPRF_8822B) & BIT_MASK_USB_SPRF_8822B)
+
+#define BIT_SHIFT_MCU_ROM_8822B 0
+#define BIT_MASK_MCU_ROM_8822B 0xf
+#define BIT_MCU_ROM_8822B(x)                                                   \
+	(((x) & BIT_MASK_MCU_ROM_8822B) << BIT_SHIFT_MCU_ROM_8822B)
+#define BIT_GET_MCU_ROM_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_MCU_ROM_8822B) & BIT_MASK_MCU_ROM_8822B)
+
+/* 2 REG_AFE_CTRL8_8822B */
+#define BIT_SYN_AGPIO_8822B BIT(20)
+#define BIT_XTAL_LP_8822B BIT(4)
+#define BIT_XTAL_GM_SEP_8822B BIT(3)
+
+#define BIT_SHIFT_XTAL_SEL_TOK_8822B 0
+#define BIT_MASK_XTAL_SEL_TOK_8822B 0x7
+#define BIT_XTAL_SEL_TOK_8822B(x)                                              \
+	(((x) & BIT_MASK_XTAL_SEL_TOK_8822B) << BIT_SHIFT_XTAL_SEL_TOK_8822B)
+#define BIT_GET_XTAL_SEL_TOK_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_XTAL_SEL_TOK_8822B) & BIT_MASK_XTAL_SEL_TOK_8822B)
+
+/* 2 REG_USB_SIE_INTF_8822B */
+#define BIT_RD_SEL_8822B BIT(31)
+#define BIT_USB_SIE_INTF_WE_V1_8822B BIT(30)
+#define BIT_USB_SIE_INTF_BYIOREG_V1_8822B BIT(29)
+#define BIT_USB_SIE_SELECT_8822B BIT(28)
+
+#define BIT_SHIFT_USB_SIE_INTF_ADDR_V1_8822B 16
+#define BIT_MASK_USB_SIE_INTF_ADDR_V1_8822B 0x1ff
+#define BIT_USB_SIE_INTF_ADDR_V1_8822B(x)                                      \
+	(((x) & BIT_MASK_USB_SIE_INTF_ADDR_V1_8822B)                           \
+	 << BIT_SHIFT_USB_SIE_INTF_ADDR_V1_8822B)
+#define BIT_GET_USB_SIE_INTF_ADDR_V1_8822B(x)                                  \
+	(((x) >> BIT_SHIFT_USB_SIE_INTF_ADDR_V1_8822B) &                       \
+	 BIT_MASK_USB_SIE_INTF_ADDR_V1_8822B)
+
+#define BIT_SHIFT_USB_SIE_INTF_RD_8822B 8
+#define BIT_MASK_USB_SIE_INTF_RD_8822B 0xff
+#define BIT_USB_SIE_INTF_RD_8822B(x)                                           \
+	(((x) & BIT_MASK_USB_SIE_INTF_RD_8822B)                                \
+	 << BIT_SHIFT_USB_SIE_INTF_RD_8822B)
+#define BIT_GET_USB_SIE_INTF_RD_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_USB_SIE_INTF_RD_8822B) &                            \
+	 BIT_MASK_USB_SIE_INTF_RD_8822B)
+
+#define BIT_SHIFT_USB_SIE_INTF_WD_8822B 0
+#define BIT_MASK_USB_SIE_INTF_WD_8822B 0xff
+#define BIT_USB_SIE_INTF_WD_8822B(x)                                           \
+	(((x) & BIT_MASK_USB_SIE_INTF_WD_8822B)                                \
+	 << BIT_SHIFT_USB_SIE_INTF_WD_8822B)
+#define BIT_GET_USB_SIE_INTF_WD_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_USB_SIE_INTF_WD_8822B) &                            \
+	 BIT_MASK_USB_SIE_INTF_WD_8822B)
+
+/* 2 REG_PCIE_MIO_INTF_8822B */
+#define BIT_PCIE_MIO_BYIOREG_8822B BIT(13)
+#define BIT_PCIE_MIO_RE_8822B BIT(12)
+
+#define BIT_SHIFT_PCIE_MIO_WE_8822B 8
+#define BIT_MASK_PCIE_MIO_WE_8822B 0xf
+#define BIT_PCIE_MIO_WE_8822B(x)                                               \
+	(((x) & BIT_MASK_PCIE_MIO_WE_8822B) << BIT_SHIFT_PCIE_MIO_WE_8822B)
+#define BIT_GET_PCIE_MIO_WE_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_PCIE_MIO_WE_8822B) & BIT_MASK_PCIE_MIO_WE_8822B)
+
+#define BIT_SHIFT_PCIE_MIO_ADDR_8822B 0
+#define BIT_MASK_PCIE_MIO_ADDR_8822B 0xff
+#define BIT_PCIE_MIO_ADDR_8822B(x)                                             \
+	(((x) & BIT_MASK_PCIE_MIO_ADDR_8822B) << BIT_SHIFT_PCIE_MIO_ADDR_8822B)
+#define BIT_GET_PCIE_MIO_ADDR_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_PCIE_MIO_ADDR_8822B) & BIT_MASK_PCIE_MIO_ADDR_8822B)
+
+/* 2 REG_PCIE_MIO_INTD_8822B */
+
+#define BIT_SHIFT_PCIE_MIO_DATA_8822B 0
+#define BIT_MASK_PCIE_MIO_DATA_8822B 0xffffffffL
+#define BIT_PCIE_MIO_DATA_8822B(x)                                             \
+	(((x) & BIT_MASK_PCIE_MIO_DATA_8822B) << BIT_SHIFT_PCIE_MIO_DATA_8822B)
+#define BIT_GET_PCIE_MIO_DATA_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_PCIE_MIO_DATA_8822B) & BIT_MASK_PCIE_MIO_DATA_8822B)
+
+/* 2 REG_WLRF1_8822B */
+
+#define BIT_SHIFT_WLRF1_CTRL_8822B 24
+#define BIT_MASK_WLRF1_CTRL_8822B 0xff
+#define BIT_WLRF1_CTRL_8822B(x)                                                \
+	(((x) & BIT_MASK_WLRF1_CTRL_8822B) << BIT_SHIFT_WLRF1_CTRL_8822B)
+#define BIT_GET_WLRF1_CTRL_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_WLRF1_CTRL_8822B) & BIT_MASK_WLRF1_CTRL_8822B)
+
+/* 2 REG_SYS_CFG1_8822B */
+
+#define BIT_SHIFT_TRP_ICFG_8822B 28
+#define BIT_MASK_TRP_ICFG_8822B 0xf
+#define BIT_TRP_ICFG_8822B(x)                                                  \
+	(((x) & BIT_MASK_TRP_ICFG_8822B) << BIT_SHIFT_TRP_ICFG_8822B)
+#define BIT_GET_TRP_ICFG_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_TRP_ICFG_8822B) & BIT_MASK_TRP_ICFG_8822B)
+
+#define BIT_RF_TYPE_ID_8822B BIT(27)
+#define BIT_BD_HCI_SEL_8822B BIT(26)
+#define BIT_BD_PKG_SEL_8822B BIT(25)
+#define BIT_SPSLDO_SEL_8822B BIT(24)
+#define BIT_RTL_ID_8822B BIT(23)
+#define BIT_PAD_HWPD_IDN_8822B BIT(22)
+#define BIT_TESTMODE_8822B BIT(20)
+
+#define BIT_SHIFT_VENDOR_ID_8822B 16
+#define BIT_MASK_VENDOR_ID_8822B 0xf
+#define BIT_VENDOR_ID_8822B(x)                                                 \
+	(((x) & BIT_MASK_VENDOR_ID_8822B) << BIT_SHIFT_VENDOR_ID_8822B)
+#define BIT_GET_VENDOR_ID_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_VENDOR_ID_8822B) & BIT_MASK_VENDOR_ID_8822B)
+
+#define BIT_SHIFT_CHIP_VER_8822B 12
+#define BIT_MASK_CHIP_VER_8822B 0xf
+#define BIT_CHIP_VER_8822B(x)                                                  \
+	(((x) & BIT_MASK_CHIP_VER_8822B) << BIT_SHIFT_CHIP_VER_8822B)
+#define BIT_GET_CHIP_VER_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_CHIP_VER_8822B) & BIT_MASK_CHIP_VER_8822B)
+
+#define BIT_BD_MAC3_8822B BIT(11)
+#define BIT_BD_MAC1_8822B BIT(10)
+#define BIT_BD_MAC2_8822B BIT(9)
+#define BIT_SIC_IDLE_8822B BIT(8)
+#define BIT_SW_OFFLOAD_EN_8822B BIT(7)
+#define BIT_OCP_SHUTDN_8822B BIT(6)
+#define BIT_V15_VLD_8822B BIT(5)
+#define BIT_PCIRSTB_8822B BIT(4)
+#define BIT_PCLK_VLD_8822B BIT(3)
+#define BIT_UCLK_VLD_8822B BIT(2)
+#define BIT_ACLK_VLD_8822B BIT(1)
+#define BIT_XCLK_VLD_8822B BIT(0)
+
+/* 2 REG_SYS_STATUS1_8822B */
+
+#define BIT_SHIFT_RF_RL_ID_8822B 28
+#define BIT_MASK_RF_RL_ID_8822B 0xf
+#define BIT_RF_RL_ID_8822B(x)                                                  \
+	(((x) & BIT_MASK_RF_RL_ID_8822B) << BIT_SHIFT_RF_RL_ID_8822B)
+#define BIT_GET_RF_RL_ID_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_RF_RL_ID_8822B) & BIT_MASK_RF_RL_ID_8822B)
+
+#define BIT_HPHY_ICFG_8822B BIT(19)
+
+#define BIT_SHIFT_SEL_0XC0_8822B 16
+#define BIT_MASK_SEL_0XC0_8822B 0x3
+#define BIT_SEL_0XC0_8822B(x)                                                  \
+	(((x) & BIT_MASK_SEL_0XC0_8822B) << BIT_SHIFT_SEL_0XC0_8822B)
+#define BIT_GET_SEL_0XC0_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_SEL_0XC0_8822B) & BIT_MASK_SEL_0XC0_8822B)
+
+#define BIT_SHIFT_HCI_SEL_V3_8822B 12
+#define BIT_MASK_HCI_SEL_V3_8822B 0x7
+#define BIT_HCI_SEL_V3_8822B(x)                                                \
+	(((x) & BIT_MASK_HCI_SEL_V3_8822B) << BIT_SHIFT_HCI_SEL_V3_8822B)
+#define BIT_GET_HCI_SEL_V3_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_HCI_SEL_V3_8822B) & BIT_MASK_HCI_SEL_V3_8822B)
+
+#define BIT_USB_OPERATION_MODE_8822B BIT(10)
+#define BIT_BT_PDN_8822B BIT(9)
+#define BIT_AUTO_WLPON_8822B BIT(8)
+#define BIT_WL_MODE_8822B BIT(7)
+#define BIT_PKG_SEL_HCI_8822B BIT(6)
+
+#define BIT_SHIFT_PAD_HCI_SEL_V1_8822B 3
+#define BIT_MASK_PAD_HCI_SEL_V1_8822B 0x7
+#define BIT_PAD_HCI_SEL_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_PAD_HCI_SEL_V1_8822B)                                 \
+	 << BIT_SHIFT_PAD_HCI_SEL_V1_8822B)
+#define BIT_GET_PAD_HCI_SEL_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_PAD_HCI_SEL_V1_8822B) &                             \
+	 BIT_MASK_PAD_HCI_SEL_V1_8822B)
+
+#define BIT_SHIFT_EFS_HCI_SEL_V1_8822B 0
+#define BIT_MASK_EFS_HCI_SEL_V1_8822B 0x7
+#define BIT_EFS_HCI_SEL_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_EFS_HCI_SEL_V1_8822B)                                 \
+	 << BIT_SHIFT_EFS_HCI_SEL_V1_8822B)
+#define BIT_GET_EFS_HCI_SEL_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_EFS_HCI_SEL_V1_8822B) &                             \
+	 BIT_MASK_EFS_HCI_SEL_V1_8822B)
+
+/* 2 REG_SYS_STATUS2_8822B */
+#define BIT_SIO_ALDN_8822B BIT(19)
+#define BIT_USB_ALDN_8822B BIT(18)
+#define BIT_PCI_ALDN_8822B BIT(17)
+#define BIT_SYS_ALDN_8822B BIT(16)
+
+#define BIT_SHIFT_EPVID1_8822B 8
+#define BIT_MASK_EPVID1_8822B 0xff
+#define BIT_EPVID1_8822B(x)                                                    \
+	(((x) & BIT_MASK_EPVID1_8822B) << BIT_SHIFT_EPVID1_8822B)
+#define BIT_GET_EPVID1_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_EPVID1_8822B) & BIT_MASK_EPVID1_8822B)
+
+#define BIT_SHIFT_EPVID0_8822B 0
+#define BIT_MASK_EPVID0_8822B 0xff
+#define BIT_EPVID0_8822B(x)                                                    \
+	(((x) & BIT_MASK_EPVID0_8822B) << BIT_SHIFT_EPVID0_8822B)
+#define BIT_GET_EPVID0_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_EPVID0_8822B) & BIT_MASK_EPVID0_8822B)
+
+/* 2 REG_SYS_CFG2_8822B */
+#define BIT_HCI_SEL_EMBEDDED_8822B BIT(8)
+
+#define BIT_SHIFT_HW_ID_8822B 0
+#define BIT_MASK_HW_ID_8822B 0xff
+#define BIT_HW_ID_8822B(x)                                                     \
+	(((x) & BIT_MASK_HW_ID_8822B) << BIT_SHIFT_HW_ID_8822B)
+#define BIT_GET_HW_ID_8822B(x)                                                 \
+	(((x) >> BIT_SHIFT_HW_ID_8822B) & BIT_MASK_HW_ID_8822B)
+
+/* 2 REG_SYS_CFG3_8822B */
+#define BIT_PWC_MA33V_8822B BIT(15)
+#define BIT_PWC_MA12V_8822B BIT(14)
+#define BIT_PWC_MD12V_8822B BIT(13)
+#define BIT_PWC_PD12V_8822B BIT(12)
+#define BIT_PWC_UD12V_8822B BIT(11)
+#define BIT_ISO_MA2MD_8822B BIT(1)
+#define BIT_ISO_MD2PP_8822B BIT(0)
+
+/* 2 REG_SYS_CFG4_8822B */
+
+/* 2 REG_SYS_CFG5_8822B */
+#define BIT_LPS_STATUS_8822B BIT(3)
+#define BIT_HCI_TXDMA_BUSY_8822B BIT(2)
+#define BIT_HCI_TXDMA_ALLOW_8822B BIT(1)
+#define BIT_FW_CTRL_HCI_TXDMA_EN_8822B BIT(0)
+
+/* 2 REG_CPU_DMEM_CON_8822B */
+#define BIT_WDT_OPT_IOWRAPPER_8822B BIT(19)
+#define BIT_ANA_PORT_IDLE_8822B BIT(18)
+#define BIT_MAC_PORT_IDLE_8822B BIT(17)
+#define BIT_WL_PLATFORM_RST_8822B BIT(16)
+#define BIT_WL_SECURITY_CLK_8822B BIT(15)
+
+#define BIT_SHIFT_CPU_DMEM_CON_8822B 0
+#define BIT_MASK_CPU_DMEM_CON_8822B 0xff
+#define BIT_CPU_DMEM_CON_8822B(x)                                              \
+	(((x) & BIT_MASK_CPU_DMEM_CON_8822B) << BIT_SHIFT_CPU_DMEM_CON_8822B)
+#define BIT_GET_CPU_DMEM_CON_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_CPU_DMEM_CON_8822B) & BIT_MASK_CPU_DMEM_CON_8822B)
+
+/* 2 REG_BOOT_REASON_8822B */
+
+#define BIT_SHIFT_BOOT_REASON_8822B 0
+#define BIT_MASK_BOOT_REASON_8822B 0x7
+#define BIT_BOOT_REASON_8822B(x)                                               \
+	(((x) & BIT_MASK_BOOT_REASON_8822B) << BIT_SHIFT_BOOT_REASON_8822B)
+#define BIT_GET_BOOT_REASON_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_BOOT_REASON_8822B) & BIT_MASK_BOOT_REASON_8822B)
+
+/* 2 REG_NFCPAD_CTRL_8822B */
+#define BIT_PAD_SHUTDW_8822B BIT(18)
+#define BIT_SYSON_NFC_PAD_8822B BIT(17)
+#define BIT_NFC_INT_PAD_CTRL_8822B BIT(16)
+#define BIT_NFC_RFDIS_PAD_CTRL_8822B BIT(15)
+#define BIT_NFC_CLK_PAD_CTRL_8822B BIT(14)
+#define BIT_NFC_DATA_PAD_CTRL_8822B BIT(13)
+#define BIT_NFC_PAD_PULL_CTRL_8822B BIT(12)
+
+#define BIT_SHIFT_NFCPAD_IO_SEL_8822B 8
+#define BIT_MASK_NFCPAD_IO_SEL_8822B 0xf
+#define BIT_NFCPAD_IO_SEL_8822B(x)                                             \
+	(((x) & BIT_MASK_NFCPAD_IO_SEL_8822B) << BIT_SHIFT_NFCPAD_IO_SEL_8822B)
+#define BIT_GET_NFCPAD_IO_SEL_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_NFCPAD_IO_SEL_8822B) & BIT_MASK_NFCPAD_IO_SEL_8822B)
+
+#define BIT_SHIFT_NFCPAD_OUT_8822B 4
+#define BIT_MASK_NFCPAD_OUT_8822B 0xf
+#define BIT_NFCPAD_OUT_8822B(x)                                                \
+	(((x) & BIT_MASK_NFCPAD_OUT_8822B) << BIT_SHIFT_NFCPAD_OUT_8822B)
+#define BIT_GET_NFCPAD_OUT_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_NFCPAD_OUT_8822B) & BIT_MASK_NFCPAD_OUT_8822B)
+
+#define BIT_SHIFT_NFCPAD_IN_8822B 0
+#define BIT_MASK_NFCPAD_IN_8822B 0xf
+#define BIT_NFCPAD_IN_8822B(x)                                                 \
+	(((x) & BIT_MASK_NFCPAD_IN_8822B) << BIT_SHIFT_NFCPAD_IN_8822B)
+#define BIT_GET_NFCPAD_IN_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_NFCPAD_IN_8822B) & BIT_MASK_NFCPAD_IN_8822B)
+
+/* 2 REG_HIMR2_8822B */
+#define BIT_BCNDMAINT_P4_MSK_8822B BIT(31)
+#define BIT_BCNDMAINT_P3_MSK_8822B BIT(30)
+#define BIT_BCNDMAINT_P2_MSK_8822B BIT(29)
+#define BIT_BCNDMAINT_P1_MSK_8822B BIT(28)
+#define BIT_ATIMEND7_MSK_8822B BIT(22)
+#define BIT_ATIMEND6_MSK_8822B BIT(21)
+#define BIT_ATIMEND5_MSK_8822B BIT(20)
+#define BIT_ATIMEND4_MSK_8822B BIT(19)
+#define BIT_ATIMEND3_MSK_8822B BIT(18)
+#define BIT_ATIMEND2_MSK_8822B BIT(17)
+#define BIT_ATIMEND1_MSK_8822B BIT(16)
+#define BIT_TXBCN7OK_MSK_8822B BIT(14)
+#define BIT_TXBCN6OK_MSK_8822B BIT(13)
+#define BIT_TXBCN5OK_MSK_8822B BIT(12)
+#define BIT_TXBCN4OK_MSK_8822B BIT(11)
+#define BIT_TXBCN3OK_MSK_8822B BIT(10)
+#define BIT_TXBCN2OK_MSK_8822B BIT(9)
+#define BIT_TXBCN1OK_MSK_V1_8822B BIT(8)
+#define BIT_TXBCN7ERR_MSK_8822B BIT(6)
+#define BIT_TXBCN6ERR_MSK_8822B BIT(5)
+#define BIT_TXBCN5ERR_MSK_8822B BIT(4)
+#define BIT_TXBCN4ERR_MSK_8822B BIT(3)
+#define BIT_TXBCN3ERR_MSK_8822B BIT(2)
+#define BIT_TXBCN2ERR_MSK_8822B BIT(1)
+#define BIT_TXBCN1ERR_MSK_V1_8822B BIT(0)
+
+/* 2 REG_HISR2_8822B */
+#define BIT_BCNDMAINT_P4_8822B BIT(31)
+#define BIT_BCNDMAINT_P3_8822B BIT(30)
+#define BIT_BCNDMAINT_P2_8822B BIT(29)
+#define BIT_BCNDMAINT_P1_8822B BIT(28)
+#define BIT_ATIMEND7_8822B BIT(22)
+#define BIT_ATIMEND6_8822B BIT(21)
+#define BIT_ATIMEND5_8822B BIT(20)
+#define BIT_ATIMEND4_8822B BIT(19)
+#define BIT_ATIMEND3_8822B BIT(18)
+#define BIT_ATIMEND2_8822B BIT(17)
+#define BIT_ATIMEND1_8822B BIT(16)
+#define BIT_TXBCN7OK_8822B BIT(14)
+#define BIT_TXBCN6OK_8822B BIT(13)
+#define BIT_TXBCN5OK_8822B BIT(12)
+#define BIT_TXBCN4OK_8822B BIT(11)
+#define BIT_TXBCN3OK_8822B BIT(10)
+#define BIT_TXBCN2OK_8822B BIT(9)
+#define BIT_TXBCN1OK_8822B BIT(8)
+#define BIT_TXBCN7ERR_8822B BIT(6)
+#define BIT_TXBCN6ERR_8822B BIT(5)
+#define BIT_TXBCN5ERR_8822B BIT(4)
+#define BIT_TXBCN4ERR_8822B BIT(3)
+#define BIT_TXBCN3ERR_8822B BIT(2)
+#define BIT_TXBCN2ERR_8822B BIT(1)
+#define BIT_TXBCN1ERR_8822B BIT(0)
+
+/* 2 REG_HIMR3_8822B */
+#define BIT_WDT_PLATFORM_INT_MSK_8822B BIT(18)
+#define BIT_WDT_CPU_INT_MSK_8822B BIT(17)
+#define BIT_SETH2CDOK_MASK_8822B BIT(16)
+#define BIT_H2C_CMD_FULL_MASK_8822B BIT(15)
+#define BIT_PWR_INT_127_MASK_8822B BIT(14)
+#define BIT_TXSHORTCUT_TXDESUPDATEOK_MASK_8822B BIT(13)
+#define BIT_TXSHORTCUT_BKUPDATEOK_MASK_8822B BIT(12)
+#define BIT_TXSHORTCUT_BEUPDATEOK_MASK_8822B BIT(11)
+#define BIT_TXSHORTCUT_VIUPDATEOK_MAS_8822B BIT(10)
+#define BIT_TXSHORTCUT_VOUPDATEOK_MASK_8822B BIT(9)
+#define BIT_PWR_INT_127_MASK_V1_8822B BIT(8)
+#define BIT_PWR_INT_126TO96_MASK_8822B BIT(7)
+#define BIT_PWR_INT_95TO64_MASK_8822B BIT(6)
+#define BIT_PWR_INT_63TO32_MASK_8822B BIT(5)
+#define BIT_PWR_INT_31TO0_MASK_8822B BIT(4)
+#define BIT_DDMA0_LP_INT_MSK_8822B BIT(1)
+#define BIT_DDMA0_HP_INT_MSK_8822B BIT(0)
+
+/* 2 REG_HISR3_8822B */
+#define BIT_WDT_PLATFORM_INT_8822B BIT(18)
+#define BIT_WDT_CPU_INT_8822B BIT(17)
+#define BIT_SETH2CDOK_8822B BIT(16)
+#define BIT_H2C_CMD_FULL_8822B BIT(15)
+#define BIT_PWR_INT_127_8822B BIT(14)
+#define BIT_TXSHORTCUT_TXDESUPDATEOK_8822B BIT(13)
+#define BIT_TXSHORTCUT_BKUPDATEOK_8822B BIT(12)
+#define BIT_TXSHORTCUT_BEUPDATEOK_8822B BIT(11)
+#define BIT_TXSHORTCUT_VIUPDATEOK_8822B BIT(10)
+#define BIT_TXSHORTCUT_VOUPDATEOK_8822B BIT(9)
+#define BIT_PWR_INT_127_V1_8822B BIT(8)
+#define BIT_PWR_INT_126TO96_8822B BIT(7)
+#define BIT_PWR_INT_95TO64_8822B BIT(6)
+#define BIT_PWR_INT_63TO32_8822B BIT(5)
+#define BIT_PWR_INT_31TO0_8822B BIT(4)
+#define BIT_DDMA0_LP_INT_8822B BIT(1)
+#define BIT_DDMA0_HP_INT_8822B BIT(0)
+
+/* 2 REG_SW_MDIO_8822B */
+#define BIT_DIS_TIMEOUT_IO_8822B BIT(24)
+
+/* 2 REG_SW_FLUSH_8822B */
+#define BIT_FLUSH_HOLDN_EN_8822B BIT(25)
+#define BIT_FLUSH_WR_EN_8822B BIT(24)
+#define BIT_SW_FLASH_CONTROL_8822B BIT(23)
+#define BIT_SW_FLASH_WEN_E_8822B BIT(19)
+#define BIT_SW_FLASH_HOLDN_E_8822B BIT(18)
+#define BIT_SW_FLASH_SO_E_8822B BIT(17)
+#define BIT_SW_FLASH_SI_E_8822B BIT(16)
+#define BIT_SW_FLASH_SK_O_8822B BIT(13)
+#define BIT_SW_FLASH_CEN_O_8822B BIT(12)
+#define BIT_SW_FLASH_WEN_O_8822B BIT(11)
+#define BIT_SW_FLASH_HOLDN_O_8822B BIT(10)
+#define BIT_SW_FLASH_SO_O_8822B BIT(9)
+#define BIT_SW_FLASH_SI_O_8822B BIT(8)
+#define BIT_SW_FLASH_WEN_I_8822B BIT(3)
+#define BIT_SW_FLASH_HOLDN_I_8822B BIT(2)
+#define BIT_SW_FLASH_SO_I_8822B BIT(1)
+#define BIT_SW_FLASH_SI_I_8822B BIT(0)
+
+/* 2 REG_H2C_PKT_READADDR_8822B */
+
+#define BIT_SHIFT_H2C_PKT_READADDR_8822B 0
+#define BIT_MASK_H2C_PKT_READADDR_8822B 0x3ffff
+#define BIT_H2C_PKT_READADDR_8822B(x)                                          \
+	(((x) & BIT_MASK_H2C_PKT_READADDR_8822B)                               \
+	 << BIT_SHIFT_H2C_PKT_READADDR_8822B)
+#define BIT_GET_H2C_PKT_READADDR_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_H2C_PKT_READADDR_8822B) &                           \
+	 BIT_MASK_H2C_PKT_READADDR_8822B)
+
+/* 2 REG_H2C_PKT_WRITEADDR_8822B */
+
+#define BIT_SHIFT_H2C_PKT_WRITEADDR_8822B 0
+#define BIT_MASK_H2C_PKT_WRITEADDR_8822B 0x3ffff
+#define BIT_H2C_PKT_WRITEADDR_8822B(x)                                         \
+	(((x) & BIT_MASK_H2C_PKT_WRITEADDR_8822B)                              \
+	 << BIT_SHIFT_H2C_PKT_WRITEADDR_8822B)
+#define BIT_GET_H2C_PKT_WRITEADDR_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_H2C_PKT_WRITEADDR_8822B) &                          \
+	 BIT_MASK_H2C_PKT_WRITEADDR_8822B)
+
+/* 2 REG_MEM_PWR_CRTL_8822B */
+#define BIT_MEM_BB_SD_8822B BIT(17)
+#define BIT_MEM_BB_DS_8822B BIT(16)
+#define BIT_MEM_BT_DS_8822B BIT(10)
+#define BIT_MEM_SDIO_LS_8822B BIT(9)
+#define BIT_MEM_SDIO_DS_8822B BIT(8)
+#define BIT_MEM_USB_LS_8822B BIT(7)
+#define BIT_MEM_USB_DS_8822B BIT(6)
+#define BIT_MEM_PCI_LS_8822B BIT(5)
+#define BIT_MEM_PCI_DS_8822B BIT(4)
+#define BIT_MEM_WLMAC_LS_8822B BIT(3)
+#define BIT_MEM_WLMAC_DS_8822B BIT(2)
+#define BIT_MEM_WLMCU_LS_8822B BIT(1)
+#define BIT_MEM_WLMCU_DS_8822B BIT(0)
+
+/* 2 REG_FW_DBG0_8822B */
+
+#define BIT_SHIFT_FW_DBG0_8822B 0
+#define BIT_MASK_FW_DBG0_8822B 0xffffffffL
+#define BIT_FW_DBG0_8822B(x)                                                   \
+	(((x) & BIT_MASK_FW_DBG0_8822B) << BIT_SHIFT_FW_DBG0_8822B)
+#define BIT_GET_FW_DBG0_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_FW_DBG0_8822B) & BIT_MASK_FW_DBG0_8822B)
+
+/* 2 REG_FW_DBG1_8822B */
+
+#define BIT_SHIFT_FW_DBG1_8822B 0
+#define BIT_MASK_FW_DBG1_8822B 0xffffffffL
+#define BIT_FW_DBG1_8822B(x)                                                   \
+	(((x) & BIT_MASK_FW_DBG1_8822B) << BIT_SHIFT_FW_DBG1_8822B)
+#define BIT_GET_FW_DBG1_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_FW_DBG1_8822B) & BIT_MASK_FW_DBG1_8822B)
+
+/* 2 REG_FW_DBG2_8822B */
+
+#define BIT_SHIFT_FW_DBG2_8822B 0
+#define BIT_MASK_FW_DBG2_8822B 0xffffffffL
+#define BIT_FW_DBG2_8822B(x)                                                   \
+	(((x) & BIT_MASK_FW_DBG2_8822B) << BIT_SHIFT_FW_DBG2_8822B)
+#define BIT_GET_FW_DBG2_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_FW_DBG2_8822B) & BIT_MASK_FW_DBG2_8822B)
+
+/* 2 REG_FW_DBG3_8822B */
+
+#define BIT_SHIFT_FW_DBG3_8822B 0
+#define BIT_MASK_FW_DBG3_8822B 0xffffffffL
+#define BIT_FW_DBG3_8822B(x)                                                   \
+	(((x) & BIT_MASK_FW_DBG3_8822B) << BIT_SHIFT_FW_DBG3_8822B)
+#define BIT_GET_FW_DBG3_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_FW_DBG3_8822B) & BIT_MASK_FW_DBG3_8822B)
+
+/* 2 REG_FW_DBG4_8822B */
+
+#define BIT_SHIFT_FW_DBG4_8822B 0
+#define BIT_MASK_FW_DBG4_8822B 0xffffffffL
+#define BIT_FW_DBG4_8822B(x)                                                   \
+	(((x) & BIT_MASK_FW_DBG4_8822B) << BIT_SHIFT_FW_DBG4_8822B)
+#define BIT_GET_FW_DBG4_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_FW_DBG4_8822B) & BIT_MASK_FW_DBG4_8822B)
+
+/* 2 REG_FW_DBG5_8822B */
+
+#define BIT_SHIFT_FW_DBG5_8822B 0
+#define BIT_MASK_FW_DBG5_8822B 0xffffffffL
+#define BIT_FW_DBG5_8822B(x)                                                   \
+	(((x) & BIT_MASK_FW_DBG5_8822B) << BIT_SHIFT_FW_DBG5_8822B)
+#define BIT_GET_FW_DBG5_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_FW_DBG5_8822B) & BIT_MASK_FW_DBG5_8822B)
+
+/* 2 REG_FW_DBG6_8822B */
+
+#define BIT_SHIFT_FW_DBG6_8822B 0
+#define BIT_MASK_FW_DBG6_8822B 0xffffffffL
+#define BIT_FW_DBG6_8822B(x)                                                   \
+	(((x) & BIT_MASK_FW_DBG6_8822B) << BIT_SHIFT_FW_DBG6_8822B)
+#define BIT_GET_FW_DBG6_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_FW_DBG6_8822B) & BIT_MASK_FW_DBG6_8822B)
+
+/* 2 REG_FW_DBG7_8822B */
+
+#define BIT_SHIFT_FW_DBG7_8822B 0
+#define BIT_MASK_FW_DBG7_8822B 0xffffffffL
+#define BIT_FW_DBG7_8822B(x)                                                   \
+	(((x) & BIT_MASK_FW_DBG7_8822B) << BIT_SHIFT_FW_DBG7_8822B)
+#define BIT_GET_FW_DBG7_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_FW_DBG7_8822B) & BIT_MASK_FW_DBG7_8822B)
+
+/* 2 REG_NOT_VALID_8822B */
+
+/* 2 REG_CR_8822B */
+
+#define BIT_SHIFT_LBMODE_8822B 24
+#define BIT_MASK_LBMODE_8822B 0x1f
+#define BIT_LBMODE_8822B(x)                                                    \
+	(((x) & BIT_MASK_LBMODE_8822B) << BIT_SHIFT_LBMODE_8822B)
+#define BIT_GET_LBMODE_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_LBMODE_8822B) & BIT_MASK_LBMODE_8822B)
+
+#define BIT_SHIFT_NETYPE1_8822B 18
+#define BIT_MASK_NETYPE1_8822B 0x3
+#define BIT_NETYPE1_8822B(x)                                                   \
+	(((x) & BIT_MASK_NETYPE1_8822B) << BIT_SHIFT_NETYPE1_8822B)
+#define BIT_GET_NETYPE1_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_NETYPE1_8822B) & BIT_MASK_NETYPE1_8822B)
+
+#define BIT_SHIFT_NETYPE0_8822B 16
+#define BIT_MASK_NETYPE0_8822B 0x3
+#define BIT_NETYPE0_8822B(x)                                                   \
+	(((x) & BIT_MASK_NETYPE0_8822B) << BIT_SHIFT_NETYPE0_8822B)
+#define BIT_GET_NETYPE0_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_NETYPE0_8822B) & BIT_MASK_NETYPE0_8822B)
+
+#define BIT_I2C_MAILBOX_EN_8822B BIT(12)
+#define BIT_SHCUT_EN_8822B BIT(11)
+#define BIT_32K_CAL_TMR_EN_8822B BIT(10)
+#define BIT_MAC_SEC_EN_8822B BIT(9)
+#define BIT_ENSWBCN_8822B BIT(8)
+#define BIT_MACRXEN_8822B BIT(7)
+#define BIT_MACTXEN_8822B BIT(6)
+#define BIT_SCHEDULE_EN_8822B BIT(5)
+#define BIT_PROTOCOL_EN_8822B BIT(4)
+#define BIT_RXDMA_EN_8822B BIT(3)
+#define BIT_TXDMA_EN_8822B BIT(2)
+#define BIT_HCI_RXDMA_EN_8822B BIT(1)
+#define BIT_HCI_TXDMA_EN_8822B BIT(0)
+
+/* 2 REG_PKT_BUFF_ACCESS_CTRL_8822B */
+
+#define BIT_SHIFT_PKT_BUFF_ACCESS_CTRL_8822B 0
+#define BIT_MASK_PKT_BUFF_ACCESS_CTRL_8822B 0xff
+#define BIT_PKT_BUFF_ACCESS_CTRL_8822B(x)                                      \
+	(((x) & BIT_MASK_PKT_BUFF_ACCESS_CTRL_8822B)                           \
+	 << BIT_SHIFT_PKT_BUFF_ACCESS_CTRL_8822B)
+#define BIT_GET_PKT_BUFF_ACCESS_CTRL_8822B(x)                                  \
+	(((x) >> BIT_SHIFT_PKT_BUFF_ACCESS_CTRL_8822B) &                       \
+	 BIT_MASK_PKT_BUFF_ACCESS_CTRL_8822B)
+
+/* 2 REG_TSF_CLK_STATE_8822B */
+#define BIT_TSF_CLK_STABLE_8822B BIT(15)
+
+/* 2 REG_TXDMA_PQ_MAP_8822B */
+
+#define BIT_SHIFT_TXDMA_HIQ_MAP_8822B 14
+#define BIT_MASK_TXDMA_HIQ_MAP_8822B 0x3
+#define BIT_TXDMA_HIQ_MAP_8822B(x)                                             \
+	(((x) & BIT_MASK_TXDMA_HIQ_MAP_8822B) << BIT_SHIFT_TXDMA_HIQ_MAP_8822B)
+#define BIT_GET_TXDMA_HIQ_MAP_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_TXDMA_HIQ_MAP_8822B) & BIT_MASK_TXDMA_HIQ_MAP_8822B)
+
+#define BIT_SHIFT_TXDMA_MGQ_MAP_8822B 12
+#define BIT_MASK_TXDMA_MGQ_MAP_8822B 0x3
+#define BIT_TXDMA_MGQ_MAP_8822B(x)                                             \
+	(((x) & BIT_MASK_TXDMA_MGQ_MAP_8822B) << BIT_SHIFT_TXDMA_MGQ_MAP_8822B)
+#define BIT_GET_TXDMA_MGQ_MAP_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_TXDMA_MGQ_MAP_8822B) & BIT_MASK_TXDMA_MGQ_MAP_8822B)
+
+#define BIT_SHIFT_TXDMA_BKQ_MAP_8822B 10
+#define BIT_MASK_TXDMA_BKQ_MAP_8822B 0x3
+#define BIT_TXDMA_BKQ_MAP_8822B(x)                                             \
+	(((x) & BIT_MASK_TXDMA_BKQ_MAP_8822B) << BIT_SHIFT_TXDMA_BKQ_MAP_8822B)
+#define BIT_GET_TXDMA_BKQ_MAP_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_TXDMA_BKQ_MAP_8822B) & BIT_MASK_TXDMA_BKQ_MAP_8822B)
+
+#define BIT_SHIFT_TXDMA_BEQ_MAP_8822B 8
+#define BIT_MASK_TXDMA_BEQ_MAP_8822B 0x3
+#define BIT_TXDMA_BEQ_MAP_8822B(x)                                             \
+	(((x) & BIT_MASK_TXDMA_BEQ_MAP_8822B) << BIT_SHIFT_TXDMA_BEQ_MAP_8822B)
+#define BIT_GET_TXDMA_BEQ_MAP_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_TXDMA_BEQ_MAP_8822B) & BIT_MASK_TXDMA_BEQ_MAP_8822B)
+
+#define BIT_SHIFT_TXDMA_VIQ_MAP_8822B 6
+#define BIT_MASK_TXDMA_VIQ_MAP_8822B 0x3
+#define BIT_TXDMA_VIQ_MAP_8822B(x)                                             \
+	(((x) & BIT_MASK_TXDMA_VIQ_MAP_8822B) << BIT_SHIFT_TXDMA_VIQ_MAP_8822B)
+#define BIT_GET_TXDMA_VIQ_MAP_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_TXDMA_VIQ_MAP_8822B) & BIT_MASK_TXDMA_VIQ_MAP_8822B)
+
+#define BIT_SHIFT_TXDMA_VOQ_MAP_8822B 4
+#define BIT_MASK_TXDMA_VOQ_MAP_8822B 0x3
+#define BIT_TXDMA_VOQ_MAP_8822B(x)                                             \
+	(((x) & BIT_MASK_TXDMA_VOQ_MAP_8822B) << BIT_SHIFT_TXDMA_VOQ_MAP_8822B)
+#define BIT_GET_TXDMA_VOQ_MAP_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_TXDMA_VOQ_MAP_8822B) & BIT_MASK_TXDMA_VOQ_MAP_8822B)
+
+#define BIT_RXDMA_AGG_EN_8822B BIT(2)
+#define BIT_RXSHFT_EN_8822B BIT(1)
+#define BIT_RXDMA_ARBBW_EN_8822B BIT(0)
+
+/* 2 REG_TRXFF_BNDY_8822B */
+
+#define BIT_SHIFT_RXFFOVFL_RSV_V2_8822B 8
+#define BIT_MASK_RXFFOVFL_RSV_V2_8822B 0xf
+#define BIT_RXFFOVFL_RSV_V2_8822B(x)                                           \
+	(((x) & BIT_MASK_RXFFOVFL_RSV_V2_8822B)                                \
+	 << BIT_SHIFT_RXFFOVFL_RSV_V2_8822B)
+#define BIT_GET_RXFFOVFL_RSV_V2_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_RXFFOVFL_RSV_V2_8822B) &                            \
+	 BIT_MASK_RXFFOVFL_RSV_V2_8822B)
+
+#define BIT_SHIFT_TXPKTBUF_PGBNDY_8822B 0
+#define BIT_MASK_TXPKTBUF_PGBNDY_8822B 0xff
+#define BIT_TXPKTBUF_PGBNDY_8822B(x)                                           \
+	(((x) & BIT_MASK_TXPKTBUF_PGBNDY_8822B)                                \
+	 << BIT_SHIFT_TXPKTBUF_PGBNDY_8822B)
+#define BIT_GET_TXPKTBUF_PGBNDY_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_TXPKTBUF_PGBNDY_8822B) &                            \
+	 BIT_MASK_TXPKTBUF_PGBNDY_8822B)
+
+/* 2 REG_PTA_I2C_MBOX_8822B */
+
+/* 2 REG_NOT_VALID_8822B */
+
+#define BIT_SHIFT_I2C_M_STATUS_8822B 8
+#define BIT_MASK_I2C_M_STATUS_8822B 0xf
+#define BIT_I2C_M_STATUS_8822B(x)                                              \
+	(((x) & BIT_MASK_I2C_M_STATUS_8822B) << BIT_SHIFT_I2C_M_STATUS_8822B)
+#define BIT_GET_I2C_M_STATUS_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_I2C_M_STATUS_8822B) & BIT_MASK_I2C_M_STATUS_8822B)
+
+#define BIT_SHIFT_I2C_M_BUS_GNT_FW_8822B 4
+#define BIT_MASK_I2C_M_BUS_GNT_FW_8822B 0x7
+#define BIT_I2C_M_BUS_GNT_FW_8822B(x)                                          \
+	(((x) & BIT_MASK_I2C_M_BUS_GNT_FW_8822B)                               \
+	 << BIT_SHIFT_I2C_M_BUS_GNT_FW_8822B)
+#define BIT_GET_I2C_M_BUS_GNT_FW_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_I2C_M_BUS_GNT_FW_8822B) &                           \
+	 BIT_MASK_I2C_M_BUS_GNT_FW_8822B)
+
+#define BIT_I2C_M_GNT_FW_8822B BIT(3)
+
+#define BIT_SHIFT_I2C_M_SPEED_8822B 1
+#define BIT_MASK_I2C_M_SPEED_8822B 0x3
+#define BIT_I2C_M_SPEED_8822B(x)                                               \
+	(((x) & BIT_MASK_I2C_M_SPEED_8822B) << BIT_SHIFT_I2C_M_SPEED_8822B)
+#define BIT_GET_I2C_M_SPEED_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_I2C_M_SPEED_8822B) & BIT_MASK_I2C_M_SPEED_8822B)
+
+#define BIT_I2C_M_UNLOCK_8822B BIT(0)
+
+/* 2 REG_RXFF_BNDY_8822B */
+
+/* 2 REG_NOT_VALID_8822B */
+
+#define BIT_SHIFT_RXFF0_BNDY_V2_8822B 0
+#define BIT_MASK_RXFF0_BNDY_V2_8822B 0x3ffff
+#define BIT_RXFF0_BNDY_V2_8822B(x)                                             \
+	(((x) & BIT_MASK_RXFF0_BNDY_V2_8822B) << BIT_SHIFT_RXFF0_BNDY_V2_8822B)
+#define BIT_GET_RXFF0_BNDY_V2_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_RXFF0_BNDY_V2_8822B) & BIT_MASK_RXFF0_BNDY_V2_8822B)
+
+/* 2 REG_FE1IMR_8822B */
+#define BIT_FS_RXDMA2_DONE_INT_EN_8822B BIT(28)
+#define BIT_FS_RXDONE3_INT_EN_8822B BIT(27)
+#define BIT_FS_RXDONE2_INT_EN_8822B BIT(26)
+#define BIT_FS_RX_BCN_P4_INT_EN_8822B BIT(25)
+#define BIT_FS_RX_BCN_P3_INT_EN_8822B BIT(24)
+#define BIT_FS_RX_BCN_P2_INT_EN_8822B BIT(23)
+#define BIT_FS_RX_BCN_P1_INT_EN_8822B BIT(22)
+#define BIT_FS_RX_BCN_P0_INT_EN_8822B BIT(21)
+#define BIT_FS_RX_UMD0_INT_EN_8822B BIT(20)
+#define BIT_FS_RX_UMD1_INT_EN_8822B BIT(19)
+#define BIT_FS_RX_BMD0_INT_EN_8822B BIT(18)
+#define BIT_FS_RX_BMD1_INT_EN_8822B BIT(17)
+#define BIT_FS_RXDONE_INT_EN_8822B BIT(16)
+#define BIT_FS_WWLAN_INT_EN_8822B BIT(15)
+#define BIT_FS_SOUND_DONE_INT_EN_8822B BIT(14)
+#define BIT_FS_LP_STBY_INT_EN_8822B BIT(13)
+#define BIT_FS_TRL_MTR_INT_EN_8822B BIT(12)
+#define BIT_FS_BF1_PRETO_INT_EN_8822B BIT(11)
+#define BIT_FS_BF0_PRETO_INT_EN_8822B BIT(10)
+#define BIT_FS_PTCL_RELEASE_MACID_INT_EN_8822B BIT(9)
+#define BIT_FS_LTE_COEX_EN_8822B BIT(6)
+#define BIT_FS_WLACTOFF_INT_EN_8822B BIT(5)
+#define BIT_FS_WLACTON_INT_EN_8822B BIT(4)
+#define BIT_FS_BTCMD_INT_EN_8822B BIT(3)
+#define BIT_FS_REG_MAILBOX_TO_I2C_INT_EN_8822B BIT(2)
+#define BIT_FS_TRPC_TO_INT_EN_V1_8822B BIT(1)
+#define BIT_FS_RPC_O_T_INT_EN_V1_8822B BIT(0)
+
+/* 2 REG_FE1ISR_8822B */
+#define BIT_FS_RXDMA2_DONE_INT_8822B BIT(28)
+#define BIT_FS_RXDONE3_INT_8822B BIT(27)
+#define BIT_FS_RXDONE2_INT_8822B BIT(26)
+#define BIT_FS_RX_BCN_P4_INT_8822B BIT(25)
+#define BIT_FS_RX_BCN_P3_INT_8822B BIT(24)
+#define BIT_FS_RX_BCN_P2_INT_8822B BIT(23)
+#define BIT_FS_RX_BCN_P1_INT_8822B BIT(22)
+#define BIT_FS_RX_BCN_P0_INT_8822B BIT(21)
+#define BIT_FS_RX_UMD0_INT_8822B BIT(20)
+#define BIT_FS_RX_UMD1_INT_8822B BIT(19)
+#define BIT_FS_RX_BMD0_INT_8822B BIT(18)
+#define BIT_FS_RX_BMD1_INT_8822B BIT(17)
+#define BIT_FS_RXDONE_INT_8822B BIT(16)
+#define BIT_FS_WWLAN_INT_8822B BIT(15)
+#define BIT_FS_SOUND_DONE_INT_8822B BIT(14)
+#define BIT_FS_LP_STBY_INT_8822B BIT(13)
+#define BIT_FS_TRL_MTR_INT_8822B BIT(12)
+#define BIT_FS_BF1_PRETO_INT_8822B BIT(11)
+#define BIT_FS_BF0_PRETO_INT_8822B BIT(10)
+#define BIT_FS_PTCL_RELEASE_MACID_INT_8822B BIT(9)
+#define BIT_FS_LTE_COEX_INT_8822B BIT(6)
+#define BIT_FS_WLACTOFF_INT_8822B BIT(5)
+#define BIT_FS_WLACTON_INT_8822B BIT(4)
+#define BIT_FS_BCN_RX_INT_INT_8822B BIT(3)
+#define BIT_FS_MAILBOX_TO_I2C_INT_8822B BIT(2)
+#define BIT_FS_TRPC_TO_INT_8822B BIT(1)
+#define BIT_FS_RPC_O_T_INT_8822B BIT(0)
+
+/* 2 REG_NOT_VALID_8822B */
+
+/* 2 REG_CPWM_8822B */
+#define BIT_CPWM_TOGGLING_8822B BIT(31)
+
+#define BIT_SHIFT_CPWM_MOD_8822B 24
+#define BIT_MASK_CPWM_MOD_8822B 0x7f
+#define BIT_CPWM_MOD_8822B(x)                                                  \
+	(((x) & BIT_MASK_CPWM_MOD_8822B) << BIT_SHIFT_CPWM_MOD_8822B)
+#define BIT_GET_CPWM_MOD_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_CPWM_MOD_8822B) & BIT_MASK_CPWM_MOD_8822B)
+
+/* 2 REG_FWIMR_8822B */
+#define BIT_FS_TXBCNOK_MB7_INT_EN_8822B BIT(31)
+#define BIT_FS_TXBCNOK_MB6_INT_EN_8822B BIT(30)
+#define BIT_FS_TXBCNOK_MB5_INT_EN_8822B BIT(29)
+#define BIT_FS_TXBCNOK_MB4_INT_EN_8822B BIT(28)
+#define BIT_FS_TXBCNOK_MB3_INT_EN_8822B BIT(27)
+#define BIT_FS_TXBCNOK_MB2_INT_EN_8822B BIT(26)
+#define BIT_FS_TXBCNOK_MB1_INT_EN_8822B BIT(25)
+#define BIT_FS_TXBCNOK_MB0_INT_EN_8822B BIT(24)
+#define BIT_FS_TXBCNERR_MB7_INT_EN_8822B BIT(23)
+#define BIT_FS_TXBCNERR_MB6_INT_EN_8822B BIT(22)
+#define BIT_FS_TXBCNERR_MB5_INT_EN_8822B BIT(21)
+#define BIT_FS_TXBCNERR_MB4_INT_EN_8822B BIT(20)
+#define BIT_FS_TXBCNERR_MB3_INT_EN_8822B BIT(19)
+#define BIT_FS_TXBCNERR_MB2_INT_EN_8822B BIT(18)
+#define BIT_FS_TXBCNERR_MB1_INT_EN_8822B BIT(17)
+#define BIT_FS_TXBCNERR_MB0_INT_EN_8822B BIT(16)
+#define BIT_CPU_MGQ_TXDONE_INT_EN_8822B BIT(15)
+#define BIT_SIFS_OVERSPEC_INT_EN_8822B BIT(14)
+#define BIT_FS_MGNTQ_RPTR_RELEASE_INT_EN_8822B BIT(13)
+#define BIT_FS_MGNTQFF_TO_INT_EN_8822B BIT(12)
+#define BIT_FS_DDMA1_LP_INT_EN_8822B BIT(11)
+#define BIT_FS_DDMA1_HP_INT_EN_8822B BIT(10)
+#define BIT_FS_DDMA0_LP_INT_EN_8822B BIT(9)
+#define BIT_FS_DDMA0_HP_INT_EN_8822B BIT(8)
+#define BIT_FS_TRXRPT_INT_EN_8822B BIT(7)
+#define BIT_FS_C2H_W_READY_INT_EN_8822B BIT(6)
+#define BIT_FS_HRCV_INT_EN_8822B BIT(5)
+#define BIT_FS_H2CCMD_INT_EN_8822B BIT(4)
+#define BIT_FS_TXPKTIN_INT_EN_8822B BIT(3)
+#define BIT_FS_ERRORHDL_INT_EN_8822B BIT(2)
+#define BIT_FS_TXCCX_INT_EN_8822B BIT(1)
+#define BIT_FS_TXCLOSE_INT_EN_8822B BIT(0)
+
+/* 2 REG_FWISR_8822B */
+#define BIT_FS_TXBCNOK_MB7_INT_8822B BIT(31)
+#define BIT_FS_TXBCNOK_MB6_INT_8822B BIT(30)
+#define BIT_FS_TXBCNOK_MB5_INT_8822B BIT(29)
+#define BIT_FS_TXBCNOK_MB4_INT_8822B BIT(28)
+#define BIT_FS_TXBCNOK_MB3_INT_8822B BIT(27)
+#define BIT_FS_TXBCNOK_MB2_INT_8822B BIT(26)
+#define BIT_FS_TXBCNOK_MB1_INT_8822B BIT(25)
+#define BIT_FS_TXBCNOK_MB0_INT_8822B BIT(24)
+#define BIT_FS_TXBCNERR_MB7_INT_8822B BIT(23)
+#define BIT_FS_TXBCNERR_MB6_INT_8822B BIT(22)
+#define BIT_FS_TXBCNERR_MB5_INT_8822B BIT(21)
+#define BIT_FS_TXBCNERR_MB4_INT_8822B BIT(20)
+#define BIT_FS_TXBCNERR_MB3_INT_8822B BIT(19)
+#define BIT_FS_TXBCNERR_MB2_INT_8822B BIT(18)
+#define BIT_FS_TXBCNERR_MB1_INT_8822B BIT(17)
+#define BIT_FS_TXBCNERR_MB0_INT_8822B BIT(16)
+#define BIT_CPU_MGQ_TXDONE_INT_8822B BIT(15)
+#define BIT_SIFS_OVERSPEC_INT_8822B BIT(14)
+#define BIT_FS_MGNTQ_RPTR_RELEASE_INT_8822B BIT(13)
+#define BIT_FS_MGNTQFF_TO_INT_8822B BIT(12)
+#define BIT_FS_DDMA1_LP_INT_8822B BIT(11)
+#define BIT_FS_DDMA1_HP_INT_8822B BIT(10)
+#define BIT_FS_DDMA0_LP_INT_8822B BIT(9)
+#define BIT_FS_DDMA0_HP_INT_8822B BIT(8)
+#define BIT_FS_TRXRPT_INT_8822B BIT(7)
+#define BIT_FS_C2H_W_READY_INT_8822B BIT(6)
+#define BIT_FS_HRCV_INT_8822B BIT(5)
+#define BIT_FS_H2CCMD_INT_8822B BIT(4)
+#define BIT_FS_TXPKTIN_INT_8822B BIT(3)
+#define BIT_FS_ERRORHDL_INT_8822B BIT(2)
+#define BIT_FS_TXCCX_INT_8822B BIT(1)
+#define BIT_FS_TXCLOSE_INT_8822B BIT(0)
+
+/* 2 REG_FTIMR_8822B */
+#define BIT_PS_TIMER_C_EARLY_INT_EN_8822B BIT(23)
+#define BIT_PS_TIMER_B_EARLY_INT_EN_8822B BIT(22)
+#define BIT_PS_TIMER_A_EARLY_INT_EN_8822B BIT(21)
+#define BIT_CPUMGQ_TX_TIMER_EARLY_INT_EN_8822B BIT(20)
+#define BIT_PS_TIMER_C_INT_EN_8822B BIT(19)
+#define BIT_PS_TIMER_B_INT_EN_8822B BIT(18)
+#define BIT_PS_TIMER_A_INT_EN_8822B BIT(17)
+#define BIT_CPUMGQ_TX_TIMER_INT_EN_8822B BIT(16)
+#define BIT_FS_PS_TIMEOUT2_EN_8822B BIT(15)
+#define BIT_FS_PS_TIMEOUT1_EN_8822B BIT(14)
+#define BIT_FS_PS_TIMEOUT0_EN_8822B BIT(13)
+#define BIT_FS_GTINT8_EN_8822B BIT(8)
+#define BIT_FS_GTINT7_EN_8822B BIT(7)
+#define BIT_FS_GTINT6_EN_8822B BIT(6)
+#define BIT_FS_GTINT5_EN_8822B BIT(5)
+#define BIT_FS_GTINT4_EN_8822B BIT(4)
+#define BIT_FS_GTINT3_EN_8822B BIT(3)
+#define BIT_FS_GTINT2_EN_8822B BIT(2)
+#define BIT_FS_GTINT1_EN_8822B BIT(1)
+#define BIT_FS_GTINT0_EN_8822B BIT(0)
+
+/* 2 REG_FTISR_8822B */
+#define BIT_PS_TIMER_C_EARLY__INT_8822B BIT(23)
+#define BIT_PS_TIMER_B_EARLY__INT_8822B BIT(22)
+#define BIT_PS_TIMER_A_EARLY__INT_8822B BIT(21)
+#define BIT_CPUMGQ_TX_TIMER_EARLY_INT_8822B BIT(20)
+#define BIT_PS_TIMER_C_INT_8822B BIT(19)
+#define BIT_PS_TIMER_B_INT_8822B BIT(18)
+#define BIT_PS_TIMER_A_INT_8822B BIT(17)
+#define BIT_CPUMGQ_TX_TIMER_INT_8822B BIT(16)
+#define BIT_FS_PS_TIMEOUT2_INT_8822B BIT(15)
+#define BIT_FS_PS_TIMEOUT1_INT_8822B BIT(14)
+#define BIT_FS_PS_TIMEOUT0_INT_8822B BIT(13)
+#define BIT_FS_GTINT8_INT_8822B BIT(8)
+#define BIT_FS_GTINT7_INT_8822B BIT(7)
+#define BIT_FS_GTINT6_INT_8822B BIT(6)
+#define BIT_FS_GTINT5_INT_8822B BIT(5)
+#define BIT_FS_GTINT4_INT_8822B BIT(4)
+#define BIT_FS_GTINT3_INT_8822B BIT(3)
+#define BIT_FS_GTINT2_INT_8822B BIT(2)
+#define BIT_FS_GTINT1_INT_8822B BIT(1)
+#define BIT_FS_GTINT0_INT_8822B BIT(0)
+
+/* 2 REG_PKTBUF_DBG_CTRL_8822B */
+
+#define BIT_SHIFT_PKTBUF_WRITE_EN_8822B 24
+#define BIT_MASK_PKTBUF_WRITE_EN_8822B 0xff
+#define BIT_PKTBUF_WRITE_EN_8822B(x)                                           \
+	(((x) & BIT_MASK_PKTBUF_WRITE_EN_8822B)                                \
+	 << BIT_SHIFT_PKTBUF_WRITE_EN_8822B)
+#define BIT_GET_PKTBUF_WRITE_EN_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_PKTBUF_WRITE_EN_8822B) &                            \
+	 BIT_MASK_PKTBUF_WRITE_EN_8822B)
+
+#define BIT_TXRPTBUF_DBG_8822B BIT(23)
+
+/* 2 REG_NOT_VALID_8822B */
+#define BIT_TXPKTBUF_DBG_V2_8822B BIT(20)
+#define BIT_RXPKTBUF_DBG_8822B BIT(16)
+
+#define BIT_SHIFT_PKTBUF_DBG_ADDR_8822B 0
+#define BIT_MASK_PKTBUF_DBG_ADDR_8822B 0x1fff
+#define BIT_PKTBUF_DBG_ADDR_8822B(x)                                           \
+	(((x) & BIT_MASK_PKTBUF_DBG_ADDR_8822B)                                \
+	 << BIT_SHIFT_PKTBUF_DBG_ADDR_8822B)
+#define BIT_GET_PKTBUF_DBG_ADDR_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_PKTBUF_DBG_ADDR_8822B) &                            \
+	 BIT_MASK_PKTBUF_DBG_ADDR_8822B)
+
+/* 2 REG_PKTBUF_DBG_DATA_L_8822B */
+
+#define BIT_SHIFT_PKTBUF_DBG_DATA_L_8822B 0
+#define BIT_MASK_PKTBUF_DBG_DATA_L_8822B 0xffffffffL
+#define BIT_PKTBUF_DBG_DATA_L_8822B(x)                                         \
+	(((x) & BIT_MASK_PKTBUF_DBG_DATA_L_8822B)                              \
+	 << BIT_SHIFT_PKTBUF_DBG_DATA_L_8822B)
+#define BIT_GET_PKTBUF_DBG_DATA_L_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_PKTBUF_DBG_DATA_L_8822B) &                          \
+	 BIT_MASK_PKTBUF_DBG_DATA_L_8822B)
+
+/* 2 REG_PKTBUF_DBG_DATA_H_8822B */
+
+#define BIT_SHIFT_PKTBUF_DBG_DATA_H_8822B 0
+#define BIT_MASK_PKTBUF_DBG_DATA_H_8822B 0xffffffffL
+#define BIT_PKTBUF_DBG_DATA_H_8822B(x)                                         \
+	(((x) & BIT_MASK_PKTBUF_DBG_DATA_H_8822B)                              \
+	 << BIT_SHIFT_PKTBUF_DBG_DATA_H_8822B)
+#define BIT_GET_PKTBUF_DBG_DATA_H_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_PKTBUF_DBG_DATA_H_8822B) &                          \
+	 BIT_MASK_PKTBUF_DBG_DATA_H_8822B)
+
+/* 2 REG_CPWM2_8822B */
+
+#define BIT_SHIFT_L0S_TO_RCVY_NUM_8822B 16
+#define BIT_MASK_L0S_TO_RCVY_NUM_8822B 0xff
+#define BIT_L0S_TO_RCVY_NUM_8822B(x)                                           \
+	(((x) & BIT_MASK_L0S_TO_RCVY_NUM_8822B)                                \
+	 << BIT_SHIFT_L0S_TO_RCVY_NUM_8822B)
+#define BIT_GET_L0S_TO_RCVY_NUM_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_L0S_TO_RCVY_NUM_8822B) &                            \
+	 BIT_MASK_L0S_TO_RCVY_NUM_8822B)
+
+#define BIT_CPWM2_TOGGLING_8822B BIT(15)
+
+#define BIT_SHIFT_CPWM2_MOD_8822B 0
+#define BIT_MASK_CPWM2_MOD_8822B 0x7fff
+#define BIT_CPWM2_MOD_8822B(x)                                                 \
+	(((x) & BIT_MASK_CPWM2_MOD_8822B) << BIT_SHIFT_CPWM2_MOD_8822B)
+#define BIT_GET_CPWM2_MOD_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_CPWM2_MOD_8822B) & BIT_MASK_CPWM2_MOD_8822B)
+
+/* 2 REG_NOT_VALID_8822B */
+
+/* 2 REG_TC0_CTRL_8822B */
+#define BIT_TC0INT_EN_8822B BIT(26)
+#define BIT_TC0MODE_8822B BIT(25)
+#define BIT_TC0EN_8822B BIT(24)
+
+#define BIT_SHIFT_TC0DATA_8822B 0
+#define BIT_MASK_TC0DATA_8822B 0xffffff
+#define BIT_TC0DATA_8822B(x)                                                   \
+	(((x) & BIT_MASK_TC0DATA_8822B) << BIT_SHIFT_TC0DATA_8822B)
+#define BIT_GET_TC0DATA_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_TC0DATA_8822B) & BIT_MASK_TC0DATA_8822B)
+
+/* 2 REG_TC1_CTRL_8822B */
+#define BIT_TC1INT_EN_8822B BIT(26)
+#define BIT_TC1MODE_8822B BIT(25)
+#define BIT_TC1EN_8822B BIT(24)
+
+#define BIT_SHIFT_TC1DATA_8822B 0
+#define BIT_MASK_TC1DATA_8822B 0xffffff
+#define BIT_TC1DATA_8822B(x)                                                   \
+	(((x) & BIT_MASK_TC1DATA_8822B) << BIT_SHIFT_TC1DATA_8822B)
+#define BIT_GET_TC1DATA_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_TC1DATA_8822B) & BIT_MASK_TC1DATA_8822B)
+
+/* 2 REG_TC2_CTRL_8822B */
+#define BIT_TC2INT_EN_8822B BIT(26)
+#define BIT_TC2MODE_8822B BIT(25)
+#define BIT_TC2EN_8822B BIT(24)
+
+#define BIT_SHIFT_TC2DATA_8822B 0
+#define BIT_MASK_TC2DATA_8822B 0xffffff
+#define BIT_TC2DATA_8822B(x)                                                   \
+	(((x) & BIT_MASK_TC2DATA_8822B) << BIT_SHIFT_TC2DATA_8822B)
+#define BIT_GET_TC2DATA_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_TC2DATA_8822B) & BIT_MASK_TC2DATA_8822B)
+
+/* 2 REG_TC3_CTRL_8822B */
+#define BIT_TC3INT_EN_8822B BIT(26)
+#define BIT_TC3MODE_8822B BIT(25)
+#define BIT_TC3EN_8822B BIT(24)
+
+#define BIT_SHIFT_TC3DATA_8822B 0
+#define BIT_MASK_TC3DATA_8822B 0xffffff
+#define BIT_TC3DATA_8822B(x)                                                   \
+	(((x) & BIT_MASK_TC3DATA_8822B) << BIT_SHIFT_TC3DATA_8822B)
+#define BIT_GET_TC3DATA_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_TC3DATA_8822B) & BIT_MASK_TC3DATA_8822B)
+
+/* 2 REG_TC4_CTRL_8822B */
+#define BIT_TC4INT_EN_8822B BIT(26)
+#define BIT_TC4MODE_8822B BIT(25)
+#define BIT_TC4EN_8822B BIT(24)
+
+#define BIT_SHIFT_TC4DATA_8822B 0
+#define BIT_MASK_TC4DATA_8822B 0xffffff
+#define BIT_TC4DATA_8822B(x)                                                   \
+	(((x) & BIT_MASK_TC4DATA_8822B) << BIT_SHIFT_TC4DATA_8822B)
+#define BIT_GET_TC4DATA_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_TC4DATA_8822B) & BIT_MASK_TC4DATA_8822B)
+
+/* 2 REG_TCUNIT_BASE_8822B */
+
+#define BIT_SHIFT_TCUNIT_BASE_8822B 0
+#define BIT_MASK_TCUNIT_BASE_8822B 0x3fff
+#define BIT_TCUNIT_BASE_8822B(x)                                               \
+	(((x) & BIT_MASK_TCUNIT_BASE_8822B) << BIT_SHIFT_TCUNIT_BASE_8822B)
+#define BIT_GET_TCUNIT_BASE_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_TCUNIT_BASE_8822B) & BIT_MASK_TCUNIT_BASE_8822B)
+
+/* 2 REG_TC5_CTRL_8822B */
+#define BIT_TC5INT_EN_8822B BIT(26)
+#define BIT_TC5MODE_8822B BIT(25)
+#define BIT_TC5EN_8822B BIT(24)
+
+#define BIT_SHIFT_TC5DATA_8822B 0
+#define BIT_MASK_TC5DATA_8822B 0xffffff
+#define BIT_TC5DATA_8822B(x)                                                   \
+	(((x) & BIT_MASK_TC5DATA_8822B) << BIT_SHIFT_TC5DATA_8822B)
+#define BIT_GET_TC5DATA_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_TC5DATA_8822B) & BIT_MASK_TC5DATA_8822B)
+
+/* 2 REG_TC6_CTRL_8822B */
+#define BIT_TC6INT_EN_8822B BIT(26)
+#define BIT_TC6MODE_8822B BIT(25)
+#define BIT_TC6EN_8822B BIT(24)
+
+#define BIT_SHIFT_TC6DATA_8822B 0
+#define BIT_MASK_TC6DATA_8822B 0xffffff
+#define BIT_TC6DATA_8822B(x)                                                   \
+	(((x) & BIT_MASK_TC6DATA_8822B) << BIT_SHIFT_TC6DATA_8822B)
+#define BIT_GET_TC6DATA_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_TC6DATA_8822B) & BIT_MASK_TC6DATA_8822B)
+
+/* 2 REG_MBIST_FAIL_8822B */
+
+#define BIT_SHIFT_8051_MBIST_FAIL_8822B 26
+#define BIT_MASK_8051_MBIST_FAIL_8822B 0x7
+#define BIT_8051_MBIST_FAIL_8822B(x)                                           \
+	(((x) & BIT_MASK_8051_MBIST_FAIL_8822B)                                \
+	 << BIT_SHIFT_8051_MBIST_FAIL_8822B)
+#define BIT_GET_8051_MBIST_FAIL_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_8051_MBIST_FAIL_8822B) &                            \
+	 BIT_MASK_8051_MBIST_FAIL_8822B)
+
+#define BIT_SHIFT_USB_MBIST_FAIL_8822B 24
+#define BIT_MASK_USB_MBIST_FAIL_8822B 0x3
+#define BIT_USB_MBIST_FAIL_8822B(x)                                            \
+	(((x) & BIT_MASK_USB_MBIST_FAIL_8822B)                                 \
+	 << BIT_SHIFT_USB_MBIST_FAIL_8822B)
+#define BIT_GET_USB_MBIST_FAIL_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_USB_MBIST_FAIL_8822B) &                             \
+	 BIT_MASK_USB_MBIST_FAIL_8822B)
+
+#define BIT_SHIFT_PCIE_MBIST_FAIL_8822B 16
+#define BIT_MASK_PCIE_MBIST_FAIL_8822B 0x3f
+#define BIT_PCIE_MBIST_FAIL_8822B(x)                                           \
+	(((x) & BIT_MASK_PCIE_MBIST_FAIL_8822B)                                \
+	 << BIT_SHIFT_PCIE_MBIST_FAIL_8822B)
+#define BIT_GET_PCIE_MBIST_FAIL_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_PCIE_MBIST_FAIL_8822B) &                            \
+	 BIT_MASK_PCIE_MBIST_FAIL_8822B)
+
+#define BIT_SHIFT_MAC_MBIST_FAIL_8822B 0
+#define BIT_MASK_MAC_MBIST_FAIL_8822B 0xfff
+#define BIT_MAC_MBIST_FAIL_8822B(x)                                            \
+	(((x) & BIT_MASK_MAC_MBIST_FAIL_8822B)                                 \
+	 << BIT_SHIFT_MAC_MBIST_FAIL_8822B)
+#define BIT_GET_MAC_MBIST_FAIL_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_MAC_MBIST_FAIL_8822B) &                             \
+	 BIT_MASK_MAC_MBIST_FAIL_8822B)
+
+/* 2 REG_MBIST_START_PAUSE_8822B */
+
+#define BIT_SHIFT_8051_MBIST_START_PAUSE_8822B 26
+#define BIT_MASK_8051_MBIST_START_PAUSE_8822B 0x7
+#define BIT_8051_MBIST_START_PAUSE_8822B(x)                                    \
+	(((x) & BIT_MASK_8051_MBIST_START_PAUSE_8822B)                         \
+	 << BIT_SHIFT_8051_MBIST_START_PAUSE_8822B)
+#define BIT_GET_8051_MBIST_START_PAUSE_8822B(x)                                \
+	(((x) >> BIT_SHIFT_8051_MBIST_START_PAUSE_8822B) &                     \
+	 BIT_MASK_8051_MBIST_START_PAUSE_8822B)
+
+#define BIT_SHIFT_USB_MBIST_START_PAUSE_8822B 24
+#define BIT_MASK_USB_MBIST_START_PAUSE_8822B 0x3
+#define BIT_USB_MBIST_START_PAUSE_8822B(x)                                     \
+	(((x) & BIT_MASK_USB_MBIST_START_PAUSE_8822B)                          \
+	 << BIT_SHIFT_USB_MBIST_START_PAUSE_8822B)
+#define BIT_GET_USB_MBIST_START_PAUSE_8822B(x)                                 \
+	(((x) >> BIT_SHIFT_USB_MBIST_START_PAUSE_8822B) &                      \
+	 BIT_MASK_USB_MBIST_START_PAUSE_8822B)
+
+#define BIT_SHIFT_PCIE_MBIST_START_PAUSE_8822B 16
+#define BIT_MASK_PCIE_MBIST_START_PAUSE_8822B 0x3f
+#define BIT_PCIE_MBIST_START_PAUSE_8822B(x)                                    \
+	(((x) & BIT_MASK_PCIE_MBIST_START_PAUSE_8822B)                         \
+	 << BIT_SHIFT_PCIE_MBIST_START_PAUSE_8822B)
+#define BIT_GET_PCIE_MBIST_START_PAUSE_8822B(x)                                \
+	(((x) >> BIT_SHIFT_PCIE_MBIST_START_PAUSE_8822B) &                     \
+	 BIT_MASK_PCIE_MBIST_START_PAUSE_8822B)
+
+#define BIT_SHIFT_MAC_MBIST_START_PAUSE_8822B 0
+#define BIT_MASK_MAC_MBIST_START_PAUSE_8822B 0xfff
+#define BIT_MAC_MBIST_START_PAUSE_8822B(x)                                     \
+	(((x) & BIT_MASK_MAC_MBIST_START_PAUSE_8822B)                          \
+	 << BIT_SHIFT_MAC_MBIST_START_PAUSE_8822B)
+#define BIT_GET_MAC_MBIST_START_PAUSE_8822B(x)                                 \
+	(((x) >> BIT_SHIFT_MAC_MBIST_START_PAUSE_8822B) &                      \
+	 BIT_MASK_MAC_MBIST_START_PAUSE_8822B)
+
+/* 2 REG_MBIST_DONE_8822B */
+
+#define BIT_SHIFT_8051_MBIST_DONE_8822B 26
+#define BIT_MASK_8051_MBIST_DONE_8822B 0x7
+#define BIT_8051_MBIST_DONE_8822B(x)                                           \
+	(((x) & BIT_MASK_8051_MBIST_DONE_8822B)                                \
+	 << BIT_SHIFT_8051_MBIST_DONE_8822B)
+#define BIT_GET_8051_MBIST_DONE_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_8051_MBIST_DONE_8822B) &                            \
+	 BIT_MASK_8051_MBIST_DONE_8822B)
+
+#define BIT_SHIFT_USB_MBIST_DONE_8822B 24
+#define BIT_MASK_USB_MBIST_DONE_8822B 0x3
+#define BIT_USB_MBIST_DONE_8822B(x)                                            \
+	(((x) & BIT_MASK_USB_MBIST_DONE_8822B)                                 \
+	 << BIT_SHIFT_USB_MBIST_DONE_8822B)
+#define BIT_GET_USB_MBIST_DONE_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_USB_MBIST_DONE_8822B) &                             \
+	 BIT_MASK_USB_MBIST_DONE_8822B)
+
+#define BIT_SHIFT_PCIE_MBIST_DONE_8822B 16
+#define BIT_MASK_PCIE_MBIST_DONE_8822B 0x3f
+#define BIT_PCIE_MBIST_DONE_8822B(x)                                           \
+	(((x) & BIT_MASK_PCIE_MBIST_DONE_8822B)                                \
+	 << BIT_SHIFT_PCIE_MBIST_DONE_8822B)
+#define BIT_GET_PCIE_MBIST_DONE_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_PCIE_MBIST_DONE_8822B) &                            \
+	 BIT_MASK_PCIE_MBIST_DONE_8822B)
+
+#define BIT_SHIFT_MAC_MBIST_DONE_8822B 0
+#define BIT_MASK_MAC_MBIST_DONE_8822B 0xfff
+#define BIT_MAC_MBIST_DONE_8822B(x)                                            \
+	(((x) & BIT_MASK_MAC_MBIST_DONE_8822B)                                 \
+	 << BIT_SHIFT_MAC_MBIST_DONE_8822B)
+#define BIT_GET_MAC_MBIST_DONE_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_MAC_MBIST_DONE_8822B) &                             \
+	 BIT_MASK_MAC_MBIST_DONE_8822B)
+
+/* 2 REG_MBIST_FAIL_NRML_8822B */
+
+#define BIT_SHIFT_MBIST_FAIL_NRML_8822B 0
+#define BIT_MASK_MBIST_FAIL_NRML_8822B 0xffffffffL
+#define BIT_MBIST_FAIL_NRML_8822B(x)                                           \
+	(((x) & BIT_MASK_MBIST_FAIL_NRML_8822B)                                \
+	 << BIT_SHIFT_MBIST_FAIL_NRML_8822B)
+#define BIT_GET_MBIST_FAIL_NRML_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_MBIST_FAIL_NRML_8822B) &                            \
+	 BIT_MASK_MBIST_FAIL_NRML_8822B)
+
+/* 2 REG_AES_DECRPT_DATA_8822B */
+
+#define BIT_SHIFT_IPS_CFG_ADDR_8822B 0
+#define BIT_MASK_IPS_CFG_ADDR_8822B 0xff
+#define BIT_IPS_CFG_ADDR_8822B(x)                                              \
+	(((x) & BIT_MASK_IPS_CFG_ADDR_8822B) << BIT_SHIFT_IPS_CFG_ADDR_8822B)
+#define BIT_GET_IPS_CFG_ADDR_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_IPS_CFG_ADDR_8822B) & BIT_MASK_IPS_CFG_ADDR_8822B)
+
+/* 2 REG_AES_DECRPT_CFG_8822B */
+
+#define BIT_SHIFT_IPS_CFG_DATA_8822B 0
+#define BIT_MASK_IPS_CFG_DATA_8822B 0xffffffffL
+#define BIT_IPS_CFG_DATA_8822B(x)                                              \
+	(((x) & BIT_MASK_IPS_CFG_DATA_8822B) << BIT_SHIFT_IPS_CFG_DATA_8822B)
+#define BIT_GET_IPS_CFG_DATA_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_IPS_CFG_DATA_8822B) & BIT_MASK_IPS_CFG_DATA_8822B)
+
+/* 2 REG_NOT_VALID_8822B */
+
+/* 2 REG_NOT_VALID_8822B */
+
+/* 2 REG_TMETER_8822B */
+#define BIT_TEMP_VALID_8822B BIT(31)
+
+#define BIT_SHIFT_TEMP_VALUE_8822B 24
+#define BIT_MASK_TEMP_VALUE_8822B 0x3f
+#define BIT_TEMP_VALUE_8822B(x)                                                \
+	(((x) & BIT_MASK_TEMP_VALUE_8822B) << BIT_SHIFT_TEMP_VALUE_8822B)
+#define BIT_GET_TEMP_VALUE_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_TEMP_VALUE_8822B) & BIT_MASK_TEMP_VALUE_8822B)
+
+#define BIT_SHIFT_REG_TMETER_TIMER_8822B 8
+#define BIT_MASK_REG_TMETER_TIMER_8822B 0xfff
+#define BIT_REG_TMETER_TIMER_8822B(x)                                          \
+	(((x) & BIT_MASK_REG_TMETER_TIMER_8822B)                               \
+	 << BIT_SHIFT_REG_TMETER_TIMER_8822B)
+#define BIT_GET_REG_TMETER_TIMER_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_REG_TMETER_TIMER_8822B) &                           \
+	 BIT_MASK_REG_TMETER_TIMER_8822B)
+
+#define BIT_SHIFT_REG_TEMP_DELTA_8822B 2
+#define BIT_MASK_REG_TEMP_DELTA_8822B 0x3f
+#define BIT_REG_TEMP_DELTA_8822B(x)                                            \
+	(((x) & BIT_MASK_REG_TEMP_DELTA_8822B)                                 \
+	 << BIT_SHIFT_REG_TEMP_DELTA_8822B)
+#define BIT_GET_REG_TEMP_DELTA_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_REG_TEMP_DELTA_8822B) &                             \
+	 BIT_MASK_REG_TEMP_DELTA_8822B)
+
+#define BIT_REG_TMETER_EN_8822B BIT(0)
+
+/* 2 REG_OSC_32K_CTRL_8822B */
+
+#define BIT_SHIFT_OSC_32K_CLKGEN_0_8822B 16
+#define BIT_MASK_OSC_32K_CLKGEN_0_8822B 0xffff
+#define BIT_OSC_32K_CLKGEN_0_8822B(x)                                          \
+	(((x) & BIT_MASK_OSC_32K_CLKGEN_0_8822B)                               \
+	 << BIT_SHIFT_OSC_32K_CLKGEN_0_8822B)
+#define BIT_GET_OSC_32K_CLKGEN_0_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_OSC_32K_CLKGEN_0_8822B) &                           \
+	 BIT_MASK_OSC_32K_CLKGEN_0_8822B)
+
+#define BIT_SHIFT_OSC_32K_RES_COMP_8822B 4
+#define BIT_MASK_OSC_32K_RES_COMP_8822B 0x3
+#define BIT_OSC_32K_RES_COMP_8822B(x)                                          \
+	(((x) & BIT_MASK_OSC_32K_RES_COMP_8822B)                               \
+	 << BIT_SHIFT_OSC_32K_RES_COMP_8822B)
+#define BIT_GET_OSC_32K_RES_COMP_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_OSC_32K_RES_COMP_8822B) &                           \
+	 BIT_MASK_OSC_32K_RES_COMP_8822B)
+
+#define BIT_OSC_32K_OUT_SEL_8822B BIT(3)
+#define BIT_ISO_WL_2_OSC_32K_8822B BIT(1)
+#define BIT_POW_CKGEN_8822B BIT(0)
+
+/* 2 REG_32K_CAL_REG1_8822B */
+#define BIT_CAL_32K_REG_WR_8822B BIT(31)
+#define BIT_CAL_32K_DBG_SEL_8822B BIT(22)
+
+#define BIT_SHIFT_CAL_32K_REG_ADDR_8822B 16
+#define BIT_MASK_CAL_32K_REG_ADDR_8822B 0x3f
+#define BIT_CAL_32K_REG_ADDR_8822B(x)                                          \
+	(((x) & BIT_MASK_CAL_32K_REG_ADDR_8822B)                               \
+	 << BIT_SHIFT_CAL_32K_REG_ADDR_8822B)
+#define BIT_GET_CAL_32K_REG_ADDR_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_CAL_32K_REG_ADDR_8822B) &                           \
+	 BIT_MASK_CAL_32K_REG_ADDR_8822B)
+
+#define BIT_SHIFT_CAL_32K_REG_DATA_8822B 0
+#define BIT_MASK_CAL_32K_REG_DATA_8822B 0xffff
+#define BIT_CAL_32K_REG_DATA_8822B(x)                                          \
+	(((x) & BIT_MASK_CAL_32K_REG_DATA_8822B)                               \
+	 << BIT_SHIFT_CAL_32K_REG_DATA_8822B)
+#define BIT_GET_CAL_32K_REG_DATA_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_CAL_32K_REG_DATA_8822B) &                           \
+	 BIT_MASK_CAL_32K_REG_DATA_8822B)
+
+/* 2 REG_NOT_VALID_8822B */
+
+/* 2 REG_C2HEVT_8822B */
+
+#define BIT_SHIFT_C2HEVT_MSG_8822B 0
+#define BIT_MASK_C2HEVT_MSG_8822B 0xffffffffffffffffffffffffffffffffL
+#define BIT_C2HEVT_MSG_8822B(x)                                                \
+	(((x) & BIT_MASK_C2HEVT_MSG_8822B) << BIT_SHIFT_C2HEVT_MSG_8822B)
+#define BIT_GET_C2HEVT_MSG_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_C2HEVT_MSG_8822B) & BIT_MASK_C2HEVT_MSG_8822B)
+
+/* 2 REG_SW_DEFINED_PAGE1_8822B */
+
+#define BIT_SHIFT_SW_DEFINED_PAGE1_8822B 0
+#define BIT_MASK_SW_DEFINED_PAGE1_8822B 0xffffffffffffffffL
+#define BIT_SW_DEFINED_PAGE1_8822B(x)                                          \
+	(((x) & BIT_MASK_SW_DEFINED_PAGE1_8822B)                               \
+	 << BIT_SHIFT_SW_DEFINED_PAGE1_8822B)
+#define BIT_GET_SW_DEFINED_PAGE1_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_SW_DEFINED_PAGE1_8822B) &                           \
+	 BIT_MASK_SW_DEFINED_PAGE1_8822B)
+
+/* 2 REG_MCUTST_I_8822B */
+
+#define BIT_SHIFT_MCUDMSG_I_8822B 0
+#define BIT_MASK_MCUDMSG_I_8822B 0xffffffffL
+#define BIT_MCUDMSG_I_8822B(x)                                                 \
+	(((x) & BIT_MASK_MCUDMSG_I_8822B) << BIT_SHIFT_MCUDMSG_I_8822B)
+#define BIT_GET_MCUDMSG_I_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_MCUDMSG_I_8822B) & BIT_MASK_MCUDMSG_I_8822B)
+
+/* 2 REG_MCUTST_II_8822B */
+
+#define BIT_SHIFT_MCUDMSG_II_8822B 0
+#define BIT_MASK_MCUDMSG_II_8822B 0xffffffffL
+#define BIT_MCUDMSG_II_8822B(x)                                                \
+	(((x) & BIT_MASK_MCUDMSG_II_8822B) << BIT_SHIFT_MCUDMSG_II_8822B)
+#define BIT_GET_MCUDMSG_II_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_MCUDMSG_II_8822B) & BIT_MASK_MCUDMSG_II_8822B)
+
+/* 2 REG_FMETHR_8822B */
+#define BIT_FMSG_INT_8822B BIT(31)
+
+#define BIT_SHIFT_FW_MSG_8822B 0
+#define BIT_MASK_FW_MSG_8822B 0xffffffffL
+#define BIT_FW_MSG_8822B(x)                                                    \
+	(((x) & BIT_MASK_FW_MSG_8822B) << BIT_SHIFT_FW_MSG_8822B)
+#define BIT_GET_FW_MSG_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_FW_MSG_8822B) & BIT_MASK_FW_MSG_8822B)
+
+/* 2 REG_HMETFR_8822B */
+
+#define BIT_SHIFT_HRCV_MSG_8822B 24
+#define BIT_MASK_HRCV_MSG_8822B 0xff
+#define BIT_HRCV_MSG_8822B(x)                                                  \
+	(((x) & BIT_MASK_HRCV_MSG_8822B) << BIT_SHIFT_HRCV_MSG_8822B)
+#define BIT_GET_HRCV_MSG_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_HRCV_MSG_8822B) & BIT_MASK_HRCV_MSG_8822B)
+
+#define BIT_INT_BOX3_8822B BIT(3)
+#define BIT_INT_BOX2_8822B BIT(2)
+#define BIT_INT_BOX1_8822B BIT(1)
+#define BIT_INT_BOX0_8822B BIT(0)
+
+/* 2 REG_HMEBOX0_8822B */
+
+#define BIT_SHIFT_HOST_MSG_0_8822B 0
+#define BIT_MASK_HOST_MSG_0_8822B 0xffffffffL
+#define BIT_HOST_MSG_0_8822B(x)                                                \
+	(((x) & BIT_MASK_HOST_MSG_0_8822B) << BIT_SHIFT_HOST_MSG_0_8822B)
+#define BIT_GET_HOST_MSG_0_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_HOST_MSG_0_8822B) & BIT_MASK_HOST_MSG_0_8822B)
+
+/* 2 REG_HMEBOX1_8822B */
+
+#define BIT_SHIFT_HOST_MSG_1_8822B 0
+#define BIT_MASK_HOST_MSG_1_8822B 0xffffffffL
+#define BIT_HOST_MSG_1_8822B(x)                                                \
+	(((x) & BIT_MASK_HOST_MSG_1_8822B) << BIT_SHIFT_HOST_MSG_1_8822B)
+#define BIT_GET_HOST_MSG_1_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_HOST_MSG_1_8822B) & BIT_MASK_HOST_MSG_1_8822B)
+
+/* 2 REG_HMEBOX2_8822B */
+
+#define BIT_SHIFT_HOST_MSG_2_8822B 0
+#define BIT_MASK_HOST_MSG_2_8822B 0xffffffffL
+#define BIT_HOST_MSG_2_8822B(x)                                                \
+	(((x) & BIT_MASK_HOST_MSG_2_8822B) << BIT_SHIFT_HOST_MSG_2_8822B)
+#define BIT_GET_HOST_MSG_2_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_HOST_MSG_2_8822B) & BIT_MASK_HOST_MSG_2_8822B)
+
+/* 2 REG_HMEBOX3_8822B */
+
+#define BIT_SHIFT_HOST_MSG_3_8822B 0
+#define BIT_MASK_HOST_MSG_3_8822B 0xffffffffL
+#define BIT_HOST_MSG_3_8822B(x)                                                \
+	(((x) & BIT_MASK_HOST_MSG_3_8822B) << BIT_SHIFT_HOST_MSG_3_8822B)
+#define BIT_GET_HOST_MSG_3_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_HOST_MSG_3_8822B) & BIT_MASK_HOST_MSG_3_8822B)
+
+/* 2 REG_LLT_INIT_8822B */
+
+#define BIT_SHIFT_LLTE_RWM_8822B 30
+#define BIT_MASK_LLTE_RWM_8822B 0x3
+#define BIT_LLTE_RWM_8822B(x)                                                  \
+	(((x) & BIT_MASK_LLTE_RWM_8822B) << BIT_SHIFT_LLTE_RWM_8822B)
+#define BIT_GET_LLTE_RWM_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_LLTE_RWM_8822B) & BIT_MASK_LLTE_RWM_8822B)
+
+#define BIT_SHIFT_LLTINI_PDATA_V1_8822B 16
+#define BIT_MASK_LLTINI_PDATA_V1_8822B 0xfff
+#define BIT_LLTINI_PDATA_V1_8822B(x)                                           \
+	(((x) & BIT_MASK_LLTINI_PDATA_V1_8822B)                                \
+	 << BIT_SHIFT_LLTINI_PDATA_V1_8822B)
+#define BIT_GET_LLTINI_PDATA_V1_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_LLTINI_PDATA_V1_8822B) &                            \
+	 BIT_MASK_LLTINI_PDATA_V1_8822B)
+
+#define BIT_SHIFT_LLTINI_HDATA_V1_8822B 0
+#define BIT_MASK_LLTINI_HDATA_V1_8822B 0xfff
+#define BIT_LLTINI_HDATA_V1_8822B(x)                                           \
+	(((x) & BIT_MASK_LLTINI_HDATA_V1_8822B)                                \
+	 << BIT_SHIFT_LLTINI_HDATA_V1_8822B)
+#define BIT_GET_LLTINI_HDATA_V1_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_LLTINI_HDATA_V1_8822B) &                            \
+	 BIT_MASK_LLTINI_HDATA_V1_8822B)
+
+/* 2 REG_LLT_INIT_ADDR_8822B */
+
+#define BIT_SHIFT_LLTINI_ADDR_V1_8822B 0
+#define BIT_MASK_LLTINI_ADDR_V1_8822B 0xfff
+#define BIT_LLTINI_ADDR_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_LLTINI_ADDR_V1_8822B)                                 \
+	 << BIT_SHIFT_LLTINI_ADDR_V1_8822B)
+#define BIT_GET_LLTINI_ADDR_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_LLTINI_ADDR_V1_8822B) &                             \
+	 BIT_MASK_LLTINI_ADDR_V1_8822B)
+
+/* 2 REG_BB_ACCESS_CTRL_8822B */
+
+#define BIT_SHIFT_BB_WRITE_READ_8822B 30
+#define BIT_MASK_BB_WRITE_READ_8822B 0x3
+#define BIT_BB_WRITE_READ_8822B(x)                                             \
+	(((x) & BIT_MASK_BB_WRITE_READ_8822B) << BIT_SHIFT_BB_WRITE_READ_8822B)
+#define BIT_GET_BB_WRITE_READ_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_BB_WRITE_READ_8822B) & BIT_MASK_BB_WRITE_READ_8822B)
+
+#define BIT_SHIFT_BB_WRITE_EN_8822B 12
+#define BIT_MASK_BB_WRITE_EN_8822B 0xf
+#define BIT_BB_WRITE_EN_8822B(x)                                               \
+	(((x) & BIT_MASK_BB_WRITE_EN_8822B) << BIT_SHIFT_BB_WRITE_EN_8822B)
+#define BIT_GET_BB_WRITE_EN_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_BB_WRITE_EN_8822B) & BIT_MASK_BB_WRITE_EN_8822B)
+
+#define BIT_SHIFT_BB_ADDR_8822B 2
+#define BIT_MASK_BB_ADDR_8822B 0x1ff
+#define BIT_BB_ADDR_8822B(x)                                                   \
+	(((x) & BIT_MASK_BB_ADDR_8822B) << BIT_SHIFT_BB_ADDR_8822B)
+#define BIT_GET_BB_ADDR_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_BB_ADDR_8822B) & BIT_MASK_BB_ADDR_8822B)
+
+#define BIT_BB_ERRACC_8822B BIT(0)
+
+/* 2 REG_BB_ACCESS_DATA_8822B */
+
+#define BIT_SHIFT_BB_DATA_8822B 0
+#define BIT_MASK_BB_DATA_8822B 0xffffffffL
+#define BIT_BB_DATA_8822B(x)                                                   \
+	(((x) & BIT_MASK_BB_DATA_8822B) << BIT_SHIFT_BB_DATA_8822B)
+#define BIT_GET_BB_DATA_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_BB_DATA_8822B) & BIT_MASK_BB_DATA_8822B)
+
+/* 2 REG_HMEBOX_E0_8822B */
+
+#define BIT_SHIFT_HMEBOX_E0_8822B 0
+#define BIT_MASK_HMEBOX_E0_8822B 0xffffffffL
+#define BIT_HMEBOX_E0_8822B(x)                                                 \
+	(((x) & BIT_MASK_HMEBOX_E0_8822B) << BIT_SHIFT_HMEBOX_E0_8822B)
+#define BIT_GET_HMEBOX_E0_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_HMEBOX_E0_8822B) & BIT_MASK_HMEBOX_E0_8822B)
+
+/* 2 REG_HMEBOX_E1_8822B */
+
+#define BIT_SHIFT_HMEBOX_E1_8822B 0
+#define BIT_MASK_HMEBOX_E1_8822B 0xffffffffL
+#define BIT_HMEBOX_E1_8822B(x)                                                 \
+	(((x) & BIT_MASK_HMEBOX_E1_8822B) << BIT_SHIFT_HMEBOX_E1_8822B)
+#define BIT_GET_HMEBOX_E1_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_HMEBOX_E1_8822B) & BIT_MASK_HMEBOX_E1_8822B)
+
+/* 2 REG_HMEBOX_E2_8822B */
+
+#define BIT_SHIFT_HMEBOX_E2_8822B 0
+#define BIT_MASK_HMEBOX_E2_8822B 0xffffffffL
+#define BIT_HMEBOX_E2_8822B(x)                                                 \
+	(((x) & BIT_MASK_HMEBOX_E2_8822B) << BIT_SHIFT_HMEBOX_E2_8822B)
+#define BIT_GET_HMEBOX_E2_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_HMEBOX_E2_8822B) & BIT_MASK_HMEBOX_E2_8822B)
+
+/* 2 REG_HMEBOX_E3_8822B */
+
+#define BIT_SHIFT_HMEBOX_E3_8822B 0
+#define BIT_MASK_HMEBOX_E3_8822B 0xffffffffL
+#define BIT_HMEBOX_E3_8822B(x)                                                 \
+	(((x) & BIT_MASK_HMEBOX_E3_8822B) << BIT_SHIFT_HMEBOX_E3_8822B)
+#define BIT_GET_HMEBOX_E3_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_HMEBOX_E3_8822B) & BIT_MASK_HMEBOX_E3_8822B)
+
+/* 2 REG_NOT_VALID_8822B */
+
+/* 2 REG_CR_EXT_8822B */
+
+#define BIT_SHIFT_PHY_REQ_DELAY_8822B 24
+#define BIT_MASK_PHY_REQ_DELAY_8822B 0xf
+#define BIT_PHY_REQ_DELAY_8822B(x)                                             \
+	(((x) & BIT_MASK_PHY_REQ_DELAY_8822B) << BIT_SHIFT_PHY_REQ_DELAY_8822B)
+#define BIT_GET_PHY_REQ_DELAY_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_PHY_REQ_DELAY_8822B) & BIT_MASK_PHY_REQ_DELAY_8822B)
+
+#define BIT_SPD_DOWN_8822B BIT(16)
+
+#define BIT_SHIFT_NETYPE4_8822B 4
+#define BIT_MASK_NETYPE4_8822B 0x3
+#define BIT_NETYPE4_8822B(x)                                                   \
+	(((x) & BIT_MASK_NETYPE4_8822B) << BIT_SHIFT_NETYPE4_8822B)
+#define BIT_GET_NETYPE4_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_NETYPE4_8822B) & BIT_MASK_NETYPE4_8822B)
+
+#define BIT_SHIFT_NETYPE3_8822B 2
+#define BIT_MASK_NETYPE3_8822B 0x3
+#define BIT_NETYPE3_8822B(x)                                                   \
+	(((x) & BIT_MASK_NETYPE3_8822B) << BIT_SHIFT_NETYPE3_8822B)
+#define BIT_GET_NETYPE3_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_NETYPE3_8822B) & BIT_MASK_NETYPE3_8822B)
+
+#define BIT_SHIFT_NETYPE2_8822B 0
+#define BIT_MASK_NETYPE2_8822B 0x3
+#define BIT_NETYPE2_8822B(x)                                                   \
+	(((x) & BIT_MASK_NETYPE2_8822B) << BIT_SHIFT_NETYPE2_8822B)
+#define BIT_GET_NETYPE2_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_NETYPE2_8822B) & BIT_MASK_NETYPE2_8822B)
+
+/* 2 REG_FWFF_8822B */
+
+#define BIT_SHIFT_PKTNUM_TH_V1_8822B 24
+#define BIT_MASK_PKTNUM_TH_V1_8822B 0xff
+#define BIT_PKTNUM_TH_V1_8822B(x)                                              \
+	(((x) & BIT_MASK_PKTNUM_TH_V1_8822B) << BIT_SHIFT_PKTNUM_TH_V1_8822B)
+#define BIT_GET_PKTNUM_TH_V1_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_PKTNUM_TH_V1_8822B) & BIT_MASK_PKTNUM_TH_V1_8822B)
+
+#define BIT_SHIFT_TIMER_TH_8822B 16
+#define BIT_MASK_TIMER_TH_8822B 0xff
+#define BIT_TIMER_TH_8822B(x)                                                  \
+	(((x) & BIT_MASK_TIMER_TH_8822B) << BIT_SHIFT_TIMER_TH_8822B)
+#define BIT_GET_TIMER_TH_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_TIMER_TH_8822B) & BIT_MASK_TIMER_TH_8822B)
+
+#define BIT_SHIFT_RXPKT1ENADDR_8822B 0
+#define BIT_MASK_RXPKT1ENADDR_8822B 0xffff
+#define BIT_RXPKT1ENADDR_8822B(x)                                              \
+	(((x) & BIT_MASK_RXPKT1ENADDR_8822B) << BIT_SHIFT_RXPKT1ENADDR_8822B)
+#define BIT_GET_RXPKT1ENADDR_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_RXPKT1ENADDR_8822B) & BIT_MASK_RXPKT1ENADDR_8822B)
+
+/* 2 REG_RXFF_PTR_V1_8822B */
+
+/* 2 REG_NOT_VALID_8822B */
+
+#define BIT_SHIFT_RXFF0_RDPTR_V2_8822B 0
+#define BIT_MASK_RXFF0_RDPTR_V2_8822B 0x3ffff
+#define BIT_RXFF0_RDPTR_V2_8822B(x)                                            \
+	(((x) & BIT_MASK_RXFF0_RDPTR_V2_8822B)                                 \
+	 << BIT_SHIFT_RXFF0_RDPTR_V2_8822B)
+#define BIT_GET_RXFF0_RDPTR_V2_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_RXFF0_RDPTR_V2_8822B) &                             \
+	 BIT_MASK_RXFF0_RDPTR_V2_8822B)
+
+/* 2 REG_RXFF_WTR_V1_8822B */
+
+/* 2 REG_NOT_VALID_8822B */
+
+#define BIT_SHIFT_RXFF0_WTPTR_V2_8822B 0
+#define BIT_MASK_RXFF0_WTPTR_V2_8822B 0x3ffff
+#define BIT_RXFF0_WTPTR_V2_8822B(x)                                            \
+	(((x) & BIT_MASK_RXFF0_WTPTR_V2_8822B)                                 \
+	 << BIT_SHIFT_RXFF0_WTPTR_V2_8822B)
+#define BIT_GET_RXFF0_WTPTR_V2_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_RXFF0_WTPTR_V2_8822B) &                             \
+	 BIT_MASK_RXFF0_WTPTR_V2_8822B)
+
+/* 2 REG_FE2IMR_8822B */
+#define BIT__FE4ISR__IND_MSK_8822B BIT(29)
+#define BIT_FS_TXSC_DESC_DONE_INT_EN_8822B BIT(28)
+#define BIT_FS_TXSC_BKDONE_INT_EN_8822B BIT(27)
+#define BIT_FS_TXSC_BEDONE_INT_EN_8822B BIT(26)
+#define BIT_FS_TXSC_VIDONE_INT_EN_8822B BIT(25)
+#define BIT_FS_TXSC_VODONE_INT_EN_8822B BIT(24)
+#define BIT_FS_ATIM_MB7_INT_EN_8822B BIT(23)
+#define BIT_FS_ATIM_MB6_INT_EN_8822B BIT(22)
+#define BIT_FS_ATIM_MB5_INT_EN_8822B BIT(21)
+#define BIT_FS_ATIM_MB4_INT_EN_8822B BIT(20)
+#define BIT_FS_ATIM_MB3_INT_EN_8822B BIT(19)
+#define BIT_FS_ATIM_MB2_INT_EN_8822B BIT(18)
+#define BIT_FS_ATIM_MB1_INT_EN_8822B BIT(17)
+#define BIT_FS_ATIM_MB0_INT_EN_8822B BIT(16)
+#define BIT_FS_TBTT4INT_EN_8822B BIT(11)
+#define BIT_FS_TBTT3INT_EN_8822B BIT(10)
+#define BIT_FS_TBTT2INT_EN_8822B BIT(9)
+#define BIT_FS_TBTT1INT_EN_8822B BIT(8)
+#define BIT_FS_TBTT0_MB7INT_EN_8822B BIT(7)
+#define BIT_FS_TBTT0_MB6INT_EN_8822B BIT(6)
+#define BIT_FS_TBTT0_MB5INT_EN_8822B BIT(5)
+#define BIT_FS_TBTT0_MB4INT_EN_8822B BIT(4)
+#define BIT_FS_TBTT0_MB3INT_EN_8822B BIT(3)
+#define BIT_FS_TBTT0_MB2INT_EN_8822B BIT(2)
+#define BIT_FS_TBTT0_MB1INT_EN_8822B BIT(1)
+#define BIT_FS_TBTT0_INT_EN_8822B BIT(0)
+
+/* 2 REG_FE2ISR_8822B */
+#define BIT__FE4ISR__IND_INT_8822B BIT(29)
+#define BIT_FS_TXSC_DESC_DONE_INT_8822B BIT(28)
+#define BIT_FS_TXSC_BKDONE_INT_8822B BIT(27)
+#define BIT_FS_TXSC_BEDONE_INT_8822B BIT(26)
+#define BIT_FS_TXSC_VIDONE_INT_8822B BIT(25)
+#define BIT_FS_TXSC_VODONE_INT_8822B BIT(24)
+#define BIT_FS_ATIM_MB7_INT_8822B BIT(23)
+#define BIT_FS_ATIM_MB6_INT_8822B BIT(22)
+#define BIT_FS_ATIM_MB5_INT_8822B BIT(21)
+#define BIT_FS_ATIM_MB4_INT_8822B BIT(20)
+#define BIT_FS_ATIM_MB3_INT_8822B BIT(19)
+#define BIT_FS_ATIM_MB2_INT_8822B BIT(18)
+#define BIT_FS_ATIM_MB1_INT_8822B BIT(17)
+#define BIT_FS_ATIM_MB0_INT_8822B BIT(16)
+#define BIT_FS_TBTT4INT_8822B BIT(11)
+#define BIT_FS_TBTT3INT_8822B BIT(10)
+#define BIT_FS_TBTT2INT_8822B BIT(9)
+#define BIT_FS_TBTT1INT_8822B BIT(8)
+#define BIT_FS_TBTT0_MB7INT_8822B BIT(7)
+#define BIT_FS_TBTT0_MB6INT_8822B BIT(6)
+#define BIT_FS_TBTT0_MB5INT_8822B BIT(5)
+#define BIT_FS_TBTT0_MB4INT_8822B BIT(4)
+#define BIT_FS_TBTT0_MB3INT_8822B BIT(3)
+#define BIT_FS_TBTT0_MB2INT_8822B BIT(2)
+#define BIT_FS_TBTT0_MB1INT_8822B BIT(1)
+#define BIT_FS_TBTT0_INT_8822B BIT(0)
+
+/* 2 REG_FE3IMR_8822B */
+#define BIT_FS_CLI3_MTI_BCNIVLEAR_INT__EN_8822B BIT(31)
+#define BIT_FS_CLI2_MTI_BCNIVLEAR_INT__EN_8822B BIT(30)
+#define BIT_FS_CLI1_MTI_BCNIVLEAR_INT__EN_8822B BIT(29)
+#define BIT_FS_CLI0_MTI_BCNIVLEAR_INT__EN_8822B BIT(28)
+#define BIT_FS_BCNDMA4_INT_EN_8822B BIT(27)
+#define BIT_FS_BCNDMA3_INT_EN_8822B BIT(26)
+#define BIT_FS_BCNDMA2_INT_EN_8822B BIT(25)
+#define BIT_FS_BCNDMA1_INT_EN_8822B BIT(24)
+#define BIT_FS_BCNDMA0_MB7_INT_EN_8822B BIT(23)
+#define BIT_FS_BCNDMA0_MB6_INT_EN_8822B BIT(22)
+#define BIT_FS_BCNDMA0_MB5_INT_EN_8822B BIT(21)
+#define BIT_FS_BCNDMA0_MB4_INT_EN_8822B BIT(20)
+#define BIT_FS_BCNDMA0_MB3_INT_EN_8822B BIT(19)
+#define BIT_FS_BCNDMA0_MB2_INT_EN_8822B BIT(18)
+#define BIT_FS_BCNDMA0_MB1_INT_EN_8822B BIT(17)
+#define BIT_FS_BCNDMA0_INT_EN_8822B BIT(16)
+#define BIT_FS_MTI_BCNIVLEAR_INT__EN_8822B BIT(15)
+#define BIT_FS_BCNERLY4_INT_EN_8822B BIT(11)
+#define BIT_FS_BCNERLY3_INT_EN_8822B BIT(10)
+#define BIT_FS_BCNERLY2_INT_EN_8822B BIT(9)
+#define BIT_FS_BCNERLY1_INT_EN_8822B BIT(8)
+#define BIT_FS_BCNERLY0_MB7INT_EN_8822B BIT(7)
+#define BIT_FS_BCNERLY0_MB6INT_EN_8822B BIT(6)
+#define BIT_FS_BCNERLY0_MB5INT_EN_8822B BIT(5)
+#define BIT_FS_BCNERLY0_MB4INT_EN_8822B BIT(4)
+#define BIT_FS_BCNERLY0_MB3INT_EN_8822B BIT(3)
+#define BIT_FS_BCNERLY0_MB2INT_EN_8822B BIT(2)
+#define BIT_FS_BCNERLY0_MB1INT_EN_8822B BIT(1)
+#define BIT_FS_BCNERLY0_INT_EN_8822B BIT(0)
+
+/* 2 REG_FE3ISR_8822B */
+#define BIT_FS_CLI3_MTI_BCNIVLEAR_INT_8822B BIT(31)
+#define BIT_FS_CLI2_MTI_BCNIVLEAR_INT_8822B BIT(30)
+#define BIT_FS_CLI1_MTI_BCNIVLEAR_INT_8822B BIT(29)
+#define BIT_FS_CLI0_MTI_BCNIVLEAR_INT_8822B BIT(28)
+#define BIT_FS_BCNDMA4_INT_8822B BIT(27)
+#define BIT_FS_BCNDMA3_INT_8822B BIT(26)
+#define BIT_FS_BCNDMA2_INT_8822B BIT(25)
+#define BIT_FS_BCNDMA1_INT_8822B BIT(24)
+#define BIT_FS_BCNDMA0_MB7_INT_8822B BIT(23)
+#define BIT_FS_BCNDMA0_MB6_INT_8822B BIT(22)
+#define BIT_FS_BCNDMA0_MB5_INT_8822B BIT(21)
+#define BIT_FS_BCNDMA0_MB4_INT_8822B BIT(20)
+#define BIT_FS_BCNDMA0_MB3_INT_8822B BIT(19)
+#define BIT_FS_BCNDMA0_MB2_INT_8822B BIT(18)
+#define BIT_FS_BCNDMA0_MB1_INT_8822B BIT(17)
+#define BIT_FS_BCNDMA0_INT_8822B BIT(16)
+#define BIT_FS_MTI_BCNIVLEAR_INT_8822B BIT(15)
+#define BIT_FS_BCNERLY4_INT_8822B BIT(11)
+#define BIT_FS_BCNERLY3_INT_8822B BIT(10)
+#define BIT_FS_BCNERLY2_INT_8822B BIT(9)
+#define BIT_FS_BCNERLY1_INT_8822B BIT(8)
+#define BIT_FS_BCNERLY0_MB7INT_8822B BIT(7)
+#define BIT_FS_BCNERLY0_MB6INT_8822B BIT(6)
+#define BIT_FS_BCNERLY0_MB5INT_8822B BIT(5)
+#define BIT_FS_BCNERLY0_MB4INT_8822B BIT(4)
+#define BIT_FS_BCNERLY0_MB3INT_8822B BIT(3)
+#define BIT_FS_BCNERLY0_MB2INT_8822B BIT(2)
+#define BIT_FS_BCNERLY0_MB1INT_8822B BIT(1)
+#define BIT_FS_BCNERLY0_INT_8822B BIT(0)
+
+/* 2 REG_FE4IMR_8822B */
+#define BIT_FS_CLI3_TXPKTIN_INT_EN_8822B BIT(19)
+#define BIT_FS_CLI2_TXPKTIN_INT_EN_8822B BIT(18)
+#define BIT_FS_CLI1_TXPKTIN_INT_EN_8822B BIT(17)
+#define BIT_FS_CLI0_TXPKTIN_INT_EN_8822B BIT(16)
+#define BIT_FS_CLI3_RX_UMD0_INT_EN_8822B BIT(15)
+#define BIT_FS_CLI3_RX_UMD1_INT_EN_8822B BIT(14)
+#define BIT_FS_CLI3_RX_BMD0_INT_EN_8822B BIT(13)
+#define BIT_FS_CLI3_RX_BMD1_INT_EN_8822B BIT(12)
+#define BIT_FS_CLI2_RX_UMD0_INT_EN_8822B BIT(11)
+#define BIT_FS_CLI2_RX_UMD1_INT_EN_8822B BIT(10)
+#define BIT_FS_CLI2_RX_BMD0_INT_EN_8822B BIT(9)
+#define BIT_FS_CLI2_RX_BMD1_INT_EN_8822B BIT(8)
+#define BIT_FS_CLI1_RX_UMD0_INT_EN_8822B BIT(7)
+#define BIT_FS_CLI1_RX_UMD1_INT_EN_8822B BIT(6)
+#define BIT_FS_CLI1_RX_BMD0_INT_EN_8822B BIT(5)
+#define BIT_FS_CLI1_RX_BMD1_INT_EN_8822B BIT(4)
+#define BIT_FS_CLI0_RX_UMD0_INT_EN_8822B BIT(3)
+#define BIT_FS_CLI0_RX_UMD1_INT_EN_8822B BIT(2)
+#define BIT_FS_CLI0_RX_BMD0_INT_EN_8822B BIT(1)
+#define BIT_FS_CLI0_RX_BMD1_INT_EN_8822B BIT(0)
+
+/* 2 REG_FE4ISR_8822B */
+#define BIT_FS_CLI3_TXPKTIN_INT_8822B BIT(19)
+#define BIT_FS_CLI2_TXPKTIN_INT_8822B BIT(18)
+#define BIT_FS_CLI1_TXPKTIN_INT_8822B BIT(17)
+#define BIT_FS_CLI0_TXPKTIN_INT_8822B BIT(16)
+#define BIT_FS_CLI3_RX_UMD0_INT_8822B BIT(15)
+#define BIT_FS_CLI3_RX_UMD1_INT_8822B BIT(14)
+#define BIT_FS_CLI3_RX_BMD0_INT_8822B BIT(13)
+#define BIT_FS_CLI3_RX_BMD1_INT_8822B BIT(12)
+#define BIT_FS_CLI2_RX_UMD0_INT_8822B BIT(11)
+#define BIT_FS_CLI2_RX_UMD1_INT_8822B BIT(10)
+#define BIT_FS_CLI2_RX_BMD0_INT_8822B BIT(9)
+#define BIT_FS_CLI2_RX_BMD1_INT_8822B BIT(8)
+#define BIT_FS_CLI1_RX_UMD0_INT_8822B BIT(7)
+#define BIT_FS_CLI1_RX_UMD1_INT_8822B BIT(6)
+#define BIT_FS_CLI1_RX_BMD0_INT_8822B BIT(5)
+#define BIT_FS_CLI1_RX_BMD1_INT_8822B BIT(4)
+#define BIT_FS_CLI0_RX_UMD0_INT_8822B BIT(3)
+#define BIT_FS_CLI0_RX_UMD1_INT_8822B BIT(2)
+#define BIT_FS_CLI0_RX_BMD0_INT_8822B BIT(1)
+#define BIT_FS_CLI0_RX_BMD1_INT_8822B BIT(0)
+
+/* 2 REG_FT1IMR_8822B */
+#define BIT__FT2ISR__IND_MSK_8822B BIT(30)
+#define BIT_FTM_PTT_INT_EN_8822B BIT(29)
+#define BIT_RXFTMREQ_INT_EN_8822B BIT(28)
+#define BIT_RXFTM_INT_EN_8822B BIT(27)
+#define BIT_TXFTM_INT_EN_8822B BIT(26)
+#define BIT_FS_H2C_CMD_OK_INT_EN_8822B BIT(25)
+#define BIT_FS_H2C_CMD_FULL_INT_EN_8822B BIT(24)
+#define BIT_FS_MACID_PWRCHANGE5_INT_EN_8822B BIT(23)
+#define BIT_FS_MACID_PWRCHANGE4_INT_EN_8822B BIT(22)
+#define BIT_FS_MACID_PWRCHANGE3_INT_EN_8822B BIT(21)
+#define BIT_FS_MACID_PWRCHANGE2_INT_EN_8822B BIT(20)
+#define BIT_FS_MACID_PWRCHANGE1_INT_EN_8822B BIT(19)
+#define BIT_FS_MACID_PWRCHANGE0_INT_EN_8822B BIT(18)
+#define BIT_FS_CTWEND2_INT_EN_8822B BIT(17)
+#define BIT_FS_CTWEND1_INT_EN_8822B BIT(16)
+#define BIT_FS_CTWEND0_INT_EN_8822B BIT(15)
+#define BIT_FS_TX_NULL1_INT_EN_8822B BIT(14)
+#define BIT_FS_TX_NULL0_INT_EN_8822B BIT(13)
+#define BIT_FS_TSF_BIT32_TOGGLE_EN_8822B BIT(12)
+#define BIT_FS_P2P_RFON2_INT_EN_8822B BIT(11)
+#define BIT_FS_P2P_RFOFF2_INT_EN_8822B BIT(10)
+#define BIT_FS_P2P_RFON1_INT_EN_8822B BIT(9)
+#define BIT_FS_P2P_RFOFF1_INT_EN_8822B BIT(8)
+#define BIT_FS_P2P_RFON0_INT_EN_8822B BIT(7)
+#define BIT_FS_P2P_RFOFF0_INT_EN_8822B BIT(6)
+#define BIT_FS_RX_UAPSDMD1_EN_8822B BIT(5)
+#define BIT_FS_RX_UAPSDMD0_EN_8822B BIT(4)
+#define BIT_FS_TRIGGER_PKT_EN_8822B BIT(3)
+#define BIT_FS_EOSP_INT_EN_8822B BIT(2)
+#define BIT_FS_RPWM2_INT_EN_8822B BIT(1)
+#define BIT_FS_RPWM_INT_EN_8822B BIT(0)
+
+/* 2 REG_FT1ISR_8822B */
+#define BIT__FT2ISR__IND_INT_8822B BIT(30)
+#define BIT_FTM_PTT_INT_8822B BIT(29)
+#define BIT_RXFTMREQ_INT_8822B BIT(28)
+#define BIT_RXFTM_INT_8822B BIT(27)
+#define BIT_TXFTM_INT_8822B BIT(26)
+#define BIT_FS_H2C_CMD_OK_INT_8822B BIT(25)
+#define BIT_FS_H2C_CMD_FULL_INT_8822B BIT(24)
+#define BIT_FS_MACID_PWRCHANGE5_INT_8822B BIT(23)
+#define BIT_FS_MACID_PWRCHANGE4_INT_8822B BIT(22)
+#define BIT_FS_MACID_PWRCHANGE3_INT_8822B BIT(21)
+#define BIT_FS_MACID_PWRCHANGE2_INT_8822B BIT(20)
+#define BIT_FS_MACID_PWRCHANGE1_INT_8822B BIT(19)
+#define BIT_FS_MACID_PWRCHANGE0_INT_8822B BIT(18)
+#define BIT_FS_CTWEND2_INT_8822B BIT(17)
+#define BIT_FS_CTWEND1_INT_8822B BIT(16)
+#define BIT_FS_CTWEND0_INT_8822B BIT(15)
+#define BIT_FS_TX_NULL1_INT_8822B BIT(14)
+#define BIT_FS_TX_NULL0_INT_8822B BIT(13)
+#define BIT_FS_TSF_BIT32_TOGGLE_INT_8822B BIT(12)
+#define BIT_FS_P2P_RFON2_INT_8822B BIT(11)
+#define BIT_FS_P2P_RFOFF2_INT_8822B BIT(10)
+#define BIT_FS_P2P_RFON1_INT_8822B BIT(9)
+#define BIT_FS_P2P_RFOFF1_INT_8822B BIT(8)
+#define BIT_FS_P2P_RFON0_INT_8822B BIT(7)
+#define BIT_FS_P2P_RFOFF0_INT_8822B BIT(6)
+#define BIT_FS_RX_UAPSDMD1_INT_8822B BIT(5)
+#define BIT_FS_RX_UAPSDMD0_INT_8822B BIT(4)
+#define BIT_FS_TRIGGER_PKT_INT_8822B BIT(3)
+#define BIT_FS_EOSP_INT_8822B BIT(2)
+#define BIT_FS_RPWM2_INT_8822B BIT(1)
+#define BIT_FS_RPWM_INT_8822B BIT(0)
+
+/* 2 REG_SPWR0_8822B */
+
+#define BIT_SHIFT_MID_31TO0_8822B 0
+#define BIT_MASK_MID_31TO0_8822B 0xffffffffL
+#define BIT_MID_31TO0_8822B(x)                                                 \
+	(((x) & BIT_MASK_MID_31TO0_8822B) << BIT_SHIFT_MID_31TO0_8822B)
+#define BIT_GET_MID_31TO0_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_MID_31TO0_8822B) & BIT_MASK_MID_31TO0_8822B)
+
+/* 2 REG_SPWR1_8822B */
+
+#define BIT_SHIFT_MID_63TO32_8822B 0
+#define BIT_MASK_MID_63TO32_8822B 0xffffffffL
+#define BIT_MID_63TO32_8822B(x)                                                \
+	(((x) & BIT_MASK_MID_63TO32_8822B) << BIT_SHIFT_MID_63TO32_8822B)
+#define BIT_GET_MID_63TO32_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_MID_63TO32_8822B) & BIT_MASK_MID_63TO32_8822B)
+
+/* 2 REG_SPWR2_8822B */
+
+#define BIT_SHIFT_MID_95O64_8822B 0
+#define BIT_MASK_MID_95O64_8822B 0xffffffffL
+#define BIT_MID_95O64_8822B(x)                                                 \
+	(((x) & BIT_MASK_MID_95O64_8822B) << BIT_SHIFT_MID_95O64_8822B)
+#define BIT_GET_MID_95O64_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_MID_95O64_8822B) & BIT_MASK_MID_95O64_8822B)
+
+/* 2 REG_SPWR3_8822B */
+
+#define BIT_SHIFT_MID_127TO96_8822B 0
+#define BIT_MASK_MID_127TO96_8822B 0xffffffffL
+#define BIT_MID_127TO96_8822B(x)                                               \
+	(((x) & BIT_MASK_MID_127TO96_8822B) << BIT_SHIFT_MID_127TO96_8822B)
+#define BIT_GET_MID_127TO96_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_MID_127TO96_8822B) & BIT_MASK_MID_127TO96_8822B)
+
+/* 2 REG_POWSEQ_8822B */
+
+#define BIT_SHIFT_SEQNUM_MID_8822B 16
+#define BIT_MASK_SEQNUM_MID_8822B 0xffff
+#define BIT_SEQNUM_MID_8822B(x)                                                \
+	(((x) & BIT_MASK_SEQNUM_MID_8822B) << BIT_SHIFT_SEQNUM_MID_8822B)
+#define BIT_GET_SEQNUM_MID_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_SEQNUM_MID_8822B) & BIT_MASK_SEQNUM_MID_8822B)
+
+#define BIT_SHIFT_REF_MID_8822B 0
+#define BIT_MASK_REF_MID_8822B 0x7f
+#define BIT_REF_MID_8822B(x)                                                   \
+	(((x) & BIT_MASK_REF_MID_8822B) << BIT_SHIFT_REF_MID_8822B)
+#define BIT_GET_REF_MID_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_REF_MID_8822B) & BIT_MASK_REF_MID_8822B)
+
+/* 2 REG_TC7_CTRL_V1_8822B */
+#define BIT_TC7INT_EN_8822B BIT(26)
+#define BIT_TC7MODE_8822B BIT(25)
+#define BIT_TC7EN_8822B BIT(24)
+
+#define BIT_SHIFT_TC7DATA_8822B 0
+#define BIT_MASK_TC7DATA_8822B 0xffffff
+#define BIT_TC7DATA_8822B(x)                                                   \
+	(((x) & BIT_MASK_TC7DATA_8822B) << BIT_SHIFT_TC7DATA_8822B)
+#define BIT_GET_TC7DATA_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_TC7DATA_8822B) & BIT_MASK_TC7DATA_8822B)
+
+/* 2 REG_TC8_CTRL_V1_8822B */
+#define BIT_TC8INT_EN_8822B BIT(26)
+#define BIT_TC8MODE_8822B BIT(25)
+#define BIT_TC8EN_8822B BIT(24)
+
+#define BIT_SHIFT_TC8DATA_8822B 0
+#define BIT_MASK_TC8DATA_8822B 0xffffff
+#define BIT_TC8DATA_8822B(x)                                                   \
+	(((x) & BIT_MASK_TC8DATA_8822B) << BIT_SHIFT_TC8DATA_8822B)
+#define BIT_GET_TC8DATA_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_TC8DATA_8822B) & BIT_MASK_TC8DATA_8822B)
+
+/* 2 REG_FT2IMR_8822B */
+#define BIT_FS_CLI3_RX_UAPSDMD1_EN_8822B BIT(31)
+#define BIT_FS_CLI3_RX_UAPSDMD0_EN_8822B BIT(30)
+#define BIT_FS_CLI3_TRIGGER_PKT_EN_8822B BIT(29)
+#define BIT_FS_CLI3_EOSP_INT_EN_8822B BIT(28)
+#define BIT_FS_CLI2_RX_UAPSDMD1_EN_8822B BIT(27)
+#define BIT_FS_CLI2_RX_UAPSDMD0_EN_8822B BIT(26)
+#define BIT_FS_CLI2_TRIGGER_PKT_EN_8822B BIT(25)
+#define BIT_FS_CLI2_EOSP_INT_EN_8822B BIT(24)
+#define BIT_FS_CLI1_RX_UAPSDMD1_EN_8822B BIT(23)
+#define BIT_FS_CLI1_RX_UAPSDMD0_EN_8822B BIT(22)
+#define BIT_FS_CLI1_TRIGGER_PKT_EN_8822B BIT(21)
+#define BIT_FS_CLI1_EOSP_INT_EN_8822B BIT(20)
+#define BIT_FS_CLI0_RX_UAPSDMD1_EN_8822B BIT(19)
+#define BIT_FS_CLI0_RX_UAPSDMD0_EN_8822B BIT(18)
+#define BIT_FS_CLI0_TRIGGER_PKT_EN_8822B BIT(17)
+#define BIT_FS_CLI0_EOSP_INT_EN_8822B BIT(16)
+#define BIT_FS_TSF_BIT32_TOGGLE_P2P2_EN_8822B BIT(9)
+#define BIT_FS_TSF_BIT32_TOGGLE_P2P1_EN_8822B BIT(8)
+#define BIT_FS_CLI3_TX_NULL1_INT_EN_8822B BIT(7)
+#define BIT_FS_CLI3_TX_NULL0_INT_EN_8822B BIT(6)
+#define BIT_FS_CLI2_TX_NULL1_INT_EN_8822B BIT(5)
+#define BIT_FS_CLI2_TX_NULL0_INT_EN_8822B BIT(4)
+#define BIT_FS_CLI1_TX_NULL1_INT_EN_8822B BIT(3)
+#define BIT_FS_CLI1_TX_NULL0_INT_EN_8822B BIT(2)
+#define BIT_FS_CLI0_TX_NULL1_INT_EN_8822B BIT(1)
+#define BIT_FS_CLI0_TX_NULL0_INT_EN_8822B BIT(0)
+
+/* 2 REG_FT2ISR_8822B */
+#define BIT_FS_CLI3_RX_UAPSDMD1_INT_8822B BIT(31)
+#define BIT_FS_CLI3_RX_UAPSDMD0_INT_8822B BIT(30)
+#define BIT_FS_CLI3_TRIGGER_PKT_INT_8822B BIT(29)
+#define BIT_FS_CLI3_EOSP_INT_8822B BIT(28)
+#define BIT_FS_CLI2_RX_UAPSDMD1_INT_8822B BIT(27)
+#define BIT_FS_CLI2_RX_UAPSDMD0_INT_8822B BIT(26)
+#define BIT_FS_CLI2_TRIGGER_PKT_INT_8822B BIT(25)
+#define BIT_FS_CLI2_EOSP_INT_8822B BIT(24)
+#define BIT_FS_CLI1_RX_UAPSDMD1_INT_8822B BIT(23)
+#define BIT_FS_CLI1_RX_UAPSDMD0_INT_8822B BIT(22)
+#define BIT_FS_CLI1_TRIGGER_PKT_INT_8822B BIT(21)
+#define BIT_FS_CLI1_EOSP_INT_8822B BIT(20)
+#define BIT_FS_CLI0_RX_UAPSDMD1_INT_8822B BIT(19)
+#define BIT_FS_CLI0_RX_UAPSDMD0_INT_8822B BIT(18)
+#define BIT_FS_CLI0_TRIGGER_PKT_INT_8822B BIT(17)
+#define BIT_FS_CLI0_EOSP_INT_8822B BIT(16)
+#define BIT_FS_TSF_BIT32_TOGGLE_P2P2_INT_8822B BIT(9)
+#define BIT_FS_TSF_BIT32_TOGGLE_P2P1_INT_8822B BIT(8)
+#define BIT_FS_CLI3_TX_NULL1_INT_8822B BIT(7)
+#define BIT_FS_CLI3_TX_NULL0_INT_8822B BIT(6)
+#define BIT_FS_CLI2_TX_NULL1_INT_8822B BIT(5)
+#define BIT_FS_CLI2_TX_NULL0_INT_8822B BIT(4)
+#define BIT_FS_CLI1_TX_NULL1_INT_8822B BIT(3)
+#define BIT_FS_CLI1_TX_NULL0_INT_8822B BIT(2)
+#define BIT_FS_CLI0_TX_NULL1_INT_8822B BIT(1)
+#define BIT_FS_CLI0_TX_NULL0_INT_8822B BIT(0)
+
+/* 2 REG_MSG2_8822B */
+
+#define BIT_SHIFT_FW_MSG2_8822B 0
+#define BIT_MASK_FW_MSG2_8822B 0xffffffffL
+#define BIT_FW_MSG2_8822B(x)                                                   \
+	(((x) & BIT_MASK_FW_MSG2_8822B) << BIT_SHIFT_FW_MSG2_8822B)
+#define BIT_GET_FW_MSG2_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_FW_MSG2_8822B) & BIT_MASK_FW_MSG2_8822B)
+
+/* 2 REG_MSG3_8822B */
+
+#define BIT_SHIFT_FW_MSG3_8822B 0
+#define BIT_MASK_FW_MSG3_8822B 0xffffffffL
+#define BIT_FW_MSG3_8822B(x)                                                   \
+	(((x) & BIT_MASK_FW_MSG3_8822B) << BIT_SHIFT_FW_MSG3_8822B)
+#define BIT_GET_FW_MSG3_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_FW_MSG3_8822B) & BIT_MASK_FW_MSG3_8822B)
+
+/* 2 REG_MSG4_8822B */
+
+#define BIT_SHIFT_FW_MSG4_8822B 0
+#define BIT_MASK_FW_MSG4_8822B 0xffffffffL
+#define BIT_FW_MSG4_8822B(x)                                                   \
+	(((x) & BIT_MASK_FW_MSG4_8822B) << BIT_SHIFT_FW_MSG4_8822B)
+#define BIT_GET_FW_MSG4_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_FW_MSG4_8822B) & BIT_MASK_FW_MSG4_8822B)
+
+/* 2 REG_MSG5_8822B */
+
+#define BIT_SHIFT_FW_MSG5_8822B 0
+#define BIT_MASK_FW_MSG5_8822B 0xffffffffL
+#define BIT_FW_MSG5_8822B(x)                                                   \
+	(((x) & BIT_MASK_FW_MSG5_8822B) << BIT_SHIFT_FW_MSG5_8822B)
+#define BIT_GET_FW_MSG5_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_FW_MSG5_8822B) & BIT_MASK_FW_MSG5_8822B)
+
+/* 2 REG_NOT_VALID_8822B */
+
+/* 2 REG_FIFOPAGE_CTRL_1_8822B */
+
+#define BIT_SHIFT_TX_OQT_HE_FREE_SPACE_V1_8822B 16
+#define BIT_MASK_TX_OQT_HE_FREE_SPACE_V1_8822B 0xff
+#define BIT_TX_OQT_HE_FREE_SPACE_V1_8822B(x)                                   \
+	(((x) & BIT_MASK_TX_OQT_HE_FREE_SPACE_V1_8822B)                        \
+	 << BIT_SHIFT_TX_OQT_HE_FREE_SPACE_V1_8822B)
+#define BIT_GET_TX_OQT_HE_FREE_SPACE_V1_8822B(x)                               \
+	(((x) >> BIT_SHIFT_TX_OQT_HE_FREE_SPACE_V1_8822B) &                    \
+	 BIT_MASK_TX_OQT_HE_FREE_SPACE_V1_8822B)
+
+#define BIT_SHIFT_TX_OQT_NL_FREE_SPACE_V1_8822B 0
+#define BIT_MASK_TX_OQT_NL_FREE_SPACE_V1_8822B 0xff
+#define BIT_TX_OQT_NL_FREE_SPACE_V1_8822B(x)                                   \
+	(((x) & BIT_MASK_TX_OQT_NL_FREE_SPACE_V1_8822B)                        \
+	 << BIT_SHIFT_TX_OQT_NL_FREE_SPACE_V1_8822B)
+#define BIT_GET_TX_OQT_NL_FREE_SPACE_V1_8822B(x)                               \
+	(((x) >> BIT_SHIFT_TX_OQT_NL_FREE_SPACE_V1_8822B) &                    \
+	 BIT_MASK_TX_OQT_NL_FREE_SPACE_V1_8822B)
+
+/* 2 REG_FIFOPAGE_CTRL_2_8822B */
+#define BIT_BCN_VALID_1_V1_8822B BIT(31)
+
+#define BIT_SHIFT_BCN_HEAD_1_V1_8822B 16
+#define BIT_MASK_BCN_HEAD_1_V1_8822B 0xfff
+#define BIT_BCN_HEAD_1_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_BCN_HEAD_1_V1_8822B) << BIT_SHIFT_BCN_HEAD_1_V1_8822B)
+#define BIT_GET_BCN_HEAD_1_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_BCN_HEAD_1_V1_8822B) & BIT_MASK_BCN_HEAD_1_V1_8822B)
+
+#define BIT_BCN_VALID_V1_8822B BIT(15)
+
+#define BIT_SHIFT_BCN_HEAD_V1_8822B 0
+#define BIT_MASK_BCN_HEAD_V1_8822B 0xfff
+#define BIT_BCN_HEAD_V1_8822B(x)                                               \
+	(((x) & BIT_MASK_BCN_HEAD_V1_8822B) << BIT_SHIFT_BCN_HEAD_V1_8822B)
+#define BIT_GET_BCN_HEAD_V1_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_BCN_HEAD_V1_8822B) & BIT_MASK_BCN_HEAD_V1_8822B)
+
+/* 2 REG_AUTO_LLT_V1_8822B */
+
+#define BIT_SHIFT_MAX_TX_PKT_FOR_USB_AND_SDIO_V1_8822B 24
+#define BIT_MASK_MAX_TX_PKT_FOR_USB_AND_SDIO_V1_8822B 0xff
+#define BIT_MAX_TX_PKT_FOR_USB_AND_SDIO_V1_8822B(x)                            \
+	(((x) & BIT_MASK_MAX_TX_PKT_FOR_USB_AND_SDIO_V1_8822B)                 \
+	 << BIT_SHIFT_MAX_TX_PKT_FOR_USB_AND_SDIO_V1_8822B)
+#define BIT_GET_MAX_TX_PKT_FOR_USB_AND_SDIO_V1_8822B(x)                        \
+	(((x) >> BIT_SHIFT_MAX_TX_PKT_FOR_USB_AND_SDIO_V1_8822B) &             \
+	 BIT_MASK_MAX_TX_PKT_FOR_USB_AND_SDIO_V1_8822B)
+
+#define BIT_SHIFT_LLT_FREE_PAGE_V1_8822B 8
+#define BIT_MASK_LLT_FREE_PAGE_V1_8822B 0xffff
+#define BIT_LLT_FREE_PAGE_V1_8822B(x)                                          \
+	(((x) & BIT_MASK_LLT_FREE_PAGE_V1_8822B)                               \
+	 << BIT_SHIFT_LLT_FREE_PAGE_V1_8822B)
+#define BIT_GET_LLT_FREE_PAGE_V1_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_LLT_FREE_PAGE_V1_8822B) &                           \
+	 BIT_MASK_LLT_FREE_PAGE_V1_8822B)
+
+#define BIT_SHIFT_BLK_DESC_NUM_8822B 4
+#define BIT_MASK_BLK_DESC_NUM_8822B 0xf
+#define BIT_BLK_DESC_NUM_8822B(x)                                              \
+	(((x) & BIT_MASK_BLK_DESC_NUM_8822B) << BIT_SHIFT_BLK_DESC_NUM_8822B)
+#define BIT_GET_BLK_DESC_NUM_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_BLK_DESC_NUM_8822B) & BIT_MASK_BLK_DESC_NUM_8822B)
+
+#define BIT_R_BCN_HEAD_SEL_8822B BIT(3)
+#define BIT_R_EN_BCN_SW_HEAD_SEL_8822B BIT(2)
+#define BIT_LLT_DBG_SEL_8822B BIT(1)
+#define BIT_AUTO_INIT_LLT_V1_8822B BIT(0)
+
+/* 2 REG_TXDMA_OFFSET_CHK_8822B */
+#define BIT_EM_CHKSUM_FIN_8822B BIT(31)
+#define BIT_EMN_PCIE_DMA_MOD_8822B BIT(30)
+#define BIT_EN_TXQUE_CLR_8822B BIT(29)
+#define BIT_EN_PCIE_FIFO_MODE_8822B BIT(28)
+
+#define BIT_SHIFT_PG_UNDER_TH_V1_8822B 16
+#define BIT_MASK_PG_UNDER_TH_V1_8822B 0xfff
+#define BIT_PG_UNDER_TH_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_PG_UNDER_TH_V1_8822B)                                 \
+	 << BIT_SHIFT_PG_UNDER_TH_V1_8822B)
+#define BIT_GET_PG_UNDER_TH_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_PG_UNDER_TH_V1_8822B) &                             \
+	 BIT_MASK_PG_UNDER_TH_V1_8822B)
+
+#define BIT_RESTORE_H2C_ADDRESS_8822B BIT(15)
+#define BIT_SDIO_TXDESC_CHKSUM_EN_8822B BIT(13)
+#define BIT_RST_RDPTR_8822B BIT(12)
+#define BIT_RST_WRPTR_8822B BIT(11)
+#define BIT_CHK_PG_TH_EN_8822B BIT(10)
+#define BIT_DROP_DATA_EN_8822B BIT(9)
+#define BIT_CHECK_OFFSET_EN_8822B BIT(8)
+
+#define BIT_SHIFT_CHECK_OFFSET_8822B 0
+#define BIT_MASK_CHECK_OFFSET_8822B 0xff
+#define BIT_CHECK_OFFSET_8822B(x)                                              \
+	(((x) & BIT_MASK_CHECK_OFFSET_8822B) << BIT_SHIFT_CHECK_OFFSET_8822B)
+#define BIT_GET_CHECK_OFFSET_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_CHECK_OFFSET_8822B) & BIT_MASK_CHECK_OFFSET_8822B)
+
+/* 2 REG_TXDMA_STATUS_8822B */
+#define BIT_HI_OQT_UDN_8822B BIT(17)
+#define BIT_HI_OQT_OVF_8822B BIT(16)
+#define BIT_PAYLOAD_CHKSUM_ERR_8822B BIT(15)
+#define BIT_PAYLOAD_UDN_8822B BIT(14)
+#define BIT_PAYLOAD_OVF_8822B BIT(13)
+#define BIT_DSC_CHKSUM_FAIL_8822B BIT(12)
+#define BIT_UNKNOWN_QSEL_8822B BIT(11)
+#define BIT_EP_QSEL_DIFF_8822B BIT(10)
+#define BIT_TX_OFFS_UNMATCH_8822B BIT(9)
+#define BIT_TXOQT_UDN_8822B BIT(8)
+#define BIT_TXOQT_OVF_8822B BIT(7)
+#define BIT_TXDMA_SFF_UDN_8822B BIT(6)
+#define BIT_TXDMA_SFF_OVF_8822B BIT(5)
+#define BIT_LLT_NULL_PG_8822B BIT(4)
+#define BIT_PAGE_UDN_8822B BIT(3)
+#define BIT_PAGE_OVF_8822B BIT(2)
+#define BIT_TXFF_PG_UDN_8822B BIT(1)
+#define BIT_TXFF_PG_OVF_8822B BIT(0)
+
+/* 2 REG_TX_DMA_DBG_8822B */
+
+/* 2 REG_TQPNT1_8822B */
+
+#define BIT_SHIFT_HPQ_HIGH_TH_V1_8822B 16
+#define BIT_MASK_HPQ_HIGH_TH_V1_8822B 0xfff
+#define BIT_HPQ_HIGH_TH_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_HPQ_HIGH_TH_V1_8822B)                                 \
+	 << BIT_SHIFT_HPQ_HIGH_TH_V1_8822B)
+#define BIT_GET_HPQ_HIGH_TH_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HPQ_HIGH_TH_V1_8822B) &                             \
+	 BIT_MASK_HPQ_HIGH_TH_V1_8822B)
+
+#define BIT_SHIFT_HPQ_LOW_TH_V1_8822B 0
+#define BIT_MASK_HPQ_LOW_TH_V1_8822B 0xfff
+#define BIT_HPQ_LOW_TH_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_HPQ_LOW_TH_V1_8822B) << BIT_SHIFT_HPQ_LOW_TH_V1_8822B)
+#define BIT_GET_HPQ_LOW_TH_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_HPQ_LOW_TH_V1_8822B) & BIT_MASK_HPQ_LOW_TH_V1_8822B)
+
+/* 2 REG_TQPNT2_8822B */
+
+#define BIT_SHIFT_NPQ_HIGH_TH_V1_8822B 16
+#define BIT_MASK_NPQ_HIGH_TH_V1_8822B 0xfff
+#define BIT_NPQ_HIGH_TH_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_NPQ_HIGH_TH_V1_8822B)                                 \
+	 << BIT_SHIFT_NPQ_HIGH_TH_V1_8822B)
+#define BIT_GET_NPQ_HIGH_TH_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_NPQ_HIGH_TH_V1_8822B) &                             \
+	 BIT_MASK_NPQ_HIGH_TH_V1_8822B)
+
+#define BIT_SHIFT_NPQ_LOW_TH_V1_8822B 0
+#define BIT_MASK_NPQ_LOW_TH_V1_8822B 0xfff
+#define BIT_NPQ_LOW_TH_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_NPQ_LOW_TH_V1_8822B) << BIT_SHIFT_NPQ_LOW_TH_V1_8822B)
+#define BIT_GET_NPQ_LOW_TH_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_NPQ_LOW_TH_V1_8822B) & BIT_MASK_NPQ_LOW_TH_V1_8822B)
+
+/* 2 REG_TQPNT3_8822B */
+
+#define BIT_SHIFT_LPQ_HIGH_TH_V1_8822B 16
+#define BIT_MASK_LPQ_HIGH_TH_V1_8822B 0xfff
+#define BIT_LPQ_HIGH_TH_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_LPQ_HIGH_TH_V1_8822B)                                 \
+	 << BIT_SHIFT_LPQ_HIGH_TH_V1_8822B)
+#define BIT_GET_LPQ_HIGH_TH_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_LPQ_HIGH_TH_V1_8822B) &                             \
+	 BIT_MASK_LPQ_HIGH_TH_V1_8822B)
+
+#define BIT_SHIFT_LPQ_LOW_TH_V1_8822B 0
+#define BIT_MASK_LPQ_LOW_TH_V1_8822B 0xfff
+#define BIT_LPQ_LOW_TH_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_LPQ_LOW_TH_V1_8822B) << BIT_SHIFT_LPQ_LOW_TH_V1_8822B)
+#define BIT_GET_LPQ_LOW_TH_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_LPQ_LOW_TH_V1_8822B) & BIT_MASK_LPQ_LOW_TH_V1_8822B)
+
+/* 2 REG_TQPNT4_8822B */
+
+#define BIT_SHIFT_EXQ_HIGH_TH_V1_8822B 16
+#define BIT_MASK_EXQ_HIGH_TH_V1_8822B 0xfff
+#define BIT_EXQ_HIGH_TH_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_EXQ_HIGH_TH_V1_8822B)                                 \
+	 << BIT_SHIFT_EXQ_HIGH_TH_V1_8822B)
+#define BIT_GET_EXQ_HIGH_TH_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_EXQ_HIGH_TH_V1_8822B) &                             \
+	 BIT_MASK_EXQ_HIGH_TH_V1_8822B)
+
+#define BIT_SHIFT_EXQ_LOW_TH_V1_8822B 0
+#define BIT_MASK_EXQ_LOW_TH_V1_8822B 0xfff
+#define BIT_EXQ_LOW_TH_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_EXQ_LOW_TH_V1_8822B) << BIT_SHIFT_EXQ_LOW_TH_V1_8822B)
+#define BIT_GET_EXQ_LOW_TH_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_EXQ_LOW_TH_V1_8822B) & BIT_MASK_EXQ_LOW_TH_V1_8822B)
+
+/* 2 REG_RQPN_CTRL_1_8822B */
+
+#define BIT_SHIFT_TXPKTNUM_H_8822B 16
+#define BIT_MASK_TXPKTNUM_H_8822B 0xffff
+#define BIT_TXPKTNUM_H_8822B(x)                                                \
+	(((x) & BIT_MASK_TXPKTNUM_H_8822B) << BIT_SHIFT_TXPKTNUM_H_8822B)
+#define BIT_GET_TXPKTNUM_H_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_TXPKTNUM_H_8822B) & BIT_MASK_TXPKTNUM_H_8822B)
+
+#define BIT_SHIFT_TXPKTNUM_V2_8822B 0
+#define BIT_MASK_TXPKTNUM_V2_8822B 0xffff
+#define BIT_TXPKTNUM_V2_8822B(x)                                               \
+	(((x) & BIT_MASK_TXPKTNUM_V2_8822B) << BIT_SHIFT_TXPKTNUM_V2_8822B)
+#define BIT_GET_TXPKTNUM_V2_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_TXPKTNUM_V2_8822B) & BIT_MASK_TXPKTNUM_V2_8822B)
+
+/* 2 REG_RQPN_CTRL_2_8822B */
+#define BIT_LD_RQPN_8822B BIT(31)
+#define BIT_EXQ_PUBLIC_DIS_V1_8822B BIT(19)
+#define BIT_NPQ_PUBLIC_DIS_V1_8822B BIT(18)
+#define BIT_LPQ_PUBLIC_DIS_V1_8822B BIT(17)
+#define BIT_HPQ_PUBLIC_DIS_V1_8822B BIT(16)
+
+/* 2 REG_FIFOPAGE_INFO_1_8822B */
+
+#define BIT_SHIFT_HPQ_AVAL_PG_V1_8822B 16
+#define BIT_MASK_HPQ_AVAL_PG_V1_8822B 0xfff
+#define BIT_HPQ_AVAL_PG_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_HPQ_AVAL_PG_V1_8822B)                                 \
+	 << BIT_SHIFT_HPQ_AVAL_PG_V1_8822B)
+#define BIT_GET_HPQ_AVAL_PG_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HPQ_AVAL_PG_V1_8822B) &                             \
+	 BIT_MASK_HPQ_AVAL_PG_V1_8822B)
+
+#define BIT_SHIFT_HPQ_V1_8822B 0
+#define BIT_MASK_HPQ_V1_8822B 0xfff
+#define BIT_HPQ_V1_8822B(x)                                                    \
+	(((x) & BIT_MASK_HPQ_V1_8822B) << BIT_SHIFT_HPQ_V1_8822B)
+#define BIT_GET_HPQ_V1_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_HPQ_V1_8822B) & BIT_MASK_HPQ_V1_8822B)
+
+/* 2 REG_FIFOPAGE_INFO_2_8822B */
+
+#define BIT_SHIFT_LPQ_AVAL_PG_V1_8822B 16
+#define BIT_MASK_LPQ_AVAL_PG_V1_8822B 0xfff
+#define BIT_LPQ_AVAL_PG_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_LPQ_AVAL_PG_V1_8822B)                                 \
+	 << BIT_SHIFT_LPQ_AVAL_PG_V1_8822B)
+#define BIT_GET_LPQ_AVAL_PG_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_LPQ_AVAL_PG_V1_8822B) &                             \
+	 BIT_MASK_LPQ_AVAL_PG_V1_8822B)
+
+#define BIT_SHIFT_LPQ_V1_8822B 0
+#define BIT_MASK_LPQ_V1_8822B 0xfff
+#define BIT_LPQ_V1_8822B(x)                                                    \
+	(((x) & BIT_MASK_LPQ_V1_8822B) << BIT_SHIFT_LPQ_V1_8822B)
+#define BIT_GET_LPQ_V1_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_LPQ_V1_8822B) & BIT_MASK_LPQ_V1_8822B)
+
+/* 2 REG_FIFOPAGE_INFO_3_8822B */
+
+#define BIT_SHIFT_NPQ_AVAL_PG_V1_8822B 16
+#define BIT_MASK_NPQ_AVAL_PG_V1_8822B 0xfff
+#define BIT_NPQ_AVAL_PG_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_NPQ_AVAL_PG_V1_8822B)                                 \
+	 << BIT_SHIFT_NPQ_AVAL_PG_V1_8822B)
+#define BIT_GET_NPQ_AVAL_PG_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_NPQ_AVAL_PG_V1_8822B) &                             \
+	 BIT_MASK_NPQ_AVAL_PG_V1_8822B)
+
+#define BIT_SHIFT_NPQ_V1_8822B 0
+#define BIT_MASK_NPQ_V1_8822B 0xfff
+#define BIT_NPQ_V1_8822B(x)                                                    \
+	(((x) & BIT_MASK_NPQ_V1_8822B) << BIT_SHIFT_NPQ_V1_8822B)
+#define BIT_GET_NPQ_V1_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_NPQ_V1_8822B) & BIT_MASK_NPQ_V1_8822B)
+
+/* 2 REG_FIFOPAGE_INFO_4_8822B */
+
+#define BIT_SHIFT_EXQ_AVAL_PG_V1_8822B 16
+#define BIT_MASK_EXQ_AVAL_PG_V1_8822B 0xfff
+#define BIT_EXQ_AVAL_PG_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_EXQ_AVAL_PG_V1_8822B)                                 \
+	 << BIT_SHIFT_EXQ_AVAL_PG_V1_8822B)
+#define BIT_GET_EXQ_AVAL_PG_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_EXQ_AVAL_PG_V1_8822B) &                             \
+	 BIT_MASK_EXQ_AVAL_PG_V1_8822B)
+
+#define BIT_SHIFT_EXQ_V1_8822B 0
+#define BIT_MASK_EXQ_V1_8822B 0xfff
+#define BIT_EXQ_V1_8822B(x)                                                    \
+	(((x) & BIT_MASK_EXQ_V1_8822B) << BIT_SHIFT_EXQ_V1_8822B)
+#define BIT_GET_EXQ_V1_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_EXQ_V1_8822B) & BIT_MASK_EXQ_V1_8822B)
+
+/* 2 REG_FIFOPAGE_INFO_5_8822B */
+
+#define BIT_SHIFT_PUBQ_AVAL_PG_V1_8822B 16
+#define BIT_MASK_PUBQ_AVAL_PG_V1_8822B 0xfff
+#define BIT_PUBQ_AVAL_PG_V1_8822B(x)                                           \
+	(((x) & BIT_MASK_PUBQ_AVAL_PG_V1_8822B)                                \
+	 << BIT_SHIFT_PUBQ_AVAL_PG_V1_8822B)
+#define BIT_GET_PUBQ_AVAL_PG_V1_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_PUBQ_AVAL_PG_V1_8822B) &                            \
+	 BIT_MASK_PUBQ_AVAL_PG_V1_8822B)
+
+#define BIT_SHIFT_PUBQ_V1_8822B 0
+#define BIT_MASK_PUBQ_V1_8822B 0xfff
+#define BIT_PUBQ_V1_8822B(x)                                                   \
+	(((x) & BIT_MASK_PUBQ_V1_8822B) << BIT_SHIFT_PUBQ_V1_8822B)
+#define BIT_GET_PUBQ_V1_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_PUBQ_V1_8822B) & BIT_MASK_PUBQ_V1_8822B)
+
+/* 2 REG_H2C_HEAD_8822B */
+
+#define BIT_SHIFT_H2C_HEAD_8822B 0
+#define BIT_MASK_H2C_HEAD_8822B 0x3ffff
+#define BIT_H2C_HEAD_8822B(x)                                                  \
+	(((x) & BIT_MASK_H2C_HEAD_8822B) << BIT_SHIFT_H2C_HEAD_8822B)
+#define BIT_GET_H2C_HEAD_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_H2C_HEAD_8822B) & BIT_MASK_H2C_HEAD_8822B)
+
+/* 2 REG_H2C_TAIL_8822B */
+
+#define BIT_SHIFT_H2C_TAIL_8822B 0
+#define BIT_MASK_H2C_TAIL_8822B 0x3ffff
+#define BIT_H2C_TAIL_8822B(x)                                                  \
+	(((x) & BIT_MASK_H2C_TAIL_8822B) << BIT_SHIFT_H2C_TAIL_8822B)
+#define BIT_GET_H2C_TAIL_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_H2C_TAIL_8822B) & BIT_MASK_H2C_TAIL_8822B)
+
+/* 2 REG_H2C_READ_ADDR_8822B */
+
+#define BIT_SHIFT_H2C_READ_ADDR_8822B 0
+#define BIT_MASK_H2C_READ_ADDR_8822B 0x3ffff
+#define BIT_H2C_READ_ADDR_8822B(x)                                             \
+	(((x) & BIT_MASK_H2C_READ_ADDR_8822B) << BIT_SHIFT_H2C_READ_ADDR_8822B)
+#define BIT_GET_H2C_READ_ADDR_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_H2C_READ_ADDR_8822B) & BIT_MASK_H2C_READ_ADDR_8822B)
+
+/* 2 REG_H2C_WR_ADDR_8822B */
+
+#define BIT_SHIFT_H2C_WR_ADDR_8822B 0
+#define BIT_MASK_H2C_WR_ADDR_8822B 0x3ffff
+#define BIT_H2C_WR_ADDR_8822B(x)                                               \
+	(((x) & BIT_MASK_H2C_WR_ADDR_8822B) << BIT_SHIFT_H2C_WR_ADDR_8822B)
+#define BIT_GET_H2C_WR_ADDR_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_H2C_WR_ADDR_8822B) & BIT_MASK_H2C_WR_ADDR_8822B)
+
+/* 2 REG_H2C_INFO_8822B */
+#define BIT_H2C_SPACE_VLD_8822B BIT(3)
+#define BIT_H2C_WR_ADDR_RST_8822B BIT(2)
+
+#define BIT_SHIFT_H2C_LEN_SEL_8822B 0
+#define BIT_MASK_H2C_LEN_SEL_8822B 0x3
+#define BIT_H2C_LEN_SEL_8822B(x)                                               \
+	(((x) & BIT_MASK_H2C_LEN_SEL_8822B) << BIT_SHIFT_H2C_LEN_SEL_8822B)
+#define BIT_GET_H2C_LEN_SEL_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_H2C_LEN_SEL_8822B) & BIT_MASK_H2C_LEN_SEL_8822B)
+
+/* 2 REG_RXDMA_AGG_PG_TH_8822B */
+
+#define BIT_SHIFT_RXDMA_AGG_OLD_MOD_8822B 24
+#define BIT_MASK_RXDMA_AGG_OLD_MOD_8822B 0xff
+#define BIT_RXDMA_AGG_OLD_MOD_8822B(x)                                         \
+	(((x) & BIT_MASK_RXDMA_AGG_OLD_MOD_8822B)                              \
+	 << BIT_SHIFT_RXDMA_AGG_OLD_MOD_8822B)
+#define BIT_GET_RXDMA_AGG_OLD_MOD_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_RXDMA_AGG_OLD_MOD_8822B) &                          \
+	 BIT_MASK_RXDMA_AGG_OLD_MOD_8822B)
+
+#define BIT_SHIFT_PKT_NUM_WOL_8822B 16
+#define BIT_MASK_PKT_NUM_WOL_8822B 0xff
+#define BIT_PKT_NUM_WOL_8822B(x)                                               \
+	(((x) & BIT_MASK_PKT_NUM_WOL_8822B) << BIT_SHIFT_PKT_NUM_WOL_8822B)
+#define BIT_GET_PKT_NUM_WOL_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_PKT_NUM_WOL_8822B) & BIT_MASK_PKT_NUM_WOL_8822B)
+
+#define BIT_SHIFT_DMA_AGG_TO_8822B 8
+#define BIT_MASK_DMA_AGG_TO_8822B 0xf
+#define BIT_DMA_AGG_TO_8822B(x)                                                \
+	(((x) & BIT_MASK_DMA_AGG_TO_8822B) << BIT_SHIFT_DMA_AGG_TO_8822B)
+#define BIT_GET_DMA_AGG_TO_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_DMA_AGG_TO_8822B) & BIT_MASK_DMA_AGG_TO_8822B)
+
+#define BIT_SHIFT_RXDMA_AGG_PG_TH_V1_8822B 0
+#define BIT_MASK_RXDMA_AGG_PG_TH_V1_8822B 0xf
+#define BIT_RXDMA_AGG_PG_TH_V1_8822B(x)                                        \
+	(((x) & BIT_MASK_RXDMA_AGG_PG_TH_V1_8822B)                             \
+	 << BIT_SHIFT_RXDMA_AGG_PG_TH_V1_8822B)
+#define BIT_GET_RXDMA_AGG_PG_TH_V1_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_RXDMA_AGG_PG_TH_V1_8822B) &                         \
+	 BIT_MASK_RXDMA_AGG_PG_TH_V1_8822B)
+
+/* 2 REG_RXPKT_NUM_8822B */
+
+#define BIT_SHIFT_RXPKT_NUM_8822B 24
+#define BIT_MASK_RXPKT_NUM_8822B 0xff
+#define BIT_RXPKT_NUM_8822B(x)                                                 \
+	(((x) & BIT_MASK_RXPKT_NUM_8822B) << BIT_SHIFT_RXPKT_NUM_8822B)
+#define BIT_GET_RXPKT_NUM_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_RXPKT_NUM_8822B) & BIT_MASK_RXPKT_NUM_8822B)
+
+#define BIT_SHIFT_FW_UPD_RDPTR19_TO_16_8822B 20
+#define BIT_MASK_FW_UPD_RDPTR19_TO_16_8822B 0xf
+#define BIT_FW_UPD_RDPTR19_TO_16_8822B(x)                                      \
+	(((x) & BIT_MASK_FW_UPD_RDPTR19_TO_16_8822B)                           \
+	 << BIT_SHIFT_FW_UPD_RDPTR19_TO_16_8822B)
+#define BIT_GET_FW_UPD_RDPTR19_TO_16_8822B(x)                                  \
+	(((x) >> BIT_SHIFT_FW_UPD_RDPTR19_TO_16_8822B) &                       \
+	 BIT_MASK_FW_UPD_RDPTR19_TO_16_8822B)
+
+#define BIT_RXDMA_REQ_8822B BIT(19)
+#define BIT_RW_RELEASE_EN_8822B BIT(18)
+#define BIT_RXDMA_IDLE_8822B BIT(17)
+#define BIT_RXPKT_RELEASE_POLL_8822B BIT(16)
+
+#define BIT_SHIFT_FW_UPD_RDPTR_8822B 0
+#define BIT_MASK_FW_UPD_RDPTR_8822B 0xffff
+#define BIT_FW_UPD_RDPTR_8822B(x)                                              \
+	(((x) & BIT_MASK_FW_UPD_RDPTR_8822B) << BIT_SHIFT_FW_UPD_RDPTR_8822B)
+#define BIT_GET_FW_UPD_RDPTR_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_FW_UPD_RDPTR_8822B) & BIT_MASK_FW_UPD_RDPTR_8822B)
+
+/* 2 REG_RXDMA_STATUS_8822B */
+#define BIT_C2H_PKT_OVF_8822B BIT(7)
+#define BIT_AGG_CONFGI_ISSUE_8822B BIT(6)
+#define BIT_FW_POLL_ISSUE_8822B BIT(5)
+#define BIT_RX_DATA_UDN_8822B BIT(4)
+#define BIT_RX_SFF_UDN_8822B BIT(3)
+#define BIT_RX_SFF_OVF_8822B BIT(2)
+#define BIT_RXPKT_OVF_8822B BIT(0)
+
+/* 2 REG_RXDMA_DPR_8822B */
+
+#define BIT_SHIFT_RDE_DEBUG_8822B 0
+#define BIT_MASK_RDE_DEBUG_8822B 0xffffffffL
+#define BIT_RDE_DEBUG_8822B(x)                                                 \
+	(((x) & BIT_MASK_RDE_DEBUG_8822B) << BIT_SHIFT_RDE_DEBUG_8822B)
+#define BIT_GET_RDE_DEBUG_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_RDE_DEBUG_8822B) & BIT_MASK_RDE_DEBUG_8822B)
+
+/* 2 REG_RXDMA_MODE_8822B */
+
+#define BIT_SHIFT_PKTNUM_TH_V2_8822B 24
+#define BIT_MASK_PKTNUM_TH_V2_8822B 0x1f
+#define BIT_PKTNUM_TH_V2_8822B(x)                                              \
+	(((x) & BIT_MASK_PKTNUM_TH_V2_8822B) << BIT_SHIFT_PKTNUM_TH_V2_8822B)
+#define BIT_GET_PKTNUM_TH_V2_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_PKTNUM_TH_V2_8822B) & BIT_MASK_PKTNUM_TH_V2_8822B)
+
+#define BIT_TXBA_BREAK_USBAGG_8822B BIT(23)
+
+#define BIT_SHIFT_PKTLEN_PARA_8822B 16
+#define BIT_MASK_PKTLEN_PARA_8822B 0x7
+#define BIT_PKTLEN_PARA_8822B(x)                                               \
+	(((x) & BIT_MASK_PKTLEN_PARA_8822B) << BIT_SHIFT_PKTLEN_PARA_8822B)
+#define BIT_GET_PKTLEN_PARA_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_PKTLEN_PARA_8822B) & BIT_MASK_PKTLEN_PARA_8822B)
+
+/* 2 REG_NOT_VALID_8822B */
+
+/* 2 REG_NOT_VALID_8822B */
+
+/* 2 REG_NOT_VALID_8822B */
+
+#define BIT_SHIFT_BURST_SIZE_8822B 4
+#define BIT_MASK_BURST_SIZE_8822B 0x3
+#define BIT_BURST_SIZE_8822B(x)                                                \
+	(((x) & BIT_MASK_BURST_SIZE_8822B) << BIT_SHIFT_BURST_SIZE_8822B)
+#define BIT_GET_BURST_SIZE_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_BURST_SIZE_8822B) & BIT_MASK_BURST_SIZE_8822B)
+
+#define BIT_SHIFT_BURST_CNT_8822B 2
+#define BIT_MASK_BURST_CNT_8822B 0x3
+#define BIT_BURST_CNT_8822B(x)                                                 \
+	(((x) & BIT_MASK_BURST_CNT_8822B) << BIT_SHIFT_BURST_CNT_8822B)
+#define BIT_GET_BURST_CNT_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_BURST_CNT_8822B) & BIT_MASK_BURST_CNT_8822B)
+
+#define BIT_DMA_MODE_8822B BIT(1)
+
+/* 2 REG_C2H_PKT_8822B */
+
+#define BIT_SHIFT_R_C2H_STR_ADDR_16_TO_19_8822B 24
+#define BIT_MASK_R_C2H_STR_ADDR_16_TO_19_8822B 0xf
+#define BIT_R_C2H_STR_ADDR_16_TO_19_8822B(x)                                   \
+	(((x) & BIT_MASK_R_C2H_STR_ADDR_16_TO_19_8822B)                        \
+	 << BIT_SHIFT_R_C2H_STR_ADDR_16_TO_19_8822B)
+#define BIT_GET_R_C2H_STR_ADDR_16_TO_19_8822B(x)                               \
+	(((x) >> BIT_SHIFT_R_C2H_STR_ADDR_16_TO_19_8822B) &                    \
+	 BIT_MASK_R_C2H_STR_ADDR_16_TO_19_8822B)
+
+#define BIT_R_C2H_PKT_REQ_8822B BIT(16)
+
+#define BIT_SHIFT_R_C2H_STR_ADDR_8822B 0
+#define BIT_MASK_R_C2H_STR_ADDR_8822B 0xffff
+#define BIT_R_C2H_STR_ADDR_8822B(x)                                            \
+	(((x) & BIT_MASK_R_C2H_STR_ADDR_8822B)                                 \
+	 << BIT_SHIFT_R_C2H_STR_ADDR_8822B)
+#define BIT_GET_R_C2H_STR_ADDR_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_R_C2H_STR_ADDR_8822B) &                             \
+	 BIT_MASK_R_C2H_STR_ADDR_8822B)
+
+/* 2 REG_FWFF_C2H_8822B */
+
+#define BIT_SHIFT_C2H_DMA_ADDR_8822B 0
+#define BIT_MASK_C2H_DMA_ADDR_8822B 0x3ffff
+#define BIT_C2H_DMA_ADDR_8822B(x)                                              \
+	(((x) & BIT_MASK_C2H_DMA_ADDR_8822B) << BIT_SHIFT_C2H_DMA_ADDR_8822B)
+#define BIT_GET_C2H_DMA_ADDR_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_C2H_DMA_ADDR_8822B) & BIT_MASK_C2H_DMA_ADDR_8822B)
+
+/* 2 REG_FWFF_CTRL_8822B */
+#define BIT_FWFF_DMAPKT_REQ_8822B BIT(31)
+
+#define BIT_SHIFT_FWFF_DMA_PKT_NUM_8822B 16
+#define BIT_MASK_FWFF_DMA_PKT_NUM_8822B 0xff
+#define BIT_FWFF_DMA_PKT_NUM_8822B(x)                                          \
+	(((x) & BIT_MASK_FWFF_DMA_PKT_NUM_8822B)                               \
+	 << BIT_SHIFT_FWFF_DMA_PKT_NUM_8822B)
+#define BIT_GET_FWFF_DMA_PKT_NUM_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_FWFF_DMA_PKT_NUM_8822B) &                           \
+	 BIT_MASK_FWFF_DMA_PKT_NUM_8822B)
+
+#define BIT_SHIFT_FWFF_STR_ADDR_8822B 0
+#define BIT_MASK_FWFF_STR_ADDR_8822B 0xffff
+#define BIT_FWFF_STR_ADDR_8822B(x)                                             \
+	(((x) & BIT_MASK_FWFF_STR_ADDR_8822B) << BIT_SHIFT_FWFF_STR_ADDR_8822B)
+#define BIT_GET_FWFF_STR_ADDR_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_FWFF_STR_ADDR_8822B) & BIT_MASK_FWFF_STR_ADDR_8822B)
+
+/* 2 REG_FWFF_PKT_INFO_8822B */
+
+#define BIT_SHIFT_FWFF_PKT_QUEUED_8822B 16
+#define BIT_MASK_FWFF_PKT_QUEUED_8822B 0xff
+#define BIT_FWFF_PKT_QUEUED_8822B(x)                                           \
+	(((x) & BIT_MASK_FWFF_PKT_QUEUED_8822B)                                \
+	 << BIT_SHIFT_FWFF_PKT_QUEUED_8822B)
+#define BIT_GET_FWFF_PKT_QUEUED_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_FWFF_PKT_QUEUED_8822B) &                            \
+	 BIT_MASK_FWFF_PKT_QUEUED_8822B)
+
+#define BIT_SHIFT_FWFF_PKT_STR_ADDR_8822B 0
+#define BIT_MASK_FWFF_PKT_STR_ADDR_8822B 0xffff
+#define BIT_FWFF_PKT_STR_ADDR_8822B(x)                                         \
+	(((x) & BIT_MASK_FWFF_PKT_STR_ADDR_8822B)                              \
+	 << BIT_SHIFT_FWFF_PKT_STR_ADDR_8822B)
+#define BIT_GET_FWFF_PKT_STR_ADDR_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_FWFF_PKT_STR_ADDR_8822B) &                          \
+	 BIT_MASK_FWFF_PKT_STR_ADDR_8822B)
+
+/* 2 REG_NOT_VALID_8822B */
+
+/* 2 REG_DDMA_CH0SA_8822B */
+
+#define BIT_SHIFT_DDMACH0_SA_8822B 0
+#define BIT_MASK_DDMACH0_SA_8822B 0xffffffffL
+#define BIT_DDMACH0_SA_8822B(x)                                                \
+	(((x) & BIT_MASK_DDMACH0_SA_8822B) << BIT_SHIFT_DDMACH0_SA_8822B)
+#define BIT_GET_DDMACH0_SA_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_DDMACH0_SA_8822B) & BIT_MASK_DDMACH0_SA_8822B)
+
+/* 2 REG_DDMA_CH0DA_8822B */
+
+#define BIT_SHIFT_DDMACH0_DA_8822B 0
+#define BIT_MASK_DDMACH0_DA_8822B 0xffffffffL
+#define BIT_DDMACH0_DA_8822B(x)                                                \
+	(((x) & BIT_MASK_DDMACH0_DA_8822B) << BIT_SHIFT_DDMACH0_DA_8822B)
+#define BIT_GET_DDMACH0_DA_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_DDMACH0_DA_8822B) & BIT_MASK_DDMACH0_DA_8822B)
+
+/* 2 REG_DDMA_CH0CTRL_8822B */
+#define BIT_DDMACH0_OWN_8822B BIT(31)
+#define BIT_DDMACH0_CHKSUM_EN_8822B BIT(29)
+#define BIT_DDMACH0_DA_W_DISABLE_8822B BIT(28)
+#define BIT_DDMACH0_CHKSUM_STS_8822B BIT(27)
+#define BIT_DDMACH0_DDMA_MODE_8822B BIT(26)
+#define BIT_DDMACH0_RESET_CHKSUM_STS_8822B BIT(25)
+#define BIT_DDMACH0_CHKSUM_CONT_8822B BIT(24)
+
+#define BIT_SHIFT_DDMACH0_DLEN_8822B 0
+#define BIT_MASK_DDMACH0_DLEN_8822B 0x3ffff
+#define BIT_DDMACH0_DLEN_8822B(x)                                              \
+	(((x) & BIT_MASK_DDMACH0_DLEN_8822B) << BIT_SHIFT_DDMACH0_DLEN_8822B)
+#define BIT_GET_DDMACH0_DLEN_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_DDMACH0_DLEN_8822B) & BIT_MASK_DDMACH0_DLEN_8822B)
+
+/* 2 REG_DDMA_CH1SA_8822B */
+
+#define BIT_SHIFT_DDMACH1_SA_8822B 0
+#define BIT_MASK_DDMACH1_SA_8822B 0xffffffffL
+#define BIT_DDMACH1_SA_8822B(x)                                                \
+	(((x) & BIT_MASK_DDMACH1_SA_8822B) << BIT_SHIFT_DDMACH1_SA_8822B)
+#define BIT_GET_DDMACH1_SA_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_DDMACH1_SA_8822B) & BIT_MASK_DDMACH1_SA_8822B)
+
+/* 2 REG_DDMA_CH1DA_8822B */
+
+#define BIT_SHIFT_DDMACH1_DA_8822B 0
+#define BIT_MASK_DDMACH1_DA_8822B 0xffffffffL
+#define BIT_DDMACH1_DA_8822B(x)                                                \
+	(((x) & BIT_MASK_DDMACH1_DA_8822B) << BIT_SHIFT_DDMACH1_DA_8822B)
+#define BIT_GET_DDMACH1_DA_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_DDMACH1_DA_8822B) & BIT_MASK_DDMACH1_DA_8822B)
+
+/* 2 REG_DDMA_CH1CTRL_8822B */
+#define BIT_DDMACH1_OWN_8822B BIT(31)
+#define BIT_DDMACH1_CHKSUM_EN_8822B BIT(29)
+#define BIT_DDMACH1_DA_W_DISABLE_8822B BIT(28)
+#define BIT_DDMACH1_CHKSUM_STS_8822B BIT(27)
+#define BIT_DDMACH1_DDMA_MODE_8822B BIT(26)
+#define BIT_DDMACH1_RESET_CHKSUM_STS_8822B BIT(25)
+#define BIT_DDMACH1_CHKSUM_CONT_8822B BIT(24)
+
+#define BIT_SHIFT_DDMACH1_DLEN_8822B 0
+#define BIT_MASK_DDMACH1_DLEN_8822B 0x3ffff
+#define BIT_DDMACH1_DLEN_8822B(x)                                              \
+	(((x) & BIT_MASK_DDMACH1_DLEN_8822B) << BIT_SHIFT_DDMACH1_DLEN_8822B)
+#define BIT_GET_DDMACH1_DLEN_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_DDMACH1_DLEN_8822B) & BIT_MASK_DDMACH1_DLEN_8822B)
+
+/* 2 REG_DDMA_CH2SA_8822B */
+
+#define BIT_SHIFT_DDMACH2_SA_8822B 0
+#define BIT_MASK_DDMACH2_SA_8822B 0xffffffffL
+#define BIT_DDMACH2_SA_8822B(x)                                                \
+	(((x) & BIT_MASK_DDMACH2_SA_8822B) << BIT_SHIFT_DDMACH2_SA_8822B)
+#define BIT_GET_DDMACH2_SA_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_DDMACH2_SA_8822B) & BIT_MASK_DDMACH2_SA_8822B)
+
+/* 2 REG_DDMA_CH2DA_8822B */
+
+#define BIT_SHIFT_DDMACH2_DA_8822B 0
+#define BIT_MASK_DDMACH2_DA_8822B 0xffffffffL
+#define BIT_DDMACH2_DA_8822B(x)                                                \
+	(((x) & BIT_MASK_DDMACH2_DA_8822B) << BIT_SHIFT_DDMACH2_DA_8822B)
+#define BIT_GET_DDMACH2_DA_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_DDMACH2_DA_8822B) & BIT_MASK_DDMACH2_DA_8822B)
+
+/* 2 REG_DDMA_CH2CTRL_8822B */
+#define BIT_DDMACH2_OWN_8822B BIT(31)
+#define BIT_DDMACH2_CHKSUM_EN_8822B BIT(29)
+#define BIT_DDMACH2_DA_W_DISABLE_8822B BIT(28)
+#define BIT_DDMACH2_CHKSUM_STS_8822B BIT(27)
+#define BIT_DDMACH2_DDMA_MODE_8822B BIT(26)
+#define BIT_DDMACH2_RESET_CHKSUM_STS_8822B BIT(25)
+#define BIT_DDMACH2_CHKSUM_CONT_8822B BIT(24)
+
+#define BIT_SHIFT_DDMACH2_DLEN_8822B 0
+#define BIT_MASK_DDMACH2_DLEN_8822B 0x3ffff
+#define BIT_DDMACH2_DLEN_8822B(x)                                              \
+	(((x) & BIT_MASK_DDMACH2_DLEN_8822B) << BIT_SHIFT_DDMACH2_DLEN_8822B)
+#define BIT_GET_DDMACH2_DLEN_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_DDMACH2_DLEN_8822B) & BIT_MASK_DDMACH2_DLEN_8822B)
+
+/* 2 REG_DDMA_CH3SA_8822B */
+
+#define BIT_SHIFT_DDMACH3_SA_8822B 0
+#define BIT_MASK_DDMACH3_SA_8822B 0xffffffffL
+#define BIT_DDMACH3_SA_8822B(x)                                                \
+	(((x) & BIT_MASK_DDMACH3_SA_8822B) << BIT_SHIFT_DDMACH3_SA_8822B)
+#define BIT_GET_DDMACH3_SA_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_DDMACH3_SA_8822B) & BIT_MASK_DDMACH3_SA_8822B)
+
+/* 2 REG_DDMA_CH3DA_8822B */
+
+#define BIT_SHIFT_DDMACH3_DA_8822B 0
+#define BIT_MASK_DDMACH3_DA_8822B 0xffffffffL
+#define BIT_DDMACH3_DA_8822B(x)                                                \
+	(((x) & BIT_MASK_DDMACH3_DA_8822B) << BIT_SHIFT_DDMACH3_DA_8822B)
+#define BIT_GET_DDMACH3_DA_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_DDMACH3_DA_8822B) & BIT_MASK_DDMACH3_DA_8822B)
+
+/* 2 REG_DDMA_CH3CTRL_8822B */
+#define BIT_DDMACH3_OWN_8822B BIT(31)
+#define BIT_DDMACH3_CHKSUM_EN_8822B BIT(29)
+#define BIT_DDMACH3_DA_W_DISABLE_8822B BIT(28)
+#define BIT_DDMACH3_CHKSUM_STS_8822B BIT(27)
+#define BIT_DDMACH3_DDMA_MODE_8822B BIT(26)
+#define BIT_DDMACH3_RESET_CHKSUM_STS_8822B BIT(25)
+#define BIT_DDMACH3_CHKSUM_CONT_8822B BIT(24)
+
+#define BIT_SHIFT_DDMACH3_DLEN_8822B 0
+#define BIT_MASK_DDMACH3_DLEN_8822B 0x3ffff
+#define BIT_DDMACH3_DLEN_8822B(x)                                              \
+	(((x) & BIT_MASK_DDMACH3_DLEN_8822B) << BIT_SHIFT_DDMACH3_DLEN_8822B)
+#define BIT_GET_DDMACH3_DLEN_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_DDMACH3_DLEN_8822B) & BIT_MASK_DDMACH3_DLEN_8822B)
+
+/* 2 REG_DDMA_CH4SA_8822B */
+
+#define BIT_SHIFT_DDMACH4_SA_8822B 0
+#define BIT_MASK_DDMACH4_SA_8822B 0xffffffffL
+#define BIT_DDMACH4_SA_8822B(x)                                                \
+	(((x) & BIT_MASK_DDMACH4_SA_8822B) << BIT_SHIFT_DDMACH4_SA_8822B)
+#define BIT_GET_DDMACH4_SA_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_DDMACH4_SA_8822B) & BIT_MASK_DDMACH4_SA_8822B)
+
+/* 2 REG_DDMA_CH4DA_8822B */
+
+#define BIT_SHIFT_DDMACH4_DA_8822B 0
+#define BIT_MASK_DDMACH4_DA_8822B 0xffffffffL
+#define BIT_DDMACH4_DA_8822B(x)                                                \
+	(((x) & BIT_MASK_DDMACH4_DA_8822B) << BIT_SHIFT_DDMACH4_DA_8822B)
+#define BIT_GET_DDMACH4_DA_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_DDMACH4_DA_8822B) & BIT_MASK_DDMACH4_DA_8822B)
+
+/* 2 REG_DDMA_CH4CTRL_8822B */
+#define BIT_DDMACH4_OWN_8822B BIT(31)
+#define BIT_DDMACH4_CHKSUM_EN_8822B BIT(29)
+#define BIT_DDMACH4_DA_W_DISABLE_8822B BIT(28)
+#define BIT_DDMACH4_CHKSUM_STS_8822B BIT(27)
+#define BIT_DDMACH4_DDMA_MODE_8822B BIT(26)
+#define BIT_DDMACH4_RESET_CHKSUM_STS_8822B BIT(25)
+#define BIT_DDMACH4_CHKSUM_CONT_8822B BIT(24)
+
+#define BIT_SHIFT_DDMACH4_DLEN_8822B 0
+#define BIT_MASK_DDMACH4_DLEN_8822B 0x3ffff
+#define BIT_DDMACH4_DLEN_8822B(x)                                              \
+	(((x) & BIT_MASK_DDMACH4_DLEN_8822B) << BIT_SHIFT_DDMACH4_DLEN_8822B)
+#define BIT_GET_DDMACH4_DLEN_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_DDMACH4_DLEN_8822B) & BIT_MASK_DDMACH4_DLEN_8822B)
+
+/* 2 REG_DDMA_CH5SA_8822B */
+
+#define BIT_SHIFT_DDMACH5_SA_8822B 0
+#define BIT_MASK_DDMACH5_SA_8822B 0xffffffffL
+#define BIT_DDMACH5_SA_8822B(x)                                                \
+	(((x) & BIT_MASK_DDMACH5_SA_8822B) << BIT_SHIFT_DDMACH5_SA_8822B)
+#define BIT_GET_DDMACH5_SA_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_DDMACH5_SA_8822B) & BIT_MASK_DDMACH5_SA_8822B)
+
+/* 2 REG_DDMA_CH5DA_8822B */
+
+#define BIT_SHIFT_DDMACH5_DA_8822B 0
+#define BIT_MASK_DDMACH5_DA_8822B 0xffffffffL
+#define BIT_DDMACH5_DA_8822B(x)                                                \
+	(((x) & BIT_MASK_DDMACH5_DA_8822B) << BIT_SHIFT_DDMACH5_DA_8822B)
+#define BIT_GET_DDMACH5_DA_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_DDMACH5_DA_8822B) & BIT_MASK_DDMACH5_DA_8822B)
+
+/* 2 REG_REG_DDMA_CH5CTRL_8822B */
+#define BIT_DDMACH5_OWN_8822B BIT(31)
+#define BIT_DDMACH5_CHKSUM_EN_8822B BIT(29)
+#define BIT_DDMACH5_DA_W_DISABLE_8822B BIT(28)
+#define BIT_DDMACH5_CHKSUM_STS_8822B BIT(27)
+#define BIT_DDMACH5_DDMA_MODE_8822B BIT(26)
+#define BIT_DDMACH5_RESET_CHKSUM_STS_8822B BIT(25)
+#define BIT_DDMACH5_CHKSUM_CONT_8822B BIT(24)
+
+#define BIT_SHIFT_DDMACH5_DLEN_8822B 0
+#define BIT_MASK_DDMACH5_DLEN_8822B 0x3ffff
+#define BIT_DDMACH5_DLEN_8822B(x)                                              \
+	(((x) & BIT_MASK_DDMACH5_DLEN_8822B) << BIT_SHIFT_DDMACH5_DLEN_8822B)
+#define BIT_GET_DDMACH5_DLEN_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_DDMACH5_DLEN_8822B) & BIT_MASK_DDMACH5_DLEN_8822B)
+
+/* 2 REG_DDMA_INT_MSK_8822B */
+#define BIT_DDMACH5_MSK_8822B BIT(5)
+#define BIT_DDMACH4_MSK_8822B BIT(4)
+#define BIT_DDMACH3_MSK_8822B BIT(3)
+#define BIT_DDMACH2_MSK_8822B BIT(2)
+#define BIT_DDMACH1_MSK_8822B BIT(1)
+#define BIT_DDMACH0_MSK_8822B BIT(0)
+
+/* 2 REG_DDMA_CHSTATUS_8822B */
+#define BIT_DDMACH5_BUSY_8822B BIT(5)
+#define BIT_DDMACH4_BUSY_8822B BIT(4)
+#define BIT_DDMACH3_BUSY_8822B BIT(3)
+#define BIT_DDMACH2_BUSY_8822B BIT(2)
+#define BIT_DDMACH1_BUSY_8822B BIT(1)
+#define BIT_DDMACH0_BUSY_8822B BIT(0)
+
+/* 2 REG_DDMA_CHKSUM_8822B */
+
+#define BIT_SHIFT_IDDMA0_CHKSUM_8822B 0
+#define BIT_MASK_IDDMA0_CHKSUM_8822B 0xffff
+#define BIT_IDDMA0_CHKSUM_8822B(x)                                             \
+	(((x) & BIT_MASK_IDDMA0_CHKSUM_8822B) << BIT_SHIFT_IDDMA0_CHKSUM_8822B)
+#define BIT_GET_IDDMA0_CHKSUM_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_IDDMA0_CHKSUM_8822B) & BIT_MASK_IDDMA0_CHKSUM_8822B)
+
+/* 2 REG_DDMA_MONITOR_8822B */
+#define BIT_IDDMA0_PERMU_UNDERFLOW_8822B BIT(14)
+#define BIT_IDDMA0_FIFO_UNDERFLOW_8822B BIT(13)
+#define BIT_IDDMA0_FIFO_OVERFLOW_8822B BIT(12)
+#define BIT_CH5_ERR_8822B BIT(5)
+#define BIT_CH4_ERR_8822B BIT(4)
+#define BIT_CH3_ERR_8822B BIT(3)
+#define BIT_CH2_ERR_8822B BIT(2)
+#define BIT_CH1_ERR_8822B BIT(1)
+#define BIT_CH0_ERR_8822B BIT(0)
+
+/* 2 REG_NOT_VALID_8822B */
+
+/* 2 REG_PCIE_CTRL_8822B */
+#define BIT_PCIEIO_PERSTB_SEL_8822B BIT(31)
+
+#define BIT_SHIFT_PCIE_MAX_RXDMA_8822B 28
+#define BIT_MASK_PCIE_MAX_RXDMA_8822B 0x7
+#define BIT_PCIE_MAX_RXDMA_8822B(x)                                            \
+	(((x) & BIT_MASK_PCIE_MAX_RXDMA_8822B)                                 \
+	 << BIT_SHIFT_PCIE_MAX_RXDMA_8822B)
+#define BIT_GET_PCIE_MAX_RXDMA_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_PCIE_MAX_RXDMA_8822B) &                             \
+	 BIT_MASK_PCIE_MAX_RXDMA_8822B)
+
+#define BIT_MULRW_8822B BIT(27)
+
+#define BIT_SHIFT_PCIE_MAX_TXDMA_8822B 24
+#define BIT_MASK_PCIE_MAX_TXDMA_8822B 0x7
+#define BIT_PCIE_MAX_TXDMA_8822B(x)                                            \
+	(((x) & BIT_MASK_PCIE_MAX_TXDMA_8822B)                                 \
+	 << BIT_SHIFT_PCIE_MAX_TXDMA_8822B)
+#define BIT_GET_PCIE_MAX_TXDMA_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_PCIE_MAX_TXDMA_8822B) &                             \
+	 BIT_MASK_PCIE_MAX_TXDMA_8822B)
+
+#define BIT_EN_CPL_TIMEOUT_PS_8822B BIT(22)
+#define BIT_REG_TXDMA_FAIL_PS_8822B BIT(21)
+#define BIT_PCIE_RST_TRXDMA_INTF_8822B BIT(20)
+#define BIT_EN_HWENTR_L1_8822B BIT(19)
+#define BIT_EN_ADV_CLKGATE_8822B BIT(18)
+#define BIT_PCIE_EN_SWENT_L23_8822B BIT(17)
+#define BIT_PCIE_EN_HWEXT_L1_8822B BIT(16)
+#define BIT_RX_CLOSE_EN_8822B BIT(15)
+#define BIT_STOP_BCNQ_8822B BIT(14)
+#define BIT_STOP_MGQ_8822B BIT(13)
+#define BIT_STOP_VOQ_8822B BIT(12)
+#define BIT_STOP_VIQ_8822B BIT(11)
+#define BIT_STOP_BEQ_8822B BIT(10)
+#define BIT_STOP_BKQ_8822B BIT(9)
+#define BIT_STOP_RXQ_8822B BIT(8)
+#define BIT_STOP_HI7Q_8822B BIT(7)
+#define BIT_STOP_HI6Q_8822B BIT(6)
+#define BIT_STOP_HI5Q_8822B BIT(5)
+#define BIT_STOP_HI4Q_8822B BIT(4)
+#define BIT_STOP_HI3Q_8822B BIT(3)
+#define BIT_STOP_HI2Q_8822B BIT(2)
+#define BIT_STOP_HI1Q_8822B BIT(1)
+#define BIT_STOP_HI0Q_8822B BIT(0)
+
+/* 2 REG_INT_MIG_8822B */
+
+#define BIT_SHIFT_TXTTIMER_MATCH_NUM_8822B 28
+#define BIT_MASK_TXTTIMER_MATCH_NUM_8822B 0xf
+#define BIT_TXTTIMER_MATCH_NUM_8822B(x)                                        \
+	(((x) & BIT_MASK_TXTTIMER_MATCH_NUM_8822B)                             \
+	 << BIT_SHIFT_TXTTIMER_MATCH_NUM_8822B)
+#define BIT_GET_TXTTIMER_MATCH_NUM_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_TXTTIMER_MATCH_NUM_8822B) &                         \
+	 BIT_MASK_TXTTIMER_MATCH_NUM_8822B)
+
+#define BIT_SHIFT_TXPKT_NUM_MATCH_8822B 24
+#define BIT_MASK_TXPKT_NUM_MATCH_8822B 0xf
+#define BIT_TXPKT_NUM_MATCH_8822B(x)                                           \
+	(((x) & BIT_MASK_TXPKT_NUM_MATCH_8822B)                                \
+	 << BIT_SHIFT_TXPKT_NUM_MATCH_8822B)
+#define BIT_GET_TXPKT_NUM_MATCH_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_TXPKT_NUM_MATCH_8822B) &                            \
+	 BIT_MASK_TXPKT_NUM_MATCH_8822B)
+
+#define BIT_SHIFT_RXTTIMER_MATCH_NUM_8822B 20
+#define BIT_MASK_RXTTIMER_MATCH_NUM_8822B 0xf
+#define BIT_RXTTIMER_MATCH_NUM_8822B(x)                                        \
+	(((x) & BIT_MASK_RXTTIMER_MATCH_NUM_8822B)                             \
+	 << BIT_SHIFT_RXTTIMER_MATCH_NUM_8822B)
+#define BIT_GET_RXTTIMER_MATCH_NUM_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_RXTTIMER_MATCH_NUM_8822B) &                         \
+	 BIT_MASK_RXTTIMER_MATCH_NUM_8822B)
+
+#define BIT_SHIFT_RXPKT_NUM_MATCH_8822B 16
+#define BIT_MASK_RXPKT_NUM_MATCH_8822B 0xf
+#define BIT_RXPKT_NUM_MATCH_8822B(x)                                           \
+	(((x) & BIT_MASK_RXPKT_NUM_MATCH_8822B)                                \
+	 << BIT_SHIFT_RXPKT_NUM_MATCH_8822B)
+#define BIT_GET_RXPKT_NUM_MATCH_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_RXPKT_NUM_MATCH_8822B) &                            \
+	 BIT_MASK_RXPKT_NUM_MATCH_8822B)
+
+#define BIT_SHIFT_MIGRATE_TIMER_8822B 0
+#define BIT_MASK_MIGRATE_TIMER_8822B 0xffff
+#define BIT_MIGRATE_TIMER_8822B(x)                                             \
+	(((x) & BIT_MASK_MIGRATE_TIMER_8822B) << BIT_SHIFT_MIGRATE_TIMER_8822B)
+#define BIT_GET_MIGRATE_TIMER_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_MIGRATE_TIMER_8822B) & BIT_MASK_MIGRATE_TIMER_8822B)
+
+/* 2 REG_BCNQ_TXBD_DESA_8822B */
+
+#define BIT_SHIFT_BCNQ_TXBD_DESA_8822B 0
+#define BIT_MASK_BCNQ_TXBD_DESA_8822B 0xffffffffffffffffL
+#define BIT_BCNQ_TXBD_DESA_8822B(x)                                            \
+	(((x) & BIT_MASK_BCNQ_TXBD_DESA_8822B)                                 \
+	 << BIT_SHIFT_BCNQ_TXBD_DESA_8822B)
+#define BIT_GET_BCNQ_TXBD_DESA_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_BCNQ_TXBD_DESA_8822B) &                             \
+	 BIT_MASK_BCNQ_TXBD_DESA_8822B)
+
+/* 2 REG_MGQ_TXBD_DESA_8822B */
+
+#define BIT_SHIFT_MGQ_TXBD_DESA_8822B 0
+#define BIT_MASK_MGQ_TXBD_DESA_8822B 0xffffffffffffffffL
+#define BIT_MGQ_TXBD_DESA_8822B(x)                                             \
+	(((x) & BIT_MASK_MGQ_TXBD_DESA_8822B) << BIT_SHIFT_MGQ_TXBD_DESA_8822B)
+#define BIT_GET_MGQ_TXBD_DESA_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_MGQ_TXBD_DESA_8822B) & BIT_MASK_MGQ_TXBD_DESA_8822B)
+
+/* 2 REG_VOQ_TXBD_DESA_8822B */
+
+#define BIT_SHIFT_VOQ_TXBD_DESA_8822B 0
+#define BIT_MASK_VOQ_TXBD_DESA_8822B 0xffffffffffffffffL
+#define BIT_VOQ_TXBD_DESA_8822B(x)                                             \
+	(((x) & BIT_MASK_VOQ_TXBD_DESA_8822B) << BIT_SHIFT_VOQ_TXBD_DESA_8822B)
+#define BIT_GET_VOQ_TXBD_DESA_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_VOQ_TXBD_DESA_8822B) & BIT_MASK_VOQ_TXBD_DESA_8822B)
+
+/* 2 REG_VIQ_TXBD_DESA_8822B */
+
+#define BIT_SHIFT_VIQ_TXBD_DESA_8822B 0
+#define BIT_MASK_VIQ_TXBD_DESA_8822B 0xffffffffffffffffL
+#define BIT_VIQ_TXBD_DESA_8822B(x)                                             \
+	(((x) & BIT_MASK_VIQ_TXBD_DESA_8822B) << BIT_SHIFT_VIQ_TXBD_DESA_8822B)
+#define BIT_GET_VIQ_TXBD_DESA_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_VIQ_TXBD_DESA_8822B) & BIT_MASK_VIQ_TXBD_DESA_8822B)
+
+/* 2 REG_BEQ_TXBD_DESA_8822B */
+
+#define BIT_SHIFT_BEQ_TXBD_DESA_8822B 0
+#define BIT_MASK_BEQ_TXBD_DESA_8822B 0xffffffffffffffffL
+#define BIT_BEQ_TXBD_DESA_8822B(x)                                             \
+	(((x) & BIT_MASK_BEQ_TXBD_DESA_8822B) << BIT_SHIFT_BEQ_TXBD_DESA_8822B)
+#define BIT_GET_BEQ_TXBD_DESA_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_BEQ_TXBD_DESA_8822B) & BIT_MASK_BEQ_TXBD_DESA_8822B)
+
+/* 2 REG_BKQ_TXBD_DESA_8822B */
+
+#define BIT_SHIFT_BKQ_TXBD_DESA_8822B 0
+#define BIT_MASK_BKQ_TXBD_DESA_8822B 0xffffffffffffffffL
+#define BIT_BKQ_TXBD_DESA_8822B(x)                                             \
+	(((x) & BIT_MASK_BKQ_TXBD_DESA_8822B) << BIT_SHIFT_BKQ_TXBD_DESA_8822B)
+#define BIT_GET_BKQ_TXBD_DESA_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_BKQ_TXBD_DESA_8822B) & BIT_MASK_BKQ_TXBD_DESA_8822B)
+
+/* 2 REG_RXQ_RXBD_DESA_8822B */
+
+#define BIT_SHIFT_RXQ_RXBD_DESA_8822B 0
+#define BIT_MASK_RXQ_RXBD_DESA_8822B 0xffffffffffffffffL
+#define BIT_RXQ_RXBD_DESA_8822B(x)                                             \
+	(((x) & BIT_MASK_RXQ_RXBD_DESA_8822B) << BIT_SHIFT_RXQ_RXBD_DESA_8822B)
+#define BIT_GET_RXQ_RXBD_DESA_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_RXQ_RXBD_DESA_8822B) & BIT_MASK_RXQ_RXBD_DESA_8822B)
+
+/* 2 REG_HI0Q_TXBD_DESA_8822B */
+
+#define BIT_SHIFT_HI0Q_TXBD_DESA_8822B 0
+#define BIT_MASK_HI0Q_TXBD_DESA_8822B 0xffffffffffffffffL
+#define BIT_HI0Q_TXBD_DESA_8822B(x)                                            \
+	(((x) & BIT_MASK_HI0Q_TXBD_DESA_8822B)                                 \
+	 << BIT_SHIFT_HI0Q_TXBD_DESA_8822B)
+#define BIT_GET_HI0Q_TXBD_DESA_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HI0Q_TXBD_DESA_8822B) &                             \
+	 BIT_MASK_HI0Q_TXBD_DESA_8822B)
+
+/* 2 REG_HI1Q_TXBD_DESA_8822B */
+
+#define BIT_SHIFT_HI1Q_TXBD_DESA_8822B 0
+#define BIT_MASK_HI1Q_TXBD_DESA_8822B 0xffffffffffffffffL
+#define BIT_HI1Q_TXBD_DESA_8822B(x)                                            \
+	(((x) & BIT_MASK_HI1Q_TXBD_DESA_8822B)                                 \
+	 << BIT_SHIFT_HI1Q_TXBD_DESA_8822B)
+#define BIT_GET_HI1Q_TXBD_DESA_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HI1Q_TXBD_DESA_8822B) &                             \
+	 BIT_MASK_HI1Q_TXBD_DESA_8822B)
+
+/* 2 REG_HI2Q_TXBD_DESA_8822B */
+
+#define BIT_SHIFT_HI2Q_TXBD_DESA_8822B 0
+#define BIT_MASK_HI2Q_TXBD_DESA_8822B 0xffffffffffffffffL
+#define BIT_HI2Q_TXBD_DESA_8822B(x)                                            \
+	(((x) & BIT_MASK_HI2Q_TXBD_DESA_8822B)                                 \
+	 << BIT_SHIFT_HI2Q_TXBD_DESA_8822B)
+#define BIT_GET_HI2Q_TXBD_DESA_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HI2Q_TXBD_DESA_8822B) &                             \
+	 BIT_MASK_HI2Q_TXBD_DESA_8822B)
+
+/* 2 REG_HI3Q_TXBD_DESA_8822B */
+
+#define BIT_SHIFT_HI3Q_TXBD_DESA_8822B 0
+#define BIT_MASK_HI3Q_TXBD_DESA_8822B 0xffffffffffffffffL
+#define BIT_HI3Q_TXBD_DESA_8822B(x)                                            \
+	(((x) & BIT_MASK_HI3Q_TXBD_DESA_8822B)                                 \
+	 << BIT_SHIFT_HI3Q_TXBD_DESA_8822B)
+#define BIT_GET_HI3Q_TXBD_DESA_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HI3Q_TXBD_DESA_8822B) &                             \
+	 BIT_MASK_HI3Q_TXBD_DESA_8822B)
+
+/* 2 REG_HI4Q_TXBD_DESA_8822B */
+
+#define BIT_SHIFT_HI4Q_TXBD_DESA_8822B 0
+#define BIT_MASK_HI4Q_TXBD_DESA_8822B 0xffffffffffffffffL
+#define BIT_HI4Q_TXBD_DESA_8822B(x)                                            \
+	(((x) & BIT_MASK_HI4Q_TXBD_DESA_8822B)                                 \
+	 << BIT_SHIFT_HI4Q_TXBD_DESA_8822B)
+#define BIT_GET_HI4Q_TXBD_DESA_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HI4Q_TXBD_DESA_8822B) &                             \
+	 BIT_MASK_HI4Q_TXBD_DESA_8822B)
+
+/* 2 REG_HI5Q_TXBD_DESA_8822B */
+
+#define BIT_SHIFT_HI5Q_TXBD_DESA_8822B 0
+#define BIT_MASK_HI5Q_TXBD_DESA_8822B 0xffffffffffffffffL
+#define BIT_HI5Q_TXBD_DESA_8822B(x)                                            \
+	(((x) & BIT_MASK_HI5Q_TXBD_DESA_8822B)                                 \
+	 << BIT_SHIFT_HI5Q_TXBD_DESA_8822B)
+#define BIT_GET_HI5Q_TXBD_DESA_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HI5Q_TXBD_DESA_8822B) &                             \
+	 BIT_MASK_HI5Q_TXBD_DESA_8822B)
+
+/* 2 REG_HI6Q_TXBD_DESA_8822B */
+
+#define BIT_SHIFT_HI6Q_TXBD_DESA_8822B 0
+#define BIT_MASK_HI6Q_TXBD_DESA_8822B 0xffffffffffffffffL
+#define BIT_HI6Q_TXBD_DESA_8822B(x)                                            \
+	(((x) & BIT_MASK_HI6Q_TXBD_DESA_8822B)                                 \
+	 << BIT_SHIFT_HI6Q_TXBD_DESA_8822B)
+#define BIT_GET_HI6Q_TXBD_DESA_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HI6Q_TXBD_DESA_8822B) &                             \
+	 BIT_MASK_HI6Q_TXBD_DESA_8822B)
+
+/* 2 REG_HI7Q_TXBD_DESA_8822B */
+
+#define BIT_SHIFT_HI7Q_TXBD_DESA_8822B 0
+#define BIT_MASK_HI7Q_TXBD_DESA_8822B 0xffffffffffffffffL
+#define BIT_HI7Q_TXBD_DESA_8822B(x)                                            \
+	(((x) & BIT_MASK_HI7Q_TXBD_DESA_8822B)                                 \
+	 << BIT_SHIFT_HI7Q_TXBD_DESA_8822B)
+#define BIT_GET_HI7Q_TXBD_DESA_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HI7Q_TXBD_DESA_8822B) &                             \
+	 BIT_MASK_HI7Q_TXBD_DESA_8822B)
+
+/* 2 REG_MGQ_TXBD_NUM_8822B */
+#define BIT_PCIE_MGQ_FLAG_8822B BIT(14)
+
+#define BIT_SHIFT_MGQ_DESC_MODE_8822B 12
+#define BIT_MASK_MGQ_DESC_MODE_8822B 0x3
+#define BIT_MGQ_DESC_MODE_8822B(x)                                             \
+	(((x) & BIT_MASK_MGQ_DESC_MODE_8822B) << BIT_SHIFT_MGQ_DESC_MODE_8822B)
+#define BIT_GET_MGQ_DESC_MODE_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_MGQ_DESC_MODE_8822B) & BIT_MASK_MGQ_DESC_MODE_8822B)
+
+#define BIT_SHIFT_MGQ_DESC_NUM_8822B 0
+#define BIT_MASK_MGQ_DESC_NUM_8822B 0xfff
+#define BIT_MGQ_DESC_NUM_8822B(x)                                              \
+	(((x) & BIT_MASK_MGQ_DESC_NUM_8822B) << BIT_SHIFT_MGQ_DESC_NUM_8822B)
+#define BIT_GET_MGQ_DESC_NUM_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_MGQ_DESC_NUM_8822B) & BIT_MASK_MGQ_DESC_NUM_8822B)
+
+/* 2 REG_RX_RXBD_NUM_8822B */
+#define BIT_SYS_32_64_8822B BIT(15)
+
+#define BIT_SHIFT_BCNQ_DESC_MODE_8822B 13
+#define BIT_MASK_BCNQ_DESC_MODE_8822B 0x3
+#define BIT_BCNQ_DESC_MODE_8822B(x)                                            \
+	(((x) & BIT_MASK_BCNQ_DESC_MODE_8822B)                                 \
+	 << BIT_SHIFT_BCNQ_DESC_MODE_8822B)
+#define BIT_GET_BCNQ_DESC_MODE_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_BCNQ_DESC_MODE_8822B) &                             \
+	 BIT_MASK_BCNQ_DESC_MODE_8822B)
+
+#define BIT_PCIE_BCNQ_FLAG_8822B BIT(12)
+
+#define BIT_SHIFT_RXQ_DESC_NUM_8822B 0
+#define BIT_MASK_RXQ_DESC_NUM_8822B 0xfff
+#define BIT_RXQ_DESC_NUM_8822B(x)                                              \
+	(((x) & BIT_MASK_RXQ_DESC_NUM_8822B) << BIT_SHIFT_RXQ_DESC_NUM_8822B)
+#define BIT_GET_RXQ_DESC_NUM_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_RXQ_DESC_NUM_8822B) & BIT_MASK_RXQ_DESC_NUM_8822B)
+
+/* 2 REG_VOQ_TXBD_NUM_8822B */
+#define BIT_PCIE_VOQ_FLAG_8822B BIT(14)
+
+#define BIT_SHIFT_VOQ_DESC_MODE_8822B 12
+#define BIT_MASK_VOQ_DESC_MODE_8822B 0x3
+#define BIT_VOQ_DESC_MODE_8822B(x)                                             \
+	(((x) & BIT_MASK_VOQ_DESC_MODE_8822B) << BIT_SHIFT_VOQ_DESC_MODE_8822B)
+#define BIT_GET_VOQ_DESC_MODE_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_VOQ_DESC_MODE_8822B) & BIT_MASK_VOQ_DESC_MODE_8822B)
+
+#define BIT_SHIFT_VOQ_DESC_NUM_8822B 0
+#define BIT_MASK_VOQ_DESC_NUM_8822B 0xfff
+#define BIT_VOQ_DESC_NUM_8822B(x)                                              \
+	(((x) & BIT_MASK_VOQ_DESC_NUM_8822B) << BIT_SHIFT_VOQ_DESC_NUM_8822B)
+#define BIT_GET_VOQ_DESC_NUM_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_VOQ_DESC_NUM_8822B) & BIT_MASK_VOQ_DESC_NUM_8822B)
+
+/* 2 REG_VIQ_TXBD_NUM_8822B */
+#define BIT_PCIE_VIQ_FLAG_8822B BIT(14)
+
+#define BIT_SHIFT_VIQ_DESC_MODE_8822B 12
+#define BIT_MASK_VIQ_DESC_MODE_8822B 0x3
+#define BIT_VIQ_DESC_MODE_8822B(x)                                             \
+	(((x) & BIT_MASK_VIQ_DESC_MODE_8822B) << BIT_SHIFT_VIQ_DESC_MODE_8822B)
+#define BIT_GET_VIQ_DESC_MODE_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_VIQ_DESC_MODE_8822B) & BIT_MASK_VIQ_DESC_MODE_8822B)
+
+#define BIT_SHIFT_VIQ_DESC_NUM_8822B 0
+#define BIT_MASK_VIQ_DESC_NUM_8822B 0xfff
+#define BIT_VIQ_DESC_NUM_8822B(x)                                              \
+	(((x) & BIT_MASK_VIQ_DESC_NUM_8822B) << BIT_SHIFT_VIQ_DESC_NUM_8822B)
+#define BIT_GET_VIQ_DESC_NUM_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_VIQ_DESC_NUM_8822B) & BIT_MASK_VIQ_DESC_NUM_8822B)
+
+/* 2 REG_BEQ_TXBD_NUM_8822B */
+#define BIT_PCIE_BEQ_FLAG_8822B BIT(14)
+
+#define BIT_SHIFT_BEQ_DESC_MODE_8822B 12
+#define BIT_MASK_BEQ_DESC_MODE_8822B 0x3
+#define BIT_BEQ_DESC_MODE_8822B(x)                                             \
+	(((x) & BIT_MASK_BEQ_DESC_MODE_8822B) << BIT_SHIFT_BEQ_DESC_MODE_8822B)
+#define BIT_GET_BEQ_DESC_MODE_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_BEQ_DESC_MODE_8822B) & BIT_MASK_BEQ_DESC_MODE_8822B)
+
+#define BIT_SHIFT_BEQ_DESC_NUM_8822B 0
+#define BIT_MASK_BEQ_DESC_NUM_8822B 0xfff
+#define BIT_BEQ_DESC_NUM_8822B(x)                                              \
+	(((x) & BIT_MASK_BEQ_DESC_NUM_8822B) << BIT_SHIFT_BEQ_DESC_NUM_8822B)
+#define BIT_GET_BEQ_DESC_NUM_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_BEQ_DESC_NUM_8822B) & BIT_MASK_BEQ_DESC_NUM_8822B)
+
+/* 2 REG_BKQ_TXBD_NUM_8822B */
+#define BIT_PCIE_BKQ_FLAG_8822B BIT(14)
+
+#define BIT_SHIFT_BKQ_DESC_MODE_8822B 12
+#define BIT_MASK_BKQ_DESC_MODE_8822B 0x3
+#define BIT_BKQ_DESC_MODE_8822B(x)                                             \
+	(((x) & BIT_MASK_BKQ_DESC_MODE_8822B) << BIT_SHIFT_BKQ_DESC_MODE_8822B)
+#define BIT_GET_BKQ_DESC_MODE_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_BKQ_DESC_MODE_8822B) & BIT_MASK_BKQ_DESC_MODE_8822B)
+
+#define BIT_SHIFT_BKQ_DESC_NUM_8822B 0
+#define BIT_MASK_BKQ_DESC_NUM_8822B 0xfff
+#define BIT_BKQ_DESC_NUM_8822B(x)                                              \
+	(((x) & BIT_MASK_BKQ_DESC_NUM_8822B) << BIT_SHIFT_BKQ_DESC_NUM_8822B)
+#define BIT_GET_BKQ_DESC_NUM_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_BKQ_DESC_NUM_8822B) & BIT_MASK_BKQ_DESC_NUM_8822B)
+
+/* 2 REG_HI0Q_TXBD_NUM_8822B */
+#define BIT_HI0Q_FLAG_8822B BIT(14)
+
+#define BIT_SHIFT_HI0Q_DESC_MODE_8822B 12
+#define BIT_MASK_HI0Q_DESC_MODE_8822B 0x3
+#define BIT_HI0Q_DESC_MODE_8822B(x)                                            \
+	(((x) & BIT_MASK_HI0Q_DESC_MODE_8822B)                                 \
+	 << BIT_SHIFT_HI0Q_DESC_MODE_8822B)
+#define BIT_GET_HI0Q_DESC_MODE_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HI0Q_DESC_MODE_8822B) &                             \
+	 BIT_MASK_HI0Q_DESC_MODE_8822B)
+
+#define BIT_SHIFT_HI0Q_DESC_NUM_8822B 0
+#define BIT_MASK_HI0Q_DESC_NUM_8822B 0xfff
+#define BIT_HI0Q_DESC_NUM_8822B(x)                                             \
+	(((x) & BIT_MASK_HI0Q_DESC_NUM_8822B) << BIT_SHIFT_HI0Q_DESC_NUM_8822B)
+#define BIT_GET_HI0Q_DESC_NUM_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_HI0Q_DESC_NUM_8822B) & BIT_MASK_HI0Q_DESC_NUM_8822B)
+
+/* 2 REG_HI1Q_TXBD_NUM_8822B */
+#define BIT_HI1Q_FLAG_8822B BIT(14)
+
+#define BIT_SHIFT_HI1Q_DESC_MODE_8822B 12
+#define BIT_MASK_HI1Q_DESC_MODE_8822B 0x3
+#define BIT_HI1Q_DESC_MODE_8822B(x)                                            \
+	(((x) & BIT_MASK_HI1Q_DESC_MODE_8822B)                                 \
+	 << BIT_SHIFT_HI1Q_DESC_MODE_8822B)
+#define BIT_GET_HI1Q_DESC_MODE_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HI1Q_DESC_MODE_8822B) &                             \
+	 BIT_MASK_HI1Q_DESC_MODE_8822B)
+
+#define BIT_SHIFT_HI1Q_DESC_NUM_8822B 0
+#define BIT_MASK_HI1Q_DESC_NUM_8822B 0xfff
+#define BIT_HI1Q_DESC_NUM_8822B(x)                                             \
+	(((x) & BIT_MASK_HI1Q_DESC_NUM_8822B) << BIT_SHIFT_HI1Q_DESC_NUM_8822B)
+#define BIT_GET_HI1Q_DESC_NUM_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_HI1Q_DESC_NUM_8822B) & BIT_MASK_HI1Q_DESC_NUM_8822B)
+
+/* 2 REG_HI2Q_TXBD_NUM_8822B */
+#define BIT_HI2Q_FLAG_8822B BIT(14)
+
+#define BIT_SHIFT_HI2Q_DESC_MODE_8822B 12
+#define BIT_MASK_HI2Q_DESC_MODE_8822B 0x3
+#define BIT_HI2Q_DESC_MODE_8822B(x)                                            \
+	(((x) & BIT_MASK_HI2Q_DESC_MODE_8822B)                                 \
+	 << BIT_SHIFT_HI2Q_DESC_MODE_8822B)
+#define BIT_GET_HI2Q_DESC_MODE_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HI2Q_DESC_MODE_8822B) &                             \
+	 BIT_MASK_HI2Q_DESC_MODE_8822B)
+
+#define BIT_SHIFT_HI2Q_DESC_NUM_8822B 0
+#define BIT_MASK_HI2Q_DESC_NUM_8822B 0xfff
+#define BIT_HI2Q_DESC_NUM_8822B(x)                                             \
+	(((x) & BIT_MASK_HI2Q_DESC_NUM_8822B) << BIT_SHIFT_HI2Q_DESC_NUM_8822B)
+#define BIT_GET_HI2Q_DESC_NUM_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_HI2Q_DESC_NUM_8822B) & BIT_MASK_HI2Q_DESC_NUM_8822B)
+
+/* 2 REG_HI3Q_TXBD_NUM_8822B */
+#define BIT_HI3Q_FLAG_8822B BIT(14)
+
+#define BIT_SHIFT_HI3Q_DESC_MODE_8822B 12
+#define BIT_MASK_HI3Q_DESC_MODE_8822B 0x3
+#define BIT_HI3Q_DESC_MODE_8822B(x)                                            \
+	(((x) & BIT_MASK_HI3Q_DESC_MODE_8822B)                                 \
+	 << BIT_SHIFT_HI3Q_DESC_MODE_8822B)
+#define BIT_GET_HI3Q_DESC_MODE_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HI3Q_DESC_MODE_8822B) &                             \
+	 BIT_MASK_HI3Q_DESC_MODE_8822B)
+
+#define BIT_SHIFT_HI3Q_DESC_NUM_8822B 0
+#define BIT_MASK_HI3Q_DESC_NUM_8822B 0xfff
+#define BIT_HI3Q_DESC_NUM_8822B(x)                                             \
+	(((x) & BIT_MASK_HI3Q_DESC_NUM_8822B) << BIT_SHIFT_HI3Q_DESC_NUM_8822B)
+#define BIT_GET_HI3Q_DESC_NUM_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_HI3Q_DESC_NUM_8822B) & BIT_MASK_HI3Q_DESC_NUM_8822B)
+
+/* 2 REG_HI4Q_TXBD_NUM_8822B */
+#define BIT_HI4Q_FLAG_8822B BIT(14)
+
+#define BIT_SHIFT_HI4Q_DESC_MODE_8822B 12
+#define BIT_MASK_HI4Q_DESC_MODE_8822B 0x3
+#define BIT_HI4Q_DESC_MODE_8822B(x)                                            \
+	(((x) & BIT_MASK_HI4Q_DESC_MODE_8822B)                                 \
+	 << BIT_SHIFT_HI4Q_DESC_MODE_8822B)
+#define BIT_GET_HI4Q_DESC_MODE_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HI4Q_DESC_MODE_8822B) &                             \
+	 BIT_MASK_HI4Q_DESC_MODE_8822B)
+
+#define BIT_SHIFT_HI4Q_DESC_NUM_8822B 0
+#define BIT_MASK_HI4Q_DESC_NUM_8822B 0xfff
+#define BIT_HI4Q_DESC_NUM_8822B(x)                                             \
+	(((x) & BIT_MASK_HI4Q_DESC_NUM_8822B) << BIT_SHIFT_HI4Q_DESC_NUM_8822B)
+#define BIT_GET_HI4Q_DESC_NUM_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_HI4Q_DESC_NUM_8822B) & BIT_MASK_HI4Q_DESC_NUM_8822B)
+
+/* 2 REG_HI5Q_TXBD_NUM_8822B */
+#define BIT_HI5Q_FLAG_8822B BIT(14)
+
+#define BIT_SHIFT_HI5Q_DESC_MODE_8822B 12
+#define BIT_MASK_HI5Q_DESC_MODE_8822B 0x3
+#define BIT_HI5Q_DESC_MODE_8822B(x)                                            \
+	(((x) & BIT_MASK_HI5Q_DESC_MODE_8822B)                                 \
+	 << BIT_SHIFT_HI5Q_DESC_MODE_8822B)
+#define BIT_GET_HI5Q_DESC_MODE_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HI5Q_DESC_MODE_8822B) &                             \
+	 BIT_MASK_HI5Q_DESC_MODE_8822B)
+
+#define BIT_SHIFT_HI5Q_DESC_NUM_8822B 0
+#define BIT_MASK_HI5Q_DESC_NUM_8822B 0xfff
+#define BIT_HI5Q_DESC_NUM_8822B(x)                                             \
+	(((x) & BIT_MASK_HI5Q_DESC_NUM_8822B) << BIT_SHIFT_HI5Q_DESC_NUM_8822B)
+#define BIT_GET_HI5Q_DESC_NUM_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_HI5Q_DESC_NUM_8822B) & BIT_MASK_HI5Q_DESC_NUM_8822B)
+
+/* 2 REG_HI6Q_TXBD_NUM_8822B */
+#define BIT_HI6Q_FLAG_8822B BIT(14)
+
+#define BIT_SHIFT_HI6Q_DESC_MODE_8822B 12
+#define BIT_MASK_HI6Q_DESC_MODE_8822B 0x3
+#define BIT_HI6Q_DESC_MODE_8822B(x)                                            \
+	(((x) & BIT_MASK_HI6Q_DESC_MODE_8822B)                                 \
+	 << BIT_SHIFT_HI6Q_DESC_MODE_8822B)
+#define BIT_GET_HI6Q_DESC_MODE_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HI6Q_DESC_MODE_8822B) &                             \
+	 BIT_MASK_HI6Q_DESC_MODE_8822B)
+
+#define BIT_SHIFT_HI6Q_DESC_NUM_8822B 0
+#define BIT_MASK_HI6Q_DESC_NUM_8822B 0xfff
+#define BIT_HI6Q_DESC_NUM_8822B(x)                                             \
+	(((x) & BIT_MASK_HI6Q_DESC_NUM_8822B) << BIT_SHIFT_HI6Q_DESC_NUM_8822B)
+#define BIT_GET_HI6Q_DESC_NUM_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_HI6Q_DESC_NUM_8822B) & BIT_MASK_HI6Q_DESC_NUM_8822B)
+
+/* 2 REG_HI7Q_TXBD_NUM_8822B */
+#define BIT_HI7Q_FLAG_8822B BIT(14)
+
+#define BIT_SHIFT_HI7Q_DESC_MODE_8822B 12
+#define BIT_MASK_HI7Q_DESC_MODE_8822B 0x3
+#define BIT_HI7Q_DESC_MODE_8822B(x)                                            \
+	(((x) & BIT_MASK_HI7Q_DESC_MODE_8822B)                                 \
+	 << BIT_SHIFT_HI7Q_DESC_MODE_8822B)
+#define BIT_GET_HI7Q_DESC_MODE_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HI7Q_DESC_MODE_8822B) &                             \
+	 BIT_MASK_HI7Q_DESC_MODE_8822B)
+
+#define BIT_SHIFT_HI7Q_DESC_NUM_8822B 0
+#define BIT_MASK_HI7Q_DESC_NUM_8822B 0xfff
+#define BIT_HI7Q_DESC_NUM_8822B(x)                                             \
+	(((x) & BIT_MASK_HI7Q_DESC_NUM_8822B) << BIT_SHIFT_HI7Q_DESC_NUM_8822B)
+#define BIT_GET_HI7Q_DESC_NUM_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_HI7Q_DESC_NUM_8822B) & BIT_MASK_HI7Q_DESC_NUM_8822B)
+
+/* 2 REG_TSFTIMER_HCI_8822B */
+
+#define BIT_SHIFT_TSFT2_HCI_8822B 16
+#define BIT_MASK_TSFT2_HCI_8822B 0xffff
+#define BIT_TSFT2_HCI_8822B(x)                                                 \
+	(((x) & BIT_MASK_TSFT2_HCI_8822B) << BIT_SHIFT_TSFT2_HCI_8822B)
+#define BIT_GET_TSFT2_HCI_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_TSFT2_HCI_8822B) & BIT_MASK_TSFT2_HCI_8822B)
+
+#define BIT_SHIFT_TSFT1_HCI_8822B 0
+#define BIT_MASK_TSFT1_HCI_8822B 0xffff
+#define BIT_TSFT1_HCI_8822B(x)                                                 \
+	(((x) & BIT_MASK_TSFT1_HCI_8822B) << BIT_SHIFT_TSFT1_HCI_8822B)
+#define BIT_GET_TSFT1_HCI_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_TSFT1_HCI_8822B) & BIT_MASK_TSFT1_HCI_8822B)
+
+/* 2 REG_BD_RWPTR_CLR_8822B */
+#define BIT_CLR_HI7Q_HW_IDX_8822B BIT(29)
+#define BIT_CLR_HI6Q_HW_IDX_8822B BIT(28)
+#define BIT_CLR_HI5Q_HW_IDX_8822B BIT(27)
+#define BIT_CLR_HI4Q_HW_IDX_8822B BIT(26)
+#define BIT_CLR_HI3Q_HW_IDX_8822B BIT(25)
+#define BIT_CLR_HI2Q_HW_IDX_8822B BIT(24)
+#define BIT_CLR_HI1Q_HW_IDX_8822B BIT(23)
+#define BIT_CLR_HI0Q_HW_IDX_8822B BIT(22)
+#define BIT_CLR_BKQ_HW_IDX_8822B BIT(21)
+#define BIT_CLR_BEQ_HW_IDX_8822B BIT(20)
+#define BIT_CLR_VIQ_HW_IDX_8822B BIT(19)
+#define BIT_CLR_VOQ_HW_IDX_8822B BIT(18)
+#define BIT_CLR_MGQ_HW_IDX_8822B BIT(17)
+#define BIT_CLR_RXQ_HW_IDX_8822B BIT(16)
+#define BIT_CLR_HI7Q_HOST_IDX_8822B BIT(13)
+#define BIT_CLR_HI6Q_HOST_IDX_8822B BIT(12)
+#define BIT_CLR_HI5Q_HOST_IDX_8822B BIT(11)
+#define BIT_CLR_HI4Q_HOST_IDX_8822B BIT(10)
+#define BIT_CLR_HI3Q_HOST_IDX_8822B BIT(9)
+#define BIT_CLR_HI2Q_HOST_IDX_8822B BIT(8)
+#define BIT_CLR_HI1Q_HOST_IDX_8822B BIT(7)
+#define BIT_CLR_HI0Q_HOST_IDX_8822B BIT(6)
+#define BIT_CLR_BKQ_HOST_IDX_8822B BIT(5)
+#define BIT_CLR_BEQ_HOST_IDX_8822B BIT(4)
+#define BIT_CLR_VIQ_HOST_IDX_8822B BIT(3)
+#define BIT_CLR_VOQ_HOST_IDX_8822B BIT(2)
+#define BIT_CLR_MGQ_HOST_IDX_8822B BIT(1)
+#define BIT_CLR_RXQ_HOST_IDX_8822B BIT(0)
+
+/* 2 REG_VOQ_TXBD_IDX_8822B */
+
+#define BIT_SHIFT_VOQ_HW_IDX_8822B 16
+#define BIT_MASK_VOQ_HW_IDX_8822B 0xfff
+#define BIT_VOQ_HW_IDX_8822B(x)                                                \
+	(((x) & BIT_MASK_VOQ_HW_IDX_8822B) << BIT_SHIFT_VOQ_HW_IDX_8822B)
+#define BIT_GET_VOQ_HW_IDX_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_VOQ_HW_IDX_8822B) & BIT_MASK_VOQ_HW_IDX_8822B)
+
+#define BIT_SHIFT_VOQ_HOST_IDX_8822B 0
+#define BIT_MASK_VOQ_HOST_IDX_8822B 0xfff
+#define BIT_VOQ_HOST_IDX_8822B(x)                                              \
+	(((x) & BIT_MASK_VOQ_HOST_IDX_8822B) << BIT_SHIFT_VOQ_HOST_IDX_8822B)
+#define BIT_GET_VOQ_HOST_IDX_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_VOQ_HOST_IDX_8822B) & BIT_MASK_VOQ_HOST_IDX_8822B)
+
+/* 2 REG_VIQ_TXBD_IDX_8822B */
+
+#define BIT_SHIFT_VIQ_HW_IDX_8822B 16
+#define BIT_MASK_VIQ_HW_IDX_8822B 0xfff
+#define BIT_VIQ_HW_IDX_8822B(x)                                                \
+	(((x) & BIT_MASK_VIQ_HW_IDX_8822B) << BIT_SHIFT_VIQ_HW_IDX_8822B)
+#define BIT_GET_VIQ_HW_IDX_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_VIQ_HW_IDX_8822B) & BIT_MASK_VIQ_HW_IDX_8822B)
+
+#define BIT_SHIFT_VIQ_HOST_IDX_8822B 0
+#define BIT_MASK_VIQ_HOST_IDX_8822B 0xfff
+#define BIT_VIQ_HOST_IDX_8822B(x)                                              \
+	(((x) & BIT_MASK_VIQ_HOST_IDX_8822B) << BIT_SHIFT_VIQ_HOST_IDX_8822B)
+#define BIT_GET_VIQ_HOST_IDX_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_VIQ_HOST_IDX_8822B) & BIT_MASK_VIQ_HOST_IDX_8822B)
+
+/* 2 REG_BEQ_TXBD_IDX_8822B */
+
+#define BIT_SHIFT_BEQ_HW_IDX_8822B 16
+#define BIT_MASK_BEQ_HW_IDX_8822B 0xfff
+#define BIT_BEQ_HW_IDX_8822B(x)                                                \
+	(((x) & BIT_MASK_BEQ_HW_IDX_8822B) << BIT_SHIFT_BEQ_HW_IDX_8822B)
+#define BIT_GET_BEQ_HW_IDX_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_BEQ_HW_IDX_8822B) & BIT_MASK_BEQ_HW_IDX_8822B)
+
+#define BIT_SHIFT_BEQ_HOST_IDX_8822B 0
+#define BIT_MASK_BEQ_HOST_IDX_8822B 0xfff
+#define BIT_BEQ_HOST_IDX_8822B(x)                                              \
+	(((x) & BIT_MASK_BEQ_HOST_IDX_8822B) << BIT_SHIFT_BEQ_HOST_IDX_8822B)
+#define BIT_GET_BEQ_HOST_IDX_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_BEQ_HOST_IDX_8822B) & BIT_MASK_BEQ_HOST_IDX_8822B)
+
+/* 2 REG_BKQ_TXBD_IDX_8822B */
+
+#define BIT_SHIFT_BKQ_HW_IDX_8822B 16
+#define BIT_MASK_BKQ_HW_IDX_8822B 0xfff
+#define BIT_BKQ_HW_IDX_8822B(x)                                                \
+	(((x) & BIT_MASK_BKQ_HW_IDX_8822B) << BIT_SHIFT_BKQ_HW_IDX_8822B)
+#define BIT_GET_BKQ_HW_IDX_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_BKQ_HW_IDX_8822B) & BIT_MASK_BKQ_HW_IDX_8822B)
+
+#define BIT_SHIFT_BKQ_HOST_IDX_8822B 0
+#define BIT_MASK_BKQ_HOST_IDX_8822B 0xfff
+#define BIT_BKQ_HOST_IDX_8822B(x)                                              \
+	(((x) & BIT_MASK_BKQ_HOST_IDX_8822B) << BIT_SHIFT_BKQ_HOST_IDX_8822B)
+#define BIT_GET_BKQ_HOST_IDX_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_BKQ_HOST_IDX_8822B) & BIT_MASK_BKQ_HOST_IDX_8822B)
+
+/* 2 REG_MGQ_TXBD_IDX_8822B */
+
+#define BIT_SHIFT_MGQ_HW_IDX_8822B 16
+#define BIT_MASK_MGQ_HW_IDX_8822B 0xfff
+#define BIT_MGQ_HW_IDX_8822B(x)                                                \
+	(((x) & BIT_MASK_MGQ_HW_IDX_8822B) << BIT_SHIFT_MGQ_HW_IDX_8822B)
+#define BIT_GET_MGQ_HW_IDX_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_MGQ_HW_IDX_8822B) & BIT_MASK_MGQ_HW_IDX_8822B)
+
+#define BIT_SHIFT_MGQ_HOST_IDX_8822B 0
+#define BIT_MASK_MGQ_HOST_IDX_8822B 0xfff
+#define BIT_MGQ_HOST_IDX_8822B(x)                                              \
+	(((x) & BIT_MASK_MGQ_HOST_IDX_8822B) << BIT_SHIFT_MGQ_HOST_IDX_8822B)
+#define BIT_GET_MGQ_HOST_IDX_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_MGQ_HOST_IDX_8822B) & BIT_MASK_MGQ_HOST_IDX_8822B)
+
+/* 2 REG_RXQ_RXBD_IDX_8822B */
+
+#define BIT_SHIFT_RXQ_HW_IDX_8822B 16
+#define BIT_MASK_RXQ_HW_IDX_8822B 0xfff
+#define BIT_RXQ_HW_IDX_8822B(x)                                                \
+	(((x) & BIT_MASK_RXQ_HW_IDX_8822B) << BIT_SHIFT_RXQ_HW_IDX_8822B)
+#define BIT_GET_RXQ_HW_IDX_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_RXQ_HW_IDX_8822B) & BIT_MASK_RXQ_HW_IDX_8822B)
+
+#define BIT_SHIFT_RXQ_HOST_IDX_8822B 0
+#define BIT_MASK_RXQ_HOST_IDX_8822B 0xfff
+#define BIT_RXQ_HOST_IDX_8822B(x)                                              \
+	(((x) & BIT_MASK_RXQ_HOST_IDX_8822B) << BIT_SHIFT_RXQ_HOST_IDX_8822B)
+#define BIT_GET_RXQ_HOST_IDX_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_RXQ_HOST_IDX_8822B) & BIT_MASK_RXQ_HOST_IDX_8822B)
+
+/* 2 REG_HI0Q_TXBD_IDX_8822B */
+
+#define BIT_SHIFT_HI0Q_HW_IDX_8822B 16
+#define BIT_MASK_HI0Q_HW_IDX_8822B 0xfff
+#define BIT_HI0Q_HW_IDX_8822B(x)                                               \
+	(((x) & BIT_MASK_HI0Q_HW_IDX_8822B) << BIT_SHIFT_HI0Q_HW_IDX_8822B)
+#define BIT_GET_HI0Q_HW_IDX_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_HI0Q_HW_IDX_8822B) & BIT_MASK_HI0Q_HW_IDX_8822B)
+
+#define BIT_SHIFT_HI0Q_HOST_IDX_8822B 0
+#define BIT_MASK_HI0Q_HOST_IDX_8822B 0xfff
+#define BIT_HI0Q_HOST_IDX_8822B(x)                                             \
+	(((x) & BIT_MASK_HI0Q_HOST_IDX_8822B) << BIT_SHIFT_HI0Q_HOST_IDX_8822B)
+#define BIT_GET_HI0Q_HOST_IDX_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_HI0Q_HOST_IDX_8822B) & BIT_MASK_HI0Q_HOST_IDX_8822B)
+
+/* 2 REG_HI1Q_TXBD_IDX_8822B */
+
+#define BIT_SHIFT_HI1Q_HW_IDX_8822B 16
+#define BIT_MASK_HI1Q_HW_IDX_8822B 0xfff
+#define BIT_HI1Q_HW_IDX_8822B(x)                                               \
+	(((x) & BIT_MASK_HI1Q_HW_IDX_8822B) << BIT_SHIFT_HI1Q_HW_IDX_8822B)
+#define BIT_GET_HI1Q_HW_IDX_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_HI1Q_HW_IDX_8822B) & BIT_MASK_HI1Q_HW_IDX_8822B)
+
+#define BIT_SHIFT_HI1Q_HOST_IDX_8822B 0
+#define BIT_MASK_HI1Q_HOST_IDX_8822B 0xfff
+#define BIT_HI1Q_HOST_IDX_8822B(x)                                             \
+	(((x) & BIT_MASK_HI1Q_HOST_IDX_8822B) << BIT_SHIFT_HI1Q_HOST_IDX_8822B)
+#define BIT_GET_HI1Q_HOST_IDX_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_HI1Q_HOST_IDX_8822B) & BIT_MASK_HI1Q_HOST_IDX_8822B)
+
+/* 2 REG_HI2Q_TXBD_IDX_8822B */
+
+#define BIT_SHIFT_HI2Q_HW_IDX_8822B 16
+#define BIT_MASK_HI2Q_HW_IDX_8822B 0xfff
+#define BIT_HI2Q_HW_IDX_8822B(x)                                               \
+	(((x) & BIT_MASK_HI2Q_HW_IDX_8822B) << BIT_SHIFT_HI2Q_HW_IDX_8822B)
+#define BIT_GET_HI2Q_HW_IDX_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_HI2Q_HW_IDX_8822B) & BIT_MASK_HI2Q_HW_IDX_8822B)
+
+#define BIT_SHIFT_HI2Q_HOST_IDX_8822B 0
+#define BIT_MASK_HI2Q_HOST_IDX_8822B 0xfff
+#define BIT_HI2Q_HOST_IDX_8822B(x)                                             \
+	(((x) & BIT_MASK_HI2Q_HOST_IDX_8822B) << BIT_SHIFT_HI2Q_HOST_IDX_8822B)
+#define BIT_GET_HI2Q_HOST_IDX_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_HI2Q_HOST_IDX_8822B) & BIT_MASK_HI2Q_HOST_IDX_8822B)
+
+/* 2 REG_HI3Q_TXBD_IDX_8822B */
+
+#define BIT_SHIFT_HI3Q_HW_IDX_8822B 16
+#define BIT_MASK_HI3Q_HW_IDX_8822B 0xfff
+#define BIT_HI3Q_HW_IDX_8822B(x)                                               \
+	(((x) & BIT_MASK_HI3Q_HW_IDX_8822B) << BIT_SHIFT_HI3Q_HW_IDX_8822B)
+#define BIT_GET_HI3Q_HW_IDX_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_HI3Q_HW_IDX_8822B) & BIT_MASK_HI3Q_HW_IDX_8822B)
+
+#define BIT_SHIFT_HI3Q_HOST_IDX_8822B 0
+#define BIT_MASK_HI3Q_HOST_IDX_8822B 0xfff
+#define BIT_HI3Q_HOST_IDX_8822B(x)                                             \
+	(((x) & BIT_MASK_HI3Q_HOST_IDX_8822B) << BIT_SHIFT_HI3Q_HOST_IDX_8822B)
+#define BIT_GET_HI3Q_HOST_IDX_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_HI3Q_HOST_IDX_8822B) & BIT_MASK_HI3Q_HOST_IDX_8822B)
+
+/* 2 REG_HI4Q_TXBD_IDX_8822B */
+
+#define BIT_SHIFT_HI4Q_HW_IDX_8822B 16
+#define BIT_MASK_HI4Q_HW_IDX_8822B 0xfff
+#define BIT_HI4Q_HW_IDX_8822B(x)                                               \
+	(((x) & BIT_MASK_HI4Q_HW_IDX_8822B) << BIT_SHIFT_HI4Q_HW_IDX_8822B)
+#define BIT_GET_HI4Q_HW_IDX_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_HI4Q_HW_IDX_8822B) & BIT_MASK_HI4Q_HW_IDX_8822B)
+
+#define BIT_SHIFT_HI4Q_HOST_IDX_8822B 0
+#define BIT_MASK_HI4Q_HOST_IDX_8822B 0xfff
+#define BIT_HI4Q_HOST_IDX_8822B(x)                                             \
+	(((x) & BIT_MASK_HI4Q_HOST_IDX_8822B) << BIT_SHIFT_HI4Q_HOST_IDX_8822B)
+#define BIT_GET_HI4Q_HOST_IDX_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_HI4Q_HOST_IDX_8822B) & BIT_MASK_HI4Q_HOST_IDX_8822B)
+
+/* 2 REG_HI5Q_TXBD_IDX_8822B */
+
+#define BIT_SHIFT_HI5Q_HW_IDX_8822B 16
+#define BIT_MASK_HI5Q_HW_IDX_8822B 0xfff
+#define BIT_HI5Q_HW_IDX_8822B(x)                                               \
+	(((x) & BIT_MASK_HI5Q_HW_IDX_8822B) << BIT_SHIFT_HI5Q_HW_IDX_8822B)
+#define BIT_GET_HI5Q_HW_IDX_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_HI5Q_HW_IDX_8822B) & BIT_MASK_HI5Q_HW_IDX_8822B)
+
+#define BIT_SHIFT_HI5Q_HOST_IDX_8822B 0
+#define BIT_MASK_HI5Q_HOST_IDX_8822B 0xfff
+#define BIT_HI5Q_HOST_IDX_8822B(x)                                             \
+	(((x) & BIT_MASK_HI5Q_HOST_IDX_8822B) << BIT_SHIFT_HI5Q_HOST_IDX_8822B)
+#define BIT_GET_HI5Q_HOST_IDX_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_HI5Q_HOST_IDX_8822B) & BIT_MASK_HI5Q_HOST_IDX_8822B)
+
+/* 2 REG_HI6Q_TXBD_IDX_8822B */
+
+#define BIT_SHIFT_HI6Q_HW_IDX_8822B 16
+#define BIT_MASK_HI6Q_HW_IDX_8822B 0xfff
+#define BIT_HI6Q_HW_IDX_8822B(x)                                               \
+	(((x) & BIT_MASK_HI6Q_HW_IDX_8822B) << BIT_SHIFT_HI6Q_HW_IDX_8822B)
+#define BIT_GET_HI6Q_HW_IDX_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_HI6Q_HW_IDX_8822B) & BIT_MASK_HI6Q_HW_IDX_8822B)
+
+#define BIT_SHIFT_HI6Q_HOST_IDX_8822B 0
+#define BIT_MASK_HI6Q_HOST_IDX_8822B 0xfff
+#define BIT_HI6Q_HOST_IDX_8822B(x)                                             \
+	(((x) & BIT_MASK_HI6Q_HOST_IDX_8822B) << BIT_SHIFT_HI6Q_HOST_IDX_8822B)
+#define BIT_GET_HI6Q_HOST_IDX_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_HI6Q_HOST_IDX_8822B) & BIT_MASK_HI6Q_HOST_IDX_8822B)
+
+/* 2 REG_HI7Q_TXBD_IDX_8822B */
+
+#define BIT_SHIFT_HI7Q_HW_IDX_8822B 16
+#define BIT_MASK_HI7Q_HW_IDX_8822B 0xfff
+#define BIT_HI7Q_HW_IDX_8822B(x)                                               \
+	(((x) & BIT_MASK_HI7Q_HW_IDX_8822B) << BIT_SHIFT_HI7Q_HW_IDX_8822B)
+#define BIT_GET_HI7Q_HW_IDX_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_HI7Q_HW_IDX_8822B) & BIT_MASK_HI7Q_HW_IDX_8822B)
+
+#define BIT_SHIFT_HI7Q_HOST_IDX_8822B 0
+#define BIT_MASK_HI7Q_HOST_IDX_8822B 0xfff
+#define BIT_HI7Q_HOST_IDX_8822B(x)                                             \
+	(((x) & BIT_MASK_HI7Q_HOST_IDX_8822B) << BIT_SHIFT_HI7Q_HOST_IDX_8822B)
+#define BIT_GET_HI7Q_HOST_IDX_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_HI7Q_HOST_IDX_8822B) & BIT_MASK_HI7Q_HOST_IDX_8822B)
+
+/* 2 REG_DBG_SEL_V1_8822B */
+
+#define BIT_SHIFT_DBG_SEL_8822B 0
+#define BIT_MASK_DBG_SEL_8822B 0xff
+#define BIT_DBG_SEL_8822B(x)                                                   \
+	(((x) & BIT_MASK_DBG_SEL_8822B) << BIT_SHIFT_DBG_SEL_8822B)
+#define BIT_GET_DBG_SEL_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_DBG_SEL_8822B) & BIT_MASK_DBG_SEL_8822B)
+
+/* 2 REG_PCIE_HRPWM1_V1_8822B */
+
+#define BIT_SHIFT_PCIE_HRPWM_8822B 0
+#define BIT_MASK_PCIE_HRPWM_8822B 0xff
+#define BIT_PCIE_HRPWM_8822B(x)                                                \
+	(((x) & BIT_MASK_PCIE_HRPWM_8822B) << BIT_SHIFT_PCIE_HRPWM_8822B)
+#define BIT_GET_PCIE_HRPWM_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_PCIE_HRPWM_8822B) & BIT_MASK_PCIE_HRPWM_8822B)
+
+/* 2 REG_PCIE_HCPWM1_V1_8822B */
+
+#define BIT_SHIFT_PCIE_HCPWM_8822B 0
+#define BIT_MASK_PCIE_HCPWM_8822B 0xff
+#define BIT_PCIE_HCPWM_8822B(x)                                                \
+	(((x) & BIT_MASK_PCIE_HCPWM_8822B) << BIT_SHIFT_PCIE_HCPWM_8822B)
+#define BIT_GET_PCIE_HCPWM_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_PCIE_HCPWM_8822B) & BIT_MASK_PCIE_HCPWM_8822B)
+
+/* 2 REG_PCIE_CTRL2_8822B */
+#define BIT_DIS_TXDMA_PRE_8822B BIT(7)
+#define BIT_DIS_RXDMA_PRE_8822B BIT(6)
+
+#define BIT_SHIFT_HPS_CLKR_PCIE_8822B 4
+#define BIT_MASK_HPS_CLKR_PCIE_8822B 0x3
+#define BIT_HPS_CLKR_PCIE_8822B(x)                                             \
+	(((x) & BIT_MASK_HPS_CLKR_PCIE_8822B) << BIT_SHIFT_HPS_CLKR_PCIE_8822B)
+#define BIT_GET_HPS_CLKR_PCIE_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_HPS_CLKR_PCIE_8822B) & BIT_MASK_HPS_CLKR_PCIE_8822B)
+
+#define BIT_PCIE_INT_8822B BIT(3)
+#define BIT_TXFLAG_EXIT_L1_EN_8822B BIT(2)
+#define BIT_EN_RXDMA_ALIGN_8822B BIT(1)
+#define BIT_EN_TXDMA_ALIGN_8822B BIT(0)
+
+/* 2 REG_PCIE_HRPWM2_V1_8822B */
+
+#define BIT_SHIFT_PCIE_HRPWM2_8822B 0
+#define BIT_MASK_PCIE_HRPWM2_8822B 0xffff
+#define BIT_PCIE_HRPWM2_8822B(x)                                               \
+	(((x) & BIT_MASK_PCIE_HRPWM2_8822B) << BIT_SHIFT_PCIE_HRPWM2_8822B)
+#define BIT_GET_PCIE_HRPWM2_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_PCIE_HRPWM2_8822B) & BIT_MASK_PCIE_HRPWM2_8822B)
+
+/* 2 REG_PCIE_HCPWM2_V1_8822B */
+
+#define BIT_SHIFT_PCIE_HCPWM2_8822B 0
+#define BIT_MASK_PCIE_HCPWM2_8822B 0xffff
+#define BIT_PCIE_HCPWM2_8822B(x)                                               \
+	(((x) & BIT_MASK_PCIE_HCPWM2_8822B) << BIT_SHIFT_PCIE_HCPWM2_8822B)
+#define BIT_GET_PCIE_HCPWM2_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_PCIE_HCPWM2_8822B) & BIT_MASK_PCIE_HCPWM2_8822B)
+
+/* 2 REG_PCIE_H2C_MSG_V1_8822B */
+
+#define BIT_SHIFT_DRV2FW_INFO_8822B 0
+#define BIT_MASK_DRV2FW_INFO_8822B 0xffffffffL
+#define BIT_DRV2FW_INFO_8822B(x)                                               \
+	(((x) & BIT_MASK_DRV2FW_INFO_8822B) << BIT_SHIFT_DRV2FW_INFO_8822B)
+#define BIT_GET_DRV2FW_INFO_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_DRV2FW_INFO_8822B) & BIT_MASK_DRV2FW_INFO_8822B)
+
+/* 2 REG_PCIE_C2H_MSG_V1_8822B */
+
+#define BIT_SHIFT_HCI_PCIE_C2H_MSG_8822B 0
+#define BIT_MASK_HCI_PCIE_C2H_MSG_8822B 0xffffffffL
+#define BIT_HCI_PCIE_C2H_MSG_8822B(x)                                          \
+	(((x) & BIT_MASK_HCI_PCIE_C2H_MSG_8822B)                               \
+	 << BIT_SHIFT_HCI_PCIE_C2H_MSG_8822B)
+#define BIT_GET_HCI_PCIE_C2H_MSG_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_HCI_PCIE_C2H_MSG_8822B) &                           \
+	 BIT_MASK_HCI_PCIE_C2H_MSG_8822B)
+
+/* 2 REG_DBI_WDATA_V1_8822B */
+
+#define BIT_SHIFT_DBI_WDATA_8822B 0
+#define BIT_MASK_DBI_WDATA_8822B 0xffffffffL
+#define BIT_DBI_WDATA_8822B(x)                                                 \
+	(((x) & BIT_MASK_DBI_WDATA_8822B) << BIT_SHIFT_DBI_WDATA_8822B)
+#define BIT_GET_DBI_WDATA_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_DBI_WDATA_8822B) & BIT_MASK_DBI_WDATA_8822B)
+
+/* 2 REG_DBI_RDATA_V1_8822B */
+
+#define BIT_SHIFT_DBI_RDATA_8822B 0
+#define BIT_MASK_DBI_RDATA_8822B 0xffffffffL
+#define BIT_DBI_RDATA_8822B(x)                                                 \
+	(((x) & BIT_MASK_DBI_RDATA_8822B) << BIT_SHIFT_DBI_RDATA_8822B)
+#define BIT_GET_DBI_RDATA_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_DBI_RDATA_8822B) & BIT_MASK_DBI_RDATA_8822B)
+
+/* 2 REG_DBI_FLAG_V1_8822B */
+#define BIT_EN_STUCK_DBG_8822B BIT(26)
+#define BIT_RX_STUCK_8822B BIT(25)
+#define BIT_TX_STUCK_8822B BIT(24)
+#define BIT_DBI_RFLAG_8822B BIT(17)
+#define BIT_DBI_WFLAG_8822B BIT(16)
+
+#define BIT_SHIFT_DBI_WREN_8822B 12
+#define BIT_MASK_DBI_WREN_8822B 0xf
+#define BIT_DBI_WREN_8822B(x)                                                  \
+	(((x) & BIT_MASK_DBI_WREN_8822B) << BIT_SHIFT_DBI_WREN_8822B)
+#define BIT_GET_DBI_WREN_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_DBI_WREN_8822B) & BIT_MASK_DBI_WREN_8822B)
+
+#define BIT_SHIFT_DBI_ADDR_8822B 0
+#define BIT_MASK_DBI_ADDR_8822B 0xfff
+#define BIT_DBI_ADDR_8822B(x)                                                  \
+	(((x) & BIT_MASK_DBI_ADDR_8822B) << BIT_SHIFT_DBI_ADDR_8822B)
+#define BIT_GET_DBI_ADDR_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_DBI_ADDR_8822B) & BIT_MASK_DBI_ADDR_8822B)
+
+/* 2 REG_MDIO_V1_8822B */
+
+#define BIT_SHIFT_MDIO_RDATA_8822B 16
+#define BIT_MASK_MDIO_RDATA_8822B 0xffff
+#define BIT_MDIO_RDATA_8822B(x)                                                \
+	(((x) & BIT_MASK_MDIO_RDATA_8822B) << BIT_SHIFT_MDIO_RDATA_8822B)
+#define BIT_GET_MDIO_RDATA_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_MDIO_RDATA_8822B) & BIT_MASK_MDIO_RDATA_8822B)
+
+#define BIT_SHIFT_MDIO_WDATA_8822B 0
+#define BIT_MASK_MDIO_WDATA_8822B 0xffff
+#define BIT_MDIO_WDATA_8822B(x)                                                \
+	(((x) & BIT_MASK_MDIO_WDATA_8822B) << BIT_SHIFT_MDIO_WDATA_8822B)
+#define BIT_GET_MDIO_WDATA_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_MDIO_WDATA_8822B) & BIT_MASK_MDIO_WDATA_8822B)
+
+/* 2 REG_PCIE_MIX_CFG_8822B */
+
+#define BIT_SHIFT_MDIO_PHY_ADDR_8822B 24
+#define BIT_MASK_MDIO_PHY_ADDR_8822B 0x1f
+#define BIT_MDIO_PHY_ADDR_8822B(x)                                             \
+	(((x) & BIT_MASK_MDIO_PHY_ADDR_8822B) << BIT_SHIFT_MDIO_PHY_ADDR_8822B)
+#define BIT_GET_MDIO_PHY_ADDR_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_MDIO_PHY_ADDR_8822B) & BIT_MASK_MDIO_PHY_ADDR_8822B)
+
+#define BIT_SHIFT_WATCH_DOG_RECORD_V1_8822B 10
+#define BIT_MASK_WATCH_DOG_RECORD_V1_8822B 0x3fff
+#define BIT_WATCH_DOG_RECORD_V1_8822B(x)                                       \
+	(((x) & BIT_MASK_WATCH_DOG_RECORD_V1_8822B)                            \
+	 << BIT_SHIFT_WATCH_DOG_RECORD_V1_8822B)
+#define BIT_GET_WATCH_DOG_RECORD_V1_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_WATCH_DOG_RECORD_V1_8822B) &                        \
+	 BIT_MASK_WATCH_DOG_RECORD_V1_8822B)
+
+#define BIT_R_IO_TIMEOUT_FLAG_V1_8822B BIT(9)
+#define BIT_EN_WATCH_DOG_8822B BIT(8)
+#define BIT_ECRC_EN_V1_8822B BIT(7)
+#define BIT_MDIO_RFLAG_V1_8822B BIT(6)
+#define BIT_MDIO_WFLAG_V1_8822B BIT(5)
+
+#define BIT_SHIFT_MDIO_REG_ADDR_V1_8822B 0
+#define BIT_MASK_MDIO_REG_ADDR_V1_8822B 0x1f
+#define BIT_MDIO_REG_ADDR_V1_8822B(x)                                          \
+	(((x) & BIT_MASK_MDIO_REG_ADDR_V1_8822B)                               \
+	 << BIT_SHIFT_MDIO_REG_ADDR_V1_8822B)
+#define BIT_GET_MDIO_REG_ADDR_V1_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_MDIO_REG_ADDR_V1_8822B) &                           \
+	 BIT_MASK_MDIO_REG_ADDR_V1_8822B)
+
+/* 2 REG_HCI_MIX_CFG_8822B */
+#define BIT_HOST_GEN2_SUPPORT_8822B BIT(20)
+
+#define BIT_SHIFT_TXDMA_ERR_FLAG_8822B 16
+#define BIT_MASK_TXDMA_ERR_FLAG_8822B 0xf
+#define BIT_TXDMA_ERR_FLAG_8822B(x)                                            \
+	(((x) & BIT_MASK_TXDMA_ERR_FLAG_8822B)                                 \
+	 << BIT_SHIFT_TXDMA_ERR_FLAG_8822B)
+#define BIT_GET_TXDMA_ERR_FLAG_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_TXDMA_ERR_FLAG_8822B) &                             \
+	 BIT_MASK_TXDMA_ERR_FLAG_8822B)
+
+#define BIT_SHIFT_EARLY_MODE_SEL_8822B 12
+#define BIT_MASK_EARLY_MODE_SEL_8822B 0xf
+#define BIT_EARLY_MODE_SEL_8822B(x)                                            \
+	(((x) & BIT_MASK_EARLY_MODE_SEL_8822B)                                 \
+	 << BIT_SHIFT_EARLY_MODE_SEL_8822B)
+#define BIT_GET_EARLY_MODE_SEL_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_EARLY_MODE_SEL_8822B) &                             \
+	 BIT_MASK_EARLY_MODE_SEL_8822B)
+
+#define BIT_EPHY_RX50_EN_8822B BIT(11)
+
+#define BIT_SHIFT_MSI_TIMEOUT_ID_V1_8822B 8
+#define BIT_MASK_MSI_TIMEOUT_ID_V1_8822B 0x7
+#define BIT_MSI_TIMEOUT_ID_V1_8822B(x)                                         \
+	(((x) & BIT_MASK_MSI_TIMEOUT_ID_V1_8822B)                              \
+	 << BIT_SHIFT_MSI_TIMEOUT_ID_V1_8822B)
+#define BIT_GET_MSI_TIMEOUT_ID_V1_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_MSI_TIMEOUT_ID_V1_8822B) &                          \
+	 BIT_MASK_MSI_TIMEOUT_ID_V1_8822B)
+
+#define BIT_RADDR_RD_8822B BIT(7)
+#define BIT_EN_MUL_TAG_8822B BIT(6)
+#define BIT_EN_EARLY_MODE_8822B BIT(5)
+#define BIT_L0S_LINK_OFF_8822B BIT(4)
+#define BIT_ACT_LINK_OFF_8822B BIT(3)
+#define BIT_EN_SLOW_MAC_TX_8822B BIT(2)
+#define BIT_EN_SLOW_MAC_RX_8822B BIT(1)
+
+/* 2 REG_STC_INT_CS_8822B(PCIE STATE CHANGE INTERRUPT CONTROL AND STATUS) */
+#define BIT_STC_INT_EN_8822B BIT(31)
+
+#define BIT_SHIFT_STC_INT_FLAG_8822B 16
+#define BIT_MASK_STC_INT_FLAG_8822B 0xff
+#define BIT_STC_INT_FLAG_8822B(x)                                              \
+	(((x) & BIT_MASK_STC_INT_FLAG_8822B) << BIT_SHIFT_STC_INT_FLAG_8822B)
+#define BIT_GET_STC_INT_FLAG_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_STC_INT_FLAG_8822B) & BIT_MASK_STC_INT_FLAG_8822B)
+
+#define BIT_SHIFT_STC_INT_IDX_8822B 8
+#define BIT_MASK_STC_INT_IDX_8822B 0x7
+#define BIT_STC_INT_IDX_8822B(x)                                               \
+	(((x) & BIT_MASK_STC_INT_IDX_8822B) << BIT_SHIFT_STC_INT_IDX_8822B)
+#define BIT_GET_STC_INT_IDX_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_STC_INT_IDX_8822B) & BIT_MASK_STC_INT_IDX_8822B)
+
+#define BIT_SHIFT_STC_INT_REALTIME_CS_8822B 0
+#define BIT_MASK_STC_INT_REALTIME_CS_8822B 0x3f
+#define BIT_STC_INT_REALTIME_CS_8822B(x)                                       \
+	(((x) & BIT_MASK_STC_INT_REALTIME_CS_8822B)                            \
+	 << BIT_SHIFT_STC_INT_REALTIME_CS_8822B)
+#define BIT_GET_STC_INT_REALTIME_CS_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_STC_INT_REALTIME_CS_8822B) &                        \
+	 BIT_MASK_STC_INT_REALTIME_CS_8822B)
+
+/* 2 REG_ST_INT_CFG_8822B(PCIE STATE CHANGE INTERRUPT CONFIGURATION) */
+#define BIT_STC_INT_GRP_EN_8822B BIT(31)
+
+#define BIT_SHIFT_STC_INT_EXPECT_LS_8822B 8
+#define BIT_MASK_STC_INT_EXPECT_LS_8822B 0x3f
+#define BIT_STC_INT_EXPECT_LS_8822B(x)                                         \
+	(((x) & BIT_MASK_STC_INT_EXPECT_LS_8822B)                              \
+	 << BIT_SHIFT_STC_INT_EXPECT_LS_8822B)
+#define BIT_GET_STC_INT_EXPECT_LS_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_STC_INT_EXPECT_LS_8822B) &                          \
+	 BIT_MASK_STC_INT_EXPECT_LS_8822B)
+
+#define BIT_SHIFT_STC_INT_EXPECT_CS_8822B 0
+#define BIT_MASK_STC_INT_EXPECT_CS_8822B 0x3f
+#define BIT_STC_INT_EXPECT_CS_8822B(x)                                         \
+	(((x) & BIT_MASK_STC_INT_EXPECT_CS_8822B)                              \
+	 << BIT_SHIFT_STC_INT_EXPECT_CS_8822B)
+#define BIT_GET_STC_INT_EXPECT_CS_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_STC_INT_EXPECT_CS_8822B) &                          \
+	 BIT_MASK_STC_INT_EXPECT_CS_8822B)
+
+/* 2 REG_CMU_DLY_CTRL_8822B(PCIE PHY CLOCK MGT UNIT DELAY CONTROL ) */
+#define BIT_CMU_DLY_EN_8822B BIT(31)
+#define BIT_CMU_DLY_MODE_8822B BIT(30)
+
+#define BIT_SHIFT_CMU_DLY_PRE_DIV_8822B 0
+#define BIT_MASK_CMU_DLY_PRE_DIV_8822B 0xff
+#define BIT_CMU_DLY_PRE_DIV_8822B(x)                                           \
+	(((x) & BIT_MASK_CMU_DLY_PRE_DIV_8822B)                                \
+	 << BIT_SHIFT_CMU_DLY_PRE_DIV_8822B)
+#define BIT_GET_CMU_DLY_PRE_DIV_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_CMU_DLY_PRE_DIV_8822B) &                            \
+	 BIT_MASK_CMU_DLY_PRE_DIV_8822B)
+
+/* 2 REG_CMU_DLY_CFG_8822B(PCIE PHY CLOCK MGT UNIT DELAY CONFIGURATION ) */
+
+#define BIT_SHIFT_CMU_DLY_LTR_A2I_8822B 24
+#define BIT_MASK_CMU_DLY_LTR_A2I_8822B 0xff
+#define BIT_CMU_DLY_LTR_A2I_8822B(x)                                           \
+	(((x) & BIT_MASK_CMU_DLY_LTR_A2I_8822B)                                \
+	 << BIT_SHIFT_CMU_DLY_LTR_A2I_8822B)
+#define BIT_GET_CMU_DLY_LTR_A2I_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_CMU_DLY_LTR_A2I_8822B) &                            \
+	 BIT_MASK_CMU_DLY_LTR_A2I_8822B)
+
+#define BIT_SHIFT_CMU_DLY_LTR_I2A_8822B 16
+#define BIT_MASK_CMU_DLY_LTR_I2A_8822B 0xff
+#define BIT_CMU_DLY_LTR_I2A_8822B(x)                                           \
+	(((x) & BIT_MASK_CMU_DLY_LTR_I2A_8822B)                                \
+	 << BIT_SHIFT_CMU_DLY_LTR_I2A_8822B)
+#define BIT_GET_CMU_DLY_LTR_I2A_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_CMU_DLY_LTR_I2A_8822B) &                            \
+	 BIT_MASK_CMU_DLY_LTR_I2A_8822B)
+
+#define BIT_SHIFT_CMU_DLY_LTR_IDLE_8822B 8
+#define BIT_MASK_CMU_DLY_LTR_IDLE_8822B 0xff
+#define BIT_CMU_DLY_LTR_IDLE_8822B(x)                                          \
+	(((x) & BIT_MASK_CMU_DLY_LTR_IDLE_8822B)                               \
+	 << BIT_SHIFT_CMU_DLY_LTR_IDLE_8822B)
+#define BIT_GET_CMU_DLY_LTR_IDLE_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_CMU_DLY_LTR_IDLE_8822B) &                           \
+	 BIT_MASK_CMU_DLY_LTR_IDLE_8822B)
+
+#define BIT_SHIFT_CMU_DLY_LTR_ACT_8822B 0
+#define BIT_MASK_CMU_DLY_LTR_ACT_8822B 0xff
+#define BIT_CMU_DLY_LTR_ACT_8822B(x)                                           \
+	(((x) & BIT_MASK_CMU_DLY_LTR_ACT_8822B)                                \
+	 << BIT_SHIFT_CMU_DLY_LTR_ACT_8822B)
+#define BIT_GET_CMU_DLY_LTR_ACT_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_CMU_DLY_LTR_ACT_8822B) &                            \
+	 BIT_MASK_CMU_DLY_LTR_ACT_8822B)
+
+/* 2 REG_H2CQ_TXBD_DESA_8822B */
+
+#define BIT_SHIFT_H2CQ_TXBD_DESA_8822B 0
+#define BIT_MASK_H2CQ_TXBD_DESA_8822B 0xffffffffffffffffL
+#define BIT_H2CQ_TXBD_DESA_8822B(x)                                            \
+	(((x) & BIT_MASK_H2CQ_TXBD_DESA_8822B)                                 \
+	 << BIT_SHIFT_H2CQ_TXBD_DESA_8822B)
+#define BIT_GET_H2CQ_TXBD_DESA_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_H2CQ_TXBD_DESA_8822B) &                             \
+	 BIT_MASK_H2CQ_TXBD_DESA_8822B)
+
+/* 2 REG_H2CQ_TXBD_NUM_8822B */
+#define BIT_PCIE_H2CQ_FLAG_8822B BIT(14)
+
+#define BIT_SHIFT_H2CQ_DESC_MODE_8822B 12
+#define BIT_MASK_H2CQ_DESC_MODE_8822B 0x3
+#define BIT_H2CQ_DESC_MODE_8822B(x)                                            \
+	(((x) & BIT_MASK_H2CQ_DESC_MODE_8822B)                                 \
+	 << BIT_SHIFT_H2CQ_DESC_MODE_8822B)
+#define BIT_GET_H2CQ_DESC_MODE_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_H2CQ_DESC_MODE_8822B) &                             \
+	 BIT_MASK_H2CQ_DESC_MODE_8822B)
+
+#define BIT_SHIFT_H2CQ_DESC_NUM_8822B 0
+#define BIT_MASK_H2CQ_DESC_NUM_8822B 0xfff
+#define BIT_H2CQ_DESC_NUM_8822B(x)                                             \
+	(((x) & BIT_MASK_H2CQ_DESC_NUM_8822B) << BIT_SHIFT_H2CQ_DESC_NUM_8822B)
+#define BIT_GET_H2CQ_DESC_NUM_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_H2CQ_DESC_NUM_8822B) & BIT_MASK_H2CQ_DESC_NUM_8822B)
+
+/* 2 REG_H2CQ_TXBD_IDX_8822B */
+
+#define BIT_SHIFT_H2CQ_HW_IDX_8822B 16
+#define BIT_MASK_H2CQ_HW_IDX_8822B 0xfff
+#define BIT_H2CQ_HW_IDX_8822B(x)                                               \
+	(((x) & BIT_MASK_H2CQ_HW_IDX_8822B) << BIT_SHIFT_H2CQ_HW_IDX_8822B)
+#define BIT_GET_H2CQ_HW_IDX_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_H2CQ_HW_IDX_8822B) & BIT_MASK_H2CQ_HW_IDX_8822B)
+
+#define BIT_SHIFT_H2CQ_HOST_IDX_8822B 0
+#define BIT_MASK_H2CQ_HOST_IDX_8822B 0xfff
+#define BIT_H2CQ_HOST_IDX_8822B(x)                                             \
+	(((x) & BIT_MASK_H2CQ_HOST_IDX_8822B) << BIT_SHIFT_H2CQ_HOST_IDX_8822B)
+#define BIT_GET_H2CQ_HOST_IDX_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_H2CQ_HOST_IDX_8822B) & BIT_MASK_H2CQ_HOST_IDX_8822B)
+
+/* 2 REG_H2CQ_CSR_8822B[31:0] (H2CQ CONTROL AND STATUS) */
+#define BIT_H2CQ_FULL_8822B BIT(31)
+#define BIT_CLR_H2CQ_HOST_IDX_8822B BIT(16)
+#define BIT_CLR_H2CQ_HW_IDX_8822B BIT(8)
+
+/* 2 REG_CHANGE_PCIE_SPEED_8822B */
+#define BIT_CHANGE_PCIE_SPEED_8822B BIT(18)
+
+#define BIT_SHIFT_GEN1_GEN2_8822B 16
+#define BIT_MASK_GEN1_GEN2_8822B 0x3
+#define BIT_GEN1_GEN2_8822B(x)                                                 \
+	(((x) & BIT_MASK_GEN1_GEN2_8822B) << BIT_SHIFT_GEN1_GEN2_8822B)
+#define BIT_GET_GEN1_GEN2_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_GEN1_GEN2_8822B) & BIT_MASK_GEN1_GEN2_8822B)
+
+#define BIT_SHIFT_AUTO_HANG_RELEASE_8822B 0
+#define BIT_MASK_AUTO_HANG_RELEASE_8822B 0x7
+#define BIT_AUTO_HANG_RELEASE_8822B(x)                                         \
+	(((x) & BIT_MASK_AUTO_HANG_RELEASE_8822B)                              \
+	 << BIT_SHIFT_AUTO_HANG_RELEASE_8822B)
+#define BIT_GET_AUTO_HANG_RELEASE_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_AUTO_HANG_RELEASE_8822B) &                          \
+	 BIT_MASK_AUTO_HANG_RELEASE_8822B)
+
+/* 2 REG_OLD_DEHANG_8822B */
+#define BIT_OLD_DEHANG_8822B BIT(1)
+
+/* 2 REG_Q0_INFO_8822B */
+
+#define BIT_SHIFT_QUEUEMACID_Q0_V1_8822B 25
+#define BIT_MASK_QUEUEMACID_Q0_V1_8822B 0x7f
+#define BIT_QUEUEMACID_Q0_V1_8822B(x)                                          \
+	(((x) & BIT_MASK_QUEUEMACID_Q0_V1_8822B)                               \
+	 << BIT_SHIFT_QUEUEMACID_Q0_V1_8822B)
+#define BIT_GET_QUEUEMACID_Q0_V1_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_QUEUEMACID_Q0_V1_8822B) &                           \
+	 BIT_MASK_QUEUEMACID_Q0_V1_8822B)
+
+#define BIT_SHIFT_QUEUEAC_Q0_V1_8822B 23
+#define BIT_MASK_QUEUEAC_Q0_V1_8822B 0x3
+#define BIT_QUEUEAC_Q0_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_QUEUEAC_Q0_V1_8822B) << BIT_SHIFT_QUEUEAC_Q0_V1_8822B)
+#define BIT_GET_QUEUEAC_Q0_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_QUEUEAC_Q0_V1_8822B) & BIT_MASK_QUEUEAC_Q0_V1_8822B)
+
+#define BIT_TIDEMPTY_Q0_V1_8822B BIT(22)
+
+#define BIT_SHIFT_TAIL_PKT_Q0_V2_8822B 11
+#define BIT_MASK_TAIL_PKT_Q0_V2_8822B 0x7ff
+#define BIT_TAIL_PKT_Q0_V2_8822B(x)                                            \
+	(((x) & BIT_MASK_TAIL_PKT_Q0_V2_8822B)                                 \
+	 << BIT_SHIFT_TAIL_PKT_Q0_V2_8822B)
+#define BIT_GET_TAIL_PKT_Q0_V2_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_TAIL_PKT_Q0_V2_8822B) &                             \
+	 BIT_MASK_TAIL_PKT_Q0_V2_8822B)
+
+#define BIT_SHIFT_HEAD_PKT_Q0_V1_8822B 0
+#define BIT_MASK_HEAD_PKT_Q0_V1_8822B 0x7ff
+#define BIT_HEAD_PKT_Q0_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_HEAD_PKT_Q0_V1_8822B)                                 \
+	 << BIT_SHIFT_HEAD_PKT_Q0_V1_8822B)
+#define BIT_GET_HEAD_PKT_Q0_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HEAD_PKT_Q0_V1_8822B) &                             \
+	 BIT_MASK_HEAD_PKT_Q0_V1_8822B)
+
+/* 2 REG_Q1_INFO_8822B */
+
+#define BIT_SHIFT_QUEUEMACID_Q1_V1_8822B 25
+#define BIT_MASK_QUEUEMACID_Q1_V1_8822B 0x7f
+#define BIT_QUEUEMACID_Q1_V1_8822B(x)                                          \
+	(((x) & BIT_MASK_QUEUEMACID_Q1_V1_8822B)                               \
+	 << BIT_SHIFT_QUEUEMACID_Q1_V1_8822B)
+#define BIT_GET_QUEUEMACID_Q1_V1_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_QUEUEMACID_Q1_V1_8822B) &                           \
+	 BIT_MASK_QUEUEMACID_Q1_V1_8822B)
+
+#define BIT_SHIFT_QUEUEAC_Q1_V1_8822B 23
+#define BIT_MASK_QUEUEAC_Q1_V1_8822B 0x3
+#define BIT_QUEUEAC_Q1_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_QUEUEAC_Q1_V1_8822B) << BIT_SHIFT_QUEUEAC_Q1_V1_8822B)
+#define BIT_GET_QUEUEAC_Q1_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_QUEUEAC_Q1_V1_8822B) & BIT_MASK_QUEUEAC_Q1_V1_8822B)
+
+#define BIT_TIDEMPTY_Q1_V1_8822B BIT(22)
+
+#define BIT_SHIFT_TAIL_PKT_Q1_V2_8822B 11
+#define BIT_MASK_TAIL_PKT_Q1_V2_8822B 0x7ff
+#define BIT_TAIL_PKT_Q1_V2_8822B(x)                                            \
+	(((x) & BIT_MASK_TAIL_PKT_Q1_V2_8822B)                                 \
+	 << BIT_SHIFT_TAIL_PKT_Q1_V2_8822B)
+#define BIT_GET_TAIL_PKT_Q1_V2_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_TAIL_PKT_Q1_V2_8822B) &                             \
+	 BIT_MASK_TAIL_PKT_Q1_V2_8822B)
+
+#define BIT_SHIFT_HEAD_PKT_Q1_V1_8822B 0
+#define BIT_MASK_HEAD_PKT_Q1_V1_8822B 0x7ff
+#define BIT_HEAD_PKT_Q1_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_HEAD_PKT_Q1_V1_8822B)                                 \
+	 << BIT_SHIFT_HEAD_PKT_Q1_V1_8822B)
+#define BIT_GET_HEAD_PKT_Q1_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HEAD_PKT_Q1_V1_8822B) &                             \
+	 BIT_MASK_HEAD_PKT_Q1_V1_8822B)
+
+/* 2 REG_Q2_INFO_8822B */
+
+#define BIT_SHIFT_QUEUEMACID_Q2_V1_8822B 25
+#define BIT_MASK_QUEUEMACID_Q2_V1_8822B 0x7f
+#define BIT_QUEUEMACID_Q2_V1_8822B(x)                                          \
+	(((x) & BIT_MASK_QUEUEMACID_Q2_V1_8822B)                               \
+	 << BIT_SHIFT_QUEUEMACID_Q2_V1_8822B)
+#define BIT_GET_QUEUEMACID_Q2_V1_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_QUEUEMACID_Q2_V1_8822B) &                           \
+	 BIT_MASK_QUEUEMACID_Q2_V1_8822B)
+
+#define BIT_SHIFT_QUEUEAC_Q2_V1_8822B 23
+#define BIT_MASK_QUEUEAC_Q2_V1_8822B 0x3
+#define BIT_QUEUEAC_Q2_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_QUEUEAC_Q2_V1_8822B) << BIT_SHIFT_QUEUEAC_Q2_V1_8822B)
+#define BIT_GET_QUEUEAC_Q2_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_QUEUEAC_Q2_V1_8822B) & BIT_MASK_QUEUEAC_Q2_V1_8822B)
+
+#define BIT_TIDEMPTY_Q2_V1_8822B BIT(22)
+
+#define BIT_SHIFT_TAIL_PKT_Q2_V2_8822B 11
+#define BIT_MASK_TAIL_PKT_Q2_V2_8822B 0x7ff
+#define BIT_TAIL_PKT_Q2_V2_8822B(x)                                            \
+	(((x) & BIT_MASK_TAIL_PKT_Q2_V2_8822B)                                 \
+	 << BIT_SHIFT_TAIL_PKT_Q2_V2_8822B)
+#define BIT_GET_TAIL_PKT_Q2_V2_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_TAIL_PKT_Q2_V2_8822B) &                             \
+	 BIT_MASK_TAIL_PKT_Q2_V2_8822B)
+
+#define BIT_SHIFT_HEAD_PKT_Q2_V1_8822B 0
+#define BIT_MASK_HEAD_PKT_Q2_V1_8822B 0x7ff
+#define BIT_HEAD_PKT_Q2_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_HEAD_PKT_Q2_V1_8822B)                                 \
+	 << BIT_SHIFT_HEAD_PKT_Q2_V1_8822B)
+#define BIT_GET_HEAD_PKT_Q2_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HEAD_PKT_Q2_V1_8822B) &                             \
+	 BIT_MASK_HEAD_PKT_Q2_V1_8822B)
+
+/* 2 REG_Q3_INFO_8822B */
+
+#define BIT_SHIFT_QUEUEMACID_Q3_V1_8822B 25
+#define BIT_MASK_QUEUEMACID_Q3_V1_8822B 0x7f
+#define BIT_QUEUEMACID_Q3_V1_8822B(x)                                          \
+	(((x) & BIT_MASK_QUEUEMACID_Q3_V1_8822B)                               \
+	 << BIT_SHIFT_QUEUEMACID_Q3_V1_8822B)
+#define BIT_GET_QUEUEMACID_Q3_V1_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_QUEUEMACID_Q3_V1_8822B) &                           \
+	 BIT_MASK_QUEUEMACID_Q3_V1_8822B)
+
+#define BIT_SHIFT_QUEUEAC_Q3_V1_8822B 23
+#define BIT_MASK_QUEUEAC_Q3_V1_8822B 0x3
+#define BIT_QUEUEAC_Q3_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_QUEUEAC_Q3_V1_8822B) << BIT_SHIFT_QUEUEAC_Q3_V1_8822B)
+#define BIT_GET_QUEUEAC_Q3_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_QUEUEAC_Q3_V1_8822B) & BIT_MASK_QUEUEAC_Q3_V1_8822B)
+
+#define BIT_TIDEMPTY_Q3_V1_8822B BIT(22)
+
+#define BIT_SHIFT_TAIL_PKT_Q3_V2_8822B 11
+#define BIT_MASK_TAIL_PKT_Q3_V2_8822B 0x7ff
+#define BIT_TAIL_PKT_Q3_V2_8822B(x)                                            \
+	(((x) & BIT_MASK_TAIL_PKT_Q3_V2_8822B)                                 \
+	 << BIT_SHIFT_TAIL_PKT_Q3_V2_8822B)
+#define BIT_GET_TAIL_PKT_Q3_V2_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_TAIL_PKT_Q3_V2_8822B) &                             \
+	 BIT_MASK_TAIL_PKT_Q3_V2_8822B)
+
+#define BIT_SHIFT_HEAD_PKT_Q3_V1_8822B 0
+#define BIT_MASK_HEAD_PKT_Q3_V1_8822B 0x7ff
+#define BIT_HEAD_PKT_Q3_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_HEAD_PKT_Q3_V1_8822B)                                 \
+	 << BIT_SHIFT_HEAD_PKT_Q3_V1_8822B)
+#define BIT_GET_HEAD_PKT_Q3_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HEAD_PKT_Q3_V1_8822B) &                             \
+	 BIT_MASK_HEAD_PKT_Q3_V1_8822B)
+
+/* 2 REG_MGQ_INFO_8822B */
+
+#define BIT_SHIFT_QUEUEMACID_MGQ_V1_8822B 25
+#define BIT_MASK_QUEUEMACID_MGQ_V1_8822B 0x7f
+#define BIT_QUEUEMACID_MGQ_V1_8822B(x)                                         \
+	(((x) & BIT_MASK_QUEUEMACID_MGQ_V1_8822B)                              \
+	 << BIT_SHIFT_QUEUEMACID_MGQ_V1_8822B)
+#define BIT_GET_QUEUEMACID_MGQ_V1_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_QUEUEMACID_MGQ_V1_8822B) &                          \
+	 BIT_MASK_QUEUEMACID_MGQ_V1_8822B)
+
+#define BIT_SHIFT_QUEUEAC_MGQ_V1_8822B 23
+#define BIT_MASK_QUEUEAC_MGQ_V1_8822B 0x3
+#define BIT_QUEUEAC_MGQ_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_QUEUEAC_MGQ_V1_8822B)                                 \
+	 << BIT_SHIFT_QUEUEAC_MGQ_V1_8822B)
+#define BIT_GET_QUEUEAC_MGQ_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_QUEUEAC_MGQ_V1_8822B) &                             \
+	 BIT_MASK_QUEUEAC_MGQ_V1_8822B)
+
+#define BIT_TIDEMPTY_MGQ_V1_8822B BIT(22)
+
+#define BIT_SHIFT_TAIL_PKT_MGQ_V2_8822B 11
+#define BIT_MASK_TAIL_PKT_MGQ_V2_8822B 0x7ff
+#define BIT_TAIL_PKT_MGQ_V2_8822B(x)                                           \
+	(((x) & BIT_MASK_TAIL_PKT_MGQ_V2_8822B)                                \
+	 << BIT_SHIFT_TAIL_PKT_MGQ_V2_8822B)
+#define BIT_GET_TAIL_PKT_MGQ_V2_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_TAIL_PKT_MGQ_V2_8822B) &                            \
+	 BIT_MASK_TAIL_PKT_MGQ_V2_8822B)
+
+#define BIT_SHIFT_HEAD_PKT_MGQ_V1_8822B 0
+#define BIT_MASK_HEAD_PKT_MGQ_V1_8822B 0x7ff
+#define BIT_HEAD_PKT_MGQ_V1_8822B(x)                                           \
+	(((x) & BIT_MASK_HEAD_PKT_MGQ_V1_8822B)                                \
+	 << BIT_SHIFT_HEAD_PKT_MGQ_V1_8822B)
+#define BIT_GET_HEAD_PKT_MGQ_V1_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_HEAD_PKT_MGQ_V1_8822B) &                            \
+	 BIT_MASK_HEAD_PKT_MGQ_V1_8822B)
+
+/* 2 REG_HIQ_INFO_8822B */
+
+#define BIT_SHIFT_QUEUEMACID_HIQ_V1_8822B 25
+#define BIT_MASK_QUEUEMACID_HIQ_V1_8822B 0x7f
+#define BIT_QUEUEMACID_HIQ_V1_8822B(x)                                         \
+	(((x) & BIT_MASK_QUEUEMACID_HIQ_V1_8822B)                              \
+	 << BIT_SHIFT_QUEUEMACID_HIQ_V1_8822B)
+#define BIT_GET_QUEUEMACID_HIQ_V1_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_QUEUEMACID_HIQ_V1_8822B) &                          \
+	 BIT_MASK_QUEUEMACID_HIQ_V1_8822B)
+
+#define BIT_SHIFT_QUEUEAC_HIQ_V1_8822B 23
+#define BIT_MASK_QUEUEAC_HIQ_V1_8822B 0x3
+#define BIT_QUEUEAC_HIQ_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_QUEUEAC_HIQ_V1_8822B)                                 \
+	 << BIT_SHIFT_QUEUEAC_HIQ_V1_8822B)
+#define BIT_GET_QUEUEAC_HIQ_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_QUEUEAC_HIQ_V1_8822B) &                             \
+	 BIT_MASK_QUEUEAC_HIQ_V1_8822B)
+
+#define BIT_TIDEMPTY_HIQ_V1_8822B BIT(22)
+
+#define BIT_SHIFT_TAIL_PKT_HIQ_V2_8822B 11
+#define BIT_MASK_TAIL_PKT_HIQ_V2_8822B 0x7ff
+#define BIT_TAIL_PKT_HIQ_V2_8822B(x)                                           \
+	(((x) & BIT_MASK_TAIL_PKT_HIQ_V2_8822B)                                \
+	 << BIT_SHIFT_TAIL_PKT_HIQ_V2_8822B)
+#define BIT_GET_TAIL_PKT_HIQ_V2_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_TAIL_PKT_HIQ_V2_8822B) &                            \
+	 BIT_MASK_TAIL_PKT_HIQ_V2_8822B)
+
+#define BIT_SHIFT_HEAD_PKT_HIQ_V1_8822B 0
+#define BIT_MASK_HEAD_PKT_HIQ_V1_8822B 0x7ff
+#define BIT_HEAD_PKT_HIQ_V1_8822B(x)                                           \
+	(((x) & BIT_MASK_HEAD_PKT_HIQ_V1_8822B)                                \
+	 << BIT_SHIFT_HEAD_PKT_HIQ_V1_8822B)
+#define BIT_GET_HEAD_PKT_HIQ_V1_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_HEAD_PKT_HIQ_V1_8822B) &                            \
+	 BIT_MASK_HEAD_PKT_HIQ_V1_8822B)
+
+/* 2 REG_BCNQ_INFO_8822B */
+
+#define BIT_SHIFT_BCNQ_HEAD_PG_V1_8822B 0
+#define BIT_MASK_BCNQ_HEAD_PG_V1_8822B 0xfff
+#define BIT_BCNQ_HEAD_PG_V1_8822B(x)                                           \
+	(((x) & BIT_MASK_BCNQ_HEAD_PG_V1_8822B)                                \
+	 << BIT_SHIFT_BCNQ_HEAD_PG_V1_8822B)
+#define BIT_GET_BCNQ_HEAD_PG_V1_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_BCNQ_HEAD_PG_V1_8822B) &                            \
+	 BIT_MASK_BCNQ_HEAD_PG_V1_8822B)
+
+/* 2 REG_TXPKT_EMPTY_8822B */
+#define BIT_BCNQ_EMPTY_8822B BIT(11)
+#define BIT_HQQ_EMPTY_8822B BIT(10)
+#define BIT_MQQ_EMPTY_8822B BIT(9)
+#define BIT_MGQ_CPU_EMPTY_8822B BIT(8)
+#define BIT_AC7Q_EMPTY_8822B BIT(7)
+#define BIT_AC6Q_EMPTY_8822B BIT(6)
+#define BIT_AC5Q_EMPTY_8822B BIT(5)
+#define BIT_AC4Q_EMPTY_8822B BIT(4)
+#define BIT_AC3Q_EMPTY_8822B BIT(3)
+#define BIT_AC2Q_EMPTY_8822B BIT(2)
+#define BIT_AC1Q_EMPTY_8822B BIT(1)
+#define BIT_AC0Q_EMPTY_8822B BIT(0)
+
+/* 2 REG_CPU_MGQ_INFO_8822B */
+#define BIT_BCN1_POLL_8822B BIT(30)
+#define BIT_CPUMGT_POLL_8822B BIT(29)
+#define BIT_BCN_POLL_8822B BIT(28)
+#define BIT_CPUMGQ_FW_NUM_V1_8822B BIT(12)
+
+#define BIT_SHIFT_FW_FREE_TAIL_V1_8822B 0
+#define BIT_MASK_FW_FREE_TAIL_V1_8822B 0xfff
+#define BIT_FW_FREE_TAIL_V1_8822B(x)                                           \
+	(((x) & BIT_MASK_FW_FREE_TAIL_V1_8822B)                                \
+	 << BIT_SHIFT_FW_FREE_TAIL_V1_8822B)
+#define BIT_GET_FW_FREE_TAIL_V1_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_FW_FREE_TAIL_V1_8822B) &                            \
+	 BIT_MASK_FW_FREE_TAIL_V1_8822B)
+
+/* 2 REG_FWHW_TXQ_CTRL_8822B */
+#define BIT_RTS_LIMIT_IN_OFDM_8822B BIT(23)
+#define BIT_EN_BCNQ_DL_8822B BIT(22)
+#define BIT_EN_RD_RESP_NAV_BK_8822B BIT(21)
+#define BIT_EN_WR_FREE_TAIL_8822B BIT(20)
+
+#define BIT_SHIFT_EN_QUEUE_RPT_8822B 8
+#define BIT_MASK_EN_QUEUE_RPT_8822B 0xff
+#define BIT_EN_QUEUE_RPT_8822B(x)                                              \
+	(((x) & BIT_MASK_EN_QUEUE_RPT_8822B) << BIT_SHIFT_EN_QUEUE_RPT_8822B)
+#define BIT_GET_EN_QUEUE_RPT_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_EN_QUEUE_RPT_8822B) & BIT_MASK_EN_QUEUE_RPT_8822B)
+
+#define BIT_EN_RTY_BK_8822B BIT(7)
+#define BIT_EN_USE_INI_RAT_8822B BIT(6)
+#define BIT_EN_RTS_NAV_BK_8822B BIT(5)
+#define BIT_DIS_SSN_CHECK_8822B BIT(4)
+#define BIT_MACID_MATCH_RTS_8822B BIT(3)
+#define BIT_EN_BCN_TRXRPT_V1_8822B BIT(2)
+#define BIT_EN_FTMACKRPT_8822B BIT(1)
+#define BIT_EN_FTMRPT_8822B BIT(0)
+
+/* 2 REG_DATAFB_SEL_8822B */
+#define BIT__R_EN_RTY_BK_COD_8822B BIT(2)
+
+#define BIT_SHIFT__R_DATA_FALLBACK_SEL_8822B 0
+#define BIT_MASK__R_DATA_FALLBACK_SEL_8822B 0x3
+#define BIT__R_DATA_FALLBACK_SEL_8822B(x)                                      \
+	(((x) & BIT_MASK__R_DATA_FALLBACK_SEL_8822B)                           \
+	 << BIT_SHIFT__R_DATA_FALLBACK_SEL_8822B)
+#define BIT_GET__R_DATA_FALLBACK_SEL_8822B(x)                                  \
+	(((x) >> BIT_SHIFT__R_DATA_FALLBACK_SEL_8822B) &                       \
+	 BIT_MASK__R_DATA_FALLBACK_SEL_8822B)
+
+/* 2 REG_BCNQ_BDNY_V1_8822B */
+
+#define BIT_SHIFT_BCNQ_PGBNDY_V1_8822B 0
+#define BIT_MASK_BCNQ_PGBNDY_V1_8822B 0xfff
+#define BIT_BCNQ_PGBNDY_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_BCNQ_PGBNDY_V1_8822B)                                 \
+	 << BIT_SHIFT_BCNQ_PGBNDY_V1_8822B)
+#define BIT_GET_BCNQ_PGBNDY_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_BCNQ_PGBNDY_V1_8822B) &                             \
+	 BIT_MASK_BCNQ_PGBNDY_V1_8822B)
+
+/* 2 REG_LIFETIME_EN_8822B */
+#define BIT_BT_INT_CPU_8822B BIT(7)
+#define BIT_BT_INT_PTA_8822B BIT(6)
+#define BIT_EN_CTRL_RTYBIT_8822B BIT(4)
+#define BIT_LIFETIME_BK_EN_8822B BIT(3)
+#define BIT_LIFETIME_BE_EN_8822B BIT(2)
+#define BIT_LIFETIME_VI_EN_8822B BIT(1)
+#define BIT_LIFETIME_VO_EN_8822B BIT(0)
+
+/* 2 REG_SPEC_SIFS_8822B */
+
+#define BIT_SHIFT_SPEC_SIFS_OFDM_PTCL_8822B 8
+#define BIT_MASK_SPEC_SIFS_OFDM_PTCL_8822B 0xff
+#define BIT_SPEC_SIFS_OFDM_PTCL_8822B(x)                                       \
+	(((x) & BIT_MASK_SPEC_SIFS_OFDM_PTCL_8822B)                            \
+	 << BIT_SHIFT_SPEC_SIFS_OFDM_PTCL_8822B)
+#define BIT_GET_SPEC_SIFS_OFDM_PTCL_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_SPEC_SIFS_OFDM_PTCL_8822B) &                        \
+	 BIT_MASK_SPEC_SIFS_OFDM_PTCL_8822B)
+
+#define BIT_SHIFT_SPEC_SIFS_CCK_PTCL_8822B 0
+#define BIT_MASK_SPEC_SIFS_CCK_PTCL_8822B 0xff
+#define BIT_SPEC_SIFS_CCK_PTCL_8822B(x)                                        \
+	(((x) & BIT_MASK_SPEC_SIFS_CCK_PTCL_8822B)                             \
+	 << BIT_SHIFT_SPEC_SIFS_CCK_PTCL_8822B)
+#define BIT_GET_SPEC_SIFS_CCK_PTCL_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_SPEC_SIFS_CCK_PTCL_8822B) &                         \
+	 BIT_MASK_SPEC_SIFS_CCK_PTCL_8822B)
+
+/* 2 REG_RETRY_LIMIT_8822B */
+
+#define BIT_SHIFT_SRL_8822B 8
+#define BIT_MASK_SRL_8822B 0x3f
+#define BIT_SRL_8822B(x) (((x) & BIT_MASK_SRL_8822B) << BIT_SHIFT_SRL_8822B)
+#define BIT_GET_SRL_8822B(x) (((x) >> BIT_SHIFT_SRL_8822B) & BIT_MASK_SRL_8822B)
+
+#define BIT_SHIFT_LRL_8822B 0
+#define BIT_MASK_LRL_8822B 0x3f
+#define BIT_LRL_8822B(x) (((x) & BIT_MASK_LRL_8822B) << BIT_SHIFT_LRL_8822B)
+#define BIT_GET_LRL_8822B(x) (((x) >> BIT_SHIFT_LRL_8822B) & BIT_MASK_LRL_8822B)
+
+/* 2 REG_TXBF_CTRL_8822B */
+#define BIT_R_ENABLE_NDPA_8822B BIT(31)
+#define BIT_USE_NDPA_PARAMETER_8822B BIT(30)
+#define BIT_R_PROP_TXBF_8822B BIT(29)
+#define BIT_R_EN_NDPA_INT_8822B BIT(28)
+#define BIT_R_TXBF1_80M_8822B BIT(27)
+#define BIT_R_TXBF1_40M_8822B BIT(26)
+#define BIT_R_TXBF1_20M_8822B BIT(25)
+
+#define BIT_SHIFT_R_TXBF1_AID_8822B 16
+#define BIT_MASK_R_TXBF1_AID_8822B 0x1ff
+#define BIT_R_TXBF1_AID_8822B(x)                                               \
+	(((x) & BIT_MASK_R_TXBF1_AID_8822B) << BIT_SHIFT_R_TXBF1_AID_8822B)
+#define BIT_GET_R_TXBF1_AID_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_R_TXBF1_AID_8822B) & BIT_MASK_R_TXBF1_AID_8822B)
+
+#define BIT_DIS_NDP_BFEN_8822B BIT(15)
+#define BIT_R_TXBCN_NOBLOCK_NDP_8822B BIT(14)
+#define BIT_R_TXBF0_80M_8822B BIT(11)
+#define BIT_R_TXBF0_40M_8822B BIT(10)
+#define BIT_R_TXBF0_20M_8822B BIT(9)
+
+#define BIT_SHIFT_R_TXBF0_AID_8822B 0
+#define BIT_MASK_R_TXBF0_AID_8822B 0x1ff
+#define BIT_R_TXBF0_AID_8822B(x)                                               \
+	(((x) & BIT_MASK_R_TXBF0_AID_8822B) << BIT_SHIFT_R_TXBF0_AID_8822B)
+#define BIT_GET_R_TXBF0_AID_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_R_TXBF0_AID_8822B) & BIT_MASK_R_TXBF0_AID_8822B)
+
+/* 2 REG_DARFRC_8822B */
+
+#define BIT_SHIFT_DARF_RC8_8822B (56 & CPU_OPT_WIDTH)
+#define BIT_MASK_DARF_RC8_8822B 0x1f
+#define BIT_DARF_RC8_8822B(x)                                                  \
+	(((x) & BIT_MASK_DARF_RC8_8822B) << BIT_SHIFT_DARF_RC8_8822B)
+#define BIT_GET_DARF_RC8_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_DARF_RC8_8822B) & BIT_MASK_DARF_RC8_8822B)
+
+#define BIT_SHIFT_DARF_RC7_8822B (48 & CPU_OPT_WIDTH)
+#define BIT_MASK_DARF_RC7_8822B 0x1f
+#define BIT_DARF_RC7_8822B(x)                                                  \
+	(((x) & BIT_MASK_DARF_RC7_8822B) << BIT_SHIFT_DARF_RC7_8822B)
+#define BIT_GET_DARF_RC7_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_DARF_RC7_8822B) & BIT_MASK_DARF_RC7_8822B)
+
+#define BIT_SHIFT_DARF_RC6_8822B (40 & CPU_OPT_WIDTH)
+#define BIT_MASK_DARF_RC6_8822B 0x1f
+#define BIT_DARF_RC6_8822B(x)                                                  \
+	(((x) & BIT_MASK_DARF_RC6_8822B) << BIT_SHIFT_DARF_RC6_8822B)
+#define BIT_GET_DARF_RC6_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_DARF_RC6_8822B) & BIT_MASK_DARF_RC6_8822B)
+
+#define BIT_SHIFT_DARF_RC5_8822B (32 & CPU_OPT_WIDTH)
+#define BIT_MASK_DARF_RC5_8822B 0x1f
+#define BIT_DARF_RC5_8822B(x)                                                  \
+	(((x) & BIT_MASK_DARF_RC5_8822B) << BIT_SHIFT_DARF_RC5_8822B)
+#define BIT_GET_DARF_RC5_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_DARF_RC5_8822B) & BIT_MASK_DARF_RC5_8822B)
+
+#define BIT_SHIFT_DARF_RC4_8822B 24
+#define BIT_MASK_DARF_RC4_8822B 0x1f
+#define BIT_DARF_RC4_8822B(x)                                                  \
+	(((x) & BIT_MASK_DARF_RC4_8822B) << BIT_SHIFT_DARF_RC4_8822B)
+#define BIT_GET_DARF_RC4_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_DARF_RC4_8822B) & BIT_MASK_DARF_RC4_8822B)
+
+#define BIT_SHIFT_DARF_RC3_8822B 16
+#define BIT_MASK_DARF_RC3_8822B 0x1f
+#define BIT_DARF_RC3_8822B(x)                                                  \
+	(((x) & BIT_MASK_DARF_RC3_8822B) << BIT_SHIFT_DARF_RC3_8822B)
+#define BIT_GET_DARF_RC3_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_DARF_RC3_8822B) & BIT_MASK_DARF_RC3_8822B)
+
+#define BIT_SHIFT_DARF_RC2_8822B 8
+#define BIT_MASK_DARF_RC2_8822B 0x1f
+#define BIT_DARF_RC2_8822B(x)                                                  \
+	(((x) & BIT_MASK_DARF_RC2_8822B) << BIT_SHIFT_DARF_RC2_8822B)
+#define BIT_GET_DARF_RC2_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_DARF_RC2_8822B) & BIT_MASK_DARF_RC2_8822B)
+
+#define BIT_SHIFT_DARF_RC1_8822B 0
+#define BIT_MASK_DARF_RC1_8822B 0x1f
+#define BIT_DARF_RC1_8822B(x)                                                  \
+	(((x) & BIT_MASK_DARF_RC1_8822B) << BIT_SHIFT_DARF_RC1_8822B)
+#define BIT_GET_DARF_RC1_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_DARF_RC1_8822B) & BIT_MASK_DARF_RC1_8822B)
+
+/* 2 REG_RARFRC_8822B */
+
+#define BIT_SHIFT_RARF_RC8_8822B (56 & CPU_OPT_WIDTH)
+#define BIT_MASK_RARF_RC8_8822B 0x1f
+#define BIT_RARF_RC8_8822B(x)                                                  \
+	(((x) & BIT_MASK_RARF_RC8_8822B) << BIT_SHIFT_RARF_RC8_8822B)
+#define BIT_GET_RARF_RC8_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_RARF_RC8_8822B) & BIT_MASK_RARF_RC8_8822B)
+
+#define BIT_SHIFT_RARF_RC7_8822B (48 & CPU_OPT_WIDTH)
+#define BIT_MASK_RARF_RC7_8822B 0x1f
+#define BIT_RARF_RC7_8822B(x)                                                  \
+	(((x) & BIT_MASK_RARF_RC7_8822B) << BIT_SHIFT_RARF_RC7_8822B)
+#define BIT_GET_RARF_RC7_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_RARF_RC7_8822B) & BIT_MASK_RARF_RC7_8822B)
+
+#define BIT_SHIFT_RARF_RC6_8822B (40 & CPU_OPT_WIDTH)
+#define BIT_MASK_RARF_RC6_8822B 0x1f
+#define BIT_RARF_RC6_8822B(x)                                                  \
+	(((x) & BIT_MASK_RARF_RC6_8822B) << BIT_SHIFT_RARF_RC6_8822B)
+#define BIT_GET_RARF_RC6_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_RARF_RC6_8822B) & BIT_MASK_RARF_RC6_8822B)
+
+#define BIT_SHIFT_RARF_RC5_8822B (32 & CPU_OPT_WIDTH)
+#define BIT_MASK_RARF_RC5_8822B 0x1f
+#define BIT_RARF_RC5_8822B(x)                                                  \
+	(((x) & BIT_MASK_RARF_RC5_8822B) << BIT_SHIFT_RARF_RC5_8822B)
+#define BIT_GET_RARF_RC5_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_RARF_RC5_8822B) & BIT_MASK_RARF_RC5_8822B)
+
+#define BIT_SHIFT_RARF_RC4_8822B 24
+#define BIT_MASK_RARF_RC4_8822B 0x1f
+#define BIT_RARF_RC4_8822B(x)                                                  \
+	(((x) & BIT_MASK_RARF_RC4_8822B) << BIT_SHIFT_RARF_RC4_8822B)
+#define BIT_GET_RARF_RC4_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_RARF_RC4_8822B) & BIT_MASK_RARF_RC4_8822B)
+
+#define BIT_SHIFT_RARF_RC3_8822B 16
+#define BIT_MASK_RARF_RC3_8822B 0x1f
+#define BIT_RARF_RC3_8822B(x)                                                  \
+	(((x) & BIT_MASK_RARF_RC3_8822B) << BIT_SHIFT_RARF_RC3_8822B)
+#define BIT_GET_RARF_RC3_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_RARF_RC3_8822B) & BIT_MASK_RARF_RC3_8822B)
+
+#define BIT_SHIFT_RARF_RC2_8822B 8
+#define BIT_MASK_RARF_RC2_8822B 0x1f
+#define BIT_RARF_RC2_8822B(x)                                                  \
+	(((x) & BIT_MASK_RARF_RC2_8822B) << BIT_SHIFT_RARF_RC2_8822B)
+#define BIT_GET_RARF_RC2_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_RARF_RC2_8822B) & BIT_MASK_RARF_RC2_8822B)
+
+#define BIT_SHIFT_RARF_RC1_8822B 0
+#define BIT_MASK_RARF_RC1_8822B 0x1f
+#define BIT_RARF_RC1_8822B(x)                                                  \
+	(((x) & BIT_MASK_RARF_RC1_8822B) << BIT_SHIFT_RARF_RC1_8822B)
+#define BIT_GET_RARF_RC1_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_RARF_RC1_8822B) & BIT_MASK_RARF_RC1_8822B)
+
+/* 2 REG_RRSR_8822B */
+
+#define BIT_SHIFT_RRSR_RSC_8822B 21
+#define BIT_MASK_RRSR_RSC_8822B 0x3
+#define BIT_RRSR_RSC_8822B(x)                                                  \
+	(((x) & BIT_MASK_RRSR_RSC_8822B) << BIT_SHIFT_RRSR_RSC_8822B)
+#define BIT_GET_RRSR_RSC_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_RRSR_RSC_8822B) & BIT_MASK_RRSR_RSC_8822B)
+
+#define BIT_RRSR_BW_8822B BIT(20)
+
+#define BIT_SHIFT_RRSC_BITMAP_8822B 0
+#define BIT_MASK_RRSC_BITMAP_8822B 0xfffff
+#define BIT_RRSC_BITMAP_8822B(x)                                               \
+	(((x) & BIT_MASK_RRSC_BITMAP_8822B) << BIT_SHIFT_RRSC_BITMAP_8822B)
+#define BIT_GET_RRSC_BITMAP_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_RRSC_BITMAP_8822B) & BIT_MASK_RRSC_BITMAP_8822B)
+
+/* 2 REG_ARFR0_8822B */
+
+#define BIT_SHIFT_ARFR0_V1_8822B 0
+#define BIT_MASK_ARFR0_V1_8822B 0xffffffffffffffffL
+#define BIT_ARFR0_V1_8822B(x)                                                  \
+	(((x) & BIT_MASK_ARFR0_V1_8822B) << BIT_SHIFT_ARFR0_V1_8822B)
+#define BIT_GET_ARFR0_V1_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_ARFR0_V1_8822B) & BIT_MASK_ARFR0_V1_8822B)
+
+/* 2 REG_ARFR1_V1_8822B */
+
+#define BIT_SHIFT_ARFR1_V1_8822B 0
+#define BIT_MASK_ARFR1_V1_8822B 0xffffffffffffffffL
+#define BIT_ARFR1_V1_8822B(x)                                                  \
+	(((x) & BIT_MASK_ARFR1_V1_8822B) << BIT_SHIFT_ARFR1_V1_8822B)
+#define BIT_GET_ARFR1_V1_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_ARFR1_V1_8822B) & BIT_MASK_ARFR1_V1_8822B)
+
+/* 2 REG_CCK_CHECK_8822B */
+#define BIT_CHECK_CCK_EN_8822B BIT(7)
+#define BIT_EN_BCN_PKT_REL_8822B BIT(6)
+#define BIT_BCN_PORT_SEL_8822B BIT(5)
+#define BIT_MOREDATA_BYPASS_8822B BIT(4)
+#define BIT_EN_CLR_CMD_REL_BCN_PKT_8822B BIT(3)
+#define BIT_R_EN_SET_MOREDATA_8822B BIT(2)
+#define BIT__R_DIS_CLEAR_MACID_RELEASE_8822B BIT(1)
+#define BIT__R_MACID_RELEASE_EN_8822B BIT(0)
+
+/* 2 REG_AMPDU_MAX_TIME_V1_8822B */
+
+#define BIT_SHIFT_AMPDU_MAX_TIME_8822B 0
+#define BIT_MASK_AMPDU_MAX_TIME_8822B 0xff
+#define BIT_AMPDU_MAX_TIME_8822B(x)                                            \
+	(((x) & BIT_MASK_AMPDU_MAX_TIME_8822B)                                 \
+	 << BIT_SHIFT_AMPDU_MAX_TIME_8822B)
+#define BIT_GET_AMPDU_MAX_TIME_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_AMPDU_MAX_TIME_8822B) &                             \
+	 BIT_MASK_AMPDU_MAX_TIME_8822B)
+
+/* 2 REG_BCNQ1_BDNY_V1_8822B */
+
+#define BIT_SHIFT_BCNQ1_PGBNDY_V1_8822B 0
+#define BIT_MASK_BCNQ1_PGBNDY_V1_8822B 0xfff
+#define BIT_BCNQ1_PGBNDY_V1_8822B(x)                                           \
+	(((x) & BIT_MASK_BCNQ1_PGBNDY_V1_8822B)                                \
+	 << BIT_SHIFT_BCNQ1_PGBNDY_V1_8822B)
+#define BIT_GET_BCNQ1_PGBNDY_V1_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_BCNQ1_PGBNDY_V1_8822B) &                            \
+	 BIT_MASK_BCNQ1_PGBNDY_V1_8822B)
+
+/* 2 REG_AMPDU_MAX_LENGTH_8822B */
+
+#define BIT_SHIFT_AMPDU_MAX_LENGTH_8822B 0
+#define BIT_MASK_AMPDU_MAX_LENGTH_8822B 0xffffffffL
+#define BIT_AMPDU_MAX_LENGTH_8822B(x)                                          \
+	(((x) & BIT_MASK_AMPDU_MAX_LENGTH_8822B)                               \
+	 << BIT_SHIFT_AMPDU_MAX_LENGTH_8822B)
+#define BIT_GET_AMPDU_MAX_LENGTH_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_AMPDU_MAX_LENGTH_8822B) &                           \
+	 BIT_MASK_AMPDU_MAX_LENGTH_8822B)
+
+/* 2 REG_ACQ_STOP_8822B */
+#define BIT_AC7Q_STOP_8822B BIT(7)
+#define BIT_AC6Q_STOP_8822B BIT(6)
+#define BIT_AC5Q_STOP_8822B BIT(5)
+#define BIT_AC4Q_STOP_8822B BIT(4)
+#define BIT_AC3Q_STOP_8822B BIT(3)
+#define BIT_AC2Q_STOP_8822B BIT(2)
+#define BIT_AC1Q_STOP_8822B BIT(1)
+#define BIT_AC0Q_STOP_8822B BIT(0)
+
+/* 2 REG_NDPA_RATE_8822B */
+
+#define BIT_SHIFT_R_NDPA_RATE_V1_8822B 0
+#define BIT_MASK_R_NDPA_RATE_V1_8822B 0xff
+#define BIT_R_NDPA_RATE_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_R_NDPA_RATE_V1_8822B)                                 \
+	 << BIT_SHIFT_R_NDPA_RATE_V1_8822B)
+#define BIT_GET_R_NDPA_RATE_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_R_NDPA_RATE_V1_8822B) &                             \
+	 BIT_MASK_R_NDPA_RATE_V1_8822B)
+
+/* 2 REG_TX_HANG_CTRL_8822B */
+#define BIT_R_EN_GNT_BT_AWAKE_8822B BIT(3)
+#define BIT_EN_EOF_V1_8822B BIT(2)
+#define BIT_DIS_OQT_BLOCK_8822B BIT(1)
+#define BIT_SEARCH_QUEUE_EN_8822B BIT(0)
+
+/* 2 REG_NDPA_OPT_CTRL_8822B */
+#define BIT_R_DIS_MACID_RELEASE_RTY_8822B BIT(5)
+
+#define BIT_SHIFT_BW_SIGTA_8822B 3
+#define BIT_MASK_BW_SIGTA_8822B 0x3
+#define BIT_BW_SIGTA_8822B(x)                                                  \
+	(((x) & BIT_MASK_BW_SIGTA_8822B) << BIT_SHIFT_BW_SIGTA_8822B)
+#define BIT_GET_BW_SIGTA_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_BW_SIGTA_8822B) & BIT_MASK_BW_SIGTA_8822B)
+
+#define BIT_EN_BAR_SIGTA_8822B BIT(2)
+
+#define BIT_SHIFT_R_NDPA_BW_8822B 0
+#define BIT_MASK_R_NDPA_BW_8822B 0x3
+#define BIT_R_NDPA_BW_8822B(x)                                                 \
+	(((x) & BIT_MASK_R_NDPA_BW_8822B) << BIT_SHIFT_R_NDPA_BW_8822B)
+#define BIT_GET_R_NDPA_BW_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_R_NDPA_BW_8822B) & BIT_MASK_R_NDPA_BW_8822B)
+
+/* 2 REG_RD_RESP_PKT_TH_8822B */
+
+#define BIT_SHIFT_RD_RESP_PKT_TH_V1_8822B 0
+#define BIT_MASK_RD_RESP_PKT_TH_V1_8822B 0x3f
+#define BIT_RD_RESP_PKT_TH_V1_8822B(x)                                         \
+	(((x) & BIT_MASK_RD_RESP_PKT_TH_V1_8822B)                              \
+	 << BIT_SHIFT_RD_RESP_PKT_TH_V1_8822B)
+#define BIT_GET_RD_RESP_PKT_TH_V1_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_RD_RESP_PKT_TH_V1_8822B) &                          \
+	 BIT_MASK_RD_RESP_PKT_TH_V1_8822B)
+
+/* 2 REG_CMDQ_INFO_8822B */
+
+#define BIT_SHIFT_QUEUEMACID_CMDQ_V1_8822B 25
+#define BIT_MASK_QUEUEMACID_CMDQ_V1_8822B 0x7f
+#define BIT_QUEUEMACID_CMDQ_V1_8822B(x)                                        \
+	(((x) & BIT_MASK_QUEUEMACID_CMDQ_V1_8822B)                             \
+	 << BIT_SHIFT_QUEUEMACID_CMDQ_V1_8822B)
+#define BIT_GET_QUEUEMACID_CMDQ_V1_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_QUEUEMACID_CMDQ_V1_8822B) &                         \
+	 BIT_MASK_QUEUEMACID_CMDQ_V1_8822B)
+
+#define BIT_SHIFT_QUEUEAC_CMDQ_V1_8822B 23
+#define BIT_MASK_QUEUEAC_CMDQ_V1_8822B 0x3
+#define BIT_QUEUEAC_CMDQ_V1_8822B(x)                                           \
+	(((x) & BIT_MASK_QUEUEAC_CMDQ_V1_8822B)                                \
+	 << BIT_SHIFT_QUEUEAC_CMDQ_V1_8822B)
+#define BIT_GET_QUEUEAC_CMDQ_V1_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_QUEUEAC_CMDQ_V1_8822B) &                            \
+	 BIT_MASK_QUEUEAC_CMDQ_V1_8822B)
+
+#define BIT_TIDEMPTY_CMDQ_V1_8822B BIT(22)
+
+#define BIT_SHIFT_TAIL_PKT_CMDQ_V2_8822B 11
+#define BIT_MASK_TAIL_PKT_CMDQ_V2_8822B 0x7ff
+#define BIT_TAIL_PKT_CMDQ_V2_8822B(x)                                          \
+	(((x) & BIT_MASK_TAIL_PKT_CMDQ_V2_8822B)                               \
+	 << BIT_SHIFT_TAIL_PKT_CMDQ_V2_8822B)
+#define BIT_GET_TAIL_PKT_CMDQ_V2_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_TAIL_PKT_CMDQ_V2_8822B) &                           \
+	 BIT_MASK_TAIL_PKT_CMDQ_V2_8822B)
+
+#define BIT_SHIFT_HEAD_PKT_CMDQ_V1_8822B 0
+#define BIT_MASK_HEAD_PKT_CMDQ_V1_8822B 0x7ff
+#define BIT_HEAD_PKT_CMDQ_V1_8822B(x)                                          \
+	(((x) & BIT_MASK_HEAD_PKT_CMDQ_V1_8822B)                               \
+	 << BIT_SHIFT_HEAD_PKT_CMDQ_V1_8822B)
+#define BIT_GET_HEAD_PKT_CMDQ_V1_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_HEAD_PKT_CMDQ_V1_8822B) &                           \
+	 BIT_MASK_HEAD_PKT_CMDQ_V1_8822B)
+
+/* 2 REG_Q4_INFO_8822B */
+
+#define BIT_SHIFT_QUEUEMACID_Q4_V1_8822B 25
+#define BIT_MASK_QUEUEMACID_Q4_V1_8822B 0x7f
+#define BIT_QUEUEMACID_Q4_V1_8822B(x)                                          \
+	(((x) & BIT_MASK_QUEUEMACID_Q4_V1_8822B)                               \
+	 << BIT_SHIFT_QUEUEMACID_Q4_V1_8822B)
+#define BIT_GET_QUEUEMACID_Q4_V1_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_QUEUEMACID_Q4_V1_8822B) &                           \
+	 BIT_MASK_QUEUEMACID_Q4_V1_8822B)
+
+#define BIT_SHIFT_QUEUEAC_Q4_V1_8822B 23
+#define BIT_MASK_QUEUEAC_Q4_V1_8822B 0x3
+#define BIT_QUEUEAC_Q4_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_QUEUEAC_Q4_V1_8822B) << BIT_SHIFT_QUEUEAC_Q4_V1_8822B)
+#define BIT_GET_QUEUEAC_Q4_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_QUEUEAC_Q4_V1_8822B) & BIT_MASK_QUEUEAC_Q4_V1_8822B)
+
+#define BIT_TIDEMPTY_Q4_V1_8822B BIT(22)
+
+#define BIT_SHIFT_TAIL_PKT_Q4_V2_8822B 11
+#define BIT_MASK_TAIL_PKT_Q4_V2_8822B 0x7ff
+#define BIT_TAIL_PKT_Q4_V2_8822B(x)                                            \
+	(((x) & BIT_MASK_TAIL_PKT_Q4_V2_8822B)                                 \
+	 << BIT_SHIFT_TAIL_PKT_Q4_V2_8822B)
+#define BIT_GET_TAIL_PKT_Q4_V2_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_TAIL_PKT_Q4_V2_8822B) &                             \
+	 BIT_MASK_TAIL_PKT_Q4_V2_8822B)
+
+#define BIT_SHIFT_HEAD_PKT_Q4_V1_8822B 0
+#define BIT_MASK_HEAD_PKT_Q4_V1_8822B 0x7ff
+#define BIT_HEAD_PKT_Q4_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_HEAD_PKT_Q4_V1_8822B)                                 \
+	 << BIT_SHIFT_HEAD_PKT_Q4_V1_8822B)
+#define BIT_GET_HEAD_PKT_Q4_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HEAD_PKT_Q4_V1_8822B) &                             \
+	 BIT_MASK_HEAD_PKT_Q4_V1_8822B)
+
+/* 2 REG_Q5_INFO_8822B */
+
+#define BIT_SHIFT_QUEUEMACID_Q5_V1_8822B 25
+#define BIT_MASK_QUEUEMACID_Q5_V1_8822B 0x7f
+#define BIT_QUEUEMACID_Q5_V1_8822B(x)                                          \
+	(((x) & BIT_MASK_QUEUEMACID_Q5_V1_8822B)                               \
+	 << BIT_SHIFT_QUEUEMACID_Q5_V1_8822B)
+#define BIT_GET_QUEUEMACID_Q5_V1_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_QUEUEMACID_Q5_V1_8822B) &                           \
+	 BIT_MASK_QUEUEMACID_Q5_V1_8822B)
+
+#define BIT_SHIFT_QUEUEAC_Q5_V1_8822B 23
+#define BIT_MASK_QUEUEAC_Q5_V1_8822B 0x3
+#define BIT_QUEUEAC_Q5_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_QUEUEAC_Q5_V1_8822B) << BIT_SHIFT_QUEUEAC_Q5_V1_8822B)
+#define BIT_GET_QUEUEAC_Q5_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_QUEUEAC_Q5_V1_8822B) & BIT_MASK_QUEUEAC_Q5_V1_8822B)
+
+#define BIT_TIDEMPTY_Q5_V1_8822B BIT(22)
+
+#define BIT_SHIFT_TAIL_PKT_Q5_V2_8822B 11
+#define BIT_MASK_TAIL_PKT_Q5_V2_8822B 0x7ff
+#define BIT_TAIL_PKT_Q5_V2_8822B(x)                                            \
+	(((x) & BIT_MASK_TAIL_PKT_Q5_V2_8822B)                                 \
+	 << BIT_SHIFT_TAIL_PKT_Q5_V2_8822B)
+#define BIT_GET_TAIL_PKT_Q5_V2_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_TAIL_PKT_Q5_V2_8822B) &                             \
+	 BIT_MASK_TAIL_PKT_Q5_V2_8822B)
+
+#define BIT_SHIFT_HEAD_PKT_Q5_V1_8822B 0
+#define BIT_MASK_HEAD_PKT_Q5_V1_8822B 0x7ff
+#define BIT_HEAD_PKT_Q5_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_HEAD_PKT_Q5_V1_8822B)                                 \
+	 << BIT_SHIFT_HEAD_PKT_Q5_V1_8822B)
+#define BIT_GET_HEAD_PKT_Q5_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HEAD_PKT_Q5_V1_8822B) &                             \
+	 BIT_MASK_HEAD_PKT_Q5_V1_8822B)
+
+/* 2 REG_Q6_INFO_8822B */
+
+#define BIT_SHIFT_QUEUEMACID_Q6_V1_8822B 25
+#define BIT_MASK_QUEUEMACID_Q6_V1_8822B 0x7f
+#define BIT_QUEUEMACID_Q6_V1_8822B(x)                                          \
+	(((x) & BIT_MASK_QUEUEMACID_Q6_V1_8822B)                               \
+	 << BIT_SHIFT_QUEUEMACID_Q6_V1_8822B)
+#define BIT_GET_QUEUEMACID_Q6_V1_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_QUEUEMACID_Q6_V1_8822B) &                           \
+	 BIT_MASK_QUEUEMACID_Q6_V1_8822B)
+
+#define BIT_SHIFT_QUEUEAC_Q6_V1_8822B 23
+#define BIT_MASK_QUEUEAC_Q6_V1_8822B 0x3
+#define BIT_QUEUEAC_Q6_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_QUEUEAC_Q6_V1_8822B) << BIT_SHIFT_QUEUEAC_Q6_V1_8822B)
+#define BIT_GET_QUEUEAC_Q6_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_QUEUEAC_Q6_V1_8822B) & BIT_MASK_QUEUEAC_Q6_V1_8822B)
+
+#define BIT_TIDEMPTY_Q6_V1_8822B BIT(22)
+
+#define BIT_SHIFT_TAIL_PKT_Q6_V2_8822B 11
+#define BIT_MASK_TAIL_PKT_Q6_V2_8822B 0x7ff
+#define BIT_TAIL_PKT_Q6_V2_8822B(x)                                            \
+	(((x) & BIT_MASK_TAIL_PKT_Q6_V2_8822B)                                 \
+	 << BIT_SHIFT_TAIL_PKT_Q6_V2_8822B)
+#define BIT_GET_TAIL_PKT_Q6_V2_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_TAIL_PKT_Q6_V2_8822B) &                             \
+	 BIT_MASK_TAIL_PKT_Q6_V2_8822B)
+
+#define BIT_SHIFT_HEAD_PKT_Q6_V1_8822B 0
+#define BIT_MASK_HEAD_PKT_Q6_V1_8822B 0x7ff
+#define BIT_HEAD_PKT_Q6_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_HEAD_PKT_Q6_V1_8822B)                                 \
+	 << BIT_SHIFT_HEAD_PKT_Q6_V1_8822B)
+#define BIT_GET_HEAD_PKT_Q6_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HEAD_PKT_Q6_V1_8822B) &                             \
+	 BIT_MASK_HEAD_PKT_Q6_V1_8822B)
+
+/* 2 REG_Q7_INFO_8822B */
+
+#define BIT_SHIFT_QUEUEMACID_Q7_V1_8822B 25
+#define BIT_MASK_QUEUEMACID_Q7_V1_8822B 0x7f
+#define BIT_QUEUEMACID_Q7_V1_8822B(x)                                          \
+	(((x) & BIT_MASK_QUEUEMACID_Q7_V1_8822B)                               \
+	 << BIT_SHIFT_QUEUEMACID_Q7_V1_8822B)
+#define BIT_GET_QUEUEMACID_Q7_V1_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_QUEUEMACID_Q7_V1_8822B) &                           \
+	 BIT_MASK_QUEUEMACID_Q7_V1_8822B)
+
+#define BIT_SHIFT_QUEUEAC_Q7_V1_8822B 23
+#define BIT_MASK_QUEUEAC_Q7_V1_8822B 0x3
+#define BIT_QUEUEAC_Q7_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_QUEUEAC_Q7_V1_8822B) << BIT_SHIFT_QUEUEAC_Q7_V1_8822B)
+#define BIT_GET_QUEUEAC_Q7_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_QUEUEAC_Q7_V1_8822B) & BIT_MASK_QUEUEAC_Q7_V1_8822B)
+
+#define BIT_TIDEMPTY_Q7_V1_8822B BIT(22)
+
+#define BIT_SHIFT_TAIL_PKT_Q7_V2_8822B 11
+#define BIT_MASK_TAIL_PKT_Q7_V2_8822B 0x7ff
+#define BIT_TAIL_PKT_Q7_V2_8822B(x)                                            \
+	(((x) & BIT_MASK_TAIL_PKT_Q7_V2_8822B)                                 \
+	 << BIT_SHIFT_TAIL_PKT_Q7_V2_8822B)
+#define BIT_GET_TAIL_PKT_Q7_V2_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_TAIL_PKT_Q7_V2_8822B) &                             \
+	 BIT_MASK_TAIL_PKT_Q7_V2_8822B)
+
+#define BIT_SHIFT_HEAD_PKT_Q7_V1_8822B 0
+#define BIT_MASK_HEAD_PKT_Q7_V1_8822B 0x7ff
+#define BIT_HEAD_PKT_Q7_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_HEAD_PKT_Q7_V1_8822B)                                 \
+	 << BIT_SHIFT_HEAD_PKT_Q7_V1_8822B)
+#define BIT_GET_HEAD_PKT_Q7_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_HEAD_PKT_Q7_V1_8822B) &                             \
+	 BIT_MASK_HEAD_PKT_Q7_V1_8822B)
+
+/* 2 REG_WMAC_LBK_BUF_HD_V1_8822B */
+
+#define BIT_SHIFT_WMAC_LBK_BUF_HEAD_V1_8822B 0
+#define BIT_MASK_WMAC_LBK_BUF_HEAD_V1_8822B 0xfff
+#define BIT_WMAC_LBK_BUF_HEAD_V1_8822B(x)                                      \
+	(((x) & BIT_MASK_WMAC_LBK_BUF_HEAD_V1_8822B)                           \
+	 << BIT_SHIFT_WMAC_LBK_BUF_HEAD_V1_8822B)
+#define BIT_GET_WMAC_LBK_BUF_HEAD_V1_8822B(x)                                  \
+	(((x) >> BIT_SHIFT_WMAC_LBK_BUF_HEAD_V1_8822B) &                       \
+	 BIT_MASK_WMAC_LBK_BUF_HEAD_V1_8822B)
+
+/* 2 REG_MGQ_BDNY_V1_8822B */
+
+#define BIT_SHIFT_MGQ_PGBNDY_V1_8822B 0
+#define BIT_MASK_MGQ_PGBNDY_V1_8822B 0xfff
+#define BIT_MGQ_PGBNDY_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_MGQ_PGBNDY_V1_8822B) << BIT_SHIFT_MGQ_PGBNDY_V1_8822B)
+#define BIT_GET_MGQ_PGBNDY_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_MGQ_PGBNDY_V1_8822B) & BIT_MASK_MGQ_PGBNDY_V1_8822B)
+
+/* 2 REG_TXRPT_CTRL_8822B */
+
+#define BIT_SHIFT_TRXRPT_TIMER_TH_8822B 24
+#define BIT_MASK_TRXRPT_TIMER_TH_8822B 0xff
+#define BIT_TRXRPT_TIMER_TH_8822B(x)                                           \
+	(((x) & BIT_MASK_TRXRPT_TIMER_TH_8822B)                                \
+	 << BIT_SHIFT_TRXRPT_TIMER_TH_8822B)
+#define BIT_GET_TRXRPT_TIMER_TH_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_TRXRPT_TIMER_TH_8822B) &                            \
+	 BIT_MASK_TRXRPT_TIMER_TH_8822B)
+
+#define BIT_SHIFT_TRXRPT_LEN_TH_8822B 16
+#define BIT_MASK_TRXRPT_LEN_TH_8822B 0xff
+#define BIT_TRXRPT_LEN_TH_8822B(x)                                             \
+	(((x) & BIT_MASK_TRXRPT_LEN_TH_8822B) << BIT_SHIFT_TRXRPT_LEN_TH_8822B)
+#define BIT_GET_TRXRPT_LEN_TH_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_TRXRPT_LEN_TH_8822B) & BIT_MASK_TRXRPT_LEN_TH_8822B)
+
+#define BIT_SHIFT_TRXRPT_READ_PTR_8822B 8
+#define BIT_MASK_TRXRPT_READ_PTR_8822B 0xff
+#define BIT_TRXRPT_READ_PTR_8822B(x)                                           \
+	(((x) & BIT_MASK_TRXRPT_READ_PTR_8822B)                                \
+	 << BIT_SHIFT_TRXRPT_READ_PTR_8822B)
+#define BIT_GET_TRXRPT_READ_PTR_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_TRXRPT_READ_PTR_8822B) &                            \
+	 BIT_MASK_TRXRPT_READ_PTR_8822B)
+
+#define BIT_SHIFT_TRXRPT_WRITE_PTR_8822B 0
+#define BIT_MASK_TRXRPT_WRITE_PTR_8822B 0xff
+#define BIT_TRXRPT_WRITE_PTR_8822B(x)                                          \
+	(((x) & BIT_MASK_TRXRPT_WRITE_PTR_8822B)                               \
+	 << BIT_SHIFT_TRXRPT_WRITE_PTR_8822B)
+#define BIT_GET_TRXRPT_WRITE_PTR_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_TRXRPT_WRITE_PTR_8822B) &                           \
+	 BIT_MASK_TRXRPT_WRITE_PTR_8822B)
+
+/* 2 REG_INIRTS_RATE_SEL_8822B */
+#define BIT_LEAG_RTS_BW_DUP_8822B BIT(5)
+
+/* 2 REG_BASIC_CFEND_RATE_8822B */
+
+#define BIT_SHIFT_BASIC_CFEND_RATE_8822B 0
+#define BIT_MASK_BASIC_CFEND_RATE_8822B 0x1f
+#define BIT_BASIC_CFEND_RATE_8822B(x)                                          \
+	(((x) & BIT_MASK_BASIC_CFEND_RATE_8822B)                               \
+	 << BIT_SHIFT_BASIC_CFEND_RATE_8822B)
+#define BIT_GET_BASIC_CFEND_RATE_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_BASIC_CFEND_RATE_8822B) &                           \
+	 BIT_MASK_BASIC_CFEND_RATE_8822B)
+
+/* 2 REG_STBC_CFEND_RATE_8822B */
+
+#define BIT_SHIFT_STBC_CFEND_RATE_8822B 0
+#define BIT_MASK_STBC_CFEND_RATE_8822B 0x1f
+#define BIT_STBC_CFEND_RATE_8822B(x)                                           \
+	(((x) & BIT_MASK_STBC_CFEND_RATE_8822B)                                \
+	 << BIT_SHIFT_STBC_CFEND_RATE_8822B)
+#define BIT_GET_STBC_CFEND_RATE_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_STBC_CFEND_RATE_8822B) &                            \
+	 BIT_MASK_STBC_CFEND_RATE_8822B)
+
+/* 2 REG_DATA_SC_8822B */
+
+#define BIT_SHIFT_TXSC_40M_8822B 4
+#define BIT_MASK_TXSC_40M_8822B 0xf
+#define BIT_TXSC_40M_8822B(x)                                                  \
+	(((x) & BIT_MASK_TXSC_40M_8822B) << BIT_SHIFT_TXSC_40M_8822B)
+#define BIT_GET_TXSC_40M_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_TXSC_40M_8822B) & BIT_MASK_TXSC_40M_8822B)
+
+#define BIT_SHIFT_TXSC_20M_8822B 0
+#define BIT_MASK_TXSC_20M_8822B 0xf
+#define BIT_TXSC_20M_8822B(x)                                                  \
+	(((x) & BIT_MASK_TXSC_20M_8822B) << BIT_SHIFT_TXSC_20M_8822B)
+#define BIT_GET_TXSC_20M_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_TXSC_20M_8822B) & BIT_MASK_TXSC_20M_8822B)
+
+/* 2 REG_MACID_SLEEP3_8822B */
+
+#define BIT_SHIFT_MACID127_96_PKTSLEEP_8822B 0
+#define BIT_MASK_MACID127_96_PKTSLEEP_8822B 0xffffffffL
+#define BIT_MACID127_96_PKTSLEEP_8822B(x)                                      \
+	(((x) & BIT_MASK_MACID127_96_PKTSLEEP_8822B)                           \
+	 << BIT_SHIFT_MACID127_96_PKTSLEEP_8822B)
+#define BIT_GET_MACID127_96_PKTSLEEP_8822B(x)                                  \
+	(((x) >> BIT_SHIFT_MACID127_96_PKTSLEEP_8822B) &                       \
+	 BIT_MASK_MACID127_96_PKTSLEEP_8822B)
+
+/* 2 REG_MACID_SLEEP1_8822B */
+
+#define BIT_SHIFT_MACID63_32_PKTSLEEP_8822B 0
+#define BIT_MASK_MACID63_32_PKTSLEEP_8822B 0xffffffffL
+#define BIT_MACID63_32_PKTSLEEP_8822B(x)                                       \
+	(((x) & BIT_MASK_MACID63_32_PKTSLEEP_8822B)                            \
+	 << BIT_SHIFT_MACID63_32_PKTSLEEP_8822B)
+#define BIT_GET_MACID63_32_PKTSLEEP_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_MACID63_32_PKTSLEEP_8822B) &                        \
+	 BIT_MASK_MACID63_32_PKTSLEEP_8822B)
+
+/* 2 REG_ARFR2_V1_8822B */
+
+#define BIT_SHIFT_ARFR2_V1_8822B 0
+#define BIT_MASK_ARFR2_V1_8822B 0xffffffffffffffffL
+#define BIT_ARFR2_V1_8822B(x)                                                  \
+	(((x) & BIT_MASK_ARFR2_V1_8822B) << BIT_SHIFT_ARFR2_V1_8822B)
+#define BIT_GET_ARFR2_V1_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_ARFR2_V1_8822B) & BIT_MASK_ARFR2_V1_8822B)
+
+/* 2 REG_ARFR3_V1_8822B */
+
+#define BIT_SHIFT_ARFR3_V1_8822B 0
+#define BIT_MASK_ARFR3_V1_8822B 0xffffffffffffffffL
+#define BIT_ARFR3_V1_8822B(x)                                                  \
+	(((x) & BIT_MASK_ARFR3_V1_8822B) << BIT_SHIFT_ARFR3_V1_8822B)
+#define BIT_GET_ARFR3_V1_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_ARFR3_V1_8822B) & BIT_MASK_ARFR3_V1_8822B)
+
+/* 2 REG_ARFR4_8822B */
+
+#define BIT_SHIFT_ARFR4_8822B 0
+#define BIT_MASK_ARFR4_8822B 0xffffffffffffffffL
+#define BIT_ARFR4_8822B(x)                                                     \
+	(((x) & BIT_MASK_ARFR4_8822B) << BIT_SHIFT_ARFR4_8822B)
+#define BIT_GET_ARFR4_8822B(x)                                                 \
+	(((x) >> BIT_SHIFT_ARFR4_8822B) & BIT_MASK_ARFR4_8822B)
+
+/* 2 REG_ARFR5_8822B */
+
+#define BIT_SHIFT_ARFR5_8822B 0
+#define BIT_MASK_ARFR5_8822B 0xffffffffffffffffL
+#define BIT_ARFR5_8822B(x)                                                     \
+	(((x) & BIT_MASK_ARFR5_8822B) << BIT_SHIFT_ARFR5_8822B)
+#define BIT_GET_ARFR5_8822B(x)                                                 \
+	(((x) >> BIT_SHIFT_ARFR5_8822B) & BIT_MASK_ARFR5_8822B)
+
+/* 2 REG_TXRPT_START_OFFSET_8822B */
+
+#define BIT_SHIFT_MACID_MURATE_OFFSET_8822B 24
+#define BIT_MASK_MACID_MURATE_OFFSET_8822B 0xff
+#define BIT_MACID_MURATE_OFFSET_8822B(x)                                       \
+	(((x) & BIT_MASK_MACID_MURATE_OFFSET_8822B)                            \
+	 << BIT_SHIFT_MACID_MURATE_OFFSET_8822B)
+#define BIT_GET_MACID_MURATE_OFFSET_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_MACID_MURATE_OFFSET_8822B) &                        \
+	 BIT_MASK_MACID_MURATE_OFFSET_8822B)
+
+#define BIT_RPTFIFO_SIZE_OPT_8822B BIT(16)
+
+#define BIT_SHIFT_MACID_CTRL_OFFSET_8822B 8
+#define BIT_MASK_MACID_CTRL_OFFSET_8822B 0xff
+#define BIT_MACID_CTRL_OFFSET_8822B(x)                                         \
+	(((x) & BIT_MASK_MACID_CTRL_OFFSET_8822B)                              \
+	 << BIT_SHIFT_MACID_CTRL_OFFSET_8822B)
+#define BIT_GET_MACID_CTRL_OFFSET_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_MACID_CTRL_OFFSET_8822B) &                          \
+	 BIT_MASK_MACID_CTRL_OFFSET_8822B)
+
+#define BIT_SHIFT_AMPDU_TXRPT_OFFSET_8822B 0
+#define BIT_MASK_AMPDU_TXRPT_OFFSET_8822B 0xff
+#define BIT_AMPDU_TXRPT_OFFSET_8822B(x)                                        \
+	(((x) & BIT_MASK_AMPDU_TXRPT_OFFSET_8822B)                             \
+	 << BIT_SHIFT_AMPDU_TXRPT_OFFSET_8822B)
+#define BIT_GET_AMPDU_TXRPT_OFFSET_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_AMPDU_TXRPT_OFFSET_8822B) &                         \
+	 BIT_MASK_AMPDU_TXRPT_OFFSET_8822B)
+
+/* 2 REG_POWER_STAGE1_8822B */
+#define BIT_PTA_WL_PRI_MASK_CPU_MGQ_8822B BIT(31)
+#define BIT_PTA_WL_PRI_MASK_BCNQ_8822B BIT(30)
+#define BIT_PTA_WL_PRI_MASK_HIQ_8822B BIT(29)
+#define BIT_PTA_WL_PRI_MASK_MGQ_8822B BIT(28)
+#define BIT_PTA_WL_PRI_MASK_BK_8822B BIT(27)
+#define BIT_PTA_WL_PRI_MASK_BE_8822B BIT(26)
+#define BIT_PTA_WL_PRI_MASK_VI_8822B BIT(25)
+#define BIT_PTA_WL_PRI_MASK_VO_8822B BIT(24)
+
+#define BIT_SHIFT_POWER_STAGE1_8822B 0
+#define BIT_MASK_POWER_STAGE1_8822B 0xffffff
+#define BIT_POWER_STAGE1_8822B(x)                                              \
+	(((x) & BIT_MASK_POWER_STAGE1_8822B) << BIT_SHIFT_POWER_STAGE1_8822B)
+#define BIT_GET_POWER_STAGE1_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_POWER_STAGE1_8822B) & BIT_MASK_POWER_STAGE1_8822B)
+
+/* 2 REG_POWER_STAGE2_8822B */
+#define BIT__R_CTRL_PKT_POW_ADJ_8822B BIT(24)
+
+#define BIT_SHIFT_POWER_STAGE2_8822B 0
+#define BIT_MASK_POWER_STAGE2_8822B 0xffffff
+#define BIT_POWER_STAGE2_8822B(x)                                              \
+	(((x) & BIT_MASK_POWER_STAGE2_8822B) << BIT_SHIFT_POWER_STAGE2_8822B)
+#define BIT_GET_POWER_STAGE2_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_POWER_STAGE2_8822B) & BIT_MASK_POWER_STAGE2_8822B)
+
+/* 2 REG_SW_AMPDU_BURST_MODE_CTRL_8822B */
+
+#define BIT_SHIFT_PAD_NUM_THRES_8822B 24
+#define BIT_MASK_PAD_NUM_THRES_8822B 0x3f
+#define BIT_PAD_NUM_THRES_8822B(x)                                             \
+	(((x) & BIT_MASK_PAD_NUM_THRES_8822B) << BIT_SHIFT_PAD_NUM_THRES_8822B)
+#define BIT_GET_PAD_NUM_THRES_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_PAD_NUM_THRES_8822B) & BIT_MASK_PAD_NUM_THRES_8822B)
+
+#define BIT_R_DMA_THIS_QUEUE_BK_8822B BIT(23)
+#define BIT_R_DMA_THIS_QUEUE_BE_8822B BIT(22)
+#define BIT_R_DMA_THIS_QUEUE_VI_8822B BIT(21)
+#define BIT_R_DMA_THIS_QUEUE_VO_8822B BIT(20)
+
+#define BIT_SHIFT_R_TOTAL_LEN_TH_8822B 8
+#define BIT_MASK_R_TOTAL_LEN_TH_8822B 0xfff
+#define BIT_R_TOTAL_LEN_TH_8822B(x)                                            \
+	(((x) & BIT_MASK_R_TOTAL_LEN_TH_8822B)                                 \
+	 << BIT_SHIFT_R_TOTAL_LEN_TH_8822B)
+#define BIT_GET_R_TOTAL_LEN_TH_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_R_TOTAL_LEN_TH_8822B) &                             \
+	 BIT_MASK_R_TOTAL_LEN_TH_8822B)
+
+#define BIT_EN_NEW_EARLY_8822B BIT(7)
+#define BIT_PRE_TX_CMD_8822B BIT(6)
+
+#define BIT_SHIFT_NUM_SCL_EN_8822B 4
+#define BIT_MASK_NUM_SCL_EN_8822B 0x3
+#define BIT_NUM_SCL_EN_8822B(x)                                                \
+	(((x) & BIT_MASK_NUM_SCL_EN_8822B) << BIT_SHIFT_NUM_SCL_EN_8822B)
+#define BIT_GET_NUM_SCL_EN_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_NUM_SCL_EN_8822B) & BIT_MASK_NUM_SCL_EN_8822B)
+
+#define BIT_BK_EN_8822B BIT(3)
+#define BIT_BE_EN_8822B BIT(2)
+#define BIT_VI_EN_8822B BIT(1)
+#define BIT_VO_EN_8822B BIT(0)
+
+/* 2 REG_PKT_LIFE_TIME_8822B */
+
+#define BIT_SHIFT_PKT_LIFTIME_BEBK_8822B 16
+#define BIT_MASK_PKT_LIFTIME_BEBK_8822B 0xffff
+#define BIT_PKT_LIFTIME_BEBK_8822B(x)                                          \
+	(((x) & BIT_MASK_PKT_LIFTIME_BEBK_8822B)                               \
+	 << BIT_SHIFT_PKT_LIFTIME_BEBK_8822B)
+#define BIT_GET_PKT_LIFTIME_BEBK_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_PKT_LIFTIME_BEBK_8822B) &                           \
+	 BIT_MASK_PKT_LIFTIME_BEBK_8822B)
+
+#define BIT_SHIFT_PKT_LIFTIME_VOVI_8822B 0
+#define BIT_MASK_PKT_LIFTIME_VOVI_8822B 0xffff
+#define BIT_PKT_LIFTIME_VOVI_8822B(x)                                          \
+	(((x) & BIT_MASK_PKT_LIFTIME_VOVI_8822B)                               \
+	 << BIT_SHIFT_PKT_LIFTIME_VOVI_8822B)
+#define BIT_GET_PKT_LIFTIME_VOVI_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_PKT_LIFTIME_VOVI_8822B) &                           \
+	 BIT_MASK_PKT_LIFTIME_VOVI_8822B)
+
+/* 2 REG_STBC_SETTING_8822B */
+
+#define BIT_SHIFT_CDEND_TXTIME_L_8822B 4
+#define BIT_MASK_CDEND_TXTIME_L_8822B 0xf
+#define BIT_CDEND_TXTIME_L_8822B(x)                                            \
+	(((x) & BIT_MASK_CDEND_TXTIME_L_8822B)                                 \
+	 << BIT_SHIFT_CDEND_TXTIME_L_8822B)
+#define BIT_GET_CDEND_TXTIME_L_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_CDEND_TXTIME_L_8822B) &                             \
+	 BIT_MASK_CDEND_TXTIME_L_8822B)
+
+#define BIT_SHIFT_NESS_8822B 2
+#define BIT_MASK_NESS_8822B 0x3
+#define BIT_NESS_8822B(x) (((x) & BIT_MASK_NESS_8822B) << BIT_SHIFT_NESS_8822B)
+#define BIT_GET_NESS_8822B(x)                                                  \
+	(((x) >> BIT_SHIFT_NESS_8822B) & BIT_MASK_NESS_8822B)
+
+#define BIT_SHIFT_STBC_CFEND_8822B 0
+#define BIT_MASK_STBC_CFEND_8822B 0x3
+#define BIT_STBC_CFEND_8822B(x)                                                \
+	(((x) & BIT_MASK_STBC_CFEND_8822B) << BIT_SHIFT_STBC_CFEND_8822B)
+#define BIT_GET_STBC_CFEND_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_STBC_CFEND_8822B) & BIT_MASK_STBC_CFEND_8822B)
+
+/* 2 REG_STBC_SETTING2_8822B */
+
+#define BIT_SHIFT_CDEND_TXTIME_H_8822B 0
+#define BIT_MASK_CDEND_TXTIME_H_8822B 0x1f
+#define BIT_CDEND_TXTIME_H_8822B(x)                                            \
+	(((x) & BIT_MASK_CDEND_TXTIME_H_8822B)                                 \
+	 << BIT_SHIFT_CDEND_TXTIME_H_8822B)
+#define BIT_GET_CDEND_TXTIME_H_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_CDEND_TXTIME_H_8822B) &                             \
+	 BIT_MASK_CDEND_TXTIME_H_8822B)
+
+/* 2 REG_QUEUE_CTRL_8822B */
+#define BIT_PTA_EDCCA_EN_8822B BIT(5)
+#define BIT_PTA_WL_TX_EN_8822B BIT(4)
+#define BIT_R_USE_DATA_BW_8822B BIT(3)
+#define BIT_TRI_PKT_INT_MODE1_8822B BIT(2)
+#define BIT_TRI_PKT_INT_MODE0_8822B BIT(1)
+#define BIT_ACQ_MODE_SEL_8822B BIT(0)
+
+/* 2 REG_SINGLE_AMPDU_CTRL_8822B */
+#define BIT_EN_SINGLE_APMDU_8822B BIT(7)
+
+/* 2 REG_PROT_MODE_CTRL_8822B */
+
+#define BIT_SHIFT_RTS_MAX_AGG_NUM_8822B 24
+#define BIT_MASK_RTS_MAX_AGG_NUM_8822B 0x3f
+#define BIT_RTS_MAX_AGG_NUM_8822B(x)                                           \
+	(((x) & BIT_MASK_RTS_MAX_AGG_NUM_8822B)                                \
+	 << BIT_SHIFT_RTS_MAX_AGG_NUM_8822B)
+#define BIT_GET_RTS_MAX_AGG_NUM_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_RTS_MAX_AGG_NUM_8822B) &                            \
+	 BIT_MASK_RTS_MAX_AGG_NUM_8822B)
+
+#define BIT_SHIFT_MAX_AGG_NUM_8822B 16
+#define BIT_MASK_MAX_AGG_NUM_8822B 0x3f
+#define BIT_MAX_AGG_NUM_8822B(x)                                               \
+	(((x) & BIT_MASK_MAX_AGG_NUM_8822B) << BIT_SHIFT_MAX_AGG_NUM_8822B)
+#define BIT_GET_MAX_AGG_NUM_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_MAX_AGG_NUM_8822B) & BIT_MASK_MAX_AGG_NUM_8822B)
+
+#define BIT_SHIFT_RTS_TXTIME_TH_8822B 8
+#define BIT_MASK_RTS_TXTIME_TH_8822B 0xff
+#define BIT_RTS_TXTIME_TH_8822B(x)                                             \
+	(((x) & BIT_MASK_RTS_TXTIME_TH_8822B) << BIT_SHIFT_RTS_TXTIME_TH_8822B)
+#define BIT_GET_RTS_TXTIME_TH_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_RTS_TXTIME_TH_8822B) & BIT_MASK_RTS_TXTIME_TH_8822B)
+
+#define BIT_SHIFT_RTS_LEN_TH_8822B 0
+#define BIT_MASK_RTS_LEN_TH_8822B 0xff
+#define BIT_RTS_LEN_TH_8822B(x)                                                \
+	(((x) & BIT_MASK_RTS_LEN_TH_8822B) << BIT_SHIFT_RTS_LEN_TH_8822B)
+#define BIT_GET_RTS_LEN_TH_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_RTS_LEN_TH_8822B) & BIT_MASK_RTS_LEN_TH_8822B)
+
+/* 2 REG_BAR_MODE_CTRL_8822B */
+
+#define BIT_SHIFT_BAR_RTY_LMT_8822B 16
+#define BIT_MASK_BAR_RTY_LMT_8822B 0x3
+#define BIT_BAR_RTY_LMT_8822B(x)                                               \
+	(((x) & BIT_MASK_BAR_RTY_LMT_8822B) << BIT_SHIFT_BAR_RTY_LMT_8822B)
+#define BIT_GET_BAR_RTY_LMT_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_BAR_RTY_LMT_8822B) & BIT_MASK_BAR_RTY_LMT_8822B)
+
+#define BIT_SHIFT_BAR_PKT_TXTIME_TH_8822B 8
+#define BIT_MASK_BAR_PKT_TXTIME_TH_8822B 0xff
+#define BIT_BAR_PKT_TXTIME_TH_8822B(x)                                         \
+	(((x) & BIT_MASK_BAR_PKT_TXTIME_TH_8822B)                              \
+	 << BIT_SHIFT_BAR_PKT_TXTIME_TH_8822B)
+#define BIT_GET_BAR_PKT_TXTIME_TH_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_BAR_PKT_TXTIME_TH_8822B) &                          \
+	 BIT_MASK_BAR_PKT_TXTIME_TH_8822B)
+
+#define BIT_BAR_EN_V1_8822B BIT(6)
+
+#define BIT_SHIFT_BAR_PKTNUM_TH_V1_8822B 0
+#define BIT_MASK_BAR_PKTNUM_TH_V1_8822B 0x3f
+#define BIT_BAR_PKTNUM_TH_V1_8822B(x)                                          \
+	(((x) & BIT_MASK_BAR_PKTNUM_TH_V1_8822B)                               \
+	 << BIT_SHIFT_BAR_PKTNUM_TH_V1_8822B)
+#define BIT_GET_BAR_PKTNUM_TH_V1_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_BAR_PKTNUM_TH_V1_8822B) &                           \
+	 BIT_MASK_BAR_PKTNUM_TH_V1_8822B)
+
+/* 2 REG_RA_TRY_RATE_AGG_LMT_8822B */
+
+#define BIT_SHIFT_RA_TRY_RATE_AGG_LMT_V1_8822B 0
+#define BIT_MASK_RA_TRY_RATE_AGG_LMT_V1_8822B 0x3f
+#define BIT_RA_TRY_RATE_AGG_LMT_V1_8822B(x)                                    \
+	(((x) & BIT_MASK_RA_TRY_RATE_AGG_LMT_V1_8822B)                         \
+	 << BIT_SHIFT_RA_TRY_RATE_AGG_LMT_V1_8822B)
+#define BIT_GET_RA_TRY_RATE_AGG_LMT_V1_8822B(x)                                \
+	(((x) >> BIT_SHIFT_RA_TRY_RATE_AGG_LMT_V1_8822B) &                     \
+	 BIT_MASK_RA_TRY_RATE_AGG_LMT_V1_8822B)
+
+/* 2 REG_MACID_SLEEP2_8822B */
+
+#define BIT_SHIFT_MACID95_64PKTSLEEP_8822B 0
+#define BIT_MASK_MACID95_64PKTSLEEP_8822B 0xffffffffL
+#define BIT_MACID95_64PKTSLEEP_8822B(x)                                        \
+	(((x) & BIT_MASK_MACID95_64PKTSLEEP_8822B)                             \
+	 << BIT_SHIFT_MACID95_64PKTSLEEP_8822B)
+#define BIT_GET_MACID95_64PKTSLEEP_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_MACID95_64PKTSLEEP_8822B) &                         \
+	 BIT_MASK_MACID95_64PKTSLEEP_8822B)
+
+/* 2 REG_MACID_SLEEP_8822B */
+
+#define BIT_SHIFT_MACID31_0_PKTSLEEP_8822B 0
+#define BIT_MASK_MACID31_0_PKTSLEEP_8822B 0xffffffffL
+#define BIT_MACID31_0_PKTSLEEP_8822B(x)                                        \
+	(((x) & BIT_MASK_MACID31_0_PKTSLEEP_8822B)                             \
+	 << BIT_SHIFT_MACID31_0_PKTSLEEP_8822B)
+#define BIT_GET_MACID31_0_PKTSLEEP_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_MACID31_0_PKTSLEEP_8822B) &                         \
+	 BIT_MASK_MACID31_0_PKTSLEEP_8822B)
+
+/* 2 REG_HW_SEQ0_8822B */
+
+#define BIT_SHIFT_HW_SSN_SEQ0_8822B 0
+#define BIT_MASK_HW_SSN_SEQ0_8822B 0xfff
+#define BIT_HW_SSN_SEQ0_8822B(x)                                               \
+	(((x) & BIT_MASK_HW_SSN_SEQ0_8822B) << BIT_SHIFT_HW_SSN_SEQ0_8822B)
+#define BIT_GET_HW_SSN_SEQ0_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_HW_SSN_SEQ0_8822B) & BIT_MASK_HW_SSN_SEQ0_8822B)
+
+/* 2 REG_HW_SEQ1_8822B */
+
+#define BIT_SHIFT_HW_SSN_SEQ1_8822B 0
+#define BIT_MASK_HW_SSN_SEQ1_8822B 0xfff
+#define BIT_HW_SSN_SEQ1_8822B(x)                                               \
+	(((x) & BIT_MASK_HW_SSN_SEQ1_8822B) << BIT_SHIFT_HW_SSN_SEQ1_8822B)
+#define BIT_GET_HW_SSN_SEQ1_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_HW_SSN_SEQ1_8822B) & BIT_MASK_HW_SSN_SEQ1_8822B)
+
+/* 2 REG_HW_SEQ2_8822B */
+
+#define BIT_SHIFT_HW_SSN_SEQ2_8822B 0
+#define BIT_MASK_HW_SSN_SEQ2_8822B 0xfff
+#define BIT_HW_SSN_SEQ2_8822B(x)                                               \
+	(((x) & BIT_MASK_HW_SSN_SEQ2_8822B) << BIT_SHIFT_HW_SSN_SEQ2_8822B)
+#define BIT_GET_HW_SSN_SEQ2_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_HW_SSN_SEQ2_8822B) & BIT_MASK_HW_SSN_SEQ2_8822B)
+
+/* 2 REG_HW_SEQ3_8822B */
+
+#define BIT_SHIFT_HW_SSN_SEQ3_8822B 0
+#define BIT_MASK_HW_SSN_SEQ3_8822B 0xfff
+#define BIT_HW_SSN_SEQ3_8822B(x)                                               \
+	(((x) & BIT_MASK_HW_SSN_SEQ3_8822B) << BIT_SHIFT_HW_SSN_SEQ3_8822B)
+#define BIT_GET_HW_SSN_SEQ3_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_HW_SSN_SEQ3_8822B) & BIT_MASK_HW_SSN_SEQ3_8822B)
+
+/* 2 REG_NULL_PKT_STATUS_V1_8822B */
+
+#define BIT_SHIFT_PTCL_TOTAL_PG_V2_8822B 2
+#define BIT_MASK_PTCL_TOTAL_PG_V2_8822B 0x3fff
+#define BIT_PTCL_TOTAL_PG_V2_8822B(x)                                          \
+	(((x) & BIT_MASK_PTCL_TOTAL_PG_V2_8822B)                               \
+	 << BIT_SHIFT_PTCL_TOTAL_PG_V2_8822B)
+#define BIT_GET_PTCL_TOTAL_PG_V2_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_PTCL_TOTAL_PG_V2_8822B) &                           \
+	 BIT_MASK_PTCL_TOTAL_PG_V2_8822B)
+
+#define BIT_TX_NULL_1_8822B BIT(1)
+#define BIT_TX_NULL_0_8822B BIT(0)
+
+/* 2 REG_PTCL_ERR_STATUS_8822B */
+#define BIT_PTCL_RATE_TABLE_INVALID_8822B BIT(7)
+#define BIT_FTM_T2R_ERROR_8822B BIT(6)
+#define BIT_PTCL_ERR0_8822B BIT(5)
+#define BIT_PTCL_ERR1_8822B BIT(4)
+#define BIT_PTCL_ERR2_8822B BIT(3)
+#define BIT_PTCL_ERR3_8822B BIT(2)
+#define BIT_PTCL_ERR4_8822B BIT(1)
+#define BIT_PTCL_ERR5_8822B BIT(0)
+
+/* 2 REG_NULL_PKT_STATUS_EXTEND_8822B */
+#define BIT_CLI3_TX_NULL_1_8822B BIT(7)
+#define BIT_CLI3_TX_NULL_0_8822B BIT(6)
+#define BIT_CLI2_TX_NULL_1_8822B BIT(5)
+#define BIT_CLI2_TX_NULL_0_8822B BIT(4)
+#define BIT_CLI1_TX_NULL_1_8822B BIT(3)
+#define BIT_CLI1_TX_NULL_0_8822B BIT(2)
+#define BIT_CLI0_TX_NULL_1_8822B BIT(1)
+#define BIT_CLI0_TX_NULL_0_8822B BIT(0)
+
+/* 2 REG_VIDEO_ENHANCEMENT_FUN_8822B */
+#define BIT_VIDEO_JUST_DROP_8822B BIT(1)
+#define BIT_VIDEO_ENHANCEMENT_FUN_EN_8822B BIT(0)
+
+/* 2 REG_BT_POLLUTE_PKT_CNT_8822B */
+
+#define BIT_SHIFT_BT_POLLUTE_PKT_CNT_8822B 0
+#define BIT_MASK_BT_POLLUTE_PKT_CNT_8822B 0xffff
+#define BIT_BT_POLLUTE_PKT_CNT_8822B(x)                                        \
+	(((x) & BIT_MASK_BT_POLLUTE_PKT_CNT_8822B)                             \
+	 << BIT_SHIFT_BT_POLLUTE_PKT_CNT_8822B)
+#define BIT_GET_BT_POLLUTE_PKT_CNT_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_BT_POLLUTE_PKT_CNT_8822B) &                         \
+	 BIT_MASK_BT_POLLUTE_PKT_CNT_8822B)
+
+/* 2 REG_NOT_VALID_8822B */
+
+/* 2 REG_PTCL_DBG_8822B */
+
+#define BIT_SHIFT_PTCL_DBG_8822B 0
+#define BIT_MASK_PTCL_DBG_8822B 0xffffffffL
+#define BIT_PTCL_DBG_8822B(x)                                                  \
+	(((x) & BIT_MASK_PTCL_DBG_8822B) << BIT_SHIFT_PTCL_DBG_8822B)
+#define BIT_GET_PTCL_DBG_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_PTCL_DBG_8822B) & BIT_MASK_PTCL_DBG_8822B)
+
+/* 2 REG_NOT_VALID_8822B */
+
+/* 2 REG_CPUMGQ_TIMER_CTRL2_8822B */
+
+#define BIT_SHIFT_TRI_HEAD_ADDR_8822B 16
+#define BIT_MASK_TRI_HEAD_ADDR_8822B 0xfff
+#define BIT_TRI_HEAD_ADDR_8822B(x)                                             \
+	(((x) & BIT_MASK_TRI_HEAD_ADDR_8822B) << BIT_SHIFT_TRI_HEAD_ADDR_8822B)
+#define BIT_GET_TRI_HEAD_ADDR_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_TRI_HEAD_ADDR_8822B) & BIT_MASK_TRI_HEAD_ADDR_8822B)
+
+#define BIT_DROP_TH_EN_8822B BIT(8)
+
+#define BIT_SHIFT_DROP_TH_8822B 0
+#define BIT_MASK_DROP_TH_8822B 0xff
+#define BIT_DROP_TH_8822B(x)                                                   \
+	(((x) & BIT_MASK_DROP_TH_8822B) << BIT_SHIFT_DROP_TH_8822B)
+#define BIT_GET_DROP_TH_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_DROP_TH_8822B) & BIT_MASK_DROP_TH_8822B)
+
+/* 2 REG_NOT_VALID_8822B */
+
+/* 2 REG_DUMMY_PAGE4_V1_8822B */
+#define BIT_BCN_EN_EXTHWSEQ_8822B BIT(1)
+#define BIT_BCN_EN_HWSEQ_8822B BIT(0)
+
+/* 2 REG_MOREDATA_8822B */
+#define BIT_MOREDATA_CTRL2_EN_V1_8822B BIT(3)
+#define BIT_MOREDATA_CTRL1_EN_V1_8822B BIT(2)
+#define BIT_PKTIN_MOREDATA_REPLACE_ENABLE_V1_8822B BIT(0)
+
+/* 2 REG_NOT_VALID_8822B */
+
+/* 2 REG_Q0_Q1_INFO_8822B */
+#define BIT_QUEUE_MACID_AC_NOT_THE_SAME_8822B BIT(31)
+
+#define BIT_SHIFT_GTAB_ID_8822B 28
+#define BIT_MASK_GTAB_ID_8822B 0x7
+#define BIT_GTAB_ID_8822B(x)                                                   \
+	(((x) & BIT_MASK_GTAB_ID_8822B) << BIT_SHIFT_GTAB_ID_8822B)
+#define BIT_GET_GTAB_ID_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_GTAB_ID_8822B) & BIT_MASK_GTAB_ID_8822B)
+
+#define BIT_SHIFT_AC1_PKT_INFO_8822B 16
+#define BIT_MASK_AC1_PKT_INFO_8822B 0xfff
+#define BIT_AC1_PKT_INFO_8822B(x)                                              \
+	(((x) & BIT_MASK_AC1_PKT_INFO_8822B) << BIT_SHIFT_AC1_PKT_INFO_8822B)
+#define BIT_GET_AC1_PKT_INFO_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_AC1_PKT_INFO_8822B) & BIT_MASK_AC1_PKT_INFO_8822B)
+
+#define BIT_QUEUE_MACID_AC_NOT_THE_SAME_V1_8822B BIT(15)
+
+#define BIT_SHIFT_GTAB_ID_V1_8822B 12
+#define BIT_MASK_GTAB_ID_V1_8822B 0x7
+#define BIT_GTAB_ID_V1_8822B(x)                                                \
+	(((x) & BIT_MASK_GTAB_ID_V1_8822B) << BIT_SHIFT_GTAB_ID_V1_8822B)
+#define BIT_GET_GTAB_ID_V1_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_GTAB_ID_V1_8822B) & BIT_MASK_GTAB_ID_V1_8822B)
+
+#define BIT_SHIFT_AC0_PKT_INFO_8822B 0
+#define BIT_MASK_AC0_PKT_INFO_8822B 0xfff
+#define BIT_AC0_PKT_INFO_8822B(x)                                              \
+	(((x) & BIT_MASK_AC0_PKT_INFO_8822B) << BIT_SHIFT_AC0_PKT_INFO_8822B)
+#define BIT_GET_AC0_PKT_INFO_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_AC0_PKT_INFO_8822B) & BIT_MASK_AC0_PKT_INFO_8822B)
+
+/* 2 REG_Q2_Q3_INFO_8822B */
+#define BIT_QUEUE_MACID_AC_NOT_THE_SAME_8822B BIT(31)
+
+#define BIT_SHIFT_GTAB_ID_8822B 28
+#define BIT_MASK_GTAB_ID_8822B 0x7
+#define BIT_GTAB_ID_8822B(x)                                                   \
+	(((x) & BIT_MASK_GTAB_ID_8822B) << BIT_SHIFT_GTAB_ID_8822B)
+#define BIT_GET_GTAB_ID_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_GTAB_ID_8822B) & BIT_MASK_GTAB_ID_8822B)
+
+#define BIT_SHIFT_AC3_PKT_INFO_8822B 16
+#define BIT_MASK_AC3_PKT_INFO_8822B 0xfff
+#define BIT_AC3_PKT_INFO_8822B(x)                                              \
+	(((x) & BIT_MASK_AC3_PKT_INFO_8822B) << BIT_SHIFT_AC3_PKT_INFO_8822B)
+#define BIT_GET_AC3_PKT_INFO_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_AC3_PKT_INFO_8822B) & BIT_MASK_AC3_PKT_INFO_8822B)
+
+#define BIT_QUEUE_MACID_AC_NOT_THE_SAME_V1_8822B BIT(15)
+
+#define BIT_SHIFT_GTAB_ID_V1_8822B 12
+#define BIT_MASK_GTAB_ID_V1_8822B 0x7
+#define BIT_GTAB_ID_V1_8822B(x)                                                \
+	(((x) & BIT_MASK_GTAB_ID_V1_8822B) << BIT_SHIFT_GTAB_ID_V1_8822B)
+#define BIT_GET_GTAB_ID_V1_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_GTAB_ID_V1_8822B) & BIT_MASK_GTAB_ID_V1_8822B)
+
+#define BIT_SHIFT_AC2_PKT_INFO_8822B 0
+#define BIT_MASK_AC2_PKT_INFO_8822B 0xfff
+#define BIT_AC2_PKT_INFO_8822B(x)                                              \
+	(((x) & BIT_MASK_AC2_PKT_INFO_8822B) << BIT_SHIFT_AC2_PKT_INFO_8822B)
+#define BIT_GET_AC2_PKT_INFO_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_AC2_PKT_INFO_8822B) & BIT_MASK_AC2_PKT_INFO_8822B)
+
+/* 2 REG_Q4_Q5_INFO_8822B */
+#define BIT_QUEUE_MACID_AC_NOT_THE_SAME_8822B BIT(31)
+
+#define BIT_SHIFT_GTAB_ID_8822B 28
+#define BIT_MASK_GTAB_ID_8822B 0x7
+#define BIT_GTAB_ID_8822B(x)                                                   \
+	(((x) & BIT_MASK_GTAB_ID_8822B) << BIT_SHIFT_GTAB_ID_8822B)
+#define BIT_GET_GTAB_ID_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_GTAB_ID_8822B) & BIT_MASK_GTAB_ID_8822B)
+
+#define BIT_SHIFT_AC5_PKT_INFO_8822B 16
+#define BIT_MASK_AC5_PKT_INFO_8822B 0xfff
+#define BIT_AC5_PKT_INFO_8822B(x)                                              \
+	(((x) & BIT_MASK_AC5_PKT_INFO_8822B) << BIT_SHIFT_AC5_PKT_INFO_8822B)
+#define BIT_GET_AC5_PKT_INFO_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_AC5_PKT_INFO_8822B) & BIT_MASK_AC5_PKT_INFO_8822B)
+
+#define BIT_QUEUE_MACID_AC_NOT_THE_SAME_V1_8822B BIT(15)
+
+#define BIT_SHIFT_GTAB_ID_V1_8822B 12
+#define BIT_MASK_GTAB_ID_V1_8822B 0x7
+#define BIT_GTAB_ID_V1_8822B(x)                                                \
+	(((x) & BIT_MASK_GTAB_ID_V1_8822B) << BIT_SHIFT_GTAB_ID_V1_8822B)
+#define BIT_GET_GTAB_ID_V1_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_GTAB_ID_V1_8822B) & BIT_MASK_GTAB_ID_V1_8822B)
+
+#define BIT_SHIFT_AC4_PKT_INFO_8822B 0
+#define BIT_MASK_AC4_PKT_INFO_8822B 0xfff
+#define BIT_AC4_PKT_INFO_8822B(x)                                              \
+	(((x) & BIT_MASK_AC4_PKT_INFO_8822B) << BIT_SHIFT_AC4_PKT_INFO_8822B)
+#define BIT_GET_AC4_PKT_INFO_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_AC4_PKT_INFO_8822B) & BIT_MASK_AC4_PKT_INFO_8822B)
+
+/* 2 REG_Q6_Q7_INFO_8822B */
+#define BIT_QUEUE_MACID_AC_NOT_THE_SAME_8822B BIT(31)
+
+#define BIT_SHIFT_GTAB_ID_8822B 28
+#define BIT_MASK_GTAB_ID_8822B 0x7
+#define BIT_GTAB_ID_8822B(x)                                                   \
+	(((x) & BIT_MASK_GTAB_ID_8822B) << BIT_SHIFT_GTAB_ID_8822B)
+#define BIT_GET_GTAB_ID_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_GTAB_ID_8822B) & BIT_MASK_GTAB_ID_8822B)
+
+#define BIT_SHIFT_AC7_PKT_INFO_8822B 16
+#define BIT_MASK_AC7_PKT_INFO_8822B 0xfff
+#define BIT_AC7_PKT_INFO_8822B(x)                                              \
+	(((x) & BIT_MASK_AC7_PKT_INFO_8822B) << BIT_SHIFT_AC7_PKT_INFO_8822B)
+#define BIT_GET_AC7_PKT_INFO_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_AC7_PKT_INFO_8822B) & BIT_MASK_AC7_PKT_INFO_8822B)
+
+#define BIT_QUEUE_MACID_AC_NOT_THE_SAME_V1_8822B BIT(15)
+
+#define BIT_SHIFT_GTAB_ID_V1_8822B 12
+#define BIT_MASK_GTAB_ID_V1_8822B 0x7
+#define BIT_GTAB_ID_V1_8822B(x)                                                \
+	(((x) & BIT_MASK_GTAB_ID_V1_8822B) << BIT_SHIFT_GTAB_ID_V1_8822B)
+#define BIT_GET_GTAB_ID_V1_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_GTAB_ID_V1_8822B) & BIT_MASK_GTAB_ID_V1_8822B)
+
+#define BIT_SHIFT_AC6_PKT_INFO_8822B 0
+#define BIT_MASK_AC6_PKT_INFO_8822B 0xfff
+#define BIT_AC6_PKT_INFO_8822B(x)                                              \
+	(((x) & BIT_MASK_AC6_PKT_INFO_8822B) << BIT_SHIFT_AC6_PKT_INFO_8822B)
+#define BIT_GET_AC6_PKT_INFO_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_AC6_PKT_INFO_8822B) & BIT_MASK_AC6_PKT_INFO_8822B)
+
+/* 2 REG_MGQ_HIQ_INFO_8822B */
+
+#define BIT_SHIFT_HIQ_PKT_INFO_8822B 16
+#define BIT_MASK_HIQ_PKT_INFO_8822B 0xfff
+#define BIT_HIQ_PKT_INFO_8822B(x)                                              \
+	(((x) & BIT_MASK_HIQ_PKT_INFO_8822B) << BIT_SHIFT_HIQ_PKT_INFO_8822B)
+#define BIT_GET_HIQ_PKT_INFO_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_HIQ_PKT_INFO_8822B) & BIT_MASK_HIQ_PKT_INFO_8822B)
+
+#define BIT_SHIFT_MGQ_PKT_INFO_8822B 0
+#define BIT_MASK_MGQ_PKT_INFO_8822B 0xfff
+#define BIT_MGQ_PKT_INFO_8822B(x)                                              \
+	(((x) & BIT_MASK_MGQ_PKT_INFO_8822B) << BIT_SHIFT_MGQ_PKT_INFO_8822B)
+#define BIT_GET_MGQ_PKT_INFO_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_MGQ_PKT_INFO_8822B) & BIT_MASK_MGQ_PKT_INFO_8822B)
+
+/* 2 REG_CMDQ_BCNQ_INFO_8822B */
+
+#define BIT_SHIFT_CMDQ_PKT_INFO_8822B 16
+#define BIT_MASK_CMDQ_PKT_INFO_8822B 0xfff
+#define BIT_CMDQ_PKT_INFO_8822B(x)                                             \
+	(((x) & BIT_MASK_CMDQ_PKT_INFO_8822B) << BIT_SHIFT_CMDQ_PKT_INFO_8822B)
+#define BIT_GET_CMDQ_PKT_INFO_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_CMDQ_PKT_INFO_8822B) & BIT_MASK_CMDQ_PKT_INFO_8822B)
+
+#define BIT_SHIFT_BCNQ_PKT_INFO_8822B 0
+#define BIT_MASK_BCNQ_PKT_INFO_8822B 0xfff
+#define BIT_BCNQ_PKT_INFO_8822B(x)                                             \
+	(((x) & BIT_MASK_BCNQ_PKT_INFO_8822B) << BIT_SHIFT_BCNQ_PKT_INFO_8822B)
+#define BIT_GET_BCNQ_PKT_INFO_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_BCNQ_PKT_INFO_8822B) & BIT_MASK_BCNQ_PKT_INFO_8822B)
+
+/* 2 REG_USEREG_SETTING_8822B */
+#define BIT_NDPA_USEREG_8822B BIT(21)
+
+#define BIT_SHIFT_RETRY_USEREG_8822B 19
+#define BIT_MASK_RETRY_USEREG_8822B 0x3
+#define BIT_RETRY_USEREG_8822B(x)                                              \
+	(((x) & BIT_MASK_RETRY_USEREG_8822B) << BIT_SHIFT_RETRY_USEREG_8822B)
+#define BIT_GET_RETRY_USEREG_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_RETRY_USEREG_8822B) & BIT_MASK_RETRY_USEREG_8822B)
+
+#define BIT_SHIFT_TRYPKT_USEREG_8822B 17
+#define BIT_MASK_TRYPKT_USEREG_8822B 0x3
+#define BIT_TRYPKT_USEREG_8822B(x)                                             \
+	(((x) & BIT_MASK_TRYPKT_USEREG_8822B) << BIT_SHIFT_TRYPKT_USEREG_8822B)
+#define BIT_GET_TRYPKT_USEREG_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_TRYPKT_USEREG_8822B) & BIT_MASK_TRYPKT_USEREG_8822B)
+
+#define BIT_CTLPKT_USEREG_8822B BIT(16)
+
+/* 2 REG_AESIV_SETTING_8822B */
+
+#define BIT_SHIFT_AESIV_OFFSET_8822B 0
+#define BIT_MASK_AESIV_OFFSET_8822B 0xfff
+#define BIT_AESIV_OFFSET_8822B(x)                                              \
+	(((x) & BIT_MASK_AESIV_OFFSET_8822B) << BIT_SHIFT_AESIV_OFFSET_8822B)
+#define BIT_GET_AESIV_OFFSET_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_AESIV_OFFSET_8822B) & BIT_MASK_AESIV_OFFSET_8822B)
+
+/* 2 REG_BF0_TIME_SETTING_8822B */
+#define BIT_BF0_TIMER_SET_8822B BIT(31)
+#define BIT_BF0_TIMER_CLR_8822B BIT(30)
+#define BIT_BF0_UPDATE_EN_8822B BIT(29)
+#define BIT_BF0_TIMER_EN_8822B BIT(28)
+
+#define BIT_SHIFT_BF0_PRETIME_OVER_8822B 16
+#define BIT_MASK_BF0_PRETIME_OVER_8822B 0xfff
+#define BIT_BF0_PRETIME_OVER_8822B(x)                                          \
+	(((x) & BIT_MASK_BF0_PRETIME_OVER_8822B)                               \
+	 << BIT_SHIFT_BF0_PRETIME_OVER_8822B)
+#define BIT_GET_BF0_PRETIME_OVER_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_BF0_PRETIME_OVER_8822B) &                           \
+	 BIT_MASK_BF0_PRETIME_OVER_8822B)
+
+#define BIT_SHIFT_BF0_LIFETIME_8822B 0
+#define BIT_MASK_BF0_LIFETIME_8822B 0xffff
+#define BIT_BF0_LIFETIME_8822B(x)                                              \
+	(((x) & BIT_MASK_BF0_LIFETIME_8822B) << BIT_SHIFT_BF0_LIFETIME_8822B)
+#define BIT_GET_BF0_LIFETIME_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_BF0_LIFETIME_8822B) & BIT_MASK_BF0_LIFETIME_8822B)
+
+/* 2 REG_BF1_TIME_SETTING_8822B */
+#define BIT_BF1_TIMER_SET_8822B BIT(31)
+#define BIT_BF1_TIMER_CLR_8822B BIT(30)
+#define BIT_BF1_UPDATE_EN_8822B BIT(29)
+#define BIT_BF1_TIMER_EN_8822B BIT(28)
+
+#define BIT_SHIFT_BF1_PRETIME_OVER_8822B 16
+#define BIT_MASK_BF1_PRETIME_OVER_8822B 0xfff
+#define BIT_BF1_PRETIME_OVER_8822B(x)                                          \
+	(((x) & BIT_MASK_BF1_PRETIME_OVER_8822B)                               \
+	 << BIT_SHIFT_BF1_PRETIME_OVER_8822B)
+#define BIT_GET_BF1_PRETIME_OVER_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_BF1_PRETIME_OVER_8822B) &                           \
+	 BIT_MASK_BF1_PRETIME_OVER_8822B)
+
+#define BIT_SHIFT_BF1_LIFETIME_8822B 0
+#define BIT_MASK_BF1_LIFETIME_8822B 0xffff
+#define BIT_BF1_LIFETIME_8822B(x)                                              \
+	(((x) & BIT_MASK_BF1_LIFETIME_8822B) << BIT_SHIFT_BF1_LIFETIME_8822B)
+#define BIT_GET_BF1_LIFETIME_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_BF1_LIFETIME_8822B) & BIT_MASK_BF1_LIFETIME_8822B)
+
+/* 2 REG_BF_TIMEOUT_EN_8822B */
+#define BIT_EN_VHT_LDPC_8822B BIT(9)
+#define BIT_EN_HT_LDPC_8822B BIT(8)
+#define BIT_BF1_TIMEOUT_EN_8822B BIT(1)
+#define BIT_BF0_TIMEOUT_EN_8822B BIT(0)
+
+/* 2 REG_MACID_RELEASE0_8822B */
+
+#define BIT_SHIFT_MACID31_0_RELEASE_8822B 0
+#define BIT_MASK_MACID31_0_RELEASE_8822B 0xffffffffL
+#define BIT_MACID31_0_RELEASE_8822B(x)                                         \
+	(((x) & BIT_MASK_MACID31_0_RELEASE_8822B)                              \
+	 << BIT_SHIFT_MACID31_0_RELEASE_8822B)
+#define BIT_GET_MACID31_0_RELEASE_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_MACID31_0_RELEASE_8822B) &                          \
+	 BIT_MASK_MACID31_0_RELEASE_8822B)
+
+/* 2 REG_MACID_RELEASE1_8822B */
+
+#define BIT_SHIFT_MACID63_32_RELEASE_8822B 0
+#define BIT_MASK_MACID63_32_RELEASE_8822B 0xffffffffL
+#define BIT_MACID63_32_RELEASE_8822B(x)                                        \
+	(((x) & BIT_MASK_MACID63_32_RELEASE_8822B)                             \
+	 << BIT_SHIFT_MACID63_32_RELEASE_8822B)
+#define BIT_GET_MACID63_32_RELEASE_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_MACID63_32_RELEASE_8822B) &                         \
+	 BIT_MASK_MACID63_32_RELEASE_8822B)
+
+/* 2 REG_MACID_RELEASE2_8822B */
+
+#define BIT_SHIFT_MACID95_64_RELEASE_8822B 0
+#define BIT_MASK_MACID95_64_RELEASE_8822B 0xffffffffL
+#define BIT_MACID95_64_RELEASE_8822B(x)                                        \
+	(((x) & BIT_MASK_MACID95_64_RELEASE_8822B)                             \
+	 << BIT_SHIFT_MACID95_64_RELEASE_8822B)
+#define BIT_GET_MACID95_64_RELEASE_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_MACID95_64_RELEASE_8822B) &                         \
+	 BIT_MASK_MACID95_64_RELEASE_8822B)
+
+/* 2 REG_MACID_RELEASE3_8822B */
+
+#define BIT_SHIFT_MACID127_96_RELEASE_8822B 0
+#define BIT_MASK_MACID127_96_RELEASE_8822B 0xffffffffL
+#define BIT_MACID127_96_RELEASE_8822B(x)                                       \
+	(((x) & BIT_MASK_MACID127_96_RELEASE_8822B)                            \
+	 << BIT_SHIFT_MACID127_96_RELEASE_8822B)
+#define BIT_GET_MACID127_96_RELEASE_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_MACID127_96_RELEASE_8822B) &                        \
+	 BIT_MASK_MACID127_96_RELEASE_8822B)
+
+/* 2 REG_MACID_RELEASE_SETTING_8822B */
+#define BIT_MACID_VALUE_8822B BIT(7)
+
+#define BIT_SHIFT_MACID_OFFSET_8822B 0
+#define BIT_MASK_MACID_OFFSET_8822B 0x7f
+#define BIT_MACID_OFFSET_8822B(x)                                              \
+	(((x) & BIT_MASK_MACID_OFFSET_8822B) << BIT_SHIFT_MACID_OFFSET_8822B)
+#define BIT_GET_MACID_OFFSET_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_MACID_OFFSET_8822B) & BIT_MASK_MACID_OFFSET_8822B)
+
+/* 2 REG_FAST_EDCA_VOVI_SETTING_8822B */
+
+#define BIT_SHIFT_VI_FAST_EDCA_TO_8822B 24
+#define BIT_MASK_VI_FAST_EDCA_TO_8822B 0xff
+#define BIT_VI_FAST_EDCA_TO_8822B(x)                                           \
+	(((x) & BIT_MASK_VI_FAST_EDCA_TO_8822B)                                \
+	 << BIT_SHIFT_VI_FAST_EDCA_TO_8822B)
+#define BIT_GET_VI_FAST_EDCA_TO_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_VI_FAST_EDCA_TO_8822B) &                            \
+	 BIT_MASK_VI_FAST_EDCA_TO_8822B)
+
+#define BIT_VI_THRESHOLD_SEL_8822B BIT(23)
+
+#define BIT_SHIFT_VI_FAST_EDCA_PKT_TH_8822B 16
+#define BIT_MASK_VI_FAST_EDCA_PKT_TH_8822B 0x7f
+#define BIT_VI_FAST_EDCA_PKT_TH_8822B(x)                                       \
+	(((x) & BIT_MASK_VI_FAST_EDCA_PKT_TH_8822B)                            \
+	 << BIT_SHIFT_VI_FAST_EDCA_PKT_TH_8822B)
+#define BIT_GET_VI_FAST_EDCA_PKT_TH_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_VI_FAST_EDCA_PKT_TH_8822B) &                        \
+	 BIT_MASK_VI_FAST_EDCA_PKT_TH_8822B)
+
+#define BIT_SHIFT_VO_FAST_EDCA_TO_8822B 8
+#define BIT_MASK_VO_FAST_EDCA_TO_8822B 0xff
+#define BIT_VO_FAST_EDCA_TO_8822B(x)                                           \
+	(((x) & BIT_MASK_VO_FAST_EDCA_TO_8822B)                                \
+	 << BIT_SHIFT_VO_FAST_EDCA_TO_8822B)
+#define BIT_GET_VO_FAST_EDCA_TO_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_VO_FAST_EDCA_TO_8822B) &                            \
+	 BIT_MASK_VO_FAST_EDCA_TO_8822B)
+
+#define BIT_VO_THRESHOLD_SEL_8822B BIT(7)
+
+#define BIT_SHIFT_VO_FAST_EDCA_PKT_TH_8822B 0
+#define BIT_MASK_VO_FAST_EDCA_PKT_TH_8822B 0x7f
+#define BIT_VO_FAST_EDCA_PKT_TH_8822B(x)                                       \
+	(((x) & BIT_MASK_VO_FAST_EDCA_PKT_TH_8822B)                            \
+	 << BIT_SHIFT_VO_FAST_EDCA_PKT_TH_8822B)
+#define BIT_GET_VO_FAST_EDCA_PKT_TH_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_VO_FAST_EDCA_PKT_TH_8822B) &                        \
+	 BIT_MASK_VO_FAST_EDCA_PKT_TH_8822B)
+
+/* 2 REG_FAST_EDCA_BEBK_SETTING_8822B */
+
+#define BIT_SHIFT_BK_FAST_EDCA_TO_8822B 24
+#define BIT_MASK_BK_FAST_EDCA_TO_8822B 0xff
+#define BIT_BK_FAST_EDCA_TO_8822B(x)                                           \
+	(((x) & BIT_MASK_BK_FAST_EDCA_TO_8822B)                                \
+	 << BIT_SHIFT_BK_FAST_EDCA_TO_8822B)
+#define BIT_GET_BK_FAST_EDCA_TO_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_BK_FAST_EDCA_TO_8822B) &                            \
+	 BIT_MASK_BK_FAST_EDCA_TO_8822B)
+
+#define BIT_BK_THRESHOLD_SEL_8822B BIT(23)
+
+#define BIT_SHIFT_BK_FAST_EDCA_PKT_TH_8822B 16
+#define BIT_MASK_BK_FAST_EDCA_PKT_TH_8822B 0x7f
+#define BIT_BK_FAST_EDCA_PKT_TH_8822B(x)                                       \
+	(((x) & BIT_MASK_BK_FAST_EDCA_PKT_TH_8822B)                            \
+	 << BIT_SHIFT_BK_FAST_EDCA_PKT_TH_8822B)
+#define BIT_GET_BK_FAST_EDCA_PKT_TH_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_BK_FAST_EDCA_PKT_TH_8822B) &                        \
+	 BIT_MASK_BK_FAST_EDCA_PKT_TH_8822B)
+
+#define BIT_SHIFT_BE_FAST_EDCA_TO_8822B 8
+#define BIT_MASK_BE_FAST_EDCA_TO_8822B 0xff
+#define BIT_BE_FAST_EDCA_TO_8822B(x)                                           \
+	(((x) & BIT_MASK_BE_FAST_EDCA_TO_8822B)                                \
+	 << BIT_SHIFT_BE_FAST_EDCA_TO_8822B)
+#define BIT_GET_BE_FAST_EDCA_TO_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_BE_FAST_EDCA_TO_8822B) &                            \
+	 BIT_MASK_BE_FAST_EDCA_TO_8822B)
+
+#define BIT_BE_THRESHOLD_SEL_8822B BIT(7)
+
+#define BIT_SHIFT_BE_FAST_EDCA_PKT_TH_8822B 0
+#define BIT_MASK_BE_FAST_EDCA_PKT_TH_8822B 0x7f
+#define BIT_BE_FAST_EDCA_PKT_TH_8822B(x)                                       \
+	(((x) & BIT_MASK_BE_FAST_EDCA_PKT_TH_8822B)                            \
+	 << BIT_SHIFT_BE_FAST_EDCA_PKT_TH_8822B)
+#define BIT_GET_BE_FAST_EDCA_PKT_TH_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_BE_FAST_EDCA_PKT_TH_8822B) &                        \
+	 BIT_MASK_BE_FAST_EDCA_PKT_TH_8822B)
+
+/* 2 REG_MACID_DROP0_8822B */
+
+#define BIT_SHIFT_MACID31_0_DROP_8822B 0
+#define BIT_MASK_MACID31_0_DROP_8822B 0xffffffffL
+#define BIT_MACID31_0_DROP_8822B(x)                                            \
+	(((x) & BIT_MASK_MACID31_0_DROP_8822B)                                 \
+	 << BIT_SHIFT_MACID31_0_DROP_8822B)
+#define BIT_GET_MACID31_0_DROP_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_MACID31_0_DROP_8822B) &                             \
+	 BIT_MASK_MACID31_0_DROP_8822B)
+
+/* 2 REG_MACID_DROP1_8822B */
+
+#define BIT_SHIFT_MACID63_32_DROP_8822B 0
+#define BIT_MASK_MACID63_32_DROP_8822B 0xffffffffL
+#define BIT_MACID63_32_DROP_8822B(x)                                           \
+	(((x) & BIT_MASK_MACID63_32_DROP_8822B)                                \
+	 << BIT_SHIFT_MACID63_32_DROP_8822B)
+#define BIT_GET_MACID63_32_DROP_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_MACID63_32_DROP_8822B) &                            \
+	 BIT_MASK_MACID63_32_DROP_8822B)
+
+/* 2 REG_MACID_DROP2_8822B */
+
+#define BIT_SHIFT_MACID95_64_DROP_8822B 0
+#define BIT_MASK_MACID95_64_DROP_8822B 0xffffffffL
+#define BIT_MACID95_64_DROP_8822B(x)                                           \
+	(((x) & BIT_MASK_MACID95_64_DROP_8822B)                                \
+	 << BIT_SHIFT_MACID95_64_DROP_8822B)
+#define BIT_GET_MACID95_64_DROP_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_MACID95_64_DROP_8822B) &                            \
+	 BIT_MASK_MACID95_64_DROP_8822B)
+
+/* 2 REG_MACID_DROP3_8822B */
+
+#define BIT_SHIFT_MACID127_96_DROP_8822B 0
+#define BIT_MASK_MACID127_96_DROP_8822B 0xffffffffL
+#define BIT_MACID127_96_DROP_8822B(x)                                          \
+	(((x) & BIT_MASK_MACID127_96_DROP_8822B)                               \
+	 << BIT_SHIFT_MACID127_96_DROP_8822B)
+#define BIT_GET_MACID127_96_DROP_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_MACID127_96_DROP_8822B) &                           \
+	 BIT_MASK_MACID127_96_DROP_8822B)
+
+/* 2 REG_R_MACID_RELEASE_SUCCESS_0_8822B */
+
+#define BIT_SHIFT_R_MACID_RELEASE_SUCCESS_0_8822B 0
+#define BIT_MASK_R_MACID_RELEASE_SUCCESS_0_8822B 0xffffffffL
+#define BIT_R_MACID_RELEASE_SUCCESS_0_8822B(x)                                 \
+	(((x) & BIT_MASK_R_MACID_RELEASE_SUCCESS_0_8822B)                      \
+	 << BIT_SHIFT_R_MACID_RELEASE_SUCCESS_0_8822B)
+#define BIT_GET_R_MACID_RELEASE_SUCCESS_0_8822B(x)                             \
+	(((x) >> BIT_SHIFT_R_MACID_RELEASE_SUCCESS_0_8822B) &                  \
+	 BIT_MASK_R_MACID_RELEASE_SUCCESS_0_8822B)
+
+/* 2 REG_R_MACID_RELEASE_SUCCESS_1_8822B */
+
+#define BIT_SHIFT_R_MACID_RELEASE_SUCCESS_1_8822B 0
+#define BIT_MASK_R_MACID_RELEASE_SUCCESS_1_8822B 0xffffffffL
+#define BIT_R_MACID_RELEASE_SUCCESS_1_8822B(x)                                 \
+	(((x) & BIT_MASK_R_MACID_RELEASE_SUCCESS_1_8822B)                      \
+	 << BIT_SHIFT_R_MACID_RELEASE_SUCCESS_1_8822B)
+#define BIT_GET_R_MACID_RELEASE_SUCCESS_1_8822B(x)                             \
+	(((x) >> BIT_SHIFT_R_MACID_RELEASE_SUCCESS_1_8822B) &                  \
+	 BIT_MASK_R_MACID_RELEASE_SUCCESS_1_8822B)
+
+/* 2 REG_R_MACID_RELEASE_SUCCESS_2_8822B */
+
+#define BIT_SHIFT_R_MACID_RELEASE_SUCCESS_2_8822B 0
+#define BIT_MASK_R_MACID_RELEASE_SUCCESS_2_8822B 0xffffffffL
+#define BIT_R_MACID_RELEASE_SUCCESS_2_8822B(x)                                 \
+	(((x) & BIT_MASK_R_MACID_RELEASE_SUCCESS_2_8822B)                      \
+	 << BIT_SHIFT_R_MACID_RELEASE_SUCCESS_2_8822B)
+#define BIT_GET_R_MACID_RELEASE_SUCCESS_2_8822B(x)                             \
+	(((x) >> BIT_SHIFT_R_MACID_RELEASE_SUCCESS_2_8822B) &                  \
+	 BIT_MASK_R_MACID_RELEASE_SUCCESS_2_8822B)
+
+/* 2 REG_R_MACID_RELEASE_SUCCESS_3_8822B */
+
+#define BIT_SHIFT_R_MACID_RELEASE_SUCCESS_3_8822B 0
+#define BIT_MASK_R_MACID_RELEASE_SUCCESS_3_8822B 0xffffffffL
+#define BIT_R_MACID_RELEASE_SUCCESS_3_8822B(x)                                 \
+	(((x) & BIT_MASK_R_MACID_RELEASE_SUCCESS_3_8822B)                      \
+	 << BIT_SHIFT_R_MACID_RELEASE_SUCCESS_3_8822B)
+#define BIT_GET_R_MACID_RELEASE_SUCCESS_3_8822B(x)                             \
+	(((x) >> BIT_SHIFT_R_MACID_RELEASE_SUCCESS_3_8822B) &                  \
+	 BIT_MASK_R_MACID_RELEASE_SUCCESS_3_8822B)
+
+/* 2 REG_MGG_FIFO_CRTL_8822B */
+#define BIT_R_MGG_FIFO_EN_8822B BIT(31)
+
+#define BIT_SHIFT_R_MGG_FIFO_PG_SIZE_8822B 28
+#define BIT_MASK_R_MGG_FIFO_PG_SIZE_8822B 0x7
+#define BIT_R_MGG_FIFO_PG_SIZE_8822B(x)                                        \
+	(((x) & BIT_MASK_R_MGG_FIFO_PG_SIZE_8822B)                             \
+	 << BIT_SHIFT_R_MGG_FIFO_PG_SIZE_8822B)
+#define BIT_GET_R_MGG_FIFO_PG_SIZE_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_R_MGG_FIFO_PG_SIZE_8822B) &                         \
+	 BIT_MASK_R_MGG_FIFO_PG_SIZE_8822B)
+
+#define BIT_SHIFT_R_MGG_FIFO_START_PG_8822B 16
+#define BIT_MASK_R_MGG_FIFO_START_PG_8822B 0xfff
+#define BIT_R_MGG_FIFO_START_PG_8822B(x)                                       \
+	(((x) & BIT_MASK_R_MGG_FIFO_START_PG_8822B)                            \
+	 << BIT_SHIFT_R_MGG_FIFO_START_PG_8822B)
+#define BIT_GET_R_MGG_FIFO_START_PG_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_R_MGG_FIFO_START_PG_8822B) &                        \
+	 BIT_MASK_R_MGG_FIFO_START_PG_8822B)
+
+#define BIT_SHIFT_R_MGG_FIFO_SIZE_8822B 14
+#define BIT_MASK_R_MGG_FIFO_SIZE_8822B 0x3
+#define BIT_R_MGG_FIFO_SIZE_8822B(x)                                           \
+	(((x) & BIT_MASK_R_MGG_FIFO_SIZE_8822B)                                \
+	 << BIT_SHIFT_R_MGG_FIFO_SIZE_8822B)
+#define BIT_GET_R_MGG_FIFO_SIZE_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_R_MGG_FIFO_SIZE_8822B) &                            \
+	 BIT_MASK_R_MGG_FIFO_SIZE_8822B)
+
+#define BIT_R_MGG_FIFO_PAUSE_8822B BIT(13)
+
+#define BIT_SHIFT_R_MGG_FIFO_RPTR_8822B 8
+#define BIT_MASK_R_MGG_FIFO_RPTR_8822B 0x1f
+#define BIT_R_MGG_FIFO_RPTR_8822B(x)                                           \
+	(((x) & BIT_MASK_R_MGG_FIFO_RPTR_8822B)                                \
+	 << BIT_SHIFT_R_MGG_FIFO_RPTR_8822B)
+#define BIT_GET_R_MGG_FIFO_RPTR_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_R_MGG_FIFO_RPTR_8822B) &                            \
+	 BIT_MASK_R_MGG_FIFO_RPTR_8822B)
+
+#define BIT_R_MGG_FIFO_OV_8822B BIT(7)
+#define BIT_R_MGG_FIFO_WPTR_ERROR_8822B BIT(6)
+#define BIT_R_EN_CPU_LIFETIME_8822B BIT(5)
+
+#define BIT_SHIFT_R_MGG_FIFO_WPTR_8822B 0
+#define BIT_MASK_R_MGG_FIFO_WPTR_8822B 0x1f
+#define BIT_R_MGG_FIFO_WPTR_8822B(x)                                           \
+	(((x) & BIT_MASK_R_MGG_FIFO_WPTR_8822B)                                \
+	 << BIT_SHIFT_R_MGG_FIFO_WPTR_8822B)
+#define BIT_GET_R_MGG_FIFO_WPTR_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_R_MGG_FIFO_WPTR_8822B) &                            \
+	 BIT_MASK_R_MGG_FIFO_WPTR_8822B)
+
+/* 2 REG_MGG_FIFO_INT_8822B */
+
+#define BIT_SHIFT_R_MGG_FIFO_INT_FLAG_8822B 16
+#define BIT_MASK_R_MGG_FIFO_INT_FLAG_8822B 0xffff
+#define BIT_R_MGG_FIFO_INT_FLAG_8822B(x)                                       \
+	(((x) & BIT_MASK_R_MGG_FIFO_INT_FLAG_8822B)                            \
+	 << BIT_SHIFT_R_MGG_FIFO_INT_FLAG_8822B)
+#define BIT_GET_R_MGG_FIFO_INT_FLAG_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_R_MGG_FIFO_INT_FLAG_8822B) &                        \
+	 BIT_MASK_R_MGG_FIFO_INT_FLAG_8822B)
+
+#define BIT_SHIFT_R_MGG_FIFO_INT_MASK_8822B 0
+#define BIT_MASK_R_MGG_FIFO_INT_MASK_8822B 0xffff
+#define BIT_R_MGG_FIFO_INT_MASK_8822B(x)                                       \
+	(((x) & BIT_MASK_R_MGG_FIFO_INT_MASK_8822B)                            \
+	 << BIT_SHIFT_R_MGG_FIFO_INT_MASK_8822B)
+#define BIT_GET_R_MGG_FIFO_INT_MASK_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_R_MGG_FIFO_INT_MASK_8822B) &                        \
+	 BIT_MASK_R_MGG_FIFO_INT_MASK_8822B)
+
+/* 2 REG_MGG_FIFO_LIFETIME_8822B */
+
+#define BIT_SHIFT_R_MGG_FIFO_LIFETIME_8822B 16
+#define BIT_MASK_R_MGG_FIFO_LIFETIME_8822B 0xffff
+#define BIT_R_MGG_FIFO_LIFETIME_8822B(x)                                       \
+	(((x) & BIT_MASK_R_MGG_FIFO_LIFETIME_8822B)                            \
+	 << BIT_SHIFT_R_MGG_FIFO_LIFETIME_8822B)
+#define BIT_GET_R_MGG_FIFO_LIFETIME_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_R_MGG_FIFO_LIFETIME_8822B) &                        \
+	 BIT_MASK_R_MGG_FIFO_LIFETIME_8822B)
+
+#define BIT_SHIFT_R_MGG_FIFO_VALID_MAP_8822B 0
+#define BIT_MASK_R_MGG_FIFO_VALID_MAP_8822B 0xffff
+#define BIT_R_MGG_FIFO_VALID_MAP_8822B(x)                                      \
+	(((x) & BIT_MASK_R_MGG_FIFO_VALID_MAP_8822B)                           \
+	 << BIT_SHIFT_R_MGG_FIFO_VALID_MAP_8822B)
+#define BIT_GET_R_MGG_FIFO_VALID_MAP_8822B(x)                                  \
+	(((x) >> BIT_SHIFT_R_MGG_FIFO_VALID_MAP_8822B) &                       \
+	 BIT_MASK_R_MGG_FIFO_VALID_MAP_8822B)
+
+/* 2 REG_R_MACID_RELEASE_SUCCESS_CLEAR_OFFSET_8822B */
+
+#define BIT_SHIFT_R_MACID_RELEASE_SUCCESS_CLEAR_OFFSET_8822B 0
+#define BIT_MASK_R_MACID_RELEASE_SUCCESS_CLEAR_OFFSET_8822B 0x7f
+#define BIT_R_MACID_RELEASE_SUCCESS_CLEAR_OFFSET_8822B(x)                      \
+	(((x) & BIT_MASK_R_MACID_RELEASE_SUCCESS_CLEAR_OFFSET_8822B)           \
+	 << BIT_SHIFT_R_MACID_RELEASE_SUCCESS_CLEAR_OFFSET_8822B)
+#define BIT_GET_R_MACID_RELEASE_SUCCESS_CLEAR_OFFSET_8822B(x)                  \
+	(((x) >> BIT_SHIFT_R_MACID_RELEASE_SUCCESS_CLEAR_OFFSET_8822B) &       \
+	 BIT_MASK_R_MACID_RELEASE_SUCCESS_CLEAR_OFFSET_8822B)
+
+/* 2 REG_MACID_SHCUT_OFFSET_8822B */
+
+#define BIT_SHIFT_MACID_SHCUT_OFFSET_V1_8822B 0
+#define BIT_MASK_MACID_SHCUT_OFFSET_V1_8822B 0xff
+#define BIT_MACID_SHCUT_OFFSET_V1_8822B(x)                                     \
+	(((x) & BIT_MASK_MACID_SHCUT_OFFSET_V1_8822B)                          \
+	 << BIT_SHIFT_MACID_SHCUT_OFFSET_V1_8822B)
+#define BIT_GET_MACID_SHCUT_OFFSET_V1_8822B(x)                                 \
+	(((x) >> BIT_SHIFT_MACID_SHCUT_OFFSET_V1_8822B) &                      \
+	 BIT_MASK_MACID_SHCUT_OFFSET_V1_8822B)
+
+/* 2 REG_MU_TX_CTL_8822B */
+#define BIT_R_EN_REVERS_GTAB_8822B BIT(6)
+
+#define BIT_SHIFT_R_MU_TABLE_VALID_8822B 0
+#define BIT_MASK_R_MU_TABLE_VALID_8822B 0x3f
+#define BIT_R_MU_TABLE_VALID_8822B(x)                                          \
+	(((x) & BIT_MASK_R_MU_TABLE_VALID_8822B)                               \
+	 << BIT_SHIFT_R_MU_TABLE_VALID_8822B)
+#define BIT_GET_R_MU_TABLE_VALID_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_R_MU_TABLE_VALID_8822B) &                           \
+	 BIT_MASK_R_MU_TABLE_VALID_8822B)
+
+/* 2 REG_MU_STA_GID_VLD_8822B */
+
+/* 2 REG_NOT_VALID_8822B */
+
+#define BIT_SHIFT_R_MU_STA_GTAB_VALID_8822B 0
+#define BIT_MASK_R_MU_STA_GTAB_VALID_8822B 0xffffffffL
+#define BIT_R_MU_STA_GTAB_VALID_8822B(x)                                       \
+	(((x) & BIT_MASK_R_MU_STA_GTAB_VALID_8822B)                            \
+	 << BIT_SHIFT_R_MU_STA_GTAB_VALID_8822B)
+#define BIT_GET_R_MU_STA_GTAB_VALID_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_R_MU_STA_GTAB_VALID_8822B) &                        \
+	 BIT_MASK_R_MU_STA_GTAB_VALID_8822B)
+
+#define BIT_SHIFT_R_MU_STA_GTAB_VALID_8822B 0
+#define BIT_MASK_R_MU_STA_GTAB_VALID_8822B 0xffffffffL
+#define BIT_R_MU_STA_GTAB_VALID_8822B(x)                                       \
+	(((x) & BIT_MASK_R_MU_STA_GTAB_VALID_8822B)                            \
+	 << BIT_SHIFT_R_MU_STA_GTAB_VALID_8822B)
+#define BIT_GET_R_MU_STA_GTAB_VALID_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_R_MU_STA_GTAB_VALID_8822B) &                        \
+	 BIT_MASK_R_MU_STA_GTAB_VALID_8822B)
+
+/* 2 REG_MU_STA_USER_POS_INFO_8822B */
+
+/* 2 REG_NOT_VALID_8822B */
+
+#define BIT_SHIFT_R_MU_STA_GTAB_POSITION_8822B 0
+#define BIT_MASK_R_MU_STA_GTAB_POSITION_8822B 0xffffffffffffffffL
+#define BIT_R_MU_STA_GTAB_POSITION_8822B(x)                                    \
+	(((x) & BIT_MASK_R_MU_STA_GTAB_POSITION_8822B)                         \
+	 << BIT_SHIFT_R_MU_STA_GTAB_POSITION_8822B)
+#define BIT_GET_R_MU_STA_GTAB_POSITION_8822B(x)                                \
+	(((x) >> BIT_SHIFT_R_MU_STA_GTAB_POSITION_8822B) &                     \
+	 BIT_MASK_R_MU_STA_GTAB_POSITION_8822B)
+
+#define BIT_SHIFT_R_MU_STA_GTAB_POSITION_8822B 0
+#define BIT_MASK_R_MU_STA_GTAB_POSITION_8822B 0xffffffffffffffffL
+#define BIT_R_MU_STA_GTAB_POSITION_8822B(x)                                    \
+	(((x) & BIT_MASK_R_MU_STA_GTAB_POSITION_8822B)                         \
+	 << BIT_SHIFT_R_MU_STA_GTAB_POSITION_8822B)
+#define BIT_GET_R_MU_STA_GTAB_POSITION_8822B(x)                                \
+	(((x) >> BIT_SHIFT_R_MU_STA_GTAB_POSITION_8822B) &                     \
+	 BIT_MASK_R_MU_STA_GTAB_POSITION_8822B)
+
+/* 2 REG_MU_TRX_DBG_CNT_8822B */
+#define BIT_MU_DNGCNT_RST_8822B BIT(20)
+
+#define BIT_SHIFT_MU_DBGCNT_SEL_8822B 16
+#define BIT_MASK_MU_DBGCNT_SEL_8822B 0xf
+#define BIT_MU_DBGCNT_SEL_8822B(x)                                             \
+	(((x) & BIT_MASK_MU_DBGCNT_SEL_8822B) << BIT_SHIFT_MU_DBGCNT_SEL_8822B)
+#define BIT_GET_MU_DBGCNT_SEL_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_MU_DBGCNT_SEL_8822B) & BIT_MASK_MU_DBGCNT_SEL_8822B)
+
+#define BIT_SHIFT_MU_DNGCNT_8822B 0
+#define BIT_MASK_MU_DNGCNT_8822B 0xffff
+#define BIT_MU_DNGCNT_8822B(x)                                                 \
+	(((x) & BIT_MASK_MU_DNGCNT_8822B) << BIT_SHIFT_MU_DNGCNT_8822B)
+#define BIT_GET_MU_DNGCNT_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_MU_DNGCNT_8822B) & BIT_MASK_MU_DNGCNT_8822B)
+
+/* 2 REG_NOT_VALID_8822B */
+
+/* 2 REG_EDCA_VO_PARAM_8822B */
+
+#define BIT_SHIFT_TXOPLIMIT_8822B 16
+#define BIT_MASK_TXOPLIMIT_8822B 0x7ff
+#define BIT_TXOPLIMIT_8822B(x)                                                 \
+	(((x) & BIT_MASK_TXOPLIMIT_8822B) << BIT_SHIFT_TXOPLIMIT_8822B)
+#define BIT_GET_TXOPLIMIT_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_TXOPLIMIT_8822B) & BIT_MASK_TXOPLIMIT_8822B)
+
+#define BIT_SHIFT_CW_8822B 8
+#define BIT_MASK_CW_8822B 0xff
+#define BIT_CW_8822B(x) (((x) & BIT_MASK_CW_8822B) << BIT_SHIFT_CW_8822B)
+#define BIT_GET_CW_8822B(x) (((x) >> BIT_SHIFT_CW_8822B) & BIT_MASK_CW_8822B)
+
+#define BIT_SHIFT_AIFS_8822B 0
+#define BIT_MASK_AIFS_8822B 0xff
+#define BIT_AIFS_8822B(x) (((x) & BIT_MASK_AIFS_8822B) << BIT_SHIFT_AIFS_8822B)
+#define BIT_GET_AIFS_8822B(x)                                                  \
+	(((x) >> BIT_SHIFT_AIFS_8822B) & BIT_MASK_AIFS_8822B)
+
+/* 2 REG_EDCA_VI_PARAM_8822B */
+
+/* 2 REG_NOT_VALID_8822B */
+
+#define BIT_SHIFT_TXOPLIMIT_8822B 16
+#define BIT_MASK_TXOPLIMIT_8822B 0x7ff
+#define BIT_TXOPLIMIT_8822B(x)                                                 \
+	(((x) & BIT_MASK_TXOPLIMIT_8822B) << BIT_SHIFT_TXOPLIMIT_8822B)
+#define BIT_GET_TXOPLIMIT_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_TXOPLIMIT_8822B) & BIT_MASK_TXOPLIMIT_8822B)
+
+#define BIT_SHIFT_CW_8822B 8
+#define BIT_MASK_CW_8822B 0xff
+#define BIT_CW_8822B(x) (((x) & BIT_MASK_CW_8822B) << BIT_SHIFT_CW_8822B)
+#define BIT_GET_CW_8822B(x) (((x) >> BIT_SHIFT_CW_8822B) & BIT_MASK_CW_8822B)
+
+#define BIT_SHIFT_AIFS_8822B 0
+#define BIT_MASK_AIFS_8822B 0xff
+#define BIT_AIFS_8822B(x) (((x) & BIT_MASK_AIFS_8822B) << BIT_SHIFT_AIFS_8822B)
+#define BIT_GET_AIFS_8822B(x)                                                  \
+	(((x) >> BIT_SHIFT_AIFS_8822B) & BIT_MASK_AIFS_8822B)
+
+/* 2 REG_EDCA_BE_PARAM_8822B */
+
+/* 2 REG_NOT_VALID_8822B */
+
+#define BIT_SHIFT_TXOPLIMIT_8822B 16
+#define BIT_MASK_TXOPLIMIT_8822B 0x7ff
+#define BIT_TXOPLIMIT_8822B(x)                                                 \
+	(((x) & BIT_MASK_TXOPLIMIT_8822B) << BIT_SHIFT_TXOPLIMIT_8822B)
+#define BIT_GET_TXOPLIMIT_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_TXOPLIMIT_8822B) & BIT_MASK_TXOPLIMIT_8822B)
+
+#define BIT_SHIFT_CW_8822B 8
+#define BIT_MASK_CW_8822B 0xff
+#define BIT_CW_8822B(x) (((x) & BIT_MASK_CW_8822B) << BIT_SHIFT_CW_8822B)
+#define BIT_GET_CW_8822B(x) (((x) >> BIT_SHIFT_CW_8822B) & BIT_MASK_CW_8822B)
+
+#define BIT_SHIFT_AIFS_8822B 0
+#define BIT_MASK_AIFS_8822B 0xff
+#define BIT_AIFS_8822B(x) (((x) & BIT_MASK_AIFS_8822B) << BIT_SHIFT_AIFS_8822B)
+#define BIT_GET_AIFS_8822B(x)                                                  \
+	(((x) >> BIT_SHIFT_AIFS_8822B) & BIT_MASK_AIFS_8822B)
+
+/* 2 REG_EDCA_BK_PARAM_8822B */
+
+/* 2 REG_NOT_VALID_8822B */
+
+#define BIT_SHIFT_TXOPLIMIT_8822B 16
+#define BIT_MASK_TXOPLIMIT_8822B 0x7ff
+#define BIT_TXOPLIMIT_8822B(x)                                                 \
+	(((x) & BIT_MASK_TXOPLIMIT_8822B) << BIT_SHIFT_TXOPLIMIT_8822B)
+#define BIT_GET_TXOPLIMIT_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_TXOPLIMIT_8822B) & BIT_MASK_TXOPLIMIT_8822B)
+
+#define BIT_SHIFT_CW_8822B 8
+#define BIT_MASK_CW_8822B 0xff
+#define BIT_CW_8822B(x) (((x) & BIT_MASK_CW_8822B) << BIT_SHIFT_CW_8822B)
+#define BIT_GET_CW_8822B(x) (((x) >> BIT_SHIFT_CW_8822B) & BIT_MASK_CW_8822B)
+
+#define BIT_SHIFT_AIFS_8822B 0
+#define BIT_MASK_AIFS_8822B 0xff
+#define BIT_AIFS_8822B(x) (((x) & BIT_MASK_AIFS_8822B) << BIT_SHIFT_AIFS_8822B)
+#define BIT_GET_AIFS_8822B(x)                                                  \
+	(((x) >> BIT_SHIFT_AIFS_8822B) & BIT_MASK_AIFS_8822B)
+
+/* 2 REG_BCNTCFG_8822B */
+
+#define BIT_SHIFT_BCNCW_MAX_8822B 12
+#define BIT_MASK_BCNCW_MAX_8822B 0xf
+#define BIT_BCNCW_MAX_8822B(x)                                                 \
+	(((x) & BIT_MASK_BCNCW_MAX_8822B) << BIT_SHIFT_BCNCW_MAX_8822B)
+#define BIT_GET_BCNCW_MAX_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_BCNCW_MAX_8822B) & BIT_MASK_BCNCW_MAX_8822B)
+
+#define BIT_SHIFT_BCNCW_MIN_8822B 8
+#define BIT_MASK_BCNCW_MIN_8822B 0xf
+#define BIT_BCNCW_MIN_8822B(x)                                                 \
+	(((x) & BIT_MASK_BCNCW_MIN_8822B) << BIT_SHIFT_BCNCW_MIN_8822B)
+#define BIT_GET_BCNCW_MIN_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_BCNCW_MIN_8822B) & BIT_MASK_BCNCW_MIN_8822B)
+
+#define BIT_SHIFT_BCNIFS_8822B 0
+#define BIT_MASK_BCNIFS_8822B 0xff
+#define BIT_BCNIFS_8822B(x)                                                    \
+	(((x) & BIT_MASK_BCNIFS_8822B) << BIT_SHIFT_BCNIFS_8822B)
+#define BIT_GET_BCNIFS_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_BCNIFS_8822B) & BIT_MASK_BCNIFS_8822B)
+
+/* 2 REG_PIFS_8822B */
+
+#define BIT_SHIFT_PIFS_8822B 0
+#define BIT_MASK_PIFS_8822B 0xff
+#define BIT_PIFS_8822B(x) (((x) & BIT_MASK_PIFS_8822B) << BIT_SHIFT_PIFS_8822B)
+#define BIT_GET_PIFS_8822B(x)                                                  \
+	(((x) >> BIT_SHIFT_PIFS_8822B) & BIT_MASK_PIFS_8822B)
+
+/* 2 REG_RDG_PIFS_8822B */
+
+#define BIT_SHIFT_RDG_PIFS_8822B 0
+#define BIT_MASK_RDG_PIFS_8822B 0xff
+#define BIT_RDG_PIFS_8822B(x)                                                  \
+	(((x) & BIT_MASK_RDG_PIFS_8822B) << BIT_SHIFT_RDG_PIFS_8822B)
+#define BIT_GET_RDG_PIFS_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_RDG_PIFS_8822B) & BIT_MASK_RDG_PIFS_8822B)
+
+/* 2 REG_SIFS_8822B */
+
+#define BIT_SHIFT_SIFS_OFDM_TRX_8822B 24
+#define BIT_MASK_SIFS_OFDM_TRX_8822B 0xff
+#define BIT_SIFS_OFDM_TRX_8822B(x)                                             \
+	(((x) & BIT_MASK_SIFS_OFDM_TRX_8822B) << BIT_SHIFT_SIFS_OFDM_TRX_8822B)
+#define BIT_GET_SIFS_OFDM_TRX_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_SIFS_OFDM_TRX_8822B) & BIT_MASK_SIFS_OFDM_TRX_8822B)
+
+#define BIT_SHIFT_SIFS_CCK_TRX_8822B 16
+#define BIT_MASK_SIFS_CCK_TRX_8822B 0xff
+#define BIT_SIFS_CCK_TRX_8822B(x)                                              \
+	(((x) & BIT_MASK_SIFS_CCK_TRX_8822B) << BIT_SHIFT_SIFS_CCK_TRX_8822B)
+#define BIT_GET_SIFS_CCK_TRX_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_SIFS_CCK_TRX_8822B) & BIT_MASK_SIFS_CCK_TRX_8822B)
+
+#define BIT_SHIFT_SIFS_OFDM_CTX_8822B 8
+#define BIT_MASK_SIFS_OFDM_CTX_8822B 0xff
+#define BIT_SIFS_OFDM_CTX_8822B(x)                                             \
+	(((x) & BIT_MASK_SIFS_OFDM_CTX_8822B) << BIT_SHIFT_SIFS_OFDM_CTX_8822B)
+#define BIT_GET_SIFS_OFDM_CTX_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_SIFS_OFDM_CTX_8822B) & BIT_MASK_SIFS_OFDM_CTX_8822B)
+
+#define BIT_SHIFT_SIFS_CCK_CTX_8822B 0
+#define BIT_MASK_SIFS_CCK_CTX_8822B 0xff
+#define BIT_SIFS_CCK_CTX_8822B(x)                                              \
+	(((x) & BIT_MASK_SIFS_CCK_CTX_8822B) << BIT_SHIFT_SIFS_CCK_CTX_8822B)
+#define BIT_GET_SIFS_CCK_CTX_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_SIFS_CCK_CTX_8822B) & BIT_MASK_SIFS_CCK_CTX_8822B)
+
+/* 2 REG_TSFTR_SYN_OFFSET_8822B */
+
+#define BIT_SHIFT_TSFTR_SNC_OFFSET_8822B 0
+#define BIT_MASK_TSFTR_SNC_OFFSET_8822B 0xffff
+#define BIT_TSFTR_SNC_OFFSET_8822B(x)                                          \
+	(((x) & BIT_MASK_TSFTR_SNC_OFFSET_8822B)                               \
+	 << BIT_SHIFT_TSFTR_SNC_OFFSET_8822B)
+#define BIT_GET_TSFTR_SNC_OFFSET_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_TSFTR_SNC_OFFSET_8822B) &                           \
+	 BIT_MASK_TSFTR_SNC_OFFSET_8822B)
+
+/* 2 REG_AGGR_BREAK_TIME_8822B */
+
+#define BIT_SHIFT_AGGR_BK_TIME_8822B 0
+#define BIT_MASK_AGGR_BK_TIME_8822B 0xff
+#define BIT_AGGR_BK_TIME_8822B(x)                                              \
+	(((x) & BIT_MASK_AGGR_BK_TIME_8822B) << BIT_SHIFT_AGGR_BK_TIME_8822B)
+#define BIT_GET_AGGR_BK_TIME_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_AGGR_BK_TIME_8822B) & BIT_MASK_AGGR_BK_TIME_8822B)
+
+/* 2 REG_SLOT_8822B */
+
+#define BIT_SHIFT_SLOT_8822B 0
+#define BIT_MASK_SLOT_8822B 0xff
+#define BIT_SLOT_8822B(x) (((x) & BIT_MASK_SLOT_8822B) << BIT_SHIFT_SLOT_8822B)
+#define BIT_GET_SLOT_8822B(x)                                                  \
+	(((x) >> BIT_SHIFT_SLOT_8822B) & BIT_MASK_SLOT_8822B)
+
+/* 2 REG_TX_PTCL_CTRL_8822B */
+#define BIT_DIS_EDCCA_8822B BIT(15)
+#define BIT_DIS_CCA_8822B BIT(14)
+#define BIT_LSIG_TXOP_TXCMD_NAV_8822B BIT(13)
+#define BIT_SIFS_BK_EN_8822B BIT(12)
+
+#define BIT_SHIFT_TXQ_NAV_MSK_8822B 8
+#define BIT_MASK_TXQ_NAV_MSK_8822B 0xf
+#define BIT_TXQ_NAV_MSK_8822B(x)                                               \
+	(((x) & BIT_MASK_TXQ_NAV_MSK_8822B) << BIT_SHIFT_TXQ_NAV_MSK_8822B)
+#define BIT_GET_TXQ_NAV_MSK_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_TXQ_NAV_MSK_8822B) & BIT_MASK_TXQ_NAV_MSK_8822B)
+
+#define BIT_DIS_CW_8822B BIT(7)
+#define BIT_NAV_END_TXOP_8822B BIT(6)
+#define BIT_RDG_END_TXOP_8822B BIT(5)
+#define BIT_AC_INBCN_HOLD_8822B BIT(4)
+#define BIT_MGTQ_TXOP_EN_8822B BIT(3)
+#define BIT_MGTQ_RTSMF_EN_8822B BIT(2)
+#define BIT_HIQ_RTSMF_EN_8822B BIT(1)
+#define BIT_BCN_RTSMF_EN_8822B BIT(0)
+
+/* 2 REG_TXPAUSE_8822B */
+#define BIT_STOP_BCN_HI_MGT_8822B BIT(7)
+#define BIT_MAC_STOPBCNQ_8822B BIT(6)
+#define BIT_MAC_STOPHIQ_8822B BIT(5)
+#define BIT_MAC_STOPMGQ_8822B BIT(4)
+#define BIT_MAC_STOPBK_8822B BIT(3)
+#define BIT_MAC_STOPBE_8822B BIT(2)
+#define BIT_MAC_STOPVI_8822B BIT(1)
+#define BIT_MAC_STOPVO_8822B BIT(0)
+
+/* 2 REG_DIS_TXREQ_CLR_8822B */
+#define BIT_DIS_BT_CCA_8822B BIT(7)
+#define BIT_DIS_TXREQ_CLR_HI_8822B BIT(5)
+#define BIT_DIS_TXREQ_CLR_MGQ_8822B BIT(4)
+#define BIT_DIS_TXREQ_CLR_VO_8822B BIT(3)
+#define BIT_DIS_TXREQ_CLR_VI_8822B BIT(2)
+#define BIT_DIS_TXREQ_CLR_BE_8822B BIT(1)
+#define BIT_DIS_TXREQ_CLR_BK_8822B BIT(0)
+
+/* 2 REG_RD_CTRL_8822B */
+#define BIT_EN_CLR_TXREQ_INCCA_8822B BIT(15)
+#define BIT_DIS_TX_OVER_BCNQ_8822B BIT(14)
+#define BIT_EN_BCNERR_INCCCA_8822B BIT(13)
+#define BIT_EDCCA_MSK_CNTDOWN_EN_8822B BIT(11)
+#define BIT_DIS_TXOP_CFE_8822B BIT(10)
+#define BIT_DIS_LSIG_CFE_8822B BIT(9)
+#define BIT_DIS_STBC_CFE_8822B BIT(8)
+#define BIT_BKQ_RD_INIT_EN_8822B BIT(7)
+#define BIT_BEQ_RD_INIT_EN_8822B BIT(6)
+#define BIT_VIQ_RD_INIT_EN_8822B BIT(5)
+#define BIT_VOQ_RD_INIT_EN_8822B BIT(4)
+#define BIT_BKQ_RD_RESP_EN_8822B BIT(3)
+#define BIT_BEQ_RD_RESP_EN_8822B BIT(2)
+#define BIT_VIQ_RD_RESP_EN_8822B BIT(1)
+#define BIT_VOQ_RD_RESP_EN_8822B BIT(0)
+
+/* 2 REG_MBSSID_CTRL_8822B */
+#define BIT_MBID_BCNQ7_EN_8822B BIT(7)
+#define BIT_MBID_BCNQ6_EN_8822B BIT(6)
+#define BIT_MBID_BCNQ5_EN_8822B BIT(5)
+#define BIT_MBID_BCNQ4_EN_8822B BIT(4)
+#define BIT_MBID_BCNQ3_EN_8822B BIT(3)
+#define BIT_MBID_BCNQ2_EN_8822B BIT(2)
+#define BIT_MBID_BCNQ1_EN_8822B BIT(1)
+#define BIT_MBID_BCNQ0_EN_8822B BIT(0)
+
+/* 2 REG_P2PPS_CTRL_8822B */
+#define BIT_P2P_CTW_ALLSTASLEEP_8822B BIT(7)
+#define BIT_P2P_OFF_DISTX_EN_8822B BIT(6)
+#define BIT_PWR_MGT_EN_8822B BIT(5)
+#define BIT_P2P_NOA1_EN_8822B BIT(2)
+#define BIT_P2P_NOA0_EN_8822B BIT(1)
+
+/* 2 REG_PKT_LIFETIME_CTRL_8822B */
+#define BIT_EN_P2P_CTWND1_8822B BIT(23)
+#define BIT_EN_BKF_CLR_TXREQ_8822B BIT(22)
+#define BIT_EN_TSFBIT32_RST_P2P_8822B BIT(21)
+#define BIT_EN_BCN_TX_BTCCA_8822B BIT(20)
+#define BIT_DIS_PKT_TX_ATIM_8822B BIT(19)
+#define BIT_DIS_BCN_DIS_CTN_8822B BIT(18)
+#define BIT_EN_NAVEND_RST_TXOP_8822B BIT(17)
+#define BIT_EN_FILTER_CCA_8822B BIT(16)
+
+#define BIT_SHIFT_CCA_FILTER_THRS_8822B 8
+#define BIT_MASK_CCA_FILTER_THRS_8822B 0xff
+#define BIT_CCA_FILTER_THRS_8822B(x)                                           \
+	(((x) & BIT_MASK_CCA_FILTER_THRS_8822B)                                \
+	 << BIT_SHIFT_CCA_FILTER_THRS_8822B)
+#define BIT_GET_CCA_FILTER_THRS_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_CCA_FILTER_THRS_8822B) &                            \
+	 BIT_MASK_CCA_FILTER_THRS_8822B)
+
+#define BIT_SHIFT_EDCCA_THRS_8822B 0
+#define BIT_MASK_EDCCA_THRS_8822B 0xff
+#define BIT_EDCCA_THRS_8822B(x)                                                \
+	(((x) & BIT_MASK_EDCCA_THRS_8822B) << BIT_SHIFT_EDCCA_THRS_8822B)
+#define BIT_GET_EDCCA_THRS_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_EDCCA_THRS_8822B) & BIT_MASK_EDCCA_THRS_8822B)
+
+/* 2 REG_P2PPS_SPEC_STATE_8822B */
+#define BIT_SPEC_POWER_STATE_8822B BIT(7)
+#define BIT_SPEC_CTWINDOW_ON_8822B BIT(6)
+#define BIT_SPEC_BEACON_AREA_ON_8822B BIT(5)
+#define BIT_SPEC_CTWIN_EARLY_DISTX_8822B BIT(4)
+#define BIT_SPEC_NOA1_OFF_PERIOD_8822B BIT(3)
+#define BIT_SPEC_FORCE_DOZE1_8822B BIT(2)
+#define BIT_SPEC_NOA0_OFF_PERIOD_8822B BIT(1)
+#define BIT_SPEC_FORCE_DOZE0_8822B BIT(0)
+
+/* 2 REG_BAR_TX_CTRL_8822B */
+
+/* 2 REG_NOT_VALID_8822B */
+
+#define BIT_SHIFT_P2PON_DIS_TXTIME_8822B 0
+#define BIT_MASK_P2PON_DIS_TXTIME_8822B 0xff
+#define BIT_P2PON_DIS_TXTIME_8822B(x)                                          \
+	(((x) & BIT_MASK_P2PON_DIS_TXTIME_8822B)                               \
+	 << BIT_SHIFT_P2PON_DIS_TXTIME_8822B)
+#define BIT_GET_P2PON_DIS_TXTIME_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_P2PON_DIS_TXTIME_8822B) &                           \
+	 BIT_MASK_P2PON_DIS_TXTIME_8822B)
+
+/* 2 REG_QUEUE_INCOL_THR_8822B */
+
+#define BIT_SHIFT_BK_QUEUE_THR_8822B 24
+#define BIT_MASK_BK_QUEUE_THR_8822B 0xff
+#define BIT_BK_QUEUE_THR_8822B(x)                                              \
+	(((x) & BIT_MASK_BK_QUEUE_THR_8822B) << BIT_SHIFT_BK_QUEUE_THR_8822B)
+#define BIT_GET_BK_QUEUE_THR_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_BK_QUEUE_THR_8822B) & BIT_MASK_BK_QUEUE_THR_8822B)
+
+#define BIT_SHIFT_BE_QUEUE_THR_8822B 16
+#define BIT_MASK_BE_QUEUE_THR_8822B 0xff
+#define BIT_BE_QUEUE_THR_8822B(x)                                              \
+	(((x) & BIT_MASK_BE_QUEUE_THR_8822B) << BIT_SHIFT_BE_QUEUE_THR_8822B)
+#define BIT_GET_BE_QUEUE_THR_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_BE_QUEUE_THR_8822B) & BIT_MASK_BE_QUEUE_THR_8822B)
+
+#define BIT_SHIFT_VI_QUEUE_THR_8822B 8
+#define BIT_MASK_VI_QUEUE_THR_8822B 0xff
+#define BIT_VI_QUEUE_THR_8822B(x)                                              \
+	(((x) & BIT_MASK_VI_QUEUE_THR_8822B) << BIT_SHIFT_VI_QUEUE_THR_8822B)
+#define BIT_GET_VI_QUEUE_THR_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_VI_QUEUE_THR_8822B) & BIT_MASK_VI_QUEUE_THR_8822B)
+
+#define BIT_SHIFT_VO_QUEUE_THR_8822B 0
+#define BIT_MASK_VO_QUEUE_THR_8822B 0xff
+#define BIT_VO_QUEUE_THR_8822B(x)                                              \
+	(((x) & BIT_MASK_VO_QUEUE_THR_8822B) << BIT_SHIFT_VO_QUEUE_THR_8822B)
+#define BIT_GET_VO_QUEUE_THR_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_VO_QUEUE_THR_8822B) & BIT_MASK_VO_QUEUE_THR_8822B)
+
+/* 2 REG_QUEUE_INCOL_EN_8822B */
+#define BIT_QUEUE_INCOL_EN_8822B BIT(16)
+
+#define BIT_SHIFT_BE_TRIGGER_NUM_8822B 12
+#define BIT_MASK_BE_TRIGGER_NUM_8822B 0xf
+#define BIT_BE_TRIGGER_NUM_8822B(x)                                            \
+	(((x) & BIT_MASK_BE_TRIGGER_NUM_8822B)                                 \
+	 << BIT_SHIFT_BE_TRIGGER_NUM_8822B)
+#define BIT_GET_BE_TRIGGER_NUM_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_BE_TRIGGER_NUM_8822B) &                             \
+	 BIT_MASK_BE_TRIGGER_NUM_8822B)
+
+#define BIT_SHIFT_BK_TRIGGER_NUM_8822B 8
+#define BIT_MASK_BK_TRIGGER_NUM_8822B 0xf
+#define BIT_BK_TRIGGER_NUM_8822B(x)                                            \
+	(((x) & BIT_MASK_BK_TRIGGER_NUM_8822B)                                 \
+	 << BIT_SHIFT_BK_TRIGGER_NUM_8822B)
+#define BIT_GET_BK_TRIGGER_NUM_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_BK_TRIGGER_NUM_8822B) &                             \
+	 BIT_MASK_BK_TRIGGER_NUM_8822B)
+
+#define BIT_SHIFT_VI_TRIGGER_NUM_8822B 4
+#define BIT_MASK_VI_TRIGGER_NUM_8822B 0xf
+#define BIT_VI_TRIGGER_NUM_8822B(x)                                            \
+	(((x) & BIT_MASK_VI_TRIGGER_NUM_8822B)                                 \
+	 << BIT_SHIFT_VI_TRIGGER_NUM_8822B)
+#define BIT_GET_VI_TRIGGER_NUM_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_VI_TRIGGER_NUM_8822B) &                             \
+	 BIT_MASK_VI_TRIGGER_NUM_8822B)
+
+#define BIT_SHIFT_VO_TRIGGER_NUM_8822B 0
+#define BIT_MASK_VO_TRIGGER_NUM_8822B 0xf
+#define BIT_VO_TRIGGER_NUM_8822B(x)                                            \
+	(((x) & BIT_MASK_VO_TRIGGER_NUM_8822B)                                 \
+	 << BIT_SHIFT_VO_TRIGGER_NUM_8822B)
+#define BIT_GET_VO_TRIGGER_NUM_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_VO_TRIGGER_NUM_8822B) &                             \
+	 BIT_MASK_VO_TRIGGER_NUM_8822B)
+
+/* 2 REG_TBTT_PROHIBIT_8822B */
+
+#define BIT_SHIFT_TBTT_HOLD_TIME_AP_8822B 8
+#define BIT_MASK_TBTT_HOLD_TIME_AP_8822B 0xfff
+#define BIT_TBTT_HOLD_TIME_AP_8822B(x)                                         \
+	(((x) & BIT_MASK_TBTT_HOLD_TIME_AP_8822B)                              \
+	 << BIT_SHIFT_TBTT_HOLD_TIME_AP_8822B)
+#define BIT_GET_TBTT_HOLD_TIME_AP_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_TBTT_HOLD_TIME_AP_8822B) &                          \
+	 BIT_MASK_TBTT_HOLD_TIME_AP_8822B)
+
+#define BIT_SHIFT_TBTT_PROHIBIT_SETUP_8822B 0
+#define BIT_MASK_TBTT_PROHIBIT_SETUP_8822B 0xf
+#define BIT_TBTT_PROHIBIT_SETUP_8822B(x)                                       \
+	(((x) & BIT_MASK_TBTT_PROHIBIT_SETUP_8822B)                            \
+	 << BIT_SHIFT_TBTT_PROHIBIT_SETUP_8822B)
+#define BIT_GET_TBTT_PROHIBIT_SETUP_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_TBTT_PROHIBIT_SETUP_8822B) &                        \
+	 BIT_MASK_TBTT_PROHIBIT_SETUP_8822B)
+
+/* 2 REG_P2PPS_STATE_8822B */
+#define BIT_POWER_STATE_8822B BIT(7)
+#define BIT_CTWINDOW_ON_8822B BIT(6)
+#define BIT_BEACON_AREA_ON_8822B BIT(5)
+#define BIT_CTWIN_EARLY_DISTX_8822B BIT(4)
+#define BIT_NOA1_OFF_PERIOD_8822B BIT(3)
+#define BIT_FORCE_DOZE1_8822B BIT(2)
+#define BIT_NOA0_OFF_PERIOD_8822B BIT(1)
+#define BIT_FORCE_DOZE0_8822B BIT(0)
+
+/* 2 REG_RD_NAV_NXT_8822B */
+
+#define BIT_SHIFT_RD_NAV_PROT_NXT_8822B 0
+#define BIT_MASK_RD_NAV_PROT_NXT_8822B 0xffff
+#define BIT_RD_NAV_PROT_NXT_8822B(x)                                           \
+	(((x) & BIT_MASK_RD_NAV_PROT_NXT_8822B)                                \
+	 << BIT_SHIFT_RD_NAV_PROT_NXT_8822B)
+#define BIT_GET_RD_NAV_PROT_NXT_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_RD_NAV_PROT_NXT_8822B) &                            \
+	 BIT_MASK_RD_NAV_PROT_NXT_8822B)
+
+/* 2 REG_NAV_PROT_LEN_8822B */
+
+#define BIT_SHIFT_NAV_PROT_LEN_8822B 0
+#define BIT_MASK_NAV_PROT_LEN_8822B 0xffff
+#define BIT_NAV_PROT_LEN_8822B(x)                                              \
+	(((x) & BIT_MASK_NAV_PROT_LEN_8822B) << BIT_SHIFT_NAV_PROT_LEN_8822B)
+#define BIT_GET_NAV_PROT_LEN_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_NAV_PROT_LEN_8822B) & BIT_MASK_NAV_PROT_LEN_8822B)
+
+/* 2 REG_BCN_CTRL_8822B */
+#define BIT_DIS_RX_BSSID_FIT_8822B BIT(6)
+#define BIT_P0_EN_TXBCN_RPT_8822B BIT(5)
+#define BIT_DIS_TSF_UDT_8822B BIT(4)
+#define BIT_EN_BCN_FUNCTION_8822B BIT(3)
+#define BIT_P0_EN_RXBCN_RPT_8822B BIT(2)
+#define BIT_EN_P2P_CTWINDOW_8822B BIT(1)
+#define BIT_EN_P2P_BCNQ_AREA_8822B BIT(0)
+
+/* 2 REG_BCN_CTRL_CLINT0_8822B */
+#define BIT_CLI0_DIS_RX_BSSID_FIT_8822B BIT(6)
+#define BIT_CLI0_DIS_TSF_UDT_8822B BIT(4)
+#define BIT_CLI0_EN_BCN_FUNCTION_8822B BIT(3)
+#define BIT_CLI0_EN_RXBCN_RPT_8822B BIT(2)
+#define BIT_CLI0_ENP2P_CTWINDOW_8822B BIT(1)
+#define BIT_CLI0_ENP2P_BCNQ_AREA_8822B BIT(0)
+
+/* 2 REG_MBID_NUM_8822B */
+#define BIT_EN_PRE_DL_BEACON_8822B BIT(3)
+
+#define BIT_SHIFT_MBID_BCN_NUM_8822B 0
+#define BIT_MASK_MBID_BCN_NUM_8822B 0x7
+#define BIT_MBID_BCN_NUM_8822B(x)                                              \
+	(((x) & BIT_MASK_MBID_BCN_NUM_8822B) << BIT_SHIFT_MBID_BCN_NUM_8822B)
+#define BIT_GET_MBID_BCN_NUM_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_MBID_BCN_NUM_8822B) & BIT_MASK_MBID_BCN_NUM_8822B)
+
+/* 2 REG_DUAL_TSF_RST_8822B */
+#define BIT_FREECNT_RST_8822B BIT(5)
+#define BIT_TSFTR_CLI3_RST_8822B BIT(4)
+#define BIT_TSFTR_CLI2_RST_8822B BIT(3)
+#define BIT_TSFTR_CLI1_RST_8822B BIT(2)
+#define BIT_TSFTR_CLI0_RST_8822B BIT(1)
+#define BIT_TSFTR_RST_8822B BIT(0)
+
+/* 2 REG_MBSSID_BCN_SPACE_8822B */
+
+#define BIT_SHIFT_BCN_TIMER_SEL_FWRD_8822B 28
+#define BIT_MASK_BCN_TIMER_SEL_FWRD_8822B 0x7
+#define BIT_BCN_TIMER_SEL_FWRD_8822B(x)                                        \
+	(((x) & BIT_MASK_BCN_TIMER_SEL_FWRD_8822B)                             \
+	 << BIT_SHIFT_BCN_TIMER_SEL_FWRD_8822B)
+#define BIT_GET_BCN_TIMER_SEL_FWRD_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_BCN_TIMER_SEL_FWRD_8822B) &                         \
+	 BIT_MASK_BCN_TIMER_SEL_FWRD_8822B)
+
+#define BIT_SHIFT_BCN_SPACE_CLINT0_8822B 16
+#define BIT_MASK_BCN_SPACE_CLINT0_8822B 0xfff
+#define BIT_BCN_SPACE_CLINT0_8822B(x)                                          \
+	(((x) & BIT_MASK_BCN_SPACE_CLINT0_8822B)                               \
+	 << BIT_SHIFT_BCN_SPACE_CLINT0_8822B)
+#define BIT_GET_BCN_SPACE_CLINT0_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_BCN_SPACE_CLINT0_8822B) &                           \
+	 BIT_MASK_BCN_SPACE_CLINT0_8822B)
+
+#define BIT_SHIFT_BCN_SPACE0_8822B 0
+#define BIT_MASK_BCN_SPACE0_8822B 0xffff
+#define BIT_BCN_SPACE0_8822B(x)                                                \
+	(((x) & BIT_MASK_BCN_SPACE0_8822B) << BIT_SHIFT_BCN_SPACE0_8822B)
+#define BIT_GET_BCN_SPACE0_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_BCN_SPACE0_8822B) & BIT_MASK_BCN_SPACE0_8822B)
+
+/* 2 REG_DRVERLYINT_8822B */
+
+#define BIT_SHIFT_DRVERLYITV_8822B 0
+#define BIT_MASK_DRVERLYITV_8822B 0xff
+#define BIT_DRVERLYITV_8822B(x)                                                \
+	(((x) & BIT_MASK_DRVERLYITV_8822B) << BIT_SHIFT_DRVERLYITV_8822B)
+#define BIT_GET_DRVERLYITV_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_DRVERLYITV_8822B) & BIT_MASK_DRVERLYITV_8822B)
+
+/* 2 REG_BCNDMATIM_8822B */
+
+#define BIT_SHIFT_BCNDMATIM_8822B 0
+#define BIT_MASK_BCNDMATIM_8822B 0xff
+#define BIT_BCNDMATIM_8822B(x)                                                 \
+	(((x) & BIT_MASK_BCNDMATIM_8822B) << BIT_SHIFT_BCNDMATIM_8822B)
+#define BIT_GET_BCNDMATIM_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_BCNDMATIM_8822B) & BIT_MASK_BCNDMATIM_8822B)
+
+/* 2 REG_ATIMWND_8822B */
+
+#define BIT_SHIFT_ATIMWND0_8822B 0
+#define BIT_MASK_ATIMWND0_8822B 0xffff
+#define BIT_ATIMWND0_8822B(x)                                                  \
+	(((x) & BIT_MASK_ATIMWND0_8822B) << BIT_SHIFT_ATIMWND0_8822B)
+#define BIT_GET_ATIMWND0_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_ATIMWND0_8822B) & BIT_MASK_ATIMWND0_8822B)
+
+/* 2 REG_USTIME_TSF_8822B */
+
+#define BIT_SHIFT_USTIME_TSF_V1_8822B 0
+#define BIT_MASK_USTIME_TSF_V1_8822B 0xff
+#define BIT_USTIME_TSF_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_USTIME_TSF_V1_8822B) << BIT_SHIFT_USTIME_TSF_V1_8822B)
+#define BIT_GET_USTIME_TSF_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_USTIME_TSF_V1_8822B) & BIT_MASK_USTIME_TSF_V1_8822B)
+
+/* 2 REG_BCN_MAX_ERR_8822B */
+
+#define BIT_SHIFT_BCN_MAX_ERR_8822B 0
+#define BIT_MASK_BCN_MAX_ERR_8822B 0xff
+#define BIT_BCN_MAX_ERR_8822B(x)                                               \
+	(((x) & BIT_MASK_BCN_MAX_ERR_8822B) << BIT_SHIFT_BCN_MAX_ERR_8822B)
+#define BIT_GET_BCN_MAX_ERR_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_BCN_MAX_ERR_8822B) & BIT_MASK_BCN_MAX_ERR_8822B)
+
+/* 2 REG_RXTSF_OFFSET_CCK_8822B */
+
+#define BIT_SHIFT_CCK_RXTSF_OFFSET_8822B 0
+#define BIT_MASK_CCK_RXTSF_OFFSET_8822B 0xff
+#define BIT_CCK_RXTSF_OFFSET_8822B(x)                                          \
+	(((x) & BIT_MASK_CCK_RXTSF_OFFSET_8822B)                               \
+	 << BIT_SHIFT_CCK_RXTSF_OFFSET_8822B)
+#define BIT_GET_CCK_RXTSF_OFFSET_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_CCK_RXTSF_OFFSET_8822B) &                           \
+	 BIT_MASK_CCK_RXTSF_OFFSET_8822B)
+
+/* 2 REG_RXTSF_OFFSET_OFDM_8822B */
+
+#define BIT_SHIFT_OFDM_RXTSF_OFFSET_8822B 0
+#define BIT_MASK_OFDM_RXTSF_OFFSET_8822B 0xff
+#define BIT_OFDM_RXTSF_OFFSET_8822B(x)                                         \
+	(((x) & BIT_MASK_OFDM_RXTSF_OFFSET_8822B)                              \
+	 << BIT_SHIFT_OFDM_RXTSF_OFFSET_8822B)
+#define BIT_GET_OFDM_RXTSF_OFFSET_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_OFDM_RXTSF_OFFSET_8822B) &                          \
+	 BIT_MASK_OFDM_RXTSF_OFFSET_8822B)
+
+/* 2 REG_TSFTR_8822B */
+
+#define BIT_SHIFT_TSF_TIMER_8822B 0
+#define BIT_MASK_TSF_TIMER_8822B 0xffffffffffffffffL
+#define BIT_TSF_TIMER_8822B(x)                                                 \
+	(((x) & BIT_MASK_TSF_TIMER_8822B) << BIT_SHIFT_TSF_TIMER_8822B)
+#define BIT_GET_TSF_TIMER_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_TSF_TIMER_8822B) & BIT_MASK_TSF_TIMER_8822B)
+
+/* 2 REG_FREERUN_CNT_8822B */
+
+#define BIT_SHIFT_FREERUN_CNT_8822B 0
+#define BIT_MASK_FREERUN_CNT_8822B 0xffffffffffffffffL
+#define BIT_FREERUN_CNT_8822B(x)                                               \
+	(((x) & BIT_MASK_FREERUN_CNT_8822B) << BIT_SHIFT_FREERUN_CNT_8822B)
+#define BIT_GET_FREERUN_CNT_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_FREERUN_CNT_8822B) & BIT_MASK_FREERUN_CNT_8822B)
+
+/* 2 REG_ATIMWND1_V1_8822B */
+
+#define BIT_SHIFT_ATIMWND1_V1_8822B 0
+#define BIT_MASK_ATIMWND1_V1_8822B 0xff
+#define BIT_ATIMWND1_V1_8822B(x)                                               \
+	(((x) & BIT_MASK_ATIMWND1_V1_8822B) << BIT_SHIFT_ATIMWND1_V1_8822B)
+#define BIT_GET_ATIMWND1_V1_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_ATIMWND1_V1_8822B) & BIT_MASK_ATIMWND1_V1_8822B)
+
+/* 2 REG_TBTT_PROHIBIT_INFRA_8822B */
+
+#define BIT_SHIFT_TBTT_PROHIBIT_INFRA_8822B 0
+#define BIT_MASK_TBTT_PROHIBIT_INFRA_8822B 0xff
+#define BIT_TBTT_PROHIBIT_INFRA_8822B(x)                                       \
+	(((x) & BIT_MASK_TBTT_PROHIBIT_INFRA_8822B)                            \
+	 << BIT_SHIFT_TBTT_PROHIBIT_INFRA_8822B)
+#define BIT_GET_TBTT_PROHIBIT_INFRA_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_TBTT_PROHIBIT_INFRA_8822B) &                        \
+	 BIT_MASK_TBTT_PROHIBIT_INFRA_8822B)
+
+/* 2 REG_CTWND_8822B */
+
+#define BIT_SHIFT_CTWND_8822B 0
+#define BIT_MASK_CTWND_8822B 0xff
+#define BIT_CTWND_8822B(x)                                                     \
+	(((x) & BIT_MASK_CTWND_8822B) << BIT_SHIFT_CTWND_8822B)
+#define BIT_GET_CTWND_8822B(x)                                                 \
+	(((x) >> BIT_SHIFT_CTWND_8822B) & BIT_MASK_CTWND_8822B)
+
+/* 2 REG_BCNIVLCUNT_8822B */
+
+#define BIT_SHIFT_BCNIVLCUNT_8822B 0
+#define BIT_MASK_BCNIVLCUNT_8822B 0x7f
+#define BIT_BCNIVLCUNT_8822B(x)                                                \
+	(((x) & BIT_MASK_BCNIVLCUNT_8822B) << BIT_SHIFT_BCNIVLCUNT_8822B)
+#define BIT_GET_BCNIVLCUNT_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_BCNIVLCUNT_8822B) & BIT_MASK_BCNIVLCUNT_8822B)
+
+/* 2 REG_BCNDROPCTRL_8822B */
+#define BIT_BEACON_DROP_EN_8822B BIT(7)
+
+#define BIT_SHIFT_BEACON_DROP_IVL_8822B 0
+#define BIT_MASK_BEACON_DROP_IVL_8822B 0x7f
+#define BIT_BEACON_DROP_IVL_8822B(x)                                           \
+	(((x) & BIT_MASK_BEACON_DROP_IVL_8822B)                                \
+	 << BIT_SHIFT_BEACON_DROP_IVL_8822B)
+#define BIT_GET_BEACON_DROP_IVL_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_BEACON_DROP_IVL_8822B) &                            \
+	 BIT_MASK_BEACON_DROP_IVL_8822B)
+
+/* 2 REG_HGQ_TIMEOUT_PERIOD_8822B */
+
+#define BIT_SHIFT_HGQ_TIMEOUT_PERIOD_8822B 0
+#define BIT_MASK_HGQ_TIMEOUT_PERIOD_8822B 0xff
+#define BIT_HGQ_TIMEOUT_PERIOD_8822B(x)                                        \
+	(((x) & BIT_MASK_HGQ_TIMEOUT_PERIOD_8822B)                             \
+	 << BIT_SHIFT_HGQ_TIMEOUT_PERIOD_8822B)
+#define BIT_GET_HGQ_TIMEOUT_PERIOD_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_HGQ_TIMEOUT_PERIOD_8822B) &                         \
+	 BIT_MASK_HGQ_TIMEOUT_PERIOD_8822B)
+
+/* 2 REG_TXCMD_TIMEOUT_PERIOD_8822B */
+
+#define BIT_SHIFT_TXCMD_TIMEOUT_PERIOD_8822B 0
+#define BIT_MASK_TXCMD_TIMEOUT_PERIOD_8822B 0xff
+#define BIT_TXCMD_TIMEOUT_PERIOD_8822B(x)                                      \
+	(((x) & BIT_MASK_TXCMD_TIMEOUT_PERIOD_8822B)                           \
+	 << BIT_SHIFT_TXCMD_TIMEOUT_PERIOD_8822B)
+#define BIT_GET_TXCMD_TIMEOUT_PERIOD_8822B(x)                                  \
+	(((x) >> BIT_SHIFT_TXCMD_TIMEOUT_PERIOD_8822B) &                       \
+	 BIT_MASK_TXCMD_TIMEOUT_PERIOD_8822B)
+
+/* 2 REG_MISC_CTRL_8822B */
+#define BIT_DIS_TRX_CAL_BCN_8822B BIT(5)
+#define BIT_DIS_TX_CAL_TBTT_8822B BIT(4)
+#define BIT_EN_FREECNT_8822B BIT(3)
+#define BIT_BCN_AGGRESSION_8822B BIT(2)
+
+#define BIT_SHIFT_DIS_SECONDARY_CCA_8822B 0
+#define BIT_MASK_DIS_SECONDARY_CCA_8822B 0x3
+#define BIT_DIS_SECONDARY_CCA_8822B(x)                                         \
+	(((x) & BIT_MASK_DIS_SECONDARY_CCA_8822B)                              \
+	 << BIT_SHIFT_DIS_SECONDARY_CCA_8822B)
+#define BIT_GET_DIS_SECONDARY_CCA_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_DIS_SECONDARY_CCA_8822B) &                          \
+	 BIT_MASK_DIS_SECONDARY_CCA_8822B)
+
+/* 2 REG_BCN_CTRL_CLINT1_8822B */
+#define BIT_CLI1_DIS_RX_BSSID_FIT_8822B BIT(6)
+#define BIT_CLI1_DIS_TSF_UDT_8822B BIT(4)
+#define BIT_CLI1_EN_BCN_FUNCTION_8822B BIT(3)
+#define BIT_CLI1_EN_RXBCN_RPT_8822B BIT(2)
+#define BIT_CLI1_ENP2P_CTWINDOW_8822B BIT(1)
+#define BIT_CLI1_ENP2P_BCNQ_AREA_8822B BIT(0)
+
+/* 2 REG_BCN_CTRL_CLINT2_8822B */
+#define BIT_CLI2_DIS_RX_BSSID_FIT_8822B BIT(6)
+#define BIT_CLI2_DIS_TSF_UDT_8822B BIT(4)
+#define BIT_CLI2_EN_BCN_FUNCTION_8822B BIT(3)
+#define BIT_CLI2_EN_RXBCN_RPT_8822B BIT(2)
+#define BIT_CLI2_ENP2P_CTWINDOW_8822B BIT(1)
+#define BIT_CLI2_ENP2P_BCNQ_AREA_8822B BIT(0)
+
+/* 2 REG_BCN_CTRL_CLINT3_8822B */
+#define BIT_CLI3_DIS_RX_BSSID_FIT_8822B BIT(6)
+#define BIT_CLI3_DIS_TSF_UDT_8822B BIT(4)
+#define BIT_CLI3_EN_BCN_FUNCTION_8822B BIT(3)
+#define BIT_CLI3_EN_RXBCN_RPT_8822B BIT(2)
+#define BIT_CLI3_ENP2P_CTWINDOW_8822B BIT(1)
+#define BIT_CLI3_ENP2P_BCNQ_AREA_8822B BIT(0)
+
+/* 2 REG_EXTEND_CTRL_8822B */
+#define BIT_EN_TSFBIT32_RST_P2P2_8822B BIT(5)
+#define BIT_EN_TSFBIT32_RST_P2P1_8822B BIT(4)
+
+#define BIT_SHIFT_PORT_SEL_8822B 0
+#define BIT_MASK_PORT_SEL_8822B 0x7
+#define BIT_PORT_SEL_8822B(x)                                                  \
+	(((x) & BIT_MASK_PORT_SEL_8822B) << BIT_SHIFT_PORT_SEL_8822B)
+#define BIT_GET_PORT_SEL_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_PORT_SEL_8822B) & BIT_MASK_PORT_SEL_8822B)
+
+/* 2 REG_P2PPS1_SPEC_STATE_8822B */
+#define BIT_P2P1_SPEC_POWER_STATE_8822B BIT(7)
+#define BIT_P2P1_SPEC_CTWINDOW_ON_8822B BIT(6)
+#define BIT_P2P1_SPEC_BCN_AREA_ON_8822B BIT(5)
+#define BIT_P2P1_SPEC_CTWIN_EARLY_DISTX_8822B BIT(4)
+#define BIT_P2P1_SPEC_NOA1_OFF_PERIOD_8822B BIT(3)
+#define BIT_P2P1_SPEC_FORCE_DOZE1_8822B BIT(2)
+#define BIT_P2P1_SPEC_NOA0_OFF_PERIOD_8822B BIT(1)
+#define BIT_P2P1_SPEC_FORCE_DOZE0_8822B BIT(0)
+
+/* 2 REG_P2PPS1_STATE_8822B */
+#define BIT_P2P1_POWER_STATE_8822B BIT(7)
+#define BIT_P2P1_CTWINDOW_ON_8822B BIT(6)
+#define BIT_P2P1_BEACON_AREA_ON_8822B BIT(5)
+#define BIT_P2P1_CTWIN_EARLY_DISTX_8822B BIT(4)
+#define BIT_P2P1_NOA1_OFF_PERIOD_8822B BIT(3)
+#define BIT_P2P1_FORCE_DOZE1_8822B BIT(2)
+#define BIT_P2P1_NOA0_OFF_PERIOD_8822B BIT(1)
+#define BIT_P2P1_FORCE_DOZE0_8822B BIT(0)
+
+/* 2 REG_P2PPS2_SPEC_STATE_8822B */
+#define BIT_P2P2_SPEC_POWER_STATE_8822B BIT(7)
+#define BIT_P2P2_SPEC_CTWINDOW_ON_8822B BIT(6)
+#define BIT_P2P2_SPEC_BCN_AREA_ON_8822B BIT(5)
+#define BIT_P2P2_SPEC_CTWIN_EARLY_DISTX_8822B BIT(4)
+#define BIT_P2P2_SPEC_NOA1_OFF_PERIOD_8822B BIT(3)
+#define BIT_P2P2_SPEC_FORCE_DOZE1_8822B BIT(2)
+#define BIT_P2P2_SPEC_NOA0_OFF_PERIOD_8822B BIT(1)
+#define BIT_P2P2_SPEC_FORCE_DOZE0_8822B BIT(0)
+
+/* 2 REG_P2PPS2_STATE_8822B */
+#define BIT_P2P2_POWER_STATE_8822B BIT(7)
+#define BIT_P2P2_CTWINDOW_ON_8822B BIT(6)
+#define BIT_P2P2_BEACON_AREA_ON_8822B BIT(5)
+#define BIT_P2P2_CTWIN_EARLY_DISTX_8822B BIT(4)
+#define BIT_P2P2_NOA1_OFF_PERIOD_8822B BIT(3)
+#define BIT_P2P2_FORCE_DOZE1_8822B BIT(2)
+#define BIT_P2P2_NOA0_OFF_PERIOD_8822B BIT(1)
+#define BIT_P2P2_FORCE_DOZE0_8822B BIT(0)
+
+/* 2 REG_PS_TIMER0_8822B */
+
+#define BIT_SHIFT_PSTIMER0_INT_8822B 5
+#define BIT_MASK_PSTIMER0_INT_8822B 0x7ffffff
+#define BIT_PSTIMER0_INT_8822B(x)                                              \
+	(((x) & BIT_MASK_PSTIMER0_INT_8822B) << BIT_SHIFT_PSTIMER0_INT_8822B)
+#define BIT_GET_PSTIMER0_INT_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_PSTIMER0_INT_8822B) & BIT_MASK_PSTIMER0_INT_8822B)
+
+/* 2 REG_PS_TIMER1_8822B */
+
+#define BIT_SHIFT_PSTIMER1_INT_8822B 5
+#define BIT_MASK_PSTIMER1_INT_8822B 0x7ffffff
+#define BIT_PSTIMER1_INT_8822B(x)                                              \
+	(((x) & BIT_MASK_PSTIMER1_INT_8822B) << BIT_SHIFT_PSTIMER1_INT_8822B)
+#define BIT_GET_PSTIMER1_INT_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_PSTIMER1_INT_8822B) & BIT_MASK_PSTIMER1_INT_8822B)
+
+/* 2 REG_PS_TIMER2_8822B */
+
+#define BIT_SHIFT_PSTIMER2_INT_8822B 5
+#define BIT_MASK_PSTIMER2_INT_8822B 0x7ffffff
+#define BIT_PSTIMER2_INT_8822B(x)                                              \
+	(((x) & BIT_MASK_PSTIMER2_INT_8822B) << BIT_SHIFT_PSTIMER2_INT_8822B)
+#define BIT_GET_PSTIMER2_INT_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_PSTIMER2_INT_8822B) & BIT_MASK_PSTIMER2_INT_8822B)
+
+/* 2 REG_TBTT_CTN_AREA_8822B */
+
+#define BIT_SHIFT_TBTT_CTN_AREA_8822B 0
+#define BIT_MASK_TBTT_CTN_AREA_8822B 0xff
+#define BIT_TBTT_CTN_AREA_8822B(x)                                             \
+	(((x) & BIT_MASK_TBTT_CTN_AREA_8822B) << BIT_SHIFT_TBTT_CTN_AREA_8822B)
+#define BIT_GET_TBTT_CTN_AREA_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_TBTT_CTN_AREA_8822B) & BIT_MASK_TBTT_CTN_AREA_8822B)
+
+/* 2 REG_FORCE_BCN_IFS_8822B */
+
+#define BIT_SHIFT_FORCE_BCN_IFS_8822B 0
+#define BIT_MASK_FORCE_BCN_IFS_8822B 0xff
+#define BIT_FORCE_BCN_IFS_8822B(x)                                             \
+	(((x) & BIT_MASK_FORCE_BCN_IFS_8822B) << BIT_SHIFT_FORCE_BCN_IFS_8822B)
+#define BIT_GET_FORCE_BCN_IFS_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_FORCE_BCN_IFS_8822B) & BIT_MASK_FORCE_BCN_IFS_8822B)
+
+/* 2 REG_TXOP_MIN_8822B */
+
+#define BIT_SHIFT_TXOP_MIN_8822B 0
+#define BIT_MASK_TXOP_MIN_8822B 0x3fff
+#define BIT_TXOP_MIN_8822B(x)                                                  \
+	(((x) & BIT_MASK_TXOP_MIN_8822B) << BIT_SHIFT_TXOP_MIN_8822B)
+#define BIT_GET_TXOP_MIN_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_TXOP_MIN_8822B) & BIT_MASK_TXOP_MIN_8822B)
+
+/* 2 REG_PRE_BKF_TIME_8822B */
+
+#define BIT_SHIFT_PRE_BKF_TIME_8822B 0
+#define BIT_MASK_PRE_BKF_TIME_8822B 0xff
+#define BIT_PRE_BKF_TIME_8822B(x)                                              \
+	(((x) & BIT_MASK_PRE_BKF_TIME_8822B) << BIT_SHIFT_PRE_BKF_TIME_8822B)
+#define BIT_GET_PRE_BKF_TIME_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_PRE_BKF_TIME_8822B) & BIT_MASK_PRE_BKF_TIME_8822B)
+
+/* 2 REG_CROSS_TXOP_CTRL_8822B */
+#define BIT_DTIM_BYPASS_8822B BIT(2)
+#define BIT_RTS_NAV_TXOP_8822B BIT(1)
+#define BIT_NOT_CROSS_TXOP_8822B BIT(0)
+
+/* 2 REG_ATIMWND2_8822B */
+
+#define BIT_SHIFT_ATIMWND2_8822B 0
+#define BIT_MASK_ATIMWND2_8822B 0xff
+#define BIT_ATIMWND2_8822B(x)                                                  \
+	(((x) & BIT_MASK_ATIMWND2_8822B) << BIT_SHIFT_ATIMWND2_8822B)
+#define BIT_GET_ATIMWND2_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_ATIMWND2_8822B) & BIT_MASK_ATIMWND2_8822B)
+
+/* 2 REG_ATIMWND3_8822B */
+
+#define BIT_SHIFT_ATIMWND3_8822B 0
+#define BIT_MASK_ATIMWND3_8822B 0xff
+#define BIT_ATIMWND3_8822B(x)                                                  \
+	(((x) & BIT_MASK_ATIMWND3_8822B) << BIT_SHIFT_ATIMWND3_8822B)
+#define BIT_GET_ATIMWND3_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_ATIMWND3_8822B) & BIT_MASK_ATIMWND3_8822B)
+
+/* 2 REG_ATIMWND4_8822B */
+
+#define BIT_SHIFT_ATIMWND4_8822B 0
+#define BIT_MASK_ATIMWND4_8822B 0xff
+#define BIT_ATIMWND4_8822B(x)                                                  \
+	(((x) & BIT_MASK_ATIMWND4_8822B) << BIT_SHIFT_ATIMWND4_8822B)
+#define BIT_GET_ATIMWND4_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_ATIMWND4_8822B) & BIT_MASK_ATIMWND4_8822B)
+
+/* 2 REG_ATIMWND5_8822B */
+
+#define BIT_SHIFT_ATIMWND5_8822B 0
+#define BIT_MASK_ATIMWND5_8822B 0xff
+#define BIT_ATIMWND5_8822B(x)                                                  \
+	(((x) & BIT_MASK_ATIMWND5_8822B) << BIT_SHIFT_ATIMWND5_8822B)
+#define BIT_GET_ATIMWND5_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_ATIMWND5_8822B) & BIT_MASK_ATIMWND5_8822B)
+
+/* 2 REG_ATIMWND6_8822B */
+
+#define BIT_SHIFT_ATIMWND6_8822B 0
+#define BIT_MASK_ATIMWND6_8822B 0xff
+#define BIT_ATIMWND6_8822B(x)                                                  \
+	(((x) & BIT_MASK_ATIMWND6_8822B) << BIT_SHIFT_ATIMWND6_8822B)
+#define BIT_GET_ATIMWND6_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_ATIMWND6_8822B) & BIT_MASK_ATIMWND6_8822B)
+
+/* 2 REG_ATIMWND7_8822B */
+
+#define BIT_SHIFT_ATIMWND7_8822B 0
+#define BIT_MASK_ATIMWND7_8822B 0xff
+#define BIT_ATIMWND7_8822B(x)                                                  \
+	(((x) & BIT_MASK_ATIMWND7_8822B) << BIT_SHIFT_ATIMWND7_8822B)
+#define BIT_GET_ATIMWND7_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_ATIMWND7_8822B) & BIT_MASK_ATIMWND7_8822B)
+
+/* 2 REG_ATIMUGT_8822B */
+
+#define BIT_SHIFT_ATIM_URGENT_8822B 0
+#define BIT_MASK_ATIM_URGENT_8822B 0xff
+#define BIT_ATIM_URGENT_8822B(x)                                               \
+	(((x) & BIT_MASK_ATIM_URGENT_8822B) << BIT_SHIFT_ATIM_URGENT_8822B)
+#define BIT_GET_ATIM_URGENT_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_ATIM_URGENT_8822B) & BIT_MASK_ATIM_URGENT_8822B)
+
+/* 2 REG_HIQ_NO_LMT_EN_8822B */
+#define BIT_HIQ_NO_LMT_EN_VAP7_8822B BIT(7)
+#define BIT_HIQ_NO_LMT_EN_VAP6_8822B BIT(6)
+#define BIT_HIQ_NO_LMT_EN_VAP5_8822B BIT(5)
+#define BIT_HIQ_NO_LMT_EN_VAP4_8822B BIT(4)
+#define BIT_HIQ_NO_LMT_EN_VAP3_8822B BIT(3)
+#define BIT_HIQ_NO_LMT_EN_VAP2_8822B BIT(2)
+#define BIT_HIQ_NO_LMT_EN_VAP1_8822B BIT(1)
+#define BIT_HIQ_NO_LMT_EN_ROOT_8822B BIT(0)
+
+/* 2 REG_DTIM_COUNTER_ROOT_8822B */
+
+#define BIT_SHIFT_DTIM_COUNT_ROOT_8822B 0
+#define BIT_MASK_DTIM_COUNT_ROOT_8822B 0xff
+#define BIT_DTIM_COUNT_ROOT_8822B(x)                                           \
+	(((x) & BIT_MASK_DTIM_COUNT_ROOT_8822B)                                \
+	 << BIT_SHIFT_DTIM_COUNT_ROOT_8822B)
+#define BIT_GET_DTIM_COUNT_ROOT_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_DTIM_COUNT_ROOT_8822B) &                            \
+	 BIT_MASK_DTIM_COUNT_ROOT_8822B)
+
+/* 2 REG_DTIM_COUNTER_VAP1_8822B */
+
+#define BIT_SHIFT_DTIM_COUNT_VAP1_8822B 0
+#define BIT_MASK_DTIM_COUNT_VAP1_8822B 0xff
+#define BIT_DTIM_COUNT_VAP1_8822B(x)                                           \
+	(((x) & BIT_MASK_DTIM_COUNT_VAP1_8822B)                                \
+	 << BIT_SHIFT_DTIM_COUNT_VAP1_8822B)
+#define BIT_GET_DTIM_COUNT_VAP1_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_DTIM_COUNT_VAP1_8822B) &                            \
+	 BIT_MASK_DTIM_COUNT_VAP1_8822B)
+
+/* 2 REG_DTIM_COUNTER_VAP2_8822B */
+
+#define BIT_SHIFT_DTIM_COUNT_VAP2_8822B 0
+#define BIT_MASK_DTIM_COUNT_VAP2_8822B 0xff
+#define BIT_DTIM_COUNT_VAP2_8822B(x)                                           \
+	(((x) & BIT_MASK_DTIM_COUNT_VAP2_8822B)                                \
+	 << BIT_SHIFT_DTIM_COUNT_VAP2_8822B)
+#define BIT_GET_DTIM_COUNT_VAP2_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_DTIM_COUNT_VAP2_8822B) &                            \
+	 BIT_MASK_DTIM_COUNT_VAP2_8822B)
+
+/* 2 REG_DTIM_COUNTER_VAP3_8822B */
+
+#define BIT_SHIFT_DTIM_COUNT_VAP3_8822B 0
+#define BIT_MASK_DTIM_COUNT_VAP3_8822B 0xff
+#define BIT_DTIM_COUNT_VAP3_8822B(x)                                           \
+	(((x) & BIT_MASK_DTIM_COUNT_VAP3_8822B)                                \
+	 << BIT_SHIFT_DTIM_COUNT_VAP3_8822B)
+#define BIT_GET_DTIM_COUNT_VAP3_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_DTIM_COUNT_VAP3_8822B) &                            \
+	 BIT_MASK_DTIM_COUNT_VAP3_8822B)
+
+/* 2 REG_DTIM_COUNTER_VAP4_8822B */
+
+#define BIT_SHIFT_DTIM_COUNT_VAP4_8822B 0
+#define BIT_MASK_DTIM_COUNT_VAP4_8822B 0xff
+#define BIT_DTIM_COUNT_VAP4_8822B(x)                                           \
+	(((x) & BIT_MASK_DTIM_COUNT_VAP4_8822B)                                \
+	 << BIT_SHIFT_DTIM_COUNT_VAP4_8822B)
+#define BIT_GET_DTIM_COUNT_VAP4_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_DTIM_COUNT_VAP4_8822B) &                            \
+	 BIT_MASK_DTIM_COUNT_VAP4_8822B)
+
+/* 2 REG_DTIM_COUNTER_VAP5_8822B */
+
+#define BIT_SHIFT_DTIM_COUNT_VAP5_8822B 0
+#define BIT_MASK_DTIM_COUNT_VAP5_8822B 0xff
+#define BIT_DTIM_COUNT_VAP5_8822B(x)                                           \
+	(((x) & BIT_MASK_DTIM_COUNT_VAP5_8822B)                                \
+	 << BIT_SHIFT_DTIM_COUNT_VAP5_8822B)
+#define BIT_GET_DTIM_COUNT_VAP5_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_DTIM_COUNT_VAP5_8822B) &                            \
+	 BIT_MASK_DTIM_COUNT_VAP5_8822B)
+
+/* 2 REG_DTIM_COUNTER_VAP6_8822B */
+
+#define BIT_SHIFT_DTIM_COUNT_VAP6_8822B 0
+#define BIT_MASK_DTIM_COUNT_VAP6_8822B 0xff
+#define BIT_DTIM_COUNT_VAP6_8822B(x)                                           \
+	(((x) & BIT_MASK_DTIM_COUNT_VAP6_8822B)                                \
+	 << BIT_SHIFT_DTIM_COUNT_VAP6_8822B)
+#define BIT_GET_DTIM_COUNT_VAP6_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_DTIM_COUNT_VAP6_8822B) &                            \
+	 BIT_MASK_DTIM_COUNT_VAP6_8822B)
+
+/* 2 REG_DTIM_COUNTER_VAP7_8822B */
+
+#define BIT_SHIFT_DTIM_COUNT_VAP7_8822B 0
+#define BIT_MASK_DTIM_COUNT_VAP7_8822B 0xff
+#define BIT_DTIM_COUNT_VAP7_8822B(x)                                           \
+	(((x) & BIT_MASK_DTIM_COUNT_VAP7_8822B)                                \
+	 << BIT_SHIFT_DTIM_COUNT_VAP7_8822B)
+#define BIT_GET_DTIM_COUNT_VAP7_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_DTIM_COUNT_VAP7_8822B) &                            \
+	 BIT_MASK_DTIM_COUNT_VAP7_8822B)
+
+/* 2 REG_DIS_ATIM_8822B */
+#define BIT_DIS_ATIM_VAP7_8822B BIT(7)
+#define BIT_DIS_ATIM_VAP6_8822B BIT(6)
+#define BIT_DIS_ATIM_VAP5_8822B BIT(5)
+#define BIT_DIS_ATIM_VAP4_8822B BIT(4)
+#define BIT_DIS_ATIM_VAP3_8822B BIT(3)
+#define BIT_DIS_ATIM_VAP2_8822B BIT(2)
+#define BIT_DIS_ATIM_VAP1_8822B BIT(1)
+#define BIT_DIS_ATIM_ROOT_8822B BIT(0)
+
+/* 2 REG_EARLY_128US_8822B */
+
+#define BIT_SHIFT_TSFT_SEL_TIMER1_8822B 3
+#define BIT_MASK_TSFT_SEL_TIMER1_8822B 0x7
+#define BIT_TSFT_SEL_TIMER1_8822B(x)                                           \
+	(((x) & BIT_MASK_TSFT_SEL_TIMER1_8822B)                                \
+	 << BIT_SHIFT_TSFT_SEL_TIMER1_8822B)
+#define BIT_GET_TSFT_SEL_TIMER1_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_TSFT_SEL_TIMER1_8822B) &                            \
+	 BIT_MASK_TSFT_SEL_TIMER1_8822B)
+
+#define BIT_SHIFT_EARLY_128US_8822B 0
+#define BIT_MASK_EARLY_128US_8822B 0x7
+#define BIT_EARLY_128US_8822B(x)                                               \
+	(((x) & BIT_MASK_EARLY_128US_8822B) << BIT_SHIFT_EARLY_128US_8822B)
+#define BIT_GET_EARLY_128US_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_EARLY_128US_8822B) & BIT_MASK_EARLY_128US_8822B)
+
+/* 2 REG_P2PPS1_CTRL_8822B */
+#define BIT_P2P1_CTW_ALLSTASLEEP_8822B BIT(7)
+#define BIT_P2P1_OFF_DISTX_EN_8822B BIT(6)
+#define BIT_P2P1_PWR_MGT_EN_8822B BIT(5)
+#define BIT_P2P1_NOA1_EN_8822B BIT(2)
+#define BIT_P2P1_NOA0_EN_8822B BIT(1)
+
+/* 2 REG_P2PPS2_CTRL_8822B */
+#define BIT_P2P2_CTW_ALLSTASLEEP_8822B BIT(7)
+#define BIT_P2P2_OFF_DISTX_EN_8822B BIT(6)
+#define BIT_P2P2_PWR_MGT_EN_8822B BIT(5)
+#define BIT_P2P2_NOA1_EN_8822B BIT(2)
+#define BIT_P2P2_NOA0_EN_8822B BIT(1)
+
+/* 2 REG_TIMER0_SRC_SEL_8822B */
+
+#define BIT_SHIFT_SYNC_CLI_SEL_8822B 4
+#define BIT_MASK_SYNC_CLI_SEL_8822B 0x7
+#define BIT_SYNC_CLI_SEL_8822B(x)                                              \
+	(((x) & BIT_MASK_SYNC_CLI_SEL_8822B) << BIT_SHIFT_SYNC_CLI_SEL_8822B)
+#define BIT_GET_SYNC_CLI_SEL_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_SYNC_CLI_SEL_8822B) & BIT_MASK_SYNC_CLI_SEL_8822B)
+
+#define BIT_SHIFT_TSFT_SEL_TIMER0_8822B 0
+#define BIT_MASK_TSFT_SEL_TIMER0_8822B 0x7
+#define BIT_TSFT_SEL_TIMER0_8822B(x)                                           \
+	(((x) & BIT_MASK_TSFT_SEL_TIMER0_8822B)                                \
+	 << BIT_SHIFT_TSFT_SEL_TIMER0_8822B)
+#define BIT_GET_TSFT_SEL_TIMER0_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_TSFT_SEL_TIMER0_8822B) &                            \
+	 BIT_MASK_TSFT_SEL_TIMER0_8822B)
+
+/* 2 REG_NOA_UNIT_SEL_8822B */
+
+#define BIT_SHIFT_NOA_UNIT2_SEL_8822B 8
+#define BIT_MASK_NOA_UNIT2_SEL_8822B 0x7
+#define BIT_NOA_UNIT2_SEL_8822B(x)                                             \
+	(((x) & BIT_MASK_NOA_UNIT2_SEL_8822B) << BIT_SHIFT_NOA_UNIT2_SEL_8822B)
+#define BIT_GET_NOA_UNIT2_SEL_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_NOA_UNIT2_SEL_8822B) & BIT_MASK_NOA_UNIT2_SEL_8822B)
+
+#define BIT_SHIFT_NOA_UNIT1_SEL_8822B 4
+#define BIT_MASK_NOA_UNIT1_SEL_8822B 0x7
+#define BIT_NOA_UNIT1_SEL_8822B(x)                                             \
+	(((x) & BIT_MASK_NOA_UNIT1_SEL_8822B) << BIT_SHIFT_NOA_UNIT1_SEL_8822B)
+#define BIT_GET_NOA_UNIT1_SEL_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_NOA_UNIT1_SEL_8822B) & BIT_MASK_NOA_UNIT1_SEL_8822B)
+
+#define BIT_SHIFT_NOA_UNIT0_SEL_8822B 0
+#define BIT_MASK_NOA_UNIT0_SEL_8822B 0x7
+#define BIT_NOA_UNIT0_SEL_8822B(x)                                             \
+	(((x) & BIT_MASK_NOA_UNIT0_SEL_8822B) << BIT_SHIFT_NOA_UNIT0_SEL_8822B)
+#define BIT_GET_NOA_UNIT0_SEL_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_NOA_UNIT0_SEL_8822B) & BIT_MASK_NOA_UNIT0_SEL_8822B)
+
+/* 2 REG_P2POFF_DIS_TXTIME_8822B */
+
+#define BIT_SHIFT_P2POFF_DIS_TXTIME_8822B 0
+#define BIT_MASK_P2POFF_DIS_TXTIME_8822B 0xff
+#define BIT_P2POFF_DIS_TXTIME_8822B(x)                                         \
+	(((x) & BIT_MASK_P2POFF_DIS_TXTIME_8822B)                              \
+	 << BIT_SHIFT_P2POFF_DIS_TXTIME_8822B)
+#define BIT_GET_P2POFF_DIS_TXTIME_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_P2POFF_DIS_TXTIME_8822B) &                          \
+	 BIT_MASK_P2POFF_DIS_TXTIME_8822B)
+
+/* 2 REG_MBSSID_BCN_SPACE2_8822B */
+
+#define BIT_SHIFT_BCN_SPACE_CLINT2_8822B 16
+#define BIT_MASK_BCN_SPACE_CLINT2_8822B 0xfff
+#define BIT_BCN_SPACE_CLINT2_8822B(x)                                          \
+	(((x) & BIT_MASK_BCN_SPACE_CLINT2_8822B)                               \
+	 << BIT_SHIFT_BCN_SPACE_CLINT2_8822B)
+#define BIT_GET_BCN_SPACE_CLINT2_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_BCN_SPACE_CLINT2_8822B) &                           \
+	 BIT_MASK_BCN_SPACE_CLINT2_8822B)
+
+#define BIT_SHIFT_BCN_SPACE_CLINT1_8822B 0
+#define BIT_MASK_BCN_SPACE_CLINT1_8822B 0xfff
+#define BIT_BCN_SPACE_CLINT1_8822B(x)                                          \
+	(((x) & BIT_MASK_BCN_SPACE_CLINT1_8822B)                               \
+	 << BIT_SHIFT_BCN_SPACE_CLINT1_8822B)
+#define BIT_GET_BCN_SPACE_CLINT1_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_BCN_SPACE_CLINT1_8822B) &                           \
+	 BIT_MASK_BCN_SPACE_CLINT1_8822B)
+
+/* 2 REG_MBSSID_BCN_SPACE3_8822B */
+
+#define BIT_SHIFT_SUB_BCN_SPACE_8822B 16
+#define BIT_MASK_SUB_BCN_SPACE_8822B 0xff
+#define BIT_SUB_BCN_SPACE_8822B(x)                                             \
+	(((x) & BIT_MASK_SUB_BCN_SPACE_8822B) << BIT_SHIFT_SUB_BCN_SPACE_8822B)
+#define BIT_GET_SUB_BCN_SPACE_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_SUB_BCN_SPACE_8822B) & BIT_MASK_SUB_BCN_SPACE_8822B)
+
+#define BIT_SHIFT_BCN_SPACE_CLINT3_8822B 0
+#define BIT_MASK_BCN_SPACE_CLINT3_8822B 0xfff
+#define BIT_BCN_SPACE_CLINT3_8822B(x)                                          \
+	(((x) & BIT_MASK_BCN_SPACE_CLINT3_8822B)                               \
+	 << BIT_SHIFT_BCN_SPACE_CLINT3_8822B)
+#define BIT_GET_BCN_SPACE_CLINT3_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_BCN_SPACE_CLINT3_8822B) &                           \
+	 BIT_MASK_BCN_SPACE_CLINT3_8822B)
+
+/* 2 REG_ACMHWCTRL_8822B */
+#define BIT_BEQ_ACM_STATUS_8822B BIT(7)
+#define BIT_VIQ_ACM_STATUS_8822B BIT(6)
+#define BIT_VOQ_ACM_STATUS_8822B BIT(5)
+#define BIT_BEQ_ACM_EN_8822B BIT(3)
+#define BIT_VIQ_ACM_EN_8822B BIT(2)
+#define BIT_VOQ_ACM_EN_8822B BIT(1)
+#define BIT_ACMHWEN_8822B BIT(0)
+
+/* 2 REG_ACMRSTCTRL_8822B */
+#define BIT_BE_ACM_RESET_USED_TIME_8822B BIT(2)
+#define BIT_VI_ACM_RESET_USED_TIME_8822B BIT(1)
+#define BIT_VO_ACM_RESET_USED_TIME_8822B BIT(0)
+
+/* 2 REG_ACMAVG_8822B */
+
+#define BIT_SHIFT_AVGPERIOD_8822B 0
+#define BIT_MASK_AVGPERIOD_8822B 0xffff
+#define BIT_AVGPERIOD_8822B(x)                                                 \
+	(((x) & BIT_MASK_AVGPERIOD_8822B) << BIT_SHIFT_AVGPERIOD_8822B)
+#define BIT_GET_AVGPERIOD_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_AVGPERIOD_8822B) & BIT_MASK_AVGPERIOD_8822B)
+
+/* 2 REG_VO_ADMTIME_8822B */
+
+#define BIT_SHIFT_VO_ADMITTED_TIME_8822B 0
+#define BIT_MASK_VO_ADMITTED_TIME_8822B 0xffff
+#define BIT_VO_ADMITTED_TIME_8822B(x)                                          \
+	(((x) & BIT_MASK_VO_ADMITTED_TIME_8822B)                               \
+	 << BIT_SHIFT_VO_ADMITTED_TIME_8822B)
+#define BIT_GET_VO_ADMITTED_TIME_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_VO_ADMITTED_TIME_8822B) &                           \
+	 BIT_MASK_VO_ADMITTED_TIME_8822B)
+
+/* 2 REG_VI_ADMTIME_8822B */
+
+#define BIT_SHIFT_VI_ADMITTED_TIME_8822B 0
+#define BIT_MASK_VI_ADMITTED_TIME_8822B 0xffff
+#define BIT_VI_ADMITTED_TIME_8822B(x)                                          \
+	(((x) & BIT_MASK_VI_ADMITTED_TIME_8822B)                               \
+	 << BIT_SHIFT_VI_ADMITTED_TIME_8822B)
+#define BIT_GET_VI_ADMITTED_TIME_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_VI_ADMITTED_TIME_8822B) &                           \
+	 BIT_MASK_VI_ADMITTED_TIME_8822B)
+
+/* 2 REG_BE_ADMTIME_8822B */
+
+#define BIT_SHIFT_BE_ADMITTED_TIME_8822B 0
+#define BIT_MASK_BE_ADMITTED_TIME_8822B 0xffff
+#define BIT_BE_ADMITTED_TIME_8822B(x)                                          \
+	(((x) & BIT_MASK_BE_ADMITTED_TIME_8822B)                               \
+	 << BIT_SHIFT_BE_ADMITTED_TIME_8822B)
+#define BIT_GET_BE_ADMITTED_TIME_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_BE_ADMITTED_TIME_8822B) &                           \
+	 BIT_MASK_BE_ADMITTED_TIME_8822B)
+
+/* 2 REG_EDCA_RANDOM_GEN_8822B */
+
+#define BIT_SHIFT_RANDOM_GEN_8822B 0
+#define BIT_MASK_RANDOM_GEN_8822B 0xffffff
+#define BIT_RANDOM_GEN_8822B(x)                                                \
+	(((x) & BIT_MASK_RANDOM_GEN_8822B) << BIT_SHIFT_RANDOM_GEN_8822B)
+#define BIT_GET_RANDOM_GEN_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_RANDOM_GEN_8822B) & BIT_MASK_RANDOM_GEN_8822B)
+
+/* 2 REG_TXCMD_NOA_SEL_8822B */
+
+#define BIT_SHIFT_NOA_SEL_8822B 4
+#define BIT_MASK_NOA_SEL_8822B 0x7
+#define BIT_NOA_SEL_8822B(x)                                                   \
+	(((x) & BIT_MASK_NOA_SEL_8822B) << BIT_SHIFT_NOA_SEL_8822B)
+#define BIT_GET_NOA_SEL_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_NOA_SEL_8822B) & BIT_MASK_NOA_SEL_8822B)
+
+#define BIT_SHIFT_TXCMD_SEG_SEL_8822B 0
+#define BIT_MASK_TXCMD_SEG_SEL_8822B 0xf
+#define BIT_TXCMD_SEG_SEL_8822B(x)                                             \
+	(((x) & BIT_MASK_TXCMD_SEG_SEL_8822B) << BIT_SHIFT_TXCMD_SEG_SEL_8822B)
+#define BIT_GET_TXCMD_SEG_SEL_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_TXCMD_SEG_SEL_8822B) & BIT_MASK_TXCMD_SEG_SEL_8822B)
+
+/* 2 REG_NOA_PARAM_8822B */
+
+#define BIT_SHIFT_NOA_COUNT_8822B (96 & CPU_OPT_WIDTH)
+#define BIT_MASK_NOA_COUNT_8822B 0xff
+#define BIT_NOA_COUNT_8822B(x)                                                 \
+	(((x) & BIT_MASK_NOA_COUNT_8822B) << BIT_SHIFT_NOA_COUNT_8822B)
+#define BIT_GET_NOA_COUNT_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_NOA_COUNT_8822B) & BIT_MASK_NOA_COUNT_8822B)
+
+#define BIT_SHIFT_NOA_START_TIME_8822B (64 & CPU_OPT_WIDTH)
+#define BIT_MASK_NOA_START_TIME_8822B 0xffffffffL
+#define BIT_NOA_START_TIME_8822B(x)                                            \
+	(((x) & BIT_MASK_NOA_START_TIME_8822B)                                 \
+	 << BIT_SHIFT_NOA_START_TIME_8822B)
+#define BIT_GET_NOA_START_TIME_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_NOA_START_TIME_8822B) &                             \
+	 BIT_MASK_NOA_START_TIME_8822B)
+
+#define BIT_SHIFT_NOA_INTERVAL_8822B (32 & CPU_OPT_WIDTH)
+#define BIT_MASK_NOA_INTERVAL_8822B 0xffffffffL
+#define BIT_NOA_INTERVAL_8822B(x)                                              \
+	(((x) & BIT_MASK_NOA_INTERVAL_8822B) << BIT_SHIFT_NOA_INTERVAL_8822B)
+#define BIT_GET_NOA_INTERVAL_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_NOA_INTERVAL_8822B) & BIT_MASK_NOA_INTERVAL_8822B)
+
+#define BIT_SHIFT_NOA_DURATION_8822B 0
+#define BIT_MASK_NOA_DURATION_8822B 0xffffffffL
+#define BIT_NOA_DURATION_8822B(x)                                              \
+	(((x) & BIT_MASK_NOA_DURATION_8822B) << BIT_SHIFT_NOA_DURATION_8822B)
+#define BIT_GET_NOA_DURATION_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_NOA_DURATION_8822B) & BIT_MASK_NOA_DURATION_8822B)
+
+/* 2 REG_P2P_RST_8822B */
+#define BIT_P2P2_PWR_RST1_8822B BIT(5)
+#define BIT_P2P2_PWR_RST0_8822B BIT(4)
+#define BIT_P2P1_PWR_RST1_8822B BIT(3)
+#define BIT_P2P1_PWR_RST0_8822B BIT(2)
+#define BIT_P2P_PWR_RST1_V1_8822B BIT(1)
+#define BIT_P2P_PWR_RST0_V1_8822B BIT(0)
+
+/* 2 REG_SCHEDULER_RST_8822B */
+#define BIT_SYNC_CLI_8822B BIT(1)
+#define BIT_SCHEDULER_RST_V1_8822B BIT(0)
+
+/* 2 REG_SCH_TXCMD_8822B */
+
+#define BIT_SHIFT_SCH_TXCMD_8822B 0
+#define BIT_MASK_SCH_TXCMD_8822B 0xffffffffL
+#define BIT_SCH_TXCMD_8822B(x)                                                 \
+	(((x) & BIT_MASK_SCH_TXCMD_8822B) << BIT_SHIFT_SCH_TXCMD_8822B)
+#define BIT_GET_SCH_TXCMD_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_SCH_TXCMD_8822B) & BIT_MASK_SCH_TXCMD_8822B)
+
+/* 2 REG_PAGE5_DUMMY_8822B */
+
+/* 2 REG_CPUMGQ_TX_TIMER_8822B */
+
+#define BIT_SHIFT_CPUMGQ_TX_TIMER_V1_8822B 0
+#define BIT_MASK_CPUMGQ_TX_TIMER_V1_8822B 0xffffffffL
+#define BIT_CPUMGQ_TX_TIMER_V1_8822B(x)                                        \
+	(((x) & BIT_MASK_CPUMGQ_TX_TIMER_V1_8822B)                             \
+	 << BIT_SHIFT_CPUMGQ_TX_TIMER_V1_8822B)
+#define BIT_GET_CPUMGQ_TX_TIMER_V1_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_CPUMGQ_TX_TIMER_V1_8822B) &                         \
+	 BIT_MASK_CPUMGQ_TX_TIMER_V1_8822B)
+
+/* 2 REG_PS_TIMER_A_8822B */
+
+#define BIT_SHIFT_PS_TIMER_A_V1_8822B 0
+#define BIT_MASK_PS_TIMER_A_V1_8822B 0xffffffffL
+#define BIT_PS_TIMER_A_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_PS_TIMER_A_V1_8822B) << BIT_SHIFT_PS_TIMER_A_V1_8822B)
+#define BIT_GET_PS_TIMER_A_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_PS_TIMER_A_V1_8822B) & BIT_MASK_PS_TIMER_A_V1_8822B)
+
+/* 2 REG_PS_TIMER_B_8822B */
+
+#define BIT_SHIFT_PS_TIMER_B_V1_8822B 0
+#define BIT_MASK_PS_TIMER_B_V1_8822B 0xffffffffL
+#define BIT_PS_TIMER_B_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_PS_TIMER_B_V1_8822B) << BIT_SHIFT_PS_TIMER_B_V1_8822B)
+#define BIT_GET_PS_TIMER_B_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_PS_TIMER_B_V1_8822B) & BIT_MASK_PS_TIMER_B_V1_8822B)
+
+/* 2 REG_PS_TIMER_C_8822B */
+
+#define BIT_SHIFT_PS_TIMER_C_V1_8822B 0
+#define BIT_MASK_PS_TIMER_C_V1_8822B 0xffffffffL
+#define BIT_PS_TIMER_C_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_PS_TIMER_C_V1_8822B) << BIT_SHIFT_PS_TIMER_C_V1_8822B)
+#define BIT_GET_PS_TIMER_C_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_PS_TIMER_C_V1_8822B) & BIT_MASK_PS_TIMER_C_V1_8822B)
+
+/* 2 REG_PS_TIMER_ABC_CPUMGQ_TIMER_CRTL_8822B */
+#define BIT_CPUMGQ_TIMER_EN_8822B BIT(31)
+#define BIT_CPUMGQ_TX_EN_8822B BIT(28)
+
+#define BIT_SHIFT_CPUMGQ_TIMER_TSF_SEL_8822B 24
+#define BIT_MASK_CPUMGQ_TIMER_TSF_SEL_8822B 0x7
+#define BIT_CPUMGQ_TIMER_TSF_SEL_8822B(x)                                      \
+	(((x) & BIT_MASK_CPUMGQ_TIMER_TSF_SEL_8822B)                           \
+	 << BIT_SHIFT_CPUMGQ_TIMER_TSF_SEL_8822B)
+#define BIT_GET_CPUMGQ_TIMER_TSF_SEL_8822B(x)                                  \
+	(((x) >> BIT_SHIFT_CPUMGQ_TIMER_TSF_SEL_8822B) &                       \
+	 BIT_MASK_CPUMGQ_TIMER_TSF_SEL_8822B)
+
+#define BIT_PS_TIMER_C_EN_8822B BIT(23)
+
+#define BIT_SHIFT_PS_TIMER_C_TSF_SEL_8822B 16
+#define BIT_MASK_PS_TIMER_C_TSF_SEL_8822B 0x7
+#define BIT_PS_TIMER_C_TSF_SEL_8822B(x)                                        \
+	(((x) & BIT_MASK_PS_TIMER_C_TSF_SEL_8822B)                             \
+	 << BIT_SHIFT_PS_TIMER_C_TSF_SEL_8822B)
+#define BIT_GET_PS_TIMER_C_TSF_SEL_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_PS_TIMER_C_TSF_SEL_8822B) &                         \
+	 BIT_MASK_PS_TIMER_C_TSF_SEL_8822B)
+
+#define BIT_PS_TIMER_B_EN_8822B BIT(15)
+
+#define BIT_SHIFT_PS_TIMER_B_TSF_SEL_8822B 8
+#define BIT_MASK_PS_TIMER_B_TSF_SEL_8822B 0x7
+#define BIT_PS_TIMER_B_TSF_SEL_8822B(x)                                        \
+	(((x) & BIT_MASK_PS_TIMER_B_TSF_SEL_8822B)                             \
+	 << BIT_SHIFT_PS_TIMER_B_TSF_SEL_8822B)
+#define BIT_GET_PS_TIMER_B_TSF_SEL_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_PS_TIMER_B_TSF_SEL_8822B) &                         \
+	 BIT_MASK_PS_TIMER_B_TSF_SEL_8822B)
+
+#define BIT_PS_TIMER_A_EN_8822B BIT(7)
+
+#define BIT_SHIFT_PS_TIMER_A_TSF_SEL_8822B 0
+#define BIT_MASK_PS_TIMER_A_TSF_SEL_8822B 0x7
+#define BIT_PS_TIMER_A_TSF_SEL_8822B(x)                                        \
+	(((x) & BIT_MASK_PS_TIMER_A_TSF_SEL_8822B)                             \
+	 << BIT_SHIFT_PS_TIMER_A_TSF_SEL_8822B)
+#define BIT_GET_PS_TIMER_A_TSF_SEL_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_PS_TIMER_A_TSF_SEL_8822B) &                         \
+	 BIT_MASK_PS_TIMER_A_TSF_SEL_8822B)
+
+/* 2 REG_CPUMGQ_TX_TIMER_EARLY_8822B */
+
+#define BIT_SHIFT_CPUMGQ_TX_TIMER_EARLY_8822B 0
+#define BIT_MASK_CPUMGQ_TX_TIMER_EARLY_8822B 0xff
+#define BIT_CPUMGQ_TX_TIMER_EARLY_8822B(x)                                     \
+	(((x) & BIT_MASK_CPUMGQ_TX_TIMER_EARLY_8822B)                          \
+	 << BIT_SHIFT_CPUMGQ_TX_TIMER_EARLY_8822B)
+#define BIT_GET_CPUMGQ_TX_TIMER_EARLY_8822B(x)                                 \
+	(((x) >> BIT_SHIFT_CPUMGQ_TX_TIMER_EARLY_8822B) &                      \
+	 BIT_MASK_CPUMGQ_TX_TIMER_EARLY_8822B)
+
+/* 2 REG_PS_TIMER_A_EARLY_8822B */
+
+#define BIT_SHIFT_PS_TIMER_A_EARLY_8822B 0
+#define BIT_MASK_PS_TIMER_A_EARLY_8822B 0xff
+#define BIT_PS_TIMER_A_EARLY_8822B(x)                                          \
+	(((x) & BIT_MASK_PS_TIMER_A_EARLY_8822B)                               \
+	 << BIT_SHIFT_PS_TIMER_A_EARLY_8822B)
+#define BIT_GET_PS_TIMER_A_EARLY_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_PS_TIMER_A_EARLY_8822B) &                           \
+	 BIT_MASK_PS_TIMER_A_EARLY_8822B)
+
+/* 2 REG_PS_TIMER_B_EARLY_8822B */
+
+#define BIT_SHIFT_PS_TIMER_B_EARLY_8822B 0
+#define BIT_MASK_PS_TIMER_B_EARLY_8822B 0xff
+#define BIT_PS_TIMER_B_EARLY_8822B(x)                                          \
+	(((x) & BIT_MASK_PS_TIMER_B_EARLY_8822B)                               \
+	 << BIT_SHIFT_PS_TIMER_B_EARLY_8822B)
+#define BIT_GET_PS_TIMER_B_EARLY_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_PS_TIMER_B_EARLY_8822B) &                           \
+	 BIT_MASK_PS_TIMER_B_EARLY_8822B)
+
+/* 2 REG_PS_TIMER_C_EARLY_8822B */
+
+#define BIT_SHIFT_PS_TIMER_C_EARLY_8822B 0
+#define BIT_MASK_PS_TIMER_C_EARLY_8822B 0xff
+#define BIT_PS_TIMER_C_EARLY_8822B(x)                                          \
+	(((x) & BIT_MASK_PS_TIMER_C_EARLY_8822B)                               \
+	 << BIT_SHIFT_PS_TIMER_C_EARLY_8822B)
+#define BIT_GET_PS_TIMER_C_EARLY_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_PS_TIMER_C_EARLY_8822B) &                           \
+	 BIT_MASK_PS_TIMER_C_EARLY_8822B)
+
+/* 2 REG_NOT_VALID_8822B */
+
+/* 2 REG_BWOPMODE_8822B (BW OPERATION MODE REGISTER) */
+
+/* 2 REG_WMAC_FWPKT_CR_8822B */
+#define BIT_FWEN_8822B BIT(7)
+#define BIT_PHYSTS_PKT_CTRL_8822B BIT(6)
+#define BIT_APPHDR_MIDSRCH_FAIL_8822B BIT(4)
+#define BIT_FWPARSING_EN_8822B BIT(3)
+
+#define BIT_SHIFT_APPEND_MHDR_LEN_8822B 0
+#define BIT_MASK_APPEND_MHDR_LEN_8822B 0x7
+#define BIT_APPEND_MHDR_LEN_8822B(x)                                           \
+	(((x) & BIT_MASK_APPEND_MHDR_LEN_8822B)                                \
+	 << BIT_SHIFT_APPEND_MHDR_LEN_8822B)
+#define BIT_GET_APPEND_MHDR_LEN_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_APPEND_MHDR_LEN_8822B) &                            \
+	 BIT_MASK_APPEND_MHDR_LEN_8822B)
+
+/* 2 REG_WMAC_CR_8822B (WMAC CR AND APSD CONTROL REGISTER) */
+#define BIT_IC_MACPHY_M_8822B BIT(0)
+
+/* 2 REG_TCR_8822B (TRANSMISSION CONFIGURATION REGISTER) */
+#define BIT_WMAC_EN_RTS_ADDR_8822B BIT(31)
+#define BIT_WMAC_DISABLE_CCK_8822B BIT(30)
+#define BIT_WMAC_RAW_LEN_8822B BIT(29)
+#define BIT_WMAC_NOTX_IN_RXNDP_8822B BIT(28)
+#define BIT_WMAC_EN_EOF_8822B BIT(27)
+#define BIT_WMAC_BF_SEL_8822B BIT(26)
+#define BIT_WMAC_ANTMODE_SEL_8822B BIT(25)
+#define BIT_WMAC_TCRPWRMGT_HWCTL_8822B BIT(24)
+#define BIT_WMAC_SMOOTH_VAL_8822B BIT(23)
+#define BIT_FETCH_MPDU_AFTER_WSEC_RDY_8822B BIT(20)
+#define BIT_WMAC_TCR_EN_20MST_8822B BIT(19)
+#define BIT_WMAC_DIS_SIGTA_8822B BIT(18)
+#define BIT_WMAC_DIS_A2B0_8822B BIT(17)
+#define BIT_WMAC_MSK_SIGBCRC_8822B BIT(16)
+#define BIT_WMAC_TCR_ERRSTEN_3_8822B BIT(15)
+#define BIT_WMAC_TCR_ERRSTEN_2_8822B BIT(14)
+#define BIT_WMAC_TCR_ERRSTEN_1_8822B BIT(13)
+#define BIT_WMAC_TCR_ERRSTEN_0_8822B BIT(12)
+#define BIT_WMAC_TCR_TXSK_PERPKT_8822B BIT(11)
+#define BIT_ICV_8822B BIT(10)
+#define BIT_CFEND_FORMAT_8822B BIT(9)
+#define BIT_CRC_8822B BIT(8)
+#define BIT_PWRBIT_OW_EN_8822B BIT(7)
+#define BIT_PWR_ST_8822B BIT(6)
+#define BIT_WMAC_TCR_UPD_TIMIE_8822B BIT(5)
+#define BIT_WMAC_TCR_UPD_HGQMD_8822B BIT(4)
+#define BIT_VHTSIGA1_TXPS_8822B BIT(3)
+#define BIT_PAD_SEL_8822B BIT(2)
+#define BIT_DIS_GCLK_8822B BIT(1)
+
+/* 2 REG_RCR_8822B (RECEIVE CONFIGURATION REGISTER) */
+#define BIT_APP_FCS_8822B BIT(31)
+#define BIT_APP_MIC_8822B BIT(30)
+#define BIT_APP_ICV_8822B BIT(29)
+#define BIT_APP_PHYSTS_8822B BIT(28)
+#define BIT_APP_BASSN_8822B BIT(27)
+#define BIT_VHT_DACK_8822B BIT(26)
+#define BIT_TCPOFLD_EN_8822B BIT(25)
+#define BIT_ENMBID_8822B BIT(24)
+#define BIT_LSIGEN_8822B BIT(23)
+#define BIT_MFBEN_8822B BIT(22)
+#define BIT_DISCHKPPDLLEN_8822B BIT(21)
+#define BIT_PKTCTL_DLEN_8822B BIT(20)
+#define BIT_TIM_PARSER_EN_8822B BIT(18)
+#define BIT_BC_MD_EN_8822B BIT(17)
+#define BIT_UC_MD_EN_8822B BIT(16)
+#define BIT_RXSK_PERPKT_8822B BIT(15)
+#define BIT_HTC_LOC_CTRL_8822B BIT(14)
+#define BIT_RPFM_CAM_ENABLE_8822B BIT(12)
+#define BIT_TA_BCN_8822B BIT(11)
+#define BIT_DISDECMYPKT_8822B BIT(10)
+#define BIT_AICV_8822B BIT(9)
+#define BIT_ACRC32_8822B BIT(8)
+#define BIT_CBSSID_BCN_8822B BIT(7)
+#define BIT_CBSSID_DATA_8822B BIT(6)
+#define BIT_APWRMGT_8822B BIT(5)
+#define BIT_ADD3_8822B BIT(4)
+#define BIT_AB_8822B BIT(3)
+#define BIT_AM_8822B BIT(2)
+#define BIT_APM_8822B BIT(1)
+#define BIT_AAP_8822B BIT(0)
+
+/* 2 REG_RX_DRVINFO_SZ_8822B (RX DRIVER INFO SIZE REGISTER) */
+#define BIT_PHYSTS_PER_PKT_MODE_8822B BIT(7)
+
+#define BIT_SHIFT_DRVINFO_SZ_V1_8822B 0
+#define BIT_MASK_DRVINFO_SZ_V1_8822B 0xf
+#define BIT_DRVINFO_SZ_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_DRVINFO_SZ_V1_8822B) << BIT_SHIFT_DRVINFO_SZ_V1_8822B)
+#define BIT_GET_DRVINFO_SZ_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_DRVINFO_SZ_V1_8822B) & BIT_MASK_DRVINFO_SZ_V1_8822B)
+
+/* 2 REG_RX_DLK_TIME_8822B (RX DEADLOCK TIME REGISTER) */
+
+#define BIT_SHIFT_RX_DLK_TIME_8822B 0
+#define BIT_MASK_RX_DLK_TIME_8822B 0xff
+#define BIT_RX_DLK_TIME_8822B(x)                                               \
+	(((x) & BIT_MASK_RX_DLK_TIME_8822B) << BIT_SHIFT_RX_DLK_TIME_8822B)
+#define BIT_GET_RX_DLK_TIME_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_RX_DLK_TIME_8822B) & BIT_MASK_RX_DLK_TIME_8822B)
+
+/* 2 REG_RX_PKT_LIMIT_8822B (RX PACKET LENGTH LIMIT REGISTER) */
+
+#define BIT_SHIFT_RXPKTLMT_8822B 0
+#define BIT_MASK_RXPKTLMT_8822B 0x3f
+#define BIT_RXPKTLMT_8822B(x)                                                  \
+	(((x) & BIT_MASK_RXPKTLMT_8822B) << BIT_SHIFT_RXPKTLMT_8822B)
+#define BIT_GET_RXPKTLMT_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_RXPKTLMT_8822B) & BIT_MASK_RXPKTLMT_8822B)
+
+/* 2 REG_MACID_8822B (MAC ID REGISTER) */
+
+#define BIT_SHIFT_MACID_8822B 0
+#define BIT_MASK_MACID_8822B 0xffffffffffffL
+#define BIT_MACID_8822B(x)                                                     \
+	(((x) & BIT_MASK_MACID_8822B) << BIT_SHIFT_MACID_8822B)
+#define BIT_GET_MACID_8822B(x)                                                 \
+	(((x) >> BIT_SHIFT_MACID_8822B) & BIT_MASK_MACID_8822B)
+
+/* 2 REG_BSSID_8822B (BSSID REGISTER) */
+
+#define BIT_SHIFT_BSSID_8822B 0
+#define BIT_MASK_BSSID_8822B 0xffffffffffffL
+#define BIT_BSSID_8822B(x)                                                     \
+	(((x) & BIT_MASK_BSSID_8822B) << BIT_SHIFT_BSSID_8822B)
+#define BIT_GET_BSSID_8822B(x)                                                 \
+	(((x) >> BIT_SHIFT_BSSID_8822B) & BIT_MASK_BSSID_8822B)
+
+/* 2 REG_MAR_8822B (MULTICAST ADDRESS REGISTER) */
+
+#define BIT_SHIFT_MAR_8822B 0
+#define BIT_MASK_MAR_8822B 0xffffffffffffffffL
+#define BIT_MAR_8822B(x) (((x) & BIT_MASK_MAR_8822B) << BIT_SHIFT_MAR_8822B)
+#define BIT_GET_MAR_8822B(x) (((x) >> BIT_SHIFT_MAR_8822B) & BIT_MASK_MAR_8822B)
+
+/* 2 REG_MBIDCAMCFG_1_8822B (MBSSID CAM CONFIGURATION REGISTER) */
+
+#define BIT_SHIFT_MBIDCAM_RWDATA_L_8822B 0
+#define BIT_MASK_MBIDCAM_RWDATA_L_8822B 0xffffffffL
+#define BIT_MBIDCAM_RWDATA_L_8822B(x)                                          \
+	(((x) & BIT_MASK_MBIDCAM_RWDATA_L_8822B)                               \
+	 << BIT_SHIFT_MBIDCAM_RWDATA_L_8822B)
+#define BIT_GET_MBIDCAM_RWDATA_L_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_MBIDCAM_RWDATA_L_8822B) &                           \
+	 BIT_MASK_MBIDCAM_RWDATA_L_8822B)
+
+/* 2 REG_MBIDCAMCFG_2_8822B (MBSSID CAM CONFIGURATION REGISTER) */
+#define BIT_MBIDCAM_POLL_8822B BIT(31)
+#define BIT_MBIDCAM_WT_EN_8822B BIT(30)
+
+#define BIT_SHIFT_MBIDCAM_ADDR_8822B 24
+#define BIT_MASK_MBIDCAM_ADDR_8822B 0x1f
+#define BIT_MBIDCAM_ADDR_8822B(x)                                              \
+	(((x) & BIT_MASK_MBIDCAM_ADDR_8822B) << BIT_SHIFT_MBIDCAM_ADDR_8822B)
+#define BIT_GET_MBIDCAM_ADDR_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_MBIDCAM_ADDR_8822B) & BIT_MASK_MBIDCAM_ADDR_8822B)
+
+#define BIT_MBIDCAM_VALID_8822B BIT(23)
+#define BIT_LSIC_TXOP_EN_8822B BIT(17)
+#define BIT_CTS_EN_8822B BIT(16)
+
+#define BIT_SHIFT_MBIDCAM_RWDATA_H_8822B 0
+#define BIT_MASK_MBIDCAM_RWDATA_H_8822B 0xffff
+#define BIT_MBIDCAM_RWDATA_H_8822B(x)                                          \
+	(((x) & BIT_MASK_MBIDCAM_RWDATA_H_8822B)                               \
+	 << BIT_SHIFT_MBIDCAM_RWDATA_H_8822B)
+#define BIT_GET_MBIDCAM_RWDATA_H_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_MBIDCAM_RWDATA_H_8822B) &                           \
+	 BIT_MASK_MBIDCAM_RWDATA_H_8822B)
+
+/* 2 REG_ZLD_NUM_8822B */
+
+#define BIT_SHIFT_ZLD_NUM_8822B 0
+#define BIT_MASK_ZLD_NUM_8822B 0xff
+#define BIT_ZLD_NUM_8822B(x)                                                   \
+	(((x) & BIT_MASK_ZLD_NUM_8822B) << BIT_SHIFT_ZLD_NUM_8822B)
+#define BIT_GET_ZLD_NUM_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_ZLD_NUM_8822B) & BIT_MASK_ZLD_NUM_8822B)
+
+/* 2 REG_UDF_THSD_8822B */
+
+#define BIT_SHIFT_UDF_THSD_8822B 0
+#define BIT_MASK_UDF_THSD_8822B 0xff
+#define BIT_UDF_THSD_8822B(x)                                                  \
+	(((x) & BIT_MASK_UDF_THSD_8822B) << BIT_SHIFT_UDF_THSD_8822B)
+#define BIT_GET_UDF_THSD_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_UDF_THSD_8822B) & BIT_MASK_UDF_THSD_8822B)
+
+/* 2 REG_WMAC_TCR_TSFT_OFS_8822B */
+
+#define BIT_SHIFT_WMAC_TCR_TSFT_OFS_8822B 0
+#define BIT_MASK_WMAC_TCR_TSFT_OFS_8822B 0xffff
+#define BIT_WMAC_TCR_TSFT_OFS_8822B(x)                                         \
+	(((x) & BIT_MASK_WMAC_TCR_TSFT_OFS_8822B)                              \
+	 << BIT_SHIFT_WMAC_TCR_TSFT_OFS_8822B)
+#define BIT_GET_WMAC_TCR_TSFT_OFS_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_WMAC_TCR_TSFT_OFS_8822B) &                          \
+	 BIT_MASK_WMAC_TCR_TSFT_OFS_8822B)
+
+/* 2 REG_MCU_TEST_2_V1_8822B */
+
+#define BIT_SHIFT_MCU_RSVD_2_V1_8822B 0
+#define BIT_MASK_MCU_RSVD_2_V1_8822B 0xffff
+#define BIT_MCU_RSVD_2_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_MCU_RSVD_2_V1_8822B) << BIT_SHIFT_MCU_RSVD_2_V1_8822B)
+#define BIT_GET_MCU_RSVD_2_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_MCU_RSVD_2_V1_8822B) & BIT_MASK_MCU_RSVD_2_V1_8822B)
+
+/* 2 REG_WMAC_TXTIMEOUT_8822B */
+
+#define BIT_SHIFT_WMAC_TXTIMEOUT_8822B 0
+#define BIT_MASK_WMAC_TXTIMEOUT_8822B 0xff
+#define BIT_WMAC_TXTIMEOUT_8822B(x)                                            \
+	(((x) & BIT_MASK_WMAC_TXTIMEOUT_8822B)                                 \
+	 << BIT_SHIFT_WMAC_TXTIMEOUT_8822B)
+#define BIT_GET_WMAC_TXTIMEOUT_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_WMAC_TXTIMEOUT_8822B) &                             \
+	 BIT_MASK_WMAC_TXTIMEOUT_8822B)
+
+/* 2 REG_STMP_THSD_8822B */
+
+#define BIT_SHIFT_STMP_THSD_8822B 0
+#define BIT_MASK_STMP_THSD_8822B 0xff
+#define BIT_STMP_THSD_8822B(x)                                                 \
+	(((x) & BIT_MASK_STMP_THSD_8822B) << BIT_SHIFT_STMP_THSD_8822B)
+#define BIT_GET_STMP_THSD_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_STMP_THSD_8822B) & BIT_MASK_STMP_THSD_8822B)
+
+/* 2 REG_MAC_SPEC_SIFS_8822B (SPECIFICATION SIFS REGISTER) */
+
+#define BIT_SHIFT_SPEC_SIFS_OFDM_8822B 8
+#define BIT_MASK_SPEC_SIFS_OFDM_8822B 0xff
+#define BIT_SPEC_SIFS_OFDM_8822B(x)                                            \
+	(((x) & BIT_MASK_SPEC_SIFS_OFDM_8822B)                                 \
+	 << BIT_SHIFT_SPEC_SIFS_OFDM_8822B)
+#define BIT_GET_SPEC_SIFS_OFDM_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_SPEC_SIFS_OFDM_8822B) &                             \
+	 BIT_MASK_SPEC_SIFS_OFDM_8822B)
+
+#define BIT_SHIFT_SPEC_SIFS_CCK_8822B 0
+#define BIT_MASK_SPEC_SIFS_CCK_8822B 0xff
+#define BIT_SPEC_SIFS_CCK_8822B(x)                                             \
+	(((x) & BIT_MASK_SPEC_SIFS_CCK_8822B) << BIT_SHIFT_SPEC_SIFS_CCK_8822B)
+#define BIT_GET_SPEC_SIFS_CCK_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_SPEC_SIFS_CCK_8822B) & BIT_MASK_SPEC_SIFS_CCK_8822B)
+
+/* 2 REG_USTIME_EDCA_8822B (US TIME TUNING FOR EDCA REGISTER) */
+
+#define BIT_SHIFT_USTIME_EDCA_V1_8822B 0
+#define BIT_MASK_USTIME_EDCA_V1_8822B 0x1ff
+#define BIT_USTIME_EDCA_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_USTIME_EDCA_V1_8822B)                                 \
+	 << BIT_SHIFT_USTIME_EDCA_V1_8822B)
+#define BIT_GET_USTIME_EDCA_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_USTIME_EDCA_V1_8822B) &                             \
+	 BIT_MASK_USTIME_EDCA_V1_8822B)
+
+/* 2 REG_RESP_SIFS_OFDM_8822B (RESPONSE SIFS FOR OFDM REGISTER) */
+
+#define BIT_SHIFT_SIFS_R2T_OFDM_8822B 8
+#define BIT_MASK_SIFS_R2T_OFDM_8822B 0xff
+#define BIT_SIFS_R2T_OFDM_8822B(x)                                             \
+	(((x) & BIT_MASK_SIFS_R2T_OFDM_8822B) << BIT_SHIFT_SIFS_R2T_OFDM_8822B)
+#define BIT_GET_SIFS_R2T_OFDM_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_SIFS_R2T_OFDM_8822B) & BIT_MASK_SIFS_R2T_OFDM_8822B)
+
+#define BIT_SHIFT_SIFS_T2T_OFDM_8822B 0
+#define BIT_MASK_SIFS_T2T_OFDM_8822B 0xff
+#define BIT_SIFS_T2T_OFDM_8822B(x)                                             \
+	(((x) & BIT_MASK_SIFS_T2T_OFDM_8822B) << BIT_SHIFT_SIFS_T2T_OFDM_8822B)
+#define BIT_GET_SIFS_T2T_OFDM_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_SIFS_T2T_OFDM_8822B) & BIT_MASK_SIFS_T2T_OFDM_8822B)
+
+/* 2 REG_RESP_SIFS_CCK_8822B (RESPONSE SIFS FOR CCK REGISTER) */
+
+#define BIT_SHIFT_SIFS_R2T_CCK_8822B 8
+#define BIT_MASK_SIFS_R2T_CCK_8822B 0xff
+#define BIT_SIFS_R2T_CCK_8822B(x)                                              \
+	(((x) & BIT_MASK_SIFS_R2T_CCK_8822B) << BIT_SHIFT_SIFS_R2T_CCK_8822B)
+#define BIT_GET_SIFS_R2T_CCK_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_SIFS_R2T_CCK_8822B) & BIT_MASK_SIFS_R2T_CCK_8822B)
+
+#define BIT_SHIFT_SIFS_T2T_CCK_8822B 0
+#define BIT_MASK_SIFS_T2T_CCK_8822B 0xff
+#define BIT_SIFS_T2T_CCK_8822B(x)                                              \
+	(((x) & BIT_MASK_SIFS_T2T_CCK_8822B) << BIT_SHIFT_SIFS_T2T_CCK_8822B)
+#define BIT_GET_SIFS_T2T_CCK_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_SIFS_T2T_CCK_8822B) & BIT_MASK_SIFS_T2T_CCK_8822B)
+
+/* 2 REG_EIFS_8822B (EIFS REGISTER) */
+
+#define BIT_SHIFT_EIFS_8822B 0
+#define BIT_MASK_EIFS_8822B 0xffff
+#define BIT_EIFS_8822B(x) (((x) & BIT_MASK_EIFS_8822B) << BIT_SHIFT_EIFS_8822B)
+#define BIT_GET_EIFS_8822B(x)                                                  \
+	(((x) >> BIT_SHIFT_EIFS_8822B) & BIT_MASK_EIFS_8822B)
+
+/* 2 REG_CTS2TO_8822B (CTS2 TIMEOUT REGISTER) */
+
+#define BIT_SHIFT_CTS2TO_8822B 0
+#define BIT_MASK_CTS2TO_8822B 0xff
+#define BIT_CTS2TO_8822B(x)                                                    \
+	(((x) & BIT_MASK_CTS2TO_8822B) << BIT_SHIFT_CTS2TO_8822B)
+#define BIT_GET_CTS2TO_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_CTS2TO_8822B) & BIT_MASK_CTS2TO_8822B)
+
+/* 2 REG_ACKTO_8822B (ACK TIMEOUT REGISTER) */
+
+#define BIT_SHIFT_ACKTO_8822B 0
+#define BIT_MASK_ACKTO_8822B 0xff
+#define BIT_ACKTO_8822B(x)                                                     \
+	(((x) & BIT_MASK_ACKTO_8822B) << BIT_SHIFT_ACKTO_8822B)
+#define BIT_GET_ACKTO_8822B(x)                                                 \
+	(((x) >> BIT_SHIFT_ACKTO_8822B) & BIT_MASK_ACKTO_8822B)
+
+/* 2 REG_NAV_CTRL_8822B (NAV CONTROL REGISTER) */
+
+#define BIT_SHIFT_NAV_UPPER_8822B 16
+#define BIT_MASK_NAV_UPPER_8822B 0xff
+#define BIT_NAV_UPPER_8822B(x)                                                 \
+	(((x) & BIT_MASK_NAV_UPPER_8822B) << BIT_SHIFT_NAV_UPPER_8822B)
+#define BIT_GET_NAV_UPPER_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_NAV_UPPER_8822B) & BIT_MASK_NAV_UPPER_8822B)
+
+#define BIT_SHIFT_RXMYRTS_NAV_8822B 8
+#define BIT_MASK_RXMYRTS_NAV_8822B 0xf
+#define BIT_RXMYRTS_NAV_8822B(x)                                               \
+	(((x) & BIT_MASK_RXMYRTS_NAV_8822B) << BIT_SHIFT_RXMYRTS_NAV_8822B)
+#define BIT_GET_RXMYRTS_NAV_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_RXMYRTS_NAV_8822B) & BIT_MASK_RXMYRTS_NAV_8822B)
+
+#define BIT_SHIFT_RTSRST_8822B 0
+#define BIT_MASK_RTSRST_8822B 0xff
+#define BIT_RTSRST_8822B(x)                                                    \
+	(((x) & BIT_MASK_RTSRST_8822B) << BIT_SHIFT_RTSRST_8822B)
+#define BIT_GET_RTSRST_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_RTSRST_8822B) & BIT_MASK_RTSRST_8822B)
+
+/* 2 REG_BACAMCMD_8822B (BLOCK ACK CAM COMMAND REGISTER) */
+#define BIT_BACAM_POLL_8822B BIT(31)
+#define BIT_BACAM_RST_8822B BIT(17)
+#define BIT_BACAM_RW_8822B BIT(16)
+
+#define BIT_SHIFT_TXSBM_8822B 14
+#define BIT_MASK_TXSBM_8822B 0x3
+#define BIT_TXSBM_8822B(x)                                                     \
+	(((x) & BIT_MASK_TXSBM_8822B) << BIT_SHIFT_TXSBM_8822B)
+#define BIT_GET_TXSBM_8822B(x)                                                 \
+	(((x) >> BIT_SHIFT_TXSBM_8822B) & BIT_MASK_TXSBM_8822B)
+
+#define BIT_SHIFT_BACAM_ADDR_8822B 0
+#define BIT_MASK_BACAM_ADDR_8822B 0x3f
+#define BIT_BACAM_ADDR_8822B(x)                                                \
+	(((x) & BIT_MASK_BACAM_ADDR_8822B) << BIT_SHIFT_BACAM_ADDR_8822B)
+#define BIT_GET_BACAM_ADDR_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_BACAM_ADDR_8822B) & BIT_MASK_BACAM_ADDR_8822B)
+
+/* 2 REG_BACAMCONTENT_8822B (BLOCK ACK CAM CONTENT REGISTER) */
+
+#define BIT_SHIFT_BA_CONTENT_H_8822B (32 & CPU_OPT_WIDTH)
+#define BIT_MASK_BA_CONTENT_H_8822B 0xffffffffL
+#define BIT_BA_CONTENT_H_8822B(x)                                              \
+	(((x) & BIT_MASK_BA_CONTENT_H_8822B) << BIT_SHIFT_BA_CONTENT_H_8822B)
+#define BIT_GET_BA_CONTENT_H_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_BA_CONTENT_H_8822B) & BIT_MASK_BA_CONTENT_H_8822B)
+
+#define BIT_SHIFT_BA_CONTENT_L_8822B 0
+#define BIT_MASK_BA_CONTENT_L_8822B 0xffffffffL
+#define BIT_BA_CONTENT_L_8822B(x)                                              \
+	(((x) & BIT_MASK_BA_CONTENT_L_8822B) << BIT_SHIFT_BA_CONTENT_L_8822B)
+#define BIT_GET_BA_CONTENT_L_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_BA_CONTENT_L_8822B) & BIT_MASK_BA_CONTENT_L_8822B)
+
+/* 2 REG_WMAC_BITMAP_CTL_8822B */
+#define BIT_BITMAP_VO_8822B BIT(7)
+#define BIT_BITMAP_VI_8822B BIT(6)
+#define BIT_BITMAP_BE_8822B BIT(5)
+#define BIT_BITMAP_BK_8822B BIT(4)
+
+#define BIT_SHIFT_BITMAP_CONDITION_8822B 2
+#define BIT_MASK_BITMAP_CONDITION_8822B 0x3
+#define BIT_BITMAP_CONDITION_8822B(x)                                          \
+	(((x) & BIT_MASK_BITMAP_CONDITION_8822B)                               \
+	 << BIT_SHIFT_BITMAP_CONDITION_8822B)
+#define BIT_GET_BITMAP_CONDITION_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_BITMAP_CONDITION_8822B) &                           \
+	 BIT_MASK_BITMAP_CONDITION_8822B)
+
+#define BIT_BITMAP_SSNBK_COUNTER_CLR_8822B BIT(1)
+#define BIT_BITMAP_FORCE_8822B BIT(0)
+
+/* 2 REG_TX_RX_8822B STATUS */
+
+#define BIT_SHIFT_RXPKT_TYPE_8822B 2
+#define BIT_MASK_RXPKT_TYPE_8822B 0x3f
+#define BIT_RXPKT_TYPE_8822B(x)                                                \
+	(((x) & BIT_MASK_RXPKT_TYPE_8822B) << BIT_SHIFT_RXPKT_TYPE_8822B)
+#define BIT_GET_RXPKT_TYPE_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_RXPKT_TYPE_8822B) & BIT_MASK_RXPKT_TYPE_8822B)
+
+#define BIT_TXACT_IND_8822B BIT(1)
+#define BIT_RXACT_IND_8822B BIT(0)
+
+/* 2 REG_WMAC_BACAM_RPMEN_8822B */
+
+#define BIT_SHIFT_BITMAP_SSNBK_COUNTER_8822B 2
+#define BIT_MASK_BITMAP_SSNBK_COUNTER_8822B 0x3f
+#define BIT_BITMAP_SSNBK_COUNTER_8822B(x)                                      \
+	(((x) & BIT_MASK_BITMAP_SSNBK_COUNTER_8822B)                           \
+	 << BIT_SHIFT_BITMAP_SSNBK_COUNTER_8822B)
+#define BIT_GET_BITMAP_SSNBK_COUNTER_8822B(x)                                  \
+	(((x) >> BIT_SHIFT_BITMAP_SSNBK_COUNTER_8822B) &                       \
+	 BIT_MASK_BITMAP_SSNBK_COUNTER_8822B)
+
+#define BIT_BITMAP_EN_8822B BIT(1)
+#define BIT_WMAC_BACAM_RPMEN_8822B BIT(0)
+
+/* 2 REG_LBDLY_8822B (LOOPBACK DELAY REGISTER) */
+
+#define BIT_SHIFT_LBDLY_8822B 0
+#define BIT_MASK_LBDLY_8822B 0x1f
+#define BIT_LBDLY_8822B(x)                                                     \
+	(((x) & BIT_MASK_LBDLY_8822B) << BIT_SHIFT_LBDLY_8822B)
+#define BIT_GET_LBDLY_8822B(x)                                                 \
+	(((x) >> BIT_SHIFT_LBDLY_8822B) & BIT_MASK_LBDLY_8822B)
+
+/* 2 REG_RXERR_RPT_8822B (RX ERROR REPORT REGISTER) */
+
+#define BIT_SHIFT_RXERR_RPT_SEL_V1_3_0_8822B 28
+#define BIT_MASK_RXERR_RPT_SEL_V1_3_0_8822B 0xf
+#define BIT_RXERR_RPT_SEL_V1_3_0_8822B(x)                                      \
+	(((x) & BIT_MASK_RXERR_RPT_SEL_V1_3_0_8822B)                           \
+	 << BIT_SHIFT_RXERR_RPT_SEL_V1_3_0_8822B)
+#define BIT_GET_RXERR_RPT_SEL_V1_3_0_8822B(x)                                  \
+	(((x) >> BIT_SHIFT_RXERR_RPT_SEL_V1_3_0_8822B) &                       \
+	 BIT_MASK_RXERR_RPT_SEL_V1_3_0_8822B)
+
+#define BIT_RXERR_RPT_RST_8822B BIT(27)
+#define BIT_RXERR_RPT_SEL_V1_4_8822B BIT(26)
+#define BIT_W1S_8822B BIT(23)
+#define BIT_UD_SELECT_BSSID_8822B BIT(22)
+
+#define BIT_SHIFT_UD_SUB_TYPE_8822B 18
+#define BIT_MASK_UD_SUB_TYPE_8822B 0xf
+#define BIT_UD_SUB_TYPE_8822B(x)                                               \
+	(((x) & BIT_MASK_UD_SUB_TYPE_8822B) << BIT_SHIFT_UD_SUB_TYPE_8822B)
+#define BIT_GET_UD_SUB_TYPE_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_UD_SUB_TYPE_8822B) & BIT_MASK_UD_SUB_TYPE_8822B)
+
+#define BIT_SHIFT_UD_TYPE_8822B 16
+#define BIT_MASK_UD_TYPE_8822B 0x3
+#define BIT_UD_TYPE_8822B(x)                                                   \
+	(((x) & BIT_MASK_UD_TYPE_8822B) << BIT_SHIFT_UD_TYPE_8822B)
+#define BIT_GET_UD_TYPE_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_UD_TYPE_8822B) & BIT_MASK_UD_TYPE_8822B)
+
+#define BIT_SHIFT_RPT_COUNTER_8822B 0
+#define BIT_MASK_RPT_COUNTER_8822B 0xffff
+#define BIT_RPT_COUNTER_8822B(x)                                               \
+	(((x) & BIT_MASK_RPT_COUNTER_8822B) << BIT_SHIFT_RPT_COUNTER_8822B)
+#define BIT_GET_RPT_COUNTER_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_RPT_COUNTER_8822B) & BIT_MASK_RPT_COUNTER_8822B)
+
+/* 2 REG_WMAC_TRXPTCL_CTL_8822B (WMAC TX/RX PROTOCOL CONTROL REGISTER) */
+
+#define BIT_SHIFT_ACKBA_TYPSEL_8822B (60 & CPU_OPT_WIDTH)
+#define BIT_MASK_ACKBA_TYPSEL_8822B 0xf
+#define BIT_ACKBA_TYPSEL_8822B(x)                                              \
+	(((x) & BIT_MASK_ACKBA_TYPSEL_8822B) << BIT_SHIFT_ACKBA_TYPSEL_8822B)
+#define BIT_GET_ACKBA_TYPSEL_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_ACKBA_TYPSEL_8822B) & BIT_MASK_ACKBA_TYPSEL_8822B)
+
+#define BIT_SHIFT_ACKBA_ACKPCHK_8822B (56 & CPU_OPT_WIDTH)
+#define BIT_MASK_ACKBA_ACKPCHK_8822B 0xf
+#define BIT_ACKBA_ACKPCHK_8822B(x)                                             \
+	(((x) & BIT_MASK_ACKBA_ACKPCHK_8822B) << BIT_SHIFT_ACKBA_ACKPCHK_8822B)
+#define BIT_GET_ACKBA_ACKPCHK_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_ACKBA_ACKPCHK_8822B) & BIT_MASK_ACKBA_ACKPCHK_8822B)
+
+#define BIT_SHIFT_ACKBAR_TYPESEL_8822B (48 & CPU_OPT_WIDTH)
+#define BIT_MASK_ACKBAR_TYPESEL_8822B 0xff
+#define BIT_ACKBAR_TYPESEL_8822B(x)                                            \
+	(((x) & BIT_MASK_ACKBAR_TYPESEL_8822B)                                 \
+	 << BIT_SHIFT_ACKBAR_TYPESEL_8822B)
+#define BIT_GET_ACKBAR_TYPESEL_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_ACKBAR_TYPESEL_8822B) &                             \
+	 BIT_MASK_ACKBAR_TYPESEL_8822B)
+
+#define BIT_SHIFT_ACKBAR_ACKPCHK_8822B (44 & CPU_OPT_WIDTH)
+#define BIT_MASK_ACKBAR_ACKPCHK_8822B 0xf
+#define BIT_ACKBAR_ACKPCHK_8822B(x)                                            \
+	(((x) & BIT_MASK_ACKBAR_ACKPCHK_8822B)                                 \
+	 << BIT_SHIFT_ACKBAR_ACKPCHK_8822B)
+#define BIT_GET_ACKBAR_ACKPCHK_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_ACKBAR_ACKPCHK_8822B) &                             \
+	 BIT_MASK_ACKBAR_ACKPCHK_8822B)
+
+#define BIT_RXBA_IGNOREA2_8822B BIT(42)
+#define BIT_EN_SAVE_ALL_TXOPADDR_8822B BIT(41)
+#define BIT_EN_TXCTS_TO_TXOPOWNER_INRXNAV_8822B BIT(40)
+#define BIT_DIS_TXBA_AMPDUFCSERR_8822B BIT(39)
+#define BIT_DIS_TXBA_RXBARINFULL_8822B BIT(38)
+#define BIT_DIS_TXCFE_INFULL_8822B BIT(37)
+#define BIT_DIS_TXCTS_INFULL_8822B BIT(36)
+#define BIT_EN_TXACKBA_IN_TX_RDG_8822B BIT(35)
+#define BIT_EN_TXACKBA_IN_TXOP_8822B BIT(34)
+#define BIT_EN_TXCTS_IN_RXNAV_8822B BIT(33)
+#define BIT_EN_TXCTS_INTXOP_8822B BIT(32)
+#define BIT_BLK_EDCA_BBSLP_8822B BIT(31)
+#define BIT_BLK_EDCA_BBSBY_8822B BIT(30)
+#define BIT_ACKTO_BLOCK_SCH_EN_8822B BIT(27)
+#define BIT_EIFS_BLOCK_SCH_EN_8822B BIT(26)
+#define BIT_PLCPCHK_RST_EIFS_8822B BIT(25)
+#define BIT_CCA_RST_EIFS_8822B BIT(24)
+#define BIT_DIS_UPD_MYRXPKTNAV_8822B BIT(23)
+#define BIT_EARLY_TXBA_8822B BIT(22)
+
+#define BIT_SHIFT_RESP_CHNBUSY_8822B 20
+#define BIT_MASK_RESP_CHNBUSY_8822B 0x3
+#define BIT_RESP_CHNBUSY_8822B(x)                                              \
+	(((x) & BIT_MASK_RESP_CHNBUSY_8822B) << BIT_SHIFT_RESP_CHNBUSY_8822B)
+#define BIT_GET_RESP_CHNBUSY_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_RESP_CHNBUSY_8822B) & BIT_MASK_RESP_CHNBUSY_8822B)
+
+#define BIT_RESP_DCTS_EN_8822B BIT(19)
+#define BIT_RESP_DCFE_EN_8822B BIT(18)
+#define BIT_RESP_SPLCPEN_8822B BIT(17)
+#define BIT_RESP_SGIEN_8822B BIT(16)
+#define BIT_RESP_LDPC_EN_8822B BIT(15)
+#define BIT_DIS_RESP_ACKINCCA_8822B BIT(14)
+#define BIT_DIS_RESP_CTSINCCA_8822B BIT(13)
+
+#define BIT_SHIFT_R_WMAC_SECOND_CCA_TIMER_8822B 10
+#define BIT_MASK_R_WMAC_SECOND_CCA_TIMER_8822B 0x7
+#define BIT_R_WMAC_SECOND_CCA_TIMER_8822B(x)                                   \
+	(((x) & BIT_MASK_R_WMAC_SECOND_CCA_TIMER_8822B)                        \
+	 << BIT_SHIFT_R_WMAC_SECOND_CCA_TIMER_8822B)
+#define BIT_GET_R_WMAC_SECOND_CCA_TIMER_8822B(x)                               \
+	(((x) >> BIT_SHIFT_R_WMAC_SECOND_CCA_TIMER_8822B) &                    \
+	 BIT_MASK_R_WMAC_SECOND_CCA_TIMER_8822B)
+
+#define BIT_SHIFT_RFMOD_8822B 7
+#define BIT_MASK_RFMOD_8822B 0x3
+#define BIT_RFMOD_8822B(x)                                                     \
+	(((x) & BIT_MASK_RFMOD_8822B) << BIT_SHIFT_RFMOD_8822B)
+#define BIT_GET_RFMOD_8822B(x)                                                 \
+	(((x) >> BIT_SHIFT_RFMOD_8822B) & BIT_MASK_RFMOD_8822B)
+
+#define BIT_SHIFT_RESP_CTS_DYNBW_SEL_8822B 5
+#define BIT_MASK_RESP_CTS_DYNBW_SEL_8822B 0x3
+#define BIT_RESP_CTS_DYNBW_SEL_8822B(x)                                        \
+	(((x) & BIT_MASK_RESP_CTS_DYNBW_SEL_8822B)                             \
+	 << BIT_SHIFT_RESP_CTS_DYNBW_SEL_8822B)
+#define BIT_GET_RESP_CTS_DYNBW_SEL_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_RESP_CTS_DYNBW_SEL_8822B) &                         \
+	 BIT_MASK_RESP_CTS_DYNBW_SEL_8822B)
+
+#define BIT_DLY_TX_WAIT_RXANTSEL_8822B BIT(4)
+#define BIT_TXRESP_BY_RXANTSEL_8822B BIT(3)
+
+#define BIT_SHIFT_ORIG_DCTS_CHK_8822B 0
+#define BIT_MASK_ORIG_DCTS_CHK_8822B 0x3
+#define BIT_ORIG_DCTS_CHK_8822B(x)                                             \
+	(((x) & BIT_MASK_ORIG_DCTS_CHK_8822B) << BIT_SHIFT_ORIG_DCTS_CHK_8822B)
+#define BIT_GET_ORIG_DCTS_CHK_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_ORIG_DCTS_CHK_8822B) & BIT_MASK_ORIG_DCTS_CHK_8822B)
+
+/* 2 REG_CAMCMD_8822B (CAM COMMAND REGISTER) */
+#define BIT_SECCAM_POLLING_8822B BIT(31)
+#define BIT_SECCAM_CLR_8822B BIT(30)
+#define BIT_MFBCAM_CLR_8822B BIT(29)
+#define BIT_SECCAM_WE_8822B BIT(16)
+
+#define BIT_SHIFT_SECCAM_ADDR_V2_8822B 0
+#define BIT_MASK_SECCAM_ADDR_V2_8822B 0x3ff
+#define BIT_SECCAM_ADDR_V2_8822B(x)                                            \
+	(((x) & BIT_MASK_SECCAM_ADDR_V2_8822B)                                 \
+	 << BIT_SHIFT_SECCAM_ADDR_V2_8822B)
+#define BIT_GET_SECCAM_ADDR_V2_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_SECCAM_ADDR_V2_8822B) &                             \
+	 BIT_MASK_SECCAM_ADDR_V2_8822B)
+
+/* 2 REG_CAMWRITE_8822B (CAM WRITE REGISTER) */
+
+#define BIT_SHIFT_CAMW_DATA_8822B 0
+#define BIT_MASK_CAMW_DATA_8822B 0xffffffffL
+#define BIT_CAMW_DATA_8822B(x)                                                 \
+	(((x) & BIT_MASK_CAMW_DATA_8822B) << BIT_SHIFT_CAMW_DATA_8822B)
+#define BIT_GET_CAMW_DATA_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_CAMW_DATA_8822B) & BIT_MASK_CAMW_DATA_8822B)
+
+/* 2 REG_CAMREAD_8822B (CAM READ REGISTER) */
+
+#define BIT_SHIFT_CAMR_DATA_8822B 0
+#define BIT_MASK_CAMR_DATA_8822B 0xffffffffL
+#define BIT_CAMR_DATA_8822B(x)                                                 \
+	(((x) & BIT_MASK_CAMR_DATA_8822B) << BIT_SHIFT_CAMR_DATA_8822B)
+#define BIT_GET_CAMR_DATA_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_CAMR_DATA_8822B) & BIT_MASK_CAMR_DATA_8822B)
+
+/* 2 REG_CAMDBG_8822B (CAM DEBUG REGISTER) */
+#define BIT_SECCAM_INFO_8822B BIT(31)
+#define BIT_SEC_KEYFOUND_8822B BIT(15)
+
+#define BIT_SHIFT_CAMDBG_SEC_TYPE_8822B 12
+#define BIT_MASK_CAMDBG_SEC_TYPE_8822B 0x7
+#define BIT_CAMDBG_SEC_TYPE_8822B(x)                                           \
+	(((x) & BIT_MASK_CAMDBG_SEC_TYPE_8822B)                                \
+	 << BIT_SHIFT_CAMDBG_SEC_TYPE_8822B)
+#define BIT_GET_CAMDBG_SEC_TYPE_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_CAMDBG_SEC_TYPE_8822B) &                            \
+	 BIT_MASK_CAMDBG_SEC_TYPE_8822B)
+
+#define BIT_CAMDBG_EXT_SECTYPE_8822B BIT(11)
+
+#define BIT_SHIFT_CAMDBG_MIC_KEY_IDX_8822B 5
+#define BIT_MASK_CAMDBG_MIC_KEY_IDX_8822B 0x1f
+#define BIT_CAMDBG_MIC_KEY_IDX_8822B(x)                                        \
+	(((x) & BIT_MASK_CAMDBG_MIC_KEY_IDX_8822B)                             \
+	 << BIT_SHIFT_CAMDBG_MIC_KEY_IDX_8822B)
+#define BIT_GET_CAMDBG_MIC_KEY_IDX_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_CAMDBG_MIC_KEY_IDX_8822B) &                         \
+	 BIT_MASK_CAMDBG_MIC_KEY_IDX_8822B)
+
+#define BIT_SHIFT_CAMDBG_SEC_KEY_IDX_8822B 0
+#define BIT_MASK_CAMDBG_SEC_KEY_IDX_8822B 0x1f
+#define BIT_CAMDBG_SEC_KEY_IDX_8822B(x)                                        \
+	(((x) & BIT_MASK_CAMDBG_SEC_KEY_IDX_8822B)                             \
+	 << BIT_SHIFT_CAMDBG_SEC_KEY_IDX_8822B)
+#define BIT_GET_CAMDBG_SEC_KEY_IDX_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_CAMDBG_SEC_KEY_IDX_8822B) &                         \
+	 BIT_MASK_CAMDBG_SEC_KEY_IDX_8822B)
+
+/* 2 REG_RXFILTER_ACTION_1_8822B */
+
+#define BIT_SHIFT_RXFILTER_ACTION_1_8822B 0
+#define BIT_MASK_RXFILTER_ACTION_1_8822B 0xff
+#define BIT_RXFILTER_ACTION_1_8822B(x)                                         \
+	(((x) & BIT_MASK_RXFILTER_ACTION_1_8822B)                              \
+	 << BIT_SHIFT_RXFILTER_ACTION_1_8822B)
+#define BIT_GET_RXFILTER_ACTION_1_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_RXFILTER_ACTION_1_8822B) &                          \
+	 BIT_MASK_RXFILTER_ACTION_1_8822B)
+
+/* 2 REG_RXFILTER_CATEGORY_1_8822B */
+
+#define BIT_SHIFT_RXFILTER_CATEGORY_1_8822B 0
+#define BIT_MASK_RXFILTER_CATEGORY_1_8822B 0xff
+#define BIT_RXFILTER_CATEGORY_1_8822B(x)                                       \
+	(((x) & BIT_MASK_RXFILTER_CATEGORY_1_8822B)                            \
+	 << BIT_SHIFT_RXFILTER_CATEGORY_1_8822B)
+#define BIT_GET_RXFILTER_CATEGORY_1_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_RXFILTER_CATEGORY_1_8822B) &                        \
+	 BIT_MASK_RXFILTER_CATEGORY_1_8822B)
+
+/* 2 REG_SECCFG_8822B (SECURITY CONFIGURATION REGISTER) */
+#define BIT_DIS_GCLK_WAPI_8822B BIT(15)
+#define BIT_DIS_GCLK_AES_8822B BIT(14)
+#define BIT_DIS_GCLK_TKIP_8822B BIT(13)
+#define BIT_AES_SEL_QC_1_8822B BIT(12)
+#define BIT_AES_SEL_QC_0_8822B BIT(11)
+#define BIT_CHK_BMC_8822B BIT(9)
+#define BIT_CHK_KEYID_8822B BIT(8)
+#define BIT_RXBCUSEDK_8822B BIT(7)
+#define BIT_TXBCUSEDK_8822B BIT(6)
+#define BIT_NOSKMC_8822B BIT(5)
+#define BIT_SKBYA2_8822B BIT(4)
+#define BIT_RXDEC_8822B BIT(3)
+#define BIT_TXENC_8822B BIT(2)
+#define BIT_RXUHUSEDK_8822B BIT(1)
+#define BIT_TXUHUSEDK_8822B BIT(0)
+
+/* 2 REG_RXFILTER_ACTION_3_8822B */
+
+#define BIT_SHIFT_RXFILTER_ACTION_3_8822B 0
+#define BIT_MASK_RXFILTER_ACTION_3_8822B 0xff
+#define BIT_RXFILTER_ACTION_3_8822B(x)                                         \
+	(((x) & BIT_MASK_RXFILTER_ACTION_3_8822B)                              \
+	 << BIT_SHIFT_RXFILTER_ACTION_3_8822B)
+#define BIT_GET_RXFILTER_ACTION_3_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_RXFILTER_ACTION_3_8822B) &                          \
+	 BIT_MASK_RXFILTER_ACTION_3_8822B)
+
+/* 2 REG_RXFILTER_CATEGORY_3_8822B */
+
+#define BIT_SHIFT_RXFILTER_CATEGORY_3_8822B 0
+#define BIT_MASK_RXFILTER_CATEGORY_3_8822B 0xff
+#define BIT_RXFILTER_CATEGORY_3_8822B(x)                                       \
+	(((x) & BIT_MASK_RXFILTER_CATEGORY_3_8822B)                            \
+	 << BIT_SHIFT_RXFILTER_CATEGORY_3_8822B)
+#define BIT_GET_RXFILTER_CATEGORY_3_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_RXFILTER_CATEGORY_3_8822B) &                        \
+	 BIT_MASK_RXFILTER_CATEGORY_3_8822B)
+
+/* 2 REG_RXFILTER_ACTION_2_8822B */
+
+#define BIT_SHIFT_RXFILTER_ACTION_2_8822B 0
+#define BIT_MASK_RXFILTER_ACTION_2_8822B 0xff
+#define BIT_RXFILTER_ACTION_2_8822B(x)                                         \
+	(((x) & BIT_MASK_RXFILTER_ACTION_2_8822B)                              \
+	 << BIT_SHIFT_RXFILTER_ACTION_2_8822B)
+#define BIT_GET_RXFILTER_ACTION_2_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_RXFILTER_ACTION_2_8822B) &                          \
+	 BIT_MASK_RXFILTER_ACTION_2_8822B)
+
+/* 2 REG_RXFILTER_CATEGORY_2_8822B */
+
+#define BIT_SHIFT_RXFILTER_CATEGORY_2_8822B 0
+#define BIT_MASK_RXFILTER_CATEGORY_2_8822B 0xff
+#define BIT_RXFILTER_CATEGORY_2_8822B(x)                                       \
+	(((x) & BIT_MASK_RXFILTER_CATEGORY_2_8822B)                            \
+	 << BIT_SHIFT_RXFILTER_CATEGORY_2_8822B)
+#define BIT_GET_RXFILTER_CATEGORY_2_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_RXFILTER_CATEGORY_2_8822B) &                        \
+	 BIT_MASK_RXFILTER_CATEGORY_2_8822B)
+
+/* 2 REG_RXFLTMAP4_8822B (RX FILTER MAP GROUP 4) */
+#define BIT_CTRLFLT15EN_FW_8822B BIT(15)
+#define BIT_CTRLFLT14EN_FW_8822B BIT(14)
+#define BIT_CTRLFLT13EN_FW_8822B BIT(13)
+#define BIT_CTRLFLT12EN_FW_8822B BIT(12)
+#define BIT_CTRLFLT11EN_FW_8822B BIT(11)
+#define BIT_CTRLFLT10EN_FW_8822B BIT(10)
+#define BIT_CTRLFLT9EN_FW_8822B BIT(9)
+#define BIT_CTRLFLT8EN_FW_8822B BIT(8)
+#define BIT_CTRLFLT7EN_FW_8822B BIT(7)
+#define BIT_CTRLFLT6EN_FW_8822B BIT(6)
+#define BIT_CTRLFLT5EN_FW_8822B BIT(5)
+#define BIT_CTRLFLT4EN_FW_8822B BIT(4)
+#define BIT_CTRLFLT3EN_FW_8822B BIT(3)
+#define BIT_CTRLFLT2EN_FW_8822B BIT(2)
+#define BIT_CTRLFLT1EN_FW_8822B BIT(1)
+#define BIT_CTRLFLT0EN_FW_8822B BIT(0)
+
+/* 2 REG_RXFLTMAP3_8822B (RX FILTER MAP GROUP 3) */
+#define BIT_MGTFLT15EN_FW_8822B BIT(15)
+#define BIT_MGTFLT14EN_FW_8822B BIT(14)
+#define BIT_MGTFLT13EN_FW_8822B BIT(13)
+#define BIT_MGTFLT12EN_FW_8822B BIT(12)
+#define BIT_MGTFLT11EN_FW_8822B BIT(11)
+#define BIT_MGTFLT10EN_FW_8822B BIT(10)
+#define BIT_MGTFLT9EN_FW_8822B BIT(9)
+#define BIT_MGTFLT8EN_FW_8822B BIT(8)
+#define BIT_MGTFLT7EN_FW_8822B BIT(7)
+#define BIT_MGTFLT6EN_FW_8822B BIT(6)
+#define BIT_MGTFLT5EN_FW_8822B BIT(5)
+#define BIT_MGTFLT4EN_FW_8822B BIT(4)
+#define BIT_MGTFLT3EN_FW_8822B BIT(3)
+#define BIT_MGTFLT2EN_FW_8822B BIT(2)
+#define BIT_MGTFLT1EN_FW_8822B BIT(1)
+#define BIT_MGTFLT0EN_FW_8822B BIT(0)
+
+/* 2 REG_RXFLTMAP6_8822B (RX FILTER MAP GROUP 3) */
+#define BIT_ACTIONFLT15EN_FW_8822B BIT(15)
+#define BIT_ACTIONFLT14EN_FW_8822B BIT(14)
+#define BIT_ACTIONFLT13EN_FW_8822B BIT(13)
+#define BIT_ACTIONFLT12EN_FW_8822B BIT(12)
+#define BIT_ACTIONFLT11EN_FW_8822B BIT(11)
+#define BIT_ACTIONFLT10EN_FW_8822B BIT(10)
+#define BIT_ACTIONFLT9EN_FW_8822B BIT(9)
+#define BIT_ACTIONFLT8EN_FW_8822B BIT(8)
+#define BIT_ACTIONFLT7EN_FW_8822B BIT(7)
+#define BIT_ACTIONFLT6EN_FW_8822B BIT(6)
+#define BIT_ACTIONFLT5EN_FW_8822B BIT(5)
+#define BIT_ACTIONFLT4EN_FW_8822B BIT(4)
+#define BIT_ACTIONFLT3EN_FW_8822B BIT(3)
+#define BIT_ACTIONFLT2EN_FW_8822B BIT(2)
+#define BIT_ACTIONFLT1EN_FW_8822B BIT(1)
+#define BIT_ACTIONFLT0EN_FW_8822B BIT(0)
+
+/* 2 REG_RXFLTMAP5_8822B (RX FILTER MAP GROUP 3) */
+#define BIT_DATAFLT15EN_FW_8822B BIT(15)
+#define BIT_DATAFLT14EN_FW_8822B BIT(14)
+#define BIT_DATAFLT13EN_FW_8822B BIT(13)
+#define BIT_DATAFLT12EN_FW_8822B BIT(12)
+#define BIT_DATAFLT11EN_FW_8822B BIT(11)
+#define BIT_DATAFLT10EN_FW_8822B BIT(10)
+#define BIT_DATAFLT9EN_FW_8822B BIT(9)
+#define BIT_DATAFLT8EN_FW_8822B BIT(8)
+#define BIT_DATAFLT7EN_FW_8822B BIT(7)
+#define BIT_DATAFLT6EN_FW_8822B BIT(6)
+#define BIT_DATAFLT5EN_FW_8822B BIT(5)
+#define BIT_DATAFLT4EN_FW_8822B BIT(4)
+#define BIT_DATAFLT3EN_FW_8822B BIT(3)
+#define BIT_DATAFLT2EN_FW_8822B BIT(2)
+#define BIT_DATAFLT1EN_FW_8822B BIT(1)
+#define BIT_DATAFLT0EN_FW_8822B BIT(0)
+
+/* 2 REG_WMMPS_UAPSD_TID_8822B (WMM POWER SAVE UAPSD TID REGISTER) */
+#define BIT_WMMPS_UAPSD_TID7_8822B BIT(7)
+#define BIT_WMMPS_UAPSD_TID6_8822B BIT(6)
+#define BIT_WMMPS_UAPSD_TID5_8822B BIT(5)
+#define BIT_WMMPS_UAPSD_TID4_8822B BIT(4)
+#define BIT_WMMPS_UAPSD_TID3_8822B BIT(3)
+#define BIT_WMMPS_UAPSD_TID2_8822B BIT(2)
+#define BIT_WMMPS_UAPSD_TID1_8822B BIT(1)
+#define BIT_WMMPS_UAPSD_TID0_8822B BIT(0)
+
+/* 2 REG_PS_RX_INFO_8822B (POWER SAVE RX INFORMATION REGISTER) */
+
+#define BIT_SHIFT_PORTSEL__PS_RX_INFO_8822B 5
+#define BIT_MASK_PORTSEL__PS_RX_INFO_8822B 0x7
+#define BIT_PORTSEL__PS_RX_INFO_8822B(x)                                       \
+	(((x) & BIT_MASK_PORTSEL__PS_RX_INFO_8822B)                            \
+	 << BIT_SHIFT_PORTSEL__PS_RX_INFO_8822B)
+#define BIT_GET_PORTSEL__PS_RX_INFO_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_PORTSEL__PS_RX_INFO_8822B) &                        \
+	 BIT_MASK_PORTSEL__PS_RX_INFO_8822B)
+
+#define BIT_RXCTRLIN0_8822B BIT(4)
+#define BIT_RXMGTIN0_8822B BIT(3)
+#define BIT_RXDATAIN2_8822B BIT(2)
+#define BIT_RXDATAIN1_8822B BIT(1)
+#define BIT_RXDATAIN0_8822B BIT(0)
+
+/* 2 REG_NAN_RX_TSF_FILTER_8822B(NAN_RX_TSF_ADDRESS_FILTER) */
+#define BIT_CHK_TSF_TA_8822B BIT(2)
+#define BIT_CHK_TSF_CBSSID_8822B BIT(1)
+#define BIT_CHK_TSF_EN_8822B BIT(0)
+
+/* 2 REG_WOW_CTRL_8822B (WAKE ON WLAN CONTROL REGISTER) */
+
+#define BIT_SHIFT_PSF_BSSIDSEL_B2B1_8822B 6
+#define BIT_MASK_PSF_BSSIDSEL_B2B1_8822B 0x3
+#define BIT_PSF_BSSIDSEL_B2B1_8822B(x)                                         \
+	(((x) & BIT_MASK_PSF_BSSIDSEL_B2B1_8822B)                              \
+	 << BIT_SHIFT_PSF_BSSIDSEL_B2B1_8822B)
+#define BIT_GET_PSF_BSSIDSEL_B2B1_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_PSF_BSSIDSEL_B2B1_8822B) &                          \
+	 BIT_MASK_PSF_BSSIDSEL_B2B1_8822B)
+
+#define BIT_WOWHCI_8822B BIT(5)
+#define BIT_PSF_BSSIDSEL_B0_8822B BIT(4)
+#define BIT_UWF_8822B BIT(3)
+#define BIT_MAGIC_8822B BIT(2)
+#define BIT_WOWEN_8822B BIT(1)
+#define BIT_FORCE_WAKEUP_8822B BIT(0)
+
+/* 2 REG_LPNAV_CTRL_8822B (LOW POWER NAV CONTROL REGISTER) */
+#define BIT_LPNAV_EN_8822B BIT(31)
+
+#define BIT_SHIFT_LPNAV_EARLY_8822B 16
+#define BIT_MASK_LPNAV_EARLY_8822B 0x7fff
+#define BIT_LPNAV_EARLY_8822B(x)                                               \
+	(((x) & BIT_MASK_LPNAV_EARLY_8822B) << BIT_SHIFT_LPNAV_EARLY_8822B)
+#define BIT_GET_LPNAV_EARLY_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_LPNAV_EARLY_8822B) & BIT_MASK_LPNAV_EARLY_8822B)
+
+#define BIT_SHIFT_LPNAV_TH_8822B 0
+#define BIT_MASK_LPNAV_TH_8822B 0xffff
+#define BIT_LPNAV_TH_8822B(x)                                                  \
+	(((x) & BIT_MASK_LPNAV_TH_8822B) << BIT_SHIFT_LPNAV_TH_8822B)
+#define BIT_GET_LPNAV_TH_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_LPNAV_TH_8822B) & BIT_MASK_LPNAV_TH_8822B)
+
+/* 2 REG_WKFMCAM_CMD_8822B (WAKEUP FRAME CAM COMMAND REGISTER) */
+#define BIT_WKFCAM_POLLING_V1_8822B BIT(31)
+#define BIT_WKFCAM_CLR_V1_8822B BIT(30)
+#define BIT_WKFCAM_WE_8822B BIT(16)
+
+#define BIT_SHIFT_WKFCAM_ADDR_V2_8822B 8
+#define BIT_MASK_WKFCAM_ADDR_V2_8822B 0xff
+#define BIT_WKFCAM_ADDR_V2_8822B(x)                                            \
+	(((x) & BIT_MASK_WKFCAM_ADDR_V2_8822B)                                 \
+	 << BIT_SHIFT_WKFCAM_ADDR_V2_8822B)
+#define BIT_GET_WKFCAM_ADDR_V2_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_WKFCAM_ADDR_V2_8822B) &                             \
+	 BIT_MASK_WKFCAM_ADDR_V2_8822B)
+
+#define BIT_SHIFT_WKFCAM_CAM_NUM_V1_8822B 0
+#define BIT_MASK_WKFCAM_CAM_NUM_V1_8822B 0xff
+#define BIT_WKFCAM_CAM_NUM_V1_8822B(x)                                         \
+	(((x) & BIT_MASK_WKFCAM_CAM_NUM_V1_8822B)                              \
+	 << BIT_SHIFT_WKFCAM_CAM_NUM_V1_8822B)
+#define BIT_GET_WKFCAM_CAM_NUM_V1_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_WKFCAM_CAM_NUM_V1_8822B) &                          \
+	 BIT_MASK_WKFCAM_CAM_NUM_V1_8822B)
+
+/* 2 REG_WKFMCAM_RWD_8822B (WAKEUP FRAME READ/WRITE DATA) */
+
+#define BIT_SHIFT_WKFMCAM_RWD_8822B 0
+#define BIT_MASK_WKFMCAM_RWD_8822B 0xffffffffL
+#define BIT_WKFMCAM_RWD_8822B(x)                                               \
+	(((x) & BIT_MASK_WKFMCAM_RWD_8822B) << BIT_SHIFT_WKFMCAM_RWD_8822B)
+#define BIT_GET_WKFMCAM_RWD_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_WKFMCAM_RWD_8822B) & BIT_MASK_WKFMCAM_RWD_8822B)
+
+/* 2 REG_RXFLTMAP1_8822B (RX FILTER MAP GROUP 1) */
+#define BIT_CTRLFLT15EN_8822B BIT(15)
+#define BIT_CTRLFLT14EN_8822B BIT(14)
+#define BIT_CTRLFLT13EN_8822B BIT(13)
+#define BIT_CTRLFLT12EN_8822B BIT(12)
+#define BIT_CTRLFLT11EN_8822B BIT(11)
+#define BIT_CTRLFLT10EN_8822B BIT(10)
+#define BIT_CTRLFLT9EN_8822B BIT(9)
+#define BIT_CTRLFLT8EN_8822B BIT(8)
+#define BIT_CTRLFLT7EN_8822B BIT(7)
+#define BIT_CTRLFLT6EN_8822B BIT(6)
+#define BIT_CTRLFLT5EN_8822B BIT(5)
+#define BIT_CTRLFLT4EN_8822B BIT(4)
+#define BIT_CTRLFLT3EN_8822B BIT(3)
+#define BIT_CTRLFLT2EN_8822B BIT(2)
+#define BIT_CTRLFLT1EN_8822B BIT(1)
+#define BIT_CTRLFLT0EN_8822B BIT(0)
+
+/* 2 REG_RXFLTMAP0_8822B (RX FILTER MAP GROUP 0) */
+#define BIT_MGTFLT15EN_8822B BIT(15)
+#define BIT_MGTFLT14EN_8822B BIT(14)
+#define BIT_MGTFLT13EN_8822B BIT(13)
+#define BIT_MGTFLT12EN_8822B BIT(12)
+#define BIT_MGTFLT11EN_8822B BIT(11)
+#define BIT_MGTFLT10EN_8822B BIT(10)
+#define BIT_MGTFLT9EN_8822B BIT(9)
+#define BIT_MGTFLT8EN_8822B BIT(8)
+#define BIT_MGTFLT7EN_8822B BIT(7)
+#define BIT_MGTFLT6EN_8822B BIT(6)
+#define BIT_MGTFLT5EN_8822B BIT(5)
+#define BIT_MGTFLT4EN_8822B BIT(4)
+#define BIT_MGTFLT3EN_8822B BIT(3)
+#define BIT_MGTFLT2EN_8822B BIT(2)
+#define BIT_MGTFLT1EN_8822B BIT(1)
+#define BIT_MGTFLT0EN_8822B BIT(0)
+
+/* 2 REG_NOT_VALID_8822B */
+
+/* 2 REG_RXFLTMAP_8822B (RX FILTER MAP GROUP 2) */
+#define BIT_DATAFLT15EN_8822B BIT(15)
+#define BIT_DATAFLT14EN_8822B BIT(14)
+#define BIT_DATAFLT13EN_8822B BIT(13)
+#define BIT_DATAFLT12EN_8822B BIT(12)
+#define BIT_DATAFLT11EN_8822B BIT(11)
+#define BIT_DATAFLT10EN_8822B BIT(10)
+#define BIT_DATAFLT9EN_8822B BIT(9)
+#define BIT_DATAFLT8EN_8822B BIT(8)
+#define BIT_DATAFLT7EN_8822B BIT(7)
+#define BIT_DATAFLT6EN_8822B BIT(6)
+#define BIT_DATAFLT5EN_8822B BIT(5)
+#define BIT_DATAFLT4EN_8822B BIT(4)
+#define BIT_DATAFLT3EN_8822B BIT(3)
+#define BIT_DATAFLT2EN_8822B BIT(2)
+#define BIT_DATAFLT1EN_8822B BIT(1)
+#define BIT_DATAFLT0EN_8822B BIT(0)
+
+/* 2 REG_BCN_PSR_RPT_8822B (BEACON PARSER REPORT REGISTER) */
+
+#define BIT_SHIFT_DTIM_CNT_8822B 24
+#define BIT_MASK_DTIM_CNT_8822B 0xff
+#define BIT_DTIM_CNT_8822B(x)                                                  \
+	(((x) & BIT_MASK_DTIM_CNT_8822B) << BIT_SHIFT_DTIM_CNT_8822B)
+#define BIT_GET_DTIM_CNT_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_DTIM_CNT_8822B) & BIT_MASK_DTIM_CNT_8822B)
+
+#define BIT_SHIFT_DTIM_PERIOD_8822B 16
+#define BIT_MASK_DTIM_PERIOD_8822B 0xff
+#define BIT_DTIM_PERIOD_8822B(x)                                               \
+	(((x) & BIT_MASK_DTIM_PERIOD_8822B) << BIT_SHIFT_DTIM_PERIOD_8822B)
+#define BIT_GET_DTIM_PERIOD_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_DTIM_PERIOD_8822B) & BIT_MASK_DTIM_PERIOD_8822B)
+
+#define BIT_DTIM_8822B BIT(15)
+#define BIT_TIM_8822B BIT(14)
+
+#define BIT_SHIFT_PS_AID_0_8822B 0
+#define BIT_MASK_PS_AID_0_8822B 0x7ff
+#define BIT_PS_AID_0_8822B(x)                                                  \
+	(((x) & BIT_MASK_PS_AID_0_8822B) << BIT_SHIFT_PS_AID_0_8822B)
+#define BIT_GET_PS_AID_0_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_PS_AID_0_8822B) & BIT_MASK_PS_AID_0_8822B)
+
+/* 2 REG_FLC_TRPC_8822B (TIMER OF FLC_RPC) */
+#define BIT_FLC_RPCT_V1_8822B BIT(7)
+#define BIT_MODE_8822B BIT(6)
+
+#define BIT_SHIFT_TRPCD_8822B 0
+#define BIT_MASK_TRPCD_8822B 0x3f
+#define BIT_TRPCD_8822B(x)                                                     \
+	(((x) & BIT_MASK_TRPCD_8822B) << BIT_SHIFT_TRPCD_8822B)
+#define BIT_GET_TRPCD_8822B(x)                                                 \
+	(((x) >> BIT_SHIFT_TRPCD_8822B) & BIT_MASK_TRPCD_8822B)
+
+/* 2 REG_FLC_PTS_8822B (PKT TYPE SELECTION OF FLC_RPC T) */
+#define BIT_CMF_8822B BIT(2)
+#define BIT_CCF_8822B BIT(1)
+#define BIT_CDF_8822B BIT(0)
+
+/* 2 REG_FLC_RPCT_8822B (FLC_RPC THRESHOLD) */
+
+#define BIT_SHIFT_FLC_RPCT_8822B 0
+#define BIT_MASK_FLC_RPCT_8822B 0xff
+#define BIT_FLC_RPCT_8822B(x)                                                  \
+	(((x) & BIT_MASK_FLC_RPCT_8822B) << BIT_SHIFT_FLC_RPCT_8822B)
+#define BIT_GET_FLC_RPCT_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_FLC_RPCT_8822B) & BIT_MASK_FLC_RPCT_8822B)
+
+/* 2 REG_FLC_RPC_8822B (FW LPS CONDITION -- RX PKT COUNTER) */
+
+#define BIT_SHIFT_FLC_RPC_8822B 0
+#define BIT_MASK_FLC_RPC_8822B 0xff
+#define BIT_FLC_RPC_8822B(x)                                                   \
+	(((x) & BIT_MASK_FLC_RPC_8822B) << BIT_SHIFT_FLC_RPC_8822B)
+#define BIT_GET_FLC_RPC_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_FLC_RPC_8822B) & BIT_MASK_FLC_RPC_8822B)
+
+/* 2 REG_RXPKTMON_CTRL_8822B */
+
+#define BIT_SHIFT_RXBKQPKT_SEQ_8822B 20
+#define BIT_MASK_RXBKQPKT_SEQ_8822B 0xf
+#define BIT_RXBKQPKT_SEQ_8822B(x)                                              \
+	(((x) & BIT_MASK_RXBKQPKT_SEQ_8822B) << BIT_SHIFT_RXBKQPKT_SEQ_8822B)
+#define BIT_GET_RXBKQPKT_SEQ_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_RXBKQPKT_SEQ_8822B) & BIT_MASK_RXBKQPKT_SEQ_8822B)
+
+#define BIT_SHIFT_RXBEQPKT_SEQ_8822B 16
+#define BIT_MASK_RXBEQPKT_SEQ_8822B 0xf
+#define BIT_RXBEQPKT_SEQ_8822B(x)                                              \
+	(((x) & BIT_MASK_RXBEQPKT_SEQ_8822B) << BIT_SHIFT_RXBEQPKT_SEQ_8822B)
+#define BIT_GET_RXBEQPKT_SEQ_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_RXBEQPKT_SEQ_8822B) & BIT_MASK_RXBEQPKT_SEQ_8822B)
+
+#define BIT_SHIFT_RXVIQPKT_SEQ_8822B 12
+#define BIT_MASK_RXVIQPKT_SEQ_8822B 0xf
+#define BIT_RXVIQPKT_SEQ_8822B(x)                                              \
+	(((x) & BIT_MASK_RXVIQPKT_SEQ_8822B) << BIT_SHIFT_RXVIQPKT_SEQ_8822B)
+#define BIT_GET_RXVIQPKT_SEQ_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_RXVIQPKT_SEQ_8822B) & BIT_MASK_RXVIQPKT_SEQ_8822B)
+
+#define BIT_SHIFT_RXVOQPKT_SEQ_8822B 8
+#define BIT_MASK_RXVOQPKT_SEQ_8822B 0xf
+#define BIT_RXVOQPKT_SEQ_8822B(x)                                              \
+	(((x) & BIT_MASK_RXVOQPKT_SEQ_8822B) << BIT_SHIFT_RXVOQPKT_SEQ_8822B)
+#define BIT_GET_RXVOQPKT_SEQ_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_RXVOQPKT_SEQ_8822B) & BIT_MASK_RXVOQPKT_SEQ_8822B)
+
+#define BIT_RXBKQPKT_ERR_8822B BIT(7)
+#define BIT_RXBEQPKT_ERR_8822B BIT(6)
+#define BIT_RXVIQPKT_ERR_8822B BIT(5)
+#define BIT_RXVOQPKT_ERR_8822B BIT(4)
+#define BIT_RXDMA_MON_EN_8822B BIT(2)
+#define BIT_RXPKT_MON_RST_8822B BIT(1)
+#define BIT_RXPKT_MON_EN_8822B BIT(0)
+
+/* 2 REG_STATE_MON_8822B */
+
+#define BIT_SHIFT_STATE_SEL_8822B 24
+#define BIT_MASK_STATE_SEL_8822B 0x1f
+#define BIT_STATE_SEL_8822B(x)                                                 \
+	(((x) & BIT_MASK_STATE_SEL_8822B) << BIT_SHIFT_STATE_SEL_8822B)
+#define BIT_GET_STATE_SEL_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_STATE_SEL_8822B) & BIT_MASK_STATE_SEL_8822B)
+
+#define BIT_SHIFT_STATE_INFO_8822B 8
+#define BIT_MASK_STATE_INFO_8822B 0xff
+#define BIT_STATE_INFO_8822B(x)                                                \
+	(((x) & BIT_MASK_STATE_INFO_8822B) << BIT_SHIFT_STATE_INFO_8822B)
+#define BIT_GET_STATE_INFO_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_STATE_INFO_8822B) & BIT_MASK_STATE_INFO_8822B)
+
+#define BIT_UPD_NXT_STATE_8822B BIT(7)
+
+#define BIT_SHIFT_CUR_STATE_8822B 0
+#define BIT_MASK_CUR_STATE_8822B 0x7f
+#define BIT_CUR_STATE_8822B(x)                                                 \
+	(((x) & BIT_MASK_CUR_STATE_8822B) << BIT_SHIFT_CUR_STATE_8822B)
+#define BIT_GET_CUR_STATE_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_CUR_STATE_8822B) & BIT_MASK_CUR_STATE_8822B)
+
+/* 2 REG_ERROR_MON_8822B */
+#define BIT_MACRX_ERR_1_8822B BIT(17)
+#define BIT_MACRX_ERR_0_8822B BIT(16)
+#define BIT_MACTX_ERR_3_8822B BIT(3)
+#define BIT_MACTX_ERR_2_8822B BIT(2)
+#define BIT_MACTX_ERR_1_8822B BIT(1)
+#define BIT_MACTX_ERR_0_8822B BIT(0)
+
+/* 2 REG_SEARCH_MACID_8822B */
+#define BIT_EN_TXRPTBUF_CLK_8822B BIT(31)
+
+#define BIT_SHIFT_INFO_INDEX_OFFSET_8822B 16
+#define BIT_MASK_INFO_INDEX_OFFSET_8822B 0x1fff
+#define BIT_INFO_INDEX_OFFSET_8822B(x)                                         \
+	(((x) & BIT_MASK_INFO_INDEX_OFFSET_8822B)                              \
+	 << BIT_SHIFT_INFO_INDEX_OFFSET_8822B)
+#define BIT_GET_INFO_INDEX_OFFSET_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_INFO_INDEX_OFFSET_8822B) &                          \
+	 BIT_MASK_INFO_INDEX_OFFSET_8822B)
+
+#define BIT_WMAC_SRCH_FIFOFULL_8822B BIT(15)
+#define BIT_DIS_INFOSRCH_8822B BIT(14)
+#define BIT_DISABLE_B0_8822B BIT(13)
+
+#define BIT_SHIFT_INFO_ADDR_OFFSET_8822B 0
+#define BIT_MASK_INFO_ADDR_OFFSET_8822B 0x1fff
+#define BIT_INFO_ADDR_OFFSET_8822B(x)                                          \
+	(((x) & BIT_MASK_INFO_ADDR_OFFSET_8822B)                               \
+	 << BIT_SHIFT_INFO_ADDR_OFFSET_8822B)
+#define BIT_GET_INFO_ADDR_OFFSET_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_INFO_ADDR_OFFSET_8822B) &                           \
+	 BIT_MASK_INFO_ADDR_OFFSET_8822B)
+
+/* 2 REG_BT_COEX_TABLE_8822B (BT-COEXISTENCE CONTROL REGISTER) */
+#define BIT_PRI_MASK_RX_RESP_8822B BIT(126)
+#define BIT_PRI_MASK_RXOFDM_8822B BIT(125)
+#define BIT_PRI_MASK_RXCCK_8822B BIT(124)
+
+#define BIT_SHIFT_PRI_MASK_TXAC_8822B (117 & CPU_OPT_WIDTH)
+#define BIT_MASK_PRI_MASK_TXAC_8822B 0x7f
+#define BIT_PRI_MASK_TXAC_8822B(x)                                             \
+	(((x) & BIT_MASK_PRI_MASK_TXAC_8822B) << BIT_SHIFT_PRI_MASK_TXAC_8822B)
+#define BIT_GET_PRI_MASK_TXAC_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_PRI_MASK_TXAC_8822B) & BIT_MASK_PRI_MASK_TXAC_8822B)
+
+#define BIT_SHIFT_PRI_MASK_NAV_8822B (109 & CPU_OPT_WIDTH)
+#define BIT_MASK_PRI_MASK_NAV_8822B 0xff
+#define BIT_PRI_MASK_NAV_8822B(x)                                              \
+	(((x) & BIT_MASK_PRI_MASK_NAV_8822B) << BIT_SHIFT_PRI_MASK_NAV_8822B)
+#define BIT_GET_PRI_MASK_NAV_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_PRI_MASK_NAV_8822B) & BIT_MASK_PRI_MASK_NAV_8822B)
+
+#define BIT_PRI_MASK_CCK_8822B BIT(108)
+#define BIT_PRI_MASK_OFDM_8822B BIT(107)
+#define BIT_PRI_MASK_RTY_8822B BIT(106)
+
+#define BIT_SHIFT_PRI_MASK_NUM_8822B (102 & CPU_OPT_WIDTH)
+#define BIT_MASK_PRI_MASK_NUM_8822B 0xf
+#define BIT_PRI_MASK_NUM_8822B(x)                                              \
+	(((x) & BIT_MASK_PRI_MASK_NUM_8822B) << BIT_SHIFT_PRI_MASK_NUM_8822B)
+#define BIT_GET_PRI_MASK_NUM_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_PRI_MASK_NUM_8822B) & BIT_MASK_PRI_MASK_NUM_8822B)
+
+#define BIT_SHIFT_PRI_MASK_TYPE_8822B (98 & CPU_OPT_WIDTH)
+#define BIT_MASK_PRI_MASK_TYPE_8822B 0xf
+#define BIT_PRI_MASK_TYPE_8822B(x)                                             \
+	(((x) & BIT_MASK_PRI_MASK_TYPE_8822B) << BIT_SHIFT_PRI_MASK_TYPE_8822B)
+#define BIT_GET_PRI_MASK_TYPE_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_PRI_MASK_TYPE_8822B) & BIT_MASK_PRI_MASK_TYPE_8822B)
+
+#define BIT_OOB_8822B BIT(97)
+#define BIT_ANT_SEL_8822B BIT(96)
+
+#define BIT_SHIFT_BREAK_TABLE_2_8822B (80 & CPU_OPT_WIDTH)
+#define BIT_MASK_BREAK_TABLE_2_8822B 0xffff
+#define BIT_BREAK_TABLE_2_8822B(x)                                             \
+	(((x) & BIT_MASK_BREAK_TABLE_2_8822B) << BIT_SHIFT_BREAK_TABLE_2_8822B)
+#define BIT_GET_BREAK_TABLE_2_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_BREAK_TABLE_2_8822B) & BIT_MASK_BREAK_TABLE_2_8822B)
+
+#define BIT_SHIFT_BREAK_TABLE_1_8822B (64 & CPU_OPT_WIDTH)
+#define BIT_MASK_BREAK_TABLE_1_8822B 0xffff
+#define BIT_BREAK_TABLE_1_8822B(x)                                             \
+	(((x) & BIT_MASK_BREAK_TABLE_1_8822B) << BIT_SHIFT_BREAK_TABLE_1_8822B)
+#define BIT_GET_BREAK_TABLE_1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_BREAK_TABLE_1_8822B) & BIT_MASK_BREAK_TABLE_1_8822B)
+
+#define BIT_SHIFT_COEX_TABLE_2_8822B (32 & CPU_OPT_WIDTH)
+#define BIT_MASK_COEX_TABLE_2_8822B 0xffffffffL
+#define BIT_COEX_TABLE_2_8822B(x)                                              \
+	(((x) & BIT_MASK_COEX_TABLE_2_8822B) << BIT_SHIFT_COEX_TABLE_2_8822B)
+#define BIT_GET_COEX_TABLE_2_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_COEX_TABLE_2_8822B) & BIT_MASK_COEX_TABLE_2_8822B)
+
+#define BIT_SHIFT_COEX_TABLE_1_8822B 0
+#define BIT_MASK_COEX_TABLE_1_8822B 0xffffffffL
+#define BIT_COEX_TABLE_1_8822B(x)                                              \
+	(((x) & BIT_MASK_COEX_TABLE_1_8822B) << BIT_SHIFT_COEX_TABLE_1_8822B)
+#define BIT_GET_COEX_TABLE_1_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_COEX_TABLE_1_8822B) & BIT_MASK_COEX_TABLE_1_8822B)
+
+/* 2 REG_RXCMD_0_8822B */
+#define BIT_RXCMD_EN_8822B BIT(31)
+
+#define BIT_SHIFT_RXCMD_INFO_8822B 0
+#define BIT_MASK_RXCMD_INFO_8822B 0x7fffffffL
+#define BIT_RXCMD_INFO_8822B(x)                                                \
+	(((x) & BIT_MASK_RXCMD_INFO_8822B) << BIT_SHIFT_RXCMD_INFO_8822B)
+#define BIT_GET_RXCMD_INFO_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_RXCMD_INFO_8822B) & BIT_MASK_RXCMD_INFO_8822B)
+
+/* 2 REG_RXCMD_1_8822B */
+
+#define BIT_SHIFT_RXCMD_PRD_8822B 0
+#define BIT_MASK_RXCMD_PRD_8822B 0xffff
+#define BIT_RXCMD_PRD_8822B(x)                                                 \
+	(((x) & BIT_MASK_RXCMD_PRD_8822B) << BIT_SHIFT_RXCMD_PRD_8822B)
+#define BIT_GET_RXCMD_PRD_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_RXCMD_PRD_8822B) & BIT_MASK_RXCMD_PRD_8822B)
+
+/* 2 REG_NOT_VALID_8822B */
+
+/* 2 REG_WMAC_RESP_TXINFO_8822B (RESPONSE TXINFO REGISTER) */
+
+#define BIT_SHIFT_WMAC_RESP_MFB_8822B 25
+#define BIT_MASK_WMAC_RESP_MFB_8822B 0x7f
+#define BIT_WMAC_RESP_MFB_8822B(x)                                             \
+	(((x) & BIT_MASK_WMAC_RESP_MFB_8822B) << BIT_SHIFT_WMAC_RESP_MFB_8822B)
+#define BIT_GET_WMAC_RESP_MFB_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_WMAC_RESP_MFB_8822B) & BIT_MASK_WMAC_RESP_MFB_8822B)
+
+#define BIT_SHIFT_WMAC_ANTINF_SEL_8822B 23
+#define BIT_MASK_WMAC_ANTINF_SEL_8822B 0x3
+#define BIT_WMAC_ANTINF_SEL_8822B(x)                                           \
+	(((x) & BIT_MASK_WMAC_ANTINF_SEL_8822B)                                \
+	 << BIT_SHIFT_WMAC_ANTINF_SEL_8822B)
+#define BIT_GET_WMAC_ANTINF_SEL_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_WMAC_ANTINF_SEL_8822B) &                            \
+	 BIT_MASK_WMAC_ANTINF_SEL_8822B)
+
+#define BIT_SHIFT_WMAC_ANTSEL_SEL_8822B 21
+#define BIT_MASK_WMAC_ANTSEL_SEL_8822B 0x3
+#define BIT_WMAC_ANTSEL_SEL_8822B(x)                                           \
+	(((x) & BIT_MASK_WMAC_ANTSEL_SEL_8822B)                                \
+	 << BIT_SHIFT_WMAC_ANTSEL_SEL_8822B)
+#define BIT_GET_WMAC_ANTSEL_SEL_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_WMAC_ANTSEL_SEL_8822B) &                            \
+	 BIT_MASK_WMAC_ANTSEL_SEL_8822B)
+
+#define BIT_SHIFT_R_WMAC_RESP_TXPOWER_8822B 18
+#define BIT_MASK_R_WMAC_RESP_TXPOWER_8822B 0x7
+#define BIT_R_WMAC_RESP_TXPOWER_8822B(x)                                       \
+	(((x) & BIT_MASK_R_WMAC_RESP_TXPOWER_8822B)                            \
+	 << BIT_SHIFT_R_WMAC_RESP_TXPOWER_8822B)
+#define BIT_GET_R_WMAC_RESP_TXPOWER_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_R_WMAC_RESP_TXPOWER_8822B) &                        \
+	 BIT_MASK_R_WMAC_RESP_TXPOWER_8822B)
+
+#define BIT_SHIFT_WMAC_RESP_TXANT_8822B 0
+#define BIT_MASK_WMAC_RESP_TXANT_8822B 0x3ffff
+#define BIT_WMAC_RESP_TXANT_8822B(x)                                           \
+	(((x) & BIT_MASK_WMAC_RESP_TXANT_8822B)                                \
+	 << BIT_SHIFT_WMAC_RESP_TXANT_8822B)
+#define BIT_GET_WMAC_RESP_TXANT_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_WMAC_RESP_TXANT_8822B) &                            \
+	 BIT_MASK_WMAC_RESP_TXANT_8822B)
+
+/* 2 REG_BBPSF_CTRL_8822B */
+#define BIT_CTL_IDLE_CLR_CSI_RPT_8822B BIT(31)
+#define BIT_WMAC_USE_NDPARATE_8822B BIT(30)
+
+#define BIT_SHIFT_WMAC_CSI_RATE_8822B 24
+#define BIT_MASK_WMAC_CSI_RATE_8822B 0x3f
+#define BIT_WMAC_CSI_RATE_8822B(x)                                             \
+	(((x) & BIT_MASK_WMAC_CSI_RATE_8822B) << BIT_SHIFT_WMAC_CSI_RATE_8822B)
+#define BIT_GET_WMAC_CSI_RATE_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_WMAC_CSI_RATE_8822B) & BIT_MASK_WMAC_CSI_RATE_8822B)
+
+#define BIT_SHIFT_WMAC_RESP_TXRATE_8822B 16
+#define BIT_MASK_WMAC_RESP_TXRATE_8822B 0xff
+#define BIT_WMAC_RESP_TXRATE_8822B(x)                                          \
+	(((x) & BIT_MASK_WMAC_RESP_TXRATE_8822B)                               \
+	 << BIT_SHIFT_WMAC_RESP_TXRATE_8822B)
+#define BIT_GET_WMAC_RESP_TXRATE_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_WMAC_RESP_TXRATE_8822B) &                           \
+	 BIT_MASK_WMAC_RESP_TXRATE_8822B)
+
+#define BIT_BBPSF_MPDUCHKEN_8822B BIT(5)
+#define BIT_BBPSF_MHCHKEN_8822B BIT(4)
+#define BIT_BBPSF_ERRCHKEN_8822B BIT(3)
+
+#define BIT_SHIFT_BBPSF_ERRTHR_8822B 0
+#define BIT_MASK_BBPSF_ERRTHR_8822B 0x7
+#define BIT_BBPSF_ERRTHR_8822B(x)                                              \
+	(((x) & BIT_MASK_BBPSF_ERRTHR_8822B) << BIT_SHIFT_BBPSF_ERRTHR_8822B)
+#define BIT_GET_BBPSF_ERRTHR_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_BBPSF_ERRTHR_8822B) & BIT_MASK_BBPSF_ERRTHR_8822B)
+
+/* 2 REG_NOT_VALID_8822B */
+
+/* 2 REG_P2P_RX_BCN_NOA_8822B (P2P RX BEACON NOA REGISTER) */
+#define BIT_NOA_PARSER_EN_8822B BIT(15)
+#define BIT_BSSID_SEL_8822B BIT(14)
+
+#define BIT_SHIFT_P2P_OUI_TYPE_8822B 0
+#define BIT_MASK_P2P_OUI_TYPE_8822B 0xff
+#define BIT_P2P_OUI_TYPE_8822B(x)                                              \
+	(((x) & BIT_MASK_P2P_OUI_TYPE_8822B) << BIT_SHIFT_P2P_OUI_TYPE_8822B)
+#define BIT_GET_P2P_OUI_TYPE_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_P2P_OUI_TYPE_8822B) & BIT_MASK_P2P_OUI_TYPE_8822B)
+
+/* 2 REG_ASSOCIATED_BFMER0_INFO_8822B (ASSOCIATED BEAMFORMER0 INFO REGISTER) */
+
+#define BIT_SHIFT_R_WMAC_TXCSI_AID0_8822B (48 & CPU_OPT_WIDTH)
+#define BIT_MASK_R_WMAC_TXCSI_AID0_8822B 0x1ff
+#define BIT_R_WMAC_TXCSI_AID0_8822B(x)                                         \
+	(((x) & BIT_MASK_R_WMAC_TXCSI_AID0_8822B)                              \
+	 << BIT_SHIFT_R_WMAC_TXCSI_AID0_8822B)
+#define BIT_GET_R_WMAC_TXCSI_AID0_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_R_WMAC_TXCSI_AID0_8822B) &                          \
+	 BIT_MASK_R_WMAC_TXCSI_AID0_8822B)
+
+#define BIT_SHIFT_R_WMAC_SOUNDING_RXADD_R0_8822B 0
+#define BIT_MASK_R_WMAC_SOUNDING_RXADD_R0_8822B 0xffffffffffffL
+#define BIT_R_WMAC_SOUNDING_RXADD_R0_8822B(x)                                  \
+	(((x) & BIT_MASK_R_WMAC_SOUNDING_RXADD_R0_8822B)                       \
+	 << BIT_SHIFT_R_WMAC_SOUNDING_RXADD_R0_8822B)
+#define BIT_GET_R_WMAC_SOUNDING_RXADD_R0_8822B(x)                              \
+	(((x) >> BIT_SHIFT_R_WMAC_SOUNDING_RXADD_R0_8822B) &                   \
+	 BIT_MASK_R_WMAC_SOUNDING_RXADD_R0_8822B)
+
+/* 2 REG_ASSOCIATED_BFMER1_INFO_8822B (ASSOCIATED BEAMFORMER1 INFO REGISTER) */
+
+#define BIT_SHIFT_R_WMAC_TXCSI_AID1_8822B (48 & CPU_OPT_WIDTH)
+#define BIT_MASK_R_WMAC_TXCSI_AID1_8822B 0x1ff
+#define BIT_R_WMAC_TXCSI_AID1_8822B(x)                                         \
+	(((x) & BIT_MASK_R_WMAC_TXCSI_AID1_8822B)                              \
+	 << BIT_SHIFT_R_WMAC_TXCSI_AID1_8822B)
+#define BIT_GET_R_WMAC_TXCSI_AID1_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_R_WMAC_TXCSI_AID1_8822B) &                          \
+	 BIT_MASK_R_WMAC_TXCSI_AID1_8822B)
+
+#define BIT_SHIFT_R_WMAC_SOUNDING_RXADD_R1_8822B 0
+#define BIT_MASK_R_WMAC_SOUNDING_RXADD_R1_8822B 0xffffffffffffL
+#define BIT_R_WMAC_SOUNDING_RXADD_R1_8822B(x)                                  \
+	(((x) & BIT_MASK_R_WMAC_SOUNDING_RXADD_R1_8822B)                       \
+	 << BIT_SHIFT_R_WMAC_SOUNDING_RXADD_R1_8822B)
+#define BIT_GET_R_WMAC_SOUNDING_RXADD_R1_8822B(x)                              \
+	(((x) >> BIT_SHIFT_R_WMAC_SOUNDING_RXADD_R1_8822B) &                   \
+	 BIT_MASK_R_WMAC_SOUNDING_RXADD_R1_8822B)
+
+/* 2 REG_TX_CSI_RPT_PARAM_BW20_8822B (TX CSI REPORT PARAMETER REGISTER) */
+
+#define BIT_SHIFT_R_WMAC_BFINFO_20M_1_8822B 16
+#define BIT_MASK_R_WMAC_BFINFO_20M_1_8822B 0xfff
+#define BIT_R_WMAC_BFINFO_20M_1_8822B(x)                                       \
+	(((x) & BIT_MASK_R_WMAC_BFINFO_20M_1_8822B)                            \
+	 << BIT_SHIFT_R_WMAC_BFINFO_20M_1_8822B)
+#define BIT_GET_R_WMAC_BFINFO_20M_1_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_R_WMAC_BFINFO_20M_1_8822B) &                        \
+	 BIT_MASK_R_WMAC_BFINFO_20M_1_8822B)
+
+#define BIT_SHIFT_R_WMAC_BFINFO_20M_0_8822B 0
+#define BIT_MASK_R_WMAC_BFINFO_20M_0_8822B 0xfff
+#define BIT_R_WMAC_BFINFO_20M_0_8822B(x)                                       \
+	(((x) & BIT_MASK_R_WMAC_BFINFO_20M_0_8822B)                            \
+	 << BIT_SHIFT_R_WMAC_BFINFO_20M_0_8822B)
+#define BIT_GET_R_WMAC_BFINFO_20M_0_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_R_WMAC_BFINFO_20M_0_8822B) &                        \
+	 BIT_MASK_R_WMAC_BFINFO_20M_0_8822B)
+
+/* 2 REG_TX_CSI_RPT_PARAM_BW40_8822B (TX CSI REPORT PARAMETER_BW40 REGISTER) */
+
+#define BIT_SHIFT_WMAC_RESP_ANTCD_8822B 0
+#define BIT_MASK_WMAC_RESP_ANTCD_8822B 0xf
+#define BIT_WMAC_RESP_ANTCD_8822B(x)                                           \
+	(((x) & BIT_MASK_WMAC_RESP_ANTCD_8822B)                                \
+	 << BIT_SHIFT_WMAC_RESP_ANTCD_8822B)
+#define BIT_GET_WMAC_RESP_ANTCD_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_WMAC_RESP_ANTCD_8822B) &                            \
+	 BIT_MASK_WMAC_RESP_ANTCD_8822B)
+
+/* 2 REG_TX_CSI_RPT_PARAM_BW80_8822B (TX CSI REPORT PARAMETER_BW80 REGISTER) */
+
+/* 2 REG_BCN_PSR_RPT2_8822B (BEACON PARSER REPORT REGISTER2) */
+
+#define BIT_SHIFT_DTIM_CNT2_8822B 24
+#define BIT_MASK_DTIM_CNT2_8822B 0xff
+#define BIT_DTIM_CNT2_8822B(x)                                                 \
+	(((x) & BIT_MASK_DTIM_CNT2_8822B) << BIT_SHIFT_DTIM_CNT2_8822B)
+#define BIT_GET_DTIM_CNT2_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_DTIM_CNT2_8822B) & BIT_MASK_DTIM_CNT2_8822B)
+
+#define BIT_SHIFT_DTIM_PERIOD2_8822B 16
+#define BIT_MASK_DTIM_PERIOD2_8822B 0xff
+#define BIT_DTIM_PERIOD2_8822B(x)                                              \
+	(((x) & BIT_MASK_DTIM_PERIOD2_8822B) << BIT_SHIFT_DTIM_PERIOD2_8822B)
+#define BIT_GET_DTIM_PERIOD2_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_DTIM_PERIOD2_8822B) & BIT_MASK_DTIM_PERIOD2_8822B)
+
+#define BIT_DTIM2_8822B BIT(15)
+#define BIT_TIM2_8822B BIT(14)
+
+#define BIT_SHIFT_PS_AID_2_8822B 0
+#define BIT_MASK_PS_AID_2_8822B 0x7ff
+#define BIT_PS_AID_2_8822B(x)                                                  \
+	(((x) & BIT_MASK_PS_AID_2_8822B) << BIT_SHIFT_PS_AID_2_8822B)
+#define BIT_GET_PS_AID_2_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_PS_AID_2_8822B) & BIT_MASK_PS_AID_2_8822B)
+
+/* 2 REG_BCN_PSR_RPT3_8822B (BEACON PARSER REPORT REGISTER3) */
+
+#define BIT_SHIFT_DTIM_CNT3_8822B 24
+#define BIT_MASK_DTIM_CNT3_8822B 0xff
+#define BIT_DTIM_CNT3_8822B(x)                                                 \
+	(((x) & BIT_MASK_DTIM_CNT3_8822B) << BIT_SHIFT_DTIM_CNT3_8822B)
+#define BIT_GET_DTIM_CNT3_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_DTIM_CNT3_8822B) & BIT_MASK_DTIM_CNT3_8822B)
+
+#define BIT_SHIFT_DTIM_PERIOD3_8822B 16
+#define BIT_MASK_DTIM_PERIOD3_8822B 0xff
+#define BIT_DTIM_PERIOD3_8822B(x)                                              \
+	(((x) & BIT_MASK_DTIM_PERIOD3_8822B) << BIT_SHIFT_DTIM_PERIOD3_8822B)
+#define BIT_GET_DTIM_PERIOD3_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_DTIM_PERIOD3_8822B) & BIT_MASK_DTIM_PERIOD3_8822B)
+
+#define BIT_DTIM3_8822B BIT(15)
+#define BIT_TIM3_8822B BIT(14)
+
+#define BIT_SHIFT_PS_AID_3_8822B 0
+#define BIT_MASK_PS_AID_3_8822B 0x7ff
+#define BIT_PS_AID_3_8822B(x)                                                  \
+	(((x) & BIT_MASK_PS_AID_3_8822B) << BIT_SHIFT_PS_AID_3_8822B)
+#define BIT_GET_PS_AID_3_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_PS_AID_3_8822B) & BIT_MASK_PS_AID_3_8822B)
+
+/* 2 REG_BCN_PSR_RPT4_8822B (BEACON PARSER REPORT REGISTER4) */
+
+#define BIT_SHIFT_DTIM_CNT4_8822B 24
+#define BIT_MASK_DTIM_CNT4_8822B 0xff
+#define BIT_DTIM_CNT4_8822B(x)                                                 \
+	(((x) & BIT_MASK_DTIM_CNT4_8822B) << BIT_SHIFT_DTIM_CNT4_8822B)
+#define BIT_GET_DTIM_CNT4_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_DTIM_CNT4_8822B) & BIT_MASK_DTIM_CNT4_8822B)
+
+#define BIT_SHIFT_DTIM_PERIOD4_8822B 16
+#define BIT_MASK_DTIM_PERIOD4_8822B 0xff
+#define BIT_DTIM_PERIOD4_8822B(x)                                              \
+	(((x) & BIT_MASK_DTIM_PERIOD4_8822B) << BIT_SHIFT_DTIM_PERIOD4_8822B)
+#define BIT_GET_DTIM_PERIOD4_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_DTIM_PERIOD4_8822B) & BIT_MASK_DTIM_PERIOD4_8822B)
+
+#define BIT_DTIM4_8822B BIT(15)
+#define BIT_TIM4_8822B BIT(14)
+
+#define BIT_SHIFT_PS_AID_4_8822B 0
+#define BIT_MASK_PS_AID_4_8822B 0x7ff
+#define BIT_PS_AID_4_8822B(x)                                                  \
+	(((x) & BIT_MASK_PS_AID_4_8822B) << BIT_SHIFT_PS_AID_4_8822B)
+#define BIT_GET_PS_AID_4_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_PS_AID_4_8822B) & BIT_MASK_PS_AID_4_8822B)
+
+/* 2 REG_A1_ADDR_MASK_8822B (A1 ADDR MASK REGISTER) */
+
+#define BIT_SHIFT_A1_ADDR_MASK_8822B 0
+#define BIT_MASK_A1_ADDR_MASK_8822B 0xffffffffL
+#define BIT_A1_ADDR_MASK_8822B(x)                                              \
+	(((x) & BIT_MASK_A1_ADDR_MASK_8822B) << BIT_SHIFT_A1_ADDR_MASK_8822B)
+#define BIT_GET_A1_ADDR_MASK_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_A1_ADDR_MASK_8822B) & BIT_MASK_A1_ADDR_MASK_8822B)
+
+/* 2 REG_MACID2_8822B (MAC ID2 REGISTER) */
+
+#define BIT_SHIFT_MACID2_8822B 0
+#define BIT_MASK_MACID2_8822B 0xffffffffffffL
+#define BIT_MACID2_8822B(x)                                                    \
+	(((x) & BIT_MASK_MACID2_8822B) << BIT_SHIFT_MACID2_8822B)
+#define BIT_GET_MACID2_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_MACID2_8822B) & BIT_MASK_MACID2_8822B)
+
+/* 2 REG_BSSID2_8822B (BSSID2 REGISTER) */
+
+#define BIT_SHIFT_BSSID2_8822B 0
+#define BIT_MASK_BSSID2_8822B 0xffffffffffffL
+#define BIT_BSSID2_8822B(x)                                                    \
+	(((x) & BIT_MASK_BSSID2_8822B) << BIT_SHIFT_BSSID2_8822B)
+#define BIT_GET_BSSID2_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_BSSID2_8822B) & BIT_MASK_BSSID2_8822B)
+
+/* 2 REG_MACID3_8822B (MAC ID3 REGISTER) */
+
+#define BIT_SHIFT_MACID3_8822B 0
+#define BIT_MASK_MACID3_8822B 0xffffffffffffL
+#define BIT_MACID3_8822B(x)                                                    \
+	(((x) & BIT_MASK_MACID3_8822B) << BIT_SHIFT_MACID3_8822B)
+#define BIT_GET_MACID3_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_MACID3_8822B) & BIT_MASK_MACID3_8822B)
+
+/* 2 REG_BSSID3_8822B (BSSID3 REGISTER) */
+
+#define BIT_SHIFT_BSSID3_8822B 0
+#define BIT_MASK_BSSID3_8822B 0xffffffffffffL
+#define BIT_BSSID3_8822B(x)                                                    \
+	(((x) & BIT_MASK_BSSID3_8822B) << BIT_SHIFT_BSSID3_8822B)
+#define BIT_GET_BSSID3_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_BSSID3_8822B) & BIT_MASK_BSSID3_8822B)
+
+/* 2 REG_MACID4_8822B (MAC ID4 REGISTER) */
+
+#define BIT_SHIFT_MACID4_8822B 0
+#define BIT_MASK_MACID4_8822B 0xffffffffffffL
+#define BIT_MACID4_8822B(x)                                                    \
+	(((x) & BIT_MASK_MACID4_8822B) << BIT_SHIFT_MACID4_8822B)
+#define BIT_GET_MACID4_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_MACID4_8822B) & BIT_MASK_MACID4_8822B)
+
+/* 2 REG_BSSID4_8822B (BSSID4 REGISTER) */
+
+#define BIT_SHIFT_BSSID4_8822B 0
+#define BIT_MASK_BSSID4_8822B 0xffffffffffffL
+#define BIT_BSSID4_8822B(x)                                                    \
+	(((x) & BIT_MASK_BSSID4_8822B) << BIT_SHIFT_BSSID4_8822B)
+#define BIT_GET_BSSID4_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_BSSID4_8822B) & BIT_MASK_BSSID4_8822B)
+
+/* 2 REG_NOA_REPORT_8822B */
+
+/* 2 REG_PWRBIT_SETTING_8822B */
+#define BIT_CLI3_PWRBIT_OW_EN_8822B BIT(7)
+#define BIT_CLI3_PWR_ST_8822B BIT(6)
+#define BIT_CLI2_PWRBIT_OW_EN_8822B BIT(5)
+#define BIT_CLI2_PWR_ST_8822B BIT(4)
+#define BIT_CLI1_PWRBIT_OW_EN_8822B BIT(3)
+#define BIT_CLI1_PWR_ST_8822B BIT(2)
+#define BIT_CLI0_PWRBIT_OW_EN_8822B BIT(1)
+#define BIT_CLI0_PWR_ST_8822B BIT(0)
+
+/* 2 REG_WMAC_MU_BF_OPTION_8822B */
+#define BIT_WMAC_RESP_NONSTA1_DIS_8822B BIT(7)
+#define BIT_BIT_WMAC_TXMU_ACKPOLICY_EN_8822B BIT(6)
+
+#define BIT_SHIFT_WMAC_TXMU_ACKPOLICY_8822B 4
+#define BIT_MASK_WMAC_TXMU_ACKPOLICY_8822B 0x3
+#define BIT_WMAC_TXMU_ACKPOLICY_8822B(x)                                       \
+	(((x) & BIT_MASK_WMAC_TXMU_ACKPOLICY_8822B)                            \
+	 << BIT_SHIFT_WMAC_TXMU_ACKPOLICY_8822B)
+#define BIT_GET_WMAC_TXMU_ACKPOLICY_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_WMAC_TXMU_ACKPOLICY_8822B) &                        \
+	 BIT_MASK_WMAC_TXMU_ACKPOLICY_8822B)
+
+#define BIT_SHIFT_WMAC_MU_BFEE_PORT_SEL_8822B 1
+#define BIT_MASK_WMAC_MU_BFEE_PORT_SEL_8822B 0x7
+#define BIT_WMAC_MU_BFEE_PORT_SEL_8822B(x)                                     \
+	(((x) & BIT_MASK_WMAC_MU_BFEE_PORT_SEL_8822B)                          \
+	 << BIT_SHIFT_WMAC_MU_BFEE_PORT_SEL_8822B)
+#define BIT_GET_WMAC_MU_BFEE_PORT_SEL_8822B(x)                                 \
+	(((x) >> BIT_SHIFT_WMAC_MU_BFEE_PORT_SEL_8822B) &                      \
+	 BIT_MASK_WMAC_MU_BFEE_PORT_SEL_8822B)
+
+#define BIT_WMAC_MU_BFEE_DIS_8822B BIT(0)
+
+/* 2 REG_NOT_VALID_8822B */
+
+#define BIT_SHIFT_WMAC_PAUSE_BB_CLR_TH_8822B 0
+#define BIT_MASK_WMAC_PAUSE_BB_CLR_TH_8822B 0xff
+#define BIT_WMAC_PAUSE_BB_CLR_TH_8822B(x)                                      \
+	(((x) & BIT_MASK_WMAC_PAUSE_BB_CLR_TH_8822B)                           \
+	 << BIT_SHIFT_WMAC_PAUSE_BB_CLR_TH_8822B)
+#define BIT_GET_WMAC_PAUSE_BB_CLR_TH_8822B(x)                                  \
+	(((x) >> BIT_SHIFT_WMAC_PAUSE_BB_CLR_TH_8822B) &                       \
+	 BIT_MASK_WMAC_PAUSE_BB_CLR_TH_8822B)
+
+/* 2 REG_WMAC_MU_ARB_8822B */
+#define BIT_WMAC_ARB_HW_ADAPT_EN_8822B BIT(7)
+#define BIT_WMAC_ARB_SW_EN_8822B BIT(6)
+
+#define BIT_SHIFT_WMAC_ARB_SW_STATE_8822B 0
+#define BIT_MASK_WMAC_ARB_SW_STATE_8822B 0x3f
+#define BIT_WMAC_ARB_SW_STATE_8822B(x)                                         \
+	(((x) & BIT_MASK_WMAC_ARB_SW_STATE_8822B)                              \
+	 << BIT_SHIFT_WMAC_ARB_SW_STATE_8822B)
+#define BIT_GET_WMAC_ARB_SW_STATE_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_WMAC_ARB_SW_STATE_8822B) &                          \
+	 BIT_MASK_WMAC_ARB_SW_STATE_8822B)
+
+/* 2 REG_WMAC_MU_OPTION_8822B */
+
+#define BIT_SHIFT_WMAC_MU_DBGSEL_8822B 5
+#define BIT_MASK_WMAC_MU_DBGSEL_8822B 0x3
+#define BIT_WMAC_MU_DBGSEL_8822B(x)                                            \
+	(((x) & BIT_MASK_WMAC_MU_DBGSEL_8822B)                                 \
+	 << BIT_SHIFT_WMAC_MU_DBGSEL_8822B)
+#define BIT_GET_WMAC_MU_DBGSEL_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_WMAC_MU_DBGSEL_8822B) &                             \
+	 BIT_MASK_WMAC_MU_DBGSEL_8822B)
+
+#define BIT_SHIFT_WMAC_MU_CPRD_TIMEOUT_8822B 0
+#define BIT_MASK_WMAC_MU_CPRD_TIMEOUT_8822B 0x1f
+#define BIT_WMAC_MU_CPRD_TIMEOUT_8822B(x)                                      \
+	(((x) & BIT_MASK_WMAC_MU_CPRD_TIMEOUT_8822B)                           \
+	 << BIT_SHIFT_WMAC_MU_CPRD_TIMEOUT_8822B)
+#define BIT_GET_WMAC_MU_CPRD_TIMEOUT_8822B(x)                                  \
+	(((x) >> BIT_SHIFT_WMAC_MU_CPRD_TIMEOUT_8822B) &                       \
+	 BIT_MASK_WMAC_MU_CPRD_TIMEOUT_8822B)
+
+/* 2 REG_WMAC_MU_BF_CTL_8822B */
+#define BIT_WMAC_INVLD_BFPRT_CHK_8822B BIT(15)
+#define BIT_WMAC_RETXBFRPTSEQ_UPD_8822B BIT(14)
+
+#define BIT_SHIFT_WMAC_MU_BFRPTSEG_SEL_8822B 12
+#define BIT_MASK_WMAC_MU_BFRPTSEG_SEL_8822B 0x3
+#define BIT_WMAC_MU_BFRPTSEG_SEL_8822B(x)                                      \
+	(((x) & BIT_MASK_WMAC_MU_BFRPTSEG_SEL_8822B)                           \
+	 << BIT_SHIFT_WMAC_MU_BFRPTSEG_SEL_8822B)
+#define BIT_GET_WMAC_MU_BFRPTSEG_SEL_8822B(x)                                  \
+	(((x) >> BIT_SHIFT_WMAC_MU_BFRPTSEG_SEL_8822B) &                       \
+	 BIT_MASK_WMAC_MU_BFRPTSEG_SEL_8822B)
+
+#define BIT_SHIFT_WMAC_MU_BF_MYAID_8822B 0
+#define BIT_MASK_WMAC_MU_BF_MYAID_8822B 0xfff
+#define BIT_WMAC_MU_BF_MYAID_8822B(x)                                          \
+	(((x) & BIT_MASK_WMAC_MU_BF_MYAID_8822B)                               \
+	 << BIT_SHIFT_WMAC_MU_BF_MYAID_8822B)
+#define BIT_GET_WMAC_MU_BF_MYAID_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_WMAC_MU_BF_MYAID_8822B) &                           \
+	 BIT_MASK_WMAC_MU_BF_MYAID_8822B)
+
+/* 2 REG_WMAC_MU_BFRPT_PARA_8822B */
+
+#define BIT_SHIFT_BIT_BFRPT_PARA_USERID_SEL_8822B 12
+#define BIT_MASK_BIT_BFRPT_PARA_USERID_SEL_8822B 0x7
+#define BIT_BIT_BFRPT_PARA_USERID_SEL_8822B(x)                                 \
+	(((x) & BIT_MASK_BIT_BFRPT_PARA_USERID_SEL_8822B)                      \
+	 << BIT_SHIFT_BIT_BFRPT_PARA_USERID_SEL_8822B)
+#define BIT_GET_BIT_BFRPT_PARA_USERID_SEL_8822B(x)                             \
+	(((x) >> BIT_SHIFT_BIT_BFRPT_PARA_USERID_SEL_8822B) &                  \
+	 BIT_MASK_BIT_BFRPT_PARA_USERID_SEL_8822B)
+
+#define BIT_SHIFT_BFRPT_PARA_8822B 0
+#define BIT_MASK_BFRPT_PARA_8822B 0xfff
+#define BIT_BFRPT_PARA_8822B(x)                                                \
+	(((x) & BIT_MASK_BFRPT_PARA_8822B) << BIT_SHIFT_BFRPT_PARA_8822B)
+#define BIT_GET_BFRPT_PARA_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_BFRPT_PARA_8822B) & BIT_MASK_BFRPT_PARA_8822B)
+
+/* 2 REG_WMAC_ASSOCIATED_MU_BFMEE2_8822B */
+#define BIT_STATUS_BFEE2_8822B BIT(10)
+#define BIT_WMAC_MU_BFEE2_EN_8822B BIT(9)
+
+#define BIT_SHIFT_WMAC_MU_BFEE2_AID_8822B 0
+#define BIT_MASK_WMAC_MU_BFEE2_AID_8822B 0x1ff
+#define BIT_WMAC_MU_BFEE2_AID_8822B(x)                                         \
+	(((x) & BIT_MASK_WMAC_MU_BFEE2_AID_8822B)                              \
+	 << BIT_SHIFT_WMAC_MU_BFEE2_AID_8822B)
+#define BIT_GET_WMAC_MU_BFEE2_AID_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_WMAC_MU_BFEE2_AID_8822B) &                          \
+	 BIT_MASK_WMAC_MU_BFEE2_AID_8822B)
+
+/* 2 REG_WMAC_ASSOCIATED_MU_BFMEE3_8822B */
+#define BIT_STATUS_BFEE3_8822B BIT(10)
+#define BIT_WMAC_MU_BFEE3_EN_8822B BIT(9)
+
+#define BIT_SHIFT_WMAC_MU_BFEE3_AID_8822B 0
+#define BIT_MASK_WMAC_MU_BFEE3_AID_8822B 0x1ff
+#define BIT_WMAC_MU_BFEE3_AID_8822B(x)                                         \
+	(((x) & BIT_MASK_WMAC_MU_BFEE3_AID_8822B)                              \
+	 << BIT_SHIFT_WMAC_MU_BFEE3_AID_8822B)
+#define BIT_GET_WMAC_MU_BFEE3_AID_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_WMAC_MU_BFEE3_AID_8822B) &                          \
+	 BIT_MASK_WMAC_MU_BFEE3_AID_8822B)
+
+/* 2 REG_WMAC_ASSOCIATED_MU_BFMEE4_8822B */
+#define BIT_STATUS_BFEE4_8822B BIT(10)
+#define BIT_WMAC_MU_BFEE4_EN_8822B BIT(9)
+
+#define BIT_SHIFT_WMAC_MU_BFEE4_AID_8822B 0
+#define BIT_MASK_WMAC_MU_BFEE4_AID_8822B 0x1ff
+#define BIT_WMAC_MU_BFEE4_AID_8822B(x)                                         \
+	(((x) & BIT_MASK_WMAC_MU_BFEE4_AID_8822B)                              \
+	 << BIT_SHIFT_WMAC_MU_BFEE4_AID_8822B)
+#define BIT_GET_WMAC_MU_BFEE4_AID_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_WMAC_MU_BFEE4_AID_8822B) &                          \
+	 BIT_MASK_WMAC_MU_BFEE4_AID_8822B)
+
+/* 2 REG_WMAC_ASSOCIATED_MU_BFMEE5_8822B */
+#define BIT_STATUS_BFEE5_8822B BIT(10)
+#define BIT_WMAC_MU_BFEE5_EN_8822B BIT(9)
+
+#define BIT_SHIFT_WMAC_MU_BFEE5_AID_8822B 0
+#define BIT_MASK_WMAC_MU_BFEE5_AID_8822B 0x1ff
+#define BIT_WMAC_MU_BFEE5_AID_8822B(x)                                         \
+	(((x) & BIT_MASK_WMAC_MU_BFEE5_AID_8822B)                              \
+	 << BIT_SHIFT_WMAC_MU_BFEE5_AID_8822B)
+#define BIT_GET_WMAC_MU_BFEE5_AID_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_WMAC_MU_BFEE5_AID_8822B) &                          \
+	 BIT_MASK_WMAC_MU_BFEE5_AID_8822B)
+
+/* 2 REG_WMAC_ASSOCIATED_MU_BFMEE6_8822B */
+#define BIT_STATUS_BFEE6_8822B BIT(10)
+#define BIT_WMAC_MU_BFEE6_EN_8822B BIT(9)
+
+#define BIT_SHIFT_WMAC_MU_BFEE6_AID_8822B 0
+#define BIT_MASK_WMAC_MU_BFEE6_AID_8822B 0x1ff
+#define BIT_WMAC_MU_BFEE6_AID_8822B(x)                                         \
+	(((x) & BIT_MASK_WMAC_MU_BFEE6_AID_8822B)                              \
+	 << BIT_SHIFT_WMAC_MU_BFEE6_AID_8822B)
+#define BIT_GET_WMAC_MU_BFEE6_AID_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_WMAC_MU_BFEE6_AID_8822B) &                          \
+	 BIT_MASK_WMAC_MU_BFEE6_AID_8822B)
+
+/* 2 REG_WMAC_ASSOCIATED_MU_BFMEE7_8822B */
+#define BIT_BIT_STATUS_BFEE4_8822B BIT(10)
+#define BIT_WMAC_MU_BFEE7_EN_8822B BIT(9)
+
+#define BIT_SHIFT_WMAC_MU_BFEE7_AID_8822B 0
+#define BIT_MASK_WMAC_MU_BFEE7_AID_8822B 0x1ff
+#define BIT_WMAC_MU_BFEE7_AID_8822B(x)                                         \
+	(((x) & BIT_MASK_WMAC_MU_BFEE7_AID_8822B)                              \
+	 << BIT_SHIFT_WMAC_MU_BFEE7_AID_8822B)
+#define BIT_GET_WMAC_MU_BFEE7_AID_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_WMAC_MU_BFEE7_AID_8822B) &                          \
+	 BIT_MASK_WMAC_MU_BFEE7_AID_8822B)
+
+/* 2 REG_NOT_VALID_8822B */
+#define BIT_RST_ALL_COUNTER_8822B BIT(31)
+
+#define BIT_SHIFT_ABORT_RX_VBON_COUNTER_8822B 16
+#define BIT_MASK_ABORT_RX_VBON_COUNTER_8822B 0xff
+#define BIT_ABORT_RX_VBON_COUNTER_8822B(x)                                     \
+	(((x) & BIT_MASK_ABORT_RX_VBON_COUNTER_8822B)                          \
+	 << BIT_SHIFT_ABORT_RX_VBON_COUNTER_8822B)
+#define BIT_GET_ABORT_RX_VBON_COUNTER_8822B(x)                                 \
+	(((x) >> BIT_SHIFT_ABORT_RX_VBON_COUNTER_8822B) &                      \
+	 BIT_MASK_ABORT_RX_VBON_COUNTER_8822B)
+
+#define BIT_SHIFT_ABORT_RX_RDRDY_COUNTER_8822B 8
+#define BIT_MASK_ABORT_RX_RDRDY_COUNTER_8822B 0xff
+#define BIT_ABORT_RX_RDRDY_COUNTER_8822B(x)                                    \
+	(((x) & BIT_MASK_ABORT_RX_RDRDY_COUNTER_8822B)                         \
+	 << BIT_SHIFT_ABORT_RX_RDRDY_COUNTER_8822B)
+#define BIT_GET_ABORT_RX_RDRDY_COUNTER_8822B(x)                                \
+	(((x) >> BIT_SHIFT_ABORT_RX_RDRDY_COUNTER_8822B) &                     \
+	 BIT_MASK_ABORT_RX_RDRDY_COUNTER_8822B)
+
+#define BIT_SHIFT_VBON_EARLY_FALLING_COUNTER_8822B 0
+#define BIT_MASK_VBON_EARLY_FALLING_COUNTER_8822B 0xff
+#define BIT_VBON_EARLY_FALLING_COUNTER_8822B(x)                                \
+	(((x) & BIT_MASK_VBON_EARLY_FALLING_COUNTER_8822B)                     \
+	 << BIT_SHIFT_VBON_EARLY_FALLING_COUNTER_8822B)
+#define BIT_GET_VBON_EARLY_FALLING_COUNTER_8822B(x)                            \
+	(((x) >> BIT_SHIFT_VBON_EARLY_FALLING_COUNTER_8822B) &                 \
+	 BIT_MASK_VBON_EARLY_FALLING_COUNTER_8822B)
+
+/* 2 REG_NOT_VALID_8822B */
+#define BIT_WMAC_PLCP_TRX_SEL_8822B BIT(31)
+
+#define BIT_SHIFT_WMAC_PLCP_RDSIG_SEL_8822B 28
+#define BIT_MASK_WMAC_PLCP_RDSIG_SEL_8822B 0x7
+#define BIT_WMAC_PLCP_RDSIG_SEL_8822B(x)                                       \
+	(((x) & BIT_MASK_WMAC_PLCP_RDSIG_SEL_8822B)                            \
+	 << BIT_SHIFT_WMAC_PLCP_RDSIG_SEL_8822B)
+#define BIT_GET_WMAC_PLCP_RDSIG_SEL_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_WMAC_PLCP_RDSIG_SEL_8822B) &                        \
+	 BIT_MASK_WMAC_PLCP_RDSIG_SEL_8822B)
+
+#define BIT_SHIFT_WMAC_RATE_IDX_8822B 24
+#define BIT_MASK_WMAC_RATE_IDX_8822B 0xf
+#define BIT_WMAC_RATE_IDX_8822B(x)                                             \
+	(((x) & BIT_MASK_WMAC_RATE_IDX_8822B) << BIT_SHIFT_WMAC_RATE_IDX_8822B)
+#define BIT_GET_WMAC_RATE_IDX_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_WMAC_RATE_IDX_8822B) & BIT_MASK_WMAC_RATE_IDX_8822B)
+
+#define BIT_SHIFT_WMAC_PLCP_RDSIG_8822B 0
+#define BIT_MASK_WMAC_PLCP_RDSIG_8822B 0xffffff
+#define BIT_WMAC_PLCP_RDSIG_8822B(x)                                           \
+	(((x) & BIT_MASK_WMAC_PLCP_RDSIG_8822B)                                \
+	 << BIT_SHIFT_WMAC_PLCP_RDSIG_8822B)
+#define BIT_GET_WMAC_PLCP_RDSIG_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_WMAC_PLCP_RDSIG_8822B) &                            \
+	 BIT_MASK_WMAC_PLCP_RDSIG_8822B)
+
+/* 2 REG_NOT_VALID_8822B */
+#define BIT_WMAC_MUTX_IDX_8822B BIT(24)
+
+#define BIT_SHIFT_WMAC_PLCP_RDSIG_8822B 0
+#define BIT_MASK_WMAC_PLCP_RDSIG_8822B 0xffffff
+#define BIT_WMAC_PLCP_RDSIG_8822B(x)                                           \
+	(((x) & BIT_MASK_WMAC_PLCP_RDSIG_8822B)                                \
+	 << BIT_SHIFT_WMAC_PLCP_RDSIG_8822B)
+#define BIT_GET_WMAC_PLCP_RDSIG_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_WMAC_PLCP_RDSIG_8822B) &                            \
+	 BIT_MASK_WMAC_PLCP_RDSIG_8822B)
+
+/* 2 REG_TRANSMIT_ADDRSS_0_8822B (TA0 REGISTER) */
+
+#define BIT_SHIFT_TA0_8822B 0
+#define BIT_MASK_TA0_8822B 0xffffffffffffL
+#define BIT_TA0_8822B(x) (((x) & BIT_MASK_TA0_8822B) << BIT_SHIFT_TA0_8822B)
+#define BIT_GET_TA0_8822B(x) (((x) >> BIT_SHIFT_TA0_8822B) & BIT_MASK_TA0_8822B)
+
+/* 2 REG_TRANSMIT_ADDRSS_1_8822B (TA1 REGISTER) */
+
+#define BIT_SHIFT_TA1_8822B 0
+#define BIT_MASK_TA1_8822B 0xffffffffffffL
+#define BIT_TA1_8822B(x) (((x) & BIT_MASK_TA1_8822B) << BIT_SHIFT_TA1_8822B)
+#define BIT_GET_TA1_8822B(x) (((x) >> BIT_SHIFT_TA1_8822B) & BIT_MASK_TA1_8822B)
+
+/* 2 REG_TRANSMIT_ADDRSS_2_8822B (TA2 REGISTER) */
+
+#define BIT_SHIFT_TA2_8822B 0
+#define BIT_MASK_TA2_8822B 0xffffffffffffL
+#define BIT_TA2_8822B(x) (((x) & BIT_MASK_TA2_8822B) << BIT_SHIFT_TA2_8822B)
+#define BIT_GET_TA2_8822B(x) (((x) >> BIT_SHIFT_TA2_8822B) & BIT_MASK_TA2_8822B)
+
+/* 2 REG_TRANSMIT_ADDRSS_3_8822B (TA3 REGISTER) */
+
+#define BIT_SHIFT_TA3_8822B 0
+#define BIT_MASK_TA3_8822B 0xffffffffffffL
+#define BIT_TA3_8822B(x) (((x) & BIT_MASK_TA3_8822B) << BIT_SHIFT_TA3_8822B)
+#define BIT_GET_TA3_8822B(x) (((x) >> BIT_SHIFT_TA3_8822B) & BIT_MASK_TA3_8822B)
+
+/* 2 REG_TRANSMIT_ADDRSS_4_8822B (TA4 REGISTER) */
+
+#define BIT_SHIFT_TA4_8822B 0
+#define BIT_MASK_TA4_8822B 0xffffffffffffL
+#define BIT_TA4_8822B(x) (((x) & BIT_MASK_TA4_8822B) << BIT_SHIFT_TA4_8822B)
+#define BIT_GET_TA4_8822B(x) (((x) >> BIT_SHIFT_TA4_8822B) & BIT_MASK_TA4_8822B)
+
+/* 2 REG_NOT_VALID_8822B */
+
+/* 2 REG_MACID1_8822B */
+
+#define BIT_SHIFT_MACID1_8822B 0
+#define BIT_MASK_MACID1_8822B 0xffffffffffffL
+#define BIT_MACID1_8822B(x)                                                    \
+	(((x) & BIT_MASK_MACID1_8822B) << BIT_SHIFT_MACID1_8822B)
+#define BIT_GET_MACID1_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_MACID1_8822B) & BIT_MASK_MACID1_8822B)
+
+/* 2 REG_BSSID1_8822B */
+
+#define BIT_SHIFT_BSSID1_8822B 0
+#define BIT_MASK_BSSID1_8822B 0xffffffffffffL
+#define BIT_BSSID1_8822B(x)                                                    \
+	(((x) & BIT_MASK_BSSID1_8822B) << BIT_SHIFT_BSSID1_8822B)
+#define BIT_GET_BSSID1_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_BSSID1_8822B) & BIT_MASK_BSSID1_8822B)
+
+/* 2 REG_BCN_PSR_RPT1_8822B */
+
+#define BIT_SHIFT_DTIM_CNT1_8822B 24
+#define BIT_MASK_DTIM_CNT1_8822B 0xff
+#define BIT_DTIM_CNT1_8822B(x)                                                 \
+	(((x) & BIT_MASK_DTIM_CNT1_8822B) << BIT_SHIFT_DTIM_CNT1_8822B)
+#define BIT_GET_DTIM_CNT1_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_DTIM_CNT1_8822B) & BIT_MASK_DTIM_CNT1_8822B)
+
+#define BIT_SHIFT_DTIM_PERIOD1_8822B 16
+#define BIT_MASK_DTIM_PERIOD1_8822B 0xff
+#define BIT_DTIM_PERIOD1_8822B(x)                                              \
+	(((x) & BIT_MASK_DTIM_PERIOD1_8822B) << BIT_SHIFT_DTIM_PERIOD1_8822B)
+#define BIT_GET_DTIM_PERIOD1_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_DTIM_PERIOD1_8822B) & BIT_MASK_DTIM_PERIOD1_8822B)
+
+#define BIT_DTIM1_8822B BIT(15)
+#define BIT_TIM1_8822B BIT(14)
+
+#define BIT_SHIFT_PS_AID_1_8822B 0
+#define BIT_MASK_PS_AID_1_8822B 0x7ff
+#define BIT_PS_AID_1_8822B(x)                                                  \
+	(((x) & BIT_MASK_PS_AID_1_8822B) << BIT_SHIFT_PS_AID_1_8822B)
+#define BIT_GET_PS_AID_1_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_PS_AID_1_8822B) & BIT_MASK_PS_AID_1_8822B)
+
+/* 2 REG_ASSOCIATED_BFMEE_SEL_8822B */
+#define BIT_TXUSER_ID1_8822B BIT(25)
+
+#define BIT_SHIFT_AID1_8822B 16
+#define BIT_MASK_AID1_8822B 0x1ff
+#define BIT_AID1_8822B(x) (((x) & BIT_MASK_AID1_8822B) << BIT_SHIFT_AID1_8822B)
+#define BIT_GET_AID1_8822B(x)                                                  \
+	(((x) >> BIT_SHIFT_AID1_8822B) & BIT_MASK_AID1_8822B)
+
+#define BIT_TXUSER_ID0_8822B BIT(9)
+
+#define BIT_SHIFT_AID0_8822B 0
+#define BIT_MASK_AID0_8822B 0x1ff
+#define BIT_AID0_8822B(x) (((x) & BIT_MASK_AID0_8822B) << BIT_SHIFT_AID0_8822B)
+#define BIT_GET_AID0_8822B(x)                                                  \
+	(((x) >> BIT_SHIFT_AID0_8822B) & BIT_MASK_AID0_8822B)
+
+/* 2 REG_SND_PTCL_CTRL_8822B */
+
+#define BIT_SHIFT_NDP_RX_STANDBY_TIMER_8822B 24
+#define BIT_MASK_NDP_RX_STANDBY_TIMER_8822B 0xff
+#define BIT_NDP_RX_STANDBY_TIMER_8822B(x)                                      \
+	(((x) & BIT_MASK_NDP_RX_STANDBY_TIMER_8822B)                           \
+	 << BIT_SHIFT_NDP_RX_STANDBY_TIMER_8822B)
+#define BIT_GET_NDP_RX_STANDBY_TIMER_8822B(x)                                  \
+	(((x) >> BIT_SHIFT_NDP_RX_STANDBY_TIMER_8822B) &                       \
+	 BIT_MASK_NDP_RX_STANDBY_TIMER_8822B)
+
+#define BIT_SHIFT_CSI_RPT_OFFSET_HT_8822B 16
+#define BIT_MASK_CSI_RPT_OFFSET_HT_8822B 0xff
+#define BIT_CSI_RPT_OFFSET_HT_8822B(x)                                         \
+	(((x) & BIT_MASK_CSI_RPT_OFFSET_HT_8822B)                              \
+	 << BIT_SHIFT_CSI_RPT_OFFSET_HT_8822B)
+#define BIT_GET_CSI_RPT_OFFSET_HT_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_CSI_RPT_OFFSET_HT_8822B) &                          \
+	 BIT_MASK_CSI_RPT_OFFSET_HT_8822B)
+
+#define BIT_SHIFT_R_WMAC_VHT_CATEGORY_8822B 8
+#define BIT_MASK_R_WMAC_VHT_CATEGORY_8822B 0xff
+#define BIT_R_WMAC_VHT_CATEGORY_8822B(x)                                       \
+	(((x) & BIT_MASK_R_WMAC_VHT_CATEGORY_8822B)                            \
+	 << BIT_SHIFT_R_WMAC_VHT_CATEGORY_8822B)
+#define BIT_GET_R_WMAC_VHT_CATEGORY_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_R_WMAC_VHT_CATEGORY_8822B) &                        \
+	 BIT_MASK_R_WMAC_VHT_CATEGORY_8822B)
+
+#define BIT_R_WMAC_USE_NSTS_8822B BIT(7)
+#define BIT_R_DISABLE_CHECK_VHTSIGB_CRC_8822B BIT(6)
+#define BIT_R_DISABLE_CHECK_VHTSIGA_CRC_8822B BIT(5)
+#define BIT_R_WMAC_BFPARAM_SEL_8822B BIT(4)
+#define BIT_R_WMAC_CSISEQ_SEL_8822B BIT(3)
+#define BIT_R_WMAC_CSI_WITHHTC_EN_8822B BIT(2)
+#define BIT_R_WMAC_HT_NDPA_EN_8822B BIT(1)
+#define BIT_R_WMAC_VHT_NDPA_EN_8822B BIT(0)
+
+/* 2 REG_RX_CSI_RPT_INFO_8822B */
+
+/* 2 REG_NS_ARP_CTRL_8822B */
+#define BIT_R_WMAC_NSARP_RSPEN_8822B BIT(15)
+#define BIT_R_WMAC_NSARP_RARP_8822B BIT(9)
+#define BIT_R_WMAC_NSARP_RIPV6_8822B BIT(8)
+
+#define BIT_SHIFT_R_WMAC_NSARP_MODEN_8822B 6
+#define BIT_MASK_R_WMAC_NSARP_MODEN_8822B 0x3
+#define BIT_R_WMAC_NSARP_MODEN_8822B(x)                                        \
+	(((x) & BIT_MASK_R_WMAC_NSARP_MODEN_8822B)                             \
+	 << BIT_SHIFT_R_WMAC_NSARP_MODEN_8822B)
+#define BIT_GET_R_WMAC_NSARP_MODEN_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_R_WMAC_NSARP_MODEN_8822B) &                         \
+	 BIT_MASK_R_WMAC_NSARP_MODEN_8822B)
+
+#define BIT_SHIFT_R_WMAC_NSARP_RSPFTP_8822B 4
+#define BIT_MASK_R_WMAC_NSARP_RSPFTP_8822B 0x3
+#define BIT_R_WMAC_NSARP_RSPFTP_8822B(x)                                       \
+	(((x) & BIT_MASK_R_WMAC_NSARP_RSPFTP_8822B)                            \
+	 << BIT_SHIFT_R_WMAC_NSARP_RSPFTP_8822B)
+#define BIT_GET_R_WMAC_NSARP_RSPFTP_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_R_WMAC_NSARP_RSPFTP_8822B) &                        \
+	 BIT_MASK_R_WMAC_NSARP_RSPFTP_8822B)
+
+#define BIT_SHIFT_R_WMAC_NSARP_RSPSEC_8822B 0
+#define BIT_MASK_R_WMAC_NSARP_RSPSEC_8822B 0xf
+#define BIT_R_WMAC_NSARP_RSPSEC_8822B(x)                                       \
+	(((x) & BIT_MASK_R_WMAC_NSARP_RSPSEC_8822B)                            \
+	 << BIT_SHIFT_R_WMAC_NSARP_RSPSEC_8822B)
+#define BIT_GET_R_WMAC_NSARP_RSPSEC_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_R_WMAC_NSARP_RSPSEC_8822B) &                        \
+	 BIT_MASK_R_WMAC_NSARP_RSPSEC_8822B)
+
+/* 2 REG_NS_ARP_INFO_8822B */
+#define BIT_REQ_IS_MCNS_8822B BIT(23)
+#define BIT_REQ_IS_UCNS_8822B BIT(22)
+#define BIT_REQ_IS_USNS_8822B BIT(21)
+#define BIT_REQ_IS_ARP_8822B BIT(20)
+#define BIT_EXPRSP_MH_WITHQC_8822B BIT(19)
+
+#define BIT_SHIFT_EXPRSP_SECTYPE_8822B 16
+#define BIT_MASK_EXPRSP_SECTYPE_8822B 0x7
+#define BIT_EXPRSP_SECTYPE_8822B(x)                                            \
+	(((x) & BIT_MASK_EXPRSP_SECTYPE_8822B)                                 \
+	 << BIT_SHIFT_EXPRSP_SECTYPE_8822B)
+#define BIT_GET_EXPRSP_SECTYPE_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_EXPRSP_SECTYPE_8822B) &                             \
+	 BIT_MASK_EXPRSP_SECTYPE_8822B)
+
+#define BIT_SHIFT_EXPRSP_CHKSM_7_TO_0_8822B 8
+#define BIT_MASK_EXPRSP_CHKSM_7_TO_0_8822B 0xff
+#define BIT_EXPRSP_CHKSM_7_TO_0_8822B(x)                                       \
+	(((x) & BIT_MASK_EXPRSP_CHKSM_7_TO_0_8822B)                            \
+	 << BIT_SHIFT_EXPRSP_CHKSM_7_TO_0_8822B)
+#define BIT_GET_EXPRSP_CHKSM_7_TO_0_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_EXPRSP_CHKSM_7_TO_0_8822B) &                        \
+	 BIT_MASK_EXPRSP_CHKSM_7_TO_0_8822B)
+
+#define BIT_SHIFT_EXPRSP_CHKSM_15_TO_8_8822B 0
+#define BIT_MASK_EXPRSP_CHKSM_15_TO_8_8822B 0xff
+#define BIT_EXPRSP_CHKSM_15_TO_8_8822B(x)                                      \
+	(((x) & BIT_MASK_EXPRSP_CHKSM_15_TO_8_8822B)                           \
+	 << BIT_SHIFT_EXPRSP_CHKSM_15_TO_8_8822B)
+#define BIT_GET_EXPRSP_CHKSM_15_TO_8_8822B(x)                                  \
+	(((x) >> BIT_SHIFT_EXPRSP_CHKSM_15_TO_8_8822B) &                       \
+	 BIT_MASK_EXPRSP_CHKSM_15_TO_8_8822B)
+
+/* 2 REG_BEAMFORMING_INFO_NSARP_V1_8822B */
+
+#define BIT_SHIFT_WMAC_ARPIP_8822B 0
+#define BIT_MASK_WMAC_ARPIP_8822B 0xffffffffL
+#define BIT_WMAC_ARPIP_8822B(x)                                                \
+	(((x) & BIT_MASK_WMAC_ARPIP_8822B) << BIT_SHIFT_WMAC_ARPIP_8822B)
+#define BIT_GET_WMAC_ARPIP_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_WMAC_ARPIP_8822B) & BIT_MASK_WMAC_ARPIP_8822B)
+
+/* 2 REG_BEAMFORMING_INFO_NSARP_8822B */
+
+#define BIT_SHIFT_BEAMFORMING_INFO_8822B 0
+#define BIT_MASK_BEAMFORMING_INFO_8822B 0xffffffffL
+#define BIT_BEAMFORMING_INFO_8822B(x)                                          \
+	(((x) & BIT_MASK_BEAMFORMING_INFO_8822B)                               \
+	 << BIT_SHIFT_BEAMFORMING_INFO_8822B)
+#define BIT_GET_BEAMFORMING_INFO_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_BEAMFORMING_INFO_8822B) &                           \
+	 BIT_MASK_BEAMFORMING_INFO_8822B)
+
+/* 2 REG_NOT_VALID_8822B */
+
+#define BIT_SHIFT_R_WMAC_IPV6_MYIPAD_8822B 0
+#define BIT_MASK_R_WMAC_IPV6_MYIPAD_8822B 0xffffffffffffffffffffffffffffffffL
+#define BIT_R_WMAC_IPV6_MYIPAD_8822B(x)                                        \
+	(((x) & BIT_MASK_R_WMAC_IPV6_MYIPAD_8822B)                             \
+	 << BIT_SHIFT_R_WMAC_IPV6_MYIPAD_8822B)
+#define BIT_GET_R_WMAC_IPV6_MYIPAD_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_R_WMAC_IPV6_MYIPAD_8822B) &                         \
+	 BIT_MASK_R_WMAC_IPV6_MYIPAD_8822B)
+
+/* 2 REG_RSVD_0X740_8822B */
+
+/* 2 REG_WMAC_RTX_CTX_SUBTYPE_CFG_8822B */
+
+#define BIT_SHIFT_R_WMAC_CTX_SUBTYPE_8822B 4
+#define BIT_MASK_R_WMAC_CTX_SUBTYPE_8822B 0xf
+#define BIT_R_WMAC_CTX_SUBTYPE_8822B(x)                                        \
+	(((x) & BIT_MASK_R_WMAC_CTX_SUBTYPE_8822B)                             \
+	 << BIT_SHIFT_R_WMAC_CTX_SUBTYPE_8822B)
+#define BIT_GET_R_WMAC_CTX_SUBTYPE_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_R_WMAC_CTX_SUBTYPE_8822B) &                         \
+	 BIT_MASK_R_WMAC_CTX_SUBTYPE_8822B)
+
+#define BIT_SHIFT_R_WMAC_RTX_SUBTYPE_8822B 0
+#define BIT_MASK_R_WMAC_RTX_SUBTYPE_8822B 0xf
+#define BIT_R_WMAC_RTX_SUBTYPE_8822B(x)                                        \
+	(((x) & BIT_MASK_R_WMAC_RTX_SUBTYPE_8822B)                             \
+	 << BIT_SHIFT_R_WMAC_RTX_SUBTYPE_8822B)
+#define BIT_GET_R_WMAC_RTX_SUBTYPE_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_R_WMAC_RTX_SUBTYPE_8822B) &                         \
+	 BIT_MASK_R_WMAC_RTX_SUBTYPE_8822B)
+
+/* 2 REG_WMAC_SWAES_CFG_8822B */
+
+/* 2 REG_BT_COEX_V2_8822B */
+#define BIT_GNT_BT_POLARITY_8822B BIT(12)
+#define BIT_GNT_BT_BYPASS_PRIORITY_8822B BIT(8)
+
+#define BIT_SHIFT_TIMER_8822B 0
+#define BIT_MASK_TIMER_8822B 0xff
+#define BIT_TIMER_8822B(x)                                                     \
+	(((x) & BIT_MASK_TIMER_8822B) << BIT_SHIFT_TIMER_8822B)
+#define BIT_GET_TIMER_8822B(x)                                                 \
+	(((x) >> BIT_SHIFT_TIMER_8822B) & BIT_MASK_TIMER_8822B)
+
+/* 2 REG_BT_COEX_8822B */
+#define BIT_R_GNT_BT_RFC_SW_8822B BIT(12)
+#define BIT_R_GNT_BT_RFC_SW_EN_8822B BIT(11)
+#define BIT_R_GNT_BT_BB_SW_8822B BIT(10)
+#define BIT_R_GNT_BT_BB_SW_EN_8822B BIT(9)
+#define BIT_R_BT_CNT_THREN_8822B BIT(8)
+
+#define BIT_SHIFT_R_BT_CNT_THR_8822B 0
+#define BIT_MASK_R_BT_CNT_THR_8822B 0xff
+#define BIT_R_BT_CNT_THR_8822B(x)                                              \
+	(((x) & BIT_MASK_R_BT_CNT_THR_8822B) << BIT_SHIFT_R_BT_CNT_THR_8822B)
+#define BIT_GET_R_BT_CNT_THR_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_R_BT_CNT_THR_8822B) & BIT_MASK_R_BT_CNT_THR_8822B)
+
+/* 2 REG_WLAN_ACT_MASK_CTRL_8822B */
+#define BIT_WLRX_TER_BY_CTL_8822B BIT(43)
+#define BIT_WLRX_TER_BY_AD_8822B BIT(42)
+#define BIT_ANT_DIVERSITY_SEL_8822B BIT(41)
+#define BIT_ANTSEL_FOR_BT_CTRL_EN_8822B BIT(40)
+#define BIT_WLACT_LOW_GNTWL_EN_8822B BIT(34)
+#define BIT_WLACT_HIGH_GNTBT_EN_8822B BIT(33)
+#define BIT_NAV_UPPER_V1_8822B BIT(32)
+
+#define BIT_SHIFT_RXMYRTS_NAV_V1_8822B 8
+#define BIT_MASK_RXMYRTS_NAV_V1_8822B 0xff
+#define BIT_RXMYRTS_NAV_V1_8822B(x)                                            \
+	(((x) & BIT_MASK_RXMYRTS_NAV_V1_8822B)                                 \
+	 << BIT_SHIFT_RXMYRTS_NAV_V1_8822B)
+#define BIT_GET_RXMYRTS_NAV_V1_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_RXMYRTS_NAV_V1_8822B) &                             \
+	 BIT_MASK_RXMYRTS_NAV_V1_8822B)
+
+#define BIT_SHIFT_RTSRST_V1_8822B 0
+#define BIT_MASK_RTSRST_V1_8822B 0xff
+#define BIT_RTSRST_V1_8822B(x)                                                 \
+	(((x) & BIT_MASK_RTSRST_V1_8822B) << BIT_SHIFT_RTSRST_V1_8822B)
+#define BIT_GET_RTSRST_V1_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_RTSRST_V1_8822B) & BIT_MASK_RTSRST_V1_8822B)
+
+/* 2 REG_BT_COEX_ENHANCED_INTR_CTRL_8822B */
+
+#define BIT_SHIFT_BT_STAT_DELAY_8822B 12
+#define BIT_MASK_BT_STAT_DELAY_8822B 0xf
+#define BIT_BT_STAT_DELAY_8822B(x)                                             \
+	(((x) & BIT_MASK_BT_STAT_DELAY_8822B) << BIT_SHIFT_BT_STAT_DELAY_8822B)
+#define BIT_GET_BT_STAT_DELAY_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_BT_STAT_DELAY_8822B) & BIT_MASK_BT_STAT_DELAY_8822B)
+
+#define BIT_SHIFT_BT_TRX_INIT_DETECT_8822B 8
+#define BIT_MASK_BT_TRX_INIT_DETECT_8822B 0xf
+#define BIT_BT_TRX_INIT_DETECT_8822B(x)                                        \
+	(((x) & BIT_MASK_BT_TRX_INIT_DETECT_8822B)                             \
+	 << BIT_SHIFT_BT_TRX_INIT_DETECT_8822B)
+#define BIT_GET_BT_TRX_INIT_DETECT_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_BT_TRX_INIT_DETECT_8822B) &                         \
+	 BIT_MASK_BT_TRX_INIT_DETECT_8822B)
+
+#define BIT_SHIFT_BT_PRI_DETECT_TO_8822B 4
+#define BIT_MASK_BT_PRI_DETECT_TO_8822B 0xf
+#define BIT_BT_PRI_DETECT_TO_8822B(x)                                          \
+	(((x) & BIT_MASK_BT_PRI_DETECT_TO_8822B)                               \
+	 << BIT_SHIFT_BT_PRI_DETECT_TO_8822B)
+#define BIT_GET_BT_PRI_DETECT_TO_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_BT_PRI_DETECT_TO_8822B) &                           \
+	 BIT_MASK_BT_PRI_DETECT_TO_8822B)
+
+#define BIT_R_GRANTALL_WLMASK_8822B BIT(3)
+#define BIT_STATIS_BT_EN_8822B BIT(2)
+#define BIT_WL_ACT_MASK_ENABLE_8822B BIT(1)
+#define BIT_ENHANCED_BT_8822B BIT(0)
+
+/* 2 REG_BT_ACT_STATISTICS_8822B */
+
+#define BIT_SHIFT_STATIS_BT_LO_RX_8822B (48 & CPU_OPT_WIDTH)
+#define BIT_MASK_STATIS_BT_LO_RX_8822B 0xffff
+#define BIT_STATIS_BT_LO_RX_8822B(x)                                           \
+	(((x) & BIT_MASK_STATIS_BT_LO_RX_8822B)                                \
+	 << BIT_SHIFT_STATIS_BT_LO_RX_8822B)
+#define BIT_GET_STATIS_BT_LO_RX_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_STATIS_BT_LO_RX_8822B) &                            \
+	 BIT_MASK_STATIS_BT_LO_RX_8822B)
+
+#define BIT_SHIFT_STATIS_BT_LO_TX_8822B (32 & CPU_OPT_WIDTH)
+#define BIT_MASK_STATIS_BT_LO_TX_8822B 0xffff
+#define BIT_STATIS_BT_LO_TX_8822B(x)                                           \
+	(((x) & BIT_MASK_STATIS_BT_LO_TX_8822B)                                \
+	 << BIT_SHIFT_STATIS_BT_LO_TX_8822B)
+#define BIT_GET_STATIS_BT_LO_TX_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_STATIS_BT_LO_TX_8822B) &                            \
+	 BIT_MASK_STATIS_BT_LO_TX_8822B)
+
+#define BIT_SHIFT_STATIS_BT_HI_RX_8822B 16
+#define BIT_MASK_STATIS_BT_HI_RX_8822B 0xffff
+#define BIT_STATIS_BT_HI_RX_8822B(x)                                           \
+	(((x) & BIT_MASK_STATIS_BT_HI_RX_8822B)                                \
+	 << BIT_SHIFT_STATIS_BT_HI_RX_8822B)
+#define BIT_GET_STATIS_BT_HI_RX_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_STATIS_BT_HI_RX_8822B) &                            \
+	 BIT_MASK_STATIS_BT_HI_RX_8822B)
+
+#define BIT_SHIFT_STATIS_BT_HI_TX_8822B 0
+#define BIT_MASK_STATIS_BT_HI_TX_8822B 0xffff
+#define BIT_STATIS_BT_HI_TX_8822B(x)                                           \
+	(((x) & BIT_MASK_STATIS_BT_HI_TX_8822B)                                \
+	 << BIT_SHIFT_STATIS_BT_HI_TX_8822B)
+#define BIT_GET_STATIS_BT_HI_TX_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_STATIS_BT_HI_TX_8822B) &                            \
+	 BIT_MASK_STATIS_BT_HI_TX_8822B)
+
+/* 2 REG_BT_STATISTICS_CONTROL_REGISTER_8822B */
+
+#define BIT_SHIFT_R_BT_CMD_RPT_8822B 16
+#define BIT_MASK_R_BT_CMD_RPT_8822B 0xffff
+#define BIT_R_BT_CMD_RPT_8822B(x)                                              \
+	(((x) & BIT_MASK_R_BT_CMD_RPT_8822B) << BIT_SHIFT_R_BT_CMD_RPT_8822B)
+#define BIT_GET_R_BT_CMD_RPT_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_R_BT_CMD_RPT_8822B) & BIT_MASK_R_BT_CMD_RPT_8822B)
+
+#define BIT_SHIFT_R_RPT_FROM_BT_8822B 8
+#define BIT_MASK_R_RPT_FROM_BT_8822B 0xff
+#define BIT_R_RPT_FROM_BT_8822B(x)                                             \
+	(((x) & BIT_MASK_R_RPT_FROM_BT_8822B) << BIT_SHIFT_R_RPT_FROM_BT_8822B)
+#define BIT_GET_R_RPT_FROM_BT_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_R_RPT_FROM_BT_8822B) & BIT_MASK_R_RPT_FROM_BT_8822B)
+
+#define BIT_SHIFT_BT_HID_ISR_SET_8822B 6
+#define BIT_MASK_BT_HID_ISR_SET_8822B 0x3
+#define BIT_BT_HID_ISR_SET_8822B(x)                                            \
+	(((x) & BIT_MASK_BT_HID_ISR_SET_8822B)                                 \
+	 << BIT_SHIFT_BT_HID_ISR_SET_8822B)
+#define BIT_GET_BT_HID_ISR_SET_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_BT_HID_ISR_SET_8822B) &                             \
+	 BIT_MASK_BT_HID_ISR_SET_8822B)
+
+#define BIT_TDMA_BT_START_NOTIFY_8822B BIT(5)
+#define BIT_ENABLE_TDMA_FW_MODE_8822B BIT(4)
+#define BIT_ENABLE_PTA_TDMA_MODE_8822B BIT(3)
+#define BIT_ENABLE_COEXIST_TAB_IN_TDMA_8822B BIT(2)
+#define BIT_GPIO2_GPIO3_EXANGE_OR_NO_BT_CCA_8822B BIT(1)
+#define BIT_RTK_BT_ENABLE_8822B BIT(0)
+
+/* 2 REG_BT_STATUS_REPORT_REGISTER_8822B */
+
+#define BIT_SHIFT_BT_PROFILE_8822B 24
+#define BIT_MASK_BT_PROFILE_8822B 0xff
+#define BIT_BT_PROFILE_8822B(x)                                                \
+	(((x) & BIT_MASK_BT_PROFILE_8822B) << BIT_SHIFT_BT_PROFILE_8822B)
+#define BIT_GET_BT_PROFILE_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_BT_PROFILE_8822B) & BIT_MASK_BT_PROFILE_8822B)
+
+#define BIT_SHIFT_BT_POWER_8822B 16
+#define BIT_MASK_BT_POWER_8822B 0xff
+#define BIT_BT_POWER_8822B(x)                                                  \
+	(((x) & BIT_MASK_BT_POWER_8822B) << BIT_SHIFT_BT_POWER_8822B)
+#define BIT_GET_BT_POWER_8822B(x)                                              \
+	(((x) >> BIT_SHIFT_BT_POWER_8822B) & BIT_MASK_BT_POWER_8822B)
+
+#define BIT_SHIFT_BT_PREDECT_STATUS_8822B 8
+#define BIT_MASK_BT_PREDECT_STATUS_8822B 0xff
+#define BIT_BT_PREDECT_STATUS_8822B(x)                                         \
+	(((x) & BIT_MASK_BT_PREDECT_STATUS_8822B)                              \
+	 << BIT_SHIFT_BT_PREDECT_STATUS_8822B)
+#define BIT_GET_BT_PREDECT_STATUS_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_BT_PREDECT_STATUS_8822B) &                          \
+	 BIT_MASK_BT_PREDECT_STATUS_8822B)
+
+#define BIT_SHIFT_BT_CMD_INFO_8822B 0
+#define BIT_MASK_BT_CMD_INFO_8822B 0xff
+#define BIT_BT_CMD_INFO_8822B(x)                                               \
+	(((x) & BIT_MASK_BT_CMD_INFO_8822B) << BIT_SHIFT_BT_CMD_INFO_8822B)
+#define BIT_GET_BT_CMD_INFO_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_BT_CMD_INFO_8822B) & BIT_MASK_BT_CMD_INFO_8822B)
+
+/* 2 REG_BT_INTERRUPT_CONTROL_REGISTER_8822B */
+#define BIT_EN_MAC_NULL_PKT_NOTIFY_8822B BIT(31)
+#define BIT_EN_WLAN_RPT_AND_BT_QUERY_8822B BIT(30)
+#define BIT_EN_BT_STSTUS_RPT_8822B BIT(29)
+#define BIT_EN_BT_POWER_8822B BIT(28)
+#define BIT_EN_BT_CHANNEL_8822B BIT(27)
+#define BIT_EN_BT_SLOT_CHANGE_8822B BIT(26)
+#define BIT_EN_BT_PROFILE_OR_HID_8822B BIT(25)
+#define BIT_WLAN_RPT_NOTIFY_8822B BIT(24)
+
+#define BIT_SHIFT_WLAN_RPT_DATA_8822B 16
+#define BIT_MASK_WLAN_RPT_DATA_8822B 0xff
+#define BIT_WLAN_RPT_DATA_8822B(x)                                             \
+	(((x) & BIT_MASK_WLAN_RPT_DATA_8822B) << BIT_SHIFT_WLAN_RPT_DATA_8822B)
+#define BIT_GET_WLAN_RPT_DATA_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_WLAN_RPT_DATA_8822B) & BIT_MASK_WLAN_RPT_DATA_8822B)
+
+#define BIT_SHIFT_CMD_ID_8822B 8
+#define BIT_MASK_CMD_ID_8822B 0xff
+#define BIT_CMD_ID_8822B(x)                                                    \
+	(((x) & BIT_MASK_CMD_ID_8822B) << BIT_SHIFT_CMD_ID_8822B)
+#define BIT_GET_CMD_ID_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_CMD_ID_8822B) & BIT_MASK_CMD_ID_8822B)
+
+#define BIT_SHIFT_BT_DATA_8822B 0
+#define BIT_MASK_BT_DATA_8822B 0xff
+#define BIT_BT_DATA_8822B(x)                                                   \
+	(((x) & BIT_MASK_BT_DATA_8822B) << BIT_SHIFT_BT_DATA_8822B)
+#define BIT_GET_BT_DATA_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_BT_DATA_8822B) & BIT_MASK_BT_DATA_8822B)
+
+/* 2 REG_WLAN_REPORT_TIME_OUT_CONTROL_REGISTER_8822B */
+
+#define BIT_SHIFT_WLAN_RPT_TO_8822B 0
+#define BIT_MASK_WLAN_RPT_TO_8822B 0xff
+#define BIT_WLAN_RPT_TO_8822B(x)                                               \
+	(((x) & BIT_MASK_WLAN_RPT_TO_8822B) << BIT_SHIFT_WLAN_RPT_TO_8822B)
+#define BIT_GET_WLAN_RPT_TO_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_WLAN_RPT_TO_8822B) & BIT_MASK_WLAN_RPT_TO_8822B)
+
+/* 2 REG_BT_ISOLATION_TABLE_REGISTER_REGISTER_8822B */
+
+#define BIT_SHIFT_ISOLATION_CHK_8822B 1
+#define BIT_MASK_ISOLATION_CHK_8822B 0x7fffffffffffffffffffL
+#define BIT_ISOLATION_CHK_8822B(x)                                             \
+	(((x) & BIT_MASK_ISOLATION_CHK_8822B) << BIT_SHIFT_ISOLATION_CHK_8822B)
+#define BIT_GET_ISOLATION_CHK_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_ISOLATION_CHK_8822B) & BIT_MASK_ISOLATION_CHK_8822B)
+
+#define BIT_ISOLATION_EN_8822B BIT(0)
+
+/* 2 REG_BT_INTERRUPT_STATUS_REGISTER_8822B */
+#define BIT_BT_HID_ISR_8822B BIT(7)
+#define BIT_BT_QUERY_ISR_8822B BIT(6)
+#define BIT_MAC_NULL_PKT_NOTIFY_ISR_8822B BIT(5)
+#define BIT_WLAN_RPT_ISR_8822B BIT(4)
+#define BIT_BT_POWER_ISR_8822B BIT(3)
+#define BIT_BT_CHANNEL_ISR_8822B BIT(2)
+#define BIT_BT_SLOT_CHANGE_ISR_8822B BIT(1)
+#define BIT_BT_PROFILE_ISR_8822B BIT(0)
+
+/* 2 REG_BT_TDMA_TIME_REGISTER_8822B */
+
+#define BIT_SHIFT_BT_TIME_8822B 6
+#define BIT_MASK_BT_TIME_8822B 0x3ffffff
+#define BIT_BT_TIME_8822B(x)                                                   \
+	(((x) & BIT_MASK_BT_TIME_8822B) << BIT_SHIFT_BT_TIME_8822B)
+#define BIT_GET_BT_TIME_8822B(x)                                               \
+	(((x) >> BIT_SHIFT_BT_TIME_8822B) & BIT_MASK_BT_TIME_8822B)
+
+#define BIT_SHIFT_BT_RPT_SAMPLE_RATE_8822B 0
+#define BIT_MASK_BT_RPT_SAMPLE_RATE_8822B 0x3f
+#define BIT_BT_RPT_SAMPLE_RATE_8822B(x)                                        \
+	(((x) & BIT_MASK_BT_RPT_SAMPLE_RATE_8822B)                             \
+	 << BIT_SHIFT_BT_RPT_SAMPLE_RATE_8822B)
+#define BIT_GET_BT_RPT_SAMPLE_RATE_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_BT_RPT_SAMPLE_RATE_8822B) &                         \
+	 BIT_MASK_BT_RPT_SAMPLE_RATE_8822B)
+
+/* 2 REG_BT_ACT_REGISTER_8822B */
+
+#define BIT_SHIFT_BT_EISR_EN_8822B 16
+#define BIT_MASK_BT_EISR_EN_8822B 0xff
+#define BIT_BT_EISR_EN_8822B(x)                                                \
+	(((x) & BIT_MASK_BT_EISR_EN_8822B) << BIT_SHIFT_BT_EISR_EN_8822B)
+#define BIT_GET_BT_EISR_EN_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_BT_EISR_EN_8822B) & BIT_MASK_BT_EISR_EN_8822B)
+
+#define BIT_BT_ACT_FALLING_ISR_8822B BIT(10)
+#define BIT_BT_ACT_RISING_ISR_8822B BIT(9)
+#define BIT_TDMA_TO_ISR_8822B BIT(8)
+
+#define BIT_SHIFT_BT_CH_8822B 0
+#define BIT_MASK_BT_CH_8822B 0xff
+#define BIT_BT_CH_8822B(x)                                                     \
+	(((x) & BIT_MASK_BT_CH_8822B) << BIT_SHIFT_BT_CH_8822B)
+#define BIT_GET_BT_CH_8822B(x)                                                 \
+	(((x) >> BIT_SHIFT_BT_CH_8822B) & BIT_MASK_BT_CH_8822B)
+
+/* 2 REG_OBFF_CTRL_BASIC_8822B */
+#define BIT_OBFF_EN_V1_8822B BIT(31)
+
+#define BIT_SHIFT_OBFF_STATE_V1_8822B 28
+#define BIT_MASK_OBFF_STATE_V1_8822B 0x3
+#define BIT_OBFF_STATE_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_OBFF_STATE_V1_8822B) << BIT_SHIFT_OBFF_STATE_V1_8822B)
+#define BIT_GET_OBFF_STATE_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_OBFF_STATE_V1_8822B) & BIT_MASK_OBFF_STATE_V1_8822B)
+
+#define BIT_OBFF_ACT_RXDMA_EN_8822B BIT(27)
+#define BIT_OBFF_BLOCK_INT_EN_8822B BIT(26)
+#define BIT_OBFF_AUTOACT_EN_8822B BIT(25)
+#define BIT_OBFF_AUTOIDLE_EN_8822B BIT(24)
+
+#define BIT_SHIFT_WAKE_MAX_PLS_8822B 20
+#define BIT_MASK_WAKE_MAX_PLS_8822B 0x7
+#define BIT_WAKE_MAX_PLS_8822B(x)                                              \
+	(((x) & BIT_MASK_WAKE_MAX_PLS_8822B) << BIT_SHIFT_WAKE_MAX_PLS_8822B)
+#define BIT_GET_WAKE_MAX_PLS_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_WAKE_MAX_PLS_8822B) & BIT_MASK_WAKE_MAX_PLS_8822B)
+
+#define BIT_SHIFT_WAKE_MIN_PLS_8822B 16
+#define BIT_MASK_WAKE_MIN_PLS_8822B 0x7
+#define BIT_WAKE_MIN_PLS_8822B(x)                                              \
+	(((x) & BIT_MASK_WAKE_MIN_PLS_8822B) << BIT_SHIFT_WAKE_MIN_PLS_8822B)
+#define BIT_GET_WAKE_MIN_PLS_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_WAKE_MIN_PLS_8822B) & BIT_MASK_WAKE_MIN_PLS_8822B)
+
+#define BIT_SHIFT_WAKE_MAX_F2F_8822B 12
+#define BIT_MASK_WAKE_MAX_F2F_8822B 0x7
+#define BIT_WAKE_MAX_F2F_8822B(x)                                              \
+	(((x) & BIT_MASK_WAKE_MAX_F2F_8822B) << BIT_SHIFT_WAKE_MAX_F2F_8822B)
+#define BIT_GET_WAKE_MAX_F2F_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_WAKE_MAX_F2F_8822B) & BIT_MASK_WAKE_MAX_F2F_8822B)
+
+#define BIT_SHIFT_WAKE_MIN_F2F_8822B 8
+#define BIT_MASK_WAKE_MIN_F2F_8822B 0x7
+#define BIT_WAKE_MIN_F2F_8822B(x)                                              \
+	(((x) & BIT_MASK_WAKE_MIN_F2F_8822B) << BIT_SHIFT_WAKE_MIN_F2F_8822B)
+#define BIT_GET_WAKE_MIN_F2F_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_WAKE_MIN_F2F_8822B) & BIT_MASK_WAKE_MIN_F2F_8822B)
+
+#define BIT_APP_CPU_ACT_V1_8822B BIT(3)
+#define BIT_APP_OBFF_V1_8822B BIT(2)
+#define BIT_APP_IDLE_V1_8822B BIT(1)
+#define BIT_APP_INIT_V1_8822B BIT(0)
+
+/* 2 REG_OBFF_CTRL2_TIMER_8822B */
+
+#define BIT_SHIFT_RX_HIGH_TIMER_IDX_8822B 24
+#define BIT_MASK_RX_HIGH_TIMER_IDX_8822B 0x7
+#define BIT_RX_HIGH_TIMER_IDX_8822B(x)                                         \
+	(((x) & BIT_MASK_RX_HIGH_TIMER_IDX_8822B)                              \
+	 << BIT_SHIFT_RX_HIGH_TIMER_IDX_8822B)
+#define BIT_GET_RX_HIGH_TIMER_IDX_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_RX_HIGH_TIMER_IDX_8822B) &                          \
+	 BIT_MASK_RX_HIGH_TIMER_IDX_8822B)
+
+#define BIT_SHIFT_RX_MED_TIMER_IDX_8822B 16
+#define BIT_MASK_RX_MED_TIMER_IDX_8822B 0x7
+#define BIT_RX_MED_TIMER_IDX_8822B(x)                                          \
+	(((x) & BIT_MASK_RX_MED_TIMER_IDX_8822B)                               \
+	 << BIT_SHIFT_RX_MED_TIMER_IDX_8822B)
+#define BIT_GET_RX_MED_TIMER_IDX_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_RX_MED_TIMER_IDX_8822B) &                           \
+	 BIT_MASK_RX_MED_TIMER_IDX_8822B)
+
+#define BIT_SHIFT_RX_LOW_TIMER_IDX_8822B 8
+#define BIT_MASK_RX_LOW_TIMER_IDX_8822B 0x7
+#define BIT_RX_LOW_TIMER_IDX_8822B(x)                                          \
+	(((x) & BIT_MASK_RX_LOW_TIMER_IDX_8822B)                               \
+	 << BIT_SHIFT_RX_LOW_TIMER_IDX_8822B)
+#define BIT_GET_RX_LOW_TIMER_IDX_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_RX_LOW_TIMER_IDX_8822B) &                           \
+	 BIT_MASK_RX_LOW_TIMER_IDX_8822B)
+
+#define BIT_SHIFT_OBFF_INT_TIMER_IDX_8822B 0
+#define BIT_MASK_OBFF_INT_TIMER_IDX_8822B 0x7
+#define BIT_OBFF_INT_TIMER_IDX_8822B(x)                                        \
+	(((x) & BIT_MASK_OBFF_INT_TIMER_IDX_8822B)                             \
+	 << BIT_SHIFT_OBFF_INT_TIMER_IDX_8822B)
+#define BIT_GET_OBFF_INT_TIMER_IDX_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_OBFF_INT_TIMER_IDX_8822B) &                         \
+	 BIT_MASK_OBFF_INT_TIMER_IDX_8822B)
+
+/* 2 REG_LTR_CTRL_BASIC_8822B */
+#define BIT_LTR_EN_V1_8822B BIT(31)
+#define BIT_LTR_HW_EN_V1_8822B BIT(30)
+#define BIT_LRT_ACT_CTS_EN_8822B BIT(29)
+#define BIT_LTR_ACT_RXPKT_EN_8822B BIT(28)
+#define BIT_LTR_ACT_RXDMA_EN_8822B BIT(27)
+#define BIT_LTR_IDLE_NO_SNOOP_8822B BIT(26)
+#define BIT_SPDUP_MGTPKT_8822B BIT(25)
+#define BIT_RX_AGG_EN_8822B BIT(24)
+#define BIT_APP_LTR_ACT_8822B BIT(23)
+#define BIT_APP_LTR_IDLE_8822B BIT(22)
+
+#define BIT_SHIFT_HIGH_RATE_TRIG_SEL_8822B 20
+#define BIT_MASK_HIGH_RATE_TRIG_SEL_8822B 0x3
+#define BIT_HIGH_RATE_TRIG_SEL_8822B(x)                                        \
+	(((x) & BIT_MASK_HIGH_RATE_TRIG_SEL_8822B)                             \
+	 << BIT_SHIFT_HIGH_RATE_TRIG_SEL_8822B)
+#define BIT_GET_HIGH_RATE_TRIG_SEL_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_HIGH_RATE_TRIG_SEL_8822B) &                         \
+	 BIT_MASK_HIGH_RATE_TRIG_SEL_8822B)
+
+#define BIT_SHIFT_MED_RATE_TRIG_SEL_8822B 18
+#define BIT_MASK_MED_RATE_TRIG_SEL_8822B 0x3
+#define BIT_MED_RATE_TRIG_SEL_8822B(x)                                         \
+	(((x) & BIT_MASK_MED_RATE_TRIG_SEL_8822B)                              \
+	 << BIT_SHIFT_MED_RATE_TRIG_SEL_8822B)
+#define BIT_GET_MED_RATE_TRIG_SEL_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_MED_RATE_TRIG_SEL_8822B) &                          \
+	 BIT_MASK_MED_RATE_TRIG_SEL_8822B)
+
+#define BIT_SHIFT_LOW_RATE_TRIG_SEL_8822B 16
+#define BIT_MASK_LOW_RATE_TRIG_SEL_8822B 0x3
+#define BIT_LOW_RATE_TRIG_SEL_8822B(x)                                         \
+	(((x) & BIT_MASK_LOW_RATE_TRIG_SEL_8822B)                              \
+	 << BIT_SHIFT_LOW_RATE_TRIG_SEL_8822B)
+#define BIT_GET_LOW_RATE_TRIG_SEL_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_LOW_RATE_TRIG_SEL_8822B) &                          \
+	 BIT_MASK_LOW_RATE_TRIG_SEL_8822B)
+
+#define BIT_SHIFT_HIGH_RATE_BD_IDX_8822B 8
+#define BIT_MASK_HIGH_RATE_BD_IDX_8822B 0x7f
+#define BIT_HIGH_RATE_BD_IDX_8822B(x)                                          \
+	(((x) & BIT_MASK_HIGH_RATE_BD_IDX_8822B)                               \
+	 << BIT_SHIFT_HIGH_RATE_BD_IDX_8822B)
+#define BIT_GET_HIGH_RATE_BD_IDX_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_HIGH_RATE_BD_IDX_8822B) &                           \
+	 BIT_MASK_HIGH_RATE_BD_IDX_8822B)
+
+#define BIT_SHIFT_LOW_RATE_BD_IDX_8822B 0
+#define BIT_MASK_LOW_RATE_BD_IDX_8822B 0x7f
+#define BIT_LOW_RATE_BD_IDX_8822B(x)                                           \
+	(((x) & BIT_MASK_LOW_RATE_BD_IDX_8822B)                                \
+	 << BIT_SHIFT_LOW_RATE_BD_IDX_8822B)
+#define BIT_GET_LOW_RATE_BD_IDX_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_LOW_RATE_BD_IDX_8822B) &                            \
+	 BIT_MASK_LOW_RATE_BD_IDX_8822B)
+
+/* 2 REG_LTR_CTRL2_TIMER_THRESHOLD_8822B */
+
+#define BIT_SHIFT_RX_EMPTY_TIMER_IDX_8822B 24
+#define BIT_MASK_RX_EMPTY_TIMER_IDX_8822B 0x7
+#define BIT_RX_EMPTY_TIMER_IDX_8822B(x)                                        \
+	(((x) & BIT_MASK_RX_EMPTY_TIMER_IDX_8822B)                             \
+	 << BIT_SHIFT_RX_EMPTY_TIMER_IDX_8822B)
+#define BIT_GET_RX_EMPTY_TIMER_IDX_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_RX_EMPTY_TIMER_IDX_8822B) &                         \
+	 BIT_MASK_RX_EMPTY_TIMER_IDX_8822B)
+
+#define BIT_SHIFT_RX_AFULL_TH_IDX_8822B 20
+#define BIT_MASK_RX_AFULL_TH_IDX_8822B 0x7
+#define BIT_RX_AFULL_TH_IDX_8822B(x)                                           \
+	(((x) & BIT_MASK_RX_AFULL_TH_IDX_8822B)                                \
+	 << BIT_SHIFT_RX_AFULL_TH_IDX_8822B)
+#define BIT_GET_RX_AFULL_TH_IDX_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_RX_AFULL_TH_IDX_8822B) &                            \
+	 BIT_MASK_RX_AFULL_TH_IDX_8822B)
+
+#define BIT_SHIFT_RX_HIGH_TH_IDX_8822B 16
+#define BIT_MASK_RX_HIGH_TH_IDX_8822B 0x7
+#define BIT_RX_HIGH_TH_IDX_8822B(x)                                            \
+	(((x) & BIT_MASK_RX_HIGH_TH_IDX_8822B)                                 \
+	 << BIT_SHIFT_RX_HIGH_TH_IDX_8822B)
+#define BIT_GET_RX_HIGH_TH_IDX_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_RX_HIGH_TH_IDX_8822B) &                             \
+	 BIT_MASK_RX_HIGH_TH_IDX_8822B)
+
+#define BIT_SHIFT_RX_MED_TH_IDX_8822B 12
+#define BIT_MASK_RX_MED_TH_IDX_8822B 0x7
+#define BIT_RX_MED_TH_IDX_8822B(x)                                             \
+	(((x) & BIT_MASK_RX_MED_TH_IDX_8822B) << BIT_SHIFT_RX_MED_TH_IDX_8822B)
+#define BIT_GET_RX_MED_TH_IDX_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_RX_MED_TH_IDX_8822B) & BIT_MASK_RX_MED_TH_IDX_8822B)
+
+#define BIT_SHIFT_RX_LOW_TH_IDX_8822B 8
+#define BIT_MASK_RX_LOW_TH_IDX_8822B 0x7
+#define BIT_RX_LOW_TH_IDX_8822B(x)                                             \
+	(((x) & BIT_MASK_RX_LOW_TH_IDX_8822B) << BIT_SHIFT_RX_LOW_TH_IDX_8822B)
+#define BIT_GET_RX_LOW_TH_IDX_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_RX_LOW_TH_IDX_8822B) & BIT_MASK_RX_LOW_TH_IDX_8822B)
+
+#define BIT_SHIFT_LTR_SPACE_IDX_8822B 4
+#define BIT_MASK_LTR_SPACE_IDX_8822B 0x3
+#define BIT_LTR_SPACE_IDX_8822B(x)                                             \
+	(((x) & BIT_MASK_LTR_SPACE_IDX_8822B) << BIT_SHIFT_LTR_SPACE_IDX_8822B)
+#define BIT_GET_LTR_SPACE_IDX_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_LTR_SPACE_IDX_8822B) & BIT_MASK_LTR_SPACE_IDX_8822B)
+
+#define BIT_SHIFT_LTR_IDLE_TIMER_IDX_8822B 0
+#define BIT_MASK_LTR_IDLE_TIMER_IDX_8822B 0x7
+#define BIT_LTR_IDLE_TIMER_IDX_8822B(x)                                        \
+	(((x) & BIT_MASK_LTR_IDLE_TIMER_IDX_8822B)                             \
+	 << BIT_SHIFT_LTR_IDLE_TIMER_IDX_8822B)
+#define BIT_GET_LTR_IDLE_TIMER_IDX_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_LTR_IDLE_TIMER_IDX_8822B) &                         \
+	 BIT_MASK_LTR_IDLE_TIMER_IDX_8822B)
+
+/* 2 REG_LTR_IDLE_LATENCY_V1_8822B */
+
+#define BIT_SHIFT_LTR_IDLE_L_8822B 0
+#define BIT_MASK_LTR_IDLE_L_8822B 0xffffffffL
+#define BIT_LTR_IDLE_L_8822B(x)                                                \
+	(((x) & BIT_MASK_LTR_IDLE_L_8822B) << BIT_SHIFT_LTR_IDLE_L_8822B)
+#define BIT_GET_LTR_IDLE_L_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_LTR_IDLE_L_8822B) & BIT_MASK_LTR_IDLE_L_8822B)
+
+/* 2 REG_LTR_ACTIVE_LATENCY_V1_8822B */
+
+#define BIT_SHIFT_LTR_ACT_L_8822B 0
+#define BIT_MASK_LTR_ACT_L_8822B 0xffffffffL
+#define BIT_LTR_ACT_L_8822B(x)                                                 \
+	(((x) & BIT_MASK_LTR_ACT_L_8822B) << BIT_SHIFT_LTR_ACT_L_8822B)
+#define BIT_GET_LTR_ACT_L_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_LTR_ACT_L_8822B) & BIT_MASK_LTR_ACT_L_8822B)
+
+/* 2 REG_ANTENNA_TRAINING_CONTROL_REGISTER_8822B */
+#define BIT_APPEND_MACID_IN_RESP_EN_8822B BIT(50)
+#define BIT_ADDR2_MATCH_EN_8822B BIT(49)
+#define BIT_ANTTRN_EN_8822B BIT(48)
+
+#define BIT_SHIFT_TRAIN_STA_ADDR_8822B 0
+#define BIT_MASK_TRAIN_STA_ADDR_8822B 0xffffffffffffL
+#define BIT_TRAIN_STA_ADDR_8822B(x)                                            \
+	(((x) & BIT_MASK_TRAIN_STA_ADDR_8822B)                                 \
+	 << BIT_SHIFT_TRAIN_STA_ADDR_8822B)
+#define BIT_GET_TRAIN_STA_ADDR_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_TRAIN_STA_ADDR_8822B) &                             \
+	 BIT_MASK_TRAIN_STA_ADDR_8822B)
+
+/* 2 REG_RSVD_0X7B4_8822B */
+
+/* 2 REG_WMAC_PKTCNT_RWD_8822B */
+
+#define BIT_SHIFT_PKTCNT_BSSIDMAP_8822B 4
+#define BIT_MASK_PKTCNT_BSSIDMAP_8822B 0xf
+#define BIT_PKTCNT_BSSIDMAP_8822B(x)                                           \
+	(((x) & BIT_MASK_PKTCNT_BSSIDMAP_8822B)                                \
+	 << BIT_SHIFT_PKTCNT_BSSIDMAP_8822B)
+#define BIT_GET_PKTCNT_BSSIDMAP_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_PKTCNT_BSSIDMAP_8822B) &                            \
+	 BIT_MASK_PKTCNT_BSSIDMAP_8822B)
+
+#define BIT_PKTCNT_CNTRST_8822B BIT(1)
+#define BIT_PKTCNT_CNTEN_8822B BIT(0)
+
+/* 2 REG_WMAC_PKTCNT_CTRL_8822B */
+#define BIT_WMAC_PKTCNT_TRST_8822B BIT(9)
+#define BIT_WMAC_PKTCNT_FEN_8822B BIT(8)
+
+#define BIT_SHIFT_WMAC_PKTCNT_CFGAD_8822B 0
+#define BIT_MASK_WMAC_PKTCNT_CFGAD_8822B 0xff
+#define BIT_WMAC_PKTCNT_CFGAD_8822B(x)                                         \
+	(((x) & BIT_MASK_WMAC_PKTCNT_CFGAD_8822B)                              \
+	 << BIT_SHIFT_WMAC_PKTCNT_CFGAD_8822B)
+#define BIT_GET_WMAC_PKTCNT_CFGAD_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_WMAC_PKTCNT_CFGAD_8822B) &                          \
+	 BIT_MASK_WMAC_PKTCNT_CFGAD_8822B)
+
+/* 2 REG_IQ_DUMP_8822B */
+
+#define BIT_SHIFT_R_WMAC_MATCH_REF_MAC_8822B (64 & CPU_OPT_WIDTH)
+#define BIT_MASK_R_WMAC_MATCH_REF_MAC_8822B 0xffffffffL
+#define BIT_R_WMAC_MATCH_REF_MAC_8822B(x)                                      \
+	(((x) & BIT_MASK_R_WMAC_MATCH_REF_MAC_8822B)                           \
+	 << BIT_SHIFT_R_WMAC_MATCH_REF_MAC_8822B)
+#define BIT_GET_R_WMAC_MATCH_REF_MAC_8822B(x)                                  \
+	(((x) >> BIT_SHIFT_R_WMAC_MATCH_REF_MAC_8822B) &                       \
+	 BIT_MASK_R_WMAC_MATCH_REF_MAC_8822B)
+
+#define BIT_SHIFT_R_WMAC_MASK_LA_MAC_8822B (32 & CPU_OPT_WIDTH)
+#define BIT_MASK_R_WMAC_MASK_LA_MAC_8822B 0xffffffffL
+#define BIT_R_WMAC_MASK_LA_MAC_8822B(x)                                        \
+	(((x) & BIT_MASK_R_WMAC_MASK_LA_MAC_8822B)                             \
+	 << BIT_SHIFT_R_WMAC_MASK_LA_MAC_8822B)
+#define BIT_GET_R_WMAC_MASK_LA_MAC_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_R_WMAC_MASK_LA_MAC_8822B) &                         \
+	 BIT_MASK_R_WMAC_MASK_LA_MAC_8822B)
+
+#define BIT_SHIFT_DUMP_OK_ADDR_8822B 15
+#define BIT_MASK_DUMP_OK_ADDR_8822B 0x1ffff
+#define BIT_DUMP_OK_ADDR_8822B(x)                                              \
+	(((x) & BIT_MASK_DUMP_OK_ADDR_8822B) << BIT_SHIFT_DUMP_OK_ADDR_8822B)
+#define BIT_GET_DUMP_OK_ADDR_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_DUMP_OK_ADDR_8822B) & BIT_MASK_DUMP_OK_ADDR_8822B)
+
+#define BIT_SHIFT_R_TRIG_TIME_SEL_8822B 8
+#define BIT_MASK_R_TRIG_TIME_SEL_8822B 0x7f
+#define BIT_R_TRIG_TIME_SEL_8822B(x)                                           \
+	(((x) & BIT_MASK_R_TRIG_TIME_SEL_8822B)                                \
+	 << BIT_SHIFT_R_TRIG_TIME_SEL_8822B)
+#define BIT_GET_R_TRIG_TIME_SEL_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_R_TRIG_TIME_SEL_8822B) &                            \
+	 BIT_MASK_R_TRIG_TIME_SEL_8822B)
+
+#define BIT_SHIFT_R_MAC_TRIG_SEL_8822B 6
+#define BIT_MASK_R_MAC_TRIG_SEL_8822B 0x3
+#define BIT_R_MAC_TRIG_SEL_8822B(x)                                            \
+	(((x) & BIT_MASK_R_MAC_TRIG_SEL_8822B)                                 \
+	 << BIT_SHIFT_R_MAC_TRIG_SEL_8822B)
+#define BIT_GET_R_MAC_TRIG_SEL_8822B(x)                                        \
+	(((x) >> BIT_SHIFT_R_MAC_TRIG_SEL_8822B) &                             \
+	 BIT_MASK_R_MAC_TRIG_SEL_8822B)
+
+#define BIT_MAC_TRIG_REG_8822B BIT(5)
+
+#define BIT_SHIFT_R_LEVEL_PULSE_SEL_8822B 3
+#define BIT_MASK_R_LEVEL_PULSE_SEL_8822B 0x3
+#define BIT_R_LEVEL_PULSE_SEL_8822B(x)                                         \
+	(((x) & BIT_MASK_R_LEVEL_PULSE_SEL_8822B)                              \
+	 << BIT_SHIFT_R_LEVEL_PULSE_SEL_8822B)
+#define BIT_GET_R_LEVEL_PULSE_SEL_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_R_LEVEL_PULSE_SEL_8822B) &                          \
+	 BIT_MASK_R_LEVEL_PULSE_SEL_8822B)
+
+#define BIT_EN_LA_MAC_8822B BIT(2)
+#define BIT_R_EN_IQDUMP_8822B BIT(1)
+#define BIT_R_IQDATA_DUMP_8822B BIT(0)
+
+/* 2 REG_WMAC_FTM_CTL_8822B */
+#define BIT_RXFTM_TXACK_SC_8822B BIT(6)
+#define BIT_RXFTM_TXACK_BW_8822B BIT(5)
+#define BIT_RXFTM_EN_8822B BIT(3)
+#define BIT_RXFTMREQ_BYDRV_8822B BIT(2)
+#define BIT_RXFTMREQ_EN_8822B BIT(1)
+#define BIT_FTM_EN_8822B BIT(0)
+
+/* 2 REG_WMAC_IQ_MDPK_FUNC_8822B */
+
+/* 2 REG_WMAC_OPTION_FUNCTION_8822B */
+
+#define BIT_SHIFT_R_WMAC_RX_FIL_LEN_8822B (64 & CPU_OPT_WIDTH)
+#define BIT_MASK_R_WMAC_RX_FIL_LEN_8822B 0xffff
+#define BIT_R_WMAC_RX_FIL_LEN_8822B(x)                                         \
+	(((x) & BIT_MASK_R_WMAC_RX_FIL_LEN_8822B)                              \
+	 << BIT_SHIFT_R_WMAC_RX_FIL_LEN_8822B)
+#define BIT_GET_R_WMAC_RX_FIL_LEN_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_R_WMAC_RX_FIL_LEN_8822B) &                          \
+	 BIT_MASK_R_WMAC_RX_FIL_LEN_8822B)
+
+#define BIT_SHIFT_R_WMAC_RXFIFO_FULL_TH_8822B (56 & CPU_OPT_WIDTH)
+#define BIT_MASK_R_WMAC_RXFIFO_FULL_TH_8822B 0xff
+#define BIT_R_WMAC_RXFIFO_FULL_TH_8822B(x)                                     \
+	(((x) & BIT_MASK_R_WMAC_RXFIFO_FULL_TH_8822B)                          \
+	 << BIT_SHIFT_R_WMAC_RXFIFO_FULL_TH_8822B)
+#define BIT_GET_R_WMAC_RXFIFO_FULL_TH_8822B(x)                                 \
+	(((x) >> BIT_SHIFT_R_WMAC_RXFIFO_FULL_TH_8822B) &                      \
+	 BIT_MASK_R_WMAC_RXFIFO_FULL_TH_8822B)
+
+#define BIT_R_WMAC_RX_SYNCFIFO_SYNC_8822B BIT(55)
+#define BIT_R_WMAC_RXRST_DLY_8822B BIT(54)
+#define BIT_R_WMAC_SRCH_TXRPT_REF_DROP_8822B BIT(53)
+#define BIT_R_WMAC_SRCH_TXRPT_UA1_8822B BIT(52)
+#define BIT_R_WMAC_SRCH_TXRPT_TYPE_8822B BIT(51)
+#define BIT_R_WMAC_NDP_RST_8822B BIT(50)
+#define BIT_R_WMAC_POWINT_EN_8822B BIT(49)
+#define BIT_R_WMAC_SRCH_TXRPT_PERPKT_8822B BIT(48)
+#define BIT_R_WMAC_SRCH_TXRPT_MID_8822B BIT(47)
+#define BIT_R_WMAC_PFIN_TOEN_8822B BIT(46)
+#define BIT_R_WMAC_FIL_SECERR_8822B BIT(45)
+#define BIT_R_WMAC_FIL_CTLPKTLEN_8822B BIT(44)
+#define BIT_R_WMAC_FIL_FCTYPE_8822B BIT(43)
+#define BIT_R_WMAC_FIL_FCPROVER_8822B BIT(42)
+#define BIT_R_WMAC_PHYSTS_SNIF_8822B BIT(41)
+#define BIT_R_WMAC_PHYSTS_PLCP_8822B BIT(40)
+#define BIT_R_MAC_TCR_VBONF_RD_8822B BIT(39)
+#define BIT_R_WMAC_TCR_MPAR_NDP_8822B BIT(38)
+#define BIT_R_WMAC_NDP_FILTER_8822B BIT(37)
+#define BIT_R_WMAC_RXLEN_SEL_8822B BIT(36)
+#define BIT_R_WMAC_RXLEN_SEL1_8822B BIT(35)
+#define BIT_R_OFDM_FILTER_8822B BIT(34)
+#define BIT_R_WMAC_CHK_OFDM_LEN_8822B BIT(33)
+#define BIT_R_WMAC_CHK_CCK_LEN_8822B BIT(32)
+
+#define BIT_SHIFT_R_OFDM_LEN_8822B 26
+#define BIT_MASK_R_OFDM_LEN_8822B 0x3f
+#define BIT_R_OFDM_LEN_8822B(x)                                                \
+	(((x) & BIT_MASK_R_OFDM_LEN_8822B) << BIT_SHIFT_R_OFDM_LEN_8822B)
+#define BIT_GET_R_OFDM_LEN_8822B(x)                                            \
+	(((x) >> BIT_SHIFT_R_OFDM_LEN_8822B) & BIT_MASK_R_OFDM_LEN_8822B)
+
+#define BIT_SHIFT_R_CCK_LEN_8822B 0
+#define BIT_MASK_R_CCK_LEN_8822B 0xffff
+#define BIT_R_CCK_LEN_8822B(x)                                                 \
+	(((x) & BIT_MASK_R_CCK_LEN_8822B) << BIT_SHIFT_R_CCK_LEN_8822B)
+#define BIT_GET_R_CCK_LEN_8822B(x)                                             \
+	(((x) >> BIT_SHIFT_R_CCK_LEN_8822B) & BIT_MASK_R_CCK_LEN_8822B)
+
+/* 2 REG_RX_FILTER_FUNCTION_8822B */
+#define BIT_R_WMAC_MHRDDY_LATCH_8822B BIT(14)
+#define BIT_R_WMAC_MHRDDY_CLR_8822B BIT(13)
+#define BIT_R_RXPKTCTL_FSM_BASED_MPDURDY1_8822B BIT(12)
+#define BIT_WMAC_DIS_VHT_PLCP_CHK_MU_8822B BIT(11)
+#define BIT_R_CHK_DELIMIT_LEN_8822B BIT(10)
+#define BIT_R_REAPTER_ADDR_MATCH_8822B BIT(9)
+#define BIT_R_RXPKTCTL_FSM_BASED_MPDURDY_8822B BIT(8)
+#define BIT_R_LATCH_MACHRDY_8822B BIT(7)
+#define BIT_R_WMAC_RXFIL_REND_8822B BIT(6)
+#define BIT_R_WMAC_MPDURDY_CLR_8822B BIT(5)
+#define BIT_R_WMAC_CLRRXSEC_8822B BIT(4)
+#define BIT_R_WMAC_RXFIL_RDEL_8822B BIT(3)
+#define BIT_R_WMAC_RXFIL_FCSE_8822B BIT(2)
+#define BIT_R_WMAC_RXFIL_MESH_DEL_8822B BIT(1)
+#define BIT_R_WMAC_RXFIL_MASKM_8822B BIT(0)
+
+/* 2 REG_NDP_SIG_8822B */
+
+#define BIT_SHIFT_R_WMAC_TXNDP_SIGB_8822B 0
+#define BIT_MASK_R_WMAC_TXNDP_SIGB_8822B 0x1fffff
+#define BIT_R_WMAC_TXNDP_SIGB_8822B(x)                                         \
+	(((x) & BIT_MASK_R_WMAC_TXNDP_SIGB_8822B)                              \
+	 << BIT_SHIFT_R_WMAC_TXNDP_SIGB_8822B)
+#define BIT_GET_R_WMAC_TXNDP_SIGB_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_R_WMAC_TXNDP_SIGB_8822B) &                          \
+	 BIT_MASK_R_WMAC_TXNDP_SIGB_8822B)
+
+/* 2 REG_TXCMD_INFO_FOR_RSP_PKT_8822B */
+
+#define BIT_SHIFT_R_MAC_DEBUG_8822B (32 & CPU_OPT_WIDTH)
+#define BIT_MASK_R_MAC_DEBUG_8822B 0xffffffffL
+#define BIT_R_MAC_DEBUG_8822B(x)                                               \
+	(((x) & BIT_MASK_R_MAC_DEBUG_8822B) << BIT_SHIFT_R_MAC_DEBUG_8822B)
+#define BIT_GET_R_MAC_DEBUG_8822B(x)                                           \
+	(((x) >> BIT_SHIFT_R_MAC_DEBUG_8822B) & BIT_MASK_R_MAC_DEBUG_8822B)
+
+#define BIT_SHIFT_R_MAC_DBG_SHIFT_8822B 8
+#define BIT_MASK_R_MAC_DBG_SHIFT_8822B 0x7
+#define BIT_R_MAC_DBG_SHIFT_8822B(x)                                           \
+	(((x) & BIT_MASK_R_MAC_DBG_SHIFT_8822B)                                \
+	 << BIT_SHIFT_R_MAC_DBG_SHIFT_8822B)
+#define BIT_GET_R_MAC_DBG_SHIFT_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_R_MAC_DBG_SHIFT_8822B) &                            \
+	 BIT_MASK_R_MAC_DBG_SHIFT_8822B)
+
+#define BIT_SHIFT_R_MAC_DBG_SEL_8822B 0
+#define BIT_MASK_R_MAC_DBG_SEL_8822B 0x3
+#define BIT_R_MAC_DBG_SEL_8822B(x)                                             \
+	(((x) & BIT_MASK_R_MAC_DBG_SEL_8822B) << BIT_SHIFT_R_MAC_DBG_SEL_8822B)
+#define BIT_GET_R_MAC_DBG_SEL_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_R_MAC_DBG_SEL_8822B) & BIT_MASK_R_MAC_DBG_SEL_8822B)
+
+/* 2 REG_RTS_ADDRESS_0_8822B */
+
+/* 2 REG_RTS_ADDRESS_1_8822B */
+
+/* 2 REG__RPFM_MAP1_8822B
+ * (RX PAYLOAD FILTER MAP FRAME TYPE CONTROL REGISTER GROUP 1
+ */
+#define BIT_DATA_RPFM15EN_8822B BIT(15)
+#define BIT_DATA_RPFM14EN_8822B BIT(14)
+#define BIT_DATA_RPFM13EN_8822B BIT(13)
+#define BIT_DATA_RPFM12EN_8822B BIT(12)
+#define BIT_DATA_RPFM11EN_8822B BIT(11)
+#define BIT_DATA_RPFM10EN_8822B BIT(10)
+#define BIT_DATA_RPFM9EN_8822B BIT(9)
+#define BIT_DATA_RPFM8EN_8822B BIT(8)
+#define BIT_DATA_RPFM7EN_8822B BIT(7)
+#define BIT_DATA_RPFM6EN_8822B BIT(6)
+#define BIT_DATA_RPFM5EN_8822B BIT(5)
+#define BIT_DATA_RPFM4EN_8822B BIT(4)
+#define BIT_DATA_RPFM3EN_8822B BIT(3)
+#define BIT_DATA_RPFM2EN_8822B BIT(2)
+#define BIT_DATA_RPFM1EN_8822B BIT(1)
+#define BIT_DATA_RPFM0EN_8822B BIT(0)
+
+/* 2 REG_WL2LTECOEX_INDIRECT_ACCESS_CTRL_V1_8822B */
+#define BIT_LTECOEX_ACCESS_START_V1_8822B BIT(31)
+#define BIT_LTECOEX_WRITE_MODE_V1_8822B BIT(30)
+#define BIT_LTECOEX_READY_BIT_V1_8822B BIT(29)
+
+#define BIT_SHIFT_WRITE_BYTE_EN_V1_8822B 16
+#define BIT_MASK_WRITE_BYTE_EN_V1_8822B 0xf
+#define BIT_WRITE_BYTE_EN_V1_8822B(x)                                          \
+	(((x) & BIT_MASK_WRITE_BYTE_EN_V1_8822B)                               \
+	 << BIT_SHIFT_WRITE_BYTE_EN_V1_8822B)
+#define BIT_GET_WRITE_BYTE_EN_V1_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_WRITE_BYTE_EN_V1_8822B) &                           \
+	 BIT_MASK_WRITE_BYTE_EN_V1_8822B)
+
+#define BIT_SHIFT_LTECOEX_REG_ADDR_V1_8822B 0
+#define BIT_MASK_LTECOEX_REG_ADDR_V1_8822B 0xffff
+#define BIT_LTECOEX_REG_ADDR_V1_8822B(x)                                       \
+	(((x) & BIT_MASK_LTECOEX_REG_ADDR_V1_8822B)                            \
+	 << BIT_SHIFT_LTECOEX_REG_ADDR_V1_8822B)
+#define BIT_GET_LTECOEX_REG_ADDR_V1_8822B(x)                                   \
+	(((x) >> BIT_SHIFT_LTECOEX_REG_ADDR_V1_8822B) &                        \
+	 BIT_MASK_LTECOEX_REG_ADDR_V1_8822B)
+
+/* 2 REG_WL2LTECOEX_INDIRECT_ACCESS_WRITE_DATA_V1_8822B */
+
+#define BIT_SHIFT_LTECOEX_W_DATA_V1_8822B 0
+#define BIT_MASK_LTECOEX_W_DATA_V1_8822B 0xffffffffL
+#define BIT_LTECOEX_W_DATA_V1_8822B(x)                                         \
+	(((x) & BIT_MASK_LTECOEX_W_DATA_V1_8822B)                              \
+	 << BIT_SHIFT_LTECOEX_W_DATA_V1_8822B)
+#define BIT_GET_LTECOEX_W_DATA_V1_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_LTECOEX_W_DATA_V1_8822B) &                          \
+	 BIT_MASK_LTECOEX_W_DATA_V1_8822B)
+
+/* 2 REG_WL2LTECOEX_INDIRECT_ACCESS_READ_DATA_V1_8822B */
+
+#define BIT_SHIFT_LTECOEX_R_DATA_V1_8822B 0
+#define BIT_MASK_LTECOEX_R_DATA_V1_8822B 0xffffffffL
+#define BIT_LTECOEX_R_DATA_V1_8822B(x)                                         \
+	(((x) & BIT_MASK_LTECOEX_R_DATA_V1_8822B)                              \
+	 << BIT_SHIFT_LTECOEX_R_DATA_V1_8822B)
+#define BIT_GET_LTECOEX_R_DATA_V1_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_LTECOEX_R_DATA_V1_8822B) &                          \
+	 BIT_MASK_LTECOEX_R_DATA_V1_8822B)
+
+/* 2 REG_NOT_VALID_8822B */
+
+/* 2 REG_SDIO_TX_CTRL_8822B */
+
+#define BIT_SHIFT_SDIO_INT_TIMEOUT_8822B 16
+#define BIT_MASK_SDIO_INT_TIMEOUT_8822B 0xffff
+#define BIT_SDIO_INT_TIMEOUT_8822B(x)                                          \
+	(((x) & BIT_MASK_SDIO_INT_TIMEOUT_8822B)                               \
+	 << BIT_SHIFT_SDIO_INT_TIMEOUT_8822B)
+#define BIT_GET_SDIO_INT_TIMEOUT_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_SDIO_INT_TIMEOUT_8822B) &                           \
+	 BIT_MASK_SDIO_INT_TIMEOUT_8822B)
+
+#define BIT_IO_ERR_STATUS_8822B BIT(15)
+#define BIT_REPLY_ERRCRC_IN_DATA_8822B BIT(9)
+#define BIT_EN_CMD53_OVERLAP_8822B BIT(8)
+#define BIT_REPLY_ERR_IN_R5_8822B BIT(7)
+#define BIT_R18A_EN_8822B BIT(6)
+#define BIT_INIT_CMD_EN_8822B BIT(5)
+#define BIT_EN_RXDMA_MASK_INT_8822B BIT(2)
+#define BIT_EN_MASK_TIMER_8822B BIT(1)
+#define BIT_CMD_ERR_STOP_INT_EN_8822B BIT(0)
+
+/* 2 REG_SDIO_HIMR_8822B */
+#define BIT_SDIO_CRCERR_MSK_8822B BIT(31)
+#define BIT_SDIO_HSISR3_IND_MSK_8822B BIT(30)
+#define BIT_SDIO_HSISR2_IND_MSK_8822B BIT(29)
+#define BIT_SDIO_HEISR_IND_MSK_8822B BIT(28)
+#define BIT_SDIO_CTWEND_MSK_8822B BIT(27)
+#define BIT_SDIO_ATIMEND_E_MSK_8822B BIT(26)
+#define BIT_SDIIO_ATIMEND_MSK_8822B BIT(25)
+#define BIT_SDIO_OCPINT_MSK_8822B BIT(24)
+#define BIT_SDIO_PSTIMEOUT_MSK_8822B BIT(23)
+#define BIT_SDIO_GTINT4_MSK_8822B BIT(22)
+#define BIT_SDIO_GTINT3_MSK_8822B BIT(21)
+#define BIT_SDIO_HSISR_IND_MSK_8822B BIT(20)
+#define BIT_SDIO_CPWM2_MSK_8822B BIT(19)
+#define BIT_SDIO_CPWM1_MSK_8822B BIT(18)
+#define BIT_SDIO_C2HCMD_INT_MSK_8822B BIT(17)
+#define BIT_SDIO_BCNERLY_INT_MSK_8822B BIT(16)
+#define BIT_SDIO_TXBCNERR_MSK_8822B BIT(7)
+#define BIT_SDIO_TXBCNOK_MSK_8822B BIT(6)
+#define BIT_SDIO_RXFOVW_MSK_8822B BIT(5)
+#define BIT_SDIO_TXFOVW_MSK_8822B BIT(4)
+#define BIT_SDIO_RXERR_MSK_8822B BIT(3)
+#define BIT_SDIO_TXERR_MSK_8822B BIT(2)
+#define BIT_SDIO_AVAL_MSK_8822B BIT(1)
+#define BIT_RX_REQUEST_MSK_8822B BIT(0)
+
+/* 2 REG_SDIO_HISR_8822B */
+#define BIT_SDIO_CRCERR_8822B BIT(31)
+#define BIT_SDIO_HSISR3_IND_8822B BIT(30)
+#define BIT_SDIO_HSISR2_IND_8822B BIT(29)
+#define BIT_SDIO_HEISR_IND_8822B BIT(28)
+#define BIT_SDIO_CTWEND_8822B BIT(27)
+#define BIT_SDIO_ATIMEND_E_8822B BIT(26)
+#define BIT_SDIO_ATIMEND_8822B BIT(25)
+#define BIT_SDIO_OCPINT_8822B BIT(24)
+#define BIT_SDIO_PSTIMEOUT_8822B BIT(23)
+#define BIT_SDIO_GTINT4_8822B BIT(22)
+#define BIT_SDIO_GTINT3_8822B BIT(21)
+#define BIT_SDIO_HSISR_IND_8822B BIT(20)
+#define BIT_SDIO_CPWM2_8822B BIT(19)
+#define BIT_SDIO_CPWM1_8822B BIT(18)
+#define BIT_SDIO_C2HCMD_INT_8822B BIT(17)
+#define BIT_SDIO_BCNERLY_INT_8822B BIT(16)
+#define BIT_SDIO_TXBCNERR_8822B BIT(7)
+#define BIT_SDIO_TXBCNOK_8822B BIT(6)
+#define BIT_SDIO_RXFOVW_8822B BIT(5)
+#define BIT_SDIO_TXFOVW_8822B BIT(4)
+#define BIT_SDIO_RXERR_8822B BIT(3)
+#define BIT_SDIO_TXERR_8822B BIT(2)
+#define BIT_SDIO_AVAL_8822B BIT(1)
+#define BIT_RX_REQUEST_8822B BIT(0)
+
+/* 2 REG_SDIO_RX_REQ_LEN_8822B */
+
+#define BIT_SHIFT_RX_REQ_LEN_V1_8822B 0
+#define BIT_MASK_RX_REQ_LEN_V1_8822B 0x3ffff
+#define BIT_RX_REQ_LEN_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_RX_REQ_LEN_V1_8822B) << BIT_SHIFT_RX_REQ_LEN_V1_8822B)
+#define BIT_GET_RX_REQ_LEN_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_RX_REQ_LEN_V1_8822B) & BIT_MASK_RX_REQ_LEN_V1_8822B)
+
+/* 2 REG_SDIO_FREE_TXPG_SEQ_V1_8822B */
+
+#define BIT_SHIFT_FREE_TXPG_SEQ_8822B 0
+#define BIT_MASK_FREE_TXPG_SEQ_8822B 0xff
+#define BIT_FREE_TXPG_SEQ_8822B(x)                                             \
+	(((x) & BIT_MASK_FREE_TXPG_SEQ_8822B) << BIT_SHIFT_FREE_TXPG_SEQ_8822B)
+#define BIT_GET_FREE_TXPG_SEQ_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_FREE_TXPG_SEQ_8822B) & BIT_MASK_FREE_TXPG_SEQ_8822B)
+
+/* 2 REG_SDIO_FREE_TXPG_8822B */
+
+#define BIT_SHIFT_MID_FREEPG_V1_8822B 16
+#define BIT_MASK_MID_FREEPG_V1_8822B 0xfff
+#define BIT_MID_FREEPG_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_MID_FREEPG_V1_8822B) << BIT_SHIFT_MID_FREEPG_V1_8822B)
+#define BIT_GET_MID_FREEPG_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_MID_FREEPG_V1_8822B) & BIT_MASK_MID_FREEPG_V1_8822B)
+
+#define BIT_SHIFT_HIQ_FREEPG_V1_8822B 0
+#define BIT_MASK_HIQ_FREEPG_V1_8822B 0xfff
+#define BIT_HIQ_FREEPG_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_HIQ_FREEPG_V1_8822B) << BIT_SHIFT_HIQ_FREEPG_V1_8822B)
+#define BIT_GET_HIQ_FREEPG_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_HIQ_FREEPG_V1_8822B) & BIT_MASK_HIQ_FREEPG_V1_8822B)
+
+/* 2 REG_SDIO_FREE_TXPG2_8822B */
+
+#define BIT_SHIFT_PUB_FREEPG_V1_8822B 16
+#define BIT_MASK_PUB_FREEPG_V1_8822B 0xfff
+#define BIT_PUB_FREEPG_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_PUB_FREEPG_V1_8822B) << BIT_SHIFT_PUB_FREEPG_V1_8822B)
+#define BIT_GET_PUB_FREEPG_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_PUB_FREEPG_V1_8822B) & BIT_MASK_PUB_FREEPG_V1_8822B)
+
+#define BIT_SHIFT_LOW_FREEPG_V1_8822B 0
+#define BIT_MASK_LOW_FREEPG_V1_8822B 0xfff
+#define BIT_LOW_FREEPG_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_LOW_FREEPG_V1_8822B) << BIT_SHIFT_LOW_FREEPG_V1_8822B)
+#define BIT_GET_LOW_FREEPG_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_LOW_FREEPG_V1_8822B) & BIT_MASK_LOW_FREEPG_V1_8822B)
+
+/* 2 REG_SDIO_OQT_FREE_TXPG_V1_8822B */
+
+#define BIT_SHIFT_NOAC_OQT_FREEPG_V1_8822B 24
+#define BIT_MASK_NOAC_OQT_FREEPG_V1_8822B 0xff
+#define BIT_NOAC_OQT_FREEPG_V1_8822B(x)                                        \
+	(((x) & BIT_MASK_NOAC_OQT_FREEPG_V1_8822B)                             \
+	 << BIT_SHIFT_NOAC_OQT_FREEPG_V1_8822B)
+#define BIT_GET_NOAC_OQT_FREEPG_V1_8822B(x)                                    \
+	(((x) >> BIT_SHIFT_NOAC_OQT_FREEPG_V1_8822B) &                         \
+	 BIT_MASK_NOAC_OQT_FREEPG_V1_8822B)
+
+#define BIT_SHIFT_AC_OQT_FREEPG_V1_8822B 16
+#define BIT_MASK_AC_OQT_FREEPG_V1_8822B 0xff
+#define BIT_AC_OQT_FREEPG_V1_8822B(x)                                          \
+	(((x) & BIT_MASK_AC_OQT_FREEPG_V1_8822B)                               \
+	 << BIT_SHIFT_AC_OQT_FREEPG_V1_8822B)
+#define BIT_GET_AC_OQT_FREEPG_V1_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_AC_OQT_FREEPG_V1_8822B) &                           \
+	 BIT_MASK_AC_OQT_FREEPG_V1_8822B)
+
+#define BIT_SHIFT_EXQ_FREEPG_V1_8822B 0
+#define BIT_MASK_EXQ_FREEPG_V1_8822B 0xfff
+#define BIT_EXQ_FREEPG_V1_8822B(x)                                             \
+	(((x) & BIT_MASK_EXQ_FREEPG_V1_8822B) << BIT_SHIFT_EXQ_FREEPG_V1_8822B)
+#define BIT_GET_EXQ_FREEPG_V1_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_EXQ_FREEPG_V1_8822B) & BIT_MASK_EXQ_FREEPG_V1_8822B)
+
+/* 2 REG_SDIO_HTSFR_INFO_8822B */
+
+#define BIT_SHIFT_HTSFR1_8822B 16
+#define BIT_MASK_HTSFR1_8822B 0xffff
+#define BIT_HTSFR1_8822B(x)                                                    \
+	(((x) & BIT_MASK_HTSFR1_8822B) << BIT_SHIFT_HTSFR1_8822B)
+#define BIT_GET_HTSFR1_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_HTSFR1_8822B) & BIT_MASK_HTSFR1_8822B)
+
+#define BIT_SHIFT_HTSFR0_8822B 0
+#define BIT_MASK_HTSFR0_8822B 0xffff
+#define BIT_HTSFR0_8822B(x)                                                    \
+	(((x) & BIT_MASK_HTSFR0_8822B) << BIT_SHIFT_HTSFR0_8822B)
+#define BIT_GET_HTSFR0_8822B(x)                                                \
+	(((x) >> BIT_SHIFT_HTSFR0_8822B) & BIT_MASK_HTSFR0_8822B)
+
+/* 2 REG_SDIO_HCPWM1_V2_8822B */
+#define BIT_TOGGLING_8822B BIT(7)
+#define BIT_ACK_8822B BIT(6)
+#define BIT_SYS_CLK_8822B BIT(0)
+
+/* 2 REG_SDIO_HCPWM2_V2_8822B */
+
+/* 2 REG_SDIO_INDIRECT_REG_CFG_8822B */
+#define BIT_INDIRECT_REG_RDY_8822B BIT(20)
+#define BIT_INDIRECT_REG_R_8822B BIT(19)
+#define BIT_INDIRECT_REG_W_8822B BIT(18)
+
+#define BIT_SHIFT_INDIRECT_REG_SIZE_8822B 16
+#define BIT_MASK_INDIRECT_REG_SIZE_8822B 0x3
+#define BIT_INDIRECT_REG_SIZE_8822B(x)                                         \
+	(((x) & BIT_MASK_INDIRECT_REG_SIZE_8822B)                              \
+	 << BIT_SHIFT_INDIRECT_REG_SIZE_8822B)
+#define BIT_GET_INDIRECT_REG_SIZE_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_INDIRECT_REG_SIZE_8822B) &                          \
+	 BIT_MASK_INDIRECT_REG_SIZE_8822B)
+
+#define BIT_SHIFT_INDIRECT_REG_ADDR_8822B 0
+#define BIT_MASK_INDIRECT_REG_ADDR_8822B 0xffff
+#define BIT_INDIRECT_REG_ADDR_8822B(x)                                         \
+	(((x) & BIT_MASK_INDIRECT_REG_ADDR_8822B)                              \
+	 << BIT_SHIFT_INDIRECT_REG_ADDR_8822B)
+#define BIT_GET_INDIRECT_REG_ADDR_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_INDIRECT_REG_ADDR_8822B) &                          \
+	 BIT_MASK_INDIRECT_REG_ADDR_8822B)
+
+/* 2 REG_SDIO_INDIRECT_REG_DATA_8822B */
+
+#define BIT_SHIFT_INDIRECT_REG_DATA_8822B 0
+#define BIT_MASK_INDIRECT_REG_DATA_8822B 0xffffffffL
+#define BIT_INDIRECT_REG_DATA_8822B(x)                                         \
+	(((x) & BIT_MASK_INDIRECT_REG_DATA_8822B)                              \
+	 << BIT_SHIFT_INDIRECT_REG_DATA_8822B)
+#define BIT_GET_INDIRECT_REG_DATA_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_INDIRECT_REG_DATA_8822B) &                          \
+	 BIT_MASK_INDIRECT_REG_DATA_8822B)
+
+/* 2 REG_SDIO_H2C_8822B */
+
+#define BIT_SHIFT_SDIO_H2C_MSG_8822B 0
+#define BIT_MASK_SDIO_H2C_MSG_8822B 0xffffffffL
+#define BIT_SDIO_H2C_MSG_8822B(x)                                              \
+	(((x) & BIT_MASK_SDIO_H2C_MSG_8822B) << BIT_SHIFT_SDIO_H2C_MSG_8822B)
+#define BIT_GET_SDIO_H2C_MSG_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_SDIO_H2C_MSG_8822B) & BIT_MASK_SDIO_H2C_MSG_8822B)
+
+/* 2 REG_SDIO_C2H_8822B */
+
+#define BIT_SHIFT_SDIO_C2H_MSG_8822B 0
+#define BIT_MASK_SDIO_C2H_MSG_8822B 0xffffffffL
+#define BIT_SDIO_C2H_MSG_8822B(x)                                              \
+	(((x) & BIT_MASK_SDIO_C2H_MSG_8822B) << BIT_SHIFT_SDIO_C2H_MSG_8822B)
+#define BIT_GET_SDIO_C2H_MSG_8822B(x)                                          \
+	(((x) >> BIT_SHIFT_SDIO_C2H_MSG_8822B) & BIT_MASK_SDIO_C2H_MSG_8822B)
+
+/* 2 REG_SDIO_HRPWM1_8822B */
+#define BIT_TOGGLING_8822B BIT(7)
+#define BIT_ACK_8822B BIT(6)
+#define BIT_32K_PERMISSION_8822B BIT(0)
+
+/* 2 REG_SDIO_HRPWM2_8822B */
+
+/* 2 REG_SDIO_HPS_CLKR_8822B */
+
+/* 2 REG_SDIO_BUS_CTRL_8822B */
+#define BIT_PAD_CLK_XHGE_EN_8822B BIT(3)
+#define BIT_INTER_CLK_EN_8822B BIT(2)
+#define BIT_EN_RPT_TXCRC_8822B BIT(1)
+#define BIT_DIS_RXDMA_STS_8822B BIT(0)
+
+/* 2 REG_SDIO_HSUS_CTRL_8822B */
+#define BIT_INTR_CTRL_8822B BIT(4)
+#define BIT_SDIO_VOLTAGE_8822B BIT(3)
+#define BIT_BYPASS_INIT_8822B BIT(2)
+#define BIT_HCI_RESUME_RDY_8822B BIT(1)
+#define BIT_HCI_SUS_REQ_8822B BIT(0)
+
+/* 2 REG_SDIO_RESPONSE_TIMER_8822B */
+
+#define BIT_SHIFT_CMDIN_2RESP_TIMER_8822B 0
+#define BIT_MASK_CMDIN_2RESP_TIMER_8822B 0xffff
+#define BIT_CMDIN_2RESP_TIMER_8822B(x)                                         \
+	(((x) & BIT_MASK_CMDIN_2RESP_TIMER_8822B)                              \
+	 << BIT_SHIFT_CMDIN_2RESP_TIMER_8822B)
+#define BIT_GET_CMDIN_2RESP_TIMER_8822B(x)                                     \
+	(((x) >> BIT_SHIFT_CMDIN_2RESP_TIMER_8822B) &                          \
+	 BIT_MASK_CMDIN_2RESP_TIMER_8822B)
+
+/* 2 REG_SDIO_CMD_CRC_8822B */
+
+#define BIT_SHIFT_SDIO_CMD_CRC_V1_8822B 0
+#define BIT_MASK_SDIO_CMD_CRC_V1_8822B 0xff
+#define BIT_SDIO_CMD_CRC_V1_8822B(x)                                           \
+	(((x) & BIT_MASK_SDIO_CMD_CRC_V1_8822B)                                \
+	 << BIT_SHIFT_SDIO_CMD_CRC_V1_8822B)
+#define BIT_GET_SDIO_CMD_CRC_V1_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_SDIO_CMD_CRC_V1_8822B) &                            \
+	 BIT_MASK_SDIO_CMD_CRC_V1_8822B)
+
+/* 2 REG_SDIO_HSISR_8822B */
+#define BIT_DRV_WLAN_INT_CLR_8822B BIT(1)
+#define BIT_DRV_WLAN_INT_8822B BIT(0)
+
+/* 2 REG_SDIO_HSIMR_8822B */
+#define BIT_HISR_MASK_8822B BIT(0)
+
+/* 2 REG_SDIO_ERR_RPT_8822B */
+#define BIT_HR_FF_OVF_8822B BIT(6)
+#define BIT_HR_FF_UDN_8822B BIT(5)
+#define BIT_TXDMA_BUSY_ERR_8822B BIT(4)
+#define BIT_TXDMA_VLD_ERR_8822B BIT(3)
+#define BIT_QSEL_UNKNOWN_ERR_8822B BIT(2)
+#define BIT_QSEL_MIS_ERR_8822B BIT(1)
+#define BIT_SDIO_OVERRD_ERR_8822B BIT(0)
+
+/* 2 REG_SDIO_CMD_ERRCNT_8822B */
+
+#define BIT_SHIFT_CMD_CRC_ERR_CNT_8822B 0
+#define BIT_MASK_CMD_CRC_ERR_CNT_8822B 0xff
+#define BIT_CMD_CRC_ERR_CNT_8822B(x)                                           \
+	(((x) & BIT_MASK_CMD_CRC_ERR_CNT_8822B)                                \
+	 << BIT_SHIFT_CMD_CRC_ERR_CNT_8822B)
+#define BIT_GET_CMD_CRC_ERR_CNT_8822B(x)                                       \
+	(((x) >> BIT_SHIFT_CMD_CRC_ERR_CNT_8822B) &                            \
+	 BIT_MASK_CMD_CRC_ERR_CNT_8822B)
+
+/* 2 REG_SDIO_DATA_ERRCNT_8822B */
+
+#define BIT_SHIFT_DATA_CRC_ERR_CNT_8822B 0
+#define BIT_MASK_DATA_CRC_ERR_CNT_8822B 0xff
+#define BIT_DATA_CRC_ERR_CNT_8822B(x)                                          \
+	(((x) & BIT_MASK_DATA_CRC_ERR_CNT_8822B)                               \
+	 << BIT_SHIFT_DATA_CRC_ERR_CNT_8822B)
+#define BIT_GET_DATA_CRC_ERR_CNT_8822B(x)                                      \
+	(((x) >> BIT_SHIFT_DATA_CRC_ERR_CNT_8822B) &                           \
+	 BIT_MASK_DATA_CRC_ERR_CNT_8822B)
+
+/* 2 REG_SDIO_CMD_ERR_CONTENT_8822B */
+
+#define BIT_SHIFT_SDIO_CMD_ERR_CONTENT_8822B 0
+#define BIT_MASK_SDIO_CMD_ERR_CONTENT_8822B 0xffffffffffL
+#define BIT_SDIO_CMD_ERR_CONTENT_8822B(x)                                      \
+	(((x) & BIT_MASK_SDIO_CMD_ERR_CONTENT_8822B)                           \
+	 << BIT_SHIFT_SDIO_CMD_ERR_CONTENT_8822B)
+#define BIT_GET_SDIO_CMD_ERR_CONTENT_8822B(x)                                  \
+	(((x) >> BIT_SHIFT_SDIO_CMD_ERR_CONTENT_8822B) &                       \
+	 BIT_MASK_SDIO_CMD_ERR_CONTENT_8822B)
+
+/* 2 REG_SDIO_CRC_ERR_IDX_8822B */
+#define BIT_D3_CRC_ERR_8822B BIT(4)
+#define BIT_D2_CRC_ERR_8822B BIT(3)
+#define BIT_D1_CRC_ERR_8822B BIT(2)
+#define BIT_D0_CRC_ERR_8822B BIT(1)
+#define BIT_CMD_CRC_ERR_8822B BIT(0)
+
+/* 2 REG_SDIO_DATA_CRC_8822B */
+
+#define BIT_SHIFT_SDIO_DATA_CRC_8822B 0
+#define BIT_MASK_SDIO_DATA_CRC_8822B 0xff
+#define BIT_SDIO_DATA_CRC_8822B(x)                                             \
+	(((x) & BIT_MASK_SDIO_DATA_CRC_8822B) << BIT_SHIFT_SDIO_DATA_CRC_8822B)
+#define BIT_GET_SDIO_DATA_CRC_8822B(x)                                         \
+	(((x) >> BIT_SHIFT_SDIO_DATA_CRC_8822B) & BIT_MASK_SDIO_DATA_CRC_8822B)
+
+/* 2 REG_SDIO_DATA_REPLY_TIME_8822B */
+
+#define BIT_SHIFT_SDIO_DATA_REPLY_TIME_8822B 0
+#define BIT_MASK_SDIO_DATA_REPLY_TIME_8822B 0x7
+#define BIT_SDIO_DATA_REPLY_TIME_8822B(x)                                      \
+	(((x) & BIT_MASK_SDIO_DATA_REPLY_TIME_8822B)                           \
+	 << BIT_SHIFT_SDIO_DATA_REPLY_TIME_8822B)
+#define BIT_GET_SDIO_DATA_REPLY_TIME_8822B(x)                                  \
+	(((x) >> BIT_SHIFT_SDIO_DATA_REPLY_TIME_8822B) &                       \
+	 BIT_MASK_SDIO_DATA_REPLY_TIME_8822B)
+
+#endif
diff --git a/drivers/staging/rtlwifi/halmac/halmac_fw_info.h b/drivers/staging/rtlwifi/halmac/halmac_fw_info.h
new file mode 100644
index 000000000000..dad8be311ff2
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_fw_info.h
@@ -0,0 +1,122 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HALMAC_FW_INFO_H_
+#define _HALMAC_FW_INFO_H_
+
+#define H2C_FORMAT_VERSION 6
+
+#define H2C_ACK_HDR_CONTENT_LENGTH 8
+#define CFG_PARAMETER_ACK_CONTENT_LENGTH 16
+#define SCAN_STATUS_RPT_CONTENT_LENGTH 4
+#define C2H_DBG_HEADER_LENGTH 4
+#define C2H_DBG_CONTENT_MAX_LENGTH 228
+
+#define C2H_DBG_CONTENT_SEQ_OFFSET 1
+
+/* Rename from FW SysHalCom_Debug_RAM.h */
+#define FW_REG_H2CPKT_DONE_SEQ 0x1C8
+#define fw_reg_wow_reason 0x1C7
+
+enum halmac_data_type {
+	HALMAC_DATA_TYPE_MAC_REG = 0x00,
+	HALMAC_DATA_TYPE_BB_REG = 0x01,
+	HALMAC_DATA_TYPE_RADIO_A = 0x02,
+	HALMAC_DATA_TYPE_RADIO_B = 0x03,
+	HALMAC_DATA_TYPE_RADIO_C = 0x04,
+	HALMAC_DATA_TYPE_RADIO_D = 0x05,
+
+	HALMAC_DATA_TYPE_DRV_DEFINE_0 = 0x80,
+	HALMAC_DATA_TYPE_DRV_DEFINE_1 = 0x81,
+	HALMAC_DATA_TYPE_DRV_DEFINE_2 = 0x82,
+	HALMAC_DATA_TYPE_DRV_DEFINE_3 = 0x83,
+	HALMAC_DATA_TYPE_UNDEFINE = 0x7FFFFFFF,
+};
+
+enum halmac_packet_id {
+	HALMAC_PACKET_PROBE_REQ = 0x00,
+	HALMAC_PACKET_SYNC_BCN = 0x01,
+	HALMAC_PACKET_DISCOVERY_BCN = 0x02,
+
+	HALMAC_PACKET_UNDEFINE = 0x7FFFFFFF,
+};
+
+/* Channel Switch Action ID */
+enum halmac_cs_action_id {
+	HALMAC_CS_ACTION_NONE = 0x00,
+	HALMAC_CS_ACTIVE_SCAN = 0x01,
+	HALMAC_CS_NAN_NONMASTER_DW = 0x02,
+	HALMAC_CS_NAN_NONMASTER_NONDW = 0x03,
+	HALMAC_CS_NAN_MASTER_NONDW = 0x04,
+	HALMAC_CS_NAN_MASTER_DW = 0x05,
+
+	HALMAC_CS_ACTION_UNDEFINE = 0x7FFFFFFF,
+};
+
+/* Channel Switch Extra Action ID */
+enum halmac_cs_extra_action_id {
+	HALMAC_CS_EXTRA_ACTION_NONE = 0x00,
+	HALMAC_CS_EXTRA_UPDATE_PROBE = 0x01,
+	HALMAC_CS_EXTRA_UPDATE_BEACON = 0x02,
+
+	HALMAC_CS_EXTRA_ACTION_UNDEFINE = 0x7FFFFFFF,
+};
+
+enum halmac_h2c_return_code {
+	HALMAC_H2C_RETURN_SUCCESS = 0x00,
+	HALMAC_H2C_RETURN_CFG_ERR_LEN = 0x01,
+	HALMAC_H2C_RETURN_CFG_ERR_CMD = 0x02,
+
+	HALMAC_H2C_RETURN_EFUSE_ERR_DUMP = 0x03,
+
+	HALMAC_H2C_RETURN_DATAPACK_ERR_FULL = 0x04, /* DMEM buffer full */
+	HALMAC_H2C_RETURN_DATAPACK_ERR_ID = 0x05, /* Invalid pack id */
+
+	HALMAC_H2C_RETURN_RUN_ERR_EMPTY =
+		0x06, /* No data in dedicated buffer */
+	HALMAC_H2C_RETURN_RUN_ERR_LEN = 0x07,
+	HALMAC_H2C_RETURN_RUN_ERR_CMD = 0x08,
+	HALMAC_H2C_RETURN_RUN_ERR_ID = 0x09, /* Invalid pack id */
+
+	HALMAC_H2C_RETURN_PACKET_ERR_FULL = 0x0A, /* DMEM buffer full */
+	HALMAC_H2C_RETURN_PACKET_ERR_ID = 0x0B, /* Invalid packet id */
+
+	HALMAC_H2C_RETURN_SCAN_ERR_FULL = 0x0C, /* DMEM buffer full */
+	HALMAC_H2C_RETURN_SCAN_ERR_PHYDM = 0x0D, /* PHYDM API return fail */
+
+	HALMAC_H2C_RETURN_ORIG_ERR_ID = 0x0E, /* Invalid original H2C cmd id */
+
+	HALMAC_H2C_RETURN_UNDEFINE = 0x7FFFFFFF,
+};
+
+enum halmac_scan_report_code {
+	HALMAC_SCAN_REPORT_DONE = 0x00,
+	HALMAC_SCAN_REPORT_ERR_PHYDM = 0x01, /* PHYDM API return fail */
+	HALMAC_SCAN_REPORT_ERR_ID = 0x02, /* Invalid ActionID */
+	HALMAC_SCAN_REPORT_ERR_TX = 0x03, /* Tx RsvdPage fail */
+
+	HALMAC_SCAN_REPORT_UNDEFINE = 0x7FFFFFFF,
+};
+
+#endif
diff --git a/drivers/staging/rtlwifi/halmac/halmac_fw_offload_c2h_nic.h b/drivers/staging/rtlwifi/halmac/halmac_fw_offload_c2h_nic.h
new file mode 100644
index 000000000000..0e99967f3663
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_fw_offload_c2h_nic.h
@@ -0,0 +1,184 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HAL_FWOFFLOADC2HFORMAT_H2C_C2H_NIC_H_
+#define _HAL_FWOFFLOADC2HFORMAT_H2C_C2H_NIC_H_
+#define C2H_SUB_CMD_ID_C2H_DBG 0X00
+#define C2H_SUB_CMD_ID_BT_COEX_INFO 0X02
+#define C2H_SUB_CMD_ID_SCAN_STATUS_RPT 0X03
+#define C2H_SUB_CMD_ID_H2C_ACK_HDR 0X01
+#define C2H_SUB_CMD_ID_CFG_PARAMETER_ACK 0X01
+#define C2H_SUB_CMD_ID_BT_COEX_ACK 0X01
+#define C2H_SUB_CMD_ID_DUMP_PHYSICAL_EFUSE_ACK 0X01
+#define C2H_SUB_CMD_ID_UPDATE_PACKET_ACK 0X01
+#define C2H_SUB_CMD_ID_UPDATE_DATAPACK_ACK 0X01
+#define C2H_SUB_CMD_ID_RUN_DATAPACK_ACK 0X01
+#define C2H_SUB_CMD_ID_CHANNEL_SWITCH_ACK 0X01
+#define C2H_SUB_CMD_ID_IQK_ACK 0X01
+#define C2H_SUB_CMD_ID_POWER_TRACKING_ACK 0X01
+#define C2H_SUB_CMD_ID_PSD_ACK 0X01
+#define C2H_SUB_CMD_ID_PSD_DATA 0X04
+#define C2H_SUB_CMD_ID_EFUSE_DATA 0X05
+#define C2H_SUB_CMD_ID_IQK_DATA 0X06
+#define C2H_SUB_CMD_ID_C2H_PKT_FTM_DBG 0X07
+#define C2H_SUB_CMD_ID_C2H_PKT_FTM_2_DBG 0X08
+#define C2H_SUB_CMD_ID_C2H_PKT_FTM_3_DBG 0X09
+#define C2H_SUB_CMD_ID_C2H_PKT_FTM_4_DBG 0X0A
+#define C2H_SUB_CMD_ID_FTMACKRPT_HDL_DBG 0X0B
+#define C2H_SUB_CMD_ID_FTMC2H_RPT 0X0C
+#define C2H_SUB_CMD_ID_DRVFTMC2H_RPT 0X0D
+#define C2H_SUB_CMD_ID_C2H_PKT_FTM_5_DBG 0X0E
+#define C2H_SUB_CMD_ID_CCX_RPT 0X0F
+#define C2H_SUB_CMD_ID_C2H_PKT_NAN_RPT 0X10
+#define H2C_SUB_CMD_ID_CFG_PARAMETER_ACK SUB_CMD_ID_CFG_PARAMETER
+#define H2C_SUB_CMD_ID_BT_COEX_ACK SUB_CMD_ID_BT_COEX
+#define H2C_SUB_CMD_ID_DUMP_PHYSICAL_EFUSE_ACK SUB_CMD_ID_DUMP_PHYSICAL_EFUSE
+#define H2C_SUB_CMD_ID_UPDATE_PACKET_ACK SUB_CMD_ID_UPDATE_PACKET
+#define H2C_SUB_CMD_ID_UPDATE_DATAPACK_ACK SUB_CMD_ID_UPDATE_DATAPACK
+#define H2C_SUB_CMD_ID_RUN_DATAPACK_ACK SUB_CMD_ID_RUN_DATAPACK
+#define H2C_SUB_CMD_ID_CHANNEL_SWITCH_ACK SUB_CMD_ID_CHANNEL_SWITCH
+#define H2C_SUB_CMD_ID_IQK_ACK SUB_CMD_ID_IQK
+#define H2C_SUB_CMD_ID_POWER_TRACKING_ACK SUB_CMD_ID_POWER_TRACKING
+#define H2C_SUB_CMD_ID_PSD_ACK SUB_CMD_ID_PSD
+#define H2C_SUB_CMD_ID_CCX_RPT SUB_CMD_ID_CCX_RPT
+#define H2C_CMD_ID_CFG_PARAMETER_ACK 0XFF
+#define H2C_CMD_ID_BT_COEX_ACK 0XFF
+#define H2C_CMD_ID_DUMP_PHYSICAL_EFUSE_ACK 0XFF
+#define H2C_CMD_ID_UPDATE_PACKET_ACK 0XFF
+#define H2C_CMD_ID_UPDATE_DATAPACK_ACK 0XFF
+#define H2C_CMD_ID_RUN_DATAPACK_ACK 0XFF
+#define H2C_CMD_ID_CHANNEL_SWITCH_ACK 0XFF
+#define H2C_CMD_ID_IQK_ACK 0XFF
+#define H2C_CMD_ID_POWER_TRACKING_ACK 0XFF
+#define H2C_CMD_ID_PSD_ACK 0XFF
+#define H2C_CMD_ID_CCX_RPT 0XFF
+#define C2H_HDR_GET_CMD_ID(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 0, 8)
+#define C2H_HDR_SET_CMD_ID(__c2h, __value)                                     \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 0, 8, __value)
+#define C2H_HDR_GET_SEQ(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 8, 8)
+#define C2H_HDR_SET_SEQ(__c2h, __value)                                        \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 8, 8, __value)
+#define C2H_HDR_GET_C2H_SUB_CMD_ID(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 16, 8)
+#define C2H_HDR_SET_C2H_SUB_CMD_ID(__c2h, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 16, 8, __value)
+#define C2H_HDR_GET_LEN(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 24, 8)
+#define C2H_HDR_SET_LEN(__c2h, __value)                                        \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 24, 8, __value)
+#define C2H_DBG_GET_DBG_MSG(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X04, 0, 8)
+#define C2H_DBG_SET_DBG_MSG(__c2h, __value)                                    \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 0, 8, __value)
+#define BT_COEX_INFO_GET_DATA_START(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X04, 0, 8)
+#define BT_COEX_INFO_SET_DATA_START(__c2h, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 0, 8, __value)
+#define SCAN_STATUS_RPT_GET_H2C_RETURN_CODE(__c2h)                             \
+	LE_BITS_TO_4BYTE(__c2h + 0X04, 0, 8)
+#define SCAN_STATUS_RPT_SET_H2C_RETURN_CODE(__c2h, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 0, 8, __value)
+#define SCAN_STATUS_RPT_GET_H2C_SEQ(__c2h)                                     \
+	LE_BITS_TO_4BYTE(__c2h + 0X04, 16, 16)
+#define SCAN_STATUS_RPT_SET_H2C_SEQ(__c2h, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 16, 16, __value)
+#define H2C_ACK_HDR_GET_H2C_RETURN_CODE(__c2h)                                 \
+	LE_BITS_TO_4BYTE(__c2h + 0X04, 0, 8)
+#define H2C_ACK_HDR_SET_H2C_RETURN_CODE(__c2h, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 0, 8, __value)
+#define H2C_ACK_HDR_GET_H2C_CMD_ID(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X04, 8, 8)
+#define H2C_ACK_HDR_SET_H2C_CMD_ID(__c2h, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 8, 8, __value)
+#define H2C_ACK_HDR_GET_H2C_SUB_CMD_ID(__c2h)                                  \
+	LE_BITS_TO_4BYTE(__c2h + 0X04, 16, 16)
+#define H2C_ACK_HDR_SET_H2C_SUB_CMD_ID(__c2h, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 16, 16, __value)
+#define H2C_ACK_HDR_GET_H2C_SEQ(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X08, 0, 16)
+#define H2C_ACK_HDR_SET_H2C_SEQ(__c2h, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X08, 0, 16, __value)
+#define CFG_PARAMETER_ACK_GET_OFFSET_ACCUMULATION(__c2h)                       \
+	LE_BITS_TO_4BYTE(__c2h + 0XC, 0, 32)
+#define CFG_PARAMETER_ACK_SET_OFFSET_ACCUMULATION(__c2h, __value)              \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0XC, 0, 32, __value)
+#define CFG_PARAMETER_ACK_GET_VALUE_ACCUMULATION(__c2h)                        \
+	LE_BITS_TO_4BYTE(__c2h + 0X10, 0, 32)
+#define CFG_PARAMETER_ACK_SET_VALUE_ACCUMULATION(__c2h, __value)               \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X10, 0, 32, __value)
+#define BT_COEX_ACK_GET_DATA_START(__c2h) LE_BITS_TO_4BYTE(__c2h + 0XC, 0, 8)
+#define BT_COEX_ACK_SET_DATA_START(__c2h, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0XC, 0, 8, __value)
+#define PSD_DATA_GET_SEGMENT_ID(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X04, 0, 7)
+#define PSD_DATA_SET_SEGMENT_ID(__c2h, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 0, 7, __value)
+#define PSD_DATA_GET_END_SEGMENT(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X04, 7, 1)
+#define PSD_DATA_SET_END_SEGMENT(__c2h, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 7, 1, __value)
+#define PSD_DATA_GET_SEGMENT_SIZE(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X04, 8, 8)
+#define PSD_DATA_SET_SEGMENT_SIZE(__c2h, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 8, 8, __value)
+#define PSD_DATA_GET_TOTAL_SIZE(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X04, 16, 16)
+#define PSD_DATA_SET_TOTAL_SIZE(__c2h, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 16, 16, __value)
+#define PSD_DATA_GET_H2C_SEQ(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X8, 0, 16)
+#define PSD_DATA_SET_H2C_SEQ(__c2h, __value)                                   \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X8, 0, 16, __value)
+#define PSD_DATA_GET_DATA_START(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X8, 16, 8)
+#define PSD_DATA_SET_DATA_START(__c2h, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X8, 16, 8, __value)
+#define EFUSE_DATA_GET_SEGMENT_ID(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X04, 0, 7)
+#define EFUSE_DATA_SET_SEGMENT_ID(__c2h, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 0, 7, __value)
+#define EFUSE_DATA_GET_END_SEGMENT(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X04, 7, 1)
+#define EFUSE_DATA_SET_END_SEGMENT(__c2h, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 7, 1, __value)
+#define EFUSE_DATA_GET_SEGMENT_SIZE(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X04, 8, 8)
+#define EFUSE_DATA_SET_SEGMENT_SIZE(__c2h, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 8, 8, __value)
+#define EFUSE_DATA_GET_TOTAL_SIZE(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X04, 16, 16)
+#define EFUSE_DATA_SET_TOTAL_SIZE(__c2h, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 16, 16, __value)
+#define EFUSE_DATA_GET_H2C_SEQ(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X8, 0, 16)
+#define EFUSE_DATA_SET_H2C_SEQ(__c2h, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X8, 0, 16, __value)
+#define EFUSE_DATA_GET_DATA_START(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X8, 16, 8)
+#define EFUSE_DATA_SET_DATA_START(__c2h, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X8, 16, 8, __value)
+#define IQK_DATA_GET_SEGMENT_ID(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X04, 0, 7)
+#define IQK_DATA_SET_SEGMENT_ID(__c2h, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 0, 7, __value)
+#define IQK_DATA_GET_END_SEGMENT(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X04, 7, 1)
+#define IQK_DATA_SET_END_SEGMENT(__c2h, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 7, 1, __value)
+#define IQK_DATA_GET_SEGMENT_SIZE(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X04, 8, 8)
+#define IQK_DATA_SET_SEGMENT_SIZE(__c2h, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 8, 8, __value)
+#define IQK_DATA_GET_TOTAL_SIZE(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X04, 16, 16)
+#define IQK_DATA_SET_TOTAL_SIZE(__c2h, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 16, 16, __value)
+#define IQK_DATA_GET_H2C_SEQ(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X8, 0, 16)
+#define IQK_DATA_SET_H2C_SEQ(__c2h, __value)                                   \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X8, 0, 16, __value)
+#define IQK_DATA_GET_DATA_START(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X8, 16, 8)
+#define IQK_DATA_SET_DATA_START(__c2h, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X8, 16, 8, __value)
+#define CCX_RPT_GET_CCX_RPT(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X4, 0, 129)
+#define CCX_RPT_SET_CCX_RPT(__c2h, __value)                                    \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X4, 0, 129, __value)
+#endif
diff --git a/drivers/staging/rtlwifi/halmac/halmac_fw_offload_h2c_nic.h b/drivers/staging/rtlwifi/halmac/halmac_fw_offload_h2c_nic.h
new file mode 100644
index 000000000000..7adc3cdb38c9
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_fw_offload_h2c_nic.h
@@ -0,0 +1,515 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HAL_FWOFFLOADH2CFORMAT_H2C_C2H_NIC_H_
+#define _HAL_FWOFFLOADH2CFORMAT_H2C_C2H_NIC_H_
+#define CMD_ID_FW_OFFLOAD_H2C 0XFF
+#define CMD_ID_CHANNEL_SWITCH 0XFF
+#define CMD_ID_DUMP_PHYSICAL_EFUSE 0XFF
+#define CMD_ID_UPDATE_BEACON_PARSING_INFO 0XFF
+#define CMD_ID_CFG_PARAMETER 0XFF
+#define CMD_ID_UPDATE_DATAPACK 0XFF
+#define CMD_ID_RUN_DATAPACK 0XFF
+#define CMD_ID_DOWNLOAD_FLASH 0XFF
+#define CMD_ID_UPDATE_PACKET 0XFF
+#define CMD_ID_GENERAL_INFO 0XFF
+#define CMD_ID_IQK 0XFF
+#define CMD_ID_POWER_TRACKING 0XFF
+#define CMD_ID_PSD 0XFF
+#define CMD_ID_P2PPS 0XFF
+#define CMD_ID_BT_COEX 0XFF
+#define CMD_ID_NAN_CTRL 0XFF
+#define CMD_ID_NAN_CHANNEL_PLAN_0 0XFF
+#define CMD_ID_NAN_CHANNEL_PLAN_1 0XFF
+#define CATEGORY_H2C_CMD_HEADER 0X00
+#define CATEGORY_FW_OFFLOAD_H2C 0X01
+#define CATEGORY_CHANNEL_SWITCH 0X01
+#define CATEGORY_DUMP_PHYSICAL_EFUSE 0X01
+#define CATEGORY_UPDATE_BEACON_PARSING_INFO 0X01
+#define CATEGORY_CFG_PARAMETER 0X01
+#define CATEGORY_UPDATE_DATAPACK 0X01
+#define CATEGORY_RUN_DATAPACK 0X01
+#define CATEGORY_DOWNLOAD_FLASH 0X01
+#define CATEGORY_UPDATE_PACKET 0X01
+#define CATEGORY_GENERAL_INFO 0X01
+#define CATEGORY_IQK 0X01
+#define CATEGORY_POWER_TRACKING 0X01
+#define CATEGORY_PSD 0X01
+#define CATEGORY_P2PPS 0X01
+#define CATEGORY_BT_COEX 0X01
+#define CATEGORY_NAN_CTRL 0X01
+#define CATEGORY_NAN_CHANNEL_PLAN_0 0X01
+#define CATEGORY_NAN_CHANNEL_PLAN_1 0X01
+#define SUB_CMD_ID_CHANNEL_SWITCH 0X02
+#define SUB_CMD_ID_DUMP_PHYSICAL_EFUSE 0X03
+#define SUB_CMD_ID_UPDATE_BEACON_PARSING_INFO 0X05
+#define SUB_CMD_ID_CFG_PARAMETER 0X08
+#define SUB_CMD_ID_UPDATE_DATAPACK 0X09
+#define SUB_CMD_ID_RUN_DATAPACK 0X0A
+#define SUB_CMD_ID_DOWNLOAD_FLASH 0X0B
+#define SUB_CMD_ID_UPDATE_PACKET 0X0C
+#define SUB_CMD_ID_GENERAL_INFO 0X0D
+#define SUB_CMD_ID_IQK 0X0E
+#define SUB_CMD_ID_POWER_TRACKING 0X0F
+#define SUB_CMD_ID_PSD 0X10
+#define SUB_CMD_ID_P2PPS 0X24
+#define SUB_CMD_ID_BT_COEX 0X60
+#define SUB_CMD_ID_NAN_CTRL 0XB2
+#define SUB_CMD_ID_NAN_CHANNEL_PLAN_0 0XB4
+#define SUB_CMD_ID_NAN_CHANNEL_PLAN_1 0XB5
+#define H2C_CMD_HEADER_GET_CATEGORY(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 7)
+#define H2C_CMD_HEADER_SET_CATEGORY(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 7, __value)
+#define H2C_CMD_HEADER_GET_ACK(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 7, 1)
+#define H2C_CMD_HEADER_SET_ACK(__h2c, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 7, 1, __value)
+#define H2C_CMD_HEADER_GET_TOTAL_LEN(__h2c)                                    \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 0, 16)
+#define H2C_CMD_HEADER_SET_TOTAL_LEN(__h2c, __value)                           \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 0, 16, __value)
+#define H2C_CMD_HEADER_GET_SEQ_NUM(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X04, 16, 16)
+#define H2C_CMD_HEADER_SET_SEQ_NUM(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 16, 16, __value)
+#define FW_OFFLOAD_H2C_GET_CATEGORY(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 7)
+#define FW_OFFLOAD_H2C_SET_CATEGORY(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 7, __value)
+#define FW_OFFLOAD_H2C_GET_ACK(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 7, 1)
+#define FW_OFFLOAD_H2C_SET_ACK(__h2c, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 7, 1, __value)
+#define FW_OFFLOAD_H2C_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 8)
+#define FW_OFFLOAD_H2C_SET_CMD_ID(__h2c, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 8, __value)
+#define FW_OFFLOAD_H2C_GET_SUB_CMD_ID(__h2c)                                   \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 16)
+#define FW_OFFLOAD_H2C_SET_SUB_CMD_ID(__h2c, __value)                          \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 16, __value)
+#define FW_OFFLOAD_H2C_GET_TOTAL_LEN(__h2c)                                    \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 0, 16)
+#define FW_OFFLOAD_H2C_SET_TOTAL_LEN(__h2c, __value)                           \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 0, 16, __value)
+#define FW_OFFLOAD_H2C_GET_SEQ_NUM(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X04, 16, 16)
+#define FW_OFFLOAD_H2C_SET_SEQ_NUM(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 16, 16, __value)
+#define CHANNEL_SWITCH_GET_SWITCH_START(__h2c)                                 \
+	LE_BITS_TO_4BYTE(__h2c + 0X08, 0, 1)
+#define CHANNEL_SWITCH_SET_SWITCH_START(__h2c, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 0, 1, __value)
+#define CHANNEL_SWITCH_GET_DEST_CH_EN(__h2c)                                   \
+	LE_BITS_TO_4BYTE(__h2c + 0X08, 1, 1)
+#define CHANNEL_SWITCH_SET_DEST_CH_EN(__h2c, __value)                          \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 1, 1, __value)
+#define CHANNEL_SWITCH_GET_ABSOLUTE_TIME(__h2c)                                \
+	LE_BITS_TO_4BYTE(__h2c + 0X08, 2, 1)
+#define CHANNEL_SWITCH_SET_ABSOLUTE_TIME(__h2c, __value)                       \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 2, 1, __value)
+#define CHANNEL_SWITCH_GET_PERIODIC_OPTION(__h2c)                              \
+	LE_BITS_TO_4BYTE(__h2c + 0X08, 3, 2)
+#define CHANNEL_SWITCH_SET_PERIODIC_OPTION(__h2c, __value)                     \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 3, 2, __value)
+#define CHANNEL_SWITCH_GET_CHANNEL_INFO_LOC(__h2c)                             \
+	LE_BITS_TO_4BYTE(__h2c + 0X08, 8, 8)
+#define CHANNEL_SWITCH_SET_CHANNEL_INFO_LOC(__h2c, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 8, 8, __value)
+#define CHANNEL_SWITCH_GET_CHANNEL_NUM(__h2c)                                  \
+	LE_BITS_TO_4BYTE(__h2c + 0X08, 16, 8)
+#define CHANNEL_SWITCH_SET_CHANNEL_NUM(__h2c, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 16, 8, __value)
+#define CHANNEL_SWITCH_GET_PRI_CH_IDX(__h2c)                                   \
+	LE_BITS_TO_4BYTE(__h2c + 0X08, 24, 4)
+#define CHANNEL_SWITCH_SET_PRI_CH_IDX(__h2c, __value)                          \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 24, 4, __value)
+#define CHANNEL_SWITCH_GET_DEST_BW(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 28, 4)
+#define CHANNEL_SWITCH_SET_DEST_BW(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 28, 4, __value)
+#define CHANNEL_SWITCH_GET_DEST_CH(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X0C, 0, 8)
+#define CHANNEL_SWITCH_SET_DEST_CH(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X0C, 0, 8, __value)
+#define CHANNEL_SWITCH_GET_NORMAL_PERIOD(__h2c)                                \
+	LE_BITS_TO_4BYTE(__h2c + 0X0C, 8, 8)
+#define CHANNEL_SWITCH_SET_NORMAL_PERIOD(__h2c, __value)                       \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X0C, 8, 8, __value)
+#define CHANNEL_SWITCH_GET_SLOW_PERIOD(__h2c)                                  \
+	LE_BITS_TO_4BYTE(__h2c + 0X0C, 16, 8)
+#define CHANNEL_SWITCH_SET_SLOW_PERIOD(__h2c, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X0C, 16, 8, __value)
+#define CHANNEL_SWITCH_GET_NORMAL_CYCLE(__h2c)                                 \
+	LE_BITS_TO_4BYTE(__h2c + 0X0C, 24, 8)
+#define CHANNEL_SWITCH_SET_NORMAL_CYCLE(__h2c, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X0C, 24, 8, __value)
+#define CHANNEL_SWITCH_GET_TSF_HIGH(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X10, 0, 32)
+#define CHANNEL_SWITCH_SET_TSF_HIGH(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X10, 0, 32, __value)
+#define CHANNEL_SWITCH_GET_TSF_LOW(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X14, 0, 32)
+#define CHANNEL_SWITCH_SET_TSF_LOW(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X14, 0, 32, __value)
+#define CHANNEL_SWITCH_GET_CHANNEL_INFO_SIZE(__h2c)                            \
+	LE_BITS_TO_4BYTE(__h2c + 0X18, 0, 16)
+#define CHANNEL_SWITCH_SET_CHANNEL_INFO_SIZE(__h2c, __value)                   \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X18, 0, 16, __value)
+#define UPDATE_BEACON_PARSING_INFO_GET_FUNC_EN(__h2c)                          \
+	LE_BITS_TO_4BYTE(__h2c + 0X08, 0, 1)
+#define UPDATE_BEACON_PARSING_INFO_SET_FUNC_EN(__h2c, __value)                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 0, 1, __value)
+#define UPDATE_BEACON_PARSING_INFO_GET_SIZE_TH(__h2c)                          \
+	LE_BITS_TO_4BYTE(__h2c + 0X08, 8, 4)
+#define UPDATE_BEACON_PARSING_INFO_SET_SIZE_TH(__h2c, __value)                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 8, 4, __value)
+#define UPDATE_BEACON_PARSING_INFO_GET_TIMEOUT(__h2c)                          \
+	LE_BITS_TO_4BYTE(__h2c + 0X08, 12, 4)
+#define UPDATE_BEACON_PARSING_INFO_SET_TIMEOUT(__h2c, __value)                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 12, 4, __value)
+#define UPDATE_BEACON_PARSING_INFO_GET_IE_ID_BMP_0(__h2c)                      \
+	LE_BITS_TO_4BYTE(__h2c + 0X0C, 0, 32)
+#define UPDATE_BEACON_PARSING_INFO_SET_IE_ID_BMP_0(__h2c, __value)             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X0C, 0, 32, __value)
+#define UPDATE_BEACON_PARSING_INFO_GET_IE_ID_BMP_1(__h2c)                      \
+	LE_BITS_TO_4BYTE(__h2c + 0X10, 0, 32)
+#define UPDATE_BEACON_PARSING_INFO_SET_IE_ID_BMP_1(__h2c, __value)             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X10, 0, 32, __value)
+#define UPDATE_BEACON_PARSING_INFO_GET_IE_ID_BMP_2(__h2c)                      \
+	LE_BITS_TO_4BYTE(__h2c + 0X14, 0, 32)
+#define UPDATE_BEACON_PARSING_INFO_SET_IE_ID_BMP_2(__h2c, __value)             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X14, 0, 32, __value)
+#define UPDATE_BEACON_PARSING_INFO_GET_IE_ID_BMP_3(__h2c)                      \
+	LE_BITS_TO_4BYTE(__h2c + 0X18, 0, 32)
+#define UPDATE_BEACON_PARSING_INFO_SET_IE_ID_BMP_3(__h2c, __value)             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X18, 0, 32, __value)
+#define UPDATE_BEACON_PARSING_INFO_GET_IE_ID_BMP_4(__h2c)                      \
+	LE_BITS_TO_4BYTE(__h2c + 0X1C, 0, 32)
+#define UPDATE_BEACON_PARSING_INFO_SET_IE_ID_BMP_4(__h2c, __value)             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X1C, 0, 32, __value)
+#define CFG_PARAMETER_GET_NUM(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 0, 16)
+#define CFG_PARAMETER_SET_NUM(__h2c, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 0, 16, __value)
+#define CFG_PARAMETER_GET_INIT_CASE(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 16, 1)
+#define CFG_PARAMETER_SET_INIT_CASE(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 16, 1, __value)
+#define CFG_PARAMETER_GET_PHY_PARAMETER_LOC(__h2c)                             \
+	LE_BITS_TO_4BYTE(__h2c + 0X08, 24, 8)
+#define CFG_PARAMETER_SET_PHY_PARAMETER_LOC(__h2c, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 24, 8, __value)
+#define UPDATE_DATAPACK_GET_SIZE(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 0, 16)
+#define UPDATE_DATAPACK_SET_SIZE(__h2c, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 0, 16, __value)
+#define UPDATE_DATAPACK_GET_DATAPACK_ID(__h2c)                                 \
+	LE_BITS_TO_4BYTE(__h2c + 0X08, 16, 8)
+#define UPDATE_DATAPACK_SET_DATAPACK_ID(__h2c, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 16, 8, __value)
+#define UPDATE_DATAPACK_GET_DATAPACK_LOC(__h2c)                                \
+	LE_BITS_TO_4BYTE(__h2c + 0X08, 24, 8)
+#define UPDATE_DATAPACK_SET_DATAPACK_LOC(__h2c, __value)                       \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 24, 8, __value)
+#define UPDATE_DATAPACK_GET_DATAPACK_SEGMENT(__h2c)                            \
+	LE_BITS_TO_4BYTE(__h2c + 0X0C, 0, 8)
+#define UPDATE_DATAPACK_SET_DATAPACK_SEGMENT(__h2c, __value)                   \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X0C, 0, 8, __value)
+#define UPDATE_DATAPACK_GET_END_SEGMENT(__h2c)                                 \
+	LE_BITS_TO_4BYTE(__h2c + 0X0C, 8, 1)
+#define UPDATE_DATAPACK_SET_END_SEGMENT(__h2c, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X0C, 8, 1, __value)
+#define RUN_DATAPACK_GET_DATAPACK_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 0, 8)
+#define RUN_DATAPACK_SET_DATAPACK_ID(__h2c, __value)                           \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 0, 8, __value)
+#define DOWNLOAD_FLASH_GET_SPI_CMD(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 0, 8)
+#define DOWNLOAD_FLASH_SET_SPI_CMD(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 0, 8, __value)
+#define DOWNLOAD_FLASH_GET_LOCATION(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 8, 16)
+#define DOWNLOAD_FLASH_SET_LOCATION(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 8, 16, __value)
+#define DOWNLOAD_FLASH_GET_SIZE(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X0C, 0, 32)
+#define DOWNLOAD_FLASH_SET_SIZE(__h2c, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X0C, 0, 32, __value)
+#define DOWNLOAD_FLASH_GET_START_ADDR(__h2c)                                   \
+	LE_BITS_TO_4BYTE(__h2c + 0X10, 0, 32)
+#define DOWNLOAD_FLASH_SET_START_ADDR(__h2c, __value)                          \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X10, 0, 32, __value)
+#define UPDATE_PACKET_GET_SIZE(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 0, 16)
+#define UPDATE_PACKET_SET_SIZE(__h2c, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 0, 16, __value)
+#define UPDATE_PACKET_GET_PACKET_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 16, 8)
+#define UPDATE_PACKET_SET_PACKET_ID(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 16, 8, __value)
+#define UPDATE_PACKET_GET_PACKET_LOC(__h2c)                                    \
+	LE_BITS_TO_4BYTE(__h2c + 0X08, 24, 8)
+#define UPDATE_PACKET_SET_PACKET_LOC(__h2c, __value)                           \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 24, 8, __value)
+#define GENERAL_INFO_GET_REF_TYPE(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 0, 8)
+#define GENERAL_INFO_SET_REF_TYPE(__h2c, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 0, 8, __value)
+#define GENERAL_INFO_GET_RF_TYPE(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 8, 9)
+#define GENERAL_INFO_SET_RF_TYPE(__h2c, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 8, 9, __value)
+#define GENERAL_INFO_GET_FW_TX_BOUNDARY(__h2c)                                 \
+	LE_BITS_TO_4BYTE(__h2c + 0X08, 16, 8)
+#define GENERAL_INFO_SET_FW_TX_BOUNDARY(__h2c, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 16, 8, __value)
+#define IQK_GET_CLEAR(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 0, 1)
+#define IQK_SET_CLEAR(__h2c, __value)                                          \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 0, 1, __value)
+#define IQK_GET_SEGMENT_IQK(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 1, 1)
+#define IQK_SET_SEGMENT_IQK(__h2c, __value)                                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 1, 1, __value)
+#define POWER_TRACKING_GET_ENABLE_A(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 0, 1)
+#define POWER_TRACKING_SET_ENABLE_A(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 0, 1, __value)
+#define POWER_TRACKING_GET_ENABLE_B(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 1, 1)
+#define POWER_TRACKING_SET_ENABLE_B(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 1, 1, __value)
+#define POWER_TRACKING_GET_ENABLE_C(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 2, 1)
+#define POWER_TRACKING_SET_ENABLE_C(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 2, 1, __value)
+#define POWER_TRACKING_GET_ENABLE_D(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 3, 1)
+#define POWER_TRACKING_SET_ENABLE_D(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 3, 1, __value)
+#define POWER_TRACKING_GET_TYPE(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 4, 3)
+#define POWER_TRACKING_SET_TYPE(__h2c, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 4, 3, __value)
+#define POWER_TRACKING_GET_BBSWING_INDEX(__h2c)                                \
+	LE_BITS_TO_4BYTE(__h2c + 0X08, 8, 8)
+#define POWER_TRACKING_SET_BBSWING_INDEX(__h2c, __value)                       \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 8, 8, __value)
+#define POWER_TRACKING_GET_TX_PWR_INDEX_A(__h2c)                               \
+	LE_BITS_TO_4BYTE(__h2c + 0X0C, 0, 8)
+#define POWER_TRACKING_SET_TX_PWR_INDEX_A(__h2c, __value)                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X0C, 0, 8, __value)
+#define POWER_TRACKING_GET_PWR_TRACKING_OFFSET_VALUE_A(__h2c)                  \
+	LE_BITS_TO_4BYTE(__h2c + 0X0C, 8, 8)
+#define POWER_TRACKING_SET_PWR_TRACKING_OFFSET_VALUE_A(__h2c, __value)         \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X0C, 8, 8, __value)
+#define POWER_TRACKING_GET_TSSI_VALUE_A(__h2c)                                 \
+	LE_BITS_TO_4BYTE(__h2c + 0X0C, 16, 8)
+#define POWER_TRACKING_SET_TSSI_VALUE_A(__h2c, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X0C, 16, 8, __value)
+#define POWER_TRACKING_GET_TX_PWR_INDEX_B(__h2c)                               \
+	LE_BITS_TO_4BYTE(__h2c + 0X10, 0, 8)
+#define POWER_TRACKING_SET_TX_PWR_INDEX_B(__h2c, __value)                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X10, 0, 8, __value)
+#define POWER_TRACKING_GET_PWR_TRACKING_OFFSET_VALUE_B(__h2c)                  \
+	LE_BITS_TO_4BYTE(__h2c + 0X10, 8, 8)
+#define POWER_TRACKING_SET_PWR_TRACKING_OFFSET_VALUE_B(__h2c, __value)         \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X10, 8, 8, __value)
+#define POWER_TRACKING_GET_TSSI_VALUE_B(__h2c)                                 \
+	LE_BITS_TO_4BYTE(__h2c + 0X10, 16, 8)
+#define POWER_TRACKING_SET_TSSI_VALUE_B(__h2c, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X10, 16, 8, __value)
+#define POWER_TRACKING_GET_TX_PWR_INDEX_C(__h2c)                               \
+	LE_BITS_TO_4BYTE(__h2c + 0X14, 0, 8)
+#define POWER_TRACKING_SET_TX_PWR_INDEX_C(__h2c, __value)                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X14, 0, 8, __value)
+#define POWER_TRACKING_GET_PWR_TRACKING_OFFSET_VALUE_C(__h2c)                  \
+	LE_BITS_TO_4BYTE(__h2c + 0X14, 8, 8)
+#define POWER_TRACKING_SET_PWR_TRACKING_OFFSET_VALUE_C(__h2c, __value)         \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X14, 8, 8, __value)
+#define POWER_TRACKING_GET_TSSI_VALUE_C(__h2c)                                 \
+	LE_BITS_TO_4BYTE(__h2c + 0X14, 16, 8)
+#define POWER_TRACKING_SET_TSSI_VALUE_C(__h2c, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X14, 16, 8, __value)
+#define POWER_TRACKING_GET_TX_PWR_INDEX_D(__h2c)                               \
+	LE_BITS_TO_4BYTE(__h2c + 0X18, 0, 8)
+#define POWER_TRACKING_SET_TX_PWR_INDEX_D(__h2c, __value)                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X18, 0, 8, __value)
+#define POWER_TRACKING_GET_PWR_TRACKING_OFFSET_VALUE_D(__h2c)                  \
+	LE_BITS_TO_4BYTE(__h2c + 0X18, 8, 8)
+#define POWER_TRACKING_SET_PWR_TRACKING_OFFSET_VALUE_D(__h2c, __value)         \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X18, 8, 8, __value)
+#define POWER_TRACKING_GET_TSSI_VALUE_D(__h2c)                                 \
+	LE_BITS_TO_4BYTE(__h2c + 0X18, 16, 8)
+#define POWER_TRACKING_SET_TSSI_VALUE_D(__h2c, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X18, 16, 8, __value)
+#define PSD_GET_START_PSD(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 0, 16)
+#define PSD_SET_START_PSD(__h2c, __value)                                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 0, 16, __value)
+#define PSD_GET_END_PSD(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 16, 16)
+#define PSD_SET_END_PSD(__h2c, __value)                                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 16, 16, __value)
+#define P2PPS_GET_OFFLOAD_EN(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 0, 1)
+#define P2PPS_SET_OFFLOAD_EN(__h2c, __value)                                   \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 0, 1, __value)
+#define P2PPS_GET_ROLE(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 1, 1)
+#define P2PPS_SET_ROLE(__h2c, __value)                                         \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 1, 1, __value)
+#define P2PPS_GET_CTWINDOW_EN(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 2, 1)
+#define P2PPS_SET_CTWINDOW_EN(__h2c, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 2, 1, __value)
+#define P2PPS_GET_NOA_EN(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 3, 1)
+#define P2PPS_SET_NOA_EN(__h2c, __value)                                       \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 3, 1, __value)
+#define P2PPS_GET_NOA_SEL(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 4, 1)
+#define P2PPS_SET_NOA_SEL(__h2c, __value)                                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 4, 1, __value)
+#define P2PPS_GET_ALLSTASLEEP(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 5, 1)
+#define P2PPS_SET_ALLSTASLEEP(__h2c, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 5, 1, __value)
+#define P2PPS_GET_DISCOVERY(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 6, 1)
+#define P2PPS_SET_DISCOVERY(__h2c, __value)                                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 6, 1, __value)
+#define P2PPS_GET_P2P_PORT_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 8, 8)
+#define P2PPS_SET_P2P_PORT_ID(__h2c, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 8, 8, __value)
+#define P2PPS_GET_P2P_GROUP(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 16, 8)
+#define P2PPS_SET_P2P_GROUP(__h2c, __value)                                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 16, 8, __value)
+#define P2PPS_GET_P2P_MACID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 24, 8)
+#define P2PPS_SET_P2P_MACID(__h2c, __value)                                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 24, 8, __value)
+#define P2PPS_GET_CTWINDOW_LENGTH(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X0C, 0, 8)
+#define P2PPS_SET_CTWINDOW_LENGTH(__h2c, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X0C, 0, 8, __value)
+#define P2PPS_GET_NOA_DURATION_PARA(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X10, 0, 32)
+#define P2PPS_SET_NOA_DURATION_PARA(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X10, 0, 32, __value)
+#define P2PPS_GET_NOA_INTERVAL_PARA(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X14, 0, 32)
+#define P2PPS_SET_NOA_INTERVAL_PARA(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X14, 0, 32, __value)
+#define P2PPS_GET_NOA_START_TIME_PARA(__h2c)                                   \
+	LE_BITS_TO_4BYTE(__h2c + 0X18, 0, 32)
+#define P2PPS_SET_NOA_START_TIME_PARA(__h2c, __value)                          \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X18, 0, 32, __value)
+#define P2PPS_GET_NOA_COUNT_PARA(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X1C, 0, 32)
+#define P2PPS_SET_NOA_COUNT_PARA(__h2c, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X1C, 0, 32, __value)
+#define BT_COEX_GET_DATA_START(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 0, 8)
+#define BT_COEX_SET_DATA_START(__h2c, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 0, 8, __value)
+#define NAN_CTRL_GET_NAN_EN(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 0, 2)
+#define NAN_CTRL_SET_NAN_EN(__h2c, __value)                                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 0, 2, __value)
+#define NAN_CTRL_GET_SUPPORT_BAND(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 8, 2)
+#define NAN_CTRL_SET_SUPPORT_BAND(__h2c, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 8, 2, __value)
+#define NAN_CTRL_GET_DISABLE_2G_DISC_BCN(__h2c)                                \
+	LE_BITS_TO_4BYTE(__h2c + 0X08, 10, 1)
+#define NAN_CTRL_SET_DISABLE_2G_DISC_BCN(__h2c, __value)                       \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 10, 1, __value)
+#define NAN_CTRL_GET_DISABLE_5G_DISC_BCN(__h2c)                                \
+	LE_BITS_TO_4BYTE(__h2c + 0X08, 11, 1)
+#define NAN_CTRL_SET_DISABLE_5G_DISC_BCN(__h2c, __value)                       \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 11, 1, __value)
+#define NAN_CTRL_GET_BCN_RSVD_PAGE_OFFSET(__h2c)                               \
+	LE_BITS_TO_4BYTE(__h2c + 0X08, 16, 8)
+#define NAN_CTRL_SET_BCN_RSVD_PAGE_OFFSET(__h2c, __value)                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 16, 8, __value)
+#define NAN_CTRL_GET_CHANNEL_2G(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X08, 24, 8)
+#define NAN_CTRL_SET_CHANNEL_2G(__h2c, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 24, 8, __value)
+#define NAN_CTRL_GET_CHANNEL_5G(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X0C, 0, 8)
+#define NAN_CTRL_SET_CHANNEL_5G(__h2c, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X0C, 0, 8, __value)
+#define NAN_CHANNEL_PLAN_0_GET_CHANNEL_NUMBER_0(__h2c)                         \
+	LE_BITS_TO_4BYTE(__h2c + 0X08, 0, 8)
+#define NAN_CHANNEL_PLAN_0_SET_CHANNEL_NUMBER_0(__h2c, __value)                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 0, 8, __value)
+#define NAN_CHANNEL_PLAN_0_GET_UNPAUSE_MACID_0(__h2c)                          \
+	LE_BITS_TO_4BYTE(__h2c + 0X08, 8, 8)
+#define NAN_CHANNEL_PLAN_0_SET_UNPAUSE_MACID_0(__h2c, __value)                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 8, 8, __value)
+#define NAN_CHANNEL_PLAN_0_GET_START_TIME_SLOT_0(__h2c)                        \
+	LE_BITS_TO_4BYTE(__h2c + 0X0C, 0, 16)
+#define NAN_CHANNEL_PLAN_0_SET_START_TIME_SLOT_0(__h2c, __value)               \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X0C, 0, 16, __value)
+#define NAN_CHANNEL_PLAN_0_GET_DURATION_0(__h2c)                               \
+	LE_BITS_TO_4BYTE(__h2c + 0X0C, 16, 16)
+#define NAN_CHANNEL_PLAN_0_SET_DURATION_0(__h2c, __value)                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X0C, 16, 16, __value)
+#define NAN_CHANNEL_PLAN_0_GET_CHANNEL_NUMBER_1(__h2c)                         \
+	LE_BITS_TO_4BYTE(__h2c + 0X10, 0, 8)
+#define NAN_CHANNEL_PLAN_0_SET_CHANNEL_NUMBER_1(__h2c, __value)                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X10, 0, 8, __value)
+#define NAN_CHANNEL_PLAN_0_GET_UNPAUSE_MACID_1(__h2c)                          \
+	LE_BITS_TO_4BYTE(__h2c + 0X10, 8, 8)
+#define NAN_CHANNEL_PLAN_0_SET_UNPAUSE_MACID_1(__h2c, __value)                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X10, 8, 8, __value)
+#define NAN_CHANNEL_PLAN_0_GET_START_TIME_SLOT_1(__h2c)                        \
+	LE_BITS_TO_4BYTE(__h2c + 0X14, 0, 16)
+#define NAN_CHANNEL_PLAN_0_SET_START_TIME_SLOT_1(__h2c, __value)               \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X14, 0, 16, __value)
+#define NAN_CHANNEL_PLAN_0_GET_DURATION_1(__h2c)                               \
+	LE_BITS_TO_4BYTE(__h2c + 0X14, 16, 16)
+#define NAN_CHANNEL_PLAN_0_SET_DURATION_1(__h2c, __value)                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X14, 16, 16, __value)
+#define NAN_CHANNEL_PLAN_0_GET_CHANNEL_NUMBER_2(__h2c)                         \
+	LE_BITS_TO_4BYTE(__h2c + 0X18, 0, 8)
+#define NAN_CHANNEL_PLAN_0_SET_CHANNEL_NUMBER_2(__h2c, __value)                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X18, 0, 8, __value)
+#define NAN_CHANNEL_PLAN_0_GET_UNPAUSE_MACID_2(__h2c)                          \
+	LE_BITS_TO_4BYTE(__h2c + 0X18, 8, 8)
+#define NAN_CHANNEL_PLAN_0_SET_UNPAUSE_MACID_2(__h2c, __value)                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X18, 8, 8, __value)
+#define NAN_CHANNEL_PLAN_0_GET_START_TIME_SLOT_2(__h2c)                        \
+	LE_BITS_TO_4BYTE(__h2c + 0X1C, 0, 16)
+#define NAN_CHANNEL_PLAN_0_SET_START_TIME_SLOT_2(__h2c, __value)               \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X1C, 0, 16, __value)
+#define NAN_CHANNEL_PLAN_0_GET_DURATION_2(__h2c)                               \
+	LE_BITS_TO_4BYTE(__h2c + 0X1C, 16, 16)
+#define NAN_CHANNEL_PLAN_0_SET_DURATION_2(__h2c, __value)                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X1C, 16, 16, __value)
+#define NAN_CHANNEL_PLAN_1_GET_CHANNEL_NUMBER_3(__h2c)                         \
+	LE_BITS_TO_4BYTE(__h2c + 0X08, 0, 8)
+#define NAN_CHANNEL_PLAN_1_SET_CHANNEL_NUMBER_3(__h2c, __value)                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 0, 8, __value)
+#define NAN_CHANNEL_PLAN_1_GET_UNPAUSE_MACID_3(__h2c)                          \
+	LE_BITS_TO_4BYTE(__h2c + 0X08, 8, 8)
+#define NAN_CHANNEL_PLAN_1_SET_UNPAUSE_MACID_3(__h2c, __value)                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X08, 8, 8, __value)
+#define NAN_CHANNEL_PLAN_1_GET_START_TIME_SLOT_3(__h2c)                        \
+	LE_BITS_TO_4BYTE(__h2c + 0X0C, 0, 16)
+#define NAN_CHANNEL_PLAN_1_SET_START_TIME_SLOT_3(__h2c, __value)               \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X0C, 0, 16, __value)
+#define NAN_CHANNEL_PLAN_1_GET_DURATION_3(__h2c)                               \
+	LE_BITS_TO_4BYTE(__h2c + 0X0C, 16, 16)
+#define NAN_CHANNEL_PLAN_1_SET_DURATION_3(__h2c, __value)                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X0C, 16, 16, __value)
+#define NAN_CHANNEL_PLAN_1_GET_CHANNEL_NUMBER_4(__h2c)                         \
+	LE_BITS_TO_4BYTE(__h2c + 0X10, 0, 8)
+#define NAN_CHANNEL_PLAN_1_SET_CHANNEL_NUMBER_4(__h2c, __value)                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X10, 0, 8, __value)
+#define NAN_CHANNEL_PLAN_1_GET_UNPAUSE_MACID_4(__h2c)                          \
+	LE_BITS_TO_4BYTE(__h2c + 0X10, 8, 8)
+#define NAN_CHANNEL_PLAN_1_SET_UNPAUSE_MACID_4(__h2c, __value)                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X10, 8, 8, __value)
+#define NAN_CHANNEL_PLAN_1_GET_START_TIME_SLOT_4(__h2c)                        \
+	LE_BITS_TO_4BYTE(__h2c + 0X14, 0, 16)
+#define NAN_CHANNEL_PLAN_1_SET_START_TIME_SLOT_4(__h2c, __value)               \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X14, 0, 16, __value)
+#define NAN_CHANNEL_PLAN_1_GET_DURATION_4(__h2c)                               \
+	LE_BITS_TO_4BYTE(__h2c + 0X14, 16, 16)
+#define NAN_CHANNEL_PLAN_1_SET_DURATION_4(__h2c, __value)                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X14, 16, 16, __value)
+#define NAN_CHANNEL_PLAN_1_GET_CHANNEL_NUMBER_5(__h2c)                         \
+	LE_BITS_TO_4BYTE(__h2c + 0X18, 0, 8)
+#define NAN_CHANNEL_PLAN_1_SET_CHANNEL_NUMBER_5(__h2c, __value)                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X18, 0, 8, __value)
+#define NAN_CHANNEL_PLAN_1_GET_UNPAUSE_MACID_5(__h2c)                          \
+	LE_BITS_TO_4BYTE(__h2c + 0X18, 8, 8)
+#define NAN_CHANNEL_PLAN_1_SET_UNPAUSE_MACID_5(__h2c, __value)                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X18, 8, 8, __value)
+#define NAN_CHANNEL_PLAN_1_GET_START_TIME_SLOT_5(__h2c)                        \
+	LE_BITS_TO_4BYTE(__h2c + 0X1C, 0, 16)
+#define NAN_CHANNEL_PLAN_1_SET_START_TIME_SLOT_5(__h2c, __value)               \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X1C, 0, 16, __value)
+#define NAN_CHANNEL_PLAN_1_GET_DURATION_5(__h2c)                               \
+	LE_BITS_TO_4BYTE(__h2c + 0X1C, 16, 16)
+#define NAN_CHANNEL_PLAN_1_SET_DURATION_5(__h2c, __value)                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X1C, 16, 16, __value)
+#endif
diff --git a/drivers/staging/rtlwifi/halmac/halmac_h2c_extra_info_nic.h b/drivers/staging/rtlwifi/halmac/halmac_h2c_extra_info_nic.h
new file mode 100644
index 000000000000..5f23cba6d067
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_h2c_extra_info_nic.h
@@ -0,0 +1,115 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HAL_H2CEXTRAINFO_H2C_C2H_NIC_H_
+#define _HAL_H2CEXTRAINFO_H2C_C2H_NIC_H_
+#define PHY_PARAMETER_INFO_GET_LENGTH(__extra_info)                            \
+	LE_BITS_TO_4BYTE(__extra_info + 0X00, 0, 8)
+#define PHY_PARAMETER_INFO_SET_LENGTH(__extra_info, __value)                   \
+	SET_BITS_TO_LE_4BYTE(__extra_info + 0X00, 0, 8, __value)
+#define PHY_PARAMETER_INFO_GET_IO_CMD(__extra_info)                            \
+	LE_BITS_TO_4BYTE(__extra_info + 0X00, 8, 7)
+#define PHY_PARAMETER_INFO_SET_IO_CMD(__extra_info, __value)                   \
+	SET_BITS_TO_LE_4BYTE(__extra_info + 0X00, 8, 7, __value)
+#define PHY_PARAMETER_INFO_GET_MSK_EN(__extra_info)                            \
+	LE_BITS_TO_4BYTE(__extra_info + 0X00, 15, 1)
+#define PHY_PARAMETER_INFO_SET_MSK_EN(__extra_info, __value)                   \
+	SET_BITS_TO_LE_4BYTE(__extra_info + 0X00, 15, 1, __value)
+#define PHY_PARAMETER_INFO_GET_LLT_PG_BNDY(__extra_info)                       \
+	LE_BITS_TO_4BYTE(__extra_info + 0X00, 16, 8)
+#define PHY_PARAMETER_INFO_SET_LLT_PG_BNDY(__extra_info, __value)              \
+	SET_BITS_TO_LE_4BYTE(__extra_info + 0X00, 16, 8, __value)
+#define PHY_PARAMETER_INFO_GET_EFUSE_RSVDPAGE_LOC(__extra_info)                \
+	LE_BITS_TO_4BYTE(__extra_info + 0X00, 16, 8)
+#define PHY_PARAMETER_INFO_SET_EFUSE_RSVDPAGE_LOC(__extra_info, __value)       \
+	SET_BITS_TO_LE_4BYTE(__extra_info + 0X00, 16, 8, __value)
+#define PHY_PARAMETER_INFO_GET_EFUSE_PATCH_EN(__extra_info)                    \
+	LE_BITS_TO_4BYTE(__extra_info + 0X00, 16, 8)
+#define PHY_PARAMETER_INFO_SET_EFUSE_PATCH_EN(__extra_info, __value)           \
+	SET_BITS_TO_LE_4BYTE(__extra_info + 0X00, 16, 8, __value)
+#define PHY_PARAMETER_INFO_GET_RF_ADDR(__extra_info)                           \
+	LE_BITS_TO_4BYTE(__extra_info + 0X00, 16, 8)
+#define PHY_PARAMETER_INFO_SET_RF_ADDR(__extra_info, __value)                  \
+	SET_BITS_TO_LE_4BYTE(__extra_info + 0X00, 16, 8, __value)
+#define PHY_PARAMETER_INFO_GET_IO_ADDR(__extra_info)                           \
+	LE_BITS_TO_4BYTE(__extra_info + 0X00, 16, 16)
+#define PHY_PARAMETER_INFO_SET_IO_ADDR(__extra_info, __value)                  \
+	SET_BITS_TO_LE_4BYTE(__extra_info + 0X00, 16, 16, __value)
+#define PHY_PARAMETER_INFO_GET_DELAY_VALUE(__extra_info)                       \
+	LE_BITS_TO_4BYTE(__extra_info + 0X00, 16, 16)
+#define PHY_PARAMETER_INFO_SET_DELAY_VALUE(__extra_info, __value)              \
+	SET_BITS_TO_LE_4BYTE(__extra_info + 0X00, 16, 16, __value)
+#define PHY_PARAMETER_INFO_GET_RF_PATH(__extra_info)                           \
+	LE_BITS_TO_4BYTE(__extra_info + 0X00, 24, 8)
+#define PHY_PARAMETER_INFO_SET_RF_PATH(__extra_info, __value)                  \
+	SET_BITS_TO_LE_4BYTE(__extra_info + 0X00, 24, 8, __value)
+#define PHY_PARAMETER_INFO_GET_DATA(__extra_info)                              \
+	LE_BITS_TO_4BYTE(__extra_info + 0X04, 0, 32)
+#define PHY_PARAMETER_INFO_SET_DATA(__extra_info, __value)                     \
+	SET_BITS_TO_LE_4BYTE(__extra_info + 0X04, 0, 32, __value)
+#define PHY_PARAMETER_INFO_GET_MASK(__extra_info)                              \
+	LE_BITS_TO_4BYTE(__extra_info + 0X08, 0, 32)
+#define PHY_PARAMETER_INFO_SET_MASK(__extra_info, __value)                     \
+	SET_BITS_TO_LE_4BYTE(__extra_info + 0X08, 0, 32, __value)
+#define CHANNEL_INFO_GET_CHANNEL(__extra_info)                                 \
+	LE_BITS_TO_4BYTE(__extra_info + 0X00, 0, 8)
+#define CHANNEL_INFO_SET_CHANNEL(__extra_info, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__extra_info + 0X00, 0, 8, __value)
+#define CHANNEL_INFO_GET_PRI_CH_IDX(__extra_info)                              \
+	LE_BITS_TO_4BYTE(__extra_info + 0X00, 8, 4)
+#define CHANNEL_INFO_SET_PRI_CH_IDX(__extra_info, __value)                     \
+	SET_BITS_TO_LE_4BYTE(__extra_info + 0X00, 8, 4, __value)
+#define CHANNEL_INFO_GET_BANDWIDTH(__extra_info)                               \
+	LE_BITS_TO_4BYTE(__extra_info + 0X00, 12, 4)
+#define CHANNEL_INFO_SET_BANDWIDTH(__extra_info, __value)                      \
+	SET_BITS_TO_LE_4BYTE(__extra_info + 0X00, 12, 4, __value)
+#define CHANNEL_INFO_GET_TIMEOUT(__extra_info)                                 \
+	LE_BITS_TO_4BYTE(__extra_info + 0X00, 16, 8)
+#define CHANNEL_INFO_SET_TIMEOUT(__extra_info, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__extra_info + 0X00, 16, 8, __value)
+#define CHANNEL_INFO_GET_ACTION_ID(__extra_info)                               \
+	LE_BITS_TO_4BYTE(__extra_info + 0X00, 24, 7)
+#define CHANNEL_INFO_SET_ACTION_ID(__extra_info, __value)                      \
+	SET_BITS_TO_LE_4BYTE(__extra_info + 0X00, 24, 7, __value)
+#define CHANNEL_INFO_GET_CH_EXTRA_INFO(__extra_info)                           \
+	LE_BITS_TO_4BYTE(__extra_info + 0X00, 31, 1)
+#define CHANNEL_INFO_SET_CH_EXTRA_INFO(__extra_info, __value)                  \
+	SET_BITS_TO_LE_4BYTE(__extra_info + 0X00, 31, 1, __value)
+#define CH_EXTRA_INFO_GET_CH_EXTRA_INFO_ID(__extra_info)                       \
+	LE_BITS_TO_4BYTE(__extra_info + 0X00, 0, 7)
+#define CH_EXTRA_INFO_SET_CH_EXTRA_INFO_ID(__extra_info, __value)              \
+	SET_BITS_TO_LE_4BYTE(__extra_info + 0X00, 0, 7, __value)
+#define CH_EXTRA_INFO_GET_CH_EXTRA_INFO(__extra_info)                          \
+	LE_BITS_TO_4BYTE(__extra_info + 0X00, 7, 1)
+#define CH_EXTRA_INFO_SET_CH_EXTRA_INFO(__extra_info, __value)                 \
+	SET_BITS_TO_LE_4BYTE(__extra_info + 0X00, 7, 1, __value)
+#define CH_EXTRA_INFO_GET_CH_EXTRA_INFO_SIZE(__extra_info)                     \
+	LE_BITS_TO_4BYTE(__extra_info + 0X00, 8, 8)
+#define CH_EXTRA_INFO_SET_CH_EXTRA_INFO_SIZE(__extra_info, __value)            \
+	SET_BITS_TO_LE_4BYTE(__extra_info + 0X00, 8, 8, __value)
+#define CH_EXTRA_INFO_GET_CH_EXTRA_INFO_DATA(__extra_info)                     \
+	LE_BITS_TO_4BYTE(__extra_info + 0X00, 16, 1)
+#define CH_EXTRA_INFO_SET_CH_EXTRA_INFO_DATA(__extra_info, __value)            \
+	SET_BITS_TO_LE_4BYTE(__extra_info + 0X00, 16, 1, __value)
+#endif
diff --git a/drivers/staging/rtlwifi/halmac/halmac_intf_phy_cmd.h b/drivers/staging/rtlwifi/halmac/halmac_intf_phy_cmd.h
new file mode 100644
index 000000000000..273d4c0e338a
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_intf_phy_cmd.h
@@ -0,0 +1,54 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef HALMAC_INTF_PHY_CMD
+#define HALMAC_INTF_PHY_CMD
+
+/* Cut mask */
+enum halmac_intf_phy_cut {
+	HALMAC_INTF_PHY_CUT_TESTCHIP = BIT(0),
+	HALMAC_INTF_PHY_CUT_A = BIT(1),
+	HALMAC_INTF_PHY_CUT_B = BIT(2),
+	HALMAC_INTF_PHY_CUT_C = BIT(3),
+	HALMAC_INTF_PHY_CUT_D = BIT(4),
+	HALMAC_INTF_PHY_CUT_E = BIT(5),
+	HALMAC_INTF_PHY_CUT_F = BIT(6),
+	HALMAC_INTF_PHY_CUT_G = BIT(7),
+	HALMAC_INTF_PHY_CUT_ALL = 0x7FFF,
+};
+
+/* IP selection */
+enum halmac_ip_sel {
+	HALMAC_IP_SEL_INTF_PHY = 0,
+	HALMAC_IP_SEL_MAC = 1,
+	HALMAC_IP_SEL_PCIE_DBI = 2,
+	HALMAC_IP_SEL_UNDEFINE = 0x7FFF,
+};
+
+/* Platform mask */
+enum halmac_intf_phy_platform {
+	HALMAC_INTF_PHY_PLATFORM_ALL = 0x7FFF,
+};
+
+#endif
diff --git a/drivers/staging/rtlwifi/halmac/halmac_original_c2h_nic.h b/drivers/staging/rtlwifi/halmac/halmac_original_c2h_nic.h
new file mode 100644
index 000000000000..4331e2ae14c2
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_original_c2h_nic.h
@@ -0,0 +1,403 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HAL_ORIGINALC2HFORMAT_H2C_C2H_NIC_H_
+#define _HAL_ORIGINALC2HFORMAT_H2C_C2H_NIC_H_
+#define CMD_ID_C2H 0X00
+#define CMD_ID_DBG 0X00
+#define CMD_ID_C2H_LB 0X01
+#define CMD_ID_C2H_SND_TXBF 0X02
+#define CMD_ID_C2H_CCX_RPT 0X03
+#define CMD_ID_C2H_AP_REQ_TXRPT 0X04
+#define CMD_ID_C2H_INITIAL_RATE_COLLECTION 0X05
+#define CMD_ID_C2H_RA_RPT 0X0C
+#define CMD_ID_C2H_SPECIAL_STATISTICS 0X0D
+#define CMD_ID_C2H_RA_PARA_RPT 0X0E
+#define CMD_ID_C2H_CUR_CHANNEL 0X10
+#define CMD_ID_C2H_GPIO_WAKEUP 0X14
+#define C2H_GET_CMD_ID(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 0, 8)
+#define C2H_SET_CMD_ID(__c2h, __value)                                         \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 0, 8, __value)
+#define C2H_GET_SEQ(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 8, 8)
+#define C2H_SET_SEQ(__c2h, __value)                                            \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 8, 8, __value)
+#define DBG_GET_CMD_ID(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 0, 8)
+#define DBG_SET_CMD_ID(__c2h, __value)                                         \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 0, 8, __value)
+#define DBG_GET_SEQ(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 8, 8)
+#define DBG_SET_SEQ(__c2h, __value)                                            \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 8, 8, __value)
+#define DBG_GET_DBG_STR1(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 16, 8)
+#define DBG_SET_DBG_STR1(__c2h, __value)                                       \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 16, 8, __value)
+#define DBG_GET_DBG_STR2(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 24, 8)
+#define DBG_SET_DBG_STR2(__c2h, __value)                                       \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 24, 8, __value)
+#define DBG_GET_DBG_STR3(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X04, 0, 8)
+#define DBG_SET_DBG_STR3(__c2h, __value)                                       \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 0, 8, __value)
+#define DBG_GET_DBG_STR4(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X04, 8, 8)
+#define DBG_SET_DBG_STR4(__c2h, __value)                                       \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 8, 8, __value)
+#define DBG_GET_DBG_STR5(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X04, 16, 8)
+#define DBG_SET_DBG_STR5(__c2h, __value)                                       \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 16, 8, __value)
+#define DBG_GET_DBG_STR6(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X04, 24, 8)
+#define DBG_SET_DBG_STR6(__c2h, __value)                                       \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 24, 8, __value)
+#define DBG_GET_DBG_STR7(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X08, 0, 8)
+#define DBG_SET_DBG_STR7(__c2h, __value)                                       \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X08, 0, 8, __value)
+#define DBG_GET_DBG_STR8(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X08, 8, 8)
+#define DBG_SET_DBG_STR8(__c2h, __value)                                       \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X08, 8, 8, __value)
+#define DBG_GET_DBG_STR9(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X08, 16, 8)
+#define DBG_SET_DBG_STR9(__c2h, __value)                                       \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X08, 16, 8, __value)
+#define DBG_GET_DBG_STR10(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X08, 24, 8)
+#define DBG_SET_DBG_STR10(__c2h, __value)                                      \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X08, 24, 8, __value)
+#define DBG_GET_DBG_STR11(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X0C, 0, 8)
+#define DBG_SET_DBG_STR11(__c2h, __value)                                      \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 0, 8, __value)
+#define DBG_GET_DBG_STR12(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X0C, 8, 8)
+#define DBG_SET_DBG_STR12(__c2h, __value)                                      \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 8, 8, __value)
+#define DBG_GET_LEN(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X0C, 16, 8)
+#define DBG_SET_LEN(__c2h, __value)                                            \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 16, 8, __value)
+#define DBG_GET_TRIGGER(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X0C, 24, 8)
+#define DBG_SET_TRIGGER(__c2h, __value)                                        \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 24, 8, __value)
+#define C2H_LB_GET_CMD_ID(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 0, 8)
+#define C2H_LB_SET_CMD_ID(__c2h, __value)                                      \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 0, 8, __value)
+#define C2H_LB_GET_SEQ(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 8, 8)
+#define C2H_LB_SET_SEQ(__c2h, __value)                                         \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 8, 8, __value)
+#define C2H_LB_GET_PAYLOAD1(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 16, 16)
+#define C2H_LB_SET_PAYLOAD1(__c2h, __value)                                    \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 16, 16, __value)
+#define C2H_LB_GET_PAYLOAD2(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X04, 0, 32)
+#define C2H_LB_SET_PAYLOAD2(__c2h, __value)                                    \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 0, 32, __value)
+#define C2H_LB_GET_LEN(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X0C, 16, 8)
+#define C2H_LB_SET_LEN(__c2h, __value)                                         \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 16, 8, __value)
+#define C2H_LB_GET_TRIGGER(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X0C, 24, 8)
+#define C2H_LB_SET_TRIGGER(__c2h, __value)                                     \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 24, 8, __value)
+#define C2H_SND_TXBF_GET_CMD_ID(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 0, 8)
+#define C2H_SND_TXBF_SET_CMD_ID(__c2h, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 0, 8, __value)
+#define C2H_SND_TXBF_GET_SEQ(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 8, 8)
+#define C2H_SND_TXBF_SET_SEQ(__c2h, __value)                                   \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 8, 8, __value)
+#define C2H_SND_TXBF_GET_SND_RESULT(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 16, 1)
+#define C2H_SND_TXBF_SET_SND_RESULT(__c2h, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 16, 1, __value)
+#define C2H_SND_TXBF_GET_LEN(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X0C, 16, 8)
+#define C2H_SND_TXBF_SET_LEN(__c2h, __value)                                   \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 16, 8, __value)
+#define C2H_SND_TXBF_GET_TRIGGER(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X0C, 24, 8)
+#define C2H_SND_TXBF_SET_TRIGGER(__c2h, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 24, 8, __value)
+#define C2H_CCX_RPT_GET_CMD_ID(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 0, 8)
+#define C2H_CCX_RPT_SET_CMD_ID(__c2h, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 0, 8, __value)
+#define C2H_CCX_RPT_GET_SEQ(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 8, 8)
+#define C2H_CCX_RPT_SET_SEQ(__c2h, __value)                                    \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 8, 8, __value)
+#define C2H_CCX_RPT_GET_QSEL(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 16, 5)
+#define C2H_CCX_RPT_SET_QSEL(__c2h, __value)                                   \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 16, 5, __value)
+#define C2H_CCX_RPT_GET_BMC(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 21, 1)
+#define C2H_CCX_RPT_SET_BMC(__c2h, __value)                                    \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 21, 1, __value)
+#define C2H_CCX_RPT_GET_LIFE_TIME_OVER(__c2h)                                  \
+	LE_BITS_TO_4BYTE(__c2h + 0X00, 22, 1)
+#define C2H_CCX_RPT_SET_LIFE_TIME_OVER(__c2h, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 22, 1, __value)
+#define C2H_CCX_RPT_GET_RETRY_OVER(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 23, 1)
+#define C2H_CCX_RPT_SET_RETRY_OVER(__c2h, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 23, 1, __value)
+#define C2H_CCX_RPT_GET_MACID(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 24, 8)
+#define C2H_CCX_RPT_SET_MACID(__c2h, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 24, 8, __value)
+#define C2H_CCX_RPT_GET_DATA_RETRY_CNT(__c2h)                                  \
+	LE_BITS_TO_4BYTE(__c2h + 0X04, 0, 6)
+#define C2H_CCX_RPT_SET_DATA_RETRY_CNT(__c2h, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 0, 6, __value)
+#define C2H_CCX_RPT_GET_QUEUE7_0(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X04, 8, 8)
+#define C2H_CCX_RPT_SET_QUEUE7_0(__c2h, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 8, 8, __value)
+#define C2H_CCX_RPT_GET_QUEUE15_8(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X04, 16, 8)
+#define C2H_CCX_RPT_SET_QUEUE15_8(__c2h, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 16, 8, __value)
+#define C2H_CCX_RPT_GET_FINAL_DATA_RATE(__c2h)                                 \
+	LE_BITS_TO_4BYTE(__c2h + 0X04, 24, 8)
+#define C2H_CCX_RPT_SET_FINAL_DATA_RATE(__c2h, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 24, 8, __value)
+#define C2H_CCX_RPT_GET_SW_DEFINE_0(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X08, 0, 8)
+#define C2H_CCX_RPT_SET_SW_DEFINE_0(__c2h, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X08, 0, 8, __value)
+#define C2H_CCX_RPT_GET_SW_DEFINE_1(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X08, 8, 4)
+#define C2H_CCX_RPT_SET_SW_DEFINE_1(__c2h, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X08, 8, 4, __value)
+#define C2H_CCX_RPT_GET_LEN(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X0C, 16, 8)
+#define C2H_CCX_RPT_SET_LEN(__c2h, __value)                                    \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 16, 8, __value)
+#define C2H_CCX_RPT_GET_TRIGGER(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X0C, 24, 8)
+#define C2H_CCX_RPT_SET_TRIGGER(__c2h, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 24, 8, __value)
+#define C2H_AP_REQ_TXRPT_GET_CMD_ID(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 0, 8)
+#define C2H_AP_REQ_TXRPT_SET_CMD_ID(__c2h, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 0, 8, __value)
+#define C2H_AP_REQ_TXRPT_GET_SEQ(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 8, 8)
+#define C2H_AP_REQ_TXRPT_SET_SEQ(__c2h, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 8, 8, __value)
+#define C2H_AP_REQ_TXRPT_GET_STA1_MACID(__c2h)                                 \
+	LE_BITS_TO_4BYTE(__c2h + 0X00, 16, 8)
+#define C2H_AP_REQ_TXRPT_SET_STA1_MACID(__c2h, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 16, 8, __value)
+#define C2H_AP_REQ_TXRPT_GET_TX_OK1_0(__c2h)                                   \
+	LE_BITS_TO_4BYTE(__c2h + 0X00, 24, 8)
+#define C2H_AP_REQ_TXRPT_SET_TX_OK1_0(__c2h, __value)                          \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 24, 8, __value)
+#define C2H_AP_REQ_TXRPT_GET_TX_OK1_1(__c2h)                                   \
+	LE_BITS_TO_4BYTE(__c2h + 0X04, 0, 8)
+#define C2H_AP_REQ_TXRPT_SET_TX_OK1_1(__c2h, __value)                          \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 0, 8, __value)
+#define C2H_AP_REQ_TXRPT_GET_TX_FAIL1_0(__c2h)                                 \
+	LE_BITS_TO_4BYTE(__c2h + 0X04, 8, 8)
+#define C2H_AP_REQ_TXRPT_SET_TX_FAIL1_0(__c2h, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 8, 8, __value)
+#define C2H_AP_REQ_TXRPT_GET_TX_FAIL1_1(__c2h)                                 \
+	LE_BITS_TO_4BYTE(__c2h + 0X04, 16, 8)
+#define C2H_AP_REQ_TXRPT_SET_TX_FAIL1_1(__c2h, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 16, 8, __value)
+#define C2H_AP_REQ_TXRPT_GET_INITIAL_RATE1(__c2h)                              \
+	LE_BITS_TO_4BYTE(__c2h + 0X04, 24, 8)
+#define C2H_AP_REQ_TXRPT_SET_INITIAL_RATE1(__c2h, __value)                     \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 24, 8, __value)
+#define C2H_AP_REQ_TXRPT_GET_STA2_MACID(__c2h)                                 \
+	LE_BITS_TO_4BYTE(__c2h + 0X08, 0, 8)
+#define C2H_AP_REQ_TXRPT_SET_STA2_MACID(__c2h, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X08, 0, 8, __value)
+#define C2H_AP_REQ_TXRPT_GET_TX_OK2_0(__c2h)                                   \
+	LE_BITS_TO_4BYTE(__c2h + 0X08, 8, 8)
+#define C2H_AP_REQ_TXRPT_SET_TX_OK2_0(__c2h, __value)                          \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X08, 8, 8, __value)
+#define C2H_AP_REQ_TXRPT_GET_TX_OK2_1(__c2h)                                   \
+	LE_BITS_TO_4BYTE(__c2h + 0X08, 16, 8)
+#define C2H_AP_REQ_TXRPT_SET_TX_OK2_1(__c2h, __value)                          \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X08, 16, 8, __value)
+#define C2H_AP_REQ_TXRPT_GET_TX_FAIL2_0(__c2h)                                 \
+	LE_BITS_TO_4BYTE(__c2h + 0X08, 24, 8)
+#define C2H_AP_REQ_TXRPT_SET_TX_FAIL2_0(__c2h, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X08, 24, 8, __value)
+#define C2H_AP_REQ_TXRPT_GET_TX_FAIL2_1(__c2h)                                 \
+	LE_BITS_TO_4BYTE(__c2h + 0X0C, 0, 8)
+#define C2H_AP_REQ_TXRPT_SET_TX_FAIL2_1(__c2h, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 0, 8, __value)
+#define C2H_AP_REQ_TXRPT_GET_INITIAL_RATE2(__c2h)                              \
+	LE_BITS_TO_4BYTE(__c2h + 0X0C, 8, 8)
+#define C2H_AP_REQ_TXRPT_SET_INITIAL_RATE2(__c2h, __value)                     \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 8, 8, __value)
+#define C2H_AP_REQ_TXRPT_GET_LEN(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X0C, 16, 8)
+#define C2H_AP_REQ_TXRPT_SET_LEN(__c2h, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 16, 8, __value)
+#define C2H_AP_REQ_TXRPT_GET_TRIGGER(__c2h)                                    \
+	LE_BITS_TO_4BYTE(__c2h + 0X0C, 24, 8)
+#define C2H_AP_REQ_TXRPT_SET_TRIGGER(__c2h, __value)                           \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 24, 8, __value)
+#define C2H_INITIAL_RATE_COLLECTION_GET_CMD_ID(__c2h)                          \
+	LE_BITS_TO_4BYTE(__c2h + 0X00, 0, 8)
+#define C2H_INITIAL_RATE_COLLECTION_SET_CMD_ID(__c2h, __value)                 \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 0, 8, __value)
+#define C2H_INITIAL_RATE_COLLECTION_GET_SEQ(__c2h)                             \
+	LE_BITS_TO_4BYTE(__c2h + 0X00, 8, 8)
+#define C2H_INITIAL_RATE_COLLECTION_SET_SEQ(__c2h, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 8, 8, __value)
+#define C2H_INITIAL_RATE_COLLECTION_GET_TRYING_BITMAP(__c2h)                   \
+	LE_BITS_TO_4BYTE(__c2h + 0X00, 16, 7)
+#define C2H_INITIAL_RATE_COLLECTION_SET_TRYING_BITMAP(__c2h, __value)          \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 16, 7, __value)
+#define C2H_INITIAL_RATE_COLLECTION_GET_INITIAL_RATE1(__c2h)                   \
+	LE_BITS_TO_4BYTE(__c2h + 0X00, 24, 8)
+#define C2H_INITIAL_RATE_COLLECTION_SET_INITIAL_RATE1(__c2h, __value)          \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 24, 8, __value)
+#define C2H_INITIAL_RATE_COLLECTION_GET_INITIAL_RATE2(__c2h)                   \
+	LE_BITS_TO_4BYTE(__c2h + 0X04, 0, 8)
+#define C2H_INITIAL_RATE_COLLECTION_SET_INITIAL_RATE2(__c2h, __value)          \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 0, 8, __value)
+#define C2H_INITIAL_RATE_COLLECTION_GET_INITIAL_RATE3(__c2h)                   \
+	LE_BITS_TO_4BYTE(__c2h + 0X04, 8, 8)
+#define C2H_INITIAL_RATE_COLLECTION_SET_INITIAL_RATE3(__c2h, __value)          \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 8, 8, __value)
+#define C2H_INITIAL_RATE_COLLECTION_GET_INITIAL_RATE4(__c2h)                   \
+	LE_BITS_TO_4BYTE(__c2h + 0X04, 16, 8)
+#define C2H_INITIAL_RATE_COLLECTION_SET_INITIAL_RATE4(__c2h, __value)          \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 16, 8, __value)
+#define C2H_INITIAL_RATE_COLLECTION_GET_INITIAL_RATE5(__c2h)                   \
+	LE_BITS_TO_4BYTE(__c2h + 0X04, 24, 8)
+#define C2H_INITIAL_RATE_COLLECTION_SET_INITIAL_RATE5(__c2h, __value)          \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 24, 8, __value)
+#define C2H_INITIAL_RATE_COLLECTION_GET_INITIAL_RATE6(__c2h)                   \
+	LE_BITS_TO_4BYTE(__c2h + 0X08, 0, 8)
+#define C2H_INITIAL_RATE_COLLECTION_SET_INITIAL_RATE6(__c2h, __value)          \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X08, 0, 8, __value)
+#define C2H_INITIAL_RATE_COLLECTION_GET_INITIAL_RATE7(__c2h)                   \
+	LE_BITS_TO_4BYTE(__c2h + 0X08, 8, 8)
+#define C2H_INITIAL_RATE_COLLECTION_SET_INITIAL_RATE7(__c2h, __value)          \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X08, 8, 8, __value)
+#define C2H_INITIAL_RATE_COLLECTION_GET_LEN(__c2h)                             \
+	LE_BITS_TO_4BYTE(__c2h + 0X0C, 16, 8)
+#define C2H_INITIAL_RATE_COLLECTION_SET_LEN(__c2h, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 16, 8, __value)
+#define C2H_INITIAL_RATE_COLLECTION_GET_TRIGGER(__c2h)                         \
+	LE_BITS_TO_4BYTE(__c2h + 0X0C, 24, 8)
+#define C2H_INITIAL_RATE_COLLECTION_SET_TRIGGER(__c2h, __value)                \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 24, 8, __value)
+#define C2H_RA_RPT_GET_CMD_ID(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 0, 8)
+#define C2H_RA_RPT_SET_CMD_ID(__c2h, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 0, 8, __value)
+#define C2H_RA_RPT_GET_SEQ(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 8, 8)
+#define C2H_RA_RPT_SET_SEQ(__c2h, __value)                                     \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 8, 8, __value)
+#define C2H_RA_RPT_GET_RATE(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 16, 8)
+#define C2H_RA_RPT_SET_RATE(__c2h, __value)                                    \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 16, 8, __value)
+#define C2H_RA_RPT_GET_MACID(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 24, 8)
+#define C2H_RA_RPT_SET_MACID(__c2h, __value)                                   \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 24, 8, __value)
+#define C2H_RA_RPT_GET_USE_LDPC(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X04, 0, 1)
+#define C2H_RA_RPT_SET_USE_LDPC(__c2h, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 0, 1, __value)
+#define C2H_RA_RPT_GET_USE_TXBF(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X04, 1, 1)
+#define C2H_RA_RPT_SET_USE_TXBF(__c2h, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 1, 1, __value)
+#define C2H_RA_RPT_GET_COLLISION_STATE(__c2h)                                  \
+	LE_BITS_TO_4BYTE(__c2h + 0X04, 8, 8)
+#define C2H_RA_RPT_SET_COLLISION_STATE(__c2h, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 8, 8, __value)
+#define C2H_RA_RPT_GET_LEN(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X0C, 16, 8)
+#define C2H_RA_RPT_SET_LEN(__c2h, __value)                                     \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 16, 8, __value)
+#define C2H_RA_RPT_GET_TRIGGER(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X0C, 24, 8)
+#define C2H_RA_RPT_SET_TRIGGER(__c2h, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 24, 8, __value)
+#define C2H_SPECIAL_STATISTICS_GET_CMD_ID(__c2h)                               \
+	LE_BITS_TO_4BYTE(__c2h + 0X00, 0, 8)
+#define C2H_SPECIAL_STATISTICS_SET_CMD_ID(__c2h, __value)                      \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 0, 8, __value)
+#define C2H_SPECIAL_STATISTICS_GET_SEQ(__c2h)                                  \
+	LE_BITS_TO_4BYTE(__c2h + 0X00, 8, 8)
+#define C2H_SPECIAL_STATISTICS_SET_SEQ(__c2h, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 8, 8, __value)
+#define C2H_SPECIAL_STATISTICS_GET_STATISTICS_IDX(__c2h)                       \
+	LE_BITS_TO_4BYTE(__c2h + 0X00, 16, 8)
+#define C2H_SPECIAL_STATISTICS_SET_STATISTICS_IDX(__c2h, __value)              \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 16, 8, __value)
+#define C2H_SPECIAL_STATISTICS_GET_DATA0(__c2h)                                \
+	LE_BITS_TO_4BYTE(__c2h + 0X00, 24, 8)
+#define C2H_SPECIAL_STATISTICS_SET_DATA0(__c2h, __value)                       \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 24, 8, __value)
+#define C2H_SPECIAL_STATISTICS_GET_DATA1(__c2h)                                \
+	LE_BITS_TO_4BYTE(__c2h + 0X04, 0, 8)
+#define C2H_SPECIAL_STATISTICS_SET_DATA1(__c2h, __value)                       \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 0, 8, __value)
+#define C2H_SPECIAL_STATISTICS_GET_DATA2(__c2h)                                \
+	LE_BITS_TO_4BYTE(__c2h + 0X04, 8, 8)
+#define C2H_SPECIAL_STATISTICS_SET_DATA2(__c2h, __value)                       \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 8, 8, __value)
+#define C2H_SPECIAL_STATISTICS_GET_DATA3(__c2h)                                \
+	LE_BITS_TO_4BYTE(__c2h + 0X04, 16, 8)
+#define C2H_SPECIAL_STATISTICS_SET_DATA3(__c2h, __value)                       \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 16, 8, __value)
+#define C2H_SPECIAL_STATISTICS_GET_DATA4(__c2h)                                \
+	LE_BITS_TO_4BYTE(__c2h + 0X04, 24, 8)
+#define C2H_SPECIAL_STATISTICS_SET_DATA4(__c2h, __value)                       \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X04, 24, 8, __value)
+#define C2H_SPECIAL_STATISTICS_GET_DATA5(__c2h)                                \
+	LE_BITS_TO_4BYTE(__c2h + 0X08, 0, 8)
+#define C2H_SPECIAL_STATISTICS_SET_DATA5(__c2h, __value)                       \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X08, 0, 8, __value)
+#define C2H_SPECIAL_STATISTICS_GET_DATA6(__c2h)                                \
+	LE_BITS_TO_4BYTE(__c2h + 0X08, 8, 8)
+#define C2H_SPECIAL_STATISTICS_SET_DATA6(__c2h, __value)                       \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X08, 8, 8, __value)
+#define C2H_SPECIAL_STATISTICS_GET_DATA7(__c2h)                                \
+	LE_BITS_TO_4BYTE(__c2h + 0X08, 16, 8)
+#define C2H_SPECIAL_STATISTICS_SET_DATA7(__c2h, __value)                       \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X08, 16, 8, __value)
+#define C2H_SPECIAL_STATISTICS_GET_LEN(__c2h)                                  \
+	LE_BITS_TO_4BYTE(__c2h + 0X0C, 16, 8)
+#define C2H_SPECIAL_STATISTICS_SET_LEN(__c2h, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 16, 8, __value)
+#define C2H_SPECIAL_STATISTICS_GET_TRIGGER(__c2h)                              \
+	LE_BITS_TO_4BYTE(__c2h + 0X0C, 24, 8)
+#define C2H_SPECIAL_STATISTICS_SET_TRIGGER(__c2h, __value)                     \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 24, 8, __value)
+#define C2H_RA_PARA_RPT_GET_CMD_ID(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 0, 8)
+#define C2H_RA_PARA_RPT_SET_CMD_ID(__c2h, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 0, 8, __value)
+#define C2H_RA_PARA_RPT_GET_SEQ(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 8, 8)
+#define C2H_RA_PARA_RPT_SET_SEQ(__c2h, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 8, 8, __value)
+#define C2H_RA_PARA_RPT_GET_LEN(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X0C, 16, 8)
+#define C2H_RA_PARA_RPT_SET_LEN(__c2h, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 16, 8, __value)
+#define C2H_RA_PARA_RPT_GET_TRIGGER(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X0C, 24, 8)
+#define C2H_RA_PARA_RPT_SET_TRIGGER(__c2h, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 24, 8, __value)
+#define C2H_CUR_CHANNEL_GET_CMD_ID(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 0, 8)
+#define C2H_CUR_CHANNEL_SET_CMD_ID(__c2h, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 0, 8, __value)
+#define C2H_CUR_CHANNEL_GET_SEQ(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 8, 8)
+#define C2H_CUR_CHANNEL_SET_SEQ(__c2h, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 8, 8, __value)
+#define C2H_CUR_CHANNEL_GET_CHANNEL_NUM(__c2h)                                 \
+	LE_BITS_TO_4BYTE(__c2h + 0X00, 16, 8)
+#define C2H_CUR_CHANNEL_SET_CHANNEL_NUM(__c2h, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 16, 8, __value)
+#define C2H_CUR_CHANNEL_GET_LEN(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X0C, 16, 8)
+#define C2H_CUR_CHANNEL_SET_LEN(__c2h, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 16, 8, __value)
+#define C2H_CUR_CHANNEL_GET_TRIGGER(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X0C, 24, 8)
+#define C2H_CUR_CHANNEL_SET_TRIGGER(__c2h, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 24, 8, __value)
+#define C2H_GPIO_WAKEUP_GET_CMD_ID(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 0, 8)
+#define C2H_GPIO_WAKEUP_SET_CMD_ID(__c2h, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 0, 8, __value)
+#define C2H_GPIO_WAKEUP_GET_SEQ(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X00, 8, 8)
+#define C2H_GPIO_WAKEUP_SET_SEQ(__c2h, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X00, 8, 8, __value)
+#define C2H_GPIO_WAKEUP_GET_LEN(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X0C, 16, 8)
+#define C2H_GPIO_WAKEUP_SET_LEN(__c2h, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 16, 8, __value)
+#define C2H_GPIO_WAKEUP_GET_TRIGGER(__c2h) LE_BITS_TO_4BYTE(__c2h + 0X0C, 24, 8)
+#define C2H_GPIO_WAKEUP_SET_TRIGGER(__c2h, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__c2h + 0X0C, 24, 8, __value)
+#endif
diff --git a/drivers/staging/rtlwifi/halmac/halmac_original_h2c_nic.h b/drivers/staging/rtlwifi/halmac/halmac_original_h2c_nic.h
new file mode 100644
index 000000000000..db7aac4de843
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_original_h2c_nic.h
@@ -0,0 +1,1011 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HAL_ORIGINALH2CFORMAT_H2C_C2H_NIC_H_
+#define _HAL_ORIGINALH2CFORMAT_H2C_C2H_NIC_H_
+#define CMD_ID_ORIGINAL_H2C 0X00
+#define CMD_ID_H2C2H_LB 0X0
+#define CMD_ID_D0_SCAN_OFFLOAD_CTRL 0X06
+#define CMD_ID_RSVD_PAGE 0X0
+#define CMD_ID_MEDIA_STATUS_RPT 0X01
+#define CMD_ID_KEEP_ALIVE 0X03
+#define CMD_ID_DISCONNECT_DECISION 0X04
+#define CMD_ID_AP_OFFLOAD 0X08
+#define CMD_ID_BCN_RSVDPAGE 0X09
+#define CMD_ID_PROBE_RSP_RSVDPAGE 0X0A
+#define CMD_ID_SET_PWR_MODE 0X00
+#define CMD_ID_PS_TUNING_PARA 0X01
+#define CMD_ID_PS_TUNING_PARA_II 0X02
+#define CMD_ID_PS_LPS_PARA 0X03
+#define CMD_ID_P2P_PS_OFFLOAD 0X04
+#define CMD_ID_PS_SCAN_EN 0X05
+#define CMD_ID_SAP_PS 0X06
+#define CMD_ID_INACTIVE_PS 0X07
+#define CMD_ID_MACID_CFG 0X00
+#define CMD_ID_TXBF 0X01
+#define CMD_ID_RSSI_SETTING 0X02
+#define CMD_ID_AP_REQ_TXRPT 0X03
+#define CMD_ID_INIT_RATE_COLLECTION 0X04
+#define CMD_ID_IQK_OFFLOAD 0X05
+#define CMD_ID_MACID_CFG_3SS 0X06
+#define CMD_ID_RA_PARA_ADJUST 0X07
+#define CMD_ID_WWLAN 0X00
+#define CMD_ID_REMOTE_WAKE_CTRL 0X01
+#define CMD_ID_AOAC_GLOBAL_INFO 0X02
+#define CMD_ID_AOAC_RSVD_PAGE 0X03
+#define CMD_ID_AOAC_RSVD_PAGE2 0X04
+#define CMD_ID_D0_SCAN_OFFLOAD_INFO 0X05
+#define CMD_ID_CHANNEL_SWITCH_OFFLOAD 0X07
+#define CMD_ID_AOAC_RSVD_PAGE3 0X08
+#define CLASS_ORIGINAL_H2C 0X00
+#define CLASS_H2C2H_LB 0X07
+#define CLASS_D0_SCAN_OFFLOAD_CTRL 0X04
+#define CLASS_RSVD_PAGE 0X0
+#define CLASS_MEDIA_STATUS_RPT 0X0
+#define CLASS_KEEP_ALIVE 0X0
+#define CLASS_DISCONNECT_DECISION 0X0
+#define CLASS_AP_OFFLOAD 0X0
+#define CLASS_BCN_RSVDPAGE 0X0
+#define CLASS_PROBE_RSP_RSVDPAGE 0X0
+#define CLASS_SET_PWR_MODE 0X01
+#define CLASS_PS_TUNING_PARA 0X01
+#define CLASS_PS_TUNING_PARA_II 0X01
+#define CLASS_PS_LPS_PARA 0X01
+#define CLASS_P2P_PS_OFFLOAD 0X01
+#define CLASS_PS_SCAN_EN 0X1
+#define CLASS_SAP_PS 0X1
+#define CLASS_INACTIVE_PS 0X1
+#define CLASS_MACID_CFG 0X2
+#define CLASS_TXBF 0X2
+#define CLASS_RSSI_SETTING 0X2
+#define CLASS_AP_REQ_TXRPT 0X2
+#define CLASS_INIT_RATE_COLLECTION 0X2
+#define CLASS_IQK_OFFLOAD 0X2
+#define CLASS_MACID_CFG_3SS 0X2
+#define CLASS_RA_PARA_ADJUST 0X02
+#define CLASS_WWLAN 0X4
+#define CLASS_REMOTE_WAKE_CTRL 0X4
+#define CLASS_AOAC_GLOBAL_INFO 0X04
+#define CLASS_AOAC_RSVD_PAGE 0X04
+#define CLASS_AOAC_RSVD_PAGE2 0X04
+#define CLASS_D0_SCAN_OFFLOAD_INFO 0X04
+#define CLASS_CHANNEL_SWITCH_OFFLOAD 0X04
+#define CLASS_AOAC_RSVD_PAGE3 0X04
+#define ORIGINAL_H2C_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define ORIGINAL_H2C_SET_CMD_ID(__h2c, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define ORIGINAL_H2C_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define ORIGINAL_H2C_SET_CLASS(__h2c, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define H2C2H_LB_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define H2C2H_LB_SET_CMD_ID(__h2c, __value)                                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define H2C2H_LB_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define H2C2H_LB_SET_CLASS(__h2c, __value)                                     \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define H2C2H_LB_GET_SEQ(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 8)
+#define H2C2H_LB_SET_SEQ(__h2c, __value)                                       \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 8, __value)
+#define H2C2H_LB_GET_PAYLOAD1(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 16)
+#define H2C2H_LB_SET_PAYLOAD1(__h2c, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 16, __value)
+#define H2C2H_LB_GET_PAYLOAD2(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X04, 0, 32)
+#define H2C2H_LB_SET_PAYLOAD2(__h2c, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 0, 32, __value)
+#define D0_SCAN_OFFLOAD_CTRL_GET_CMD_ID(__h2c)                                 \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define D0_SCAN_OFFLOAD_CTRL_SET_CMD_ID(__h2c, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define D0_SCAN_OFFLOAD_CTRL_GET_CLASS(__h2c)                                  \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define D0_SCAN_OFFLOAD_CTRL_SET_CLASS(__h2c, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define D0_SCAN_OFFLOAD_CTRL_GET_D0_SCAN_FUN_EN(__h2c)                         \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 1)
+#define D0_SCAN_OFFLOAD_CTRL_SET_D0_SCAN_FUN_EN(__h2c, __value)                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 1, __value)
+#define D0_SCAN_OFFLOAD_CTRL_GET_RTD3FUN_EN(__h2c)                             \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 9, 1)
+#define D0_SCAN_OFFLOAD_CTRL_SET_RTD3FUN_EN(__h2c, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 9, 1, __value)
+#define D0_SCAN_OFFLOAD_CTRL_GET_U3_SCAN_FUN_EN(__h2c)                         \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 10, 1)
+#define D0_SCAN_OFFLOAD_CTRL_SET_U3_SCAN_FUN_EN(__h2c, __value)                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 10, 1, __value)
+#define D0_SCAN_OFFLOAD_CTRL_GET_NLO_FUN_EN(__h2c)                             \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 11, 1)
+#define D0_SCAN_OFFLOAD_CTRL_SET_NLO_FUN_EN(__h2c, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 11, 1, __value)
+#define D0_SCAN_OFFLOAD_CTRL_GET_IPS_DEPENDENT(__h2c)                          \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 12, 1)
+#define D0_SCAN_OFFLOAD_CTRL_SET_IPS_DEPENDENT(__h2c, __value)                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 12, 1, __value)
+#define D0_SCAN_OFFLOAD_CTRL_GET_LOC_PROBE_PACKET(__h2c)                       \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 17)
+#define D0_SCAN_OFFLOAD_CTRL_SET_LOC_PROBE_PACKET(__h2c, __value)              \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 17, __value)
+#define D0_SCAN_OFFLOAD_CTRL_GET_LOC_SCAN_INFO(__h2c)                          \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 24, 8)
+#define D0_SCAN_OFFLOAD_CTRL_SET_LOC_SCAN_INFO(__h2c, __value)                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 24, 8, __value)
+#define D0_SCAN_OFFLOAD_CTRL_GET_LOC_SSID_INFO(__h2c)                          \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 0, 8)
+#define D0_SCAN_OFFLOAD_CTRL_SET_LOC_SSID_INFO(__h2c, __value)                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 0, 8, __value)
+#define RSVD_PAGE_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define RSVD_PAGE_SET_CMD_ID(__h2c, __value)                                   \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define RSVD_PAGE_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define RSVD_PAGE_SET_CLASS(__h2c, __value)                                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define RSVD_PAGE_GET_LOC_PROBE_RSP(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 8)
+#define RSVD_PAGE_SET_LOC_PROBE_RSP(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 8, __value)
+#define RSVD_PAGE_GET_LOC_PS_POLL(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 8)
+#define RSVD_PAGE_SET_LOC_PS_POLL(__h2c, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 8, __value)
+#define RSVD_PAGE_GET_LOC_NULL_DATA(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 24, 8)
+#define RSVD_PAGE_SET_LOC_NULL_DATA(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 24, 8, __value)
+#define RSVD_PAGE_GET_LOC_QOS_NULL(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X04, 0, 8)
+#define RSVD_PAGE_SET_LOC_QOS_NULL(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 0, 8, __value)
+#define RSVD_PAGE_GET_LOC_BT_QOS_NULL(__h2c)                                   \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 8, 8)
+#define RSVD_PAGE_SET_LOC_BT_QOS_NULL(__h2c, __value)                          \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 8, 8, __value)
+#define RSVD_PAGE_GET_LOC_CTS2SELF(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X04, 16, 8)
+#define RSVD_PAGE_SET_LOC_CTS2SELF(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 16, 8, __value)
+#define RSVD_PAGE_GET_LOC_LTECOEX_QOSNULL(__h2c)                               \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 24, 8)
+#define RSVD_PAGE_SET_LOC_LTECOEX_QOSNULL(__h2c, __value)                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 24, 8, __value)
+#define MEDIA_STATUS_RPT_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define MEDIA_STATUS_RPT_SET_CMD_ID(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define MEDIA_STATUS_RPT_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define MEDIA_STATUS_RPT_SET_CLASS(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define MEDIA_STATUS_RPT_GET_OP_MODE(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 1)
+#define MEDIA_STATUS_RPT_SET_OP_MODE(__h2c, __value)                           \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 1, __value)
+#define MEDIA_STATUS_RPT_GET_MACID_IN(__h2c)                                   \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 9, 1)
+#define MEDIA_STATUS_RPT_SET_MACID_IN(__h2c, __value)                          \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 9, 1, __value)
+#define MEDIA_STATUS_RPT_GET_MACID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 8)
+#define MEDIA_STATUS_RPT_SET_MACID(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 8, __value)
+#define MEDIA_STATUS_RPT_GET_MACID_END(__h2c)                                  \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 24, 8)
+#define MEDIA_STATUS_RPT_SET_MACID_END(__h2c, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 24, 8, __value)
+#define KEEP_ALIVE_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define KEEP_ALIVE_SET_CMD_ID(__h2c, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define KEEP_ALIVE_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define KEEP_ALIVE_SET_CLASS(__h2c, __value)                                   \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define KEEP_ALIVE_GET_ENABLE(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 1)
+#define KEEP_ALIVE_SET_ENABLE(__h2c, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 1, __value)
+#define KEEP_ALIVE_GET_ADOPT_USER_SETTING(__h2c)                               \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 9, 1)
+#define KEEP_ALIVE_SET_ADOPT_USER_SETTING(__h2c, __value)                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 9, 1, __value)
+#define KEEP_ALIVE_GET_PKT_TYPE(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 10, 1)
+#define KEEP_ALIVE_SET_PKT_TYPE(__h2c, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 10, 1, __value)
+#define KEEP_ALIVE_GET_KEEP_ALIVE_CHECK_PERIOD(__h2c)                          \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 8)
+#define KEEP_ALIVE_SET_KEEP_ALIVE_CHECK_PERIOD(__h2c, __value)                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 8, __value)
+#define DISCONNECT_DECISION_GET_CMD_ID(__h2c)                                  \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define DISCONNECT_DECISION_SET_CMD_ID(__h2c, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define DISCONNECT_DECISION_GET_CLASS(__h2c)                                   \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define DISCONNECT_DECISION_SET_CLASS(__h2c, __value)                          \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define DISCONNECT_DECISION_GET_ENABLE(__h2c)                                  \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 1)
+#define DISCONNECT_DECISION_SET_ENABLE(__h2c, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 1, __value)
+#define DISCONNECT_DECISION_GET_ADOPT_USER_SETTING(__h2c)                      \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 9, 1)
+#define DISCONNECT_DECISION_SET_ADOPT_USER_SETTING(__h2c, __value)             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 9, 1, __value)
+#define DISCONNECT_DECISION_GET_TRY_OK_BCN_FAIL_COUNT_EN(__h2c)                \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 10, 1)
+#define DISCONNECT_DECISION_SET_TRY_OK_BCN_FAIL_COUNT_EN(__h2c, __value)       \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 10, 1, __value)
+#define DISCONNECT_DECISION_GET_DISCONNECT_EN(__h2c)                           \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 11, 1)
+#define DISCONNECT_DECISION_SET_DISCONNECT_EN(__h2c, __value)                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 11, 1, __value)
+#define DISCONNECT_DECISION_GET_DISCON_DECISION_CHECK_PERIOD(__h2c)            \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 8)
+#define DISCONNECT_DECISION_SET_DISCON_DECISION_CHECK_PERIOD(__h2c, __value)   \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 8, __value)
+#define DISCONNECT_DECISION_GET_TRY_PKT_NUM(__h2c)                             \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 24, 8)
+#define DISCONNECT_DECISION_SET_TRY_PKT_NUM(__h2c, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 24, 8, __value)
+#define DISCONNECT_DECISION_GET_TRY_OK_BCN_FAIL_COUNT_LIMIT(__h2c)             \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 0, 8)
+#define DISCONNECT_DECISION_SET_TRY_OK_BCN_FAIL_COUNT_LIMIT(__h2c, __value)    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 0, 8, __value)
+#define AP_OFFLOAD_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define AP_OFFLOAD_SET_CMD_ID(__h2c, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define AP_OFFLOAD_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define AP_OFFLOAD_SET_CLASS(__h2c, __value)                                   \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define AP_OFFLOAD_GET_ON(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 1)
+#define AP_OFFLOAD_SET_ON(__h2c, __value)                                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 1, __value)
+#define AP_OFFLOAD_GET_CFG_MIFI_PLATFORM(__h2c)                                \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 9, 1)
+#define AP_OFFLOAD_SET_CFG_MIFI_PLATFORM(__h2c, __value)                       \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 9, 1, __value)
+#define AP_OFFLOAD_GET_LINKED(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 10, 1)
+#define AP_OFFLOAD_SET_LINKED(__h2c, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 10, 1, __value)
+#define AP_OFFLOAD_GET_EN_AUTO_WAKE(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 11, 1)
+#define AP_OFFLOAD_SET_EN_AUTO_WAKE(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 11, 1, __value)
+#define AP_OFFLOAD_GET_WAKE_FLAG(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 12, 1)
+#define AP_OFFLOAD_SET_WAKE_FLAG(__h2c, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 12, 1, __value)
+#define AP_OFFLOAD_GET_HIDDEN_ROOT(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 1)
+#define AP_OFFLOAD_SET_HIDDEN_ROOT(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 1, __value)
+#define AP_OFFLOAD_GET_HIDDEN_VAP1(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 17, 1)
+#define AP_OFFLOAD_SET_HIDDEN_VAP1(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 17, 1, __value)
+#define AP_OFFLOAD_GET_HIDDEN_VAP2(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 18, 1)
+#define AP_OFFLOAD_SET_HIDDEN_VAP2(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 18, 1, __value)
+#define AP_OFFLOAD_GET_HIDDEN_VAP3(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 19, 1)
+#define AP_OFFLOAD_SET_HIDDEN_VAP3(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 19, 1, __value)
+#define AP_OFFLOAD_GET_HIDDEN_VAP4(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 20, 1)
+#define AP_OFFLOAD_SET_HIDDEN_VAP4(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 20, 1, __value)
+#define AP_OFFLOAD_GET_DENYANY_ROOT(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 24, 1)
+#define AP_OFFLOAD_SET_DENYANY_ROOT(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 24, 1, __value)
+#define AP_OFFLOAD_GET_DENYANY_VAP1(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 25, 1)
+#define AP_OFFLOAD_SET_DENYANY_VAP1(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 25, 1, __value)
+#define AP_OFFLOAD_GET_DENYANY_VAP2(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 26, 1)
+#define AP_OFFLOAD_SET_DENYANY_VAP2(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 26, 1, __value)
+#define AP_OFFLOAD_GET_DENYANY_VAP3(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 27, 1)
+#define AP_OFFLOAD_SET_DENYANY_VAP3(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 27, 1, __value)
+#define AP_OFFLOAD_GET_DENYANY_VAP4(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 28, 1)
+#define AP_OFFLOAD_SET_DENYANY_VAP4(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 28, 1, __value)
+#define AP_OFFLOAD_GET_WAIT_TBTT_CNT(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X04, 0, 8)
+#define AP_OFFLOAD_SET_WAIT_TBTT_CNT(__h2c, __value)                           \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 0, 8, __value)
+#define AP_OFFLOAD_GET_WAKE_TIMEOUT(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X04, 8, 8)
+#define AP_OFFLOAD_SET_WAKE_TIMEOUT(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 8, 8, __value)
+#define AP_OFFLOAD_GET_LEN_IV_PAIR(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X04, 16, 8)
+#define AP_OFFLOAD_SET_LEN_IV_PAIR(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 16, 8, __value)
+#define AP_OFFLOAD_GET_LEN_IV_GRP(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X04, 24, 8)
+#define AP_OFFLOAD_SET_LEN_IV_GRP(__h2c, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 24, 8, __value)
+#define BCN_RSVDPAGE_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define BCN_RSVDPAGE_SET_CMD_ID(__h2c, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define BCN_RSVDPAGE_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define BCN_RSVDPAGE_SET_CLASS(__h2c, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define BCN_RSVDPAGE_GET_LOC_ROOT(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 8)
+#define BCN_RSVDPAGE_SET_LOC_ROOT(__h2c, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 8, __value)
+#define BCN_RSVDPAGE_GET_LOC_VAP1(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 8)
+#define BCN_RSVDPAGE_SET_LOC_VAP1(__h2c, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 8, __value)
+#define BCN_RSVDPAGE_GET_LOC_VAP2(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 24, 8)
+#define BCN_RSVDPAGE_SET_LOC_VAP2(__h2c, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 24, 8, __value)
+#define BCN_RSVDPAGE_GET_LOC_VAP3(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X04, 0, 8)
+#define BCN_RSVDPAGE_SET_LOC_VAP3(__h2c, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 0, 8, __value)
+#define BCN_RSVDPAGE_GET_LOC_VAP4(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X04, 8, 8)
+#define BCN_RSVDPAGE_SET_LOC_VAP4(__h2c, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 8, 8, __value)
+#define PROBE_RSP_RSVDPAGE_GET_CMD_ID(__h2c)                                   \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define PROBE_RSP_RSVDPAGE_SET_CMD_ID(__h2c, __value)                          \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define PROBE_RSP_RSVDPAGE_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define PROBE_RSP_RSVDPAGE_SET_CLASS(__h2c, __value)                           \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define PROBE_RSP_RSVDPAGE_GET_LOC_ROOT(__h2c)                                 \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 8)
+#define PROBE_RSP_RSVDPAGE_SET_LOC_ROOT(__h2c, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 8, __value)
+#define PROBE_RSP_RSVDPAGE_GET_LOC_VAP1(__h2c)                                 \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 8)
+#define PROBE_RSP_RSVDPAGE_SET_LOC_VAP1(__h2c, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 8, __value)
+#define PROBE_RSP_RSVDPAGE_GET_LOC_VAP2(__h2c)                                 \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 24, 8)
+#define PROBE_RSP_RSVDPAGE_SET_LOC_VAP2(__h2c, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 24, 8, __value)
+#define PROBE_RSP_RSVDPAGE_GET_LOC_VAP3(__h2c)                                 \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 0, 8)
+#define PROBE_RSP_RSVDPAGE_SET_LOC_VAP3(__h2c, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 0, 8, __value)
+#define PROBE_RSP_RSVDPAGE_GET_LOC_VAP4(__h2c)                                 \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 8, 8)
+#define PROBE_RSP_RSVDPAGE_SET_LOC_VAP4(__h2c, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 8, 8, __value)
+#define SET_PWR_MODE_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define SET_PWR_MODE_SET_CMD_ID(__h2c, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define SET_PWR_MODE_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define SET_PWR_MODE_SET_CLASS(__h2c, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define SET_PWR_MODE_GET_MODE(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 7)
+#define SET_PWR_MODE_SET_MODE(__h2c, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 7, __value)
+#define SET_PWR_MODE_GET_CLK_REQUEST(__h2c)                                    \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 15, 1)
+#define SET_PWR_MODE_SET_CLK_REQUEST(__h2c, __value)                           \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 15, 1, __value)
+#define SET_PWR_MODE_GET_RLBM(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 4)
+#define SET_PWR_MODE_SET_RLBM(__h2c, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 4, __value)
+#define SET_PWR_MODE_GET_SMART_PS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 20, 4)
+#define SET_PWR_MODE_SET_SMART_PS(__h2c, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 20, 4, __value)
+#define SET_PWR_MODE_GET_AWAKE_INTERVAL(__h2c)                                 \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 24, 8)
+#define SET_PWR_MODE_SET_AWAKE_INTERVAL(__h2c, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 24, 8, __value)
+#define SET_PWR_MODE_GET_B_ALL_QUEUE_UAPSD(__h2c)                              \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 0, 1)
+#define SET_PWR_MODE_SET_B_ALL_QUEUE_UAPSD(__h2c, __value)                     \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 0, 1, __value)
+#define SET_PWR_MODE_GET_BCN_EARLY_RPT(__h2c)                                  \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 2, 1)
+#define SET_PWR_MODE_SET_BCN_EARLY_RPT(__h2c, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 2, 1, __value)
+#define SET_PWR_MODE_GET_PORT_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X04, 5, 3)
+#define SET_PWR_MODE_SET_PORT_ID(__h2c, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 5, 3, __value)
+#define SET_PWR_MODE_GET_PWR_STATE(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X04, 8, 8)
+#define SET_PWR_MODE_SET_PWR_STATE(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 8, 8, __value)
+#define SET_PWR_MODE_GET_LOW_POWER_RX_BCN(__h2c)                               \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 16, 1)
+#define SET_PWR_MODE_SET_LOW_POWER_RX_BCN(__h2c, __value)                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 16, 1, __value)
+#define SET_PWR_MODE_GET_ANT_AUTO_SWITCH(__h2c)                                \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 17, 1)
+#define SET_PWR_MODE_SET_ANT_AUTO_SWITCH(__h2c, __value)                       \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 17, 1, __value)
+#define SET_PWR_MODE_GET_PS_ALLOW_BT_HIGH_PRIORITY(__h2c)                      \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 18, 1)
+#define SET_PWR_MODE_SET_PS_ALLOW_BT_HIGH_PRIORITY(__h2c, __value)             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 18, 1, __value)
+#define SET_PWR_MODE_GET_PROTECT_BCN(__h2c)                                    \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 19, 1)
+#define SET_PWR_MODE_SET_PROTECT_BCN(__h2c, __value)                           \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 19, 1, __value)
+#define SET_PWR_MODE_GET_SILENCE_PERIOD(__h2c)                                 \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 20, 1)
+#define SET_PWR_MODE_SET_SILENCE_PERIOD(__h2c, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 20, 1, __value)
+#define SET_PWR_MODE_GET_FAST_BT_CONNECT(__h2c)                                \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 21, 1)
+#define SET_PWR_MODE_SET_FAST_BT_CONNECT(__h2c, __value)                       \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 21, 1, __value)
+#define SET_PWR_MODE_GET_TWO_ANTENNA_EN(__h2c)                                 \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 22, 1)
+#define SET_PWR_MODE_SET_TWO_ANTENNA_EN(__h2c, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 22, 1, __value)
+#define SET_PWR_MODE_GET_ADOPT_USER_SETTING(__h2c)                             \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 24, 1)
+#define SET_PWR_MODE_SET_ADOPT_USER_SETTING(__h2c, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 24, 1, __value)
+#define SET_PWR_MODE_GET_DRV_BCN_EARLY_SHIFT(__h2c)                            \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 25, 3)
+#define SET_PWR_MODE_SET_DRV_BCN_EARLY_SHIFT(__h2c, __value)                   \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 25, 3, __value)
+#define SET_PWR_MODE_GET_DRV_BCN_EARLY_SHIFT2(__h2c)                           \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 28, 4)
+#define SET_PWR_MODE_SET_DRV_BCN_EARLY_SHIFT2(__h2c, __value)                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 28, 4, __value)
+#define PS_TUNING_PARA_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define PS_TUNING_PARA_SET_CMD_ID(__h2c, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define PS_TUNING_PARA_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define PS_TUNING_PARA_SET_CLASS(__h2c, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define PS_TUNING_PARA_GET_BCN_TO_LIMIT(__h2c)                                 \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 7)
+#define PS_TUNING_PARA_SET_BCN_TO_LIMIT(__h2c, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 7, __value)
+#define PS_TUNING_PARA_GET_DTIM_TIME_OUT(__h2c)                                \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 15, 1)
+#define PS_TUNING_PARA_SET_DTIM_TIME_OUT(__h2c, __value)                       \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 15, 1, __value)
+#define PS_TUNING_PARA_GET_PS_TIME_OUT(__h2c)                                  \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 4)
+#define PS_TUNING_PARA_SET_PS_TIME_OUT(__h2c, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 4, __value)
+#define PS_TUNING_PARA_GET_ADOPT(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 24, 8)
+#define PS_TUNING_PARA_SET_ADOPT(__h2c, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 24, 8, __value)
+#define PS_TUNING_PARA_II_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define PS_TUNING_PARA_II_SET_CMD_ID(__h2c, __value)                           \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define PS_TUNING_PARA_II_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define PS_TUNING_PARA_II_SET_CLASS(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define PS_TUNING_PARA_II_GET_BCN_TO_PERIOD(__h2c)                             \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 7)
+#define PS_TUNING_PARA_II_SET_BCN_TO_PERIOD(__h2c, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 7, __value)
+#define PS_TUNING_PARA_II_GET_ADOPT(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 15, 1)
+#define PS_TUNING_PARA_II_SET_ADOPT(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 15, 1, __value)
+#define PS_TUNING_PARA_II_GET_DRV_EARLY_IVL(__h2c)                             \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 8)
+#define PS_TUNING_PARA_II_SET_DRV_EARLY_IVL(__h2c, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 8, __value)
+#define PS_LPS_PARA_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define PS_LPS_PARA_SET_CMD_ID(__h2c, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define PS_LPS_PARA_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define PS_LPS_PARA_SET_CLASS(__h2c, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define PS_LPS_PARA_GET_LPS_CONTROL(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 8)
+#define PS_LPS_PARA_SET_LPS_CONTROL(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 8, __value)
+#define P2P_PS_OFFLOAD_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define P2P_PS_OFFLOAD_SET_CMD_ID(__h2c, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define P2P_PS_OFFLOAD_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define P2P_PS_OFFLOAD_SET_CLASS(__h2c, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define P2P_PS_OFFLOAD_GET_OFFLOAD_EN(__h2c)                                   \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 1)
+#define P2P_PS_OFFLOAD_SET_OFFLOAD_EN(__h2c, __value)                          \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 1, __value)
+#define P2P_PS_OFFLOAD_GET_ROLE(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 9, 1)
+#define P2P_PS_OFFLOAD_SET_ROLE(__h2c, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 9, 1, __value)
+#define P2P_PS_OFFLOAD_GET_CTWINDOW_EN(__h2c)                                  \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 10, 1)
+#define P2P_PS_OFFLOAD_SET_CTWINDOW_EN(__h2c, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 10, 1, __value)
+#define P2P_PS_OFFLOAD_GET_NOA0_EN(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 11, 1)
+#define P2P_PS_OFFLOAD_SET_NOA0_EN(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 11, 1, __value)
+#define P2P_PS_OFFLOAD_GET_NOA1_EN(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 12, 1)
+#define P2P_PS_OFFLOAD_SET_NOA1_EN(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 12, 1, __value)
+#define P2P_PS_OFFLOAD_GET_ALL_STA_SLEEP(__h2c)                                \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 13, 1)
+#define P2P_PS_OFFLOAD_SET_ALL_STA_SLEEP(__h2c, __value)                       \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 13, 1, __value)
+#define P2P_PS_OFFLOAD_GET_DISCOVERY(__h2c)                                    \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 14, 1)
+#define P2P_PS_OFFLOAD_SET_DISCOVERY(__h2c, __value)                           \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 14, 1, __value)
+#define PS_SCAN_EN_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define PS_SCAN_EN_SET_CMD_ID(__h2c, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define PS_SCAN_EN_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define PS_SCAN_EN_SET_CLASS(__h2c, __value)                                   \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define PS_SCAN_EN_GET_ENABLE(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 1)
+#define PS_SCAN_EN_SET_ENABLE(__h2c, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 1, __value)
+#define SAP_PS_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define SAP_PS_SET_CMD_ID(__h2c, __value)                                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define SAP_PS_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define SAP_PS_SET_CLASS(__h2c, __value)                                       \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define SAP_PS_GET_ENABLE(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 1)
+#define SAP_PS_SET_ENABLE(__h2c, __value)                                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 1, __value)
+#define SAP_PS_GET_EN_PS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 9, 1)
+#define SAP_PS_SET_EN_PS(__h2c, __value)                                       \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 9, 1, __value)
+#define SAP_PS_GET_EN_LP_RX(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 10, 1)
+#define SAP_PS_SET_EN_LP_RX(__h2c, __value)                                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 10, 1, __value)
+#define SAP_PS_GET_MANUAL_32K(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 11, 1)
+#define SAP_PS_SET_MANUAL_32K(__h2c, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 11, 1, __value)
+#define SAP_PS_GET_DURATION(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 8)
+#define SAP_PS_SET_DURATION(__h2c, __value)                                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 8, __value)
+#define INACTIVE_PS_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define INACTIVE_PS_SET_CMD_ID(__h2c, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define INACTIVE_PS_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define INACTIVE_PS_SET_CLASS(__h2c, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define INACTIVE_PS_GET_ENABLE(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 1)
+#define INACTIVE_PS_SET_ENABLE(__h2c, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 1, __value)
+#define INACTIVE_PS_GET_IGNORE_PS_CONDITION(__h2c)                             \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 9, 1)
+#define INACTIVE_PS_SET_IGNORE_PS_CONDITION(__h2c, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 9, 1, __value)
+#define INACTIVE_PS_GET_FREQUENCY(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 8)
+#define INACTIVE_PS_SET_FREQUENCY(__h2c, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 8, __value)
+#define INACTIVE_PS_GET_DURATION(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 24, 8)
+#define INACTIVE_PS_SET_DURATION(__h2c, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 24, 8, __value)
+#define MACID_CFG_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define MACID_CFG_SET_CMD_ID(__h2c, __value)                                   \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define MACID_CFG_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define MACID_CFG_SET_CLASS(__h2c, __value)                                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define MACID_CFG_GET_MAC_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 8)
+#define MACID_CFG_SET_MAC_ID(__h2c, __value)                                   \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 8, __value)
+#define MACID_CFG_GET_RATE_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 5)
+#define MACID_CFG_SET_RATE_ID(__h2c, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 5, __value)
+#define MACID_CFG_GET_SGI(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 23, 1)
+#define MACID_CFG_SET_SGI(__h2c, __value)                                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 23, 1, __value)
+#define MACID_CFG_GET_BW(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 24, 2)
+#define MACID_CFG_SET_BW(__h2c, __value)                                       \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 24, 2, __value)
+#define MACID_CFG_GET_LDPC_CAP(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 26, 1)
+#define MACID_CFG_SET_LDPC_CAP(__h2c, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 26, 1, __value)
+#define MACID_CFG_GET_NO_UPDATE(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 27, 1)
+#define MACID_CFG_SET_NO_UPDATE(__h2c, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 27, 1, __value)
+#define MACID_CFG_GET_WHT_EN(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 28, 2)
+#define MACID_CFG_SET_WHT_EN(__h2c, __value)                                   \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 28, 2, __value)
+#define MACID_CFG_GET_DISPT(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 30, 1)
+#define MACID_CFG_SET_DISPT(__h2c, __value)                                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 30, 1, __value)
+#define MACID_CFG_GET_DISRA(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 31, 1)
+#define MACID_CFG_SET_DISRA(__h2c, __value)                                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 31, 1, __value)
+#define MACID_CFG_GET_RATE_MASK7_0(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X04, 0, 8)
+#define MACID_CFG_SET_RATE_MASK7_0(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 0, 8, __value)
+#define MACID_CFG_GET_RATE_MASK15_8(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X04, 8, 8)
+#define MACID_CFG_SET_RATE_MASK15_8(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 8, 8, __value)
+#define MACID_CFG_GET_RATE_MASK23_16(__h2c)                                    \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 16, 8)
+#define MACID_CFG_SET_RATE_MASK23_16(__h2c, __value)                           \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 16, 8, __value)
+#define MACID_CFG_GET_RATE_MASK31_24(__h2c)                                    \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 24, 8)
+#define MACID_CFG_SET_RATE_MASK31_24(__h2c, __value)                           \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 24, 8, __value)
+#define TXBF_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define TXBF_SET_CMD_ID(__h2c, __value)                                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define TXBF_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define TXBF_SET_CLASS(__h2c, __value)                                         \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define TXBF_GET_NDPA0_HEAD_PAGE(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 8)
+#define TXBF_SET_NDPA0_HEAD_PAGE(__h2c, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 8, __value)
+#define TXBF_GET_NDPA1_HEAD_PAGE(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 8)
+#define TXBF_SET_NDPA1_HEAD_PAGE(__h2c, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 8, __value)
+#define TXBF_GET_PERIOD_0(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 24, 8)
+#define TXBF_SET_PERIOD_0(__h2c, __value)                                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 24, 8, __value)
+#define RSSI_SETTING_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define RSSI_SETTING_SET_CMD_ID(__h2c, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define RSSI_SETTING_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define RSSI_SETTING_SET_CLASS(__h2c, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define RSSI_SETTING_GET_MAC_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 8)
+#define RSSI_SETTING_SET_MAC_ID(__h2c, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 8, __value)
+#define RSSI_SETTING_GET_RSSI(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 24, 7)
+#define RSSI_SETTING_SET_RSSI(__h2c, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 24, 7, __value)
+#define RSSI_SETTING_GET_RA_INFO(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X04, 0, 8)
+#define RSSI_SETTING_SET_RA_INFO(__h2c, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 0, 8, __value)
+#define AP_REQ_TXRPT_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define AP_REQ_TXRPT_SET_CMD_ID(__h2c, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define AP_REQ_TXRPT_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define AP_REQ_TXRPT_SET_CLASS(__h2c, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define AP_REQ_TXRPT_GET_STA1_MACID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 8)
+#define AP_REQ_TXRPT_SET_STA1_MACID(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 8, __value)
+#define AP_REQ_TXRPT_GET_STA2_MACID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 8)
+#define AP_REQ_TXRPT_SET_STA2_MACID(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 8, __value)
+#define AP_REQ_TXRPT_GET_RTY_OK_TOTAL(__h2c)                                   \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 24, 1)
+#define AP_REQ_TXRPT_SET_RTY_OK_TOTAL(__h2c, __value)                          \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 24, 1, __value)
+#define AP_REQ_TXRPT_GET_RTY_CNT_MACID(__h2c)                                  \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 25, 1)
+#define AP_REQ_TXRPT_SET_RTY_CNT_MACID(__h2c, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 25, 1, __value)
+#define INIT_RATE_COLLECTION_GET_CMD_ID(__h2c)                                 \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define INIT_RATE_COLLECTION_SET_CMD_ID(__h2c, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define INIT_RATE_COLLECTION_GET_CLASS(__h2c)                                  \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define INIT_RATE_COLLECTION_SET_CLASS(__h2c, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define INIT_RATE_COLLECTION_GET_STA1_MACID(__h2c)                             \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 8)
+#define INIT_RATE_COLLECTION_SET_STA1_MACID(__h2c, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 8, __value)
+#define INIT_RATE_COLLECTION_GET_STA2_MACID(__h2c)                             \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 8)
+#define INIT_RATE_COLLECTION_SET_STA2_MACID(__h2c, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 8, __value)
+#define INIT_RATE_COLLECTION_GET_STA3_MACID(__h2c)                             \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 24, 8)
+#define INIT_RATE_COLLECTION_SET_STA3_MACID(__h2c, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 24, 8, __value)
+#define INIT_RATE_COLLECTION_GET_STA4_MACID(__h2c)                             \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 0, 8)
+#define INIT_RATE_COLLECTION_SET_STA4_MACID(__h2c, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 0, 8, __value)
+#define INIT_RATE_COLLECTION_GET_STA5_MACID(__h2c)                             \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 8, 8)
+#define INIT_RATE_COLLECTION_SET_STA5_MACID(__h2c, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 8, 8, __value)
+#define INIT_RATE_COLLECTION_GET_STA6_MACID(__h2c)                             \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 16, 8)
+#define INIT_RATE_COLLECTION_SET_STA6_MACID(__h2c, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 16, 8, __value)
+#define INIT_RATE_COLLECTION_GET_STA7_MACID(__h2c)                             \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 24, 8)
+#define INIT_RATE_COLLECTION_SET_STA7_MACID(__h2c, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 24, 8, __value)
+#define IQK_OFFLOAD_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define IQK_OFFLOAD_SET_CMD_ID(__h2c, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define IQK_OFFLOAD_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define IQK_OFFLOAD_SET_CLASS(__h2c, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define IQK_OFFLOAD_GET_CHANNEL(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 8)
+#define IQK_OFFLOAD_SET_CHANNEL(__h2c, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 8, __value)
+#define IQK_OFFLOAD_GET_BWBAND(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 8)
+#define IQK_OFFLOAD_SET_BWBAND(__h2c, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 8, __value)
+#define IQK_OFFLOAD_GET_EXTPALNA(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 24, 8)
+#define IQK_OFFLOAD_SET_EXTPALNA(__h2c, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 24, 8, __value)
+#define MACID_CFG_3SS_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define MACID_CFG_3SS_SET_CMD_ID(__h2c, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define MACID_CFG_3SS_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define MACID_CFG_3SS_SET_CLASS(__h2c, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define MACID_CFG_3SS_GET_MACID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 8)
+#define MACID_CFG_3SS_SET_MACID(__h2c, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 8, __value)
+#define MACID_CFG_3SS_GET_RATE_MASK_39_32(__h2c)                               \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 0, 8)
+#define MACID_CFG_3SS_SET_RATE_MASK_39_32(__h2c, __value)                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 0, 8, __value)
+#define MACID_CFG_3SS_GET_RATE_MASK_47_40(__h2c)                               \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 8, 8)
+#define MACID_CFG_3SS_SET_RATE_MASK_47_40(__h2c, __value)                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 8, 8, __value)
+#define RA_PARA_ADJUST_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define RA_PARA_ADJUST_SET_CMD_ID(__h2c, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define RA_PARA_ADJUST_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define RA_PARA_ADJUST_SET_CLASS(__h2c, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define RA_PARA_ADJUST_GET_MAC_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 8)
+#define RA_PARA_ADJUST_SET_MAC_ID(__h2c, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 8, __value)
+#define RA_PARA_ADJUST_GET_PARAMETER_INDEX(__h2c)                              \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 8)
+#define RA_PARA_ADJUST_SET_PARAMETER_INDEX(__h2c, __value)                     \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 8, __value)
+#define RA_PARA_ADJUST_GET_RATE_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 24, 8)
+#define RA_PARA_ADJUST_SET_RATE_ID(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 24, 8, __value)
+#define RA_PARA_ADJUST_GET_VALUE_BYTE0(__h2c)                                  \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 0, 8)
+#define RA_PARA_ADJUST_SET_VALUE_BYTE0(__h2c, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 0, 8, __value)
+#define RA_PARA_ADJUST_GET_VALUE_BYTE1(__h2c)                                  \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 8, 8)
+#define RA_PARA_ADJUST_SET_VALUE_BYTE1(__h2c, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 8, 8, __value)
+#define RA_PARA_ADJUST_GET_ASK_FW_FOR_FW_PARA(__h2c)                           \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 16, 8)
+#define RA_PARA_ADJUST_SET_ASK_FW_FOR_FW_PARA(__h2c, __value)                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 16, 8, __value)
+#define WWLAN_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define WWLAN_SET_CMD_ID(__h2c, __value)                                       \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define WWLAN_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define WWLAN_SET_CLASS(__h2c, __value)                                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define WWLAN_GET_FUNC_EN(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 1)
+#define WWLAN_SET_FUNC_EN(__h2c, __value)                                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 1, __value)
+#define WWLAN_GET_PATTERM_MAT_EN(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 9, 1)
+#define WWLAN_SET_PATTERM_MAT_EN(__h2c, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 9, 1, __value)
+#define WWLAN_GET_MAGIC_PKT_EN(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 10, 1)
+#define WWLAN_SET_MAGIC_PKT_EN(__h2c, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 10, 1, __value)
+#define WWLAN_GET_UNICAST_WAKEUP_EN(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 11, 1)
+#define WWLAN_SET_UNICAST_WAKEUP_EN(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 11, 1, __value)
+#define WWLAN_GET_ALL_PKT_DROP(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 12, 1)
+#define WWLAN_SET_ALL_PKT_DROP(__h2c, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 12, 1, __value)
+#define WWLAN_GET_GPIO_ACTIVE(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 13, 1)
+#define WWLAN_SET_GPIO_ACTIVE(__h2c, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 13, 1, __value)
+#define WWLAN_GET_REKEY_WAKEUP_EN(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 14, 1)
+#define WWLAN_SET_REKEY_WAKEUP_EN(__h2c, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 14, 1, __value)
+#define WWLAN_GET_DEAUTH_WAKEUP_EN(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 15, 1)
+#define WWLAN_SET_DEAUTH_WAKEUP_EN(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 15, 1, __value)
+#define WWLAN_GET_GPIO_NUM(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 7)
+#define WWLAN_SET_GPIO_NUM(__h2c, __value)                                     \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 7, __value)
+#define WWLAN_GET_DATAPIN_WAKEUP_EN(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 23, 1)
+#define WWLAN_SET_DATAPIN_WAKEUP_EN(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 23, 1, __value)
+#define WWLAN_GET_GPIO_DURATION(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 24, 8)
+#define WWLAN_SET_GPIO_DURATION(__h2c, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 24, 8, __value)
+#define WWLAN_GET_GPIO_PLUS_EN(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X04, 0, 1)
+#define WWLAN_SET_GPIO_PLUS_EN(__h2c, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 0, 1, __value)
+#define WWLAN_GET_GPIO_PULSE_COUNT(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X04, 1, 7)
+#define WWLAN_SET_GPIO_PULSE_COUNT(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 1, 7, __value)
+#define WWLAN_GET_DISABLE_UPHY(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X04, 8, 1)
+#define WWLAN_SET_DISABLE_UPHY(__h2c, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 8, 1, __value)
+#define WWLAN_GET_HST2DEV_EN(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X04, 9, 1)
+#define WWLAN_SET_HST2DEV_EN(__h2c, __value)                                   \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 9, 1, __value)
+#define WWLAN_GET_GPIO_DURATION_MS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X04, 10, 1)
+#define WWLAN_SET_GPIO_DURATION_MS(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 10, 1, __value)
+#define REMOTE_WAKE_CTRL_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define REMOTE_WAKE_CTRL_SET_CMD_ID(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define REMOTE_WAKE_CTRL_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define REMOTE_WAKE_CTRL_SET_CLASS(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define REMOTE_WAKE_CTRL_GET_REMOTE_WAKE_CTRL_EN(__h2c)                        \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 1)
+#define REMOTE_WAKE_CTRL_SET_REMOTE_WAKE_CTRL_EN(__h2c, __value)               \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 1, __value)
+#define REMOTE_WAKE_CTRL_GET_ARP_EN(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 9, 1)
+#define REMOTE_WAKE_CTRL_SET_ARP_EN(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 9, 1, __value)
+#define REMOTE_WAKE_CTRL_GET_NDP_EN(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 10, 1)
+#define REMOTE_WAKE_CTRL_SET_NDP_EN(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 10, 1, __value)
+#define REMOTE_WAKE_CTRL_GET_GTK_EN(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 11, 1)
+#define REMOTE_WAKE_CTRL_SET_GTK_EN(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 11, 1, __value)
+#define REMOTE_WAKE_CTRL_GET_NLO_OFFLOAD_EN(__h2c)                             \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 12, 1)
+#define REMOTE_WAKE_CTRL_SET_NLO_OFFLOAD_EN(__h2c, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 12, 1, __value)
+#define REMOTE_WAKE_CTRL_GET_REAL_WOW_V1_EN(__h2c)                             \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 13, 1)
+#define REMOTE_WAKE_CTRL_SET_REAL_WOW_V1_EN(__h2c, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 13, 1, __value)
+#define REMOTE_WAKE_CTRL_GET_REAL_WOW_V2_EN(__h2c)                             \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 14, 1)
+#define REMOTE_WAKE_CTRL_SET_REAL_WOW_V2_EN(__h2c, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 14, 1, __value)
+#define REMOTE_WAKE_CTRL_GET_FW_UNICAST(__h2c)                                 \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 15, 1)
+#define REMOTE_WAKE_CTRL_SET_FW_UNICAST(__h2c, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 15, 1, __value)
+#define REMOTE_WAKE_CTRL_GET_P2P_OFFLOAD_EN(__h2c)                             \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 1)
+#define REMOTE_WAKE_CTRL_SET_P2P_OFFLOAD_EN(__h2c, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 1, __value)
+#define REMOTE_WAKE_CTRL_GET_RUNTIME_PM_EN(__h2c)                              \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 17, 1)
+#define REMOTE_WAKE_CTRL_SET_RUNTIME_PM_EN(__h2c, __value)                     \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 17, 1, __value)
+#define REMOTE_WAKE_CTRL_GET_NET_BIOS_DROP_EN(__h2c)                           \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 18, 1)
+#define REMOTE_WAKE_CTRL_SET_NET_BIOS_DROP_EN(__h2c, __value)                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 18, 1, __value)
+#define REMOTE_WAKE_CTRL_GET_ARP_ACTION(__h2c)                                 \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 24, 1)
+#define REMOTE_WAKE_CTRL_SET_ARP_ACTION(__h2c, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 24, 1, __value)
+#define REMOTE_WAKE_CTRL_GET_FW_PARSING_UNTIL_WAKEUP(__h2c)                    \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 28, 1)
+#define REMOTE_WAKE_CTRL_SET_FW_PARSING_UNTIL_WAKEUP(__h2c, __value)           \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 28, 1, __value)
+#define REMOTE_WAKE_CTRL_GET_FW_PARSING_AFTER_WAKEUP(__h2c)                    \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 29, 1)
+#define REMOTE_WAKE_CTRL_SET_FW_PARSING_AFTER_WAKEUP(__h2c, __value)           \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 29, 1, __value)
+#define AOAC_GLOBAL_INFO_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define AOAC_GLOBAL_INFO_SET_CMD_ID(__h2c, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define AOAC_GLOBAL_INFO_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define AOAC_GLOBAL_INFO_SET_CLASS(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define AOAC_GLOBAL_INFO_GET_PAIR_WISE_ENC_ALG(__h2c)                          \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 8)
+#define AOAC_GLOBAL_INFO_SET_PAIR_WISE_ENC_ALG(__h2c, __value)                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 8, __value)
+#define AOAC_GLOBAL_INFO_GET_GROUP_ENC_ALG(__h2c)                              \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 8)
+#define AOAC_GLOBAL_INFO_SET_GROUP_ENC_ALG(__h2c, __value)                     \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 8, __value)
+#define AOAC_RSVD_PAGE_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define AOAC_RSVD_PAGE_SET_CMD_ID(__h2c, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define AOAC_RSVD_PAGE_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define AOAC_RSVD_PAGE_SET_CLASS(__h2c, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define AOAC_RSVD_PAGE_GET_LOC_REMOTE_CTRL_INFO(__h2c)                         \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 8)
+#define AOAC_RSVD_PAGE_SET_LOC_REMOTE_CTRL_INFO(__h2c, __value)                \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 8, __value)
+#define AOAC_RSVD_PAGE_GET_LOC_ARP_RESPONSE(__h2c)                             \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 8)
+#define AOAC_RSVD_PAGE_SET_LOC_ARP_RESPONSE(__h2c, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 8, __value)
+#define AOAC_RSVD_PAGE_GET_LOC_NEIGHBOR_ADVERTISEMENT(__h2c)                   \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 24, 8)
+#define AOAC_RSVD_PAGE_SET_LOC_NEIGHBOR_ADVERTISEMENT(__h2c, __value)          \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 24, 8, __value)
+#define AOAC_RSVD_PAGE_GET_LOC_GTK_RSP(__h2c)                                  \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 0, 8)
+#define AOAC_RSVD_PAGE_SET_LOC_GTK_RSP(__h2c, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 0, 8, __value)
+#define AOAC_RSVD_PAGE_GET_LOC_GTK_INFO(__h2c)                                 \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 8, 8)
+#define AOAC_RSVD_PAGE_SET_LOC_GTK_INFO(__h2c, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 8, 8, __value)
+#define AOAC_RSVD_PAGE_GET_LOC_GTK_EXT_MEM(__h2c)                              \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 16, 8)
+#define AOAC_RSVD_PAGE_SET_LOC_GTK_EXT_MEM(__h2c, __value)                     \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 16, 8, __value)
+#define AOAC_RSVD_PAGE_GET_LOC_NDP_INFO(__h2c)                                 \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 24, 8)
+#define AOAC_RSVD_PAGE_SET_LOC_NDP_INFO(__h2c, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 24, 8, __value)
+#define AOAC_RSVD_PAGE2_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define AOAC_RSVD_PAGE2_SET_CMD_ID(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define AOAC_RSVD_PAGE2_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define AOAC_RSVD_PAGE2_SET_CLASS(__h2c, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define AOAC_RSVD_PAGE2_GET_LOC_ROUTER_SOLICATION(__h2c)                       \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 8)
+#define AOAC_RSVD_PAGE2_SET_LOC_ROUTER_SOLICATION(__h2c, __value)              \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 8, __value)
+#define AOAC_RSVD_PAGE2_GET_LOC_BUBBLE_PACKET(__h2c)                           \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 8)
+#define AOAC_RSVD_PAGE2_SET_LOC_BUBBLE_PACKET(__h2c, __value)                  \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 8, __value)
+#define AOAC_RSVD_PAGE2_GET_LOC_TEREDO_INFO(__h2c)                             \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 24, 8)
+#define AOAC_RSVD_PAGE2_SET_LOC_TEREDO_INFO(__h2c, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 24, 8, __value)
+#define AOAC_RSVD_PAGE2_GET_LOC_REALWOW_INFO(__h2c)                            \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 0, 8)
+#define AOAC_RSVD_PAGE2_SET_LOC_REALWOW_INFO(__h2c, __value)                   \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 0, 8, __value)
+#define AOAC_RSVD_PAGE2_GET_LOC_KEEP_ALIVE_PKT(__h2c)                          \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 8, 8)
+#define AOAC_RSVD_PAGE2_SET_LOC_KEEP_ALIVE_PKT(__h2c, __value)                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 8, 8, __value)
+#define AOAC_RSVD_PAGE2_GET_LOC_ACK_PATTERN(__h2c)                             \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 16, 8)
+#define AOAC_RSVD_PAGE2_SET_LOC_ACK_PATTERN(__h2c, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 16, 8, __value)
+#define AOAC_RSVD_PAGE2_GET_LOC_WAKEUP_PATTERN(__h2c)                          \
+	LE_BITS_TO_4BYTE(__h2c + 0X04, 24, 8)
+#define AOAC_RSVD_PAGE2_SET_LOC_WAKEUP_PATTERN(__h2c, __value)                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X04, 24, 8, __value)
+#define D0_SCAN_OFFLOAD_INFO_GET_CMD_ID(__h2c)                                 \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define D0_SCAN_OFFLOAD_INFO_SET_CMD_ID(__h2c, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define D0_SCAN_OFFLOAD_INFO_GET_CLASS(__h2c)                                  \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define D0_SCAN_OFFLOAD_INFO_SET_CLASS(__h2c, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define D0_SCAN_OFFLOAD_INFO_GET_LOC_CHANNEL_INFO(__h2c)                       \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 8)
+#define D0_SCAN_OFFLOAD_INFO_SET_LOC_CHANNEL_INFO(__h2c, __value)              \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 8, __value)
+#define CHANNEL_SWITCH_OFFLOAD_GET_CMD_ID(__h2c)                               \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define CHANNEL_SWITCH_OFFLOAD_SET_CMD_ID(__h2c, __value)                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define CHANNEL_SWITCH_OFFLOAD_GET_CLASS(__h2c)                                \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define CHANNEL_SWITCH_OFFLOAD_SET_CLASS(__h2c, __value)                       \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define CHANNEL_SWITCH_OFFLOAD_GET_CHANNEL_NUM(__h2c)                          \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 8)
+#define CHANNEL_SWITCH_OFFLOAD_SET_CHANNEL_NUM(__h2c, __value)                 \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 8, __value)
+#define CHANNEL_SWITCH_OFFLOAD_GET_EN_RFE(__h2c)                               \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 8)
+#define CHANNEL_SWITCH_OFFLOAD_SET_EN_RFE(__h2c, __value)                      \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 8, __value)
+#define CHANNEL_SWITCH_OFFLOAD_GET_RFE_TYPE(__h2c)                             \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 24, 8)
+#define CHANNEL_SWITCH_OFFLOAD_SET_RFE_TYPE(__h2c, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 24, 8, __value)
+#define AOAC_RSVD_PAGE3_GET_CMD_ID(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 0, 5)
+#define AOAC_RSVD_PAGE3_SET_CMD_ID(__h2c, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 0, 5, __value)
+#define AOAC_RSVD_PAGE3_GET_CLASS(__h2c) LE_BITS_TO_4BYTE(__h2c + 0X00, 5, 3)
+#define AOAC_RSVD_PAGE3_SET_CLASS(__h2c, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 5, 3, __value)
+#define AOAC_RSVD_PAGE3_GET_LOC_NLO_INFO(__h2c)                                \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 8, 8)
+#define AOAC_RSVD_PAGE3_SET_LOC_NLO_INFO(__h2c, __value)                       \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 8, 8, __value)
+#define AOAC_RSVD_PAGE3_GET_LOC_AOAC_REPORT(__h2c)                             \
+	LE_BITS_TO_4BYTE(__h2c + 0X00, 16, 8)
+#define AOAC_RSVD_PAGE3_SET_LOC_AOAC_REPORT(__h2c, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__h2c + 0X00, 16, 8, __value)
+#endif
diff --git a/drivers/staging/rtlwifi/halmac/halmac_pcie_reg.h b/drivers/staging/rtlwifi/halmac/halmac_pcie_reg.h
new file mode 100644
index 000000000000..41780676508e
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_pcie_reg.h
@@ -0,0 +1,28 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef __HALMAC_PCIE_REG_H__
+#define __HALMAC_PCIE_REG_H__
+
+#endif /* __HALMAC_PCIE_REG_H__ */
diff --git a/drivers/staging/rtlwifi/halmac/halmac_pwr_seq_cmd.h b/drivers/staging/rtlwifi/halmac/halmac_pwr_seq_cmd.h
new file mode 100644
index 000000000000..13a65a4754b0
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_pwr_seq_cmd.h
@@ -0,0 +1,116 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef HALMAC_POWER_SEQUENCE_CMD
+#define HALMAC_POWER_SEQUENCE_CMD
+
+#include "halmac_2_platform.h"
+#include "halmac_type.h"
+
+#define HALMAC_POLLING_READY_TIMEOUT_COUNT 20000
+
+/* The value of cmd : 4 bits */
+
+/* offset : the read register offset
+ * msk : the mask of the read value
+ * value : N/A, left by 0
+ * Note : dirver shall implement this function by read & msk
+ */
+#define HALMAC_PWR_CMD_READ 0x00
+/*
+ * offset: the read register offset
+ * msk: the mask of the write bits
+ * value: write value
+ * Note: driver shall implement this cmd by read & msk after write
+ */
+#define HALMAC_PWR_CMD_WRITE 0x01
+/* offset: the read register offset
+ * msk: the mask of the polled value
+ * value: the value to be polled, masked by the msd field.
+ * Note: driver shall implement this cmd by
+ * do{
+ * if( (Read(offset) & msk) == (value & msk) )
+ * break;
+ * } while(not timeout);
+ */
+#define HALMAC_PWR_CMD_POLLING 0x02
+/* offset: the value to delay
+ * msk: N/A
+ * value: the unit of delay, 0: us, 1: ms
+ */
+#define HALMAC_PWR_CMD_DELAY 0x03
+/* offset: N/A
+ * msk: N/A
+ * value: N/A
+ */
+#define HALMAC_PWR_CMD_END 0x04
+
+/* The value of base : 4 bits */
+
+/* define the base address of each block */
+#define HALMAC_PWR_BASEADDR_MAC 0x00
+#define HALMAC_PWR_BASEADDR_USB 0x01
+#define HALMAC_PWR_BASEADDR_PCIE 0x02
+#define HALMAC_PWR_BASEADDR_SDIO 0x03
+
+/* The value of interface_msk : 4 bits */
+#define HALMAC_PWR_INTF_SDIO_MSK BIT(0)
+#define HALMAC_PWR_INTF_USB_MSK BIT(1)
+#define HALMAC_PWR_INTF_PCI_MSK BIT(2)
+#define HALMAC_PWR_INTF_ALL_MSK (BIT(0) | BIT(1) | BIT(2) | BIT(3))
+
+/* The value of fab_msk : 4 bits */
+#define HALMAC_PWR_FAB_TSMC_MSK BIT(0)
+#define HALMAC_PWR_FAB_UMC_MSK BIT(1)
+#define HALMAC_PWR_FAB_ALL_MSK (BIT(0) | BIT(1) | BIT(2) | BIT(3))
+
+/* The value of cut_msk : 8 bits */
+#define HALMAC_PWR_CUT_TESTCHIP_MSK BIT(0)
+#define HALMAC_PWR_CUT_A_MSK BIT(1)
+#define HALMAC_PWR_CUT_B_MSK BIT(2)
+#define HALMAC_PWR_CUT_C_MSK BIT(3)
+#define HALMAC_PWR_CUT_D_MSK BIT(4)
+#define HALMAC_PWR_CUT_E_MSK BIT(5)
+#define HALMAC_PWR_CUT_F_MSK BIT(6)
+#define HALMAC_PWR_CUT_G_MSK BIT(7)
+#define HALMAC_PWR_CUT_ALL_MSK 0xFF
+
+enum halmac_pwrseq_cmd_delay_unit_ {
+	HALMAC_PWRSEQ_DELAY_US,
+	HALMAC_PWRSEQ_DELAY_MS,
+};
+
+/*Don't care endian issue, because element of pwer seq vector is fixed address*/
+struct halmac_wl_pwr_cfg_ {
+	u16 offset;
+	u8 cut_msk;
+	u8 fab_msk : 4;
+	u8 interface_msk : 4;
+	u8 base : 4;
+	u8 cmd : 4;
+	u8 msk;
+	u8 value;
+};
+
+#endif
diff --git a/drivers/staging/rtlwifi/halmac/halmac_reg2.h b/drivers/staging/rtlwifi/halmac/halmac_reg2.h
new file mode 100644
index 000000000000..bebf974ed949
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_reg2.h
@@ -0,0 +1,1132 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef __HALMAC_COM_REG_H__
+#define __HALMAC_COM_REG_H__
+/*-------------------------Modification Log-----------------------------------
+ *	For Page0, it is based on Combo_And_WL_Only_Page0_Reg.xls SVN524
+ *	The supported IC are 8723A, 8881A, 8723B, 8192E, 8881A
+ *	8812A and 8188E is not included in page0 register
+ *
+ *	For other pages, it is based on MAC_Register.doc SVN502
+ *	Most IC is the same with 8812A
+ *-------------------------Modification Log-----------------------------------
+ */
+
+/*--------------------------Include File--------------------------------------*/
+/*--------------------------Include File--------------------------------------*/
+
+#define REG_SYS_ISO_CTRL				0x0000
+
+#define REG_SDIO_TX_CTRL				0x10250000
+
+#define REG_SYS_FUNC_EN				0x0002
+#define REG_SYS_PW_CTRL				0x0004
+#define REG_SYS_CLK_CTRL				0x0008
+#define REG_SYS_EEPROM_CTRL				0x000A
+#define REG_EE_VPD					0x000C
+#define REG_SYS_SWR_CTRL1				0x0010
+#define REG_SYS_SWR_CTRL2				0x0014
+
+#define REG_SDIO_HIMR					0x10250014
+
+#define REG_SYS_SWR_CTRL3				0x0018
+
+#define REG_SDIO_HISR					0x10250018
+
+#define REG_RSV_CTRL					0x001C
+
+#define REG_SDIO_RX_REQ_LEN				0x1025001C
+
+#define REG_RF_CTRL					0x001F
+
+#define REG_SDIO_FREE_TXPG_SEQ_V1			0x1025001F
+
+#define REG_AFE_LDO_CTRL				0x0020
+
+#define REG_SDIO_FREE_TXPG				0x10250020
+
+#define REG_AFE_CTRL1					0x0024
+
+#define REG_SDIO_FREE_TXPG2				0x10250024
+
+#define REG_AFE_CTRL2					0x0028
+
+#define REG_SDIO_OQT_FREE_TXPG_V1			0x10250028
+
+#define REG_AFE_CTRL3					0x002C
+#define REG_EFUSE_CTRL					0x0030
+
+#define REG_SDIO_HTSFR_INFO				0x10250030
+
+#define REG_LDO_EFUSE_CTRL				0x0034
+#define REG_PWR_OPTION_CTRL				0x0038
+
+#define REG_SDIO_HCPWM1_V2				0x10250038
+#define REG_SDIO_HCPWM2_V2				0x1025003A
+
+#define REG_CAL_TIMER					0x003C
+#define REG_ACLK_MON					0x003E
+#define REG_GPIO_MUXCFG				0x0040
+
+#define REG_SDIO_INDIRECT_REG_CFG			0x10250040
+
+#define REG_GPIO_PIN_CTRL				0x0044
+
+#define REG_SDIO_INDIRECT_REG_DATA			0x10250044
+
+#define REG_GPIO_INTM					0x0048
+#define REG_LED_CFG					0x004C
+#define REG_FSIMR					0x0050
+#define REG_FSISR					0x0054
+#define REG_HSIMR					0x0058
+#define REG_HSISR					0x005C
+#define REG_GPIO_EXT_CTRL				0x0060
+
+#define REG_SDIO_H2C					0x10250060
+
+#define REG_PAD_CTRL1					0x0064
+
+#define REG_SDIO_C2H					0x10250064
+
+#define REG_WL_BT_PWR_CTRL				0x0068
+
+#define REG_SDM_DEBUG					0x006C
+
+#define REG_SYS_SDIO_CTRL				0x0070
+
+#define REG_HCI_OPT_CTRL				0x0074
+
+#define REG_AFE_CTRL4					0x0078
+
+#define REG_LDO_SWR_CTRL				0x007C
+
+#define REG_MCUFW_CTRL					0x0080
+
+#define REG_SDIO_HRPWM1				0x10250080
+#define REG_SDIO_HRPWM2				0x10250082
+
+#define REG_MCU_TST_CFG				0x0084
+
+#define REG_SDIO_HPS_CLKR				0x10250084
+#define REG_SDIO_BUS_CTRL				0x10250085
+
+#define REG_SDIO_HSUS_CTRL				0x10250086
+
+#define REG_HMEBOX_E0_E1				0x0088
+
+#define REG_SDIO_RESPONSE_TIMER			0x10250088
+
+#define REG_SDIO_CMD_CRC				0x1025008A
+
+#define REG_HMEBOX_E2_E3				0x008C
+#define REG_WLLPS_CTRL					0x0090
+
+#define REG_SDIO_HSISR					0x10250090
+#define REG_SDIO_HSIMR					0x10250091
+
+#define REG_AFE_CTRL5					0x0094
+
+#define REG_GPIO_DEBOUNCE_CTRL				0x0098
+#define REG_RPWM2					0x009C
+#define REG_SYSON_FSM_MON				0x00A0
+
+#define REG_AFE_CTRL6					0x00A4
+
+#define REG_PMC_DBG_CTRL1				0x00A8
+
+#define REG_AFE_CTRL7					0x00AC
+
+#define REG_HIMR0					0x00B0
+#define REG_HISR0					0x00B4
+#define REG_HIMR1					0x00B8
+#define REG_HISR1					0x00BC
+#define REG_DBG_PORT_SEL				0x00C0
+
+#define REG_SDIO_ERR_RPT				0x102500C0
+#define REG_SDIO_CMD_ERRCNT				0x102500C1
+#define REG_SDIO_DATA_ERRCNT				0x102500C2
+
+#define REG_PAD_CTRL2					0x00C4
+
+#define REG_SDIO_CMD_ERR_CONTENT			0x102500C4
+
+#define REG_SDIO_CRC_ERR_IDX				0x102500C9
+#define REG_SDIO_DATA_CRC				0x102500CA
+#define REG_SDIO_DATA_REPLY_TIME			0x102500CB
+
+#define REG_PMC_DBG_CTRL2				0x00CC
+#define REG_BIST_CTRL					0x00D0
+#define REG_BIST_RPT					0x00D4
+#define REG_MEM_CTRL					0x00D8
+
+#define REG_AFE_CTRL8					0x00DC
+
+#define REG_USB_SIE_INTF				0x00E0
+#define REG_PCIE_MIO_INTF				0x00E4
+#define REG_PCIE_MIO_INTD				0x00E8
+
+#define REG_WLRF1					0x00EC
+
+#define REG_SYS_CFG1					0x00F0
+#define REG_SYS_STATUS1				0x00F4
+#define REG_SYS_STATUS2				0x00F8
+#define REG_SYS_CFG2					0x00FC
+#define REG_CR						0x0100
+
+#define REG_PKT_BUFF_ACCESS_CTRL			0x0106
+
+#define REG_TSF_CLK_STATE				0x0108
+#define REG_TXDMA_PQ_MAP				0x010C
+#define REG_TRXFF_BNDY					0x0114
+
+#define REG_PTA_I2C_MBOX				0x0118
+
+#define REG_RXFF_BNDY					0x011C
+
+#define REG_FE1IMR					0x0120
+
+#define REG_FE1ISR					0x0124
+
+#define REG_CPWM					0x012C
+#define REG_FWIMR					0x0130
+#define REG_FWISR					0x0134
+#define REG_FTIMR					0x0138
+#define REG_FTISR					0x013C
+#define REG_PKTBUF_DBG_CTRL				0x0140
+#define REG_PKTBUF_DBG_DATA_L				0x0144
+#define REG_PKTBUF_DBG_DATA_H				0x0148
+#define REG_CPWM2					0x014C
+#define REG_TC0_CTRL					0x0150
+#define REG_TC1_CTRL					0x0154
+#define REG_TC2_CTRL					0x0158
+#define REG_TC3_CTRL					0x015C
+#define REG_TC4_CTRL					0x0160
+#define REG_TCUNIT_BASE				0x0164
+#define REG_TC5_CTRL					0x0168
+#define REG_TC6_CTRL					0x016C
+#define REG_MBIST_FAIL					0x0170
+#define REG_MBIST_START_PAUSE				0x0174
+#define REG_MBIST_DONE					0x0178
+
+#define REG_MBIST_FAIL_NRML				0x017C
+
+#define REG_AES_DECRPT_DATA				0x0180
+#define REG_AES_DECRPT_CFG				0x0184
+
+#define REG_TMETER					0x0190
+#define REG_OSC_32K_CTRL				0x0194
+#define REG_32K_CAL_REG1				0x0198
+#define REG_C2HEVT					0x01A0
+
+#define REG_C2HEVT_1					0x01A4
+#define REG_C2HEVT_2					0x01A8
+#define REG_C2HEVT_3					0x01AC
+
+#define REG_SW_DEFINED_PAGE1				0x01B8
+
+#define REG_MCUTST_I					0x01C0
+#define REG_MCUTST_II					0x01C4
+#define REG_FMETHR					0x01C8
+#define REG_HMETFR					0x01CC
+#define REG_HMEBOX0					0x01D0
+#define REG_HMEBOX1					0x01D4
+#define REG_HMEBOX2					0x01D8
+#define REG_HMEBOX3					0x01DC
+#define REG_LLT_INIT					0x01E0
+
+#define REG_LLT_INIT_ADDR				0x01E4
+
+#define REG_BB_ACCESS_CTRL				0x01E8
+#define REG_BB_ACCESS_DATA				0x01EC
+#define REG_HMEBOX_E0					0x01F0
+#define REG_HMEBOX_E1					0x01F4
+#define REG_HMEBOX_E2					0x01F8
+#define REG_HMEBOX_E3					0x01FC
+
+#define REG_FIFOPAGE_CTRL_1				0x0200
+
+#define REG_FIFOPAGE_CTRL_2				0x0204
+
+#define REG_AUTO_LLT_V1				0x0208
+
+#define REG_TXDMA_OFFSET_CHK				0x020C
+#define REG_TXDMA_STATUS				0x0210
+
+#define REG_TX_DMA_DBG					0x0214
+
+#define REG_TQPNT1					0x0218
+#define REG_TQPNT2					0x021C
+
+#define REG_TQPNT3					0x0220
+
+#define REG_TQPNT4					0x0224
+
+#define REG_RQPN_CTRL_1				0x0228
+#define REG_RQPN_CTRL_2				0x022C
+#define REG_FIFOPAGE_INFO_1				0x0230
+#define REG_FIFOPAGE_INFO_2				0x0234
+#define REG_FIFOPAGE_INFO_3				0x0238
+#define REG_FIFOPAGE_INFO_4				0x023C
+#define REG_FIFOPAGE_INFO_5				0x0240
+
+#define REG_H2C_HEAD					0x0244
+#define REG_H2C_TAIL					0x0248
+#define REG_H2C_READ_ADDR				0x024C
+#define REG_H2C_WR_ADDR				0x0250
+#define REG_H2C_INFO					0x0254
+
+#define REG_RXDMA_AGG_PG_TH				0x0280
+#define REG_RXPKT_NUM					0x0284
+#define REG_RXDMA_STATUS				0x0288
+#define REG_RXDMA_DPR					0x028C
+#define REG_RXDMA_MODE					0x0290
+#define REG_C2H_PKT					0x0294
+
+#define REG_FWFF_C2H					0x0298
+#define REG_FWFF_CTRL					0x029C
+#define REG_FWFF_PKT_INFO				0x02A0
+
+#define REG_PCIE_CTRL					0x0300
+
+#define REG_INT_MIG					0x0304
+#define REG_BCNQ_TXBD_DESA				0x0308
+#define REG_MGQ_TXBD_DESA				0x0310
+#define REG_VOQ_TXBD_DESA				0x0318
+#define REG_VIQ_TXBD_DESA				0x0320
+#define REG_BEQ_TXBD_DESA				0x0328
+#define REG_BKQ_TXBD_DESA				0x0330
+#define REG_RXQ_RXBD_DESA				0x0338
+#define REG_HI0Q_TXBD_DESA				0x0340
+#define REG_HI1Q_TXBD_DESA				0x0348
+#define REG_HI2Q_TXBD_DESA				0x0350
+#define REG_HI3Q_TXBD_DESA				0x0358
+#define REG_HI4Q_TXBD_DESA				0x0360
+#define REG_HI5Q_TXBD_DESA				0x0368
+#define REG_HI6Q_TXBD_DESA				0x0370
+#define REG_HI7Q_TXBD_DESA				0x0378
+#define REG_MGQ_TXBD_NUM				0x0380
+#define REG_RX_RXBD_NUM				0x0382
+#define REG_VOQ_TXBD_NUM				0x0384
+#define REG_VIQ_TXBD_NUM				0x0386
+#define REG_BEQ_TXBD_NUM				0x0388
+#define REG_BKQ_TXBD_NUM				0x038A
+#define REG_HI0Q_TXBD_NUM				0x038C
+#define REG_HI1Q_TXBD_NUM				0x038E
+#define REG_HI2Q_TXBD_NUM				0x0390
+#define REG_HI3Q_TXBD_NUM				0x0392
+#define REG_HI4Q_TXBD_NUM				0x0394
+#define REG_HI5Q_TXBD_NUM				0x0396
+#define REG_HI6Q_TXBD_NUM				0x0398
+#define REG_HI7Q_TXBD_NUM				0x039A
+#define REG_TSFTIMER_HCI				0x039C
+#define REG_BD_RWPTR_CLR				0x039C
+#define REG_VOQ_TXBD_IDX				0x03A0
+#define REG_VIQ_TXBD_IDX				0x03A4
+#define REG_BEQ_TXBD_IDX				0x03A8
+#define REG_BKQ_TXBD_IDX				0x03AC
+#define REG_MGQ_TXBD_IDX				0x03B0
+#define REG_RXQ_RXBD_IDX				0x03B4
+#define REG_HI0Q_TXBD_IDX				0x03B8
+#define REG_HI1Q_TXBD_IDX				0x03BC
+#define REG_HI2Q_TXBD_IDX				0x03C0
+#define REG_HI3Q_TXBD_IDX				0x03C4
+#define REG_HI4Q_TXBD_IDX				0x03C8
+#define REG_HI5Q_TXBD_IDX				0x03CC
+#define REG_HI6Q_TXBD_IDX				0x03D0
+#define REG_HI7Q_TXBD_IDX				0x03D4
+
+#define REG_DBG_SEL_V1					0x03D8
+
+#define REG_PCIE_HRPWM1_V1				0x03D9
+
+#define REG_PCIE_HCPWM1_V1				0x03DA
+
+#define REG_PCIE_CTRL2					0x03DB
+
+#define REG_PCIE_HRPWM2_V1				0x03DC
+
+#define REG_PCIE_HCPWM2_V1				0x03DE
+
+#define REG_PCIE_H2C_MSG_V1				0x03E0
+
+#define REG_PCIE_C2H_MSG_V1				0x03E4
+
+#define REG_DBI_WDATA_V1				0x03E8
+
+#define REG_DBI_RDATA_V1				0x03EC
+
+#define REG_DBI_FLAG_V1				0x03F0
+
+#define REG_MDIO_V1					0x03F4
+
+#define REG_PCIE_MIX_CFG				0x03F8
+
+#define REG_HCI_MIX_CFG				0x03FC
+
+#define REG_Q0_INFO					0x0400
+#define REG_Q1_INFO					0x0404
+#define REG_Q2_INFO					0x0408
+#define REG_Q3_INFO					0x040C
+#define REG_MGQ_INFO					0x0410
+#define REG_HIQ_INFO					0x0414
+#define REG_BCNQ_INFO					0x0418
+#define REG_TXPKT_EMPTY				0x041A
+#define REG_CPU_MGQ_INFO				0x041C
+#define REG_FWHW_TXQ_CTRL				0x0420
+
+#define REG_DATAFB_SEL					0x0423
+
+#define REG_BCNQ_BDNY_V1				0x0424
+
+#define REG_LIFETIME_EN				0x0426
+
+#define REG_SPEC_SIFS					0x0428
+#define REG_RETRY_LIMIT				0x042A
+#define REG_TXBF_CTRL					0x042C
+#define REG_DARFRC					0x0430
+#define REG_RARFRC					0x0438
+#define REG_RRSR					0x0440
+#define REG_ARFR0					0x0444
+#define REG_ARFR1_V1					0x044C
+#define REG_CCK_CHECK					0x0454
+
+#define REG_AMPDU_MAX_TIME_V1				0x0455
+
+#define REG_BCNQ1_BDNY_V1				0x0456
+
+#define REG_AMPDU_MAX_LENGTH				0x0458
+#define REG_ACQ_STOP					0x045C
+
+#define REG_NDPA_RATE					0x045D
+
+#define REG_TX_HANG_CTRL				0x045E
+#define REG_NDPA_OPT_CTRL				0x045F
+
+#define REG_RD_RESP_PKT_TH				0x0463
+#define REG_CMDQ_INFO					0x0464
+#define REG_Q4_INFO					0x0468
+#define REG_Q5_INFO					0x046C
+#define REG_Q6_INFO					0x0470
+#define REG_Q7_INFO					0x0474
+
+#define REG_WMAC_LBK_BUF_HD_V1				0x0478
+#define REG_MGQ_BDNY_V1				0x047A
+
+#define REG_TXRPT_CTRL					0x047C
+#define REG_INIRTS_RATE_SEL				0x0480
+#define REG_BASIC_CFEND_RATE				0x0481
+#define REG_STBC_CFEND_RATE				0x0482
+#define REG_DATA_SC					0x0483
+#define REG_MACID_SLEEP3				0x0484
+#define REG_MACID_SLEEP1				0x0488
+#define REG_ARFR2_V1					0x048C
+#define REG_ARFR3_V1					0x0494
+#define REG_ARFR4					0x049C
+#define REG_ARFR5					0x04A4
+#define REG_TXRPT_START_OFFSET				0x04AC
+
+#define REG_POWER_STAGE1				0x04B4
+
+#define REG_POWER_STAGE2				0x04B8
+
+#define REG_SW_AMPDU_BURST_MODE_CTRL			0x04BC
+#define REG_PKT_LIFE_TIME				0x04C0
+#define REG_STBC_SETTING				0x04C4
+#define REG_STBC_SETTING2				0x04C5
+#define REG_QUEUE_CTRL					0x04C6
+#define REG_SINGLE_AMPDU_CTRL				0x04C7
+#define REG_PROT_MODE_CTRL				0x04C8
+#define REG_BAR_MODE_CTRL				0x04CC
+#define REG_RA_TRY_RATE_AGG_LMT			0x04CF
+#define REG_MACID_SLEEP2				0x04D0
+#define REG_MACID_SLEEP				0x04D4
+
+#define REG_HW_SEQ0					0x04D8
+#define REG_HW_SEQ1					0x04DA
+#define REG_HW_SEQ2					0x04DC
+#define REG_HW_SEQ3					0x04DE
+
+#define REG_NULL_PKT_STATUS_V1				0x04E0
+
+#define REG_PTCL_ERR_STATUS				0x04E2
+
+#define REG_NULL_PKT_STATUS_EXTEND			0x04E3
+
+#define REG_VIDEO_ENHANCEMENT_FUN			0x04E4
+
+#define REG_BT_POLLUTE_PKT_CNT				0x04E8
+#define REG_PTCL_DBG					0x04EC
+
+#define REG_CPUMGQ_TIMER_CTRL2				0x04F4
+
+#define REG_DUMMY_PAGE4_V1				0x04FC
+#define REG_MOREDATA					0x04FE
+
+#define REG_EDCA_VO_PARAM				0x0500
+#define REG_EDCA_VI_PARAM				0x0504
+#define REG_EDCA_BE_PARAM				0x0508
+#define REG_EDCA_BK_PARAM				0x050C
+#define REG_BCNTCFG					0x0510
+#define REG_PIFS					0x0512
+#define REG_RDG_PIFS					0x0513
+#define REG_SIFS					0x0514
+#define REG_TSFTR_SYN_OFFSET				0x0518
+#define REG_AGGR_BREAK_TIME				0x051A
+#define REG_SLOT					0x051B
+#define REG_TX_PTCL_CTRL				0x0520
+#define REG_TXPAUSE					0x0522
+#define REG_DIS_TXREQ_CLR				0x0523
+#define REG_RD_CTRL					0x0524
+#define REG_MBSSID_CTRL				0x0526
+#define REG_P2PPS_CTRL					0x0527
+#define REG_PKT_LIFETIME_CTRL				0x0528
+#define REG_P2PPS_SPEC_STATE				0x052B
+
+#define REG_BAR_TX_CTRL				0x0530
+
+#define REG_QUEUE_INCOL_THR				0x0538
+#define REG_QUEUE_INCOL_EN				0x053C
+
+#define REG_TBTT_PROHIBIT				0x0540
+#define REG_P2PPS_STATE				0x0543
+#define REG_RD_NAV_NXT					0x0544
+#define REG_NAV_PROT_LEN				0x0546
+
+#define REG_BCN_CTRL					0x0550
+
+#define REG_BCN_CTRL_CLINT0				0x0551
+
+#define REG_MBID_NUM					0x0552
+#define REG_DUAL_TSF_RST				0x0553
+#define REG_MBSSID_BCN_SPACE				0x0554
+#define REG_DRVERLYINT					0x0558
+#define REG_BCNDMATIM					0x0559
+#define REG_ATIMWND					0x055A
+#define REG_USTIME_TSF					0x055C
+#define REG_BCN_MAX_ERR				0x055D
+#define REG_RXTSF_OFFSET_CCK				0x055E
+#define REG_RXTSF_OFFSET_OFDM				0x055F
+#define REG_TSFTR					0x0560
+
+#define REG_FREERUN_CNT				0x0568
+
+#define REG_ATIMWND1_V1				0x0570
+
+#define REG_TBTT_PROHIBIT_INFRA			0x0571
+
+#define REG_CTWND					0x0572
+#define REG_BCNIVLCUNT					0x0573
+#define REG_BCNDROPCTRL				0x0574
+#define REG_HGQ_TIMEOUT_PERIOD				0x0575
+
+#define REG_TXCMD_TIMEOUT_PERIOD			0x0576
+#define REG_MISC_CTRL					0x0577
+#define REG_BCN_CTRL_CLINT1				0x0578
+#define REG_BCN_CTRL_CLINT2				0x0579
+#define REG_BCN_CTRL_CLINT3				0x057A
+
+#define REG_EXTEND_CTRL				0x057B
+
+#define REG_P2PPS1_SPEC_STATE				0x057C
+#define REG_P2PPS1_STATE				0x057D
+#define REG_P2PPS2_SPEC_STATE				0x057E
+#define REG_P2PPS2_STATE				0x057F
+
+#define REG_PS_TIMER0					0x0580
+
+#define REG_PS_TIMER1					0x0584
+
+#define REG_PS_TIMER2					0x0588
+
+#define REG_TBTT_CTN_AREA				0x058C
+#define REG_FORCE_BCN_IFS				0x058E
+#define REG_TXOP_MIN					0x0590
+#define REG_PRE_BKF_TIME				0x0592
+#define REG_CROSS_TXOP_CTRL				0x0593
+
+#define REG_ATIMWND2					0x05A0
+#define REG_ATIMWND3					0x05A1
+#define REG_ATIMWND4					0x05A2
+#define REG_ATIMWND5					0x05A3
+#define REG_ATIMWND6					0x05A4
+#define REG_ATIMWND7					0x05A5
+#define REG_ATIMUGT					0x05A6
+#define REG_HIQ_NO_LMT_EN				0x05A7
+#define REG_DTIM_COUNTER_ROOT				0x05A8
+#define REG_DTIM_COUNTER_VAP1				0x05A9
+#define REG_DTIM_COUNTER_VAP2				0x05AA
+#define REG_DTIM_COUNTER_VAP3				0x05AB
+#define REG_DTIM_COUNTER_VAP4				0x05AC
+#define REG_DTIM_COUNTER_VAP5				0x05AD
+#define REG_DTIM_COUNTER_VAP6				0x05AE
+#define REG_DTIM_COUNTER_VAP7				0x05AF
+#define REG_DIS_ATIM					0x05B0
+
+#define REG_EARLY_128US				0x05B1
+#define REG_P2PPS1_CTRL				0x05B2
+#define REG_P2PPS2_CTRL				0x05B3
+#define REG_TIMER0_SRC_SEL				0x05B4
+#define REG_NOA_UNIT_SEL				0x05B5
+#define REG_P2POFF_DIS_TXTIME				0x05B7
+#define REG_MBSSID_BCN_SPACE2				0x05B8
+#define REG_MBSSID_BCN_SPACE3				0x05BC
+
+#define REG_ACMHWCTRL					0x05C0
+#define REG_ACMRSTCTRL					0x05C1
+#define REG_ACMAVG					0x05C2
+#define REG_VO_ADMTIME					0x05C4
+#define REG_VI_ADMTIME					0x05C6
+#define REG_BE_ADMTIME					0x05C8
+#define REG_EDCA_RANDOM_GEN				0x05CC
+#define REG_TXCMD_NOA_SEL				0x05CF
+#define REG_NOA_PARAM					0x05E0
+
+#define REG_P2P_RST					0x05F0
+#define REG_SCHEDULER_RST				0x05F1
+
+#define REG_SCH_TXCMD					0x05F8
+#define REG_PAGE5_DUMMY				0x05FC
+#define REG_WMAC_CR					0x0600
+
+#define REG_WMAC_FWPKT_CR				0x0601
+
+#define REG_BWOPMODE					0x0603
+
+#define REG_TCR					0x0604
+#define REG_RCR					0x0608
+#define REG_RX_PKT_LIMIT				0x060C
+#define REG_RX_DLK_TIME				0x060D
+#define REG_RX_DRVINFO_SZ				0x060F
+#define REG_MACID					0x0610
+#define REG_BSSID					0x0618
+#define REG_MAR					0x0620
+#define REG_MBIDCAMCFG_1				0x0628
+#define REG_MBIDCAMCFG_2				0x062C
+
+#define REG_WMAC_TCR_TSFT_OFS				0x0630
+#define REG_UDF_THSD					0x0632
+#define REG_ZLD_NUM					0x0633
+
+#define REG_STMP_THSD					0x0634
+#define REG_WMAC_TXTIMEOUT				0x0635
+#define REG_MCU_TEST_2_V1				0x0636
+
+#define REG_USTIME_EDCA				0x0638
+
+#define REG_MAC_SPEC_SIFS				0x063A
+#define REG_RESP_SIFS_CCK				0x063C
+#define REG_RESP_SIFS_OFDM				0x063E
+#define REG_ACKTO					0x0640
+#define REG_CTS2TO					0x0641
+#define REG_EIFS					0x0642
+
+#define REG_NAV_CTRL					0x0650
+#define REG_BACAMCMD					0x0654
+#define REG_BACAMCONTENT				0x0658
+#define REG_LBDLY					0x0660
+
+#define REG_WMAC_BACAM_RPMEN				0x0661
+
+#define REG_TX_RX					0x0662
+
+#define REG_WMAC_BITMAP_CTL				0x0663
+
+#define REG_RXERR_RPT					0x0664
+#define REG_WMAC_TRXPTCL_CTL				0x0668
+#define REG_CAMCMD					0x0670
+#define REG_CAMWRITE					0x0674
+#define REG_CAMREAD					0x0678
+#define REG_CAMDBG					0x067C
+#define REG_SECCFG					0x0680
+
+#define REG_RXFILTER_CATEGORY_1			0x0682
+#define REG_RXFILTER_ACTION_1				0x0683
+#define REG_RXFILTER_CATEGORY_2			0x0684
+#define REG_RXFILTER_ACTION_2				0x0685
+#define REG_RXFILTER_CATEGORY_3			0x0686
+#define REG_RXFILTER_ACTION_3				0x0687
+#define REG_RXFLTMAP3					0x0688
+#define REG_RXFLTMAP4					0x068A
+#define REG_RXFLTMAP5					0x068C
+#define REG_RXFLTMAP6					0x068E
+
+#define REG_WOW_CTRL					0x0690
+
+#define REG_NAN_RX_TSF_FILTER				0x0691
+
+#define REG_PS_RX_INFO					0x0692
+#define REG_WMMPS_UAPSD_TID				0x0693
+#define REG_LPNAV_CTRL					0x0694
+
+#define REG_WKFMCAM_CMD				0x0698
+#define REG_WKFMCAM_RWD				0x069C
+
+#define REG_RXFLTMAP0					0x06A0
+#define REG_RXFLTMAP1					0x06A2
+#define REG_RXFLTMAP					0x06A4
+#define REG_BCN_PSR_RPT				0x06A8
+
+#define REG_FLC_RPC					0x06AC
+#define REG_FLC_RPCT					0x06AD
+#define REG_FLC_PTS					0x06AE
+#define REG_FLC_TRPC					0x06AF
+
+#define REG_RXPKTMON_CTRL				0x06B0
+
+#define REG_STATE_MON					0x06B4
+
+#define REG_ERROR_MON					0x06B8
+#define REG_SEARCH_MACID				0x06BC
+
+#define REG_BT_COEX_TABLE				0x06C0
+
+#define REG_RXCMD_0					0x06D0
+#define REG_RXCMD_1					0x06D4
+
+#define REG_WMAC_RESP_TXINFO				0x06D8
+
+#define REG_BBPSF_CTRL					0x06DC
+
+#define REG_P2P_RX_BCN_NOA				0x06E0
+#define REG_ASSOCIATED_BFMER0_INFO			0x06E4
+#define REG_ASSOCIATED_BFMER1_INFO			0x06EC
+#define REG_TX_CSI_RPT_PARAM_BW20			0x06F4
+#define REG_TX_CSI_RPT_PARAM_BW40			0x06F8
+#define REG_TX_CSI_RPT_PARAM_BW80			0x06FC
+#define REG_MACID1					0x0700
+
+#define REG_BSSID1					0x0708
+
+#define REG_BCN_PSR_RPT1				0x0710
+#define REG_ASSOCIATED_BFMEE_SEL			0x0714
+#define REG_SND_PTCL_CTRL				0x0718
+#define REG_RX_CSI_RPT_INFO				0x071C
+#define REG_NS_ARP_CTRL				0x0720
+#define REG_NS_ARP_INFO				0x0724
+
+#define REG_BEAMFORMING_INFO_NSARP_V1			0x0728
+
+#define REG_BEAMFORMING_INFO_NSARP			0x072C
+
+#define REG_WMAC_RTX_CTX_SUBTYPE_CFG			0x0750
+
+#define REG_WMAC_SWAES_CFG				0x0760
+
+#define REG_BT_COEX_V2					0x0762
+
+#define REG_BT_COEX					0x0764
+
+#define REG_WLAN_ACT_MASK_CTRL				0x0768
+
+#define REG_BT_COEX_ENHANCED_INTR_CTRL			0x076E
+
+#define REG_BT_ACT_STATISTICS				0x0770
+
+#define REG_BT_STATISTICS_CONTROL_REGISTER		0x0778
+
+#define REG_BT_STATUS_REPORT_REGISTER			0x077C
+
+#define REG_BT_INTERRUPT_CONTROL_REGISTER		0x0780
+
+#define REG_WLAN_REPORT_TIME_OUT_CONTROL_REGISTER	0x0784
+
+#define REG_BT_ISOLATION_TABLE_REGISTER_REGISTER	0x0785
+
+#define REG_BT_INTERRUPT_STATUS_REGISTER		0x078F
+
+#define REG_BT_TDMA_TIME_REGISTER			0x0790
+
+#define REG_BT_ACT_REGISTER				0x0794
+
+#define REG_OBFF_CTRL_BASIC				0x0798
+
+#define REG_OBFF_CTRL2_TIMER				0x079C
+
+#define REG_LTR_CTRL_BASIC				0x07A0
+
+#define REG_LTR_CTRL2_TIMER_THRESHOLD			0x07A4
+
+#define REG_LTR_IDLE_LATENCY_V1			0x07A8
+#define REG_LTR_ACTIVE_LATENCY_V1			0x07AC
+
+#define REG_ANTENNA_TRAINING_CONTROL_REGISTER		0x07B0
+
+#define REG_WMAC_PKTCNT_RWD				0x07B8
+#define REG_WMAC_PKTCNT_CTRL				0x07BC
+
+#define REG_IQ_DUMP					0x07C0
+
+#define REG_WMAC_FTM_CTL				0x07CC
+
+#define REG_WMAC_IQ_MDPK_FUNC				0x07CE
+
+#define REG_WMAC_OPTION_FUNCTION			0x07D0
+
+#define REG_RX_FILTER_FUNCTION				0x07DA
+
+#define REG_NDP_SIG					0x07E0
+#define REG_TXCMD_INFO_FOR_RSP_PKT			0x07E4
+
+#define REG_RTS_ADDRESS_0				0x07F0
+
+#define REG_RTS_ADDRESS_1				0x07F8
+
+#define REG__RPFM_MAP1					0x07FE
+
+#define REG_SYS_CFG3					0x1000
+#define REG_SYS_CFG4					0x1034
+
+#define REG_SYS_CFG5					0x1070
+
+#define REG_CPU_DMEM_CON				0x1080
+
+#define REG_BOOT_REASON				0x1088
+#define REG_NFCPAD_CTRL				0x10A8
+
+#define REG_HIMR2					0x10B0
+#define REG_HISR2					0x10B4
+#define REG_HIMR3					0x10B8
+#define REG_HISR3					0x10BC
+#define REG_SW_MDIO					0x10C0
+#define REG_SW_FLUSH					0x10C4
+
+#define REG_H2C_PKT_READADDR				0x10D0
+#define REG_H2C_PKT_WRITEADDR				0x10D4
+
+#define REG_MEM_PWR_CRTL				0x10D8
+
+#define REG_FW_DBG0					0x10E0
+#define REG_FW_DBG1					0x10E4
+#define REG_FW_DBG2					0x10E8
+#define REG_FW_DBG3					0x10EC
+#define REG_FW_DBG4					0x10F0
+#define REG_FW_DBG5					0x10F4
+#define REG_FW_DBG6					0x10F8
+#define REG_FW_DBG7					0x10FC
+#define REG_CR_EXT					0x1100
+#define REG_FWFF					0x1114
+
+#define REG_RXFF_PTR_V1				0x1118
+#define REG_RXFF_WTR_V1				0x111C
+
+#define REG_FE2IMR					0x1120
+#define REG_FE2ISR					0x1124
+#define REG_FE3IMR					0x1128
+#define REG_FE3ISR					0x112C
+#define REG_FE4IMR					0x1130
+#define REG_FE4ISR					0x1134
+#define REG_FT1IMR					0x1138
+#define REG_FT1ISR					0x113C
+#define REG_SPWR0					0x1140
+#define REG_SPWR1					0x1144
+#define REG_SPWR2					0x1148
+#define REG_SPWR3					0x114C
+#define REG_POWSEQ					0x1150
+
+#define REG_TC7_CTRL_V1				0x1158
+#define REG_TC8_CTRL_V1				0x115C
+
+#define REG_FT2IMR					0x11E0
+#define REG_FT2ISR					0x11E4
+
+#define REG_MSG2					0x11F0
+#define REG_MSG3					0x11F4
+#define REG_MSG4					0x11F8
+#define REG_MSG5					0x11FC
+#define REG_DDMA_CH0SA					0x1200
+#define REG_DDMA_CH0DA					0x1204
+#define REG_DDMA_CH0CTRL				0x1208
+#define REG_DDMA_CH1SA					0x1210
+#define REG_DDMA_CH1DA					0x1214
+#define REG_DDMA_CH1CTRL				0x1218
+#define REG_DDMA_CH2SA					0x1220
+#define REG_DDMA_CH2DA					0x1224
+#define REG_DDMA_CH2CTRL				0x1228
+#define REG_DDMA_CH3SA					0x1230
+#define REG_DDMA_CH3DA					0x1234
+#define REG_DDMA_CH3CTRL				0x1238
+#define REG_DDMA_CH4SA					0x1240
+#define REG_DDMA_CH4DA					0x1244
+#define REG_DDMA_CH4CTRL				0x1248
+#define REG_DDMA_CH5SA					0x1250
+#define REG_DDMA_CH5DA					0x1254
+
+#define REG_REG_DDMA_CH5CTRL				0x1258
+
+#define REG_DDMA_INT_MSK				0x12E0
+#define REG_DDMA_CHSTATUS				0x12E8
+#define REG_DDMA_CHKSUM				0x12F0
+#define REG_DDMA_MONITOR				0x12FC
+
+#define REG_STC_INT_CS					0x1300
+#define REG_ST_INT_CFG					0x1304
+#define REG_CMU_DLY_CTRL				0x1310
+#define REG_CMU_DLY_CFG				0x1314
+#define REG_H2CQ_TXBD_DESA				0x1320
+#define REG_H2CQ_TXBD_NUM				0x1328
+#define REG_H2CQ_TXBD_IDX				0x132C
+#define REG_H2CQ_CSR					0x1330
+
+#define REG_CHANGE_PCIE_SPEED				0x1350
+
+#define REG_OLD_DEHANG					0x13F4
+
+#define REG_Q0_Q1_INFO					0x1400
+#define REG_Q2_Q3_INFO					0x1404
+#define REG_Q4_Q5_INFO					0x1408
+#define REG_Q6_Q7_INFO					0x140C
+#define REG_MGQ_HIQ_INFO				0x1410
+#define REG_CMDQ_BCNQ_INFO				0x1414
+#define REG_USEREG_SETTING				0x1420
+#define REG_AESIV_SETTING				0x1424
+#define REG_BF0_TIME_SETTING				0x1428
+#define REG_BF1_TIME_SETTING				0x142C
+#define REG_BF_TIMEOUT_EN				0x1430
+#define REG_MACID_RELEASE0				0x1434
+#define REG_MACID_RELEASE1				0x1438
+#define REG_MACID_RELEASE2				0x143C
+#define REG_MACID_RELEASE3				0x1440
+#define REG_MACID_RELEASE_SETTING			0x1444
+#define REG_FAST_EDCA_VOVI_SETTING			0x1448
+#define REG_FAST_EDCA_BEBK_SETTING			0x144C
+#define REG_MACID_DROP0				0x1450
+#define REG_MACID_DROP1				0x1454
+#define REG_MACID_DROP2				0x1458
+#define REG_MACID_DROP3				0x145C
+
+#define REG_R_MACID_RELEASE_SUCCESS_0			0x1460
+#define REG_R_MACID_RELEASE_SUCCESS_1			0x1464
+#define REG_R_MACID_RELEASE_SUCCESS_2			0x1468
+#define REG_R_MACID_RELEASE_SUCCESS_3			0x146C
+#define REG_MGG_FIFO_CRTL				0x1470
+#define REG_MGG_FIFO_INT				0x1474
+#define REG_MGG_FIFO_LIFETIME				0x1478
+#define REG_R_MACID_RELEASE_SUCCESS_CLEAR_OFFSET	0x147C
+
+#define REG_MACID_SHCUT_OFFSET				0x1480
+
+#define REG_MU_TX_CTL					0x14C0
+#define REG_MU_STA_GID_VLD				0x14C4
+#define REG_MU_STA_USER_POS_INFO			0x14C8
+#define REG_MU_TRX_DBG_CNT				0x14D0
+
+#define REG_CPUMGQ_TX_TIMER				0x1500
+#define REG_PS_TIMER_A					0x1504
+#define REG_PS_TIMER_B					0x1508
+#define REG_PS_TIMER_C					0x150C
+#define REG_PS_TIMER_ABC_CPUMGQ_TIMER_CRTL		0x1510
+#define REG_CPUMGQ_TX_TIMER_EARLY			0x1514
+#define REG_PS_TIMER_A_EARLY				0x1515
+#define REG_PS_TIMER_B_EARLY				0x1516
+#define REG_PS_TIMER_C_EARLY				0x1517
+
+#define REG_BCN_PSR_RPT2				0x1600
+#define REG_BCN_PSR_RPT3				0x1604
+#define REG_BCN_PSR_RPT4				0x1608
+#define REG_A1_ADDR_MASK				0x160C
+#define REG_MACID2					0x1620
+#define REG_BSSID2					0x1628
+#define REG_MACID3					0x1630
+#define REG_BSSID3					0x1638
+#define REG_MACID4					0x1640
+#define REG_BSSID4					0x1648
+
+#define REG_NOA_REPORT					0x1650
+#define REG_PWRBIT_SETTING				0x1660
+#define REG_WMAC_MU_BF_OPTION				0x167C
+
+#define REG_WMAC_MU_ARB				0x167E
+#define REG_WMAC_MU_OPTION				0x167F
+#define REG_WMAC_MU_BF_CTL				0x1680
+
+#define REG_WMAC_MU_BFRPT_PARA				0x1682
+
+#define REG_WMAC_ASSOCIATED_MU_BFMEE2			0x1684
+#define REG_WMAC_ASSOCIATED_MU_BFMEE3			0x1686
+#define REG_WMAC_ASSOCIATED_MU_BFMEE4			0x1688
+#define REG_WMAC_ASSOCIATED_MU_BFMEE5			0x168A
+#define REG_WMAC_ASSOCIATED_MU_BFMEE6			0x168C
+#define REG_WMAC_ASSOCIATED_MU_BFMEE7			0x168E
+
+#define REG_TRANSMIT_ADDRSS_0				0x16A0
+#define REG_TRANSMIT_ADDRSS_1				0x16A8
+#define REG_TRANSMIT_ADDRSS_2				0x16B0
+#define REG_TRANSMIT_ADDRSS_3				0x16B8
+#define REG_TRANSMIT_ADDRSS_4				0x16C0
+
+#define REG_WL2LTECOEX_INDIRECT_ACCESS_CTRL_V1	0x1700
+#define REG_WL2LTECOEX_INDIRECT_ACCESS_WRITE_DATA_V1	0x1704
+#define REG_WL2LTECOEX_INDIRECT_ACCESS_READ_DATA_V1	0x1708
+
+/* ----------------------------------------------------- */
+/*	*/
+/* 0xFB00h ~ 0xFCFFh	TX/RX packet buffer affress */
+/*	*/
+/* ----------------------------------------------------- */
+#define REG_RXPKTBUF_STARTADDR	0xFB00
+#define REG_TXPKTBUF_STARTADDR	0xFC00
+
+/* ----------------------------------------------------- */
+/*	*/
+/* 0xFD00h ~ 0xFDFFh	8051 CPU Local REG */
+/*	*/
+/* ----------------------------------------------------- */
+#define REG_SYS_CTRL		0xFD00
+#define REG_PONSTS_RPT1		0xFD01
+#define REG_PONSTS_RPT2		0xFD02
+#define REG_PONSTS_RPT3		0xFD03
+#define REG_PONSTS_RPT4		0xFD04	/* 0x84 */
+#define REG_PONSTS_RPT5		0xFD05	/* 0x85 */
+#define REG_8051ERRFLAG		0xFD08
+#define REG_8051ERRFLAG_MASK	0xFD09
+#define REG_TXADDRH		0xFD10	/* Tx Packet High address */
+#define REG_RXADDRH		0xFD11	/* Rx Packet High address */
+#define REG_TXADDRH_EXT		0xFD12	/* 0xFD12[0] : for 8051 access txpktbuf
+					 * high64k as external register
+					 */
+
+#define REG_U3_STATE		0xFD48	/* (Read only)
+					 * [7:4] : usb3 changed last state.
+					 * [3:0] : usb3 state
+					 */
+
+/* for MAILBOX */
+#define REG_OUTDATA0		0xFD50
+#define REG_OUTDATA1		0xFD54
+#define REG_OUTRDY		0xFD58	/* bit[0] : OutReady,
+					 * bit[1] : OutEmptyIntEn
+					 */
+
+#define REG_INDATA0		0xFD60
+#define REG_INDATA1		0xFD64
+#define REG_INRDY		0xFD68	/* bit[0] : InReady,
+					 * bit[1] : InRdyIntEn
+					 */
+
+/* MCU ERROR debug REG */
+#define REG_MCUERR_PCLSB	0xFD90	/* PC[7:0] */
+#define REG_MCUERR_PCMSB	0xFD91	/* PC[15:8] */
+#define REG_MCUERR_ACC		0xFD92
+#define REG_MCUERR_B		0xFD93
+#define REG_MCUERR_DPTRLSB	0xFD94	/* DPTR[7:0] */
+#define REG_MCUERR_DPTRMSB	0xFD95	/* DPTR[15:8] */
+#define REG_MCUERR_SP		0xFD96	/* SP[7:0] */
+#define REG_MCUERR_IE		0xFD97	/* IE[7:0] */
+#define REG_MCUERR_EIE		0xFD98	/* EIE[7:0] */
+#define REG_VERA_SIM		0xFD9F
+/* 0xFD99~0xFD9F are reserved.. */
+
+/* ----------------------------------------------------- */
+/*	*/
+/* 0xFE00h ~ 0xFEFFh	USB Configuration */
+/*	*/
+/* ----------------------------------------------------- */
+
+/* RTS5101 USB Register Definition */
+#define REG_USB_SETUP_DEC_INT		0xFE00
+#define REG_USB_DMACTL			0xFE01
+#define REG_USB_IRQSTAT0		0xFE02
+#define REG_USB_IRQSTAT1		0xFE03
+#define REG_USB_IRQEN0			0xFE04
+#define REG_USB_IRQEN1			0xFE05
+#define REG_USB_AUTOPTRL		0xFE06
+#define REG_USB_AUTOPTRH		0xFE07
+#define REG_USB_AUTODAT			0xFE08
+
+#define REG_USB_SCRATCH0		0xFE09
+#define REG_USB_SCRATCH1		0xFE0A
+#define REG_USB_SEEPROM			0xFE0B
+#define REG_USB_GPIO0			0xFE0C
+#define REG_USB_GPIO0DIR		0xFE0D
+#define REG_USB_CLKSEL			0xFE0E
+#define REG_USB_BOOTCTL			0xFE0F
+
+#define REG_USB_USBCTL			0xFE10
+#define REG_USB_USBSTAT			0xFE11
+#define REG_USB_DEVADDR			0xFE12
+#define REG_USB_USBTEST			0xFE13
+#define REG_USB_FNUM0			0xFE14
+#define REG_USB_FNUM1			0xFE15
+
+#define REG_USB_EP_IDX			0xFE20
+#define REG_USB_EP_CFG			0xFE21
+#define REG_USB_EP_CTL			0xFE22
+#define REG_USB_EP_STAT			0xFE23
+#define REG_USB_EP_IRQ			0xFE24
+#define REG_USB_EP_IRQEN		0xFE25
+#define REG_USB_EP_MAXPKT0		0xFE26
+#define REG_USB_EP_MAXPKT1		0xFE27
+#define REG_USB_EP_DAT			0xFE28
+#define REG_USB_EP_BC0			0xFE29
+#define REG_USB_EP_BC1			0xFE2A
+#define REG_USB_EP_TC0			0xFE2B
+#define REG_USB_EP_TC1			0xFE2C
+#define REG_USB_EP_TC2			0xFE2D
+#define REG_USB_EP_CTL2			0xFE2E
+
+#define REG_USB_INFO			0xFE17
+#define REG_USB_SPECIAL_OPTION		0xFE55
+#define REG_USB_DMA_AGG_TO		0xFE5B
+#define REG_USB_AGG_TO			0xFE5C
+#define REG_USB_AGG_TH			0xFE5D
+
+#define REG_USB_VID			0xFE60
+#define REG_USB_PID			0xFE62
+#define REG_USB_OPT			0xFE64
+#define REG_USB_CONFIG			0xFE65	/* RX EP setting.
+						 * 0xFE65 Bit[3:0] : RXQ,
+						 *        Bit[7:4] : INTQ
+						 */
+						/* TX EP setting.
+						 * 0xFE66 Bit[3:0] : TXQ0,
+						 *        Bit[7:4] : TXQ1,
+						 * 0xFE67 Bit[3:0] : TXQ2
+						 */
+#define REG_USB_PHY_PARA1 0xFE68 /* Bit[7:4]: XCVR_SEN	(USB PHY 0xE2[7:4]),
+				  * Bit[3:0]: XCVR_SH	(USB PHY 0xE2[3:0])
+				  */
+#define REG_USB_PHY_PARA2 0xFE69 /* Bit[7:5]: XCVR_BG	(USB PHY 0xE3[5:3]),
+				  * Bit[4:2]: XCVR_DR	(USB PHY 0xE3[2:0]),
+				  * Bit[1]: SE0_LVL	(USB PHY 0xE5[7]),
+				  * Bit[0]: FORCE_XTL_ON (USB PHY 0xE5[1])
+				  */
+#define REG_USB_PHY_PARA3 0xFE6A /* Bit[7:5]: XCVR_SRC	(USB PHY 0xE5[4:2]),
+				  * Bit[4]: LATE_DLLEN	(USB PHY 0xF0[4]),
+				  * Bit[3]: HS_LP_MODE	(USB PHY 0xF0[3]),
+				  * Bit[2]: UTMI_POS_OUT (USB PHY 0xF1 [7]),
+				  * Bit[1:0]: TX_DELAY	(USB PHY 0xF1 [2:1])
+				  */
+#define REG_USB_PHY_PARA4		0xFE6B	/* (USB PHY 0xE7[7:0]) */
+#define REG_USB_OPT2			0xFE6C
+#define REG_USB_MAC_ADDR			0xFE70	/* 0xFE70~0xFE75 */
+#define REG_USB_MANUFACTURE_SETTING	0xFE80	/* 0xFE80~0xFE90 Max: 32 bytes*/
+#define REG_USB_PRODUCT_STRING		0xFEA0	/* 0xFEA0~0xFECF Max: 48 bytes*/
+#define REG_USB_SERIAL_NUMBER_STRING	0xFED0	/* 0xFED0~0xFEDF Max: 12 bytes*/
+
+#define REG_USB_ALTERNATE_SETTING	0xFE4F
+#define REG_USB_INT_BINTERVAL		0xFE6E
+#define REG_USB_GPS_EP_CONFIG		0xFE6D
+
+#endif	/* __HALMAC_COM_REG_H__ */
diff --git a/drivers/staging/rtlwifi/halmac/halmac_reg_8822b.h b/drivers/staging/rtlwifi/halmac/halmac_reg_8822b.h
new file mode 100644
index 000000000000..4bc59b127412
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_reg_8822b.h
@@ -0,0 +1,728 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef __INC_HALMAC_REG_8822B_H
+#define __INC_HALMAC_REG_8822B_H
+
+#define REG_SYS_ISO_CTRL_8822B 0x0000
+#define REG_SYS_FUNC_EN_8822B 0x0002
+#define REG_SYS_PW_CTRL_8822B 0x0004
+#define REG_SYS_CLK_CTRL_8822B 0x0008
+#define REG_SYS_EEPROM_CTRL_8822B 0x000A
+#define REG_EE_VPD_8822B 0x000C
+#define REG_SYS_SWR_CTRL1_8822B 0x0010
+#define REG_SYS_SWR_CTRL2_8822B 0x0014
+#define REG_SYS_SWR_CTRL3_8822B 0x0018
+#define REG_RSV_CTRL_8822B 0x001C
+#define REG_RF_CTRL_8822B 0x001F
+#define REG_AFE_LDO_CTRL_8822B 0x0020
+#define REG_AFE_CTRL1_8822B 0x0024
+#define REG_AFE_CTRL2_8822B 0x0028
+#define REG_AFE_CTRL3_8822B 0x002C
+#define REG_EFUSE_CTRL_8822B 0x0030
+#define REG_LDO_EFUSE_CTRL_8822B 0x0034
+#define REG_PWR_OPTION_CTRL_8822B 0x0038
+#define REG_CAL_TIMER_8822B 0x003C
+#define REG_ACLK_MON_8822B 0x003E
+#define REG_GPIO_MUXCFG_8822B 0x0040
+#define REG_GPIO_PIN_CTRL_8822B 0x0044
+#define REG_GPIO_INTM_8822B 0x0048
+#define REG_LED_CFG_8822B 0x004C
+#define REG_FSIMR_8822B 0x0050
+#define REG_FSISR_8822B 0x0054
+#define REG_HSIMR_8822B 0x0058
+#define REG_HSISR_8822B 0x005C
+#define REG_GPIO_EXT_CTRL_8822B 0x0060
+#define REG_PAD_CTRL1_8822B 0x0064
+#define REG_WL_BT_PWR_CTRL_8822B 0x0068
+#define REG_SDM_DEBUG_8822B 0x006C
+#define REG_SYS_SDIO_CTRL_8822B 0x0070
+#define REG_HCI_OPT_CTRL_8822B 0x0074
+#define REG_AFE_CTRL4_8822B 0x0078
+#define REG_LDO_SWR_CTRL_8822B 0x007C
+#define REG_MCUFW_CTRL_8822B 0x0080
+#define REG_MCU_TST_CFG_8822B 0x0084
+#define REG_HMEBOX_E0_E1_8822B 0x0088
+#define REG_HMEBOX_E2_E3_8822B 0x008C
+#define REG_WLLPS_CTRL_8822B 0x0090
+#define REG_AFE_CTRL5_8822B 0x0094
+#define REG_GPIO_DEBOUNCE_CTRL_8822B 0x0098
+#define REG_RPWM2_8822B 0x009C
+#define REG_SYSON_FSM_MON_8822B 0x00A0
+#define REG_AFE_CTRL6_8822B 0x00A4
+#define REG_PMC_DBG_CTRL1_8822B 0x00A8
+#define REG_AFE_CTRL7_8822B 0x00AC
+#define REG_HIMR0_8822B 0x00B0
+#define REG_HISR0_8822B 0x00B4
+#define REG_HIMR1_8822B 0x00B8
+#define REG_HISR1_8822B 0x00BC
+#define REG_DBG_PORT_SEL_8822B 0x00C0
+#define REG_PAD_CTRL2_8822B 0x00C4
+#define REG_PMC_DBG_CTRL2_8822B 0x00CC
+#define REG_BIST_CTRL_8822B 0x00D0
+#define REG_BIST_RPT_8822B 0x00D4
+#define REG_MEM_CTRL_8822B 0x00D8
+#define REG_AFE_CTRL8_8822B 0x00DC
+#define REG_USB_SIE_INTF_8822B 0x00E0
+#define REG_PCIE_MIO_INTF_8822B 0x00E4
+#define REG_PCIE_MIO_INTD_8822B 0x00E8
+#define REG_WLRF1_8822B 0x00EC
+#define REG_SYS_CFG1_8822B 0x00F0
+#define REG_SYS_STATUS1_8822B 0x00F4
+#define REG_SYS_STATUS2_8822B 0x00F8
+#define REG_SYS_CFG2_8822B 0x00FC
+#define REG_SYS_CFG3_8822B 0x1000
+#define REG_SYS_CFG4_8822B 0x1034
+#define REG_SYS_CFG5_8822B 0x1070
+#define REG_CPU_DMEM_CON_8822B 0x1080
+#define REG_BOOT_REASON_8822B 0x1088
+#define REG_NFCPAD_CTRL_8822B 0x10A8
+#define REG_HIMR2_8822B 0x10B0
+#define REG_HISR2_8822B 0x10B4
+#define REG_HIMR3_8822B 0x10B8
+#define REG_HISR3_8822B 0x10BC
+#define REG_SW_MDIO_8822B 0x10C0
+#define REG_SW_FLUSH_8822B 0x10C4
+#define REG_H2C_PKT_READADDR_8822B 0x10D0
+#define REG_H2C_PKT_WRITEADDR_8822B 0x10D4
+#define REG_MEM_PWR_CRTL_8822B 0x10D8
+#define REG_FW_DBG0_8822B 0x10E0
+#define REG_FW_DBG1_8822B 0x10E4
+#define REG_FW_DBG2_8822B 0x10E8
+#define REG_FW_DBG3_8822B 0x10EC
+#define REG_FW_DBG4_8822B 0x10F0
+#define REG_FW_DBG5_8822B 0x10F4
+#define REG_FW_DBG6_8822B 0x10F8
+#define REG_FW_DBG7_8822B 0x10FC
+#define REG_CR_8822B 0x0100
+#define REG_PKT_BUFF_ACCESS_CTRL_8822B 0x0106
+#define REG_TSF_CLK_STATE_8822B 0x0108
+#define REG_TXDMA_PQ_MAP_8822B 0x010C
+#define REG_TRXFF_BNDY_8822B 0x0114
+#define REG_PTA_I2C_MBOX_8822B 0x0118
+#define REG_RXFF_BNDY_8822B 0x011C
+#define REG_FE1IMR_8822B 0x0120
+#define REG_FE1ISR_8822B 0x0124
+#define REG_CPWM_8822B 0x012C
+#define REG_FWIMR_8822B 0x0130
+#define REG_FWISR_8822B 0x0134
+#define REG_FTIMR_8822B 0x0138
+#define REG_FTISR_8822B 0x013C
+#define REG_PKTBUF_DBG_CTRL_8822B 0x0140
+#define REG_PKTBUF_DBG_DATA_L_8822B 0x0144
+#define REG_PKTBUF_DBG_DATA_H_8822B 0x0148
+#define REG_CPWM2_8822B 0x014C
+#define REG_TC0_CTRL_8822B 0x0150
+#define REG_TC1_CTRL_8822B 0x0154
+#define REG_TC2_CTRL_8822B 0x0158
+#define REG_TC3_CTRL_8822B 0x015C
+#define REG_TC4_CTRL_8822B 0x0160
+#define REG_TCUNIT_BASE_8822B 0x0164
+#define REG_TC5_CTRL_8822B 0x0168
+#define REG_TC6_CTRL_8822B 0x016C
+#define REG_MBIST_FAIL_8822B 0x0170
+#define REG_MBIST_START_PAUSE_8822B 0x0174
+#define REG_MBIST_DONE_8822B 0x0178
+#define REG_MBIST_FAIL_NRML_8822B 0x017C
+#define REG_AES_DECRPT_DATA_8822B 0x0180
+#define REG_AES_DECRPT_CFG_8822B 0x0184
+#define REG_TMETER_8822B 0x0190
+#define REG_OSC_32K_CTRL_8822B 0x0194
+#define REG_32K_CAL_REG1_8822B 0x0198
+#define REG_C2HEVT_8822B 0x01A0
+#define REG_SW_DEFINED_PAGE1_8822B 0x01B8
+#define REG_MCUTST_I_8822B 0x01C0
+#define REG_MCUTST_II_8822B 0x01C4
+#define REG_FMETHR_8822B 0x01C8
+#define REG_HMETFR_8822B 0x01CC
+#define REG_HMEBOX0_8822B 0x01D0
+#define REG_HMEBOX1_8822B 0x01D4
+#define REG_HMEBOX2_8822B 0x01D8
+#define REG_HMEBOX3_8822B 0x01DC
+#define REG_LLT_INIT_8822B 0x01E0
+#define REG_LLT_INIT_ADDR_8822B 0x01E4
+#define REG_BB_ACCESS_CTRL_8822B 0x01E8
+#define REG_BB_ACCESS_DATA_8822B 0x01EC
+#define REG_HMEBOX_E0_8822B 0x01F0
+#define REG_HMEBOX_E1_8822B 0x01F4
+#define REG_HMEBOX_E2_8822B 0x01F8
+#define REG_HMEBOX_E3_8822B 0x01FC
+#define REG_CR_EXT_8822B 0x1100
+#define REG_FWFF_8822B 0x1114
+#define REG_RXFF_PTR_V1_8822B 0x1118
+#define REG_RXFF_WTR_V1_8822B 0x111C
+#define REG_FE2IMR_8822B 0x1120
+#define REG_FE2ISR_8822B 0x1124
+#define REG_FE3IMR_8822B 0x1128
+#define REG_FE3ISR_8822B 0x112C
+#define REG_FE4IMR_8822B 0x1130
+#define REG_FE4ISR_8822B 0x1134
+#define REG_FT1IMR_8822B 0x1138
+#define REG_FT1ISR_8822B 0x113C
+#define REG_SPWR0_8822B 0x1140
+#define REG_SPWR1_8822B 0x1144
+#define REG_SPWR2_8822B 0x1148
+#define REG_SPWR3_8822B 0x114C
+#define REG_POWSEQ_8822B 0x1150
+#define REG_TC7_CTRL_V1_8822B 0x1158
+#define REG_TC8_CTRL_V1_8822B 0x115C
+#define REG_FT2IMR_8822B 0x11E0
+#define REG_FT2ISR_8822B 0x11E4
+#define REG_MSG2_8822B 0x11F0
+#define REG_MSG3_8822B 0x11F4
+#define REG_MSG4_8822B 0x11F8
+#define REG_MSG5_8822B 0x11FC
+#define REG_FIFOPAGE_CTRL_1_8822B 0x0200
+#define REG_FIFOPAGE_CTRL_2_8822B 0x0204
+#define REG_AUTO_LLT_V1_8822B 0x0208
+#define REG_TXDMA_OFFSET_CHK_8822B 0x020C
+#define REG_TXDMA_STATUS_8822B 0x0210
+#define REG_TX_DMA_DBG_8822B 0x0214
+#define REG_TQPNT1_8822B 0x0218
+#define REG_TQPNT2_8822B 0x021C
+#define REG_TQPNT3_8822B 0x0220
+#define REG_TQPNT4_8822B 0x0224
+#define REG_RQPN_CTRL_1_8822B 0x0228
+#define REG_RQPN_CTRL_2_8822B 0x022C
+#define REG_FIFOPAGE_INFO_1_8822B 0x0230
+#define REG_FIFOPAGE_INFO_2_8822B 0x0234
+#define REG_FIFOPAGE_INFO_3_8822B 0x0238
+#define REG_FIFOPAGE_INFO_4_8822B 0x023C
+#define REG_FIFOPAGE_INFO_5_8822B 0x0240
+#define REG_H2C_HEAD_8822B 0x0244
+#define REG_H2C_TAIL_8822B 0x0248
+#define REG_H2C_READ_ADDR_8822B 0x024C
+#define REG_H2C_WR_ADDR_8822B 0x0250
+#define REG_H2C_INFO_8822B 0x0254
+#define REG_RXDMA_AGG_PG_TH_8822B 0x0280
+#define REG_RXPKT_NUM_8822B 0x0284
+#define REG_RXDMA_STATUS_8822B 0x0288
+#define REG_RXDMA_DPR_8822B 0x028C
+#define REG_RXDMA_MODE_8822B 0x0290
+#define REG_C2H_PKT_8822B 0x0294
+#define REG_FWFF_C2H_8822B 0x0298
+#define REG_FWFF_CTRL_8822B 0x029C
+#define REG_FWFF_PKT_INFO_8822B 0x02A0
+#define REG_DDMA_CH0SA_8822B 0x1200
+#define REG_DDMA_CH0DA_8822B 0x1204
+#define REG_DDMA_CH0CTRL_8822B 0x1208
+#define REG_DDMA_CH1SA_8822B 0x1210
+#define REG_DDMA_CH1DA_8822B 0x1214
+#define REG_DDMA_CH1CTRL_8822B 0x1218
+#define REG_DDMA_CH2SA_8822B 0x1220
+#define REG_DDMA_CH2DA_8822B 0x1224
+#define REG_DDMA_CH2CTRL_8822B 0x1228
+#define REG_DDMA_CH3SA_8822B 0x1230
+#define REG_DDMA_CH3DA_8822B 0x1234
+#define REG_DDMA_CH3CTRL_8822B 0x1238
+#define REG_DDMA_CH4SA_8822B 0x1240
+#define REG_DDMA_CH4DA_8822B 0x1244
+#define REG_DDMA_CH4CTRL_8822B 0x1248
+#define REG_DDMA_CH5SA_8822B 0x1250
+#define REG_DDMA_CH5DA_8822B 0x1254
+#define REG_REG_DDMA_CH5CTRL_8822B 0x1258
+#define REG_DDMA_INT_MSK_8822B 0x12E0
+#define REG_DDMA_CHSTATUS_8822B 0x12E8
+#define REG_DDMA_CHKSUM_8822B 0x12F0
+#define REG_DDMA_MONITOR_8822B 0x12FC
+#define REG_PCIE_CTRL_8822B 0x0300
+#define REG_INT_MIG_8822B 0x0304
+#define REG_BCNQ_TXBD_DESA_8822B 0x0308
+#define REG_MGQ_TXBD_DESA_8822B 0x0310
+#define REG_VOQ_TXBD_DESA_8822B 0x0318
+#define REG_VIQ_TXBD_DESA_8822B 0x0320
+#define REG_BEQ_TXBD_DESA_8822B 0x0328
+#define REG_BKQ_TXBD_DESA_8822B 0x0330
+#define REG_RXQ_RXBD_DESA_8822B 0x0338
+#define REG_HI0Q_TXBD_DESA_8822B 0x0340
+#define REG_HI1Q_TXBD_DESA_8822B 0x0348
+#define REG_HI2Q_TXBD_DESA_8822B 0x0350
+#define REG_HI3Q_TXBD_DESA_8822B 0x0358
+#define REG_HI4Q_TXBD_DESA_8822B 0x0360
+#define REG_HI5Q_TXBD_DESA_8822B 0x0368
+#define REG_HI6Q_TXBD_DESA_8822B 0x0370
+#define REG_HI7Q_TXBD_DESA_8822B 0x0378
+#define REG_MGQ_TXBD_NUM_8822B 0x0380
+#define REG_RX_RXBD_NUM_8822B 0x0382
+#define REG_VOQ_TXBD_NUM_8822B 0x0384
+#define REG_VIQ_TXBD_NUM_8822B 0x0386
+#define REG_BEQ_TXBD_NUM_8822B 0x0388
+#define REG_BKQ_TXBD_NUM_8822B 0x038A
+#define REG_HI0Q_TXBD_NUM_8822B 0x038C
+#define REG_HI1Q_TXBD_NUM_8822B 0x038E
+#define REG_HI2Q_TXBD_NUM_8822B 0x0390
+#define REG_HI3Q_TXBD_NUM_8822B 0x0392
+#define REG_HI4Q_TXBD_NUM_8822B 0x0394
+#define REG_HI5Q_TXBD_NUM_8822B 0x0396
+#define REG_HI6Q_TXBD_NUM_8822B 0x0398
+#define REG_HI7Q_TXBD_NUM_8822B 0x039A
+#define REG_TSFTIMER_HCI_8822B 0x039C
+#define REG_BD_RWPTR_CLR_8822B 0x039C
+#define REG_VOQ_TXBD_IDX_8822B 0x03A0
+#define REG_VIQ_TXBD_IDX_8822B 0x03A4
+#define REG_BEQ_TXBD_IDX_8822B 0x03A8
+#define REG_BKQ_TXBD_IDX_8822B 0x03AC
+#define REG_MGQ_TXBD_IDX_8822B 0x03B0
+#define REG_RXQ_RXBD_IDX_8822B 0x03B4
+#define REG_HI0Q_TXBD_IDX_8822B 0x03B8
+#define REG_HI1Q_TXBD_IDX_8822B 0x03BC
+#define REG_HI2Q_TXBD_IDX_8822B 0x03C0
+#define REG_HI3Q_TXBD_IDX_8822B 0x03C4
+#define REG_HI4Q_TXBD_IDX_8822B 0x03C8
+#define REG_HI5Q_TXBD_IDX_8822B 0x03CC
+#define REG_HI6Q_TXBD_IDX_8822B 0x03D0
+#define REG_HI7Q_TXBD_IDX_8822B 0x03D4
+#define REG_DBG_SEL_V1_8822B 0x03D8
+#define REG_PCIE_HRPWM1_V1_8822B 0x03D9
+#define REG_PCIE_HCPWM1_V1_8822B 0x03DA
+#define REG_PCIE_CTRL2_8822B 0x03DB
+#define REG_PCIE_HRPWM2_V1_8822B 0x03DC
+#define REG_PCIE_HCPWM2_V1_8822B 0x03DE
+#define REG_PCIE_H2C_MSG_V1_8822B 0x03E0
+#define REG_PCIE_C2H_MSG_V1_8822B 0x03E4
+#define REG_DBI_WDATA_V1_8822B 0x03E8
+#define REG_DBI_RDATA_V1_8822B 0x03EC
+#define REG_DBI_FLAG_V1_8822B 0x03F0
+#define REG_MDIO_V1_8822B 0x03F4
+#define REG_PCIE_MIX_CFG_8822B 0x03F8
+#define REG_HCI_MIX_CFG_8822B 0x03FC
+#define REG_STC_INT_CS_8822B 0x1300
+#define REG_ST_INT_CFG_8822B 0x1304
+#define REG_CMU_DLY_CTRL_8822B 0x1310
+#define REG_CMU_DLY_CFG_8822B 0x1314
+#define REG_H2CQ_TXBD_DESA_8822B 0x1320
+#define REG_H2CQ_TXBD_NUM_8822B 0x1328
+#define REG_H2CQ_TXBD_IDX_8822B 0x132C
+#define REG_H2CQ_CSR_8822B 0x1330
+#define REG_CHANGE_PCIE_SPEED_8822B 0x1350
+#define REG_OLD_DEHANG_8822B 0x13F4
+#define REG_Q0_INFO_8822B 0x0400
+#define REG_Q1_INFO_8822B 0x0404
+#define REG_Q2_INFO_8822B 0x0408
+#define REG_Q3_INFO_8822B 0x040C
+#define REG_MGQ_INFO_8822B 0x0410
+#define REG_HIQ_INFO_8822B 0x0414
+#define REG_BCNQ_INFO_8822B 0x0418
+#define REG_TXPKT_EMPTY_8822B 0x041A
+#define REG_CPU_MGQ_INFO_8822B 0x041C
+#define REG_FWHW_TXQ_CTRL_8822B 0x0420
+#define REG_DATAFB_SEL_8822B 0x0423
+#define REG_BCNQ_BDNY_V1_8822B 0x0424
+#define REG_LIFETIME_EN_8822B 0x0426
+#define REG_SPEC_SIFS_8822B 0x0428
+#define REG_RETRY_LIMIT_8822B 0x042A
+#define REG_TXBF_CTRL_8822B 0x042C
+#define REG_DARFRC_8822B 0x0430
+#define REG_RARFRC_8822B 0x0438
+#define REG_RRSR_8822B 0x0440
+#define REG_ARFR0_8822B 0x0444
+#define REG_ARFR1_V1_8822B 0x044C
+#define REG_CCK_CHECK_8822B 0x0454
+#define REG_AMPDU_MAX_TIME_V1_8822B 0x0455
+#define REG_BCNQ1_BDNY_V1_8822B 0x0456
+#define REG_AMPDU_MAX_LENGTH_8822B 0x0458
+#define REG_ACQ_STOP_8822B 0x045C
+#define REG_NDPA_RATE_8822B 0x045D
+#define REG_TX_HANG_CTRL_8822B 0x045E
+#define REG_NDPA_OPT_CTRL_8822B 0x045F
+#define REG_RD_RESP_PKT_TH_8822B 0x0463
+#define REG_CMDQ_INFO_8822B 0x0464
+#define REG_Q4_INFO_8822B 0x0468
+#define REG_Q5_INFO_8822B 0x046C
+#define REG_Q6_INFO_8822B 0x0470
+#define REG_Q7_INFO_8822B 0x0474
+#define REG_WMAC_LBK_BUF_HD_V1_8822B 0x0478
+#define REG_MGQ_BDNY_V1_8822B 0x047A
+#define REG_TXRPT_CTRL_8822B 0x047C
+#define REG_INIRTS_RATE_SEL_8822B 0x0480
+#define REG_BASIC_CFEND_RATE_8822B 0x0481
+#define REG_STBC_CFEND_RATE_8822B 0x0482
+#define REG_DATA_SC_8822B 0x0483
+#define REG_MACID_SLEEP3_8822B 0x0484
+#define REG_MACID_SLEEP1_8822B 0x0488
+#define REG_ARFR2_V1_8822B 0x048C
+#define REG_ARFR3_V1_8822B 0x0494
+#define REG_ARFR4_8822B 0x049C
+#define REG_ARFR5_8822B 0x04A4
+#define REG_TXRPT_START_OFFSET_8822B 0x04AC
+#define REG_POWER_STAGE1_8822B 0x04B4
+#define REG_POWER_STAGE2_8822B 0x04B8
+#define REG_SW_AMPDU_BURST_MODE_CTRL_8822B 0x04BC
+#define REG_PKT_LIFE_TIME_8822B 0x04C0
+#define REG_STBC_SETTING_8822B 0x04C4
+#define REG_STBC_SETTING2_8822B 0x04C5
+#define REG_QUEUE_CTRL_8822B 0x04C6
+#define REG_SINGLE_AMPDU_CTRL_8822B 0x04C7
+#define REG_PROT_MODE_CTRL_8822B 0x04C8
+#define REG_BAR_MODE_CTRL_8822B 0x04CC
+#define REG_RA_TRY_RATE_AGG_LMT_8822B 0x04CF
+#define REG_MACID_SLEEP2_8822B 0x04D0
+#define REG_MACID_SLEEP_8822B 0x04D4
+#define REG_HW_SEQ0_8822B 0x04D8
+#define REG_HW_SEQ1_8822B 0x04DA
+#define REG_HW_SEQ2_8822B 0x04DC
+#define REG_HW_SEQ3_8822B 0x04DE
+#define REG_NULL_PKT_STATUS_V1_8822B 0x04E0
+#define REG_PTCL_ERR_STATUS_8822B 0x04E2
+#define REG_NULL_PKT_STATUS_EXTEND_8822B 0x04E3
+#define REG_VIDEO_ENHANCEMENT_FUN_8822B 0x04E4
+#define REG_BT_POLLUTE_PKT_CNT_8822B 0x04E8
+#define REG_PTCL_DBG_8822B 0x04EC
+#define REG_CPUMGQ_TIMER_CTRL2_8822B 0x04F4
+#define REG_DUMMY_PAGE4_V1_8822B 0x04FC
+#define REG_MOREDATA_8822B 0x04FE
+#define REG_Q0_Q1_INFO_8822B 0x1400
+#define REG_Q2_Q3_INFO_8822B 0x1404
+#define REG_Q4_Q5_INFO_8822B 0x1408
+#define REG_Q6_Q7_INFO_8822B 0x140C
+#define REG_MGQ_HIQ_INFO_8822B 0x1410
+#define REG_CMDQ_BCNQ_INFO_8822B 0x1414
+#define REG_USEREG_SETTING_8822B 0x1420
+#define REG_AESIV_SETTING_8822B 0x1424
+#define REG_BF0_TIME_SETTING_8822B 0x1428
+#define REG_BF1_TIME_SETTING_8822B 0x142C
+#define REG_BF_TIMEOUT_EN_8822B 0x1430
+#define REG_MACID_RELEASE0_8822B 0x1434
+#define REG_MACID_RELEASE1_8822B 0x1438
+#define REG_MACID_RELEASE2_8822B 0x143C
+#define REG_MACID_RELEASE3_8822B 0x1440
+#define REG_MACID_RELEASE_SETTING_8822B 0x1444
+#define REG_FAST_EDCA_VOVI_SETTING_8822B 0x1448
+#define REG_FAST_EDCA_BEBK_SETTING_8822B 0x144C
+#define REG_MACID_DROP0_8822B 0x1450
+#define REG_MACID_DROP1_8822B 0x1454
+#define REG_MACID_DROP2_8822B 0x1458
+#define REG_MACID_DROP3_8822B 0x145C
+#define REG_R_MACID_RELEASE_SUCCESS_0_8822B 0x1460
+#define REG_R_MACID_RELEASE_SUCCESS_1_8822B 0x1464
+#define REG_R_MACID_RELEASE_SUCCESS_2_8822B 0x1468
+#define REG_R_MACID_RELEASE_SUCCESS_3_8822B 0x146C
+#define REG_MGG_FIFO_CRTL_8822B 0x1470
+#define REG_MGG_FIFO_INT_8822B 0x1474
+#define REG_MGG_FIFO_LIFETIME_8822B 0x1478
+#define REG_R_MACID_RELEASE_SUCCESS_CLEAR_OFFSET_8822B 0x147C
+#define REG_MACID_SHCUT_OFFSET_8822B 0x1480
+#define REG_MU_TX_CTL_8822B 0x14C0
+#define REG_MU_STA_GID_VLD_8822B 0x14C4
+#define REG_MU_STA_USER_POS_INFO_8822B 0x14C8
+#define REG_MU_TRX_DBG_CNT_8822B 0x14D0
+#define REG_EDCA_VO_PARAM_8822B 0x0500
+#define REG_EDCA_VI_PARAM_8822B 0x0504
+#define REG_EDCA_BE_PARAM_8822B 0x0508
+#define REG_EDCA_BK_PARAM_8822B 0x050C
+#define REG_BCNTCFG_8822B 0x0510
+#define REG_PIFS_8822B 0x0512
+#define REG_RDG_PIFS_8822B 0x0513
+#define REG_SIFS_8822B 0x0514
+#define REG_TSFTR_SYN_OFFSET_8822B 0x0518
+#define REG_AGGR_BREAK_TIME_8822B 0x051A
+#define REG_SLOT_8822B 0x051B
+#define REG_TX_PTCL_CTRL_8822B 0x0520
+#define REG_TXPAUSE_8822B 0x0522
+#define REG_DIS_TXREQ_CLR_8822B 0x0523
+#define REG_RD_CTRL_8822B 0x0524
+#define REG_MBSSID_CTRL_8822B 0x0526
+#define REG_P2PPS_CTRL_8822B 0x0527
+#define REG_PKT_LIFETIME_CTRL_8822B 0x0528
+#define REG_P2PPS_SPEC_STATE_8822B 0x052B
+#define REG_BAR_TX_CTRL_8822B 0x0530
+#define REG_QUEUE_INCOL_THR_8822B 0x0538
+#define REG_QUEUE_INCOL_EN_8822B 0x053C
+#define REG_TBTT_PROHIBIT_8822B 0x0540
+#define REG_P2PPS_STATE_8822B 0x0543
+#define REG_RD_NAV_NXT_8822B 0x0544
+#define REG_NAV_PROT_LEN_8822B 0x0546
+#define REG_BCN_CTRL_8822B 0x0550
+#define REG_BCN_CTRL_CLINT0_8822B 0x0551
+#define REG_MBID_NUM_8822B 0x0552
+#define REG_DUAL_TSF_RST_8822B 0x0553
+#define REG_MBSSID_BCN_SPACE_8822B 0x0554
+#define REG_DRVERLYINT_8822B 0x0558
+#define REG_BCNDMATIM_8822B 0x0559
+#define REG_ATIMWND_8822B 0x055A
+#define REG_USTIME_TSF_8822B 0x055C
+#define REG_BCN_MAX_ERR_8822B 0x055D
+#define REG_RXTSF_OFFSET_CCK_8822B 0x055E
+#define REG_RXTSF_OFFSET_OFDM_8822B 0x055F
+#define REG_TSFTR_8822B 0x0560
+#define REG_FREERUN_CNT_8822B 0x0568
+#define REG_ATIMWND1_V1_8822B 0x0570
+#define REG_TBTT_PROHIBIT_INFRA_8822B 0x0571
+#define REG_CTWND_8822B 0x0572
+#define REG_BCNIVLCUNT_8822B 0x0573
+#define REG_BCNDROPCTRL_8822B 0x0574
+#define REG_HGQ_TIMEOUT_PERIOD_8822B 0x0575
+#define REG_TXCMD_TIMEOUT_PERIOD_8822B 0x0576
+#define REG_MISC_CTRL_8822B 0x0577
+#define REG_BCN_CTRL_CLINT1_8822B 0x0578
+#define REG_BCN_CTRL_CLINT2_8822B 0x0579
+#define REG_BCN_CTRL_CLINT3_8822B 0x057A
+#define REG_EXTEND_CTRL_8822B 0x057B
+#define REG_P2PPS1_SPEC_STATE_8822B 0x057C
+#define REG_P2PPS1_STATE_8822B 0x057D
+#define REG_P2PPS2_SPEC_STATE_8822B 0x057E
+#define REG_P2PPS2_STATE_8822B 0x057F
+#define REG_PS_TIMER0_8822B 0x0580
+#define REG_PS_TIMER1_8822B 0x0584
+#define REG_PS_TIMER2_8822B 0x0588
+#define REG_TBTT_CTN_AREA_8822B 0x058C
+#define REG_FORCE_BCN_IFS_8822B 0x058E
+#define REG_TXOP_MIN_8822B 0x0590
+#define REG_PRE_BKF_TIME_8822B 0x0592
+#define REG_CROSS_TXOP_CTRL_8822B 0x0593
+#define REG_ATIMWND2_8822B 0x05A0
+#define REG_ATIMWND3_8822B 0x05A1
+#define REG_ATIMWND4_8822B 0x05A2
+#define REG_ATIMWND5_8822B 0x05A3
+#define REG_ATIMWND6_8822B 0x05A4
+#define REG_ATIMWND7_8822B 0x05A5
+#define REG_ATIMUGT_8822B 0x05A6
+#define REG_HIQ_NO_LMT_EN_8822B 0x05A7
+#define REG_DTIM_COUNTER_ROOT_8822B 0x05A8
+#define REG_DTIM_COUNTER_VAP1_8822B 0x05A9
+#define REG_DTIM_COUNTER_VAP2_8822B 0x05AA
+#define REG_DTIM_COUNTER_VAP3_8822B 0x05AB
+#define REG_DTIM_COUNTER_VAP4_8822B 0x05AC
+#define REG_DTIM_COUNTER_VAP5_8822B 0x05AD
+#define REG_DTIM_COUNTER_VAP6_8822B 0x05AE
+#define REG_DTIM_COUNTER_VAP7_8822B 0x05AF
+#define REG_DIS_ATIM_8822B 0x05B0
+#define REG_EARLY_128US_8822B 0x05B1
+#define REG_P2PPS1_CTRL_8822B 0x05B2
+#define REG_P2PPS2_CTRL_8822B 0x05B3
+#define REG_TIMER0_SRC_SEL_8822B 0x05B4
+#define REG_NOA_UNIT_SEL_8822B 0x05B5
+#define REG_P2POFF_DIS_TXTIME_8822B 0x05B7
+#define REG_MBSSID_BCN_SPACE2_8822B 0x05B8
+#define REG_MBSSID_BCN_SPACE3_8822B 0x05BC
+#define REG_ACMHWCTRL_8822B 0x05C0
+#define REG_ACMRSTCTRL_8822B 0x05C1
+#define REG_ACMAVG_8822B 0x05C2
+#define REG_VO_ADMTIME_8822B 0x05C4
+#define REG_VI_ADMTIME_8822B 0x05C6
+#define REG_BE_ADMTIME_8822B 0x05C8
+#define REG_EDCA_RANDOM_GEN_8822B 0x05CC
+#define REG_TXCMD_NOA_SEL_8822B 0x05CF
+#define REG_NOA_PARAM_8822B 0x05E0
+#define REG_P2P_RST_8822B 0x05F0
+#define REG_SCHEDULER_RST_8822B 0x05F1
+#define REG_SCH_TXCMD_8822B 0x05F8
+#define REG_PAGE5_DUMMY_8822B 0x05FC
+#define REG_CPUMGQ_TX_TIMER_8822B 0x1500
+#define REG_PS_TIMER_A_8822B 0x1504
+#define REG_PS_TIMER_B_8822B 0x1508
+#define REG_PS_TIMER_C_8822B 0x150C
+#define REG_PS_TIMER_ABC_CPUMGQ_TIMER_CRTL_8822B 0x1510
+#define REG_CPUMGQ_TX_TIMER_EARLY_8822B 0x1514
+#define REG_PS_TIMER_A_EARLY_8822B 0x1515
+#define REG_PS_TIMER_B_EARLY_8822B 0x1516
+#define REG_PS_TIMER_C_EARLY_8822B 0x1517
+#define REG_WMAC_CR_8822B 0x0600
+#define REG_WMAC_FWPKT_CR_8822B 0x0601
+#define REG_BWOPMODE_8822B 0x0603
+#define REG_TCR_8822B 0x0604
+#define REG_RCR_8822B 0x0608
+#define REG_RX_PKT_LIMIT_8822B 0x060C
+#define REG_RX_DLK_TIME_8822B 0x060D
+#define REG_RX_DRVINFO_SZ_8822B 0x060F
+#define REG_MACID_8822B 0x0610
+#define REG_BSSID_8822B 0x0618
+#define REG_MAR_8822B 0x0620
+#define REG_MBIDCAMCFG_1_8822B 0x0628
+#define REG_MBIDCAMCFG_2_8822B 0x062C
+#define REG_WMAC_TCR_TSFT_OFS_8822B 0x0630
+#define REG_UDF_THSD_8822B 0x0632
+#define REG_ZLD_NUM_8822B 0x0633
+#define REG_STMP_THSD_8822B 0x0634
+#define REG_WMAC_TXTIMEOUT_8822B 0x0635
+#define REG_MCU_TEST_2_V1_8822B 0x0636
+#define REG_USTIME_EDCA_8822B 0x0638
+#define REG_MAC_SPEC_SIFS_8822B 0x063A
+#define REG_RESP_SIFS_CCK_8822B 0x063C
+#define REG_RESP_SIFS_OFDM_8822B 0x063E
+#define REG_ACKTO_8822B 0x0640
+#define REG_CTS2TO_8822B 0x0641
+#define REG_EIFS_8822B 0x0642
+#define REG_NAV_CTRL_8822B 0x0650
+#define REG_BACAMCMD_8822B 0x0654
+#define REG_BACAMCONTENT_8822B 0x0658
+#define REG_LBDLY_8822B 0x0660
+#define REG_WMAC_BACAM_RPMEN_8822B 0x0661
+#define REG_TX_RX_8822B 0x0662
+#define REG_WMAC_BITMAP_CTL_8822B 0x0663
+#define REG_RXERR_RPT_8822B 0x0664
+#define REG_WMAC_TRXPTCL_CTL_8822B 0x0668
+#define REG_CAMCMD_8822B 0x0670
+#define REG_CAMWRITE_8822B 0x0674
+#define REG_CAMREAD_8822B 0x0678
+#define REG_CAMDBG_8822B 0x067C
+#define REG_SECCFG_8822B 0x0680
+#define REG_RXFILTER_CATEGORY_1_8822B 0x0682
+#define REG_RXFILTER_ACTION_1_8822B 0x0683
+#define REG_RXFILTER_CATEGORY_2_8822B 0x0684
+#define REG_RXFILTER_ACTION_2_8822B 0x0685
+#define REG_RXFILTER_CATEGORY_3_8822B 0x0686
+#define REG_RXFILTER_ACTION_3_8822B 0x0687
+#define REG_RXFLTMAP3_8822B 0x0688
+#define REG_RXFLTMAP4_8822B 0x068A
+#define REG_RXFLTMAP5_8822B 0x068C
+#define REG_RXFLTMAP6_8822B 0x068E
+#define REG_WOW_CTRL_8822B 0x0690
+#define REG_NAN_RX_TSF_FILTER_8822B 0x0691
+#define REG_PS_RX_INFO_8822B 0x0692
+#define REG_WMMPS_UAPSD_TID_8822B 0x0693
+#define REG_LPNAV_CTRL_8822B 0x0694
+#define REG_WKFMCAM_CMD_8822B 0x0698
+#define REG_WKFMCAM_RWD_8822B 0x069C
+#define REG_RXFLTMAP0_8822B 0x06A0
+#define REG_RXFLTMAP1_8822B 0x06A2
+#define REG_RXFLTMAP_8822B 0x06A4
+#define REG_BCN_PSR_RPT_8822B 0x06A8
+#define REG_FLC_RPC_8822B 0x06AC
+#define REG_FLC_RPCT_8822B 0x06AD
+#define REG_FLC_PTS_8822B 0x06AE
+#define REG_FLC_TRPC_8822B 0x06AF
+#define REG_RXPKTMON_CTRL_8822B 0x06B0
+#define REG_STATE_MON_8822B 0x06B4
+#define REG_ERROR_MON_8822B 0x06B8
+#define REG_SEARCH_MACID_8822B 0x06BC
+#define REG_BT_COEX_TABLE_8822B 0x06C0
+#define REG_RXCMD_0_8822B 0x06D0
+#define REG_RXCMD_1_8822B 0x06D4
+#define REG_WMAC_RESP_TXINFO_8822B 0x06D8
+#define REG_BBPSF_CTRL_8822B 0x06DC
+#define REG_P2P_RX_BCN_NOA_8822B 0x06E0
+#define REG_ASSOCIATED_BFMER0_INFO_8822B 0x06E4
+#define REG_ASSOCIATED_BFMER1_INFO_8822B 0x06EC
+#define REG_TX_CSI_RPT_PARAM_BW20_8822B 0x06F4
+#define REG_TX_CSI_RPT_PARAM_BW40_8822B 0x06F8
+#define REG_TX_CSI_RPT_PARAM_BW80_8822B 0x06FC
+#define REG_BCN_PSR_RPT2_8822B 0x1600
+#define REG_BCN_PSR_RPT3_8822B 0x1604
+#define REG_BCN_PSR_RPT4_8822B 0x1608
+#define REG_A1_ADDR_MASK_8822B 0x160C
+#define REG_MACID2_8822B 0x1620
+#define REG_BSSID2_8822B 0x1628
+#define REG_MACID3_8822B 0x1630
+#define REG_BSSID3_8822B 0x1638
+#define REG_MACID4_8822B 0x1640
+#define REG_BSSID4_8822B 0x1648
+#define REG_NOA_REPORT_8822B 0x1650
+#define REG_PWRBIT_SETTING_8822B 0x1660
+#define REG_WMAC_MU_BF_OPTION_8822B 0x167C
+#define REG_WMAC_MU_ARB_8822B 0x167E
+#define REG_WMAC_MU_OPTION_8822B 0x167F
+#define REG_WMAC_MU_BF_CTL_8822B 0x1680
+#define REG_WMAC_MU_BFRPT_PARA_8822B 0x1682
+#define REG_WMAC_ASSOCIATED_MU_BFMEE2_8822B 0x1684
+#define REG_WMAC_ASSOCIATED_MU_BFMEE3_8822B 0x1686
+#define REG_WMAC_ASSOCIATED_MU_BFMEE4_8822B 0x1688
+#define REG_WMAC_ASSOCIATED_MU_BFMEE5_8822B 0x168A
+#define REG_WMAC_ASSOCIATED_MU_BFMEE6_8822B 0x168C
+#define REG_WMAC_ASSOCIATED_MU_BFMEE7_8822B 0x168E
+#define REG_TRANSMIT_ADDRSS_0_8822B 0x16A0
+#define REG_TRANSMIT_ADDRSS_1_8822B 0x16A8
+#define REG_TRANSMIT_ADDRSS_2_8822B 0x16B0
+#define REG_TRANSMIT_ADDRSS_3_8822B 0x16B8
+#define REG_TRANSMIT_ADDRSS_4_8822B 0x16C0
+#define REG_MACID1_8822B 0x0700
+#define REG_BSSID1_8822B 0x0708
+#define REG_BCN_PSR_RPT1_8822B 0x0710
+#define REG_ASSOCIATED_BFMEE_SEL_8822B 0x0714
+#define REG_SND_PTCL_CTRL_8822B 0x0718
+#define REG_RX_CSI_RPT_INFO_8822B 0x071C
+#define REG_NS_ARP_CTRL_8822B 0x0720
+#define REG_NS_ARP_INFO_8822B 0x0724
+#define REG_BEAMFORMING_INFO_NSARP_V1_8822B 0x0728
+#define REG_BEAMFORMING_INFO_NSARP_8822B 0x072C
+#define REG_WMAC_RTX_CTX_SUBTYPE_CFG_8822B 0x0750
+#define REG_WMAC_SWAES_CFG_8822B 0x0760
+#define REG_BT_COEX_V2_8822B 0x0762
+#define REG_BT_COEX_8822B 0x0764
+#define REG_WLAN_ACT_MASK_CTRL_8822B 0x0768
+#define REG_BT_COEX_ENHANCED_INTR_CTRL_8822B 0x076E
+#define REG_BT_ACT_STATISTICS_8822B 0x0770
+#define REG_BT_STATISTICS_CONTROL_REGISTER_8822B 0x0778
+#define REG_BT_STATUS_REPORT_REGISTER_8822B 0x077C
+#define REG_BT_INTERRUPT_CONTROL_REGISTER_8822B 0x0780
+#define REG_WLAN_REPORT_TIME_OUT_CONTROL_REGISTER_8822B 0x0784
+#define REG_BT_ISOLATION_TABLE_REGISTER_REGISTER_8822B 0x0785
+#define REG_BT_INTERRUPT_STATUS_REGISTER_8822B 0x078F
+#define REG_BT_TDMA_TIME_REGISTER_8822B 0x0790
+#define REG_BT_ACT_REGISTER_8822B 0x0794
+#define REG_OBFF_CTRL_BASIC_8822B 0x0798
+#define REG_OBFF_CTRL2_TIMER_8822B 0x079C
+#define REG_LTR_CTRL_BASIC_8822B 0x07A0
+#define REG_LTR_CTRL2_TIMER_THRESHOLD_8822B 0x07A4
+#define REG_LTR_IDLE_LATENCY_V1_8822B 0x07A8
+#define REG_LTR_ACTIVE_LATENCY_V1_8822B 0x07AC
+#define REG_ANTENNA_TRAINING_CONTROL_REGISTER_8822B 0x07B0
+#define REG_WMAC_PKTCNT_RWD_8822B 0x07B8
+#define REG_WMAC_PKTCNT_CTRL_8822B 0x07BC
+#define REG_IQ_DUMP_8822B 0x07C0
+#define REG_WMAC_FTM_CTL_8822B 0x07CC
+#define REG_WMAC_IQ_MDPK_FUNC_8822B 0x07CE
+#define REG_WMAC_OPTION_FUNCTION_8822B 0x07D0
+#define REG_RX_FILTER_FUNCTION_8822B 0x07DA
+#define REG_NDP_SIG_8822B 0x07E0
+#define REG_TXCMD_INFO_FOR_RSP_PKT_8822B 0x07E4
+#define REG_RTS_ADDRESS_0_8822B 0x07F0
+#define REG_RTS_ADDRESS_1_8822B 0x07F8
+#define REG__RPFM_MAP1_8822B 0x07FE
+#define REG_WL2LTECOEX_INDIRECT_ACCESS_CTRL_V1_8822B 0x1700
+#define REG_WL2LTECOEX_INDIRECT_ACCESS_WRITE_DATA_V1_8822B 0x1704
+#define REG_WL2LTECOEX_INDIRECT_ACCESS_READ_DATA_V1_8822B 0x1708
+#define REG_SDIO_TX_CTRL_8822B 0x10250000
+#define REG_SDIO_HIMR_8822B 0x10250014
+#define REG_SDIO_HISR_8822B 0x10250018
+#define REG_SDIO_RX_REQ_LEN_8822B 0x1025001C
+#define REG_SDIO_FREE_TXPG_SEQ_V1_8822B 0x1025001F
+#define REG_SDIO_FREE_TXPG_8822B 0x10250020
+#define REG_SDIO_FREE_TXPG2_8822B 0x10250024
+#define REG_SDIO_OQT_FREE_TXPG_V1_8822B 0x10250028
+#define REG_SDIO_HTSFR_INFO_8822B 0x10250030
+#define REG_SDIO_HCPWM1_V2_8822B 0x10250038
+#define REG_SDIO_HCPWM2_V2_8822B 0x1025003A
+#define REG_SDIO_INDIRECT_REG_CFG_8822B 0x10250040
+#define REG_SDIO_INDIRECT_REG_DATA_8822B 0x10250044
+#define REG_SDIO_H2C_8822B 0x10250060
+#define REG_SDIO_C2H_8822B 0x10250064
+#define REG_SDIO_HRPWM1_8822B 0x10250080
+#define REG_SDIO_HRPWM2_8822B 0x10250082
+#define REG_SDIO_HPS_CLKR_8822B 0x10250084
+#define REG_SDIO_BUS_CTRL_8822B 0x10250085
+#define REG_SDIO_HSUS_CTRL_8822B 0x10250086
+#define REG_SDIO_RESPONSE_TIMER_8822B 0x10250088
+#define REG_SDIO_CMD_CRC_8822B 0x1025008A
+#define REG_SDIO_HSISR_8822B 0x10250090
+#define REG_SDIO_HSIMR_8822B 0x10250091
+#define REG_SDIO_ERR_RPT_8822B 0x102500C0
+#define REG_SDIO_CMD_ERRCNT_8822B 0x102500C1
+#define REG_SDIO_DATA_ERRCNT_8822B 0x102500C2
+#define REG_SDIO_CMD_ERR_CONTENT_8822B 0x102500C4
+#define REG_SDIO_CRC_ERR_IDX_8822B 0x102500C9
+#define REG_SDIO_DATA_CRC_8822B 0x102500CA
+#define REG_SDIO_DATA_REPLY_TIME_8822B 0x102500CB
+
+#endif
diff --git a/drivers/staging/rtlwifi/halmac/halmac_rx_bd_chip.h b/drivers/staging/rtlwifi/halmac/halmac_rx_bd_chip.h
new file mode 100644
index 000000000000..59ff1fecf73f
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_rx_bd_chip.h
@@ -0,0 +1,48 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HALMAC_RX_BD_CHIP_H_
+#define _HALMAC_RX_BD_CHIP_H_
+
+/*TXBD_DW0*/
+
+#define GET_RX_BD_RXFAIL_8822B(__rx_bd) GET_RX_BD_RXFAIL(__rx_bd)
+#define GET_RX_BD_TOTALRXPKTSIZE_8822B(__rx_bd)                                \
+	GET_RX_BD_TOTALRXPKTSIZE(__rx_bd)
+#define GET_RX_BD_RXTAG_8822B(__rx_bd) GET_RX_BD_RXTAG(__rx_bd)
+#define GET_RX_BD_FS_8822B(__rx_bd) GET_RX_BD_FS(__rx_bd)
+#define GET_RX_BD_LS_8822B(__rx_bd) GET_RX_BD_LS(__rx_bd)
+#define GET_RX_BD_RXBUFFSIZE_8822B(__rx_bd) GET_RX_BD_RXBUFFSIZE(__rx_bd)
+
+/*TXBD_DW1*/
+
+#define GET_RX_BD_PHYSICAL_ADDR_LOW_8822B(__rx_bd)                             \
+	GET_RX_BD_PHYSICAL_ADDR_LOW(__rx_bd)
+
+/*TXBD_DW2*/
+
+#define GET_RX_BD_PHYSICAL_ADDR_HIGH_8822B(__rx_bd)                            \
+	GET_RX_BD_PHYSICAL_ADDR_HIGH(__rx_bd)
+
+#endif
diff --git a/drivers/staging/rtlwifi/halmac/halmac_rx_bd_nic.h b/drivers/staging/rtlwifi/halmac/halmac_rx_bd_nic.h
new file mode 100644
index 000000000000..62817d808fbb
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_rx_bd_nic.h
@@ -0,0 +1,48 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HALMAC_RX_BD_NIC_H_
+#define _HALMAC_RX_BD_NIC_H_
+
+/*TXBD_DW0*/
+
+#define GET_RX_BD_RXFAIL(__rx_bd) LE_BITS_TO_4BYTE(__rx_bd + 0x00, 31, 1)
+#define GET_RX_BD_TOTALRXPKTSIZE(__rx_bd)                                      \
+	LE_BITS_TO_4BYTE(__rx_bd + 0x00, 16, 13)
+#define GET_RX_BD_RXTAG(__rx_bd) LE_BITS_TO_4BYTE(__rx_bd + 0x00, 16, 13)
+#define GET_RX_BD_FS(__rx_bd) LE_BITS_TO_4BYTE(__rx_bd + 0x00, 15, 1)
+#define GET_RX_BD_LS(__rx_bd) LE_BITS_TO_4BYTE(__rx_bd + 0x00, 14, 1)
+#define GET_RX_BD_RXBUFFSIZE(__rx_bd) LE_BITS_TO_4BYTE(__rx_bd + 0x00, 0, 14)
+
+/*TXBD_DW1*/
+
+#define GET_RX_BD_PHYSICAL_ADDR_LOW(__rx_bd)                                   \
+	LE_BITS_TO_4BYTE(__rx_bd + 0x04, 0, 32)
+
+/*TXBD_DW2*/
+
+#define GET_RX_BD_PHYSICAL_ADDR_HIGH(__rx_bd)                                  \
+	LE_BITS_TO_4BYTE(__rx_bd + 0x08, 0, 32)
+
+#endif
diff --git a/drivers/staging/rtlwifi/halmac/halmac_rx_desc_chip.h b/drivers/staging/rtlwifi/halmac/halmac_rx_desc_chip.h
new file mode 100644
index 000000000000..442120a14839
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_rx_desc_chip.h
@@ -0,0 +1,118 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HALMAC_RX_DESC_CHIP_H_
+#define _HALMAC_RX_DESC_CHIP_H_
+
+/*RXDESC_WORD0*/
+
+#define GET_RX_DESC_EOR_8822B(__rx_desc) GET_RX_DESC_EOR(__rx_desc)
+#define GET_RX_DESC_PHYPKTIDC_8822B(__rx_desc) GET_RX_DESC_PHYPKTIDC(__rx_desc)
+#define GET_RX_DESC_SWDEC_8822B(__rx_desc) GET_RX_DESC_SWDEC(__rx_desc)
+#define GET_RX_DESC_PHYST_8822B(__rx_desc) GET_RX_DESC_PHYST(__rx_desc)
+#define GET_RX_DESC_SHIFT_8822B(__rx_desc) GET_RX_DESC_SHIFT(__rx_desc)
+#define GET_RX_DESC_QOS_8822B(__rx_desc) GET_RX_DESC_QOS(__rx_desc)
+#define GET_RX_DESC_SECURITY_8822B(__rx_desc) GET_RX_DESC_SECURITY(__rx_desc)
+#define GET_RX_DESC_DRV_INFO_SIZE_8822B(__rx_desc)                             \
+	GET_RX_DESC_DRV_INFO_SIZE(__rx_desc)
+#define GET_RX_DESC_ICV_ERR_8822B(__rx_desc) GET_RX_DESC_ICV_ERR(__rx_desc)
+#define GET_RX_DESC_CRC32_8822B(__rx_desc) GET_RX_DESC_CRC32(__rx_desc)
+#define GET_RX_DESC_PKT_LEN_8822B(__rx_desc) GET_RX_DESC_PKT_LEN(__rx_desc)
+
+/*RXDESC_WORD1*/
+
+#define GET_RX_DESC_BC_8822B(__rx_desc) GET_RX_DESC_BC(__rx_desc)
+#define GET_RX_DESC_MC_8822B(__rx_desc) GET_RX_DESC_MC(__rx_desc)
+#define GET_RX_DESC_TY_PE_8822B(__rx_desc) GET_RX_DESC_TY_PE(__rx_desc)
+#define GET_RX_DESC_MF_8822B(__rx_desc) GET_RX_DESC_MF(__rx_desc)
+#define GET_RX_DESC_MD_8822B(__rx_desc) GET_RX_DESC_MD(__rx_desc)
+#define GET_RX_DESC_PWR_8822B(__rx_desc) GET_RX_DESC_PWR(__rx_desc)
+#define GET_RX_DESC_PAM_8822B(__rx_desc) GET_RX_DESC_PAM(__rx_desc)
+#define GET_RX_DESC_CHK_VLD_8822B(__rx_desc) GET_RX_DESC_CHK_VLD(__rx_desc)
+#define GET_RX_DESC_RX_IS_TCP_UDP_8822B(__rx_desc)                             \
+	GET_RX_DESC_RX_IS_TCP_UDP(__rx_desc)
+#define GET_RX_DESC_RX_IPV_8822B(__rx_desc) GET_RX_DESC_RX_IPV(__rx_desc)
+#define GET_RX_DESC_CHKERR_8822B(__rx_desc) GET_RX_DESC_CHKERR(__rx_desc)
+#define GET_RX_DESC_PAGGR_8822B(__rx_desc) GET_RX_DESC_PAGGR(__rx_desc)
+#define GET_RX_DESC_RXID_MATCH_8822B(__rx_desc)                                \
+	GET_RX_DESC_RXID_MATCH(__rx_desc)
+#define GET_RX_DESC_AMSDU_8822B(__rx_desc) GET_RX_DESC_AMSDU(__rx_desc)
+#define GET_RX_DESC_MACID_VLD_8822B(__rx_desc) GET_RX_DESC_MACID_VLD(__rx_desc)
+#define GET_RX_DESC_TID_8822B(__rx_desc) GET_RX_DESC_TID(__rx_desc)
+#define GET_RX_DESC_EXT_SECTYPE_8822B(__rx_desc)                               \
+	GET_RX_DESC_EXT_SECTYPE(__rx_desc)
+#define GET_RX_DESC_MACID_8822B(__rx_desc) GET_RX_DESC_MACID(__rx_desc)
+
+/*RXDESC_WORD2*/
+
+#define GET_RX_DESC_FCS_OK_8822B(__rx_desc) GET_RX_DESC_FCS_OK(__rx_desc)
+#define GET_RX_DESC_PPDU_CNT_8822B(__rx_desc) GET_RX_DESC_PPDU_CNT(__rx_desc)
+#define GET_RX_DESC_C2H_8822B(__rx_desc) GET_RX_DESC_C2H(__rx_desc)
+#define GET_RX_DESC_HWRSVD_8822B(__rx_desc) GET_RX_DESC_HWRSVD(__rx_desc)
+#define GET_RX_DESC_WLANHD_IV_LEN_8822B(__rx_desc)                             \
+	GET_RX_DESC_WLANHD_IV_LEN(__rx_desc)
+#define GET_RX_DESC_RX_IS_QOS_8822B(__rx_desc) GET_RX_DESC_RX_IS_QOS(__rx_desc)
+#define GET_RX_DESC_FRAG_8822B(__rx_desc) GET_RX_DESC_FRAG(__rx_desc)
+#define GET_RX_DESC_SEQ_8822B(__rx_desc) GET_RX_DESC_SEQ(__rx_desc)
+
+/*RXDESC_WORD3*/
+
+#define GET_RX_DESC_MAGIC_WAKE_8822B(__rx_desc)                                \
+	GET_RX_DESC_MAGIC_WAKE(__rx_desc)
+#define GET_RX_DESC_UNICAST_WAKE_8822B(__rx_desc)                              \
+	GET_RX_DESC_UNICAST_WAKE(__rx_desc)
+#define GET_RX_DESC_PATTERN_MATCH_8822B(__rx_desc)                             \
+	GET_RX_DESC_PATTERN_MATCH(__rx_desc)
+#define GET_RX_DESC_RXPAYLOAD_MATCH_8822B(__rx_desc)                           \
+	GET_RX_DESC_RXPAYLOAD_MATCH(__rx_desc)
+#define GET_RX_DESC_RXPAYLOAD_ID_8822B(__rx_desc)                              \
+	GET_RX_DESC_RXPAYLOAD_ID(__rx_desc)
+#define GET_RX_DESC_DMA_AGG_NUM_8822B(__rx_desc)                               \
+	GET_RX_DESC_DMA_AGG_NUM(__rx_desc)
+#define GET_RX_DESC_BSSID_FIT_1_0_8822B(__rx_desc)                             \
+	GET_RX_DESC_BSSID_FIT_1_0(__rx_desc)
+#define GET_RX_DESC_EOSP_8822B(__rx_desc) GET_RX_DESC_EOSP(__rx_desc)
+#define GET_RX_DESC_HTC_8822B(__rx_desc) GET_RX_DESC_HTC(__rx_desc)
+#define GET_RX_DESC_BSSID_FIT_4_2_8822B(__rx_desc)                             \
+	GET_RX_DESC_BSSID_FIT_4_2(__rx_desc)
+#define GET_RX_DESC_RX_RATE_8822B(__rx_desc) GET_RX_DESC_RX_RATE(__rx_desc)
+
+/*RXDESC_WORD4*/
+
+#define GET_RX_DESC_A1_FIT_8822B(__rx_desc) GET_RX_DESC_A1_FIT(__rx_desc)
+#define GET_RX_DESC_MACID_RPT_BUFF_8822B(__rx_desc)                            \
+	GET_RX_DESC_MACID_RPT_BUFF(__rx_desc)
+#define GET_RX_DESC_RX_PRE_NDP_VLD_8822B(__rx_desc)                            \
+	GET_RX_DESC_RX_PRE_NDP_VLD(__rx_desc)
+#define GET_RX_DESC_RX_SCRAMBLER_8822B(__rx_desc)                              \
+	GET_RX_DESC_RX_SCRAMBLER(__rx_desc)
+#define GET_RX_DESC_RX_EOF_8822B(__rx_desc) GET_RX_DESC_RX_EOF(__rx_desc)
+#define GET_RX_DESC_PATTERN_IDX_8822B(__rx_desc)                               \
+	GET_RX_DESC_PATTERN_IDX(__rx_desc)
+
+/*RXDESC_WORD5*/
+
+#define GET_RX_DESC_TSFL_8822B(__rx_desc) GET_RX_DESC_TSFL(__rx_desc)
+
+#endif
diff --git a/drivers/staging/rtlwifi/halmac/halmac_rx_desc_nic.h b/drivers/staging/rtlwifi/halmac/halmac_rx_desc_nic.h
new file mode 100644
index 000000000000..8256c3605072
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_rx_desc_nic.h
@@ -0,0 +1,133 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HALMAC_RX_DESC_NIC_H_
+#define _HALMAC_RX_DESC_NIC_H_
+
+/*RXDESC_WORD0*/
+
+#define GET_RX_DESC_EOR(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x00, 30, 1)
+#define GET_RX_DESC_PHYPKTIDC(__rx_desc)                                       \
+	LE_BITS_TO_4BYTE(__rx_desc + 0x00, 28, 1)
+#define GET_RX_DESC_SWDEC(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x00, 27, 1)
+#define GET_RX_DESC_PHYST(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x00, 26, 1)
+#define GET_RX_DESC_SHIFT(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x00, 24, 2)
+#define GET_RX_DESC_QOS(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x00, 23, 1)
+#define GET_RX_DESC_SECURITY(__rx_desc)                                        \
+	LE_BITS_TO_4BYTE(__rx_desc + 0x00, 20, 3)
+#define GET_RX_DESC_DRV_INFO_SIZE(__rx_desc)                                   \
+	LE_BITS_TO_4BYTE(__rx_desc + 0x00, 16, 4)
+#define GET_RX_DESC_ICV_ERR(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x00, 15, 1)
+#define GET_RX_DESC_CRC32(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x00, 14, 1)
+#define GET_RX_DESC_PKT_LEN(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x00, 0, 14)
+
+/*RXDESC_WORD1*/
+
+#define GET_RX_DESC_BC(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x04, 31, 1)
+#define GET_RX_DESC_MC(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x04, 30, 1)
+#define GET_RX_DESC_TY_PE(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x04, 28, 2)
+#define GET_RX_DESC_MF(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x04, 27, 1)
+#define GET_RX_DESC_MD(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x04, 26, 1)
+#define GET_RX_DESC_PWR(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x04, 25, 1)
+#define GET_RX_DESC_PAM(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x04, 24, 1)
+#define GET_RX_DESC_CHK_VLD(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x04, 23, 1)
+#define GET_RX_DESC_RX_IS_TCP_UDP(__rx_desc)                                   \
+	LE_BITS_TO_4BYTE(__rx_desc + 0x04, 22, 1)
+#define GET_RX_DESC_RX_IPV(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x04, 21, 1)
+#define GET_RX_DESC_CHKERR(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x04, 20, 1)
+#define GET_RX_DESC_PAGGR(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x04, 15, 1)
+#define GET_RX_DESC_RXID_MATCH(__rx_desc)                                      \
+	LE_BITS_TO_4BYTE(__rx_desc + 0x04, 14, 1)
+#define GET_RX_DESC_AMSDU(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x04, 13, 1)
+#define GET_RX_DESC_MACID_VLD(__rx_desc)                                       \
+	LE_BITS_TO_4BYTE(__rx_desc + 0x04, 12, 1)
+#define GET_RX_DESC_TID(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x04, 8, 4)
+
+#define GET_RX_DESC_EXT_SECTYPE(__rx_desc)                                     \
+	LE_BITS_TO_4BYTE(__rx_desc + 0x04, 7, 1)
+
+#define GET_RX_DESC_MACID(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x04, 0, 7)
+
+/*RXDESC_WORD2*/
+
+#define GET_RX_DESC_FCS_OK(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x08, 31, 1)
+
+#define GET_RX_DESC_PPDU_CNT(__rx_desc)                                        \
+	LE_BITS_TO_4BYTE(__rx_desc + 0x08, 29, 2)
+
+#define GET_RX_DESC_C2H(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x08, 28, 1)
+#define GET_RX_DESC_HWRSVD(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x08, 24, 4)
+#define GET_RX_DESC_WLANHD_IV_LEN(__rx_desc)                                   \
+	LE_BITS_TO_4BYTE(__rx_desc + 0x08, 18, 6)
+#define GET_RX_DESC_RX_IS_QOS(__rx_desc)                                       \
+	LE_BITS_TO_4BYTE(__rx_desc + 0x08, 16, 1)
+#define GET_RX_DESC_FRAG(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x08, 12, 4)
+#define GET_RX_DESC_SEQ(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x08, 0, 12)
+
+/*RXDESC_WORD3*/
+
+#define GET_RX_DESC_MAGIC_WAKE(__rx_desc)                                      \
+	LE_BITS_TO_4BYTE(__rx_desc + 0x0C, 31, 1)
+#define GET_RX_DESC_UNICAST_WAKE(__rx_desc)                                    \
+	LE_BITS_TO_4BYTE(__rx_desc + 0x0C, 30, 1)
+#define GET_RX_DESC_PATTERN_MATCH(__rx_desc)                                   \
+	LE_BITS_TO_4BYTE(__rx_desc + 0x0C, 29, 1)
+
+#define GET_RX_DESC_RXPAYLOAD_MATCH(__rx_desc)                                 \
+	LE_BITS_TO_4BYTE(__rx_desc + 0x0C, 28, 1)
+#define GET_RX_DESC_RXPAYLOAD_ID(__rx_desc)                                    \
+	LE_BITS_TO_4BYTE(__rx_desc + 0x0C, 24, 4)
+
+#define GET_RX_DESC_DMA_AGG_NUM(__rx_desc)                                     \
+	LE_BITS_TO_4BYTE(__rx_desc + 0x0C, 16, 8)
+#define GET_RX_DESC_BSSID_FIT_1_0(__rx_desc)                                   \
+	LE_BITS_TO_4BYTE(__rx_desc + 0x0C, 12, 2)
+#define GET_RX_DESC_EOSP(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x0C, 11, 1)
+#define GET_RX_DESC_HTC(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x0C, 10, 1)
+
+#define GET_RX_DESC_BSSID_FIT_4_2(__rx_desc)                                   \
+	LE_BITS_TO_4BYTE(__rx_desc + 0x0C, 7, 3)
+
+#define GET_RX_DESC_RX_RATE(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x0C, 0, 7)
+
+/*RXDESC_WORD4*/
+
+#define GET_RX_DESC_A1_FIT(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x10, 24, 5)
+
+#define GET_RX_DESC_MACID_RPT_BUFF(__rx_desc)                                  \
+	LE_BITS_TO_4BYTE(__rx_desc + 0x10, 17, 7)
+#define GET_RX_DESC_RX_PRE_NDP_VLD(__rx_desc)                                  \
+	LE_BITS_TO_4BYTE(__rx_desc + 0x10, 16, 1)
+#define GET_RX_DESC_RX_SCRAMBLER(__rx_desc)                                    \
+	LE_BITS_TO_4BYTE(__rx_desc + 0x10, 9, 7)
+#define GET_RX_DESC_RX_EOF(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x10, 8, 1)
+
+#define GET_RX_DESC_PATTERN_IDX(__rx_desc)                                     \
+	LE_BITS_TO_4BYTE(__rx_desc + 0x10, 0, 8)
+
+/*RXDESC_WORD5*/
+
+#define GET_RX_DESC_TSFL(__rx_desc) LE_BITS_TO_4BYTE(__rx_desc + 0x14, 0, 32)
+
+#endif
diff --git a/drivers/staging/rtlwifi/halmac/halmac_sdio_reg.h b/drivers/staging/rtlwifi/halmac/halmac_sdio_reg.h
new file mode 100644
index 000000000000..8967699e3784
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_sdio_reg.h
@@ -0,0 +1,62 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef __HALMAC_SDIO_REG_H__
+#define __HALMAC_SDIO_REG_H__
+
+/* SDIO CMD address mapping */
+
+#define HALMAC_SDIO_4BYTE_LEN_MASK 0x1FFF
+#define HALMAC_SDIO_LOCAL_MSK 0x0FFF
+#define HALMAC_WLAN_MAC_REG_MSK 0xFFFF
+#define HALMAC_WLAN_IOREG_MSK 0xFFFF
+
+/* Sdio address for SDIO Local Reg, TRX FIFO, MAC Reg */
+enum halmac_sdio_cmd_addr {
+	HALMAC_SDIO_CMD_ADDR_SDIO_REG = 0,
+	HALMAC_SDIO_CMD_ADDR_MAC_REG = 8,
+	HALMAC_SDIO_CMD_ADDR_TXFF_HIGH = 4,
+	HALMAC_SDIO_CMD_ADDR_TXFF_LOW = 6,
+	HALMAC_SDIO_CMD_ADDR_TXFF_NORMAL = 5,
+	HALMAC_SDIO_CMD_ADDR_TXFF_EXTRA = 7,
+	HALMAC_SDIO_CMD_ADDR_RXFF = 7,
+};
+
+/* IO Bus domain address mapping */
+#define SDIO_LOCAL_OFFSET 0x10250000
+#define WLAN_IOREG_OFFSET 0x10260000
+#define FW_FIFO_OFFSET 0x10270000
+#define TX_HIQ_OFFSET 0x10310000
+#define TX_MIQ_OFFSET 0x10320000
+#define TX_LOQ_OFFSET 0x10330000
+#define TX_EXQ_OFFSET 0x10350000
+#define RX_RXOFF_OFFSET 0x10340000
+
+/* Get TX WLAN FIFO information in CMD53 addr  */
+#define GET_WLAN_TXFF_DEVICE_ID(__cmd53_addr)                                  \
+	LE_BITS_TO_4BYTE((u32 *)__cmd53_addr, 13, 4)
+#define GET_WLAN_TXFF_PKT_SIZE(__cmd53_addr)                                   \
+	(LE_BITS_TO_4BYTE((u32 *)__cmd53_addr, 0, 13) << 2)
+
+#endif /* __HALMAC_SDIO_REG_H__ */
diff --git a/drivers/staging/rtlwifi/halmac/halmac_tx_bd_chip.h b/drivers/staging/rtlwifi/halmac/halmac_tx_bd_chip.h
new file mode 100644
index 000000000000..d5c9da247ca3
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_tx_bd_chip.h
@@ -0,0 +1,118 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HALMAC_TX_BD_CHIP_H_
+#define _HALMAC_TX_BD_CHIP_H_
+
+/*TXBD_DW0*/
+
+#define SET_TX_BD_OWN_8822B(__tx_bd, __value) SET_TX_BD_OWN(__tx_bd, __value)
+#define GET_TX_BD_OWN_8822B(__tx_bd) GET_TX_BD_OWN(__tx_bd)
+#define SET_TX_BD_PSB_8822B(__tx_bd, __value) SET_TX_BD_PSB(__tx_bd, __value)
+#define GET_TX_BD_PSB_8822B(__tx_bd) GET_TX_BD_PSB(__tx_bd)
+#define SET_TX_BD_TX_BUFF_SIZE0_8822B(__tx_bd, __value)                        \
+	SET_TX_BD_TX_BUFF_SIZE0(__tx_bd, __value)
+#define GET_TX_BD_TX_BUFF_SIZE0_8822B(__tx_bd) GET_TX_BD_TX_BUFF_SIZE0(__tx_bd)
+
+/*TXBD_DW1*/
+
+#define SET_TX_BD_PHYSICAL_ADDR0_LOW_8822B(__tx_bd, __value)                   \
+	SET_TX_BD_PHYSICAL_ADDR0_LOW(__tx_bd, __value)
+#define GET_TX_BD_PHYSICAL_ADDR0_LOW_8822B(__tx_bd)                            \
+	GET_TX_BD_PHYSICAL_ADDR0_LOW(__tx_bd)
+
+/*TXBD_DW2*/
+
+#define SET_TX_BD_PHYSICAL_ADDR0_HIGH_8822B(__tx_bd, __value)                  \
+	SET_TX_BD_PHYSICAL_ADDR0_HIGH(__tx_bd, __value)
+#define GET_TX_BD_PHYSICAL_ADDR0_HIGH_8822B(__tx_bd)                           \
+	GET_TX_BD_PHYSICAL_ADDR0_HIGH(__tx_bd)
+
+/*TXBD_DW4*/
+
+#define SET_TX_BD_A1_8822B(__tx_bd, __value) SET_TX_BD_A1(__tx_bd, __value)
+#define GET_TX_BD_A1_8822B(__tx_bd) GET_TX_BD_A1(__tx_bd)
+#define SET_TX_BD_TX_BUFF_SIZE1_8822B(__tx_bd, __value)                        \
+	SET_TX_BD_TX_BUFF_SIZE1(__tx_bd, __value)
+#define GET_TX_BD_TX_BUFF_SIZE1_8822B(__tx_bd) GET_TX_BD_TX_BUFF_SIZE1(__tx_bd)
+
+/*TXBD_DW5*/
+
+#define SET_TX_BD_PHYSICAL_ADDR1_LOW_8822B(__tx_bd, __value)                   \
+	SET_TX_BD_PHYSICAL_ADDR1_LOW(__tx_bd, __value)
+#define GET_TX_BD_PHYSICAL_ADDR1_LOW_8822B(__tx_bd)                            \
+	GET_TX_BD_PHYSICAL_ADDR1_LOW(__tx_bd)
+
+/*TXBD_DW6*/
+
+#define SET_TX_BD_PHYSICAL_ADDR1_HIGH_8822B(__tx_bd, __value)                  \
+	SET_TX_BD_PHYSICAL_ADDR1_HIGH(__tx_bd, __value)
+#define GET_TX_BD_PHYSICAL_ADDR1_HIGH_8822B(__tx_bd)                           \
+	GET_TX_BD_PHYSICAL_ADDR1_HIGH(__tx_bd)
+
+/*TXBD_DW8*/
+
+#define SET_TX_BD_A2_8822B(__tx_bd, __value) SET_TX_BD_A2(__tx_bd, __value)
+#define GET_TX_BD_A2_8822B(__tx_bd) GET_TX_BD_A2(__tx_bd)
+#define SET_TX_BD_TX_BUFF_SIZE2_8822B(__tx_bd, __value)                        \
+	SET_TX_BD_TX_BUFF_SIZE2(__tx_bd, __value)
+#define GET_TX_BD_TX_BUFF_SIZE2_8822B(__tx_bd) GET_TX_BD_TX_BUFF_SIZE2(__tx_bd)
+
+/*TXBD_DW9*/
+
+#define SET_TX_BD_PHYSICAL_ADDR2_LOW_8822B(__tx_bd, __value)                   \
+	SET_TX_BD_PHYSICAL_ADDR2_LOW(__tx_bd, __value)
+#define GET_TX_BD_PHYSICAL_ADDR2_LOW_8822B(__tx_bd)                            \
+	GET_TX_BD_PHYSICAL_ADDR2_LOW(__tx_bd)
+
+/*TXBD_DW10*/
+
+#define SET_TX_BD_PHYSICAL_ADDR2_HIGH_8822B(__tx_bd, __value)                  \
+	SET_TX_BD_PHYSICAL_ADDR2_HIGH(__tx_bd, __value)
+#define GET_TX_BD_PHYSICAL_ADDR2_HIGH_8822B(__tx_bd)                           \
+	GET_TX_BD_PHYSICAL_ADDR2_HIGH(__tx_bd)
+
+/*TXBD_DW12*/
+
+#define SET_TX_BD_A3_8822B(__tx_bd, __value) SET_TX_BD_A3(__tx_bd, __value)
+#define GET_TX_BD_A3_8822B(__tx_bd) GET_TX_BD_A3(__tx_bd)
+#define SET_TX_BD_TX_BUFF_SIZE3_8822B(__tx_bd, __value)                        \
+	SET_TX_BD_TX_BUFF_SIZE3(__tx_bd, __value)
+#define GET_TX_BD_TX_BUFF_SIZE3_8822B(__tx_bd) GET_TX_BD_TX_BUFF_SIZE3(__tx_bd)
+
+/*TXBD_DW13*/
+
+#define SET_TX_BD_PHYSICAL_ADDR3_LOW_8822B(__tx_bd, __value)                   \
+	SET_TX_BD_PHYSICAL_ADDR3_LOW(__tx_bd, __value)
+#define GET_TX_BD_PHYSICAL_ADDR3_LOW_8822B(__tx_bd)                            \
+	GET_TX_BD_PHYSICAL_ADDR3_LOW(__tx_bd)
+
+/*TXBD_DW14*/
+
+#define SET_TX_BD_PHYSICAL_ADDR3_HIGH_8822B(__tx_bd, __value)                  \
+	SET_TX_BD_PHYSICAL_ADDR3_HIGH(__tx_bd, __value)
+#define GET_TX_BD_PHYSICAL_ADDR3_HIGH_8822B(__tx_bd)                           \
+	GET_TX_BD_PHYSICAL_ADDR3_HIGH(__tx_bd)
+
+#endif
diff --git a/drivers/staging/rtlwifi/halmac/halmac_tx_bd_nic.h b/drivers/staging/rtlwifi/halmac/halmac_tx_bd_nic.h
new file mode 100644
index 000000000000..43c2261ab083
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_tx_bd_nic.h
@@ -0,0 +1,123 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HALMAC_TX_BD_NIC_H_
+#define _HALMAC_TX_BD_NIC_H_
+
+/*TXBD_DW0*/
+
+#define SET_TX_BD_OWN(__tx_bd, __value)                                        \
+	SET_BITS_TO_LE_4BYTE(__tx_bd + 0x00, 31, 1, __value)
+#define GET_TX_BD_OWN(__tx_bd) LE_BITS_TO_4BYTE(__tx_bd + 0x00, 31, 1)
+#define SET_TX_BD_PSB(__tx_bd, __value)                                        \
+	SET_BITS_TO_LE_4BYTE(__tx_bd + 0x00, 16, 8, __value)
+#define GET_TX_BD_PSB(__tx_bd) LE_BITS_TO_4BYTE(__tx_bd + 0x00, 16, 8)
+#define SET_TX_BD_TX_BUFF_SIZE0(__tx_bd, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__tx_bd + 0x00, 0, 16, __value)
+#define GET_TX_BD_TX_BUFF_SIZE0(__tx_bd) LE_BITS_TO_4BYTE(__tx_bd + 0x00, 0, 16)
+
+/*TXBD_DW1*/
+
+#define SET_TX_BD_PHYSICAL_ADDR0_LOW(__tx_bd, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__tx_bd + 0x04, 0, 32, __value)
+#define GET_TX_BD_PHYSICAL_ADDR0_LOW(__tx_bd)                                  \
+	LE_BITS_TO_4BYTE(__tx_bd + 0x04, 0, 32)
+
+/*TXBD_DW2*/
+
+#define SET_TX_BD_PHYSICAL_ADDR0_HIGH(__tx_bd, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__tx_bd + 0x08, 0, 32, __value)
+#define GET_TX_BD_PHYSICAL_ADDR0_HIGH(__tx_bd)                                 \
+	LE_BITS_TO_4BYTE(__tx_bd + 0x08, 0, 32)
+
+/*TXBD_DW4*/
+
+#define SET_TX_BD_A1(__tx_bd, __value)                                         \
+	SET_BITS_TO_LE_4BYTE(__tx_bd + 0x10, 31, 1, __value)
+#define GET_TX_BD_A1(__tx_bd) LE_BITS_TO_4BYTE(__tx_bd + 0x10, 31, 1)
+#define SET_TX_BD_TX_BUFF_SIZE1(__tx_bd, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__tx_bd + 0x10, 0, 16, __value)
+#define GET_TX_BD_TX_BUFF_SIZE1(__tx_bd) LE_BITS_TO_4BYTE(__tx_bd + 0x10, 0, 16)
+
+/*TXBD_DW5*/
+
+#define SET_TX_BD_PHYSICAL_ADDR1_LOW(__tx_bd, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__tx_bd + 0x14, 0, 32, __value)
+#define GET_TX_BD_PHYSICAL_ADDR1_LOW(__tx_bd)                                  \
+	LE_BITS_TO_4BYTE(__tx_bd + 0x14, 0, 32)
+
+/*TXBD_DW6*/
+
+#define SET_TX_BD_PHYSICAL_ADDR1_HIGH(__tx_bd, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__tx_bd + 0x18, 0, 32, __value)
+#define GET_TX_BD_PHYSICAL_ADDR1_HIGH(__tx_bd)                                 \
+	LE_BITS_TO_4BYTE(__tx_bd + 0x18, 0, 32)
+
+/*TXBD_DW8*/
+
+#define SET_TX_BD_A2(__tx_bd, __value)                                         \
+	SET_BITS_TO_LE_4BYTE(__tx_bd + 0x20, 31, 1, __value)
+#define GET_TX_BD_A2(__tx_bd) LE_BITS_TO_4BYTE(__tx_bd + 0x20, 31, 1)
+#define SET_TX_BD_TX_BUFF_SIZE2(__tx_bd, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__tx_bd + 0x20, 0, 16, __value)
+#define GET_TX_BD_TX_BUFF_SIZE2(__tx_bd) LE_BITS_TO_4BYTE(__tx_bd + 0x20, 0, 16)
+
+/*TXBD_DW9*/
+
+#define SET_TX_BD_PHYSICAL_ADDR2_LOW(__tx_bd, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__tx_bd + 0x24, 0, 32, __value)
+#define GET_TX_BD_PHYSICAL_ADDR2_LOW(__tx_bd)                                  \
+	LE_BITS_TO_4BYTE(__tx_bd + 0x24, 0, 32)
+
+/*TXBD_DW10*/
+
+#define SET_TX_BD_PHYSICAL_ADDR2_HIGH(__tx_bd, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__tx_bd + 0x28, 0, 32, __value)
+#define GET_TX_BD_PHYSICAL_ADDR2_HIGH(__tx_bd)                                 \
+	LE_BITS_TO_4BYTE(__tx_bd + 0x28, 0, 32)
+
+/*TXBD_DW12*/
+
+#define SET_TX_BD_A3(__tx_bd, __value)                                         \
+	SET_BITS_TO_LE_4BYTE(__tx_bd + 0x30, 31, 1, __value)
+#define GET_TX_BD_A3(__tx_bd) LE_BITS_TO_4BYTE(__tx_bd + 0x30, 31, 1)
+#define SET_TX_BD_TX_BUFF_SIZE3(__tx_bd, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__tx_bd + 0x30, 0, 16, __value)
+#define GET_TX_BD_TX_BUFF_SIZE3(__tx_bd) LE_BITS_TO_4BYTE(__tx_bd + 0x30, 0, 16)
+
+/*TXBD_DW13*/
+
+#define SET_TX_BD_PHYSICAL_ADDR3_LOW(__tx_bd, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__tx_bd + 0x34, 0, 32, __value)
+#define GET_TX_BD_PHYSICAL_ADDR3_LOW(__tx_bd)                                  \
+	LE_BITS_TO_4BYTE(__tx_bd + 0x34, 0, 32)
+
+/*TXBD_DW14*/
+
+#define SET_TX_BD_PHYSICAL_ADDR3_HIGH(__tx_bd, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__tx_bd + 0x38, 0, 32, __value)
+#define GET_TX_BD_PHYSICAL_ADDR3_HIGH(__tx_bd)                                 \
+	LE_BITS_TO_4BYTE(__tx_bd + 0x38, 0, 32)
+
+#endif
diff --git a/drivers/staging/rtlwifi/halmac/halmac_tx_desc_chip.h b/drivers/staging/rtlwifi/halmac/halmac_tx_desc_chip.h
new file mode 100644
index 000000000000..fd1aa39c4bed
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_tx_desc_chip.h
@@ -0,0 +1,444 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HALMAC_TX_DESC_CHIP_H_
+#define _HALMAC_TX_DESC_CHIP_H_
+
+/*TXDESC_WORD0*/
+
+#define SET_TX_DESC_DISQSELSEQ_8822B(__tx_desc, __value)                       \
+	SET_TX_DESC_DISQSELSEQ(__tx_desc, __value)
+#define GET_TX_DESC_DISQSELSEQ_8822B(__tx_desc)                                \
+	GET_TX_DESC_DISQSELSEQ(__tx_desc)
+#define SET_TX_DESC_GF_8822B(__tx_desc, __value)                               \
+	SET_TX_DESC_GF(__tx_desc, __value)
+#define GET_TX_DESC_GF_8822B(__tx_desc) GET_TX_DESC_GF(__tx_desc)
+#define SET_TX_DESC_NO_ACM_8822B(__tx_desc, __value)                           \
+	SET_TX_DESC_NO_ACM(__tx_desc, __value)
+#define GET_TX_DESC_NO_ACM_8822B(__tx_desc) GET_TX_DESC_NO_ACM(__tx_desc)
+#define SET_TX_DESC_BCNPKT_TSF_CTRL_8822B(__tx_desc, __value)                  \
+	SET_TX_DESC_BCNPKT_TSF_CTRL(__tx_desc, __value)
+#define GET_TX_DESC_BCNPKT_TSF_CTRL_8822B(__tx_desc)                           \
+	GET_TX_DESC_BCNPKT_TSF_CTRL(__tx_desc)
+#define SET_TX_DESC_AMSDU_PAD_EN_8822B(__tx_desc, __value)                     \
+	SET_TX_DESC_AMSDU_PAD_EN(__tx_desc, __value)
+#define GET_TX_DESC_AMSDU_PAD_EN_8822B(__tx_desc)                              \
+	GET_TX_DESC_AMSDU_PAD_EN(__tx_desc)
+#define SET_TX_DESC_LS_8822B(__tx_desc, __value)                               \
+	SET_TX_DESC_LS(__tx_desc, __value)
+#define GET_TX_DESC_LS_8822B(__tx_desc) GET_TX_DESC_LS(__tx_desc)
+#define SET_TX_DESC_HTC_8822B(__tx_desc, __value)                              \
+	SET_TX_DESC_HTC(__tx_desc, __value)
+#define GET_TX_DESC_HTC_8822B(__tx_desc) GET_TX_DESC_HTC(__tx_desc)
+#define SET_TX_DESC_BMC_8822B(__tx_desc, __value)                              \
+	SET_TX_DESC_BMC(__tx_desc, __value)
+#define GET_TX_DESC_BMC_8822B(__tx_desc) GET_TX_DESC_BMC(__tx_desc)
+#define SET_TX_DESC_OFFSET_8822B(__tx_desc, __value)                           \
+	SET_TX_DESC_OFFSET(__tx_desc, __value)
+#define GET_TX_DESC_OFFSET_8822B(__tx_desc) GET_TX_DESC_OFFSET(__tx_desc)
+#define SET_TX_DESC_TXPKTSIZE_8822B(__tx_desc, __value)                        \
+	SET_TX_DESC_TXPKTSIZE(__tx_desc, __value)
+#define GET_TX_DESC_TXPKTSIZE_8822B(__tx_desc) GET_TX_DESC_TXPKTSIZE(__tx_desc)
+
+/*TXDESC_WORD1*/
+
+#define SET_TX_DESC_MOREDATA_8822B(__tx_desc, __value)                         \
+	SET_TX_DESC_MOREDATA(__tx_desc, __value)
+#define GET_TX_DESC_MOREDATA_8822B(__tx_desc) GET_TX_DESC_MOREDATA(__tx_desc)
+#define SET_TX_DESC_PKT_OFFSET_8822B(__tx_desc, __value)                       \
+	SET_TX_DESC_PKT_OFFSET(__tx_desc, __value)
+#define GET_TX_DESC_PKT_OFFSET_8822B(__tx_desc)                                \
+	GET_TX_DESC_PKT_OFFSET(__tx_desc)
+#define SET_TX_DESC_SEC_TYPE_8822B(__tx_desc, __value)                         \
+	SET_TX_DESC_SEC_TYPE(__tx_desc, __value)
+#define GET_TX_DESC_SEC_TYPE_8822B(__tx_desc) GET_TX_DESC_SEC_TYPE(__tx_desc)
+#define SET_TX_DESC_EN_DESC_ID_8822B(__tx_desc, __value)                       \
+	SET_TX_DESC_EN_DESC_ID(__tx_desc, __value)
+#define GET_TX_DESC_EN_DESC_ID_8822B(__tx_desc)                                \
+	GET_TX_DESC_EN_DESC_ID(__tx_desc)
+#define SET_TX_DESC_RATE_ID_8822B(__tx_desc, __value)                          \
+	SET_TX_DESC_RATE_ID(__tx_desc, __value)
+#define GET_TX_DESC_RATE_ID_8822B(__tx_desc) GET_TX_DESC_RATE_ID(__tx_desc)
+#define SET_TX_DESC_PIFS_8822B(__tx_desc, __value)                             \
+	SET_TX_DESC_PIFS(__tx_desc, __value)
+#define GET_TX_DESC_PIFS_8822B(__tx_desc) GET_TX_DESC_PIFS(__tx_desc)
+#define SET_TX_DESC_LSIG_TXOP_EN_8822B(__tx_desc, __value)                     \
+	SET_TX_DESC_LSIG_TXOP_EN(__tx_desc, __value)
+#define GET_TX_DESC_LSIG_TXOP_EN_8822B(__tx_desc)                              \
+	GET_TX_DESC_LSIG_TXOP_EN(__tx_desc)
+#define SET_TX_DESC_RD_NAV_EXT_8822B(__tx_desc, __value)                       \
+	SET_TX_DESC_RD_NAV_EXT(__tx_desc, __value)
+#define GET_TX_DESC_RD_NAV_EXT_8822B(__tx_desc)                                \
+	GET_TX_DESC_RD_NAV_EXT(__tx_desc)
+#define SET_TX_DESC_QSEL_8822B(__tx_desc, __value)                             \
+	SET_TX_DESC_QSEL(__tx_desc, __value)
+#define GET_TX_DESC_QSEL_8822B(__tx_desc) GET_TX_DESC_QSEL(__tx_desc)
+#define SET_TX_DESC_MACID_8822B(__tx_desc, __value)                            \
+	SET_TX_DESC_MACID(__tx_desc, __value)
+#define GET_TX_DESC_MACID_8822B(__tx_desc) GET_TX_DESC_MACID(__tx_desc)
+
+/*TXDESC_WORD2*/
+
+#define SET_TX_DESC_HW_AES_IV_8822B(__tx_desc, __value)                        \
+	SET_TX_DESC_HW_AES_IV(__tx_desc, __value)
+#define GET_TX_DESC_HW_AES_IV_8822B(__tx_desc) GET_TX_DESC_HW_AES_IV(__tx_desc)
+#define SET_TX_DESC_FTM_EN_8822B(__tx_desc, __value)                           \
+	SET_TX_DESC_FTM_EN(__tx_desc, __value)
+#define GET_TX_DESC_FTM_EN_8822B(__tx_desc) GET_TX_DESC_FTM_EN(__tx_desc)
+#define SET_TX_DESC_G_ID_8822B(__tx_desc, __value)                             \
+	SET_TX_DESC_G_ID(__tx_desc, __value)
+#define GET_TX_DESC_G_ID_8822B(__tx_desc) GET_TX_DESC_G_ID(__tx_desc)
+#define SET_TX_DESC_BT_NULL_8822B(__tx_desc, __value)                          \
+	SET_TX_DESC_BT_NULL(__tx_desc, __value)
+#define GET_TX_DESC_BT_NULL_8822B(__tx_desc) GET_TX_DESC_BT_NULL(__tx_desc)
+#define SET_TX_DESC_AMPDU_DENSITY_8822B(__tx_desc, __value)                    \
+	SET_TX_DESC_AMPDU_DENSITY(__tx_desc, __value)
+#define GET_TX_DESC_AMPDU_DENSITY_8822B(__tx_desc)                             \
+	GET_TX_DESC_AMPDU_DENSITY(__tx_desc)
+#define SET_TX_DESC_SPE_RPT_8822B(__tx_desc, __value)                          \
+	SET_TX_DESC_SPE_RPT(__tx_desc, __value)
+#define GET_TX_DESC_SPE_RPT_8822B(__tx_desc) GET_TX_DESC_SPE_RPT(__tx_desc)
+#define SET_TX_DESC_RAW_8822B(__tx_desc, __value)                              \
+	SET_TX_DESC_RAW(__tx_desc, __value)
+#define GET_TX_DESC_RAW_8822B(__tx_desc) GET_TX_DESC_RAW(__tx_desc)
+#define SET_TX_DESC_MOREFRAG_8822B(__tx_desc, __value)                         \
+	SET_TX_DESC_MOREFRAG(__tx_desc, __value)
+#define GET_TX_DESC_MOREFRAG_8822B(__tx_desc) GET_TX_DESC_MOREFRAG(__tx_desc)
+#define SET_TX_DESC_BK_8822B(__tx_desc, __value)                               \
+	SET_TX_DESC_BK(__tx_desc, __value)
+#define GET_TX_DESC_BK_8822B(__tx_desc) GET_TX_DESC_BK(__tx_desc)
+#define SET_TX_DESC_NULL_1_8822B(__tx_desc, __value)                           \
+	SET_TX_DESC_NULL_1(__tx_desc, __value)
+#define GET_TX_DESC_NULL_1_8822B(__tx_desc) GET_TX_DESC_NULL_1(__tx_desc)
+#define SET_TX_DESC_NULL_0_8822B(__tx_desc, __value)                           \
+	SET_TX_DESC_NULL_0(__tx_desc, __value)
+#define GET_TX_DESC_NULL_0_8822B(__tx_desc) GET_TX_DESC_NULL_0(__tx_desc)
+#define SET_TX_DESC_RDG_EN_8822B(__tx_desc, __value)                           \
+	SET_TX_DESC_RDG_EN(__tx_desc, __value)
+#define GET_TX_DESC_RDG_EN_8822B(__tx_desc) GET_TX_DESC_RDG_EN(__tx_desc)
+#define SET_TX_DESC_AGG_EN_8822B(__tx_desc, __value)                           \
+	SET_TX_DESC_AGG_EN(__tx_desc, __value)
+#define GET_TX_DESC_AGG_EN_8822B(__tx_desc) GET_TX_DESC_AGG_EN(__tx_desc)
+#define SET_TX_DESC_CCA_RTS_8822B(__tx_desc, __value)                          \
+	SET_TX_DESC_CCA_RTS(__tx_desc, __value)
+#define GET_TX_DESC_CCA_RTS_8822B(__tx_desc) GET_TX_DESC_CCA_RTS(__tx_desc)
+#define SET_TX_DESC_TRI_FRAME_8822B(__tx_desc, __value)                        \
+	SET_TX_DESC_TRI_FRAME(__tx_desc, __value)
+#define GET_TX_DESC_TRI_FRAME_8822B(__tx_desc) GET_TX_DESC_TRI_FRAME(__tx_desc)
+#define SET_TX_DESC_P_AID_8822B(__tx_desc, __value)                            \
+	SET_TX_DESC_P_AID(__tx_desc, __value)
+#define GET_TX_DESC_P_AID_8822B(__tx_desc) GET_TX_DESC_P_AID(__tx_desc)
+
+/*TXDESC_WORD3*/
+
+#define SET_TX_DESC_AMPDU_MAX_TIME_8822B(__tx_desc, __value)                   \
+	SET_TX_DESC_AMPDU_MAX_TIME(__tx_desc, __value)
+#define GET_TX_DESC_AMPDU_MAX_TIME_8822B(__tx_desc)                            \
+	GET_TX_DESC_AMPDU_MAX_TIME(__tx_desc)
+#define SET_TX_DESC_NDPA_8822B(__tx_desc, __value)                             \
+	SET_TX_DESC_NDPA(__tx_desc, __value)
+#define GET_TX_DESC_NDPA_8822B(__tx_desc) GET_TX_DESC_NDPA(__tx_desc)
+#define SET_TX_DESC_MAX_AGG_NUM_8822B(__tx_desc, __value)                      \
+	SET_TX_DESC_MAX_AGG_NUM(__tx_desc, __value)
+#define GET_TX_DESC_MAX_AGG_NUM_8822B(__tx_desc)                               \
+	GET_TX_DESC_MAX_AGG_NUM(__tx_desc)
+#define SET_TX_DESC_USE_MAX_TIME_EN_8822B(__tx_desc, __value)                  \
+	SET_TX_DESC_USE_MAX_TIME_EN(__tx_desc, __value)
+#define GET_TX_DESC_USE_MAX_TIME_EN_8822B(__tx_desc)                           \
+	GET_TX_DESC_USE_MAX_TIME_EN(__tx_desc)
+#define SET_TX_DESC_NAVUSEHDR_8822B(__tx_desc, __value)                        \
+	SET_TX_DESC_NAVUSEHDR(__tx_desc, __value)
+#define GET_TX_DESC_NAVUSEHDR_8822B(__tx_desc) GET_TX_DESC_NAVUSEHDR(__tx_desc)
+#define SET_TX_DESC_CHK_EN_8822B(__tx_desc, __value)                           \
+	SET_TX_DESC_CHK_EN(__tx_desc, __value)
+#define GET_TX_DESC_CHK_EN_8822B(__tx_desc) GET_TX_DESC_CHK_EN(__tx_desc)
+#define SET_TX_DESC_HW_RTS_EN_8822B(__tx_desc, __value)                        \
+	SET_TX_DESC_HW_RTS_EN(__tx_desc, __value)
+#define GET_TX_DESC_HW_RTS_EN_8822B(__tx_desc) GET_TX_DESC_HW_RTS_EN(__tx_desc)
+#define SET_TX_DESC_RTSEN_8822B(__tx_desc, __value)                            \
+	SET_TX_DESC_RTSEN(__tx_desc, __value)
+#define GET_TX_DESC_RTSEN_8822B(__tx_desc) GET_TX_DESC_RTSEN(__tx_desc)
+#define SET_TX_DESC_CTS2SELF_8822B(__tx_desc, __value)                         \
+	SET_TX_DESC_CTS2SELF(__tx_desc, __value)
+#define GET_TX_DESC_CTS2SELF_8822B(__tx_desc) GET_TX_DESC_CTS2SELF(__tx_desc)
+#define SET_TX_DESC_DISDATAFB_8822B(__tx_desc, __value)                        \
+	SET_TX_DESC_DISDATAFB(__tx_desc, __value)
+#define GET_TX_DESC_DISDATAFB_8822B(__tx_desc) GET_TX_DESC_DISDATAFB(__tx_desc)
+#define SET_TX_DESC_DISRTSFB_8822B(__tx_desc, __value)                         \
+	SET_TX_DESC_DISRTSFB(__tx_desc, __value)
+#define GET_TX_DESC_DISRTSFB_8822B(__tx_desc) GET_TX_DESC_DISRTSFB(__tx_desc)
+#define SET_TX_DESC_USE_RATE_8822B(__tx_desc, __value)                         \
+	SET_TX_DESC_USE_RATE(__tx_desc, __value)
+#define GET_TX_DESC_USE_RATE_8822B(__tx_desc) GET_TX_DESC_USE_RATE(__tx_desc)
+#define SET_TX_DESC_HW_SSN_SEL_8822B(__tx_desc, __value)                       \
+	SET_TX_DESC_HW_SSN_SEL(__tx_desc, __value)
+#define GET_TX_DESC_HW_SSN_SEL_8822B(__tx_desc)                                \
+	GET_TX_DESC_HW_SSN_SEL(__tx_desc)
+#define SET_TX_DESC_WHEADER_LEN_8822B(__tx_desc, __value)                      \
+	SET_TX_DESC_WHEADER_LEN(__tx_desc, __value)
+#define GET_TX_DESC_WHEADER_LEN_8822B(__tx_desc)                               \
+	GET_TX_DESC_WHEADER_LEN(__tx_desc)
+
+/*TXDESC_WORD4*/
+
+#define SET_TX_DESC_PCTS_MASK_IDX_8822B(__tx_desc, __value)                    \
+	SET_TX_DESC_PCTS_MASK_IDX(__tx_desc, __value)
+#define GET_TX_DESC_PCTS_MASK_IDX_8822B(__tx_desc)                             \
+	GET_TX_DESC_PCTS_MASK_IDX(__tx_desc)
+#define SET_TX_DESC_PCTS_EN_8822B(__tx_desc, __value)                          \
+	SET_TX_DESC_PCTS_EN(__tx_desc, __value)
+#define GET_TX_DESC_PCTS_EN_8822B(__tx_desc) GET_TX_DESC_PCTS_EN(__tx_desc)
+#define SET_TX_DESC_RTSRATE_8822B(__tx_desc, __value)                          \
+	SET_TX_DESC_RTSRATE(__tx_desc, __value)
+#define GET_TX_DESC_RTSRATE_8822B(__tx_desc) GET_TX_DESC_RTSRATE(__tx_desc)
+#define SET_TX_DESC_RTS_DATA_RTY_LMT_8822B(__tx_desc, __value)                 \
+	SET_TX_DESC_RTS_DATA_RTY_LMT(__tx_desc, __value)
+#define GET_TX_DESC_RTS_DATA_RTY_LMT_8822B(__tx_desc)                          \
+	GET_TX_DESC_RTS_DATA_RTY_LMT(__tx_desc)
+#define SET_TX_DESC_RTY_LMT_EN_8822B(__tx_desc, __value)                       \
+	SET_TX_DESC_RTY_LMT_EN(__tx_desc, __value)
+#define GET_TX_DESC_RTY_LMT_EN_8822B(__tx_desc)                                \
+	GET_TX_DESC_RTY_LMT_EN(__tx_desc)
+#define SET_TX_DESC_RTS_RTY_LOWEST_RATE_8822B(__tx_desc, __value)              \
+	SET_TX_DESC_RTS_RTY_LOWEST_RATE(__tx_desc, __value)
+#define GET_TX_DESC_RTS_RTY_LOWEST_RATE_8822B(__tx_desc)                       \
+	GET_TX_DESC_RTS_RTY_LOWEST_RATE(__tx_desc)
+#define SET_TX_DESC_DATA_RTY_LOWEST_RATE_8822B(__tx_desc, __value)             \
+	SET_TX_DESC_DATA_RTY_LOWEST_RATE(__tx_desc, __value)
+#define GET_TX_DESC_DATA_RTY_LOWEST_RATE_8822B(__tx_desc)                      \
+	GET_TX_DESC_DATA_RTY_LOWEST_RATE(__tx_desc)
+#define SET_TX_DESC_TRY_RATE_8822B(__tx_desc, __value)                         \
+	SET_TX_DESC_TRY_RATE(__tx_desc, __value)
+#define GET_TX_DESC_TRY_RATE_8822B(__tx_desc) GET_TX_DESC_TRY_RATE(__tx_desc)
+#define SET_TX_DESC_DATARATE_8822B(__tx_desc, __value)                         \
+	SET_TX_DESC_DATARATE(__tx_desc, __value)
+#define GET_TX_DESC_DATARATE_8822B(__tx_desc) GET_TX_DESC_DATARATE(__tx_desc)
+
+/*TXDESC_WORD5*/
+
+#define SET_TX_DESC_POLLUTED_8822B(__tx_desc, __value)                         \
+	SET_TX_DESC_POLLUTED(__tx_desc, __value)
+#define GET_TX_DESC_POLLUTED_8822B(__tx_desc) GET_TX_DESC_POLLUTED(__tx_desc)
+#define SET_TX_DESC_TXPWR_OFSET_8822B(__tx_desc, __value)                      \
+	SET_TX_DESC_TXPWR_OFSET(__tx_desc, __value)
+#define GET_TX_DESC_TXPWR_OFSET_8822B(__tx_desc)                               \
+	GET_TX_DESC_TXPWR_OFSET(__tx_desc)
+#define SET_TX_DESC_TX_ANT_8822B(__tx_desc, __value)                           \
+	SET_TX_DESC_TX_ANT(__tx_desc, __value)
+#define GET_TX_DESC_TX_ANT_8822B(__tx_desc) GET_TX_DESC_TX_ANT(__tx_desc)
+#define SET_TX_DESC_PORT_ID_8822B(__tx_desc, __value)                          \
+	SET_TX_DESC_PORT_ID(__tx_desc, __value)
+#define GET_TX_DESC_PORT_ID_8822B(__tx_desc) GET_TX_DESC_PORT_ID(__tx_desc)
+#define SET_TX_DESC_MULTIPLE_PORT_8822B(__tx_desc, __value)                    \
+	SET_TX_DESC_MULTIPLE_PORT(__tx_desc, __value)
+#define GET_TX_DESC_MULTIPLE_PORT_8822B(__tx_desc)                             \
+	GET_TX_DESC_MULTIPLE_PORT(__tx_desc)
+#define SET_TX_DESC_SIGNALING_TAPKT_EN_8822B(__tx_desc, __value)               \
+	SET_TX_DESC_SIGNALING_TAPKT_EN(__tx_desc, __value)
+#define GET_TX_DESC_SIGNALING_TAPKT_EN_8822B(__tx_desc)                        \
+	GET_TX_DESC_SIGNALING_TAPKT_EN(__tx_desc)
+#define SET_TX_DESC_RTS_SC_8822B(__tx_desc, __value)                           \
+	SET_TX_DESC_RTS_SC(__tx_desc, __value)
+#define GET_TX_DESC_RTS_SC_8822B(__tx_desc) GET_TX_DESC_RTS_SC(__tx_desc)
+#define SET_TX_DESC_RTS_SHORT_8822B(__tx_desc, __value)                        \
+	SET_TX_DESC_RTS_SHORT(__tx_desc, __value)
+#define GET_TX_DESC_RTS_SHORT_8822B(__tx_desc) GET_TX_DESC_RTS_SHORT(__tx_desc)
+#define SET_TX_DESC_VCS_STBC_8822B(__tx_desc, __value)                         \
+	SET_TX_DESC_VCS_STBC(__tx_desc, __value)
+#define GET_TX_DESC_VCS_STBC_8822B(__tx_desc) GET_TX_DESC_VCS_STBC(__tx_desc)
+#define SET_TX_DESC_DATA_STBC_8822B(__tx_desc, __value)                        \
+	SET_TX_DESC_DATA_STBC(__tx_desc, __value)
+#define GET_TX_DESC_DATA_STBC_8822B(__tx_desc) GET_TX_DESC_DATA_STBC(__tx_desc)
+#define SET_TX_DESC_DATA_LDPC_8822B(__tx_desc, __value)                        \
+	SET_TX_DESC_DATA_LDPC(__tx_desc, __value)
+#define GET_TX_DESC_DATA_LDPC_8822B(__tx_desc) GET_TX_DESC_DATA_LDPC(__tx_desc)
+#define SET_TX_DESC_DATA_BW_8822B(__tx_desc, __value)                          \
+	SET_TX_DESC_DATA_BW(__tx_desc, __value)
+#define GET_TX_DESC_DATA_BW_8822B(__tx_desc) GET_TX_DESC_DATA_BW(__tx_desc)
+#define SET_TX_DESC_DATA_SHORT_8822B(__tx_desc, __value)                       \
+	SET_TX_DESC_DATA_SHORT(__tx_desc, __value)
+#define GET_TX_DESC_DATA_SHORT_8822B(__tx_desc)                                \
+	GET_TX_DESC_DATA_SHORT(__tx_desc)
+#define SET_TX_DESC_DATA_SC_8822B(__tx_desc, __value)                          \
+	SET_TX_DESC_DATA_SC(__tx_desc, __value)
+#define GET_TX_DESC_DATA_SC_8822B(__tx_desc) GET_TX_DESC_DATA_SC(__tx_desc)
+
+/*TXDESC_WORD6*/
+
+#define SET_TX_DESC_ANTSEL_D_8822B(__tx_desc, __value)                         \
+	SET_TX_DESC_ANTSEL_D(__tx_desc, __value)
+#define GET_TX_DESC_ANTSEL_D_8822B(__tx_desc) GET_TX_DESC_ANTSEL_D(__tx_desc)
+#define SET_TX_DESC_ANT_MAPD_8822B(__tx_desc, __value)                         \
+	SET_TX_DESC_ANT_MAPD(__tx_desc, __value)
+#define GET_TX_DESC_ANT_MAPD_8822B(__tx_desc) GET_TX_DESC_ANT_MAPD(__tx_desc)
+#define SET_TX_DESC_ANT_MAPC_8822B(__tx_desc, __value)                         \
+	SET_TX_DESC_ANT_MAPC(__tx_desc, __value)
+#define GET_TX_DESC_ANT_MAPC_8822B(__tx_desc) GET_TX_DESC_ANT_MAPC(__tx_desc)
+#define SET_TX_DESC_ANT_MAPB_8822B(__tx_desc, __value)                         \
+	SET_TX_DESC_ANT_MAPB(__tx_desc, __value)
+#define GET_TX_DESC_ANT_MAPB_8822B(__tx_desc) GET_TX_DESC_ANT_MAPB(__tx_desc)
+#define SET_TX_DESC_ANT_MAPA_8822B(__tx_desc, __value)                         \
+	SET_TX_DESC_ANT_MAPA(__tx_desc, __value)
+#define GET_TX_DESC_ANT_MAPA_8822B(__tx_desc) GET_TX_DESC_ANT_MAPA(__tx_desc)
+#define SET_TX_DESC_ANTSEL_C_8822B(__tx_desc, __value)                         \
+	SET_TX_DESC_ANTSEL_C(__tx_desc, __value)
+#define GET_TX_DESC_ANTSEL_C_8822B(__tx_desc) GET_TX_DESC_ANTSEL_C(__tx_desc)
+#define SET_TX_DESC_ANTSEL_B_8822B(__tx_desc, __value)                         \
+	SET_TX_DESC_ANTSEL_B(__tx_desc, __value)
+#define GET_TX_DESC_ANTSEL_B_8822B(__tx_desc) GET_TX_DESC_ANTSEL_B(__tx_desc)
+#define SET_TX_DESC_ANTSEL_A_8822B(__tx_desc, __value)                         \
+	SET_TX_DESC_ANTSEL_A(__tx_desc, __value)
+#define GET_TX_DESC_ANTSEL_A_8822B(__tx_desc) GET_TX_DESC_ANTSEL_A(__tx_desc)
+#define SET_TX_DESC_MBSSID_8822B(__tx_desc, __value)                           \
+	SET_TX_DESC_MBSSID(__tx_desc, __value)
+#define GET_TX_DESC_MBSSID_8822B(__tx_desc) GET_TX_DESC_MBSSID(__tx_desc)
+#define SET_TX_DESC_SW_DEFINE_8822B(__tx_desc, __value)                        \
+	SET_TX_DESC_SW_DEFINE(__tx_desc, __value)
+#define GET_TX_DESC_SW_DEFINE_8822B(__tx_desc) GET_TX_DESC_SW_DEFINE(__tx_desc)
+
+/*TXDESC_WORD7*/
+
+#define SET_TX_DESC_DMA_TXAGG_NUM_8822B(__tx_desc, __value)                    \
+	SET_TX_DESC_DMA_TXAGG_NUM(__tx_desc, __value)
+#define GET_TX_DESC_DMA_TXAGG_NUM_8822B(__tx_desc)                             \
+	GET_TX_DESC_DMA_TXAGG_NUM(__tx_desc)
+#define SET_TX_DESC_FINAL_DATA_RATE_8822B(__tx_desc, __value)                  \
+	SET_TX_DESC_FINAL_DATA_RATE(__tx_desc, __value)
+#define GET_TX_DESC_FINAL_DATA_RATE_8822B(__tx_desc)                           \
+	GET_TX_DESC_FINAL_DATA_RATE(__tx_desc)
+#define SET_TX_DESC_NTX_MAP_8822B(__tx_desc, __value)                          \
+	SET_TX_DESC_NTX_MAP(__tx_desc, __value)
+#define GET_TX_DESC_NTX_MAP_8822B(__tx_desc) GET_TX_DESC_NTX_MAP(__tx_desc)
+#define SET_TX_DESC_TX_BUFF_SIZE_8822B(__tx_desc, __value)                     \
+	SET_TX_DESC_TX_BUFF_SIZE(__tx_desc, __value)
+#define GET_TX_DESC_TX_BUFF_SIZE_8822B(__tx_desc)                              \
+	GET_TX_DESC_TX_BUFF_SIZE(__tx_desc)
+#define SET_TX_DESC_TXDESC_CHECKSUM_8822B(__tx_desc, __value)                  \
+	SET_TX_DESC_TXDESC_CHECKSUM(__tx_desc, __value)
+#define GET_TX_DESC_TXDESC_CHECKSUM_8822B(__tx_desc)                           \
+	GET_TX_DESC_TXDESC_CHECKSUM(__tx_desc)
+#define SET_TX_DESC_TIMESTAMP_8822B(__tx_desc, __value)                        \
+	SET_TX_DESC_TIMESTAMP(__tx_desc, __value)
+#define GET_TX_DESC_TIMESTAMP_8822B(__tx_desc) GET_TX_DESC_TIMESTAMP(__tx_desc)
+
+/*TXDESC_WORD8*/
+
+#define SET_TX_DESC_TXWIFI_CP_8822B(__tx_desc, __value)                        \
+	SET_TX_DESC_TXWIFI_CP(__tx_desc, __value)
+#define GET_TX_DESC_TXWIFI_CP_8822B(__tx_desc) GET_TX_DESC_TXWIFI_CP(__tx_desc)
+#define SET_TX_DESC_MAC_CP_8822B(__tx_desc, __value)                           \
+	SET_TX_DESC_MAC_CP(__tx_desc, __value)
+#define GET_TX_DESC_MAC_CP_8822B(__tx_desc) GET_TX_DESC_MAC_CP(__tx_desc)
+#define SET_TX_DESC_STW_PKTRE_DIS_8822B(__tx_desc, __value)                    \
+	SET_TX_DESC_STW_PKTRE_DIS(__tx_desc, __value)
+#define GET_TX_DESC_STW_PKTRE_DIS_8822B(__tx_desc)                             \
+	GET_TX_DESC_STW_PKTRE_DIS(__tx_desc)
+#define SET_TX_DESC_STW_RB_DIS_8822B(__tx_desc, __value)                       \
+	SET_TX_DESC_STW_RB_DIS(__tx_desc, __value)
+#define GET_TX_DESC_STW_RB_DIS_8822B(__tx_desc)                                \
+	GET_TX_DESC_STW_RB_DIS(__tx_desc)
+#define SET_TX_DESC_STW_RATE_DIS_8822B(__tx_desc, __value)                     \
+	SET_TX_DESC_STW_RATE_DIS(__tx_desc, __value)
+#define GET_TX_DESC_STW_RATE_DIS_8822B(__tx_desc)                              \
+	GET_TX_DESC_STW_RATE_DIS(__tx_desc)
+#define SET_TX_DESC_STW_ANT_DIS_8822B(__tx_desc, __value)                      \
+	SET_TX_DESC_STW_ANT_DIS(__tx_desc, __value)
+#define GET_TX_DESC_STW_ANT_DIS_8822B(__tx_desc)                               \
+	GET_TX_DESC_STW_ANT_DIS(__tx_desc)
+#define SET_TX_DESC_STW_EN_8822B(__tx_desc, __value)                           \
+	SET_TX_DESC_STW_EN(__tx_desc, __value)
+#define GET_TX_DESC_STW_EN_8822B(__tx_desc) GET_TX_DESC_STW_EN(__tx_desc)
+#define SET_TX_DESC_SMH_EN_8822B(__tx_desc, __value)                           \
+	SET_TX_DESC_SMH_EN(__tx_desc, __value)
+#define GET_TX_DESC_SMH_EN_8822B(__tx_desc) GET_TX_DESC_SMH_EN(__tx_desc)
+#define SET_TX_DESC_TAILPAGE_L_8822B(__tx_desc, __value)                       \
+	SET_TX_DESC_TAILPAGE_L(__tx_desc, __value)
+#define GET_TX_DESC_TAILPAGE_L_8822B(__tx_desc)                                \
+	GET_TX_DESC_TAILPAGE_L(__tx_desc)
+#define SET_TX_DESC_SDIO_DMASEQ_8822B(__tx_desc, __value)                      \
+	SET_TX_DESC_SDIO_DMASEQ(__tx_desc, __value)
+#define GET_TX_DESC_SDIO_DMASEQ_8822B(__tx_desc)                               \
+	GET_TX_DESC_SDIO_DMASEQ(__tx_desc)
+#define SET_TX_DESC_NEXTHEADPAGE_L_8822B(__tx_desc, __value)                   \
+	SET_TX_DESC_NEXTHEADPAGE_L(__tx_desc, __value)
+#define GET_TX_DESC_NEXTHEADPAGE_L_8822B(__tx_desc)                            \
+	GET_TX_DESC_NEXTHEADPAGE_L(__tx_desc)
+#define SET_TX_DESC_EN_HWSEQ_8822B(__tx_desc, __value)                         \
+	SET_TX_DESC_EN_HWSEQ(__tx_desc, __value)
+#define GET_TX_DESC_EN_HWSEQ_8822B(__tx_desc) GET_TX_DESC_EN_HWSEQ(__tx_desc)
+#define SET_TX_DESC_EN_HWEXSEQ_8822B(__tx_desc, __value)                       \
+	SET_TX_DESC_EN_HWEXSEQ(__tx_desc, __value)
+#define GET_TX_DESC_EN_HWEXSEQ_8822B(__tx_desc)                                \
+	GET_TX_DESC_EN_HWEXSEQ(__tx_desc)
+#define SET_TX_DESC_DATA_RC_8822B(__tx_desc, __value)                          \
+	SET_TX_DESC_DATA_RC(__tx_desc, __value)
+#define GET_TX_DESC_DATA_RC_8822B(__tx_desc) GET_TX_DESC_DATA_RC(__tx_desc)
+#define SET_TX_DESC_BAR_RTY_TH_8822B(__tx_desc, __value)                       \
+	SET_TX_DESC_BAR_RTY_TH(__tx_desc, __value)
+#define GET_TX_DESC_BAR_RTY_TH_8822B(__tx_desc)                                \
+	GET_TX_DESC_BAR_RTY_TH(__tx_desc)
+#define SET_TX_DESC_RTS_RC_8822B(__tx_desc, __value)                           \
+	SET_TX_DESC_RTS_RC(__tx_desc, __value)
+#define GET_TX_DESC_RTS_RC_8822B(__tx_desc) GET_TX_DESC_RTS_RC(__tx_desc)
+
+/*TXDESC_WORD9*/
+
+#define SET_TX_DESC_TAILPAGE_H_8822B(__tx_desc, __value)                       \
+	SET_TX_DESC_TAILPAGE_H(__tx_desc, __value)
+#define GET_TX_DESC_TAILPAGE_H_8822B(__tx_desc)                                \
+	GET_TX_DESC_TAILPAGE_H(__tx_desc)
+#define SET_TX_DESC_NEXTHEADPAGE_H_8822B(__tx_desc, __value)                   \
+	SET_TX_DESC_NEXTHEADPAGE_H(__tx_desc, __value)
+#define GET_TX_DESC_NEXTHEADPAGE_H_8822B(__tx_desc)                            \
+	GET_TX_DESC_NEXTHEADPAGE_H(__tx_desc)
+#define SET_TX_DESC_SW_SEQ_8822B(__tx_desc, __value)                           \
+	SET_TX_DESC_SW_SEQ(__tx_desc, __value)
+#define GET_TX_DESC_SW_SEQ_8822B(__tx_desc) GET_TX_DESC_SW_SEQ(__tx_desc)
+#define SET_TX_DESC_TXBF_PATH_8822B(__tx_desc, __value)                        \
+	SET_TX_DESC_TXBF_PATH(__tx_desc, __value)
+#define GET_TX_DESC_TXBF_PATH_8822B(__tx_desc) GET_TX_DESC_TXBF_PATH(__tx_desc)
+#define SET_TX_DESC_PADDING_LEN_8822B(__tx_desc, __value)                      \
+	SET_TX_DESC_PADDING_LEN(__tx_desc, __value)
+#define GET_TX_DESC_PADDING_LEN_8822B(__tx_desc)                               \
+	GET_TX_DESC_PADDING_LEN(__tx_desc)
+#define SET_TX_DESC_GROUP_BIT_IE_OFFSET_8822B(__tx_desc, __value)              \
+	SET_TX_DESC_GROUP_BIT_IE_OFFSET(__tx_desc, __value)
+#define GET_TX_DESC_GROUP_BIT_IE_OFFSET_8822B(__tx_desc)                       \
+	GET_TX_DESC_GROUP_BIT_IE_OFFSET(__tx_desc)
+
+/*WORD10*/
+
+#define SET_TX_DESC_MU_DATARATE_8822B(__tx_desc, __value)                      \
+	SET_TX_DESC_MU_DATARATE(__tx_desc, __value)
+#define GET_TX_DESC_MU_DATARATE_8822B(__tx_desc)                               \
+	GET_TX_DESC_MU_DATARATE(__tx_desc)
+#define SET_TX_DESC_MU_RC_8822B(__tx_desc, __value)                            \
+	SET_TX_DESC_MU_RC(__tx_desc, __value)
+#define GET_TX_DESC_MU_RC_8822B(__tx_desc) GET_TX_DESC_MU_RC(__tx_desc)
+#define SET_TX_DESC_SND_PKT_SEL_8822B(__tx_desc, __value)                      \
+	SET_TX_DESC_SND_PKT_SEL(__tx_desc, __value)
+#define GET_TX_DESC_SND_PKT_SEL_8822B(__tx_desc)                               \
+	GET_TX_DESC_SND_PKT_SEL(__tx_desc)
+
+#endif
diff --git a/drivers/staging/rtlwifi/halmac/halmac_tx_desc_nic.h b/drivers/staging/rtlwifi/halmac/halmac_tx_desc_nic.h
new file mode 100644
index 000000000000..02177c5faddf
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_tx_desc_nic.h
@@ -0,0 +1,506 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HALMAC_TX_DESC_NIC_H_
+#define _HALMAC_TX_DESC_NIC_H_
+
+/*TXDESC_WORD0*/
+
+#define SET_TX_DESC_DISQSELSEQ(__tx_desc, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x00, 31, 1, __value)
+#define GET_TX_DESC_DISQSELSEQ(__tx_desc)                                      \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x00, 31, 1)
+
+#define SET_TX_DESC_GF(__tx_desc, __value)                                     \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x00, 30, 1, __value)
+#define GET_TX_DESC_GF(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x00, 30, 1)
+#define SET_TX_DESC_NO_ACM(__tx_desc, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x00, 29, 1, __value)
+#define GET_TX_DESC_NO_ACM(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x00, 29, 1)
+
+#define SET_TX_DESC_BCNPKT_TSF_CTRL(__tx_desc, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x00, 28, 1, __value)
+#define GET_TX_DESC_BCNPKT_TSF_CTRL(__tx_desc)                                 \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x00, 28, 1)
+
+#define SET_TX_DESC_AMSDU_PAD_EN(__tx_desc, __value)                           \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x00, 27, 1, __value)
+#define GET_TX_DESC_AMSDU_PAD_EN(__tx_desc)                                    \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x00, 27, 1)
+
+#define SET_TX_DESC_LS(__tx_desc, __value)                                     \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x00, 26, 1, __value)
+#define GET_TX_DESC_LS(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x00, 26, 1)
+#define SET_TX_DESC_HTC(__tx_desc, __value)                                    \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x00, 25, 1, __value)
+#define GET_TX_DESC_HTC(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x00, 25, 1)
+#define SET_TX_DESC_BMC(__tx_desc, __value)                                    \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x00, 24, 1, __value)
+#define GET_TX_DESC_BMC(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x00, 24, 1)
+#define SET_TX_DESC_OFFSET(__tx_desc, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x00, 16, 8, __value)
+#define GET_TX_DESC_OFFSET(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x00, 16, 8)
+#define SET_TX_DESC_TXPKTSIZE(__tx_desc, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x00, 0, 16, __value)
+#define GET_TX_DESC_TXPKTSIZE(__tx_desc)                                       \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x00, 0, 16)
+
+/*TXDESC_WORD1*/
+
+#define SET_TX_DESC_MOREDATA(__tx_desc, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x04, 29, 1, __value)
+#define GET_TX_DESC_MOREDATA(__tx_desc)                                        \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x04, 29, 1)
+#define SET_TX_DESC_PKT_OFFSET(__tx_desc, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x04, 24, 5, __value)
+#define GET_TX_DESC_PKT_OFFSET(__tx_desc)                                      \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x04, 24, 5)
+#define SET_TX_DESC_SEC_TYPE(__tx_desc, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x04, 22, 2, __value)
+#define GET_TX_DESC_SEC_TYPE(__tx_desc)                                        \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x04, 22, 2)
+#define SET_TX_DESC_EN_DESC_ID(__tx_desc, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x04, 21, 1, __value)
+#define GET_TX_DESC_EN_DESC_ID(__tx_desc)                                      \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x04, 21, 1)
+#define SET_TX_DESC_RATE_ID(__tx_desc, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x04, 16, 5, __value)
+#define GET_TX_DESC_RATE_ID(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x04, 16, 5)
+#define SET_TX_DESC_PIFS(__tx_desc, __value)                                   \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x04, 15, 1, __value)
+#define GET_TX_DESC_PIFS(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x04, 15, 1)
+#define SET_TX_DESC_LSIG_TXOP_EN(__tx_desc, __value)                           \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x04, 14, 1, __value)
+#define GET_TX_DESC_LSIG_TXOP_EN(__tx_desc)                                    \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x04, 14, 1)
+#define SET_TX_DESC_RD_NAV_EXT(__tx_desc, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x04, 13, 1, __value)
+#define GET_TX_DESC_RD_NAV_EXT(__tx_desc)                                      \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x04, 13, 1)
+#define SET_TX_DESC_QSEL(__tx_desc, __value)                                   \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x04, 8, 5, __value)
+#define GET_TX_DESC_QSEL(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x04, 8, 5)
+#define SET_TX_DESC_MACID(__tx_desc, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x04, 0, 7, __value)
+#define GET_TX_DESC_MACID(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x04, 0, 7)
+
+/*TXDESC_WORD2*/
+
+#define SET_TX_DESC_HW_AES_IV(__tx_desc, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x08, 31, 1, __value)
+#define GET_TX_DESC_HW_AES_IV(__tx_desc)                                       \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x08, 31, 1)
+
+#define SET_TX_DESC_FTM_EN(__tx_desc, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x08, 30, 1, __value)
+#define GET_TX_DESC_FTM_EN(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x08, 30, 1)
+
+#define SET_TX_DESC_G_ID(__tx_desc, __value)                                   \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x08, 24, 6, __value)
+#define GET_TX_DESC_G_ID(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x08, 24, 6)
+#define SET_TX_DESC_BT_NULL(__tx_desc, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x08, 23, 1, __value)
+#define GET_TX_DESC_BT_NULL(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x08, 23, 1)
+#define SET_TX_DESC_AMPDU_DENSITY(__tx_desc, __value)                          \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x08, 20, 3, __value)
+#define GET_TX_DESC_AMPDU_DENSITY(__tx_desc)                                   \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x08, 20, 3)
+#ifdef SET_TX_DESC_SPE_RPT
+#undef SET_TX_DESC_SPE_RPT
+#endif
+#define SET_TX_DESC_SPE_RPT(__tx_desc, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x08, 19, 1, __value)
+#define GET_TX_DESC_SPE_RPT(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x08, 19, 1)
+#define SET_TX_DESC_RAW(__tx_desc, __value)                                    \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x08, 18, 1, __value)
+#define GET_TX_DESC_RAW(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x08, 18, 1)
+#define SET_TX_DESC_MOREFRAG(__tx_desc, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x08, 17, 1, __value)
+#define GET_TX_DESC_MOREFRAG(__tx_desc)                                        \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x08, 17, 1)
+#define SET_TX_DESC_BK(__tx_desc, __value)                                     \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x08, 16, 1, __value)
+#define GET_TX_DESC_BK(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x08, 16, 1)
+#define SET_TX_DESC_NULL_1(__tx_desc, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x08, 15, 1, __value)
+#define GET_TX_DESC_NULL_1(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x08, 15, 1)
+#define SET_TX_DESC_NULL_0(__tx_desc, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x08, 14, 1, __value)
+#define GET_TX_DESC_NULL_0(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x08, 14, 1)
+#define SET_TX_DESC_RDG_EN(__tx_desc, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x08, 13, 1, __value)
+#define GET_TX_DESC_RDG_EN(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x08, 13, 1)
+#define SET_TX_DESC_AGG_EN(__tx_desc, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x08, 12, 1, __value)
+#define GET_TX_DESC_AGG_EN(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x08, 12, 1)
+#define SET_TX_DESC_CCA_RTS(__tx_desc, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x08, 10, 2, __value)
+#define GET_TX_DESC_CCA_RTS(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x08, 10, 2)
+
+#define SET_TX_DESC_TRI_FRAME(__tx_desc, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x08, 9, 1, __value)
+#define GET_TX_DESC_TRI_FRAME(__tx_desc)                                       \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x08, 9, 1)
+
+#define SET_TX_DESC_P_AID(__tx_desc, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x08, 0, 9, __value)
+#define GET_TX_DESC_P_AID(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x08, 0, 9)
+
+/*TXDESC_WORD3*/
+
+#define SET_TX_DESC_AMPDU_MAX_TIME(__tx_desc, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x0C, 24, 8, __value)
+#define GET_TX_DESC_AMPDU_MAX_TIME(__tx_desc)                                  \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x0C, 24, 8)
+#define SET_TX_DESC_NDPA(__tx_desc, __value)                                   \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x0C, 22, 2, __value)
+#define GET_TX_DESC_NDPA(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x0C, 22, 2)
+#define SET_TX_DESC_MAX_AGG_NUM(__tx_desc, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x0C, 17, 5, __value)
+#define GET_TX_DESC_MAX_AGG_NUM(__tx_desc)                                     \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x0C, 17, 5)
+#define SET_TX_DESC_USE_MAX_TIME_EN(__tx_desc, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x0C, 16, 1, __value)
+#define GET_TX_DESC_USE_MAX_TIME_EN(__tx_desc)                                 \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x0C, 16, 1)
+#define SET_TX_DESC_NAVUSEHDR(__tx_desc, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x0C, 15, 1, __value)
+#define GET_TX_DESC_NAVUSEHDR(__tx_desc)                                       \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x0C, 15, 1)
+
+#define SET_TX_DESC_CHK_EN(__tx_desc, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x0C, 14, 1, __value)
+#define GET_TX_DESC_CHK_EN(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x0C, 14, 1)
+
+#define SET_TX_DESC_HW_RTS_EN(__tx_desc, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x0C, 13, 1, __value)
+#define GET_TX_DESC_HW_RTS_EN(__tx_desc)                                       \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x0C, 13, 1)
+#define SET_TX_DESC_RTSEN(__tx_desc, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x0C, 12, 1, __value)
+#define GET_TX_DESC_RTSEN(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x0C, 12, 1)
+#define SET_TX_DESC_CTS2SELF(__tx_desc, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x0C, 11, 1, __value)
+#define GET_TX_DESC_CTS2SELF(__tx_desc)                                        \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x0C, 11, 1)
+#define SET_TX_DESC_DISDATAFB(__tx_desc, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x0C, 10, 1, __value)
+#define GET_TX_DESC_DISDATAFB(__tx_desc)                                       \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x0C, 10, 1)
+#define SET_TX_DESC_DISRTSFB(__tx_desc, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x0C, 9, 1, __value)
+#define GET_TX_DESC_DISRTSFB(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x0C, 9, 1)
+#define SET_TX_DESC_USE_RATE(__tx_desc, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x0C, 8, 1, __value)
+#define GET_TX_DESC_USE_RATE(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x0C, 8, 1)
+#define SET_TX_DESC_HW_SSN_SEL(__tx_desc, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x0C, 6, 2, __value)
+#define GET_TX_DESC_HW_SSN_SEL(__tx_desc)                                      \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x0C, 6, 2)
+
+#define SET_TX_DESC_WHEADER_LEN(__tx_desc, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x0C, 0, 5, __value)
+#define GET_TX_DESC_WHEADER_LEN(__tx_desc)                                     \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x0C, 0, 5)
+
+/*TXDESC_WORD4*/
+
+#define SET_TX_DESC_PCTS_MASK_IDX(__tx_desc, __value)                          \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x10, 30, 2, __value)
+#define GET_TX_DESC_PCTS_MASK_IDX(__tx_desc)                                   \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x10, 30, 2)
+#define SET_TX_DESC_PCTS_EN(__tx_desc, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x10, 29, 1, __value)
+#define GET_TX_DESC_PCTS_EN(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x10, 29, 1)
+#define SET_TX_DESC_RTSRATE(__tx_desc, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x10, 24, 5, __value)
+#define GET_TX_DESC_RTSRATE(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x10, 24, 5)
+#define SET_TX_DESC_RTS_DATA_RTY_LMT(__tx_desc, __value)                       \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x10, 18, 6, __value)
+#define GET_TX_DESC_RTS_DATA_RTY_LMT(__tx_desc)                                \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x10, 18, 6)
+#define SET_TX_DESC_RTY_LMT_EN(__tx_desc, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x10, 17, 1, __value)
+#define GET_TX_DESC_RTY_LMT_EN(__tx_desc)                                      \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x10, 17, 1)
+#define SET_TX_DESC_RTS_RTY_LOWEST_RATE(__tx_desc, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x10, 13, 4, __value)
+#define GET_TX_DESC_RTS_RTY_LOWEST_RATE(__tx_desc)                             \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x10, 13, 4)
+#define SET_TX_DESC_DATA_RTY_LOWEST_RATE(__tx_desc, __value)                   \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x10, 8, 5, __value)
+#define GET_TX_DESC_DATA_RTY_LOWEST_RATE(__tx_desc)                            \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x10, 8, 5)
+#define SET_TX_DESC_TRY_RATE(__tx_desc, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x10, 7, 1, __value)
+#define GET_TX_DESC_TRY_RATE(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x10, 7, 1)
+#define SET_TX_DESC_DATARATE(__tx_desc, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x10, 0, 7, __value)
+#define GET_TX_DESC_DATARATE(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x10, 0, 7)
+
+/*TXDESC_WORD5*/
+
+#define SET_TX_DESC_POLLUTED(__tx_desc, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x14, 31, 1, __value)
+#define GET_TX_DESC_POLLUTED(__tx_desc)                                        \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x14, 31, 1)
+
+#define SET_TX_DESC_TXPWR_OFSET(__tx_desc, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x14, 28, 3, __value)
+#define GET_TX_DESC_TXPWR_OFSET(__tx_desc)                                     \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x14, 28, 3)
+#define SET_TX_DESC_TX_ANT(__tx_desc, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x14, 24, 4, __value)
+#define GET_TX_DESC_TX_ANT(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x14, 24, 4)
+#define SET_TX_DESC_PORT_ID(__tx_desc, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x14, 21, 3, __value)
+#define GET_TX_DESC_PORT_ID(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x14, 21, 3)
+
+#define SET_TX_DESC_MULTIPLE_PORT(__tx_desc, __value)                          \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x14, 18, 3, __value)
+#define GET_TX_DESC_MULTIPLE_PORT(__tx_desc)                                   \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x14, 18, 3)
+
+#define SET_TX_DESC_SIGNALING_TAPKT_EN(__tx_desc, __value)                     \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x14, 17, 1, __value)
+#define GET_TX_DESC_SIGNALING_TAPKT_EN(__tx_desc)                              \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x14, 17, 1)
+
+#define SET_TX_DESC_RTS_SC(__tx_desc, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x14, 13, 4, __value)
+#define GET_TX_DESC_RTS_SC(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x14, 13, 4)
+#define SET_TX_DESC_RTS_SHORT(__tx_desc, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x14, 12, 1, __value)
+#define GET_TX_DESC_RTS_SHORT(__tx_desc)                                       \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x14, 12, 1)
+
+#define SET_TX_DESC_VCS_STBC(__tx_desc, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x14, 10, 2, __value)
+#define GET_TX_DESC_VCS_STBC(__tx_desc)                                        \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x14, 10, 2)
+
+#define SET_TX_DESC_DATA_STBC(__tx_desc, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x14, 8, 2, __value)
+#define GET_TX_DESC_DATA_STBC(__tx_desc)                                       \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x14, 8, 2)
+
+#define SET_TX_DESC_DATA_LDPC(__tx_desc, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x14, 7, 1, __value)
+#define GET_TX_DESC_DATA_LDPC(__tx_desc)                                       \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x14, 7, 1)
+
+#define SET_TX_DESC_DATA_BW(__tx_desc, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x14, 5, 2, __value)
+#define GET_TX_DESC_DATA_BW(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x14, 5, 2)
+#define SET_TX_DESC_DATA_SHORT(__tx_desc, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x14, 4, 1, __value)
+#define GET_TX_DESC_DATA_SHORT(__tx_desc)                                      \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x14, 4, 1)
+#define SET_TX_DESC_DATA_SC(__tx_desc, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x14, 0, 4, __value)
+#define GET_TX_DESC_DATA_SC(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x14, 0, 4)
+
+/*TXDESC_WORD6*/
+
+#define SET_TX_DESC_ANTSEL_D(__tx_desc, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x18, 30, 2, __value)
+#define GET_TX_DESC_ANTSEL_D(__tx_desc)                                        \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x18, 30, 2)
+#define SET_TX_DESC_ANT_MAPD(__tx_desc, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x18, 28, 2, __value)
+#define GET_TX_DESC_ANT_MAPD(__tx_desc)                                        \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x18, 28, 2)
+#define SET_TX_DESC_ANT_MAPC(__tx_desc, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x18, 26, 2, __value)
+#define GET_TX_DESC_ANT_MAPC(__tx_desc)                                        \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x18, 26, 2)
+#define SET_TX_DESC_ANT_MAPB(__tx_desc, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x18, 24, 2, __value)
+#define GET_TX_DESC_ANT_MAPB(__tx_desc)                                        \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x18, 24, 2)
+#define SET_TX_DESC_ANT_MAPA(__tx_desc, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x18, 22, 2, __value)
+#define GET_TX_DESC_ANT_MAPA(__tx_desc)                                        \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x18, 22, 2)
+#define SET_TX_DESC_ANTSEL_C(__tx_desc, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x18, 20, 2, __value)
+#define GET_TX_DESC_ANTSEL_C(__tx_desc)                                        \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x18, 20, 2)
+#define SET_TX_DESC_ANTSEL_B(__tx_desc, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x18, 18, 2, __value)
+#define GET_TX_DESC_ANTSEL_B(__tx_desc)                                        \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x18, 18, 2)
+
+#define SET_TX_DESC_ANTSEL_A(__tx_desc, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x18, 16, 2, __value)
+#define GET_TX_DESC_ANTSEL_A(__tx_desc)                                        \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x18, 16, 2)
+#define SET_TX_DESC_MBSSID(__tx_desc, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x18, 12, 4, __value)
+#define GET_TX_DESC_MBSSID(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x18, 12, 4)
+#ifdef SET_TX_DESC_SW_DEFINE
+#undef SET_TX_DESC_SW_DEFINE
+#endif
+#define SET_TX_DESC_SW_DEFINE(__tx_desc, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x18, 0, 12, __value)
+#define GET_TX_DESC_SW_DEFINE(__tx_desc)                                       \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x18, 0, 12)
+
+/*TXDESC_WORD7*/
+
+#define SET_TX_DESC_DMA_TXAGG_NUM(__tx_desc, __value)                          \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x1C, 24, 8, __value)
+#define GET_TX_DESC_DMA_TXAGG_NUM(__tx_desc)                                   \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x1C, 24, 8)
+
+#define SET_TX_DESC_FINAL_DATA_RATE(__tx_desc, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x1C, 24, 8, __value)
+#define GET_TX_DESC_FINAL_DATA_RATE(__tx_desc)                                 \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x1C, 24, 8)
+#define SET_TX_DESC_NTX_MAP(__tx_desc, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x1C, 20, 4, __value)
+#define GET_TX_DESC_NTX_MAP(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x1C, 20, 4)
+
+#define SET_TX_DESC_TX_BUFF_SIZE(__tx_desc, __value)                           \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x1C, 0, 16, __value)
+#define GET_TX_DESC_TX_BUFF_SIZE(__tx_desc)                                    \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x1C, 0, 16)
+#define SET_TX_DESC_TXDESC_CHECKSUM(__tx_desc, __value)                        \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x1C, 0, 16, __value)
+#define GET_TX_DESC_TXDESC_CHECKSUM(__tx_desc)                                 \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x1C, 0, 16)
+#define SET_TX_DESC_TIMESTAMP(__tx_desc, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x1C, 0, 16, __value)
+#define GET_TX_DESC_TIMESTAMP(__tx_desc)                                       \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x1C, 0, 16)
+
+/*TXDESC_WORD8*/
+
+#define SET_TX_DESC_TXWIFI_CP(__tx_desc, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x20, 31, 1, __value)
+#define GET_TX_DESC_TXWIFI_CP(__tx_desc)                                       \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x20, 31, 1)
+#define SET_TX_DESC_MAC_CP(__tx_desc, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x20, 30, 1, __value)
+#define GET_TX_DESC_MAC_CP(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x20, 30, 1)
+#define SET_TX_DESC_STW_PKTRE_DIS(__tx_desc, __value)                          \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x20, 29, 1, __value)
+#define GET_TX_DESC_STW_PKTRE_DIS(__tx_desc)                                   \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x20, 29, 1)
+#define SET_TX_DESC_STW_RB_DIS(__tx_desc, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x20, 28, 1, __value)
+#define GET_TX_DESC_STW_RB_DIS(__tx_desc)                                      \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x20, 28, 1)
+#define SET_TX_DESC_STW_RATE_DIS(__tx_desc, __value)                           \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x20, 27, 1, __value)
+#define GET_TX_DESC_STW_RATE_DIS(__tx_desc)                                    \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x20, 27, 1)
+#define SET_TX_DESC_STW_ANT_DIS(__tx_desc, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x20, 26, 1, __value)
+#define GET_TX_DESC_STW_ANT_DIS(__tx_desc)                                     \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x20, 26, 1)
+#define SET_TX_DESC_STW_EN(__tx_desc, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x20, 25, 1, __value)
+#define GET_TX_DESC_STW_EN(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x20, 25, 1)
+#define SET_TX_DESC_SMH_EN(__tx_desc, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x20, 24, 1, __value)
+#define GET_TX_DESC_SMH_EN(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x20, 24, 1)
+
+#define SET_TX_DESC_TAILPAGE_L(__tx_desc, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x20, 24, 8, __value)
+#define GET_TX_DESC_TAILPAGE_L(__tx_desc)                                      \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x20, 24, 8)
+
+#define SET_TX_DESC_SDIO_DMASEQ(__tx_desc, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x20, 16, 8, __value)
+#define GET_TX_DESC_SDIO_DMASEQ(__tx_desc)                                     \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x20, 16, 8)
+
+#define SET_TX_DESC_NEXTHEADPAGE_L(__tx_desc, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x20, 16, 8, __value)
+#define GET_TX_DESC_NEXTHEADPAGE_L(__tx_desc)                                  \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x20, 16, 8)
+#define SET_TX_DESC_EN_HWSEQ(__tx_desc, __value)                               \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x20, 15, 1, __value)
+#define GET_TX_DESC_EN_HWSEQ(__tx_desc)                                        \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x20, 15, 1)
+
+#define SET_TX_DESC_EN_HWEXSEQ(__tx_desc, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x20, 14, 1, __value)
+#define GET_TX_DESC_EN_HWEXSEQ(__tx_desc)                                      \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x20, 14, 1)
+
+#define SET_TX_DESC_DATA_RC(__tx_desc, __value)                                \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x20, 8, 6, __value)
+#define GET_TX_DESC_DATA_RC(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x20, 8, 6)
+#define SET_TX_DESC_BAR_RTY_TH(__tx_desc, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x20, 6, 2, __value)
+#define GET_TX_DESC_BAR_RTY_TH(__tx_desc)                                      \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x20, 6, 2)
+#define SET_TX_DESC_RTS_RC(__tx_desc, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x20, 0, 6, __value)
+#define GET_TX_DESC_RTS_RC(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x20, 0, 6)
+
+/*TXDESC_WORD9*/
+
+#define SET_TX_DESC_TAILPAGE_H(__tx_desc, __value)                             \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x24, 28, 4, __value)
+#define GET_TX_DESC_TAILPAGE_H(__tx_desc)                                      \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x24, 28, 4)
+#define SET_TX_DESC_NEXTHEADPAGE_H(__tx_desc, __value)                         \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x24, 24, 4, __value)
+#define GET_TX_DESC_NEXTHEADPAGE_H(__tx_desc)                                  \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x24, 24, 4)
+
+#define SET_TX_DESC_SW_SEQ(__tx_desc, __value)                                 \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x24, 12, 12, __value)
+#define GET_TX_DESC_SW_SEQ(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x24, 12, 12)
+#define SET_TX_DESC_TXBF_PATH(__tx_desc, __value)                              \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x24, 11, 1, __value)
+#define GET_TX_DESC_TXBF_PATH(__tx_desc)                                       \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x24, 11, 1)
+#define SET_TX_DESC_PADDING_LEN(__tx_desc, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x24, 0, 11, __value)
+#define GET_TX_DESC_PADDING_LEN(__tx_desc)                                     \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x24, 0, 11)
+#define SET_TX_DESC_GROUP_BIT_IE_OFFSET(__tx_desc, __value)                    \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x24, 0, 8, __value)
+#define GET_TX_DESC_GROUP_BIT_IE_OFFSET(__tx_desc)                             \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x24, 0, 8)
+
+/*WORD10*/
+
+#define SET_TX_DESC_MU_DATARATE(__tx_desc, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x28, 8, 8, __value)
+#define GET_TX_DESC_MU_DATARATE(__tx_desc)                                     \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x28, 8, 8)
+#define SET_TX_DESC_MU_RC(__tx_desc, __value)                                  \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x28, 4, 4, __value)
+#define GET_TX_DESC_MU_RC(__tx_desc) LE_BITS_TO_4BYTE(__tx_desc + 0x28, 4, 4)
+#define SET_TX_DESC_SND_PKT_SEL(__tx_desc, __value)                            \
+	SET_BITS_TO_LE_4BYTE(__tx_desc + 0x28, 0, 2, __value)
+#define GET_TX_DESC_SND_PKT_SEL(__tx_desc)                                     \
+	LE_BITS_TO_4BYTE(__tx_desc + 0x28, 0, 2)
+
+#endif
diff --git a/drivers/staging/rtlwifi/halmac/halmac_type.h b/drivers/staging/rtlwifi/halmac/halmac_type.h
new file mode 100644
index 000000000000..0bf842435080
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_type.h
@@ -0,0 +1,1934 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _HALMAC_TYPE_H_
+#define _HALMAC_TYPE_H_
+
+#include "halmac_2_platform.h"
+#include "halmac_fw_info.h"
+#include "halmac_intf_phy_cmd.h"
+
+#define HALMAC_SCAN_CH_NUM_MAX 28
+#define HALMAC_BCN_IE_BMP_SIZE 24 /* ID0~ID191, 192/8=24 */
+#define HALMAC_PHY_PARAMETER_SIZE 12
+#define HALMAC_PHY_PARAMETER_MAX_NUM 128
+#define HALMAC_MAX_SSID_LEN 32
+#define HALMAC_SUPPORT_NLO_NUM 16
+#define HALMAC_SUPPORT_PROBE_REQ_NUM 8
+#define HALMC_DDMA_POLLING_COUNT 1000
+#define API_ARRAY_SIZE 32
+
+/* platform api */
+#define PLATFORM_SDIO_CMD52_READ                                               \
+	halmac_adapter->halmac_platform_api->SDIO_CMD52_READ
+#define PLATFORM_SDIO_CMD53_READ_8                                             \
+	halmac_adapter->halmac_platform_api->SDIO_CMD53_READ_8
+#define PLATFORM_SDIO_CMD53_READ_16                                            \
+	halmac_adapter->halmac_platform_api->SDIO_CMD53_READ_16
+#define PLATFORM_SDIO_CMD53_READ_32                                            \
+	halmac_adapter->halmac_platform_api->SDIO_CMD53_READ_32
+#define PLATFORM_SDIO_CMD53_READ_N                                             \
+	halmac_adapter->halmac_platform_api->SDIO_CMD53_READ_N
+#define PLATFORM_SDIO_CMD52_WRITE                                              \
+	halmac_adapter->halmac_platform_api->SDIO_CMD52_WRITE
+#define PLATFORM_SDIO_CMD53_WRITE_8                                            \
+	halmac_adapter->halmac_platform_api->SDIO_CMD53_WRITE_8
+#define PLATFORM_SDIO_CMD53_WRITE_16                                           \
+	halmac_adapter->halmac_platform_api->SDIO_CMD53_WRITE_16
+#define PLATFORM_SDIO_CMD53_WRITE_32                                           \
+	halmac_adapter->halmac_platform_api->SDIO_CMD53_WRITE_32
+
+#define PLATFORM_REG_READ_8 halmac_adapter->halmac_platform_api->REG_READ_8
+#define PLATFORM_REG_READ_16 halmac_adapter->halmac_platform_api->REG_READ_16
+#define PLATFORM_REG_READ_32 halmac_adapter->halmac_platform_api->REG_READ_32
+#define PLATFORM_REG_WRITE_8 halmac_adapter->halmac_platform_api->REG_WRITE_8
+#define PLATFORM_REG_WRITE_16 halmac_adapter->halmac_platform_api->REG_WRITE_16
+#define PLATFORM_REG_WRITE_32 halmac_adapter->halmac_platform_api->REG_WRITE_32
+
+#define PLATFORM_SEND_RSVD_PAGE                                                \
+	halmac_adapter->halmac_platform_api->SEND_RSVD_PAGE
+#define PLATFORM_SEND_H2C_PKT halmac_adapter->halmac_platform_api->SEND_H2C_PKT
+
+#define PLATFORM_EVENT_INDICATION                                              \
+	halmac_adapter->halmac_platform_api->EVENT_INDICATION
+
+#define HALMAC_RT_TRACE(drv_adapter, comp, level, fmt, ...)                    \
+	RT_TRACE(drv_adapter, COMP_HALMAC, level, fmt, ##__VA_ARGS__)
+
+#define HALMAC_REG_READ_8 halmac_api->halmac_reg_read_8
+#define HALMAC_REG_READ_16 halmac_api->halmac_reg_read_16
+#define HALMAC_REG_READ_32 halmac_api->halmac_reg_read_32
+#define HALMAC_REG_WRITE_8 halmac_api->halmac_reg_write_8
+#define HALMAC_REG_WRITE_16 halmac_api->halmac_reg_write_16
+#define HALMAC_REG_WRITE_32 halmac_api->halmac_reg_write_32
+#define HALMAC_REG_SDIO_CMD53_READ_N halmac_api->halmac_reg_sdio_cmd53_read_n
+
+/* Swap Little-endian <-> Big-endia*/
+
+/*1->Little endian 0->Big endian*/
+#if HALMAC_SYSTEM_ENDIAN
+#else
+#endif
+
+#define HALMAC_ALIGN(x, a) HALMAC_ALIGN_MASK(x, (a) - 1)
+#define HALMAC_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask))
+
+/* HALMAC API return status*/
+enum halmac_ret_status {
+	HALMAC_RET_SUCCESS = 0x00,
+	HALMAC_RET_SUCCESS_ENQUEUE = 0x01,
+	HALMAC_RET_PLATFORM_API_NULL = 0x02,
+	HALMAC_RET_EFUSE_SIZE_INCORRECT = 0x03,
+	HALMAC_RET_MALLOC_FAIL = 0x04,
+	HALMAC_RET_ADAPTER_INVALID = 0x05,
+	HALMAC_RET_ITF_INCORRECT = 0x06,
+	HALMAC_RET_DLFW_FAIL = 0x07,
+	HALMAC_RET_PORT_NOT_SUPPORT = 0x08,
+	HALMAC_RET_TRXMODE_NOT_SUPPORT = 0x09,
+	HALMAC_RET_INIT_LLT_FAIL = 0x0A,
+	HALMAC_RET_POWER_STATE_INVALID = 0x0B,
+	HALMAC_RET_H2C_ACK_NOT_RECEIVED = 0x0C,
+	HALMAC_RET_DL_RSVD_PAGE_FAIL = 0x0D,
+	HALMAC_RET_EFUSE_R_FAIL = 0x0E,
+	HALMAC_RET_EFUSE_W_FAIL = 0x0F,
+	HALMAC_RET_H2C_SW_RES_FAIL = 0x10,
+	HALMAC_RET_SEND_H2C_FAIL = 0x11,
+	HALMAC_RET_PARA_NOT_SUPPORT = 0x12,
+	HALMAC_RET_PLATFORM_API_INCORRECT = 0x13,
+	HALMAC_RET_ENDIAN_ERR = 0x14,
+	HALMAC_RET_FW_SIZE_ERR = 0x15,
+	HALMAC_RET_TRX_MODE_NOT_SUPPORT = 0x16,
+	HALMAC_RET_FAIL = 0x17,
+	HALMAC_RET_CHANGE_PS_FAIL = 0x18,
+	HALMAC_RET_CFG_PARA_FAIL = 0x19,
+	HALMAC_RET_UPDATE_PROBE_FAIL = 0x1A,
+	HALMAC_RET_SCAN_FAIL = 0x1B,
+	HALMAC_RET_STOP_SCAN_FAIL = 0x1C,
+	HALMAC_RET_BCN_PARSER_CMD_FAIL = 0x1D,
+	HALMAC_RET_POWER_ON_FAIL = 0x1E,
+	HALMAC_RET_POWER_OFF_FAIL = 0x1F,
+	HALMAC_RET_RX_AGG_MODE_FAIL = 0x20,
+	HALMAC_RET_DATA_BUF_NULL = 0x21,
+	HALMAC_RET_DATA_SIZE_INCORRECT = 0x22,
+	HALMAC_RET_QSEL_INCORRECT = 0x23,
+	HALMAC_RET_DMA_MAP_INCORRECT = 0x24,
+	HALMAC_RET_SEND_ORIGINAL_H2C_FAIL = 0x25,
+	HALMAC_RET_DDMA_FAIL = 0x26,
+	HALMAC_RET_FW_CHECKSUM_FAIL = 0x27,
+	HALMAC_RET_PWRSEQ_POLLING_FAIL = 0x28,
+	HALMAC_RET_PWRSEQ_CMD_INCORRECT = 0x29,
+	HALMAC_RET_WRITE_DATA_FAIL = 0x2A,
+	HALMAC_RET_DUMP_FIFOSIZE_INCORRECT = 0x2B,
+	HALMAC_RET_NULL_POINTER = 0x2C,
+	HALMAC_RET_PROBE_NOT_FOUND = 0x2D,
+	HALMAC_RET_FW_NO_MEMORY = 0x2E,
+	HALMAC_RET_H2C_STATUS_ERR = 0x2F,
+	HALMAC_RET_GET_H2C_SPACE_ERR = 0x30,
+	HALMAC_RET_H2C_SPACE_FULL = 0x31,
+	HALMAC_RET_DATAPACK_NO_FOUND = 0x32,
+	HALMAC_RET_CANNOT_FIND_H2C_RESOURCE = 0x33,
+	HALMAC_RET_TX_DMA_ERR = 0x34,
+	HALMAC_RET_RX_DMA_ERR = 0x35,
+	HALMAC_RET_CHIP_NOT_SUPPORT = 0x36,
+	HALMAC_RET_FREE_SPACE_NOT_ENOUGH = 0x37,
+	HALMAC_RET_CH_SW_SEQ_WRONG = 0x38,
+	HALMAC_RET_CH_SW_NO_BUF = 0x39,
+	HALMAC_RET_SW_CASE_NOT_SUPPORT = 0x3A,
+	HALMAC_RET_CONVERT_SDIO_OFFSET_FAIL = 0x3B,
+	HALMAC_RET_INVALID_SOUNDING_SETTING = 0x3C,
+	HALMAC_RET_GEN_INFO_NOT_SENT = 0x3D,
+	HALMAC_RET_STATE_INCORRECT = 0x3E,
+	HALMAC_RET_H2C_BUSY = 0x3F,
+	HALMAC_RET_INVALID_FEATURE_ID = 0x40,
+	HALMAC_RET_BUFFER_TOO_SMALL = 0x41,
+	HALMAC_RET_ZERO_LEN_RSVD_PACKET = 0x42,
+	HALMAC_RET_BUSY_STATE = 0x43,
+	HALMAC_RET_ERROR_STATE = 0x44,
+	HALMAC_RET_API_INVALID = 0x45,
+	HALMAC_RET_POLLING_BCN_VALID_FAIL = 0x46,
+	HALMAC_RET_SDIO_LEAVE_SUSPEND_FAIL = 0x47,
+	HALMAC_RET_EEPROM_PARSING_FAIL = 0x48,
+	HALMAC_RET_EFUSE_NOT_ENOUGH = 0x49,
+	HALMAC_RET_WRONG_ARGUMENT = 0x4A,
+	HALMAC_RET_NOT_SUPPORT = 0x4B,
+	HALMAC_RET_C2H_NOT_HANDLED = 0x4C,
+	HALMAC_RET_PARA_SENDING = 0x4D,
+	HALMAC_RET_CFG_DLFW_SIZE_FAIL = 0x4E,
+	HALMAC_RET_CFG_TXFIFO_PAGE_FAIL = 0x4F,
+	HALMAC_RET_SWITCH_CASE_ERROR = 0x50,
+	HALMAC_RET_EFUSE_BANK_INCORRECT = 0x51,
+	HALMAC_RET_SWITCH_EFUSE_BANK_FAIL = 0x52,
+	HALMAC_RET_USB_MODE_UNCHANGE = 0x53,
+	HALMAC_RET_NO_DLFW = 0x54,
+	HALMAC_RET_USB2_3_SWITCH_UNSUPPORT = 0x55,
+	HALMAC_RET_BIP_NO_SUPPORT = 0x56,
+	HALMAC_RET_ENTRY_INDEX_ERROR = 0x57,
+	HALMAC_RET_ENTRY_KEY_ID_ERROR = 0x58,
+	HALMAC_RET_DRV_DL_ERR = 0x59,
+	HALMAC_RET_OQT_NOT_ENOUGH = 0x5A,
+	HALMAC_RET_PWR_UNCHANGE = 0x5B,
+	HALMAC_RET_FW_NO_SUPPORT = 0x60,
+	HALMAC_RET_TXFIFO_NO_EMPTY = 0x61,
+};
+
+enum halmac_mac_clock_hw_def {
+	HALMAC_MAC_CLOCK_HW_DEF_80M = 0,
+	HALMAC_MAC_CLOCK_HW_DEF_40M = 1,
+	HALMAC_MAC_CLOCK_HW_DEF_20M = 2,
+};
+
+/* Rx aggregation parameters */
+enum halmac_normal_rxagg_th_to {
+	HALMAC_NORMAL_RXAGG_THRESHOLD = 0xFF,
+	HALMAC_NORMAL_RXAGG_TIMEOUT = 0x01,
+};
+
+enum halmac_loopback_rxagg_th_to {
+	HALMAC_LOOPBACK_RXAGG_THRESHOLD = 0xFF,
+	HALMAC_LOOPBACK_RXAGG_TIMEOUT = 0x01,
+};
+
+/* Chip ID*/
+enum halmac_chip_id {
+	HALMAC_CHIP_ID_8822B = 0,
+	HALMAC_CHIP_ID_8821C = 1,
+	HALMAC_CHIP_ID_8814B = 2,
+	HALMAC_CHIP_ID_8197F = 3,
+	HALMAC_CHIP_ID_UNDEFINE = 0x7F,
+};
+
+enum halmac_chip_id_hw_def {
+	HALMAC_CHIP_ID_HW_DEF_8723A = 0x01,
+	HALMAC_CHIP_ID_HW_DEF_8188E = 0x02,
+	HALMAC_CHIP_ID_HW_DEF_8881A = 0x03,
+	HALMAC_CHIP_ID_HW_DEF_8812A = 0x04,
+	HALMAC_CHIP_ID_HW_DEF_8821A = 0x05,
+	HALMAC_CHIP_ID_HW_DEF_8723B = 0x06,
+	HALMAC_CHIP_ID_HW_DEF_8192E = 0x07,
+	HALMAC_CHIP_ID_HW_DEF_8814A = 0x08,
+	HALMAC_CHIP_ID_HW_DEF_8821C = 0x09,
+	HALMAC_CHIP_ID_HW_DEF_8822B = 0x0A,
+	HALMAC_CHIP_ID_HW_DEF_8703B = 0x0B,
+	HALMAC_CHIP_ID_HW_DEF_8188F = 0x0C,
+	HALMAC_CHIP_ID_HW_DEF_8192F = 0x0D,
+	HALMAC_CHIP_ID_HW_DEF_8197F = 0x0E,
+	HALMAC_CHIP_ID_HW_DEF_8723D = 0x0F,
+	HALMAC_CHIP_ID_HW_DEF_8814B = 0x10,
+	HALMAC_CHIP_ID_HW_DEF_UNDEFINE = 0x7F,
+	HALMAC_CHIP_ID_HW_DEF_PS = 0xEA,
+};
+
+/* Chip Version*/
+enum halmac_chip_ver {
+	HALMAC_CHIP_VER_A_CUT = 0x00,
+	HALMAC_CHIP_VER_B_CUT = 0x01,
+	HALMAC_CHIP_VER_C_CUT = 0x02,
+	HALMAC_CHIP_VER_D_CUT = 0x03,
+	HALMAC_CHIP_VER_E_CUT = 0x04,
+	HALMAC_CHIP_VER_F_CUT = 0x05,
+	HALMAC_CHIP_VER_TEST = 0xFF,
+	HALMAC_CHIP_VER_UNDEFINE = 0x7FFF,
+};
+
+/* Network type select */
+enum halmac_network_type_select {
+	HALMAC_NETWORK_NO_LINK = 0,
+	HALMAC_NETWORK_ADHOC = 1,
+	HALMAC_NETWORK_INFRASTRUCTURE = 2,
+	HALMAC_NETWORK_AP = 3,
+	HALMAC_NETWORK_UNDEFINE = 0x7F,
+};
+
+/* Transfer mode select */
+enum halmac_trnsfer_mode_select {
+	HALMAC_TRNSFER_NORMAL = 0x0,
+	HALMAC_TRNSFER_LOOPBACK_DIRECT = 0xB,
+	HALMAC_TRNSFER_LOOPBACK_DELAY = 0x3,
+	HALMAC_TRNSFER_UNDEFINE = 0x7F,
+};
+
+/* Queue select */
+enum halmac_dma_mapping {
+	HALMAC_DMA_MAPPING_EXTRA = 0,
+	HALMAC_DMA_MAPPING_LOW = 1,
+	HALMAC_DMA_MAPPING_NORMAL = 2,
+	HALMAC_DMA_MAPPING_HIGH = 3,
+	HALMAC_DMA_MAPPING_UNDEFINE = 0x7F,
+};
+
+#define HALMAC_MAP2_HQ HALMAC_DMA_MAPPING_HIGH
+#define HALMAC_MAP2_NQ HALMAC_DMA_MAPPING_NORMAL
+#define HALMAC_MAP2_LQ HALMAC_DMA_MAPPING_LOW
+#define HALMAC_MAP2_EXQ HALMAC_DMA_MAPPING_EXTRA
+#define HALMAC_MAP2_UNDEF HALMAC_DMA_MAPPING_UNDEFINE
+
+/* TXDESC queue select TID */
+enum halmac_txdesc_queue_tid {
+	HALMAC_TXDESC_QSEL_TID0 = 0,
+	HALMAC_TXDESC_QSEL_TID1 = 1,
+	HALMAC_TXDESC_QSEL_TID2 = 2,
+	HALMAC_TXDESC_QSEL_TID3 = 3,
+	HALMAC_TXDESC_QSEL_TID4 = 4,
+	HALMAC_TXDESC_QSEL_TID5 = 5,
+	HALMAC_TXDESC_QSEL_TID6 = 6,
+	HALMAC_TXDESC_QSEL_TID7 = 7,
+	HALMAC_TXDESC_QSEL_TID8 = 8,
+	HALMAC_TXDESC_QSEL_TID9 = 9,
+	HALMAC_TXDESC_QSEL_TIDA = 10,
+	HALMAC_TXDESC_QSEL_TIDB = 11,
+	HALMAC_TXDESC_QSEL_TIDC = 12,
+	HALMAC_TXDESC_QSEL_TIDD = 13,
+	HALMAC_TXDESC_QSEL_TIDE = 14,
+	HALMAC_TXDESC_QSEL_TIDF = 15,
+
+	HALMAC_TXDESC_QSEL_BEACON = 0x10,
+	HALMAC_TXDESC_QSEL_HIGH = 0x11,
+	HALMAC_TXDESC_QSEL_MGT = 0x12,
+	HALMAC_TXDESC_QSEL_H2C_CMD = 0x13,
+
+	HALMAC_TXDESC_QSEL_UNDEFINE = 0x7F,
+};
+
+enum halmac_ptcl_queue {
+	HALMAC_PTCL_QUEUE_VO = 0x0,
+	HALMAC_PTCL_QUEUE_VI = 0x1,
+	HALMAC_PTCL_QUEUE_BE = 0x2,
+	HALMAC_PTCL_QUEUE_BK = 0x3,
+	HALMAC_PTCL_QUEUE_MG = 0x4,
+	HALMAC_PTCL_QUEUE_HI = 0x5,
+	HALMAC_PTCL_QUEUE_NUM = 0x6,
+	HALMAC_PTCL_QUEUE_UNDEFINE = 0x7F,
+};
+
+enum halmac_queue_select {
+	HALMAC_QUEUE_SELECT_VO = HALMAC_TXDESC_QSEL_TID6,
+	HALMAC_QUEUE_SELECT_VI = HALMAC_TXDESC_QSEL_TID4,
+	HALMAC_QUEUE_SELECT_BE = HALMAC_TXDESC_QSEL_TID0,
+	HALMAC_QUEUE_SELECT_BK = HALMAC_TXDESC_QSEL_TID1,
+	HALMAC_QUEUE_SELECT_VO_V2 = HALMAC_TXDESC_QSEL_TID7,
+	HALMAC_QUEUE_SELECT_VI_V2 = HALMAC_TXDESC_QSEL_TID5,
+	HALMAC_QUEUE_SELECT_BE_V2 = HALMAC_TXDESC_QSEL_TID3,
+	HALMAC_QUEUE_SELECT_BK_V2 = HALMAC_TXDESC_QSEL_TID2,
+	HALMAC_QUEUE_SELECT_BCN = HALMAC_TXDESC_QSEL_BEACON,
+	HALMAC_QUEUE_SELECT_HIGH = HALMAC_TXDESC_QSEL_HIGH,
+	HALMAC_QUEUE_SELECT_MGNT = HALMAC_TXDESC_QSEL_MGT,
+	HALMAC_QUEUE_SELECT_CMD = HALMAC_TXDESC_QSEL_H2C_CMD,
+	HALMAC_QUEUE_SELECT_UNDEFINE = 0x7F,
+};
+
+/* USB burst size */
+enum halmac_usb_burst_size {
+	HALMAC_USB_BURST_SIZE_3_0 = 0x0,
+	HALMAC_USB_BURST_SIZE_2_0_HSPEED = 0x1,
+	HALMAC_USB_BURST_SIZE_2_0_FSPEED = 0x2,
+	HALMAC_USB_BURST_SIZE_2_0_OTHERS = 0x3,
+	HALMAC_USB_BURST_SIZE_UNDEFINE = 0x7F,
+};
+
+/* HAL API  function parameters*/
+enum halmac_interface {
+	HALMAC_INTERFACE_PCIE = 0x0,
+	HALMAC_INTERFACE_USB = 0x1,
+	HALMAC_INTERFACE_SDIO = 0x2,
+	HALMAC_INTERFACE_AXI = 0x3,
+	HALMAC_INTERFACE_UNDEFINE = 0x7F,
+};
+
+enum halmac_rx_agg_mode {
+	HALMAC_RX_AGG_MODE_NONE = 0x0,
+	HALMAC_RX_AGG_MODE_DMA = 0x1,
+	HALMAC_RX_AGG_MODE_USB = 0x2,
+	HALMAC_RX_AGG_MODE_UNDEFINE = 0x7F,
+};
+
+struct halmac_rxagg_th {
+	u8 drv_define;
+	u8 timeout;
+	u8 size;
+};
+
+struct halmac_rxagg_cfg {
+	enum halmac_rx_agg_mode mode;
+	struct halmac_rxagg_th threshold;
+};
+
+enum halmac_mac_power {
+	HALMAC_MAC_POWER_OFF = 0x0,
+	HALMAC_MAC_POWER_ON = 0x1,
+	HALMAC_MAC_POWER_UNDEFINE = 0x7F,
+};
+
+enum halmac_ps_state {
+	HALMAC_PS_STATE_ACT = 0x0,
+	HALMAC_PS_STATE_LPS = 0x1,
+	HALMAC_PS_STATE_IPS = 0x2,
+	HALMAC_PS_STATE_UNDEFINE = 0x7F,
+};
+
+enum halmac_trx_mode {
+	HALMAC_TRX_MODE_NORMAL = 0x0,
+	HALMAC_TRX_MODE_TRXSHARE = 0x1,
+	HALMAC_TRX_MODE_WMM = 0x2,
+	HALMAC_TRX_MODE_P2P = 0x3,
+	HALMAC_TRX_MODE_LOOPBACK = 0x4,
+	HALMAC_TRX_MODE_DELAY_LOOPBACK = 0x5,
+	HALMAC_TRX_MODE_MAX = 0x6,
+	HALMAC_TRX_MODE_WMM_LINUX = 0x7E,
+	HALMAC_TRX_MODE_UNDEFINE = 0x7F,
+};
+
+enum halmac_wireless_mode {
+	HALMAC_WIRELESS_MODE_B = 0x0,
+	HALMAC_WIRELESS_MODE_G = 0x1,
+	HALMAC_WIRELESS_MODE_N = 0x2,
+	HALMAC_WIRELESS_MODE_AC = 0x3,
+	HALMAC_WIRELESS_MODE_UNDEFINE = 0x7F,
+};
+
+enum halmac_bw {
+	HALMAC_BW_20 = 0x00,
+	HALMAC_BW_40 = 0x01,
+	HALMAC_BW_80 = 0x02,
+	HALMAC_BW_160 = 0x03,
+	HALMAC_BW_5 = 0x04,
+	HALMAC_BW_10 = 0x05,
+	HALMAC_BW_MAX = 0x06,
+	HALMAC_BW_UNDEFINE = 0x7F,
+};
+
+enum halmac_efuse_read_cfg {
+	HALMAC_EFUSE_R_AUTO = 0x00,
+	HALMAC_EFUSE_R_DRV = 0x01,
+	HALMAC_EFUSE_R_FW = 0x02,
+	HALMAC_EFUSE_R_UNDEFINE = 0x7F,
+};
+
+enum halmac_dlfw_mem {
+	HALMAC_DLFW_MEM_EMEM = 0x00,
+	HALMAC_DLFW_MEM_UNDEFINE = 0x7F,
+};
+
+struct halmac_tx_desc {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+	u32 dword6;
+	u32 dword7;
+	u32 dword8;
+	u32 dword9;
+	u32 dword10;
+	u32 dword11;
+};
+
+struct halmac_rx_desc {
+	u32 dword0;
+	u32 dword1;
+	u32 dword2;
+	u32 dword3;
+	u32 dword4;
+	u32 dword5;
+};
+
+struct halmac_fwlps_option {
+	u8 mode;
+	u8 clk_request;
+	u8 rlbm;
+	u8 smart_ps;
+	u8 awake_interval;
+	u8 all_queue_uapsd;
+	u8 pwr_state;
+	u8 low_pwr_rx_beacon;
+	u8 ant_auto_switch;
+	u8 ps_allow_bt_high_priority;
+	u8 protect_bcn;
+	u8 silence_period;
+	u8 fast_bt_connect;
+	u8 two_antenna_en;
+	u8 adopt_user_setting;
+	u8 drv_bcn_early_shift;
+	bool enter_32K;
+};
+
+struct halmac_fwips_option {
+	u8 adopt_user_setting;
+};
+
+struct halmac_wowlan_option {
+	u8 adopt_user_setting;
+};
+
+struct halmac_bcn_ie_info {
+	u8 func_en;
+	u8 size_th;
+	u8 timeout;
+	u8 ie_bmp[HALMAC_BCN_IE_BMP_SIZE];
+};
+
+enum halmac_reg_type {
+	HALMAC_REG_TYPE_MAC = 0x0,
+	HALMAC_REG_TYPE_BB = 0x1,
+	HALMAC_REG_TYPE_RF = 0x2,
+	HALMAC_REG_TYPE_UNDEFINE = 0x7F,
+};
+
+enum halmac_parameter_cmd {
+	/* HALMAC_PARAMETER_CMD_LLT				= 0x1, */
+	/* HALMAC_PARAMETER_CMD_R_EFUSE			= 0x2, */
+	/* HALMAC_PARAMETER_CMD_EFUSE_PATCH	= 0x3, */
+	HALMAC_PARAMETER_CMD_MAC_W8 = 0x4,
+	HALMAC_PARAMETER_CMD_MAC_W16 = 0x5,
+	HALMAC_PARAMETER_CMD_MAC_W32 = 0x6,
+	HALMAC_PARAMETER_CMD_RF_W = 0x7,
+	HALMAC_PARAMETER_CMD_BB_W8 = 0x8,
+	HALMAC_PARAMETER_CMD_BB_W16 = 0x9,
+	HALMAC_PARAMETER_CMD_BB_W32 = 0XA,
+	HALMAC_PARAMETER_CMD_DELAY_US = 0X10,
+	HALMAC_PARAMETER_CMD_DELAY_MS = 0X11,
+	HALMAC_PARAMETER_CMD_END = 0XFF,
+};
+
+union halmac_parameter_content {
+	struct _MAC_REG_W {
+		u32 value;
+		u32 msk;
+		u16 offset;
+		u8 msk_en;
+	} MAC_REG_W;
+	struct _BB_REG_W {
+		u32 value;
+		u32 msk;
+		u16 offset;
+		u8 msk_en;
+	} BB_REG_W;
+	struct _RF_REG_W {
+		u32 value;
+		u32 msk;
+		u8 offset;
+		u8 msk_en;
+		u8 rf_path;
+	} RF_REG_W;
+	struct _DELAY_TIME {
+		u32 rsvd1;
+		u32 rsvd2;
+		u16 delay_time;
+		u8 rsvd3;
+	} DELAY_TIME;
+};
+
+struct halmac_phy_parameter_info {
+	enum halmac_parameter_cmd cmd_id;
+	union halmac_parameter_content content;
+};
+
+struct halmac_h2c_info {
+	u16 h2c_seq_num; /* H2C sequence number */
+	u8 in_use; /* 0 : empty 1 : used */
+	enum halmac_h2c_return_code status;
+};
+
+struct halmac_pg_efuse_info {
+	u8 *efuse_map;
+	u32 efuse_map_size;
+	u8 *efuse_mask;
+	u32 efuse_mask_size;
+};
+
+struct halmac_txagg_buff_info {
+	u8 *tx_agg_buf;
+	u8 *curr_pkt_buf;
+	u32 avai_buf_size;
+	u32 total_pkt_size;
+	u8 agg_num;
+};
+
+struct halmac_config_para_info {
+	u32 para_buf_size; /* Parameter buffer size */
+	u8 *cfg_para_buf; /* Buffer for config parameter */
+	u8 *para_buf_w; /* Write pointer of the parameter buffer */
+	u32 para_num; /* Parameter numbers in parameter buffer */
+	u32 avai_para_buf_size; /* Free size of parameter buffer */
+	u32 offset_accumulation;
+	u32 value_accumulation;
+	enum halmac_data_type data_type; /*DataType which is passed to FW*/
+	u8 datapack_segment; /*DataPack Segment, from segment0...*/
+	bool full_fifo_mode; /* Used full tx fifo to save cfg parameter */
+};
+
+struct halmac_hw_config_info {
+	u32 efuse_size; /* Record efuse size */
+	u32 eeprom_size; /* Record eeprom size */
+	u32 bt_efuse_size; /* Record BT efuse size */
+	u32 tx_fifo_size; /* Record tx fifo size */
+	u32 rx_fifo_size; /* Record rx fifo size */
+	u8 txdesc_size; /* Record tx desc size */
+	u8 rxdesc_size; /* Record rx desc size */
+	u32 page_size; /* Record page size */
+	u16 tx_align_size;
+	u8 page_size_2_power;
+	u8 cam_entry_num; /* Record CAM entry number */
+};
+
+struct halmac_sdio_free_space {
+	u16 high_queue_number; /* Free space of HIQ */
+	u16 normal_queue_number; /* Free space of MIDQ */
+	u16 low_queue_number; /* Free space of LOWQ */
+	u16 public_queue_number; /* Free space of PUBQ */
+	u16 extra_queue_number; /* Free space of EXBQ */
+	u8 ac_oqt_number;
+	u8 non_ac_oqt_number;
+	u8 ac_empty;
+};
+
+enum hal_fifo_sel {
+	HAL_FIFO_SEL_TX,
+	HAL_FIFO_SEL_RX,
+	HAL_FIFO_SEL_RSVD_PAGE,
+	HAL_FIFO_SEL_REPORT,
+	HAL_FIFO_SEL_LLT,
+};
+
+enum halmac_drv_info {
+	HALMAC_DRV_INFO_NONE, /* No information is appended in rx_pkt */
+	HALMAC_DRV_INFO_PHY_STATUS, /* PHY status is appended after rx_desc */
+	HALMAC_DRV_INFO_PHY_SNIFFER, /* PHY status and sniffer info appended */
+	HALMAC_DRV_INFO_PHY_PLCP, /* PHY status and plcp header are appended */
+	HALMAC_DRV_INFO_UNDEFINE,
+};
+
+struct halmac_bt_coex_cmd {
+	u8 element_id;
+	u8 op_code;
+	u8 op_code_ver;
+	u8 req_num;
+	u8 data0;
+	u8 data1;
+	u8 data2;
+	u8 data3;
+	u8 data4;
+};
+
+enum halmac_pri_ch_idx {
+	HALMAC_CH_IDX_UNDEFINE = 0,
+	HALMAC_CH_IDX_1 = 1,
+	HALMAC_CH_IDX_2 = 2,
+	HALMAC_CH_IDX_3 = 3,
+	HALMAC_CH_IDX_4 = 4,
+	HALMAC_CH_IDX_MAX = 5,
+};
+
+struct halmac_ch_info {
+	enum halmac_cs_action_id action_id;
+	enum halmac_bw bw;
+	enum halmac_pri_ch_idx pri_ch_idx;
+	u8 channel;
+	u8 timeout;
+	u8 extra_info;
+};
+
+struct halmac_ch_extra_info {
+	u8 extra_info;
+	enum halmac_cs_extra_action_id extra_action_id;
+	u8 extra_info_size;
+	u8 *extra_info_data;
+};
+
+enum halmac_cs_periodic_option {
+	HALMAC_CS_PERIODIC_NONE,
+	HALMAC_CS_PERIODIC_NORMAL,
+	HALMAC_CS_PERIODIC_2_PHASE,
+	HALMAC_CS_PERIODIC_SEAMLESS,
+};
+
+struct halmac_ch_switch_option {
+	enum halmac_bw dest_bw;
+	enum halmac_cs_periodic_option periodic_option;
+	enum halmac_pri_ch_idx dest_pri_ch_idx;
+	/* u32 tsf_high; */
+	u32 tsf_low;
+	bool switch_en;
+	u8 dest_ch_en;
+	u8 absolute_time_en;
+	u8 dest_ch;
+	u8 normal_period;
+	u8 normal_cycle;
+	u8 phase_2_period;
+};
+
+struct halmac_fw_version {
+	u16 version;
+	u8 sub_version;
+	u8 sub_index;
+	u16 h2c_version;
+};
+
+enum halmac_rf_type {
+	HALMAC_RF_1T2R = 0,
+	HALMAC_RF_2T4R = 1,
+	HALMAC_RF_2T2R = 2,
+	HALMAC_RF_2T3R = 3,
+	HALMAC_RF_1T1R = 4,
+	HALMAC_RF_2T2R_GREEN = 5,
+	HALMAC_RF_3T3R = 6,
+	HALMAC_RF_3T4R = 7,
+	HALMAC_RF_4T4R = 8,
+	HALMAC_RF_MAX_TYPE = 0xF,
+};
+
+struct halmac_general_info {
+	u8 rfe_type;
+	enum halmac_rf_type rf_type;
+};
+
+struct halmac_pwr_tracking_para {
+	u8 enable;
+	u8 tx_pwr_index;
+	u8 pwr_tracking_offset_value;
+	u8 tssi_value;
+};
+
+struct halmac_pwr_tracking_option {
+	u8 type;
+	u8 bbswing_index;
+	struct halmac_pwr_tracking_para
+		pwr_tracking_para[4]; /* pathA, pathB, pathC, pathD */
+};
+
+struct halmac_nlo_cfg {
+	u8 num_of_ssid;
+	u8 num_of_hidden_ap;
+	u8 rsvd[2];
+	u32 pattern_check;
+	u32 rsvd1;
+	u32 rsvd2;
+	u8 ssid_len[HALMAC_SUPPORT_NLO_NUM];
+	u8 chiper_type[HALMAC_SUPPORT_NLO_NUM];
+	u8 rsvd3[HALMAC_SUPPORT_NLO_NUM];
+	u8 loc_probe_req[HALMAC_SUPPORT_PROBE_REQ_NUM];
+	u8 rsvd4[56];
+	u8 ssid[HALMAC_SUPPORT_NLO_NUM][HALMAC_MAX_SSID_LEN];
+};
+
+enum halmac_data_rate {
+	HALMAC_CCK1,
+	HALMAC_CCK2,
+	HALMAC_CCK5_5,
+	HALMAC_CCK11,
+	HALMAC_OFDM6,
+	HALMAC_OFDM9,
+	HALMAC_OFDM12,
+	HALMAC_OFDM18,
+	HALMAC_OFDM24,
+	HALMAC_OFDM36,
+	HALMAC_OFDM48,
+	HALMAC_OFDM54,
+	HALMAC_MCS0,
+	HALMAC_MCS1,
+	HALMAC_MCS2,
+	HALMAC_MCS3,
+	HALMAC_MCS4,
+	HALMAC_MCS5,
+	HALMAC_MCS6,
+	HALMAC_MCS7,
+	HALMAC_MCS8,
+	HALMAC_MCS9,
+	HALMAC_MCS10,
+	HALMAC_MCS11,
+	HALMAC_MCS12,
+	HALMAC_MCS13,
+	HALMAC_MCS14,
+	HALMAC_MCS15,
+	HALMAC_MCS16,
+	HALMAC_MCS17,
+	HALMAC_MCS18,
+	HALMAC_MCS19,
+	HALMAC_MCS20,
+	HALMAC_MCS21,
+	HALMAC_MCS22,
+	HALMAC_MCS23,
+	HALMAC_MCS24,
+	HALMAC_MCS25,
+	HALMAC_MCS26,
+	HALMAC_MCS27,
+	HALMAC_MCS28,
+	HALMAC_MCS29,
+	HALMAC_MCS30,
+	HALMAC_MCS31,
+	HALMAC_VHT_NSS1_MCS0,
+	HALMAC_VHT_NSS1_MCS1,
+	HALMAC_VHT_NSS1_MCS2,
+	HALMAC_VHT_NSS1_MCS3,
+	HALMAC_VHT_NSS1_MCS4,
+	HALMAC_VHT_NSS1_MCS5,
+	HALMAC_VHT_NSS1_MCS6,
+	HALMAC_VHT_NSS1_MCS7,
+	HALMAC_VHT_NSS1_MCS8,
+	HALMAC_VHT_NSS1_MCS9,
+	HALMAC_VHT_NSS2_MCS0,
+	HALMAC_VHT_NSS2_MCS1,
+	HALMAC_VHT_NSS2_MCS2,
+	HALMAC_VHT_NSS2_MCS3,
+	HALMAC_VHT_NSS2_MCS4,
+	HALMAC_VHT_NSS2_MCS5,
+	HALMAC_VHT_NSS2_MCS6,
+	HALMAC_VHT_NSS2_MCS7,
+	HALMAC_VHT_NSS2_MCS8,
+	HALMAC_VHT_NSS2_MCS9,
+	HALMAC_VHT_NSS3_MCS0,
+	HALMAC_VHT_NSS3_MCS1,
+	HALMAC_VHT_NSS3_MCS2,
+	HALMAC_VHT_NSS3_MCS3,
+	HALMAC_VHT_NSS3_MCS4,
+	HALMAC_VHT_NSS3_MCS5,
+	HALMAC_VHT_NSS3_MCS6,
+	HALMAC_VHT_NSS3_MCS7,
+	HALMAC_VHT_NSS3_MCS8,
+	HALMAC_VHT_NSS3_MCS9,
+	HALMAC_VHT_NSS4_MCS0,
+	HALMAC_VHT_NSS4_MCS1,
+	HALMAC_VHT_NSS4_MCS2,
+	HALMAC_VHT_NSS4_MCS3,
+	HALMAC_VHT_NSS4_MCS4,
+	HALMAC_VHT_NSS4_MCS5,
+	HALMAC_VHT_NSS4_MCS6,
+	HALMAC_VHT_NSS4_MCS7,
+	HALMAC_VHT_NSS4_MCS8,
+	HALMAC_VHT_NSS4_MCS9
+};
+
+enum halmac_rf_path {
+	HALMAC_RF_PATH_A,
+	HALMAC_RF_PATH_B,
+	HALMAC_RF_PATH_C,
+	HALMAC_RF_PATH_D
+};
+
+enum halmac_snd_pkt_sel {
+	HALMAC_UNI_NDPA,
+	HALMAC_BMC_NDPA,
+	HALMAC_NON_FINAL_BFRPRPOLL,
+	HALMAC_FINAL_BFRPTPOLL,
+};
+
+enum hal_security_type {
+	HAL_SECURITY_TYPE_NONE = 0,
+	HAL_SECURITY_TYPE_WEP40 = 1,
+	HAL_SECURITY_TYPE_WEP104 = 2,
+	HAL_SECURITY_TYPE_TKIP = 3,
+	HAL_SECURITY_TYPE_AES128 = 4,
+	HAL_SECURITY_TYPE_WAPI = 5,
+	HAL_SECURITY_TYPE_AES256 = 6,
+	HAL_SECURITY_TYPE_GCMP128 = 7,
+	HAL_SECURITY_TYPE_GCMP256 = 8,
+	HAL_SECURITY_TYPE_GCMSMS4 = 9,
+	HAL_SECURITY_TYPE_BIP = 10,
+	HAL_SECURITY_TYPE_UNDEFINE = 0x7F,
+};
+
+enum hal_intf_phy {
+	HAL_INTF_PHY_USB2 = 0,
+	HAL_INTF_PHY_USB3 = 1,
+	HAL_INTF_PHY_PCIE_GEN1 = 2,
+	HAL_INTF_PHY_PCIE_GEN2 = 3,
+	HAL_INTF_PHY_UNDEFINE = 0x7F,
+};
+
+enum halmac_dbg_msg_info {
+	HALMAC_DBG_ERR,
+	HALMAC_DBG_WARN,
+	HALMAC_DBG_TRACE,
+};
+
+enum halmac_dbg_msg_type {
+	HALMAC_MSG_INIT,
+	HALMAC_MSG_EFUSE,
+	HALMAC_MSG_FW,
+	HALMAC_MSG_H2C,
+	HALMAC_MSG_PWR,
+	HALMAC_MSG_SND,
+	HALMAC_MSG_COMMON,
+	HALMAC_MSG_DBI,
+	HALMAC_MSG_MDIO,
+	HALMAC_MSG_USB
+};
+
+enum halmac_cmd_process_status {
+	HALMAC_CMD_PROCESS_IDLE = 0x01, /* Init status */
+	HALMAC_CMD_PROCESS_SENDING = 0x02, /* Wait ack */
+	HALMAC_CMD_PROCESS_RCVD = 0x03, /* Rcvd ack */
+	HALMAC_CMD_PROCESS_DONE = 0x04, /* Event done */
+	HALMAC_CMD_PROCESS_ERROR = 0x05, /* Return code error */
+	HALMAC_CMD_PROCESS_UNDEFINE = 0x7F,
+};
+
+enum halmac_feature_id {
+	HALMAC_FEATURE_CFG_PARA, /* Support */
+	HALMAC_FEATURE_DUMP_PHYSICAL_EFUSE, /* Support */
+	HALMAC_FEATURE_DUMP_LOGICAL_EFUSE, /* Support */
+	HALMAC_FEATURE_UPDATE_PACKET, /* Support */
+	HALMAC_FEATURE_UPDATE_DATAPACK,
+	HALMAC_FEATURE_RUN_DATAPACK,
+	HALMAC_FEATURE_CHANNEL_SWITCH, /* Support */
+	HALMAC_FEATURE_IQK, /* Support */
+	HALMAC_FEATURE_POWER_TRACKING, /* Support */
+	HALMAC_FEATURE_PSD, /* Support */
+	HALMAC_FEATURE_ALL, /* Support, only for reset */
+};
+
+enum halmac_drv_rsvd_pg_num {
+	HALMAC_RSVD_PG_NUM16, /* 2K */
+	HALMAC_RSVD_PG_NUM24, /* 3K */
+	HALMAC_RSVD_PG_NUM32, /* 4K */
+};
+
+enum halmac_pcie_cfg {
+	HALMAC_PCIE_GEN1,
+	HALMAC_PCIE_GEN2,
+	HALMAC_PCIE_CFG_UNDEFINE,
+};
+
+enum halmac_portid {
+	HALMAC_PORTID0 = 0,
+	HALMAC_PORTID1 = 1,
+	HALMAC_PORTID2 = 2,
+	HALMAC_PORTID3 = 3,
+	HALMAC_PORTID4 = 4,
+	HALMAC_PORTIDMAX
+};
+
+struct halmac_p2pps {
+	/*DW0*/
+	u8 offload_en : 1;
+	u8 role : 1;
+	u8 ctwindow_en : 1;
+	u8 noa_en : 1;
+	u8 noa_sel : 1;
+	u8 all_sta_sleep : 1;
+	u8 discovery : 1;
+	u8 rsvd2 : 1;
+	u8 p2p_port_id;
+	u8 p2p_group;
+	u8 p2p_macid;
+
+	/*DW1*/
+	u8 ctwindow_length;
+	u8 rsvd3;
+	u8 rsvd4;
+	u8 rsvd5;
+
+	/*DW2*/
+	u32 noa_duration_para;
+
+	/*DW3*/
+	u32 noa_interval_para;
+
+	/*DW4*/
+	u32 noa_start_time_para;
+
+	/*DW5*/
+	u32 noa_count_para;
+};
+
+/* Platform API setting */
+struct halmac_platform_api {
+	/* R/W register */
+	u8 (*SDIO_CMD52_READ)(void *driver_adapter, u32 offset);
+	u8 (*SDIO_CMD53_READ_8)(void *driver_adapter, u32 offset);
+	u16 (*SDIO_CMD53_READ_16)(void *driver_adapter, u32 offset);
+	u32 (*SDIO_CMD53_READ_32)(void *driver_adapter, u32 offset);
+	u8 (*SDIO_CMD53_READ_N)(void *driver_adapter, u32 offset, u32 size,
+				u8 *data);
+	void (*SDIO_CMD52_WRITE)(void *driver_adapter, u32 offset, u8 value);
+	void (*SDIO_CMD53_WRITE_8)(void *driver_adapter, u32 offset, u8 value);
+	void (*SDIO_CMD53_WRITE_16)(void *driver_adapter, u32 offset,
+				    u16 value);
+	void (*SDIO_CMD53_WRITE_32)(void *driver_adapter, u32 offset,
+				    u32 value);
+	u8 (*REG_READ_8)(void *driver_adapter, u32 offset);
+	u16 (*REG_READ_16)(void *driver_adapter, u32 offset);
+	u32 (*REG_READ_32)(void *driver_adapter, u32 offset);
+	void (*REG_WRITE_8)(void *driver_adapter, u32 offset, u8 value);
+	void (*REG_WRITE_16)(void *driver_adapter, u32 offset, u16 value);
+	void (*REG_WRITE_32)(void *driver_adapter, u32 offset, u32 value);
+
+	/* send buf to reserved page, the tx_desc is not included in buf,
+	 * driver need to fill tx_desc with qsel = bcn
+	 */
+	bool (*SEND_RSVD_PAGE)(void *driver_adapter, u8 *buf, u32 size);
+	/* send buf to h2c queue, the tx_desc is not included in buf,
+	 * driver need to fill tx_desc with qsel = h2c
+	 */
+	bool (*SEND_H2C_PKT)(void *driver_adapter, u8 *buf, u32 size);
+
+	bool (*EVENT_INDICATION)(void *driver_adapter,
+				 enum halmac_feature_id feature_id,
+				 enum halmac_cmd_process_status process_status,
+				 u8 *buf, u32 size);
+};
+
+/*1->Little endian 0->Big endian*/
+#if HALMAC_SYSTEM_ENDIAN
+
+#else
+
+#endif
+
+/* User can not use members in address_l_h, use address[6] is mandatory */
+union halmac_wlan_addr {
+	u8 address[6]; /* WLAN address (MACID, BSSID, Brodcast ID).
+			* address[0] is lowest, address[5] is highest
+			*/
+	struct {
+		union {
+			u32 address_low;
+			__le32 le_address_low;
+			u8 address_low_b[4];
+		};
+		union {
+			u16 address_high;
+			__le16 le_address_high;
+			u8 address_high_b[2];
+		};
+	} address_l_h;
+};
+
+enum halmac_snd_role {
+	HAL_BFER = 0,
+	HAL_BFEE = 1,
+};
+
+enum halmac_csi_seg_len {
+	HAL_CSI_SEG_4K = 0,
+	HAL_CSI_SEG_8K = 1,
+	HAL_CSI_SEG_11K = 2,
+};
+
+struct halmac_cfg_mumimo_para {
+	enum halmac_snd_role role;
+	bool sounding_sts[6];
+	u16 grouping_bitmap;
+	bool mu_tx_en;
+	u32 given_gid_tab[2];
+	u32 given_user_pos[4];
+};
+
+struct halmac_su_bfer_init_para {
+	u8 userid;
+	u16 paid;
+	u16 csi_para;
+	union halmac_wlan_addr bfer_address;
+};
+
+struct halmac_mu_bfee_init_para {
+	u8 userid;
+	u16 paid;
+	u32 user_position_l;
+	u32 user_position_h;
+};
+
+struct halmac_mu_bfer_init_para {
+	u16 paid;
+	u16 csi_para;
+	u16 my_aid;
+	enum halmac_csi_seg_len csi_length_sel;
+	union halmac_wlan_addr bfer_address;
+};
+
+struct halmac_snd_info {
+	u16 paid;
+	u8 userid;
+	enum halmac_data_rate ndpa_rate;
+	u16 csi_para;
+	u16 my_aid;
+	enum halmac_data_rate csi_rate;
+	enum halmac_csi_seg_len csi_length_sel;
+	enum halmac_snd_role role;
+	union halmac_wlan_addr bfer_address;
+	enum halmac_bw bw;
+	u8 txbf_en;
+	struct halmac_su_bfer_init_para *su_bfer_init;
+	struct halmac_mu_bfer_init_para *mu_bfer_init;
+	struct halmac_mu_bfee_init_para *mu_bfee_init;
+};
+
+struct halmac_cs_info {
+	u8 *ch_info_buf;
+	u8 *ch_info_buf_w;
+	u8 extra_info_en;
+	u32 buf_size; /* buffer size */
+	u32 avai_buf_size; /* buffer size */
+	u32 total_size;
+	u32 accu_timeout;
+	u32 ch_num;
+};
+
+struct halmac_restore_info {
+	u32 mac_register;
+	u32 value;
+	u8 length;
+};
+
+struct halmac_event_trigger {
+	u32 physical_efuse_map : 1;
+	u32 logical_efuse_map : 1;
+	u32 rsvd1 : 28;
+};
+
+struct halmac_h2c_header_info {
+	u16 sub_cmd_id;
+	u16 content_size;
+	bool ack;
+};
+
+enum halmac_dlfw_state {
+	HALMAC_DLFW_NONE = 0,
+	HALMAC_DLFW_DONE = 1,
+	HALMAC_GEN_INFO_SENT = 2,
+	HALMAC_DLFW_UNDEFINED = 0x7F,
+};
+
+enum halmac_efuse_cmd_construct_state {
+	HALMAC_EFUSE_CMD_CONSTRUCT_IDLE = 0,
+	HALMAC_EFUSE_CMD_CONSTRUCT_BUSY = 1,
+	HALMAC_EFUSE_CMD_CONSTRUCT_H2C_SENT = 2,
+	HALMAC_EFUSE_CMD_CONSTRUCT_STATE_NUM = 3,
+	HALMAC_EFUSE_CMD_CONSTRUCT_UNDEFINED = 0x7F,
+};
+
+enum halmac_cfg_para_cmd_construct_state {
+	HALMAC_CFG_PARA_CMD_CONSTRUCT_IDLE = 0,
+	HALMAC_CFG_PARA_CMD_CONSTRUCT_CONSTRUCTING = 1,
+	HALMAC_CFG_PARA_CMD_CONSTRUCT_H2C_SENT = 2,
+	HALMAC_CFG_PARA_CMD_CONSTRUCT_NUM = 3,
+	HALMAC_CFG_PARA_CMD_CONSTRUCT_UNDEFINED = 0x7F,
+};
+
+enum halmac_scan_cmd_construct_state {
+	HALMAC_SCAN_CMD_CONSTRUCT_IDLE = 0,
+	HALMAC_SCAN_CMD_CONSTRUCT_BUFFER_CLEARED = 1,
+	HALMAC_SCAN_CMD_CONSTRUCT_CONSTRUCTING = 2,
+	HALMAC_SCAN_CMD_CONSTRUCT_H2C_SENT = 3,
+	HALMAC_SCAN_CMD_CONSTRUCT_STATE_NUM = 4,
+	HALMAC_SCAN_CMD_CONSTRUCT_UNDEFINED = 0x7F,
+};
+
+enum halmac_api_state {
+	HALMAC_API_STATE_INIT = 0,
+	HALMAC_API_STATE_HALT = 1,
+	HALMAC_API_STATE_UNDEFINED = 0x7F,
+};
+
+struct halmac_efuse_state_set {
+	enum halmac_efuse_cmd_construct_state efuse_cmd_construct_state;
+	enum halmac_cmd_process_status process_status;
+	u8 fw_return_code;
+	u16 seq_num;
+};
+
+struct halmac_cfg_para_state_set {
+	enum halmac_cfg_para_cmd_construct_state cfg_para_cmd_construct_state;
+	enum halmac_cmd_process_status process_status;
+	u8 fw_return_code;
+	u16 seq_num;
+};
+
+struct halmac_scan_state_set {
+	enum halmac_scan_cmd_construct_state scan_cmd_construct_state;
+	enum halmac_cmd_process_status process_status;
+	u8 fw_return_code;
+	u16 seq_num;
+};
+
+struct halmac_update_packet_state_set {
+	enum halmac_cmd_process_status process_status;
+	u8 fw_return_code;
+	u16 seq_num;
+};
+
+struct halmac_iqk_state_set {
+	enum halmac_cmd_process_status process_status;
+	u8 fw_return_code;
+	u16 seq_num;
+};
+
+struct halmac_power_tracking_state_set {
+	enum halmac_cmd_process_status process_status;
+	u8 fw_return_code;
+	u16 seq_num;
+};
+
+struct halmac_psd_state_set {
+	enum halmac_cmd_process_status process_status;
+	u16 data_size;
+	u16 segment_size;
+	u8 *data;
+	u8 fw_return_code;
+	u16 seq_num;
+};
+
+struct halmac_state {
+	struct halmac_efuse_state_set
+		efuse_state_set; /* State machine + cmd process status */
+	struct halmac_cfg_para_state_set
+		cfg_para_state_set; /* State machine + cmd process status */
+	struct halmac_scan_state_set
+		scan_state_set; /* State machine + cmd process status */
+	struct halmac_update_packet_state_set
+		update_packet_set; /* cmd process status */
+	struct halmac_iqk_state_set iqk_set; /* cmd process status */
+	struct halmac_power_tracking_state_set
+		power_tracking_set; /* cmd process status */
+	struct halmac_psd_state_set psd_set; /* cmd process status */
+	enum halmac_api_state api_state; /* Halmac api state */
+	enum halmac_mac_power mac_power; /* 0 : power off, 1 : power on*/
+	enum halmac_ps_state ps_state; /* power saving state */
+	enum halmac_dlfw_state dlfw_state; /* download FW state */
+};
+
+struct halmac_ver {
+	u8 major_ver;
+	u8 prototype_ver;
+	u8 minor_ver;
+};
+
+enum halmac_api_id {
+	/*stuff, need to be the 1st*/
+	HALMAC_API_STUFF = 0x0,
+	/*stuff, need to be the 1st*/
+	HALMAC_API_MAC_POWER_SWITCH = 0x1,
+	HALMAC_API_DOWNLOAD_FIRMWARE = 0x2,
+	HALMAC_API_CFG_MAC_ADDR = 0x3,
+	HALMAC_API_CFG_BSSID = 0x4,
+	HALMAC_API_CFG_MULTICAST_ADDR = 0x5,
+	HALMAC_API_PRE_INIT_SYSTEM_CFG = 0x6,
+	HALMAC_API_INIT_SYSTEM_CFG = 0x7,
+	HALMAC_API_INIT_TRX_CFG = 0x8,
+	HALMAC_API_CFG_RX_AGGREGATION = 0x9,
+	HALMAC_API_INIT_PROTOCOL_CFG = 0xA,
+	HALMAC_API_INIT_EDCA_CFG = 0xB,
+	HALMAC_API_CFG_OPERATION_MODE = 0xC,
+	HALMAC_API_CFG_CH_BW = 0xD,
+	HALMAC_API_CFG_BW = 0xE,
+	HALMAC_API_INIT_WMAC_CFG = 0xF,
+	HALMAC_API_INIT_MAC_CFG = 0x10,
+	HALMAC_API_INIT_SDIO_CFG = 0x11,
+	HALMAC_API_INIT_USB_CFG = 0x12,
+	HALMAC_API_INIT_PCIE_CFG = 0x13,
+	HALMAC_API_INIT_INTERFACE_CFG = 0x14,
+	HALMAC_API_DEINIT_SDIO_CFG = 0x15,
+	HALMAC_API_DEINIT_USB_CFG = 0x16,
+	HALMAC_API_DEINIT_PCIE_CFG = 0x17,
+	HALMAC_API_DEINIT_INTERFACE_CFG = 0x18,
+	HALMAC_API_GET_EFUSE_SIZE = 0x19,
+	HALMAC_API_DUMP_EFUSE_MAP = 0x1A,
+	HALMAC_API_WRITE_EFUSE = 0x1B,
+	HALMAC_API_READ_EFUSE = 0x1C,
+	HALMAC_API_GET_LOGICAL_EFUSE_SIZE = 0x1D,
+	HALMAC_API_DUMP_LOGICAL_EFUSE_MAP = 0x1E,
+	HALMAC_API_WRITE_LOGICAL_EFUSE = 0x1F,
+	HALMAC_API_READ_LOGICAL_EFUSE = 0x20,
+	HALMAC_API_PG_EFUSE_BY_MAP = 0x21,
+	HALMAC_API_GET_C2H_INFO = 0x22,
+	HALMAC_API_CFG_FWLPS_OPTION = 0x23,
+	HALMAC_API_CFG_FWIPS_OPTION = 0x24,
+	HALMAC_API_ENTER_WOWLAN = 0x25,
+	HALMAC_API_LEAVE_WOWLAN = 0x26,
+	HALMAC_API_ENTER_PS = 0x27,
+	HALMAC_API_LEAVE_PS = 0x28,
+	HALMAC_API_H2C_LB = 0x29,
+	HALMAC_API_DEBUG = 0x2A,
+	HALMAC_API_CFG_PARAMETER = 0x2B,
+	HALMAC_API_UPDATE_PACKET = 0x2C,
+	HALMAC_API_BCN_IE_FILTER = 0x2D,
+	HALMAC_API_REG_READ_8 = 0x2E,
+	HALMAC_API_REG_WRITE_8 = 0x2F,
+	HALMAC_API_REG_READ_16 = 0x30,
+	HALMAC_API_REG_WRITE_16 = 0x31,
+	HALMAC_API_REG_READ_32 = 0x32,
+	HALMAC_API_REG_WRITE_32 = 0x33,
+	HALMAC_API_TX_ALLOWED_SDIO = 0x34,
+	HALMAC_API_SET_BULKOUT_NUM = 0x35,
+	HALMAC_API_GET_SDIO_TX_ADDR = 0x36,
+	HALMAC_API_GET_USB_BULKOUT_ID = 0x37,
+	HALMAC_API_TIMER_2S = 0x38,
+	HALMAC_API_FILL_TXDESC_CHECKSUM = 0x39,
+	HALMAC_API_SEND_ORIGINAL_H2C = 0x3A,
+	HALMAC_API_UPDATE_DATAPACK = 0x3B,
+	HALMAC_API_RUN_DATAPACK = 0x3C,
+	HALMAC_API_CFG_DRV_INFO = 0x3D,
+	HALMAC_API_SEND_BT_COEX = 0x3E,
+	HALMAC_API_VERIFY_PLATFORM_API = 0x3F,
+	HALMAC_API_GET_FIFO_SIZE = 0x40,
+	HALMAC_API_DUMP_FIFO = 0x41,
+	HALMAC_API_CFG_TXBF = 0x42,
+	HALMAC_API_CFG_MUMIMO = 0x43,
+	HALMAC_API_CFG_SOUNDING = 0x44,
+	HALMAC_API_DEL_SOUNDING = 0x45,
+	HALMAC_API_SU_BFER_ENTRY_INIT = 0x46,
+	HALMAC_API_SU_BFEE_ENTRY_INIT = 0x47,
+	HALMAC_API_MU_BFER_ENTRY_INIT = 0x48,
+	HALMAC_API_MU_BFEE_ENTRY_INIT = 0x49,
+	HALMAC_API_SU_BFER_ENTRY_DEL = 0x4A,
+	HALMAC_API_SU_BFEE_ENTRY_DEL = 0x4B,
+	HALMAC_API_MU_BFER_ENTRY_DEL = 0x4C,
+	HALMAC_API_MU_BFEE_ENTRY_DEL = 0x4D,
+
+	HALMAC_API_ADD_CH_INFO = 0x4E,
+	HALMAC_API_ADD_EXTRA_CH_INFO = 0x4F,
+	HALMAC_API_CTRL_CH_SWITCH = 0x50,
+	HALMAC_API_CLEAR_CH_INFO = 0x51,
+
+	HALMAC_API_SEND_GENERAL_INFO = 0x52,
+	HALMAC_API_START_IQK = 0x53,
+	HALMAC_API_CTRL_PWR_TRACKING = 0x54,
+	HALMAC_API_PSD = 0x55,
+	HALMAC_API_CFG_TX_AGG_ALIGN = 0x56,
+
+	HALMAC_API_QUERY_STATE = 0x57,
+	HALMAC_API_RESET_FEATURE = 0x58,
+	HALMAC_API_CHECK_FW_STATUS = 0x59,
+	HALMAC_API_DUMP_FW_DMEM = 0x5A,
+	HALMAC_API_CFG_MAX_DL_SIZE = 0x5B,
+
+	HALMAC_API_INIT_OBJ = 0x5C,
+	HALMAC_API_DEINIT_OBJ = 0x5D,
+	HALMAC_API_CFG_LA_MODE = 0x5E,
+	HALMAC_API_GET_HW_VALUE = 0x5F,
+	HALMAC_API_SET_HW_VALUE = 0x60,
+	HALMAC_API_CFG_DRV_RSVD_PG_NUM = 0x61,
+	HALMAC_API_SWITCH_EFUSE_BANK = 0x62,
+	HALMAC_API_WRITE_EFUSE_BT = 0x63,
+	HALMAC_API_DUMP_EFUSE_MAP_BT = 0x64,
+	HALMAC_API_DL_DRV_RSVD_PG = 0x65,
+	HALMAC_API_PCIE_SWITCH = 0x66,
+	HALMAC_API_PHY_CFG = 0x67,
+	HALMAC_API_CFG_RX_FIFO_EXPANDING_MODE = 0x68,
+	HALMAC_API_CFG_CSI_RATE = 0x69,
+	HALMAC_API_MAX
+};
+
+struct halmac_api_record {
+	enum halmac_api_id api_array[API_ARRAY_SIZE];
+	u8 array_wptr;
+};
+
+enum halmac_la_mode {
+	HALMAC_LA_MODE_DISABLE = 0,
+	HALMAC_LA_MODE_PARTIAL = 1,
+	HALMAC_LA_MODE_FULL = 2,
+	HALMAC_LA_MODE_UNDEFINE = 0x7F,
+};
+
+enum halmac_rx_fifo_expanding_mode {
+	HALMAC_RX_FIFO_EXPANDING_MODE_DISABLE = 0,
+	HALMAC_RX_FIFO_EXPANDING_MODE_1_BLOCK = 1,
+	HALMAC_RX_FIFO_EXPANDING_MODE_2_BLOCK = 2,
+	HALMAC_RX_FIFO_EXPANDING_MODE_3_BLOCK = 3,
+	HALMAC_RX_FIFO_EXPANDING_MODE_UNDEFINE = 0x7F,
+};
+
+enum halmac_sdio_cmd53_4byte_mode {
+	HALMAC_SDIO_CMD53_4BYTE_MODE_DISABLE = 0,
+	HALMAC_SDIO_CMD53_4BYTE_MODE_RW = 1,
+	HALMAC_SDIO_CMD53_4BYTE_MODE_R = 2,
+	HALMAC_SDIO_CMD53_4BYTE_MODE_W = 3,
+	HALMAC_SDIO_CMD53_4BYTE_MODE_UNDEFINE = 0x7F,
+};
+
+enum halmac_usb_mode {
+	HALMAC_USB_MODE_U2 = 1,
+	HALMAC_USB_MODE_U3 = 2,
+};
+
+enum halmac_hw_id {
+	/* Get HW value */
+	HALMAC_HW_RQPN_MAPPING = 0x00,
+	HALMAC_HW_EFUSE_SIZE = 0x01,
+	HALMAC_HW_EEPROM_SIZE = 0x02,
+	HALMAC_HW_BT_BANK_EFUSE_SIZE = 0x03,
+	HALMAC_HW_BT_BANK1_EFUSE_SIZE = 0x04,
+	HALMAC_HW_BT_BANK2_EFUSE_SIZE = 0x05,
+	HALMAC_HW_TXFIFO_SIZE = 0x06,
+	HALMAC_HW_RSVD_PG_BNDY = 0x07,
+	HALMAC_HW_CAM_ENTRY_NUM = 0x08,
+	HALMAC_HW_IC_VERSION = 0x09,
+	HALMAC_HW_PAGE_SIZE = 0x0A,
+	HALMAC_HW_TX_AGG_ALIGN_SIZE = 0x0B,
+	HALMAC_HW_RX_AGG_ALIGN_SIZE = 0x0C,
+	HALMAC_HW_DRV_INFO_SIZE = 0x0D,
+	HALMAC_HW_TXFF_ALLOCATION = 0x0E,
+	HALMAC_HW_RSVD_EFUSE_SIZE = 0x0F,
+	HALMAC_HW_FW_HDR_SIZE = 0x10,
+	HALMAC_HW_TX_DESC_SIZE = 0x11,
+	HALMAC_HW_RX_DESC_SIZE = 0x12,
+	HALMAC_HW_WLAN_EFUSE_AVAILABLE_SIZE = 0x13,
+	/* Set HW value */
+	HALMAC_HW_USB_MODE = 0x60,
+	HALMAC_HW_SEQ_EN = 0x61,
+	HALMAC_HW_BANDWIDTH = 0x62,
+	HALMAC_HW_CHANNEL = 0x63,
+	HALMAC_HW_PRI_CHANNEL_IDX = 0x64,
+	HALMAC_HW_EN_BB_RF = 0x65,
+	HALMAC_HW_SDIO_TX_PAGE_THRESHOLD = 0x66,
+	HALMAC_HW_AMPDU_CONFIG = 0x67,
+
+	HALMAC_HW_ID_UNDEFINE = 0x7F,
+};
+
+enum halmac_efuse_bank {
+	HALMAC_EFUSE_BANK_WIFI = 0,
+	HALMAC_EFUSE_BANK_BT = 1,
+	HALMAC_EFUSE_BANK_BT_1 = 2,
+	HALMAC_EFUSE_BANK_BT_2 = 3,
+	HALMAC_EFUSE_BANK_MAX,
+	HALMAC_EFUSE_BANK_UNDEFINE = 0X7F,
+};
+
+struct halmac_txff_allocation {
+	u16 tx_fifo_pg_num;
+	u16 rsvd_pg_num;
+	u16 rsvd_drv_pg_num;
+	u16 ac_q_pg_num;
+	u16 high_queue_pg_num;
+	u16 low_queue_pg_num;
+	u16 normal_queue_pg_num;
+	u16 extra_queue_pg_num;
+	u16 pub_queue_pg_num;
+	u16 rsvd_pg_bndy;
+	u16 rsvd_drv_pg_bndy;
+	u16 rsvd_h2c_extra_info_pg_bndy;
+	u16 rsvd_h2c_queue_pg_bndy;
+	u16 rsvd_cpu_instr_pg_bndy;
+	u16 rsvd_fw_txbuff_pg_bndy;
+	enum halmac_la_mode la_mode;
+	enum halmac_rx_fifo_expanding_mode rx_fifo_expanding_mode;
+};
+
+struct halmac_rqpn_map {
+	enum halmac_dma_mapping dma_map_vo;
+	enum halmac_dma_mapping dma_map_vi;
+	enum halmac_dma_mapping dma_map_be;
+	enum halmac_dma_mapping dma_map_bk;
+	enum halmac_dma_mapping dma_map_mg;
+	enum halmac_dma_mapping dma_map_hi;
+};
+
+struct halmac_security_setting {
+	u8 tx_encryption;
+	u8 rx_decryption;
+	u8 bip_enable;
+};
+
+struct halmac_cam_entry_info {
+	enum hal_security_type security_type;
+	u32 key[4];
+	u32 key_ext[4];
+	u8 mac_address[6];
+	u8 unicast;
+	u8 key_id;
+	u8 valid;
+};
+
+struct halmac_cam_entry_format {
+	u16 key_id : 2;
+	u16 type : 3;
+	u16 mic : 1;
+	u16 grp : 1;
+	u16 spp_mode : 1;
+	u16 rpt_md : 1;
+	u16 ext_sectype : 1;
+	u16 mgnt : 1;
+	u16 rsvd1 : 4;
+	u16 valid : 1;
+	u8 mac_address[6];
+	u32 key[4];
+	u32 rsvd[2];
+};
+
+struct halmac_tx_page_threshold_info {
+	u32 threshold;
+	enum halmac_dma_mapping dma_queue_sel;
+};
+
+struct halmac_ampdu_config {
+	u8 max_agg_num;
+};
+
+struct halmac_port_cfg {
+	u8 port0_sync_tsf;
+	u8 port1_sync_tsf;
+};
+
+struct halmac_rqpn_ {
+	enum halmac_trx_mode mode;
+	enum halmac_dma_mapping dma_map_vo;
+	enum halmac_dma_mapping dma_map_vi;
+	enum halmac_dma_mapping dma_map_be;
+	enum halmac_dma_mapping dma_map_bk;
+	enum halmac_dma_mapping dma_map_mg;
+	enum halmac_dma_mapping dma_map_hi;
+};
+
+struct halmac_pg_num_ {
+	enum halmac_trx_mode mode;
+	u16 hq_num;
+	u16 nq_num;
+	u16 lq_num;
+	u16 exq_num;
+	u16 gap_num; /*used for loopback mode*/
+};
+
+struct halmac_intf_phy_para_ {
+	u16 offset;
+	u16 value;
+	u16 ip_sel;
+	u16 cut;
+	u16 plaform;
+};
+
+struct halmac_iqk_para_ {
+	u8 clear;
+	u8 segment_iqk;
+};
+
+/* Hal mac adapter */
+struct halmac_adapter {
+	/* Dma mapping of protocol queues */
+	enum halmac_dma_mapping halmac_ptcl_queue[HALMAC_PTCL_QUEUE_NUM];
+	/* low power state option */
+	struct halmac_fwlps_option fwlps_option;
+	/* mac address information, suppot 2 ports */
+	union halmac_wlan_addr hal_mac_addr[HALMAC_PORTIDMAX];
+	/* bss address information, suppot 2 ports */
+	union halmac_wlan_addr hal_bss_addr[HALMAC_PORTIDMAX];
+	/* Protect h2c_packet_seq packet*/
+	spinlock_t h2c_seq_lock;
+	/* Protect Efuse map memory of halmac_adapter */
+	spinlock_t efuse_lock;
+	struct halmac_config_para_info config_para_info;
+	struct halmac_cs_info ch_sw_info;
+	struct halmac_event_trigger event_trigger;
+	/* HW related information */
+	struct halmac_hw_config_info hw_config_info;
+	struct halmac_sdio_free_space sdio_free_space;
+	struct halmac_snd_info snd_info;
+	/* Backup HalAdapter address */
+	void *hal_adapter_backup;
+	/* Driver or FW adapter address. Do not write this memory*/
+	void *driver_adapter;
+	u8 *hal_efuse_map;
+	/* Record function pointer of halmac api */
+	void *halmac_api;
+	/* Record function pointer of platform api */
+	struct halmac_platform_api *halmac_platform_api;
+	/* Record efuse used memory */
+	u32 efuse_end;
+	u32 h2c_buf_free_space;
+	u32 h2c_buff_size;
+	u32 max_download_size;
+	/* Chip ID, 8822B, 8821C... */
+	enum halmac_chip_id chip_id;
+	/* A cut, B cut... */
+	enum halmac_chip_ver chip_version;
+	struct halmac_fw_version fw_version;
+	struct halmac_state halmac_state;
+	/* Interface information, get from driver */
+	enum halmac_interface halmac_interface;
+	/* Noraml, WMM, P2P, LoopBack... */
+	enum halmac_trx_mode trx_mode;
+	struct halmac_txff_allocation txff_allocation;
+	u8 h2c_packet_seq; /* current h2c packet sequence number */
+	u16 ack_h2c_packet_seq; /*the acked h2c packet sequence number */
+	bool hal_efuse_map_valid;
+	u8 efuse_segment_size;
+	u8 rpwm_record; /* record rpwm value */
+	bool low_clk; /*LPS 32K or IPS 32K*/
+	u8 halmac_bulkout_num; /* USB bulkout num */
+	struct halmac_api_record api_record; /* API record */
+	bool gen_info_valid;
+	struct halmac_general_info general_info;
+	u8 drv_info_size;
+	enum halmac_sdio_cmd53_4byte_mode sdio_cmd53_4byte;
+};
+
+/* Function pointer of  Hal mac API */
+struct halmac_api {
+	enum halmac_ret_status (*halmac_mac_power_switch)(
+		struct halmac_adapter *halmac_adapter,
+		enum halmac_mac_power halmac_power);
+	enum halmac_ret_status (*halmac_download_firmware)(
+		struct halmac_adapter *halmac_adapter, u8 *hamacl_fw,
+		u32 halmac_fw_size);
+	enum halmac_ret_status (*halmac_free_download_firmware)(
+		struct halmac_adapter *halmac_adapter,
+		enum halmac_dlfw_mem dlfw_mem, u8 *hamacl_fw,
+		u32 halmac_fw_size);
+	enum halmac_ret_status (*halmac_get_fw_version)(
+		struct halmac_adapter *halmac_adapter,
+		struct halmac_fw_version *fw_version);
+	enum halmac_ret_status (*halmac_cfg_mac_addr)(
+		struct halmac_adapter *halmac_adapter, u8 halmac_port,
+		union halmac_wlan_addr *hal_address);
+	enum halmac_ret_status (*halmac_cfg_bssid)(
+		struct halmac_adapter *halmac_adapter, u8 halmac_port,
+		union halmac_wlan_addr *hal_address);
+	enum halmac_ret_status (*halmac_cfg_multicast_addr)(
+		struct halmac_adapter *halmac_adapter,
+		union halmac_wlan_addr *hal_address);
+	enum halmac_ret_status (*halmac_pre_init_system_cfg)(
+		struct halmac_adapter *halmac_adapter);
+	enum halmac_ret_status (*halmac_init_system_cfg)(
+		struct halmac_adapter *halmac_adapter);
+	enum halmac_ret_status (*halmac_init_trx_cfg)(
+		struct halmac_adapter *halmac_adapter,
+		enum halmac_trx_mode mode);
+	enum halmac_ret_status (*halmac_init_h2c)(
+		struct halmac_adapter *halmac_adapter);
+	enum halmac_ret_status (*halmac_cfg_rx_aggregation)(
+		struct halmac_adapter *halmac_adapter,
+		struct halmac_rxagg_cfg *phalmac_rxagg_cfg);
+	enum halmac_ret_status (*halmac_init_protocol_cfg)(
+		struct halmac_adapter *halmac_adapter);
+	enum halmac_ret_status (*halmac_init_edca_cfg)(
+		struct halmac_adapter *halmac_adapter);
+	enum halmac_ret_status (*halmac_cfg_operation_mode)(
+		struct halmac_adapter *halmac_adapter,
+		enum halmac_wireless_mode wireless_mode);
+	enum halmac_ret_status (*halmac_cfg_ch_bw)(
+		struct halmac_adapter *halmac_adapter, u8 channel,
+		enum halmac_pri_ch_idx pri_ch_idx, enum halmac_bw bw);
+	enum halmac_ret_status (*halmac_cfg_bw)(
+		struct halmac_adapter *halmac_adapter, enum halmac_bw bw);
+	enum halmac_ret_status (*halmac_init_wmac_cfg)(
+		struct halmac_adapter *halmac_adapter);
+	enum halmac_ret_status (*halmac_init_mac_cfg)(
+		struct halmac_adapter *halmac_adapter,
+		enum halmac_trx_mode mode);
+	enum halmac_ret_status (*halmac_init_sdio_cfg)(
+		struct halmac_adapter *halmac_adapter);
+	enum halmac_ret_status (*halmac_init_usb_cfg)(
+		struct halmac_adapter *halmac_adapter);
+	enum halmac_ret_status (*halmac_init_pcie_cfg)(
+		struct halmac_adapter *halmac_adapter);
+	enum halmac_ret_status (*halmac_init_interface_cfg)(
+		struct halmac_adapter *halmac_adapter);
+	enum halmac_ret_status (*halmac_deinit_sdio_cfg)(
+		struct halmac_adapter *halmac_adapter);
+	enum halmac_ret_status (*halmac_deinit_usb_cfg)(
+		struct halmac_adapter *halmac_adapter);
+	enum halmac_ret_status (*halmac_deinit_pcie_cfg)(
+		struct halmac_adapter *halmac_adapter);
+	enum halmac_ret_status (*halmac_deinit_interface_cfg)(
+		struct halmac_adapter *halmac_adapter);
+	enum halmac_ret_status (*halmac_get_efuse_size)(
+		struct halmac_adapter *halmac_adapter, u32 *halmac_size);
+	enum halmac_ret_status (*halmac_get_efuse_available_size)(
+		struct halmac_adapter *halmac_adapter, u32 *halmac_size);
+	enum halmac_ret_status (*halmac_dump_efuse_map)(
+		struct halmac_adapter *halmac_adapter,
+		enum halmac_efuse_read_cfg cfg);
+	enum halmac_ret_status (*halmac_dump_efuse_map_bt)(
+		struct halmac_adapter *halmac_adapter,
+		enum halmac_efuse_bank halmac_efues_bank, u32 bt_efuse_map_size,
+		u8 *bt_efuse_map);
+	enum halmac_ret_status (*halmac_write_efuse)(
+		struct halmac_adapter *halmac_adapter, u32 halmac_offset,
+		u8 halmac_value);
+	enum halmac_ret_status (*halmac_read_efuse)(
+		struct halmac_adapter *halmac_adapter, u32 halmac_offset,
+		u8 *value);
+	enum halmac_ret_status (*halmac_switch_efuse_bank)(
+		struct halmac_adapter *halmac_adapter,
+		enum halmac_efuse_bank halmac_efues_bank);
+	enum halmac_ret_status (*halmac_write_efuse_bt)(
+		struct halmac_adapter *halmac_adapter, u32 halmac_offset,
+		u8 halmac_value, enum halmac_efuse_bank halmac_efues_bank);
+	enum halmac_ret_status (*halmac_get_logical_efuse_size)(
+		struct halmac_adapter *halmac_adapter, u32 *halmac_size);
+	enum halmac_ret_status (*halmac_dump_logical_efuse_map)(
+		struct halmac_adapter *halmac_adapter,
+		enum halmac_efuse_read_cfg cfg);
+	enum halmac_ret_status (*halmac_write_logical_efuse)(
+		struct halmac_adapter *halmac_adapter, u32 halmac_offset,
+		u8 halmac_value);
+	enum halmac_ret_status (*halmac_read_logical_efuse)(
+		struct halmac_adapter *halmac_adapter, u32 halmac_offset,
+		u8 *value);
+	enum halmac_ret_status (*halmac_pg_efuse_by_map)(
+		struct halmac_adapter *halmac_adapter,
+		struct halmac_pg_efuse_info *pg_efuse_info,
+		enum halmac_efuse_read_cfg cfg);
+	enum halmac_ret_status (*halmac_get_c2h_info)(
+		struct halmac_adapter *halmac_adapter, u8 *halmac_buf,
+		u32 halmac_size);
+	enum halmac_ret_status (*halmac_cfg_fwlps_option)(
+		struct halmac_adapter *halmac_adapter,
+		struct halmac_fwlps_option *lps_option);
+	enum halmac_ret_status (*halmac_cfg_fwips_option)(
+		struct halmac_adapter *halmac_adapter,
+		struct halmac_fwips_option *ips_option);
+	enum halmac_ret_status (*halmac_enter_wowlan)(
+		struct halmac_adapter *halmac_adapter,
+		struct halmac_wowlan_option *wowlan_option);
+	enum halmac_ret_status (*halmac_leave_wowlan)(
+		struct halmac_adapter *halmac_adapter);
+	enum halmac_ret_status (*halmac_enter_ps)(
+		struct halmac_adapter *halmac_adapter,
+		enum halmac_ps_state ps_state);
+	enum halmac_ret_status (*halmac_leave_ps)(
+		struct halmac_adapter *halmac_adapter);
+	enum halmac_ret_status (*halmac_h2c_lb)(
+		struct halmac_adapter *halmac_adapter);
+	enum halmac_ret_status (*halmac_debug)(
+		struct halmac_adapter *halmac_adapter);
+	enum halmac_ret_status (*halmac_cfg_parameter)(
+		struct halmac_adapter *halmac_adapter,
+		struct halmac_phy_parameter_info *para_info, u8 full_fifo);
+	enum halmac_ret_status (*halmac_update_packet)(
+		struct halmac_adapter *halmac_adapter,
+		enum halmac_packet_id pkt_id, u8 *pkt, u32 pkt_size);
+	enum halmac_ret_status (*halmac_bcn_ie_filter)(
+		struct halmac_adapter *halmac_adapter,
+		struct halmac_bcn_ie_info *bcn_ie_info);
+	u8 (*halmac_reg_read_8)(struct halmac_adapter *halmac_adapter,
+				u32 halmac_offset);
+	enum halmac_ret_status (*halmac_reg_write_8)(
+		struct halmac_adapter *halmac_adapter, u32 halmac_offset,
+		u8 halmac_data);
+	u16 (*halmac_reg_read_16)(struct halmac_adapter *halmac_adapter,
+				  u32 halmac_offset);
+	enum halmac_ret_status (*halmac_reg_write_16)(
+		struct halmac_adapter *halmac_adapter, u32 halmac_offset,
+		u16 halmac_data);
+	u32 (*halmac_reg_read_32)(struct halmac_adapter *halmac_adapter,
+				  u32 halmac_offset);
+	u32 (*halmac_reg_read_indirect_32)(
+		struct halmac_adapter *halmac_adapter, u32 halmac_offset);
+	u8 (*halmac_reg_sdio_cmd53_read_n)(
+		struct halmac_adapter *halmac_adapter, u32 halmac_offset,
+		u32 halmac_size, u8 *halmac_data);
+	enum halmac_ret_status (*halmac_reg_write_32)(
+		struct halmac_adapter *halmac_adapter, u32 halmac_offset,
+		u32 halmac_data);
+	enum halmac_ret_status (*halmac_tx_allowed_sdio)(
+		struct halmac_adapter *halmac_adapter, u8 *halmac_buf,
+		u32 halmac_size);
+	enum halmac_ret_status (*halmac_set_bulkout_num)(
+		struct halmac_adapter *halmac_adapter, u8 bulkout_num);
+	enum halmac_ret_status (*halmac_get_sdio_tx_addr)(
+		struct halmac_adapter *halmac_adapter, u8 *halmac_buf,
+		u32 halmac_size, u32 *pcmd53_addr);
+	enum halmac_ret_status (*halmac_get_usb_bulkout_id)(
+		struct halmac_adapter *halmac_adapter, u8 *halmac_buf,
+		u32 halmac_size, u8 *bulkout_id);
+	enum halmac_ret_status (*halmac_timer_2s)(
+		struct halmac_adapter *halmac_adapter);
+	enum halmac_ret_status (*halmac_fill_txdesc_checksum)(
+		struct halmac_adapter *halmac_adapter, u8 *cur_desc);
+	enum halmac_ret_status (*halmac_update_datapack)(
+		struct halmac_adapter *halmac_adapter,
+		enum halmac_data_type halmac_data_type,
+		struct halmac_phy_parameter_info *para_info);
+	enum halmac_ret_status (*halmac_run_datapack)(
+		struct halmac_adapter *halmac_adapter,
+		enum halmac_data_type halmac_data_type);
+	enum halmac_ret_status (*halmac_cfg_drv_info)(
+		struct halmac_adapter *halmac_adapter,
+		enum halmac_drv_info halmac_drv_info);
+	enum halmac_ret_status (*halmac_send_bt_coex)(
+		struct halmac_adapter *halmac_adapter, u8 *bt_buf, u32 bt_size,
+		u8 ack);
+	enum halmac_ret_status (*halmac_verify_platform_api)(
+		struct halmac_adapter *halmac_adapte);
+	u32 (*halmac_get_fifo_size)(struct halmac_adapter *halmac_adapter,
+				    enum hal_fifo_sel halmac_fifo_sel);
+	enum halmac_ret_status (*halmac_dump_fifo)(
+		struct halmac_adapter *halmac_adapter,
+		enum hal_fifo_sel halmac_fifo_sel, u32 halmac_start_addr,
+		u32 halmac_fifo_dump_size, u8 *fifo_map);
+	enum halmac_ret_status (*halmac_cfg_txbf)(
+		struct halmac_adapter *halmac_adapter, u8 userid,
+		enum halmac_bw bw, u8 txbf_en);
+	enum halmac_ret_status (*halmac_cfg_mumimo)(
+		struct halmac_adapter *halmac_adapter,
+		struct halmac_cfg_mumimo_para *cfgmu);
+	enum halmac_ret_status (*halmac_cfg_sounding)(
+		struct halmac_adapter *halmac_adapter,
+		enum halmac_snd_role role, enum halmac_data_rate datarate);
+	enum halmac_ret_status (*halmac_del_sounding)(
+		struct halmac_adapter *halmac_adapter,
+		enum halmac_snd_role role);
+	enum halmac_ret_status (*halmac_su_bfer_entry_init)(
+		struct halmac_adapter *halmac_adapter,
+		struct halmac_su_bfer_init_para *su_bfer_init);
+	enum halmac_ret_status (*halmac_su_bfee_entry_init)(
+		struct halmac_adapter *halmac_adapter, u8 userid, u16 paid);
+	enum halmac_ret_status (*halmac_mu_bfer_entry_init)(
+		struct halmac_adapter *halmac_adapter,
+		struct halmac_mu_bfer_init_para *mu_bfer_init);
+	enum halmac_ret_status (*halmac_mu_bfee_entry_init)(
+		struct halmac_adapter *halmac_adapter,
+		struct halmac_mu_bfee_init_para *mu_bfee_init);
+	enum halmac_ret_status (*halmac_su_bfer_entry_del)(
+		struct halmac_adapter *halmac_adapter, u8 userid);
+	enum halmac_ret_status (*halmac_su_bfee_entry_del)(
+		struct halmac_adapter *halmac_adapter, u8 userid);
+	enum halmac_ret_status (*halmac_mu_bfer_entry_del)(
+		struct halmac_adapter *halmac_adapter);
+	enum halmac_ret_status (*halmac_mu_bfee_entry_del)(
+		struct halmac_adapter *halmac_adapter, u8 userid);
+	enum halmac_ret_status (*halmac_add_ch_info)(
+		struct halmac_adapter *halmac_adapter,
+		struct halmac_ch_info *ch_info);
+	enum halmac_ret_status (*halmac_add_extra_ch_info)(
+		struct halmac_adapter *halmac_adapter,
+		struct halmac_ch_extra_info *ch_extra_info);
+	enum halmac_ret_status (*halmac_ctrl_ch_switch)(
+		struct halmac_adapter *halmac_adapter,
+		struct halmac_ch_switch_option *cs_option);
+	enum halmac_ret_status (*halmac_p2pps)(
+		struct halmac_adapter *halmac_adapter,
+		struct halmac_p2pps *p2p_ps);
+	enum halmac_ret_status (*halmac_clear_ch_info)(
+		struct halmac_adapter *halmac_adapter);
+	enum halmac_ret_status (*halmac_send_general_info)(
+		struct halmac_adapter *halmac_adapter,
+		struct halmac_general_info *pg_general_info);
+	enum halmac_ret_status (*halmac_start_iqk)(
+		struct halmac_adapter *halmac_adapter,
+		struct halmac_iqk_para_ *iqk_para);
+	enum halmac_ret_status (*halmac_ctrl_pwr_tracking)(
+		struct halmac_adapter *halmac_adapter,
+		struct halmac_pwr_tracking_option *pwr_tracking_opt);
+	enum halmac_ret_status (*halmac_psd)(
+		struct halmac_adapter *halmac_adapter, u16 start_psd,
+		u16 end_psd);
+	enum halmac_ret_status (*halmac_cfg_tx_agg_align)(
+		struct halmac_adapter *halmac_adapter, u8 enable,
+		u16 align_size);
+	enum halmac_ret_status (*halmac_query_status)(
+		struct halmac_adapter *halmac_adapter,
+		enum halmac_feature_id feature_id,
+		enum halmac_cmd_process_status *process_status, u8 *data,
+		u32 *size);
+	enum halmac_ret_status (*halmac_reset_feature)(
+		struct halmac_adapter *halmac_adapter,
+		enum halmac_feature_id feature_id);
+	enum halmac_ret_status (*halmac_check_fw_status)(
+		struct halmac_adapter *halmac_adapter, bool *fw_status);
+	enum halmac_ret_status (*halmac_dump_fw_dmem)(
+		struct halmac_adapter *halmac_adapter, u8 *dmem, u32 *size);
+	enum halmac_ret_status (*halmac_cfg_max_dl_size)(
+		struct halmac_adapter *halmac_adapter, u32 size);
+	enum halmac_ret_status (*halmac_cfg_la_mode)(
+		struct halmac_adapter *halmac_adapter,
+		enum halmac_la_mode la_mode);
+	enum halmac_ret_status (*halmac_cfg_rx_fifo_expanding_mode)(
+		struct halmac_adapter *halmac_adapter,
+		enum halmac_rx_fifo_expanding_mode rx_fifo_expanding_mode);
+	enum halmac_ret_status (*halmac_config_security)(
+		struct halmac_adapter *halmac_adapter,
+		struct halmac_security_setting *sec_setting);
+	u8 (*halmac_get_used_cam_entry_num)(
+		struct halmac_adapter *halmac_adapter,
+		enum hal_security_type sec_type);
+	enum halmac_ret_status (*halmac_write_cam)(
+		struct halmac_adapter *halmac_adapter, u32 entry_index,
+		struct halmac_cam_entry_info *cam_entry_info);
+	enum halmac_ret_status (*halmac_read_cam_entry)(
+		struct halmac_adapter *halmac_adapter, u32 entry_index,
+		struct halmac_cam_entry_format *content);
+	enum halmac_ret_status (*halmac_clear_cam_entry)(
+		struct halmac_adapter *halmac_adapter, u32 entry_index);
+	enum halmac_ret_status (*halmac_get_hw_value)(
+		struct halmac_adapter *halmac_adapter, enum halmac_hw_id hw_id,
+		void *pvalue);
+	enum halmac_ret_status (*halmac_set_hw_value)(
+		struct halmac_adapter *halmac_adapter, enum halmac_hw_id hw_id,
+		void *pvalue);
+	enum halmac_ret_status (*halmac_cfg_drv_rsvd_pg_num)(
+		struct halmac_adapter *halmac_adapter,
+		enum halmac_drv_rsvd_pg_num pg_num);
+	enum halmac_ret_status (*halmac_get_chip_version)(
+		struct halmac_adapter *halmac_adapter,
+		struct halmac_ver *version);
+	enum halmac_ret_status (*halmac_chk_txdesc)(
+		struct halmac_adapter *halmac_adapter, u8 *halmac_buf,
+		u32 halmac_size);
+	enum halmac_ret_status (*halmac_dl_drv_rsvd_page)(
+		struct halmac_adapter *halmac_adapter, u8 pg_offset,
+		u8 *hal_buf, u32 size);
+	enum halmac_ret_status (*halmac_pcie_switch)(
+		struct halmac_adapter *halmac_adapter,
+		enum halmac_pcie_cfg pcie_cfg);
+	enum halmac_ret_status (*halmac_phy_cfg)(
+		struct halmac_adapter *halmac_adapter,
+		enum halmac_intf_phy_platform platform);
+	enum halmac_ret_status (*halmac_cfg_csi_rate)(
+		struct halmac_adapter *halmac_adapter, u8 rssi, u8 current_rate,
+		u8 fixrate_en, u8 *new_rate);
+	enum halmac_ret_status (*halmac_sdio_cmd53_4byte)(
+		struct halmac_adapter *halmac_adapter,
+		enum halmac_sdio_cmd53_4byte_mode cmd53_4byte_mode);
+	enum halmac_ret_status (*halmac_interface_integration_tuning)(
+		struct halmac_adapter *halmac_adapter);
+	enum halmac_ret_status (*halmac_txfifo_is_empty)(
+		struct halmac_adapter *halmac_adapter, u32 chk_num);
+};
+
+#define HALMAC_GET_API(phalmac_adapter)                                        \
+	((struct halmac_api *)phalmac_adapter->halmac_api)
+
+static inline enum halmac_ret_status
+halmac_adapter_validate(struct halmac_adapter *halmac_adapter)
+{
+	if ((!halmac_adapter) ||
+	    (halmac_adapter->hal_adapter_backup != halmac_adapter))
+		return HALMAC_RET_ADAPTER_INVALID;
+
+	return HALMAC_RET_SUCCESS;
+}
+
+static inline enum halmac_ret_status
+halmac_api_validate(struct halmac_adapter *halmac_adapter)
+{
+	if (halmac_adapter->halmac_state.api_state != HALMAC_API_STATE_INIT)
+		return HALMAC_RET_API_INVALID;
+
+	return HALMAC_RET_SUCCESS;
+}
+
+static inline enum halmac_ret_status
+halmac_fw_validate(struct halmac_adapter *halmac_adapter)
+{
+	if (halmac_adapter->halmac_state.dlfw_state != HALMAC_DLFW_DONE &&
+	    halmac_adapter->halmac_state.dlfw_state != HALMAC_GEN_INFO_SENT)
+		return HALMAC_RET_NO_DLFW;
+
+	return HALMAC_RET_SUCCESS;
+}
+
+#endif
diff --git a/drivers/staging/rtlwifi/halmac/halmac_usb_reg.h b/drivers/staging/rtlwifi/halmac/halmac_usb_reg.h
new file mode 100644
index 000000000000..d6e721ea7463
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/halmac_usb_reg.h
@@ -0,0 +1,28 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef __HALMAC_USB_REG_H__
+#define __HALMAC_USB_REG_H__
+
+#endif /* __HALMAC_USB_REG_H__ */
diff --git a/drivers/staging/rtlwifi/halmac/rtl_halmac.c b/drivers/staging/rtlwifi/halmac/rtl_halmac.c
new file mode 100644
index 000000000000..6448a8bfc14b
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/rtl_halmac.c
@@ -0,0 +1,1384 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#include "halmac_api.h"
+#include "rtl_halmac.h"
+#include <linux/module.h>
+#include <linux/vmalloc.h>
+
+#define DEFAULT_INDICATOR_TIMELMT msecs_to_jiffies(1000) /* ms */
+#define FIRMWARE_MAX_SIZE HALMAC_FW_SIZE_MAX_88XX
+
+static struct rtl_halmac_ops rtl_halmac_operation = {
+	.halmac_init_adapter = rtl_halmac_init_adapter,
+	.halmac_deinit_adapter = rtl_halmac_deinit_adapter,
+	.halmac_init_hal = rtl_halmac_init_hal,
+	.halmac_deinit_hal = rtl_halmac_deinit_hal,
+	.halmac_poweron = rtl_halmac_poweron,
+	.halmac_poweroff = rtl_halmac_poweroff,
+
+	.halmac_phy_power_switch = rtl_halmac_phy_power_switch,
+	.halmac_set_mac_address = rtl_halmac_set_mac_address,
+	.halmac_set_bssid = rtl_halmac_set_bssid,
+
+	.halmac_get_physical_efuse_size = rtl_halmac_get_physical_efuse_size,
+	.halmac_read_physical_efuse_map = rtl_halmac_read_physical_efuse_map,
+	.halmac_get_logical_efuse_size = rtl_halmac_get_logical_efuse_size,
+	.halmac_read_logical_efuse_map = rtl_halmac_read_logical_efuse_map,
+
+	.halmac_set_bandwidth = rtl_halmac_set_bandwidth,
+
+	.halmac_c2h_handle = rtl_halmac_c2h_handle,
+
+	.halmac_chk_txdesc = rtl_halmac_chk_txdesc,
+};
+
+struct rtl_halmac_ops *rtl_halmac_get_ops_pointer(void)
+{
+	return &rtl_halmac_operation;
+}
+EXPORT_SYMBOL(rtl_halmac_get_ops_pointer);
+
+/*
+ * Driver API for HALMAC operations
+ */
+
+static u8 _halmac_reg_read_8(void *p, u32 offset)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)p;
+
+	return rtl_read_byte(rtlpriv, offset);
+}
+
+static u16 _halmac_reg_read_16(void *p, u32 offset)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)p;
+
+	return rtl_read_word(rtlpriv, offset);
+}
+
+static u32 _halmac_reg_read_32(void *p, u32 offset)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)p;
+
+	return rtl_read_dword(rtlpriv, offset);
+}
+
+static void _halmac_reg_write_8(void *p, u32 offset, u8 val)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)p;
+
+	rtl_write_byte(rtlpriv, offset, val);
+}
+
+static void _halmac_reg_write_16(void *p, u32 offset, u16 val)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)p;
+
+	rtl_write_word(rtlpriv, offset, val);
+}
+
+static void _halmac_reg_write_32(void *p, u32 offset, u32 val)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)p;
+
+	rtl_write_dword(rtlpriv, offset, val);
+}
+
+static bool _halmac_write_data_rsvd_page(void *p, u8 *buf, u32 size)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)p;
+
+	if (rtlpriv->cfg->ops->halmac_cb_write_data_rsvd_page &&
+	    rtlpriv->cfg->ops->halmac_cb_write_data_rsvd_page(rtlpriv, buf,
+							      size))
+		return true;
+
+	return false;
+}
+
+static bool _halmac_write_data_h2c(void *p, u8 *buf, u32 size)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)p;
+
+	if (rtlpriv->cfg->ops->halmac_cb_write_data_h2c &&
+	    rtlpriv->cfg->ops->halmac_cb_write_data_h2c(rtlpriv, buf, size))
+		return true;
+
+	return false;
+}
+
+static const char *const RTL_HALMAC_FEATURE_NAME[] = {
+	"HALMAC_FEATURE_CFG_PARA",
+	"HALMAC_FEATURE_DUMP_PHYSICAL_EFUSE",
+	"HALMAC_FEATURE_DUMP_LOGICAL_EFUSE",
+	"HALMAC_FEATURE_UPDATE_PACKET",
+	"HALMAC_FEATURE_UPDATE_DATAPACK",
+	"HALMAC_FEATURE_RUN_DATAPACK",
+	"HALMAC_FEATURE_CHANNEL_SWITCH",
+	"HALMAC_FEATURE_IQK",
+	"HALMAC_FEATURE_POWER_TRACKING",
+	"HALMAC_FEATURE_PSD",
+	"HALMAC_FEATURE_ALL"};
+
+static inline bool is_valid_id_status(struct rtl_priv *rtlpriv,
+				      enum halmac_feature_id id,
+				      enum halmac_cmd_process_status status)
+{
+	switch (id) {
+	case HALMAC_FEATURE_CFG_PARA:
+		RT_TRACE(rtlpriv, COMP_HALMAC, DBG_LOUD, "%s: %s\n", __func__,
+			 RTL_HALMAC_FEATURE_NAME[id]);
+		break;
+	case HALMAC_FEATURE_DUMP_PHYSICAL_EFUSE:
+		RT_TRACE(rtlpriv, COMP_HALMAC, DBG_LOUD, "%s: %s\n", __func__,
+			 RTL_HALMAC_FEATURE_NAME[id]);
+		if (status != HALMAC_CMD_PROCESS_DONE) {
+			RT_TRACE(rtlpriv, COMP_HALMAC, DBG_LOUD,
+				 "%s: <WARN> id(%d) unspecified status(%d)!\n",
+				 __func__, id, status);
+		}
+		break;
+	case HALMAC_FEATURE_DUMP_LOGICAL_EFUSE:
+		RT_TRACE(rtlpriv, COMP_HALMAC, DBG_LOUD, "%s: %s\n", __func__,
+			 RTL_HALMAC_FEATURE_NAME[id]);
+		if (status != HALMAC_CMD_PROCESS_DONE) {
+			RT_TRACE(rtlpriv, COMP_HALMAC, DBG_LOUD,
+				 "%s: <WARN> id(%d) unspecified status(%d)!\n",
+				 __func__, id, status);
+		}
+		break;
+	case HALMAC_FEATURE_UPDATE_PACKET:
+		RT_TRACE(rtlpriv, COMP_HALMAC, DBG_LOUD, "%s: %s\n", __func__,
+			 RTL_HALMAC_FEATURE_NAME[id]);
+		break;
+	case HALMAC_FEATURE_UPDATE_DATAPACK:
+		RT_TRACE(rtlpriv, COMP_HALMAC, DBG_LOUD, "%s: %s\n", __func__,
+			 RTL_HALMAC_FEATURE_NAME[id]);
+		break;
+	case HALMAC_FEATURE_RUN_DATAPACK:
+		RT_TRACE(rtlpriv, COMP_HALMAC, DBG_LOUD, "%s: %s\n", __func__,
+			 RTL_HALMAC_FEATURE_NAME[id]);
+		break;
+	case HALMAC_FEATURE_CHANNEL_SWITCH:
+		RT_TRACE(rtlpriv, COMP_HALMAC, DBG_LOUD, "%s: %s\n", __func__,
+			 RTL_HALMAC_FEATURE_NAME[id]);
+		break;
+	case HALMAC_FEATURE_IQK:
+		RT_TRACE(rtlpriv, COMP_HALMAC, DBG_LOUD, "%s: %s\n", __func__,
+			 RTL_HALMAC_FEATURE_NAME[id]);
+		break;
+	case HALMAC_FEATURE_POWER_TRACKING:
+		RT_TRACE(rtlpriv, COMP_HALMAC, DBG_LOUD, "%s: %s\n", __func__,
+			 RTL_HALMAC_FEATURE_NAME[id]);
+		break;
+	case HALMAC_FEATURE_PSD:
+		RT_TRACE(rtlpriv, COMP_HALMAC, DBG_LOUD, "%s: %s\n", __func__,
+			 RTL_HALMAC_FEATURE_NAME[id]);
+		break;
+	case HALMAC_FEATURE_ALL:
+		RT_TRACE(rtlpriv, COMP_HALMAC, DBG_LOUD, "%s: %s\n", __func__,
+			 RTL_HALMAC_FEATURE_NAME[id]);
+		break;
+	default:
+		RT_TRACE(rtlpriv, COMP_HALMAC, DBG_LOUD,
+			 "%s: unknown feature id(%d)\n", __func__, id);
+		return false;
+	}
+
+	return true;
+}
+
+static int init_halmac_event_with_waittime(struct rtl_priv *rtlpriv,
+					   enum halmac_feature_id id, u8 *buf,
+					   u32 size, u32 time)
+{
+	struct completion *comp;
+
+	if (!rtlpriv->halmac.indicator[id].comp) {
+		comp = kzalloc(sizeof(*comp), GFP_KERNEL);
+		if (!comp)
+			return -1;
+	} else {
+		RT_TRACE(rtlpriv, COMP_HALMAC, DBG_LOUD,
+			 "%s: <WARN> id(%d) sctx is not NULL!!\n", __func__,
+			 id);
+		comp = rtlpriv->halmac.indicator[id].comp;
+		rtlpriv->halmac.indicator[id].comp = NULL;
+	}
+
+	init_completion(comp);
+	rtlpriv->halmac.indicator[id].wait_ms = time;
+
+	rtlpriv->halmac.indicator[id].buffer = buf;
+	rtlpriv->halmac.indicator[id].buf_size = size;
+	rtlpriv->halmac.indicator[id].ret_size = 0;
+	rtlpriv->halmac.indicator[id].status = 0;
+	/* fill sctx at least to sure other variables are all ready! */
+	rtlpriv->halmac.indicator[id].comp = comp;
+
+	return 0;
+}
+
+static inline int init_halmac_event(struct rtl_priv *rtlpriv,
+				    enum halmac_feature_id id, u8 *buf,
+				    u32 size)
+{
+	return init_halmac_event_with_waittime(rtlpriv, id, buf, size,
+					       DEFAULT_INDICATOR_TIMELMT);
+}
+
+static void free_halmac_event(struct rtl_priv *rtlpriv,
+			      enum halmac_feature_id id)
+{
+	struct completion *comp;
+
+	if (!rtlpriv->halmac.indicator[id].comp)
+		return;
+
+	comp = rtlpriv->halmac.indicator[id].comp;
+	rtlpriv->halmac.indicator[id].comp = NULL;
+	kfree(comp);
+}
+
+static int wait_halmac_event(struct rtl_priv *rtlpriv,
+			     enum halmac_feature_id id)
+{
+	struct completion *comp;
+	int ret;
+
+	comp = rtlpriv->halmac.indicator[id].comp;
+	if (!comp)
+		return -1;
+
+	ret = wait_for_completion_timeout(
+		comp, rtlpriv->halmac.indicator[id].wait_ms);
+	free_halmac_event(rtlpriv, id);
+	if (ret > 0)
+		return 0;
+
+	return -1;
+}
+
+/*
+ * Return:
+ *	Always return true, HALMAC don't care the return value.
+ */
+static bool
+_halmac_event_indication(void *p, enum halmac_feature_id feature_id,
+			 enum halmac_cmd_process_status process_status, u8 *buf,
+			 u32 size)
+{
+	struct rtl_priv *rtlpriv;
+	struct rtl_halmac_indicator *tbl, *indicator;
+	struct completion *comp;
+	u32 cpsz;
+	bool ret;
+
+	rtlpriv = (struct rtl_priv *)p;
+	tbl = rtlpriv->halmac.indicator;
+
+	ret = is_valid_id_status(rtlpriv, feature_id, process_status);
+	if (!ret)
+		goto exit;
+
+	indicator = &tbl[feature_id];
+	indicator->status = process_status;
+	indicator->ret_size = size;
+	if (!indicator->comp) {
+		RT_TRACE(rtlpriv, COMP_HALMAC, DBG_LOUD,
+			 "%s: No feature id(%d) waiting!!\n", __func__,
+			 feature_id);
+		goto exit;
+	}
+	comp = indicator->comp;
+
+	if (process_status == HALMAC_CMD_PROCESS_ERROR) {
+		RT_TRACE(rtlpriv, COMP_HALMAC, DBG_LOUD,
+			 "%s: Something wrong id(%d)!!\n", __func__,
+			 feature_id);
+		complete(comp); /* may provide error code */
+		goto exit;
+	}
+
+	if (size > indicator->buf_size) {
+		RT_TRACE(
+			rtlpriv, COMP_HALMAC, DBG_LOUD,
+			"%s: <WARN> id(%d) buffer is not enough(%d<%d), data will be truncated!\n",
+			__func__, feature_id, indicator->buf_size, size);
+		cpsz = indicator->buf_size;
+	} else {
+		cpsz = size;
+	}
+
+	if (cpsz && indicator->buffer)
+		memcpy(indicator->buffer, buf, cpsz);
+
+	complete(comp);
+
+exit:
+	return true;
+}
+
+static struct halmac_platform_api rtl_halmac_platform_api = {
+	/* R/W register */
+	.REG_READ_8 = _halmac_reg_read_8,
+	.REG_READ_16 = _halmac_reg_read_16,
+	.REG_READ_32 = _halmac_reg_read_32,
+	.REG_WRITE_8 = _halmac_reg_write_8,
+	.REG_WRITE_16 = _halmac_reg_write_16,
+	.REG_WRITE_32 = _halmac_reg_write_32,
+
+	/* Write data */
+	/* impletement in HAL-IC level */
+	.SEND_RSVD_PAGE = _halmac_write_data_rsvd_page,
+	.SEND_H2C_PKT = _halmac_write_data_h2c,
+
+	.EVENT_INDICATION = _halmac_event_indication,
+};
+
+static int init_priv(struct rtl_halmac *halmac)
+{
+	struct rtl_halmac_indicator *indicator;
+	u32 count, size;
+
+	halmac->send_general_info = 0;
+
+	count = HALMAC_FEATURE_ALL + 1;
+	size = sizeof(*indicator) * count;
+	indicator = kzalloc(size, GFP_KERNEL);
+	if (!indicator)
+		return -1;
+	halmac->indicator = indicator;
+
+	return 0;
+}
+
+static void deinit_priv(struct rtl_halmac *halmac)
+{
+	struct rtl_halmac_indicator *indicator;
+
+	indicator = halmac->indicator;
+	halmac->indicator = NULL;
+	kfree(indicator);
+}
+
+int rtl_halmac_init_adapter(struct rtl_priv *rtlpriv)
+{
+	struct halmac_adapter *halmac;
+	struct halmac_api *api;
+	enum halmac_interface intf;
+	enum halmac_ret_status status;
+	int err = 0;
+	struct halmac_platform_api *pf_api = &rtl_halmac_platform_api;
+
+	halmac = rtlpriv_to_halmac(rtlpriv);
+	if (halmac) {
+		err = 0;
+		goto out;
+	}
+
+	err = init_priv(&rtlpriv->halmac);
+	if (err)
+		goto out;
+
+	intf = HALMAC_INTERFACE_PCIE;
+	status = halmac_init_adapter(rtlpriv, pf_api, intf, &halmac, &api);
+	if (status != HALMAC_RET_SUCCESS) {
+		RT_TRACE(rtlpriv, COMP_HALMAC, DBG_LOUD,
+			 "%s: halmac_init_adapter fail!(status=%d)\n", __func__,
+			 status);
+		err = -1;
+		goto out;
+	}
+
+	rtlpriv->halmac.internal = halmac;
+
+out:
+	if (err)
+		rtl_halmac_deinit_adapter(rtlpriv);
+
+	return err;
+}
+
+int rtl_halmac_deinit_adapter(struct rtl_priv *rtlpriv)
+{
+	struct halmac_adapter *halmac;
+	enum halmac_ret_status status;
+	int err = 0;
+
+	halmac = rtlpriv_to_halmac(rtlpriv);
+	if (!halmac) {
+		err = 0;
+		goto out;
+	}
+
+	deinit_priv(&rtlpriv->halmac);
+
+	halmac_halt_api(halmac);
+
+	status = halmac_deinit_adapter(halmac);
+	rtlpriv->halmac.internal = NULL;
+	if (status != HALMAC_RET_SUCCESS) {
+		err = -1;
+		goto out;
+	}
+
+out:
+	return err;
+}
+
+int rtl_halmac_poweron(struct rtl_priv *rtlpriv)
+{
+	struct halmac_adapter *halmac;
+	struct halmac_api *api;
+	enum halmac_ret_status status;
+	int err = -1;
+
+	halmac = rtlpriv_to_halmac(rtlpriv);
+	if (!halmac)
+		goto out;
+
+	api = HALMAC_GET_API(halmac);
+
+	status = api->halmac_pre_init_system_cfg(halmac);
+	if (status != HALMAC_RET_SUCCESS)
+		goto out;
+
+	status = api->halmac_mac_power_switch(halmac, HALMAC_MAC_POWER_ON);
+	if (status != HALMAC_RET_SUCCESS)
+		goto out;
+
+	status = api->halmac_init_system_cfg(halmac);
+	if (status != HALMAC_RET_SUCCESS)
+		goto out;
+
+	err = 0;
+out:
+	return err;
+}
+
+int rtl_halmac_poweroff(struct rtl_priv *rtlpriv)
+{
+	struct halmac_adapter *halmac;
+	struct halmac_api *api;
+	enum halmac_ret_status status;
+	int err = -1;
+
+	halmac = rtlpriv_to_halmac(rtlpriv);
+	if (!halmac)
+		goto out;
+
+	api = HALMAC_GET_API(halmac);
+
+	status = api->halmac_mac_power_switch(halmac, HALMAC_MAC_POWER_OFF);
+	if (status != HALMAC_RET_SUCCESS)
+		goto out;
+
+	err = 0;
+out:
+	return err;
+}
+
+/*
+ * Note:
+ *	When this function return, the register REG_RCR may be changed.
+ */
+int rtl_halmac_config_rx_info(struct rtl_priv *rtlpriv,
+			      enum halmac_drv_info info)
+{
+	struct halmac_adapter *halmac;
+	struct halmac_api *api;
+	enum halmac_ret_status status;
+	int err = -1;
+
+	halmac = rtlpriv_to_halmac(rtlpriv);
+	api = HALMAC_GET_API(halmac);
+
+	status = api->halmac_cfg_drv_info(halmac, info);
+	if (status != HALMAC_RET_SUCCESS)
+		goto out;
+
+	err = 0;
+out:
+	return err;
+}
+
+static enum halmac_ret_status init_mac_flow(struct rtl_priv *rtlpriv)
+{
+	struct halmac_adapter *halmac;
+	struct halmac_api *api;
+	enum halmac_ret_status status;
+	u8 wifi_test = 0;
+	int err;
+
+	halmac = rtlpriv_to_halmac(rtlpriv);
+	api = HALMAC_GET_API(halmac);
+
+	if (wifi_test)
+		status = api->halmac_init_mac_cfg(halmac, HALMAC_TRX_MODE_WMM);
+	else
+		status = api->halmac_init_mac_cfg(halmac,
+						  HALMAC_TRX_MODE_NORMAL);
+	if (status != HALMAC_RET_SUCCESS)
+		goto out;
+
+	err = rtl_halmac_rx_agg_switch(rtlpriv, true);
+	if (err)
+		goto out;
+
+	if (rtlpriv->cfg->maps[RTL_RC_VHT_RATE_1SS_MCS7])
+		status = api->halmac_cfg_operation_mode(
+			halmac, HALMAC_WIRELESS_MODE_AC);
+	else if (rtlpriv->cfg->maps[RTL_RC_HT_RATEMCS7])
+		status = api->halmac_cfg_operation_mode(halmac,
+							HALMAC_WIRELESS_MODE_N);
+	else if (rtlpriv->cfg->maps[RTL_RC_OFDM_RATE6M])
+		status = api->halmac_cfg_operation_mode(halmac,
+							HALMAC_WIRELESS_MODE_G);
+	else
+		status = api->halmac_cfg_operation_mode(halmac,
+							HALMAC_WIRELESS_MODE_B);
+	if (status != HALMAC_RET_SUCCESS)
+		goto out;
+
+out:
+	return status;
+}
+
+static inline enum halmac_rf_type _rf_type_drv2halmac(enum rf_type rf_drv)
+{
+	enum halmac_rf_type rf_mac;
+
+	switch (rf_drv) {
+	case RF_1T2R:
+		rf_mac = HALMAC_RF_1T2R;
+		break;
+	case RF_2T2R:
+		rf_mac = HALMAC_RF_2T2R;
+		break;
+	case RF_1T1R:
+		rf_mac = HALMAC_RF_1T1R;
+		break;
+	case RF_2T2R_GREEN:
+		rf_mac = HALMAC_RF_2T2R_GREEN;
+		break;
+	default:
+		rf_mac = (enum halmac_rf_type)rf_drv;
+		break;
+	}
+
+	return rf_mac;
+}
+
+static int _send_general_info(struct rtl_priv *rtlpriv)
+{
+	struct halmac_adapter *halmac;
+	struct halmac_api *api;
+	struct halmac_general_info info;
+	enum halmac_ret_status status;
+
+	halmac = rtlpriv_to_halmac(rtlpriv);
+	if (!halmac)
+		return -1;
+	api = HALMAC_GET_API(halmac);
+
+	memset(&info, 0, sizeof(info));
+	info.rfe_type = rtlpriv->rtlhal.rfe_type;
+	info.rf_type = _rf_type_drv2halmac(rtlpriv->phy.rf_type);
+
+	status = api->halmac_send_general_info(halmac, &info);
+	switch (status) {
+	case HALMAC_RET_SUCCESS:
+		break;
+	case HALMAC_RET_NO_DLFW:
+		RT_TRACE(rtlpriv, COMP_HALMAC, DBG_WARNING,
+			 "%s: halmac_send_general_info() fail because fw not dl!\n",
+			 __func__);
+	/* fallthrough here */
+	default:
+		return -1;
+	}
+
+	return 0;
+}
+
+/*
+ * Notices:
+ *	Make sure
+ *	1. rtl_hal_get_hwreg(HW_VAR_RF_TYPE)
+ *	2. HAL_DATA_TYPE.rfe_type
+ *	already ready for use before calling this function.
+ */
+static int _halmac_init_hal(struct rtl_priv *rtlpriv, u8 *fw, u32 fwsize)
+{
+	struct halmac_adapter *halmac;
+	struct halmac_api *api;
+	enum halmac_ret_status status;
+	bool ok;
+	bool fw_ok = false;
+	int err, err_ret = -1;
+
+	halmac = rtlpriv_to_halmac(rtlpriv);
+	if (!halmac)
+		goto out;
+	api = HALMAC_GET_API(halmac);
+
+	/* StatePowerOff */
+
+	/* SKIP: halmac_init_adapter (Already done before) */
+
+	/* halmac_pre_Init_system_cfg */
+	/* halmac_mac_power_switch(on) */
+	/* halmac_Init_system_cfg */
+	err = rtl_halmac_poweron(rtlpriv);
+	if (err)
+		goto out;
+
+	/* StatePowerOn */
+
+	/* DownloadFW */
+	rtlpriv->halmac.send_general_info = 0;
+	if (fw && fwsize) {
+		err = rtl_halmac_dlfw(rtlpriv, fw, fwsize);
+		if (err)
+			goto out;
+		fw_ok = true;
+	}
+
+	/* InitMACFlow */
+	status = init_mac_flow(rtlpriv);
+	if (status != HALMAC_RET_SUCCESS)
+		goto out;
+
+	/* halmac_send_general_info */
+	if (fw_ok) {
+		rtlpriv->halmac.send_general_info = 0;
+		err = _send_general_info(rtlpriv);
+		if (err)
+			goto out;
+	} else {
+		rtlpriv->halmac.send_general_info = 1;
+	}
+
+	/* Init Phy parameter-MAC */
+	if (rtlpriv->cfg->ops->halmac_cb_init_mac_register)
+		ok = rtlpriv->cfg->ops->halmac_cb_init_mac_register(rtlpriv);
+	else
+		ok = false;
+
+	if (!ok)
+		goto out;
+
+	/* StateMacInitialized */
+
+	/* halmac_cfg_drv_info */
+	err = rtl_halmac_config_rx_info(rtlpriv, HALMAC_DRV_INFO_PHY_STATUS);
+	if (err)
+		goto out;
+
+	/* halmac_set_hw_value(HALMAC_HW_EN_BB_RF) */
+	/* Init BB, RF */
+	if (rtlpriv->cfg->ops->halmac_cb_init_bb_rf_register)
+		ok = rtlpriv->cfg->ops->halmac_cb_init_bb_rf_register(rtlpriv);
+	else
+		ok = false;
+
+	if (!ok)
+		goto out;
+
+	status = api->halmac_init_interface_cfg(halmac);
+	if (status != HALMAC_RET_SUCCESS)
+		goto out;
+
+	/* SKIP: halmac_verify_platform_api */
+	/* SKIP: halmac_h2c_lb */
+
+	/* StateRxIdle */
+
+	err_ret = 0;
+out:
+	return err_ret;
+}
+
+int rtl_halmac_init_hal(struct rtl_priv *rtlpriv)
+{
+	if (!rtlpriv->rtlhal.pfirmware || rtlpriv->rtlhal.fwsize == 0)
+		return -1;
+
+	return _halmac_init_hal(rtlpriv, rtlpriv->rtlhal.pfirmware,
+				rtlpriv->rtlhal.fwsize);
+}
+
+int rtl_halmac_deinit_hal(struct rtl_priv *rtlpriv)
+{
+	struct halmac_adapter *halmac;
+	struct halmac_api *api;
+	enum halmac_ret_status status;
+	int err = -1;
+
+	halmac = rtlpriv_to_halmac(rtlpriv);
+	if (!halmac)
+		goto out;
+	api = HALMAC_GET_API(halmac);
+
+	status = api->halmac_deinit_interface_cfg(halmac);
+	if (status != HALMAC_RET_SUCCESS)
+		goto out;
+
+	/* rtw_hal_power_off(adapter); */
+	status = api->halmac_mac_power_switch(halmac, HALMAC_MAC_POWER_OFF);
+	if (status != HALMAC_RET_SUCCESS)
+		goto out;
+
+	err = 0;
+out:
+	return err;
+}
+
+int rtl_halmac_self_verify(struct rtl_priv *rtlpriv)
+{
+	struct halmac_adapter *mac;
+	struct halmac_api *api;
+	enum halmac_ret_status status;
+	int err = -1;
+
+	mac = rtlpriv_to_halmac(rtlpriv);
+	api = HALMAC_GET_API(mac);
+
+	status = api->halmac_verify_platform_api(mac);
+	if (status != HALMAC_RET_SUCCESS)
+		goto out;
+
+	status = api->halmac_h2c_lb(mac);
+	if (status != HALMAC_RET_SUCCESS)
+		goto out;
+
+	err = 0;
+out:
+	return err;
+}
+
+int rtl_halmac_dlfw(struct rtl_priv *rtlpriv, u8 *fw, u32 fwsize)
+{
+	struct halmac_adapter *mac;
+	struct halmac_api *api;
+	enum halmac_ret_status status;
+	struct halmac_fw_version fw_version;
+	int err = 0;
+
+	mac = rtlpriv_to_halmac(rtlpriv);
+	api = HALMAC_GET_API(mac);
+
+	if ((!fw) || (!fwsize))
+		return -1;
+
+	/* 1. Driver Stop Tx */
+	/* ToDo */
+
+	/* 2. Driver Check Tx FIFO is empty */
+	/* ToDo */
+
+	/* 3. Config MAX download size */
+	api->halmac_cfg_max_dl_size(mac, 0x1000);
+
+	/* 4. Download Firmware */
+	mac->h2c_packet_seq = 0;
+	status = api->halmac_download_firmware(mac, fw, fwsize);
+	if (status != HALMAC_RET_SUCCESS)
+		return -1;
+
+	status = api->halmac_get_fw_version(mac, &fw_version);
+	if (status == HALMAC_RET_SUCCESS) {
+		rtlpriv->rtlhal.fw_version = fw_version.version;
+		rtlpriv->rtlhal.fw_subversion =
+			(fw_version.sub_version << 8) | (fw_version.sub_index);
+
+		RT_TRACE(
+			rtlpriv, COMP_HALMAC, DBG_DMESG,
+			"halmac report firmware version %04X.%04X\n",
+			rtlpriv->rtlhal.fw_version,
+			rtlpriv->rtlhal.fw_subversion);
+	}
+
+	if (rtlpriv->halmac.send_general_info) {
+		rtlpriv->halmac.send_general_info = 0;
+		err = _send_general_info(rtlpriv);
+	}
+
+	/* 5. Driver resume TX if needed */
+	/* ToDo */
+
+	/* 6. Reset driver variables if needed */
+	/*hal->LastHMEBoxNum = 0;*/
+
+	return err;
+}
+
+/*
+ * Description:
+ *	Power on/off BB/RF domain.
+ *
+ * Parameters:
+ *	enable	true/false for power on/off
+ *
+ * Return:
+ *	0	Success
+ *	others	Fail
+ */
+int rtl_halmac_phy_power_switch(struct rtl_priv *rtlpriv, u8 enable)
+{
+	struct halmac_adapter *halmac;
+	struct halmac_api *api;
+	enum halmac_ret_status status;
+
+	halmac = rtlpriv_to_halmac(rtlpriv);
+	if (!halmac)
+		return -1;
+	api = HALMAC_GET_API(halmac);
+
+	status = api->halmac_set_hw_value(halmac, HALMAC_HW_EN_BB_RF, &enable);
+	if (status != HALMAC_RET_SUCCESS)
+		return -1;
+
+	return 0;
+}
+
+static bool _is_fw_read_cmd_down(struct rtl_priv *rtlpriv, u8 msgbox_num)
+{
+	bool read_down = false;
+	int retry_cnts = 100;
+	u8 valid;
+
+	RT_TRACE(rtlpriv, COMP_HALMAC, DBG_LOUD,
+		 "%s, reg_1cc(%x), msg_box(%d)...\n", __func__,
+		 rtl_read_byte(rtlpriv, REG_HMETFR), msgbox_num);
+
+	do {
+		valid = rtl_read_byte(rtlpriv, REG_HMETFR) & BIT(msgbox_num);
+		if (valid == 0)
+			read_down = true;
+		else
+			schedule();
+	} while ((!read_down) && (retry_cnts--));
+
+	return read_down;
+}
+
+int rtl_halmac_send_h2c(struct rtl_priv *rtlpriv, u8 *h2c)
+{
+	u8 h2c_box_num = 0;
+	u32 msgbox_addr = 0;
+	u32 msgbox_ex_addr = 0;
+	__le32 h2c_cmd = 0;
+	__le32 h2c_cmd_ex = 0;
+	s32 ret = -1;
+	unsigned long flag = 0;
+	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+
+	if (!h2c) {
+		RT_TRACE(rtlpriv, COMP_HALMAC, DBG_LOUD, "%s: pbuf is NULL\n",
+			 __func__);
+		return ret;
+	}
+
+	spin_lock_irqsave(&rtlpriv->locks.h2c_lock, flag);
+
+	/* pay attention to if race condition happened in  H2C cmd setting */
+	h2c_box_num = rtlhal->last_hmeboxnum;
+
+	if (!_is_fw_read_cmd_down(rtlpriv, h2c_box_num)) {
+		RT_TRACE(rtlpriv, COMP_HALMAC, DBG_LOUD,
+			 " fw read cmd failed...\n");
+		goto exit;
+	}
+
+	/* Write Ext command(byte 4 -7) */
+	msgbox_ex_addr = REG_HMEBOX_E0 + (h2c_box_num * EX_MESSAGE_BOX_SIZE);
+	memcpy((u8 *)(&h2c_cmd_ex), h2c + 4, EX_MESSAGE_BOX_SIZE);
+	rtl_write_dword(rtlpriv, msgbox_ex_addr, le32_to_cpu(h2c_cmd_ex));
+
+	/* Write command (byte 0 -3 ) */
+	msgbox_addr = REG_HMEBOX0 + (h2c_box_num * MESSAGE_BOX_SIZE);
+	memcpy((u8 *)(&h2c_cmd), h2c, 4);
+	rtl_write_dword(rtlpriv, msgbox_addr, le32_to_cpu(h2c_cmd));
+
+	/* update last msg box number */
+	rtlhal->last_hmeboxnum = (h2c_box_num + 1) % MAX_H2C_BOX_NUMS;
+	ret = 0;
+
+exit:
+	spin_unlock_irqrestore(&rtlpriv->locks.h2c_lock, flag);
+	return ret;
+}
+
+int rtl_halmac_c2h_handle(struct rtl_priv *rtlpriv, u8 *c2h, u32 size)
+{
+	struct halmac_adapter *mac;
+	struct halmac_api *api;
+	enum halmac_ret_status status;
+
+	mac = rtlpriv_to_halmac(rtlpriv);
+	api = HALMAC_GET_API(mac);
+
+	status = api->halmac_get_c2h_info(mac, c2h, size);
+	if (status != HALMAC_RET_SUCCESS)
+		return -1;
+
+	return 0;
+}
+
+int rtl_halmac_get_physical_efuse_size(struct rtl_priv *rtlpriv, u32 *size)
+{
+	struct halmac_adapter *mac;
+	struct halmac_api *api;
+	enum halmac_ret_status status;
+	u32 val;
+
+	mac = rtlpriv_to_halmac(rtlpriv);
+	api = HALMAC_GET_API(mac);
+
+	status = api->halmac_get_efuse_size(mac, &val);
+	if (status != HALMAC_RET_SUCCESS)
+		return -1;
+
+	*size = val;
+	return 0;
+}
+
+int rtl_halmac_read_physical_efuse_map(struct rtl_priv *rtlpriv, u8 *map,
+				       u32 size)
+{
+	struct halmac_adapter *mac;
+	struct halmac_api *api;
+	enum halmac_ret_status status;
+	enum halmac_feature_id id;
+	int ret;
+
+	mac = rtlpriv_to_halmac(rtlpriv);
+	api = HALMAC_GET_API(mac);
+	id = HALMAC_FEATURE_DUMP_PHYSICAL_EFUSE;
+
+	ret = init_halmac_event(rtlpriv, id, map, size);
+	if (ret)
+		return -1;
+
+	status = api->halmac_dump_efuse_map(mac, HALMAC_EFUSE_R_DRV);
+	if (status != HALMAC_RET_SUCCESS) {
+		free_halmac_event(rtlpriv, id);
+		return -1;
+	}
+
+	ret = wait_halmac_event(rtlpriv, id);
+	if (ret)
+		return -1;
+
+	return 0;
+}
+
+int rtl_halmac_read_physical_efuse(struct rtl_priv *rtlpriv, u32 offset,
+				   u32 cnt, u8 *data)
+{
+	struct halmac_adapter *mac;
+	struct halmac_api *api;
+	enum halmac_ret_status status;
+	u8 v;
+	u32 i;
+
+	mac = rtlpriv_to_halmac(rtlpriv);
+	api = HALMAC_GET_API(mac);
+
+	for (i = 0; i < cnt; i++) {
+		status = api->halmac_read_efuse(mac, offset + i, &v);
+		if (status != HALMAC_RET_SUCCESS)
+			return -1;
+		data[i] = v;
+	}
+
+	return 0;
+}
+
+int rtl_halmac_write_physical_efuse(struct rtl_priv *rtlpriv, u32 offset,
+				    u32 cnt, u8 *data)
+{
+	struct halmac_adapter *mac;
+	struct halmac_api *api;
+	enum halmac_ret_status status;
+	u32 i;
+
+	mac = rtlpriv_to_halmac(rtlpriv);
+	api = HALMAC_GET_API(mac);
+
+	for (i = 0; i < cnt; i++) {
+		status = api->halmac_write_efuse(mac, offset + i, data[i]);
+		if (status != HALMAC_RET_SUCCESS)
+			return -1;
+	}
+
+	return 0;
+}
+
+int rtl_halmac_get_logical_efuse_size(struct rtl_priv *rtlpriv, u32 *size)
+{
+	struct halmac_adapter *mac;
+	struct halmac_api *api;
+	enum halmac_ret_status status;
+	u32 val;
+
+	mac = rtlpriv_to_halmac(rtlpriv);
+	api = HALMAC_GET_API(mac);
+
+	status = api->halmac_get_logical_efuse_size(mac, &val);
+	if (status != HALMAC_RET_SUCCESS)
+		return -1;
+
+	*size = val;
+	return 0;
+}
+
+int rtl_halmac_read_logical_efuse_map(struct rtl_priv *rtlpriv, u8 *map,
+				      u32 size)
+{
+	struct halmac_adapter *mac;
+	struct halmac_api *api;
+	enum halmac_ret_status status;
+	enum halmac_feature_id id;
+	int ret;
+
+	mac = rtlpriv_to_halmac(rtlpriv);
+	api = HALMAC_GET_API(mac);
+	id = HALMAC_FEATURE_DUMP_LOGICAL_EFUSE;
+
+	ret = init_halmac_event(rtlpriv, id, map, size);
+	if (ret)
+		return -1;
+
+	status = api->halmac_dump_logical_efuse_map(mac, HALMAC_EFUSE_R_AUTO);
+	if (status != HALMAC_RET_SUCCESS) {
+		free_halmac_event(rtlpriv, id);
+		return -1;
+	}
+
+	ret = wait_halmac_event(rtlpriv, id);
+	if (ret)
+		return -1;
+
+	return 0;
+}
+
+int rtl_halmac_write_logical_efuse_map(struct rtl_priv *rtlpriv, u8 *map,
+				       u32 size, u8 *maskmap, u32 masksize)
+{
+	struct halmac_adapter *mac;
+	struct halmac_api *api;
+	struct halmac_pg_efuse_info pginfo;
+	enum halmac_ret_status status;
+
+	mac = rtlpriv_to_halmac(rtlpriv);
+	api = HALMAC_GET_API(mac);
+
+	pginfo.efuse_map = map;
+	pginfo.efuse_map_size = size;
+	pginfo.efuse_mask = maskmap;
+	pginfo.efuse_mask_size = masksize;
+
+	status = api->halmac_pg_efuse_by_map(mac, &pginfo, HALMAC_EFUSE_R_AUTO);
+	if (status != HALMAC_RET_SUCCESS)
+		return -1;
+
+	return 0;
+}
+
+int rtl_halmac_read_logical_efuse(struct rtl_priv *rtlpriv, u32 offset, u32 cnt,
+				  u8 *data)
+{
+	struct halmac_adapter *mac;
+	struct halmac_api *api;
+	enum halmac_ret_status status;
+	u8 v;
+	u32 i;
+
+	mac = rtlpriv_to_halmac(rtlpriv);
+	api = HALMAC_GET_API(mac);
+
+	for (i = 0; i < cnt; i++) {
+		status = api->halmac_read_logical_efuse(mac, offset + i, &v);
+		if (status != HALMAC_RET_SUCCESS)
+			return -1;
+		data[i] = v;
+	}
+
+	return 0;
+}
+
+int rtl_halmac_write_logical_efuse(struct rtl_priv *rtlpriv, u32 offset,
+				   u32 cnt, u8 *data)
+{
+	struct halmac_adapter *mac;
+	struct halmac_api *api;
+	enum halmac_ret_status status;
+	u32 i;
+
+	mac = rtlpriv_to_halmac(rtlpriv);
+	api = HALMAC_GET_API(mac);
+
+	for (i = 0; i < cnt; i++) {
+		status = api->halmac_write_logical_efuse(mac, offset + i,
+							 data[i]);
+		if (status != HALMAC_RET_SUCCESS)
+			return -1;
+	}
+
+	return 0;
+}
+
+int rtl_halmac_set_mac_address(struct rtl_priv *rtlpriv, u8 hwport, u8 *addr)
+{
+	struct halmac_adapter *halmac;
+	struct halmac_api *api;
+	u8 port;
+	union halmac_wlan_addr hwa;
+	enum halmac_ret_status status;
+	int err = -1;
+
+	halmac = rtlpriv_to_halmac(rtlpriv);
+	api = HALMAC_GET_API(halmac);
+
+	port = hwport;
+	memset(&hwa, 0, sizeof(hwa));
+	memcpy(hwa.address, addr, 6);
+
+	status = api->halmac_cfg_mac_addr(halmac, port, &hwa);
+	if (status != HALMAC_RET_SUCCESS)
+		goto out;
+
+	err = 0;
+out:
+	return err;
+}
+
+int rtl_halmac_set_bssid(struct rtl_priv *rtlpriv, u8 hwport, u8 *addr)
+{
+	struct halmac_adapter *halmac;
+	struct halmac_api *api;
+	u8 port;
+	union halmac_wlan_addr hwa;
+	enum halmac_ret_status status;
+	int err = -1;
+
+	halmac = rtlpriv_to_halmac(rtlpriv);
+	api = HALMAC_GET_API(halmac);
+	port = hwport;
+
+	memset(&hwa, 0, sizeof(union halmac_wlan_addr));
+	memcpy(hwa.address, addr, 6);
+	status = api->halmac_cfg_bssid(halmac, port, &hwa);
+	if (status != HALMAC_RET_SUCCESS)
+		goto out;
+
+	err = 0;
+out:
+	return err;
+}
+
+int rtl_halmac_set_bandwidth(struct rtl_priv *rtlpriv, u8 channel,
+			     u8 pri_ch_idx, u8 bw)
+{
+	struct halmac_adapter *mac;
+	struct halmac_api *api;
+	enum halmac_ret_status status;
+
+	mac = rtlpriv_to_halmac(rtlpriv);
+	api = HALMAC_GET_API(mac);
+
+	status = api->halmac_cfg_ch_bw(mac, channel, pri_ch_idx, bw);
+	if (status != HALMAC_RET_SUCCESS)
+		return -1;
+
+	return 0;
+}
+
+int rtl_halmac_get_hw_value(struct rtl_priv *rtlpriv, enum halmac_hw_id hw_id,
+			    void *pvalue)
+{
+	struct halmac_adapter *mac;
+	struct halmac_api *api;
+	enum halmac_ret_status status;
+
+	mac = rtlpriv_to_halmac(rtlpriv);
+	api = HALMAC_GET_API(mac);
+
+	status = api->halmac_get_hw_value(mac, hw_id, pvalue);
+	if (status != HALMAC_RET_SUCCESS)
+		return -1;
+
+	return 0;
+}
+
+int rtl_halmac_dump_fifo(struct rtl_priv *rtlpriv,
+			 enum hal_fifo_sel halmac_fifo_sel)
+{
+	struct halmac_adapter *mac;
+	struct halmac_api *api;
+	enum halmac_ret_status status;
+	u8 *pfifo_map = NULL;
+	u32 fifo_size = 0;
+	s8 ret = 0;
+
+	mac = rtlpriv_to_halmac(rtlpriv);
+	api = HALMAC_GET_API(mac);
+
+	fifo_size = api->halmac_get_fifo_size(mac, halmac_fifo_sel);
+	if (fifo_size)
+		pfifo_map = vmalloc(fifo_size);
+	if (!pfifo_map)
+		return -1;
+
+	status = api->halmac_dump_fifo(mac, halmac_fifo_sel, 0, fifo_size,
+				       pfifo_map);
+
+	if (status != HALMAC_RET_SUCCESS) {
+		ret = -1;
+		goto _exit;
+	}
+
+_exit:
+	if (pfifo_map)
+		vfree(pfifo_map);
+	return ret;
+}
+
+int rtl_halmac_rx_agg_switch(struct rtl_priv *rtlpriv, bool enable)
+{
+	struct halmac_adapter *halmac;
+	struct halmac_api *api;
+	struct halmac_rxagg_cfg rxaggcfg;
+	enum halmac_ret_status status;
+	int err = -1;
+
+	halmac = rtlpriv_to_halmac(rtlpriv);
+	api = HALMAC_GET_API(halmac);
+	memset((void *)&rxaggcfg, 0, sizeof(rxaggcfg));
+
+	if (enable) {
+		/* enable RX agg. */
+		/* PCIE do nothing */
+	} else {
+		/* disable RX agg. */
+		rxaggcfg.mode = HALMAC_RX_AGG_MODE_NONE;
+	}
+
+	status = api->halmac_cfg_rx_aggregation(halmac, &rxaggcfg);
+	if (status != HALMAC_RET_SUCCESS)
+		goto out;
+
+	err = 0;
+out:
+	return err;
+}
+
+int rtl_halmac_get_wow_reason(struct rtl_priv *rtlpriv, u8 *reason)
+{
+	u8 val8;
+	int err = -1;
+
+	val8 = rtl_read_byte(rtlpriv, 0x1C7);
+	if (val8 == 0xEA)
+		goto out;
+
+	*reason = val8;
+	err = 0;
+out:
+	return err;
+}
+
+/*
+ * Description:
+ *	Get RX driver info size. RX driver info is a small memory space between
+ *	scriptor and RX payload.
+ *
+ *	+-------------------------+
+ *	| RX descriptor           |
+ *	| usually 24 bytes        |
+ *	+-------------------------+
+ *	| RX driver info          |
+ *	| depends on driver cfg   |
+ *	+-------------------------+
+ *	| RX paylad               |
+ *	|                         |
+ *	+-------------------------+
+ *
+ * Parameter:
+ *	d	pointer to struct dvobj_priv of driver
+ *	sz	rx driver info size in bytes.
+ *
+ * Rteurn:
+ *	0	Success
+ *	other	Fail
+ */
+int rtl_halmac_get_drv_info_sz(struct rtl_priv *rtlpriv, u8 *sz)
+{
+	/*	enum halmac_ret_status status; */
+	u8 dw = 6; /* max number */
+
+	*sz = dw * 8;
+	return 0;
+}
+
+int rtl_halmac_get_rsvd_drv_pg_bndy(struct rtl_priv *rtlpriv, u16 *drv_pg)
+{
+	enum halmac_ret_status status;
+	struct halmac_adapter *halmac = rtlpriv_to_halmac(rtlpriv);
+	struct halmac_api *api = HALMAC_GET_API(halmac);
+
+	status = api->halmac_get_hw_value(halmac, HALMAC_HW_RSVD_PG_BNDY,
+					  drv_pg);
+	if (status != HALMAC_RET_SUCCESS)
+		return -1;
+
+	return 0;
+}
+
+int rtl_halmac_chk_txdesc(struct rtl_priv *rtlpriv, u8 *txdesc, u32 size)
+{
+	struct halmac_adapter *mac;
+	struct halmac_api *api;
+	enum halmac_ret_status status;
+
+	mac = rtlpriv_to_halmac(rtlpriv);
+	api = HALMAC_GET_API(mac);
+
+	status = api->halmac_chk_txdesc(mac, txdesc, size);
+
+	if (status != HALMAC_RET_SUCCESS)
+		return -1;
+
+	return 0;
+}
+
+MODULE_AUTHOR("Realtek WlanFAE	<wlanfae@realtek.com>");
+MODULE_AUTHOR("Larry Finger	<Larry.FInger@lwfinger.net>");
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Realtek 802.11n PCI wireless core");
diff --git a/drivers/staging/rtlwifi/halmac/rtl_halmac.h b/drivers/staging/rtlwifi/halmac/rtl_halmac.h
new file mode 100644
index 000000000000..51a3684f30d8
--- /dev/null
+++ b/drivers/staging/rtlwifi/halmac/rtl_halmac.h
@@ -0,0 +1,94 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef _RTL_HALMAC_H_
+#define _RTL_HALMAC_H_
+
+#include "halmac_api.h"
+
+#define rtlpriv_to_halmac(priv)                                                \
+	((struct halmac_adapter *)((priv)->halmac.internal))
+
+/* for H2C cmd */
+#define MAX_H2C_BOX_NUMS 4
+#define MESSAGE_BOX_SIZE 4
+#define EX_MESSAGE_BOX_SIZE 4
+
+/* HALMAC API for Driver(HAL) */
+int rtl_halmac_init_adapter(struct rtl_priv *rtlpriv);
+int rtl_halmac_deinit_adapter(struct rtl_priv *rtlpriv);
+int rtl_halmac_poweron(struct rtl_priv *rtlpriv);
+int rtl_halmac_poweroff(struct rtl_priv *rtlpriv);
+int rtl_halmac_init_hal(struct rtl_priv *rtlpriv);
+int rtl_halmac_init_hal_fw(struct rtl_priv *rtlpriv, u8 *fw, u32 fwsize);
+int rtl_halmac_init_hal_fw_file(struct rtl_priv *rtlpriv, u8 *fwpath);
+int rtl_halmac_deinit_hal(struct rtl_priv *rtlpriv);
+int rtl_halmac_self_verify(struct rtl_priv *rtlpriv);
+int rtl_halmac_dlfw(struct rtl_priv *rtlpriv, u8 *fw, u32 fwsize);
+int rtl_halmac_dlfw_from_file(struct rtl_priv *rtlpriv, u8 *fwpath);
+int rtl_halmac_phy_power_switch(struct rtl_priv *rtlpriv, u8 enable);
+int rtl_halmac_send_h2c(struct rtl_priv *rtlpriv, u8 *h2c);
+int rtl_halmac_c2h_handle(struct rtl_priv *rtlpriv, u8 *c2h, u32 size);
+
+int rtl_halmac_get_physical_efuse_size(struct rtl_priv *rtlpriv, u32 *size);
+int rtl_halmac_read_physical_efuse_map(struct rtl_priv *rtlpriv, u8 *map,
+				       u32 size);
+int rtl_halmac_read_physical_efuse(struct rtl_priv *rtlpriv, u32 offset,
+				   u32 cnt, u8 *data);
+int rtl_halmac_write_physical_efuse(struct rtl_priv *rtlpriv, u32 offset,
+				    u32 cnt, u8 *data);
+int rtl_halmac_get_logical_efuse_size(struct rtl_priv *rtlpriv, u32 *size);
+int rtl_halmac_read_logical_efuse_map(struct rtl_priv *rtlpriv, u8 *map,
+				      u32 size);
+int rtl_halmac_write_logical_efuse_map(struct rtl_priv *rtlpriv, u8 *map,
+				       u32 size, u8 *maskmap, u32 masksize);
+int rtl_halmac_read_logical_efuse(struct rtl_priv *rtlpriv, u32 offset, u32 cnt,
+				  u8 *data);
+int rtl_halmac_write_logical_efuse(struct rtl_priv *rtlpriv, u32 offset,
+				   u32 cnt, u8 *data);
+
+int rtl_halmac_config_rx_info(struct rtl_priv *rtlpriv, enum halmac_drv_info);
+int rtl_halmac_set_mac_address(struct rtl_priv *rtlpriv, u8 hwport, u8 *addr);
+int rtl_halmac_set_bssid(struct rtl_priv *d, u8 hwport, u8 *addr);
+
+int rtl_halmac_set_bandwidth(struct rtl_priv *rtlpriv, u8 channel,
+			     u8 pri_ch_idx, u8 bw);
+int rtl_halmac_rx_agg_switch(struct rtl_priv *rtlpriv, bool enable);
+int rtl_halmac_get_hw_value(struct rtl_priv *d, enum halmac_hw_id hw_id,
+			    void *pvalue);
+int rtl_halmac_dump_fifo(struct rtl_priv *rtlpriv,
+			 enum hal_fifo_sel halmac_fifo_sel);
+
+int rtl_halmac_get_wow_reason(struct rtl_priv *rtlpriv, u8 *reason);
+int rtl_halmac_get_drv_info_sz(struct rtl_priv *d, u8 *sz);
+
+int rtl_halmac_get_rsvd_drv_pg_bndy(struct rtl_priv *dvobj, u16 *drv_pg);
+int rtl_halmac_download_rsvd_page(struct rtl_priv *dvobj, u8 pg_offset,
+				  u8 *pbuf, u32 size);
+
+int rtl_halmac_chk_txdesc(struct rtl_priv *rtlpriv, u8 *txdesc, u32 size);
+
+struct rtl_halmac_ops *rtl_halmac_get_ops_pointer(void);
+
+#endif /* _RTL_HALMAC_H_ */
diff --git a/drivers/staging/rtlwifi/pci.c b/drivers/staging/rtlwifi/pci.c
new file mode 100644
index 000000000000..4035b8835bd1
--- /dev/null
+++ b/drivers/staging/rtlwifi/pci.c
@@ -0,0 +1,2508 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2012  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#include "wifi.h"
+#include "core.h"
+#include "pci.h"
+#include "base.h"
+#include "ps.h"
+#include "efuse.h"
+#include <linux/interrupt.h>
+#include <linux/export.h>
+#include <linux/kmemleak.h>
+#include <linux/module.h>
+
+MODULE_AUTHOR("lizhaoming	<chaoming_li@realsil.com.cn>");
+MODULE_AUTHOR("Realtek WlanFAE	<wlanfae@realtek.com>");
+MODULE_AUTHOR("Larry Finger	<Larry.FInger@lwfinger.net>");
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("PCI basic driver for rtlwifi");
+
+static const u16 pcibridge_vendors[PCI_BRIDGE_VENDOR_MAX] = {
+	INTEL_VENDOR_ID,
+	ATI_VENDOR_ID,
+	AMD_VENDOR_ID,
+	SIS_VENDOR_ID
+};
+
+static const u8 ac_to_hwq[] = {
+	VO_QUEUE,
+	VI_QUEUE,
+	BE_QUEUE,
+	BK_QUEUE
+};
+
+static u8 _rtl_mac_to_hwqueue(struct ieee80211_hw *hw,
+			      struct sk_buff *skb)
+{
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	__le16 fc = rtl_get_fc(skb);
+	u8 queue_index = skb_get_queue_mapping(skb);
+	struct ieee80211_hdr *hdr;
+
+	if (unlikely(ieee80211_is_beacon(fc)))
+		return BEACON_QUEUE;
+	if (ieee80211_is_mgmt(fc) || ieee80211_is_ctl(fc))
+		return MGNT_QUEUE;
+	if (rtlhal->hw_type == HARDWARE_TYPE_RTL8192SE)
+		if (ieee80211_is_nullfunc(fc))
+			return HIGH_QUEUE;
+	if (rtlhal->hw_type == HARDWARE_TYPE_RTL8822BE) {
+		hdr = rtl_get_hdr(skb);
+
+		if (is_multicast_ether_addr(hdr->addr1) ||
+		    is_broadcast_ether_addr(hdr->addr1))
+			return HIGH_QUEUE;
+	}
+
+	return ac_to_hwq[queue_index];
+}
+
+/* Update PCI dependent default settings*/
+static void _rtl_pci_update_default_setting(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci_priv *pcipriv = rtl_pcipriv(hw);
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	u8 pcibridge_vendor = pcipriv->ndis_adapter.pcibridge_vendor;
+	u8 init_aspm;
+
+	ppsc->reg_rfps_level = 0;
+	ppsc->support_aspm = false;
+
+	/*Update PCI ASPM setting */
+	ppsc->const_amdpci_aspm = rtlpci->const_amdpci_aspm;
+	switch (rtlpci->const_pci_aspm) {
+	case 0:
+		/*No ASPM */
+		break;
+
+	case 1:
+		/*ASPM dynamically enabled/disable. */
+		ppsc->reg_rfps_level |= RT_RF_LPS_LEVEL_ASPM;
+		break;
+
+	case 2:
+		/*ASPM with Clock Req dynamically enabled/disable. */
+		ppsc->reg_rfps_level |= (RT_RF_LPS_LEVEL_ASPM |
+					 RT_RF_OFF_LEVL_CLK_REQ);
+		break;
+
+	case 3:
+		/* Always enable ASPM and Clock Req
+		 * from initialization to halt.
+		 */
+		ppsc->reg_rfps_level &= ~(RT_RF_LPS_LEVEL_ASPM);
+		ppsc->reg_rfps_level |= (RT_RF_PS_LEVEL_ALWAYS_ASPM |
+					 RT_RF_OFF_LEVL_CLK_REQ);
+		break;
+
+	case 4:
+		/* Always enable ASPM without Clock Req
+		 * from initialization to halt.
+		 */
+		ppsc->reg_rfps_level &= ~(RT_RF_LPS_LEVEL_ASPM |
+					  RT_RF_OFF_LEVL_CLK_REQ);
+		ppsc->reg_rfps_level |= RT_RF_PS_LEVEL_ALWAYS_ASPM;
+		break;
+	}
+
+	ppsc->reg_rfps_level |= RT_RF_OFF_LEVL_HALT_NIC;
+
+	/*Update Radio OFF setting */
+	switch (rtlpci->const_hwsw_rfoff_d3) {
+	case 1:
+		if (ppsc->reg_rfps_level & RT_RF_LPS_LEVEL_ASPM)
+			ppsc->reg_rfps_level |= RT_RF_OFF_LEVL_ASPM;
+		break;
+
+	case 2:
+		if (ppsc->reg_rfps_level & RT_RF_LPS_LEVEL_ASPM)
+			ppsc->reg_rfps_level |= RT_RF_OFF_LEVL_ASPM;
+		ppsc->reg_rfps_level |= RT_RF_OFF_LEVL_HALT_NIC;
+		break;
+
+	case 3:
+		ppsc->reg_rfps_level |= RT_RF_OFF_LEVL_PCI_D3;
+		break;
+	}
+
+	/*Set HW definition to determine if it supports ASPM. */
+	switch (rtlpci->const_support_pciaspm) {
+	case 0:{
+		/*Not support ASPM. */
+		bool support_aspm = false;
+
+		ppsc->support_aspm = support_aspm;
+		break;
+	}
+	case 1:{
+		/*Support ASPM. */
+		bool support_aspm = true;
+		bool support_backdoor = true;
+
+		ppsc->support_aspm = support_aspm;
+
+		/*if (priv->oem_id == RT_CID_TOSHIBA &&
+		 * !priv->ndis_adapter.amd_l1_patch)
+		 *  support_backdoor = false;
+		 */
+
+		ppsc->support_backdoor = support_backdoor;
+
+		break;
+	}
+	case 2:
+		/*ASPM value set by chipset. */
+		if (pcibridge_vendor == PCI_BRIDGE_VENDOR_INTEL) {
+			bool support_aspm = true;
+
+			ppsc->support_aspm = support_aspm;
+		}
+		break;
+	default:
+		pr_err("switch case %#x not processed\n",
+		       rtlpci->const_support_pciaspm);
+		break;
+	}
+
+	/* toshiba aspm issue, toshiba will set aspm selfly
+	 * so we should not set aspm in driver
+	 */
+	pci_read_config_byte(rtlpci->pdev, 0x80, &init_aspm);
+	if (rtlpriv->rtlhal.hw_type == HARDWARE_TYPE_RTL8192SE &&
+	    init_aspm == 0x43)
+		ppsc->support_aspm = false;
+}
+
+static bool _rtl_pci_platform_switch_device_pci_aspm(
+			struct ieee80211_hw *hw,
+			u8 value)
+{
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+
+	if (rtlhal->hw_type != HARDWARE_TYPE_RTL8192SE)
+		value |= 0x40;
+
+	pci_write_config_byte(rtlpci->pdev, 0x80, value);
+
+	return false;
+}
+
+/*When we set 0x01 to enable clk request. Set 0x0 to disable clk req.*/
+static void _rtl_pci_switch_clk_req(struct ieee80211_hw *hw, u8 value)
+{
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+
+	pci_write_config_byte(rtlpci->pdev, 0x81, value);
+
+	if (rtlhal->hw_type == HARDWARE_TYPE_RTL8192SE)
+		udelay(100);
+}
+
+/*Disable RTL8192SE ASPM & Disable Pci Bridge ASPM*/
+static void rtl_pci_disable_aspm(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci_priv *pcipriv = rtl_pcipriv(hw);
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	u8 pcibridge_vendor = pcipriv->ndis_adapter.pcibridge_vendor;
+	u8 num4bytes = pcipriv->ndis_adapter.num4bytes;
+	/*Retrieve original configuration settings. */
+	u8 linkctrl_reg = pcipriv->ndis_adapter.linkctrl_reg;
+	u16 pcibridge_linkctrlreg = pcipriv->ndis_adapter.pcibridge_linkctrlreg;
+	u16 aspmlevel = 0;
+	u8 tmp_u1b = 0;
+
+	if (!ppsc->support_aspm)
+		return;
+
+	if (pcibridge_vendor == PCI_BRIDGE_VENDOR_UNKNOWN) {
+		RT_TRACE(rtlpriv, COMP_POWER, DBG_TRACE,
+			 "PCI(Bridge) UNKNOWN\n");
+
+		return;
+	}
+
+	if (ppsc->reg_rfps_level & RT_RF_OFF_LEVL_CLK_REQ) {
+		RT_CLEAR_PS_LEVEL(ppsc, RT_RF_OFF_LEVL_CLK_REQ);
+		_rtl_pci_switch_clk_req(hw, 0x0);
+	}
+
+	/*for promising device will in L0 state after an I/O. */
+	pci_read_config_byte(rtlpci->pdev, 0x80, &tmp_u1b);
+
+	/*Set corresponding value. */
+	aspmlevel |= BIT(0) | BIT(1);
+	linkctrl_reg &= ~aspmlevel;
+	pcibridge_linkctrlreg &= ~(BIT(0) | BIT(1));
+
+	_rtl_pci_platform_switch_device_pci_aspm(hw, linkctrl_reg);
+	udelay(50);
+
+	/*4 Disable Pci Bridge ASPM */
+	pci_write_config_byte(rtlpci->pdev, (num4bytes << 2),
+			      pcibridge_linkctrlreg);
+
+	udelay(50);
+}
+
+/*
+ *Enable RTL8192SE ASPM & Enable Pci Bridge ASPM for
+ *power saving We should follow the sequence to enable
+ *RTL8192SE first then enable Pci Bridge ASPM
+ *or the system will show bluescreen.
+ */
+static void rtl_pci_enable_aspm(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci_priv *pcipriv = rtl_pcipriv(hw);
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	u8 pcibridge_vendor = pcipriv->ndis_adapter.pcibridge_vendor;
+	u8 num4bytes = pcipriv->ndis_adapter.num4bytes;
+	u16 aspmlevel;
+	u8 u_pcibridge_aspmsetting;
+	u8 u_device_aspmsetting;
+
+	if (!ppsc->support_aspm)
+		return;
+
+	if (pcibridge_vendor == PCI_BRIDGE_VENDOR_UNKNOWN) {
+		RT_TRACE(rtlpriv, COMP_POWER, DBG_TRACE,
+			 "PCI(Bridge) UNKNOWN\n");
+		return;
+	}
+
+	/*4 Enable Pci Bridge ASPM */
+
+	u_pcibridge_aspmsetting =
+	    pcipriv->ndis_adapter.pcibridge_linkctrlreg |
+	    rtlpci->const_hostpci_aspm_setting;
+
+	if (pcibridge_vendor == PCI_BRIDGE_VENDOR_INTEL)
+		u_pcibridge_aspmsetting &= ~BIT(0);
+
+	pci_write_config_byte(rtlpci->pdev, (num4bytes << 2),
+			      u_pcibridge_aspmsetting);
+
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+		 "PlatformEnableASPM(): Write reg[%x] = %x\n",
+		 (pcipriv->ndis_adapter.pcibridge_pciehdr_offset + 0x10),
+		 u_pcibridge_aspmsetting);
+
+	udelay(50);
+
+	/*Get ASPM level (with/without Clock Req) */
+	aspmlevel = rtlpci->const_devicepci_aspm_setting;
+	u_device_aspmsetting = pcipriv->ndis_adapter.linkctrl_reg;
+
+	/*_rtl_pci_platform_switch_device_pci_aspm(dev,*/
+	/*(priv->ndis_adapter.linkctrl_reg | ASPMLevel)); */
+
+	u_device_aspmsetting |= aspmlevel;
+
+	_rtl_pci_platform_switch_device_pci_aspm(hw, u_device_aspmsetting);
+
+	if (ppsc->reg_rfps_level & RT_RF_OFF_LEVL_CLK_REQ) {
+		_rtl_pci_switch_clk_req(hw, (ppsc->reg_rfps_level &
+					     RT_RF_OFF_LEVL_CLK_REQ) ? 1 : 0);
+		RT_SET_PS_LEVEL(ppsc, RT_RF_OFF_LEVL_CLK_REQ);
+	}
+	udelay(100);
+}
+
+static bool rtl_pci_get_amd_l1_patch(struct ieee80211_hw *hw)
+{
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+
+	bool status = false;
+	u8 offset_e0;
+	unsigned int offset_e4;
+
+	pci_write_config_byte(rtlpci->pdev, 0xe0, 0xa0);
+
+	pci_read_config_byte(rtlpci->pdev, 0xe0, &offset_e0);
+
+	if (offset_e0 == 0xA0) {
+		pci_read_config_dword(rtlpci->pdev, 0xe4, &offset_e4);
+		if (offset_e4 & BIT(23))
+			status = true;
+	}
+
+	return status;
+}
+
+static bool rtl_pci_check_buddy_priv(struct ieee80211_hw *hw,
+				     struct rtl_priv **buddy_priv)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci_priv *pcipriv = rtl_pcipriv(hw);
+	bool find_buddy_priv = false;
+	struct rtl_priv *tpriv;
+	struct rtl_pci_priv *tpcipriv = NULL;
+
+	if (!list_empty(&rtlpriv->glb_var->glb_priv_list)) {
+		list_for_each_entry(tpriv, &rtlpriv->glb_var->glb_priv_list,
+				    list) {
+			tpcipriv = (struct rtl_pci_priv *)tpriv->priv;
+			RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+				 "pcipriv->ndis_adapter.funcnumber %x\n",
+				pcipriv->ndis_adapter.funcnumber);
+			RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+				 "tpcipriv->ndis_adapter.funcnumber %x\n",
+				tpcipriv->ndis_adapter.funcnumber);
+
+			if ((pcipriv->ndis_adapter.busnumber ==
+			     tpcipriv->ndis_adapter.busnumber) &&
+			    (pcipriv->ndis_adapter.devnumber ==
+			    tpcipriv->ndis_adapter.devnumber) &&
+			    (pcipriv->ndis_adapter.funcnumber !=
+			    tpcipriv->ndis_adapter.funcnumber)) {
+				find_buddy_priv = true;
+				break;
+			}
+		}
+	}
+
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+		 "find_buddy_priv %d\n", find_buddy_priv);
+
+	if (find_buddy_priv)
+		*buddy_priv = tpriv;
+
+	return find_buddy_priv;
+}
+
+static void rtl_pci_get_linkcontrol_field(struct ieee80211_hw *hw)
+{
+	struct rtl_pci_priv *pcipriv = rtl_pcipriv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(pcipriv);
+	u8 capabilityoffset = pcipriv->ndis_adapter.pcibridge_pciehdr_offset;
+	u8 linkctrl_reg;
+	u8 num4bbytes;
+
+	num4bbytes = (capabilityoffset + 0x10) / 4;
+
+	/*Read  Link Control Register */
+	pci_read_config_byte(rtlpci->pdev, (num4bbytes << 2), &linkctrl_reg);
+
+	pcipriv->ndis_adapter.pcibridge_linkctrlreg = linkctrl_reg;
+}
+
+static void rtl_pci_parse_configuration(struct pci_dev *pdev,
+					struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci_priv *pcipriv = rtl_pcipriv(hw);
+
+	u8 tmp;
+	u16 linkctrl_reg;
+
+	/*Link Control Register */
+	pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &linkctrl_reg);
+	pcipriv->ndis_adapter.linkctrl_reg = (u8)linkctrl_reg;
+
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE, "Link Control Register =%x\n",
+		 pcipriv->ndis_adapter.linkctrl_reg);
+
+	pci_read_config_byte(pdev, 0x98, &tmp);
+	tmp |= BIT(4);
+	pci_write_config_byte(pdev, 0x98, tmp);
+
+	tmp = 0x17;
+	pci_write_config_byte(pdev, 0x70f, tmp);
+}
+
+static void rtl_pci_init_aspm(struct ieee80211_hw *hw)
+{
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+
+	_rtl_pci_update_default_setting(hw);
+
+	if (ppsc->reg_rfps_level & RT_RF_PS_LEVEL_ALWAYS_ASPM) {
+		/*Always enable ASPM & Clock Req. */
+		rtl_pci_enable_aspm(hw);
+		RT_SET_PS_LEVEL(ppsc, RT_RF_PS_LEVEL_ALWAYS_ASPM);
+	}
+}
+
+static void _rtl_pci_io_handler_init(struct device *dev,
+				     struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	rtlpriv->io.dev = dev;
+
+	rtlpriv->io.write8_async = pci_write8_async;
+	rtlpriv->io.write16_async = pci_write16_async;
+	rtlpriv->io.write32_async = pci_write32_async;
+
+	rtlpriv->io.read8_sync = pci_read8_sync;
+	rtlpriv->io.read16_sync = pci_read16_sync;
+	rtlpriv->io.read32_sync = pci_read32_sync;
+}
+
+static bool _rtl_update_earlymode_info(struct ieee80211_hw *hw,
+				       struct sk_buff *skb,
+				       struct rtl_tcb_desc *tcb_desc, u8 tid)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	struct sk_buff *next_skb;
+	u8 additionlen = FCS_LEN;
+
+	/* here open is 4, wep/tkip is 8, aes is 12*/
+	if (info->control.hw_key)
+		additionlen += info->control.hw_key->icv_len;
+
+	/* The most skb num is 6 */
+	tcb_desc->empkt_num = 0;
+	spin_lock_bh(&rtlpriv->locks.waitq_lock);
+	skb_queue_walk(&rtlpriv->mac80211.skb_waitq[tid], next_skb) {
+		struct ieee80211_tx_info *next_info;
+
+		next_info = IEEE80211_SKB_CB(next_skb);
+		if (next_info->flags & IEEE80211_TX_CTL_AMPDU) {
+			tcb_desc->empkt_len[tcb_desc->empkt_num] =
+				next_skb->len + additionlen;
+			tcb_desc->empkt_num++;
+		} else {
+			break;
+		}
+
+		if (skb_queue_is_last(&rtlpriv->mac80211.skb_waitq[tid],
+				      next_skb))
+			break;
+
+		if (tcb_desc->empkt_num >= rtlhal->max_earlymode_num)
+			break;
+	}
+	spin_unlock_bh(&rtlpriv->locks.waitq_lock);
+
+	return true;
+}
+
+/* just for early mode now */
+static void _rtl_pci_tx_chk_waitq(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	struct sk_buff *skb = NULL;
+	struct ieee80211_tx_info *info = NULL;
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	int tid;
+
+	if (!rtlpriv->rtlhal.earlymode_enable)
+		return;
+
+	if (rtlpriv->dm.supp_phymode_switch &&
+	    (rtlpriv->easy_concurrent_ctl.switch_in_process ||
+	    (rtlpriv->buddy_priv &&
+	    rtlpriv->buddy_priv->easy_concurrent_ctl.switch_in_process)))
+		return;
+	/* we just use em for BE/BK/VI/VO */
+	for (tid = 7; tid >= 0; tid--) {
+		u8 hw_queue = ac_to_hwq[rtl_tid_to_ac(tid)];
+		struct rtl8192_tx_ring *ring = &rtlpci->tx_ring[hw_queue];
+
+		while (!mac->act_scanning &&
+		       rtlpriv->psc.rfpwr_state == ERFON) {
+			struct rtl_tcb_desc tcb_desc;
+
+			memset(&tcb_desc, 0, sizeof(struct rtl_tcb_desc));
+			spin_lock_bh(&rtlpriv->locks.waitq_lock);
+			if (!skb_queue_empty(&mac->skb_waitq[tid]) &&
+			    (ring->entries - skb_queue_len(&ring->queue) >
+			     rtlhal->max_earlymode_num)) {
+				skb = skb_dequeue(&mac->skb_waitq[tid]);
+			} else {
+				spin_unlock_bh(&rtlpriv->locks.waitq_lock);
+				break;
+			}
+			spin_unlock_bh(&rtlpriv->locks.waitq_lock);
+
+			/* Some macaddr can't do early mode. like
+			 * multicast/broadcast/no_qos data
+			 */
+			info = IEEE80211_SKB_CB(skb);
+			if (info->flags & IEEE80211_TX_CTL_AMPDU)
+				_rtl_update_earlymode_info(hw, skb,
+							   &tcb_desc, tid);
+
+			rtlpriv->intf_ops->adapter_tx(hw, NULL, skb, &tcb_desc);
+		}
+	}
+}
+
+static void _rtl_pci_tx_isr(struct ieee80211_hw *hw, int prio)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+
+	struct rtl8192_tx_ring *ring = &rtlpci->tx_ring[prio];
+
+	while (skb_queue_len(&ring->queue)) {
+		struct sk_buff *skb;
+		struct ieee80211_tx_info *info;
+		__le16 fc;
+		u8 tid;
+		u8 *entry;
+
+		if (rtlpriv->use_new_trx_flow)
+			entry = (u8 *)(&ring->buffer_desc[ring->idx]);
+		else
+			entry = (u8 *)(&ring->desc[ring->idx]);
+
+		if (!rtlpriv->cfg->ops->is_tx_desc_closed(hw, prio, ring->idx))
+			return;
+		ring->idx = (ring->idx + 1) % ring->entries;
+
+		skb = __skb_dequeue(&ring->queue);
+		pci_unmap_single(rtlpci->pdev,
+				 rtlpriv->cfg->ops->get_desc(hw, (u8 *)entry, true,
+				 HW_DESC_TXBUFF_ADDR),
+				 skb->len, PCI_DMA_TODEVICE);
+
+		/* remove early mode header */
+		if (rtlpriv->rtlhal.earlymode_enable)
+			skb_pull(skb, EM_HDR_LEN);
+
+		RT_TRACE(rtlpriv, (COMP_INTR | COMP_SEND), DBG_TRACE,
+			 "new ring->idx:%d, free: skb_queue_len:%d, free: seq:%x\n",
+			 ring->idx,
+			 skb_queue_len(&ring->queue),
+			 *(u16 *)(skb->data + 22));
+
+		if (prio == TXCMD_QUEUE) {
+			dev_kfree_skb(skb);
+			goto tx_status_ok;
+		}
+
+		/* for sw LPS, just after NULL skb send out, we can
+		 * sure AP knows we are sleeping, we should not let
+		 * rf sleep
+		 */
+		fc = rtl_get_fc(skb);
+		if (ieee80211_is_nullfunc(fc)) {
+			if (ieee80211_has_pm(fc)) {
+				rtlpriv->mac80211.offchan_delay = true;
+				rtlpriv->psc.state_inap = true;
+			} else {
+				rtlpriv->psc.state_inap = false;
+			}
+		}
+		if (ieee80211_is_action(fc)) {
+			struct ieee80211_mgmt *action_frame =
+				(struct ieee80211_mgmt *)skb->data;
+			if (action_frame->u.action.u.ht_smps.action ==
+			    WLAN_HT_ACTION_SMPS) {
+				dev_kfree_skb(skb);
+				goto tx_status_ok;
+			}
+		}
+
+		/* update tid tx pkt num */
+		tid = rtl_get_tid(skb);
+		if (tid <= 7)
+			rtlpriv->link_info.tidtx_inperiod[tid]++;
+
+		info = IEEE80211_SKB_CB(skb);
+		ieee80211_tx_info_clear_status(info);
+
+		info->flags |= IEEE80211_TX_STAT_ACK;
+		/*info->status.rates[0].count = 1; */
+
+		ieee80211_tx_status_irqsafe(hw, skb);
+
+		if ((ring->entries - skb_queue_len(&ring->queue)) <= 4) {
+			RT_TRACE(rtlpriv, COMP_ERR, DBG_DMESG,
+				 "more desc left, wake skb_queue@%d, ring->idx = %d, skb_queue_len = 0x%x\n",
+				 prio, ring->idx,
+				 skb_queue_len(&ring->queue));
+
+			ieee80211_wake_queue(hw, skb_get_queue_mapping (skb));
+		}
+tx_status_ok:
+		skb = NULL;
+	}
+
+	if (((rtlpriv->link_info.num_rx_inperiod +
+	      rtlpriv->link_info.num_tx_inperiod) > 8) ||
+	      (rtlpriv->link_info.num_rx_inperiod > 2))
+		rtl_lps_leave(hw);
+}
+
+static int _rtl_pci_init_one_rxdesc(struct ieee80211_hw *hw,
+				    struct sk_buff *new_skb, u8 *entry,
+				    int rxring_idx, int desc_idx)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	u32 bufferaddress;
+	u8 tmp_one = 1;
+	struct sk_buff *skb;
+
+	if (likely(new_skb)) {
+		skb = new_skb;
+		goto remap;
+	}
+	skb = dev_alloc_skb(rtlpci->rxbuffersize);
+	if (!skb)
+		return 0;
+
+remap:
+	/* just set skb->cb to mapping addr for pci_unmap_single use */
+	*((dma_addr_t *)skb->cb) =
+		pci_map_single(rtlpci->pdev, skb_tail_pointer(skb),
+			       rtlpci->rxbuffersize, PCI_DMA_FROMDEVICE);
+	bufferaddress = *((dma_addr_t *)skb->cb);
+	if (pci_dma_mapping_error(rtlpci->pdev, bufferaddress))
+		return 0;
+	rtlpci->rx_ring[rxring_idx].rx_buf[desc_idx] = skb;
+	if (rtlpriv->use_new_trx_flow) {
+		/* skb->cb may be 64 bit address */
+		rtlpriv->cfg->ops->set_desc(hw, (u8 *)entry, false,
+					    HW_DESC_RX_PREPARE,
+					    (u8 *)(dma_addr_t *)skb->cb);
+	} else {
+		rtlpriv->cfg->ops->set_desc(hw, (u8 *)entry, false,
+					    HW_DESC_RXBUFF_ADDR,
+					    (u8 *)&bufferaddress);
+		rtlpriv->cfg->ops->set_desc(hw, (u8 *)entry, false,
+					    HW_DESC_RXPKT_LEN,
+					    (u8 *)&rtlpci->rxbuffersize);
+		rtlpriv->cfg->ops->set_desc(hw, (u8 *)entry, false,
+					    HW_DESC_RXOWN,
+					    (u8 *)&tmp_one);
+	}
+	return 1;
+}
+
+/* inorder to receive 8K AMSDU we have set skb to
+ * 9100bytes in init rx ring, but if this packet is
+ * not a AMSDU, this large packet will be sent to
+ * TCP/IP directly, this cause big packet ping fail
+ * like: "ping -s 65507", so here we will realloc skb
+ * based on the true size of packet, Mac80211
+ * Probably will do it better, but does not yet.
+ *
+ * Some platform will fail when alloc skb sometimes.
+ * in this condition, we will send the old skb to
+ * mac80211 directly, this will not cause any other
+ * issues, but only this packet will be lost by TCP/IP
+ */
+static void _rtl_pci_rx_to_mac80211(struct ieee80211_hw *hw,
+				    struct sk_buff *skb,
+				    struct ieee80211_rx_status rx_status)
+{
+	if (unlikely(!rtl_action_proc(hw, skb, false))) {
+		dev_kfree_skb_any(skb);
+	} else {
+		struct sk_buff *uskb = NULL;
+
+		uskb = dev_alloc_skb(skb->len + 128);
+		if (likely(uskb)) {
+			memcpy(IEEE80211_SKB_RXCB(uskb), &rx_status,
+			       sizeof(rx_status));
+			skb_put_data(uskb, skb->data, skb->len);
+			dev_kfree_skb_any(skb);
+			ieee80211_rx_irqsafe(hw, uskb);
+		} else {
+			ieee80211_rx_irqsafe(hw, skb);
+		}
+	}
+}
+
+/*hsisr interrupt handler*/
+static void _rtl_pci_hs_interrupt(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+
+	rtl_write_byte(rtlpriv, rtlpriv->cfg->maps[MAC_HSISR],
+		       rtl_read_byte(rtlpriv, rtlpriv->cfg->maps[MAC_HSISR]) |
+		       rtlpci->sys_irq_mask);
+}
+
+static void _rtl_pci_rx_interrupt(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	int rxring_idx = RTL_PCI_RX_MPDU_QUEUE;
+	struct ieee80211_rx_status rx_status = { 0 };
+	unsigned int count = rtlpci->rxringcount;
+	u8 own;
+	u8 tmp_one;
+	bool unicast = false;
+	u8 hw_queue = 0;
+	unsigned int rx_remained_cnt = 0;
+	struct rtl_stats stats = {
+		.signal = 0,
+		.rate = 0,
+	};
+
+	/*RX NORMAL PKT */
+	while (count--) {
+		struct ieee80211_hdr *hdr;
+		__le16 fc;
+		u16 len;
+		/*rx buffer descriptor */
+		struct rtl_rx_buffer_desc *buffer_desc = NULL;
+		/*if use new trx flow, it means wifi info */
+		struct rtl_rx_desc *pdesc = NULL;
+		/*rx pkt */
+		struct sk_buff *skb = rtlpci->rx_ring[rxring_idx].rx_buf[
+				      rtlpci->rx_ring[rxring_idx].idx];
+		struct sk_buff *new_skb;
+
+		if (rtlpriv->use_new_trx_flow) {
+			if (rx_remained_cnt == 0)
+				rx_remained_cnt =
+				rtlpriv->cfg->ops->rx_desc_buff_remained_cnt(hw,
+								      hw_queue);
+			if (rx_remained_cnt == 0)
+				return;
+			buffer_desc = &rtlpci->rx_ring[rxring_idx].buffer_desc[
+				rtlpci->rx_ring[rxring_idx].idx];
+			pdesc = (struct rtl_rx_desc *)skb->data;
+		} else {	/* rx descriptor */
+			pdesc = &rtlpci->rx_ring[rxring_idx].desc[
+				rtlpci->rx_ring[rxring_idx].idx];
+
+			own = (u8)rtlpriv->cfg->ops->get_desc(hw, (u8 *)pdesc,
+							      false,
+							      HW_DESC_OWN);
+			if (own) /* wait data to be filled by hardware */
+				return;
+		}
+
+		/* Reaching this point means: data is filled already
+		 * AAAAAAttention !!!
+		 * We can NOT access 'skb' before 'pci_unmap_single'
+		 */
+		pci_unmap_single(rtlpci->pdev, *((dma_addr_t *)skb->cb),
+				 rtlpci->rxbuffersize, PCI_DMA_FROMDEVICE);
+
+		/* get a new skb - if fail, old one will be reused */
+		new_skb = dev_alloc_skb(rtlpci->rxbuffersize);
+		if (unlikely(!new_skb))
+			goto no_new;
+		memset(&rx_status, 0, sizeof(rx_status));
+		rtlpriv->cfg->ops->query_rx_desc(hw, &stats,
+						 &rx_status, (u8 *)pdesc, skb);
+
+		if (rtlpriv->use_new_trx_flow)
+			rtlpriv->cfg->ops->rx_check_dma_ok(hw,
+							   (u8 *)buffer_desc,
+							   hw_queue);
+
+		len = rtlpriv->cfg->ops->get_desc(hw, (u8 *)pdesc, false,
+						  HW_DESC_RXPKT_LEN);
+
+		if (skb->end - skb->tail > len) {
+			skb_put(skb, len);
+			if (rtlpriv->use_new_trx_flow)
+				skb_reserve(skb, stats.rx_drvinfo_size +
+					    stats.rx_bufshift + 24);
+			else
+				skb_reserve(skb, stats.rx_drvinfo_size +
+					    stats.rx_bufshift);
+		} else {
+			RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING,
+				 "skb->end - skb->tail = %d, len is %d\n",
+				 skb->end - skb->tail, len);
+			dev_kfree_skb_any(skb);
+			goto new_trx_end;
+		}
+		/* handle command packet here */
+		if (rtlpriv->cfg->ops->rx_command_packet &&
+		    rtlpriv->cfg->ops->rx_command_packet(hw, &stats, skb)) {
+			dev_kfree_skb_any(skb);
+			goto new_trx_end;
+		}
+
+		/*
+		 * NOTICE This can not be use for mac80211,
+		 * this is done in mac80211 code,
+		 * if done here sec DHCP will fail
+		 * skb_trim(skb, skb->len - 4);
+		 */
+
+		hdr = rtl_get_hdr(skb);
+		fc = rtl_get_fc(skb);
+
+		if (!stats.crc && !stats.hwerror) {
+			memcpy(IEEE80211_SKB_RXCB(skb), &rx_status,
+			       sizeof(rx_status));
+
+			if (is_broadcast_ether_addr(hdr->addr1)) {
+				;/*TODO*/
+			} else if (is_multicast_ether_addr(hdr->addr1)) {
+				;/*TODO*/
+			} else {
+				unicast = true;
+				rtlpriv->stats.rxbytesunicast += skb->len;
+			}
+			rtl_is_special_data(hw, skb, false, true);
+
+			if (ieee80211_is_data(fc)) {
+				rtlpriv->cfg->ops->led_control(hw, LED_CTL_RX);
+				if (unicast)
+					rtlpriv->link_info.num_rx_inperiod++;
+			}
+
+			rtl_collect_scan_list(hw, skb);
+
+			/* static bcn for roaming */
+			rtl_beacon_statistic(hw, skb);
+			rtl_p2p_info(hw, (void *)skb->data, skb->len);
+			/* for sw lps */
+			rtl_swlps_beacon(hw, (void *)skb->data, skb->len);
+			rtl_recognize_peer(hw, (void *)skb->data, skb->len);
+			if ((rtlpriv->mac80211.opmode == NL80211_IFTYPE_AP) &&
+			    (rtlpriv->rtlhal.current_bandtype ==
+			     BAND_ON_2_4G) &&
+			    (ieee80211_is_beacon(fc) ||
+			     ieee80211_is_probe_resp(fc))) {
+				dev_kfree_skb_any(skb);
+			} else {
+				rtl_check_beacon_key(hw, (void *)skb->data,
+						     skb->len);
+				_rtl_pci_rx_to_mac80211(hw, skb, rx_status);
+			}
+		} else {
+			dev_kfree_skb_any(skb);
+		}
+new_trx_end:
+		if (rtlpriv->use_new_trx_flow) {
+			rtlpci->rx_ring[hw_queue].next_rx_rp += 1;
+			rtlpci->rx_ring[hw_queue].next_rx_rp %=
+					RTL_PCI_MAX_RX_COUNT;
+
+			rx_remained_cnt--;
+			rtl_write_word(rtlpriv, 0x3B4,
+				       rtlpci->rx_ring[hw_queue].next_rx_rp);
+		}
+		if (((rtlpriv->link_info.num_rx_inperiod +
+		      rtlpriv->link_info.num_tx_inperiod) > 8) ||
+		      (rtlpriv->link_info.num_rx_inperiod > 2))
+			rtl_lps_leave(hw);
+		skb = new_skb;
+no_new:
+		if (rtlpriv->use_new_trx_flow) {
+			_rtl_pci_init_one_rxdesc(hw, skb, (u8 *)buffer_desc,
+						 rxring_idx,
+						 rtlpci->rx_ring[rxring_idx].idx);
+		} else {
+			_rtl_pci_init_one_rxdesc(hw, skb, (u8 *)pdesc,
+						 rxring_idx,
+						 rtlpci->rx_ring[rxring_idx].idx);
+			if (rtlpci->rx_ring[rxring_idx].idx ==
+			    rtlpci->rxringcount - 1)
+				rtlpriv->cfg->ops->set_desc(hw, (u8 *)pdesc,
+							    false,
+							    HW_DESC_RXERO,
+							    (u8 *)&tmp_one);
+		}
+		rtlpci->rx_ring[rxring_idx].idx =
+				(rtlpci->rx_ring[rxring_idx].idx + 1) %
+				rtlpci->rxringcount;
+	}
+}
+
+static irqreturn_t _rtl_pci_interrupt(int irq, void *dev_id)
+{
+	struct ieee80211_hw *hw = dev_id;
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	unsigned long flags;
+	u32 inta = 0;
+	u32 intb = 0;
+	u32 intc = 0;
+	u32 intd = 0;
+	irqreturn_t ret = IRQ_HANDLED;
+
+	if (rtlpci->irq_enabled == 0)
+		return ret;
+
+	spin_lock_irqsave(&rtlpriv->locks.irq_th_lock, flags);
+	rtlpriv->cfg->ops->disable_interrupt(hw);
+
+	/*read ISR: 4/8bytes */
+	rtlpriv->cfg->ops->interrupt_recognized(hw, &inta, &intb, &intc, &intd);
+
+	/*Shared IRQ or HW disappeared */
+	if (!inta || inta == 0xffff)
+		goto done;
+
+	/*<1> beacon related */
+	if (inta & rtlpriv->cfg->maps[RTL_IMR_TBDOK]) {
+		RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
+			 "beacon ok interrupt!\n");
+	}
+
+	if (unlikely(inta & rtlpriv->cfg->maps[RTL_IMR_TBDER])) {
+		RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
+			 "beacon err interrupt!\n");
+	}
+
+	if (inta & rtlpriv->cfg->maps[RTL_IMR_BDOK])
+		RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE, "beacon interrupt!\n");
+
+	if (inta & rtlpriv->cfg->maps[RTL_IMR_BCNINT]) {
+		RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
+			 "prepare beacon for interrupt!\n");
+		tasklet_schedule(&rtlpriv->works.irq_prepare_bcn_tasklet);
+	}
+
+	/*<2> Tx related */
+	if (unlikely(intb & rtlpriv->cfg->maps[RTL_IMR_TXFOVW]))
+		RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING, "IMR_TXFOVW!\n");
+
+	if (inta & rtlpriv->cfg->maps[RTL_IMR_MGNTDOK]) {
+		RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
+			 "Manage ok interrupt!\n");
+		_rtl_pci_tx_isr(hw, MGNT_QUEUE);
+	}
+
+	if (inta & rtlpriv->cfg->maps[RTL_IMR_HIGHDOK]) {
+		RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
+			 "HIGH_QUEUE ok interrupt!\n");
+		_rtl_pci_tx_isr(hw, HIGH_QUEUE);
+	}
+
+	if (inta & rtlpriv->cfg->maps[RTL_IMR_BKDOK]) {
+		rtlpriv->link_info.num_tx_inperiod++;
+
+		RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
+			 "BK Tx OK interrupt!\n");
+		_rtl_pci_tx_isr(hw, BK_QUEUE);
+	}
+
+	if (inta & rtlpriv->cfg->maps[RTL_IMR_BEDOK]) {
+		rtlpriv->link_info.num_tx_inperiod++;
+
+		RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
+			 "BE TX OK interrupt!\n");
+		_rtl_pci_tx_isr(hw, BE_QUEUE);
+	}
+
+	if (inta & rtlpriv->cfg->maps[RTL_IMR_VIDOK]) {
+		rtlpriv->link_info.num_tx_inperiod++;
+
+		RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
+			 "VI TX OK interrupt!\n");
+		_rtl_pci_tx_isr(hw, VI_QUEUE);
+	}
+
+	if (inta & rtlpriv->cfg->maps[RTL_IMR_VODOK]) {
+		rtlpriv->link_info.num_tx_inperiod++;
+
+		RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
+			 "Vo TX OK interrupt!\n");
+		_rtl_pci_tx_isr(hw, VO_QUEUE);
+	}
+
+	if (rtlhal->hw_type == HARDWARE_TYPE_RTL8822BE) {
+		if (intd & rtlpriv->cfg->maps[RTL_IMR_H2CDOK]) {
+			rtlpriv->link_info.num_tx_inperiod++;
+
+			RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
+				 "H2C TX OK interrupt!\n");
+			_rtl_pci_tx_isr(hw, H2C_QUEUE);
+		}
+	}
+
+	if (rtlhal->hw_type == HARDWARE_TYPE_RTL8192SE) {
+		if (inta & rtlpriv->cfg->maps[RTL_IMR_COMDOK]) {
+			rtlpriv->link_info.num_tx_inperiod++;
+
+			RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
+				 "CMD TX OK interrupt!\n");
+			_rtl_pci_tx_isr(hw, TXCMD_QUEUE);
+		}
+	}
+
+	/*<3> Rx related */
+	if (inta & rtlpriv->cfg->maps[RTL_IMR_ROK]) {
+		RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE, "Rx ok interrupt!\n");
+		_rtl_pci_rx_interrupt(hw);
+	}
+
+	if (unlikely(inta & rtlpriv->cfg->maps[RTL_IMR_RDU])) {
+		RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING,
+			 "rx descriptor unavailable!\n");
+		_rtl_pci_rx_interrupt(hw);
+	}
+
+	if (unlikely(intb & rtlpriv->cfg->maps[RTL_IMR_RXFOVW])) {
+		RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING, "rx overflow !\n");
+		_rtl_pci_rx_interrupt(hw);
+	}
+
+	/*<4> fw related*/
+	if (rtlhal->hw_type == HARDWARE_TYPE_RTL8723AE) {
+		if (inta & rtlpriv->cfg->maps[RTL_IMR_C2HCMD]) {
+			RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
+				 "firmware interrupt!\n");
+			queue_delayed_work(rtlpriv->works.rtl_wq,
+					   &rtlpriv->works.fwevt_wq, 0);
+		}
+	}
+
+	/*<5> hsisr related*/
+	/* Only 8188EE & 8723BE Supported.
+	 * If Other ICs Come in, System will corrupt,
+	 * because maps[RTL_IMR_HSISR_IND] & maps[MAC_HSISR]
+	 * are not initialized
+	 */
+	if (rtlhal->hw_type == HARDWARE_TYPE_RTL8188EE ||
+	    rtlhal->hw_type == HARDWARE_TYPE_RTL8723BE) {
+		if (unlikely(inta & rtlpriv->cfg->maps[RTL_IMR_HSISR_IND])) {
+			RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
+				 "hsisr interrupt!\n");
+			_rtl_pci_hs_interrupt(hw);
+		}
+	}
+
+	if (rtlpriv->rtlhal.earlymode_enable)
+		tasklet_schedule(&rtlpriv->works.irq_tasklet);
+
+done:
+	rtlpriv->cfg->ops->enable_interrupt(hw);
+	spin_unlock_irqrestore(&rtlpriv->locks.irq_th_lock, flags);
+	return ret;
+}
+
+static void _rtl_pci_irq_tasklet(struct ieee80211_hw *hw)
+{
+	_rtl_pci_tx_chk_waitq(hw);
+}
+
+static void _rtl_pci_prepare_bcn_tasklet(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct rtl8192_tx_ring *ring = NULL;
+	struct ieee80211_hdr *hdr = NULL;
+	struct ieee80211_tx_info *info = NULL;
+	struct sk_buff *pskb = NULL;
+	struct rtl_tx_desc *pdesc = NULL;
+	struct rtl_tcb_desc tcb_desc;
+	/*This is for new trx flow*/
+	struct rtl_tx_buffer_desc *pbuffer_desc = NULL;
+	u8 temp_one = 1;
+	u8 *entry;
+
+	memset(&tcb_desc, 0, sizeof(struct rtl_tcb_desc));
+	ring = &rtlpci->tx_ring[BEACON_QUEUE];
+	pskb = __skb_dequeue(&ring->queue);
+	if (rtlpriv->use_new_trx_flow)
+		entry = (u8 *)(&ring->buffer_desc[ring->idx]);
+	else
+		entry = (u8 *)(&ring->desc[ring->idx]);
+	if (pskb) {
+		pci_unmap_single(rtlpci->pdev,
+				 rtlpriv->cfg->ops->get_desc(
+				 hw, (u8 *)entry, true, HW_DESC_TXBUFF_ADDR),
+				 pskb->len, PCI_DMA_TODEVICE);
+		kfree_skb(pskb);
+	}
+
+	/*NB: the beacon data buffer must be 32-bit aligned. */
+	pskb = ieee80211_beacon_get(hw, mac->vif);
+	if (!pskb)
+		return;
+	hdr = rtl_get_hdr(pskb);
+	info = IEEE80211_SKB_CB(pskb);
+	pdesc = &ring->desc[0];
+	if (rtlpriv->use_new_trx_flow)
+		pbuffer_desc = &ring->buffer_desc[0];
+
+	rtlpriv->cfg->ops->fill_tx_desc(hw, hdr, (u8 *)pdesc,
+					(u8 *)pbuffer_desc, info, NULL, pskb,
+					BEACON_QUEUE, &tcb_desc);
+
+	__skb_queue_tail(&ring->queue, pskb);
+
+	if (rtlpriv->use_new_trx_flow) {
+		temp_one = 4;
+		rtlpriv->cfg->ops->set_desc(hw, (u8 *)pbuffer_desc, true,
+					    HW_DESC_OWN, (u8 *)&temp_one);
+	} else {
+		rtlpriv->cfg->ops->set_desc(hw, (u8 *)pdesc, true, HW_DESC_OWN,
+					    &temp_one);
+	}
+}
+
+static void _rtl_pci_init_trx_var(struct ieee80211_hw *hw)
+{
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+	u8 i;
+	u16 desc_num;
+
+	if (rtlhal->hw_type == HARDWARE_TYPE_RTL8192EE)
+		desc_num = TX_DESC_NUM_92E;
+	else if (rtlhal->hw_type == HARDWARE_TYPE_RTL8822BE)
+		desc_num = TX_DESC_NUM_8822B;
+	else
+		desc_num = RT_TXDESC_NUM;
+
+	for (i = 0; i < RTL_PCI_MAX_TX_QUEUE_COUNT; i++)
+		rtlpci->txringcount[i] = desc_num;
+
+	/*
+	 *we just alloc 2 desc for beacon queue,
+	 *because we just need first desc in hw beacon.
+	 */
+	rtlpci->txringcount[BEACON_QUEUE] = 2;
+
+	/*BE queue need more descriptor for performance
+	 *consideration or, No more tx desc will happen,
+	 *and may cause mac80211 mem leakage.
+	 */
+	if (!rtl_priv(hw)->use_new_trx_flow)
+		rtlpci->txringcount[BE_QUEUE] = RT_TXDESC_NUM_BE_QUEUE;
+
+	rtlpci->rxbuffersize = 9100;	/*2048/1024; */
+	rtlpci->rxringcount = RTL_PCI_MAX_RX_COUNT;	/*64; */
+}
+
+static void _rtl_pci_init_struct(struct ieee80211_hw *hw,
+				 struct pci_dev *pdev)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+
+	rtlpci->up_first_time = true;
+	rtlpci->being_init_adapter = false;
+
+	rtlhal->hw = hw;
+	rtlpci->pdev = pdev;
+
+	/*Tx/Rx related var */
+	_rtl_pci_init_trx_var(hw);
+
+	/*IBSS*/
+	mac->beacon_interval = 100;
+
+	/*AMPDU*/
+	mac->min_space_cfg = 0;
+	mac->max_mss_density = 0;
+	/*set sane AMPDU defaults */
+	mac->current_ampdu_density = 7;
+	mac->current_ampdu_factor = 3;
+
+	/*Retry Limit*/
+	mac->retry_short = 7;
+	mac->retry_long = 7;
+
+	/*QOS*/
+	rtlpci->acm_method = EACMWAY2_SW;
+
+	/*task */
+	tasklet_init(&rtlpriv->works.irq_tasklet,
+		     (void (*)(unsigned long))_rtl_pci_irq_tasklet,
+		     (unsigned long)hw);
+	tasklet_init(&rtlpriv->works.irq_prepare_bcn_tasklet,
+		     (void (*)(unsigned long))_rtl_pci_prepare_bcn_tasklet,
+		     (unsigned long)hw);
+	INIT_WORK(&rtlpriv->works.lps_change_work,
+		  rtl_lps_change_work_callback);
+}
+
+static int _rtl_pci_init_tx_ring(struct ieee80211_hw *hw,
+				 unsigned int prio, unsigned int entries)
+{
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_tx_buffer_desc *buffer_desc;
+	struct rtl_tx_desc *desc;
+	dma_addr_t buffer_desc_dma, desc_dma;
+	u32 nextdescaddress;
+	int i;
+
+	/* alloc tx buffer desc for new trx flow*/
+	if (rtlpriv->use_new_trx_flow) {
+		buffer_desc =
+		   pci_zalloc_consistent(rtlpci->pdev,
+					 sizeof(*buffer_desc) * entries,
+					 &buffer_desc_dma);
+
+		if (!buffer_desc || (unsigned long)buffer_desc & 0xFF) {
+			pr_err("Cannot allocate TX ring (prio = %d)\n",
+			       prio);
+			return -ENOMEM;
+		}
+
+		rtlpci->tx_ring[prio].buffer_desc = buffer_desc;
+		rtlpci->tx_ring[prio].buffer_desc_dma = buffer_desc_dma;
+
+		rtlpci->tx_ring[prio].cur_tx_rp = 0;
+		rtlpci->tx_ring[prio].cur_tx_wp = 0;
+	}
+
+	/* alloc dma for this ring */
+	desc = pci_zalloc_consistent(rtlpci->pdev,
+				     sizeof(*desc) * entries, &desc_dma);
+
+	if (!desc || (unsigned long)desc & 0xFF) {
+		pr_err("Cannot allocate TX ring (prio = %d)\n", prio);
+		return -ENOMEM;
+	}
+
+	rtlpci->tx_ring[prio].desc = desc;
+	rtlpci->tx_ring[prio].dma = desc_dma;
+
+	rtlpci->tx_ring[prio].idx = 0;
+	rtlpci->tx_ring[prio].entries = entries;
+	skb_queue_head_init(&rtlpci->tx_ring[prio].queue);
+
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD, "queue:%d, ring_addr:%p\n",
+		 prio, desc);
+
+	/* init every desc in this ring */
+	if (!rtlpriv->use_new_trx_flow) {
+		for (i = 0; i < entries; i++) {
+			nextdescaddress = (u32)desc_dma +
+					  ((i +	1) % entries) *
+					  sizeof(*desc);
+
+			rtlpriv->cfg->ops->set_desc(hw, (u8 *)&desc[i],
+						    true,
+						    HW_DESC_TX_NEXTDESC_ADDR,
+						    (u8 *)&nextdescaddress);
+		}
+	}
+	return 0;
+}
+
+static int _rtl_pci_init_rx_ring(struct ieee80211_hw *hw, int rxring_idx)
+{
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	int i;
+
+	if (rtlpriv->use_new_trx_flow) {
+		struct rtl_rx_buffer_desc *entry = NULL;
+		/* alloc dma for this ring */
+		rtlpci->rx_ring[rxring_idx].buffer_desc =
+		    pci_zalloc_consistent(rtlpci->pdev,
+					  sizeof(*rtlpci->rx_ring[rxring_idx].buffer_desc) *
+						 rtlpci->rxringcount,
+					  &rtlpci->rx_ring[rxring_idx].dma);
+		if (!rtlpci->rx_ring[rxring_idx].buffer_desc ||
+		    (ulong)rtlpci->rx_ring[rxring_idx].buffer_desc & 0xFF) {
+			pr_err("Cannot allocate RX ring\n");
+			return -ENOMEM;
+		}
+
+		/* init every desc in this ring */
+		rtlpci->rx_ring[rxring_idx].idx = 0;
+		for (i = 0; i < rtlpci->rxringcount; i++) {
+			entry = &rtlpci->rx_ring[rxring_idx].buffer_desc[i];
+			if (!_rtl_pci_init_one_rxdesc(hw, NULL, (u8 *)entry,
+						      rxring_idx, i))
+				return -ENOMEM;
+		}
+	} else {
+		struct rtl_rx_desc *entry = NULL;
+		u8 tmp_one = 1;
+		/* alloc dma for this ring */
+		rtlpci->rx_ring[rxring_idx].desc =
+		    pci_zalloc_consistent(rtlpci->pdev,
+					  sizeof(*rtlpci->rx_ring[rxring_idx].desc) *
+					  rtlpci->rxringcount,
+					  &rtlpci->rx_ring[rxring_idx].dma);
+		if (!rtlpci->rx_ring[rxring_idx].desc ||
+		    (unsigned long)rtlpci->rx_ring[rxring_idx].desc & 0xFF) {
+			pr_err("Cannot allocate RX ring\n");
+			return -ENOMEM;
+		}
+
+		/* init every desc in this ring */
+		rtlpci->rx_ring[rxring_idx].idx = 0;
+
+		for (i = 0; i < rtlpci->rxringcount; i++) {
+			entry = &rtlpci->rx_ring[rxring_idx].desc[i];
+			if (!_rtl_pci_init_one_rxdesc(hw, NULL, (u8 *)entry,
+						      rxring_idx, i))
+				return -ENOMEM;
+		}
+
+		rtlpriv->cfg->ops->set_desc(hw, (u8 *)entry, false,
+					    HW_DESC_RXERO, &tmp_one);
+	}
+	return 0;
+}
+
+static void _rtl_pci_free_tx_ring(struct ieee80211_hw *hw,
+				  unsigned int prio)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	struct rtl8192_tx_ring *ring = &rtlpci->tx_ring[prio];
+
+	/* free every desc in this ring */
+	while (skb_queue_len(&ring->queue)) {
+		u8 *entry;
+		struct sk_buff *skb = __skb_dequeue(&ring->queue);
+
+		if (rtlpriv->use_new_trx_flow)
+			entry = (u8 *)(&ring->buffer_desc[ring->idx]);
+		else
+			entry = (u8 *)(&ring->desc[ring->idx]);
+
+		pci_unmap_single(rtlpci->pdev,
+				 rtlpriv->cfg->ops->get_desc(hw, (u8 *)entry,
+						   true,
+						   HW_DESC_TXBUFF_ADDR),
+				 skb->len, PCI_DMA_TODEVICE);
+		kfree_skb(skb);
+		ring->idx = (ring->idx + 1) % ring->entries;
+	}
+
+	/* free dma of this ring */
+	pci_free_consistent(rtlpci->pdev,
+			    sizeof(*ring->desc) * ring->entries,
+			    ring->desc, ring->dma);
+	ring->desc = NULL;
+	if (rtlpriv->use_new_trx_flow) {
+		pci_free_consistent(rtlpci->pdev,
+				    sizeof(*ring->buffer_desc) * ring->entries,
+				    ring->buffer_desc, ring->buffer_desc_dma);
+		ring->buffer_desc = NULL;
+	}
+}
+
+static void _rtl_pci_free_rx_ring(struct ieee80211_hw *hw, int rxring_idx)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	int i;
+
+	/* free every desc in this ring */
+	for (i = 0; i < rtlpci->rxringcount; i++) {
+		struct sk_buff *skb = rtlpci->rx_ring[rxring_idx].rx_buf[i];
+
+		if (!skb)
+			continue;
+		pci_unmap_single(rtlpci->pdev, *((dma_addr_t *)skb->cb),
+				 rtlpci->rxbuffersize, PCI_DMA_FROMDEVICE);
+		kfree_skb(skb);
+	}
+
+	/* free dma of this ring */
+	if (rtlpriv->use_new_trx_flow) {
+		pci_free_consistent(rtlpci->pdev,
+				    sizeof(*rtlpci->rx_ring[rxring_idx].buffer_desc) *
+				    rtlpci->rxringcount,
+				    rtlpci->rx_ring[rxring_idx].buffer_desc,
+				    rtlpci->rx_ring[rxring_idx].dma);
+		rtlpci->rx_ring[rxring_idx].buffer_desc = NULL;
+	} else {
+		pci_free_consistent(rtlpci->pdev,
+				    sizeof(*rtlpci->rx_ring[rxring_idx].desc) *
+				    rtlpci->rxringcount,
+				    rtlpci->rx_ring[rxring_idx].desc,
+				    rtlpci->rx_ring[rxring_idx].dma);
+		rtlpci->rx_ring[rxring_idx].desc = NULL;
+	}
+}
+
+static int _rtl_pci_init_trx_ring(struct ieee80211_hw *hw)
+{
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	int ret;
+	int i, rxring_idx;
+
+	/* rxring_idx 0:RX_MPDU_QUEUE
+	 * rxring_idx 1:RX_CMD_QUEUE
+	 */
+	for (rxring_idx = 0; rxring_idx < RTL_PCI_MAX_RX_QUEUE; rxring_idx++) {
+		ret = _rtl_pci_init_rx_ring(hw, rxring_idx);
+		if (ret)
+			return ret;
+	}
+
+	for (i = 0; i < RTL_PCI_MAX_TX_QUEUE_COUNT; i++) {
+		ret = _rtl_pci_init_tx_ring(hw, i, rtlpci->txringcount[i]);
+		if (ret)
+			goto err_free_rings;
+	}
+
+	return 0;
+
+err_free_rings:
+	for (rxring_idx = 0; rxring_idx < RTL_PCI_MAX_RX_QUEUE; rxring_idx++)
+		_rtl_pci_free_rx_ring(hw, rxring_idx);
+
+	for (i = 0; i < RTL_PCI_MAX_TX_QUEUE_COUNT; i++)
+		if (rtlpci->tx_ring[i].desc ||
+		    rtlpci->tx_ring[i].buffer_desc)
+			_rtl_pci_free_tx_ring(hw, i);
+
+	return 1;
+}
+
+static int _rtl_pci_deinit_trx_ring(struct ieee80211_hw *hw)
+{
+	u32 i, rxring_idx;
+
+	/*free rx rings */
+	for (rxring_idx = 0; rxring_idx < RTL_PCI_MAX_RX_QUEUE; rxring_idx++)
+		_rtl_pci_free_rx_ring(hw, rxring_idx);
+
+	/*free tx rings */
+	for (i = 0; i < RTL_PCI_MAX_TX_QUEUE_COUNT; i++)
+		_rtl_pci_free_tx_ring(hw, i);
+
+	return 0;
+}
+
+int rtl_pci_reset_trx_ring(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	int i, rxring_idx;
+	unsigned long flags;
+	u8 tmp_one = 1;
+	u32 bufferaddress;
+	/* rxring_idx 0:RX_MPDU_QUEUE */
+	/* rxring_idx 1:RX_CMD_QUEUE */
+	for (rxring_idx = 0; rxring_idx < RTL_PCI_MAX_RX_QUEUE; rxring_idx++) {
+		/* force the rx_ring[RX_MPDU_QUEUE/
+		 * RX_CMD_QUEUE].idx to the first one
+		 * new trx flow, do nothing
+		 */
+		if (!rtlpriv->use_new_trx_flow &&
+		    rtlpci->rx_ring[rxring_idx].desc) {
+			struct rtl_rx_desc *entry = NULL;
+
+			rtlpci->rx_ring[rxring_idx].idx = 0;
+			for (i = 0; i < rtlpci->rxringcount; i++) {
+				entry = &rtlpci->rx_ring[rxring_idx].desc[i];
+				bufferaddress =
+				  rtlpriv->cfg->ops->get_desc(hw, (u8 *)entry,
+				  false, HW_DESC_RXBUFF_ADDR);
+				memset((u8 *)entry, 0,
+				       sizeof(*rtlpci->rx_ring
+				       [rxring_idx].desc));/*clear one entry*/
+				if (rtlpriv->use_new_trx_flow) {
+					/* This is deadcode */
+					rtlpriv->cfg->ops->set_desc(hw,
+					    (u8 *)entry, false,
+					    HW_DESC_RX_PREPARE,
+					    (u8 *)&bufferaddress);
+				} else {
+					rtlpriv->cfg->ops->set_desc(hw,
+					    (u8 *)entry, false,
+					    HW_DESC_RXBUFF_ADDR,
+					    (u8 *)&bufferaddress);
+					rtlpriv->cfg->ops->set_desc(hw,
+					    (u8 *)entry, false,
+					    HW_DESC_RXPKT_LEN,
+					    (u8 *)&rtlpci->rxbuffersize);
+					rtlpriv->cfg->ops->set_desc(hw,
+					    (u8 *)entry, false,
+					    HW_DESC_RXOWN,
+					    (u8 *)&tmp_one);
+				}
+			}
+			rtlpriv->cfg->ops->set_desc(hw, (u8 *)entry, false,
+					    HW_DESC_RXERO, (u8 *)&tmp_one);
+		}
+		rtlpci->rx_ring[rxring_idx].idx = 0;
+	}
+
+	/*
+	 *after reset, release previous pending packet,
+	 *and force the  tx idx to the first one
+	 */
+	spin_lock_irqsave(&rtlpriv->locks.irq_th_lock, flags);
+	for (i = 0; i < RTL_PCI_MAX_TX_QUEUE_COUNT; i++) {
+		if (rtlpci->tx_ring[i].desc ||
+		    rtlpci->tx_ring[i].buffer_desc) {
+			struct rtl8192_tx_ring *ring = &rtlpci->tx_ring[i];
+
+			while (skb_queue_len(&ring->queue)) {
+				u8 *entry;
+				struct sk_buff *skb =
+					__skb_dequeue(&ring->queue);
+				if (rtlpriv->use_new_trx_flow)
+					entry = (u8 *)(&ring->buffer_desc
+								[ring->idx]);
+				else
+					entry = (u8 *)(&ring->desc[ring->idx]);
+
+				pci_unmap_single(rtlpci->pdev,
+				     rtlpriv->cfg->ops->get_desc(hw, (u8 *)entry,
+					 true, HW_DESC_TXBUFF_ADDR),
+					 skb->len, PCI_DMA_TODEVICE);
+				dev_kfree_skb_irq(skb);
+				ring->idx = (ring->idx + 1) % ring->entries;
+			}
+
+			if (rtlpriv->use_new_trx_flow) {
+				rtlpci->tx_ring[i].cur_tx_rp = 0;
+				rtlpci->tx_ring[i].cur_tx_wp = 0;
+			}
+
+			ring->idx = 0;
+			ring->entries = rtlpci->txringcount[i];
+		}
+	}
+	spin_unlock_irqrestore(&rtlpriv->locks.irq_th_lock, flags);
+
+	return 0;
+}
+
+static bool rtl_pci_tx_chk_waitq_insert(struct ieee80211_hw *hw,
+					struct ieee80211_sta *sta,
+					struct sk_buff *skb)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_sta_info *sta_entry = NULL;
+	u8 tid = rtl_get_tid(skb);
+	__le16 fc = rtl_get_fc(skb);
+
+	if (!sta)
+		return false;
+	sta_entry = (struct rtl_sta_info *)sta->drv_priv;
+
+	if (!rtlpriv->rtlhal.earlymode_enable)
+		return false;
+	if (ieee80211_is_nullfunc(fc))
+		return false;
+	if (ieee80211_is_qos_nullfunc(fc))
+		return false;
+	if (ieee80211_is_pspoll(fc))
+		return false;
+	if (sta_entry->tids[tid].agg.agg_state != RTL_AGG_OPERATIONAL)
+		return false;
+	if (_rtl_mac_to_hwqueue(hw, skb) > VO_QUEUE)
+		return false;
+	if (tid > 7)
+		return false;
+
+	/* maybe every tid should be checked */
+	if (!rtlpriv->link_info.higher_busytxtraffic[tid])
+		return false;
+
+	spin_lock_bh(&rtlpriv->locks.waitq_lock);
+	skb_queue_tail(&rtlpriv->mac80211.skb_waitq[tid], skb);
+	spin_unlock_bh(&rtlpriv->locks.waitq_lock);
+
+	return true;
+}
+
+static int rtl_pci_tx(struct ieee80211_hw *hw,
+		      struct ieee80211_sta *sta,
+		      struct sk_buff *skb,
+		      struct rtl_tcb_desc *ptcb_desc)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_sta_info *sta_entry = NULL;
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	struct rtl8192_tx_ring *ring;
+	struct rtl_tx_desc *pdesc;
+	struct rtl_tx_buffer_desc *ptx_bd_desc = NULL;
+	u16 idx;
+	u8 hw_queue = _rtl_mac_to_hwqueue(hw, skb);
+	unsigned long flags;
+	struct ieee80211_hdr *hdr = rtl_get_hdr(skb);
+	__le16 fc = rtl_get_fc(skb);
+	u8 *pda_addr = hdr->addr1;
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	/*ssn */
+	u8 tid = 0;
+	u16 seq_number = 0;
+	u8 own;
+	u8 temp_one = 1;
+
+	if (ieee80211_is_mgmt(fc))
+		rtl_tx_mgmt_proc(hw, skb);
+
+	if (rtlpriv->psc.sw_ps_enabled) {
+		if (ieee80211_is_data(fc) && !ieee80211_is_nullfunc(fc) &&
+		    !ieee80211_has_pm(fc))
+			hdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_PM);
+	}
+
+	rtl_action_proc(hw, skb, true);
+
+	if (is_multicast_ether_addr(pda_addr))
+		rtlpriv->stats.txbytesmulticast += skb->len;
+	else if (is_broadcast_ether_addr(pda_addr))
+		rtlpriv->stats.txbytesbroadcast += skb->len;
+	else
+		rtlpriv->stats.txbytesunicast += skb->len;
+
+	spin_lock_irqsave(&rtlpriv->locks.irq_th_lock, flags);
+	ring = &rtlpci->tx_ring[hw_queue];
+	if (hw_queue != BEACON_QUEUE) {
+		if (rtlpriv->use_new_trx_flow)
+			idx = ring->cur_tx_wp;
+		else
+			idx = (ring->idx + skb_queue_len(&ring->queue)) %
+			      ring->entries;
+	} else {
+		idx = 0;
+	}
+
+	pdesc = &ring->desc[idx];
+	if (rtlpriv->use_new_trx_flow) {
+		ptx_bd_desc = &ring->buffer_desc[idx];
+	} else {
+		own = (u8)rtlpriv->cfg->ops->get_desc(hw, (u8 *)pdesc,
+				true, HW_DESC_OWN);
+
+		if ((own == 1) && (hw_queue != BEACON_QUEUE)) {
+			RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING,
+				 "No more TX desc@%d, ring->idx = %d, idx = %d, skb_queue_len = 0x%x\n",
+				 hw_queue, ring->idx, idx,
+				 skb_queue_len(&ring->queue));
+
+			spin_unlock_irqrestore(&rtlpriv->locks.irq_th_lock,
+					       flags);
+			return skb->len;
+		}
+	}
+
+	if (rtlpriv->cfg->ops->get_available_desc &&
+	    rtlpriv->cfg->ops->get_available_desc(hw, hw_queue) == 0) {
+		RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING,
+			 "get_available_desc fail\n");
+		spin_unlock_irqrestore(&rtlpriv->locks.irq_th_lock, flags);
+		return skb->len;
+	}
+
+	if (ieee80211_is_data_qos(fc)) {
+		tid = rtl_get_tid(skb);
+		if (sta) {
+			sta_entry = (struct rtl_sta_info *)sta->drv_priv;
+			seq_number = (le16_to_cpu(hdr->seq_ctrl) &
+				      IEEE80211_SCTL_SEQ) >> 4;
+			seq_number += 1;
+
+			if (!ieee80211_has_morefrags(hdr->frame_control))
+				sta_entry->tids[tid].seq_number = seq_number;
+		}
+	}
+
+	if (ieee80211_is_data(fc))
+		rtlpriv->cfg->ops->led_control(hw, LED_CTL_TX);
+
+	rtlpriv->cfg->ops->fill_tx_desc(hw, hdr, (u8 *)pdesc,
+			(u8 *)ptx_bd_desc, info, sta, skb, hw_queue, ptcb_desc);
+
+	__skb_queue_tail(&ring->queue, skb);
+
+	if (rtlpriv->use_new_trx_flow) {
+		rtlpriv->cfg->ops->set_desc(hw, (u8 *)pdesc, true,
+					    HW_DESC_OWN, &hw_queue);
+	} else {
+		rtlpriv->cfg->ops->set_desc(hw, (u8 *)pdesc, true,
+					    HW_DESC_OWN, &temp_one);
+	}
+
+	if ((ring->entries - skb_queue_len(&ring->queue)) < 2 &&
+	    hw_queue != BEACON_QUEUE) {
+		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
+			 "less desc left, stop skb_queue@%d, ring->idx = %d, idx = %d, skb_queue_len = 0x%x\n",
+			 hw_queue, ring->idx, idx,
+			 skb_queue_len(&ring->queue));
+
+		ieee80211_stop_queue(hw, skb_get_queue_mapping(skb));
+	}
+
+	spin_unlock_irqrestore(&rtlpriv->locks.irq_th_lock, flags);
+
+	rtlpriv->cfg->ops->tx_polling(hw, hw_queue);
+
+	return 0;
+}
+
+static void rtl_pci_flush(struct ieee80211_hw *hw, u32 queues, bool drop)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci_priv *pcipriv = rtl_pcipriv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	u16 i = 0;
+	int queue_id;
+	struct rtl8192_tx_ring *ring;
+
+	if (mac->skip_scan)
+		return;
+
+	for (queue_id = RTL_PCI_MAX_TX_QUEUE_COUNT - 1; queue_id >= 0;) {
+		u32 queue_len;
+
+		if (((queues >> queue_id) & 0x1) == 0) {
+			queue_id--;
+			continue;
+		}
+		ring = &pcipriv->dev.tx_ring[queue_id];
+		queue_len = skb_queue_len(&ring->queue);
+		if (queue_len == 0 || queue_id == BEACON_QUEUE ||
+		    queue_id == TXCMD_QUEUE) {
+			queue_id--;
+			continue;
+		} else {
+			msleep(20);
+			i++;
+		}
+
+		/* we just wait 1s for all queues */
+		if (rtlpriv->psc.rfpwr_state == ERFOFF ||
+		    is_hal_stop(rtlhal) || i >= 200)
+			return;
+	}
+}
+
+static void rtl_pci_deinit(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+
+	_rtl_pci_deinit_trx_ring(hw);
+
+	synchronize_irq(rtlpci->pdev->irq);
+	tasklet_kill(&rtlpriv->works.irq_tasklet);
+	cancel_work_sync(&rtlpriv->works.lps_change_work);
+
+	flush_workqueue(rtlpriv->works.rtl_wq);
+	destroy_workqueue(rtlpriv->works.rtl_wq);
+}
+
+static int rtl_pci_init(struct ieee80211_hw *hw, struct pci_dev *pdev)
+{
+	int err;
+
+	_rtl_pci_init_struct(hw, pdev);
+
+	err = _rtl_pci_init_trx_ring(hw);
+	if (err) {
+		pr_err("tx ring initialization failed\n");
+		return err;
+	}
+
+	return 0;
+}
+
+static int rtl_pci_start(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	struct rtl_mac *rtlmac = rtl_mac(rtl_priv(hw));
+
+	int err;
+
+	rtl_pci_reset_trx_ring(hw);
+
+	rtlpci->driver_is_goingto_unload = false;
+	if (rtlpriv->cfg->ops->get_btc_status &&
+	    rtlpriv->cfg->ops->get_btc_status()) {
+		rtlpriv->btcoexist.btc_info.ap_num = 36;
+		rtlpriv->btcoexist.btc_ops->btc_init_variables(rtlpriv);
+		rtlpriv->btcoexist.btc_ops->btc_init_hal_vars(rtlpriv);
+	} else if (rtlpriv->btcoexist.btc_ops) {
+		rtlpriv->btcoexist.btc_ops->btc_init_variables_wifi_only(
+								rtlpriv);
+	}
+
+	err = rtlpriv->cfg->ops->hw_init(hw);
+	if (err) {
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
+			 "Failed to config hardware!\n");
+		return err;
+	}
+	rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_RETRY_LIMIT,
+			&rtlmac->retry_long);
+
+	rtlpriv->cfg->ops->enable_interrupt(hw);
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD, "enable_interrupt OK\n");
+
+	rtl_init_rx_config(hw);
+
+	/*should be after adapter start and interrupt enable. */
+	set_hal_start(rtlhal);
+
+	RT_CLEAR_PS_LEVEL(ppsc, RT_RF_OFF_LEVL_HALT_NIC);
+
+	rtlpci->up_first_time = false;
+
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "%s OK\n", __func__);
+	return 0;
+}
+
+static void rtl_pci_stop(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	unsigned long flags;
+	u8 rf_timeout = 0;
+
+	if (rtlpriv->cfg->ops->get_btc_status())
+		rtlpriv->btcoexist.btc_ops->btc_halt_notify(rtlpriv);
+
+	if (rtlpriv->btcoexist.btc_ops)
+		rtlpriv->btcoexist.btc_ops->btc_deinit_variables(rtlpriv);
+
+	/*
+	 *should be before disable interrupt&adapter
+	 *and will do it immediately.
+	 */
+	set_hal_stop(rtlhal);
+
+	rtlpci->driver_is_goingto_unload = true;
+	rtlpriv->cfg->ops->disable_interrupt(hw);
+	cancel_work_sync(&rtlpriv->works.lps_change_work);
+
+	spin_lock_irqsave(&rtlpriv->locks.rf_ps_lock, flags);
+	while (ppsc->rfchange_inprogress) {
+		spin_unlock_irqrestore(&rtlpriv->locks.rf_ps_lock, flags);
+		if (rf_timeout > 100) {
+			spin_lock_irqsave(&rtlpriv->locks.rf_ps_lock, flags);
+			break;
+		}
+		mdelay(1);
+		rf_timeout++;
+		spin_lock_irqsave(&rtlpriv->locks.rf_ps_lock, flags);
+	}
+	ppsc->rfchange_inprogress = true;
+	spin_unlock_irqrestore(&rtlpriv->locks.rf_ps_lock, flags);
+
+	rtlpriv->cfg->ops->hw_disable(hw);
+	/* some things are not needed if firmware not available */
+	if (!rtlpriv->max_fw_size)
+		return;
+	rtlpriv->cfg->ops->led_control(hw, LED_CTL_POWER_OFF);
+
+	spin_lock_irqsave(&rtlpriv->locks.rf_ps_lock, flags);
+	ppsc->rfchange_inprogress = false;
+	spin_unlock_irqrestore(&rtlpriv->locks.rf_ps_lock, flags);
+
+	rtl_pci_enable_aspm(hw);
+}
+
+static bool _rtl_pci_find_adapter(struct pci_dev *pdev,
+				  struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci_priv *pcipriv = rtl_pcipriv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	struct pci_dev *bridge_pdev = pdev->bus->self;
+	u16 venderid;
+	u16 deviceid;
+	u8 revisionid;
+	u16 irqline;
+	u8 tmp;
+
+	pcipriv->ndis_adapter.pcibridge_vendor = PCI_BRIDGE_VENDOR_UNKNOWN;
+	venderid = pdev->vendor;
+	deviceid = pdev->device;
+	pci_read_config_byte(pdev, 0x8, &revisionid);
+	pci_read_config_word(pdev, 0x3C, &irqline);
+
+	/* PCI ID 0x10ec:0x8192 occurs for both RTL8192E, which uses
+	 * r8192e_pci, and RTL8192SE, which uses this driver. If the
+	 * revision ID is RTL_PCI_REVISION_ID_8192PCIE (0x01), then
+	 * the correct driver is r8192e_pci, thus this routine should
+	 * return false.
+	 */
+	if (deviceid == RTL_PCI_8192SE_DID &&
+	    revisionid == RTL_PCI_REVISION_ID_8192PCIE)
+		return false;
+
+	if (deviceid == RTL_PCI_8192_DID ||
+	    deviceid == RTL_PCI_0044_DID ||
+	    deviceid == RTL_PCI_0047_DID ||
+	    deviceid == RTL_PCI_8192SE_DID ||
+	    deviceid == RTL_PCI_8174_DID ||
+	    deviceid == RTL_PCI_8173_DID ||
+	    deviceid == RTL_PCI_8172_DID ||
+	    deviceid == RTL_PCI_8171_DID) {
+		switch (revisionid) {
+		case RTL_PCI_REVISION_ID_8192PCIE:
+			RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
+				 "8192 PCI-E is found - vid/did=%x/%x\n",
+				 venderid, deviceid);
+			rtlhal->hw_type = HARDWARE_TYPE_RTL8192E;
+			return false;
+		case RTL_PCI_REVISION_ID_8192SE:
+			RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
+				 "8192SE is found - vid/did=%x/%x\n",
+				 venderid, deviceid);
+			rtlhal->hw_type = HARDWARE_TYPE_RTL8192SE;
+			break;
+		default:
+			RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING,
+				 "Err: Unknown device - vid/did=%x/%x\n",
+				 venderid, deviceid);
+			rtlhal->hw_type = HARDWARE_TYPE_RTL8192SE;
+			break;
+		}
+	} else if (deviceid == RTL_PCI_8723AE_DID) {
+		rtlhal->hw_type = HARDWARE_TYPE_RTL8723AE;
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
+			 "8723AE PCI-E is found - vid/did=%x/%x\n",
+			 venderid, deviceid);
+	} else if (deviceid == RTL_PCI_8192CET_DID ||
+		   deviceid == RTL_PCI_8192CE_DID ||
+		   deviceid == RTL_PCI_8191CE_DID ||
+		   deviceid == RTL_PCI_8188CE_DID) {
+		rtlhal->hw_type = HARDWARE_TYPE_RTL8192CE;
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
+			 "8192C PCI-E is found - vid/did=%x/%x\n",
+			 venderid, deviceid);
+	} else if (deviceid == RTL_PCI_8192DE_DID ||
+		   deviceid == RTL_PCI_8192DE_DID2) {
+		rtlhal->hw_type = HARDWARE_TYPE_RTL8192DE;
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
+			 "8192D PCI-E is found - vid/did=%x/%x\n",
+			 venderid, deviceid);
+	} else if (deviceid == RTL_PCI_8188EE_DID) {
+		rtlhal->hw_type = HARDWARE_TYPE_RTL8188EE;
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+			 "Find adapter, Hardware type is 8188EE\n");
+	} else if (deviceid == RTL_PCI_8723BE_DID) {
+		rtlhal->hw_type = HARDWARE_TYPE_RTL8723BE;
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+			 "Find adapter, Hardware type is 8723BE\n");
+	} else if (deviceid == RTL_PCI_8192EE_DID) {
+		rtlhal->hw_type = HARDWARE_TYPE_RTL8192EE;
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+			 "Find adapter, Hardware type is 8192EE\n");
+	} else if (deviceid == RTL_PCI_8821AE_DID) {
+		rtlhal->hw_type = HARDWARE_TYPE_RTL8821AE;
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+			 "Find adapter, Hardware type is 8821AE\n");
+	} else if (deviceid == RTL_PCI_8812AE_DID) {
+		rtlhal->hw_type = HARDWARE_TYPE_RTL8812AE;
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+			 "Find adapter, Hardware type is 8812AE\n");
+	} else if (deviceid == RTL_PCI_8822BE_DID) {
+		rtlhal->hw_type = HARDWARE_TYPE_RTL8822BE;
+		rtlhal->bandset = BAND_ON_BOTH;
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+			 "Find adapter, Hardware type is 8822BE\n");
+	} else {
+		RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING,
+			 "Err: Unknown device - vid/did=%x/%x\n",
+			 venderid, deviceid);
+
+		rtlhal->hw_type = RTL_DEFAULT_HARDWARE_TYPE;
+	}
+
+	if (rtlhal->hw_type == HARDWARE_TYPE_RTL8192DE) {
+		if (revisionid == 0 || revisionid == 1) {
+			if (revisionid == 0) {
+				RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+					 "Find 92DE MAC0\n");
+				rtlhal->interfaceindex = 0;
+			} else if (revisionid == 1) {
+				RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+					 "Find 92DE MAC1\n");
+				rtlhal->interfaceindex = 1;
+			}
+		} else {
+			RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+				 "Unknown device - VendorID/DeviceID=%x/%x, Revision=%x\n",
+				 venderid, deviceid, revisionid);
+			rtlhal->interfaceindex = 0;
+		}
+	}
+
+	switch (rtlhal->hw_type) {
+	case HARDWARE_TYPE_RTL8192EE:
+	case HARDWARE_TYPE_RTL8822BE:
+		/* use new trx flow */
+		rtlpriv->use_new_trx_flow = true;
+		break;
+
+	default:
+		rtlpriv->use_new_trx_flow = false;
+		break;
+	}
+
+	/*find bus info */
+	pcipriv->ndis_adapter.busnumber = pdev->bus->number;
+	pcipriv->ndis_adapter.devnumber = PCI_SLOT(pdev->devfn);
+	pcipriv->ndis_adapter.funcnumber = PCI_FUNC(pdev->devfn);
+
+	/*find bridge info */
+	pcipriv->ndis_adapter.pcibridge_vendor = PCI_BRIDGE_VENDOR_UNKNOWN;
+	/* some ARM have no bridge_pdev and will crash here
+	 * so we should check if bridge_pdev is NULL
+	 */
+	if (bridge_pdev) {
+		/*find bridge info if available */
+		pcipriv->ndis_adapter.pcibridge_vendorid = bridge_pdev->vendor;
+		for (tmp = 0; tmp < PCI_BRIDGE_VENDOR_MAX; tmp++) {
+			if (bridge_pdev->vendor == pcibridge_vendors[tmp]) {
+				pcipriv->ndis_adapter.pcibridge_vendor = tmp;
+				RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
+					 "Pci Bridge Vendor is found index: %d\n",
+					 tmp);
+				break;
+			}
+		}
+	}
+
+	if (pcipriv->ndis_adapter.pcibridge_vendor !=
+		PCI_BRIDGE_VENDOR_UNKNOWN) {
+		pcipriv->ndis_adapter.pcibridge_busnum =
+		    bridge_pdev->bus->number;
+		pcipriv->ndis_adapter.pcibridge_devnum =
+		    PCI_SLOT(bridge_pdev->devfn);
+		pcipriv->ndis_adapter.pcibridge_funcnum =
+		    PCI_FUNC(bridge_pdev->devfn);
+		pcipriv->ndis_adapter.pcibridge_pciehdr_offset =
+		    pci_pcie_cap(bridge_pdev);
+		pcipriv->ndis_adapter.num4bytes =
+		    (pcipriv->ndis_adapter.pcibridge_pciehdr_offset + 0x10) / 4;
+
+		rtl_pci_get_linkcontrol_field(hw);
+
+		if (pcipriv->ndis_adapter.pcibridge_vendor ==
+		    PCI_BRIDGE_VENDOR_AMD) {
+			pcipriv->ndis_adapter.amd_l1_patch =
+			    rtl_pci_get_amd_l1_patch(hw);
+		}
+	}
+
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
+		 "pcidev busnumber:devnumber:funcnumber:vendor:link_ctl %d:%d:%d:%x:%x\n",
+		 pcipriv->ndis_adapter.busnumber,
+		 pcipriv->ndis_adapter.devnumber,
+		 pcipriv->ndis_adapter.funcnumber,
+		 pdev->vendor, pcipriv->ndis_adapter.linkctrl_reg);
+
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
+		 "pci_bridge busnumber:devnumber:funcnumber:vendor:pcie_cap:link_ctl_reg:amd %d:%d:%d:%x:%x:%x:%x\n",
+		 pcipriv->ndis_adapter.pcibridge_busnum,
+		 pcipriv->ndis_adapter.pcibridge_devnum,
+		 pcipriv->ndis_adapter.pcibridge_funcnum,
+		 pcibridge_vendors[pcipriv->ndis_adapter.pcibridge_vendor],
+		 pcipriv->ndis_adapter.pcibridge_pciehdr_offset,
+		 pcipriv->ndis_adapter.pcibridge_linkctrlreg,
+		 pcipriv->ndis_adapter.amd_l1_patch);
+
+	rtl_pci_parse_configuration(pdev, hw);
+	list_add_tail(&rtlpriv->list, &rtlpriv->glb_var->glb_priv_list);
+
+	return true;
+}
+
+static int rtl_pci_intr_mode_msi(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci_priv *pcipriv = rtl_pcipriv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(pcipriv);
+	int ret;
+
+	ret = pci_enable_msi(rtlpci->pdev);
+	if (ret < 0)
+		return ret;
+
+	ret = request_irq(rtlpci->pdev->irq, &_rtl_pci_interrupt,
+			  IRQF_SHARED, KBUILD_MODNAME, hw);
+	if (ret < 0) {
+		pci_disable_msi(rtlpci->pdev);
+		return ret;
+	}
+
+	rtlpci->using_msi = true;
+
+	RT_TRACE(rtlpriv, COMP_INIT | COMP_INTR, DBG_DMESG,
+		 "MSI Interrupt Mode!\n");
+	return 0;
+}
+
+static int rtl_pci_intr_mode_legacy(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci_priv *pcipriv = rtl_pcipriv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(pcipriv);
+	int ret;
+
+	ret = request_irq(rtlpci->pdev->irq, &_rtl_pci_interrupt,
+			  IRQF_SHARED, KBUILD_MODNAME, hw);
+	if (ret < 0)
+		return ret;
+
+	rtlpci->using_msi = false;
+	RT_TRACE(rtlpriv, COMP_INIT | COMP_INTR, DBG_DMESG,
+		 "Pin-based Interrupt Mode!\n");
+	return 0;
+}
+
+static int rtl_pci_intr_mode_decide(struct ieee80211_hw *hw)
+{
+	struct rtl_pci_priv *pcipriv = rtl_pcipriv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(pcipriv);
+	int ret;
+
+	if (rtlpci->msi_support) {
+		ret = rtl_pci_intr_mode_msi(hw);
+		if (ret < 0)
+			ret = rtl_pci_intr_mode_legacy(hw);
+	} else {
+		ret = rtl_pci_intr_mode_legacy(hw);
+	}
+	return ret;
+}
+
+static void platform_enable_dma64(struct pci_dev *pdev, bool dma64)
+{
+	u8	value;
+
+	pci_read_config_byte(pdev, 0x719, &value);
+
+	/* 0x719 Bit5 is DMA64 bit fetch. */
+	if (dma64)
+		value |= BIT(5);
+	else
+		value &= ~BIT(5);
+
+	pci_write_config_byte(pdev, 0x719, value);
+}
+
+int rtl_pci_probe(struct pci_dev *pdev,
+		  const struct pci_device_id *id)
+{
+	struct ieee80211_hw *hw = NULL;
+
+	struct rtl_priv *rtlpriv = NULL;
+	struct rtl_pci_priv *pcipriv = NULL;
+	struct rtl_pci *rtlpci;
+	unsigned long pmem_start, pmem_len, pmem_flags;
+	int err;
+
+	err = rtl_core_module_init();
+	if (err)
+		return err;
+	err = pci_enable_device(pdev);
+	if (err) {
+		WARN_ONCE(true, "%s : Cannot enable new PCI device\n",
+			  pci_name(pdev));
+		return err;
+	}
+
+	if (((struct rtl_hal_cfg *)(id->driver_data))->mod_params->dma64 &&
+	    !pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
+		if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
+			WARN_ONCE(true,
+				  "Unable to obtain 64bit DMA for consistent allocations\n");
+			err = -ENOMEM;
+			goto fail1;
+		}
+
+		platform_enable_dma64(pdev, true);
+	} else if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(32))) {
+		if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32))) {
+			WARN_ONCE(true,
+				  "rtlwifi: Unable to obtain 32bit DMA for consistent allocations\n");
+			err = -ENOMEM;
+			goto fail1;
+		}
+
+		platform_enable_dma64(pdev, false);
+	}
+
+	pci_set_master(pdev);
+
+	hw = ieee80211_alloc_hw(sizeof(struct rtl_pci_priv) +
+				sizeof(struct rtl_priv), &rtl_ops);
+	if (!hw) {
+		WARN_ONCE(true,
+			  "%s : ieee80211 alloc failed\n", pci_name(pdev));
+		err = -ENOMEM;
+		goto fail1;
+	}
+
+	SET_IEEE80211_DEV(hw, &pdev->dev);
+	pci_set_drvdata(pdev, hw);
+
+	rtlpriv = hw->priv;
+	rtlpriv->hw = hw;
+	pcipriv = (void *)rtlpriv->priv;
+	pcipriv->dev.pdev = pdev;
+	init_completion(&rtlpriv->firmware_loading_complete);
+	/*proximity init here*/
+	rtlpriv->proximity.proxim_on = false;
+
+	pcipriv = (void *)rtlpriv->priv;
+	pcipriv->dev.pdev = pdev;
+
+	/* init cfg & intf_ops */
+	rtlpriv->rtlhal.interface = INTF_PCI;
+	rtlpriv->cfg = (struct rtl_hal_cfg *)(id->driver_data);
+	rtlpriv->intf_ops = &rtl_pci_ops;
+	rtlpriv->glb_var = &rtl_global_var;
+
+	/* MEM map */
+	err = pci_request_regions(pdev, KBUILD_MODNAME);
+	if (err) {
+		WARN_ONCE(true, "rtlwifi: Can't obtain PCI resources\n");
+		goto fail1;
+	}
+
+	pmem_start = pci_resource_start(pdev, rtlpriv->cfg->bar_id);
+	pmem_len = pci_resource_len(pdev, rtlpriv->cfg->bar_id);
+	pmem_flags = pci_resource_flags(pdev, rtlpriv->cfg->bar_id);
+
+	/*shared mem start */
+	rtlpriv->io.pci_mem_start =
+			(unsigned long)pci_iomap(pdev,
+			rtlpriv->cfg->bar_id, pmem_len);
+	if (rtlpriv->io.pci_mem_start == 0) {
+		WARN_ONCE(true, "rtlwifi: Can't map PCI mem\n");
+		err = -ENOMEM;
+		goto fail2;
+	}
+
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
+		 "mem mapped space: start: 0x%08lx len:%08lx flags:%08lx, after map:0x%08lx\n",
+		 pmem_start, pmem_len, pmem_flags,
+		 rtlpriv->io.pci_mem_start);
+
+	/* Disable Clk Request */
+	pci_write_config_byte(pdev, 0x81, 0);
+	/* leave D3 mode */
+	pci_write_config_byte(pdev, 0x44, 0);
+	pci_write_config_byte(pdev, 0x04, 0x06);
+	pci_write_config_byte(pdev, 0x04, 0x07);
+
+	/* find adapter */
+	if (!_rtl_pci_find_adapter(pdev, hw)) {
+		err = -ENODEV;
+		goto fail2;
+	}
+
+	/* Init IO handler */
+	_rtl_pci_io_handler_init(&pdev->dev, hw);
+
+	/*like read eeprom and so on */
+	rtlpriv->cfg->ops->read_eeprom_info(hw);
+
+	if (rtlpriv->cfg->ops->init_sw_vars(hw)) {
+		pr_err("Can't init_sw_vars\n");
+		err = -ENODEV;
+		goto fail3;
+	}
+	rtlpriv->cfg->ops->init_sw_leds(hw);
+
+	/*aspm */
+	rtl_pci_init_aspm(hw);
+
+	/* Init mac80211 sw */
+	err = rtl_init_core(hw);
+	if (err) {
+		pr_err("Can't allocate sw for mac80211\n");
+		goto fail3;
+	}
+
+	/* Init PCI sw */
+	err = rtl_pci_init(hw, pdev);
+	if (err) {
+		pr_err("Failed to init PCI\n");
+		goto fail3;
+	}
+
+	err = ieee80211_register_hw(hw);
+	if (err) {
+		pr_err("Can't register mac80211 hw.\n");
+		err = -ENODEV;
+		goto fail3;
+	}
+	rtlpriv->mac80211.mac80211_registered = 1;
+
+	/* add for debug */
+	rtl_debug_add_one(hw);
+
+	/*init rfkill */
+	rtl_init_rfkill(hw);	/* Init PCI sw */
+
+	rtlpci = rtl_pcidev(pcipriv);
+	err = rtl_pci_intr_mode_decide(hw);
+	if (err) {
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
+			 "%s: failed to register IRQ handler\n",
+			 wiphy_name(hw->wiphy));
+		goto fail3;
+	}
+	rtlpci->irq_alloc = 1;
+
+	set_bit(RTL_STATUS_INTERFACE_START, &rtlpriv->status);
+	return 0;
+
+fail3:
+	pci_set_drvdata(pdev, NULL);
+	rtl_deinit_core(hw);
+
+fail2:
+	if (rtlpriv->io.pci_mem_start != 0)
+		pci_iounmap(pdev, (void __iomem *)rtlpriv->io.pci_mem_start);
+
+	pci_release_regions(pdev);
+	complete(&rtlpriv->firmware_loading_complete);
+
+fail1:
+	if (hw)
+		ieee80211_free_hw(hw);
+	pci_disable_device(pdev);
+
+	return err;
+}
+
+void rtl_pci_disconnect(struct pci_dev *pdev)
+{
+	struct ieee80211_hw *hw = pci_get_drvdata(pdev);
+	struct rtl_pci_priv *pcipriv = rtl_pcipriv(hw);
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(pcipriv);
+	struct rtl_mac *rtlmac = rtl_mac(rtlpriv);
+
+	/* just in case driver is removed before firmware callback */
+	wait_for_completion(&rtlpriv->firmware_loading_complete);
+	clear_bit(RTL_STATUS_INTERFACE_START, &rtlpriv->status);
+
+	/* remove form debug */
+	rtl_debug_remove_one(hw);
+
+	/*ieee80211_unregister_hw will call ops_stop */
+	if (rtlmac->mac80211_registered == 1) {
+		ieee80211_unregister_hw(hw);
+		rtlmac->mac80211_registered = 0;
+	} else {
+		rtl_deinit_deferred_work(hw);
+		rtlpriv->intf_ops->adapter_stop(hw);
+	}
+	rtlpriv->cfg->ops->disable_interrupt(hw);
+
+	/*deinit rfkill */
+	rtl_deinit_rfkill(hw);
+
+	rtl_pci_deinit(hw);
+	rtl_deinit_core(hw);
+	rtlpriv->cfg->ops->deinit_sw_vars(hw);
+
+	if (rtlpci->irq_alloc) {
+		free_irq(rtlpci->pdev->irq, hw);
+		rtlpci->irq_alloc = 0;
+	}
+
+	if (rtlpci->using_msi)
+		pci_disable_msi(rtlpci->pdev);
+
+	list_del(&rtlpriv->list);
+	if (rtlpriv->io.pci_mem_start != 0) {
+		pci_iounmap(pdev, (void __iomem *)rtlpriv->io.pci_mem_start);
+		pci_release_regions(pdev);
+	}
+
+	pci_disable_device(pdev);
+
+	rtl_pci_disable_aspm(hw);
+
+	pci_set_drvdata(pdev, NULL);
+
+	ieee80211_free_hw(hw);
+	rtl_core_module_exit();
+}
+
+#ifdef CONFIG_PM_SLEEP
+/***************************************
+ * kernel pci power state define:
+ * PCI_D0         ((pci_power_t __force) 0)
+ * PCI_D1         ((pci_power_t __force) 1)
+ * PCI_D2         ((pci_power_t __force) 2)
+ * PCI_D3hot      ((pci_power_t __force) 3)
+ * PCI_D3cold     ((pci_power_t __force) 4)
+ * PCI_UNKNOWN    ((pci_power_t __force) 5)
+
+ * This function is called when system
+ * goes into suspend state mac80211 will
+ * call rtl_mac_stop() from the mac80211
+ * suspend function first, So there is
+ * no need to call hw_disable here.
+ ****************************************/
+int rtl_pci_suspend(struct device *dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct ieee80211_hw *hw = pci_get_drvdata(pdev);
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	rtlpriv->cfg->ops->hw_suspend(hw);
+	rtl_deinit_rfkill(hw);
+
+	return 0;
+}
+
+int rtl_pci_resume(struct device *dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct ieee80211_hw *hw = pci_get_drvdata(pdev);
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	rtlpriv->cfg->ops->hw_resume(hw);
+	rtl_init_rfkill(hw);
+	return 0;
+}
+#endif /* CONFIG_PM_SLEEP */
+
+const struct rtl_intf_ops rtl_pci_ops = {
+	.read_efuse_byte = read_efuse_byte,
+	.adapter_start = rtl_pci_start,
+	.adapter_stop = rtl_pci_stop,
+	.check_buddy_priv = rtl_pci_check_buddy_priv,
+	.adapter_tx = rtl_pci_tx,
+	.flush = rtl_pci_flush,
+	.reset_trx_ring = rtl_pci_reset_trx_ring,
+	.waitq_insert = rtl_pci_tx_chk_waitq_insert,
+
+	.disable_aspm = rtl_pci_disable_aspm,
+	.enable_aspm = rtl_pci_enable_aspm,
+};
diff --git a/drivers/staging/rtlwifi/pci.h b/drivers/staging/rtlwifi/pci.h
new file mode 100644
index 000000000000..3fb56c845a61
--- /dev/null
+++ b/drivers/staging/rtlwifi/pci.h
@@ -0,0 +1,329 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2012  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __RTL_PCI_H__
+#define __RTL_PCI_H__
+
+#include <linux/pci.h>
+/* 1: MSDU packet queue,
+ * 2: Rx Command Queue
+ */
+#define RTL_PCI_RX_MPDU_QUEUE			0
+#define RTL_PCI_RX_CMD_QUEUE			1
+#define RTL_PCI_MAX_RX_QUEUE			2
+
+#define RTL_PCI_MAX_RX_COUNT			512/*64*/
+#define RTL_PCI_MAX_TX_QUEUE_COUNT		9
+
+#define RT_TXDESC_NUM				128
+#define TX_DESC_NUM_92E				512
+#define TX_DESC_NUM_8822B			512
+#define RT_TXDESC_NUM_BE_QUEUE			256
+
+#define BK_QUEUE				0
+#define BE_QUEUE				1
+#define VI_QUEUE				2
+#define VO_QUEUE				3
+#define BEACON_QUEUE				4
+#define TXCMD_QUEUE				5
+#define MGNT_QUEUE				6
+#define HIGH_QUEUE				7
+#define HCCA_QUEUE				8
+#define H2C_QUEUE				TXCMD_QUEUE	/* In 8822B */
+
+#define RTL_PCI_DEVICE(vend, dev, cfg)  \
+	.vendor = (vend), \
+	.device = (dev), \
+	.subvendor = PCI_ANY_ID, \
+	.subdevice = PCI_ANY_ID,\
+	.driver_data = (kernel_ulong_t)&(cfg)
+
+#define INTEL_VENDOR_ID				0x8086
+#define SIS_VENDOR_ID				0x1039
+#define ATI_VENDOR_ID				0x1002
+#define ATI_DEVICE_ID				0x7914
+#define AMD_VENDOR_ID				0x1022
+
+#define PCI_MAX_BRIDGE_NUMBER			255
+#define PCI_MAX_DEVICES				32
+#define PCI_MAX_FUNCTION			8
+
+#define PCI_CONF_ADDRESS	0x0CF8	/*PCI Configuration Space Address */
+#define PCI_CONF_DATA		0x0CFC	/*PCI Configuration Space Data */
+
+#define PCI_CLASS_BRIDGE_DEV		0x06
+#define PCI_SUBCLASS_BR_PCI_TO_PCI	0x04
+#define PCI_CAPABILITY_ID_PCI_EXPRESS	0x10
+#define PCI_CAP_ID_EXP			0x10
+
+#define U1DONTCARE			0xFF
+#define U2DONTCARE			0xFFFF
+#define U4DONTCARE			0xFFFFFFFF
+
+#define RTL_PCI_8192_DID	0x8192	/*8192 PCI-E */
+#define RTL_PCI_8192SE_DID	0x8192	/*8192 SE */
+#define RTL_PCI_8174_DID	0x8174	/*8192 SE */
+#define RTL_PCI_8173_DID	0x8173	/*8191 SE Crab */
+#define RTL_PCI_8172_DID	0x8172	/*8191 SE RE */
+#define RTL_PCI_8171_DID	0x8171	/*8191 SE Unicron */
+#define RTL_PCI_8723AE_DID	0x8723	/*8723AE */
+#define RTL_PCI_0045_DID	0x0045	/*8190 PCI for Ceraga */
+#define RTL_PCI_0046_DID	0x0046	/*8190 Cardbus for Ceraga */
+#define RTL_PCI_0044_DID	0x0044	/*8192e PCIE for Ceraga */
+#define RTL_PCI_0047_DID	0x0047	/*8192e Express Card for Ceraga */
+#define RTL_PCI_700F_DID	0x700F
+#define RTL_PCI_701F_DID	0x701F
+#define RTL_PCI_DLINK_DID	0x3304
+#define RTL_PCI_8723AE_DID	0x8723	/*8723e */
+#define RTL_PCI_8192CET_DID	0x8191	/*8192ce */
+#define RTL_PCI_8192CE_DID	0x8178	/*8192ce */
+#define RTL_PCI_8191CE_DID	0x8177	/*8192ce */
+#define RTL_PCI_8188CE_DID	0x8176	/*8192ce */
+#define RTL_PCI_8192CU_DID	0x8191	/*8192ce */
+#define RTL_PCI_8192DE_DID	0x8193	/*8192de */
+#define RTL_PCI_8192DE_DID2	0x002B	/*92DE*/
+#define RTL_PCI_8188EE_DID	0x8179  /*8188ee*/
+#define RTL_PCI_8723BE_DID	0xB723  /*8723be*/
+#define RTL_PCI_8192EE_DID	0x818B	/*8192ee*/
+#define RTL_PCI_8821AE_DID	0x8821	/*8821ae*/
+#define RTL_PCI_8812AE_DID	0x8812	/*8812ae*/
+#define RTL_PCI_8822BE_DID	0xB822	/*8822be*/
+
+/*8192 support 16 pages of IO registers*/
+#define RTL_MEM_MAPPED_IO_RANGE_8190PCI		0x1000
+#define RTL_MEM_MAPPED_IO_RANGE_8192PCIE	0x4000
+#define RTL_MEM_MAPPED_IO_RANGE_8192SE		0x4000
+#define RTL_MEM_MAPPED_IO_RANGE_8192CE		0x4000
+#define RTL_MEM_MAPPED_IO_RANGE_8192DE		0x4000
+
+#define RTL_PCI_REVISION_ID_8190PCI		0x00
+#define RTL_PCI_REVISION_ID_8192PCIE		0x01
+#define RTL_PCI_REVISION_ID_8192SE		0x10
+#define RTL_PCI_REVISION_ID_8192CE		0x1
+#define RTL_PCI_REVISION_ID_8192DE		0x0
+
+#define RTL_DEFAULT_HARDWARE_TYPE	HARDWARE_TYPE_RTL8192CE
+
+enum pci_bridge_vendor {
+	PCI_BRIDGE_VENDOR_INTEL = 0x0,	/*0b'0000,0001 */
+	PCI_BRIDGE_VENDOR_ATI,		/*0b'0000,0010*/
+	PCI_BRIDGE_VENDOR_AMD,		/*0b'0000,0100*/
+	PCI_BRIDGE_VENDOR_SIS,		/*0b'0000,1000*/
+	PCI_BRIDGE_VENDOR_UNKNOWN,	/*0b'0100,0000*/
+	PCI_BRIDGE_VENDOR_MAX,
+};
+
+struct rtl_pci_capabilities_header {
+	u8 capability_id;
+	u8 next;
+};
+
+/* In new TRX flow, Buffer_desc is new concept
+ * But TX wifi info == TX descriptor in old flow
+ * RX wifi info == RX descriptor in old flow
+ */
+struct rtl_tx_buffer_desc {
+	u32 dword[4 * (1 << (BUFDESC_SEG_NUM + 1))];
+} __packed;
+
+struct rtl_tx_desc {
+	u32 dword[16];
+} __packed;
+
+struct rtl_rx_buffer_desc { /*rx buffer desc*/
+	u32 dword[4];
+} __packed;
+
+struct rtl_rx_desc { /*old: rx desc new: rx wifi info*/
+	u32 dword[8];
+} __packed;
+
+struct rtl_tx_cmd_desc {
+	u32 dword[16];
+} __packed;
+
+struct rtl8192_tx_ring {
+	struct rtl_tx_desc *desc;
+	dma_addr_t dma;
+	unsigned int idx;
+	unsigned int entries;
+	struct sk_buff_head queue;
+	/*add for new trx flow*/
+	struct rtl_tx_buffer_desc *buffer_desc; /*tx buffer descriptor*/
+	dma_addr_t buffer_desc_dma; /*tx bufferd desc dma memory*/
+	u16 cur_tx_wp; /* current_tx_write_point */
+	u16 cur_tx_rp; /* current_tx_read_point */
+};
+
+struct rtl8192_rx_ring {
+	struct rtl_rx_desc *desc;
+	dma_addr_t dma;
+	unsigned int idx;
+	struct sk_buff *rx_buf[RTL_PCI_MAX_RX_COUNT];
+	/*add for new trx flow*/
+	struct rtl_rx_buffer_desc *buffer_desc; /*rx buffer descriptor*/
+	u16 next_rx_rp; /* next_rx_read_point */
+};
+
+struct rtl_pci {
+	struct pci_dev *pdev;
+	bool irq_enabled;
+
+	bool driver_is_goingto_unload;
+	bool up_first_time;
+	bool first_init;
+	bool being_init_adapter;
+	bool init_ready;
+
+	/*Tx */
+	struct rtl8192_tx_ring tx_ring[RTL_PCI_MAX_TX_QUEUE_COUNT];
+	int txringcount[RTL_PCI_MAX_TX_QUEUE_COUNT];
+	u32 transmit_config;
+
+	/*Rx */
+	struct rtl8192_rx_ring rx_ring[RTL_PCI_MAX_RX_QUEUE];
+	int rxringcount;
+	u16 rxbuffersize;
+	u32 receive_config;
+
+	/*irq */
+	u8 irq_alloc;
+	u32 irq_mask[4];	/* 0-1: normal, 2: unused, 3: h2c */
+	u32 sys_irq_mask;
+
+	/*Bcn control register setting */
+	u32 reg_bcn_ctrl_val;
+
+	 /*ASPM*/ u8 const_pci_aspm;
+	u8 const_amdpci_aspm;
+	u8 const_hwsw_rfoff_d3;
+	u8 const_support_pciaspm;
+	/*pci-e bridge */
+	u8 const_hostpci_aspm_setting;
+	/*pci-e device */
+	u8 const_devicepci_aspm_setting;
+	/* If it supports ASPM, Offset[560h] = 0x40,
+	 * otherwise Offset[560h] = 0x00.
+	 */
+	bool support_aspm;
+	bool support_backdoor;
+
+	/*QOS & EDCA */
+	enum acm_method acm_method;
+
+	u16 shortretry_limit;
+	u16 longretry_limit;
+
+	/* MSI support */
+	bool msi_support;
+	bool using_msi;
+	/* interrupt clear before set */
+	bool int_clear;
+};
+
+struct mp_adapter {
+	u8 linkctrl_reg;
+
+	u8 busnumber;
+	u8 devnumber;
+	u8 funcnumber;
+
+	u8 pcibridge_busnum;
+	u8 pcibridge_devnum;
+	u8 pcibridge_funcnum;
+
+	u8 pcibridge_vendor;
+	u16 pcibridge_vendorid;
+	u16 pcibridge_deviceid;
+
+	u8 num4bytes;
+
+	u8 pcibridge_pciehdr_offset;
+	u8 pcibridge_linkctrlreg;
+
+	bool amd_l1_patch;
+};
+
+struct rtl_pci_priv {
+	struct bt_coexist_info bt_coexist;
+	struct rtl_led_ctl ledctl;
+	struct rtl_pci dev;
+	struct mp_adapter ndis_adapter;
+};
+
+#define rtl_pcipriv(hw)		(((struct rtl_pci_priv *)(rtl_priv(hw))->priv))
+#define rtl_pcidev(pcipriv)	(&((pcipriv)->dev))
+
+int rtl_pci_reset_trx_ring(struct ieee80211_hw *hw);
+
+extern const struct rtl_intf_ops rtl_pci_ops;
+
+int rtl_pci_probe(struct pci_dev *pdev,
+		  const struct pci_device_id *id);
+void rtl_pci_disconnect(struct pci_dev *pdev);
+#ifdef CONFIG_PM_SLEEP
+int rtl_pci_suspend(struct device *dev);
+int rtl_pci_resume(struct device *dev);
+#endif /* CONFIG_PM_SLEEP */
+static inline u8 pci_read8_sync(struct rtl_priv *rtlpriv, u32 addr)
+{
+	return readb((u8 __iomem *)rtlpriv->io.pci_mem_start + addr);
+}
+
+static inline u16 pci_read16_sync(struct rtl_priv *rtlpriv, u32 addr)
+{
+	return readw((u8 __iomem *)rtlpriv->io.pci_mem_start + addr);
+}
+
+static inline u32 pci_read32_sync(struct rtl_priv *rtlpriv, u32 addr)
+{
+	return readl((u8 __iomem *)rtlpriv->io.pci_mem_start + addr);
+}
+
+static inline void pci_write8_async(struct rtl_priv *rtlpriv, u32 addr, u8 val)
+{
+	writeb(val, (u8 __iomem *)rtlpriv->io.pci_mem_start + addr);
+}
+
+static inline void pci_write16_async(struct rtl_priv *rtlpriv,
+				     u32 addr, u16 val)
+{
+	writew(val, (u8 __iomem *)rtlpriv->io.pci_mem_start + addr);
+}
+
+static inline void pci_write32_async(struct rtl_priv *rtlpriv,
+				     u32 addr, u32 val)
+{
+	writel(val, (u8 __iomem *)rtlpriv->io.pci_mem_start + addr);
+}
+
+static inline u16 calc_fifo_space(u16 rp, u16 wp, u16 size)
+{
+	if (rp <= wp)
+		return size - 1 + rp - wp;
+	return rp - wp - 1;
+}
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/halphyrf_ce.c b/drivers/staging/rtlwifi/phydm/halphyrf_ce.c
new file mode 100644
index 000000000000..684e383201d6
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/halphyrf_ce.c
@@ -0,0 +1,965 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#include "mp_precomp.h"
+#include "phydm_precomp.h"
+
+#define CALCULATE_SWINGTALBE_OFFSET(_offset, _direction, _size,                \
+				    _delta_thermal)                            \
+	do {                                                                   \
+		for (_offset = 0; _offset < _size; _offset++) {                \
+			if (_delta_thermal <                                   \
+			    thermal_threshold[_direction][_offset]) {          \
+				if (_offset != 0)                              \
+					_offset--;                             \
+				break;                                         \
+			}                                                      \
+		}                                                              \
+		if (_offset >= _size)                                          \
+			_offset = _size - 1;                                   \
+	} while (0)
+
+static inline void phydm_set_calibrate_info_up(
+	struct phy_dm_struct *dm, struct txpwrtrack_cfg *c, u8 delta,
+	struct dm_rf_calibration_struct *cali_info,
+	u8 *delta_swing_table_idx_tup_a, u8 *delta_swing_table_idx_tup_b,
+	u8 *delta_swing_table_idx_tup_c, u8 *delta_swing_table_idx_tup_d)
+{
+	u8 p = 0;
+
+	for (p = ODM_RF_PATH_A; p < c->rf_path_count; p++) {
+		cali_info->delta_power_index_last[p] =
+			cali_info->delta_power_index
+				[p]; /*recording poer index offset*/
+		switch (p) {
+		case ODM_RF_PATH_B:
+			ODM_RT_TRACE(dm, ODM_COMP_TX_PWR_TRACK,
+				     "delta_swing_table_idx_tup_b[%d] = %d\n",
+				     delta, delta_swing_table_idx_tup_b[delta]);
+
+			cali_info->delta_power_index[p] =
+				delta_swing_table_idx_tup_b[delta];
+			/*Record delta swing for mix mode pwr tracking*/
+			cali_info->absolute_ofdm_swing_idx[p] =
+				delta_swing_table_idx_tup_b[delta];
+			ODM_RT_TRACE(
+				dm, ODM_COMP_TX_PWR_TRACK,
+				"******Temp is higher and cali_info->absolute_ofdm_swing_idx[ODM_RF_PATH_B] = %d\n",
+				cali_info->absolute_ofdm_swing_idx[p]);
+			break;
+
+		case ODM_RF_PATH_C:
+			ODM_RT_TRACE(dm, ODM_COMP_TX_PWR_TRACK,
+				     "delta_swing_table_idx_tup_c[%d] = %d\n",
+				     delta, delta_swing_table_idx_tup_c[delta]);
+
+			cali_info->delta_power_index[p] =
+				delta_swing_table_idx_tup_c[delta];
+			/*Record delta swing for mix mode pwr tracking*/
+			cali_info->absolute_ofdm_swing_idx[p] =
+				delta_swing_table_idx_tup_c[delta];
+			ODM_RT_TRACE(
+				dm, ODM_COMP_TX_PWR_TRACK,
+				"******Temp is higher and cali_info->absolute_ofdm_swing_idx[ODM_RF_PATH_C] = %d\n",
+				cali_info->absolute_ofdm_swing_idx[p]);
+			break;
+
+		case ODM_RF_PATH_D:
+			ODM_RT_TRACE(dm, ODM_COMP_TX_PWR_TRACK,
+				     "delta_swing_table_idx_tup_d[%d] = %d\n",
+				     delta, delta_swing_table_idx_tup_d[delta]);
+
+			cali_info->delta_power_index[p] =
+				delta_swing_table_idx_tup_d[delta];
+			/*Record delta swing for mix mode pwr tracking*/
+			cali_info->absolute_ofdm_swing_idx[p] =
+				delta_swing_table_idx_tup_d[delta];
+			ODM_RT_TRACE(
+				dm, ODM_COMP_TX_PWR_TRACK,
+				"******Temp is higher and cali_info->absolute_ofdm_swing_idx[ODM_RF_PATH_D] = %d\n",
+				cali_info->absolute_ofdm_swing_idx[p]);
+			break;
+
+		default:
+			ODM_RT_TRACE(dm, ODM_COMP_TX_PWR_TRACK,
+				     "delta_swing_table_idx_tup_a[%d] = %d\n",
+				     delta, delta_swing_table_idx_tup_a[delta]);
+
+			cali_info->delta_power_index[p] =
+				delta_swing_table_idx_tup_a[delta];
+			/*Record delta swing for mix mode pwr tracking*/
+			cali_info->absolute_ofdm_swing_idx[p] =
+				delta_swing_table_idx_tup_a[delta];
+			ODM_RT_TRACE(
+				dm, ODM_COMP_TX_PWR_TRACK,
+				"******Temp is higher and cali_info->absolute_ofdm_swing_idx[ODM_RF_PATH_A] = %d\n",
+				cali_info->absolute_ofdm_swing_idx[p]);
+			break;
+		}
+	}
+}
+
+static inline void phydm_set_calibrate_info_down(
+	struct phy_dm_struct *dm, struct txpwrtrack_cfg *c, u8 delta,
+	struct dm_rf_calibration_struct *cali_info,
+	u8 *delta_swing_table_idx_tdown_a, u8 *delta_swing_table_idx_tdown_b,
+	u8 *delta_swing_table_idx_tdown_c, u8 *delta_swing_table_idx_tdown_d)
+{
+	u8 p = 0;
+
+	for (p = ODM_RF_PATH_A; p < c->rf_path_count; p++) {
+		cali_info->delta_power_index_last[p] =
+			cali_info->delta_power_index
+				[p]; /*recording poer index offset*/
+
+		switch (p) {
+		case ODM_RF_PATH_B:
+			ODM_RT_TRACE(dm, ODM_COMP_TX_PWR_TRACK,
+				     "delta_swing_table_idx_tdown_b[%d] = %d\n",
+				     delta,
+				     delta_swing_table_idx_tdown_b[delta]);
+			cali_info->delta_power_index[p] =
+				-1 * delta_swing_table_idx_tdown_b[delta];
+			/*Record delta swing for mix mode pwr tracking*/
+			cali_info->absolute_ofdm_swing_idx[p] =
+				-1 * delta_swing_table_idx_tdown_b[delta];
+			ODM_RT_TRACE(
+				dm, ODM_COMP_TX_PWR_TRACK,
+				"******Temp is lower and cali_info->absolute_ofdm_swing_idx[ODM_RF_PATH_B] = %d\n",
+				cali_info->absolute_ofdm_swing_idx[p]);
+			break;
+
+		case ODM_RF_PATH_C:
+			ODM_RT_TRACE(dm, ODM_COMP_TX_PWR_TRACK,
+				     "delta_swing_table_idx_tdown_c[%d] = %d\n",
+				     delta,
+				     delta_swing_table_idx_tdown_c[delta]);
+			cali_info->delta_power_index[p] =
+				-1 * delta_swing_table_idx_tdown_c[delta];
+			/*Record delta swing for mix mode pwr tracking*/
+			cali_info->absolute_ofdm_swing_idx[p] =
+				-1 * delta_swing_table_idx_tdown_c[delta];
+			ODM_RT_TRACE(
+				dm, ODM_COMP_TX_PWR_TRACK,
+				"******Temp is lower and cali_info->absolute_ofdm_swing_idx[ODM_RF_PATH_C] = %d\n",
+				cali_info->absolute_ofdm_swing_idx[p]);
+			break;
+
+		case ODM_RF_PATH_D:
+			ODM_RT_TRACE(dm, ODM_COMP_TX_PWR_TRACK,
+				     "delta_swing_table_idx_tdown_d[%d] = %d\n",
+				     delta,
+				     delta_swing_table_idx_tdown_d[delta]);
+			cali_info->delta_power_index[p] =
+				-1 * delta_swing_table_idx_tdown_d[delta];
+			/*Record delta swing for mix mode pwr tracking*/
+			cali_info->absolute_ofdm_swing_idx[p] =
+				-1 * delta_swing_table_idx_tdown_d[delta];
+			ODM_RT_TRACE(
+				dm, ODM_COMP_TX_PWR_TRACK,
+				"******Temp is lower and cali_info->absolute_ofdm_swing_idx[ODM_RF_PATH_D] = %d\n",
+				cali_info->absolute_ofdm_swing_idx[p]);
+			break;
+
+		default:
+			ODM_RT_TRACE(dm, ODM_COMP_TX_PWR_TRACK,
+				     "delta_swing_table_idx_tdown_a[%d] = %d\n",
+				     delta,
+				     delta_swing_table_idx_tdown_a[delta]);
+			cali_info->delta_power_index[p] =
+				-1 * delta_swing_table_idx_tdown_a[delta];
+			/*Record delta swing for mix mode pwr tracking*/
+			cali_info->absolute_ofdm_swing_idx[p] =
+				-1 * delta_swing_table_idx_tdown_a[delta];
+			ODM_RT_TRACE(
+				dm, ODM_COMP_TX_PWR_TRACK,
+				"******Temp is lower and cali_info->absolute_ofdm_swing_idx[ODM_RF_PATH_A] = %d\n",
+				cali_info->absolute_ofdm_swing_idx[p]);
+			break;
+		}
+	}
+}
+
+static inline void phydm_odm_tx_power_set(struct phy_dm_struct *dm,
+					  struct txpwrtrack_cfg *c,
+					  u8 indexforchannel, u8 flag)
+{
+	u8 p = 0;
+
+	if (dm->support_ic_type == ODM_RTL8188E ||
+	    dm->support_ic_type == ODM_RTL8192E ||
+	    dm->support_ic_type == ODM_RTL8821 ||
+	    dm->support_ic_type == ODM_RTL8812 ||
+	    dm->support_ic_type == ODM_RTL8723B ||
+	    dm->support_ic_type == ODM_RTL8814A ||
+	    dm->support_ic_type == ODM_RTL8703B ||
+	    dm->support_ic_type == ODM_RTL8188F ||
+	    dm->support_ic_type == ODM_RTL8822B ||
+	    dm->support_ic_type == ODM_RTL8723D ||
+	    dm->support_ic_type == ODM_RTL8821C ||
+	    dm->support_ic_type == ODM_RTL8710B) { /* JJ ADD 20161014 */
+
+		ODM_RT_TRACE(
+			dm, ODM_COMP_TX_PWR_TRACK,
+			"**********Enter POWER Tracking MIX_MODE**********\n");
+		for (p = ODM_RF_PATH_A; p < c->rf_path_count; p++) {
+			if (flag == 0)
+				(*c->odm_tx_pwr_track_set_pwr)(dm, MIX_MODE, p,
+							       0);
+			else
+				(*c->odm_tx_pwr_track_set_pwr)(dm, MIX_MODE, p,
+							       indexforchannel);
+		}
+	} else {
+		ODM_RT_TRACE(
+			dm, ODM_COMP_TX_PWR_TRACK,
+			"**********Enter POWER Tracking BBSWING_MODE**********\n");
+		for (p = ODM_RF_PATH_A; p < c->rf_path_count; p++)
+			(*c->odm_tx_pwr_track_set_pwr)(dm, BBSWING, p,
+						       indexforchannel);
+	}
+}
+
+void configure_txpower_track(void *dm_void, struct txpwrtrack_cfg *config)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	/* JJ ADD 20161014 */
+
+	if (dm->support_ic_type == ODM_RTL8822B)
+		configure_txpower_track_8822b(config);
+}
+
+/* **********************************************************************
+ * <20121113, Kordan> This function should be called when tx_agc changed.
+ * Otherwise the previous compensation is gone, because we record the
+ * delta of temperature between two TxPowerTracking watch dogs.
+ *
+ * NOTE: If Tx BB swing or Tx scaling is varified during run-time, still
+ * need to call this function.
+ * ***********************************************************************/
+void odm_clear_txpowertracking_state(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)dm->adapter;
+	struct rtl_efuse *rtlefu = rtl_efuse(rtlpriv);
+	u8 p = 0;
+	struct dm_rf_calibration_struct *cali_info = &dm->rf_calibrate_info;
+
+	cali_info->bb_swing_idx_cck_base = cali_info->default_cck_index;
+	cali_info->bb_swing_idx_cck = cali_info->default_cck_index;
+	dm->rf_calibrate_info.CCK_index = 0;
+
+	for (p = ODM_RF_PATH_A; p < MAX_RF_PATH; ++p) {
+		cali_info->bb_swing_idx_ofdm_base[p] =
+			cali_info->default_ofdm_index;
+		cali_info->bb_swing_idx_ofdm[p] = cali_info->default_ofdm_index;
+		cali_info->OFDM_index[p] = cali_info->default_ofdm_index;
+
+		cali_info->power_index_offset[p] = 0;
+		cali_info->delta_power_index[p] = 0;
+		cali_info->delta_power_index_last[p] = 0;
+
+		cali_info->absolute_ofdm_swing_idx[p] =
+			0; /* Initial Mix mode power tracking*/
+		cali_info->remnant_ofdm_swing_idx[p] = 0;
+		cali_info->kfree_offset[p] = 0;
+	}
+
+	cali_info->modify_tx_agc_flag_path_a =
+		false; /*Initial at Modify Tx Scaling mode*/
+	cali_info->modify_tx_agc_flag_path_b =
+		false; /*Initial at Modify Tx Scaling mode*/
+	cali_info->modify_tx_agc_flag_path_c =
+		false; /*Initial at Modify Tx Scaling mode*/
+	cali_info->modify_tx_agc_flag_path_d =
+		false; /*Initial at Modify Tx Scaling mode*/
+	cali_info->remnant_cck_swing_idx = 0;
+	cali_info->thermal_value = rtlefu->eeprom_thermalmeter;
+
+	cali_info->modify_tx_agc_value_cck = 0; /* modify by Mingzhi.Guo */
+	cali_info->modify_tx_agc_value_ofdm = 0; /* modify by Mingzhi.Guo */
+}
+
+void odm_txpowertracking_callback_thermal_meter(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)dm->adapter;
+	struct rtl_efuse *rtlefu = rtl_efuse(rtlpriv);
+	void *adapter = dm->adapter;
+
+	struct dm_rf_calibration_struct *cali_info = &dm->rf_calibrate_info;
+
+	u8 thermal_value = 0, delta, delta_LCK, delta_IQK, p = 0, i = 0;
+	s8 diff_DPK[4]; /* use 'for..loop' to initialize */
+	u8 thermal_value_avg_count = 0;
+	u32 thermal_value_avg = 0, regc80, regcd0, regcd4, regab4;
+
+	/* OFDM BB Swing should be less than +3.0dB (required by Arthur) */
+	u8 OFDM_min_index = 0;
+	/* get_right_chnl_place_for_iqk(hal_data->current_channel) */
+	u8 indexforchannel = 0;
+	u8 power_tracking_type = 0; /* no specify type */
+	u8 xtal_offset_eanble = 0;
+
+	struct txpwrtrack_cfg c;
+
+	/* 4 1. The following TWO tables decide the final index of
+	 *      OFDM/CCK swing table.
+	 */
+	u8 *delta_swing_table_idx_tup_a = NULL;
+	u8 *delta_swing_table_idx_tdown_a = NULL;
+	u8 *delta_swing_table_idx_tup_b = NULL;
+	u8 *delta_swing_table_idx_tdown_b = NULL;
+	/*for 8814 add by Yu Chen*/
+	u8 *delta_swing_table_idx_tup_c = NULL;
+	u8 *delta_swing_table_idx_tdown_c = NULL;
+	u8 *delta_swing_table_idx_tup_d = NULL;
+	u8 *delta_swing_table_idx_tdown_d = NULL;
+	/*for Xtal Offset by James.Tung*/
+	s8 *delta_swing_table_xtal_up = NULL;
+	s8 *delta_swing_table_xtal_down = NULL;
+
+	/* 4 2. Initialization ( 7 steps in total ) */
+
+	configure_txpower_track(dm, &c);
+
+	(*c.get_delta_swing_table)(dm, (u8 **)&delta_swing_table_idx_tup_a,
+				   (u8 **)&delta_swing_table_idx_tdown_a,
+				   (u8 **)&delta_swing_table_idx_tup_b,
+				   (u8 **)&delta_swing_table_idx_tdown_b);
+
+	if (dm->support_ic_type & ODM_RTL8814A) /*for 8814 path C & D*/
+		(*c.get_delta_swing_table8814only)(
+			dm, (u8 **)&delta_swing_table_idx_tup_c,
+			(u8 **)&delta_swing_table_idx_tdown_c,
+			(u8 **)&delta_swing_table_idx_tup_d,
+			(u8 **)&delta_swing_table_idx_tdown_d);
+	/* JJ ADD 20161014 */
+	if (dm->support_ic_type &
+	    (ODM_RTL8703B | ODM_RTL8723D | ODM_RTL8710B)) /*for Xtal Offset*/
+		(*c.get_delta_swing_xtal_table)(
+			dm, (s8 **)&delta_swing_table_xtal_up,
+			(s8 **)&delta_swing_table_xtal_down);
+
+	cali_info->txpowertracking_callback_cnt++; /*cosa add for debug*/
+	cali_info->is_txpowertracking_init = true;
+
+	/*cali_info->txpowertrack_control = hal_data->txpowertrack_control;
+	 *<Kordan> We should keep updating ctrl variable according to HalData.
+	 *<Kordan> rf_calibrate_info.rega24 will be initialized when
+	 *ODM HW configuring, but MP configures with para files.
+	 */
+	if (dm->mp_mode)
+		cali_info->rega24 = 0x090e1317;
+
+	ODM_RT_TRACE(
+		dm, ODM_COMP_TX_PWR_TRACK,
+		"===>%s\n cali_info->bb_swing_idx_cck_base: %d, cali_info->bb_swing_idx_ofdm_base[A]: %d, cali_info->default_ofdm_index: %d\n",
+		__func__, cali_info->bb_swing_idx_cck_base,
+		cali_info->bb_swing_idx_ofdm_base[ODM_RF_PATH_A],
+		cali_info->default_ofdm_index);
+
+	ODM_RT_TRACE(
+		dm, ODM_COMP_TX_PWR_TRACK,
+		"cali_info->txpowertrack_control=%d,  rtlefu->eeprom_thermalmeter %d\n",
+		cali_info->txpowertrack_control, rtlefu->eeprom_thermalmeter);
+
+	thermal_value =
+		(u8)odm_get_rf_reg(dm, ODM_RF_PATH_A, c.thermal_reg_addr,
+				   0xfc00); /* 0x42: RF Reg[15:10] 88E */
+
+	/*add log by zhao he, check c80/c94/c14/ca0 value*/
+	if (dm->support_ic_type == ODM_RTL8723D) {
+		regc80 = odm_get_bb_reg(dm, 0xc80, MASKDWORD);
+		regcd0 = odm_get_bb_reg(dm, 0xcd0, MASKDWORD);
+		regcd4 = odm_get_bb_reg(dm, 0xcd4, MASKDWORD);
+		regab4 = odm_get_bb_reg(dm, 0xab4, 0x000007FF);
+		ODM_RT_TRACE(
+			dm, ODM_COMP_CALIBRATION,
+			"0xc80 = 0x%x 0xcd0 = 0x%x 0xcd4 = 0x%x 0xab4 = 0x%x\n",
+			regc80, regcd0, regcd4, regab4);
+	}
+	/* JJ ADD 20161014 */
+	if (dm->support_ic_type == ODM_RTL8710B) {
+		regc80 = odm_get_bb_reg(dm, 0xc80, MASKDWORD);
+		regcd0 = odm_get_bb_reg(dm, 0xcd0, MASKDWORD);
+		regcd4 = odm_get_bb_reg(dm, 0xcd4, MASKDWORD);
+		regab4 = odm_get_bb_reg(dm, 0xab4, 0x000007FF);
+		ODM_RT_TRACE(
+			dm, ODM_COMP_CALIBRATION,
+			"0xc80 = 0x%x 0xcd0 = 0x%x 0xcd4 = 0x%x 0xab4 = 0x%x\n",
+			regc80, regcd0, regcd4, regab4);
+	}
+
+	if (!cali_info->txpowertrack_control)
+		return;
+
+	/*4 3. Initialize ThermalValues of rf_calibrate_info*/
+
+	if (cali_info->is_reloadtxpowerindex)
+		ODM_RT_TRACE(dm, ODM_COMP_TX_PWR_TRACK,
+			     "reload ofdm index for band switch\n");
+
+	/*4 4. Calculate average thermal meter*/
+
+	cali_info->thermal_value_avg[cali_info->thermal_value_avg_index] =
+		thermal_value;
+	cali_info->thermal_value_avg_index++;
+	if (cali_info->thermal_value_avg_index ==
+	    c.average_thermal_num) /*Average times =  c.average_thermal_num*/
+		cali_info->thermal_value_avg_index = 0;
+
+	for (i = 0; i < c.average_thermal_num; i++) {
+		if (cali_info->thermal_value_avg[i]) {
+			thermal_value_avg += cali_info->thermal_value_avg[i];
+			thermal_value_avg_count++;
+		}
+	}
+
+	if (thermal_value_avg_count) {
+		/* Calculate Average thermal_value after average enough times */
+		thermal_value =
+			(u8)(thermal_value_avg / thermal_value_avg_count);
+		cali_info->thermal_value_delta =
+			thermal_value - rtlefu->eeprom_thermalmeter;
+		ODM_RT_TRACE(
+			dm, ODM_COMP_TX_PWR_TRACK,
+			"AVG Thermal Meter = 0x%X, EFUSE Thermal base = 0x%X\n",
+			thermal_value, rtlefu->eeprom_thermalmeter);
+	}
+
+	/* 4 5. Calculate delta, delta_LCK, delta_IQK. */
+
+	/* "delta" is used to determine whether thermal value changes or not*/
+	delta = (thermal_value > cali_info->thermal_value) ?
+			(thermal_value - cali_info->thermal_value) :
+			(cali_info->thermal_value - thermal_value);
+	delta_LCK = (thermal_value > cali_info->thermal_value_lck) ?
+			    (thermal_value - cali_info->thermal_value_lck) :
+			    (cali_info->thermal_value_lck - thermal_value);
+	delta_IQK = (thermal_value > cali_info->thermal_value_iqk) ?
+			    (thermal_value - cali_info->thermal_value_iqk) :
+			    (cali_info->thermal_value_iqk - thermal_value);
+
+	if (cali_info->thermal_value_iqk ==
+	    0xff) { /*no PG, use thermal value for IQK*/
+		cali_info->thermal_value_iqk = thermal_value;
+		delta_IQK =
+			(thermal_value > cali_info->thermal_value_iqk) ?
+				(thermal_value - cali_info->thermal_value_iqk) :
+				(cali_info->thermal_value_iqk - thermal_value);
+		ODM_RT_TRACE(dm, ODM_COMP_TX_PWR_TRACK,
+			     "no PG, use thermal_value for IQK\n");
+	}
+
+	for (p = ODM_RF_PATH_A; p < c.rf_path_count; p++)
+		diff_DPK[p] = (s8)thermal_value - (s8)cali_info->dpk_thermal[p];
+
+	/*4 6. If necessary, do LCK.*/
+
+	if (!(dm->support_ic_type &
+	      ODM_RTL8821)) { /*no PG, do LCK at initial status*/
+		if (cali_info->thermal_value_lck == 0xff) {
+			ODM_RT_TRACE(dm, ODM_COMP_TX_PWR_TRACK,
+				     "no PG, do LCK\n");
+			cali_info->thermal_value_lck = thermal_value;
+
+			/*Use RTLCK, so close power tracking driver LCK*/
+			if (!(dm->support_ic_type & ODM_RTL8814A) &&
+			    c.phy_lc_calibrate)
+				(*c.phy_lc_calibrate)(dm);
+
+			delta_LCK =
+				(thermal_value > cali_info->thermal_value_lck) ?
+					(thermal_value -
+					 cali_info->thermal_value_lck) :
+					(cali_info->thermal_value_lck -
+					 thermal_value);
+		}
+
+		ODM_RT_TRACE(dm, ODM_COMP_TX_PWR_TRACK,
+			     "(delta, delta_LCK, delta_IQK) = (%d, %d, %d)\n",
+			     delta, delta_LCK, delta_IQK);
+
+		/*Delta temperature is equal to or larger than 20 centigrade.*/
+		if (delta_LCK >= c.threshold_iqk) {
+			ODM_RT_TRACE(dm, ODM_COMP_TX_PWR_TRACK,
+				     "delta_LCK(%d) >= threshold_iqk(%d)\n",
+				     delta_LCK, c.threshold_iqk);
+			cali_info->thermal_value_lck = thermal_value;
+
+			/*Use RTLCK, so close power tracking driver LCK*/
+			if (!(dm->support_ic_type & ODM_RTL8814A) &&
+			    c.phy_lc_calibrate)
+				(*c.phy_lc_calibrate)(dm);
+		}
+	}
+
+	/*3 7. If necessary, move the index of swing table to adjust Tx power.*/
+
+	if (delta > 0 && cali_info->txpowertrack_control) {
+		/* "delta" here is used to record the abs value of difference.*/
+		delta = thermal_value > rtlefu->eeprom_thermalmeter ?
+				(thermal_value - rtlefu->eeprom_thermalmeter) :
+				(rtlefu->eeprom_thermalmeter - thermal_value);
+		if (delta >= TXPWR_TRACK_TABLE_SIZE)
+			delta = TXPWR_TRACK_TABLE_SIZE - 1;
+
+		/*4 7.1 The Final Power index = BaseIndex + power_index_offset*/
+
+		if (thermal_value > rtlefu->eeprom_thermalmeter) {
+			phydm_set_calibrate_info_up(
+				dm, &c, delta, cali_info,
+				delta_swing_table_idx_tup_a,
+				delta_swing_table_idx_tup_b,
+				delta_swing_table_idx_tup_c,
+				delta_swing_table_idx_tup_d);
+			/* JJ ADD 20161014 */
+			if (dm->support_ic_type &
+			    (ODM_RTL8703B | ODM_RTL8723D | ODM_RTL8710B)) {
+				/*Save xtal_offset from Xtal table*/
+
+				/*recording last Xtal offset*/
+				cali_info->xtal_offset_last =
+					cali_info->xtal_offset;
+				ODM_RT_TRACE(
+					dm, ODM_COMP_TX_PWR_TRACK,
+					"[Xtal] delta_swing_table_xtal_up[%d] = %d\n",
+					delta,
+					delta_swing_table_xtal_up[delta]);
+				cali_info->xtal_offset =
+					delta_swing_table_xtal_up[delta];
+				xtal_offset_eanble =
+					(cali_info->xtal_offset_last ==
+					 cali_info->xtal_offset) ?
+						0 :
+						1;
+			}
+
+		} else {
+			phydm_set_calibrate_info_down(
+				dm, &c, delta, cali_info,
+				delta_swing_table_idx_tdown_a,
+				delta_swing_table_idx_tdown_b,
+				delta_swing_table_idx_tdown_c,
+				delta_swing_table_idx_tdown_d);
+			/* JJ ADD 20161014 */
+			if (dm->support_ic_type &
+			    (ODM_RTL8703B | ODM_RTL8723D | ODM_RTL8710B)) {
+				/*Save xtal_offset from Xtal table*/
+
+				/*recording last Xtal offset*/
+				cali_info->xtal_offset_last =
+					cali_info->xtal_offset;
+				ODM_RT_TRACE(
+					dm, ODM_COMP_TX_PWR_TRACK,
+					"[Xtal] delta_swing_table_xtal_down[%d] = %d\n",
+					delta,
+					delta_swing_table_xtal_down[delta]);
+				cali_info->xtal_offset =
+					delta_swing_table_xtal_down[delta];
+				xtal_offset_eanble =
+					(cali_info->xtal_offset_last ==
+					 cali_info->xtal_offset) ?
+						0 :
+						1;
+			}
+		}
+
+		for (p = ODM_RF_PATH_A; p < c.rf_path_count; p++) {
+			ODM_RT_TRACE(
+				dm, ODM_COMP_TX_PWR_TRACK,
+				"\n\n=========================== [path-%d] Calculating power_index_offset===========================\n",
+				p);
+
+			if (cali_info->delta_power_index[p] ==
+			    cali_info->delta_power_index_last[p]) {
+				/* If Thermal value changes but lookup table
+				 * value still the same
+				 */
+				cali_info->power_index_offset[p] = 0;
+			} else {
+				/*Power idx diff between 2 times Pwr Tracking*/
+				cali_info->power_index_offset[p] =
+					cali_info->delta_power_index[p] -
+					cali_info->delta_power_index_last[p];
+			}
+
+			ODM_RT_TRACE(
+				dm, ODM_COMP_TX_PWR_TRACK,
+				"[path-%d] power_index_offset(%d) = delta_power_index(%d) - delta_power_index_last(%d)\n",
+				p, cali_info->power_index_offset[p],
+				cali_info->delta_power_index[p],
+				cali_info->delta_power_index_last[p]);
+
+			cali_info->OFDM_index[p] =
+				cali_info->bb_swing_idx_ofdm_base[p] +
+				cali_info->power_index_offset[p];
+			cali_info->CCK_index =
+				cali_info->bb_swing_idx_cck_base +
+				cali_info->power_index_offset[p];
+
+			cali_info->bb_swing_idx_cck = cali_info->CCK_index;
+			cali_info->bb_swing_idx_ofdm[p] =
+				cali_info->OFDM_index[p];
+
+			/*******Print BB Swing base and index Offset**********/
+
+			ODM_RT_TRACE(
+				dm, ODM_COMP_TX_PWR_TRACK,
+				"The 'CCK' final index(%d) = BaseIndex(%d) + power_index_offset(%d)\n",
+				cali_info->bb_swing_idx_cck,
+				cali_info->bb_swing_idx_cck_base,
+				cali_info->power_index_offset[p]);
+			ODM_RT_TRACE(
+				dm, ODM_COMP_TX_PWR_TRACK,
+				"The 'OFDM' final index(%d) = BaseIndex[%d](%d) + power_index_offset(%d)\n",
+				cali_info->bb_swing_idx_ofdm[p], p,
+				cali_info->bb_swing_idx_ofdm_base[p],
+				cali_info->power_index_offset[p]);
+
+			/*4 7.1 Handle boundary conditions of index.*/
+
+			if (cali_info->OFDM_index[p] >
+			    c.swing_table_size_ofdm - 1)
+				cali_info->OFDM_index[p] =
+					c.swing_table_size_ofdm - 1;
+			else if (cali_info->OFDM_index[p] <= OFDM_min_index)
+				cali_info->OFDM_index[p] = OFDM_min_index;
+		}
+
+		ODM_RT_TRACE(
+			dm, ODM_COMP_TX_PWR_TRACK,
+			"\n\n========================================================================================================\n");
+
+		if (cali_info->CCK_index > c.swing_table_size_cck - 1)
+			cali_info->CCK_index = c.swing_table_size_cck - 1;
+		else if (cali_info->CCK_index <= 0)
+			cali_info->CCK_index = 0;
+	} else {
+		ODM_RT_TRACE(
+			dm, ODM_COMP_TX_PWR_TRACK,
+			"The thermal meter is unchanged or TxPowerTracking OFF(%d): thermal_value: %d, cali_info->thermal_value: %d\n",
+			cali_info->txpowertrack_control, thermal_value,
+			cali_info->thermal_value);
+
+		for (p = ODM_RF_PATH_A; p < c.rf_path_count; p++)
+			cali_info->power_index_offset[p] = 0;
+	}
+
+	/*Print Swing base & current*/
+	ODM_RT_TRACE(
+		dm, ODM_COMP_TX_PWR_TRACK,
+		"TxPowerTracking: [CCK] Swing Current index: %d, Swing base index: %d\n",
+		cali_info->CCK_index, cali_info->bb_swing_idx_cck_base);
+
+	for (p = ODM_RF_PATH_A; p < c.rf_path_count; p++)
+		ODM_RT_TRACE(
+			dm, ODM_COMP_TX_PWR_TRACK,
+			"TxPowerTracking: [OFDM] Swing Current index: %d, Swing base index[%d]: %d\n",
+			cali_info->OFDM_index[p], p,
+			cali_info->bb_swing_idx_ofdm_base[p]);
+
+	if ((dm->support_ic_type & ODM_RTL8814A)) {
+		ODM_RT_TRACE(dm, ODM_COMP_TX_PWR_TRACK,
+			     "power_tracking_type=%d\n", power_tracking_type);
+
+		if (power_tracking_type == 0) {
+			ODM_RT_TRACE(
+				dm, ODM_COMP_TX_PWR_TRACK,
+				"**********Enter POWER Tracking MIX_MODE**********\n");
+			for (p = ODM_RF_PATH_A; p < c.rf_path_count; p++)
+				(*c.odm_tx_pwr_track_set_pwr)(dm, MIX_MODE, p,
+							      0);
+		} else if (power_tracking_type == 1) {
+			ODM_RT_TRACE(
+				dm, ODM_COMP_TX_PWR_TRACK,
+				"**********Enter POWER Tracking MIX(2G) TSSI(5G) MODE**********\n");
+			for (p = ODM_RF_PATH_A; p < c.rf_path_count; p++)
+				(*c.odm_tx_pwr_track_set_pwr)(
+					dm, MIX_2G_TSSI_5G_MODE, p, 0);
+		} else if (power_tracking_type == 2) {
+			ODM_RT_TRACE(
+				dm, ODM_COMP_TX_PWR_TRACK,
+				"**********Enter POWER Tracking MIX(5G) TSSI(2G)MODE**********\n");
+			for (p = ODM_RF_PATH_A; p < c.rf_path_count; p++)
+				(*c.odm_tx_pwr_track_set_pwr)(
+					dm, MIX_5G_TSSI_2G_MODE, p, 0);
+		} else if (power_tracking_type == 3) {
+			ODM_RT_TRACE(
+				dm, ODM_COMP_TX_PWR_TRACK,
+				"**********Enter POWER Tracking TSSI MODE**********\n");
+			for (p = ODM_RF_PATH_A; p < c.rf_path_count; p++)
+				(*c.odm_tx_pwr_track_set_pwr)(dm, TSSI_MODE, p,
+							      0);
+		}
+		/*Record last Power Tracking Thermal value*/
+		cali_info->thermal_value = thermal_value;
+
+	} else if ((cali_info->power_index_offset[ODM_RF_PATH_A] != 0 ||
+		    cali_info->power_index_offset[ODM_RF_PATH_B] != 0 ||
+		    cali_info->power_index_offset[ODM_RF_PATH_C] != 0 ||
+		    cali_info->power_index_offset[ODM_RF_PATH_D] != 0) &&
+		   cali_info->txpowertrack_control &&
+		   (rtlefu->eeprom_thermalmeter != 0xff)) {
+		/* 4 7.2 Configure the Swing Table to adjust Tx Power. */
+
+		/*Always true after Tx Power is adjusted by power tracking.*/
+		cali_info->is_tx_power_changed = true;
+		/* 2012/04/23 MH According to Luke's suggestion, we can not
+		 * write BB digital to increase TX power. Otherwise, EVM will
+		 * be bad.
+		 */
+		/* 2012/04/25 MH Add for tx power tracking to set tx power in
+		 * tx agc for 88E.
+		 */
+		if (thermal_value > cali_info->thermal_value) {
+			for (p = ODM_RF_PATH_A; p < c.rf_path_count; p++) {
+				/* print temperature increasing */
+				ODM_RT_TRACE(
+					dm, ODM_COMP_TX_PWR_TRACK,
+					"Temperature Increasing(%d): delta_pi: %d, delta_t: %d, Now_t: %d, EFUSE_t: %d, Last_t: %d\n",
+					p, cali_info->power_index_offset[p],
+					delta, thermal_value,
+					rtlefu->eeprom_thermalmeter,
+					cali_info->thermal_value);
+			}
+		} else if (thermal_value <
+			   cali_info->thermal_value) { /*Low temperature*/
+			for (p = ODM_RF_PATH_A; p < c.rf_path_count; p++) {
+				/* print temperature decreasing */
+				ODM_RT_TRACE(
+					dm, ODM_COMP_TX_PWR_TRACK,
+					"Temperature Decreasing(%d): delta_pi: %d, delta_t: %d, Now_t: %d, EFUSE_t: %d, Last_t: %d\n",
+					p, cali_info->power_index_offset[p],
+					delta, thermal_value,
+					rtlefu->eeprom_thermalmeter,
+					cali_info->thermal_value);
+			}
+		}
+
+		if (thermal_value > rtlefu->eeprom_thermalmeter) {
+			ODM_RT_TRACE(
+				dm, ODM_COMP_TX_PWR_TRACK,
+				"Temperature(%d) higher than PG value(%d)\n",
+				thermal_value, rtlefu->eeprom_thermalmeter);
+
+			phydm_odm_tx_power_set(dm, &c, indexforchannel, 0);
+		} else {
+			ODM_RT_TRACE(
+				dm, ODM_COMP_TX_PWR_TRACK,
+				"Temperature(%d) lower than PG value(%d)\n",
+				thermal_value, rtlefu->eeprom_thermalmeter);
+			phydm_odm_tx_power_set(dm, &c, indexforchannel, 1);
+		}
+
+		/*Record last time Power Tracking result as base.*/
+		cali_info->bb_swing_idx_cck_base = cali_info->bb_swing_idx_cck;
+
+		for (p = ODM_RF_PATH_A; p < c.rf_path_count; p++)
+			cali_info->bb_swing_idx_ofdm_base[p] =
+				cali_info->bb_swing_idx_ofdm[p];
+
+		ODM_RT_TRACE(
+			dm, ODM_COMP_TX_PWR_TRACK,
+			"cali_info->thermal_value = %d thermal_value= %d\n",
+			cali_info->thermal_value, thermal_value);
+
+		/*Record last Power Tracking Thermal value*/
+		cali_info->thermal_value = thermal_value;
+	}
+
+	if (dm->support_ic_type == ODM_RTL8703B ||
+	    dm->support_ic_type == ODM_RTL8723D ||
+	    dm->support_ic_type == ODM_RTL8710B) { /* JJ ADD 20161014 */
+
+		if (xtal_offset_eanble != 0 &&
+		    cali_info->txpowertrack_control &&
+		    (rtlefu->eeprom_thermalmeter != 0xff)) {
+			ODM_RT_TRACE(
+				dm, ODM_COMP_TX_PWR_TRACK,
+				"**********Enter Xtal Tracking**********\n");
+
+			if (thermal_value > rtlefu->eeprom_thermalmeter) {
+				ODM_RT_TRACE(
+					dm, ODM_COMP_TX_PWR_TRACK,
+					"Temperature(%d) higher than PG value(%d)\n",
+					thermal_value,
+					rtlefu->eeprom_thermalmeter);
+				(*c.odm_txxtaltrack_set_xtal)(dm);
+			} else {
+				ODM_RT_TRACE(
+					dm, ODM_COMP_TX_PWR_TRACK,
+					"Temperature(%d) lower than PG value(%d)\n",
+					thermal_value,
+					rtlefu->eeprom_thermalmeter);
+				(*c.odm_txxtaltrack_set_xtal)(dm);
+			}
+		}
+		ODM_RT_TRACE(dm, ODM_COMP_TX_PWR_TRACK,
+			     "**********End Xtal Tracking**********\n");
+	}
+
+	if (!IS_HARDWARE_TYPE_8723B(adapter)) {
+		/* Delta temperature is equal to or larger than 20 centigrade
+		 * (When threshold is 8).
+		 */
+		if (delta_IQK >= c.threshold_iqk) {
+			ODM_RT_TRACE(dm, ODM_COMP_TX_PWR_TRACK,
+				     "delta_IQK(%d) >= threshold_iqk(%d)\n",
+				     delta_IQK, c.threshold_iqk);
+			if (!cali_info->is_iqk_in_progress)
+				(*c.do_iqk)(dm, delta_IQK, thermal_value, 8);
+		}
+	}
+	if (cali_info->dpk_thermal[ODM_RF_PATH_A] != 0) {
+		if (diff_DPK[ODM_RF_PATH_A] >= c.threshold_dpk) {
+			odm_set_bb_reg(dm, 0x82c, BIT(31), 0x1);
+			odm_set_bb_reg(
+				dm, 0xcc4,
+				BIT(14) | BIT(13) | BIT(12) | BIT(11) | BIT(10),
+				(diff_DPK[ODM_RF_PATH_A] / c.threshold_dpk));
+			odm_set_bb_reg(dm, 0x82c, BIT(31), 0x0);
+		} else if ((diff_DPK[ODM_RF_PATH_A] <= -1 * c.threshold_dpk)) {
+			s32 value = 0x20 +
+				    (diff_DPK[ODM_RF_PATH_A] / c.threshold_dpk);
+
+			odm_set_bb_reg(dm, 0x82c, BIT(31), 0x1);
+			odm_set_bb_reg(dm, 0xcc4, BIT(14) | BIT(13) | BIT(12) |
+							  BIT(11) | BIT(10),
+				       value);
+			odm_set_bb_reg(dm, 0x82c, BIT(31), 0x0);
+		} else {
+			odm_set_bb_reg(dm, 0x82c, BIT(31), 0x1);
+			odm_set_bb_reg(dm, 0xcc4, BIT(14) | BIT(13) | BIT(12) |
+							  BIT(11) | BIT(10),
+				       0);
+			odm_set_bb_reg(dm, 0x82c, BIT(31), 0x0);
+		}
+	}
+	if (cali_info->dpk_thermal[ODM_RF_PATH_B] != 0) {
+		if (diff_DPK[ODM_RF_PATH_B] >= c.threshold_dpk) {
+			odm_set_bb_reg(dm, 0x82c, BIT(31), 0x1);
+			odm_set_bb_reg(
+				dm, 0xec4,
+				BIT(14) | BIT(13) | BIT(12) | BIT(11) | BIT(10),
+				(diff_DPK[ODM_RF_PATH_B] / c.threshold_dpk));
+			odm_set_bb_reg(dm, 0x82c, BIT(31), 0x0);
+		} else if ((diff_DPK[ODM_RF_PATH_B] <= -1 * c.threshold_dpk)) {
+			s32 value = 0x20 +
+				    (diff_DPK[ODM_RF_PATH_B] / c.threshold_dpk);
+
+			odm_set_bb_reg(dm, 0x82c, BIT(31), 0x1);
+			odm_set_bb_reg(dm, 0xec4, BIT(14) | BIT(13) | BIT(12) |
+							  BIT(11) | BIT(10),
+				       value);
+			odm_set_bb_reg(dm, 0x82c, BIT(31), 0x0);
+		} else {
+			odm_set_bb_reg(dm, 0x82c, BIT(31), 0x1);
+			odm_set_bb_reg(dm, 0xec4, BIT(14) | BIT(13) | BIT(12) |
+							  BIT(11) | BIT(10),
+				       0);
+			odm_set_bb_reg(dm, 0x82c, BIT(31), 0x0);
+		}
+	}
+
+	ODM_RT_TRACE(dm, ODM_COMP_TX_PWR_TRACK, "<===%s\n", __func__);
+
+	cali_info->tx_powercount = 0;
+}
+
+/* 3============================================================
+ * 3 IQ Calibration
+ * 3============================================================
+ */
+
+void odm_reset_iqk_result(void *dm_void) { return; }
+
+u8 odm_get_right_chnl_place_for_iqk(u8 chnl)
+{
+	u8 channel_all[ODM_TARGET_CHNL_NUM_2G_5G] = {
+		1,   2,   3,   4,   5,   6,   7,   8,   9,   10,  11,  12,
+		13,  14,  36,  38,  40,  42,  44,  46,  48,  50,  52,  54,
+		56,  58,  60,  62,  64,  100, 102, 104, 106, 108, 110, 112,
+		114, 116, 118, 120, 122, 124, 126, 128, 130, 132, 134, 136,
+		138, 140, 149, 151, 153, 155, 157, 159, 161, 163, 165};
+	u8 place = chnl;
+
+	if (chnl > 14) {
+		for (place = 14; place < sizeof(channel_all); place++) {
+			if (channel_all[place] == chnl)
+				return place - 13;
+		}
+	}
+	return 0;
+}
+
+static void odm_iq_calibrate(struct phy_dm_struct *dm)
+{
+	void *adapter = dm->adapter;
+
+	if (IS_HARDWARE_TYPE_8812AU(adapter))
+		return;
+
+	if (dm->is_linked) {
+		if ((*dm->channel != dm->pre_channel) &&
+		    (!*dm->is_scan_in_process)) {
+			dm->pre_channel = *dm->channel;
+			dm->linked_interval = 0;
+		}
+
+		if (dm->linked_interval < 3)
+			dm->linked_interval++;
+
+		if (dm->linked_interval == 2) {
+			if (IS_HARDWARE_TYPE_8814A(adapter))
+				;
+
+			else if (IS_HARDWARE_TYPE_8822B(adapter))
+				phy_iq_calibrate_8822b(dm, false);
+		}
+	} else {
+		dm->linked_interval = 0;
+	}
+}
+
+void phydm_rf_init(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	odm_txpowertracking_init(dm);
+
+	odm_clear_txpowertracking_state(dm);
+}
+
+void phydm_rf_watchdog(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	odm_txpowertracking_check(dm);
+	if (dm->support_ic_type & ODM_IC_11AC_SERIES)
+		odm_iq_calibrate(dm);
+}
diff --git a/drivers/staging/rtlwifi/phydm/halphyrf_ce.h b/drivers/staging/rtlwifi/phydm/halphyrf_ce.h
new file mode 100644
index 000000000000..e5d6257efb2b
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/halphyrf_ce.h
@@ -0,0 +1,85 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __HAL_PHY_RF_H__
+#define __HAL_PHY_RF_H__
+
+#include "phydm_kfree.h"
+
+#include "rtl8822b/phydm_iqk_8822b.h"
+
+#include "phydm_powertracking_ce.h"
+
+enum spur_cal_method { PLL_RESET, AFE_PHASE_SEL };
+
+enum pwrtrack_method {
+	BBSWING,
+	TXAGC,
+	MIX_MODE,
+	TSSI_MODE,
+	MIX_2G_TSSI_5G_MODE,
+	MIX_5G_TSSI_2G_MODE
+};
+
+typedef void (*func_set_pwr)(void *, enum pwrtrack_method, u8, u8);
+typedef void (*func_iqk)(void *, u8, u8, u8);
+typedef void (*func_lck)(void *);
+typedef void (*func_swing)(void *, u8 **, u8 **, u8 **, u8 **);
+typedef void (*func_swing8814only)(void *, u8 **, u8 **, u8 **, u8 **);
+typedef void (*func_swing_xtal)(void *, s8 **, s8 **);
+typedef void (*func_set_xtal)(void *);
+
+struct txpwrtrack_cfg {
+	u8 swing_table_size_cck;
+	u8 swing_table_size_ofdm;
+	u8 threshold_iqk;
+	u8 threshold_dpk;
+	u8 average_thermal_num;
+	u8 rf_path_count;
+	u32 thermal_reg_addr;
+	func_set_pwr odm_tx_pwr_track_set_pwr;
+	func_iqk do_iqk;
+	func_lck phy_lc_calibrate;
+	func_swing get_delta_swing_table;
+	func_swing8814only get_delta_swing_table8814only;
+	func_swing_xtal get_delta_swing_xtal_table;
+	func_set_xtal odm_txxtaltrack_set_xtal;
+};
+
+void configure_txpower_track(void *dm_void, struct txpwrtrack_cfg *config);
+
+void odm_clear_txpowertracking_state(void *dm_void);
+
+void odm_txpowertracking_callback_thermal_meter(void *dm);
+
+#define ODM_TARGET_CHNL_NUM_2G_5G 59
+
+void odm_reset_iqk_result(void *dm_void);
+u8 odm_get_right_chnl_place_for_iqk(u8 chnl);
+
+void phydm_rf_init(void *dm_void);
+void phydm_rf_watchdog(void *dm_void);
+
+#endif /*  #ifndef __HAL_PHY_RF_H__ */
diff --git a/drivers/staging/rtlwifi/phydm/mp_precomp.h b/drivers/staging/rtlwifi/phydm/mp_precomp.h
new file mode 100644
index 000000000000..b313de511ed6
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/mp_precomp.h
@@ -0,0 +1,24 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
diff --git a/drivers/staging/rtlwifi/phydm/phydm.c b/drivers/staging/rtlwifi/phydm/phydm.c
new file mode 100644
index 000000000000..37888c3087a4
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm.c
@@ -0,0 +1,1986 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+/* ************************************************************
+ * include files
+ * *************************************************************/
+
+#include "mp_precomp.h"
+#include "phydm_precomp.h"
+
+static const u16 db_invert_table[12][8] = {
+	{1, 1, 1, 2, 2, 2, 2, 3},
+	{3, 3, 4, 4, 4, 5, 6, 6},
+	{7, 8, 9, 10, 11, 13, 14, 16},
+	{18, 20, 22, 25, 28, 32, 35, 40},
+	{45, 50, 56, 63, 71, 79, 89, 100},
+	{112, 126, 141, 158, 178, 200, 224, 251},
+	{282, 316, 355, 398, 447, 501, 562, 631},
+	{708, 794, 891, 1000, 1122, 1259, 1413, 1585},
+	{1778, 1995, 2239, 2512, 2818, 3162, 3548, 3981},
+	{4467, 5012, 5623, 6310, 7079, 7943, 8913, 10000},
+	{11220, 12589, 14125, 15849, 17783, 19953, 22387, 25119},
+	{28184, 31623, 35481, 39811, 44668, 50119, 56234, 65535},
+};
+
+/* ************************************************************
+ * Local Function predefine.
+ * *************************************************************/
+
+/* START------------COMMON INFO RELATED--------------- */
+
+static void odm_update_power_training_state(struct phy_dm_struct *dm);
+
+/* ************************************************************
+ * 3 Export Interface
+ * *************************************************************/
+
+/*Y = 10*log(X)*/
+s32 odm_pwdb_conversion(s32 X, u32 total_bit, u32 decimal_bit)
+{
+	s32 Y, integer = 0, decimal = 0;
+	u32 i;
+
+	if (X == 0)
+		X = 1; /* log2(x), x can't be 0 */
+
+	for (i = (total_bit - 1); i > 0; i--) {
+		if (X & BIT(i)) {
+			integer = i;
+			if (i > 0) {
+				/* decimal is 0.5dB*3=1.5dB~=2dB */
+				decimal = (X & BIT(i - 1)) ? 2 : 0;
+			}
+			break;
+		}
+	}
+
+	Y = 3 * (integer - decimal_bit) + decimal; /* 10*log(x)=3*log2(x), */
+
+	return Y;
+}
+
+s32 odm_sign_conversion(s32 value, u32 total_bit)
+{
+	if (value & BIT(total_bit - 1))
+		value -= BIT(total_bit);
+	return value;
+}
+
+void phydm_seq_sorting(void *dm_void, u32 *value, u32 *rank_idx, u32 *idx_out,
+		       u8 seq_length)
+{
+	u8 i = 0, j = 0;
+	u32 tmp_a, tmp_b;
+	u32 tmp_idx_a, tmp_idx_b;
+
+	for (i = 0; i < seq_length; i++) {
+		rank_idx[i] = i;
+		/**/
+	}
+
+	for (i = 0; i < (seq_length - 1); i++) {
+		for (j = 0; j < (seq_length - 1 - i); j++) {
+			tmp_a = value[j];
+			tmp_b = value[j + 1];
+
+			tmp_idx_a = rank_idx[j];
+			tmp_idx_b = rank_idx[j + 1];
+
+			if (tmp_a < tmp_b) {
+				value[j] = tmp_b;
+				value[j + 1] = tmp_a;
+
+				rank_idx[j] = tmp_idx_b;
+				rank_idx[j + 1] = tmp_idx_a;
+			}
+		}
+	}
+
+	for (i = 0; i < seq_length; i++) {
+		idx_out[rank_idx[i]] = i + 1;
+		/**/
+	}
+}
+
+void odm_init_mp_driver_status(struct phy_dm_struct *dm)
+{
+	dm->mp_mode = false;
+}
+
+static void odm_update_mp_driver_status(struct phy_dm_struct *dm)
+{
+	/* Do nothing. */
+}
+
+static void phydm_init_trx_antenna_setting(struct phy_dm_struct *dm)
+{
+	/*#if (RTL8814A_SUPPORT == 1)*/
+
+	if (dm->support_ic_type & (ODM_RTL8814A)) {
+		u8 rx_ant = 0, tx_ant = 0;
+
+		rx_ant = (u8)odm_get_bb_reg(dm, ODM_REG(BB_RX_PATH, dm),
+					    ODM_BIT(BB_RX_PATH, dm));
+		tx_ant = (u8)odm_get_bb_reg(dm, ODM_REG(BB_TX_PATH, dm),
+					    ODM_BIT(BB_TX_PATH, dm));
+		dm->tx_ant_status = (tx_ant & 0xf);
+		dm->rx_ant_status = (rx_ant & 0xf);
+	} else if (dm->support_ic_type & (ODM_RTL8723D | ODM_RTL8821C |
+					  ODM_RTL8710B)) { /* JJ ADD 20161014 */
+		dm->tx_ant_status = 0x1;
+		dm->rx_ant_status = 0x1;
+	}
+	/*#endif*/
+}
+
+static void phydm_traffic_load_decision(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	/*---TP & Trafic-load calculation---*/
+
+	if (dm->last_tx_ok_cnt > *dm->num_tx_bytes_unicast)
+		dm->last_tx_ok_cnt = *dm->num_tx_bytes_unicast;
+
+	if (dm->last_rx_ok_cnt > *dm->num_rx_bytes_unicast)
+		dm->last_rx_ok_cnt = *dm->num_rx_bytes_unicast;
+
+	dm->cur_tx_ok_cnt = *dm->num_tx_bytes_unicast - dm->last_tx_ok_cnt;
+	dm->cur_rx_ok_cnt = *dm->num_rx_bytes_unicast - dm->last_rx_ok_cnt;
+	dm->last_tx_ok_cnt = *dm->num_tx_bytes_unicast;
+	dm->last_rx_ok_cnt = *dm->num_rx_bytes_unicast;
+
+	dm->tx_tp = ((dm->tx_tp) >> 1) +
+		    (u32)(((dm->cur_tx_ok_cnt) >> 18) >>
+			  1); /* <<3(8bit), >>20(10^6,M), >>1(2sec)*/
+	dm->rx_tp = ((dm->rx_tp) >> 1) +
+		    (u32)(((dm->cur_rx_ok_cnt) >> 18) >>
+			  1); /* <<3(8bit), >>20(10^6,M), >>1(2sec)*/
+	dm->total_tp = dm->tx_tp + dm->rx_tp;
+
+	dm->pre_traffic_load = dm->traffic_load;
+
+	if (dm->cur_tx_ok_cnt > 1875000 ||
+	    dm->cur_rx_ok_cnt >
+		    1875000) { /* ( 1.875M * 8bit ) / 2sec= 7.5M bits /sec )*/
+
+		dm->traffic_load = TRAFFIC_HIGH;
+		/**/
+	} else if (
+		dm->cur_tx_ok_cnt > 500000 ||
+		dm->cur_rx_ok_cnt >
+			500000) { /*( 0.5M * 8bit ) / 2sec =  2M bits /sec )*/
+
+		dm->traffic_load = TRAFFIC_MID;
+		/**/
+	} else if (
+		dm->cur_tx_ok_cnt > 100000 ||
+		dm->cur_rx_ok_cnt >
+			100000) { /*( 0.1M * 8bit ) / 2sec =  0.4M bits /sec )*/
+
+		dm->traffic_load = TRAFFIC_LOW;
+		/**/
+	} else {
+		dm->traffic_load = TRAFFIC_ULTRA_LOW;
+		/**/
+	}
+}
+
+static void phydm_config_ofdm_tx_path(struct phy_dm_struct *dm, u32 path) {}
+
+void phydm_config_ofdm_rx_path(struct phy_dm_struct *dm, u32 path)
+{
+	u8 ofdm_rx_path = 0;
+
+	if (dm->support_ic_type & (ODM_RTL8192E)) {
+	} else if (dm->support_ic_type & (ODM_RTL8812 | ODM_RTL8822B)) {
+		if (path == PHYDM_A) {
+			ofdm_rx_path = 1;
+			/**/
+		} else if (path == PHYDM_B) {
+			ofdm_rx_path = 2;
+			/**/
+		} else if (path == PHYDM_AB) {
+			ofdm_rx_path = 3;
+			/**/
+		}
+
+		odm_set_bb_reg(dm, 0x808, MASKBYTE0,
+			       ((ofdm_rx_path << 4) | ofdm_rx_path));
+	}
+}
+
+static void phydm_config_cck_rx_antenna_init(struct phy_dm_struct *dm) {}
+
+static void phydm_config_cck_rx_path(struct phy_dm_struct *dm, u8 path,
+				     u8 path_div_en)
+{
+}
+
+void phydm_config_trx_path(void *dm_void, u32 *const dm_value, u32 *_used,
+			   char *output, u32 *_out_len)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 used = *_used;
+	u32 out_len = *_out_len;
+
+	/* CCK */
+	if (dm_value[0] == 0) {
+		if (dm_value[1] == 1) { /*TX*/
+			if (dm_value[2] == 1)
+				odm_set_bb_reg(dm, 0xa04, 0xf0000000, 0x8);
+			else if (dm_value[2] == 2)
+				odm_set_bb_reg(dm, 0xa04, 0xf0000000, 0x4);
+			else if (dm_value[2] == 3)
+				odm_set_bb_reg(dm, 0xa04, 0xf0000000, 0xc);
+		} else if (dm_value[1] == 2) { /*RX*/
+
+			phydm_config_cck_rx_antenna_init(dm);
+
+			if (dm_value[2] == 1)
+				phydm_config_cck_rx_path(dm, PHYDM_A,
+							 CCA_PATHDIV_DISABLE);
+			else if (dm_value[2] == 2)
+				phydm_config_cck_rx_path(dm, PHYDM_B,
+							 CCA_PATHDIV_DISABLE);
+			else if (dm_value[2] == 3 &&
+				 dm_value[3] == 1) /*enable path diversity*/
+				phydm_config_cck_rx_path(dm, PHYDM_AB,
+							 CCA_PATHDIV_ENABLE);
+			else if (dm_value[2] == 3 && dm_value[3] != 1)
+				phydm_config_cck_rx_path(dm, PHYDM_B,
+							 CCA_PATHDIV_DISABLE);
+		}
+	}
+	/* OFDM */
+	else if (dm_value[0] == 1) {
+		if (dm_value[1] == 1) { /*TX*/
+			phydm_config_ofdm_tx_path(dm, dm_value[2]);
+			/**/
+		} else if (dm_value[1] == 2) { /*RX*/
+			phydm_config_ofdm_rx_path(dm, dm_value[2]);
+			/**/
+		}
+	}
+
+	PHYDM_SNPRINTF(
+		output + used, out_len - used,
+		"PHYDM Set path [%s] [%s] = [%s%s%s%s]\n",
+		(dm_value[0] == 1) ? "OFDM" : "CCK",
+		(dm_value[1] == 1) ? "TX" : "RX",
+		(dm_value[2] & 0x1) ? "A" : "", (dm_value[2] & 0x2) ? "B" : "",
+		(dm_value[2] & 0x4) ? "C" : "", (dm_value[2] & 0x8) ? "D" : "");
+}
+
+static void phydm_init_cck_setting(struct phy_dm_struct *dm)
+{
+	dm->is_cck_high_power = (bool)odm_get_bb_reg(
+		dm, ODM_REG(CCK_RPT_FORMAT, dm), ODM_BIT(CCK_RPT_FORMAT, dm));
+
+	/* JJ ADD 20161014 */
+	/* JJ ADD 20161014 */
+	if (dm->support_ic_type & (ODM_RTL8723D | ODM_RTL8822B | ODM_RTL8197F |
+				   ODM_RTL8821C | ODM_RTL8710B))
+		dm->cck_new_agc = odm_get_bb_reg(dm, 0xa9c, BIT(17)) ?
+					  true :
+					  false; /*1: new agc  0: old agc*/
+	else
+		dm->cck_new_agc = false;
+}
+
+static void phydm_init_soft_ml_setting(struct phy_dm_struct *dm)
+{
+	if (!dm->mp_mode) {
+		if (dm->support_ic_type & ODM_RTL8822B)
+			odm_set_bb_reg(dm, 0x19a8, MASKDWORD, 0xc10a0000);
+	}
+}
+
+static void phydm_init_hw_info_by_rfe(struct phy_dm_struct *dm)
+{
+	if (dm->support_ic_type & ODM_RTL8822B)
+		phydm_init_hw_info_by_rfe_type_8822b(dm);
+}
+
+static void odm_common_info_self_init(struct phy_dm_struct *dm)
+{
+	phydm_init_cck_setting(dm);
+	dm->rf_path_rx_enable = (u8)odm_get_bb_reg(dm, ODM_REG(BB_RX_PATH, dm),
+						   ODM_BIT(BB_RX_PATH, dm));
+	odm_init_mp_driver_status(dm);
+	phydm_init_trx_antenna_setting(dm);
+	phydm_init_soft_ml_setting(dm);
+
+	dm->phydm_period = PHYDM_WATCH_DOG_PERIOD;
+	dm->phydm_sys_up_time = 0;
+
+	if (dm->support_ic_type & ODM_IC_1SS)
+		dm->num_rf_path = 1;
+	else if (dm->support_ic_type & ODM_IC_2SS)
+		dm->num_rf_path = 2;
+	else if (dm->support_ic_type & ODM_IC_3SS)
+		dm->num_rf_path = 3;
+	else if (dm->support_ic_type & ODM_IC_4SS)
+		dm->num_rf_path = 4;
+
+	dm->tx_rate = 0xFF;
+
+	dm->number_linked_client = 0;
+	dm->pre_number_linked_client = 0;
+	dm->number_active_client = 0;
+	dm->pre_number_active_client = 0;
+
+	dm->last_tx_ok_cnt = 0;
+	dm->last_rx_ok_cnt = 0;
+	dm->tx_tp = 0;
+	dm->rx_tp = 0;
+	dm->total_tp = 0;
+	dm->traffic_load = TRAFFIC_LOW;
+
+	dm->nbi_set_result = 0;
+	dm->is_init_hw_info_by_rfe = false;
+	dm->pre_dbg_priority = BB_DBGPORT_RELEASE;
+}
+
+static void odm_common_info_self_update(struct phy_dm_struct *dm)
+{
+	u8 entry_cnt = 0, num_active_client = 0;
+	u32 i, one_entry_macid = 0;
+	struct rtl_sta_info *entry;
+
+	/* THis variable cannot be used because it is wrong*/
+	if (*dm->band_width == ODM_BW40M) {
+		if (*dm->sec_ch_offset == 1)
+			dm->control_channel = *dm->channel - 2;
+		else if (*dm->sec_ch_offset == 2)
+			dm->control_channel = *dm->channel + 2;
+	} else {
+		dm->control_channel = *dm->channel;
+	}
+
+	for (i = 0; i < ODM_ASSOCIATE_ENTRY_NUM; i++) {
+		entry = dm->odm_sta_info[i];
+		if (IS_STA_VALID(entry)) {
+			entry_cnt++;
+			if (entry_cnt == 1)
+				one_entry_macid = i;
+		}
+	}
+
+	if (entry_cnt == 1) {
+		dm->is_one_entry_only = true;
+		dm->one_entry_macid = one_entry_macid;
+	} else {
+		dm->is_one_entry_only = false;
+	}
+
+	dm->pre_number_linked_client = dm->number_linked_client;
+	dm->pre_number_active_client = dm->number_active_client;
+
+	dm->number_linked_client = entry_cnt;
+	dm->number_active_client = num_active_client;
+
+	/* Update MP driver status*/
+	odm_update_mp_driver_status(dm);
+
+	/*Traffic load information update*/
+	phydm_traffic_load_decision(dm);
+
+	dm->phydm_sys_up_time += dm->phydm_period;
+}
+
+static void odm_common_info_self_reset(struct phy_dm_struct *dm)
+{
+	dm->phy_dbg_info.num_qry_beacon_pkt = 0;
+}
+
+void *phydm_get_structure(struct phy_dm_struct *dm, u8 structure_type)
+
+{
+	void *p_struct = NULL;
+
+	switch (structure_type) {
+	case PHYDM_FALSEALMCNT:
+		p_struct = &dm->false_alm_cnt;
+		break;
+
+	case PHYDM_CFOTRACK:
+		p_struct = &dm->dm_cfo_track;
+		break;
+
+	case PHYDM_ADAPTIVITY:
+		p_struct = &dm->adaptivity;
+		break;
+
+	default:
+		break;
+	}
+
+	return p_struct;
+}
+
+static void odm_hw_setting(struct phy_dm_struct *dm)
+{
+	if (dm->support_ic_type & ODM_RTL8822B)
+		phydm_hwsetting_8822b(dm);
+}
+
+static void phydm_supportability_init(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 support_ability = 0;
+
+	if (dm->support_ic_type != ODM_RTL8821C)
+		return;
+
+	switch (dm->support_ic_type) {
+	/*---------------AC Series-------------------*/
+
+	case ODM_RTL8822B:
+		support_ability |= ODM_BB_DIG | ODM_BB_FA_CNT | ODM_BB_CCK_PD |
+				   ODM_BB_CFO_TRACKING | ODM_BB_RATE_ADAPTIVE |
+				   ODM_BB_RSSI_MONITOR | ODM_BB_RA_MASK |
+				   ODM_RF_TX_PWR_TRACK;
+		break;
+
+	default:
+		support_ability |= ODM_BB_DIG | ODM_BB_FA_CNT | ODM_BB_CCK_PD |
+				   ODM_BB_CFO_TRACKING | ODM_BB_RATE_ADAPTIVE |
+				   ODM_BB_RSSI_MONITOR | ODM_BB_RA_MASK |
+				   ODM_RF_TX_PWR_TRACK;
+
+		ODM_RT_TRACE(dm, ODM_COMP_UNCOND,
+			     "[Warning] Supportability Init Warning !!!\n");
+		break;
+	}
+
+	if (*dm->enable_antdiv)
+		support_ability |= ODM_BB_ANT_DIV;
+
+	if (*dm->enable_adaptivity) {
+		ODM_RT_TRACE(dm, ODM_COMP_INIT,
+			     "ODM adaptivity is set to Enabled!!!\n");
+
+		support_ability |= ODM_BB_ADAPTIVITY;
+
+	} else {
+		ODM_RT_TRACE(dm, ODM_COMP_INIT,
+			     "ODM adaptivity is set to disnabled!!!\n");
+		/**/
+	}
+
+	ODM_RT_TRACE(dm, ODM_COMP_INIT, "PHYDM support_ability = ((0x%x))\n",
+		     support_ability);
+	odm_cmn_info_init(dm, ODM_CMNINFO_ABILITY, support_ability);
+}
+
+/*
+ * 2011/09/21 MH Add to describe different team necessary resource allocate??
+ */
+void odm_dm_init(struct phy_dm_struct *dm)
+{
+	phydm_supportability_init(dm);
+	odm_common_info_self_init(dm);
+	odm_dig_init(dm);
+	phydm_nhm_counter_statistics_init(dm);
+	phydm_adaptivity_init(dm);
+	phydm_ra_info_init(dm);
+	odm_rate_adaptive_mask_init(dm);
+	odm_cfo_tracking_init(dm);
+	odm_edca_turbo_init(dm);
+	odm_rssi_monitor_init(dm);
+	phydm_rf_init(dm);
+	odm_txpowertracking_init(dm);
+
+	if (dm->support_ic_type & ODM_RTL8822B)
+		phydm_txcurrentcalibration(dm);
+
+	odm_antenna_diversity_init(dm);
+	odm_auto_channel_select_init(dm);
+	odm_dynamic_tx_power_init(dm);
+	phydm_init_ra_info(dm);
+	adc_smp_init(dm);
+
+	phydm_beamforming_init(dm);
+
+	if (dm->support_ic_type & ODM_IC_11N_SERIES) {
+		/* 11n series */
+		odm_dynamic_bb_power_saving_init(dm);
+	}
+
+	phydm_psd_init(dm);
+}
+
+void odm_dm_reset(struct phy_dm_struct *dm)
+{
+	struct dig_thres *dig_tab = &dm->dm_dig_table;
+
+	odm_ant_div_reset(dm);
+	phydm_set_edcca_threshold_api(dm, dig_tab->cur_ig_value);
+}
+
+void phydm_support_ability_debug(void *dm_void, u32 *const dm_value, u32 *_used,
+				 char *output, u32 *_out_len)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 pre_support_ability;
+	u32 used = *_used;
+	u32 out_len = *_out_len;
+
+	pre_support_ability = dm->support_ability;
+	PHYDM_SNPRINTF(output + used, out_len - used, "\n%s\n",
+		       "================================");
+	if (dm_value[0] == 100) {
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "[Supportability] PhyDM Selection\n");
+		PHYDM_SNPRINTF(output + used, out_len - used, "%s\n",
+			       "================================");
+		PHYDM_SNPRINTF(
+			output + used, out_len - used, "00. (( %s ))DIG\n",
+			((dm->support_ability & ODM_BB_DIG) ? ("V") : (".")));
+		PHYDM_SNPRINTF(
+			output + used, out_len - used, "01. (( %s ))RA_MASK\n",
+			((dm->support_ability & ODM_BB_RA_MASK) ? ("V") :
+								  (".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "02. (( %s ))DYNAMIC_TXPWR\n",
+			       ((dm->support_ability & ODM_BB_DYNAMIC_TXPWR) ?
+					("V") :
+					(".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "03. (( %s ))FA_CNT\n",
+			       ((dm->support_ability & ODM_BB_FA_CNT) ? ("V") :
+									(".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "04. (( %s ))RSSI_MONITOR\n",
+			       ((dm->support_ability & ODM_BB_RSSI_MONITOR) ?
+					("V") :
+					(".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "05. (( %s ))CCK_PD\n",
+			       ((dm->support_ability & ODM_BB_CCK_PD) ? ("V") :
+									(".")));
+		PHYDM_SNPRINTF(
+			output + used, out_len - used, "06. (( %s ))ANT_DIV\n",
+			((dm->support_ability & ODM_BB_ANT_DIV) ? ("V") :
+								  (".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "08. (( %s ))PWR_TRAIN\n",
+			       ((dm->support_ability & ODM_BB_PWR_TRAIN) ?
+					("V") :
+					(".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "09. (( %s ))RATE_ADAPTIVE\n",
+			       ((dm->support_ability & ODM_BB_RATE_ADAPTIVE) ?
+					("V") :
+					(".")));
+		PHYDM_SNPRINTF(
+			output + used, out_len - used, "10. (( %s ))PATH_DIV\n",
+			((dm->support_ability & ODM_BB_PATH_DIV) ? ("V") :
+								   (".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "13. (( %s ))ADAPTIVITY\n",
+			       ((dm->support_ability & ODM_BB_ADAPTIVITY) ?
+					("V") :
+					(".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "14. (( %s ))struct cfo_tracking\n",
+			       ((dm->support_ability & ODM_BB_CFO_TRACKING) ?
+					("V") :
+					(".")));
+		PHYDM_SNPRINTF(
+			output + used, out_len - used, "15. (( %s ))NHM_CNT\n",
+			((dm->support_ability & ODM_BB_NHM_CNT) ? ("V") :
+								  (".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "16. (( %s ))PRIMARY_CCA\n",
+			       ((dm->support_ability & ODM_BB_PRIMARY_CCA) ?
+					("V") :
+					(".")));
+		PHYDM_SNPRINTF(
+			output + used, out_len - used, "17. (( %s ))TXBF\n",
+			((dm->support_ability & ODM_BB_TXBF) ? ("V") : (".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "18. (( %s ))DYNAMIC_ARFR\n",
+			       ((dm->support_ability & ODM_BB_DYNAMIC_ARFR) ?
+					("V") :
+					(".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "20. (( %s ))EDCA_TURBO\n",
+			       ((dm->support_ability & ODM_MAC_EDCA_TURBO) ?
+					("V") :
+					(".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "21. (( %s ))DYNAMIC_RX_PATH\n",
+			       ((dm->support_ability & ODM_BB_DYNAMIC_RX_PATH) ?
+					("V") :
+					(".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "24. (( %s ))TX_PWR_TRACK\n",
+			       ((dm->support_ability & ODM_RF_TX_PWR_TRACK) ?
+					("V") :
+					(".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "25. (( %s ))RX_GAIN_TRACK\n",
+			       ((dm->support_ability & ODM_RF_RX_GAIN_TRACK) ?
+					("V") :
+					(".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "26. (( %s ))RF_CALIBRATION\n",
+			       ((dm->support_ability & ODM_RF_CALIBRATION) ?
+					("V") :
+					(".")));
+		PHYDM_SNPRINTF(output + used, out_len - used, "%s\n",
+			       "================================");
+	} else {
+		if (dm_value[1] == 1) { /* enable */
+			dm->support_ability |= BIT(dm_value[0]);
+		} else if (dm_value[1] == 2) /* disable */
+			dm->support_ability &= ~(BIT(dm_value[0]));
+		else {
+			PHYDM_SNPRINTF(output + used, out_len - used, "%s\n",
+				       "[Warning!!!]  1:enable,  2:disable");
+		}
+	}
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "pre-support_ability  =  0x%x\n", pre_support_ability);
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "Curr-support_ability =  0x%x\n", dm->support_ability);
+	PHYDM_SNPRINTF(output + used, out_len - used, "%s\n",
+		       "================================");
+}
+
+void phydm_watchdog_mp(struct phy_dm_struct *dm) {}
+/*
+ * 2011/09/20 MH This is the entry pointer for all team to execute HW outsrc DM.
+ * You can not add any dummy function here, be care, you can only use DM struct
+ * to perform any new ODM_DM.
+ */
+void odm_dm_watchdog(struct phy_dm_struct *dm)
+{
+	odm_common_info_self_update(dm);
+	phydm_basic_dbg_message(dm);
+	odm_hw_setting(dm);
+
+	odm_false_alarm_counter_statistics(dm);
+	phydm_noisy_detection(dm);
+
+	odm_rssi_monitor_check(dm);
+
+	if (*dm->is_power_saving) {
+		odm_dig_by_rssi_lps(dm);
+		phydm_adaptivity(dm);
+		odm_antenna_diversity(
+			dm); /*enable AntDiv in PS mode, request from SD4 Jeff*/
+		ODM_RT_TRACE(dm, ODM_COMP_COMMON,
+			     "DMWatchdog in power saving mode\n");
+		return;
+	}
+
+	phydm_check_adaptivity(dm);
+	odm_update_power_training_state(dm);
+	odm_DIG(dm);
+	phydm_adaptivity(dm);
+	odm_cck_packet_detection_thresh(dm);
+
+	phydm_ra_info_watchdog(dm);
+	odm_edca_turbo_check(dm);
+	odm_cfo_tracking(dm);
+	odm_dynamic_tx_power(dm);
+	odm_antenna_diversity(dm);
+
+	phydm_beamforming_watchdog(dm);
+
+	phydm_rf_watchdog(dm);
+
+	odm_dtc(dm);
+
+	odm_common_info_self_reset(dm);
+}
+
+/*
+ * Init /.. Fixed HW value. Only init time.
+ */
+void odm_cmn_info_init(struct phy_dm_struct *dm, enum odm_cmninfo cmn_info,
+		       u32 value)
+{
+	/* This section is used for init value */
+	switch (cmn_info) {
+	/* Fixed ODM value. */
+	case ODM_CMNINFO_ABILITY:
+		dm->support_ability = (u32)value;
+		break;
+
+	case ODM_CMNINFO_RF_TYPE:
+		dm->rf_type = (u8)value;
+		break;
+
+	case ODM_CMNINFO_PLATFORM:
+		dm->support_platform = (u8)value;
+		break;
+
+	case ODM_CMNINFO_INTERFACE:
+		dm->support_interface = (u8)value;
+		break;
+
+	case ODM_CMNINFO_MP_TEST_CHIP:
+		dm->is_mp_chip = (u8)value;
+		break;
+
+	case ODM_CMNINFO_IC_TYPE:
+		dm->support_ic_type = value;
+		break;
+
+	case ODM_CMNINFO_CUT_VER:
+		dm->cut_version = (u8)value;
+		break;
+
+	case ODM_CMNINFO_FAB_VER:
+		dm->fab_version = (u8)value;
+		break;
+
+	case ODM_CMNINFO_RFE_TYPE:
+		dm->rfe_type = (u8)value;
+		phydm_init_hw_info_by_rfe(dm);
+		break;
+
+	case ODM_CMNINFO_RF_ANTENNA_TYPE:
+		dm->ant_div_type = (u8)value;
+		break;
+
+	case ODM_CMNINFO_WITH_EXT_ANTENNA_SWITCH:
+		dm->with_extenal_ant_switch = (u8)value;
+		break;
+
+	case ODM_CMNINFO_BE_FIX_TX_ANT:
+		dm->dm_fat_table.b_fix_tx_ant = (u8)value;
+		break;
+
+	case ODM_CMNINFO_BOARD_TYPE:
+		if (!dm->is_init_hw_info_by_rfe)
+			dm->board_type = (u8)value;
+		break;
+
+	case ODM_CMNINFO_PACKAGE_TYPE:
+		if (!dm->is_init_hw_info_by_rfe)
+			dm->package_type = (u8)value;
+		break;
+
+	case ODM_CMNINFO_EXT_LNA:
+		if (!dm->is_init_hw_info_by_rfe)
+			dm->ext_lna = (u8)value;
+		break;
+
+	case ODM_CMNINFO_5G_EXT_LNA:
+		if (!dm->is_init_hw_info_by_rfe)
+			dm->ext_lna_5g = (u8)value;
+		break;
+
+	case ODM_CMNINFO_EXT_PA:
+		if (!dm->is_init_hw_info_by_rfe)
+			dm->ext_pa = (u8)value;
+		break;
+
+	case ODM_CMNINFO_5G_EXT_PA:
+		if (!dm->is_init_hw_info_by_rfe)
+			dm->ext_pa_5g = (u8)value;
+		break;
+
+	case ODM_CMNINFO_GPA:
+		if (!dm->is_init_hw_info_by_rfe)
+			dm->type_gpa = (u16)value;
+		break;
+
+	case ODM_CMNINFO_APA:
+		if (!dm->is_init_hw_info_by_rfe)
+			dm->type_apa = (u16)value;
+		break;
+
+	case ODM_CMNINFO_GLNA:
+		if (!dm->is_init_hw_info_by_rfe)
+			dm->type_glna = (u16)value;
+		break;
+
+	case ODM_CMNINFO_ALNA:
+		if (!dm->is_init_hw_info_by_rfe)
+			dm->type_alna = (u16)value;
+		break;
+
+	case ODM_CMNINFO_EXT_TRSW:
+		if (!dm->is_init_hw_info_by_rfe)
+			dm->ext_trsw = (u8)value;
+		break;
+	case ODM_CMNINFO_EXT_LNA_GAIN:
+		dm->ext_lna_gain = (u8)value;
+		break;
+	case ODM_CMNINFO_PATCH_ID:
+		dm->patch_id = (u8)value;
+		break;
+	case ODM_CMNINFO_BINHCT_TEST:
+		dm->is_in_hct_test = (bool)value;
+		break;
+	case ODM_CMNINFO_BWIFI_TEST:
+		dm->wifi_test = (u8)value;
+		break;
+	case ODM_CMNINFO_SMART_CONCURRENT:
+		dm->is_dual_mac_smart_concurrent = (bool)value;
+		break;
+	case ODM_CMNINFO_DOMAIN_CODE_2G:
+		dm->odm_regulation_2_4g = (u8)value;
+		break;
+	case ODM_CMNINFO_DOMAIN_CODE_5G:
+		dm->odm_regulation_5g = (u8)value;
+		break;
+	case ODM_CMNINFO_CONFIG_BB_RF:
+		dm->config_bbrf = (bool)value;
+		break;
+	case ODM_CMNINFO_IQKFWOFFLOAD:
+		dm->iqk_fw_offload = (u8)value;
+		break;
+	case ODM_CMNINFO_IQKPAOFF:
+		dm->rf_calibrate_info.is_iqk_pa_off = (bool)value;
+		break;
+	case ODM_CMNINFO_REGRFKFREEENABLE:
+		dm->rf_calibrate_info.reg_rf_kfree_enable = (u8)value;
+		break;
+	case ODM_CMNINFO_RFKFREEENABLE:
+		dm->rf_calibrate_info.rf_kfree_enable = (u8)value;
+		break;
+	case ODM_CMNINFO_NORMAL_RX_PATH_CHANGE:
+		dm->normal_rx_path = (u8)value;
+		break;
+	case ODM_CMNINFO_EFUSE0X3D8:
+		dm->efuse0x3d8 = (u8)value;
+		break;
+	case ODM_CMNINFO_EFUSE0X3D7:
+		dm->efuse0x3d7 = (u8)value;
+		break;
+	/* To remove the compiler warning, must add an empty default statement
+	 * to handle the other values.
+	 */
+	default:
+		/* do nothing */
+		break;
+	}
+}
+
+void odm_cmn_info_hook(struct phy_dm_struct *dm, enum odm_cmninfo cmn_info,
+		       void *value)
+{
+	/*  */
+	/* Hook call by reference pointer. */
+	/*  */
+	switch (cmn_info) {
+	/*  */
+	/* Dynamic call by reference pointer. */
+	/*  */
+	case ODM_CMNINFO_MAC_PHY_MODE:
+		dm->mac_phy_mode = (u8 *)value;
+		break;
+
+	case ODM_CMNINFO_TX_UNI:
+		dm->num_tx_bytes_unicast = (u64 *)value;
+		break;
+
+	case ODM_CMNINFO_RX_UNI:
+		dm->num_rx_bytes_unicast = (u64 *)value;
+		break;
+
+	case ODM_CMNINFO_WM_MODE:
+		dm->wireless_mode = (u8 *)value;
+		break;
+
+	case ODM_CMNINFO_BAND:
+		dm->band_type = (u8 *)value;
+		break;
+
+	case ODM_CMNINFO_SEC_CHNL_OFFSET:
+		dm->sec_ch_offset = (u8 *)value;
+		break;
+
+	case ODM_CMNINFO_SEC_MODE:
+		dm->security = (u8 *)value;
+		break;
+
+	case ODM_CMNINFO_BW:
+		dm->band_width = (u8 *)value;
+		break;
+
+	case ODM_CMNINFO_CHNL:
+		dm->channel = (u8 *)value;
+		break;
+
+	case ODM_CMNINFO_DMSP_GET_VALUE:
+		dm->is_get_value_from_other_mac = (bool *)value;
+		break;
+
+	case ODM_CMNINFO_BUDDY_ADAPTOR:
+		dm->buddy_adapter = (void **)value;
+		break;
+
+	case ODM_CMNINFO_DMSP_IS_MASTER:
+		dm->is_master_of_dmsp = (bool *)value;
+		break;
+
+	case ODM_CMNINFO_SCAN:
+		dm->is_scan_in_process = (bool *)value;
+		break;
+
+	case ODM_CMNINFO_POWER_SAVING:
+		dm->is_power_saving = (bool *)value;
+		break;
+
+	case ODM_CMNINFO_ONE_PATH_CCA:
+		dm->one_path_cca = (u8 *)value;
+		break;
+
+	case ODM_CMNINFO_DRV_STOP:
+		dm->is_driver_stopped = (bool *)value;
+		break;
+
+	case ODM_CMNINFO_PNP_IN:
+		dm->is_driver_is_going_to_pnp_set_power_sleep = (bool *)value;
+		break;
+
+	case ODM_CMNINFO_INIT_ON:
+		dm->pinit_adpt_in_progress = (bool *)value;
+		break;
+
+	case ODM_CMNINFO_ANT_TEST:
+		dm->antenna_test = (u8 *)value;
+		break;
+
+	case ODM_CMNINFO_NET_CLOSED:
+		dm->is_net_closed = (bool *)value;
+		break;
+
+	case ODM_CMNINFO_FORCED_RATE:
+		dm->forced_data_rate = (u16 *)value;
+		break;
+	case ODM_CMNINFO_ANT_DIV:
+		dm->enable_antdiv = (u8 *)value;
+		break;
+	case ODM_CMNINFO_ADAPTIVITY:
+		dm->enable_adaptivity = (u8 *)value;
+		break;
+	case ODM_CMNINFO_FORCED_IGI_LB:
+		dm->pu1_forced_igi_lb = (u8 *)value;
+		break;
+
+	case ODM_CMNINFO_P2P_LINK:
+		dm->dm_dig_table.is_p2p_in_process = (u8 *)value;
+		break;
+
+	case ODM_CMNINFO_IS1ANTENNA:
+		dm->is_1_antenna = (bool *)value;
+		break;
+
+	case ODM_CMNINFO_RFDEFAULTPATH:
+		dm->rf_default_path = (u8 *)value;
+		break;
+
+	case ODM_CMNINFO_FCS_MODE:
+		dm->is_fcs_mode_enable = (bool *)value;
+		break;
+	/*add by YuChen for beamforming PhyDM*/
+	case ODM_CMNINFO_HUBUSBMODE:
+		dm->hub_usb_mode = (u8 *)value;
+		break;
+	case ODM_CMNINFO_FWDWRSVDPAGEINPROGRESS:
+		dm->is_fw_dw_rsvd_page_in_progress = (bool *)value;
+		break;
+	case ODM_CMNINFO_TX_TP:
+		dm->current_tx_tp = (u32 *)value;
+		break;
+	case ODM_CMNINFO_RX_TP:
+		dm->current_rx_tp = (u32 *)value;
+		break;
+	case ODM_CMNINFO_SOUNDING_SEQ:
+		dm->sounding_seq = (u8 *)value;
+		break;
+	case ODM_CMNINFO_FORCE_TX_ANT_BY_TXDESC:
+		dm->dm_fat_table.p_force_tx_ant_by_desc = (u8 *)value;
+		break;
+	case ODM_CMNINFO_SET_S0S1_DEFAULT_ANTENNA:
+		dm->dm_fat_table.p_default_s0_s1 = (u8 *)value;
+		break;
+
+	default:
+		/*do nothing*/
+		break;
+	}
+}
+
+void odm_cmn_info_ptr_array_hook(struct phy_dm_struct *dm,
+				 enum odm_cmninfo cmn_info, u16 index,
+				 void *value)
+{
+	/*Hook call by reference pointer.*/
+	switch (cmn_info) {
+	/*Dynamic call by reference pointer.	*/
+	case ODM_CMNINFO_STA_STATUS:
+		dm->odm_sta_info[index] = (struct rtl_sta_info *)value;
+
+		if (IS_STA_VALID(dm->odm_sta_info[index]))
+			dm->platform2phydm_macid_table[index] = index;
+
+		break;
+	/* To remove the compiler warning, must add an empty default statement
+	 * to handle the other values.
+	 */
+	default:
+		/* do nothing */
+		break;
+	}
+}
+
+/*
+ * Update band/CHannel/.. The values are dynamic but non-per-packet.
+ */
+void odm_cmn_info_update(struct phy_dm_struct *dm, u32 cmn_info, u64 value)
+{
+	/* This init variable may be changed in run time. */
+	switch (cmn_info) {
+	case ODM_CMNINFO_LINK_IN_PROGRESS:
+		dm->is_link_in_process = (bool)value;
+		break;
+
+	case ODM_CMNINFO_ABILITY:
+		dm->support_ability = (u32)value;
+		break;
+
+	case ODM_CMNINFO_RF_TYPE:
+		dm->rf_type = (u8)value;
+		break;
+
+	case ODM_CMNINFO_WIFI_DIRECT:
+		dm->is_wifi_direct = (bool)value;
+		break;
+
+	case ODM_CMNINFO_WIFI_DISPLAY:
+		dm->is_wifi_display = (bool)value;
+		break;
+
+	case ODM_CMNINFO_LINK:
+		dm->is_linked = (bool)value;
+		break;
+
+	case ODM_CMNINFO_CMW500LINK:
+		dm->is_linkedcmw500 = (bool)value;
+		break;
+
+	case ODM_CMNINFO_LPSPG:
+		dm->is_in_lps_pg = (bool)value;
+		break;
+
+	case ODM_CMNINFO_STATION_STATE:
+		dm->bsta_state = (bool)value;
+		break;
+
+	case ODM_CMNINFO_RSSI_MIN:
+		dm->rssi_min = (u8)value;
+		break;
+
+	case ODM_CMNINFO_DBG_COMP:
+		dm->debug_components = (u32)value;
+		break;
+
+	case ODM_CMNINFO_DBG_LEVEL:
+		dm->debug_level = (u32)value;
+		break;
+	case ODM_CMNINFO_RA_THRESHOLD_HIGH:
+		dm->rate_adaptive.high_rssi_thresh = (u8)value;
+		break;
+
+	case ODM_CMNINFO_RA_THRESHOLD_LOW:
+		dm->rate_adaptive.low_rssi_thresh = (u8)value;
+		break;
+	/* The following is for BT HS mode and BT coexist mechanism. */
+	case ODM_CMNINFO_BT_ENABLED:
+		dm->is_bt_enabled = (bool)value;
+		break;
+
+	case ODM_CMNINFO_BT_HS_CONNECT_PROCESS:
+		dm->is_bt_connect_process = (bool)value;
+		break;
+
+	case ODM_CMNINFO_BT_HS_RSSI:
+		dm->bt_hs_rssi = (u8)value;
+		break;
+
+	case ODM_CMNINFO_BT_OPERATION:
+		dm->is_bt_hs_operation = (bool)value;
+		break;
+
+	case ODM_CMNINFO_BT_LIMITED_DIG:
+		dm->is_bt_limited_dig = (bool)value;
+		break;
+
+	case ODM_CMNINFO_BT_DIG:
+		dm->bt_hs_dig_val = (u8)value;
+		break;
+
+	case ODM_CMNINFO_BT_BUSY:
+		dm->is_bt_busy = (bool)value;
+		break;
+
+	case ODM_CMNINFO_BT_DISABLE_EDCA:
+		dm->is_bt_disable_edca_turbo = (bool)value;
+		break;
+
+	case ODM_CMNINFO_AP_TOTAL_NUM:
+		dm->ap_total_num = (u8)value;
+		break;
+
+	case ODM_CMNINFO_POWER_TRAINING:
+		dm->is_disable_power_training = (bool)value;
+		break;
+
+	default:
+		/* do nothing */
+		break;
+	}
+}
+
+u32 phydm_cmn_info_query(struct phy_dm_struct *dm,
+			 enum phydm_info_query info_type)
+{
+	struct false_alarm_stat *false_alm_cnt =
+		(struct false_alarm_stat *)phydm_get_structure(
+			dm, PHYDM_FALSEALMCNT);
+
+	switch (info_type) {
+	case PHYDM_INFO_FA_OFDM:
+		return false_alm_cnt->cnt_ofdm_fail;
+
+	case PHYDM_INFO_FA_CCK:
+		return false_alm_cnt->cnt_cck_fail;
+
+	case PHYDM_INFO_FA_TOTAL:
+		return false_alm_cnt->cnt_all;
+
+	case PHYDM_INFO_CCA_OFDM:
+		return false_alm_cnt->cnt_ofdm_cca;
+
+	case PHYDM_INFO_CCA_CCK:
+		return false_alm_cnt->cnt_cck_cca;
+
+	case PHYDM_INFO_CCA_ALL:
+		return false_alm_cnt->cnt_cca_all;
+
+	case PHYDM_INFO_CRC32_OK_VHT:
+		return false_alm_cnt->cnt_vht_crc32_ok;
+
+	case PHYDM_INFO_CRC32_OK_HT:
+		return false_alm_cnt->cnt_ht_crc32_ok;
+
+	case PHYDM_INFO_CRC32_OK_LEGACY:
+		return false_alm_cnt->cnt_ofdm_crc32_ok;
+
+	case PHYDM_INFO_CRC32_OK_CCK:
+		return false_alm_cnt->cnt_cck_crc32_ok;
+
+	case PHYDM_INFO_CRC32_ERROR_VHT:
+		return false_alm_cnt->cnt_vht_crc32_error;
+
+	case PHYDM_INFO_CRC32_ERROR_HT:
+		return false_alm_cnt->cnt_ht_crc32_error;
+
+	case PHYDM_INFO_CRC32_ERROR_LEGACY:
+		return false_alm_cnt->cnt_ofdm_crc32_error;
+
+	case PHYDM_INFO_CRC32_ERROR_CCK:
+		return false_alm_cnt->cnt_cck_crc32_error;
+
+	case PHYDM_INFO_EDCCA_FLAG:
+		return false_alm_cnt->edcca_flag;
+
+	case PHYDM_INFO_OFDM_ENABLE:
+		return false_alm_cnt->ofdm_block_enable;
+
+	case PHYDM_INFO_CCK_ENABLE:
+		return false_alm_cnt->cck_block_enable;
+
+	case PHYDM_INFO_DBG_PORT_0:
+		return false_alm_cnt->dbg_port0;
+
+	default:
+		return 0xffffffff;
+	}
+}
+
+void odm_init_all_timers(struct phy_dm_struct *dm) {}
+
+void odm_cancel_all_timers(struct phy_dm_struct *dm) {}
+
+void odm_release_all_timers(struct phy_dm_struct *dm) {}
+
+/* 3============================================================
+ * 3 Tx Power Tracking
+ * 3============================================================
+ */
+
+/* need to ODM CE Platform
+ * move to here for ANT detection mechanism using
+ */
+
+u32 odm_convert_to_db(u32 value)
+{
+	u8 i;
+	u8 j;
+	u32 dB;
+
+	value = value & 0xFFFF;
+
+	for (i = 0; i < 12; i++) {
+		if (value <= db_invert_table[i][7])
+			break;
+	}
+
+	if (i >= 12)
+		return 96; /* maximum 96 dB */
+
+	for (j = 0; j < 8; j++) {
+		if (value <= db_invert_table[i][j])
+			break;
+	}
+
+	dB = (i << 3) + j + 1;
+
+	return dB;
+}
+
+u32 odm_convert_to_linear(u32 value)
+{
+	u8 i;
+	u8 j;
+	u32 linear;
+
+	/* 1dB~96dB */
+
+	value = value & 0xFF;
+
+	i = (u8)((value - 1) >> 3);
+	j = (u8)(value - 1) - (i << 3);
+
+	linear = db_invert_table[i][j];
+
+	return linear;
+}
+
+/*
+ * ODM multi-port consideration, added by Roger, 2013.10.01.
+ */
+void odm_asoc_entry_init(struct phy_dm_struct *dm) {}
+
+/* Justin: According to the current RRSI to adjust Response Frame TX power */
+void odm_dtc(struct phy_dm_struct *dm) {}
+
+static void odm_update_power_training_state(struct phy_dm_struct *dm)
+{
+	struct false_alarm_stat *false_alm_cnt =
+		(struct false_alarm_stat *)phydm_get_structure(
+			dm, PHYDM_FALSEALMCNT);
+	struct dig_thres *dig_tab = &dm->dm_dig_table;
+	u32 score = 0;
+
+	if (!(dm->support_ability & ODM_BB_PWR_TRAIN))
+		return;
+
+	ODM_RT_TRACE(dm, ODM_COMP_RA_MASK, "%s()============>\n", __func__);
+	dm->is_change_state = false;
+
+	/* Debug command */
+	if (dm->force_power_training_state) {
+		if (dm->force_power_training_state == 1 &&
+		    !dm->is_disable_power_training) {
+			dm->is_change_state = true;
+			dm->is_disable_power_training = true;
+		} else if (dm->force_power_training_state == 2 &&
+			   dm->is_disable_power_training) {
+			dm->is_change_state = true;
+			dm->is_disable_power_training = false;
+		}
+
+		dm->PT_score = 0;
+		dm->phy_dbg_info.num_qry_phy_status_ofdm = 0;
+		dm->phy_dbg_info.num_qry_phy_status_cck = 0;
+		ODM_RT_TRACE(dm, ODM_COMP_RA_MASK,
+			     "%s(): force_power_training_state = %d\n",
+			     __func__, dm->force_power_training_state);
+		return;
+	}
+
+	if (!dm->is_linked)
+		return;
+
+	/* First connect */
+	if ((dm->is_linked) && !dig_tab->is_media_connect_0) {
+		dm->PT_score = 0;
+		dm->is_change_state = true;
+		dm->phy_dbg_info.num_qry_phy_status_ofdm = 0;
+		dm->phy_dbg_info.num_qry_phy_status_cck = 0;
+		ODM_RT_TRACE(dm, ODM_COMP_RA_MASK, "%s(): First Connect\n",
+			     __func__);
+		return;
+	}
+
+	/* Compute score */
+	if (dm->nhm_cnt_0 >= 215) {
+		score = 2;
+	} else if (dm->nhm_cnt_0 >= 190) {
+		score = 1; /* unknown state */
+	} else {
+		u32 rx_pkt_cnt;
+
+		rx_pkt_cnt = (u32)(dm->phy_dbg_info.num_qry_phy_status_ofdm) +
+			     (u32)(dm->phy_dbg_info.num_qry_phy_status_cck);
+
+		if ((false_alm_cnt->cnt_cca_all > 31 && rx_pkt_cnt > 31) &&
+		    (false_alm_cnt->cnt_cca_all >= rx_pkt_cnt)) {
+			if ((rx_pkt_cnt + (rx_pkt_cnt >> 1)) <=
+			    false_alm_cnt->cnt_cca_all)
+				score = 0;
+			else if ((rx_pkt_cnt + (rx_pkt_cnt >> 2)) <=
+				 false_alm_cnt->cnt_cca_all)
+				score = 1;
+			else
+				score = 2;
+		}
+		ODM_RT_TRACE(dm, ODM_COMP_RA_MASK,
+			     "%s(): rx_pkt_cnt = %d, cnt_cca_all = %d\n",
+			     __func__, rx_pkt_cnt, false_alm_cnt->cnt_cca_all);
+	}
+	ODM_RT_TRACE(
+		dm, ODM_COMP_RA_MASK,
+		"%s(): num_qry_phy_status_ofdm = %d, num_qry_phy_status_cck = %d\n",
+		__func__, (u32)(dm->phy_dbg_info.num_qry_phy_status_ofdm),
+		(u32)(dm->phy_dbg_info.num_qry_phy_status_cck));
+	ODM_RT_TRACE(dm, ODM_COMP_RA_MASK, "%s(): nhm_cnt_0 = %d, score = %d\n",
+		     __func__, dm->nhm_cnt_0, score);
+
+	/* smoothing */
+	dm->PT_score = (score << 4) + (dm->PT_score >> 1) + (dm->PT_score >> 2);
+	score = (dm->PT_score + 32) >> 6;
+	ODM_RT_TRACE(dm, ODM_COMP_RA_MASK,
+		     "%s(): PT_score = %d, score after smoothing = %d\n",
+		     __func__, dm->PT_score, score);
+
+	/* mode decision */
+	if (score == 2) {
+		if (dm->is_disable_power_training) {
+			dm->is_change_state = true;
+			dm->is_disable_power_training = false;
+			ODM_RT_TRACE(dm, ODM_COMP_RA_MASK,
+				     "%s(): Change state\n", __func__);
+		}
+		ODM_RT_TRACE(dm, ODM_COMP_RA_MASK,
+			     "%s(): Enable Power Training\n", __func__);
+	} else if (score == 0) {
+		if (!dm->is_disable_power_training) {
+			dm->is_change_state = true;
+			dm->is_disable_power_training = true;
+			ODM_RT_TRACE(dm, ODM_COMP_RA_MASK,
+				     "%s(): Change state\n", __func__);
+		}
+		ODM_RT_TRACE(dm, ODM_COMP_RA_MASK,
+			     "%s(): Disable Power Training\n", __func__);
+	}
+
+	dm->phy_dbg_info.num_qry_phy_status_ofdm = 0;
+	dm->phy_dbg_info.num_qry_phy_status_cck = 0;
+}
+
+/*===========================================================*/
+/* The following is for compile only*/
+/*===========================================================*/
+/*#define TARGET_CHNL_NUM_2G_5G	59*/
+/*===========================================================*/
+
+void phydm_noisy_detection(struct phy_dm_struct *dm)
+{
+	u32 total_fa_cnt, total_cca_cnt;
+	u32 score = 0, i, score_smooth;
+
+	total_cca_cnt = dm->false_alm_cnt.cnt_cca_all;
+	total_fa_cnt = dm->false_alm_cnt.cnt_all;
+
+	for (i = 0; i <= 16; i++) {
+		if (total_fa_cnt * 16 >= total_cca_cnt * (16 - i)) {
+			score = 16 - i;
+			break;
+		}
+	}
+
+	/* noisy_decision_smooth = noisy_decision_smooth>>1 + (score<<3)>>1; */
+	dm->noisy_decision_smooth =
+		(dm->noisy_decision_smooth >> 1) + (score << 2);
+
+	/* Round the noisy_decision_smooth: +"3" comes from (2^3)/2-1 */
+	score_smooth = (total_cca_cnt >= 300) ?
+			       ((dm->noisy_decision_smooth + 3) >> 3) :
+			       0;
+
+	dm->noisy_decision = (score_smooth >= 3) ? 1 : 0;
+	ODM_RT_TRACE(
+		dm, ODM_COMP_NOISY_DETECT,
+		"[NoisyDetection] total_cca_cnt=%d, total_fa_cnt=%d, noisy_decision_smooth=%d, score=%d, score_smooth=%d, dm->noisy_decision=%d\n",
+		total_cca_cnt, total_fa_cnt, dm->noisy_decision_smooth, score,
+		score_smooth, dm->noisy_decision);
+}
+
+void phydm_set_ext_switch(void *dm_void, u32 *const dm_value, u32 *_used,
+			  char *output, u32 *_out_len)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 ext_ant_switch = dm_value[0];
+
+	if (dm->support_ic_type & (ODM_RTL8821 | ODM_RTL8881A)) {
+		/*Output Pin Settings*/
+		odm_set_mac_reg(dm, 0x4C, BIT(23),
+				0); /*select DPDT_P and DPDT_N as output pin*/
+		odm_set_mac_reg(dm, 0x4C, BIT(24), 1); /*by WLAN control*/
+
+		odm_set_bb_reg(dm, 0xCB4, 0xF, 7); /*DPDT_P = 1b'0*/
+		odm_set_bb_reg(dm, 0xCB4, 0xF0, 7); /*DPDT_N = 1b'0*/
+
+		if (ext_ant_switch == MAIN_ANT) {
+			odm_set_bb_reg(dm, 0xCB4, (BIT(29) | BIT(28)), 1);
+			ODM_RT_TRACE(
+				dm, ODM_COMP_API,
+				"***8821A set ant switch = 2b'01 (Main)\n");
+		} else if (ext_ant_switch == AUX_ANT) {
+			odm_set_bb_reg(dm, 0xCB4, BIT(29) | BIT(28), 2);
+			ODM_RT_TRACE(dm, ODM_COMP_API,
+				     "***8821A set ant switch = 2b'10 (Aux)\n");
+		}
+	}
+}
+
+static void phydm_csi_mask_enable(void *dm_void, u32 enable)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 reg_value = 0;
+
+	reg_value = (enable == CSI_MASK_ENABLE) ? 1 : 0;
+
+	if (dm->support_ic_type & ODM_IC_11N_SERIES) {
+		odm_set_bb_reg(dm, 0xD2C, BIT(28), reg_value);
+		ODM_RT_TRACE(dm, ODM_COMP_API,
+			     "Enable CSI Mask:  Reg 0xD2C[28] = ((0x%x))\n",
+			     reg_value);
+
+	} else if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+		odm_set_bb_reg(dm, 0x874, BIT(0), reg_value);
+		ODM_RT_TRACE(dm, ODM_COMP_API,
+			     "Enable CSI Mask:  Reg 0x874[0] = ((0x%x))\n",
+			     reg_value);
+	}
+}
+
+static void phydm_clean_all_csi_mask(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	if (dm->support_ic_type & ODM_IC_11N_SERIES) {
+		odm_set_bb_reg(dm, 0xD40, MASKDWORD, 0);
+		odm_set_bb_reg(dm, 0xD44, MASKDWORD, 0);
+		odm_set_bb_reg(dm, 0xD48, MASKDWORD, 0);
+		odm_set_bb_reg(dm, 0xD4c, MASKDWORD, 0);
+
+	} else if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+		odm_set_bb_reg(dm, 0x880, MASKDWORD, 0);
+		odm_set_bb_reg(dm, 0x884, MASKDWORD, 0);
+		odm_set_bb_reg(dm, 0x888, MASKDWORD, 0);
+		odm_set_bb_reg(dm, 0x88c, MASKDWORD, 0);
+		odm_set_bb_reg(dm, 0x890, MASKDWORD, 0);
+		odm_set_bb_reg(dm, 0x894, MASKDWORD, 0);
+		odm_set_bb_reg(dm, 0x898, MASKDWORD, 0);
+		odm_set_bb_reg(dm, 0x89c, MASKDWORD, 0);
+	}
+}
+
+static void phydm_set_csi_mask_reg(void *dm_void, u32 tone_idx_tmp,
+				   u8 tone_direction)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u8 byte_offset, bit_offset;
+	u32 target_reg;
+	u8 reg_tmp_value;
+	u32 tone_num = 64;
+	u32 tone_num_shift = 0;
+	u32 csi_mask_reg_p = 0, csi_mask_reg_n = 0;
+
+	/* calculate real tone idx*/
+	if ((tone_idx_tmp % 10) >= 5)
+		tone_idx_tmp += 10;
+
+	tone_idx_tmp = (tone_idx_tmp / 10);
+
+	if (dm->support_ic_type & ODM_IC_11N_SERIES) {
+		tone_num = 64;
+		csi_mask_reg_p = 0xD40;
+		csi_mask_reg_n = 0xD48;
+
+	} else if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+		tone_num = 128;
+		csi_mask_reg_p = 0x880;
+		csi_mask_reg_n = 0x890;
+	}
+
+	if (tone_direction == FREQ_POSITIVE) {
+		if (tone_idx_tmp >= (tone_num - 1))
+			tone_idx_tmp = (tone_num - 1);
+
+		byte_offset = (u8)(tone_idx_tmp >> 3);
+		bit_offset = (u8)(tone_idx_tmp & 0x7);
+		target_reg = csi_mask_reg_p + byte_offset;
+
+	} else {
+		tone_num_shift = tone_num;
+
+		if (tone_idx_tmp >= tone_num)
+			tone_idx_tmp = tone_num;
+
+		tone_idx_tmp = tone_num - tone_idx_tmp;
+
+		byte_offset = (u8)(tone_idx_tmp >> 3);
+		bit_offset = (u8)(tone_idx_tmp & 0x7);
+		target_reg = csi_mask_reg_n + byte_offset;
+	}
+
+	reg_tmp_value = odm_read_1byte(dm, target_reg);
+	ODM_RT_TRACE(dm, ODM_COMP_API,
+		     "Pre Mask tone idx[%d]:  Reg0x%x = ((0x%x))\n",
+		     (tone_idx_tmp + tone_num_shift), target_reg,
+		     reg_tmp_value);
+	reg_tmp_value |= BIT(bit_offset);
+	odm_write_1byte(dm, target_reg, reg_tmp_value);
+	ODM_RT_TRACE(dm, ODM_COMP_API,
+		     "New Mask tone idx[%d]:  Reg0x%x = ((0x%x))\n",
+		     (tone_idx_tmp + tone_num_shift), target_reg,
+		     reg_tmp_value);
+}
+
+static void phydm_set_nbi_reg(void *dm_void, u32 tone_idx_tmp, u32 bw)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 nbi_table_128[NBI_TABLE_SIZE_128] = {
+		25, 55, 85, 115, 135, 155, 185, 205, 225, 245,
+		/*1~10*/ /*tone_idx X 10*/
+		265, 285, 305, 335, 355, 375, 395, 415, 435, 455, /*11~20*/
+		485, 505, 525, 555, 585, 615, 635}; /*21~27*/
+
+	u32 nbi_table_256[NBI_TABLE_SIZE_256] = {
+		25,   55,   85,   115,  135,  155,  175,  195,  225,
+		245, /*1~10*/
+		265,  285,  305,  325,  345,  365,  385,  405,  425,
+		445, /*11~20*/
+		465,  485,  505,  525,  545,  565,  585,  605,  625,
+		645, /*21~30*/
+		665,  695,  715,  735,  755,  775,  795,  815,  835,
+		855, /*31~40*/
+		875,  895,  915,  935,  955,  975,  995,  1015, 1035,
+		1055, /*41~50*/
+		1085, 1105, 1125, 1145, 1175, 1195, 1225, 1255, 1275}; /*51~59*/
+
+	u32 reg_idx = 0;
+	u32 i;
+	u8 nbi_table_idx = FFT_128_TYPE;
+
+	if (dm->support_ic_type & ODM_IC_11N_SERIES) {
+		nbi_table_idx = FFT_128_TYPE;
+	} else if (dm->support_ic_type & ODM_IC_11AC_1_SERIES) {
+		nbi_table_idx = FFT_256_TYPE;
+	} else if (dm->support_ic_type & ODM_IC_11AC_2_SERIES) {
+		if (bw == 80)
+			nbi_table_idx = FFT_256_TYPE;
+		else /*20M, 40M*/
+			nbi_table_idx = FFT_128_TYPE;
+	}
+
+	if (nbi_table_idx == FFT_128_TYPE) {
+		for (i = 0; i < NBI_TABLE_SIZE_128; i++) {
+			if (tone_idx_tmp < nbi_table_128[i]) {
+				reg_idx = i + 1;
+				break;
+			}
+		}
+
+	} else if (nbi_table_idx == FFT_256_TYPE) {
+		for (i = 0; i < NBI_TABLE_SIZE_256; i++) {
+			if (tone_idx_tmp < nbi_table_256[i]) {
+				reg_idx = i + 1;
+				break;
+			}
+		}
+	}
+
+	if (dm->support_ic_type & ODM_IC_11N_SERIES) {
+		odm_set_bb_reg(dm, 0xc40, 0x1f000000, reg_idx);
+		ODM_RT_TRACE(dm, ODM_COMP_API,
+			     "Set tone idx:  Reg0xC40[28:24] = ((0x%x))\n",
+			     reg_idx);
+		/**/
+	} else {
+		odm_set_bb_reg(dm, 0x87c, 0xfc000, reg_idx);
+		ODM_RT_TRACE(dm, ODM_COMP_API,
+			     "Set tone idx: Reg0x87C[19:14] = ((0x%x))\n",
+			     reg_idx);
+		/**/
+	}
+}
+
+static void phydm_nbi_enable(void *dm_void, u32 enable)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 reg_value = 0;
+
+	reg_value = (enable == NBI_ENABLE) ? 1 : 0;
+
+	if (dm->support_ic_type & ODM_IC_11N_SERIES) {
+		odm_set_bb_reg(dm, 0xc40, BIT(9), reg_value);
+		ODM_RT_TRACE(dm, ODM_COMP_API,
+			     "Enable NBI Reg0xC40[9] = ((0x%x))\n", reg_value);
+
+	} else if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+		odm_set_bb_reg(dm, 0x87c, BIT(13), reg_value);
+		ODM_RT_TRACE(dm, ODM_COMP_API,
+			     "Enable NBI Reg0x87C[13] = ((0x%x))\n", reg_value);
+	}
+}
+
+static u8 phydm_calculate_fc(void *dm_void, u32 channel, u32 bw, u32 second_ch,
+			     u32 *fc_in)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 fc = *fc_in;
+	u32 start_ch_per_40m[NUM_START_CH_40M + 1] = {
+		36,  44,  52,  60,  100, 108, 116,     124,
+		132, 140, 149, 157, 165, 173, 173 + 8,
+	};
+	u32 start_ch_per_80m[NUM_START_CH_80M + 1] = {
+		36, 52, 100, 116, 132, 149, 165, 165 + 16,
+	};
+	u32 *start_ch = &start_ch_per_40m[0];
+	u32 num_start_channel = NUM_START_CH_40M;
+	u32 channel_offset = 0;
+	u32 i;
+
+	/*2.4G*/
+	if (channel <= 14 && channel > 0) {
+		if (bw == 80)
+			return SET_ERROR;
+
+		fc = 2412 + (channel - 1) * 5;
+
+		if (bw == 40 && (second_ch == PHYDM_ABOVE)) {
+			if (channel >= 10) {
+				ODM_RT_TRACE(
+					dm, ODM_COMP_API,
+					"CH = ((%d)), Scnd_CH = ((%d)) Error setting\n",
+					channel, second_ch);
+				return SET_ERROR;
+			}
+			fc += 10;
+		} else if (bw == 40 && (second_ch == PHYDM_BELOW)) {
+			if (channel <= 2) {
+				ODM_RT_TRACE(
+					dm, ODM_COMP_API,
+					"CH = ((%d)), Scnd_CH = ((%d)) Error setting\n",
+					channel, second_ch);
+				return SET_ERROR;
+			}
+			fc -= 10;
+		}
+	}
+	/*5G*/
+	else if (channel >= 36 && channel <= 177) {
+		if (bw == 20) {
+			fc = 5180 + (channel - 36) * 5;
+			*fc_in = fc;
+			return SET_SUCCESS;
+		}
+
+		if (bw == 40) {
+			num_start_channel = NUM_START_CH_40M;
+			start_ch = &start_ch_per_40m[0];
+			channel_offset = CH_OFFSET_40M;
+		} else if (bw == 80) {
+			num_start_channel = NUM_START_CH_80M;
+			start_ch = &start_ch_per_80m[0];
+			channel_offset = CH_OFFSET_80M;
+		}
+
+		for (i = 0; i < num_start_channel; i++) {
+			if (channel < start_ch[i + 1]) {
+				channel = start_ch[i] + channel_offset;
+				break;
+			}
+		}
+
+		ODM_RT_TRACE(dm, ODM_COMP_API, "Mod_CH = ((%d))\n", channel);
+
+		fc = 5180 + (channel - 36) * 5;
+
+	} else {
+		ODM_RT_TRACE(dm, ODM_COMP_API, "CH = ((%d)) Error setting\n",
+			     channel);
+		return SET_ERROR;
+	}
+
+	*fc_in = fc;
+
+	return SET_SUCCESS;
+}
+
+static u8 phydm_calculate_intf_distance(void *dm_void, u32 bw, u32 fc,
+					u32 f_interference,
+					u32 *tone_idx_tmp_in)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 bw_up, bw_low;
+	u32 int_distance;
+	u32 tone_idx_tmp;
+	u8 set_result = SET_NO_NEED;
+
+	bw_up = fc + bw / 2;
+	bw_low = fc - bw / 2;
+
+	ODM_RT_TRACE(dm, ODM_COMP_API,
+		     "[f_l, fc, fh] = [ %d, %d, %d ], f_int = ((%d))\n", bw_low,
+		     fc, bw_up, f_interference);
+
+	if ((f_interference >= bw_low) && (f_interference <= bw_up)) {
+		int_distance = (fc >= f_interference) ? (fc - f_interference) :
+							(f_interference - fc);
+		tone_idx_tmp =
+			(int_distance << 5); /* =10*(int_distance /0.3125) */
+		ODM_RT_TRACE(
+			dm, ODM_COMP_API,
+			"int_distance = ((%d MHz)) Mhz, tone_idx_tmp = ((%d.%d))\n",
+			int_distance, (tone_idx_tmp / 10), (tone_idx_tmp % 10));
+		*tone_idx_tmp_in = tone_idx_tmp;
+		set_result = SET_SUCCESS;
+	}
+
+	return set_result;
+}
+
+static u8 phydm_csi_mask_setting(void *dm_void, u32 enable, u32 channel, u32 bw,
+				 u32 f_interference, u32 second_ch)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 fc;
+	u8 tone_direction;
+	u32 tone_idx_tmp;
+	u8 set_result = SET_SUCCESS;
+
+	if (enable == CSI_MASK_DISABLE) {
+		set_result = SET_SUCCESS;
+		phydm_clean_all_csi_mask(dm);
+
+	} else {
+		ODM_RT_TRACE(
+			dm, ODM_COMP_API,
+			"[Set CSI MASK_] CH = ((%d)), BW = ((%d)), f_intf = ((%d)), Scnd_CH = ((%s))\n",
+			channel, bw, f_interference,
+			(((bw == 20) || (channel > 14)) ?
+				 "Don't care" :
+				 (second_ch == PHYDM_ABOVE) ? "H" : "L"));
+
+		/*calculate fc*/
+		if (phydm_calculate_fc(dm, channel, bw, second_ch, &fc) ==
+		    SET_ERROR) {
+			set_result = SET_ERROR;
+		} else {
+			/*calculate interference distance*/
+			if (phydm_calculate_intf_distance(
+				    dm, bw, fc, f_interference,
+				    &tone_idx_tmp) == SET_SUCCESS) {
+				tone_direction = (f_interference >= fc) ?
+							 FREQ_POSITIVE :
+							 FREQ_NEGATIVE;
+				phydm_set_csi_mask_reg(dm, tone_idx_tmp,
+						       tone_direction);
+				set_result = SET_SUCCESS;
+			} else {
+				set_result = SET_NO_NEED;
+			}
+		}
+	}
+
+	if (set_result == SET_SUCCESS)
+		phydm_csi_mask_enable(dm, enable);
+	else
+		phydm_csi_mask_enable(dm, CSI_MASK_DISABLE);
+
+	return set_result;
+}
+
+u8 phydm_nbi_setting(void *dm_void, u32 enable, u32 channel, u32 bw,
+		     u32 f_interference, u32 second_ch)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 fc;
+	u32 tone_idx_tmp;
+	u8 set_result = SET_SUCCESS;
+
+	if (enable == NBI_DISABLE) {
+		set_result = SET_SUCCESS;
+	} else {
+		ODM_RT_TRACE(
+			dm, ODM_COMP_API,
+			"[Set NBI] CH = ((%d)), BW = ((%d)), f_intf = ((%d)), Scnd_CH = ((%s))\n",
+			channel, bw, f_interference,
+			(((second_ch == PHYDM_DONT_CARE) || (bw == 20) ||
+			  (channel > 14)) ?
+				 "Don't care" :
+				 (second_ch == PHYDM_ABOVE) ? "H" : "L"));
+
+		/*calculate fc*/
+		if (phydm_calculate_fc(dm, channel, bw, second_ch, &fc) ==
+		    SET_ERROR) {
+			set_result = SET_ERROR;
+		} else {
+			/*calculate interference distance*/
+			if (phydm_calculate_intf_distance(
+				    dm, bw, fc, f_interference,
+				    &tone_idx_tmp) == SET_SUCCESS) {
+				phydm_set_nbi_reg(dm, tone_idx_tmp, bw);
+				set_result = SET_SUCCESS;
+			} else {
+				set_result = SET_NO_NEED;
+			}
+		}
+	}
+
+	if (set_result == SET_SUCCESS)
+		phydm_nbi_enable(dm, enable);
+	else
+		phydm_nbi_enable(dm, NBI_DISABLE);
+
+	return set_result;
+}
+
+void phydm_api_debug(void *dm_void, u32 function_map, u32 *const dm_value,
+		     u32 *_used, char *output, u32 *_out_len)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 used = *_used;
+	u32 out_len = *_out_len;
+	u32 channel = dm_value[1];
+	u32 bw = dm_value[2];
+	u32 f_interference = dm_value[3];
+	u32 second_ch = dm_value[4];
+	u8 set_result = 0;
+
+	/*PHYDM_API_NBI*/
+	/*--------------------------------------------------------------------*/
+	if (function_map == PHYDM_API_NBI) {
+		if (dm_value[0] == 100) {
+			PHYDM_SNPRINTF(
+				output + used, out_len - used,
+				"[HELP-NBI]  EN(on=1, off=2)   CH   BW(20/40/80)  f_intf(Mhz)    Scnd_CH(L=1, H=2)\n");
+			return;
+
+		} else if (dm_value[0] == NBI_ENABLE) {
+			PHYDM_SNPRINTF(
+				output + used, out_len - used,
+				"[Enable NBI] CH = ((%d)), BW = ((%d)), f_intf = ((%d)), Scnd_CH = ((%s))\n",
+				channel, bw, f_interference,
+				((second_ch == PHYDM_DONT_CARE) || (bw == 20) ||
+				 (channel > 14)) ?
+					"Don't care" :
+					((second_ch == PHYDM_ABOVE) ? "H" :
+								      "L"));
+			set_result =
+				phydm_nbi_setting(dm, NBI_ENABLE, channel, bw,
+						  f_interference, second_ch);
+
+		} else if (dm_value[0] == NBI_DISABLE) {
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "[Disable NBI]\n");
+			set_result =
+				phydm_nbi_setting(dm, NBI_DISABLE, channel, bw,
+						  f_interference, second_ch);
+
+		} else {
+			set_result = SET_ERROR;
+		}
+
+		PHYDM_SNPRINTF(
+			output + used, out_len - used, "[NBI set result: %s]\n",
+			(set_result == SET_SUCCESS) ?
+				"Success" :
+				((set_result == SET_NO_NEED) ? "No need" :
+							       "Error"));
+	}
+
+	/*PHYDM_CSI_MASK*/
+	/*--------------------------------------------------------------------*/
+	else if (function_map == PHYDM_API_CSI_MASK) {
+		if (dm_value[0] == 100) {
+			PHYDM_SNPRINTF(
+				output + used, out_len - used,
+				"[HELP-CSI MASK]  EN(on=1, off=2)   CH   BW(20/40/80)  f_intf(Mhz)    Scnd_CH(L=1, H=2)\n");
+			return;
+
+		} else if (dm_value[0] == CSI_MASK_ENABLE) {
+			PHYDM_SNPRINTF(
+				output + used, out_len - used,
+				"[Enable CSI MASK] CH = ((%d)), BW = ((%d)), f_intf = ((%d)), Scnd_CH = ((%s))\n",
+				channel, bw, f_interference,
+				(channel > 14) ?
+					"Don't care" :
+					(((second_ch == PHYDM_DONT_CARE) ||
+					  (bw == 20) || (channel > 14)) ?
+						 "H" :
+						 "L"));
+			set_result = phydm_csi_mask_setting(
+				dm, CSI_MASK_ENABLE, channel, bw,
+				f_interference, second_ch);
+
+		} else if (dm_value[0] == CSI_MASK_DISABLE) {
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "[Disable CSI MASK]\n");
+			set_result = phydm_csi_mask_setting(
+				dm, CSI_MASK_DISABLE, channel, bw,
+				f_interference, second_ch);
+
+		} else {
+			set_result = SET_ERROR;
+		}
+
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "[CSI MASK set result: %s]\n",
+			       (set_result == SET_SUCCESS) ?
+				       "Success" :
+				       ((set_result == SET_NO_NEED) ?
+						"No need" :
+						"Error"));
+	}
+}
diff --git a/drivers/staging/rtlwifi/phydm/phydm.h b/drivers/staging/rtlwifi/phydm/phydm.h
new file mode 100644
index 000000000000..5812ff427ead
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm.h
@@ -0,0 +1,946 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __HALDMOUTSRC_H__
+#define __HALDMOUTSRC_H__
+
+/*============================================================*/
+/*include files*/
+/*============================================================*/
+#include "phydm_pre_define.h"
+#include "phydm_dig.h"
+#include "phydm_edcaturbocheck.h"
+#include "phydm_antdiv.h"
+#include "phydm_dynamicbbpowersaving.h"
+#include "phydm_rainfo.h"
+#include "phydm_dynamictxpower.h"
+#include "phydm_cfotracking.h"
+#include "phydm_acs.h"
+#include "phydm_adaptivity.h"
+#include "phydm_iqk.h"
+#include "phydm_dfs.h"
+#include "phydm_ccx.h"
+#include "txbf/phydm_hal_txbf_api.h"
+
+#include "phydm_adc_sampling.h"
+#include "phydm_dynamic_rx_path.h"
+#include "phydm_psd.h"
+
+#include "phydm_beamforming.h"
+
+#include "phydm_noisemonitor.h"
+#include "halphyrf_ce.h"
+
+/*============================================================*/
+/*Definition */
+/*============================================================*/
+
+/* Traffic load decision */
+#define TRAFFIC_ULTRA_LOW 1
+#define TRAFFIC_LOW 2
+#define TRAFFIC_MID 3
+#define TRAFFIC_HIGH 4
+
+#define NONE 0
+
+/*NBI API------------------------------------*/
+#define NBI_ENABLE 1
+#define NBI_DISABLE 2
+
+#define NBI_TABLE_SIZE_128 27
+#define NBI_TABLE_SIZE_256 59
+
+#define NUM_START_CH_80M 7
+#define NUM_START_CH_40M 14
+
+#define CH_OFFSET_40M 2
+#define CH_OFFSET_80M 6
+
+/*CSI MASK API------------------------------------*/
+#define CSI_MASK_ENABLE 1
+#define CSI_MASK_DISABLE 2
+
+/*------------------------------------------------*/
+
+#define FFT_128_TYPE 1
+#define FFT_256_TYPE 2
+
+#define SET_SUCCESS 1
+#define SET_ERROR 2
+#define SET_NO_NEED 3
+
+#define FREQ_POSITIVE 1
+#define FREQ_NEGATIVE 2
+
+#define PHYDM_WATCH_DOG_PERIOD 2
+
+/*============================================================*/
+/*structure and define*/
+/*============================================================*/
+
+/*2011/09/20 MH Add for AP/ADSLpseudo DM structuer requirement.*/
+/*We need to remove to other position???*/
+
+struct rtl8192cd_priv {
+	u8 temp;
+};
+
+struct dyn_primary_cca {
+	u8 pri_cca_flag;
+	u8 intf_flag;
+	u8 intf_type;
+	u8 dup_rts_flag;
+	u8 monitor_flag;
+	u8 ch_offset;
+	u8 mf_state;
+};
+
+#define dm_type_by_fw 0
+#define dm_type_by_driver 1
+
+/*Declare for common info*/
+
+#define IQK_THRESHOLD 8
+#define DPK_THRESHOLD 4
+
+struct dm_phy_status_info {
+	/*  */
+	/* Be care, if you want to add any element please insert between */
+	/* rx_pwdb_all & signal_strength. */
+	/*  */
+	u8 rx_pwdb_all;
+	u8 signal_quality; /* in 0-100 index. */
+	s8 rx_mimo_signal_quality[4]; /* per-path's EVM translate to 0~100% */
+	u8 rx_mimo_evm_dbm[4]; /* per-path's original EVM (dbm) */
+	u8 rx_mimo_signal_strength[4]; /* in 0~100 index */
+	s16 cfo_short[4]; /* per-path's cfo_short */
+	s16 cfo_tail[4]; /* per-path's cfo_tail */
+	s8 rx_power; /* in dBm Translate from PWdB */
+	s8 recv_signal_power; /* Real power in dBm for this packet,
+			       * no beautification and aggregation.
+			       * Keep this raw info to be used for the other
+			       * procedures.
+			       */
+	u8 bt_rx_rssi_percentage;
+	u8 signal_strength; /* in 0-100 index. */
+	s8 rx_pwr[4]; /* per-path's pwdb */
+	s8 rx_snr[4]; /* per-path's SNR	*/
+	/* s8      BB_Backup[13];                   backup reg. */
+	u8 rx_count : 2; /* RX path counter---*/
+	u8 band_width : 2;
+	u8 rxsc : 4; /* sub-channel---*/
+	u8 bt_coex_pwr_adjust;
+	u8 channel; /* channel number---*/
+	bool is_mu_packet; /* is MU packet or not---*/
+	bool is_beamformed; /* BF packet---*/
+};
+
+struct dm_per_pkt_info {
+	u8 data_rate;
+	u8 station_id;
+	bool is_packet_match_bssid;
+	bool is_packet_to_self;
+	bool is_packet_beacon;
+	bool is_to_self;
+	u8 ppdu_cnt;
+};
+
+struct odm_phy_dbg_info {
+	/*ODM Write,debug info*/
+	s8 rx_snr_db[4];
+	u32 num_qry_phy_status;
+	u32 num_qry_phy_status_cck;
+	u32 num_qry_phy_status_ofdm;
+	u32 num_qry_mu_pkt;
+	u32 num_qry_bf_pkt;
+	u32 num_qry_mu_vht_pkt[40];
+	u32 num_qry_vht_pkt[40];
+	bool is_ldpc_pkt;
+	bool is_stbc_pkt;
+	u8 num_of_ppdu[4];
+	u8 gid_num[4];
+	u8 num_qry_beacon_pkt;
+	/* Others */
+	s32 rx_evm[4];
+};
+
+/*2011/20/20 MH For MP driver RT_WLAN_STA =  struct rtl_sta_info*/
+/*Please declare below ODM relative info in your STA info structure.*/
+
+struct odm_sta_info {
+	/*Driver Write*/
+	bool is_used; /*record the sta status link or not?*/
+	u8 iot_peer; /*Enum value.	HT_IOT_PEER_E*/
+
+	/*ODM Write*/
+	/*PHY_STATUS_INFO*/
+	u8 rssi_path[4];
+	u8 rssi_ave;
+	u8 RXEVM[4];
+	u8 RXSNR[4];
+};
+
+enum odm_cmninfo {
+	/*Fixed value*/
+	/*-----------HOOK BEFORE REG INIT-----------*/
+	ODM_CMNINFO_PLATFORM = 0,
+	ODM_CMNINFO_ABILITY,
+	ODM_CMNINFO_INTERFACE,
+	ODM_CMNINFO_MP_TEST_CHIP,
+	ODM_CMNINFO_IC_TYPE,
+	ODM_CMNINFO_CUT_VER,
+	ODM_CMNINFO_FAB_VER,
+	ODM_CMNINFO_RF_TYPE,
+	ODM_CMNINFO_RFE_TYPE,
+	ODM_CMNINFO_BOARD_TYPE,
+	ODM_CMNINFO_PACKAGE_TYPE,
+	ODM_CMNINFO_EXT_LNA,
+	ODM_CMNINFO_5G_EXT_LNA,
+	ODM_CMNINFO_EXT_PA,
+	ODM_CMNINFO_5G_EXT_PA,
+	ODM_CMNINFO_GPA,
+	ODM_CMNINFO_APA,
+	ODM_CMNINFO_GLNA,
+	ODM_CMNINFO_ALNA,
+	ODM_CMNINFO_EXT_TRSW,
+	ODM_CMNINFO_DPK_EN,
+	ODM_CMNINFO_EXT_LNA_GAIN,
+	ODM_CMNINFO_PATCH_ID,
+	ODM_CMNINFO_BINHCT_TEST,
+	ODM_CMNINFO_BWIFI_TEST,
+	ODM_CMNINFO_SMART_CONCURRENT,
+	ODM_CMNINFO_CONFIG_BB_RF,
+	ODM_CMNINFO_DOMAIN_CODE_2G,
+	ODM_CMNINFO_DOMAIN_CODE_5G,
+	ODM_CMNINFO_IQKFWOFFLOAD,
+	ODM_CMNINFO_IQKPAOFF,
+	ODM_CMNINFO_HUBUSBMODE,
+	ODM_CMNINFO_FWDWRSVDPAGEINPROGRESS,
+	ODM_CMNINFO_TX_TP,
+	ODM_CMNINFO_RX_TP,
+	ODM_CMNINFO_SOUNDING_SEQ,
+	ODM_CMNINFO_REGRFKFREEENABLE,
+	ODM_CMNINFO_RFKFREEENABLE,
+	ODM_CMNINFO_NORMAL_RX_PATH_CHANGE,
+	ODM_CMNINFO_EFUSE0X3D8,
+	ODM_CMNINFO_EFUSE0X3D7,
+	/*-----------HOOK BEFORE REG INIT-----------*/
+
+	/*Dynamic value:*/
+
+	/*--------- POINTER REFERENCE-----------*/
+	ODM_CMNINFO_MAC_PHY_MODE,
+	ODM_CMNINFO_TX_UNI,
+	ODM_CMNINFO_RX_UNI,
+	ODM_CMNINFO_WM_MODE,
+	ODM_CMNINFO_BAND,
+	ODM_CMNINFO_SEC_CHNL_OFFSET,
+	ODM_CMNINFO_SEC_MODE,
+	ODM_CMNINFO_BW,
+	ODM_CMNINFO_CHNL,
+	ODM_CMNINFO_FORCED_RATE,
+	ODM_CMNINFO_ANT_DIV,
+	ODM_CMNINFO_ADAPTIVITY,
+	ODM_CMNINFO_DMSP_GET_VALUE,
+	ODM_CMNINFO_BUDDY_ADAPTOR,
+	ODM_CMNINFO_DMSP_IS_MASTER,
+	ODM_CMNINFO_SCAN,
+	ODM_CMNINFO_POWER_SAVING,
+	ODM_CMNINFO_ONE_PATH_CCA,
+	ODM_CMNINFO_DRV_STOP,
+	ODM_CMNINFO_PNP_IN,
+	ODM_CMNINFO_INIT_ON,
+	ODM_CMNINFO_ANT_TEST,
+	ODM_CMNINFO_NET_CLOSED,
+	ODM_CMNINFO_FORCED_IGI_LB,
+	ODM_CMNINFO_P2P_LINK,
+	ODM_CMNINFO_FCS_MODE,
+	ODM_CMNINFO_IS1ANTENNA,
+	ODM_CMNINFO_RFDEFAULTPATH,
+	ODM_CMNINFO_DFS_MASTER_ENABLE,
+	ODM_CMNINFO_FORCE_TX_ANT_BY_TXDESC,
+	ODM_CMNINFO_SET_S0S1_DEFAULT_ANTENNA,
+	/*--------- POINTER REFERENCE-----------*/
+
+	/*------------CALL BY VALUE-------------*/
+	ODM_CMNINFO_WIFI_DIRECT,
+	ODM_CMNINFO_WIFI_DISPLAY,
+	ODM_CMNINFO_LINK_IN_PROGRESS,
+	ODM_CMNINFO_LINK,
+	ODM_CMNINFO_CMW500LINK,
+	ODM_CMNINFO_LPSPG,
+	ODM_CMNINFO_STATION_STATE,
+	ODM_CMNINFO_RSSI_MIN,
+	ODM_CMNINFO_DBG_COMP,
+	ODM_CMNINFO_DBG_LEVEL,
+	ODM_CMNINFO_RA_THRESHOLD_HIGH,
+	ODM_CMNINFO_RA_THRESHOLD_LOW,
+	ODM_CMNINFO_RF_ANTENNA_TYPE,
+	ODM_CMNINFO_WITH_EXT_ANTENNA_SWITCH,
+	ODM_CMNINFO_BE_FIX_TX_ANT,
+	ODM_CMNINFO_BT_ENABLED,
+	ODM_CMNINFO_BT_HS_CONNECT_PROCESS,
+	ODM_CMNINFO_BT_HS_RSSI,
+	ODM_CMNINFO_BT_OPERATION,
+	ODM_CMNINFO_BT_LIMITED_DIG,
+	ODM_CMNINFO_BT_DIG,
+	ODM_CMNINFO_BT_BUSY,
+	ODM_CMNINFO_BT_DISABLE_EDCA,
+	ODM_CMNINFO_AP_TOTAL_NUM,
+	ODM_CMNINFO_POWER_TRAINING,
+	ODM_CMNINFO_DFS_REGION_DOMAIN,
+	/*------------CALL BY VALUE-------------*/
+
+	/*Dynamic ptr array hook itms.*/
+	ODM_CMNINFO_STA_STATUS,
+	ODM_CMNINFO_MAX,
+
+};
+
+enum phydm_info_query {
+	PHYDM_INFO_FA_OFDM,
+	PHYDM_INFO_FA_CCK,
+	PHYDM_INFO_FA_TOTAL,
+	PHYDM_INFO_CCA_OFDM,
+	PHYDM_INFO_CCA_CCK,
+	PHYDM_INFO_CCA_ALL,
+	PHYDM_INFO_CRC32_OK_VHT,
+	PHYDM_INFO_CRC32_OK_HT,
+	PHYDM_INFO_CRC32_OK_LEGACY,
+	PHYDM_INFO_CRC32_OK_CCK,
+	PHYDM_INFO_CRC32_ERROR_VHT,
+	PHYDM_INFO_CRC32_ERROR_HT,
+	PHYDM_INFO_CRC32_ERROR_LEGACY,
+	PHYDM_INFO_CRC32_ERROR_CCK,
+	PHYDM_INFO_EDCCA_FLAG,
+	PHYDM_INFO_OFDM_ENABLE,
+	PHYDM_INFO_CCK_ENABLE,
+	PHYDM_INFO_DBG_PORT_0
+};
+
+enum phydm_api {
+	PHYDM_API_NBI = 1,
+	PHYDM_API_CSI_MASK,
+
+};
+
+/*2011/10/20 MH Define ODM support ability.  ODM_CMNINFO_ABILITY*/
+enum odm_ability {
+	/*BB ODM section BIT 0-19*/
+	ODM_BB_DIG = BIT(0),
+	ODM_BB_RA_MASK = BIT(1),
+	ODM_BB_DYNAMIC_TXPWR = BIT(2),
+	ODM_BB_FA_CNT = BIT(3),
+	ODM_BB_RSSI_MONITOR = BIT(4),
+	ODM_BB_CCK_PD = BIT(5),
+	ODM_BB_ANT_DIV = BIT(6),
+	ODM_BB_PWR_TRAIN = BIT(8),
+	ODM_BB_RATE_ADAPTIVE = BIT(9),
+	ODM_BB_PATH_DIV = BIT(10),
+	ODM_BB_ADAPTIVITY = BIT(13),
+	ODM_BB_CFO_TRACKING = BIT(14),
+	ODM_BB_NHM_CNT = BIT(15),
+	ODM_BB_PRIMARY_CCA = BIT(16),
+	ODM_BB_TXBF = BIT(17),
+	ODM_BB_DYNAMIC_ARFR = BIT(18),
+
+	ODM_MAC_EDCA_TURBO = BIT(20),
+	ODM_BB_DYNAMIC_RX_PATH = BIT(21),
+
+	/*RF ODM section BIT 24-31*/
+	ODM_RF_TX_PWR_TRACK = BIT(24),
+	ODM_RF_RX_GAIN_TRACK = BIT(25),
+	ODM_RF_CALIBRATION = BIT(26),
+
+};
+
+/*ODM_CMNINFO_ONE_PATH_CCA*/
+enum odm_cca_path {
+	ODM_CCA_2R = 0,
+	ODM_CCA_1R_A = 1,
+	ODM_CCA_1R_B = 2,
+};
+
+enum cca_pathdiv_en {
+	CCA_PATHDIV_DISABLE = 0,
+	CCA_PATHDIV_ENABLE = 1,
+
+};
+
+enum phy_reg_pg_type {
+	PHY_REG_PG_RELATIVE_VALUE = 0,
+	PHY_REG_PG_EXACT_VALUE = 1
+};
+
+/*2011/09/22 MH Copy from SD4 defined structure.
+ *We use to support PHY DM integration.
+ */
+
+struct phy_dm_struct {
+	/*Add for different team use temporarily*/
+	void *adapter; /*For CE/NIC team*/
+	struct rtl8192cd_priv *priv; /*For AP/ADSL team*/
+	/*When you use adapter or priv pointer,
+	 *you must make sure the pointer is ready.
+	 */
+	bool odm_ready;
+
+	struct rtl8192cd_priv fake_priv;
+
+	enum phy_reg_pg_type phy_reg_pg_value_type;
+	u8 phy_reg_pg_version;
+
+	u32 debug_components;
+	u32 fw_debug_components;
+	u32 debug_level;
+
+	u32 num_qry_phy_status_all; /*CCK + OFDM*/
+	u32 last_num_qry_phy_status_all;
+	u32 rx_pwdb_ave;
+	bool MPDIG_2G; /*off MPDIG*/
+	u8 times_2g;
+	bool is_init_hw_info_by_rfe;
+
+	/*------ ODM HANDLE, DRIVER NEEDS NOT TO HOOK------*/
+	bool is_cck_high_power;
+	u8 rf_path_rx_enable;
+	u8 control_channel;
+	/*------ ODM HANDLE, DRIVER NEEDS NOT TO HOOK------*/
+
+	/* 1  COMMON INFORMATION */
+
+	/*Init value*/
+	/*-----------HOOK BEFORE REG INIT-----------*/
+	/*ODM Platform info AP/ADSL/CE/MP = 1/2/3/4*/
+	u8 support_platform;
+	/* ODM Platform info WIN/AP/CE = 1/2/3 */
+	u8 normal_rx_path;
+	/*ODM Support Ability DIG/RATR/TX_PWR_TRACK/ ... = 1/2/3/...*/
+	u32 support_ability;
+	/*ODM PCIE/USB/SDIO = 1/2/3*/
+	u8 support_interface;
+	/*ODM composite or independent. Bit oriented/ 92C+92D+ .... or
+	 *any other type = 1/2/3/...
+	 */
+	u32 support_ic_type;
+	/*cut version TestChip/A-cut/B-cut... = 0/1/2/3/...*/
+	u8 cut_version;
+	/*Fab version TSMC/UMC = 0/1*/
+	u8 fab_version;
+	/*RF type 4T4R/3T3R/2T2R/1T2R/1T1R/...*/
+	u8 rf_type;
+	u8 rfe_type;
+	/*Board type Normal/HighPower/MiniCard/SLIM/Combo/... = 0/1/2/3/4/...*/
+	/*Enable Function DPK OFF/ON = 0/1*/
+	u8 dpk_en;
+	u8 board_type;
+	u8 package_type;
+	u16 type_glna;
+	u16 type_gpa;
+	u16 type_alna;
+	u16 type_apa;
+	/*with external LNA  NO/Yes = 0/1*/
+	u8 ext_lna; /*2G*/
+	u8 ext_lna_5g; /*5G*/
+	/*with external PA  NO/Yes = 0/1*/
+	u8 ext_pa; /*2G*/
+	u8 ext_pa_5g; /*5G*/
+	/*with Efuse number*/
+	u8 efuse0x3d7;
+	u8 efuse0x3d8;
+	/*with external TRSW  NO/Yes = 0/1*/
+	u8 ext_trsw;
+	u8 ext_lna_gain; /*2G*/
+	u8 patch_id; /*Customer ID*/
+	bool is_in_hct_test;
+	u8 wifi_test;
+
+	bool is_dual_mac_smart_concurrent;
+	u32 bk_support_ability;
+	u8 ant_div_type;
+	u8 with_extenal_ant_switch;
+	bool config_bbrf;
+	u8 odm_regulation_2_4g;
+	u8 odm_regulation_5g;
+	u8 iqk_fw_offload;
+	bool cck_new_agc;
+	u8 phydm_period;
+	u32 phydm_sys_up_time;
+	u8 num_rf_path;
+	/*-----------HOOK BEFORE REG INIT-----------*/
+
+	/*Dynamic value*/
+
+	/*--------- POINTER REFERENCE-----------*/
+
+	u8 u1_byte_temp;
+	bool BOOLEAN_temp;
+	void *PADAPTER_temp;
+
+	/*MAC PHY mode SMSP/DMSP/DMDP = 0/1/2*/
+	u8 *mac_phy_mode;
+	/*TX Unicast byte count*/
+	u64 *num_tx_bytes_unicast;
+	/*RX Unicast byte count*/
+	u64 *num_rx_bytes_unicast;
+	/*Wireless mode B/G/A/N = BIT0/BIT1/BIT2/BIT3*/
+	u8 *wireless_mode;
+	/*Frequence band 2.4G/5G = 0/1*/
+	u8 *band_type;
+	/*Secondary channel offset don't_care/below/above = 0/1/2*/
+	u8 *sec_ch_offset;
+	/*security mode Open/WEP/AES/TKIP = 0/1/2/3*/
+	u8 *security;
+	/*BW info 20M/40M/80M = 0/1/2*/
+	u8 *band_width;
+	/*Central channel location Ch1/Ch2/....*/
+	u8 *channel; /*central channel number*/
+	bool dpk_done;
+	/*Common info for 92D DMSP*/
+
+	bool *is_get_value_from_other_mac;
+	void **buddy_adapter;
+	bool *is_master_of_dmsp; /* MAC0: master, MAC1: slave */
+	/*Common info for status*/
+	bool *is_scan_in_process;
+	bool *is_power_saving;
+	/*CCA path 2-path/path-A/path-B = 0/1/2; using enum odm_cca_path.*/
+	u8 *one_path_cca;
+	u8 *antenna_test;
+	bool *is_net_closed;
+	u8 *pu1_forced_igi_lb;
+	bool *is_fcs_mode_enable;
+	/*--------- For 8723B IQK-----------*/
+	bool *is_1_antenna;
+	u8 *rf_default_path;
+	/* 0:S1, 1:S0 */
+
+	/*--------- POINTER REFERENCE-----------*/
+	u16 *forced_data_rate;
+	u8 *enable_antdiv;
+	u8 *enable_adaptivity;
+	u8 *hub_usb_mode;
+	bool *is_fw_dw_rsvd_page_in_progress;
+	u32 *current_tx_tp;
+	u32 *current_rx_tp;
+	u8 *sounding_seq;
+	/*------------CALL BY VALUE-------------*/
+	bool is_link_in_process;
+	bool is_wifi_direct;
+	bool is_wifi_display;
+	bool is_linked;
+	bool is_linkedcmw500;
+	bool is_in_lps_pg;
+	bool bsta_state;
+	u8 rssi_min;
+	u8 interface_index; /*Add for 92D  dual MAC: 0--Mac0 1--Mac1*/
+	bool is_mp_chip;
+	bool is_one_entry_only;
+	bool mp_mode;
+	u32 one_entry_macid;
+	u8 pre_number_linked_client;
+	u8 number_linked_client;
+	u8 pre_number_active_client;
+	u8 number_active_client;
+	/*Common info for BTDM*/
+	bool is_bt_enabled; /*BT is enabled*/
+	bool is_bt_connect_process; /*BT HS is under connection progress.*/
+	u8 bt_hs_rssi; /*BT HS mode wifi rssi value.*/
+	bool is_bt_hs_operation; /*BT HS mode is under progress*/
+	u8 bt_hs_dig_val; /*use BT rssi to decide the DIG value*/
+	bool is_bt_disable_edca_turbo; /*Under some condition, don't enable*/
+	bool is_bt_busy; /*BT is busy.*/
+	bool is_bt_limited_dig; /*BT is busy.*/
+	bool is_disable_phy_api;
+	/*------------CALL BY VALUE-------------*/
+	u8 rssi_a;
+	u8 rssi_b;
+	u8 rssi_c;
+	u8 rssi_d;
+	u64 rssi_trsw;
+	u64 rssi_trsw_h;
+	u64 rssi_trsw_l;
+	u64 rssi_trsw_iso;
+	u8 tx_ant_status;
+	u8 rx_ant_status;
+	u8 cck_lna_idx;
+	u8 cck_vga_idx;
+	u8 curr_station_id;
+	u8 ofdm_agc_idx[4];
+
+	u8 rx_rate;
+	bool is_noisy_state;
+	u8 tx_rate;
+	u8 linked_interval;
+	u8 pre_channel;
+	u32 txagc_offset_value_a;
+	bool is_txagc_offset_positive_a;
+	u32 txagc_offset_value_b;
+	bool is_txagc_offset_positive_b;
+	u32 tx_tp;
+	u32 rx_tp;
+	u32 total_tp;
+	u64 cur_tx_ok_cnt;
+	u64 cur_rx_ok_cnt;
+	u64 last_tx_ok_cnt;
+	u64 last_rx_ok_cnt;
+	u32 bb_swing_offset_a;
+	bool is_bb_swing_offset_positive_a;
+	u32 bb_swing_offset_b;
+	bool is_bb_swing_offset_positive_b;
+	u8 igi_lower_bound;
+	u8 igi_upper_bound;
+	u8 antdiv_rssi;
+	u8 fat_comb_a;
+	u8 fat_comb_b;
+	u8 antdiv_intvl;
+	u8 ant_type;
+	u8 pre_ant_type;
+	u8 antdiv_period;
+	u8 evm_antdiv_period;
+	u8 antdiv_select;
+	u8 path_select;
+	u8 antdiv_evm_en;
+	u8 bdc_holdstate;
+	u8 ndpa_period;
+	bool h2c_rarpt_connect;
+	bool cck_agc_report_type;
+
+	u8 dm_dig_max_TH;
+	u8 dm_dig_min_TH;
+	u8 print_agc;
+	u8 traffic_load;
+	u8 pre_traffic_load;
+	/*8821C Antenna BTG/WLG/WLA Select*/
+	u8 current_rf_set_8821c;
+	u8 default_rf_set_8821c;
+	/*For Adaptivtiy*/
+	u16 nhm_cnt_0;
+	u16 nhm_cnt_1;
+	s8 TH_L2H_default;
+	s8 th_edcca_hl_diff_default;
+	s8 th_l2h_ini;
+	s8 th_edcca_hl_diff;
+	s8 th_l2h_ini_mode2;
+	s8 th_edcca_hl_diff_mode2;
+	bool carrier_sense_enable;
+	u8 adaptivity_igi_upper;
+	bool adaptivity_flag;
+	u8 dc_backoff;
+	bool adaptivity_enable;
+	u8 ap_total_num;
+	bool edcca_enable;
+	u8 pre_dbg_priority;
+	struct adaptivity_statistics adaptivity;
+	/*For Adaptivtiy*/
+	u8 last_usb_hub;
+	u8 tx_bf_data_rate;
+
+	u8 nbi_set_result;
+
+	u8 c2h_cmd_start;
+	u8 fw_debug_trace[60];
+	u8 pre_c2h_seq;
+	bool fw_buff_is_enpty;
+	u32 data_frame_num;
+
+	/*for noise detection*/
+	bool noisy_decision; /*b_noisy*/
+	bool pre_b_noisy;
+	u32 noisy_decision_smooth;
+	bool is_disable_dym_ecs;
+
+	struct odm_noise_monitor noise_level;
+	/*Define STA info.*/
+	/*odm_sta_info*/
+	/*2012/01/12 MH For MP,
+	 *we need to reduce one array pointer for default port.??
+	 */
+	struct rtl_sta_info *odm_sta_info[ODM_ASSOCIATE_ENTRY_NUM];
+	u16 platform2phydm_macid_table[ODM_ASSOCIATE_ENTRY_NUM];
+	/* platform_macid_table[platform_macid] = phydm_macid */
+	s32 accumulate_pwdb[ODM_ASSOCIATE_ENTRY_NUM];
+
+	/*2012/02/14 MH Add to share 88E ra with other SW team.*/
+	/*We need to colelct all support abilit to a proper area.*/
+
+	bool ra_support88e;
+
+	struct odm_phy_dbg_info phy_dbg_info;
+
+	/*ODM Structure*/
+	struct fast_antenna_training dm_fat_table;
+	struct dig_thres dm_dig_table;
+	struct dyn_pwr_saving dm_ps_table;
+	struct dyn_primary_cca dm_pri_cca;
+	struct ra_table dm_ra_table;
+	struct false_alarm_stat false_alm_cnt;
+	struct false_alarm_stat flase_alm_cnt_buddy_adapter;
+	struct sw_antenna_switch dm_swat_table;
+	struct cfo_tracking dm_cfo_track;
+	struct acs_info dm_acs;
+	struct ccx_info dm_ccx_info;
+	struct psd_info dm_psd_table;
+
+	struct rt_adcsmp adcsmp;
+
+	struct dm_iqk_info IQK_info;
+
+	struct edca_turbo dm_edca_table;
+	u32 WMMEDCA_BE;
+
+	bool *is_driver_stopped;
+	bool *is_driver_is_going_to_pnp_set_power_sleep;
+	bool *pinit_adpt_in_progress;
+
+	/*PSD*/
+	bool is_user_assign_level;
+	u8 RSSI_BT; /*come from BT*/
+	bool is_psd_in_process;
+	bool is_psd_active;
+	bool is_dm_initial_gain_enable;
+
+	/*MPT DIG*/
+	struct timer_list mpt_dig_timer;
+
+	/*for rate adaptive, in fact,  88c/92c fw will handle this*/
+	u8 is_use_ra_mask;
+
+	/* for dynamic SoML control */
+	bool bsomlenabled;
+
+	struct odm_rate_adaptive rate_adaptive;
+	struct dm_rf_calibration_struct rf_calibrate_info;
+	u32 n_iqk_cnt;
+	u32 n_iqk_ok_cnt;
+	u32 n_iqk_fail_cnt;
+
+	/*Power Training*/
+	u8 force_power_training_state;
+	bool is_change_state;
+	u32 PT_score;
+	u64 ofdm_rx_cnt;
+	u64 cck_rx_cnt;
+	bool is_disable_power_training;
+	u8 dynamic_tx_high_power_lvl;
+	u8 last_dtp_lvl;
+	u32 tx_agc_ofdm_18_6;
+	u8 rx_pkt_type;
+
+	/*ODM relative time.*/
+	struct timer_list path_div_switch_timer;
+	/*2011.09.27 add for path Diversity*/
+	struct timer_list cck_path_diversity_timer;
+	struct timer_list fast_ant_training_timer;
+	struct timer_list sbdcnt_timer;
+
+	/*ODM relative workitem.*/
+};
+
+enum phydm_structure_type {
+	PHYDM_FALSEALMCNT,
+	PHYDM_CFOTRACK,
+	PHYDM_ADAPTIVITY,
+	PHYDM_ROMINFO,
+
+};
+
+enum odm_rf_content {
+	odm_radioa_txt = 0x1000,
+	odm_radiob_txt = 0x1001,
+	odm_radioc_txt = 0x1002,
+	odm_radiod_txt = 0x1003
+};
+
+enum odm_bb_config_type {
+	CONFIG_BB_PHY_REG,
+	CONFIG_BB_AGC_TAB,
+	CONFIG_BB_AGC_TAB_2G,
+	CONFIG_BB_AGC_TAB_5G,
+	CONFIG_BB_PHY_REG_PG,
+	CONFIG_BB_PHY_REG_MP,
+	CONFIG_BB_AGC_TAB_DIFF,
+};
+
+enum odm_rf_config_type {
+	CONFIG_RF_RADIO,
+	CONFIG_RF_TXPWR_LMT,
+};
+
+enum odm_fw_config_type {
+	CONFIG_FW_NIC,
+	CONFIG_FW_NIC_2,
+	CONFIG_FW_AP,
+	CONFIG_FW_AP_2,
+	CONFIG_FW_MP,
+	CONFIG_FW_WOWLAN,
+	CONFIG_FW_WOWLAN_2,
+	CONFIG_FW_AP_WOWLAN,
+	CONFIG_FW_BT,
+};
+
+/*status code*/
+enum rt_status {
+	RT_STATUS_SUCCESS,
+	RT_STATUS_FAILURE,
+	RT_STATUS_PENDING,
+	RT_STATUS_RESOURCE,
+	RT_STATUS_INVALID_CONTEXT,
+	RT_STATUS_INVALID_PARAMETER,
+	RT_STATUS_NOT_SUPPORT,
+	RT_STATUS_OS_API_FAILED,
+};
+
+/*===========================================================*/
+/*AGC RX High Power mode*/
+/*===========================================================*/
+#define lna_low_gain_1 0x64
+#define lna_low_gain_2 0x5A
+#define lna_low_gain_3 0x58
+
+#define FA_RXHP_TH1 5000
+#define FA_RXHP_TH2 1500
+#define FA_RXHP_TH3 800
+#define FA_RXHP_TH4 600
+#define FA_RXHP_TH5 500
+
+enum dm_1r_cca {
+	CCA_1R = 0,
+	CCA_2R = 1,
+	CCA_MAX = 2,
+};
+
+enum dm_rf {
+	rf_save = 0,
+	rf_normal = 1,
+	RF_MAX = 2,
+};
+
+/*check Sta pointer valid or not*/
+
+#define IS_STA_VALID(sta) (sta)
+
+u32 odm_convert_to_db(u32 value);
+
+u32 odm_convert_to_linear(u32 value);
+
+s32 odm_pwdb_conversion(s32 X, u32 total_bit, u32 decimal_bit);
+
+s32 odm_sign_conversion(s32 value, u32 total_bit);
+
+void odm_init_mp_driver_status(struct phy_dm_struct *dm);
+
+void phydm_txcurrentcalibration(struct phy_dm_struct *dm);
+
+void phydm_seq_sorting(void *dm_void, u32 *value, u32 *rank_idx, u32 *idx_out,
+		       u8 seq_length);
+
+void odm_dm_init(struct phy_dm_struct *dm);
+
+void odm_dm_reset(struct phy_dm_struct *dm);
+
+void phydm_support_ability_debug(void *dm_void, u32 *const dm_value, u32 *_used,
+				 char *output, u32 *_out_len);
+
+void phydm_config_ofdm_rx_path(struct phy_dm_struct *dm, u32 path);
+
+void phydm_config_trx_path(void *dm_void, u32 *const dm_value, u32 *_used,
+			   char *output, u32 *_out_len);
+
+void odm_dm_watchdog(struct phy_dm_struct *dm);
+
+void phydm_watchdog_mp(struct phy_dm_struct *dm);
+
+void odm_cmn_info_init(struct phy_dm_struct *dm, enum odm_cmninfo cmn_info,
+		       u32 value);
+
+void odm_cmn_info_hook(struct phy_dm_struct *dm, enum odm_cmninfo cmn_info,
+		       void *value);
+
+void odm_cmn_info_ptr_array_hook(struct phy_dm_struct *dm,
+				 enum odm_cmninfo cmn_info, u16 index,
+				 void *value);
+
+void odm_cmn_info_update(struct phy_dm_struct *dm, u32 cmn_info, u64 value);
+
+u32 phydm_cmn_info_query(struct phy_dm_struct *dm,
+			 enum phydm_info_query info_type);
+
+void odm_init_all_timers(struct phy_dm_struct *dm);
+
+void odm_cancel_all_timers(struct phy_dm_struct *dm);
+
+void odm_release_all_timers(struct phy_dm_struct *dm);
+
+void odm_asoc_entry_init(struct phy_dm_struct *dm);
+
+void *phydm_get_structure(struct phy_dm_struct *dm, u8 structure_type);
+
+/*===========================================================*/
+/* The following is for compile only*/
+/*===========================================================*/
+
+#define IS_HARDWARE_TYPE_8188E(_adapter) false
+#define IS_HARDWARE_TYPE_8188F(_adapter) false
+#define IS_HARDWARE_TYPE_8703B(_adapter) false
+#define IS_HARDWARE_TYPE_8723D(_adapter) false
+#define IS_HARDWARE_TYPE_8821C(_adapter) false
+#define IS_HARDWARE_TYPE_8812AU(_adapter) false
+#define IS_HARDWARE_TYPE_8814A(_adapter) false
+#define IS_HARDWARE_TYPE_8814AU(_adapter) false
+#define IS_HARDWARE_TYPE_8814AE(_adapter) false
+#define IS_HARDWARE_TYPE_8814AS(_adapter) false
+#define IS_HARDWARE_TYPE_8723BU(_adapter) false
+#define IS_HARDWARE_TYPE_8822BU(_adapter) false
+#define IS_HARDWARE_TYPE_8822BS(_adapter) false
+#define IS_HARDWARE_TYPE_JAGUAR(_adapter)                                      \
+	(IS_HARDWARE_TYPE_8812(_adapter) || IS_HARDWARE_TYPE_8821(_adapter))
+#define IS_HARDWARE_TYPE_8723AE(_adapter) false
+#define IS_HARDWARE_TYPE_8192C(_adapter) false
+#define IS_HARDWARE_TYPE_8192D(_adapter) false
+#define RF_T_METER_92D 0x42
+
+#define GET_RX_STATUS_DESC_RX_MCS(__prx_status_desc)                           \
+	LE_BITS_TO_1BYTE(__prx_status_desc + 12, 0, 6)
+
+#define REG_CONFIG_RAM64X16 0xb2c
+
+#define TARGET_CHNL_NUM_2G_5G 59
+
+/* *********************************************************** */
+
+void odm_dtc(struct phy_dm_struct *dm);
+
+void phydm_noisy_detection(struct phy_dm_struct *dm);
+
+void phydm_set_ext_switch(void *dm_void, u32 *const dm_value, u32 *_used,
+			  char *output, u32 *_out_len);
+
+void phydm_api_debug(void *dm_void, u32 function_map, u32 *const dm_value,
+		     u32 *_used, char *output, u32 *_out_len);
+
+u8 phydm_nbi_setting(void *dm_void, u32 enable, u32 channel, u32 bw,
+		     u32 f_interference, u32 second_ch);
+#endif /* __HALDMOUTSRC_H__ */
diff --git a/drivers/staging/rtlwifi/phydm/phydm_acs.c b/drivers/staging/rtlwifi/phydm/phydm_acs.c
new file mode 100644
index 000000000000..eae5a0a24b9b
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_acs.c
@@ -0,0 +1,200 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+/* ************************************************************
+ * include files
+ * *************************************************************/
+#include "mp_precomp.h"
+#include "phydm_precomp.h"
+
+u8 odm_get_auto_channel_select_result(void *dm_void, u8 band)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct acs_info *acs = &dm->dm_acs;
+	u8 result;
+
+	if (band == ODM_BAND_2_4G) {
+		ODM_RT_TRACE(
+			dm, ODM_COMP_ACS,
+			"[struct acs_info] %s(): clean_channel_2g(%d)\n",
+			__func__, acs->clean_channel_2g);
+		result = (u8)acs->clean_channel_2g;
+	} else {
+		ODM_RT_TRACE(
+			dm, ODM_COMP_ACS,
+			"[struct acs_info] %s(): clean_channel_5g(%d)\n",
+			__func__, acs->clean_channel_5g);
+		result = (u8)acs->clean_channel_5g;
+	}
+
+	return result;
+}
+
+static void odm_auto_channel_select_setting(void *dm_void, bool is_enable)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u16 period = 0x2710; /* 40ms in default */
+	u16 nhm_type = 0x7;
+
+	ODM_RT_TRACE(dm, ODM_COMP_ACS, "%s()=========>\n", __func__);
+
+	if (is_enable) {
+		/* 20 ms */
+		period = 0x1388;
+		nhm_type = 0x1;
+	}
+
+	if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+		/* PHY parameters initialize for ac series */
+
+		/* 0x990[31:16]=0x2710
+		 * Time duration for NHM unit: 4us, 0x2710=40ms
+		 */
+		odm_write_2byte(dm, ODM_REG_CCX_PERIOD_11AC + 2, period);
+	} else if (dm->support_ic_type & ODM_IC_11N_SERIES) {
+		/* PHY parameters initialize for n series */
+
+		/* 0x894[31:16]=0x2710
+		 * Time duration for NHM unit: 4us, 0x2710=40ms
+		 */
+		odm_write_2byte(dm, ODM_REG_CCX_PERIOD_11N + 2, period);
+	}
+}
+
+void odm_auto_channel_select_init(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct acs_info *acs = &dm->dm_acs;
+	u8 i;
+
+	if (!(dm->support_ability & ODM_BB_NHM_CNT))
+		return;
+
+	if (acs->is_force_acs_result)
+		return;
+
+	ODM_RT_TRACE(dm, ODM_COMP_ACS, "%s()=========>\n", __func__);
+
+	acs->clean_channel_2g = 1;
+	acs->clean_channel_5g = 36;
+
+	for (i = 0; i < ODM_MAX_CHANNEL_2G; ++i) {
+		acs->channel_info_2g[0][i] = 0;
+		acs->channel_info_2g[1][i] = 0;
+	}
+
+	if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+		for (i = 0; i < ODM_MAX_CHANNEL_5G; ++i) {
+			acs->channel_info_5g[0][i] = 0;
+			acs->channel_info_5g[1][i] = 0;
+		}
+	}
+}
+
+void odm_auto_channel_select_reset(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct acs_info *acs = &dm->dm_acs;
+
+	if (!(dm->support_ability & ODM_BB_NHM_CNT))
+		return;
+
+	if (acs->is_force_acs_result)
+		return;
+
+	ODM_RT_TRACE(dm, ODM_COMP_ACS, "%s()=========>\n", __func__);
+
+	odm_auto_channel_select_setting(dm, true); /* for 20ms measurement */
+	phydm_nhm_counter_statistics_reset(dm);
+}
+
+void odm_auto_channel_select(void *dm_void, u8 channel)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct acs_info *acs = &dm->dm_acs;
+	u8 channel_idx = 0, search_idx = 0;
+	u16 max_score = 0;
+
+	if (!(dm->support_ability & ODM_BB_NHM_CNT)) {
+		ODM_RT_TRACE(
+			dm, ODM_COMP_DIG,
+			"%s(): Return: support_ability ODM_BB_NHM_CNT is disabled\n",
+			__func__);
+		return;
+	}
+
+	if (acs->is_force_acs_result) {
+		ODM_RT_TRACE(
+			dm, ODM_COMP_DIG,
+			"%s(): Force 2G clean channel = %d, 5G clean channel = %d\n",
+			__func__, acs->clean_channel_2g, acs->clean_channel_5g);
+		return;
+	}
+
+	ODM_RT_TRACE(dm, ODM_COMP_ACS, "%s(): channel = %d=========>\n",
+		     __func__, channel);
+
+	phydm_get_nhm_counter_statistics(dm);
+	odm_auto_channel_select_setting(dm, false);
+
+	if (channel >= 1 && channel <= 14) {
+		channel_idx = channel - 1;
+		acs->channel_info_2g[1][channel_idx]++;
+
+		if (acs->channel_info_2g[1][channel_idx] >= 2)
+			acs->channel_info_2g[0][channel_idx] =
+				(acs->channel_info_2g[0][channel_idx] >> 1) +
+				(acs->channel_info_2g[0][channel_idx] >> 2) +
+				(dm->nhm_cnt_0 >> 2);
+		else
+			acs->channel_info_2g[0][channel_idx] = dm->nhm_cnt_0;
+
+		ODM_RT_TRACE(dm, ODM_COMP_ACS, "%s(): nhm_cnt_0 = %d\n",
+			     __func__, dm->nhm_cnt_0);
+		ODM_RT_TRACE(
+			dm, ODM_COMP_ACS,
+			"%s(): Channel_Info[0][%d] = %d, Channel_Info[1][%d] = %d\n",
+			__func__, channel_idx,
+			acs->channel_info_2g[0][channel_idx], channel_idx,
+			acs->channel_info_2g[1][channel_idx]);
+
+		for (search_idx = 0; search_idx < ODM_MAX_CHANNEL_2G;
+		     search_idx++) {
+			if (acs->channel_info_2g[1][search_idx] != 0 &&
+			    acs->channel_info_2g[0][search_idx] >= max_score) {
+				max_score = acs->channel_info_2g[0][search_idx];
+				acs->clean_channel_2g = search_idx + 1;
+			}
+		}
+		ODM_RT_TRACE(
+			dm, ODM_COMP_ACS,
+			"(1)%s(): 2G: clean_channel_2g = %d, max_score = %d\n",
+			__func__, acs->clean_channel_2g, max_score);
+
+	} else if (channel >= 36) {
+		/* Need to do */
+		acs->clean_channel_5g = channel;
+	}
+}
diff --git a/drivers/staging/rtlwifi/phydm/phydm_acs.h b/drivers/staging/rtlwifi/phydm/phydm_acs.h
new file mode 100644
index 000000000000..51d72b72bd6f
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_acs.h
@@ -0,0 +1,57 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __PHYDMACS_H__
+#define __PHYDMACS_H__
+
+#define ACS_VERSION "1.1" /*20150729 by YuChen*/
+#define CLM_VERSION "1.0"
+
+#define ODM_MAX_CHANNEL_2G 14
+#define ODM_MAX_CHANNEL_5G 24
+
+/* For phydm_auto_channel_select_setting_ap() */
+#define STORE_DEFAULT_NHM_SETTING 0
+#define RESTORE_DEFAULT_NHM_SETTING 1
+#define ACS_NHM_SETTING 2
+
+struct acs_info {
+	bool is_force_acs_result;
+	u8 clean_channel_2g;
+	u8 clean_channel_5g;
+	/* channel_info[1]: channel score, channel_info[2]:channel_scan_times */
+	u16 channel_info_2g[2][ODM_MAX_CHANNEL_2G];
+	u16 channel_info_5g[2][ODM_MAX_CHANNEL_5G];
+};
+
+void odm_auto_channel_select_init(void *dm_void);
+
+void odm_auto_channel_select_reset(void *dm_void);
+
+void odm_auto_channel_select(void *dm_void, u8 channel);
+
+u8 odm_get_auto_channel_select_result(void *dm_void, u8 band);
+
+#endif /* #ifndef	__PHYDMACS_H__ */
diff --git a/drivers/staging/rtlwifi/phydm/phydm_adaptivity.c b/drivers/staging/rtlwifi/phydm/phydm_adaptivity.c
new file mode 100644
index 000000000000..4f9e267409f6
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_adaptivity.c
@@ -0,0 +1,941 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+/* ************************************************************
+ * include files
+ * *************************************************************/
+#include "mp_precomp.h"
+#include "phydm_precomp.h"
+
+void phydm_check_adaptivity(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct adaptivity_statistics *adaptivity =
+		(struct adaptivity_statistics *)phydm_get_structure(
+			dm, PHYDM_ADAPTIVITY);
+
+	if (dm->support_ability & ODM_BB_ADAPTIVITY) {
+		if (adaptivity->dynamic_link_adaptivity ||
+		    adaptivity->acs_for_adaptivity) {
+			if (dm->is_linked && !adaptivity->is_check) {
+				phydm_nhm_counter_statistics(dm);
+				phydm_check_environment(dm);
+			} else if (!dm->is_linked) {
+				adaptivity->is_check = false;
+			}
+		} else {
+			dm->adaptivity_enable = true;
+
+			if (dm->support_ic_type & (ODM_IC_11AC_GAIN_IDX_EDCCA |
+						   ODM_IC_11N_GAIN_IDX_EDCCA))
+				dm->adaptivity_flag = false;
+			else
+				dm->adaptivity_flag = true;
+		}
+	} else {
+		dm->adaptivity_enable = false;
+		dm->adaptivity_flag = false;
+	}
+}
+
+void phydm_nhm_counter_statistics_init(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	if (dm->support_ic_type & ODM_IC_11N_SERIES) {
+		/*PHY parameters initialize for n series*/
+
+		/*0x894[31:16]=0x0xC350
+		 *Time duration for NHM unit: us, 0xc350=200ms
+		 */
+		odm_write_2byte(dm, ODM_REG_CCX_PERIOD_11N + 2, 0xC350);
+		/*0x890[31:16]=0xffff		th_9, th_10*/
+		odm_write_2byte(dm, ODM_REG_NHM_TH9_TH10_11N + 2, 0xffff);
+		/*0x898=0xffffff52		th_3, th_2, th_1, th_0*/
+		odm_write_4byte(dm, ODM_REG_NHM_TH3_TO_TH0_11N, 0xffffff50);
+		/*0x89c=0xffffffff		th_7, th_6, th_5, th_4*/
+		odm_write_4byte(dm, ODM_REG_NHM_TH7_TO_TH4_11N, 0xffffffff);
+		/*0xe28[7:0]=0xff		th_8*/
+		odm_set_bb_reg(dm, ODM_REG_FPGA0_IQK_11N, MASKBYTE0, 0xff);
+		/*0x890[10:8]=1		ignoreCCA ignore PHYTXON enable CCX*/
+		odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11N,
+			       BIT(10) | BIT(9) | BIT(8), 0x1);
+		/*0xc0c[7]=1			max power among all RX ants*/
+		odm_set_bb_reg(dm, ODM_REG_OFDM_FA_RSTC_11N, BIT(7), 0x1);
+	} else if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+		/*PHY parameters initialize for ac series*/
+
+		odm_write_2byte(dm, ODM_REG_CCX_PERIOD_11AC + 2, 0xC350);
+		/*0x994[31:16]=0xffff		th_9, th_10*/
+		odm_write_2byte(dm, ODM_REG_NHM_TH9_TH10_11AC + 2, 0xffff);
+		/*0x998=0xffffff52		th_3, th_2, th_1, th_0*/
+		odm_write_4byte(dm, ODM_REG_NHM_TH3_TO_TH0_11AC, 0xffffff50);
+		/*0x99c=0xffffffff		th_7, th_6, th_5, th_4*/
+		odm_write_4byte(dm, ODM_REG_NHM_TH7_TO_TH4_11AC, 0xffffffff);
+		/*0x9a0[7:0]=0xff		th_8*/
+		odm_set_bb_reg(dm, ODM_REG_NHM_TH8_11AC, MASKBYTE0, 0xff);
+		/*0x994[10:8]=1		ignoreCCA ignore PHYTXON enable CCX*/
+		odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11AC,
+			       BIT(8) | BIT(9) | BIT(10), 0x1);
+		/*0x9e8[7]=1			max power among all RX ants*/
+		odm_set_bb_reg(dm, ODM_REG_NHM_9E8_11AC, BIT(0), 0x1);
+	}
+}
+
+void phydm_nhm_counter_statistics(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	if (!(dm->support_ability & ODM_BB_NHM_CNT))
+		return;
+
+	/*Get NHM report*/
+	phydm_get_nhm_counter_statistics(dm);
+
+	/*Reset NHM counter*/
+	phydm_nhm_counter_statistics_reset(dm);
+}
+
+void phydm_get_nhm_counter_statistics(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 value32 = 0;
+
+	if (dm->support_ic_type & ODM_IC_11AC_SERIES)
+		value32 = odm_get_bb_reg(dm, ODM_REG_NHM_CNT_11AC, MASKDWORD);
+	else if (dm->support_ic_type & ODM_IC_11N_SERIES)
+		value32 = odm_get_bb_reg(dm, ODM_REG_NHM_CNT_11N, MASKDWORD);
+
+	dm->nhm_cnt_0 = (u8)(value32 & MASKBYTE0);
+	dm->nhm_cnt_1 = (u8)((value32 & MASKBYTE1) >> 8);
+}
+
+void phydm_nhm_counter_statistics_reset(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	if (dm->support_ic_type & ODM_IC_11N_SERIES) {
+		odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11N, BIT(1), 0);
+		odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11N, BIT(1), 1);
+	} else if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+		odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11AC, BIT(1), 0);
+		odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11AC, BIT(1), 1);
+	}
+}
+
+void phydm_set_edcca_threshold(void *dm_void, s8 H2L, s8 L2H)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	if (dm->support_ic_type & ODM_IC_11N_SERIES)
+		odm_set_bb_reg(dm, REG_OFDM_0_ECCA_THRESHOLD,
+			       MASKBYTE2 | MASKBYTE0,
+			       (u32)((u8)L2H | (u8)H2L << 16));
+	else if (dm->support_ic_type & ODM_IC_11AC_SERIES)
+		odm_set_bb_reg(dm, REG_FPGA0_XB_LSSI_READ_BACK, MASKLWORD,
+			       (u16)((u8)L2H | (u8)H2L << 8));
+}
+
+static void phydm_set_lna(void *dm_void, enum phydm_set_lna type)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	if (dm->support_ic_type & (ODM_RTL8188E | ODM_RTL8192E)) {
+		if (type == phydm_disable_lna) {
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0xef, 0x80000, 0x1);
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x30, 0xfffff,
+				       0x18000); /*select Rx mode*/
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x31, 0xfffff,
+				       0x0000f);
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x32, 0xfffff,
+				       0x37f82); /*disable LNA*/
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0xef, 0x80000, 0x0);
+			if (dm->rf_type > ODM_1T1R) {
+				odm_set_rf_reg(dm, ODM_RF_PATH_B, 0xef, 0x80000,
+					       0x1);
+				odm_set_rf_reg(dm, ODM_RF_PATH_B, 0x30, 0xfffff,
+					       0x18000);
+				odm_set_rf_reg(dm, ODM_RF_PATH_B, 0x31, 0xfffff,
+					       0x0000f);
+				odm_set_rf_reg(dm, ODM_RF_PATH_B, 0x32, 0xfffff,
+					       0x37f82);
+				odm_set_rf_reg(dm, ODM_RF_PATH_B, 0xef, 0x80000,
+					       0x0);
+			}
+		} else if (type == phydm_enable_lna) {
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0xef, 0x80000, 0x1);
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x30, 0xfffff,
+				       0x18000); /*select Rx mode*/
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x31, 0xfffff,
+				       0x0000f);
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x32, 0xfffff,
+				       0x77f82); /*back to normal*/
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0xef, 0x80000, 0x0);
+			if (dm->rf_type > ODM_1T1R) {
+				odm_set_rf_reg(dm, ODM_RF_PATH_B, 0xef, 0x80000,
+					       0x1);
+				odm_set_rf_reg(dm, ODM_RF_PATH_B, 0x30, 0xfffff,
+					       0x18000);
+				odm_set_rf_reg(dm, ODM_RF_PATH_B, 0x31, 0xfffff,
+					       0x0000f);
+				odm_set_rf_reg(dm, ODM_RF_PATH_B, 0x32, 0xfffff,
+					       0x77f82);
+				odm_set_rf_reg(dm, ODM_RF_PATH_B, 0xef, 0x80000,
+					       0x0);
+			}
+		}
+	} else if (dm->support_ic_type & ODM_RTL8723B) {
+		if (type == phydm_disable_lna) {
+			/*S0*/
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0xef, 0x80000, 0x1);
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x30, 0xfffff,
+				       0x18000); /*select Rx mode*/
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x31, 0xfffff,
+				       0x0001f);
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x32, 0xfffff,
+				       0xe6137); /*disable LNA*/
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0xef, 0x80000, 0x0);
+			/*S1*/
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0xed, 0x00020, 0x1);
+			odm_set_rf_reg(
+				dm, ODM_RF_PATH_A, 0x43, 0xfffff,
+				0x3008d); /*select Rx mode and disable LNA*/
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0xed, 0x00020, 0x0);
+		} else if (type == phydm_enable_lna) {
+			/*S0*/
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0xef, 0x80000, 0x1);
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x30, 0xfffff,
+				       0x18000); /*select Rx mode*/
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x31, 0xfffff,
+				       0x0001f);
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x32, 0xfffff,
+				       0xe6177); /*disable LNA*/
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0xef, 0x80000, 0x0);
+			/*S1*/
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0xed, 0x00020, 0x1);
+			odm_set_rf_reg(
+				dm, ODM_RF_PATH_A, 0x43, 0xfffff,
+				0x300bd); /*select Rx mode and disable LNA*/
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0xed, 0x00020, 0x0);
+		}
+
+	} else if (dm->support_ic_type & ODM_RTL8812) {
+		if (type == phydm_disable_lna) {
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0xef, 0x80000, 0x1);
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x30, 0xfffff,
+				       0x18000); /*select Rx mode*/
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x31, 0xfffff,
+				       0x3f7ff);
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x32, 0xfffff,
+				       0xc22bf); /*disable LNA*/
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0xef, 0x80000, 0x0);
+			if (dm->rf_type > ODM_1T1R) {
+				odm_set_rf_reg(dm, ODM_RF_PATH_B, 0xef, 0x80000,
+					       0x1);
+				odm_set_rf_reg(dm, ODM_RF_PATH_B, 0x30, 0xfffff,
+					       0x18000); /*select Rx mode*/
+				odm_set_rf_reg(dm, ODM_RF_PATH_B, 0x31, 0xfffff,
+					       0x3f7ff);
+				odm_set_rf_reg(dm, ODM_RF_PATH_B, 0x32, 0xfffff,
+					       0xc22bf); /*disable LNA*/
+				odm_set_rf_reg(dm, ODM_RF_PATH_B, 0xef, 0x80000,
+					       0x0);
+			}
+		} else if (type == phydm_enable_lna) {
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0xef, 0x80000, 0x1);
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x30, 0xfffff,
+				       0x18000); /*select Rx mode*/
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x31, 0xfffff,
+				       0x3f7ff);
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x32, 0xfffff,
+				       0xc26bf); /*disable LNA*/
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0xef, 0x80000, 0x0);
+			if (dm->rf_type > ODM_1T1R) {
+				odm_set_rf_reg(dm, ODM_RF_PATH_B, 0xef, 0x80000,
+					       0x1);
+				odm_set_rf_reg(dm, ODM_RF_PATH_B, 0x30, 0xfffff,
+					       0x18000); /*select Rx mode*/
+				odm_set_rf_reg(dm, ODM_RF_PATH_B, 0x31, 0xfffff,
+					       0x3f7ff);
+				odm_set_rf_reg(dm, ODM_RF_PATH_B, 0x32, 0xfffff,
+					       0xc26bf); /*disable LNA*/
+				odm_set_rf_reg(dm, ODM_RF_PATH_B, 0xef, 0x80000,
+					       0x0);
+			}
+		}
+	} else if (dm->support_ic_type & (ODM_RTL8821 | ODM_RTL8881A)) {
+		if (type == phydm_disable_lna) {
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0xef, 0x80000, 0x1);
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x30, 0xfffff,
+				       0x18000); /*select Rx mode*/
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x31, 0xfffff,
+				       0x0002f);
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x32, 0xfffff,
+				       0xfb09b); /*disable LNA*/
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0xef, 0x80000, 0x0);
+		} else if (type == phydm_enable_lna) {
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0xef, 0x80000, 0x1);
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x30, 0xfffff,
+				       0x18000); /*select Rx mode*/
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x31, 0xfffff,
+				       0x0002f);
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x32, 0xfffff,
+				       0xfb0bb); /*disable LNA*/
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, 0xef, 0x80000, 0x0);
+		}
+	}
+}
+
+void phydm_set_trx_mux(void *dm_void, enum phydm_trx_mux_type tx_mode,
+		       enum phydm_trx_mux_type rx_mode)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	if (dm->support_ic_type & ODM_IC_11N_SERIES) {
+		/*set TXmod to standby mode to remove outside noise affect*/
+		odm_set_bb_reg(dm, ODM_REG_CCK_RPT_FORMAT_11N,
+			       BIT(3) | BIT(2) | BIT(1), tx_mode);
+		/*set RXmod to standby mode to remove outside noise affect*/
+		odm_set_bb_reg(dm, ODM_REG_CCK_RPT_FORMAT_11N,
+			       BIT(22) | BIT(21) | BIT(20), rx_mode);
+		if (dm->rf_type > ODM_1T1R) {
+			/*set TXmod to standby mode to rm outside noise affect*/
+			odm_set_bb_reg(dm, ODM_REG_CCK_RPT_FORMAT_11N_B,
+				       BIT(3) | BIT(2) | BIT(1), tx_mode);
+			/*set RXmod to standby mode to rm outside noise affect*/
+			odm_set_bb_reg(dm, ODM_REG_CCK_RPT_FORMAT_11N_B,
+				       BIT(22) | BIT(21) | BIT(20), rx_mode);
+		}
+	} else if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+		/*set TXmod to standby mode to remove outside noise affect*/
+		odm_set_bb_reg(dm, ODM_REG_TRMUX_11AC,
+			       BIT(11) | BIT(10) | BIT(9) | BIT(8), tx_mode);
+		/*set RXmod to standby mode to remove outside noise affect*/
+		odm_set_bb_reg(dm, ODM_REG_TRMUX_11AC,
+			       BIT(7) | BIT(6) | BIT(5) | BIT(4), rx_mode);
+		if (dm->rf_type > ODM_1T1R) {
+			/*set TXmod to standby mode to rm outside noise affect*/
+			odm_set_bb_reg(dm, ODM_REG_TRMUX_11AC_B,
+				       BIT(11) | BIT(10) | BIT(9) | BIT(8),
+				       tx_mode);
+			/*set RXmod to standby mode to rm outside noise affect*/
+			odm_set_bb_reg(dm, ODM_REG_TRMUX_11AC_B,
+				       BIT(7) | BIT(6) | BIT(5) | BIT(4),
+				       rx_mode);
+		}
+	}
+}
+
+void phydm_mac_edcca_state(void *dm_void, enum phydm_mac_edcca_type state)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	if (state == phydm_ignore_edcca) {
+		/*ignore EDCCA	reg520[15]=1*/
+		odm_set_mac_reg(dm, REG_TX_PTCL_CTRL, BIT(15), 1);
+	} else { /*don't set MAC ignore EDCCA signal*/
+		/*don't ignore EDCCA	 reg520[15]=0*/
+		odm_set_mac_reg(dm, REG_TX_PTCL_CTRL, BIT(15), 0);
+	}
+	ODM_RT_TRACE(dm, PHYDM_COMP_ADAPTIVITY, "EDCCA enable state = %d\n",
+		     state);
+}
+
+bool phydm_cal_nhm_cnt(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u16 base = 0;
+
+	base = dm->nhm_cnt_0 + dm->nhm_cnt_1;
+
+	if (base != 0) {
+		dm->nhm_cnt_0 = ((dm->nhm_cnt_0) << 8) / base;
+		dm->nhm_cnt_1 = ((dm->nhm_cnt_1) << 8) / base;
+	}
+	if ((dm->nhm_cnt_0 - dm->nhm_cnt_1) >= 100)
+		return true; /*clean environment*/
+	else
+		return false; /*noisy environment*/
+}
+
+void phydm_check_environment(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct adaptivity_statistics *adaptivity =
+		(struct adaptivity_statistics *)phydm_get_structure(
+			dm, PHYDM_ADAPTIVITY);
+	bool is_clean_environment = false;
+
+	if (adaptivity->is_first_link) {
+		if (dm->support_ic_type &
+		    (ODM_IC_11AC_GAIN_IDX_EDCCA | ODM_IC_11N_GAIN_IDX_EDCCA))
+			dm->adaptivity_flag = false;
+		else
+			dm->adaptivity_flag = true;
+
+		adaptivity->is_first_link = false;
+		return;
+	}
+
+	if (adaptivity->nhm_wait < 3) { /*Start enter NHM after 4 nhm_wait*/
+		adaptivity->nhm_wait++;
+		phydm_nhm_counter_statistics(dm);
+		return;
+	}
+
+	phydm_nhm_counter_statistics(dm);
+	is_clean_environment = phydm_cal_nhm_cnt(dm);
+
+	if (is_clean_environment) {
+		dm->th_l2h_ini =
+			adaptivity->th_l2h_ini_backup; /*adaptivity mode*/
+		dm->th_edcca_hl_diff = adaptivity->th_edcca_hl_diff_backup;
+
+		dm->adaptivity_enable = true;
+
+		if (dm->support_ic_type &
+		    (ODM_IC_11AC_GAIN_IDX_EDCCA | ODM_IC_11N_GAIN_IDX_EDCCA))
+			dm->adaptivity_flag = false;
+		else
+			dm->adaptivity_flag = true;
+	} else {
+		if (!adaptivity->acs_for_adaptivity) {
+			dm->th_l2h_ini = dm->th_l2h_ini_mode2; /*mode2*/
+			dm->th_edcca_hl_diff = dm->th_edcca_hl_diff_mode2;
+
+			dm->adaptivity_flag = false;
+			dm->adaptivity_enable = false;
+		}
+	}
+
+	adaptivity->nhm_wait = 0;
+	adaptivity->is_first_link = true;
+	adaptivity->is_check = true;
+}
+
+void phydm_search_pwdb_lower_bound(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct adaptivity_statistics *adaptivity =
+		(struct adaptivity_statistics *)phydm_get_structure(
+			dm, PHYDM_ADAPTIVITY);
+	u32 value32 = 0, reg_value32 = 0;
+	u8 cnt, try_count = 0;
+	u8 tx_edcca1 = 0, tx_edcca0 = 0;
+	bool is_adjust = true;
+	s8 th_l2h_dmc, th_h2l_dmc, igi_target = 0x32;
+	s8 diff;
+	u8 IGI = adaptivity->igi_base + 30 + (u8)dm->th_l2h_ini -
+		 (u8)dm->th_edcca_hl_diff;
+
+	if (dm->support_ic_type & (ODM_RTL8723B | ODM_RTL8188E | ODM_RTL8192E |
+				   ODM_RTL8812 | ODM_RTL8821 | ODM_RTL8881A)) {
+		phydm_set_lna(dm, phydm_disable_lna);
+	} else {
+		phydm_set_trx_mux(dm, phydm_standby_mode, phydm_standby_mode);
+		odm_pause_dig(dm, PHYDM_PAUSE, PHYDM_PAUSE_LEVEL_0, 0x7e);
+	}
+
+	diff = igi_target - (s8)IGI;
+	th_l2h_dmc = dm->th_l2h_ini + diff;
+	if (th_l2h_dmc > 10)
+		th_l2h_dmc = 10;
+	th_h2l_dmc = th_l2h_dmc - dm->th_edcca_hl_diff;
+
+	phydm_set_edcca_threshold(dm, th_h2l_dmc, th_l2h_dmc);
+	ODM_delay_ms(30);
+
+	while (is_adjust) {
+		if (dm->support_ic_type & ODM_IC_11N_SERIES) {
+			odm_set_bb_reg(dm, ODM_REG_DBG_RPT_11N, MASKDWORD, 0x0);
+			reg_value32 =
+				odm_get_bb_reg(dm, ODM_REG_RPT_11N, MASKDWORD);
+		} else if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+			odm_set_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD,
+				       0x0);
+			reg_value32 =
+				odm_get_bb_reg(dm, ODM_REG_RPT_11AC, MASKDWORD);
+		}
+		while (reg_value32 & BIT(3) && try_count < 3) {
+			ODM_delay_ms(3);
+			try_count = try_count + 1;
+			if (dm->support_ic_type & ODM_IC_11N_SERIES)
+				reg_value32 = odm_get_bb_reg(
+					dm, ODM_REG_RPT_11N, MASKDWORD);
+			else if (dm->support_ic_type & ODM_IC_11AC_SERIES)
+				reg_value32 = odm_get_bb_reg(
+					dm, ODM_REG_RPT_11AC, MASKDWORD);
+		}
+		try_count = 0;
+
+		for (cnt = 0; cnt < 20; cnt++) {
+			if (dm->support_ic_type & ODM_IC_11N_SERIES) {
+				odm_set_bb_reg(dm, ODM_REG_DBG_RPT_11N,
+					       MASKDWORD, 0x208);
+				value32 = odm_get_bb_reg(dm, ODM_REG_RPT_11N,
+							 MASKDWORD);
+			} else if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+				odm_set_bb_reg(dm, ODM_REG_DBG_RPT_11AC,
+					       MASKDWORD, 0x209);
+				value32 = odm_get_bb_reg(dm, ODM_REG_RPT_11AC,
+							 MASKDWORD);
+			}
+			if (value32 & BIT(30) &&
+			    (dm->support_ic_type &
+			     (ODM_RTL8723B | ODM_RTL8188E)))
+				tx_edcca1 = tx_edcca1 + 1;
+			else if (value32 & BIT(29))
+				tx_edcca1 = tx_edcca1 + 1;
+			else
+				tx_edcca0 = tx_edcca0 + 1;
+		}
+
+		if (tx_edcca1 > 1) {
+			IGI = IGI - 1;
+			th_l2h_dmc = th_l2h_dmc + 1;
+			if (th_l2h_dmc > 10)
+				th_l2h_dmc = 10;
+			th_h2l_dmc = th_l2h_dmc - dm->th_edcca_hl_diff;
+
+			phydm_set_edcca_threshold(dm, th_h2l_dmc, th_l2h_dmc);
+			if (th_l2h_dmc == 10) {
+				is_adjust = false;
+				adaptivity->h2l_lb = th_h2l_dmc;
+				adaptivity->l2h_lb = th_l2h_dmc;
+				dm->adaptivity_igi_upper = IGI;
+			}
+
+			tx_edcca1 = 0;
+			tx_edcca0 = 0;
+
+		} else {
+			is_adjust = false;
+			adaptivity->h2l_lb = th_h2l_dmc;
+			adaptivity->l2h_lb = th_l2h_dmc;
+			dm->adaptivity_igi_upper = IGI;
+			tx_edcca1 = 0;
+			tx_edcca0 = 0;
+		}
+	}
+
+	dm->adaptivity_igi_upper = dm->adaptivity_igi_upper - dm->dc_backoff;
+	adaptivity->h2l_lb = adaptivity->h2l_lb + dm->dc_backoff;
+	adaptivity->l2h_lb = adaptivity->l2h_lb + dm->dc_backoff;
+
+	if (dm->support_ic_type & (ODM_RTL8723B | ODM_RTL8188E | ODM_RTL8192E |
+				   ODM_RTL8812 | ODM_RTL8821 | ODM_RTL8881A)) {
+		phydm_set_lna(dm, phydm_enable_lna);
+	} else {
+		phydm_set_trx_mux(dm, phydm_tx_mode, phydm_rx_mode);
+		odm_pause_dig(dm, PHYDM_RESUME, PHYDM_PAUSE_LEVEL_0, NONE);
+	}
+
+	phydm_set_edcca_threshold(dm, 0x7f, 0x7f); /*resume to no link state*/
+}
+
+static bool phydm_re_search_condition(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u8 adaptivity_igi_upper;
+	u8 count = 0;
+
+	adaptivity_igi_upper = dm->adaptivity_igi_upper + dm->dc_backoff;
+
+	if (adaptivity_igi_upper <= 0x26 && count < 3) {
+		count = count + 1;
+		return true;
+	}
+
+	return false;
+}
+
+void phydm_adaptivity_info_init(void *dm_void, enum phydm_adapinfo cmn_info,
+				u32 value)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct adaptivity_statistics *adaptivity =
+		(struct adaptivity_statistics *)phydm_get_structure(
+			dm, PHYDM_ADAPTIVITY);
+
+	switch (cmn_info) {
+	case PHYDM_ADAPINFO_CARRIER_SENSE_ENABLE:
+		dm->carrier_sense_enable = (bool)value;
+		break;
+
+	case PHYDM_ADAPINFO_DCBACKOFF:
+		dm->dc_backoff = (u8)value;
+		break;
+
+	case PHYDM_ADAPINFO_DYNAMICLINKADAPTIVITY:
+		adaptivity->dynamic_link_adaptivity = (bool)value;
+		break;
+
+	case PHYDM_ADAPINFO_TH_L2H_INI:
+		dm->th_l2h_ini = (s8)value;
+		break;
+
+	case PHYDM_ADAPINFO_TH_EDCCA_HL_DIFF:
+		dm->th_edcca_hl_diff = (s8)value;
+		break;
+
+	case PHYDM_ADAPINFO_AP_NUM_TH:
+		adaptivity->ap_num_th = (u8)value;
+		break;
+
+	default:
+		break;
+	}
+}
+
+void phydm_adaptivity_init(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct adaptivity_statistics *adaptivity =
+		(struct adaptivity_statistics *)phydm_get_structure(
+			dm, PHYDM_ADAPTIVITY);
+	s8 igi_target = 0x32;
+
+	if (!dm->carrier_sense_enable) {
+		if (dm->th_l2h_ini == 0)
+			dm->th_l2h_ini = 0xf5;
+	} else {
+		dm->th_l2h_ini = 0xa;
+	}
+
+	if (dm->th_edcca_hl_diff == 0)
+		dm->th_edcca_hl_diff = 7;
+	if (dm->wifi_test || dm->mp_mode) {
+		/*even no adaptivity, we still enable EDCCA, AP use mib ctrl*/
+		dm->edcca_enable = false;
+	} else {
+		dm->edcca_enable = true;
+	}
+
+	dm->adaptivity_igi_upper = 0;
+	dm->adaptivity_enable =
+		false; /*use this flag to decide enable or disable*/
+
+	dm->th_l2h_ini_mode2 = 20;
+	dm->th_edcca_hl_diff_mode2 = 8;
+	adaptivity->th_l2h_ini_backup = dm->th_l2h_ini;
+	adaptivity->th_edcca_hl_diff_backup = dm->th_edcca_hl_diff;
+
+	adaptivity->igi_base = 0x32;
+	adaptivity->igi_target = 0x1c;
+	adaptivity->h2l_lb = 0;
+	adaptivity->l2h_lb = 0;
+	adaptivity->nhm_wait = 0;
+	adaptivity->is_check = false;
+	adaptivity->is_first_link = true;
+	adaptivity->adajust_igi_level = 0;
+	adaptivity->is_stop_edcca = false;
+	adaptivity->backup_h2l = 0;
+	adaptivity->backup_l2h = 0;
+
+	phydm_mac_edcca_state(dm, phydm_dont_ignore_edcca);
+
+	/*Search pwdB lower bound*/
+	if (dm->support_ic_type & ODM_IC_11N_SERIES)
+		odm_set_bb_reg(dm, ODM_REG_DBG_RPT_11N, MASKDWORD, 0x208);
+	else if (dm->support_ic_type & ODM_IC_11AC_SERIES)
+		odm_set_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD, 0x209);
+
+	if (dm->support_ic_type & ODM_IC_11N_GAIN_IDX_EDCCA) {
+		if (dm->support_ic_type & ODM_RTL8197F) {
+			/*set to page B1*/
+			odm_set_bb_reg(dm, ODM_REG_PAGE_B1_97F, BIT(30), 0x1);
+			/*0:rx_dfir, 1: dcnf_out, 2 :rx_iq, 3: rx_nbi_nf_out*/
+			odm_set_bb_reg(dm, ODM_REG_EDCCA_DCNF_97F,
+				       BIT(27) | BIT(26), 0x1);
+			odm_set_bb_reg(dm, ODM_REG_PAGE_B1_97F, BIT(30), 0x0);
+		} else {
+			/*0:rx_dfir, 1: dcnf_out, 2 :rx_iq, 3: rx_nbi_nf_out*/
+			odm_set_bb_reg(dm, ODM_REG_EDCCA_DCNF_11N,
+				       BIT(21) | BIT(20), 0x1);
+		}
+	}
+	/*8814a no need to find pwdB lower bound, maybe*/
+	if (dm->support_ic_type & ODM_IC_11AC_GAIN_IDX_EDCCA) {
+		/*0:rx_dfir, 1: dcnf_out, 2 :rx_iq, 3: rx_nbi_nf_out*/
+		odm_set_bb_reg(dm, ODM_REG_ACBB_EDCCA_ENHANCE,
+			       BIT(29) | BIT(28), 0x1);
+	}
+
+	if (!(dm->support_ic_type &
+	      (ODM_IC_11AC_GAIN_IDX_EDCCA | ODM_IC_11N_GAIN_IDX_EDCCA))) {
+		phydm_search_pwdb_lower_bound(dm);
+		if (phydm_re_search_condition(dm))
+			phydm_search_pwdb_lower_bound(dm);
+	}
+
+	/*we need to consider PwdB upper bound for 8814 later IC*/
+	adaptivity->adajust_igi_level =
+		(u8)((dm->th_l2h_ini + igi_target) - pwdb_upper_bound +
+		     dfir_loss); /*IGI = L2H - PwdB - dfir_loss*/
+
+	ODM_RT_TRACE(
+		dm, PHYDM_COMP_ADAPTIVITY,
+		"th_l2h_ini = 0x%x, th_edcca_hl_diff = 0x%x, adaptivity->adajust_igi_level = 0x%x\n",
+		dm->th_l2h_ini, dm->th_edcca_hl_diff,
+		adaptivity->adajust_igi_level);
+
+	/*Check this later on Windows*/
+	/*phydm_set_edcca_threshold_api(dm, dig_tab->cur_ig_value);*/
+}
+
+void phydm_adaptivity(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct dig_thres *dig_tab = &dm->dm_dig_table;
+	u8 IGI = dig_tab->cur_ig_value;
+	s8 th_l2h_dmc, th_h2l_dmc;
+	s8 diff = 0, igi_target;
+	struct adaptivity_statistics *adaptivity =
+		(struct adaptivity_statistics *)phydm_get_structure(
+			dm, PHYDM_ADAPTIVITY);
+
+	if (!dm->edcca_enable || adaptivity->is_stop_edcca) {
+		ODM_RT_TRACE(dm, PHYDM_COMP_ADAPTIVITY, "Disable EDCCA!!!\n");
+		return;
+	}
+
+	if (!(dm->support_ability & ODM_BB_ADAPTIVITY)) {
+		ODM_RT_TRACE(dm, PHYDM_COMP_ADAPTIVITY,
+			     "adaptivity disable, enable EDCCA mode!!!\n");
+		dm->th_l2h_ini = dm->th_l2h_ini_mode2;
+		dm->th_edcca_hl_diff = dm->th_edcca_hl_diff_mode2;
+	}
+
+	ODM_RT_TRACE(dm, PHYDM_COMP_ADAPTIVITY, "%s() =====>\n", __func__);
+	ODM_RT_TRACE(dm, PHYDM_COMP_ADAPTIVITY,
+		     "igi_base=0x%x, th_l2h_ini = %d, th_edcca_hl_diff = %d\n",
+		     adaptivity->igi_base, dm->th_l2h_ini,
+		     dm->th_edcca_hl_diff);
+	if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+		/*fix AC series when enable EDCCA hang issue*/
+		odm_set_bb_reg(dm, 0x800, BIT(10), 1); /*ADC_mask disable*/
+		odm_set_bb_reg(dm, 0x800, BIT(10), 0); /*ADC_mask enable*/
+	}
+	if (*dm->band_width == ODM_BW20M) /*CHANNEL_WIDTH_20*/
+		igi_target = adaptivity->igi_base;
+	else if (*dm->band_width == ODM_BW40M)
+		igi_target = adaptivity->igi_base + 2;
+	else if (*dm->band_width == ODM_BW80M)
+		igi_target = adaptivity->igi_base + 2;
+	else
+		igi_target = adaptivity->igi_base;
+	adaptivity->igi_target = (u8)igi_target;
+
+	ODM_RT_TRACE(
+		dm, PHYDM_COMP_ADAPTIVITY,
+		"band_width=%s, igi_target=0x%x, dynamic_link_adaptivity = %d, acs_for_adaptivity = %d\n",
+		(*dm->band_width == ODM_BW80M) ?
+			"80M" :
+			((*dm->band_width == ODM_BW40M) ? "40M" : "20M"),
+		igi_target, adaptivity->dynamic_link_adaptivity,
+		adaptivity->acs_for_adaptivity);
+	ODM_RT_TRACE(
+		dm, PHYDM_COMP_ADAPTIVITY,
+		"rssi_min = %d, adaptivity->adajust_igi_level= 0x%x, adaptivity_flag = %d, adaptivity_enable = %d\n",
+		dm->rssi_min, adaptivity->adajust_igi_level,
+		dm->adaptivity_flag, dm->adaptivity_enable);
+
+	if (adaptivity->dynamic_link_adaptivity && (!dm->is_linked) &&
+	    !dm->adaptivity_enable) {
+		phydm_set_edcca_threshold(dm, 0x7f, 0x7f);
+		ODM_RT_TRACE(
+			dm, PHYDM_COMP_ADAPTIVITY,
+			"In DynamicLink mode(noisy) and No link, Turn off EDCCA!!\n");
+		return;
+	}
+
+	if (dm->support_ic_type &
+	    (ODM_IC_11AC_GAIN_IDX_EDCCA | ODM_IC_11N_GAIN_IDX_EDCCA)) {
+		if ((adaptivity->adajust_igi_level > IGI) &&
+		    dm->adaptivity_enable)
+			diff = adaptivity->adajust_igi_level - IGI;
+
+		th_l2h_dmc = dm->th_l2h_ini - diff + igi_target;
+		th_h2l_dmc = th_l2h_dmc - dm->th_edcca_hl_diff;
+	} else {
+		diff = igi_target - (s8)IGI;
+		th_l2h_dmc = dm->th_l2h_ini + diff;
+		if (th_l2h_dmc > 10 && dm->adaptivity_enable)
+			th_l2h_dmc = 10;
+
+		th_h2l_dmc = th_l2h_dmc - dm->th_edcca_hl_diff;
+
+		/*replace lower bound to prevent EDCCA always equal 1*/
+		if (th_h2l_dmc < adaptivity->h2l_lb)
+			th_h2l_dmc = adaptivity->h2l_lb;
+		if (th_l2h_dmc < adaptivity->l2h_lb)
+			th_l2h_dmc = adaptivity->l2h_lb;
+	}
+	ODM_RT_TRACE(dm, PHYDM_COMP_ADAPTIVITY,
+		     "IGI=0x%x, th_l2h_dmc = %d, th_h2l_dmc = %d\n", IGI,
+		     th_l2h_dmc, th_h2l_dmc);
+	ODM_RT_TRACE(
+		dm, PHYDM_COMP_ADAPTIVITY,
+		"adaptivity_igi_upper=0x%x, h2l_lb = 0x%x, l2h_lb = 0x%x\n",
+		dm->adaptivity_igi_upper, adaptivity->h2l_lb,
+		adaptivity->l2h_lb);
+
+	phydm_set_edcca_threshold(dm, th_h2l_dmc, th_l2h_dmc);
+
+	if (dm->adaptivity_enable)
+		odm_set_mac_reg(dm, REG_RD_CTRL, BIT(11), 1);
+}
+
+/*This is for solving USB can't Tx problem due to USB3.0 interference in 2.4G*/
+void phydm_pause_edcca(void *dm_void, bool is_pasue_edcca)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct adaptivity_statistics *adaptivity =
+		(struct adaptivity_statistics *)phydm_get_structure(
+			dm, PHYDM_ADAPTIVITY);
+	struct dig_thres *dig_tab = &dm->dm_dig_table;
+	u8 IGI = dig_tab->cur_ig_value;
+	s8 diff = 0;
+
+	if (is_pasue_edcca) {
+		adaptivity->is_stop_edcca = true;
+
+		if (dm->support_ic_type &
+		    (ODM_IC_11AC_GAIN_IDX_EDCCA | ODM_IC_11N_GAIN_IDX_EDCCA)) {
+			if (adaptivity->adajust_igi_level > IGI)
+				diff = adaptivity->adajust_igi_level - IGI;
+
+			adaptivity->backup_l2h =
+				dm->th_l2h_ini - diff + adaptivity->igi_target;
+			adaptivity->backup_h2l =
+				adaptivity->backup_l2h - dm->th_edcca_hl_diff;
+		} else {
+			diff = adaptivity->igi_target - (s8)IGI;
+			adaptivity->backup_l2h = dm->th_l2h_ini + diff;
+			if (adaptivity->backup_l2h > 10)
+				adaptivity->backup_l2h = 10;
+
+			adaptivity->backup_h2l =
+				adaptivity->backup_l2h - dm->th_edcca_hl_diff;
+
+			/*replace lower bound to prevent EDCCA always equal 1*/
+			if (adaptivity->backup_h2l < adaptivity->h2l_lb)
+				adaptivity->backup_h2l = adaptivity->h2l_lb;
+			if (adaptivity->backup_l2h < adaptivity->l2h_lb)
+				adaptivity->backup_l2h = adaptivity->l2h_lb;
+		}
+		ODM_RT_TRACE(
+			dm, PHYDM_COMP_ADAPTIVITY,
+			"pauseEDCCA : L2Hbak = 0x%x, H2Lbak = 0x%x, IGI = 0x%x\n",
+			adaptivity->backup_l2h, adaptivity->backup_h2l, IGI);
+
+		/*Disable EDCCA*/
+		phydm_pause_edcca_work_item_callback(dm);
+
+	} else {
+		adaptivity->is_stop_edcca = false;
+		ODM_RT_TRACE(
+			dm, PHYDM_COMP_ADAPTIVITY,
+			"resumeEDCCA : L2Hbak = 0x%x, H2Lbak = 0x%x, IGI = 0x%x\n",
+			adaptivity->backup_l2h, adaptivity->backup_h2l, IGI);
+		/*Resume EDCCA*/
+		phydm_resume_edcca_work_item_callback(dm);
+	}
+}
+
+void phydm_pause_edcca_work_item_callback(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	if (dm->support_ic_type & ODM_IC_11N_SERIES)
+		odm_set_bb_reg(dm, REG_OFDM_0_ECCA_THRESHOLD,
+			       MASKBYTE2 | MASKBYTE0, (u32)(0x7f | 0x7f << 16));
+	else if (dm->support_ic_type & ODM_IC_11AC_SERIES)
+		odm_set_bb_reg(dm, REG_FPGA0_XB_LSSI_READ_BACK, MASKLWORD,
+			       (u16)(0x7f | 0x7f << 8));
+}
+
+void phydm_resume_edcca_work_item_callback(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct adaptivity_statistics *adaptivity =
+		(struct adaptivity_statistics *)phydm_get_structure(
+			dm, PHYDM_ADAPTIVITY);
+
+	if (dm->support_ic_type & ODM_IC_11N_SERIES)
+		odm_set_bb_reg(dm, REG_OFDM_0_ECCA_THRESHOLD,
+			       MASKBYTE2 | MASKBYTE0,
+			       (u32)((u8)adaptivity->backup_l2h |
+				     (u8)adaptivity->backup_h2l << 16));
+	else if (dm->support_ic_type & ODM_IC_11AC_SERIES)
+		odm_set_bb_reg(dm, REG_FPGA0_XB_LSSI_READ_BACK, MASKLWORD,
+			       (u16)((u8)adaptivity->backup_l2h |
+				     (u8)adaptivity->backup_h2l << 8));
+}
+
+void phydm_set_edcca_threshold_api(void *dm_void, u8 IGI)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct adaptivity_statistics *adaptivity =
+		(struct adaptivity_statistics *)phydm_get_structure(
+			dm, PHYDM_ADAPTIVITY);
+	s8 th_l2h_dmc, th_h2l_dmc;
+	s8 diff = 0, igi_target = 0x32;
+
+	if (dm->support_ability & ODM_BB_ADAPTIVITY) {
+		if (dm->support_ic_type &
+		    (ODM_IC_11AC_GAIN_IDX_EDCCA | ODM_IC_11N_GAIN_IDX_EDCCA)) {
+			if (adaptivity->adajust_igi_level > IGI)
+				diff = adaptivity->adajust_igi_level - IGI;
+
+			th_l2h_dmc = dm->th_l2h_ini - diff + igi_target;
+			th_h2l_dmc = th_l2h_dmc - dm->th_edcca_hl_diff;
+		} else {
+			diff = igi_target - (s8)IGI;
+			th_l2h_dmc = dm->th_l2h_ini + diff;
+			if (th_l2h_dmc > 10)
+				th_l2h_dmc = 10;
+
+			th_h2l_dmc = th_l2h_dmc - dm->th_edcca_hl_diff;
+
+			/*replace lower bound to prevent EDCCA always equal 1*/
+			if (th_h2l_dmc < adaptivity->h2l_lb)
+				th_h2l_dmc = adaptivity->h2l_lb;
+			if (th_l2h_dmc < adaptivity->l2h_lb)
+				th_l2h_dmc = adaptivity->l2h_lb;
+		}
+		ODM_RT_TRACE(
+			dm, PHYDM_COMP_ADAPTIVITY,
+			"API :IGI=0x%x, th_l2h_dmc = %d, th_h2l_dmc = %d\n",
+			IGI, th_l2h_dmc, th_h2l_dmc);
+		ODM_RT_TRACE(
+			dm, PHYDM_COMP_ADAPTIVITY,
+			"API :adaptivity_igi_upper=0x%x, h2l_lb = 0x%x, l2h_lb = 0x%x\n",
+			dm->adaptivity_igi_upper, adaptivity->h2l_lb,
+			adaptivity->l2h_lb);
+
+		phydm_set_edcca_threshold(dm, th_h2l_dmc, th_l2h_dmc);
+	}
+}
diff --git a/drivers/staging/rtlwifi/phydm/phydm_adaptivity.h b/drivers/staging/rtlwifi/phydm/phydm_adaptivity.h
new file mode 100644
index 000000000000..fdb39b4f9df2
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_adaptivity.h
@@ -0,0 +1,119 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __PHYDMADAPTIVITY_H__
+#define __PHYDMADAPTIVITY_H__
+
+/*20160902 changed by Kevin, refine method for searching pwdb lower bound*/
+#define ADAPTIVITY_VERSION "9.3.5"
+
+#define pwdb_upper_bound 7
+#define dfir_loss 5
+
+enum phydm_adapinfo {
+	PHYDM_ADAPINFO_CARRIER_SENSE_ENABLE = 0,
+	PHYDM_ADAPINFO_DCBACKOFF,
+	PHYDM_ADAPINFO_DYNAMICLINKADAPTIVITY,
+	PHYDM_ADAPINFO_TH_L2H_INI,
+	PHYDM_ADAPINFO_TH_EDCCA_HL_DIFF,
+	PHYDM_ADAPINFO_AP_NUM_TH
+
+};
+
+enum phydm_set_lna {
+	phydm_disable_lna = 0,
+	phydm_enable_lna = 1,
+};
+
+enum phydm_trx_mux_type {
+	phydm_shutdown = 0,
+	phydm_standby_mode = 1,
+	phydm_tx_mode = 2,
+	phydm_rx_mode = 3
+};
+
+enum phydm_mac_edcca_type {
+	phydm_ignore_edcca = 0,
+	phydm_dont_ignore_edcca = 1
+};
+
+struct adaptivity_statistics {
+	s8 th_l2h_ini_backup;
+	s8 th_edcca_hl_diff_backup;
+	s8 igi_base;
+	u8 igi_target;
+	u8 nhm_wait;
+	s8 h2l_lb;
+	s8 l2h_lb;
+	bool is_first_link;
+	bool is_check;
+	bool dynamic_link_adaptivity;
+	u8 ap_num_th;
+	u8 adajust_igi_level;
+	bool acs_for_adaptivity;
+	s8 backup_l2h;
+	s8 backup_h2l;
+	bool is_stop_edcca;
+};
+
+void phydm_pause_edcca(void *dm_void, bool is_pasue_edcca);
+
+void phydm_check_adaptivity(void *dm_void);
+
+void phydm_check_environment(void *dm_void);
+
+void phydm_nhm_counter_statistics_init(void *dm_void);
+
+void phydm_nhm_counter_statistics(void *dm_void);
+
+void phydm_nhm_counter_statistics_reset(void *dm_void);
+
+void phydm_get_nhm_counter_statistics(void *dm_void);
+
+void phydm_mac_edcca_state(void *dm_void, enum phydm_mac_edcca_type state);
+
+void phydm_set_edcca_threshold(void *dm_void, s8 H2L, s8 L2H);
+
+void phydm_set_trx_mux(void *dm_void, enum phydm_trx_mux_type tx_mode,
+		       enum phydm_trx_mux_type rx_mode);
+
+bool phydm_cal_nhm_cnt(void *dm_void);
+
+void phydm_search_pwdb_lower_bound(void *dm_void);
+
+void phydm_adaptivity_info_init(void *dm_void, enum phydm_adapinfo cmn_info,
+				u32 value);
+
+void phydm_adaptivity_init(void *dm_void);
+
+void phydm_adaptivity(void *dm_void);
+
+void phydm_set_edcca_threshold_api(void *dm_void, u8 IGI);
+
+void phydm_pause_edcca_work_item_callback(void *dm_void);
+
+void phydm_resume_edcca_work_item_callback(void *dm_void);
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/phydm_adc_sampling.c b/drivers/staging/rtlwifi/phydm/phydm_adc_sampling.c
new file mode 100644
index 000000000000..158dd5d05de4
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_adc_sampling.c
@@ -0,0 +1,628 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#include "mp_precomp.h"
+#include "phydm_precomp.h"
+
+static bool phydm_la_buffer_allocate(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct rt_adcsmp *adc_smp = &dm->adcsmp;
+	struct rt_adcsmp_string *adc_smp_buf = &adc_smp->adc_smp_buf;
+	bool ret = false;
+
+	ODM_RT_TRACE(dm, ODM_COMP_UNCOND, "[LA mode BufferAllocate]\n");
+
+	if (adc_smp_buf->length == 0) {
+		odm_allocate_memory(dm, (void **)&adc_smp_buf->octet,
+				    adc_smp_buf->buffer_size);
+		if (!adc_smp_buf->octet) {
+			ret = false;
+		} else {
+			adc_smp_buf->length = adc_smp_buf->buffer_size;
+			ret = true;
+		}
+	}
+
+	return ret;
+}
+
+static void phydm_la_get_tx_pkt_buf(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct rt_adcsmp *adc_smp = &dm->adcsmp;
+	struct rt_adcsmp_string *adc_smp_buf = &adc_smp->adc_smp_buf;
+	u32 i = 0, value32, data_l = 0, data_h = 0;
+	u32 addr, finish_addr;
+	u32 end_addr = (adc_smp_buf->start_pos + adc_smp_buf->buffer_size) -
+		       1; /*end_addr = 0x3ffff;*/
+	bool is_round_up;
+	static u32 page = 0xFF;
+	u32 smp_cnt = 0, smp_number = 0, addr_8byte = 0;
+
+	odm_memory_set(dm, adc_smp_buf->octet, 0, adc_smp_buf->length);
+	odm_write_1byte(dm, 0x0106, 0x69);
+
+	ODM_RT_TRACE(dm, ODM_COMP_UNCOND, "GetTxPktBuf\n");
+
+	value32 = odm_read_4byte(dm, 0x7c0);
+	is_round_up = (bool)((value32 & BIT(31)) >> 31);
+	/*Reg7C0[30:16]: finish addr (unit: 8byte)*/
+	finish_addr = (value32 & 0x7FFF0000) >> 16;
+
+	if (is_round_up) {
+		addr = (finish_addr + 1) << 3;
+		ODM_RT_TRACE(
+			dm, ODM_COMP_UNCOND,
+			"is_round_up = ((%d)), finish_addr=((0x%x)), 0x7c0=((0x%x))\n",
+			is_round_up, finish_addr, value32);
+		/*Byte to 64Byte*/
+		smp_number = ((adc_smp_buf->buffer_size) >> 3);
+	} else {
+		addr = adc_smp_buf->start_pos;
+
+		addr_8byte = addr >> 3;
+		if (addr_8byte > finish_addr)
+			smp_number = addr_8byte - finish_addr;
+		else
+			smp_number = finish_addr - addr_8byte;
+
+		ODM_RT_TRACE(
+			dm, ODM_COMP_UNCOND,
+			"is_round_up = ((%d)), finish_addr=((0x%x * 8Byte)), Start_Addr = ((0x%x * 8Byte)), smp_number = ((%d))\n",
+			is_round_up, finish_addr, addr_8byte, smp_number);
+	}
+
+	if (dm->support_ic_type & ODM_RTL8197F) {
+		/*64K byte*/
+		for (addr = 0x0, i = 0; addr < end_addr; addr += 8, i += 2) {
+			if ((addr & 0xfff) == 0)
+				odm_set_bb_reg(dm, 0x0140, MASKLWORD,
+					       0x780 + (addr >> 12));
+			data_l = odm_get_bb_reg(dm, 0x8000 + (addr & 0xfff),
+						MASKDWORD);
+			data_h = odm_get_bb_reg(dm, 0x8000 + (addr & 0xfff) + 4,
+						MASKDWORD);
+
+			ODM_RT_TRACE(dm, ODM_COMP_UNCOND, "%08x%08x\n", data_h,
+				     data_l);
+		}
+	} else {
+		while (addr != (finish_addr << 3)) {
+			if (page != (addr >> 12)) {
+				/*Reg140=0x780+(addr>>12),
+				 *addr=0x30~0x3F, total 16 pages
+				 */
+				page = (addr >> 12);
+			}
+			odm_set_bb_reg(dm, 0x0140, MASKLWORD, 0x780 + page);
+
+			/*pDataL = 0x8000+(addr&0xfff);*/
+			data_l = odm_get_bb_reg(dm, 0x8000 + (addr & 0xfff),
+						MASKDWORD);
+			data_h = odm_get_bb_reg(dm, 0x8000 + (addr & 0xfff) + 4,
+						MASKDWORD);
+
+			adc_smp_buf->octet[i] = data_h;
+			adc_smp_buf->octet[i + 1] = data_l;
+
+			ODM_RT_TRACE(dm, ODM_COMP_UNCOND, "%08x%08x\n", data_h,
+				     data_l);
+
+			i = i + 2;
+
+			if ((addr + 8) >= end_addr)
+				addr = adc_smp_buf->start_pos;
+			else
+				addr = addr + 8;
+
+			smp_cnt++;
+			if (smp_cnt >= (smp_number - 1))
+				break;
+		}
+		ODM_RT_TRACE(dm, ODM_COMP_UNCOND, "smp_cnt = ((%d))\n",
+			     smp_cnt);
+	}
+}
+
+static void phydm_la_mode_set_mac_iq_dump(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct rt_adcsmp *adc_smp = &dm->adcsmp;
+	u32 reg_value;
+
+	odm_write_1byte(dm, 0x7c0, 0); /*clear all 0x7c0*/
+	odm_set_mac_reg(dm, 0x7c0, BIT(0), 1); /*Enable LA mode HW block*/
+
+	if (adc_smp->la_trig_mode == PHYDM_MAC_TRIG) {
+		adc_smp->is_bb_trigger = 0;
+		odm_set_mac_reg(dm, 0x7c0, BIT(2),
+				1); /*polling bit for MAC mode*/
+		odm_set_mac_reg(
+			dm, 0x7c0, BIT(4) | BIT(3),
+			adc_smp->la_trigger_edge); /*trigger mode for MAC*/
+
+		ODM_RT_TRACE(
+			dm, ODM_COMP_UNCOND,
+			"[MAC_trig] ref_mask = ((0x%x)), ref_value = ((0x%x)), dbg_port = ((0x%x))\n",
+			adc_smp->la_mac_ref_mask, adc_smp->la_trig_sig_sel,
+			adc_smp->la_dbg_port);
+		/*[Set MAC Debug Port]*/
+		odm_set_mac_reg(dm, 0xF4, BIT(16), 1);
+		odm_set_mac_reg(dm, 0x38, 0xff0000, adc_smp->la_dbg_port);
+		odm_set_mac_reg(dm, 0x7c4, MASKDWORD, adc_smp->la_mac_ref_mask);
+		odm_set_mac_reg(dm, 0x7c8, MASKDWORD, adc_smp->la_trig_sig_sel);
+
+	} else {
+		adc_smp->is_bb_trigger = 1;
+		odm_set_mac_reg(dm, 0x7c0, BIT(1),
+				1); /*polling bit for BB ADC mode*/
+
+		if (adc_smp->la_trig_mode == PHYDM_ADC_MAC_TRIG) {
+			odm_set_mac_reg(
+				dm, 0x7c0, BIT(3),
+				1); /*polling bit for MAC trigger event*/
+			odm_set_mac_reg(dm, 0x7c0, BIT(7) | BIT(6),
+					adc_smp->la_trig_sig_sel);
+
+			if (adc_smp->la_trig_sig_sel == ADCSMP_TRIG_REG)
+				odm_set_mac_reg(
+					dm, 0x7c0, BIT(5),
+					1); /* manual trigger 0x7C0[5] = 0->1*/
+		}
+	}
+
+	reg_value = odm_get_bb_reg(dm, 0x7c0, 0xff);
+	ODM_RT_TRACE(dm, ODM_COMP_UNCOND,
+		     "4. [Set MAC IQ dump] 0x7c0[7:0] = ((0x%x))\n", reg_value);
+}
+
+static void phydm_la_mode_set_dma_type(void *dm_void, u8 la_dma_type)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	ODM_RT_TRACE(dm, ODM_COMP_UNCOND,
+		     "2. [LA mode DMA setting] Dma_type = ((%d))\n",
+		     la_dma_type);
+
+	if (dm->support_ic_type & ODM_N_ANTDIV_SUPPORT)
+		odm_set_bb_reg(dm, 0x9a0, 0xf00, la_dma_type); /*0x9A0[11:8]*/
+	else
+		odm_set_bb_reg(dm, odm_adc_trigger_jaguar2, 0xf00,
+			       la_dma_type); /*0x95C[11:8]*/
+}
+
+static void phydm_adc_smp_start(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct rt_adcsmp *adc_smp = &dm->adcsmp;
+	u8 tmp_u1b;
+	u8 while_cnt = 0;
+	u8 polling_ok = false, target_polling_bit;
+
+	phydm_la_mode_bb_setting(dm);
+	phydm_la_mode_set_dma_type(dm, adc_smp->la_dma_type);
+	phydm_la_mode_set_trigger_time(dm, adc_smp->la_trigger_time);
+
+	if (dm->support_ic_type & ODM_RTL8197F) {
+		odm_set_bb_reg(dm, 0xd00, BIT(26), 0x1);
+	} else { /*for 8814A and 8822B?*/
+		odm_write_1byte(dm, 0x198c, 0x7);
+		odm_write_1byte(dm, 0x8b4, 0x80);
+		/* odm_set_bb_reg(dm, 0x8b4, BIT(7), 1); */
+	}
+
+	phydm_la_mode_set_mac_iq_dump(dm);
+	/* return; */
+
+	target_polling_bit = (adc_smp->is_bb_trigger) ? BIT(1) : BIT(2);
+	do { /*Poll time always use 100ms, when it exceed 2s, break while loop*/
+		tmp_u1b = odm_read_1byte(dm, 0x7c0);
+
+		if (adc_smp->adc_smp_state != ADCSMP_STATE_SET) {
+			ODM_RT_TRACE(
+				dm, ODM_COMP_UNCOND,
+				"[state Error] adc_smp_state != ADCSMP_STATE_SET\n");
+			break;
+
+		} else if (tmp_u1b & target_polling_bit) {
+			ODM_delay_ms(100);
+			while_cnt = while_cnt + 1;
+			continue;
+		} else {
+			ODM_RT_TRACE(dm, ODM_COMP_UNCOND,
+				     "[LA Query OK] polling_bit=((0x%x))\n",
+				     target_polling_bit);
+			polling_ok = true;
+			if (dm->support_ic_type & ODM_RTL8197F)
+				odm_set_bb_reg(dm, 0x7c0, BIT(0), 0x0);
+			break;
+		}
+	} while (while_cnt < 20);
+
+	if (adc_smp->adc_smp_state == ADCSMP_STATE_SET) {
+		if (polling_ok)
+			phydm_la_get_tx_pkt_buf(dm);
+		else
+			ODM_RT_TRACE(dm, ODM_COMP_UNCOND,
+				     "[Polling timeout]\n");
+	}
+
+	if (adc_smp->adc_smp_state == ADCSMP_STATE_SET)
+		adc_smp->adc_smp_state = ADCSMP_STATE_QUERY;
+
+	ODM_RT_TRACE(dm, ODM_COMP_UNCOND,
+		     "[LA mode] LA_pattern_count = ((%d))\n",
+		     adc_smp->la_count);
+
+	adc_smp_stop(dm);
+
+	if (adc_smp->la_count == 0) {
+		ODM_RT_TRACE(dm, ODM_COMP_UNCOND,
+			     "LA Dump finished ---------->\n\n\n");
+		/**/
+	} else {
+		adc_smp->la_count--;
+		ODM_RT_TRACE(dm, ODM_COMP_UNCOND,
+			     "LA Dump more ---------->\n\n\n");
+		adc_smp_set(dm, adc_smp->la_trig_mode, adc_smp->la_trig_sig_sel,
+			    adc_smp->la_dma_type, adc_smp->la_trigger_time, 0);
+	}
+}
+
+void adc_smp_set(void *dm_void, u8 trig_mode, u32 trig_sig_sel,
+		 u8 dma_data_sig_sel, u32 trigger_time, u16 polling_time)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	bool is_set_success = true;
+	struct rt_adcsmp *adc_smp = &dm->adcsmp;
+
+	adc_smp->la_trig_mode = trig_mode;
+	adc_smp->la_trig_sig_sel = trig_sig_sel;
+	adc_smp->la_dma_type = dma_data_sig_sel;
+	adc_smp->la_trigger_time = trigger_time;
+
+	if (adc_smp->adc_smp_state != ADCSMP_STATE_IDLE)
+		is_set_success = false;
+	else if (adc_smp->adc_smp_buf.length == 0)
+		is_set_success = phydm_la_buffer_allocate(dm);
+
+	if (is_set_success) {
+		adc_smp->adc_smp_state = ADCSMP_STATE_SET;
+
+		ODM_RT_TRACE(dm, ODM_COMP_UNCOND,
+			     "[LA Set Success] LA_State=((%d))\n",
+			     adc_smp->adc_smp_state);
+
+		phydm_adc_smp_start(dm);
+	} else {
+		ODM_RT_TRACE(dm, ODM_COMP_UNCOND,
+			     "[LA Set Fail] LA_State=((%d))\n",
+			     adc_smp->adc_smp_state);
+	}
+}
+
+void adc_smp_query(void *dm_void, void *output, u32 out_len, u32 *pused)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct rt_adcsmp *adc_smp = &dm->adcsmp;
+	struct rt_adcsmp_string *adc_smp_buf = &adc_smp->adc_smp_buf;
+	u32 used = *pused;
+	u32 i;
+
+	ODM_RT_TRACE(dm, ODM_COMP_UNCOND, "%s adc_smp_state %d", __func__,
+		     adc_smp->adc_smp_state);
+
+	for (i = 0; i < (adc_smp_buf->length >> 2) - 2; i += 2) {
+		PHYDM_SNPRINTF(output + used, out_len - used, "%08x%08x\n",
+			       adc_smp_buf->octet[i],
+			       adc_smp_buf->octet[i + 1]);
+	}
+
+	PHYDM_SNPRINTF(output + used, out_len - used, "\n");
+	*pused = used;
+}
+
+s32 adc_smp_get_sample_counts(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct rt_adcsmp *adc_smp = &dm->adcsmp;
+	struct rt_adcsmp_string *adc_smp_buf = &adc_smp->adc_smp_buf;
+
+	return (adc_smp_buf->length >> 2) - 2;
+}
+
+s32 adc_smp_query_single_data(void *dm_void, void *output, u32 out_len,
+			      u32 index)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct rt_adcsmp *adc_smp = &dm->adcsmp;
+	struct rt_adcsmp_string *adc_smp_buf = &adc_smp->adc_smp_buf;
+	u32 used = 0;
+
+	if (adc_smp->adc_smp_state != ADCSMP_STATE_QUERY) {
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "Error: la data is not ready yet ...\n");
+		return -1;
+	}
+
+	if (index < ((adc_smp_buf->length >> 2) - 2)) {
+		PHYDM_SNPRINTF(output + used, out_len - used, "%08x%08x\n",
+			       adc_smp_buf->octet[index],
+			       adc_smp_buf->octet[index + 1]);
+	}
+	return 0;
+}
+
+void adc_smp_stop(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct rt_adcsmp *adc_smp = &dm->adcsmp;
+
+	adc_smp->adc_smp_state = ADCSMP_STATE_IDLE;
+	ODM_RT_TRACE(dm, ODM_COMP_UNCOND, "[LA_Stop] LA_state = ((%d))\n",
+		     adc_smp->adc_smp_state);
+}
+
+void adc_smp_init(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct rt_adcsmp *adc_smp = &dm->adcsmp;
+	struct rt_adcsmp_string *adc_smp_buf = &adc_smp->adc_smp_buf;
+
+	adc_smp->adc_smp_state = ADCSMP_STATE_IDLE;
+
+	if (dm->support_ic_type & ODM_RTL8814A) {
+		adc_smp_buf->start_pos = 0x30000;
+		adc_smp_buf->buffer_size = 0x10000;
+	} else if (dm->support_ic_type & ODM_RTL8822B) {
+		adc_smp_buf->start_pos = 0x20000;
+		adc_smp_buf->buffer_size = 0x20000;
+	} else if (dm->support_ic_type & ODM_RTL8197F) {
+		adc_smp_buf->start_pos = 0x00000;
+		adc_smp_buf->buffer_size = 0x10000;
+	} else if (dm->support_ic_type & ODM_RTL8821C) {
+		adc_smp_buf->start_pos = 0x8000;
+		adc_smp_buf->buffer_size = 0x8000;
+	}
+}
+
+void adc_smp_de_init(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct rt_adcsmp *adc_smp = &dm->adcsmp;
+	struct rt_adcsmp_string *adc_smp_buf = &adc_smp->adc_smp_buf;
+
+	adc_smp_stop(dm);
+
+	if (adc_smp_buf->length != 0x0) {
+		odm_free_memory(dm, adc_smp_buf->octet, adc_smp_buf->length);
+		adc_smp_buf->length = 0x0;
+	}
+}
+
+void phydm_la_mode_bb_setting(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct rt_adcsmp *adc_smp = &dm->adcsmp;
+
+	u8 trig_mode = adc_smp->la_trig_mode;
+	u32 trig_sig_sel = adc_smp->la_trig_sig_sel;
+	u32 dbg_port = adc_smp->la_dbg_port;
+	u8 is_trigger_edge = adc_smp->la_trigger_edge;
+	u8 sampling_rate = adc_smp->la_smp_rate;
+
+	ODM_RT_TRACE(
+		dm, ODM_COMP_UNCOND,
+		"1. [LA mode bb_setting] trig_mode = ((%d)), dbg_port = ((0x%x)), Trig_Edge = ((%d)), smp_rate = ((%d)), Trig_Sel = ((0x%x))\n",
+		trig_mode, dbg_port, is_trigger_edge, sampling_rate,
+		trig_sig_sel);
+
+	if (trig_mode == PHYDM_MAC_TRIG)
+		trig_sig_sel = 0; /*ignore this setting*/
+
+	if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+		if (trig_mode == PHYDM_ADC_RF0_TRIG) {
+			/*DBGOUT_RFC_a[31:0]*/
+			odm_set_bb_reg(dm, 0x8f8,
+				       BIT(25) | BIT(24) | BIT(23) | BIT(22),
+				       9);
+		} else if (trig_mode == PHYDM_ADC_RF1_TRIG) {
+			/*DBGOUT_RFC_b[31:0]*/
+			odm_set_bb_reg(dm, 0x8f8,
+				       BIT(25) | BIT(24) | BIT(23) | BIT(22),
+				       8);
+		} else {
+			odm_set_bb_reg(dm, 0x8f8,
+				       BIT(25) | BIT(24) | BIT(23) | BIT(22),
+				       0);
+		}
+		/*
+		 *	(0:) '{ofdm_dbg[31:0]}'
+		 *	(1:) '{cca,crc32_fail,dbg_ofdm[29:0]}'
+		 *	(2:) '{vbon,crc32_fail,dbg_ofdm[29:0]}'
+		 *	(3:) '{cca,crc32_ok,dbg_ofdm[29:0]}'
+		 *	(4:) '{vbon,crc32_ok,dbg_ofdm[29:0]}'
+		 *	(5:) '{dbg_iqk_anta}'
+		 *	(6:) '{cca,ofdm_crc_ok,dbg_dp_anta[29:0]}'
+		 *	(7:) '{dbg_iqk_antb}'
+		 *	(8:) '{DBGOUT_RFC_b[31:0]}'
+		 *	(9:) '{DBGOUT_RFC_a[31:0]}'
+		 *	(a:) '{dbg_ofdm}'
+		 *	(b:) '{dbg_cck}'
+		 */
+
+		/*disable dbg clk gating*/
+		odm_set_bb_reg(dm, 0x198C, BIT(2) | BIT(1) | BIT(0), 7);
+
+		/*0x95C[4:0], BB debug port bit*/
+		odm_set_bb_reg(dm, 0x95C, 0x1f, trig_sig_sel);
+		odm_set_bb_reg(dm, 0x8FC, MASKDWORD, dbg_port);
+		/*0: posedge, 1: negedge*/
+		odm_set_bb_reg(dm, 0x95C, BIT(31), is_trigger_edge);
+		odm_set_bb_reg(dm, 0x95c, 0xe0, sampling_rate);
+		/*	(0:) '80MHz'
+		 *	(1:) '40MHz'
+		 *	(2:) '20MHz'
+		 *	(3:) '10MHz'
+		 *	(4:) '5MHz'
+		 *	(5:) '2.5MHz'
+		 *	(6:) '1.25MHz'
+		 *	(7:) '160MHz (for BW160 ic)'
+		 */
+	} else {
+		/*0x9A0[4:0], BB debug port bit*/
+		odm_set_bb_reg(dm, 0x9a0, 0x1f, trig_sig_sel);
+		odm_set_bb_reg(dm, 0x908, MASKDWORD, dbg_port);
+		/*0: posedge, 1: negedge*/
+		odm_set_bb_reg(dm, 0x9A0, BIT(31), is_trigger_edge);
+		odm_set_bb_reg(dm, 0x9A0, 0xe0, sampling_rate);
+		/*	(0:) '80MHz'
+		 *	(1:) '40MHz'
+		 *	(2:) '20MHz'
+		 *	(3:) '10MHz'
+		 *	(4:) '5MHz'
+		 *	(5:) '2.5MHz'
+		 *	(6:) '1.25MHz'
+		 *	(7:) '160MHz (for BW160 ic)'
+		 */
+	}
+}
+
+void phydm_la_mode_set_trigger_time(void *dm_void, u32 trigger_time_mu_sec)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u8 trigger_time_unit_num;
+	u32 time_unit = 0;
+
+	if (trigger_time_mu_sec < 128)
+		time_unit = 0; /*unit: 1mu sec*/
+	else if (trigger_time_mu_sec < 256)
+		time_unit = 1; /*unit: 2mu sec*/
+	else if (trigger_time_mu_sec < 512)
+		time_unit = 2; /*unit: 4mu sec*/
+	else if (trigger_time_mu_sec < 1024)
+		time_unit = 3; /*unit: 8mu sec*/
+	else if (trigger_time_mu_sec < 2048)
+		time_unit = 4; /*unit: 16mu sec*/
+	else if (trigger_time_mu_sec < 4096)
+		time_unit = 5; /*unit: 32mu sec*/
+	else if (trigger_time_mu_sec < 8192)
+		time_unit = 6; /*unit: 64mu sec*/
+
+	trigger_time_unit_num = (u8)(trigger_time_mu_sec >> time_unit);
+
+	ODM_RT_TRACE(
+		dm, ODM_COMP_UNCOND,
+		"3. [Set Trigger Time] Trig_Time = ((%d)) * unit = ((2^%d us))\n",
+		trigger_time_unit_num, time_unit);
+
+	odm_set_mac_reg(dm, 0x7cc, BIT(20) | BIT(19) | BIT(18), time_unit);
+	odm_set_mac_reg(dm, 0x7c0, 0x7f00, (trigger_time_unit_num & 0x7f));
+}
+
+void phydm_lamode_trigger_setting(void *dm_void, char input[][16], u32 *_used,
+				  char *output, u32 *_out_len, u32 input_num)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct rt_adcsmp *adc_smp = &dm->adcsmp;
+	u8 trig_mode, dma_data_sig_sel;
+	u32 trig_sig_sel;
+	bool is_enable_la_mode;
+	u32 trigger_time_mu_sec;
+	char help[] = "-h";
+	u32 var1[10] = {0};
+	u32 used = *_used;
+	u32 out_len = *_out_len;
+
+	if (dm->support_ic_type & PHYDM_IC_SUPPORT_LA_MODE) {
+		PHYDM_SSCANF(input[1], DCMD_DECIMAL, &var1[0]);
+		is_enable_la_mode = (bool)var1[0];
+		/*dbg_print("echo cmd input_num = %d\n", input_num);*/
+
+		if ((strcmp(input[1], help) == 0)) {
+			PHYDM_SNPRINTF(
+				output + used, out_len - used,
+				"{En} {0:BB,1:BB_MAC,2:RF0,3:RF1,4:MAC}\n {BB:dbg_port[bit],BB_MAC:0-ok/1-fail/2-cca,MAC:ref} {DMA type} {TrigTime}\n {polling_time/ref_mask} {dbg_port} {0:P_Edge, 1:N_Edge} {SpRate:0-80M,1-40M,2-20M} {Capture num}\n");
+			/**/
+		} else if ((is_enable_la_mode == 1)) {
+			PHYDM_SSCANF(input[2], DCMD_DECIMAL, &var1[1]);
+
+			trig_mode = (u8)var1[1];
+
+			if (trig_mode == PHYDM_MAC_TRIG)
+				PHYDM_SSCANF(input[3], DCMD_HEX, &var1[2]);
+			else
+				PHYDM_SSCANF(input[3], DCMD_DECIMAL, &var1[2]);
+			trig_sig_sel = var1[2];
+
+			PHYDM_SSCANF(input[4], DCMD_DECIMAL, &var1[3]);
+			PHYDM_SSCANF(input[5], DCMD_DECIMAL, &var1[4]);
+			PHYDM_SSCANF(input[6], DCMD_HEX, &var1[5]);
+			PHYDM_SSCANF(input[7], DCMD_HEX, &var1[6]);
+			PHYDM_SSCANF(input[8], DCMD_DECIMAL, &var1[7]);
+			PHYDM_SSCANF(input[9], DCMD_DECIMAL, &var1[8]);
+			PHYDM_SSCANF(input[10], DCMD_DECIMAL, &var1[9]);
+
+			dma_data_sig_sel = (u8)var1[3];
+			trigger_time_mu_sec = var1[4]; /*unit: us*/
+
+			adc_smp->la_mac_ref_mask = var1[5];
+			adc_smp->la_dbg_port = var1[6];
+			adc_smp->la_trigger_edge = (u8)var1[7];
+			adc_smp->la_smp_rate = (u8)(var1[8] & 0x7);
+			adc_smp->la_count = var1[9];
+
+			ODM_RT_TRACE(
+				dm, ODM_COMP_UNCOND,
+				"echo lamode %d %d %d %d %d %d %x %d %d %d\n",
+				var1[0], var1[1], var1[2], var1[3], var1[4],
+				var1[5], var1[6], var1[7], var1[8], var1[9]);
+
+			PHYDM_SNPRINTF(
+				output + used, out_len - used,
+				"a.En= ((1)),  b.mode = ((%d)), c.Trig_Sel = ((0x%x)), d.Dma_type = ((%d))\n",
+				trig_mode, trig_sig_sel, dma_data_sig_sel);
+			PHYDM_SNPRINTF(
+				output + used, out_len - used,
+				"e.Trig_Time = ((%dus)), f.mac_ref_mask = ((0x%x)), g.dbg_port = ((0x%x))\n",
+				trigger_time_mu_sec, adc_smp->la_mac_ref_mask,
+				adc_smp->la_dbg_port);
+			PHYDM_SNPRINTF(
+				output + used, out_len - used,
+				"h.Trig_edge = ((%d)), i.smp rate = ((%d MHz)), j.Cap_num = ((%d))\n",
+				adc_smp->la_trigger_edge,
+				(80 >> adc_smp->la_smp_rate),
+				adc_smp->la_count);
+
+			adc_smp_set(dm, trig_mode, trig_sig_sel,
+				    dma_data_sig_sel, trigger_time_mu_sec, 0);
+
+		} else {
+			adc_smp_stop(dm);
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "Disable LA mode\n");
+		}
+	}
+}
diff --git a/drivers/staging/rtlwifi/phydm/phydm_adc_sampling.h b/drivers/staging/rtlwifi/phydm/phydm_adc_sampling.h
new file mode 100644
index 000000000000..460931489be3
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_adc_sampling.h
@@ -0,0 +1,96 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef __INC_ADCSMP_H
+#define __INC_ADCSMP_H
+
+#define DYNAMIC_LA_MODE "1.0" /*2016.07.15  Dino */
+
+struct rt_adcsmp_string {
+	u32 *octet;
+	u32 length;
+	u32 buffer_size;
+	u32 start_pos;
+};
+
+enum rt_adcsmp_trig_sel {
+	PHYDM_ADC_BB_TRIG = 0,
+	PHYDM_ADC_MAC_TRIG = 1,
+	PHYDM_ADC_RF0_TRIG = 2,
+	PHYDM_ADC_RF1_TRIG = 3,
+	PHYDM_MAC_TRIG = 4
+};
+
+enum rt_adcsmp_trig_sig_sel {
+	ADCSMP_TRIG_CRCOK = 0,
+	ADCSMP_TRIG_CRCFAIL = 1,
+	ADCSMP_TRIG_CCA = 2,
+	ADCSMP_TRIG_REG = 3
+};
+
+enum rt_adcsmp_state {
+	ADCSMP_STATE_IDLE = 0,
+	ADCSMP_STATE_SET = 1,
+	ADCSMP_STATE_QUERY = 2
+};
+
+struct rt_adcsmp {
+	struct rt_adcsmp_string adc_smp_buf;
+	enum rt_adcsmp_state adc_smp_state;
+	u8 la_trig_mode;
+	u32 la_trig_sig_sel;
+	u8 la_dma_type;
+	u32 la_trigger_time;
+	u32 la_mac_ref_mask;
+	u32 la_dbg_port;
+	u8 la_trigger_edge;
+	u8 la_smp_rate;
+	u32 la_count;
+	u8 is_bb_trigger;
+	u8 la_work_item_index;
+};
+
+void adc_smp_set(void *dm_void, u8 trig_mode, u32 trig_sig_sel,
+		 u8 dma_data_sig_sel, u32 trigger_time, u16 polling_time);
+
+void adc_smp_query(void *dm_void, void *output, u32 out_len, u32 *pused);
+
+s32 adc_smp_get_sample_counts(void *dm_void);
+
+s32 adc_smp_query_single_data(void *dm_void, void *output, u32 out_len,
+			      u32 index);
+
+void adc_smp_stop(void *dm_void);
+
+void adc_smp_init(void *dm_void);
+
+void adc_smp_de_init(void *dm_void);
+
+void phydm_la_mode_bb_setting(void *dm_void);
+
+void phydm_la_mode_set_trigger_time(void *dm_void, u32 trigger_time_mu_sec);
+
+void phydm_lamode_trigger_setting(void *dm_void, char input[][16], u32 *_used,
+				  char *output, u32 *_out_len, u32 input_num);
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/phydm_antdiv.c b/drivers/staging/rtlwifi/phydm/phydm_antdiv.c
new file mode 100644
index 000000000000..39d3c6947556
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_antdiv.c
@@ -0,0 +1,83 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+/* ************************************************************
+ * include files
+ * *************************************************************/
+
+#include "mp_precomp.h"
+#include "phydm_precomp.h"
+
+/* ******************************************************
+ * when antenna test utility is on or some testing need to disable antenna
+ * diversity, call this function to disable all ODM related mechanisms which
+ * will switch antenna.
+ * *******************************************************/
+void odm_stop_antenna_switch_dm(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	/* disable ODM antenna diversity */
+	dm->support_ability &= ~ODM_BB_ANT_DIV;
+	ODM_RT_TRACE(dm, ODM_COMP_ANT_DIV, "STOP Antenna Diversity\n");
+}
+
+void phydm_enable_antenna_diversity(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	dm->support_ability |= ODM_BB_ANT_DIV;
+	ODM_RT_TRACE(dm, ODM_COMP_ANT_DIV,
+		     "AntDiv is enabled & Re-Init AntDiv\n");
+	odm_antenna_diversity_init(dm);
+}
+
+void odm_set_ant_config(void *dm_void, u8 ant_setting /* 0=A, 1=B, 2=C, .... */
+			)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	if (dm->support_ic_type == ODM_RTL8723B) {
+		if (ant_setting == 0) /* ant A*/
+			odm_set_bb_reg(dm, 0x948, MASKDWORD, 0x00000000);
+		else if (ant_setting == 1)
+			odm_set_bb_reg(dm, 0x948, MASKDWORD, 0x00000280);
+	} else if (dm->support_ic_type == ODM_RTL8723D) {
+		if (ant_setting == 0) /* ant A*/
+			odm_set_bb_reg(dm, 0x948, MASKLWORD, 0x0000);
+		else if (ant_setting == 1)
+			odm_set_bb_reg(dm, 0x948, MASKLWORD, 0x0280);
+	}
+}
+
+/* ****************************************************** */
+
+void odm_sw_ant_div_rest_after_link(void *dm_void) {}
+
+void odm_ant_div_reset(void *dm_void) {}
+
+void odm_antenna_diversity_init(void *dm_void) {}
+
+void odm_antenna_diversity(void *dm_void) {}
diff --git a/drivers/staging/rtlwifi/phydm/phydm_antdiv.h b/drivers/staging/rtlwifi/phydm/phydm_antdiv.h
new file mode 100644
index 000000000000..ebbff2f56c5e
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_antdiv.h
@@ -0,0 +1,301 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __PHYDMANTDIV_H__
+#define __PHYDMANTDIV_H__
+
+/* 2.0 2014.11.04
+ * 2.1 2015.01.13 Dino
+ * 2.2 2015.01.16 Dino
+ * 3.1 2015.07.29 YuChen, remove 92c 92d 8723a
+ * 3.2 2015.08.11 Stanley, disable antenna diversity when BT is enable for 8723B
+ * 3.3 2015.08.12 Stanley. 8723B does not need to check the antenna is control
+ *		  by BT, because antenna diversity only works when BT is disable
+ *		  or radio off
+ * 3.4 2015.08.28 Dino  1.Add 8821A Smart Antenna 2. Add 8188F SW S0S1 Antenna
+ *		  Diversity
+ * 3.5 2015.10.07 Stanley  Always check antenna detection result from BT-coex.
+ *		  for 8723B, not from PHYDM
+ * 3.6 2015.11.16 Stanley
+ * 3.7 2015.11.20 Dino Add SmartAnt FAT Patch
+ * 3.8 2015.12.21 Dino, Add SmartAnt dynamic training packet num
+ * 3.9 2016.01.05 Dino, Add SmartAnt cmd for converting single & two smtant, and
+ *		  add cmd for adjust truth table
+ */
+#define ANTDIV_VERSION "3.9"
+
+/* 1 ============================================================
+ * 1  Definition
+ * 1 ============================================================
+ */
+
+#define ANTDIV_INIT 0xff
+#define MAIN_ANT 1 /*ant A or ant Main   or S1*/
+#define AUX_ANT 2 /*AntB or ant Aux   or S0*/
+#define MAX_ANT 3 /* 3 for AP using*/
+
+#define ANT1_2G 0 /* = ANT2_5G	for 8723D  BTG S1 RX S0S1 diversity for 8723D,
+		   * TX fixed at S1
+		   */
+#define ANT2_2G 1 /* = ANT1_5G	for 8723D  BTG S0  RX S0S1 diversity for 8723D,
+		   * TX fixed at S1
+		   */
+/*smart antenna*/
+#define SUPPORT_RF_PATH_NUM 4
+#define SUPPORT_BEAM_PATTERN_NUM 4
+#define NUM_ANTENNA_8821A 2
+
+#define SUPPORT_BEAM_SET_PATTERN_NUM 8
+
+#define NO_FIX_TX_ANT 0
+#define FIX_TX_AT_MAIN 1
+#define FIX_AUX_AT_MAIN 2
+
+/* Antenna Diversty Control type */
+#define ODM_AUTO_ANT 0
+#define ODM_FIX_MAIN_ANT 1
+#define ODM_FIX_AUX_ANT 2
+
+#define ODM_N_ANTDIV_SUPPORT                                                   \
+	(ODM_RTL8188E | ODM_RTL8192E | ODM_RTL8723B | ODM_RTL8188F |           \
+	 ODM_RTL8723D | ODM_RTL8195A)
+#define ODM_AC_ANTDIV_SUPPORT                                                  \
+	(ODM_RTL8821 | ODM_RTL8881A | ODM_RTL8812 | ODM_RTL8821C |             \
+	 ODM_RTL8822B | ODM_RTL8814B)
+#define ODM_ANTDIV_SUPPORT (ODM_N_ANTDIV_SUPPORT | ODM_AC_ANTDIV_SUPPORT)
+#define ODM_SMART_ANT_SUPPORT (ODM_RTL8188E | ODM_RTL8192E)
+#define ODM_HL_SMART_ANT_TYPE1_SUPPORT (ODM_RTL8821 | ODM_RTL8822B)
+
+#define ODM_ANTDIV_2G_SUPPORT_IC                                               \
+	(ODM_RTL8188E | ODM_RTL8192E | ODM_RTL8723B | ODM_RTL8881A |           \
+	 ODM_RTL8188F | ODM_RTL8723D)
+#define ODM_ANTDIV_5G_SUPPORT_IC                                               \
+	(ODM_RTL8821 | ODM_RTL8881A | ODM_RTL8812 | ODM_RTL8821C)
+
+#define ODM_EVM_ENHANCE_ANTDIV_SUPPORT_IC (ODM_RTL8192E)
+
+#define ODM_ANTDIV_2G BIT(0)
+#define ODM_ANTDIV_5G BIT(1)
+
+#define ANTDIV_ON 1
+#define ANTDIV_OFF 0
+
+#define FAT_ON 1
+#define FAT_OFF 0
+
+#define TX_BY_DESC 1
+#define TX_BY_REG 0
+
+#define RSSI_METHOD 0
+#define EVM_METHOD 1
+#define CRC32_METHOD 2
+
+#define INIT_ANTDIV_TIMMER 0
+#define CANCEL_ANTDIV_TIMMER 1
+#define RELEASE_ANTDIV_TIMMER 2
+
+#define CRC32_FAIL 1
+#define CRC32_OK 0
+
+#define evm_rssi_th_high 25
+#define evm_rssi_th_low 20
+
+#define NORMAL_STATE_MIAN 1
+#define NORMAL_STATE_AUX 2
+#define TRAINING_STATE 3
+
+#define FORCE_RSSI_DIFF 10
+
+#define CSI_ON 1
+#define CSI_OFF 0
+
+#define DIVON_CSIOFF 1
+#define DIVOFF_CSION 2
+
+#define BDC_DIV_TRAIN_STATE 0
+#define bdc_bfer_train_state 1
+#define BDC_DECISION_STATE 2
+#define BDC_BF_HOLD_STATE 3
+#define BDC_DIV_HOLD_STATE 4
+
+#define BDC_MODE_1 1
+#define BDC_MODE_2 2
+#define BDC_MODE_3 3
+#define BDC_MODE_4 4
+#define BDC_MODE_NULL 0xff
+
+/*SW S0S1 antenna diversity*/
+#define SWAW_STEP_INIT 0xff
+#define SWAW_STEP_PEEK 0
+#define SWAW_STEP_DETERMINE 1
+
+#define RSSI_CHECK_RESET_PERIOD 10
+#define RSSI_CHECK_THRESHOLD 50
+
+/*Hong Lin Smart antenna*/
+#define HL_SMTANT_2WIRE_DATA_LEN 24
+
+/* 1 ============================================================
+ * 1  structure
+ * 1 ============================================================
+ */
+
+struct sw_antenna_switch {
+	u8 double_chk_flag; /*If current antenna RSSI > "RSSI_CHECK_THRESHOLD",
+			     *than check this antenna again
+			     */
+	u8 try_flag;
+	s32 pre_rssi;
+	u8 cur_antenna;
+	u8 pre_antenna;
+	u8 rssi_trying;
+	u8 reset_idx;
+	u8 train_time;
+	u8 train_time_flag; /*base on RSSI difference between two antennas*/
+	struct timer_list phydm_sw_antenna_switch_timer;
+	u32 pkt_cnt_sw_ant_div_by_ctrl_frame;
+	bool is_sw_ant_div_by_ctrl_frame;
+
+	/* AntDect (Before link Antenna Switch check) need to be moved*/
+	u16 single_ant_counter;
+	u16 dual_ant_counter;
+	u16 aux_fail_detec_counter;
+	u16 retry_counter;
+	u8 swas_no_link_state;
+	u32 swas_no_link_bk_reg948;
+	bool ANTA_ON; /*To indicate ant A is or not*/
+	bool ANTB_ON; /*To indicate ant B is on or not*/
+	bool pre_aux_fail_detec;
+	bool rssi_ant_dect_result;
+	u8 ant_5g;
+	u8 ant_2g;
+};
+
+struct fast_antenna_training {
+	u8 bssid[6];
+	u8 antsel_rx_keep_0;
+	u8 antsel_rx_keep_1;
+	u8 antsel_rx_keep_2;
+	u8 antsel_rx_keep_3;
+	u32 ant_sum_rssi[7];
+	u32 ant_rssi_cnt[7];
+	u32 ant_ave_rssi[7];
+	u8 fat_state;
+	u32 train_idx;
+	u8 antsel_a[ODM_ASSOCIATE_ENTRY_NUM];
+	u8 antsel_b[ODM_ASSOCIATE_ENTRY_NUM];
+	u8 antsel_c[ODM_ASSOCIATE_ENTRY_NUM];
+	u16 main_ant_sum[ODM_ASSOCIATE_ENTRY_NUM];
+	u16 aux_ant_sum[ODM_ASSOCIATE_ENTRY_NUM];
+	u16 main_ant_cnt[ODM_ASSOCIATE_ENTRY_NUM];
+	u16 aux_ant_cnt[ODM_ASSOCIATE_ENTRY_NUM];
+	u16 main_ant_sum_cck[ODM_ASSOCIATE_ENTRY_NUM];
+	u16 aux_ant_sum_cck[ODM_ASSOCIATE_ENTRY_NUM];
+	u16 main_ant_cnt_cck[ODM_ASSOCIATE_ENTRY_NUM];
+	u16 aux_ant_cnt_cck[ODM_ASSOCIATE_ENTRY_NUM];
+	u8 rx_idle_ant;
+	u8 ant_div_on_off;
+	bool is_become_linked;
+	u32 min_max_rssi;
+	u8 idx_ant_div_counter_2g;
+	u8 idx_ant_div_counter_5g;
+	u8 ant_div_2g_5g;
+
+	u32 cck_ctrl_frame_cnt_main;
+	u32 cck_ctrl_frame_cnt_aux;
+	u32 ofdm_ctrl_frame_cnt_main;
+	u32 ofdm_ctrl_frame_cnt_aux;
+	u32 main_ant_ctrl_frame_sum;
+	u32 aux_ant_ctrl_frame_sum;
+	u32 main_ant_ctrl_frame_cnt;
+	u32 aux_ant_ctrl_frame_cnt;
+	u8 b_fix_tx_ant;
+	bool fix_ant_bfee;
+	bool enable_ctrl_frame_antdiv;
+	bool use_ctrl_frame_antdiv;
+	u8 hw_antsw_occur;
+	u8 *p_force_tx_ant_by_desc;
+	u8 force_tx_ant_by_desc; /*A temp value, will hook to driver team's
+				  *outer parameter later
+				  */
+	u8 *p_default_s0_s1;
+	u8 default_s0_s1;
+};
+
+/* 1 ============================================================
+ * 1  enumeration
+ * 1 ============================================================
+ */
+
+/*Fast antenna training*/
+enum fat_state {
+	FAT_BEFORE_LINK_STATE = 0,
+	FAT_PREPARE_STATE = 1,
+	FAT_TRAINING_STATE = 2,
+	FAT_DECISION_STATE = 3
+};
+
+enum ant_div_type {
+	NO_ANTDIV = 0xFF,
+	CG_TRX_HW_ANTDIV = 0x01,
+	CGCS_RX_HW_ANTDIV = 0x02,
+	FIXED_HW_ANTDIV = 0x03,
+	CG_TRX_SMART_ANTDIV = 0x04,
+	CGCS_RX_SW_ANTDIV = 0x05,
+	/*8723B intrnal switch S0 S1*/
+	S0S1_SW_ANTDIV = 0x06,
+	/*TRX S0S1 diversity for 8723D*/
+	S0S1_TRX_HW_ANTDIV = 0x07,
+	/*Hong-Lin Smart antenna use for 8821AE which is a 2 ant. entitys, and
+	 *each ant. is equipped with 4 antenna patterns
+	 */
+	HL_SW_SMART_ANT_TYPE1 = 0x10,
+	/*Hong-Bo Smart antenna use for 8822B which is a 2 ant. entitys*/
+	HL_SW_SMART_ANT_TYPE2 = 0x11,
+};
+
+/* 1 ============================================================
+ * 1  function prototype
+ * 1 ============================================================
+ */
+
+void odm_stop_antenna_switch_dm(void *dm_void);
+
+void phydm_enable_antenna_diversity(void *dm_void);
+
+void odm_set_ant_config(void *dm_void, u8 ant_setting /* 0=A, 1=B, 2=C, .... */
+			);
+
+#define sw_ant_div_rest_after_link odm_sw_ant_div_rest_after_link
+
+void odm_sw_ant_div_rest_after_link(void *dm_void);
+
+void odm_ant_div_reset(void *dm_void);
+
+void odm_antenna_diversity_init(void *dm_void);
+
+void odm_antenna_diversity(void *dm_void);
+
+#endif /*#ifndef	__ODMANTDIV_H__*/
diff --git a/drivers/staging/rtlwifi/phydm/phydm_beamforming.h b/drivers/staging/rtlwifi/phydm/phydm_beamforming.h
new file mode 100644
index 000000000000..adc04ba4e218
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_beamforming.h
@@ -0,0 +1,48 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef __INC_PHYDM_BEAMFORMING_H
+#define __INC_PHYDM_BEAMFORMING_H
+
+/*Beamforming Related*/
+#include "txbf/halcomtxbf.h"
+#include "txbf/haltxbfjaguar.h"
+#include "txbf/haltxbf8822b.h"
+#include "txbf/haltxbfinterface.h"
+
+#define beamforming_gid_paid(adapter, tcb)
+#define phydm_acting_determine(dm, type) false
+#define beamforming_enter(dm, sta_idx)
+#define beamforming_leave(dm, RA)
+#define beamforming_end_fw(dm)
+#define beamforming_control_v1(dm, RA, AID, mode, BW, rate) true
+#define beamforming_control_v2(dm, idx, mode, BW, period) true
+#define phydm_beamforming_end_sw(dm, _status)
+#define beamforming_timer_callback(dm)
+#define phydm_beamforming_init(dm)
+#define phydm_beamforming_control_v2(dm, _idx, _mode, _BW, _period) false
+#define beamforming_watchdog(dm)
+#define phydm_beamforming_watchdog(dm)
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/phydm_ccx.c b/drivers/staging/rtlwifi/phydm/phydm_ccx.c
new file mode 100644
index 000000000000..2e0dc68757dc
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_ccx.c
@@ -0,0 +1,457 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#include "mp_precomp.h"
+#include "phydm_precomp.h"
+
+/*Set NHM period, threshold, disable ignore cca or not,
+ *disable ignore txon or not
+ */
+void phydm_nhm_setting(void *dm_void, u8 nhm_setting)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct ccx_info *ccx_info = &dm->dm_ccx_info;
+
+	if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+		if (nhm_setting == SET_NHM_SETTING) {
+			/*Set inexclude_cca, inexclude_txon*/
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11AC, BIT(9),
+				       ccx_info->nhm_inexclude_cca);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11AC, BIT(10),
+				       ccx_info->nhm_inexclude_txon);
+
+			/*Set NHM period*/
+			odm_set_bb_reg(dm, ODM_REG_CCX_PERIOD_11AC, MASKHWORD,
+				       ccx_info->NHM_period);
+
+			/*Set NHM threshold*/
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH3_TO_TH0_11AC,
+				       MASKBYTE0, ccx_info->NHM_th[0]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH3_TO_TH0_11AC,
+				       MASKBYTE1, ccx_info->NHM_th[1]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH3_TO_TH0_11AC,
+				       MASKBYTE2, ccx_info->NHM_th[2]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH3_TO_TH0_11AC,
+				       MASKBYTE3, ccx_info->NHM_th[3]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH7_TO_TH4_11AC,
+				       MASKBYTE0, ccx_info->NHM_th[4]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH7_TO_TH4_11AC,
+				       MASKBYTE1, ccx_info->NHM_th[5]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH7_TO_TH4_11AC,
+				       MASKBYTE2, ccx_info->NHM_th[6]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH7_TO_TH4_11AC,
+				       MASKBYTE3, ccx_info->NHM_th[7]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH8_11AC, MASKBYTE0,
+				       ccx_info->NHM_th[8]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11AC, MASKBYTE2,
+				       ccx_info->NHM_th[9]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11AC, MASKBYTE3,
+				       ccx_info->NHM_th[10]);
+
+			/*CCX EN*/
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11AC, BIT(8),
+				       CCX_EN);
+		} else if (nhm_setting == STORE_NHM_SETTING) {
+			/*Store prev. disable_ignore_cca, disable_ignore_txon*/
+			ccx_info->NHM_inexclude_cca_restore =
+				(enum nhm_inexclude_cca)odm_get_bb_reg(
+					dm, ODM_REG_NHM_TH9_TH10_11AC, BIT(9));
+			ccx_info->NHM_inexclude_txon_restore =
+				(enum nhm_inexclude_txon)odm_get_bb_reg(
+					dm, ODM_REG_NHM_TH9_TH10_11AC, BIT(10));
+
+			/*Store pervious NHM period*/
+			ccx_info->NHM_period_restore = (u16)odm_get_bb_reg(
+				dm, ODM_REG_CCX_PERIOD_11AC, MASKHWORD);
+
+			/*Store NHM threshold*/
+			ccx_info->NHM_th_restore[0] = (u8)odm_get_bb_reg(
+				dm, ODM_REG_NHM_TH3_TO_TH0_11AC, MASKBYTE0);
+			ccx_info->NHM_th_restore[1] = (u8)odm_get_bb_reg(
+				dm, ODM_REG_NHM_TH3_TO_TH0_11AC, MASKBYTE1);
+			ccx_info->NHM_th_restore[2] = (u8)odm_get_bb_reg(
+				dm, ODM_REG_NHM_TH3_TO_TH0_11AC, MASKBYTE2);
+			ccx_info->NHM_th_restore[3] = (u8)odm_get_bb_reg(
+				dm, ODM_REG_NHM_TH3_TO_TH0_11AC, MASKBYTE3);
+			ccx_info->NHM_th_restore[4] = (u8)odm_get_bb_reg(
+				dm, ODM_REG_NHM_TH7_TO_TH4_11AC, MASKBYTE0);
+			ccx_info->NHM_th_restore[5] = (u8)odm_get_bb_reg(
+				dm, ODM_REG_NHM_TH7_TO_TH4_11AC, MASKBYTE1);
+			ccx_info->NHM_th_restore[6] = (u8)odm_get_bb_reg(
+				dm, ODM_REG_NHM_TH7_TO_TH4_11AC, MASKBYTE2);
+			ccx_info->NHM_th_restore[7] = (u8)odm_get_bb_reg(
+				dm, ODM_REG_NHM_TH7_TO_TH4_11AC, MASKBYTE3);
+			ccx_info->NHM_th_restore[8] = (u8)odm_get_bb_reg(
+				dm, ODM_REG_NHM_TH8_11AC, MASKBYTE0);
+			ccx_info->NHM_th_restore[9] = (u8)odm_get_bb_reg(
+				dm, ODM_REG_NHM_TH9_TH10_11AC, MASKBYTE2);
+			ccx_info->NHM_th_restore[10] = (u8)odm_get_bb_reg(
+				dm, ODM_REG_NHM_TH9_TH10_11AC, MASKBYTE3);
+		} else if (nhm_setting == RESTORE_NHM_SETTING) {
+			/*Set disable_ignore_cca, disable_ignore_txon*/
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11AC, BIT(9),
+				       ccx_info->NHM_inexclude_cca_restore);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11AC, BIT(10),
+				       ccx_info->NHM_inexclude_txon_restore);
+
+			/*Set NHM period*/
+			odm_set_bb_reg(dm, ODM_REG_CCX_PERIOD_11AC, MASKHWORD,
+				       ccx_info->NHM_period);
+
+			/*Set NHM threshold*/
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH3_TO_TH0_11AC,
+				       MASKBYTE0, ccx_info->NHM_th_restore[0]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH3_TO_TH0_11AC,
+				       MASKBYTE1, ccx_info->NHM_th_restore[1]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH3_TO_TH0_11AC,
+				       MASKBYTE2, ccx_info->NHM_th_restore[2]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH3_TO_TH0_11AC,
+				       MASKBYTE3, ccx_info->NHM_th_restore[3]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH7_TO_TH4_11AC,
+				       MASKBYTE0, ccx_info->NHM_th_restore[4]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH7_TO_TH4_11AC,
+				       MASKBYTE1, ccx_info->NHM_th_restore[5]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH7_TO_TH4_11AC,
+				       MASKBYTE2, ccx_info->NHM_th_restore[6]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH7_TO_TH4_11AC,
+				       MASKBYTE3, ccx_info->NHM_th_restore[7]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH8_11AC, MASKBYTE0,
+				       ccx_info->NHM_th_restore[8]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11AC, MASKBYTE2,
+				       ccx_info->NHM_th_restore[9]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11AC, MASKBYTE3,
+				       ccx_info->NHM_th_restore[10]);
+		} else {
+			return;
+		}
+	}
+
+	else if (dm->support_ic_type & ODM_IC_11N_SERIES) {
+		if (nhm_setting == SET_NHM_SETTING) {
+			/*Set disable_ignore_cca, disable_ignore_txon*/
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11N, BIT(9),
+				       ccx_info->nhm_inexclude_cca);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11N, BIT(10),
+				       ccx_info->nhm_inexclude_txon);
+
+			/*Set NHM period*/
+			odm_set_bb_reg(dm, ODM_REG_CCX_PERIOD_11N, MASKHWORD,
+				       ccx_info->NHM_period);
+
+			/*Set NHM threshold*/
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH3_TO_TH0_11N,
+				       MASKBYTE0, ccx_info->NHM_th[0]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH3_TO_TH0_11N,
+				       MASKBYTE1, ccx_info->NHM_th[1]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH3_TO_TH0_11N,
+				       MASKBYTE2, ccx_info->NHM_th[2]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH3_TO_TH0_11N,
+				       MASKBYTE3, ccx_info->NHM_th[3]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH7_TO_TH4_11N,
+				       MASKBYTE0, ccx_info->NHM_th[4]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH7_TO_TH4_11N,
+				       MASKBYTE1, ccx_info->NHM_th[5]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH7_TO_TH4_11N,
+				       MASKBYTE2, ccx_info->NHM_th[6]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH7_TO_TH4_11N,
+				       MASKBYTE3, ccx_info->NHM_th[7]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH8_11N, MASKBYTE0,
+				       ccx_info->NHM_th[8]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11N, MASKBYTE2,
+				       ccx_info->NHM_th[9]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11N, MASKBYTE3,
+				       ccx_info->NHM_th[10]);
+
+			/*CCX EN*/
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11N, BIT(8),
+				       CCX_EN);
+		} else if (nhm_setting == STORE_NHM_SETTING) {
+			/*Store prev. disable_ignore_cca, disable_ignore_txon*/
+			ccx_info->NHM_inexclude_cca_restore =
+				(enum nhm_inexclude_cca)odm_get_bb_reg(
+					dm, ODM_REG_NHM_TH9_TH10_11N, BIT(9));
+			ccx_info->NHM_inexclude_txon_restore =
+				(enum nhm_inexclude_txon)odm_get_bb_reg(
+					dm, ODM_REG_NHM_TH9_TH10_11N, BIT(10));
+
+			/*Store pervious NHM period*/
+			ccx_info->NHM_period_restore = (u16)odm_get_bb_reg(
+				dm, ODM_REG_CCX_PERIOD_11N, MASKHWORD);
+
+			/*Store NHM threshold*/
+			ccx_info->NHM_th_restore[0] = (u8)odm_get_bb_reg(
+				dm, ODM_REG_NHM_TH3_TO_TH0_11N, MASKBYTE0);
+			ccx_info->NHM_th_restore[1] = (u8)odm_get_bb_reg(
+				dm, ODM_REG_NHM_TH3_TO_TH0_11N, MASKBYTE1);
+			ccx_info->NHM_th_restore[2] = (u8)odm_get_bb_reg(
+				dm, ODM_REG_NHM_TH3_TO_TH0_11N, MASKBYTE2);
+			ccx_info->NHM_th_restore[3] = (u8)odm_get_bb_reg(
+				dm, ODM_REG_NHM_TH3_TO_TH0_11N, MASKBYTE3);
+			ccx_info->NHM_th_restore[4] = (u8)odm_get_bb_reg(
+				dm, ODM_REG_NHM_TH7_TO_TH4_11N, MASKBYTE0);
+			ccx_info->NHM_th_restore[5] = (u8)odm_get_bb_reg(
+				dm, ODM_REG_NHM_TH7_TO_TH4_11N, MASKBYTE1);
+			ccx_info->NHM_th_restore[6] = (u8)odm_get_bb_reg(
+				dm, ODM_REG_NHM_TH7_TO_TH4_11N, MASKBYTE2);
+			ccx_info->NHM_th_restore[7] = (u8)odm_get_bb_reg(
+				dm, ODM_REG_NHM_TH7_TO_TH4_11N, MASKBYTE3);
+			ccx_info->NHM_th_restore[8] = (u8)odm_get_bb_reg(
+				dm, ODM_REG_NHM_TH8_11N, MASKBYTE0);
+			ccx_info->NHM_th_restore[9] = (u8)odm_get_bb_reg(
+				dm, ODM_REG_NHM_TH9_TH10_11N, MASKBYTE2);
+			ccx_info->NHM_th_restore[10] = (u8)odm_get_bb_reg(
+				dm, ODM_REG_NHM_TH9_TH10_11N, MASKBYTE3);
+		} else if (nhm_setting == RESTORE_NHM_SETTING) {
+			/*Set disable_ignore_cca, disable_ignore_txon*/
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11N, BIT(9),
+				       ccx_info->NHM_inexclude_cca_restore);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11N, BIT(10),
+				       ccx_info->NHM_inexclude_txon_restore);
+
+			/*Set NHM period*/
+			odm_set_bb_reg(dm, ODM_REG_CCX_PERIOD_11N, MASKHWORD,
+				       ccx_info->NHM_period_restore);
+
+			/*Set NHM threshold*/
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH3_TO_TH0_11N,
+				       MASKBYTE0, ccx_info->NHM_th_restore[0]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH3_TO_TH0_11N,
+				       MASKBYTE1, ccx_info->NHM_th_restore[1]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH3_TO_TH0_11N,
+				       MASKBYTE2, ccx_info->NHM_th_restore[2]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH3_TO_TH0_11N,
+				       MASKBYTE3, ccx_info->NHM_th_restore[3]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH7_TO_TH4_11N,
+				       MASKBYTE0, ccx_info->NHM_th_restore[4]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH7_TO_TH4_11N,
+				       MASKBYTE1, ccx_info->NHM_th_restore[5]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH7_TO_TH4_11N,
+				       MASKBYTE2, ccx_info->NHM_th_restore[6]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH7_TO_TH4_11N,
+				       MASKBYTE3, ccx_info->NHM_th_restore[7]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH8_11N, MASKBYTE0,
+				       ccx_info->NHM_th_restore[8]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11N, MASKBYTE2,
+				       ccx_info->NHM_th_restore[9]);
+			odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11N, MASKBYTE3,
+				       ccx_info->NHM_th_restore[10]);
+		} else {
+			return;
+		}
+	}
+}
+
+void phydm_nhm_trigger(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+		/*Trigger NHM*/
+		odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11AC, BIT(1), 0);
+		odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11AC, BIT(1), 1);
+	} else if (dm->support_ic_type & ODM_IC_11N_SERIES) {
+		/*Trigger NHM*/
+		odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11N, BIT(1), 0);
+		odm_set_bb_reg(dm, ODM_REG_NHM_TH9_TH10_11N, BIT(1), 1);
+	}
+}
+
+void phydm_get_nhm_result(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 value32;
+	struct ccx_info *ccx_info = &dm->dm_ccx_info;
+
+	if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+		value32 = odm_read_4byte(dm, ODM_REG_NHM_CNT_11AC);
+		ccx_info->NHM_result[0] = (u8)(value32 & MASKBYTE0);
+		ccx_info->NHM_result[1] = (u8)((value32 & MASKBYTE1) >> 8);
+		ccx_info->NHM_result[2] = (u8)((value32 & MASKBYTE2) >> 16);
+		ccx_info->NHM_result[3] = (u8)((value32 & MASKBYTE3) >> 24);
+
+		value32 = odm_read_4byte(dm, ODM_REG_NHM_CNT7_TO_CNT4_11AC);
+		ccx_info->NHM_result[4] = (u8)(value32 & MASKBYTE0);
+		ccx_info->NHM_result[5] = (u8)((value32 & MASKBYTE1) >> 8);
+		ccx_info->NHM_result[6] = (u8)((value32 & MASKBYTE2) >> 16);
+		ccx_info->NHM_result[7] = (u8)((value32 & MASKBYTE3) >> 24);
+
+		value32 = odm_read_4byte(dm, ODM_REG_NHM_CNT11_TO_CNT8_11AC);
+		ccx_info->NHM_result[8] = (u8)(value32 & MASKBYTE0);
+		ccx_info->NHM_result[9] = (u8)((value32 & MASKBYTE1) >> 8);
+		ccx_info->NHM_result[10] = (u8)((value32 & MASKBYTE2) >> 16);
+		ccx_info->NHM_result[11] = (u8)((value32 & MASKBYTE3) >> 24);
+
+		/*Get NHM duration*/
+		value32 = odm_read_4byte(dm, ODM_REG_NHM_DUR_READY_11AC);
+		ccx_info->NHM_duration = (u16)(value32 & MASKLWORD);
+	}
+
+	else if (dm->support_ic_type & ODM_IC_11N_SERIES) {
+		value32 = odm_read_4byte(dm, ODM_REG_NHM_CNT_11N);
+		ccx_info->NHM_result[0] = (u8)(value32 & MASKBYTE0);
+		ccx_info->NHM_result[1] = (u8)((value32 & MASKBYTE1) >> 8);
+		ccx_info->NHM_result[2] = (u8)((value32 & MASKBYTE2) >> 16);
+		ccx_info->NHM_result[3] = (u8)((value32 & MASKBYTE3) >> 24);
+
+		value32 = odm_read_4byte(dm, ODM_REG_NHM_CNT7_TO_CNT4_11N);
+		ccx_info->NHM_result[4] = (u8)(value32 & MASKBYTE0);
+		ccx_info->NHM_result[5] = (u8)((value32 & MASKBYTE1) >> 8);
+		ccx_info->NHM_result[6] = (u8)((value32 & MASKBYTE2) >> 16);
+		ccx_info->NHM_result[7] = (u8)((value32 & MASKBYTE3) >> 24);
+
+		value32 = odm_read_4byte(dm, ODM_REG_NHM_CNT9_TO_CNT8_11N);
+		ccx_info->NHM_result[8] = (u8)((value32 & MASKBYTE2) >> 16);
+		ccx_info->NHM_result[9] = (u8)((value32 & MASKBYTE3) >> 24);
+
+		value32 = odm_read_4byte(dm, ODM_REG_NHM_CNT10_TO_CNT11_11N);
+		ccx_info->NHM_result[10] = (u8)((value32 & MASKBYTE2) >> 16);
+		ccx_info->NHM_result[11] = (u8)((value32 & MASKBYTE3) >> 24);
+
+		/*Get NHM duration*/
+		value32 = odm_read_4byte(dm, ODM_REG_NHM_CNT10_TO_CNT11_11N);
+		ccx_info->NHM_duration = (u16)(value32 & MASKLWORD);
+	}
+}
+
+bool phydm_check_nhm_ready(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 value32 = 0;
+	u8 i;
+	bool ret = false;
+
+	if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+		value32 =
+			odm_get_bb_reg(dm, ODM_REG_CLM_RESULT_11AC, MASKDWORD);
+
+		for (i = 0; i < 200; i++) {
+			ODM_delay_ms(1);
+			if (odm_get_bb_reg(dm, ODM_REG_NHM_DUR_READY_11AC,
+					   BIT(17))) {
+				ret = 1;
+				break;
+			}
+		}
+	}
+
+	else if (dm->support_ic_type & ODM_IC_11N_SERIES) {
+		value32 = odm_get_bb_reg(dm, ODM_REG_CLM_READY_11N, MASKDWORD);
+
+		for (i = 0; i < 200; i++) {
+			ODM_delay_ms(1);
+			if (odm_get_bb_reg(dm, ODM_REG_NHM_DUR_READY_11AC,
+					   BIT(17))) {
+				ret = 1;
+				break;
+			}
+		}
+	}
+	return ret;
+}
+
+void phydm_clm_setting(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct ccx_info *ccx_info = &dm->dm_ccx_info;
+
+	if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+		odm_set_bb_reg(dm, ODM_REG_CCX_PERIOD_11AC, MASKLWORD,
+			       ccx_info->CLM_period); /*4us sample 1 time*/
+		odm_set_bb_reg(dm, ODM_REG_CLM_11AC, BIT(8),
+			       0x1); /*Enable CCX for CLM*/
+
+	} else if (dm->support_ic_type & ODM_IC_11N_SERIES) {
+		odm_set_bb_reg(dm, ODM_REG_CCX_PERIOD_11N, MASKLWORD,
+			       ccx_info->CLM_period); /*4us sample 1 time*/
+		odm_set_bb_reg(dm, ODM_REG_CLM_11N, BIT(8),
+			       0x1); /*Enable CCX for CLM*/
+	}
+
+	ODM_RT_TRACE(dm, ODM_COMP_CCX, "[%s] : CLM period = %dus\n", __func__,
+		     ccx_info->CLM_period * 4);
+}
+
+void phydm_clm_trigger(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+		odm_set_bb_reg(dm, ODM_REG_CLM_11AC, BIT(0),
+			       0x0); /*Trigger CLM*/
+		odm_set_bb_reg(dm, ODM_REG_CLM_11AC, BIT(0), 0x1);
+	} else if (dm->support_ic_type & ODM_IC_11N_SERIES) {
+		odm_set_bb_reg(dm, ODM_REG_CLM_11N, BIT(0),
+			       0x0); /*Trigger CLM*/
+		odm_set_bb_reg(dm, ODM_REG_CLM_11N, BIT(0), 0x1);
+	}
+}
+
+bool phydm_check_cl_mready(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 value32 = 0;
+	bool ret = false;
+
+	if (dm->support_ic_type & ODM_IC_11AC_SERIES)
+		value32 = odm_get_bb_reg(
+			dm, ODM_REG_CLM_RESULT_11AC,
+			MASKDWORD); /*make sure CLM calc is ready*/
+	else if (dm->support_ic_type & ODM_IC_11N_SERIES)
+		value32 = odm_get_bb_reg(
+			dm, ODM_REG_CLM_READY_11N,
+			MASKDWORD); /*make sure CLM calc is ready*/
+
+	if ((dm->support_ic_type & ODM_IC_11AC_SERIES) && (value32 & BIT(16)))
+		ret = true;
+	else if ((dm->support_ic_type & ODM_IC_11N_SERIES) &&
+		 (value32 & BIT(16)))
+		ret = true;
+	else
+		ret = false;
+
+	ODM_RT_TRACE(dm, ODM_COMP_CCX, "[%s] : CLM ready = %d\n", __func__,
+		     ret);
+
+	return ret;
+}
+
+void phydm_get_cl_mresult(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct ccx_info *ccx_info = &dm->dm_ccx_info;
+
+	u32 value32 = 0;
+
+	if (dm->support_ic_type & ODM_IC_11AC_SERIES)
+		value32 = odm_get_bb_reg(dm, ODM_REG_CLM_RESULT_11AC,
+					 MASKDWORD); /*read CLM calc result*/
+	else if (dm->support_ic_type & ODM_IC_11N_SERIES)
+		value32 = odm_get_bb_reg(dm, ODM_REG_CLM_RESULT_11N,
+					 MASKDWORD); /*read CLM calc result*/
+
+	ccx_info->CLM_result = (u16)(value32 & MASKLWORD);
+
+	ODM_RT_TRACE(dm, ODM_COMP_CCX, "[%s] : CLM result = %dus\n", __func__,
+		     ccx_info->CLM_result * 4);
+}
diff --git a/drivers/staging/rtlwifi/phydm/phydm_ccx.h b/drivers/staging/rtlwifi/phydm/phydm_ccx.h
new file mode 100644
index 000000000000..a3517f4642f9
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_ccx.h
@@ -0,0 +1,83 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef __PHYDMCCX_H__
+#define __PHYDMCCX_H__
+
+#define CCX_EN 1
+
+#define SET_NHM_SETTING 0
+#define STORE_NHM_SETTING 1
+#define RESTORE_NHM_SETTING 2
+
+enum nhm_inexclude_cca { NHM_EXCLUDE_CCA, NHM_INCLUDE_CCA };
+
+enum nhm_inexclude_txon { NHM_EXCLUDE_TXON, NHM_INCLUDE_TXON };
+
+struct ccx_info {
+	/*Settings*/
+	u8 NHM_th[11];
+	u16 NHM_period; /* 4us per unit */
+	u16 CLM_period; /* 4us per unit */
+	enum nhm_inexclude_txon nhm_inexclude_txon;
+	enum nhm_inexclude_cca nhm_inexclude_cca;
+
+	/*Previous Settings*/
+	u8 NHM_th_restore[11];
+	u16 NHM_period_restore; /* 4us per unit */
+	u16 CLM_period_restore; /* 4us per unit */
+	enum nhm_inexclude_txon NHM_inexclude_txon_restore;
+	enum nhm_inexclude_cca NHM_inexclude_cca_restore;
+
+	/*Report*/
+	u8 NHM_result[12];
+	u16 NHM_duration;
+	u16 CLM_result;
+
+	bool echo_NHM_en;
+	bool echo_CLM_en;
+	u8 echo_IGI;
+};
+
+/*NHM*/
+
+void phydm_nhm_setting(void *dm_void, u8 nhm_setting);
+
+void phydm_nhm_trigger(void *dm_void);
+
+void phydm_get_nhm_result(void *dm_void);
+
+bool phydm_check_nhm_ready(void *dm_void);
+
+/*CLM*/
+
+void phydm_clm_setting(void *dm_void);
+
+void phydm_clm_trigger(void *dm_void);
+
+bool phydm_check_cl_mready(void *dm_void);
+
+void phydm_get_cl_mresult(void *dm_void);
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/phydm_cfotracking.c b/drivers/staging/rtlwifi/phydm/phydm_cfotracking.c
new file mode 100644
index 000000000000..2ec8444f31a7
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_cfotracking.c
@@ -0,0 +1,343 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#include "mp_precomp.h"
+#include "phydm_precomp.h"
+
+static void odm_set_crystal_cap(void *dm_void, u8 crystal_cap)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct cfo_tracking *cfo_track =
+		(struct cfo_tracking *)phydm_get_structure(dm, PHYDM_CFOTRACK);
+
+	if (cfo_track->crystal_cap == crystal_cap)
+		return;
+
+	cfo_track->crystal_cap = crystal_cap;
+
+	if (dm->support_ic_type & (ODM_RTL8188E | ODM_RTL8188F)) {
+		/* write 0x24[22:17] = 0x24[16:11] = crystal_cap */
+		crystal_cap = crystal_cap & 0x3F;
+		odm_set_bb_reg(dm, REG_AFE_XTAL_CTRL, 0x007ff800,
+			       (crystal_cap | (crystal_cap << 6)));
+	} else if (dm->support_ic_type & ODM_RTL8812) {
+		/* write 0x2C[30:25] = 0x2C[24:19] = crystal_cap */
+		crystal_cap = crystal_cap & 0x3F;
+		odm_set_bb_reg(dm, REG_MAC_PHY_CTRL, 0x7FF80000,
+			       (crystal_cap | (crystal_cap << 6)));
+	} else if ((dm->support_ic_type & (ODM_RTL8703B | ODM_RTL8723B |
+					   ODM_RTL8192E | ODM_RTL8821))) {
+		/* 0x2C[23:18] = 0x2C[17:12] = crystal_cap */
+		crystal_cap = crystal_cap & 0x3F;
+		odm_set_bb_reg(dm, REG_MAC_PHY_CTRL, 0x00FFF000,
+			       (crystal_cap | (crystal_cap << 6)));
+	} else if (dm->support_ic_type & ODM_RTL8814A) {
+		/* write 0x2C[26:21] = 0x2C[20:15] = crystal_cap */
+		crystal_cap = crystal_cap & 0x3F;
+		odm_set_bb_reg(dm, REG_MAC_PHY_CTRL, 0x07FF8000,
+			       (crystal_cap | (crystal_cap << 6)));
+	} else if (dm->support_ic_type & (ODM_RTL8822B | ODM_RTL8821C)) {
+		/* write 0x24[30:25] = 0x28[6:1] = crystal_cap */
+		crystal_cap = crystal_cap & 0x3F;
+		odm_set_bb_reg(dm, REG_AFE_XTAL_CTRL, 0x7e000000, crystal_cap);
+		odm_set_bb_reg(dm, REG_AFE_PLL_CTRL, 0x7e, crystal_cap);
+	} else {
+		ODM_RT_TRACE(dm, ODM_COMP_CFO_TRACKING,
+			     "%s(): Use default setting.\n", __func__);
+		odm_set_bb_reg(dm, REG_MAC_PHY_CTRL, 0xFFF000,
+			       (crystal_cap | (crystal_cap << 6)));
+	}
+
+	ODM_RT_TRACE(dm, ODM_COMP_CFO_TRACKING, "%s(): crystal_cap = 0x%x\n",
+		     __func__, crystal_cap);
+
+	/* JJ modified 20161115 */
+}
+
+static u8 odm_get_default_crytaltal_cap(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u8 crystal_cap = 0x20;
+
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)dm->adapter;
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtlpriv);
+
+	crystal_cap = rtlefuse->crystalcap;
+
+	crystal_cap = crystal_cap & 0x3f;
+
+	return crystal_cap;
+}
+
+static void odm_set_atc_status(void *dm_void, bool atc_status)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct cfo_tracking *cfo_track =
+		(struct cfo_tracking *)phydm_get_structure(dm, PHYDM_CFOTRACK);
+
+	if (cfo_track->is_atc_status == atc_status)
+		return;
+
+	odm_set_bb_reg(dm, ODM_REG(BB_ATC, dm), ODM_BIT(BB_ATC, dm),
+		       atc_status);
+	cfo_track->is_atc_status = atc_status;
+}
+
+static bool odm_get_atc_status(void *dm_void)
+{
+	bool atc_status;
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	atc_status = (bool)odm_get_bb_reg(dm, ODM_REG(BB_ATC, dm),
+					  ODM_BIT(BB_ATC, dm));
+	return atc_status;
+}
+
+void odm_cfo_tracking_reset(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct cfo_tracking *cfo_track =
+		(struct cfo_tracking *)phydm_get_structure(dm, PHYDM_CFOTRACK);
+
+	cfo_track->def_x_cap = odm_get_default_crytaltal_cap(dm);
+	cfo_track->is_adjust = true;
+
+	if (cfo_track->crystal_cap > cfo_track->def_x_cap) {
+		odm_set_crystal_cap(dm, cfo_track->crystal_cap - 1);
+		ODM_RT_TRACE(dm, ODM_COMP_CFO_TRACKING,
+			     "%s(): approch default value (0x%x)\n", __func__,
+			     cfo_track->crystal_cap);
+	} else if (cfo_track->crystal_cap < cfo_track->def_x_cap) {
+		odm_set_crystal_cap(dm, cfo_track->crystal_cap + 1);
+		ODM_RT_TRACE(dm, ODM_COMP_CFO_TRACKING,
+			     "%s(): approch default value (0x%x)\n", __func__,
+			     cfo_track->crystal_cap);
+	}
+
+	odm_set_atc_status(dm, true);
+}
+
+void odm_cfo_tracking_init(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct cfo_tracking *cfo_track =
+		(struct cfo_tracking *)phydm_get_structure(dm, PHYDM_CFOTRACK);
+
+	cfo_track->crystal_cap = odm_get_default_crytaltal_cap(dm);
+	cfo_track->def_x_cap = cfo_track->crystal_cap;
+	cfo_track->is_atc_status = odm_get_atc_status(dm);
+	cfo_track->is_adjust = true;
+	ODM_RT_TRACE(dm, ODM_COMP_CFO_TRACKING, "%s()=========>\n", __func__);
+	ODM_RT_TRACE(dm, ODM_COMP_CFO_TRACKING,
+		     "%s(): is_atc_status = %d, crystal_cap = 0x%x\n", __func__,
+		     cfo_track->is_atc_status, cfo_track->def_x_cap);
+
+	/* Crystal cap. control by WiFi */
+	if (dm->support_ic_type & ODM_RTL8822B)
+		odm_set_bb_reg(dm, 0x10, 0x40, 0x1);
+}
+
+void odm_cfo_tracking(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct cfo_tracking *cfo_track =
+		(struct cfo_tracking *)phydm_get_structure(dm, PHYDM_CFOTRACK);
+	s32 cfo_ave = 0;
+	u32 cfo_rpt_sum, cfo_khz_avg[4] = {0};
+	s32 cfo_ave_diff;
+	s8 crystal_cap = cfo_track->crystal_cap;
+	u8 adjust_xtal = 1, i, valid_path_cnt = 0;
+
+	/* 4 Support ability */
+	if (!(dm->support_ability & ODM_BB_CFO_TRACKING)) {
+		ODM_RT_TRACE(
+			dm, ODM_COMP_CFO_TRACKING,
+			"%s(): Return: support_ability ODM_BB_CFO_TRACKING is disabled\n",
+			__func__);
+		return;
+	}
+
+	ODM_RT_TRACE(dm, ODM_COMP_CFO_TRACKING, "%s()=========>\n", __func__);
+
+	if (!dm->is_linked || !dm->is_one_entry_only) {
+		/* 4 No link or more than one entry */
+		odm_cfo_tracking_reset(dm);
+		ODM_RT_TRACE(
+			dm, ODM_COMP_CFO_TRACKING,
+			"%s(): Reset: is_linked = %d, is_one_entry_only = %d\n",
+			__func__, dm->is_linked, dm->is_one_entry_only);
+	} else {
+		/* 3 1. CFO Tracking */
+		/* 4 1.1 No new packet */
+		if (cfo_track->packet_count == cfo_track->packet_count_pre) {
+			ODM_RT_TRACE(dm, ODM_COMP_CFO_TRACKING,
+				     "%s(): packet counter doesn't change\n",
+				     __func__);
+			return;
+		}
+		cfo_track->packet_count_pre = cfo_track->packet_count;
+
+		/* 4 1.2 Calculate CFO */
+		for (i = 0; i < dm->num_rf_path; i++) {
+			if (cfo_track->CFO_cnt[i] == 0)
+				continue;
+
+			valid_path_cnt++;
+			cfo_rpt_sum =
+				(u32)((cfo_track->CFO_tail[i] < 0) ?
+					      (0 - cfo_track->CFO_tail[i]) :
+					      cfo_track->CFO_tail[i]);
+			cfo_khz_avg[i] = CFO_HW_RPT_2_MHZ(cfo_rpt_sum) /
+					 cfo_track->CFO_cnt[i];
+
+			ODM_RT_TRACE(
+				dm, ODM_COMP_CFO_TRACKING,
+				"[path %d] cfo_rpt_sum = (( %d )), CFO_cnt = (( %d )) , CFO_avg= (( %s%d )) kHz\n",
+				i, cfo_rpt_sum, cfo_track->CFO_cnt[i],
+				((cfo_track->CFO_tail[i] < 0) ? "-" : " "),
+				cfo_khz_avg[i]);
+		}
+
+		for (i = 0; i < valid_path_cnt; i++) {
+			if (cfo_track->CFO_tail[i] < 0) {
+				/* */
+				cfo_ave += (0 - (s32)cfo_khz_avg[i]);
+			} else {
+				cfo_ave += (s32)cfo_khz_avg[i];
+			}
+		}
+
+		if (valid_path_cnt >= 2)
+			cfo_ave = cfo_ave / valid_path_cnt;
+
+		ODM_RT_TRACE(dm, ODM_COMP_CFO_TRACKING,
+			     "valid_path_cnt = ((%d)), cfo_ave = ((%d kHz))\n",
+			     valid_path_cnt, cfo_ave);
+
+		/*reset counter*/
+		for (i = 0; i < dm->num_rf_path; i++) {
+			cfo_track->CFO_tail[i] = 0;
+			cfo_track->CFO_cnt[i] = 0;
+		}
+
+		/* 4 1.3 Avoid abnormal large CFO */
+		cfo_ave_diff = (cfo_track->CFO_ave_pre >= cfo_ave) ?
+				       (cfo_track->CFO_ave_pre - cfo_ave) :
+				       (cfo_ave - cfo_track->CFO_ave_pre);
+		if (cfo_ave_diff > 20 && cfo_track->large_cfo_hit == 0 &&
+		    !cfo_track->is_adjust) {
+			ODM_RT_TRACE(dm, ODM_COMP_CFO_TRACKING,
+				     "%s(): first large CFO hit\n", __func__);
+			cfo_track->large_cfo_hit = 1;
+			return;
+		}
+
+		cfo_track->large_cfo_hit = 0;
+		cfo_track->CFO_ave_pre = cfo_ave;
+
+		/* 4 1.4 Dynamic Xtal threshold */
+		if (!cfo_track->is_adjust) {
+			if (cfo_ave > CFO_TH_XTAL_HIGH ||
+			    cfo_ave < (-CFO_TH_XTAL_HIGH))
+				cfo_track->is_adjust = true;
+		} else {
+			if (cfo_ave < CFO_TH_XTAL_LOW &&
+			    cfo_ave > (-CFO_TH_XTAL_LOW))
+				cfo_track->is_adjust = false;
+		}
+
+		/* 4 1.5 BT case: Disable CFO tracking */
+		if (dm->is_bt_enabled) {
+			cfo_track->is_adjust = false;
+			odm_set_crystal_cap(dm, cfo_track->def_x_cap);
+			ODM_RT_TRACE(dm, ODM_COMP_CFO_TRACKING,
+				     "%s(): Disable CFO tracking for BT!!\n",
+				     __func__);
+		}
+
+		/* 4 1.7 Adjust Crystal Cap. */
+		if (cfo_track->is_adjust) {
+			if (cfo_ave > CFO_TH_XTAL_LOW)
+				crystal_cap = crystal_cap + adjust_xtal;
+			else if (cfo_ave < (-CFO_TH_XTAL_LOW))
+				crystal_cap = crystal_cap - adjust_xtal;
+
+			if (crystal_cap > 0x3f)
+				crystal_cap = 0x3f;
+			else if (crystal_cap < 0)
+				crystal_cap = 0;
+
+			odm_set_crystal_cap(dm, (u8)crystal_cap);
+		}
+		ODM_RT_TRACE(
+			dm, ODM_COMP_CFO_TRACKING,
+			"%s(): Crystal cap = 0x%x, Default Crystal cap = 0x%x\n",
+			__func__, cfo_track->crystal_cap, cfo_track->def_x_cap);
+
+		if (dm->support_ic_type & ODM_IC_11AC_SERIES)
+			return;
+
+		/* 3 2. Dynamic ATC switch */
+		if (cfo_ave < CFO_TH_ATC && cfo_ave > -CFO_TH_ATC) {
+			odm_set_atc_status(dm, false);
+			ODM_RT_TRACE(dm, ODM_COMP_CFO_TRACKING,
+				     "%s(): Disable ATC!!\n", __func__);
+		} else {
+			odm_set_atc_status(dm, true);
+			ODM_RT_TRACE(dm, ODM_COMP_CFO_TRACKING,
+				     "%s(): Enable ATC!!\n", __func__);
+		}
+	}
+}
+
+void odm_parsing_cfo(void *dm_void, void *pktinfo_void, s8 *pcfotail, u8 num_ss)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct dm_per_pkt_info *pktinfo =
+		(struct dm_per_pkt_info *)pktinfo_void;
+	struct cfo_tracking *cfo_track =
+		(struct cfo_tracking *)phydm_get_structure(dm, PHYDM_CFOTRACK);
+	u8 i;
+
+	if (!(dm->support_ability & ODM_BB_CFO_TRACKING))
+		return;
+
+	if (pktinfo->is_packet_match_bssid) {
+		if (num_ss > dm->num_rf_path) /*For fool proof*/
+			num_ss = dm->num_rf_path;
+
+		/* 3 Update CFO report for path-A & path-B */
+		/* Only paht-A and path-B have CFO tail and short CFO */
+		for (i = 0; i < num_ss; i++) {
+			cfo_track->CFO_tail[i] += pcfotail[i];
+			cfo_track->CFO_cnt[i]++;
+		}
+
+		/* 3 Update packet counter */
+		if (cfo_track->packet_count == 0xffffffff)
+			cfo_track->packet_count = 0;
+		else
+			cfo_track->packet_count++;
+	}
+}
diff --git a/drivers/staging/rtlwifi/phydm/phydm_cfotracking.h b/drivers/staging/rtlwifi/phydm/phydm_cfotracking.h
new file mode 100644
index 000000000000..e8436a31019d
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_cfotracking.h
@@ -0,0 +1,60 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __PHYDMCFOTRACK_H__
+#define __PHYDMCFOTRACK_H__
+
+#define CFO_TRACKING_VERSION "1.4" /*2015.10.01	Stanley, Modify for 8822B*/
+
+#define CFO_TH_XTAL_HIGH 20 /* kHz */
+#define CFO_TH_XTAL_LOW 10 /* kHz */
+#define CFO_TH_ATC 80 /* kHz */
+
+struct cfo_tracking {
+	bool is_atc_status;
+	bool large_cfo_hit;
+	bool is_adjust;
+	u8 crystal_cap;
+	u8 def_x_cap;
+	s32 CFO_tail[4];
+	u32 CFO_cnt[4];
+	s32 CFO_ave_pre;
+	u32 packet_count;
+	u32 packet_count_pre;
+
+	bool is_force_xtal_cap;
+	bool is_reset;
+};
+
+void odm_cfo_tracking_reset(void *dm_void);
+
+void odm_cfo_tracking_init(void *dm_void);
+
+void odm_cfo_tracking(void *dm_void);
+
+void odm_parsing_cfo(void *dm_void, void *pktinfo_void, s8 *pcfotail,
+		     u8 num_ss);
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/phydm_debug.c b/drivers/staging/rtlwifi/phydm/phydm_debug.c
new file mode 100644
index 000000000000..a5f90afdae9b
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_debug.c
@@ -0,0 +1,2910 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+/* ************************************************************
+ * include files
+ * *************************************************************/
+
+#include "mp_precomp.h"
+#include "phydm_precomp.h"
+
+bool phydm_api_set_txagc(struct phy_dm_struct *, u32, enum odm_rf_radio_path,
+			 u8, bool);
+static inline void phydm_check_dmval_txagc(struct phy_dm_struct *dm, u32 used,
+					   u32 out_len, u32 *const dm_value,
+					   char *output)
+{
+	if ((u8)dm_value[2] != 0xff) {
+		if (phydm_api_set_txagc(dm, dm_value[3],
+					(enum odm_rf_radio_path)dm_value[1],
+					(u8)dm_value[2], true))
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "  %s%d   %s%x%s%x\n", "Write path-",
+				       dm_value[1], "rate index-0x",
+				       dm_value[2], " = 0x", dm_value[3]);
+		else
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "  %s%d   %s%x%s\n", "Write path-",
+				       (dm_value[1] & 0x1), "rate index-0x",
+				       (dm_value[2] & 0x7f), " fail");
+	} else {
+		u8 i;
+		u32 power_index;
+		bool status = true;
+
+		power_index = (dm_value[3] & 0x3f);
+
+		if (dm->support_ic_type & (ODM_RTL8822B | ODM_RTL8821C)) {
+			power_index = (power_index << 24) |
+				      (power_index << 16) | (power_index << 8) |
+				      (power_index);
+			for (i = 0; i < ODM_RATEVHTSS2MCS9; i += 4)
+				status = (status &
+					  phydm_api_set_txagc(
+						  dm, power_index,
+						  (enum odm_rf_radio_path)
+							  dm_value[1],
+						  i, false));
+		} else if (dm->support_ic_type & ODM_RTL8197F) {
+			for (i = 0; i <= ODM_RATEMCS15; i++)
+				status = (status &
+					  phydm_api_set_txagc(
+						  dm, power_index,
+						  (enum odm_rf_radio_path)
+							  dm_value[1],
+						  i, false));
+		}
+
+		if (status)
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "  %s%d   %s%x\n",
+				       "Write all TXAGC of path-", dm_value[1],
+				       " = 0x", dm_value[3]);
+		else
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "  %s%d   %s\n",
+				       "Write all TXAGC of path-", dm_value[1],
+				       " fail");
+	}
+}
+
+static inline void phydm_print_nhm_trigger(char *output, u32 used, u32 out_len,
+					   struct ccx_info *ccx_info)
+{
+	int i;
+
+	for (i = 0; i <= 10; i++) {
+		if (i == 5)
+			PHYDM_SNPRINTF(
+				output + used, out_len - used,
+				"\r\n NHM_th[%d] = 0x%x, echo_IGI = 0x%x", i,
+				ccx_info->NHM_th[i], ccx_info->echo_IGI);
+		else if (i == 10)
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "\r\n NHM_th[%d] = 0x%x\n", i,
+				       ccx_info->NHM_th[i]);
+		else
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "\r\n NHM_th[%d] = 0x%x", i,
+				       ccx_info->NHM_th[i]);
+	}
+}
+
+static inline void phydm_print_nhm_result(char *output, u32 used, u32 out_len,
+					  struct ccx_info *ccx_info)
+{
+	int i;
+
+	for (i = 0; i <= 11; i++) {
+		if (i == 5)
+			PHYDM_SNPRINTF(
+				output + used, out_len - used,
+				"\r\n nhm_result[%d] = %d, echo_IGI = 0x%x", i,
+				ccx_info->NHM_result[i], ccx_info->echo_IGI);
+		else if (i == 11)
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "\r\n nhm_result[%d] = %d\n", i,
+				       ccx_info->NHM_result[i]);
+		else
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "\r\n nhm_result[%d] = %d", i,
+				       ccx_info->NHM_result[i]);
+	}
+}
+
+static inline void phydm_print_csi(struct phy_dm_struct *dm, u32 used,
+				   u32 out_len, char *output)
+{
+	int index, ptr;
+	u32 dword_h, dword_l;
+
+	for (index = 0; index < 80; index++) {
+		ptr = index + 256;
+
+		if (ptr > 311)
+			ptr -= 312;
+
+		odm_set_bb_reg(dm, 0x1910, 0x03FF0000, ptr); /*Select Address*/
+		dword_h = odm_get_bb_reg(dm, 0xF74, MASKDWORD);
+		dword_l = odm_get_bb_reg(dm, 0xF5C, MASKDWORD);
+
+		if (index % 2 == 0)
+			PHYDM_SNPRINTF(
+				output + used, out_len - used,
+				"%02x  %02x  %02x  %02x  %02x  %02x  %02x  %02x\n",
+				dword_l & MASKBYTE0, (dword_l & MASKBYTE1) >> 8,
+				(dword_l & MASKBYTE2) >> 16,
+				(dword_l & MASKBYTE3) >> 24,
+				dword_h & MASKBYTE0, (dword_h & MASKBYTE1) >> 8,
+				(dword_h & MASKBYTE2) >> 16,
+				(dword_h & MASKBYTE3) >> 24);
+		else
+			PHYDM_SNPRINTF(
+				output + used, out_len - used,
+				"%02x  %02x  %02x  %02x  %02x  %02x  %02x  %02x\n",
+				dword_l & MASKBYTE0, (dword_l & MASKBYTE1) >> 8,
+				(dword_l & MASKBYTE2) >> 16,
+				(dword_l & MASKBYTE3) >> 24,
+				dword_h & MASKBYTE0, (dword_h & MASKBYTE1) >> 8,
+				(dword_h & MASKBYTE2) >> 16,
+				(dword_h & MASKBYTE3) >> 24);
+	}
+}
+
+void phydm_init_debug_setting(struct phy_dm_struct *dm)
+{
+	dm->debug_level = ODM_DBG_TRACE;
+
+	dm->fw_debug_components = 0;
+	dm->debug_components =
+
+		0;
+
+	dm->fw_buff_is_enpty = true;
+	dm->pre_c2h_seq = 0;
+}
+
+u8 phydm_set_bb_dbg_port(void *dm_void, u8 curr_dbg_priority, u32 debug_port)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u8 dbg_port_result = false;
+
+	if (curr_dbg_priority > dm->pre_dbg_priority) {
+		if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+			odm_set_bb_reg(dm, 0x8fc, MASKDWORD, debug_port);
+			/**/
+		} else /*if (dm->support_ic_type & ODM_IC_11N_SERIES)*/ {
+			odm_set_bb_reg(dm, 0x908, MASKDWORD, debug_port);
+			/**/
+		}
+		ODM_RT_TRACE(
+			dm, ODM_COMP_API,
+			"DbgPort set success, Reg((0x%x)), Cur_priority=((%d)), Pre_priority=((%d))\n",
+			debug_port, curr_dbg_priority, dm->pre_dbg_priority);
+		dm->pre_dbg_priority = curr_dbg_priority;
+		dbg_port_result = true;
+	}
+
+	return dbg_port_result;
+}
+
+void phydm_release_bb_dbg_port(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	dm->pre_dbg_priority = BB_DBGPORT_RELEASE;
+	ODM_RT_TRACE(dm, ODM_COMP_API, "Release BB dbg_port\n");
+}
+
+u32 phydm_get_bb_dbg_port_value(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 dbg_port_value = 0;
+
+	if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+		dbg_port_value = odm_get_bb_reg(dm, 0xfa0, MASKDWORD);
+		/**/
+	} else /*if (dm->support_ic_type & ODM_IC_11N_SERIES)*/ {
+		dbg_port_value = odm_get_bb_reg(dm, 0xdf4, MASKDWORD);
+		/**/
+	}
+	ODM_RT_TRACE(dm, ODM_COMP_API, "dbg_port_value = 0x%x\n",
+		     dbg_port_value);
+	return dbg_port_value;
+}
+
+static void phydm_bb_rx_hang_info(void *dm_void, u32 *_used, char *output,
+				  u32 *_out_len)
+{
+	u32 value32 = 0;
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 used = *_used;
+	u32 out_len = *_out_len;
+
+	if (dm->support_ic_type & ODM_IC_11N_SERIES)
+		return;
+
+	value32 = odm_get_bb_reg(dm, 0xF80, MASKDWORD);
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s = 0x%x",
+		       "rptreg of sc/bw/ht/...", value32);
+
+	if (dm->support_ic_type & ODM_RTL8822B)
+		odm_set_bb_reg(dm, 0x198c, BIT(2) | BIT(1) | BIT(0), 7);
+
+	/* dbg_port = basic state machine */
+	{
+		odm_set_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD, 0x000);
+		value32 = odm_get_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "0x8fc", value32);
+
+		value32 = odm_get_bb_reg(dm, ODM_REG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "basic state machine",
+			       value32);
+	}
+
+	/* dbg_port = state machine */
+	{
+		odm_set_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD, 0x007);
+		value32 = odm_get_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "0x8fc", value32);
+
+		value32 = odm_get_bb_reg(dm, ODM_REG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "state machine", value32);
+	}
+
+	/* dbg_port = CCA-related*/
+	{
+		odm_set_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD, 0x204);
+		value32 = odm_get_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "0x8fc", value32);
+
+		value32 = odm_get_bb_reg(dm, ODM_REG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "CCA-related", value32);
+	}
+
+	/* dbg_port = edcca/rxd*/
+	{
+		odm_set_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD, 0x278);
+		value32 = odm_get_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "0x8fc", value32);
+
+		value32 = odm_get_bb_reg(dm, ODM_REG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "edcca/rxd", value32);
+	}
+
+	/* dbg_port = rx_state/mux_state/ADC_MASK_OFDM*/
+	{
+		odm_set_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD, 0x290);
+		value32 = odm_get_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "0x8fc", value32);
+
+		value32 = odm_get_bb_reg(dm, ODM_REG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x",
+			       "rx_state/mux_state/ADC_MASK_OFDM", value32);
+	}
+
+	/* dbg_port = bf-related*/
+	{
+		odm_set_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD, 0x2B2);
+		value32 = odm_get_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "0x8fc", value32);
+
+		value32 = odm_get_bb_reg(dm, ODM_REG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "bf-related", value32);
+	}
+
+	/* dbg_port = bf-related*/
+	{
+		odm_set_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD, 0x2B8);
+		value32 = odm_get_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "0x8fc", value32);
+
+		value32 = odm_get_bb_reg(dm, ODM_REG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "bf-related", value32);
+	}
+
+	/* dbg_port = txon/rxd*/
+	{
+		odm_set_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD, 0xA03);
+		value32 = odm_get_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "0x8fc", value32);
+
+		value32 = odm_get_bb_reg(dm, ODM_REG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "txon/rxd", value32);
+	}
+
+	/* dbg_port = l_rate/l_length*/
+	{
+		odm_set_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD, 0xA0B);
+		value32 = odm_get_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "0x8fc", value32);
+
+		value32 = odm_get_bb_reg(dm, ODM_REG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "l_rate/l_length", value32);
+	}
+
+	/* dbg_port = rxd/rxd_hit*/
+	{
+		odm_set_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD, 0xA0D);
+		value32 = odm_get_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "0x8fc", value32);
+
+		value32 = odm_get_bb_reg(dm, ODM_REG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "rxd/rxd_hit", value32);
+	}
+
+	/* dbg_port = dis_cca*/
+	{
+		odm_set_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD, 0xAA0);
+		value32 = odm_get_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "0x8fc", value32);
+
+		value32 = odm_get_bb_reg(dm, ODM_REG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "dis_cca", value32);
+	}
+
+	/* dbg_port = tx*/
+	{
+		odm_set_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD, 0xAB0);
+		value32 = odm_get_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "0x8fc", value32);
+
+		value32 = odm_get_bb_reg(dm, ODM_REG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "tx", value32);
+	}
+
+	/* dbg_port = rx plcp*/
+	{
+		odm_set_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD, 0xAD0);
+		value32 = odm_get_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "0x8fc", value32);
+
+		value32 = odm_get_bb_reg(dm, ODM_REG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "rx plcp", value32);
+
+		odm_set_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD, 0xAD1);
+		value32 = odm_get_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "0x8fc", value32);
+
+		value32 = odm_get_bb_reg(dm, ODM_REG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "rx plcp", value32);
+
+		odm_set_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD, 0xAD2);
+		value32 = odm_get_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "0x8fc", value32);
+
+		value32 = odm_get_bb_reg(dm, ODM_REG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "rx plcp", value32);
+
+		odm_set_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD, 0xAD3);
+		value32 = odm_get_bb_reg(dm, ODM_REG_DBG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "0x8fc", value32);
+
+		value32 = odm_get_bb_reg(dm, ODM_REG_RPT_11AC, MASKDWORD);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = 0x%x", "rx plcp", value32);
+	}
+}
+
+static void phydm_bb_debug_info_n_series(void *dm_void, u32 *_used,
+					 char *output, u32 *_out_len)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 used = *_used;
+	u32 out_len = *_out_len;
+
+	u32 value32 = 0, value32_1 = 0;
+	u8 rf_gain_a = 0, rf_gain_b = 0, rf_gain_c = 0, rf_gain_d = 0;
+	u8 rx_snr_a = 0, rx_snr_b = 0, rx_snr_c = 0, rx_snr_d = 0;
+
+	s8 rxevm_0 = 0, rxevm_1 = 0;
+	s32 short_cfo_a = 0, short_cfo_b = 0, long_cfo_a = 0, long_cfo_b = 0;
+	s32 scfo_a = 0, scfo_b = 0, avg_cfo_a = 0, avg_cfo_b = 0;
+	s32 cfo_end_a = 0, cfo_end_b = 0, acq_cfo_a = 0, acq_cfo_b = 0;
+
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s\n",
+		       "BB Report Info");
+
+	/*AGC result*/
+	value32 = odm_get_bb_reg(dm, 0xdd0, MASKDWORD);
+	rf_gain_a = (u8)(value32 & 0x3f);
+	rf_gain_a = rf_gain_a << 1;
+
+	rf_gain_b = (u8)((value32 >> 8) & 0x3f);
+	rf_gain_b = rf_gain_b << 1;
+
+	rf_gain_c = (u8)((value32 >> 16) & 0x3f);
+	rf_gain_c = rf_gain_c << 1;
+
+	rf_gain_d = (u8)((value32 >> 24) & 0x3f);
+	rf_gain_d = rf_gain_d << 1;
+
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "\r\n %-35s = %d / %d / %d / %d",
+		       "OFDM RX RF Gain(A/B/C/D)", rf_gain_a, rf_gain_b,
+		       rf_gain_c, rf_gain_d);
+
+	/*SNR report*/
+	value32 = odm_get_bb_reg(dm, 0xdd4, MASKDWORD);
+	rx_snr_a = (u8)(value32 & 0xff);
+	rx_snr_a = rx_snr_a >> 1;
+
+	rx_snr_b = (u8)((value32 >> 8) & 0xff);
+	rx_snr_b = rx_snr_b >> 1;
+
+	rx_snr_c = (u8)((value32 >> 16) & 0xff);
+	rx_snr_c = rx_snr_c >> 1;
+
+	rx_snr_d = (u8)((value32 >> 24) & 0xff);
+	rx_snr_d = rx_snr_d >> 1;
+
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "\r\n %-35s = %d / %d / %d / %d", "RXSNR(A/B/C/D, dB)",
+		       rx_snr_a, rx_snr_b, rx_snr_c, rx_snr_d);
+
+	/* PostFFT related info*/
+	value32 = odm_get_bb_reg(dm, 0xdd8, MASKDWORD);
+
+	rxevm_0 = (s8)((value32 & MASKBYTE2) >> 16);
+	rxevm_0 /= 2;
+	if (rxevm_0 < -63)
+		rxevm_0 = 0;
+
+	rxevm_1 = (s8)((value32 & MASKBYTE3) >> 24);
+	rxevm_1 /= 2;
+	if (rxevm_1 < -63)
+		rxevm_1 = 0;
+
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s = %d / %d",
+		       "RXEVM (1ss/2ss)", rxevm_0, rxevm_1);
+
+	/*CFO Report Info*/
+	odm_set_bb_reg(dm, 0xd00, BIT(26), 1);
+
+	/*Short CFO*/
+	value32 = odm_get_bb_reg(dm, 0xdac, MASKDWORD);
+	value32_1 = odm_get_bb_reg(dm, 0xdb0, MASKDWORD);
+
+	short_cfo_b = (s32)(value32 & 0xfff); /*S(12,11)*/
+	short_cfo_a = (s32)((value32 & 0x0fff0000) >> 16);
+
+	long_cfo_b = (s32)(value32_1 & 0x1fff); /*S(13,12)*/
+	long_cfo_a = (s32)((value32_1 & 0x1fff0000) >> 16);
+
+	/*SFO 2's to dec*/
+	if (short_cfo_a > 2047)
+		short_cfo_a = short_cfo_a - 4096;
+	if (short_cfo_b > 2047)
+		short_cfo_b = short_cfo_b - 4096;
+
+	short_cfo_a = (short_cfo_a * 312500) / 2048;
+	short_cfo_b = (short_cfo_b * 312500) / 2048;
+
+	/*LFO 2's to dec*/
+
+	if (long_cfo_a > 4095)
+		long_cfo_a = long_cfo_a - 8192;
+
+	if (long_cfo_b > 4095)
+		long_cfo_b = long_cfo_b - 8192;
+
+	long_cfo_a = long_cfo_a * 312500 / 4096;
+	long_cfo_b = long_cfo_b * 312500 / 4096;
+
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s",
+		       "CFO Report Info");
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s = %d / %d",
+		       "Short CFO(Hz) <A/B>", short_cfo_a, short_cfo_b);
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s = %d / %d",
+		       "Long CFO(Hz) <A/B>", long_cfo_a, long_cfo_b);
+
+	/*SCFO*/
+	value32 = odm_get_bb_reg(dm, 0xdb8, MASKDWORD);
+	value32_1 = odm_get_bb_reg(dm, 0xdb4, MASKDWORD);
+
+	scfo_b = (s32)(value32 & 0x7ff); /*S(11,10)*/
+	scfo_a = (s32)((value32 & 0x07ff0000) >> 16);
+
+	if (scfo_a > 1023)
+		scfo_a = scfo_a - 2048;
+
+	if (scfo_b > 1023)
+		scfo_b = scfo_b - 2048;
+
+	scfo_a = scfo_a * 312500 / 1024;
+	scfo_b = scfo_b * 312500 / 1024;
+
+	avg_cfo_b = (s32)(value32_1 & 0x1fff); /*S(13,12)*/
+	avg_cfo_a = (s32)((value32_1 & 0x1fff0000) >> 16);
+
+	if (avg_cfo_a > 4095)
+		avg_cfo_a = avg_cfo_a - 8192;
+
+	if (avg_cfo_b > 4095)
+		avg_cfo_b = avg_cfo_b - 8192;
+
+	avg_cfo_a = avg_cfo_a * 312500 / 4096;
+	avg_cfo_b = avg_cfo_b * 312500 / 4096;
+
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s = %d / %d",
+		       "value SCFO(Hz) <A/B>", scfo_a, scfo_b);
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s = %d / %d",
+		       "Avg CFO(Hz) <A/B>", avg_cfo_a, avg_cfo_b);
+
+	value32 = odm_get_bb_reg(dm, 0xdbc, MASKDWORD);
+	value32_1 = odm_get_bb_reg(dm, 0xde0, MASKDWORD);
+
+	cfo_end_b = (s32)(value32 & 0x1fff); /*S(13,12)*/
+	cfo_end_a = (s32)((value32 & 0x1fff0000) >> 16);
+
+	if (cfo_end_a > 4095)
+		cfo_end_a = cfo_end_a - 8192;
+
+	if (cfo_end_b > 4095)
+		cfo_end_b = cfo_end_b - 8192;
+
+	cfo_end_a = cfo_end_a * 312500 / 4096;
+	cfo_end_b = cfo_end_b * 312500 / 4096;
+
+	acq_cfo_b = (s32)(value32_1 & 0x1fff); /*S(13,12)*/
+	acq_cfo_a = (s32)((value32_1 & 0x1fff0000) >> 16);
+
+	if (acq_cfo_a > 4095)
+		acq_cfo_a = acq_cfo_a - 8192;
+
+	if (acq_cfo_b > 4095)
+		acq_cfo_b = acq_cfo_b - 8192;
+
+	acq_cfo_a = acq_cfo_a * 312500 / 4096;
+	acq_cfo_b = acq_cfo_b * 312500 / 4096;
+
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s = %d / %d",
+		       "End CFO(Hz) <A/B>", cfo_end_a, cfo_end_b);
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s = %d / %d",
+		       "ACQ CFO(Hz) <A/B>", acq_cfo_a, acq_cfo_b);
+}
+
+static void phydm_bb_debug_info(void *dm_void, u32 *_used, char *output,
+				u32 *_out_len)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 used = *_used;
+	u32 out_len = *_out_len;
+
+	char *tmp_string = NULL;
+
+	u8 rx_ht_bw, rx_vht_bw, rxsc, rx_ht, rx_bw;
+	static u8 v_rx_bw;
+	u32 value32, value32_1, value32_2, value32_3;
+	s32 sfo_a, sfo_b, sfo_c, sfo_d;
+	s32 lfo_a, lfo_b, lfo_c, lfo_d;
+	static u8 MCSS, tail, parity, rsv, vrsv, idx, smooth, htsound, agg,
+		stbc, vstbc, fec, fecext, sgi, sgiext, htltf, vgid, v_nsts,
+		vtxops, vrsv2, vbrsv, bf, vbcrc;
+	static u16 h_length, htcrc8, length;
+	static u16 vpaid;
+	static u16 v_length, vhtcrc8, v_mcss, v_tail, vb_tail;
+	static u8 hmcss, hrx_bw;
+
+	u8 pwdb;
+	s8 rxevm_0, rxevm_1, rxevm_2;
+	u8 rf_gain_path_a, rf_gain_path_b, rf_gain_path_c, rf_gain_path_d;
+	u8 rx_snr_path_a, rx_snr_path_b, rx_snr_path_c, rx_snr_path_d;
+	s32 sig_power;
+
+	const char *L_rate[8] = {"6M",  "9M",  "12M", "18M",
+				 "24M", "36M", "48M", "54M"};
+
+	if (dm->support_ic_type & ODM_IC_11N_SERIES) {
+		phydm_bb_debug_info_n_series(dm, &used, output, &out_len);
+		return;
+	}
+
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s\n",
+		       "BB Report Info");
+
+	/*BW & mode Detection*/
+
+	value32 = odm_get_bb_reg(dm, 0xf80, MASKDWORD);
+	value32_2 = value32;
+	rx_ht_bw = (u8)(value32 & 0x1);
+	rx_vht_bw = (u8)((value32 >> 1) & 0x3);
+	rxsc = (u8)(value32 & 0x78);
+	value32_1 = (value32 & 0x180) >> 7;
+	rx_ht = (u8)(value32_1);
+
+	rx_bw = 0;
+
+	if (rx_ht == 2) {
+		if (rx_vht_bw == 0)
+			tmp_string = "20M";
+		else if (rx_vht_bw == 1)
+			tmp_string = "40M";
+		else
+			tmp_string = "80M";
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s %s %s", "mode", "VHT", tmp_string);
+		rx_bw = rx_vht_bw;
+	} else if (rx_ht == 1) {
+		if (rx_ht_bw == 0)
+			tmp_string = "20M";
+		else if (rx_ht_bw == 1)
+			tmp_string = "40M";
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s %s %s", "mode", "HT", tmp_string);
+		rx_bw = rx_ht_bw;
+	} else {
+		PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s %s",
+			       "mode", "Legacy");
+	}
+	if (rx_ht != 0) {
+		if (rxsc == 0)
+			tmp_string = "duplicate/full bw";
+		else if (rxsc == 1)
+			tmp_string = "usc20-1";
+		else if (rxsc == 2)
+			tmp_string = "lsc20-1";
+		else if (rxsc == 3)
+			tmp_string = "usc20-2";
+		else if (rxsc == 4)
+			tmp_string = "lsc20-2";
+		else if (rxsc == 9)
+			tmp_string = "usc40";
+		else if (rxsc == 10)
+			tmp_string = "lsc40";
+		PHYDM_SNPRINTF(output + used, out_len - used, "  %-35s",
+			       tmp_string);
+	}
+
+	/* RX signal power and AGC related info*/
+
+	value32 = odm_get_bb_reg(dm, 0xF90, MASKDWORD);
+	pwdb = (u8)((value32 & MASKBYTE1) >> 8);
+	pwdb = pwdb >> 1;
+	sig_power = -110 + pwdb;
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s = %d",
+		       "OFDM RX Signal Power(dB)", sig_power);
+
+	value32 = odm_get_bb_reg(dm, 0xd14, MASKDWORD);
+	rx_snr_path_a = (u8)(value32 & 0xFF) >> 1;
+	rf_gain_path_a = (s8)((value32 & MASKBYTE1) >> 8);
+	rf_gain_path_a *= 2;
+	value32 = odm_get_bb_reg(dm, 0xd54, MASKDWORD);
+	rx_snr_path_b = (u8)(value32 & 0xFF) >> 1;
+	rf_gain_path_b = (s8)((value32 & MASKBYTE1) >> 8);
+	rf_gain_path_b *= 2;
+	value32 = odm_get_bb_reg(dm, 0xd94, MASKDWORD);
+	rx_snr_path_c = (u8)(value32 & 0xFF) >> 1;
+	rf_gain_path_c = (s8)((value32 & MASKBYTE1) >> 8);
+	rf_gain_path_c *= 2;
+	value32 = odm_get_bb_reg(dm, 0xdd4, MASKDWORD);
+	rx_snr_path_d = (u8)(value32 & 0xFF) >> 1;
+	rf_gain_path_d = (s8)((value32 & MASKBYTE1) >> 8);
+	rf_gain_path_d *= 2;
+
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "\r\n %-35s = %d / %d / %d / %d",
+		       "OFDM RX RF Gain(A/B/C/D)", rf_gain_path_a,
+		       rf_gain_path_b, rf_gain_path_c, rf_gain_path_d);
+
+	/* RX counter related info*/
+
+	value32 = odm_get_bb_reg(dm, 0xF08, MASKDWORD);
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s = %d",
+		       "OFDM CCA counter", ((value32 & 0xFFFF0000) >> 16));
+
+	value32 = odm_get_bb_reg(dm, 0xFD0, MASKDWORD);
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s = %d",
+		       "OFDM SBD Fail counter", value32 & 0xFFFF);
+
+	value32 = odm_get_bb_reg(dm, 0xFC4, MASKDWORD);
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s = %d / %d",
+		       "VHT SIGA/SIGB CRC8 Fail counter", value32 & 0xFFFF,
+		       ((value32 & 0xFFFF0000) >> 16));
+
+	value32 = odm_get_bb_reg(dm, 0xFCC, MASKDWORD);
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s = %d",
+		       "CCK CCA counter", value32 & 0xFFFF);
+
+	value32 = odm_get_bb_reg(dm, 0xFBC, MASKDWORD);
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s = %d / %d",
+		       "LSIG (parity Fail/rate Illegal) counter",
+		       value32 & 0xFFFF, ((value32 & 0xFFFF0000) >> 16));
+
+	value32_1 = odm_get_bb_reg(dm, 0xFC8, MASKDWORD);
+	value32_2 = odm_get_bb_reg(dm, 0xFC0, MASKDWORD);
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s = %d / %d",
+		       "HT/VHT MCS NOT SUPPORT counter",
+		       ((value32_2 & 0xFFFF0000) >> 16), value32_1 & 0xFFFF);
+
+	/* PostFFT related info*/
+	value32 = odm_get_bb_reg(dm, 0xF8c, MASKDWORD);
+	rxevm_0 = (s8)((value32 & MASKBYTE2) >> 16);
+	rxevm_0 /= 2;
+	if (rxevm_0 < -63)
+		rxevm_0 = 0;
+
+	rxevm_1 = (s8)((value32 & MASKBYTE3) >> 24);
+	rxevm_1 /= 2;
+	value32 = odm_get_bb_reg(dm, 0xF88, MASKDWORD);
+	rxevm_2 = (s8)((value32 & MASKBYTE2) >> 16);
+	rxevm_2 /= 2;
+
+	if (rxevm_1 < -63)
+		rxevm_1 = 0;
+	if (rxevm_2 < -63)
+		rxevm_2 = 0;
+
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "\r\n %-35s = %d / %d / %d", "RXEVM (1ss/2ss/3ss)",
+		       rxevm_0, rxevm_1, rxevm_2);
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "\r\n %-35s = %d / %d / %d / %d", "RXSNR(A/B/C/D, dB)",
+		       rx_snr_path_a, rx_snr_path_b, rx_snr_path_c,
+		       rx_snr_path_d);
+
+	value32 = odm_get_bb_reg(dm, 0xF8C, MASKDWORD);
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s = %d / %d",
+		       "CSI_1st /CSI_2nd", value32 & 0xFFFF,
+		       ((value32 & 0xFFFF0000) >> 16));
+
+	/*BW & mode Detection*/
+
+	/*Reset Page F counter*/
+	odm_set_bb_reg(dm, 0xB58, BIT(0), 1);
+	odm_set_bb_reg(dm, 0xB58, BIT(0), 0);
+
+	/*CFO Report Info*/
+	/*Short CFO*/
+	value32 = odm_get_bb_reg(dm, 0xd0c, MASKDWORD);
+	value32_1 = odm_get_bb_reg(dm, 0xd4c, MASKDWORD);
+	value32_2 = odm_get_bb_reg(dm, 0xd8c, MASKDWORD);
+	value32_3 = odm_get_bb_reg(dm, 0xdcc, MASKDWORD);
+
+	sfo_a = (s32)(value32 & 0xfff);
+	sfo_b = (s32)(value32_1 & 0xfff);
+	sfo_c = (s32)(value32_2 & 0xfff);
+	sfo_d = (s32)(value32_3 & 0xfff);
+
+	lfo_a = (s32)(value32 >> 16);
+	lfo_b = (s32)(value32_1 >> 16);
+	lfo_c = (s32)(value32_2 >> 16);
+	lfo_d = (s32)(value32_3 >> 16);
+
+	/*SFO 2's to dec*/
+	if (sfo_a > 2047)
+		sfo_a = sfo_a - 4096;
+	sfo_a = (sfo_a * 312500) / 2048;
+	if (sfo_b > 2047)
+		sfo_b = sfo_b - 4096;
+	sfo_b = (sfo_b * 312500) / 2048;
+	if (sfo_c > 2047)
+		sfo_c = sfo_c - 4096;
+	sfo_c = (sfo_c * 312500) / 2048;
+	if (sfo_d > 2047)
+		sfo_d = sfo_d - 4096;
+	sfo_d = (sfo_d * 312500) / 2048;
+
+	/*LFO 2's to dec*/
+
+	if (lfo_a > 4095)
+		lfo_a = lfo_a - 8192;
+
+	if (lfo_b > 4095)
+		lfo_b = lfo_b - 8192;
+
+	if (lfo_c > 4095)
+		lfo_c = lfo_c - 8192;
+
+	if (lfo_d > 4095)
+		lfo_d = lfo_d - 8192;
+	lfo_a = lfo_a * 312500 / 4096;
+	lfo_b = lfo_b * 312500 / 4096;
+	lfo_c = lfo_c * 312500 / 4096;
+	lfo_d = lfo_d * 312500 / 4096;
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s",
+		       "CFO Report Info");
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "\r\n %-35s = %d / %d / %d /%d",
+		       "Short CFO(Hz) <A/B/C/D>", sfo_a, sfo_b, sfo_c, sfo_d);
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "\r\n %-35s = %d / %d / %d /%d",
+		       "Long CFO(Hz) <A/B/C/D>", lfo_a, lfo_b, lfo_c, lfo_d);
+
+	/*SCFO*/
+	value32 = odm_get_bb_reg(dm, 0xd10, MASKDWORD);
+	value32_1 = odm_get_bb_reg(dm, 0xd50, MASKDWORD);
+	value32_2 = odm_get_bb_reg(dm, 0xd90, MASKDWORD);
+	value32_3 = odm_get_bb_reg(dm, 0xdd0, MASKDWORD);
+
+	sfo_a = (s32)(value32 & 0x7ff);
+	sfo_b = (s32)(value32_1 & 0x7ff);
+	sfo_c = (s32)(value32_2 & 0x7ff);
+	sfo_d = (s32)(value32_3 & 0x7ff);
+
+	if (sfo_a > 1023)
+		sfo_a = sfo_a - 2048;
+
+	if (sfo_b > 2047)
+		sfo_b = sfo_b - 4096;
+
+	if (sfo_c > 2047)
+		sfo_c = sfo_c - 4096;
+
+	if (sfo_d > 2047)
+		sfo_d = sfo_d - 4096;
+
+	sfo_a = sfo_a * 312500 / 1024;
+	sfo_b = sfo_b * 312500 / 1024;
+	sfo_c = sfo_c * 312500 / 1024;
+	sfo_d = sfo_d * 312500 / 1024;
+
+	lfo_a = (s32)(value32 >> 16);
+	lfo_b = (s32)(value32_1 >> 16);
+	lfo_c = (s32)(value32_2 >> 16);
+	lfo_d = (s32)(value32_3 >> 16);
+
+	if (lfo_a > 4095)
+		lfo_a = lfo_a - 8192;
+
+	if (lfo_b > 4095)
+		lfo_b = lfo_b - 8192;
+
+	if (lfo_c > 4095)
+		lfo_c = lfo_c - 8192;
+
+	if (lfo_d > 4095)
+		lfo_d = lfo_d - 8192;
+	lfo_a = lfo_a * 312500 / 4096;
+	lfo_b = lfo_b * 312500 / 4096;
+	lfo_c = lfo_c * 312500 / 4096;
+	lfo_d = lfo_d * 312500 / 4096;
+
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "\r\n %-35s = %d / %d / %d /%d",
+		       "value SCFO(Hz) <A/B/C/D>", sfo_a, sfo_b, sfo_c, sfo_d);
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "\r\n %-35s = %d / %d / %d /%d", "ACQ CFO(Hz) <A/B/C/D>",
+		       lfo_a, lfo_b, lfo_c, lfo_d);
+
+	value32 = odm_get_bb_reg(dm, 0xd14, MASKDWORD);
+	value32_1 = odm_get_bb_reg(dm, 0xd54, MASKDWORD);
+	value32_2 = odm_get_bb_reg(dm, 0xd94, MASKDWORD);
+	value32_3 = odm_get_bb_reg(dm, 0xdd4, MASKDWORD);
+
+	lfo_a = (s32)(value32 >> 16);
+	lfo_b = (s32)(value32_1 >> 16);
+	lfo_c = (s32)(value32_2 >> 16);
+	lfo_d = (s32)(value32_3 >> 16);
+
+	if (lfo_a > 4095)
+		lfo_a = lfo_a - 8192;
+
+	if (lfo_b > 4095)
+		lfo_b = lfo_b - 8192;
+
+	if (lfo_c > 4095)
+		lfo_c = lfo_c - 8192;
+
+	if (lfo_d > 4095)
+		lfo_d = lfo_d - 8192;
+
+	lfo_a = lfo_a * 312500 / 4096;
+	lfo_b = lfo_b * 312500 / 4096;
+	lfo_c = lfo_c * 312500 / 4096;
+	lfo_d = lfo_d * 312500 / 4096;
+
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "\r\n %-35s = %d / %d / %d /%d", "End CFO(Hz) <A/B/C/D>",
+		       lfo_a, lfo_b, lfo_c, lfo_d);
+
+	value32 = odm_get_bb_reg(dm, 0xf20, MASKDWORD); /*L SIG*/
+
+	tail = (u8)((value32 & 0xfc0000) >> 16);
+	parity = (u8)((value32 & 0x20000) >> 16);
+	length = (u16)((value32 & 0x1ffe00) >> 8);
+	rsv = (u8)(value32 & 0x10);
+	MCSS = (u8)(value32 & 0x0f);
+
+	switch (MCSS) {
+	case 0x0b:
+		idx = 0;
+		break;
+	case 0x0f:
+		idx = 1;
+		break;
+	case 0x0a:
+		idx = 2;
+		break;
+	case 0x0e:
+		idx = 3;
+		break;
+	case 0x09:
+		idx = 4;
+		break;
+	case 0x08:
+		idx = 5;
+		break;
+	case 0x0c:
+		idx = 6;
+		break;
+	default:
+		idx = 6;
+		break;
+	}
+
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s", "L-SIG");
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s : %s", "rate",
+		       L_rate[idx]);
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "\r\n %-35s = %x / %x / %x", "Rsv/length/parity", rsv,
+		       rx_bw, length);
+
+	value32 = odm_get_bb_reg(dm, 0xf2c, MASKDWORD); /*HT SIG*/
+	if (rx_ht == 1) {
+		hmcss = (u8)(value32 & 0x7F);
+		hrx_bw = (u8)(value32 & 0x80);
+		h_length = (u16)((value32 >> 8) & 0xffff);
+	}
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s", "HT-SIG1");
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "\r\n %-35s = %x / %x / %x", "MCS/BW/length", hmcss,
+		       hrx_bw, h_length);
+
+	value32 = odm_get_bb_reg(dm, 0xf30, MASKDWORD); /*HT SIG*/
+
+	if (rx_ht == 1) {
+		smooth = (u8)(value32 & 0x01);
+		htsound = (u8)(value32 & 0x02);
+		rsv = (u8)(value32 & 0x04);
+		agg = (u8)(value32 & 0x08);
+		stbc = (u8)(value32 & 0x30);
+		fec = (u8)(value32 & 0x40);
+		sgi = (u8)(value32 & 0x80);
+		htltf = (u8)((value32 & 0x300) >> 8);
+		htcrc8 = (u16)((value32 & 0x3fc00) >> 8);
+		tail = (u8)((value32 & 0xfc0000) >> 16);
+	}
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s", "HT-SIG2");
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "\r\n %-35s = %x / %x / %x / %x / %x / %x",
+		       "Smooth/NoSound/Rsv/Aggregate/STBC/LDPC", smooth,
+		       htsound, rsv, agg, stbc, fec);
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "\r\n %-35s = %x / %x / %x / %x",
+		       "SGI/E-HT-LTFs/CRC/tail", sgi, htltf, htcrc8, tail);
+
+	value32 = odm_get_bb_reg(dm, 0xf2c, MASKDWORD); /*VHT SIG A1*/
+	if (rx_ht == 2) {
+		/* value32 = odm_get_bb_reg(dm, 0xf2c,MASKDWORD);*/
+		v_rx_bw = (u8)(value32 & 0x03);
+		vrsv = (u8)(value32 & 0x04);
+		vstbc = (u8)(value32 & 0x08);
+		vgid = (u8)((value32 & 0x3f0) >> 4);
+		v_nsts = (u8)(((value32 & 0x1c00) >> 8) + 1);
+		vpaid = (u16)(value32 & 0x3fe);
+		vtxops = (u8)((value32 & 0x400000) >> 20);
+		vrsv2 = (u8)((value32 & 0x800000) >> 20);
+	}
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s",
+		       "VHT-SIG-A1");
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "\r\n %-35s = %x / %x / %x / %x / %x / %x / %x / %x",
+		       "BW/Rsv1/STBC/GID/Nsts/PAID/TXOPPS/Rsv2", v_rx_bw, vrsv,
+		       vstbc, vgid, v_nsts, vpaid, vtxops, vrsv2);
+
+	value32 = odm_get_bb_reg(dm, 0xf30, MASKDWORD); /*VHT SIG*/
+
+	if (rx_ht == 2) {
+		/*value32 = odm_get_bb_reg(dm, 0xf30,MASKDWORD); */ /*VHT SIG*/
+
+		/* sgi=(u8)(value32&0x01); */
+		sgiext = (u8)(value32 & 0x03);
+		/* fec = (u8)(value32&0x04); */
+		fecext = (u8)(value32 & 0x0C);
+
+		v_mcss = (u8)(value32 & 0xf0);
+		bf = (u8)((value32 & 0x100) >> 8);
+		vrsv = (u8)((value32 & 0x200) >> 8);
+		vhtcrc8 = (u16)((value32 & 0x3fc00) >> 8);
+		v_tail = (u8)((value32 & 0xfc0000) >> 16);
+	}
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s",
+		       "VHT-SIG-A2");
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "\r\n %-35s = %x / %x / %x / %x / %x / %x / %x",
+		       "SGI/FEC/MCS/BF/Rsv/CRC/tail", sgiext, fecext, v_mcss,
+		       bf, vrsv, vhtcrc8, v_tail);
+
+	value32 = odm_get_bb_reg(dm, 0xf34, MASKDWORD); /*VHT SIG*/
+	{
+		v_length = (u16)(value32 & 0x1fffff);
+		vbrsv = (u8)((value32 & 0x600000) >> 20);
+		vb_tail = (u16)((value32 & 0x1f800000) >> 20);
+		vbcrc = (u8)((value32 & 0x80000000) >> 28);
+	}
+	PHYDM_SNPRINTF(output + used, out_len - used, "\r\n %-35s",
+		       "VHT-SIG-B");
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "\r\n %-35s = %x / %x / %x / %x", "length/Rsv/tail/CRC",
+		       v_length, vbrsv, vb_tail, vbcrc);
+
+	/*for Condition number*/
+	if (dm->support_ic_type & ODM_RTL8822B) {
+		s32 condition_num = 0;
+		char *factor = NULL;
+
+		/*enable report condition number*/
+		odm_set_bb_reg(dm, 0x1988, BIT(22), 0x1);
+
+		condition_num = odm_get_bb_reg(dm, 0xf84, MASKDWORD);
+		condition_num = (condition_num & 0x3ffff) >> 4;
+
+		if (*dm->band_width == ODM_BW80M) {
+			factor = "256/234";
+		} else if (*dm->band_width == ODM_BW40M) {
+			factor = "128/108";
+		} else if (*dm->band_width == ODM_BW20M) {
+			if (rx_ht == 2 || rx_ht == 1)
+				factor = "64/52"; /*HT or VHT*/
+			else
+				factor = "64/48"; /*legacy*/
+		}
+
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n %-35s = %d (factor = %s)",
+			       "Condition number", condition_num, factor);
+	}
+}
+
+void phydm_basic_dbg_message(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct false_alarm_stat *false_alm_cnt =
+		(struct false_alarm_stat *)phydm_get_structure(
+			dm, PHYDM_FALSEALMCNT);
+	struct cfo_tracking *cfo_track =
+		(struct cfo_tracking *)phydm_get_structure(dm, PHYDM_CFOTRACK);
+	struct dig_thres *dig_tab = &dm->dm_dig_table;
+	struct ra_table *ra_tab = &dm->dm_ra_table;
+	u16 macid, phydm_macid, client_cnt = 0;
+	struct rtl_sta_info *entry;
+	s32 tmp_val = 0;
+	u8 tmp_val_u1 = 0;
+
+	ODM_RT_TRACE(dm, ODM_COMP_COMMON,
+		     "[PHYDM Common MSG] System up time: ((%d sec))----->\n",
+		     dm->phydm_sys_up_time);
+
+	if (dm->is_linked) {
+		ODM_RT_TRACE(dm, ODM_COMP_COMMON,
+			     "ID=%d, BW=((%d)), CH=((%d))\n",
+			     dm->curr_station_id, 20 << *dm->band_width,
+			     *dm->channel);
+
+		/*Print RX rate*/
+		if (dm->rx_rate <= ODM_RATE11M)
+			ODM_RT_TRACE(
+				dm, ODM_COMP_COMMON,
+				"[CCK AGC Report] LNA_idx = 0x%x, VGA_idx = 0x%x\n",
+				dm->cck_lna_idx, dm->cck_vga_idx);
+		else
+			ODM_RT_TRACE(
+				dm, ODM_COMP_COMMON,
+				"[OFDM AGC Report] { 0x%x, 0x%x, 0x%x, 0x%x }\n",
+				dm->ofdm_agc_idx[0], dm->ofdm_agc_idx[1],
+				dm->ofdm_agc_idx[2], dm->ofdm_agc_idx[3]);
+
+		ODM_RT_TRACE(dm, ODM_COMP_COMMON,
+			     "RSSI: { %d,  %d,  %d,  %d },    rx_rate:",
+			     (dm->rssi_a == 0xff) ? 0 : dm->rssi_a,
+			     (dm->rssi_b == 0xff) ? 0 : dm->rssi_b,
+			     (dm->rssi_c == 0xff) ? 0 : dm->rssi_c,
+			     (dm->rssi_d == 0xff) ? 0 : dm->rssi_d);
+
+		phydm_print_rate(dm, dm->rx_rate, ODM_COMP_COMMON);
+
+		/*Print TX rate*/
+		for (macid = 0; macid < ODM_ASSOCIATE_ENTRY_NUM; macid++) {
+			entry = dm->odm_sta_info[macid];
+			if (!IS_STA_VALID(entry))
+				continue;
+
+			phydm_macid = (dm->platform2phydm_macid_table[macid]);
+			ODM_RT_TRACE(dm, ODM_COMP_COMMON, "TXRate [%d]:",
+				     macid);
+			phydm_print_rate(dm, ra_tab->link_tx_rate[macid],
+					 ODM_COMP_COMMON);
+
+			client_cnt++;
+
+			if (client_cnt == dm->number_linked_client)
+				break;
+		}
+
+		ODM_RT_TRACE(
+			dm, ODM_COMP_COMMON,
+			"TP { TX, RX, total} = {%d, %d, %d }Mbps, traffic_load = (%d))\n",
+			dm->tx_tp, dm->rx_tp, dm->total_tp, dm->traffic_load);
+
+		tmp_val_u1 =
+			(cfo_track->crystal_cap > cfo_track->def_x_cap) ?
+				(cfo_track->crystal_cap -
+				 cfo_track->def_x_cap) :
+				(cfo_track->def_x_cap - cfo_track->crystal_cap);
+		ODM_RT_TRACE(
+			dm, ODM_COMP_COMMON,
+			"CFO_avg = ((%d kHz)) , CrystalCap_tracking = ((%s%d))\n",
+			cfo_track->CFO_ave_pre,
+			((cfo_track->crystal_cap > cfo_track->def_x_cap) ? "+" :
+									   "-"),
+			tmp_val_u1);
+
+		/* Condition number */
+		if (dm->support_ic_type == ODM_RTL8822B) {
+			tmp_val = phydm_get_condition_number_8822B(dm);
+			ODM_RT_TRACE(dm, ODM_COMP_COMMON,
+				     "Condition number = ((%d))\n", tmp_val);
+		}
+
+		/*STBC or LDPC pkt*/
+		ODM_RT_TRACE(dm, ODM_COMP_COMMON, "LDPC = %s, STBC = %s\n",
+			     (dm->phy_dbg_info.is_ldpc_pkt) ? "Y" : "N",
+			     (dm->phy_dbg_info.is_stbc_pkt) ? "Y" : "N");
+	} else {
+		ODM_RT_TRACE(dm, ODM_COMP_COMMON, "No Link !!!\n");
+	}
+
+	ODM_RT_TRACE(dm, ODM_COMP_COMMON,
+		     "[CCA Cnt] {CCK, OFDM, Total} = {%d, %d, %d}\n",
+		     false_alm_cnt->cnt_cck_cca, false_alm_cnt->cnt_ofdm_cca,
+		     false_alm_cnt->cnt_cca_all);
+
+	ODM_RT_TRACE(dm, ODM_COMP_COMMON,
+		     "[FA Cnt] {CCK, OFDM, Total} = {%d, %d, %d}\n",
+		     false_alm_cnt->cnt_cck_fail, false_alm_cnt->cnt_ofdm_fail,
+		     false_alm_cnt->cnt_all);
+
+	if (dm->support_ic_type & ODM_IC_11N_SERIES)
+		ODM_RT_TRACE(
+			dm, ODM_COMP_COMMON,
+			"[OFDM FA Detail] Parity_Fail = (( %d )), Rate_Illegal = (( %d )), CRC8_fail = (( %d )), Mcs_fail = (( %d )), Fast_Fsync = (( %d )), SB_Search_fail = (( %d ))\n",
+			false_alm_cnt->cnt_parity_fail,
+			false_alm_cnt->cnt_rate_illegal,
+			false_alm_cnt->cnt_crc8_fail,
+			false_alm_cnt->cnt_mcs_fail,
+			false_alm_cnt->cnt_fast_fsync,
+			false_alm_cnt->cnt_sb_search_fail);
+
+	ODM_RT_TRACE(
+		dm, ODM_COMP_COMMON,
+		"is_linked = %d, Num_client = %d, rssi_min = %d, current_igi = 0x%x, bNoisy=%d\n\n",
+		dm->is_linked, dm->number_linked_client, dm->rssi_min,
+		dig_tab->cur_ig_value, dm->noisy_decision);
+}
+
+void phydm_basic_profile(void *dm_void, u32 *_used, char *output, u32 *_out_len)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	char *cut = NULL;
+	char *ic_type = NULL;
+	u32 used = *_used;
+	u32 out_len = *_out_len;
+	u32 date = 0;
+	char *commit_by = NULL;
+	u32 release_ver = 0;
+
+	PHYDM_SNPRINTF(output + used, out_len - used, "%-35s\n",
+		       "% Basic Profile %");
+
+	if (dm->support_ic_type == ODM_RTL8188E) {
+	} else if (dm->support_ic_type == ODM_RTL8822B) {
+		ic_type = "RTL8822B";
+		date = RELEASE_DATE_8822B;
+		commit_by = COMMIT_BY_8822B;
+		release_ver = RELEASE_VERSION_8822B;
+	}
+
+	/* JJ ADD 20161014 */
+
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "  %-35s: %s (MP Chip: %s)\n", "IC type", ic_type,
+		       dm->is_mp_chip ? "Yes" : "No");
+
+	if (dm->cut_version == ODM_CUT_A)
+		cut = "A";
+	else if (dm->cut_version == ODM_CUT_B)
+		cut = "B";
+	else if (dm->cut_version == ODM_CUT_C)
+		cut = "C";
+	else if (dm->cut_version == ODM_CUT_D)
+		cut = "D";
+	else if (dm->cut_version == ODM_CUT_E)
+		cut = "E";
+	else if (dm->cut_version == ODM_CUT_F)
+		cut = "F";
+	else if (dm->cut_version == ODM_CUT_I)
+		cut = "I";
+	PHYDM_SNPRINTF(output + used, out_len - used, "  %-35s: %s\n",
+		       "cut version", cut);
+	PHYDM_SNPRINTF(output + used, out_len - used, "  %-35s: %d\n",
+		       "PHY Parameter version", odm_get_hw_img_version(dm));
+	PHYDM_SNPRINTF(output + used, out_len - used, "  %-35s: %d\n",
+		       "PHY Parameter Commit date", date);
+	PHYDM_SNPRINTF(output + used, out_len - used, "  %-35s: %s\n",
+		       "PHY Parameter Commit by", commit_by);
+	PHYDM_SNPRINTF(output + used, out_len - used, "  %-35s: %d\n",
+		       "PHY Parameter Release version", release_ver);
+
+	{
+		struct rtl_priv *rtlpriv = (struct rtl_priv *)dm->adapter;
+		struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "  %-35s: %d (Subversion: %d)\n", "FW version",
+			       rtlhal->fw_version, rtlhal->fw_subversion);
+	}
+	/* 1 PHY DM version List */
+	PHYDM_SNPRINTF(output + used, out_len - used, "%-35s\n",
+		       "% PHYDM version %");
+	PHYDM_SNPRINTF(output + used, out_len - used, "  %-35s: %s\n",
+		       "Code base", PHYDM_CODE_BASE);
+	PHYDM_SNPRINTF(output + used, out_len - used, "  %-35s: %s\n",
+		       "Release Date", PHYDM_RELEASE_DATE);
+	PHYDM_SNPRINTF(output + used, out_len - used, "  %-35s: %s\n",
+		       "adaptivity", ADAPTIVITY_VERSION);
+	PHYDM_SNPRINTF(output + used, out_len - used, "  %-35s: %s\n", "DIG",
+		       DIG_VERSION);
+	PHYDM_SNPRINTF(output + used, out_len - used, "  %-35s: %s\n",
+		       "Dynamic BB PowerSaving", DYNAMIC_BBPWRSAV_VERSION);
+	PHYDM_SNPRINTF(output + used, out_len - used, "  %-35s: %s\n",
+		       "CFO Tracking", CFO_TRACKING_VERSION);
+	PHYDM_SNPRINTF(output + used, out_len - used, "  %-35s: %s\n",
+		       "Antenna Diversity", ANTDIV_VERSION);
+	PHYDM_SNPRINTF(output + used, out_len - used, "  %-35s: %s\n",
+		       "Power Tracking", POWRTRACKING_VERSION);
+	PHYDM_SNPRINTF(output + used, out_len - used, "  %-35s: %s\n",
+		       "Dynamic TxPower", DYNAMIC_TXPWR_VERSION);
+	PHYDM_SNPRINTF(output + used, out_len - used, "  %-35s: %s\n",
+		       "RA Info", RAINFO_VERSION);
+	PHYDM_SNPRINTF(output + used, out_len - used, "  %-35s: %s\n",
+		       "Auto channel Selection", ACS_VERSION);
+	PHYDM_SNPRINTF(output + used, out_len - used, "  %-35s: %s\n",
+		       "EDCA Turbo", EDCATURBO_VERSION);
+	PHYDM_SNPRINTF(output + used, out_len - used, "  %-35s: %s\n",
+		       "LA mode", DYNAMIC_LA_MODE);
+	PHYDM_SNPRINTF(output + used, out_len - used, "  %-35s: %s\n",
+		       "Dynamic RX path", DYNAMIC_RX_PATH_VERSION);
+
+	if (dm->support_ic_type & ODM_RTL8822B)
+		PHYDM_SNPRINTF(output + used, out_len - used, "  %-35s: %s\n",
+			       "PHY config 8822B", PHY_CONFIG_VERSION_8822B);
+
+	*_used = used;
+	*_out_len = out_len;
+}
+
+void phydm_fw_trace_en_h2c(void *dm_void, bool enable, u32 fw_debug_component,
+			   u32 monitor_mode, u32 macid)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u8 h2c_parameter[7] = {0};
+	u8 cmd_length;
+
+	if (dm->support_ic_type & PHYDM_IC_3081_SERIES) {
+		h2c_parameter[0] = enable;
+		h2c_parameter[1] = (u8)(fw_debug_component & MASKBYTE0);
+		h2c_parameter[2] = (u8)((fw_debug_component & MASKBYTE1) >> 8);
+		h2c_parameter[3] = (u8)((fw_debug_component & MASKBYTE2) >> 16);
+		h2c_parameter[4] = (u8)((fw_debug_component & MASKBYTE3) >> 24);
+		h2c_parameter[5] = (u8)monitor_mode;
+		h2c_parameter[6] = (u8)macid;
+		cmd_length = 7;
+
+	} else {
+		h2c_parameter[0] = enable;
+		h2c_parameter[1] = (u8)monitor_mode;
+		h2c_parameter[2] = (u8)macid;
+		cmd_length = 3;
+	}
+
+	ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "---->\n");
+	if (monitor_mode == 0)
+		ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE,
+			     "[H2C] FW_debug_en: (( %d ))\n", enable);
+	else
+		ODM_RT_TRACE(
+			dm, ODM_FW_DEBUG_TRACE,
+			"[H2C] FW_debug_en: (( %d )), mode: (( %d )), macid: (( %d ))\n",
+			enable, monitor_mode, macid);
+	odm_fill_h2c_cmd(dm, PHYDM_H2C_FW_TRACE_EN, cmd_length, h2c_parameter);
+}
+
+bool phydm_api_set_txagc(struct phy_dm_struct *dm, u32 power_index,
+			 enum odm_rf_radio_path path, u8 hw_rate,
+			 bool is_single_rate)
+{
+	bool ret = false;
+
+	if (dm->support_ic_type & (ODM_RTL8822B | ODM_RTL8821C)) {
+		if (is_single_rate) {
+			if (dm->support_ic_type == ODM_RTL8822B)
+				ret = phydm_write_txagc_1byte_8822b(
+					dm, power_index, path, hw_rate);
+
+		} else {
+			if (dm->support_ic_type == ODM_RTL8822B)
+				ret = config_phydm_write_txagc_8822b(
+					dm, power_index, path, hw_rate);
+		}
+	}
+
+	return ret;
+}
+
+static u8 phydm_api_get_txagc(struct phy_dm_struct *dm,
+			      enum odm_rf_radio_path path, u8 hw_rate)
+{
+	u8 ret = 0;
+
+	if (dm->support_ic_type & ODM_RTL8822B)
+		ret = config_phydm_read_txagc_8822b(dm, path, hw_rate);
+
+	return ret;
+}
+
+static bool phydm_api_switch_bw_channel(struct phy_dm_struct *dm, u8 central_ch,
+					u8 primary_ch_idx,
+					enum odm_bw bandwidth)
+{
+	bool ret = false;
+
+	if (dm->support_ic_type & ODM_RTL8822B)
+		ret = config_phydm_switch_channel_bw_8822b(
+			dm, central_ch, primary_ch_idx, bandwidth);
+
+	return ret;
+}
+
+bool phydm_api_trx_mode(struct phy_dm_struct *dm, enum odm_rf_path tx_path,
+			enum odm_rf_path rx_path, bool is_tx2_path)
+{
+	bool ret = false;
+
+	if (dm->support_ic_type & ODM_RTL8822B)
+		ret = config_phydm_trx_mode_8822b(dm, tx_path, rx_path,
+						  is_tx2_path);
+
+	return ret;
+}
+
+static void phydm_get_per_path_txagc(void *dm_void, u8 path, u32 *_used,
+				     char *output, u32 *_out_len)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u8 rate_idx;
+	u8 txagc;
+	u32 used = *_used;
+	u32 out_len = *_out_len;
+
+	if (((dm->support_ic_type & (ODM_RTL8822B | ODM_RTL8197F)) &&
+	     (path <= ODM_RF_PATH_B)) ||
+	    ((dm->support_ic_type & (ODM_RTL8821C)) &&
+	     (path <= ODM_RF_PATH_A))) {
+		for (rate_idx = 0; rate_idx <= 0x53; rate_idx++) {
+			if (rate_idx == ODM_RATE1M)
+				PHYDM_SNPRINTF(output + used, out_len - used,
+					       "  %-35s\n", "CCK====>");
+			else if (rate_idx == ODM_RATE6M)
+				PHYDM_SNPRINTF(output + used, out_len - used,
+					       "\n  %-35s\n", "OFDM====>");
+			else if (rate_idx == ODM_RATEMCS0)
+				PHYDM_SNPRINTF(output + used, out_len - used,
+					       "\n  %-35s\n", "HT 1ss====>");
+			else if (rate_idx == ODM_RATEMCS8)
+				PHYDM_SNPRINTF(output + used, out_len - used,
+					       "\n  %-35s\n", "HT 2ss====>");
+			else if (rate_idx == ODM_RATEMCS16)
+				PHYDM_SNPRINTF(output + used, out_len - used,
+					       "\n  %-35s\n", "HT 3ss====>");
+			else if (rate_idx == ODM_RATEMCS24)
+				PHYDM_SNPRINTF(output + used, out_len - used,
+					       "\n  %-35s\n", "HT 4ss====>");
+			else if (rate_idx == ODM_RATEVHTSS1MCS0)
+				PHYDM_SNPRINTF(output + used, out_len - used,
+					       "\n  %-35s\n", "VHT 1ss====>");
+			else if (rate_idx == ODM_RATEVHTSS2MCS0)
+				PHYDM_SNPRINTF(output + used, out_len - used,
+					       "\n  %-35s\n", "VHT 2ss====>");
+			else if (rate_idx == ODM_RATEVHTSS3MCS0)
+				PHYDM_SNPRINTF(output + used, out_len - used,
+					       "\n  %-35s\n", "VHT 3ss====>");
+			else if (rate_idx == ODM_RATEVHTSS4MCS0)
+				PHYDM_SNPRINTF(output + used, out_len - used,
+					       "\n  %-35s\n", "VHT 4ss====>");
+
+			txagc = phydm_api_get_txagc(
+				dm, (enum odm_rf_radio_path)path, rate_idx);
+			if (config_phydm_read_txagc_check(txagc))
+				PHYDM_SNPRINTF(output + used, out_len - used,
+					       "  0x%02x    ", txagc);
+			else
+				PHYDM_SNPRINTF(output + used, out_len - used,
+					       "  0x%s    ", "xx");
+		}
+	}
+}
+
+static void phydm_get_txagc(void *dm_void, u32 *_used, char *output,
+			    u32 *_out_len)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 used = *_used;
+	u32 out_len = *_out_len;
+
+	/* path-A */
+	PHYDM_SNPRINTF(output + used, out_len - used, "%-35s\n",
+		       "path-A====================");
+	phydm_get_per_path_txagc(dm, ODM_RF_PATH_A, _used, output, _out_len);
+
+	/* path-B */
+	PHYDM_SNPRINTF(output + used, out_len - used, "\n%-35s\n",
+		       "path-B====================");
+	phydm_get_per_path_txagc(dm, ODM_RF_PATH_B, _used, output, _out_len);
+
+	/* path-C */
+	PHYDM_SNPRINTF(output + used, out_len - used, "\n%-35s\n",
+		       "path-C====================");
+	phydm_get_per_path_txagc(dm, ODM_RF_PATH_C, _used, output, _out_len);
+
+	/* path-D */
+	PHYDM_SNPRINTF(output + used, out_len - used, "\n%-35s\n",
+		       "path-D====================");
+	phydm_get_per_path_txagc(dm, ODM_RF_PATH_D, _used, output, _out_len);
+}
+
+static void phydm_set_txagc(void *dm_void, u32 *const dm_value, u32 *_used,
+			    char *output, u32 *_out_len)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 used = *_used;
+	u32 out_len = *_out_len;
+
+	/*dm_value[1] = path*/
+	/*dm_value[2] = hw_rate*/
+	/*dm_value[3] = power_index*/
+
+	if (dm->support_ic_type &
+	    (ODM_RTL8822B | ODM_RTL8197F | ODM_RTL8821C)) {
+		if (dm_value[1] <= 1) {
+			phydm_check_dmval_txagc(dm, used, out_len, dm_value,
+						output);
+		} else {
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "  %s%d   %s%x%s\n", "Write path-",
+				       (dm_value[1] & 0x1), "rate index-0x",
+				       (dm_value[2] & 0x7f), " fail");
+		}
+	}
+}
+
+static void phydm_debug_trace(void *dm_void, u32 *const dm_value, u32 *_used,
+			      char *output, u32 *_out_len)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 pre_debug_components, one = 1;
+	u32 used = *_used;
+	u32 out_len = *_out_len;
+
+	pre_debug_components = dm->debug_components;
+
+	PHYDM_SNPRINTF(output + used, out_len - used, "\n%s\n",
+		       "================================");
+	if (dm_value[0] == 100) {
+		PHYDM_SNPRINTF(output + used, out_len - used, "%s\n",
+			       "[Debug Message] PhyDM Selection");
+		PHYDM_SNPRINTF(output + used, out_len - used, "%s\n",
+			       "================================");
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "00. (( %s ))DIG\n",
+			       ((dm->debug_components & ODM_COMP_DIG) ? ("V") :
+									(".")));
+		PHYDM_SNPRINTF(
+			output + used, out_len - used, "01. (( %s ))RA_MASK\n",
+			((dm->debug_components & ODM_COMP_RA_MASK) ? ("V") :
+								     (".")));
+		PHYDM_SNPRINTF(
+			output + used, out_len - used,
+			"02. (( %s ))DYNAMIC_TXPWR\n",
+			((dm->debug_components & ODM_COMP_DYNAMIC_TXPWR) ?
+				 ("V") :
+				 (".")));
+		PHYDM_SNPRINTF(
+			output + used, out_len - used, "03. (( %s ))FA_CNT\n",
+			((dm->debug_components & ODM_COMP_FA_CNT) ? ("V") :
+								    (".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "04. (( %s ))RSSI_MONITOR\n",
+			       ((dm->debug_components & ODM_COMP_RSSI_MONITOR) ?
+					("V") :
+					(".")));
+		PHYDM_SNPRINTF(
+			output + used, out_len - used, "05. (( %s ))SNIFFER\n",
+			((dm->debug_components & ODM_COMP_SNIFFER) ? ("V") :
+								     (".")));
+		PHYDM_SNPRINTF(
+			output + used, out_len - used, "06. (( %s ))ANT_DIV\n",
+			((dm->debug_components & ODM_COMP_ANT_DIV) ? ("V") :
+								     (".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "07. (( %s ))DFS\n",
+			       ((dm->debug_components & ODM_COMP_DFS) ? ("V") :
+									(".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "08. (( %s ))NOISY_DETECT\n",
+			       ((dm->debug_components & ODM_COMP_NOISY_DETECT) ?
+					("V") :
+					(".")));
+		PHYDM_SNPRINTF(
+			output + used, out_len - used,
+			"09. (( %s ))RATE_ADAPTIVE\n",
+			((dm->debug_components & ODM_COMP_RATE_ADAPTIVE) ?
+				 ("V") :
+				 (".")));
+		PHYDM_SNPRINTF(
+			output + used, out_len - used, "10. (( %s ))PATH_DIV\n",
+			((dm->debug_components & ODM_COMP_PATH_DIV) ? ("V") :
+								      (".")));
+		PHYDM_SNPRINTF(
+			output + used, out_len - used,
+			"12. (( %s ))DYNAMIC_PRICCA\n",
+			((dm->debug_components & ODM_COMP_DYNAMIC_PRICCA) ?
+				 ("V") :
+				 (".")));
+		PHYDM_SNPRINTF(
+			output + used, out_len - used, "14. (( %s ))MP\n",
+			((dm->debug_components & ODM_COMP_MP) ? ("V") : (".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "15. (( %s ))struct cfo_tracking\n",
+			       ((dm->debug_components & ODM_COMP_CFO_TRACKING) ?
+					("V") :
+					(".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "16. (( %s ))struct acs_info\n",
+			       ((dm->debug_components & ODM_COMP_ACS) ? ("V") :
+									(".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "17. (( %s ))ADAPTIVITY\n",
+			       ((dm->debug_components & PHYDM_COMP_ADAPTIVITY) ?
+					("V") :
+					(".")));
+		PHYDM_SNPRINTF(
+			output + used, out_len - used, "18. (( %s ))RA_DBG\n",
+			((dm->debug_components & PHYDM_COMP_RA_DBG) ? ("V") :
+								      (".")));
+		PHYDM_SNPRINTF(
+			output + used, out_len - used, "19. (( %s ))TXBF\n",
+			((dm->debug_components & PHYDM_COMP_TXBF) ? ("V") :
+								    (".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "20. (( %s ))EDCA_TURBO\n",
+			       ((dm->debug_components & ODM_COMP_EDCA_TURBO) ?
+					("V") :
+					(".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "22. (( %s ))FW_DEBUG_TRACE\n",
+			       ((dm->debug_components & ODM_FW_DEBUG_TRACE) ?
+					("V") :
+					(".")));
+
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "24. (( %s ))TX_PWR_TRACK\n",
+			       ((dm->debug_components & ODM_COMP_TX_PWR_TRACK) ?
+					("V") :
+					(".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "26. (( %s ))CALIBRATION\n",
+			       ((dm->debug_components & ODM_COMP_CALIBRATION) ?
+					("V") :
+					(".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "28. (( %s ))PHY_CONFIG\n",
+			       ((dm->debug_components & ODM_PHY_CONFIG) ?
+					("V") :
+					(".")));
+		PHYDM_SNPRINTF(
+			output + used, out_len - used, "29. (( %s ))INIT\n",
+			((dm->debug_components & ODM_COMP_INIT) ? ("V") :
+								  (".")));
+		PHYDM_SNPRINTF(
+			output + used, out_len - used, "30. (( %s ))COMMON\n",
+			((dm->debug_components & ODM_COMP_COMMON) ? ("V") :
+								    (".")));
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "31. (( %s ))API\n",
+			       ((dm->debug_components & ODM_COMP_API) ? ("V") :
+									(".")));
+		PHYDM_SNPRINTF(output + used, out_len - used, "%s\n",
+			       "================================");
+
+	} else if (dm_value[0] == 101) {
+		dm->debug_components = 0;
+		PHYDM_SNPRINTF(output + used, out_len - used, "%s\n",
+			       "Disable all debug components");
+	} else {
+		if (dm_value[1] == 1) /*enable*/
+			dm->debug_components |= (one << dm_value[0]);
+		else if (dm_value[1] == 2) /*disable*/
+			dm->debug_components &= ~(one << dm_value[0]);
+		else
+			PHYDM_SNPRINTF(output + used, out_len - used, "%s\n",
+				       "[Warning!!!]  1:enable,  2:disable");
+	}
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "pre-DbgComponents = 0x%x\n", pre_debug_components);
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "Curr-DbgComponents = 0x%x\n", dm->debug_components);
+	PHYDM_SNPRINTF(output + used, out_len - used, "%s\n",
+		       "================================");
+}
+
+static void phydm_fw_debug_trace(void *dm_void, u32 *const dm_value, u32 *_used,
+				 char *output, u32 *_out_len)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 pre_fw_debug_components, one = 1;
+	u32 used = *_used;
+	u32 out_len = *_out_len;
+
+	pre_fw_debug_components = dm->fw_debug_components;
+
+	PHYDM_SNPRINTF(output + used, out_len - used, "\n%s\n",
+		       "================================");
+	if (dm_value[0] == 100) {
+		PHYDM_SNPRINTF(output + used, out_len - used, "%s\n",
+			       "[FW Debug Component]");
+		PHYDM_SNPRINTF(output + used, out_len - used, "%s\n",
+			       "================================");
+		PHYDM_SNPRINTF(
+			output + used, out_len - used, "00. (( %s ))RA\n",
+			((dm->fw_debug_components & PHYDM_FW_COMP_RA) ? ("V") :
+									(".")));
+
+		if (dm->support_ic_type & PHYDM_IC_3081_SERIES) {
+			PHYDM_SNPRINTF(
+				output + used, out_len - used,
+				"01. (( %s ))MU\n",
+				((dm->fw_debug_components & PHYDM_FW_COMP_MU) ?
+					 ("V") :
+					 (".")));
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "02. (( %s ))path Div\n",
+				       ((dm->fw_debug_components &
+					 PHYDM_FW_COMP_PHY_CONFIG) ?
+						("V") :
+						(".")));
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "03. (( %s ))Phy Config\n",
+				       ((dm->fw_debug_components &
+					 PHYDM_FW_COMP_PHY_CONFIG) ?
+						("V") :
+						(".")));
+		}
+		PHYDM_SNPRINTF(output + used, out_len - used, "%s\n",
+			       "================================");
+
+	} else {
+		if (dm_value[0] == 101) {
+			dm->fw_debug_components = 0;
+			PHYDM_SNPRINTF(output + used, out_len - used, "%s\n",
+				       "Clear all fw debug components");
+		} else {
+			if (dm_value[1] == 1) /*enable*/
+				dm->fw_debug_components |= (one << dm_value[0]);
+			else if (dm_value[1] == 2) /*disable*/
+				dm->fw_debug_components &=
+					~(one << dm_value[0]);
+			else
+				PHYDM_SNPRINTF(
+					output + used, out_len - used, "%s\n",
+					"[Warning!!!]  1:enable,  2:disable");
+		}
+
+		if (dm->fw_debug_components == 0) {
+			dm->debug_components &= ~ODM_FW_DEBUG_TRACE;
+			phydm_fw_trace_en_h2c(
+				dm, false, dm->fw_debug_components, dm_value[2],
+				dm_value[3]); /*H2C to enable C2H Msg*/
+		} else {
+			dm->debug_components |= ODM_FW_DEBUG_TRACE;
+			phydm_fw_trace_en_h2c(
+				dm, true, dm->fw_debug_components, dm_value[2],
+				dm_value[3]); /*H2C to enable C2H Msg*/
+		}
+	}
+}
+
+static void phydm_dump_bb_reg(void *dm_void, u32 *_used, char *output,
+			      u32 *_out_len)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 addr = 0;
+	u32 used = *_used;
+	u32 out_len = *_out_len;
+
+	/* For Nseries IC we only need to dump page8 to pageF using 3 digits*/
+	for (addr = 0x800; addr < 0xfff; addr += 4) {
+		if (dm->support_ic_type & ODM_IC_11N_SERIES)
+			PHYDM_VAST_INFO_SNPRINTF(
+				output + used, out_len - used,
+				"0x%03x 0x%08x\n", addr,
+				odm_get_bb_reg(dm, addr, MASKDWORD));
+		else
+			PHYDM_VAST_INFO_SNPRINTF(
+				output + used, out_len - used,
+				"0x%04x 0x%08x\n", addr,
+				odm_get_bb_reg(dm, addr, MASKDWORD));
+	}
+
+	if (dm->support_ic_type &
+	    (ODM_RTL8822B | ODM_RTL8814A | ODM_RTL8821C)) {
+		if (dm->rf_type > ODM_2T2R) {
+			for (addr = 0x1800; addr < 0x18ff; addr += 4)
+				PHYDM_VAST_INFO_SNPRINTF(
+					output + used, out_len - used,
+					"0x%04x 0x%08x\n", addr,
+					odm_get_bb_reg(dm, addr, MASKDWORD));
+		}
+
+		if (dm->rf_type > ODM_3T3R) {
+			for (addr = 0x1a00; addr < 0x1aff; addr += 4)
+				PHYDM_VAST_INFO_SNPRINTF(
+					output + used, out_len - used,
+					"0x%04x 0x%08x\n", addr,
+					odm_get_bb_reg(dm, addr, MASKDWORD));
+		}
+
+		for (addr = 0x1900; addr < 0x19ff; addr += 4)
+			PHYDM_VAST_INFO_SNPRINTF(
+				output + used, out_len - used,
+				"0x%04x 0x%08x\n", addr,
+				odm_get_bb_reg(dm, addr, MASKDWORD));
+
+		for (addr = 0x1c00; addr < 0x1cff; addr += 4)
+			PHYDM_VAST_INFO_SNPRINTF(
+				output + used, out_len - used,
+				"0x%04x 0x%08x\n", addr,
+				odm_get_bb_reg(dm, addr, MASKDWORD));
+
+		for (addr = 0x1f00; addr < 0x1fff; addr += 4)
+			PHYDM_VAST_INFO_SNPRINTF(
+				output + used, out_len - used,
+				"0x%04x 0x%08x\n", addr,
+				odm_get_bb_reg(dm, addr, MASKDWORD));
+	}
+}
+
+static void phydm_dump_all_reg(void *dm_void, u32 *_used, char *output,
+			       u32 *_out_len)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 addr = 0;
+	u32 used = *_used;
+	u32 out_len = *_out_len;
+
+	/* dump MAC register */
+	PHYDM_VAST_INFO_SNPRINTF(output + used, out_len - used,
+				 "MAC==========\n");
+	for (addr = 0; addr < 0x7ff; addr += 4)
+		PHYDM_VAST_INFO_SNPRINTF(output + used, out_len - used,
+					 "0x%04x 0x%08x\n", addr,
+					 odm_get_bb_reg(dm, addr, MASKDWORD));
+
+	for (addr = 0x1000; addr < 0x17ff; addr += 4)
+		PHYDM_VAST_INFO_SNPRINTF(output + used, out_len - used,
+					 "0x%04x 0x%08x\n", addr,
+					 odm_get_bb_reg(dm, addr, MASKDWORD));
+
+	/* dump BB register */
+	PHYDM_VAST_INFO_SNPRINTF(output + used, out_len - used,
+				 "BB==========\n");
+	phydm_dump_bb_reg(dm, &used, output, &out_len);
+
+	/* dump RF register */
+	PHYDM_VAST_INFO_SNPRINTF(output + used, out_len - used,
+				 "RF-A==========\n");
+	for (addr = 0; addr < 0xFF; addr++)
+		PHYDM_VAST_INFO_SNPRINTF(output + used, out_len - used,
+					 "0x%02x 0x%05x\n", addr,
+					 odm_get_rf_reg(dm, ODM_RF_PATH_A, addr,
+							RFREGOFFSETMASK));
+
+	if (dm->rf_type > ODM_1T1R) {
+		PHYDM_VAST_INFO_SNPRINTF(output + used, out_len - used,
+					 "RF-B==========\n");
+		for (addr = 0; addr < 0xFF; addr++)
+			PHYDM_VAST_INFO_SNPRINTF(
+				output + used, out_len - used,
+				"0x%02x 0x%05x\n", addr,
+				odm_get_rf_reg(dm, ODM_RF_PATH_B, addr,
+					       RFREGOFFSETMASK));
+	}
+
+	if (dm->rf_type > ODM_2T2R) {
+		PHYDM_VAST_INFO_SNPRINTF(output + used, out_len - used,
+					 "RF-C==========\n");
+		for (addr = 0; addr < 0xFF; addr++)
+			PHYDM_VAST_INFO_SNPRINTF(
+				output + used, out_len - used,
+				"0x%02x 0x%05x\n", addr,
+				odm_get_rf_reg(dm, ODM_RF_PATH_C, addr,
+					       RFREGOFFSETMASK));
+	}
+
+	if (dm->rf_type > ODM_3T3R) {
+		PHYDM_VAST_INFO_SNPRINTF(output + used, out_len - used,
+					 "RF-D==========\n");
+		for (addr = 0; addr < 0xFF; addr++)
+			PHYDM_VAST_INFO_SNPRINTF(
+				output + used, out_len - used,
+				"0x%02x 0x%05x\n", addr,
+				odm_get_rf_reg(dm, ODM_RF_PATH_D, addr,
+					       RFREGOFFSETMASK));
+	}
+}
+
+static void phydm_enable_big_jump(struct phy_dm_struct *dm, bool state)
+{
+	struct dig_thres *dig_tab = &dm->dm_dig_table;
+
+	if (!state) {
+		dm->dm_dig_table.enable_adjust_big_jump = false;
+		odm_set_bb_reg(dm, 0x8c8, 0xfe,
+			       ((dig_tab->big_jump_step3 << 5) |
+				(dig_tab->big_jump_step2 << 3) |
+				dig_tab->big_jump_step1));
+	} else {
+		dm->dm_dig_table.enable_adjust_big_jump = true;
+	}
+}
+
+static void phydm_show_rx_rate(struct phy_dm_struct *dm, u32 *_used,
+			       char *output, u32 *_out_len)
+{
+	u32 used = *_used;
+	u32 out_len = *_out_len;
+
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "=====Rx SU rate Statistics=====\n");
+	PHYDM_SNPRINTF(
+		output + used, out_len - used,
+		"1SS MCS0 = %d, 1SS MCS1 = %d, 1SS MCS2 = %d, 1SS MCS 3 = %d\n",
+		dm->phy_dbg_info.num_qry_vht_pkt[0],
+		dm->phy_dbg_info.num_qry_vht_pkt[1],
+		dm->phy_dbg_info.num_qry_vht_pkt[2],
+		dm->phy_dbg_info.num_qry_vht_pkt[3]);
+	PHYDM_SNPRINTF(
+		output + used, out_len - used,
+		"1SS MCS4 = %d, 1SS MCS5 = %d, 1SS MCS6 = %d, 1SS MCS 7 = %d\n",
+		dm->phy_dbg_info.num_qry_vht_pkt[4],
+		dm->phy_dbg_info.num_qry_vht_pkt[5],
+		dm->phy_dbg_info.num_qry_vht_pkt[6],
+		dm->phy_dbg_info.num_qry_vht_pkt[7]);
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "1SS MCS8 = %d, 1SS MCS9 = %d\n",
+		       dm->phy_dbg_info.num_qry_vht_pkt[8],
+		       dm->phy_dbg_info.num_qry_vht_pkt[9]);
+	PHYDM_SNPRINTF(
+		output + used, out_len - used,
+		"2SS MCS0 = %d, 2SS MCS1 = %d, 2SS MCS2 = %d, 2SS MCS 3 = %d\n",
+		dm->phy_dbg_info.num_qry_vht_pkt[10],
+		dm->phy_dbg_info.num_qry_vht_pkt[11],
+		dm->phy_dbg_info.num_qry_vht_pkt[12],
+		dm->phy_dbg_info.num_qry_vht_pkt[13]);
+	PHYDM_SNPRINTF(
+		output + used, out_len - used,
+		"2SS MCS4 = %d, 2SS MCS5 = %d, 2SS MCS6 = %d, 2SS MCS 7 = %d\n",
+		dm->phy_dbg_info.num_qry_vht_pkt[14],
+		dm->phy_dbg_info.num_qry_vht_pkt[15],
+		dm->phy_dbg_info.num_qry_vht_pkt[16],
+		dm->phy_dbg_info.num_qry_vht_pkt[17]);
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "2SS MCS8 = %d, 2SS MCS9 = %d\n",
+		       dm->phy_dbg_info.num_qry_vht_pkt[18],
+		       dm->phy_dbg_info.num_qry_vht_pkt[19]);
+
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "=====Rx MU rate Statistics=====\n");
+	PHYDM_SNPRINTF(
+		output + used, out_len - used,
+		"1SS MCS0 = %d, 1SS MCS1 = %d, 1SS MCS2 = %d, 1SS MCS 3 = %d\n",
+		dm->phy_dbg_info.num_qry_mu_vht_pkt[0],
+		dm->phy_dbg_info.num_qry_mu_vht_pkt[1],
+		dm->phy_dbg_info.num_qry_mu_vht_pkt[2],
+		dm->phy_dbg_info.num_qry_mu_vht_pkt[3]);
+	PHYDM_SNPRINTF(
+		output + used, out_len - used,
+		"1SS MCS4 = %d, 1SS MCS5 = %d, 1SS MCS6 = %d, 1SS MCS 7 = %d\n",
+		dm->phy_dbg_info.num_qry_mu_vht_pkt[4],
+		dm->phy_dbg_info.num_qry_mu_vht_pkt[5],
+		dm->phy_dbg_info.num_qry_mu_vht_pkt[6],
+		dm->phy_dbg_info.num_qry_mu_vht_pkt[7]);
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "1SS MCS8 = %d, 1SS MCS9 = %d\n",
+		       dm->phy_dbg_info.num_qry_mu_vht_pkt[8],
+		       dm->phy_dbg_info.num_qry_mu_vht_pkt[9]);
+	PHYDM_SNPRINTF(
+		output + used, out_len - used,
+		"2SS MCS0 = %d, 2SS MCS1 = %d, 2SS MCS2 = %d, 2SS MCS 3 = %d\n",
+		dm->phy_dbg_info.num_qry_mu_vht_pkt[10],
+		dm->phy_dbg_info.num_qry_mu_vht_pkt[11],
+		dm->phy_dbg_info.num_qry_mu_vht_pkt[12],
+		dm->phy_dbg_info.num_qry_mu_vht_pkt[13]);
+	PHYDM_SNPRINTF(
+		output + used, out_len - used,
+		"2SS MCS4 = %d, 2SS MCS5 = %d, 2SS MCS6 = %d, 2SS MCS 7 = %d\n",
+		dm->phy_dbg_info.num_qry_mu_vht_pkt[14],
+		dm->phy_dbg_info.num_qry_mu_vht_pkt[15],
+		dm->phy_dbg_info.num_qry_mu_vht_pkt[16],
+		dm->phy_dbg_info.num_qry_mu_vht_pkt[17]);
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "2SS MCS8 = %d, 2SS MCS9 = %d\n",
+		       dm->phy_dbg_info.num_qry_mu_vht_pkt[18],
+		       dm->phy_dbg_info.num_qry_mu_vht_pkt[19]);
+}
+
+struct phydm_command {
+	char name[16];
+	u8 id;
+};
+
+enum PHYDM_CMD_ID {
+	PHYDM_HELP,
+	PHYDM_DEMO,
+	PHYDM_RA,
+	PHYDM_PROFILE,
+	PHYDM_ANTDIV,
+	PHYDM_PATHDIV,
+	PHYDM_DEBUG,
+	PHYDM_FW_DEBUG,
+	PHYDM_SUPPORT_ABILITY,
+	PHYDM_GET_TXAGC,
+	PHYDM_SET_TXAGC,
+	PHYDM_SMART_ANT,
+	PHYDM_API,
+	PHYDM_TRX_PATH,
+	PHYDM_LA_MODE,
+	PHYDM_DUMP_REG,
+	PHYDM_MU_MIMO,
+	PHYDM_HANG,
+	PHYDM_BIG_JUMP,
+	PHYDM_SHOW_RXRATE,
+	PHYDM_NBI_EN,
+	PHYDM_CSI_MASK_EN,
+	PHYDM_DFS,
+	PHYDM_IQK,
+	PHYDM_NHM,
+	PHYDM_CLM,
+	PHYDM_BB_INFO,
+	PHYDM_TXBF,
+	PHYDM_PAUSE_DIG_EN,
+	PHYDM_H2C,
+	PHYDM_ANT_SWITCH,
+	PHYDM_DYNAMIC_RA_PATH,
+	PHYDM_PSD,
+	PHYDM_DEBUG_PORT
+};
+
+static struct phydm_command phy_dm_ary[] = {
+	{"-h", PHYDM_HELP}, /*do not move this element to other position*/
+	{"demo", PHYDM_DEMO}, /*do not move this element to other position*/
+	{"ra", PHYDM_RA},
+	{"profile", PHYDM_PROFILE},
+	{"antdiv", PHYDM_ANTDIV},
+	{"pathdiv", PHYDM_PATHDIV},
+	{"dbg", PHYDM_DEBUG},
+	{"fw_dbg", PHYDM_FW_DEBUG},
+	{"ability", PHYDM_SUPPORT_ABILITY},
+	{"get_txagc", PHYDM_GET_TXAGC},
+	{"set_txagc", PHYDM_SET_TXAGC},
+	{"smtant", PHYDM_SMART_ANT},
+	{"api", PHYDM_API},
+	{"trxpath", PHYDM_TRX_PATH},
+	{"lamode", PHYDM_LA_MODE},
+	{"dumpreg", PHYDM_DUMP_REG},
+	{"mu", PHYDM_MU_MIMO},
+	{"hang", PHYDM_HANG},
+	{"bigjump", PHYDM_BIG_JUMP},
+	{"rxrate", PHYDM_SHOW_RXRATE},
+	{"nbi", PHYDM_NBI_EN},
+	{"csi_mask", PHYDM_CSI_MASK_EN},
+	{"dfs", PHYDM_DFS},
+	{"iqk", PHYDM_IQK},
+	{"nhm", PHYDM_NHM},
+	{"clm", PHYDM_CLM},
+	{"bbinfo", PHYDM_BB_INFO},
+	{"txbf", PHYDM_TXBF},
+	{"pause_dig", PHYDM_PAUSE_DIG_EN},
+	{"h2c", PHYDM_H2C},
+	{"ant_switch", PHYDM_ANT_SWITCH},
+	{"drp", PHYDM_DYNAMIC_RA_PATH},
+	{"psd", PHYDM_PSD},
+	{"dbgport", PHYDM_DEBUG_PORT},
+};
+
+void phydm_cmd_parser(struct phy_dm_struct *dm, char input[][MAX_ARGV],
+		      u32 input_num, u8 flag, char *output, u32 out_len)
+{
+	u32 used = 0;
+	u8 id = 0;
+	int var1[10] = {0};
+	int i, input_idx = 0, phydm_ary_size;
+	char help[] = "-h";
+
+	bool is_enable_dbg_mode;
+	u8 central_ch, primary_ch_idx, bandwidth;
+
+	if (flag == 0) {
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "GET, nothing to print\n");
+		return;
+	}
+
+	PHYDM_SNPRINTF(output + used, out_len - used, "\n");
+
+	/* Parsing Cmd ID */
+	if (input_num) {
+		phydm_ary_size =
+			sizeof(phy_dm_ary) / sizeof(struct phydm_command);
+		for (i = 0; i < phydm_ary_size; i++) {
+			if (strcmp(phy_dm_ary[i].name, input[0]) == 0) {
+				id = phy_dm_ary[i].id;
+				break;
+			}
+		}
+		if (i == phydm_ary_size) {
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "SET, command not found!\n");
+			return;
+		}
+	}
+
+	switch (id) {
+	case PHYDM_HELP: {
+		PHYDM_SNPRINTF(output + used, out_len - used, "BB cmd ==>\n");
+		for (i = 0; i < phydm_ary_size - 2; i++) {
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "  %-5d: %s\n", i,
+				       phy_dm_ary[i + 2].name);
+			/**/
+		}
+	} break;
+
+	case PHYDM_DEMO: { /*echo demo 10 0x3a z abcde >cmd*/
+		u32 directory = 0;
+
+		char char_temp;
+
+		PHYDM_SSCANF(input[1], DCMD_DECIMAL, &directory);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "Decimal value = %d\n", directory);
+		PHYDM_SSCANF(input[2], DCMD_HEX, &directory);
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "Hex value = 0x%x\n", directory);
+		PHYDM_SSCANF(input[3], DCMD_CHAR, &char_temp);
+		PHYDM_SNPRINTF(output + used, out_len - used, "Char = %c\n",
+			       char_temp);
+		PHYDM_SNPRINTF(output + used, out_len - used, "String = %s\n",
+			       input[4]);
+	} break;
+
+	case PHYDM_RA:
+
+		for (i = 0; i < 5; i++) {
+			if (input[i + 1]) {
+				PHYDM_SSCANF(input[i + 1], DCMD_DECIMAL,
+					     &var1[i]);
+
+				input_idx++;
+			}
+		}
+
+		if (input_idx >= 1) {
+			phydm_RA_debug_PCR(dm, (u32 *)var1, &used, output,
+					   &out_len);
+		}
+
+		break;
+
+	case PHYDM_ANTDIV:
+
+		for (i = 0; i < 5; i++) {
+			if (input[i + 1]) {
+				PHYDM_SSCANF(input[i + 1], DCMD_HEX, &var1[i]);
+
+				input_idx++;
+			}
+		}
+
+		break;
+
+	case PHYDM_PATHDIV:
+
+		for (i = 0; i < 5; i++) {
+			if (input[i + 1]) {
+				PHYDM_SSCANF(input[i + 1], DCMD_HEX, &var1[i]);
+
+				input_idx++;
+			}
+		}
+
+		break;
+
+	case PHYDM_DEBUG:
+
+		for (i = 0; i < 5; i++) {
+			if (input[i + 1]) {
+				PHYDM_SSCANF(input[i + 1], DCMD_DECIMAL,
+					     &var1[i]);
+
+				input_idx++;
+			}
+		}
+
+		if (input_idx >= 1) {
+			phydm_debug_trace(dm, (u32 *)var1, &used, output,
+					  &out_len);
+		}
+
+		break;
+
+	case PHYDM_FW_DEBUG:
+
+		for (i = 0; i < 5; i++) {
+			if (input[i + 1]) {
+				PHYDM_SSCANF(input[i + 1], DCMD_DECIMAL,
+					     &var1[i]);
+				input_idx++;
+			}
+		}
+
+		if (input_idx >= 1)
+			phydm_fw_debug_trace(dm, (u32 *)var1, &used, output,
+					     &out_len);
+
+		break;
+
+	case PHYDM_SUPPORT_ABILITY:
+
+		for (i = 0; i < 5; i++) {
+			if (input[i + 1]) {
+				PHYDM_SSCANF(input[i + 1], DCMD_DECIMAL,
+					     &var1[i]);
+
+				input_idx++;
+			}
+		}
+
+		if (input_idx >= 1) {
+			phydm_support_ability_debug(dm, (u32 *)var1, &used,
+						    output, &out_len);
+		}
+
+		break;
+
+	case PHYDM_SMART_ANT:
+
+		for (i = 0; i < 5; i++) {
+			if (input[i + 1]) {
+				PHYDM_SSCANF(input[i + 1], DCMD_HEX, &var1[i]);
+				input_idx++;
+			}
+		}
+
+		break;
+
+	case PHYDM_API:
+		if (!(dm->support_ic_type &
+		      (ODM_RTL8822B | ODM_RTL8197F | ODM_RTL8821C))) {
+			PHYDM_SNPRINTF(
+				output + used, out_len - used,
+				"This IC doesn't support PHYDM API function\n");
+		}
+
+		for (i = 0; i < 4; i++) {
+			if (input[i + 1])
+				PHYDM_SSCANF(input[i + 1], DCMD_DECIMAL,
+					     &var1[i]);
+		}
+
+		is_enable_dbg_mode = (bool)var1[0];
+		central_ch = (u8)var1[1];
+		primary_ch_idx = (u8)var1[2];
+		bandwidth = (enum odm_bw)var1[3];
+
+		if (is_enable_dbg_mode) {
+			dm->is_disable_phy_api = false;
+			phydm_api_switch_bw_channel(dm, central_ch,
+						    primary_ch_idx,
+						    (enum odm_bw)bandwidth);
+			dm->is_disable_phy_api = true;
+			PHYDM_SNPRINTF(
+				output + used, out_len - used,
+				"central_ch = %d, primary_ch_idx = %d, bandwidth = %d\n",
+				central_ch, primary_ch_idx, bandwidth);
+		} else {
+			dm->is_disable_phy_api = false;
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "Disable API debug mode\n");
+		}
+		break;
+
+	case PHYDM_PROFILE: /*echo profile, >cmd*/
+		phydm_basic_profile(dm, &used, output, &out_len);
+		break;
+
+	case PHYDM_GET_TXAGC:
+		phydm_get_txagc(dm, &used, output, &out_len);
+		break;
+
+	case PHYDM_SET_TXAGC: {
+		bool is_enable_dbg_mode;
+
+		for (i = 0; i < 5; i++) {
+			if (input[i + 1]) {
+				PHYDM_SSCANF(input[i + 1], DCMD_HEX, &var1[i]);
+				input_idx++;
+			}
+		}
+
+		if ((strcmp(input[1], help) == 0)) {
+			PHYDM_SNPRINTF(
+				output + used, out_len - used,
+				"{En} {pathA~D(0~3)} {rate_idx(Hex), All_rate:0xff} {txagc_idx (Hex)}\n");
+			/**/
+
+		} else {
+			is_enable_dbg_mode = (bool)var1[0];
+			if (is_enable_dbg_mode) {
+				dm->is_disable_phy_api = false;
+				phydm_set_txagc(dm, (u32 *)var1, &used, output,
+						&out_len);
+				dm->is_disable_phy_api = true;
+			} else {
+				dm->is_disable_phy_api = false;
+				PHYDM_SNPRINTF(output + used, out_len - used,
+					       "Disable API debug mode\n");
+			}
+		}
+	} break;
+
+	case PHYDM_TRX_PATH:
+
+		for (i = 0; i < 4; i++) {
+			if (input[i + 1])
+				PHYDM_SSCANF(input[i + 1], DCMD_DECIMAL,
+					     &var1[i]);
+		}
+		if (dm->support_ic_type & (ODM_RTL8822B | ODM_RTL8197F)) {
+			u8 tx_path, rx_path;
+			bool is_enable_dbg_mode, is_tx2_path;
+
+			is_enable_dbg_mode = (bool)var1[0];
+			tx_path = (u8)var1[1];
+			rx_path = (u8)var1[2];
+			is_tx2_path = (bool)var1[3];
+
+			if (is_enable_dbg_mode) {
+				dm->is_disable_phy_api = false;
+				phydm_api_trx_mode(
+					dm, (enum odm_rf_path)tx_path,
+					(enum odm_rf_path)rx_path, is_tx2_path);
+				dm->is_disable_phy_api = true;
+				PHYDM_SNPRINTF(
+					output + used, out_len - used,
+					"tx_path = 0x%x, rx_path = 0x%x, is_tx2_path = %d\n",
+					tx_path, rx_path, is_tx2_path);
+			} else {
+				dm->is_disable_phy_api = false;
+				PHYDM_SNPRINTF(output + used, out_len - used,
+					       "Disable API debug mode\n");
+			}
+		} else {
+			phydm_config_trx_path(dm, (u32 *)var1, &used, output,
+					      &out_len);
+		}
+		break;
+
+	case PHYDM_LA_MODE:
+
+		dm->support_ability &= ~(ODM_BB_FA_CNT);
+		phydm_lamode_trigger_setting(dm, &input[0], &used, output,
+					     &out_len, input_num);
+		dm->support_ability |= ODM_BB_FA_CNT;
+
+		break;
+
+	case PHYDM_DUMP_REG: {
+		u8 type = 0;
+
+		if (input[1]) {
+			PHYDM_SSCANF(input[1], DCMD_DECIMAL, &var1[0]);
+			type = (u8)var1[0];
+		}
+
+		if (type == 0)
+			phydm_dump_bb_reg(dm, &used, output, &out_len);
+		else if (type == 1)
+			phydm_dump_all_reg(dm, &used, output, &out_len);
+	} break;
+
+	case PHYDM_MU_MIMO:
+
+		if (input[1])
+			PHYDM_SSCANF(input[1], DCMD_DECIMAL, &var1[0]);
+		else
+			var1[0] = 0;
+
+		if (var1[0] == 1) {
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "Get MU BFee CSI\n");
+			odm_set_bb_reg(dm, 0x9e8, BIT(17) | BIT(16),
+				       2); /*Read BFee*/
+			odm_set_bb_reg(dm, 0x1910, BIT(15),
+				       1); /*Select BFee's CSI report*/
+			odm_set_bb_reg(dm, 0x19b8, BIT(6),
+				       1); /*set as CSI report*/
+			odm_set_bb_reg(dm, 0x19a8, 0xFFFF,
+				       0xFFFF); /*disable gated_clk*/
+			phydm_print_csi(dm, used, out_len, output);
+
+		} else if (var1[0] == 2) {
+			PHYDM_SSCANF(input[2], DCMD_DECIMAL, &var1[1]);
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "Get MU BFer's STA%d CSI\n", var1[1]);
+			odm_set_bb_reg(dm, 0x9e8, BIT(24), 0); /*Read BFer*/
+			odm_set_bb_reg(dm, 0x9e8, BIT(25),
+				       1); /*enable Read/Write RAM*/
+			odm_set_bb_reg(dm, 0x9e8, BIT(30) | BIT(29) | BIT(28),
+				       var1[1]); /*read which STA's CSI report*/
+			odm_set_bb_reg(dm, 0x1910, BIT(15),
+				       0); /*select BFer's CSI*/
+			odm_set_bb_reg(dm, 0x19e0, 0x00003FC0,
+				       0xFF); /*disable gated_clk*/
+			phydm_print_csi(dm, used, out_len, output);
+		}
+		break;
+
+	case PHYDM_BIG_JUMP: {
+		if (input[1]) {
+			PHYDM_SSCANF(input[1], DCMD_DECIMAL, &var1[0]);
+			phydm_enable_big_jump(dm, (bool)(var1[0]));
+		} else {
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "unknown command!\n");
+		}
+		break;
+	}
+
+	case PHYDM_HANG:
+		phydm_bb_rx_hang_info(dm, &used, output, &out_len);
+		break;
+
+	case PHYDM_SHOW_RXRATE: {
+		u8 rate_idx;
+
+		if (input[1])
+			PHYDM_SSCANF(input[1], DCMD_DECIMAL, &var1[0]);
+
+		if (var1[0] == 1) {
+			phydm_show_rx_rate(dm, &used, output, &out_len);
+		} else {
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "Reset Rx rate counter\n");
+
+			for (rate_idx = 0; rate_idx < 40; rate_idx++) {
+				dm->phy_dbg_info.num_qry_vht_pkt[rate_idx] = 0;
+				dm->phy_dbg_info.num_qry_mu_vht_pkt[rate_idx] =
+					0;
+			}
+		}
+	} break;
+
+	case PHYDM_NBI_EN:
+
+		for (i = 0; i < 5; i++) {
+			if (input[i + 1]) {
+				PHYDM_SSCANF(input[i + 1], DCMD_DECIMAL,
+					     &var1[i]);
+				input_idx++;
+			}
+		}
+
+		if (input_idx >= 1) {
+			phydm_api_debug(dm, PHYDM_API_NBI, (u32 *)var1, &used,
+					output, &out_len);
+			/**/
+		}
+
+		break;
+
+	case PHYDM_CSI_MASK_EN:
+
+		for (i = 0; i < 5; i++) {
+			if (input[i + 1]) {
+				PHYDM_SSCANF(input[i + 1], DCMD_DECIMAL,
+					     &var1[i]);
+				input_idx++;
+			}
+		}
+
+		if (input_idx >= 1) {
+			phydm_api_debug(dm, PHYDM_API_CSI_MASK, (u32 *)var1,
+					&used, output, &out_len);
+			/**/
+		}
+
+		break;
+
+	case PHYDM_DFS:
+		break;
+
+	case PHYDM_IQK:
+		break;
+
+	case PHYDM_NHM: {
+		u8 target_rssi;
+		u16 nhm_period = 0xC350; /* 200ms */
+		u8 IGI;
+		struct ccx_info *ccx_info = &dm->dm_ccx_info;
+
+		PHYDM_SSCANF(input[1], DCMD_DECIMAL, &var1[0]);
+
+		if (input_num == 1) {
+			ccx_info->echo_NHM_en = false;
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "\r\n Trigger NHM: echo nhm 1\n");
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "\r (Exclude CCA)\n");
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "\r Trigger NHM: echo nhm 2\n");
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "\r (Include CCA)\n");
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "\r Get NHM results: echo nhm 3\n");
+
+			return;
+		}
+
+		/* NMH trigger */
+		if ((var1[0] <= 2) && (var1[0] != 0)) {
+			ccx_info->echo_NHM_en = true;
+			ccx_info->echo_IGI =
+				(u8)odm_get_bb_reg(dm, 0xC50, MASKBYTE0);
+
+			target_rssi = ccx_info->echo_IGI - 10;
+
+			ccx_info->NHM_th[0] = (target_rssi - 15 + 10) * 2;
+
+			for (i = 1; i <= 10; i++)
+				ccx_info->NHM_th[i] =
+					ccx_info->NHM_th[0] + 6 * i;
+
+			/* 4 1. store previous NHM setting */
+			phydm_nhm_setting(dm, STORE_NHM_SETTING);
+
+			/* 4 2. Set NHM period, 0x990[31:16]=0xC350,
+			 * Time duration for NHM unit: 4us, 0xC350=200ms
+			 */
+			ccx_info->NHM_period = nhm_period;
+
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "\r\n Monitor NHM for %d us",
+				       nhm_period * 4);
+
+			/* 4 3. Set NHM inexclude_txon, inexclude_cca, ccx_en */
+
+			ccx_info->nhm_inexclude_cca = (var1[0] == 1) ?
+							      NHM_EXCLUDE_CCA :
+							      NHM_INCLUDE_CCA;
+			ccx_info->nhm_inexclude_txon = NHM_EXCLUDE_TXON;
+
+			phydm_nhm_setting(dm, SET_NHM_SETTING);
+			phydm_print_nhm_trigger(output, used, out_len,
+						ccx_info);
+
+			/* 4 4. Trigger NHM */
+			phydm_nhm_trigger(dm);
+		}
+
+		/*Get NHM results*/
+		else if (var1[0] == 3) {
+			IGI = (u8)odm_get_bb_reg(dm, 0xC50, MASKBYTE0);
+
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "\r\n Cur_IGI = 0x%x", IGI);
+
+			phydm_get_nhm_result(dm);
+
+			/* 4 Resotre NHM setting */
+			phydm_nhm_setting(dm, RESTORE_NHM_SETTING);
+			phydm_print_nhm_result(output, used, out_len, ccx_info);
+
+			ccx_info->echo_NHM_en = false;
+		} else {
+			ccx_info->echo_NHM_en = false;
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "\r\n Trigger NHM: echo nhm 1\n");
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "\r (Exclude CCA)\n");
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "\r Trigger NHM: echo nhm 2\n");
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "\r (Include CCA)\n");
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "\r Get NHM results: echo nhm 3\n");
+
+			return;
+		}
+	} break;
+
+	case PHYDM_CLM: {
+		struct ccx_info *ccx_info = &dm->dm_ccx_info;
+
+		PHYDM_SSCANF(input[1], DCMD_DECIMAL, &var1[0]);
+
+		if (input_num == 1) {
+			ccx_info->echo_CLM_en = false;
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "\r\n Trigger CLM: echo clm 1\n");
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "\r Get CLM results: echo clm 2\n");
+			return;
+		}
+
+		/* Set & trigger CLM */
+		if (var1[0] == 1) {
+			ccx_info->echo_CLM_en = true;
+			ccx_info->CLM_period = 0xC350; /*100ms*/
+			phydm_clm_setting(dm);
+			phydm_clm_trigger(dm);
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "\r\n Monitor CLM for 200ms\n");
+		}
+
+		/* Get CLM results */
+		else if (var1[0] == 2) {
+			ccx_info->echo_CLM_en = false;
+			phydm_get_cl_mresult(dm);
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "\r\n CLM_result = %d us\n",
+				       ccx_info->CLM_result * 4);
+
+		} else {
+			ccx_info->echo_CLM_en = false;
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "\n\r Error command !\n");
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "\r Trigger CLM: echo clm 1\n");
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "\r Get CLM results: echo clm 2\n");
+		}
+	} break;
+
+	case PHYDM_BB_INFO: {
+		s32 value32 = 0;
+
+		phydm_bb_debug_info(dm, &used, output, &out_len);
+
+		if (dm->support_ic_type & ODM_RTL8822B && input[1]) {
+			PHYDM_SSCANF(input[1], DCMD_DECIMAL, &var1[0]);
+			odm_set_bb_reg(dm, 0x1988, 0x003fff00, var1[0]);
+			value32 = odm_get_bb_reg(dm, 0xf84, MASKDWORD);
+			value32 = (value32 & 0xff000000) >> 24;
+			PHYDM_SNPRINTF(
+				output + used, out_len - used,
+				"\r\n %-35s = condition num = %d, subcarriers = %d\n",
+				"Over condition num subcarrier", var1[0],
+				value32);
+			odm_set_bb_reg(dm, 0x1988, BIT(22),
+				       0x0); /*disable report condition number*/
+		}
+	} break;
+
+	case PHYDM_TXBF: {
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "\r\n no TxBF !!\n");
+	} break;
+
+	case PHYDM_PAUSE_DIG_EN:
+
+		for (i = 0; i < 5; i++) {
+			if (input[i + 1]) {
+				PHYDM_SSCANF(input[i + 1], DCMD_HEX, &var1[i]);
+				input_idx++;
+			}
+		}
+
+		if (input_idx >= 1) {
+			if (var1[0] == 0) {
+				odm_pause_dig(dm, PHYDM_PAUSE,
+					      PHYDM_PAUSE_LEVEL_7, (u8)var1[1]);
+				PHYDM_SNPRINTF(output + used, out_len - used,
+					       "Set IGI_value = ((%x))\n",
+					       var1[1]);
+			} else if (var1[0] == 1) {
+				odm_pause_dig(dm, PHYDM_RESUME,
+					      PHYDM_PAUSE_LEVEL_7, (u8)var1[1]);
+				PHYDM_SNPRINTF(output + used, out_len - used,
+					       "Resume IGI_value\n");
+			} else {
+				PHYDM_SNPRINTF(
+					output + used, out_len - used,
+					"echo  (1:pause, 2resume)  (IGI_value)\n");
+			}
+		}
+		break;
+	case PHYDM_H2C:
+
+		for (i = 0; i < 8; i++) {
+			if (input[i + 1]) {
+				PHYDM_SSCANF(input[i + 1], DCMD_HEX, &var1[i]);
+				input_idx++;
+			}
+		}
+
+		if (input_idx >= 1)
+			phydm_h2C_debug(dm, (u32 *)var1, &used, output,
+					&out_len);
+
+		break;
+
+	case PHYDM_ANT_SWITCH:
+
+		for (i = 0; i < 8; i++) {
+			if (input[i + 1]) {
+				PHYDM_SSCANF(input[i + 1], DCMD_DECIMAL,
+					     &var1[i]);
+				input_idx++;
+			}
+		}
+
+		if (input_idx >= 1) {
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "Not Support IC");
+		}
+
+		break;
+
+	case PHYDM_DYNAMIC_RA_PATH:
+
+		PHYDM_SNPRINTF(output + used, out_len - used, "Not Support IC");
+
+		break;
+
+	case PHYDM_PSD:
+
+		phydm_psd_debug(dm, &input[0], &used, output, &out_len,
+				input_num);
+
+		break;
+
+	case PHYDM_DEBUG_PORT: {
+		u32 dbg_port_value;
+
+		PHYDM_SSCANF(input[1], DCMD_HEX, &var1[0]);
+
+		if (phydm_set_bb_dbg_port(dm, BB_DBGPORT_PRIORITY_3,
+					  var1[0])) { /*set debug port to 0x0*/
+
+			dbg_port_value = phydm_get_bb_dbg_port_value(dm);
+			phydm_release_bb_dbg_port(dm);
+
+			PHYDM_SNPRINTF(output + used, out_len - used,
+				       "Debug Port[0x%x] = ((0x%x))\n", var1[1],
+				       dbg_port_value);
+		}
+	} break;
+
+	default:
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "SET, unknown command!\n");
+		break;
+	}
+}
+
+s32 phydm_cmd(struct phy_dm_struct *dm, char *input, u32 in_len, u8 flag,
+	      char *output, u32 out_len)
+{
+	char *token;
+	u32 argc = 0;
+	char argv[MAX_ARGC][MAX_ARGV];
+
+	do {
+		token = strsep(&input, ", ");
+		if (token) {
+			strcpy(argv[argc], token);
+			argc++;
+		} else {
+			break;
+		}
+	} while (argc < MAX_ARGC);
+
+	if (argc == 1)
+		argv[0][strlen(argv[0]) - 1] = '\0';
+
+	phydm_cmd_parser(dm, argv, argc, flag, output, out_len);
+
+	return 0;
+}
+
+void phydm_fw_trace_handler(void *dm_void, u8 *cmd_buf, u8 cmd_len)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	/*u8	debug_trace_11byte[60];*/
+	u8 freg_num, c2h_seq, buf_0 = 0;
+
+	if (!(dm->support_ic_type & PHYDM_IC_3081_SERIES))
+		return;
+
+	if (cmd_len > 12)
+		return;
+
+	buf_0 = cmd_buf[0];
+	freg_num = (buf_0 & 0xf);
+	c2h_seq = (buf_0 & 0xf0) >> 4;
+
+	if ((c2h_seq != dm->pre_c2h_seq) && !dm->fw_buff_is_enpty) {
+		dm->fw_debug_trace[dm->c2h_cmd_start] = '\0';
+		ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE,
+			     "[FW Dbg Queue Overflow] %s\n",
+			     dm->fw_debug_trace);
+		dm->c2h_cmd_start = 0;
+	}
+
+	if ((cmd_len - 1) > (60 - dm->c2h_cmd_start)) {
+		dm->fw_debug_trace[dm->c2h_cmd_start] = '\0';
+		ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE,
+			     "[FW Dbg Queue error: wrong C2H length] %s\n",
+			     dm->fw_debug_trace);
+		dm->c2h_cmd_start = 0;
+		return;
+	}
+
+	strncpy((char *)&dm->fw_debug_trace[dm->c2h_cmd_start],
+		(char *)&cmd_buf[1], (cmd_len - 1));
+	dm->c2h_cmd_start += (cmd_len - 1);
+	dm->fw_buff_is_enpty = false;
+
+	if (freg_num == 0 || dm->c2h_cmd_start >= 60) {
+		if (dm->c2h_cmd_start < 60)
+			dm->fw_debug_trace[dm->c2h_cmd_start] = '\0';
+		else
+			dm->fw_debug_trace[59] = '\0';
+
+		ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "[FW DBG Msg] %s\n",
+			     dm->fw_debug_trace);
+		/*dbg_print("[FW DBG Msg] %s\n", dm->fw_debug_trace);*/
+		dm->c2h_cmd_start = 0;
+		dm->fw_buff_is_enpty = true;
+	}
+
+	dm->pre_c2h_seq = c2h_seq;
+}
+
+void phydm_fw_trace_handler_code(void *dm_void, u8 *buffer, u8 cmd_len)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u8 function = buffer[0];
+	u8 dbg_num = buffer[1];
+	u16 content_0 = (((u16)buffer[3]) << 8) | ((u16)buffer[2]);
+	u16 content_1 = (((u16)buffer[5]) << 8) | ((u16)buffer[4]);
+	u16 content_2 = (((u16)buffer[7]) << 8) | ((u16)buffer[6]);
+	u16 content_3 = (((u16)buffer[9]) << 8) | ((u16)buffer[8]);
+	u16 content_4 = (((u16)buffer[11]) << 8) | ((u16)buffer[10]);
+
+	if (cmd_len > 12)
+		ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE,
+			     "[FW Msg] Invalid cmd length (( %d )) >12\n",
+			     cmd_len);
+
+	/*--------------------------------------------*/
+	ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE,
+		     "[FW][general][%d, %d, %d] = {%d, %d, %d, %d}\n", function,
+		     dbg_num, content_0, content_1, content_2, content_3,
+		     content_4);
+	/*--------------------------------------------*/
+}
+
+void phydm_fw_trace_handler_8051(void *dm_void, u8 *buffer, u8 cmd_len)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	int i = 0;
+	u8 extend_c2h_sub_id = 0, extend_c2h_dbg_len = 0,
+	   extend_c2h_dbg_seq = 0;
+	u8 fw_debug_trace[128];
+	u8 *extend_c2h_dbg_content = NULL;
+
+	if (cmd_len > 127)
+		return;
+
+	extend_c2h_sub_id = buffer[0];
+	extend_c2h_dbg_len = buffer[1];
+	extend_c2h_dbg_content = buffer + 2; /*DbgSeq+DbgContent  for show HEX*/
+
+go_backfor_aggre_dbg_pkt:
+	i = 0;
+	extend_c2h_dbg_seq = buffer[2];
+	extend_c2h_dbg_content = buffer + 3;
+
+	for (;; i++) {
+		fw_debug_trace[i] = extend_c2h_dbg_content[i];
+		if (extend_c2h_dbg_content[i + 1] == '\0') {
+			fw_debug_trace[i + 1] = '\0';
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "[FW DBG Msg] %s",
+				     &fw_debug_trace[0]);
+			break;
+		} else if (extend_c2h_dbg_content[i] == '\n') {
+			fw_debug_trace[i + 1] = '\0';
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "[FW DBG Msg] %s",
+				     &fw_debug_trace[0]);
+			buffer = extend_c2h_dbg_content + i + 3;
+			goto go_backfor_aggre_dbg_pkt;
+		}
+	}
+}
diff --git a/drivers/staging/rtlwifi/phydm/phydm_debug.h b/drivers/staging/rtlwifi/phydm/phydm_debug.h
new file mode 100644
index 000000000000..f442f7c19595
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_debug.h
@@ -0,0 +1,175 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __ODM_DBG_H__
+#define __ODM_DBG_H__
+
+/*#define DEBUG_VERSION	"1.1"*/ /*2015.07.29 YuChen*/
+/*#define DEBUG_VERSION	"1.2"*/ /*2015.08.28 Dino*/
+#define DEBUG_VERSION "1.3" /*2016.04.28 YuChen*/
+#define ODM_DBG_TRACE 5
+
+/*FW DBG MSG*/
+#define RATE_DECISION BIT(0)
+#define INIT_RA_TABLE BIT(1)
+#define RATE_UP BIT(2)
+#define RATE_DOWN BIT(3)
+#define TRY_DONE BIT(4)
+#define RA_H2C BIT(5)
+#define F_RATE_AP_RPT BIT(7)
+
+/* -----------------------------------------------------------------------------
+ * Define the tracing components
+ *
+ * -----------------------------------------------------------------------------
+ */
+/*BB FW Functions*/
+#define PHYDM_FW_COMP_RA BIT(0)
+#define PHYDM_FW_COMP_MU BIT(1)
+#define PHYDM_FW_COMP_PATH_DIV BIT(2)
+#define PHYDM_FW_COMP_PHY_CONFIG BIT(3)
+
+/*BB Driver Functions*/
+#define ODM_COMP_DIG BIT(0)
+#define ODM_COMP_RA_MASK BIT(1)
+#define ODM_COMP_DYNAMIC_TXPWR BIT(2)
+#define ODM_COMP_FA_CNT BIT(3)
+#define ODM_COMP_RSSI_MONITOR BIT(4)
+#define ODM_COMP_SNIFFER BIT(5)
+#define ODM_COMP_ANT_DIV BIT(6)
+#define ODM_COMP_DFS BIT(7)
+#define ODM_COMP_NOISY_DETECT BIT(8)
+#define ODM_COMP_RATE_ADAPTIVE BIT(9)
+#define ODM_COMP_PATH_DIV BIT(10)
+#define ODM_COMP_CCX BIT(11)
+
+#define ODM_COMP_DYNAMIC_PRICCA BIT(12)
+/*BIT13 TBD*/
+#define ODM_COMP_MP BIT(14)
+#define ODM_COMP_CFO_TRACKING BIT(15)
+#define ODM_COMP_ACS BIT(16)
+#define PHYDM_COMP_ADAPTIVITY BIT(17)
+#define PHYDM_COMP_RA_DBG BIT(18)
+#define PHYDM_COMP_TXBF BIT(19)
+/* MAC Functions */
+#define ODM_COMP_EDCA_TURBO BIT(20)
+#define ODM_COMP_DYNAMIC_RX_PATH BIT(21)
+#define ODM_FW_DEBUG_TRACE BIT(22)
+/* RF Functions */
+/*BIT23 TBD*/
+#define ODM_COMP_TX_PWR_TRACK BIT(24)
+/*BIT25 TBD*/
+#define ODM_COMP_CALIBRATION BIT(26)
+/* Common Functions */
+/*BIT27 TBD*/
+#define ODM_PHY_CONFIG BIT(28)
+#define ODM_COMP_INIT BIT(29)
+#define ODM_COMP_COMMON BIT(30)
+#define ODM_COMP_API BIT(31)
+
+#define ODM_COMP_UNCOND 0xFFFFFFFF
+
+/*------------------------Export Marco Definition---------------------------*/
+
+#define config_phydm_read_txagc_check(data) (data != INVALID_TXAGC_DATA)
+
+#define ODM_RT_TRACE(dm, comp, fmt, ...)                                       \
+	do {                                                                   \
+		if (((comp) & dm->debug_components) ||                         \
+		    ((comp) == ODM_COMP_UNCOND))                               \
+			RT_TRACE(dm->adapter, COMP_PHYDM, DBG_DMESG, fmt,      \
+				 ##__VA_ARGS__);                               \
+	} while (0)
+
+#define BB_DBGPORT_PRIORITY_3 3 /*Debug function (the highest priority)*/
+#define BB_DBGPORT_PRIORITY_2 2 /*Check hang function & Strong function*/
+#define BB_DBGPORT_PRIORITY_1 1 /*Watch dog function*/
+#define BB_DBGPORT_RELEASE 0 /*Init value (the lowest priority)*/
+
+void phydm_init_debug_setting(struct phy_dm_struct *dm);
+
+u8 phydm_set_bb_dbg_port(void *dm_void, u8 curr_dbg_priority, u32 debug_port);
+
+void phydm_release_bb_dbg_port(void *dm_void);
+
+u32 phydm_get_bb_dbg_port_value(void *dm_void);
+
+void phydm_basic_dbg_message(void *dm_void);
+
+#define PHYDM_DBGPRINT 0
+#define MAX_ARGC 20
+#define MAX_ARGV 16
+#define DCMD_DECIMAL "%d"
+#define DCMD_CHAR "%c"
+#define DCMD_HEX "%x"
+
+#define PHYDM_SSCANF(x, y, z)                                                  \
+	do {                                                                   \
+		if (sscanf(x, y, z) != 1)                                      \
+			ODM_RT_TRACE(dm, ODM_COMP_UNCOND,                      \
+				     "%s:%d sscanf fail!", __func__,           \
+				     __LINE__);                                \
+	} while (0)
+
+#define PHYDM_VAST_INFO_SNPRINTF(msg, ...)                                     \
+	do {                                                                   \
+		snprintf(msg, ##__VA_ARGS__);                                  \
+		ODM_RT_TRACE(dm, ODM_COMP_UNCOND, output);                     \
+	} while (0)
+
+#if (PHYDM_DBGPRINT == 1)
+#define PHYDM_SNPRINTF(msg, ...)                                               \
+	do {                                                                   \
+		snprintf(msg, ##__VA_ARGS__);                                  \
+		ODM_RT_TRACE(dm, ODM_COMP_UNCOND, output);                     \
+	} while (0)
+#else
+#define PHYDM_SNPRINTF(msg, ...)                                               \
+	do {                                                                   \
+		if (out_len > used)                                            \
+			used += snprintf(msg, ##__VA_ARGS__);                  \
+	} while (0)
+#endif
+
+void phydm_basic_profile(void *dm_void, u32 *_used, char *output,
+			 u32 *_out_len);
+s32 phydm_cmd(struct phy_dm_struct *dm, char *input, u32 in_len, u8 flag,
+	      char *output, u32 out_len);
+void phydm_cmd_parser(struct phy_dm_struct *dm, char input[][16], u32 input_num,
+		      u8 flag, char *output, u32 out_len);
+
+bool phydm_api_trx_mode(struct phy_dm_struct *dm, enum odm_rf_path tx_path,
+			enum odm_rf_path rx_path, bool is_tx2_path);
+
+void phydm_fw_trace_en_h2c(void *dm_void, bool enable, u32 fw_debug_component,
+			   u32 monitor_mode, u32 macid);
+
+void phydm_fw_trace_handler(void *dm_void, u8 *cmd_buf, u8 cmd_len);
+
+void phydm_fw_trace_handler_code(void *dm_void, u8 *buffer, u8 cmd_len);
+
+void phydm_fw_trace_handler_8051(void *dm_void, u8 *cmd_buf, u8 cmd_len);
+
+#endif /* __ODM_DBG_H__ */
diff --git a/drivers/staging/rtlwifi/phydm/phydm_dfs.h b/drivers/staging/rtlwifi/phydm/phydm_dfs.h
new file mode 100644
index 000000000000..59a1d08cf381
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_dfs.h
@@ -0,0 +1,59 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __PHYDM_DFS_H__
+#define __PHYDM_DFS_H__
+
+#define DFS_VERSION "0.0"
+
+/* ============================================================
+ *  Definition
+ * ============================================================
+ */
+
+/* ============================================================
+ * 1  structure
+ * ============================================================
+ */
+
+/* ============================================================
+ *  enumeration
+ * ============================================================
+ */
+
+enum phydm_dfs_region_domain {
+	PHYDM_DFS_DOMAIN_UNKNOWN = 0,
+	PHYDM_DFS_DOMAIN_FCC = 1,
+	PHYDM_DFS_DOMAIN_MKK = 2,
+	PHYDM_DFS_DOMAIN_ETSI = 3,
+};
+
+/* ============================================================
+ *  function prototype
+ * ============================================================
+ */
+#define phydm_dfs_master_enabled(dm) false
+
+#endif /*#ifndef __PHYDM_DFS_H__ */
diff --git a/drivers/staging/rtlwifi/phydm/phydm_dig.c b/drivers/staging/rtlwifi/phydm/phydm_dig.c
new file mode 100644
index 000000000000..31a4f3fcad19
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_dig.c
@@ -0,0 +1,1535 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+/* ************************************************************
+ * include files
+ * *************************************************************/
+#include "mp_precomp.h"
+#include "phydm_precomp.h"
+
+static int get_igi_for_diff(int);
+
+static inline void phydm_check_ap_write_dig(struct phy_dm_struct *dm,
+					    u8 current_igi)
+{
+	switch (*dm->one_path_cca) {
+	case ODM_CCA_2R:
+		odm_set_bb_reg(dm, ODM_REG(IGI_A, dm), ODM_BIT(IGI, dm),
+			       current_igi);
+
+		if (dm->rf_type > ODM_1T1R)
+			odm_set_bb_reg(dm, ODM_REG(IGI_B, dm), ODM_BIT(IGI, dm),
+				       current_igi);
+		break;
+	case ODM_CCA_1R_A:
+		odm_set_bb_reg(dm, ODM_REG(IGI_A, dm), ODM_BIT(IGI, dm),
+			       current_igi);
+		if (dm->rf_type != ODM_1T1R)
+			odm_set_bb_reg(dm, ODM_REG(IGI_B, dm), ODM_BIT(IGI, dm),
+				       get_igi_for_diff(current_igi));
+		break;
+	case ODM_CCA_1R_B:
+		odm_set_bb_reg(dm, ODM_REG(IGI_B, dm), ODM_BIT(IGI, dm),
+			       get_igi_for_diff(current_igi));
+		if (dm->rf_type != ODM_1T1R)
+			odm_set_bb_reg(dm, ODM_REG(IGI_A, dm), ODM_BIT(IGI, dm),
+				       current_igi);
+		break;
+	}
+}
+
+static inline u8 phydm_get_current_igi(u8 dig_max_of_min, u8 rssi_min,
+				       u8 current_igi)
+{
+	if (rssi_min < dig_max_of_min) {
+		if (current_igi < rssi_min)
+			return rssi_min;
+	} else {
+		if (current_igi < dig_max_of_min)
+			return dig_max_of_min;
+	}
+	return current_igi;
+}
+
+void odm_change_dynamic_init_gain_thresh(void *dm_void, u32 dm_type,
+					 u32 dm_value)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct dig_thres *dig_tab = &dm->dm_dig_table;
+
+	if (dm_type == DIG_TYPE_THRESH_HIGH) {
+		dig_tab->rssi_high_thresh = dm_value;
+	} else if (dm_type == DIG_TYPE_THRESH_LOW) {
+		dig_tab->rssi_low_thresh = dm_value;
+	} else if (dm_type == DIG_TYPE_ENABLE) {
+		dig_tab->dig_enable_flag = true;
+	} else if (dm_type == DIG_TYPE_DISABLE) {
+		dig_tab->dig_enable_flag = false;
+	} else if (dm_type == DIG_TYPE_BACKOFF) {
+		if (dm_value > 30)
+			dm_value = 30;
+		dig_tab->backoff_val = (u8)dm_value;
+	} else if (dm_type == DIG_TYPE_RX_GAIN_MIN) {
+		if (dm_value == 0)
+			dm_value = 0x1;
+		dig_tab->rx_gain_range_min = (u8)dm_value;
+	} else if (dm_type == DIG_TYPE_RX_GAIN_MAX) {
+		if (dm_value > 0x50)
+			dm_value = 0x50;
+		dig_tab->rx_gain_range_max = (u8)dm_value;
+	}
+} /* dm_change_dynamic_init_gain_thresh */
+
+static int get_igi_for_diff(int value_IGI)
+{
+#define ONERCCA_LOW_TH 0x30
+#define ONERCCA_LOW_DIFF 8
+
+	if (value_IGI < ONERCCA_LOW_TH) {
+		if ((ONERCCA_LOW_TH - value_IGI) < ONERCCA_LOW_DIFF)
+			return ONERCCA_LOW_TH;
+		else
+			return value_IGI + ONERCCA_LOW_DIFF;
+	}
+
+	return value_IGI;
+}
+
+static void odm_fa_threshold_check(void *dm_void, bool is_dfs_band,
+				   bool is_performance, u32 rx_tp, u32 tx_tp,
+				   u32 *dm_FA_thres)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	if (dm->is_linked && (is_performance || is_dfs_band)) {
+		/*For NIC*/
+		dm_FA_thres[0] = DM_DIG_FA_TH0;
+		dm_FA_thres[1] = DM_DIG_FA_TH1;
+		dm_FA_thres[2] = DM_DIG_FA_TH2;
+	} else {
+		if (is_dfs_band) {
+			/* For DFS band and no link */
+			dm_FA_thres[0] = 250;
+			dm_FA_thres[1] = 1000;
+			dm_FA_thres[2] = 2000;
+		} else {
+			dm_FA_thres[0] = 2000;
+			dm_FA_thres[1] = 4000;
+			dm_FA_thres[2] = 5000;
+		}
+	}
+}
+
+static u8 odm_forbidden_igi_check(void *dm_void, u8 dig_dynamic_min,
+				  u8 current_igi)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct dig_thres *dig_tab = &dm->dm_dig_table;
+	struct false_alarm_stat *fa_cnt =
+		(struct false_alarm_stat *)phydm_get_structure(
+			dm, PHYDM_FALSEALMCNT);
+	u8 rx_gain_range_min = dig_tab->rx_gain_range_min;
+
+	if (dig_tab->large_fa_timeout) {
+		if (--dig_tab->large_fa_timeout == 0)
+			dig_tab->large_fa_hit = 0;
+	}
+
+	if (fa_cnt->cnt_all > 10000) {
+		ODM_RT_TRACE(dm, ODM_COMP_DIG,
+			     "%s(): Abnormally false alarm case.\n", __func__);
+
+		if (dig_tab->large_fa_hit != 3)
+			dig_tab->large_fa_hit++;
+
+		if (dig_tab->forbidden_igi < current_igi) {
+			dig_tab->forbidden_igi = current_igi;
+			dig_tab->large_fa_hit = 1;
+			dig_tab->large_fa_timeout = LARGE_FA_TIMEOUT;
+		}
+
+		if (dig_tab->large_fa_hit >= 3) {
+			if ((dig_tab->forbidden_igi + 2) >
+			    dig_tab->rx_gain_range_max)
+				rx_gain_range_min = dig_tab->rx_gain_range_max;
+			else
+				rx_gain_range_min =
+					(dig_tab->forbidden_igi + 2);
+			dig_tab->recover_cnt = 1800;
+			ODM_RT_TRACE(
+				dm, ODM_COMP_DIG,
+				"%s(): Abnormally false alarm case: recover_cnt = %d\n",
+				__func__, dig_tab->recover_cnt);
+		}
+	}
+
+	else if (fa_cnt->cnt_all > 2000) {
+		ODM_RT_TRACE(dm, ODM_COMP_DIG,
+			     "Abnormally false alarm case.\n");
+		ODM_RT_TRACE(
+			dm, ODM_COMP_DIG,
+			"cnt_all=%d, cnt_all_pre=%d, current_igi=0x%x, pre_ig_value=0x%x\n",
+			fa_cnt->cnt_all, fa_cnt->cnt_all_pre, current_igi,
+			dig_tab->pre_ig_value);
+
+		/* fa_cnt->cnt_all = 1.1875*fa_cnt->cnt_all_pre */
+		if ((fa_cnt->cnt_all >
+		     (fa_cnt->cnt_all_pre + (fa_cnt->cnt_all_pre >> 3) +
+		      (fa_cnt->cnt_all_pre >> 4))) &&
+		    (current_igi < dig_tab->pre_ig_value)) {
+			if (dig_tab->large_fa_hit != 3)
+				dig_tab->large_fa_hit++;
+
+			if (dig_tab->forbidden_igi < current_igi) {
+				ODM_RT_TRACE(
+					dm, ODM_COMP_DIG,
+					"Updating forbidden_igi by current_igi, forbidden_igi=0x%x, current_igi=0x%x\n",
+					dig_tab->forbidden_igi, current_igi);
+
+				dig_tab->forbidden_igi = current_igi;
+				dig_tab->large_fa_hit = 1;
+				dig_tab->large_fa_timeout = LARGE_FA_TIMEOUT;
+			}
+		}
+
+		if (dig_tab->large_fa_hit >= 3) {
+			ODM_RT_TRACE(
+				dm, ODM_COMP_DIG,
+				"FaHit is greater than 3, rx_gain_range_max=0x%x, rx_gain_range_min=0x%x, forbidden_igi=0x%x\n",
+				dig_tab->rx_gain_range_max, rx_gain_range_min,
+				dig_tab->forbidden_igi);
+
+			if ((dig_tab->forbidden_igi + 1) >
+			    dig_tab->rx_gain_range_max)
+				rx_gain_range_min = dig_tab->rx_gain_range_max;
+			else
+				rx_gain_range_min =
+					(dig_tab->forbidden_igi + 1);
+
+			dig_tab->recover_cnt = 1200;
+			ODM_RT_TRACE(
+				dm, ODM_COMP_DIG,
+				"Abnormally false alarm case: recover_cnt = %d,  rx_gain_range_min = 0x%x\n",
+				dig_tab->recover_cnt, rx_gain_range_min);
+		}
+	} else {
+		if (dig_tab->recover_cnt != 0) {
+			dig_tab->recover_cnt--;
+			ODM_RT_TRACE(dm, ODM_COMP_DIG,
+				     "%s(): Normal Case: recover_cnt = %d\n",
+				     __func__, dig_tab->recover_cnt);
+			return rx_gain_range_min;
+		}
+
+		if (dig_tab->large_fa_hit >= 3) {
+			dig_tab->large_fa_hit = 0;
+			return rx_gain_range_min;
+		}
+
+		if ((dig_tab->forbidden_igi - 2) <
+		    dig_dynamic_min) { /* DM_DIG_MIN) */
+			dig_tab->forbidden_igi =
+				dig_dynamic_min; /* DM_DIG_MIN; */
+			rx_gain_range_min = dig_dynamic_min; /* DM_DIG_MIN; */
+			ODM_RT_TRACE(dm, ODM_COMP_DIG,
+				     "%s(): Normal Case: At Lower Bound\n",
+				     __func__);
+		} else {
+			if (dig_tab->large_fa_hit == 0) {
+				dig_tab->forbidden_igi -= 2;
+				rx_gain_range_min =
+					(dig_tab->forbidden_igi + 2);
+				ODM_RT_TRACE(
+					dm, ODM_COMP_DIG,
+					"%s(): Normal Case: Approach Lower Bound\n",
+					__func__);
+			}
+		}
+	}
+
+	return rx_gain_range_min;
+}
+
+static void phydm_set_big_jump_step(void *dm_void, u8 current_igi)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct dig_thres *dig_tab = &dm->dm_dig_table;
+	u8 step1[8] = {24, 30, 40, 50, 60, 70, 80, 90};
+	u8 i;
+
+	if (dig_tab->enable_adjust_big_jump == 0)
+		return;
+
+	for (i = 0; i <= dig_tab->big_jump_step1; i++) {
+		if ((current_igi + step1[i]) >
+		    dig_tab->big_jump_lmt[dig_tab->agc_table_idx]) {
+			if (i != 0)
+				i = i - 1;
+			break;
+		} else if (i == dig_tab->big_jump_step1) {
+			break;
+		}
+	}
+	if (dm->support_ic_type & ODM_RTL8822B)
+		odm_set_bb_reg(dm, 0x8c8, 0xe, i);
+	else if (dm->support_ic_type & ODM_RTL8197F)
+		odm_set_bb_reg(dm, ODM_REG_BB_AGC_SET_2_11N, 0xe, i);
+
+	ODM_RT_TRACE(dm, ODM_COMP_DIG,
+		     "%s(): bigjump = %d (ori = 0x%x), LMT=0x%x\n", __func__, i,
+		     dig_tab->big_jump_step1,
+		     dig_tab->big_jump_lmt[dig_tab->agc_table_idx]);
+}
+
+void odm_write_dig(void *dm_void, u8 current_igi)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct dig_thres *dig_tab = &dm->dm_dig_table;
+
+	if (dig_tab->is_stop_dig) {
+		ODM_RT_TRACE(dm, ODM_COMP_DIG, "%s(): Stop Writing IGI\n",
+			     __func__);
+		return;
+	}
+
+	ODM_RT_TRACE(dm, ODM_COMP_DIG,
+		     "%s(): ODM_REG(IGI_A,dm)=0x%x, ODM_BIT(IGI,dm)=0x%x\n",
+		     __func__, ODM_REG(IGI_A, dm), ODM_BIT(IGI, dm));
+
+	/* 1 Check initial gain by upper bound */
+	if ((!dig_tab->is_psd_in_progress) && dm->is_linked) {
+		if (current_igi > dig_tab->rx_gain_range_max) {
+			ODM_RT_TRACE(
+				dm, ODM_COMP_DIG,
+				"%s(): current_igi(0x%02x) is larger than upper bound !!\n",
+				__func__, current_igi);
+			current_igi = dig_tab->rx_gain_range_max;
+		}
+		if (dm->support_ability & ODM_BB_ADAPTIVITY &&
+		    dm->adaptivity_flag) {
+			if (current_igi > dm->adaptivity_igi_upper)
+				current_igi = dm->adaptivity_igi_upper;
+
+			ODM_RT_TRACE(
+				dm, ODM_COMP_DIG,
+				"%s(): adaptivity case: Force upper bound to 0x%x !!!!!!\n",
+				__func__, current_igi);
+		}
+	}
+
+	if (dig_tab->cur_ig_value != current_igi) {
+		/* Modify big jump step for 8822B and 8197F */
+		if (dm->support_ic_type & (ODM_RTL8822B | ODM_RTL8197F))
+			phydm_set_big_jump_step(dm, current_igi);
+
+		/* Set IGI value of CCK for new CCK AGC */
+		if (dm->cck_new_agc) {
+			if (dm->support_ic_type & ODM_IC_PHY_STATUE_NEW_TYPE)
+				odm_set_bb_reg(dm, 0xa0c, 0x00003f00,
+					       (current_igi >> 1));
+		}
+
+		/*Add by YuChen for USB IO too slow issue*/
+		if ((dm->support_ability & ODM_BB_ADAPTIVITY) &&
+		    (current_igi > dig_tab->cur_ig_value)) {
+			dig_tab->cur_ig_value = current_igi;
+			phydm_adaptivity(dm);
+		}
+
+		/* 1 Set IGI value */
+		if (dm->support_platform & (ODM_WIN | ODM_CE)) {
+			odm_set_bb_reg(dm, ODM_REG(IGI_A, dm), ODM_BIT(IGI, dm),
+				       current_igi);
+
+			if (dm->rf_type > ODM_1T1R)
+				odm_set_bb_reg(dm, ODM_REG(IGI_B, dm),
+					       ODM_BIT(IGI, dm), current_igi);
+
+		} else if (dm->support_platform & (ODM_AP)) {
+			phydm_check_ap_write_dig(dm, current_igi);
+		}
+
+		dig_tab->cur_ig_value = current_igi;
+	}
+
+	ODM_RT_TRACE(dm, ODM_COMP_DIG, "%s(): current_igi(0x%02x).\n", __func__,
+		     current_igi);
+}
+
+void odm_pause_dig(void *dm_void, enum phydm_pause_type pause_type,
+		   enum phydm_pause_level pause_level, u8 igi_value)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct dig_thres *dig_tab = &dm->dm_dig_table;
+	s8 max_level;
+
+	ODM_RT_TRACE(dm, ODM_COMP_DIG, "%s()=========> level = %d\n", __func__,
+		     pause_level);
+
+	if ((dig_tab->pause_dig_level == 0) &&
+	    (!(dm->support_ability & ODM_BB_DIG) ||
+	     !(dm->support_ability & ODM_BB_FA_CNT))) {
+		ODM_RT_TRACE(
+			dm, ODM_COMP_DIG,
+			"%s(): Return: support_ability DIG or FA is disabled !!\n",
+			__func__);
+		return;
+	}
+
+	if (pause_level > DM_DIG_MAX_PAUSE_TYPE) {
+		ODM_RT_TRACE(dm, ODM_COMP_DIG,
+			     "%s(): Return: Wrong pause level !!\n", __func__);
+		return;
+	}
+
+	ODM_RT_TRACE(dm, ODM_COMP_DIG,
+		     "%s(): pause level = 0x%x, Current value = 0x%x\n",
+		     __func__, dig_tab->pause_dig_level, igi_value);
+	ODM_RT_TRACE(
+		dm, ODM_COMP_DIG,
+		"%s(): pause value = 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n",
+		__func__, dig_tab->pause_dig_value[7],
+		dig_tab->pause_dig_value[6], dig_tab->pause_dig_value[5],
+		dig_tab->pause_dig_value[4], dig_tab->pause_dig_value[3],
+		dig_tab->pause_dig_value[2], dig_tab->pause_dig_value[1],
+		dig_tab->pause_dig_value[0]);
+
+	switch (pause_type) {
+	/* Pause DIG */
+	case PHYDM_PAUSE: {
+		/* Disable DIG */
+		odm_cmn_info_update(dm, ODM_CMNINFO_ABILITY,
+				    dm->support_ability & (~ODM_BB_DIG));
+		ODM_RT_TRACE(dm, ODM_COMP_DIG, "%s(): Pause DIG !!\n",
+			     __func__);
+
+		/* Backup IGI value */
+		if (dig_tab->pause_dig_level == 0) {
+			dig_tab->igi_backup = dig_tab->cur_ig_value;
+			ODM_RT_TRACE(
+				dm, ODM_COMP_DIG,
+				"%s(): Backup IGI  = 0x%x, new IGI = 0x%x\n",
+				__func__, dig_tab->igi_backup, igi_value);
+		}
+
+		/* Record IGI value */
+		dig_tab->pause_dig_value[pause_level] = igi_value;
+
+		/* Update pause level */
+		dig_tab->pause_dig_level =
+			(dig_tab->pause_dig_level | BIT(pause_level));
+
+		/* Write new IGI value */
+		if (BIT(pause_level + 1) > dig_tab->pause_dig_level) {
+			odm_write_dig(dm, igi_value);
+			ODM_RT_TRACE(dm, ODM_COMP_DIG,
+				     "%s(): IGI of higher level = 0x%x\n",
+				     __func__, igi_value);
+		}
+		break;
+	}
+	/* Resume DIG */
+	case PHYDM_RESUME: {
+		/* check if the level is illegal or not */
+		if ((dig_tab->pause_dig_level & (BIT(pause_level))) != 0) {
+			dig_tab->pause_dig_level = dig_tab->pause_dig_level &
+						   (~(BIT(pause_level)));
+			dig_tab->pause_dig_value[pause_level] = 0;
+			ODM_RT_TRACE(dm, ODM_COMP_DIG, "%s(): Resume DIG !!\n",
+				     __func__);
+		} else {
+			ODM_RT_TRACE(dm, ODM_COMP_DIG,
+				     "%s(): Wrong resume level !!\n", __func__);
+			break;
+		}
+
+		/* Resume DIG */
+		if (dig_tab->pause_dig_level == 0) {
+			/* Write backup IGI value */
+			odm_write_dig(dm, dig_tab->igi_backup);
+			dig_tab->is_ignore_dig = true;
+			ODM_RT_TRACE(dm, ODM_COMP_DIG,
+				     "%s(): Write original IGI = 0x%x\n",
+				     __func__, dig_tab->igi_backup);
+
+			/* Enable DIG */
+			odm_cmn_info_update(dm, ODM_CMNINFO_ABILITY,
+					    dm->support_ability | ODM_BB_DIG);
+			break;
+		}
+
+		if (BIT(pause_level) <= dig_tab->pause_dig_level)
+			break;
+
+		/* Calculate the maximum level now */
+		for (max_level = (pause_level - 1); max_level >= 0;
+		     max_level--) {
+			if ((dig_tab->pause_dig_level & BIT(max_level)) > 0)
+				break;
+		}
+
+		/* write IGI of lower level */
+		odm_write_dig(dm, dig_tab->pause_dig_value[max_level]);
+		ODM_RT_TRACE(dm, ODM_COMP_DIG,
+			     "%s(): Write IGI (0x%x) of level (%d)\n", __func__,
+			     dig_tab->pause_dig_value[max_level], max_level);
+		break;
+	}
+	default:
+		ODM_RT_TRACE(dm, ODM_COMP_DIG, "%s(): Wrong  type !!\n",
+			     __func__);
+		break;
+	}
+
+	ODM_RT_TRACE(dm, ODM_COMP_DIG,
+		     "%s(): pause level = 0x%x, Current value = 0x%x\n",
+		     __func__, dig_tab->pause_dig_level, igi_value);
+	ODM_RT_TRACE(
+		dm, ODM_COMP_DIG,
+		"%s(): pause value = 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n",
+		__func__, dig_tab->pause_dig_value[7],
+		dig_tab->pause_dig_value[6], dig_tab->pause_dig_value[5],
+		dig_tab->pause_dig_value[4], dig_tab->pause_dig_value[3],
+		dig_tab->pause_dig_value[2], dig_tab->pause_dig_value[1],
+		dig_tab->pause_dig_value[0]);
+}
+
+static bool odm_dig_abort(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct dig_thres *dig_tab = &dm->dm_dig_table;
+
+	/* support_ability */
+	if (!(dm->support_ability & ODM_BB_FA_CNT)) {
+		ODM_RT_TRACE(
+			dm, ODM_COMP_DIG,
+			"%s(): Return: support_ability ODM_BB_FA_CNT is disabled\n",
+			__func__);
+		return true;
+	}
+
+	/* support_ability */
+	if (!(dm->support_ability & ODM_BB_DIG)) {
+		ODM_RT_TRACE(
+			dm, ODM_COMP_DIG,
+			"%s(): Return: support_ability ODM_BB_DIG is disabled\n",
+			__func__);
+		return true;
+	}
+
+	/* ScanInProcess */
+	if (*dm->is_scan_in_process) {
+		ODM_RT_TRACE(dm, ODM_COMP_DIG,
+			     "%s(): Return: In Scan Progress\n", __func__);
+		return true;
+	}
+
+	if (dig_tab->is_ignore_dig) {
+		dig_tab->is_ignore_dig = false;
+		ODM_RT_TRACE(dm, ODM_COMP_DIG, "%s(): Return: Ignore DIG\n",
+			     __func__);
+		return true;
+	}
+
+	/* add by Neil Chen to avoid PSD is processing */
+	if (!dm->is_dm_initial_gain_enable) {
+		ODM_RT_TRACE(dm, ODM_COMP_DIG,
+			     "%s(): Return: PSD is Processing\n", __func__);
+		return true;
+	}
+
+	return false;
+}
+
+void odm_dig_init(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct dig_thres *dig_tab = &dm->dm_dig_table;
+	u32 ret_value;
+	u8 i;
+
+	dig_tab->is_stop_dig = false;
+	dig_tab->is_ignore_dig = false;
+	dig_tab->is_psd_in_progress = false;
+	dig_tab->cur_ig_value =
+		(u8)odm_get_bb_reg(dm, ODM_REG(IGI_A, dm), ODM_BIT(IGI, dm));
+	dig_tab->pre_ig_value = 0;
+	dig_tab->rssi_low_thresh = DM_DIG_THRESH_LOW;
+	dig_tab->rssi_high_thresh = DM_DIG_THRESH_HIGH;
+	dig_tab->fa_low_thresh = DM_FALSEALARM_THRESH_LOW;
+	dig_tab->fa_high_thresh = DM_FALSEALARM_THRESH_HIGH;
+	dig_tab->backoff_val = DM_DIG_BACKOFF_DEFAULT;
+	dig_tab->backoff_val_range_max = DM_DIG_BACKOFF_MAX;
+	dig_tab->backoff_val_range_min = DM_DIG_BACKOFF_MIN;
+	dig_tab->pre_cck_cca_thres = 0xFF;
+	dig_tab->cur_cck_cca_thres = 0x83;
+	dig_tab->forbidden_igi = DM_DIG_MIN_NIC;
+	dig_tab->large_fa_hit = 0;
+	dig_tab->large_fa_timeout = 0;
+	dig_tab->recover_cnt = 0;
+	dig_tab->is_media_connect_0 = false;
+	dig_tab->is_media_connect_1 = false;
+
+	/*To initialize dm->is_dm_initial_gain_enable==false to avoid DIG err*/
+	dm->is_dm_initial_gain_enable = true;
+
+	dig_tab->dig_dynamic_min_0 = DM_DIG_MIN_NIC;
+	dig_tab->dig_dynamic_min_1 = DM_DIG_MIN_NIC;
+
+	/* To Initi BT30 IGI */
+	dig_tab->bt30_cur_igi = 0x32;
+
+	odm_memory_set(dm, dig_tab->pause_dig_value, 0,
+		       (DM_DIG_MAX_PAUSE_TYPE + 1));
+	dig_tab->pause_dig_level = 0;
+	odm_memory_set(dm, dig_tab->pause_cckpd_value, 0,
+		       (DM_DIG_MAX_PAUSE_TYPE + 1));
+	dig_tab->pause_cckpd_level = 0;
+
+	if (dm->board_type & (ODM_BOARD_EXT_PA | ODM_BOARD_EXT_LNA)) {
+		dig_tab->rx_gain_range_max = DM_DIG_MAX_NIC;
+		dig_tab->rx_gain_range_min = DM_DIG_MIN_NIC;
+	} else {
+		dig_tab->rx_gain_range_max = DM_DIG_MAX_NIC;
+		dig_tab->rx_gain_range_min = DM_DIG_MIN_NIC;
+	}
+
+	dig_tab->enable_adjust_big_jump = 1;
+	if (dm->support_ic_type & ODM_RTL8822B) {
+		ret_value = odm_get_bb_reg(dm, 0x8c8, MASKLWORD);
+		dig_tab->big_jump_step1 = (u8)(ret_value & 0xe) >> 1;
+		dig_tab->big_jump_step2 = (u8)(ret_value & 0x30) >> 4;
+		dig_tab->big_jump_step3 = (u8)(ret_value & 0xc0) >> 6;
+
+	} else if (dm->support_ic_type & ODM_RTL8197F) {
+		ret_value =
+			odm_get_bb_reg(dm, ODM_REG_BB_AGC_SET_2_11N, MASKLWORD);
+		dig_tab->big_jump_step1 = (u8)(ret_value & 0xe) >> 1;
+		dig_tab->big_jump_step2 = (u8)(ret_value & 0x30) >> 4;
+		dig_tab->big_jump_step3 = (u8)(ret_value & 0xc0) >> 6;
+	}
+	if (dm->support_ic_type & (ODM_RTL8822B | ODM_RTL8197F)) {
+		for (i = 0; i < sizeof(dig_tab->big_jump_lmt); i++) {
+			if (dig_tab->big_jump_lmt[i] == 0)
+				dig_tab->big_jump_lmt[i] =
+					0x64; /* Set -10dBm as default value */
+		}
+	}
+}
+
+void odm_DIG(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	/* Common parameters */
+	struct dig_thres *dig_tab = &dm->dm_dig_table;
+	struct false_alarm_stat *fa_cnt =
+		(struct false_alarm_stat *)phydm_get_structure(
+			dm, PHYDM_FALSEALMCNT);
+	bool first_connect, first_dis_connect;
+	u8 dig_max_of_min, dig_dynamic_min;
+	u8 dm_dig_max, dm_dig_min;
+	u8 current_igi = dig_tab->cur_ig_value;
+	u8 offset;
+	u32 dm_FA_thres[3];
+	u32 tx_tp = 0, rx_tp = 0;
+	bool is_dfs_band = false;
+	bool is_performance = true, is_first_tp_target = false,
+	     is_first_coverage = false;
+
+	if (odm_dig_abort(dm))
+		return;
+
+	ODM_RT_TRACE(dm, ODM_COMP_DIG, "DIG Start===>\n");
+
+	/* 1 Update status */
+	{
+		dig_dynamic_min = dig_tab->dig_dynamic_min_0;
+		first_connect = (dm->is_linked) && !dig_tab->is_media_connect_0;
+		first_dis_connect =
+			(!dm->is_linked) && dig_tab->is_media_connect_0;
+	}
+
+	/* 1 Boundary Decision */
+	{
+		/* 2 For WIN\CE */
+		if (dm->support_ic_type >= ODM_RTL8188E)
+			dm_dig_max = 0x5A;
+		else
+			dm_dig_max = DM_DIG_MAX_NIC;
+
+		if (dm->support_ic_type != ODM_RTL8821)
+			dm_dig_min = DM_DIG_MIN_NIC;
+		else
+			dm_dig_min = 0x1C;
+
+		dig_max_of_min = DM_DIG_MAX_AP;
+
+		/* Modify lower bound for DFS band */
+		if ((((*dm->channel >= 52) && (*dm->channel <= 64)) ||
+		     ((*dm->channel >= 100) && (*dm->channel <= 140))) &&
+		    phydm_dfs_master_enabled(dm)) {
+			is_dfs_band = true;
+			if (*dm->band_width == ODM_BW20M)
+				dm_dig_min = DM_DIG_MIN_AP_DFS + 2;
+			else
+				dm_dig_min = DM_DIG_MIN_AP_DFS;
+			ODM_RT_TRACE(dm, ODM_COMP_DIG,
+				     "DIG: ====== In DFS band ======\n");
+		}
+	}
+	ODM_RT_TRACE(dm, ODM_COMP_DIG,
+		     "DIG: Absolutly upper bound = 0x%x, lower bound = 0x%x\n",
+		     dm_dig_max, dm_dig_min);
+
+	if (dm->pu1_forced_igi_lb && (*dm->pu1_forced_igi_lb > 0)) {
+		ODM_RT_TRACE(dm, ODM_COMP_DIG, "DIG: Force IGI lb to: 0x%02x\n",
+			     *dm->pu1_forced_igi_lb);
+		dm_dig_min = *dm->pu1_forced_igi_lb;
+		dm_dig_max = (dm_dig_min <= dm_dig_max) ? (dm_dig_max) :
+							  (dm_dig_min + 1);
+	}
+
+	/* 1 Adjust boundary by RSSI */
+	if (dm->is_linked && is_performance) {
+		/* 2 Modify DIG upper bound */
+		/* 4 Modify DIG upper bound for 92E, 8723A\B, 8821 & 8812 BT */
+		if ((dm->support_ic_type & (ODM_RTL8192E | ODM_RTL8723B |
+					    ODM_RTL8812 | ODM_RTL8821)) &&
+		    (dm->is_bt_limited_dig == 1)) {
+			offset = 10;
+			ODM_RT_TRACE(
+				dm, ODM_COMP_DIG,
+				"DIG: Coex. case: Force upper bound to RSSI + %d\n",
+				offset);
+		} else {
+			offset = 15;
+		}
+
+		if ((dm->rssi_min + offset) > dm_dig_max)
+			dig_tab->rx_gain_range_max = dm_dig_max;
+		else if ((dm->rssi_min + offset) < dm_dig_min)
+			dig_tab->rx_gain_range_max = dm_dig_min;
+		else
+			dig_tab->rx_gain_range_max = dm->rssi_min + offset;
+
+		/* 2 Modify DIG lower bound */
+		/* if(dm->is_one_entry_only) */
+		{
+			if (dm->rssi_min < dm_dig_min)
+				dig_dynamic_min = dm_dig_min;
+			else if (dm->rssi_min > dig_max_of_min)
+				dig_dynamic_min = dig_max_of_min;
+			else
+				dig_dynamic_min = dm->rssi_min;
+
+			if (is_dfs_band) {
+				dig_dynamic_min = dm_dig_min;
+				ODM_RT_TRACE(
+					dm, ODM_COMP_DIG,
+					"DIG: DFS band: Force lower bound to 0x%x after link\n",
+					dm_dig_min);
+			}
+		}
+	} else {
+		if (is_performance && is_dfs_band) {
+			dig_tab->rx_gain_range_max = 0x28;
+			ODM_RT_TRACE(
+				dm, ODM_COMP_DIG,
+				"DIG: DFS band: Force upper bound to 0x%x before link\n",
+				dig_tab->rx_gain_range_max);
+		} else {
+			if (is_performance)
+				dig_tab->rx_gain_range_max = DM_DIG_MAX_OF_MIN;
+			else
+				dig_tab->rx_gain_range_max = dm_dig_max;
+		}
+		dig_dynamic_min = dm_dig_min;
+	}
+
+	/* 1 Force Lower Bound for AntDiv */
+	if (dm->is_linked && !dm->is_one_entry_only &&
+	    (dm->support_ic_type & ODM_ANTDIV_SUPPORT) &&
+	    (dm->support_ability & ODM_BB_ANT_DIV)) {
+		if (dm->ant_div_type == CG_TRX_HW_ANTDIV ||
+		    dm->ant_div_type == CG_TRX_SMART_ANTDIV) {
+			if (dig_tab->ant_div_rssi_max > dig_max_of_min)
+				dig_dynamic_min = dig_max_of_min;
+			else
+				dig_dynamic_min = (u8)dig_tab->ant_div_rssi_max;
+			ODM_RT_TRACE(
+				dm, ODM_COMP_DIG,
+				"DIG: AntDiv case: Force lower bound to 0x%x\n",
+				dig_dynamic_min);
+			ODM_RT_TRACE(dm, ODM_COMP_DIG,
+				     "DIG: AntDiv case: rssi_max = 0x%x\n",
+				     dig_tab->ant_div_rssi_max);
+		}
+	}
+	ODM_RT_TRACE(
+		dm, ODM_COMP_DIG,
+		"DIG: Adjust boundary by RSSI Upper bound = 0x%x, Lower bound = 0x%x\n",
+		dig_tab->rx_gain_range_max, dig_dynamic_min);
+	ODM_RT_TRACE(
+		dm, ODM_COMP_DIG,
+		"DIG: Link status: is_linked = %d, RSSI = %d, bFirstConnect = %d, bFirsrDisConnect = %d\n",
+		dm->is_linked, dm->rssi_min, first_connect, first_dis_connect);
+
+	/* 1 Modify DIG lower bound, deal with abnormal case */
+	/* 2 Abnormal false alarm case */
+	if (is_dfs_band) {
+		dig_tab->rx_gain_range_min = dig_dynamic_min;
+	} else {
+		if (!dm->is_linked) {
+			dig_tab->rx_gain_range_min = dig_dynamic_min;
+
+			if (first_dis_connect)
+				dig_tab->forbidden_igi = dig_dynamic_min;
+		} else {
+			dig_tab->rx_gain_range_min = odm_forbidden_igi_check(
+				dm, dig_dynamic_min, current_igi);
+		}
+	}
+
+	/* 2 Abnormal # beacon case */
+	if (dm->is_linked && !first_connect) {
+		ODM_RT_TRACE(dm, ODM_COMP_DIG, "Beacon Num (%d)\n",
+			     dm->phy_dbg_info.num_qry_beacon_pkt);
+		if ((dm->phy_dbg_info.num_qry_beacon_pkt < 5) &&
+		    (dm->bsta_state)) {
+			dig_tab->rx_gain_range_min = 0x1c;
+			ODM_RT_TRACE(
+				dm, ODM_COMP_DIG,
+				"DIG: Abnrormal #beacon (%d) case in STA mode: Force lower bound to 0x%x\n",
+				dm->phy_dbg_info.num_qry_beacon_pkt,
+				dig_tab->rx_gain_range_min);
+		}
+	}
+
+	/* 2 Abnormal lower bound case */
+	if (dig_tab->rx_gain_range_min > dig_tab->rx_gain_range_max) {
+		dig_tab->rx_gain_range_min = dig_tab->rx_gain_range_max;
+		ODM_RT_TRACE(
+			dm, ODM_COMP_DIG,
+			"DIG: Abnrormal lower bound case: Force lower bound to 0x%x\n",
+			dig_tab->rx_gain_range_min);
+	}
+
+	/* 1 False alarm threshold decision */
+	odm_fa_threshold_check(dm, is_dfs_band, is_performance, rx_tp, tx_tp,
+			       dm_FA_thres);
+	ODM_RT_TRACE(dm, ODM_COMP_DIG,
+		     "DIG: False alarm threshold = %d, %d, %d\n",
+		     dm_FA_thres[0], dm_FA_thres[1], dm_FA_thres[2]);
+
+	/* 1 Adjust initial gain by false alarm */
+	if (dm->is_linked && is_performance) {
+		/* 2 After link */
+		ODM_RT_TRACE(dm, ODM_COMP_DIG, "DIG: Adjust IGI after link\n");
+
+		if (is_first_tp_target || (first_connect && is_performance)) {
+			dig_tab->large_fa_hit = 0;
+
+			if (is_dfs_band) {
+				u8 rssi = dm->rssi_min;
+
+				current_igi =
+					(dm->rssi_min > 0x28) ? 0x28 : rssi;
+				ODM_RT_TRACE(
+					dm, ODM_COMP_DIG,
+					"DIG: DFS band: One-shot to 0x28 upmost\n");
+			} else {
+				current_igi = phydm_get_current_igi(
+					dig_max_of_min, dm->rssi_min,
+					current_igi);
+			}
+
+			ODM_RT_TRACE(
+				dm, ODM_COMP_DIG,
+				"DIG: First connect case: IGI does on-shot to 0x%x\n",
+				current_igi);
+
+		} else {
+			if (fa_cnt->cnt_all > dm_FA_thres[2])
+				current_igi = current_igi + 4;
+			else if (fa_cnt->cnt_all > dm_FA_thres[1])
+				current_igi = current_igi + 2;
+			else if (fa_cnt->cnt_all < dm_FA_thres[0])
+				current_igi = current_igi - 2;
+
+			/* 4 Abnormal # beacon case */
+			if ((dm->phy_dbg_info.num_qry_beacon_pkt < 5) &&
+			    (fa_cnt->cnt_all < DM_DIG_FA_TH1) &&
+			    (dm->bsta_state)) {
+				current_igi = dig_tab->rx_gain_range_min;
+				ODM_RT_TRACE(
+					dm, ODM_COMP_DIG,
+					"DIG: Abnormal #beacon (%d) case: IGI does one-shot to 0x%x\n",
+					dm->phy_dbg_info.num_qry_beacon_pkt,
+					current_igi);
+			}
+		}
+	} else {
+		/* 2 Before link */
+		ODM_RT_TRACE(dm, ODM_COMP_DIG, "DIG: Adjust IGI before link\n");
+
+		if (first_dis_connect || is_first_coverage) {
+			current_igi = dm_dig_min;
+			ODM_RT_TRACE(
+				dm, ODM_COMP_DIG,
+				"DIG: First disconnect case: IGI does on-shot to lower bound\n");
+		} else {
+			if (fa_cnt->cnt_all > dm_FA_thres[2])
+				current_igi = current_igi + 4;
+			else if (fa_cnt->cnt_all > dm_FA_thres[1])
+				current_igi = current_igi + 2;
+			else if (fa_cnt->cnt_all < dm_FA_thres[0])
+				current_igi = current_igi - 2;
+		}
+	}
+
+	/* 1 Check initial gain by upper/lower bound */
+	if (current_igi < dig_tab->rx_gain_range_min)
+		current_igi = dig_tab->rx_gain_range_min;
+
+	if (current_igi > dig_tab->rx_gain_range_max)
+		current_igi = dig_tab->rx_gain_range_max;
+
+	ODM_RT_TRACE(dm, ODM_COMP_DIG, "DIG: cur_ig_value=0x%x, TotalFA = %d\n",
+		     current_igi, fa_cnt->cnt_all);
+
+	/* 1 Update status */
+	if (dm->is_bt_hs_operation) {
+		if (dm->is_linked) {
+			if (dig_tab->bt30_cur_igi > (current_igi))
+				odm_write_dig(dm, current_igi);
+			else
+				odm_write_dig(dm, dig_tab->bt30_cur_igi);
+
+			dig_tab->is_media_connect_0 = dm->is_linked;
+			dig_tab->dig_dynamic_min_0 = dig_dynamic_min;
+		} else {
+			if (dm->is_link_in_process)
+				odm_write_dig(dm, 0x1c);
+			else if (dm->is_bt_connect_process)
+				odm_write_dig(dm, 0x28);
+			else
+				odm_write_dig(dm, dig_tab->bt30_cur_igi);
+		}
+	} else { /* BT is not using */
+		odm_write_dig(dm, current_igi);
+		dig_tab->is_media_connect_0 = dm->is_linked;
+		dig_tab->dig_dynamic_min_0 = dig_dynamic_min;
+	}
+	ODM_RT_TRACE(dm, ODM_COMP_DIG, "DIG end\n");
+}
+
+void odm_dig_by_rssi_lps(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct false_alarm_stat *fa_cnt =
+		(struct false_alarm_stat *)phydm_get_structure(
+			dm, PHYDM_FALSEALMCNT);
+
+	u8 rssi_lower = DM_DIG_MIN_NIC; /* 0x1E or 0x1C */
+	u8 current_igi = dm->rssi_min;
+
+	if (odm_dig_abort(dm))
+		return;
+
+	current_igi = current_igi + RSSI_OFFSET_DIG;
+
+	ODM_RT_TRACE(dm, ODM_COMP_DIG, "%s()==>\n", __func__);
+
+	/* Using FW PS mode to make IGI */
+	/* Adjust by  FA in LPS MODE */
+	if (fa_cnt->cnt_all > DM_DIG_FA_TH2_LPS)
+		current_igi = current_igi + 4;
+	else if (fa_cnt->cnt_all > DM_DIG_FA_TH1_LPS)
+		current_igi = current_igi + 2;
+	else if (fa_cnt->cnt_all < DM_DIG_FA_TH0_LPS)
+		current_igi = current_igi - 2;
+
+	/* Lower bound checking */
+
+	/* RSSI Lower bound check */
+	if ((dm->rssi_min - 10) > DM_DIG_MIN_NIC)
+		rssi_lower = (dm->rssi_min - 10);
+	else
+		rssi_lower = DM_DIG_MIN_NIC;
+
+	/* Upper and Lower Bound checking */
+	if (current_igi > DM_DIG_MAX_NIC)
+		current_igi = DM_DIG_MAX_NIC;
+	else if (current_igi < rssi_lower)
+		current_igi = rssi_lower;
+
+	ODM_RT_TRACE(dm, ODM_COMP_DIG, "%s(): fa_cnt->cnt_all = %d\n", __func__,
+		     fa_cnt->cnt_all);
+	ODM_RT_TRACE(dm, ODM_COMP_DIG, "%s(): dm->rssi_min = %d\n", __func__,
+		     dm->rssi_min);
+	ODM_RT_TRACE(dm, ODM_COMP_DIG, "%s(): current_igi = 0x%x\n", __func__,
+		     current_igi);
+
+	odm_write_dig(
+		dm,
+		current_igi); /* odm_write_dig(dm, dig_tab->cur_ig_value); */
+}
+
+/* 3============================================================
+ * 3 FASLE ALARM CHECK
+ * 3============================================================
+ */
+
+void odm_false_alarm_counter_statistics(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct false_alarm_stat *false_alm_cnt =
+		(struct false_alarm_stat *)phydm_get_structure(
+			dm, PHYDM_FALSEALMCNT);
+	struct rt_adcsmp *adc_smp = &dm->adcsmp;
+	u32 ret_value;
+
+	if (!(dm->support_ability & ODM_BB_FA_CNT))
+		return;
+
+	ODM_RT_TRACE(dm, ODM_COMP_FA_CNT, "%s()======>\n", __func__);
+
+	if (dm->support_ic_type & ODM_IC_11N_SERIES) {
+		/* hold ofdm counter */
+		odm_set_bb_reg(dm, ODM_REG_OFDM_FA_HOLDC_11N, BIT(31),
+			       1); /* hold page C counter */
+		odm_set_bb_reg(dm, ODM_REG_OFDM_FA_RSTD_11N, BIT(31),
+			       1); /* hold page D counter */
+
+		ret_value = odm_get_bb_reg(dm, ODM_REG_OFDM_FA_TYPE1_11N,
+					   MASKDWORD);
+		false_alm_cnt->cnt_fast_fsync = (ret_value & 0xffff);
+		false_alm_cnt->cnt_sb_search_fail =
+			((ret_value & 0xffff0000) >> 16);
+
+		ret_value = odm_get_bb_reg(dm, ODM_REG_OFDM_FA_TYPE2_11N,
+					   MASKDWORD);
+		false_alm_cnt->cnt_ofdm_cca = (ret_value & 0xffff);
+		false_alm_cnt->cnt_parity_fail =
+			((ret_value & 0xffff0000) >> 16);
+
+		ret_value = odm_get_bb_reg(dm, ODM_REG_OFDM_FA_TYPE3_11N,
+					   MASKDWORD);
+		false_alm_cnt->cnt_rate_illegal = (ret_value & 0xffff);
+		false_alm_cnt->cnt_crc8_fail = ((ret_value & 0xffff0000) >> 16);
+
+		ret_value = odm_get_bb_reg(dm, ODM_REG_OFDM_FA_TYPE4_11N,
+					   MASKDWORD);
+		false_alm_cnt->cnt_mcs_fail = (ret_value & 0xffff);
+
+		false_alm_cnt->cnt_ofdm_fail =
+			false_alm_cnt->cnt_parity_fail +
+			false_alm_cnt->cnt_rate_illegal +
+			false_alm_cnt->cnt_crc8_fail +
+			false_alm_cnt->cnt_mcs_fail +
+			false_alm_cnt->cnt_fast_fsync +
+			false_alm_cnt->cnt_sb_search_fail;
+
+		/* read CCK CRC32 counter */
+		false_alm_cnt->cnt_cck_crc32_error = odm_get_bb_reg(
+			dm, ODM_REG_CCK_CRC32_ERROR_CNT_11N, MASKDWORD);
+		false_alm_cnt->cnt_cck_crc32_ok = odm_get_bb_reg(
+			dm, ODM_REG_CCK_CRC32_OK_CNT_11N, MASKDWORD);
+
+		/* read OFDM CRC32 counter */
+		ret_value = odm_get_bb_reg(dm, ODM_REG_OFDM_CRC32_CNT_11N,
+					   MASKDWORD);
+		false_alm_cnt->cnt_ofdm_crc32_error =
+			(ret_value & 0xffff0000) >> 16;
+		false_alm_cnt->cnt_ofdm_crc32_ok = ret_value & 0xffff;
+
+		/* read HT CRC32 counter */
+		ret_value =
+			odm_get_bb_reg(dm, ODM_REG_HT_CRC32_CNT_11N, MASKDWORD);
+		false_alm_cnt->cnt_ht_crc32_error =
+			(ret_value & 0xffff0000) >> 16;
+		false_alm_cnt->cnt_ht_crc32_ok = ret_value & 0xffff;
+
+		/* read VHT CRC32 counter */
+		false_alm_cnt->cnt_vht_crc32_error = 0;
+		false_alm_cnt->cnt_vht_crc32_ok = 0;
+
+		{
+			/* hold cck counter */
+			odm_set_bb_reg(dm, ODM_REG_CCK_FA_RST_11N, BIT(12), 1);
+			odm_set_bb_reg(dm, ODM_REG_CCK_FA_RST_11N, BIT(14), 1);
+
+			ret_value = odm_get_bb_reg(dm, ODM_REG_CCK_FA_LSB_11N,
+						   MASKBYTE0);
+			false_alm_cnt->cnt_cck_fail = ret_value;
+
+			ret_value = odm_get_bb_reg(dm, ODM_REG_CCK_FA_MSB_11N,
+						   MASKBYTE3);
+			false_alm_cnt->cnt_cck_fail += (ret_value & 0xff) << 8;
+
+			ret_value = odm_get_bb_reg(dm, ODM_REG_CCK_CCA_CNT_11N,
+						   MASKDWORD);
+			false_alm_cnt->cnt_cck_cca =
+				((ret_value & 0xFF) << 8) |
+				((ret_value & 0xFF00) >> 8);
+		}
+
+		false_alm_cnt->cnt_all_pre = false_alm_cnt->cnt_all;
+
+		false_alm_cnt->cnt_all = (false_alm_cnt->cnt_fast_fsync +
+					  false_alm_cnt->cnt_sb_search_fail +
+					  false_alm_cnt->cnt_parity_fail +
+					  false_alm_cnt->cnt_rate_illegal +
+					  false_alm_cnt->cnt_crc8_fail +
+					  false_alm_cnt->cnt_mcs_fail +
+					  false_alm_cnt->cnt_cck_fail);
+
+		false_alm_cnt->cnt_cca_all = false_alm_cnt->cnt_ofdm_cca +
+					     false_alm_cnt->cnt_cck_cca;
+
+		if (dm->support_ic_type >= ODM_RTL8188E) {
+			/*reset false alarm counter registers*/
+			odm_set_bb_reg(dm, ODM_REG_OFDM_FA_RSTC_11N, BIT(31),
+				       1);
+			odm_set_bb_reg(dm, ODM_REG_OFDM_FA_RSTC_11N, BIT(31),
+				       0);
+			odm_set_bb_reg(dm, ODM_REG_OFDM_FA_RSTD_11N, BIT(27),
+				       1);
+			odm_set_bb_reg(dm, ODM_REG_OFDM_FA_RSTD_11N, BIT(27),
+				       0);
+
+			/*update ofdm counter*/
+			odm_set_bb_reg(dm, ODM_REG_OFDM_FA_HOLDC_11N, BIT(31),
+				       0); /*update page C counter*/
+			odm_set_bb_reg(dm, ODM_REG_OFDM_FA_RSTD_11N, BIT(31),
+				       0); /*update page D counter*/
+
+			/*reset CCK CCA counter*/
+			odm_set_bb_reg(dm, ODM_REG_CCK_FA_RST_11N,
+				       BIT(13) | BIT(12), 0);
+			odm_set_bb_reg(dm, ODM_REG_CCK_FA_RST_11N,
+				       BIT(13) | BIT(12), 2);
+
+			/*reset CCK FA counter*/
+			odm_set_bb_reg(dm, ODM_REG_CCK_FA_RST_11N,
+				       BIT(15) | BIT(14), 0);
+			odm_set_bb_reg(dm, ODM_REG_CCK_FA_RST_11N,
+				       BIT(15) | BIT(14), 2);
+
+			/*reset CRC32 counter*/
+			odm_set_bb_reg(dm, ODM_REG_PAGE_F_RST_11N, BIT(16), 1);
+			odm_set_bb_reg(dm, ODM_REG_PAGE_F_RST_11N, BIT(16), 0);
+		}
+
+		/* Get debug port 0 */
+		odm_set_bb_reg(dm, ODM_REG_DBG_RPT_11N, MASKDWORD, 0x0);
+		false_alm_cnt->dbg_port0 =
+			odm_get_bb_reg(dm, ODM_REG_RPT_11N, MASKDWORD);
+
+		/* Get EDCCA flag */
+		odm_set_bb_reg(dm, ODM_REG_DBG_RPT_11N, MASKDWORD, 0x208);
+		false_alm_cnt->edcca_flag =
+			(bool)odm_get_bb_reg(dm, ODM_REG_RPT_11N, BIT(30));
+
+		ODM_RT_TRACE(
+			dm, ODM_COMP_FA_CNT,
+			"[OFDM FA Detail] Parity_Fail = (( %d )), Rate_Illegal = (( %d )), CRC8_fail = (( %d )), Mcs_fail = (( %d )), Fast_Fsync = (( %d )), SB_Search_fail = (( %d ))\n",
+			false_alm_cnt->cnt_parity_fail,
+			false_alm_cnt->cnt_rate_illegal,
+			false_alm_cnt->cnt_crc8_fail,
+			false_alm_cnt->cnt_mcs_fail,
+			false_alm_cnt->cnt_fast_fsync,
+			false_alm_cnt->cnt_sb_search_fail);
+	}
+
+	if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+		u32 cck_enable;
+
+		/* read OFDM FA counter */
+		false_alm_cnt->cnt_ofdm_fail =
+			odm_get_bb_reg(dm, ODM_REG_OFDM_FA_11AC, MASKLWORD);
+
+		/* Read CCK FA counter */
+		false_alm_cnt->cnt_cck_fail =
+			odm_get_bb_reg(dm, ODM_REG_CCK_FA_11AC, MASKLWORD);
+
+		/* read CCK/OFDM CCA counter */
+		ret_value =
+			odm_get_bb_reg(dm, ODM_REG_CCK_CCA_CNT_11AC, MASKDWORD);
+		false_alm_cnt->cnt_ofdm_cca = (ret_value & 0xffff0000) >> 16;
+		false_alm_cnt->cnt_cck_cca = ret_value & 0xffff;
+
+		/* read CCK CRC32 counter */
+		ret_value = odm_get_bb_reg(dm, ODM_REG_CCK_CRC32_CNT_11AC,
+					   MASKDWORD);
+		false_alm_cnt->cnt_cck_crc32_error =
+			(ret_value & 0xffff0000) >> 16;
+		false_alm_cnt->cnt_cck_crc32_ok = ret_value & 0xffff;
+
+		/* read OFDM CRC32 counter */
+		ret_value = odm_get_bb_reg(dm, ODM_REG_OFDM_CRC32_CNT_11AC,
+					   MASKDWORD);
+		false_alm_cnt->cnt_ofdm_crc32_error =
+			(ret_value & 0xffff0000) >> 16;
+		false_alm_cnt->cnt_ofdm_crc32_ok = ret_value & 0xffff;
+
+		/* read HT CRC32 counter */
+		ret_value = odm_get_bb_reg(dm, ODM_REG_HT_CRC32_CNT_11AC,
+					   MASKDWORD);
+		false_alm_cnt->cnt_ht_crc32_error =
+			(ret_value & 0xffff0000) >> 16;
+		false_alm_cnt->cnt_ht_crc32_ok = ret_value & 0xffff;
+
+		/* read VHT CRC32 counter */
+		ret_value = odm_get_bb_reg(dm, ODM_REG_VHT_CRC32_CNT_11AC,
+					   MASKDWORD);
+		false_alm_cnt->cnt_vht_crc32_error =
+			(ret_value & 0xffff0000) >> 16;
+		false_alm_cnt->cnt_vht_crc32_ok = ret_value & 0xffff;
+
+		/* reset OFDM FA counter */
+		odm_set_bb_reg(dm, ODM_REG_OFDM_FA_RST_11AC, BIT(17), 1);
+		odm_set_bb_reg(dm, ODM_REG_OFDM_FA_RST_11AC, BIT(17), 0);
+
+		/* reset CCK FA counter */
+		odm_set_bb_reg(dm, ODM_REG_CCK_FA_RST_11AC, BIT(15), 0);
+		odm_set_bb_reg(dm, ODM_REG_CCK_FA_RST_11AC, BIT(15), 1);
+
+		/* reset CCA counter */
+		odm_set_bb_reg(dm, ODM_REG_RST_RPT_11AC, BIT(0), 1);
+		odm_set_bb_reg(dm, ODM_REG_RST_RPT_11AC, BIT(0), 0);
+
+		cck_enable =
+			odm_get_bb_reg(dm, ODM_REG_BB_RX_PATH_11AC, BIT(28));
+		if (cck_enable) { /* if(*dm->band_type == ODM_BAND_2_4G) */
+			false_alm_cnt->cnt_all = false_alm_cnt->cnt_ofdm_fail +
+						 false_alm_cnt->cnt_cck_fail;
+			false_alm_cnt->cnt_cca_all =
+				false_alm_cnt->cnt_cck_cca +
+				false_alm_cnt->cnt_ofdm_cca;
+		} else {
+			false_alm_cnt->cnt_all = false_alm_cnt->cnt_ofdm_fail;
+			false_alm_cnt->cnt_cca_all =
+				false_alm_cnt->cnt_ofdm_cca;
+		}
+
+		if (adc_smp->adc_smp_state == ADCSMP_STATE_IDLE) {
+			if (phydm_set_bb_dbg_port(
+				    dm, BB_DBGPORT_PRIORITY_1,
+				    0x0)) { /*set debug port to 0x0*/
+				false_alm_cnt->dbg_port0 =
+					phydm_get_bb_dbg_port_value(dm);
+				phydm_release_bb_dbg_port(dm);
+			}
+
+			if (phydm_set_bb_dbg_port(
+				    dm, BB_DBGPORT_PRIORITY_1,
+				    0x209)) { /*set debug port to 0x0*/
+				false_alm_cnt->edcca_flag =
+					(bool)((phydm_get_bb_dbg_port_value(
+							dm) &
+						BIT(30)) >>
+					       30);
+				phydm_release_bb_dbg_port(dm);
+			}
+		}
+	}
+
+	false_alm_cnt->cnt_crc32_error_all =
+		false_alm_cnt->cnt_vht_crc32_error +
+		false_alm_cnt->cnt_ht_crc32_error +
+		false_alm_cnt->cnt_ofdm_crc32_error +
+		false_alm_cnt->cnt_cck_crc32_error;
+	false_alm_cnt->cnt_crc32_ok_all = false_alm_cnt->cnt_vht_crc32_ok +
+					  false_alm_cnt->cnt_ht_crc32_ok +
+					  false_alm_cnt->cnt_ofdm_crc32_ok +
+					  false_alm_cnt->cnt_cck_crc32_ok;
+
+	ODM_RT_TRACE(dm, ODM_COMP_FA_CNT,
+		     "[CCA Cnt] {CCK, OFDM, Total} = {%d, %d, %d}\n",
+		     false_alm_cnt->cnt_cck_cca, false_alm_cnt->cnt_ofdm_cca,
+		     false_alm_cnt->cnt_cca_all);
+
+	ODM_RT_TRACE(dm, ODM_COMP_FA_CNT,
+		     "[FA Cnt] {CCK, OFDM, Total} = {%d, %d, %d}\n",
+		     false_alm_cnt->cnt_cck_fail, false_alm_cnt->cnt_ofdm_fail,
+		     false_alm_cnt->cnt_all);
+
+	ODM_RT_TRACE(dm, ODM_COMP_FA_CNT,
+		     "[CCK]  CRC32 {error, ok}= {%d, %d}\n",
+		     false_alm_cnt->cnt_cck_crc32_error,
+		     false_alm_cnt->cnt_cck_crc32_ok);
+	ODM_RT_TRACE(dm, ODM_COMP_FA_CNT, "[OFDM]CRC32 {error, ok}= {%d, %d}\n",
+		     false_alm_cnt->cnt_ofdm_crc32_error,
+		     false_alm_cnt->cnt_ofdm_crc32_ok);
+	ODM_RT_TRACE(dm, ODM_COMP_FA_CNT,
+		     "[ HT ]  CRC32 {error, ok}= {%d, %d}\n",
+		     false_alm_cnt->cnt_ht_crc32_error,
+		     false_alm_cnt->cnt_ht_crc32_ok);
+	ODM_RT_TRACE(dm, ODM_COMP_FA_CNT,
+		     "[VHT]  CRC32 {error, ok}= {%d, %d}\n",
+		     false_alm_cnt->cnt_vht_crc32_error,
+		     false_alm_cnt->cnt_vht_crc32_ok);
+	ODM_RT_TRACE(dm, ODM_COMP_FA_CNT,
+		     "[VHT]  CRC32 {error, ok}= {%d, %d}\n",
+		     false_alm_cnt->cnt_crc32_error_all,
+		     false_alm_cnt->cnt_crc32_ok_all);
+	ODM_RT_TRACE(dm, ODM_COMP_FA_CNT,
+		     "FA_Cnt: Dbg port 0x0 = 0x%x, EDCCA = %d\n\n",
+		     false_alm_cnt->dbg_port0, false_alm_cnt->edcca_flag);
+}
+
+/* 3============================================================
+ * 3 CCK Packet Detect threshold
+ * 3============================================================
+ */
+
+void odm_pause_cck_packet_detection(void *dm_void,
+				    enum phydm_pause_type pause_type,
+				    enum phydm_pause_level pause_level,
+				    u8 cck_pd_threshold)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct dig_thres *dig_tab = &dm->dm_dig_table;
+	s8 max_level;
+
+	ODM_RT_TRACE(dm, ODM_COMP_DIG, "%s()=========> level = %d\n", __func__,
+		     pause_level);
+
+	if ((dig_tab->pause_cckpd_level == 0) &&
+	    (!(dm->support_ability & ODM_BB_CCK_PD) ||
+	     !(dm->support_ability & ODM_BB_FA_CNT))) {
+		ODM_RT_TRACE(
+			dm, ODM_COMP_DIG,
+			"Return: support_ability ODM_BB_CCK_PD or ODM_BB_FA_CNT is disabled\n");
+		return;
+	}
+
+	if (pause_level > DM_DIG_MAX_PAUSE_TYPE) {
+		ODM_RT_TRACE(dm, ODM_COMP_DIG,
+			     "%s(): Return: Wrong pause level !!\n", __func__);
+		return;
+	}
+
+	ODM_RT_TRACE(dm, ODM_COMP_DIG,
+		     "%s(): pause level = 0x%x, Current value = 0x%x\n",
+		     __func__, dig_tab->pause_cckpd_level, cck_pd_threshold);
+	ODM_RT_TRACE(
+		dm, ODM_COMP_DIG,
+		"%s(): pause value = 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n",
+		__func__, dig_tab->pause_cckpd_value[7],
+		dig_tab->pause_cckpd_value[6], dig_tab->pause_cckpd_value[5],
+		dig_tab->pause_cckpd_value[4], dig_tab->pause_cckpd_value[3],
+		dig_tab->pause_cckpd_value[2], dig_tab->pause_cckpd_value[1],
+		dig_tab->pause_cckpd_value[0]);
+
+	switch (pause_type) {
+	/* Pause CCK Packet Detection threshold */
+	case PHYDM_PAUSE: {
+		/* Disable CCK PD */
+		odm_cmn_info_update(dm, ODM_CMNINFO_ABILITY,
+				    dm->support_ability & (~ODM_BB_CCK_PD));
+		ODM_RT_TRACE(dm, ODM_COMP_DIG,
+			     "%s(): Pause CCK packet detection threshold !!\n",
+			     __func__);
+
+		/*Backup original CCK PD threshold decided by CCK PD mechanism*/
+		if (dig_tab->pause_cckpd_level == 0) {
+			dig_tab->cck_pd_backup = dig_tab->cur_cck_cca_thres;
+			ODM_RT_TRACE(
+				dm, ODM_COMP_DIG,
+				"%s(): Backup CCKPD  = 0x%x, new CCKPD = 0x%x\n",
+				__func__, dig_tab->cck_pd_backup,
+				cck_pd_threshold);
+		}
+
+		/* Update pause level */
+		dig_tab->pause_cckpd_level =
+			(dig_tab->pause_cckpd_level | BIT(pause_level));
+
+		/* Record CCK PD threshold */
+		dig_tab->pause_cckpd_value[pause_level] = cck_pd_threshold;
+
+		/* Write new CCK PD threshold */
+		if (BIT(pause_level + 1) > dig_tab->pause_cckpd_level) {
+			odm_write_cck_cca_thres(dm, cck_pd_threshold);
+			ODM_RT_TRACE(dm, ODM_COMP_DIG,
+				     "%s(): CCKPD of higher level = 0x%x\n",
+				     __func__, cck_pd_threshold);
+		}
+		break;
+	}
+	/* Resume CCK Packet Detection threshold */
+	case PHYDM_RESUME: {
+		/* check if the level is illegal or not */
+		if ((dig_tab->pause_cckpd_level & (BIT(pause_level))) != 0) {
+			dig_tab->pause_cckpd_level =
+				dig_tab->pause_cckpd_level &
+				(~(BIT(pause_level)));
+			dig_tab->pause_cckpd_value[pause_level] = 0;
+			ODM_RT_TRACE(dm, ODM_COMP_DIG,
+				     "%s(): Resume CCK PD !!\n", __func__);
+		} else {
+			ODM_RT_TRACE(dm, ODM_COMP_DIG,
+				     "%s(): Wrong resume level !!\n", __func__);
+			break;
+		}
+
+		/* Resume DIG */
+		if (dig_tab->pause_cckpd_level == 0) {
+			/* Write backup IGI value */
+			odm_write_cck_cca_thres(dm, dig_tab->cck_pd_backup);
+			/* dig_tab->is_ignore_dig = true; */
+			ODM_RT_TRACE(dm, ODM_COMP_DIG,
+				     "%s(): Write original CCKPD = 0x%x\n",
+				     __func__, dig_tab->cck_pd_backup);
+
+			/* Enable DIG */
+			odm_cmn_info_update(dm, ODM_CMNINFO_ABILITY,
+					    dm->support_ability |
+						    ODM_BB_CCK_PD);
+			break;
+		}
+
+		if (BIT(pause_level) <= dig_tab->pause_cckpd_level)
+			break;
+
+		/* Calculate the maximum level now */
+		for (max_level = (pause_level - 1); max_level >= 0;
+		     max_level--) {
+			if ((dig_tab->pause_cckpd_level & BIT(max_level)) > 0)
+				break;
+		}
+
+		/* write CCKPD of lower level */
+		odm_write_cck_cca_thres(dm,
+					dig_tab->pause_cckpd_value[max_level]);
+		ODM_RT_TRACE(dm, ODM_COMP_DIG,
+			     "%s(): Write CCKPD (0x%x) of level (%d)\n",
+			     __func__, dig_tab->pause_cckpd_value[max_level],
+			     max_level);
+		break;
+	}
+	default:
+		ODM_RT_TRACE(dm, ODM_COMP_DIG, "%s(): Wrong  type !!\n",
+			     __func__);
+		break;
+	}
+
+	ODM_RT_TRACE(dm, ODM_COMP_DIG,
+		     "%s(): pause level = 0x%x, Current value = 0x%x\n",
+		     __func__, dig_tab->pause_cckpd_level, cck_pd_threshold);
+	ODM_RT_TRACE(
+		dm, ODM_COMP_DIG,
+		"%s(): pause value = 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n",
+		__func__, dig_tab->pause_cckpd_value[7],
+		dig_tab->pause_cckpd_value[6], dig_tab->pause_cckpd_value[5],
+		dig_tab->pause_cckpd_value[4], dig_tab->pause_cckpd_value[3],
+		dig_tab->pause_cckpd_value[2], dig_tab->pause_cckpd_value[1],
+		dig_tab->pause_cckpd_value[0]);
+}
+
+void odm_cck_packet_detection_thresh(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct dig_thres *dig_tab = &dm->dm_dig_table;
+	struct false_alarm_stat *false_alm_cnt =
+		(struct false_alarm_stat *)phydm_get_structure(
+			dm, PHYDM_FALSEALMCNT);
+	u8 cur_cck_cca_thres = dig_tab->cur_cck_cca_thres, rssi_thd = 35;
+
+	if ((!(dm->support_ability & ODM_BB_CCK_PD)) ||
+	    (!(dm->support_ability & ODM_BB_FA_CNT))) {
+		ODM_RT_TRACE(dm, ODM_COMP_DIG, "CCK_PD: return==========\n");
+		return;
+	}
+
+	if (dm->ext_lna)
+		return;
+
+	ODM_RT_TRACE(dm, ODM_COMP_DIG, "CCK_PD: ==========>\n");
+
+	if (dig_tab->cck_fa_ma == 0xffffffff)
+		dig_tab->cck_fa_ma = false_alm_cnt->cnt_cck_fail;
+	else
+		dig_tab->cck_fa_ma =
+			((dig_tab->cck_fa_ma << 1) + dig_tab->cck_fa_ma +
+			 false_alm_cnt->cnt_cck_fail) >>
+			2;
+
+	ODM_RT_TRACE(dm, ODM_COMP_DIG, "CCK_PD: CCK FA moving average = %d\n",
+		     dig_tab->cck_fa_ma);
+
+	if (dm->is_linked) {
+		if (dm->rssi_min > rssi_thd) {
+			cur_cck_cca_thres = 0xcd;
+		} else if (dm->rssi_min > 20) {
+			if (dig_tab->cck_fa_ma >
+			    ((DM_DIG_FA_TH1 >> 1) + (DM_DIG_FA_TH1 >> 3)))
+				cur_cck_cca_thres = 0xcd;
+			else if (dig_tab->cck_fa_ma < (DM_DIG_FA_TH0 >> 1))
+				cur_cck_cca_thres = 0x83;
+		} else if (dm->rssi_min > 7) {
+			cur_cck_cca_thres = 0x83;
+		} else {
+			cur_cck_cca_thres = 0x40;
+		}
+
+	} else {
+		if (dig_tab->cck_fa_ma > 0x400)
+			cur_cck_cca_thres = 0x83;
+		else if (dig_tab->cck_fa_ma < 0x200)
+			cur_cck_cca_thres = 0x40;
+	}
+
+	{
+		odm_write_cck_cca_thres(dm, cur_cck_cca_thres);
+	}
+
+	ODM_RT_TRACE(dm, ODM_COMP_DIG, "CCK_PD: cck_cca_th=((0x%x))\n\n",
+		     cur_cck_cca_thres);
+}
+
+void odm_write_cck_cca_thres(void *dm_void, u8 cur_cck_cca_thres)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct dig_thres *dig_tab = &dm->dm_dig_table;
+
+	if (dig_tab->cur_cck_cca_thres !=
+	    cur_cck_cca_thres) { /* modify by Guo.Mingzhi 2012-01-03 */
+		odm_write_1byte(dm, ODM_REG(CCK_CCA, dm), cur_cck_cca_thres);
+		dig_tab->cck_fa_ma = 0xffffffff;
+	}
+	dig_tab->pre_cck_cca_thres = dig_tab->cur_cck_cca_thres;
+	dig_tab->cur_cck_cca_thres = cur_cck_cca_thres;
+}
+
+bool phydm_dig_go_up_check(void *dm_void)
+{
+	bool ret = true;
+
+	return ret;
+}
diff --git a/drivers/staging/rtlwifi/phydm/phydm_dig.h b/drivers/staging/rtlwifi/phydm/phydm_dig.h
new file mode 100644
index 000000000000..af70aaec3b19
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_dig.h
@@ -0,0 +1,241 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __PHYDMDIG_H__
+#define __PHYDMDIG_H__
+
+#define DIG_VERSION "1.32" /* 2016.09.02  YuChen. add CCK PD for 8197F*/
+
+/* Pause DIG & CCKPD */
+#define DM_DIG_MAX_PAUSE_TYPE 0x7
+
+enum dig_goupcheck_level {
+	DIG_GOUPCHECK_LEVEL_0,
+	DIG_GOUPCHECK_LEVEL_1,
+	DIG_GOUPCHECK_LEVEL_2
+
+};
+
+struct dig_thres {
+	bool is_stop_dig; /* for debug */
+	bool is_ignore_dig;
+	bool is_psd_in_progress;
+
+	u8 dig_enable_flag;
+	u8 dig_ext_port_stage;
+
+	int rssi_low_thresh;
+	int rssi_high_thresh;
+
+	u32 fa_low_thresh;
+	u32 fa_high_thresh;
+
+	u8 cur_sta_connect_state;
+	u8 pre_sta_connect_state;
+	u8 cur_multi_sta_connect_state;
+
+	u8 pre_ig_value;
+	u8 cur_ig_value;
+	u8 backup_ig_value; /* MP DIG */
+	u8 bt30_cur_igi;
+	u8 igi_backup;
+
+	s8 backoff_val;
+	s8 backoff_val_range_max;
+	s8 backoff_val_range_min;
+	u8 rx_gain_range_max;
+	u8 rx_gain_range_min;
+	u8 rssi_val_min;
+
+	u8 pre_cck_cca_thres;
+	u8 cur_cck_cca_thres;
+	u8 pre_cck_pd_state;
+	u8 cur_cck_pd_state;
+	u8 cck_pd_backup;
+	u8 pause_cckpd_level;
+	u8 pause_cckpd_value[DM_DIG_MAX_PAUSE_TYPE + 1];
+
+	u8 large_fa_hit;
+	u8 large_fa_timeout; /*if (large_fa_hit), monitor "large_fa_timeout"
+			      *sec, if timeout, large_fa_hit=0
+			      */
+	u8 forbidden_igi;
+	u32 recover_cnt;
+
+	u8 dig_dynamic_min_0;
+	u8 dig_dynamic_min_1;
+	bool is_media_connect_0;
+	bool is_media_connect_1;
+
+	u32 ant_div_rssi_max;
+	u32 rssi_max;
+
+	u8 *is_p2p_in_process;
+
+	u8 pause_dig_level;
+	u8 pause_dig_value[DM_DIG_MAX_PAUSE_TYPE + 1];
+
+	u32 cck_fa_ma;
+	enum dig_goupcheck_level dig_go_up_check_level;
+	u8 aaa_default;
+
+	u8 rf_gain_idx;
+	u8 agc_table_idx;
+	u8 big_jump_lmt[16];
+	u8 enable_adjust_big_jump : 1;
+	u8 big_jump_step1 : 3;
+	u8 big_jump_step2 : 2;
+	u8 big_jump_step3 : 2;
+};
+
+struct false_alarm_stat {
+	u32 cnt_parity_fail;
+	u32 cnt_rate_illegal;
+	u32 cnt_crc8_fail;
+	u32 cnt_mcs_fail;
+	u32 cnt_ofdm_fail;
+	u32 cnt_ofdm_fail_pre; /* For RTL8881A */
+	u32 cnt_cck_fail;
+	u32 cnt_all;
+	u32 cnt_all_pre;
+	u32 cnt_fast_fsync;
+	u32 cnt_sb_search_fail;
+	u32 cnt_ofdm_cca;
+	u32 cnt_cck_cca;
+	u32 cnt_cca_all;
+	u32 cnt_bw_usc; /* Gary */
+	u32 cnt_bw_lsc; /* Gary */
+	u32 cnt_cck_crc32_error;
+	u32 cnt_cck_crc32_ok;
+	u32 cnt_ofdm_crc32_error;
+	u32 cnt_ofdm_crc32_ok;
+	u32 cnt_ht_crc32_error;
+	u32 cnt_ht_crc32_ok;
+	u32 cnt_vht_crc32_error;
+	u32 cnt_vht_crc32_ok;
+	u32 cnt_crc32_error_all;
+	u32 cnt_crc32_ok_all;
+	bool cck_block_enable;
+	bool ofdm_block_enable;
+	u32 dbg_port0;
+	bool edcca_flag;
+};
+
+enum dm_dig_op {
+	DIG_TYPE_THRESH_HIGH = 0,
+	DIG_TYPE_THRESH_LOW = 1,
+	DIG_TYPE_BACKOFF = 2,
+	DIG_TYPE_RX_GAIN_MIN = 3,
+	DIG_TYPE_RX_GAIN_MAX = 4,
+	DIG_TYPE_ENABLE = 5,
+	DIG_TYPE_DISABLE = 6,
+	DIG_OP_TYPE_MAX
+};
+
+enum phydm_pause_type { PHYDM_PAUSE = BIT(0), PHYDM_RESUME = BIT(1) };
+
+enum phydm_pause_level {
+	/* number of pause level can't exceed DM_DIG_MAX_PAUSE_TYPE */
+	PHYDM_PAUSE_LEVEL_0 = 0,
+	PHYDM_PAUSE_LEVEL_1 = 1,
+	PHYDM_PAUSE_LEVEL_2 = 2,
+	PHYDM_PAUSE_LEVEL_3 = 3,
+	PHYDM_PAUSE_LEVEL_4 = 4,
+	PHYDM_PAUSE_LEVEL_5 = 5,
+	PHYDM_PAUSE_LEVEL_6 = 6,
+	PHYDM_PAUSE_LEVEL_7 = DM_DIG_MAX_PAUSE_TYPE /* maximum level */
+};
+
+#define DM_DIG_THRESH_HIGH 40
+#define DM_DIG_THRESH_LOW 35
+
+#define DM_FALSEALARM_THRESH_LOW 400
+#define DM_FALSEALARM_THRESH_HIGH 1000
+
+#define DM_DIG_MAX_NIC 0x3e
+#define DM_DIG_MIN_NIC 0x20
+#define DM_DIG_MAX_OF_MIN_NIC 0x3e
+
+#define DM_DIG_MAX_AP 0x3e
+#define DM_DIG_MIN_AP 0x20
+#define DM_DIG_MAX_OF_MIN 0x2A /* 0x32 */
+#define DM_DIG_MIN_AP_DFS 0x20
+
+#define DM_DIG_MAX_NIC_HP 0x46
+#define DM_DIG_MIN_NIC_HP 0x2e
+
+#define DM_DIG_MAX_AP_HP 0x42
+#define DM_DIG_MIN_AP_HP 0x30
+
+/* vivi 92c&92d has different definition, 20110504
+ * this is for 92c
+ */
+#define DM_DIG_FA_TH0 0x200 /* 0x20 */
+
+#define DM_DIG_FA_TH1 0x300
+#define DM_DIG_FA_TH2 0x400
+/* this is for 92d */
+#define DM_DIG_FA_TH0_92D 0x100
+#define DM_DIG_FA_TH1_92D 0x400
+#define DM_DIG_FA_TH2_92D 0x600
+
+#define DM_DIG_BACKOFF_MAX 12
+#define DM_DIG_BACKOFF_MIN -4
+#define DM_DIG_BACKOFF_DEFAULT 10
+
+#define DM_DIG_FA_TH0_LPS 4 /* -> 4 in lps */
+#define DM_DIG_FA_TH1_LPS 15 /* -> 15 lps */
+#define DM_DIG_FA_TH2_LPS 30 /* -> 30 lps */
+#define RSSI_OFFSET_DIG 0x05
+#define LARGE_FA_TIMEOUT 60
+
+void odm_change_dynamic_init_gain_thresh(void *dm_void, u32 dm_type,
+					 u32 dm_value);
+
+void odm_write_dig(void *dm_void, u8 current_igi);
+
+void odm_pause_dig(void *dm_void, enum phydm_pause_type pause_type,
+		   enum phydm_pause_level pause_level, u8 igi_value);
+
+void odm_dig_init(void *dm_void);
+
+void odm_DIG(void *dm_void);
+
+void odm_dig_by_rssi_lps(void *dm_void);
+
+void odm_false_alarm_counter_statistics(void *dm_void);
+
+void odm_pause_cck_packet_detection(void *dm_void,
+				    enum phydm_pause_type pause_type,
+				    enum phydm_pause_level pause_level,
+				    u8 cck_pd_threshold);
+
+void odm_cck_packet_detection_thresh(void *dm_void);
+
+void odm_write_cck_cca_thres(void *dm_void, u8 cur_cck_cca_thres);
+
+bool phydm_dig_go_up_check(void *dm_void);
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/phydm_dynamic_rx_path.h b/drivers/staging/rtlwifi/phydm/phydm_dynamic_rx_path.h
new file mode 100644
index 000000000000..9f3cb2468c02
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_dynamic_rx_path.h
@@ -0,0 +1,37 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __PHYDMDYMICRXPATH_H__
+#define __PHYDMDYMICRXPATH_H__
+
+#define DYNAMIC_RX_PATH_VERSION "1.0" /*2016.07.15  Dino */
+
+#define DRP_RSSI_TH 35
+
+#define INIT_DRP_TIMMER 0
+#define CANCEL_DRP_TIMMER 1
+#define RELEASE_DRP_TIMMER 2
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/phydm_dynamicbbpowersaving.c b/drivers/staging/rtlwifi/phydm/phydm_dynamicbbpowersaving.c
new file mode 100644
index 000000000000..7661c499aeb1
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_dynamicbbpowersaving.c
@@ -0,0 +1,129 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+/* ************************************************************
+ * include files
+ * *************************************************************/
+#include "mp_precomp.h"
+#include "phydm_precomp.h"
+
+static inline void phydm_update_rf_state(struct phy_dm_struct *dm,
+					 struct dyn_pwr_saving *dm_ps_table,
+					 int _rssi_up_bound,
+					 int _rssi_low_bound,
+					 int _is_force_in_normal)
+{
+	if (_is_force_in_normal) {
+		dm_ps_table->cur_rf_state = rf_normal;
+		return;
+	}
+
+	if (dm->rssi_min == 0xFF) {
+		dm_ps_table->cur_rf_state = RF_MAX;
+		return;
+	}
+
+	if (dm_ps_table->pre_rf_state == rf_normal) {
+		if (dm->rssi_min >= _rssi_up_bound)
+			dm_ps_table->cur_rf_state = rf_save;
+		else
+			dm_ps_table->cur_rf_state = rf_normal;
+	} else {
+		if (dm->rssi_min <= _rssi_low_bound)
+			dm_ps_table->cur_rf_state = rf_normal;
+		else
+			dm_ps_table->cur_rf_state = rf_save;
+	}
+}
+
+void odm_dynamic_bb_power_saving_init(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct dyn_pwr_saving *dm_ps_table = &dm->dm_ps_table;
+
+	dm_ps_table->pre_cca_state = CCA_MAX;
+	dm_ps_table->cur_cca_state = CCA_MAX;
+	dm_ps_table->pre_rf_state = RF_MAX;
+	dm_ps_table->cur_rf_state = RF_MAX;
+	dm_ps_table->rssi_val_min = 0;
+	dm_ps_table->initialize = 0;
+}
+
+void odm_rf_saving(void *dm_void, u8 is_force_in_normal)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct dyn_pwr_saving *dm_ps_table = &dm->dm_ps_table;
+	u8 rssi_up_bound = 30;
+	u8 rssi_low_bound = 25;
+
+	if (dm->patch_id == 40) { /* RT_CID_819x_FUNAI_TV */
+		rssi_up_bound = 50;
+		rssi_low_bound = 45;
+	}
+	if (dm_ps_table->initialize == 0) {
+		dm_ps_table->reg874 =
+			(odm_get_bb_reg(dm, 0x874, MASKDWORD) & 0x1CC000) >> 14;
+		dm_ps_table->regc70 =
+			(odm_get_bb_reg(dm, 0xc70, MASKDWORD) & BIT(3)) >> 3;
+		dm_ps_table->reg85c =
+			(odm_get_bb_reg(dm, 0x85c, MASKDWORD) & 0xFF000000) >>
+			24;
+		dm_ps_table->rega74 =
+			(odm_get_bb_reg(dm, 0xa74, MASKDWORD) & 0xF000) >> 12;
+		/* Reg818 = phy_query_bb_reg(adapter, 0x818, MASKDWORD); */
+		dm_ps_table->initialize = 1;
+	}
+
+	phydm_update_rf_state(dm, dm_ps_table, rssi_up_bound, rssi_low_bound,
+			      is_force_in_normal);
+
+	if (dm_ps_table->pre_rf_state != dm_ps_table->cur_rf_state) {
+		if (dm_ps_table->cur_rf_state == rf_save) {
+			odm_set_bb_reg(dm, 0x874, 0x1C0000,
+				       0x2); /* reg874[20:18]=3'b010 */
+			odm_set_bb_reg(dm, 0xc70, BIT(3),
+				       0); /* regc70[3]=1'b0 */
+			odm_set_bb_reg(dm, 0x85c, 0xFF000000,
+				       0x63); /* reg85c[31:24]=0x63 */
+			odm_set_bb_reg(dm, 0x874, 0xC000,
+				       0x2); /* reg874[15:14]=2'b10 */
+			odm_set_bb_reg(dm, 0xa74, 0xF000,
+				       0x3); /* RegA75[7:4]=0x3 */
+			odm_set_bb_reg(dm, 0x818, BIT(28),
+				       0x0); /* Reg818[28]=1'b0 */
+			odm_set_bb_reg(dm, 0x818, BIT(28),
+				       0x1); /* Reg818[28]=1'b1 */
+		} else {
+			odm_set_bb_reg(dm, 0x874, 0x1CC000,
+				       dm_ps_table->reg874);
+			odm_set_bb_reg(dm, 0xc70, BIT(3), dm_ps_table->regc70);
+			odm_set_bb_reg(dm, 0x85c, 0xFF000000,
+				       dm_ps_table->reg85c);
+			odm_set_bb_reg(dm, 0xa74, 0xF000, dm_ps_table->rega74);
+			odm_set_bb_reg(dm, 0x818, BIT(28), 0x0);
+		}
+		dm_ps_table->pre_rf_state = dm_ps_table->cur_rf_state;
+	}
+}
diff --git a/drivers/staging/rtlwifi/phydm/phydm_dynamicbbpowersaving.h b/drivers/staging/rtlwifi/phydm/phydm_dynamicbbpowersaving.h
new file mode 100644
index 000000000000..e7394c475395
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_dynamicbbpowersaving.h
@@ -0,0 +1,50 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __PHYDMDYNAMICBBPOWERSAVING_H__
+#define __PHYDMDYNAMICBBPOWERSAVING_H__
+
+#define DYNAMIC_BBPWRSAV_VERSION "1.1"
+
+struct dyn_pwr_saving {
+	u8 pre_cca_state;
+	u8 cur_cca_state;
+
+	u8 pre_rf_state;
+	u8 cur_rf_state;
+
+	int rssi_val_min;
+
+	u8 initialize;
+	u32 reg874, regc70, reg85c, rega74;
+};
+
+#define dm_rf_saving odm_rf_saving
+
+void odm_rf_saving(void *dm_void, u8 is_force_in_normal);
+
+void odm_dynamic_bb_power_saving_init(void *dm_void);
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/phydm_dynamictxpower.c b/drivers/staging/rtlwifi/phydm/phydm_dynamictxpower.c
new file mode 100644
index 000000000000..ebb43342b80b
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_dynamictxpower.c
@@ -0,0 +1,102 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+/* ************************************************************
+ * include files
+ * *************************************************************/
+#include "mp_precomp.h"
+#include "phydm_precomp.h"
+
+void odm_dynamic_tx_power_init(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	dm->last_dtp_lvl = tx_high_pwr_level_normal;
+	dm->dynamic_tx_high_power_lvl = tx_high_pwr_level_normal;
+	dm->tx_agc_ofdm_18_6 =
+		odm_get_bb_reg(dm, 0xC24, MASKDWORD); /*TXAGC {18M 12M 9M 6M}*/
+}
+
+void odm_dynamic_tx_power_save_power_index(void *dm_void) {}
+
+void odm_dynamic_tx_power_restore_power_index(void *dm_void) {}
+
+void odm_dynamic_tx_power_write_power_index(void *dm_void, u8 value)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u8 index;
+	u32 power_index_reg[6] = {0xc90, 0xc91, 0xc92, 0xc98, 0xc99, 0xc9a};
+
+	for (index = 0; index < 6; index++)
+		odm_write_1byte(dm, power_index_reg[index], value);
+}
+
+static void odm_dynamic_tx_power_nic_ce(void *dm_void) {}
+
+void odm_dynamic_tx_power(void *dm_void)
+{
+	/*  */
+	/* For AP/ADSL use struct rtl8192cd_priv* */
+	/* For CE/NIC use struct void* */
+	/*  */
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	if (!(dm->support_ability & ODM_BB_DYNAMIC_TXPWR))
+		return;
+	/* 2011/09/29 MH In HW integration first stage, we provide 4 different
+	 * handle to operate at the same time.
+	 * In the stage2/3, we need to prive universal interface and merge all
+	 * HW dynamic mechanism.
+	 */
+	switch (dm->support_platform) {
+	case ODM_WIN:
+		odm_dynamic_tx_power_nic(dm);
+		break;
+	case ODM_CE:
+		odm_dynamic_tx_power_nic_ce(dm);
+		break;
+	case ODM_AP:
+		odm_dynamic_tx_power_ap(dm);
+		break;
+	default:
+		break;
+	}
+}
+
+void odm_dynamic_tx_power_nic(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	if (!(dm->support_ability & ODM_BB_DYNAMIC_TXPWR))
+		return;
+}
+
+void odm_dynamic_tx_power_ap(void *dm_void
+
+			     )
+{
+}
+
+void odm_dynamic_tx_power_8821(void *dm_void, u8 *desc, u8 mac_id) {}
diff --git a/drivers/staging/rtlwifi/phydm/phydm_dynamictxpower.h b/drivers/staging/rtlwifi/phydm/phydm_dynamictxpower.h
new file mode 100644
index 000000000000..10bad1209db2
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_dynamictxpower.h
@@ -0,0 +1,64 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __PHYDMDYNAMICTXPOWER_H__
+#define __PHYDMDYNAMICTXPOWER_H__
+
+/*#define DYNAMIC_TXPWR_VERSION	"1.0"*/
+/*#define DYNAMIC_TXPWR_VERSION	"1.3" */ /*2015.08.26, Add 8814 Dynamic TX pwr*/
+#define DYNAMIC_TXPWR_VERSION "1.4" /*2015.11.06,Add CE 8821A Dynamic TX pwr*/
+
+#define TX_POWER_NEAR_FIELD_THRESH_LVL2 74
+#define TX_POWER_NEAR_FIELD_THRESH_LVL1 60
+
+#define tx_high_pwr_level_normal 0
+#define tx_high_pwr_level_level1 1
+#define tx_high_pwr_level_level2 2
+
+#define tx_high_pwr_level_bt1 3
+#define tx_high_pwr_level_bt2 4
+#define tx_high_pwr_level_15 5
+#define tx_high_pwr_level_35 6
+#define tx_high_pwr_level_50 7
+#define tx_high_pwr_level_70 8
+#define tx_high_pwr_level_100 9
+
+void odm_dynamic_tx_power_init(void *dm_void);
+
+void odm_dynamic_tx_power_restore_power_index(void *dm_void);
+
+void odm_dynamic_tx_power_nic(void *dm_void);
+
+void odm_dynamic_tx_power_save_power_index(void *dm_void);
+
+void odm_dynamic_tx_power_write_power_index(void *dm_void, u8 value);
+
+void odm_dynamic_tx_power_8821(void *dm_void, u8 *desc, u8 mac_id);
+
+void odm_dynamic_tx_power(void *dm_void);
+
+void odm_dynamic_tx_power_ap(void *dm_void);
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/phydm_edcaturbocheck.c b/drivers/staging/rtlwifi/phydm/phydm_edcaturbocheck.c
new file mode 100644
index 000000000000..753a9b9834e4
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_edcaturbocheck.c
@@ -0,0 +1,139 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+/* ************************************************************
+ * include files
+ * *************************************************************/
+#include "mp_precomp.h"
+#include "phydm_precomp.h"
+
+void odm_edca_turbo_init(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	dm->dm_edca_table.is_current_turbo_edca = false;
+	dm->dm_edca_table.is_cur_rdl_state = false;
+
+	ODM_RT_TRACE(dm, ODM_COMP_EDCA_TURBO, "Orginial VO PARAM: 0x%x\n",
+		     odm_read_4byte(dm, ODM_EDCA_VO_PARAM));
+	ODM_RT_TRACE(dm, ODM_COMP_EDCA_TURBO, "Orginial VI PARAM: 0x%x\n",
+		     odm_read_4byte(dm, ODM_EDCA_VI_PARAM));
+	ODM_RT_TRACE(dm, ODM_COMP_EDCA_TURBO, "Orginial BE PARAM: 0x%x\n",
+		     odm_read_4byte(dm, ODM_EDCA_BE_PARAM));
+	ODM_RT_TRACE(dm, ODM_COMP_EDCA_TURBO, "Orginial BK PARAM: 0x%x\n",
+		     odm_read_4byte(dm, ODM_EDCA_BK_PARAM));
+
+} /* ODM_InitEdcaTurbo */
+
+void odm_edca_turbo_check(void *dm_void)
+{
+	/* For AP/ADSL use struct rtl8192cd_priv* */
+	/* For CE/NIC use struct void* */
+
+	/* 2011/09/29 MH In HW integration first stage, we provide 4 different
+	 * handle to operate at the same time.
+	 * In the stage2/3, we need to prive universal interface and merge all
+	 * HW dynamic mechanism.
+	 */
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	ODM_RT_TRACE(dm, ODM_COMP_EDCA_TURBO,
+		     "%s========================>\n", __func__);
+
+	if (!(dm->support_ability & ODM_MAC_EDCA_TURBO))
+		return;
+
+	switch (dm->support_platform) {
+	case ODM_WIN:
+
+		break;
+
+	case ODM_CE:
+		odm_edca_turbo_check_ce(dm);
+		break;
+	}
+	ODM_RT_TRACE(dm, ODM_COMP_EDCA_TURBO,
+		     "<========================%s\n", __func__);
+
+} /* odm_CheckEdcaTurbo */
+
+void odm_edca_turbo_check_ce(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)dm->adapter;
+	u64 cur_txok_cnt = 0;
+	u64 cur_rxok_cnt = 0;
+	u32 edca_be_ul = 0x5ea42b;
+	u32 edca_be_dl = 0x5ea42b;
+	u32 edca_be = 0x5ea42b;
+	bool is_cur_rdlstate;
+	bool edca_turbo_on = false;
+
+	if (dm->wifi_test)
+		return;
+
+	if (!dm->is_linked) {
+		rtlpriv->dm.is_any_nonbepkts = false;
+		return;
+	}
+
+	if (rtlpriv->dm.dbginfo.num_non_be_pkt > 0x100)
+		rtlpriv->dm.is_any_nonbepkts = true;
+	rtlpriv->dm.dbginfo.num_non_be_pkt = 0;
+
+	cur_txok_cnt = rtlpriv->stats.txbytesunicast_inperiod;
+	cur_rxok_cnt = rtlpriv->stats.rxbytesunicast_inperiod;
+
+	/*b_bias_on_rx = false;*/
+	edca_turbo_on = ((!rtlpriv->dm.is_any_nonbepkts) &&
+			 (!rtlpriv->dm.disable_framebursting)) ?
+				true :
+				false;
+
+	if (rtlpriv->mac80211.mode == WIRELESS_MODE_B)
+		goto label_exit;
+
+	if (edca_turbo_on) {
+		is_cur_rdlstate =
+			(cur_rxok_cnt > cur_txok_cnt * 4) ? true : false;
+
+		edca_be = is_cur_rdlstate ? edca_be_dl : edca_be_ul;
+		rtl_write_dword(rtlpriv, REG_EDCA_BE_PARAM_8822B, edca_be);
+		rtlpriv->dm.is_cur_rdlstate = is_cur_rdlstate;
+		rtlpriv->dm.current_turbo_edca = true;
+	} else {
+		if (rtlpriv->dm.current_turbo_edca) {
+			u8 tmp = AC0_BE;
+
+			rtlpriv->cfg->ops->set_hw_reg(rtlpriv->hw,
+						      HW_VAR_AC_PARAM,
+						      (u8 *)(&tmp));
+			rtlpriv->dm.current_turbo_edca = false;
+		}
+	}
+
+label_exit:
+	rtlpriv->dm.is_any_nonbepkts = false;
+}
diff --git a/drivers/staging/rtlwifi/phydm/phydm_edcaturbocheck.h b/drivers/staging/rtlwifi/phydm/phydm_edcaturbocheck.h
new file mode 100644
index 000000000000..5845b108a001
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_edcaturbocheck.h
@@ -0,0 +1,44 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __PHYDMEDCATURBOCHECK_H__
+#define __PHYDMEDCATURBOCHECK_H__
+
+/*#define EDCATURBO_VERSION	"2.1"*/
+#define EDCATURBO_VERSION "2.3" /*2015.07.29 by YuChen*/
+
+struct edca_turbo {
+	bool is_current_turbo_edca;
+	bool is_cur_rdl_state;
+
+	u32 prv_traffic_idx; /* edca turbo */
+};
+
+void odm_edca_turbo_check(void *dm_void);
+void odm_edca_turbo_init(void *dm_void);
+
+void odm_edca_turbo_check_ce(void *dm_void);
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/phydm_features.h b/drivers/staging/rtlwifi/phydm/phydm_features.h
new file mode 100644
index 000000000000..37f6f0cd7235
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_features.h
@@ -0,0 +1,33 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __PHYDM_FEATURES_H__
+#define __PHYDM_FEATURES
+
+/*phydm debyg report & tools*/
+
+/*Antenna Diversity*/
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/phydm_hwconfig.c b/drivers/staging/rtlwifi/phydm/phydm_hwconfig.c
new file mode 100644
index 000000000000..0a1f11a926e4
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_hwconfig.c
@@ -0,0 +1,1928 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+/* ************************************************************
+ * include files
+ * *************************************************************/
+
+#include "mp_precomp.h"
+#include "phydm_precomp.h"
+
+#define READ_AND_CONFIG_MP(ic, txt) (odm_read_and_config_mp_##ic##txt(dm))
+#define READ_AND_CONFIG_TC(ic, txt) (odm_read_and_config_tc_##ic##txt(dm))
+
+#define READ_AND_CONFIG READ_AND_CONFIG_MP
+
+#define READ_FIRMWARE_MP(ic, txt)                                              \
+	(odm_read_firmware_mp_##ic##txt(dm, p_firmware, size))
+#define READ_FIRMWARE_TC(ic, txt)                                              \
+	(odm_read_firmware_tc_##ic##txt(dm, p_firmware, size))
+
+#define READ_FIRMWARE READ_FIRMWARE_MP
+
+#define GET_VERSION_MP(ic, txt) (odm_get_version_mp_##ic##txt())
+#define GET_VERSION_TC(ic, txt) (odm_get_version_tc_##ic##txt())
+
+#define GET_VERSION(ic, txt) GET_VERSION_MP(ic, txt)
+
+static u32 phydm_process_rssi_pwdb(struct phy_dm_struct *dm,
+				   struct rtl_sta_info *entry,
+				   struct dm_per_pkt_info *pktinfo,
+				   u32 undecorated_smoothed_ofdm,
+				   u32 undecorated_smoothed_cck)
+{
+	u32 weighting = 0, undecorated_smoothed_pwdb;
+	/* 2011.07.28 LukeLee: modified to prevent unstable CCK RSSI */
+
+	if (entry->rssi_stat.ofdm_pkt ==
+	    64) { /* speed up when all packets are OFDM*/
+		undecorated_smoothed_pwdb = undecorated_smoothed_ofdm;
+		ODM_RT_TRACE(dm, ODM_COMP_RSSI_MONITOR,
+			     "PWDB_0[%d] = (( %d ))\n", pktinfo->station_id,
+			     undecorated_smoothed_cck);
+	} else {
+		if (entry->rssi_stat.valid_bit < 64)
+			entry->rssi_stat.valid_bit++;
+
+		if (entry->rssi_stat.valid_bit == 64) {
+			weighting = ((entry->rssi_stat.ofdm_pkt) > 4) ?
+					    64 :
+					    (entry->rssi_stat.ofdm_pkt << 4);
+			undecorated_smoothed_pwdb =
+				(weighting * undecorated_smoothed_ofdm +
+				 (64 - weighting) * undecorated_smoothed_cck) >>
+				6;
+			ODM_RT_TRACE(dm, ODM_COMP_RSSI_MONITOR,
+				     "PWDB_1[%d] = (( %d )), W = (( %d ))\n",
+				     pktinfo->station_id,
+				     undecorated_smoothed_cck, weighting);
+		} else {
+			if (entry->rssi_stat.valid_bit != 0)
+				undecorated_smoothed_pwdb =
+					(entry->rssi_stat.ofdm_pkt *
+						 undecorated_smoothed_ofdm +
+					 (entry->rssi_stat.valid_bit -
+					  entry->rssi_stat.ofdm_pkt) *
+						 undecorated_smoothed_cck) /
+					entry->rssi_stat.valid_bit;
+			else
+				undecorated_smoothed_pwdb = 0;
+
+			ODM_RT_TRACE(
+				dm, ODM_COMP_RSSI_MONITOR,
+				"PWDB_2[%d] = (( %d )), ofdm_pkt = (( %d )), Valid_Bit = (( %d ))\n",
+				pktinfo->station_id, undecorated_smoothed_cck,
+				entry->rssi_stat.ofdm_pkt,
+				entry->rssi_stat.valid_bit);
+		}
+	}
+
+	return undecorated_smoothed_pwdb;
+}
+
+static u32 phydm_process_rssi_cck(struct phy_dm_struct *dm,
+				  struct dm_phy_status_info *phy_info,
+				  struct rtl_sta_info *entry,
+				  u32 undecorated_smoothed_cck)
+{
+	u32 rssi_ave;
+	u8 i;
+
+	rssi_ave = phy_info->rx_pwdb_all;
+	dm->rssi_a = (u8)phy_info->rx_pwdb_all;
+	dm->rssi_b = 0xFF;
+	dm->rssi_c = 0xFF;
+	dm->rssi_d = 0xFF;
+
+	if (entry->rssi_stat.cck_pkt <= 63)
+		entry->rssi_stat.cck_pkt++;
+
+	/* 1 Process CCK RSSI */
+	if (undecorated_smoothed_cck <= 0) { /* initialize */
+		undecorated_smoothed_cck = phy_info->rx_pwdb_all;
+		entry->rssi_stat.cck_sum_power =
+			(u16)phy_info->rx_pwdb_all; /*reset*/
+		entry->rssi_stat.cck_pkt = 1; /*reset*/
+		ODM_RT_TRACE(dm, ODM_COMP_RSSI_MONITOR, "CCK_INIT: (( %d ))\n",
+			     undecorated_smoothed_cck);
+	} else if (entry->rssi_stat.cck_pkt <= CCK_RSSI_INIT_COUNT) {
+		entry->rssi_stat.cck_sum_power =
+			entry->rssi_stat.cck_sum_power +
+			(u16)phy_info->rx_pwdb_all;
+		undecorated_smoothed_cck = entry->rssi_stat.cck_sum_power /
+					   entry->rssi_stat.cck_pkt;
+
+		ODM_RT_TRACE(
+			dm, ODM_COMP_RSSI_MONITOR,
+			"CCK_0: (( %d )), SumPow = (( %d )), cck_pkt = (( %d ))\n",
+			undecorated_smoothed_cck,
+			entry->rssi_stat.cck_sum_power,
+			entry->rssi_stat.cck_pkt);
+	} else {
+		if (phy_info->rx_pwdb_all > (u32)undecorated_smoothed_cck) {
+			undecorated_smoothed_cck =
+				(((undecorated_smoothed_cck) *
+				  (RX_SMOOTH_FACTOR - 1)) +
+				 (phy_info->rx_pwdb_all)) /
+				(RX_SMOOTH_FACTOR);
+			undecorated_smoothed_cck = undecorated_smoothed_cck + 1;
+			ODM_RT_TRACE(dm, ODM_COMP_RSSI_MONITOR,
+				     "CCK_1: (( %d ))\n",
+				     undecorated_smoothed_cck);
+		} else {
+			undecorated_smoothed_cck =
+				(((undecorated_smoothed_cck) *
+				  (RX_SMOOTH_FACTOR - 1)) +
+				 (phy_info->rx_pwdb_all)) /
+				(RX_SMOOTH_FACTOR);
+			ODM_RT_TRACE(dm, ODM_COMP_RSSI_MONITOR,
+				     "CCK_2: (( %d ))\n",
+				     undecorated_smoothed_cck);
+		}
+	}
+
+	i = 63;
+	entry->rssi_stat.ofdm_pkt -=
+		(u8)((entry->rssi_stat.packet_map >> i) & BIT(0));
+	entry->rssi_stat.packet_map = entry->rssi_stat.packet_map << 1;
+	return undecorated_smoothed_cck;
+}
+
+static u32 phydm_process_rssi_ofdm(struct phy_dm_struct *dm,
+				   struct dm_phy_status_info *phy_info,
+				   struct rtl_sta_info *entry,
+				   u32 undecorated_smoothed_ofdm)
+{
+	u32 rssi_ave;
+	u8 rssi_max, rssi_min, i;
+
+	if (dm->support_ic_type & (ODM_RTL8814A | ODM_RTL8822B)) {
+		u8 rx_count = 0;
+		u32 rssi_linear = 0;
+
+		if (dm->rx_ant_status & ODM_RF_A) {
+			dm->rssi_a = phy_info->rx_mimo_signal_strength
+					     [ODM_RF_PATH_A];
+			rx_count++;
+			rssi_linear += odm_convert_to_linear(
+				phy_info->rx_mimo_signal_strength
+					[ODM_RF_PATH_A]);
+		} else {
+			dm->rssi_a = 0;
+		}
+
+		if (dm->rx_ant_status & ODM_RF_B) {
+			dm->rssi_b = phy_info->rx_mimo_signal_strength
+					     [ODM_RF_PATH_B];
+			rx_count++;
+			rssi_linear += odm_convert_to_linear(
+				phy_info->rx_mimo_signal_strength
+					[ODM_RF_PATH_B]);
+		} else {
+			dm->rssi_b = 0;
+		}
+
+		if (dm->rx_ant_status & ODM_RF_C) {
+			dm->rssi_c = phy_info->rx_mimo_signal_strength
+					     [ODM_RF_PATH_C];
+			rx_count++;
+			rssi_linear += odm_convert_to_linear(
+				phy_info->rx_mimo_signal_strength
+					[ODM_RF_PATH_C]);
+		} else {
+			dm->rssi_c = 0;
+		}
+
+		if (dm->rx_ant_status & ODM_RF_D) {
+			dm->rssi_d = phy_info->rx_mimo_signal_strength
+					     [ODM_RF_PATH_D];
+			rx_count++;
+			rssi_linear += odm_convert_to_linear(
+				phy_info->rx_mimo_signal_strength
+					[ODM_RF_PATH_D]);
+		} else {
+			dm->rssi_d = 0;
+		}
+
+		/* Calculate average RSSI */
+		switch (rx_count) {
+		case 2:
+			rssi_linear = (rssi_linear >> 1);
+			break;
+		case 3:
+			/* rssi_linear/3 ~ rssi_linear*11/32 */
+			rssi_linear = ((rssi_linear) + (rssi_linear << 1) +
+				       (rssi_linear << 3)) >>
+				      5;
+			break;
+		case 4:
+			rssi_linear = (rssi_linear >> 2);
+			break;
+		}
+
+		rssi_ave = odm_convert_to_db(rssi_linear);
+	} else {
+		if (phy_info->rx_mimo_signal_strength[ODM_RF_PATH_B] == 0) {
+			rssi_ave = phy_info->rx_mimo_signal_strength
+					   [ODM_RF_PATH_A];
+			dm->rssi_a = phy_info->rx_mimo_signal_strength
+					     [ODM_RF_PATH_A];
+			dm->rssi_b = 0;
+		} else {
+			dm->rssi_a = phy_info->rx_mimo_signal_strength
+					     [ODM_RF_PATH_A];
+			dm->rssi_b = phy_info->rx_mimo_signal_strength
+					     [ODM_RF_PATH_B];
+
+			if (phy_info->rx_mimo_signal_strength[ODM_RF_PATH_A] >
+			    phy_info->rx_mimo_signal_strength[ODM_RF_PATH_B]) {
+				rssi_max = phy_info->rx_mimo_signal_strength
+						   [ODM_RF_PATH_A];
+				rssi_min = phy_info->rx_mimo_signal_strength
+						   [ODM_RF_PATH_B];
+			} else {
+				rssi_max = phy_info->rx_mimo_signal_strength
+						   [ODM_RF_PATH_B];
+				rssi_min = phy_info->rx_mimo_signal_strength
+						   [ODM_RF_PATH_A];
+			}
+			if ((rssi_max - rssi_min) < 3)
+				rssi_ave = rssi_max;
+			else if ((rssi_max - rssi_min) < 6)
+				rssi_ave = rssi_max - 1;
+			else if ((rssi_max - rssi_min) < 10)
+				rssi_ave = rssi_max - 2;
+			else
+				rssi_ave = rssi_max - 3;
+		}
+	}
+
+	/* 1 Process OFDM RSSI */
+	if (undecorated_smoothed_ofdm <= 0) { /* initialize */
+		undecorated_smoothed_ofdm = phy_info->rx_pwdb_all;
+		ODM_RT_TRACE(dm, ODM_COMP_RSSI_MONITOR, "OFDM_INIT: (( %d ))\n",
+			     undecorated_smoothed_ofdm);
+	} else {
+		if (phy_info->rx_pwdb_all > (u32)undecorated_smoothed_ofdm) {
+			undecorated_smoothed_ofdm =
+				(((undecorated_smoothed_ofdm) *
+				  (RX_SMOOTH_FACTOR - 1)) +
+				 (rssi_ave)) /
+				(RX_SMOOTH_FACTOR);
+			undecorated_smoothed_ofdm =
+				undecorated_smoothed_ofdm + 1;
+			ODM_RT_TRACE(dm, ODM_COMP_RSSI_MONITOR,
+				     "OFDM_1: (( %d ))\n",
+				     undecorated_smoothed_ofdm);
+		} else {
+			undecorated_smoothed_ofdm =
+				(((undecorated_smoothed_ofdm) *
+				  (RX_SMOOTH_FACTOR - 1)) +
+				 (rssi_ave)) /
+				(RX_SMOOTH_FACTOR);
+			ODM_RT_TRACE(dm, ODM_COMP_RSSI_MONITOR,
+				     "OFDM_2: (( %d ))\n",
+				     undecorated_smoothed_ofdm);
+		}
+	}
+
+	if (entry->rssi_stat.ofdm_pkt != 64) {
+		i = 63;
+		entry->rssi_stat.ofdm_pkt -=
+			(u8)(((entry->rssi_stat.packet_map >> i) & BIT(0)) - 1);
+	}
+
+	entry->rssi_stat.packet_map =
+		(entry->rssi_stat.packet_map << 1) | BIT(0);
+	return undecorated_smoothed_ofdm;
+}
+
+static u8 odm_evm_db_to_percentage(s8);
+static u8 odm_evm_dbm_jaguar_series(s8);
+
+static inline u32 phydm_get_rssi_average(struct phy_dm_struct *dm,
+					 struct dm_phy_status_info *phy_info)
+{
+	u8 rssi_max = 0, rssi_min = 0;
+
+	dm->rssi_a = phy_info->rx_mimo_signal_strength[ODM_RF_PATH_A];
+	dm->rssi_b = phy_info->rx_mimo_signal_strength[ODM_RF_PATH_B];
+
+	if (phy_info->rx_mimo_signal_strength[ODM_RF_PATH_A] >
+	    phy_info->rx_mimo_signal_strength[ODM_RF_PATH_B]) {
+		rssi_max = phy_info->rx_mimo_signal_strength[ODM_RF_PATH_A];
+		rssi_min = phy_info->rx_mimo_signal_strength[ODM_RF_PATH_B];
+	} else {
+		rssi_max = phy_info->rx_mimo_signal_strength[ODM_RF_PATH_B];
+		rssi_min = phy_info->rx_mimo_signal_strength[ODM_RF_PATH_A];
+	}
+	if ((rssi_max - rssi_min) < 3)
+		return rssi_max;
+	else if ((rssi_max - rssi_min) < 6)
+		return rssi_max - 1;
+	else if ((rssi_max - rssi_min) < 10)
+		return rssi_max - 2;
+	else
+		return rssi_max - 3;
+}
+
+static inline u8 phydm_get_evm_dbm(u8 i, u8 EVM,
+				   struct phy_status_rpt_8812 *phy_sta_rpt,
+				   struct dm_phy_status_info *phy_info)
+{
+	if (i < ODM_RF_PATH_C)
+		return odm_evm_dbm_jaguar_series(phy_sta_rpt->rxevm[i]);
+	else
+		return odm_evm_dbm_jaguar_series(phy_sta_rpt->rxevm_cd[i - 2]);
+	/*RT_DISP(FRX, RX_PHY_SQ, ("RXRATE=%x RXEVM=%x EVM=%s%d\n",*/
+	/*pktinfo->data_rate, phy_sta_rpt->rxevm[i], "%", EVM));*/
+}
+
+static inline u8 phydm_get_odm_evm(u8 i, struct dm_per_pkt_info *pktinfo,
+				   struct phy_status_rpt_8812 *phy_sta_rpt)
+{
+	u8 evm = 0;
+
+	if (pktinfo->data_rate >= ODM_RATE6M &&
+	    pktinfo->data_rate <= ODM_RATE54M) {
+		if (i == ODM_RF_PATH_A) {
+			evm = odm_evm_db_to_percentage(
+				(phy_sta_rpt->sigevm)); /*dbm*/
+			evm += 20;
+			if (evm > 100)
+				evm = 100;
+		}
+	} else {
+		if (i < ODM_RF_PATH_C) {
+			if (phy_sta_rpt->rxevm[i] == -128)
+				phy_sta_rpt->rxevm[i] = -25;
+			evm = odm_evm_db_to_percentage(
+				(phy_sta_rpt->rxevm[i])); /*dbm*/
+		} else {
+			if (phy_sta_rpt->rxevm_cd[i - 2] == -128)
+				phy_sta_rpt->rxevm_cd[i - 2] = -25;
+			evm = odm_evm_db_to_percentage(
+				(phy_sta_rpt->rxevm_cd[i - 2])); /*dbm*/
+		}
+	}
+
+	return evm;
+}
+
+static inline s8 phydm_get_rx_pwr(u8 LNA_idx, u8 VGA_idx, u8 cck_highpwr)
+{
+	switch (LNA_idx) {
+	case 7:
+		if (VGA_idx <= 27)
+			return -100 + 2 * (27 - VGA_idx); /*VGA_idx = 27~2*/
+		else
+			return -100;
+		break;
+	case 6:
+		return -48 + 2 * (2 - VGA_idx); /*VGA_idx = 2~0*/
+	case 5:
+		return -42 + 2 * (7 - VGA_idx); /*VGA_idx = 7~5*/
+	case 4:
+		return -36 + 2 * (7 - VGA_idx); /*VGA_idx = 7~4*/
+	case 3:
+		return -24 + 2 * (7 - VGA_idx); /*VGA_idx = 7~0*/
+	case 2:
+		if (cck_highpwr)
+			return -12 + 2 * (5 - VGA_idx); /*VGA_idx = 5~0*/
+		else
+			return -6 + 2 * (5 - VGA_idx);
+		break;
+	case 1:
+		return 8 - 2 * VGA_idx;
+	case 0:
+		return 14 - 2 * VGA_idx;
+	default:
+		break;
+	}
+	return 0;
+}
+
+static inline u8 phydm_adjust_pwdb(u8 cck_highpwr, u8 pwdb_all)
+{
+	if (!cck_highpwr) {
+		if (pwdb_all >= 80)
+			return ((pwdb_all - 80) << 1) + ((pwdb_all - 80) >> 1) +
+			       80;
+		else if ((pwdb_all <= 78) && (pwdb_all >= 20))
+			return pwdb_all + 3;
+		if (pwdb_all > 100)
+			return 100;
+	}
+	return pwdb_all;
+}
+
+static inline u8
+phydm_get_signal_quality_8812(struct dm_phy_status_info *phy_info,
+			      struct phy_dm_struct *dm,
+			      struct phy_status_rpt_8812 *phy_sta_rpt)
+{
+	u8 sq_rpt;
+
+	if (phy_info->rx_pwdb_all > 40 && !dm->is_in_hct_test)
+		return 100;
+
+	sq_rpt = phy_sta_rpt->pwdb_all;
+
+	if (sq_rpt > 64)
+		return 0;
+	else if (sq_rpt < 20)
+		return 100;
+	else
+		return ((64 - sq_rpt) * 100) / 44;
+}
+
+static inline u8
+phydm_get_signal_quality_8192(struct dm_phy_status_info *phy_info,
+			      struct phy_dm_struct *dm,
+			      struct phy_status_rpt_8192cd *phy_sta_rpt)
+{
+	u8 sq_rpt;
+
+	if (phy_info->rx_pwdb_all > 40 && !dm->is_in_hct_test)
+		return 100;
+
+	sq_rpt = phy_sta_rpt->cck_sig_qual_ofdm_pwdb_all;
+
+	if (sq_rpt > 64)
+		return 0;
+	else if (sq_rpt < 20)
+		return 100;
+	else
+		return ((64 - sq_rpt) * 100) / 44;
+}
+
+static u8 odm_query_rx_pwr_percentage(s8 ant_power)
+{
+	if ((ant_power <= -100) || (ant_power >= 20))
+		return 0;
+	else if (ant_power >= 0)
+		return 100;
+	else
+		return 100 + ant_power;
+}
+
+/*
+ * 2012/01/12 MH MOve some signal strength smooth method to MP HAL layer.
+ * IF other SW team do not support the feature, remove this section.??
+ */
+
+s32 odm_signal_scale_mapping(struct phy_dm_struct *dm, s32 curr_sig)
+{
+	{
+		return curr_sig;
+	}
+}
+
+static u8 odm_sq_process_patch_rt_cid_819x_lenovo(struct phy_dm_struct *dm,
+						  u8 is_cck_rate, u8 pwdb_all,
+						  u8 path, u8 RSSI)
+{
+	u8 sq = 0;
+	return sq;
+}
+
+static u8 odm_evm_db_to_percentage(s8 value)
+{
+	/* -33dB~0dB to 0%~99% */
+	s8 ret_val;
+
+	ret_val = value;
+	ret_val /= 2;
+
+	if (ret_val >= 0)
+		ret_val = 0;
+
+	if (ret_val <= -33)
+		ret_val = -33;
+
+	ret_val = 0 - ret_val;
+	ret_val *= 3;
+
+	if (ret_val == 99)
+		ret_val = 100;
+
+	return (u8)ret_val;
+}
+
+static u8 odm_evm_dbm_jaguar_series(s8 value)
+{
+	s8 ret_val = value;
+
+	/* -33dB~0dB to 33dB ~ 0dB */
+	if (ret_val == -128)
+		ret_val = 127;
+	else if (ret_val < 0)
+		ret_val = 0 - ret_val;
+
+	ret_val = ret_val >> 1;
+	return (u8)ret_val;
+}
+
+static s16 odm_cfo(s8 value)
+{
+	s16 ret_val;
+
+	if (value < 0) {
+		ret_val = 0 - value;
+		ret_val = (ret_val << 1) + (ret_val >> 1); /* *2.5~=312.5/2^7 */
+		ret_val =
+			ret_val | BIT(12); /* set bit12 as 1 for negative cfo */
+	} else {
+		ret_val = value;
+		ret_val = (ret_val << 1) + (ret_val >> 1); /* *2.5~=312.5/2^7 */
+	}
+	return ret_val;
+}
+
+static u8 phydm_rate_to_num_ss(struct phy_dm_struct *dm, u8 data_rate)
+{
+	u8 num_ss = 1;
+
+	if (data_rate <= ODM_RATE54M)
+		num_ss = 1;
+	else if (data_rate <= ODM_RATEMCS31)
+		num_ss = ((data_rate - ODM_RATEMCS0) >> 3) + 1;
+	else if (data_rate <= ODM_RATEVHTSS1MCS9)
+		num_ss = 1;
+	else if (data_rate <= ODM_RATEVHTSS2MCS9)
+		num_ss = 2;
+	else if (data_rate <= ODM_RATEVHTSS3MCS9)
+		num_ss = 3;
+	else if (data_rate <= ODM_RATEVHTSS4MCS9)
+		num_ss = 4;
+
+	return num_ss;
+}
+
+static void odm_rx_phy_status92c_series_parsing(
+	struct phy_dm_struct *dm, struct dm_phy_status_info *phy_info,
+	u8 *phy_status, struct dm_per_pkt_info *pktinfo)
+{
+	u8 i, max_spatial_stream;
+	s8 rx_pwr[4], rx_pwr_all = 0;
+	u8 EVM, pwdb_all = 0, pwdb_all_bt;
+	u8 RSSI, total_rssi = 0;
+	bool is_cck_rate = false;
+	u8 rf_rx_num = 0;
+	u8 LNA_idx = 0;
+	u8 VGA_idx = 0;
+	u8 cck_agc_rpt;
+	u8 num_ss;
+	struct phy_status_rpt_8192cd *phy_sta_rpt =
+		(struct phy_status_rpt_8192cd *)phy_status;
+
+	is_cck_rate = (pktinfo->data_rate <= ODM_RATE11M) ? true : false;
+
+	if (pktinfo->is_to_self)
+		dm->curr_station_id = pktinfo->station_id;
+
+	phy_info->rx_mimo_signal_quality[ODM_RF_PATH_A] = -1;
+	phy_info->rx_mimo_signal_quality[ODM_RF_PATH_B] = -1;
+
+	if (is_cck_rate) {
+		dm->phy_dbg_info.num_qry_phy_status_cck++;
+		cck_agc_rpt = phy_sta_rpt->cck_agc_rpt_ofdm_cfosho_a;
+
+		if (dm->support_ic_type & (ODM_RTL8703B)) {
+		} else { /*3 bit LNA*/
+
+			LNA_idx = ((cck_agc_rpt & 0xE0) >> 5);
+			VGA_idx = (cck_agc_rpt & 0x1F);
+		}
+
+		ODM_RT_TRACE(
+			dm, ODM_COMP_RSSI_MONITOR,
+			"ext_lna_gain (( %d )), LNA_idx: (( 0x%x )), VGA_idx: (( 0x%x )), rx_pwr_all: (( %d ))\n",
+			dm->ext_lna_gain, LNA_idx, VGA_idx, rx_pwr_all);
+
+		if (dm->board_type & ODM_BOARD_EXT_LNA)
+			rx_pwr_all -= dm->ext_lna_gain;
+
+		pwdb_all = odm_query_rx_pwr_percentage(rx_pwr_all);
+
+		if (pktinfo->is_to_self) {
+			dm->cck_lna_idx = LNA_idx;
+			dm->cck_vga_idx = VGA_idx;
+		}
+		phy_info->rx_pwdb_all = pwdb_all;
+
+		phy_info->bt_rx_rssi_percentage = pwdb_all;
+		phy_info->recv_signal_power = rx_pwr_all;
+		/* (3) Get Signal Quality (EVM) */
+		{
+			u8 sq;
+
+			sq = phydm_get_signal_quality_8192(phy_info, dm,
+							   phy_sta_rpt);
+			phy_info->signal_quality = sq;
+			phy_info->rx_mimo_signal_quality[ODM_RF_PATH_A] = sq;
+			phy_info->rx_mimo_signal_quality[ODM_RF_PATH_B] = -1;
+		}
+
+		for (i = ODM_RF_PATH_A; i < ODM_RF_PATH_MAX; i++) {
+			if (i == 0)
+				phy_info->rx_mimo_signal_strength[0] = pwdb_all;
+			else
+				phy_info->rx_mimo_signal_strength[1] = 0;
+		}
+	} else { /* 2 is OFDM rate */
+		dm->phy_dbg_info.num_qry_phy_status_ofdm++;
+
+		/*  */
+		/* (1)Get RSSI for HT rate */
+		/*  */
+
+		for (i = ODM_RF_PATH_A; i < ODM_RF_PATH_MAX; i++) {
+			/* 2008/01/30 MH we will judge RF RX path now. */
+			if (dm->rf_path_rx_enable & BIT(i))
+				rf_rx_num++;
+			/* else */
+			/* continue; */
+
+			rx_pwr[i] =
+				((phy_sta_rpt->path_agc[i].gain & 0x3F) * 2) -
+				110;
+
+			if (pktinfo->is_to_self) {
+				dm->ofdm_agc_idx[i] =
+					(phy_sta_rpt->path_agc[i].gain & 0x3F);
+				/**/
+			}
+
+			phy_info->rx_pwr[i] = rx_pwr[i];
+
+			/* Translate DBM to percentage. */
+			RSSI = odm_query_rx_pwr_percentage(rx_pwr[i]);
+			total_rssi += RSSI;
+
+			phy_info->rx_mimo_signal_strength[i] = (u8)RSSI;
+
+			/* Get Rx snr value in DB */
+			dm->phy_dbg_info.rx_snr_db[i] =
+				(s32)(phy_sta_rpt->path_rxsnr[i] / 2);
+			phy_info->rx_snr[i] = dm->phy_dbg_info.rx_snr_db[i];
+
+			/* Record Signal Strength for next packet */
+			/* if(pktinfo->is_packet_match_bssid) */
+			{
+			}
+		}
+
+		/*  */
+		/* (2)PWDB, Average PWDB calcuated by hardware (for RA) */
+		/*  */
+		rx_pwr_all = (((phy_sta_rpt->cck_sig_qual_ofdm_pwdb_all) >> 1) &
+			      0x7f) -
+			     110;
+
+		pwdb_all = odm_query_rx_pwr_percentage(rx_pwr_all);
+		pwdb_all_bt = pwdb_all;
+
+		phy_info->rx_pwdb_all = pwdb_all;
+		phy_info->bt_rx_rssi_percentage = pwdb_all_bt;
+		phy_info->rx_power = rx_pwr_all;
+		phy_info->recv_signal_power = rx_pwr_all;
+
+		if ((dm->support_platform == ODM_WIN) && (dm->patch_id == 19)) {
+			/* do nothing */
+		} else if ((dm->support_platform == ODM_WIN) &&
+			   (dm->patch_id == 25)) {
+			/* do nothing */
+		} else { /* mgnt_info->customer_id != RT_CID_819X_LENOVO */
+			/*  */
+			/* (3)EVM of HT rate */
+			/*  */
+			if (pktinfo->data_rate >= ODM_RATEMCS8 &&
+			    pktinfo->data_rate <= ODM_RATEMCS15) {
+				/* both spatial stream make sense */
+				max_spatial_stream = 2;
+			} else {
+				/* only spatial stream 1 makes sense */
+				max_spatial_stream = 1;
+			}
+
+			for (i = 0; i < max_spatial_stream; i++) {
+				/*Don't use shift operation like "rx_evmX >>= 1"
+				 *because the compilor of free build environment
+				 *fill most significant bit to "zero" when doing
+				 *shifting operation which may change a negative
+				 *value to positive one, then the dbm value
+				 *(which is supposed to be negative)  is not
+				 *correct anymore.
+				 */
+				EVM = odm_evm_db_to_percentage(
+					(phy_sta_rpt
+						 ->stream_rxevm[i])); /* dbm */
+
+				/* Fill value in RFD, Get the first spatial
+				 * stream only
+				 */
+				if (i == ODM_RF_PATH_A)
+					phy_info->signal_quality =
+						(u8)(EVM & 0xff);
+				phy_info->rx_mimo_signal_quality[i] =
+					(u8)(EVM & 0xff);
+			}
+		}
+
+		num_ss = phydm_rate_to_num_ss(dm, pktinfo->data_rate);
+		odm_parsing_cfo(dm, pktinfo, phy_sta_rpt->path_cfotail, num_ss);
+	}
+	/* UI BSS List signal strength(in percentage), make it good looking,
+	 * from 0~100.
+	 */
+	/* It is assigned to the BSS List in GetValueFromBeaconOrProbeRsp(). */
+	if (is_cck_rate) {
+		phy_info->signal_strength = (u8)(
+			odm_signal_scale_mapping(dm, pwdb_all)); /*pwdb_all;*/
+	} else {
+		if (rf_rx_num != 0) {
+			phy_info->signal_strength =
+				(u8)(odm_signal_scale_mapping(dm, total_rssi /=
+								  rf_rx_num));
+		}
+	}
+
+	/* For 92C/92D HW (Hybrid) Antenna Diversity */
+}
+
+static void
+odm_rx_phy_bw_jaguar_series_parsing(struct dm_phy_status_info *phy_info,
+				    struct dm_per_pkt_info *pktinfo,
+				    struct phy_status_rpt_8812 *phy_sta_rpt)
+{
+	if (pktinfo->data_rate <= ODM_RATE54M) {
+		switch (phy_sta_rpt->r_RFMOD) {
+		case 1:
+			if (phy_sta_rpt->sub_chnl == 0)
+				phy_info->band_width = 1;
+			else
+				phy_info->band_width = 0;
+			break;
+
+		case 2:
+			if (phy_sta_rpt->sub_chnl == 0)
+				phy_info->band_width = 2;
+			else if (phy_sta_rpt->sub_chnl == 9 ||
+				 phy_sta_rpt->sub_chnl == 10)
+				phy_info->band_width = 1;
+			else
+				phy_info->band_width = 0;
+			break;
+
+		default:
+		case 0:
+			phy_info->band_width = 0;
+			break;
+		}
+	}
+}
+
+static void odm_rx_phy_status_jaguar_series_parsing(
+	struct phy_dm_struct *dm, struct dm_phy_status_info *phy_info,
+	u8 *phy_status, struct dm_per_pkt_info *pktinfo)
+{
+	u8 i, max_spatial_stream;
+	s8 rx_pwr[4], rx_pwr_all = 0;
+	u8 EVM = 0, evm_dbm, pwdb_all = 0, pwdb_all_bt;
+	u8 RSSI, avg_rssi = 0, best_rssi = 0, second_rssi = 0;
+	u8 is_cck_rate = 0;
+	u8 rf_rx_num = 0;
+	u8 cck_highpwr = 0;
+	u8 LNA_idx, VGA_idx;
+	struct phy_status_rpt_8812 *phy_sta_rpt =
+		(struct phy_status_rpt_8812 *)phy_status;
+	struct fast_antenna_training *fat_tab = &dm->dm_fat_table;
+	u8 num_ss;
+
+	odm_rx_phy_bw_jaguar_series_parsing(phy_info, pktinfo, phy_sta_rpt);
+
+	if (pktinfo->data_rate <= ODM_RATE11M)
+		is_cck_rate = true;
+	else
+		is_cck_rate = false;
+
+	if (pktinfo->is_to_self)
+		dm->curr_station_id = pktinfo->station_id;
+	else
+		dm->curr_station_id = 0xff;
+
+	phy_info->rx_mimo_signal_quality[ODM_RF_PATH_A] = -1;
+	phy_info->rx_mimo_signal_quality[ODM_RF_PATH_B] = -1;
+	phy_info->rx_mimo_signal_quality[ODM_RF_PATH_C] = -1;
+	phy_info->rx_mimo_signal_quality[ODM_RF_PATH_D] = -1;
+
+	if (is_cck_rate) {
+		u8 cck_agc_rpt;
+
+		dm->phy_dbg_info.num_qry_phy_status_cck++;
+
+		/*(1)Hardware does not provide RSSI for CCK*/
+		/*(2)PWDB, Average PWDB calculated by hardware (for RA)*/
+
+		cck_highpwr = dm->is_cck_high_power;
+
+		cck_agc_rpt = phy_sta_rpt->cfosho[0];
+		LNA_idx = ((cck_agc_rpt & 0xE0) >> 5);
+		VGA_idx = (cck_agc_rpt & 0x1F);
+
+		if (dm->support_ic_type == ODM_RTL8812) {
+			rx_pwr_all =
+				phydm_get_rx_pwr(LNA_idx, VGA_idx, cck_highpwr);
+			rx_pwr_all += 6;
+			pwdb_all = odm_query_rx_pwr_percentage(rx_pwr_all);
+			pwdb_all = phydm_adjust_pwdb(cck_highpwr, pwdb_all);
+
+		} else if (dm->support_ic_type & (ODM_RTL8821 | ODM_RTL8881A)) {
+			s8 pout = -6;
+
+			switch (LNA_idx) {
+			case 5:
+				rx_pwr_all = pout - 32 - (2 * VGA_idx);
+				break;
+			case 4:
+				rx_pwr_all = pout - 24 - (2 * VGA_idx);
+				break;
+			case 2:
+				rx_pwr_all = pout - 11 - (2 * VGA_idx);
+				break;
+			case 1:
+				rx_pwr_all = pout + 5 - (2 * VGA_idx);
+				break;
+			case 0:
+				rx_pwr_all = pout + 21 - (2 * VGA_idx);
+				break;
+			}
+			pwdb_all = odm_query_rx_pwr_percentage(rx_pwr_all);
+		} else if (dm->support_ic_type == ODM_RTL8814A ||
+			   dm->support_ic_type == ODM_RTL8822B) {
+			s8 pout = -6;
+
+			switch (LNA_idx) {
+			/*CCK only use LNA: 2, 3, 5, 7*/
+			case 7:
+				rx_pwr_all = pout - 32 - (2 * VGA_idx);
+				break;
+			case 5:
+				rx_pwr_all = pout - 22 - (2 * VGA_idx);
+				break;
+			case 3:
+				rx_pwr_all = pout - 2 - (2 * VGA_idx);
+				break;
+			case 2:
+				rx_pwr_all = pout + 5 - (2 * VGA_idx);
+				break;
+			default:
+				break;
+			}
+			pwdb_all = odm_query_rx_pwr_percentage(rx_pwr_all);
+		}
+
+		dm->cck_lna_idx = LNA_idx;
+		dm->cck_vga_idx = VGA_idx;
+		phy_info->rx_pwdb_all = pwdb_all;
+		phy_info->bt_rx_rssi_percentage = pwdb_all;
+		phy_info->recv_signal_power = rx_pwr_all;
+		/*(3) Get Signal Quality (EVM)*/
+		{
+			u8 sq;
+
+			if ((dm->support_platform == ODM_WIN) &&
+			    (dm->patch_id == RT_CID_819X_LENOVO))
+				sq = odm_sq_process_patch_rt_cid_819x_lenovo(
+					dm, is_cck_rate, pwdb_all, 0, 0);
+			else
+				sq = phydm_get_signal_quality_8812(phy_info, dm,
+								   phy_sta_rpt);
+
+			phy_info->signal_quality = sq;
+			phy_info->rx_mimo_signal_quality[ODM_RF_PATH_A] = sq;
+		}
+
+		for (i = ODM_RF_PATH_A; i < ODM_RF_PATH_MAX_JAGUAR; i++) {
+			if (i == 0)
+				phy_info->rx_mimo_signal_strength[0] = pwdb_all;
+			else
+				phy_info->rx_mimo_signal_strength[i] = 0;
+		}
+	} else {
+		/*is OFDM rate*/
+		fat_tab->hw_antsw_occur = phy_sta_rpt->hw_antsw_occur;
+
+		dm->phy_dbg_info.num_qry_phy_status_ofdm++;
+
+		/*(1)Get RSSI for OFDM rate*/
+
+		for (i = ODM_RF_PATH_A; i < ODM_RF_PATH_MAX_JAGUAR; i++) {
+			/*2008/01/30 MH we will judge RF RX path now.*/
+			if (dm->rf_path_rx_enable & BIT(i))
+				rf_rx_num++;
+			/*2012.05.25 LukeLee: Testchip AGC report is wrong,
+			 *it should be restored back to old formula in MP chip
+			 */
+			if (i < ODM_RF_PATH_C)
+				rx_pwr[i] = (phy_sta_rpt->gain_trsw[i] & 0x7F) -
+					    110;
+			else
+				rx_pwr[i] = (phy_sta_rpt->gain_trsw_cd[i - 2] &
+					     0x7F) -
+					    110;
+
+			phy_info->rx_pwr[i] = rx_pwr[i];
+
+			/* Translate DBM to percentage. */
+			RSSI = odm_query_rx_pwr_percentage(rx_pwr[i]);
+
+			/*total_rssi += RSSI;*/
+			/*Get the best two RSSI*/
+			if (RSSI > best_rssi && RSSI > second_rssi) {
+				second_rssi = best_rssi;
+				best_rssi = RSSI;
+			} else if (RSSI > second_rssi && RSSI <= best_rssi) {
+				second_rssi = RSSI;
+			}
+
+			phy_info->rx_mimo_signal_strength[i] = (u8)RSSI;
+
+			/*Get Rx snr value in DB*/
+			if (i < ODM_RF_PATH_C)
+				phy_info->rx_snr[i] =
+					dm->phy_dbg_info.rx_snr_db[i] =
+						phy_sta_rpt->rxsnr[i] / 2;
+			else if (dm->support_ic_type &
+				 (ODM_RTL8814A | ODM_RTL8822B))
+				phy_info->rx_snr[i] = dm->phy_dbg_info
+							      .rx_snr_db[i] =
+					phy_sta_rpt->csi_current[i - 2] / 2;
+
+			/*(2) CFO_short  & CFO_tail*/
+			if (i < ODM_RF_PATH_C) {
+				phy_info->cfo_short[i] =
+					odm_cfo((phy_sta_rpt->cfosho[i]));
+				phy_info->cfo_tail[i] =
+					odm_cfo((phy_sta_rpt->cfotail[i]));
+			}
+		}
+
+		/*(3)PWDB, Average PWDB calculated by hardware (for RA)*/
+
+		/*2012.05.25 LukeLee: Testchip AGC report is wrong, it should be
+		 *restored back to old formula in MP chip
+		 */
+		if ((dm->support_ic_type &
+		     (ODM_RTL8812 | ODM_RTL8821 | ODM_RTL8881A)) &&
+		    (!dm->is_mp_chip))
+			rx_pwr_all = (phy_sta_rpt->pwdb_all & 0x7f) - 110;
+		else
+			rx_pwr_all = (((phy_sta_rpt->pwdb_all) >> 1) & 0x7f) -
+				     110; /*OLD FORMULA*/
+
+		pwdb_all = odm_query_rx_pwr_percentage(rx_pwr_all);
+		pwdb_all_bt = pwdb_all;
+
+		phy_info->rx_pwdb_all = pwdb_all;
+		phy_info->bt_rx_rssi_percentage = pwdb_all_bt;
+		phy_info->rx_power = rx_pwr_all;
+		phy_info->recv_signal_power = rx_pwr_all;
+
+		if ((dm->support_platform == ODM_WIN) && (dm->patch_id == 19)) {
+			/*do nothing*/
+		} else {
+			/*mgnt_info->customer_id != RT_CID_819X_LENOVO*/
+
+			/*(4)EVM of OFDM rate*/
+
+			if ((pktinfo->data_rate >= ODM_RATEMCS8) &&
+			    (pktinfo->data_rate <= ODM_RATEMCS15))
+				max_spatial_stream = 2;
+			else if ((pktinfo->data_rate >= ODM_RATEVHTSS2MCS0) &&
+				 (pktinfo->data_rate <= ODM_RATEVHTSS2MCS9))
+				max_spatial_stream = 2;
+			else if ((pktinfo->data_rate >= ODM_RATEMCS16) &&
+				 (pktinfo->data_rate <= ODM_RATEMCS23))
+				max_spatial_stream = 3;
+			else if ((pktinfo->data_rate >= ODM_RATEVHTSS3MCS0) &&
+				 (pktinfo->data_rate <= ODM_RATEVHTSS3MCS9))
+				max_spatial_stream = 3;
+			else
+				max_spatial_stream = 1;
+
+			for (i = 0; i < max_spatial_stream; i++) {
+				/*Don't use shift operation like "rx_evmX >>= 1"
+				 *because the compilor of free build environment
+				 *fill most significant bit to "zero" when doing
+				 *shifting operation which may change a negative
+				 *value to positive one, then the dbm value
+				 *(which is supposed to be negative) is not
+				 *correct anymore.
+				 */
+
+				EVM = phydm_get_odm_evm(i, pktinfo,
+							phy_sta_rpt);
+				evm_dbm = phydm_get_evm_dbm(i, EVM, phy_sta_rpt,
+							    phy_info);
+				phy_info->rx_mimo_signal_quality[i] = EVM;
+				phy_info->rx_mimo_evm_dbm[i] = evm_dbm;
+			}
+		}
+
+		num_ss = phydm_rate_to_num_ss(dm, pktinfo->data_rate);
+		odm_parsing_cfo(dm, pktinfo, phy_sta_rpt->cfotail, num_ss);
+	}
+
+	/*UI BSS List signal strength(in percentage), make it good looking,
+	 *from 0~100.
+	 */
+	/*It is assigned to the BSS List in GetValueFromBeaconOrProbeRsp().*/
+	if (is_cck_rate) {
+		phy_info->signal_strength = (u8)(
+			odm_signal_scale_mapping(dm, pwdb_all)); /*pwdb_all;*/
+	} else {
+		if (rf_rx_num != 0) {
+			/* 2015/01 Sean, use the best two RSSI only,
+			 * suggested by Ynlin and ChenYu.
+			 */
+			if (rf_rx_num == 1)
+				avg_rssi = best_rssi;
+			else
+				avg_rssi = (best_rssi + second_rssi) / 2;
+			phy_info->signal_strength =
+				(u8)(odm_signal_scale_mapping(dm, avg_rssi));
+		}
+	}
+	dm->rx_pwdb_ave = dm->rx_pwdb_ave + phy_info->rx_pwdb_all;
+
+	dm->dm_fat_table.antsel_rx_keep_0 = phy_sta_rpt->antidx_anta;
+	dm->dm_fat_table.antsel_rx_keep_1 = phy_sta_rpt->antidx_antb;
+	dm->dm_fat_table.antsel_rx_keep_2 = phy_sta_rpt->antidx_antc;
+	dm->dm_fat_table.antsel_rx_keep_3 = phy_sta_rpt->antidx_antd;
+}
+
+void phydm_reset_rssi_for_dm(struct phy_dm_struct *dm, u8 station_id)
+{
+	struct rtl_sta_info *entry;
+
+	entry = dm->odm_sta_info[station_id];
+
+	if (!IS_STA_VALID(entry))
+		return;
+
+	ODM_RT_TRACE(dm, ODM_COMP_RSSI_MONITOR,
+		     "Reset RSSI for macid = (( %d ))\n", station_id);
+
+	entry->rssi_stat.undecorated_smoothed_cck = -1;
+	entry->rssi_stat.undecorated_smoothed_ofdm = -1;
+	entry->rssi_stat.undecorated_smoothed_pwdb = -1;
+	entry->rssi_stat.ofdm_pkt = 0;
+	entry->rssi_stat.cck_pkt = 0;
+	entry->rssi_stat.cck_sum_power = 0;
+	entry->rssi_stat.is_send_rssi = RA_RSSI_STATE_INIT;
+	entry->rssi_stat.packet_map = 0;
+	entry->rssi_stat.valid_bit = 0;
+}
+
+void odm_init_rssi_for_dm(struct phy_dm_struct *dm) {}
+
+static void odm_process_rssi_for_dm(struct phy_dm_struct *dm,
+				    struct dm_phy_status_info *phy_info,
+				    struct dm_per_pkt_info *pktinfo)
+{
+	s32 undecorated_smoothed_pwdb, undecorated_smoothed_cck,
+		undecorated_smoothed_ofdm;
+	u8 is_cck_rate = 0;
+	u8 send_rssi_2_fw = 0;
+	struct rtl_sta_info *entry;
+
+	if (pktinfo->station_id >= ODM_ASSOCIATE_ENTRY_NUM)
+		return;
+
+	/* 2012/05/30 MH/Luke.Lee Add some description */
+	/* In windows driver: AP/IBSS mode STA */
+	entry = dm->odm_sta_info[pktinfo->station_id];
+
+	if (!IS_STA_VALID(entry))
+		return;
+
+	{
+		if ((!pktinfo->is_packet_match_bssid)) /*data frame only*/
+			return;
+	}
+
+	if (pktinfo->is_packet_beacon)
+		dm->phy_dbg_info.num_qry_beacon_pkt++;
+
+	is_cck_rate = (pktinfo->data_rate <= ODM_RATE11M) ? true : false;
+	dm->rx_rate = pktinfo->data_rate;
+
+	/* --------------Statistic for antenna/path diversity---------------- */
+
+	/* -----------------Smart Antenna Debug Message------------------ */
+
+	undecorated_smoothed_cck = entry->rssi_stat.undecorated_smoothed_cck;
+	undecorated_smoothed_ofdm = entry->rssi_stat.undecorated_smoothed_ofdm;
+	undecorated_smoothed_pwdb = entry->rssi_stat.undecorated_smoothed_pwdb;
+
+	if (pktinfo->is_packet_to_self || pktinfo->is_packet_beacon) {
+		if (!is_cck_rate) /* ofdm rate */
+			undecorated_smoothed_ofdm = phydm_process_rssi_ofdm(
+				dm, phy_info, entry, undecorated_smoothed_ofdm);
+		else
+			undecorated_smoothed_cck = phydm_process_rssi_cck(
+				dm, phy_info, entry, undecorated_smoothed_cck);
+
+		undecorated_smoothed_pwdb = phydm_process_rssi_pwdb(
+			dm, entry, pktinfo, undecorated_smoothed_ofdm,
+			undecorated_smoothed_cck);
+
+		if ((entry->rssi_stat.ofdm_pkt >= 1 ||
+		     entry->rssi_stat.cck_pkt >= 5) &&
+		    (entry->rssi_stat.is_send_rssi == RA_RSSI_STATE_INIT)) {
+			send_rssi_2_fw = 1;
+			entry->rssi_stat.is_send_rssi = RA_RSSI_STATE_SEND;
+		}
+
+		entry->rssi_stat.undecorated_smoothed_cck =
+			undecorated_smoothed_cck;
+		entry->rssi_stat.undecorated_smoothed_ofdm =
+			undecorated_smoothed_ofdm;
+		entry->rssi_stat.undecorated_smoothed_pwdb =
+			undecorated_smoothed_pwdb;
+
+		if (send_rssi_2_fw) { /* Trigger init rate by RSSI */
+
+			if (entry->rssi_stat.ofdm_pkt != 0)
+				entry->rssi_stat.undecorated_smoothed_pwdb =
+					undecorated_smoothed_ofdm;
+
+			ODM_RT_TRACE(
+				dm, ODM_COMP_RSSI_MONITOR,
+				"[Send to FW] PWDB = (( %d )), ofdm_pkt = (( %d )), cck_pkt = (( %d ))\n",
+				undecorated_smoothed_pwdb,
+				entry->rssi_stat.ofdm_pkt,
+				entry->rssi_stat.cck_pkt);
+		}
+	}
+}
+
+/*
+ * Endianness before calling this API
+ */
+static void odm_phy_status_query_92c_series(struct phy_dm_struct *dm,
+					    struct dm_phy_status_info *phy_info,
+					    u8 *phy_status,
+					    struct dm_per_pkt_info *pktinfo)
+{
+	odm_rx_phy_status92c_series_parsing(dm, phy_info, phy_status, pktinfo);
+	odm_process_rssi_for_dm(dm, phy_info, pktinfo);
+}
+
+/*
+ * Endianness before calling this API
+ */
+
+static void odm_phy_status_query_jaguar_series(
+	struct phy_dm_struct *dm, struct dm_phy_status_info *phy_info,
+	u8 *phy_status, struct dm_per_pkt_info *pktinfo)
+{
+	odm_rx_phy_status_jaguar_series_parsing(dm, phy_info, phy_status,
+						pktinfo);
+	odm_process_rssi_for_dm(dm, phy_info, pktinfo);
+}
+
+void odm_phy_status_query(struct phy_dm_struct *dm,
+			  struct dm_phy_status_info *phy_info, u8 *phy_status,
+			  struct dm_per_pkt_info *pktinfo)
+{
+	if (dm->support_ic_type & ODM_IC_PHY_STATUE_NEW_TYPE) {
+		phydm_rx_phy_status_new_type(dm, phy_status, pktinfo, phy_info);
+		return;
+	}
+
+	if (dm->support_ic_type & ODM_IC_11AC_SERIES)
+		odm_phy_status_query_jaguar_series(dm, phy_info, phy_status,
+						   pktinfo);
+
+	if (dm->support_ic_type & ODM_IC_11N_SERIES)
+		odm_phy_status_query_92c_series(dm, phy_info, phy_status,
+						pktinfo);
+}
+
+/* For future use. */
+void odm_mac_status_query(struct phy_dm_struct *dm, u8 *mac_status, u8 mac_id,
+			  bool is_packet_match_bssid, bool is_packet_to_self,
+			  bool is_packet_beacon)
+{
+	/* 2011/10/19 Driver team will handle in the future. */
+}
+
+/*
+ * If you want to add a new IC, Please follow below template and generate
+ * a new one.
+ */
+
+enum hal_status
+odm_config_rf_with_header_file(struct phy_dm_struct *dm,
+			       enum odm_rf_config_type config_type,
+			       enum odm_rf_radio_path e_rf_path)
+{
+	ODM_RT_TRACE(dm, ODM_COMP_INIT,
+		     "===>%s (%s)\n", __func__,
+		     (dm->is_mp_chip) ? "MPChip" : "TestChip");
+	ODM_RT_TRACE(
+		dm, ODM_COMP_INIT,
+		"dm->support_platform: 0x%X, dm->support_interface: 0x%X, dm->board_type: 0x%X\n",
+		dm->support_platform, dm->support_interface, dm->board_type);
+
+	/* 1 AP doesn't use PHYDM power tracking table in these ICs */
+	/* JJ ADD 20161014 */
+
+	/* 1 All platforms support */
+	if (dm->support_ic_type == ODM_RTL8822B) {
+		if (config_type == CONFIG_RF_RADIO) {
+			if (e_rf_path == ODM_RF_PATH_A)
+				READ_AND_CONFIG_MP(8822b, _radioa);
+			else if (e_rf_path == ODM_RF_PATH_B)
+				READ_AND_CONFIG_MP(8822b, _radiob);
+		} else if (config_type == CONFIG_RF_TXPWR_LMT) {
+			if (dm->rfe_type == 5)
+				READ_AND_CONFIG_MP(8822b, _txpwr_lmt_type5);
+			else
+				READ_AND_CONFIG_MP(8822b, _txpwr_lmt);
+		}
+	}
+
+	return HAL_STATUS_SUCCESS;
+}
+
+enum hal_status
+odm_config_rf_with_tx_pwr_track_header_file(struct phy_dm_struct *dm)
+{
+	ODM_RT_TRACE(dm, ODM_COMP_INIT,
+		     "===>%s (%s)\n", __func__,
+		     (dm->is_mp_chip) ? "MPChip" : "TestChip");
+	ODM_RT_TRACE(
+		dm, ODM_COMP_INIT,
+		"dm->support_platform: 0x%X, dm->support_interface: 0x%X, dm->board_type: 0x%X\n",
+		dm->support_platform, dm->support_interface, dm->board_type);
+
+	/* 1 AP doesn't use PHYDM power tracking table in these ICs */
+	/* JJ ADD 20161014 */
+
+	/* 1 All platforms support */
+
+	if (dm->support_ic_type == ODM_RTL8822B) {
+		if (dm->rfe_type == 0)
+			READ_AND_CONFIG_MP(8822b, _txpowertrack_type0);
+		else if (dm->rfe_type == 1)
+			READ_AND_CONFIG_MP(8822b, _txpowertrack_type1);
+		else if (dm->rfe_type == 2)
+			READ_AND_CONFIG_MP(8822b, _txpowertrack_type2);
+		else if ((dm->rfe_type == 3) || (dm->rfe_type == 5))
+			READ_AND_CONFIG_MP(8822b, _txpowertrack_type3_type5);
+		else if (dm->rfe_type == 4)
+			READ_AND_CONFIG_MP(8822b, _txpowertrack_type4);
+		else if (dm->rfe_type == 6)
+			READ_AND_CONFIG_MP(8822b, _txpowertrack_type6);
+		else if (dm->rfe_type == 7)
+			READ_AND_CONFIG_MP(8822b, _txpowertrack_type7);
+		else if (dm->rfe_type == 8)
+			READ_AND_CONFIG_MP(8822b, _txpowertrack_type8);
+		else if (dm->rfe_type == 9)
+			READ_AND_CONFIG_MP(8822b, _txpowertrack_type9);
+		else
+			READ_AND_CONFIG_MP(8822b, _txpowertrack);
+	}
+
+	return HAL_STATUS_SUCCESS;
+}
+
+enum hal_status
+odm_config_bb_with_header_file(struct phy_dm_struct *dm,
+			       enum odm_bb_config_type config_type)
+{
+	/* 1 AP doesn't use PHYDM initialization in these ICs */
+	/* JJ ADD 20161014 */
+
+	/* 1 All platforms support */
+	if (dm->support_ic_type == ODM_RTL8822B) {
+		if (config_type == CONFIG_BB_PHY_REG)
+			READ_AND_CONFIG_MP(8822b, _phy_reg);
+		else if (config_type == CONFIG_BB_AGC_TAB)
+			READ_AND_CONFIG_MP(8822b, _agc_tab);
+		else if (config_type == CONFIG_BB_PHY_REG_PG)
+			READ_AND_CONFIG_MP(8822b, _phy_reg_pg);
+		/*else if (config_type == CONFIG_BB_PHY_REG_MP)*/
+		/*READ_AND_CONFIG_MP(8822b, _phy_reg_mp);*/
+	}
+
+	return HAL_STATUS_SUCCESS;
+}
+
+enum hal_status odm_config_mac_with_header_file(struct phy_dm_struct *dm)
+{
+	ODM_RT_TRACE(dm, ODM_COMP_INIT,
+		     "===>%s (%s)\n", __func__,
+		     (dm->is_mp_chip) ? "MPChip" : "TestChip");
+	ODM_RT_TRACE(
+		dm, ODM_COMP_INIT,
+		"dm->support_platform: 0x%X, dm->support_interface: 0x%X, dm->board_type: 0x%X\n",
+		dm->support_platform, dm->support_interface, dm->board_type);
+
+	/* 1 AP doesn't use PHYDM initialization in these ICs */
+	/* JJ ADD 20161014 */
+
+	/* 1 All platforms support */
+	if (dm->support_ic_type == ODM_RTL8822B)
+		READ_AND_CONFIG_MP(8822b, _mac_reg);
+
+	return HAL_STATUS_SUCCESS;
+}
+
+enum hal_status
+odm_config_fw_with_header_file(struct phy_dm_struct *dm,
+			       enum odm_fw_config_type config_type,
+			       u8 *p_firmware, u32 *size)
+{
+	return HAL_STATUS_SUCCESS;
+}
+
+u32 odm_get_hw_img_version(struct phy_dm_struct *dm)
+{
+	u32 version = 0;
+
+	/* 1 AP doesn't use PHYDM initialization in these ICs */
+	/* JJ ADD 20161014 */
+
+	/*1 All platforms support*/
+	if (dm->support_ic_type == ODM_RTL8822B)
+		version = GET_VERSION_MP(8822b, _mac_reg);
+
+	return version;
+}
+
+/* For 8822B only!! need to move to FW finally */
+/*==============================================*/
+
+bool phydm_query_is_mu_api(struct phy_dm_struct *phydm, u8 ppdu_idx,
+			   u8 *p_data_rate, u8 *p_gid)
+{
+	u8 data_rate = 0, gid = 0;
+	bool is_mu = false;
+
+	data_rate = phydm->phy_dbg_info.num_of_ppdu[ppdu_idx];
+	gid = phydm->phy_dbg_info.gid_num[ppdu_idx];
+
+	if (data_rate & BIT(7)) {
+		is_mu = true;
+		data_rate = data_rate & ~(BIT(7));
+	} else {
+		is_mu = false;
+	}
+
+	*p_data_rate = data_rate;
+	*p_gid = gid;
+
+	return is_mu;
+}
+
+static void phydm_rx_statistic_cal(struct phy_dm_struct *phydm, u8 *phy_status,
+				   struct dm_per_pkt_info *pktinfo)
+{
+	struct phy_status_rpt_jaguar2_type1 *phy_sta_rpt =
+		(struct phy_status_rpt_jaguar2_type1 *)phy_status;
+	u8 date_rate = pktinfo->data_rate & ~(BIT(7));
+
+	if ((phy_sta_rpt->gid != 0) && (phy_sta_rpt->gid != 63)) {
+		if (date_rate >= ODM_RATEVHTSS1MCS0) {
+			phydm->phy_dbg_info
+				.num_qry_mu_vht_pkt[date_rate - 0x2C]++;
+			phydm->phy_dbg_info.num_of_ppdu[pktinfo->ppdu_cnt] =
+				date_rate | BIT(7);
+			phydm->phy_dbg_info.gid_num[pktinfo->ppdu_cnt] =
+				phy_sta_rpt->gid;
+		}
+
+	} else {
+		if (date_rate >= ODM_RATEVHTSS1MCS0) {
+			phydm->phy_dbg_info.num_qry_vht_pkt[date_rate - 0x2C]++;
+			phydm->phy_dbg_info.num_of_ppdu[pktinfo->ppdu_cnt] =
+				date_rate;
+			phydm->phy_dbg_info.gid_num[pktinfo->ppdu_cnt] =
+				phy_sta_rpt->gid;
+		}
+	}
+}
+
+static void phydm_reset_phy_info(struct phy_dm_struct *phydm,
+				 struct dm_phy_status_info *phy_info)
+{
+	phy_info->rx_pwdb_all = 0;
+	phy_info->signal_quality = 0;
+	phy_info->band_width = 0;
+	phy_info->rx_count = 0;
+	odm_memory_set(phydm, phy_info->rx_mimo_signal_quality, 0, 4);
+	odm_memory_set(phydm, phy_info->rx_mimo_signal_strength, 0, 4);
+	odm_memory_set(phydm, phy_info->rx_snr, 0, 4);
+
+	phy_info->rx_power = -110;
+	phy_info->recv_signal_power = -110;
+	phy_info->bt_rx_rssi_percentage = 0;
+	phy_info->signal_strength = 0;
+	phy_info->bt_coex_pwr_adjust = 0;
+	phy_info->channel = 0;
+	phy_info->is_mu_packet = 0;
+	phy_info->is_beamformed = 0;
+	phy_info->rxsc = 0;
+	odm_memory_set(phydm, phy_info->rx_pwr, -110, 4);
+	odm_memory_set(phydm, phy_info->rx_mimo_evm_dbm, 0, 4);
+	odm_memory_set(phydm, phy_info->cfo_short, 0, 8);
+	odm_memory_set(phydm, phy_info->cfo_tail, 0, 8);
+}
+
+static void phydm_set_per_path_phy_info(u8 rx_path, s8 rx_pwr, s8 rx_evm,
+					s8 cfo_tail, s8 rx_snr,
+					struct dm_phy_status_info *phy_info)
+{
+	u8 evm_dbm = 0;
+	u8 evm_percentage = 0;
+
+	/* SNR is S(8,1), EVM is S(8,1), CFO is S(8,7) */
+
+	if (rx_evm < 0) {
+		/* Calculate EVM in dBm */
+		evm_dbm = ((u8)(0 - rx_evm) >> 1);
+
+		/* Calculate EVM in percentage */
+		if (evm_dbm >= 33)
+			evm_percentage = 100;
+		else
+			evm_percentage = (evm_dbm << 1) + (evm_dbm);
+	}
+
+	phy_info->rx_pwr[rx_path] = rx_pwr;
+	phy_info->rx_mimo_evm_dbm[rx_path] = evm_dbm;
+
+	/* CFO = CFO_tail * 312.5 / 2^7 ~= CFO tail * 39/512 (kHz)*/
+	phy_info->cfo_tail[rx_path] = cfo_tail;
+	phy_info->cfo_tail[rx_path] = ((phy_info->cfo_tail[rx_path] << 5) +
+				       (phy_info->cfo_tail[rx_path] << 2) +
+				       (phy_info->cfo_tail[rx_path] << 1) +
+				       (phy_info->cfo_tail[rx_path])) >>
+				      9;
+
+	phy_info->rx_mimo_signal_strength[rx_path] =
+		odm_query_rx_pwr_percentage(rx_pwr);
+	phy_info->rx_mimo_signal_quality[rx_path] = evm_percentage;
+	phy_info->rx_snr[rx_path] = rx_snr >> 1;
+}
+
+static void phydm_set_common_phy_info(s8 rx_power, u8 channel,
+				      bool is_beamformed, bool is_mu_packet,
+				      u8 bandwidth, u8 signal_quality, u8 rxsc,
+				      struct dm_phy_status_info *phy_info)
+{
+	phy_info->rx_power = rx_power; /* RSSI in dB */
+	phy_info->recv_signal_power = rx_power; /* RSSI in dB */
+	phy_info->channel = channel; /* channel number */
+	phy_info->is_beamformed = is_beamformed; /* apply BF */
+	phy_info->is_mu_packet = is_mu_packet; /* MU packet */
+	phy_info->rxsc = rxsc;
+	phy_info->rx_pwdb_all =
+		odm_query_rx_pwr_percentage(rx_power); /* RSSI in percentage */
+	phy_info->signal_quality = signal_quality; /* signal quality */
+	phy_info->band_width = bandwidth; /* bandwidth */
+}
+
+static void phydm_get_rx_phy_status_type0(struct phy_dm_struct *dm,
+					  u8 *phy_status,
+					  struct dm_per_pkt_info *pktinfo,
+					  struct dm_phy_status_info *phy_info)
+{
+	/* type 0 is used for cck packet */
+
+	struct phy_status_rpt_jaguar2_type0 *phy_sta_rpt =
+		(struct phy_status_rpt_jaguar2_type0 *)phy_status;
+	u8 sq = 0;
+	s8 rx_power = phy_sta_rpt->pwdb - 110;
+
+	/* JJ ADD 20161014 */
+
+	/* Calculate Signal Quality*/
+	if (pktinfo->is_packet_match_bssid) {
+		if (phy_sta_rpt->signal_quality >= 64) {
+			sq = 0;
+		} else if (phy_sta_rpt->signal_quality <= 20) {
+			sq = 100;
+		} else {
+			/* mapping to 2~99% */
+			sq = 64 - phy_sta_rpt->signal_quality;
+			sq = ((sq << 3) + sq) >> 2;
+		}
+	}
+
+	/* Modify CCK PWDB if old AGC */
+	if (!dm->cck_new_agc) {
+		u8 lna_idx, vga_idx;
+
+		lna_idx = ((phy_sta_rpt->lna_h << 3) | phy_sta_rpt->lna_l);
+		vga_idx = phy_sta_rpt->vga;
+
+		/* JJ ADD 20161014 */
+
+		/* Need to do !! */
+		/*if (dm->support_ic_type & ODM_RTL8822B) */
+		/*rx_power = odm_CCKRSSI_8822B(LNA_idx, VGA_idx);*/
+	}
+
+	/* Update CCK packet counter */
+	dm->phy_dbg_info.num_qry_phy_status_cck++;
+
+	/*CCK no STBC and LDPC*/
+	dm->phy_dbg_info.is_ldpc_pkt = false;
+	dm->phy_dbg_info.is_stbc_pkt = false;
+
+	/* Update Common information */
+	phydm_set_common_phy_info(rx_power, phy_sta_rpt->channel, false, false,
+				  ODM_BW20M, sq, phy_sta_rpt->rxsc, phy_info);
+
+	/* Update CCK pwdb */
+	/* Update per-path information */
+	phydm_set_per_path_phy_info(ODM_RF_PATH_A, rx_power, 0, 0, 0, phy_info);
+
+	dm->dm_fat_table.antsel_rx_keep_0 = phy_sta_rpt->antidx_a;
+	dm->dm_fat_table.antsel_rx_keep_1 = phy_sta_rpt->antidx_b;
+	dm->dm_fat_table.antsel_rx_keep_2 = phy_sta_rpt->antidx_c;
+	dm->dm_fat_table.antsel_rx_keep_3 = phy_sta_rpt->antidx_d;
+}
+
+static void phydm_get_rx_phy_status_type1(struct phy_dm_struct *dm,
+					  u8 *phy_status,
+					  struct dm_per_pkt_info *pktinfo,
+					  struct dm_phy_status_info *phy_info)
+{
+	/* type 1 is used for ofdm packet */
+
+	struct phy_status_rpt_jaguar2_type1 *phy_sta_rpt =
+		(struct phy_status_rpt_jaguar2_type1 *)phy_status;
+	s8 rx_pwr_db = -120;
+	u8 i, rxsc, bw = ODM_BW20M, rx_count = 0;
+	bool is_mu;
+	u8 num_ss;
+
+	/* Update OFDM packet counter */
+	dm->phy_dbg_info.num_qry_phy_status_ofdm++;
+
+	/* Update per-path information */
+	for (i = ODM_RF_PATH_A; i < ODM_RF_PATH_MAX_JAGUAR; i++) {
+		if (dm->rx_ant_status & BIT(i)) {
+			s8 rx_path_pwr_db;
+
+			/* RX path counter */
+			rx_count++;
+
+			/* Update per-path information
+			 * (RSSI_dB RSSI_percentage EVM SNR CFO sq)
+			 */
+			/* EVM report is reported by stream, not path */
+			rx_path_pwr_db = phy_sta_rpt->pwdb[i] -
+					 110; /* per-path pwdb in dB domain */
+			phydm_set_per_path_phy_info(
+				i, rx_path_pwr_db,
+				phy_sta_rpt->rxevm[rx_count - 1],
+				phy_sta_rpt->cfo_tail[i], phy_sta_rpt->rxsnr[i],
+				phy_info);
+
+			/* search maximum pwdb */
+			if (rx_path_pwr_db > rx_pwr_db)
+				rx_pwr_db = rx_path_pwr_db;
+		}
+	}
+
+	/* mapping RX counter from 1~4 to 0~3 */
+	if (rx_count > 0)
+		phy_info->rx_count = rx_count - 1;
+
+	/* Check if MU packet or not */
+	if ((phy_sta_rpt->gid != 0) && (phy_sta_rpt->gid != 63)) {
+		is_mu = true;
+		dm->phy_dbg_info.num_qry_mu_pkt++;
+	} else {
+		is_mu = false;
+	}
+
+	/* count BF packet */
+	dm->phy_dbg_info.num_qry_bf_pkt =
+		dm->phy_dbg_info.num_qry_bf_pkt + phy_sta_rpt->beamformed;
+
+	/*STBC or LDPC pkt*/
+	dm->phy_dbg_info.is_ldpc_pkt = phy_sta_rpt->ldpc;
+	dm->phy_dbg_info.is_stbc_pkt = phy_sta_rpt->stbc;
+
+	/* Check sub-channel */
+	if ((pktinfo->data_rate > ODM_RATE11M) &&
+	    (pktinfo->data_rate < ODM_RATEMCS0))
+		rxsc = phy_sta_rpt->l_rxsc;
+	else
+		rxsc = phy_sta_rpt->ht_rxsc;
+
+	/* Check RX bandwidth */
+	if (dm->support_ic_type & ODM_RTL8822B) {
+		if ((rxsc >= 1) && (rxsc <= 8))
+			bw = ODM_BW20M;
+		else if ((rxsc >= 9) && (rxsc <= 12))
+			bw = ODM_BW40M;
+		else if (rxsc >= 13)
+			bw = ODM_BW80M;
+		else
+			bw = phy_sta_rpt->rf_mode;
+	} else if (dm->support_ic_type & (ODM_RTL8197F | ODM_RTL8723D |
+					  ODM_RTL8710B)) { /* JJ ADD 20161014 */
+		if (phy_sta_rpt->rf_mode == 0)
+			bw = ODM_BW20M;
+		else if ((rxsc == 1) || (rxsc == 2))
+			bw = ODM_BW20M;
+		else
+			bw = ODM_BW40M;
+	}
+
+	/* Update packet information */
+	phydm_set_common_phy_info(
+		rx_pwr_db, phy_sta_rpt->channel, (bool)phy_sta_rpt->beamformed,
+		is_mu, bw, odm_evm_db_to_percentage(phy_sta_rpt->rxevm[0]),
+		rxsc, phy_info);
+
+	num_ss = phydm_rate_to_num_ss(dm, pktinfo->data_rate);
+
+	odm_parsing_cfo(dm, pktinfo, phy_sta_rpt->cfo_tail, num_ss);
+	dm->dm_fat_table.antsel_rx_keep_0 = phy_sta_rpt->antidx_a;
+	dm->dm_fat_table.antsel_rx_keep_1 = phy_sta_rpt->antidx_b;
+	dm->dm_fat_table.antsel_rx_keep_2 = phy_sta_rpt->antidx_c;
+	dm->dm_fat_table.antsel_rx_keep_3 = phy_sta_rpt->antidx_d;
+
+	if (pktinfo->is_packet_match_bssid) {
+		/* */
+		phydm_rx_statistic_cal(dm, phy_status, pktinfo);
+	}
+}
+
+static void phydm_get_rx_phy_status_type2(struct phy_dm_struct *dm,
+					  u8 *phy_status,
+					  struct dm_per_pkt_info *pktinfo,
+					  struct dm_phy_status_info *phy_info)
+{
+	struct phy_status_rpt_jaguar2_type2 *phy_sta_rpt =
+		(struct phy_status_rpt_jaguar2_type2 *)phy_status;
+	s8 rx_pwr_db = -120;
+	u8 i, rxsc, bw = ODM_BW20M, rx_count = 0;
+
+	/* Update OFDM packet counter */
+	dm->phy_dbg_info.num_qry_phy_status_ofdm++;
+
+	/* Update per-path information */
+	for (i = ODM_RF_PATH_A; i < ODM_RF_PATH_MAX_JAGUAR; i++) {
+		if (dm->rx_ant_status & BIT(i)) {
+			s8 rx_path_pwr_db;
+
+			/* RX path counter */
+			rx_count++;
+
+			/* Update per-path information
+			 * (RSSI_dB RSSI_percentage EVM SNR CFO sq)
+			 */
+			rx_path_pwr_db = phy_sta_rpt->pwdb[i] -
+					 110; /* per-path pwdb in dB domain */
+
+			phydm_set_per_path_phy_info(i, rx_path_pwr_db, 0, 0, 0,
+						    phy_info);
+
+			/* search maximum pwdb */
+			if (rx_path_pwr_db > rx_pwr_db)
+				rx_pwr_db = rx_path_pwr_db;
+		}
+	}
+
+	/* mapping RX counter from 1~4 to 0~3 */
+	if (rx_count > 0)
+		phy_info->rx_count = rx_count - 1;
+
+	/* Check RX sub-channel */
+	if ((pktinfo->data_rate > ODM_RATE11M) &&
+	    (pktinfo->data_rate < ODM_RATEMCS0))
+		rxsc = phy_sta_rpt->l_rxsc;
+	else
+		rxsc = phy_sta_rpt->ht_rxsc;
+
+	/*STBC or LDPC pkt*/
+	dm->phy_dbg_info.is_ldpc_pkt = phy_sta_rpt->ldpc;
+	dm->phy_dbg_info.is_stbc_pkt = phy_sta_rpt->stbc;
+
+	/* Check RX bandwidth */
+	/* the BW information of sc=0 is useless, because there is
+	 * no information of RF mode
+	 */
+
+	if (dm->support_ic_type & ODM_RTL8822B) {
+		if ((rxsc >= 1) && (rxsc <= 8))
+			bw = ODM_BW20M;
+		else if ((rxsc >= 9) && (rxsc <= 12))
+			bw = ODM_BW40M;
+		else if (rxsc >= 13)
+			bw = ODM_BW80M;
+		else
+			bw = ODM_BW20M;
+	} else if (dm->support_ic_type & (ODM_RTL8197F | ODM_RTL8723D |
+					  ODM_RTL8710B)) { /* JJ ADD 20161014 */
+		if (rxsc == 3)
+			bw = ODM_BW40M;
+		else if ((rxsc == 1) || (rxsc == 2))
+			bw = ODM_BW20M;
+		else
+			bw = ODM_BW20M;
+	}
+
+	/* Update packet information */
+	phydm_set_common_phy_info(rx_pwr_db, phy_sta_rpt->channel,
+				  (bool)phy_sta_rpt->beamformed, false, bw, 0,
+				  rxsc, phy_info);
+}
+
+static void
+phydm_process_rssi_for_dm_new_type(struct phy_dm_struct *dm,
+				   struct dm_phy_status_info *phy_info,
+				   struct dm_per_pkt_info *pktinfo)
+{
+	s32 undecorated_smoothed_pwdb, accumulate_pwdb;
+	u32 rssi_ave;
+	u8 i;
+	struct rtl_sta_info *entry;
+	u8 scaling_factor = 4;
+
+	if (pktinfo->station_id >= ODM_ASSOCIATE_ENTRY_NUM)
+		return;
+
+	entry = dm->odm_sta_info[pktinfo->station_id];
+
+	if (!IS_STA_VALID(entry))
+		return;
+
+	if ((!pktinfo->is_packet_match_bssid)) /*data frame only*/
+		return;
+
+	if (pktinfo->is_packet_beacon)
+		dm->phy_dbg_info.num_qry_beacon_pkt++;
+
+	if (pktinfo->is_packet_to_self || pktinfo->is_packet_beacon) {
+		u32 rssi_linear = 0;
+
+		dm->rx_rate = pktinfo->data_rate;
+		undecorated_smoothed_pwdb =
+			entry->rssi_stat.undecorated_smoothed_pwdb;
+		accumulate_pwdb = dm->accumulate_pwdb[pktinfo->station_id];
+		dm->rssi_a = phy_info->rx_mimo_signal_strength[ODM_RF_PATH_A];
+		dm->rssi_b = phy_info->rx_mimo_signal_strength[ODM_RF_PATH_B];
+		dm->rssi_c = phy_info->rx_mimo_signal_strength[ODM_RF_PATH_C];
+		dm->rssi_d = phy_info->rx_mimo_signal_strength[ODM_RF_PATH_D];
+
+		for (i = ODM_RF_PATH_A; i < ODM_RF_PATH_MAX_JAGUAR; i++) {
+			if (phy_info->rx_mimo_signal_strength[i] != 0)
+				rssi_linear += odm_convert_to_linear(
+					phy_info->rx_mimo_signal_strength[i]);
+		}
+
+		switch (phy_info->rx_count + 1) {
+		case 2:
+			rssi_linear = (rssi_linear >> 1);
+			break;
+		case 3:
+			/* rssi_linear/3 ~ rssi_linear*11/32 */
+			rssi_linear = ((rssi_linear) + (rssi_linear << 1) +
+				       (rssi_linear << 3)) >>
+				      5;
+			break;
+		case 4:
+			rssi_linear = (rssi_linear >> 2);
+			break;
+		}
+		rssi_ave = odm_convert_to_db(rssi_linear);
+
+		if (undecorated_smoothed_pwdb <= 0) {
+			accumulate_pwdb =
+				(phy_info->rx_pwdb_all << scaling_factor);
+			undecorated_smoothed_pwdb = phy_info->rx_pwdb_all;
+		} else {
+			accumulate_pwdb = accumulate_pwdb -
+					  (accumulate_pwdb >> scaling_factor) +
+					  rssi_ave;
+			undecorated_smoothed_pwdb =
+				(accumulate_pwdb +
+				 (1 << (scaling_factor - 1))) >>
+				scaling_factor;
+		}
+
+		entry->rssi_stat.undecorated_smoothed_pwdb =
+			undecorated_smoothed_pwdb;
+		dm->accumulate_pwdb[pktinfo->station_id] = accumulate_pwdb;
+	}
+}
+
+void phydm_rx_phy_status_new_type(struct phy_dm_struct *phydm, u8 *phy_status,
+				  struct dm_per_pkt_info *pktinfo,
+				  struct dm_phy_status_info *phy_info)
+{
+	u8 phy_status_type = (*phy_status & 0xf);
+
+	/* Memory reset */
+	phydm_reset_phy_info(phydm, phy_info);
+
+	/* Phy status parsing */
+	switch (phy_status_type) {
+	case 0: {
+		phydm_get_rx_phy_status_type0(phydm, phy_status, pktinfo,
+					      phy_info);
+		break;
+	}
+	case 1: {
+		phydm_get_rx_phy_status_type1(phydm, phy_status, pktinfo,
+					      phy_info);
+		break;
+	}
+	case 2: {
+		phydm_get_rx_phy_status_type2(phydm, phy_status, pktinfo,
+					      phy_info);
+		break;
+	}
+	default:
+		return;
+	}
+
+	/* Update signal strength to UI, and phy_info->rx_pwdb_all is the
+	 * maximum RSSI of all path
+	 */
+	phy_info->signal_strength =
+		(u8)(odm_signal_scale_mapping(phydm, phy_info->rx_pwdb_all));
+
+	/* Calculate average RSSI and smoothed RSSI */
+	phydm_process_rssi_for_dm_new_type(phydm, phy_info, pktinfo);
+}
+
+u32 query_phydm_trx_capability(struct phy_dm_struct *dm)
+{
+	u32 value32 = 0xFFFFFFFF;
+
+	return value32;
+}
+
+u32 query_phydm_stbc_capability(struct phy_dm_struct *dm)
+{
+	u32 value32 = 0xFFFFFFFF;
+
+	return value32;
+}
+
+u32 query_phydm_ldpc_capability(struct phy_dm_struct *dm)
+{
+	u32 value32 = 0xFFFFFFFF;
+
+	return value32;
+}
+
+u32 query_phydm_txbf_parameters(struct phy_dm_struct *dm)
+{
+	u32 value32 = 0xFFFFFFFF;
+
+	return value32;
+}
+
+u32 query_phydm_txbf_capability(struct phy_dm_struct *dm)
+{
+	u32 value32 = 0xFFFFFFFF;
+
+	return value32;
+}
diff --git a/drivers/staging/rtlwifi/phydm/phydm_hwconfig.h b/drivers/staging/rtlwifi/phydm/phydm_hwconfig.h
new file mode 100644
index 000000000000..ec94c61df2b9
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_hwconfig.h
@@ -0,0 +1,510 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __HALHWOUTSRC_H__
+#define __HALHWOUTSRC_H__
+
+/*--------------------------Define -------------------------------------------*/
+#define CCK_RSSI_INIT_COUNT 5
+
+#define RA_RSSI_STATE_INIT 0
+#define RA_RSSI_STATE_SEND 1
+#define RA_RSSI_STATE_HOLD 2
+
+#define CFO_HW_RPT_2_MHZ(val) ((val << 1) + (val >> 1))
+/* ((X* 3125)  / 10)>>7 = (X*10)>>2 = X*2.5 = X<<1 + X>>1  */
+
+#define AGC_DIFF_CONFIG_MP(ic, band)                                           \
+	(odm_read_and_config_mp_##ic##_agc_tab_diff(                           \
+		dm, array_mp_##ic##_agc_tab_diff_##band,                       \
+		sizeof(array_mp_##ic##_agc_tab_diff_##band) / sizeof(u32)))
+#define AGC_DIFF_CONFIG_TC(ic, band)                                           \
+	(odm_read_and_config_tc_##ic##_agc_tab_diff(                           \
+		dm, array_tc_##ic##_agc_tab_diff_##band,                       \
+		sizeof(array_tc_##ic##_agc_tab_diff_##band) / sizeof(u32)))
+
+#define AGC_DIFF_CONFIG(ic, band)                                              \
+	do {                                                                   \
+		if (dm->is_mp_chip)                                            \
+			AGC_DIFF_CONFIG_MP(ic, band);                          \
+		else                                                           \
+			AGC_DIFF_CONFIG_TC(ic, band);                          \
+	} while (0)
+
+/* ************************************************************
+ * structure and define
+ * *************************************************************/
+
+struct phy_rx_agc_info {
+#if (ODM_ENDIAN_TYPE == ODM_ENDIAN_LITTLE)
+	u8 gain : 7, trsw : 1;
+#else
+	u8 trsw : 1, gain : 7;
+#endif
+};
+
+struct phy_status_rpt_8192cd {
+	struct phy_rx_agc_info path_agc[2];
+	u8 ch_corr[2];
+	u8 cck_sig_qual_ofdm_pwdb_all;
+	u8 cck_agc_rpt_ofdm_cfosho_a;
+	u8 cck_rpt_b_ofdm_cfosho_b;
+	u8 rsvd_1; /*ch_corr_msb;*/
+	u8 noise_power_db_msb;
+	s8 path_cfotail[2];
+	u8 pcts_mask[2];
+	s8 stream_rxevm[2];
+	u8 path_rxsnr[2];
+	u8 noise_power_db_lsb;
+	u8 rsvd_2[3];
+	u8 stream_csi[2];
+	u8 stream_target_csi[2];
+	s8 sig_evm;
+	u8 rsvd_3;
+
+#if (ODM_ENDIAN_TYPE == ODM_ENDIAN_LITTLE)
+	u8 antsel_rx_keep_2 : 1; /*ex_intf_flg:1;*/
+	u8 sgi_en : 1;
+	u8 rxsc : 2;
+	u8 idle_long : 1;
+	u8 r_ant_train_en : 1;
+	u8 ant_sel_b : 1;
+	u8 ant_sel : 1;
+#else /*_BIG_ENDIAN_	*/
+	u8 ant_sel : 1;
+	u8 ant_sel_b : 1;
+	u8 r_ant_train_en : 1;
+	u8 idle_long : 1;
+	u8 rxsc : 2;
+	u8 sgi_en : 1;
+	u8 antsel_rx_keep_2 : 1; /*ex_intf_flg:1;*/
+#endif
+};
+
+struct phy_status_rpt_8812 {
+	/*	DWORD 0*/
+	u8 gain_trsw[2]; /*path-A and path-B {TRSW, gain[6:0] }*/
+	u8 chl_num_LSB; /*channel number[7:0]*/
+#if (ODM_ENDIAN_TYPE == ODM_ENDIAN_LITTLE)
+	u8 chl_num_MSB : 2; /*channel number[9:8]*/
+	u8 sub_chnl : 4; /*sub-channel location[3:0]*/
+	u8 r_RFMOD : 2; /*RF mode[1:0]*/
+#else /*_BIG_ENDIAN_	*/
+	u8 r_RFMOD : 2;
+	u8 sub_chnl : 4;
+	u8 chl_num_MSB : 2;
+#endif
+
+	/*	DWORD 1*/
+	u8 pwdb_all; /*CCK signal quality / OFDM pwdb all*/
+	s8 cfosho[2]; /*DW1 byte 1 DW1 byte2 */
+/*CCK AGC report and CCK_BB_Power / OFDM path-A and path-B short CFO*/
+#if (ODM_ENDIAN_TYPE == ODM_ENDIAN_LITTLE)
+	/*this should be checked again
+	 *because the definition of 8812 and 8814 is different
+	 */
+	u8 resvd_0 : 6;
+	u8 bt_RF_ch_MSB : 2; /*8812A:2'b0, 8814A: bt rf channel keep[7:6]*/
+#else /*_BIG_ENDIAN_*/
+	u8 bt_RF_ch_MSB : 2;
+	u8 resvd_0 : 6;
+#endif
+
+/*	DWORD 2*/
+#if (ODM_ENDIAN_TYPE == ODM_ENDIAN_LITTLE)
+	u8 ant_div_sw_a : 1; /*8812A: ant_div_sw_a, 8814A: 1'b0*/
+	u8 ant_div_sw_b : 1; /*8812A: ant_div_sw_b, 8814A: 1'b0*/
+	u8 bt_RF_ch_LSB : 6; /*8812A: 6'b0, 8814A: bt rf channel keep[5:0]*/
+#else /*_BIG_ENDIAN_	*/
+	u8 bt_RF_ch_LSB : 6;
+	u8 ant_div_sw_b : 1;
+	u8 ant_div_sw_a : 1;
+#endif
+	s8 cfotail[2]; /*DW2 byte 1 DW2 byte 2	path-A and path-B CFO tail*/
+	u8 PCTS_MSK_RPT_0; /*PCTS mask report[7:0]*/
+	u8 PCTS_MSK_RPT_1; /*PCTS mask report[15:8]*/
+
+	/*	DWORD 3*/
+	s8 rxevm[2]; /*DW3 byte 1 DW3 byte 2	stream 1 and stream 2 RX EVM*/
+	s8 rxsnr[2]; /*DW3 byte 3 DW4 byte 0	path-A and path-B RX SNR*/
+
+	/*	DWORD 4*/
+	u8 PCTS_MSK_RPT_2; /*PCTS mask report[23:16]*/
+#if (ODM_ENDIAN_TYPE == ODM_ENDIAN_LITTLE)
+	u8 PCTS_MSK_RPT_3 : 6; /*PCTS mask report[29:24]*/
+	u8 pcts_rpt_valid : 1; /*pcts_rpt_valid*/
+	u8 resvd_1 : 1; /*1'b0*/
+#else /*_BIG_ENDIAN_*/
+	u8 resvd_1 : 1;
+	u8 pcts_rpt_valid : 1;
+	u8 PCTS_MSK_RPT_3 : 6;
+#endif
+	s8 rxevm_cd[2]; /*DW 4 byte 3 DW5 byte 0 */
+	/* 8812A: 16'b0, 8814A: stream 3 and stream 4 RX EVM*/
+
+	/*	DWORD 5*/
+	u8 csi_current[2]; /*DW5 byte 1 DW5 byte 2 */
+	/* 8812A: stream 1 and 2 CSI, 8814A: path-C and path-D RX SNR*/
+	u8 gain_trsw_cd[2]; /*DW5 byte 3 DW6 byte 0 */
+	/* path-C and path-D {TRSW, gain[6:0] }*/
+
+	/*	DWORD 6*/
+	s8 sigevm; /*signal field EVM*/
+#if (ODM_ENDIAN_TYPE == ODM_ENDIAN_LITTLE)
+	u8 antidx_antc : 3; /*8812A: 3'b0	8814A: antidx_antc[2:0]*/
+	u8 antidx_antd : 3; /*8812A: 3'b0	8814A: antidx_antd[2:0]*/
+	u8 dpdt_ctrl_keep : 1; /*8812A: 1'b0	8814A: dpdt_ctrl_keep*/
+	u8 GNT_BT_keep : 1; /*8812A: 1'b0	8814A: GNT_BT_keep*/
+#else /*_BIG_ENDIAN_*/
+	u8 GNT_BT_keep : 1;
+	u8 dpdt_ctrl_keep : 1;
+	u8 antidx_antd : 3;
+	u8 antidx_antc : 3;
+#endif
+#if (ODM_ENDIAN_TYPE == ODM_ENDIAN_LITTLE)
+	u8 antidx_anta : 3; /*antidx_anta[2:0]*/
+	u8 antidx_antb : 3; /*antidx_antb[2:0]*/
+	u8 hw_antsw_occur : 2; /*1'b0*/
+#else /*_BIG_ENDIAN_*/
+	u8 hw_antsw_occur : 2;
+	u8 antidx_antb : 3;
+	u8 antidx_anta : 3;
+#endif
+};
+
+void phydm_reset_rssi_for_dm(struct phy_dm_struct *dm, u8 station_id);
+
+void odm_init_rssi_for_dm(struct phy_dm_struct *dm);
+
+void odm_phy_status_query(struct phy_dm_struct *dm,
+			  struct dm_phy_status_info *phy_info, u8 *phy_status,
+			  struct dm_per_pkt_info *pktinfo);
+
+void odm_mac_status_query(struct phy_dm_struct *dm, u8 *mac_status, u8 mac_id,
+			  bool is_packet_match_bssid, bool is_packet_to_self,
+			  bool is_packet_beacon);
+
+enum hal_status
+odm_config_rf_with_tx_pwr_track_header_file(struct phy_dm_struct *dm);
+
+enum hal_status
+odm_config_rf_with_header_file(struct phy_dm_struct *dm,
+			       enum odm_rf_config_type config_type,
+			       enum odm_rf_radio_path e_rf_path);
+
+enum hal_status
+odm_config_bb_with_header_file(struct phy_dm_struct *dm,
+			       enum odm_bb_config_type config_type);
+
+enum hal_status odm_config_mac_with_header_file(struct phy_dm_struct *dm);
+
+enum hal_status
+odm_config_fw_with_header_file(struct phy_dm_struct *dm,
+			       enum odm_fw_config_type config_type,
+			       u8 *p_firmware, u32 *size);
+
+u32 odm_get_hw_img_version(struct phy_dm_struct *dm);
+
+s32 odm_signal_scale_mapping(struct phy_dm_struct *dm, s32 curr_sig);
+
+/*For 8822B only!! need to move to FW finally */
+/*==============================================*/
+void phydm_rx_phy_status_new_type(struct phy_dm_struct *phydm, u8 *phy_status,
+				  struct dm_per_pkt_info *pktinfo,
+				  struct dm_phy_status_info *phy_info);
+
+bool phydm_query_is_mu_api(struct phy_dm_struct *phydm, u8 ppdu_idx,
+			   u8 *p_data_rate, u8 *p_gid);
+
+struct phy_status_rpt_jaguar2_type0 {
+	/* DW0 */
+	u8 page_num;
+	u8 pwdb;
+#if (ODM_ENDIAN_TYPE == ODM_ENDIAN_LITTLE)
+	u8 gain : 6;
+	u8 rsvd_0 : 1;
+	u8 trsw : 1;
+#else
+	u8 trsw : 1;
+	u8 rsvd_0 : 1;
+	u8 gain : 6;
+#endif
+	u8 rsvd_1;
+
+	/* DW1 */
+	u8 rsvd_2;
+#if (ODM_ENDIAN_TYPE == ODM_ENDIAN_LITTLE)
+	u8 rxsc : 4;
+	u8 agc_table : 4;
+#else
+	u8 agc_table : 4;
+	u8 rxsc : 4;
+#endif
+	u8 channel;
+	u8 band;
+
+	/* DW2 */
+	u16 length;
+#if (ODM_ENDIAN_TYPE == ODM_ENDIAN_LITTLE)
+	u8 antidx_a : 3;
+	u8 antidx_b : 3;
+	u8 rsvd_3 : 2;
+	u8 antidx_c : 3;
+	u8 antidx_d : 3;
+	u8 rsvd_4 : 2;
+#else
+	u8 rsvd_3 : 2;
+	u8 antidx_b : 3;
+	u8 antidx_a : 3;
+	u8 rsvd_4 : 2;
+	u8 antidx_d : 3;
+	u8 antidx_c : 3;
+#endif
+
+	/* DW3 */
+	u8 signal_quality;
+#if (ODM_ENDIAN_TYPE == ODM_ENDIAN_LITTLE)
+	u8 vga : 5;
+	u8 lna_l : 3;
+	u8 bb_power : 6;
+	u8 rsvd_9 : 1;
+	u8 lna_h : 1;
+#else
+	u8 lna_l : 3;
+	u8 vga : 5;
+	u8 lna_h : 1;
+	u8 rsvd_9 : 1;
+	u8 bb_power : 6;
+#endif
+	u8 rsvd_5;
+
+	/* DW4 */
+	u32 rsvd_6;
+
+	/* DW5 */
+	u32 rsvd_7;
+
+	/* DW6 */
+	u32 rsvd_8;
+};
+
+struct phy_status_rpt_jaguar2_type1 {
+	/* DW0 and DW1 */
+	u8 page_num;
+	u8 pwdb[4];
+#if (ODM_ENDIAN_TYPE == ODM_ENDIAN_LITTLE)
+	u8 l_rxsc : 4;
+	u8 ht_rxsc : 4;
+#else
+	u8 ht_rxsc : 4;
+	u8 l_rxsc : 4;
+#endif
+	u8 channel;
+#if (ODM_ENDIAN_TYPE == ODM_ENDIAN_LITTLE)
+	u8 band : 2;
+	u8 rsvd_0 : 1;
+	u8 hw_antsw_occu : 1;
+	u8 gnt_bt : 1;
+	u8 ldpc : 1;
+	u8 stbc : 1;
+	u8 beamformed : 1;
+#else
+	u8 beamformed : 1;
+	u8 stbc : 1;
+	u8 ldpc : 1;
+	u8 gnt_bt : 1;
+	u8 hw_antsw_occu : 1;
+	u8 rsvd_0 : 1;
+	u8 band : 2;
+#endif
+
+	/* DW2 */
+	u16 lsig_length;
+#if (ODM_ENDIAN_TYPE == ODM_ENDIAN_LITTLE)
+	u8 antidx_a : 3;
+	u8 antidx_b : 3;
+	u8 rsvd_1 : 2;
+	u8 antidx_c : 3;
+	u8 antidx_d : 3;
+	u8 rsvd_2 : 2;
+#else
+	u8 rsvd_1 : 2;
+	u8 antidx_b : 3;
+	u8 antidx_a : 3;
+	u8 rsvd_2 : 2;
+	u8 antidx_d : 3;
+	u8 antidx_c : 3;
+#endif
+
+	/* DW3 */
+	u8 paid;
+#if (ODM_ENDIAN_TYPE == ODM_ENDIAN_LITTLE)
+	u8 paid_msb : 1;
+	u8 gid : 6;
+	u8 rsvd_3 : 1;
+#else
+	u8 rsvd_3 : 1;
+	u8 gid : 6;
+	u8 paid_msb : 1;
+#endif
+	u8 intf_pos;
+#if (ODM_ENDIAN_TYPE == ODM_ENDIAN_LITTLE)
+	u8 intf_pos_msb : 1;
+	u8 rsvd_4 : 2;
+	u8 nb_intf_flag : 1;
+	u8 rf_mode : 2;
+	u8 rsvd_5 : 2;
+#else
+	u8 rsvd_5 : 2;
+	u8 rf_mode : 2;
+	u8 nb_intf_flag : 1;
+	u8 rsvd_4 : 2;
+	u8 intf_pos_msb : 1;
+#endif
+
+	/* DW4 */
+	s8 rxevm[4]; /* s(8,1) */
+
+	/* DW5 */
+	s8 cfo_tail[4]; /* s(8,7) */
+
+	/* DW6 */
+	s8 rxsnr[4]; /* s(8,1) */
+};
+
+struct phy_status_rpt_jaguar2_type2 {
+	/* DW0 ane DW1 */
+	u8 page_num;
+	u8 pwdb[4];
+#if (ODM_ENDIAN_TYPE == ODM_ENDIAN_LITTLE)
+	u8 l_rxsc : 4;
+	u8 ht_rxsc : 4;
+#else
+	u8 ht_rxsc : 4;
+	u8 l_rxsc : 4;
+#endif
+	u8 channel;
+#if (ODM_ENDIAN_TYPE == ODM_ENDIAN_LITTLE)
+	u8 band : 2;
+	u8 rsvd_0 : 1;
+	u8 hw_antsw_occu : 1;
+	u8 gnt_bt : 1;
+	u8 ldpc : 1;
+	u8 stbc : 1;
+	u8 beamformed : 1;
+#else
+	u8 beamformed : 1;
+	u8 stbc : 1;
+	u8 ldpc : 1;
+	u8 gnt_bt : 1;
+	u8 hw_antsw_occu : 1;
+	u8 rsvd_0 : 1;
+	u8 band : 2;
+#endif
+
+/* DW2 */
+#if (ODM_ENDIAN_TYPE == ODM_ENDIAN_LITTLE)
+	u8 shift_l_map : 6;
+	u8 rsvd_1 : 2;
+#else
+	u8 rsvd_1 : 2;
+	u8 shift_l_map : 6;
+#endif
+	u8 cnt_pw2cca;
+#if (ODM_ENDIAN_TYPE == ODM_ENDIAN_LITTLE)
+	u8 agc_table_a : 4;
+	u8 agc_table_b : 4;
+	u8 agc_table_c : 4;
+	u8 agc_table_d : 4;
+#else
+	u8 agc_table_b : 4;
+	u8 agc_table_a : 4;
+	u8 agc_table_d : 4;
+	u8 agc_table_c : 4;
+#endif
+
+	/* DW3 ~ DW6*/
+	u8 cnt_cca2agc_rdy;
+#if (ODM_ENDIAN_TYPE == ODM_ENDIAN_LITTLE)
+	u8 gain_a : 6;
+	u8 rsvd_2 : 1;
+	u8 trsw_a : 1;
+	u8 gain_b : 6;
+	u8 rsvd_3 : 1;
+	u8 trsw_b : 1;
+	u8 gain_c : 6;
+	u8 rsvd_4 : 1;
+	u8 trsw_c : 1;
+	u8 gain_d : 6;
+	u8 rsvd_5 : 1;
+	u8 trsw_d : 1;
+	u8 aagc_step_a : 2;
+	u8 aagc_step_b : 2;
+	u8 aagc_step_c : 2;
+	u8 aagc_step_d : 2;
+#else
+	u8 trsw_a : 1;
+	u8 rsvd_2 : 1;
+	u8 gain_a : 6;
+	u8 trsw_b : 1;
+	u8 rsvd_3 : 1;
+	u8 gain_b : 6;
+	u8 trsw_c : 1;
+	u8 rsvd_4 : 1;
+	u8 gain_c : 6;
+	u8 trsw_d : 1;
+	u8 rsvd_5 : 1;
+	u8 gain_d : 6;
+	u8 aagc_step_d : 2;
+	u8 aagc_step_c : 2;
+	u8 aagc_step_b : 2;
+	u8 aagc_step_a : 2;
+#endif
+	u8 ht_aagc_gain[4];
+	u8 dagc_gain[4];
+#if (ODM_ENDIAN_TYPE == ODM_ENDIAN_LITTLE)
+	u8 counter : 6;
+	u8 rsvd_6 : 2;
+	u8 syn_count : 5;
+	u8 rsvd_7 : 3;
+#else
+	u8 rsvd_6 : 2;
+	u8 counter : 6;
+	u8 rsvd_7 : 3;
+	u8 syn_count : 5;
+#endif
+};
+
+u32 query_phydm_trx_capability(struct phy_dm_struct *dm);
+
+u32 query_phydm_stbc_capability(struct phy_dm_struct *dm);
+
+u32 query_phydm_ldpc_capability(struct phy_dm_struct *dm);
+
+u32 query_phydm_txbf_parameters(struct phy_dm_struct *dm);
+
+u32 query_phydm_txbf_capability(struct phy_dm_struct *dm);
+
+#endif /*#ifndef	__HALHWOUTSRC_H__*/
diff --git a/drivers/staging/rtlwifi/phydm/phydm_interface.c b/drivers/staging/rtlwifi/phydm/phydm_interface.c
new file mode 100644
index 000000000000..102576a46c04
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_interface.c
@@ -0,0 +1,341 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+/* ************************************************************
+ * include files
+ * *************************************************************/
+
+#include "mp_precomp.h"
+#include "phydm_precomp.h"
+
+/*
+ * ODM IO Relative API.
+ */
+
+u8 odm_read_1byte(struct phy_dm_struct *dm, u32 reg_addr)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)dm->adapter;
+
+	return rtl_read_byte(rtlpriv, reg_addr);
+}
+
+u16 odm_read_2byte(struct phy_dm_struct *dm, u32 reg_addr)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)dm->adapter;
+
+	return rtl_read_word(rtlpriv, reg_addr);
+}
+
+u32 odm_read_4byte(struct phy_dm_struct *dm, u32 reg_addr)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)dm->adapter;
+
+	return rtl_read_dword(rtlpriv, reg_addr);
+}
+
+void odm_write_1byte(struct phy_dm_struct *dm, u32 reg_addr, u8 data)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)dm->adapter;
+
+	rtl_write_byte(rtlpriv, reg_addr, data);
+}
+
+void odm_write_2byte(struct phy_dm_struct *dm, u32 reg_addr, u16 data)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)dm->adapter;
+
+	rtl_write_word(rtlpriv, reg_addr, data);
+}
+
+void odm_write_4byte(struct phy_dm_struct *dm, u32 reg_addr, u32 data)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)dm->adapter;
+
+	rtl_write_dword(rtlpriv, reg_addr, data);
+}
+
+void odm_set_mac_reg(struct phy_dm_struct *dm, u32 reg_addr, u32 bit_mask,
+		     u32 data)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)dm->adapter;
+
+	rtl_set_bbreg(rtlpriv->hw, reg_addr, bit_mask, data);
+}
+
+u32 odm_get_mac_reg(struct phy_dm_struct *dm, u32 reg_addr, u32 bit_mask)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)dm->adapter;
+
+	return rtl_get_bbreg(rtlpriv->hw, reg_addr, bit_mask);
+}
+
+void odm_set_bb_reg(struct phy_dm_struct *dm, u32 reg_addr, u32 bit_mask,
+		    u32 data)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)dm->adapter;
+
+	rtl_set_bbreg(rtlpriv->hw, reg_addr, bit_mask, data);
+}
+
+u32 odm_get_bb_reg(struct phy_dm_struct *dm, u32 reg_addr, u32 bit_mask)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)dm->adapter;
+
+	return rtl_get_bbreg(rtlpriv->hw, reg_addr, bit_mask);
+}
+
+void odm_set_rf_reg(struct phy_dm_struct *dm, enum odm_rf_radio_path e_rf_path,
+		    u32 reg_addr, u32 bit_mask, u32 data)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)dm->adapter;
+
+	rtl_set_rfreg(rtlpriv->hw, (enum radio_path)e_rf_path, reg_addr,
+		      bit_mask, data);
+}
+
+u32 odm_get_rf_reg(struct phy_dm_struct *dm, enum odm_rf_radio_path e_rf_path,
+		   u32 reg_addr, u32 bit_mask)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)dm->adapter;
+
+	return rtl_get_rfreg(rtlpriv->hw, (enum radio_path)e_rf_path, reg_addr,
+			     bit_mask);
+}
+
+/*
+ * ODM Memory relative API.
+ */
+void odm_allocate_memory(struct phy_dm_struct *dm, void **ptr, u32 length)
+{
+	*ptr = kmalloc(length, GFP_ATOMIC);
+}
+
+/* length could be ignored, used to detect memory leakage. */
+void odm_free_memory(struct phy_dm_struct *dm, void *ptr, u32 length)
+{
+	kfree(ptr);
+}
+
+void odm_move_memory(struct phy_dm_struct *dm, void *p_dest, void *src,
+		     u32 length)
+{
+	memcpy(p_dest, src, length);
+}
+
+void odm_memory_set(struct phy_dm_struct *dm, void *pbuf, s8 value, u32 length)
+{
+	memset(pbuf, value, length);
+}
+
+s32 odm_compare_memory(struct phy_dm_struct *dm, void *p_buf1, void *buf2,
+		       u32 length)
+{
+	return memcmp(p_buf1, buf2, length);
+}
+
+/*
+ * ODM MISC relative API.
+ */
+void odm_acquire_spin_lock(struct phy_dm_struct *dm, enum rt_spinlock_type type)
+{
+}
+
+void odm_release_spin_lock(struct phy_dm_struct *dm, enum rt_spinlock_type type)
+{
+}
+
+/*
+ * ODM Timer relative API.
+ */
+void odm_stall_execution(u32 us_delay) { udelay(us_delay); }
+
+void ODM_delay_ms(u32 ms) { mdelay(ms); }
+
+void ODM_delay_us(u32 us) { udelay(us); }
+
+void ODM_sleep_ms(u32 ms) { msleep(ms); }
+
+void ODM_sleep_us(u32 us) { usleep_range(us, us + 1); }
+
+void odm_set_timer(struct phy_dm_struct *dm, struct timer_list *timer,
+		   u32 ms_delay)
+{
+	mod_timer(timer, jiffies + msecs_to_jiffies(ms_delay));
+}
+
+void odm_initialize_timer(struct phy_dm_struct *dm, struct timer_list *timer,
+			  void *call_back_func, void *context,
+			  const char *sz_id)
+{
+	init_timer(timer);
+	timer->function = call_back_func;
+	timer->data = (unsigned long)dm;
+	/*mod_timer(timer, jiffies+RTL_MILISECONDS_TO_JIFFIES(10));	*/
+}
+
+void odm_cancel_timer(struct phy_dm_struct *dm, struct timer_list *timer)
+{
+	del_timer(timer);
+}
+
+void odm_release_timer(struct phy_dm_struct *dm, struct timer_list *timer) {}
+
+static u8 phydm_trans_h2c_id(struct phy_dm_struct *dm, u8 phydm_h2c_id)
+{
+	u8 platform_h2c_id = phydm_h2c_id;
+
+	switch (phydm_h2c_id) {
+	/* 1 [0] */
+	case ODM_H2C_RSSI_REPORT:
+
+		break;
+
+	/* 1 [3] */
+	case ODM_H2C_WIFI_CALIBRATION:
+
+		break;
+
+	/* 1 [4] */
+	case ODM_H2C_IQ_CALIBRATION:
+
+		break;
+	/* 1 [5] */
+	case ODM_H2C_RA_PARA_ADJUST:
+
+		break;
+
+	/* 1 [6] */
+	case PHYDM_H2C_DYNAMIC_TX_PATH:
+
+		break;
+
+	/* [7]*/
+	case PHYDM_H2C_FW_TRACE_EN:
+
+		platform_h2c_id = 0x49;
+
+		break;
+
+	case PHYDM_H2C_TXBF:
+		break;
+
+	case PHYDM_H2C_MU:
+		platform_h2c_id = 0x4a; /*H2C_MU*/
+		break;
+
+	default:
+		platform_h2c_id = phydm_h2c_id;
+		break;
+	}
+
+	return platform_h2c_id;
+}
+
+/*ODM FW relative API.*/
+
+void odm_fill_h2c_cmd(struct phy_dm_struct *dm, u8 phydm_h2c_id, u32 cmd_len,
+		      u8 *cmd_buffer)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)dm->adapter;
+	u8 platform_h2c_id;
+
+	platform_h2c_id = phydm_trans_h2c_id(dm, phydm_h2c_id);
+
+	ODM_RT_TRACE(dm, PHYDM_COMP_RA_DBG,
+		     "[H2C]  platform_h2c_id = ((0x%x))\n", platform_h2c_id);
+
+	rtlpriv->cfg->ops->fill_h2c_cmd(rtlpriv->hw, platform_h2c_id, cmd_len,
+					cmd_buffer);
+}
+
+u8 phydm_c2H_content_parsing(void *dm_void, u8 c2h_cmd_id, u8 c2h_cmd_len,
+			     u8 *tmp_buf)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u8 extend_c2h_sub_id = 0;
+	u8 find_c2h_cmd = true;
+
+	switch (c2h_cmd_id) {
+	case PHYDM_C2H_DBG:
+		phydm_fw_trace_handler(dm, tmp_buf, c2h_cmd_len);
+		break;
+
+	case PHYDM_C2H_RA_RPT:
+		phydm_c2h_ra_report_handler(dm, tmp_buf, c2h_cmd_len);
+		break;
+
+	case PHYDM_C2H_RA_PARA_RPT:
+		odm_c2h_ra_para_report_handler(dm, tmp_buf, c2h_cmd_len);
+		break;
+
+	case PHYDM_C2H_DYNAMIC_TX_PATH_RPT:
+		break;
+
+	case PHYDM_C2H_IQK_FINISH:
+		break;
+
+	case PHYDM_C2H_DBG_CODE:
+		phydm_fw_trace_handler_code(dm, tmp_buf, c2h_cmd_len);
+		break;
+
+	case PHYDM_C2H_EXTEND:
+		extend_c2h_sub_id = tmp_buf[0];
+		if (extend_c2h_sub_id == PHYDM_EXTEND_C2H_DBG_PRINT)
+			phydm_fw_trace_handler_8051(dm, tmp_buf, c2h_cmd_len);
+
+		break;
+
+	default:
+		find_c2h_cmd = false;
+		break;
+	}
+
+	return find_c2h_cmd;
+}
+
+u64 odm_get_current_time(struct phy_dm_struct *dm) { return jiffies; }
+
+u64 odm_get_progressing_time(struct phy_dm_struct *dm, u64 start_time)
+{
+	return jiffies_to_msecs(jiffies - (u32)start_time);
+}
+
+void odm_set_tx_power_index_by_rate_section(struct phy_dm_struct *dm,
+					    u8 rf_path, u8 channel,
+					    u8 rate_section)
+{
+	void *adapter = dm->adapter;
+
+	phy_set_tx_power_index_by_rs(adapter, channel, rf_path, rate_section);
+}
+
+u8 odm_get_tx_power_index(struct phy_dm_struct *dm, u8 rf_path, u8 tx_rate,
+			  u8 band_width, u8 channel)
+{
+	void *adapter = dm->adapter;
+
+	return phy_get_tx_power_index(adapter, (enum odm_rf_radio_path)rf_path,
+				      tx_rate, band_width, channel);
+}
diff --git a/drivers/staging/rtlwifi/phydm/phydm_interface.h b/drivers/staging/rtlwifi/phydm/phydm_interface.h
new file mode 100644
index 000000000000..d315c79c962a
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_interface.h
@@ -0,0 +1,205 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __ODM_INTERFACE_H__
+#define __ODM_INTERFACE_H__
+
+#define INTERFACE_VERSION "1.1" /*2015.07.29  YuChen*/
+
+/*
+ * =========== Constant/Structure/Enum/... Define
+ */
+
+/*
+ * =========== Macro Define
+ */
+
+#define _reg_all(_name) ODM_##_name
+#define _reg_ic(_name, _ic) ODM_##_name##_ic
+#define _bit_all(_name) BIT_##_name
+#define _bit_ic(_name, _ic) BIT_##_name##_ic
+
+/* _cat: implemented by Token-Pasting Operator. */
+
+/*===================================
+ *
+ * #define ODM_REG_DIG_11N	0xC50
+ * #define ODM_REG_DIG_11AC	0xDDD
+ *
+ * ODM_REG(DIG,_pdm_odm)
+ * ===================================
+ */
+
+#define _reg_11N(_name) ODM_REG_##_name##_11N
+#define _reg_11AC(_name) ODM_REG_##_name##_11AC
+#define _bit_11N(_name) ODM_BIT_##_name##_11N
+#define _bit_11AC(_name) ODM_BIT_##_name##_11AC
+
+#define _cat(_name, _ic_type, _func)                                           \
+	(((_ic_type) & ODM_IC_11N_SERIES) ? _func##_11N(_name) :               \
+					    _func##_11AC(_name))
+
+/* _name: name of register or bit.
+ * Example: "ODM_REG(R_A_AGC_CORE1, dm)"
+ * gets "ODM_R_A_AGC_CORE1" or "ODM_R_A_AGC_CORE1_8192C",
+ * depends on support_ic_type.
+ */
+#define ODM_REG(_name, _pdm_odm) _cat(_name, _pdm_odm->support_ic_type, _reg)
+#define ODM_BIT(_name, _pdm_odm) _cat(_name, _pdm_odm->support_ic_type, _bit)
+enum phydm_h2c_cmd {
+	PHYDM_H2C_TXBF = 0x41,
+	ODM_H2C_RSSI_REPORT = 0x42,
+	ODM_H2C_IQ_CALIBRATION = 0x45,
+	ODM_H2C_RA_PARA_ADJUST = 0x47,
+	PHYDM_H2C_DYNAMIC_TX_PATH = 0x48,
+	PHYDM_H2C_FW_TRACE_EN = 0x49,
+	ODM_H2C_WIFI_CALIBRATION = 0x6d,
+	PHYDM_H2C_MU = 0x4a,
+	ODM_MAX_H2CCMD
+};
+
+enum phydm_c2h_evt {
+	PHYDM_C2H_DBG = 0,
+	PHYDM_C2H_LB = 1,
+	PHYDM_C2H_XBF = 2,
+	PHYDM_C2H_TX_REPORT = 3,
+	PHYDM_C2H_INFO = 9,
+	PHYDM_C2H_BT_MP = 11,
+	PHYDM_C2H_RA_RPT = 12,
+	PHYDM_C2H_RA_PARA_RPT = 14,
+	PHYDM_C2H_DYNAMIC_TX_PATH_RPT = 15,
+	PHYDM_C2H_IQK_FINISH = 17, /*0x11*/
+	PHYDM_C2H_DBG_CODE = 0xFE,
+	PHYDM_C2H_EXTEND = 0xFF,
+};
+
+enum phydm_extend_c2h_evt {
+	PHYDM_EXTEND_C2H_DBG_PRINT = 0
+
+};
+
+/*
+ * =========== Extern Variable ??? It should be forbidden.
+ */
+
+/*
+ * =========== EXtern Function Prototype
+ */
+
+u8 odm_read_1byte(struct phy_dm_struct *dm, u32 reg_addr);
+
+u16 odm_read_2byte(struct phy_dm_struct *dm, u32 reg_addr);
+
+u32 odm_read_4byte(struct phy_dm_struct *dm, u32 reg_addr);
+
+void odm_write_1byte(struct phy_dm_struct *dm, u32 reg_addr, u8 data);
+
+void odm_write_2byte(struct phy_dm_struct *dm, u32 reg_addr, u16 data);
+
+void odm_write_4byte(struct phy_dm_struct *dm, u32 reg_addr, u32 data);
+
+void odm_set_mac_reg(struct phy_dm_struct *dm, u32 reg_addr, u32 bit_mask,
+		     u32 data);
+
+u32 odm_get_mac_reg(struct phy_dm_struct *dm, u32 reg_addr, u32 bit_mask);
+
+void odm_set_bb_reg(struct phy_dm_struct *dm, u32 reg_addr, u32 bit_mask,
+		    u32 data);
+
+u32 odm_get_bb_reg(struct phy_dm_struct *dm, u32 reg_addr, u32 bit_mask);
+
+void odm_set_rf_reg(struct phy_dm_struct *dm, enum odm_rf_radio_path e_rf_path,
+		    u32 reg_addr, u32 bit_mask, u32 data);
+
+u32 odm_get_rf_reg(struct phy_dm_struct *dm, enum odm_rf_radio_path e_rf_path,
+		   u32 reg_addr, u32 bit_mask);
+
+/*
+ * Memory Relative Function.
+ */
+void odm_allocate_memory(struct phy_dm_struct *dm, void **ptr, u32 length);
+void odm_free_memory(struct phy_dm_struct *dm, void *ptr, u32 length);
+
+void odm_move_memory(struct phy_dm_struct *dm, void *p_dest, void *src,
+		     u32 length);
+
+s32 odm_compare_memory(struct phy_dm_struct *dm, void *p_buf1, void *buf2,
+		       u32 length);
+
+void odm_memory_set(struct phy_dm_struct *dm, void *pbuf, s8 value, u32 length);
+
+/*
+ * ODM MISC-spin lock relative API.
+ */
+void odm_acquire_spin_lock(struct phy_dm_struct *dm,
+			   enum rt_spinlock_type type);
+
+void odm_release_spin_lock(struct phy_dm_struct *dm,
+			   enum rt_spinlock_type type);
+
+/*
+ * ODM Timer relative API.
+ */
+void odm_stall_execution(u32 us_delay);
+
+void ODM_delay_ms(u32 ms);
+
+void ODM_delay_us(u32 us);
+
+void ODM_sleep_ms(u32 ms);
+
+void ODM_sleep_us(u32 us);
+
+void odm_set_timer(struct phy_dm_struct *dm, struct timer_list *timer,
+		   u32 ms_delay);
+
+void odm_initialize_timer(struct phy_dm_struct *dm, struct timer_list *timer,
+			  void *call_back_func, void *context,
+			  const char *sz_id);
+
+void odm_cancel_timer(struct phy_dm_struct *dm, struct timer_list *timer);
+
+void odm_release_timer(struct phy_dm_struct *dm, struct timer_list *timer);
+
+/*
+ * ODM FW relative API.
+ */
+void odm_fill_h2c_cmd(struct phy_dm_struct *dm, u8 element_id, u32 cmd_len,
+		      u8 *cmd_buffer);
+
+u8 phydm_c2H_content_parsing(void *dm_void, u8 c2h_cmd_id, u8 c2h_cmd_len,
+			     u8 *tmp_buf);
+
+u64 odm_get_current_time(struct phy_dm_struct *dm);
+u64 odm_get_progressing_time(struct phy_dm_struct *dm, u64 start_time);
+
+void odm_set_tx_power_index_by_rate_section(struct phy_dm_struct *dm,
+					    u8 rf_path, u8 channel,
+					    u8 rate_section);
+
+u8 odm_get_tx_power_index(struct phy_dm_struct *dm, u8 rf_path, u8 tx_rate,
+			  u8 band_width, u8 channel);
+
+#endif /* __ODM_INTERFACE_H__ */
diff --git a/drivers/staging/rtlwifi/phydm/phydm_iqk.h b/drivers/staging/rtlwifi/phydm/phydm_iqk.h
new file mode 100644
index 000000000000..0d45bf099aeb
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_iqk.h
@@ -0,0 +1,76 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __PHYDMIQK_H__
+#define __PHYDMIQK_H__
+
+/*--------------------------Define Parameters-------------------------------*/
+#define LOK_delay 1
+#define WBIQK_delay 10
+#define TX_IQK 0
+#define RX_IQK 1
+#define TXIQK 0
+#define RXIQK1 1
+#define RXIQK2 2
+#define GSRXK1 0
+#define GSRXK2 1
+#define kcount_limit_80m 2
+#define kcount_limit_others 4
+#define rxiqk_gs_limit 4
+
+#define NUM 4
+/*----------------------End Define Parameters-------------------------------*/
+
+struct dm_iqk_info {
+	bool lok_fail[NUM];
+	bool iqk_fail[2][NUM];
+	u32 iqc_matrix[2][NUM];
+	u8 iqk_times;
+	u32 rf_reg18;
+	u32 lna_idx;
+	u8 rxiqk_step;
+	u8 tmp1bcc;
+	u8 kcount;
+
+	u32 iqk_channel[2];
+	bool iqk_fail_report[2][4][2]; /*channel/path/TRX(TX:0, RX:1) */
+	u32 iqk_cfir_real[2][4][2]
+			 [8]; /*channel / path / TRX(TX:0, RX:1) / CFIR_real*/
+	u32 iqk_cfir_imag[2][4][2]
+			 [8]; /*channel / path / TRX(TX:0, RX:1) / CFIR_imag*/
+	u8 retry_count[2][4][3]; /* channel / path / (TXK:0, RXK1:1, RXK2:2) */
+	u8 gs_retry_count[2][4][2]; /* channel / path / (GSRXK1:0, GSRXK2:1) */
+	u8 rxiqk_fail_code[2][4]; /* channel / path
+				   * 0:SRXK1 fail, 1:RXK1 fail 2:RXK2 fail
+				   */
+	u32 lok_idac[2][4]; /*channel / path*/
+	u16 rxiqk_agc[2][4]; /*channel / path*/
+	u32 bypass_iqk[2][4]; /*channel / 0xc94/0xe94*/
+	u32 tmp_gntwl;
+	bool is_btg;
+	bool isbnd;
+};
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/phydm_kfree.c b/drivers/staging/rtlwifi/phydm/phydm_kfree.c
new file mode 100644
index 000000000000..5f3582341806
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_kfree.c
@@ -0,0 +1,228 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+/*============================================================*/
+/*include files*/
+/*============================================================*/
+#include "mp_precomp.h"
+#include "phydm_precomp.h"
+
+/*<YuChen, 150720> Add for KFree Feature Requested by RF David.*/
+/*This is a phydm API*/
+
+static void phydm_set_kfree_to_rf_8814a(void *dm_void, u8 e_rf_path, u8 data)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct dm_rf_calibration_struct *cali_info = &dm->rf_calibrate_info;
+	bool is_odd;
+
+	if ((data % 2) != 0) { /*odd->positive*/
+		data = data - 1;
+		odm_set_rf_reg(dm, e_rf_path, REG_RF_TX_GAIN_OFFSET, BIT(19),
+			       1);
+		is_odd = true;
+	} else { /*even->negative*/
+		odm_set_rf_reg(dm, e_rf_path, REG_RF_TX_GAIN_OFFSET, BIT(19),
+			       0);
+		is_odd = false;
+	}
+	ODM_RT_TRACE(dm, ODM_COMP_MP, "%s(): RF_0x55[19]= %d\n", __func__,
+		     is_odd);
+	switch (data) {
+	case 0:
+		odm_set_rf_reg(dm, e_rf_path, REG_RF_TX_GAIN_OFFSET, BIT(14),
+			       0);
+		odm_set_rf_reg(dm, e_rf_path, REG_RF_TX_GAIN_OFFSET,
+			       BIT(17) | BIT(16) | BIT(15), 0);
+		cali_info->kfree_offset[e_rf_path] = 0;
+		break;
+	case 2:
+		odm_set_rf_reg(dm, e_rf_path, REG_RF_TX_GAIN_OFFSET, BIT(14),
+			       1);
+		odm_set_rf_reg(dm, e_rf_path, REG_RF_TX_GAIN_OFFSET,
+			       BIT(17) | BIT(16) | BIT(15), 0);
+		cali_info->kfree_offset[e_rf_path] = 0;
+		break;
+	case 4:
+		odm_set_rf_reg(dm, e_rf_path, REG_RF_TX_GAIN_OFFSET, BIT(14),
+			       0);
+		odm_set_rf_reg(dm, e_rf_path, REG_RF_TX_GAIN_OFFSET,
+			       BIT(17) | BIT(16) | BIT(15), 1);
+		cali_info->kfree_offset[e_rf_path] = 1;
+		break;
+	case 6:
+		odm_set_rf_reg(dm, e_rf_path, REG_RF_TX_GAIN_OFFSET, BIT(14),
+			       1);
+		odm_set_rf_reg(dm, e_rf_path, REG_RF_TX_GAIN_OFFSET,
+			       BIT(17) | BIT(16) | BIT(15), 1);
+		cali_info->kfree_offset[e_rf_path] = 1;
+		break;
+	case 8:
+		odm_set_rf_reg(dm, e_rf_path, REG_RF_TX_GAIN_OFFSET, BIT(14),
+			       0);
+		odm_set_rf_reg(dm, e_rf_path, REG_RF_TX_GAIN_OFFSET,
+			       BIT(17) | BIT(16) | BIT(15), 2);
+		cali_info->kfree_offset[e_rf_path] = 2;
+		break;
+	case 10:
+		odm_set_rf_reg(dm, e_rf_path, REG_RF_TX_GAIN_OFFSET, BIT(14),
+			       1);
+		odm_set_rf_reg(dm, e_rf_path, REG_RF_TX_GAIN_OFFSET,
+			       BIT(17) | BIT(16) | BIT(15), 2);
+		cali_info->kfree_offset[e_rf_path] = 2;
+		break;
+	case 12:
+		odm_set_rf_reg(dm, e_rf_path, REG_RF_TX_GAIN_OFFSET, BIT(14),
+			       0);
+		odm_set_rf_reg(dm, e_rf_path, REG_RF_TX_GAIN_OFFSET,
+			       BIT(17) | BIT(16) | BIT(15), 3);
+		cali_info->kfree_offset[e_rf_path] = 3;
+		break;
+	case 14:
+		odm_set_rf_reg(dm, e_rf_path, REG_RF_TX_GAIN_OFFSET, BIT(14),
+			       1);
+		odm_set_rf_reg(dm, e_rf_path, REG_RF_TX_GAIN_OFFSET,
+			       BIT(17) | BIT(16) | BIT(15), 3);
+		cali_info->kfree_offset[e_rf_path] = 3;
+		break;
+	case 16:
+		odm_set_rf_reg(dm, e_rf_path, REG_RF_TX_GAIN_OFFSET, BIT(14),
+			       0);
+		odm_set_rf_reg(dm, e_rf_path, REG_RF_TX_GAIN_OFFSET,
+			       BIT(17) | BIT(16) | BIT(15), 4);
+		cali_info->kfree_offset[e_rf_path] = 4;
+		break;
+	case 18:
+		odm_set_rf_reg(dm, e_rf_path, REG_RF_TX_GAIN_OFFSET, BIT(14),
+			       1);
+		odm_set_rf_reg(dm, e_rf_path, REG_RF_TX_GAIN_OFFSET,
+			       BIT(17) | BIT(16) | BIT(15), 4);
+		cali_info->kfree_offset[e_rf_path] = 4;
+		break;
+	case 20:
+		odm_set_rf_reg(dm, e_rf_path, REG_RF_TX_GAIN_OFFSET, BIT(14),
+			       0);
+		odm_set_rf_reg(dm, e_rf_path, REG_RF_TX_GAIN_OFFSET,
+			       BIT(17) | BIT(16) | BIT(15), 5);
+		cali_info->kfree_offset[e_rf_path] = 5;
+		break;
+
+	default:
+		break;
+	}
+
+	if (!is_odd) {
+		/*that means Kfree offset is negative, we need to record it.*/
+		cali_info->kfree_offset[e_rf_path] =
+			(-1) * cali_info->kfree_offset[e_rf_path];
+		ODM_RT_TRACE(dm, ODM_COMP_MP, "%s(): kfree_offset = %d\n",
+			     __func__, cali_info->kfree_offset[e_rf_path]);
+	} else {
+		ODM_RT_TRACE(dm, ODM_COMP_MP, "%s(): kfree_offset = %d\n",
+			     __func__, cali_info->kfree_offset[e_rf_path]);
+	}
+}
+
+static void phydm_set_kfree_to_rf(void *dm_void, u8 e_rf_path, u8 data)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	if (dm->support_ic_type & ODM_RTL8814A)
+		phydm_set_kfree_to_rf_8814a(dm, e_rf_path, data);
+}
+
+void phydm_config_kfree(void *dm_void, u8 channel_to_sw, u8 *kfree_table)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct dm_rf_calibration_struct *cali_info = &dm->rf_calibrate_info;
+	u8 rfpath = 0, max_rf_path = 0;
+	u8 channel_idx = 0;
+
+	if (dm->support_ic_type & ODM_RTL8814A)
+		max_rf_path = 4; /*0~3*/
+	else if (dm->support_ic_type &
+		 (ODM_RTL8812 | ODM_RTL8192E | ODM_RTL8822B))
+		max_rf_path = 2; /*0~1*/
+	else
+		max_rf_path = 1;
+
+	ODM_RT_TRACE(dm, ODM_COMP_MP, "===>%s()\n", __func__);
+
+	if (cali_info->reg_rf_kfree_enable == 2) {
+		ODM_RT_TRACE(dm, ODM_COMP_MP,
+			     "%s(): reg_rf_kfree_enable == 2, Disable\n",
+			     __func__);
+		return;
+	}
+
+	if (cali_info->reg_rf_kfree_enable != 1 &&
+	    cali_info->reg_rf_kfree_enable != 0) {
+		ODM_RT_TRACE(dm, ODM_COMP_MP, "<===%s()\n", __func__);
+		return;
+	}
+
+	ODM_RT_TRACE(dm, ODM_COMP_MP, "%s(): reg_rf_kfree_enable == true\n",
+		     __func__);
+	/*Make sure the targetval is defined*/
+	if (((cali_info->reg_rf_kfree_enable == 1) &&
+	     (kfree_table[0] != 0xFF)) ||
+	    cali_info->rf_kfree_enable) {
+		/*if kfree_table[0] == 0xff, means no Kfree*/
+		if (*dm->band_type == ODM_BAND_2_4G) {
+			if (channel_to_sw <= 14 && channel_to_sw >= 1)
+				channel_idx = PHYDM_2G;
+		} else if (*dm->band_type == ODM_BAND_5G) {
+			if (channel_to_sw >= 36 && channel_to_sw <= 48)
+				channel_idx = PHYDM_5GLB1;
+			if (channel_to_sw >= 52 && channel_to_sw <= 64)
+				channel_idx = PHYDM_5GLB2;
+			if (channel_to_sw >= 100 && channel_to_sw <= 120)
+				channel_idx = PHYDM_5GMB1;
+			if (channel_to_sw >= 124 && channel_to_sw <= 144)
+				channel_idx = PHYDM_5GMB2;
+			if (channel_to_sw >= 149 && channel_to_sw <= 177)
+				channel_idx = PHYDM_5GHB;
+		}
+
+		for (rfpath = ODM_RF_PATH_A; rfpath < max_rf_path; rfpath++) {
+			ODM_RT_TRACE(dm, ODM_COMP_MP, "%s(): PATH_%d: %#x\n",
+				     __func__, rfpath,
+				     kfree_table[channel_idx * max_rf_path +
+						 rfpath]);
+			phydm_set_kfree_to_rf(
+				dm, rfpath,
+				kfree_table[channel_idx * max_rf_path +
+					    rfpath]);
+		}
+	} else {
+		ODM_RT_TRACE(
+			dm, ODM_COMP_MP,
+			"%s(): targetval not defined, Don't execute KFree Process.\n",
+			__func__);
+		return;
+	}
+
+	ODM_RT_TRACE(dm, ODM_COMP_MP, "<===%s()\n", __func__);
+}
diff --git a/drivers/staging/rtlwifi/phydm/phydm_kfree.h b/drivers/staging/rtlwifi/phydm/phydm_kfree.h
new file mode 100644
index 000000000000..1ee60059afc1
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_kfree.h
@@ -0,0 +1,42 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __PHYDMKFREE_H__
+#define __PHYDKFREE_H__
+
+#define KFREE_VERSION "1.0"
+
+enum phydm_kfree_channeltosw {
+	PHYDM_2G = 0,
+	PHYDM_5GLB1 = 1,
+	PHYDM_5GLB2 = 2,
+	PHYDM_5GMB1 = 3,
+	PHYDM_5GMB2 = 4,
+	PHYDM_5GHB = 5,
+};
+
+void phydm_config_kfree(void *dm_void, u8 channel_to_sw, u8 *kfree_table);
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/phydm_noisemonitor.c b/drivers/staging/rtlwifi/phydm/phydm_noisemonitor.c
new file mode 100644
index 000000000000..8d79a5add1b4
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_noisemonitor.c
@@ -0,0 +1,330 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+/* ************************************************************
+ * include files
+ * *************************************************************/
+#include "mp_precomp.h"
+#include "phydm_precomp.h"
+#include "phydm_noisemonitor.h"
+
+/* *************************************************
+ * This function is for inband noise test utility only
+ * To obtain the inband noise level(dbm), do the following.
+ * 1. disable DIG and Power Saving
+ * 2. Set initial gain = 0x1a
+ * 3. Stop updating idle time pwer report (for driver read)
+ *	- 0x80c[25]
+ *
+ * **************************************************/
+
+#define VALID_MIN -35
+#define VALID_MAX 10
+#define VALID_CNT 5
+
+static inline void phydm_set_noise_data_sum(struct noise_level *noise_data,
+					    u8 max_rf_path)
+{
+	u8 rf_path;
+
+	for (rf_path = ODM_RF_PATH_A; rf_path < max_rf_path; rf_path++) {
+		if (noise_data->valid_cnt[rf_path])
+			noise_data->sum[rf_path] /=
+				noise_data->valid_cnt[rf_path];
+		else
+			noise_data->sum[rf_path] = 0;
+	}
+}
+
+static s16 odm_inband_noise_monitor_n_series(struct phy_dm_struct *dm,
+					     u8 is_pause_dig, u8 igi_value,
+					     u32 max_time)
+{
+	u32 tmp4b;
+	u8 max_rf_path = 0, rf_path;
+	u8 reg_c50, reg_c58, valid_done = 0;
+	struct noise_level noise_data;
+	u64 start = 0, func_start = 0, func_end = 0;
+
+	func_start = odm_get_current_time(dm);
+	dm->noise_level.noise_all = 0;
+
+	if ((dm->rf_type == ODM_1T2R) || (dm->rf_type == ODM_2T2R))
+		max_rf_path = 2;
+	else
+		max_rf_path = 1;
+
+	ODM_RT_TRACE(dm, ODM_COMP_COMMON, "%s() ==>\n", __func__);
+
+	odm_memory_set(dm, &noise_data, 0, sizeof(struct noise_level));
+
+	/*  */
+	/* step 1. Disable DIG && Set initial gain. */
+	/*  */
+
+	if (is_pause_dig)
+		odm_pause_dig(dm, PHYDM_PAUSE, PHYDM_PAUSE_LEVEL_1, igi_value);
+	/*  */
+	/* step 2. Disable all power save for read registers */
+	/*  */
+	/* dcmd_DebugControlPowerSave(adapter, PSDisable); */
+
+	/*  */
+	/* step 3. Get noise power level */
+	/*  */
+	start = odm_get_current_time(dm);
+	while (1) {
+		/* Stop updating idle time pwer report (for driver read) */
+		odm_set_bb_reg(dm, REG_FPGA0_TX_GAIN_STAGE, BIT(25), 1);
+
+		/* Read Noise Floor Report */
+		tmp4b = odm_get_bb_reg(dm, 0x8f8, MASKDWORD);
+		ODM_RT_TRACE(dm, ODM_COMP_COMMON,
+			     "Noise Floor Report (0x8f8) = 0x%08x\n", tmp4b);
+
+		/* update idle time pwer report per 5us */
+		odm_set_bb_reg(dm, REG_FPGA0_TX_GAIN_STAGE, BIT(25), 0);
+
+		noise_data.value[ODM_RF_PATH_A] = (u8)(tmp4b & 0xff);
+		noise_data.value[ODM_RF_PATH_B] = (u8)((tmp4b & 0xff00) >> 8);
+
+		ODM_RT_TRACE(dm, ODM_COMP_COMMON,
+			     "value_a = 0x%x(%d), value_b = 0x%x(%d)\n",
+			     noise_data.value[ODM_RF_PATH_A],
+			     noise_data.value[ODM_RF_PATH_A],
+			     noise_data.value[ODM_RF_PATH_B],
+			     noise_data.value[ODM_RF_PATH_B]);
+
+		for (rf_path = ODM_RF_PATH_A; rf_path < max_rf_path;
+		     rf_path++) {
+			noise_data.sval[rf_path] =
+				(s8)noise_data.value[rf_path];
+			noise_data.sval[rf_path] /= 2;
+		}
+
+		ODM_RT_TRACE(dm, ODM_COMP_COMMON, "sval_a = %d, sval_b = %d\n",
+			     noise_data.sval[ODM_RF_PATH_A],
+			     noise_data.sval[ODM_RF_PATH_B]);
+
+		for (rf_path = ODM_RF_PATH_A; rf_path < max_rf_path;
+		     rf_path++) {
+			if (!(noise_data.valid_cnt[rf_path] < VALID_CNT) ||
+			    !(noise_data.sval[rf_path] < VALID_MAX &&
+			      noise_data.sval[rf_path] >= VALID_MIN)) {
+				continue;
+			}
+
+			noise_data.valid_cnt[rf_path]++;
+			noise_data.sum[rf_path] += noise_data.sval[rf_path];
+			ODM_RT_TRACE(dm, ODM_COMP_COMMON,
+				     "rf_path:%d Valid sval = %d\n", rf_path,
+				     noise_data.sval[rf_path]);
+			ODM_RT_TRACE(dm, ODM_COMP_COMMON, "Sum of sval = %d,\n",
+				     noise_data.sum[rf_path]);
+			if (noise_data.valid_cnt[rf_path] == VALID_CNT) {
+				valid_done++;
+				ODM_RT_TRACE(
+					dm, ODM_COMP_COMMON,
+					"After divided, rf_path:%d,sum = %d\n",
+					rf_path, noise_data.sum[rf_path]);
+			}
+		}
+
+		if ((valid_done == max_rf_path) ||
+		    (odm_get_progressing_time(dm, start) > max_time)) {
+			phydm_set_noise_data_sum(&noise_data, max_rf_path);
+			break;
+		}
+	}
+	reg_c50 = (u8)odm_get_bb_reg(dm, REG_OFDM_0_XA_AGC_CORE1, MASKBYTE0);
+	reg_c50 &= ~BIT(7);
+	ODM_RT_TRACE(dm, ODM_COMP_COMMON, "0x%x = 0x%02x(%d)\n",
+		     REG_OFDM_0_XA_AGC_CORE1, reg_c50, reg_c50);
+	dm->noise_level.noise[ODM_RF_PATH_A] =
+		(u8)(-110 + reg_c50 + noise_data.sum[ODM_RF_PATH_A]);
+	dm->noise_level.noise_all += dm->noise_level.noise[ODM_RF_PATH_A];
+
+	if (max_rf_path == 2) {
+		reg_c58 = (u8)odm_get_bb_reg(dm, REG_OFDM_0_XB_AGC_CORE1,
+					     MASKBYTE0);
+		reg_c58 &= ~BIT(7);
+		ODM_RT_TRACE(dm, ODM_COMP_COMMON, "0x%x = 0x%02x(%d)\n",
+			     REG_OFDM_0_XB_AGC_CORE1, reg_c58, reg_c58);
+		dm->noise_level.noise[ODM_RF_PATH_B] =
+			(u8)(-110 + reg_c58 + noise_data.sum[ODM_RF_PATH_B]);
+		dm->noise_level.noise_all +=
+			dm->noise_level.noise[ODM_RF_PATH_B];
+	}
+	dm->noise_level.noise_all /= max_rf_path;
+
+	ODM_RT_TRACE(dm, ODM_COMP_COMMON, "noise_a = %d, noise_b = %d\n",
+		     dm->noise_level.noise[ODM_RF_PATH_A],
+		     dm->noise_level.noise[ODM_RF_PATH_B]);
+
+	/*  */
+	/* step 4. Recover the Dig */
+	/*  */
+	if (is_pause_dig)
+		odm_pause_dig(dm, PHYDM_RESUME, PHYDM_PAUSE_LEVEL_1, igi_value);
+	func_end = odm_get_progressing_time(dm, func_start);
+
+	ODM_RT_TRACE(dm, ODM_COMP_COMMON, "%s() <==\n", __func__);
+	return dm->noise_level.noise_all;
+}
+
+static s16 odm_inband_noise_monitor_ac_series(struct phy_dm_struct *dm,
+					      u8 is_pause_dig, u8 igi_value,
+					      u32 max_time)
+{
+	s32 rxi_buf_anta, rxq_buf_anta; /*rxi_buf_antb, rxq_buf_antb;*/
+	s32 value32, pwdb_A = 0, sval, noise, sum;
+	bool pd_flag;
+	u8 valid_cnt;
+	u64 start = 0, func_start = 0, func_end = 0;
+
+	if (!(dm->support_ic_type & (ODM_RTL8812 | ODM_RTL8821 | ODM_RTL8814A)))
+		return 0;
+
+	func_start = odm_get_current_time(dm);
+	dm->noise_level.noise_all = 0;
+
+	ODM_RT_TRACE(dm, ODM_COMP_COMMON, "%s() ==>\n", __func__);
+
+	/* step 1. Disable DIG && Set initial gain. */
+	if (is_pause_dig)
+		odm_pause_dig(dm, PHYDM_PAUSE, PHYDM_PAUSE_LEVEL_1, igi_value);
+
+	/* step 2. Disable all power save for read registers */
+	/*dcmd_DebugControlPowerSave(adapter, PSDisable); */
+
+	/* step 3. Get noise power level */
+	start = odm_get_current_time(dm);
+
+	/* reset counters */
+	sum = 0;
+	valid_cnt = 0;
+
+	/* step 3. Get noise power level */
+	while (1) {
+		/*Set IGI=0x1C */
+		odm_write_dig(dm, 0x1C);
+		/*stop CK320&CK88 */
+		odm_set_bb_reg(dm, 0x8B4, BIT(6), 1);
+		/*Read path-A */
+		odm_set_bb_reg(dm, 0x8FC, MASKDWORD, 0x200); /*set debug port*/
+		value32 = odm_get_bb_reg(dm, 0xFA0,
+					 MASKDWORD); /*read debug port*/
+
+		rxi_buf_anta = (value32 & 0xFFC00) >>
+			       10; /*rxi_buf_anta=RegFA0[19:10]*/
+		rxq_buf_anta = value32 & 0x3FF; /*rxq_buf_anta=RegFA0[19:10]*/
+
+		pd_flag = (bool)((value32 & BIT(31)) >> 31);
+
+		/*Not in packet detection period or Tx state */
+		if ((!pd_flag) || (rxi_buf_anta != 0x200)) {
+			/*sign conversion*/
+			rxi_buf_anta = odm_sign_conversion(rxi_buf_anta, 10);
+			rxq_buf_anta = odm_sign_conversion(rxq_buf_anta, 10);
+
+			pwdb_A = odm_pwdb_conversion(
+				rxi_buf_anta * rxi_buf_anta +
+					rxq_buf_anta * rxq_buf_anta,
+				20, 18); /*S(10,9)*S(10,9)=S(20,18)*/
+
+			ODM_RT_TRACE(
+				dm, ODM_COMP_COMMON,
+				"pwdb_A= %d dB, rxi_buf_anta= 0x%x, rxq_buf_anta= 0x%x\n",
+				pwdb_A, rxi_buf_anta & 0x3FF,
+				rxq_buf_anta & 0x3FF);
+		}
+		/*Start CK320&CK88*/
+		odm_set_bb_reg(dm, 0x8B4, BIT(6), 0);
+		/*BB Reset*/
+		odm_write_1byte(dm, 0x02, odm_read_1byte(dm, 0x02) & (~BIT(0)));
+		odm_write_1byte(dm, 0x02, odm_read_1byte(dm, 0x02) | BIT(0));
+		/*PMAC Reset*/
+		odm_write_1byte(dm, 0xB03,
+				odm_read_1byte(dm, 0xB03) & (~BIT(0)));
+		odm_write_1byte(dm, 0xB03, odm_read_1byte(dm, 0xB03) | BIT(0));
+		/*CCK Reset*/
+		if (odm_read_1byte(dm, 0x80B) & BIT(4)) {
+			odm_write_1byte(dm, 0x80B,
+					odm_read_1byte(dm, 0x80B) & (~BIT(4)));
+			odm_write_1byte(dm, 0x80B,
+					odm_read_1byte(dm, 0x80B) | BIT(4));
+		}
+
+		sval = pwdb_A;
+
+		if ((sval < 0 && sval >= -27) && (valid_cnt < VALID_CNT)) {
+			valid_cnt++;
+			sum += sval;
+			ODM_RT_TRACE(dm, ODM_COMP_COMMON, "Valid sval = %d\n",
+				     sval);
+			ODM_RT_TRACE(dm, ODM_COMP_COMMON, "Sum of sval = %d,\n",
+				     sum);
+			if ((valid_cnt >= VALID_CNT) ||
+			    (odm_get_progressing_time(dm, start) > max_time)) {
+				sum /= VALID_CNT;
+				ODM_RT_TRACE(dm, ODM_COMP_COMMON,
+					     "After divided, sum = %d\n", sum);
+				break;
+			}
+		}
+	}
+
+	/*ADC backoff is 12dB,*/
+	/*Ptarget=0x1C-110=-82dBm*/
+	noise = sum + 12 + 0x1C - 110;
+
+	/*Offset*/
+	noise = noise - 3;
+	ODM_RT_TRACE(dm, ODM_COMP_COMMON, "noise = %d\n", noise);
+	dm->noise_level.noise_all = (s16)noise;
+
+	/* step 4. Recover the Dig*/
+	if (is_pause_dig)
+		odm_pause_dig(dm, PHYDM_RESUME, PHYDM_PAUSE_LEVEL_1, igi_value);
+
+	func_end = odm_get_progressing_time(dm, func_start);
+
+	ODM_RT_TRACE(dm, ODM_COMP_COMMON, "%s() <==\n", __func__);
+
+	return dm->noise_level.noise_all;
+}
+
+s16 odm_inband_noise_monitor(void *dm_void, u8 is_pause_dig, u8 igi_value,
+			     u32 max_time)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	if (dm->support_ic_type & ODM_IC_11AC_SERIES)
+		return odm_inband_noise_monitor_ac_series(dm, is_pause_dig,
+							  igi_value, max_time);
+	else
+		return odm_inband_noise_monitor_n_series(dm, is_pause_dig,
+							 igi_value, max_time);
+}
diff --git a/drivers/staging/rtlwifi/phydm/phydm_noisemonitor.h b/drivers/staging/rtlwifi/phydm/phydm_noisemonitor.h
new file mode 100644
index 000000000000..a711b7954985
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_noisemonitor.h
@@ -0,0 +1,46 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef __ODMNOISEMONITOR_H__
+#define __ODMNOISEMONITOR_H__
+
+#define ODM_MAX_CHANNEL_NUM 38 /* 14+24 */
+struct noise_level {
+	u8 value[MAX_RF_PATH];
+	s8 sval[MAX_RF_PATH];
+
+	s32 sum[MAX_RF_PATH];
+	u8 valid[MAX_RF_PATH];
+	u8 valid_cnt[MAX_RF_PATH];
+};
+
+struct odm_noise_monitor {
+	s8 noise[MAX_RF_PATH];
+	s16 noise_all;
+};
+
+s16 odm_inband_noise_monitor(void *dm_void, u8 is_pause_dig, u8 igi_value,
+			     u32 max_time);
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/phydm_powertracking_ce.c b/drivers/staging/rtlwifi/phydm/phydm_powertracking_ce.c
new file mode 100644
index 000000000000..48e73eb1622b
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_powertracking_ce.c
@@ -0,0 +1,644 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+/*============================================================	*/
+/* include files						*/
+/*============================================================	*/
+#include "mp_precomp.h"
+#include "phydm_precomp.h"
+
+/* ************************************************************
+ * Global var
+ * *************************************************************/
+
+u32 ofdm_swing_table[OFDM_TABLE_SIZE] = {
+	0x7f8001fe, /* 0, +6.0dB */
+	0x788001e2, /* 1, +5.5dB */
+	0x71c001c7, /* 2, +5.0dB*/
+	0x6b8001ae, /* 3, +4.5dB*/
+	0x65400195, /* 4, +4.0dB*/
+	0x5fc0017f, /* 5, +3.5dB*/
+	0x5a400169, /* 6, +3.0dB*/
+	0x55400155, /* 7, +2.5dB*/
+	0x50800142, /* 8, +2.0dB*/
+	0x4c000130, /* 9, +1.5dB*/
+	0x47c0011f, /* 10, +1.0dB*/
+	0x43c0010f, /* 11, +0.5dB*/
+	0x40000100, /* 12, +0dB*/
+	0x3c8000f2, /* 13, -0.5dB*/
+	0x390000e4, /* 14, -1.0dB*/
+	0x35c000d7, /* 15, -1.5dB*/
+	0x32c000cb, /* 16, -2.0dB*/
+	0x300000c0, /* 17, -2.5dB*/
+	0x2d4000b5, /* 18, -3.0dB*/
+	0x2ac000ab, /* 19, -3.5dB*/
+	0x288000a2, /* 20, -4.0dB*/
+	0x26000098, /* 21, -4.5dB*/
+	0x24000090, /* 22, -5.0dB*/
+	0x22000088, /* 23, -5.5dB*/
+	0x20000080, /* 24, -6.0dB*/
+	0x1e400079, /* 25, -6.5dB*/
+	0x1c800072, /* 26, -7.0dB*/
+	0x1b00006c, /* 27. -7.5dB*/
+	0x19800066, /* 28, -8.0dB*/
+	0x18000060, /* 29, -8.5dB*/
+	0x16c0005b, /* 30, -9.0dB*/
+	0x15800056, /* 31, -9.5dB*/
+	0x14400051, /* 32, -10.0dB*/
+	0x1300004c, /* 33, -10.5dB*/
+	0x12000048, /* 34, -11.0dB*/
+	0x11000044, /* 35, -11.5dB*/
+	0x10000040, /* 36, -12.0dB*/
+};
+
+u8 cck_swing_table_ch1_ch13[CCK_TABLE_SIZE][8] = {
+	{0x36, 0x35, 0x2e, 0x25, 0x1c, 0x12, 0x09, 0x04}, /* 0, +0dB */
+	{0x33, 0x32, 0x2b, 0x23, 0x1a, 0x11, 0x08, 0x04}, /* 1, -0.5dB */
+	{0x30, 0x2f, 0x29, 0x21, 0x19, 0x10, 0x08, 0x03}, /* 2, -1.0dB*/
+	{0x2d, 0x2d, 0x27, 0x1f, 0x18, 0x0f, 0x08, 0x03}, /* 3, -1.5dB*/
+	{0x2b, 0x2a, 0x25, 0x1e, 0x16, 0x0e, 0x07, 0x03}, /* 4, -2.0dB */
+	{0x28, 0x28, 0x22, 0x1c, 0x15, 0x0d, 0x07, 0x03}, /* 5, -2.5dB*/
+	{0x26, 0x25, 0x21, 0x1b, 0x14, 0x0d, 0x06, 0x03}, /* 6, -3.0dB*/
+	{0x24, 0x23, 0x1f, 0x19, 0x13, 0x0c, 0x06, 0x03}, /* 7, -3.5dB*/
+	{0x22, 0x21, 0x1d, 0x18, 0x11, 0x0b, 0x06, 0x02}, /* 8, -4.0dB */
+	{0x20, 0x20, 0x1b, 0x16, 0x11, 0x08, 0x05, 0x02}, /* 9, -4.5dB*/
+	{0x1f, 0x1e, 0x1a, 0x15, 0x10, 0x0a, 0x05, 0x02}, /* 10, -5.0dB */
+	{0x1d, 0x1c, 0x18, 0x14, 0x0f, 0x0a, 0x05, 0x02}, /* 11, -5.5dB*/
+	{0x1b, 0x1a, 0x17, 0x13, 0x0e, 0x09, 0x04,
+	 0x02}, /* 12, -6.0dB <== default */
+	{0x1a, 0x19, 0x16, 0x12, 0x0d, 0x09, 0x04, 0x02}, /* 13, -6.5dB*/
+	{0x18, 0x17, 0x15, 0x11, 0x0c, 0x08, 0x04, 0x02}, /* 14, -7.0dB */
+	{0x17, 0x16, 0x13, 0x10, 0x0c, 0x08, 0x04, 0x02}, /* 15, -7.5dB*/
+	{0x16, 0x15, 0x12, 0x0f, 0x0b, 0x07, 0x04, 0x01}, /* 16, -8.0dB */
+	{0x14, 0x14, 0x11, 0x0e, 0x0b, 0x07, 0x03, 0x02}, /* 17, -8.5dB*/
+	{0x13, 0x13, 0x10, 0x0d, 0x0a, 0x06, 0x03, 0x01}, /* 18, -9.0dB */
+	{0x12, 0x12, 0x0f, 0x0c, 0x09, 0x06, 0x03, 0x01}, /* 19, -9.5dB*/
+	{0x11, 0x11, 0x0f, 0x0c, 0x09, 0x06, 0x03, 0x01}, /* 20, -10.0dB*/
+	{0x10, 0x10, 0x0e, 0x0b, 0x08, 0x05, 0x03, 0x01}, /* 21, -10.5dB*/
+	{0x0f, 0x0f, 0x0d, 0x0b, 0x08, 0x05, 0x03, 0x01}, /* 22, -11.0dB*/
+	{0x0e, 0x0e, 0x0c, 0x0a, 0x08, 0x05, 0x02, 0x01}, /* 23, -11.5dB*/
+	{0x0d, 0x0d, 0x0c, 0x0a, 0x07, 0x05, 0x02, 0x01}, /* 24, -12.0dB*/
+	{0x0d, 0x0c, 0x0b, 0x09, 0x07, 0x04, 0x02, 0x01}, /* 25, -12.5dB*/
+	{0x0c, 0x0c, 0x0a, 0x09, 0x06, 0x04, 0x02, 0x01}, /* 26, -13.0dB*/
+	{0x0b, 0x0b, 0x0a, 0x08, 0x06, 0x04, 0x02, 0x01}, /* 27, -13.5dB*/
+	{0x0b, 0x0a, 0x09, 0x08, 0x06, 0x04, 0x02, 0x01}, /* 28, -14.0dB*/
+	{0x0a, 0x0a, 0x09, 0x07, 0x05, 0x03, 0x02, 0x01}, /* 29, -14.5dB*/
+	{0x0a, 0x09, 0x08, 0x07, 0x05, 0x03, 0x02, 0x01}, /* 30, -15.0dB*/
+	{0x09, 0x09, 0x08, 0x06, 0x05, 0x03, 0x01, 0x01}, /* 31, -15.5dB*/
+	{0x09, 0x08, 0x07, 0x06, 0x04, 0x03, 0x01, 0x01} /* 32, -16.0dB*/
+};
+
+u8 cck_swing_table_ch14[CCK_TABLE_SIZE][8] = {
+	{0x36, 0x35, 0x2e, 0x1b, 0x00, 0x00, 0x00, 0x00}, /* 0, +0dB */
+	{0x33, 0x32, 0x2b, 0x19, 0x00, 0x00, 0x00, 0x00}, /* 1, -0.5dB */
+	{0x30, 0x2f, 0x29, 0x18, 0x00, 0x00, 0x00, 0x00}, /* 2, -1.0dB */
+	{0x2d, 0x2d, 0x17, 0x17, 0x00, 0x00, 0x00, 0x00}, /* 3, -1.5dB*/
+	{0x2b, 0x2a, 0x25, 0x15, 0x00, 0x00, 0x00, 0x00}, /* 4, -2.0dB */
+	{0x28, 0x28, 0x24, 0x14, 0x00, 0x00, 0x00, 0x00}, /* 5, -2.5dB*/
+	{0x26, 0x25, 0x21, 0x13, 0x00, 0x00, 0x00, 0x00}, /* 6, -3.0dB */
+	{0x24, 0x23, 0x1f, 0x12, 0x00, 0x00, 0x00, 0x00}, /* 7, -3.5dB */
+	{0x22, 0x21, 0x1d, 0x11, 0x00, 0x00, 0x00, 0x00}, /* 8, -4.0dB */
+	{0x20, 0x20, 0x1b, 0x10, 0x00, 0x00, 0x00, 0x00}, /* 9, -4.5dB*/
+	{0x1f, 0x1e, 0x1a, 0x0f, 0x00, 0x00, 0x00, 0x00}, /* 10, -5.0dB */
+	{0x1d, 0x1c, 0x18, 0x0e, 0x00, 0x00, 0x00, 0x00}, /* 11, -5.5dB*/
+	{0x1b, 0x1a, 0x17, 0x0e, 0x00, 0x00, 0x00,
+	 0x00}, /* 12, -6.0dB  <== default*/
+	{0x1a, 0x19, 0x16, 0x0d, 0x00, 0x00, 0x00, 0x00}, /* 13, -6.5dB */
+	{0x18, 0x17, 0x15, 0x0c, 0x00, 0x00, 0x00, 0x00}, /* 14, -7.0dB */
+	{0x17, 0x16, 0x13, 0x0b, 0x00, 0x00, 0x00, 0x00}, /* 15, -7.5dB*/
+	{0x16, 0x15, 0x12, 0x0b, 0x00, 0x00, 0x00, 0x00}, /* 16, -8.0dB */
+	{0x14, 0x14, 0x11, 0x0a, 0x00, 0x00, 0x00, 0x00}, /* 17, -8.5dB*/
+	{0x13, 0x13, 0x10, 0x0a, 0x00, 0x00, 0x00, 0x00}, /* 18, -9.0dB */
+	{0x12, 0x12, 0x0f, 0x09, 0x00, 0x00, 0x00, 0x00}, /* 19, -9.5dB*/
+	{0x11, 0x11, 0x0f, 0x09, 0x00, 0x00, 0x00, 0x00}, /* 20, -10.0dB*/
+	{0x10, 0x10, 0x0e, 0x08, 0x00, 0x00, 0x00, 0x00}, /* 21, -10.5dB*/
+	{0x0f, 0x0f, 0x0d, 0x08, 0x00, 0x00, 0x00, 0x00}, /* 22, -11.0dB*/
+	{0x0e, 0x0e, 0x0c, 0x07, 0x00, 0x00, 0x00, 0x00}, /* 23, -11.5dB*/
+	{0x0d, 0x0d, 0x0c, 0x07, 0x00, 0x00, 0x00, 0x00}, /* 24, -12.0dB*/
+	{0x0d, 0x0c, 0x0b, 0x06, 0x00, 0x00, 0x00, 0x00}, /* 25, -12.5dB*/
+	{0x0c, 0x0c, 0x0a, 0x06, 0x00, 0x00, 0x00, 0x00}, /* 26, -13.0dB*/
+	{0x0b, 0x0b, 0x0a, 0x06, 0x00, 0x00, 0x00, 0x00}, /* 27, -13.5dB*/
+	{0x0b, 0x0a, 0x09, 0x05, 0x00, 0x00, 0x00, 0x00}, /* 28, -14.0dB*/
+	{0x0a, 0x0a, 0x09, 0x05, 0x00, 0x00, 0x00, 0x00}, /* 29, -14.5dB*/
+	{0x0a, 0x09, 0x08, 0x05, 0x00, 0x00, 0x00, 0x00}, /* 30, -15.0dB*/
+	{0x09, 0x09, 0x08, 0x05, 0x00, 0x00, 0x00, 0x00}, /* 31, -15.5dB*/
+	{0x09, 0x08, 0x07, 0x04, 0x00, 0x00, 0x00, 0x00} /* 32, -16.0dB*/
+};
+
+u32 ofdm_swing_table_new[OFDM_TABLE_SIZE] = {
+	0x0b40002d, /* 0,  -15.0dB	*/
+	0x0c000030, /* 1,  -14.5dB*/
+	0x0cc00033, /* 2,  -14.0dB*/
+	0x0d800036, /* 3,  -13.5dB*/
+	0x0e400039, /* 4,  -13.0dB */
+	0x0f00003c, /* 5,  -12.5dB*/
+	0x10000040, /* 6,  -12.0dB*/
+	0x11000044, /* 7,  -11.5dB*/
+	0x12000048, /* 8,  -11.0dB*/
+	0x1300004c, /* 9,  -10.5dB*/
+	0x14400051, /* 10, -10.0dB*/
+	0x15800056, /* 11, -9.5dB*/
+	0x16c0005b, /* 12, -9.0dB*/
+	0x18000060, /* 13, -8.5dB*/
+	0x19800066, /* 14, -8.0dB*/
+	0x1b00006c, /* 15, -7.5dB*/
+	0x1c800072, /* 16, -7.0dB*/
+	0x1e400079, /* 17, -6.5dB*/
+	0x20000080, /* 18, -6.0dB*/
+	0x22000088, /* 19, -5.5dB*/
+	0x24000090, /* 20, -5.0dB*/
+	0x26000098, /* 21, -4.5dB*/
+	0x288000a2, /* 22, -4.0dB*/
+	0x2ac000ab, /* 23, -3.5dB*/
+	0x2d4000b5, /* 24, -3.0dB*/
+	0x300000c0, /* 25, -2.5dB*/
+	0x32c000cb, /* 26, -2.0dB*/
+	0x35c000d7, /* 27, -1.5dB*/
+	0x390000e4, /* 28, -1.0dB*/
+	0x3c8000f2, /* 29, -0.5dB*/
+	0x40000100, /* 30, +0dB*/
+	0x43c0010f, /* 31, +0.5dB*/
+	0x47c0011f, /* 32, +1.0dB*/
+	0x4c000130, /* 33, +1.5dB*/
+	0x50800142, /* 34, +2.0dB*/
+	0x55400155, /* 35, +2.5dB*/
+	0x5a400169, /* 36, +3.0dB*/
+	0x5fc0017f, /* 37, +3.5dB*/
+	0x65400195, /* 38, +4.0dB*/
+	0x6b8001ae, /* 39, +4.5dB*/
+	0x71c001c7, /* 40, +5.0dB*/
+	0x788001e2, /* 41, +5.5dB*/
+	0x7f8001fe /* 42, +6.0dB*/
+};
+
+u8 cck_swing_table_ch1_ch14_88f[CCK_TABLE_SIZE_88F][16] = {
+	{0x44, 0x42, 0x3C, 0x33, 0x28, 0x1C, 0x13, 0x0B, 0x05, 0x02, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-16dB*/
+	{0x48, 0x46, 0x3F, 0x36, 0x2A, 0x1E, 0x14, 0x0B, 0x05, 0x02, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-15.5dB*/
+	{0x4D, 0x4A, 0x43, 0x39, 0x2C, 0x20, 0x15, 0x0C, 0x06, 0x02, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-15dB*/
+	{0x51, 0x4F, 0x47, 0x3C, 0x2F, 0x22, 0x16, 0x0D, 0x06, 0x02, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-14.5dB*/
+	{0x56, 0x53, 0x4B, 0x40, 0x32, 0x24, 0x17, 0x0E, 0x06, 0x02, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-14dB*/
+	{0x5B, 0x58, 0x50, 0x43, 0x35, 0x26, 0x19, 0x0E, 0x07, 0x02, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-13.5dB*/
+	{0x60, 0x5D, 0x54, 0x47, 0x38, 0x28, 0x1A, 0x0F, 0x07, 0x02, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-13dB*/
+	{0x66, 0x63, 0x59, 0x4C, 0x3B, 0x2B, 0x1C, 0x10, 0x08, 0x02, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-12.5dB*/
+	{0x6C, 0x69, 0x5F, 0x50, 0x3F, 0x2D, 0x1E, 0x11, 0x08, 0x03, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-12dB*/
+	{0x73, 0x6F, 0x64, 0x55, 0x42, 0x30, 0x1F, 0x12, 0x08, 0x03, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-11.5dB*/
+	{0x79, 0x76, 0x6A, 0x5A, 0x46, 0x33, 0x21, 0x13, 0x09, 0x03, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-11dB*/
+	{0x81, 0x7C, 0x71, 0x5F, 0x4A, 0x36, 0x23, 0x14, 0x0A, 0x03, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-10.5dB*/
+	{0x88, 0x84, 0x77, 0x65, 0x4F, 0x39, 0x25, 0x15, 0x0A, 0x03, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-10dB*/
+	{0x90, 0x8C, 0x7E, 0x6B, 0x54, 0x3C, 0x27, 0x17, 0x0B, 0x03, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-9.5dB*/
+	{0x99, 0x94, 0x86, 0x71, 0x58, 0x40, 0x2A, 0x18, 0x0B, 0x04, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-9dB*/
+	{0xA2, 0x9D, 0x8E, 0x78, 0x5E, 0x43, 0x2C, 0x19, 0x0C, 0x04, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-8.5dB*/
+	{0xAC, 0xA6, 0x96, 0x7F, 0x63, 0x47, 0x2F, 0x1B, 0x0D, 0x04, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-8dB*/
+	{0xB6, 0xB0, 0x9F, 0x87, 0x69, 0x4C, 0x32, 0x1D, 0x0D, 0x04, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-7.5dB*/
+	{0xC1, 0xBA, 0xA8, 0x8F, 0x6F, 0x50, 0x35, 0x1E, 0x0E, 0x04, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-7dB*/
+	{0xCC, 0xC5, 0xB2, 0x97, 0x76, 0x55, 0x38, 0x20, 0x0F, 0x05, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-6.5dB*/
+	{0xD8, 0xD1, 0xBD, 0xA0, 0x7D, 0x5A, 0x3B, 0x22, 0x10, 0x05, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00} /*-6dB*/
+};
+
+u8 cck_swing_table_ch1_ch13_88f[CCK_TABLE_SIZE_88F][16] = {
+	{0x44, 0x42, 0x3C, 0x33, 0x28, 0x1C, 0x13, 0x0B, 0x05, 0x02, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-16dB*/
+	{0x48, 0x46, 0x3F, 0x36, 0x2A, 0x1E, 0x14, 0x0B, 0x05, 0x02, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-15.5dB*/
+	{0x4D, 0x4A, 0x43, 0x39, 0x2C, 0x20, 0x15, 0x0C, 0x06, 0x02, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-15dB*/
+	{0x51, 0x4F, 0x47, 0x3C, 0x2F, 0x22, 0x16, 0x0D, 0x06, 0x02, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-14.5dB*/
+	{0x56, 0x53, 0x4B, 0x40, 0x32, 0x24, 0x17, 0x0E, 0x06, 0x02, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-14dB*/
+	{0x5B, 0x58, 0x50, 0x43, 0x35, 0x26, 0x19, 0x0E, 0x07, 0x02, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-13.5dB*/
+	{0x60, 0x5D, 0x54, 0x47, 0x38, 0x28, 0x1A, 0x0F, 0x07, 0x02, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-13dB*/
+	{0x66, 0x63, 0x59, 0x4C, 0x3B, 0x2B, 0x1C, 0x10, 0x08, 0x02, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-12.5dB*/
+	{0x6C, 0x69, 0x5F, 0x50, 0x3F, 0x2D, 0x1E, 0x11, 0x08, 0x03, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-12dB*/
+	{0x73, 0x6F, 0x64, 0x55, 0x42, 0x30, 0x1F, 0x12, 0x08, 0x03, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-11.5dB*/
+	{0x79, 0x76, 0x6A, 0x5A, 0x46, 0x33, 0x21, 0x13, 0x09, 0x03, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-11dB*/
+	{0x81, 0x7C, 0x71, 0x5F, 0x4A, 0x36, 0x23, 0x14, 0x0A, 0x03, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-10.5dB*/
+	{0x88, 0x84, 0x77, 0x65, 0x4F, 0x39, 0x25, 0x15, 0x0A, 0x03, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-10dB*/
+	{0x90, 0x8C, 0x7E, 0x6B, 0x54, 0x3C, 0x27, 0x17, 0x0B, 0x03, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-9.5dB*/
+	{0x99, 0x94, 0x86, 0x71, 0x58, 0x40, 0x2A, 0x18, 0x0B, 0x04, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-9dB*/
+	{0xA2, 0x9D, 0x8E, 0x78, 0x5E, 0x43, 0x2C, 0x19, 0x0C, 0x04, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-8.5dB*/
+	{0xAC, 0xA6, 0x96, 0x7F, 0x63, 0x47, 0x2F, 0x1B, 0x0D, 0x04, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-8dB*/
+	{0xB6, 0xB0, 0x9F, 0x87, 0x69, 0x4C, 0x32, 0x1D, 0x0D, 0x04, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-7.5dB*/
+	{0xC1, 0xBA, 0xA8, 0x8F, 0x6F, 0x50, 0x35, 0x1E, 0x0E, 0x04, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-7dB*/
+	{0xCC, 0xC5, 0xB2, 0x97, 0x76, 0x55, 0x38, 0x20, 0x0F, 0x05, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-6.5dB*/
+	{0xD8, 0xD1, 0xBD, 0xA0, 0x7D, 0x5A, 0x3B, 0x22, 0x10, 0x05, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00} /*-6dB*/
+};
+
+u8 cck_swing_table_ch14_88f[CCK_TABLE_SIZE_88F][16] = {
+	{0x44, 0x42, 0x3C, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-16dB*/
+	{0x48, 0x46, 0x3F, 0x2A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-15.5dB*/
+	{0x4D, 0x4A, 0x43, 0x2C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-15dB*/
+	{0x51, 0x4F, 0x47, 0x2F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-14.5dB*/
+	{0x56, 0x53, 0x4B, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-14dB*/
+	{0x5B, 0x58, 0x50, 0x35, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-13.5dB*/
+	{0x60, 0x5D, 0x54, 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-13dB*/
+	{0x66, 0x63, 0x59, 0x3B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-12.5dB*/
+	{0x6C, 0x69, 0x5F, 0x3F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-12dB*/
+	{0x73, 0x6F, 0x64, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-11.5dB*/
+	{0x79, 0x76, 0x6A, 0x46, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-11dB*/
+	{0x81, 0x7C, 0x71, 0x4A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-10.5dB*/
+	{0x88, 0x84, 0x77, 0x4F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-10dB*/
+	{0x90, 0x8C, 0x7E, 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-9.5dB*/
+	{0x99, 0x94, 0x86, 0x58, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-9dB*/
+	{0xA2, 0x9D, 0x8E, 0x5E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-8.5dB*/
+	{0xAC, 0xA6, 0x96, 0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-8dB*/
+	{0xB6, 0xB0, 0x9F, 0x69, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-7.5dB*/
+	{0xC1, 0xBA, 0xA8, 0x6F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-7dB*/
+	{0xCC, 0xC5, 0xB2, 0x76, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00}, /*-6.5dB*/
+	{0xD8, 0xD1, 0xBD, 0x7D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	 0x00, 0x00, 0x00, 0x00} /*-6dB*/
+};
+
+u8 cck_swing_table_ch1_ch13_new[CCK_TABLE_SIZE][8] = {
+	{0x09, 0x08, 0x07, 0x06, 0x04, 0x03, 0x01, 0x01}, /*  0, -16.0dB*/
+	{0x09, 0x09, 0x08, 0x06, 0x05, 0x03, 0x01, 0x01}, /*   1, -15.5dB*/
+	{0x0a, 0x09, 0x08, 0x07, 0x05, 0x03, 0x02, 0x01}, /*  2, -15.0dB*/
+	{0x0a, 0x0a, 0x09, 0x07, 0x05, 0x03, 0x02, 0x01}, /*   3, -14.5dB*/
+	{0x0b, 0x0a, 0x09, 0x08, 0x06, 0x04, 0x02, 0x01}, /*   4, -14.0dB*/
+	{0x0b, 0x0b, 0x0a, 0x08, 0x06, 0x04, 0x02, 0x01}, /*   5, -13.5dB*/
+	{0x0c, 0x0c, 0x0a, 0x09, 0x06, 0x04, 0x02, 0x01}, /*   6, -13.0dB*/
+	{0x0d, 0x0c, 0x0b, 0x09, 0x07, 0x04, 0x02, 0x01}, /*   7, -12.5dB*/
+	{0x0d, 0x0d, 0x0c, 0x0a, 0x07, 0x05, 0x02, 0x01}, /*  8, -12.0dB*/
+	{0x0e, 0x0e, 0x0c, 0x0a, 0x08, 0x05, 0x02, 0x01}, /*   9, -11.5dB*/
+	{0x0f, 0x0f, 0x0d, 0x0b, 0x08, 0x05, 0x03, 0x01}, /*  10, -11.0dB*/
+	{0x10, 0x10, 0x0e, 0x0b, 0x08, 0x05, 0x03, 0x01}, /*  11, -10.5dB*/
+	{0x11, 0x11, 0x0f, 0x0c, 0x09, 0x06, 0x03, 0x01}, /*  12, -10.0dB*/
+	{0x12, 0x12, 0x0f, 0x0c, 0x09, 0x06, 0x03, 0x01}, /*  13, -9.5dB*/
+	{0x13, 0x13, 0x10, 0x0d, 0x0a, 0x06, 0x03, 0x01}, /*  14, -9.0dB */
+	{0x14, 0x14, 0x11, 0x0e, 0x0b, 0x07, 0x03, 0x02}, /*  15, -8.5dB*/
+	{0x16, 0x15, 0x12, 0x0f, 0x0b, 0x07, 0x04, 0x01}, /*  16, -8.0dB */
+	{0x17, 0x16, 0x13, 0x10, 0x0c, 0x08, 0x04, 0x02}, /*  17, -7.5dB*/
+	{0x18, 0x17, 0x15, 0x11, 0x0c, 0x08, 0x04, 0x02}, /*  18, -7.0dB */
+	{0x1a, 0x19, 0x16, 0x12, 0x0d, 0x09, 0x04, 0x02}, /*  19, -6.5dB*/
+	{0x1b, 0x1a, 0x17, 0x13, 0x0e, 0x09, 0x04, 0x02}, /*20, -6.0dB */
+	{0x1d, 0x1c, 0x18, 0x14, 0x0f, 0x0a, 0x05, 0x02}, /*  21, -5.5dB*/
+	{0x1f, 0x1e, 0x1a, 0x15, 0x10, 0x0a, 0x05, 0x02}, /* 22, -5.0dB */
+	{0x20, 0x20, 0x1b, 0x16, 0x11, 0x08, 0x05, 0x02}, /*  23, -4.5dB*/
+	{0x22, 0x21, 0x1d, 0x18, 0x11, 0x0b, 0x06, 0x02}, /*  24, -4.0dB */
+	{0x24, 0x23, 0x1f, 0x19, 0x13, 0x0c, 0x06, 0x03}, /*  25, -3.5dB*/
+	{0x26, 0x25, 0x21, 0x1b, 0x14, 0x0d, 0x06, 0x03}, /*  26, -3.0dB*/
+	{0x28, 0x28, 0x22, 0x1c, 0x15, 0x0d, 0x07, 0x03}, /*  27, -2.5dB*/
+	{0x2b, 0x2a, 0x25, 0x1e, 0x16, 0x0e, 0x07, 0x03}, /*  28, -2.0dB */
+	{0x2d, 0x2d, 0x27, 0x1f, 0x18, 0x0f, 0x08, 0x03}, /*  29, -1.5dB*/
+	{0x30, 0x2f, 0x29, 0x21, 0x19, 0x10, 0x08, 0x03}, /*  30, -1.0dB*/
+	{0x33, 0x32, 0x2b, 0x23, 0x1a, 0x11, 0x08, 0x04}, /*  31, -0.5dB*/
+	{0x36, 0x35, 0x2e, 0x25, 0x1c, 0x12, 0x09, 0x04} /*  32, +0dB*/
+};
+
+u8 cck_swing_table_ch14_new[CCK_TABLE_SIZE][8] = {
+	{0x09, 0x08, 0x07, 0x04, 0x00, 0x00, 0x00, 0x00}, /*  0, -16.0dB*/
+	{0x09, 0x09, 0x08, 0x05, 0x00, 0x00, 0x00, 0x00}, /* 1, -15.5dB*/
+	{0x0a, 0x09, 0x08, 0x05, 0x00, 0x00, 0x00, 0x00}, /*  2, -15.0dB*/
+	{0x0a, 0x0a, 0x09, 0x05, 0x00, 0x00, 0x00, 0x00}, /* 3, -14.5dB*/
+	{0x0b, 0x0a, 0x09, 0x05, 0x00, 0x00, 0x00, 0x00}, /*  4, -14.0dB*/
+	{0x0b, 0x0b, 0x0a, 0x06, 0x00, 0x00, 0x00, 0x00}, /*5, -13.5dB*/
+	{0x0c, 0x0c, 0x0a, 0x06, 0x00, 0x00, 0x00, 0x00}, /* 6, -13.0dB*/
+	{0x0d, 0x0c, 0x0b, 0x06, 0x00, 0x00, 0x00, 0x00}, /*  7, -12.5dB*/
+	{0x0d, 0x0d, 0x0c, 0x07, 0x00, 0x00, 0x00, 0x00}, /* 8, -12.0dB*/
+	{0x0e, 0x0e, 0x0c, 0x07, 0x00, 0x00, 0x00, 0x00}, /* 9, -11.5dB*/
+	{0x0f, 0x0f, 0x0d, 0x08, 0x00, 0x00, 0x00, 0x00}, /* 10, -11.0dB*/
+	{0x10, 0x10, 0x0e, 0x08, 0x00, 0x00, 0x00, 0x00}, /*11, -10.5dB*/
+	{0x11, 0x11, 0x0f, 0x09, 0x00, 0x00, 0x00, 0x00}, /* 12, -10.0dB*/
+	{0x12, 0x12, 0x0f, 0x09, 0x00, 0x00, 0x00, 0x00}, /* 13, -9.5dB*/
+	{0x13, 0x13, 0x10, 0x0a, 0x00, 0x00, 0x00, 0x00}, /*14, -9.0dB */
+	{0x14, 0x14, 0x11, 0x0a, 0x00, 0x00, 0x00, 0x00}, /* 15, -8.5dB*/
+	{0x16, 0x15, 0x12, 0x0b, 0x00, 0x00, 0x00, 0x00}, /* 16, -8.0dB */
+	{0x17, 0x16, 0x13, 0x0b, 0x00, 0x00, 0x00, 0x00}, /* 17, -7.5dB*/
+	{0x18, 0x17, 0x15, 0x0c, 0x00, 0x00, 0x00, 0x00}, /* 18, -7.0dB */
+	{0x1a, 0x19, 0x16, 0x0d, 0x00, 0x00, 0x00, 0x00}, /* 19, -6.5dB */
+	{0x1b, 0x1a, 0x17, 0x0e, 0x00, 0x00, 0x00, 0x00}, /* 20, -6.0dB */
+	{0x1d, 0x1c, 0x18, 0x0e, 0x00, 0x00, 0x00, 0x00}, /* 21, -5.5dB*/
+	{0x1f, 0x1e, 0x1a, 0x0f, 0x00, 0x00, 0x00, 0x00}, /* 22, -5.0dB */
+	{0x20, 0x20, 0x1b, 0x10, 0x00, 0x00, 0x00, 0x00}, /*23, -4.5dB*/
+	{0x22, 0x21, 0x1d, 0x11, 0x00, 0x00, 0x00, 0x00}, /* 24, -4.0dB */
+	{0x24, 0x23, 0x1f, 0x12, 0x00, 0x00, 0x00, 0x00}, /* 25, -3.5dB */
+	{0x26, 0x25, 0x21, 0x13, 0x00, 0x00, 0x00, 0x00}, /* 26, -3.0dB */
+	{0x28, 0x28, 0x24, 0x14, 0x00, 0x00, 0x00, 0x00}, /*27, -2.5dB*/
+	{0x2b, 0x2a, 0x25, 0x15, 0x00, 0x00, 0x00, 0x00}, /* 28, -2.0dB */
+	{0x2d, 0x2d, 0x17, 0x17, 0x00, 0x00, 0x00, 0x00}, /*29, -1.5dB*/
+	{0x30, 0x2f, 0x29, 0x18, 0x00, 0x00, 0x00, 0x00}, /* 30, -1.0dB */
+	{0x33, 0x32, 0x2b, 0x19, 0x00, 0x00, 0x00, 0x00}, /* 31, -0.5dB */
+	{0x36, 0x35, 0x2e, 0x1b, 0x00, 0x00, 0x00, 0x00} /* 32, +0dB	*/
+};
+
+u32 cck_swing_table_ch1_ch14_8723d[CCK_TABLE_SIZE_8723D] = {
+	0x0CD, /*0 ,    -20dB*/
+	0x0D9, 0x0E6, 0x0F3, 0x102, 0x111, 0x121, 0x132, 0x144, 0x158, 0x16C,
+	0x182, 0x198, 0x1B1, 0x1CA, 0x1E5, 0x202, 0x221, 0x241, 0x263, 0x287,
+	0x2AE, 0x2D6, 0x301, 0x32F, 0x35F, 0x392, 0x3C9, 0x402, 0x43F, 0x47F,
+	0x4C3, 0x50C, 0x558, 0x5A9, 0x5FF, 0x65A, 0x6BA, 0x720, 0x78C, 0x7FF,
+};
+
+/* JJ ADD 20161014 */
+u32 cck_swing_table_ch1_ch14_8710b[CCK_TABLE_SIZE_8710B] = {
+	0x0CD, /*0 ,    -20dB*/
+	0x0D9, 0x0E6, 0x0F3, 0x102, 0x111, 0x121, 0x132, 0x144, 0x158, 0x16C,
+	0x182, 0x198, 0x1B1, 0x1CA, 0x1E5, 0x202, 0x221, 0x241, 0x263, 0x287,
+	0x2AE, 0x2D6, 0x301, 0x32F, 0x35F, 0x392, 0x3C9, 0x402, 0x43F, 0x47F,
+	0x4C3, 0x50C, 0x558, 0x5A9, 0x5FF, 0x65A, 0x6BA, 0x720, 0x78C, 0x7FF,
+};
+
+u32 tx_scaling_table_jaguar[TXSCALE_TABLE_SIZE] = {
+	0x081, /* 0,  -12.0dB*/
+	0x088, /* 1,  -11.5dB*/
+	0x090, /* 2,  -11.0dB*/
+	0x099, /* 3,  -10.5dB*/
+	0x0A2, /* 4,  -10.0dB*/
+	0x0AC, /* 5,  -9.5dB*/
+	0x0B6, /* 6,  -9.0dB*/
+	0x0C0, /*7,  -8.5dB*/
+	0x0CC, /* 8,  -8.0dB*/
+	0x0D8, /* 9,  -7.5dB*/
+	0x0E5, /* 10, -7.0dB*/
+	0x0F2, /* 11, -6.5dB*/
+	0x101, /* 12, -6.0dB*/
+	0x110, /* 13, -5.5dB*/
+	0x120, /* 14, -5.0dB*/
+	0x131, /* 15, -4.5dB*/
+	0x143, /* 16, -4.0dB*/
+	0x156, /* 17, -3.5dB*/
+	0x16A, /* 18, -3.0dB*/
+	0x180, /* 19, -2.5dB*/
+	0x197, /* 20, -2.0dB*/
+	0x1AF, /* 21, -1.5dB*/
+	0x1C8, /* 22, -1.0dB*/
+	0x1E3, /* 23, -0.5dB*/
+	0x200, /* 24, +0  dB*/
+	0x21E, /* 25, +0.5dB*/
+	0x23E, /* 26, +1.0dB*/
+	0x261, /* 27, +1.5dB*/
+	0x285, /* 28, +2.0dB*/
+	0x2AB, /* 29, +2.5dB*/
+	0x2D3, /*30, +3.0dB*/
+	0x2FE, /* 31, +3.5dB*/
+	0x32B, /* 32, +4.0dB*/
+	0x35C, /* 33, +4.5dB*/
+	0x38E, /* 34, +5.0dB*/
+	0x3C4, /* 35, +5.5dB*/
+	0x3FE /* 36, +6.0dB	*/
+};
+
+void odm_txpowertracking_init(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	odm_txpowertracking_thermal_meter_init(dm);
+}
+
+static u8 get_swing_index(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u8 i = 0;
+	u32 bb_swing;
+	u32 swing_table_size;
+	u32 *swing_table;
+
+	if (dm->support_ic_type == ODM_RTL8188E ||
+	    dm->support_ic_type == ODM_RTL8723B ||
+	    dm->support_ic_type == ODM_RTL8192E ||
+	    dm->support_ic_type == ODM_RTL8188F ||
+	    dm->support_ic_type == ODM_RTL8703B) {
+		bb_swing = odm_get_bb_reg(dm, REG_OFDM_0_XA_TX_IQ_IMBALANCE,
+					  0xFFC00000);
+
+		swing_table = ofdm_swing_table_new;
+		swing_table_size = OFDM_TABLE_SIZE;
+	} else {
+		{
+			bb_swing = 0;
+			swing_table = ofdm_swing_table;
+			swing_table_size = OFDM_TABLE_SIZE;
+		}
+	}
+
+	for (i = 0; i < swing_table_size; ++i) {
+		u32 table_value = swing_table[i];
+
+		if (table_value >= 0x100000)
+			table_value >>= 22;
+		if (bb_swing == table_value)
+			break;
+	}
+	return i;
+}
+
+void odm_txpowertracking_thermal_meter_init(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u8 default_swing_index = get_swing_index(dm);
+	u8 p = 0;
+	struct dm_rf_calibration_struct *cali_info = &dm->rf_calibrate_info;
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)dm->adapter;
+	struct rtl_efuse *rtlefu = rtl_efuse(rtlpriv);
+
+	cali_info->is_txpowertracking = true;
+	cali_info->tx_powercount = 0;
+	cali_info->is_txpowertracking_init = false;
+
+	if (!dm->mp_mode)
+		cali_info->txpowertrack_control = true;
+	else
+		cali_info->txpowertrack_control = false;
+
+	if (!dm->mp_mode)
+		cali_info->txpowertrack_control = true;
+
+	ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION, "dm txpowertrack_control = %d\n",
+		     cali_info->txpowertrack_control);
+
+	/* dm->rf_calibrate_info.txpowertrack_control = true; */
+	cali_info->thermal_value = rtlefu->eeprom_thermalmeter;
+	cali_info->thermal_value_iqk = rtlefu->eeprom_thermalmeter;
+	cali_info->thermal_value_lck = rtlefu->eeprom_thermalmeter;
+
+	if (!cali_info->default_bb_swing_index_flag) {
+		/*The index of "0 dB" in SwingTable.*/
+		if (dm->support_ic_type == ODM_RTL8188E ||
+		    dm->support_ic_type == ODM_RTL8723B ||
+		    dm->support_ic_type == ODM_RTL8192E ||
+		    dm->support_ic_type == ODM_RTL8703B) {
+			cali_info->default_ofdm_index =
+				(default_swing_index >= OFDM_TABLE_SIZE) ?
+					30 :
+					default_swing_index;
+			cali_info->default_cck_index = 20;
+		} else if (dm->support_ic_type ==
+			   ODM_RTL8188F) { /*add by Mingzhi.Guo  2015-03-23*/
+			cali_info->default_ofdm_index = 28; /*OFDM: -1dB*/
+			cali_info->default_cck_index = 20; /*CCK:-6dB*/
+		} else if (dm->support_ic_type ==
+			   ODM_RTL8723D) { /*add by zhaohe  2015-10-27*/
+			cali_info->default_ofdm_index = 28; /*OFDM: -1dB*/
+			cali_info->default_cck_index = 28; /*CCK:   -6dB*/
+		} else if (dm->support_ic_type ==
+			   ODM_RTL8710B) { /* JJ ADD 20161014 */
+			cali_info->default_ofdm_index = 28; /*OFDM: -1dB*/
+			cali_info->default_cck_index = 28; /*CCK:   -6dB*/
+		} else {
+			cali_info->default_ofdm_index =
+				(default_swing_index >= TXSCALE_TABLE_SIZE) ?
+					24 :
+					default_swing_index;
+			cali_info->default_cck_index = 24;
+		}
+		cali_info->default_bb_swing_index_flag = true;
+	}
+
+	cali_info->bb_swing_idx_cck_base = cali_info->default_cck_index;
+	cali_info->CCK_index = cali_info->default_cck_index;
+
+	for (p = ODM_RF_PATH_A; p < MAX_RF_PATH; ++p) {
+		cali_info->bb_swing_idx_ofdm_base[p] =
+			cali_info->default_ofdm_index;
+		cali_info->OFDM_index[p] = cali_info->default_ofdm_index;
+		cali_info->delta_power_index[p] = 0;
+		cali_info->delta_power_index_last[p] = 0;
+		cali_info->power_index_offset[p] = 0;
+	}
+	cali_info->modify_tx_agc_value_ofdm = 0;
+	cali_info->modify_tx_agc_value_cck = 0;
+}
+
+void odm_txpowertracking_check(void *dm_void)
+{
+	/* 2011/09/29 MH In HW integration first stage, we provide 4 different
+	 * handle to operate at the same time.
+	 * In the stage2/3, we need to prive universal interface and merge all
+	 * HW dynamic mechanism.
+	 */
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	switch (dm->support_platform) {
+	case ODM_WIN:
+		odm_txpowertracking_check_mp(dm);
+		break;
+
+	case ODM_CE:
+		odm_txpowertracking_check_ce(dm);
+		break;
+
+	case ODM_AP:
+		odm_txpowertracking_check_ap(dm);
+		break;
+
+	default:
+		break;
+	}
+}
+
+void odm_txpowertracking_check_ce(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	void *adapter = dm->adapter;
+
+	if (!(dm->support_ability & ODM_RF_TX_PWR_TRACK))
+		return;
+
+	if (!dm->rf_calibrate_info.tm_trigger) {
+		if (IS_HARDWARE_TYPE_8188E(adapter) ||
+		    IS_HARDWARE_TYPE_8188F(adapter) ||
+		    IS_HARDWARE_TYPE_8192E(adapter) ||
+		    IS_HARDWARE_TYPE_8723B(adapter) ||
+		    IS_HARDWARE_TYPE_JAGUAR(adapter) ||
+		    IS_HARDWARE_TYPE_8814A(adapter) ||
+		    IS_HARDWARE_TYPE_8703B(adapter) ||
+		    IS_HARDWARE_TYPE_8723D(adapter) ||
+		    IS_HARDWARE_TYPE_8822B(adapter) ||
+		    IS_HARDWARE_TYPE_8821C(adapter) ||
+		    (dm->support_ic_type == ODM_RTL8710B)) /* JJ ADD 20161014 */
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, RF_T_METER_NEW,
+				       (BIT(17) | BIT(16)), 0x03);
+		else
+			odm_set_rf_reg(dm, ODM_RF_PATH_A, RF_T_METER_OLD,
+				       RFREGOFFSETMASK, 0x60);
+
+		dm->rf_calibrate_info.tm_trigger = 1;
+		return;
+	}
+
+	odm_txpowertracking_callback_thermal_meter(dm);
+	dm->rf_calibrate_info.tm_trigger = 0;
+}
+
+void odm_txpowertracking_check_mp(void *dm_void) {}
+
+void odm_txpowertracking_check_ap(void *dm_void) {}
diff --git a/drivers/staging/rtlwifi/phydm/phydm_powertracking_ce.h b/drivers/staging/rtlwifi/phydm/phydm_powertracking_ce.h
new file mode 100644
index 000000000000..757d7720d931
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_powertracking_ce.h
@@ -0,0 +1,293 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __PHYDMPOWERTRACKING_H__
+#define __PHYDMPOWERTRACKING_H__
+
+#define POWRTRACKING_VERSION "1.1"
+
+#define DPK_DELTA_MAPPING_NUM 13
+#define index_mapping_HP_NUM 15
+#define OFDM_TABLE_SIZE 43
+#define CCK_TABLE_SIZE 33
+#define CCK_TABLE_SIZE_88F 21
+#define TXSCALE_TABLE_SIZE 37
+#define CCK_TABLE_SIZE_8723D 41
+/* JJ ADD 20161014 */
+#define CCK_TABLE_SIZE_8710B 41
+
+#define TXPWR_TRACK_TABLE_SIZE 30
+#define DELTA_SWINGIDX_SIZE 30
+#define DELTA_SWINTSSI_SIZE 61
+#define BAND_NUM 4
+
+#define AVG_THERMAL_NUM 8
+#define HP_THERMAL_NUM 8
+#define IQK_MAC_REG_NUM 4
+#define IQK_ADDA_REG_NUM 16
+#define IQK_BB_REG_NUM_MAX 10
+
+#define IQK_BB_REG_NUM 9
+
+#define iqk_matrix_reg_num 8
+
+extern u32 ofdm_swing_table[OFDM_TABLE_SIZE];
+extern u8 cck_swing_table_ch1_ch13[CCK_TABLE_SIZE][8];
+extern u8 cck_swing_table_ch14[CCK_TABLE_SIZE][8];
+
+extern u32 ofdm_swing_table_new[OFDM_TABLE_SIZE];
+extern u8 cck_swing_table_ch1_ch13_new[CCK_TABLE_SIZE][8];
+extern u8 cck_swing_table_ch14_new[CCK_TABLE_SIZE][8];
+extern u8 cck_swing_table_ch1_ch14_88f[CCK_TABLE_SIZE_88F][16];
+extern u8 cck_swing_table_ch1_ch13_88f[CCK_TABLE_SIZE_88F][16];
+extern u8 cck_swing_table_ch14_88f[CCK_TABLE_SIZE_88F][16];
+extern u32 cck_swing_table_ch1_ch14_8723d[CCK_TABLE_SIZE_8723D];
+/* JJ ADD 20161014 */
+extern u32 cck_swing_table_ch1_ch14_8710b[CCK_TABLE_SIZE_8710B];
+
+extern u32 tx_scaling_table_jaguar[TXSCALE_TABLE_SIZE];
+
+/* <20121018, Kordan> In case fail to read TxPowerTrack.txt,
+ * we use the table of 88E as the default table.
+ */
+
+#define dm_check_txpowertracking odm_txpowertracking_check
+
+struct iqk_matrix_regs_setting {
+	bool is_iqk_done;
+	s32 value[3][iqk_matrix_reg_num];
+	bool is_bw_iqk_result_saved[3];
+};
+
+struct dm_rf_calibration_struct {
+	/* for tx power tracking */
+
+	u32 rega24; /* for TempCCK */
+	s32 rege94;
+	s32 rege9c;
+	s32 regeb4;
+	s32 regebc;
+
+	u8 tx_powercount;
+	bool is_txpowertracking_init;
+	bool is_txpowertracking;
+	/* for mp mode, turn off txpwrtracking as default */
+	u8 txpowertrack_control;
+	u8 tm_trigger;
+	u8 internal_pa_5g[2]; /* pathA / pathB */
+
+	u8 thermal_meter
+		[2]; /* thermal_meter, index 0 for RFIC0, and 1 for RFIC1 */
+	u8 thermal_value;
+	u8 thermal_value_lck;
+	u8 thermal_value_iqk;
+	s8 thermal_value_delta; /* delta of thermal_value and efuse thermal */
+	u8 thermal_value_dpk;
+	u8 thermal_value_avg[AVG_THERMAL_NUM];
+	u8 thermal_value_avg_index;
+	u8 thermal_value_rx_gain;
+	u8 thermal_value_crystal;
+	u8 thermal_value_dpk_store;
+	u8 thermal_value_dpk_track;
+	bool txpowertracking_in_progress;
+
+	bool is_reloadtxpowerindex;
+	u8 is_rf_pi_enable;
+	u32 txpowertracking_callback_cnt; /* cosa add for debug */
+
+	/* ---------------------- Tx power Tracking ------------------------- */
+	u8 is_cck_in_ch14;
+	u8 CCK_index;
+	u8 OFDM_index[MAX_RF_PATH];
+	s8 power_index_offset[MAX_RF_PATH];
+	s8 delta_power_index[MAX_RF_PATH];
+	s8 delta_power_index_last[MAX_RF_PATH];
+	bool is_tx_power_changed;
+	s8 xtal_offset;
+	s8 xtal_offset_last;
+
+	u8 thermal_value_hp[HP_THERMAL_NUM];
+	u8 thermal_value_hp_index;
+	struct iqk_matrix_regs_setting
+		iqk_matrix_reg_setting[IQK_MATRIX_SETTINGS_NUM];
+	u8 delta_lck;
+	s8 bb_swing_diff_2g, bb_swing_diff_5g; /* Unit: dB */
+	u8 delta_swing_table_idx_2g_cck_a_p[DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_table_idx_2g_cck_a_n[DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_table_idx_2g_cck_b_p[DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_table_idx_2g_cck_b_n[DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_table_idx_2g_cck_c_p[DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_table_idx_2g_cck_c_n[DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_table_idx_2g_cck_d_p[DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_table_idx_2g_cck_d_n[DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_table_idx_2ga_p[DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_table_idx_2ga_n[DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_table_idx_2gb_p[DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_table_idx_2gb_n[DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_table_idx_2gc_p[DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_table_idx_2gc_n[DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_table_idx_2gd_p[DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_table_idx_2gd_n[DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_table_idx_5ga_p[BAND_NUM][DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_table_idx_5ga_n[BAND_NUM][DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_table_idx_5gb_p[BAND_NUM][DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_table_idx_5gb_n[BAND_NUM][DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_table_idx_5gc_p[BAND_NUM][DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_table_idx_5gc_n[BAND_NUM][DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_table_idx_5gd_p[BAND_NUM][DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_table_idx_5gd_n[BAND_NUM][DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_tssi_table_2g_cck_a[DELTA_SWINTSSI_SIZE];
+	u8 delta_swing_tssi_table_2g_cck_b[DELTA_SWINTSSI_SIZE];
+	u8 delta_swing_tssi_table_2g_cck_c[DELTA_SWINTSSI_SIZE];
+	u8 delta_swing_tssi_table_2g_cck_d[DELTA_SWINTSSI_SIZE];
+	u8 delta_swing_tssi_table_2ga[DELTA_SWINTSSI_SIZE];
+	u8 delta_swing_tssi_table_2gb[DELTA_SWINTSSI_SIZE];
+	u8 delta_swing_tssi_table_2gc[DELTA_SWINTSSI_SIZE];
+	u8 delta_swing_tssi_table_2gd[DELTA_SWINTSSI_SIZE];
+	u8 delta_swing_tssi_table_5ga[BAND_NUM][DELTA_SWINTSSI_SIZE];
+	u8 delta_swing_tssi_table_5gb[BAND_NUM][DELTA_SWINTSSI_SIZE];
+	u8 delta_swing_tssi_table_5gc[BAND_NUM][DELTA_SWINTSSI_SIZE];
+	u8 delta_swing_tssi_table_5gd[BAND_NUM][DELTA_SWINTSSI_SIZE];
+	s8 delta_swing_table_xtal_p[DELTA_SWINGIDX_SIZE];
+	s8 delta_swing_table_xtal_n[DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_table_idx_2ga_p_8188e[DELTA_SWINGIDX_SIZE];
+	u8 delta_swing_table_idx_2ga_n_8188e[DELTA_SWINGIDX_SIZE];
+
+	u8 bb_swing_idx_ofdm[MAX_RF_PATH];
+	u8 bb_swing_idx_ofdm_current;
+	u8 bb_swing_idx_ofdm_base[MAX_RF_PATH];
+	bool default_bb_swing_index_flag;
+	bool bb_swing_flag_ofdm;
+	u8 bb_swing_idx_cck;
+	u8 bb_swing_idx_cck_current;
+	u8 bb_swing_idx_cck_base;
+	u8 default_ofdm_index;
+	u8 default_cck_index;
+	bool bb_swing_flag_cck;
+
+	s8 absolute_ofdm_swing_idx[MAX_RF_PATH];
+	s8 remnant_ofdm_swing_idx[MAX_RF_PATH];
+	s8 absolute_cck_swing_idx[MAX_RF_PATH];
+	s8 remnant_cck_swing_idx;
+	s8 modify_tx_agc_value; /*Remnat compensate value at tx_agc */
+	bool modify_tx_agc_flag_path_a;
+	bool modify_tx_agc_flag_path_b;
+	bool modify_tx_agc_flag_path_c;
+	bool modify_tx_agc_flag_path_d;
+	bool modify_tx_agc_flag_path_a_cck;
+
+	s8 kfree_offset[MAX_RF_PATH];
+
+	/* ------------------------------------------------------------------ */
+
+	/* for IQK */
+	u32 regc04;
+	u32 reg874;
+	u32 regc08;
+	u32 regb68;
+	u32 regb6c;
+	u32 reg870;
+	u32 reg860;
+	u32 reg864;
+
+	bool is_iqk_initialized;
+	bool is_lck_in_progress;
+	bool is_antenna_detected;
+	bool is_need_iqk;
+	bool is_iqk_in_progress;
+	bool is_iqk_pa_off;
+	u8 delta_iqk;
+	u32 ADDA_backup[IQK_ADDA_REG_NUM];
+	u32 IQK_MAC_backup[IQK_MAC_REG_NUM];
+	u32 IQK_BB_backup_recover[9];
+	/* { {S1: 0xc94, 0xc80, 0xc4c} , {S0: 0xc9c, 0xc88, 0xc4c}} */
+	u32 IQK_BB_backup[IQK_BB_REG_NUM];
+	u32 tx_iqc_8723b[2][3][2];
+	/* { {S1: 0xc14, 0xca0} ,           {S0: 0xc14, 0xca0}} */
+	u32 rx_iqc_8723b[2][2][2];
+	/* { {S1: 0xc94, 0xc80, 0xc4c} , {S0: 0xc9c, 0xc88, 0xc4c}}*/
+	u32 tx_iqc_8703b[3][2];
+	/* { {S1: 0xc14, 0xca0} ,           {S0: 0xc14, 0xca0}}*/
+	u32 rx_iqc_8703b[2][2];
+	/* { {S1: 0xc94, 0xc80, 0xc4c} , {S0: 0xc9c, 0xc88, 0xc4c}}*/
+	u32 tx_iqc_8723d[2][3][2];
+	/* { {S1: 0xc14, 0xca0} ,           {S0: 0xc14, 0xca0}}*/
+	u32 rx_iqc_8723d[2][2][2];
+	/* JJ ADD 20161014 */
+	/* { {S1: 0xc94, 0xc80, 0xc4c} , {S0: 0xc9c, 0xc88, 0xc4c}}*/
+	u32 tx_iqc_8710b[2][3][2];
+	/* { {S1: 0xc14, 0xca0} ,           {S0: 0xc14, 0xca0}}*/
+	u32 rx_iqc_8710b[2][2][2];
+
+	u8 iqk_step;
+	u8 kcount;
+	u8 retry_count[4][2]; /* [4]: path ABCD, [2] TXK, RXK */
+	bool is_mp_mode;
+
+	/* <James> IQK time measurement */
+	u64 iqk_start_time;
+	u64 iqk_progressing_time;
+	u64 iqk_total_progressing_time;
+
+	u32 lok_result;
+
+	/* for APK */
+	u32 ap_koutput[2][2]; /* path A/B; output1_1a/output1_2a */
+	u8 is_ap_kdone;
+	u8 is_apk_thermal_meter_ignore;
+
+	/* DPK */
+	bool is_dpk_fail;
+	u8 is_dp_done;
+	u8 is_dp_path_aok;
+	u8 is_dp_path_bok;
+
+	u32 tx_lok[2];
+	u32 dpk_tx_agc;
+	s32 dpk_gain;
+	u32 dpk_thermal[4];
+	s8 modify_tx_agc_value_ofdm;
+	s8 modify_tx_agc_value_cck;
+
+	/*Add by Yuchen for Kfree Phydm*/
+	u8 reg_rf_kfree_enable; /*for registry*/
+	u8 rf_kfree_enable; /*for efuse enable check*/
+};
+
+void odm_txpowertracking_check(void *dm_void);
+
+void odm_txpowertracking_init(void *dm_void);
+
+void odm_txpowertracking_check_ap(void *dm_void);
+
+void odm_txpowertracking_thermal_meter_init(void *dm_void);
+
+void odm_txpowertracking_init(void *dm_void);
+
+void odm_txpowertracking_check_mp(void *dm_void);
+
+void odm_txpowertracking_check_ce(void *dm_void);
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/phydm_pre_define.h b/drivers/staging/rtlwifi/phydm/phydm_pre_define.h
new file mode 100644
index 000000000000..6c301fe87b3d
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_pre_define.h
@@ -0,0 +1,613 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __PHYDMPREDEFINE_H__
+#define __PHYDMPREDEFINE_H__
+
+/* 1 ============================================================
+ * 1  Definition
+ * 1 ============================================================
+ */
+
+#define PHYDM_CODE_BASE "PHYDM_TRUNK"
+#define PHYDM_RELEASE_DATE "00000000"
+
+/* Max path of IC */
+#define MAX_PATH_NUM_8188E 1
+#define MAX_PATH_NUM_8192E 2
+#define MAX_PATH_NUM_8723B 1
+#define MAX_PATH_NUM_8812A 2
+#define MAX_PATH_NUM_8821A 1
+#define MAX_PATH_NUM_8814A 4
+#define MAX_PATH_NUM_8822B 2
+#define MAX_PATH_NUM_8821B 2
+#define MAX_PATH_NUM_8703B 1
+#define MAX_PATH_NUM_8188F 1
+#define MAX_PATH_NUM_8723D 1
+#define MAX_PATH_NUM_8197F 2
+#define MAX_PATH_NUM_8821C 1
+/* JJ ADD 20161014 */
+#define MAX_PATH_NUM_8710B 1
+
+/* Max RF path */
+#define ODM_RF_PATH_MAX 2
+#define ODM_RF_PATH_MAX_JAGUAR 4
+
+/*Bit define path*/
+#define PHYDM_A BIT(0)
+#define PHYDM_B BIT(1)
+#define PHYDM_C BIT(2)
+#define PHYDM_D BIT(3)
+#define PHYDM_AB (BIT(0) | BIT(1))
+#define PHYDM_AC (BIT(0) | BIT(2))
+#define PHYDM_AD (BIT(0) | BIT(3))
+#define PHYDM_BC (BIT(1) | BIT(2))
+#define PHYDM_BD (BIT(1) | BIT(3))
+#define PHYDM_CD (BIT(2) | BIT(3))
+#define PHYDM_ABC (BIT(0) | BIT(1) | BIT(2))
+#define PHYDM_ABD (BIT(0) | BIT(1) | BIT(3))
+#define PHYDM_ACD (BIT(0) | BIT(2) | BIT(3))
+#define PHYDM_BCD (BIT(1) | BIT(2) | BIT(3))
+#define PHYDM_ABCD (BIT(0) | BIT(1) | BIT(2) | BIT(3))
+
+/* number of entry */
+/* defined in wifi.h (32+1) */
+#define ODM_ASSOCIATE_ENTRY_NUM ASSOCIATE_ENTRY_NUM
+
+#define RX_SMOOTH_FACTOR 20
+
+/* -----MGN rate--------------------------------- */
+
+enum ODM_MGN_RATE {
+	ODM_MGN_1M = 0x02,
+	ODM_MGN_2M = 0x04,
+	ODM_MGN_5_5M = 0x0B,
+	ODM_MGN_6M = 0x0C,
+	ODM_MGN_9M = 0x12,
+	ODM_MGN_11M = 0x16,
+	ODM_MGN_12M = 0x18,
+	ODM_MGN_18M = 0x24,
+	ODM_MGN_24M = 0x30,
+	ODM_MGN_36M = 0x48,
+	ODM_MGN_48M = 0x60,
+	ODM_MGN_54M = 0x6C,
+	ODM_MGN_MCS32 = 0x7F,
+	ODM_MGN_MCS0,
+	ODM_MGN_MCS1,
+	ODM_MGN_MCS2,
+	ODM_MGN_MCS3,
+	ODM_MGN_MCS4,
+	ODM_MGN_MCS5,
+	ODM_MGN_MCS6,
+	ODM_MGN_MCS7,
+	ODM_MGN_MCS8,
+	ODM_MGN_MCS9,
+	ODM_MGN_MCS10,
+	ODM_MGN_MCS11,
+	ODM_MGN_MCS12,
+	ODM_MGN_MCS13,
+	ODM_MGN_MCS14,
+	ODM_MGN_MCS15,
+	ODM_MGN_MCS16,
+	ODM_MGN_MCS17,
+	ODM_MGN_MCS18,
+	ODM_MGN_MCS19,
+	ODM_MGN_MCS20,
+	ODM_MGN_MCS21,
+	ODM_MGN_MCS22,
+	ODM_MGN_MCS23,
+	ODM_MGN_MCS24,
+	ODM_MGN_MCS25,
+	ODM_MGN_MCS26,
+	ODM_MGN_MCS27,
+	ODM_MGN_MCS28,
+	ODM_MGN_MCS29,
+	ODM_MGN_MCS30,
+	ODM_MGN_MCS31,
+	ODM_MGN_VHT1SS_MCS0,
+	ODM_MGN_VHT1SS_MCS1,
+	ODM_MGN_VHT1SS_MCS2,
+	ODM_MGN_VHT1SS_MCS3,
+	ODM_MGN_VHT1SS_MCS4,
+	ODM_MGN_VHT1SS_MCS5,
+	ODM_MGN_VHT1SS_MCS6,
+	ODM_MGN_VHT1SS_MCS7,
+	ODM_MGN_VHT1SS_MCS8,
+	ODM_MGN_VHT1SS_MCS9,
+	ODM_MGN_VHT2SS_MCS0,
+	ODM_MGN_VHT2SS_MCS1,
+	ODM_MGN_VHT2SS_MCS2,
+	ODM_MGN_VHT2SS_MCS3,
+	ODM_MGN_VHT2SS_MCS4,
+	ODM_MGN_VHT2SS_MCS5,
+	ODM_MGN_VHT2SS_MCS6,
+	ODM_MGN_VHT2SS_MCS7,
+	ODM_MGN_VHT2SS_MCS8,
+	ODM_MGN_VHT2SS_MCS9,
+	ODM_MGN_VHT3SS_MCS0,
+	ODM_MGN_VHT3SS_MCS1,
+	ODM_MGN_VHT3SS_MCS2,
+	ODM_MGN_VHT3SS_MCS3,
+	ODM_MGN_VHT3SS_MCS4,
+	ODM_MGN_VHT3SS_MCS5,
+	ODM_MGN_VHT3SS_MCS6,
+	ODM_MGN_VHT3SS_MCS7,
+	ODM_MGN_VHT3SS_MCS8,
+	ODM_MGN_VHT3SS_MCS9,
+	ODM_MGN_VHT4SS_MCS0,
+	ODM_MGN_VHT4SS_MCS1,
+	ODM_MGN_VHT4SS_MCS2,
+	ODM_MGN_VHT4SS_MCS3,
+	ODM_MGN_VHT4SS_MCS4,
+	ODM_MGN_VHT4SS_MCS5,
+	ODM_MGN_VHT4SS_MCS6,
+	ODM_MGN_VHT4SS_MCS7,
+	ODM_MGN_VHT4SS_MCS8,
+	ODM_MGN_VHT4SS_MCS9,
+	ODM_MGN_UNKNOWN
+};
+
+#define ODM_MGN_MCS0_SG 0xc0
+#define ODM_MGN_MCS1_SG 0xc1
+#define ODM_MGN_MCS2_SG 0xc2
+#define ODM_MGN_MCS3_SG 0xc3
+#define ODM_MGN_MCS4_SG 0xc4
+#define ODM_MGN_MCS5_SG 0xc5
+#define ODM_MGN_MCS6_SG 0xc6
+#define ODM_MGN_MCS7_SG 0xc7
+#define ODM_MGN_MCS8_SG 0xc8
+#define ODM_MGN_MCS9_SG 0xc9
+#define ODM_MGN_MCS10_SG 0xca
+#define ODM_MGN_MCS11_SG 0xcb
+#define ODM_MGN_MCS12_SG 0xcc
+#define ODM_MGN_MCS13_SG 0xcd
+#define ODM_MGN_MCS14_SG 0xce
+#define ODM_MGN_MCS15_SG 0xcf
+
+/* -----DESC rate--------------------------------- */
+
+#define ODM_RATEMCS15_SG 0x1c
+#define ODM_RATEMCS32 0x20
+
+/* CCK Rates, TxHT = 0 */
+#define ODM_RATE1M 0x00
+#define ODM_RATE2M 0x01
+#define ODM_RATE5_5M 0x02
+#define ODM_RATE11M 0x03
+/* OFDM Rates, TxHT = 0 */
+#define ODM_RATE6M 0x04
+#define ODM_RATE9M 0x05
+#define ODM_RATE12M 0x06
+#define ODM_RATE18M 0x07
+#define ODM_RATE24M 0x08
+#define ODM_RATE36M 0x09
+#define ODM_RATE48M 0x0A
+#define ODM_RATE54M 0x0B
+/* MCS Rates, TxHT = 1 */
+#define ODM_RATEMCS0 0x0C
+#define ODM_RATEMCS1 0x0D
+#define ODM_RATEMCS2 0x0E
+#define ODM_RATEMCS3 0x0F
+#define ODM_RATEMCS4 0x10
+#define ODM_RATEMCS5 0x11
+#define ODM_RATEMCS6 0x12
+#define ODM_RATEMCS7 0x13
+#define ODM_RATEMCS8 0x14
+#define ODM_RATEMCS9 0x15
+#define ODM_RATEMCS10 0x16
+#define ODM_RATEMCS11 0x17
+#define ODM_RATEMCS12 0x18
+#define ODM_RATEMCS13 0x19
+#define ODM_RATEMCS14 0x1A
+#define ODM_RATEMCS15 0x1B
+#define ODM_RATEMCS16 0x1C
+#define ODM_RATEMCS17 0x1D
+#define ODM_RATEMCS18 0x1E
+#define ODM_RATEMCS19 0x1F
+#define ODM_RATEMCS20 0x20
+#define ODM_RATEMCS21 0x21
+#define ODM_RATEMCS22 0x22
+#define ODM_RATEMCS23 0x23
+#define ODM_RATEMCS24 0x24
+#define ODM_RATEMCS25 0x25
+#define ODM_RATEMCS26 0x26
+#define ODM_RATEMCS27 0x27
+#define ODM_RATEMCS28 0x28
+#define ODM_RATEMCS29 0x29
+#define ODM_RATEMCS30 0x2A
+#define ODM_RATEMCS31 0x2B
+#define ODM_RATEVHTSS1MCS0 0x2C
+#define ODM_RATEVHTSS1MCS1 0x2D
+#define ODM_RATEVHTSS1MCS2 0x2E
+#define ODM_RATEVHTSS1MCS3 0x2F
+#define ODM_RATEVHTSS1MCS4 0x30
+#define ODM_RATEVHTSS1MCS5 0x31
+#define ODM_RATEVHTSS1MCS6 0x32
+#define ODM_RATEVHTSS1MCS7 0x33
+#define ODM_RATEVHTSS1MCS8 0x34
+#define ODM_RATEVHTSS1MCS9 0x35
+#define ODM_RATEVHTSS2MCS0 0x36
+#define ODM_RATEVHTSS2MCS1 0x37
+#define ODM_RATEVHTSS2MCS2 0x38
+#define ODM_RATEVHTSS2MCS3 0x39
+#define ODM_RATEVHTSS2MCS4 0x3A
+#define ODM_RATEVHTSS2MCS5 0x3B
+#define ODM_RATEVHTSS2MCS6 0x3C
+#define ODM_RATEVHTSS2MCS7 0x3D
+#define ODM_RATEVHTSS2MCS8 0x3E
+#define ODM_RATEVHTSS2MCS9 0x3F
+#define ODM_RATEVHTSS3MCS0 0x40
+#define ODM_RATEVHTSS3MCS1 0x41
+#define ODM_RATEVHTSS3MCS2 0x42
+#define ODM_RATEVHTSS3MCS3 0x43
+#define ODM_RATEVHTSS3MCS4 0x44
+#define ODM_RATEVHTSS3MCS5 0x45
+#define ODM_RATEVHTSS3MCS6 0x46
+#define ODM_RATEVHTSS3MCS7 0x47
+#define ODM_RATEVHTSS3MCS8 0x48
+#define ODM_RATEVHTSS3MCS9 0x49
+#define ODM_RATEVHTSS4MCS0 0x4A
+#define ODM_RATEVHTSS4MCS1 0x4B
+#define ODM_RATEVHTSS4MCS2 0x4C
+#define ODM_RATEVHTSS4MCS3 0x4D
+#define ODM_RATEVHTSS4MCS4 0x4E
+#define ODM_RATEVHTSS4MCS5 0x4F
+#define ODM_RATEVHTSS4MCS6 0x50
+#define ODM_RATEVHTSS4MCS7 0x51
+#define ODM_RATEVHTSS4MCS8 0x52
+#define ODM_RATEVHTSS4MCS9 0x53
+
+#define ODM_NUM_RATE_IDX (ODM_RATEVHTSS4MCS9 + 1)
+
+/* 1 ============================================================
+ * 1  enumeration
+ * 1 ============================================================
+ */
+
+/*	ODM_CMNINFO_INTERFACE */
+enum odm_interface {
+	ODM_ITRF_PCIE = 0x1,
+	ODM_ITRF_USB = 0x2,
+	ODM_ITRF_SDIO = 0x4,
+	ODM_ITRF_ALL = 0x7,
+};
+
+/* ODM_CMNINFO_IC_TYPE */
+enum odm_ic_type {
+	ODM_RTL8188E = BIT(0),
+	ODM_RTL8812 = BIT(1),
+	ODM_RTL8821 = BIT(2),
+	ODM_RTL8192E = BIT(3),
+	ODM_RTL8723B = BIT(4),
+	ODM_RTL8814A = BIT(5),
+	ODM_RTL8881A = BIT(6),
+	ODM_RTL8822B = BIT(7),
+	ODM_RTL8703B = BIT(8),
+	ODM_RTL8195A = BIT(9),
+	ODM_RTL8188F = BIT(10),
+	ODM_RTL8723D = BIT(11),
+	ODM_RTL8197F = BIT(12),
+	ODM_RTL8821C = BIT(13),
+	ODM_RTL8814B = BIT(14),
+	ODM_RTL8198F = BIT(15),
+	/* JJ ADD 20161014 */
+	ODM_RTL8710B = BIT(16),
+};
+
+/* JJ ADD 20161014 */
+#define ODM_IC_1SS                                                             \
+	(ODM_RTL8188E | ODM_RTL8188F | ODM_RTL8723B | ODM_RTL8703B |           \
+	 ODM_RTL8723D | ODM_RTL8881A | ODM_RTL8821 | ODM_RTL8821C |            \
+	 ODM_RTL8195A | ODM_RTL8710B)
+#define ODM_IC_2SS (ODM_RTL8192E | ODM_RTL8197F | ODM_RTL8812 | ODM_RTL8822B)
+#define ODM_IC_3SS (ODM_RTL8814A)
+#define ODM_IC_4SS (ODM_RTL8814B | ODM_RTL8198F)
+
+/* JJ ADD 20161014 */
+#define ODM_IC_11N_SERIES                                                      \
+	(ODM_RTL8188E | ODM_RTL8192E | ODM_RTL8723B | ODM_RTL8703B |           \
+	 ODM_RTL8188F | ODM_RTL8723D | ODM_RTL8197F | ODM_RTL8710B)
+#define ODM_IC_11AC_SERIES                                                     \
+	(ODM_RTL8812 | ODM_RTL8821 | ODM_RTL8814A | ODM_RTL8881A |             \
+	 ODM_RTL8822B | ODM_RTL8821C)
+#define ODM_IC_11AC_1_SERIES (ODM_RTL8812 | ODM_RTL8821 | ODM_RTL8881A)
+#define ODM_IC_11AC_2_SERIES (ODM_RTL8814A | ODM_RTL8822B | ODM_RTL8821C)
+#define ODM_IC_TXBF_SUPPORT                                                    \
+	(ODM_RTL8192E | ODM_RTL8812 | ODM_RTL8821 | ODM_RTL8814A |             \
+	 ODM_RTL8881A | ODM_RTL8822B | ODM_RTL8197F | ODM_RTL8821C)
+#define ODM_IC_11N_GAIN_IDX_EDCCA                                              \
+	(ODM_RTL8195A | ODM_RTL8703B | ODM_RTL8188F | ODM_RTL8723D |           \
+	 ODM_RTL8197F | ODM_RTL8710B)
+#define ODM_IC_11AC_GAIN_IDX_EDCCA (ODM_RTL8814A | ODM_RTL8822B | ODM_RTL8821C)
+#define ODM_IC_PHY_STATUE_NEW_TYPE                                             \
+	(ODM_RTL8197F | ODM_RTL8822B | ODM_RTL8723D | ODM_RTL8821C |           \
+	 ODM_RTL8710B)
+
+#define PHYDM_IC_8051_SERIES                                                   \
+	(ODM_RTL8881A | ODM_RTL8812 | ODM_RTL8821 | ODM_RTL8188E |             \
+	 ODM_RTL8192E | ODM_RTL8723B | ODM_RTL8703B | ODM_RTL8188F)
+#define PHYDM_IC_3081_SERIES                                                   \
+	(ODM_RTL8814A | ODM_RTL8822B | ODM_RTL8197F | ODM_RTL8821C)
+
+#define PHYDM_IC_SUPPORT_LA_MODE                                               \
+	(ODM_RTL8814A | ODM_RTL8822B | ODM_RTL8197F | ODM_RTL8821C)
+
+/* JJ ADD 20161014 */
+
+/* ODM_CMNINFO_CUT_VER */
+enum odm_cut_version {
+	ODM_CUT_A = 0,
+	ODM_CUT_B = 1,
+	ODM_CUT_C = 2,
+	ODM_CUT_D = 3,
+	ODM_CUT_E = 4,
+	ODM_CUT_F = 5,
+
+	ODM_CUT_I = 8,
+	ODM_CUT_J = 9,
+	ODM_CUT_K = 10,
+	ODM_CUT_TEST = 15,
+};
+
+/* ODM_CMNINFO_FAB_VER */
+enum odm_fab {
+	ODM_TSMC = 0,
+	ODM_UMC = 1,
+};
+
+/* ODM_CMNINFO_RF_TYPE
+ *
+ * For example 1T2R (A+AB = BIT(0)|BIT(4)|BIT(5))
+ */
+enum odm_rf_path {
+	ODM_RF_A = BIT(0),
+	ODM_RF_B = BIT(1),
+	ODM_RF_C = BIT(2),
+	ODM_RF_D = BIT(3),
+};
+
+enum odm_rf_tx_num {
+	ODM_1T = 1,
+	ODM_2T = 2,
+	ODM_3T = 3,
+	ODM_4T = 4,
+};
+
+enum odm_rf_type {
+	ODM_1T1R,
+	ODM_1T2R,
+	ODM_2T2R,
+	ODM_2T2R_GREEN,
+	ODM_2T3R,
+	ODM_2T4R,
+	ODM_3T3R,
+	ODM_3T4R,
+	ODM_4T4R,
+	ODM_XTXR
+};
+
+enum odm_mac_phy_mode {
+	ODM_SMSP = 0,
+	ODM_DMSP = 1,
+	ODM_DMDP = 2,
+};
+
+enum odm_bt_coexist {
+	ODM_BT_BUSY = 1,
+	ODM_BT_ON = 2,
+	ODM_BT_OFF = 3,
+	ODM_BT_NONE = 4,
+};
+
+/* ODM_CMNINFO_OP_MODE */
+enum odm_operation_mode {
+	ODM_NO_LINK = BIT(0),
+	ODM_LINK = BIT(1),
+	ODM_SCAN = BIT(2),
+	ODM_POWERSAVE = BIT(3),
+	ODM_AP_MODE = BIT(4),
+	ODM_CLIENT_MODE = BIT(5),
+	ODM_AD_HOC = BIT(6),
+	ODM_WIFI_DIRECT = BIT(7),
+	ODM_WIFI_DISPLAY = BIT(8),
+};
+
+/* ODM_CMNINFO_WM_MODE */
+enum odm_wireless_mode {
+	ODM_WM_UNKNOWN = 0x0,
+	ODM_WM_B = BIT(0),
+	ODM_WM_G = BIT(1),
+	ODM_WM_A = BIT(2),
+	ODM_WM_N24G = BIT(3),
+	ODM_WM_N5G = BIT(4),
+	ODM_WM_AUTO = BIT(5),
+	ODM_WM_AC = BIT(6),
+};
+
+/* ODM_CMNINFO_BAND */
+enum odm_band_type {
+	ODM_BAND_2_4G = 0,
+	ODM_BAND_5G,
+	ODM_BAND_ON_BOTH,
+	ODM_BANDMAX
+};
+
+/* ODM_CMNINFO_SEC_CHNL_OFFSET */
+enum phydm_sec_chnl_offset {
+	PHYDM_DONT_CARE = 0,
+	PHYDM_BELOW = 1,
+	PHYDM_ABOVE = 2
+};
+
+/* ODM_CMNINFO_SEC_MODE */
+enum odm_security {
+	ODM_SEC_OPEN = 0,
+	ODM_SEC_WEP40 = 1,
+	ODM_SEC_TKIP = 2,
+	ODM_SEC_RESERVE = 3,
+	ODM_SEC_AESCCMP = 4,
+	ODM_SEC_WEP104 = 5,
+	ODM_WEP_WPA_MIXED = 6, /* WEP + WPA */
+	ODM_SEC_SMS4 = 7,
+};
+
+/* ODM_CMNINFO_BW */
+enum odm_bw {
+	ODM_BW20M = 0,
+	ODM_BW40M = 1,
+	ODM_BW80M = 2,
+	ODM_BW160M = 3,
+	ODM_BW5M = 4,
+	ODM_BW10M = 5,
+	ODM_BW_MAX = 6
+};
+
+/* ODM_CMNINFO_CHNL */
+
+/* ODM_CMNINFO_BOARD_TYPE */
+enum odm_board_type {
+	ODM_BOARD_DEFAULT = 0, /* The DEFAULT case. */
+	ODM_BOARD_MINICARD = BIT(0), /* 0 = non-mini card, 1= mini card. */
+	ODM_BOARD_SLIM = BIT(1), /* 0 = non-slim card, 1 = slim card */
+	ODM_BOARD_BT = BIT(2), /* 0 = without BT card, 1 = with BT */
+	ODM_BOARD_EXT_PA =
+		BIT(3), /* 0 = no 2G ext-PA, 1 = existing 2G ext-PA */
+	ODM_BOARD_EXT_LNA =
+		BIT(4), /* 0 = no 2G ext-LNA, 1 = existing 2G ext-LNA */
+	ODM_BOARD_EXT_TRSW =
+		BIT(5), /* 0 = no ext-TRSW, 1 = existing ext-TRSW */
+	ODM_BOARD_EXT_PA_5G =
+		BIT(6), /* 0 = no 5G ext-PA, 1 = existing 5G ext-PA */
+	ODM_BOARD_EXT_LNA_5G =
+		BIT(7), /* 0 = no 5G ext-LNA, 1 = existing 5G ext-LNA */
+};
+
+enum odm_package_type {
+	ODM_PACKAGE_DEFAULT = 0,
+	ODM_PACKAGE_QFN68 = BIT(0),
+	ODM_PACKAGE_TFBGA90 = BIT(1),
+	ODM_PACKAGE_TFBGA79 = BIT(2),
+};
+
+enum odm_type_gpa {
+	TYPE_GPA0 = 0x0000,
+	TYPE_GPA1 = 0x0055,
+	TYPE_GPA2 = 0x00AA,
+	TYPE_GPA3 = 0x00FF,
+	TYPE_GPA4 = 0x5500,
+	TYPE_GPA5 = 0x5555,
+	TYPE_GPA6 = 0x55AA,
+	TYPE_GPA7 = 0x55FF,
+	TYPE_GPA8 = 0xAA00,
+	TYPE_GPA9 = 0xAA55,
+	TYPE_GPA10 = 0xAAAA,
+	TYPE_GPA11 = 0xAAFF,
+	TYPE_GPA12 = 0xFF00,
+	TYPE_GPA13 = 0xFF55,
+	TYPE_GPA14 = 0xFFAA,
+	TYPE_GPA15 = 0xFFFF,
+};
+
+enum odm_type_apa {
+	TYPE_APA0 = 0x0000,
+	TYPE_APA1 = 0x0055,
+	TYPE_APA2 = 0x00AA,
+	TYPE_APA3 = 0x00FF,
+	TYPE_APA4 = 0x5500,
+	TYPE_APA5 = 0x5555,
+	TYPE_APA6 = 0x55AA,
+	TYPE_APA7 = 0x55FF,
+	TYPE_APA8 = 0xAA00,
+	TYPE_APA9 = 0xAA55,
+	TYPE_APA10 = 0xAAAA,
+	TYPE_APA11 = 0xAAFF,
+	TYPE_APA12 = 0xFF00,
+	TYPE_APA13 = 0xFF55,
+	TYPE_APA14 = 0xFFAA,
+	TYPE_APA15 = 0xFFFF,
+};
+
+enum odm_type_glna {
+	TYPE_GLNA0 = 0x0000,
+	TYPE_GLNA1 = 0x0055,
+	TYPE_GLNA2 = 0x00AA,
+	TYPE_GLNA3 = 0x00FF,
+	TYPE_GLNA4 = 0x5500,
+	TYPE_GLNA5 = 0x5555,
+	TYPE_GLNA6 = 0x55AA,
+	TYPE_GLNA7 = 0x55FF,
+	TYPE_GLNA8 = 0xAA00,
+	TYPE_GLNA9 = 0xAA55,
+	TYPE_GLNA10 = 0xAAAA,
+	TYPE_GLNA11 = 0xAAFF,
+	TYPE_GLNA12 = 0xFF00,
+	TYPE_GLNA13 = 0xFF55,
+	TYPE_GLNA14 = 0xFFAA,
+	TYPE_GLNA15 = 0xFFFF,
+};
+
+enum odm_type_alna {
+	TYPE_ALNA0 = 0x0000,
+	TYPE_ALNA1 = 0x0055,
+	TYPE_ALNA2 = 0x00AA,
+	TYPE_ALNA3 = 0x00FF,
+	TYPE_ALNA4 = 0x5500,
+	TYPE_ALNA5 = 0x5555,
+	TYPE_ALNA6 = 0x55AA,
+	TYPE_ALNA7 = 0x55FF,
+	TYPE_ALNA8 = 0xAA00,
+	TYPE_ALNA9 = 0xAA55,
+	TYPE_ALNA10 = 0xAAAA,
+	TYPE_ALNA11 = 0xAAFF,
+	TYPE_ALNA12 = 0xFF00,
+	TYPE_ALNA13 = 0xFF55,
+	TYPE_ALNA14 = 0xFFAA,
+	TYPE_ALNA15 = 0xFFFF,
+};
+
+enum odm_rf_radio_path {
+	ODM_RF_PATH_A = 0, /* Radio path A */
+	ODM_RF_PATH_B = 1, /* Radio path B */
+	ODM_RF_PATH_C = 2, /* Radio path C */
+	ODM_RF_PATH_D = 3, /* Radio path D */
+	ODM_RF_PATH_AB,
+	ODM_RF_PATH_AC,
+	ODM_RF_PATH_AD,
+	ODM_RF_PATH_BC,
+	ODM_RF_PATH_BD,
+	ODM_RF_PATH_CD,
+	ODM_RF_PATH_ABC,
+	ODM_RF_PATH_ACD,
+	ODM_RF_PATH_BCD,
+	ODM_RF_PATH_ABCD,
+	/* ODM_RF_PATH_MAX,    */ /* Max RF number 90 support */
+};
+
+enum odm_parameter_init {
+	ODM_PRE_SETTING = 0,
+	ODM_POST_SETTING = 1,
+};
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/phydm_precomp.h b/drivers/staging/rtlwifi/phydm/phydm_precomp.h
new file mode 100644
index 000000000000..bada15c4d2d8
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_precomp.h
@@ -0,0 +1,85 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __ODM_PRECOMP_H__
+#define __ODM_PRECOMP_H__
+
+#include "phydm_types.h"
+
+/* 2 Config Flags and Structs - defined by each ODM type */
+
+#include "../wifi.h"
+#include "rtl_phydm.h"
+
+/* 2 OutSrc Header Files */
+
+#include "phydm.h"
+#include "phydm_hwconfig.h"
+#include "phydm_debug.h"
+#include "phydm_regdefine11ac.h"
+#include "phydm_regdefine11n.h"
+#include "phydm_interface.h"
+#include "phydm_reg.h"
+
+#include "phydm_adc_sampling.h"
+
+/* JJ ADD 20161014 */
+
+#include "../halmac/halmac_reg2.h"
+
+#define LDPC_HT_ENABLE_RX BIT(0)
+#define LDPC_HT_ENABLE_TX BIT(1)
+#define LDPC_HT_TEST_TX_ENABLE BIT(2)
+#define LDPC_HT_CAP_TX BIT(3)
+
+#define STBC_HT_ENABLE_RX BIT(0)
+#define STBC_HT_ENABLE_TX BIT(1)
+#define STBC_HT_TEST_TX_ENABLE BIT(2)
+#define STBC_HT_CAP_TX BIT(3)
+
+#define LDPC_VHT_ENABLE_RX BIT(0)
+#define LDPC_VHT_ENABLE_TX BIT(1)
+#define LDPC_VHT_TEST_TX_ENABLE BIT(2)
+#define LDPC_VHT_CAP_TX BIT(3)
+
+#define STBC_VHT_ENABLE_RX BIT(0)
+#define STBC_VHT_ENABLE_TX BIT(1)
+#define STBC_VHT_TEST_TX_ENABLE BIT(2)
+#define STBC_VHT_CAP_TX BIT(3)
+
+#include "rtl8822b/halhwimg8822b_mac.h"
+#include "rtl8822b/halhwimg8822b_rf.h"
+#include "rtl8822b/halhwimg8822b_bb.h"
+#include "rtl8822b/phydm_regconfig8822b.h"
+#include "rtl8822b/halphyrf_8822b.h"
+#include "rtl8822b/phydm_rtl8822b.h"
+#include "rtl8822b/phydm_hal_api8822b.h"
+#include "rtl8822b/version_rtl8822b.h"
+
+#include "../halmac/halmac_reg_8822b.h"
+
+/* JJ ADD 20161014 */
+
+#endif /* __ODM_PRECOMP_H__ */
diff --git a/drivers/staging/rtlwifi/phydm/phydm_psd.c b/drivers/staging/rtlwifi/phydm/phydm_psd.c
new file mode 100644
index 000000000000..48f8776bc8f9
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_psd.c
@@ -0,0 +1,422 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+/*============================================================
+ * include files
+ *============================================================
+ */
+#include "mp_precomp.h"
+#include "phydm_precomp.h"
+
+u32 phydm_get_psd_data(void *dm_void, u32 psd_tone_idx, u32 igi)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct psd_info *dm_psd_table = &dm->dm_psd_table;
+	u32 psd_report = 0;
+
+	odm_set_bb_reg(dm, dm_psd_table->psd_reg, 0x3ff, psd_tone_idx);
+
+	odm_set_bb_reg(dm, dm_psd_table->psd_reg, BIT(22),
+		       1); /*PSD trigger start*/
+	ODM_delay_us(10);
+	odm_set_bb_reg(dm, dm_psd_table->psd_reg, BIT(22),
+		       0); /*PSD trigger stop*/
+
+	psd_report = odm_get_bb_reg(dm, dm_psd_table->psd_report_reg, 0xffff);
+	psd_report = odm_convert_to_db(psd_report) + igi;
+
+	return psd_report;
+}
+
+static u8 phydm_psd_stop_trx(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u32 i;
+	u8 trx_idle_success = false;
+	u32 dbg_port_value = 0;
+
+	/*[Stop TRX]----------------------------------------------------------*/
+	if (!phydm_set_bb_dbg_port(dm, BB_DBGPORT_PRIORITY_3,
+				   0x0)) /*set debug port to 0x0*/
+		return STOP_TRX_FAIL;
+
+	for (i = 0; i < 10000; i++) {
+		dbg_port_value = phydm_get_bb_dbg_port_value(dm);
+		if ((dbg_port_value & (BIT(17) | BIT(3))) ==
+		    0) /* PHYTXON && CCA_all */ {
+			ODM_RT_TRACE(dm, ODM_COMP_API,
+				     "PSD wait for ((%d)) times\n", i);
+
+			trx_idle_success = true;
+			break;
+		}
+	}
+
+	if (trx_idle_success) {
+		/*pause all TX queue*/
+		odm_set_bb_reg(dm, 0x520, 0xff0000, 0xff);
+
+		if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+			/*disable CCK block*/
+			odm_set_bb_reg(dm, 0x808, BIT(28), 0);
+			/*disable OFDM RX CCA*/
+			odm_set_bb_reg(dm, 0x838, BIT(1), 1);
+		} else {
+			/*TBD*/
+			/* disable whole CCK block */
+			odm_set_bb_reg(dm, 0x800, BIT(24), 0);
+			/*[ Set IQK Matrix = 0 ] equivalent to [ Turn off CCA]*/
+			odm_set_bb_reg(dm, 0xC14, MASKDWORD, 0x0);
+		}
+
+	} else {
+		return STOP_TRX_FAIL;
+	}
+
+	phydm_release_bb_dbg_port(dm);
+
+	return STOP_TRX_SUCCESS;
+}
+
+static u8 psd_result_cali_tone_8821[7] = {21, 28, 33, 93, 98, 105, 127};
+static u8 psd_result_cali_val_8821[7] = {67, 69, 71, 72, 71, 69, 67};
+
+void phydm_psd(void *dm_void, u32 igi, u16 start_point, u16 stop_point)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct psd_info *dm_psd_table = &dm->dm_psd_table;
+	u32 i = 0, mod_tone_idx;
+	u32 t = 0;
+	u16 fft_max_half_bw;
+	u32 psd_igi_a_reg;
+	u32 psd_igi_b_reg;
+	u16 psd_fc_channel = dm_psd_table->psd_fc_channel;
+	u8 ag_rf_mode_reg = 0;
+	u8 rf_reg18_9_8 = 0;
+	u32 psd_result_tmp = 0;
+	u8 psd_result = 0;
+	u8 psd_result_cali_tone[7] = {0};
+	u8 psd_result_cali_val[7] = {0};
+	u8 noise_table_idx = 0;
+
+	if (dm->support_ic_type == ODM_RTL8821) {
+		odm_move_memory(dm, psd_result_cali_tone,
+				psd_result_cali_tone_8821, 7);
+		odm_move_memory(dm, psd_result_cali_val,
+				psd_result_cali_val_8821, 7);
+	}
+
+	dm_psd_table->psd_in_progress = 1;
+
+	/*[Stop DIG]*/
+	dm->support_ability &= ~(ODM_BB_DIG);
+	dm->support_ability &= ~(ODM_BB_FA_CNT);
+
+	ODM_RT_TRACE(dm, ODM_COMP_API, "PSD Start =>\n");
+
+	if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+		psd_igi_a_reg = 0xc50;
+		psd_igi_b_reg = 0xe50;
+	} else {
+		psd_igi_a_reg = 0xc50;
+		psd_igi_b_reg = 0xc58;
+	}
+
+	/*[back up IGI]*/
+	dm_psd_table->initial_gain_backup =
+		odm_get_bb_reg(dm, psd_igi_a_reg, 0xff);
+	odm_set_bb_reg(dm, psd_igi_a_reg, 0xff,
+		       0x6e); /*IGI target at 0dBm & make it can't CCA*/
+	odm_set_bb_reg(dm, psd_igi_b_reg, 0xff,
+		       0x6e); /*IGI target at 0dBm & make it can't CCA*/
+	ODM_delay_us(10);
+
+	if (phydm_psd_stop_trx(dm) == STOP_TRX_FAIL) {
+		ODM_RT_TRACE(dm, ODM_COMP_API, "STOP_TRX_FAIL\n");
+		return;
+	}
+
+	/*[Set IGI]*/
+	odm_set_bb_reg(dm, psd_igi_a_reg, 0xff, igi);
+	odm_set_bb_reg(dm, psd_igi_b_reg, 0xff, igi);
+
+	/*[Backup RF Reg]*/
+	dm_psd_table->rf_0x18_bkp =
+		odm_get_rf_reg(dm, ODM_RF_PATH_A, 0x18, RFREGOFFSETMASK);
+
+	if (psd_fc_channel > 14) {
+		rf_reg18_9_8 = 1;
+
+		if (psd_fc_channel >= 36 && psd_fc_channel <= 64)
+			ag_rf_mode_reg = 0x1;
+		else if (psd_fc_channel >= 100 && psd_fc_channel <= 140)
+			ag_rf_mode_reg = 0x3;
+		else if (psd_fc_channel > 140)
+			ag_rf_mode_reg = 0x5;
+	}
+
+	/* Set RF fc*/
+	odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x18, 0xff, psd_fc_channel);
+	odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x18, 0x300, rf_reg18_9_8);
+	/*2b'11: 20MHz, 2b'10: 40MHz, 2b'01: 80MHz */
+	odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x18, 0xc00,
+		       dm_psd_table->psd_bw_rf_reg);
+	/* Set RF ag fc mode*/
+	odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x18, 0xf0000, ag_rf_mode_reg);
+
+	ODM_RT_TRACE(dm, ODM_COMP_API, "0xc50=((0x%x))\n",
+		     odm_get_bb_reg(dm, 0xc50, MASKDWORD));
+	ODM_RT_TRACE(dm, ODM_COMP_API, "RF0x18=((0x%x))\n",
+		     odm_get_rf_reg(dm, ODM_RF_PATH_A, 0x18, RFREGOFFSETMASK));
+
+	/*[Stop 3-wires]*/
+	if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+		odm_set_bb_reg(dm, 0xc00, 0xf, 0x4); /*	hardware 3-wire off */
+		odm_set_bb_reg(dm, 0xe00, 0xf, 0x4); /*	hardware 3-wire off */
+	} else {
+		odm_set_bb_reg(dm, 0x88c, 0xf00000,
+			       0xf); /* 3 wire Disable    88c[23:20]=0xf */
+	}
+	ODM_delay_us(10);
+
+	if (stop_point > (dm_psd_table->fft_smp_point - 1))
+		stop_point = (dm_psd_table->fft_smp_point - 1);
+
+	if (start_point > (dm_psd_table->fft_smp_point - 1))
+		start_point = (dm_psd_table->fft_smp_point - 1);
+
+	if (start_point > stop_point)
+		stop_point = start_point;
+
+	if (stop_point > 127) /* limit of psd_result[128] */
+		stop_point = 127;
+
+	for (i = start_point; i <= stop_point; i++) {
+		fft_max_half_bw = (dm_psd_table->fft_smp_point) >> 1;
+
+		if (i < fft_max_half_bw)
+			mod_tone_idx = i + fft_max_half_bw;
+		else
+			mod_tone_idx = i - fft_max_half_bw;
+
+		psd_result_tmp = 0;
+		for (t = 0; t < dm_psd_table->sw_avg_time; t++)
+			psd_result_tmp +=
+				phydm_get_psd_data(dm, mod_tone_idx, igi);
+		psd_result =
+			(u8)((psd_result_tmp / dm_psd_table->sw_avg_time)) -
+			dm_psd_table->psd_pwr_common_offset;
+
+		if (dm_psd_table->fft_smp_point == 128 &&
+		    (dm_psd_table->noise_k_en)) {
+			if (i > psd_result_cali_tone[noise_table_idx])
+				noise_table_idx++;
+
+			if (noise_table_idx > 6)
+				noise_table_idx = 6;
+
+			if (psd_result >= psd_result_cali_val[noise_table_idx])
+				psd_result =
+					psd_result -
+					psd_result_cali_val[noise_table_idx];
+			else
+				psd_result = 0;
+
+			dm_psd_table->psd_result[i] = psd_result;
+		}
+
+		ODM_RT_TRACE(dm, ODM_COMP_API, "[%d] N_cali = %d, PSD = %d\n",
+			     mod_tone_idx, psd_result_cali_val[noise_table_idx],
+			     psd_result);
+	}
+
+	/*[Start 3-wires]*/
+	if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+		odm_set_bb_reg(dm, 0xc00, 0xf, 0x7); /*	hardware 3-wire on */
+		odm_set_bb_reg(dm, 0xe00, 0xf, 0x7); /*	hardware 3-wire on */
+	} else {
+		odm_set_bb_reg(dm, 0x88c, 0xf00000,
+			       0x0); /* 3 wire enable    88c[23:20]=0x0 */
+	}
+	ODM_delay_us(10);
+
+	/*[Revert Reg]*/
+	odm_set_bb_reg(dm, 0x520, 0xff0000, 0x0); /*start all TX queue*/
+	odm_set_bb_reg(dm, 0x808, BIT(28), 1); /*enable CCK block*/
+	odm_set_bb_reg(dm, 0x838, BIT(1), 0); /*enable OFDM RX CCA*/
+
+	odm_set_bb_reg(dm, psd_igi_a_reg, 0xff,
+		       dm_psd_table->initial_gain_backup);
+	odm_set_bb_reg(dm, psd_igi_b_reg, 0xff,
+		       dm_psd_table->initial_gain_backup);
+
+	odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x18, RFREGOFFSETMASK,
+		       dm_psd_table->rf_0x18_bkp);
+
+	ODM_RT_TRACE(dm, ODM_COMP_API, "PSD finished\n\n");
+
+	dm->support_ability |= ODM_BB_DIG;
+	dm->support_ability |= ODM_BB_FA_CNT;
+	dm_psd_table->psd_in_progress = 0;
+}
+
+void phydm_psd_para_setting(void *dm_void, u8 sw_avg_time, u8 hw_avg_time,
+			    u8 i_q_setting, u16 fft_smp_point, u8 ant_sel,
+			    u8 psd_input, u8 channel, u8 noise_k_en)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct psd_info *dm_psd_table = &dm->dm_psd_table;
+	u8 fft_smp_point_idx = 0;
+
+	dm_psd_table->fft_smp_point = fft_smp_point;
+
+	if (sw_avg_time == 0)
+		sw_avg_time = 1;
+
+	dm_psd_table->sw_avg_time = sw_avg_time;
+	dm_psd_table->psd_fc_channel = channel;
+	dm_psd_table->noise_k_en = noise_k_en;
+
+	if (fft_smp_point == 128)
+		fft_smp_point_idx = 0;
+	else if (fft_smp_point == 256)
+		fft_smp_point_idx = 1;
+	else if (fft_smp_point == 512)
+		fft_smp_point_idx = 2;
+	else if (fft_smp_point == 1024)
+		fft_smp_point_idx = 3;
+
+	if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+		odm_set_bb_reg(dm, 0x910, BIT(11) | BIT(10), i_q_setting);
+		odm_set_bb_reg(dm, 0x910, BIT(13) | BIT(12), hw_avg_time);
+		odm_set_bb_reg(dm, 0x910, BIT(15) | BIT(14), fft_smp_point_idx);
+		odm_set_bb_reg(dm, 0x910, BIT(17) | BIT(16), ant_sel);
+		odm_set_bb_reg(dm, 0x910, BIT(23), psd_input);
+	}
+
+	/*bw = (*dm->band_width); //ODM_BW20M */
+	/*channel = *(dm->channel);*/
+}
+
+void phydm_psd_init(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct psd_info *dm_psd_table = &dm->dm_psd_table;
+
+	ODM_RT_TRACE(dm, ODM_COMP_API, "PSD para init\n");
+
+	dm_psd_table->psd_in_progress = false;
+
+	if (dm->support_ic_type & ODM_IC_11AC_SERIES) {
+		dm_psd_table->psd_reg = 0x910;
+		dm_psd_table->psd_report_reg = 0xF44;
+
+		if (ODM_IC_11AC_2_SERIES)
+			dm_psd_table->psd_bw_rf_reg =
+				1; /*2b'11: 20MHz, 2b'10: 40MHz, 2b'01: 80MHz */
+		else
+			dm_psd_table->psd_bw_rf_reg =
+				2; /*2b'11: 20MHz, 2b'10: 40MHz, 2b'01: 80MHz */
+
+	} else {
+		dm_psd_table->psd_reg = 0x808;
+		dm_psd_table->psd_report_reg = 0x8B4;
+		dm_psd_table->psd_bw_rf_reg =
+			2; /*2b'11: 20MHz, 2b'10: 40MHz, 2b'01: 80MHz */
+	}
+
+	if (dm->support_ic_type == ODM_RTL8812)
+		dm_psd_table->psd_pwr_common_offset = 0;
+	else if (dm->support_ic_type == ODM_RTL8821)
+		dm_psd_table->psd_pwr_common_offset = 0;
+	else
+		dm_psd_table->psd_pwr_common_offset = 0;
+
+	phydm_psd_para_setting(dm, 1, 2, 3, 128, 0, 0, 7, 0);
+	/*phydm_psd(dm, 0x3c, 0, 127);*/ /* target at -50dBm */
+}
+
+void phydm_psd_debug(void *dm_void, char input[][16], u32 *_used, char *output,
+		     u32 *_out_len, u32 input_num)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	char help[] = "-h";
+	u32 var1[10] = {0};
+	u32 used = *_used;
+	u32 out_len = *_out_len;
+	u8 i;
+
+	if ((strcmp(input[1], help) == 0)) {
+		PHYDM_SNPRINTF(
+			output + used, out_len - used,
+			"{0} {sw_avg} {hw_avg 0:3} {1:I,2:Q,3:IQ} {fft_point: 128*(1:4)} {path_sel 0~3} {0:ADC, 1:RXIQC} {CH} {noise_k}\n");
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "{1} {IGI(hex)} {start_point} {stop_point}\n");
+		return;
+	}
+
+	PHYDM_SSCANF(input[1], DCMD_DECIMAL, &var1[0]);
+
+	if (var1[0] == 0) {
+		for (i = 1; i < 10; i++) {
+			if (input[i + 1])
+				PHYDM_SSCANF(input[i + 1], DCMD_DECIMAL,
+					     &var1[i]);
+		}
+
+		PHYDM_SNPRINTF(
+			output + used, out_len - used,
+			"sw_avg_time=((%d)), hw_avg_time=((%d)), IQ=((%d)), fft=((%d)), path=((%d)), input =((%d)) ch=((%d)), noise_k=((%d))\n",
+			var1[1], var1[2], var1[3], var1[4], var1[5], var1[6],
+			(u8)var1[7], (u8)var1[8]);
+		phydm_psd_para_setting(dm, (u8)var1[1], (u8)var1[2],
+				       (u8)var1[3], (u16)var1[4], (u8)var1[5],
+				       (u8)var1[6], (u8)var1[7], (u8)var1[8]);
+
+	} else if (var1[0] == 1) {
+		PHYDM_SSCANF(input[2], DCMD_HEX, &var1[1]);
+		PHYDM_SSCANF(input[3], DCMD_DECIMAL, &var1[2]);
+		PHYDM_SSCANF(input[4], DCMD_DECIMAL, &var1[3]);
+		PHYDM_SNPRINTF(
+			output + used, out_len - used,
+			"IGI=((0x%x)), start_point=((%d)), stop_point=((%d))\n",
+			var1[1], var1[2], var1[3]);
+		dm->debug_components |= ODM_COMP_API;
+		phydm_psd(dm, var1[1], (u16)var1[2], (u16)var1[3]);
+		dm->debug_components &= (~ODM_COMP_API);
+	}
+}
+
+u8 phydm_get_psd_result_table(void *dm_void, int index)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct psd_info *dm_psd_table = &dm->dm_psd_table;
+	u8 temp_result = 0;
+
+	if (index < 128)
+		temp_result = dm_psd_table->psd_result[index];
+
+	return temp_result;
+}
diff --git a/drivers/staging/rtlwifi/phydm/phydm_psd.h b/drivers/staging/rtlwifi/phydm/phydm_psd.h
new file mode 100644
index 000000000000..aeb70751d80b
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_psd.h
@@ -0,0 +1,67 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __PHYDMPSD_H__
+#define __PHYDMPSD_H__
+
+/*#define PSD_VERSION	"1.0"*/ /*2016.09.22  Dino*/
+#define PSD_VERSION "1.1" /*2016.10.07  Dino, Add Option for PSD Tone index
+			   *Selection
+			   */
+
+#define STOP_TRX_SUCCESS 1
+#define STOP_TRX_FAIL 0
+
+struct psd_info {
+	u8 psd_in_progress;
+	u32 psd_reg;
+	u32 psd_report_reg;
+	u8 psd_pwr_common_offset;
+	u16 sw_avg_time;
+	u16 fft_smp_point;
+	u32 initial_gain_backup;
+	u32 rf_0x18_bkp;
+	u16 psd_fc_channel;
+	u32 psd_bw_rf_reg;
+	u8 psd_result[128];
+	u8 noise_k_en;
+};
+
+u32 phydm_get_psd_data(void *dm_void, u32 psd_tone_idx, u32 igi);
+
+void phydm_psd_debug(void *dm_void, char input[][16], u32 *_used, char *output,
+		     u32 *_out_len, u32 input_num);
+
+void phydm_psd(void *dm_void, u32 igi, u16 start_point, u16 stop_point);
+
+void phydm_psd_para_setting(void *dm_void, u8 sw_avg_time, u8 hw_avg_time,
+			    u8 i_q_setting, u16 fft_smp_point, u8 ant_sel,
+			    u8 psd_input, u8 channel, u8 noise_k_en);
+
+void phydm_psd_init(void *dm_void);
+
+u8 phydm_get_psd_result_table(void *dm_void, int index);
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/phydm_rainfo.c b/drivers/staging/rtlwifi/phydm/phydm_rainfo.c
new file mode 100644
index 000000000000..8c08c76d4eda
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_rainfo.c
@@ -0,0 +1,1208 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+/* ************************************************************
+ * include files
+ * *************************************************************/
+#include "mp_precomp.h"
+#include "phydm_precomp.h"
+
+void phydm_h2C_debug(void *dm_void, u32 *const dm_value, u32 *_used,
+		     char *output, u32 *_out_len)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u8 h2c_parameter[H2C_MAX_LENGTH] = {0};
+	u8 phydm_h2c_id = (u8)dm_value[0];
+	u8 i;
+	u32 used = *_used;
+	u32 out_len = *_out_len;
+
+	PHYDM_SNPRINTF(output + used, out_len - used,
+		       "Phydm Send H2C_ID (( 0x%x))\n", phydm_h2c_id);
+	for (i = 0; i < H2C_MAX_LENGTH; i++) {
+		h2c_parameter[i] = (u8)dm_value[i + 1];
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "H2C: Byte[%d] = ((0x%x))\n", i,
+			       h2c_parameter[i]);
+	}
+
+	odm_fill_h2c_cmd(dm, phydm_h2c_id, H2C_MAX_LENGTH, h2c_parameter);
+}
+
+void phydm_RA_debug_PCR(void *dm_void, u32 *const dm_value, u32 *_used,
+			char *output, u32 *_out_len)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct ra_table *ra_tab = &dm->dm_ra_table;
+	u32 used = *_used;
+	u32 out_len = *_out_len;
+
+	if (dm_value[0] == 100) {
+		PHYDM_SNPRINTF(
+			output + used, out_len - used,
+			"[Get] PCR RA_threshold_offset = (( %s%d ))\n",
+			((ra_tab->RA_threshold_offset == 0) ?
+				 " " :
+				 ((ra_tab->RA_offset_direction) ? "+" : "-")),
+			ra_tab->RA_threshold_offset);
+		/**/
+	} else if (dm_value[0] == 0) {
+		ra_tab->RA_offset_direction = 0;
+		ra_tab->RA_threshold_offset = (u8)dm_value[1];
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "[Set] PCR RA_threshold_offset = (( -%d ))\n",
+			       ra_tab->RA_threshold_offset);
+	} else if (dm_value[0] == 1) {
+		ra_tab->RA_offset_direction = 1;
+		ra_tab->RA_threshold_offset = (u8)dm_value[1];
+		PHYDM_SNPRINTF(output + used, out_len - used,
+			       "[Set] PCR RA_threshold_offset = (( +%d ))\n",
+			       ra_tab->RA_threshold_offset);
+	} else {
+		PHYDM_SNPRINTF(output + used, out_len - used, "[Set] Error\n");
+		/**/
+	}
+}
+
+void odm_c2h_ra_para_report_handler(void *dm_void, u8 *cmd_buf, u8 cmd_len)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	u8 para_idx = cmd_buf[0]; /*Retry Penalty, NH, NL*/
+	u8 i;
+
+	ODM_RT_TRACE(dm, PHYDM_COMP_RA_DBG,
+		     "[ From FW C2H RA Para ]  cmd_buf[0]= (( %d ))\n",
+		     cmd_buf[0]);
+
+	if (para_idx == RADBG_DEBUG_MONITOR1) {
+		ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE,
+			     "-------------------------------\n");
+		if (dm->support_ic_type & PHYDM_IC_3081_SERIES) {
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  %d\n",
+				     "RSSI =", cmd_buf[1]);
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  0x%x\n",
+				     "rate =", cmd_buf[2] & 0x7f);
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  %d\n",
+				     "SGI =", (cmd_buf[2] & 0x80) >> 7);
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  %d\n",
+				     "BW =", cmd_buf[3]);
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  %d\n",
+				     "BW_max =", cmd_buf[4]);
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  0x%x\n",
+				     "multi_rate0 =", cmd_buf[5]);
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  0x%x\n",
+				     "multi_rate1 =", cmd_buf[6]);
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  %d\n",
+				     "DISRA =", cmd_buf[7]);
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  %d\n",
+				     "VHT_EN =", cmd_buf[8]);
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  %d\n",
+				     "SGI_support =", cmd_buf[9]);
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  %d\n",
+				     "try_ness =", cmd_buf[10]);
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  0x%x\n",
+				     "pre_rate =", cmd_buf[11]);
+		} else {
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  %d\n",
+				     "RSSI =", cmd_buf[1]);
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  %x\n",
+				     "BW =", cmd_buf[2]);
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  %d\n",
+				     "DISRA =", cmd_buf[3]);
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  %d\n",
+				     "VHT_EN =", cmd_buf[4]);
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  %d\n",
+				     "Hightest rate =", cmd_buf[5]);
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  0x%x\n",
+				     "Lowest rate =", cmd_buf[6]);
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  0x%x\n",
+				     "SGI_support =", cmd_buf[7]);
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  %d\n",
+				     "Rate_ID =", cmd_buf[8]);
+			;
+		}
+		ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE,
+			     "-------------------------------\n");
+	} else if (para_idx == RADBG_DEBUG_MONITOR2) {
+		ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE,
+			     "-------------------------------\n");
+		if (dm->support_ic_type & PHYDM_IC_3081_SERIES) {
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  %d\n",
+				     "rate_id =", cmd_buf[1]);
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  0x%x\n",
+				     "highest_rate =", cmd_buf[2]);
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  0x%x\n",
+				     "lowest_rate =", cmd_buf[3]);
+
+			for (i = 4; i <= 11; i++)
+				ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE,
+					     "RAMASK =  0x%x\n", cmd_buf[i]);
+		} else {
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE,
+				     "%5s  %x%x  %x%x  %x%x  %x%x\n",
+				     "RA Mask:", cmd_buf[8], cmd_buf[7],
+				     cmd_buf[6], cmd_buf[5], cmd_buf[4],
+				     cmd_buf[3], cmd_buf[2], cmd_buf[1]);
+		}
+		ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE,
+			     "-------------------------------\n");
+	} else if (para_idx == RADBG_DEBUG_MONITOR3) {
+		for (i = 0; i < (cmd_len - 1); i++)
+			ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE,
+				     "content[%d] = %d\n", i, cmd_buf[1 + i]);
+	} else if (para_idx == RADBG_DEBUG_MONITOR4) {
+		ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  {%d.%d}\n",
+			     "RA version =", cmd_buf[1], cmd_buf[2]);
+	} else if (para_idx == RADBG_DEBUG_MONITOR5) {
+		ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  0x%x\n",
+			     "Current rate =", cmd_buf[1]);
+		ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  %d\n",
+			     "Retry ratio =", cmd_buf[2]);
+		ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  %d\n",
+			     "rate down ratio =", cmd_buf[3]);
+		ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  0x%x\n",
+			     "highest rate =", cmd_buf[4]);
+		ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  {0x%x 0x%x}\n",
+			     "Muti-try =", cmd_buf[5], cmd_buf[6]);
+		ODM_RT_TRACE(dm, ODM_FW_DEBUG_TRACE, "%5s  0x%x%x%x%x%x\n",
+			     "RA mask =", cmd_buf[11], cmd_buf[10], cmd_buf[9],
+			     cmd_buf[8], cmd_buf[7]);
+	}
+}
+
+void phydm_ra_dynamic_retry_count(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	if (!(dm->support_ability & ODM_BB_DYNAMIC_ARFR))
+		return;
+
+	if (dm->pre_b_noisy != dm->noisy_decision) {
+		if (dm->noisy_decision) {
+			ODM_RT_TRACE(dm, ODM_COMP_RATE_ADAPTIVE,
+				     "->Noisy Env. RA fallback value\n");
+			odm_set_mac_reg(dm, 0x430, MASKDWORD, 0x0);
+			odm_set_mac_reg(dm, 0x434, MASKDWORD, 0x04030201);
+		} else {
+			ODM_RT_TRACE(dm, ODM_COMP_RATE_ADAPTIVE,
+				     "->Clean Env. RA fallback value\n");
+			odm_set_mac_reg(dm, 0x430, MASKDWORD, 0x01000000);
+			odm_set_mac_reg(dm, 0x434, MASKDWORD, 0x06050402);
+		}
+		dm->pre_b_noisy = dm->noisy_decision;
+	}
+}
+
+void phydm_ra_dynamic_retry_limit(void *dm_void) {}
+
+void phydm_print_rate(void *dm_void, u8 rate, u32 dbg_component)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u8 legacy_table[12] = {1, 2, 5, 11, 6, 9, 12, 18, 24, 36, 48, 54};
+	u8 rate_idx = rate & 0x7f; /*remove bit7 SGI*/
+	u8 vht_en = (rate_idx >= ODM_RATEVHTSS1MCS0) ? 1 : 0;
+	u8 b_sgi = (rate & 0x80) >> 7;
+
+	ODM_RT_TRACE(dm, dbg_component, "( %s%s%s%s%d%s%s)\n",
+		     ((rate_idx >= ODM_RATEVHTSS1MCS0) &&
+		      (rate_idx <= ODM_RATEVHTSS1MCS9)) ?
+			     "VHT 1ss  " :
+			     "",
+		     ((rate_idx >= ODM_RATEVHTSS2MCS0) &&
+		      (rate_idx <= ODM_RATEVHTSS2MCS9)) ?
+			     "VHT 2ss " :
+			     "",
+		     ((rate_idx >= ODM_RATEVHTSS3MCS0) &&
+		      (rate_idx <= ODM_RATEVHTSS3MCS9)) ?
+			     "VHT 3ss " :
+			     "",
+		     (rate_idx >= ODM_RATEMCS0) ? "MCS " : "",
+		     (vht_en) ? ((rate_idx - ODM_RATEVHTSS1MCS0) % 10) :
+				((rate_idx >= ODM_RATEMCS0) ?
+					 (rate_idx - ODM_RATEMCS0) :
+					 ((rate_idx <= ODM_RATE54M) ?
+						  legacy_table[rate_idx] :
+						  0)),
+		     (b_sgi) ? "-S" : "  ",
+		     (rate_idx >= ODM_RATEMCS0) ? "" : "M");
+}
+
+void phydm_c2h_ra_report_handler(void *dm_void, u8 *cmd_buf, u8 cmd_len)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct ra_table *ra_tab = &dm->dm_ra_table;
+	u8 macid = cmd_buf[1];
+	u8 rate = cmd_buf[0];
+	u8 rate_idx = rate & 0x7f; /*remove bit7 SGI*/
+	u8 rate_order;
+
+	if (cmd_len >= 4) {
+		if (cmd_buf[3] == 0) {
+			ODM_RT_TRACE(dm, ODM_COMP_RATE_ADAPTIVE,
+				     "TX Init-rate Update[%d]:", macid);
+			/**/
+		} else if (cmd_buf[3] == 0xff) {
+			ODM_RT_TRACE(dm, ODM_COMP_RATE_ADAPTIVE,
+				     "FW Level: Fix rate[%d]:", macid);
+			/**/
+		} else if (cmd_buf[3] == 1) {
+			ODM_RT_TRACE(dm, ODM_COMP_RATE_ADAPTIVE,
+				     "Try Success[%d]:", macid);
+			/**/
+		} else if (cmd_buf[3] == 2) {
+			ODM_RT_TRACE(dm, ODM_COMP_RATE_ADAPTIVE,
+				     "Try Fail & Try Again[%d]:", macid);
+			/**/
+		} else if (cmd_buf[3] == 3) {
+			ODM_RT_TRACE(dm, ODM_COMP_RATE_ADAPTIVE,
+				     "rate Back[%d]:", macid);
+			/**/
+		} else if (cmd_buf[3] == 4) {
+			ODM_RT_TRACE(dm, ODM_COMP_RATE_ADAPTIVE,
+				     "start rate by RSSI[%d]:", macid);
+			/**/
+		} else if (cmd_buf[3] == 5) {
+			ODM_RT_TRACE(dm, ODM_COMP_RATE_ADAPTIVE,
+				     "Try rate[%d]:", macid);
+			/**/
+		}
+	} else {
+		ODM_RT_TRACE(dm, ODM_COMP_RATE_ADAPTIVE, "Tx rate Update[%d]:",
+			     macid);
+		/**/
+	}
+
+	phydm_print_rate(dm, rate, ODM_COMP_RATE_ADAPTIVE);
+
+	ra_tab->link_tx_rate[macid] = rate;
+
+	/*trigger power training*/
+
+	rate_order = phydm_rate_order_compute(dm, rate_idx);
+
+	if ((dm->is_one_entry_only) ||
+	    ((rate_order > ra_tab->highest_client_tx_order) &&
+	     (ra_tab->power_tracking_flag == 1))) {
+		phydm_update_pwr_track(dm, rate_idx);
+		ra_tab->power_tracking_flag = 0;
+	}
+
+	/*trigger dynamic rate ID*/
+}
+
+void odm_rssi_monitor_init(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct ra_table *ra_tab = &dm->dm_ra_table;
+
+	ra_tab->firstconnect = false;
+}
+
+void odm_ra_post_action_on_assoc(void *dm_void) {}
+
+void phydm_init_ra_info(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	if (dm->support_ic_type == ODM_RTL8822B) {
+		u32 ret_value;
+
+		ret_value = odm_get_bb_reg(dm, 0x4c8, MASKBYTE2);
+		odm_set_bb_reg(dm, 0x4cc, MASKBYTE3, (ret_value - 1));
+	}
+}
+
+void phydm_modify_RA_PCR_threshold(void *dm_void, u8 RA_offset_direction,
+				   u8 RA_threshold_offset
+
+				   )
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct ra_table *ra_tab = &dm->dm_ra_table;
+
+	ra_tab->RA_offset_direction = RA_offset_direction;
+	ra_tab->RA_threshold_offset = RA_threshold_offset;
+	ODM_RT_TRACE(dm, ODM_COMP_RA_MASK,
+		     "Set RA_threshold_offset = (( %s%d ))\n",
+		     ((RA_threshold_offset == 0) ?
+			      " " :
+			      ((RA_offset_direction) ? "+" : "-")),
+		     RA_threshold_offset);
+}
+
+static void odm_rssi_monitor_check_mp(void *dm_void) {}
+
+static void odm_rssi_monitor_check_ce(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct ra_table *ra_tab = &dm->dm_ra_table;
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)dm->adapter;
+	struct rtl_mac *mac = rtl_mac(rtlpriv);
+	struct rtl_sta_info *entry;
+	int i;
+	int tmp_entry_min_pwdb = 0xff;
+	unsigned long cur_tx_ok_cnt = 0, cur_rx_ok_cnt = 0;
+	u8 UL_DL_STATE = 0, STBC_TX = 0, tx_bf_en = 0;
+	u8 h2c_parameter[H2C_0X42_LENGTH] = {0};
+	u8 cmdlen = H2C_0X42_LENGTH;
+	u8 macid = 0;
+
+	if (!dm->is_linked)
+		return;
+
+	for (i = 0; i < ODM_ASSOCIATE_ENTRY_NUM; i++) {
+		entry = (struct rtl_sta_info *)dm->odm_sta_info[i];
+		if (!IS_STA_VALID(entry))
+			continue;
+
+		if (is_multicast_ether_addr(entry->mac_addr) ||
+		    is_broadcast_ether_addr(entry->mac_addr))
+			continue;
+
+		if (entry->rssi_stat.undecorated_smoothed_pwdb == (-1))
+			continue;
+
+		/* calculate min_pwdb */
+		if (entry->rssi_stat.undecorated_smoothed_pwdb <
+		    tmp_entry_min_pwdb)
+			tmp_entry_min_pwdb =
+				entry->rssi_stat.undecorated_smoothed_pwdb;
+
+		/* report RSSI */
+		cur_tx_ok_cnt = rtlpriv->stats.txbytesunicast_inperiod;
+		cur_rx_ok_cnt = rtlpriv->stats.rxbytesunicast_inperiod;
+
+		if (cur_rx_ok_cnt > (cur_tx_ok_cnt * 6))
+			UL_DL_STATE = 1;
+		else
+			UL_DL_STATE = 0;
+
+		if (mac->opmode == NL80211_IFTYPE_AP ||
+		    mac->opmode == NL80211_IFTYPE_ADHOC) {
+			struct ieee80211_sta *sta = container_of(
+				(void *)entry, struct ieee80211_sta, drv_priv);
+			macid = sta->aid + 1;
+		}
+
+		h2c_parameter[0] = macid;
+		h2c_parameter[2] =
+			entry->rssi_stat.undecorated_smoothed_pwdb & 0x7F;
+
+		if (UL_DL_STATE)
+			h2c_parameter[3] |= RAINFO_BE_RX_STATE;
+
+		if (tx_bf_en)
+			h2c_parameter[3] |= RAINFO_BF_STATE;
+		if (STBC_TX)
+			h2c_parameter[3] |= RAINFO_STBC_STATE;
+		if (dm->noisy_decision)
+			h2c_parameter[3] |= RAINFO_NOISY_STATE;
+
+		if (entry->rssi_stat.is_send_rssi == RA_RSSI_STATE_SEND) {
+			h2c_parameter[3] |= RAINFO_INIT_RSSI_RATE_STATE;
+			entry->rssi_stat.is_send_rssi = RA_RSSI_STATE_HOLD;
+		}
+
+		h2c_parameter[4] = (ra_tab->RA_threshold_offset & 0x7f) |
+				   (ra_tab->RA_offset_direction << 7);
+
+		odm_fill_h2c_cmd(dm, ODM_H2C_RSSI_REPORT, cmdlen,
+				 h2c_parameter);
+	}
+
+	if (tmp_entry_min_pwdb != 0xff)
+		dm->rssi_min = tmp_entry_min_pwdb;
+}
+
+static void odm_rssi_monitor_check_ap(void *dm_void) {}
+
+void odm_rssi_monitor_check(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	if (!(dm->support_ability & ODM_BB_RSSI_MONITOR))
+		return;
+
+	switch (dm->support_platform) {
+	case ODM_WIN:
+		odm_rssi_monitor_check_mp(dm);
+		break;
+
+	case ODM_CE:
+		odm_rssi_monitor_check_ce(dm);
+		break;
+
+	case ODM_AP:
+		odm_rssi_monitor_check_ap(dm);
+		break;
+
+	default:
+		break;
+	}
+}
+
+void odm_rate_adaptive_mask_init(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct odm_rate_adaptive *odm_ra = &dm->rate_adaptive;
+
+	odm_ra->type = dm_type_by_driver;
+	if (odm_ra->type == dm_type_by_driver)
+		dm->is_use_ra_mask = true;
+	else
+		dm->is_use_ra_mask = false;
+
+	odm_ra->ratr_state = DM_RATR_STA_INIT;
+
+	odm_ra->ldpc_thres = 35;
+	odm_ra->is_use_ldpc = false;
+
+	odm_ra->high_rssi_thresh = 50;
+	odm_ra->low_rssi_thresh = 20;
+}
+
+/*-----------------------------------------------------------------------------
+ * Function:	odm_refresh_rate_adaptive_mask()
+ *
+ * Overview:	Update rate table mask according to rssi
+ *
+ * Input:		NONE
+ *
+ * Output:		NONE
+ *
+ * Return:		NONE
+ *
+ * Revised History:
+ *	When		Who		Remark
+ *	05/27/2009	hpfan	Create version 0.
+ *
+ *---------------------------------------------------------------------------
+ */
+void odm_refresh_rate_adaptive_mask(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct ra_table *ra_tab = &dm->dm_ra_table;
+
+	if (!dm->is_linked)
+		return;
+
+	if (!(dm->support_ability & ODM_BB_RA_MASK)) {
+		ODM_RT_TRACE(dm, ODM_COMP_RA_MASK,
+			     "%s(): Return cos not supported\n", __func__);
+		return;
+	}
+
+	ra_tab->force_update_ra_mask_count++;
+	/* 2011/09/29 MH In HW integration first stage, we provide 4 different
+	 * handle to operate at the same time.
+	 * In the stage2/3, we need to prive universal interface and merge all
+	 * HW dynamic mechanism.
+	 */
+	switch (dm->support_platform) {
+	case ODM_WIN:
+		odm_refresh_rate_adaptive_mask_mp(dm);
+		break;
+
+	case ODM_CE:
+		odm_refresh_rate_adaptive_mask_ce(dm);
+		break;
+
+	case ODM_AP:
+		odm_refresh_rate_adaptive_mask_apadsl(dm);
+		break;
+	}
+}
+
+static u8 phydm_trans_platform_bw(void *dm_void, u8 BW)
+{
+	if (BW == HT_CHANNEL_WIDTH_20)
+		BW = PHYDM_BW_20;
+
+	else if (BW == HT_CHANNEL_WIDTH_20_40)
+		BW = PHYDM_BW_40;
+
+	else if (BW == HT_CHANNEL_WIDTH_80)
+		BW = PHYDM_BW_80;
+
+	return BW;
+}
+
+static u8 phydm_trans_platform_rf_type(void *dm_void, u8 rf_type)
+{
+	if (rf_type == RF_1T2R)
+		rf_type = PHYDM_RF_1T2R;
+
+	else if (rf_type == RF_2T4R)
+		rf_type = PHYDM_RF_2T4R;
+
+	else if (rf_type == RF_2T2R)
+		rf_type = PHYDM_RF_2T2R;
+
+	else if (rf_type == RF_1T1R)
+		rf_type = PHYDM_RF_1T1R;
+
+	else if (rf_type == RF_2T2R_GREEN)
+		rf_type = PHYDM_RF_2T2R_GREEN;
+
+	else if (rf_type == RF_3T3R)
+		rf_type = PHYDM_RF_3T3R;
+
+	else if (rf_type == RF_4T4R)
+		rf_type = PHYDM_RF_4T4R;
+
+	else if (rf_type == RF_2T3R)
+		rf_type = PHYDM_RF_1T2R;
+
+	else if (rf_type == RF_3T4R)
+		rf_type = PHYDM_RF_3T4R;
+
+	return rf_type;
+}
+
+static u32 phydm_trans_platform_wireless_mode(void *dm_void, u32 wireless_mode)
+{
+	return wireless_mode;
+}
+
+u8 phydm_vht_en_mapping(void *dm_void, u32 wireless_mode)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u8 vht_en_out = 0;
+
+	if ((wireless_mode == PHYDM_WIRELESS_MODE_AC_5G) ||
+	    (wireless_mode == PHYDM_WIRELESS_MODE_AC_24G) ||
+	    (wireless_mode == PHYDM_WIRELESS_MODE_AC_ONLY)) {
+		vht_en_out = 1;
+		/**/
+	}
+
+	ODM_RT_TRACE(dm, ODM_COMP_RA_MASK,
+		     "wireless_mode= (( 0x%x )), VHT_EN= (( %d ))\n",
+		     wireless_mode, vht_en_out);
+	return vht_en_out;
+}
+
+u8 phydm_rate_id_mapping(void *dm_void, u32 wireless_mode, u8 rf_type, u8 bw)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u8 rate_id_idx = 0;
+	u8 phydm_BW;
+	u8 phydm_rf_type;
+
+	phydm_BW = phydm_trans_platform_bw(dm, bw);
+	phydm_rf_type = phydm_trans_platform_rf_type(dm, rf_type);
+	wireless_mode = phydm_trans_platform_wireless_mode(dm, wireless_mode);
+
+	ODM_RT_TRACE(
+		dm, ODM_COMP_RA_MASK,
+		"wireless_mode= (( 0x%x )), rf_type = (( 0x%x )), BW = (( 0x%x ))\n",
+		wireless_mode, phydm_rf_type, phydm_BW);
+
+	switch (wireless_mode) {
+	case PHYDM_WIRELESS_MODE_N_24G: {
+		if (phydm_BW == PHYDM_BW_40) {
+			if (phydm_rf_type == PHYDM_RF_1T1R)
+				rate_id_idx = PHYDM_BGN_40M_1SS;
+			else if (phydm_rf_type == PHYDM_RF_2T2R)
+				rate_id_idx = PHYDM_BGN_40M_2SS;
+			else
+				rate_id_idx = PHYDM_ARFR5_N_3SS;
+
+		} else {
+			if (phydm_rf_type == PHYDM_RF_1T1R)
+				rate_id_idx = PHYDM_BGN_20M_1SS;
+			else if (phydm_rf_type == PHYDM_RF_2T2R)
+				rate_id_idx = PHYDM_BGN_20M_2SS;
+			else
+				rate_id_idx = PHYDM_ARFR5_N_3SS;
+		}
+	} break;
+
+	case PHYDM_WIRELESS_MODE_N_5G: {
+		if (phydm_rf_type == PHYDM_RF_1T1R)
+			rate_id_idx = PHYDM_GN_N1SS;
+		else if (phydm_rf_type == PHYDM_RF_2T2R)
+			rate_id_idx = PHYDM_GN_N2SS;
+		else
+			rate_id_idx = PHYDM_ARFR5_N_3SS;
+	}
+
+	break;
+
+	case PHYDM_WIRELESS_MODE_G:
+		rate_id_idx = PHYDM_BG;
+		break;
+
+	case PHYDM_WIRELESS_MODE_A:
+		rate_id_idx = PHYDM_G;
+		break;
+
+	case PHYDM_WIRELESS_MODE_B:
+		rate_id_idx = PHYDM_B_20M;
+		break;
+
+	case PHYDM_WIRELESS_MODE_AC_5G:
+	case PHYDM_WIRELESS_MODE_AC_ONLY: {
+		if (phydm_rf_type == PHYDM_RF_1T1R)
+			rate_id_idx = PHYDM_ARFR1_AC_1SS;
+		else if (phydm_rf_type == PHYDM_RF_2T2R)
+			rate_id_idx = PHYDM_ARFR0_AC_2SS;
+		else
+			rate_id_idx = PHYDM_ARFR4_AC_3SS;
+	} break;
+
+	case PHYDM_WIRELESS_MODE_AC_24G: {
+		/*Becareful to set "Lowest rate" while using PHYDM_ARFR4_AC_3SS
+		 *in 2.4G/5G
+		 */
+		if (phydm_BW >= PHYDM_BW_80) {
+			if (phydm_rf_type == PHYDM_RF_1T1R)
+				rate_id_idx = PHYDM_ARFR1_AC_1SS;
+			else if (phydm_rf_type == PHYDM_RF_2T2R)
+				rate_id_idx = PHYDM_ARFR0_AC_2SS;
+			else
+				rate_id_idx = PHYDM_ARFR4_AC_3SS;
+		} else {
+			if (phydm_rf_type == PHYDM_RF_1T1R)
+				rate_id_idx = PHYDM_ARFR2_AC_2G_1SS;
+			else if (phydm_rf_type == PHYDM_RF_2T2R)
+				rate_id_idx = PHYDM_ARFR3_AC_2G_2SS;
+			else
+				rate_id_idx = PHYDM_ARFR4_AC_3SS;
+		}
+	} break;
+
+	default:
+		rate_id_idx = 0;
+		break;
+	}
+
+	ODM_RT_TRACE(dm, ODM_COMP_RA_MASK, "RA rate ID = (( 0x%x ))\n",
+		     rate_id_idx);
+
+	return rate_id_idx;
+}
+
+void phydm_update_hal_ra_mask(void *dm_void, u32 wireless_mode, u8 rf_type,
+			      u8 BW, u8 mimo_ps_enable, u8 disable_cck_rate,
+			      u32 *ratr_bitmap_msb_in, u32 *ratr_bitmap_lsb_in,
+			      u8 tx_rate_level)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u8 phydm_rf_type;
+	u8 phydm_BW;
+	u32 ratr_bitmap = *ratr_bitmap_lsb_in,
+	    ratr_bitmap_msb = *ratr_bitmap_msb_in;
+
+	wireless_mode = phydm_trans_platform_wireless_mode(dm, wireless_mode);
+
+	phydm_rf_type = phydm_trans_platform_rf_type(dm, rf_type);
+	phydm_BW = phydm_trans_platform_bw(dm, BW);
+
+	ODM_RT_TRACE(dm, ODM_COMP_RA_MASK,
+		     "Platfoem original RA Mask = (( 0x %x | %x ))\n",
+		     ratr_bitmap_msb, ratr_bitmap);
+
+	switch (wireless_mode) {
+	case PHYDM_WIRELESS_MODE_B: {
+		ratr_bitmap &= 0x0000000f;
+	} break;
+
+	case PHYDM_WIRELESS_MODE_G: {
+		ratr_bitmap &= 0x00000ff5;
+	} break;
+
+	case PHYDM_WIRELESS_MODE_A: {
+		ratr_bitmap &= 0x00000ff0;
+	} break;
+
+	case PHYDM_WIRELESS_MODE_N_24G:
+	case PHYDM_WIRELESS_MODE_N_5G: {
+		if (mimo_ps_enable)
+			phydm_rf_type = PHYDM_RF_1T1R;
+
+		if (phydm_rf_type == PHYDM_RF_1T1R) {
+			if (phydm_BW == PHYDM_BW_40)
+				ratr_bitmap &= 0x000ff015;
+			else
+				ratr_bitmap &= 0x000ff005;
+		} else if (phydm_rf_type == PHYDM_RF_2T2R ||
+			   phydm_rf_type == PHYDM_RF_2T4R ||
+			   phydm_rf_type == PHYDM_RF_2T3R) {
+			if (phydm_BW == PHYDM_BW_40)
+				ratr_bitmap &= 0x0ffff015;
+			else
+				ratr_bitmap &= 0x0ffff005;
+		} else { /*3T*/
+
+			ratr_bitmap &= 0xfffff015;
+			ratr_bitmap_msb &= 0xf;
+		}
+	} break;
+
+	case PHYDM_WIRELESS_MODE_AC_24G: {
+		if (phydm_rf_type == PHYDM_RF_1T1R) {
+			ratr_bitmap &= 0x003ff015;
+		} else if (phydm_rf_type == PHYDM_RF_2T2R ||
+			   phydm_rf_type == PHYDM_RF_2T4R ||
+			   phydm_rf_type == PHYDM_RF_2T3R) {
+			ratr_bitmap &= 0xfffff015;
+		} else { /*3T*/
+
+			ratr_bitmap &= 0xfffff010;
+			ratr_bitmap_msb &= 0x3ff;
+		}
+
+		if (phydm_BW ==
+		    PHYDM_BW_20) { /* AC 20MHz doesn't support MCS9 */
+			ratr_bitmap &= 0x7fdfffff;
+			ratr_bitmap_msb &= 0x1ff;
+		}
+	} break;
+
+	case PHYDM_WIRELESS_MODE_AC_5G: {
+		if (phydm_rf_type == PHYDM_RF_1T1R) {
+			ratr_bitmap &= 0x003ff010;
+		} else if (phydm_rf_type == PHYDM_RF_2T2R ||
+			   phydm_rf_type == PHYDM_RF_2T4R ||
+			   phydm_rf_type == PHYDM_RF_2T3R) {
+			ratr_bitmap &= 0xfffff010;
+		} else { /*3T*/
+
+			ratr_bitmap &= 0xfffff010;
+			ratr_bitmap_msb &= 0x3ff;
+		}
+
+		if (phydm_BW ==
+		    PHYDM_BW_20) { /* AC 20MHz doesn't support MCS9 */
+			ratr_bitmap &= 0x7fdfffff;
+			ratr_bitmap_msb &= 0x1ff;
+		}
+	} break;
+
+	default:
+		break;
+	}
+
+	if (wireless_mode != PHYDM_WIRELESS_MODE_B) {
+		if (tx_rate_level == 0)
+			ratr_bitmap &= 0xffffffff;
+		else if (tx_rate_level == 1)
+			ratr_bitmap &= 0xfffffff0;
+		else if (tx_rate_level == 2)
+			ratr_bitmap &= 0xffffefe0;
+		else if (tx_rate_level == 3)
+			ratr_bitmap &= 0xffffcfc0;
+		else if (tx_rate_level == 4)
+			ratr_bitmap &= 0xffff8f80;
+		else if (tx_rate_level >= 5)
+			ratr_bitmap &= 0xffff0f00;
+	}
+
+	if (disable_cck_rate)
+		ratr_bitmap &= 0xfffffff0;
+
+	ODM_RT_TRACE(
+		dm, ODM_COMP_RA_MASK,
+		"wireless_mode= (( 0x%x )), rf_type = (( 0x%x )), BW = (( 0x%x )), MimoPs_en = (( %d )), tx_rate_level= (( 0x%x ))\n",
+		wireless_mode, phydm_rf_type, phydm_BW, mimo_ps_enable,
+		tx_rate_level);
+
+	*ratr_bitmap_lsb_in = ratr_bitmap;
+	*ratr_bitmap_msb_in = ratr_bitmap_msb;
+	ODM_RT_TRACE(dm, ODM_COMP_RA_MASK,
+		     "Phydm modified RA Mask = (( 0x %x | %x ))\n",
+		     *ratr_bitmap_msb_in, *ratr_bitmap_lsb_in);
+}
+
+u8 phydm_RA_level_decision(void *dm_void, u32 rssi, u8 ratr_state)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u8 ra_rate_floor_table[RA_FLOOR_TABLE_SIZE] = {
+		20, 34, 38, 42,
+		46, 50, 100}; /*MCS0 ~ MCS4 , VHT1SS MCS0 ~ MCS4 , G 6M~24M*/
+	u8 new_ratr_state = 0;
+	u8 i;
+
+	ODM_RT_TRACE(
+		dm, ODM_COMP_RA_MASK,
+		"curr RA level = ((%d)), Rate_floor_table ori [ %d , %d, %d , %d, %d, %d]\n",
+		ratr_state, ra_rate_floor_table[0], ra_rate_floor_table[1],
+		ra_rate_floor_table[2], ra_rate_floor_table[3],
+		ra_rate_floor_table[4], ra_rate_floor_table[5]);
+
+	for (i = 0; i < RA_FLOOR_TABLE_SIZE; i++) {
+		if (i >= (ratr_state))
+			ra_rate_floor_table[i] += RA_FLOOR_UP_GAP;
+	}
+
+	ODM_RT_TRACE(
+		dm, ODM_COMP_RA_MASK,
+		"RSSI = ((%d)), Rate_floor_table_mod [ %d , %d, %d , %d, %d, %d]\n",
+		rssi, ra_rate_floor_table[0], ra_rate_floor_table[1],
+		ra_rate_floor_table[2], ra_rate_floor_table[3],
+		ra_rate_floor_table[4], ra_rate_floor_table[5]);
+
+	for (i = 0; i < RA_FLOOR_TABLE_SIZE; i++) {
+		if (rssi < ra_rate_floor_table[i]) {
+			new_ratr_state = i;
+			break;
+		}
+	}
+
+	return new_ratr_state;
+}
+
+void odm_refresh_rate_adaptive_mask_mp(void *dm_void) {}
+
+void odm_refresh_rate_adaptive_mask_ce(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct ra_table *ra_tab = &dm->dm_ra_table;
+	void *adapter = dm->adapter;
+	u32 i;
+	struct rtl_sta_info *entry;
+	u8 ratr_state_new;
+
+	if (!dm->is_use_ra_mask) {
+		ODM_RT_TRACE(
+			dm, ODM_COMP_RA_MASK,
+			"<---- %s(): driver does not control rate adaptive mask\n",
+			__func__);
+		return;
+	}
+
+	for (i = 0; i < ODM_ASSOCIATE_ENTRY_NUM; i++) {
+		entry = dm->odm_sta_info[i];
+
+		if (!IS_STA_VALID(entry))
+			continue;
+
+		if (is_multicast_ether_addr(entry->mac_addr))
+			continue;
+		else if (is_broadcast_ether_addr(entry->mac_addr))
+			continue;
+
+		ratr_state_new = phydm_RA_level_decision(
+			dm, entry->rssi_stat.undecorated_smoothed_pwdb,
+			entry->rssi_level);
+
+		if ((entry->rssi_level != ratr_state_new) ||
+		    (ra_tab->force_update_ra_mask_count >=
+		     FORCED_UPDATE_RAMASK_PERIOD)) {
+			ra_tab->force_update_ra_mask_count = 0;
+			ODM_RT_TRACE(
+				dm, ODM_COMP_RA_MASK,
+				"Update Tx RA Level: ((%x)) -> ((%x)),  RSSI = ((%d))\n",
+				entry->rssi_level, ratr_state_new,
+				entry->rssi_stat.undecorated_smoothed_pwdb);
+
+			entry->rssi_level = ratr_state_new;
+			rtl_hal_update_ra_mask(adapter, entry,
+					       entry->rssi_level);
+		} else {
+			ODM_RT_TRACE(dm, ODM_COMP_RA_MASK,
+				     "Stay in RA level  = (( %d ))\n\n",
+				     ratr_state_new);
+			/**/
+		}
+	}
+}
+
+void odm_refresh_rate_adaptive_mask_apadsl(void *dm_void) {}
+
+void odm_refresh_basic_rate_mask(void *dm_void) {}
+
+u8 phydm_rate_order_compute(void *dm_void, u8 rate_idx)
+{
+	u8 rate_order = 0;
+
+	if (rate_idx >= ODM_RATEVHTSS4MCS0) {
+		rate_idx -= ODM_RATEVHTSS4MCS0;
+		/**/
+	} else if (rate_idx >= ODM_RATEVHTSS3MCS0) {
+		rate_idx -= ODM_RATEVHTSS3MCS0;
+		/**/
+	} else if (rate_idx >= ODM_RATEVHTSS2MCS0) {
+		rate_idx -= ODM_RATEVHTSS2MCS0;
+		/**/
+	} else if (rate_idx >= ODM_RATEVHTSS1MCS0) {
+		rate_idx -= ODM_RATEVHTSS1MCS0;
+		/**/
+	} else if (rate_idx >= ODM_RATEMCS24) {
+		rate_idx -= ODM_RATEMCS24;
+		/**/
+	} else if (rate_idx >= ODM_RATEMCS16) {
+		rate_idx -= ODM_RATEMCS16;
+		/**/
+	} else if (rate_idx >= ODM_RATEMCS8) {
+		rate_idx -= ODM_RATEMCS8;
+		/**/
+	}
+	rate_order = rate_idx;
+
+	return rate_order;
+}
+
+static void phydm_ra_common_info_update(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct ra_table *ra_tab = &dm->dm_ra_table;
+	u16 macid;
+	u8 rate_order_tmp;
+	u8 cnt = 0;
+
+	ra_tab->highest_client_tx_order = 0;
+	ra_tab->power_tracking_flag = 1;
+
+	if (dm->number_linked_client != 0) {
+		for (macid = 0; macid < ODM_ASSOCIATE_ENTRY_NUM; macid++) {
+			rate_order_tmp = phydm_rate_order_compute(
+				dm, ((ra_tab->link_tx_rate[macid]) & 0x7f));
+
+			if (rate_order_tmp >=
+			    (ra_tab->highest_client_tx_order)) {
+				ra_tab->highest_client_tx_order =
+					rate_order_tmp;
+				ra_tab->highest_client_tx_rate_order = macid;
+			}
+
+			cnt++;
+
+			if (cnt == dm->number_linked_client)
+				break;
+		}
+		ODM_RT_TRACE(
+			dm, ODM_COMP_RATE_ADAPTIVE,
+			"MACID[%d], Highest Tx order Update for power traking: %d\n",
+			(ra_tab->highest_client_tx_rate_order),
+			(ra_tab->highest_client_tx_order));
+	}
+}
+
+void phydm_ra_info_watchdog(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	phydm_ra_common_info_update(dm);
+	phydm_ra_dynamic_retry_limit(dm);
+	phydm_ra_dynamic_retry_count(dm);
+	odm_refresh_rate_adaptive_mask(dm);
+	odm_refresh_basic_rate_mask(dm);
+}
+
+void phydm_ra_info_init(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct ra_table *ra_tab = &dm->dm_ra_table;
+
+	ra_tab->highest_client_tx_rate_order = 0;
+	ra_tab->highest_client_tx_order = 0;
+	ra_tab->RA_threshold_offset = 0;
+	ra_tab->RA_offset_direction = 0;
+}
+
+u8 odm_find_rts_rate(void *dm_void, u8 tx_rate, bool is_erp_protect)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	u8 rts_ini_rate = ODM_RATE6M;
+
+	if (is_erp_protect) { /* use CCK rate as RTS*/
+		rts_ini_rate = ODM_RATE1M;
+	} else {
+		switch (tx_rate) {
+		case ODM_RATEVHTSS3MCS9:
+		case ODM_RATEVHTSS3MCS8:
+		case ODM_RATEVHTSS3MCS7:
+		case ODM_RATEVHTSS3MCS6:
+		case ODM_RATEVHTSS3MCS5:
+		case ODM_RATEVHTSS3MCS4:
+		case ODM_RATEVHTSS3MCS3:
+		case ODM_RATEVHTSS2MCS9:
+		case ODM_RATEVHTSS2MCS8:
+		case ODM_RATEVHTSS2MCS7:
+		case ODM_RATEVHTSS2MCS6:
+		case ODM_RATEVHTSS2MCS5:
+		case ODM_RATEVHTSS2MCS4:
+		case ODM_RATEVHTSS2MCS3:
+		case ODM_RATEVHTSS1MCS9:
+		case ODM_RATEVHTSS1MCS8:
+		case ODM_RATEVHTSS1MCS7:
+		case ODM_RATEVHTSS1MCS6:
+		case ODM_RATEVHTSS1MCS5:
+		case ODM_RATEVHTSS1MCS4:
+		case ODM_RATEVHTSS1MCS3:
+		case ODM_RATEMCS15:
+		case ODM_RATEMCS14:
+		case ODM_RATEMCS13:
+		case ODM_RATEMCS12:
+		case ODM_RATEMCS11:
+		case ODM_RATEMCS7:
+		case ODM_RATEMCS6:
+		case ODM_RATEMCS5:
+		case ODM_RATEMCS4:
+		case ODM_RATEMCS3:
+		case ODM_RATE54M:
+		case ODM_RATE48M:
+		case ODM_RATE36M:
+		case ODM_RATE24M:
+			rts_ini_rate = ODM_RATE24M;
+			break;
+		case ODM_RATEVHTSS3MCS2:
+		case ODM_RATEVHTSS3MCS1:
+		case ODM_RATEVHTSS2MCS2:
+		case ODM_RATEVHTSS2MCS1:
+		case ODM_RATEVHTSS1MCS2:
+		case ODM_RATEVHTSS1MCS1:
+		case ODM_RATEMCS10:
+		case ODM_RATEMCS9:
+		case ODM_RATEMCS2:
+		case ODM_RATEMCS1:
+		case ODM_RATE18M:
+		case ODM_RATE12M:
+			rts_ini_rate = ODM_RATE12M;
+			break;
+		case ODM_RATEVHTSS3MCS0:
+		case ODM_RATEVHTSS2MCS0:
+		case ODM_RATEVHTSS1MCS0:
+		case ODM_RATEMCS8:
+		case ODM_RATEMCS0:
+		case ODM_RATE9M:
+		case ODM_RATE6M:
+			rts_ini_rate = ODM_RATE6M;
+			break;
+		case ODM_RATE11M:
+		case ODM_RATE5_5M:
+		case ODM_RATE2M:
+		case ODM_RATE1M:
+			rts_ini_rate = ODM_RATE1M;
+			break;
+		default:
+			rts_ini_rate = ODM_RATE6M;
+			break;
+		}
+	}
+
+	if (*dm->band_type == 1) {
+		if (rts_ini_rate < ODM_RATE6M)
+			rts_ini_rate = ODM_RATE6M;
+	}
+	return rts_ini_rate;
+}
+
+static void odm_set_ra_dm_arfb_by_noisy(struct phy_dm_struct *dm) {}
+
+void odm_update_noisy_state(void *dm_void, bool is_noisy_state_from_c2h)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	/* JJ ADD 20161014 */
+	if (dm->support_ic_type == ODM_RTL8821 ||
+	    dm->support_ic_type == ODM_RTL8812 ||
+	    dm->support_ic_type == ODM_RTL8723B ||
+	    dm->support_ic_type == ODM_RTL8192E ||
+	    dm->support_ic_type == ODM_RTL8188E ||
+	    dm->support_ic_type == ODM_RTL8723D ||
+	    dm->support_ic_type == ODM_RTL8710B)
+		dm->is_noisy_state = is_noisy_state_from_c2h;
+	odm_set_ra_dm_arfb_by_noisy(dm);
+};
+
+void phydm_update_pwr_track(void *dm_void, u8 rate)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	ODM_RT_TRACE(dm, ODM_COMP_TX_PWR_TRACK, "Pwr Track Get rate=0x%x\n",
+		     rate);
+
+	dm->tx_rate = rate;
+}
+
+/* RA_MASK_PHYDMLIZE, will delete it later*/
+
+bool odm_ra_state_check(void *dm_void, s32 rssi, bool is_force_update,
+			u8 *ra_tr_state)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct odm_rate_adaptive *ra = &dm->rate_adaptive;
+	const u8 go_up_gap = 5;
+	u8 high_rssi_thresh_for_ra = ra->high_rssi_thresh;
+	u8 low_rssi_thresh_for_ra = ra->low_rssi_thresh;
+	u8 ratr_state;
+
+	ODM_RT_TRACE(dm, ODM_COMP_RA_MASK,
+		     "RSSI= (( %d )), Current_RSSI_level = (( %d ))\n", rssi,
+		     *ra_tr_state);
+	ODM_RT_TRACE(dm, ODM_COMP_RA_MASK,
+		     "[Ori RA RSSI Thresh]  High= (( %d )), Low = (( %d ))\n",
+		     high_rssi_thresh_for_ra, low_rssi_thresh_for_ra);
+	/* threshold Adjustment:
+	 * when RSSI state trends to go up one or two levels, make sure RSSI is
+	 * high enough. Here go_up_gap is added to solve the boundary's level
+	 * alternation issue.
+	 */
+
+	switch (*ra_tr_state) {
+	case DM_RATR_STA_INIT:
+	case DM_RATR_STA_HIGH:
+		break;
+
+	case DM_RATR_STA_MIDDLE:
+		high_rssi_thresh_for_ra += go_up_gap;
+		break;
+
+	case DM_RATR_STA_LOW:
+		high_rssi_thresh_for_ra += go_up_gap;
+		low_rssi_thresh_for_ra += go_up_gap;
+		break;
+
+	default:
+		WARN_ONCE(true, "wrong rssi level setting %d !", *ra_tr_state);
+		break;
+	}
+
+	/* Decide ratr_state by RSSI.*/
+	if (rssi > high_rssi_thresh_for_ra)
+		ratr_state = DM_RATR_STA_HIGH;
+	else if (rssi > low_rssi_thresh_for_ra)
+		ratr_state = DM_RATR_STA_MIDDLE;
+
+	else
+		ratr_state = DM_RATR_STA_LOW;
+	ODM_RT_TRACE(dm, ODM_COMP_RA_MASK,
+		     "[Mod RA RSSI Thresh]  High= (( %d )), Low = (( %d ))\n",
+		     high_rssi_thresh_for_ra, low_rssi_thresh_for_ra);
+
+	if (*ra_tr_state != ratr_state || is_force_update) {
+		ODM_RT_TRACE(dm, ODM_COMP_RA_MASK,
+			     "[RSSI Level Update] %d->%d\n", *ra_tr_state,
+			     ratr_state);
+		*ra_tr_state = ratr_state;
+		return true;
+	}
+
+	return false;
+}
diff --git a/drivers/staging/rtlwifi/phydm/phydm_rainfo.h b/drivers/staging/rtlwifi/phydm/phydm_rainfo.h
new file mode 100644
index 000000000000..c14ed9bda0af
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_rainfo.h
@@ -0,0 +1,269 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __PHYDMRAINFO_H__
+#define __PHYDMRAINFO_H__
+
+/*#define RAINFO_VERSION	"2.0"*/ /*2014.11.04*/
+/*#define RAINFO_VERSION	"3.0"*/ /*2015.01.13 Dino*/
+/*#define RAINFO_VERSION	"3.1"*/ /*2015.01.14 Dino*/
+/*#define RAINFO_VERSION	"3.3"*/ /*2015.07.29 YuChen*/
+/*#define RAINFO_VERSION	"3.4"*/ /*2015.12.15 Stanley*/
+/*#define RAINFO_VERSION	"4.0"*/ /*2016.03.24 Dino, Add more RA mask
+					  *state and Phydm-lize partial ra mask
+					  *function
+					  */
+/*#define RAINFO_VERSION	"4.1"*/ /*2016.04.20 Dino, Add new function to
+					  *adjust PCR RA threshold
+					  */
+/*#define RAINFO_VERSION	"4.2"*/ /*2016.05.17 Dino, Add H2C debug cmd */
+#define RAINFO_VERSION "4.3" /*2016.07.11 Dino, Fix RA hang in CCK 1M problem*/
+
+#define FORCED_UPDATE_RAMASK_PERIOD 5
+
+#define H2C_0X42_LENGTH 5
+#define H2C_MAX_LENGTH 7
+
+#define RA_FLOOR_UP_GAP 3
+#define RA_FLOOR_TABLE_SIZE 7
+
+#define ACTIVE_TP_THRESHOLD 150
+#define RA_RETRY_DESCEND_NUM 2
+#define RA_RETRY_LIMIT_LOW 4
+#define RA_RETRY_LIMIT_HIGH 32
+
+#define RAINFO_BE_RX_STATE BIT(0) /* 1:RX    */ /* ULDL */
+#define RAINFO_STBC_STATE BIT(1)
+/* #define RAINFO_LDPC_STATE			BIT2 */
+#define RAINFO_NOISY_STATE BIT(2) /* set by Noisy_Detection */
+#define RAINFO_SHURTCUT_STATE BIT(3)
+#define RAINFO_SHURTCUT_FLAG BIT(4)
+#define RAINFO_INIT_RSSI_RATE_STATE BIT(5)
+#define RAINFO_BF_STATE BIT(6)
+#define RAINFO_BE_TX_STATE BIT(7) /* 1:TX */
+
+#define RA_MASK_CCK 0xf
+#define RA_MASK_OFDM 0xff0
+#define RA_MASK_HT1SS 0xff000
+#define RA_MASK_HT2SS 0xff00000
+/*#define	RA_MASK_MCS3SS	*/
+#define RA_MASK_HT4SS 0xff0
+#define RA_MASK_VHT1SS 0x3ff000
+#define RA_MASK_VHT2SS 0xffc00000
+
+#define RA_FIRST_MACID 0
+
+#define ap_init_rate_adaptive_state odm_rate_adaptive_state_ap_init
+
+#define DM_RATR_STA_INIT 0
+#define DM_RATR_STA_HIGH 1
+#define DM_RATR_STA_MIDDLE 2
+#define DM_RATR_STA_LOW 3
+#define DM_RATR_STA_ULTRA_LOW 4
+
+enum phydm_ra_arfr_num {
+	ARFR_0_RATE_ID = 0x9,
+	ARFR_1_RATE_ID = 0xa,
+	ARFR_2_RATE_ID = 0xb,
+	ARFR_3_RATE_ID = 0xc,
+	ARFR_4_RATE_ID = 0xd,
+	ARFR_5_RATE_ID = 0xe
+};
+
+enum phydm_ra_dbg_para {
+	RADBG_PCR_TH_OFFSET = 0,
+	RADBG_RTY_PENALTY = 1,
+	RADBG_N_HIGH = 2,
+	RADBG_N_LOW = 3,
+	RADBG_TRATE_UP_TABLE = 4,
+	RADBG_TRATE_DOWN_TABLE = 5,
+	RADBG_TRYING_NECESSARY = 6,
+	RADBG_TDROPING_NECESSARY = 7,
+	RADBG_RATE_UP_RTY_RATIO = 8,
+	RADBG_RATE_DOWN_RTY_RATIO = 9, /* u8 */
+
+	RADBG_DEBUG_MONITOR1 = 0xc,
+	RADBG_DEBUG_MONITOR2 = 0xd,
+	RADBG_DEBUG_MONITOR3 = 0xe,
+	RADBG_DEBUG_MONITOR4 = 0xf,
+	RADBG_DEBUG_MONITOR5 = 0x10,
+	NUM_RA_PARA
+};
+
+enum phydm_wireless_mode {
+	PHYDM_WIRELESS_MODE_UNKNOWN = 0x00,
+	PHYDM_WIRELESS_MODE_A = 0x01,
+	PHYDM_WIRELESS_MODE_B = 0x02,
+	PHYDM_WIRELESS_MODE_G = 0x04,
+	PHYDM_WIRELESS_MODE_AUTO = 0x08,
+	PHYDM_WIRELESS_MODE_N_24G = 0x10,
+	PHYDM_WIRELESS_MODE_N_5G = 0x20,
+	PHYDM_WIRELESS_MODE_AC_5G = 0x40,
+	PHYDM_WIRELESS_MODE_AC_24G = 0x80,
+	PHYDM_WIRELESS_MODE_AC_ONLY = 0x100,
+	PHYDM_WIRELESS_MODE_MAX = 0x800,
+	PHYDM_WIRELESS_MODE_ALL = 0xFFFF
+};
+
+enum phydm_rateid_idx {
+	PHYDM_BGN_40M_2SS = 0,
+	PHYDM_BGN_40M_1SS = 1,
+	PHYDM_BGN_20M_2SS = 2,
+	PHYDM_BGN_20M_1SS = 3,
+	PHYDM_GN_N2SS = 4,
+	PHYDM_GN_N1SS = 5,
+	PHYDM_BG = 6,
+	PHYDM_G = 7,
+	PHYDM_B_20M = 8,
+	PHYDM_ARFR0_AC_2SS = 9,
+	PHYDM_ARFR1_AC_1SS = 10,
+	PHYDM_ARFR2_AC_2G_1SS = 11,
+	PHYDM_ARFR3_AC_2G_2SS = 12,
+	PHYDM_ARFR4_AC_3SS = 13,
+	PHYDM_ARFR5_N_3SS = 14
+};
+
+enum phydm_rf_type_def {
+	PHYDM_RF_1T1R = 0,
+	PHYDM_RF_1T2R,
+	PHYDM_RF_2T2R,
+	PHYDM_RF_2T2R_GREEN,
+	PHYDM_RF_2T3R,
+	PHYDM_RF_2T4R,
+	PHYDM_RF_3T3R,
+	PHYDM_RF_3T4R,
+	PHYDM_RF_4T4R,
+	PHYDM_RF_MAX_TYPE
+};
+
+enum phydm_bw {
+	PHYDM_BW_20 = 0,
+	PHYDM_BW_40,
+	PHYDM_BW_80,
+	PHYDM_BW_80_80,
+	PHYDM_BW_160,
+	PHYDM_BW_10,
+	PHYDM_BW_5
+};
+
+struct ra_table {
+	u8 firstconnect;
+
+	u8 link_tx_rate[ODM_ASSOCIATE_ENTRY_NUM];
+	u8 highest_client_tx_order;
+	u16 highest_client_tx_rate_order;
+	u8 power_tracking_flag;
+	u8 RA_threshold_offset;
+	u8 RA_offset_direction;
+	u8 force_update_ra_mask_count;
+};
+
+struct odm_rate_adaptive {
+	/* dm_type_by_fw/dm_type_by_driver */
+	u8 type;
+	/* if RSSI > high_rssi_thresh	=> ratr_state is DM_RATR_STA_HIGH */
+	u8 high_rssi_thresh;
+	/* if RSSI <= low_rssi_thresh	=> ratr_state is DM_RATR_STA_LOW */
+	u8 low_rssi_thresh;
+	/* Cur RSSI level, DM_RATR_STA_HIGH/DM_RATR_STA_MIDDLE/DM_RATR_STA_LOW*/
+	u8 ratr_state;
+
+	/* if RSSI > ldpc_thres => switch from LPDC to BCC */
+	u8 ldpc_thres;
+	bool is_lower_rts_rate;
+
+	bool is_use_ldpc;
+};
+
+void phydm_h2C_debug(void *dm_void, u32 *const dm_value, u32 *_used,
+		     char *output, u32 *_out_len);
+
+void phydm_RA_debug_PCR(void *dm_void, u32 *const dm_value, u32 *_used,
+			char *output, u32 *_out_len);
+
+void odm_c2h_ra_para_report_handler(void *dm_void, u8 *cmd_buf, u8 cmd_len);
+
+void odm_ra_para_adjust(void *dm_void);
+
+void phydm_ra_dynamic_retry_count(void *dm_void);
+
+void phydm_ra_dynamic_retry_limit(void *dm_void);
+
+void phydm_ra_dynamic_rate_id_on_assoc(void *dm_void, u8 wireless_mode,
+				       u8 init_rate_id);
+
+void phydm_print_rate(void *dm_void, u8 rate, u32 dbg_component);
+
+void phydm_c2h_ra_report_handler(void *dm_void, u8 *cmd_buf, u8 cmd_len);
+
+u8 phydm_rate_order_compute(void *dm_void, u8 rate_idx);
+
+void phydm_ra_info_watchdog(void *dm_void);
+
+void phydm_ra_info_init(void *dm_void);
+
+void odm_rssi_monitor_init(void *dm_void);
+
+void phydm_modify_RA_PCR_threshold(void *dm_void, u8 RA_offset_direction,
+				   u8 RA_threshold_offset);
+
+void odm_rssi_monitor_check(void *dm_void);
+
+void phydm_init_ra_info(void *dm_void);
+
+u8 phydm_vht_en_mapping(void *dm_void, u32 wireless_mode);
+
+u8 phydm_rate_id_mapping(void *dm_void, u32 wireless_mode, u8 rf_type, u8 bw);
+
+void phydm_update_hal_ra_mask(void *dm_void, u32 wireless_mode, u8 rf_type,
+			      u8 BW, u8 mimo_ps_enable, u8 disable_cck_rate,
+			      u32 *ratr_bitmap_msb_in, u32 *ratr_bitmap_in,
+			      u8 tx_rate_level);
+
+void odm_rate_adaptive_mask_init(void *dm_void);
+
+void odm_refresh_rate_adaptive_mask(void *dm_void);
+
+void odm_refresh_rate_adaptive_mask_mp(void *dm_void);
+
+void odm_refresh_rate_adaptive_mask_ce(void *dm_void);
+
+void odm_refresh_rate_adaptive_mask_apadsl(void *dm_void);
+
+u8 phydm_RA_level_decision(void *dm_void, u32 rssi, u8 ratr_state);
+
+bool odm_ra_state_check(void *dm_void, s32 RSSI, bool is_force_update,
+			u8 *ra_tr_state);
+
+void odm_refresh_basic_rate_mask(void *dm_void);
+void odm_ra_post_action_on_assoc(void *dm);
+
+u8 odm_find_rts_rate(void *dm_void, u8 tx_rate, bool is_erp_protect);
+
+void odm_update_noisy_state(void *dm_void, bool is_noisy_state_from_c2h);
+
+void phydm_update_pwr_track(void *dm_void, u8 rate);
+
+#endif /*#ifndef	__ODMRAINFO_H__*/
diff --git a/drivers/staging/rtlwifi/phydm/phydm_reg.h b/drivers/staging/rtlwifi/phydm/phydm_reg.h
new file mode 100644
index 000000000000..d9d878e4c925
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_reg.h
@@ -0,0 +1,151 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+/* ************************************************************
+ * File Name: odm_reg.h
+ *
+ * Description:
+ *
+ * This file is for general register definition.
+ *
+ *
+ * *************************************************************/
+#ifndef __HAL_ODM_REG_H__
+#define __HAL_ODM_REG_H__
+
+/*
+ * Register Definition
+ */
+
+/* MAC REG */
+#define ODM_BB_RESET 0x002
+#define ODM_DUMMY 0x4fe
+#define RF_T_METER_OLD 0x24
+#define RF_T_METER_NEW 0x42
+
+#define ODM_EDCA_VO_PARAM 0x500
+#define ODM_EDCA_VI_PARAM 0x504
+#define ODM_EDCA_BE_PARAM 0x508
+#define ODM_EDCA_BK_PARAM 0x50C
+#define ODM_TXPAUSE 0x522
+
+/* LTE_COEX */
+#define REG_LTECOEX_CTRL 0x07C0
+#define REG_LTECOEX_WRITE_DATA 0x07C4
+#define REG_LTECOEX_READ_DATA 0x07C8
+#define REG_LTECOEX_PATH_CONTROL 0x70
+
+/* BB REG */
+#define ODM_FPGA_PHY0_PAGE8 0x800
+#define ODM_PSD_SETTING 0x808
+#define ODM_AFE_SETTING 0x818
+#define ODM_TXAGC_B_6_18 0x830
+#define ODM_TXAGC_B_24_54 0x834
+#define ODM_TXAGC_B_MCS32_5 0x838
+#define ODM_TXAGC_B_MCS0_MCS3 0x83c
+#define ODM_TXAGC_B_MCS4_MCS7 0x848
+#define ODM_TXAGC_B_MCS8_MCS11 0x84c
+#define ODM_ANALOG_REGISTER 0x85c
+#define ODM_RF_INTERFACE_OUTPUT 0x860
+#define ODM_TXAGC_B_MCS12_MCS15 0x868
+#define ODM_TXAGC_B_11_A_2_11 0x86c
+#define ODM_AD_DA_LSB_MASK 0x874
+#define ODM_ENABLE_3_WIRE 0x88c
+#define ODM_PSD_REPORT 0x8b4
+#define ODM_R_ANT_SELECT 0x90c
+#define ODM_CCK_ANT_SELECT 0xa07
+#define ODM_CCK_PD_THRESH 0xa0a
+#define ODM_CCK_RF_REG1 0xa11
+#define ODM_CCK_MATCH_FILTER 0xa20
+#define ODM_CCK_RAKE_MAC 0xa2e
+#define ODM_CCK_CNT_RESET 0xa2d
+#define ODM_CCK_TX_DIVERSITY 0xa2f
+#define ODM_CCK_FA_CNT_MSB 0xa5b
+#define ODM_CCK_FA_CNT_LSB 0xa5c
+#define ODM_CCK_NEW_FUNCTION 0xa75
+#define ODM_OFDM_PHY0_PAGE_C 0xc00
+#define ODM_OFDM_RX_ANT 0xc04
+#define ODM_R_A_RXIQI 0xc14
+#define ODM_R_A_AGC_CORE1 0xc50
+#define ODM_R_A_AGC_CORE2 0xc54
+#define ODM_R_B_AGC_CORE1 0xc58
+#define ODM_R_AGC_PAR 0xc70
+#define ODM_R_HTSTF_AGC_PAR 0xc7c
+#define ODM_TX_PWR_TRAINING_A 0xc90
+#define ODM_TX_PWR_TRAINING_B 0xc98
+#define ODM_OFDM_FA_CNT1 0xcf0
+#define ODM_OFDM_PHY0_PAGE_D 0xd00
+#define ODM_OFDM_FA_CNT2 0xda0
+#define ODM_OFDM_FA_CNT3 0xda4
+#define ODM_OFDM_FA_CNT4 0xda8
+#define ODM_TXAGC_A_6_18 0xe00
+#define ODM_TXAGC_A_24_54 0xe04
+#define ODM_TXAGC_A_1_MCS32 0xe08
+#define ODM_TXAGC_A_MCS0_MCS3 0xe10
+#define ODM_TXAGC_A_MCS4_MCS7 0xe14
+#define ODM_TXAGC_A_MCS8_MCS11 0xe18
+#define ODM_TXAGC_A_MCS12_MCS15 0xe1c
+
+/* RF REG */
+#define ODM_GAIN_SETTING 0x00
+#define ODM_CHANNEL 0x18
+#define ODM_RF_T_METER 0x24
+#define ODM_RF_T_METER_92D 0x42
+#define ODM_RF_T_METER_88E 0x42
+#define ODM_RF_T_METER_92E 0x42
+#define ODM_RF_T_METER_8812 0x42
+#define REG_RF_TX_GAIN_OFFSET 0x55
+
+/* ant Detect Reg */
+#define ODM_DPDT 0x300
+
+/* PSD Init */
+#define ODM_PSDREG 0x808
+
+/* 92D path Div */
+#define PATHDIV_REG 0xB30
+#define PATHDIV_TRI 0xBA0
+
+/*
+ * Bitmap Definition
+ */
+
+#define BIT_FA_RESET BIT(0)
+
+#define REG_OFDM_0_XA_TX_IQ_IMBALANCE 0xC80
+#define REG_OFDM_0_ECCA_THRESHOLD 0xC4C
+#define REG_FPGA0_XB_LSSI_READ_BACK 0x8A4
+#define REG_FPGA0_TX_GAIN_STAGE 0x80C
+#define REG_OFDM_0_XA_AGC_CORE1 0xC50
+#define REG_OFDM_0_XB_AGC_CORE1 0xC58
+#define REG_A_TX_SCALE_JAGUAR 0xC1C
+#define REG_B_TX_SCALE_JAGUAR 0xE1C
+
+#define REG_AFE_XTAL_CTRL 0x0024
+#define REG_AFE_PLL_CTRL 0x0028
+#define REG_MAC_PHY_CTRL 0x002C
+
+#define RF_CHNLBW 0x18
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/phydm_regdefine11ac.h b/drivers/staging/rtlwifi/phydm/phydm_regdefine11ac.h
new file mode 100644
index 000000000000..28d48415ac99
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_regdefine11ac.h
@@ -0,0 +1,94 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __ODM_REGDEFINE11AC_H__
+#define __ODM_REGDEFINE11AC_H__
+
+/* 2 RF REG LIST */
+
+/* 2 BB REG LIST */
+/* PAGE 8 */
+#define ODM_REG_CCK_RPT_FORMAT_11AC 0x804
+#define ODM_REG_BB_RX_PATH_11AC 0x808
+#define ODM_REG_BB_TX_PATH_11AC 0x80c
+#define ODM_REG_BB_ATC_11AC 0x860
+#define ODM_REG_EDCCA_POWER_CAL 0x8dc
+#define ODM_REG_DBG_RPT_11AC 0x8fc
+/* PAGE 9 */
+#define ODM_REG_EDCCA_DOWN_OPT 0x900
+#define ODM_REG_ACBB_EDCCA_ENHANCE 0x944
+#define odm_adc_trigger_jaguar2 0x95C /*ADC sample mode*/
+#define ODM_REG_OFDM_FA_RST_11AC 0x9A4
+#define ODM_REG_CCX_PERIOD_11AC 0x990
+#define ODM_REG_NHM_TH9_TH10_11AC 0x994
+#define ODM_REG_CLM_11AC 0x994
+#define ODM_REG_NHM_TH3_TO_TH0_11AC 0x998
+#define ODM_REG_NHM_TH7_TO_TH4_11AC 0x99c
+#define ODM_REG_NHM_TH8_11AC 0x9a0
+#define ODM_REG_NHM_9E8_11AC 0x9e8
+#define ODM_REG_CSI_CONTENT_VALUE 0x9b4
+/* PAGE A */
+#define ODM_REG_CCK_CCA_11AC 0xA0A
+#define ODM_REG_CCK_FA_RST_11AC 0xA2C
+#define ODM_REG_CCK_FA_11AC 0xA5C
+/* PAGE B */
+#define ODM_REG_RST_RPT_11AC 0xB58
+/* PAGE C */
+#define ODM_REG_TRMUX_11AC 0xC08
+#define ODM_REG_IGI_A_11AC 0xC50
+/* PAGE E */
+#define ODM_REG_IGI_B_11AC 0xE50
+#define ODM_REG_TRMUX_11AC_B 0xE08
+/* PAGE F */
+#define ODM_REG_CCK_CRC32_CNT_11AC 0xF04
+#define ODM_REG_CCK_CCA_CNT_11AC 0xF08
+#define ODM_REG_VHT_CRC32_CNT_11AC 0xF0c
+#define ODM_REG_HT_CRC32_CNT_11AC 0xF10
+#define ODM_REG_OFDM_CRC32_CNT_11AC 0xF14
+#define ODM_REG_OFDM_FA_11AC 0xF48
+#define ODM_REG_RPT_11AC 0xfa0
+#define ODM_REG_CLM_RESULT_11AC 0xfa4
+#define ODM_REG_NHM_CNT_11AC 0xfa8
+#define ODM_REG_NHM_DUR_READY_11AC 0xfb4
+
+#define ODM_REG_NHM_CNT7_TO_CNT4_11AC 0xfac
+#define ODM_REG_NHM_CNT11_TO_CNT8_11AC 0xfb0
+#define ODM_REG_OFDM_FA_TYPE2_11AC 0xFD0
+/* PAGE 18 */
+#define ODM_REG_IGI_C_11AC 0x1850
+/* PAGE 1A */
+#define ODM_REG_IGI_D_11AC 0x1A50
+
+/* 2 MAC REG LIST */
+#define ODM_REG_RESP_TX_11AC 0x6D8
+
+/* DIG Related */
+#define ODM_BIT_IGI_11AC 0xFFFFFFFF
+#define ODM_BIT_CCK_RPT_FORMAT_11AC BIT(16)
+#define ODM_BIT_BB_RX_PATH_11AC 0xF
+#define ODM_BIT_BB_TX_PATH_11AC 0xF
+#define ODM_BIT_BB_ATC_11AC BIT(14)
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/phydm_regdefine11n.h b/drivers/staging/rtlwifi/phydm/phydm_regdefine11n.h
new file mode 100644
index 000000000000..0b6581c50ab3
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_regdefine11n.h
@@ -0,0 +1,213 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __ODM_REGDEFINE11N_H__
+#define __ODM_REGDEFINE11N_H__
+
+/* 2 RF REG LIST */
+#define ODM_REG_RF_MODE_11N 0x00
+#define ODM_REG_RF_0B_11N 0x0B
+#define ODM_REG_CHNBW_11N 0x18
+#define ODM_REG_T_METER_11N 0x24
+#define ODM_REG_RF_25_11N 0x25
+#define ODM_REG_RF_26_11N 0x26
+#define ODM_REG_RF_27_11N 0x27
+#define ODM_REG_RF_2B_11N 0x2B
+#define ODM_REG_RF_2C_11N 0x2C
+#define ODM_REG_RXRF_A3_11N 0x3C
+#define ODM_REG_T_METER_92D_11N 0x42
+#define ODM_REG_T_METER_88E_11N 0x42
+
+/* 2 BB REG LIST */
+/* PAGE 8 */
+#define ODM_REG_BB_CTRL_11N 0x800
+#define ODM_REG_RF_PIN_11N 0x804
+#define ODM_REG_PSD_CTRL_11N 0x808
+#define ODM_REG_TX_ANT_CTRL_11N 0x80C
+#define ODM_REG_BB_PWR_SAV5_11N 0x818
+#define ODM_REG_CCK_RPT_FORMAT_11N 0x824
+#define ODM_REG_CCK_RPT_FORMAT_11N_B 0x82C
+#define ODM_REG_RX_DEFAULT_A_11N 0x858
+#define ODM_REG_RX_DEFAULT_B_11N 0x85A
+#define ODM_REG_BB_PWR_SAV3_11N 0x85C
+#define ODM_REG_ANTSEL_CTRL_11N 0x860
+#define ODM_REG_RX_ANT_CTRL_11N 0x864
+#define ODM_REG_PIN_CTRL_11N 0x870
+#define ODM_REG_BB_PWR_SAV1_11N 0x874
+#define ODM_REG_ANTSEL_PATH_11N 0x878
+#define ODM_REG_BB_3WIRE_11N 0x88C
+#define ODM_REG_SC_CNT_11N 0x8C4
+#define ODM_REG_PSD_DATA_11N 0x8B4
+#define ODM_REG_CCX_PERIOD_11N 0x894
+#define ODM_REG_NHM_TH9_TH10_11N 0x890
+#define ODM_REG_CLM_11N 0x890
+#define ODM_REG_NHM_TH3_TO_TH0_11N 0x898
+#define ODM_REG_NHM_TH7_TO_TH4_11N 0x89c
+#define ODM_REG_NHM_TH8_11N 0xe28
+#define ODM_REG_CLM_READY_11N 0x8b4
+#define ODM_REG_CLM_RESULT_11N 0x8d0
+#define ODM_REG_NHM_CNT_11N 0x8d8
+
+/* For struct acs_info, Jeffery, 2014-12-26 */
+#define ODM_REG_NHM_CNT7_TO_CNT4_11N 0x8dc
+#define ODM_REG_NHM_CNT9_TO_CNT8_11N 0x8d0
+#define ODM_REG_NHM_CNT10_TO_CNT11_11N 0x8d4
+
+/* PAGE 9 */
+#define ODM_REG_BB_CTRL_PAGE9_11N 0x900
+#define ODM_REG_DBG_RPT_11N 0x908
+#define ODM_REG_BB_TX_PATH_11N 0x90c
+#define ODM_REG_ANT_MAPPING1_11N 0x914
+#define ODM_REG_ANT_MAPPING2_11N 0x918
+#define ODM_REG_EDCCA_DOWN_OPT_11N 0x948
+#define ODM_REG_RX_DFIR_MOD_97F 0x948
+
+/* PAGE A */
+#define ODM_REG_CCK_ANTDIV_PARA1_11N 0xA00
+#define ODM_REG_CCK_ANT_SEL_11N 0xA04
+#define ODM_REG_CCK_CCA_11N 0xA0A
+#define ODM_REG_CCK_ANTDIV_PARA2_11N 0xA0C
+#define ODM_REG_CCK_ANTDIV_PARA3_11N 0xA10
+#define ODM_REG_CCK_ANTDIV_PARA4_11N 0xA14
+#define ODM_REG_CCK_FILTER_PARA1_11N 0xA22
+#define ODM_REG_CCK_FILTER_PARA2_11N 0xA23
+#define ODM_REG_CCK_FILTER_PARA3_11N 0xA24
+#define ODM_REG_CCK_FILTER_PARA4_11N 0xA25
+#define ODM_REG_CCK_FILTER_PARA5_11N 0xA26
+#define ODM_REG_CCK_FILTER_PARA6_11N 0xA27
+#define ODM_REG_CCK_FILTER_PARA7_11N 0xA28
+#define ODM_REG_CCK_FILTER_PARA8_11N 0xA29
+#define ODM_REG_CCK_FA_RST_11N 0xA2C
+#define ODM_REG_CCK_FA_MSB_11N 0xA58
+#define ODM_REG_CCK_FA_LSB_11N 0xA5C
+#define ODM_REG_CCK_CCA_CNT_11N 0xA60
+#define ODM_REG_BB_PWR_SAV4_11N 0xA74
+/* PAGE B */
+#define ODM_REG_LNA_SWITCH_11N 0xB2C
+#define ODM_REG_PATH_SWITCH_11N 0xB30
+#define ODM_REG_RSSI_CTRL_11N 0xB38
+#define ODM_REG_CONFIG_ANTA_11N 0xB68
+#define ODM_REG_RSSI_BT_11N 0xB9C
+#define ODM_REG_RXCK_RFMOD 0xBB0
+#define ODM_REG_EDCCA_DCNF_97F 0xBC0
+
+/* PAGE C */
+#define ODM_REG_OFDM_FA_HOLDC_11N 0xC00
+#define ODM_REG_BB_RX_PATH_11N 0xC04
+#define ODM_REG_TRMUX_11N 0xC08
+#define ODM_REG_OFDM_FA_RSTC_11N 0xC0C
+#define ODM_REG_DOWNSAM_FACTOR_11N 0xC10
+#define ODM_REG_RXIQI_MATRIX_11N 0xC14
+#define ODM_REG_TXIQK_MATRIX_LSB1_11N 0xC4C
+#define ODM_REG_IGI_A_11N 0xC50
+#define ODM_REG_ANTDIV_PARA2_11N 0xC54
+#define ODM_REG_IGI_B_11N 0xC58
+#define ODM_REG_ANTDIV_PARA3_11N 0xC5C
+#define ODM_REG_L1SBD_PD_CH_11N 0XC6C
+#define ODM_REG_BB_PWR_SAV2_11N 0xC70
+#define ODM_REG_BB_AGC_SET_2_11N 0xc74
+#define ODM_REG_RX_OFF_11N 0xC7C
+#define ODM_REG_TXIQK_MATRIXA_11N 0xC80
+#define ODM_REG_TXIQK_MATRIXB_11N 0xC88
+#define ODM_REG_TXIQK_MATRIXA_LSB2_11N 0xC94
+#define ODM_REG_TXIQK_MATRIXB_LSB2_11N 0xC9C
+#define ODM_REG_RXIQK_MATRIX_LSB_11N 0xCA0
+#define ODM_REG_ANTDIV_PARA1_11N 0xCA4
+#define ODM_REG_SMALL_BANDWIDTH_11N 0xCE4
+#define ODM_REG_OFDM_FA_TYPE1_11N 0xCF0
+/* PAGE D */
+#define ODM_REG_OFDM_FA_RSTD_11N 0xD00
+#define ODM_REG_BB_RX_ANT_11N 0xD04
+#define ODM_REG_BB_ATC_11N 0xD2C
+#define ODM_REG_OFDM_FA_TYPE2_11N 0xDA0
+#define ODM_REG_OFDM_FA_TYPE3_11N 0xDA4
+#define ODM_REG_OFDM_FA_TYPE4_11N 0xDA8
+#define ODM_REG_RPT_11N 0xDF4
+/* PAGE E */
+#define ODM_REG_TXAGC_A_6_18_11N 0xE00
+#define ODM_REG_TXAGC_A_24_54_11N 0xE04
+#define ODM_REG_TXAGC_A_1_MCS32_11N 0xE08
+#define ODM_REG_TXAGC_A_MCS0_3_11N 0xE10
+#define ODM_REG_TXAGC_A_MCS4_7_11N 0xE14
+#define ODM_REG_TXAGC_A_MCS8_11_11N 0xE18
+#define ODM_REG_TXAGC_A_MCS12_15_11N 0xE1C
+#define ODM_REG_EDCCA_DCNF_11N 0xE24
+#define ODM_REG_TAP_UPD_97F 0xE24
+#define ODM_REG_FPGA0_IQK_11N 0xE28
+#define ODM_REG_PAGE_B1_97F 0xE28
+#define ODM_REG_TXIQK_TONE_A_11N 0xE30
+#define ODM_REG_RXIQK_TONE_A_11N 0xE34
+#define ODM_REG_TXIQK_PI_A_11N 0xE38
+#define ODM_REG_RXIQK_PI_A_11N 0xE3C
+#define ODM_REG_TXIQK_11N 0xE40
+#define ODM_REG_RXIQK_11N 0xE44
+#define ODM_REG_IQK_AGC_PTS_11N 0xE48
+#define ODM_REG_IQK_AGC_RSP_11N 0xE4C
+#define ODM_REG_BLUETOOTH_11N 0xE6C
+#define ODM_REG_RX_WAIT_CCA_11N 0xE70
+#define ODM_REG_TX_CCK_RFON_11N 0xE74
+#define ODM_REG_TX_CCK_BBON_11N 0xE78
+#define ODM_REG_OFDM_RFON_11N 0xE7C
+#define ODM_REG_OFDM_BBON_11N 0xE80
+#define ODM_REG_TX2RX_11N 0xE84
+#define ODM_REG_TX2TX_11N 0xE88
+#define ODM_REG_RX_CCK_11N 0xE8C
+#define ODM_REG_RX_OFDM_11N 0xED0
+#define ODM_REG_RX_WAIT_RIFS_11N 0xED4
+#define ODM_REG_RX2RX_11N 0xED8
+#define ODM_REG_STANDBY_11N 0xEDC
+#define ODM_REG_SLEEP_11N 0xEE0
+#define ODM_REG_PMPD_ANAEN_11N 0xEEC
+/* PAGE F */
+#define ODM_REG_PAGE_F_RST_11N 0xF14
+#define ODM_REG_IGI_C_11N 0xF84
+#define ODM_REG_IGI_D_11N 0xF88
+#define ODM_REG_CCK_CRC32_ERROR_CNT_11N 0xF84
+#define ODM_REG_CCK_CRC32_OK_CNT_11N 0xF88
+#define ODM_REG_HT_CRC32_CNT_11N 0xF90
+#define ODM_REG_OFDM_CRC32_CNT_11N 0xF94
+
+/* 2 MAC REG LIST */
+#define ODM_REG_BB_RST_11N 0x02
+#define ODM_REG_ANTSEL_PIN_11N 0x4C
+#define ODM_REG_EARLY_MODE_11N 0x4D0
+#define ODM_REG_RSSI_MONITOR_11N 0x4FE
+#define ODM_REG_EDCA_VO_11N 0x500
+#define ODM_REG_EDCA_VI_11N 0x504
+#define ODM_REG_EDCA_BE_11N 0x508
+#define ODM_REG_EDCA_BK_11N 0x50C
+#define ODM_REG_TXPAUSE_11N 0x522
+#define ODM_REG_RESP_TX_11N 0x6D8
+#define ODM_REG_ANT_TRAIN_PARA1_11N 0x7b0
+#define ODM_REG_ANT_TRAIN_PARA2_11N 0x7b4
+
+/* DIG Related */
+#define ODM_BIT_IGI_11N 0x0000007F
+#define ODM_BIT_CCK_RPT_FORMAT_11N BIT(9)
+#define ODM_BIT_BB_RX_PATH_11N 0xF
+#define ODM_BIT_BB_TX_PATH_11N 0xF
+#define ODM_BIT_BB_ATC_11N BIT(11)
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/phydm_types.h b/drivers/staging/rtlwifi/phydm/phydm_types.h
new file mode 100644
index 000000000000..a34ebe876528
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/phydm_types.h
@@ -0,0 +1,130 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef __ODM_TYPES_H__
+#define __ODM_TYPES_H__
+
+/*Define Different SW team support*/
+#define ODM_AP 0x01 /*BIT0*/
+#define ODM_CE 0x04 /*BIT2*/
+#define ODM_WIN 0x08 /*BIT3*/
+#define ODM_ADSL 0x10 /*BIT4*/
+#define ODM_IOT 0x20 /*BIT5*/
+
+/*Deifne HW endian support*/
+#define ODM_ENDIAN_BIG 0
+#define ODM_ENDIAN_LITTLE 1
+
+#define GET_PDM_ODM(__padapter)                                                \
+	((struct phy_dm_struct *)(&(GET_HAL_DATA(__padapter))->odmpriv))
+
+enum hal_status {
+	HAL_STATUS_SUCCESS,
+	HAL_STATUS_FAILURE,
+};
+
+/*
+ * Declare for ODM spin lock definition temporarily fro compile pass.
+ */
+enum rt_spinlock_type {
+	RT_TX_SPINLOCK = 1,
+	RT_RX_SPINLOCK = 2,
+	RT_RM_SPINLOCK = 3,
+	RT_CAM_SPINLOCK = 4,
+	RT_SCAN_SPINLOCK = 5,
+	RT_LOG_SPINLOCK = 7,
+	RT_BW_SPINLOCK = 8,
+	RT_CHNLOP_SPINLOCK = 9,
+	RT_RF_OPERATE_SPINLOCK = 10,
+	RT_INITIAL_SPINLOCK = 11,
+	RT_RF_STATE_SPINLOCK =
+		12, /* For RF state. Added by Bruce, 2007-10-30. */
+	/* Shall we define Ndis 6.2 SpinLock Here ? */
+	RT_PORT_SPINLOCK = 16,
+	RT_VNIC_SPINLOCK = 17,
+	RT_HVL_SPINLOCK = 18,
+	RT_H2C_SPINLOCK = 20, /* For H2C cmd. Added by tynli. 2009.11.09. */
+
+	rt_bt_data_spinlock = 25,
+
+	RT_WAPI_OPTION_SPINLOCK = 26,
+	RT_WAPI_RX_SPINLOCK = 27,
+
+	/* add for 92D CCK control issue */
+	RT_CCK_PAGEA_SPINLOCK = 28,
+	RT_BUFFER_SPINLOCK = 29,
+	RT_CHANNEL_AND_BANDWIDTH_SPINLOCK = 30,
+	RT_GEN_TEMP_BUF_SPINLOCK = 31,
+	RT_AWB_SPINLOCK = 32,
+	RT_FW_PS_SPINLOCK = 33,
+	RT_HW_TIMER_SPIN_LOCK = 34,
+	RT_MPT_WI_SPINLOCK = 35,
+	RT_P2P_SPIN_LOCK = 36, /* Protect P2P context */
+	RT_DBG_SPIN_LOCK = 37,
+	RT_IQK_SPINLOCK = 38,
+	RT_PENDED_OID_SPINLOCK = 39,
+	RT_CHNLLIST_SPINLOCK = 40,
+	RT_INDIC_SPINLOCK = 41, /* protect indication */
+	RT_RFD_SPINLOCK = 42,
+	RT_SYNC_IO_CNT_SPINLOCK = 43,
+	RT_LAST_SPINLOCK,
+};
+
+#include <asm/byteorder.h>
+
+#if defined(__LITTLE_ENDIAN)
+#define ODM_ENDIAN_TYPE ODM_ENDIAN_LITTLE
+#elif defined(__BIG_ENDIAN)
+#define ODM_ENDIAN_TYPE ODM_ENDIAN_BIG
+#else
+#error
+#endif
+
+#define COND_ELSE 2
+#define COND_ENDIF 3
+
+#define MASKBYTE0 0xff
+#define MASKBYTE1 0xff00
+#define MASKBYTE2 0xff0000
+#define MASKBYTE3 0xff000000
+#define MASKHWORD 0xffff0000
+#define MASKLWORD 0x0000ffff
+#define MASKDWORD 0xffffffff
+#define MASK7BITS 0x7f
+#define MASK12BITS 0xfff
+#define MASKH4BITS 0xf0000000
+#define MASK20BITS 0xfffff
+#define MASKOFDM_D 0xffc00000
+#define MASKCCK 0x3f3f3f3f
+#define RFREGOFFSETMASK 0xfffff
+#define MASKH3BYTES 0xffffff00
+#define MASKL3BYTES 0x00ffffff
+#define MASKBYTE2HIGHNIBBLE 0x00f00000
+#define MASKBYTE3LOWNIBBLE 0x0f000000
+#define MASKL3BYTES 0x00ffffff
+#define RFREGOFFSETMASK 0xfffff
+
+#include "phydm_features.h"
+
+#endif /* __ODM_TYPES_H__ */
diff --git a/drivers/staging/rtlwifi/phydm/rtl8822b/halhwimg8822b_bb.c b/drivers/staging/rtlwifi/phydm/rtl8822b/halhwimg8822b_bb.c
new file mode 100644
index 000000000000..4e7946019fcb
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/rtl8822b/halhwimg8822b_bb.c
@@ -0,0 +1,1969 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+/*Image2HeaderVersion: 3.2*/
+#include "../mp_precomp.h"
+#include "../phydm_precomp.h"
+
+static bool check_positive(struct phy_dm_struct *dm, const u32 condition1,
+			   const u32 condition2, const u32 condition3,
+			   const u32 condition4)
+{
+	u8 _board_type = ((dm->board_type & BIT(4)) >> 4) << 0 | /* _GLNA*/
+			 ((dm->board_type & BIT(3)) >> 3) << 1 | /* _GPA*/
+			 ((dm->board_type & BIT(7)) >> 7) << 2 | /* _ALNA*/
+			 ((dm->board_type & BIT(6)) >> 6) << 3 | /* _APA */
+			 ((dm->board_type & BIT(2)) >> 2) << 4; /* _BT*/
+
+	u32 cond1 = condition1, cond2 = condition2, cond3 = condition3,
+	    cond4 = condition4;
+
+	u8 cut_version_for_para =
+		(dm->cut_version == ODM_CUT_A) ? 14 : dm->cut_version;
+	u8 pkg_type_for_para = (dm->package_type == 0) ? 14 : dm->package_type;
+
+	u32 driver1 = cut_version_for_para << 24 |
+		      (dm->support_interface & 0xF0) << 16 |
+		      dm->support_platform << 16 | pkg_type_for_para << 12 |
+		      (dm->support_interface & 0x0F) << 8 | _board_type;
+
+	u32 driver2 = (dm->type_glna & 0xFF) << 0 | (dm->type_gpa & 0xFF) << 8 |
+		      (dm->type_alna & 0xFF) << 16 |
+		      (dm->type_apa & 0xFF) << 24;
+
+	u32 driver3 = 0;
+
+	u32 driver4 = (dm->type_glna & 0xFF00) >> 8 | (dm->type_gpa & 0xFF00) |
+		      (dm->type_alna & 0xFF00) << 8 |
+		      (dm->type_apa & 0xFF00) << 16;
+
+	ODM_RT_TRACE(
+		dm, ODM_COMP_INIT,
+		"===> %s (cond1, cond2, cond3, cond4) = (0x%X 0x%X 0x%X 0x%X)\n",
+		__func__, cond1, cond2, cond3, cond4);
+	ODM_RT_TRACE(
+		dm, ODM_COMP_INIT,
+		"===> %s (driver1, driver2, driver3, driver4) = (0x%X 0x%X 0x%X 0x%X)\n",
+		__func__, driver1, driver2, driver3, driver4);
+
+	ODM_RT_TRACE(dm, ODM_COMP_INIT,
+		     "	(Platform, Interface) = (0x%X, 0x%X)\n",
+		     dm->support_platform, dm->support_interface);
+	ODM_RT_TRACE(dm, ODM_COMP_INIT,
+		     "	(Board, Package) = (0x%X, 0x%X)\n",
+		     dm->board_type, dm->package_type);
+
+	/*============== value Defined Check ===============*/
+	/*QFN type [15:12] and cut version [27:24] need to do value check*/
+
+	if (((cond1 & 0x0000F000) != 0) &&
+	    ((cond1 & 0x0000F000) != (driver1 & 0x0000F000)))
+		return false;
+	if (((cond1 & 0x0F000000) != 0) &&
+	    ((cond1 & 0x0F000000) != (driver1 & 0x0F000000)))
+		return false;
+
+	/*=============== Bit Defined Check ================*/
+	/* We don't care [31:28] */
+
+	cond1 &= 0x00FF0FFF;
+	driver1 &= 0x00FF0FFF;
+
+	if ((cond1 & driver1) == cond1) {
+		u32 bit_mask = 0;
+
+		if ((cond1 & 0x0F) == 0) /* board_type is DONTCARE*/
+			return true;
+
+		if ((cond1 & BIT(0)) != 0) /*GLNA*/
+			bit_mask |= 0x000000FF;
+		if ((cond1 & BIT(1)) != 0) /*GPA*/
+			bit_mask |= 0x0000FF00;
+		if ((cond1 & BIT(2)) != 0) /*ALNA*/
+			bit_mask |= 0x00FF0000;
+		if ((cond1 & BIT(3)) != 0) /*APA*/
+			bit_mask |= 0xFF000000;
+
+		if (((cond2 & bit_mask) == (driver2 & bit_mask)) &&
+		    ((cond4 & bit_mask) ==
+		     (driver4 &
+		      bit_mask))) /* board_type of each RF path is matched*/
+			return true;
+		else
+			return false;
+	} else {
+		return false;
+	}
+}
+
+/******************************************************************************
+ *                           agc_tab.TXT
+ ******************************************************************************/
+
+static u32 array_mp_8822b_agc_tab[] = {
+	0x8000100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x81C,      0xFF000003,
+	0x81C,      0xF5000003, 0x81C,      0xF4020003, 0x81C,      0xF3040003,
+	0x81C,      0xF2060003, 0x81C,      0xF1080003, 0x81C,      0xF00A0003,
+	0x81C,      0xEF0C0003, 0x81C,      0xEE0E0003, 0x81C,      0xED100003,
+	0x81C,      0xEC120003, 0x81C,      0xEB140003, 0x81C,      0xEA160003,
+	0x81C,      0xE9180003, 0x81C,      0xE81A0003, 0x81C,      0xE71C0003,
+	0x81C,      0xE61E0003, 0x81C,      0xE5200003, 0x81C,      0xE4220003,
+	0x81C,      0xE3240003, 0x81C,      0xE2260003, 0x81C,      0xE1280003,
+	0x81C,      0xE02A0003, 0x81C,      0xC32C0003, 0x81C,      0xC22E0003,
+	0x81C,      0xC1300003, 0x81C,      0xC0320003, 0x81C,      0xA4340003,
+	0x81C,      0xA3360003, 0x81C,      0xA2380003, 0x81C,      0xA13A0003,
+	0x81C,      0xA03C0003, 0x81C,      0x823E0003, 0x81C,      0x81400003,
+	0x81C,      0x80420003, 0x81C,      0x64440003, 0x81C,      0x63460003,
+	0x81C,      0x62480003, 0x81C,      0x614A0003, 0x81C,      0x604C0003,
+	0x81C,      0x454E0003, 0x81C,      0x44500003, 0x81C,      0x43520003,
+	0x81C,      0x42540003, 0x81C,      0x41560003, 0x81C,      0x40580003,
+	0x81C,      0x055A0003, 0x81C,      0x045C0003, 0x81C,      0x035E0003,
+	0x81C,      0x02600003, 0x81C,      0x01620003, 0x81C,      0x00640003,
+	0x81C,      0x00660003, 0x81C,      0x00680003, 0x81C,      0x006A0003,
+	0x81C,      0x006C0003, 0x81C,      0x006E0003, 0x81C,      0x00700003,
+	0x81C,      0x00720003, 0x81C,      0x00740003, 0x81C,      0x00760003,
+	0x81C,      0x00780003, 0x81C,      0x007A0003, 0x81C,      0x007C0003,
+	0x81C,      0x007E0003, 0x9000100f, 0x05050505, 0x40000000, 0x00000000,
+	0x81C,      0xFF000003, 0x81C,      0xF5000003, 0x81C,      0xF4020003,
+	0x81C,      0xF3040003, 0x81C,      0xF2060003, 0x81C,      0xF1080003,
+	0x81C,      0xF00A0003, 0x81C,      0xEF0C0003, 0x81C,      0xEE0E0003,
+	0x81C,      0xED100003, 0x81C,      0xEC120003, 0x81C,      0xEB140003,
+	0x81C,      0xEA160003, 0x81C,      0xE9180003, 0x81C,      0xE81A0003,
+	0x81C,      0xE71C0003, 0x81C,      0xE61E0003, 0x81C,      0xE5200003,
+	0x81C,      0xE4220003, 0x81C,      0xE3240003, 0x81C,      0xE2260003,
+	0x81C,      0xE1280003, 0x81C,      0xE02A0003, 0x81C,      0xC32C0003,
+	0x81C,      0xC22E0003, 0x81C,      0xC1300003, 0x81C,      0xC0320003,
+	0x81C,      0xA4340003, 0x81C,      0xA3360003, 0x81C,      0xA2380003,
+	0x81C,      0xA13A0003, 0x81C,      0xA03C0003, 0x81C,      0x823E0003,
+	0x81C,      0x81400003, 0x81C,      0x80420003, 0x81C,      0x64440003,
+	0x81C,      0x63460003, 0x81C,      0x62480003, 0x81C,      0x614A0003,
+	0x81C,      0x604C0003, 0x81C,      0x454E0003, 0x81C,      0x44500003,
+	0x81C,      0x43520003, 0x81C,      0x42540003, 0x81C,      0x41560003,
+	0x81C,      0x40580003, 0x81C,      0x055A0003, 0x81C,      0x045C0003,
+	0x81C,      0x035E0003, 0x81C,      0x02600003, 0x81C,      0x01620003,
+	0x81C,      0x00640003, 0x81C,      0x00660003, 0x81C,      0x00680003,
+	0x81C,      0x006A0003, 0x81C,      0x006C0003, 0x81C,      0x006E0003,
+	0x81C,      0x00700003, 0x81C,      0x00720003, 0x81C,      0x00740003,
+	0x81C,      0x00760003, 0x81C,      0x00780003, 0x81C,      0x007A0003,
+	0x81C,      0x007C0003, 0x81C,      0x007E0003, 0x9000100f, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xFF000003, 0x81C,      0xF5000003,
+	0x81C,      0xF4020003, 0x81C,      0xF3040003, 0x81C,      0xF2060003,
+	0x81C,      0xF1080003, 0x81C,      0xF00A0003, 0x81C,      0xEF0C0003,
+	0x81C,      0xEE0E0003, 0x81C,      0xED100003, 0x81C,      0xEC120003,
+	0x81C,      0xEB140003, 0x81C,      0xEA160003, 0x81C,      0xE9180003,
+	0x81C,      0xE81A0003, 0x81C,      0xE71C0003, 0x81C,      0xE61E0003,
+	0x81C,      0xE5200003, 0x81C,      0xE4220003, 0x81C,      0xE3240003,
+	0x81C,      0xE2260003, 0x81C,      0xE1280003, 0x81C,      0xE02A0003,
+	0x81C,      0xC32C0003, 0x81C,      0xC22E0003, 0x81C,      0xC1300003,
+	0x81C,      0xC0320003, 0x81C,      0xA4340003, 0x81C,      0xA3360003,
+	0x81C,      0xA2380003, 0x81C,      0xA13A0003, 0x81C,      0xA03C0003,
+	0x81C,      0x823E0003, 0x81C,      0x81400003, 0x81C,      0x80420003,
+	0x81C,      0x64440003, 0x81C,      0x63460003, 0x81C,      0x62480003,
+	0x81C,      0x614A0003, 0x81C,      0x604C0003, 0x81C,      0x454E0003,
+	0x81C,      0x44500003, 0x81C,      0x43520003, 0x81C,      0x42540003,
+	0x81C,      0x41560003, 0x81C,      0x40580003, 0x81C,      0x055A0003,
+	0x81C,      0x045C0003, 0x81C,      0x035E0003, 0x81C,      0x02600003,
+	0x81C,      0x01620003, 0x81C,      0x00640003, 0x81C,      0x00660003,
+	0x81C,      0x00680003, 0x81C,      0x006A0003, 0x81C,      0x006C0003,
+	0x81C,      0x006E0003, 0x81C,      0x00700003, 0x81C,      0x00720003,
+	0x81C,      0x00740003, 0x81C,      0x00760003, 0x81C,      0x00780003,
+	0x81C,      0x007A0003, 0x81C,      0x007C0003, 0x81C,      0x007E0003,
+	0x9000200f, 0x00000000, 0x40000000, 0x00000000, 0x81C,      0xFF000003,
+	0x81C,      0xF5000003, 0x81C,      0xF4020003, 0x81C,      0xF3040003,
+	0x81C,      0xF2060003, 0x81C,      0xF1080003, 0x81C,      0xF00A0003,
+	0x81C,      0xEF0C0003, 0x81C,      0xEE0E0003, 0x81C,      0xED100003,
+	0x81C,      0xEC120003, 0x81C,      0xEB140003, 0x81C,      0xEA160003,
+	0x81C,      0xE9180003, 0x81C,      0xE81A0003, 0x81C,      0xE71C0003,
+	0x81C,      0xE61E0003, 0x81C,      0xE5200003, 0x81C,      0xE4220003,
+	0x81C,      0xE3240003, 0x81C,      0xE2260003, 0x81C,      0xE1280003,
+	0x81C,      0xE02A0003, 0x81C,      0xC32C0003, 0x81C,      0xC22E0003,
+	0x81C,      0xC1300003, 0x81C,      0xC0320003, 0x81C,      0xA4340003,
+	0x81C,      0xA3360003, 0x81C,      0xA2380003, 0x81C,      0xA13A0003,
+	0x81C,      0xA03C0003, 0x81C,      0x823E0003, 0x81C,      0x81400003,
+	0x81C,      0x80420003, 0x81C,      0x64440003, 0x81C,      0x63460003,
+	0x81C,      0x62480003, 0x81C,      0x614A0003, 0x81C,      0x604C0003,
+	0x81C,      0x454E0003, 0x81C,      0x44500003, 0x81C,      0x43520003,
+	0x81C,      0x42540003, 0x81C,      0x41560003, 0x81C,      0x40580003,
+	0x81C,      0x055A0003, 0x81C,      0x045C0003, 0x81C,      0x035E0003,
+	0x81C,      0x02600003, 0x81C,      0x01620003, 0x81C,      0x00640003,
+	0x81C,      0x00660003, 0x81C,      0x00680003, 0x81C,      0x006A0003,
+	0x81C,      0x006C0003, 0x81C,      0x006E0003, 0x81C,      0x00700003,
+	0x81C,      0x00720003, 0x81C,      0x00740003, 0x81C,      0x00760003,
+	0x81C,      0x00780003, 0x81C,      0x007A0003, 0x81C,      0x007C0003,
+	0x81C,      0x007E0003, 0x9000200c, 0x00000000, 0x40000000, 0x00000000,
+	0x81C,      0xFF000003, 0x81C,      0xFD000003, 0x81C,      0xFC020003,
+	0x81C,      0xFB040003, 0x81C,      0xFA060003, 0x81C,      0xF9080003,
+	0x81C,      0xF80A0003, 0x81C,      0xF70C0003, 0x81C,      0xF60E0003,
+	0x81C,      0xF5100003, 0x81C,      0xF4120003, 0x81C,      0xF3140003,
+	0x81C,      0xF2160003, 0x81C,      0xF1180003, 0x81C,      0xF01A0003,
+	0x81C,      0xEF1C0003, 0x81C,      0xEE1E0003, 0x81C,      0xED200003,
+	0x81C,      0xEC220003, 0x81C,      0xEB240003, 0x81C,      0xEA260003,
+	0x81C,      0xE9280003, 0x81C,      0xE82A0003, 0x81C,      0xE72C0003,
+	0x81C,      0xE62E0003, 0x81C,      0xE5300003, 0x81C,      0xC8320003,
+	0x81C,      0xC7340003, 0x81C,      0xC6360003, 0x81C,      0xC5380003,
+	0x81C,      0xC43A0003, 0x81C,      0xC33C0003, 0x81C,      0xC23E0003,
+	0x81C,      0xC1400003, 0x81C,      0xC0420003, 0x81C,      0xA5440003,
+	0x81C,      0xA4460003, 0x81C,      0xA3480003, 0x81C,      0xA24A0003,
+	0x81C,      0xA14C0003, 0x81C,      0x834E0003, 0x81C,      0x82500003,
+	0x81C,      0x81520003, 0x81C,      0x80540003, 0x81C,      0x65560003,
+	0x81C,      0x64580003, 0x81C,      0x635A0003, 0x81C,      0x625C0003,
+	0x81C,      0x435E0003, 0x81C,      0x42600003, 0x81C,      0x41620003,
+	0x81C,      0x40640003, 0x81C,      0x06660003, 0x81C,      0x05680003,
+	0x81C,      0x046A0003, 0x81C,      0x036C0003, 0x81C,      0x026E0003,
+	0x81C,      0x01700003, 0x81C,      0x00720003, 0x81C,      0x00740003,
+	0x81C,      0x00760003, 0x81C,      0x00780003, 0x81C,      0x007A0003,
+	0x81C,      0x007C0003, 0x81C,      0x007E0003, 0x90012100, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xFF000003, 0x81C,      0xFE000003,
+	0x81C,      0xFD020003, 0x81C,      0xFC040003, 0x81C,      0xFB060003,
+	0x81C,      0xFA080003, 0x81C,      0xF90A0003, 0x81C,      0xF80C0003,
+	0x81C,      0xF70E0003, 0x81C,      0xF6100003, 0x81C,      0xF5120003,
+	0x81C,      0xF4140003, 0x81C,      0xF3160003, 0x81C,      0xF2180003,
+	0x81C,      0xF11A0003, 0x81C,      0xF01C0003, 0x81C,      0xEF1E0003,
+	0x81C,      0xEE200003, 0x81C,      0xED220003, 0x81C,      0xEC240003,
+	0x81C,      0xEB260003, 0x81C,      0xEA280003, 0x81C,      0xE92A0003,
+	0x81C,      0xE82C0003, 0x81C,      0xE72E0003, 0x81C,      0xE6300003,
+	0x81C,      0xE5320003, 0x81C,      0xC8340003, 0x81C,      0xC7360003,
+	0x81C,      0xC6380003, 0x81C,      0xC53A0003, 0x81C,      0xC43C0003,
+	0x81C,      0xC33E0003, 0x81C,      0xC2400003, 0x81C,      0xC1420003,
+	0x81C,      0xC0440003, 0x81C,      0xA3460003, 0x81C,      0xA2480003,
+	0x81C,      0xA14A0003, 0x81C,      0xA04C0003, 0x81C,      0x824E0003,
+	0x81C,      0x81500003, 0x81C,      0x80520003, 0x81C,      0x64540003,
+	0x81C,      0x63560003, 0x81C,      0x62580003, 0x81C,      0x445A0003,
+	0x81C,      0x435C0003, 0x81C,      0x425E0003, 0x81C,      0x41600003,
+	0x81C,      0x40620003, 0x81C,      0x05640003, 0x81C,      0x04660003,
+	0x81C,      0x03680003, 0x81C,      0x026A0003, 0x81C,      0x016C0003,
+	0x81C,      0x006E0003, 0x81C,      0x00700003, 0x81C,      0x00720003,
+	0x81C,      0x00740003, 0x81C,      0x00760003, 0x81C,      0x00780003,
+	0x81C,      0x007A0003, 0x81C,      0x007C0003, 0x81C,      0x007E0003,
+	0x90001004, 0x00000000, 0x40000000, 0x00000000, 0x81C,      0xFF000003,
+	0x81C,      0xF5000003, 0x81C,      0xF4020003, 0x81C,      0xF3040003,
+	0x81C,      0xF2060003, 0x81C,      0xF1080003, 0x81C,      0xF00A0003,
+	0x81C,      0xEF0C0003, 0x81C,      0xEE0E0003, 0x81C,      0xED100003,
+	0x81C,      0xEC120003, 0x81C,      0xEB140003, 0x81C,      0xEA160003,
+	0x81C,      0xE9180003, 0x81C,      0xE81A0003, 0x81C,      0xE71C0003,
+	0x81C,      0xE61E0003, 0x81C,      0xE5200003, 0x81C,      0xE4220003,
+	0x81C,      0xE3240003, 0x81C,      0xE2260003, 0x81C,      0xE1280003,
+	0x81C,      0xE02A0003, 0x81C,      0xC32C0003, 0x81C,      0xC22E0003,
+	0x81C,      0xC1300003, 0x81C,      0xC0320003, 0x81C,      0xA4340003,
+	0x81C,      0xA3360003, 0x81C,      0xA2380003, 0x81C,      0xA13A0003,
+	0x81C,      0xA03C0003, 0x81C,      0x823E0003, 0x81C,      0x81400003,
+	0x81C,      0x80420003, 0x81C,      0x64440003, 0x81C,      0x63460003,
+	0x81C,      0x62480003, 0x81C,      0x614A0003, 0x81C,      0x604C0003,
+	0x81C,      0x454E0003, 0x81C,      0x44500003, 0x81C,      0x43520003,
+	0x81C,      0x42540003, 0x81C,      0x41560003, 0x81C,      0x40580003,
+	0x81C,      0x055A0003, 0x81C,      0x045C0003, 0x81C,      0x035E0003,
+	0x81C,      0x02600003, 0x81C,      0x01620003, 0x81C,      0x00640003,
+	0x81C,      0x00660003, 0x81C,      0x00680003, 0x81C,      0x006A0003,
+	0x81C,      0x006C0003, 0x81C,      0x006E0003, 0x81C,      0x00700003,
+	0x81C,      0x00720003, 0x81C,      0x00740003, 0x81C,      0x00760003,
+	0x81C,      0x00780003, 0x81C,      0x007A0003, 0x81C,      0x007C0003,
+	0x81C,      0x007E0003, 0x90011000, 0x00000000, 0x40000000, 0x00000000,
+	0x81C,      0xFF000003, 0x81C,      0xFE000003, 0x81C,      0xFD020003,
+	0x81C,      0xFC040003, 0x81C,      0xFB060003, 0x81C,      0xFA080003,
+	0x81C,      0xF90A0003, 0x81C,      0xF80C0003, 0x81C,      0xF70E0003,
+	0x81C,      0xF6100003, 0x81C,      0xF5120003, 0x81C,      0xF4140003,
+	0x81C,      0xF3160003, 0x81C,      0xF2180003, 0x81C,      0xF11A0003,
+	0x81C,      0xF01C0003, 0x81C,      0xEF1E0003, 0x81C,      0xEE200003,
+	0x81C,      0xED220003, 0x81C,      0xEC240003, 0x81C,      0xEB260003,
+	0x81C,      0xEA280003, 0x81C,      0xE92A0003, 0x81C,      0xE82C0003,
+	0x81C,      0xE72E0003, 0x81C,      0xE6300003, 0x81C,      0xE5320003,
+	0x81C,      0xC8340003, 0x81C,      0xC7360003, 0x81C,      0xC6380003,
+	0x81C,      0xC53A0003, 0x81C,      0xC43C0003, 0x81C,      0xC33E0003,
+	0x81C,      0xC2400003, 0x81C,      0xC1420003, 0x81C,      0xC0440003,
+	0x81C,      0xA3460003, 0x81C,      0xA2480003, 0x81C,      0xA14A0003,
+	0x81C,      0xA04C0003, 0x81C,      0x824E0003, 0x81C,      0x81500003,
+	0x81C,      0x80520003, 0x81C,      0x64540003, 0x81C,      0x63560003,
+	0x81C,      0x62580003, 0x81C,      0x445A0003, 0x81C,      0x435C0003,
+	0x81C,      0x425E0003, 0x81C,      0x41600003, 0x81C,      0x40620003,
+	0x81C,      0x05640003, 0x81C,      0x04660003, 0x81C,      0x03680003,
+	0x81C,      0x026A0003, 0x81C,      0x016C0003, 0x81C,      0x006E0003,
+	0x81C,      0x00700003, 0x81C,      0x00720003, 0x81C,      0x00740003,
+	0x81C,      0x00760003, 0x81C,      0x00780003, 0x81C,      0x007A0003,
+	0x81C,      0x007C0003, 0x81C,      0x007E0003, 0x90002100, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xFF000003, 0x81C,      0xFD000003,
+	0x81C,      0xFC020003, 0x81C,      0xFB040003, 0x81C,      0xFA060003,
+	0x81C,      0xF9080003, 0x81C,      0xF80A0003, 0x81C,      0xF70C0003,
+	0x81C,      0xF60E0003, 0x81C,      0xF5100003, 0x81C,      0xF4120003,
+	0x81C,      0xF3140003, 0x81C,      0xF2160003, 0x81C,      0xF1180003,
+	0x81C,      0xF01A0003, 0x81C,      0xEF1C0003, 0x81C,      0xEE1E0003,
+	0x81C,      0xED200003, 0x81C,      0xEC220003, 0x81C,      0xEB240003,
+	0x81C,      0xEA260003, 0x81C,      0xE9280003, 0x81C,      0xE82A0003,
+	0x81C,      0xE72C0003, 0x81C,      0xE62E0003, 0x81C,      0xE5300003,
+	0x81C,      0xC8320003, 0x81C,      0xC7340003, 0x81C,      0xC6360003,
+	0x81C,      0xC5380003, 0x81C,      0xC43A0003, 0x81C,      0xC33C0003,
+	0x81C,      0xC23E0003, 0x81C,      0xC1400003, 0x81C,      0xC0420003,
+	0x81C,      0xA5440003, 0x81C,      0xA4460003, 0x81C,      0xA3480003,
+	0x81C,      0xA24A0003, 0x81C,      0xA14C0003, 0x81C,      0x834E0003,
+	0x81C,      0x82500003, 0x81C,      0x81520003, 0x81C,      0x80540003,
+	0x81C,      0x65560003, 0x81C,      0x64580003, 0x81C,      0x635A0003,
+	0x81C,      0x625C0003, 0x81C,      0x435E0003, 0x81C,      0x42600003,
+	0x81C,      0x41620003, 0x81C,      0x40640003, 0x81C,      0x06660003,
+	0x81C,      0x05680003, 0x81C,      0x046A0003, 0x81C,      0x036C0003,
+	0x81C,      0x026E0003, 0x81C,      0x01700003, 0x81C,      0x00720003,
+	0x81C,      0x00740003, 0x81C,      0x00760003, 0x81C,      0x00780003,
+	0x81C,      0x007A0003, 0x81C,      0x007C0003, 0x81C,      0x007E0003,
+	0x90002000, 0x00000000, 0x40000000, 0x00000000, 0x81C,      0xFF000003,
+	0x81C,      0xFD000003, 0x81C,      0xFC020003, 0x81C,      0xFB040003,
+	0x81C,      0xFA060003, 0x81C,      0xF9080003, 0x81C,      0xF80A0003,
+	0x81C,      0xF70C0003, 0x81C,      0xF60E0003, 0x81C,      0xF5100003,
+	0x81C,      0xF4120003, 0x81C,      0xF3140003, 0x81C,      0xF2160003,
+	0x81C,      0xF1180003, 0x81C,      0xF01A0003, 0x81C,      0xEF1C0003,
+	0x81C,      0xEE1E0003, 0x81C,      0xED200003, 0x81C,      0xEC220003,
+	0x81C,      0xEB240003, 0x81C,      0xEA260003, 0x81C,      0xE9280003,
+	0x81C,      0xE82A0003, 0x81C,      0xE72C0003, 0x81C,      0xE62E0003,
+	0x81C,      0xE5300003, 0x81C,      0xC8320003, 0x81C,      0xC7340003,
+	0x81C,      0xC6360003, 0x81C,      0xC5380003, 0x81C,      0xC43A0003,
+	0x81C,      0xC33C0003, 0x81C,      0xC23E0003, 0x81C,      0xC1400003,
+	0x81C,      0xC0420003, 0x81C,      0xA5440003, 0x81C,      0xA4460003,
+	0x81C,      0xA3480003, 0x81C,      0xA24A0003, 0x81C,      0xA14C0003,
+	0x81C,      0x834E0003, 0x81C,      0x82500003, 0x81C,      0x81520003,
+	0x81C,      0x80540003, 0x81C,      0x65560003, 0x81C,      0x64580003,
+	0x81C,      0x635A0003, 0x81C,      0x625C0003, 0x81C,      0x435E0003,
+	0x81C,      0x42600003, 0x81C,      0x41620003, 0x81C,      0x40640003,
+	0x81C,      0x06660003, 0x81C,      0x05680003, 0x81C,      0x046A0003,
+	0x81C,      0x036C0003, 0x81C,      0x026E0003, 0x81C,      0x01700003,
+	0x81C,      0x00720003, 0x81C,      0x00740003, 0x81C,      0x00760003,
+	0x81C,      0x00780003, 0x81C,      0x007A0003, 0x81C,      0x007C0003,
+	0x81C,      0x007E0003, 0xA0000000, 0x00000000, 0x81C,      0xFF000003,
+	0x81C,      0xFE000003, 0x81C,      0xFD020003, 0x81C,      0xFC040003,
+	0x81C,      0xFB060003, 0x81C,      0xFA080003, 0x81C,      0xF90A0003,
+	0x81C,      0xF80C0003, 0x81C,      0xF70E0003, 0x81C,      0xF6100003,
+	0x81C,      0xF5120003, 0x81C,      0xF4140003, 0x81C,      0xF3160003,
+	0x81C,      0xF2180003, 0x81C,      0xF11A0003, 0x81C,      0xF01C0003,
+	0x81C,      0xEF1E0003, 0x81C,      0xEE200003, 0x81C,      0xED220003,
+	0x81C,      0xEC240003, 0x81C,      0xEB260003, 0x81C,      0xEA280003,
+	0x81C,      0xE92A0003, 0x81C,      0xE82C0003, 0x81C,      0xE72E0003,
+	0x81C,      0xE6300003, 0x81C,      0xE5320003, 0x81C,      0xC8340003,
+	0x81C,      0xC7360003, 0x81C,      0xC6380003, 0x81C,      0xC53A0003,
+	0x81C,      0xC43C0003, 0x81C,      0xC33E0003, 0x81C,      0xC2400003,
+	0x81C,      0xC1420003, 0x81C,      0xC0440003, 0x81C,      0xA3460003,
+	0x81C,      0xA2480003, 0x81C,      0xA14A0003, 0x81C,      0xA04C0003,
+	0x81C,      0x824E0003, 0x81C,      0x81500003, 0x81C,      0x80520003,
+	0x81C,      0x64540003, 0x81C,      0x63560003, 0x81C,      0x62580003,
+	0x81C,      0x445A0003, 0x81C,      0x435C0003, 0x81C,      0x425E0003,
+	0x81C,      0x41600003, 0x81C,      0x40620003, 0x81C,      0x05640003,
+	0x81C,      0x04660003, 0x81C,      0x03680003, 0x81C,      0x026A0003,
+	0x81C,      0x016C0003, 0x81C,      0x006E0003, 0x81C,      0x00700003,
+	0x81C,      0x00720003, 0x81C,      0x00740003, 0x81C,      0x00760003,
+	0x81C,      0x00780003, 0x81C,      0x007A0003, 0x81C,      0x007C0003,
+	0x81C,      0x007E0003, 0xB0000000, 0x00000000, 0x8000100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x81C,      0xF8000103, 0x81C,      0xF7020103,
+	0x81C,      0xF6040103, 0x81C,      0xF5060103, 0x81C,      0xF4080103,
+	0x81C,      0xF30A0103, 0x81C,      0xF20C0103, 0x81C,      0xF10E0103,
+	0x81C,      0xF0100103, 0x81C,      0xEF120103, 0x81C,      0xEE140103,
+	0x81C,      0xED160103, 0x81C,      0xEC180103, 0x81C,      0xEB1A0103,
+	0x81C,      0xEA1C0103, 0x81C,      0xE91E0103, 0x81C,      0xE8200103,
+	0x81C,      0xE7220103, 0x81C,      0xE6240103, 0x81C,      0xE5260103,
+	0x81C,      0xE4280103, 0x81C,      0xE32A0103, 0x81C,      0xE22C0103,
+	0x81C,      0xC32E0103, 0x81C,      0xC2300103, 0x81C,      0xC1320103,
+	0x81C,      0xA3340103, 0x81C,      0xA2360103, 0x81C,      0xA1380103,
+	0x81C,      0xA03A0103, 0x81C,      0x823C0103, 0x81C,      0x813E0103,
+	0x81C,      0x80400103, 0x81C,      0x64420103, 0x81C,      0x63440103,
+	0x81C,      0x62460103, 0x81C,      0x61480103, 0x81C,      0x434A0103,
+	0x81C,      0x424C0103, 0x81C,      0x414E0103, 0x81C,      0x40500103,
+	0x81C,      0x22520103, 0x81C,      0x21540103, 0x81C,      0x20560103,
+	0x81C,      0x04580103, 0x81C,      0x035A0103, 0x81C,      0x025C0103,
+	0x81C,      0x015E0103, 0x81C,      0x00600103, 0x81C,      0x00620103,
+	0x81C,      0x00640103, 0x81C,      0x00660103, 0x81C,      0x00680103,
+	0x81C,      0x006A0103, 0x81C,      0x006C0103, 0x81C,      0x006E0103,
+	0x81C,      0x00700103, 0x81C,      0x00720103, 0x81C,      0x00740103,
+	0x81C,      0x00760103, 0x81C,      0x00780103, 0x81C,      0x007A0103,
+	0x81C,      0x007C0103, 0x81C,      0x007E0103, 0x9000100f, 0x05050505,
+	0x40000000, 0x00000000, 0x81C,      0xFA000103, 0x81C,      0xF9020103,
+	0x81C,      0xF8040103, 0x81C,      0xF7060103, 0x81C,      0xF6080103,
+	0x81C,      0xF50A0103, 0x81C,      0xF40C0103, 0x81C,      0xF30E0103,
+	0x81C,      0xF2100103, 0x81C,      0xF1120103, 0x81C,      0xF0140103,
+	0x81C,      0xEF160103, 0x81C,      0xEE180103, 0x81C,      0xED1A0103,
+	0x81C,      0xEC1C0103, 0x81C,      0xEB1E0103, 0x81C,      0xEA200103,
+	0x81C,      0xE9220103, 0x81C,      0xE8240103, 0x81C,      0xE7260103,
+	0x81C,      0xE6280103, 0x81C,      0xE52A0103, 0x81C,      0xC42C0103,
+	0x81C,      0xC32E0103, 0x81C,      0xC2300103, 0x81C,      0xC1320103,
+	0x81C,      0xA4340103, 0x81C,      0xA3360103, 0x81C,      0xA2380103,
+	0x81C,      0xA13A0103, 0x81C,      0x833C0103, 0x81C,      0x823E0103,
+	0x81C,      0x81400103, 0x81C,      0x63420103, 0x81C,      0x62440103,
+	0x81C,      0x61460103, 0x81C,      0x60480103, 0x81C,      0x424A0103,
+	0x81C,      0x414C0103, 0x81C,      0x404E0103, 0x81C,      0x22500103,
+	0x81C,      0x21520103, 0x81C,      0x20540103, 0x81C,      0x03560103,
+	0x81C,      0x02580103, 0x81C,      0x015A0103, 0x81C,      0x005C0103,
+	0x81C,      0x005E0103, 0x81C,      0x00600103, 0x81C,      0x00620103,
+	0x81C,      0x00640103, 0x81C,      0x00660103, 0x81C,      0x00680103,
+	0x81C,      0x006A0103, 0x81C,      0x006C0103, 0x81C,      0x006E0103,
+	0x81C,      0x00700103, 0x81C,      0x00720103, 0x81C,      0x00740103,
+	0x81C,      0x00760103, 0x81C,      0x00780103, 0x81C,      0x007A0103,
+	0x81C,      0x007C0103, 0x81C,      0x007E0103, 0x9000100f, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xF8000103, 0x81C,      0xF7020103,
+	0x81C,      0xF6040103, 0x81C,      0xF5060103, 0x81C,      0xF4080103,
+	0x81C,      0xF30A0103, 0x81C,      0xF20C0103, 0x81C,      0xF10E0103,
+	0x81C,      0xF0100103, 0x81C,      0xEF120103, 0x81C,      0xEE140103,
+	0x81C,      0xED160103, 0x81C,      0xEC180103, 0x81C,      0xEB1A0103,
+	0x81C,      0xEA1C0103, 0x81C,      0xE91E0103, 0x81C,      0xE8200103,
+	0x81C,      0xE7220103, 0x81C,      0xE6240103, 0x81C,      0xE5260103,
+	0x81C,      0xE4280103, 0x81C,      0xE32A0103, 0x81C,      0xC32C0103,
+	0x81C,      0xC22E0103, 0x81C,      0xC1300103, 0x81C,      0xC0320103,
+	0x81C,      0xA3340103, 0x81C,      0xA2360103, 0x81C,      0xA1380103,
+	0x81C,      0xA03A0103, 0x81C,      0x823C0103, 0x81C,      0x813E0103,
+	0x81C,      0x80400103, 0x81C,      0x63420103, 0x81C,      0x62440103,
+	0x81C,      0x61460103, 0x81C,      0x60480103, 0x81C,      0x424A0103,
+	0x81C,      0x414C0103, 0x81C,      0x404E0103, 0x81C,      0x06500103,
+	0x81C,      0x05520103, 0x81C,      0x04540103, 0x81C,      0x03560103,
+	0x81C,      0x02580103, 0x81C,      0x015A0103, 0x81C,      0x005C0103,
+	0x81C,      0x005E0103, 0x81C,      0x00600103, 0x81C,      0x00620103,
+	0x81C,      0x00640103, 0x81C,      0x00660103, 0x81C,      0x00680103,
+	0x81C,      0x006A0103, 0x81C,      0x006C0103, 0x81C,      0x006E0103,
+	0x81C,      0x00700103, 0x81C,      0x00720103, 0x81C,      0x00740103,
+	0x81C,      0x00760103, 0x81C,      0x00780103, 0x81C,      0x007A0103,
+	0x81C,      0x007C0103, 0x81C,      0x007E0103, 0x9000200f, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xF8000103, 0x81C,      0xF7020103,
+	0x81C,      0xF6040103, 0x81C,      0xF5060103, 0x81C,      0xF4080103,
+	0x81C,      0xF30A0103, 0x81C,      0xF20C0103, 0x81C,      0xF10E0103,
+	0x81C,      0xF0100103, 0x81C,      0xEF120103, 0x81C,      0xEE140103,
+	0x81C,      0xED160103, 0x81C,      0xEC180103, 0x81C,      0xEB1A0103,
+	0x81C,      0xEA1C0103, 0x81C,      0xE91E0103, 0x81C,      0xE8200103,
+	0x81C,      0xE7220103, 0x81C,      0xE6240103, 0x81C,      0xE5260103,
+	0x81C,      0xE4280103, 0x81C,      0xE32A0103, 0x81C,      0xC32C0103,
+	0x81C,      0xC22E0103, 0x81C,      0xC1300103, 0x81C,      0xC0320103,
+	0x81C,      0xA3340103, 0x81C,      0xA2360103, 0x81C,      0xA1380103,
+	0x81C,      0xA03A0103, 0x81C,      0x823C0103, 0x81C,      0x813E0103,
+	0x81C,      0x80400103, 0x81C,      0x63420103, 0x81C,      0x62440103,
+	0x81C,      0x61460103, 0x81C,      0x60480103, 0x81C,      0x424A0103,
+	0x81C,      0x414C0103, 0x81C,      0x404E0103, 0x81C,      0x22500103,
+	0x81C,      0x21520103, 0x81C,      0x20540103, 0x81C,      0x03560103,
+	0x81C,      0x02580103, 0x81C,      0x015A0103, 0x81C,      0x005C0103,
+	0x81C,      0x005E0103, 0x81C,      0x00600103, 0x81C,      0x00620103,
+	0x81C,      0x00640103, 0x81C,      0x00660103, 0x81C,      0x00680103,
+	0x81C,      0x006A0103, 0x81C,      0x006C0103, 0x81C,      0x006E0103,
+	0x81C,      0x00700103, 0x81C,      0x00720103, 0x81C,      0x00740103,
+	0x81C,      0x00760103, 0x81C,      0x00780103, 0x81C,      0x007A0103,
+	0x81C,      0x007C0103, 0x81C,      0x007E0103, 0x9000200c, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xF8000103, 0x81C,      0xF7020103,
+	0x81C,      0xF6040103, 0x81C,      0xF5060103, 0x81C,      0xF4080103,
+	0x81C,      0xF30A0103, 0x81C,      0xF20C0103, 0x81C,      0xF10E0103,
+	0x81C,      0xF0100103, 0x81C,      0xEF120103, 0x81C,      0xEE140103,
+	0x81C,      0xED160103, 0x81C,      0xEC180103, 0x81C,      0xEB1A0103,
+	0x81C,      0xEA1C0103, 0x81C,      0xE91E0103, 0x81C,      0xE8200103,
+	0x81C,      0xE7220103, 0x81C,      0xE6240103, 0x81C,      0xE5260103,
+	0x81C,      0xE4280103, 0x81C,      0xE32A0103, 0x81C,      0xC32C0103,
+	0x81C,      0xC22E0103, 0x81C,      0xC1300103, 0x81C,      0xC0320103,
+	0x81C,      0xA3340103, 0x81C,      0xA2360103, 0x81C,      0xA1380103,
+	0x81C,      0xA03A0103, 0x81C,      0x823C0103, 0x81C,      0x813E0103,
+	0x81C,      0x80400103, 0x81C,      0x63420103, 0x81C,      0x62440103,
+	0x81C,      0x61460103, 0x81C,      0x60480103, 0x81C,      0x424A0103,
+	0x81C,      0x414C0103, 0x81C,      0x404E0103, 0x81C,      0x22500103,
+	0x81C,      0x21520103, 0x81C,      0x20540103, 0x81C,      0x03560103,
+	0x81C,      0x02580103, 0x81C,      0x015A0103, 0x81C,      0x005C0103,
+	0x81C,      0x005E0103, 0x81C,      0x00600103, 0x81C,      0x00620103,
+	0x81C,      0x00640103, 0x81C,      0x00660103, 0x81C,      0x00680103,
+	0x81C,      0x006A0103, 0x81C,      0x006C0103, 0x81C,      0x006E0103,
+	0x81C,      0x00700103, 0x81C,      0x00720103, 0x81C,      0x00740103,
+	0x81C,      0x00760103, 0x81C,      0x00780103, 0x81C,      0x007A0103,
+	0x81C,      0x007C0103, 0x81C,      0x007E0103, 0x90012100, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xFD000103, 0x81C,      0xFC020103,
+	0x81C,      0xFB040103, 0x81C,      0xFA060103, 0x81C,      0xF9080103,
+	0x81C,      0xF80A0103, 0x81C,      0xF70C0103, 0x81C,      0xF60E0103,
+	0x81C,      0xF5100103, 0x81C,      0xF4120103, 0x81C,      0xF3140103,
+	0x81C,      0xF2160103, 0x81C,      0xF1180103, 0x81C,      0xF01A0103,
+	0x81C,      0xEF1C0103, 0x81C,      0xEE1E0103, 0x81C,      0xED200103,
+	0x81C,      0xEC220103, 0x81C,      0xEB240103, 0x81C,      0xEA260103,
+	0x81C,      0xE9280103, 0x81C,      0xE82A0103, 0x81C,      0xE72C0103,
+	0x81C,      0xE62E0103, 0x81C,      0xE5300103, 0x81C,      0xE4320103,
+	0x81C,      0xE3340103, 0x81C,      0xC6360103, 0x81C,      0xC5380103,
+	0x81C,      0xC43A0103, 0x81C,      0xC33C0103, 0x81C,      0xC23E0103,
+	0x81C,      0xA5400103, 0x81C,      0xA4420103, 0x81C,      0xA3440103,
+	0x81C,      0xA2460103, 0x81C,      0xA1480103, 0x81C,      0x834A0103,
+	0x81C,      0x824C0103, 0x81C,      0x814E0103, 0x81C,      0x63500103,
+	0x81C,      0x62520103, 0x81C,      0x61540103, 0x81C,      0x43560103,
+	0x81C,      0x42580103, 0x81C,      0x245A0103, 0x81C,      0x235C0103,
+	0x81C,      0x225E0103, 0x81C,      0x21600103, 0x81C,      0x04620103,
+	0x81C,      0x03640103, 0x81C,      0x02660103, 0x81C,      0x01680103,
+	0x81C,      0x006A0103, 0x81C,      0x006C0103, 0x81C,      0x006E0103,
+	0x81C,      0x00700103, 0x81C,      0x00720103, 0x81C,      0x00740103,
+	0x81C,      0x00760103, 0x81C,      0x00780103, 0x81C,      0x007A0103,
+	0x81C,      0x007C0103, 0x81C,      0x007E0103, 0x90001004, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xF8000103, 0x81C,      0xF7020103,
+	0x81C,      0xF6040103, 0x81C,      0xF5060103, 0x81C,      0xF4080103,
+	0x81C,      0xF30A0103, 0x81C,      0xF20C0103, 0x81C,      0xF10E0103,
+	0x81C,      0xF0100103, 0x81C,      0xEF120103, 0x81C,      0xEE140103,
+	0x81C,      0xED160103, 0x81C,      0xEC180103, 0x81C,      0xEB1A0103,
+	0x81C,      0xEA1C0103, 0x81C,      0xE91E0103, 0x81C,      0xE8200103,
+	0x81C,      0xE7220103, 0x81C,      0xE6240103, 0x81C,      0xE5260103,
+	0x81C,      0xE4280103, 0x81C,      0xE32A0103, 0x81C,      0xE22C0103,
+	0x81C,      0xC32E0103, 0x81C,      0xC2300103, 0x81C,      0xC1320103,
+	0x81C,      0xA3340103, 0x81C,      0xA2360103, 0x81C,      0xA1380103,
+	0x81C,      0xA03A0103, 0x81C,      0x823C0103, 0x81C,      0x813E0103,
+	0x81C,      0x80400103, 0x81C,      0x64420103, 0x81C,      0x63440103,
+	0x81C,      0x62460103, 0x81C,      0x61480103, 0x81C,      0x434A0103,
+	0x81C,      0x424C0103, 0x81C,      0x414E0103, 0x81C,      0x40500103,
+	0x81C,      0x22520103, 0x81C,      0x21540103, 0x81C,      0x20560103,
+	0x81C,      0x04580103, 0x81C,      0x035A0103, 0x81C,      0x025C0103,
+	0x81C,      0x015E0103, 0x81C,      0x00600103, 0x81C,      0x00620103,
+	0x81C,      0x00640103, 0x81C,      0x00660103, 0x81C,      0x00680103,
+	0x81C,      0x006A0103, 0x81C,      0x006C0103, 0x81C,      0x006E0103,
+	0x81C,      0x00700103, 0x81C,      0x00720103, 0x81C,      0x00740103,
+	0x81C,      0x00760103, 0x81C,      0x00780103, 0x81C,      0x007A0103,
+	0x81C,      0x007C0103, 0x81C,      0x007E0103, 0x90011000, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xFD000103, 0x81C,      0xFC020103,
+	0x81C,      0xFB040103, 0x81C,      0xFA060103, 0x81C,      0xF9080103,
+	0x81C,      0xF80A0103, 0x81C,      0xF70C0103, 0x81C,      0xF60E0103,
+	0x81C,      0xF5100103, 0x81C,      0xF4120103, 0x81C,      0xF3140103,
+	0x81C,      0xF2160103, 0x81C,      0xF1180103, 0x81C,      0xF01A0103,
+	0x81C,      0xEE1C0103, 0x81C,      0xED1E0103, 0x81C,      0xEC200103,
+	0x81C,      0xEB220103, 0x81C,      0xEA240103, 0x81C,      0xE9260103,
+	0x81C,      0xE8280103, 0x81C,      0xE72A0103, 0x81C,      0xE62C0103,
+	0x81C,      0xE52E0103, 0x81C,      0xE4300103, 0x81C,      0xE3320103,
+	0x81C,      0xE2340103, 0x81C,      0xC5360103, 0x81C,      0xC4380103,
+	0x81C,      0xC33A0103, 0x81C,      0xC23C0103, 0x81C,      0xA53E0103,
+	0x81C,      0xA4400103, 0x81C,      0xA3420103, 0x81C,      0xA2440103,
+	0x81C,      0xA1460103, 0x81C,      0x83480103, 0x81C,      0x824A0103,
+	0x81C,      0x814C0103, 0x81C,      0x804E0103, 0x81C,      0x63500103,
+	0x81C,      0x62520103, 0x81C,      0x61540103, 0x81C,      0x43560103,
+	0x81C,      0x42580103, 0x81C,      0x415A0103, 0x81C,      0x405C0103,
+	0x81C,      0x225E0103, 0x81C,      0x21600103, 0x81C,      0x20620103,
+	0x81C,      0x03640103, 0x81C,      0x02660103, 0x81C,      0x01680103,
+	0x81C,      0x006A0103, 0x81C,      0x006C0103, 0x81C,      0x006E0103,
+	0x81C,      0x00700103, 0x81C,      0x00720103, 0x81C,      0x00740103,
+	0x81C,      0x00760103, 0x81C,      0x00780103, 0x81C,      0x007A0103,
+	0x81C,      0x007C0103, 0x81C,      0x007E0103, 0x90002100, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xFD000103, 0x81C,      0xFC020103,
+	0x81C,      0xFB040103, 0x81C,      0xFA060103, 0x81C,      0xF9080103,
+	0x81C,      0xF80A0103, 0x81C,      0xF70C0103, 0x81C,      0xF60E0103,
+	0x81C,      0xF5100103, 0x81C,      0xF4120103, 0x81C,      0xF3140103,
+	0x81C,      0xF2160103, 0x81C,      0xF1180103, 0x81C,      0xF01A0103,
+	0x81C,      0xEF1C0103, 0x81C,      0xEE1E0103, 0x81C,      0xED200103,
+	0x81C,      0xEC220103, 0x81C,      0xEB240103, 0x81C,      0xEA260103,
+	0x81C,      0xE9280103, 0x81C,      0xE82A0103, 0x81C,      0xE72C0103,
+	0x81C,      0xE62E0103, 0x81C,      0xE5300103, 0x81C,      0xE4320103,
+	0x81C,      0xE3340103, 0x81C,      0xE2360103, 0x81C,      0xC5380103,
+	0x81C,      0xC43A0103, 0x81C,      0xC33C0103, 0x81C,      0xC23E0103,
+	0x81C,      0xA5400103, 0x81C,      0xA4420103, 0x81C,      0xA3440103,
+	0x81C,      0xA2460103, 0x81C,      0xA1480103, 0x81C,      0x834A0103,
+	0x81C,      0x824C0103, 0x81C,      0x814E0103, 0x81C,      0x64500103,
+	0x81C,      0x63520103, 0x81C,      0x62540103, 0x81C,      0x61560103,
+	0x81C,      0x42580103, 0x81C,      0x415A0103, 0x81C,      0x405C0103,
+	0x81C,      0x065E0103, 0x81C,      0x05600103, 0x81C,      0x04620103,
+	0x81C,      0x03640103, 0x81C,      0x02660103, 0x81C,      0x01680103,
+	0x81C,      0x006A0103, 0x81C,      0x006C0103, 0x81C,      0x006E0103,
+	0x81C,      0x00700103, 0x81C,      0x00720103, 0x81C,      0x00740103,
+	0x81C,      0x00760103, 0x81C,      0x00780103, 0x81C,      0x007A0103,
+	0x81C,      0x007C0103, 0x81C,      0x007E0103, 0x90002000, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xFE000103, 0x81C,      0xFD020103,
+	0x81C,      0xFC040103, 0x81C,      0xFB060103, 0x81C,      0xFA080103,
+	0x81C,      0xF90A0103, 0x81C,      0xF80C0103, 0x81C,      0xF70E0103,
+	0x81C,      0xF6100103, 0x81C,      0xF5120103, 0x81C,      0xF4140103,
+	0x81C,      0xF3160103, 0x81C,      0xF2180103, 0x81C,      0xF11A0103,
+	0x81C,      0xF01C0103, 0x81C,      0xEF1E0103, 0x81C,      0xEE200103,
+	0x81C,      0xED220103, 0x81C,      0xEC240103, 0x81C,      0xEB260103,
+	0x81C,      0xEA280103, 0x81C,      0xE92A0103, 0x81C,      0xE82C0103,
+	0x81C,      0xE72E0103, 0x81C,      0xE6300103, 0x81C,      0xE5320103,
+	0x81C,      0xE4340103, 0x81C,      0xE3360103, 0x81C,      0xC6380103,
+	0x81C,      0xC53A0103, 0x81C,      0xC43C0103, 0x81C,      0xC33E0103,
+	0x81C,      0xA5400103, 0x81C,      0xA4420103, 0x81C,      0xA3440103,
+	0x81C,      0xA2460103, 0x81C,      0xA1480103, 0x81C,      0xA04A0103,
+	0x81C,      0x824C0103, 0x81C,      0x814E0103, 0x81C,      0x80500103,
+	0x81C,      0x64520103, 0x81C,      0x63540103, 0x81C,      0x62560103,
+	0x81C,      0x61580103, 0x81C,      0x605A0103, 0x81C,      0x235C0103,
+	0x81C,      0x225E0103, 0x81C,      0x21600103, 0x81C,      0x20620103,
+	0x81C,      0x03640103, 0x81C,      0x02660103, 0x81C,      0x01680103,
+	0x81C,      0x006A0103, 0x81C,      0x006C0103, 0x81C,      0x006E0103,
+	0x81C,      0x00700103, 0x81C,      0x00720103, 0x81C,      0x00740103,
+	0x81C,      0x00760103, 0x81C,      0x00780103, 0x81C,      0x007A0103,
+	0x81C,      0x007C0103, 0x81C,      0x007E0103, 0xA0000000, 0x00000000,
+	0x81C,      0xFE000103, 0x81C,      0xFD020103, 0x81C,      0xFC040103,
+	0x81C,      0xFB060103, 0x81C,      0xFA080103, 0x81C,      0xF90A0103,
+	0x81C,      0xF80C0103, 0x81C,      0xF70E0103, 0x81C,      0xF6100103,
+	0x81C,      0xF5120103, 0x81C,      0xF4140103, 0x81C,      0xF3160103,
+	0x81C,      0xF2180103, 0x81C,      0xF11A0103, 0x81C,      0xF01C0103,
+	0x81C,      0xEF1E0103, 0x81C,      0xEE200103, 0x81C,      0xED220103,
+	0x81C,      0xEC240103, 0x81C,      0xEB260103, 0x81C,      0xEA280103,
+	0x81C,      0xE92A0103, 0x81C,      0xE82C0103, 0x81C,      0xE72E0103,
+	0x81C,      0xE6300103, 0x81C,      0xE5320103, 0x81C,      0xE4340103,
+	0x81C,      0xE3360103, 0x81C,      0xC6380103, 0x81C,      0xC53A0103,
+	0x81C,      0xC43C0103, 0x81C,      0xC33E0103, 0x81C,      0xA5400103,
+	0x81C,      0xA4420103, 0x81C,      0xA3440103, 0x81C,      0xA2460103,
+	0x81C,      0xA1480103, 0x81C,      0xA04A0103, 0x81C,      0x824C0103,
+	0x81C,      0x814E0103, 0x81C,      0x80500103, 0x81C,      0x64520103,
+	0x81C,      0x63540103, 0x81C,      0x62560103, 0x81C,      0x61580103,
+	0x81C,      0x605A0103, 0x81C,      0x235C0103, 0x81C,      0x225E0103,
+	0x81C,      0x21600103, 0x81C,      0x20620103, 0x81C,      0x03640103,
+	0x81C,      0x02660103, 0x81C,      0x01680103, 0x81C,      0x006A0103,
+	0x81C,      0x006C0103, 0x81C,      0x006E0103, 0x81C,      0x00700103,
+	0x81C,      0x00720103, 0x81C,      0x00740103, 0x81C,      0x00760103,
+	0x81C,      0x00780103, 0x81C,      0x007A0103, 0x81C,      0x007C0103,
+	0x81C,      0x007E0103, 0xB0000000, 0x00000000, 0x8000100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x81C,      0xF8000203, 0x81C,      0xF7020203,
+	0x81C,      0xF6040203, 0x81C,      0xF5060203, 0x81C,      0xF4080203,
+	0x81C,      0xF30A0203, 0x81C,      0xF20C0203, 0x81C,      0xF10E0203,
+	0x81C,      0xF0100203, 0x81C,      0xEF120203, 0x81C,      0xEE140203,
+	0x81C,      0xED160203, 0x81C,      0xEC180203, 0x81C,      0xEB1A0203,
+	0x81C,      0xEA1C0203, 0x81C,      0xE91E0203, 0x81C,      0xE8200203,
+	0x81C,      0xE7220203, 0x81C,      0xE6240203, 0x81C,      0xE5260203,
+	0x81C,      0xE4280203, 0x81C,      0xE32A0203, 0x81C,      0xC42C0203,
+	0x81C,      0xC32E0203, 0x81C,      0xC2300203, 0x81C,      0xC1320203,
+	0x81C,      0xA3340203, 0x81C,      0xA2360203, 0x81C,      0xA1380203,
+	0x81C,      0xA03A0203, 0x81C,      0x823C0203, 0x81C,      0x813E0203,
+	0x81C,      0x80400203, 0x81C,      0x65420203, 0x81C,      0x64440203,
+	0x81C,      0x63460203, 0x81C,      0x62480203, 0x81C,      0x614A0203,
+	0x81C,      0x424C0203, 0x81C,      0x414E0203, 0x81C,      0x40500203,
+	0x81C,      0x22520203, 0x81C,      0x21540203, 0x81C,      0x20560203,
+	0x81C,      0x04580203, 0x81C,      0x035A0203, 0x81C,      0x025C0203,
+	0x81C,      0x015E0203, 0x81C,      0x00600203, 0x81C,      0x00620203,
+	0x81C,      0x00640203, 0x81C,      0x00660203, 0x81C,      0x00680203,
+	0x81C,      0x006A0203, 0x81C,      0x006C0203, 0x81C,      0x006E0203,
+	0x81C,      0x00700203, 0x81C,      0x00720203, 0x81C,      0x00740203,
+	0x81C,      0x00760203, 0x81C,      0x00780203, 0x81C,      0x007A0203,
+	0x81C,      0x007C0203, 0x81C,      0x007E0203, 0x9000100f, 0x05050505,
+	0x40000000, 0x00000000, 0x81C,      0xF9000203, 0x81C,      0xF8020203,
+	0x81C,      0xF7040203, 0x81C,      0xF6060203, 0x81C,      0xF5080203,
+	0x81C,      0xF40A0203, 0x81C,      0xF30C0203, 0x81C,      0xF20E0203,
+	0x81C,      0xF1100203, 0x81C,      0xF0120203, 0x81C,      0xEF140203,
+	0x81C,      0xEE160203, 0x81C,      0xED180203, 0x81C,      0xEC1A0203,
+	0x81C,      0xEB1C0203, 0x81C,      0xEA1E0203, 0x81C,      0xE9200203,
+	0x81C,      0xE8220203, 0x81C,      0xE7240203, 0x81C,      0xE6260203,
+	0x81C,      0xE5280203, 0x81C,      0xC42A0203, 0x81C,      0xC32C0203,
+	0x81C,      0xC22E0203, 0x81C,      0xC1300203, 0x81C,      0xC0320203,
+	0x81C,      0xA3340203, 0x81C,      0xA2360203, 0x81C,      0xA1380203,
+	0x81C,      0xA03A0203, 0x81C,      0x823C0203, 0x81C,      0x813E0203,
+	0x81C,      0x80400203, 0x81C,      0x64420203, 0x81C,      0x63440203,
+	0x81C,      0x62460203, 0x81C,      0x61480203, 0x81C,      0x604A0203,
+	0x81C,      0x414C0203, 0x81C,      0x404E0203, 0x81C,      0x22500203,
+	0x81C,      0x21520203, 0x81C,      0x20540203, 0x81C,      0x03560203,
+	0x81C,      0x02580203, 0x81C,      0x015A0203, 0x81C,      0x005C0203,
+	0x81C,      0x005E0203, 0x81C,      0x00600203, 0x81C,      0x00620203,
+	0x81C,      0x00640203, 0x81C,      0x00660203, 0x81C,      0x00680203,
+	0x81C,      0x006A0203, 0x81C,      0x006C0203, 0x81C,      0x006E0203,
+	0x81C,      0x00700203, 0x81C,      0x00720203, 0x81C,      0x00740203,
+	0x81C,      0x00760203, 0x81C,      0x00780203, 0x81C,      0x007A0203,
+	0x81C,      0x007C0203, 0x81C,      0x007E0203, 0x9000100f, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xF7000203, 0x81C,      0xF6020203,
+	0x81C,      0xF5040203, 0x81C,      0xF4060203, 0x81C,      0xF3080203,
+	0x81C,      0xF20A0203, 0x81C,      0xF10C0203, 0x81C,      0xF00E0203,
+	0x81C,      0xEF100203, 0x81C,      0xEE120203, 0x81C,      0xED140203,
+	0x81C,      0xEC160203, 0x81C,      0xEB180203, 0x81C,      0xEA1A0203,
+	0x81C,      0xE91C0203, 0x81C,      0xE81E0203, 0x81C,      0xE7200203,
+	0x81C,      0xE6220203, 0x81C,      0xE5240203, 0x81C,      0xE4260203,
+	0x81C,      0xE3280203, 0x81C,      0xC42A0203, 0x81C,      0xC32C0203,
+	0x81C,      0xC22E0203, 0x81C,      0xC1300203, 0x81C,      0xC0320203,
+	0x81C,      0xA3340203, 0x81C,      0xA2360203, 0x81C,      0xA1380203,
+	0x81C,      0xA03A0203, 0x81C,      0x823C0203, 0x81C,      0x813E0203,
+	0x81C,      0x80400203, 0x81C,      0x63420203, 0x81C,      0x62440203,
+	0x81C,      0x61460203, 0x81C,      0x60480203, 0x81C,      0x424A0203,
+	0x81C,      0x414C0203, 0x81C,      0x404E0203, 0x81C,      0x06500203,
+	0x81C,      0x05520203, 0x81C,      0x04540203, 0x81C,      0x03560203,
+	0x81C,      0x02580203, 0x81C,      0x015A0203, 0x81C,      0x005C0203,
+	0x81C,      0x005E0203, 0x81C,      0x00600203, 0x81C,      0x00620203,
+	0x81C,      0x00640203, 0x81C,      0x00660203, 0x81C,      0x00680203,
+	0x81C,      0x006A0203, 0x81C,      0x006C0203, 0x81C,      0x006E0203,
+	0x81C,      0x00700203, 0x81C,      0x00720203, 0x81C,      0x00740203,
+	0x81C,      0x00760203, 0x81C,      0x00780203, 0x81C,      0x007A0203,
+	0x81C,      0x007C0203, 0x81C,      0x007E0203, 0x9000200f, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xF7000203, 0x81C,      0xF6020203,
+	0x81C,      0xF5040203, 0x81C,      0xF4060203, 0x81C,      0xF3080203,
+	0x81C,      0xF20A0203, 0x81C,      0xF10C0203, 0x81C,      0xF00E0203,
+	0x81C,      0xEF100203, 0x81C,      0xEE120203, 0x81C,      0xED140203,
+	0x81C,      0xEC160203, 0x81C,      0xEB180203, 0x81C,      0xEA1A0203,
+	0x81C,      0xE91C0203, 0x81C,      0xE81E0203, 0x81C,      0xE7200203,
+	0x81C,      0xE6220203, 0x81C,      0xE5240203, 0x81C,      0xE4260203,
+	0x81C,      0xE3280203, 0x81C,      0xC42A0203, 0x81C,      0xC32C0203,
+	0x81C,      0xC22E0203, 0x81C,      0xC1300203, 0x81C,      0xC0320203,
+	0x81C,      0xA3340203, 0x81C,      0xA2360203, 0x81C,      0xA1380203,
+	0x81C,      0xA03A0203, 0x81C,      0x823C0203, 0x81C,      0x813E0203,
+	0x81C,      0x80400203, 0x81C,      0x64420203, 0x81C,      0x63440203,
+	0x81C,      0x62460203, 0x81C,      0x61480203, 0x81C,      0x604A0203,
+	0x81C,      0x414C0203, 0x81C,      0x404E0203, 0x81C,      0x22500203,
+	0x81C,      0x21520203, 0x81C,      0x20540203, 0x81C,      0x03560203,
+	0x81C,      0x02580203, 0x81C,      0x015A0203, 0x81C,      0x005C0203,
+	0x81C,      0x005E0203, 0x81C,      0x00600203, 0x81C,      0x00620203,
+	0x81C,      0x00640203, 0x81C,      0x00660203, 0x81C,      0x00680203,
+	0x81C,      0x006A0203, 0x81C,      0x006C0203, 0x81C,      0x006E0203,
+	0x81C,      0x00700203, 0x81C,      0x00720203, 0x81C,      0x00740203,
+	0x81C,      0x00760203, 0x81C,      0x00780203, 0x81C,      0x007A0203,
+	0x81C,      0x007C0203, 0x81C,      0x007E0203, 0x9000200c, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xF7000203, 0x81C,      0xF6020203,
+	0x81C,      0xF5040203, 0x81C,      0xF4060203, 0x81C,      0xF3080203,
+	0x81C,      0xF20A0203, 0x81C,      0xF10C0203, 0x81C,      0xF00E0203,
+	0x81C,      0xEF100203, 0x81C,      0xEE120203, 0x81C,      0xED140203,
+	0x81C,      0xEC160203, 0x81C,      0xEB180203, 0x81C,      0xEA1A0203,
+	0x81C,      0xE91C0203, 0x81C,      0xE81E0203, 0x81C,      0xE7200203,
+	0x81C,      0xE6220203, 0x81C,      0xE5240203, 0x81C,      0xE4260203,
+	0x81C,      0xE3280203, 0x81C,      0xC42A0203, 0x81C,      0xC32C0203,
+	0x81C,      0xC22E0203, 0x81C,      0xC1300203, 0x81C,      0xC0320203,
+	0x81C,      0xA3340203, 0x81C,      0xA2360203, 0x81C,      0xA1380203,
+	0x81C,      0xA03A0203, 0x81C,      0x823C0203, 0x81C,      0x813E0203,
+	0x81C,      0x80400203, 0x81C,      0x64420203, 0x81C,      0x63440203,
+	0x81C,      0x62460203, 0x81C,      0x61480203, 0x81C,      0x604A0203,
+	0x81C,      0x414C0203, 0x81C,      0x404E0203, 0x81C,      0x22500203,
+	0x81C,      0x21520203, 0x81C,      0x20540203, 0x81C,      0x03560203,
+	0x81C,      0x02580203, 0x81C,      0x015A0203, 0x81C,      0x005C0203,
+	0x81C,      0x005E0203, 0x81C,      0x00600203, 0x81C,      0x00620203,
+	0x81C,      0x00640203, 0x81C,      0x00660203, 0x81C,      0x00680203,
+	0x81C,      0x006A0203, 0x81C,      0x006C0203, 0x81C,      0x006E0203,
+	0x81C,      0x00700203, 0x81C,      0x00720203, 0x81C,      0x00740203,
+	0x81C,      0x00760203, 0x81C,      0x00780203, 0x81C,      0x007A0203,
+	0x81C,      0x007C0203, 0x81C,      0x007E0203, 0x90012100, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xFB000203, 0x81C,      0xFA020203,
+	0x81C,      0xF9040203, 0x81C,      0xF8060203, 0x81C,      0xF7080203,
+	0x81C,      0xF60A0203, 0x81C,      0xF50C0203, 0x81C,      0xF40E0203,
+	0x81C,      0xF3100203, 0x81C,      0xF2120203, 0x81C,      0xF1140203,
+	0x81C,      0xF0160203, 0x81C,      0xEF180203, 0x81C,      0xEE1A0203,
+	0x81C,      0xED1C0203, 0x81C,      0xEC1E0203, 0x81C,      0xEB200203,
+	0x81C,      0xEA220203, 0x81C,      0xE9240203, 0x81C,      0xE8260203,
+	0x81C,      0xE7280203, 0x81C,      0xE62A0203, 0x81C,      0xE52C0203,
+	0x81C,      0xE42E0203, 0x81C,      0xE3300203, 0x81C,      0xE2320203,
+	0x81C,      0xC6340203, 0x81C,      0xC5360203, 0x81C,      0xC4380203,
+	0x81C,      0xC33A0203, 0x81C,      0xC23C0203, 0x81C,      0xC13E0203,
+	0x81C,      0xC0400203, 0x81C,      0xA3420203, 0x81C,      0xA2440203,
+	0x81C,      0xA1460203, 0x81C,      0xA0480203, 0x81C,      0x824A0203,
+	0x81C,      0x814C0203, 0x81C,      0x804E0203, 0x81C,      0x63500203,
+	0x81C,      0x62520203, 0x81C,      0x61540203, 0x81C,      0x60560203,
+	0x81C,      0x24580203, 0x81C,      0x235A0203, 0x81C,      0x225C0203,
+	0x81C,      0x215E0203, 0x81C,      0x20600203, 0x81C,      0x03620203,
+	0x81C,      0x02640203, 0x81C,      0x01660203, 0x81C,      0x00680203,
+	0x81C,      0x006A0203, 0x81C,      0x006C0203, 0x81C,      0x006E0203,
+	0x81C,      0x00700203, 0x81C,      0x00720203, 0x81C,      0x00740203,
+	0x81C,      0x00760203, 0x81C,      0x00780203, 0x81C,      0x007A0203,
+	0x81C,      0x007C0203, 0x81C,      0x007E0203, 0x90001004, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xF8000203, 0x81C,      0xF7020203,
+	0x81C,      0xF6040203, 0x81C,      0xF5060203, 0x81C,      0xF4080203,
+	0x81C,      0xF30A0203, 0x81C,      0xF20C0203, 0x81C,      0xF10E0203,
+	0x81C,      0xF0100203, 0x81C,      0xEF120203, 0x81C,      0xEE140203,
+	0x81C,      0xED160203, 0x81C,      0xEC180203, 0x81C,      0xEB1A0203,
+	0x81C,      0xEA1C0203, 0x81C,      0xE91E0203, 0x81C,      0xE8200203,
+	0x81C,      0xE7220203, 0x81C,      0xE6240203, 0x81C,      0xE5260203,
+	0x81C,      0xE4280203, 0x81C,      0xE32A0203, 0x81C,      0xC42C0203,
+	0x81C,      0xC32E0203, 0x81C,      0xC2300203, 0x81C,      0xC1320203,
+	0x81C,      0xA3340203, 0x81C,      0xA2360203, 0x81C,      0xA1380203,
+	0x81C,      0xA03A0203, 0x81C,      0x823C0203, 0x81C,      0x813E0203,
+	0x81C,      0x80400203, 0x81C,      0x65420203, 0x81C,      0x64440203,
+	0x81C,      0x63460203, 0x81C,      0x62480203, 0x81C,      0x614A0203,
+	0x81C,      0x424C0203, 0x81C,      0x414E0203, 0x81C,      0x40500203,
+	0x81C,      0x22520203, 0x81C,      0x21540203, 0x81C,      0x20560203,
+	0x81C,      0x04580203, 0x81C,      0x035A0203, 0x81C,      0x025C0203,
+	0x81C,      0x015E0203, 0x81C,      0x00600203, 0x81C,      0x00620203,
+	0x81C,      0x00640203, 0x81C,      0x00660203, 0x81C,      0x00680203,
+	0x81C,      0x006A0203, 0x81C,      0x006C0203, 0x81C,      0x006E0203,
+	0x81C,      0x00700203, 0x81C,      0x00720203, 0x81C,      0x00740203,
+	0x81C,      0x00760203, 0x81C,      0x00780203, 0x81C,      0x007A0203,
+	0x81C,      0x007C0203, 0x81C,      0x007E0203, 0x90011000, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xFC000203, 0x81C,      0xFB020203,
+	0x81C,      0xFA040203, 0x81C,      0xF9060203, 0x81C,      0xF8080203,
+	0x81C,      0xF70A0203, 0x81C,      0xF60C0203, 0x81C,      0xF50E0203,
+	0x81C,      0xF4100203, 0x81C,      0xF3120203, 0x81C,      0xF2140203,
+	0x81C,      0xF1160203, 0x81C,      0xF0180203, 0x81C,      0xEE1A0203,
+	0x81C,      0xED1C0203, 0x81C,      0xEC1E0203, 0x81C,      0xEB200203,
+	0x81C,      0xEA220203, 0x81C,      0xE9240203, 0x81C,      0xE8260203,
+	0x81C,      0xE7280203, 0x81C,      0xE62A0203, 0x81C,      0xE52C0203,
+	0x81C,      0xE42E0203, 0x81C,      0xE3300203, 0x81C,      0xE2320203,
+	0x81C,      0xC6340203, 0x81C,      0xC5360203, 0x81C,      0xC4380203,
+	0x81C,      0xC33A0203, 0x81C,      0xA63C0203, 0x81C,      0xA53E0203,
+	0x81C,      0xA4400203, 0x81C,      0xA3420203, 0x81C,      0xA2440203,
+	0x81C,      0xA1460203, 0x81C,      0x83480203, 0x81C,      0x824A0203,
+	0x81C,      0x814C0203, 0x81C,      0x804E0203, 0x81C,      0x63500203,
+	0x81C,      0x62520203, 0x81C,      0x61540203, 0x81C,      0x42560203,
+	0x81C,      0x41580203, 0x81C,      0x405A0203, 0x81C,      0x225C0203,
+	0x81C,      0x215E0203, 0x81C,      0x20600203, 0x81C,      0x04620203,
+	0x81C,      0x03640203, 0x81C,      0x02660203, 0x81C,      0x01680203,
+	0x81C,      0x006A0203, 0x81C,      0x006C0203, 0x81C,      0x006E0203,
+	0x81C,      0x00700203, 0x81C,      0x00720203, 0x81C,      0x00740203,
+	0x81C,      0x00760203, 0x81C,      0x00780203, 0x81C,      0x007A0203,
+	0x81C,      0x007C0203, 0x81C,      0x007E0203, 0x90002100, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xFC000203, 0x81C,      0xFB020203,
+	0x81C,      0xFA040203, 0x81C,      0xF9060203, 0x81C,      0xF8080203,
+	0x81C,      0xF70A0203, 0x81C,      0xF60C0203, 0x81C,      0xF50E0203,
+	0x81C,      0xF4100203, 0x81C,      0xF3120203, 0x81C,      0xF2140203,
+	0x81C,      0xF1160203, 0x81C,      0xF0180203, 0x81C,      0xEF1A0203,
+	0x81C,      0xEE1C0203, 0x81C,      0xED1E0203, 0x81C,      0xEC200203,
+	0x81C,      0xEB220203, 0x81C,      0xEA240203, 0x81C,      0xE9260203,
+	0x81C,      0xE8280203, 0x81C,      0xE72A0203, 0x81C,      0xE62C0203,
+	0x81C,      0xE52E0203, 0x81C,      0xE4300203, 0x81C,      0xE3320203,
+	0x81C,      0xE2340203, 0x81C,      0xE1360203, 0x81C,      0xC5380203,
+	0x81C,      0xC43A0203, 0x81C,      0xC33C0203, 0x81C,      0xC23E0203,
+	0x81C,      0xC1400203, 0x81C,      0xA3420203, 0x81C,      0xA2440203,
+	0x81C,      0xA1460203, 0x81C,      0xA0480203, 0x81C,      0x834A0203,
+	0x81C,      0x824C0203, 0x81C,      0x814E0203, 0x81C,      0x64500203,
+	0x81C,      0x63520203, 0x81C,      0x62540203, 0x81C,      0x61560203,
+	0x81C,      0x25580203, 0x81C,      0x245A0203, 0x81C,      0x235C0203,
+	0x81C,      0x225E0203, 0x81C,      0x21600203, 0x81C,      0x04620203,
+	0x81C,      0x03640203, 0x81C,      0x02660203, 0x81C,      0x01680203,
+	0x81C,      0x006A0203, 0x81C,      0x006C0203, 0x81C,      0x006E0203,
+	0x81C,      0x00700203, 0x81C,      0x00720203, 0x81C,      0x00740203,
+	0x81C,      0x00760203, 0x81C,      0x00780203, 0x81C,      0x007A0203,
+	0x81C,      0x007C0203, 0x81C,      0x007E0203, 0x90002000, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xFC000203, 0x81C,      0xFB020203,
+	0x81C,      0xFA040203, 0x81C,      0xF9060203, 0x81C,      0xF8080203,
+	0x81C,      0xF70A0203, 0x81C,      0xF60C0203, 0x81C,      0xF50E0203,
+	0x81C,      0xF4100203, 0x81C,      0xF3120203, 0x81C,      0xF2140203,
+	0x81C,      0xF1160203, 0x81C,      0xF0180203, 0x81C,      0xEF1A0203,
+	0x81C,      0xEE1C0203, 0x81C,      0xED1E0203, 0x81C,      0xEC200203,
+	0x81C,      0xEB220203, 0x81C,      0xEA240203, 0x81C,      0xE9260203,
+	0x81C,      0xE8280203, 0x81C,      0xE72A0203, 0x81C,      0xE62C0203,
+	0x81C,      0xE52E0203, 0x81C,      0xE4300203, 0x81C,      0xE3320203,
+	0x81C,      0xE2340203, 0x81C,      0xC6360203, 0x81C,      0xC5380203,
+	0x81C,      0xC43A0203, 0x81C,      0xC33C0203, 0x81C,      0xA63E0203,
+	0x81C,      0xA5400203, 0x81C,      0xA4420203, 0x81C,      0xA3440203,
+	0x81C,      0xA2460203, 0x81C,      0xA1480203, 0x81C,      0x834A0203,
+	0x81C,      0x824C0203, 0x81C,      0x814E0203, 0x81C,      0x64500203,
+	0x81C,      0x63520203, 0x81C,      0x62540203, 0x81C,      0x61560203,
+	0x81C,      0x60580203, 0x81C,      0x405A0203, 0x81C,      0x215C0203,
+	0x81C,      0x205E0203, 0x81C,      0x03600203, 0x81C,      0x02620203,
+	0x81C,      0x01640203, 0x81C,      0x00660203, 0x81C,      0x00680203,
+	0x81C,      0x006A0203, 0x81C,      0x006C0203, 0x81C,      0x006E0203,
+	0x81C,      0x00700203, 0x81C,      0x00720203, 0x81C,      0x00740203,
+	0x81C,      0x00760203, 0x81C,      0x00780203, 0x81C,      0x007A0203,
+	0x81C,      0x007C0203, 0x81C,      0x007E0203, 0xA0000000, 0x00000000,
+	0x81C,      0xFD000203, 0x81C,      0xFC020203, 0x81C,      0xFB040203,
+	0x81C,      0xFA060203, 0x81C,      0xF9080203, 0x81C,      0xF80A0203,
+	0x81C,      0xF70C0203, 0x81C,      0xF60E0203, 0x81C,      0xF5100203,
+	0x81C,      0xF4120203, 0x81C,      0xF3140203, 0x81C,      0xF2160203,
+	0x81C,      0xF1180203, 0x81C,      0xF01A0203, 0x81C,      0xEF1C0203,
+	0x81C,      0xEE1E0203, 0x81C,      0xED200203, 0x81C,      0xEC220203,
+	0x81C,      0xEB240203, 0x81C,      0xEA260203, 0x81C,      0xE9280203,
+	0x81C,      0xE82A0203, 0x81C,      0xE72C0203, 0x81C,      0xE62E0203,
+	0x81C,      0xE5300203, 0x81C,      0xE4320203, 0x81C,      0xE3340203,
+	0x81C,      0xC6360203, 0x81C,      0xC5380203, 0x81C,      0xC43A0203,
+	0x81C,      0xC33C0203, 0x81C,      0xA63E0203, 0x81C,      0xA5400203,
+	0x81C,      0xA4420203, 0x81C,      0xA3440203, 0x81C,      0xA2460203,
+	0x81C,      0xA1480203, 0x81C,      0x834A0203, 0x81C,      0x824C0203,
+	0x81C,      0x814E0203, 0x81C,      0x64500203, 0x81C,      0x63520203,
+	0x81C,      0x62540203, 0x81C,      0x61560203, 0x81C,      0x60580203,
+	0x81C,      0x235A0203, 0x81C,      0x225C0203, 0x81C,      0x215E0203,
+	0x81C,      0x20600203, 0x81C,      0x03620203, 0x81C,      0x02640203,
+	0x81C,      0x01660203, 0x81C,      0x00680203, 0x81C,      0x006A0203,
+	0x81C,      0x006C0203, 0x81C,      0x006E0203, 0x81C,      0x00700203,
+	0x81C,      0x00720203, 0x81C,      0x00740203, 0x81C,      0x00760203,
+	0x81C,      0x00780203, 0x81C,      0x007A0203, 0x81C,      0x007C0203,
+	0x81C,      0x007E0203, 0xB0000000, 0x00000000, 0x8000100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x81C,      0xF8000303, 0x81C,      0xF7020303,
+	0x81C,      0xF6040303, 0x81C,      0xF5060303, 0x81C,      0xF4080303,
+	0x81C,      0xF30A0303, 0x81C,      0xF20C0303, 0x81C,      0xF10E0303,
+	0x81C,      0xF0100303, 0x81C,      0xEF120303, 0x81C,      0xEE140303,
+	0x81C,      0xED160303, 0x81C,      0xEC180303, 0x81C,      0xEB1A0303,
+	0x81C,      0xEA1C0303, 0x81C,      0xE91E0303, 0x81C,      0xCA200303,
+	0x81C,      0xC9220303, 0x81C,      0xC8240303, 0x81C,      0xC7260303,
+	0x81C,      0xC6280303, 0x81C,      0xC52A0303, 0x81C,      0xC42C0303,
+	0x81C,      0xC32E0303, 0x81C,      0xC2300303, 0x81C,      0xC1320303,
+	0x81C,      0xA3340303, 0x81C,      0xA2360303, 0x81C,      0xA1380303,
+	0x81C,      0xA03A0303, 0x81C,      0x823C0303, 0x81C,      0x813E0303,
+	0x81C,      0x80400303, 0x81C,      0x65420303, 0x81C,      0x64440303,
+	0x81C,      0x63460303, 0x81C,      0x62480303, 0x81C,      0x614A0303,
+	0x81C,      0x424C0303, 0x81C,      0x414E0303, 0x81C,      0x40500303,
+	0x81C,      0x22520303, 0x81C,      0x21540303, 0x81C,      0x20560303,
+	0x81C,      0x04580303, 0x81C,      0x035A0303, 0x81C,      0x025C0303,
+	0x81C,      0x015E0303, 0x81C,      0x00600303, 0x81C,      0x00620303,
+	0x81C,      0x00640303, 0x81C,      0x00660303, 0x81C,      0x00680303,
+	0x81C,      0x006A0303, 0x81C,      0x006C0303, 0x81C,      0x006E0303,
+	0x81C,      0x00700303, 0x81C,      0x00720303, 0x81C,      0x00740303,
+	0x81C,      0x00760303, 0x81C,      0x00780303, 0x81C,      0x007A0303,
+	0x81C,      0x007C0303, 0x81C,      0x007E0303, 0x9000100f, 0x05050505,
+	0x40000000, 0x00000000, 0x81C,      0xF9000303, 0x81C,      0xF8020303,
+	0x81C,      0xF7040303, 0x81C,      0xF6060303, 0x81C,      0xF5080303,
+	0x81C,      0xF40A0303, 0x81C,      0xF30C0303, 0x81C,      0xF20E0303,
+	0x81C,      0xF1100303, 0x81C,      0xF0120303, 0x81C,      0xEF140303,
+	0x81C,      0xEE160303, 0x81C,      0xED180303, 0x81C,      0xEC1A0303,
+	0x81C,      0xEB1C0303, 0x81C,      0xEA1E0303, 0x81C,      0xC9200303,
+	0x81C,      0xC8220303, 0x81C,      0xC7240303, 0x81C,      0xC6260303,
+	0x81C,      0xC5280303, 0x81C,      0xC42A0303, 0x81C,      0xC32C0303,
+	0x81C,      0xC22E0303, 0x81C,      0xC1300303, 0x81C,      0xC0320303,
+	0x81C,      0xA3340303, 0x81C,      0xA2360303, 0x81C,      0xA1380303,
+	0x81C,      0xA03A0303, 0x81C,      0x823C0303, 0x81C,      0x813E0303,
+	0x81C,      0x80400303, 0x81C,      0x64420303, 0x81C,      0x63440303,
+	0x81C,      0x62460303, 0x81C,      0x61480303, 0x81C,      0x604A0303,
+	0x81C,      0x414C0303, 0x81C,      0x404E0303, 0x81C,      0x22500303,
+	0x81C,      0x21520303, 0x81C,      0x20540303, 0x81C,      0x03560303,
+	0x81C,      0x02580303, 0x81C,      0x015A0303, 0x81C,      0x005C0303,
+	0x81C,      0x005E0303, 0x81C,      0x00600303, 0x81C,      0x00620303,
+	0x81C,      0x00640303, 0x81C,      0x00660303, 0x81C,      0x00680303,
+	0x81C,      0x006A0303, 0x81C,      0x006C0303, 0x81C,      0x006E0303,
+	0x81C,      0x00700303, 0x81C,      0x00720303, 0x81C,      0x00740303,
+	0x81C,      0x00760303, 0x81C,      0x00780303, 0x81C,      0x007A0303,
+	0x81C,      0x007C0303, 0x81C,      0x007E0303, 0x9000100f, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xF7000303, 0x81C,      0xF6020303,
+	0x81C,      0xF5040303, 0x81C,      0xF4060303, 0x81C,      0xF3080303,
+	0x81C,      0xF20A0303, 0x81C,      0xF10C0303, 0x81C,      0xF00E0303,
+	0x81C,      0xEF100303, 0x81C,      0xEE120303, 0x81C,      0xED140303,
+	0x81C,      0xEC160303, 0x81C,      0xEB180303, 0x81C,      0xEA1A0303,
+	0x81C,      0xE91C0303, 0x81C,      0xCA1E0303, 0x81C,      0xC9200303,
+	0x81C,      0xC8220303, 0x81C,      0xC7240303, 0x81C,      0xC6260303,
+	0x81C,      0xC5280303, 0x81C,      0xC42A0303, 0x81C,      0xC32C0303,
+	0x81C,      0xC22E0303, 0x81C,      0xC1300303, 0x81C,      0xA4320303,
+	0x81C,      0xA3340303, 0x81C,      0xA2360303, 0x81C,      0xA1380303,
+	0x81C,      0xA03A0303, 0x81C,      0x823C0303, 0x81C,      0x813E0303,
+	0x81C,      0x80400303, 0x81C,      0x64420303, 0x81C,      0x63440303,
+	0x81C,      0x62460303, 0x81C,      0x61480303, 0x81C,      0x604A0303,
+	0x81C,      0x414C0303, 0x81C,      0x404E0303, 0x81C,      0x06500303,
+	0x81C,      0x05520303, 0x81C,      0x04540303, 0x81C,      0x03560303,
+	0x81C,      0x02580303, 0x81C,      0x015A0303, 0x81C,      0x005C0303,
+	0x81C,      0x005E0303, 0x81C,      0x00600303, 0x81C,      0x00620303,
+	0x81C,      0x00640303, 0x81C,      0x00660303, 0x81C,      0x00680303,
+	0x81C,      0x006A0303, 0x81C,      0x006C0303, 0x81C,      0x006E0303,
+	0x81C,      0x00700303, 0x81C,      0x00720303, 0x81C,      0x00740303,
+	0x81C,      0x00760303, 0x81C,      0x00780303, 0x81C,      0x007A0303,
+	0x81C,      0x007C0303, 0x81C,      0x007E0303, 0x9000200f, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xF7000303, 0x81C,      0xF6020303,
+	0x81C,      0xF5040303, 0x81C,      0xF4060303, 0x81C,      0xF3080303,
+	0x81C,      0xF20A0303, 0x81C,      0xF10C0303, 0x81C,      0xF00E0303,
+	0x81C,      0xEF100303, 0x81C,      0xEE120303, 0x81C,      0xED140303,
+	0x81C,      0xEC160303, 0x81C,      0xEB180303, 0x81C,      0xEA1A0303,
+	0x81C,      0xE91C0303, 0x81C,      0xCA1E0303, 0x81C,      0xC9200303,
+	0x81C,      0xC8220303, 0x81C,      0xC7240303, 0x81C,      0xC6260303,
+	0x81C,      0xC5280303, 0x81C,      0xC42A0303, 0x81C,      0xC32C0303,
+	0x81C,      0xC22E0303, 0x81C,      0xC1300303, 0x81C,      0xA4320303,
+	0x81C,      0xA3340303, 0x81C,      0xA2360303, 0x81C,      0xA1380303,
+	0x81C,      0xA03A0303, 0x81C,      0x823C0303, 0x81C,      0x813E0303,
+	0x81C,      0x80400303, 0x81C,      0x64420303, 0x81C,      0x63440303,
+	0x81C,      0x62460303, 0x81C,      0x61480303, 0x81C,      0x604A0303,
+	0x81C,      0x414C0303, 0x81C,      0x404E0303, 0x81C,      0x22500303,
+	0x81C,      0x21520303, 0x81C,      0x20540303, 0x81C,      0x03560303,
+	0x81C,      0x02580303, 0x81C,      0x015A0303, 0x81C,      0x005C0303,
+	0x81C,      0x005E0303, 0x81C,      0x00600303, 0x81C,      0x00620303,
+	0x81C,      0x00640303, 0x81C,      0x00660303, 0x81C,      0x00680303,
+	0x81C,      0x006A0303, 0x81C,      0x006C0303, 0x81C,      0x006E0303,
+	0x81C,      0x00700303, 0x81C,      0x00720303, 0x81C,      0x00740303,
+	0x81C,      0x00760303, 0x81C,      0x00780303, 0x81C,      0x007A0303,
+	0x81C,      0x007C0303, 0x81C,      0x007E0303, 0x9000200c, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xF7000303, 0x81C,      0xF6020303,
+	0x81C,      0xF5040303, 0x81C,      0xF4060303, 0x81C,      0xF3080303,
+	0x81C,      0xF20A0303, 0x81C,      0xF10C0303, 0x81C,      0xF00E0303,
+	0x81C,      0xEF100303, 0x81C,      0xEE120303, 0x81C,      0xED140303,
+	0x81C,      0xEC160303, 0x81C,      0xEB180303, 0x81C,      0xEA1A0303,
+	0x81C,      0xE91C0303, 0x81C,      0xCA1E0303, 0x81C,      0xC9200303,
+	0x81C,      0xC8220303, 0x81C,      0xC7240303, 0x81C,      0xC6260303,
+	0x81C,      0xC5280303, 0x81C,      0xC42A0303, 0x81C,      0xC32C0303,
+	0x81C,      0xC22E0303, 0x81C,      0xC1300303, 0x81C,      0xA4320303,
+	0x81C,      0xA3340303, 0x81C,      0xA2360303, 0x81C,      0xA1380303,
+	0x81C,      0xA03A0303, 0x81C,      0x823C0303, 0x81C,      0x813E0303,
+	0x81C,      0x80400303, 0x81C,      0x64420303, 0x81C,      0x63440303,
+	0x81C,      0x62460303, 0x81C,      0x61480303, 0x81C,      0x604A0303,
+	0x81C,      0x414C0303, 0x81C,      0x404E0303, 0x81C,      0x22500303,
+	0x81C,      0x21520303, 0x81C,      0x20540303, 0x81C,      0x03560303,
+	0x81C,      0x02580303, 0x81C,      0x015A0303, 0x81C,      0x005C0303,
+	0x81C,      0x005E0303, 0x81C,      0x00600303, 0x81C,      0x00620303,
+	0x81C,      0x00640303, 0x81C,      0x00660303, 0x81C,      0x00680303,
+	0x81C,      0x006A0303, 0x81C,      0x006C0303, 0x81C,      0x006E0303,
+	0x81C,      0x00700303, 0x81C,      0x00720303, 0x81C,      0x00740303,
+	0x81C,      0x00760303, 0x81C,      0x00780303, 0x81C,      0x007A0303,
+	0x81C,      0x007C0303, 0x81C,      0x007E0303, 0x90012100, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xFB000303, 0x81C,      0xFA020303,
+	0x81C,      0xF9040303, 0x81C,      0xF8060303, 0x81C,      0xF7080303,
+	0x81C,      0xF60A0303, 0x81C,      0xF50C0303, 0x81C,      0xF40E0303,
+	0x81C,      0xF3100303, 0x81C,      0xF2120303, 0x81C,      0xF1140303,
+	0x81C,      0xF0160303, 0x81C,      0xEF180303, 0x81C,      0xEE1A0303,
+	0x81C,      0xED1C0303, 0x81C,      0xEC1E0303, 0x81C,      0xEB200303,
+	0x81C,      0xEA220303, 0x81C,      0xE9240303, 0x81C,      0xE8260303,
+	0x81C,      0xE7280303, 0x81C,      0xE62A0303, 0x81C,      0xE52C0303,
+	0x81C,      0xE42E0303, 0x81C,      0xE3300303, 0x81C,      0xE2320303,
+	0x81C,      0xC6340303, 0x81C,      0xC5360303, 0x81C,      0xC4380303,
+	0x81C,      0xC33A0303, 0x81C,      0xC23C0303, 0x81C,      0xC13E0303,
+	0x81C,      0xA4400303, 0x81C,      0xA3420303, 0x81C,      0xA2440303,
+	0x81C,      0xA1460303, 0x81C,      0x83480303, 0x81C,      0x824A0303,
+	0x81C,      0x814C0303, 0x81C,      0x804E0303, 0x81C,      0x63500303,
+	0x81C,      0x62520303, 0x81C,      0x43540303, 0x81C,      0x42560303,
+	0x81C,      0x41580303, 0x81C,      0x235A0303, 0x81C,      0x225C0303,
+	0x81C,      0x215E0303, 0x81C,      0x20600303, 0x81C,      0x04620303,
+	0x81C,      0x03640303, 0x81C,      0x02660303, 0x81C,      0x01680303,
+	0x81C,      0x006A0303, 0x81C,      0x006C0303, 0x81C,      0x006E0303,
+	0x81C,      0x00700303, 0x81C,      0x00720303, 0x81C,      0x00740303,
+	0x81C,      0x00760303, 0x81C,      0x00780303, 0x81C,      0x007A0303,
+	0x81C,      0x007C0303, 0x81C,      0x007E0303, 0x90001004, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xF8000303, 0x81C,      0xF7020303,
+	0x81C,      0xF6040303, 0x81C,      0xF5060303, 0x81C,      0xF4080303,
+	0x81C,      0xF30A0303, 0x81C,      0xF20C0303, 0x81C,      0xF10E0303,
+	0x81C,      0xF0100303, 0x81C,      0xEF120303, 0x81C,      0xEE140303,
+	0x81C,      0xED160303, 0x81C,      0xEC180303, 0x81C,      0xEB1A0303,
+	0x81C,      0xEA1C0303, 0x81C,      0xE91E0303, 0x81C,      0xCA200303,
+	0x81C,      0xC9220303, 0x81C,      0xC8240303, 0x81C,      0xC7260303,
+	0x81C,      0xC6280303, 0x81C,      0xC52A0303, 0x81C,      0xC42C0303,
+	0x81C,      0xC32E0303, 0x81C,      0xC2300303, 0x81C,      0xC1320303,
+	0x81C,      0xA3340303, 0x81C,      0xA2360303, 0x81C,      0xA1380303,
+	0x81C,      0xA03A0303, 0x81C,      0x823C0303, 0x81C,      0x813E0303,
+	0x81C,      0x80400303, 0x81C,      0x65420303, 0x81C,      0x64440303,
+	0x81C,      0x63460303, 0x81C,      0x62480303, 0x81C,      0x614A0303,
+	0x81C,      0x424C0303, 0x81C,      0x414E0303, 0x81C,      0x40500303,
+	0x81C,      0x22520303, 0x81C,      0x21540303, 0x81C,      0x20560303,
+	0x81C,      0x04580303, 0x81C,      0x035A0303, 0x81C,      0x025C0303,
+	0x81C,      0x015E0303, 0x81C,      0x00600303, 0x81C,      0x00620303,
+	0x81C,      0x00640303, 0x81C,      0x00660303, 0x81C,      0x00680303,
+	0x81C,      0x006A0303, 0x81C,      0x006C0303, 0x81C,      0x006E0303,
+	0x81C,      0x00700303, 0x81C,      0x00720303, 0x81C,      0x00740303,
+	0x81C,      0x00760303, 0x81C,      0x00780303, 0x81C,      0x007A0303,
+	0x81C,      0x007C0303, 0x81C,      0x007E0303, 0x90011000, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xFB000303, 0x81C,      0xFA020303,
+	0x81C,      0xF9040303, 0x81C,      0xF8060303, 0x81C,      0xF7080303,
+	0x81C,      0xF60A0303, 0x81C,      0xF50C0303, 0x81C,      0xF40E0303,
+	0x81C,      0xF3100303, 0x81C,      0xF2120303, 0x81C,      0xF1140303,
+	0x81C,      0xF0160303, 0x81C,      0xEE180303, 0x81C,      0xED1A0303,
+	0x81C,      0xEC1C0303, 0x81C,      0xEB1E0303, 0x81C,      0xEA200303,
+	0x81C,      0xE9220303, 0x81C,      0xE8240303, 0x81C,      0xE7260303,
+	0x81C,      0xE6280303, 0x81C,      0xE52A0303, 0x81C,      0xE42C0303,
+	0x81C,      0xE32E0303, 0x81C,      0xE2300303, 0x81C,      0xE1320303,
+	0x81C,      0xC6340303, 0x81C,      0xC5360303, 0x81C,      0xC4380303,
+	0x81C,      0xC33A0303, 0x81C,      0xA63C0303, 0x81C,      0xA53E0303,
+	0x81C,      0xA4400303, 0x81C,      0xA3420303, 0x81C,      0xA2440303,
+	0x81C,      0xA1460303, 0x81C,      0x83480303, 0x81C,      0x824A0303,
+	0x81C,      0x814C0303, 0x81C,      0x804E0303, 0x81C,      0x63500303,
+	0x81C,      0x62520303, 0x81C,      0x61540303, 0x81C,      0x42560303,
+	0x81C,      0x41580303, 0x81C,      0x405A0303, 0x81C,      0x225C0303,
+	0x81C,      0x215E0303, 0x81C,      0x20600303, 0x81C,      0x04620303,
+	0x81C,      0x03640303, 0x81C,      0x02660303, 0x81C,      0x01680303,
+	0x81C,      0x006A0303, 0x81C,      0x006C0303, 0x81C,      0x006E0303,
+	0x81C,      0x00700303, 0x81C,      0x00720303, 0x81C,      0x00740303,
+	0x81C,      0x00760303, 0x81C,      0x00780303, 0x81C,      0x007A0303,
+	0x81C,      0x007C0303, 0x81C,      0x007E0303, 0x90002100, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xFB000303, 0x81C,      0xFA020303,
+	0x81C,      0xF9040303, 0x81C,      0xF8060303, 0x81C,      0xF7080303,
+	0x81C,      0xF60A0303, 0x81C,      0xF50C0303, 0x81C,      0xF40E0303,
+	0x81C,      0xF3100303, 0x81C,      0xF2120303, 0x81C,      0xF1140303,
+	0x81C,      0xF0160303, 0x81C,      0xEF180303, 0x81C,      0xEE1A0303,
+	0x81C,      0xED1C0303, 0x81C,      0xEC1E0303, 0x81C,      0xEB200303,
+	0x81C,      0xEA220303, 0x81C,      0xE9240303, 0x81C,      0xE8260303,
+	0x81C,      0xE7280303, 0x81C,      0xE62A0303, 0x81C,      0xE52C0303,
+	0x81C,      0xE42E0303, 0x81C,      0xE3300303, 0x81C,      0xE2320303,
+	0x81C,      0xE1340303, 0x81C,      0xC5360303, 0x81C,      0xC4380303,
+	0x81C,      0xC33A0303, 0x81C,      0xC23C0303, 0x81C,      0xC13E0303,
+	0x81C,      0xA4400303, 0x81C,      0xA3420303, 0x81C,      0xA2440303,
+	0x81C,      0xA1460303, 0x81C,      0x83480303, 0x81C,      0x824A0303,
+	0x81C,      0x814C0303, 0x81C,      0x804E0303, 0x81C,      0x64500303,
+	0x81C,      0x63520303, 0x81C,      0x62540303, 0x81C,      0x61560303,
+	0x81C,      0x60580303, 0x81C,      0x235A0303, 0x81C,      0x225C0303,
+	0x81C,      0x215E0303, 0x81C,      0x20600303, 0x81C,      0x04620303,
+	0x81C,      0x03640303, 0x81C,      0x02660303, 0x81C,      0x01680303,
+	0x81C,      0x006A0303, 0x81C,      0x006C0303, 0x81C,      0x006E0303,
+	0x81C,      0x00700303, 0x81C,      0x00720303, 0x81C,      0x00740303,
+	0x81C,      0x00760303, 0x81C,      0x00780303, 0x81C,      0x007A0303,
+	0x81C,      0x007C0303, 0x81C,      0x007E0303, 0x90002000, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xFC000303, 0x81C,      0xFB020303,
+	0x81C,      0xFA040303, 0x81C,      0xF9060303, 0x81C,      0xF8080303,
+	0x81C,      0xF70A0303, 0x81C,      0xF60C0303, 0x81C,      0xF50E0303,
+	0x81C,      0xF4100303, 0x81C,      0xF3120303, 0x81C,      0xF2140303,
+	0x81C,      0xF1160303, 0x81C,      0xF0180303, 0x81C,      0xEF1A0303,
+	0x81C,      0xEE1C0303, 0x81C,      0xED1E0303, 0x81C,      0xEC200303,
+	0x81C,      0xEB220303, 0x81C,      0xEA240303, 0x81C,      0xE9260303,
+	0x81C,      0xE8280303, 0x81C,      0xE72A0303, 0x81C,      0xE62C0303,
+	0x81C,      0xE52E0303, 0x81C,      0xE4300303, 0x81C,      0xE3320303,
+	0x81C,      0xE2340303, 0x81C,      0xC6360303, 0x81C,      0xC5380303,
+	0x81C,      0xC43A0303, 0x81C,      0xC33C0303, 0x81C,      0xA63E0303,
+	0x81C,      0xA5400303, 0x81C,      0xA4420303, 0x81C,      0xA3440303,
+	0x81C,      0xA2460303, 0x81C,      0x84480303, 0x81C,      0x834A0303,
+	0x81C,      0x824C0303, 0x81C,      0x814E0303, 0x81C,      0x80500303,
+	0x81C,      0x63520303, 0x81C,      0x62540303, 0x81C,      0x61560303,
+	0x81C,      0x60580303, 0x81C,      0x225A0303, 0x81C,      0x055C0303,
+	0x81C,      0x045E0303, 0x81C,      0x03600303, 0x81C,      0x02620303,
+	0x81C,      0x01640303, 0x81C,      0x00660303, 0x81C,      0x00680303,
+	0x81C,      0x006A0303, 0x81C,      0x006C0303, 0x81C,      0x006E0303,
+	0x81C,      0x00700303, 0x81C,      0x00720303, 0x81C,      0x00740303,
+	0x81C,      0x00760303, 0x81C,      0x00780303, 0x81C,      0x007A0303,
+	0x81C,      0x007C0303, 0x81C,      0x007E0303, 0xA0000000, 0x00000000,
+	0x81C,      0xFC000303, 0x81C,      0xFB020303, 0x81C,      0xFA040303,
+	0x81C,      0xF9060303, 0x81C,      0xF8080303, 0x81C,      0xF70A0303,
+	0x81C,      0xF60C0303, 0x81C,      0xF50E0303, 0x81C,      0xF4100303,
+	0x81C,      0xF3120303, 0x81C,      0xF2140303, 0x81C,      0xF1160303,
+	0x81C,      0xF0180303, 0x81C,      0xEF1A0303, 0x81C,      0xEE1C0303,
+	0x81C,      0xED1E0303, 0x81C,      0xEC200303, 0x81C,      0xEB220303,
+	0x81C,      0xEA240303, 0x81C,      0xE9260303, 0x81C,      0xE8280303,
+	0x81C,      0xE72A0303, 0x81C,      0xE62C0303, 0x81C,      0xE52E0303,
+	0x81C,      0xE4300303, 0x81C,      0xE3320303, 0x81C,      0xE2340303,
+	0x81C,      0xC6360303, 0x81C,      0xC5380303, 0x81C,      0xC43A0303,
+	0x81C,      0xC33C0303, 0x81C,      0xA63E0303, 0x81C,      0xA5400303,
+	0x81C,      0xA4420303, 0x81C,      0xA3440303, 0x81C,      0xA2460303,
+	0x81C,      0x84480303, 0x81C,      0x834A0303, 0x81C,      0x824C0303,
+	0x81C,      0x814E0303, 0x81C,      0x80500303, 0x81C,      0x63520303,
+	0x81C,      0x62540303, 0x81C,      0x61560303, 0x81C,      0x60580303,
+	0x81C,      0x235A0303, 0x81C,      0x225C0303, 0x81C,      0x215E0303,
+	0x81C,      0x20600303, 0x81C,      0x03620303, 0x81C,      0x02640303,
+	0x81C,      0x01660303, 0x81C,      0x00680303, 0x81C,      0x006A0303,
+	0x81C,      0x006C0303, 0x81C,      0x006E0303, 0x81C,      0x00700303,
+	0x81C,      0x00720303, 0x81C,      0x00740303, 0x81C,      0x00760303,
+	0x81C,      0x00780303, 0x81C,      0x007A0303, 0x81C,      0x007C0303,
+	0x81C,      0x007E0303, 0xB0000000, 0x00000000, 0x8000100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x81C,      0xFF000403, 0x81C,      0xF5000403,
+	0x81C,      0xF4020403, 0x81C,      0xF3040403, 0x81C,      0xF2060403,
+	0x81C,      0xF1080403, 0x81C,      0xF00A0403, 0x81C,      0xEF0C0403,
+	0x81C,      0xEE0E0403, 0x81C,      0xED100403, 0x81C,      0xEC120403,
+	0x81C,      0xEB140403, 0x81C,      0xEA160403, 0x81C,      0xE9180403,
+	0x81C,      0xE81A0403, 0x81C,      0xE71C0403, 0x81C,      0xE61E0403,
+	0x81C,      0xE5200403, 0x81C,      0xE4220403, 0x81C,      0xE3240403,
+	0x81C,      0xE2260403, 0x81C,      0xE1280403, 0x81C,      0xE02A0403,
+	0x81C,      0xC32C0403, 0x81C,      0xC22E0403, 0x81C,      0xC1300403,
+	0x81C,      0xC0320403, 0x81C,      0xA4340403, 0x81C,      0xA3360403,
+	0x81C,      0xA2380403, 0x81C,      0xA13A0403, 0x81C,      0xA03C0403,
+	0x81C,      0x823E0403, 0x81C,      0x81400403, 0x81C,      0x80420403,
+	0x81C,      0x64440403, 0x81C,      0x63460403, 0x81C,      0x62480403,
+	0x81C,      0x614A0403, 0x81C,      0x604C0403, 0x81C,      0x454E0403,
+	0x81C,      0x44500403, 0x81C,      0x43520403, 0x81C,      0x42540403,
+	0x81C,      0x41560403, 0x81C,      0x40580403, 0x81C,      0x055A0403,
+	0x81C,      0x045C0403, 0x81C,      0x035E0403, 0x81C,      0x02600403,
+	0x81C,      0x01620403, 0x81C,      0x00640403, 0x81C,      0x00660403,
+	0x81C,      0x00680403, 0x81C,      0x006A0403, 0x81C,      0x006C0403,
+	0x81C,      0x006E0403, 0x81C,      0x00700403, 0x81C,      0x00720403,
+	0x81C,      0x00740403, 0x81C,      0x00760403, 0x81C,      0x00780403,
+	0x81C,      0x007A0403, 0x81C,      0x007C0403, 0x81C,      0x007E0403,
+	0x9000100f, 0x05050505, 0x40000000, 0x00000000, 0x81C,      0xFF000403,
+	0x81C,      0xF5000403, 0x81C,      0xF4020403, 0x81C,      0xF3040403,
+	0x81C,      0xF2060403, 0x81C,      0xF1080403, 0x81C,      0xF00A0403,
+	0x81C,      0xEF0C0403, 0x81C,      0xEE0E0403, 0x81C,      0xED100403,
+	0x81C,      0xEC120403, 0x81C,      0xEB140403, 0x81C,      0xEA160403,
+	0x81C,      0xE9180403, 0x81C,      0xE81A0403, 0x81C,      0xE71C0403,
+	0x81C,      0xE61E0403, 0x81C,      0xE5200403, 0x81C,      0xE4220403,
+	0x81C,      0xE3240403, 0x81C,      0xE2260403, 0x81C,      0xE1280403,
+	0x81C,      0xE02A0403, 0x81C,      0xC32C0403, 0x81C,      0xC22E0403,
+	0x81C,      0xC1300403, 0x81C,      0xC0320403, 0x81C,      0xA4340403,
+	0x81C,      0xA3360403, 0x81C,      0xA2380403, 0x81C,      0xA13A0403,
+	0x81C,      0xA03C0403, 0x81C,      0x823E0403, 0x81C,      0x81400403,
+	0x81C,      0x80420403, 0x81C,      0x64440403, 0x81C,      0x63460403,
+	0x81C,      0x62480403, 0x81C,      0x614A0403, 0x81C,      0x604C0403,
+	0x81C,      0x454E0403, 0x81C,      0x44500403, 0x81C,      0x43520403,
+	0x81C,      0x42540403, 0x81C,      0x41560403, 0x81C,      0x40580403,
+	0x81C,      0x055A0403, 0x81C,      0x045C0403, 0x81C,      0x035E0403,
+	0x81C,      0x02600403, 0x81C,      0x01620403, 0x81C,      0x00640403,
+	0x81C,      0x00660403, 0x81C,      0x00680403, 0x81C,      0x006A0403,
+	0x81C,      0x006C0403, 0x81C,      0x006E0403, 0x81C,      0x00700403,
+	0x81C,      0x00720403, 0x81C,      0x00740403, 0x81C,      0x00760403,
+	0x81C,      0x00780403, 0x81C,      0x007A0403, 0x81C,      0x007C0403,
+	0x81C,      0x007E0403, 0x9000100f, 0x00000000, 0x40000000, 0x00000000,
+	0x81C,      0xFF000403, 0x81C,      0xF5000403, 0x81C,      0xF4020403,
+	0x81C,      0xF3040403, 0x81C,      0xF2060403, 0x81C,      0xF1080403,
+	0x81C,      0xF00A0403, 0x81C,      0xEF0C0403, 0x81C,      0xEE0E0403,
+	0x81C,      0xED100403, 0x81C,      0xEC120403, 0x81C,      0xEB140403,
+	0x81C,      0xEA160403, 0x81C,      0xE9180403, 0x81C,      0xE81A0403,
+	0x81C,      0xE71C0403, 0x81C,      0xE61E0403, 0x81C,      0xE5200403,
+	0x81C,      0xE4220403, 0x81C,      0xE3240403, 0x81C,      0xE2260403,
+	0x81C,      0xE1280403, 0x81C,      0xE02A0403, 0x81C,      0xC32C0403,
+	0x81C,      0xC22E0403, 0x81C,      0xC1300403, 0x81C,      0xC0320403,
+	0x81C,      0xA4340403, 0x81C,      0xA3360403, 0x81C,      0xA2380403,
+	0x81C,      0xA13A0403, 0x81C,      0xA03C0403, 0x81C,      0x823E0403,
+	0x81C,      0x81400403, 0x81C,      0x80420403, 0x81C,      0x64440403,
+	0x81C,      0x63460403, 0x81C,      0x62480403, 0x81C,      0x614A0403,
+	0x81C,      0x604C0403, 0x81C,      0x454E0403, 0x81C,      0x44500403,
+	0x81C,      0x43520403, 0x81C,      0x42540403, 0x81C,      0x41560403,
+	0x81C,      0x40580403, 0x81C,      0x055A0403, 0x81C,      0x045C0403,
+	0x81C,      0x035E0403, 0x81C,      0x02600403, 0x81C,      0x01620403,
+	0x81C,      0x00640403, 0x81C,      0x00660403, 0x81C,      0x00680403,
+	0x81C,      0x006A0403, 0x81C,      0x006C0403, 0x81C,      0x006E0403,
+	0x81C,      0x00700403, 0x81C,      0x00720403, 0x81C,      0x00740403,
+	0x81C,      0x00760403, 0x81C,      0x00780403, 0x81C,      0x007A0403,
+	0x81C,      0x007C0403, 0x81C,      0x007E0403, 0x9000200f, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xFF000403, 0x81C,      0xF5000403,
+	0x81C,      0xF4020403, 0x81C,      0xF3040403, 0x81C,      0xF2060403,
+	0x81C,      0xF1080403, 0x81C,      0xF00A0403, 0x81C,      0xEF0C0403,
+	0x81C,      0xEE0E0403, 0x81C,      0xED100403, 0x81C,      0xEC120403,
+	0x81C,      0xEB140403, 0x81C,      0xEA160403, 0x81C,      0xE9180403,
+	0x81C,      0xE81A0403, 0x81C,      0xE71C0403, 0x81C,      0xE61E0403,
+	0x81C,      0xE5200403, 0x81C,      0xE4220403, 0x81C,      0xE3240403,
+	0x81C,      0xE2260403, 0x81C,      0xE1280403, 0x81C,      0xE02A0403,
+	0x81C,      0xC32C0403, 0x81C,      0xC22E0403, 0x81C,      0xC1300403,
+	0x81C,      0xC0320403, 0x81C,      0xA4340403, 0x81C,      0xA3360403,
+	0x81C,      0xA2380403, 0x81C,      0xA13A0403, 0x81C,      0xA03C0403,
+	0x81C,      0x823E0403, 0x81C,      0x81400403, 0x81C,      0x80420403,
+	0x81C,      0x64440403, 0x81C,      0x63460403, 0x81C,      0x62480403,
+	0x81C,      0x614A0403, 0x81C,      0x604C0403, 0x81C,      0x454E0403,
+	0x81C,      0x44500403, 0x81C,      0x43520403, 0x81C,      0x42540403,
+	0x81C,      0x41560403, 0x81C,      0x40580403, 0x81C,      0x055A0403,
+	0x81C,      0x045C0403, 0x81C,      0x035E0403, 0x81C,      0x02600403,
+	0x81C,      0x01620403, 0x81C,      0x00640403, 0x81C,      0x00660403,
+	0x81C,      0x00680403, 0x81C,      0x006A0403, 0x81C,      0x006C0403,
+	0x81C,      0x006E0403, 0x81C,      0x00700403, 0x81C,      0x00720403,
+	0x81C,      0x00740403, 0x81C,      0x00760403, 0x81C,      0x00780403,
+	0x81C,      0x007A0403, 0x81C,      0x007C0403, 0x81C,      0x007E0403,
+	0x9000200c, 0x00000000, 0x40000000, 0x00000000, 0x81C,      0xFF000403,
+	0x81C,      0xFF000403, 0x81C,      0xFF020403, 0x81C,      0xFE040403,
+	0x81C,      0xFD060403, 0x81C,      0xFC080403, 0x81C,      0xFB0A0403,
+	0x81C,      0xFA0C0403, 0x81C,      0xF90E0403, 0x81C,      0xF8100403,
+	0x81C,      0xF7120403, 0x81C,      0xF6140403, 0x81C,      0xF5160403,
+	0x81C,      0xF4180403, 0x81C,      0xF31A0403, 0x81C,      0xF21C0403,
+	0x81C,      0xD51E0403, 0x81C,      0xD4200403, 0x81C,      0xD3220403,
+	0x81C,      0xD2240403, 0x81C,      0xB6260403, 0x81C,      0xB5280403,
+	0x81C,      0xB42A0403, 0x81C,      0xB32C0403, 0x81C,      0xB22E0403,
+	0x81C,      0xB1300403, 0x81C,      0xB0320403, 0x81C,      0xAF340403,
+	0x81C,      0xAE360403, 0x81C,      0xAD380403, 0x81C,      0xAC3A0403,
+	0x81C,      0xAB3C0403, 0x81C,      0xAA3E0403, 0x81C,      0xA9400403,
+	0x81C,      0xA8420403, 0x81C,      0xA7440403, 0x81C,      0xA6460403,
+	0x81C,      0xA5480403, 0x81C,      0xA44A0403, 0x81C,      0xA34C0403,
+	0x81C,      0x854E0403, 0x81C,      0x84500403, 0x81C,      0x83520403,
+	0x81C,      0x82540403, 0x81C,      0x81560403, 0x81C,      0x80580403,
+	0x81C,      0x485A0403, 0x81C,      0x475C0403, 0x81C,      0x465E0403,
+	0x81C,      0x45600403, 0x81C,      0x44620403, 0x81C,      0x0A640403,
+	0x81C,      0x09660403, 0x81C,      0x08680403, 0x81C,      0x076A0403,
+	0x81C,      0x066C0403, 0x81C,      0x056E0403, 0x81C,      0x04700403,
+	0x81C,      0x03720403, 0x81C,      0x02740403, 0x81C,      0x01760403,
+	0x81C,      0x00780403, 0x81C,      0x007A0403, 0x81C,      0x007C0403,
+	0x81C,      0x007E0403, 0x90012100, 0x00000000, 0x40000000, 0x00000000,
+	0x81C,      0xFF000403, 0x81C,      0xFF000403, 0x81C,      0xFF020403,
+	0x81C,      0xFE040403, 0x81C,      0xFD060403, 0x81C,      0xFC080403,
+	0x81C,      0xFB0A0403, 0x81C,      0xFA0C0403, 0x81C,      0xF90E0403,
+	0x81C,      0xF8100403, 0x81C,      0xF7120403, 0x81C,      0xF6140403,
+	0x81C,      0xF5160403, 0x81C,      0xF4180403, 0x81C,      0xF31A0403,
+	0x81C,      0xF21C0403, 0x81C,      0xD51E0403, 0x81C,      0xD4200403,
+	0x81C,      0xD3220403, 0x81C,      0xD2240403, 0x81C,      0xB6260403,
+	0x81C,      0xB5280403, 0x81C,      0xB42A0403, 0x81C,      0xB32C0403,
+	0x81C,      0xB22E0403, 0x81C,      0xB1300403, 0x81C,      0xB0320403,
+	0x81C,      0xAF340403, 0x81C,      0xAE360403, 0x81C,      0xAD380403,
+	0x81C,      0xAC3A0403, 0x81C,      0xAB3C0403, 0x81C,      0xAA3E0403,
+	0x81C,      0xA9400403, 0x81C,      0xA8420403, 0x81C,      0xA7440403,
+	0x81C,      0xA6460403, 0x81C,      0xA5480403, 0x81C,      0xA44A0403,
+	0x81C,      0xA34C0403, 0x81C,      0x854E0403, 0x81C,      0x84500403,
+	0x81C,      0x83520403, 0x81C,      0x82540403, 0x81C,      0x81560403,
+	0x81C,      0x80580403, 0x81C,      0x485A0403, 0x81C,      0x475C0403,
+	0x81C,      0x465E0403, 0x81C,      0x45600403, 0x81C,      0x44620403,
+	0x81C,      0x0A640403, 0x81C,      0x09660403, 0x81C,      0x08680403,
+	0x81C,      0x076A0403, 0x81C,      0x066C0403, 0x81C,      0x056E0403,
+	0x81C,      0x04700403, 0x81C,      0x03720403, 0x81C,      0x02740403,
+	0x81C,      0x01760403, 0x81C,      0x00780403, 0x81C,      0x007A0403,
+	0x81C,      0x007C0403, 0x81C,      0x007E0403, 0x90001004, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xFF000403, 0x81C,      0xF5000403,
+	0x81C,      0xF4020403, 0x81C,      0xF3040403, 0x81C,      0xF2060403,
+	0x81C,      0xF1080403, 0x81C,      0xF00A0403, 0x81C,      0xEF0C0403,
+	0x81C,      0xEE0E0403, 0x81C,      0xED100403, 0x81C,      0xEC120403,
+	0x81C,      0xEB140403, 0x81C,      0xEA160403, 0x81C,      0xE9180403,
+	0x81C,      0xE81A0403, 0x81C,      0xE71C0403, 0x81C,      0xE61E0403,
+	0x81C,      0xE5200403, 0x81C,      0xE4220403, 0x81C,      0xE3240403,
+	0x81C,      0xE2260403, 0x81C,      0xE1280403, 0x81C,      0xE02A0403,
+	0x81C,      0xC32C0403, 0x81C,      0xC22E0403, 0x81C,      0xC1300403,
+	0x81C,      0xC0320403, 0x81C,      0xA4340403, 0x81C,      0xA3360403,
+	0x81C,      0xA2380403, 0x81C,      0xA13A0403, 0x81C,      0xA03C0403,
+	0x81C,      0x823E0403, 0x81C,      0x81400403, 0x81C,      0x80420403,
+	0x81C,      0x64440403, 0x81C,      0x63460403, 0x81C,      0x62480403,
+	0x81C,      0x614A0403, 0x81C,      0x604C0403, 0x81C,      0x454E0403,
+	0x81C,      0x44500403, 0x81C,      0x43520403, 0x81C,      0x42540403,
+	0x81C,      0x41560403, 0x81C,      0x40580403, 0x81C,      0x055A0403,
+	0x81C,      0x045C0403, 0x81C,      0x035E0403, 0x81C,      0x02600403,
+	0x81C,      0x01620403, 0x81C,      0x00640403, 0x81C,      0x00660403,
+	0x81C,      0x00680403, 0x81C,      0x006A0403, 0x81C,      0x006C0403,
+	0x81C,      0x006E0403, 0x81C,      0x00700403, 0x81C,      0x00720403,
+	0x81C,      0x00740403, 0x81C,      0x00760403, 0x81C,      0x00780403,
+	0x81C,      0x007A0403, 0x81C,      0x007C0403, 0x81C,      0x007E0403,
+	0x90011000, 0x00000000, 0x40000000, 0x00000000, 0x81C,      0xFF000403,
+	0x81C,      0xFF000403, 0x81C,      0xFF020403, 0x81C,      0xFE040403,
+	0x81C,      0xFD060403, 0x81C,      0xFC080403, 0x81C,      0xFB0A0403,
+	0x81C,      0xFA0C0403, 0x81C,      0xF90E0403, 0x81C,      0xF8100403,
+	0x81C,      0xF7120403, 0x81C,      0xF6140403, 0x81C,      0xF5160403,
+	0x81C,      0xF4180403, 0x81C,      0xF31A0403, 0x81C,      0xF21C0403,
+	0x81C,      0xD51E0403, 0x81C,      0xD4200403, 0x81C,      0xD3220403,
+	0x81C,      0xD2240403, 0x81C,      0xB6260403, 0x81C,      0xB5280403,
+	0x81C,      0xB42A0403, 0x81C,      0xB32C0403, 0x81C,      0xB22E0403,
+	0x81C,      0xB1300403, 0x81C,      0xB0320403, 0x81C,      0xAF340403,
+	0x81C,      0xAE360403, 0x81C,      0xAD380403, 0x81C,      0xAC3A0403,
+	0x81C,      0xAB3C0403, 0x81C,      0xAA3E0403, 0x81C,      0xA9400403,
+	0x81C,      0xA8420403, 0x81C,      0xA7440403, 0x81C,      0xA6460403,
+	0x81C,      0xA5480403, 0x81C,      0xA44A0403, 0x81C,      0xA34C0403,
+	0x81C,      0x854E0403, 0x81C,      0x84500403, 0x81C,      0x83520403,
+	0x81C,      0x82540403, 0x81C,      0x81560403, 0x81C,      0x80580403,
+	0x81C,      0x485A0403, 0x81C,      0x475C0403, 0x81C,      0x465E0403,
+	0x81C,      0x45600403, 0x81C,      0x44620403, 0x81C,      0x0A640403,
+	0x81C,      0x09660403, 0x81C,      0x08680403, 0x81C,      0x076A0403,
+	0x81C,      0x066C0403, 0x81C,      0x056E0403, 0x81C,      0x04700403,
+	0x81C,      0x03720403, 0x81C,      0x02740403, 0x81C,      0x01760403,
+	0x81C,      0x00780403, 0x81C,      0x007A0403, 0x81C,      0x007C0403,
+	0x81C,      0x007E0403, 0x90002100, 0x00000000, 0x40000000, 0x00000000,
+	0x81C,      0xFF000403, 0x81C,      0xFF000403, 0x81C,      0xFF020403,
+	0x81C,      0xFE040403, 0x81C,      0xFD060403, 0x81C,      0xFC080403,
+	0x81C,      0xFB0A0403, 0x81C,      0xFA0C0403, 0x81C,      0xF90E0403,
+	0x81C,      0xF8100403, 0x81C,      0xF7120403, 0x81C,      0xF6140403,
+	0x81C,      0xF5160403, 0x81C,      0xF4180403, 0x81C,      0xF31A0403,
+	0x81C,      0xF21C0403, 0x81C,      0xD51E0403, 0x81C,      0xD4200403,
+	0x81C,      0xD3220403, 0x81C,      0xD2240403, 0x81C,      0xB6260403,
+	0x81C,      0xB5280403, 0x81C,      0xB42A0403, 0x81C,      0xB32C0403,
+	0x81C,      0xB22E0403, 0x81C,      0xB1300403, 0x81C,      0xB0320403,
+	0x81C,      0xAF340403, 0x81C,      0xAE360403, 0x81C,      0xAD380403,
+	0x81C,      0xAC3A0403, 0x81C,      0xAB3C0403, 0x81C,      0xAA3E0403,
+	0x81C,      0xA9400403, 0x81C,      0xA8420403, 0x81C,      0xA7440403,
+	0x81C,      0xA6460403, 0x81C,      0xA5480403, 0x81C,      0xA44A0403,
+	0x81C,      0xA34C0403, 0x81C,      0x854E0403, 0x81C,      0x84500403,
+	0x81C,      0x83520403, 0x81C,      0x82540403, 0x81C,      0x81560403,
+	0x81C,      0x80580403, 0x81C,      0x485A0403, 0x81C,      0x475C0403,
+	0x81C,      0x465E0403, 0x81C,      0x45600403, 0x81C,      0x44620403,
+	0x81C,      0x0A640403, 0x81C,      0x09660403, 0x81C,      0x08680403,
+	0x81C,      0x076A0403, 0x81C,      0x066C0403, 0x81C,      0x056E0403,
+	0x81C,      0x04700403, 0x81C,      0x03720403, 0x81C,      0x02740403,
+	0x81C,      0x01760403, 0x81C,      0x00780403, 0x81C,      0x007A0403,
+	0x81C,      0x007C0403, 0x81C,      0x007E0403, 0x90002000, 0x00000000,
+	0x40000000, 0x00000000, 0x81C,      0xFF000403, 0x81C,      0xFF000403,
+	0x81C,      0xFF020403, 0x81C,      0xFE040403, 0x81C,      0xFD060403,
+	0x81C,      0xFC080403, 0x81C,      0xFB0A0403, 0x81C,      0xFA0C0403,
+	0x81C,      0xF90E0403, 0x81C,      0xF8100403, 0x81C,      0xF7120403,
+	0x81C,      0xF6140403, 0x81C,      0xF5160403, 0x81C,      0xF4180403,
+	0x81C,      0xF31A0403, 0x81C,      0xF21C0403, 0x81C,      0xD51E0403,
+	0x81C,      0xD4200403, 0x81C,      0xD3220403, 0x81C,      0xD2240403,
+	0x81C,      0xB6260403, 0x81C,      0xB5280403, 0x81C,      0xB42A0403,
+	0x81C,      0xB32C0403, 0x81C,      0xB22E0403, 0x81C,      0xB1300403,
+	0x81C,      0xB0320403, 0x81C,      0xAF340403, 0x81C,      0xAE360403,
+	0x81C,      0xAD380403, 0x81C,      0xAC3A0403, 0x81C,      0xAB3C0403,
+	0x81C,      0xAA3E0403, 0x81C,      0xA9400403, 0x81C,      0xA8420403,
+	0x81C,      0xA7440403, 0x81C,      0xA6460403, 0x81C,      0xA5480403,
+	0x81C,      0xA44A0403, 0x81C,      0xA34C0403, 0x81C,      0x854E0403,
+	0x81C,      0x84500403, 0x81C,      0x83520403, 0x81C,      0x82540403,
+	0x81C,      0x81560403, 0x81C,      0x80580403, 0x81C,      0x485A0403,
+	0x81C,      0x475C0403, 0x81C,      0x465E0403, 0x81C,      0x45600403,
+	0x81C,      0x44620403, 0x81C,      0x0A640403, 0x81C,      0x09660403,
+	0x81C,      0x08680403, 0x81C,      0x076A0403, 0x81C,      0x066C0403,
+	0x81C,      0x056E0403, 0x81C,      0x04700403, 0x81C,      0x03720403,
+	0x81C,      0x02740403, 0x81C,      0x01760403, 0x81C,      0x00780403,
+	0x81C,      0x007A0403, 0x81C,      0x007C0403, 0x81C,      0x007E0403,
+	0xA0000000, 0x00000000, 0x81C,      0xFF000403, 0x81C,      0xFF000403,
+	0x81C,      0xFF020403, 0x81C,      0xFE040403, 0x81C,      0xFD060403,
+	0x81C,      0xFC080403, 0x81C,      0xFB0A0403, 0x81C,      0xFA0C0403,
+	0x81C,      0xF90E0403, 0x81C,      0xF8100403, 0x81C,      0xF7120403,
+	0x81C,      0xF6140403, 0x81C,      0xF5160403, 0x81C,      0xF4180403,
+	0x81C,      0xF31A0403, 0x81C,      0xF21C0403, 0x81C,      0xD51E0403,
+	0x81C,      0xD4200403, 0x81C,      0xD3220403, 0x81C,      0xD2240403,
+	0x81C,      0xB6260403, 0x81C,      0xB5280403, 0x81C,      0xB42A0403,
+	0x81C,      0xB32C0403, 0x81C,      0xB22E0403, 0x81C,      0xB1300403,
+	0x81C,      0xB0320403, 0x81C,      0xAF340403, 0x81C,      0xAE360403,
+	0x81C,      0xAD380403, 0x81C,      0xAC3A0403, 0x81C,      0xAB3C0403,
+	0x81C,      0xAA3E0403, 0x81C,      0xA9400403, 0x81C,      0xA8420403,
+	0x81C,      0xA7440403, 0x81C,      0xA6460403, 0x81C,      0xA5480403,
+	0x81C,      0xA44A0403, 0x81C,      0xA34C0403, 0x81C,      0x854E0403,
+	0x81C,      0x84500403, 0x81C,      0x83520403, 0x81C,      0x82540403,
+	0x81C,      0x81560403, 0x81C,      0x80580403, 0x81C,      0x485A0403,
+	0x81C,      0x475C0403, 0x81C,      0x465E0403, 0x81C,      0x45600403,
+	0x81C,      0x44620403, 0x81C,      0x0A640403, 0x81C,      0x09660403,
+	0x81C,      0x08680403, 0x81C,      0x076A0403, 0x81C,      0x066C0403,
+	0x81C,      0x056E0403, 0x81C,      0x04700403, 0x81C,      0x03720403,
+	0x81C,      0x02740403, 0x81C,      0x01760403, 0x81C,      0x00780403,
+	0x81C,      0x007A0403, 0x81C,      0x007C0403, 0x81C,      0x007E0403,
+	0xB0000000, 0x00000000, 0xC50,      0x00000022, 0xC50,      0x00000020,
+	0xE50,      0x00000022, 0xE50,      0x00000020,
+
+};
+
+void odm_read_and_config_mp_8822b_agc_tab(struct phy_dm_struct *dm)
+{
+	u32 i = 0;
+	u8 c_cond;
+	bool is_matched = true, is_skipped = false;
+	u32 array_len = sizeof(array_mp_8822b_agc_tab) / sizeof(u32);
+	u32 *array = array_mp_8822b_agc_tab;
+
+	u32 v1 = 0, v2 = 0, pre_v1 = 0, pre_v2 = 0;
+
+	ODM_RT_TRACE(dm, ODM_COMP_INIT,
+		     "===> %s\n", __func__);
+
+	for (; (i + 1) < array_len; i = i + 2) {
+		v1 = array[i];
+		v2 = array[i + 1];
+
+		if (v1 & BIT(31)) { /* positive condition*/
+			c_cond = (u8)((v1 & (BIT(29) | BIT(28))) >> 28);
+			if (c_cond == COND_ENDIF) { /*end*/
+				is_matched = true;
+				is_skipped = false;
+				ODM_RT_TRACE(dm, ODM_COMP_INIT, "ENDIF\n");
+			} else if (c_cond == COND_ELSE) { /*else*/
+				is_matched = is_skipped ? false : true;
+				ODM_RT_TRACE(dm, ODM_COMP_INIT, "ELSE\n");
+			} else { /*if , else if*/
+				pre_v1 = v1;
+				pre_v2 = v2;
+				ODM_RT_TRACE(dm, ODM_COMP_INIT,
+					     "IF or ELSE IF\n");
+			}
+		} else if (v1 & BIT(30)) { /*negative condition*/
+			if (is_skipped) {
+				is_matched = false;
+				continue;
+			}
+
+			if (check_positive(dm, pre_v1, pre_v2, v1, v2)) {
+				is_matched = true;
+				is_skipped = true;
+			} else {
+				is_matched = false;
+				is_skipped = false;
+			}
+		} else if (is_matched) {
+			odm_config_bb_agc_8822b(dm, v1, MASKDWORD, v2);
+		}
+	}
+}
+
+u32 odm_get_version_mp_8822b_agc_tab(void) { return 67; }
+
+/******************************************************************************
+ *                           phy_reg.TXT
+ ******************************************************************************/
+
+static u32 array_mp_8822b_phy_reg[] = {
+	0x800,  0x9020D010, 0x804,  0x800181A0, 0x808,  0x0E028233,
+	0x80C,  0x10000013, 0x810,  0x21101263, 0x814,  0x020C3D10,
+	0x818,  0x84A10385, 0x81C,  0x1E1E081F, 0x820,  0x0001AAAA,
+	0x824,  0x00030FE0, 0x828,  0x0000CCCC, 0x82C,  0x75CB7010,
+	0x830,  0x79A0EA2A, 0x834,  0x072E6986, 0x838,  0x87766441,
+	0x83C,  0x9194B2B6, 0x840,  0x171740E0, 0x844,  0x4D3D7CDB,
+	0x848,  0x4AD0408B, 0x84C,  0x6AFBF7A5, 0x850,  0x28A74706,
+	0x854,  0x0001520C, 0x858,  0x4060C000, 0x85C,  0x74010160,
+	0x860,  0x68A7C321, 0x864,  0x79F27432, 0x868,  0x8CA7A314,
+	0x86C,  0x778C2878, 0x870,  0x77777777, 0x874,  0x27612C2E,
+	0x878,  0xC0003152, 0x87C,  0x5C8FC000, 0x880,  0x00000000,
+	0x884,  0x00000000, 0x888,  0x00000000, 0x88C,  0x00000000,
+	0x890,  0x00000000, 0x894,  0x00000000, 0x898,  0x00000000,
+	0x89C,  0x00000000, 0x8A0,  0x00000013, 0x8A4,  0x7F7F7F7F,
+	0x8A8,  0x2202033E, 0x8AC,  0xF00F000A, 0x8B0,  0x00000600,
+	0x8B4,  0x000FC080, 0x8B8,  0xEC0057F7, 0x8BC,  0xACB520A3,
+	0x8C0,  0xFFE04020, 0x8C4,  0x47C00000, 0x8C8,  0x000251A5,
+	0x8CC,  0x08108000, 0x8D0,  0x0000B800, 0x8D4,  0x860308A0,
+	0x8D8,  0x21095612, 0x8DC,  0x00000000, 0x8E0,  0x32D16777,
+	0x8E4,  0x4C098935, 0x8E8,  0xFFFFC42C, 0x8EC,  0x99999999,
+	0x8F0,  0x00009999, 0x8F4,  0x00D80FA1, 0x8F8,  0x40000080,
+	0x8FC,  0x00000130, 0x900,  0x00800000, 0x904,  0x00000000,
+	0x908,  0x00000000, 0x90C,  0xD3000000, 0x910,  0x0000FC00,
+	0x914,  0xC6380000, 0x918,  0x1C1028C0, 0x91C,  0x64B11A1C,
+	0x920,  0xE0767233, 0x924,  0x855A2500, 0x928,  0x4AB0E4E4,
+	0x92C,  0xFFFEB200, 0x930,  0xFFFFFFFE, 0x934,  0x001FFFFF,
+	0x938,  0x00008480, 0x93C,  0xE41C0642, 0x940,  0x0E470430,
+	0x944,  0x00000000, 0x948,  0xAC000000, 0x94C,  0x10000083,
+	0x950,  0x32010080, 0x954,  0x84510080, 0x958,  0x00000001,
+	0x95C,  0x04248000, 0x960,  0x00000000, 0x964,  0x00000000,
+	0x968,  0x00000000, 0x96C,  0x00000000, 0x970,  0x00001FFF,
+	0x974,  0x44000FFF, 0x978,  0x00000000, 0x97C,  0x00000000,
+	0x980,  0x00000000, 0x984,  0x00000000, 0x988,  0x00000000,
+	0x98C,  0x23440000, 0x990,  0x27100000, 0x994,  0xFFFF0100,
+	0x998,  0xFFFFFF5C, 0x99C,  0xFFFFFFFF, 0x9A0,  0x000000FF,
+	0x9A4,  0x80000088, 0x9A8,  0x0C2F0000, 0x9AC,  0x01560000,
+	0x9B0,  0x70000000, 0x9B4,  0x00000000, 0x9B8,  0x00000000,
+	0x9BC,  0x00000000, 0x9C0,  0x00000000, 0x9C4,  0x00000000,
+	0x9C8,  0x00000000, 0x9CC,  0x00000000, 0x9D0,  0x00000000,
+	0x9D4,  0x00000000, 0x9D8,  0x00000000, 0x9DC,  0x00000000,
+	0x9E0,  0x00000000, 0x9E4,  0x02000402, 0x9E8,  0x000022D4,
+	0x9EC,  0x00000000, 0x9F0,  0x00010080, 0x9F4,  0x00000000,
+	0x9F8,  0x00000000, 0x9FC,  0xEFFFF7F7, 0xA00,  0x00D047C8,
+	0xA04,  0x81FF800C, 0xA08,  0x8C838300, 0xA0C,  0x2E20100F,
+	0xA10,  0x9500BB78, 0xA14,  0x1114D028, 0xA18,  0x00881117,
+	0xA1C,  0x89140F00, 0xA20,  0x84880000, 0xA24,  0x384F6577,
+	0xA28,  0x00001525, 0xA2C,  0x00920000, 0xA70,  0x101FFF00,
+	0xA74,  0x00000148, 0xA78,  0x00000900, 0xA7C,  0x225B0606,
+	0xA80,  0x218675B2, 0xA84,  0x80208C00, 0xA88,  0x040C0000,
+	0xA8C,  0x12345678, 0xA90,  0xABCDEF00, 0xA94,  0x001B1B89,
+	0xA98,  0x030A0000, 0xA9C,  0x00060000, 0xAA0,  0x00000000,
+	0xAA4,  0x0004000F, 0xAA8,  0x00000200, 0xB00,  0xE1000440,
+	0xB04,  0x00800000, 0xB08,  0xFF02030B, 0xB0C,  0x01EAA406,
+	0xB10,  0x00030690, 0xB14,  0x006000FA, 0xB18,  0x00000002,
+	0xB1C,  0x00000002, 0xB20,  0x4B00001F, 0xB24,  0x4E8E3E40,
+	0xB28,  0x03020100, 0xB2C,  0x07060504, 0xB30,  0x0B0A0908,
+	0xB34,  0x0F0E0D0C, 0xB38,  0x13121110, 0xB3C,  0x0000003A,
+	0xB40,  0x00000000, 0xB44,  0x80000000, 0xB48,  0x3F0000FA,
+	0xB4C,  0x88C80020, 0xB50,  0x00000000, 0xB54,  0x00004241,
+	0xB58,  0xE0008208, 0xB5C,  0x41EFFFF9, 0xB60,  0x00000000,
+	0xB64,  0x00200063, 0xB68,  0x0000003A, 0xB6C,  0x00000102,
+	0xB70,  0x4E6D1870, 0xB74,  0x03020100, 0xB78,  0x07060504,
+	0xB7C,  0x0B0A0908, 0xB80,  0x0F0E0D0C, 0xB84,  0x13121110,
+	0xB88,  0x00000000, 0xB8C,  0x00000000, 0xC00,  0x00000007,
+	0xC04,  0x00000020, 0xC08,  0x60403231, 0xC0C,  0x00012345,
+	0xC10,  0x00000100, 0xC14,  0x01000000, 0xC18,  0x00000000,
+	0xC1C,  0x40040053, 0xC20,  0x40020103, 0xC24,  0x00000000,
+	0xC28,  0x00000000, 0xC2C,  0x00000000, 0xC30,  0x00000000,
+	0xC34,  0x00000000, 0xC38,  0x00000000, 0xC3C,  0x00000000,
+	0xC40,  0x00000000, 0xC44,  0x00000000, 0xC48,  0x00000000,
+	0xC4C,  0x00000000, 0xC50,  0x00000020, 0xC54,  0x00000000,
+	0xC58,  0xD8020402, 0xC5C,  0xDE000120, 0xC68,  0x5979993F,
+	0xC6C,  0x0000122A, 0xC70,  0x99795979, 0xC74,  0x99795979,
+	0xC78,  0x99799979, 0xC7C,  0x99791979, 0xC80,  0x19791979,
+	0xC84,  0x19791979, 0xC88,  0x00000000, 0xC8C,  0x07000000,
+	0xC94,  0x01000100, 0xC98,  0x201C8000, 0xC9C,  0x00000000,
+	0xCA0,  0x0000A555, 0xCA4,  0x08040201, 0xCA8,  0x80402010,
+	0xCAC,  0x00000000, 0xCB0,  0x77777777, 0xCB4,  0x00007777,
+	0xCB8,  0x00000000, 0xCBC,  0x00000000, 0xCC0,  0x00000000,
+	0xCC4,  0x00000000, 0xCC8,  0x00000000, 0xCCC,  0x00000000,
+	0xCD0,  0x00000000, 0xCD4,  0x00000000, 0xCD8,  0x00000000,
+	0xCDC,  0x00000000, 0xCE0,  0x00000000, 0xCE4,  0x00000000,
+	0xCE8,  0x00000000, 0xCEC,  0x00000000, 0xE00,  0x00000007,
+	0xE04,  0x00000020, 0xE08,  0x60403231, 0xE0C,  0x00012345,
+	0xE10,  0x00000100, 0xE14,  0x01000000, 0xE18,  0x00000000,
+	0xE1C,  0x40040053, 0xE20,  0x40020103, 0xE24,  0x00000000,
+	0xE28,  0x00000000, 0xE2C,  0x00000000, 0xE30,  0x00000000,
+	0xE34,  0x00000000, 0xE38,  0x00000000, 0xE3C,  0x00000000,
+	0xE40,  0x00000000, 0xE44,  0x00000000, 0xE48,  0x00000000,
+	0xE4C,  0x00000000, 0xE50,  0x00000020, 0xE54,  0x00000000,
+	0xE58,  0xD8020402, 0xE5C,  0xDE000120, 0xE68,  0x5979993F,
+	0xE6C,  0x0000122A, 0xE70,  0x99795979, 0xE74,  0x99795979,
+	0xE78,  0x99799979, 0xE7C,  0x99791979, 0xE80,  0x19791979,
+	0xE84,  0x19791979, 0xE88,  0x00000000, 0xE8C,  0x07000000,
+	0xE94,  0x01000100, 0xE98,  0x201C8000, 0xE9C,  0x00000000,
+	0xEA0,  0x0000A555, 0xEA4,  0x08040201, 0xEA8,  0x80402010,
+	0xEAC,  0x00000000, 0xEB0,  0x77777777, 0xEB4,  0x00007777,
+	0xEB8,  0x00000000, 0xEBC,  0x00000000, 0xEC0,  0x00000000,
+	0xEC4,  0x00000000, 0xEC8,  0x00000000, 0xECC,  0x00000000,
+	0xED0,  0x00000000, 0xED4,  0x00000000, 0xED8,  0x00000000,
+	0xEDC,  0x00000000, 0xEE0,  0x00000000, 0xEE4,  0x00000000,
+	0xEE8,  0x00000000, 0xEEC,  0x00000000, 0x1900, 0x00000000,
+	0x1904, 0x00238000, 0x1908, 0x00000000, 0x190C, 0x00000000,
+	0x1910, 0x00000000, 0x1914, 0x00000000, 0x1918, 0x00000000,
+	0x191C, 0x00000000, 0x1920, 0x00000000, 0x1924, 0x00000000,
+	0x1928, 0x00000000, 0x192C, 0x00000000, 0x1930, 0x00000000,
+	0x1934, 0x00000000, 0x1938, 0x00000000, 0x193C, 0x00000000,
+	0x1940, 0x00000000, 0x1944, 0x00000000, 0x1948, 0x00000000,
+	0x194C, 0x00000000, 0x1950, 0x00000000, 0x1954, 0x00000000,
+	0x1958, 0x00000000, 0x195C, 0x00000000, 0x1960, 0x00000000,
+	0x1964, 0x00000000, 0x1968, 0x00000000, 0x196C, 0x00000000,
+	0x1970, 0x00000000, 0x1974, 0x00000000, 0x1978, 0x00000000,
+	0x197C, 0x00000000, 0x1980, 0x00000000, 0x1984, 0x03000000,
+	0x1988, 0x21401E88, 0x198C, 0x00004000, 0x1990, 0x00000000,
+	0x1994, 0x00000000, 0x1998, 0x00000053, 0x199C, 0x00000000,
+	0x19A0, 0x00000000, 0x19A4, 0x00000000, 0x19A8, 0x00000000,
+	0x19AC, 0x0E47E47F, 0x19B0, 0x00000000, 0x19B4, 0x0E47E47F,
+	0x19B8, 0x00000000, 0x19BC, 0x00000000, 0x19C0, 0x00000000,
+	0x19C4, 0x00000000, 0x19C8, 0x00000000, 0x19CC, 0x00000000,
+	0x19D0, 0x00000000, 0x19D4, 0xAAAAAAAA, 0x19D8, 0x00000AAA,
+	0x19DC, 0x133E0F37, 0x19E0, 0x00000000, 0x19E4, 0x00000000,
+	0x19E8, 0x00000000, 0x19EC, 0x00000000, 0x19F0, 0x00000000,
+	0x19F4, 0x00000000, 0x19F8, 0x01A00000, 0x19FC, 0x00000000,
+	0x1C00, 0x00000100, 0x1C04, 0x01000000, 0x1C08, 0x00000100,
+	0x1C0C, 0x01000000, 0x1C10, 0x00000100, 0x1C14, 0x01000000,
+	0x1C18, 0x00000100, 0x1C1C, 0x01000000, 0x1C20, 0x00000100,
+	0x1C24, 0x01000000, 0x1C28, 0x00000100, 0x1C2C, 0x01000000,
+	0x1C30, 0x00000100, 0x1C34, 0x01000000, 0x1C38, 0x00000000,
+	0x1C3C, 0x00000000, 0x1C40, 0x000C0100, 0x1C44, 0x000000F3,
+	0x1C48, 0x1A8249A8, 0x1C4C, 0x1461C826, 0x1C50, 0x0001469E,
+	0x1C54, 0x58D158D1, 0x1C58, 0x04490088, 0x1C5C, 0x04004400,
+	0x1C60, 0x00000000, 0x1C64, 0x04004400, 0x1C68, 0x00000100,
+	0x1C6C, 0x01000000, 0x1C70, 0x00000100, 0x1C74, 0x01000000,
+	0x1C78, 0x00000000, 0x1C7C, 0x00000010, 0x1C80, 0x5FFF5FFF,
+	0x1C84, 0x5FFF5FFF, 0x1C88, 0x5FFF5FFF, 0x1C8C, 0x5FFF5FFF,
+	0x1C90, 0x5FFF5FFF, 0x1C94, 0x5FFF5FFF, 0x1C98, 0x5FFF5FFF,
+	0x1C9C, 0x5FFF5FFF, 0x1CA0, 0x00000100, 0x1CA4, 0x01000000,
+	0x1CA8, 0x00000100, 0x1CAC, 0x5FFF5FFF, 0x1CB0, 0x00000100,
+	0x1CB4, 0x01000000, 0x1CB8, 0x00000000, 0x1CBC, 0x00000000,
+	0x1CC0, 0x00000100, 0x1CC4, 0x01000000, 0x1CC8, 0x00000100,
+	0x1CCC, 0x01000000, 0x1CD0, 0x00000100, 0x1CD4, 0x01000000,
+	0x1CD8, 0x00000100, 0x1CDC, 0x01000000, 0x1CE0, 0x00000100,
+	0x1CE4, 0x01000000, 0x1CE8, 0x00000100, 0x1CEC, 0x01000000,
+	0x1CF0, 0x00000100, 0x1CF4, 0x01000000, 0x1CF8, 0x00000000,
+	0x1CFC, 0x00000000, 0xC60,  0x70038040, 0xC60,  0x70038040,
+	0xC60,  0x70146040, 0xC60,  0x70246040, 0xC60,  0x70346040,
+	0xC60,  0x70446040, 0xC60,  0x70532040, 0xC60,  0x70646040,
+	0xC60,  0x70738040, 0xC60,  0x70838040, 0xC60,  0x70938040,
+	0xC60,  0x70A38040, 0xC60,  0x70B36040, 0xC60,  0x70C06040,
+	0xC60,  0x70D06040, 0xC60,  0x70E76040, 0xC60,  0x70F06040,
+	0xE60,  0x70038040, 0xE60,  0x70038040, 0xE60,  0x70146040,
+	0xE60,  0x70246040, 0xE60,  0x70346040, 0xE60,  0x70446040,
+	0xE60,  0x70532040, 0xE60,  0x70646040, 0xE60,  0x70738040,
+	0xE60,  0x70838040, 0xE60,  0x70938040, 0xE60,  0x70A38040,
+	0xE60,  0x70B36040, 0xE60,  0x70C06040, 0xE60,  0x70D06040,
+	0xE60,  0x70E76040, 0xE60,  0x70F06040, 0xC64,  0x00800000,
+	0xC64,  0x08800001, 0xC64,  0x00800002, 0xC64,  0x00800003,
+	0xC64,  0x00800004, 0xC64,  0x00800005, 0xC64,  0x00800006,
+	0xC64,  0x08800007, 0xC64,  0x00004000, 0xE64,  0x00800000,
+	0xE64,  0x08800001, 0xE64,  0x00800002, 0xE64,  0x00800003,
+	0xE64,  0x00800004, 0xE64,  0x00800005, 0xE64,  0x00800006,
+	0xE64,  0x08800007, 0xE64,  0x00004000, 0x1B00, 0xF8000008,
+	0x1B00, 0xF80A7008, 0x1B00, 0xF8015008, 0x1B00, 0xF8000008,
+	0x1B04, 0xE24629D2, 0x1B08, 0x00000080, 0x1B0C, 0x00000000,
+	0x1B10, 0x00010C00, 0x1B14, 0x00000000, 0x1B18, 0x00292903,
+	0x1B1C, 0xA2193C32, 0x1B20, 0x01840008, 0x1B24, 0x01860008,
+	0x1B28, 0x80060300, 0x1B2C, 0x00000003, 0x1B30, 0x20000000,
+	0x1B34, 0x00000800, 0x1B3C, 0x20000000, 0x1BC0, 0x01000000,
+	0x1BCC, 0x00000000, 0x1B00, 0xF800000A, 0x1B1C, 0xA2193C32,
+	0x1B20, 0x01840008, 0x1B24, 0x01860008, 0x1B28, 0x80060300,
+	0x1B2C, 0x00000003, 0x1B30, 0x20000000, 0x1B34, 0x00000800,
+	0x1B3C, 0x20000000, 0x1BC0, 0x01000000, 0x1BCC, 0x00000000,
+	0x1B00, 0xF8000000, 0x1B80, 0x00000007, 0x1B80, 0x090A0005,
+	0x1B80, 0x090A0007, 0x1B80, 0x0FFE0015, 0x1B80, 0x0FFE0017,
+	0x1B80, 0x00220025, 0x1B80, 0x00220027, 0x1B80, 0x00040035,
+	0x1B80, 0x00040037, 0x1B80, 0x05C00045, 0x1B80, 0x05C00047,
+	0x1B80, 0x00070055, 0x1B80, 0x00070057, 0x1B80, 0x64000065,
+	0x1B80, 0x64000067, 0x1B80, 0x00020075, 0x1B80, 0x00020077,
+	0x1B80, 0x00080085, 0x1B80, 0x00080087, 0x1B80, 0x80000095,
+	0x1B80, 0x80000097, 0x1B80, 0x090800A5, 0x1B80, 0x090800A7,
+	0x1B80, 0x0F0200B5, 0x1B80, 0x0F0200B7, 0x1B80, 0x002200C5,
+	0x1B80, 0x002200C7, 0x1B80, 0x000400D5, 0x1B80, 0x000400D7,
+	0x1B80, 0x05C000E5, 0x1B80, 0x05C000E7, 0x1B80, 0x000700F5,
+	0x1B80, 0x000700F7, 0x1B80, 0x64020105, 0x1B80, 0x64020107,
+	0x1B80, 0x00020115, 0x1B80, 0x00020117, 0x1B80, 0x00040125,
+	0x1B80, 0x00040127, 0x1B80, 0x4A000135, 0x1B80, 0x4A000137,
+	0x1B80, 0x4B040145, 0x1B80, 0x4B040147, 0x1B80, 0x85030155,
+	0x1B80, 0x85030157, 0x1B80, 0x40090165, 0x1B80, 0x40090167,
+	0x1B80, 0xE0210175, 0x1B80, 0xE0210177, 0x1B80, 0x4B050185,
+	0x1B80, 0x4B050187, 0x1B80, 0x86030195, 0x1B80, 0x86030197,
+	0x1B80, 0x400B01A5, 0x1B80, 0x400B01A7, 0x1B80, 0xE02101B5,
+	0x1B80, 0xE02101B7, 0x1B80, 0x4B0001C5, 0x1B80, 0x4B0001C7,
+	0x1B80, 0x000701D5, 0x1B80, 0x000701D7, 0x1B80, 0x4C0001E5,
+	0x1B80, 0x4C0001E7, 0x1B80, 0x000401F5, 0x1B80, 0x000401F7,
+	0x1B80, 0x30000205, 0x1B80, 0x30000207, 0x1B80, 0xFE000215,
+	0x1B80, 0xFE000217, 0x1B80, 0xFF000225, 0x1B80, 0xFF000227,
+	0x1B80, 0xE1750235, 0x1B80, 0xE1750237, 0x1B80, 0xF00D0245,
+	0x1B80, 0xF00D0247, 0x1B80, 0xF10D0255, 0x1B80, 0xF10D0257,
+	0x1B80, 0xF20D0265, 0x1B80, 0xF20D0267, 0x1B80, 0xF30D0275,
+	0x1B80, 0xF30D0277, 0x1B80, 0xF40D0285, 0x1B80, 0xF40D0287,
+	0x1B80, 0xF50D0295, 0x1B80, 0xF50D0297, 0x1B80, 0xF60D02A5,
+	0x1B80, 0xF60D02A7, 0x1B80, 0xF70D02B5, 0x1B80, 0xF70D02B7,
+	0x1B80, 0xF80D02C5, 0x1B80, 0xF80D02C7, 0x1B80, 0xF90D02D5,
+	0x1B80, 0xF90D02D7, 0x1B80, 0xFA0D02E5, 0x1B80, 0xFA0D02E7,
+	0x1B80, 0xFB0D02F5, 0x1B80, 0xFB0D02F7, 0x1B80, 0x00010305,
+	0x1B80, 0x00010307, 0x1B80, 0x303D0315, 0x1B80, 0x303D0317,
+	0x1B80, 0x30550325, 0x1B80, 0x30550327, 0x1B80, 0x30A00335,
+	0x1B80, 0x30A00337, 0x1B80, 0x30A30345, 0x1B80, 0x30A30347,
+	0x1B80, 0x30570355, 0x1B80, 0x30570357, 0x1B80, 0x30620365,
+	0x1B80, 0x30620367, 0x1B80, 0x306D0375, 0x1B80, 0x306D0377,
+	0x1B80, 0x30AD0385, 0x1B80, 0x30AD0387, 0x1B80, 0x30A70395,
+	0x1B80, 0x30A70397, 0x1B80, 0x30BB03A5, 0x1B80, 0x30BB03A7,
+	0x1B80, 0x30C603B5, 0x1B80, 0x30C603B7, 0x1B80, 0x30D103C5,
+	0x1B80, 0x30D103C7, 0x1B80, 0xE11403D5, 0x1B80, 0xE11403D7,
+	0x1B80, 0x4D0403E5, 0x1B80, 0x4D0403E7, 0x1B80, 0x208003F5,
+	0x1B80, 0x208003F7, 0x1B80, 0x00000405, 0x1B80, 0x00000407,
+	0x1B80, 0x4D000415, 0x1B80, 0x4D000417, 0x1B80, 0x55070425,
+	0x1B80, 0x55070427, 0x1B80, 0xE10C0435, 0x1B80, 0xE10C0437,
+	0x1B80, 0xE10C0445, 0x1B80, 0xE10C0447, 0x1B80, 0x4D040455,
+	0x1B80, 0x4D040457, 0x1B80, 0x20880465, 0x1B80, 0x20880467,
+	0x1B80, 0x02000475, 0x1B80, 0x02000477, 0x1B80, 0x4D000485,
+	0x1B80, 0x4D000487, 0x1B80, 0x550F0495, 0x1B80, 0x550F0497,
+	0x1B80, 0xE10C04A5, 0x1B80, 0xE10C04A7, 0x1B80, 0x4F0204B5,
+	0x1B80, 0x4F0204B7, 0x1B80, 0x4E0004C5, 0x1B80, 0x4E0004C7,
+	0x1B80, 0x530204D5, 0x1B80, 0x530204D7, 0x1B80, 0x520104E5,
+	0x1B80, 0x520104E7, 0x1B80, 0xE11004F5, 0x1B80, 0xE11004F7,
+	0x1B80, 0x4D080505, 0x1B80, 0x4D080507, 0x1B80, 0x57100515,
+	0x1B80, 0x57100517, 0x1B80, 0x57000525, 0x1B80, 0x57000527,
+	0x1B80, 0x4D000535, 0x1B80, 0x4D000537, 0x1B80, 0x00010545,
+	0x1B80, 0x00010547, 0x1B80, 0xE1140555, 0x1B80, 0xE1140557,
+	0x1B80, 0x00010565, 0x1B80, 0x00010567, 0x1B80, 0x30770575,
+	0x1B80, 0x30770577, 0x1B80, 0x00230585, 0x1B80, 0x00230587,
+	0x1B80, 0xE1680595, 0x1B80, 0xE1680597, 0x1B80, 0x000205A5,
+	0x1B80, 0x000205A7, 0x1B80, 0x54E905B5, 0x1B80, 0x54E905B7,
+	0x1B80, 0x0BA605C5, 0x1B80, 0x0BA605C7, 0x1B80, 0x002305D5,
+	0x1B80, 0x002305D7, 0x1B80, 0xE16805E5, 0x1B80, 0xE16805E7,
+	0x1B80, 0x000205F5, 0x1B80, 0x000205F7, 0x1B80, 0x4D300605,
+	0x1B80, 0x4D300607, 0x1B80, 0x30900615, 0x1B80, 0x30900617,
+	0x1B80, 0x30730625, 0x1B80, 0x30730627, 0x1B80, 0x00220635,
+	0x1B80, 0x00220637, 0x1B80, 0xE1680645, 0x1B80, 0xE1680647,
+	0x1B80, 0x00020655, 0x1B80, 0x00020657, 0x1B80, 0x54E80665,
+	0x1B80, 0x54E80667, 0x1B80, 0x0BA60675, 0x1B80, 0x0BA60677,
+	0x1B80, 0x00220685, 0x1B80, 0x00220687, 0x1B80, 0xE1680695,
+	0x1B80, 0xE1680697, 0x1B80, 0x000206A5, 0x1B80, 0x000206A7,
+	0x1B80, 0x4D3006B5, 0x1B80, 0x4D3006B7, 0x1B80, 0x309006C5,
+	0x1B80, 0x309006C7, 0x1B80, 0x63F106D5, 0x1B80, 0x63F106D7,
+	0x1B80, 0xE11406E5, 0x1B80, 0xE11406E7, 0x1B80, 0xE16806F5,
+	0x1B80, 0xE16806F7, 0x1B80, 0x63F40705, 0x1B80, 0x63F40707,
+	0x1B80, 0xE1140715, 0x1B80, 0xE1140717, 0x1B80, 0xE1680725,
+	0x1B80, 0xE1680727, 0x1B80, 0x0BA80735, 0x1B80, 0x0BA80737,
+	0x1B80, 0x63F80745, 0x1B80, 0x63F80747, 0x1B80, 0xE1140755,
+	0x1B80, 0xE1140757, 0x1B80, 0xE1680765, 0x1B80, 0xE1680767,
+	0x1B80, 0x0BA90775, 0x1B80, 0x0BA90777, 0x1B80, 0x63FC0785,
+	0x1B80, 0x63FC0787, 0x1B80, 0xE1140795, 0x1B80, 0xE1140797,
+	0x1B80, 0xE16807A5, 0x1B80, 0xE16807A7, 0x1B80, 0x63FF07B5,
+	0x1B80, 0x63FF07B7, 0x1B80, 0xE11407C5, 0x1B80, 0xE11407C7,
+	0x1B80, 0xE16807D5, 0x1B80, 0xE16807D7, 0x1B80, 0x630007E5,
+	0x1B80, 0x630007E7, 0x1B80, 0xE11407F5, 0x1B80, 0xE11407F7,
+	0x1B80, 0xE1680805, 0x1B80, 0xE1680807, 0x1B80, 0x63030815,
+	0x1B80, 0x63030817, 0x1B80, 0xE1140825, 0x1B80, 0xE1140827,
+	0x1B80, 0xE1680835, 0x1B80, 0xE1680837, 0x1B80, 0xF4D40845,
+	0x1B80, 0xF4D40847, 0x1B80, 0x63070855, 0x1B80, 0x63070857,
+	0x1B80, 0xE1140865, 0x1B80, 0xE1140867, 0x1B80, 0xE1680875,
+	0x1B80, 0xE1680877, 0x1B80, 0xF5DB0885, 0x1B80, 0xF5DB0887,
+	0x1B80, 0x630B0895, 0x1B80, 0x630B0897, 0x1B80, 0xE11408A5,
+	0x1B80, 0xE11408A7, 0x1B80, 0xE16808B5, 0x1B80, 0xE16808B7,
+	0x1B80, 0x630E08C5, 0x1B80, 0x630E08C7, 0x1B80, 0xE11408D5,
+	0x1B80, 0xE11408D7, 0x1B80, 0xE16808E5, 0x1B80, 0xE16808E7,
+	0x1B80, 0x4D3008F5, 0x1B80, 0x4D3008F7, 0x1B80, 0x55010905,
+	0x1B80, 0x55010907, 0x1B80, 0x57040915, 0x1B80, 0x57040917,
+	0x1B80, 0x57000925, 0x1B80, 0x57000927, 0x1B80, 0x96000935,
+	0x1B80, 0x96000937, 0x1B80, 0x57080945, 0x1B80, 0x57080947,
+	0x1B80, 0x57000955, 0x1B80, 0x57000957, 0x1B80, 0x95000965,
+	0x1B80, 0x95000967, 0x1B80, 0x4D000975, 0x1B80, 0x4D000977,
+	0x1B80, 0x6C070985, 0x1B80, 0x6C070987, 0x1B80, 0x7B200995,
+	0x1B80, 0x7B200997, 0x1B80, 0x7A0009A5, 0x1B80, 0x7A0009A7,
+	0x1B80, 0x790009B5, 0x1B80, 0x790009B7, 0x1B80, 0x7F2009C5,
+	0x1B80, 0x7F2009C7, 0x1B80, 0x7E0009D5, 0x1B80, 0x7E0009D7,
+	0x1B80, 0x7D0009E5, 0x1B80, 0x7D0009E7, 0x1B80, 0x000109F5,
+	0x1B80, 0x000109F7, 0x1B80, 0x62850A05, 0x1B80, 0x62850A07,
+	0x1B80, 0xE1140A15, 0x1B80, 0xE1140A17, 0x1B80, 0x00010A25,
+	0x1B80, 0x00010A27, 0x1B80, 0x5C320A35, 0x1B80, 0x5C320A37,
+	0x1B80, 0xE1640A45, 0x1B80, 0xE1640A47, 0x1B80, 0xE1420A55,
+	0x1B80, 0xE1420A57, 0x1B80, 0x00010A65, 0x1B80, 0x00010A67,
+	0x1B80, 0x5C320A75, 0x1B80, 0x5C320A77, 0x1B80, 0x63F40A85,
+	0x1B80, 0x63F40A87, 0x1B80, 0x62850A95, 0x1B80, 0x62850A97,
+	0x1B80, 0x0BB00AA5, 0x1B80, 0x0BB00AA7, 0x1B80, 0xE1140AB5,
+	0x1B80, 0xE1140AB7, 0x1B80, 0xE1680AC5, 0x1B80, 0xE1680AC7,
+	0x1B80, 0x5C320AD5, 0x1B80, 0x5C320AD7, 0x1B80, 0x63FC0AE5,
+	0x1B80, 0x63FC0AE7, 0x1B80, 0x62850AF5, 0x1B80, 0x62850AF7,
+	0x1B80, 0x0BB10B05, 0x1B80, 0x0BB10B07, 0x1B80, 0xE1140B15,
+	0x1B80, 0xE1140B17, 0x1B80, 0xE1680B25, 0x1B80, 0xE1680B27,
+	0x1B80, 0x63030B35, 0x1B80, 0x63030B37, 0x1B80, 0xE1140B45,
+	0x1B80, 0xE1140B47, 0x1B80, 0xE1680B55, 0x1B80, 0xE1680B57,
+	0x1B80, 0xF7040B65, 0x1B80, 0xF7040B67, 0x1B80, 0x630B0B75,
+	0x1B80, 0x630B0B77, 0x1B80, 0xE1140B85, 0x1B80, 0xE1140B87,
+	0x1B80, 0xE1680B95, 0x1B80, 0xE1680B97, 0x1B80, 0x00010BA5,
+	0x1B80, 0x00010BA7, 0x1B80, 0x30DF0BB5, 0x1B80, 0x30DF0BB7,
+	0x1B80, 0x00230BC5, 0x1B80, 0x00230BC7, 0x1B80, 0xE16D0BD5,
+	0x1B80, 0xE16D0BD7, 0x1B80, 0x00020BE5, 0x1B80, 0x00020BE7,
+	0x1B80, 0x54E90BF5, 0x1B80, 0x54E90BF7, 0x1B80, 0x0BA60C05,
+	0x1B80, 0x0BA60C07, 0x1B80, 0x00230C15, 0x1B80, 0x00230C17,
+	0x1B80, 0xE16D0C25, 0x1B80, 0xE16D0C27, 0x1B80, 0x00020C35,
+	0x1B80, 0x00020C37, 0x1B80, 0x4D100C45, 0x1B80, 0x4D100C47,
+	0x1B80, 0x30900C55, 0x1B80, 0x30900C57, 0x1B80, 0x30D90C65,
+	0x1B80, 0x30D90C67, 0x1B80, 0x00220C75, 0x1B80, 0x00220C77,
+	0x1B80, 0xE16D0C85, 0x1B80, 0xE16D0C87, 0x1B80, 0x00020C95,
+	0x1B80, 0x00020C97, 0x1B80, 0x54E80CA5, 0x1B80, 0x54E80CA7,
+	0x1B80, 0x0BA60CB5, 0x1B80, 0x0BA60CB7, 0x1B80, 0x00220CC5,
+	0x1B80, 0x00220CC7, 0x1B80, 0xE16D0CD5, 0x1B80, 0xE16D0CD7,
+	0x1B80, 0x00020CE5, 0x1B80, 0x00020CE7, 0x1B80, 0x4D100CF5,
+	0x1B80, 0x4D100CF7, 0x1B80, 0x30900D05, 0x1B80, 0x30900D07,
+	0x1B80, 0x5C320D15, 0x1B80, 0x5C320D17, 0x1B80, 0x54F00D25,
+	0x1B80, 0x54F00D27, 0x1B80, 0x67F10D35, 0x1B80, 0x67F10D37,
+	0x1B80, 0xE1420D45, 0x1B80, 0xE1420D47, 0x1B80, 0xE16D0D55,
+	0x1B80, 0xE16D0D57, 0x1B80, 0x67F40D65, 0x1B80, 0x67F40D67,
+	0x1B80, 0xE1420D75, 0x1B80, 0xE1420D77, 0x1B80, 0xE16D0D85,
+	0x1B80, 0xE16D0D87, 0x1B80, 0x5C320D95, 0x1B80, 0x5C320D97,
+	0x1B80, 0x54F10DA5, 0x1B80, 0x54F10DA7, 0x1B80, 0x0BA80DB5,
+	0x1B80, 0x0BA80DB7, 0x1B80, 0x67F80DC5, 0x1B80, 0x67F80DC7,
+	0x1B80, 0xE1420DD5, 0x1B80, 0xE1420DD7, 0x1B80, 0xE16D0DE5,
+	0x1B80, 0xE16D0DE7, 0x1B80, 0x5C320DF5, 0x1B80, 0x5C320DF7,
+	0x1B80, 0x54F10E05, 0x1B80, 0x54F10E07, 0x1B80, 0x0BA90E15,
+	0x1B80, 0x0BA90E17, 0x1B80, 0x67FC0E25, 0x1B80, 0x67FC0E27,
+	0x1B80, 0xE1420E35, 0x1B80, 0xE1420E37, 0x1B80, 0xE16D0E45,
+	0x1B80, 0xE16D0E47, 0x1B80, 0x67FF0E55, 0x1B80, 0x67FF0E57,
+	0x1B80, 0xE1420E65, 0x1B80, 0xE1420E67, 0x1B80, 0xE16D0E75,
+	0x1B80, 0xE16D0E77, 0x1B80, 0x5C320E85, 0x1B80, 0x5C320E87,
+	0x1B80, 0x54F20E95, 0x1B80, 0x54F20E97, 0x1B80, 0x67000EA5,
+	0x1B80, 0x67000EA7, 0x1B80, 0xE1420EB5, 0x1B80, 0xE1420EB7,
+	0x1B80, 0xE16D0EC5, 0x1B80, 0xE16D0EC7, 0x1B80, 0x67030ED5,
+	0x1B80, 0x67030ED7, 0x1B80, 0xE1420EE5, 0x1B80, 0xE1420EE7,
+	0x1B80, 0xE16D0EF5, 0x1B80, 0xE16D0EF7, 0x1B80, 0xF9CC0F05,
+	0x1B80, 0xF9CC0F07, 0x1B80, 0x67070F15, 0x1B80, 0x67070F17,
+	0x1B80, 0xE1420F25, 0x1B80, 0xE1420F27, 0x1B80, 0xE16D0F35,
+	0x1B80, 0xE16D0F37, 0x1B80, 0xFAD30F45, 0x1B80, 0xFAD30F47,
+	0x1B80, 0x5C320F55, 0x1B80, 0x5C320F57, 0x1B80, 0x54F30F65,
+	0x1B80, 0x54F30F67, 0x1B80, 0x670B0F75, 0x1B80, 0x670B0F77,
+	0x1B80, 0xE1420F85, 0x1B80, 0xE1420F87, 0x1B80, 0xE16D0F95,
+	0x1B80, 0xE16D0F97, 0x1B80, 0x670E0FA5, 0x1B80, 0x670E0FA7,
+	0x1B80, 0xE1420FB5, 0x1B80, 0xE1420FB7, 0x1B80, 0xE16D0FC5,
+	0x1B80, 0xE16D0FC7, 0x1B80, 0x4D100FD5, 0x1B80, 0x4D100FD7,
+	0x1B80, 0x30900FE5, 0x1B80, 0x30900FE7, 0x1B80, 0x00010FF5,
+	0x1B80, 0x00010FF7, 0x1B80, 0x7B241005, 0x1B80, 0x7B241007,
+	0x1B80, 0x7A401015, 0x1B80, 0x7A401017, 0x1B80, 0x79001025,
+	0x1B80, 0x79001027, 0x1B80, 0x55031035, 0x1B80, 0x55031037,
+	0x1B80, 0x310C1045, 0x1B80, 0x310C1047, 0x1B80, 0x7B1C1055,
+	0x1B80, 0x7B1C1057, 0x1B80, 0x7A401065, 0x1B80, 0x7A401067,
+	0x1B80, 0x550B1075, 0x1B80, 0x550B1077, 0x1B80, 0x310C1085,
+	0x1B80, 0x310C1087, 0x1B80, 0x7B201095, 0x1B80, 0x7B201097,
+	0x1B80, 0x7A0010A5, 0x1B80, 0x7A0010A7, 0x1B80, 0x551310B5,
+	0x1B80, 0x551310B7, 0x1B80, 0x740110C5, 0x1B80, 0x740110C7,
+	0x1B80, 0x740010D5, 0x1B80, 0x740010D7, 0x1B80, 0x8E0010E5,
+	0x1B80, 0x8E0010E7, 0x1B80, 0x000110F5, 0x1B80, 0x000110F7,
+	0x1B80, 0x57021105, 0x1B80, 0x57021107, 0x1B80, 0x57001115,
+	0x1B80, 0x57001117, 0x1B80, 0x97001125, 0x1B80, 0x97001127,
+	0x1B80, 0x00011135, 0x1B80, 0x00011137, 0x1B80, 0x4F781145,
+	0x1B80, 0x4F781147, 0x1B80, 0x53881155, 0x1B80, 0x53881157,
+	0x1B80, 0xE1221165, 0x1B80, 0xE1221167, 0x1B80, 0x54801175,
+	0x1B80, 0x54801177, 0x1B80, 0x54001185, 0x1B80, 0x54001187,
+	0x1B80, 0xE1221195, 0x1B80, 0xE1221197, 0x1B80, 0x548111A5,
+	0x1B80, 0x548111A7, 0x1B80, 0x540011B5, 0x1B80, 0x540011B7,
+	0x1B80, 0xE12211C5, 0x1B80, 0xE12211C7, 0x1B80, 0x548211D5,
+	0x1B80, 0x548211D7, 0x1B80, 0x540011E5, 0x1B80, 0x540011E7,
+	0x1B80, 0xE12D11F5, 0x1B80, 0xE12D11F7, 0x1B80, 0xBF1D1205,
+	0x1B80, 0xBF1D1207, 0x1B80, 0x301D1215, 0x1B80, 0x301D1217,
+	0x1B80, 0xE1001225, 0x1B80, 0xE1001227, 0x1B80, 0xE1051235,
+	0x1B80, 0xE1051237, 0x1B80, 0xE1091245, 0x1B80, 0xE1091247,
+	0x1B80, 0xE1101255, 0x1B80, 0xE1101257, 0x1B80, 0xE1641265,
+	0x1B80, 0xE1641267, 0x1B80, 0x55131275, 0x1B80, 0x55131277,
+	0x1B80, 0xE10C1285, 0x1B80, 0xE10C1287, 0x1B80, 0x55151295,
+	0x1B80, 0x55151297, 0x1B80, 0xE11012A5, 0x1B80, 0xE11012A7,
+	0x1B80, 0xE16412B5, 0x1B80, 0xE16412B7, 0x1B80, 0x000112C5,
+	0x1B80, 0x000112C7, 0x1B80, 0x54BF12D5, 0x1B80, 0x54BF12D7,
+	0x1B80, 0x54C012E5, 0x1B80, 0x54C012E7, 0x1B80, 0x54A312F5,
+	0x1B80, 0x54A312F7, 0x1B80, 0x54C11305, 0x1B80, 0x54C11307,
+	0x1B80, 0x54A41315, 0x1B80, 0x54A41317, 0x1B80, 0x4C181325,
+	0x1B80, 0x4C181327, 0x1B80, 0xBF071335, 0x1B80, 0xBF071337,
+	0x1B80, 0x54C21345, 0x1B80, 0x54C21347, 0x1B80, 0x54A41355,
+	0x1B80, 0x54A41357, 0x1B80, 0xBF041365, 0x1B80, 0xBF041367,
+	0x1B80, 0x54C11375, 0x1B80, 0x54C11377, 0x1B80, 0x54A31385,
+	0x1B80, 0x54A31387, 0x1B80, 0xBF011395, 0x1B80, 0xBF011397,
+	0x1B80, 0xE17213A5, 0x1B80, 0xE17213A7, 0x1B80, 0x54DF13B5,
+	0x1B80, 0x54DF13B7, 0x1B80, 0x000113C5, 0x1B80, 0x000113C7,
+	0x1B80, 0x54BF13D5, 0x1B80, 0x54BF13D7, 0x1B80, 0x54E513E5,
+	0x1B80, 0x54E513E7, 0x1B80, 0x050A13F5, 0x1B80, 0x050A13F7,
+	0x1B80, 0x54DF1405, 0x1B80, 0x54DF1407, 0x1B80, 0x00011415,
+	0x1B80, 0x00011417, 0x1B80, 0x7F201425, 0x1B80, 0x7F201427,
+	0x1B80, 0x7E001435, 0x1B80, 0x7E001437, 0x1B80, 0x7D001445,
+	0x1B80, 0x7D001447, 0x1B80, 0x55011455, 0x1B80, 0x55011457,
+	0x1B80, 0x5C311465, 0x1B80, 0x5C311467, 0x1B80, 0xE10C1475,
+	0x1B80, 0xE10C1477, 0x1B80, 0xE1101485, 0x1B80, 0xE1101487,
+	0x1B80, 0x54801495, 0x1B80, 0x54801497, 0x1B80, 0x540014A5,
+	0x1B80, 0x540014A7, 0x1B80, 0xE10C14B5, 0x1B80, 0xE10C14B7,
+	0x1B80, 0xE11014C5, 0x1B80, 0xE11014C7, 0x1B80, 0x548114D5,
+	0x1B80, 0x548114D7, 0x1B80, 0x540014E5, 0x1B80, 0x540014E7,
+	0x1B80, 0xE10C14F5, 0x1B80, 0xE10C14F7, 0x1B80, 0xE1101505,
+	0x1B80, 0xE1101507, 0x1B80, 0x54821515, 0x1B80, 0x54821517,
+	0x1B80, 0x54001525, 0x1B80, 0x54001527, 0x1B80, 0xE12D1535,
+	0x1B80, 0xE12D1537, 0x1B80, 0xBFE91545, 0x1B80, 0xBFE91547,
+	0x1B80, 0x301D1555, 0x1B80, 0x301D1557, 0x1B80, 0x00231565,
+	0x1B80, 0x00231567, 0x1B80, 0x7B201575, 0x1B80, 0x7B201577,
+	0x1B80, 0x7A001585, 0x1B80, 0x7A001587, 0x1B80, 0x79001595,
+	0x1B80, 0x79001597, 0x1B80, 0xE16815A5, 0x1B80, 0xE16815A7,
+	0x1B80, 0x000215B5, 0x1B80, 0x000215B7, 0x1B80, 0x000115C5,
+	0x1B80, 0x000115C7, 0x1B80, 0x002215D5, 0x1B80, 0x002215D7,
+	0x1B80, 0x7B2015E5, 0x1B80, 0x7B2015E7, 0x1B80, 0x7A0015F5,
+	0x1B80, 0x7A0015F7, 0x1B80, 0x79001605, 0x1B80, 0x79001607,
+	0x1B80, 0xE1681615, 0x1B80, 0xE1681617, 0x1B80, 0x00021625,
+	0x1B80, 0x00021627, 0x1B80, 0x00011635, 0x1B80, 0x00011637,
+	0x1B80, 0x549F1645, 0x1B80, 0x549F1647, 0x1B80, 0x54FF1655,
+	0x1B80, 0x54FF1657, 0x1B80, 0x54001665, 0x1B80, 0x54001667,
+	0x1B80, 0x00011675, 0x1B80, 0x00011677, 0x1B80, 0x5C311685,
+	0x1B80, 0x5C311687, 0x1B80, 0x07141695, 0x1B80, 0x07141697,
+	0x1B80, 0x540016A5, 0x1B80, 0x540016A7, 0x1B80, 0x5C3216B5,
+	0x1B80, 0x5C3216B7, 0x1B80, 0x000116C5, 0x1B80, 0x000116C7,
+	0x1B80, 0x5C3216D5, 0x1B80, 0x5C3216D7, 0x1B80, 0x071416E5,
+	0x1B80, 0x071416E7, 0x1B80, 0x540016F5, 0x1B80, 0x540016F7,
+	0x1B80, 0x5C311705, 0x1B80, 0x5C311707, 0x1B80, 0x00011715,
+	0x1B80, 0x00011717, 0x1B80, 0x4C981725, 0x1B80, 0x4C981727,
+	0x1B80, 0x4C181735, 0x1B80, 0x4C181737, 0x1B80, 0x00011745,
+	0x1B80, 0x00011747, 0x1B80, 0x5C321755, 0x1B80, 0x5C321757,
+	0x1B80, 0x62841765, 0x1B80, 0x62841767, 0x1B80, 0x66861775,
+	0x1B80, 0x66861777, 0x1B80, 0x6C031785, 0x1B80, 0x6C031787,
+	0x1B80, 0x7B201795, 0x1B80, 0x7B201797, 0x1B80, 0x7A0017A5,
+	0x1B80, 0x7A0017A7, 0x1B80, 0x790017B5, 0x1B80, 0x790017B7,
+	0x1B80, 0x7F2017C5, 0x1B80, 0x7F2017C7, 0x1B80, 0x7E0017D5,
+	0x1B80, 0x7E0017D7, 0x1B80, 0x7D0017E5, 0x1B80, 0x7D0017E7,
+	0x1B80, 0x090117F5, 0x1B80, 0x090117F7, 0x1B80, 0x0C011805,
+	0x1B80, 0x0C011807, 0x1B80, 0x0BA61815, 0x1B80, 0x0BA61817,
+	0x1B80, 0x00011825, 0x1B80, 0x00011827, 0x1B80, 0x00000006,
+	0x1B80, 0x00000002,
+
+};
+
+void odm_read_and_config_mp_8822b_phy_reg(struct phy_dm_struct *dm)
+{
+	u32 i = 0;
+	u8 c_cond;
+	bool is_matched = true, is_skipped = false;
+	u32 array_len = sizeof(array_mp_8822b_phy_reg) / sizeof(u32);
+	u32 *array = array_mp_8822b_phy_reg;
+
+	u32 v1 = 0, v2 = 0, pre_v1 = 0, pre_v2 = 0;
+
+	ODM_RT_TRACE(dm, ODM_COMP_INIT,
+		     "===> %s\n", __func__);
+
+	for (; (i + 1) < array_len; i = i + 2) {
+		v1 = array[i];
+		v2 = array[i + 1];
+
+		if (v1 & BIT(31)) { /* positive condition*/
+			c_cond = (u8)((v1 & (BIT(29) | BIT(28))) >> 28);
+			if (c_cond == COND_ENDIF) { /*end*/
+				is_matched = true;
+				is_skipped = false;
+				ODM_RT_TRACE(dm, ODM_COMP_INIT, "ENDIF\n");
+			} else if (c_cond == COND_ELSE) { /*else*/
+				is_matched = is_skipped ? false : true;
+				ODM_RT_TRACE(dm, ODM_COMP_INIT, "ELSE\n");
+			} else { /*if , else if*/
+				pre_v1 = v1;
+				pre_v2 = v2;
+				ODM_RT_TRACE(dm, ODM_COMP_INIT,
+					     "IF or ELSE IF\n");
+			}
+		} else if (v1 & BIT(30)) { /*negative condition*/
+			if (is_skipped) {
+				is_matched = false;
+				continue;
+			}
+
+			if (check_positive(dm, pre_v1, pre_v2, v1, v2)) {
+				is_matched = true;
+				is_skipped = true;
+			} else {
+				is_matched = false;
+				is_skipped = false;
+			}
+		} else if (is_matched) {
+			odm_config_bb_phy_8822b(dm, v1, MASKDWORD, v2);
+		}
+	}
+}
+
+u32 odm_get_version_mp_8822b_phy_reg(void) { return 67; }
+
+/******************************************************************************
+ *                           phy_reg_pg.TXT
+ ******************************************************************************/
+
+static u32 array_mp_8822b_phy_reg_pg[] = {
+	0, 0, 0, 0x00000c20, 0xffffffff, 0x32343638,
+	0, 0, 0, 0x00000c24, 0xffffffff, 0x36384042,
+	0, 0, 0, 0x00000c28, 0xffffffff, 0x28303234,
+	0, 0, 0, 0x00000c2c, 0xffffffff, 0x34363840,
+	0, 0, 0, 0x00000c30, 0xffffffff, 0x26283032,
+	0, 0, 1, 0x00000c34, 0xffffffff, 0x34363840,
+	0, 0, 1, 0x00000c38, 0xffffffff, 0x26283032,
+	0, 0, 0, 0x00000c3c, 0xffffffff, 0x34363840,
+	0, 0, 0, 0x00000c40, 0xffffffff, 0x26283032,
+	0, 0, 0, 0x00000c44, 0xffffffff, 0x38402224,
+	0, 0, 1, 0x00000c48, 0xffffffff, 0x30323436,
+	0, 0, 1, 0x00000c4c, 0xffffffff, 0x22242628,
+	0, 1, 0, 0x00000e20, 0xffffffff, 0x32343638,
+	0, 1, 0, 0x00000e24, 0xffffffff, 0x36384042,
+	0, 1, 0, 0x00000e28, 0xffffffff, 0x28303234,
+	0, 1, 0, 0x00000e2c, 0xffffffff, 0x34363840,
+	0, 1, 0, 0x00000e30, 0xffffffff, 0x26283032,
+	0, 1, 1, 0x00000e34, 0xffffffff, 0x34363840,
+	0, 1, 1, 0x00000e38, 0xffffffff, 0x26283032,
+	0, 1, 0, 0x00000e3c, 0xffffffff, 0x34363840,
+	0, 1, 0, 0x00000e40, 0xffffffff, 0x26283032,
+	0, 1, 0, 0x00000e44, 0xffffffff, 0x38402224,
+	0, 1, 1, 0x00000e48, 0xffffffff, 0x30323436,
+	0, 1, 1, 0x00000e4c, 0xffffffff, 0x22242628,
+	1, 0, 0, 0x00000c24, 0xffffffff, 0x34363840,
+	1, 0, 0, 0x00000c28, 0xffffffff, 0x26283032,
+	1, 0, 0, 0x00000c2c, 0xffffffff, 0x32343638,
+	1, 0, 0, 0x00000c30, 0xffffffff, 0x24262830,
+	1, 0, 1, 0x00000c34, 0xffffffff, 0x32343638,
+	1, 0, 1, 0x00000c38, 0xffffffff, 0x24262830,
+	1, 0, 0, 0x00000c3c, 0xffffffff, 0x32343638,
+	1, 0, 0, 0x00000c40, 0xffffffff, 0x24262830,
+	1, 0, 0, 0x00000c44, 0xffffffff, 0x36382022,
+	1, 0, 1, 0x00000c48, 0xffffffff, 0x28303234,
+	1, 0, 1, 0x00000c4c, 0xffffffff, 0x20222426,
+	1, 1, 0, 0x00000e24, 0xffffffff, 0x34363840,
+	1, 1, 0, 0x00000e28, 0xffffffff, 0x26283032,
+	1, 1, 0, 0x00000e2c, 0xffffffff, 0x32343638,
+	1, 1, 0, 0x00000e30, 0xffffffff, 0x24262830,
+	1, 1, 1, 0x00000e34, 0xffffffff, 0x32343638,
+	1, 1, 1, 0x00000e38, 0xffffffff, 0x24262830,
+	1, 1, 0, 0x00000e3c, 0xffffffff, 0x32343638,
+	1, 1, 0, 0x00000e40, 0xffffffff, 0x24262830,
+	1, 1, 0, 0x00000e44, 0xffffffff, 0x36382022,
+	1, 1, 1, 0x00000e48, 0xffffffff, 0x28303234,
+	1, 1, 1, 0x00000e4c, 0xffffffff, 0x20222426,
+};
+
+void odm_read_and_config_mp_8822b_phy_reg_pg(struct phy_dm_struct *dm)
+{
+	u32 i = 0;
+	u32 array_len = sizeof(array_mp_8822b_phy_reg_pg) / sizeof(u32);
+	u32 *array = array_mp_8822b_phy_reg_pg;
+
+	ODM_RT_TRACE(dm, ODM_COMP_INIT,
+		     "===> %s\n", __func__);
+
+	dm->phy_reg_pg_version = 1;
+	dm->phy_reg_pg_value_type = PHY_REG_PG_EXACT_VALUE;
+
+	for (i = 0; i < array_len; i += 6) {
+		u32 v1 = array[i];
+		u32 v2 = array[i + 1];
+		u32 v3 = array[i + 2];
+		u32 v4 = array[i + 3];
+		u32 v5 = array[i + 4];
+		u32 v6 = array[i + 5];
+
+		odm_config_bb_phy_reg_pg_8822b(dm, v1, v2, v3, v4, v5, v6);
+	}
+}
diff --git a/drivers/staging/rtlwifi/phydm/rtl8822b/halhwimg8822b_bb.h b/drivers/staging/rtlwifi/phydm/rtl8822b/halhwimg8822b_bb.h
new file mode 100644
index 000000000000..53431998b47e
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/rtl8822b/halhwimg8822b_bb.h
@@ -0,0 +1,54 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+/*Image2HeaderVersion: 3.2*/
+#ifndef __INC_MP_BB_HW_IMG_8822B_H
+#define __INC_MP_BB_HW_IMG_8822B_H
+
+/******************************************************************************
+ *                           agc_tab.TXT
+ ******************************************************************************/
+
+void odm_read_and_config_mp_8822b_agc_tab(/* tc: Test Chip, mp: mp Chip*/
+					  struct phy_dm_struct *dm);
+u32 odm_get_version_mp_8822b_agc_tab(void);
+
+/******************************************************************************
+ *                           phy_reg.TXT
+ ******************************************************************************/
+
+void odm_read_and_config_mp_8822b_phy_reg(/* tc: Test Chip, mp: mp Chip*/
+					  struct phy_dm_struct *dm);
+u32 odm_get_version_mp_8822b_phy_reg(void);
+
+/******************************************************************************
+ *                           phy_reg_pg.TXT
+ ******************************************************************************/
+
+void odm_read_and_config_mp_8822b_phy_reg_pg(/* tc: Test Chip, mp: mp Chip*/
+					     struct phy_dm_struct *dm);
+u32 odm_get_version_mp_8822b_phy_reg_pg(void);
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/rtl8822b/halhwimg8822b_mac.c b/drivers/staging/rtlwifi/phydm/rtl8822b/halhwimg8822b_mac.c
new file mode 100644
index 000000000000..1a9daed2e609
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/rtl8822b/halhwimg8822b_mac.c
@@ -0,0 +1,222 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+/*Image2HeaderVersion: 3.2*/
+#include "../mp_precomp.h"
+#include "../phydm_precomp.h"
+
+static bool check_positive(struct phy_dm_struct *dm, const u32 condition1,
+			   const u32 condition2, const u32 condition3,
+			   const u32 condition4)
+{
+	u8 _board_type = ((dm->board_type & BIT(4)) >> 4) << 0 | /* _GLNA*/
+			 ((dm->board_type & BIT(3)) >> 3) << 1 | /* _GPA*/
+			 ((dm->board_type & BIT(7)) >> 7) << 2 | /* _ALNA*/
+			 ((dm->board_type & BIT(6)) >> 6) << 3 | /* _APA */
+			 ((dm->board_type & BIT(2)) >> 2) << 4; /* _BT*/
+
+	u32 cond1 = condition1, cond2 = condition2, cond3 = condition3,
+	    cond4 = condition4;
+
+	u8 cut_version_for_para =
+		(dm->cut_version == ODM_CUT_A) ? 14 : dm->cut_version;
+	u8 pkg_type_for_para = (dm->package_type == 0) ? 14 : dm->package_type;
+
+	u32 driver1 = cut_version_for_para << 24 |
+		      (dm->support_interface & 0xF0) << 16 |
+		      dm->support_platform << 16 | pkg_type_for_para << 12 |
+		      (dm->support_interface & 0x0F) << 8 | _board_type;
+
+	u32 driver2 = (dm->type_glna & 0xFF) << 0 | (dm->type_gpa & 0xFF) << 8 |
+		      (dm->type_alna & 0xFF) << 16 |
+		      (dm->type_apa & 0xFF) << 24;
+
+	u32 driver3 = 0;
+
+	u32 driver4 = (dm->type_glna & 0xFF00) >> 8 | (dm->type_gpa & 0xFF00) |
+		      (dm->type_alna & 0xFF00) << 8 |
+		      (dm->type_apa & 0xFF00) << 16;
+
+	ODM_RT_TRACE(
+		dm, ODM_COMP_INIT,
+		"===> %s (cond1, cond2, cond3, cond4) = (0x%X 0x%X 0x%X 0x%X)\n",
+		__func__, cond1, cond2, cond3, cond4);
+	ODM_RT_TRACE(
+		dm, ODM_COMP_INIT,
+		"===> %s (driver1, driver2, driver3, driver4) = (0x%X 0x%X 0x%X 0x%X)\n",
+		__func__, driver1, driver2, driver3, driver4);
+
+	ODM_RT_TRACE(dm, ODM_COMP_INIT,
+		     "	(Platform, Interface) = (0x%X, 0x%X)\n",
+		     dm->support_platform, dm->support_interface);
+	ODM_RT_TRACE(dm, ODM_COMP_INIT,
+		     "	(Board, Package) = (0x%X, 0x%X)\n",
+		     dm->board_type, dm->package_type);
+
+	/*============== value Defined Check ===============*/
+	/*QFN type [15:12] and cut version [27:24] need to do value check*/
+
+	if (((cond1 & 0x0000F000) != 0) &&
+	    ((cond1 & 0x0000F000) != (driver1 & 0x0000F000)))
+		return false;
+	if (((cond1 & 0x0F000000) != 0) &&
+	    ((cond1 & 0x0F000000) != (driver1 & 0x0F000000)))
+		return false;
+
+	/*=============== Bit Defined Check ================*/
+	/* We don't care [31:28] */
+
+	cond1 &= 0x00FF0FFF;
+	driver1 &= 0x00FF0FFF;
+
+	if ((cond1 & driver1) == cond1) {
+		u32 bit_mask = 0;
+
+		if ((cond1 & 0x0F) == 0) /* board_type is DONTCARE*/
+			return true;
+
+		if ((cond1 & BIT(0)) != 0) /*GLNA*/
+			bit_mask |= 0x000000FF;
+		if ((cond1 & BIT(1)) != 0) /*GPA*/
+			bit_mask |= 0x0000FF00;
+		if ((cond1 & BIT(2)) != 0) /*ALNA*/
+			bit_mask |= 0x00FF0000;
+		if ((cond1 & BIT(3)) != 0) /*APA*/
+			bit_mask |= 0xFF000000;
+
+		if (((cond2 & bit_mask) == (driver2 & bit_mask)) &&
+		    ((cond4 & bit_mask) ==
+		     (driver4 &
+		      bit_mask))) /* board_type of each RF path is matched*/
+			return true;
+		else
+			return false;
+	} else {
+		return false;
+	}
+}
+
+/******************************************************************************
+ *                           mac_reg.TXT
+ ******************************************************************************/
+
+static u32 array_mp_8822b_mac_reg[] = {
+	0x029,  0x000000F9, 0x420,  0x00000080, 0x421,  0x0000000F,
+	0x428,  0x0000000A, 0x429,  0x00000010, 0x430,  0x00000000,
+	0x431,  0x00000000, 0x432,  0x00000000, 0x433,  0x00000001,
+	0x434,  0x00000004, 0x435,  0x00000005, 0x436,  0x00000007,
+	0x437,  0x00000008, 0x43C,  0x00000004, 0x43D,  0x00000005,
+	0x43E,  0x00000007, 0x43F,  0x00000008, 0x440,  0x0000005D,
+	0x441,  0x00000001, 0x442,  0x00000000, 0x444,  0x00000010,
+	0x445,  0x000000F0, 0x446,  0x00000001, 0x447,  0x000000FE,
+	0x448,  0x00000000, 0x449,  0x00000000, 0x44A,  0x00000000,
+	0x44B,  0x00000040, 0x44C,  0x00000010, 0x44D,  0x000000F0,
+	0x44E,  0x0000003F, 0x44F,  0x00000000, 0x450,  0x00000000,
+	0x451,  0x00000000, 0x452,  0x00000000, 0x453,  0x00000040,
+	0x455,  0x00000070, 0x45E,  0x00000004, 0x49C,  0x00000010,
+	0x49D,  0x000000F0, 0x49E,  0x00000000, 0x49F,  0x00000006,
+	0x4A0,  0x000000E0, 0x4A1,  0x00000003, 0x4A2,  0x00000000,
+	0x4A3,  0x00000040, 0x4A4,  0x00000015, 0x4A5,  0x000000F0,
+	0x4A6,  0x00000000, 0x4A7,  0x00000006, 0x4A8,  0x000000E0,
+	0x4A9,  0x00000000, 0x4AA,  0x00000000, 0x4AB,  0x00000000,
+	0x7DA,  0x00000008, 0x1448, 0x00000006, 0x144A, 0x00000006,
+	0x144C, 0x00000006, 0x144E, 0x00000006, 0x4C8,  0x000000FF,
+	0x4C9,  0x00000008, 0x4CA,  0x00000020, 0x4CB,  0x00000020,
+	0x4CC,  0x000000FF, 0x4CD,  0x000000FF, 0x4CE,  0x00000001,
+	0x4CF,  0x00000008, 0x500,  0x00000026, 0x501,  0x000000A2,
+	0x502,  0x0000002F, 0x503,  0x00000000, 0x504,  0x00000028,
+	0x505,  0x000000A3, 0x506,  0x0000005E, 0x507,  0x00000000,
+	0x508,  0x0000002B, 0x509,  0x000000A4, 0x50A,  0x0000005E,
+	0x50B,  0x00000000, 0x50C,  0x0000004F, 0x50D,  0x000000A4,
+	0x50E,  0x00000000, 0x50F,  0x00000000, 0x512,  0x0000001C,
+	0x514,  0x0000000A, 0x516,  0x0000000A, 0x521,  0x0000002F,
+	0x525,  0x0000004F, 0x551,  0x00000010, 0x559,  0x00000002,
+	0x55C,  0x00000050, 0x55D,  0x000000FF, 0x577,  0x0000000B,
+	0x5BE,  0x00000064, 0x605,  0x00000030, 0x608,  0x0000000E,
+	0x609,  0x00000022, 0x60C,  0x00000018, 0x6A0,  0x000000FF,
+	0x6A1,  0x000000FF, 0x6A2,  0x000000FF, 0x6A3,  0x000000FF,
+	0x6A4,  0x000000FF, 0x6A5,  0x000000FF, 0x6DE,  0x00000084,
+	0x620,  0x000000FF, 0x621,  0x000000FF, 0x622,  0x000000FF,
+	0x623,  0x000000FF, 0x624,  0x000000FF, 0x625,  0x000000FF,
+	0x626,  0x000000FF, 0x627,  0x000000FF, 0x638,  0x00000050,
+	0x63C,  0x0000000A, 0x63D,  0x0000000A, 0x63E,  0x0000000E,
+	0x63F,  0x0000000E, 0x640,  0x00000040, 0x642,  0x00000040,
+	0x643,  0x00000000, 0x652,  0x000000C8, 0x66E,  0x00000005,
+	0x718,  0x00000040, 0x7D4,  0x00000098,
+
+};
+
+void odm_read_and_config_mp_8822b_mac_reg(struct phy_dm_struct *dm)
+{
+	u32 i = 0;
+	u8 c_cond;
+	bool is_matched = true, is_skipped = false;
+	u32 array_len = sizeof(array_mp_8822b_mac_reg) / sizeof(u32);
+	u32 *array = array_mp_8822b_mac_reg;
+
+	u32 v1 = 0, v2 = 0, pre_v1 = 0, pre_v2 = 0;
+
+	ODM_RT_TRACE(dm, ODM_COMP_INIT,
+		     "===> %s\n", __func__);
+
+	for (; (i + 1) < array_len; i = i + 2) {
+		v1 = array[i];
+		v2 = array[i + 1];
+
+		if (v1 & BIT(31)) { /* positive condition*/
+			c_cond = (u8)((v1 & (BIT(29) | BIT(28))) >> 28);
+			if (c_cond == COND_ENDIF) { /*end*/
+				is_matched = true;
+				is_skipped = false;
+				ODM_RT_TRACE(dm, ODM_COMP_INIT, "ENDIF\n");
+			} else if (c_cond == COND_ELSE) { /*else*/
+				is_matched = is_skipped ? false : true;
+				ODM_RT_TRACE(dm, ODM_COMP_INIT, "ELSE\n");
+			} else { /*if , else if*/
+				pre_v1 = v1;
+				pre_v2 = v2;
+				ODM_RT_TRACE(dm, ODM_COMP_INIT,
+					     "IF or ELSE IF\n");
+			}
+		} else if (v1 & BIT(30)) { /*negative condition*/
+			if (is_skipped) {
+				is_matched = false;
+				continue;
+			}
+
+			if (check_positive(dm, pre_v1, pre_v2, v1, v2)) {
+				is_matched = true;
+				is_skipped = true;
+			} else {
+				is_matched = false;
+				is_skipped = false;
+			}
+		} else if (is_matched) {
+			odm_config_mac_8822b(dm, v1, (u8)v2);
+		}
+	}
+}
+
+u32 odm_get_version_mp_8822b_mac_reg(void) { return 67; }
diff --git a/drivers/staging/rtlwifi/phydm/rtl8822b/halhwimg8822b_mac.h b/drivers/staging/rtlwifi/phydm/rtl8822b/halhwimg8822b_mac.h
new file mode 100644
index 000000000000..d02fdd7a4a53
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/rtl8822b/halhwimg8822b_mac.h
@@ -0,0 +1,38 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+/*Image2HeaderVersion: 3.2*/
+#ifndef __INC_MP_MAC_HW_IMG_8822B_H
+#define __INC_MP_MAC_HW_IMG_8822B_H
+
+/******************************************************************************
+ *                           mac_reg.TXT
+ ******************************************************************************/
+
+void odm_read_and_config_mp_8822b_mac_reg(/* tc: Test Chip, mp: mp Chip*/
+					  struct phy_dm_struct *dm);
+u32 odm_get_version_mp_8822b_mac_reg(void);
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/rtl8822b/halhwimg8822b_rf.c b/drivers/staging/rtlwifi/phydm/rtl8822b/halhwimg8822b_rf.c
new file mode 100644
index 000000000000..84cdc0644207
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/rtl8822b/halhwimg8822b_rf.c
@@ -0,0 +1,4744 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+/*Image2HeaderVersion: 3.2*/
+#include "../mp_precomp.h"
+#include "../phydm_precomp.h"
+
+static bool check_positive(struct phy_dm_struct *dm, const u32 condition1,
+			   const u32 condition2, const u32 condition3,
+			   const u32 condition4)
+{
+	u8 _board_type = ((dm->board_type & BIT(4)) >> 4) << 0 | /* _GLNA*/
+			 ((dm->board_type & BIT(3)) >> 3) << 1 | /* _GPA*/
+			 ((dm->board_type & BIT(7)) >> 7) << 2 | /* _ALNA*/
+			 ((dm->board_type & BIT(6)) >> 6) << 3 | /* _APA */
+			 ((dm->board_type & BIT(2)) >> 2) << 4; /* _BT*/
+
+	u32 cond1 = condition1, cond2 = condition2, cond3 = condition3,
+	    cond4 = condition4;
+
+	u8 cut_version_for_para =
+		(dm->cut_version == ODM_CUT_A) ? 14 : dm->cut_version;
+	u8 pkg_type_for_para = (dm->package_type == 0) ? 14 : dm->package_type;
+
+	u32 driver1 = cut_version_for_para << 24 |
+		      (dm->support_interface & 0xF0) << 16 |
+		      dm->support_platform << 16 | pkg_type_for_para << 12 |
+		      (dm->support_interface & 0x0F) << 8 | _board_type;
+
+	u32 driver2 = (dm->type_glna & 0xFF) << 0 | (dm->type_gpa & 0xFF) << 8 |
+		      (dm->type_alna & 0xFF) << 16 |
+		      (dm->type_apa & 0xFF) << 24;
+
+	u32 driver3 = 0;
+
+	u32 driver4 = (dm->type_glna & 0xFF00) >> 8 | (dm->type_gpa & 0xFF00) |
+		      (dm->type_alna & 0xFF00) << 8 |
+		      (dm->type_apa & 0xFF00) << 16;
+
+	ODM_RT_TRACE(
+		dm, ODM_COMP_INIT,
+		"===> %s (cond1, cond2, cond3, cond4) = (0x%X 0x%X 0x%X 0x%X)\n",
+		__func__, cond1, cond2, cond3, cond4);
+	ODM_RT_TRACE(
+		dm, ODM_COMP_INIT,
+		"===> %s (driver1, driver2, driver3, driver4) = (0x%X 0x%X 0x%X 0x%X)\n",
+		__func__, driver1, driver2, driver3, driver4);
+
+	ODM_RT_TRACE(dm, ODM_COMP_INIT,
+		     "	(Platform, Interface) = (0x%X, 0x%X)\n",
+		     dm->support_platform, dm->support_interface);
+	ODM_RT_TRACE(dm, ODM_COMP_INIT,
+		     "	(Board, Package) = (0x%X, 0x%X)\n",
+		     dm->board_type, dm->package_type);
+
+	/*============== value Defined Check ===============*/
+	/*QFN type [15:12] and cut version [27:24] need to do value check*/
+
+	if (((cond1 & 0x0000F000) != 0) &&
+	    ((cond1 & 0x0000F000) != (driver1 & 0x0000F000)))
+		return false;
+	if (((cond1 & 0x0F000000) != 0) &&
+	    ((cond1 & 0x0F000000) != (driver1 & 0x0F000000)))
+		return false;
+
+	/*=============== Bit Defined Check ================*/
+	/* We don't care [31:28] */
+
+	cond1 &= 0x00FF0FFF;
+	driver1 &= 0x00FF0FFF;
+
+	if ((cond1 & driver1) == cond1) {
+		u32 bit_mask = 0;
+
+		if ((cond1 & 0x0F) == 0) /* board_type is DONTCARE*/
+			return true;
+
+		if ((cond1 & BIT(0)) != 0) /*GLNA*/
+			bit_mask |= 0x000000FF;
+		if ((cond1 & BIT(1)) != 0) /*GPA*/
+			bit_mask |= 0x0000FF00;
+		if ((cond1 & BIT(2)) != 0) /*ALNA*/
+			bit_mask |= 0x00FF0000;
+		if ((cond1 & BIT(3)) != 0) /*APA*/
+			bit_mask |= 0xFF000000;
+
+		if (((cond2 & bit_mask) == (driver2 & bit_mask)) &&
+		    ((cond4 & bit_mask) ==
+		     (driver4 &
+		      bit_mask))) /* board_type of each RF path is matched*/
+			return true;
+		else
+			return false;
+	} else {
+		return false;
+	}
+}
+
+/******************************************************************************
+ *                           radioa.TXT
+ ******************************************************************************/
+
+static u32 array_mp_8822b_radioa[] = {
+	0x000,      0x00030000, 0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x001,      0x0004002D, 0x9300100f, 0x05050505, 0x40000000, 0x00000000,
+	0x001,      0x0004002D, 0x9300100f, 0x00000000, 0x40000000, 0x00000000,
+	0x001,      0x0004002D, 0x9300200f, 0x00000000, 0x40000000, 0x00000000,
+	0x001,      0x0004002D, 0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x001,      0x0004002D, 0x9000100f, 0x05050505, 0x40000000, 0x00000000,
+	0x001,      0x0004002D, 0x9000100f, 0x00000000, 0x40000000, 0x00000000,
+	0x001,      0x0004002D, 0x9000200f, 0x00000000, 0x40000000, 0x00000000,
+	0x001,      0x0004002D, 0x9300200c, 0x00000000, 0x40000000, 0x00000000,
+	0x001,      0x00040029, 0x93012100, 0x00000000, 0x40000000, 0x00000000,
+	0x001,      0x00040029, 0x93002100, 0x00000000, 0x40000000, 0x00000000,
+	0x001,      0x00040029, 0x9000200c, 0x00000000, 0x40000000, 0x00000000,
+	0x001,      0x00040029, 0x90001004, 0x00000000, 0x40000000, 0x00000000,
+	0x001,      0x00040029, 0x93002000, 0x00000000, 0x40000000, 0x00000000,
+	0x001,      0x00040029, 0x90002100, 0x00000000, 0x40000000, 0x00000000,
+	0x001,      0x00040029, 0x90002000, 0x00000000, 0x40000000, 0x00000000,
+	0x001,      0x00040029, 0xA0000000, 0x00000000, 0x001,      0x00040029,
+	0xB0000000, 0x00000000, 0x018,      0x00010D24, 0x0EF,      0x00080000,
+	0x033,      0x00000002, 0x03E,      0x0000003F, 0x03F,      0x000C0F4E,
+	0x033,      0x00000001, 0x03E,      0x00000034, 0x03F,      0x0004080E,
+	0x0EF,      0x00080000, 0x0DF,      0x00002449, 0x033,      0x00000024,
+	0x03E,      0x0000003F, 0x03F,      0x00060FDE, 0x0EF,      0x00000000,
+	0x0EF,      0x00080000, 0x033,      0x00000025, 0x03E,      0x00000037,
+	0x03F,      0x0007EFCE, 0x0EF,      0x00000000, 0x0EF,      0x00080000,
+	0x033,      0x00000026, 0x03E,      0x00000037, 0x03F,      0x000DEFCE,
+	0x0EF,      0x00000000, 0x07F,      0x00000000, 0x8300100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x0B0,      0x000FF0F8, 0x9300100f, 0x05050505,
+	0x40000000, 0x00000000, 0x0B0,      0x000FF0F8, 0x9300100f, 0x00000000,
+	0x40000000, 0x00000000, 0x0B0,      0x000FF0F8, 0x9300200f, 0x00000000,
+	0x40000000, 0x00000000, 0x0B0,      0x000FB0F8, 0x9000100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x0B0,      0x000FF0F8, 0x9000100f, 0x05050505,
+	0x40000000, 0x00000000, 0x0B0,      0x000FF0F8, 0x9000100f, 0x00000000,
+	0x40000000, 0x00000000, 0x0B0,      0x000FF0F8, 0x9000200f, 0x00000000,
+	0x40000000, 0x00000000, 0x0B0,      0x000FB0F8, 0x9300200c, 0x00000000,
+	0x40000000, 0x00000000, 0x0B0,      0x000FB0F8, 0x93012100, 0x00000000,
+	0x40000000, 0x00000000, 0x0B0,      0x000FB0F8, 0x93002100, 0x00000000,
+	0x40000000, 0x00000000, 0x0B0,      0x000FB0F8, 0x93011000, 0x00000000,
+	0x40000000, 0x00000000, 0x0B0,      0x000FF0F8, 0x9000200c, 0x00000000,
+	0x40000000, 0x00000000, 0x0B0,      0x000FB0F8, 0x90001004, 0x00000000,
+	0x40000000, 0x00000000, 0x0B0,      0x000FF0F8, 0x93002000, 0x00000000,
+	0x40000000, 0x00000000, 0x0B0,      0x000FB0F8, 0x93001000, 0x00000000,
+	0x40000000, 0x00000000, 0x0B0,      0x000FF0F8, 0x90002100, 0x00000000,
+	0x40000000, 0x00000000, 0x0B0,      0x000FB0F8, 0x90002000, 0x00000000,
+	0x40000000, 0x00000000, 0x0B0,      0x000FB0F8, 0xA0000000, 0x00000000,
+	0x0B0,      0x000FF0F8, 0xB0000000, 0x00000000, 0x0B1,      0x0007DBE4,
+	0x0B2,      0x000225D1, 0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x0B3,      0x000FC760, 0x9300100f, 0x05050505, 0x40000000, 0x00000000,
+	0x0B3,      0x000FC760, 0x9300100f, 0x00000000, 0x40000000, 0x00000000,
+	0x0B3,      0x000FC760, 0x9300200f, 0x00000000, 0x40000000, 0x00000000,
+	0x0B3,      0x000FC760, 0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x0B3,      0x000FC760, 0x9000100f, 0x05050505, 0x40000000, 0x00000000,
+	0x0B3,      0x000FC760, 0x9000100f, 0x00000000, 0x40000000, 0x00000000,
+	0x0B3,      0x000FC760, 0x9000200f, 0x00000000, 0x40000000, 0x00000000,
+	0x0B3,      0x000FC760, 0x9300200c, 0x00000000, 0x40000000, 0x00000000,
+	0x0B3,      0x000FC760, 0x93012100, 0x00000000, 0x40000000, 0x00000000,
+	0x0B3,      0x000FC760, 0x93002100, 0x00000000, 0x40000000, 0x00000000,
+	0x0B3,      0x0007C330, 0xA0000000, 0x00000000, 0x0B3,      0x000FC760,
+	0xB0000000, 0x00000000, 0x0B4,      0x00099DD0, 0x0B5,      0x000400FC,
+	0x0B6,      0x000187F0, 0x0B7,      0x00030018, 0x0B8,      0x00080800,
+	0x0B9,      0x00000000, 0x0BA,      0x00008000, 0x0BB,      0x00000000,
+	0x0BC,      0x00040030, 0x0BD,      0x00000000, 0x0BE,      0x00000000,
+	0x0BF,      0x00000000, 0x0C0,      0x00000000, 0x0C1,      0x00000000,
+	0x0C2,      0x00000000, 0x0C3,      0x00000000, 0x0C4,      0x00002402,
+	0x0C5,      0x00000009, 0x0C6,      0x00040299, 0x0C7,      0x00055555,
+	0x0C8,      0x0000C16C, 0x0C9,      0x0001C140, 0x0CA,      0x00000000,
+	0x0CB,      0x00000000, 0x0CC,      0x00000000, 0x0CD,      0x00000000,
+	0x0CE,      0x00090C00, 0x0CF,      0x0006D200, 0x0DF,      0x00000009,
+	0x018,      0x00010524, 0x089,      0x00000207, 0x8300100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x08A,      0x000FF186, 0x9300100f, 0x05050505,
+	0x40000000, 0x00000000, 0x08A,      0x000FE186, 0x9300100f, 0x00000000,
+	0x40000000, 0x00000000, 0x08A,      0x000FF186, 0x9300200f, 0x00000000,
+	0x40000000, 0x00000000, 0x08A,      0x000FF186, 0x9000100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x08A,      0x000FF186, 0x9000100f, 0x05050505,
+	0x40000000, 0x00000000, 0x08A,      0x000FE186, 0x9000100f, 0x00000000,
+	0x40000000, 0x00000000, 0x08A,      0x000FF186, 0x9000200f, 0x00000000,
+	0x40000000, 0x00000000, 0x08A,      0x000FF186, 0x9300200c, 0x00000000,
+	0x40000000, 0x00000000, 0x08A,      0x000FF186, 0x93012100, 0x00000000,
+	0x40000000, 0x00000000, 0x08A,      0x000FF186, 0x93002100, 0x00000000,
+	0x40000000, 0x00000000, 0x08A,      0x000FE186, 0xA0000000, 0x00000000,
+	0x08A,      0x000FF186, 0xB0000000, 0x00000000, 0x08B,      0x00061E3C,
+	0x08C,      0x000112C7, 0x08D,      0x000F4988, 0x08E,      0x00064D40,
+	0x0EF,      0x00020000, 0x033,      0x00000007, 0x8300100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x9300100f, 0x05050505,
+	0x40000000, 0x00000000, 0x03E,      0x00004080, 0x9300100f, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x9300200f, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x9000100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x9000100f, 0x05050505,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x9000100f, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x9000200f, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x9300200c, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x93012100, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004000, 0x93002100, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004000, 0x93011000, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004000, 0x9000200c, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x90001004, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0xA0000000, 0x00000000,
+	0x03E,      0x00004000, 0xB0000000, 0x00000000, 0x8300100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9300100f, 0x05050505,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9300100f, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9300200f, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9000100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9000100f, 0x05050505,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9000100f, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9000200f, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9300200c, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x93012100, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x93002100, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000DFF86, 0x93011000, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9000200c, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x90001004, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x93002000, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C0006, 0x93001000, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0xA0000000, 0x00000000,
+	0x03F,      0x000C3186, 0xB0000000, 0x00000000, 0x033,      0x00000006,
+	0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x03E,      0x00004080,
+	0x9300100f, 0x05050505, 0x40000000, 0x00000000, 0x03E,      0x00004080,
+	0x9300100f, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00004080,
+	0x9300200f, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00004080,
+	0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x03E,      0x00004080,
+	0x9000100f, 0x05050505, 0x40000000, 0x00000000, 0x03E,      0x00004040,
+	0xA0000000, 0x00000000, 0x03E,      0x00004080, 0xB0000000, 0x00000000,
+	0x03F,      0x000C3186, 0x033,      0x00000005, 0x8300100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x03E,      0x000040C8, 0x9300100f, 0x05050505,
+	0x40000000, 0x00000000, 0x03E,      0x000040C8, 0x9300100f, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x000040C8, 0x9300200f, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x000040C8, 0x9000100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x03E,      0x000040C8, 0x9000100f, 0x05050505,
+	0x40000000, 0x00000000, 0x03E,      0x00004084, 0xA0000000, 0x00000000,
+	0x03E,      0x000040C8, 0xB0000000, 0x00000000, 0x03F,      0x000C3186,
+	0x033,      0x00000004, 0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x03E,      0x00004190, 0x9300100f, 0x05050505, 0x40000000, 0x00000000,
+	0x03E,      0x00004190, 0x9300100f, 0x00000000, 0x40000000, 0x00000000,
+	0x03E,      0x00004190, 0x9300200f, 0x00000000, 0x40000000, 0x00000000,
+	0x03E,      0x00004190, 0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x03E,      0x00004190, 0x9000100f, 0x05050505, 0x40000000, 0x00000000,
+	0x03E,      0x00004108, 0xA0000000, 0x00000000, 0x03E,      0x00004190,
+	0xB0000000, 0x00000000, 0x03F,      0x000C3186, 0x033,      0x00000003,
+	0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x03E,      0x00004998,
+	0x9300100f, 0x05050505, 0x40000000, 0x00000000, 0x03E,      0x00004998,
+	0x9300100f, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00004998,
+	0x9300200f, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00004998,
+	0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x03E,      0x00004998,
+	0x9000100f, 0x05050505, 0x40000000, 0x00000000, 0x03E,      0x0000490C,
+	0xA0000000, 0x00000000, 0x03E,      0x00004998, 0xB0000000, 0x00000000,
+	0x03F,      0x000C3186, 0x033,      0x00000002, 0x8300100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x03E,      0x00005840, 0x9300100f, 0x05050505,
+	0x40000000, 0x00000000, 0x03E,      0x00005840, 0x9300100f, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00005840, 0x9300200f, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00005840, 0x9000100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x03E,      0x00005840, 0x9000100f, 0x05050505,
+	0x40000000, 0x00000000, 0x03E,      0x00005E00, 0xA0000000, 0x00000000,
+	0x03E,      0x00005840, 0xB0000000, 0x00000000, 0x03F,      0x000C3186,
+	0x033,      0x00000001, 0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x03E,      0x000058C2, 0x9300100f, 0x05050505, 0x40000000, 0x00000000,
+	0x03E,      0x000058C2, 0x9300100f, 0x00000000, 0x40000000, 0x00000000,
+	0x03E,      0x000058C2, 0x9300200f, 0x00000000, 0x40000000, 0x00000000,
+	0x03E,      0x000058C2, 0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x03E,      0x000058C2, 0x9000100f, 0x05050505, 0x40000000, 0x00000000,
+	0x03E,      0x00005862, 0xA0000000, 0x00000000, 0x03E,      0x000058C2,
+	0xB0000000, 0x00000000, 0x03F,      0x000C3186, 0x033,      0x00000000,
+	0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x03E,      0x00005930,
+	0x9300100f, 0x05050505, 0x40000000, 0x00000000, 0x03E,      0x00005930,
+	0x9300100f, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00005930,
+	0x9300200f, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00005930,
+	0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x03E,      0x00005930,
+	0x9000100f, 0x05050505, 0x40000000, 0x00000000, 0x03E,      0x00005948,
+	0xA0000000, 0x00000000, 0x03E,      0x00005930, 0xB0000000, 0x00000000,
+	0x03F,      0x000C3186, 0x033,      0x0000000F, 0x8300100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x9300100f, 0x05050505,
+	0x40000000, 0x00000000, 0x03E,      0x00004080, 0x9300100f, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x9300200f, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x9000100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x9000100f, 0x05050505,
+	0x40000000, 0x00000000, 0x03E,      0x00004080, 0x9000100f, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x9000200f, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x9300200c, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x93012100, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004000, 0x93002100, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004000, 0x93011000, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004000, 0x9000200c, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x90001004, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0xA0000000, 0x00000000,
+	0x03E,      0x00004000, 0xB0000000, 0x00000000, 0x8300100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9300100f, 0x05050505,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9300100f, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9300200f, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9000100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9000100f, 0x05050505,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9000100f, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9000200f, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9300200c, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x93012100, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000DFF86, 0x93002100, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000DFF86, 0x93011000, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9000200c, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x90001004, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x93002000, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C0006, 0x93001000, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0xA0000000, 0x00000000,
+	0x03F,      0x000C3186, 0xB0000000, 0x00000000, 0x033,      0x0000000E,
+	0x03E,      0x00004080, 0x03F,      0x000C3186, 0x033,      0x0000000D,
+	0x03E,      0x000040C8, 0x03F,      0x000C3186, 0x033,      0x0000000C,
+	0x03E,      0x00004190, 0x03F,      0x000C3186, 0x033,      0x0000000B,
+	0x03E,      0x00004998, 0x03F,      0x000C3186, 0x033,      0x0000000A,
+	0x03E,      0x00005840, 0x03F,      0x000C3186, 0x033,      0x00000009,
+	0x03E,      0x000058C2, 0x03F,      0x000C3186, 0x033,      0x00000008,
+	0x03E,      0x00005930, 0x03F,      0x000C3186, 0x033,      0x00000017,
+	0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x03E,      0x00004040,
+	0x9300100f, 0x05050505, 0x40000000, 0x00000000, 0x03E,      0x00004080,
+	0x9300100f, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00004040,
+	0x9300200f, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00004040,
+	0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x03E,      0x00004040,
+	0x9000100f, 0x05050505, 0x40000000, 0x00000000, 0x03E,      0x00004080,
+	0x9000100f, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00004040,
+	0x9000200f, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00004040,
+	0x9300200c, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00004040,
+	0x93012100, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00004000,
+	0x93002100, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00004000,
+	0x93011000, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00004000,
+	0x9000200c, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00004040,
+	0x90001004, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00004040,
+	0xA0000000, 0x00000000, 0x03E,      0x00004000, 0xB0000000, 0x00000000,
+	0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x9300100f, 0x05050505, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x9300100f, 0x00000000, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x9300200f, 0x00000000, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x9000100f, 0x05050505, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x9000100f, 0x00000000, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x9000200f, 0x00000000, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x9300200c, 0x00000000, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x93012100, 0x00000000, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x93002100, 0x00000000, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x93011000, 0x00000000, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x9000200c, 0x00000000, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x90001004, 0x00000000, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x93002000, 0x00000000, 0x40000000, 0x00000000, 0x03F,      0x000C0006,
+	0x93001000, 0x00000000, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0xA0000000, 0x00000000, 0x03F,      0x000C3186, 0xB0000000, 0x00000000,
+	0x033,      0x00000016, 0x03E,      0x00004080, 0x03F,      0x000C3186,
+	0x033,      0x00000015, 0x03E,      0x000040C8, 0x03F,      0x000C3186,
+	0x033,      0x00000014, 0x03E,      0x00004190, 0x03F,      0x000C3186,
+	0x033,      0x00000013, 0x03E,      0x00004998, 0x03F,      0x000C3186,
+	0x033,      0x00000012, 0x03E,      0x00005840, 0x03F,      0x000C3186,
+	0x033,      0x00000011, 0x03E,      0x000058C2, 0x03F,      0x000C3186,
+	0x033,      0x00000010, 0x03E,      0x00005930, 0x03F,      0x000C3186,
+	0x0EF,      0x00000000, 0x0EF,      0x00004000, 0x033,      0x00000000,
+	0x03F,      0x0000000A, 0x033,      0x00000001, 0x8300100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x9300100f, 0x05050505,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x9300100f, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x9300200f, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x9000100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x9000100f, 0x05050505,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x9000100f, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x9000200f, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x9300200c, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x93012100, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000005, 0x93002100, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000006, 0x93011000, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000005, 0x9000200c, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x90001004, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x93002000, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x93001000, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000005, 0x90002100, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x90002000, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0xA0000000, 0x00000000,
+	0x03F,      0x00000005, 0xB0000000, 0x00000000, 0x033,      0x00000002,
+	0x03F,      0x00000000, 0x0EF,      0x00000000, 0x018,      0x00000401,
+	0x084,      0x00001209, 0x086,      0x000001A0, 0x8300100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x087,      0x00068080, 0x9300100f, 0x05050505,
+	0x40000000, 0x00000000, 0x087,      0x00068080, 0x9300100f, 0x00000000,
+	0x40000000, 0x00000000, 0x087,      0x00068080, 0x9300200f, 0x00000000,
+	0x40000000, 0x00000000, 0x087,      0x00068080, 0x9000100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x087,      0x00068080, 0x9000100f, 0x05050505,
+	0x40000000, 0x00000000, 0x087,      0x00068080, 0x9000100f, 0x00000000,
+	0x40000000, 0x00000000, 0x087,      0x00068080, 0x9000200f, 0x00000000,
+	0x40000000, 0x00000000, 0x087,      0x00068080, 0xA0000000, 0x00000000,
+	0x087,      0x000E8180, 0xB0000000, 0x00000000, 0x088,      0x00070020,
+	0x0DE,      0x00000010, 0x0EF,      0x00008000, 0x033,      0x0000000F,
+	0x03F,      0x0000003C, 0x033,      0x0000000E, 0x03F,      0x00000038,
+	0x033,      0x0000000D, 0x03F,      0x00000030, 0x033,      0x0000000C,
+	0x03F,      0x00000028, 0x033,      0x0000000B, 0x03F,      0x00000020,
+	0x033,      0x0000000A, 0x03F,      0x00000018, 0x033,      0x00000009,
+	0x03F,      0x00000010, 0x033,      0x00000008, 0x03F,      0x00000008,
+	0x033,      0x00000007, 0x03F,      0x0000003C, 0x033,      0x00000006,
+	0x03F,      0x00000038, 0x033,      0x00000005, 0x03F,      0x00000030,
+	0x033,      0x00000004, 0x03F,      0x00000028, 0x033,      0x00000003,
+	0x03F,      0x00000020, 0x033,      0x00000002, 0x03F,      0x00000018,
+	0x033,      0x00000001, 0x03F,      0x00000010, 0x033,      0x00000000,
+	0x03F,      0x00000008, 0x0EF,      0x00000000, 0x0B8,      0x00080A00,
+	0x0B0,      0x000FF0FA, 0x0FE,      0x00000000, 0x0CA,      0x00080000,
+	0x0C9,      0x0001C141, 0x0FE,      0x00000000, 0x0B0,      0x000FF0F8,
+	0x018,      0x00018D24, 0xFFE,      0x00000000, 0xFFE,      0x00000000,
+	0xFFE,      0x00000000, 0xFFE,      0x00000000, 0x018,      0x00010D24,
+	0x01B,      0x00075A40, 0x0EE,      0x00000002, 0x033,      0x00000000,
+	0x03F,      0x00000004, 0x033,      0x00000001, 0x03F,      0x00000004,
+	0x033,      0x00000002, 0x03F,      0x00000004, 0x033,      0x00000003,
+	0x03F,      0x00000004, 0x033,      0x00000004, 0x03F,      0x00000004,
+	0x033,      0x00000005, 0x03F,      0x00000006, 0x033,      0x00000006,
+	0x03F,      0x00000002, 0x033,      0x00000007, 0x03F,      0x00000000,
+	0x0EE,      0x00000000, 0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x061,      0x0005D4A0, 0x062,      0x0000D203, 0x063,      0x00000062,
+	0x9300100f, 0x05050505, 0x40000000, 0x00000000, 0x061,      0x0005D4A0,
+	0x062,      0x0000D203, 0x063,      0x00000062, 0x9300100f, 0x00000000,
+	0x40000000, 0x00000000, 0x061,      0x0005D4A0, 0x062,      0x0000D203,
+	0x063,      0x00000062, 0x9300200f, 0x00000000, 0x40000000, 0x00000000,
+	0x061,      0x0005D2A1, 0x062,      0x0000D3A2, 0x063,      0x00000062,
+	0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x061,      0x0005D4A0,
+	0x062,      0x0000D203, 0x063,      0x00000062, 0x9000100f, 0x05050505,
+	0x40000000, 0x00000000, 0x061,      0x0005D4A0, 0x062,      0x0000D203,
+	0x063,      0x00000062, 0x9000100f, 0x00000000, 0x40000000, 0x00000000,
+	0x061,      0x0005D4A0, 0x062,      0x0000D203, 0x063,      0x00000062,
+	0x9000200f, 0x00000000, 0x40000000, 0x00000000, 0x061,      0x0005D2A1,
+	0x062,      0x0000D3A2, 0x063,      0x00000062, 0x9300200c, 0x00000000,
+	0x40000000, 0x00000000, 0x061,      0x0005D2A1, 0x062,      0x0000D3A2,
+	0x063,      0x00000062, 0x93012100, 0x00000000, 0x40000000, 0x00000000,
+	0x061,      0x0005D301, 0x062,      0x0000D303, 0x063,      0x00000002,
+	0x93002100, 0x00000000, 0x40000000, 0x00000000, 0x061,      0x0005D301,
+	0x062,      0x0000D303, 0x063,      0x00000002, 0x93011000, 0x00000000,
+	0x40000000, 0x00000000, 0x061,      0x0005D3D1, 0x062,      0x0000D3A2,
+	0x063,      0x00000002, 0x9000200c, 0x00000000, 0x40000000, 0x00000000,
+	0x061,      0x0005D2A1, 0x062,      0x0000D3A2, 0x063,      0x00000062,
+	0x90001004, 0x00000000, 0x40000000, 0x00000000, 0x061,      0x0005D3D1,
+	0x062,      0x0000D3A2, 0x063,      0x00000002, 0x93002000, 0x00000000,
+	0x40000000, 0x00000000, 0x061,      0x0005D301, 0x062,      0x0000D303,
+	0x063,      0x00000002, 0x93001000, 0x00000000, 0x40000000, 0x00000000,
+	0x061,      0x0005D3D1, 0x062,      0x0000D3A2, 0x063,      0x00000002,
+	0x90002100, 0x00000000, 0x40000000, 0x00000000, 0x061,      0x0005D301,
+	0x062,      0x0000D303, 0x063,      0x00000002, 0x90002000, 0x00000000,
+	0x40000000, 0x00000000, 0x061,      0x0005D301, 0x062,      0x0000D303,
+	0x063,      0x00000002, 0xA0000000, 0x00000000, 0x061,      0x0005D3D0,
+	0x062,      0x0000D303, 0x063,      0x00000002, 0xB0000000, 0x00000000,
+	0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x0EF,      0x00000200,
+	0x030,      0x000004A3, 0x030,      0x000014A3, 0x030,      0x000024A3,
+	0x030,      0x000034A3, 0x030,      0x000044A3, 0x030,      0x000054A3,
+	0x030,      0x000064A3, 0x030,      0x000074A3, 0x030,      0x000084A3,
+	0x030,      0x000094A3, 0x030,      0x0000A4A3, 0x030,      0x0000B4A3,
+	0x0EF,      0x00000000, 0x9300100f, 0x05050505, 0x40000000, 0x00000000,
+	0x0EF,      0x00000200, 0x030,      0x000004A3, 0x030,      0x000014A3,
+	0x030,      0x000024A3, 0x030,      0x000034A3, 0x030,      0x000044A3,
+	0x030,      0x000054A3, 0x030,      0x000064A3, 0x030,      0x000074A3,
+	0x030,      0x000084A3, 0x030,      0x000094A3, 0x030,      0x0000A4A3,
+	0x030,      0x0000B4A3, 0x0EF,      0x00000000, 0x9300100f, 0x00000000,
+	0x40000000, 0x00000000, 0x0EF,      0x00000200, 0x030,      0x000004A3,
+	0x030,      0x000014A3, 0x030,      0x000024A3, 0x030,      0x000034A3,
+	0x030,      0x000044A3, 0x030,      0x000054A3, 0x030,      0x000064A3,
+	0x030,      0x000074A3, 0x030,      0x000084A3, 0x030,      0x000094A3,
+	0x030,      0x0000A4A3, 0x030,      0x0000B4A3, 0x0EF,      0x00000000,
+	0x9300200f, 0x00000000, 0x40000000, 0x00000000, 0x0EF,      0x00000200,
+	0x030,      0x000002A6, 0x030,      0x000012A6, 0x030,      0x000022A6,
+	0x030,      0x000032A6, 0x030,      0x000042A6, 0x030,      0x000052A6,
+	0x030,      0x000062A6, 0x030,      0x000072A6, 0x030,      0x000082A6,
+	0x030,      0x000092A6, 0x030,      0x0000A2A6, 0x030,      0x0000B2A6,
+	0x0EF,      0x00000000, 0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x0EF,      0x00000200, 0x030,      0x000004A0, 0x030,      0x000014A0,
+	0x030,      0x000024A0, 0x030,      0x000034A0, 0x030,      0x000044A0,
+	0x030,      0x000054A0, 0x030,      0x000064A0, 0x030,      0x000074A0,
+	0x030,      0x000084A0, 0x030,      0x000094A0, 0x030,      0x0000A4A0,
+	0x030,      0x0000B4A0, 0x0EF,      0x00000000, 0x9000100f, 0x05050505,
+	0x40000000, 0x00000000, 0x0EF,      0x00000200, 0x030,      0x000004A0,
+	0x030,      0x000014A0, 0x030,      0x000024A0, 0x030,      0x000034A0,
+	0x030,      0x000044A0, 0x030,      0x000054A0, 0x030,      0x000064A0,
+	0x030,      0x000074A0, 0x030,      0x000084A0, 0x030,      0x000094A0,
+	0x030,      0x0000A4A0, 0x030,      0x0000B4A0, 0x0EF,      0x00000000,
+	0x9000100f, 0x00000000, 0x40000000, 0x00000000, 0x0EF,      0x00000200,
+	0x030,      0x000004A0, 0x030,      0x000014A0, 0x030,      0x000024A0,
+	0x030,      0x000034A0, 0x030,      0x000044A0, 0x030,      0x000054A0,
+	0x030,      0x000064A0, 0x030,      0x000074A0, 0x030,      0x000084A0,
+	0x030,      0x000094A0, 0x030,      0x0000A4A0, 0x030,      0x0000B4A0,
+	0x0EF,      0x00000000, 0x9000200f, 0x00000000, 0x40000000, 0x00000000,
+	0x0EF,      0x00000200, 0x030,      0x000002A1, 0x030,      0x000012A1,
+	0x030,      0x000022A1, 0x030,      0x000032A1, 0x030,      0x000042A1,
+	0x030,      0x000052A1, 0x030,      0x000062A1, 0x030,      0x000072A1,
+	0x030,      0x000082A1, 0x030,      0x000092A1, 0x030,      0x0000A2A1,
+	0x030,      0x0000B2A1, 0x0EF,      0x00000000, 0x9300200c, 0x00000000,
+	0x40000000, 0x00000000, 0x0EF,      0x00000200, 0x030,      0x000002A6,
+	0x030,      0x000012A6, 0x030,      0x000022A6, 0x030,      0x000032A6,
+	0x030,      0x000042A6, 0x030,      0x000052A6, 0x030,      0x000062A6,
+	0x030,      0x000072A6, 0x030,      0x000082A6, 0x030,      0x000092A6,
+	0x030,      0x0000A2A6, 0x030,      0x0000B2A6, 0x0EF,      0x00000000,
+	0x93012100, 0x00000000, 0x40000000, 0x00000000, 0x0EF,      0x00000200,
+	0x030,      0x00000384, 0x030,      0x00001384, 0x030,      0x00002384,
+	0x030,      0x00003384, 0x030,      0x00004425, 0x030,      0x00005425,
+	0x030,      0x00006425, 0x030,      0x00007425, 0x030,      0x000083A4,
+	0x030,      0x000093A4, 0x030,      0x0000A3A4, 0x030,      0x0000B3A4,
+	0x0EF,      0x00000000, 0x93002100, 0x00000000, 0x40000000, 0x00000000,
+	0x0EF,      0x00000200, 0x030,      0x000003A3, 0x030,      0x000013A3,
+	0x030,      0x000023A3, 0x030,      0x000033A3, 0x030,      0x00004355,
+	0x030,      0x00005355, 0x030,      0x00006355, 0x030,      0x00007355,
+	0x030,      0x00008314, 0x030,      0x00009314, 0x030,      0x0000A314,
+	0x030,      0x0000B314, 0x0EF,      0x00000000, 0x93011000, 0x00000000,
+	0x40000000, 0x00000000, 0x0EF,      0x00000200, 0x030,      0x000003A1,
+	0x030,      0x000013A1, 0x030,      0x000023A1, 0x030,      0x000033A1,
+	0x030,      0x000043A3, 0x030,      0x000053A3, 0x030,      0x000063A3,
+	0x030,      0x000073A3, 0x030,      0x000083A5, 0x030,      0x000093A5,
+	0x030,      0x0000A3A5, 0x030,      0x0000B3A5, 0x0EF,      0x00000000,
+	0x9000200c, 0x00000000, 0x40000000, 0x00000000, 0x0EF,      0x00000200,
+	0x030,      0x000002A1, 0x030,      0x000012A1, 0x030,      0x000022A1,
+	0x030,      0x000032A1, 0x030,      0x000042A1, 0x030,      0x000052A1,
+	0x030,      0x000062A1, 0x030,      0x000072A1, 0x030,      0x000082A1,
+	0x030,      0x000092A1, 0x030,      0x0000A2A1, 0x030,      0x0000B2A1,
+	0x0EF,      0x00000000, 0x90001004, 0x00000000, 0x40000000, 0x00000000,
+	0x0EF,      0x00000200, 0x030,      0x00000463, 0x030,      0x00001463,
+	0x030,      0x00002463, 0x030,      0x00003463, 0x030,      0x00004545,
+	0x030,      0x00005545, 0x030,      0x00006545, 0x030,      0x00007545,
+	0x030,      0x00008565, 0x030,      0x00009565, 0x030,      0x0000A565,
+	0x030,      0x0000B565, 0x0EF,      0x00000000, 0x93002000, 0x00000000,
+	0x40000000, 0x00000000, 0x0EF,      0x00000200, 0x030,      0x00000303,
+	0x030,      0x00001303, 0x030,      0x00002303, 0x030,      0x00003303,
+	0x030,      0x000043A4, 0x030,      0x000053A4, 0x030,      0x000063A4,
+	0x030,      0x000073A4, 0x030,      0x00008365, 0x030,      0x00009365,
+	0x030,      0x0000A365, 0x030,      0x0000B365, 0x0EF,      0x00000000,
+	0x93001000, 0x00000000, 0x40000000, 0x00000000, 0x0EF,      0x00000200,
+	0x030,      0x000003A2, 0x030,      0x000013A2, 0x030,      0x000023A2,
+	0x030,      0x000033A2, 0x030,      0x00004343, 0x030,      0x00005343,
+	0x030,      0x00006343, 0x030,      0x00007343, 0x030,      0x00008364,
+	0x030,      0x00009364, 0x030,      0x0000A364, 0x030,      0x0000B364,
+	0x0EF,      0x00000000, 0x90002100, 0x00000000, 0x40000000, 0x00000000,
+	0x0EF,      0x00000200, 0x030,      0x000003A0, 0x030,      0x000013A0,
+	0x030,      0x000023A0, 0x030,      0x000033A0, 0x030,      0x00004430,
+	0x030,      0x00005430, 0x030,      0x00006430, 0x030,      0x00007430,
+	0x030,      0x00008372, 0x030,      0x00009372, 0x030,      0x0000A372,
+	0x030,      0x0000B372, 0x0EF,      0x00000000, 0x90002000, 0x00000000,
+	0x40000000, 0x00000000, 0x0EF,      0x00000200, 0x030,      0x000003A0,
+	0x030,      0x000013A0, 0x030,      0x000023A0, 0x030,      0x000033A0,
+	0x030,      0x000043A1, 0x030,      0x000053A1, 0x030,      0x000063A1,
+	0x030,      0x000073A1, 0x030,      0x000083A2, 0x030,      0x000093A2,
+	0x030,      0x0000A3A2, 0x030,      0x0000B3A2, 0x0EF,      0x00000000,
+	0xA0000000, 0x00000000, 0x0EF,      0x00000200, 0x030,      0x000003D0,
+	0x030,      0x000013D0, 0x030,      0x000023D0, 0x030,      0x000033D0,
+	0x030,      0x000043D0, 0x030,      0x000053D0, 0x030,      0x000063D0,
+	0x030,      0x000073D0, 0x030,      0x000083D0, 0x030,      0x000093D0,
+	0x030,      0x0000A3D0, 0x030,      0x0000B3D0, 0x0EF,      0x00000000,
+	0xB0000000, 0x00000000, 0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x0EF,      0x00000080, 0x030,      0x00000203, 0x030,      0x00001203,
+	0x030,      0x00002203, 0x030,      0x00003203, 0x030,      0x00004203,
+	0x030,      0x00005203, 0x030,      0x00006203, 0x030,      0x00007203,
+	0x030,      0x00008203, 0x030,      0x00009203, 0x030,      0x0000A203,
+	0x030,      0x0000B203, 0x9300100f, 0x05050505, 0x40000000, 0x00000000,
+	0x0EF,      0x00000080, 0x030,      0x00000203, 0x030,      0x00001203,
+	0x030,      0x00002203, 0x030,      0x00003203, 0x030,      0x00004203,
+	0x030,      0x00005203, 0x030,      0x00006203, 0x030,      0x00007203,
+	0x030,      0x00008203, 0x030,      0x00009203, 0x030,      0x0000A203,
+	0x030,      0x0000B203, 0x9300100f, 0x00000000, 0x40000000, 0x00000000,
+	0x0EF,      0x00000080, 0x030,      0x00000203, 0x030,      0x00001203,
+	0x030,      0x00002203, 0x030,      0x00003203, 0x030,      0x00004203,
+	0x030,      0x00005203, 0x030,      0x00006203, 0x030,      0x00007203,
+	0x030,      0x00008203, 0x030,      0x00009203, 0x030,      0x0000A203,
+	0x030,      0x0000B203, 0x9300200f, 0x00000000, 0x40000000, 0x00000000,
+	0x0EF,      0x00000080, 0x030,      0x000003A2, 0x030,      0x000013A2,
+	0x030,      0x000023A2, 0x030,      0x000033A2, 0x030,      0x000043A2,
+	0x030,      0x000053A2, 0x030,      0x000063A2, 0x030,      0x000073A2,
+	0x030,      0x000083A2, 0x030,      0x000093A2, 0x030,      0x0000A3A2,
+	0x030,      0x0000B3A2, 0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x0EF,      0x00000080, 0x030,      0x00000203, 0x030,      0x00001203,
+	0x030,      0x00002203, 0x030,      0x00003203, 0x030,      0x00004203,
+	0x030,      0x00005203, 0x030,      0x00006203, 0x030,      0x00007203,
+	0x030,      0x00008203, 0x030,      0x00009203, 0x030,      0x0000A203,
+	0x030,      0x0000B203, 0x9000100f, 0x05050505, 0x40000000, 0x00000000,
+	0x0EF,      0x00000080, 0x030,      0x00000203, 0x030,      0x00001203,
+	0x030,      0x00002203, 0x030,      0x00003203, 0x030,      0x00004203,
+	0x030,      0x00005203, 0x030,      0x00006203, 0x030,      0x00007203,
+	0x030,      0x00008203, 0x030,      0x00009203, 0x030,      0x0000A203,
+	0x030,      0x0000B203, 0x9000100f, 0x00000000, 0x40000000, 0x00000000,
+	0x0EF,      0x00000080, 0x030,      0x00000203, 0x030,      0x00001203,
+	0x030,      0x00002203, 0x030,      0x00003203, 0x030,      0x00004203,
+	0x030,      0x00005203, 0x030,      0x00006203, 0x030,      0x00007203,
+	0x030,      0x00008203, 0x030,      0x00009203, 0x030,      0x0000A203,
+	0x030,      0x0000B203, 0x9000200f, 0x00000000, 0x40000000, 0x00000000,
+	0x0EF,      0x00000080, 0x030,      0x000003A2, 0x030,      0x000013A2,
+	0x030,      0x000023A2, 0x030,      0x000033A2, 0x030,      0x000043A2,
+	0x030,      0x000053A2, 0x030,      0x000063A2, 0x030,      0x000073A2,
+	0x030,      0x000083A2, 0x030,      0x000093A2, 0x030,      0x0000A3A2,
+	0x030,      0x0000B3A2, 0x9300200c, 0x00000000, 0x40000000, 0x00000000,
+	0x0EF,      0x00000080, 0x030,      0x000003A2, 0x030,      0x000013A2,
+	0x030,      0x000023A2, 0x030,      0x000033A2, 0x030,      0x000043A2,
+	0x030,      0x000053A2, 0x030,      0x000063A2, 0x030,      0x000073A2,
+	0x030,      0x000083A2, 0x030,      0x000093A2, 0x030,      0x0000A3A2,
+	0x030,      0x0000B3A2, 0x93012100, 0x00000000, 0x40000000, 0x00000000,
+	0x0EF,      0x00000080, 0x030,      0x000003A3, 0x030,      0x000013A3,
+	0x030,      0x000023A3, 0x030,      0x000033A3, 0x030,      0x000043A3,
+	0x030,      0x000053A3, 0x030,      0x000063A3, 0x030,      0x000073A3,
+	0x030,      0x000083A3, 0x030,      0x000093A3, 0x030,      0x0000A3A3,
+	0x030,      0x0000B3A3, 0x93002100, 0x00000000, 0x40000000, 0x00000000,
+	0x0EF,      0x00000080, 0x030,      0x000003A2, 0x030,      0x000013A2,
+	0x030,      0x000023A2, 0x030,      0x000033A2, 0x030,      0x000043A2,
+	0x030,      0x000053A2, 0x030,      0x000063A2, 0x030,      0x000073A2,
+	0x030,      0x000083A2, 0x030,      0x000093A2, 0x030,      0x0000A3A2,
+	0x030,      0x0000B3A2, 0x93011000, 0x00000000, 0x40000000, 0x00000000,
+	0x0EF,      0x00000080, 0x030,      0x000003A2, 0x030,      0x000013A2,
+	0x030,      0x000023A2, 0x030,      0x000033A2, 0x030,      0x000043A2,
+	0x030,      0x000053A2, 0x030,      0x000063A2, 0x030,      0x000073A2,
+	0x030,      0x000083A2, 0x030,      0x000093A2, 0x030,      0x0000A3A2,
+	0x030,      0x0000B3A2, 0x9000200c, 0x00000000, 0x40000000, 0x00000000,
+	0x0EF,      0x00000080, 0x030,      0x000003A2, 0x030,      0x000013A2,
+	0x030,      0x000023A2, 0x030,      0x000033A2, 0x030,      0x000043A2,
+	0x030,      0x000053A2, 0x030,      0x000063A2, 0x030,      0x000073A2,
+	0x030,      0x000083A2, 0x030,      0x000093A2, 0x030,      0x0000A3A2,
+	0x030,      0x0000B3A2, 0x90001004, 0x00000000, 0x40000000, 0x00000000,
+	0x0EF,      0x00000080, 0x030,      0x000003A2, 0x030,      0x000013A2,
+	0x030,      0x000023A2, 0x030,      0x000033A2, 0x030,      0x000043A2,
+	0x030,      0x000053A2, 0x030,      0x000063A2, 0x030,      0x000073A2,
+	0x030,      0x000083A2, 0x030,      0x000093A2, 0x030,      0x0000A3A2,
+	0x030,      0x0000B3A2, 0x93002000, 0x00000000, 0x40000000, 0x00000000,
+	0x0EF,      0x00000080, 0x030,      0x000003A2, 0x030,      0x000013A2,
+	0x030,      0x000023A2, 0x030,      0x000033A2, 0x030,      0x000043A2,
+	0x030,      0x000053A2, 0x030,      0x000063A2, 0x030,      0x000073A2,
+	0x030,      0x000083A2, 0x030,      0x000093A2, 0x030,      0x0000A3A2,
+	0x030,      0x0000B3A2, 0x93001000, 0x00000000, 0x40000000, 0x00000000,
+	0x0EF,      0x00000080, 0x030,      0x000003A2, 0x030,      0x000013A2,
+	0x030,      0x000023A2, 0x030,      0x000033A2, 0x030,      0x000043A2,
+	0x030,      0x000053A2, 0x030,      0x000063A2, 0x030,      0x000073A2,
+	0x030,      0x000083A2, 0x030,      0x000093A2, 0x030,      0x0000A3A2,
+	0x030,      0x0000B3A2, 0x90002100, 0x00000000, 0x40000000, 0x00000000,
+	0x0EF,      0x00000080, 0x030,      0x000003A2, 0x030,      0x000013A2,
+	0x030,      0x000023A2, 0x030,      0x000033A2, 0x030,      0x000043A2,
+	0x030,      0x000053A2, 0x030,      0x000063A2, 0x030,      0x000073A2,
+	0x030,      0x000083A2, 0x030,      0x000093A2, 0x030,      0x0000A3A2,
+	0x030,      0x0000B3A2, 0x90002000, 0x00000000, 0x40000000, 0x00000000,
+	0x0EF,      0x00000080, 0x030,      0x000003A2, 0x030,      0x000013A2,
+	0x030,      0x000023A2, 0x030,      0x000033A2, 0x030,      0x000043A2,
+	0x030,      0x000053A2, 0x030,      0x000063A2, 0x030,      0x000073A2,
+	0x030,      0x000083A2, 0x030,      0x000093A2, 0x030,      0x0000A3A2,
+	0x030,      0x0000B3A2, 0xA0000000, 0x00000000, 0x0EF,      0x00000080,
+	0x030,      0x000003A2, 0x030,      0x000013A2, 0x030,      0x000023A2,
+	0x030,      0x000033A2, 0x030,      0x000043A2, 0x030,      0x000053A2,
+	0x030,      0x000063A2, 0x030,      0x000073A2, 0x030,      0x000083A2,
+	0x030,      0x000093A2, 0x030,      0x0000A3A2, 0x030,      0x0000B3A2,
+	0xB0000000, 0x00000000, 0x0EF,      0x00000000, 0x8300100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x0EF,      0x00000040, 0x030,      0x00000645,
+	0x030,      0x00001333, 0x030,      0x00002011, 0x030,      0x00004000,
+	0x030,      0x00005000, 0x030,      0x00006000, 0x9300100f, 0x05050505,
+	0x40000000, 0x00000000, 0x0EF,      0x00000040, 0x030,      0x00000645,
+	0x030,      0x00001333, 0x030,      0x00002011, 0x030,      0x00004000,
+	0x030,      0x00005000, 0x030,      0x00006000, 0x9300100f, 0x00000000,
+	0x40000000, 0x00000000, 0x0EF,      0x00000040, 0x030,      0x00000645,
+	0x030,      0x00001333, 0x030,      0x00002011, 0x030,      0x00004000,
+	0x030,      0x00005000, 0x030,      0x00006000, 0x9300200f, 0x00000000,
+	0x40000000, 0x00000000, 0x0EF,      0x00000040, 0x030,      0x00000645,
+	0x030,      0x00001333, 0x030,      0x00002011, 0x030,      0x00004777,
+	0x030,      0x00005777, 0x030,      0x00006777, 0x9000100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x0EF,      0x00000040, 0x030,      0x00000645,
+	0x030,      0x00001333, 0x030,      0x00002011, 0x030,      0x00004000,
+	0x030,      0x00005000, 0x030,      0x00006000, 0x9000100f, 0x05050505,
+	0x40000000, 0x00000000, 0x0EF,      0x00000040, 0x030,      0x00000645,
+	0x030,      0x00001333, 0x030,      0x00002011, 0x030,      0x00004000,
+	0x030,      0x00005000, 0x030,      0x00006000, 0x9000100f, 0x00000000,
+	0x40000000, 0x00000000, 0x0EF,      0x00000040, 0x030,      0x00000645,
+	0x030,      0x00001333, 0x030,      0x00002011, 0x030,      0x00004000,
+	0x030,      0x00005000, 0x030,      0x00006000, 0x9000200f, 0x00000000,
+	0x40000000, 0x00000000, 0x0EF,      0x00000040, 0x030,      0x00000645,
+	0x030,      0x00001333, 0x030,      0x00002011, 0x030,      0x00004000,
+	0x030,      0x00005000, 0x030,      0x00006000, 0x9300200c, 0x00000000,
+	0x40000000, 0x00000000, 0x0EF,      0x00000040, 0x030,      0x00000645,
+	0x030,      0x00001333, 0x030,      0x00002011, 0x030,      0x00004777,
+	0x030,      0x00005777, 0x030,      0x00006777, 0x93012100, 0x00000000,
+	0x40000000, 0x00000000, 0x0EF,      0x00000040, 0x030,      0x00000660,
+	0x030,      0x00001443, 0x030,      0x00002221, 0x030,      0x00004777,
+	0x030,      0x00005777, 0x030,      0x00006777, 0x93002100, 0x00000000,
+	0x40000000, 0x00000000, 0x0EF,      0x00000040, 0x030,      0x00000776,
+	0x030,      0x00001455, 0x030,      0x00002325, 0x030,      0x00004777,
+	0x030,      0x00005777, 0x030,      0x00006777, 0x93011000, 0x00000000,
+	0x40000000, 0x00000000, 0x0EF,      0x00000040, 0x030,      0x00000764,
+	0x030,      0x00001632, 0x030,      0x00002421, 0x030,      0x00004000,
+	0x030,      0x00005000, 0x030,      0x00006000, 0x9000200c, 0x00000000,
+	0x40000000, 0x00000000, 0x0EF,      0x00000040, 0x030,      0x00000645,
+	0x030,      0x00001333, 0x030,      0x00002011, 0x030,      0x00004000,
+	0x030,      0x00005000, 0x030,      0x00006000, 0x90001004, 0x00000000,
+	0x40000000, 0x00000000, 0x0EF,      0x00000040, 0x030,      0x00000764,
+	0x030,      0x00001632, 0x030,      0x00002421, 0x030,      0x00004000,
+	0x030,      0x00005000, 0x030,      0x00006000, 0x93002000, 0x00000000,
+	0x40000000, 0x00000000, 0x0EF,      0x00000040, 0x030,      0x00000777,
+	0x030,      0x00001442, 0x030,      0x00002222, 0x030,      0x00004777,
+	0x030,      0x00005777, 0x030,      0x00006777, 0x93001000, 0x00000000,
+	0x40000000, 0x00000000, 0x0EF,      0x00000040, 0x030,      0x00000764,
+	0x030,      0x00001632, 0x030,      0x00002421, 0x030,      0x00004000,
+	0x030,      0x00005000, 0x030,      0x00006000, 0x90002100, 0x00000000,
+	0x40000000, 0x00000000, 0x0EF,      0x00000040, 0x030,      0x00000775,
+	0x030,      0x00001343, 0x030,      0x00002210, 0x030,      0x00004000,
+	0x030,      0x00005000, 0x030,      0x00006000, 0x90002000, 0x00000000,
+	0x40000000, 0x00000000, 0x0EF,      0x00000040, 0x030,      0x00000775,
+	0x030,      0x00001422, 0x030,      0x00002210, 0x030,      0x00004000,
+	0x030,      0x00005000, 0x030,      0x00006000, 0xA0000000, 0x00000000,
+	0x0EF,      0x00000040, 0x030,      0x00000764, 0x030,      0x00001632,
+	0x030,      0x00002421, 0x030,      0x00004000, 0x030,      0x00005000,
+	0x030,      0x00006000, 0xB0000000, 0x00000000, 0x0EF,      0x00000000,
+	0x0EF,      0x00000800, 0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x033,      0x00000020, 0x03F,      0x00000007, 0x033,      0x00000021,
+	0x03F,      0x0000000A, 0x033,      0x00000022, 0x03F,      0x0000000D,
+	0x033,      0x00000023, 0x03F,      0x0000002A, 0x033,      0x00000024,
+	0x03F,      0x0000002D, 0x033,      0x00000025, 0x03F,      0x00000030,
+	0x033,      0x00000026, 0x03F,      0x0000006D, 0x033,      0x00000027,
+	0x03F,      0x00000070, 0x033,      0x00000028, 0x03F,      0x000000ED,
+	0x033,      0x00000029, 0x03F,      0x000000F0, 0x033,      0x0000002A,
+	0x03F,      0x000000F3, 0x9300100f, 0x05050505, 0x40000000, 0x00000000,
+	0x033,      0x00000020, 0x03F,      0x00000007, 0x033,      0x00000021,
+	0x03F,      0x0000000A, 0x033,      0x00000022, 0x03F,      0x0000000D,
+	0x033,      0x00000023, 0x03F,      0x0000002A, 0x033,      0x00000024,
+	0x03F,      0x0000002D, 0x033,      0x00000025, 0x03F,      0x00000030,
+	0x033,      0x00000026, 0x03F,      0x0000006D, 0x033,      0x00000027,
+	0x03F,      0x00000070, 0x033,      0x00000028, 0x03F,      0x000000ED,
+	0x033,      0x00000029, 0x03F,      0x000000F0, 0x033,      0x0000002A,
+	0x03F,      0x000000F3, 0x9300100f, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000020, 0x03F,      0x00000007, 0x033,      0x00000021,
+	0x03F,      0x0000000A, 0x033,      0x00000022, 0x03F,      0x0000000D,
+	0x033,      0x00000023, 0x03F,      0x0000002A, 0x033,      0x00000024,
+	0x03F,      0x0000002D, 0x033,      0x00000025, 0x03F,      0x00000030,
+	0x033,      0x00000026, 0x03F,      0x0000006D, 0x033,      0x00000027,
+	0x03F,      0x00000070, 0x033,      0x00000028, 0x03F,      0x000000ED,
+	0x033,      0x00000029, 0x03F,      0x000000F0, 0x033,      0x0000002A,
+	0x03F,      0x000000F3, 0x9300200f, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000020, 0x03F,      0x00000005, 0x033,      0x00000021,
+	0x03F,      0x00000008, 0x033,      0x00000022, 0x03F,      0x0000000B,
+	0x033,      0x00000023, 0x03F,      0x0000000E, 0x033,      0x00000024,
+	0x03F,      0x0000002B, 0x033,      0x00000025, 0x03F,      0x00000068,
+	0x033,      0x00000026, 0x03F,      0x0000006B, 0x033,      0x00000027,
+	0x03F,      0x0000006E, 0x033,      0x00000028, 0x03F,      0x00000071,
+	0x033,      0x00000029, 0x03F,      0x00000074, 0x033,      0x0000002A,
+	0x03F,      0x00000077, 0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x033,      0x00000020, 0x03F,      0x00000007, 0x033,      0x00000021,
+	0x03F,      0x0000000A, 0x033,      0x00000022, 0x03F,      0x0000000D,
+	0x033,      0x00000023, 0x03F,      0x0000002A, 0x033,      0x00000024,
+	0x03F,      0x0000002D, 0x033,      0x00000025, 0x03F,      0x00000030,
+	0x033,      0x00000026, 0x03F,      0x0000006D, 0x033,      0x00000027,
+	0x03F,      0x00000070, 0x033,      0x00000028, 0x03F,      0x000000ED,
+	0x033,      0x00000029, 0x03F,      0x000000F0, 0x033,      0x0000002A,
+	0x03F,      0x000000F3, 0x9000100f, 0x05050505, 0x40000000, 0x00000000,
+	0x033,      0x00000020, 0x03F,      0x00000007, 0x033,      0x00000021,
+	0x03F,      0x0000000A, 0x033,      0x00000022, 0x03F,      0x0000000D,
+	0x033,      0x00000023, 0x03F,      0x0000002A, 0x033,      0x00000024,
+	0x03F,      0x0000002D, 0x033,      0x00000025, 0x03F,      0x00000030,
+	0x033,      0x00000026, 0x03F,      0x0000006D, 0x033,      0x00000027,
+	0x03F,      0x00000070, 0x033,      0x00000028, 0x03F,      0x000000ED,
+	0x033,      0x00000029, 0x03F,      0x000000F0, 0x033,      0x0000002A,
+	0x03F,      0x000000F3, 0x9000100f, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000020, 0x03F,      0x00000007, 0x033,      0x00000021,
+	0x03F,      0x0000000A, 0x033,      0x00000022, 0x03F,      0x0000000D,
+	0x033,      0x00000023, 0x03F,      0x0000002A, 0x033,      0x00000024,
+	0x03F,      0x0000002D, 0x033,      0x00000025, 0x03F,      0x00000030,
+	0x033,      0x00000026, 0x03F,      0x0000006D, 0x033,      0x00000027,
+	0x03F,      0x00000070, 0x033,      0x00000028, 0x03F,      0x000000ED,
+	0x033,      0x00000029, 0x03F,      0x000000F0, 0x033,      0x0000002A,
+	0x03F,      0x000000F3, 0x9000200f, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000020, 0x03F,      0x00000005, 0x033,      0x00000021,
+	0x03F,      0x00000008, 0x033,      0x00000022, 0x03F,      0x0000000B,
+	0x033,      0x00000023, 0x03F,      0x0000000E, 0x033,      0x00000024,
+	0x03F,      0x0000002B, 0x033,      0x00000025, 0x03F,      0x00000068,
+	0x033,      0x00000026, 0x03F,      0x0000006B, 0x033,      0x00000027,
+	0x03F,      0x0000006E, 0x033,      0x00000028, 0x03F,      0x00000071,
+	0x033,      0x00000029, 0x03F,      0x00000074, 0x033,      0x0000002A,
+	0x03F,      0x00000077, 0x9300200c, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000020, 0x03F,      0x00000005, 0x033,      0x00000021,
+	0x03F,      0x00000008, 0x033,      0x00000022, 0x03F,      0x0000000B,
+	0x033,      0x00000023, 0x03F,      0x0000000E, 0x033,      0x00000024,
+	0x03F,      0x0000002B, 0x033,      0x00000025, 0x03F,      0x00000068,
+	0x033,      0x00000026, 0x03F,      0x0000006B, 0x033,      0x00000027,
+	0x03F,      0x0000006E, 0x033,      0x00000028, 0x03F,      0x00000071,
+	0x033,      0x00000029, 0x03F,      0x00000074, 0x033,      0x0000002A,
+	0x03F,      0x00000077, 0x93012100, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000020, 0x03F,      0x00000C0C, 0x033,      0x00000021,
+	0x03F,      0x00000C29, 0x033,      0x00000022, 0x03F,      0x00000C2C,
+	0x033,      0x00000023, 0x03F,      0x00000C69, 0x033,      0x00000024,
+	0x03F,      0x00000CA8, 0x033,      0x00000025, 0x03F,      0x00000CE8,
+	0x033,      0x00000026, 0x03F,      0x00000CEB, 0x033,      0x00000027,
+	0x03F,      0x00000CEE, 0x033,      0x00000028, 0x03F,      0x00000CF1,
+	0x033,      0x00000029, 0x03F,      0x00000CF4, 0x033,      0x0000002A,
+	0x03F,      0x00000CF7, 0x93002100, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000020, 0x03F,      0x0000042B, 0x033,      0x00000021,
+	0x03F,      0x0000082A, 0x033,      0x00000022, 0x03F,      0x00000849,
+	0x033,      0x00000023, 0x03F,      0x0000084C, 0x033,      0x00000024,
+	0x03F,      0x00000C4C, 0x033,      0x00000025, 0x03F,      0x00000CA9,
+	0x033,      0x00000026, 0x03F,      0x00000CEA, 0x033,      0x00000027,
+	0x03F,      0x00000CED, 0x033,      0x00000028, 0x03F,      0x00000CF0,
+	0x033,      0x00000029, 0x03F,      0x00000CF3, 0x033,      0x0000002A,
+	0x03F,      0x00000CF6, 0x93011000, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000020, 0x03F,      0x00000C09, 0x033,      0x00000021,
+	0x03F,      0x00000C0C, 0x033,      0x00000022, 0x03F,      0x00000C0F,
+	0x033,      0x00000023, 0x03F,      0x00000C2C, 0x033,      0x00000024,
+	0x03F,      0x00000C2F, 0x033,      0x00000025, 0x03F,      0x00000C8A,
+	0x033,      0x00000026, 0x03F,      0x00000C8D, 0x033,      0x00000027,
+	0x03F,      0x00000C90, 0x033,      0x00000028, 0x03F,      0x00000CD0,
+	0x033,      0x00000029, 0x03F,      0x00000CF2, 0x033,      0x0000002A,
+	0x03F,      0x00000CF5, 0x9000200c, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000020, 0x03F,      0x00000005, 0x033,      0x00000021,
+	0x03F,      0x00000008, 0x033,      0x00000022, 0x03F,      0x0000000B,
+	0x033,      0x00000023, 0x03F,      0x0000000E, 0x033,      0x00000024,
+	0x03F,      0x0000002B, 0x033,      0x00000025, 0x03F,      0x00000068,
+	0x033,      0x00000026, 0x03F,      0x0000006B, 0x033,      0x00000027,
+	0x03F,      0x0000006E, 0x033,      0x00000028, 0x03F,      0x00000071,
+	0x033,      0x00000029, 0x03F,      0x00000074, 0x033,      0x0000002A,
+	0x03F,      0x00000077, 0x90001004, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000020, 0x03F,      0x00000C09, 0x033,      0x00000021,
+	0x03F,      0x00000C0C, 0x033,      0x00000022, 0x03F,      0x00000C0F,
+	0x033,      0x00000023, 0x03F,      0x00000C2C, 0x033,      0x00000024,
+	0x03F,      0x00000C2F, 0x033,      0x00000025, 0x03F,      0x00000C8A,
+	0x033,      0x00000026, 0x03F,      0x00000C8D, 0x033,      0x00000027,
+	0x03F,      0x00000C90, 0x033,      0x00000028, 0x03F,      0x00000CD0,
+	0x033,      0x00000029, 0x03F,      0x00000CF2, 0x033,      0x0000002A,
+	0x03F,      0x00000CF5, 0x93002000, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000020, 0x03F,      0x00000429, 0x033,      0x00000021,
+	0x03F,      0x00000828, 0x033,      0x00000022, 0x03F,      0x00000847,
+	0x033,      0x00000023, 0x03F,      0x0000084A, 0x033,      0x00000024,
+	0x03F,      0x00000C4B, 0x033,      0x00000025, 0x03F,      0x00000C8A,
+	0x033,      0x00000026, 0x03F,      0x00000CEA, 0x033,      0x00000027,
+	0x03F,      0x00000CED, 0x033,      0x00000028, 0x03F,      0x00000CF0,
+	0x033,      0x00000029, 0x03F,      0x00000CF3, 0x033,      0x0000002A,
+	0x03F,      0x00000CF6, 0x93001000, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000020, 0x03F,      0x00000C09, 0x033,      0x00000021,
+	0x03F,      0x00000C0C, 0x033,      0x00000022, 0x03F,      0x00000C0F,
+	0x033,      0x00000023, 0x03F,      0x00000C2C, 0x033,      0x00000024,
+	0x03F,      0x00000C2F, 0x033,      0x00000025, 0x03F,      0x00000C8A,
+	0x033,      0x00000026, 0x03F,      0x00000C8D, 0x033,      0x00000027,
+	0x03F,      0x00000C90, 0x033,      0x00000028, 0x03F,      0x00000CD0,
+	0x033,      0x00000029, 0x03F,      0x00000CF2, 0x033,      0x0000002A,
+	0x03F,      0x00000CF5, 0x90002100, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000020, 0x03F,      0x0000042B, 0x033,      0x00000021,
+	0x03F,      0x0000082A, 0x033,      0x00000022, 0x03F,      0x00000849,
+	0x033,      0x00000023, 0x03F,      0x0000084C, 0x033,      0x00000024,
+	0x03F,      0x00000C4C, 0x033,      0x00000025, 0x03F,      0x00000C8A,
+	0x033,      0x00000026, 0x03F,      0x00000C8D, 0x033,      0x00000027,
+	0x03F,      0x00000CEB, 0x033,      0x00000028, 0x03F,      0x00000CEE,
+	0x033,      0x00000029, 0x03F,      0x00000CF1, 0x033,      0x0000002A,
+	0x03F,      0x00000CF4, 0x90002000, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000020, 0x03F,      0x0000042B, 0x033,      0x00000021,
+	0x03F,      0x0000082A, 0x033,      0x00000022, 0x03F,      0x00000849,
+	0x033,      0x00000023, 0x03F,      0x0000084C, 0x033,      0x00000024,
+	0x03F,      0x00000C4C, 0x033,      0x00000025, 0x03F,      0x00000C8A,
+	0x033,      0x00000026, 0x03F,      0x00000C8D, 0x033,      0x00000027,
+	0x03F,      0x00000CEB, 0x033,      0x00000028, 0x03F,      0x00000CEE,
+	0x033,      0x00000029, 0x03F,      0x00000CF1, 0x033,      0x0000002A,
+	0x03F,      0x00000CF4, 0xA0000000, 0x00000000, 0x033,      0x00000020,
+	0x03F,      0x00000C09, 0x033,      0x00000021, 0x03F,      0x00000C0C,
+	0x033,      0x00000022, 0x03F,      0x00000C0F, 0x033,      0x00000023,
+	0x03F,      0x00000C2C, 0x033,      0x00000024, 0x03F,      0x00000C2F,
+	0x033,      0x00000025, 0x03F,      0x00000C8A, 0x033,      0x00000026,
+	0x03F,      0x00000C8D, 0x033,      0x00000027, 0x03F,      0x00000C90,
+	0x033,      0x00000028, 0x03F,      0x00000CD0, 0x033,      0x00000029,
+	0x03F,      0x00000CF2, 0x033,      0x0000002A, 0x03F,      0x00000CF5,
+	0xB0000000, 0x00000000, 0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x033,      0x00000060, 0x03F,      0x00000007, 0x033,      0x00000061,
+	0x03F,      0x0000000A, 0x033,      0x00000062, 0x03F,      0x0000000D,
+	0x033,      0x00000063, 0x03F,      0x0000002A, 0x033,      0x00000064,
+	0x03F,      0x0000002D, 0x033,      0x00000065, 0x03F,      0x00000030,
+	0x033,      0x00000066, 0x03F,      0x0000006D, 0x033,      0x00000067,
+	0x03F,      0x00000070, 0x033,      0x00000068, 0x03F,      0x000000ED,
+	0x033,      0x00000069, 0x03F,      0x000000F0, 0x033,      0x0000006A,
+	0x03F,      0x000000F3, 0x9300100f, 0x05050505, 0x40000000, 0x00000000,
+	0x033,      0x00000060, 0x03F,      0x00000007, 0x033,      0x00000061,
+	0x03F,      0x0000000A, 0x033,      0x00000062, 0x03F,      0x0000000D,
+	0x033,      0x00000063, 0x03F,      0x0000002A, 0x033,      0x00000064,
+	0x03F,      0x0000002D, 0x033,      0x00000065, 0x03F,      0x00000030,
+	0x033,      0x00000066, 0x03F,      0x0000006D, 0x033,      0x00000067,
+	0x03F,      0x00000070, 0x033,      0x00000068, 0x03F,      0x000000ED,
+	0x033,      0x00000069, 0x03F,      0x000000F0, 0x033,      0x0000006A,
+	0x03F,      0x000000F3, 0x9300100f, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000060, 0x03F,      0x00000007, 0x033,      0x00000061,
+	0x03F,      0x0000000A, 0x033,      0x00000062, 0x03F,      0x0000000D,
+	0x033,      0x00000063, 0x03F,      0x0000002A, 0x033,      0x00000064,
+	0x03F,      0x0000002D, 0x033,      0x00000065, 0x03F,      0x00000030,
+	0x033,      0x00000066, 0x03F,      0x0000006D, 0x033,      0x00000067,
+	0x03F,      0x00000070, 0x033,      0x00000068, 0x03F,      0x000000ED,
+	0x033,      0x00000069, 0x03F,      0x000000F0, 0x033,      0x0000006A,
+	0x03F,      0x000000F3, 0x9300200f, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000060, 0x03F,      0x00000005, 0x033,      0x00000061,
+	0x03F,      0x00000008, 0x033,      0x00000062, 0x03F,      0x0000000B,
+	0x033,      0x00000063, 0x03F,      0x0000000E, 0x033,      0x00000064,
+	0x03F,      0x0000002B, 0x033,      0x00000065, 0x03F,      0x00000068,
+	0x033,      0x00000066, 0x03F,      0x0000006B, 0x033,      0x00000067,
+	0x03F,      0x0000006E, 0x033,      0x00000068, 0x03F,      0x00000071,
+	0x033,      0x00000069, 0x03F,      0x00000074, 0x033,      0x0000006A,
+	0x03F,      0x00000077, 0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x033,      0x00000060, 0x03F,      0x00000007, 0x033,      0x00000061,
+	0x03F,      0x0000000A, 0x033,      0x00000062, 0x03F,      0x0000000D,
+	0x033,      0x00000063, 0x03F,      0x0000002A, 0x033,      0x00000064,
+	0x03F,      0x0000002D, 0x033,      0x00000065, 0x03F,      0x00000030,
+	0x033,      0x00000066, 0x03F,      0x0000006D, 0x033,      0x00000067,
+	0x03F,      0x00000070, 0x033,      0x00000068, 0x03F,      0x000000ED,
+	0x033,      0x00000069, 0x03F,      0x000000F0, 0x033,      0x0000006A,
+	0x03F,      0x000000F3, 0x9000100f, 0x05050505, 0x40000000, 0x00000000,
+	0x033,      0x00000060, 0x03F,      0x00000007, 0x033,      0x00000061,
+	0x03F,      0x0000000A, 0x033,      0x00000062, 0x03F,      0x0000000D,
+	0x033,      0x00000063, 0x03F,      0x0000002A, 0x033,      0x00000064,
+	0x03F,      0x0000002D, 0x033,      0x00000065, 0x03F,      0x00000030,
+	0x033,      0x00000066, 0x03F,      0x0000006D, 0x033,      0x00000067,
+	0x03F,      0x00000070, 0x033,      0x00000068, 0x03F,      0x000000ED,
+	0x033,      0x00000069, 0x03F,      0x000000F0, 0x033,      0x0000006A,
+	0x03F,      0x000000F3, 0x9000100f, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000060, 0x03F,      0x00000007, 0x033,      0x00000061,
+	0x03F,      0x0000000A, 0x033,      0x00000062, 0x03F,      0x0000000D,
+	0x033,      0x00000063, 0x03F,      0x0000002A, 0x033,      0x00000064,
+	0x03F,      0x0000002D, 0x033,      0x00000065, 0x03F,      0x00000030,
+	0x033,      0x00000066, 0x03F,      0x0000006D, 0x033,      0x00000067,
+	0x03F,      0x00000070, 0x033,      0x00000068, 0x03F,      0x000000ED,
+	0x033,      0x00000069, 0x03F,      0x000000F0, 0x033,      0x0000006A,
+	0x03F,      0x000000F3, 0x9000200f, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000060, 0x03F,      0x00000005, 0x033,      0x00000061,
+	0x03F,      0x00000008, 0x033,      0x00000062, 0x03F,      0x0000000B,
+	0x033,      0x00000063, 0x03F,      0x0000000E, 0x033,      0x00000064,
+	0x03F,      0x0000002B, 0x033,      0x00000065, 0x03F,      0x00000068,
+	0x033,      0x00000066, 0x03F,      0x0000006B, 0x033,      0x00000067,
+	0x03F,      0x0000006E, 0x033,      0x00000068, 0x03F,      0x00000071,
+	0x033,      0x00000069, 0x03F,      0x00000074, 0x033,      0x0000006A,
+	0x03F,      0x00000077, 0x9300200c, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000060, 0x03F,      0x00000005, 0x033,      0x00000061,
+	0x03F,      0x00000008, 0x033,      0x00000062, 0x03F,      0x0000000B,
+	0x033,      0x00000063, 0x03F,      0x0000000E, 0x033,      0x00000064,
+	0x03F,      0x0000002B, 0x033,      0x00000065, 0x03F,      0x00000068,
+	0x033,      0x00000066, 0x03F,      0x0000006B, 0x033,      0x00000067,
+	0x03F,      0x0000006E, 0x033,      0x00000068, 0x03F,      0x00000071,
+	0x033,      0x00000069, 0x03F,      0x00000074, 0x033,      0x0000006A,
+	0x03F,      0x00000077, 0x93012100, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000060, 0x03F,      0x0000080B, 0x033,      0x00000061,
+	0x03F,      0x0000080E, 0x033,      0x00000062, 0x03F,      0x00000848,
+	0x033,      0x00000063, 0x03F,      0x00000869, 0x033,      0x00000064,
+	0x03F,      0x000008A9, 0x033,      0x00000065, 0x03F,      0x00000CE8,
+	0x033,      0x00000066, 0x03F,      0x00000CEB, 0x033,      0x00000067,
+	0x03F,      0x00000CEE, 0x033,      0x00000068, 0x03F,      0x00000CF1,
+	0x033,      0x00000069, 0x03F,      0x00000CF4, 0x033,      0x0000006A,
+	0x03F,      0x00000CF7, 0x93002100, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000060, 0x03F,      0x0000042B, 0x033,      0x00000061,
+	0x03F,      0x0000082A, 0x033,      0x00000062, 0x03F,      0x00000849,
+	0x033,      0x00000063, 0x03F,      0x0000084C, 0x033,      0x00000064,
+	0x03F,      0x00000C4C, 0x033,      0x00000065, 0x03F,      0x00000CA9,
+	0x033,      0x00000066, 0x03F,      0x00000CEA, 0x033,      0x00000067,
+	0x03F,      0x00000CED, 0x033,      0x00000068, 0x03F,      0x00000CF0,
+	0x033,      0x00000069, 0x03F,      0x00000CF3, 0x033,      0x0000006A,
+	0x03F,      0x00000CF6, 0x93011000, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000060, 0x03F,      0x00000C0A, 0x033,      0x00000061,
+	0x03F,      0x00000C0D, 0x033,      0x00000062, 0x03F,      0x00000C2A,
+	0x033,      0x00000063, 0x03F,      0x00000C2D, 0x033,      0x00000064,
+	0x03F,      0x00000C6A, 0x033,      0x00000065, 0x03F,      0x00000CAA,
+	0x033,      0x00000066, 0x03F,      0x00000CAD, 0x033,      0x00000067,
+	0x03F,      0x00000CB0, 0x033,      0x00000068, 0x03F,      0x00000CF1,
+	0x033,      0x00000069, 0x03F,      0x00000CF4, 0x033,      0x0000006A,
+	0x03F,      0x00000CF7, 0x9000200c, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000060, 0x03F,      0x00000005, 0x033,      0x00000061,
+	0x03F,      0x00000008, 0x033,      0x00000062, 0x03F,      0x0000000B,
+	0x033,      0x00000063, 0x03F,      0x0000000E, 0x033,      0x00000064,
+	0x03F,      0x0000002B, 0x033,      0x00000065, 0x03F,      0x00000068,
+	0x033,      0x00000066, 0x03F,      0x0000006B, 0x033,      0x00000067,
+	0x03F,      0x0000006E, 0x033,      0x00000068, 0x03F,      0x00000071,
+	0x033,      0x00000069, 0x03F,      0x00000074, 0x033,      0x0000006A,
+	0x03F,      0x00000077, 0x90001004, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000060, 0x03F,      0x00000C0A, 0x033,      0x00000061,
+	0x03F,      0x00000C0D, 0x033,      0x00000062, 0x03F,      0x00000C2A,
+	0x033,      0x00000063, 0x03F,      0x00000C2D, 0x033,      0x00000064,
+	0x03F,      0x00000C6A, 0x033,      0x00000065, 0x03F,      0x00000CAA,
+	0x033,      0x00000066, 0x03F,      0x00000CAD, 0x033,      0x00000067,
+	0x03F,      0x00000CB0, 0x033,      0x00000068, 0x03F,      0x00000CF1,
+	0x033,      0x00000069, 0x03F,      0x00000CF4, 0x033,      0x0000006A,
+	0x03F,      0x00000CF7, 0x93002000, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000060, 0x03F,      0x00000429, 0x033,      0x00000061,
+	0x03F,      0x00000828, 0x033,      0x00000062, 0x03F,      0x00000847,
+	0x033,      0x00000063, 0x03F,      0x0000084A, 0x033,      0x00000064,
+	0x03F,      0x00000C4B, 0x033,      0x00000065, 0x03F,      0x00000C8A,
+	0x033,      0x00000066, 0x03F,      0x00000CEA, 0x033,      0x00000067,
+	0x03F,      0x00000CED, 0x033,      0x00000068, 0x03F,      0x00000CF0,
+	0x033,      0x00000069, 0x03F,      0x00000CF3, 0x033,      0x0000006A,
+	0x03F,      0x00000CF6, 0x93001000, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000060, 0x03F,      0x00000C0A, 0x033,      0x00000061,
+	0x03F,      0x00000C0D, 0x033,      0x00000062, 0x03F,      0x00000C2A,
+	0x033,      0x00000063, 0x03F,      0x00000C2D, 0x033,      0x00000064,
+	0x03F,      0x00000C6A, 0x033,      0x00000065, 0x03F,      0x00000CAA,
+	0x033,      0x00000066, 0x03F,      0x00000CAD, 0x033,      0x00000067,
+	0x03F,      0x00000CB0, 0x033,      0x00000068, 0x03F,      0x00000CF1,
+	0x033,      0x00000069, 0x03F,      0x00000CF4, 0x033,      0x0000006A,
+	0x03F,      0x00000CF7, 0x90002100, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000060, 0x03F,      0x0000042C, 0x033,      0x00000061,
+	0x03F,      0x0000082B, 0x033,      0x00000062, 0x03F,      0x0000084A,
+	0x033,      0x00000063, 0x03F,      0x0000084D, 0x033,      0x00000064,
+	0x03F,      0x00000C4D, 0x033,      0x00000065, 0x03F,      0x00000C8B,
+	0x033,      0x00000066, 0x03F,      0x00000C8E, 0x033,      0x00000067,
+	0x03F,      0x00000CEC, 0x033,      0x00000068, 0x03F,      0x00000CEF,
+	0x033,      0x00000069, 0x03F,      0x00000CF2, 0x033,      0x0000006A,
+	0x03F,      0x00000CF5, 0x90002000, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000060, 0x03F,      0x0000042C, 0x033,      0x00000061,
+	0x03F,      0x0000082B, 0x033,      0x00000062, 0x03F,      0x0000084A,
+	0x033,      0x00000063, 0x03F,      0x0000084D, 0x033,      0x00000064,
+	0x03F,      0x00000C4D, 0x033,      0x00000065, 0x03F,      0x00000C8B,
+	0x033,      0x00000066, 0x03F,      0x00000C8E, 0x033,      0x00000067,
+	0x03F,      0x00000CEC, 0x033,      0x00000068, 0x03F,      0x00000CEF,
+	0x033,      0x00000069, 0x03F,      0x00000CF2, 0x033,      0x0000006A,
+	0x03F,      0x00000CF5, 0xA0000000, 0x00000000, 0x033,      0x00000060,
+	0x03F,      0x00000C0A, 0x033,      0x00000061, 0x03F,      0x00000C0D,
+	0x033,      0x00000062, 0x03F,      0x00000C2A, 0x033,      0x00000063,
+	0x03F,      0x00000C2D, 0x033,      0x00000064, 0x03F,      0x00000C6A,
+	0x033,      0x00000065, 0x03F,      0x00000CAA, 0x033,      0x00000066,
+	0x03F,      0x00000CAD, 0x033,      0x00000067, 0x03F,      0x00000CB0,
+	0x033,      0x00000068, 0x03F,      0x00000CF1, 0x033,      0x00000069,
+	0x03F,      0x00000CF4, 0x033,      0x0000006A, 0x03F,      0x00000CF7,
+	0xB0000000, 0x00000000, 0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x033,      0x000000A0, 0x03F,      0x00000007, 0x033,      0x000000A1,
+	0x03F,      0x0000000A, 0x033,      0x000000A2, 0x03F,      0x0000000D,
+	0x033,      0x000000A3, 0x03F,      0x0000002A, 0x033,      0x000000A4,
+	0x03F,      0x0000002D, 0x033,      0x000000A5, 0x03F,      0x00000030,
+	0x033,      0x000000A6, 0x03F,      0x0000006D, 0x033,      0x000000A7,
+	0x03F,      0x00000070, 0x033,      0x000000A8, 0x03F,      0x000000ED,
+	0x033,      0x000000A9, 0x03F,      0x000000F0, 0x033,      0x000000AA,
+	0x03F,      0x000000F3, 0x9300100f, 0x05050505, 0x40000000, 0x00000000,
+	0x033,      0x000000A0, 0x03F,      0x00000007, 0x033,      0x000000A1,
+	0x03F,      0x0000000A, 0x033,      0x000000A2, 0x03F,      0x0000000D,
+	0x033,      0x000000A3, 0x03F,      0x0000002A, 0x033,      0x000000A4,
+	0x03F,      0x0000002D, 0x033,      0x000000A5, 0x03F,      0x00000030,
+	0x033,      0x000000A6, 0x03F,      0x0000006D, 0x033,      0x000000A7,
+	0x03F,      0x00000070, 0x033,      0x000000A8, 0x03F,      0x000000ED,
+	0x033,      0x000000A9, 0x03F,      0x000000F0, 0x033,      0x000000AA,
+	0x03F,      0x000000F3, 0x9300100f, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x000000A0, 0x03F,      0x00000007, 0x033,      0x000000A1,
+	0x03F,      0x0000000A, 0x033,      0x000000A2, 0x03F,      0x0000000D,
+	0x033,      0x000000A3, 0x03F,      0x0000002A, 0x033,      0x000000A4,
+	0x03F,      0x0000002D, 0x033,      0x000000A5, 0x03F,      0x00000030,
+	0x033,      0x000000A6, 0x03F,      0x0000006D, 0x033,      0x000000A7,
+	0x03F,      0x00000070, 0x033,      0x000000A8, 0x03F,      0x000000ED,
+	0x033,      0x000000A9, 0x03F,      0x000000F0, 0x033,      0x000000AA,
+	0x03F,      0x000000F3, 0x9300200f, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x000000A0, 0x03F,      0x00000005, 0x033,      0x000000A1,
+	0x03F,      0x00000008, 0x033,      0x000000A2, 0x03F,      0x0000000B,
+	0x033,      0x000000A3, 0x03F,      0x0000000E, 0x033,      0x000000A4,
+	0x03F,      0x00000047, 0x033,      0x000000A5, 0x03F,      0x0000004A,
+	0x033,      0x000000A6, 0x03F,      0x0000004D, 0x033,      0x000000A7,
+	0x03F,      0x00000050, 0x033,      0x000000A8, 0x03F,      0x00000053,
+	0x033,      0x000000A9, 0x03F,      0x00000056, 0x033,      0x000000AA,
+	0x03F,      0x00000094, 0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x033,      0x000000A0, 0x03F,      0x00000007, 0x033,      0x000000A1,
+	0x03F,      0x0000000A, 0x033,      0x000000A2, 0x03F,      0x0000000D,
+	0x033,      0x000000A3, 0x03F,      0x0000002A, 0x033,      0x000000A4,
+	0x03F,      0x0000002D, 0x033,      0x000000A5, 0x03F,      0x00000030,
+	0x033,      0x000000A6, 0x03F,      0x0000006D, 0x033,      0x000000A7,
+	0x03F,      0x00000070, 0x033,      0x000000A8, 0x03F,      0x000000ED,
+	0x033,      0x000000A9, 0x03F,      0x000000F0, 0x033,      0x000000AA,
+	0x03F,      0x000000F3, 0x9000100f, 0x05050505, 0x40000000, 0x00000000,
+	0x033,      0x000000A0, 0x03F,      0x00000007, 0x033,      0x000000A1,
+	0x03F,      0x0000000A, 0x033,      0x000000A2, 0x03F,      0x0000000D,
+	0x033,      0x000000A3, 0x03F,      0x0000002A, 0x033,      0x000000A4,
+	0x03F,      0x0000002D, 0x033,      0x000000A5, 0x03F,      0x00000030,
+	0x033,      0x000000A6, 0x03F,      0x0000006D, 0x033,      0x000000A7,
+	0x03F,      0x00000070, 0x033,      0x000000A8, 0x03F,      0x000000ED,
+	0x033,      0x000000A9, 0x03F,      0x000000F0, 0x033,      0x000000AA,
+	0x03F,      0x000000F3, 0x9000100f, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x000000A0, 0x03F,      0x00000007, 0x033,      0x000000A1,
+	0x03F,      0x0000000A, 0x033,      0x000000A2, 0x03F,      0x0000000D,
+	0x033,      0x000000A3, 0x03F,      0x0000002A, 0x033,      0x000000A4,
+	0x03F,      0x0000002D, 0x033,      0x000000A5, 0x03F,      0x00000030,
+	0x033,      0x000000A6, 0x03F,      0x0000006D, 0x033,      0x000000A7,
+	0x03F,      0x00000070, 0x033,      0x000000A8, 0x03F,      0x000000ED,
+	0x033,      0x000000A9, 0x03F,      0x000000F0, 0x033,      0x000000AA,
+	0x03F,      0x000000F3, 0x9000200f, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x000000A0, 0x03F,      0x00000005, 0x033,      0x000000A1,
+	0x03F,      0x00000008, 0x033,      0x000000A2, 0x03F,      0x0000000B,
+	0x033,      0x000000A3, 0x03F,      0x0000000E, 0x033,      0x000000A4,
+	0x03F,      0x00000047, 0x033,      0x000000A5, 0x03F,      0x0000004A,
+	0x033,      0x000000A6, 0x03F,      0x0000004D, 0x033,      0x000000A7,
+	0x03F,      0x00000050, 0x033,      0x000000A8, 0x03F,      0x00000053,
+	0x033,      0x000000A9, 0x03F,      0x00000056, 0x033,      0x000000AA,
+	0x03F,      0x00000094, 0x9300200c, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x000000A0, 0x03F,      0x00000005, 0x033,      0x000000A1,
+	0x03F,      0x00000008, 0x033,      0x000000A2, 0x03F,      0x0000000B,
+	0x033,      0x000000A3, 0x03F,      0x0000000E, 0x033,      0x000000A4,
+	0x03F,      0x00000047, 0x033,      0x000000A5, 0x03F,      0x0000004A,
+	0x033,      0x000000A6, 0x03F,      0x0000004D, 0x033,      0x000000A7,
+	0x03F,      0x00000050, 0x033,      0x000000A8, 0x03F,      0x00000053,
+	0x033,      0x000000A9, 0x03F,      0x00000056, 0x033,      0x000000AA,
+	0x03F,      0x00000094, 0x93012100, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x000000A0, 0x03F,      0x00000C0A, 0x033,      0x000000A1,
+	0x03F,      0x00000C0D, 0x033,      0x000000A2, 0x03F,      0x00000C2A,
+	0x033,      0x000000A3, 0x03F,      0x00000C2D, 0x033,      0x000000A4,
+	0x03F,      0x00000C6A, 0x033,      0x000000A5, 0x03F,      0x00000CE8,
+	0x033,      0x000000A6, 0x03F,      0x00000CEB, 0x033,      0x000000A7,
+	0x03F,      0x00000CEE, 0x033,      0x000000A8, 0x03F,      0x00000CF1,
+	0x033,      0x000000A9, 0x03F,      0x00000CF4, 0x033,      0x000000AA,
+	0x03F,      0x00000CF7, 0x93002100, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x000000A0, 0x03F,      0x0000042A, 0x033,      0x000000A1,
+	0x03F,      0x00000829, 0x033,      0x000000A2, 0x03F,      0x00000848,
+	0x033,      0x000000A3, 0x03F,      0x0000084B, 0x033,      0x000000A4,
+	0x03F,      0x00000C4C, 0x033,      0x000000A5, 0x03F,      0x00000CA9,
+	0x033,      0x000000A6, 0x03F,      0x00000CEA, 0x033,      0x000000A7,
+	0x03F,      0x00000CED, 0x033,      0x000000A8, 0x03F,      0x00000CF0,
+	0x033,      0x000000A9, 0x03F,      0x00000CF3, 0x033,      0x000000AA,
+	0x03F,      0x00000CF6, 0x93011000, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x000000A0, 0x03F,      0x00000C09, 0x033,      0x000000A1,
+	0x03F,      0x00000C0C, 0x033,      0x000000A2, 0x03F,      0x00000C0F,
+	0x033,      0x000000A3, 0x03F,      0x00000C2C, 0x033,      0x000000A4,
+	0x03F,      0x00000C2F, 0x033,      0x000000A5, 0x03F,      0x00000C8A,
+	0x033,      0x000000A6, 0x03F,      0x00000C8D, 0x033,      0x000000A7,
+	0x03F,      0x00000C90, 0x033,      0x000000A8, 0x03F,      0x00000CEF,
+	0x033,      0x000000A9, 0x03F,      0x00000CF2, 0x033,      0x000000AA,
+	0x03F,      0x00000CF5, 0x9000200c, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x000000A0, 0x03F,      0x00000005, 0x033,      0x000000A1,
+	0x03F,      0x00000008, 0x033,      0x000000A2, 0x03F,      0x0000000B,
+	0x033,      0x000000A3, 0x03F,      0x0000000E, 0x033,      0x000000A4,
+	0x03F,      0x00000047, 0x033,      0x000000A5, 0x03F,      0x0000004A,
+	0x033,      0x000000A6, 0x03F,      0x0000004D, 0x033,      0x000000A7,
+	0x03F,      0x00000050, 0x033,      0x000000A8, 0x03F,      0x00000053,
+	0x033,      0x000000A9, 0x03F,      0x00000056, 0x033,      0x000000AA,
+	0x03F,      0x00000094, 0x90001004, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x000000A0, 0x03F,      0x00000C09, 0x033,      0x000000A1,
+	0x03F,      0x00000C0C, 0x033,      0x000000A2, 0x03F,      0x00000C0F,
+	0x033,      0x000000A3, 0x03F,      0x00000C2C, 0x033,      0x000000A4,
+	0x03F,      0x00000C2F, 0x033,      0x000000A5, 0x03F,      0x00000C8A,
+	0x033,      0x000000A6, 0x03F,      0x00000C8D, 0x033,      0x000000A7,
+	0x03F,      0x00000C90, 0x033,      0x000000A8, 0x03F,      0x00000CEF,
+	0x033,      0x000000A9, 0x03F,      0x00000CF2, 0x033,      0x000000AA,
+	0x03F,      0x00000CF5, 0x93002000, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x000000A0, 0x03F,      0x00000429, 0x033,      0x000000A1,
+	0x03F,      0x00000828, 0x033,      0x000000A2, 0x03F,      0x00000847,
+	0x033,      0x000000A3, 0x03F,      0x0000084A, 0x033,      0x000000A4,
+	0x03F,      0x00000C4B, 0x033,      0x000000A5, 0x03F,      0x00000C8A,
+	0x033,      0x000000A6, 0x03F,      0x00000CEA, 0x033,      0x000000A7,
+	0x03F,      0x00000CED, 0x033,      0x000000A8, 0x03F,      0x00000CF0,
+	0x033,      0x000000A9, 0x03F,      0x00000CF3, 0x033,      0x000000AA,
+	0x03F,      0x00000CF6, 0x93001000, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x000000A0, 0x03F,      0x00000C09, 0x033,      0x000000A1,
+	0x03F,      0x00000C0C, 0x033,      0x000000A2, 0x03F,      0x00000C0F,
+	0x033,      0x000000A3, 0x03F,      0x00000C2C, 0x033,      0x000000A4,
+	0x03F,      0x00000C2F, 0x033,      0x000000A5, 0x03F,      0x00000C8A,
+	0x033,      0x000000A6, 0x03F,      0x00000C8D, 0x033,      0x000000A7,
+	0x03F,      0x00000C90, 0x033,      0x000000A8, 0x03F,      0x00000CEF,
+	0x033,      0x000000A9, 0x03F,      0x00000CF2, 0x033,      0x000000AA,
+	0x03F,      0x00000CF5, 0x90002100, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x000000A0, 0x03F,      0x0000042A, 0x033,      0x000000A1,
+	0x03F,      0x00000829, 0x033,      0x000000A2, 0x03F,      0x00000848,
+	0x033,      0x000000A3, 0x03F,      0x0000084B, 0x033,      0x000000A4,
+	0x03F,      0x00000C4C, 0x033,      0x000000A5, 0x03F,      0x00000C8A,
+	0x033,      0x000000A6, 0x03F,      0x00000C8D, 0x033,      0x000000A7,
+	0x03F,      0x00000CEB, 0x033,      0x000000A8, 0x03F,      0x00000CEE,
+	0x033,      0x000000A9, 0x03F,      0x00000CF1, 0x033,      0x000000AA,
+	0x03F,      0x00000CF4, 0x90002000, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x000000A0, 0x03F,      0x0000042A, 0x033,      0x000000A1,
+	0x03F,      0x00000829, 0x033,      0x000000A2, 0x03F,      0x00000848,
+	0x033,      0x000000A3, 0x03F,      0x0000084B, 0x033,      0x000000A4,
+	0x03F,      0x00000C4C, 0x033,      0x000000A5, 0x03F,      0x00000C8A,
+	0x033,      0x000000A6, 0x03F,      0x00000C8D, 0x033,      0x000000A7,
+	0x03F,      0x00000CEB, 0x033,      0x000000A8, 0x03F,      0x00000CEE,
+	0x033,      0x000000A9, 0x03F,      0x00000CF1, 0x033,      0x000000AA,
+	0x03F,      0x00000CF4, 0xA0000000, 0x00000000, 0x033,      0x000000A0,
+	0x03F,      0x00000C09, 0x033,      0x000000A1, 0x03F,      0x00000C0C,
+	0x033,      0x000000A2, 0x03F,      0x00000C0F, 0x033,      0x000000A3,
+	0x03F,      0x00000C2C, 0x033,      0x000000A4, 0x03F,      0x00000C2F,
+	0x033,      0x000000A5, 0x03F,      0x00000C8A, 0x033,      0x000000A6,
+	0x03F,      0x00000C8D, 0x033,      0x000000A7, 0x03F,      0x00000C90,
+	0x033,      0x000000A8, 0x03F,      0x00000CEF, 0x033,      0x000000A9,
+	0x03F,      0x00000CF2, 0x033,      0x000000AA, 0x03F,      0x00000CF5,
+	0xB0000000, 0x00000000, 0x0EF,      0x00000000, 0x0EF,      0x00000400,
+	0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x033,      0x00000000,
+	0x03F,      0x0000047C, 0x033,      0x00000001, 0x03F,      0x0000047C,
+	0x033,      0x00000002, 0x03F,      0x0000047C, 0x033,      0x00000003,
+	0x03F,      0x0000047C, 0x9300100f, 0x05050505, 0x40000000, 0x00000000,
+	0x033,      0x00000000, 0x03F,      0x0000047C, 0x033,      0x00000001,
+	0x03F,      0x0000047C, 0x033,      0x00000002, 0x03F,      0x0000047C,
+	0x033,      0x00000003, 0x03F,      0x0000047C, 0x9300100f, 0x00000000,
+	0x40000000, 0x00000000, 0x033,      0x00000000, 0x03F,      0x0000047C,
+	0x033,      0x00000001, 0x03F,      0x0000047C, 0x033,      0x00000002,
+	0x03F,      0x0000047C, 0x033,      0x00000003, 0x03F,      0x0000047C,
+	0x9300200f, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000000,
+	0x03F,      0x0000047C, 0x033,      0x00000001, 0x03F,      0x0000047C,
+	0x033,      0x00000002, 0x03F,      0x0000047C, 0x033,      0x00000003,
+	0x03F,      0x0000047C, 0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x033,      0x00000000, 0x03F,      0x0000047C, 0x033,      0x00000001,
+	0x03F,      0x0000047C, 0x033,      0x00000002, 0x03F,      0x0000047C,
+	0x033,      0x00000003, 0x03F,      0x0000047C, 0x9000100f, 0x05050505,
+	0x40000000, 0x00000000, 0x033,      0x00000000, 0x03F,      0x0000047C,
+	0x033,      0x00000001, 0x03F,      0x0000047C, 0x033,      0x00000002,
+	0x03F,      0x0000047C, 0x033,      0x00000003, 0x03F,      0x0000047C,
+	0x9000100f, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000000,
+	0x03F,      0x0000047C, 0x033,      0x00000001, 0x03F,      0x0000047C,
+	0x033,      0x00000002, 0x03F,      0x0000047C, 0x033,      0x00000003,
+	0x03F,      0x0000047C, 0x9000200f, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000000, 0x03F,      0x0000047C, 0x033,      0x00000001,
+	0x03F,      0x0000047C, 0x033,      0x00000002, 0x03F,      0x0000047C,
+	0x033,      0x00000003, 0x03F,      0x0000047C, 0xA0000000, 0x00000000,
+	0x033,      0x00000000, 0x03F,      0x000004BB, 0x033,      0x00000001,
+	0x03F,      0x000004BB, 0x033,      0x00000002, 0x03F,      0x000004BB,
+	0x033,      0x00000003, 0x03F,      0x000004BB, 0xB0000000, 0x00000000,
+	0x0EF,      0x00000000, 0x0EF,      0x00000100, 0x8300100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x033,      0x00000000, 0x03F,      0x00001726,
+	0x033,      0x00000001, 0x03F,      0x00001726, 0x033,      0x00000002,
+	0x03F,      0x00001726, 0x033,      0x00000003, 0x03F,      0x00001726,
+	0x9300100f, 0x05050505, 0x40000000, 0x00000000, 0x033,      0x00000000,
+	0x03F,      0x00001726, 0x033,      0x00000001, 0x03F,      0x00001726,
+	0x033,      0x00000002, 0x03F,      0x00001726, 0x033,      0x00000003,
+	0x03F,      0x00001726, 0x9300100f, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000000, 0x03F,      0x00001726, 0x033,      0x00000001,
+	0x03F,      0x00001726, 0x033,      0x00000002, 0x03F,      0x00001726,
+	0x033,      0x00000003, 0x03F,      0x00001726, 0x9300200f, 0x00000000,
+	0x40000000, 0x00000000, 0x033,      0x00000000, 0x03F,      0x00001726,
+	0x033,      0x00000001, 0x03F,      0x00001726, 0x033,      0x00000002,
+	0x03F,      0x00001726, 0x033,      0x00000003, 0x03F,      0x00001726,
+	0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x033,      0x00000000,
+	0x03F,      0x00001726, 0x033,      0x00000001, 0x03F,      0x00001726,
+	0x033,      0x00000002, 0x03F,      0x00001726, 0x033,      0x00000003,
+	0x03F,      0x00001726, 0x9000100f, 0x05050505, 0x40000000, 0x00000000,
+	0x033,      0x00000000, 0x03F,      0x00001726, 0x033,      0x00000001,
+	0x03F,      0x00001726, 0x033,      0x00000002, 0x03F,      0x00001726,
+	0x033,      0x00000003, 0x03F,      0x00001726, 0x9000100f, 0x00000000,
+	0x40000000, 0x00000000, 0x033,      0x00000000, 0x03F,      0x00001726,
+	0x033,      0x00000001, 0x03F,      0x00001726, 0x033,      0x00000002,
+	0x03F,      0x00001726, 0x033,      0x00000003, 0x03F,      0x00001726,
+	0x9000200f, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000000,
+	0x03F,      0x00001726, 0x033,      0x00000001, 0x03F,      0x00001726,
+	0x033,      0x00000002, 0x03F,      0x00001726, 0x033,      0x00000003,
+	0x03F,      0x00001726, 0xA0000000, 0x00000000, 0x033,      0x00000000,
+	0x03F,      0x00000F34, 0x033,      0x00000001, 0x03F,      0x00000F34,
+	0x033,      0x00000002, 0x03F,      0x00000F34, 0x033,      0x00000003,
+	0x03F,      0x00000F34, 0xB0000000, 0x00000000, 0x0EF,      0x00000000,
+	0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x081,      0x0000F400,
+	0x087,      0x00016040, 0x051,      0x00000808, 0x052,      0x00098002,
+	0x053,      0x0000FA47, 0x054,      0x00058032, 0x056,      0x00051000,
+	0x057,      0x0000CE0A, 0x058,      0x00082030, 0x9300100f, 0x05050505,
+	0x40000000, 0x00000000, 0x081,      0x0000F400, 0x087,      0x00016040,
+	0x051,      0x00000808, 0x052,      0x00098002, 0x053,      0x0000FA47,
+	0x054,      0x00058032, 0x056,      0x00051000, 0x057,      0x0000CE0A,
+	0x058,      0x00082030, 0x9300100f, 0x00000000, 0x40000000, 0x00000000,
+	0x081,      0x0000F400, 0x087,      0x00016040, 0x051,      0x00000808,
+	0x052,      0x00098002, 0x053,      0x0000FA47, 0x054,      0x00058032,
+	0x056,      0x00051000, 0x057,      0x0000CE0A, 0x058,      0x00082030,
+	0x9300200f, 0x00000000, 0x40000000, 0x00000000, 0x081,      0x0000F400,
+	0x087,      0x00016040, 0x051,      0x00000808, 0x052,      0x00098002,
+	0x053,      0x0000FA47, 0x054,      0x00058032, 0x056,      0x00051000,
+	0x057,      0x0000CE0A, 0x058,      0x00082030, 0x9000100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x081,      0x0000F400, 0x087,      0x00016040,
+	0x051,      0x00000808, 0x052,      0x00098002, 0x053,      0x0000FA47,
+	0x054,      0x00058032, 0x056,      0x00051000, 0x057,      0x0000CE0A,
+	0x058,      0x00082030, 0x9000100f, 0x05050505, 0x40000000, 0x00000000,
+	0x081,      0x0000F400, 0x087,      0x00016040, 0x051,      0x00000808,
+	0x052,      0x00098002, 0x053,      0x0000FA47, 0x054,      0x00058032,
+	0x056,      0x00051000, 0x057,      0x0000CE0A, 0x058,      0x00082030,
+	0x9000100f, 0x00000000, 0x40000000, 0x00000000, 0x081,      0x0000F400,
+	0x087,      0x00016040, 0x051,      0x00000808, 0x052,      0x00098002,
+	0x053,      0x0000FA47, 0x054,      0x00058032, 0x056,      0x00051000,
+	0x057,      0x0000CE0A, 0x058,      0x00082030, 0x9000200f, 0x00000000,
+	0x40000000, 0x00000000, 0x081,      0x0000F400, 0x087,      0x00016040,
+	0x051,      0x00000808, 0x052,      0x00098002, 0x053,      0x0000FA47,
+	0x054,      0x00058032, 0x056,      0x00051000, 0x057,      0x0000CE0A,
+	0x058,      0x00082030, 0xA0000000, 0x00000000, 0x081,      0x0000F000,
+	0x087,      0x00016040, 0x051,      0x00000C00, 0x052,      0x0007C241,
+	0x053,      0x0001C069, 0x054,      0x00078032, 0x057,      0x0000CE0A,
+	0x058,      0x00058750, 0xB0000000, 0x00000000, 0x0EF,      0x00000800,
+	0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x033,      0x00000000,
+	0x03F,      0x00000003, 0x033,      0x00000001, 0x03F,      0x00000006,
+	0x033,      0x00000002, 0x03F,      0x00000009, 0x033,      0x00000003,
+	0x03F,      0x00000026, 0x033,      0x00000004, 0x03F,      0x00000029,
+	0x033,      0x00000005, 0x03F,      0x0000002C, 0x033,      0x00000006,
+	0x03F,      0x0000002F, 0x033,      0x00000007, 0x03F,      0x00000033,
+	0x033,      0x00000008, 0x03F,      0x00000036, 0x033,      0x00000009,
+	0x03F,      0x00000039, 0x033,      0x0000000A, 0x03F,      0x0000003C,
+	0x9300100f, 0x05050505, 0x40000000, 0x00000000, 0x033,      0x00000000,
+	0x03F,      0x00000003, 0x033,      0x00000001, 0x03F,      0x00000006,
+	0x033,      0x00000002, 0x03F,      0x00000009, 0x033,      0x00000003,
+	0x03F,      0x00000026, 0x033,      0x00000004, 0x03F,      0x00000029,
+	0x033,      0x00000005, 0x03F,      0x0000002C, 0x033,      0x00000006,
+	0x03F,      0x0000002F, 0x033,      0x00000007, 0x03F,      0x00000033,
+	0x033,      0x00000008, 0x03F,      0x00000036, 0x033,      0x00000009,
+	0x03F,      0x00000039, 0x033,      0x0000000A, 0x03F,      0x0000003C,
+	0x9300100f, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000000,
+	0x03F,      0x00000003, 0x033,      0x00000001, 0x03F,      0x00000006,
+	0x033,      0x00000002, 0x03F,      0x00000009, 0x033,      0x00000003,
+	0x03F,      0x00000026, 0x033,      0x00000004, 0x03F,      0x00000029,
+	0x033,      0x00000005, 0x03F,      0x0000002C, 0x033,      0x00000006,
+	0x03F,      0x0000002F, 0x033,      0x00000007, 0x03F,      0x00000033,
+	0x033,      0x00000008, 0x03F,      0x00000036, 0x033,      0x00000009,
+	0x03F,      0x00000039, 0x033,      0x0000000A, 0x03F,      0x0000003C,
+	0x9300200f, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000000,
+	0x03F,      0x00000003, 0x033,      0x00000001, 0x03F,      0x00000006,
+	0x033,      0x00000002, 0x03F,      0x00000009, 0x033,      0x00000003,
+	0x03F,      0x00000026, 0x033,      0x00000004, 0x03F,      0x00000029,
+	0x033,      0x00000005, 0x03F,      0x0000002C, 0x033,      0x00000006,
+	0x03F,      0x0000002F, 0x033,      0x00000007, 0x03F,      0x00000033,
+	0x033,      0x00000008, 0x03F,      0x00000036, 0x033,      0x00000009,
+	0x03F,      0x00000039, 0x033,      0x0000000A, 0x03F,      0x0000003C,
+	0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x033,      0x00000000,
+	0x03F,      0x00000003, 0x033,      0x00000001, 0x03F,      0x00000006,
+	0x033,      0x00000002, 0x03F,      0x00000009, 0x033,      0x00000003,
+	0x03F,      0x00000026, 0x033,      0x00000004, 0x03F,      0x00000029,
+	0x033,      0x00000005, 0x03F,      0x0000002C, 0x033,      0x00000006,
+	0x03F,      0x0000002F, 0x033,      0x00000007, 0x03F,      0x00000033,
+	0x033,      0x00000008, 0x03F,      0x00000036, 0x033,      0x00000009,
+	0x03F,      0x00000039, 0x033,      0x0000000A, 0x03F,      0x0000003C,
+	0x9000100f, 0x05050505, 0x40000000, 0x00000000, 0x033,      0x00000000,
+	0x03F,      0x00000003, 0x033,      0x00000001, 0x03F,      0x00000006,
+	0x033,      0x00000002, 0x03F,      0x00000009, 0x033,      0x00000003,
+	0x03F,      0x00000026, 0x033,      0x00000004, 0x03F,      0x00000029,
+	0x033,      0x00000005, 0x03F,      0x0000002C, 0x033,      0x00000006,
+	0x03F,      0x0000002F, 0x033,      0x00000007, 0x03F,      0x00000033,
+	0x033,      0x00000008, 0x03F,      0x00000036, 0x033,      0x00000009,
+	0x03F,      0x00000039, 0x033,      0x0000000A, 0x03F,      0x0000003C,
+	0x9000100f, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000000,
+	0x03F,      0x00000003, 0x033,      0x00000001, 0x03F,      0x00000006,
+	0x033,      0x00000002, 0x03F,      0x00000009, 0x033,      0x00000003,
+	0x03F,      0x00000026, 0x033,      0x00000004, 0x03F,      0x00000029,
+	0x033,      0x00000005, 0x03F,      0x0000002C, 0x033,      0x00000006,
+	0x03F,      0x0000002F, 0x033,      0x00000007, 0x03F,      0x00000033,
+	0x033,      0x00000008, 0x03F,      0x00000036, 0x033,      0x00000009,
+	0x03F,      0x00000039, 0x033,      0x0000000A, 0x03F,      0x0000003C,
+	0x9000200f, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000000,
+	0x03F,      0x00000003, 0x033,      0x00000001, 0x03F,      0x00000006,
+	0x033,      0x00000002, 0x03F,      0x00000009, 0x033,      0x00000003,
+	0x03F,      0x00000026, 0x033,      0x00000004, 0x03F,      0x00000029,
+	0x033,      0x00000005, 0x03F,      0x0000002C, 0x033,      0x00000006,
+	0x03F,      0x0000002F, 0x033,      0x00000007, 0x03F,      0x00000033,
+	0x033,      0x00000008, 0x03F,      0x00000036, 0x033,      0x00000009,
+	0x03F,      0x00000039, 0x033,      0x0000000A, 0x03F,      0x0000003C,
+	0xA0000000, 0x00000000, 0x033,      0x00000000, 0x03F,      0x0005142C,
+	0x033,      0x00000001, 0x03F,      0x0005144B, 0x033,      0x00000002,
+	0x03F,      0x0005144E, 0x033,      0x00000003, 0x03F,      0x00051C69,
+	0x033,      0x00000004, 0x03F,      0x00051C6C, 0x033,      0x00000005,
+	0x03F,      0x00051C6F, 0x033,      0x00000006, 0x03F,      0x00051CEB,
+	0x033,      0x00000007, 0x03F,      0x00051CEE, 0x033,      0x00000008,
+	0x03F,      0x00051CF1, 0x033,      0x00000009, 0x03F,      0x00051CF4,
+	0x033,      0x0000000A, 0x03F,      0x00051CF7, 0xB0000000, 0x00000000,
+	0x0EF,      0x00000000, 0x0EF,      0x00000010, 0x033,      0x00000000,
+	0x008,      0x0009C060, 0x033,      0x00000001, 0x008,      0x0009C060,
+	0x0EF,      0x00000000, 0x033,      0x000000A2, 0x0EF,      0x00080000,
+	0x03E,      0x0000593F, 0x03F,      0x000C0F4F, 0x0EF,      0x00000000,
+	0x033,      0x000000A3, 0x0EF,      0x00080000, 0x03E,      0x00005934,
+	0x03F,      0x0005AFCF, 0x0EF,      0x00000000,
+
+};
+
+void odm_read_and_config_mp_8822b_radioa(struct phy_dm_struct *dm)
+{
+	u32 i = 0;
+	u8 c_cond;
+	bool is_matched = true, is_skipped = false;
+	u32 array_len = sizeof(array_mp_8822b_radioa) / sizeof(u32);
+	u32 *array = array_mp_8822b_radioa;
+
+	u32 v1 = 0, v2 = 0, pre_v1 = 0, pre_v2 = 0;
+
+	ODM_RT_TRACE(dm, ODM_COMP_INIT,
+		     "===> %s\n", __func__);
+
+	for (; (i + 1) < array_len; i = i + 2) {
+		v1 = array[i];
+		v2 = array[i + 1];
+
+		if (v1 & BIT(31)) { /* positive condition*/
+			c_cond = (u8)((v1 & (BIT(29) | BIT(28))) >> 28);
+			if (c_cond == COND_ENDIF) { /*end*/
+				is_matched = true;
+				is_skipped = false;
+				ODM_RT_TRACE(dm, ODM_COMP_INIT, "ENDIF\n");
+			} else if (c_cond == COND_ELSE) { /*else*/
+				is_matched = is_skipped ? false : true;
+				ODM_RT_TRACE(dm, ODM_COMP_INIT, "ELSE\n");
+			} else { /*if , else if*/
+				pre_v1 = v1;
+				pre_v2 = v2;
+				ODM_RT_TRACE(dm, ODM_COMP_INIT,
+					     "IF or ELSE IF\n");
+			}
+		} else if (v1 & BIT(30)) { /*negative condition*/
+			if (is_skipped) {
+				is_matched = false;
+				continue;
+			}
+
+			if (check_positive(dm, pre_v1, pre_v2, v1, v2)) {
+				is_matched = true;
+				is_skipped = true;
+			} else {
+				is_matched = false;
+				is_skipped = false;
+			}
+		} else if (is_matched) {
+			odm_config_rf_radio_a_8822b(dm, v1, v2);
+		}
+	}
+}
+
+u32 odm_get_version_mp_8822b_radioa(void) { return 67; }
+
+/******************************************************************************
+ *                           radiob.TXT
+ ******************************************************************************/
+
+static u32 array_mp_8822b_radiob[] = {
+	0x000,      0x00030000, 0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x001,      0x0004002D, 0x9300100f, 0x05050505, 0x40000000, 0x00000000,
+	0x001,      0x0004002D, 0x9300100f, 0x00000000, 0x40000000, 0x00000000,
+	0x001,      0x0004002D, 0x9300200f, 0x00000000, 0x40000000, 0x00000000,
+	0x001,      0x0004002D, 0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x001,      0x0004002D, 0x9000100f, 0x05050505, 0x40000000, 0x00000000,
+	0x001,      0x0004002D, 0x9000100f, 0x00000000, 0x40000000, 0x00000000,
+	0x001,      0x0004002D, 0x9000200f, 0x00000000, 0x40000000, 0x00000000,
+	0x001,      0x0004002D, 0x9300200c, 0x00000000, 0x40000000, 0x00000000,
+	0x001,      0x00040029, 0x93012100, 0x00000000, 0x40000000, 0x00000000,
+	0x001,      0x00040029, 0x93002100, 0x00000000, 0x40000000, 0x00000000,
+	0x001,      0x00040029, 0x9000200c, 0x00000000, 0x40000000, 0x00000000,
+	0x001,      0x00040029, 0x90001004, 0x00000000, 0x40000000, 0x00000000,
+	0x001,      0x00040029, 0x93002000, 0x00000000, 0x40000000, 0x00000000,
+	0x001,      0x00040029, 0x90002100, 0x00000000, 0x40000000, 0x00000000,
+	0x001,      0x00040029, 0x90002000, 0x00000000, 0x40000000, 0x00000000,
+	0x001,      0x00040029, 0xA0000000, 0x00000000, 0x001,      0x00040029,
+	0xB0000000, 0x00000000, 0x018,      0x00010D24, 0x0EF,      0x00080000,
+	0x033,      0x00000002, 0x03E,      0x0000003F, 0x03F,      0x000C0F4E,
+	0x033,      0x00000001, 0x03E,      0x00000034, 0x03F,      0x0004080E,
+	0x0EF,      0x00080000, 0x0DF,      0x00002449, 0x033,      0x00000024,
+	0x03E,      0x0000003F, 0x03F,      0x00060FDE, 0x0EF,      0x00000000,
+	0x0EF,      0x00080000, 0x033,      0x00000025, 0x03E,      0x00000037,
+	0x03F,      0x0007EFCE, 0x0EF,      0x00000000, 0x0EF,      0x00080000,
+	0x033,      0x00000026, 0x03E,      0x00000037, 0x03F,      0x000DEFCE,
+	0x0EF,      0x00000000, 0x0DF,      0x00000009, 0x018,      0x00010524,
+	0x089,      0x00000207, 0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x08A,      0x000FF186, 0x9300100f, 0x05050505, 0x40000000, 0x00000000,
+	0x08A,      0x000FE186, 0x9300100f, 0x00000000, 0x40000000, 0x00000000,
+	0x08A,      0x000FF186, 0x9300200f, 0x00000000, 0x40000000, 0x00000000,
+	0x08A,      0x000FF186, 0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x08A,      0x000FF186, 0x9000100f, 0x05050505, 0x40000000, 0x00000000,
+	0x08A,      0x000FE186, 0xA0000000, 0x00000000, 0x08A,      0x000FF186,
+	0xB0000000, 0x00000000, 0x08B,      0x00061E3C, 0x08C,      0x000112C7,
+	0x08D,      0x000F4988, 0x08E,      0x00064D40, 0x0EF,      0x00020000,
+	0x033,      0x00000007, 0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x03E,      0x00004040, 0x9300100f, 0x05050505, 0x40000000, 0x00000000,
+	0x03E,      0x00004080, 0x9300100f, 0x00000000, 0x40000000, 0x00000000,
+	0x03E,      0x00004040, 0x9300200f, 0x00000000, 0x40000000, 0x00000000,
+	0x03E,      0x00004040, 0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x03E,      0x00004040, 0x9000100f, 0x05050505, 0x40000000, 0x00000000,
+	0x03E,      0x00004080, 0x9000100f, 0x00000000, 0x40000000, 0x00000000,
+	0x03E,      0x00004040, 0x9000200f, 0x00000000, 0x40000000, 0x00000000,
+	0x03E,      0x00004040, 0x9300200c, 0x00000000, 0x40000000, 0x00000000,
+	0x03E,      0x00004040, 0x93012100, 0x00000000, 0x40000000, 0x00000000,
+	0x03E,      0x00004000, 0x93002100, 0x00000000, 0x40000000, 0x00000000,
+	0x03E,      0x00004000, 0x93011000, 0x00000000, 0x40000000, 0x00000000,
+	0x03E,      0x00004000, 0x9000200c, 0x00000000, 0x40000000, 0x00000000,
+	0x03E,      0x00004040, 0x90001004, 0x00000000, 0x40000000, 0x00000000,
+	0x03E,      0x00004040, 0x93002000, 0x00000000, 0x40000000, 0x00000000,
+	0x03E,      0x00004000, 0xA0000000, 0x00000000, 0x03E,      0x00004000,
+	0xB0000000, 0x00000000, 0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x03F,      0x000C3186, 0x9300100f, 0x05050505, 0x40000000, 0x00000000,
+	0x03F,      0x000C3186, 0x9300100f, 0x00000000, 0x40000000, 0x00000000,
+	0x03F,      0x000C3186, 0x9300200f, 0x00000000, 0x40000000, 0x00000000,
+	0x03F,      0x000C3186, 0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x03F,      0x000C3186, 0x9000100f, 0x05050505, 0x40000000, 0x00000000,
+	0x03F,      0x000C3186, 0x9000100f, 0x00000000, 0x40000000, 0x00000000,
+	0x03F,      0x000C3186, 0x9000200f, 0x00000000, 0x40000000, 0x00000000,
+	0x03F,      0x000C3186, 0x9300200c, 0x00000000, 0x40000000, 0x00000000,
+	0x03F,      0x000C3186, 0x93012100, 0x00000000, 0x40000000, 0x00000000,
+	0x03F,      0x000C3186, 0x93002100, 0x00000000, 0x40000000, 0x00000000,
+	0x03F,      0x000C0006, 0x93011000, 0x00000000, 0x40000000, 0x00000000,
+	0x03F,      0x000C3186, 0x9000200c, 0x00000000, 0x40000000, 0x00000000,
+	0x03F,      0x000C3186, 0x90001004, 0x00000000, 0x40000000, 0x00000000,
+	0x03F,      0x000C3186, 0x93002000, 0x00000000, 0x40000000, 0x00000000,
+	0x03F,      0x000C0006, 0x93001000, 0x00000000, 0x40000000, 0x00000000,
+	0x03F,      0x000C3186, 0xA0000000, 0x00000000, 0x03F,      0x000C3186,
+	0xB0000000, 0x00000000, 0x033,      0x00000006, 0x03E,      0x00004080,
+	0x03F,      0x000C3186, 0x033,      0x00000005, 0x03E,      0x000040C8,
+	0x03F,      0x000C3186, 0x033,      0x00000004, 0x03E,      0x00004190,
+	0x03F,      0x000C3186, 0x033,      0x00000003, 0x03E,      0x00004998,
+	0x03F,      0x000C3186, 0x033,      0x00000002, 0x03E,      0x00005840,
+	0x03F,      0x000C3186, 0x033,      0x00000001, 0x03E,      0x000058C2,
+	0x03F,      0x000C3186, 0x033,      0x00000000, 0x03E,      0x00005930,
+	0x03F,      0x000C3186, 0x033,      0x0000000F, 0x8300100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x9300100f, 0x05050505,
+	0x40000000, 0x00000000, 0x03E,      0x00004080, 0x9300100f, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x9300200f, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x9000100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x9000100f, 0x05050505,
+	0x40000000, 0x00000000, 0x03E,      0x00004080, 0x9000100f, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x9000200f, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x9300200c, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x93012100, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004000, 0x93002100, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004000, 0x93011000, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004000, 0x9000200c, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x90001004, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004040, 0x93002000, 0x00000000,
+	0x40000000, 0x00000000, 0x03E,      0x00004000, 0xA0000000, 0x00000000,
+	0x03E,      0x00004000, 0xB0000000, 0x00000000, 0x8300100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9300100f, 0x05050505,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9300100f, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9300200f, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9000100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9000100f, 0x05050505,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9000100f, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9000200f, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9300200c, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x93012100, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x93002100, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x93011000, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x9000200c, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x90001004, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0x93002000, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C0006, 0x93001000, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x000C3186, 0xA0000000, 0x00000000,
+	0x03F,      0x000C3186, 0xB0000000, 0x00000000, 0x033,      0x0000000E,
+	0x03E,      0x00004080, 0x03F,      0x000C3186, 0x033,      0x0000000D,
+	0x03E,      0x000040C8, 0x03F,      0x000C3186, 0x033,      0x0000000C,
+	0x03E,      0x00004190, 0x03F,      0x000C3186, 0x033,      0x0000000B,
+	0x03E,      0x00004998, 0x03F,      0x000C3186, 0x033,      0x0000000A,
+	0x03E,      0x00005840, 0x03F,      0x000C3186, 0x033,      0x00000009,
+	0x03E,      0x000058C2, 0x03F,      0x000C3186, 0x033,      0x00000008,
+	0x03E,      0x00005930, 0x03F,      0x000C3186, 0x033,      0x00000017,
+	0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x03E,      0x00004040,
+	0x9300100f, 0x05050505, 0x40000000, 0x00000000, 0x03E,      0x00004080,
+	0x9300100f, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00004040,
+	0x9300200f, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00004040,
+	0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x03E,      0x00004040,
+	0x9000100f, 0x05050505, 0x40000000, 0x00000000, 0x03E,      0x00004080,
+	0x9000100f, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00004040,
+	0x9000200f, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00004040,
+	0x9300200c, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00004040,
+	0x93012100, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00004000,
+	0x93002100, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00004000,
+	0x93011000, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00004000,
+	0x9000200c, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00004040,
+	0x90001004, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00004040,
+	0x93002000, 0x00000000, 0x40000000, 0x00000000, 0x03E,      0x00004000,
+	0xA0000000, 0x00000000, 0x03E,      0x00004000, 0xB0000000, 0x00000000,
+	0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x9300100f, 0x05050505, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x9300100f, 0x00000000, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x9300200f, 0x00000000, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x9000100f, 0x05050505, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x9000100f, 0x00000000, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x9000200f, 0x00000000, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x9300200c, 0x00000000, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x93012100, 0x00000000, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x93002100, 0x00000000, 0x40000000, 0x00000000, 0x03F,      0x000DFF86,
+	0x93011000, 0x00000000, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x9000200c, 0x00000000, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x90001004, 0x00000000, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0x93002000, 0x00000000, 0x40000000, 0x00000000, 0x03F,      0x000C0006,
+	0x93001000, 0x00000000, 0x40000000, 0x00000000, 0x03F,      0x000C3186,
+	0xA0000000, 0x00000000, 0x03F,      0x000C3186, 0xB0000000, 0x00000000,
+	0x033,      0x00000016, 0x03E,      0x00004080, 0x03F,      0x000C3186,
+	0x033,      0x00000015, 0x03E,      0x000040C8, 0x03F,      0x000C3186,
+	0x033,      0x00000014, 0x03E,      0x00004190, 0x03F,      0x000C3186,
+	0x033,      0x00000013, 0x03E,      0x00004998, 0x03F,      0x000C3186,
+	0x033,      0x00000012, 0x03E,      0x00005840, 0x03F,      0x000C3186,
+	0x033,      0x00000011, 0x03E,      0x000058C2, 0x03F,      0x000C3186,
+	0x033,      0x00000010, 0x03E,      0x00005930, 0x03F,      0x000C3186,
+	0x0EF,      0x00000000, 0x0EF,      0x00004000, 0x033,      0x00000000,
+	0x03F,      0x0000000A, 0x033,      0x00000001, 0x8300100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x9300100f, 0x05050505,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x9300100f, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x9300200f, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x9000100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x9000100f, 0x05050505,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x9000100f, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x9000200f, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x9300200c, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x93012100, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000002, 0x93002100, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x93011000, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000005, 0x9000200c, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x90001004, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x93002000, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x93001000, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000005, 0x90002100, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0x90002000, 0x00000000,
+	0x40000000, 0x00000000, 0x03F,      0x00000000, 0xA0000000, 0x00000000,
+	0x03F,      0x00000005, 0xB0000000, 0x00000000, 0x033,      0x00000002,
+	0x03F,      0x00000000, 0x0EF,      0x00000000, 0x018,      0x00000401,
+	0x084,      0x00001209, 0x086,      0x000001A0, 0x8300100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x087,      0x00068080, 0x9300100f, 0x05050505,
+	0x40000000, 0x00000000, 0x087,      0x00068080, 0x9300100f, 0x00000000,
+	0x40000000, 0x00000000, 0x087,      0x00068080, 0x9300200f, 0x00000000,
+	0x40000000, 0x00000000, 0x087,      0x00068080, 0x9000100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x087,      0x00068080, 0x9000100f, 0x05050505,
+	0x40000000, 0x00000000, 0x087,      0x00068080, 0x9000100f, 0x00000000,
+	0x40000000, 0x00000000, 0x087,      0x00068080, 0x9000200f, 0x00000000,
+	0x40000000, 0x00000000, 0x087,      0x00068080, 0xA0000000, 0x00000000,
+	0x087,      0x000E8180, 0xB0000000, 0x00000000, 0x088,      0x00070020,
+	0x0DE,      0x00000010, 0x0EF,      0x00008000, 0x033,      0x0000000F,
+	0x03F,      0x0000003C, 0x033,      0x0000000E, 0x03F,      0x00000038,
+	0x033,      0x0000000D, 0x03F,      0x00000030, 0x033,      0x0000000C,
+	0x03F,      0x00000028, 0x033,      0x0000000B, 0x03F,      0x00000020,
+	0x033,      0x0000000A, 0x03F,      0x00000018, 0x033,      0x00000009,
+	0x03F,      0x00000010, 0x033,      0x00000008, 0x03F,      0x00000008,
+	0x033,      0x00000007, 0x03F,      0x0000003C, 0x033,      0x00000006,
+	0x03F,      0x00000038, 0x033,      0x00000005, 0x03F,      0x00000030,
+	0x033,      0x00000004, 0x03F,      0x00000028, 0x033,      0x00000003,
+	0x03F,      0x00000020, 0x033,      0x00000002, 0x03F,      0x00000018,
+	0x033,      0x00000001, 0x03F,      0x00000010, 0x033,      0x00000000,
+	0x03F,      0x00000008, 0x0EF,      0x00000000, 0x018,      0x00018D24,
+	0xFFE,      0x00000000, 0xFFE,      0x00000000, 0xFFE,      0x00000000,
+	0xFFE,      0x00000000, 0x018,      0x00010D24, 0x01B,      0x00075A40,
+	0x0EE,      0x00000002, 0x033,      0x00000000, 0x03F,      0x00000004,
+	0x033,      0x00000001, 0x03F,      0x00000004, 0x033,      0x00000002,
+	0x03F,      0x00000004, 0x033,      0x00000003, 0x03F,      0x00000004,
+	0x033,      0x00000004, 0x03F,      0x00000004, 0x033,      0x00000005,
+	0x03F,      0x00000006, 0x033,      0x00000006, 0x03F,      0x00000002,
+	0x033,      0x00000007, 0x03F,      0x00000000, 0x0EE,      0x00000000,
+	0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x061,      0x0005D4A0,
+	0x062,      0x0000D203, 0x063,      0x00000062, 0x9300100f, 0x05050505,
+	0x40000000, 0x00000000, 0x061,      0x0005D4A0, 0x062,      0x0000D203,
+	0x063,      0x00000062, 0x9300100f, 0x00000000, 0x40000000, 0x00000000,
+	0x061,      0x0005D4A0, 0x062,      0x0000D203, 0x063,      0x00000062,
+	0x9300200f, 0x00000000, 0x40000000, 0x00000000, 0x061,      0x0005D2A1,
+	0x062,      0x0000D3A2, 0x063,      0x00000062, 0x9000100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x061,      0x0005D4A0, 0x062,      0x0000D203,
+	0x063,      0x00000062, 0x9000100f, 0x05050505, 0x40000000, 0x00000000,
+	0x061,      0x0005D4A0, 0x062,      0x0000D203, 0x063,      0x00000062,
+	0x9000100f, 0x00000000, 0x40000000, 0x00000000, 0x061,      0x0005D4A0,
+	0x062,      0x0000D203, 0x063,      0x00000062, 0x9000200f, 0x00000000,
+	0x40000000, 0x00000000, 0x061,      0x0005D2A1, 0x062,      0x0000D3A2,
+	0x063,      0x00000062, 0x9300200c, 0x00000000, 0x40000000, 0x00000000,
+	0x061,      0x0005D2A1, 0x062,      0x0000D3A2, 0x063,      0x00000062,
+	0x93012100, 0x00000000, 0x40000000, 0x00000000, 0x061,      0x0005D2A1,
+	0x062,      0x0000D3A2, 0x063,      0x00000002, 0x93002100, 0x00000000,
+	0x40000000, 0x00000000, 0x061,      0x0005D2A1, 0x062,      0x0000D3A2,
+	0x063,      0x00000002, 0x93011000, 0x00000000, 0x40000000, 0x00000000,
+	0x061,      0x0005D3D1, 0x062,      0x0000D3A2, 0x063,      0x00000002,
+	0x9000200c, 0x00000000, 0x40000000, 0x00000000, 0x061,      0x0005D2A1,
+	0x062,      0x0000D3A2, 0x063,      0x00000062, 0x90001004, 0x00000000,
+	0x40000000, 0x00000000, 0x061,      0x0005D3D1, 0x062,      0x0000D3A2,
+	0x063,      0x00000002, 0x93002000, 0x00000000, 0x40000000, 0x00000000,
+	0x061,      0x0005D2A1, 0x062,      0x0000D3A2, 0x063,      0x00000002,
+	0x93001000, 0x00000000, 0x40000000, 0x00000000, 0x061,      0x0005D3D1,
+	0x062,      0x0000D3A2, 0x063,      0x00000002, 0x90002100, 0x00000000,
+	0x40000000, 0x00000000, 0x061,      0x0005D2A1, 0x062,      0x0000D3A2,
+	0x063,      0x00000002, 0x90002000, 0x00000000, 0x40000000, 0x00000000,
+	0x061,      0x0005D2A1, 0x062,      0x0000D3A2, 0x063,      0x00000002,
+	0xA0000000, 0x00000000, 0x061,      0x0005D3D0, 0x062,      0x0000D303,
+	0x063,      0x00000002, 0xB0000000, 0x00000000, 0x0EF,      0x00000200,
+	0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x030,      0x000004A3,
+	0x030,      0x000014A3, 0x030,      0x000024A3, 0x030,      0x000034A3,
+	0x030,      0x000044A3, 0x030,      0x000054A3, 0x030,      0x000064A3,
+	0x030,      0x000074A3, 0x030,      0x000084A3, 0x030,      0x000094A3,
+	0x030,      0x0000A4A3, 0x030,      0x0000B4A3, 0x9300100f, 0x05050505,
+	0x40000000, 0x00000000, 0x030,      0x000004A3, 0x030,      0x000014A3,
+	0x030,      0x000024A3, 0x030,      0x000034A3, 0x030,      0x000044A3,
+	0x030,      0x000054A3, 0x030,      0x000064A3, 0x030,      0x000074A3,
+	0x030,      0x000084A3, 0x030,      0x000094A3, 0x030,      0x0000A4A3,
+	0x030,      0x0000B4A3, 0x9300100f, 0x00000000, 0x40000000, 0x00000000,
+	0x030,      0x000004A3, 0x030,      0x000014A3, 0x030,      0x000024A3,
+	0x030,      0x000034A3, 0x030,      0x000044A3, 0x030,      0x000054A3,
+	0x030,      0x000064A3, 0x030,      0x000074A3, 0x030,      0x000084A3,
+	0x030,      0x000094A3, 0x030,      0x0000A4A3, 0x030,      0x0000B4A3,
+	0x9300200f, 0x00000000, 0x40000000, 0x00000000, 0x030,      0x000002A6,
+	0x030,      0x000012A6, 0x030,      0x000022A6, 0x030,      0x000032A6,
+	0x030,      0x000042A6, 0x030,      0x000052A6, 0x030,      0x000062A6,
+	0x030,      0x000072A6, 0x030,      0x000082A6, 0x030,      0x000092A6,
+	0x030,      0x0000A2A6, 0x030,      0x0000B2A6, 0x9000100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x030,      0x000004A0, 0x030,      0x000014A0,
+	0x030,      0x000024A0, 0x030,      0x000034A0, 0x030,      0x000044A0,
+	0x030,      0x000054A0, 0x030,      0x000064A0, 0x030,      0x000074A0,
+	0x030,      0x000084A0, 0x030,      0x000094A0, 0x030,      0x0000A4A0,
+	0x030,      0x0000B4A0, 0x9000100f, 0x05050505, 0x40000000, 0x00000000,
+	0x030,      0x000004A0, 0x030,      0x000014A0, 0x030,      0x000024A0,
+	0x030,      0x000034A0, 0x030,      0x000044A0, 0x030,      0x000054A0,
+	0x030,      0x000064A0, 0x030,      0x000074A0, 0x030,      0x000084A0,
+	0x030,      0x000094A0, 0x030,      0x0000A4A0, 0x030,      0x0000B4A0,
+	0x9000100f, 0x00000000, 0x40000000, 0x00000000, 0x030,      0x000004A0,
+	0x030,      0x000014A0, 0x030,      0x000024A0, 0x030,      0x000034A0,
+	0x030,      0x000044A0, 0x030,      0x000054A0, 0x030,      0x000064A0,
+	0x030,      0x000074A0, 0x030,      0x000084A0, 0x030,      0x000094A0,
+	0x030,      0x0000A4A0, 0x030,      0x0000B4A0, 0x9000200f, 0x00000000,
+	0x40000000, 0x00000000, 0x030,      0x000002A1, 0x030,      0x000012A1,
+	0x030,      0x000022A1, 0x030,      0x000032A1, 0x030,      0x000042A1,
+	0x030,      0x000052A1, 0x030,      0x000062A1, 0x030,      0x000072A1,
+	0x030,      0x000082A1, 0x030,      0x000092A1, 0x030,      0x0000A2A1,
+	0x030,      0x0000B2A1, 0x9300200c, 0x00000000, 0x40000000, 0x00000000,
+	0x030,      0x000002A6, 0x030,      0x000012A6, 0x030,      0x000022A6,
+	0x030,      0x000032A6, 0x030,      0x000042A6, 0x030,      0x000052A6,
+	0x030,      0x000062A6, 0x030,      0x000072A6, 0x030,      0x000082A6,
+	0x030,      0x000092A6, 0x030,      0x0000A2A6, 0x030,      0x0000B2A6,
+	0x93012100, 0x00000000, 0x40000000, 0x00000000, 0x030,      0x000002F4,
+	0x030,      0x000012F4, 0x030,      0x000022F4, 0x030,      0x000032F4,
+	0x030,      0x00004365, 0x030,      0x00005365, 0x030,      0x00006365,
+	0x030,      0x00007365, 0x030,      0x000082A4, 0x030,      0x000092A4,
+	0x030,      0x0000A2A4, 0x030,      0x0000B2A4, 0x93002100, 0x00000000,
+	0x40000000, 0x00000000, 0x030,      0x000004A4, 0x030,      0x000014A4,
+	0x030,      0x000024A4, 0x030,      0x000034A4, 0x030,      0x000043A4,
+	0x030,      0x000053A4, 0x030,      0x000063A4, 0x030,      0x000073A4,
+	0x030,      0x000083A5, 0x030,      0x000093A5, 0x030,      0x0000A3A5,
+	0x030,      0x0000B3A5, 0x93011000, 0x00000000, 0x40000000, 0x00000000,
+	0x030,      0x000003A1, 0x030,      0x000013A1, 0x030,      0x000023A1,
+	0x030,      0x000033A1, 0x030,      0x000043A4, 0x030,      0x000053A4,
+	0x030,      0x000063A4, 0x030,      0x000073A4, 0x030,      0x000083A6,
+	0x030,      0x000093A6, 0x030,      0x0000A3A6, 0x030,      0x0000B3A6,
+	0x9000200c, 0x00000000, 0x40000000, 0x00000000, 0x030,      0x000002A1,
+	0x030,      0x000012A1, 0x030,      0x000022A1, 0x030,      0x000032A1,
+	0x030,      0x000042A1, 0x030,      0x000052A1, 0x030,      0x000062A1,
+	0x030,      0x000072A1, 0x030,      0x000082A1, 0x030,      0x000092A1,
+	0x030,      0x0000A2A1, 0x030,      0x0000B2A1, 0x90001004, 0x00000000,
+	0x40000000, 0x00000000, 0x030,      0x00000382, 0x030,      0x00001382,
+	0x030,      0x00002382, 0x030,      0x00003382, 0x030,      0x00004445,
+	0x030,      0x00005445, 0x030,      0x00006445, 0x030,      0x00007445,
+	0x030,      0x00008425, 0x030,      0x00009425, 0x030,      0x0000A425,
+	0x030,      0x0000B425, 0x93002000, 0x00000000, 0x40000000, 0x00000000,
+	0x030,      0x00000303, 0x030,      0x00001303, 0x030,      0x00002303,
+	0x030,      0x00003303, 0x030,      0x000043A4, 0x030,      0x000053A4,
+	0x030,      0x000063A4, 0x030,      0x000073A4, 0x030,      0x00008365,
+	0x030,      0x00009365, 0x030,      0x0000A365, 0x030,      0x0000B365,
+	0x93001000, 0x00000000, 0x40000000, 0x00000000, 0x030,      0x000003A1,
+	0x030,      0x000013A1, 0x030,      0x000023A1, 0x030,      0x000033A1,
+	0x030,      0x00004364, 0x030,      0x00005364, 0x030,      0x00006364,
+	0x030,      0x00007364, 0x030,      0x00008564, 0x030,      0x00009564,
+	0x030,      0x0000A564, 0x030,      0x0000B564, 0x90002100, 0x00000000,
+	0x40000000, 0x00000000, 0x030,      0x000004A1, 0x030,      0x000014A1,
+	0x030,      0x000024A1, 0x030,      0x000034A1, 0x030,      0x000043A1,
+	0x030,      0x000053A1, 0x030,      0x000063A1, 0x030,      0x000073A1,
+	0x030,      0x000083A1, 0x030,      0x000093A1, 0x030,      0x0000A3A1,
+	0x030,      0x0000B3A1, 0x90002000, 0x00000000, 0x40000000, 0x00000000,
+	0x030,      0x000004A0, 0x030,      0x000014A0, 0x030,      0x000024A0,
+	0x030,      0x000034A0, 0x030,      0x000043A1, 0x030,      0x000053A1,
+	0x030,      0x000063A1, 0x030,      0x000073A1, 0x030,      0x000083A2,
+	0x030,      0x000093A2, 0x030,      0x0000A3A2, 0x030,      0x0000B3A2,
+	0xA0000000, 0x00000000, 0x030,      0x000002D0, 0x030,      0x000012D0,
+	0x030,      0x000022D0, 0x030,      0x000032D0, 0x030,      0x000042D0,
+	0x030,      0x000052D0, 0x030,      0x000062D0, 0x030,      0x000072D0,
+	0x030,      0x000082D0, 0x030,      0x000092D0, 0x030,      0x0000A2D0,
+	0x030,      0x0000B2D0, 0xB0000000, 0x00000000, 0x0EF,      0x00000000,
+	0x0EF,      0x00000080, 0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x030,      0x00000203, 0x030,      0x00001203, 0x030,      0x00002203,
+	0x030,      0x00003203, 0x030,      0x00004203, 0x030,      0x00005203,
+	0x030,      0x00006203, 0x030,      0x00007203, 0x030,      0x00008203,
+	0x030,      0x00009203, 0x030,      0x0000A203, 0x030,      0x0000B203,
+	0x9300100f, 0x05050505, 0x40000000, 0x00000000, 0x030,      0x00000203,
+	0x030,      0x00001203, 0x030,      0x00002203, 0x030,      0x00003203,
+	0x030,      0x00004203, 0x030,      0x00005203, 0x030,      0x00006203,
+	0x030,      0x00007203, 0x030,      0x00008203, 0x030,      0x00009203,
+	0x030,      0x0000A203, 0x030,      0x0000B203, 0x9300100f, 0x00000000,
+	0x40000000, 0x00000000, 0x030,      0x00000203, 0x030,      0x00001203,
+	0x030,      0x00002203, 0x030,      0x00003203, 0x030,      0x00004203,
+	0x030,      0x00005203, 0x030,      0x00006203, 0x030,      0x00007203,
+	0x030,      0x00008203, 0x030,      0x00009203, 0x030,      0x0000A203,
+	0x030,      0x0000B203, 0x9300200f, 0x00000000, 0x40000000, 0x00000000,
+	0x030,      0x000003A2, 0x030,      0x000013A2, 0x030,      0x000023A2,
+	0x030,      0x000033A2, 0x030,      0x000043A2, 0x030,      0x000053A2,
+	0x030,      0x000063A2, 0x030,      0x000073A2, 0x030,      0x000083A2,
+	0x030,      0x000093A2, 0x030,      0x0000A3A2, 0x030,      0x0000B3A2,
+	0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x030,      0x00000203,
+	0x030,      0x00001203, 0x030,      0x00002203, 0x030,      0x00003203,
+	0x030,      0x00004203, 0x030,      0x00005203, 0x030,      0x00006203,
+	0x030,      0x00007203, 0x030,      0x00008203, 0x030,      0x00009203,
+	0x030,      0x0000A203, 0x030,      0x0000B203, 0x9000100f, 0x05050505,
+	0x40000000, 0x00000000, 0x030,      0x00000203, 0x030,      0x00001203,
+	0x030,      0x00002203, 0x030,      0x00003203, 0x030,      0x00004203,
+	0x030,      0x00005203, 0x030,      0x00006203, 0x030,      0x00007203,
+	0x030,      0x00008203, 0x030,      0x00009203, 0x030,      0x0000A203,
+	0x030,      0x0000B203, 0x9000100f, 0x00000000, 0x40000000, 0x00000000,
+	0x030,      0x00000203, 0x030,      0x00001203, 0x030,      0x00002203,
+	0x030,      0x00003203, 0x030,      0x00004203, 0x030,      0x00005203,
+	0x030,      0x00006203, 0x030,      0x00007203, 0x030,      0x00008203,
+	0x030,      0x00009203, 0x030,      0x0000A203, 0x030,      0x0000B203,
+	0x9000200f, 0x00000000, 0x40000000, 0x00000000, 0x030,      0x000003A2,
+	0x030,      0x000013A2, 0x030,      0x000023A2, 0x030,      0x000033A2,
+	0x030,      0x000043A2, 0x030,      0x000053A2, 0x030,      0x000063A2,
+	0x030,      0x000073A2, 0x030,      0x000083A2, 0x030,      0x000093A2,
+	0x030,      0x0000A3A2, 0x030,      0x0000B3A2, 0x9300200c, 0x00000000,
+	0x40000000, 0x00000000, 0x030,      0x000003A2, 0x030,      0x000013A2,
+	0x030,      0x000023A2, 0x030,      0x000033A2, 0x030,      0x000043A2,
+	0x030,      0x000053A2, 0x030,      0x000063A2, 0x030,      0x000073A2,
+	0x030,      0x000083A2, 0x030,      0x000093A2, 0x030,      0x0000A3A2,
+	0x030,      0x0000B3A2, 0x93012100, 0x00000000, 0x40000000, 0x00000000,
+	0x030,      0x000003A3, 0x030,      0x000013A3, 0x030,      0x000023A3,
+	0x030,      0x000033A3, 0x030,      0x000043A4, 0x030,      0x000053A4,
+	0x030,      0x000063A4, 0x030,      0x000073A4, 0x030,      0x000083A3,
+	0x030,      0x000093A3, 0x030,      0x0000A3A3, 0x030,      0x0000B3A3,
+	0x93002100, 0x00000000, 0x40000000, 0x00000000, 0x030,      0x000003A2,
+	0x030,      0x000013A2, 0x030,      0x000023A2, 0x030,      0x000033A2,
+	0x030,      0x000043A2, 0x030,      0x000053A2, 0x030,      0x000063A2,
+	0x030,      0x000073A2, 0x030,      0x000083A2, 0x030,      0x000093A2,
+	0x030,      0x0000A3A2, 0x030,      0x0000B3A2, 0x93011000, 0x00000000,
+	0x40000000, 0x00000000, 0x030,      0x000003A2, 0x030,      0x000013A2,
+	0x030,      0x000023A2, 0x030,      0x000033A2, 0x030,      0x000043A2,
+	0x030,      0x000053A2, 0x030,      0x000063A2, 0x030,      0x000073A2,
+	0x030,      0x000083A2, 0x030,      0x000093A2, 0x030,      0x0000A3A2,
+	0x030,      0x0000B3A2, 0x9000200c, 0x00000000, 0x40000000, 0x00000000,
+	0x030,      0x000003A2, 0x030,      0x000013A2, 0x030,      0x000023A2,
+	0x030,      0x000033A2, 0x030,      0x000043A2, 0x030,      0x000053A2,
+	0x030,      0x000063A2, 0x030,      0x000073A2, 0x030,      0x000083A2,
+	0x030,      0x000093A2, 0x030,      0x0000A3A2, 0x030,      0x0000B3A2,
+	0x90001004, 0x00000000, 0x40000000, 0x00000000, 0x030,      0x000003A2,
+	0x030,      0x000013A2, 0x030,      0x000023A2, 0x030,      0x000033A2,
+	0x030,      0x000043A2, 0x030,      0x000053A2, 0x030,      0x000063A2,
+	0x030,      0x000073A2, 0x030,      0x000083A2, 0x030,      0x000093A2,
+	0x030,      0x0000A3A2, 0x030,      0x0000B3A2, 0x93002000, 0x00000000,
+	0x40000000, 0x00000000, 0x030,      0x000003A2, 0x030,      0x000013A2,
+	0x030,      0x000023A2, 0x030,      0x000033A2, 0x030,      0x000043A2,
+	0x030,      0x000053A2, 0x030,      0x000063A2, 0x030,      0x000073A2,
+	0x030,      0x000083A2, 0x030,      0x000093A2, 0x030,      0x0000A3A2,
+	0x030,      0x0000B3A2, 0x93001000, 0x00000000, 0x40000000, 0x00000000,
+	0x030,      0x000003A2, 0x030,      0x000013A2, 0x030,      0x000023A2,
+	0x030,      0x000033A2, 0x030,      0x000043A2, 0x030,      0x000053A2,
+	0x030,      0x000063A2, 0x030,      0x000073A2, 0x030,      0x000083A2,
+	0x030,      0x000093A2, 0x030,      0x0000A3A2, 0x030,      0x0000B3A2,
+	0x90002100, 0x00000000, 0x40000000, 0x00000000, 0x030,      0x000003A2,
+	0x030,      0x000013A2, 0x030,      0x000023A2, 0x030,      0x000033A2,
+	0x030,      0x000043A2, 0x030,      0x000053A2, 0x030,      0x000063A2,
+	0x030,      0x000073A2, 0x030,      0x000083A2, 0x030,      0x000093A2,
+	0x030,      0x0000A3A2, 0x030,      0x0000B3A2, 0x90002000, 0x00000000,
+	0x40000000, 0x00000000, 0x030,      0x000003A2, 0x030,      0x000013A2,
+	0x030,      0x000023A2, 0x030,      0x000033A2, 0x030,      0x000043A2,
+	0x030,      0x000053A2, 0x030,      0x000063A2, 0x030,      0x000073A2,
+	0x030,      0x000083A2, 0x030,      0x000093A2, 0x030,      0x0000A3A2,
+	0x030,      0x0000B3A2, 0xA0000000, 0x00000000, 0x030,      0x000003A2,
+	0x030,      0x000013A2, 0x030,      0x000023A2, 0x030,      0x000033A2,
+	0x030,      0x000043A2, 0x030,      0x000053A2, 0x030,      0x000063A2,
+	0x030,      0x000073A2, 0x030,      0x000083A2, 0x030,      0x000093A2,
+	0x030,      0x0000A3A2, 0x030,      0x0000B3A2, 0xB0000000, 0x00000000,
+	0x0EF,      0x00000000, 0x0EF,      0x00000040, 0x8300100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x030,      0x00000645, 0x030,      0x00001333,
+	0x030,      0x00002011, 0x030,      0x00004000, 0x030,      0x00005000,
+	0x030,      0x00006000, 0x9300100f, 0x05050505, 0x40000000, 0x00000000,
+	0x030,      0x00000645, 0x030,      0x00001333, 0x030,      0x00002011,
+	0x030,      0x00004000, 0x030,      0x00005000, 0x030,      0x00006000,
+	0x9300100f, 0x00000000, 0x40000000, 0x00000000, 0x030,      0x00000645,
+	0x030,      0x00001333, 0x030,      0x00002011, 0x030,      0x00004000,
+	0x030,      0x00005000, 0x030,      0x00006000, 0x9300200f, 0x00000000,
+	0x40000000, 0x00000000, 0x030,      0x00000645, 0x030,      0x00001333,
+	0x030,      0x00002011, 0x030,      0x00004777, 0x030,      0x00005777,
+	0x030,      0x00006777, 0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x030,      0x00000645, 0x030,      0x00001333, 0x030,      0x00002011,
+	0x030,      0x00004000, 0x030,      0x00005000, 0x030,      0x00006000,
+	0x9000100f, 0x05050505, 0x40000000, 0x00000000, 0x030,      0x00000645,
+	0x030,      0x00001333, 0x030,      0x00002011, 0x030,      0x00004000,
+	0x030,      0x00005000, 0x030,      0x00006000, 0x9000100f, 0x00000000,
+	0x40000000, 0x00000000, 0x030,      0x00000645, 0x030,      0x00001333,
+	0x030,      0x00002011, 0x030,      0x00004000, 0x030,      0x00005000,
+	0x030,      0x00006000, 0x9000200f, 0x00000000, 0x40000000, 0x00000000,
+	0x030,      0x00000645, 0x030,      0x00001333, 0x030,      0x00002011,
+	0x030,      0x00004000, 0x030,      0x00005000, 0x030,      0x00006000,
+	0x9300200c, 0x00000000, 0x40000000, 0x00000000, 0x030,      0x00000645,
+	0x030,      0x00001333, 0x030,      0x00002011, 0x030,      0x00004777,
+	0x030,      0x00005777, 0x030,      0x00006777, 0x93012100, 0x00000000,
+	0x40000000, 0x00000000, 0x030,      0x00000660, 0x030,      0x00001341,
+	0x030,      0x00002220, 0x030,      0x00004777, 0x030,      0x00005777,
+	0x030,      0x00006777, 0x93002100, 0x00000000, 0x40000000, 0x00000000,
+	0x030,      0x00000764, 0x030,      0x00001452, 0x030,      0x00002220,
+	0x030,      0x00004777, 0x030,      0x00005777, 0x030,      0x00006777,
+	0x93011000, 0x00000000, 0x40000000, 0x00000000, 0x030,      0x00000764,
+	0x030,      0x00001632, 0x030,      0x00002421, 0x030,      0x00004000,
+	0x030,      0x00005000, 0x030,      0x00006000, 0x9000200c, 0x00000000,
+	0x40000000, 0x00000000, 0x030,      0x00000645, 0x030,      0x00001333,
+	0x030,      0x00002011, 0x030,      0x00004000, 0x030,      0x00005000,
+	0x030,      0x00006000, 0x90001004, 0x00000000, 0x40000000, 0x00000000,
+	0x030,      0x00000764, 0x030,      0x00001632, 0x030,      0x00002421,
+	0x030,      0x00004000, 0x030,      0x00005000, 0x030,      0x00006000,
+	0x93002000, 0x00000000, 0x40000000, 0x00000000, 0x030,      0x00000777,
+	0x030,      0x00001442, 0x030,      0x00002222, 0x030,      0x00004777,
+	0x030,      0x00005777, 0x030,      0x00006777, 0x93001000, 0x00000000,
+	0x40000000, 0x00000000, 0x030,      0x00000764, 0x030,      0x00001632,
+	0x030,      0x00002421, 0x030,      0x00004000, 0x030,      0x00005000,
+	0x030,      0x00006000, 0x90002100, 0x00000000, 0x40000000, 0x00000000,
+	0x030,      0x00000775, 0x030,      0x00001222, 0x030,      0x00002210,
+	0x030,      0x00004000, 0x030,      0x00005000, 0x030,      0x00006000,
+	0x90002000, 0x00000000, 0x40000000, 0x00000000, 0x030,      0x00000775,
+	0x030,      0x00001422, 0x030,      0x00002210, 0x030,      0x00004000,
+	0x030,      0x00005000, 0x030,      0x00006000, 0xA0000000, 0x00000000,
+	0x030,      0x00000764, 0x030,      0x00001632, 0x030,      0x00002421,
+	0x030,      0x00004000, 0x030,      0x00005000, 0x030,      0x00006000,
+	0xB0000000, 0x00000000, 0x0EF,      0x00000000, 0x0EF,      0x00000800,
+	0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x033,      0x00000020,
+	0x03F,      0x00000007, 0x033,      0x00000021, 0x03F,      0x0000000A,
+	0x033,      0x00000022, 0x03F,      0x0000000D, 0x033,      0x00000023,
+	0x03F,      0x0000002A, 0x033,      0x00000024, 0x03F,      0x0000002D,
+	0x033,      0x00000025, 0x03F,      0x00000030, 0x033,      0x00000026,
+	0x03F,      0x0000006D, 0x033,      0x00000027, 0x03F,      0x00000070,
+	0x033,      0x00000028, 0x03F,      0x000000ED, 0x033,      0x00000029,
+	0x03F,      0x000000F0, 0x033,      0x0000002A, 0x03F,      0x000000F3,
+	0x9300100f, 0x05050505, 0x40000000, 0x00000000, 0x033,      0x00000020,
+	0x03F,      0x00000007, 0x033,      0x00000021, 0x03F,      0x0000000A,
+	0x033,      0x00000022, 0x03F,      0x0000000D, 0x033,      0x00000023,
+	0x03F,      0x0000002A, 0x033,      0x00000024, 0x03F,      0x0000002D,
+	0x033,      0x00000025, 0x03F,      0x00000030, 0x033,      0x00000026,
+	0x03F,      0x0000006D, 0x033,      0x00000027, 0x03F,      0x00000070,
+	0x033,      0x00000028, 0x03F,      0x000000ED, 0x033,      0x00000029,
+	0x03F,      0x000000F0, 0x033,      0x0000002A, 0x03F,      0x000000F3,
+	0x9300100f, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000020,
+	0x03F,      0x00000007, 0x033,      0x00000021, 0x03F,      0x0000000A,
+	0x033,      0x00000022, 0x03F,      0x0000000D, 0x033,      0x00000023,
+	0x03F,      0x0000002A, 0x033,      0x00000024, 0x03F,      0x0000002D,
+	0x033,      0x00000025, 0x03F,      0x00000030, 0x033,      0x00000026,
+	0x03F,      0x0000006D, 0x033,      0x00000027, 0x03F,      0x00000070,
+	0x033,      0x00000028, 0x03F,      0x000000ED, 0x033,      0x00000029,
+	0x03F,      0x000000F0, 0x033,      0x0000002A, 0x03F,      0x000000F3,
+	0x9300200f, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000020,
+	0x03F,      0x00000005, 0x033,      0x00000021, 0x03F,      0x00000008,
+	0x033,      0x00000022, 0x03F,      0x0000000B, 0x033,      0x00000023,
+	0x03F,      0x0000000E, 0x033,      0x00000024, 0x03F,      0x0000002B,
+	0x033,      0x00000025, 0x03F,      0x00000068, 0x033,      0x00000026,
+	0x03F,      0x0000006B, 0x033,      0x00000027, 0x03F,      0x0000006E,
+	0x033,      0x00000028, 0x03F,      0x00000071, 0x033,      0x00000029,
+	0x03F,      0x00000074, 0x033,      0x0000002A, 0x03F,      0x00000077,
+	0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x033,      0x00000020,
+	0x03F,      0x00000007, 0x033,      0x00000021, 0x03F,      0x0000000A,
+	0x033,      0x00000022, 0x03F,      0x0000000D, 0x033,      0x00000023,
+	0x03F,      0x0000002A, 0x033,      0x00000024, 0x03F,      0x0000002D,
+	0x033,      0x00000025, 0x03F,      0x00000030, 0x033,      0x00000026,
+	0x03F,      0x0000006D, 0x033,      0x00000027, 0x03F,      0x00000070,
+	0x033,      0x00000028, 0x03F,      0x000000ED, 0x033,      0x00000029,
+	0x03F,      0x000000F0, 0x033,      0x0000002A, 0x03F,      0x000000F3,
+	0x9000100f, 0x05050505, 0x40000000, 0x00000000, 0x033,      0x00000020,
+	0x03F,      0x00000007, 0x033,      0x00000021, 0x03F,      0x0000000A,
+	0x033,      0x00000022, 0x03F,      0x0000000D, 0x033,      0x00000023,
+	0x03F,      0x0000002A, 0x033,      0x00000024, 0x03F,      0x0000002D,
+	0x033,      0x00000025, 0x03F,      0x00000030, 0x033,      0x00000026,
+	0x03F,      0x0000006D, 0x033,      0x00000027, 0x03F,      0x00000070,
+	0x033,      0x00000028, 0x03F,      0x000000ED, 0x033,      0x00000029,
+	0x03F,      0x000000F0, 0x033,      0x0000002A, 0x03F,      0x000000F3,
+	0x9000100f, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000020,
+	0x03F,      0x00000007, 0x033,      0x00000021, 0x03F,      0x0000000A,
+	0x033,      0x00000022, 0x03F,      0x0000000D, 0x033,      0x00000023,
+	0x03F,      0x0000002A, 0x033,      0x00000024, 0x03F,      0x0000002D,
+	0x033,      0x00000025, 0x03F,      0x00000030, 0x033,      0x00000026,
+	0x03F,      0x0000006D, 0x033,      0x00000027, 0x03F,      0x00000070,
+	0x033,      0x00000028, 0x03F,      0x000000ED, 0x033,      0x00000029,
+	0x03F,      0x000000F0, 0x033,      0x0000002A, 0x03F,      0x000000F3,
+	0x9000200f, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000020,
+	0x03F,      0x00000005, 0x033,      0x00000021, 0x03F,      0x00000008,
+	0x033,      0x00000022, 0x03F,      0x0000000B, 0x033,      0x00000023,
+	0x03F,      0x0000000E, 0x033,      0x00000024, 0x03F,      0x0000002B,
+	0x033,      0x00000025, 0x03F,      0x00000068, 0x033,      0x00000026,
+	0x03F,      0x0000006B, 0x033,      0x00000027, 0x03F,      0x0000006E,
+	0x033,      0x00000028, 0x03F,      0x00000071, 0x033,      0x00000029,
+	0x03F,      0x00000074, 0x033,      0x0000002A, 0x03F,      0x00000077,
+	0x9300200c, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000020,
+	0x03F,      0x00000005, 0x033,      0x00000021, 0x03F,      0x00000008,
+	0x033,      0x00000022, 0x03F,      0x0000000B, 0x033,      0x00000023,
+	0x03F,      0x0000000E, 0x033,      0x00000024, 0x03F,      0x0000002B,
+	0x033,      0x00000025, 0x03F,      0x00000068, 0x033,      0x00000026,
+	0x03F,      0x0000006B, 0x033,      0x00000027, 0x03F,      0x0000006E,
+	0x033,      0x00000028, 0x03F,      0x00000071, 0x033,      0x00000029,
+	0x03F,      0x00000074, 0x033,      0x0000002A, 0x03F,      0x00000077,
+	0x93012100, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000020,
+	0x03F,      0x00000828, 0x033,      0x00000021, 0x03F,      0x0000082B,
+	0x033,      0x00000022, 0x03F,      0x00000868, 0x033,      0x00000023,
+	0x03F,      0x00000889, 0x033,      0x00000024, 0x03F,      0x000008AA,
+	0x033,      0x00000025, 0x03F,      0x00000CE8, 0x033,      0x00000026,
+	0x03F,      0x00000CEB, 0x033,      0x00000027, 0x03F,      0x00000CEE,
+	0x033,      0x00000028, 0x03F,      0x00000CF1, 0x033,      0x00000029,
+	0x03F,      0x00000CF4, 0x033,      0x0000002A, 0x03F,      0x00000CF7,
+	0x93002100, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000020,
+	0x03F,      0x0000042A, 0x033,      0x00000021, 0x03F,      0x00000829,
+	0x033,      0x00000022, 0x03F,      0x00000848, 0x033,      0x00000023,
+	0x03F,      0x0000084B, 0x033,      0x00000024, 0x03F,      0x00000C4C,
+	0x033,      0x00000025, 0x03F,      0x00000C8B, 0x033,      0x00000026,
+	0x03F,      0x00000CEA, 0x033,      0x00000027, 0x03F,      0x00000CED,
+	0x033,      0x00000028, 0x03F,      0x00000CF0, 0x033,      0x00000029,
+	0x03F,      0x00000CF3, 0x033,      0x0000002A, 0x03F,      0x00000CF6,
+	0x93011000, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000020,
+	0x03F,      0x00000C09, 0x033,      0x00000021, 0x03F,      0x00000C0C,
+	0x033,      0x00000022, 0x03F,      0x00000C0F, 0x033,      0x00000023,
+	0x03F,      0x00000C2C, 0x033,      0x00000024, 0x03F,      0x00000C2F,
+	0x033,      0x00000025, 0x03F,      0x00000C8A, 0x033,      0x00000026,
+	0x03F,      0x00000C8D, 0x033,      0x00000027, 0x03F,      0x00000C90,
+	0x033,      0x00000028, 0x03F,      0x00000CD0, 0x033,      0x00000029,
+	0x03F,      0x00000CF2, 0x033,      0x0000002A, 0x03F,      0x00000CF5,
+	0x9000200c, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000020,
+	0x03F,      0x00000005, 0x033,      0x00000021, 0x03F,      0x00000008,
+	0x033,      0x00000022, 0x03F,      0x0000000B, 0x033,      0x00000023,
+	0x03F,      0x0000000E, 0x033,      0x00000024, 0x03F,      0x0000002B,
+	0x033,      0x00000025, 0x03F,      0x00000068, 0x033,      0x00000026,
+	0x03F,      0x0000006B, 0x033,      0x00000027, 0x03F,      0x0000006E,
+	0x033,      0x00000028, 0x03F,      0x00000071, 0x033,      0x00000029,
+	0x03F,      0x00000074, 0x033,      0x0000002A, 0x03F,      0x00000077,
+	0x90001004, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000020,
+	0x03F,      0x00000C09, 0x033,      0x00000021, 0x03F,      0x00000C0C,
+	0x033,      0x00000022, 0x03F,      0x00000C0F, 0x033,      0x00000023,
+	0x03F,      0x00000C2C, 0x033,      0x00000024, 0x03F,      0x00000C2F,
+	0x033,      0x00000025, 0x03F,      0x00000C8A, 0x033,      0x00000026,
+	0x03F,      0x00000C8D, 0x033,      0x00000027, 0x03F,      0x00000C90,
+	0x033,      0x00000028, 0x03F,      0x00000CD0, 0x033,      0x00000029,
+	0x03F,      0x00000CF2, 0x033,      0x0000002A, 0x03F,      0x00000CF5,
+	0x93002000, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000020,
+	0x03F,      0x00000429, 0x033,      0x00000021, 0x03F,      0x00000828,
+	0x033,      0x00000022, 0x03F,      0x00000847, 0x033,      0x00000023,
+	0x03F,      0x0000084A, 0x033,      0x00000024, 0x03F,      0x00000C4B,
+	0x033,      0x00000025, 0x03F,      0x00000C8A, 0x033,      0x00000026,
+	0x03F,      0x00000CEA, 0x033,      0x00000027, 0x03F,      0x00000CED,
+	0x033,      0x00000028, 0x03F,      0x00000CF0, 0x033,      0x00000029,
+	0x03F,      0x00000CF3, 0x033,      0x0000002A, 0x03F,      0x00000CF6,
+	0x93001000, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000020,
+	0x03F,      0x00000C09, 0x033,      0x00000021, 0x03F,      0x00000C0C,
+	0x033,      0x00000022, 0x03F,      0x00000C0F, 0x033,      0x00000023,
+	0x03F,      0x00000C2C, 0x033,      0x00000024, 0x03F,      0x00000C2F,
+	0x033,      0x00000025, 0x03F,      0x00000C8A, 0x033,      0x00000026,
+	0x03F,      0x00000C8D, 0x033,      0x00000027, 0x03F,      0x00000C90,
+	0x033,      0x00000028, 0x03F,      0x00000CD0, 0x033,      0x00000029,
+	0x03F,      0x00000CF2, 0x033,      0x0000002A, 0x03F,      0x00000CF5,
+	0x90002100, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000020,
+	0x03F,      0x0000042B, 0x033,      0x00000021, 0x03F,      0x0000082A,
+	0x033,      0x00000022, 0x03F,      0x00000849, 0x033,      0x00000023,
+	0x03F,      0x0000084C, 0x033,      0x00000024, 0x03F,      0x00000C4C,
+	0x033,      0x00000025, 0x03F,      0x00000C8A, 0x033,      0x00000026,
+	0x03F,      0x00000C8D, 0x033,      0x00000027, 0x03F,      0x00000CEB,
+	0x033,      0x00000028, 0x03F,      0x00000CEE, 0x033,      0x00000029,
+	0x03F,      0x00000CF1, 0x033,      0x0000002A, 0x03F,      0x00000CF4,
+	0x90002000, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000020,
+	0x03F,      0x0000042B, 0x033,      0x00000021, 0x03F,      0x0000082A,
+	0x033,      0x00000022, 0x03F,      0x00000849, 0x033,      0x00000023,
+	0x03F,      0x0000084C, 0x033,      0x00000024, 0x03F,      0x00000C4C,
+	0x033,      0x00000025, 0x03F,      0x00000C8A, 0x033,      0x00000026,
+	0x03F,      0x00000C8D, 0x033,      0x00000027, 0x03F,      0x00000CEB,
+	0x033,      0x00000028, 0x03F,      0x00000CEE, 0x033,      0x00000029,
+	0x03F,      0x00000CF1, 0x033,      0x0000002A, 0x03F,      0x00000CF4,
+	0xA0000000, 0x00000000, 0x033,      0x00000020, 0x03F,      0x00000C09,
+	0x033,      0x00000021, 0x03F,      0x00000C0C, 0x033,      0x00000022,
+	0x03F,      0x00000C0F, 0x033,      0x00000023, 0x03F,      0x00000C2C,
+	0x033,      0x00000024, 0x03F,      0x00000C2F, 0x033,      0x00000025,
+	0x03F,      0x00000C8A, 0x033,      0x00000026, 0x03F,      0x00000C8D,
+	0x033,      0x00000027, 0x03F,      0x00000C90, 0x033,      0x00000028,
+	0x03F,      0x00000CD0, 0x033,      0x00000029, 0x03F,      0x00000CF2,
+	0x033,      0x0000002A, 0x03F,      0x00000CF5, 0xB0000000, 0x00000000,
+	0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x033,      0x00000060,
+	0x03F,      0x00000007, 0x033,      0x00000061, 0x03F,      0x0000000A,
+	0x033,      0x00000062, 0x03F,      0x0000000D, 0x033,      0x00000063,
+	0x03F,      0x0000002A, 0x033,      0x00000064, 0x03F,      0x0000002D,
+	0x033,      0x00000065, 0x03F,      0x00000030, 0x033,      0x00000066,
+	0x03F,      0x0000006D, 0x033,      0x00000067, 0x03F,      0x00000070,
+	0x033,      0x00000068, 0x03F,      0x000000ED, 0x033,      0x00000069,
+	0x03F,      0x000000F0, 0x033,      0x0000006A, 0x03F,      0x000000F3,
+	0x9300100f, 0x05050505, 0x40000000, 0x00000000, 0x033,      0x00000060,
+	0x03F,      0x00000007, 0x033,      0x00000061, 0x03F,      0x0000000A,
+	0x033,      0x00000062, 0x03F,      0x0000000D, 0x033,      0x00000063,
+	0x03F,      0x0000002A, 0x033,      0x00000064, 0x03F,      0x0000002D,
+	0x033,      0x00000065, 0x03F,      0x00000030, 0x033,      0x00000066,
+	0x03F,      0x0000006D, 0x033,      0x00000067, 0x03F,      0x00000070,
+	0x033,      0x00000068, 0x03F,      0x000000ED, 0x033,      0x00000069,
+	0x03F,      0x000000F0, 0x033,      0x0000006A, 0x03F,      0x000000F3,
+	0x9300100f, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000060,
+	0x03F,      0x00000007, 0x033,      0x00000061, 0x03F,      0x0000000A,
+	0x033,      0x00000062, 0x03F,      0x0000000D, 0x033,      0x00000063,
+	0x03F,      0x0000002A, 0x033,      0x00000064, 0x03F,      0x0000002D,
+	0x033,      0x00000065, 0x03F,      0x00000030, 0x033,      0x00000066,
+	0x03F,      0x0000006D, 0x033,      0x00000067, 0x03F,      0x00000070,
+	0x033,      0x00000068, 0x03F,      0x000000ED, 0x033,      0x00000069,
+	0x03F,      0x000000F0, 0x033,      0x0000006A, 0x03F,      0x000000F3,
+	0x9300200f, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000060,
+	0x03F,      0x00000005, 0x033,      0x00000061, 0x03F,      0x00000008,
+	0x033,      0x00000062, 0x03F,      0x0000000B, 0x033,      0x00000063,
+	0x03F,      0x0000000E, 0x033,      0x00000064, 0x03F,      0x0000002B,
+	0x033,      0x00000065, 0x03F,      0x00000068, 0x033,      0x00000066,
+	0x03F,      0x0000006B, 0x033,      0x00000067, 0x03F,      0x0000006E,
+	0x033,      0x00000068, 0x03F,      0x00000071, 0x033,      0x00000069,
+	0x03F,      0x00000074, 0x033,      0x0000006A, 0x03F,      0x00000077,
+	0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x033,      0x00000060,
+	0x03F,      0x00000007, 0x033,      0x00000061, 0x03F,      0x0000000A,
+	0x033,      0x00000062, 0x03F,      0x0000000D, 0x033,      0x00000063,
+	0x03F,      0x0000002A, 0x033,      0x00000064, 0x03F,      0x0000002D,
+	0x033,      0x00000065, 0x03F,      0x00000030, 0x033,      0x00000066,
+	0x03F,      0x0000006D, 0x033,      0x00000067, 0x03F,      0x00000070,
+	0x033,      0x00000068, 0x03F,      0x000000ED, 0x033,      0x00000069,
+	0x03F,      0x000000F0, 0x033,      0x0000006A, 0x03F,      0x000000F3,
+	0x9000100f, 0x05050505, 0x40000000, 0x00000000, 0x033,      0x00000060,
+	0x03F,      0x00000007, 0x033,      0x00000061, 0x03F,      0x0000000A,
+	0x033,      0x00000062, 0x03F,      0x0000000D, 0x033,      0x00000063,
+	0x03F,      0x0000002A, 0x033,      0x00000064, 0x03F,      0x0000002D,
+	0x033,      0x00000065, 0x03F,      0x00000030, 0x033,      0x00000066,
+	0x03F,      0x0000006D, 0x033,      0x00000067, 0x03F,      0x00000070,
+	0x033,      0x00000068, 0x03F,      0x000000ED, 0x033,      0x00000069,
+	0x03F,      0x000000F0, 0x033,      0x0000006A, 0x03F,      0x000000F3,
+	0x9000100f, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000060,
+	0x03F,      0x00000007, 0x033,      0x00000061, 0x03F,      0x0000000A,
+	0x033,      0x00000062, 0x03F,      0x0000000D, 0x033,      0x00000063,
+	0x03F,      0x0000002A, 0x033,      0x00000064, 0x03F,      0x0000002D,
+	0x033,      0x00000065, 0x03F,      0x00000030, 0x033,      0x00000066,
+	0x03F,      0x0000006D, 0x033,      0x00000067, 0x03F,      0x00000070,
+	0x033,      0x00000068, 0x03F,      0x000000ED, 0x033,      0x00000069,
+	0x03F,      0x000000F0, 0x033,      0x0000006A, 0x03F,      0x000000F3,
+	0x9000200f, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000060,
+	0x03F,      0x00000005, 0x033,      0x00000061, 0x03F,      0x00000008,
+	0x033,      0x00000062, 0x03F,      0x0000000B, 0x033,      0x00000063,
+	0x03F,      0x0000000E, 0x033,      0x00000064, 0x03F,      0x0000002B,
+	0x033,      0x00000065, 0x03F,      0x00000068, 0x033,      0x00000066,
+	0x03F,      0x0000006B, 0x033,      0x00000067, 0x03F,      0x0000006E,
+	0x033,      0x00000068, 0x03F,      0x00000071, 0x033,      0x00000069,
+	0x03F,      0x00000074, 0x033,      0x0000006A, 0x03F,      0x00000077,
+	0x9300200c, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000060,
+	0x03F,      0x00000005, 0x033,      0x00000061, 0x03F,      0x00000008,
+	0x033,      0x00000062, 0x03F,      0x0000000B, 0x033,      0x00000063,
+	0x03F,      0x0000000E, 0x033,      0x00000064, 0x03F,      0x0000002B,
+	0x033,      0x00000065, 0x03F,      0x00000068, 0x033,      0x00000066,
+	0x03F,      0x0000006B, 0x033,      0x00000067, 0x03F,      0x0000006E,
+	0x033,      0x00000068, 0x03F,      0x00000071, 0x033,      0x00000069,
+	0x03F,      0x00000074, 0x033,      0x0000006A, 0x03F,      0x00000077,
+	0x93012100, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000060,
+	0x03F,      0x00000842, 0x033,      0x00000061, 0x03F,      0x00000845,
+	0x033,      0x00000062, 0x03F,      0x00000866, 0x033,      0x00000063,
+	0x03F,      0x000008A6, 0x033,      0x00000064, 0x03F,      0x000008C8,
+	0x033,      0x00000065, 0x03F,      0x00000CE8, 0x033,      0x00000066,
+	0x03F,      0x00000CEB, 0x033,      0x00000067, 0x03F,      0x00000CEE,
+	0x033,      0x00000068, 0x03F,      0x00000CF1, 0x033,      0x00000069,
+	0x03F,      0x00000CF4, 0x033,      0x0000006A, 0x03F,      0x00000CF7,
+	0x93002100, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000060,
+	0x03F,      0x0000042A, 0x033,      0x00000061, 0x03F,      0x00000829,
+	0x033,      0x00000062, 0x03F,      0x00000848, 0x033,      0x00000063,
+	0x03F,      0x0000084B, 0x033,      0x00000064, 0x03F,      0x00000C69,
+	0x033,      0x00000065, 0x03F,      0x00000CA9, 0x033,      0x00000066,
+	0x03F,      0x00000CEA, 0x033,      0x00000067, 0x03F,      0x00000CED,
+	0x033,      0x00000068, 0x03F,      0x00000CF0, 0x033,      0x00000069,
+	0x03F,      0x00000CF3, 0x033,      0x0000006A, 0x03F,      0x00000CF6,
+	0x93011000, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000060,
+	0x03F,      0x00000C0A, 0x033,      0x00000061, 0x03F,      0x00000C0D,
+	0x033,      0x00000062, 0x03F,      0x00000C2A, 0x033,      0x00000063,
+	0x03F,      0x00000C2D, 0x033,      0x00000064, 0x03F,      0x00000C6A,
+	0x033,      0x00000065, 0x03F,      0x00000CAA, 0x033,      0x00000066,
+	0x03F,      0x00000CAD, 0x033,      0x00000067, 0x03F,      0x00000CB0,
+	0x033,      0x00000068, 0x03F,      0x00000CF1, 0x033,      0x00000069,
+	0x03F,      0x00000CF4, 0x033,      0x0000006A, 0x03F,      0x00000CF7,
+	0x9000200c, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000060,
+	0x03F,      0x00000005, 0x033,      0x00000061, 0x03F,      0x00000008,
+	0x033,      0x00000062, 0x03F,      0x0000000B, 0x033,      0x00000063,
+	0x03F,      0x0000000E, 0x033,      0x00000064, 0x03F,      0x0000002B,
+	0x033,      0x00000065, 0x03F,      0x00000068, 0x033,      0x00000066,
+	0x03F,      0x0000006B, 0x033,      0x00000067, 0x03F,      0x0000006E,
+	0x033,      0x00000068, 0x03F,      0x00000071, 0x033,      0x00000069,
+	0x03F,      0x00000074, 0x033,      0x0000006A, 0x03F,      0x00000077,
+	0x90001004, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000060,
+	0x03F,      0x00000C0A, 0x033,      0x00000061, 0x03F,      0x00000C0D,
+	0x033,      0x00000062, 0x03F,      0x00000C2A, 0x033,      0x00000063,
+	0x03F,      0x00000C2D, 0x033,      0x00000064, 0x03F,      0x00000C6A,
+	0x033,      0x00000065, 0x03F,      0x00000CAA, 0x033,      0x00000066,
+	0x03F,      0x00000CAD, 0x033,      0x00000067, 0x03F,      0x00000CB0,
+	0x033,      0x00000068, 0x03F,      0x00000CF1, 0x033,      0x00000069,
+	0x03F,      0x00000CF4, 0x033,      0x0000006A, 0x03F,      0x00000CF7,
+	0x93002000, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000060,
+	0x03F,      0x00000429, 0x033,      0x00000061, 0x03F,      0x00000828,
+	0x033,      0x00000062, 0x03F,      0x00000847, 0x033,      0x00000063,
+	0x03F,      0x0000084A, 0x033,      0x00000064, 0x03F,      0x00000C4B,
+	0x033,      0x00000065, 0x03F,      0x00000C8A, 0x033,      0x00000066,
+	0x03F,      0x00000CEA, 0x033,      0x00000067, 0x03F,      0x00000CED,
+	0x033,      0x00000068, 0x03F,      0x00000CF0, 0x033,      0x00000069,
+	0x03F,      0x00000CF3, 0x033,      0x0000006A, 0x03F,      0x00000CF6,
+	0x93001000, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000060,
+	0x03F,      0x00000C0A, 0x033,      0x00000061, 0x03F,      0x00000C0D,
+	0x033,      0x00000062, 0x03F,      0x00000C2A, 0x033,      0x00000063,
+	0x03F,      0x00000C2D, 0x033,      0x00000064, 0x03F,      0x00000C6A,
+	0x033,      0x00000065, 0x03F,      0x00000CAA, 0x033,      0x00000066,
+	0x03F,      0x00000CAD, 0x033,      0x00000067, 0x03F,      0x00000CB0,
+	0x033,      0x00000068, 0x03F,      0x00000CF1, 0x033,      0x00000069,
+	0x03F,      0x00000CF4, 0x033,      0x0000006A, 0x03F,      0x00000CF7,
+	0x90002100, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000060,
+	0x03F,      0x0000042C, 0x033,      0x00000061, 0x03F,      0x0000082B,
+	0x033,      0x00000062, 0x03F,      0x0000084A, 0x033,      0x00000063,
+	0x03F,      0x0000084D, 0x033,      0x00000064, 0x03F,      0x00000C4E,
+	0x033,      0x00000065, 0x03F,      0x00000C8C, 0x033,      0x00000066,
+	0x03F,      0x00000C8F, 0x033,      0x00000067, 0x03F,      0x00000CEC,
+	0x033,      0x00000068, 0x03F,      0x00000CEF, 0x033,      0x00000069,
+	0x03F,      0x00000CF2, 0x033,      0x0000006A, 0x03F,      0x00000CF5,
+	0x90002000, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000060,
+	0x03F,      0x0000042C, 0x033,      0x00000061, 0x03F,      0x0000082B,
+	0x033,      0x00000062, 0x03F,      0x0000084A, 0x033,      0x00000063,
+	0x03F,      0x0000084D, 0x033,      0x00000064, 0x03F,      0x00000C4E,
+	0x033,      0x00000065, 0x03F,      0x00000C8C, 0x033,      0x00000066,
+	0x03F,      0x00000C8F, 0x033,      0x00000067, 0x03F,      0x00000CEC,
+	0x033,      0x00000068, 0x03F,      0x00000CEF, 0x033,      0x00000069,
+	0x03F,      0x00000CF2, 0x033,      0x0000006A, 0x03F,      0x00000CF5,
+	0xA0000000, 0x00000000, 0x033,      0x00000060, 0x03F,      0x00000C0A,
+	0x033,      0x00000061, 0x03F,      0x00000C0D, 0x033,      0x00000062,
+	0x03F,      0x00000C2A, 0x033,      0x00000063, 0x03F,      0x00000C2D,
+	0x033,      0x00000064, 0x03F,      0x00000C6A, 0x033,      0x00000065,
+	0x03F,      0x00000CAA, 0x033,      0x00000066, 0x03F,      0x00000CAD,
+	0x033,      0x00000067, 0x03F,      0x00000CB0, 0x033,      0x00000068,
+	0x03F,      0x00000CF1, 0x033,      0x00000069, 0x03F,      0x00000CF4,
+	0x033,      0x0000006A, 0x03F,      0x00000CF7, 0xB0000000, 0x00000000,
+	0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x033,      0x000000A0,
+	0x03F,      0x00000007, 0x033,      0x000000A1, 0x03F,      0x0000000A,
+	0x033,      0x000000A2, 0x03F,      0x0000000D, 0x033,      0x000000A3,
+	0x03F,      0x0000002A, 0x033,      0x000000A4, 0x03F,      0x0000002D,
+	0x033,      0x000000A5, 0x03F,      0x00000030, 0x033,      0x000000A6,
+	0x03F,      0x0000006D, 0x033,      0x000000A7, 0x03F,      0x00000070,
+	0x033,      0x000000A8, 0x03F,      0x000000ED, 0x033,      0x000000A9,
+	0x03F,      0x000000F0, 0x033,      0x000000AA, 0x03F,      0x000000F3,
+	0x9300100f, 0x05050505, 0x40000000, 0x00000000, 0x033,      0x000000A0,
+	0x03F,      0x00000007, 0x033,      0x000000A1, 0x03F,      0x0000000A,
+	0x033,      0x000000A2, 0x03F,      0x0000000D, 0x033,      0x000000A3,
+	0x03F,      0x0000002A, 0x033,      0x000000A4, 0x03F,      0x0000002D,
+	0x033,      0x000000A5, 0x03F,      0x00000030, 0x033,      0x000000A6,
+	0x03F,      0x0000006D, 0x033,      0x000000A7, 0x03F,      0x00000070,
+	0x033,      0x000000A8, 0x03F,      0x000000ED, 0x033,      0x000000A9,
+	0x03F,      0x000000F0, 0x033,      0x000000AA, 0x03F,      0x000000F3,
+	0x9300100f, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x000000A0,
+	0x03F,      0x00000007, 0x033,      0x000000A1, 0x03F,      0x0000000A,
+	0x033,      0x000000A2, 0x03F,      0x0000000D, 0x033,      0x000000A3,
+	0x03F,      0x0000002A, 0x033,      0x000000A4, 0x03F,      0x0000002D,
+	0x033,      0x000000A5, 0x03F,      0x00000030, 0x033,      0x000000A6,
+	0x03F,      0x0000006D, 0x033,      0x000000A7, 0x03F,      0x00000070,
+	0x033,      0x000000A8, 0x03F,      0x000000ED, 0x033,      0x000000A9,
+	0x03F,      0x000000F0, 0x033,      0x000000AA, 0x03F,      0x000000F3,
+	0x9300200f, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x000000A0,
+	0x03F,      0x00000005, 0x033,      0x000000A1, 0x03F,      0x00000008,
+	0x033,      0x000000A2, 0x03F,      0x0000000B, 0x033,      0x000000A3,
+	0x03F,      0x0000000E, 0x033,      0x000000A4, 0x03F,      0x00000047,
+	0x033,      0x000000A5, 0x03F,      0x0000004A, 0x033,      0x000000A6,
+	0x03F,      0x0000004D, 0x033,      0x000000A7, 0x03F,      0x00000050,
+	0x033,      0x000000A8, 0x03F,      0x00000053, 0x033,      0x000000A9,
+	0x03F,      0x00000056, 0x033,      0x000000AA, 0x03F,      0x00000094,
+	0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x033,      0x000000A0,
+	0x03F,      0x00000007, 0x033,      0x000000A1, 0x03F,      0x0000000A,
+	0x033,      0x000000A2, 0x03F,      0x0000000D, 0x033,      0x000000A3,
+	0x03F,      0x0000002A, 0x033,      0x000000A4, 0x03F,      0x0000002D,
+	0x033,      0x000000A5, 0x03F,      0x00000030, 0x033,      0x000000A6,
+	0x03F,      0x0000006D, 0x033,      0x000000A7, 0x03F,      0x00000070,
+	0x033,      0x000000A8, 0x03F,      0x000000ED, 0x033,      0x000000A9,
+	0x03F,      0x000000F0, 0x033,      0x000000AA, 0x03F,      0x000000F3,
+	0x9000100f, 0x05050505, 0x40000000, 0x00000000, 0x033,      0x000000A0,
+	0x03F,      0x00000007, 0x033,      0x000000A1, 0x03F,      0x0000000A,
+	0x033,      0x000000A2, 0x03F,      0x0000000D, 0x033,      0x000000A3,
+	0x03F,      0x0000002A, 0x033,      0x000000A4, 0x03F,      0x0000002D,
+	0x033,      0x000000A5, 0x03F,      0x00000030, 0x033,      0x000000A6,
+	0x03F,      0x0000006D, 0x033,      0x000000A7, 0x03F,      0x00000070,
+	0x033,      0x000000A8, 0x03F,      0x000000ED, 0x033,      0x000000A9,
+	0x03F,      0x000000F0, 0x033,      0x000000AA, 0x03F,      0x000000F3,
+	0x9000100f, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x000000A0,
+	0x03F,      0x00000007, 0x033,      0x000000A1, 0x03F,      0x0000000A,
+	0x033,      0x000000A2, 0x03F,      0x0000000D, 0x033,      0x000000A3,
+	0x03F,      0x0000002A, 0x033,      0x000000A4, 0x03F,      0x0000002D,
+	0x033,      0x000000A5, 0x03F,      0x00000030, 0x033,      0x000000A6,
+	0x03F,      0x0000006D, 0x033,      0x000000A7, 0x03F,      0x00000070,
+	0x033,      0x000000A8, 0x03F,      0x000000ED, 0x033,      0x000000A9,
+	0x03F,      0x000000F0, 0x033,      0x000000AA, 0x03F,      0x000000F3,
+	0x9000200f, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x000000A0,
+	0x03F,      0x00000005, 0x033,      0x000000A1, 0x03F,      0x00000008,
+	0x033,      0x000000A2, 0x03F,      0x0000000B, 0x033,      0x000000A3,
+	0x03F,      0x0000000E, 0x033,      0x000000A4, 0x03F,      0x00000047,
+	0x033,      0x000000A5, 0x03F,      0x0000004A, 0x033,      0x000000A6,
+	0x03F,      0x0000004D, 0x033,      0x000000A7, 0x03F,      0x00000050,
+	0x033,      0x000000A8, 0x03F,      0x00000053, 0x033,      0x000000A9,
+	0x03F,      0x00000056, 0x033,      0x000000AA, 0x03F,      0x00000094,
+	0x9300200c, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x000000A0,
+	0x03F,      0x00000005, 0x033,      0x000000A1, 0x03F,      0x00000008,
+	0x033,      0x000000A2, 0x03F,      0x0000000B, 0x033,      0x000000A3,
+	0x03F,      0x0000000E, 0x033,      0x000000A4, 0x03F,      0x00000047,
+	0x033,      0x000000A5, 0x03F,      0x0000004A, 0x033,      0x000000A6,
+	0x03F,      0x0000004D, 0x033,      0x000000A7, 0x03F,      0x00000050,
+	0x033,      0x000000A8, 0x03F,      0x00000053, 0x033,      0x000000A9,
+	0x03F,      0x00000056, 0x033,      0x000000AA, 0x03F,      0x00000094,
+	0x93012100, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x000000A0,
+	0x03F,      0x00000826, 0x033,      0x000000A1, 0x03F,      0x00000829,
+	0x033,      0x000000A2, 0x03F,      0x0000082C, 0x033,      0x000000A3,
+	0x03F,      0x0000082F, 0x033,      0x000000A4, 0x03F,      0x0000086C,
+	0x033,      0x000000A5, 0x03F,      0x00000CE8, 0x033,      0x000000A6,
+	0x03F,      0x00000CEB, 0x033,      0x000000A7, 0x03F,      0x00000CEE,
+	0x033,      0x000000A8, 0x03F,      0x00000CF1, 0x033,      0x000000A9,
+	0x03F,      0x00000CF4, 0x033,      0x000000AA, 0x03F,      0x00000CF7,
+	0x93002100, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x000000A0,
+	0x03F,      0x0000042A, 0x033,      0x000000A1, 0x03F,      0x00000829,
+	0x033,      0x000000A2, 0x03F,      0x00000848, 0x033,      0x000000A3,
+	0x03F,      0x0000084B, 0x033,      0x000000A4, 0x03F,      0x00000C4C,
+	0x033,      0x000000A5, 0x03F,      0x00000CA9, 0x033,      0x000000A6,
+	0x03F,      0x00000CEA, 0x033,      0x000000A7, 0x03F,      0x00000CED,
+	0x033,      0x000000A8, 0x03F,      0x00000CF0, 0x033,      0x000000A9,
+	0x03F,      0x00000CF3, 0x033,      0x000000AA, 0x03F,      0x00000CF6,
+	0x93011000, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x000000A0,
+	0x03F,      0x00000C09, 0x033,      0x000000A1, 0x03F,      0x00000C0C,
+	0x033,      0x000000A2, 0x03F,      0x00000C0F, 0x033,      0x000000A3,
+	0x03F,      0x00000C2C, 0x033,      0x000000A4, 0x03F,      0x00000C2F,
+	0x033,      0x000000A5, 0x03F,      0x00000C8A, 0x033,      0x000000A6,
+	0x03F,      0x00000C8D, 0x033,      0x000000A7, 0x03F,      0x00000C90,
+	0x033,      0x000000A8, 0x03F,      0x00000CEF, 0x033,      0x000000A9,
+	0x03F,      0x00000CF2, 0x033,      0x000000AA, 0x03F,      0x00000CF5,
+	0x9000200c, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x000000A0,
+	0x03F,      0x00000005, 0x033,      0x000000A1, 0x03F,      0x00000008,
+	0x033,      0x000000A2, 0x03F,      0x0000000B, 0x033,      0x000000A3,
+	0x03F,      0x0000000E, 0x033,      0x000000A4, 0x03F,      0x00000047,
+	0x033,      0x000000A5, 0x03F,      0x0000004A, 0x033,      0x000000A6,
+	0x03F,      0x0000004D, 0x033,      0x000000A7, 0x03F,      0x00000050,
+	0x033,      0x000000A8, 0x03F,      0x00000053, 0x033,      0x000000A9,
+	0x03F,      0x00000056, 0x033,      0x000000AA, 0x03F,      0x00000094,
+	0x90001004, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x000000A0,
+	0x03F,      0x00000C09, 0x033,      0x000000A1, 0x03F,      0x00000C0C,
+	0x033,      0x000000A2, 0x03F,      0x00000C0F, 0x033,      0x000000A3,
+	0x03F,      0x00000C2C, 0x033,      0x000000A4, 0x03F,      0x00000C2F,
+	0x033,      0x000000A5, 0x03F,      0x00000C8A, 0x033,      0x000000A6,
+	0x03F,      0x00000C8D, 0x033,      0x000000A7, 0x03F,      0x00000C90,
+	0x033,      0x000000A8, 0x03F,      0x00000CEF, 0x033,      0x000000A9,
+	0x03F,      0x00000CF2, 0x033,      0x000000AA, 0x03F,      0x00000CF5,
+	0x93002000, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x000000A0,
+	0x03F,      0x00000429, 0x033,      0x000000A1, 0x03F,      0x00000828,
+	0x033,      0x000000A2, 0x03F,      0x00000847, 0x033,      0x000000A3,
+	0x03F,      0x0000084A, 0x033,      0x000000A4, 0x03F,      0x00000C4B,
+	0x033,      0x000000A5, 0x03F,      0x00000C8A, 0x033,      0x000000A6,
+	0x03F,      0x00000CEA, 0x033,      0x000000A7, 0x03F,      0x00000CED,
+	0x033,      0x000000A8, 0x03F,      0x00000CF0, 0x033,      0x000000A9,
+	0x03F,      0x00000CF3, 0x033,      0x000000AA, 0x03F,      0x00000CF6,
+	0x93001000, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x000000A0,
+	0x03F,      0x00000C09, 0x033,      0x000000A1, 0x03F,      0x00000C0C,
+	0x033,      0x000000A2, 0x03F,      0x00000C0F, 0x033,      0x000000A3,
+	0x03F,      0x00000C2C, 0x033,      0x000000A4, 0x03F,      0x00000C2F,
+	0x033,      0x000000A5, 0x03F,      0x00000C8A, 0x033,      0x000000A6,
+	0x03F,      0x00000C8D, 0x033,      0x000000A7, 0x03F,      0x00000C90,
+	0x033,      0x000000A8, 0x03F,      0x00000CEF, 0x033,      0x000000A9,
+	0x03F,      0x00000CF2, 0x033,      0x000000AA, 0x03F,      0x00000CF5,
+	0x90002100, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x000000A0,
+	0x03F,      0x0000042A, 0x033,      0x000000A1, 0x03F,      0x00000829,
+	0x033,      0x000000A2, 0x03F,      0x00000848, 0x033,      0x000000A3,
+	0x03F,      0x0000084B, 0x033,      0x000000A4, 0x03F,      0x00000C4C,
+	0x033,      0x000000A5, 0x03F,      0x00000C8A, 0x033,      0x000000A6,
+	0x03F,      0x00000C8D, 0x033,      0x000000A7, 0x03F,      0x00000CEC,
+	0x033,      0x000000A8, 0x03F,      0x00000CEF, 0x033,      0x000000A9,
+	0x03F,      0x00000CF2, 0x033,      0x000000AA, 0x03F,      0x00000CF5,
+	0x90002000, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x000000A0,
+	0x03F,      0x0000042A, 0x033,      0x000000A1, 0x03F,      0x00000829,
+	0x033,      0x000000A2, 0x03F,      0x00000848, 0x033,      0x000000A3,
+	0x03F,      0x0000084B, 0x033,      0x000000A4, 0x03F,      0x00000C4C,
+	0x033,      0x000000A5, 0x03F,      0x00000C8A, 0x033,      0x000000A6,
+	0x03F,      0x00000C8D, 0x033,      0x000000A7, 0x03F,      0x00000CEC,
+	0x033,      0x000000A8, 0x03F,      0x00000CEF, 0x033,      0x000000A9,
+	0x03F,      0x00000CF2, 0x033,      0x000000AA, 0x03F,      0x00000CF5,
+	0xA0000000, 0x00000000, 0x033,      0x000000A0, 0x03F,      0x00000C09,
+	0x033,      0x000000A1, 0x03F,      0x00000C0C, 0x033,      0x000000A2,
+	0x03F,      0x00000C0F, 0x033,      0x000000A3, 0x03F,      0x00000C2C,
+	0x033,      0x000000A4, 0x03F,      0x00000C2F, 0x033,      0x000000A5,
+	0x03F,      0x00000C8A, 0x033,      0x000000A6, 0x03F,      0x00000C8D,
+	0x033,      0x000000A7, 0x03F,      0x00000C90, 0x033,      0x000000A8,
+	0x03F,      0x00000CEF, 0x033,      0x000000A9, 0x03F,      0x00000CF2,
+	0x033,      0x000000AA, 0x03F,      0x00000CF5, 0xB0000000, 0x00000000,
+	0x0EF,      0x00000000, 0x0EF,      0x00000400, 0x8300100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x033,      0x00000000, 0x03F,      0x0000265A,
+	0x033,      0x00000001, 0x03F,      0x0000265A, 0x033,      0x00000002,
+	0x03F,      0x0000265A, 0x033,      0x00000003, 0x03F,      0x0000265A,
+	0x9300100f, 0x05050505, 0x40000000, 0x00000000, 0x033,      0x00000000,
+	0x03F,      0x0000265A, 0x033,      0x00000001, 0x03F,      0x0000265A,
+	0x033,      0x00000002, 0x03F,      0x0000265A, 0x033,      0x00000003,
+	0x03F,      0x0000265A, 0x9300100f, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000000, 0x03F,      0x0000265A, 0x033,      0x00000001,
+	0x03F,      0x0000265A, 0x033,      0x00000002, 0x03F,      0x0000265A,
+	0x033,      0x00000003, 0x03F,      0x0000265A, 0x9300200f, 0x00000000,
+	0x40000000, 0x00000000, 0x033,      0x00000000, 0x03F,      0x0000265A,
+	0x033,      0x00000001, 0x03F,      0x0000265A, 0x033,      0x00000002,
+	0x03F,      0x0000265A, 0x033,      0x00000003, 0x03F,      0x0000265A,
+	0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000, 0x033,      0x00000000,
+	0x03F,      0x0000265A, 0x033,      0x00000001, 0x03F,      0x0000265A,
+	0x033,      0x00000002, 0x03F,      0x0000265A, 0x033,      0x00000003,
+	0x03F,      0x0000265A, 0x9000100f, 0x05050505, 0x40000000, 0x00000000,
+	0x033,      0x00000000, 0x03F,      0x0000265A, 0x033,      0x00000001,
+	0x03F,      0x0000265A, 0x033,      0x00000002, 0x03F,      0x0000265A,
+	0x033,      0x00000003, 0x03F,      0x0000265A, 0x9000100f, 0x00000000,
+	0x40000000, 0x00000000, 0x033,      0x00000000, 0x03F,      0x0000265A,
+	0x033,      0x00000001, 0x03F,      0x0000265A, 0x033,      0x00000002,
+	0x03F,      0x0000265A, 0x033,      0x00000003, 0x03F,      0x0000265A,
+	0x9000200f, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000000,
+	0x03F,      0x0000265A, 0x033,      0x00000001, 0x03F,      0x0000265A,
+	0x033,      0x00000002, 0x03F,      0x0000265A, 0x033,      0x00000003,
+	0x03F,      0x0000265A, 0xA0000000, 0x00000000, 0x033,      0x00000000,
+	0x03F,      0x000004BB, 0x033,      0x00000001, 0x03F,      0x000004BB,
+	0x033,      0x00000002, 0x03F,      0x000004BB, 0x033,      0x00000003,
+	0x03F,      0x000004BB, 0xB0000000, 0x00000000, 0x0EF,      0x00000000,
+	0x0EF,      0x00000100, 0x8300100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x033,      0x00000000, 0x03F,      0x00000745, 0x033,      0x00000001,
+	0x03F,      0x00000745, 0x033,      0x00000002, 0x03F,      0x00000745,
+	0x033,      0x00000003, 0x03F,      0x00000745, 0x9300100f, 0x05050505,
+	0x40000000, 0x00000000, 0x033,      0x00000000, 0x03F,      0x00000745,
+	0x033,      0x00000001, 0x03F,      0x00000745, 0x033,      0x00000002,
+	0x03F,      0x00000745, 0x033,      0x00000003, 0x03F,      0x00000745,
+	0x9300100f, 0x00000000, 0x40000000, 0x00000000, 0x033,      0x00000000,
+	0x03F,      0x00000745, 0x033,      0x00000001, 0x03F,      0x00000745,
+	0x033,      0x00000002, 0x03F,      0x00000745, 0x033,      0x00000003,
+	0x03F,      0x00000745, 0x9300200f, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000000, 0x03F,      0x00000745, 0x033,      0x00000001,
+	0x03F,      0x00000745, 0x033,      0x00000002, 0x03F,      0x00000745,
+	0x033,      0x00000003, 0x03F,      0x00000745, 0x9000100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x033,      0x00000000, 0x03F,      0x00000745,
+	0x033,      0x00000001, 0x03F,      0x00000745, 0x033,      0x00000002,
+	0x03F,      0x00000745, 0x033,      0x00000003, 0x03F,      0x00000745,
+	0x9000100f, 0x05050505, 0x40000000, 0x00000000, 0x033,      0x00000000,
+	0x03F,      0x00000745, 0x033,      0x00000001, 0x03F,      0x00000745,
+	0x033,      0x00000002, 0x03F,      0x00000745, 0x033,      0x00000003,
+	0x03F,      0x00000745, 0x9000100f, 0x00000000, 0x40000000, 0x00000000,
+	0x033,      0x00000000, 0x03F,      0x00000745, 0x033,      0x00000001,
+	0x03F,      0x00000745, 0x033,      0x00000002, 0x03F,      0x00000745,
+	0x033,      0x00000003, 0x03F,      0x00000745, 0x9000200f, 0x00000000,
+	0x40000000, 0x00000000, 0x033,      0x00000000, 0x03F,      0x00000745,
+	0x033,      0x00000001, 0x03F,      0x00000745, 0x033,      0x00000002,
+	0x03F,      0x00000745, 0x033,      0x00000003, 0x03F,      0x00000745,
+	0xA0000000, 0x00000000, 0x033,      0x00000000, 0x03F,      0x00000F34,
+	0x033,      0x00000001, 0x03F,      0x00000F34, 0x033,      0x00000002,
+	0x03F,      0x00000F34, 0x033,      0x00000003, 0x03F,      0x00000F34,
+	0xB0000000, 0x00000000, 0x0EF,      0x00000000, 0x8300100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x081,      0x0000F400, 0x087,      0x00016040,
+	0x051,      0x00000808, 0x052,      0x00098002, 0x053,      0x0000FA47,
+	0x054,      0x00058032, 0x056,      0x00051000, 0x057,      0x0000CE0A,
+	0x058,      0x00082030, 0x9300100f, 0x05050505, 0x40000000, 0x00000000,
+	0x081,      0x0000F400, 0x087,      0x00016040, 0x051,      0x00000808,
+	0x052,      0x00098002, 0x053,      0x0000FA47, 0x054,      0x00058032,
+	0x056,      0x00051000, 0x057,      0x0000CE0A, 0x058,      0x00082030,
+	0x9300100f, 0x00000000, 0x40000000, 0x00000000, 0x081,      0x0000F400,
+	0x087,      0x00016040, 0x051,      0x00000808, 0x052,      0x00098002,
+	0x053,      0x0000FA47, 0x054,      0x00058032, 0x056,      0x00051000,
+	0x057,      0x0000CE0A, 0x058,      0x00082030, 0x9300200f, 0x00000000,
+	0x40000000, 0x00000000, 0x081,      0x0000F400, 0x087,      0x00016040,
+	0x051,      0x00000808, 0x052,      0x00098002, 0x053,      0x0000FA47,
+	0x054,      0x00058032, 0x056,      0x00051000, 0x057,      0x0000CE0A,
+	0x058,      0x00082030, 0x9000100f, 0x0a0a0a0a, 0x40000000, 0x00000000,
+	0x081,      0x0000F400, 0x087,      0x00016040, 0x051,      0x00000808,
+	0x052,      0x00098002, 0x053,      0x0000FA47, 0x054,      0x00058032,
+	0x056,      0x00051000, 0x057,      0x0000CE0A, 0x058,      0x00082030,
+	0x9000100f, 0x05050505, 0x40000000, 0x00000000, 0x081,      0x0000F400,
+	0x087,      0x00016040, 0x051,      0x00000808, 0x052,      0x00098002,
+	0x053,      0x0000FA47, 0x054,      0x00058032, 0x056,      0x00051000,
+	0x057,      0x0000CE0A, 0x058,      0x00082030, 0x9000100f, 0x00000000,
+	0x40000000, 0x00000000, 0x081,      0x0000F400, 0x087,      0x00016040,
+	0x051,      0x00000808, 0x052,      0x00098002, 0x053,      0x0000FA47,
+	0x054,      0x00058032, 0x056,      0x00051000, 0x057,      0x0000CE0A,
+	0x058,      0x00082030, 0x9000200f, 0x00000000, 0x40000000, 0x00000000,
+	0x081,      0x0000F400, 0x087,      0x00016040, 0x051,      0x00000808,
+	0x052,      0x00098002, 0x053,      0x0000FA47, 0x054,      0x00058032,
+	0x056,      0x00051000, 0x057,      0x0000CE0A, 0x058,      0x00082030,
+	0xA0000000, 0x00000000, 0x081,      0x0000F000, 0x087,      0x00016040,
+	0x051,      0x00000C00, 0x052,      0x0007C241, 0x053,      0x0001C069,
+	0x054,      0x00078032, 0x057,      0x0000CE0A, 0x058,      0x00058750,
+	0xB0000000, 0x00000000, 0x0EF,      0x00000800, 0x8300100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x033,      0x00000000, 0x03F,      0x00000003,
+	0x033,      0x00000001, 0x03F,      0x00000006, 0x033,      0x00000002,
+	0x03F,      0x00000009, 0x033,      0x00000003, 0x03F,      0x00000026,
+	0x033,      0x00000004, 0x03F,      0x00000029, 0x033,      0x00000005,
+	0x03F,      0x0000002C, 0x033,      0x00000006, 0x03F,      0x0000002F,
+	0x033,      0x00000007, 0x03F,      0x00000033, 0x033,      0x00000008,
+	0x03F,      0x00000036, 0x033,      0x00000009, 0x03F,      0x00000039,
+	0x033,      0x0000000A, 0x03F,      0x0000003C, 0x9300100f, 0x05050505,
+	0x40000000, 0x00000000, 0x033,      0x00000000, 0x03F,      0x00000003,
+	0x033,      0x00000001, 0x03F,      0x00000006, 0x033,      0x00000002,
+	0x03F,      0x00000009, 0x033,      0x00000003, 0x03F,      0x00000026,
+	0x033,      0x00000004, 0x03F,      0x00000029, 0x033,      0x00000005,
+	0x03F,      0x0000002C, 0x033,      0x00000006, 0x03F,      0x0000002F,
+	0x033,      0x00000007, 0x03F,      0x00000033, 0x033,      0x00000008,
+	0x03F,      0x00000036, 0x033,      0x00000009, 0x03F,      0x00000039,
+	0x033,      0x0000000A, 0x03F,      0x0000003C, 0x9300100f, 0x00000000,
+	0x40000000, 0x00000000, 0x033,      0x00000000, 0x03F,      0x00000003,
+	0x033,      0x00000001, 0x03F,      0x00000006, 0x033,      0x00000002,
+	0x03F,      0x00000009, 0x033,      0x00000003, 0x03F,      0x00000026,
+	0x033,      0x00000004, 0x03F,      0x00000029, 0x033,      0x00000005,
+	0x03F,      0x0000002C, 0x033,      0x00000006, 0x03F,      0x0000002F,
+	0x033,      0x00000007, 0x03F,      0x00000033, 0x033,      0x00000008,
+	0x03F,      0x00000036, 0x033,      0x00000009, 0x03F,      0x00000039,
+	0x033,      0x0000000A, 0x03F,      0x0000003C, 0x9300200f, 0x00000000,
+	0x40000000, 0x00000000, 0x033,      0x00000000, 0x03F,      0x00000003,
+	0x033,      0x00000001, 0x03F,      0x00000006, 0x033,      0x00000002,
+	0x03F,      0x00000009, 0x033,      0x00000003, 0x03F,      0x00000026,
+	0x033,      0x00000004, 0x03F,      0x00000029, 0x033,      0x00000005,
+	0x03F,      0x0000002C, 0x033,      0x00000006, 0x03F,      0x0000002F,
+	0x033,      0x00000007, 0x03F,      0x00000033, 0x033,      0x00000008,
+	0x03F,      0x00000036, 0x033,      0x00000009, 0x03F,      0x00000039,
+	0x033,      0x0000000A, 0x03F,      0x0000003C, 0x9000100f, 0x0a0a0a0a,
+	0x40000000, 0x00000000, 0x033,      0x00000000, 0x03F,      0x00000003,
+	0x033,      0x00000001, 0x03F,      0x00000006, 0x033,      0x00000002,
+	0x03F,      0x00000009, 0x033,      0x00000003, 0x03F,      0x00000026,
+	0x033,      0x00000004, 0x03F,      0x00000029, 0x033,      0x00000005,
+	0x03F,      0x0000002C, 0x033,      0x00000006, 0x03F,      0x0000002F,
+	0x033,      0x00000007, 0x03F,      0x00000033, 0x033,      0x00000008,
+	0x03F,      0x00000036, 0x033,      0x00000009, 0x03F,      0x00000039,
+	0x033,      0x0000000A, 0x03F,      0x0000003C, 0x9000100f, 0x05050505,
+	0x40000000, 0x00000000, 0x033,      0x00000000, 0x03F,      0x00000003,
+	0x033,      0x00000001, 0x03F,      0x00000006, 0x033,      0x00000002,
+	0x03F,      0x00000009, 0x033,      0x00000003, 0x03F,      0x00000026,
+	0x033,      0x00000004, 0x03F,      0x00000029, 0x033,      0x00000005,
+	0x03F,      0x0000002C, 0x033,      0x00000006, 0x03F,      0x0000002F,
+	0x033,      0x00000007, 0x03F,      0x00000033, 0x033,      0x00000008,
+	0x03F,      0x00000036, 0x033,      0x00000009, 0x03F,      0x00000039,
+	0x033,      0x0000000A, 0x03F,      0x0000003C, 0x9000100f, 0x00000000,
+	0x40000000, 0x00000000, 0x033,      0x00000000, 0x03F,      0x00000003,
+	0x033,      0x00000001, 0x03F,      0x00000006, 0x033,      0x00000002,
+	0x03F,      0x00000009, 0x033,      0x00000003, 0x03F,      0x00000026,
+	0x033,      0x00000004, 0x03F,      0x00000029, 0x033,      0x00000005,
+	0x03F,      0x0000002C, 0x033,      0x00000006, 0x03F,      0x0000002F,
+	0x033,      0x00000007, 0x03F,      0x00000033, 0x033,      0x00000008,
+	0x03F,      0x00000036, 0x033,      0x00000009, 0x03F,      0x00000039,
+	0x033,      0x0000000A, 0x03F,      0x0000003C, 0x9000200f, 0x00000000,
+	0x40000000, 0x00000000, 0x033,      0x00000000, 0x03F,      0x00000003,
+	0x033,      0x00000001, 0x03F,      0x00000006, 0x033,      0x00000002,
+	0x03F,      0x00000009, 0x033,      0x00000003, 0x03F,      0x00000026,
+	0x033,      0x00000004, 0x03F,      0x00000029, 0x033,      0x00000005,
+	0x03F,      0x0000002C, 0x033,      0x00000006, 0x03F,      0x0000002F,
+	0x033,      0x00000007, 0x03F,      0x00000033, 0x033,      0x00000008,
+	0x03F,      0x00000036, 0x033,      0x00000009, 0x03F,      0x00000039,
+	0x033,      0x0000000A, 0x03F,      0x0000003C, 0xA0000000, 0x00000000,
+	0x033,      0x00000000, 0x03F,      0x0005142C, 0x033,      0x00000001,
+	0x03F,      0x0005142F, 0x033,      0x00000002, 0x03F,      0x00051432,
+	0x033,      0x00000003, 0x03F,      0x00051C87, 0x033,      0x00000004,
+	0x03F,      0x00051C8A, 0x033,      0x00000005, 0x03F,      0x00051C8D,
+	0x033,      0x00000006, 0x03F,      0x00051CEB, 0x033,      0x00000007,
+	0x03F,      0x00051CEE, 0x033,      0x00000008, 0x03F,      0x00051CF1,
+	0x033,      0x00000009, 0x03F,      0x00051CF4, 0x033,      0x0000000A,
+	0x03F,      0x00051CF7, 0xB0000000, 0x00000000, 0x0EF,      0x00000000,
+	0x0EF,      0x00000010, 0x033,      0x00000000, 0x008,      0x0009C060,
+	0x033,      0x00000001, 0x008,      0x0009C060, 0x0EF,      0x00000000,
+	0x033,      0x000000A2, 0x0EF,      0x00080000, 0x03E,      0x0000593F,
+	0x03F,      0x000C0F4F, 0x0EF,      0x00000000, 0x033,      0x000000A3,
+	0x0EF,      0x00080000, 0x03E,      0x00005934, 0x03F,      0x0005AFCF,
+	0x0EF,      0x00000000,
+
+};
+
+void odm_read_and_config_mp_8822b_radiob(struct phy_dm_struct *dm)
+{
+	u32 i = 0;
+	u8 c_cond;
+	bool is_matched = true, is_skipped = false;
+	u32 array_len = sizeof(array_mp_8822b_radiob) / sizeof(u32);
+	u32 *array = array_mp_8822b_radiob;
+
+	u32 v1 = 0, v2 = 0, pre_v1 = 0, pre_v2 = 0;
+
+	ODM_RT_TRACE(dm, ODM_COMP_INIT,
+		     "===> %s\n", __func__);
+
+	for (; (i + 1) < array_len; i = i + 2) {
+		v1 = array[i];
+		v2 = array[i + 1];
+
+		if (v1 & BIT(31)) { /* positive condition*/
+			c_cond = (u8)((v1 & (BIT(29) | BIT(28))) >> 28);
+			if (c_cond == COND_ENDIF) { /*end*/
+				is_matched = true;
+				is_skipped = false;
+				ODM_RT_TRACE(dm, ODM_COMP_INIT, "ENDIF\n");
+			} else if (c_cond == COND_ELSE) { /*else*/
+				is_matched = is_skipped ? false : true;
+				ODM_RT_TRACE(dm, ODM_COMP_INIT, "ELSE\n");
+			} else { /*if , else if*/
+				pre_v1 = v1;
+				pre_v2 = v2;
+				ODM_RT_TRACE(dm, ODM_COMP_INIT,
+					     "IF or ELSE IF\n");
+			}
+		} else if (v1 & BIT(30)) { /*negative condition*/
+			if (is_skipped) {
+				is_matched = false;
+				continue;
+			}
+
+			if (check_positive(dm, pre_v1, pre_v2, v1, v2)) {
+				is_matched = true;
+				is_skipped = true;
+			} else {
+				is_matched = false;
+				is_skipped = false;
+			}
+		} else if (is_matched) {
+			odm_config_rf_radio_b_8822b(dm, v1, v2);
+		}
+	}
+}
+
+u32 odm_get_version_mp_8822b_radiob(void) { return 67; }
+
+/******************************************************************************
+ *                           txpowertrack.TXT
+ ******************************************************************************/
+
+static u8 delta_swing_index_mp_5gb_n_txpwrtrack_8822b[][DELTA_SWINGIDX_SIZE] = {
+	{0,  1,  2,  3,  3,  4,  5,  6,  6,  7,  8,  9,  9,  10, 10,
+	 11, 11, 12, 12, 12, 13, 13, 14, 14, 14, 15, 15, 15, 15, 15},
+	{0, 1,  1,  2,  2,  3,  3,  4,  5,  5,  6,  7,  7,  8,  8,
+	 9, 10, 10, 11, 12, 12, 13, 13, 14, 14, 14, 14, 14, 14, 14},
+	{0, 1,  2,  2,  3,  3,  4,  4,  5,  6,  6,  7,  7,  8,  9,
+	 9, 10, 10, 11, 12, 12, 13, 13, 14, 14, 14, 14, 14, 14, 14},
+};
+
+static u8 delta_swing_index_mp_5gb_p_txpwrtrack_8822b[][DELTA_SWINGIDX_SIZE] = {
+	{0,  1,  2,  2,  3,  4,  5,  6,  7,  8,  8,  9,  10, 11, 11,
+	 12, 13, 14, 15, 15, 16, 17, 18, 18, 19, 19, 19, 19, 19, 19},
+	{0,  1,  2,  2,  3,  4,  5,  6,  6,  7,  8,  8,  9,  9,  10,
+	 11, 12, 12, 13, 14, 15, 16, 17, 17, 18, 18, 18, 18, 18, 18},
+	{0,  1,  2,  2,  3,  4,  5,  5,  6,  6,  7,  8,  8,  9,  10,
+	 10, 11, 12, 13, 14, 15, 15, 16, 16, 17, 17, 17, 17, 17, 17},
+};
+
+static u8 delta_swing_index_mp_5ga_n_txpwrtrack_8822b[][DELTA_SWINGIDX_SIZE] = {
+	{0,  1,  2,  2,  3,  3,  4,  5,  6,  7,  8,  8,  9,  9,  10,
+	 11, 11, 12, 12, 12, 13, 13, 14, 14, 14, 15, 15, 15, 15, 15},
+	{0,  1,  2,  2,  3,  4,  5,  5,  6,  6,  7,  7,  8,  8,  9,
+	 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 14, 14, 14, 14, 14},
+	{0, 1,  2,  2,  3,  4,  4,  5,  6,  6,  7,  7,  8,  8,  9,
+	 9, 10, 10, 11, 12, 12, 13, 13, 14, 14, 14, 14, 14, 14, 14},
+};
+
+static u8 delta_swing_index_mp_5ga_p_txpwrtrack_8822b[][DELTA_SWINGIDX_SIZE] = {
+	{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  9,  9,  10, 11,
+	 12, 13, 14, 14, 15, 16, 17, 18, 19, 19, 20, 20, 20, 20, 20},
+	{0,  1,  2,  2,  3,  4,  4,  5,  6,  6,  7,  7,  8,  9,  9,
+	 10, 11, 11, 12, 13, 14, 15, 16, 16, 17, 17, 18, 18, 18, 18},
+	{0,  1,  2,  3,  3,  4,  5,  5,  6,  6,  7,  8,  8,  9,  10,
+	 11, 12, 12, 13, 14, 15, 15, 16, 17, 17, 18, 18, 18, 18, 18},
+};
+
+static u8 delta_swing_index_mp_2gb_n_txpwrtrack_8822b[] = {
+	0,  1,  2,  3,  3,  4,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+	13, 13, 14, 15, 16, 16, 17, 18, 18, 18, 18, 18, 18, 18, 18};
+
+static u8 delta_swing_index_mp_2gb_p_txpwrtrack_8822b[] = {
+	0,  1,  1,  2,  3,  4,  4,  5,  6,  7,  7,  8,  9,  10, 11,
+	12, 12, 13, 14, 15, 16, 17, 17, 18, 19, 20, 21, 22, 22, 22};
+
+static u8 delta_swing_index_mp_2ga_n_txpwrtrack_8822b[] = {
+	0,  1,  2,  3,  3,  4,  5,  6,  6,  7,  8,  9,  10, 11, 12,
+	13, 13, 14, 15, 16, 17, 18, 18, 18, 18, 18, 18, 18, 18, 18};
+
+static u8 delta_swing_index_mp_2ga_p_txpwrtrack_8822b[] = {
+	0,  1,  2,  3,  3,  4,  5,  6,  6,  7,  8,  9,  9,  10, 11,
+	12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 22};
+
+static u8 delta_swing_index_mp_2g_cck_b_n_txpwrtrack_8822b[] = {
+	0,  1,  1,  2,  3,  4,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+	13, 13, 14, 15, 16, 16, 17, 17, 17, 17, 17, 17, 17, 17, 17};
+
+static u8 delta_swing_index_mp_2g_cck_b_p_txpwrtrack_8822b[] = {
+	0,  1,  1,  2,  3,  4,  4,  5,  6,  7,  8,  9,  9,  10, 11,
+	12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 22};
+
+static u8 delta_swing_index_mp_2g_cck_a_n_txpwrtrack_8822b[] = {
+	0,  1,  2,  3,  3,  4,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+	13, 13, 14, 15, 16, 17, 18, 18, 18, 18, 18, 18, 18, 18, 18};
+
+static u8 delta_swing_index_mp_2g_cck_a_p_txpwrtrack_8822b[] = {
+	0,  1,  2,  3,  3,  4,  5,  6,  6,  7,  8,  9,  9,  10, 11,
+	12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 22};
+
+void odm_read_and_config_mp_8822b_txpowertrack(struct phy_dm_struct *dm)
+{
+	struct dm_rf_calibration_struct *cali_info = &dm->rf_calibrate_info;
+
+	ODM_RT_TRACE(dm, ODM_COMP_INIT, "===> ODM_ReadAndConfig_MP_mp_8822b\n");
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2ga_p,
+			delta_swing_index_mp_2ga_p_txpwrtrack_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2ga_n,
+			delta_swing_index_mp_2ga_n_txpwrtrack_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2gb_p,
+			delta_swing_index_mp_2gb_p_txpwrtrack_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2gb_n,
+			delta_swing_index_mp_2gb_n_txpwrtrack_8822b,
+			DELTA_SWINGIDX_SIZE);
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_a_p,
+			delta_swing_index_mp_2g_cck_a_p_txpwrtrack_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_a_n,
+			delta_swing_index_mp_2g_cck_a_n_txpwrtrack_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_b_p,
+			delta_swing_index_mp_2g_cck_b_p_txpwrtrack_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_b_n,
+			delta_swing_index_mp_2g_cck_b_n_txpwrtrack_8822b,
+			DELTA_SWINGIDX_SIZE);
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5ga_p,
+			delta_swing_index_mp_5ga_p_txpwrtrack_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5ga_n,
+			delta_swing_index_mp_5ga_n_txpwrtrack_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5gb_p,
+			delta_swing_index_mp_5gb_p_txpwrtrack_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5gb_n,
+			delta_swing_index_mp_5gb_n_txpwrtrack_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+}
+
+/******************************************************************************
+ *                           txpowertrack_type0.TXT
+ ******************************************************************************/
+
+static u8 delta_swing_index_mp_5gb_n_txpwrtrack_type0_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  7,
+		 8, 8, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15},
+		{0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  8,
+		 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 14, 14},
+		{0, 1, 1,  2,  2,  3,  3,  4,  5,  5,  6,  7,  7,  8,  8,
+		 9, 9, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 15, 15},
+};
+
+static u8 delta_swing_index_mp_5gb_p_txpwrtrack_type0_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0, 1, 1, 2, 2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  7,
+		 8, 8, 9, 9, 10, 10, 11, 12, 12, 13, 13, 14, 14, 15, 15},
+		{0, 1, 1, 2, 2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  7,
+		 8, 8, 9, 9, 10, 11, 11, 12, 13, 13, 14, 14, 15, 15, 15},
+		{0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  7,
+		 8, 8, 9, 10, 10, 11, 12, 13, 14, 14, 15, 15, 15, 16, 16},
+};
+
+static u8 delta_swing_index_mp_5ga_n_txpwrtrack_type0_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0, 1, 1, 2, 2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  7,
+		 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 13, 14, 14},
+		{0, 1, 1, 2, 2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  7,
+		 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 14},
+		{0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  7,
+		 8, 8, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 14, 14},
+};
+
+static u8 delta_swing_index_mp_5ga_p_txpwrtrack_type0_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0, 1, 1, 2, 2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  7,
+		 8, 8, 9, 9, 10, 10, 11, 12, 13, 13, 14, 14, 15, 15, 15},
+		{0, 1, 1, 2, 2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  7,
+		 8, 8, 9, 9, 10, 10, 11, 12, 12, 13, 13, 14, 14, 15, 15},
+		{0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  7,
+		 8, 9, 9, 10, 10, 11, 11, 12, 13, 13, 14, 14, 15, 15, 15},
+};
+
+static u8 delta_swing_index_mp_2gb_n_txpwrtrack_type0_8822b[] = {
+	0,  1,  2,  3,  3,  4,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+	13, 13, 14, 15, 16, 16, 17, 18, 18, 18, 18, 18, 18, 18, 18};
+
+static u8 delta_swing_index_mp_2gb_p_txpwrtrack_type0_8822b[] = {
+	0,  1,  1,  2,  3,  4,  4,  5,  6,  7,  7,  8,  9,  10, 11,
+	12, 12, 13, 14, 15, 16, 17, 17, 18, 19, 20, 21, 22, 22, 22};
+
+static u8 delta_swing_index_mp_2ga_n_txpwrtrack_type0_8822b[] = {
+	0,  1,  2,  3,  3,  4,  5,  6,  6,  7,  8,  9,  10, 11, 12,
+	13, 13, 14, 15, 16, 17, 18, 18, 18, 18, 18, 18, 18, 18, 18};
+
+static u8 delta_swing_index_mp_2ga_p_txpwrtrack_type0_8822b[] = {
+	0,  1,  2,  3,  3,  4,  5,  6,  6,  7,  8,  9,  9,  10, 11,
+	12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 22};
+
+static u8 delta_swing_index_mp_2g_cck_b_n_txpwrtrack_type0_8822b[] = {
+	0,  1,  1,  2,  3,  4,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+	13, 13, 14, 15, 16, 16, 17, 17, 17, 17, 17, 17, 17, 17, 17};
+
+static u8 delta_swing_index_mp_2g_cck_b_p_txpwrtrack_type0_8822b[] = {
+	0,  1,  1,  2,  3,  4,  4,  5,  6,  7,  8,  9,  9,  10, 11,
+	12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 22};
+
+static u8 delta_swing_index_mp_2g_cck_a_n_txpwrtrack_type0_8822b[] = {
+	0,  1,  2,  3,  3,  4,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+	13, 13, 14, 15, 16, 17, 18, 18, 18, 18, 18, 18, 18, 18, 18};
+
+static u8 delta_swing_index_mp_2g_cck_a_p_txpwrtrack_type0_8822b[] = {
+	0,  1,  2,  3,  3,  4,  5,  6,  6,  7,  8,  9,  9,  10, 11,
+	12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 22};
+
+void odm_read_and_config_mp_8822b_txpowertrack_type0(struct phy_dm_struct *dm)
+{
+	struct dm_rf_calibration_struct *cali_info = &dm->rf_calibrate_info;
+
+	ODM_RT_TRACE(dm, ODM_COMP_INIT, "===> ODM_ReadAndConfig_MP_mp_8822b\n");
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2ga_p,
+			delta_swing_index_mp_2ga_p_txpwrtrack_type0_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2ga_n,
+			delta_swing_index_mp_2ga_n_txpwrtrack_type0_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2gb_p,
+			delta_swing_index_mp_2gb_p_txpwrtrack_type0_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2gb_n,
+			delta_swing_index_mp_2gb_n_txpwrtrack_type0_8822b,
+			DELTA_SWINGIDX_SIZE);
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_a_p,
+			delta_swing_index_mp_2g_cck_a_p_txpwrtrack_type0_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_a_n,
+			delta_swing_index_mp_2g_cck_a_n_txpwrtrack_type0_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_b_p,
+			delta_swing_index_mp_2g_cck_b_p_txpwrtrack_type0_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_b_n,
+			delta_swing_index_mp_2g_cck_b_n_txpwrtrack_type0_8822b,
+			DELTA_SWINGIDX_SIZE);
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5ga_p,
+			delta_swing_index_mp_5ga_p_txpwrtrack_type0_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5ga_n,
+			delta_swing_index_mp_5ga_n_txpwrtrack_type0_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5gb_p,
+			delta_swing_index_mp_5gb_p_txpwrtrack_type0_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5gb_n,
+			delta_swing_index_mp_5gb_n_txpwrtrack_type0_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+}
+
+/******************************************************************************
+ *                           txpowertrack_type1.TXT
+ ******************************************************************************/
+
+static u8 delta_swing_index_mp_5gb_n_txpwrtrack_type1_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0,  1,  2,  3,  3,  4,  5,  6,  6,  7,  8,  9,  9,  10, 10,
+		 11, 11, 12, 12, 12, 13, 13, 14, 14, 14, 15, 15, 15, 15, 15},
+		{0, 1,  1,  2,  2,  3,  3,  4,  5,  5,  6,  7,  7,  8,  8,
+		 9, 10, 10, 11, 12, 12, 13, 13, 14, 14, 14, 14, 14, 14, 14},
+		{0, 1,  2,  2,  3,  3,  4,  4,  5,  6,  6,  7,  7,  8,  9,
+		 9, 10, 10, 11, 12, 12, 13, 13, 14, 14, 14, 14, 14, 14, 14},
+};
+
+static u8 delta_swing_index_mp_5gb_p_txpwrtrack_type1_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0,  1,  2,  2,  3,  4,  5,  6,  7,  8,  8,  9,  10, 11, 11,
+		 12, 13, 14, 15, 15, 16, 17, 18, 18, 19, 19, 19, 19, 19, 19},
+		{0,  1,  2,  2,  3,  4,  5,  6,  6,  7,  8,  8,  9,  9,  10,
+		 11, 12, 12, 13, 14, 15, 16, 17, 17, 18, 18, 18, 18, 18, 18},
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  6,  7,  8,  8,  9,  10,
+		 10, 11, 12, 13, 14, 15, 15, 16, 16, 17, 17, 17, 17, 17, 17},
+};
+
+static u8 delta_swing_index_mp_5ga_n_txpwrtrack_type1_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0,  1,  2,  2,  3,  3,  4,  5,  6,  7,  8,  8,  9,  9,  10,
+		 11, 11, 12, 12, 12, 13, 13, 14, 14, 14, 15, 15, 15, 15, 15},
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  6,  7,  7,  8,  8,  9,
+		 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 14, 14, 14, 14, 14},
+		{0, 1,  2,  2,  3,  4,  4,  5,  6,  6,  7,  7,  8,  8,  9,
+		 9, 10, 10, 11, 12, 12, 13, 13, 14, 14, 14, 14, 14, 14, 14},
+};
+
+static u8 delta_swing_index_mp_5ga_p_txpwrtrack_type1_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  9,  9,  10, 11,
+		 12, 13, 14, 14, 15, 16, 17, 18, 19, 19, 20, 20, 20, 20, 20},
+		{0,  1,  2,  2,  3,  4,  4,  5,  6,  6,  7,  7,  8,  9,  9,
+		 10, 11, 11, 12, 13, 14, 15, 16, 16, 17, 17, 18, 18, 18, 18},
+		{0,  1,  2,  3,  3,  4,  5,  5,  6,  6,  7,  8,  8,  9,  10,
+		 11, 12, 12, 13, 14, 15, 15, 16, 17, 17, 18, 18, 18, 18, 18},
+};
+
+static u8 delta_swing_index_mp_2gb_n_txpwrtrack_type1_8822b[] = {
+	0,  1,  2,  3,  3,  4,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+	13, 13, 14, 15, 16, 16, 17, 18, 18, 18, 18, 18, 18, 18, 18};
+
+static u8 delta_swing_index_mp_2gb_p_txpwrtrack_type1_8822b[] = {
+	0,  1,  1,  2,  3,  4,  4,  5,  6,  7,  7,  8,  9,  10, 11,
+	12, 12, 13, 14, 15, 16, 17, 17, 18, 19, 20, 21, 22, 22, 22};
+
+static u8 delta_swing_index_mp_2ga_n_txpwrtrack_type1_8822b[] = {
+	0,  1,  2,  3,  3,  4,  5,  6,  6,  7,  8,  9,  10, 11, 12,
+	13, 13, 14, 15, 16, 17, 18, 18, 18, 18, 18, 18, 18, 18, 18};
+
+static u8 delta_swing_index_mp_2ga_p_txpwrtrack_type1_8822b[] = {
+	0,  1,  2,  3,  3,  4,  5,  6,  6,  7,  8,  9,  9,  10, 11,
+	12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 22};
+
+static u8 delta_swing_index_mp_2g_cck_b_n_txpwrtrack_type1_8822b[] = {
+	0,  1,  1,  2,  3,  4,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+	13, 13, 14, 15, 16, 16, 17, 17, 17, 17, 17, 17, 17, 17, 17};
+
+static u8 delta_swing_index_mp_2g_cck_b_p_txpwrtrack_type1_8822b[] = {
+	0,  1,  1,  2,  3,  4,  4,  5,  6,  7,  8,  9,  9,  10, 11,
+	12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 22};
+
+static u8 delta_swing_index_mp_2g_cck_a_n_txpwrtrack_type1_8822b[] = {
+	0,  1,  2,  3,  3,  4,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+	13, 13, 14, 15, 16, 17, 18, 18, 18, 18, 18, 18, 18, 18, 18};
+
+static u8 delta_swing_index_mp_2g_cck_a_p_txpwrtrack_type1_8822b[] = {
+	0,  1,  2,  3,  3,  4,  5,  6,  6,  7,  8,  9,  9,  10, 11,
+	12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 22};
+
+void odm_read_and_config_mp_8822b_txpowertrack_type1(struct phy_dm_struct *dm)
+{
+	struct dm_rf_calibration_struct *cali_info = &dm->rf_calibrate_info;
+
+	ODM_RT_TRACE(dm, ODM_COMP_INIT, "===> ODM_ReadAndConfig_MP_mp_8822b\n");
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2ga_p,
+			delta_swing_index_mp_2ga_p_txpwrtrack_type1_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2ga_n,
+			delta_swing_index_mp_2ga_n_txpwrtrack_type1_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2gb_p,
+			delta_swing_index_mp_2gb_p_txpwrtrack_type1_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2gb_n,
+			delta_swing_index_mp_2gb_n_txpwrtrack_type1_8822b,
+			DELTA_SWINGIDX_SIZE);
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_a_p,
+			delta_swing_index_mp_2g_cck_a_p_txpwrtrack_type1_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_a_n,
+			delta_swing_index_mp_2g_cck_a_n_txpwrtrack_type1_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_b_p,
+			delta_swing_index_mp_2g_cck_b_p_txpwrtrack_type1_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_b_n,
+			delta_swing_index_mp_2g_cck_b_n_txpwrtrack_type1_8822b,
+			DELTA_SWINGIDX_SIZE);
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5ga_p,
+			delta_swing_index_mp_5ga_p_txpwrtrack_type1_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5ga_n,
+			delta_swing_index_mp_5ga_n_txpwrtrack_type1_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5gb_p,
+			delta_swing_index_mp_5gb_p_txpwrtrack_type1_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5gb_n,
+			delta_swing_index_mp_5gb_n_txpwrtrack_type1_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+}
+
+/******************************************************************************
+ *                           txpowertrack_type2.TXT
+ ******************************************************************************/
+
+static u8 delta_swing_index_mp_5gb_n_txpwrtrack_type2_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  8,  9,  10, 11,
+		 11, 12, 13, 14, 14, 15, 16, 17, 17, 18, 19, 20, 20, 21, 22},
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  8,  9,  10, 11,
+		 11, 12, 13, 14, 14, 15, 16, 17, 17, 18, 19, 20, 20, 21, 22},
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  8,  9,  10, 11,
+		 11, 12, 13, 14, 14, 15, 16, 17, 17, 18, 19, 20, 20, 21, 22},
+};
+
+static u8 delta_swing_index_mp_5gb_p_txpwrtrack_type2_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  9,  9,  10, 11,
+		 12, 13, 14, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 23},
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  9,  9,  10, 11,
+		 12, 13, 14, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 23},
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  9,  9,  10, 11,
+		 12, 13, 14, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 23},
+};
+
+static u8 delta_swing_index_mp_5ga_n_txpwrtrack_type2_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  8,  9,  10, 11,
+		 11, 12, 13, 14, 14, 15, 16, 17, 17, 18, 19, 20, 20, 21, 22},
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  8,  9,  10, 11,
+		 11, 12, 13, 14, 14, 15, 16, 17, 17, 18, 19, 20, 20, 21, 22},
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  8,  9,  10, 11,
+		 11, 12, 13, 14, 14, 15, 16, 17, 17, 18, 19, 20, 20, 21, 22},
+};
+
+static u8 delta_swing_index_mp_5ga_p_txpwrtrack_type2_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  9,  9,  10, 11,
+		 12, 13, 14, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 23},
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  9,  9,  10, 11,
+		 12, 13, 14, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 23},
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  9,  9,  10, 11,
+		 12, 13, 14, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 23},
+};
+
+static u8 delta_swing_index_mp_2gb_n_txpwrtrack_type2_8822b[] = {
+	0, 1, 1, 1, 2, 2, 3, 3, 3, 4,  4,  5,  5,  5,  6,
+	6, 7, 7, 7, 8, 8, 9, 9, 9, 10, 10, 11, 11, 11, 12};
+
+static u8 delta_swing_index_mp_2gb_p_txpwrtrack_type2_8822b[] = {
+	0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+	8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15};
+
+static u8 delta_swing_index_mp_2ga_n_txpwrtrack_type2_8822b[] = {
+	0, 1, 1, 1, 2, 2, 3, 3, 3, 4,  4,  5,  5,  5,  6,
+	6, 7, 7, 7, 8, 8, 9, 9, 9, 10, 10, 11, 11, 11, 12};
+
+static u8 delta_swing_index_mp_2ga_p_txpwrtrack_type2_8822b[] = {
+	0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+	8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15};
+
+static u8 delta_swing_index_mp_2g_cck_b_n_txpwrtrack_type2_8822b[] = {
+	0, 1, 1, 1, 2, 2, 3, 3, 3, 4,  4,  5,  5,  5,  6,
+	6, 7, 7, 7, 8, 8, 9, 9, 9, 10, 10, 11, 11, 11, 12};
+
+static u8 delta_swing_index_mp_2g_cck_b_p_txpwrtrack_type2_8822b[] = {
+	0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+	8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15};
+
+static u8 delta_swing_index_mp_2g_cck_a_n_txpwrtrack_type2_8822b[] = {
+	0, 1, 1, 1, 2, 2, 3, 3, 3, 4,  4,  5,  5,  5,  6,
+	6, 7, 7, 7, 8, 8, 9, 9, 9, 10, 10, 11, 11, 11, 12};
+
+static u8 delta_swing_index_mp_2g_cck_a_p_txpwrtrack_type2_8822b[] = {
+	0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+	8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15};
+
+void odm_read_and_config_mp_8822b_txpowertrack_type2(struct phy_dm_struct *dm)
+{
+	struct dm_rf_calibration_struct *cali_info = &dm->rf_calibrate_info;
+
+	ODM_RT_TRACE(dm, ODM_COMP_INIT, "===> ODM_ReadAndConfig_MP_mp_8822b\n");
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2ga_p,
+			delta_swing_index_mp_2ga_p_txpwrtrack_type2_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2ga_n,
+			delta_swing_index_mp_2ga_n_txpwrtrack_type2_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2gb_p,
+			delta_swing_index_mp_2gb_p_txpwrtrack_type2_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2gb_n,
+			delta_swing_index_mp_2gb_n_txpwrtrack_type2_8822b,
+			DELTA_SWINGIDX_SIZE);
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_a_p,
+			delta_swing_index_mp_2g_cck_a_p_txpwrtrack_type2_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_a_n,
+			delta_swing_index_mp_2g_cck_a_n_txpwrtrack_type2_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_b_p,
+			delta_swing_index_mp_2g_cck_b_p_txpwrtrack_type2_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_b_n,
+			delta_swing_index_mp_2g_cck_b_n_txpwrtrack_type2_8822b,
+			DELTA_SWINGIDX_SIZE);
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5ga_p,
+			delta_swing_index_mp_5ga_p_txpwrtrack_type2_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5ga_n,
+			delta_swing_index_mp_5ga_n_txpwrtrack_type2_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5gb_p,
+			delta_swing_index_mp_5gb_p_txpwrtrack_type2_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5gb_n,
+			delta_swing_index_mp_5gb_n_txpwrtrack_type2_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+}
+
+/******************************************************************************
+ *                           txpowertrack_type3_type5.TXT
+ ******************************************************************************/
+
+static u8 delta_swing_index_mp_5gb_n_txpwrtrack_type3_type5_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+		 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15},
+		{0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+		 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15},
+		{0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+		 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15},
+};
+
+static u8 delta_swing_index_mp_5gb_p_txpwrtrack_type3_type5_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0, 1, 1,  2,  2,  3,  3,  4,  5,  5,  6,  6,  7,  7,  8,
+		 9, 9, 10, 10, 11, 11, 12, 13, 13, 14, 14, 15, 15, 16, 17},
+		{0, 1, 1,  2,  2,  3,  3,  4,  5,  5,  6,  6,  7,  7,  8,
+		 9, 9, 10, 10, 11, 11, 12, 13, 13, 14, 14, 15, 15, 16, 17},
+		{0, 1, 1,  2,  2,  3,  3,  4,  5,  5,  6,  6,  7,  7,  8,
+		 9, 9, 10, 10, 11, 11, 12, 13, 13, 14, 14, 15, 15, 16, 17},
+};
+
+static u8 delta_swing_index_mp_5ga_n_txpwrtrack_type3_type5_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+		 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15},
+		{0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+		 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15},
+		{0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+		 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15},
+};
+
+static u8 delta_swing_index_mp_5ga_p_txpwrtrack_type3_type5_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0, 1, 1,  2,  2,  3,  3,  4,  5,  5,  6,  6,  7,  7,  8,
+		 9, 9, 10, 10, 11, 11, 12, 13, 13, 14, 14, 15, 15, 16, 17},
+		{0, 1, 1,  2,  2,  3,  3,  4,  5,  5,  6,  6,  7,  7,  8,
+		 9, 9, 10, 10, 11, 11, 12, 13, 13, 14, 14, 15, 15, 16, 17},
+		{0, 1, 1,  2,  2,  3,  3,  4,  5,  5,  6,  6,  7,  7,  8,
+		 9, 9, 10, 10, 11, 11, 12, 13, 13, 14, 14, 15, 15, 16, 17},
+};
+
+static u8 delta_swing_index_mp_2gb_n_txpwrtrack_type3_type5_8822b[] = {
+	0, 1, 1, 1, 2, 2, 3, 3, 3, 4,  4,  5,  5,  5,  6,
+	6, 7, 7, 7, 8, 8, 9, 9, 9, 10, 10, 11, 11, 11, 12};
+
+static u8 delta_swing_index_mp_2gb_p_txpwrtrack_type3_type5_8822b[] = {
+	0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+	8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15};
+
+static u8 delta_swing_index_mp_2ga_n_txpwrtrack_type3_type5_8822b[] = {
+	0, 1, 1, 1, 2, 2, 3, 3, 3, 4,  4,  5,  5,  5,  6,
+	6, 7, 7, 7, 8, 8, 9, 9, 9, 10, 10, 11, 11, 11, 12};
+
+static u8 delta_swing_index_mp_2ga_p_txpwrtrack_type3_type5_8822b[] = {
+	0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+	8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15};
+
+static u8 delta_swing_index_mp_2g_cck_b_n_txpwrtrack_type3_type5_8822b[] = {
+	0, 1, 1, 1, 2, 2, 3, 3, 3, 4,  4,  5,  5,  5,  6,
+	6, 7, 7, 7, 8, 8, 9, 9, 9, 10, 10, 11, 11, 11, 12};
+
+static u8 delta_swing_index_mp_2g_cck_b_p_txpwrtrack_type3_type5_8822b[] = {
+	0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+	8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15};
+
+static u8 delta_swing_index_mp_2g_cck_a_n_txpwrtrack_type3_type5_8822b[] = {
+	0, 1, 1, 1, 2, 2, 3, 3, 3, 4,  4,  5,  5,  5,  6,
+	6, 7, 7, 7, 8, 8, 9, 9, 9, 10, 10, 11, 11, 11, 12};
+
+static u8 delta_swing_index_mp_2g_cck_a_p_txpwrtrack_type3_type5_8822b[] = {
+	0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+	8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15};
+
+void odm_read_and_config_mp_8822b_txpowertrack_type3_type5(
+	struct phy_dm_struct *dm)
+{
+	struct dm_rf_calibration_struct *cali_info = &dm->rf_calibrate_info;
+
+	ODM_RT_TRACE(dm, ODM_COMP_INIT, "===> ODM_ReadAndConfig_MP_mp_8822b\n");
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2ga_p,
+			delta_swing_index_mp_2ga_p_txpwrtrack_type3_type5_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2ga_n,
+			delta_swing_index_mp_2ga_n_txpwrtrack_type3_type5_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2gb_p,
+			delta_swing_index_mp_2gb_p_txpwrtrack_type3_type5_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2gb_n,
+			delta_swing_index_mp_2gb_n_txpwrtrack_type3_type5_8822b,
+			DELTA_SWINGIDX_SIZE);
+
+	odm_move_memory(
+		dm, cali_info->delta_swing_table_idx_2g_cck_a_p,
+		delta_swing_index_mp_2g_cck_a_p_txpwrtrack_type3_type5_8822b,
+		DELTA_SWINGIDX_SIZE);
+	odm_move_memory(
+		dm, cali_info->delta_swing_table_idx_2g_cck_a_n,
+		delta_swing_index_mp_2g_cck_a_n_txpwrtrack_type3_type5_8822b,
+		DELTA_SWINGIDX_SIZE);
+	odm_move_memory(
+		dm, cali_info->delta_swing_table_idx_2g_cck_b_p,
+		delta_swing_index_mp_2g_cck_b_p_txpwrtrack_type3_type5_8822b,
+		DELTA_SWINGIDX_SIZE);
+	odm_move_memory(
+		dm, cali_info->delta_swing_table_idx_2g_cck_b_n,
+		delta_swing_index_mp_2g_cck_b_n_txpwrtrack_type3_type5_8822b,
+		DELTA_SWINGIDX_SIZE);
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5ga_p,
+			delta_swing_index_mp_5ga_p_txpwrtrack_type3_type5_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5ga_n,
+			delta_swing_index_mp_5ga_n_txpwrtrack_type3_type5_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5gb_p,
+			delta_swing_index_mp_5gb_p_txpwrtrack_type3_type5_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5gb_n,
+			delta_swing_index_mp_5gb_n_txpwrtrack_type3_type5_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+}
+
+/******************************************************************************
+ *                           txpowertrack_type4.TXT
+ ******************************************************************************/
+
+static u8 delta_swing_index_mp_5gb_n_txpwrtrack_type4_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  8,  9,  10, 11,
+		 11, 12, 13, 14, 14, 15, 16, 17, 17, 18, 19, 20, 20, 21, 22},
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  8,  9,  10, 11,
+		 11, 12, 13, 14, 14, 15, 16, 17, 17, 18, 19, 20, 20, 21, 22},
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  8,  9,  10, 11,
+		 11, 12, 13, 14, 14, 15, 16, 17, 17, 18, 19, 20, 20, 21, 22},
+};
+
+static u8 delta_swing_index_mp_5gb_p_txpwrtrack_type4_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  9,  9,  10, 11,
+		 12, 13, 14, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 23},
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  9,  9,  10, 11,
+		 12, 13, 14, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 23},
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  9,  9,  10, 11,
+		 12, 13, 14, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 23},
+};
+
+static u8 delta_swing_index_mp_5ga_n_txpwrtrack_type4_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  8,  9,  10, 11,
+		 11, 12, 13, 14, 14, 15, 16, 17, 17, 18, 19, 20, 20, 21, 22},
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  8,  9,  10, 11,
+		 11, 12, 13, 14, 14, 15, 16, 17, 17, 18, 19, 20, 20, 21, 22},
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  8,  9,  10, 11,
+		 11, 12, 13, 14, 14, 15, 16, 17, 17, 18, 19, 20, 20, 21, 22},
+};
+
+static u8 delta_swing_index_mp_5ga_p_txpwrtrack_type4_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  9,  9,  10, 11,
+		 12, 13, 14, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 23},
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  9,  9,  10, 11,
+		 12, 13, 14, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 23},
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  7,  8,  9,  9,  10, 11,
+		 12, 13, 14, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 23},
+};
+
+static u8 delta_swing_index_mp_2gb_n_txpwrtrack_type4_8822b[] = {
+	0, 1, 1, 1, 2, 2, 3, 3, 3, 4,  4,  5,  5,  5,  6,
+	6, 7, 7, 7, 8, 8, 9, 9, 9, 10, 10, 11, 11, 11, 12};
+
+static u8 delta_swing_index_mp_2gb_p_txpwrtrack_type4_8822b[] = {
+	0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+	8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15};
+
+static u8 delta_swing_index_mp_2ga_n_txpwrtrack_type4_8822b[] = {
+	0, 1, 1, 1, 2, 2, 3, 3, 3, 4,  4,  5,  5,  5,  6,
+	6, 7, 7, 7, 8, 8, 9, 9, 9, 10, 10, 11, 11, 11, 12};
+
+static u8 delta_swing_index_mp_2ga_p_txpwrtrack_type4_8822b[] = {
+	0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+	8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15};
+
+static u8 delta_swing_index_mp_2g_cck_b_n_txpwrtrack_type4_8822b[] = {
+	0, 1, 1, 1, 2, 2, 3, 3, 3, 4,  4,  5,  5,  5,  6,
+	6, 7, 7, 7, 8, 8, 9, 9, 9, 10, 10, 11, 11, 11, 12};
+
+static u8 delta_swing_index_mp_2g_cck_b_p_txpwrtrack_type4_8822b[] = {
+	0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+	8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15};
+
+static u8 delta_swing_index_mp_2g_cck_a_n_txpwrtrack_type4_8822b[] = {
+	0, 1, 1, 1, 2, 2, 3, 3, 3, 4,  4,  5,  5,  5,  6,
+	6, 7, 7, 7, 8, 8, 9, 9, 9, 10, 10, 11, 11, 11, 12};
+
+static u8 delta_swing_index_mp_2g_cck_a_p_txpwrtrack_type4_8822b[] = {
+	0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+	8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15};
+
+void odm_read_and_config_mp_8822b_txpowertrack_type4(struct phy_dm_struct *dm)
+{
+	struct dm_rf_calibration_struct *cali_info = &dm->rf_calibrate_info;
+
+	ODM_RT_TRACE(dm, ODM_COMP_INIT, "===> ODM_ReadAndConfig_MP_mp_8822b\n");
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2ga_p,
+			delta_swing_index_mp_2ga_p_txpwrtrack_type4_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2ga_n,
+			delta_swing_index_mp_2ga_n_txpwrtrack_type4_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2gb_p,
+			delta_swing_index_mp_2gb_p_txpwrtrack_type4_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2gb_n,
+			delta_swing_index_mp_2gb_n_txpwrtrack_type4_8822b,
+			DELTA_SWINGIDX_SIZE);
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_a_p,
+			delta_swing_index_mp_2g_cck_a_p_txpwrtrack_type4_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_a_n,
+			delta_swing_index_mp_2g_cck_a_n_txpwrtrack_type4_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_b_p,
+			delta_swing_index_mp_2g_cck_b_p_txpwrtrack_type4_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_b_n,
+			delta_swing_index_mp_2g_cck_b_n_txpwrtrack_type4_8822b,
+			DELTA_SWINGIDX_SIZE);
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5ga_p,
+			delta_swing_index_mp_5ga_p_txpwrtrack_type4_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5ga_n,
+			delta_swing_index_mp_5ga_n_txpwrtrack_type4_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5gb_p,
+			delta_swing_index_mp_5gb_p_txpwrtrack_type4_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5gb_n,
+			delta_swing_index_mp_5gb_n_txpwrtrack_type4_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+}
+
+/******************************************************************************
+ *                           txpowertrack_type6.TXT
+ ******************************************************************************/
+
+static u8 delta_swing_index_mp_5gb_n_txpwrtrack_type6_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0,  1,  2,  3,  3,  4,  5,  6,  6,  7,  8,  9,  9,  10, 10,
+		 11, 11, 12, 12, 12, 13, 13, 14, 14, 14, 15, 15, 15, 15, 15},
+		{0,  1,  2,  3,  4,  5,  5,  6,  7,  7,  8,  9,  9,  10, 10,
+		 11, 12, 12, 13, 14, 14, 15, 15, 16, 16, 16, 16, 16, 16, 16},
+		{0,  1,  2,  3,  4,  4,  5,  5,  6,  7,  8,  9,  10, 11, 12,
+		 12, 13, 13, 14, 15, 15, 16, 16, 17, 17, 17, 17, 17, 17, 17},
+};
+
+static u8 delta_swing_index_mp_5gb_p_txpwrtrack_type6_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0,  1,  2,  2,  3,  4,  5,  6,  7,  8,  8,  9,  10, 11, 11,
+		 12, 13, 14, 15, 15, 16, 17, 18, 18, 19, 19, 19, 19, 19, 19},
+		{0,  1,  2,  2,  3,  4,  5,  6,  7,  8,  9,  9,  11, 11, 12,
+		 13, 14, 15, 16, 17, 18, 19, 20, 20, 21, 21, 21, 21, 21, 21},
+		{0,  1,  2,  3,  4,  5,  6,  6,  7,  7,  8,  9,  10, 11, 12,
+		 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 20, 20, 21, 21, 21},
+};
+
+static u8 delta_swing_index_mp_5ga_n_txpwrtrack_type6_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0,  1,  2,  2,  3,  3,  4,  5,  6,  7,  8,  9,  10, 10, 11,
+		 12, 12, 13, 14, 14, 15, 15, 16, 16, 16, 17, 17, 17, 17, 17},
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  6,  7,  8,  9,  9,  10,
+		 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 15, 15, 15, 15, 15},
+		{0,  1,  2,  2,  3,  4,  4,  5,  6,  6,  7,  8,  9,  9,  10,
+		 11, 12, 12, 13, 14, 14, 15, 15, 16, 16, 16, 16, 16, 16, 16},
+};
+
+static u8 delta_swing_index_mp_5ga_p_txpwrtrack_type6_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0,  1,  2,  2,  3,  4,  5,  6,  7,  8,  9,  10, 10, 11, 12,
+		 13, 14, 15, 15, 16, 17, 18, 19, 20, 20, 21, 21, 21, 21, 21},
+		{0,  1,  2,  2,  3,  4,  4,  5,  7,  7,  8,  9,  10, 11, 11,
+		 12, 13, 13, 14, 15, 16, 17, 18, 18, 19, 19, 20, 20, 21, 21},
+		{0,  1,  2,  3,  3,  4,  5,  5,  6,  7,  8,  9,  10, 11, 12,
+		 13, 14, 14, 15, 16, 17, 17, 18, 19, 19, 20, 20, 20, 20, 20},
+};
+
+static u8 delta_swing_index_mp_2gb_n_txpwrtrack_type6_8822b[] = {
+	0,  1,  2,  3,  3,  4,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+	13, 13, 14, 15, 16, 16, 17, 18, 18, 18, 18, 18, 18, 18, 18};
+
+static u8 delta_swing_index_mp_2gb_p_txpwrtrack_type6_8822b[] = {
+	0,  1,  1,  2,  3,  4,  4,  5,  6,  7,  7,  8,  9,  10, 11,
+	12, 12, 13, 14, 15, 16, 17, 17, 18, 19, 20, 21, 22, 22, 22};
+
+static u8 delta_swing_index_mp_2ga_n_txpwrtrack_type6_8822b[] = {
+	0,  1,  2,  3,  3,  4,  5,  6,  6,  7,  8,  9,  10, 11, 12,
+	13, 13, 14, 15, 16, 17, 18, 18, 18, 18, 18, 18, 18, 18, 18};
+
+static u8 delta_swing_index_mp_2ga_p_txpwrtrack_type6_8822b[] = {
+	0,  1,  2,  3,  3,  4,  5,  6,  6,  7,  8,  9,  9,  10, 11,
+	12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 22};
+
+static u8 delta_swing_index_mp_2g_cck_b_n_txpwrtrack_type6_8822b[] = {
+	0,  1,  1,  2,  3,  4,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+	13, 13, 14, 15, 16, 16, 17, 17, 17, 17, 17, 17, 17, 17, 17};
+
+static u8 delta_swing_index_mp_2g_cck_b_p_txpwrtrack_type6_8822b[] = {
+	0,  1,  1,  2,  3,  4,  4,  5,  6,  7,  8,  9,  9,  10, 11,
+	12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 22};
+
+static u8 delta_swing_index_mp_2g_cck_a_n_txpwrtrack_type6_8822b[] = {
+	0,  1,  2,  3,  3,  4,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+	13, 13, 14, 15, 16, 17, 18, 18, 18, 18, 18, 18, 18, 18, 18};
+
+static u8 delta_swing_index_mp_2g_cck_a_p_txpwrtrack_type6_8822b[] = {
+	0,  1,  2,  3,  3,  4,  5,  6,  6,  7,  8,  9,  9,  10, 11,
+	12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 22};
+
+void odm_read_and_config_mp_8822b_txpowertrack_type6(struct phy_dm_struct *dm)
+{
+	struct dm_rf_calibration_struct *cali_info = &dm->rf_calibrate_info;
+
+	ODM_RT_TRACE(dm, ODM_COMP_INIT, "===> ODM_ReadAndConfig_MP_mp_8822b\n");
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2ga_p,
+			delta_swing_index_mp_2ga_p_txpwrtrack_type6_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2ga_n,
+			delta_swing_index_mp_2ga_n_txpwrtrack_type6_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2gb_p,
+			delta_swing_index_mp_2gb_p_txpwrtrack_type6_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2gb_n,
+			delta_swing_index_mp_2gb_n_txpwrtrack_type6_8822b,
+			DELTA_SWINGIDX_SIZE);
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_a_p,
+			delta_swing_index_mp_2g_cck_a_p_txpwrtrack_type6_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_a_n,
+			delta_swing_index_mp_2g_cck_a_n_txpwrtrack_type6_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_b_p,
+			delta_swing_index_mp_2g_cck_b_p_txpwrtrack_type6_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_b_n,
+			delta_swing_index_mp_2g_cck_b_n_txpwrtrack_type6_8822b,
+			DELTA_SWINGIDX_SIZE);
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5ga_p,
+			delta_swing_index_mp_5ga_p_txpwrtrack_type6_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5ga_n,
+			delta_swing_index_mp_5ga_n_txpwrtrack_type6_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5gb_p,
+			delta_swing_index_mp_5gb_p_txpwrtrack_type6_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5gb_n,
+			delta_swing_index_mp_5gb_n_txpwrtrack_type6_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+}
+
+/******************************************************************************
+ *                           txpowertrack_type7.TXT
+ ******************************************************************************/
+
+static u8 delta_swing_index_mp_5gb_n_txpwrtrack_type7_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0,  1,  2,  3,  3,  4,  5,  6,  6,  7,  8,  9,  9,  10, 10,
+		 11, 11, 12, 12, 12, 13, 13, 14, 14, 14, 15, 15, 15, 15, 15},
+		{0,  1,  2,  3,  4,  5,  5,  6,  7,  7,  8,  9,  9,  10, 10,
+		 11, 12, 12, 13, 14, 14, 15, 15, 16, 16, 16, 16, 16, 16, 16},
+		{0,  1,  2,  3,  4,  4,  5,  5,  6,  7,  8,  9,  10, 11, 12,
+		 12, 13, 13, 14, 15, 15, 16, 16, 17, 17, 17, 17, 17, 17, 17},
+};
+
+static u8 delta_swing_index_mp_5gb_p_txpwrtrack_type7_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0,  1,  2,  2,  3,  4,  5,  6,  7,  8,  8,  9,  10, 11, 11,
+		 12, 13, 14, 15, 15, 16, 17, 18, 18, 19, 19, 19, 19, 19, 19},
+		{0,  1,  2,  2,  3,  4,  5,  6,  7,  8,  9,  9,  11, 11, 12,
+		 13, 14, 15, 16, 17, 18, 19, 20, 20, 21, 21, 21, 21, 21, 21},
+		{0,  1,  2,  3,  4,  5,  6,  6,  7,  7,  8,  9,  10, 11, 12,
+		 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 20, 20, 21, 21, 21},
+};
+
+static u8 delta_swing_index_mp_5ga_n_txpwrtrack_type7_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0,  1,  2,  2,  3,  3,  4,  5,  6,  7,  8,  9,  10, 10, 11,
+		 12, 12, 13, 14, 14, 15, 15, 16, 16, 16, 17, 17, 17, 17, 17},
+		{0,  1,  2,  2,  3,  4,  5,  5,  6,  6,  7,  8,  9,  9,  10,
+		 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 15, 15, 15, 15, 15},
+		{0,  1,  2,  2,  3,  4,  4,  5,  6,  6,  7,  8,  9,  9,  10,
+		 11, 12, 12, 13, 14, 14, 15, 15, 16, 16, 16, 16, 16, 16, 16},
+};
+
+static u8 delta_swing_index_mp_5ga_p_txpwrtrack_type7_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0,  1,  2,  2,  3,  4,  5,  6,  7,  8,  9,  10, 10, 11, 12,
+		 13, 14, 15, 15, 16, 17, 18, 19, 20, 20, 21, 21, 21, 21, 21},
+		{0,  1,  2,  2,  3,  4,  4,  5,  7,  7,  8,  9,  10, 11, 11,
+		 12, 13, 13, 14, 15, 16, 17, 18, 18, 19, 19, 20, 20, 21, 21},
+		{0,  1,  2,  3,  3,  4,  5,  5,  6,  7,  8,  9,  10, 11, 12,
+		 13, 14, 14, 15, 16, 17, 17, 18, 19, 19, 20, 20, 20, 20, 20},
+};
+
+static u8 delta_swing_index_mp_2gb_n_txpwrtrack_type7_8822b[] = {
+	0,  1,  2,  3,  3,  4,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+	13, 13, 14, 15, 16, 16, 17, 18, 18, 18, 18, 18, 18, 18, 18};
+
+static u8 delta_swing_index_mp_2gb_p_txpwrtrack_type7_8822b[] = {
+	0,  1,  1,  2,  3,  4,  4,  5,  6,  7,  7,  8,  9,  10, 11,
+	12, 12, 13, 14, 15, 16, 17, 17, 18, 19, 20, 21, 22, 22, 22};
+
+static u8 delta_swing_index_mp_2ga_n_txpwrtrack_type7_8822b[] = {
+	0,  1,  2,  3,  3,  4,  5,  6,  6,  7,  8,  9,  10, 11, 12,
+	13, 13, 14, 15, 16, 17, 18, 18, 18, 18, 18, 18, 18, 18, 18};
+
+static u8 delta_swing_index_mp_2ga_p_txpwrtrack_type7_8822b[] = {
+	0,  1,  2,  3,  3,  4,  5,  6,  6,  7,  8,  9,  9,  10, 11,
+	12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 22};
+
+static u8 delta_swing_index_mp_2g_cck_b_n_txpwrtrack_type7_8822b[] = {
+	0,  1,  1,  2,  3,  4,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+	13, 13, 14, 15, 16, 16, 17, 17, 17, 17, 17, 17, 17, 17, 17};
+
+static u8 delta_swing_index_mp_2g_cck_b_p_txpwrtrack_type7_8822b[] = {
+	0,  1,  1,  2,  3,  4,  4,  5,  6,  7,  8,  9,  9,  10, 11,
+	12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 22};
+
+static u8 delta_swing_index_mp_2g_cck_a_n_txpwrtrack_type7_8822b[] = {
+	0,  1,  2,  3,  3,  4,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+	13, 13, 14, 15, 16, 17, 18, 18, 18, 18, 18, 18, 18, 18, 18};
+
+static u8 delta_swing_index_mp_2g_cck_a_p_txpwrtrack_type7_8822b[] = {
+	0,  1,  2,  3,  3,  4,  5,  6,  6,  7,  8,  9,  9,  10, 11,
+	12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 22};
+
+void odm_read_and_config_mp_8822b_txpowertrack_type7(struct phy_dm_struct *dm)
+{
+	struct dm_rf_calibration_struct *cali_info = &dm->rf_calibrate_info;
+
+	ODM_RT_TRACE(dm, ODM_COMP_INIT, "===> ODM_ReadAndConfig_MP_mp_8822b\n");
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2ga_p,
+			delta_swing_index_mp_2ga_p_txpwrtrack_type7_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2ga_n,
+			delta_swing_index_mp_2ga_n_txpwrtrack_type7_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2gb_p,
+			delta_swing_index_mp_2gb_p_txpwrtrack_type7_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2gb_n,
+			delta_swing_index_mp_2gb_n_txpwrtrack_type7_8822b,
+			DELTA_SWINGIDX_SIZE);
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_a_p,
+			delta_swing_index_mp_2g_cck_a_p_txpwrtrack_type7_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_a_n,
+			delta_swing_index_mp_2g_cck_a_n_txpwrtrack_type7_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_b_p,
+			delta_swing_index_mp_2g_cck_b_p_txpwrtrack_type7_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_b_n,
+			delta_swing_index_mp_2g_cck_b_n_txpwrtrack_type7_8822b,
+			DELTA_SWINGIDX_SIZE);
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5ga_p,
+			delta_swing_index_mp_5ga_p_txpwrtrack_type7_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5ga_n,
+			delta_swing_index_mp_5ga_n_txpwrtrack_type7_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5gb_p,
+			delta_swing_index_mp_5gb_p_txpwrtrack_type7_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5gb_n,
+			delta_swing_index_mp_5gb_n_txpwrtrack_type7_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+}
+
+/******************************************************************************
+ *                           txpowertrack_type8.TXT
+ ******************************************************************************/
+
+static u8 delta_swing_index_mp_5gb_n_txpwrtrack_type8_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+		 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15},
+		{0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+		 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15},
+		{0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+		 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15},
+};
+
+static u8 delta_swing_index_mp_5gb_p_txpwrtrack_type8_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0, 1, 1,  2,  2,  3,  3,  4,  5,  5,  6,  6,  7,  7,  8,
+		 9, 9, 10, 10, 11, 11, 12, 13, 13, 14, 14, 15, 15, 16, 17},
+		{0, 1, 1,  2,  2,  3,  3,  4,  5,  5,  6,  6,  7,  7,  8,
+		 9, 9, 10, 10, 11, 11, 12, 13, 13, 14, 14, 15, 15, 16, 17},
+		{0, 1, 1,  2,  2,  3,  3,  4,  5,  5,  6,  6,  7,  7,  8,
+		 9, 9, 10, 10, 11, 11, 12, 13, 13, 14, 14, 15, 15, 16, 17},
+};
+
+static u8 delta_swing_index_mp_5ga_n_txpwrtrack_type8_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+		 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15},
+		{0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+		 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15},
+		{0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+		 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15},
+};
+
+static u8 delta_swing_index_mp_5ga_p_txpwrtrack_type8_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0, 1, 1,  2,  2,  3,  3,  4,  5,  5,  6,  6,  7,  7,  8,
+		 9, 9, 10, 10, 11, 11, 12, 13, 13, 14, 14, 15, 15, 16, 17},
+		{0, 1, 1,  2,  2,  3,  3,  4,  5,  5,  6,  6,  7,  7,  8,
+		 9, 9, 10, 10, 11, 11, 12, 13, 13, 14, 14, 15, 15, 16, 17},
+		{0, 1, 1,  2,  2,  3,  3,  4,  5,  5,  6,  6,  7,  7,  8,
+		 9, 9, 10, 10, 11, 11, 12, 13, 13, 14, 14, 15, 15, 16, 17},
+};
+
+static u8 delta_swing_index_mp_2gb_n_txpwrtrack_type8_8822b[] = {
+	0, 1, 1, 1, 2, 2, 3, 3, 3, 4,  4,  5,  5,  5,  6,
+	6, 7, 7, 7, 8, 8, 9, 9, 9, 10, 10, 11, 11, 11, 12};
+
+static u8 delta_swing_index_mp_2gb_p_txpwrtrack_type8_8822b[] = {
+	0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+	8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15};
+
+static u8 delta_swing_index_mp_2ga_n_txpwrtrack_type8_8822b[] = {
+	0, 1, 1, 1, 2, 2, 3, 3, 3, 4,  4,  5,  5,  5,  6,
+	6, 7, 7, 7, 8, 8, 9, 9, 9, 10, 10, 11, 11, 11, 12};
+
+static u8 delta_swing_index_mp_2ga_p_txpwrtrack_type8_8822b[] = {
+	0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+	8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15};
+
+static u8 delta_swing_index_mp_2g_cck_b_n_txpwrtrack_type8_8822b[] = {
+	0, 1, 1, 1, 2, 2, 3, 3, 3, 4,  4,  5,  5,  5,  6,
+	6, 7, 7, 7, 8, 8, 9, 9, 9, 10, 10, 11, 11, 11, 12};
+
+static u8 delta_swing_index_mp_2g_cck_b_p_txpwrtrack_type8_8822b[] = {
+	0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+	8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15};
+
+static u8 delta_swing_index_mp_2g_cck_a_n_txpwrtrack_type8_8822b[] = {
+	0, 1, 1, 1, 2, 2, 3, 3, 3, 4,  4,  5,  5,  5,  6,
+	6, 7, 7, 7, 8, 8, 9, 9, 9, 10, 10, 11, 11, 11, 12};
+
+static u8 delta_swing_index_mp_2g_cck_a_p_txpwrtrack_type8_8822b[] = {
+	0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  7,  7,  8,
+	8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15};
+
+void odm_read_and_config_mp_8822b_txpowertrack_type8(struct phy_dm_struct *dm)
+{
+	struct dm_rf_calibration_struct *cali_info = &dm->rf_calibrate_info;
+
+	ODM_RT_TRACE(dm, ODM_COMP_INIT, "===> ODM_ReadAndConfig_MP_mp_8822b\n");
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2ga_p,
+			delta_swing_index_mp_2ga_p_txpwrtrack_type8_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2ga_n,
+			delta_swing_index_mp_2ga_n_txpwrtrack_type8_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2gb_p,
+			delta_swing_index_mp_2gb_p_txpwrtrack_type8_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2gb_n,
+			delta_swing_index_mp_2gb_n_txpwrtrack_type8_8822b,
+			DELTA_SWINGIDX_SIZE);
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_a_p,
+			delta_swing_index_mp_2g_cck_a_p_txpwrtrack_type8_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_a_n,
+			delta_swing_index_mp_2g_cck_a_n_txpwrtrack_type8_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_b_p,
+			delta_swing_index_mp_2g_cck_b_p_txpwrtrack_type8_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_b_n,
+			delta_swing_index_mp_2g_cck_b_n_txpwrtrack_type8_8822b,
+			DELTA_SWINGIDX_SIZE);
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5ga_p,
+			delta_swing_index_mp_5ga_p_txpwrtrack_type8_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5ga_n,
+			delta_swing_index_mp_5ga_n_txpwrtrack_type8_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5gb_p,
+			delta_swing_index_mp_5gb_p_txpwrtrack_type8_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5gb_n,
+			delta_swing_index_mp_5gb_n_txpwrtrack_type8_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+}
+
+/******************************************************************************
+ *                           txpowertrack_type9.TXT
+ ******************************************************************************/
+
+static u8 delta_swing_index_mp_5gb_n_txpwrtrack_type9_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  7,
+		 8, 8, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15},
+		{0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  8,
+		 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 14, 14},
+		{0, 1, 1,  2,  2,  3,  3,  4,  5,  5,  6,  7,  7,  8,  8,
+		 9, 9, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 15, 15},
+};
+
+static u8 delta_swing_index_mp_5gb_p_txpwrtrack_type9_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0, 1, 1, 2, 2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  7,
+		 8, 8, 9, 9, 10, 10, 11, 12, 12, 13, 13, 14, 14, 15, 15},
+		{0, 1, 1, 2, 2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  7,
+		 8, 8, 9, 9, 10, 11, 11, 12, 13, 13, 14, 14, 15, 15, 15},
+		{0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  7,
+		 8, 8, 9, 10, 10, 11, 12, 13, 14, 14, 15, 15, 15, 16, 16},
+};
+
+static u8 delta_swing_index_mp_5ga_n_txpwrtrack_type9_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0, 1, 1, 2, 2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  7,
+		 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 13, 14, 14},
+		{0, 1, 1, 2, 2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  7,
+		 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 14},
+		{0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  7,
+		 8, 8, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 14, 14},
+};
+
+static u8 delta_swing_index_mp_5ga_p_txpwrtrack_type9_8822b
+	[][DELTA_SWINGIDX_SIZE] = {
+		{0, 1, 1, 2, 2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  7,
+		 8, 8, 9, 9, 10, 10, 11, 12, 13, 13, 14, 14, 15, 15, 15},
+		{0, 1, 1, 2, 2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  7,
+		 8, 8, 9, 9, 10, 10, 11, 12, 12, 13, 13, 14, 14, 15, 15},
+		{0, 1, 1, 2,  2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  7,
+		 8, 9, 9, 10, 10, 11, 11, 12, 13, 13, 14, 14, 15, 15, 15},
+};
+
+static u8 delta_swing_index_mp_2gb_n_txpwrtrack_type9_8822b[] = {
+	0,  1,  2,  3,  3,  4,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+	13, 13, 14, 15, 16, 16, 17, 18, 18, 18, 18, 18, 18, 18, 18};
+
+static u8 delta_swing_index_mp_2gb_p_txpwrtrack_type9_8822b[] = {
+	0,  1,  1,  2,  3,  4,  4,  5,  6,  7,  7,  8,  9,  10, 11,
+	12, 12, 13, 14, 15, 16, 17, 17, 18, 19, 20, 21, 22, 22, 22};
+
+static u8 delta_swing_index_mp_2ga_n_txpwrtrack_type9_8822b[] = {
+	0,  1,  2,  3,  3,  4,  5,  6,  6,  7,  8,  9,  10, 11, 12,
+	13, 13, 14, 15, 16, 17, 18, 18, 18, 18, 18, 18, 18, 18, 18};
+
+static u8 delta_swing_index_mp_2ga_p_txpwrtrack_type9_8822b[] = {
+	0,  1,  2,  3,  3,  4,  5,  6,  6,  7,  8,  9,  9,  10, 11,
+	12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 22};
+
+static u8 delta_swing_index_mp_2g_cck_b_n_txpwrtrack_type9_8822b[] = {
+	0,  1,  1,  2,  3,  4,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+	13, 13, 14, 15, 16, 16, 17, 17, 17, 17, 17, 17, 17, 17, 17};
+
+static u8 delta_swing_index_mp_2g_cck_b_p_txpwrtrack_type9_8822b[] = {
+	0,  1,  1,  2,  3,  4,  4,  5,  6,  7,  8,  9,  9,  10, 11,
+	12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 22};
+
+static u8 delta_swing_index_mp_2g_cck_a_n_txpwrtrack_type9_8822b[] = {
+	0,  1,  2,  3,  3,  4,  4,  5,  6,  7,  8,  9,  10, 11, 12,
+	13, 13, 14, 15, 16, 17, 18, 18, 18, 18, 18, 18, 18, 18, 18};
+
+static u8 delta_swing_index_mp_2g_cck_a_p_txpwrtrack_type9_8822b[] = {
+	0,  1,  2,  3,  3,  4,  5,  6,  6,  7,  8,  9,  9,  10, 11,
+	12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 22, 22};
+
+void odm_read_and_config_mp_8822b_txpowertrack_type9(struct phy_dm_struct *dm)
+{
+	struct dm_rf_calibration_struct *cali_info = &dm->rf_calibrate_info;
+
+	ODM_RT_TRACE(dm, ODM_COMP_INIT, "===> ODM_ReadAndConfig_MP_mp_8822b\n");
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2ga_p,
+			delta_swing_index_mp_2ga_p_txpwrtrack_type9_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2ga_n,
+			delta_swing_index_mp_2ga_n_txpwrtrack_type9_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2gb_p,
+			delta_swing_index_mp_2gb_p_txpwrtrack_type9_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2gb_n,
+			delta_swing_index_mp_2gb_n_txpwrtrack_type9_8822b,
+			DELTA_SWINGIDX_SIZE);
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_a_p,
+			delta_swing_index_mp_2g_cck_a_p_txpwrtrack_type9_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_a_n,
+			delta_swing_index_mp_2g_cck_a_n_txpwrtrack_type9_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_b_p,
+			delta_swing_index_mp_2g_cck_b_p_txpwrtrack_type9_8822b,
+			DELTA_SWINGIDX_SIZE);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_2g_cck_b_n,
+			delta_swing_index_mp_2g_cck_b_n_txpwrtrack_type9_8822b,
+			DELTA_SWINGIDX_SIZE);
+
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5ga_p,
+			delta_swing_index_mp_5ga_p_txpwrtrack_type9_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5ga_n,
+			delta_swing_index_mp_5ga_n_txpwrtrack_type9_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5gb_p,
+			delta_swing_index_mp_5gb_p_txpwrtrack_type9_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+	odm_move_memory(dm, cali_info->delta_swing_table_idx_5gb_n,
+			delta_swing_index_mp_5gb_n_txpwrtrack_type9_8822b,
+			DELTA_SWINGIDX_SIZE * 3);
+}
+
+/******************************************************************************
+ *                           txpwr_lmt.TXT
+ ******************************************************************************/
+
+static const char *const array_mp_8822b_txpwr_lmt[] = {
+	"FCC",  "2.4G", "20M",  "CCK",  "1T",   "01",   "32",   "ETSI", "2.4G",
+	"20M",  "CCK",  "1T",   "01",   "28",   "MKK",  "2.4G", "20M",  "CCK",
+	"1T",   "01",   "30",   "FCC",  "2.4G", "20M",  "CCK",  "1T",   "02",
+	"32",   "ETSI", "2.4G", "20M",  "CCK",  "1T",   "02",   "28",   "MKK",
+	"2.4G", "20M",  "CCK",  "1T",   "02",   "30",   "FCC",  "2.4G", "20M",
+	"CCK",  "1T",   "03",   "32",   "ETSI", "2.4G", "20M",  "CCK",  "1T",
+	"03",   "28",   "MKK",  "2.4G", "20M",  "CCK",  "1T",   "03",   "30",
+	"FCC",  "2.4G", "20M",  "CCK",  "1T",   "04",   "32",   "ETSI", "2.4G",
+	"20M",  "CCK",  "1T",   "04",   "28",   "MKK",  "2.4G", "20M",  "CCK",
+	"1T",   "04",   "30",   "FCC",  "2.4G", "20M",  "CCK",  "1T",   "05",
+	"32",   "ETSI", "2.4G", "20M",  "CCK",  "1T",   "05",   "28",   "MKK",
+	"2.4G", "20M",  "CCK",  "1T",   "05",   "30",   "FCC",  "2.4G", "20M",
+	"CCK",  "1T",   "06",   "32",   "ETSI", "2.4G", "20M",  "CCK",  "1T",
+	"06",   "28",   "MKK",  "2.4G", "20M",  "CCK",  "1T",   "06",   "30",
+	"FCC",  "2.4G", "20M",  "CCK",  "1T",   "07",   "32",   "ETSI", "2.4G",
+	"20M",  "CCK",  "1T",   "07",   "28",   "MKK",  "2.4G", "20M",  "CCK",
+	"1T",   "07",   "30",   "FCC",  "2.4G", "20M",  "CCK",  "1T",   "08",
+	"32",   "ETSI", "2.4G", "20M",  "CCK",  "1T",   "08",   "28",   "MKK",
+	"2.4G", "20M",  "CCK",  "1T",   "08",   "30",   "FCC",  "2.4G", "20M",
+	"CCK",  "1T",   "09",   "32",   "ETSI", "2.4G", "20M",  "CCK",  "1T",
+	"09",   "28",   "MKK",  "2.4G", "20M",  "CCK",  "1T",   "09",   "30",
+	"FCC",  "2.4G", "20M",  "CCK",  "1T",   "10",   "32",   "ETSI", "2.4G",
+	"20M",  "CCK",  "1T",   "10",   "28",   "MKK",  "2.4G", "20M",  "CCK",
+	"1T",   "10",   "30",   "FCC",  "2.4G", "20M",  "CCK",  "1T",   "11",
+	"32",   "ETSI", "2.4G", "20M",  "CCK",  "1T",   "11",   "28",   "MKK",
+	"2.4G", "20M",  "CCK",  "1T",   "11",   "30",   "FCC",  "2.4G", "20M",
+	"CCK",  "1T",   "12",   "26",   "ETSI", "2.4G", "20M",  "CCK",  "1T",
+	"12",   "28",   "MKK",  "2.4G", "20M",  "CCK",  "1T",   "12",   "30",
+	"FCC",  "2.4G", "20M",  "CCK",  "1T",   "13",   "20",   "ETSI", "2.4G",
+	"20M",  "CCK",  "1T",   "13",   "28",   "MKK",  "2.4G", "20M",  "CCK",
+	"1T",   "13",   "28",   "FCC",  "2.4G", "20M",  "CCK",  "1T",   "14",
+	"63",   "ETSI", "2.4G", "20M",  "CCK",  "1T",   "14",   "63",   "MKK",
+	"2.4G", "20M",  "CCK",  "1T",   "14",   "32",   "FCC",  "2.4G", "20M",
+	"OFDM", "1T",   "01",   "26",   "ETSI", "2.4G", "20M",  "OFDM", "1T",
+	"01",   "30",   "MKK",  "2.4G", "20M",  "OFDM", "1T",   "01",   "34",
+	"FCC",  "2.4G", "20M",  "OFDM", "1T",   "02",   "30",   "ETSI", "2.4G",
+	"20M",  "OFDM", "1T",   "02",   "30",   "MKK",  "2.4G", "20M",  "OFDM",
+	"1T",   "02",   "34",   "FCC",  "2.4G", "20M",  "OFDM", "1T",   "03",
+	"32",   "ETSI", "2.4G", "20M",  "OFDM", "1T",   "03",   "30",   "MKK",
+	"2.4G", "20M",  "OFDM", "1T",   "03",   "34",   "FCC",  "2.4G", "20M",
+	"OFDM", "1T",   "04",   "34",   "ETSI", "2.4G", "20M",  "OFDM", "1T",
+	"04",   "30",   "MKK",  "2.4G", "20M",  "OFDM", "1T",   "04",   "34",
+	"FCC",  "2.4G", "20M",  "OFDM", "1T",   "05",   "34",   "ETSI", "2.4G",
+	"20M",  "OFDM", "1T",   "05",   "30",   "MKK",  "2.4G", "20M",  "OFDM",
+	"1T",   "05",   "34",   "FCC",  "2.4G", "20M",  "OFDM", "1T",   "06",
+	"34",   "ETSI", "2.4G", "20M",  "OFDM", "1T",   "06",   "30",   "MKK",
+	"2.4G", "20M",  "OFDM", "1T",   "06",   "34",   "FCC",  "2.4G", "20M",
+	"OFDM", "1T",   "07",   "34",   "ETSI", "2.4G", "20M",  "OFDM", "1T",
+	"07",   "30",   "MKK",  "2.4G", "20M",  "OFDM", "1T",   "07",   "34",
+	"FCC",  "2.4G", "20M",  "OFDM", "1T",   "08",   "34",   "ETSI", "2.4G",
+	"20M",  "OFDM", "1T",   "08",   "30",   "MKK",  "2.4G", "20M",  "OFDM",
+	"1T",   "08",   "34",   "FCC",  "2.4G", "20M",  "OFDM", "1T",   "09",
+	"32",   "ETSI", "2.4G", "20M",  "OFDM", "1T",   "09",   "30",   "MKK",
+	"2.4G", "20M",  "OFDM", "1T",   "09",   "34",   "FCC",  "2.4G", "20M",
+	"OFDM", "1T",   "10",   "30",   "ETSI", "2.4G", "20M",  "OFDM", "1T",
+	"10",   "30",   "MKK",  "2.4G", "20M",  "OFDM", "1T",   "10",   "34",
+	"FCC",  "2.4G", "20M",  "OFDM", "1T",   "11",   "28",   "ETSI", "2.4G",
+	"20M",  "OFDM", "1T",   "11",   "30",   "MKK",  "2.4G", "20M",  "OFDM",
+	"1T",   "11",   "34",   "FCC",  "2.4G", "20M",  "OFDM", "1T",   "12",
+	"22",   "ETSI", "2.4G", "20M",  "OFDM", "1T",   "12",   "30",   "MKK",
+	"2.4G", "20M",  "OFDM", "1T",   "12",   "34",   "FCC",  "2.4G", "20M",
+	"OFDM", "1T",   "13",   "14",   "ETSI", "2.4G", "20M",  "OFDM", "1T",
+	"13",   "30",   "MKK",  "2.4G", "20M",  "OFDM", "1T",   "13",   "34",
+	"FCC",  "2.4G", "20M",  "OFDM", "1T",   "14",   "63",   "ETSI", "2.4G",
+	"20M",  "OFDM", "1T",   "14",   "63",   "MKK",  "2.4G", "20M",  "OFDM",
+	"1T",   "14",   "63",   "FCC",  "2.4G", "20M",  "HT",   "1T",   "01",
+	"26",   "ETSI", "2.4G", "20M",  "HT",   "1T",   "01",   "30",   "MKK",
+	"2.4G", "20M",  "HT",   "1T",   "01",   "34",   "FCC",  "2.4G", "20M",
+	"HT",   "1T",   "02",   "30",   "ETSI", "2.4G", "20M",  "HT",   "1T",
+	"02",   "30",   "MKK",  "2.4G", "20M",  "HT",   "1T",   "02",   "34",
+	"FCC",  "2.4G", "20M",  "HT",   "1T",   "03",   "32",   "ETSI", "2.4G",
+	"20M",  "HT",   "1T",   "03",   "30",   "MKK",  "2.4G", "20M",  "HT",
+	"1T",   "03",   "34",   "FCC",  "2.4G", "20M",  "HT",   "1T",   "04",
+	"34",   "ETSI", "2.4G", "20M",  "HT",   "1T",   "04",   "30",   "MKK",
+	"2.4G", "20M",  "HT",   "1T",   "04",   "34",   "FCC",  "2.4G", "20M",
+	"HT",   "1T",   "05",   "34",   "ETSI", "2.4G", "20M",  "HT",   "1T",
+	"05",   "30",   "MKK",  "2.4G", "20M",  "HT",   "1T",   "05",   "34",
+	"FCC",  "2.4G", "20M",  "HT",   "1T",   "06",   "34",   "ETSI", "2.4G",
+	"20M",  "HT",   "1T",   "06",   "30",   "MKK",  "2.4G", "20M",  "HT",
+	"1T",   "06",   "34",   "FCC",  "2.4G", "20M",  "HT",   "1T",   "07",
+	"34",   "ETSI", "2.4G", "20M",  "HT",   "1T",   "07",   "30",   "MKK",
+	"2.4G", "20M",  "HT",   "1T",   "07",   "34",   "FCC",  "2.4G", "20M",
+	"HT",   "1T",   "08",   "34",   "ETSI", "2.4G", "20M",  "HT",   "1T",
+	"08",   "30",   "MKK",  "2.4G", "20M",  "HT",   "1T",   "08",   "34",
+	"FCC",  "2.4G", "20M",  "HT",   "1T",   "09",   "32",   "ETSI", "2.4G",
+	"20M",  "HT",   "1T",   "09",   "30",   "MKK",  "2.4G", "20M",  "HT",
+	"1T",   "09",   "34",   "FCC",  "2.4G", "20M",  "HT",   "1T",   "10",
+	"30",   "ETSI", "2.4G", "20M",  "HT",   "1T",   "10",   "30",   "MKK",
+	"2.4G", "20M",  "HT",   "1T",   "10",   "34",   "FCC",  "2.4G", "20M",
+	"HT",   "1T",   "11",   "26",   "ETSI", "2.4G", "20M",  "HT",   "1T",
+	"11",   "30",   "MKK",  "2.4G", "20M",  "HT",   "1T",   "11",   "34",
+	"FCC",  "2.4G", "20M",  "HT",   "1T",   "12",   "20",   "ETSI", "2.4G",
+	"20M",  "HT",   "1T",   "12",   "30",   "MKK",  "2.4G", "20M",  "HT",
+	"1T",   "12",   "34",   "FCC",  "2.4G", "20M",  "HT",   "1T",   "13",
+	"14",   "ETSI", "2.4G", "20M",  "HT",   "1T",   "13",   "30",   "MKK",
+	"2.4G", "20M",  "HT",   "1T",   "13",   "34",   "FCC",  "2.4G", "20M",
+	"HT",   "1T",   "14",   "63",   "ETSI", "2.4G", "20M",  "HT",   "1T",
+	"14",   "63",   "MKK",  "2.4G", "20M",  "HT",   "1T",   "14",   "63",
+	"FCC",  "2.4G", "20M",  "HT",   "2T",   "01",   "26",   "ETSI", "2.4G",
+	"20M",  "HT",   "2T",   "01",   "18",   "MKK",  "2.4G", "20M",  "HT",
+	"2T",   "01",   "30",   "FCC",  "2.4G", "20M",  "HT",   "2T",   "02",
+	"28",   "ETSI", "2.4G", "20M",  "HT",   "2T",   "02",   "18",   "MKK",
+	"2.4G", "20M",  "HT",   "2T",   "02",   "30",   "FCC",  "2.4G", "20M",
+	"HT",   "2T",   "03",   "30",   "ETSI", "2.4G", "20M",  "HT",   "2T",
+	"03",   "18",   "MKK",  "2.4G", "20M",  "HT",   "2T",   "03",   "30",
+	"FCC",  "2.4G", "20M",  "HT",   "2T",   "04",   "30",   "ETSI", "2.4G",
+	"20M",  "HT",   "2T",   "04",   "18",   "MKK",  "2.4G", "20M",  "HT",
+	"2T",   "04",   "30",   "FCC",  "2.4G", "20M",  "HT",   "2T",   "05",
+	"32",   "ETSI", "2.4G", "20M",  "HT",   "2T",   "05",   "18",   "MKK",
+	"2.4G", "20M",  "HT",   "2T",   "05",   "30",   "FCC",  "2.4G", "20M",
+	"HT",   "2T",   "06",   "32",   "ETSI", "2.4G", "20M",  "HT",   "2T",
+	"06",   "18",   "MKK",  "2.4G", "20M",  "HT",   "2T",   "06",   "30",
+	"FCC",  "2.4G", "20M",  "HT",   "2T",   "07",   "32",   "ETSI", "2.4G",
+	"20M",  "HT",   "2T",   "07",   "18",   "MKK",  "2.4G", "20M",  "HT",
+	"2T",   "07",   "30",   "FCC",  "2.4G", "20M",  "HT",   "2T",   "08",
+	"30",   "ETSI", "2.4G", "20M",  "HT",   "2T",   "08",   "18",   "MKK",
+	"2.4G", "20M",  "HT",   "2T",   "08",   "30",   "FCC",  "2.4G", "20M",
+	"HT",   "2T",   "09",   "30",   "ETSI", "2.4G", "20M",  "HT",   "2T",
+	"09",   "18",   "MKK",  "2.4G", "20M",  "HT",   "2T",   "09",   "30",
+	"FCC",  "2.4G", "20M",  "HT",   "2T",   "10",   "28",   "ETSI", "2.4G",
+	"20M",  "HT",   "2T",   "10",   "18",   "MKK",  "2.4G", "20M",  "HT",
+	"2T",   "10",   "30",   "FCC",  "2.4G", "20M",  "HT",   "2T",   "11",
+	"26",   "ETSI", "2.4G", "20M",  "HT",   "2T",   "11",   "18",   "MKK",
+	"2.4G", "20M",  "HT",   "2T",   "11",   "30",   "FCC",  "2.4G", "20M",
+	"HT",   "2T",   "12",   "20",   "ETSI", "2.4G", "20M",  "HT",   "2T",
+	"12",   "18",   "MKK",  "2.4G", "20M",  "HT",   "2T",   "12",   "30",
+	"FCC",  "2.4G", "20M",  "HT",   "2T",   "13",   "14",   "ETSI", "2.4G",
+	"20M",  "HT",   "2T",   "13",   "18",   "MKK",  "2.4G", "20M",  "HT",
+	"2T",   "13",   "30",   "FCC",  "2.4G", "20M",  "HT",   "2T",   "14",
+	"63",   "ETSI", "2.4G", "20M",  "HT",   "2T",   "14",   "63",   "MKK",
+	"2.4G", "20M",  "HT",   "2T",   "14",   "63",   "FCC",  "2.4G", "40M",
+	"HT",   "1T",   "01",   "63",   "ETSI", "2.4G", "40M",  "HT",   "1T",
+	"01",   "63",   "MKK",  "2.4G", "40M",  "HT",   "1T",   "01",   "63",
+	"FCC",  "2.4G", "40M",  "HT",   "1T",   "02",   "63",   "ETSI", "2.4G",
+	"40M",  "HT",   "1T",   "02",   "63",   "MKK",  "2.4G", "40M",  "HT",
+	"1T",   "02",   "63",   "FCC",  "2.4G", "40M",  "HT",   "1T",   "03",
+	"26",   "ETSI", "2.4G", "40M",  "HT",   "1T",   "03",   "30",   "MKK",
+	"2.4G", "40M",  "HT",   "1T",   "03",   "34",   "FCC",  "2.4G", "40M",
+	"HT",   "1T",   "04",   "26",   "ETSI", "2.4G", "40M",  "HT",   "1T",
+	"04",   "30",   "MKK",  "2.4G", "40M",  "HT",   "1T",   "04",   "34",
+	"FCC",  "2.4G", "40M",  "HT",   "1T",   "05",   "30",   "ETSI", "2.4G",
+	"40M",  "HT",   "1T",   "05",   "30",   "MKK",  "2.4G", "40M",  "HT",
+	"1T",   "05",   "34",   "FCC",  "2.4G", "40M",  "HT",   "1T",   "06",
+	"32",   "ETSI", "2.4G", "40M",  "HT",   "1T",   "06",   "30",   "MKK",
+	"2.4G", "40M",  "HT",   "1T",   "06",   "34",   "FCC",  "2.4G", "40M",
+	"HT",   "1T",   "07",   "30",   "ETSI", "2.4G", "40M",  "HT",   "1T",
+	"07",   "30",   "MKK",  "2.4G", "40M",  "HT",   "1T",   "07",   "34",
+	"FCC",  "2.4G", "40M",  "HT",   "1T",   "08",   "26",   "ETSI", "2.4G",
+	"40M",  "HT",   "1T",   "08",   "30",   "MKK",  "2.4G", "40M",  "HT",
+	"1T",   "08",   "34",   "FCC",  "2.4G", "40M",  "HT",   "1T",   "09",
+	"26",   "ETSI", "2.4G", "40M",  "HT",   "1T",   "09",   "30",   "MKK",
+	"2.4G", "40M",  "HT",   "1T",   "09",   "34",   "FCC",  "2.4G", "40M",
+	"HT",   "1T",   "10",   "20",   "ETSI", "2.4G", "40M",  "HT",   "1T",
+	"10",   "30",   "MKK",  "2.4G", "40M",  "HT",   "1T",   "10",   "34",
+	"FCC",  "2.4G", "40M",  "HT",   "1T",   "11",   "14",   "ETSI", "2.4G",
+	"40M",  "HT",   "1T",   "11",   "30",   "MKK",  "2.4G", "40M",  "HT",
+	"1T",   "11",   "34",   "FCC",  "2.4G", "40M",  "HT",   "1T",   "12",
+	"63",   "ETSI", "2.4G", "40M",  "HT",   "1T",   "12",   "63",   "MKK",
+	"2.4G", "40M",  "HT",   "1T",   "12",   "63",   "FCC",  "2.4G", "40M",
+	"HT",   "1T",   "13",   "63",   "ETSI", "2.4G", "40M",  "HT",   "1T",
+	"13",   "63",   "MKK",  "2.4G", "40M",  "HT",   "1T",   "13",   "63",
+	"FCC",  "2.4G", "40M",  "HT",   "1T",   "14",   "63",   "ETSI", "2.4G",
+	"40M",  "HT",   "1T",   "14",   "63",   "MKK",  "2.4G", "40M",  "HT",
+	"1T",   "14",   "63",   "FCC",  "2.4G", "40M",  "HT",   "2T",   "01",
+	"63",   "ETSI", "2.4G", "40M",  "HT",   "2T",   "01",   "63",   "MKK",
+	"2.4G", "40M",  "HT",   "2T",   "01",   "63",   "FCC",  "2.4G", "40M",
+	"HT",   "2T",   "02",   "63",   "ETSI", "2.4G", "40M",  "HT",   "2T",
+	"02",   "63",   "MKK",  "2.4G", "40M",  "HT",   "2T",   "02",   "63",
+	"FCC",  "2.4G", "40M",  "HT",   "2T",   "03",   "24",   "ETSI", "2.4G",
+	"40M",  "HT",   "2T",   "03",   "18",   "MKK",  "2.4G", "40M",  "HT",
+	"2T",   "03",   "30",   "FCC",  "2.4G", "40M",  "HT",   "2T",   "04",
+	"24",   "ETSI", "2.4G", "40M",  "HT",   "2T",   "04",   "18",   "MKK",
+	"2.4G", "40M",  "HT",   "2T",   "04",   "30",   "FCC",  "2.4G", "40M",
+	"HT",   "2T",   "05",   "26",   "ETSI", "2.4G", "40M",  "HT",   "2T",
+	"05",   "18",   "MKK",  "2.4G", "40M",  "HT",   "2T",   "05",   "30",
+	"FCC",  "2.4G", "40M",  "HT",   "2T",   "06",   "28",   "ETSI", "2.4G",
+	"40M",  "HT",   "2T",   "06",   "18",   "MKK",  "2.4G", "40M",  "HT",
+	"2T",   "06",   "30",   "FCC",  "2.4G", "40M",  "HT",   "2T",   "07",
+	"26",   "ETSI", "2.4G", "40M",  "HT",   "2T",   "07",   "18",   "MKK",
+	"2.4G", "40M",  "HT",   "2T",   "07",   "30",   "FCC",  "2.4G", "40M",
+	"HT",   "2T",   "08",   "26",   "ETSI", "2.4G", "40M",  "HT",   "2T",
+	"08",   "18",   "MKK",  "2.4G", "40M",  "HT",   "2T",   "08",   "30",
+	"FCC",  "2.4G", "40M",  "HT",   "2T",   "09",   "26",   "ETSI", "2.4G",
+	"40M",  "HT",   "2T",   "09",   "18",   "MKK",  "2.4G", "40M",  "HT",
+	"2T",   "09",   "30",   "FCC",  "2.4G", "40M",  "HT",   "2T",   "10",
+	"20",   "ETSI", "2.4G", "40M",  "HT",   "2T",   "10",   "18",   "MKK",
+	"2.4G", "40M",  "HT",   "2T",   "10",   "30",   "FCC",  "2.4G", "40M",
+	"HT",   "2T",   "11",   "14",   "ETSI", "2.4G", "40M",  "HT",   "2T",
+	"11",   "18",   "MKK",  "2.4G", "40M",  "HT",   "2T",   "11",   "30",
+	"FCC",  "2.4G", "40M",  "HT",   "2T",   "12",   "63",   "ETSI", "2.4G",
+	"40M",  "HT",   "2T",   "12",   "63",   "MKK",  "2.4G", "40M",  "HT",
+	"2T",   "12",   "63",   "FCC",  "2.4G", "40M",  "HT",   "2T",   "13",
+	"63",   "ETSI", "2.4G", "40M",  "HT",   "2T",   "13",   "63",   "MKK",
+	"2.4G", "40M",  "HT",   "2T",   "13",   "63",   "FCC",  "2.4G", "40M",
+	"HT",   "2T",   "14",   "63",   "ETSI", "2.4G", "40M",  "HT",   "2T",
+	"14",   "63",   "MKK",  "2.4G", "40M",  "HT",   "2T",   "14",   "63",
+	"FCC",  "5G",   "20M",  "OFDM", "1T",   "36",   "30",   "ETSI", "5G",
+	"20M",  "OFDM", "1T",   "36",   "32",   "MKK",  "5G",   "20M",  "OFDM",
+	"1T",   "36",   "30",   "FCC",  "5G",   "20M",  "OFDM", "1T",   "40",
+	"32",   "ETSI", "5G",   "20M",  "OFDM", "1T",   "40",   "32",   "MKK",
+	"5G",   "20M",  "OFDM", "1T",   "40",   "30",   "FCC",  "5G",   "20M",
+	"OFDM", "1T",   "44",   "32",   "ETSI", "5G",   "20M",  "OFDM", "1T",
+	"44",   "32",   "MKK",  "5G",   "20M",  "OFDM", "1T",   "44",   "30",
+	"FCC",  "5G",   "20M",  "OFDM", "1T",   "48",   "32",   "ETSI", "5G",
+	"20M",  "OFDM", "1T",   "48",   "32",   "MKK",  "5G",   "20M",  "OFDM",
+	"1T",   "48",   "30",   "FCC",  "5G",   "20M",  "OFDM", "1T",   "52",
+	"32",   "ETSI", "5G",   "20M",  "OFDM", "1T",   "52",   "32",   "MKK",
+	"5G",   "20M",  "OFDM", "1T",   "52",   "28",   "FCC",  "5G",   "20M",
+	"OFDM", "1T",   "56",   "32",   "ETSI", "5G",   "20M",  "OFDM", "1T",
+	"56",   "32",   "MKK",  "5G",   "20M",  "OFDM", "1T",   "56",   "28",
+	"FCC",  "5G",   "20M",  "OFDM", "1T",   "60",   "32",   "ETSI", "5G",
+	"20M",  "OFDM", "1T",   "60",   "32",   "MKK",  "5G",   "20M",  "OFDM",
+	"1T",   "60",   "28",   "FCC",  "5G",   "20M",  "OFDM", "1T",   "64",
+	"28",   "ETSI", "5G",   "20M",  "OFDM", "1T",   "64",   "32",   "MKK",
+	"5G",   "20M",  "OFDM", "1T",   "64",   "28",   "FCC",  "5G",   "20M",
+	"OFDM", "1T",   "100",  "26",   "ETSI", "5G",   "20M",  "OFDM", "1T",
+	"100",  "32",   "MKK",  "5G",   "20M",  "OFDM", "1T",   "100",  "32",
+	"FCC",  "5G",   "20M",  "OFDM", "1T",   "104",  "32",   "ETSI", "5G",
+	"20M",  "OFDM", "1T",   "104",  "32",   "MKK",  "5G",   "20M",  "OFDM",
+	"1T",   "104",  "32",   "FCC",  "5G",   "20M",  "OFDM", "1T",   "108",
+	"32",   "ETSI", "5G",   "20M",  "OFDM", "1T",   "108",  "32",   "MKK",
+	"5G",   "20M",  "OFDM", "1T",   "108",  "32",   "FCC",  "5G",   "20M",
+	"OFDM", "1T",   "112",  "32",   "ETSI", "5G",   "20M",  "OFDM", "1T",
+	"112",  "32",   "MKK",  "5G",   "20M",  "OFDM", "1T",   "112",  "32",
+	"FCC",  "5G",   "20M",  "OFDM", "1T",   "116",  "32",   "ETSI", "5G",
+	"20M",  "OFDM", "1T",   "116",  "32",   "MKK",  "5G",   "20M",  "OFDM",
+	"1T",   "116",  "32",   "FCC",  "5G",   "20M",  "OFDM", "1T",   "120",
+	"32",   "ETSI", "5G",   "20M",  "OFDM", "1T",   "120",  "32",   "MKK",
+	"5G",   "20M",  "OFDM", "1T",   "120",  "32",   "FCC",  "5G",   "20M",
+	"OFDM", "1T",   "124",  "32",   "ETSI", "5G",   "20M",  "OFDM", "1T",
+	"124",  "32",   "MKK",  "5G",   "20M",  "OFDM", "1T",   "124",  "32",
+	"FCC",  "5G",   "20M",  "OFDM", "1T",   "128",  "32",   "ETSI", "5G",
+	"20M",  "OFDM", "1T",   "128",  "32",   "MKK",  "5G",   "20M",  "OFDM",
+	"1T",   "128",  "32",   "FCC",  "5G",   "20M",  "OFDM", "1T",   "132",
+	"32",   "ETSI", "5G",   "20M",  "OFDM", "1T",   "132",  "32",   "MKK",
+	"5G",   "20M",  "OFDM", "1T",   "132",  "32",   "FCC",  "5G",   "20M",
+	"OFDM", "1T",   "136",  "32",   "ETSI", "5G",   "20M",  "OFDM", "1T",
+	"136",  "32",   "MKK",  "5G",   "20M",  "OFDM", "1T",   "136",  "32",
+	"FCC",  "5G",   "20M",  "OFDM", "1T",   "140",  "28",   "ETSI", "5G",
+	"20M",  "OFDM", "1T",   "140",  "32",   "MKK",  "5G",   "20M",  "OFDM",
+	"1T",   "140",  "32",   "FCC",  "5G",   "20M",  "OFDM", "1T",   "144",
+	"28",   "ETSI", "5G",   "20M",  "OFDM", "1T",   "144",  "32",   "MKK",
+	"5G",   "20M",  "OFDM", "1T",   "144",  "63",   "FCC",  "5G",   "20M",
+	"OFDM", "1T",   "149",  "32",   "ETSI", "5G",   "20M",  "OFDM", "1T",
+	"149",  "63",   "MKK",  "5G",   "20M",  "OFDM", "1T",   "149",  "63",
+	"FCC",  "5G",   "20M",  "OFDM", "1T",   "153",  "32",   "ETSI", "5G",
+	"20M",  "OFDM", "1T",   "153",  "63",   "MKK",  "5G",   "20M",  "OFDM",
+	"1T",   "153",  "63",   "FCC",  "5G",   "20M",  "OFDM", "1T",   "157",
+	"32",   "ETSI", "5G",   "20M",  "OFDM", "1T",   "157",  "63",   "MKK",
+	"5G",   "20M",  "OFDM", "1T",   "157",  "63",   "FCC",  "5G",   "20M",
+	"OFDM", "1T",   "161",  "32",   "ETSI", "5G",   "20M",  "OFDM", "1T",
+	"161",  "63",   "MKK",  "5G",   "20M",  "OFDM", "1T",   "161",  "63",
+	"FCC",  "5G",   "20M",  "OFDM", "1T",   "165",  "32",   "ETSI", "5G",
+	"20M",  "OFDM", "1T",   "165",  "63",   "MKK",  "5G",   "20M",  "OFDM",
+	"1T",   "165",  "63",   "FCC",  "5G",   "20M",  "HT",   "1T",   "36",
+	"30",   "ETSI", "5G",   "20M",  "HT",   "1T",   "36",   "32",   "MKK",
+	"5G",   "20M",  "HT",   "1T",   "36",   "28",   "FCC",  "5G",   "20M",
+	"HT",   "1T",   "40",   "32",   "ETSI", "5G",   "20M",  "HT",   "1T",
+	"40",   "32",   "MKK",  "5G",   "20M",  "HT",   "1T",   "40",   "28",
+	"FCC",  "5G",   "20M",  "HT",   "1T",   "44",   "32",   "ETSI", "5G",
+	"20M",  "HT",   "1T",   "44",   "32",   "MKK",  "5G",   "20M",  "HT",
+	"1T",   "44",   "28",   "FCC",  "5G",   "20M",  "HT",   "1T",   "48",
+	"32",   "ETSI", "5G",   "20M",  "HT",   "1T",   "48",   "32",   "MKK",
+	"5G",   "20M",  "HT",   "1T",   "48",   "28",   "FCC",  "5G",   "20M",
+	"HT",   "1T",   "52",   "32",   "ETSI", "5G",   "20M",  "HT",   "1T",
+	"52",   "32",   "MKK",  "5G",   "20M",  "HT",   "1T",   "52",   "28",
+	"FCC",  "5G",   "20M",  "HT",   "1T",   "56",   "32",   "ETSI", "5G",
+	"20M",  "HT",   "1T",   "56",   "32",   "MKK",  "5G",   "20M",  "HT",
+	"1T",   "56",   "28",   "FCC",  "5G",   "20M",  "HT",   "1T",   "60",
+	"32",   "ETSI", "5G",   "20M",  "HT",   "1T",   "60",   "32",   "MKK",
+	"5G",   "20M",  "HT",   "1T",   "60",   "28",   "FCC",  "5G",   "20M",
+	"HT",   "1T",   "64",   "28",   "ETSI", "5G",   "20M",  "HT",   "1T",
+	"64",   "32",   "MKK",  "5G",   "20M",  "HT",   "1T",   "64",   "28",
+	"FCC",  "5G",   "20M",  "HT",   "1T",   "100",  "26",   "ETSI", "5G",
+	"20M",  "HT",   "1T",   "100",  "32",   "MKK",  "5G",   "20M",  "HT",
+	"1T",   "100",  "32",   "FCC",  "5G",   "20M",  "HT",   "1T",   "104",
+	"32",   "ETSI", "5G",   "20M",  "HT",   "1T",   "104",  "32",   "MKK",
+	"5G",   "20M",  "HT",   "1T",   "104",  "32",   "FCC",  "5G",   "20M",
+	"HT",   "1T",   "108",  "32",   "ETSI", "5G",   "20M",  "HT",   "1T",
+	"108",  "32",   "MKK",  "5G",   "20M",  "HT",   "1T",   "108",  "32",
+	"FCC",  "5G",   "20M",  "HT",   "1T",   "112",  "32",   "ETSI", "5G",
+	"20M",  "HT",   "1T",   "112",  "32",   "MKK",  "5G",   "20M",  "HT",
+	"1T",   "112",  "32",   "FCC",  "5G",   "20M",  "HT",   "1T",   "116",
+	"32",   "ETSI", "5G",   "20M",  "HT",   "1T",   "116",  "32",   "MKK",
+	"5G",   "20M",  "HT",   "1T",   "116",  "32",   "FCC",  "5G",   "20M",
+	"HT",   "1T",   "120",  "32",   "ETSI", "5G",   "20M",  "HT",   "1T",
+	"120",  "32",   "MKK",  "5G",   "20M",  "HT",   "1T",   "120",  "32",
+	"FCC",  "5G",   "20M",  "HT",   "1T",   "124",  "32",   "ETSI", "5G",
+	"20M",  "HT",   "1T",   "124",  "32",   "MKK",  "5G",   "20M",  "HT",
+	"1T",   "124",  "32",   "FCC",  "5G",   "20M",  "HT",   "1T",   "128",
+	"32",   "ETSI", "5G",   "20M",  "HT",   "1T",   "128",  "32",   "MKK",
+	"5G",   "20M",  "HT",   "1T",   "128",  "32",   "FCC",  "5G",   "20M",
+	"HT",   "1T",   "132",  "32",   "ETSI", "5G",   "20M",  "HT",   "1T",
+	"132",  "32",   "MKK",  "5G",   "20M",  "HT",   "1T",   "132",  "32",
+	"FCC",  "5G",   "20M",  "HT",   "1T",   "136",  "32",   "ETSI", "5G",
+	"20M",  "HT",   "1T",   "136",  "32",   "MKK",  "5G",   "20M",  "HT",
+	"1T",   "136",  "32",   "FCC",  "5G",   "20M",  "HT",   "1T",   "140",
+	"26",   "ETSI", "5G",   "20M",  "HT",   "1T",   "140",  "32",   "MKK",
+	"5G",   "20M",  "HT",   "1T",   "140",  "32",   "FCC",  "5G",   "20M",
+	"HT",   "1T",   "144",  "26",   "ETSI", "5G",   "20M",  "HT",   "1T",
+	"144",  "63",   "MKK",  "5G",   "20M",  "HT",   "1T",   "144",  "63",
+	"FCC",  "5G",   "20M",  "HT",   "1T",   "149",  "32",   "ETSI", "5G",
+	"20M",  "HT",   "1T",   "149",  "63",   "MKK",  "5G",   "20M",  "HT",
+	"1T",   "149",  "63",   "FCC",  "5G",   "20M",  "HT",   "1T",   "153",
+	"32",   "ETSI", "5G",   "20M",  "HT",   "1T",   "153",  "63",   "MKK",
+	"5G",   "20M",  "HT",   "1T",   "153",  "63",   "FCC",  "5G",   "20M",
+	"HT",   "1T",   "157",  "32",   "ETSI", "5G",   "20M",  "HT",   "1T",
+	"157",  "63",   "MKK",  "5G",   "20M",  "HT",   "1T",   "157",  "63",
+	"FCC",  "5G",   "20M",  "HT",   "1T",   "161",  "32",   "ETSI", "5G",
+	"20M",  "HT",   "1T",   "161",  "63",   "MKK",  "5G",   "20M",  "HT",
+	"1T",   "161",  "63",   "FCC",  "5G",   "20M",  "HT",   "1T",   "165",
+	"32",   "ETSI", "5G",   "20M",  "HT",   "1T",   "165",  "63",   "MKK",
+	"5G",   "20M",  "HT",   "1T",   "165",  "63",   "FCC",  "5G",   "20M",
+	"HT",   "2T",   "36",   "28",   "ETSI", "5G",   "20M",  "HT",   "2T",
+	"36",   "20",   "MKK",  "5G",   "20M",  "HT",   "2T",   "36",   "22",
+	"FCC",  "5G",   "20M",  "HT",   "2T",   "40",   "30",   "ETSI", "5G",
+	"20M",  "HT",   "2T",   "40",   "20",   "MKK",  "5G",   "20M",  "HT",
+	"2T",   "40",   "22",   "FCC",  "5G",   "20M",  "HT",   "2T",   "44",
+	"30",   "ETSI", "5G",   "20M",  "HT",   "2T",   "44",   "20",   "MKK",
+	"5G",   "20M",  "HT",   "2T",   "44",   "22",   "FCC",  "5G",   "20M",
+	"HT",   "2T",   "48",   "30",   "ETSI", "5G",   "20M",  "HT",   "2T",
+	"48",   "20",   "MKK",  "5G",   "20M",  "HT",   "2T",   "48",   "22",
+	"FCC",  "5G",   "20M",  "HT",   "2T",   "52",   "30",   "ETSI", "5G",
+	"20M",  "HT",   "2T",   "52",   "20",   "MKK",  "5G",   "20M",  "HT",
+	"2T",   "52",   "22",   "FCC",  "5G",   "20M",  "HT",   "2T",   "56",
+	"30",   "ETSI", "5G",   "20M",  "HT",   "2T",   "56",   "20",   "MKK",
+	"5G",   "20M",  "HT",   "2T",   "56",   "22",   "FCC",  "5G",   "20M",
+	"HT",   "2T",   "60",   "30",   "ETSI", "5G",   "20M",  "HT",   "2T",
+	"60",   "20",   "MKK",  "5G",   "20M",  "HT",   "2T",   "60",   "22",
+	"FCC",  "5G",   "20M",  "HT",   "2T",   "64",   "28",   "ETSI", "5G",
+	"20M",  "HT",   "2T",   "64",   "20",   "MKK",  "5G",   "20M",  "HT",
+	"2T",   "64",   "22",   "FCC",  "5G",   "20M",  "HT",   "2T",   "100",
+	"26",   "ETSI", "5G",   "20M",  "HT",   "2T",   "100",  "20",   "MKK",
+	"5G",   "20M",  "HT",   "2T",   "100",  "30",   "FCC",  "5G",   "20M",
+	"HT",   "2T",   "104",  "30",   "ETSI", "5G",   "20M",  "HT",   "2T",
+	"104",  "20",   "MKK",  "5G",   "20M",  "HT",   "2T",   "104",  "30",
+	"FCC",  "5G",   "20M",  "HT",   "2T",   "108",  "32",   "ETSI", "5G",
+	"20M",  "HT",   "2T",   "108",  "20",   "MKK",  "5G",   "20M",  "HT",
+	"2T",   "108",  "30",   "FCC",  "5G",   "20M",  "HT",   "2T",   "112",
+	"32",   "ETSI", "5G",   "20M",  "HT",   "2T",   "112",  "20",   "MKK",
+	"5G",   "20M",  "HT",   "2T",   "112",  "30",   "FCC",  "5G",   "20M",
+	"HT",   "2T",   "116",  "32",   "ETSI", "5G",   "20M",  "HT",   "2T",
+	"116",  "20",   "MKK",  "5G",   "20M",  "HT",   "2T",   "116",  "30",
+	"FCC",  "5G",   "20M",  "HT",   "2T",   "120",  "32",   "ETSI", "5G",
+	"20M",  "HT",   "2T",   "120",  "20",   "MKK",  "5G",   "20M",  "HT",
+	"2T",   "120",  "30",   "FCC",  "5G",   "20M",  "HT",   "2T",   "124",
+	"32",   "ETSI", "5G",   "20M",  "HT",   "2T",   "124",  "20",   "MKK",
+	"5G",   "20M",  "HT",   "2T",   "124",  "30",   "FCC",  "5G",   "20M",
+	"HT",   "2T",   "128",  "32",   "ETSI", "5G",   "20M",  "HT",   "2T",
+	"128",  "20",   "MKK",  "5G",   "20M",  "HT",   "2T",   "128",  "30",
+	"FCC",  "5G",   "20M",  "HT",   "2T",   "132",  "32",   "ETSI", "5G",
+	"20M",  "HT",   "2T",   "132",  "20",   "MKK",  "5G",   "20M",  "HT",
+	"2T",   "132",  "30",   "FCC",  "5G",   "20M",  "HT",   "2T",   "136",
+	"30",   "ETSI", "5G",   "20M",  "HT",   "2T",   "136",  "20",   "MKK",
+	"5G",   "20M",  "HT",   "2T",   "136",  "30",   "FCC",  "5G",   "20M",
+	"HT",   "2T",   "140",  "26",   "ETSI", "5G",   "20M",  "HT",   "2T",
+	"140",  "20",   "MKK",  "5G",   "20M",  "HT",   "2T",   "140",  "30",
+	"FCC",  "5G",   "20M",  "HT",   "2T",   "144",  "26",   "ETSI", "5G",
+	"20M",  "HT",   "2T",   "144",  "63",   "MKK",  "5G",   "20M",  "HT",
+	"2T",   "144",  "63",   "FCC",  "5G",   "20M",  "HT",   "2T",   "149",
+	"32",   "ETSI", "5G",   "20M",  "HT",   "2T",   "149",  "63",   "MKK",
+	"5G",   "20M",  "HT",   "2T",   "149",  "63",   "FCC",  "5G",   "20M",
+	"HT",   "2T",   "153",  "32",   "ETSI", "5G",   "20M",  "HT",   "2T",
+	"153",  "63",   "MKK",  "5G",   "20M",  "HT",   "2T",   "153",  "63",
+	"FCC",  "5G",   "20M",  "HT",   "2T",   "157",  "32",   "ETSI", "5G",
+	"20M",  "HT",   "2T",   "157",  "63",   "MKK",  "5G",   "20M",  "HT",
+	"2T",   "157",  "63",   "FCC",  "5G",   "20M",  "HT",   "2T",   "161",
+	"32",   "ETSI", "5G",   "20M",  "HT",   "2T",   "161",  "63",   "MKK",
+	"5G",   "20M",  "HT",   "2T",   "161",  "63",   "FCC",  "5G",   "20M",
+	"HT",   "2T",   "165",  "32",   "ETSI", "5G",   "20M",  "HT",   "2T",
+	"165",  "63",   "MKK",  "5G",   "20M",  "HT",   "2T",   "165",  "63",
+	"FCC",  "5G",   "40M",  "HT",   "1T",   "38",   "22",   "ETSI", "5G",
+	"40M",  "HT",   "1T",   "38",   "30",   "MKK",  "5G",   "40M",  "HT",
+	"1T",   "38",   "30",   "FCC",  "5G",   "40M",  "HT",   "1T",   "46",
+	"30",   "ETSI", "5G",   "40M",  "HT",   "1T",   "46",   "30",   "MKK",
+	"5G",   "40M",  "HT",   "1T",   "46",   "30",   "FCC",  "5G",   "40M",
+	"HT",   "1T",   "54",   "30",   "ETSI", "5G",   "40M",  "HT",   "1T",
+	"54",   "30",   "MKK",  "5G",   "40M",  "HT",   "1T",   "54",   "30",
+	"FCC",  "5G",   "40M",  "HT",   "1T",   "62",   "24",   "ETSI", "5G",
+	"40M",  "HT",   "1T",   "62",   "30",   "MKK",  "5G",   "40M",  "HT",
+	"1T",   "62",   "30",   "FCC",  "5G",   "40M",  "HT",   "1T",   "102",
+	"24",   "ETSI", "5G",   "40M",  "HT",   "1T",   "102",  "30",   "MKK",
+	"5G",   "40M",  "HT",   "1T",   "102",  "30",   "FCC",  "5G",   "40M",
+	"HT",   "1T",   "110",  "30",   "ETSI", "5G",   "40M",  "HT",   "1T",
+	"110",  "30",   "MKK",  "5G",   "40M",  "HT",   "1T",   "110",  "30",
+	"FCC",  "5G",   "40M",  "HT",   "1T",   "118",  "30",   "ETSI", "5G",
+	"40M",  "HT",   "1T",   "118",  "30",   "MKK",  "5G",   "40M",  "HT",
+	"1T",   "118",  "30",   "FCC",  "5G",   "40M",  "HT",   "1T",   "126",
+	"30",   "ETSI", "5G",   "40M",  "HT",   "1T",   "126",  "30",   "MKK",
+	"5G",   "40M",  "HT",   "1T",   "126",  "30",   "FCC",  "5G",   "40M",
+	"HT",   "1T",   "134",  "30",   "ETSI", "5G",   "40M",  "HT",   "1T",
+	"134",  "30",   "MKK",  "5G",   "40M",  "HT",   "1T",   "134",  "30",
+	"FCC",  "5G",   "40M",  "HT",   "1T",   "142",  "30",   "ETSI", "5G",
+	"40M",  "HT",   "1T",   "142",  "63",   "MKK",  "5G",   "40M",  "HT",
+	"1T",   "142",  "63",   "FCC",  "5G",   "40M",  "HT",   "1T",   "151",
+	"30",   "ETSI", "5G",   "40M",  "HT",   "1T",   "151",  "63",   "MKK",
+	"5G",   "40M",  "HT",   "1T",   "151",  "63",   "FCC",  "5G",   "40M",
+	"HT",   "1T",   "159",  "30",   "ETSI", "5G",   "40M",  "HT",   "1T",
+	"159",  "63",   "MKK",  "5G",   "40M",  "HT",   "1T",   "159",  "63",
+	"FCC",  "5G",   "40M",  "HT",   "2T",   "38",   "20",   "ETSI", "5G",
+	"40M",  "HT",   "2T",   "38",   "20",   "MKK",  "5G",   "40M",  "HT",
+	"2T",   "38",   "22",   "FCC",  "5G",   "40M",  "HT",   "2T",   "46",
+	"30",   "ETSI", "5G",   "40M",  "HT",   "2T",   "46",   "20",   "MKK",
+	"5G",   "40M",  "HT",   "2T",   "46",   "22",   "FCC",  "5G",   "40M",
+	"HT",   "2T",   "54",   "30",   "ETSI", "5G",   "40M",  "HT",   "2T",
+	"54",   "20",   "MKK",  "5G",   "40M",  "HT",   "2T",   "54",   "22",
+	"FCC",  "5G",   "40M",  "HT",   "2T",   "62",   "22",   "ETSI", "5G",
+	"40M",  "HT",   "2T",   "62",   "20",   "MKK",  "5G",   "40M",  "HT",
+	"2T",   "62",   "22",   "FCC",  "5G",   "40M",  "HT",   "2T",   "102",
+	"22",   "ETSI", "5G",   "40M",  "HT",   "2T",   "102",  "20",   "MKK",
+	"5G",   "40M",  "HT",   "2T",   "102",  "30",   "FCC",  "5G",   "40M",
+	"HT",   "2T",   "110",  "30",   "ETSI", "5G",   "40M",  "HT",   "2T",
+	"110",  "20",   "MKK",  "5G",   "40M",  "HT",   "2T",   "110",  "30",
+	"FCC",  "5G",   "40M",  "HT",   "2T",   "118",  "30",   "ETSI", "5G",
+	"40M",  "HT",   "2T",   "118",  "20",   "MKK",  "5G",   "40M",  "HT",
+	"2T",   "118",  "30",   "FCC",  "5G",   "40M",  "HT",   "2T",   "126",
+	"30",   "ETSI", "5G",   "40M",  "HT",   "2T",   "126",  "20",   "MKK",
+	"5G",   "40M",  "HT",   "2T",   "126",  "30",   "FCC",  "5G",   "40M",
+	"HT",   "2T",   "134",  "30",   "ETSI", "5G",   "40M",  "HT",   "2T",
+	"134",  "20",   "MKK",  "5G",   "40M",  "HT",   "2T",   "134",  "30",
+	"FCC",  "5G",   "40M",  "HT",   "2T",   "142",  "30",   "ETSI", "5G",
+	"40M",  "HT",   "2T",   "142",  "63",   "MKK",  "5G",   "40M",  "HT",
+	"2T",   "142",  "63",   "FCC",  "5G",   "40M",  "HT",   "2T",   "151",
+	"30",   "ETSI", "5G",   "40M",  "HT",   "2T",   "151",  "63",   "MKK",
+	"5G",   "40M",  "HT",   "2T",   "151",  "63",   "FCC",  "5G",   "40M",
+	"HT",   "2T",   "159",  "30",   "ETSI", "5G",   "40M",  "HT",   "2T",
+	"159",  "63",   "MKK",  "5G",   "40M",  "HT",   "2T",   "159",  "63",
+	"FCC",  "5G",   "80M",  "VHT",  "1T",   "42",   "20",   "ETSI", "5G",
+	"80M",  "VHT",  "1T",   "42",   "30",   "MKK",  "5G",   "80M",  "VHT",
+	"1T",   "42",   "28",   "FCC",  "5G",   "80M",  "VHT",  "1T",   "58",
+	"20",   "ETSI", "5G",   "80M",  "VHT",  "1T",   "58",   "30",   "MKK",
+	"5G",   "80M",  "VHT",  "1T",   "58",   "28",   "FCC",  "5G",   "80M",
+	"VHT",  "1T",   "106",  "20",   "ETSI", "5G",   "80M",  "VHT",  "1T",
+	"106",  "30",   "MKK",  "5G",   "80M",  "VHT",  "1T",   "106",  "30",
+	"FCC",  "5G",   "80M",  "VHT",  "1T",   "122",  "30",   "ETSI", "5G",
+	"80M",  "VHT",  "1T",   "122",  "30",   "MKK",  "5G",   "80M",  "VHT",
+	"1T",   "122",  "30",   "FCC",  "5G",   "80M",  "VHT",  "1T",   "138",
+	"30",   "ETSI", "5G",   "80M",  "VHT",  "1T",   "138",  "63",   "MKK",
+	"5G",   "80M",  "VHT",  "1T",   "138",  "63",   "FCC",  "5G",   "80M",
+	"VHT",  "1T",   "155",  "30",   "ETSI", "5G",   "80M",  "VHT",  "1T",
+	"155",  "63",   "MKK",  "5G",   "80M",  "VHT",  "1T",   "155",  "63",
+	"FCC",  "5G",   "80M",  "VHT",  "2T",   "42",   "18",   "ETSI", "5G",
+	"80M",  "VHT",  "2T",   "42",   "20",   "MKK",  "5G",   "80M",  "VHT",
+	"2T",   "42",   "22",   "FCC",  "5G",   "80M",  "VHT",  "2T",   "58",
+	"18",   "ETSI", "5G",   "80M",  "VHT",  "2T",   "58",   "20",   "MKK",
+	"5G",   "80M",  "VHT",  "2T",   "58",   "22",   "FCC",  "5G",   "80M",
+	"VHT",  "2T",   "106",  "20",   "ETSI", "5G",   "80M",  "VHT",  "2T",
+	"106",  "20",   "MKK",  "5G",   "80M",  "VHT",  "2T",   "106",  "30",
+	"FCC",  "5G",   "80M",  "VHT",  "2T",   "122",  "30",   "ETSI", "5G",
+	"80M",  "VHT",  "2T",   "122",  "20",   "MKK",  "5G",   "80M",  "VHT",
+	"2T",   "122",  "30",   "FCC",  "5G",   "80M",  "VHT",  "2T",   "138",
+	"30",   "ETSI", "5G",   "80M",  "VHT",  "2T",   "138",  "63",   "MKK",
+	"5G",   "80M",  "VHT",  "2T",   "138",  "63",   "FCC",  "5G",   "80M",
+	"VHT",  "2T",   "155",  "30",   "ETSI", "5G",   "80M",  "VHT",  "2T",
+	"155",  "63",   "MKK",  "5G",   "80M",  "VHT",  "2T",   "155",  "63"};
+
+void odm_read_and_config_mp_8822b_txpwr_lmt(struct phy_dm_struct *dm)
+{
+	u32 i = 0;
+	u32 array_len = sizeof(array_mp_8822b_txpwr_lmt) / sizeof(u8 *);
+	u8 **array = (u8 **)array_mp_8822b_txpwr_lmt;
+
+	ODM_RT_TRACE(dm, ODM_COMP_INIT,
+		     "===> %s\n", __func__);
+
+	for (i = 0; i < array_len; i += 7) {
+		u8 *regulation = array[i];
+		u8 *band = array[i + 1];
+		u8 *bandwidth = array[i + 2];
+		u8 *rate = array[i + 3];
+		u8 *rf_path = array[i + 4];
+		u8 *chnl = array[i + 5];
+		u8 *val = array[i + 6];
+
+		odm_config_bb_txpwr_lmt_8822b(dm, regulation, band, bandwidth,
+					      rate, rf_path, chnl, val);
+	}
+}
+
+/******************************************************************************
+*                           txpwr_lmt_type5.TXT
+******************************************************************************/
+
+static const char *const array_mp_8822b_txpwr_lmt_type5[] = {
+	"FCC",  "2.4G", "20M",  "CCK",  "1T",   "01",   "32",   "ETSI", "2.4G",
+	"20M",  "CCK",  "1T",   "01",   "28",   "MKK",  "2.4G", "20M",  "CCK",
+	"1T",   "01",   "30",   "FCC",  "2.4G", "20M",  "CCK",  "1T",   "02",
+	"32",   "ETSI", "2.4G", "20M",  "CCK",  "1T",   "02",   "28",   "MKK",
+	"2.4G", "20M",  "CCK",  "1T",   "02",   "30",   "FCC",  "2.4G", "20M",
+	"CCK",  "1T",   "03",   "32",   "ETSI", "2.4G", "20M",  "CCK",  "1T",
+	"03",   "28",   "MKK",  "2.4G", "20M",  "CCK",  "1T",   "03",   "30",
+	"FCC",  "2.4G", "20M",  "CCK",  "1T",   "04",   "32",   "ETSI", "2.4G",
+	"20M",  "CCK",  "1T",   "04",   "28",   "MKK",  "2.4G", "20M",  "CCK",
+	"1T",   "04",   "30",   "FCC",  "2.4G", "20M",  "CCK",  "1T",   "05",
+	"32",   "ETSI", "2.4G", "20M",  "CCK",  "1T",   "05",   "28",   "MKK",
+	"2.4G", "20M",  "CCK",  "1T",   "05",   "30",   "FCC",  "2.4G", "20M",
+	"CCK",  "1T",   "06",   "32",   "ETSI", "2.4G", "20M",  "CCK",  "1T",
+	"06",   "28",   "MKK",  "2.4G", "20M",  "CCK",  "1T",   "06",   "30",
+	"FCC",  "2.4G", "20M",  "CCK",  "1T",   "07",   "32",   "ETSI", "2.4G",
+	"20M",  "CCK",  "1T",   "07",   "28",   "MKK",  "2.4G", "20M",  "CCK",
+	"1T",   "07",   "30",   "FCC",  "2.4G", "20M",  "CCK",  "1T",   "08",
+	"32",   "ETSI", "2.4G", "20M",  "CCK",  "1T",   "08",   "28",   "MKK",
+	"2.4G", "20M",  "CCK",  "1T",   "08",   "30",   "FCC",  "2.4G", "20M",
+	"CCK",  "1T",   "09",   "32",   "ETSI", "2.4G", "20M",  "CCK",  "1T",
+	"09",   "28",   "MKK",  "2.4G", "20M",  "CCK",  "1T",   "09",   "30",
+	"FCC",  "2.4G", "20M",  "CCK",  "1T",   "10",   "32",   "ETSI", "2.4G",
+	"20M",  "CCK",  "1T",   "10",   "28",   "MKK",  "2.4G", "20M",  "CCK",
+	"1T",   "10",   "30",   "FCC",  "2.4G", "20M",  "CCK",  "1T",   "11",
+	"32",   "ETSI", "2.4G", "20M",  "CCK",  "1T",   "11",   "28",   "MKK",
+	"2.4G", "20M",  "CCK",  "1T",   "11",   "30",   "FCC",  "2.4G", "20M",
+	"CCK",  "1T",   "12",   "26",   "ETSI", "2.4G", "20M",  "CCK",  "1T",
+	"12",   "28",   "MKK",  "2.4G", "20M",  "CCK",  "1T",   "12",   "30",
+	"FCC",  "2.4G", "20M",  "CCK",  "1T",   "13",   "20",   "ETSI", "2.4G",
+	"20M",  "CCK",  "1T",   "13",   "28",   "MKK",  "2.4G", "20M",  "CCK",
+	"1T",   "13",   "28",   "FCC",  "2.4G", "20M",  "CCK",  "1T",   "14",
+	"63",   "ETSI", "2.4G", "20M",  "CCK",  "1T",   "14",   "63",   "MKK",
+	"2.4G", "20M",  "CCK",  "1T",   "14",   "32",   "FCC",  "2.4G", "20M",
+	"OFDM", "1T",   "01",   "26",   "ETSI", "2.4G", "20M",  "OFDM", "1T",
+	"01",   "30",   "MKK",  "2.4G", "20M",  "OFDM", "1T",   "01",   "34",
+	"FCC",  "2.4G", "20M",  "OFDM", "1T",   "02",   "30",   "ETSI", "2.4G",
+	"20M",  "OFDM", "1T",   "02",   "30",   "MKK",  "2.4G", "20M",  "OFDM",
+	"1T",   "02",   "34",   "FCC",  "2.4G", "20M",  "OFDM", "1T",   "03",
+	"32",   "ETSI", "2.4G", "20M",  "OFDM", "1T",   "03",   "30",   "MKK",
+	"2.4G", "20M",  "OFDM", "1T",   "03",   "34",   "FCC",  "2.4G", "20M",
+	"OFDM", "1T",   "04",   "34",   "ETSI", "2.4G", "20M",  "OFDM", "1T",
+	"04",   "30",   "MKK",  "2.4G", "20M",  "OFDM", "1T",   "04",   "34",
+	"FCC",  "2.4G", "20M",  "OFDM", "1T",   "05",   "34",   "ETSI", "2.4G",
+	"20M",  "OFDM", "1T",   "05",   "30",   "MKK",  "2.4G", "20M",  "OFDM",
+	"1T",   "05",   "34",   "FCC",  "2.4G", "20M",  "OFDM", "1T",   "06",
+	"34",   "ETSI", "2.4G", "20M",  "OFDM", "1T",   "06",   "30",   "MKK",
+	"2.4G", "20M",  "OFDM", "1T",   "06",   "34",   "FCC",  "2.4G", "20M",
+	"OFDM", "1T",   "07",   "34",   "ETSI", "2.4G", "20M",  "OFDM", "1T",
+	"07",   "30",   "MKK",  "2.4G", "20M",  "OFDM", "1T",   "07",   "34",
+	"FCC",  "2.4G", "20M",  "OFDM", "1T",   "08",   "34",   "ETSI", "2.4G",
+	"20M",  "OFDM", "1T",   "08",   "30",   "MKK",  "2.4G", "20M",  "OFDM",
+	"1T",   "08",   "34",   "FCC",  "2.4G", "20M",  "OFDM", "1T",   "09",
+	"32",   "ETSI", "2.4G", "20M",  "OFDM", "1T",   "09",   "30",   "MKK",
+	"2.4G", "20M",  "OFDM", "1T",   "09",   "34",   "FCC",  "2.4G", "20M",
+	"OFDM", "1T",   "10",   "30",   "ETSI", "2.4G", "20M",  "OFDM", "1T",
+	"10",   "30",   "MKK",  "2.4G", "20M",  "OFDM", "1T",   "10",   "34",
+	"FCC",  "2.4G", "20M",  "OFDM", "1T",   "11",   "28",   "ETSI", "2.4G",
+	"20M",  "OFDM", "1T",   "11",   "30",   "MKK",  "2.4G", "20M",  "OFDM",
+	"1T",   "11",   "34",   "FCC",  "2.4G", "20M",  "OFDM", "1T",   "12",
+	"22",   "ETSI", "2.4G", "20M",  "OFDM", "1T",   "12",   "30",   "MKK",
+	"2.4G", "20M",  "OFDM", "1T",   "12",   "34",   "FCC",  "2.4G", "20M",
+	"OFDM", "1T",   "13",   "14",   "ETSI", "2.4G", "20M",  "OFDM", "1T",
+	"13",   "30",   "MKK",  "2.4G", "20M",  "OFDM", "1T",   "13",   "34",
+	"FCC",  "2.4G", "20M",  "OFDM", "1T",   "14",   "63",   "ETSI", "2.4G",
+	"20M",  "OFDM", "1T",   "14",   "63",   "MKK",  "2.4G", "20M",  "OFDM",
+	"1T",   "14",   "63",   "FCC",  "2.4G", "20M",  "HT",   "1T",   "01",
+	"26",   "ETSI", "2.4G", "20M",  "HT",   "1T",   "01",   "30",   "MKK",
+	"2.4G", "20M",  "HT",   "1T",   "01",   "34",   "FCC",  "2.4G", "20M",
+	"HT",   "1T",   "02",   "30",   "ETSI", "2.4G", "20M",  "HT",   "1T",
+	"02",   "30",   "MKK",  "2.4G", "20M",  "HT",   "1T",   "02",   "34",
+	"FCC",  "2.4G", "20M",  "HT",   "1T",   "03",   "32",   "ETSI", "2.4G",
+	"20M",  "HT",   "1T",   "03",   "30",   "MKK",  "2.4G", "20M",  "HT",
+	"1T",   "03",   "34",   "FCC",  "2.4G", "20M",  "HT",   "1T",   "04",
+	"34",   "ETSI", "2.4G", "20M",  "HT",   "1T",   "04",   "30",   "MKK",
+	"2.4G", "20M",  "HT",   "1T",   "04",   "34",   "FCC",  "2.4G", "20M",
+	"HT",   "1T",   "05",   "34",   "ETSI", "2.4G", "20M",  "HT",   "1T",
+	"05",   "30",   "MKK",  "2.4G", "20M",  "HT",   "1T",   "05",   "34",
+	"FCC",  "2.4G", "20M",  "HT",   "1T",   "06",   "34",   "ETSI", "2.4G",
+	"20M",  "HT",   "1T",   "06",   "30",   "MKK",  "2.4G", "20M",  "HT",
+	"1T",   "06",   "34",   "FCC",  "2.4G", "20M",  "HT",   "1T",   "07",
+	"34",   "ETSI", "2.4G", "20M",  "HT",   "1T",   "07",   "30",   "MKK",
+	"2.4G", "20M",  "HT",   "1T",   "07",   "34",   "FCC",  "2.4G", "20M",
+	"HT",   "1T",   "08",   "34",   "ETSI", "2.4G", "20M",  "HT",   "1T",
+	"08",   "30",   "MKK",  "2.4G", "20M",  "HT",   "1T",   "08",   "34",
+	"FCC",  "2.4G", "20M",  "HT",   "1T",   "09",   "32",   "ETSI", "2.4G",
+	"20M",  "HT",   "1T",   "09",   "30",   "MKK",  "2.4G", "20M",  "HT",
+	"1T",   "09",   "34",   "FCC",  "2.4G", "20M",  "HT",   "1T",   "10",
+	"30",   "ETSI", "2.4G", "20M",  "HT",   "1T",   "10",   "30",   "MKK",
+	"2.4G", "20M",  "HT",   "1T",   "10",   "34",   "FCC",  "2.4G", "20M",
+	"HT",   "1T",   "11",   "26",   "ETSI", "2.4G", "20M",  "HT",   "1T",
+	"11",   "30",   "MKK",  "2.4G", "20M",  "HT",   "1T",   "11",   "34",
+	"FCC",  "2.4G", "20M",  "HT",   "1T",   "12",   "20",   "ETSI", "2.4G",
+	"20M",  "HT",   "1T",   "12",   "30",   "MKK",  "2.4G", "20M",  "HT",
+	"1T",   "12",   "34",   "FCC",  "2.4G", "20M",  "HT",   "1T",   "13",
+	"14",   "ETSI", "2.4G", "20M",  "HT",   "1T",   "13",   "30",   "MKK",
+	"2.4G", "20M",  "HT",   "1T",   "13",   "34",   "FCC",  "2.4G", "20M",
+	"HT",   "1T",   "14",   "63",   "ETSI", "2.4G", "20M",  "HT",   "1T",
+	"14",   "63",   "MKK",  "2.4G", "20M",  "HT",   "1T",   "14",   "63",
+	"FCC",  "2.4G", "20M",  "HT",   "2T",   "01",   "26",   "ETSI", "2.4G",
+	"20M",  "HT",   "2T",   "01",   "18",   "MKK",  "2.4G", "20M",  "HT",
+	"2T",   "01",   "30",   "FCC",  "2.4G", "20M",  "HT",   "2T",   "02",
+	"28",   "ETSI", "2.4G", "20M",  "HT",   "2T",   "02",   "18",   "MKK",
+	"2.4G", "20M",  "HT",   "2T",   "02",   "30",   "FCC",  "2.4G", "20M",
+	"HT",   "2T",   "03",   "30",   "ETSI", "2.4G", "20M",  "HT",   "2T",
+	"03",   "18",   "MKK",  "2.4G", "20M",  "HT",   "2T",   "03",   "30",
+	"FCC",  "2.4G", "20M",  "HT",   "2T",   "04",   "30",   "ETSI", "2.4G",
+	"20M",  "HT",   "2T",   "04",   "18",   "MKK",  "2.4G", "20M",  "HT",
+	"2T",   "04",   "30",   "FCC",  "2.4G", "20M",  "HT",   "2T",   "05",
+	"32",   "ETSI", "2.4G", "20M",  "HT",   "2T",   "05",   "18",   "MKK",
+	"2.4G", "20M",  "HT",   "2T",   "05",   "30",   "FCC",  "2.4G", "20M",
+	"HT",   "2T",   "06",   "32",   "ETSI", "2.4G", "20M",  "HT",   "2T",
+	"06",   "18",   "MKK",  "2.4G", "20M",  "HT",   "2T",   "06",   "30",
+	"FCC",  "2.4G", "20M",  "HT",   "2T",   "07",   "32",   "ETSI", "2.4G",
+	"20M",  "HT",   "2T",   "07",   "18",   "MKK",  "2.4G", "20M",  "HT",
+	"2T",   "07",   "30",   "FCC",  "2.4G", "20M",  "HT",   "2T",   "08",
+	"30",   "ETSI", "2.4G", "20M",  "HT",   "2T",   "08",   "18",   "MKK",
+	"2.4G", "20M",  "HT",   "2T",   "08",   "30",   "FCC",  "2.4G", "20M",
+	"HT",   "2T",   "09",   "30",   "ETSI", "2.4G", "20M",  "HT",   "2T",
+	"09",   "18",   "MKK",  "2.4G", "20M",  "HT",   "2T",   "09",   "30",
+	"FCC",  "2.4G", "20M",  "HT",   "2T",   "10",   "28",   "ETSI", "2.4G",
+	"20M",  "HT",   "2T",   "10",   "18",   "MKK",  "2.4G", "20M",  "HT",
+	"2T",   "10",   "30",   "FCC",  "2.4G", "20M",  "HT",   "2T",   "11",
+	"26",   "ETSI", "2.4G", "20M",  "HT",   "2T",   "11",   "18",   "MKK",
+	"2.4G", "20M",  "HT",   "2T",   "11",   "30",   "FCC",  "2.4G", "20M",
+	"HT",   "2T",   "12",   "20",   "ETSI", "2.4G", "20M",  "HT",   "2T",
+	"12",   "18",   "MKK",  "2.4G", "20M",  "HT",   "2T",   "12",   "30",
+	"FCC",  "2.4G", "20M",  "HT",   "2T",   "13",   "14",   "ETSI", "2.4G",
+	"20M",  "HT",   "2T",   "13",   "18",   "MKK",  "2.4G", "20M",  "HT",
+	"2T",   "13",   "30",   "FCC",  "2.4G", "20M",  "HT",   "2T",   "14",
+	"63",   "ETSI", "2.4G", "20M",  "HT",   "2T",   "14",   "63",   "MKK",
+	"2.4G", "20M",  "HT",   "2T",   "14",   "63",   "FCC",  "2.4G", "40M",
+	"HT",   "1T",   "01",   "63",   "ETSI", "2.4G", "40M",  "HT",   "1T",
+	"01",   "63",   "MKK",  "2.4G", "40M",  "HT",   "1T",   "01",   "63",
+	"FCC",  "2.4G", "40M",  "HT",   "1T",   "02",   "63",   "ETSI", "2.4G",
+	"40M",  "HT",   "1T",   "02",   "63",   "MKK",  "2.4G", "40M",  "HT",
+	"1T",   "02",   "63",   "FCC",  "2.4G", "40M",  "HT",   "1T",   "03",
+	"26",   "ETSI", "2.4G", "40M",  "HT",   "1T",   "03",   "30",   "MKK",
+	"2.4G", "40M",  "HT",   "1T",   "03",   "34",   "FCC",  "2.4G", "40M",
+	"HT",   "1T",   "04",   "26",   "ETSI", "2.4G", "40M",  "HT",   "1T",
+	"04",   "30",   "MKK",  "2.4G", "40M",  "HT",   "1T",   "04",   "34",
+	"FCC",  "2.4G", "40M",  "HT",   "1T",   "05",   "30",   "ETSI", "2.4G",
+	"40M",  "HT",   "1T",   "05",   "30",   "MKK",  "2.4G", "40M",  "HT",
+	"1T",   "05",   "34",   "FCC",  "2.4G", "40M",  "HT",   "1T",   "06",
+	"32",   "ETSI", "2.4G", "40M",  "HT",   "1T",   "06",   "30",   "MKK",
+	"2.4G", "40M",  "HT",   "1T",   "06",   "34",   "FCC",  "2.4G", "40M",
+	"HT",   "1T",   "07",   "30",   "ETSI", "2.4G", "40M",  "HT",   "1T",
+	"07",   "30",   "MKK",  "2.4G", "40M",  "HT",   "1T",   "07",   "34",
+	"FCC",  "2.4G", "40M",  "HT",   "1T",   "08",   "26",   "ETSI", "2.4G",
+	"40M",  "HT",   "1T",   "08",   "30",   "MKK",  "2.4G", "40M",  "HT",
+	"1T",   "08",   "34",   "FCC",  "2.4G", "40M",  "HT",   "1T",   "09",
+	"26",   "ETSI", "2.4G", "40M",  "HT",   "1T",   "09",   "30",   "MKK",
+	"2.4G", "40M",  "HT",   "1T",   "09",   "34",   "FCC",  "2.4G", "40M",
+	"HT",   "1T",   "10",   "20",   "ETSI", "2.4G", "40M",  "HT",   "1T",
+	"10",   "30",   "MKK",  "2.4G", "40M",  "HT",   "1T",   "10",   "34",
+	"FCC",  "2.4G", "40M",  "HT",   "1T",   "11",   "14",   "ETSI", "2.4G",
+	"40M",  "HT",   "1T",   "11",   "30",   "MKK",  "2.4G", "40M",  "HT",
+	"1T",   "11",   "34",   "FCC",  "2.4G", "40M",  "HT",   "1T",   "12",
+	"63",   "ETSI", "2.4G", "40M",  "HT",   "1T",   "12",   "63",   "MKK",
+	"2.4G", "40M",  "HT",   "1T",   "12",   "63",   "FCC",  "2.4G", "40M",
+	"HT",   "1T",   "13",   "63",   "ETSI", "2.4G", "40M",  "HT",   "1T",
+	"13",   "63",   "MKK",  "2.4G", "40M",  "HT",   "1T",   "13",   "63",
+	"FCC",  "2.4G", "40M",  "HT",   "1T",   "14",   "63",   "ETSI", "2.4G",
+	"40M",  "HT",   "1T",   "14",   "63",   "MKK",  "2.4G", "40M",  "HT",
+	"1T",   "14",   "63",   "FCC",  "2.4G", "40M",  "HT",   "2T",   "01",
+	"63",   "ETSI", "2.4G", "40M",  "HT",   "2T",   "01",   "63",   "MKK",
+	"2.4G", "40M",  "HT",   "2T",   "01",   "63",   "FCC",  "2.4G", "40M",
+	"HT",   "2T",   "02",   "63",   "ETSI", "2.4G", "40M",  "HT",   "2T",
+	"02",   "63",   "MKK",  "2.4G", "40M",  "HT",   "2T",   "02",   "63",
+	"FCC",  "2.4G", "40M",  "HT",   "2T",   "03",   "24",   "ETSI", "2.4G",
+	"40M",  "HT",   "2T",   "03",   "18",   "MKK",  "2.4G", "40M",  "HT",
+	"2T",   "03",   "30",   "FCC",  "2.4G", "40M",  "HT",   "2T",   "04",
+	"24",   "ETSI", "2.4G", "40M",  "HT",   "2T",   "04",   "18",   "MKK",
+	"2.4G", "40M",  "HT",   "2T",   "04",   "30",   "FCC",  "2.4G", "40M",
+	"HT",   "2T",   "05",   "26",   "ETSI", "2.4G", "40M",  "HT",   "2T",
+	"05",   "18",   "MKK",  "2.4G", "40M",  "HT",   "2T",   "05",   "30",
+	"FCC",  "2.4G", "40M",  "HT",   "2T",   "06",   "28",   "ETSI", "2.4G",
+	"40M",  "HT",   "2T",   "06",   "18",   "MKK",  "2.4G", "40M",  "HT",
+	"2T",   "06",   "30",   "FCC",  "2.4G", "40M",  "HT",   "2T",   "07",
+	"26",   "ETSI", "2.4G", "40M",  "HT",   "2T",   "07",   "18",   "MKK",
+	"2.4G", "40M",  "HT",   "2T",   "07",   "30",   "FCC",  "2.4G", "40M",
+	"HT",   "2T",   "08",   "26",   "ETSI", "2.4G", "40M",  "HT",   "2T",
+	"08",   "18",   "MKK",  "2.4G", "40M",  "HT",   "2T",   "08",   "30",
+	"FCC",  "2.4G", "40M",  "HT",   "2T",   "09",   "26",   "ETSI", "2.4G",
+	"40M",  "HT",   "2T",   "09",   "18",   "MKK",  "2.4G", "40M",  "HT",
+	"2T",   "09",   "30",   "FCC",  "2.4G", "40M",  "HT",   "2T",   "10",
+	"20",   "ETSI", "2.4G", "40M",  "HT",   "2T",   "10",   "18",   "MKK",
+	"2.4G", "40M",  "HT",   "2T",   "10",   "30",   "FCC",  "2.4G", "40M",
+	"HT",   "2T",   "11",   "14",   "ETSI", "2.4G", "40M",  "HT",   "2T",
+	"11",   "18",   "MKK",  "2.4G", "40M",  "HT",   "2T",   "11",   "30",
+	"FCC",  "2.4G", "40M",  "HT",   "2T",   "12",   "63",   "ETSI", "2.4G",
+	"40M",  "HT",   "2T",   "12",   "63",   "MKK",  "2.4G", "40M",  "HT",
+	"2T",   "12",   "63",   "FCC",  "2.4G", "40M",  "HT",   "2T",   "13",
+	"63",   "ETSI", "2.4G", "40M",  "HT",   "2T",   "13",   "63",   "MKK",
+	"2.4G", "40M",  "HT",   "2T",   "13",   "63",   "FCC",  "2.4G", "40M",
+	"HT",   "2T",   "14",   "63",   "ETSI", "2.4G", "40M",  "HT",   "2T",
+	"14",   "63",   "MKK",  "2.4G", "40M",  "HT",   "2T",   "14",   "63",
+	"FCC",  "5G",   "20M",  "OFDM", "1T",   "36",   "30",   "ETSI", "5G",
+	"20M",  "OFDM", "1T",   "36",   "32",   "MKK",  "5G",   "20M",  "OFDM",
+	"1T",   "36",   "30",   "FCC",  "5G",   "20M",  "OFDM", "1T",   "40",
+	"32",   "ETSI", "5G",   "20M",  "OFDM", "1T",   "40",   "32",   "MKK",
+	"5G",   "20M",  "OFDM", "1T",   "40",   "30",   "FCC",  "5G",   "20M",
+	"OFDM", "1T",   "44",   "32",   "ETSI", "5G",   "20M",  "OFDM", "1T",
+	"44",   "32",   "MKK",  "5G",   "20M",  "OFDM", "1T",   "44",   "30",
+	"FCC",  "5G",   "20M",  "OFDM", "1T",   "48",   "32",   "ETSI", "5G",
+	"20M",  "OFDM", "1T",   "48",   "32",   "MKK",  "5G",   "20M",  "OFDM",
+	"1T",   "48",   "30",   "FCC",  "5G",   "20M",  "OFDM", "1T",   "52",
+	"32",   "ETSI", "5G",   "20M",  "OFDM", "1T",   "52",   "32",   "MKK",
+	"5G",   "20M",  "OFDM", "1T",   "52",   "28",   "FCC",  "5G",   "20M",
+	"OFDM", "1T",   "56",   "32",   "ETSI", "5G",   "20M",  "OFDM", "1T",
+	"56",   "32",   "MKK",  "5G",   "20M",  "OFDM", "1T",   "56",   "28",
+	"FCC",  "5G",   "20M",  "OFDM", "1T",   "60",   "32",   "ETSI", "5G",
+	"20M",  "OFDM", "1T",   "60",   "32",   "MKK",  "5G",   "20M",  "OFDM",
+	"1T",   "60",   "28",   "FCC",  "5G",   "20M",  "OFDM", "1T",   "64",
+	"28",   "ETSI", "5G",   "20M",  "OFDM", "1T",   "64",   "32",   "MKK",
+	"5G",   "20M",  "OFDM", "1T",   "64",   "28",   "FCC",  "5G",   "20M",
+	"OFDM", "1T",   "100",  "26",   "ETSI", "5G",   "20M",  "OFDM", "1T",
+	"100",  "32",   "MKK",  "5G",   "20M",  "OFDM", "1T",   "100",  "32",
+	"FCC",  "5G",   "20M",  "OFDM", "1T",   "104",  "32",   "ETSI", "5G",
+	"20M",  "OFDM", "1T",   "104",  "32",   "MKK",  "5G",   "20M",  "OFDM",
+	"1T",   "104",  "32",   "FCC",  "5G",   "20M",  "OFDM", "1T",   "108",
+	"32",   "ETSI", "5G",   "20M",  "OFDM", "1T",   "108",  "32",   "MKK",
+	"5G",   "20M",  "OFDM", "1T",   "108",  "32",   "FCC",  "5G",   "20M",
+	"OFDM", "1T",   "112",  "32",   "ETSI", "5G",   "20M",  "OFDM", "1T",
+	"112",  "32",   "MKK",  "5G",   "20M",  "OFDM", "1T",   "112",  "32",
+	"FCC",  "5G",   "20M",  "OFDM", "1T",   "116",  "32",   "ETSI", "5G",
+	"20M",  "OFDM", "1T",   "116",  "32",   "MKK",  "5G",   "20M",  "OFDM",
+	"1T",   "116",  "32",   "FCC",  "5G",   "20M",  "OFDM", "1T",   "120",
+	"32",   "ETSI", "5G",   "20M",  "OFDM", "1T",   "120",  "32",   "MKK",
+	"5G",   "20M",  "OFDM", "1T",   "120",  "32",   "FCC",  "5G",   "20M",
+	"OFDM", "1T",   "124",  "32",   "ETSI", "5G",   "20M",  "OFDM", "1T",
+	"124",  "32",   "MKK",  "5G",   "20M",  "OFDM", "1T",   "124",  "32",
+	"FCC",  "5G",   "20M",  "OFDM", "1T",   "128",  "32",   "ETSI", "5G",
+	"20M",  "OFDM", "1T",   "128",  "32",   "MKK",  "5G",   "20M",  "OFDM",
+	"1T",   "128",  "32",   "FCC",  "5G",   "20M",  "OFDM", "1T",   "132",
+	"32",   "ETSI", "5G",   "20M",  "OFDM", "1T",   "132",  "32",   "MKK",
+	"5G",   "20M",  "OFDM", "1T",   "132",  "32",   "FCC",  "5G",   "20M",
+	"OFDM", "1T",   "136",  "32",   "ETSI", "5G",   "20M",  "OFDM", "1T",
+	"136",  "32",   "MKK",  "5G",   "20M",  "OFDM", "1T",   "136",  "32",
+	"FCC",  "5G",   "20M",  "OFDM", "1T",   "140",  "28",   "ETSI", "5G",
+	"20M",  "OFDM", "1T",   "140",  "32",   "MKK",  "5G",   "20M",  "OFDM",
+	"1T",   "140",  "32",   "FCC",  "5G",   "20M",  "OFDM", "1T",   "144",
+	"28",   "ETSI", "5G",   "20M",  "OFDM", "1T",   "144",  "32",   "MKK",
+	"5G",   "20M",  "OFDM", "1T",   "144",  "63",   "FCC",  "5G",   "20M",
+	"OFDM", "1T",   "149",  "32",   "ETSI", "5G",   "20M",  "OFDM", "1T",
+	"149",  "63",   "MKK",  "5G",   "20M",  "OFDM", "1T",   "149",  "63",
+	"FCC",  "5G",   "20M",  "OFDM", "1T",   "153",  "32",   "ETSI", "5G",
+	"20M",  "OFDM", "1T",   "153",  "63",   "MKK",  "5G",   "20M",  "OFDM",
+	"1T",   "153",  "63",   "FCC",  "5G",   "20M",  "OFDM", "1T",   "157",
+	"32",   "ETSI", "5G",   "20M",  "OFDM", "1T",   "157",  "63",   "MKK",
+	"5G",   "20M",  "OFDM", "1T",   "157",  "63",   "FCC",  "5G",   "20M",
+	"OFDM", "1T",   "161",  "32",   "ETSI", "5G",   "20M",  "OFDM", "1T",
+	"161",  "63",   "MKK",  "5G",   "20M",  "OFDM", "1T",   "161",  "63",
+	"FCC",  "5G",   "20M",  "OFDM", "1T",   "165",  "32",   "ETSI", "5G",
+	"20M",  "OFDM", "1T",   "165",  "63",   "MKK",  "5G",   "20M",  "OFDM",
+	"1T",   "165",  "63",   "FCC",  "5G",   "20M",  "HT",   "1T",   "36",
+	"30",   "ETSI", "5G",   "20M",  "HT",   "1T",   "36",   "32",   "MKK",
+	"5G",   "20M",  "HT",   "1T",   "36",   "28",   "FCC",  "5G",   "20M",
+	"HT",   "1T",   "40",   "32",   "ETSI", "5G",   "20M",  "HT",   "1T",
+	"40",   "32",   "MKK",  "5G",   "20M",  "HT",   "1T",   "40",   "28",
+	"FCC",  "5G",   "20M",  "HT",   "1T",   "44",   "32",   "ETSI", "5G",
+	"20M",  "HT",   "1T",   "44",   "32",   "MKK",  "5G",   "20M",  "HT",
+	"1T",   "44",   "28",   "FCC",  "5G",   "20M",  "HT",   "1T",   "48",
+	"32",   "ETSI", "5G",   "20M",  "HT",   "1T",   "48",   "32",   "MKK",
+	"5G",   "20M",  "HT",   "1T",   "48",   "28",   "FCC",  "5G",   "20M",
+	"HT",   "1T",   "52",   "32",   "ETSI", "5G",   "20M",  "HT",   "1T",
+	"52",   "32",   "MKK",  "5G",   "20M",  "HT",   "1T",   "52",   "28",
+	"FCC",  "5G",   "20M",  "HT",   "1T",   "56",   "32",   "ETSI", "5G",
+	"20M",  "HT",   "1T",   "56",   "32",   "MKK",  "5G",   "20M",  "HT",
+	"1T",   "56",   "28",   "FCC",  "5G",   "20M",  "HT",   "1T",   "60",
+	"32",   "ETSI", "5G",   "20M",  "HT",   "1T",   "60",   "32",   "MKK",
+	"5G",   "20M",  "HT",   "1T",   "60",   "28",   "FCC",  "5G",   "20M",
+	"HT",   "1T",   "64",   "28",   "ETSI", "5G",   "20M",  "HT",   "1T",
+	"64",   "32",   "MKK",  "5G",   "20M",  "HT",   "1T",   "64",   "28",
+	"FCC",  "5G",   "20M",  "HT",   "1T",   "100",  "26",   "ETSI", "5G",
+	"20M",  "HT",   "1T",   "100",  "32",   "MKK",  "5G",   "20M",  "HT",
+	"1T",   "100",  "32",   "FCC",  "5G",   "20M",  "HT",   "1T",   "104",
+	"32",   "ETSI", "5G",   "20M",  "HT",   "1T",   "104",  "32",   "MKK",
+	"5G",   "20M",  "HT",   "1T",   "104",  "32",   "FCC",  "5G",   "20M",
+	"HT",   "1T",   "108",  "32",   "ETSI", "5G",   "20M",  "HT",   "1T",
+	"108",  "32",   "MKK",  "5G",   "20M",  "HT",   "1T",   "108",  "32",
+	"FCC",  "5G",   "20M",  "HT",   "1T",   "112",  "32",   "ETSI", "5G",
+	"20M",  "HT",   "1T",   "112",  "32",   "MKK",  "5G",   "20M",  "HT",
+	"1T",   "112",  "32",   "FCC",  "5G",   "20M",  "HT",   "1T",   "116",
+	"32",   "ETSI", "5G",   "20M",  "HT",   "1T",   "116",  "32",   "MKK",
+	"5G",   "20M",  "HT",   "1T",   "116",  "32",   "FCC",  "5G",   "20M",
+	"HT",   "1T",   "120",  "32",   "ETSI", "5G",   "20M",  "HT",   "1T",
+	"120",  "32",   "MKK",  "5G",   "20M",  "HT",   "1T",   "120",  "32",
+	"FCC",  "5G",   "20M",  "HT",   "1T",   "124",  "32",   "ETSI", "5G",
+	"20M",  "HT",   "1T",   "124",  "32",   "MKK",  "5G",   "20M",  "HT",
+	"1T",   "124",  "32",   "FCC",  "5G",   "20M",  "HT",   "1T",   "128",
+	"32",   "ETSI", "5G",   "20M",  "HT",   "1T",   "128",  "32",   "MKK",
+	"5G",   "20M",  "HT",   "1T",   "128",  "32",   "FCC",  "5G",   "20M",
+	"HT",   "1T",   "132",  "32",   "ETSI", "5G",   "20M",  "HT",   "1T",
+	"132",  "32",   "MKK",  "5G",   "20M",  "HT",   "1T",   "132",  "32",
+	"FCC",  "5G",   "20M",  "HT",   "1T",   "136",  "32",   "ETSI", "5G",
+	"20M",  "HT",   "1T",   "136",  "32",   "MKK",  "5G",   "20M",  "HT",
+	"1T",   "136",  "32",   "FCC",  "5G",   "20M",  "HT",   "1T",   "140",
+	"26",   "ETSI", "5G",   "20M",  "HT",   "1T",   "140",  "32",   "MKK",
+	"5G",   "20M",  "HT",   "1T",   "140",  "32",   "FCC",  "5G",   "20M",
+	"HT",   "1T",   "144",  "26",   "ETSI", "5G",   "20M",  "HT",   "1T",
+	"144",  "63",   "MKK",  "5G",   "20M",  "HT",   "1T",   "144",  "63",
+	"FCC",  "5G",   "20M",  "HT",   "1T",   "149",  "32",   "ETSI", "5G",
+	"20M",  "HT",   "1T",   "149",  "63",   "MKK",  "5G",   "20M",  "HT",
+	"1T",   "149",  "63",   "FCC",  "5G",   "20M",  "HT",   "1T",   "153",
+	"32",   "ETSI", "5G",   "20M",  "HT",   "1T",   "153",  "63",   "MKK",
+	"5G",   "20M",  "HT",   "1T",   "153",  "63",   "FCC",  "5G",   "20M",
+	"HT",   "1T",   "157",  "32",   "ETSI", "5G",   "20M",  "HT",   "1T",
+	"157",  "63",   "MKK",  "5G",   "20M",  "HT",   "1T",   "157",  "63",
+	"FCC",  "5G",   "20M",  "HT",   "1T",   "161",  "32",   "ETSI", "5G",
+	"20M",  "HT",   "1T",   "161",  "63",   "MKK",  "5G",   "20M",  "HT",
+	"1T",   "161",  "63",   "FCC",  "5G",   "20M",  "HT",   "1T",   "165",
+	"32",   "ETSI", "5G",   "20M",  "HT",   "1T",   "165",  "63",   "MKK",
+	"5G",   "20M",  "HT",   "1T",   "165",  "63",   "FCC",  "5G",   "20M",
+	"HT",   "2T",   "36",   "28",   "ETSI", "5G",   "20M",  "HT",   "2T",
+	"36",   "20",   "MKK",  "5G",   "20M",  "HT",   "2T",   "36",   "22",
+	"FCC",  "5G",   "20M",  "HT",   "2T",   "40",   "30",   "ETSI", "5G",
+	"20M",  "HT",   "2T",   "40",   "20",   "MKK",  "5G",   "20M",  "HT",
+	"2T",   "40",   "22",   "FCC",  "5G",   "20M",  "HT",   "2T",   "44",
+	"30",   "ETSI", "5G",   "20M",  "HT",   "2T",   "44",   "20",   "MKK",
+	"5G",   "20M",  "HT",   "2T",   "44",   "22",   "FCC",  "5G",   "20M",
+	"HT",   "2T",   "48",   "30",   "ETSI", "5G",   "20M",  "HT",   "2T",
+	"48",   "20",   "MKK",  "5G",   "20M",  "HT",   "2T",   "48",   "22",
+	"FCC",  "5G",   "20M",  "HT",   "2T",   "52",   "30",   "ETSI", "5G",
+	"20M",  "HT",   "2T",   "52",   "20",   "MKK",  "5G",   "20M",  "HT",
+	"2T",   "52",   "22",   "FCC",  "5G",   "20M",  "HT",   "2T",   "56",
+	"30",   "ETSI", "5G",   "20M",  "HT",   "2T",   "56",   "20",   "MKK",
+	"5G",   "20M",  "HT",   "2T",   "56",   "22",   "FCC",  "5G",   "20M",
+	"HT",   "2T",   "60",   "30",   "ETSI", "5G",   "20M",  "HT",   "2T",
+	"60",   "20",   "MKK",  "5G",   "20M",  "HT",   "2T",   "60",   "22",
+	"FCC",  "5G",   "20M",  "HT",   "2T",   "64",   "28",   "ETSI", "5G",
+	"20M",  "HT",   "2T",   "64",   "20",   "MKK",  "5G",   "20M",  "HT",
+	"2T",   "64",   "22",   "FCC",  "5G",   "20M",  "HT",   "2T",   "100",
+	"26",   "ETSI", "5G",   "20M",  "HT",   "2T",   "100",  "20",   "MKK",
+	"5G",   "20M",  "HT",   "2T",   "100",  "30",   "FCC",  "5G",   "20M",
+	"HT",   "2T",   "104",  "30",   "ETSI", "5G",   "20M",  "HT",   "2T",
+	"104",  "20",   "MKK",  "5G",   "20M",  "HT",   "2T",   "104",  "30",
+	"FCC",  "5G",   "20M",  "HT",   "2T",   "108",  "32",   "ETSI", "5G",
+	"20M",  "HT",   "2T",   "108",  "20",   "MKK",  "5G",   "20M",  "HT",
+	"2T",   "108",  "30",   "FCC",  "5G",   "20M",  "HT",   "2T",   "112",
+	"32",   "ETSI", "5G",   "20M",  "HT",   "2T",   "112",  "20",   "MKK",
+	"5G",   "20M",  "HT",   "2T",   "112",  "30",   "FCC",  "5G",   "20M",
+	"HT",   "2T",   "116",  "32",   "ETSI", "5G",   "20M",  "HT",   "2T",
+	"116",  "20",   "MKK",  "5G",   "20M",  "HT",   "2T",   "116",  "30",
+	"FCC",  "5G",   "20M",  "HT",   "2T",   "120",  "32",   "ETSI", "5G",
+	"20M",  "HT",   "2T",   "120",  "20",   "MKK",  "5G",   "20M",  "HT",
+	"2T",   "120",  "30",   "FCC",  "5G",   "20M",  "HT",   "2T",   "124",
+	"32",   "ETSI", "5G",   "20M",  "HT",   "2T",   "124",  "20",   "MKK",
+	"5G",   "20M",  "HT",   "2T",   "124",  "30",   "FCC",  "5G",   "20M",
+	"HT",   "2T",   "128",  "32",   "ETSI", "5G",   "20M",  "HT",   "2T",
+	"128",  "20",   "MKK",  "5G",   "20M",  "HT",   "2T",   "128",  "30",
+	"FCC",  "5G",   "20M",  "HT",   "2T",   "132",  "32",   "ETSI", "5G",
+	"20M",  "HT",   "2T",   "132",  "20",   "MKK",  "5G",   "20M",  "HT",
+	"2T",   "132",  "30",   "FCC",  "5G",   "20M",  "HT",   "2T",   "136",
+	"30",   "ETSI", "5G",   "20M",  "HT",   "2T",   "136",  "20",   "MKK",
+	"5G",   "20M",  "HT",   "2T",   "136",  "30",   "FCC",  "5G",   "20M",
+	"HT",   "2T",   "140",  "26",   "ETSI", "5G",   "20M",  "HT",   "2T",
+	"140",  "20",   "MKK",  "5G",   "20M",  "HT",   "2T",   "140",  "30",
+	"FCC",  "5G",   "20M",  "HT",   "2T",   "144",  "26",   "ETSI", "5G",
+	"20M",  "HT",   "2T",   "144",  "63",   "MKK",  "5G",   "20M",  "HT",
+	"2T",   "144",  "63",   "FCC",  "5G",   "20M",  "HT",   "2T",   "149",
+	"32",   "ETSI", "5G",   "20M",  "HT",   "2T",   "149",  "63",   "MKK",
+	"5G",   "20M",  "HT",   "2T",   "149",  "63",   "FCC",  "5G",   "20M",
+	"HT",   "2T",   "153",  "32",   "ETSI", "5G",   "20M",  "HT",   "2T",
+	"153",  "63",   "MKK",  "5G",   "20M",  "HT",   "2T",   "153",  "63",
+	"FCC",  "5G",   "20M",  "HT",   "2T",   "157",  "32",   "ETSI", "5G",
+	"20M",  "HT",   "2T",   "157",  "63",   "MKK",  "5G",   "20M",  "HT",
+	"2T",   "157",  "63",   "FCC",  "5G",   "20M",  "HT",   "2T",   "161",
+	"32",   "ETSI", "5G",   "20M",  "HT",   "2T",   "161",  "63",   "MKK",
+	"5G",   "20M",  "HT",   "2T",   "161",  "63",   "FCC",  "5G",   "20M",
+	"HT",   "2T",   "165",  "32",   "ETSI", "5G",   "20M",  "HT",   "2T",
+	"165",  "63",   "MKK",  "5G",   "20M",  "HT",   "2T",   "165",  "63",
+	"FCC",  "5G",   "40M",  "HT",   "1T",   "38",   "22",   "ETSI", "5G",
+	"40M",  "HT",   "1T",   "38",   "30",   "MKK",  "5G",   "40M",  "HT",
+	"1T",   "38",   "30",   "FCC",  "5G",   "40M",  "HT",   "1T",   "46",
+	"30",   "ETSI", "5G",   "40M",  "HT",   "1T",   "46",   "30",   "MKK",
+	"5G",   "40M",  "HT",   "1T",   "46",   "30",   "FCC",  "5G",   "40M",
+	"HT",   "1T",   "54",   "30",   "ETSI", "5G",   "40M",  "HT",   "1T",
+	"54",   "30",   "MKK",  "5G",   "40M",  "HT",   "1T",   "54",   "30",
+	"FCC",  "5G",   "40M",  "HT",   "1T",   "62",   "24",   "ETSI", "5G",
+	"40M",  "HT",   "1T",   "62",   "30",   "MKK",  "5G",   "40M",  "HT",
+	"1T",   "62",   "30",   "FCC",  "5G",   "40M",  "HT",   "1T",   "102",
+	"24",   "ETSI", "5G",   "40M",  "HT",   "1T",   "102",  "30",   "MKK",
+	"5G",   "40M",  "HT",   "1T",   "102",  "30",   "FCC",  "5G",   "40M",
+	"HT",   "1T",   "110",  "30",   "ETSI", "5G",   "40M",  "HT",   "1T",
+	"110",  "30",   "MKK",  "5G",   "40M",  "HT",   "1T",   "110",  "30",
+	"FCC",  "5G",   "40M",  "HT",   "1T",   "118",  "30",   "ETSI", "5G",
+	"40M",  "HT",   "1T",   "118",  "30",   "MKK",  "5G",   "40M",  "HT",
+	"1T",   "118",  "30",   "FCC",  "5G",   "40M",  "HT",   "1T",   "126",
+	"30",   "ETSI", "5G",   "40M",  "HT",   "1T",   "126",  "30",   "MKK",
+	"5G",   "40M",  "HT",   "1T",   "126",  "30",   "FCC",  "5G",   "40M",
+	"HT",   "1T",   "134",  "30",   "ETSI", "5G",   "40M",  "HT",   "1T",
+	"134",  "30",   "MKK",  "5G",   "40M",  "HT",   "1T",   "134",  "30",
+	"FCC",  "5G",   "40M",  "HT",   "1T",   "142",  "30",   "ETSI", "5G",
+	"40M",  "HT",   "1T",   "142",  "63",   "MKK",  "5G",   "40M",  "HT",
+	"1T",   "142",  "63",   "FCC",  "5G",   "40M",  "HT",   "1T",   "151",
+	"30",   "ETSI", "5G",   "40M",  "HT",   "1T",   "151",  "63",   "MKK",
+	"5G",   "40M",  "HT",   "1T",   "151",  "63",   "FCC",  "5G",   "40M",
+	"HT",   "1T",   "159",  "30",   "ETSI", "5G",   "40M",  "HT",   "1T",
+	"159",  "63",   "MKK",  "5G",   "40M",  "HT",   "1T",   "159",  "63",
+	"FCC",  "5G",   "40M",  "HT",   "2T",   "38",   "20",   "ETSI", "5G",
+	"40M",  "HT",   "2T",   "38",   "20",   "MKK",  "5G",   "40M",  "HT",
+	"2T",   "38",   "22",   "FCC",  "5G",   "40M",  "HT",   "2T",   "46",
+	"30",   "ETSI", "5G",   "40M",  "HT",   "2T",   "46",   "20",   "MKK",
+	"5G",   "40M",  "HT",   "2T",   "46",   "22",   "FCC",  "5G",   "40M",
+	"HT",   "2T",   "54",   "30",   "ETSI", "5G",   "40M",  "HT",   "2T",
+	"54",   "20",   "MKK",  "5G",   "40M",  "HT",   "2T",   "54",   "22",
+	"FCC",  "5G",   "40M",  "HT",   "2T",   "62",   "22",   "ETSI", "5G",
+	"40M",  "HT",   "2T",   "62",   "20",   "MKK",  "5G",   "40M",  "HT",
+	"2T",   "62",   "22",   "FCC",  "5G",   "40M",  "HT",   "2T",   "102",
+	"22",   "ETSI", "5G",   "40M",  "HT",   "2T",   "102",  "20",   "MKK",
+	"5G",   "40M",  "HT",   "2T",   "102",  "30",   "FCC",  "5G",   "40M",
+	"HT",   "2T",   "110",  "30",   "ETSI", "5G",   "40M",  "HT",   "2T",
+	"110",  "20",   "MKK",  "5G",   "40M",  "HT",   "2T",   "110",  "30",
+	"FCC",  "5G",   "40M",  "HT",   "2T",   "118",  "30",   "ETSI", "5G",
+	"40M",  "HT",   "2T",   "118",  "20",   "MKK",  "5G",   "40M",  "HT",
+	"2T",   "118",  "30",   "FCC",  "5G",   "40M",  "HT",   "2T",   "126",
+	"30",   "ETSI", "5G",   "40M",  "HT",   "2T",   "126",  "20",   "MKK",
+	"5G",   "40M",  "HT",   "2T",   "126",  "30",   "FCC",  "5G",   "40M",
+	"HT",   "2T",   "134",  "30",   "ETSI", "5G",   "40M",  "HT",   "2T",
+	"134",  "20",   "MKK",  "5G",   "40M",  "HT",   "2T",   "134",  "30",
+	"FCC",  "5G",   "40M",  "HT",   "2T",   "142",  "30",   "ETSI", "5G",
+	"40M",  "HT",   "2T",   "142",  "63",   "MKK",  "5G",   "40M",  "HT",
+	"2T",   "142",  "63",   "FCC",  "5G",   "40M",  "HT",   "2T",   "151",
+	"30",   "ETSI", "5G",   "40M",  "HT",   "2T",   "151",  "63",   "MKK",
+	"5G",   "40M",  "HT",   "2T",   "151",  "63",   "FCC",  "5G",   "40M",
+	"HT",   "2T",   "159",  "30",   "ETSI", "5G",   "40M",  "HT",   "2T",
+	"159",  "63",   "MKK",  "5G",   "40M",  "HT",   "2T",   "159",  "63",
+	"FCC",  "5G",   "80M",  "VHT",  "1T",   "42",   "20",   "ETSI", "5G",
+	"80M",  "VHT",  "1T",   "42",   "30",   "MKK",  "5G",   "80M",  "VHT",
+	"1T",   "42",   "28",   "FCC",  "5G",   "80M",  "VHT",  "1T",   "58",
+	"20",   "ETSI", "5G",   "80M",  "VHT",  "1T",   "58",   "30",   "MKK",
+	"5G",   "80M",  "VHT",  "1T",   "58",   "28",   "FCC",  "5G",   "80M",
+	"VHT",  "1T",   "106",  "20",   "ETSI", "5G",   "80M",  "VHT",  "1T",
+	"106",  "30",   "MKK",  "5G",   "80M",  "VHT",  "1T",   "106",  "30",
+	"FCC",  "5G",   "80M",  "VHT",  "1T",   "122",  "30",   "ETSI", "5G",
+	"80M",  "VHT",  "1T",   "122",  "30",   "MKK",  "5G",   "80M",  "VHT",
+	"1T",   "122",  "30",   "FCC",  "5G",   "80M",  "VHT",  "1T",   "138",
+	"30",   "ETSI", "5G",   "80M",  "VHT",  "1T",   "138",  "63",   "MKK",
+	"5G",   "80M",  "VHT",  "1T",   "138",  "63",   "FCC",  "5G",   "80M",
+	"VHT",  "1T",   "155",  "30",   "ETSI", "5G",   "80M",  "VHT",  "1T",
+	"155",  "63",   "MKK",  "5G",   "80M",  "VHT",  "1T",   "155",  "63",
+	"FCC",  "5G",   "80M",  "VHT",  "2T",   "42",   "18",   "ETSI", "5G",
+	"80M",  "VHT",  "2T",   "42",   "20",   "MKK",  "5G",   "80M",  "VHT",
+	"2T",   "42",   "22",   "FCC",  "5G",   "80M",  "VHT",  "2T",   "58",
+	"18",   "ETSI", "5G",   "80M",  "VHT",  "2T",   "58",   "20",   "MKK",
+	"5G",   "80M",  "VHT",  "2T",   "58",   "22",   "FCC",  "5G",   "80M",
+	"VHT",  "2T",   "106",  "20",   "ETSI", "5G",   "80M",  "VHT",  "2T",
+	"106",  "20",   "MKK",  "5G",   "80M",  "VHT",  "2T",   "106",  "30",
+	"FCC",  "5G",   "80M",  "VHT",  "2T",   "122",  "30",   "ETSI", "5G",
+	"80M",  "VHT",  "2T",   "122",  "20",   "MKK",  "5G",   "80M",  "VHT",
+	"2T",   "122",  "30",   "FCC",  "5G",   "80M",  "VHT",  "2T",   "138",
+	"30",   "ETSI", "5G",   "80M",  "VHT",  "2T",   "138",  "63",   "MKK",
+	"5G",   "80M",  "VHT",  "2T",   "138",  "63",   "FCC",  "5G",   "80M",
+	"VHT",  "2T",   "155",  "30",   "ETSI", "5G",   "80M",  "VHT",  "2T",
+	"155",  "63",   "MKK",  "5G",   "80M",  "VHT",  "2T",   "155",  "63"};
+
+void odm_read_and_config_mp_8822b_txpwr_lmt_type5(struct phy_dm_struct *dm)
+{
+	u32 i = 0;
+	u32 array_len = sizeof(array_mp_8822b_txpwr_lmt_type5) / sizeof(u8 *);
+	u8 **array = (u8 **)array_mp_8822b_txpwr_lmt_type5;
+
+	ODM_RT_TRACE(dm, ODM_COMP_INIT,
+		     "===> odm_read_and_config_mp_8822b_txpwr_lmt_type5\n");
+
+	for (i = 0; i < array_len; i += 7) {
+		u8 *regulation = array[i];
+		u8 *band = array[i + 1];
+		u8 *bandwidth = array[i + 2];
+		u8 *rate = array[i + 3];
+		u8 *rf_path = array[i + 4];
+		u8 *chnl = array[i + 5];
+		u8 *val = array[i + 6];
+
+		odm_config_bb_txpwr_lmt_8822b(dm, regulation, band, bandwidth,
+					      rate, rf_path, chnl, val);
+	}
+}
diff --git a/drivers/staging/rtlwifi/phydm/rtl8822b/halhwimg8822b_rf.h b/drivers/staging/rtlwifi/phydm/rtl8822b/halhwimg8822b_rf.h
new file mode 100644
index 000000000000..1340fa9f369b
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/rtl8822b/halhwimg8822b_rf.h
@@ -0,0 +1,129 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+/*Image2HeaderVersion: 3.2*/
+#ifndef __INC_MP_RF_HW_IMG_8822B_H
+#define __INC_MP_RF_HW_IMG_8822B_H
+
+/******************************************************************************
+ *                           radioa.TXT
+ ******************************************************************************/
+
+void odm_read_and_config_mp_8822b_radioa(struct phy_dm_struct *dm);
+u32 odm_get_version_mp_8822b_radioa(void);
+
+/******************************************************************************
+ *                           radiob.TXT
+ ******************************************************************************/
+
+void odm_read_and_config_mp_8822b_radiob(struct phy_dm_struct *dm);
+u32 odm_get_version_mp_8822b_radiob(void);
+
+/******************************************************************************
+ *                           txpowertrack.TXT
+ ******************************************************************************/
+
+void odm_read_and_config_mp_8822b_txpowertrack(struct phy_dm_struct *dm);
+u32 odm_get_version_mp_8822b_txpowertrack(void);
+
+/******************************************************************************
+ *                           txpowertrack_type0.TXT
+ ******************************************************************************/
+
+void odm_read_and_config_mp_8822b_txpowertrack_type0(struct phy_dm_struct *dm);
+u32 odm_get_version_mp_8822b_txpowertrack_type0(void);
+
+/******************************************************************************
+ *                           txpowertrack_type1.TXT
+ ******************************************************************************/
+
+void odm_read_and_config_mp_8822b_txpowertrack_type1(struct phy_dm_struct *dm);
+u32 odm_get_version_mp_8822b_txpowertrack_type1(void);
+
+/******************************************************************************
+ *                           txpowertrack_type2.TXT
+ ******************************************************************************/
+
+void odm_read_and_config_mp_8822b_txpowertrack_type2(struct phy_dm_struct *dm);
+u32 odm_get_version_mp_8822b_txpowertrack_type2(void);
+
+/******************************************************************************
+ *                           txpowertrack_type3_type5.TXT
+ ******************************************************************************/
+
+void odm_read_and_config_mp_8822b_txpowertrack_type3_type5(
+	struct phy_dm_struct *dm);
+u32 odm_get_version_mp_8822b_txpowertrack_type3_type5(void);
+
+/******************************************************************************
+ *                           txpowertrack_type4.TXT
+ ******************************************************************************/
+
+void odm_read_and_config_mp_8822b_txpowertrack_type4(struct phy_dm_struct *dm);
+u32 odm_get_version_mp_8822b_txpowertrack_type4(void);
+
+/******************************************************************************
+ *                           txpowertrack_type6.TXT
+ ******************************************************************************/
+
+void odm_read_and_config_mp_8822b_txpowertrack_type6(struct phy_dm_struct *dm);
+u32 odm_get_version_mp_8822b_txpowertrack_type6(void);
+
+/******************************************************************************
+ *                           txpowertrack_type7.TXT
+ ******************************************************************************/
+
+void odm_read_and_config_mp_8822b_txpowertrack_type7(struct phy_dm_struct *dm);
+u32 odm_get_version_mp_8822b_txpowertrack_type7(void);
+
+/******************************************************************************
+ *                           txpowertrack_type8.TXT
+ *****************************************************************************/
+
+void odm_read_and_config_mp_8822b_txpowertrack_type8(struct phy_dm_struct *dm);
+u32 odm_get_version_mp_8822b_txpowertrack_type8(void);
+
+/******************************************************************************
+ *                           txpowertrack_type9.TXT
+ ******************************************************************************/
+
+void odm_read_and_config_mp_8822b_txpowertrack_type9(struct phy_dm_struct *dm);
+u32 odm_get_version_mp_8822b_txpowertrack_type9(void);
+
+/******************************************************************************
+ *                           txpwr_lmt.TXT
+ ******************************************************************************/
+
+void odm_read_and_config_mp_8822b_txpwr_lmt(struct phy_dm_struct *dm);
+u32 odm_get_version_mp_8822b_txpwr_lmt(void);
+
+/******************************************************************************
+ *                           txpwr_lmt_type5.TXT
+ ******************************************************************************/
+
+void odm_read_and_config_mp_8822b_txpwr_lmt_type5(struct phy_dm_struct *dm);
+u32 odm_get_version_mp_8822b_txpwr_lmt_type5(void);
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/rtl8822b/halphyrf_8822b.c b/drivers/staging/rtlwifi/phydm/rtl8822b/halphyrf_8822b.c
new file mode 100644
index 000000000000..ae3e2278fefd
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/rtl8822b/halphyrf_8822b.c
@@ -0,0 +1,351 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#include "../mp_precomp.h"
+#include "../phydm_precomp.h"
+
+static bool
+get_mix_mode_tx_agc_bb_swing_offset_8822b(void *dm_void,
+					  enum pwrtrack_method method,
+					  u8 rf_path, u8 tx_power_index_offest)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct dm_rf_calibration_struct *cali_info = &dm->rf_calibrate_info;
+
+	u8 bb_swing_upper_bound = cali_info->default_ofdm_index + 10;
+	u8 bb_swing_lower_bound = 0;
+
+	s8 tx_agc_index = 0;
+	u8 tx_bb_swing_index = cali_info->default_ofdm_index;
+
+	ODM_RT_TRACE(
+		dm, ODM_COMP_TX_PWR_TRACK,
+		"Path_%d cali_info->absolute_ofdm_swing_idx[rf_path]=%d, tx_power_index_offest=%d\n",
+		rf_path, cali_info->absolute_ofdm_swing_idx[rf_path],
+		tx_power_index_offest);
+
+	if (tx_power_index_offest > 0XF)
+		tx_power_index_offest = 0XF;
+
+	if (cali_info->absolute_ofdm_swing_idx[rf_path] >= 0 &&
+	    cali_info->absolute_ofdm_swing_idx[rf_path] <=
+		    tx_power_index_offest) {
+		tx_agc_index = cali_info->absolute_ofdm_swing_idx[rf_path];
+		tx_bb_swing_index = cali_info->default_ofdm_index;
+	} else if (cali_info->absolute_ofdm_swing_idx[rf_path] >
+		   tx_power_index_offest) {
+		tx_agc_index = tx_power_index_offest;
+		cali_info->remnant_ofdm_swing_idx[rf_path] =
+			cali_info->absolute_ofdm_swing_idx[rf_path] -
+			tx_power_index_offest;
+		tx_bb_swing_index = cali_info->default_ofdm_index +
+				    cali_info->remnant_ofdm_swing_idx[rf_path];
+
+		if (tx_bb_swing_index > bb_swing_upper_bound)
+			tx_bb_swing_index = bb_swing_upper_bound;
+	} else {
+		tx_agc_index = 0;
+
+		if (cali_info->default_ofdm_index >
+		    (cali_info->absolute_ofdm_swing_idx[rf_path] * (-1)))
+			tx_bb_swing_index =
+				cali_info->default_ofdm_index +
+				cali_info->absolute_ofdm_swing_idx[rf_path];
+		else
+			tx_bb_swing_index = bb_swing_lower_bound;
+
+		if (tx_bb_swing_index < bb_swing_lower_bound)
+			tx_bb_swing_index = bb_swing_lower_bound;
+	}
+
+	cali_info->absolute_ofdm_swing_idx[rf_path] = tx_agc_index;
+	cali_info->bb_swing_idx_ofdm[rf_path] = tx_bb_swing_index;
+
+	ODM_RT_TRACE(
+		dm, ODM_COMP_TX_PWR_TRACK,
+		"MixMode Offset Path_%d   cali_info->absolute_ofdm_swing_idx[rf_path]=%d   cali_info->bb_swing_idx_ofdm[rf_path]=%d   tx_power_index_offest=%d\n",
+		rf_path, cali_info->absolute_ofdm_swing_idx[rf_path],
+		cali_info->bb_swing_idx_ofdm[rf_path], tx_power_index_offest);
+
+	return true;
+}
+
+void odm_tx_pwr_track_set_pwr8822b(void *dm_void, enum pwrtrack_method method,
+				   u8 rf_path, u8 channel_mapped_index)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct dm_rf_calibration_struct *cali_info = &dm->rf_calibrate_info;
+	u8 tx_power_index_offest = 0;
+	u8 tx_power_index = 0;
+
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)dm->adapter;
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	u8 channel = rtlphy->current_channel;
+	u8 band_width = rtlphy->current_chan_bw;
+	u8 tx_rate = 0xFF;
+
+	if (!dm->mp_mode) {
+		u16 rate = *dm->forced_data_rate;
+
+		if (!rate) /*auto rate*/
+			tx_rate = dm->tx_rate;
+		else /*force rate*/
+			tx_rate = (u8)rate;
+	}
+
+	ODM_RT_TRACE(dm, ODM_COMP_TX_PWR_TRACK, "Call:%s tx_rate=0x%X\n",
+		     __func__, tx_rate);
+
+	ODM_RT_TRACE(dm, ODM_COMP_TX_PWR_TRACK,
+		     "pRF->default_ofdm_index=%d   pRF->default_cck_index=%d\n",
+		     cali_info->default_ofdm_index,
+		     cali_info->default_cck_index);
+
+	ODM_RT_TRACE(
+		dm, ODM_COMP_TX_PWR_TRACK,
+		"pRF->absolute_ofdm_swing_idx=%d   pRF->remnant_ofdm_swing_idx=%d   pRF->absolute_cck_swing_idx=%d   pRF->remnant_cck_swing_idx=%d   rf_path=%d\n",
+		cali_info->absolute_ofdm_swing_idx[rf_path],
+		cali_info->remnant_ofdm_swing_idx[rf_path],
+		cali_info->absolute_cck_swing_idx[rf_path],
+		cali_info->remnant_cck_swing_idx, rf_path);
+
+	if (dm->number_linked_client != 0)
+		tx_power_index = odm_get_tx_power_index(
+			dm, (enum odm_rf_radio_path)rf_path, tx_rate,
+			band_width, channel);
+
+	if (tx_power_index >= 63)
+		tx_power_index = 63;
+
+	tx_power_index_offest = 63 - tx_power_index;
+
+	ODM_RT_TRACE(dm, ODM_COMP_TX_PWR_TRACK,
+		     "tx_power_index=%d tx_power_index_offest=%d rf_path=%d\n",
+		     tx_power_index, tx_power_index_offest, rf_path);
+
+	if (method ==
+	    BBSWING) { /*use for mp driver clean power tracking status*/
+		switch (rf_path) {
+		case ODM_RF_PATH_A:
+			odm_set_bb_reg(
+				dm, 0xC94, (BIT(29) | BIT(28) | BIT(27) |
+					    BIT(26) | BIT(25)),
+				cali_info->absolute_ofdm_swing_idx[rf_path]);
+			odm_set_bb_reg(
+				dm, REG_A_TX_SCALE_JAGUAR, 0xFFE00000,
+				tx_scaling_table_jaguar
+					[cali_info
+						 ->bb_swing_idx_ofdm[rf_path]]);
+			break;
+		case ODM_RF_PATH_B:
+			odm_set_bb_reg(
+				dm, 0xE94, (BIT(29) | BIT(28) | BIT(27) |
+					    BIT(26) | BIT(25)),
+				cali_info->absolute_ofdm_swing_idx[rf_path]);
+			odm_set_bb_reg(
+				dm, REG_B_TX_SCALE_JAGUAR, 0xFFE00000,
+				tx_scaling_table_jaguar
+					[cali_info
+						 ->bb_swing_idx_ofdm[rf_path]]);
+			break;
+
+		default:
+			break;
+		}
+	} else if (method == MIX_MODE) {
+		switch (rf_path) {
+		case ODM_RF_PATH_A:
+			get_mix_mode_tx_agc_bb_swing_offset_8822b(
+				dm, method, rf_path, tx_power_index_offest);
+			odm_set_bb_reg(
+				dm, 0xC94, (BIT(29) | BIT(28) | BIT(27) |
+					    BIT(26) | BIT(25)),
+				cali_info->absolute_ofdm_swing_idx[rf_path]);
+			odm_set_bb_reg(
+				dm, REG_A_TX_SCALE_JAGUAR, 0xFFE00000,
+				tx_scaling_table_jaguar
+					[cali_info
+						 ->bb_swing_idx_ofdm[rf_path]]);
+
+			ODM_RT_TRACE(
+				dm, ODM_COMP_TX_PWR_TRACK,
+				"TXAGC(0xC94)=0x%x BBSwing(0xc1c)=0x%x BBSwingIndex=%d rf_path=%d\n",
+				odm_get_bb_reg(dm, 0xC94,
+					       (BIT(29) | BIT(28) | BIT(27) |
+						BIT(26) | BIT(25))),
+				odm_get_bb_reg(dm, 0xc1c, 0xFFE00000),
+				cali_info->bb_swing_idx_ofdm[rf_path], rf_path);
+			break;
+
+		case ODM_RF_PATH_B:
+			get_mix_mode_tx_agc_bb_swing_offset_8822b(
+				dm, method, rf_path, tx_power_index_offest);
+			odm_set_bb_reg(
+				dm, 0xE94, (BIT(29) | BIT(28) | BIT(27) |
+					    BIT(26) | BIT(25)),
+				cali_info->absolute_ofdm_swing_idx[rf_path]);
+			odm_set_bb_reg(
+				dm, REG_B_TX_SCALE_JAGUAR, 0xFFE00000,
+				tx_scaling_table_jaguar
+					[cali_info
+						 ->bb_swing_idx_ofdm[rf_path]]);
+
+			ODM_RT_TRACE(
+				dm, ODM_COMP_TX_PWR_TRACK,
+				"TXAGC(0xE94)=0x%x BBSwing(0xe1c)=0x%x BBSwingIndex=%d rf_path=%d\n",
+				odm_get_bb_reg(dm, 0xE94,
+					       (BIT(29) | BIT(28) | BIT(27) |
+						BIT(26) | BIT(25))),
+				odm_get_bb_reg(dm, 0xe1c, 0xFFE00000),
+				cali_info->bb_swing_idx_ofdm[rf_path], rf_path);
+			break;
+
+		default:
+			break;
+		}
+	}
+}
+
+void get_delta_swing_table_8822b(void *dm_void, u8 **temperature_up_a,
+				 u8 **temperature_down_a, u8 **temperature_up_b,
+				 u8 **temperature_down_b)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct dm_rf_calibration_struct *cali_info = &dm->rf_calibrate_info;
+
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)dm->adapter;
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	u8 channel = rtlphy->current_channel;
+
+	*temperature_up_a = cali_info->delta_swing_table_idx_2ga_p;
+	*temperature_down_a = cali_info->delta_swing_table_idx_2ga_n;
+	*temperature_up_b = cali_info->delta_swing_table_idx_2gb_p;
+	*temperature_down_b = cali_info->delta_swing_table_idx_2gb_n;
+
+	if (channel >= 36 && channel <= 64) {
+		*temperature_up_a = cali_info->delta_swing_table_idx_5ga_p[0];
+		*temperature_down_a = cali_info->delta_swing_table_idx_5ga_n[0];
+		*temperature_up_b = cali_info->delta_swing_table_idx_5gb_p[0];
+		*temperature_down_b = cali_info->delta_swing_table_idx_5gb_n[0];
+	} else if (channel >= 100 && channel <= 144) {
+		*temperature_up_a = cali_info->delta_swing_table_idx_5ga_p[1];
+		*temperature_down_a = cali_info->delta_swing_table_idx_5ga_n[1];
+		*temperature_up_b = cali_info->delta_swing_table_idx_5gb_p[1];
+		*temperature_down_b = cali_info->delta_swing_table_idx_5gb_n[1];
+	} else if (channel >= 149 && channel <= 177) {
+		*temperature_up_a = cali_info->delta_swing_table_idx_5ga_p[2];
+		*temperature_down_a = cali_info->delta_swing_table_idx_5ga_n[2];
+		*temperature_up_b = cali_info->delta_swing_table_idx_5gb_p[2];
+		*temperature_down_b = cali_info->delta_swing_table_idx_5gb_n[2];
+	}
+}
+
+static void _phy_lc_calibrate_8822b(struct phy_dm_struct *dm)
+{
+	u32 lc_cal = 0, cnt = 0;
+
+	/*backup RF0x18*/
+	lc_cal = odm_get_rf_reg(dm, ODM_RF_PATH_A, RF_CHNLBW, RFREGOFFSETMASK);
+
+	/*Start LCK*/
+	odm_set_rf_reg(dm, ODM_RF_PATH_A, RF_CHNLBW, RFREGOFFSETMASK,
+		       lc_cal | 0x08000);
+
+	ODM_delay_ms(100);
+
+	for (cnt = 0; cnt < 100; cnt++) {
+		if (odm_get_rf_reg(dm, ODM_RF_PATH_A, RF_CHNLBW, 0x8000) != 0x1)
+			break;
+		ODM_delay_ms(10);
+	}
+
+	/*Recover channel number*/
+	odm_set_rf_reg(dm, ODM_RF_PATH_A, RF_CHNLBW, RFREGOFFSETMASK, lc_cal);
+}
+
+void phy_lc_calibrate_8822b(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	bool is_start_cont_tx = false, is_single_tone = false,
+	     is_carrier_suppression = false;
+	u64 start_time;
+	u64 progressing_time;
+
+	if (is_start_cont_tx || is_single_tone || is_carrier_suppression) {
+		ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+			     "[LCK]continues TX ing !!! LCK return\n");
+		return;
+	}
+
+	start_time = odm_get_current_time(dm);
+	_phy_lc_calibrate_8822b(dm);
+	progressing_time = odm_get_progressing_time(dm, start_time);
+	ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+		     "[LCK]LCK progressing_time = %lld\n", progressing_time);
+}
+
+void configure_txpower_track_8822b(struct txpwrtrack_cfg *config)
+{
+	config->swing_table_size_cck = TXSCALE_TABLE_SIZE;
+	config->swing_table_size_ofdm = TXSCALE_TABLE_SIZE;
+	config->threshold_iqk = IQK_THRESHOLD;
+	config->threshold_dpk = DPK_THRESHOLD;
+	config->average_thermal_num = AVG_THERMAL_NUM_8822B;
+	config->rf_path_count = MAX_PATH_NUM_8822B;
+	config->thermal_reg_addr = RF_T_METER_8822B;
+
+	config->odm_tx_pwr_track_set_pwr = odm_tx_pwr_track_set_pwr8822b;
+	config->do_iqk = do_iqk_8822b;
+	config->phy_lc_calibrate = phy_lc_calibrate_8822b;
+
+	config->get_delta_swing_table = get_delta_swing_table_8822b;
+}
+
+void phy_set_rf_path_switch_8822b(struct phy_dm_struct *dm, bool is_main)
+{
+	/*BY SY Request */
+	odm_set_bb_reg(dm, 0x4C, (BIT(24) | BIT(23)), 0x2);
+	odm_set_bb_reg(dm, 0x974, 0xff, 0xff);
+
+	/*odm_set_bb_reg(dm, 0x1991, 0x3, 0x0);*/
+	odm_set_bb_reg(dm, 0x1990, (BIT(9) | BIT(8)), 0x0);
+
+	/*odm_set_bb_reg(dm, 0xCBE, 0x8, 0x0);*/
+	odm_set_bb_reg(dm, 0xCBC, BIT(19), 0x0);
+
+	odm_set_bb_reg(dm, 0xCB4, 0xff, 0x77);
+
+	odm_set_bb_reg(dm, 0x70, MASKBYTE3, 0x0e);
+	odm_set_bb_reg(dm, 0x1704, MASKDWORD, 0x0000ff00);
+	odm_set_bb_reg(dm, 0x1700, MASKDWORD, 0xc00f0038);
+
+	if (is_main) {
+		/*odm_set_bb_reg(dm, 0xCBD, 0x3, 0x2);		WiFi */
+		odm_set_bb_reg(dm, 0xCBC, (BIT(9) | BIT(8)), 0x2); /*WiFi */
+	} else {
+		/*odm_set_bb_reg(dm, 0xCBD, 0x3, 0x1);	 BT*/
+		odm_set_bb_reg(dm, 0xCBC, (BIT(9) | BIT(8)), 0x1); /*BT*/
+	}
+}
diff --git a/drivers/staging/rtlwifi/phydm/rtl8822b/halphyrf_8822b.h b/drivers/staging/rtlwifi/phydm/rtl8822b/halphyrf_8822b.h
new file mode 100644
index 000000000000..4f3bfe316ee9
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/rtl8822b/halphyrf_8822b.h
@@ -0,0 +1,45 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __HAL_PHY_RF_8822B_H__
+#define __HAL_PHY_RF_8822B_H__
+
+#define AVG_THERMAL_NUM_8822B 4
+#define RF_T_METER_8822B 0x42
+
+void configure_txpower_track_8822b(struct txpwrtrack_cfg *config);
+
+void odm_tx_pwr_track_set_pwr8822b(void *dm_void, enum pwrtrack_method method,
+				   u8 rf_path, u8 channel_mapped_index);
+
+void get_delta_swing_table_8822b(void *dm_void, u8 **temperature_up_a,
+				 u8 **temperature_down_a, u8 **temperature_up_b,
+				 u8 **temperature_down_b);
+
+void phy_lc_calibrate_8822b(void *dm_void);
+
+void phy_set_rf_path_switch_8822b(struct phy_dm_struct *dm, bool is_main);
+
+#endif /* #ifndef __HAL_PHY_RF_8822B_H__ */
diff --git a/drivers/staging/rtlwifi/phydm/rtl8822b/phydm_hal_api8822b.c b/drivers/staging/rtlwifi/phydm/rtl8822b/phydm_hal_api8822b.c
new file mode 100644
index 000000000000..26d1022e851c
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/rtl8822b/phydm_hal_api8822b.c
@@ -0,0 +1,1815 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#include "../mp_precomp.h"
+#include "../phydm_precomp.h"
+
+/* ======================================================================== */
+/* These following functions can be used for PHY DM only*/
+
+static u32 reg82c_8822b;
+static u32 reg838_8822b;
+static u32 reg830_8822b;
+static u32 reg83c_8822b;
+static u32 rega20_8822b;
+static u32 rega24_8822b;
+static u32 rega28_8822b;
+static enum odm_bw bw_8822b;
+static u8 central_ch_8822b;
+
+static u32 cca_ifem_ccut[12][4] = {
+	/*20M*/
+	{0x75D97010, 0x75D97010, 0x75D97010, 0x75D97010}, /*Reg82C*/
+	{0x00000000, 0x79a0ea2c, 0x00000000, 0x00000000}, /*Reg830*/
+	{0x00000000, 0x00000000, 0x00000000, 0x00000000}, /*Reg838*/
+	{0x00000000, 0x00000000, 0x00000000, 0x00000000}, /*Reg83C*/
+	/*40M*/
+	{0x75D97010, 0x75D97010, 0x75D97010, 0x75D97010}, /*Reg82C*/
+	{0x00000000, 0x79a0ea2c, 0x00000000, 0x79a0ea28}, /*Reg830*/
+	{0x87765541, 0x87766341, 0x87765541, 0x87766341}, /*Reg838*/
+	{0x00000000, 0x00000000, 0x00000000, 0x00000000}, /*Reg83C*/
+	/*80M*/
+	{0x75C97010, 0x75C97010, 0x75C97010, 0x75C97010}, /*Reg82C*/
+	{0x00000000, 0x00000000, 0x00000000, 0x00000000}, /*Reg830*/
+	{0x00000000, 0x87746641, 0x00000000, 0x87746641}, /*Reg838*/
+	{0x00000000, 0x00000000, 0x00000000, 0x00000000},
+}; /*Reg83C*/
+static u32 cca_efem_ccut[12][4] = {
+	/*20M*/
+	{0x75A76010, 0x75A76010, 0x75A76010, 0x75A75010}, /*Reg82C*/
+	{0x00000000, 0x79a0ea2c, 0x00000000, 0x00000000}, /*Reg830*/
+	{0x87766651, 0x87766431, 0x87766451, 0x87766431}, /*Reg838*/
+	{0x9194b2b9, 0x9194b2b9, 0x9194b2b9, 0x9194b2b9}, /*Reg83C*/
+	/*40M*/
+	{0x75A85010, 0x75A75010, 0x75A85010, 0x75A75010}, /*Reg82C*/
+	{0x00000000, 0x79a0ea2c, 0x00000000, 0x00000000}, /*Reg830*/
+	{0x87766431, 0x87766431, 0x87766431, 0x87766431}, /*Reg838*/
+	{0x00000000, 0x00000000, 0x00000000, 0x00000000}, /*Reg83C*/
+	/*80M*/
+	{0x76BA7010, 0x75BA7010, 0x76BA7010, 0x75BA7010}, /*Reg82C*/
+	{0x79a0ea28, 0x00000000, 0x79a0ea28, 0x00000000}, /*Reg830*/
+	{0x87766431, 0x87766431, 0x87766431, 0x87766431}, /*Reg838*/
+	{0x00000000, 0x00000000, 0x00000000, 0x00000000},
+}; /*Reg83C*/
+static u32 cca_ifem_ccut_rfetype5[12][4] = {
+	/*20M*/
+	{0x75D97010, 0x75D97010, 0x75D97010, 0x75D97010}, /*Reg82C*/
+	{0x00000000, 0x79a0ea2c, 0x00000000, 0x00000000}, /*Reg830*/
+	{0x00000000, 0x00000000, 0x87766461, 0x87766461}, /*Reg838*/
+	{0x00000000, 0x00000000, 0x00000000, 0x00000000}, /*Reg83C*/
+	/*40M*/
+	{0x75D97010, 0x75D97010, 0x75D97010, 0x75D97010}, /*Reg82C*/
+	{0x00000000, 0x79a0ea2c, 0x00000000, 0x79a0ea28}, /*Reg830*/
+	{0x87765541, 0x87766341, 0x87765541, 0x87766341}, /*Reg838*/
+	{0x00000000, 0x00000000, 0x00000000, 0x00000000}, /*Reg83C*/
+	/*80M*/
+	{0x75C97010, 0x75C97010, 0x75C97010, 0x75C97010}, /*Reg82C*/
+	{0x00000000, 0x00000000, 0x00000000, 0x00000000}, /*Reg830*/
+	{0x00000000, 0x76666641, 0x00000000, 0x76666641}, /*Reg838*/
+	{0x00000000, 0x00000000, 0x00000000, 0x00000000},
+}; /*Reg83C*/
+static u32 cca_ifem_ccut_rfetype3[12][4] = {
+	/*20M*/
+	{0x75D97010, 0x75D97010, 0x75D97010, 0x75D97010}, /*Reg82C*/
+	{0x00000000, 0x79a0ea2c, 0x00000000, 0x00000000}, /*Reg830*/
+	{0x00000000, 0x00000000, 0x87766461, 0x87766461}, /*Reg838*/
+	{0x00000000, 0x00000000, 0x00000000, 0x00000000}, /*Reg83C*/
+	/*40M*/
+	{0x75D97010, 0x75D97010, 0x75D97010, 0x75D97010}, /*Reg82C*/
+	{0x00000000, 0x79a0ea2c, 0x00000000, 0x79a0ea28}, /*Reg830*/
+	{0x87765541, 0x87766341, 0x87765541, 0x87766341}, /*Reg838*/
+	{0x00000000, 0x00000000, 0x00000000, 0x00000000}, /*Reg83C*/
+	/*80M*/
+	{0x75C97010, 0x75C97010, 0x75C97010, 0x75C97010}, /*Reg82C*/
+	{0x00000000, 0x00000000, 0x00000000, 0x00000000}, /*Reg830*/
+	{0x00000000, 0x76666641, 0x00000000, 0x76666641}, /*Reg838*/
+	{0x00000000, 0x00000000, 0x00000000, 0x00000000},
+}; /*Reg83C*/
+
+static inline u32 phydm_check_bit_mask(u32 bit_mask, u32 data_original,
+				       u32 data)
+{
+	u8 bit_shift;
+
+	if (bit_mask != 0xfffff) {
+		for (bit_shift = 0; bit_shift <= 19; bit_shift++) {
+			if (((bit_mask >> bit_shift) & 0x1) == 1)
+				break;
+		}
+		return ((data_original) & (~bit_mask)) | (data << bit_shift);
+	}
+	return data;
+}
+
+static bool phydm_rfe_8822b(struct phy_dm_struct *dm, u8 channel)
+{
+	if (dm->rfe_type == 4) {
+		/* Default setting is in PHY parameters */
+
+		if (channel <= 14) {
+			/* signal source */
+			odm_set_bb_reg(dm, 0xcb0, (MASKBYTE2 | MASKLWORD),
+				       0x745774);
+			odm_set_bb_reg(dm, 0xeb0, (MASKBYTE2 | MASKLWORD),
+				       0x745774);
+			odm_set_bb_reg(dm, 0xcb4, MASKBYTE1, 0x57);
+			odm_set_bb_reg(dm, 0xeb4, MASKBYTE1, 0x57);
+
+			/* inverse or not */
+			odm_set_bb_reg(dm, 0xcbc, (BIT(5) | BIT(4) | BIT(3) |
+						   BIT(2) | BIT(1) | BIT(0)),
+				       0x8);
+			odm_set_bb_reg(dm, 0xcbc, (BIT(11) | BIT(10)), 0x2);
+			odm_set_bb_reg(dm, 0xebc, (BIT(5) | BIT(4) | BIT(3) |
+						   BIT(2) | BIT(1) | BIT(0)),
+				       0x8);
+			odm_set_bb_reg(dm, 0xebc, (BIT(11) | BIT(10)), 0x2);
+
+			/* antenna switch table */
+			if ((dm->rx_ant_status == (ODM_RF_A | ODM_RF_B)) ||
+			    (dm->tx_ant_status == (ODM_RF_A | ODM_RF_B))) {
+				/* 2TX or 2RX */
+				odm_set_bb_reg(dm, 0xca0, MASKLWORD, 0xf050);
+				odm_set_bb_reg(dm, 0xea0, MASKLWORD, 0xf050);
+			} else if (dm->rx_ant_status == dm->tx_ant_status) {
+				/* TXA+RXA or TXB+RXB */
+				odm_set_bb_reg(dm, 0xca0, MASKLWORD, 0xf055);
+				odm_set_bb_reg(dm, 0xea0, MASKLWORD, 0xf055);
+			} else {
+				/* TXB+RXA or TXA+RXB */
+				odm_set_bb_reg(dm, 0xca0, MASKLWORD, 0xf550);
+				odm_set_bb_reg(dm, 0xea0, MASKLWORD, 0xf550);
+			}
+
+		} else if (channel > 35) {
+			/* signal source */
+			odm_set_bb_reg(dm, 0xcb0, (MASKBYTE2 | MASKLWORD),
+				       0x477547);
+			odm_set_bb_reg(dm, 0xeb0, (MASKBYTE2 | MASKLWORD),
+				       0x477547);
+			odm_set_bb_reg(dm, 0xcb4, MASKBYTE1, 0x75);
+			odm_set_bb_reg(dm, 0xeb4, MASKBYTE1, 0x75);
+
+			/* inverse or not */
+			odm_set_bb_reg(dm, 0xcbc, (BIT(5) | BIT(4) | BIT(3) |
+						   BIT(2) | BIT(1) | BIT(0)),
+				       0x0);
+			odm_set_bb_reg(dm, 0xcbc, (BIT(11) | BIT(10)), 0x0);
+			odm_set_bb_reg(dm, 0xebc, (BIT(5) | BIT(4) | BIT(3) |
+						   BIT(2) | BIT(1) | BIT(0)),
+				       0x0);
+			odm_set_bb_reg(dm, 0xebc, (BIT(11) | BIT(10)), 0x0);
+
+			/* antenna switch table */
+			if ((dm->rx_ant_status == (ODM_RF_A | ODM_RF_B)) ||
+			    (dm->tx_ant_status == (ODM_RF_A | ODM_RF_B))) {
+				/* 2TX or 2RX */
+				odm_set_bb_reg(dm, 0xca0, MASKLWORD, 0xa501);
+				odm_set_bb_reg(dm, 0xea0, MASKLWORD, 0xa501);
+			} else if (dm->rx_ant_status == dm->tx_ant_status) {
+				/* TXA+RXA or TXB+RXB */
+				odm_set_bb_reg(dm, 0xca0, MASKLWORD, 0xa500);
+				odm_set_bb_reg(dm, 0xea0, MASKLWORD, 0xa500);
+			} else {
+				/* TXB+RXA or TXA+RXB */
+				odm_set_bb_reg(dm, 0xca0, MASKLWORD, 0xa005);
+				odm_set_bb_reg(dm, 0xea0, MASKLWORD, 0xa005);
+			}
+		} else {
+			return false;
+		}
+
+	} else if ((dm->rfe_type == 1) || (dm->rfe_type == 2) ||
+		   (dm->rfe_type == 7) || (dm->rfe_type == 9)) {
+		/* eFem */
+		if (((dm->cut_version == ODM_CUT_A) ||
+		     (dm->cut_version == ODM_CUT_B)) &&
+		    (dm->rfe_type < 2)) {
+			if (channel <= 14) {
+				/* signal source */
+				odm_set_bb_reg(dm, 0xcb0,
+					       (MASKBYTE2 | MASKLWORD),
+					       0x704570);
+				odm_set_bb_reg(dm, 0xeb0,
+					       (MASKBYTE2 | MASKLWORD),
+					       0x704570);
+				odm_set_bb_reg(dm, 0xcb4, MASKBYTE1, 0x45);
+				odm_set_bb_reg(dm, 0xeb4, MASKBYTE1, 0x45);
+			} else if (channel > 35) {
+				odm_set_bb_reg(dm, 0xcb0,
+					       (MASKBYTE2 | MASKLWORD),
+					       0x174517);
+				odm_set_bb_reg(dm, 0xeb0,
+					       (MASKBYTE2 | MASKLWORD),
+					       0x174517);
+				odm_set_bb_reg(dm, 0xcb4, MASKBYTE1, 0x45);
+				odm_set_bb_reg(dm, 0xeb4, MASKBYTE1, 0x45);
+			} else {
+				return false;
+			}
+
+			/* delay 400ns for PAPE */
+			odm_set_bb_reg(dm, 0x810,
+				       MASKBYTE3 | BIT(20) | BIT(21) | BIT(22) |
+					       BIT(23),
+				       0x211);
+
+			/* antenna switch table */
+			odm_set_bb_reg(dm, 0xca0, MASKLWORD, 0xa555);
+			odm_set_bb_reg(dm, 0xea0, MASKLWORD, 0xa555);
+
+			/* inverse or not */
+			odm_set_bb_reg(dm, 0xcbc, (BIT(5) | BIT(4) | BIT(3) |
+						   BIT(2) | BIT(1) | BIT(0)),
+				       0x0);
+			odm_set_bb_reg(dm, 0xcbc, (BIT(11) | BIT(10)), 0x0);
+			odm_set_bb_reg(dm, 0xebc, (BIT(5) | BIT(4) | BIT(3) |
+						   BIT(2) | BIT(1) | BIT(0)),
+				       0x0);
+			odm_set_bb_reg(dm, 0xebc, (BIT(11) | BIT(10)), 0x0);
+
+			ODM_RT_TRACE(
+				dm, ODM_PHY_CONFIG,
+				"%s: Using old RFE control pin setting for A-cut and B-cut\n",
+				__func__);
+		} else {
+			if (channel <= 14) {
+				/* signal source */
+				odm_set_bb_reg(dm, 0xcb0,
+					       (MASKBYTE2 | MASKLWORD),
+					       0x705770);
+				odm_set_bb_reg(dm, 0xeb0,
+					       (MASKBYTE2 | MASKLWORD),
+					       0x705770);
+				odm_set_bb_reg(dm, 0xcb4, MASKBYTE1, 0x57);
+				odm_set_bb_reg(dm, 0xeb4, MASKBYTE1, 0x57);
+				odm_set_bb_reg(dm, 0xcb8, BIT(4), 0);
+				odm_set_bb_reg(dm, 0xeb8, BIT(4), 0);
+			} else if (channel > 35) {
+				/* signal source */
+				odm_set_bb_reg(dm, 0xcb0,
+					       (MASKBYTE2 | MASKLWORD),
+					       0x177517);
+				odm_set_bb_reg(dm, 0xeb0,
+					       (MASKBYTE2 | MASKLWORD),
+					       0x177517);
+				odm_set_bb_reg(dm, 0xcb4, MASKBYTE1, 0x75);
+				odm_set_bb_reg(dm, 0xeb4, MASKBYTE1, 0x75);
+				odm_set_bb_reg(dm, 0xcb8, BIT(5), 0);
+				odm_set_bb_reg(dm, 0xeb8, BIT(5), 0);
+			} else {
+				return false;
+			}
+
+			/* inverse or not */
+			odm_set_bb_reg(dm, 0xcbc, (BIT(5) | BIT(4) | BIT(3) |
+						   BIT(2) | BIT(1) | BIT(0)),
+				       0x0);
+			odm_set_bb_reg(dm, 0xcbc, (BIT(11) | BIT(10)), 0x0);
+			odm_set_bb_reg(dm, 0xebc, (BIT(5) | BIT(4) | BIT(3) |
+						   BIT(2) | BIT(1) | BIT(0)),
+				       0x0);
+			odm_set_bb_reg(dm, 0xebc, (BIT(11) | BIT(10)), 0x0);
+
+			/* antenna switch table */
+			if ((dm->rx_ant_status == (ODM_RF_A | ODM_RF_B)) ||
+			    (dm->tx_ant_status == (ODM_RF_A | ODM_RF_B))) {
+				/* 2TX or 2RX */
+				odm_set_bb_reg(dm, 0xca0, MASKLWORD, 0xa501);
+				odm_set_bb_reg(dm, 0xea0, MASKLWORD, 0xa501);
+			} else if (dm->rx_ant_status == dm->tx_ant_status) {
+				/* TXA+RXA or TXB+RXB */
+				odm_set_bb_reg(dm, 0xca0, MASKLWORD, 0xa500);
+				odm_set_bb_reg(dm, 0xea0, MASKLWORD, 0xa500);
+			} else {
+				/* TXB+RXA or TXA+RXB */
+				odm_set_bb_reg(dm, 0xca0, MASKLWORD, 0xa005);
+				odm_set_bb_reg(dm, 0xea0, MASKLWORD, 0xa005);
+			}
+		}
+	} else if ((dm->rfe_type == 0) || (dm->rfe_type == 3) ||
+		   (dm->rfe_type == 5) || (dm->rfe_type == 6) ||
+		   (dm->rfe_type == 8) || (dm->rfe_type == 10)) {
+		/* iFEM */
+		if (channel <= 14) {
+			/* signal source */
+
+			odm_set_bb_reg(dm, 0xcb0, (MASKBYTE2 | MASKLWORD),
+				       0x745774);
+			odm_set_bb_reg(dm, 0xeb0, (MASKBYTE2 | MASKLWORD),
+				       0x745774);
+			odm_set_bb_reg(dm, 0xcb4, MASKBYTE1, 0x57);
+			odm_set_bb_reg(dm, 0xeb4, MASKBYTE1, 0x57);
+
+		} else if (channel > 35) {
+			/* signal source */
+
+			odm_set_bb_reg(dm, 0xcb0, (MASKBYTE2 | MASKLWORD),
+				       0x477547);
+			odm_set_bb_reg(dm, 0xeb0, (MASKBYTE2 | MASKLWORD),
+				       0x477547);
+			odm_set_bb_reg(dm, 0xcb4, MASKBYTE1, 0x75);
+			odm_set_bb_reg(dm, 0xeb4, MASKBYTE1, 0x75);
+
+		} else {
+			return false;
+		}
+
+		/* inverse or not */
+		odm_set_bb_reg(dm, 0xcbc, (BIT(5) | BIT(4) | BIT(3) | BIT(2) |
+					   BIT(1) | BIT(0)),
+			       0x0);
+		odm_set_bb_reg(dm, 0xcbc, (BIT(11) | BIT(10)), 0x0);
+		odm_set_bb_reg(dm, 0xebc, (BIT(5) | BIT(4) | BIT(3) | BIT(2) |
+					   BIT(1) | BIT(0)),
+			       0x0);
+		odm_set_bb_reg(dm, 0xebc, (BIT(11) | BIT(10)), 0x0);
+
+		/* antenna switch table */
+		if (channel <= 14) {
+			if ((dm->rx_ant_status == (ODM_RF_A | ODM_RF_B)) ||
+			    (dm->tx_ant_status == (ODM_RF_A | ODM_RF_B))) {
+				/* 2TX or 2RX */
+				odm_set_bb_reg(dm, 0xca0, MASKLWORD, 0xa501);
+				odm_set_bb_reg(dm, 0xea0, MASKLWORD, 0xa501);
+			} else if (dm->rx_ant_status == dm->tx_ant_status) {
+				/* TXA+RXA or TXB+RXB */
+				odm_set_bb_reg(dm, 0xca0, MASKLWORD, 0xa500);
+				odm_set_bb_reg(dm, 0xea0, MASKLWORD, 0xa500);
+			} else {
+				/* TXB+RXA or TXA+RXB */
+				odm_set_bb_reg(dm, 0xca0, MASKLWORD, 0xa005);
+				odm_set_bb_reg(dm, 0xea0, MASKLWORD, 0xa005);
+			}
+		} else if (channel > 35) {
+			odm_set_bb_reg(dm, 0xca0, MASKLWORD, 0xa5a5);
+			odm_set_bb_reg(dm, 0xea0, MASKLWORD, 0xa5a5);
+		}
+	}
+
+	/* chip top mux */
+	odm_set_bb_reg(dm, 0x64, BIT(29) | BIT(28), 0x3);
+	odm_set_bb_reg(dm, 0x4c, BIT(26) | BIT(25), 0x0);
+	odm_set_bb_reg(dm, 0x40, BIT(2), 0x1);
+
+	/* from s0 or s1 */
+	odm_set_bb_reg(dm, 0x1990,
+		       (BIT(5) | BIT(4) | BIT(3) | BIT(2) | BIT(1) | BIT(0)),
+		       0x30);
+	odm_set_bb_reg(dm, 0x1990, (BIT(11) | BIT(10)), 0x3);
+
+	/* input or output */
+	odm_set_bb_reg(dm, 0x974,
+		       (BIT(5) | BIT(4) | BIT(3) | BIT(2) | BIT(1) | BIT(0)),
+		       0x3f);
+	odm_set_bb_reg(dm, 0x974, (BIT(11) | BIT(10)), 0x3);
+
+	ODM_RT_TRACE(
+		dm, ODM_PHY_CONFIG,
+		"%s: Update RFE control pin setting (ch%d, tx_path 0x%x, rx_path 0x%x)\n",
+		__func__, channel, dm->tx_ant_status, dm->rx_ant_status);
+
+	return true;
+}
+
+static void phydm_ccapar_by_rfe_8822b(struct phy_dm_struct *dm)
+{
+	u32 cca_ifem[12][4], cca_efem[12][4];
+	u8 row, col;
+	u32 reg82c, reg830, reg838, reg83c;
+
+	if (dm->cut_version == ODM_CUT_A)
+		return;
+	{
+		odm_move_memory(dm, cca_efem, cca_efem_ccut, 48 * 4);
+		if (dm->rfe_type == 5)
+			odm_move_memory(dm, cca_ifem, cca_ifem_ccut_rfetype5,
+					48 * 4);
+		else if (dm->rfe_type == 3)
+			odm_move_memory(dm, cca_ifem, cca_ifem_ccut_rfetype3,
+					48 * 4);
+		else
+			odm_move_memory(dm, cca_ifem, cca_ifem_ccut, 48 * 4);
+
+		ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+			     "%s: Update CCA parameters for Ccut\n", __func__);
+	}
+
+	if (bw_8822b == ODM_BW20M)
+		row = 0;
+	else if (bw_8822b == ODM_BW40M)
+		row = 4;
+	else
+		row = 8;
+
+	if (central_ch_8822b <= 14) {
+		if ((dm->rx_ant_status == ODM_RF_A) ||
+		    (dm->rx_ant_status == ODM_RF_B))
+			col = 0;
+		else
+			col = 1;
+	} else {
+		if ((dm->rx_ant_status == ODM_RF_A) ||
+		    (dm->rx_ant_status == ODM_RF_B))
+			col = 2;
+		else
+			col = 3;
+	}
+
+	if ((dm->rfe_type == 1) || (dm->rfe_type == 4) || (dm->rfe_type == 6) ||
+	    (dm->rfe_type == 7)) {
+		/*eFEM => RFE type 1 & RFE type 4 & RFE type 6 & RFE type 7*/
+		reg82c = (cca_efem[row][col] != 0) ? cca_efem[row][col] :
+						     reg82c_8822b;
+		reg830 = (cca_efem[row + 1][col] != 0) ?
+				 cca_efem[row + 1][col] :
+				 reg830_8822b;
+		reg838 = (cca_efem[row + 2][col] != 0) ?
+				 cca_efem[row + 2][col] :
+				 reg838_8822b;
+		reg83c = (cca_efem[row + 3][col] != 0) ?
+				 cca_efem[row + 3][col] :
+				 reg83c_8822b;
+	} else if ((dm->rfe_type == 2) || (dm->rfe_type == 9)) {
+		/*5G eFEM, 2G iFEM => RFE type 2, 5G eFEM => RFE type 9 */
+		if (central_ch_8822b <= 14) {
+			reg82c = (cca_ifem[row][col] != 0) ?
+					 cca_ifem[row][col] :
+					 reg82c_8822b;
+			reg830 = (cca_ifem[row + 1][col] != 0) ?
+					 cca_ifem[row + 1][col] :
+					 reg830_8822b;
+			reg838 = (cca_ifem[row + 2][col] != 0) ?
+					 cca_ifem[row + 2][col] :
+					 reg838_8822b;
+			reg83c = (cca_ifem[row + 3][col] != 0) ?
+					 cca_ifem[row + 3][col] :
+					 reg83c_8822b;
+		} else {
+			reg82c = (cca_efem[row][col] != 0) ?
+					 cca_efem[row][col] :
+					 reg82c_8822b;
+			reg830 = (cca_efem[row + 1][col] != 0) ?
+					 cca_efem[row + 1][col] :
+					 reg830_8822b;
+			reg838 = (cca_efem[row + 2][col] != 0) ?
+					 cca_efem[row + 2][col] :
+					 reg838_8822b;
+			reg83c = (cca_efem[row + 3][col] != 0) ?
+					 cca_efem[row + 3][col] :
+					 reg83c_8822b;
+		}
+	} else {
+		/* iFEM =>RFE type 3 & RFE type 5 & RFE type 0 & RFE type 8 &
+		 * RFE type 10
+		 */
+		reg82c = (cca_ifem[row][col] != 0) ? cca_ifem[row][col] :
+						     reg82c_8822b;
+		reg830 = (cca_ifem[row + 1][col] != 0) ?
+				 cca_ifem[row + 1][col] :
+				 reg830_8822b;
+		reg838 = (cca_ifem[row + 2][col] != 0) ?
+				 cca_ifem[row + 2][col] :
+				 reg838_8822b;
+		reg83c = (cca_ifem[row + 3][col] != 0) ?
+				 cca_ifem[row + 3][col] :
+				 reg83c_8822b;
+	}
+
+	odm_set_bb_reg(dm, 0x82c, MASKDWORD, reg82c);
+	odm_set_bb_reg(dm, 0x830, MASKDWORD, reg830);
+	odm_set_bb_reg(dm, 0x838, MASKDWORD, reg838);
+	odm_set_bb_reg(dm, 0x83c, MASKDWORD, reg83c);
+	ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+		     "%s: (Pkt%d, Intf%d, RFE%d), row = %d, col = %d\n",
+		     __func__, dm->package_type, dm->support_interface,
+		     dm->rfe_type, row, col);
+}
+
+static void phydm_ccapar_by_bw_8822b(struct phy_dm_struct *dm,
+				     enum odm_bw bandwidth)
+{
+	u32 reg82c;
+
+	if (dm->cut_version != ODM_CUT_A)
+		return;
+
+	/* A-cut */
+	reg82c = odm_get_bb_reg(dm, 0x82c, MASKDWORD);
+
+	if (bandwidth == ODM_BW20M) {
+		/* 82c[15:12] = 4 */
+		/* 82c[27:24] = 6 */
+
+		reg82c &= (~(0x0f00f000));
+		reg82c |= ((0x4) << 12);
+		reg82c |= ((0x6) << 24);
+	} else if (bandwidth == ODM_BW40M) {
+		/* 82c[19:16] = 9 */
+		/* 82c[27:24] = 6 */
+
+		reg82c &= (~(0x0f0f0000));
+		reg82c |= ((0x9) << 16);
+		reg82c |= ((0x6) << 24);
+	} else if (bandwidth == ODM_BW80M) {
+		/* 82c[15:12] 7 */
+		/* 82c[19:16] b */
+		/* 82c[23:20] d */
+		/* 82c[27:24] 3 */
+
+		reg82c &= (~(0x0ffff000));
+		reg82c |= ((0xdb7) << 12);
+		reg82c |= ((0x3) << 24);
+	}
+
+	odm_set_bb_reg(dm, 0x82c, MASKDWORD, reg82c);
+	ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+		     "%s(): Update CCA parameters for Acut\n", __func__);
+}
+
+static void phydm_ccapar_by_rxpath_8822b(struct phy_dm_struct *dm)
+{
+	if (dm->cut_version != ODM_CUT_A)
+		return;
+
+	if ((dm->rx_ant_status == ODM_RF_A) ||
+	    (dm->rx_ant_status == ODM_RF_B)) {
+		/* 838[7:4] = 8 */
+		/* 838[11:8] = 7 */
+		/* 838[15:12] = 6 */
+		/* 838[19:16] = 7 */
+		/* 838[23:20] = 7 */
+		/* 838[27:24] = 7 */
+		odm_set_bb_reg(dm, 0x838, 0x0ffffff0, 0x777678);
+	} else {
+		/* 838[7:4] = 3 */
+		/* 838[11:8] = 3 */
+		/* 838[15:12] = 6 */
+		/* 838[19:16] = 6 */
+		/* 838[23:20] = 7 */
+		/* 838[27:24] = 7 */
+		odm_set_bb_reg(dm, 0x838, 0x0ffffff0, 0x776633);
+	}
+	ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+		     "%s(): Update CCA parameters for Acut\n", __func__);
+}
+
+static void phydm_rxdfirpar_by_bw_8822b(struct phy_dm_struct *dm,
+					enum odm_bw bandwidth)
+{
+	if (bandwidth == ODM_BW40M) {
+		/* RX DFIR for BW40 */
+		odm_set_bb_reg(dm, 0x948, BIT(29) | BIT(28), 0x1);
+		odm_set_bb_reg(dm, 0x94c, BIT(29) | BIT(28), 0x0);
+		odm_set_bb_reg(dm, 0xc20, BIT(31), 0x0);
+		odm_set_bb_reg(dm, 0xe20, BIT(31), 0x0);
+	} else if (bandwidth == ODM_BW80M) {
+		/* RX DFIR for BW80 */
+		odm_set_bb_reg(dm, 0x948, BIT(29) | BIT(28), 0x2);
+		odm_set_bb_reg(dm, 0x94c, BIT(29) | BIT(28), 0x1);
+		odm_set_bb_reg(dm, 0xc20, BIT(31), 0x0);
+		odm_set_bb_reg(dm, 0xe20, BIT(31), 0x0);
+	} else {
+		/* RX DFIR for BW20, BW10 and BW5*/
+		odm_set_bb_reg(dm, 0x948, BIT(29) | BIT(28), 0x2);
+		odm_set_bb_reg(dm, 0x94c, BIT(29) | BIT(28), 0x2);
+		odm_set_bb_reg(dm, 0xc20, BIT(31), 0x1);
+		odm_set_bb_reg(dm, 0xe20, BIT(31), 0x1);
+	}
+}
+
+bool phydm_write_txagc_1byte_8822b(struct phy_dm_struct *dm, u32 power_index,
+				   enum odm_rf_radio_path path, u8 hw_rate)
+{
+	u32 offset_txagc[2] = {0x1d00, 0x1d80};
+	u8 rate_idx = (hw_rate & 0xfc), i;
+	u8 rate_offset = (hw_rate & 0x3);
+	u32 txagc_content = 0x0;
+
+	/* For debug command only!!!! */
+
+	/* Error handling */
+	if ((path > ODM_RF_PATH_B) || (hw_rate > 0x53)) {
+		ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+			     "%s(): unsupported path (%d)\n", __func__, path);
+		return false;
+	}
+
+	/* For HW limitation, We can't write TXAGC once a byte. */
+	for (i = 0; i < 4; i++) {
+		if (i != rate_offset)
+			txagc_content =
+				txagc_content | (config_phydm_read_txagc_8822b(
+							 dm, path, rate_idx + i)
+						 << (i << 3));
+		else
+			txagc_content = txagc_content |
+					((power_index & 0x3f) << (i << 3));
+	}
+	odm_set_bb_reg(dm, (offset_txagc[path] + rate_idx), MASKDWORD,
+		       txagc_content);
+
+	ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+		     "%s(): path-%d rate index 0x%x (0x%x) = 0x%x\n", __func__,
+		     path, hw_rate, (offset_txagc[path] + hw_rate),
+		     power_index);
+	return true;
+}
+
+void phydm_init_hw_info_by_rfe_type_8822b(struct phy_dm_struct *dm)
+{
+	u16 mask_path_a = 0x0303;
+	u16 mask_path_b = 0x0c0c;
+	/*u16	mask_path_c = 0x3030;*/
+	/*u16	mask_path_d = 0xc0c0;*/
+
+	dm->is_init_hw_info_by_rfe = false;
+
+	if ((dm->rfe_type == 1) || (dm->rfe_type == 6) || (dm->rfe_type == 7)) {
+		odm_cmn_info_init(dm, ODM_CMNINFO_BOARD_TYPE,
+				  (ODM_BOARD_EXT_LNA | ODM_BOARD_EXT_LNA_5G |
+				   ODM_BOARD_EXT_PA | ODM_BOARD_EXT_PA_5G));
+
+		if (dm->rfe_type == 6) {
+			odm_cmn_info_init(
+				dm, ODM_CMNINFO_GPA,
+				(TYPE_GPA1 & (mask_path_a | mask_path_b)));
+			odm_cmn_info_init(
+				dm, ODM_CMNINFO_APA,
+				(TYPE_APA1 & (mask_path_a | mask_path_b)));
+			odm_cmn_info_init(
+				dm, ODM_CMNINFO_GLNA,
+				(TYPE_GLNA1 & (mask_path_a | mask_path_b)));
+			odm_cmn_info_init(
+				dm, ODM_CMNINFO_ALNA,
+				(TYPE_ALNA1 & (mask_path_a | mask_path_b)));
+		} else if (dm->rfe_type == 7) {
+			odm_cmn_info_init(
+				dm, ODM_CMNINFO_GPA,
+				(TYPE_GPA2 & (mask_path_a | mask_path_b)));
+			odm_cmn_info_init(
+				dm, ODM_CMNINFO_APA,
+				(TYPE_APA2 & (mask_path_a | mask_path_b)));
+			odm_cmn_info_init(
+				dm, ODM_CMNINFO_GLNA,
+				(TYPE_GLNA2 & (mask_path_a | mask_path_b)));
+			odm_cmn_info_init(
+				dm, ODM_CMNINFO_ALNA,
+				(TYPE_ALNA2 & (mask_path_a | mask_path_b)));
+		} else {
+			odm_cmn_info_init(
+				dm, ODM_CMNINFO_GPA,
+				(TYPE_GPA0 & (mask_path_a | mask_path_b)));
+			odm_cmn_info_init(
+				dm, ODM_CMNINFO_APA,
+				(TYPE_APA0 & (mask_path_a | mask_path_b)));
+			odm_cmn_info_init(
+				dm, ODM_CMNINFO_GLNA,
+				(TYPE_GLNA0 & (mask_path_a | mask_path_b)));
+			odm_cmn_info_init(
+				dm, ODM_CMNINFO_ALNA,
+				(TYPE_ALNA0 & (mask_path_a | mask_path_b)));
+		}
+
+		odm_cmn_info_init(dm, ODM_CMNINFO_PACKAGE_TYPE, 1);
+
+		odm_cmn_info_init(dm, ODM_CMNINFO_EXT_LNA, true);
+		odm_cmn_info_init(dm, ODM_CMNINFO_5G_EXT_LNA, true);
+		odm_cmn_info_init(dm, ODM_CMNINFO_EXT_PA, true);
+		odm_cmn_info_init(dm, ODM_CMNINFO_5G_EXT_PA, true);
+	} else if (dm->rfe_type == 2) {
+		odm_cmn_info_init(dm, ODM_CMNINFO_BOARD_TYPE,
+				  (ODM_BOARD_EXT_LNA_5G | ODM_BOARD_EXT_PA_5G));
+		odm_cmn_info_init(dm, ODM_CMNINFO_APA,
+				  (TYPE_APA0 & (mask_path_a | mask_path_b)));
+		odm_cmn_info_init(dm, ODM_CMNINFO_ALNA,
+				  (TYPE_ALNA0 & (mask_path_a | mask_path_b)));
+
+		odm_cmn_info_init(dm, ODM_CMNINFO_PACKAGE_TYPE, 2);
+
+		odm_cmn_info_init(dm, ODM_CMNINFO_EXT_LNA, false);
+		odm_cmn_info_init(dm, ODM_CMNINFO_5G_EXT_LNA, true);
+		odm_cmn_info_init(dm, ODM_CMNINFO_EXT_PA, false);
+		odm_cmn_info_init(dm, ODM_CMNINFO_5G_EXT_PA, true);
+	} else if (dm->rfe_type == 9) {
+		odm_cmn_info_init(dm, ODM_CMNINFO_BOARD_TYPE,
+				  (ODM_BOARD_EXT_LNA_5G));
+		odm_cmn_info_init(dm, ODM_CMNINFO_ALNA,
+				  (TYPE_ALNA0 & (mask_path_a | mask_path_b)));
+
+		odm_cmn_info_init(dm, ODM_CMNINFO_PACKAGE_TYPE, 1);
+
+		odm_cmn_info_init(dm, ODM_CMNINFO_EXT_LNA, false);
+		odm_cmn_info_init(dm, ODM_CMNINFO_5G_EXT_LNA, true);
+		odm_cmn_info_init(dm, ODM_CMNINFO_EXT_PA, false);
+		odm_cmn_info_init(dm, ODM_CMNINFO_5G_EXT_PA, false);
+	} else if ((dm->rfe_type == 3) || (dm->rfe_type == 5)) {
+		/* RFE type 3: 8822BS\8822BU TFBGA iFEM */
+		/* RFE type 5: 8822BE TFBGA iFEM */
+		odm_cmn_info_init(dm, ODM_CMNINFO_BOARD_TYPE, 0);
+
+		odm_cmn_info_init(dm, ODM_CMNINFO_PACKAGE_TYPE, 2);
+
+		odm_cmn_info_init(dm, ODM_CMNINFO_EXT_LNA, false);
+		odm_cmn_info_init(dm, ODM_CMNINFO_5G_EXT_LNA, false);
+		odm_cmn_info_init(dm, ODM_CMNINFO_EXT_PA, false);
+		odm_cmn_info_init(dm, ODM_CMNINFO_5G_EXT_PA, false);
+	} else if (dm->rfe_type == 4) {
+		odm_cmn_info_init(dm, ODM_CMNINFO_BOARD_TYPE,
+				  (ODM_BOARD_EXT_LNA | ODM_BOARD_EXT_LNA_5G |
+				   ODM_BOARD_EXT_PA | ODM_BOARD_EXT_PA_5G));
+		odm_cmn_info_init(dm, ODM_CMNINFO_GPA,
+				  (TYPE_GPA0 & (mask_path_a | mask_path_b)));
+		odm_cmn_info_init(dm, ODM_CMNINFO_APA,
+				  (TYPE_APA0 & (mask_path_a | mask_path_b)));
+		odm_cmn_info_init(dm, ODM_CMNINFO_GLNA,
+				  (TYPE_GLNA0 & (mask_path_a | mask_path_b)));
+		odm_cmn_info_init(dm, ODM_CMNINFO_ALNA,
+				  (TYPE_ALNA0 & (mask_path_a | mask_path_b)));
+
+		odm_cmn_info_init(dm, ODM_CMNINFO_PACKAGE_TYPE, 2);
+
+		odm_cmn_info_init(dm, ODM_CMNINFO_EXT_LNA, true);
+		odm_cmn_info_init(dm, ODM_CMNINFO_5G_EXT_LNA, true);
+		odm_cmn_info_init(dm, ODM_CMNINFO_EXT_PA, true);
+		odm_cmn_info_init(dm, ODM_CMNINFO_5G_EXT_PA, true);
+	} else if (dm->rfe_type == 8) {
+		/* RFE type 8: TFBGA iFEM AP */
+		odm_cmn_info_init(dm, ODM_CMNINFO_BOARD_TYPE, 0);
+
+		odm_cmn_info_init(dm, ODM_CMNINFO_PACKAGE_TYPE, 2);
+
+		odm_cmn_info_init(dm, ODM_CMNINFO_EXT_LNA, false);
+		odm_cmn_info_init(dm, ODM_CMNINFO_5G_EXT_LNA, false);
+		odm_cmn_info_init(dm, ODM_CMNINFO_EXT_PA, false);
+		odm_cmn_info_init(dm, ODM_CMNINFO_5G_EXT_PA, false);
+	} else {
+		/* RFE Type 0 & 9 & 10: QFN iFEM */
+		odm_cmn_info_init(dm, ODM_CMNINFO_BOARD_TYPE, 0);
+
+		odm_cmn_info_init(dm, ODM_CMNINFO_PACKAGE_TYPE, 1);
+
+		odm_cmn_info_init(dm, ODM_CMNINFO_EXT_LNA, false);
+		odm_cmn_info_init(dm, ODM_CMNINFO_5G_EXT_LNA, false);
+		odm_cmn_info_init(dm, ODM_CMNINFO_EXT_PA, false);
+		odm_cmn_info_init(dm, ODM_CMNINFO_5G_EXT_PA, false);
+	}
+
+	dm->is_init_hw_info_by_rfe = true;
+
+	ODM_RT_TRACE(
+		dm, ODM_PHY_CONFIG,
+		"%s(): RFE type (%d), Board type (0x%x), Package type (%d)\n",
+		__func__, dm->rfe_type, dm->board_type, dm->package_type);
+	ODM_RT_TRACE(
+		dm, ODM_PHY_CONFIG,
+		"%s(): 5G ePA (%d), 5G eLNA (%d), 2G ePA (%d), 2G eLNA (%d)\n",
+		__func__, dm->ext_pa_5g, dm->ext_lna_5g, dm->ext_pa,
+		dm->ext_lna);
+	ODM_RT_TRACE(
+		dm, ODM_PHY_CONFIG,
+		"%s(): 5G PA type (%d), 5G LNA type (%d), 2G PA type (%d), 2G LNA type (%d)\n",
+		__func__, dm->type_apa, dm->type_alna, dm->type_gpa,
+		dm->type_glna);
+}
+
+s32 phydm_get_condition_number_8822B(struct phy_dm_struct *dm)
+{
+	s32 ret_val;
+
+	odm_set_bb_reg(dm, 0x1988, BIT(22), 0x1);
+	ret_val =
+		(s32)odm_get_bb_reg(dm, 0xf84, (BIT(17) | BIT(16) | MASKLWORD));
+
+	if (bw_8822b == 0) {
+		ret_val = ret_val << (8 - 4);
+		ret_val = ret_val / 234;
+	} else if (bw_8822b == 1) {
+		ret_val = ret_val << (7 - 4);
+		ret_val = ret_val / 108;
+	} else if (bw_8822b == 2) {
+		ret_val = ret_val << (6 - 4);
+		ret_val = ret_val / 52;
+	}
+
+	return ret_val;
+}
+
+/* ======================================================================== */
+
+/* ======================================================================== */
+/* These following functions can be used by driver*/
+
+u32 config_phydm_read_rf_reg_8822b(struct phy_dm_struct *dm,
+				   enum odm_rf_radio_path rf_path, u32 reg_addr,
+				   u32 bit_mask)
+{
+	u32 readback_value, direct_addr;
+	u32 offset_read_rf[2] = {0x2800, 0x2c00};
+	u32 power_RF[2] = {0x1c, 0xec};
+
+	/* Error handling.*/
+	if (rf_path > ODM_RF_PATH_B) {
+		ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+			     "%s(): unsupported path (%d)\n", __func__,
+			     rf_path);
+		return INVALID_RF_DATA;
+	}
+
+	/*  Error handling. Check if RF power is enable or not */
+	/*  0xffffffff means RF power is disable */
+	if (odm_get_mac_reg(dm, power_RF[rf_path], MASKBYTE3) != 0x7) {
+		ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+			     "%s(): Read fail, RF is disabled\n", __func__);
+		return INVALID_RF_DATA;
+	}
+
+	/* Calculate offset */
+	reg_addr &= 0xff;
+	direct_addr = offset_read_rf[rf_path] + (reg_addr << 2);
+
+	/* RF register only has 20bits */
+	bit_mask &= RFREGOFFSETMASK;
+
+	/* Read RF register directly */
+	readback_value = odm_get_bb_reg(dm, direct_addr, bit_mask);
+	ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+		     "%s(): RF-%d 0x%x = 0x%x, bit mask = 0x%x\n", __func__,
+		     rf_path, reg_addr, readback_value, bit_mask);
+	return readback_value;
+}
+
+bool config_phydm_write_rf_reg_8822b(struct phy_dm_struct *dm,
+				     enum odm_rf_radio_path rf_path,
+				     u32 reg_addr, u32 bit_mask, u32 data)
+{
+	u32 data_and_addr = 0, data_original = 0;
+	u32 offset_write_rf[2] = {0xc90, 0xe90};
+	u32 power_RF[2] = {0x1c, 0xec};
+
+	/* Error handling.*/
+	if (rf_path > ODM_RF_PATH_B) {
+		ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+			     "%s(): unsupported path (%d)\n", __func__,
+			     rf_path);
+		return false;
+	}
+
+	/* Read RF register content first */
+	reg_addr &= 0xff;
+	bit_mask = bit_mask & RFREGOFFSETMASK;
+
+	if (bit_mask != RFREGOFFSETMASK) {
+		data_original = config_phydm_read_rf_reg_8822b(
+			dm, rf_path, reg_addr, RFREGOFFSETMASK);
+
+		/* Error handling. RF is disabled */
+		if (!config_phydm_read_rf_check_8822b(data_original)) {
+			ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+				     "%s(): Write fail, RF is disable\n",
+				     __func__);
+			return false;
+		}
+
+		/* check bit mask */
+		data = phydm_check_bit_mask(bit_mask, data_original, data);
+	} else if (odm_get_mac_reg(dm, power_RF[rf_path], MASKBYTE3) != 0x7) {
+		ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+			     "%s(): Write fail, RF is disabled\n", __func__);
+		return false;
+	}
+
+	/* Put write addr in [27:20]  and write data in [19:00] */
+	data_and_addr = ((reg_addr << 20) | (data & 0x000fffff)) & 0x0fffffff;
+
+	/* Write operation */
+	odm_set_bb_reg(dm, offset_write_rf[rf_path], MASKDWORD, data_and_addr);
+	ODM_RT_TRACE(
+		dm, ODM_PHY_CONFIG,
+		"%s(): RF-%d 0x%x = 0x%x (original: 0x%x), bit mask = 0x%x\n",
+		__func__, rf_path, reg_addr, data, data_original, bit_mask);
+	return true;
+}
+
+bool config_phydm_write_txagc_8822b(struct phy_dm_struct *dm, u32 power_index,
+				    enum odm_rf_radio_path path, u8 hw_rate)
+{
+	u32 offset_txagc[2] = {0x1d00, 0x1d80};
+	u8 rate_idx = (hw_rate & 0xfc);
+
+	/* Input need to be HW rate index, not driver rate index!!!! */
+
+	if (dm->is_disable_phy_api) {
+		ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+			     "%s(): disable PHY API for debug!!\n", __func__);
+		return true;
+	}
+
+	/* Error handling */
+	if ((path > ODM_RF_PATH_B) || (hw_rate > 0x53)) {
+		ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+			     "%s(): unsupported path (%d)\n", __func__, path);
+		return false;
+	}
+
+	/* driver need to construct a 4-byte power index */
+	odm_set_bb_reg(dm, (offset_txagc[path] + rate_idx), MASKDWORD,
+		       power_index);
+
+	ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+		     "%s(): path-%d rate index 0x%x (0x%x) = 0x%x\n", __func__,
+		     path, hw_rate, (offset_txagc[path] + hw_rate),
+		     power_index);
+	return true;
+}
+
+u8 config_phydm_read_txagc_8822b(struct phy_dm_struct *dm,
+				 enum odm_rf_radio_path path, u8 hw_rate)
+{
+	u8 read_back_data;
+
+	/* Input need to be HW rate index, not driver rate index!!!! */
+
+	/* Error handling */
+	if ((path > ODM_RF_PATH_B) || (hw_rate > 0x53)) {
+		ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+			     "%s(): unsupported path (%d)\n", __func__, path);
+		return INVALID_TXAGC_DATA;
+	}
+
+	/* Disable TX AGC report */
+	odm_set_bb_reg(dm, 0x1998, BIT(16), 0x0); /* need to check */
+
+	/* Set data rate index (bit0~6) and path index (bit7) */
+	odm_set_bb_reg(dm, 0x1998, MASKBYTE0, (hw_rate | (path << 7)));
+
+	/* Enable TXAGC report */
+	odm_set_bb_reg(dm, 0x1998, BIT(16), 0x1);
+
+	/* Read TX AGC report */
+	read_back_data = (u8)odm_get_bb_reg(dm, 0xd30, 0x7f0000);
+
+	/* Driver have to disable TXAGC report after reading TXAGC
+	 * (ref. user guide v11)
+	 */
+	odm_set_bb_reg(dm, 0x1998, BIT(16), 0x0);
+
+	ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+		     "%s(): path-%d rate index 0x%x = 0x%x\n", __func__, path,
+		     hw_rate, read_back_data);
+	return read_back_data;
+}
+
+bool config_phydm_switch_band_8822b(struct phy_dm_struct *dm, u8 central_ch)
+{
+	u32 rf_reg18;
+	bool rf_reg_status = true;
+
+	ODM_RT_TRACE(dm, ODM_PHY_CONFIG, "%s()======================>\n",
+		     __func__);
+
+	if (dm->is_disable_phy_api) {
+		ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+			     "%s(): disable PHY API for debug!!\n", __func__);
+		return true;
+	}
+
+	rf_reg18 = config_phydm_read_rf_reg_8822b(dm, ODM_RF_PATH_A, 0x18,
+						  RFREGOFFSETMASK);
+	rf_reg_status =
+		rf_reg_status & config_phydm_read_rf_check_8822b(rf_reg18);
+
+	if (central_ch <= 14) {
+		/* 2.4G */
+
+		/* Enable CCK block */
+		odm_set_bb_reg(dm, 0x808, BIT(28), 0x1);
+
+		/* Disable MAC CCK check */
+		odm_set_bb_reg(dm, 0x454, BIT(7), 0x0);
+
+		/* Disable BB CCK check */
+		odm_set_bb_reg(dm, 0xa80, BIT(18), 0x0);
+
+		/*CCA Mask*/
+		odm_set_bb_reg(dm, 0x814, 0x0000FC00, 15); /*default value*/
+
+		/* RF band */
+		rf_reg18 = (rf_reg18 & (~(BIT(16) | BIT(9) | BIT(8))));
+
+		/* RxHP dynamic control */
+		if ((dm->rfe_type == 2) || (dm->rfe_type == 3) ||
+		    (dm->rfe_type == 5)) {
+			odm_set_bb_reg(dm, 0x8cc, MASKDWORD, 0x08108492);
+			odm_set_bb_reg(dm, 0x8d8, MASKDWORD, 0x29095612);
+		}
+
+	} else if (central_ch > 35) {
+		/* 5G */
+
+		/* Enable BB CCK check */
+		odm_set_bb_reg(dm, 0xa80, BIT(18), 0x1);
+
+		/* Enable CCK check */
+		odm_set_bb_reg(dm, 0x454, BIT(7), 0x1);
+
+		/* Disable CCK block */
+		odm_set_bb_reg(dm, 0x808, BIT(28), 0x0);
+
+		/*CCA Mask*/
+		odm_set_bb_reg(dm, 0x814, 0x0000FC00, 15); /*default value*/
+
+		/* RF band */
+		rf_reg18 = (rf_reg18 & (~(BIT(16) | BIT(9) | BIT(8))));
+		rf_reg18 = (rf_reg18 | BIT(8) | BIT(16));
+
+		/* RxHP dynamic control */
+		if ((dm->rfe_type == 2) || (dm->rfe_type == 3) ||
+		    (dm->rfe_type == 5)) {
+			odm_set_bb_reg(dm, 0x8cc, MASKDWORD, 0x08100000);
+			odm_set_bb_reg(dm, 0x8d8, MASKDWORD, 0x21095612);
+		}
+
+	} else {
+		ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+			     "%s(): Fail to switch band (ch: %d)\n", __func__,
+			     central_ch);
+		return false;
+	}
+
+	rf_reg_status = rf_reg_status & config_phydm_write_rf_reg_8822b(
+						dm, ODM_RF_PATH_A, 0x18,
+						RFREGOFFSETMASK, rf_reg18);
+
+	if (dm->rf_type > ODM_1T1R)
+		rf_reg_status =
+			rf_reg_status & config_phydm_write_rf_reg_8822b(
+						dm, ODM_RF_PATH_B, 0x18,
+						RFREGOFFSETMASK, rf_reg18);
+
+	if (!phydm_rfe_8822b(dm, central_ch))
+		return false;
+
+	if (!rf_reg_status) {
+		ODM_RT_TRACE(
+			dm, ODM_PHY_CONFIG,
+			"%s(): Fail to switch band (ch: %d), because writing RF register is fail\n",
+			__func__, central_ch);
+		return false;
+	}
+
+	ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+		     "%s(): Success to switch band (ch: %d)\n", __func__,
+		     central_ch);
+	return true;
+}
+
+bool config_phydm_switch_channel_8822b(struct phy_dm_struct *dm, u8 central_ch)
+{
+	struct dig_thres *dig_tab = &dm->dm_dig_table;
+	u32 rf_reg18 = 0, rf_reg_b8 = 0, rf_reg_be = 0xff;
+	bool rf_reg_status = true;
+	u8 low_band[15] = {0x7, 0x6, 0x6, 0x5, 0x0, 0x0, 0x7, 0xff,
+			   0x6, 0x5, 0x0, 0x0, 0x7, 0x6, 0x6};
+	u8 middle_band[23] = {0x6, 0x5, 0x0, 0x0, 0x7, 0x6, 0x6, 0xff,
+			      0x0, 0x0, 0x7, 0x6, 0x6, 0x5, 0x0, 0xff,
+			      0x7, 0x6, 0x6, 0x5, 0x0, 0x0, 0x7};
+	u8 high_band[15] = {0x5, 0x5, 0x0, 0x7, 0x7, 0x6, 0x5, 0xff,
+			    0x0, 0x7, 0x7, 0x6, 0x5, 0x5, 0x0};
+
+	ODM_RT_TRACE(dm, ODM_PHY_CONFIG, "%s()====================>\n",
+		     __func__);
+
+	if (dm->is_disable_phy_api) {
+		ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+			     "%s(): disable PHY API for debug!!\n", __func__);
+		return true;
+	}
+
+	central_ch_8822b = central_ch;
+	rf_reg18 = config_phydm_read_rf_reg_8822b(dm, ODM_RF_PATH_A, 0x18,
+						  RFREGOFFSETMASK);
+	rf_reg_status =
+		rf_reg_status & config_phydm_read_rf_check_8822b(rf_reg18);
+	rf_reg18 = (rf_reg18 & (~(BIT(18) | BIT(17) | MASKBYTE0)));
+
+	if (dm->cut_version == ODM_CUT_A) {
+		rf_reg_b8 = config_phydm_read_rf_reg_8822b(
+			dm, ODM_RF_PATH_A, 0xb8, RFREGOFFSETMASK);
+		rf_reg_status = rf_reg_status &
+				config_phydm_read_rf_check_8822b(rf_reg_b8);
+	}
+
+	/* Switch band and channel */
+	if (central_ch <= 14) {
+		/* 2.4G */
+
+		/* 1. RF band and channel*/
+		rf_reg18 = (rf_reg18 | central_ch);
+
+		/* 2. AGC table selection */
+		odm_set_bb_reg(dm, 0x958, 0x1f, 0x0);
+		dig_tab->agc_table_idx = 0x0;
+
+		/* 3. Set central frequency for clock offset tracking */
+		odm_set_bb_reg(dm, 0x860, 0x1ffe0000, 0x96a);
+
+		/* Fix A-cut LCK fail issue @ 5285MHz~5375MHz, 0xb8[19]=0x0 */
+		if (dm->cut_version == ODM_CUT_A)
+			rf_reg_b8 = rf_reg_b8 | BIT(19);
+
+		/* CCK TX filter parameters */
+		if (central_ch == 14) {
+			odm_set_bb_reg(dm, 0xa20, MASKHWORD, 0x8488);
+			odm_set_bb_reg(dm, 0xa24, MASKDWORD, 0x00006577);
+			odm_set_bb_reg(dm, 0xa28, MASKLWORD, 0x0000);
+		} else {
+			odm_set_bb_reg(dm, 0xa20, MASKHWORD,
+				       (rega20_8822b >> 16));
+			odm_set_bb_reg(dm, 0xa24, MASKDWORD, rega24_8822b);
+			odm_set_bb_reg(dm, 0xa28, MASKLWORD,
+				       (rega28_8822b & MASKLWORD));
+		}
+
+	} else if (central_ch > 35) {
+		/* 5G */
+
+		/* 1. RF band and channel*/
+		rf_reg18 = (rf_reg18 | central_ch);
+
+		/* 2. AGC table selection */
+		if ((central_ch >= 36) && (central_ch <= 64)) {
+			odm_set_bb_reg(dm, 0x958, 0x1f, 0x1);
+			dig_tab->agc_table_idx = 0x1;
+		} else if ((central_ch >= 100) && (central_ch <= 144)) {
+			odm_set_bb_reg(dm, 0x958, 0x1f, 0x2);
+			dig_tab->agc_table_idx = 0x2;
+		} else if (central_ch >= 149) {
+			odm_set_bb_reg(dm, 0x958, 0x1f, 0x3);
+			dig_tab->agc_table_idx = 0x3;
+		} else {
+			ODM_RT_TRACE(
+				dm, ODM_PHY_CONFIG,
+				"%s(): Fail to switch channel (AGC) (ch: %d)\n",
+				__func__, central_ch);
+			return false;
+		}
+
+		/* 3. Set central frequency for clock offset tracking */
+		if ((central_ch >= 36) && (central_ch <= 48)) {
+			odm_set_bb_reg(dm, 0x860, 0x1ffe0000, 0x494);
+		} else if ((central_ch >= 52) && (central_ch <= 64)) {
+			odm_set_bb_reg(dm, 0x860, 0x1ffe0000, 0x453);
+		} else if ((central_ch >= 100) && (central_ch <= 116)) {
+			odm_set_bb_reg(dm, 0x860, 0x1ffe0000, 0x452);
+		} else if ((central_ch >= 118) && (central_ch <= 177)) {
+			odm_set_bb_reg(dm, 0x860, 0x1ffe0000, 0x412);
+		} else {
+			ODM_RT_TRACE(
+				dm, ODM_PHY_CONFIG,
+				"%s(): Fail to switch channel (fc_area) (ch: %d)\n",
+				__func__, central_ch);
+			return false;
+		}
+
+		/* Fix A-cut LCK fail issue @ 5285MHz~5375MHz, 0xb8[19]=0x0 */
+		if (dm->cut_version == ODM_CUT_A) {
+			if ((central_ch >= 57) && (central_ch <= 75))
+				rf_reg_b8 = rf_reg_b8 & (~BIT(19));
+			else
+				rf_reg_b8 = rf_reg_b8 | BIT(19);
+		}
+	} else {
+		ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+			     "%s(): Fail to switch channel (ch: %d)\n",
+			     __func__, central_ch);
+		return false;
+	}
+
+	/* Modify IGI for MP driver to aviod PCIE interference */
+	if (dm->mp_mode && ((dm->rfe_type == 3) || (dm->rfe_type == 5))) {
+		if (central_ch == 14)
+			odm_write_dig(dm, 0x26);
+		else
+			odm_write_dig(dm, 0x20);
+	}
+
+	/* Modify the setting of register 0xBE to reduce phase noise */
+	if (central_ch <= 14)
+		rf_reg_be = 0x0;
+	else if ((central_ch >= 36) && (central_ch <= 64))
+		rf_reg_be = low_band[(central_ch - 36) >> 1];
+	else if ((central_ch >= 100) && (central_ch <= 144))
+		rf_reg_be = middle_band[(central_ch - 100) >> 1];
+	else if ((central_ch >= 149) && (central_ch <= 177))
+		rf_reg_be = high_band[(central_ch - 149) >> 1];
+	else
+		rf_reg_be = 0xff;
+
+	if (rf_reg_be != 0xff) {
+		rf_reg_status =
+			rf_reg_status & config_phydm_write_rf_reg_8822b(
+						dm, ODM_RF_PATH_A, 0xbe,
+						(BIT(17) | BIT(16) | BIT(15)),
+						rf_reg_be);
+	} else {
+		ODM_RT_TRACE(
+			dm, ODM_PHY_CONFIG,
+			"%s(): Fail to switch channel (ch: %d, Phase noise)\n",
+			__func__, central_ch);
+		return false;
+	}
+
+	/* Fix channel 144 issue, ask by RFSI Alvin*/
+	/* 00 when freq < 5400;  01 when 5400<=freq<=5720; 10 when freq > 5720;
+	 * 2G don't care
+	 */
+	/* need to set 0xdf[18]=1 before writing RF18 when channel 144 */
+	if (central_ch == 144) {
+		rf_reg_status = rf_reg_status &
+				config_phydm_write_rf_reg_8822b(
+					dm, ODM_RF_PATH_A, 0xdf, BIT(18), 0x1);
+		rf_reg18 = (rf_reg18 | BIT(17));
+	} else {
+		rf_reg_status = rf_reg_status &
+				config_phydm_write_rf_reg_8822b(
+					dm, ODM_RF_PATH_A, 0xdf, BIT(18), 0x0);
+
+		if (central_ch > 144)
+			rf_reg18 = (rf_reg18 | BIT(18));
+		else if (central_ch >= 80)
+			rf_reg18 = (rf_reg18 | BIT(17));
+	}
+
+	rf_reg_status = rf_reg_status & config_phydm_write_rf_reg_8822b(
+						dm, ODM_RF_PATH_A, 0x18,
+						RFREGOFFSETMASK, rf_reg18);
+
+	if (dm->cut_version == ODM_CUT_A)
+		rf_reg_status =
+			rf_reg_status & config_phydm_write_rf_reg_8822b(
+						dm, ODM_RF_PATH_A, 0xb8,
+						RFREGOFFSETMASK, rf_reg_b8);
+
+	if (dm->rf_type > ODM_1T1R) {
+		rf_reg_status =
+			rf_reg_status & config_phydm_write_rf_reg_8822b(
+						dm, ODM_RF_PATH_B, 0x18,
+						RFREGOFFSETMASK, rf_reg18);
+
+		if (dm->cut_version == ODM_CUT_A)
+			rf_reg_status = rf_reg_status &
+					config_phydm_write_rf_reg_8822b(
+						dm, ODM_RF_PATH_B, 0xb8,
+						RFREGOFFSETMASK, rf_reg_b8);
+	}
+
+	if (!rf_reg_status) {
+		ODM_RT_TRACE(
+			dm, ODM_PHY_CONFIG,
+			"%s(): Fail to switch channel (ch: %d), because writing RF register is fail\n",
+			__func__, central_ch);
+		return false;
+	}
+
+	phydm_ccapar_by_rfe_8822b(dm);
+	ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+		     "%s(): Success to switch channel (ch: %d)\n", __func__,
+		     central_ch);
+	return true;
+}
+
+bool config_phydm_switch_bandwidth_8822b(struct phy_dm_struct *dm,
+					 u8 primary_ch_idx,
+					 enum odm_bw bandwidth)
+{
+	u32 rf_reg18;
+	bool rf_reg_status = true;
+	u8 IGI = 0;
+
+	ODM_RT_TRACE(dm, ODM_PHY_CONFIG, "%s()===================>\n",
+		     __func__);
+
+	if (dm->is_disable_phy_api) {
+		ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+			     "%s(): disable PHY API for debug!!\n", __func__);
+		return true;
+	}
+
+	/* Error handling */
+	if ((bandwidth >= ODM_BW_MAX) ||
+	    ((bandwidth == ODM_BW40M) && (primary_ch_idx > 2)) ||
+	    ((bandwidth == ODM_BW80M) && (primary_ch_idx > 4))) {
+		ODM_RT_TRACE(
+			dm, ODM_PHY_CONFIG,
+			"%s(): Fail to switch bandwidth (bw: %d, primary ch: %d)\n",
+			__func__, bandwidth, primary_ch_idx);
+		return false;
+	}
+
+	bw_8822b = bandwidth;
+	rf_reg18 = config_phydm_read_rf_reg_8822b(dm, ODM_RF_PATH_A, 0x18,
+						  RFREGOFFSETMASK);
+	rf_reg_status =
+		rf_reg_status & config_phydm_read_rf_check_8822b(rf_reg18);
+
+	/* Switch bandwidth */
+	switch (bandwidth) {
+	case ODM_BW20M: {
+		/* Small BW([7:6]) = 0, primary channel ([5:2]) = 0,
+		 * rf mode([1:0]) = 20M
+		 */
+		odm_set_bb_reg(dm, 0x8ac, MASKBYTE0, ODM_BW20M);
+
+		/* ADC clock = 160M clock for BW20 */
+		odm_set_bb_reg(dm, 0x8ac, (BIT(9) | BIT(8)), 0x0);
+		odm_set_bb_reg(dm, 0x8ac, BIT(16), 0x1);
+
+		/* DAC clock = 160M clock for BW20 */
+		odm_set_bb_reg(dm, 0x8ac, (BIT(21) | BIT(20)), 0x0);
+		odm_set_bb_reg(dm, 0x8ac, BIT(28), 0x1);
+
+		/* ADC buffer clock */
+		odm_set_bb_reg(dm, 0x8c4, BIT(30), 0x1);
+
+		/* RF bandwidth */
+		rf_reg18 = (rf_reg18 | BIT(11) | BIT(10));
+
+		break;
+	}
+	case ODM_BW40M: {
+		/* Small BW([7:6]) = 0, primary channel ([5:2]) = sub-channel,
+		 * rf mode([1:0]) = 40M
+		 */
+		odm_set_bb_reg(dm, 0x8ac, MASKBYTE0,
+			       (((primary_ch_idx & 0xf) << 2) | ODM_BW40M));
+
+		/* CCK primary channel */
+		if (primary_ch_idx == 1)
+			odm_set_bb_reg(dm, 0xa00, BIT(4), primary_ch_idx);
+		else
+			odm_set_bb_reg(dm, 0xa00, BIT(4), 0);
+
+		/* ADC clock = 160M clock for BW40 */
+		odm_set_bb_reg(dm, 0x8ac, (BIT(11) | BIT(10)), 0x0);
+		odm_set_bb_reg(dm, 0x8ac, BIT(17), 0x1);
+
+		/* DAC clock = 160M clock for BW20 */
+		odm_set_bb_reg(dm, 0x8ac, (BIT(23) | BIT(22)), 0x0);
+		odm_set_bb_reg(dm, 0x8ac, BIT(29), 0x1);
+
+		/* ADC buffer clock */
+		odm_set_bb_reg(dm, 0x8c4, BIT(30), 0x1);
+
+		/* RF bandwidth */
+		rf_reg18 = (rf_reg18 & (~(BIT(11) | BIT(10))));
+		rf_reg18 = (rf_reg18 | BIT(11));
+
+		break;
+	}
+	case ODM_BW80M: {
+		/* Small BW([7:6]) = 0, primary channel ([5:2]) = sub-channel,
+		 * rf mode([1:0]) = 80M
+		 */
+		odm_set_bb_reg(dm, 0x8ac, MASKBYTE0,
+			       (((primary_ch_idx & 0xf) << 2) | ODM_BW80M));
+
+		/* ADC clock = 160M clock for BW80 */
+		odm_set_bb_reg(dm, 0x8ac, (BIT(13) | BIT(12)), 0x0);
+		odm_set_bb_reg(dm, 0x8ac, BIT(18), 0x1);
+
+		/* DAC clock = 160M clock for BW20 */
+		odm_set_bb_reg(dm, 0x8ac, (BIT(25) | BIT(24)), 0x0);
+		odm_set_bb_reg(dm, 0x8ac, BIT(30), 0x1);
+
+		/* ADC buffer clock */
+		odm_set_bb_reg(dm, 0x8c4, BIT(30), 0x1);
+
+		/* RF bandwidth */
+		rf_reg18 = (rf_reg18 & (~(BIT(11) | BIT(10))));
+		rf_reg18 = (rf_reg18 | BIT(10));
+
+		break;
+	}
+	case ODM_BW5M: {
+		/* Small BW([7:6]) = 1, primary channel ([5:2]) = 0,
+		 * rf mode([1:0]) = 20M
+		 */
+		odm_set_bb_reg(dm, 0x8ac, MASKBYTE0, (BIT(6) | ODM_BW20M));
+
+		/* ADC clock = 40M clock */
+		odm_set_bb_reg(dm, 0x8ac, (BIT(9) | BIT(8)), 0x2);
+		odm_set_bb_reg(dm, 0x8ac, BIT(16), 0x0);
+
+		/* DAC clock = 160M clock for BW20 */
+		odm_set_bb_reg(dm, 0x8ac, (BIT(21) | BIT(20)), 0x2);
+		odm_set_bb_reg(dm, 0x8ac, BIT(28), 0x0);
+
+		/* ADC buffer clock */
+		odm_set_bb_reg(dm, 0x8c4, BIT(30), 0x0);
+		odm_set_bb_reg(dm, 0x8c8, BIT(31), 0x1);
+
+		/* RF bandwidth */
+		rf_reg18 = (rf_reg18 | BIT(11) | BIT(10));
+
+		break;
+	}
+	case ODM_BW10M: {
+		/* Small BW([7:6]) = 1, primary channel ([5:2]) = 0,
+		 * rf mode([1:0]) = 20M
+		 */
+		odm_set_bb_reg(dm, 0x8ac, MASKBYTE0, (BIT(7) | ODM_BW20M));
+
+		/* ADC clock = 80M clock */
+		odm_set_bb_reg(dm, 0x8ac, (BIT(9) | BIT(8)), 0x3);
+		odm_set_bb_reg(dm, 0x8ac, BIT(16), 0x0);
+
+		/* DAC clock = 160M clock for BW20 */
+		odm_set_bb_reg(dm, 0x8ac, (BIT(21) | BIT(20)), 0x3);
+		odm_set_bb_reg(dm, 0x8ac, BIT(28), 0x0);
+
+		/* ADC buffer clock */
+		odm_set_bb_reg(dm, 0x8c4, BIT(30), 0x0);
+		odm_set_bb_reg(dm, 0x8c8, BIT(31), 0x1);
+
+		/* RF bandwidth */
+		rf_reg18 = (rf_reg18 | BIT(11) | BIT(10));
+
+		break;
+	}
+	default:
+		ODM_RT_TRACE(
+			dm, ODM_PHY_CONFIG,
+			"%s(): Fail to switch bandwidth (bw: %d, primary ch: %d)\n",
+			__func__, bandwidth, primary_ch_idx);
+	}
+
+	/* Write RF register */
+	rf_reg_status = rf_reg_status & config_phydm_write_rf_reg_8822b(
+						dm, ODM_RF_PATH_A, 0x18,
+						RFREGOFFSETMASK, rf_reg18);
+
+	if (dm->rf_type > ODM_1T1R)
+		rf_reg_status =
+			rf_reg_status & config_phydm_write_rf_reg_8822b(
+						dm, ODM_RF_PATH_B, 0x18,
+						RFREGOFFSETMASK, rf_reg18);
+
+	if (!rf_reg_status) {
+		ODM_RT_TRACE(
+			dm, ODM_PHY_CONFIG,
+			"%s(): Fail to switch bandwidth (bw: %d, primary ch: %d), because writing RF register is fail\n",
+			__func__, bandwidth, primary_ch_idx);
+		return false;
+	}
+
+	/* Modify RX DFIR parameters */
+	phydm_rxdfirpar_by_bw_8822b(dm, bandwidth);
+
+	/* Modify CCA parameters */
+	phydm_ccapar_by_bw_8822b(dm, bandwidth);
+	phydm_ccapar_by_rfe_8822b(dm);
+
+	/* Toggle RX path to avoid RX dead zone issue */
+	odm_set_bb_reg(dm, 0x808, MASKBYTE0, 0x0);
+	odm_set_bb_reg(dm, 0x808, MASKBYTE0,
+		       (dm->rx_ant_status | (dm->rx_ant_status << 4)));
+
+	/* Toggle IGI to let RF enter RX mode */
+	IGI = (u8)odm_get_bb_reg(dm, ODM_REG(IGI_A, dm), ODM_BIT(IGI, dm));
+	odm_set_bb_reg(dm, ODM_REG(IGI_A, dm), ODM_BIT(IGI, dm), IGI - 2);
+	odm_set_bb_reg(dm, ODM_REG(IGI_B, dm), ODM_BIT(IGI, dm), IGI - 2);
+	odm_set_bb_reg(dm, ODM_REG(IGI_A, dm), ODM_BIT(IGI, dm), IGI);
+	odm_set_bb_reg(dm, ODM_REG(IGI_B, dm), ODM_BIT(IGI, dm), IGI);
+
+	ODM_RT_TRACE(
+		dm, ODM_PHY_CONFIG,
+		"%s(): Success to switch bandwidth (bw: %d, primary ch: %d)\n",
+		__func__, bandwidth, primary_ch_idx);
+	return true;
+}
+
+bool config_phydm_switch_channel_bw_8822b(struct phy_dm_struct *dm,
+					  u8 central_ch, u8 primary_ch_idx,
+					  enum odm_bw bandwidth)
+{
+	/* Switch band */
+	if (!config_phydm_switch_band_8822b(dm, central_ch))
+		return false;
+
+	/* Switch channel */
+	if (!config_phydm_switch_channel_8822b(dm, central_ch))
+		return false;
+
+	/* Switch bandwidth */
+	if (!config_phydm_switch_bandwidth_8822b(dm, primary_ch_idx, bandwidth))
+		return false;
+
+	return true;
+}
+
+bool config_phydm_trx_mode_8822b(struct phy_dm_struct *dm,
+				 enum odm_rf_path tx_path,
+				 enum odm_rf_path rx_path, bool is_tx2_path)
+{
+	bool rf_reg_status = true;
+	u8 IGI;
+	u32 rf_reg33 = 0;
+	u16 counter = 0;
+
+	ODM_RT_TRACE(dm, ODM_PHY_CONFIG, "%s()=====================>\n",
+		     __func__);
+
+	if (dm->is_disable_phy_api) {
+		ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+			     "%s(): disable PHY API for debug!!\n", __func__);
+		return true;
+	}
+
+	if ((tx_path & (~(ODM_RF_A | ODM_RF_B))) != 0) {
+		ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+			     "%s(): Wrong TX setting (TX: 0x%x)\n", __func__,
+			     tx_path);
+		return false;
+	}
+
+	if ((rx_path & (~(ODM_RF_A | ODM_RF_B))) != 0) {
+		ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+			     "%s(): Wrong RX setting (RX: 0x%x)\n", __func__,
+			     rx_path);
+		return false;
+	}
+
+	/* RF mode of path-A and path-B */
+	/* Cannot shut down path-A, beacause synthesizer will be shut down when
+	 * path-A is in shut down mode
+	 */
+	if ((tx_path | rx_path) & ODM_RF_A)
+		odm_set_bb_reg(dm, 0xc08, MASKLWORD, 0x3231);
+	else
+		odm_set_bb_reg(dm, 0xc08, MASKLWORD, 0x1111);
+
+	if ((tx_path | rx_path) & ODM_RF_B)
+		odm_set_bb_reg(dm, 0xe08, MASKLWORD, 0x3231);
+	else
+		odm_set_bb_reg(dm, 0xe08, MASKLWORD, 0x1111);
+
+	/* Set TX antenna by Nsts */
+	odm_set_bb_reg(dm, 0x93c, (BIT(19) | BIT(18)), 0x3);
+	odm_set_bb_reg(dm, 0x80c, (BIT(29) | BIT(28)), 0x1);
+
+	/* Control CCK TX path by 0xa07[7] */
+	odm_set_bb_reg(dm, 0x80c, BIT(30), 0x1);
+
+	/* TX logic map and TX path en for Nsts = 1, and CCK TX path*/
+	if (tx_path & ODM_RF_A) {
+		odm_set_bb_reg(dm, 0x93c, 0xfff00000, 0x001);
+		odm_set_bb_reg(dm, 0xa04, 0xf0000000, 0x8);
+	} else if (tx_path & ODM_RF_B) {
+		odm_set_bb_reg(dm, 0x93c, 0xfff00000, 0x002);
+		odm_set_bb_reg(dm, 0xa04, 0xf0000000, 0x4);
+	}
+
+	/* TX logic map and TX path en for Nsts = 2*/
+	if ((tx_path == ODM_RF_A) || (tx_path == ODM_RF_B))
+		odm_set_bb_reg(dm, 0x940, 0xfff0, 0x01);
+	else
+		odm_set_bb_reg(dm, 0x940, 0xfff0, 0x43);
+
+	/* TX path enable */
+	odm_set_bb_reg(dm, 0x80c, MASKBYTE0, ((tx_path << 4) | tx_path));
+
+	/* Tx2path for 1ss */
+	if (!((tx_path == ODM_RF_A) || (tx_path == ODM_RF_B))) {
+		if (is_tx2_path || dm->mp_mode) {
+			/* 2Tx for OFDM */
+			odm_set_bb_reg(dm, 0x93c, 0xfff00000, 0x043);
+
+			/* 2Tx for CCK */
+			odm_set_bb_reg(dm, 0xa04, 0xf0000000, 0xc);
+		}
+	}
+
+	/* Always disable MRC for CCK CCA */
+	odm_set_bb_reg(dm, 0xa2c, BIT(22), 0x0);
+
+	/* Always disable MRC for CCK barker */
+	odm_set_bb_reg(dm, 0xa2c, BIT(18), 0x0);
+
+	/* CCK RX 1st and 2nd path setting*/
+	if (rx_path & ODM_RF_A)
+		odm_set_bb_reg(dm, 0xa04, 0x0f000000, 0x0);
+	else if (rx_path & ODM_RF_B)
+		odm_set_bb_reg(dm, 0xa04, 0x0f000000, 0x5);
+
+	/* RX path enable */
+	odm_set_bb_reg(dm, 0x808, MASKBYTE0, ((rx_path << 4) | rx_path));
+
+	if ((rx_path == ODM_RF_A) || (rx_path == ODM_RF_B)) {
+		/* 1R */
+
+		/* Disable MRC for CCA */
+		/* odm_set_bb_reg(dm, 0xa2c, BIT22, 0x0); */
+
+		/* Disable MRC for barker */
+		/* odm_set_bb_reg(dm, 0xa2c, BIT18, 0x0); */
+
+		/* Disable CCK antenna diversity */
+		/* odm_set_bb_reg(dm, 0xa00, BIT15, 0x0); */
+
+		/* Disable Antenna weighting */
+		odm_set_bb_reg(dm, 0x1904, BIT(16), 0x0);
+		odm_set_bb_reg(dm, 0x800, BIT(28), 0x0);
+		odm_set_bb_reg(dm, 0x850, BIT(23), 0x0);
+	} else {
+		/* 2R */
+
+		/* Enable MRC for CCA */
+		/* odm_set_bb_reg(dm, 0xa2c, BIT22, 0x1); */
+
+		/* Enable MRC for barker */
+		/* odm_set_bb_reg(dm, 0xa2c, BIT18, 0x1); */
+
+		/* Disable CCK antenna diversity */
+		/* odm_set_bb_reg(dm, 0xa00, BIT15, 0x0); */
+
+		/* Enable Antenna weighting */
+		odm_set_bb_reg(dm, 0x1904, BIT(16), 0x1);
+		odm_set_bb_reg(dm, 0x800, BIT(28), 0x1);
+		odm_set_bb_reg(dm, 0x850, BIT(23), 0x1);
+	}
+
+	/* Update TXRX antenna status for PHYDM */
+	dm->tx_ant_status = (tx_path & 0x3);
+	dm->rx_ant_status = (rx_path & 0x3);
+
+	/* MP driver need to support path-B TX\RX */
+
+	while (1) {
+		counter++;
+		rf_reg_status =
+			rf_reg_status & config_phydm_write_rf_reg_8822b(
+						dm, ODM_RF_PATH_A, 0xef,
+						RFREGOFFSETMASK, 0x80000);
+		rf_reg_status =
+			rf_reg_status & config_phydm_write_rf_reg_8822b(
+						dm, ODM_RF_PATH_A, 0x33,
+						RFREGOFFSETMASK, 0x00001);
+
+		ODM_delay_us(2);
+		rf_reg33 = config_phydm_read_rf_reg_8822b(
+			dm, ODM_RF_PATH_A, 0x33, RFREGOFFSETMASK);
+
+		if ((rf_reg33 == 0x00001) &&
+		    (config_phydm_read_rf_check_8822b(rf_reg33)))
+			break;
+		else if (counter == 100) {
+			ODM_RT_TRACE(
+				dm, ODM_PHY_CONFIG,
+				"%s(): Fail to set TRx mode setting, because writing RF mode table is fail\n",
+				__func__);
+			return false;
+		}
+	}
+
+	if ((dm->mp_mode) || *dm->antenna_test || (dm->normal_rx_path)) {
+		/* 0xef 0x80000  0x33 0x00001  0x3e 0x00034  0x3f 0x4080e
+		 * 0xef 0x00000    suggested by Lucas
+		 */
+		rf_reg_status =
+			rf_reg_status & config_phydm_write_rf_reg_8822b(
+						dm, ODM_RF_PATH_A, 0xef,
+						RFREGOFFSETMASK, 0x80000);
+		rf_reg_status =
+			rf_reg_status & config_phydm_write_rf_reg_8822b(
+						dm, ODM_RF_PATH_A, 0x33,
+						RFREGOFFSETMASK, 0x00001);
+		rf_reg_status =
+			rf_reg_status & config_phydm_write_rf_reg_8822b(
+						dm, ODM_RF_PATH_A, 0x3e,
+						RFREGOFFSETMASK, 0x00034);
+		rf_reg_status =
+			rf_reg_status & config_phydm_write_rf_reg_8822b(
+						dm, ODM_RF_PATH_A, 0x3f,
+						RFREGOFFSETMASK, 0x4080e);
+		rf_reg_status =
+			rf_reg_status & config_phydm_write_rf_reg_8822b(
+						dm, ODM_RF_PATH_A, 0xef,
+						RFREGOFFSETMASK, 0x00000);
+		ODM_RT_TRACE(
+			dm, ODM_PHY_CONFIG,
+			"%s(): MP mode or Antenna test mode!! support path-B TX and RX\n",
+			__func__);
+	} else {
+		/* 0xef 0x80000  0x33 0x00001  0x3e 0x00034  0x3f 0x4080c
+		 * 0xef 0x00000
+		 */
+		rf_reg_status =
+			rf_reg_status & config_phydm_write_rf_reg_8822b(
+						dm, ODM_RF_PATH_A, 0xef,
+						RFREGOFFSETMASK, 0x80000);
+		rf_reg_status =
+			rf_reg_status & config_phydm_write_rf_reg_8822b(
+						dm, ODM_RF_PATH_A, 0x33,
+						RFREGOFFSETMASK, 0x00001);
+		rf_reg_status =
+			rf_reg_status & config_phydm_write_rf_reg_8822b(
+						dm, ODM_RF_PATH_A, 0x3e,
+						RFREGOFFSETMASK, 0x00034);
+		rf_reg_status =
+			rf_reg_status & config_phydm_write_rf_reg_8822b(
+						dm, ODM_RF_PATH_A, 0x3f,
+						RFREGOFFSETMASK, 0x4080c);
+		rf_reg_status =
+			rf_reg_status & config_phydm_write_rf_reg_8822b(
+						dm, ODM_RF_PATH_A, 0xef,
+						RFREGOFFSETMASK, 0x00000);
+		ODM_RT_TRACE(
+			dm, ODM_PHY_CONFIG,
+			"%s(): Normal mode!! Do not support path-B TX and RX\n",
+			__func__);
+	}
+
+	rf_reg_status = rf_reg_status & config_phydm_write_rf_reg_8822b(
+						dm, ODM_RF_PATH_A, 0xef,
+						RFREGOFFSETMASK, 0x00000);
+
+	if (!rf_reg_status) {
+		ODM_RT_TRACE(
+			dm, ODM_PHY_CONFIG,
+			"%s(): Fail to set TRx mode setting (TX: 0x%x, RX: 0x%x), because writing RF register is fail\n",
+			__func__, tx_path, rx_path);
+		return false;
+	}
+
+	/* Toggle IGI to let RF enter RX mode,
+	 * because BB doesn't send 3-wire command when RX path is enable
+	 */
+	IGI = (u8)odm_get_bb_reg(dm, ODM_REG(IGI_A, dm), ODM_BIT(IGI, dm));
+	odm_write_dig(dm, IGI - 2);
+	odm_write_dig(dm, IGI);
+
+	/* Modify CCA parameters */
+	phydm_ccapar_by_rxpath_8822b(dm);
+	phydm_ccapar_by_rfe_8822b(dm);
+	phydm_rfe_8822b(dm, central_ch_8822b);
+
+	ODM_RT_TRACE(
+		dm, ODM_PHY_CONFIG,
+		"%s(): Success to set TRx mode setting (TX: 0x%x, RX: 0x%x)\n",
+		__func__, tx_path, rx_path);
+	return true;
+}
+
+bool config_phydm_parameter_init(struct phy_dm_struct *dm,
+				 enum odm_parameter_init type)
+{
+	if (type == ODM_PRE_SETTING) {
+		odm_set_bb_reg(dm, 0x808, (BIT(28) | BIT(29)), 0x0);
+		ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+			     "%s(): Pre setting: disable OFDM and CCK block\n",
+			     __func__);
+	} else if (type == ODM_POST_SETTING) {
+		odm_set_bb_reg(dm, 0x808, (BIT(28) | BIT(29)), 0x3);
+		ODM_RT_TRACE(dm, ODM_PHY_CONFIG,
+			     "%s(): Post setting: enable OFDM and CCK block\n",
+			     __func__);
+		reg82c_8822b = odm_get_bb_reg(dm, 0x82c, MASKDWORD);
+		reg838_8822b = odm_get_bb_reg(dm, 0x838, MASKDWORD);
+		reg830_8822b = odm_get_bb_reg(dm, 0x830, MASKDWORD);
+		reg83c_8822b = odm_get_bb_reg(dm, 0x83c, MASKDWORD);
+		rega20_8822b = odm_get_bb_reg(dm, 0xa20, MASKDWORD);
+		rega24_8822b = odm_get_bb_reg(dm, 0xa24, MASKDWORD);
+		rega28_8822b = odm_get_bb_reg(dm, 0xa28, MASKDWORD);
+	} else {
+		ODM_RT_TRACE(dm, ODM_PHY_CONFIG, "%s(): Wrong type!!\n",
+			     __func__);
+		return false;
+	}
+
+	return true;
+}
+
+/* ======================================================================== */
diff --git a/drivers/staging/rtlwifi/phydm/rtl8822b/phydm_hal_api8822b.h b/drivers/staging/rtlwifi/phydm/rtl8822b/phydm_hal_api8822b.h
new file mode 100644
index 000000000000..279ef06298e2
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/rtl8822b/phydm_hal_api8822b.h
@@ -0,0 +1,84 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef __INC_PHYDM_API_H_8822B__
+#define __INC_PHYDM_API_H_8822B__
+
+/*2016.08.01 (HW user guide version: R27, SW user guide version: R05,
+ *            Modification: R31)
+ */
+#define PHY_CONFIG_VERSION_8822B "27.5.31"
+
+#define INVALID_RF_DATA 0xffffffff
+#define INVALID_TXAGC_DATA 0xff
+
+#define config_phydm_read_rf_check_8822b(data) (data != INVALID_RF_DATA)
+#define config_phydm_read_txagc_check_8822b(data) (data != INVALID_TXAGC_DATA)
+
+u32 config_phydm_read_rf_reg_8822b(struct phy_dm_struct *dm,
+				   enum odm_rf_radio_path rf_path, u32 reg_addr,
+				   u32 bit_mask);
+
+bool config_phydm_write_rf_reg_8822b(struct phy_dm_struct *dm,
+				     enum odm_rf_radio_path rf_path,
+				     u32 reg_addr, u32 bit_mask, u32 data);
+
+bool config_phydm_write_txagc_8822b(struct phy_dm_struct *dm, u32 power_index,
+				    enum odm_rf_radio_path path, u8 hw_rate);
+
+u8 config_phydm_read_txagc_8822b(struct phy_dm_struct *dm,
+				 enum odm_rf_radio_path path, u8 hw_rate);
+
+bool config_phydm_switch_band_8822b(struct phy_dm_struct *dm, u8 central_ch);
+
+bool config_phydm_switch_channel_8822b(struct phy_dm_struct *dm, u8 central_ch);
+
+bool config_phydm_switch_bandwidth_8822b(struct phy_dm_struct *dm,
+					 u8 primary_ch_idx,
+					 enum odm_bw bandwidth);
+
+bool config_phydm_switch_channel_bw_8822b(struct phy_dm_struct *dm,
+					  u8 central_ch, u8 primary_ch_idx,
+					  enum odm_bw bandwidth);
+
+bool config_phydm_trx_mode_8822b(struct phy_dm_struct *dm,
+				 enum odm_rf_path tx_path,
+				 enum odm_rf_path rx_path, bool is_tx2_path);
+
+bool config_phydm_parameter_init(struct phy_dm_struct *dm,
+				 enum odm_parameter_init type);
+
+/* ======================================================================== */
+/* These following functions can be used for PHY DM only*/
+
+bool phydm_write_txagc_1byte_8822b(struct phy_dm_struct *dm, u32 power_index,
+				   enum odm_rf_radio_path path, u8 hw_rate);
+
+void phydm_init_hw_info_by_rfe_type_8822b(struct phy_dm_struct *dm);
+
+s32 phydm_get_condition_number_8822B(struct phy_dm_struct *dm);
+
+/* ======================================================================== */
+
+#endif /*  __INC_PHYDM_API_H_8822B__ */
diff --git a/drivers/staging/rtlwifi/phydm/rtl8822b/phydm_iqk_8822b.c b/drivers/staging/rtlwifi/phydm/rtl8822b/phydm_iqk_8822b.c
new file mode 100644
index 000000000000..d320311213cc
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/rtl8822b/phydm_iqk_8822b.c
@@ -0,0 +1,1410 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#include "../mp_precomp.h"
+#include "../phydm_precomp.h"
+
+/*---------------------------Define Local Constant---------------------------*/
+
+static bool _iqk_rx_iqk_by_path_8822b(void *, u8);
+
+static inline void phydm_set_iqk_info(struct phy_dm_struct *dm,
+				      struct dm_iqk_info *iqk_info, u8 status)
+{
+	bool KFAIL = true;
+
+	while (1) {
+		KFAIL = _iqk_rx_iqk_by_path_8822b(dm, ODM_RF_PATH_A);
+		if (status == 0)
+			ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+				     "[IQK]S0RXK KFail = 0x%x\n", KFAIL);
+		else if (status == 1)
+			ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+				     "[IQK]S1RXK KFail = 0x%x\n", KFAIL);
+		if (iqk_info->rxiqk_step == 5) {
+			dm->rf_calibrate_info.iqk_step++;
+			iqk_info->rxiqk_step = 1;
+			if (KFAIL && status == 0)
+				ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+					     "[IQK]S0RXK fail code: %d!!!\n",
+					     iqk_info->rxiqk_fail_code
+						     [0][ODM_RF_PATH_A]);
+			else if (KFAIL && status == 1)
+				ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+					     "[IQK]S1RXK fail code: %d!!!\n",
+					     iqk_info->rxiqk_fail_code
+						     [0][ODM_RF_PATH_A]);
+			break;
+		}
+	}
+
+	iqk_info->kcount++;
+}
+
+static inline void phydm_init_iqk_information(struct dm_iqk_info *iqk_info)
+{
+	u8 i, j, k, m;
+
+	for (i = 0; i < 2; i++) {
+		iqk_info->iqk_channel[i] = 0x0;
+
+		for (j = 0; j < SS_8822B; j++) {
+			iqk_info->lok_idac[i][j] = 0x0;
+			iqk_info->rxiqk_agc[i][j] = 0x0;
+			iqk_info->bypass_iqk[i][j] = 0x0;
+
+			for (k = 0; k < 2; k++) {
+				iqk_info->iqk_fail_report[i][j][k] = true;
+				for (m = 0; m < 8; m++) {
+					iqk_info->iqk_cfir_real[i][j][k][m] =
+						0x0;
+					iqk_info->iqk_cfir_imag[i][j][k][m] =
+						0x0;
+				}
+			}
+
+			for (k = 0; k < 3; k++)
+				iqk_info->retry_count[i][j][k] = 0x0;
+		}
+	}
+}
+
+static inline void phydm_backup_iqk_information(struct dm_iqk_info *iqk_info)
+{
+	u8 i, j, k;
+
+	iqk_info->iqk_channel[1] = iqk_info->iqk_channel[0];
+	for (i = 0; i < 2; i++) {
+		iqk_info->lok_idac[1][i] = iqk_info->lok_idac[0][i];
+		iqk_info->rxiqk_agc[1][i] = iqk_info->rxiqk_agc[0][i];
+		iqk_info->bypass_iqk[1][i] = iqk_info->bypass_iqk[0][i];
+		iqk_info->rxiqk_fail_code[1][i] =
+			iqk_info->rxiqk_fail_code[0][i];
+		for (j = 0; j < 2; j++) {
+			iqk_info->iqk_fail_report[1][i][j] =
+				iqk_info->iqk_fail_report[0][i][j];
+			for (k = 0; k < 8; k++) {
+				iqk_info->iqk_cfir_real[1][i][j][k] =
+					iqk_info->iqk_cfir_real[0][i][j][k];
+				iqk_info->iqk_cfir_imag[1][i][j][k] =
+					iqk_info->iqk_cfir_imag[0][i][j][k];
+			}
+		}
+	}
+
+	for (i = 0; i < 4; i++) {
+		iqk_info->rxiqk_fail_code[0][i] = 0x0;
+		iqk_info->rxiqk_agc[0][i] = 0x0;
+		for (j = 0; j < 2; j++) {
+			iqk_info->iqk_fail_report[0][i][j] = true;
+			iqk_info->gs_retry_count[0][i][j] = 0x0;
+		}
+		for (j = 0; j < 3; j++)
+			iqk_info->retry_count[0][i][j] = 0x0;
+	}
+}
+
+static inline void phydm_set_iqk_cfir(struct phy_dm_struct *dm,
+				      struct dm_iqk_info *iqk_info, u8 path)
+{
+	u8 idx, i;
+	u32 tmp;
+
+	for (idx = 0; idx < 2; idx++) {
+		odm_set_bb_reg(dm, 0x1b00, MASKDWORD, 0xf8000008 | path << 1);
+
+		if (idx == 0)
+			odm_set_bb_reg(dm, 0x1b0c, BIT(13) | BIT(12), 0x3);
+		else
+			odm_set_bb_reg(dm, 0x1b0c, BIT(13) | BIT(12), 0x1);
+
+		odm_set_bb_reg(dm, 0x1bd4,
+			       BIT(20) | BIT(19) | BIT(18) | BIT(17) | BIT(16),
+			       0x10);
+
+		for (i = 0; i < 8; i++) {
+			odm_set_bb_reg(dm, 0x1bd8, MASKDWORD,
+				       0xe0000001 + (i * 4));
+			tmp = odm_get_bb_reg(dm, 0x1bfc, MASKDWORD);
+			iqk_info->iqk_cfir_real[0][path][idx][i] =
+				(tmp & 0x0fff0000) >> 16;
+			iqk_info->iqk_cfir_imag[0][path][idx][i] = tmp & 0xfff;
+		}
+	}
+}
+
+static inline void phydm_get_read_counter(struct phy_dm_struct *dm)
+{
+	u32 counter = 0x0;
+
+	while (1) {
+		if (((odm_read_4byte(dm, 0x1bf0) >> 24) == 0x7f) ||
+		    (counter > 300))
+			break;
+
+		counter++;
+		ODM_delay_ms(1);
+	}
+
+	ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION, "[IQK]counter = %d\n", counter);
+}
+
+/*---------------------------Define Local Constant---------------------------*/
+
+void do_iqk_8822b(void *dm_void, u8 delta_thermal_index, u8 thermal_value,
+		  u8 threshold)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	odm_reset_iqk_result(dm);
+
+	dm->rf_calibrate_info.thermal_value_iqk = thermal_value;
+
+	phy_iq_calibrate_8822b(dm, true);
+}
+
+static void _iqk_fill_iqk_report_8822b(void *dm_void, u8 channel)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct dm_iqk_info *iqk_info = &dm->IQK_info;
+	u32 tmp1 = 0x0, tmp2 = 0x0, tmp3 = 0x0;
+	u8 i;
+
+	for (i = 0; i < SS_8822B; i++) {
+		tmp1 = tmp1 +
+		       ((iqk_info->iqk_fail_report[channel][i][TX_IQK] & 0x1)
+			<< i);
+		tmp2 = tmp2 +
+		       ((iqk_info->iqk_fail_report[channel][i][RX_IQK] & 0x1)
+			<< (i + 4));
+		tmp3 = tmp3 + ((iqk_info->rxiqk_fail_code[channel][i] & 0x3)
+			       << (i * 2 + 8));
+	}
+	odm_write_4byte(dm, 0x1b00, 0xf8000008);
+	odm_set_bb_reg(dm, 0x1bf0, 0x0000ffff, tmp1 | tmp2 | tmp3);
+
+	for (i = 0; i < 2; i++)
+		odm_write_4byte(
+			dm, 0x1be8 + (i * 4),
+			(iqk_info->rxiqk_agc[channel][(i * 2) + 1] << 16) |
+				iqk_info->rxiqk_agc[channel][i * 2]);
+}
+
+static void _iqk_backup_mac_bb_8822b(struct phy_dm_struct *dm, u32 *MAC_backup,
+				     u32 *BB_backup, u32 *backup_mac_reg,
+				     u32 *backup_bb_reg)
+{
+	u32 i;
+
+	for (i = 0; i < MAC_REG_NUM_8822B; i++)
+		MAC_backup[i] = odm_read_4byte(dm, backup_mac_reg[i]);
+
+	for (i = 0; i < BB_REG_NUM_8822B; i++)
+		BB_backup[i] = odm_read_4byte(dm, backup_bb_reg[i]);
+}
+
+static void _iqk_backup_rf_8822b(struct phy_dm_struct *dm, u32 RF_backup[][2],
+				 u32 *backup_rf_reg)
+{
+	u32 i;
+
+	for (i = 0; i < RF_REG_NUM_8822B; i++) {
+		RF_backup[i][ODM_RF_PATH_A] = odm_get_rf_reg(
+			dm, ODM_RF_PATH_A, backup_rf_reg[i], RFREGOFFSETMASK);
+		RF_backup[i][ODM_RF_PATH_B] = odm_get_rf_reg(
+			dm, ODM_RF_PATH_B, backup_rf_reg[i], RFREGOFFSETMASK);
+	}
+}
+
+static void _iqk_agc_bnd_int_8822b(struct phy_dm_struct *dm)
+{
+	/*initialize RX AGC bnd, it must do after bbreset*/
+	odm_write_4byte(dm, 0x1b00, 0xf8000008);
+	odm_write_4byte(dm, 0x1b00, 0xf80a7008);
+	odm_write_4byte(dm, 0x1b00, 0xf8015008);
+	odm_write_4byte(dm, 0x1b00, 0xf8000008);
+}
+
+static void _iqk_bb_reset_8822b(struct phy_dm_struct *dm)
+{
+	bool cca_ing = false;
+	u32 count = 0;
+
+	odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x0, RFREGOFFSETMASK, 0x10000);
+	odm_set_rf_reg(dm, ODM_RF_PATH_B, 0x0, RFREGOFFSETMASK, 0x10000);
+
+	while (1) {
+		odm_write_4byte(dm, 0x8fc, 0x0);
+		odm_set_bb_reg(dm, 0x198c, 0x7, 0x7);
+		cca_ing = (bool)odm_get_bb_reg(dm, 0xfa0, BIT(3));
+
+		if (count > 30)
+			cca_ing = false;
+
+		if (cca_ing) {
+			ODM_delay_ms(1);
+			count++;
+		} else {
+			odm_write_1byte(dm, 0x808, 0x0); /*RX ant off*/
+			odm_set_bb_reg(dm, 0xa04,
+				       BIT(27) | BIT(26) | BIT(25) | BIT(24),
+				       0x0); /*CCK RX path off*/
+
+			/*BBreset*/
+			odm_set_bb_reg(dm, 0x0, BIT(16), 0x0);
+			odm_set_bb_reg(dm, 0x0, BIT(16), 0x1);
+
+			if (odm_get_bb_reg(dm, 0x660, BIT(16)))
+				odm_write_4byte(dm, 0x6b4, 0x89000006);
+			break;
+		}
+	}
+}
+
+static void _iqk_afe_setting_8822b(struct phy_dm_struct *dm, bool do_iqk)
+{
+	if (do_iqk) {
+		odm_write_4byte(dm, 0xc60, 0x50000000);
+		odm_write_4byte(dm, 0xc60, 0x70070040);
+		odm_write_4byte(dm, 0xe60, 0x50000000);
+		odm_write_4byte(dm, 0xe60, 0x70070040);
+
+		odm_write_4byte(dm, 0xc58, 0xd8000402);
+		odm_write_4byte(dm, 0xc5c, 0xd1000120);
+		odm_write_4byte(dm, 0xc6c, 0x00000a15);
+		odm_write_4byte(dm, 0xe58, 0xd8000402);
+		odm_write_4byte(dm, 0xe5c, 0xd1000120);
+		odm_write_4byte(dm, 0xe6c, 0x00000a15);
+		_iqk_bb_reset_8822b(dm);
+	} else {
+		odm_write_4byte(dm, 0xc60, 0x50000000);
+		odm_write_4byte(dm, 0xc60, 0x70038040);
+		odm_write_4byte(dm, 0xe60, 0x50000000);
+		odm_write_4byte(dm, 0xe60, 0x70038040);
+
+		odm_write_4byte(dm, 0xc58, 0xd8020402);
+		odm_write_4byte(dm, 0xc5c, 0xde000120);
+		odm_write_4byte(dm, 0xc6c, 0x0000122a);
+		odm_write_4byte(dm, 0xe58, 0xd8020402);
+		odm_write_4byte(dm, 0xe5c, 0xde000120);
+		odm_write_4byte(dm, 0xe6c, 0x0000122a);
+	}
+}
+
+static void _iqk_restore_mac_bb_8822b(struct phy_dm_struct *dm, u32 *MAC_backup,
+				      u32 *BB_backup, u32 *backup_mac_reg,
+				      u32 *backup_bb_reg)
+{
+	u32 i;
+
+	for (i = 0; i < MAC_REG_NUM_8822B; i++)
+		odm_write_4byte(dm, backup_mac_reg[i], MAC_backup[i]);
+	for (i = 0; i < BB_REG_NUM_8822B; i++)
+		odm_write_4byte(dm, backup_bb_reg[i], BB_backup[i]);
+}
+
+static void _iqk_restore_rf_8822b(struct phy_dm_struct *dm, u32 *backup_rf_reg,
+				  u32 RF_backup[][2])
+{
+	u32 i;
+
+	odm_set_rf_reg(dm, ODM_RF_PATH_A, 0xef, RFREGOFFSETMASK, 0x0);
+	odm_set_rf_reg(dm, ODM_RF_PATH_B, 0xef, RFREGOFFSETMASK, 0x0);
+	/*0xdf[4]=0*/
+	odm_set_rf_reg(dm, ODM_RF_PATH_A, 0xdf, RFREGOFFSETMASK,
+		       RF_backup[0][ODM_RF_PATH_A] & (~BIT(4)));
+	odm_set_rf_reg(dm, ODM_RF_PATH_B, 0xdf, RFREGOFFSETMASK,
+		       RF_backup[0][ODM_RF_PATH_B] & (~BIT(4)));
+
+	for (i = 1; i < RF_REG_NUM_8822B; i++) {
+		odm_set_rf_reg(dm, ODM_RF_PATH_A, backup_rf_reg[i],
+			       RFREGOFFSETMASK, RF_backup[i][ODM_RF_PATH_A]);
+		odm_set_rf_reg(dm, ODM_RF_PATH_B, backup_rf_reg[i],
+			       RFREGOFFSETMASK, RF_backup[i][ODM_RF_PATH_B]);
+	}
+}
+
+static void _iqk_backup_iqk_8822b(struct phy_dm_struct *dm, u8 step)
+{
+	struct dm_iqk_info *iqk_info = &dm->IQK_info;
+	u8 path;
+	u16 iqk_apply[2] = {0xc94, 0xe94};
+
+	if (step == 0x0) {
+		phydm_backup_iqk_information(iqk_info);
+	} else {
+		iqk_info->iqk_channel[0] = iqk_info->rf_reg18;
+		for (path = 0; path < 2; path++) {
+			iqk_info->lok_idac[0][path] =
+				odm_get_rf_reg(dm, (enum odm_rf_radio_path)path,
+					       0x58, RFREGOFFSETMASK);
+			iqk_info->bypass_iqk[0][path] =
+				odm_get_bb_reg(dm, iqk_apply[path], MASKDWORD);
+
+			phydm_set_iqk_cfir(dm, iqk_info, path);
+			odm_set_bb_reg(dm, 0x1bd8, MASKDWORD, 0x0);
+			odm_set_bb_reg(dm, 0x1b0c, BIT(13) | BIT(12), 0x0);
+		}
+	}
+}
+
+static void _iqk_reload_iqk_setting_8822b(
+	struct phy_dm_struct *dm, u8 channel,
+	u8 reload_idx /*1: reload TX, 2: reload LO, TX, RX*/
+	)
+{
+	struct dm_iqk_info *iqk_info = &dm->IQK_info;
+	u8 i, path, idx;
+	u16 iqk_apply[2] = {0xc94, 0xe94};
+
+	for (path = 0; path < 2; path++) {
+		if (reload_idx == 2) {
+			odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0xdf,
+				       BIT(4), 0x1);
+			odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x58,
+				       RFREGOFFSETMASK,
+				       iqk_info->lok_idac[channel][path]);
+		}
+
+		for (idx = 0; idx < reload_idx; idx++) {
+			odm_set_bb_reg(dm, 0x1b00, MASKDWORD,
+				       0xf8000008 | path << 1);
+			odm_set_bb_reg(dm, 0x1b2c, MASKDWORD, 0x7);
+			odm_set_bb_reg(dm, 0x1b38, MASKDWORD, 0x20000000);
+			odm_set_bb_reg(dm, 0x1b3c, MASKDWORD, 0x20000000);
+			odm_set_bb_reg(dm, 0x1bcc, MASKDWORD, 0x00000000);
+
+			if (idx == 0)
+				odm_set_bb_reg(dm, 0x1b0c, BIT(13) | BIT(12),
+					       0x3);
+			else
+				odm_set_bb_reg(dm, 0x1b0c, BIT(13) | BIT(12),
+					       0x1);
+
+			odm_set_bb_reg(dm, 0x1bd4, BIT(20) | BIT(19) | BIT(18) |
+							   BIT(17) | BIT(16),
+				       0x10);
+
+			for (i = 0; i < 8; i++) {
+				odm_write_4byte(
+					dm, 0x1bd8,
+					((0xc0000000 >> idx) + 0x3) + (i * 4) +
+						(iqk_info->iqk_cfir_real
+							 [channel][path][idx][i]
+						 << 9));
+				odm_write_4byte(
+					dm, 0x1bd8,
+					((0xc0000000 >> idx) + 0x1) + (i * 4) +
+						(iqk_info->iqk_cfir_imag
+							 [channel][path][idx][i]
+						 << 9));
+			}
+		}
+		odm_set_bb_reg(dm, iqk_apply[path], MASKDWORD,
+			       iqk_info->bypass_iqk[channel][path]);
+
+		odm_set_bb_reg(dm, 0x1bd8, MASKDWORD, 0x0);
+		odm_set_bb_reg(dm, 0x1b0c, BIT(13) | BIT(12), 0x0);
+	}
+}
+
+static bool _iqk_reload_iqk_8822b(struct phy_dm_struct *dm, bool reset)
+{
+	struct dm_iqk_info *iqk_info = &dm->IQK_info;
+	u8 i;
+	bool reload = false;
+
+	if (reset) {
+		for (i = 0; i < 2; i++)
+			iqk_info->iqk_channel[i] = 0x0;
+	} else {
+		iqk_info->rf_reg18 = odm_get_rf_reg(dm, ODM_RF_PATH_A, 0x18,
+						    RFREGOFFSETMASK);
+
+		for (i = 0; i < 2; i++) {
+			if (iqk_info->rf_reg18 == iqk_info->iqk_channel[i]) {
+				_iqk_reload_iqk_setting_8822b(dm, i, 2);
+				_iqk_fill_iqk_report_8822b(dm, i);
+				ODM_RT_TRACE(
+					dm, ODM_COMP_CALIBRATION,
+					"[IQK]reload IQK result before!!!!\n");
+				reload = true;
+			}
+		}
+	}
+	return reload;
+}
+
+static void _iqk_rfe_setting_8822b(struct phy_dm_struct *dm, bool ext_pa_on)
+{
+	if (ext_pa_on) {
+		/*RFE setting*/
+		odm_write_4byte(dm, 0xcb0, 0x77777777);
+		odm_write_4byte(dm, 0xcb4, 0x00007777);
+		odm_write_4byte(dm, 0xcbc, 0x0000083B);
+		odm_write_4byte(dm, 0xeb0, 0x77777777);
+		odm_write_4byte(dm, 0xeb4, 0x00007777);
+		odm_write_4byte(dm, 0xebc, 0x0000083B);
+		ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+			     "[IQK]external PA on!!!!\n");
+	} else {
+		/*RFE setting*/
+		odm_write_4byte(dm, 0xcb0, 0x77777777);
+		odm_write_4byte(dm, 0xcb4, 0x00007777);
+		odm_write_4byte(dm, 0xcbc, 0x00000100);
+		odm_write_4byte(dm, 0xeb0, 0x77777777);
+		odm_write_4byte(dm, 0xeb4, 0x00007777);
+		odm_write_4byte(dm, 0xebc, 0x00000100);
+	}
+}
+
+static void _iqk_rf_setting_8822b(struct phy_dm_struct *dm)
+{
+	u8 path;
+	u32 tmp;
+
+	odm_write_4byte(dm, 0x1b00, 0xf8000008);
+	odm_write_4byte(dm, 0x1bb8, 0x00000000);
+
+	for (path = 0; path < 2; path++) {
+		/*0xdf:B11 = 1,B4 = 0, B1 = 1*/
+		tmp = odm_get_rf_reg(dm, (enum odm_rf_radio_path)path, 0xdf,
+				     RFREGOFFSETMASK);
+		tmp = (tmp & (~BIT(4))) | BIT(1) | BIT(11);
+		odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0xdf,
+			       RFREGOFFSETMASK, tmp);
+
+		/*release 0x56 TXBB*/
+		odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x65,
+			       RFREGOFFSETMASK, 0x09000);
+
+		if (*dm->band_type == ODM_BAND_5G) {
+			odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0xef,
+				       BIT(19), 0x1);
+			odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x33,
+				       RFREGOFFSETMASK, 0x00026);
+			odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x3e,
+				       RFREGOFFSETMASK, 0x00037);
+			odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x3f,
+				       RFREGOFFSETMASK, 0xdefce);
+			odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0xef,
+				       BIT(19), 0x0);
+		} else {
+			odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0xef,
+				       BIT(19), 0x1);
+			odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x33,
+				       RFREGOFFSETMASK, 0x00026);
+			odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x3e,
+				       RFREGOFFSETMASK, 0x00037);
+			odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x3f,
+				       RFREGOFFSETMASK, 0x5efce);
+			odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0xef,
+				       BIT(19), 0x0);
+		}
+	}
+}
+
+static void _iqk_configure_macbb_8822b(struct phy_dm_struct *dm)
+{
+	/*MACBB register setting*/
+	odm_write_1byte(dm, 0x522, 0x7f);
+	odm_set_bb_reg(dm, 0x550, BIT(11) | BIT(3), 0x0);
+	odm_set_bb_reg(dm, 0x90c, BIT(15),
+		       0x1); /*0x90c[15]=1: dac_buf reset selection*/
+	odm_set_bb_reg(dm, 0x9a4, BIT(31),
+		       0x0); /*0x9a4[31]=0: Select da clock*/
+	/*0xc94[0]=1, 0xe94[0]=1: let tx through iqk*/
+	odm_set_bb_reg(dm, 0xc94, BIT(0), 0x1);
+	odm_set_bb_reg(dm, 0xe94, BIT(0), 0x1);
+	/* 3-wire off*/
+	odm_write_4byte(dm, 0xc00, 0x00000004);
+	odm_write_4byte(dm, 0xe00, 0x00000004);
+}
+
+static void _iqk_lok_setting_8822b(struct phy_dm_struct *dm, u8 path)
+{
+	odm_write_4byte(dm, 0x1b00, 0xf8000008 | path << 1);
+	odm_write_4byte(dm, 0x1bcc, 0x9);
+	odm_write_1byte(dm, 0x1b23, 0x00);
+
+	switch (*dm->band_type) {
+	case ODM_BAND_2_4G:
+		odm_write_1byte(dm, 0x1b2b, 0x00);
+		odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x56,
+			       RFREGOFFSETMASK, 0x50df2);
+		odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x8f,
+			       RFREGOFFSETMASK, 0xadc00);
+		/* WE_LUT_TX_LOK*/
+		odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0xef, BIT(4),
+			       0x1);
+		odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x33,
+			       BIT(1) | BIT(0), 0x0);
+		break;
+	case ODM_BAND_5G:
+		odm_write_1byte(dm, 0x1b2b, 0x80);
+		odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x56,
+			       RFREGOFFSETMASK, 0x5086c);
+		odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x8f,
+			       RFREGOFFSETMASK, 0xa9c00);
+		/* WE_LUT_TX_LOK*/
+		odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0xef, BIT(4),
+			       0x1);
+		odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x33,
+			       BIT(1) | BIT(0), 0x1);
+		break;
+	}
+}
+
+static void _iqk_txk_setting_8822b(struct phy_dm_struct *dm, u8 path)
+{
+	odm_write_4byte(dm, 0x1b00, 0xf8000008 | path << 1);
+	odm_write_4byte(dm, 0x1bcc, 0x9);
+	odm_write_4byte(dm, 0x1b20, 0x01440008);
+
+	if (path == 0x0)
+		odm_write_4byte(dm, 0x1b00, 0xf800000a);
+	else
+		odm_write_4byte(dm, 0x1b00, 0xf8000008);
+	odm_write_4byte(dm, 0x1bcc, 0x3f);
+
+	switch (*dm->band_type) {
+	case ODM_BAND_2_4G:
+		odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x56,
+			       RFREGOFFSETMASK, 0x50df2);
+		odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x8f,
+			       RFREGOFFSETMASK, 0xadc00);
+		odm_write_1byte(dm, 0x1b2b, 0x00);
+		break;
+	case ODM_BAND_5G:
+		odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x56,
+			       RFREGOFFSETMASK, 0x500ef);
+		odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x8f,
+			       RFREGOFFSETMASK, 0xa9c00);
+		odm_write_1byte(dm, 0x1b2b, 0x80);
+		break;
+	}
+}
+
+static void _iqk_rxk1_setting_8822b(struct phy_dm_struct *dm, u8 path)
+{
+	odm_write_4byte(dm, 0x1b00, 0xf8000008 | path << 1);
+
+	switch (*dm->band_type) {
+	case ODM_BAND_2_4G:
+		odm_write_1byte(dm, 0x1bcc, 0x9);
+		odm_write_1byte(dm, 0x1b2b, 0x00);
+		odm_write_4byte(dm, 0x1b20, 0x01450008);
+		odm_write_4byte(dm, 0x1b24, 0x01460c88);
+		odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x56,
+			       RFREGOFFSETMASK, 0x510e0);
+		odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x8f,
+			       RFREGOFFSETMASK, 0xacc00);
+		break;
+	case ODM_BAND_5G:
+		odm_write_1byte(dm, 0x1bcc, 0x09);
+		odm_write_1byte(dm, 0x1b2b, 0x80);
+		odm_write_4byte(dm, 0x1b20, 0x00850008);
+		odm_write_4byte(dm, 0x1b24, 0x00460048);
+		odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x56,
+			       RFREGOFFSETMASK, 0x510e0);
+		odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x8f,
+			       RFREGOFFSETMASK, 0xadc00);
+		break;
+	}
+}
+
+static void _iqk_rxk2_setting_8822b(struct phy_dm_struct *dm, u8 path,
+				    bool is_gs)
+{
+	struct dm_iqk_info *iqk_info = &dm->IQK_info;
+
+	odm_write_4byte(dm, 0x1b00, 0xf8000008 | path << 1);
+
+	switch (*dm->band_type) {
+	case ODM_BAND_2_4G:
+		if (is_gs)
+			iqk_info->tmp1bcc = 0x12;
+		odm_write_1byte(dm, 0x1bcc, iqk_info->tmp1bcc);
+		odm_write_1byte(dm, 0x1b2b, 0x00);
+		odm_write_4byte(dm, 0x1b20, 0x01450008);
+		odm_write_4byte(dm, 0x1b24, 0x01460848);
+		odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x56,
+			       RFREGOFFSETMASK, 0x510e0);
+		odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x8f,
+			       RFREGOFFSETMASK, 0xa9c00);
+		break;
+	case ODM_BAND_5G:
+		if (is_gs) {
+			if (path == ODM_RF_PATH_A)
+				iqk_info->tmp1bcc = 0x12;
+			else
+				iqk_info->tmp1bcc = 0x09;
+		}
+		odm_write_1byte(dm, 0x1bcc, iqk_info->tmp1bcc);
+		odm_write_1byte(dm, 0x1b2b, 0x80);
+		odm_write_4byte(dm, 0x1b20, 0x00850008);
+		odm_write_4byte(dm, 0x1b24, 0x00460848);
+		odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x56,
+			       RFREGOFFSETMASK, 0x51060);
+		odm_set_rf_reg(dm, (enum odm_rf_radio_path)path, 0x8f,
+			       RFREGOFFSETMASK, 0xa9c00);
+		break;
+	}
+}
+
+static bool _iqk_check_cal_8822b(struct phy_dm_struct *dm, u32 IQK_CMD)
+{
+	bool notready = true, fail = true;
+	u32 delay_count = 0x0;
+
+	while (notready) {
+		if (odm_read_4byte(dm, 0x1b00) == (IQK_CMD & 0xffffff0f)) {
+			fail = (bool)odm_get_bb_reg(dm, 0x1b08, BIT(26));
+			notready = false;
+		} else {
+			ODM_delay_ms(1);
+			delay_count++;
+		}
+
+		if (delay_count >= 50) {
+			fail = true;
+			ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+				     "[IQK]IQK timeout!!!\n");
+			break;
+		}
+	}
+	ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION, "[IQK]delay count = 0x%x!!!\n",
+		     delay_count);
+	return fail;
+}
+
+static bool _iqk_rx_iqk_gain_search_fail_8822b(struct phy_dm_struct *dm,
+					       u8 path, u8 step)
+{
+	struct dm_iqk_info *iqk_info = &dm->IQK_info;
+	bool fail = true;
+	u32 IQK_CMD = 0x0, rf_reg0, tmp, bb_idx;
+	u8 IQMUX[4] = {0x9, 0x12, 0x1b, 0x24};
+	u8 idx;
+
+	for (idx = 0; idx < 4; idx++)
+		if (iqk_info->tmp1bcc == IQMUX[idx])
+			break;
+
+	odm_write_4byte(dm, 0x1b00, 0xf8000008 | path << 1);
+	odm_write_4byte(dm, 0x1bcc, iqk_info->tmp1bcc);
+
+	if (step == RXIQK1)
+		ODM_RT_TRACE(
+			dm, ODM_COMP_CALIBRATION,
+			"[IQK]============ S%d RXIQK GainSearch ============\n",
+			path);
+
+	if (step == RXIQK1)
+		IQK_CMD = 0xf8000208 | (1 << (path + 4));
+	else
+		IQK_CMD = 0xf8000308 | (1 << (path + 4));
+
+	ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION, "[IQK]S%d GS%d_Trigger = 0x%x\n",
+		     path, step, IQK_CMD);
+
+	odm_write_4byte(dm, 0x1b00, IQK_CMD);
+	odm_write_4byte(dm, 0x1b00, IQK_CMD + 0x1);
+	ODM_delay_ms(GS_delay_8822B);
+	fail = _iqk_check_cal_8822b(dm, IQK_CMD);
+
+	if (step == RXIQK2) {
+		rf_reg0 = odm_get_rf_reg(dm, (enum odm_rf_radio_path)path, 0x0,
+					 RFREGOFFSETMASK);
+		odm_write_4byte(dm, 0x1b00, 0xf8000008 | path << 1);
+		ODM_RT_TRACE(
+			dm, ODM_COMP_CALIBRATION,
+			"[IQK]S%d ==> RF0x0 = 0x%x, tmp1bcc = 0x%x, idx = %d, 0x1b3c = 0x%x\n",
+			path, rf_reg0, iqk_info->tmp1bcc, idx,
+			odm_read_4byte(dm, 0x1b3c));
+		tmp = (rf_reg0 & 0x1fe0) >> 5;
+		iqk_info->lna_idx = tmp >> 5;
+		bb_idx = tmp & 0x1f;
+		if (bb_idx == 0x1) {
+			if (iqk_info->lna_idx != 0x0)
+				iqk_info->lna_idx--;
+			else if (idx != 3)
+				idx++;
+			else
+				iqk_info->isbnd = true;
+			fail = true;
+		} else if (bb_idx == 0xa) {
+			if (idx != 0)
+				idx--;
+			else if (iqk_info->lna_idx != 0x7)
+				iqk_info->lna_idx++;
+			else
+				iqk_info->isbnd = true;
+			fail = true;
+		} else {
+			fail = false;
+		}
+
+		if (iqk_info->isbnd)
+			fail = false;
+
+		iqk_info->tmp1bcc = IQMUX[idx];
+
+		if (fail) {
+			odm_write_4byte(dm, 0x1b00, 0xf8000008 | path << 1);
+			odm_write_4byte(
+				dm, 0x1b24,
+				(odm_read_4byte(dm, 0x1b24) & 0xffffe3ff) |
+					(iqk_info->lna_idx << 10));
+		}
+	}
+
+	return fail;
+}
+
+static bool _lok_one_shot_8822b(void *dm_void, u8 path)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct dm_iqk_info *iqk_info = &dm->IQK_info;
+	u8 delay_count = 0;
+	bool LOK_notready = false;
+	u32 LOK_temp = 0;
+	u32 IQK_CMD = 0x0;
+
+	ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+		     "[IQK]==========S%d LOK ==========\n", path);
+
+	IQK_CMD = 0xf8000008 | (1 << (4 + path));
+
+	ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION, "[IQK]LOK_Trigger = 0x%x\n",
+		     IQK_CMD);
+
+	odm_write_4byte(dm, 0x1b00, IQK_CMD);
+	odm_write_4byte(dm, 0x1b00, IQK_CMD + 1);
+	/*LOK: CMD ID = 0	{0xf8000018, 0xf8000028}*/
+	/*LOK: CMD ID = 0	{0xf8000019, 0xf8000029}*/
+	ODM_delay_ms(LOK_delay_8822B);
+
+	delay_count = 0;
+	LOK_notready = true;
+
+	while (LOK_notready) {
+		if (odm_read_4byte(dm, 0x1b00) == (IQK_CMD & 0xffffff0f))
+			LOK_notready = false;
+		else
+			LOK_notready = true;
+
+		if (LOK_notready) {
+			ODM_delay_ms(1);
+			delay_count++;
+		}
+
+		if (delay_count >= 50) {
+			ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+				     "[IQK]S%d LOK timeout!!!\n", path);
+			break;
+		}
+	}
+
+	ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+		     "[IQK]S%d ==> delay_count = 0x%x\n", path, delay_count);
+	if (ODM_COMP_CALIBRATION) {
+		if (!LOK_notready) {
+			LOK_temp =
+				odm_get_rf_reg(dm, (enum odm_rf_radio_path)path,
+					       0x58, RFREGOFFSETMASK);
+			ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+				     "[IQK]0x58 = 0x%x\n", LOK_temp);
+		} else {
+			ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+				     "[IQK]==>S%d LOK Fail!!!\n", path);
+		}
+	}
+	iqk_info->lok_fail[path] = LOK_notready;
+	return LOK_notready;
+}
+
+static bool _iqk_one_shot_8822b(void *dm_void, u8 path, u8 idx)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct dm_iqk_info *iqk_info = &dm->IQK_info;
+	u8 delay_count = 0;
+	bool notready = true, fail = true;
+	u32 IQK_CMD = 0x0;
+	u16 iqk_apply[2] = {0xc94, 0xe94};
+
+	if (idx == TXIQK)
+		ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+			     "[IQK]============ S%d WBTXIQK ============\n",
+			     path);
+	else if (idx == RXIQK1)
+		ODM_RT_TRACE(
+			dm, ODM_COMP_CALIBRATION,
+			"[IQK]============ S%d WBRXIQK STEP1============\n",
+			path);
+	else
+		ODM_RT_TRACE(
+			dm, ODM_COMP_CALIBRATION,
+			"[IQK]============ S%d WBRXIQK STEP2============\n",
+			path);
+
+	if (idx == TXIQK) {
+		IQK_CMD = 0xf8000008 | ((*dm->band_width + 4) << 8) |
+			  (1 << (path + 4));
+		ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+			     "[IQK]TXK_Trigger = 0x%x\n", IQK_CMD);
+		/*{0xf8000418, 0xf800042a} ==> 20 WBTXK (CMD = 4)*/
+		/*{0xf8000518, 0xf800052a} ==> 40 WBTXK (CMD = 5)*/
+		/*{0xf8000618, 0xf800062a} ==> 80 WBTXK (CMD = 6)*/
+	} else if (idx == RXIQK1) {
+		if (*dm->band_width == 2)
+			IQK_CMD = 0xf8000808 | (1 << (path + 4));
+		else
+			IQK_CMD = 0xf8000708 | (1 << (path + 4));
+		ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+			     "[IQK]RXK1_Trigger = 0x%x\n", IQK_CMD);
+		/*{0xf8000718, 0xf800072a} ==> 20 WBTXK (CMD = 7)*/
+		/*{0xf8000718, 0xf800072a} ==> 40 WBTXK (CMD = 7)*/
+		/*{0xf8000818, 0xf800082a} ==> 80 WBTXK (CMD = 8)*/
+	} else if (idx == RXIQK2) {
+		IQK_CMD = 0xf8000008 | ((*dm->band_width + 9) << 8) |
+			  (1 << (path + 4));
+		ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+			     "[IQK]RXK2_Trigger = 0x%x\n", IQK_CMD);
+		/*{0xf8000918, 0xf800092a} ==> 20 WBRXK (CMD = 9)*/
+		/*{0xf8000a18, 0xf8000a2a} ==> 40 WBRXK (CMD = 10)*/
+		/*{0xf8000b18, 0xf8000b2a} ==> 80 WBRXK (CMD = 11)*/
+		odm_write_4byte(dm, 0x1b00, 0xf8000008 | path << 1);
+		odm_write_4byte(dm, 0x1b24,
+				(odm_read_4byte(dm, 0x1b24) & 0xffffe3ff) |
+					((iqk_info->lna_idx & 0x7) << 10));
+	}
+	odm_write_4byte(dm, 0x1b00, IQK_CMD);
+	odm_write_4byte(dm, 0x1b00, IQK_CMD + 0x1);
+	ODM_delay_ms(WBIQK_delay_8822B);
+
+	while (notready) {
+		if (odm_read_4byte(dm, 0x1b00) == (IQK_CMD & 0xffffff0f))
+			notready = false;
+		else
+			notready = true;
+
+		if (notready) {
+			ODM_delay_ms(1);
+			delay_count++;
+		} else {
+			fail = (bool)odm_get_bb_reg(dm, 0x1b08, BIT(26));
+			break;
+		}
+
+		if (delay_count >= 50) {
+			ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+				     "[IQK]S%d IQK timeout!!!\n", path);
+			break;
+		}
+	}
+
+	if (dm->debug_components & ODM_COMP_CALIBRATION) {
+		odm_write_4byte(dm, 0x1b00, 0xf8000008 | path << 1);
+		ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+			     "[IQK]S%d ==> 0x1b00 = 0x%x, 0x1b08 = 0x%x\n",
+			     path, odm_read_4byte(dm, 0x1b00),
+			     odm_read_4byte(dm, 0x1b08));
+		ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+			     "[IQK]S%d ==> delay_count = 0x%x\n", path,
+			     delay_count);
+		if (idx != TXIQK)
+			ODM_RT_TRACE(
+				dm, ODM_COMP_CALIBRATION,
+				"[IQK]S%d ==> RF0x0 = 0x%x, RF0x56 = 0x%x\n",
+				path,
+				odm_get_rf_reg(dm, (enum odm_rf_radio_path)path,
+					       0x0, RFREGOFFSETMASK),
+				odm_get_rf_reg(dm, (enum odm_rf_radio_path)path,
+					       0x56, RFREGOFFSETMASK));
+	}
+
+	odm_write_4byte(dm, 0x1b00, 0xf8000008 | path << 1);
+
+	if (idx == TXIQK)
+		if (fail)
+			odm_set_bb_reg(dm, iqk_apply[path], BIT(0), 0x0);
+
+	if (idx == RXIQK2) {
+		iqk_info->rxiqk_agc[0][path] =
+			(u16)(((odm_get_rf_reg(dm, (enum odm_rf_radio_path)path,
+					       0x0, RFREGOFFSETMASK) >>
+				5) &
+			       0xff) |
+			      (iqk_info->tmp1bcc << 8));
+
+		odm_write_4byte(dm, 0x1b38, 0x20000000);
+
+		if (!fail)
+			odm_set_bb_reg(dm, iqk_apply[path], (BIT(11) | BIT(10)),
+				       0x1);
+		else
+			odm_set_bb_reg(dm, iqk_apply[path], (BIT(11) | BIT(10)),
+				       0x0);
+	}
+
+	if (idx == TXIQK)
+		iqk_info->iqk_fail_report[0][path][TXIQK] = fail;
+	else
+		iqk_info->iqk_fail_report[0][path][RXIQK] = fail;
+
+	return fail;
+}
+
+static bool _iqk_rx_iqk_by_path_8822b(void *dm_void, u8 path)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct dm_iqk_info *iqk_info = &dm->IQK_info;
+	bool KFAIL = true, gonext;
+
+	switch (iqk_info->rxiqk_step) {
+	case 1: /*gain search_RXK1*/
+		_iqk_rxk1_setting_8822b(dm, path);
+		gonext = false;
+		while (1) {
+			KFAIL = _iqk_rx_iqk_gain_search_fail_8822b(dm, path,
+								   RXIQK1);
+			if (KFAIL &&
+			    (iqk_info->gs_retry_count[0][path][GSRXK1] < 2))
+				iqk_info->gs_retry_count[0][path][GSRXK1]++;
+			else if (KFAIL) {
+				iqk_info->rxiqk_fail_code[0][path] = 0;
+				iqk_info->rxiqk_step = 5;
+				gonext = true;
+			} else {
+				iqk_info->rxiqk_step++;
+				gonext = true;
+			}
+			if (gonext)
+				break;
+		}
+		break;
+	case 2: /*gain search_RXK2*/
+		_iqk_rxk2_setting_8822b(dm, path, true);
+		iqk_info->isbnd = false;
+		while (1) {
+			KFAIL = _iqk_rx_iqk_gain_search_fail_8822b(dm, path,
+								   RXIQK2);
+			if (KFAIL &&
+			    (iqk_info->gs_retry_count[0][path][GSRXK2] <
+			     rxiqk_gs_limit)) {
+				iqk_info->gs_retry_count[0][path][GSRXK2]++;
+			} else {
+				iqk_info->rxiqk_step++;
+				break;
+			}
+		}
+		break;
+	case 3: /*RXK1*/
+		_iqk_rxk1_setting_8822b(dm, path);
+		gonext = false;
+		while (1) {
+			KFAIL = _iqk_one_shot_8822b(dm, path, RXIQK1);
+			if (KFAIL &&
+			    (iqk_info->retry_count[0][path][RXIQK1] < 2))
+				iqk_info->retry_count[0][path][RXIQK1]++;
+			else if (KFAIL) {
+				iqk_info->rxiqk_fail_code[0][path] = 1;
+				iqk_info->rxiqk_step = 5;
+				gonext = true;
+			} else {
+				iqk_info->rxiqk_step++;
+				gonext = true;
+			}
+			if (gonext)
+				break;
+		}
+		break;
+	case 4: /*RXK2*/
+		_iqk_rxk2_setting_8822b(dm, path, false);
+		gonext = false;
+		while (1) {
+			KFAIL = _iqk_one_shot_8822b(dm, path, RXIQK2);
+			if (KFAIL &&
+			    (iqk_info->retry_count[0][path][RXIQK2] < 2))
+				iqk_info->retry_count[0][path][RXIQK2]++;
+			else if (KFAIL) {
+				iqk_info->rxiqk_fail_code[0][path] = 2;
+				iqk_info->rxiqk_step = 5;
+				gonext = true;
+			} else {
+				iqk_info->rxiqk_step++;
+				gonext = true;
+			}
+			if (gonext)
+				break;
+		}
+		break;
+	}
+	return KFAIL;
+}
+
+static void _iqk_iqk_by_path_8822b(void *dm_void, bool segment_iqk)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct dm_iqk_info *iqk_info = &dm->IQK_info;
+	bool KFAIL = true;
+	u8 i, kcount_limit;
+
+	if (*dm->band_width == 2)
+		kcount_limit = kcount_limit_80m;
+	else
+		kcount_limit = kcount_limit_others;
+
+	while (1) {
+		switch (dm->rf_calibrate_info.iqk_step) {
+		case 1: /*S0 LOK*/
+			_iqk_lok_setting_8822b(dm, ODM_RF_PATH_A);
+			_lok_one_shot_8822b(dm, ODM_RF_PATH_A);
+			dm->rf_calibrate_info.iqk_step++;
+			break;
+		case 2: /*S1 LOK*/
+			_iqk_lok_setting_8822b(dm, ODM_RF_PATH_B);
+			_lok_one_shot_8822b(dm, ODM_RF_PATH_B);
+			dm->rf_calibrate_info.iqk_step++;
+			break;
+		case 3: /*S0 TXIQK*/
+			_iqk_txk_setting_8822b(dm, ODM_RF_PATH_A);
+			KFAIL = _iqk_one_shot_8822b(dm, ODM_RF_PATH_A, TXIQK);
+			iqk_info->kcount++;
+			ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+				     "[IQK]S0TXK KFail = 0x%x\n", KFAIL);
+
+			if (KFAIL &&
+			    (iqk_info->retry_count[0][ODM_RF_PATH_A][TXIQK] <
+			     3))
+				iqk_info->retry_count[0][ODM_RF_PATH_A]
+						     [TXIQK]++;
+			else
+				dm->rf_calibrate_info.iqk_step++;
+			break;
+		case 4: /*S1 TXIQK*/
+			_iqk_txk_setting_8822b(dm, ODM_RF_PATH_B);
+			KFAIL = _iqk_one_shot_8822b(dm, ODM_RF_PATH_B, TXIQK);
+			iqk_info->kcount++;
+			ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+				     "[IQK]S1TXK KFail = 0x%x\n", KFAIL);
+			if (KFAIL &&
+			    iqk_info->retry_count[0][ODM_RF_PATH_B][TXIQK] < 3)
+				iqk_info->retry_count[0][ODM_RF_PATH_B]
+						     [TXIQK]++;
+			else
+				dm->rf_calibrate_info.iqk_step++;
+			break;
+		case 5: /*S0 RXIQK*/
+			phydm_set_iqk_info(dm, iqk_info, 0);
+			break;
+		case 6: /*S1 RXIQK*/
+			phydm_set_iqk_info(dm, iqk_info, 1);
+			break;
+		}
+
+		if (dm->rf_calibrate_info.iqk_step == 7) {
+			ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+				     "[IQK]==========LOK summary ==========\n");
+			ODM_RT_TRACE(
+				dm, ODM_COMP_CALIBRATION,
+				"[IQK]PathA_LOK_notready = %d, PathB_LOK1_notready = %d\n",
+				iqk_info->lok_fail[ODM_RF_PATH_A],
+				iqk_info->lok_fail[ODM_RF_PATH_B]);
+			ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+				     "[IQK]==========IQK summary ==========\n");
+			ODM_RT_TRACE(
+				dm, ODM_COMP_CALIBRATION,
+				"[IQK]PathA_TXIQK_fail = %d, PathB_TXIQK_fail = %d\n",
+				iqk_info->iqk_fail_report[0][ODM_RF_PATH_A]
+							 [TXIQK],
+				iqk_info->iqk_fail_report[0][ODM_RF_PATH_B]
+							 [TXIQK]);
+			ODM_RT_TRACE(
+				dm, ODM_COMP_CALIBRATION,
+				"[IQK]PathA_RXIQK_fail = %d, PathB_RXIQK_fail = %d\n",
+				iqk_info->iqk_fail_report[0][ODM_RF_PATH_A]
+							 [RXIQK],
+				iqk_info->iqk_fail_report[0][ODM_RF_PATH_B]
+							 [RXIQK]);
+			ODM_RT_TRACE(
+				dm, ODM_COMP_CALIBRATION,
+				"[IQK]PathA_TXIQK_retry = %d, PathB_TXIQK_retry = %d\n",
+				iqk_info->retry_count[0][ODM_RF_PATH_A][TXIQK],
+				iqk_info->retry_count[0][ODM_RF_PATH_B][TXIQK]);
+			ODM_RT_TRACE(
+				dm, ODM_COMP_CALIBRATION,
+				"[IQK]PathA_RXK1_retry = %d, PathA_RXK2_retry = %d, PathB_RXK1_retry = %d, PathB_RXK2_retry = %d\n",
+				iqk_info->retry_count[0][ODM_RF_PATH_A][RXIQK1],
+				iqk_info->retry_count[0][ODM_RF_PATH_A][RXIQK2],
+				iqk_info->retry_count[0][ODM_RF_PATH_B][RXIQK1],
+				iqk_info->retry_count[0][ODM_RF_PATH_B]
+						     [RXIQK2]);
+			ODM_RT_TRACE(
+				dm, ODM_COMP_CALIBRATION,
+				"[IQK]PathA_GS1_retry = %d, PathA_GS2_retry = %d, PathB_GS1_retry = %d, PathB_GS2_retry = %d\n",
+				iqk_info->gs_retry_count[0][ODM_RF_PATH_A]
+							[GSRXK1],
+				iqk_info->gs_retry_count[0][ODM_RF_PATH_A]
+							[GSRXK2],
+				iqk_info->gs_retry_count[0][ODM_RF_PATH_B]
+							[GSRXK1],
+				iqk_info->gs_retry_count[0][ODM_RF_PATH_B]
+							[GSRXK2]);
+			for (i = 0; i < 2; i++) {
+				odm_write_4byte(dm, 0x1b00,
+						0xf8000008 | i << 1);
+				odm_write_4byte(dm, 0x1b2c, 0x7);
+				odm_write_4byte(dm, 0x1bcc, 0x0);
+			}
+			break;
+		}
+
+		if (segment_iqk && (iqk_info->kcount == kcount_limit))
+			break;
+	}
+}
+
+static void _iqk_start_iqk_8822b(struct phy_dm_struct *dm, bool segment_iqk)
+{
+	u32 tmp;
+
+	/*GNT_WL = 1*/
+	tmp = odm_get_rf_reg(dm, ODM_RF_PATH_A, 0x1, RFREGOFFSETMASK);
+	tmp = tmp | BIT(5) | BIT(0);
+	odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x1, RFREGOFFSETMASK, tmp);
+
+	tmp = odm_get_rf_reg(dm, ODM_RF_PATH_B, 0x1, RFREGOFFSETMASK);
+	tmp = tmp | BIT(5) | BIT(0);
+	odm_set_rf_reg(dm, ODM_RF_PATH_B, 0x1, RFREGOFFSETMASK, tmp);
+
+	_iqk_iqk_by_path_8822b(dm, segment_iqk);
+}
+
+static void _iq_calibrate_8822b_init(void *dm_void)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+	struct dm_iqk_info *iqk_info = &dm->IQK_info;
+	u8 i, j;
+
+	if (iqk_info->iqk_times == 0) {
+		ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+			     "[IQK]=====>PHY_IQCalibrate_8822B_Init\n");
+
+		for (i = 0; i < SS_8822B; i++) {
+			for (j = 0; j < 2; j++) {
+				iqk_info->lok_fail[i] = true;
+				iqk_info->iqk_fail[j][i] = true;
+				iqk_info->iqc_matrix[j][i] = 0x20000000;
+			}
+		}
+
+		phydm_init_iqk_information(iqk_info);
+	}
+}
+
+static void _phy_iq_calibrate_8822b(struct phy_dm_struct *dm, bool reset)
+{
+	u32 MAC_backup[MAC_REG_NUM_8822B], BB_backup[BB_REG_NUM_8822B],
+		RF_backup[RF_REG_NUM_8822B][SS_8822B];
+	u32 backup_mac_reg[MAC_REG_NUM_8822B] = {0x520, 0x550};
+	u32 backup_bb_reg[BB_REG_NUM_8822B] = {
+		0x808, 0x90c, 0xc00, 0xcb0,  0xcb4, 0xcbc, 0xe00,
+		0xeb0, 0xeb4, 0xebc, 0x1990, 0x9a4, 0xa04};
+	u32 backup_rf_reg[RF_REG_NUM_8822B] = {0xdf, 0x8f, 0x65, 0x0, 0x1};
+	bool segment_iqk = false, is_mp = false;
+
+	struct dm_iqk_info *iqk_info = &dm->IQK_info;
+
+	if (dm->mp_mode)
+		is_mp = true;
+	else if (dm->is_linked)
+		segment_iqk = true;
+
+	if (!is_mp)
+		if (_iqk_reload_iqk_8822b(dm, reset))
+			return;
+
+	ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+		     "[IQK]==========IQK strat!!!!!==========\n");
+
+	ODM_RT_TRACE(
+		dm, ODM_COMP_CALIBRATION,
+		"[IQK]band_type = %s, band_width = %d, ExtPA2G = %d, ext_pa_5g = %d\n",
+		(*dm->band_type == ODM_BAND_5G) ? "5G" : "2G", *dm->band_width,
+		dm->ext_pa, dm->ext_pa_5g);
+	ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+		     "[IQK]Interface = %d, cut_version = %x\n",
+		     dm->support_interface, dm->cut_version);
+
+	iqk_info->iqk_times++;
+
+	iqk_info->kcount = 0;
+	dm->rf_calibrate_info.iqk_total_progressing_time = 0;
+	dm->rf_calibrate_info.iqk_step = 1;
+	iqk_info->rxiqk_step = 1;
+
+	_iqk_backup_iqk_8822b(dm, 0);
+	_iqk_backup_mac_bb_8822b(dm, MAC_backup, BB_backup, backup_mac_reg,
+				 backup_bb_reg);
+	_iqk_backup_rf_8822b(dm, RF_backup, backup_rf_reg);
+
+	while (1) {
+		if (!is_mp)
+			dm->rf_calibrate_info.iqk_start_time =
+				odm_get_current_time(dm);
+
+		_iqk_configure_macbb_8822b(dm);
+		_iqk_afe_setting_8822b(dm, true);
+		_iqk_rfe_setting_8822b(dm, false);
+		_iqk_agc_bnd_int_8822b(dm);
+		_iqk_rf_setting_8822b(dm);
+
+		_iqk_start_iqk_8822b(dm, segment_iqk);
+
+		_iqk_afe_setting_8822b(dm, false);
+		_iqk_restore_mac_bb_8822b(dm, MAC_backup, BB_backup,
+					  backup_mac_reg, backup_bb_reg);
+		_iqk_restore_rf_8822b(dm, backup_rf_reg, RF_backup);
+
+		if (!is_mp) {
+			dm->rf_calibrate_info.iqk_progressing_time =
+				odm_get_progressing_time(
+					dm,
+					dm->rf_calibrate_info.iqk_start_time);
+			dm->rf_calibrate_info.iqk_total_progressing_time +=
+				odm_get_progressing_time(
+					dm,
+					dm->rf_calibrate_info.iqk_start_time);
+			ODM_RT_TRACE(
+				dm, ODM_COMP_CALIBRATION,
+				"[IQK]IQK progressing_time = %lld ms\n",
+				dm->rf_calibrate_info.iqk_progressing_time);
+		}
+
+		if (dm->rf_calibrate_info.iqk_step == 7)
+			break;
+
+		iqk_info->kcount = 0;
+		ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION, "[IQK]delay 50ms!!!\n");
+		ODM_delay_ms(50);
+	};
+
+	_iqk_backup_iqk_8822b(dm, 1);
+	_iqk_fill_iqk_report_8822b(dm, 0);
+
+	if (!is_mp)
+		ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+			     "[IQK]Total IQK progressing_time = %lld ms\n",
+			     dm->rf_calibrate_info.iqk_total_progressing_time);
+
+	ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+		     "[IQK]==========IQK end!!!!!==========\n");
+}
+
+static void _phy_iq_calibrate_by_fw_8822b(void *dm_void, u8 clear) {}
+
+/*IQK version:v3.3, NCTL v0.6*/
+/*1.The new gainsearch method for RXIQK*/
+/*2.The new format of IQK report register: 0x1be8/0x1bec*/
+/*3. add the option of segment IQK*/
+void phy_iq_calibrate_8822b(void *dm_void, bool clear)
+{
+	struct phy_dm_struct *dm = (struct phy_dm_struct *)dm_void;
+
+	dm->iqk_fw_offload = 0;
+
+	/*FW IQK*/
+	if (dm->iqk_fw_offload) {
+		if (!dm->rf_calibrate_info.is_iqk_in_progress) {
+			odm_acquire_spin_lock(dm, RT_IQK_SPINLOCK);
+			dm->rf_calibrate_info.is_iqk_in_progress = true;
+			odm_release_spin_lock(dm, RT_IQK_SPINLOCK);
+
+			dm->rf_calibrate_info.iqk_start_time =
+				odm_get_current_time(dm);
+
+			odm_write_4byte(dm, 0x1b00, 0xf8000008);
+			odm_set_bb_reg(dm, 0x1bf0, 0xff000000, 0xff);
+			ODM_RT_TRACE(dm, ODM_COMP_CALIBRATION,
+				     "[IQK]0x1bf0 = 0x%x\n",
+				     odm_read_4byte(dm, 0x1bf0));
+
+			_phy_iq_calibrate_by_fw_8822b(dm, clear);
+			phydm_get_read_counter(dm);
+
+			dm->rf_calibrate_info.iqk_progressing_time =
+				odm_get_progressing_time(
+					dm,
+					dm->rf_calibrate_info.iqk_start_time);
+
+			ODM_RT_TRACE(
+				dm, ODM_COMP_CALIBRATION,
+				"[IQK]IQK progressing_time = %lld ms\n",
+				dm->rf_calibrate_info.iqk_progressing_time);
+
+			odm_acquire_spin_lock(dm, RT_IQK_SPINLOCK);
+			dm->rf_calibrate_info.is_iqk_in_progress = false;
+			odm_release_spin_lock(dm, RT_IQK_SPINLOCK);
+		} else {
+			ODM_RT_TRACE(
+				dm, ODM_COMP_CALIBRATION,
+				"== Return the IQK CMD, because the IQK in Progress ==\n");
+		}
+
+	} else {
+		_iq_calibrate_8822b_init(dm_void);
+
+		if (!dm->rf_calibrate_info.is_iqk_in_progress) {
+			odm_acquire_spin_lock(dm, RT_IQK_SPINLOCK);
+			dm->rf_calibrate_info.is_iqk_in_progress = true;
+			odm_release_spin_lock(dm, RT_IQK_SPINLOCK);
+			if (dm->mp_mode)
+				dm->rf_calibrate_info.iqk_start_time =
+					odm_get_current_time(dm);
+
+			_phy_iq_calibrate_8822b(dm, clear);
+			if (dm->mp_mode) {
+				dm->rf_calibrate_info.iqk_progressing_time =
+					odm_get_progressing_time(
+						dm, dm->rf_calibrate_info
+							    .iqk_start_time);
+				ODM_RT_TRACE(
+					dm, ODM_COMP_CALIBRATION,
+					"[IQK]IQK progressing_time = %lld ms\n",
+					dm->rf_calibrate_info
+						.iqk_progressing_time);
+			}
+			odm_acquire_spin_lock(dm, RT_IQK_SPINLOCK);
+			dm->rf_calibrate_info.is_iqk_in_progress = false;
+			odm_release_spin_lock(dm, RT_IQK_SPINLOCK);
+		} else {
+			ODM_RT_TRACE(
+				dm, ODM_COMP_CALIBRATION,
+				"[IQK]== Return the IQK CMD, because the IQK in Progress ==\n");
+		}
+	}
+}
diff --git a/drivers/staging/rtlwifi/phydm/rtl8822b/phydm_iqk_8822b.h b/drivers/staging/rtlwifi/phydm/rtl8822b/phydm_iqk_8822b.h
new file mode 100644
index 000000000000..ea19deb512d5
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/rtl8822b/phydm_iqk_8822b.h
@@ -0,0 +1,48 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef __PHYDM_IQK_8822B_H__
+#define __PHYDM_IQK_8822B_H__
+
+/*--------------------------Define Parameters-------------------------------*/
+#define MAC_REG_NUM_8822B 2
+#define BB_REG_NUM_8822B 13
+#define RF_REG_NUM_8822B 5
+
+#define LOK_delay_8822B 2
+#define GS_delay_8822B 2
+#define WBIQK_delay_8822B 2
+
+#define TXIQK 0
+#define RXIQK 1
+#define SS_8822B 2
+
+/*------------------------End Define Parameters-------------------------------*/
+
+void do_iqk_8822b(void *dm_void, u8 delta_thermal_index, u8 thermal_value,
+		  u8 threshold);
+
+void phy_iq_calibrate_8822b(void *dm_void, bool clear);
+
+#endif /* #ifndef __PHYDM_IQK_8822B_H__*/
diff --git a/drivers/staging/rtlwifi/phydm/rtl8822b/phydm_regconfig8822b.c b/drivers/staging/rtlwifi/phydm/rtl8822b/phydm_regconfig8822b.c
new file mode 100644
index 000000000000..644fca822c61
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/rtl8822b/phydm_regconfig8822b.c
@@ -0,0 +1,168 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#include "../mp_precomp.h"
+#include "../phydm_precomp.h"
+
+void odm_config_rf_reg_8822b(struct phy_dm_struct *dm, u32 addr, u32 data,
+			     enum odm_rf_radio_path RF_PATH, u32 reg_addr)
+{
+	if (addr == 0xffe) {
+		ODM_sleep_ms(50);
+	} else if (addr == 0xfe) {
+		ODM_delay_us(100);
+	} else {
+		odm_set_rf_reg(dm, RF_PATH, reg_addr, RFREGOFFSETMASK, data);
+
+		/* Add 1us delay between BB/RF register setting. */
+		ODM_delay_us(1);
+	}
+}
+
+void odm_config_rf_radio_a_8822b(struct phy_dm_struct *dm, u32 addr, u32 data)
+{
+	u32 content = 0x1000; /* RF_Content: radioa_txt */
+	u32 maskfor_phy_set = (u32)(content & 0xE000);
+
+	odm_config_rf_reg_8822b(dm, addr, data, ODM_RF_PATH_A,
+				addr | maskfor_phy_set);
+
+	ODM_RT_TRACE(
+		dm, ODM_COMP_INIT,
+		"===> odm_config_rf_with_header_file: [RadioA] %08X %08X\n",
+		addr, data);
+}
+
+void odm_config_rf_radio_b_8822b(struct phy_dm_struct *dm, u32 addr, u32 data)
+{
+	u32 content = 0x1001; /* RF_Content: radiob_txt */
+	u32 maskfor_phy_set = (u32)(content & 0xE000);
+
+	odm_config_rf_reg_8822b(dm, addr, data, ODM_RF_PATH_B,
+				addr | maskfor_phy_set);
+
+	ODM_RT_TRACE(
+		dm, ODM_COMP_INIT,
+		"===> odm_config_rf_with_header_file: [RadioB] %08X %08X\n",
+		addr, data);
+}
+
+void odm_config_mac_8822b(struct phy_dm_struct *dm, u32 addr, u8 data)
+{
+	odm_write_1byte(dm, addr, data);
+	ODM_RT_TRACE(
+		dm, ODM_COMP_INIT,
+		"===> odm_config_mac_with_header_file: [MAC_REG] %08X %08X\n",
+		addr, data);
+}
+
+void odm_update_agc_big_jump_lmt_8822b(struct phy_dm_struct *dm, u32 addr,
+				       u32 data)
+{
+	struct dig_thres *dig_tab = &dm->dm_dig_table;
+	u8 rf_gain_idx = (u8)((data & 0xFF000000) >> 24);
+	u8 bb_gain_idx = (u8)((data & 0x00ff0000) >> 16);
+	u8 agc_table_idx = (u8)((data & 0x00000f00) >> 8);
+	static bool is_limit;
+
+	if (addr != 0x81c)
+		return;
+
+	if (bb_gain_idx > 0x3c) {
+		if ((rf_gain_idx == dig_tab->rf_gain_idx) && !is_limit) {
+			is_limit = true;
+			dig_tab->big_jump_lmt[agc_table_idx] = bb_gain_idx - 2;
+			ODM_RT_TRACE(
+				dm, ODM_COMP_DIG,
+				"===> [AGC_TAB] big_jump_lmt [%d] = 0x%x\n",
+				agc_table_idx,
+				dig_tab->big_jump_lmt[agc_table_idx]);
+		}
+	} else {
+		is_limit = false;
+	}
+
+	dig_tab->rf_gain_idx = rf_gain_idx;
+}
+
+void odm_config_bb_agc_8822b(struct phy_dm_struct *dm, u32 addr, u32 bitmask,
+			     u32 data)
+{
+	odm_update_agc_big_jump_lmt_8822b(dm, addr, data);
+
+	odm_set_bb_reg(dm, addr, bitmask, data);
+
+	/* Add 1us delay between BB/RF register setting. */
+	ODM_delay_us(1);
+
+	ODM_RT_TRACE(dm, ODM_COMP_INIT, "===> %s: [AGC_TAB] %08X %08X\n",
+		     __func__, addr, data);
+}
+
+void odm_config_bb_phy_reg_pg_8822b(struct phy_dm_struct *dm, u32 band,
+				    u32 rf_path, u32 tx_num, u32 addr,
+				    u32 bitmask, u32 data)
+{
+	if (addr == 0xfe || addr == 0xffe) {
+		ODM_sleep_ms(50);
+	} else {
+		phy_store_tx_power_by_rate(dm->adapter, band, rf_path, tx_num,
+					   addr, bitmask, data);
+	}
+	ODM_RT_TRACE(dm, ODM_COMP_INIT, "===> %s: [PHY_REG] %08X %08X %08X\n",
+		     __func__, addr, bitmask, data);
+}
+
+void odm_config_bb_phy_8822b(struct phy_dm_struct *dm, u32 addr, u32 bitmask,
+			     u32 data)
+{
+	if (addr == 0xfe)
+		ODM_sleep_ms(50);
+	else if (addr == 0xfd)
+		ODM_delay_ms(5);
+	else if (addr == 0xfc)
+		ODM_delay_ms(1);
+	else if (addr == 0xfb)
+		ODM_delay_us(50);
+	else if (addr == 0xfa)
+		ODM_delay_us(5);
+	else if (addr == 0xf9)
+		ODM_delay_us(1);
+	else
+		odm_set_bb_reg(dm, addr, bitmask, data);
+
+	/* Add 1us delay between BB/RF register setting. */
+	ODM_delay_us(1);
+	ODM_RT_TRACE(dm, ODM_COMP_INIT, "===> %s: [PHY_REG] %08X %08X\n",
+		     __func__, addr, data);
+}
+
+void odm_config_bb_txpwr_lmt_8822b(struct phy_dm_struct *dm, u8 *regulation,
+				   u8 *band, u8 *bandwidth, u8 *rate_section,
+				   u8 *rf_path, u8 *channel, u8 *power_limit)
+{
+	phy_set_tx_power_limit(dm, regulation, band, bandwidth, rate_section,
+			       rf_path, channel, power_limit);
+}
diff --git a/drivers/staging/rtlwifi/phydm/rtl8822b/phydm_regconfig8822b.h b/drivers/staging/rtlwifi/phydm/rtl8822b/phydm_regconfig8822b.h
new file mode 100644
index 000000000000..4817cf6b1ed9
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/rtl8822b/phydm_regconfig8822b.h
@@ -0,0 +1,54 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef __INC_ODM_REGCONFIG_H_8822B
+#define __INC_ODM_REGCONFIG_H_8822B
+
+void odm_config_rf_reg_8822b(struct phy_dm_struct *dm, u32 addr, u32 data,
+			     enum odm_rf_radio_path RF_PATH, u32 reg_addr);
+
+void odm_config_rf_radio_a_8822b(struct phy_dm_struct *dm, u32 addr, u32 data);
+
+void odm_config_rf_radio_b_8822b(struct phy_dm_struct *dm, u32 addr, u32 data);
+
+void odm_config_mac_8822b(struct phy_dm_struct *dm, u32 addr, u8 data);
+
+void odm_update_agc_big_jump_lmt_8822b(struct phy_dm_struct *dm, u32 addr,
+				       u32 data);
+
+void odm_config_bb_agc_8822b(struct phy_dm_struct *dm, u32 addr, u32 bitmask,
+			     u32 data);
+
+void odm_config_bb_phy_reg_pg_8822b(struct phy_dm_struct *dm, u32 band,
+				    u32 rf_path, u32 tx_num, u32 addr,
+				    u32 bitmask, u32 data);
+
+void odm_config_bb_phy_8822b(struct phy_dm_struct *dm, u32 addr, u32 bitmask,
+			     u32 data);
+
+void odm_config_bb_txpwr_lmt_8822b(struct phy_dm_struct *dm, u8 *regulation,
+				   u8 *band, u8 *bandwidth, u8 *rate_section,
+				   u8 *rf_path, u8 *channel, u8 *power_limit);
+
+#endif /* RTL8822B_SUPPORT == 1*/
diff --git a/drivers/staging/rtlwifi/phydm/rtl8822b/phydm_rtl8822b.c b/drivers/staging/rtlwifi/phydm/rtl8822b/phydm_rtl8822b.c
new file mode 100644
index 000000000000..59adabda09de
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/rtl8822b/phydm_rtl8822b.c
@@ -0,0 +1,225 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#include "../mp_precomp.h"
+#include "../phydm_precomp.h"
+
+static void phydm_dynamic_switch_htstf_mumimo_8822b(struct phy_dm_struct *dm)
+{
+	/*if rssi > 40dBm, enable HT-STF gain controller,
+	 *otherwise, if rssi < 40dBm, disable the controller
+	 */
+	/*add by Chun-Hung Ho 20160711 */
+	if (dm->rssi_min >= 40)
+		odm_set_bb_reg(dm, 0x8d8, BIT(17), 0x1);
+	else if (dm->rssi_min < 35)
+		odm_set_bb_reg(dm, 0x8d8, BIT(17), 0x0);
+
+	ODM_RT_TRACE(dm, ODM_COMP_COMMON, "%s, rssi_min = %d\n", __func__,
+		     dm->rssi_min);
+}
+
+static void _set_tx_a_cali_value(struct phy_dm_struct *dm, u8 rf_path,
+				 u8 offset, u8 tx_a_bias_offset)
+{
+	u32 modi_tx_a_value = 0;
+	u8 tmp1_byte = 0;
+	bool is_minus = false;
+	u8 comp_value = 0;
+
+	switch (offset) {
+	case 0x0:
+		odm_set_rf_reg(dm, rf_path, 0x18, 0xFFFFF, 0X10124);
+		break;
+	case 0x1:
+		odm_set_rf_reg(dm, rf_path, 0x18, 0xFFFFF, 0X10524);
+		break;
+	case 0x2:
+		odm_set_rf_reg(dm, rf_path, 0x18, 0xFFFFF, 0X10924);
+		break;
+	case 0x3:
+		odm_set_rf_reg(dm, rf_path, 0x18, 0xFFFFF, 0X10D24);
+		break;
+	case 0x4:
+		odm_set_rf_reg(dm, rf_path, 0x18, 0xFFFFF, 0X30164);
+		break;
+	case 0x5:
+		odm_set_rf_reg(dm, rf_path, 0x18, 0xFFFFF, 0X30564);
+		break;
+	case 0x6:
+		odm_set_rf_reg(dm, rf_path, 0x18, 0xFFFFF, 0X30964);
+		break;
+	case 0x7:
+		odm_set_rf_reg(dm, rf_path, 0x18, 0xFFFFF, 0X30D64);
+		break;
+	case 0x8:
+		odm_set_rf_reg(dm, rf_path, 0x18, 0xFFFFF, 0X50195);
+		break;
+	case 0x9:
+		odm_set_rf_reg(dm, rf_path, 0x18, 0xFFFFF, 0X50595);
+		break;
+	case 0xa:
+		odm_set_rf_reg(dm, rf_path, 0x18, 0xFFFFF, 0X50995);
+		break;
+	case 0xb:
+		odm_set_rf_reg(dm, rf_path, 0x18, 0xFFFFF, 0X50D95);
+		break;
+	default:
+		ODM_RT_TRACE(dm, ODM_COMP_COMMON,
+			     "Invalid TxA band offset...\n");
+		return;
+	}
+
+	/* Get TxA value */
+	modi_tx_a_value = odm_get_rf_reg(dm, rf_path, 0x61, 0xFFFFF);
+	tmp1_byte = (u8)modi_tx_a_value & (BIT(3) | BIT(2) | BIT(1) | BIT(0));
+
+	/* check how much need to calibration */
+	switch (tx_a_bias_offset) {
+	case 0xF6:
+		is_minus = true;
+		comp_value = 3;
+		break;
+
+	case 0xF4:
+		is_minus = true;
+		comp_value = 2;
+		break;
+
+	case 0xF2:
+		is_minus = true;
+		comp_value = 1;
+		break;
+
+	case 0xF3:
+		is_minus = false;
+		comp_value = 1;
+		break;
+
+	case 0xF5:
+		is_minus = false;
+		comp_value = 2;
+		break;
+
+	case 0xF7:
+		is_minus = false;
+		comp_value = 3;
+		break;
+
+	case 0xF9:
+		is_minus = false;
+		comp_value = 4;
+		break;
+
+	/* do nothing case */
+	case 0xF0:
+	default:
+		ODM_RT_TRACE(dm, ODM_COMP_COMMON,
+			     "No need to do TxA bias current calibration\n");
+		return;
+	}
+
+	/* calc correct value to calibrate */
+	if (is_minus) {
+		if (tmp1_byte >= comp_value) {
+			tmp1_byte -= comp_value;
+			/*modi_tx_a_value += tmp1_byte;*/
+		} else {
+			tmp1_byte = 0;
+		}
+	} else {
+		tmp1_byte += comp_value;
+		if (tmp1_byte >= 7)
+			tmp1_byte = 7;
+	}
+
+	/* Write back to RF reg */
+	odm_set_rf_reg(dm, rf_path, 0x30, 0xFFFF,
+		       (offset << 12 | (modi_tx_a_value & 0xFF0) | tmp1_byte));
+}
+
+static void _txa_bias_cali_4_each_path(struct phy_dm_struct *dm, u8 rf_path,
+				       u8 efuse_value)
+{
+	/* switch on set TxA bias */
+	odm_set_rf_reg(dm, rf_path, 0xEF, 0xFFFFF, 0x200);
+
+	/* Set 12 sets of TxA value */
+	_set_tx_a_cali_value(dm, rf_path, 0x0, efuse_value);
+	_set_tx_a_cali_value(dm, rf_path, 0x1, efuse_value);
+	_set_tx_a_cali_value(dm, rf_path, 0x2, efuse_value);
+	_set_tx_a_cali_value(dm, rf_path, 0x3, efuse_value);
+	_set_tx_a_cali_value(dm, rf_path, 0x4, efuse_value);
+	_set_tx_a_cali_value(dm, rf_path, 0x5, efuse_value);
+	_set_tx_a_cali_value(dm, rf_path, 0x6, efuse_value);
+	_set_tx_a_cali_value(dm, rf_path, 0x7, efuse_value);
+	_set_tx_a_cali_value(dm, rf_path, 0x8, efuse_value);
+	_set_tx_a_cali_value(dm, rf_path, 0x9, efuse_value);
+	_set_tx_a_cali_value(dm, rf_path, 0xa, efuse_value);
+	_set_tx_a_cali_value(dm, rf_path, 0xb, efuse_value);
+
+	/* switch off set TxA bias */
+	odm_set_rf_reg(dm, rf_path, 0xEF, 0xFFFFF, 0x0);
+}
+
+/*
+ * for 8822B PCIE D-cut patch only
+ * Normal driver and MP driver need this patch
+ */
+
+void phydm_txcurrentcalibration(struct phy_dm_struct *dm)
+{
+	u8 efuse0x3D8, efuse0x3D7;
+	u32 orig_rf0x18_path_a = 0, orig_rf0x18_path_b = 0;
+
+	/* save original 0x18 value */
+	orig_rf0x18_path_a = odm_get_rf_reg(dm, ODM_RF_PATH_A, 0x18, 0xFFFFF);
+	orig_rf0x18_path_b = odm_get_rf_reg(dm, ODM_RF_PATH_B, 0x18, 0xFFFFF);
+
+	/* define efuse content */
+	efuse0x3D8 = dm->efuse0x3d8;
+	efuse0x3D7 = dm->efuse0x3d7;
+
+	/* check efuse content to judge whether need to calibration or not */
+	if (efuse0x3D7 == 0xFF) {
+		ODM_RT_TRACE(
+			dm, ODM_COMP_COMMON,
+			"efuse content 0x3D7 == 0xFF, No need to do TxA cali\n");
+		return;
+	}
+
+	/* write RF register for calibration */
+	_txa_bias_cali_4_each_path(dm, ODM_RF_PATH_A, efuse0x3D7);
+	_txa_bias_cali_4_each_path(dm, ODM_RF_PATH_B, efuse0x3D8);
+
+	/* restore original 0x18 value */
+	odm_set_rf_reg(dm, ODM_RF_PATH_A, 0x18, 0xFFFFF, orig_rf0x18_path_a);
+	odm_set_rf_reg(dm, ODM_RF_PATH_B, 0x18, 0xFFFFF, orig_rf0x18_path_b);
+}
+
+void phydm_hwsetting_8822b(struct phy_dm_struct *dm)
+{
+	phydm_dynamic_switch_htstf_mumimo_8822b(dm);
+}
diff --git a/drivers/staging/rtlwifi/phydm/rtl8822b/phydm_rtl8822b.h b/drivers/staging/rtlwifi/phydm/rtl8822b/phydm_rtl8822b.h
new file mode 100644
index 000000000000..af91a6f958ed
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/rtl8822b/phydm_rtl8822b.h
@@ -0,0 +1,30 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef __ODM_RTL8822B_H__
+#define __ODM_RTL8822B_H__
+
+void phydm_hwsetting_8822b(struct phy_dm_struct *dm);
+
+#endif /* #define __ODM_RTL8822B_H__ */
diff --git a/drivers/staging/rtlwifi/phydm/rtl8822b/version_rtl8822b.h b/drivers/staging/rtlwifi/phydm/rtl8822b/version_rtl8822b.h
new file mode 100644
index 000000000000..ad0d32fce0a9
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/rtl8822b/version_rtl8822b.h
@@ -0,0 +1,34 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+/*RTL8822B PHY Parameters*/
+/*
+ * [Caution]
+ * Since 01/Aug/2015, the commit rules will be simplified.
+ * You do not need to fill up the version.h anymore,
+ * only the maintenance supervisor fills it before formal release.
+ */
+#define RELEASE_DATE_8822B 20161103
+#define COMMIT_BY_8822B "BB_JOE"
+#define RELEASE_VERSION_8822B 67
diff --git a/drivers/staging/rtlwifi/phydm/rtl_phydm.c b/drivers/staging/rtlwifi/phydm/rtl_phydm.c
new file mode 100644
index 000000000000..85e490d3601f
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/rtl_phydm.c
@@ -0,0 +1,874 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#include "mp_precomp.h"
+#include "phydm_precomp.h"
+#include <linux/module.h>
+
+static int _rtl_phydm_init_com_info(struct rtl_priv *rtlpriv,
+				    enum odm_ic_type ic_type,
+				    struct rtl_phydm_params *params)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	struct rtl_mac *mac = rtl_mac(rtlpriv);
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtlpriv);
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtlpriv);
+	u8 odm_board_type = ODM_BOARD_DEFAULT;
+	u32 support_ability;
+	int i;
+
+	dm->adapter = (void *)rtlpriv;
+
+	odm_cmn_info_init(dm, ODM_CMNINFO_PLATFORM, ODM_CE);
+
+	odm_cmn_info_init(dm, ODM_CMNINFO_IC_TYPE, ic_type);
+
+	odm_cmn_info_init(dm, ODM_CMNINFO_INTERFACE, ODM_ITRF_PCIE);
+
+	odm_cmn_info_init(dm, ODM_CMNINFO_MP_TEST_CHIP, params->mp_chip);
+
+	odm_cmn_info_init(dm, ODM_CMNINFO_PATCH_ID, rtlhal->oem_id);
+
+	odm_cmn_info_init(dm, ODM_CMNINFO_BWIFI_TEST, 1);
+
+	if (rtlphy->rf_type == RF_1T1R)
+		odm_cmn_info_init(dm, ODM_CMNINFO_RF_TYPE, ODM_1T1R);
+	else if (rtlphy->rf_type == RF_1T2R)
+		odm_cmn_info_init(dm, ODM_CMNINFO_RF_TYPE, ODM_1T2R);
+	else if (rtlphy->rf_type == RF_2T2R)
+		odm_cmn_info_init(dm, ODM_CMNINFO_RF_TYPE, ODM_2T2R);
+	else if (rtlphy->rf_type == RF_2T2R_GREEN)
+		odm_cmn_info_init(dm, ODM_CMNINFO_RF_TYPE, ODM_2T2R_GREEN);
+	else if (rtlphy->rf_type == RF_2T3R)
+		odm_cmn_info_init(dm, ODM_CMNINFO_RF_TYPE, ODM_2T3R);
+	else if (rtlphy->rf_type == RF_2T4R)
+		odm_cmn_info_init(dm, ODM_CMNINFO_RF_TYPE, ODM_2T4R);
+	else if (rtlphy->rf_type == RF_3T3R)
+		odm_cmn_info_init(dm, ODM_CMNINFO_RF_TYPE, ODM_3T3R);
+	else if (rtlphy->rf_type == RF_3T4R)
+		odm_cmn_info_init(dm, ODM_CMNINFO_RF_TYPE, ODM_3T4R);
+	else if (rtlphy->rf_type == RF_4T4R)
+		odm_cmn_info_init(dm, ODM_CMNINFO_RF_TYPE, ODM_4T4R);
+	else
+		odm_cmn_info_init(dm, ODM_CMNINFO_RF_TYPE, ODM_XTXR);
+
+	/* 1 ======= BoardType: ODM_CMNINFO_BOARD_TYPE ======= */
+	if (rtlhal->external_lna_2g != 0) {
+		odm_board_type |= ODM_BOARD_EXT_LNA;
+		odm_cmn_info_init(dm, ODM_CMNINFO_EXT_LNA, 1);
+	}
+	if (rtlhal->external_lna_5g != 0) {
+		odm_board_type |= ODM_BOARD_EXT_LNA_5G;
+		odm_cmn_info_init(dm, ODM_CMNINFO_5G_EXT_LNA, 1);
+	}
+	if (rtlhal->external_pa_2g != 0) {
+		odm_board_type |= ODM_BOARD_EXT_PA;
+		odm_cmn_info_init(dm, ODM_CMNINFO_EXT_PA, 1);
+	}
+	if (rtlhal->external_pa_5g != 0) {
+		odm_board_type |= ODM_BOARD_EXT_PA_5G;
+		odm_cmn_info_init(dm, ODM_CMNINFO_5G_EXT_PA, 1);
+	}
+	if (rtlpriv->cfg->ops->get_btc_status())
+		odm_board_type |= ODM_BOARD_BT;
+
+	odm_cmn_info_init(dm, ODM_CMNINFO_BOARD_TYPE, odm_board_type);
+	/* 1 ============== End of BoardType ============== */
+
+	odm_cmn_info_init(dm, ODM_CMNINFO_GPA, rtlhal->type_gpa);
+	odm_cmn_info_init(dm, ODM_CMNINFO_APA, rtlhal->type_apa);
+	odm_cmn_info_init(dm, ODM_CMNINFO_GLNA, rtlhal->type_glna);
+	odm_cmn_info_init(dm, ODM_CMNINFO_ALNA, rtlhal->type_alna);
+
+	odm_cmn_info_init(dm, ODM_CMNINFO_RFE_TYPE, rtlhal->rfe_type);
+
+	odm_cmn_info_init(dm, ODM_CMNINFO_EXT_TRSW, 0);
+
+	/*Add by YuChen for kfree init*/
+	odm_cmn_info_init(dm, ODM_CMNINFO_REGRFKFREEENABLE, 2);
+	odm_cmn_info_init(dm, ODM_CMNINFO_RFKFREEENABLE, 0);
+
+	/*Antenna diversity relative parameters*/
+	odm_cmn_info_hook(dm, ODM_CMNINFO_ANT_DIV,
+			  &rtlefuse->antenna_div_cfg);
+	odm_cmn_info_init(dm, ODM_CMNINFO_RF_ANTENNA_TYPE,
+			  rtlefuse->antenna_div_type);
+	odm_cmn_info_init(dm, ODM_CMNINFO_BE_FIX_TX_ANT, 0);
+	odm_cmn_info_init(dm, ODM_CMNINFO_WITH_EXT_ANTENNA_SWITCH, 0);
+
+	/* (8822B) efuse 0x3D7 & 0x3D8 for TX PA bias */
+	odm_cmn_info_init(dm, ODM_CMNINFO_EFUSE0X3D7, params->efuse0x3d7);
+	odm_cmn_info_init(dm, ODM_CMNINFO_EFUSE0X3D8, params->efuse0x3d8);
+
+	/*Add by YuChen for adaptivity init*/
+	odm_cmn_info_hook(dm, ODM_CMNINFO_ADAPTIVITY,
+			  &rtlpriv->phydm.adaptivity_en);
+	phydm_adaptivity_info_init(dm, PHYDM_ADAPINFO_CARRIER_SENSE_ENABLE,
+				   false);
+	phydm_adaptivity_info_init(dm, PHYDM_ADAPINFO_DCBACKOFF, 0);
+	phydm_adaptivity_info_init(dm, PHYDM_ADAPINFO_DYNAMICLINKADAPTIVITY,
+				   false);
+	phydm_adaptivity_info_init(dm, PHYDM_ADAPINFO_TH_L2H_INI, 0);
+	phydm_adaptivity_info_init(dm, PHYDM_ADAPINFO_TH_EDCCA_HL_DIFF, 0);
+
+	odm_cmn_info_init(dm, ODM_CMNINFO_IQKFWOFFLOAD, 0);
+
+	/* Pointer reference */
+	odm_cmn_info_hook(dm, ODM_CMNINFO_TX_UNI,
+			  &rtlpriv->stats.txbytesunicast);
+	odm_cmn_info_hook(dm, ODM_CMNINFO_RX_UNI,
+			  &rtlpriv->stats.rxbytesunicast);
+	odm_cmn_info_hook(dm, ODM_CMNINFO_BAND, &rtlhal->current_bandtype);
+	odm_cmn_info_hook(dm, ODM_CMNINFO_FORCED_RATE,
+			  &rtlpriv->phydm.forced_data_rate);
+	odm_cmn_info_hook(dm, ODM_CMNINFO_FORCED_IGI_LB,
+			  &rtlpriv->phydm.forced_igi_lb);
+
+	odm_cmn_info_hook(dm, ODM_CMNINFO_SEC_CHNL_OFFSET,
+			  &mac->cur_40_prime_sc);
+	odm_cmn_info_hook(dm, ODM_CMNINFO_BW, &rtlphy->current_chan_bw);
+	odm_cmn_info_hook(dm, ODM_CMNINFO_CHNL, &rtlphy->current_channel);
+
+	odm_cmn_info_hook(dm, ODM_CMNINFO_SCAN, &mac->act_scanning);
+	odm_cmn_info_hook(dm, ODM_CMNINFO_POWER_SAVING,
+			  &ppsc->dot11_psmode); /* may add new boolean flag */
+	/*Add by Yuchen for phydm beamforming*/
+	odm_cmn_info_hook(dm, ODM_CMNINFO_TX_TP,
+			  &rtlpriv->stats.txbytesunicast_inperiod_tp);
+	odm_cmn_info_hook(dm, ODM_CMNINFO_RX_TP,
+			  &rtlpriv->stats.rxbytesunicast_inperiod_tp);
+	odm_cmn_info_hook(dm, ODM_CMNINFO_ANT_TEST,
+			  &rtlpriv->phydm.antenna_test);
+	for (i = 0; i < ODM_ASSOCIATE_ENTRY_NUM; i++)
+		odm_cmn_info_ptr_array_hook(dm, ODM_CMNINFO_STA_STATUS, i,
+					    NULL);
+
+	phydm_init_debug_setting(dm);
+
+	odm_cmn_info_init(dm, ODM_CMNINFO_FAB_VER, params->fab_ver);
+	odm_cmn_info_init(dm, ODM_CMNINFO_CUT_VER, params->cut_ver);
+
+	/* after ifup, ability is updated again */
+	support_ability = ODM_RF_CALIBRATION | ODM_RF_TX_PWR_TRACK;
+	odm_cmn_info_update(dm, ODM_CMNINFO_ABILITY, support_ability);
+
+	return 0;
+}
+
+static int rtl_phydm_init_priv(struct rtl_priv *rtlpriv,
+			       struct rtl_phydm_params *params)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+	enum odm_ic_type ic;
+
+	if (IS_HARDWARE_TYPE_8822B(rtlpriv))
+		ic = ODM_RTL8822B;
+	else
+		return 0;
+
+	rtlpriv->phydm.internal =
+		kzalloc(sizeof(struct phy_dm_struct), GFP_KERNEL);
+
+	_rtl_phydm_init_com_info(rtlpriv, ic, params);
+
+	odm_init_all_timers(dm);
+
+	return 1;
+}
+
+static int rtl_phydm_deinit_priv(struct rtl_priv *rtlpriv)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+
+	odm_cancel_all_timers(dm);
+
+	kfree(rtlpriv->phydm.internal);
+	rtlpriv->phydm.internal = NULL;
+
+	return 0;
+}
+
+static bool rtl_phydm_load_txpower_by_rate(struct rtl_priv *rtlpriv)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+	enum hal_status status;
+
+	status = odm_config_bb_with_header_file(dm, CONFIG_BB_PHY_REG_PG);
+	if (status != HAL_STATUS_SUCCESS)
+		return false;
+
+	return true;
+}
+
+static bool rtl_phydm_load_txpower_limit(struct rtl_priv *rtlpriv)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+	enum hal_status status;
+
+	if (IS_HARDWARE_TYPE_8822B(rtlpriv)) {
+		odm_read_and_config_mp_8822b_txpwr_lmt(dm);
+	} else {
+		status = odm_config_rf_with_header_file(dm, CONFIG_RF_TXPWR_LMT,
+							0);
+		if (status != HAL_STATUS_SUCCESS)
+			return false;
+	}
+
+	return true;
+}
+
+static int rtl_phydm_init_dm(struct rtl_priv *rtlpriv)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+	u32 support_ability = 0;
+
+	/* clang-format off */
+	support_ability = 0
+			| ODM_BB_DIG
+			| ODM_BB_RA_MASK
+			| ODM_BB_DYNAMIC_TXPWR
+			| ODM_BB_FA_CNT
+			| ODM_BB_RSSI_MONITOR
+			| ODM_BB_CCK_PD
+	/*		| ODM_BB_PWR_SAVE*/
+			| ODM_BB_CFO_TRACKING
+			| ODM_MAC_EDCA_TURBO
+			| ODM_RF_TX_PWR_TRACK
+			| ODM_RF_CALIBRATION
+			| ODM_BB_NHM_CNT
+	/*		| ODM_BB_PWR_TRAIN*/
+			;
+	/* clang-format on */
+
+	odm_cmn_info_update(dm, ODM_CMNINFO_ABILITY, support_ability);
+
+	odm_dm_init(dm);
+
+	return 0;
+}
+
+static int rtl_phydm_deinit_dm(struct rtl_priv *rtlpriv)
+{
+	return 0;
+}
+
+static int rtl_phydm_reset_dm(struct rtl_priv *rtlpriv)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+
+	odm_dm_reset(dm);
+
+	return 0;
+}
+
+static bool rtl_phydm_parameter_init(struct rtl_priv *rtlpriv, bool post)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+
+	if (IS_HARDWARE_TYPE_8822B(rtlpriv))
+		return config_phydm_parameter_init(dm, post ? ODM_POST_SETTING :
+							      ODM_PRE_SETTING);
+
+	return false;
+}
+
+static bool rtl_phydm_phy_bb_config(struct rtl_priv *rtlpriv)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+	enum hal_status status;
+
+	status = odm_config_bb_with_header_file(dm, CONFIG_BB_PHY_REG);
+	if (status != HAL_STATUS_SUCCESS)
+		return false;
+
+	status = odm_config_bb_with_header_file(dm, CONFIG_BB_AGC_TAB);
+	if (status != HAL_STATUS_SUCCESS)
+		return false;
+
+	return true;
+}
+
+static bool rtl_phydm_phy_rf_config(struct rtl_priv *rtlpriv)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	enum hal_status status;
+	enum odm_rf_radio_path rfpath;
+
+	for (rfpath = 0; rfpath < rtlphy->num_total_rfpath; rfpath++) {
+		status = odm_config_rf_with_header_file(dm, CONFIG_RF_RADIO,
+							rfpath);
+		if (status != HAL_STATUS_SUCCESS)
+			return false;
+	}
+
+	status = odm_config_rf_with_tx_pwr_track_header_file(dm);
+	if (status != HAL_STATUS_SUCCESS)
+		return false;
+
+	return true;
+}
+
+static bool rtl_phydm_phy_mac_config(struct rtl_priv *rtlpriv)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+	enum hal_status status;
+
+	status = odm_config_mac_with_header_file(dm);
+	if (status != HAL_STATUS_SUCCESS)
+		return false;
+
+	return true;
+}
+
+static bool rtl_phydm_trx_mode(struct rtl_priv *rtlpriv,
+			       enum radio_mask tx_path, enum radio_mask rx_path,
+			       bool is_tx2_path)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+
+	if (IS_HARDWARE_TYPE_8822B(rtlpriv))
+		return config_phydm_trx_mode_8822b(dm,
+						   (enum odm_rf_path)tx_path,
+						   (enum odm_rf_path)rx_path,
+						   is_tx2_path);
+
+	return false;
+}
+
+static bool rtl_phydm_watchdog(struct rtl_priv *rtlpriv)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+	struct rtl_mac *mac = rtl_mac(rtlpriv);
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtlpriv);
+	bool fw_current_inpsmode = false;
+	bool fw_ps_awake = true;
+	u8 is_linked = false;
+	u8 bsta_state = false;
+	u8 is_bt_enabled = false;
+
+	/* check whether do watchdog */
+	rtlpriv->cfg->ops->get_hw_reg(rtlpriv->hw, HW_VAR_FW_PSMODE_STATUS,
+				      (u8 *)(&fw_current_inpsmode));
+	rtlpriv->cfg->ops->get_hw_reg(rtlpriv->hw, HW_VAR_FWLPS_RF_ON,
+				      (u8 *)(&fw_ps_awake));
+	if (ppsc->p2p_ps_info.p2p_ps_mode)
+		fw_ps_awake = false;
+
+	if ((ppsc->rfpwr_state == ERFON) &&
+	    ((!fw_current_inpsmode) && fw_ps_awake) &&
+	    (!ppsc->rfchange_inprogress))
+		;
+	else
+		return false;
+
+	/* update common info before doing watchdog */
+	if (mac->link_state >= MAC80211_LINKED) {
+		is_linked = true;
+		if (mac->vif && mac->vif->type == NL80211_IFTYPE_STATION)
+			bsta_state = true;
+	}
+
+	if (rtlpriv->cfg->ops->get_btc_status())
+		is_bt_enabled = !rtlpriv->btcoexist.btc_ops->btc_is_bt_disabled(
+			rtlpriv);
+
+	odm_cmn_info_update(dm, ODM_CMNINFO_LINK, is_linked);
+	odm_cmn_info_update(dm, ODM_CMNINFO_STATION_STATE, bsta_state);
+	odm_cmn_info_update(dm, ODM_CMNINFO_BT_ENABLED, is_bt_enabled);
+
+	/* do watchdog */
+	odm_dm_watchdog(dm);
+
+	return true;
+}
+
+static bool rtl_phydm_switch_band(struct rtl_priv *rtlpriv, u8 central_ch)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+
+	if (IS_HARDWARE_TYPE_8822B(rtlpriv))
+		return config_phydm_switch_band_8822b(dm, central_ch);
+
+	return false;
+}
+
+static bool rtl_phydm_switch_channel(struct rtl_priv *rtlpriv, u8 central_ch)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+
+	if (IS_HARDWARE_TYPE_8822B(rtlpriv))
+		return config_phydm_switch_channel_8822b(dm, central_ch);
+
+	return false;
+}
+
+static bool rtl_phydm_switch_bandwidth(struct rtl_priv *rtlpriv,
+				       u8 primary_ch_idx,
+				       enum ht_channel_width bandwidth)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+	enum odm_bw odm_bw = (enum odm_bw)bandwidth;
+
+	if (IS_HARDWARE_TYPE_8822B(rtlpriv))
+		return config_phydm_switch_bandwidth_8822b(dm, primary_ch_idx,
+							   odm_bw);
+
+	return false;
+}
+
+static bool rtl_phydm_iq_calibrate(struct rtl_priv *rtlpriv)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+
+	if (IS_HARDWARE_TYPE_8822B(rtlpriv))
+		phy_iq_calibrate_8822b(dm, false);
+	else
+		return false;
+
+	return true;
+}
+
+static bool rtl_phydm_clear_txpowertracking_state(struct rtl_priv *rtlpriv)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+
+	odm_clear_txpowertracking_state(dm);
+
+	return true;
+}
+
+static bool rtl_phydm_pause_dig(struct rtl_priv *rtlpriv, bool pause)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+
+	if (pause)
+		odm_pause_dig(dm, PHYDM_PAUSE, PHYDM_PAUSE_LEVEL_0, 0x1e);
+	else /* resume */
+		odm_pause_dig(dm, PHYDM_RESUME, PHYDM_PAUSE_LEVEL_0, 0xff);
+
+	return true;
+}
+
+static u32 rtl_phydm_read_rf_reg(struct rtl_priv *rtlpriv,
+				 enum radio_path rfpath, u32 addr, u32 mask)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+	enum odm_rf_radio_path odm_rfpath = (enum odm_rf_radio_path)rfpath;
+
+	if (IS_HARDWARE_TYPE_8822B(rtlpriv))
+		return config_phydm_read_rf_reg_8822b(dm, odm_rfpath, addr,
+						      mask);
+
+	return -1;
+}
+
+static bool rtl_phydm_write_rf_reg(struct rtl_priv *rtlpriv,
+				   enum radio_path rfpath, u32 addr, u32 mask,
+				   u32 data)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+	enum odm_rf_radio_path odm_rfpath = (enum odm_rf_radio_path)rfpath;
+
+	if (IS_HARDWARE_TYPE_8822B(rtlpriv))
+		return config_phydm_write_rf_reg_8822b(dm, odm_rfpath, addr,
+						       mask, data);
+
+	return false;
+}
+
+static u8 rtl_phydm_read_txagc(struct rtl_priv *rtlpriv, enum radio_path rfpath,
+			       u8 hw_rate)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+	enum odm_rf_radio_path odm_rfpath = (enum odm_rf_radio_path)rfpath;
+
+	if (IS_HARDWARE_TYPE_8822B(rtlpriv))
+		return config_phydm_read_txagc_8822b(dm, odm_rfpath, hw_rate);
+
+	return -1;
+}
+
+static bool rtl_phydm_write_txagc(struct rtl_priv *rtlpriv, u32 power_index,
+				  enum radio_path rfpath, u8 hw_rate)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+	enum odm_rf_radio_path odm_rfpath = (enum odm_rf_radio_path)rfpath;
+
+	if (IS_HARDWARE_TYPE_8822B(rtlpriv))
+		return config_phydm_write_txagc_8822b(dm, power_index,
+						      odm_rfpath, hw_rate);
+
+	return false;
+}
+
+static bool rtl_phydm_c2h_content_parsing(struct rtl_priv *rtlpriv, u8 cmd_id,
+					  u8 cmd_len, u8 *content)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+
+	if (phydm_c2H_content_parsing(dm, cmd_id, cmd_len, content))
+		return true;
+
+	return false;
+}
+
+static bool rtl_phydm_query_phy_status(struct rtl_priv *rtlpriv, u8 *phystrpt,
+				       struct ieee80211_hdr *hdr,
+				       struct rtl_stats *pstatus)
+{
+	/* NOTE: phystrpt may be NULL, and need to fill default value */
+
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtlpriv);
+	struct rtl_mac *mac = rtl_mac(rtlpriv);
+	struct dm_per_pkt_info pktinfo; /* input of pydm */
+	struct dm_phy_status_info phy_info; /* output of phydm */
+	__le16 fc = hdr->frame_control;
+
+	/* fill driver pstatus */
+	ether_addr_copy(pstatus->psaddr, ieee80211_get_SA(hdr));
+
+	/* fill pktinfo */
+	memset(&pktinfo, 0, sizeof(pktinfo));
+
+	pktinfo.data_rate = pstatus->rate;
+
+	if (rtlpriv->mac80211.opmode == NL80211_IFTYPE_STATION) {
+		pktinfo.station_id = 0;
+	} else {
+		/* TODO: use rtl_find_sta() to find ID */
+		pktinfo.station_id = 0xFF;
+	}
+
+	pktinfo.is_packet_match_bssid =
+		(!ieee80211_is_ctl(fc) &&
+		 (ether_addr_equal(mac->bssid,
+				   ieee80211_has_tods(fc) ?
+					   hdr->addr1 :
+					   ieee80211_has_fromds(fc) ?
+					   hdr->addr2 :
+					   hdr->addr3)) &&
+		 (!pstatus->hwerror) && (!pstatus->crc) && (!pstatus->icv));
+	pktinfo.is_packet_to_self =
+		pktinfo.is_packet_match_bssid &&
+		(ether_addr_equal(hdr->addr1, rtlefuse->dev_addr));
+	pktinfo.is_to_self = (!pstatus->icv) && (!pstatus->crc) &&
+			     (ether_addr_equal(hdr->addr1, rtlefuse->dev_addr));
+	pktinfo.is_packet_beacon = (ieee80211_is_beacon(fc) ? true : false);
+
+	/* query phy status */
+	if (phystrpt)
+		odm_phy_status_query(dm, &phy_info, phystrpt, &pktinfo);
+	else
+		memset(&phy_info, 0, sizeof(phy_info));
+
+	/* copy phy_info from phydm to driver */
+	pstatus->rx_pwdb_all = phy_info.rx_pwdb_all;
+	pstatus->bt_rx_rssi_percentage = phy_info.bt_rx_rssi_percentage;
+	pstatus->recvsignalpower = phy_info.recv_signal_power;
+	pstatus->signalquality = phy_info.signal_quality;
+	pstatus->rx_mimo_signalquality[0] = phy_info.rx_mimo_signal_quality[0];
+	pstatus->rx_mimo_signalquality[1] = phy_info.rx_mimo_signal_quality[1];
+	pstatus->rx_packet_bw =
+		phy_info.band_width; /* HT_CHANNEL_WIDTH_20 <- ODM_BW20M */
+
+	/* fill driver pstatus */
+	pstatus->packet_matchbssid = pktinfo.is_packet_match_bssid;
+	pstatus->packet_toself = pktinfo.is_packet_to_self;
+	pstatus->packet_beacon = pktinfo.is_packet_beacon;
+
+	return true;
+}
+
+static u8 rtl_phydm_rate_id_mapping(struct rtl_priv *rtlpriv,
+				    enum wireless_mode wireless_mode,
+				    enum rf_type rf_type,
+				    enum ht_channel_width bw)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+
+	return phydm_rate_id_mapping(dm, wireless_mode, rf_type, bw);
+}
+
+static bool rtl_phydm_get_ra_bitmap(struct rtl_priv *rtlpriv,
+				    enum wireless_mode wireless_mode,
+				    enum rf_type rf_type,
+				    enum ht_channel_width bw,
+				    u8 tx_rate_level, /* 0~6 */
+				    u32 *tx_bitmap_msb,
+				    u32 *tx_bitmap_lsb)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+	const u8 mimo_ps_enable = 0;
+	const u8 disable_cck_rate = 0;
+
+	phydm_update_hal_ra_mask(dm, wireless_mode, rf_type, bw, mimo_ps_enable,
+				 disable_cck_rate, tx_bitmap_msb, tx_bitmap_lsb,
+				 tx_rate_level);
+
+	return true;
+}
+
+static u8 _rtl_phydm_get_macid(struct rtl_priv *rtlpriv,
+			       struct ieee80211_sta *sta)
+{
+	struct rtl_mac *mac = rtl_mac(rtlpriv);
+
+	if (mac->opmode == NL80211_IFTYPE_STATION ||
+	    mac->opmode == NL80211_IFTYPE_MESH_POINT) {
+		return 0;
+	} else if (mac->opmode == NL80211_IFTYPE_AP ||
+		   mac->opmode == NL80211_IFTYPE_ADHOC)
+		return sta->aid + 1;
+
+	return 0;
+}
+
+static bool rtl_phydm_add_sta(struct rtl_priv *rtlpriv,
+			      struct ieee80211_sta *sta)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+	struct rtl_sta_info *sta_entry = (struct rtl_sta_info *)sta->drv_priv;
+	u8 mac_id = _rtl_phydm_get_macid(rtlpriv, sta);
+
+	odm_cmn_info_ptr_array_hook(dm, ODM_CMNINFO_STA_STATUS, mac_id,
+				    sta_entry);
+
+	return true;
+}
+
+static bool rtl_phydm_del_sta(struct rtl_priv *rtlpriv,
+			      struct ieee80211_sta *sta)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+	u8 mac_id = _rtl_phydm_get_macid(rtlpriv, sta);
+
+	odm_cmn_info_ptr_array_hook(dm, ODM_CMNINFO_STA_STATUS, mac_id, NULL);
+
+	return true;
+}
+
+static u32 rtl_phydm_get_version(struct rtl_priv *rtlpriv)
+{
+	u32 ver = 0;
+
+	if (IS_HARDWARE_TYPE_8822B(rtlpriv))
+		ver = RELEASE_VERSION_8822B;
+
+	return ver;
+}
+
+static bool rtl_phydm_modify_ra_pcr_threshold(struct rtl_priv *rtlpriv,
+					      u8 ra_offset_direction,
+					      u8 ra_threshold_offset)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+
+	phydm_modify_RA_PCR_threshold(dm, ra_offset_direction,
+				      ra_threshold_offset);
+
+	return true;
+}
+
+static u32 rtl_phydm_query_counter(struct rtl_priv *rtlpriv,
+				   const char *info_type)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+	static const struct query_entry {
+		const char *query_name;
+		enum phydm_info_query query_id;
+	} query_table[] = {
+#define QUERY_ENTRY(name)	{#name, name}
+		QUERY_ENTRY(PHYDM_INFO_FA_OFDM),
+		QUERY_ENTRY(PHYDM_INFO_FA_CCK),
+		QUERY_ENTRY(PHYDM_INFO_CCA_OFDM),
+		QUERY_ENTRY(PHYDM_INFO_CCA_CCK),
+		QUERY_ENTRY(PHYDM_INFO_CRC32_OK_CCK),
+		QUERY_ENTRY(PHYDM_INFO_CRC32_OK_LEGACY),
+		QUERY_ENTRY(PHYDM_INFO_CRC32_OK_HT),
+		QUERY_ENTRY(PHYDM_INFO_CRC32_OK_VHT),
+		QUERY_ENTRY(PHYDM_INFO_CRC32_ERROR_CCK),
+		QUERY_ENTRY(PHYDM_INFO_CRC32_ERROR_LEGACY),
+		QUERY_ENTRY(PHYDM_INFO_CRC32_ERROR_HT),
+		QUERY_ENTRY(PHYDM_INFO_CRC32_ERROR_VHT),
+	};
+#define QUERY_TABLE_SIZE	ARRAY_SIZE(query_table)
+
+	int i;
+	const struct query_entry *entry;
+
+	if (!strcmp(info_type, "IQK_TOTAL"))
+		return dm->n_iqk_cnt;
+
+	if (!strcmp(info_type, "IQK_OK"))
+		return dm->n_iqk_ok_cnt;
+
+	if (!strcmp(info_type, "IQK_FAIL"))
+		return dm->n_iqk_fail_cnt;
+
+	for (i = 0; i < QUERY_TABLE_SIZE; i++) {
+		entry = &query_table[i];
+
+		if (!strcmp(info_type, entry->query_name))
+			return phydm_cmn_info_query(dm, entry->query_id);
+	}
+
+	pr_err("Unrecognized info_type:%s!!!!:\n", info_type);
+
+	return 0xDEADDEAD;
+}
+
+static bool rtl_phydm_debug_cmd(struct rtl_priv *rtlpriv, char *in, u32 in_len,
+				char *out, u32 out_len)
+{
+	struct phy_dm_struct *dm = rtlpriv_to_phydm(rtlpriv);
+
+	phydm_cmd(dm, in, in_len, 1, out, out_len);
+
+	return true;
+}
+
+static struct rtl_phydm_ops rtl_phydm_operation = {
+	/* init/deinit priv */
+	.phydm_init_priv = rtl_phydm_init_priv,
+	.phydm_deinit_priv = rtl_phydm_deinit_priv,
+	.phydm_load_txpower_by_rate = rtl_phydm_load_txpower_by_rate,
+	.phydm_load_txpower_limit = rtl_phydm_load_txpower_limit,
+
+	/* init hw */
+	.phydm_init_dm = rtl_phydm_init_dm,
+	.phydm_deinit_dm = rtl_phydm_deinit_dm,
+	.phydm_reset_dm = rtl_phydm_reset_dm,
+	.phydm_parameter_init = rtl_phydm_parameter_init,
+	.phydm_phy_bb_config = rtl_phydm_phy_bb_config,
+	.phydm_phy_rf_config = rtl_phydm_phy_rf_config,
+	.phydm_phy_mac_config = rtl_phydm_phy_mac_config,
+	.phydm_trx_mode = rtl_phydm_trx_mode,
+
+	/* watchdog */
+	.phydm_watchdog = rtl_phydm_watchdog,
+
+	/* channel */
+	.phydm_switch_band = rtl_phydm_switch_band,
+	.phydm_switch_channel = rtl_phydm_switch_channel,
+	.phydm_switch_bandwidth = rtl_phydm_switch_bandwidth,
+	.phydm_iq_calibrate = rtl_phydm_iq_calibrate,
+	.phydm_clear_txpowertracking_state =
+		rtl_phydm_clear_txpowertracking_state,
+	.phydm_pause_dig = rtl_phydm_pause_dig,
+
+	/* read/write reg */
+	.phydm_read_rf_reg = rtl_phydm_read_rf_reg,
+	.phydm_write_rf_reg = rtl_phydm_write_rf_reg,
+	.phydm_read_txagc = rtl_phydm_read_txagc,
+	.phydm_write_txagc = rtl_phydm_write_txagc,
+
+	/* RX */
+	.phydm_c2h_content_parsing = rtl_phydm_c2h_content_parsing,
+	.phydm_query_phy_status = rtl_phydm_query_phy_status,
+
+	/* TX */
+	.phydm_rate_id_mapping = rtl_phydm_rate_id_mapping,
+	.phydm_get_ra_bitmap = rtl_phydm_get_ra_bitmap,
+
+	/* STA */
+	.phydm_add_sta = rtl_phydm_add_sta,
+	.phydm_del_sta = rtl_phydm_del_sta,
+
+	/* BTC */
+	.phydm_get_version = rtl_phydm_get_version,
+	.phydm_modify_ra_pcr_threshold = rtl_phydm_modify_ra_pcr_threshold,
+	.phydm_query_counter = rtl_phydm_query_counter,
+
+	/* debug */
+	.phydm_debug_cmd = rtl_phydm_debug_cmd,
+};
+
+struct rtl_phydm_ops *rtl_phydm_get_ops_pointer(void)
+{
+	return &rtl_phydm_operation;
+}
+EXPORT_SYMBOL(rtl_phydm_get_ops_pointer);
+
+/* ********************************************************
+ * Define phydm callout function in below
+ * ********************************************************
+ */
+
+u8 phy_get_tx_power_index(void *adapter, u8 rf_path, u8 rate,
+			  enum ht_channel_width bandwidth, u8 channel)
+{
+	/* rate: DESC_RATE1M */
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)adapter;
+
+	return rtlpriv->cfg->ops->get_txpower_index(rtlpriv->hw, rf_path, rate,
+						    bandwidth, channel);
+}
+
+void phy_set_tx_power_index_by_rs(void *adapter, u8 ch, u8 path, u8 rs)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)adapter;
+
+	return rtlpriv->cfg->ops->set_tx_power_index_by_rs(rtlpriv->hw, ch,
+							   path, rs);
+}
+
+void phy_store_tx_power_by_rate(void *adapter, u32 band, u32 rfpath, u32 txnum,
+				u32 regaddr, u32 bitmask, u32 data)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)adapter;
+
+	rtlpriv->cfg->ops->store_tx_power_by_rate(
+		rtlpriv->hw, band, rfpath, txnum, regaddr, bitmask, data);
+}
+
+void phy_set_tx_power_limit(void *dm, u8 *regulation, u8 *band, u8 *bandwidth,
+			    u8 *rate_section, u8 *rf_path, u8 *channel,
+			    u8 *power_limit)
+{
+	struct rtl_priv *rtlpriv =
+		(struct rtl_priv *)((struct phy_dm_struct *)dm)->adapter;
+
+	rtlpriv->cfg->ops->phy_set_txpower_limit(rtlpriv->hw, regulation, band,
+						 bandwidth, rate_section,
+						 rf_path, channel, power_limit);
+}
+
+void rtl_hal_update_ra_mask(void *adapter, struct rtl_sta_info *psta,
+			    u8 rssi_level)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)adapter;
+	struct ieee80211_sta *sta =
+		container_of((void *)psta, struct ieee80211_sta, drv_priv);
+
+	rtlpriv->cfg->ops->update_rate_tbl(rtlpriv->hw, sta, rssi_level, false);
+}
+
+MODULE_AUTHOR("Realtek WlanFAE	<wlanfae@realtek.com>");
+MODULE_AUTHOR("Larry Finger	<Larry.FInger@lwfinger.net>");
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Realtek 802.11n PCI wireless core");
diff --git a/drivers/staging/rtlwifi/phydm/rtl_phydm.h b/drivers/staging/rtlwifi/phydm/rtl_phydm.h
new file mode 100644
index 000000000000..483d2418699b
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/rtl_phydm.h
@@ -0,0 +1,45 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef __RTL_PHYDM_H__
+#define __RTL_PHYDM_H__
+
+struct rtl_phydm_ops *rtl_phydm_get_ops_pointer(void);
+
+#define rtlpriv_to_phydm(priv)                                                 \
+	((struct phy_dm_struct *)((priv)->phydm.internal))
+
+u8 phy_get_tx_power_index(void *adapter, u8 rf_path, u8 rate,
+			  enum ht_channel_width bandwidth, u8 channel);
+void phy_set_tx_power_index_by_rs(void *adapter, u8 ch, u8 path, u8 rs);
+void phy_store_tx_power_by_rate(void *adapter, u32 band, u32 rfpath, u32 txnum,
+				u32 regaddr, u32 bitmask, u32 data);
+void phy_set_tx_power_limit(void *dm, u8 *regulation, u8 *band, u8 *bandwidth,
+			    u8 *rate_section, u8 *rf_path, u8 *channel,
+			    u8 *power_limit);
+
+void rtl_hal_update_ra_mask(void *adapter, struct rtl_sta_info *psta,
+			    u8 rssi_level);
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/txbf/halcomtxbf.h b/drivers/staging/rtlwifi/phydm/txbf/halcomtxbf.h
new file mode 100644
index 000000000000..6cacca12d792
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/txbf/halcomtxbf.h
@@ -0,0 +1,67 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef __HAL_COM_TXBF_H__
+#define __HAL_COM_TXBF_H__
+
+enum txbf_set_type {
+	TXBF_SET_SOUNDING_ENTER,
+	TXBF_SET_SOUNDING_LEAVE,
+	TXBF_SET_SOUNDING_RATE,
+	TXBF_SET_SOUNDING_STATUS,
+	TXBF_SET_SOUNDING_FW_NDPA,
+	TXBF_SET_SOUNDING_CLK,
+	TXBF_SET_TX_PATH_RESET,
+	TXBF_SET_GET_TX_RATE
+};
+
+enum txbf_get_type {
+	TXBF_GET_EXPLICIT_BEAMFORMEE,
+	TXBF_GET_EXPLICIT_BEAMFORMER,
+	TXBF_GET_MU_MIMO_STA,
+	TXBF_GET_MU_MIMO_AP
+};
+
+/* 2 HAL TXBF related */
+struct _HAL_TXBF_INFO {
+	u8 txbf_idx;
+	u8 ndpa_idx;
+	u8 BW;
+	u8 rate;
+
+	struct timer_list txbf_fw_ndpa_timer;
+};
+
+#define hal_com_txbf_beamform_init(dm_void) NULL
+#define hal_com_txbf_config_gtab(dm_void) NULL
+#define hal_com_txbf_enter_work_item_callback(_adapter) NULL
+#define hal_com_txbf_leave_work_item_callback(_adapter) NULL
+#define hal_com_txbf_fw_ndpa_work_item_callback(_adapter) NULL
+#define hal_com_txbf_clk_work_item_callback(_adapter) NULL
+#define hal_com_txbf_rate_work_item_callback(_adapter) NULL
+#define hal_com_txbf_fw_ndpa_timer_callback(_adapter) NULL
+#define hal_com_txbf_status_work_item_callback(_adapter) NULL
+#define hal_com_txbf_get(_adapter, _get_type, _pout_buf)
+
+#endif /*  #ifndef __HAL_COM_TXBF_H__ */
diff --git a/drivers/staging/rtlwifi/phydm/txbf/haltxbf8822b.h b/drivers/staging/rtlwifi/phydm/txbf/haltxbf8822b.h
new file mode 100644
index 000000000000..5c92c4326f7e
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/txbf/haltxbf8822b.h
@@ -0,0 +1,39 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef __HAL_TXBF_8822B_H__
+#define __HAL_TXBF_8822B_H__
+
+#define hal_txbf_8822b_enter(dm_void, idx)
+#define hal_txbf_8822b_leave(dm_void, idx)
+#define hal_txbf_8822b_status(dm_void, idx)
+#define hal_txbf_8822b_fw_txbf(dm_void, idx)
+#define hal_txbf_8822b_config_gtab(dm_void)
+
+void phydm_8822btxbf_rfmode(void *dm_void, u8 su_bfee_cnt, u8 mu_bfee_cnt);
+
+void phydm_8822b_sutxbfer_workaroud(void *dm_void, bool enable_su_bfer, u8 nc,
+				    u8 nr, u8 ng, u8 CB, u8 BW, bool is_vht);
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/txbf/haltxbfinterface.h b/drivers/staging/rtlwifi/phydm/txbf/haltxbfinterface.h
new file mode 100644
index 000000000000..82aeac1ff3e0
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/txbf/haltxbfinterface.h
@@ -0,0 +1,38 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef __HAL_TXBF_INTERFACE_H__
+#define __HAL_TXBF_INTERFACE_H__
+
+#define beamforming_get_ndpa_frame(dm, _pdu_os)
+#define beamforming_get_report_frame(adapter, precv_frame) RT_STATUS_FAILURE
+#define send_fw_ht_ndpa_packet(dm_void, RA, BW)
+#define send_sw_ht_ndpa_packet(dm_void, RA, BW)
+#define send_fw_vht_ndpa_packet(dm_void, RA, AID, BW)
+#define send_sw_vht_ndpa_packet(dm_void, RA, AID, BW)
+#define send_sw_vht_gid_mgnt_frame(dm_void, RA, idx)
+#define send_sw_vht_bf_report_poll(dm_void, RA, is_final_poll)
+#define send_sw_vht_mu_ndpa_packet(dm_void, BW)
+
+#endif
diff --git a/drivers/staging/rtlwifi/phydm/txbf/haltxbfjaguar.h b/drivers/staging/rtlwifi/phydm/txbf/haltxbfjaguar.h
new file mode 100644
index 000000000000..c5ddd9cb9cd5
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/txbf/haltxbfjaguar.h
@@ -0,0 +1,36 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef __HAL_TXBF_JAGUAR_H__
+#define __HAL_TXBF_JAGUAR_H__
+
+#define hal_txbf_8812a_set_ndpa_rate(dm_void, BW, rate)
+#define hal_txbf_jaguar_enter(dm_void, idx)
+#define hal_txbf_jaguar_leave(dm_void, idx)
+#define hal_txbf_jaguar_status(dm_void, idx)
+#define hal_txbf_jaguar_fw_txbf(dm_void, idx)
+#define hal_txbf_jaguar_patch(dm_void, operation)
+#define hal_txbf_jaguar_clk_8812a(dm_void)
+
+#endif /*  #ifndef __HAL_TXBF_JAGUAR_H__ */
diff --git a/drivers/staging/rtlwifi/phydm/txbf/phydm_hal_txbf_api.h b/drivers/staging/rtlwifi/phydm/txbf/phydm_hal_txbf_api.h
new file mode 100644
index 000000000000..41358fce2875
--- /dev/null
+++ b/drivers/staging/rtlwifi/phydm/txbf/phydm_hal_txbf_api.h
@@ -0,0 +1,41 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2007 - 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#ifndef __PHYDM_HAL_TXBF_API_H__
+#define __PHYDM_HAL_TXBF_API_H__
+
+#define tx_bf_nr(a, b) ((a > b) ? (b) : (a))
+
+u8 beamforming_get_htndp_tx_rate(void *dm_void, u8 comp_steering_num_of_bfer);
+
+u8 beamforming_get_vht_ndp_tx_rate(void *dm_void, u8 comp_steering_num_of_bfer);
+
+u8 phydm_get_beamforming_sounding_info(void *dm_void, u16 *troughput,
+				       u8 total_bfee_num, u8 *tx_rate);
+
+u8 phydm_get_ndpa_rate(void *dm_void);
+
+u8 phydm_get_mu_bfee_snding_decision(void *dm_void, u16 throughput);
+
+#endif
diff --git a/drivers/staging/rtlwifi/ps.c b/drivers/staging/rtlwifi/ps.c
new file mode 100644
index 000000000000..9172cee45f74
--- /dev/null
+++ b/drivers/staging/rtlwifi/ps.c
@@ -0,0 +1,1007 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2012  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#include "wifi.h"
+#include "base.h"
+#include "ps.h"
+#include <linux/export.h>
+#include "btcoexist/rtl_btc.h"
+
+bool rtl_ps_enable_nic(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	struct rtl_mac *rtlmac = rtl_mac(rtl_priv(hw));
+
+	/*<1> reset trx ring */
+	if (rtlhal->interface == INTF_PCI)
+		rtlpriv->intf_ops->reset_trx_ring(hw);
+
+	if (is_hal_stop(rtlhal))
+		RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING,
+			 "Driver is already down!\n");
+
+	/*<2> Enable Adapter */
+	if (rtlpriv->cfg->ops->hw_init(hw))
+		return false;
+	rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_RETRY_LIMIT,
+			&rtlmac->retry_long);
+	RT_CLEAR_PS_LEVEL(ppsc, RT_RF_OFF_LEVL_HALT_NIC);
+
+	/*<2.1> Switch Channel & Bandwidth to last rtl_op_config setting*/
+	rtlpriv->cfg->ops->switch_channel(hw);
+	rtlpriv->cfg->ops->set_channel_access(hw);
+	rtlpriv->cfg->ops->set_bw_mode(hw,
+			cfg80211_get_chandef_type(&hw->conf.chandef));
+
+	/*<3> Enable Interrupt */
+	rtlpriv->cfg->ops->enable_interrupt(hw);
+
+	/*<enable timer> */
+	rtl_watch_dog_timer_callback((unsigned long)hw);
+
+	return true;
+}
+
+bool rtl_ps_disable_nic(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	/*<1> Stop all timer */
+	rtl_deinit_deferred_work(hw);
+
+	/*<2> Disable Interrupt */
+	rtlpriv->cfg->ops->disable_interrupt(hw);
+	tasklet_kill(&rtlpriv->works.irq_tasklet);
+
+	/*<3> Disable Adapter */
+	rtlpriv->cfg->ops->hw_disable(hw);
+
+	return true;
+}
+
+static bool rtl_ps_set_rf_state(struct ieee80211_hw *hw,
+				enum rf_pwrstate state_toset,
+				u32 changesource)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	enum rf_pwrstate rtstate;
+	bool actionallowed = false;
+	u16 rfwait_cnt = 0;
+
+	/*Only one thread can change
+	 *the RF state at one time, and others
+	 *should wait to be executed.
+	 */
+	while (true) {
+		spin_lock(&rtlpriv->locks.rf_ps_lock);
+		if (ppsc->rfchange_inprogress) {
+			spin_unlock(&rtlpriv->locks.rf_ps_lock);
+
+			RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING,
+				 "RF Change in progress! Wait to set..state_toset(%d).\n",
+				  state_toset);
+
+			/* Set RF after the previous action is done.  */
+			while (ppsc->rfchange_inprogress) {
+				rfwait_cnt++;
+				mdelay(1);
+				/*Wait too long, return false to avoid
+				 *to be stuck here.
+				 */
+				if (rfwait_cnt > 100)
+					return false;
+			}
+		} else {
+			ppsc->rfchange_inprogress = true;
+			spin_unlock(&rtlpriv->locks.rf_ps_lock);
+			break;
+		}
+	}
+
+	rtstate = ppsc->rfpwr_state;
+
+	switch (state_toset) {
+	case ERFON:
+		ppsc->rfoff_reason &= (~changesource);
+
+		if ((changesource == RF_CHANGE_BY_HW) &&
+		    (ppsc->hwradiooff)) {
+			ppsc->hwradiooff = false;
+		}
+
+		if (!ppsc->rfoff_reason) {
+			ppsc->rfoff_reason = 0;
+			actionallowed = true;
+		}
+		break;
+	case ERFOFF:
+		if ((changesource == RF_CHANGE_BY_HW) && !ppsc->hwradiooff)
+			ppsc->hwradiooff = true;
+
+		ppsc->rfoff_reason |= changesource;
+		actionallowed = true;
+		break;
+	case ERFSLEEP:
+		ppsc->rfoff_reason |= changesource;
+		actionallowed = true;
+		break;
+	default:
+		pr_err("switch case %#x not processed\n", state_toset);
+		break;
+	}
+
+	if (actionallowed)
+		rtlpriv->cfg->ops->set_rf_power_state(hw, state_toset);
+
+	spin_lock(&rtlpriv->locks.rf_ps_lock);
+	ppsc->rfchange_inprogress = false;
+	spin_unlock(&rtlpriv->locks.rf_ps_lock);
+
+	return actionallowed;
+}
+
+static void _rtl_ps_inactive_ps(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+
+	ppsc->swrf_processing = true;
+
+	if (ppsc->inactive_pwrstate == ERFON &&
+	    rtlhal->interface == INTF_PCI) {
+		if ((ppsc->reg_rfps_level & RT_RF_OFF_LEVL_ASPM) &&
+		    RT_IN_PS_LEVEL(ppsc, RT_PS_LEVEL_ASPM) &&
+		    rtlhal->interface == INTF_PCI) {
+			rtlpriv->intf_ops->disable_aspm(hw);
+			RT_CLEAR_PS_LEVEL(ppsc, RT_PS_LEVEL_ASPM);
+		}
+	}
+
+	rtl_ps_set_rf_state(hw, ppsc->inactive_pwrstate,
+			    RF_CHANGE_BY_IPS);
+
+	if (ppsc->inactive_pwrstate == ERFOFF &&
+	    rtlhal->interface == INTF_PCI) {
+		if (ppsc->reg_rfps_level & RT_RF_OFF_LEVL_ASPM &&
+		    !RT_IN_PS_LEVEL(ppsc, RT_PS_LEVEL_ASPM)) {
+			rtlpriv->intf_ops->enable_aspm(hw);
+			RT_SET_PS_LEVEL(ppsc, RT_PS_LEVEL_ASPM);
+		}
+	}
+
+	ppsc->swrf_processing = false;
+}
+
+void rtl_ips_nic_off_wq_callback(void *data)
+{
+	struct rtl_works *rtlworks =
+	    container_of_dwork_rtl(data, struct rtl_works, ips_nic_off_wq);
+	struct ieee80211_hw *hw = rtlworks->hw;
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	enum rf_pwrstate rtstate;
+
+	if (mac->opmode != NL80211_IFTYPE_STATION) {
+		RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING,
+			 "not station return\n");
+		return;
+	}
+
+	if (mac->p2p_in_use)
+		return;
+
+	if (mac->link_state > MAC80211_NOLINK)
+		return;
+
+	if (is_hal_stop(rtlhal))
+		return;
+
+	if (rtlpriv->sec.being_setkey)
+		return;
+
+	if (rtlpriv->cfg->ops->bt_coex_off_before_lps)
+		rtlpriv->cfg->ops->bt_coex_off_before_lps(hw);
+
+	if (ppsc->inactiveps) {
+		rtstate = ppsc->rfpwr_state;
+
+		/*
+		 *Do not enter IPS in the following conditions:
+		 *(1) RF is already OFF or Sleep
+		 *(2) swrf_processing (indicates the IPS is still under going)
+		 *(3) Connectted (only disconnected can trigger IPS)
+		 *(4) IBSS (send Beacon)
+		 *(5) AP mode (send Beacon)
+		 *(6) monitor mode (rcv packet)
+		 */
+
+		if (rtstate == ERFON &&
+		    !ppsc->swrf_processing &&
+		    (mac->link_state == MAC80211_NOLINK) &&
+		    !mac->act_scanning) {
+			RT_TRACE(rtlpriv, COMP_RF, DBG_TRACE,
+				 "IPSEnter(): Turn off RF\n");
+
+			ppsc->inactive_pwrstate = ERFOFF;
+			ppsc->in_powersavemode = true;
+
+			/* call before RF off */
+			if (rtlpriv->cfg->ops->get_btc_status())
+				rtlpriv->btcoexist.btc_ops->btc_ips_notify(rtlpriv,
+									ppsc->inactive_pwrstate);
+
+			/*rtl_pci_reset_trx_ring(hw); */
+			_rtl_ps_inactive_ps(hw);
+		}
+	}
+}
+
+void rtl_ips_nic_off(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	/* because when link with ap, mac80211 will ask us
+	 * to disable nic quickly after scan before linking,
+	 * this will cause link failed, so we delay 100ms here
+	 */
+	queue_delayed_work(rtlpriv->works.rtl_wq,
+			   &rtlpriv->works.ips_nic_off_wq, MSECS(100));
+}
+
+/* NOTICE: any opmode should exc nic_on, or disable without
+ * nic_on may something wrong, like adhoc TP
+ */
+void rtl_ips_nic_on(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	enum rf_pwrstate rtstate;
+
+	cancel_delayed_work(&rtlpriv->works.ips_nic_off_wq);
+
+	mutex_lock(&rtlpriv->locks.ips_mutex);
+	if (ppsc->inactiveps) {
+		rtstate = ppsc->rfpwr_state;
+
+		if (rtstate != ERFON &&
+		    !ppsc->swrf_processing &&
+		    ppsc->rfoff_reason <= RF_CHANGE_BY_IPS) {
+			ppsc->inactive_pwrstate = ERFON;
+			ppsc->in_powersavemode = false;
+			_rtl_ps_inactive_ps(hw);
+			/* call after RF on */
+			if (rtlpriv->phydm.ops)
+				rtlpriv->phydm.ops->phydm_reset_dm(rtlpriv);
+			if (rtlpriv->cfg->ops->get_btc_status())
+				rtlpriv->btcoexist.btc_ops->btc_ips_notify(rtlpriv,
+									ppsc->inactive_pwrstate);
+		}
+	}
+	mutex_unlock(&rtlpriv->locks.ips_mutex);
+}
+
+/*for FW LPS*/
+
+/*
+ *Determine if we can set Fw into PS mode
+ *in current condition.Return TRUE if it
+ *can enter PS mode.
+ */
+static bool rtl_get_fwlps_doze(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	u32 ps_timediff;
+
+	ps_timediff = jiffies_to_msecs(jiffies -
+				       ppsc->last_delaylps_stamp_jiffies);
+
+	if (ps_timediff < 2000) {
+		RT_TRACE(rtlpriv, COMP_POWER, DBG_LOUD,
+			 "Delay enter Fw LPS for DHCP, ARP, or EAPOL exchanging state\n");
+		return false;
+	}
+
+	if (mac->link_state != MAC80211_LINKED)
+		return false;
+
+	if (mac->opmode == NL80211_IFTYPE_ADHOC)
+		return false;
+
+	return true;
+}
+
+/* Change current and default preamble mode.*/
+void rtl_lps_set_psmode(struct ieee80211_hw *hw, u8 rt_psmode)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	bool enter_fwlps;
+
+	if (mac->opmode == NL80211_IFTYPE_ADHOC)
+		return;
+
+	if (mac->link_state != MAC80211_LINKED)
+		return;
+
+	if (ppsc->dot11_psmode == rt_psmode && rt_psmode == EACTIVE)
+		return;
+
+	/* Update power save mode configured. */
+	ppsc->dot11_psmode = rt_psmode;
+
+	/*
+	 *<FW control LPS>
+	 *1. Enter PS mode
+	 *   Set RPWM to Fw to turn RF off and send H2C fw_pwrmode
+	 *   cmd to set Fw into PS mode.
+	 *2. Leave PS mode
+	 *   Send H2C fw_pwrmode cmd to Fw to set Fw into Active
+	 *   mode and set RPWM to turn RF on.
+	 */
+
+	if ((ppsc->fwctrl_lps) && ppsc->report_linked) {
+		if (ppsc->dot11_psmode == EACTIVE) {
+			RT_TRACE(rtlpriv, COMP_RF, DBG_DMESG,
+				 "FW LPS leave ps_mode:%x\n",
+				  FW_PS_ACTIVE_MODE);
+			enter_fwlps = false;
+			ppsc->pwr_mode = FW_PS_ACTIVE_MODE;
+			ppsc->smart_ps = 0;
+			rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_FW_LPS_ACTION,
+						      (u8 *)(&enter_fwlps));
+			if (ppsc->p2p_ps_info.opp_ps)
+				rtl_p2p_ps_cmd(hw, P2P_PS_ENABLE);
+
+			if (rtlpriv->cfg->ops->get_btc_status())
+				rtlpriv->btcoexist.btc_ops->btc_lps_notify(rtlpriv, rt_psmode);
+		} else {
+			if (rtl_get_fwlps_doze(hw)) {
+				RT_TRACE(rtlpriv, COMP_RF, DBG_DMESG,
+					 "FW LPS enter ps_mode:%x\n",
+					 ppsc->fwctrl_psmode);
+				if (rtlpriv->cfg->ops->get_btc_status())
+					rtlpriv->btcoexist.btc_ops->btc_lps_notify(rtlpriv, rt_psmode);
+				enter_fwlps = true;
+				ppsc->pwr_mode = ppsc->fwctrl_psmode;
+				ppsc->smart_ps = 2;
+				rtlpriv->cfg->ops->set_hw_reg(hw,
+							HW_VAR_FW_LPS_ACTION,
+							(u8 *)(&enter_fwlps));
+
+			} else {
+				/* Reset the power save related parameters. */
+				ppsc->dot11_psmode = EACTIVE;
+			}
+		}
+	}
+}
+
+/* Interrupt safe routine to enter the leisure power save mode.*/
+static void rtl_lps_enter_core(struct ieee80211_hw *hw)
+{
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	if (!ppsc->fwctrl_lps)
+		return;
+
+	if (rtlpriv->sec.being_setkey)
+		return;
+
+	if (rtlpriv->link_info.busytraffic)
+		return;
+
+	/*sleep after linked 10s, to let DHCP and 4-way handshake ok enough!! */
+	if (mac->cnt_after_linked < 5)
+		return;
+
+	if (mac->opmode == NL80211_IFTYPE_ADHOC)
+		return;
+
+	if (mac->link_state != MAC80211_LINKED)
+		return;
+
+	mutex_lock(&rtlpriv->locks.lps_mutex);
+
+	/* Don't need to check (ppsc->dot11_psmode == EACTIVE), because
+	 * bt_ccoexist may ask to enter lps.
+	 * In normal case, this constraint move to rtl_lps_set_psmode().
+	 */
+	RT_TRACE(rtlpriv, COMP_POWER, DBG_LOUD,
+		 "Enter 802.11 power save mode...\n");
+	rtl_lps_set_psmode(hw, EAUTOPS);
+
+	mutex_unlock(&rtlpriv->locks.lps_mutex);
+}
+
+/* Interrupt safe routine to leave the leisure power save mode.*/
+static void rtl_lps_leave_core(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+
+	mutex_lock(&rtlpriv->locks.lps_mutex);
+
+	if (ppsc->fwctrl_lps) {
+		if (ppsc->dot11_psmode != EACTIVE) {
+			/*FIX ME */
+			/*rtlpriv->cfg->ops->enable_interrupt(hw); */
+
+			if (ppsc->reg_rfps_level & RT_RF_LPS_LEVEL_ASPM &&
+			    RT_IN_PS_LEVEL(ppsc, RT_PS_LEVEL_ASPM) &&
+			    rtlhal->interface == INTF_PCI) {
+				rtlpriv->intf_ops->disable_aspm(hw);
+				RT_CLEAR_PS_LEVEL(ppsc, RT_PS_LEVEL_ASPM);
+			}
+
+			RT_TRACE(rtlpriv, COMP_POWER, DBG_LOUD,
+				 "Busy Traffic,Leave 802.11 power save..\n");
+
+			rtl_lps_set_psmode(hw, EACTIVE);
+		}
+	}
+	mutex_unlock(&rtlpriv->locks.lps_mutex);
+}
+
+/* For sw LPS*/
+void rtl_swlps_beacon(struct ieee80211_hw *hw, void *data, unsigned int len)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct ieee80211_hdr *hdr = data;
+	struct ieee80211_tim_ie *tim_ie;
+	u8 *tim;
+	u8 tim_len;
+	bool u_buffed;
+	bool m_buffed;
+
+	if (mac->opmode != NL80211_IFTYPE_STATION)
+		return;
+
+	if (!rtlpriv->psc.swctrl_lps)
+		return;
+
+	if (rtlpriv->mac80211.link_state != MAC80211_LINKED)
+		return;
+
+	if (!rtlpriv->psc.sw_ps_enabled)
+		return;
+
+	if (rtlpriv->psc.fwctrl_lps)
+		return;
+
+	if (likely(!(hw->conf.flags & IEEE80211_CONF_PS)))
+		return;
+
+	/* check if this really is a beacon */
+	if (!ieee80211_is_beacon(hdr->frame_control))
+		return;
+
+	/* min. beacon length + FCS_LEN */
+	if (len <= 40 + FCS_LEN)
+		return;
+
+	/* and only beacons from the associated BSSID, please */
+	if (!ether_addr_equal_64bits(hdr->addr3, rtlpriv->mac80211.bssid))
+		return;
+
+	rtlpriv->psc.last_beacon = jiffies;
+
+	tim = rtl_find_ie(data, len - FCS_LEN, WLAN_EID_TIM);
+	if (!tim)
+		return;
+
+	if (tim[1] < sizeof(*tim_ie))
+		return;
+
+	tim_len = tim[1];
+	tim_ie = (struct ieee80211_tim_ie *)&tim[2];
+
+	if (!WARN_ON_ONCE(!hw->conf.ps_dtim_period))
+		rtlpriv->psc.dtim_counter = tim_ie->dtim_count;
+
+	/* Check whenever the PHY can be turned off again. */
+
+	/* 1. What about buffered unicast traffic for our AID? */
+	u_buffed = ieee80211_check_tim(tim_ie, tim_len,
+				       rtlpriv->mac80211.assoc_id);
+
+	/* 2. Maybe the AP wants to send multicast/broadcast data? */
+	m_buffed = tim_ie->bitmap_ctrl & 0x01;
+	rtlpriv->psc.multi_buffered = m_buffed;
+
+	/* unicast will process by mac80211 through
+	 * set ~IEEE80211_CONF_PS, So we just check
+	 * multicast frames here
+	 */
+	if (!m_buffed) {
+		/* back to low-power land. and delay is
+		 * prevent null power save frame tx fail
+		 */
+		queue_delayed_work(rtlpriv->works.rtl_wq,
+				   &rtlpriv->works.ps_work, MSECS(5));
+	} else {
+		RT_TRACE(rtlpriv, COMP_POWER, DBG_DMESG,
+			 "u_bufferd: %x, m_buffered: %x\n", u_buffed, m_buffed);
+	}
+}
+
+void rtl_swlps_rf_awake(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+
+	if (!rtlpriv->psc.swctrl_lps)
+		return;
+	if (mac->link_state != MAC80211_LINKED)
+		return;
+
+	if (ppsc->reg_rfps_level & RT_RF_LPS_LEVEL_ASPM &&
+	    RT_IN_PS_LEVEL(ppsc, RT_PS_LEVEL_ASPM)) {
+		rtlpriv->intf_ops->disable_aspm(hw);
+		RT_CLEAR_PS_LEVEL(ppsc, RT_PS_LEVEL_ASPM);
+	}
+
+	mutex_lock(&rtlpriv->locks.lps_mutex);
+	rtl_ps_set_rf_state(hw, ERFON, RF_CHANGE_BY_PS);
+	mutex_unlock(&rtlpriv->locks.lps_mutex);
+}
+
+void rtl_swlps_rfon_wq_callback(void *data)
+{
+	struct rtl_works *rtlworks =
+	    container_of_dwork_rtl(data, struct rtl_works, ps_rfon_wq);
+	struct ieee80211_hw *hw = rtlworks->hw;
+
+	rtl_swlps_rf_awake(hw);
+}
+
+void rtl_swlps_rf_sleep(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	u8 sleep_intv;
+
+	if (!rtlpriv->psc.sw_ps_enabled)
+		return;
+
+	if ((rtlpriv->sec.being_setkey) ||
+	    (mac->opmode == NL80211_IFTYPE_ADHOC))
+		return;
+
+	/*sleep after linked 10s, to let DHCP and 4-way handshake ok enough!! */
+	if ((mac->link_state != MAC80211_LINKED) || (mac->cnt_after_linked < 5))
+		return;
+
+	if (rtlpriv->link_info.busytraffic)
+		return;
+
+	spin_lock(&rtlpriv->locks.rf_ps_lock);
+	if (rtlpriv->psc.rfchange_inprogress) {
+		spin_unlock(&rtlpriv->locks.rf_ps_lock);
+		return;
+	}
+	spin_unlock(&rtlpriv->locks.rf_ps_lock);
+
+	mutex_lock(&rtlpriv->locks.lps_mutex);
+	rtl_ps_set_rf_state(hw, ERFSLEEP, RF_CHANGE_BY_PS);
+	mutex_unlock(&rtlpriv->locks.lps_mutex);
+
+	if (ppsc->reg_rfps_level & RT_RF_OFF_LEVL_ASPM &&
+	    !RT_IN_PS_LEVEL(ppsc, RT_PS_LEVEL_ASPM)) {
+		rtlpriv->intf_ops->enable_aspm(hw);
+		RT_SET_PS_LEVEL(ppsc, RT_PS_LEVEL_ASPM);
+	}
+
+	/* here is power save alg, when this beacon is DTIM
+	 * we will set sleep time to dtim_period * n;
+	 * when this beacon is not DTIM, we will set sleep
+	 * time to sleep_intv = rtlpriv->psc.dtim_counter or
+	 * MAX_SW_LPS_SLEEP_INTV(default set to 5)
+	 */
+
+	if (rtlpriv->psc.dtim_counter == 0) {
+		if (hw->conf.ps_dtim_period == 1)
+			sleep_intv = hw->conf.ps_dtim_period * 2;
+		else
+			sleep_intv = hw->conf.ps_dtim_period;
+	} else {
+		sleep_intv = rtlpriv->psc.dtim_counter;
+	}
+
+	if (sleep_intv > MAX_SW_LPS_SLEEP_INTV)
+		sleep_intv = MAX_SW_LPS_SLEEP_INTV;
+
+	/* this print should always be dtim_conter = 0 &
+	 * sleep  = dtim_period, that meaons, we should
+	 * awake before every dtim
+	 */
+	RT_TRACE(rtlpriv, COMP_POWER, DBG_DMESG,
+		 "dtim_counter:%x will sleep :%d beacon_intv\n",
+		  rtlpriv->psc.dtim_counter, sleep_intv);
+
+	/* we tested that 40ms is enough for sw & hw sw delay */
+	queue_delayed_work(rtlpriv->works.rtl_wq, &rtlpriv->works.ps_rfon_wq,
+			   MSECS(sleep_intv *
+				 mac->vif->bss_conf.beacon_int - 40));
+}
+
+void rtl_lps_change_work_callback(struct work_struct *work)
+{
+	struct rtl_works *rtlworks =
+	    container_of(work, struct rtl_works, lps_change_work);
+	struct ieee80211_hw *hw = rtlworks->hw;
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	if (rtlpriv->enter_ps)
+		rtl_lps_enter_core(hw);
+	else
+		rtl_lps_leave_core(hw);
+}
+
+void rtl_lps_enter(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	if (!in_interrupt())
+		return rtl_lps_enter_core(hw);
+	rtlpriv->enter_ps = true;
+	schedule_work(&rtlpriv->works.lps_change_work);
+}
+
+void rtl_lps_leave(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	if (!in_interrupt())
+		return rtl_lps_leave_core(hw);
+	rtlpriv->enter_ps = false;
+	schedule_work(&rtlpriv->works.lps_change_work);
+}
+
+void rtl_swlps_wq_callback(void *data)
+{
+	struct rtl_works *rtlworks = container_of_dwork_rtl(data,
+				     struct rtl_works,
+				     ps_work);
+	struct ieee80211_hw *hw = rtlworks->hw;
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	bool ps = false;
+
+	ps = (hw->conf.flags & IEEE80211_CONF_PS);
+
+	/* we can sleep after ps null send ok */
+	if (rtlpriv->psc.state_inap) {
+		rtl_swlps_rf_sleep(hw);
+
+		if (rtlpriv->psc.state && !ps) {
+			rtlpriv->psc.sleep_ms = jiffies_to_msecs(jiffies -
+						 rtlpriv->psc.last_action);
+		}
+
+		if (ps)
+			rtlpriv->psc.last_slept = jiffies;
+
+		rtlpriv->psc.last_action = jiffies;
+		rtlpriv->psc.state = ps;
+	}
+}
+
+static void rtl_p2p_noa_ie(struct ieee80211_hw *hw, void *data,
+			   unsigned int len)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct ieee80211_mgmt *mgmt = data;
+	struct rtl_p2p_ps_info *p2pinfo = &rtlpriv->psc.p2p_ps_info;
+	u8 *pos, *end, *ie;
+	u16 noa_len;
+	static u8 p2p_oui_ie_type[4] = {0x50, 0x6f, 0x9a, 0x09};
+	u8 noa_num, index, i, noa_index = 0;
+	bool find_p2p_ie = false, find_p2p_ps_ie = false;
+
+	pos = (u8 *)mgmt->u.beacon.variable;
+	end = data + len;
+	ie = NULL;
+
+	while (pos + 1 < end) {
+		if (pos + 2 + pos[1] > end)
+			return;
+
+		if (pos[0] == 221 && pos[1] > 4) {
+			if (memcmp(&pos[2], p2p_oui_ie_type, 4) == 0) {
+				ie = pos + 2 + 4;
+				break;
+			}
+		}
+		pos += 2 + pos[1];
+	}
+
+	if (!ie)
+		return;
+	find_p2p_ie = true;
+	/*to find noa ie*/
+	while (ie + 1 < end) {
+		noa_len = READEF2BYTE((__le16 *)&ie[1]);
+		if (ie + 3 + ie[1] > end)
+			return;
+
+		if (ie[0] == 12) {
+			find_p2p_ps_ie = true;
+			if ((noa_len - 2) % 13 != 0) {
+				RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+					 "P2P notice of absence: invalid length.%d\n",
+					 noa_len);
+				return;
+			}
+			noa_num = (noa_len - 2) / 13;
+			noa_index = ie[3];
+			if (rtlpriv->psc.p2p_ps_info.p2p_ps_mode ==
+			    P2P_PS_NONE || noa_index != p2pinfo->noa_index) {
+				RT_TRACE(rtlpriv, COMP_FW, DBG_LOUD,
+					 "update NOA ie.\n");
+				p2pinfo->noa_index = noa_index;
+				p2pinfo->opp_ps = (ie[4] >> 7);
+				p2pinfo->ctwindow = ie[4] & 0x7F;
+				p2pinfo->noa_num = noa_num;
+				index = 5;
+				for (i = 0; i < noa_num; i++) {
+					p2pinfo->noa_count_type[i] =
+					    READEF1BYTE(ie + index);
+					index += 1;
+					p2pinfo->noa_duration[i] =
+					     READEF4BYTE((__le32 *)ie + index);
+					index += 4;
+					p2pinfo->noa_interval[i] =
+					    READEF4BYTE((__le32 *)ie + index);
+					index += 4;
+					p2pinfo->noa_start_time[i] =
+					     READEF4BYTE((__le32 *)ie + index);
+					index += 4;
+				}
+
+				if (p2pinfo->opp_ps == 1) {
+					p2pinfo->p2p_ps_mode = P2P_PS_CTWINDOW;
+					/* Driver should wait LPS entering
+					 * CTWindow
+					 */
+					if (rtlpriv->psc.fw_current_inpsmode)
+						rtl_p2p_ps_cmd(hw,
+							       P2P_PS_ENABLE);
+				} else if (p2pinfo->noa_num > 0) {
+					p2pinfo->p2p_ps_mode = P2P_PS_NOA;
+					rtl_p2p_ps_cmd(hw, P2P_PS_ENABLE);
+				} else if (p2pinfo->p2p_ps_mode > P2P_PS_NONE) {
+					rtl_p2p_ps_cmd(hw, P2P_PS_DISABLE);
+				}
+			}
+			break;
+		}
+		ie += 3 + noa_len;
+	}
+
+	if (find_p2p_ie) {
+		if ((p2pinfo->p2p_ps_mode > P2P_PS_NONE) &&
+		    (!find_p2p_ps_ie))
+			rtl_p2p_ps_cmd(hw, P2P_PS_DISABLE);
+	}
+}
+
+static void rtl_p2p_action_ie(struct ieee80211_hw *hw, void *data,
+			      unsigned int len)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct ieee80211_mgmt *mgmt = data;
+	struct rtl_p2p_ps_info *p2pinfo = &rtlpriv->psc.p2p_ps_info;
+	u8 noa_num, index, i, noa_index = 0;
+	u8 *pos, *end, *ie;
+	u16 noa_len;
+	static u8 p2p_oui_ie_type[4] = {0x50, 0x6f, 0x9a, 0x09};
+
+	pos = (u8 *)&mgmt->u.action.category;
+	end = data + len;
+	ie = NULL;
+
+	if (pos[0] == 0x7f) {
+		if (memcmp(&pos[1], p2p_oui_ie_type, 4) == 0)
+			ie = pos + 3 + 4;
+	}
+
+	if (!ie)
+		return;
+
+	RT_TRACE(rtlpriv, COMP_FW, DBG_LOUD, "action frame find P2P IE.\n");
+	/*to find noa ie*/
+	while (ie + 1 < end) {
+		noa_len = READEF2BYTE((__le16 *)&ie[1]);
+		if (ie + 3 + ie[1] > end)
+			return;
+
+		if (ie[0] == 12) {
+			RT_TRACE(rtlpriv, COMP_FW, DBG_LOUD, "find NOA IE.\n");
+			RT_PRINT_DATA(rtlpriv, COMP_FW, DBG_LOUD, "noa ie ",
+				      ie, noa_len);
+			if ((noa_len - 2) % 13 != 0) {
+				RT_TRACE(rtlpriv, COMP_FW, DBG_LOUD,
+					 "P2P notice of absence: invalid length.%d\n",
+					 noa_len);
+				return;
+			}
+			noa_num = (noa_len - 2) / 13;
+			noa_index = ie[3];
+			if (rtlpriv->psc.p2p_ps_info.p2p_ps_mode ==
+			    P2P_PS_NONE || noa_index != p2pinfo->noa_index) {
+				p2pinfo->noa_index = noa_index;
+				p2pinfo->opp_ps = (ie[4] >> 7);
+				p2pinfo->ctwindow = ie[4] & 0x7F;
+				p2pinfo->noa_num = noa_num;
+				index = 5;
+				for (i = 0; i < noa_num; i++) {
+					p2pinfo->noa_count_type[i] =
+					    READEF1BYTE(ie + index);
+					index += 1;
+					p2pinfo->noa_duration[i] =
+					     READEF4BYTE((__le32 *)ie + index);
+					index += 4;
+					p2pinfo->noa_interval[i] =
+					     READEF4BYTE((__le32 *)ie + index);
+					index += 4;
+					p2pinfo->noa_start_time[i] =
+					     READEF4BYTE((__le32 *)ie + index);
+					index += 4;
+				}
+
+				if (p2pinfo->opp_ps == 1) {
+					p2pinfo->p2p_ps_mode = P2P_PS_CTWINDOW;
+					/* Driver should wait LPS entering
+					 * CTWindow
+					 */
+					if (rtlpriv->psc.fw_current_inpsmode)
+						rtl_p2p_ps_cmd(hw,
+							       P2P_PS_ENABLE);
+				} else if (p2pinfo->noa_num > 0) {
+					p2pinfo->p2p_ps_mode = P2P_PS_NOA;
+					rtl_p2p_ps_cmd(hw, P2P_PS_ENABLE);
+				} else if (p2pinfo->p2p_ps_mode > P2P_PS_NONE) {
+					rtl_p2p_ps_cmd(hw, P2P_PS_DISABLE);
+				}
+			}
+			break;
+		}
+		ie += 3 + noa_len;
+	}
+}
+
+void rtl_p2p_ps_cmd(struct ieee80211_hw *hw, u8 p2p_ps_state)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_ps_ctl *rtlps = rtl_psc(rtl_priv(hw));
+	struct rtl_p2p_ps_info  *p2pinfo = &rtlpriv->psc.p2p_ps_info;
+
+	RT_TRACE(rtlpriv, COMP_FW, DBG_LOUD, "p2p state %x\n", p2p_ps_state);
+	switch (p2p_ps_state) {
+	case P2P_PS_DISABLE:
+		p2pinfo->p2p_ps_state = p2p_ps_state;
+		rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_H2C_FW_P2P_PS_OFFLOAD,
+					      &p2p_ps_state);
+		p2pinfo->noa_index = 0;
+		p2pinfo->ctwindow = 0;
+		p2pinfo->opp_ps = 0;
+		p2pinfo->noa_num = 0;
+		p2pinfo->p2p_ps_mode = P2P_PS_NONE;
+		if (rtlps->fw_current_inpsmode) {
+			if (rtlps->smart_ps == 0) {
+				rtlps->smart_ps = 2;
+				rtlpriv->cfg->ops->set_hw_reg(hw,
+					 HW_VAR_H2C_FW_PWRMODE,
+					 &rtlps->pwr_mode);
+			}
+		}
+		break;
+	case P2P_PS_ENABLE:
+		if (p2pinfo->p2p_ps_mode > P2P_PS_NONE) {
+			p2pinfo->p2p_ps_state = p2p_ps_state;
+
+			if (p2pinfo->ctwindow > 0) {
+				if (rtlps->smart_ps != 0) {
+					rtlps->smart_ps = 0;
+					rtlpriv->cfg->ops->set_hw_reg(hw,
+						 HW_VAR_H2C_FW_PWRMODE,
+						 &rtlps->pwr_mode);
+				}
+			}
+			rtlpriv->cfg->ops->set_hw_reg(hw,
+				 HW_VAR_H2C_FW_P2P_PS_OFFLOAD,
+				 &p2p_ps_state);
+		}
+		break;
+	case P2P_PS_SCAN:
+	case P2P_PS_SCAN_DONE:
+	case P2P_PS_ALLSTASLEEP:
+		if (p2pinfo->p2p_ps_mode > P2P_PS_NONE) {
+			p2pinfo->p2p_ps_state = p2p_ps_state;
+			rtlpriv->cfg->ops->set_hw_reg(hw,
+				 HW_VAR_H2C_FW_P2P_PS_OFFLOAD,
+				 &p2p_ps_state);
+		}
+		break;
+	default:
+		break;
+	}
+	RT_TRACE(rtlpriv, COMP_FW, DBG_LOUD,
+		 "ctwindow %x oppps %x\n",
+		 p2pinfo->ctwindow, p2pinfo->opp_ps);
+	RT_TRACE(rtlpriv, COMP_FW, DBG_LOUD,
+		 "count %x duration %x index %x interval %x start time %x noa num %x\n",
+		 p2pinfo->noa_count_type[0],
+		 p2pinfo->noa_duration[0],
+		 p2pinfo->noa_index,
+		 p2pinfo->noa_interval[0],
+		 p2pinfo->noa_start_time[0],
+		 p2pinfo->noa_num);
+	RT_TRACE(rtlpriv, COMP_FW, DBG_LOUD, "end\n");
+}
+
+void rtl_p2p_info(struct ieee80211_hw *hw, void *data, unsigned int len)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct ieee80211_hdr *hdr = data;
+
+	if (!mac->p2p)
+		return;
+	if (mac->link_state != MAC80211_LINKED)
+		return;
+	/* min. beacon length + FCS_LEN */
+	if (len <= 40 + FCS_LEN)
+		return;
+
+	/* and only beacons from the associated BSSID, please */
+	if (!ether_addr_equal_64bits(hdr->addr3, rtlpriv->mac80211.bssid))
+		return;
+
+	/* check if this really is a beacon */
+	if (!(ieee80211_is_beacon(hdr->frame_control) ||
+	      ieee80211_is_probe_resp(hdr->frame_control) ||
+	      ieee80211_is_action(hdr->frame_control)))
+		return;
+
+	if (ieee80211_is_action(hdr->frame_control))
+		rtl_p2p_action_ie(hw, data, len - FCS_LEN);
+	else
+		rtl_p2p_noa_ie(hw, data, len - FCS_LEN);
+}
diff --git a/drivers/staging/rtlwifi/ps.h b/drivers/staging/rtlwifi/ps.h
new file mode 100644
index 000000000000..6c187daced4a
--- /dev/null
+++ b/drivers/staging/rtlwifi/ps.h
@@ -0,0 +1,50 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2012  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __REALTEK_RTL_PCI_PS_H__
+#define __REALTEK_RTL_PCI_PS_H__
+
+#define MAX_SW_LPS_SLEEP_INTV	5
+
+bool rtl_ps_enable_nic(struct ieee80211_hw *hw);
+bool rtl_ps_disable_nic(struct ieee80211_hw *hw);
+void rtl_ips_nic_off(struct ieee80211_hw *hw);
+void rtl_ips_nic_on(struct ieee80211_hw *hw);
+void rtl_ips_nic_off_wq_callback(void *data);
+void rtl_lps_enter(struct ieee80211_hw *hw);
+void rtl_lps_leave(struct ieee80211_hw *hw);
+
+void rtl_lps_set_psmode(struct ieee80211_hw *hw, u8 rt_psmode);
+
+void rtl_swlps_beacon(struct ieee80211_hw *hw, void *data, unsigned int len);
+void rtl_swlps_wq_callback(void *data);
+void rtl_swlps_rfon_wq_callback(void *data);
+void rtl_swlps_rf_awake(struct ieee80211_hw *hw);
+void rtl_swlps_rf_sleep(struct ieee80211_hw *hw);
+void rtl_p2p_ps_cmd(struct ieee80211_hw *hw, u8 p2p_ps_state);
+void rtl_p2p_info(struct ieee80211_hw *hw, void *data, unsigned int len);
+void rtl_lps_change_work_callback(struct work_struct *work);
+
+#endif
diff --git a/drivers/staging/rtlwifi/pwrseqcmd.h b/drivers/staging/rtlwifi/pwrseqcmd.h
new file mode 100644
index 000000000000..f411b7ebb08f
--- /dev/null
+++ b/drivers/staging/rtlwifi/pwrseqcmd.h
@@ -0,0 +1,94 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2012  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __RTL8723E_PWRSEQCMD_H__
+#define __RTL8723E_PWRSEQCMD_H__
+
+#include "wifi.h"
+/*---------------------------------------------
+ * 3 The value of cmd: 4 bits
+ *---------------------------------------------
+ */
+#define    PWR_CMD_READ		0x00
+#define    PWR_CMD_WRITE	0x01
+#define    PWR_CMD_POLLING	0x02
+#define    PWR_CMD_DELAY	0x03
+#define    PWR_CMD_END		0x04
+
+/* define the base address of each block */
+#define   PWR_BASEADDR_MAC	0x00
+#define   PWR_BASEADDR_USB	0x01
+#define   PWR_BASEADDR_PCIE	0x02
+#define   PWR_BASEADDR_SDIO	0x03
+
+#define	PWR_INTF_SDIO_MSK	BIT(0)
+#define	PWR_INTF_USB_MSK	BIT(1)
+#define	PWR_INTF_PCI_MSK	BIT(2)
+#define	PWR_INTF_ALL_MSK	(BIT(0) | BIT(1) | BIT(2) | BIT(3))
+
+#define	PWR_FAB_TSMC_MSK	BIT(0)
+#define	PWR_FAB_UMC_MSK		BIT(1)
+#define	PWR_FAB_ALL_MSK		(BIT(0) | BIT(1) | BIT(2) | BIT(3))
+
+#define	PWR_CUT_TESTCHIP_MSK	BIT(0)
+#define	PWR_CUT_A_MSK		BIT(1)
+#define	PWR_CUT_B_MSK		BIT(2)
+#define	PWR_CUT_C_MSK		BIT(3)
+#define	PWR_CUT_D_MSK		BIT(4)
+#define	PWR_CUT_E_MSK		BIT(5)
+#define	PWR_CUT_F_MSK		BIT(6)
+#define	PWR_CUT_G_MSK		BIT(7)
+#define	PWR_CUT_ALL_MSK		0xFF
+
+enum pwrseq_delay_unit {
+	PWRSEQ_DELAY_US,
+	PWRSEQ_DELAY_MS,
+};
+
+struct wlan_pwr_cfg {
+	u16 offset;
+	u8 cut_msk;
+	u8 fab_msk:4;
+	u8 interface_msk:4;
+	u8 base:4;
+	u8 cmd:4;
+	u8 msk;
+	u8 value;
+};
+
+#define	GET_PWR_CFG_OFFSET(__PWR_CMD)	(__PWR_CMD.offset)
+#define	GET_PWR_CFG_CUT_MASK(__PWR_CMD)	(__PWR_CMD.cut_msk)
+#define	GET_PWR_CFG_FAB_MASK(__PWR_CMD)	(__PWR_CMD.fab_msk)
+#define	GET_PWR_CFG_INTF_MASK(__PWR_CMD)	(__PWR_CMD.interface_msk)
+#define	GET_PWR_CFG_BASE(__PWR_CMD)	(__PWR_CMD.base)
+#define	GET_PWR_CFG_CMD(__PWR_CMD)	(__PWR_CMD.cmd)
+#define	GET_PWR_CFG_MASK(__PWR_CMD)	(__PWR_CMD.msk)
+#define	GET_PWR_CFG_VALUE(__PWR_CMD)	(__PWR_CMD.value)
+
+bool rtl_hal_pwrseqcmdparsing(struct rtl_priv *rtlpriv, u8 cut_version,
+			      u8 fab_version, u8 interface_type,
+			      struct wlan_pwr_cfg pwrcfgcmd[]);
+
+#endif
diff --git a/drivers/staging/rtlwifi/rc.c b/drivers/staging/rtlwifi/rc.c
new file mode 100644
index 000000000000..65de0c7b5a67
--- /dev/null
+++ b/drivers/staging/rtlwifi/rc.c
@@ -0,0 +1,322 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2012  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#include "wifi.h"
+#include "base.h"
+#include "rc.h"
+
+/*
+ *Finds the highest rate index we can use
+ *if skb is special data like DHCP/EAPOL, we set should
+ *it to lowest rate CCK_1M, otherwise we set rate to
+ *highest rate based on wireless mode used for iwconfig
+ *show Tx rate.
+ */
+static u8 _rtl_rc_get_highest_rix(struct rtl_priv *rtlpriv,
+				  struct ieee80211_sta *sta,
+				  struct sk_buff *skb, bool not_data)
+{
+	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	struct rtl_sta_info *sta_entry = NULL;
+	u16 wireless_mode = 0;
+	u8 nss;	/* NSS -1 */
+
+	if (get_rf_type(rtlphy) >= RF_4T4R)
+		nss = 3;
+	else if (get_rf_type(rtlphy) >= RF_3T3R)
+		nss = 2;
+	else if (get_rf_type(rtlphy) >= RF_2T2R)
+		nss = 1;
+	else
+		nss = 0;
+
+	/*
+	 *this rate is no use for true rate, firmware
+	 *will control rate at all it just used for
+	 *1.show in iwconfig in B/G mode
+	 *2.in rtl_get_tcb_desc when we check rate is
+	 *      1M we will not use FW rate but user rate.
+	 */
+
+	if (sta) {
+		sta_entry = (struct rtl_sta_info *)sta->drv_priv;
+		wireless_mode = sta_entry->wireless_mode;
+	}
+
+	if (rtl_is_special_data(rtlpriv->mac80211.hw, skb, true, false) ||
+	    not_data) {
+		return 0;
+	}
+	if (rtlhal->current_bandtype == BAND_ON_2_4G) {
+		if (wireless_mode == WIRELESS_MODE_B) {
+			return B_MODE_MAX_RIX;
+		} else if (wireless_mode == WIRELESS_MODE_G) {
+			return G_MODE_MAX_RIX;
+		} else if (wireless_mode == WIRELESS_MODE_N_24G) {
+			if (nss == 0)
+				return N_MODE_MCS7_RIX;
+			else
+				return N_MODE_MCS15_RIX;
+		} else if (wireless_mode == WIRELESS_MODE_AC_24G) {
+			if (sta->bandwidth == IEEE80211_STA_RX_BW_20)
+				return AC_MODE_MCS8_RIX | (nss << 4);
+			else
+				return AC_MODE_MCS9_RIX | (nss << 4);
+		}
+		return 0;
+	}
+	if (wireless_mode == WIRELESS_MODE_A) {
+		return A_MODE_MAX_RIX;
+	} else if (wireless_mode == WIRELESS_MODE_N_5G) {
+		if (nss == 0)
+			return N_MODE_MCS7_RIX;
+		else
+			return N_MODE_MCS15_RIX;
+	} else if (wireless_mode == WIRELESS_MODE_AC_5G) {
+		if (sta->bandwidth == IEEE80211_STA_RX_BW_20)
+			return AC_MODE_MCS8_RIX | (nss << 4);
+		else
+			return AC_MODE_MCS9_RIX | (nss << 4);
+	}
+	return 0;
+}
+
+static void _rtl_rc_rate_set_series(struct rtl_priv *rtlpriv,
+				    struct ieee80211_sta *sta,
+				    struct ieee80211_tx_rate *rate,
+				    struct ieee80211_tx_rate_control *txrc,
+				    u8 tries, s8 rix, int rtsctsenable,
+				    bool not_data)
+{
+	struct rtl_mac *mac = rtl_mac(rtlpriv);
+	struct rtl_sta_info *sta_entry = NULL;
+	u16 wireless_mode = 0;
+	u8 sgi_20 = 0, sgi_40 = 0, sgi_80 = 0;
+
+	if (sta) {
+		sgi_20 = sta->ht_cap.cap & IEEE80211_HT_CAP_SGI_20;
+		sgi_40 = sta->ht_cap.cap & IEEE80211_HT_CAP_SGI_40;
+		sgi_80 = sta->vht_cap.cap & IEEE80211_VHT_CAP_SHORT_GI_80;
+		sta_entry = (struct rtl_sta_info *)sta->drv_priv;
+		wireless_mode = sta_entry->wireless_mode;
+	}
+	rate->count = tries;
+	rate->idx = rix >= 0x00 ? rix : 0x00;
+	if (((rtlpriv->rtlhal.hw_type == HARDWARE_TYPE_RTL8812AE) ||
+	     (rtlpriv->rtlhal.hw_type == HARDWARE_TYPE_RTL8822BE)) &&
+	    wireless_mode == WIRELESS_MODE_AC_5G)
+		rate->idx |= 0x10;/*2NSS for 8812AE, 8822BE*/
+
+	if (!not_data) {
+		if (txrc->short_preamble)
+			rate->flags |= IEEE80211_TX_RC_USE_SHORT_PREAMBLE;
+		if (mac->opmode == NL80211_IFTYPE_AP ||
+		    mac->opmode == NL80211_IFTYPE_ADHOC) {
+			if (sta && (sta->ht_cap.cap &
+				    IEEE80211_HT_CAP_SUP_WIDTH_20_40))
+				rate->flags |= IEEE80211_TX_RC_40_MHZ_WIDTH;
+			if (sta && (sta->vht_cap.vht_supported))
+				rate->flags |= IEEE80211_TX_RC_80_MHZ_WIDTH;
+		} else {
+			if (mac->bw_80)
+				rate->flags |= IEEE80211_TX_RC_80_MHZ_WIDTH;
+			else if (mac->bw_40)
+				rate->flags |= IEEE80211_TX_RC_40_MHZ_WIDTH;
+		}
+
+		if (sgi_20 || sgi_40 || sgi_80)
+			rate->flags |= IEEE80211_TX_RC_SHORT_GI;
+		if (sta && sta->ht_cap.ht_supported &&
+		    ((wireless_mode == WIRELESS_MODE_N_5G) ||
+		     (wireless_mode == WIRELESS_MODE_N_24G)))
+			rate->flags |= IEEE80211_TX_RC_MCS;
+		if (sta && sta->vht_cap.vht_supported &&
+		    (wireless_mode == WIRELESS_MODE_AC_5G ||
+		     wireless_mode == WIRELESS_MODE_AC_24G ||
+		     wireless_mode == WIRELESS_MODE_AC_ONLY))
+			rate->flags |= IEEE80211_TX_RC_VHT_MCS;
+	}
+}
+
+static void rtl_get_rate(void *ppriv, struct ieee80211_sta *sta,
+			 void *priv_sta,
+			 struct ieee80211_tx_rate_control *txrc)
+{
+	struct rtl_priv *rtlpriv = ppriv;
+	struct sk_buff *skb = txrc->skb;
+	struct ieee80211_tx_info *tx_info = IEEE80211_SKB_CB(skb);
+	struct ieee80211_tx_rate *rates = tx_info->control.rates;
+	__le16 fc = rtl_get_fc(skb);
+	u8 try_per_rate, i, rix;
+	bool not_data = !ieee80211_is_data(fc);
+
+	if (rate_control_send_low(sta, priv_sta, txrc))
+		return;
+
+	rix = _rtl_rc_get_highest_rix(rtlpriv, sta, skb, not_data);
+	try_per_rate = 1;
+	_rtl_rc_rate_set_series(rtlpriv, sta, &rates[0], txrc,
+				try_per_rate, rix, 1, not_data);
+
+	if (!not_data) {
+		for (i = 1; i < 4; i++)
+			_rtl_rc_rate_set_series(rtlpriv, sta, &rates[i],
+						txrc, i, (rix - i), 1,
+						not_data);
+	}
+}
+
+static bool _rtl_tx_aggr_check(struct rtl_priv *rtlpriv,
+			       struct rtl_sta_info *sta_entry, u16 tid)
+{
+	struct rtl_mac *mac = rtl_mac(rtlpriv);
+
+	if (mac->act_scanning)
+		return false;
+
+	if (mac->opmode == NL80211_IFTYPE_STATION &&
+	    mac->cnt_after_linked < 3)
+		return false;
+
+	if (sta_entry->tids[tid].agg.agg_state == RTL_AGG_STOP)
+		return true;
+
+	return false;
+}
+
+/*mac80211 Rate Control callbacks*/
+static void rtl_tx_status(void *ppriv,
+			  struct ieee80211_supported_band *sband,
+			  struct ieee80211_sta *sta, void *priv_sta,
+			  struct sk_buff *skb)
+{
+	struct rtl_priv *rtlpriv = ppriv;
+	struct rtl_mac *mac = rtl_mac(rtlpriv);
+	struct ieee80211_hdr *hdr = rtl_get_hdr(skb);
+	__le16 fc = rtl_get_fc(skb);
+	struct rtl_sta_info *sta_entry;
+
+	if (!priv_sta || !ieee80211_is_data(fc))
+		return;
+
+	if (rtl_is_special_data(mac->hw, skb, true, true))
+		return;
+
+	if (is_multicast_ether_addr(ieee80211_get_DA(hdr)) ||
+	    is_broadcast_ether_addr(ieee80211_get_DA(hdr)))
+		return;
+
+	if (sta) {
+		/* Check if aggregation has to be enabled for this tid */
+		sta_entry = (struct rtl_sta_info *)sta->drv_priv;
+		if ((sta->ht_cap.ht_supported) &&
+		    !(skb->protocol == cpu_to_be16(ETH_P_PAE))) {
+			if (ieee80211_is_data_qos(fc)) {
+				u8 tid = rtl_get_tid(skb);
+
+				if (_rtl_tx_aggr_check(rtlpriv, sta_entry,
+						       tid)) {
+					sta_entry->tids[tid].agg.agg_state =
+						RTL_AGG_PROGRESS;
+					ieee80211_start_tx_ba_session(sta, tid,
+								      5000);
+				}
+			}
+		}
+	}
+}
+
+static void rtl_rate_init(void *ppriv,
+			  struct ieee80211_supported_band *sband,
+			  struct cfg80211_chan_def *chandef,
+			  struct ieee80211_sta *sta, void *priv_sta)
+{
+}
+
+static void rtl_rate_update(void *ppriv,
+			    struct ieee80211_supported_band *sband,
+			    struct cfg80211_chan_def *chandef,
+			    struct ieee80211_sta *sta, void *priv_sta,
+			    u32 changed)
+{
+}
+
+static void *rtl_rate_alloc(struct ieee80211_hw *hw, struct dentry *debugfsdir)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	return rtlpriv;
+}
+
+static void rtl_rate_free(void *rtlpriv)
+{
+}
+
+static void *rtl_rate_alloc_sta(void *ppriv,
+				struct ieee80211_sta *sta, gfp_t gfp)
+{
+	struct rtl_priv *rtlpriv = ppriv;
+	struct rtl_rate_priv *rate_priv;
+
+	rate_priv = kzalloc(sizeof(*rate_priv), gfp);
+	if (!rate_priv) {
+		pr_err("Unable to allocate private rc structure\n");
+		return NULL;
+	}
+
+	rtlpriv->rate_priv = rate_priv;
+
+	return rate_priv;
+}
+
+static void rtl_rate_free_sta(void *rtlpriv,
+			      struct ieee80211_sta *sta, void *priv_sta)
+{
+	struct rtl_rate_priv *rate_priv = priv_sta;
+
+	kfree(rate_priv);
+}
+
+static const struct rate_control_ops rtl_rate_ops = {
+	.name = "rtl_rc",
+	.alloc = rtl_rate_alloc,
+	.free = rtl_rate_free,
+	.alloc_sta = rtl_rate_alloc_sta,
+	.free_sta = rtl_rate_free_sta,
+	.rate_init = rtl_rate_init,
+	.rate_update = rtl_rate_update,
+	.tx_status = rtl_tx_status,
+	.get_rate = rtl_get_rate,
+};
+
+int rtl_rate_control_register(void)
+{
+	return ieee80211_rate_control_register(&rtl_rate_ops);
+}
+
+void rtl_rate_control_unregister(void)
+{
+	ieee80211_rate_control_unregister(&rtl_rate_ops);
+}
diff --git a/drivers/staging/rtlwifi/rc.h b/drivers/staging/rtlwifi/rc.h
new file mode 100644
index 000000000000..dcc8520866b7
--- /dev/null
+++ b/drivers/staging/rtlwifi/rc.h
@@ -0,0 +1,49 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2012  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __RTL_RC_H__
+#define __RTL_RC_H__
+
+#define B_MODE_MAX_RIX 3
+#define G_MODE_MAX_RIX 11
+#define A_MODE_MAX_RIX 7
+
+/* in mac80211 mcs0-mcs15 is idx0-idx15*/
+#define N_MODE_MCS7_RIX 7
+#define N_MODE_MCS15_RIX 15
+
+/* in mac80211 vht mcs0-9 is in [3:0], nss is in [:4] */
+#define AC_MODE_MCS7_RIX 7
+#define AC_MODE_MCS8_RIX 8
+#define AC_MODE_MCS9_RIX 9
+
+struct rtl_rate_priv {
+	u8 ht_cap;
+};
+
+int rtl_rate_control_register(void);
+void rtl_rate_control_unregister(void);
+
+#endif
diff --git a/drivers/staging/rtlwifi/regd.c b/drivers/staging/rtlwifi/regd.c
new file mode 100644
index 000000000000..e0a3ff85edb6
--- /dev/null
+++ b/drivers/staging/rtlwifi/regd.c
@@ -0,0 +1,469 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2012  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#include "wifi.h"
+#include "regd.h"
+
+static struct country_code_to_enum_rd allcountries[] = {
+	{COUNTRY_CODE_FCC, "US"},
+	{COUNTRY_CODE_IC, "US"},
+	{COUNTRY_CODE_ETSI, "EC"},
+	{COUNTRY_CODE_SPAIN, "EC"},
+	{COUNTRY_CODE_FRANCE, "EC"},
+	{COUNTRY_CODE_MKK, "JP"},
+	{COUNTRY_CODE_MKK1, "JP"},
+	{COUNTRY_CODE_ISRAEL, "EC"},
+	{COUNTRY_CODE_TELEC, "JP"},
+	{COUNTRY_CODE_MIC, "JP"},
+	{COUNTRY_CODE_GLOBAL_DOMAIN, "JP"},
+	{COUNTRY_CODE_WORLD_WIDE_13, "EC"},
+	{COUNTRY_CODE_TELEC_NETGEAR, "EC"},
+	{COUNTRY_CODE_WORLD_WIDE_13_5G_ALL, "US"},
+};
+
+/*Only these channels all allow active
+ *scan on all world regulatory domains
+ */
+#define RTL819x_2GHZ_CH01_11	\
+	REG_RULE(2412 - 10, 2462 + 10, 40, 0, 20, 0)
+
+/*We enable active scan on these a case
+ *by case basis by regulatory domain
+ */
+#define RTL819x_2GHZ_CH12_13	\
+	REG_RULE(2467 - 10, 2472 + 10, 40, 0, 20,\
+	NL80211_RRF_PASSIVE_SCAN)
+
+#define RTL819x_2GHZ_CH14	\
+	REG_RULE(2484 - 10, 2484 + 10, 40, 0, 20, \
+	NL80211_RRF_PASSIVE_SCAN | \
+	NL80211_RRF_NO_OFDM)
+
+/* 5G chan 36 - chan 64*/
+#define RTL819x_5GHZ_5150_5350	\
+	REG_RULE(5150 - 10, 5350 + 10, 80, 0, 30, 0)
+/* 5G chan 100 - chan 165*/
+#define RTL819x_5GHZ_5470_5850	\
+	REG_RULE(5470 - 10, 5850 + 10, 80, 0, 30, 0)
+/* 5G chan 149 - chan 165*/
+#define RTL819x_5GHZ_5725_5850	\
+	REG_RULE(5725 - 10, 5850 + 10, 80, 0, 30, 0)
+
+#define RTL819x_5GHZ_ALL	\
+	(RTL819x_5GHZ_5150_5350, RTL819x_5GHZ_5470_5850)
+
+static const struct ieee80211_regdomain rtl_regdom_11 = {
+	.n_reg_rules = 1,
+	.alpha2 = "99",
+	.reg_rules = {
+		RTL819x_2GHZ_CH01_11,
+	}
+};
+
+static const struct ieee80211_regdomain rtl_regdom_12_13 = {
+	.n_reg_rules = 2,
+	.alpha2 = "99",
+	.reg_rules = {
+		RTL819x_2GHZ_CH01_11,
+		RTL819x_2GHZ_CH12_13,
+	}
+};
+
+static const struct ieee80211_regdomain rtl_regdom_no_midband = {
+	.n_reg_rules = 3,
+	.alpha2 = "99",
+	.reg_rules = {
+		RTL819x_2GHZ_CH01_11,
+		RTL819x_5GHZ_5150_5350,
+		RTL819x_5GHZ_5725_5850,
+	}
+};
+
+static const struct ieee80211_regdomain rtl_regdom_60_64 = {
+	.n_reg_rules = 3,
+	.alpha2 = "99",
+	.reg_rules = {
+		RTL819x_2GHZ_CH01_11,
+		RTL819x_2GHZ_CH12_13,
+		RTL819x_5GHZ_5725_5850,
+	}
+};
+
+static const struct ieee80211_regdomain rtl_regdom_14_60_64 = {
+	.n_reg_rules = 4,
+	.alpha2 = "99",
+	.reg_rules = {
+		RTL819x_2GHZ_CH01_11,
+		RTL819x_2GHZ_CH12_13,
+		RTL819x_2GHZ_CH14,
+		RTL819x_5GHZ_5725_5850,
+	}
+};
+
+static const struct ieee80211_regdomain rtl_regdom_12_13_5g_all = {
+	.n_reg_rules = 4,
+	.alpha2 = "99",
+	.reg_rules = {
+		RTL819x_2GHZ_CH01_11,
+		RTL819x_2GHZ_CH12_13,
+		RTL819x_5GHZ_5150_5350,
+		RTL819x_5GHZ_5470_5850,
+	}
+};
+
+static const struct ieee80211_regdomain rtl_regdom_14 = {
+	.n_reg_rules = 3,
+	.alpha2 = "99",
+	.reg_rules = {
+		RTL819x_2GHZ_CH01_11,
+		RTL819x_2GHZ_CH12_13,
+		RTL819x_2GHZ_CH14,
+	}
+};
+
+static bool _rtl_is_radar_freq(u16 center_freq)
+{
+	return center_freq >= 5260 && center_freq <= 5700;
+}
+
+static void _rtl_reg_apply_beaconing_flags(struct wiphy *wiphy,
+					   enum nl80211_reg_initiator initiator)
+{
+	enum nl80211_band band;
+	struct ieee80211_supported_band *sband;
+	const struct ieee80211_reg_rule *reg_rule;
+	struct ieee80211_channel *ch;
+	unsigned int i;
+
+	for (band = 0; band < NUM_NL80211_BANDS; band++) {
+		if (!wiphy->bands[band])
+			continue;
+
+		sband = wiphy->bands[band];
+
+		for (i = 0; i < sband->n_channels; i++) {
+			ch = &sband->channels[i];
+			if (_rtl_is_radar_freq(ch->center_freq) ||
+			    (ch->flags & IEEE80211_CHAN_RADAR))
+				continue;
+			if (initiator == NL80211_REGDOM_SET_BY_COUNTRY_IE) {
+				reg_rule = freq_reg_info(wiphy,
+							 ch->center_freq);
+				if (IS_ERR(reg_rule))
+					continue;
+				/*
+				 *If 11d had a rule for this channel ensure
+				 *we enable adhoc/beaconing if it allows us to
+				 *use it. Note that we would have disabled it
+				 *by applying our static world regdomain by
+				 *default during init, prior to calling our
+				 *regulatory_hint().
+				 */
+
+				if (!(reg_rule->flags & NL80211_RRF_NO_IBSS))
+					ch->flags &= ~IEEE80211_CHAN_NO_IBSS;
+				if (!(reg_rule->flags &
+				      NL80211_RRF_PASSIVE_SCAN))
+					ch->flags &=
+					    ~IEEE80211_CHAN_PASSIVE_SCAN;
+			} else {
+				if (ch->beacon_found)
+					ch->flags &= ~(IEEE80211_CHAN_NO_IBSS |
+						   IEEE80211_CHAN_PASSIVE_SCAN);
+			}
+		}
+	}
+}
+
+/* Allows active scan scan on Ch 12 and 13 */
+static void _rtl_reg_apply_active_scan_flags(struct wiphy *wiphy,
+					     enum nl80211_reg_initiator
+					     initiator)
+{
+	struct ieee80211_supported_band *sband;
+	struct ieee80211_channel *ch;
+	const struct ieee80211_reg_rule *reg_rule;
+
+	if (!wiphy->bands[NL80211_BAND_2GHZ])
+		return;
+	sband = wiphy->bands[NL80211_BAND_2GHZ];
+
+	/*
+	 *If no country IE has been received always enable active scan
+	 *on these channels. This is only done for specific regulatory SKUs
+	 */
+	if (initiator != NL80211_REGDOM_SET_BY_COUNTRY_IE) {
+		ch = &sband->channels[11];	/* CH 12 */
+		if (ch->flags & IEEE80211_CHAN_PASSIVE_SCAN)
+			ch->flags &= ~IEEE80211_CHAN_PASSIVE_SCAN;
+		ch = &sband->channels[12];	/* CH 13 */
+		if (ch->flags & IEEE80211_CHAN_PASSIVE_SCAN)
+			ch->flags &= ~IEEE80211_CHAN_PASSIVE_SCAN;
+		return;
+	}
+
+	/*If a country IE has been received check its rule for this
+	 *channel first before enabling active scan. The passive scan
+	 *would have been enforced by the initial processing of our
+	 *custom regulatory domain.
+	 */
+
+	ch = &sband->channels[11];	/* CH 12 */
+	reg_rule = freq_reg_info(wiphy, ch->center_freq);
+	if (!IS_ERR(reg_rule)) {
+		if (!(reg_rule->flags & NL80211_RRF_PASSIVE_SCAN))
+			if (ch->flags & IEEE80211_CHAN_PASSIVE_SCAN)
+				ch->flags &= ~IEEE80211_CHAN_PASSIVE_SCAN;
+	}
+
+	ch = &sband->channels[12];	/* CH 13 */
+	reg_rule = freq_reg_info(wiphy, ch->center_freq);
+	if (!IS_ERR(reg_rule)) {
+		if (!(reg_rule->flags & NL80211_RRF_PASSIVE_SCAN))
+			if (ch->flags & IEEE80211_CHAN_PASSIVE_SCAN)
+				ch->flags &= ~IEEE80211_CHAN_PASSIVE_SCAN;
+	}
+}
+
+/*
+ *Always apply Radar/DFS rules on
+ *freq range 5260 MHz - 5700 MHz
+ */
+static void _rtl_reg_apply_radar_flags(struct wiphy *wiphy)
+{
+	struct ieee80211_supported_band *sband;
+	struct ieee80211_channel *ch;
+	unsigned int i;
+
+	if (!wiphy->bands[NL80211_BAND_5GHZ])
+		return;
+
+	sband = wiphy->bands[NL80211_BAND_5GHZ];
+
+	for (i = 0; i < sband->n_channels; i++) {
+		ch = &sband->channels[i];
+		if (!_rtl_is_radar_freq(ch->center_freq))
+			continue;
+
+		/*
+		 *We always enable radar detection/DFS on this
+		 *frequency range. Additionally we also apply on
+		 *this frequency range:
+		 *- If STA mode does not yet have DFS supports disable
+		 * active scanning
+		 *- If adhoc mode does not support DFS yet then disable
+		 * adhoc in the frequency.
+		 *- If AP mode does not yet support radar detection/DFS
+		 *do not allow AP mode
+		 */
+		if (!(ch->flags & IEEE80211_CHAN_DISABLED))
+			ch->flags |= IEEE80211_CHAN_RADAR |
+			    IEEE80211_CHAN_NO_IBSS |
+			    IEEE80211_CHAN_PASSIVE_SCAN;
+	}
+}
+
+static void _rtl_reg_apply_world_flags(struct wiphy *wiphy,
+				       enum nl80211_reg_initiator initiator,
+				       struct rtl_regulatory *reg)
+{
+	_rtl_reg_apply_beaconing_flags(wiphy, initiator);
+	_rtl_reg_apply_active_scan_flags(wiphy, initiator);
+}
+
+static void _rtl_dump_channel_map(struct wiphy *wiphy)
+{
+	enum nl80211_band band;
+	struct ieee80211_supported_band *sband;
+	struct ieee80211_channel *ch;
+	unsigned int i;
+
+	for (band = 0; band < NUM_NL80211_BANDS; band++) {
+		if (!wiphy->bands[band])
+			continue;
+		sband = wiphy->bands[band];
+		for (i = 0; i < sband->n_channels; i++)
+			ch = &sband->channels[i];
+	}
+}
+
+static int _rtl_reg_notifier_apply(struct wiphy *wiphy,
+				   struct regulatory_request *request,
+				   struct rtl_regulatory *reg)
+{
+	/* We always apply this */
+	_rtl_reg_apply_radar_flags(wiphy);
+
+	switch (request->initiator) {
+	case NL80211_REGDOM_SET_BY_DRIVER:
+	case NL80211_REGDOM_SET_BY_CORE:
+	case NL80211_REGDOM_SET_BY_USER:
+		break;
+	case NL80211_REGDOM_SET_BY_COUNTRY_IE:
+		_rtl_reg_apply_world_flags(wiphy, request->initiator, reg);
+		break;
+	}
+
+	_rtl_dump_channel_map(wiphy);
+
+	return 0;
+}
+
+static const struct ieee80211_regdomain *_rtl_regdomain_select(
+						struct rtl_regulatory *reg)
+{
+	switch (reg->country_code) {
+	case COUNTRY_CODE_FCC:
+		return &rtl_regdom_no_midband;
+	case COUNTRY_CODE_IC:
+		return &rtl_regdom_11;
+	case COUNTRY_CODE_TELEC_NETGEAR:
+		return &rtl_regdom_60_64;
+	case COUNTRY_CODE_ETSI:
+	case COUNTRY_CODE_SPAIN:
+	case COUNTRY_CODE_FRANCE:
+	case COUNTRY_CODE_ISRAEL:
+		return &rtl_regdom_12_13;
+	case COUNTRY_CODE_MKK:
+	case COUNTRY_CODE_MKK1:
+	case COUNTRY_CODE_TELEC:
+	case COUNTRY_CODE_MIC:
+		return &rtl_regdom_14_60_64;
+	case COUNTRY_CODE_GLOBAL_DOMAIN:
+		return &rtl_regdom_14;
+	case COUNTRY_CODE_WORLD_WIDE_13:
+	case COUNTRY_CODE_WORLD_WIDE_13_5G_ALL:
+		return &rtl_regdom_12_13_5g_all;
+	default:
+		return &rtl_regdom_no_midband;
+	}
+}
+
+static int _rtl_regd_init_wiphy(struct rtl_regulatory *reg,
+				struct wiphy *wiphy,
+				void (*reg_notifier)(struct wiphy *wiphy,
+						     struct regulatory_request *
+						     request))
+{
+	const struct ieee80211_regdomain *regd;
+
+	wiphy->reg_notifier = reg_notifier;
+
+	wiphy->regulatory_flags |= REGULATORY_CUSTOM_REG;
+	wiphy->regulatory_flags &= ~REGULATORY_STRICT_REG;
+	wiphy->regulatory_flags &= ~REGULATORY_DISABLE_BEACON_HINTS;
+	regd = _rtl_regdomain_select(reg);
+	wiphy_apply_custom_regulatory(wiphy, regd);
+	_rtl_reg_apply_radar_flags(wiphy);
+	_rtl_reg_apply_world_flags(wiphy, NL80211_REGDOM_SET_BY_DRIVER, reg);
+	return 0;
+}
+
+static struct country_code_to_enum_rd *_rtl_regd_find_country(u16 countrycode)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(allcountries); i++) {
+		if (allcountries[i].countrycode == countrycode)
+			return &allcountries[i];
+	}
+	return NULL;
+}
+
+static u8 channel_plan_to_country_code(u8 channelplan)
+{
+	switch (channelplan) {
+	case 0x20:
+	case 0x21:
+		return COUNTRY_CODE_WORLD_WIDE_13;
+	case 0x22:
+		return COUNTRY_CODE_IC;
+	case 0x25:
+		return COUNTRY_CODE_ETSI;
+	case 0x32:
+		return COUNTRY_CODE_TELEC_NETGEAR;
+	case 0x41:
+		return COUNTRY_CODE_GLOBAL_DOMAIN;
+	case 0x7f:
+		return COUNTRY_CODE_WORLD_WIDE_13_5G_ALL;
+	default:
+		return COUNTRY_CODE_MAX; /*Error*/
+	}
+}
+
+int rtl_regd_init(struct ieee80211_hw *hw,
+		  void (*reg_notifier)(struct wiphy *wiphy,
+				       struct regulatory_request *request))
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct wiphy *wiphy = hw->wiphy;
+	struct country_code_to_enum_rd *country = NULL;
+
+	if (!wiphy || !&rtlpriv->regd)
+		return -EINVAL;
+
+	/* init country_code from efuse channel plan */
+	rtlpriv->regd.country_code =
+		channel_plan_to_country_code(rtlpriv->efuse.channel_plan);
+
+	RT_TRACE(rtlpriv, COMP_REGD, DBG_DMESG,
+		 "rtl: EEPROM regdomain: 0x%0x country code: %d\n",
+		 rtlpriv->efuse.channel_plan, rtlpriv->regd.country_code);
+
+	if (rtlpriv->regd.country_code >= COUNTRY_CODE_MAX) {
+		RT_TRACE(rtlpriv, COMP_REGD, DBG_DMESG,
+			 "rtl: EEPROM indicates invalid country code, world wide 13 should be used\n");
+
+		rtlpriv->regd.country_code = COUNTRY_CODE_WORLD_WIDE_13;
+	}
+
+	country = _rtl_regd_find_country(rtlpriv->regd.country_code);
+
+	if (country) {
+		rtlpriv->regd.alpha2[0] = country->iso_name[0];
+		rtlpriv->regd.alpha2[1] = country->iso_name[1];
+	} else {
+		rtlpriv->regd.alpha2[0] = '0';
+		rtlpriv->regd.alpha2[1] = '0';
+	}
+
+	RT_TRACE(rtlpriv, COMP_REGD, DBG_TRACE,
+		 "rtl: Country alpha2 being used: %c%c\n",
+		  rtlpriv->regd.alpha2[0], rtlpriv->regd.alpha2[1]);
+
+	_rtl_regd_init_wiphy(&rtlpriv->regd, wiphy, reg_notifier);
+
+	return 0;
+}
+
+void rtl_reg_notifier(struct wiphy *wiphy, struct regulatory_request *request)
+{
+	struct ieee80211_hw *hw = wiphy_to_ieee80211_hw(wiphy);
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	RT_TRACE(rtlpriv, COMP_REGD, DBG_LOUD, "\n");
+
+	_rtl_reg_notifier_apply(wiphy, request, &rtlpriv->regd);
+}
diff --git a/drivers/staging/rtlwifi/regd.h b/drivers/staging/rtlwifi/regd.h
new file mode 100644
index 000000000000..5626015a6d0d
--- /dev/null
+++ b/drivers/staging/rtlwifi/regd.h
@@ -0,0 +1,63 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2012  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __RTL_REGD_H__
+#define __RTL_REGD_H__
+
+/* for kernel 3.14 , both value are changed to IEEE80211_CHAN_NO_IR*/
+#define IEEE80211_CHAN_NO_IBSS IEEE80211_CHAN_NO_IR
+#define IEEE80211_CHAN_PASSIVE_SCAN IEEE80211_CHAN_NO_IR
+
+struct country_code_to_enum_rd {
+	u16 countrycode;
+	const char *iso_name;
+};
+
+enum country_code_type_t {
+	COUNTRY_CODE_FCC = 0,
+	COUNTRY_CODE_IC = 1,
+	COUNTRY_CODE_ETSI = 2,
+	COUNTRY_CODE_SPAIN = 3,
+	COUNTRY_CODE_FRANCE = 4,
+	COUNTRY_CODE_MKK = 5,
+	COUNTRY_CODE_MKK1 = 6,
+	COUNTRY_CODE_ISRAEL = 7,
+	COUNTRY_CODE_TELEC = 8,
+	COUNTRY_CODE_MIC = 9,
+	COUNTRY_CODE_GLOBAL_DOMAIN = 10,
+	COUNTRY_CODE_WORLD_WIDE_13 = 11,
+	COUNTRY_CODE_TELEC_NETGEAR = 12,
+	COUNTRY_CODE_WORLD_WIDE_13_5G_ALL = 13,
+
+	/*add new channel plan above this line */
+	COUNTRY_CODE_MAX
+};
+
+int rtl_regd_init(struct ieee80211_hw *hw,
+		  void (*reg_notifier)(struct wiphy *wiphy,
+				       struct regulatory_request *request));
+void rtl_reg_notifier(struct wiphy *wiphy, struct regulatory_request *request);
+
+#endif
diff --git a/drivers/staging/rtlwifi/rtl8822be/Makefile b/drivers/staging/rtlwifi/rtl8822be/Makefile
new file mode 100644
index 000000000000..d535ff8febf1
--- /dev/null
+++ b/drivers/staging/rtlwifi/rtl8822be/Makefile
@@ -0,0 +1,7 @@
+rtl8822be-objs :=		\
+		fw.o		\
+		hw.o		\
+		led.o		\
+		phy.o		\
+		sw.o		\
+		trx.o
diff --git a/drivers/staging/rtlwifi/rtl8822be/def.h b/drivers/staging/rtlwifi/rtl8822be/def.h
new file mode 100644
index 000000000000..7942ddfdcf43
--- /dev/null
+++ b/drivers/staging/rtlwifi/rtl8822be/def.h
@@ -0,0 +1,82 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __RTL8822B_DEF_H__
+#define __RTL8822B_DEF_H__
+
+#define RX_DESC_NUM_8822BE	512
+
+#define HAL_PRIME_CHNL_OFFSET_DONT_CARE	0
+#define HAL_PRIME_CHNL_OFFSET_LOWER	1
+#define HAL_PRIME_CHNL_OFFSET_UPPER	2
+
+#define RX_MPDU_QUEUE	0
+
+#define IS_HT_RATE(_rate) (_rate >= DESC_RATEMCS0)
+#define IS_CCK_RATE(_rate) (_rate >= DESC_RATE1M && _rate <= DESC_RATE11M)
+#define IS_OFDM_RATE(_rate) (_rate >= DESC_RATE6M && _rate <= DESC_RATE54M)
+#define IS_1T_RATE(_rate)                                                      \
+	((_rate >= DESC_RATE1M && _rate <= DESC_RATEMCS7) ||                   \
+	 (_rate >= DESC_RATEVHT1SS_MCS0 && _rate <= DESC_RATEVHT1SS_MCS9))
+#define IS_2T_RATE(_rate)                                                      \
+	((_rate >= DESC_RATEMCS8 && _rate <= DESC_RATEMCS15) ||                \
+	 (_rate >= DESC_RATEVHT2SS_MCS0 && _rate <= DESC_RATEVHT2SS_MCS9))
+
+#define IS_1T_RATESEC(_rs)                                                     \
+	((_rs == CCK) || (_rs == OFDM) || (_rs == HT_MCS0_MCS7) ||             \
+	 (_rs == VHT_1SSMCS0_1SSMCS9))
+#define IS_2T_RATESEC(_rs)                                                     \
+	((_rs == HT_MCS8_MCS15) || (_rs == VHT_2SSMCS0_2SSMCS9))
+
+enum rx_packet_type {
+	NORMAL_RX,
+	C2H_PACKET,
+};
+
+enum rtl_desc_qsel {
+	QSLT_BK	= 0x2,
+	QSLT_BE	= 0x0,
+	QSLT_VI	= 0x5,
+	QSLT_VO	= 0x7,
+	QSLT_BEACON	= 0x10,
+	QSLT_HIGH	= 0x11,
+	QSLT_MGNT	= 0x12,
+	QSLT_CMD	= 0x13,
+};
+
+enum vht_data_sc {
+	VHT_DATA_SC_DONOT_CARE	= 0,
+	VHT_DATA_SC_20_UPPER_OF_80MHZ	= 1,
+	VHT_DATA_SC_20_LOWER_OF_80MHZ	= 2,
+	VHT_DATA_SC_20_UPPERST_OF_80MHZ	= 3,
+	VHT_DATA_SC_20_LOWEST_OF_80MHZ	= 4,
+	VHT_DATA_SC_20_RECV1	= 5,
+	VHT_DATA_SC_20_RECV2	= 6,
+	VHT_DATA_SC_20_RECV3	= 7,
+	VHT_DATA_SC_20_RECV4	= 8,
+	VHT_DATA_SC_40_UPPER_OF_80MHZ	= 9,
+	VHT_DATA_SC_40_LOWER_OF_80MHZ	= 10,
+};
+#endif
diff --git a/drivers/staging/rtlwifi/rtl8822be/fw.c b/drivers/staging/rtlwifi/rtl8822be/fw.c
new file mode 100644
index 000000000000..8e24da16752c
--- /dev/null
+++ b/drivers/staging/rtlwifi/rtl8822be/fw.c
@@ -0,0 +1,968 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#include "../wifi.h"
+#include "../pci.h"
+#include "../base.h"
+#include "reg.h"
+#include "def.h"
+#include "fw.h"
+
+static bool _rtl8822be_check_fw_read_last_h2c(struct ieee80211_hw *hw,
+					      u8 boxnum)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 val_hmetfr;
+	bool result = false;
+
+	val_hmetfr = rtl_read_byte(rtlpriv, REG_HMETFR_8822B);
+	if (((val_hmetfr >> boxnum) & BIT(0)) == 0)
+		result = true;
+	return result;
+}
+
+static void _rtl8822be_fill_h2c_command(struct ieee80211_hw *hw, u8 element_id,
+					u32 cmd_len, u8 *cmdbuffer)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	u8 boxnum;
+	u16 box_reg = 0, box_extreg = 0;
+	u8 u1b_tmp;
+	bool isfw_read;
+	u8 buf_index = 0;
+	bool bwrite_success = false;
+	u8 wait_h2c_limmit = 100;
+	u8 boxcontent[4], boxextcontent[4];
+	u32 h2c_waitcounter = 0;
+	unsigned long flag;
+	u8 idx;
+
+	/* 1. Prevent race condition in setting H2C cmd.
+	 * (copy from MgntActSet_RF_State().)
+	 */
+	while (true) {
+		spin_lock_irqsave(&rtlpriv->locks.h2c_lock, flag);
+		if (rtlhal->h2c_setinprogress) {
+			RT_TRACE(rtlpriv, COMP_CMD, DBG_LOUD,
+				 "H2C set in progress! wait..H2C_ID=%d.\n",
+				 element_id);
+
+			while (rtlhal->h2c_setinprogress) {
+				spin_unlock_irqrestore(&rtlpriv->locks.h2c_lock,
+						       flag);
+				h2c_waitcounter++;
+				RT_TRACE(rtlpriv, COMP_CMD, DBG_LOUD,
+					 "Wait 100 us (%d times)...\n",
+					 h2c_waitcounter);
+				udelay(100);
+
+				if (h2c_waitcounter > 1000)
+					return;
+				spin_lock_irqsave(&rtlpriv->locks.h2c_lock,
+						  flag);
+			}
+			spin_unlock_irqrestore(&rtlpriv->locks.h2c_lock, flag);
+		} else {
+			rtlhal->h2c_setinprogress = true;
+			spin_unlock_irqrestore(&rtlpriv->locks.h2c_lock, flag);
+			break;
+		}
+	}
+
+	while (!bwrite_success) {
+		/* 2. Find the last BOX number which has been writen. */
+		boxnum = rtlhal->last_hmeboxnum;
+		switch (boxnum) {
+		case 0:
+			box_reg = REG_HMEBOX0_8822B;
+			box_extreg = REG_HMEBOX_E0_8822B;
+			break;
+		case 1:
+			box_reg = REG_HMEBOX1_8822B;
+			box_extreg = REG_HMEBOX_E1_8822B;
+			break;
+		case 2:
+			box_reg = REG_HMEBOX2_8822B;
+			box_extreg = REG_HMEBOX_E2_8822B;
+			break;
+		case 3:
+			box_reg = REG_HMEBOX3_8822B;
+			box_extreg = REG_HMEBOX_E3_8822B;
+			break;
+		default:
+			RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
+				 "switch case not process\n");
+			break;
+		}
+
+		/* 3. Check if the box content is empty. */
+		u1b_tmp = rtl_read_byte(rtlpriv, REG_CR_8822B);
+
+		if (u1b_tmp == 0xea) {
+			if (rtl_read_byte(rtlpriv, REG_TXDMA_STATUS_8822B) ==
+				    0xea ||
+			    rtl_read_byte(rtlpriv, REG_TXPKT_EMPTY_8822B) ==
+				    0xea)
+				rtl_write_byte(rtlpriv, REG_SYS_CFG1_8822B + 3,
+					       0xff);
+
+			RT_TRACE(rtlpriv, COMP_CMD, DBG_LOUD,
+				 "REG_CR is unavaliable\n");
+			break;
+		}
+
+		wait_h2c_limmit = 100;
+		isfw_read = _rtl8822be_check_fw_read_last_h2c(hw, boxnum);
+		while (!isfw_read) {
+			wait_h2c_limmit--;
+			if (wait_h2c_limmit == 0) {
+				RT_TRACE(rtlpriv, COMP_CMD, DBG_WARNING,
+					 "Wait too long for FW clear MB%d!!!\n",
+					 boxnum);
+				break;
+			}
+			udelay(10);
+			isfw_read =
+				_rtl8822be_check_fw_read_last_h2c(hw, boxnum);
+			u1b_tmp = rtl_read_byte(rtlpriv, 0x130);
+			RT_TRACE(rtlpriv, COMP_CMD, DBG_LOUD,
+				 "Waiting for FW clear MB%d!!! 0x130 = %2x\n",
+				 boxnum, u1b_tmp);
+		}
+
+		/* If Fw has not read the last H2C cmd,
+		 * break and give up this H2C.
+		 */
+		if (!isfw_read) {
+			RT_TRACE(rtlpriv, COMP_CMD, DBG_LOUD,
+				 "Write H2C reg BOX[%d] fail,Fw don't read.\n",
+				 boxnum);
+			break;
+		}
+		/* 4. Fill the H2C cmd into box */
+		memset(boxcontent, 0, sizeof(boxcontent));
+		memset(boxextcontent, 0, sizeof(boxextcontent));
+		boxcontent[0] = element_id;
+		RT_TRACE(rtlpriv, COMP_CMD, DBG_LOUD,
+			 "Write element_id box_reg(%4x) = %2x\n", box_reg,
+			 element_id);
+
+		switch (cmd_len) {
+		case 1:
+		case 2:
+		case 3:
+			/*boxcontent[0] &= ~(BIT(7));*/
+			memcpy((u8 *)(boxcontent) + 1, cmdbuffer + buf_index,
+			       cmd_len);
+
+			for (idx = 0; idx < 4; idx++) {
+				rtl_write_byte(rtlpriv, box_reg + idx,
+					       boxcontent[idx]);
+			}
+			break;
+		case 4:
+		case 5:
+		case 6:
+		case 7:
+			/*boxcontent[0] |= (BIT(7));*/
+			memcpy((u8 *)(boxextcontent), cmdbuffer + buf_index + 3,
+			       cmd_len - 3);
+			memcpy((u8 *)(boxcontent) + 1, cmdbuffer + buf_index,
+			       3);
+
+			for (idx = 0; idx < 4; idx++) {
+				rtl_write_byte(rtlpriv, box_extreg + idx,
+					       boxextcontent[idx]);
+			}
+
+			for (idx = 0; idx < 4; idx++) {
+				rtl_write_byte(rtlpriv, box_reg + idx,
+					       boxcontent[idx]);
+			}
+			break;
+		default:
+			RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
+				 "switch case not process\n");
+			break;
+		}
+
+		bwrite_success = true;
+
+		rtlhal->last_hmeboxnum = boxnum + 1;
+		if (rtlhal->last_hmeboxnum == 4)
+			rtlhal->last_hmeboxnum = 0;
+
+		RT_TRACE(rtlpriv, COMP_CMD, DBG_LOUD,
+			 "pHalData->last_hmeboxnum  = %d\n",
+			 rtlhal->last_hmeboxnum);
+	}
+
+	spin_lock_irqsave(&rtlpriv->locks.h2c_lock, flag);
+	rtlhal->h2c_setinprogress = false;
+	spin_unlock_irqrestore(&rtlpriv->locks.h2c_lock, flag);
+
+	RT_TRACE(rtlpriv, COMP_CMD, DBG_LOUD, "go out\n");
+}
+
+void rtl8822be_fill_h2c_cmd(struct ieee80211_hw *hw, u8 element_id, u32 cmd_len,
+			    u8 *cmdbuffer)
+{
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 tmp_cmdbuf[8];
+
+	if (!rtlhal->fw_ready) {
+		WARN_ONCE(true,
+			  "return H2C cmd because of Fw download fail!!!\n");
+		return;
+	}
+
+	memset(tmp_cmdbuf, 0, 8);
+	memcpy(tmp_cmdbuf, cmdbuffer, cmd_len);
+
+	RT_TRACE(rtlpriv, COMP_CMD, DBG_DMESG,
+		 "h2c cmd: len=%d %02X%02X%02X%02X %02X%02X%02X%02X\n", cmd_len,
+		 tmp_cmdbuf[2], tmp_cmdbuf[1], tmp_cmdbuf[0], element_id,
+		 tmp_cmdbuf[6], tmp_cmdbuf[5], tmp_cmdbuf[4], tmp_cmdbuf[3]);
+
+	_rtl8822be_fill_h2c_command(hw, element_id, cmd_len, tmp_cmdbuf);
+}
+
+void rtl8822be_set_default_port_id_cmd(struct ieee80211_hw *hw)
+{
+	u8 h2c_set_default_port_id[H2C_DEFAULT_PORT_ID_LEN];
+
+	SET_H2CCMD_DFTPID_PORT_ID(h2c_set_default_port_id, 0);
+	SET_H2CCMD_DFTPID_MAC_ID(h2c_set_default_port_id, 0);
+
+	rtl8822be_fill_h2c_cmd(hw, H2C_8822B_DEFAULT_PORT_ID,
+			       H2C_DEFAULT_PORT_ID_LEN,
+			       h2c_set_default_port_id);
+}
+
+void rtl8822be_set_fw_pwrmode_cmd(struct ieee80211_hw *hw, u8 mode)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 u1_h2c_set_pwrmode[H2C_8822B_PWEMODE_LENGTH] = {0};
+	static u8 prev_h2c[H2C_8822B_PWEMODE_LENGTH] = {0};
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	u8 rlbm, power_state = 0, byte5 = 0;
+	u8 awake_intvl; /* DTIM = (awake_intvl - 1) */
+	u8 smart_ps = 0;
+	struct rtl_btc_ops *btc_ops = rtlpriv->btcoexist.btc_ops;
+	bool bt_ctrl_lps = (rtlpriv->cfg->ops->get_btc_status() ?
+			    btc_ops->btc_is_bt_ctrl_lps(rtlpriv) : false);
+	bool bt_lps_on = (rtlpriv->cfg->ops->get_btc_status() ?
+			  btc_ops->btc_is_bt_lps_on(rtlpriv) : false);
+
+	memset(u1_h2c_set_pwrmode, 0, H2C_8822B_PWEMODE_LENGTH);
+
+	if (bt_ctrl_lps)
+		mode = (bt_lps_on ? FW_PS_MIN_MODE : FW_PS_ACTIVE_MODE);
+
+	RT_TRACE(rtlpriv, COMP_POWER, DBG_DMESG, "FW LPS mode = %d (coex:%d)\n",
+		 mode, bt_ctrl_lps);
+
+	switch (mode) {
+	case FW_PS_MIN_MODE:
+		rlbm = 0;
+		awake_intvl = 2;
+		smart_ps = ppsc->smart_ps;
+		break;
+	case FW_PS_MAX_MODE:
+		rlbm = 1;
+		awake_intvl = 2;
+		smart_ps = ppsc->smart_ps;
+		break;
+	case FW_PS_DTIM_MODE:
+		rlbm = 2;
+		awake_intvl = ppsc->reg_max_lps_awakeintvl;
+		/*
+		 * hw->conf.ps_dtim_period or mac->vif->bss_conf.dtim_period
+		 * is only used in swlps.
+		 */
+		smart_ps = ppsc->smart_ps;
+		break;
+	case FW_PS_ACTIVE_MODE:
+		rlbm = 0;
+		awake_intvl = 1;
+		break;
+	default:
+		rlbm = 2;
+		awake_intvl = 4;
+		smart_ps = ppsc->smart_ps;
+		break;
+	}
+
+	if (rtlpriv->mac80211.p2p) {
+		awake_intvl = 2;
+		rlbm = 1;
+	}
+
+	if (mode == FW_PS_ACTIVE_MODE) {
+		byte5 = 0x40;
+		power_state = FW_PWR_STATE_ACTIVE;
+	} else {
+		if (bt_ctrl_lps) {
+			byte5 = btc_ops->btc_get_lps_val(rtlpriv);
+			power_state = btc_ops->btc_get_rpwm_val(rtlpriv);
+
+			if ((rlbm == 2) && (byte5 & BIT(4))) {
+				/* Keep awake interval to 1 to prevent from
+				 * decreasing coex performance
+				 */
+				awake_intvl = 2;
+				rlbm = 2;
+			}
+			smart_ps = 0;
+		} else {
+			byte5 = 0x40;
+			power_state = FW_PWR_STATE_RF_OFF;
+		}
+	}
+
+	SET_H2CCMD_PWRMODE_PARM_MODE(u1_h2c_set_pwrmode, ((mode) ? 1 : 0));
+	SET_H2CCMD_PWRMODE_PARM_RLBM(u1_h2c_set_pwrmode, rlbm);
+	SET_H2CCMD_PWRMODE_PARM_SMART_PS(u1_h2c_set_pwrmode, smart_ps);
+	SET_H2CCMD_PWRMODE_PARM_AWAKE_INTERVAL(u1_h2c_set_pwrmode, awake_intvl);
+	SET_H2CCMD_PWRMODE_PARM_ALL_QUEUE_UAPSD(u1_h2c_set_pwrmode, 0);
+	SET_H2CCMD_PWRMODE_PARM_PWR_STATE(u1_h2c_set_pwrmode, power_state);
+	SET_H2CCMD_PWRMODE_PARM_BYTE5(u1_h2c_set_pwrmode, byte5);
+
+	RT_PRINT_DATA(rtlpriv, COMP_CMD, DBG_DMESG,
+		      "rtl8822be_set_fw_pwrmode(): u1_h2c_set_pwrmode\n",
+		      u1_h2c_set_pwrmode, H2C_8822B_PWEMODE_LENGTH);
+	if (rtlpriv->cfg->ops->get_btc_status())
+		btc_ops->btc_record_pwr_mode(rtlpriv, u1_h2c_set_pwrmode,
+					     H2C_8822B_PWEMODE_LENGTH);
+
+	if (!memcmp(prev_h2c, u1_h2c_set_pwrmode, H2C_8822B_PWEMODE_LENGTH))
+		return;
+	memcpy(prev_h2c, u1_h2c_set_pwrmode, H2C_8822B_PWEMODE_LENGTH);
+
+	rtl8822be_set_default_port_id_cmd(hw);
+	rtl8822be_fill_h2c_cmd(hw, H2C_8822B_SETPWRMODE,
+			       H2C_8822B_PWEMODE_LENGTH, u1_h2c_set_pwrmode);
+}
+
+void rtl8822be_set_fw_media_status_rpt_cmd(struct ieee80211_hw *hw, u8 mstatus)
+{
+	u8 parm[4] = {0, 0, 0, 0};
+	/* parm[0]: bit0=0-->Disconnect, bit0=1-->Connect
+	 *          bit1=0-->update Media Status to MACID
+	 *          bit1=1-->update Media Status from MACID to MACID_End
+	 * parm[1]: MACID, if this is INFRA_STA, MacID = 0
+	 * parm[2]: MACID_End
+	 * parm[3]: bit2-0: port ID
+	 */
+
+	SET_H2CCMD_MSRRPT_PARM_OPMODE(parm, mstatus);
+	SET_H2CCMD_MSRRPT_PARM_MACID_IND(parm, 0);
+
+	rtl8822be_fill_h2c_cmd(hw, H2C_8822B_MSRRPT, 4, parm);
+}
+
+static bool _rtl8822be_send_bcn_or_cmd_packet(struct ieee80211_hw *hw,
+					      struct sk_buff *skb, u8 hw_queue)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	struct rtl8192_tx_ring *ring;
+	struct rtl_tx_desc *pdesc;
+	struct rtl_tx_buffer_desc *pbd_desc;
+	unsigned long flags;
+	struct sk_buff *pskb = NULL;
+	u8 *pdesc_or_bddesc;
+	dma_addr_t dma_addr;
+
+	if (hw_queue != BEACON_QUEUE && hw_queue != H2C_QUEUE)
+		return false;
+
+	ring = &rtlpci->tx_ring[hw_queue];
+
+	spin_lock_irqsave(&rtlpriv->locks.irq_th_lock, flags);
+
+	if (hw_queue == BEACON_QUEUE) {
+		pdesc = &ring->desc[0];
+		pbd_desc = &ring->buffer_desc[0];
+		pdesc_or_bddesc = (u8 *)pbd_desc;
+
+		/* free previous beacon queue */
+		pskb = __skb_dequeue(&ring->queue);
+
+		if (!pskb)
+			goto free_prev_skb_done;
+
+		dma_addr = rtlpriv->cfg->ops->get_desc(
+				hw, (u8 *)pbd_desc, true, HW_DESC_TXBUFF_ADDR);
+
+		pci_unmap_single(rtlpci->pdev, dma_addr, skb->len,
+				 PCI_DMA_TODEVICE);
+		kfree_skb(pskb);
+
+free_prev_skb_done:
+		;
+
+	} else { /* hw_queue == TXCMD_QUEUE */
+		if (rtlpriv->cfg->ops->get_available_desc(hw, hw_queue) == 0) {
+			RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING,
+				 "get_available_desc fail hw_queue=%d\n",
+				 hw_queue);
+			spin_unlock_irqrestore(&rtlpriv->locks.irq_th_lock,
+					       flags);
+			return false;
+		}
+
+		pdesc = &ring->desc[ring->cur_tx_wp];
+		pbd_desc = &ring->buffer_desc[ring->cur_tx_wp];
+		pdesc_or_bddesc = (u8 *)pdesc;
+	}
+
+	rtlpriv->cfg->ops->fill_tx_special_desc(hw, (u8 *)pdesc, (u8 *)pbd_desc,
+						skb, hw_queue);
+
+	__skb_queue_tail(&ring->queue, skb);
+
+	rtlpriv->cfg->ops->set_desc(hw, (u8 *)pdesc_or_bddesc, true,
+				    HW_DESC_OWN, (u8 *)&hw_queue);
+
+	spin_unlock_irqrestore(&rtlpriv->locks.irq_th_lock, flags);
+
+	rtlpriv->cfg->ops->tx_polling(hw, hw_queue);
+
+	return true;
+}
+
+bool rtl8822b_halmac_cb_write_data_rsvd_page(struct rtl_priv *rtlpriv, u8 *buf,
+					     u32 size)
+{
+	struct sk_buff *skb = NULL;
+	u8 u1b_tmp;
+	int count;
+
+	skb = dev_alloc_skb(size);
+	memcpy((u8 *)skb_put(skb, size), buf, size);
+
+	if (!_rtl8822be_send_bcn_or_cmd_packet(rtlpriv->hw, skb, BEACON_QUEUE))
+		return false;
+
+	/* These code isn't actually need, because halmac will check
+	 * BCN_VALID
+	 */
+
+	/* Polling Beacon Queue to send Beacon */
+	u1b_tmp = rtl_read_byte(rtlpriv, REG_RX_RXBD_NUM_8822B + 1);
+	count = 0;
+	while ((count < 20) && (u1b_tmp & BIT(4))) {
+		count++;
+		udelay(10);
+		u1b_tmp = rtl_read_byte(rtlpriv, REG_RX_RXBD_NUM_8822B + 1);
+	}
+
+	if (count >= 20)
+		pr_err("%s polling beacon fail\n", __func__);
+
+	return true;
+}
+
+bool rtl8822b_halmac_cb_write_data_h2c(struct rtl_priv *rtlpriv, u8 *buf,
+				       u32 size)
+{
+	struct sk_buff *skb = NULL;
+
+	/* without GFP_DMA, pci_map_single() may not work */
+	skb = __netdev_alloc_skb(NULL, size, GFP_ATOMIC | GFP_DMA);
+	memcpy((u8 *)skb_put(skb, size), buf, size);
+
+	return _rtl8822be_send_bcn_or_cmd_packet(rtlpriv->hw, skb, H2C_QUEUE);
+}
+
+/* Rsvd page HALMAC_RSVD_DRV_PGNUM_8822B occupies 16 page (2048 byte) */
+#define BEACON_PG	0 /* ->1 */
+#define PSPOLL_PG	2
+#define NULL_PG	3
+#define PROBERSP_PG	4 /* ->5 */
+#define QOS_NULL_PG	6
+#define BT_QOS_NULL_PG	7
+
+#define TOTAL_RESERVED_PKT_LEN	1024
+
+static u8 reserved_page_packet[TOTAL_RESERVED_PKT_LEN] = {/* page size = 128 */
+	/* page 0 beacon */
+	0x80, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF,
+	0xFF, 0xFF, 0x00, 0xE0, 0x4C, 0x02, 0xB1, 0x78,
+	0xEC, 0x1A, 0x59, 0x0B, 0xAD, 0xD4, 0x20, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x64, 0x00, 0x10, 0x04, 0x00, 0x05, 0x54, 0x65,
+	0x73, 0x74, 0x32, 0x01, 0x08, 0x82, 0x84, 0x0B,
+	0x16, 0x24, 0x30, 0x48, 0x6C, 0x03, 0x01, 0x06,
+	0x06, 0x02, 0x00, 0x00, 0x2A, 0x01, 0x02, 0x32,
+	0x04, 0x0C, 0x12, 0x18, 0x60, 0x2D, 0x1A, 0x6C,
+	0x09, 0x03, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x3D, 0x00, 0xDD, 0x07, 0x00, 0xE0, 0x4C,
+	0x02, 0x02, 0x00, 0x00, 0xDD, 0x18, 0x00, 0x50,
+	0xF2, 0x01, 0x01, 0x00, 0x00, 0x50, 0xF2, 0x04,
+	0x01, 0x00, 0x00, 0x50, 0xF2, 0x04, 0x01, 0x00,
+
+	/* page 1 beacon */
+	0x00, 0x50, 0xF2, 0x02, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x10, 0x00, 0x30, 0x84, 0x00, 0x12, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x81, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+
+	/* page 2  ps-poll */
+	0xA4, 0x10, 0x01, 0xC0, 0xEC, 0x1A, 0x59, 0x0B,
+	0xAD, 0xD4, 0x00, 0xE0, 0x4C, 0x02, 0xB1, 0x78,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x18, 0x00, 0x30, 0x84, 0x00, 0x12, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+
+	/* page 3  null */
+	0x48, 0x01, 0x00, 0x00, 0xEC, 0x1A, 0x59, 0x0B,
+	0xAD, 0xD4, 0x00, 0xE0, 0x4C, 0x02, 0xB1, 0x78,
+	0xEC, 0x1A, 0x59, 0x0B, 0xAD, 0xD4, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x72, 0x00, 0x30, 0x84, 0x00, 0x12, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+
+	/* page 4  probe_resp */
+	0x50, 0x00, 0x00, 0x00, 0x00, 0x40, 0x10, 0x10,
+	0x00, 0x03, 0x00, 0xE0, 0x4C, 0x76, 0x00, 0x42,
+	0x00, 0x40, 0x10, 0x10, 0x00, 0x03, 0x00, 0x00,
+	0x9E, 0x46, 0x15, 0x32, 0x27, 0xF2, 0x2D, 0x00,
+	0x64, 0x00, 0x00, 0x04, 0x00, 0x0C, 0x6C, 0x69,
+	0x6E, 0x6B, 0x73, 0x79, 0x73, 0x5F, 0x77, 0x6C,
+	0x61, 0x6E, 0x01, 0x04, 0x82, 0x84, 0x8B, 0x96,
+	0x03, 0x01, 0x01, 0x06, 0x02, 0x00, 0x00, 0x2A,
+	0x01, 0x00, 0x32, 0x08, 0x24, 0x30, 0x48, 0x6C,
+	0x0C, 0x12, 0x18, 0x60, 0x2D, 0x1A, 0x6C, 0x18,
+	0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x3D, 0x00, 0xDD, 0x06, 0x00, 0xE0, 0x4C, 0x02,
+	0x01, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+
+	/* page 5  probe_resp */
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x1A, 0x00, 0x30, 0x84, 0x00, 0x12, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+
+	/* page 6 qos null data */
+	0xC8, 0x01, 0x00, 0x00, 0x84, 0xC9, 0xB2, 0xA7,
+	0xB3, 0x6E, 0x00, 0xE0, 0x4C, 0x02, 0x51, 0x02,
+	0x84, 0xC9, 0xB2, 0xA7, 0xB3, 0x6E, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x1A, 0x00, 0x30, 0x84, 0x00, 0x12, 0x00, 0x00,
+	0x00, 0x00, 0x80, 0x00, 0x00, 0x01, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+
+	/* page 7 BT-qos null data */
+	0xC8, 0x01, 0x00, 0x00, 0x84, 0xC9, 0xB2, 0xA7,
+	0xB3, 0x6E, 0x00, 0xE0, 0x4C, 0x02, 0x51, 0x02,
+	0x84, 0xC9, 0xB2, 0xA7, 0xB3, 0x6E, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+};
+
+void rtl8822be_set_fw_rsvdpagepkt(struct ieee80211_hw *hw, bool b_dl_finished)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct sk_buff *skb = NULL;
+
+	u32 totalpacketlen;
+	bool rtstatus;
+	u8 u1_rsvd_page_loc[7] = {0};
+	bool b_dlok = false;
+
+	u8 *beacon;
+	u8 *p_pspoll;
+	u8 *nullfunc;
+	u8 *p_probersp;
+	u8 *qosnull;
+	u8 *btqosnull;
+
+	memset(u1_rsvd_page_loc, 0, sizeof(u1_rsvd_page_loc));
+
+	/*---------------------------------------------------------
+	 *			(1) beacon
+	 *---------------------------------------------------------
+	 */
+	beacon = &reserved_page_packet[BEACON_PG * 128];
+	SET_80211_HDR_ADDRESS2(beacon, mac->mac_addr);
+	SET_80211_HDR_ADDRESS3(beacon, mac->bssid);
+
+	/*-------------------------------------------------------
+	 *			(2) ps-poll
+	 *--------------------------------------------------------
+	 */
+	p_pspoll = &reserved_page_packet[PSPOLL_PG * 128];
+	SET_80211_PS_POLL_AID(p_pspoll, (mac->assoc_id | 0xc000));
+	SET_80211_PS_POLL_BSSID(p_pspoll, mac->bssid);
+	SET_80211_PS_POLL_TA(p_pspoll, mac->mac_addr);
+
+	SET_H2CCMD_RSVDPAGE_LOC_PSPOLL(u1_rsvd_page_loc, PSPOLL_PG);
+
+	/*--------------------------------------------------------
+	 *			(3) null data
+	 *---------------------------------------------------------
+	 */
+	nullfunc = &reserved_page_packet[NULL_PG * 128];
+	SET_80211_HDR_ADDRESS1(nullfunc, mac->bssid);
+	SET_80211_HDR_ADDRESS2(nullfunc, mac->mac_addr);
+	SET_80211_HDR_ADDRESS3(nullfunc, mac->bssid);
+
+	SET_H2CCMD_RSVDPAGE_LOC_NULL_DATA(u1_rsvd_page_loc, NULL_PG);
+
+	/*---------------------------------------------------------
+	 *			(4) probe response
+	 *----------------------------------------------------------
+	 */
+	p_probersp = &reserved_page_packet[PROBERSP_PG * 128];
+	SET_80211_HDR_ADDRESS1(p_probersp, mac->bssid);
+	SET_80211_HDR_ADDRESS2(p_probersp, mac->mac_addr);
+	SET_80211_HDR_ADDRESS3(p_probersp, mac->bssid);
+
+	SET_H2CCMD_RSVDPAGE_LOC_PROBE_RSP(u1_rsvd_page_loc, PROBERSP_PG);
+
+	/*---------------------------------------------------------
+	 *			(5) QoS null data
+	 *----------------------------------------------------------
+	 */
+	qosnull = &reserved_page_packet[QOS_NULL_PG * 128];
+	SET_80211_HDR_ADDRESS1(qosnull, mac->bssid);
+	SET_80211_HDR_ADDRESS2(qosnull, mac->mac_addr);
+	SET_80211_HDR_ADDRESS3(qosnull, mac->bssid);
+
+	SET_H2CCMD_RSVDPAGE_LOC_QOS_NULL_DATA(u1_rsvd_page_loc, QOS_NULL_PG);
+
+	/*---------------------------------------------------------
+	 *			(6) BT QoS null data
+	 *----------------------------------------------------------
+	 */
+	btqosnull = &reserved_page_packet[BT_QOS_NULL_PG * 128];
+	SET_80211_HDR_ADDRESS1(btqosnull, mac->bssid);
+	SET_80211_HDR_ADDRESS2(btqosnull, mac->mac_addr);
+	SET_80211_HDR_ADDRESS3(btqosnull, mac->bssid);
+
+	SET_H2CCMD_RSVDPAGE_LOC_BT_QOS_NULL_DATA(u1_rsvd_page_loc,
+						 BT_QOS_NULL_PG);
+
+	totalpacketlen = TOTAL_RESERVED_PKT_LEN;
+
+	RT_PRINT_DATA(rtlpriv, COMP_CMD, DBG_LOUD,
+		      "rtl8822be_set_fw_rsvdpagepkt(): HW_VAR_SET_TX_CMD: ALL\n",
+		      &reserved_page_packet[0], totalpacketlen);
+	RT_PRINT_DATA(rtlpriv, COMP_CMD, DBG_LOUD,
+		      "rtl8822be_set_fw_rsvdpagepkt(): HW_VAR_SET_TX_CMD: ALL\n",
+		      u1_rsvd_page_loc, 3);
+
+	skb = dev_alloc_skb(totalpacketlen);
+	memcpy((u8 *)skb_put(skb, totalpacketlen), &reserved_page_packet,
+	       totalpacketlen);
+
+	rtstatus = _rtl8822be_send_bcn_or_cmd_packet(hw, skb, BEACON_QUEUE);
+
+	if (rtstatus)
+		b_dlok = true;
+
+	if (b_dlok) {
+		RT_TRACE(rtlpriv, COMP_POWER, DBG_LOUD,
+			 "Set RSVD page location to Fw.\n");
+		RT_PRINT_DATA(rtlpriv, COMP_CMD, DBG_LOUD, "H2C_RSVDPAGE:\n",
+			      u1_rsvd_page_loc, 3);
+		rtl8822be_fill_h2c_cmd(hw, H2C_8822B_RSVDPAGE,
+				       sizeof(u1_rsvd_page_loc),
+				       u1_rsvd_page_loc);
+	} else
+		RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING,
+			 "Set RSVD page location to Fw FAIL!!!!!!.\n");
+}
+
+/* Should check FW support p2p or not. */
+static void rtl8822be_set_p2p_ctw_period_cmd(struct ieee80211_hw *hw,
+					     u8 ctwindow)
+{
+	u8 u1_ctwindow_period[1] = {ctwindow};
+
+	rtl8822be_fill_h2c_cmd(hw, H2C_8822B_P2P_PS_CTW_CMD, 1,
+			       u1_ctwindow_period);
+}
+
+void rtl8822be_set_p2p_ps_offload_cmd(struct ieee80211_hw *hw, u8 p2p_ps_state)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_ps_ctl *rtlps = rtl_psc(rtl_priv(hw));
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	struct rtl_p2p_ps_info *p2pinfo = &rtlps->p2p_ps_info;
+	struct p2p_ps_offload_t *p2p_ps_offload = &rtlhal->p2p_ps_offload;
+	u8 i;
+	u16 ctwindow;
+	u32 start_time, tsf_low;
+
+	switch (p2p_ps_state) {
+	case P2P_PS_DISABLE:
+		RT_TRACE(rtlpriv, COMP_FW, DBG_LOUD, "P2P_PS_DISABLE\n");
+		memset(p2p_ps_offload, 0, sizeof(*p2p_ps_offload));
+		break;
+	case P2P_PS_ENABLE:
+		RT_TRACE(rtlpriv, COMP_FW, DBG_LOUD, "P2P_PS_ENABLE\n");
+		/* update CTWindow value. */
+		if (p2pinfo->ctwindow > 0) {
+			p2p_ps_offload->ctwindow_en = 1;
+			ctwindow = p2pinfo->ctwindow;
+			rtl8822be_set_p2p_ctw_period_cmd(hw, ctwindow);
+		}
+		/* hw only support 2 set of NoA */
+		for (i = 0; i < p2pinfo->noa_num; i++) {
+			/* To control the register setting for which NOA*/
+			rtl_write_byte(rtlpriv, 0x5cf, (i << 4));
+			if (i == 0)
+				p2p_ps_offload->noa0_en = 1;
+			else
+				p2p_ps_offload->noa1_en = 1;
+			/* config P2P NoA Descriptor Register */
+			rtl_write_dword(rtlpriv, 0x5E0,
+					p2pinfo->noa_duration[i]);
+			rtl_write_dword(rtlpriv, 0x5E4,
+					p2pinfo->noa_interval[i]);
+
+			/*Get Current TSF value */
+			tsf_low = rtl_read_dword(rtlpriv, REG_TSFTR_8822B);
+
+			start_time = p2pinfo->noa_start_time[i];
+			if (p2pinfo->noa_count_type[i] != 1) {
+				while (start_time <= (tsf_low + (50 * 1024))) {
+					start_time += p2pinfo->noa_interval[i];
+					if (p2pinfo->noa_count_type[i] != 255)
+						p2pinfo->noa_count_type[i]--;
+				}
+			}
+			rtl_write_dword(rtlpriv, 0x5E8, start_time);
+			rtl_write_dword(rtlpriv, 0x5EC,
+					p2pinfo->noa_count_type[i]);
+		}
+		if ((p2pinfo->opp_ps == 1) || (p2pinfo->noa_num > 0)) {
+			/* rst p2p circuit */
+			rtl_write_byte(rtlpriv, REG_DUAL_TSF_RST_8822B, BIT(4));
+			p2p_ps_offload->offload_en = 1;
+
+			if (rtlpriv->mac80211.p2p == P2P_ROLE_GO) {
+				p2p_ps_offload->role = 1;
+				p2p_ps_offload->allstasleep = 0;
+			} else {
+				p2p_ps_offload->role = 0;
+			}
+			p2p_ps_offload->discovery = 0;
+		}
+		break;
+	case P2P_PS_SCAN:
+		RT_TRACE(rtlpriv, COMP_FW, DBG_LOUD, "P2P_PS_SCAN\n");
+		p2p_ps_offload->discovery = 1;
+		break;
+	case P2P_PS_SCAN_DONE:
+		RT_TRACE(rtlpriv, COMP_FW, DBG_LOUD, "P2P_PS_SCAN_DONE\n");
+		p2p_ps_offload->discovery = 0;
+		p2pinfo->p2p_ps_state = P2P_PS_ENABLE;
+		break;
+	default:
+		break;
+	}
+
+	rtl8822be_fill_h2c_cmd(hw, H2C_8822B_P2P_PS_OFFLOAD, 1,
+			       (u8 *)p2p_ps_offload);
+}
+
+static
+void rtl8822be_c2h_content_parsing_ext(struct ieee80211_hw *hw,
+				       u8 c2h_sub_cmd_id,
+				       u8 c2h_cmd_len,
+				       u8 *c2h_content_buf)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_halmac_ops *halmac_ops;
+
+	switch (c2h_sub_cmd_id) {
+	case 0x0F:
+		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
+			 "[C2H], C2H_8822BE_TX_REPORT!\n");
+		rtl_tx_report_handler(hw, c2h_content_buf, c2h_cmd_len);
+		break;
+	default:
+		/* indicate c2h pkt + rx desc to halmac */
+		halmac_ops = rtlpriv->halmac.ops;
+		halmac_ops->halmac_c2h_handle(rtlpriv,
+					      c2h_content_buf - 24 - 2 - 2,
+					      c2h_cmd_len + 24 + 2 + 2);
+		break;
+	}
+}
+
+void rtl8822be_c2h_content_parsing(struct ieee80211_hw *hw, u8 c2h_cmd_id,
+				   u8 c2h_cmd_len, u8 *tmp_buf)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_btc_ops *btc_ops = rtlpriv->btcoexist.btc_ops;
+
+	if (c2h_cmd_id == 0xFF) {
+		rtl8822be_c2h_content_parsing_ext(hw, tmp_buf[0],
+						  c2h_cmd_len - 2,
+						  tmp_buf + 2);
+		return;
+	}
+
+	switch (c2h_cmd_id) {
+	case C2H_8822B_DBG:
+		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
+			 "[C2H], C2H_8822BE_DBG!!\n");
+		break;
+	case C2H_8822B_TXBF:
+		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
+			 "[C2H], C2H_8822B_TXBF!!\n");
+		break;
+	case C2H_8822B_BT_INFO:
+		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
+			 "[C2H], C2H_8822BE_BT_INFO!!\n");
+		if (rtlpriv->cfg->ops->get_btc_status())
+			btc_ops->btc_btinfo_notify(rtlpriv, tmp_buf,
+						   c2h_cmd_len);
+		break;
+	case C2H_8822B_BT_MP:
+		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
+			 "[C2H], C2H_8822BE_BT_MP!!\n");
+		if (rtlpriv->cfg->ops->get_btc_status())
+			btc_ops->btc_btmpinfo_notify(rtlpriv, tmp_buf,
+						     c2h_cmd_len);
+		break;
+	default:
+		if (!rtlpriv->phydm.ops->phydm_c2h_content_parsing(
+			    rtlpriv, c2h_cmd_id, c2h_cmd_len, tmp_buf))
+			break;
+
+		RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
+			 "[C2H], Unknown packet!! CmdId(%#X)!\n", c2h_cmd_id);
+		break;
+	}
+}
+
+void rtl8822be_c2h_packet_handler(struct ieee80211_hw *hw, u8 *buffer, u8 len)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 c2h_cmd_id = 0, c2h_cmd_seq = 0, c2h_cmd_len = 0;
+	u8 *tmp_buf = NULL;
+
+	c2h_cmd_id = buffer[0];
+	c2h_cmd_seq = buffer[1];
+	c2h_cmd_len = len - 2;
+	tmp_buf = buffer + 2;
+
+	RT_TRACE(rtlpriv, COMP_FW, DBG_TRACE,
+		 "[C2H packet], c2hCmdId=0x%x, c2hCmdSeq=0x%x, c2hCmdLen=%d\n",
+		 c2h_cmd_id, c2h_cmd_seq, c2h_cmd_len);
+
+	RT_PRINT_DATA(rtlpriv, COMP_FW, DBG_TRACE,
+		      "[C2H packet], Content Hex:\n", tmp_buf, c2h_cmd_len);
+
+	switch (c2h_cmd_id) {
+	case C2H_8822B_BT_INFO:
+	case C2H_8822B_BT_MP:
+		rtl_c2hcmd_enqueue(hw, c2h_cmd_id, c2h_cmd_len, tmp_buf);
+		break;
+	default:
+		rtl8822be_c2h_content_parsing(hw, c2h_cmd_id, c2h_cmd_len,
+					      tmp_buf);
+		break;
+	}
+}
diff --git a/drivers/staging/rtlwifi/rtl8822be/fw.h b/drivers/staging/rtlwifi/rtl8822be/fw.h
new file mode 100644
index 000000000000..3ad7a66e80a3
--- /dev/null
+++ b/drivers/staging/rtlwifi/rtl8822be/fw.h
@@ -0,0 +1,198 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __RTL8822B__FW__H__
+#define __RTL8822B__FW__H__
+
+#define USE_OLD_WOWLAN_DEBUG_FW	0
+
+#define H2C_8822B_RSVDPAGE_LOC_LEN	5
+#define H2C_8822B_PWEMODE_LENGTH	7
+#define H2C_8822B_JOINBSSRPT_LENGTH	1
+#define H2C_8822B_AP_OFFLOAD_LENGTH	3
+#define H2C_8822B_WOWLAN_LENGTH	3
+#define H2C_8822B_KEEP_ALIVE_CTRL_LENGTH	3
+#if (USE_OLD_WOWLAN_DEBUG_FW == 0)
+#define H2C_8822B_REMOTE_WAKE_CTRL_LEN	1
+#else
+#define H2C_8822B_REMOTE_WAKE_CTRL_LEN	3
+#endif
+#define H2C_8822B_AOAC_GLOBAL_INFO_LEN	2
+#define H2C_8822B_AOAC_RSVDPAGE_LOC_LEN	7
+#define H2C_DEFAULT_PORT_ID_LEN	2
+
+/* Fw PS state for RPWM.
+ *BIT[2:0] = HW state
+ *BIT[3] = Protocol PS state,  1: register active state, 0: register sleep state
+ *BIT[4] = sub-state
+ */
+#define FW_PS_RF_ON	BIT(2)
+#define FW_PS_REGISTER_ACTIVE	BIT(3)
+
+#define FW_PS_ACK	BIT(6)
+#define FW_PS_TOGGLE	BIT(7)
+
+/* 8822B RPWM value*/
+/* BIT[0] = 1: 32k, 0: 40M*/
+#define FW_PS_CLOCK_OFF	BIT(0) /* 32k */
+#define FW_PS_CLOCK_ON	0 /* 40M */
+
+#define FW_PS_STATE_MASK	(0x0F)
+#define FW_PS_STATE_HW_MASK	(0x07)
+#define FW_PS_STATE_INT_MASK	(0x3F)
+
+#define FW_PS_STATE(x) (FW_PS_STATE_MASK & (x))
+
+#define FW_PS_STATE_ALL_ON_8822B	(FW_PS_CLOCK_ON)
+#define FW_PS_STATE_RF_ON_8822B	(FW_PS_CLOCK_ON)
+#define FW_PS_STATE_RF_OFF_8822B	(FW_PS_CLOCK_ON)
+#define FW_PS_STATE_RF_OFF_LOW_PWR	(FW_PS_CLOCK_OFF)
+
+/* For 8822B H2C PwrMode Cmd ID 5.*/
+#define FW_PWR_STATE_ACTIVE ((FW_PS_RF_ON) | (FW_PS_REGISTER_ACTIVE))
+#define FW_PWR_STATE_RF_OFF	0
+
+#define FW_PS_IS_ACK(x) ((x) & FW_PS_ACK)
+
+#define IS_IN_LOW_POWER_STATE_8822B(fw_ps_state)                               \
+	(FW_PS_STATE(fw_ps_state) == FW_PS_CLOCK_OFF)
+
+#define FW_PWR_STATE_ACTIVE ((FW_PS_RF_ON) | (FW_PS_REGISTER_ACTIVE))
+#define FW_PWR_STATE_RF_OFF	0
+
+enum rtl8822b_h2c_cmd {
+	H2C_8822B_RSVDPAGE	= 0,
+	H2C_8822B_MSRRPT	= 1,
+	H2C_8822B_SCAN	= 2,
+	H2C_8822B_KEEP_ALIVE_CTRL	= 3,
+	H2C_8822B_DISCONNECT_DECISION	= 4,
+#if (USE_OLD_WOWLAN_DEBUG_FW == 1)
+	H2C_8822B_WO_WLAN	= 5,
+#endif
+	H2C_8822B_INIT_OFFLOAD	= 6,
+#if (USE_OLD_WOWLAN_DEBUG_FW == 1)
+	H2C_8822B_REMOTE_WAKE_CTRL	= 7,
+#endif
+	H2C_8822B_AP_OFFLOAD	= 8,
+	H2C_8822B_BCN_RSVDPAGE	= 9,
+	H2C_8822B_PROBERSP_RSVDPAGE	= 10,
+
+	H2C_8822B_SETPWRMODE	= 0x20,
+	H2C_8822B_PS_TUNING_PARA	= 0x21,
+	H2C_8822B_PS_TUNING_PARA2	= 0x22,
+	H2C_8822B_PS_LPS_PARA	= 0x23,
+	H2C_8822B_P2P_PS_OFFLOAD	= 024,
+	H2C_8822B_DEFAULT_PORT_ID	= 0x2C,
+
+#if (USE_OLD_WOWLAN_DEBUG_FW == 0)
+	H2C_8822B_WO_WLAN	= 0x80,
+	H2C_8822B_REMOTE_WAKE_CTRL	= 0x81,
+	H2C_8822B_AOAC_GLOBAL_INFO	= 0x82,
+	H2C_8822B_AOAC_RSVDPAGE	= 0x83,
+#endif
+	H2C_8822B_MACID_CFG	= 0x40,
+	H2C_8822B_RSSI_REPORT	= 0x42,
+	H2C_8822B_MACID_CFG_3SS	= 0x46,
+	/*Not defined CTW CMD for P2P yet*/
+	H2C_8822B_P2P_PS_CTW_CMD	= 0x99,
+	MAX_8822B_H2CCMD
+};
+
+enum rtl8822b_c2h_evt {
+	C2H_8822B_DBG	= 0x00,
+	C2H_8822B_LB	= 0x01,
+	C2H_8822B_TXBF	= 0x02,
+	C2H_8822B_TX_REPORT	= 0x03,
+	C2H_8822B_BT_INFO	= 0x09,
+	C2H_8822B_BT_MP	= 0x0B,
+	C2H_8822B_RA_RPT	= 0x0C,
+	MAX_8822B_C2HEVENT
+};
+
+/* H2C: 0x20 */
+#define SET_H2CCMD_PWRMODE_PARM_MODE(__ph2ccmd, __val)                         \
+	SET_BITS_TO_LE_1BYTE(__ph2ccmd, 0, 7, __val)
+#define SET_H2CCMD_PWRMODE_PARM_CLK_REQ(__ph2ccmd, __val)                      \
+	SET_BITS_TO_LE_1BYTE(__ph2ccmd, 7, 1, __val)
+#define SET_H2CCMD_PWRMODE_PARM_RLBM(__ph2ccmd, __val)                         \
+	SET_BITS_TO_LE_1BYTE((__ph2ccmd) + 1, 0, 4, __val)
+#define SET_H2CCMD_PWRMODE_PARM_SMART_PS(__ph2ccmd, __val)                     \
+	SET_BITS_TO_LE_1BYTE((__ph2ccmd) + 1, 4, 4, __val)
+#define SET_H2CCMD_PWRMODE_PARM_AWAKE_INTERVAL(__ph2ccmd, __val)               \
+	SET_BITS_TO_LE_1BYTE((__ph2ccmd) + 2, 0, 8, __val)
+#define SET_H2CCMD_PWRMODE_PARM_ALL_QUEUE_UAPSD(__ph2ccmd, __val)              \
+	SET_BITS_TO_LE_1BYTE((__ph2ccmd) + 3, 0, 1, __val)
+#define SET_H2CCMD_PWRMODE_PARM_BCN_EARLY_RPT(__ph2ccmd, __val)                \
+	SET_BITS_TO_LE_1BYTE((__ph2ccmd) + 3, 2, 1, __val)
+#define SET_H2CCMD_PWRMODE_PARM_PORT_ID(__ph2ccmd, __val)                      \
+	SET_BITS_TO_LE_1BYTE((__ph2ccmd) + 3, 5, 3, __val)
+#define SET_H2CCMD_PWRMODE_PARM_PWR_STATE(__ph2ccmd, __val)                    \
+	SET_BITS_TO_LE_1BYTE((__ph2ccmd) + 4, 0, 8, __val)
+#define SET_H2CCMD_PWRMODE_PARM_BYTE5(__ph2ccmd, __val)                        \
+	SET_BITS_TO_LE_1BYTE((__ph2ccmd) + 5, 0, 8, __val)
+
+/* H2C: 0x00 */
+#define SET_H2CCMD_RSVDPAGE_LOC_PROBE_RSP(__ph2ccmd, __val)                    \
+	SET_BITS_TO_LE_1BYTE(__ph2ccmd, 0, 8, __val)
+#define SET_H2CCMD_RSVDPAGE_LOC_PSPOLL(__ph2ccmd, __val)                       \
+	SET_BITS_TO_LE_1BYTE((__ph2ccmd) + 1, 0, 8, __val)
+#define SET_H2CCMD_RSVDPAGE_LOC_NULL_DATA(__ph2ccmd, __val)                    \
+	SET_BITS_TO_LE_1BYTE((__ph2ccmd) + 2, 0, 8, __val)
+#define SET_H2CCMD_RSVDPAGE_LOC_QOS_NULL_DATA(__ph2ccmd, __val)                \
+	SET_BITS_TO_LE_1BYTE((__ph2ccmd) + 3, 0, 8, __val)
+#define SET_H2CCMD_RSVDPAGE_LOC_BT_QOS_NULL_DATA(__ph2ccmd, __val)             \
+	SET_BITS_TO_LE_1BYTE((__ph2ccmd) + 4, 0, 8, __val)
+
+/* H2C: 0x01 */
+#define SET_H2CCMD_MSRRPT_PARM_OPMODE(__ph2ccmd, __val)                        \
+	SET_BITS_TO_LE_1BYTE(__ph2ccmd, 0, 1, __val)
+#define SET_H2CCMD_MSRRPT_PARM_MACID_IND(__ph2ccmd, __val)                     \
+	SET_BITS_TO_LE_1BYTE(__ph2ccmd, 1, 1, __val)
+#define SET_H2CCMD_MSRRPT_PARM_MACID(__ph2ccmd, __val)                         \
+	SET_BITS_TO_LE_1BYTE(__ph2ccmd + 1, 0, 8, __val)
+#define SET_H2CCMD_MSRRPT_PARM_MACID_END(__ph2ccmd, __val)                     \
+	SET_BITS_TO_LE_1BYTE(__ph2ccmd + 2, 0, 8, __val)
+
+/* H2C: 0x2C */
+#define SET_H2CCMD_DFTPID_PORT_ID(__ph2ccmd, __val)                            \
+	SET_BITS_TO_LE_1BYTE(((u8 *)(__ph2ccmd)), 0, 8, (__val))
+#define SET_H2CCMD_DFTPID_MAC_ID(__ph2ccmd, __val)                             \
+	SET_BITS_TO_LE_1BYTE(((u8 *)(__ph2ccmd)) + 1, 0, 8, (__val))
+
+void rtl8822be_fill_h2c_cmd(struct ieee80211_hw *hw, u8 element_id, u32 cmd_len,
+			    u8 *cmdbuffer);
+void rtl8822be_set_default_port_id_cmd(struct ieee80211_hw *hw);
+void rtl8822be_set_fw_pwrmode_cmd(struct ieee80211_hw *hw, u8 mode);
+void rtl8822be_set_fw_media_status_rpt_cmd(struct ieee80211_hw *hw, u8 mstatus);
+void rtl8822be_set_fw_rsvdpagepkt(struct ieee80211_hw *hw, bool b_dl_finished);
+void rtl8822be_set_p2p_ps_offload_cmd(struct ieee80211_hw *hw, u8 p2p_ps_state);
+void rtl8822be_c2h_packet_handler(struct ieee80211_hw *hw, u8 *buffer, u8 len);
+void rtl8822be_c2h_content_parsing(struct ieee80211_hw *hw, u8 c2h_cmd_id,
+				   u8 c2h_cmd_len, u8 *tmp_buf);
+bool rtl8822b_halmac_cb_write_data_rsvd_page(struct rtl_priv *rtlpriv, u8 *buf,
+					     u32 size);
+bool rtl8822b_halmac_cb_write_data_h2c(struct rtl_priv *rtlpriv, u8 *buf,
+				       u32 size);
+#endif
diff --git a/drivers/staging/rtlwifi/rtl8822be/hw.c b/drivers/staging/rtlwifi/rtl8822be/hw.c
new file mode 100644
index 000000000000..74386003044f
--- /dev/null
+++ b/drivers/staging/rtlwifi/rtl8822be/hw.c
@@ -0,0 +1,2441 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#include "../wifi.h"
+#include "../efuse.h"
+#include "../base.h"
+#include "../regd.h"
+#include "../cam.h"
+#include "../ps.h"
+#include "../pci.h"
+#include "reg.h"
+#include "def.h"
+#include "phy.h"
+#include "fw.h"
+#include "led.h"
+#include "hw.h"
+
+#define LLT_CONFIG	5
+
+u8 rtl_channel5g[CHANNEL_MAX_NUMBER_5G] = {
+	36,  38,  40,  42,  44,  46,  48, /* Band 1 */
+	52,  54,  56,  58,  60,  62,  64, /* Band 2 */
+	100, 102, 104, 106, 108, 110, 112, /* Band 3 */
+	116, 118, 120, 122, 124, 126, 128, /* Band 3 */
+	132, 134, 136, 138, 140, 142, 144, /* Band 3 */
+	149, 151, 153, 155, 157, 159, 161, /* Band 4 */
+	165, 167, 169, 171, 173, 175, 177}; /* Band 4 */
+u8 rtl_channel5g_80m[CHANNEL_MAX_NUMBER_5G_80M] = {42,  58,  106, 122,
+						   138, 155, 171};
+
+static void _rtl8822be_set_bcn_ctrl_reg(struct ieee80211_hw *hw, u8 set_bits,
+					u8 clear_bits)
+{
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	rtlpci->reg_bcn_ctrl_val |= set_bits;
+	rtlpci->reg_bcn_ctrl_val &= ~clear_bits;
+
+	rtl_write_byte(rtlpriv, REG_BCN_CTRL_8822B,
+		       (u8)rtlpci->reg_bcn_ctrl_val);
+}
+
+static void _rtl8822be_stop_tx_beacon(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 tmp;
+
+	tmp = rtl_read_byte(rtlpriv, REG_FWHW_TXQ_CTRL_8822B + 2);
+	rtl_write_byte(rtlpriv, REG_FWHW_TXQ_CTRL_8822B + 2, tmp & (~BIT(6)));
+	rtl_write_byte(rtlpriv, REG_TBTT_PROHIBIT_8822B + 1, 0x64);
+	tmp = rtl_read_byte(rtlpriv, REG_TBTT_PROHIBIT_8822B + 2);
+	tmp &= ~(BIT(0));
+	rtl_write_byte(rtlpriv, REG_TBTT_PROHIBIT_8822B + 2, tmp);
+}
+
+static void _rtl8822be_resume_tx_beacon(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 tmp;
+
+	tmp = rtl_read_byte(rtlpriv, REG_FWHW_TXQ_CTRL_8822B + 2);
+	rtl_write_byte(rtlpriv, REG_FWHW_TXQ_CTRL_8822B + 2, tmp | BIT(6));
+	rtl_write_byte(rtlpriv, REG_TBTT_PROHIBIT_8822B + 1, 0xff);
+	tmp = rtl_read_byte(rtlpriv, REG_TBTT_PROHIBIT_8822B + 2);
+	tmp |= BIT(0);
+	rtl_write_byte(rtlpriv, REG_TBTT_PROHIBIT_8822B + 2, tmp);
+}
+
+static void _rtl8822be_enable_bcn_sub_func(struct ieee80211_hw *hw)
+{
+	_rtl8822be_set_bcn_ctrl_reg(hw, 0, BIT(1));
+}
+
+static void _rtl8822be_disable_bcn_sub_func(struct ieee80211_hw *hw)
+{
+	_rtl8822be_set_bcn_ctrl_reg(hw, BIT(1), 0);
+}
+
+static void _rtl8822be_set_fw_clock_on(struct ieee80211_hw *hw, u8 rpwm_val,
+				       bool b_need_turn_off_ckk)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	u32 count = 0, isr_regaddr, content;
+	bool b_schedule_timer = b_need_turn_off_ckk;
+
+	if (!rtlhal->fw_ready)
+		return;
+	if (!rtlpriv->psc.fw_current_inpsmode)
+		return;
+
+	while (1) {
+		spin_lock_bh(&rtlpriv->locks.fw_ps_lock);
+		if (rtlhal->fw_clk_change_in_progress) {
+			while (rtlhal->fw_clk_change_in_progress) {
+				spin_unlock_bh(&rtlpriv->locks.fw_ps_lock);
+				count++;
+				udelay(100);
+				if (count > 1000)
+					return;
+				spin_lock_bh(&rtlpriv->locks.fw_ps_lock);
+			}
+			spin_unlock_bh(&rtlpriv->locks.fw_ps_lock);
+		} else {
+			rtlhal->fw_clk_change_in_progress = false;
+			spin_unlock_bh(&rtlpriv->locks.fw_ps_lock);
+			break;
+		}
+	}
+
+	if (IS_IN_LOW_POWER_STATE_8822B(rtlhal->fw_ps_state)) {
+		rtlpriv->cfg->ops->get_hw_reg(hw, HW_VAR_SET_RPWM,
+					      (u8 *)(&rpwm_val));
+		if (FW_PS_IS_ACK(rpwm_val)) {
+			isr_regaddr = REG_HISR0_8822B;
+			content = rtl_read_dword(rtlpriv, isr_regaddr);
+			while (!(content & IMR_CPWM) && (count < 500)) {
+				udelay(50);
+				count++;
+				content = rtl_read_dword(rtlpriv, isr_regaddr);
+			}
+
+			if (content & IMR_CPWM) {
+				rtl_write_word(rtlpriv, isr_regaddr, 0x0100);
+				rtlhal->fw_ps_state = FW_PS_STATE_RF_ON_8822B;
+				RT_TRACE(rtlpriv, COMP_POWER, DBG_LOUD,
+					 "Receive CPWM INT!!! PSState = %X\n",
+					 rtlhal->fw_ps_state);
+			}
+		}
+
+		spin_lock_bh(&rtlpriv->locks.fw_ps_lock);
+		rtlhal->fw_clk_change_in_progress = false;
+		spin_unlock_bh(&rtlpriv->locks.fw_ps_lock);
+		if (b_schedule_timer) {
+			mod_timer(&rtlpriv->works.fw_clockoff_timer,
+				  jiffies + MSECS(10));
+		}
+
+	} else {
+		spin_lock_bh(&rtlpriv->locks.fw_ps_lock);
+		rtlhal->fw_clk_change_in_progress = false;
+		spin_unlock_bh(&rtlpriv->locks.fw_ps_lock);
+	}
+}
+
+static void _rtl8822be_set_fw_clock_off(struct ieee80211_hw *hw, u8 rpwm_val)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	struct rtl8192_tx_ring *ring;
+	enum rf_pwrstate rtstate;
+	bool b_schedule_timer = false;
+	u8 queue;
+
+	if (!rtlhal->fw_ready)
+		return;
+	if (!rtlpriv->psc.fw_current_inpsmode)
+		return;
+	if (!rtlhal->allow_sw_to_change_hwclc)
+		return;
+
+	rtlpriv->cfg->ops->get_hw_reg(hw, HW_VAR_RF_STATE, (u8 *)(&rtstate));
+	if (rtstate == ERFOFF || rtlpriv->psc.inactive_pwrstate == ERFOFF)
+		return;
+
+	for (queue = 0; queue < RTL_PCI_MAX_TX_QUEUE_COUNT; queue++) {
+		ring = &rtlpci->tx_ring[queue];
+		if (skb_queue_len(&ring->queue)) {
+			b_schedule_timer = true;
+			break;
+		}
+	}
+
+	if (b_schedule_timer) {
+		mod_timer(&rtlpriv->works.fw_clockoff_timer,
+			  jiffies + MSECS(10));
+		return;
+	}
+
+	if (FW_PS_STATE(rtlhal->fw_ps_state) != FW_PS_STATE_RF_OFF_LOW_PWR) {
+		spin_lock_bh(&rtlpriv->locks.fw_ps_lock);
+		if (!rtlhal->fw_clk_change_in_progress) {
+			rtlhal->fw_clk_change_in_progress = true;
+			spin_unlock_bh(&rtlpriv->locks.fw_ps_lock);
+			rtlhal->fw_ps_state = FW_PS_STATE(rpwm_val);
+			rtl_write_word(rtlpriv, REG_HISR0_8822B, 0x0100);
+			rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_SET_RPWM,
+						      (u8 *)(&rpwm_val));
+			spin_lock_bh(&rtlpriv->locks.fw_ps_lock);
+			rtlhal->fw_clk_change_in_progress = false;
+			spin_unlock_bh(&rtlpriv->locks.fw_ps_lock);
+		} else {
+			spin_unlock_bh(&rtlpriv->locks.fw_ps_lock);
+			mod_timer(&rtlpriv->works.fw_clockoff_timer,
+				  jiffies + MSECS(10));
+		}
+	}
+}
+
+static void _rtl8822be_set_fw_ps_rf_on(struct ieee80211_hw *hw)
+{
+	u8 rpwm_val = 0;
+
+	rpwm_val |= (FW_PS_STATE_RF_OFF_8822B | FW_PS_ACK);
+	_rtl8822be_set_fw_clock_on(hw, rpwm_val, true);
+}
+
+static void _rtl8822be_set_fw_ps_rf_off_low_power(struct ieee80211_hw *hw)
+{
+	u8 rpwm_val = 0;
+
+	rpwm_val |= FW_PS_STATE_RF_OFF_LOW_PWR;
+	_rtl8822be_set_fw_clock_off(hw, rpwm_val);
+}
+
+void rtl8822be_fw_clk_off_timer_callback(unsigned long data)
+{
+	struct ieee80211_hw *hw = (struct ieee80211_hw *)data;
+
+	_rtl8822be_set_fw_ps_rf_off_low_power(hw);
+}
+
+static void _rtl8822be_fwlps_leave(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	bool fw_current_inps = false;
+	u8 rpwm_val = 0, fw_pwrmode = FW_PS_ACTIVE_MODE;
+
+	if (ppsc->low_power_enable) {
+		rpwm_val = (FW_PS_STATE_ALL_ON_8822B | FW_PS_ACK); /* RF on */
+		_rtl8822be_set_fw_clock_on(hw, rpwm_val, false);
+		rtlhal->allow_sw_to_change_hwclc = false;
+		rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_H2C_FW_PWRMODE,
+					      (u8 *)(&fw_pwrmode));
+		rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_FW_PSMODE_STATUS,
+					      (u8 *)(&fw_current_inps));
+	} else {
+		rpwm_val = FW_PS_STATE_ALL_ON_8822B; /* RF on */
+		rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_SET_RPWM,
+					      (u8 *)(&rpwm_val));
+		rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_H2C_FW_PWRMODE,
+					      (u8 *)(&fw_pwrmode));
+		rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_FW_PSMODE_STATUS,
+					      (u8 *)(&fw_current_inps));
+	}
+}
+
+static void _rtl8822be_fwlps_enter(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	bool fw_current_inps = true;
+	u8 rpwm_val;
+
+	if (ppsc->low_power_enable) {
+		rpwm_val = FW_PS_STATE_RF_OFF_LOW_PWR; /* RF off */
+		rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_FW_PSMODE_STATUS,
+					      (u8 *)(&fw_current_inps));
+		rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_H2C_FW_PWRMODE,
+					      (u8 *)(&ppsc->fwctrl_psmode));
+		rtlhal->allow_sw_to_change_hwclc = true;
+		_rtl8822be_set_fw_clock_off(hw, rpwm_val);
+	} else {
+		rpwm_val = FW_PS_STATE_RF_OFF_8822B; /* RF off */
+		rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_FW_PSMODE_STATUS,
+					      (u8 *)(&fw_current_inps));
+		rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_H2C_FW_PWRMODE,
+					      (u8 *)(&ppsc->fwctrl_psmode));
+		rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_SET_RPWM,
+					      (u8 *)(&rpwm_val));
+	}
+}
+
+void rtl8822be_get_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+
+	switch (variable) {
+	case HW_VAR_RCR:
+		*((u32 *)(val)) = rtlpci->receive_config;
+		break;
+	case HW_VAR_RF_STATE:
+		*((enum rf_pwrstate *)(val)) = ppsc->rfpwr_state;
+		break;
+	case HW_VAR_FWLPS_RF_ON: {
+		enum rf_pwrstate rf_state;
+		u32 val_rcr;
+
+		rtlpriv->cfg->ops->get_hw_reg(hw, HW_VAR_RF_STATE,
+					      (u8 *)(&rf_state));
+		if (rf_state == ERFOFF) {
+			*((bool *)(val)) = true;
+		} else {
+			val_rcr = rtl_read_dword(rtlpriv, REG_RCR_8822B);
+			val_rcr &= 0x00070000;
+			if (val_rcr)
+				*((bool *)(val)) = false;
+			else
+				*((bool *)(val)) = true;
+		}
+	} break;
+	case HW_VAR_FW_PSMODE_STATUS:
+		*((bool *)(val)) = ppsc->fw_current_inpsmode;
+		break;
+	case HW_VAR_CORRECT_TSF: {
+		u64 tsf;
+		u32 *ptsf_low = (u32 *)&tsf;
+		u32 *ptsf_high = ((u32 *)&tsf) + 1;
+
+		*ptsf_high = rtl_read_dword(rtlpriv, (REG_TSFTR_8822B + 4));
+		*ptsf_low = rtl_read_dword(rtlpriv, REG_TSFTR_8822B);
+
+		*((u64 *)(val)) = tsf;
+
+	} break;
+	default:
+		RT_TRACE(rtlpriv, COMP_ERR, DBG_DMESG,
+			 "switch case not process %x\n", variable);
+		break;
+	}
+}
+
+static void _rtl8822be_download_rsvd_page(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 tmp_regcr, tmp_reg422;
+	u8 bcnvalid_reg /*, txbc_reg*/;
+	u8 count = 0, dlbcn_count = 0;
+	bool b_recover = false;
+
+	/*Set REG_CR_8822B bit 8. DMA beacon by SW.*/
+	tmp_regcr = rtl_read_byte(rtlpriv, REG_CR_8822B + 1);
+	rtl_write_byte(rtlpriv, REG_CR_8822B + 1, tmp_regcr | BIT(0));
+
+	/* Disable Hw protection for a time which revserd for Hw sending beacon.
+	 * Fix download reserved page packet fail
+	 * that access collision with the protection time.
+	 * 2010.05.11. Added by tynli.
+	 */
+	_rtl8822be_set_bcn_ctrl_reg(hw, 0, BIT(3));
+	_rtl8822be_set_bcn_ctrl_reg(hw, BIT(4), 0);
+
+	/* Set FWHW_TXQ_CTRL 0x422[6]=0 to
+	 * tell Hw the packet is not a real beacon frame.
+	 */
+	tmp_reg422 = rtl_read_byte(rtlpriv, REG_FWHW_TXQ_CTRL_8822B + 2);
+	rtl_write_byte(rtlpriv, REG_FWHW_TXQ_CTRL_8822B + 2,
+		       tmp_reg422 & (~BIT(6)));
+
+	if (tmp_reg422 & BIT(6))
+		b_recover = true;
+
+	do {
+		/* Clear beacon valid check bit */
+		bcnvalid_reg =
+			rtl_read_byte(rtlpriv, REG_FIFOPAGE_CTRL_2_8822B + 1);
+		bcnvalid_reg = bcnvalid_reg | BIT(7);
+		rtl_write_byte(rtlpriv, REG_FIFOPAGE_CTRL_2_8822B + 1,
+			       bcnvalid_reg);
+
+		/* download rsvd page */
+		rtl8822be_set_fw_rsvdpagepkt(hw, false);
+
+		/* check rsvd page download OK. */
+		bcnvalid_reg =
+			rtl_read_byte(rtlpriv, REG_FIFOPAGE_CTRL_2_8822B + 1);
+
+		count = 0;
+		while (!(BIT(7) & bcnvalid_reg) && count < 20) {
+			count++;
+			udelay(50);
+			bcnvalid_reg = rtl_read_byte(
+				rtlpriv, REG_FIFOPAGE_CTRL_2_8822B + 1);
+		}
+
+		dlbcn_count++;
+	} while (!(BIT(7) & bcnvalid_reg) && dlbcn_count < 5);
+
+	if (!(BIT(7) & bcnvalid_reg))
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_WARNING,
+			 "Download RSVD page failed!\n");
+
+	/* Enable Bcn */
+	_rtl8822be_set_bcn_ctrl_reg(hw, BIT(3), 0);
+	_rtl8822be_set_bcn_ctrl_reg(hw, 0, BIT(4));
+
+	if (b_recover)
+		rtl_write_byte(rtlpriv, REG_FWHW_TXQ_CTRL_8822B + 2,
+			       tmp_reg422);
+}
+
+void rtl8822be_set_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct rtl_efuse *efuse = rtl_efuse(rtl_priv(hw));
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+
+	switch (variable) {
+	case HW_VAR_ETHER_ADDR:
+		rtlpriv->halmac.ops->halmac_set_mac_address(rtlpriv, 0, val);
+		break;
+	case HW_VAR_BASIC_RATE: {
+		u16 b_rate_cfg = ((u16 *)val)[0];
+
+		b_rate_cfg = b_rate_cfg & 0x15f;
+		b_rate_cfg |= 0x01;
+		b_rate_cfg = (b_rate_cfg | 0xd) & (~BIT(1));
+		rtl_write_byte(rtlpriv, REG_RRSR_8822B, b_rate_cfg & 0xff);
+		rtl_write_byte(rtlpriv, REG_RRSR_8822B + 1,
+			       (b_rate_cfg >> 8) & 0xff);
+	} break;
+	case HW_VAR_BSSID:
+		rtlpriv->halmac.ops->halmac_set_bssid(rtlpriv, 0, val);
+		break;
+	case HW_VAR_SIFS:
+		rtl_write_byte(rtlpriv, REG_SIFS_8822B + 1, val[0]);
+		rtl_write_byte(rtlpriv, REG_SIFS_TRX_8822B + 1, val[1]);
+
+		rtl_write_byte(rtlpriv, REG_SPEC_SIFS_8822B + 1, val[0]);
+		rtl_write_byte(rtlpriv, REG_MAC_SPEC_SIFS_8822B + 1, val[0]);
+
+		if (!mac->ht_enable)
+			rtl_write_word(rtlpriv, REG_RESP_SIFS_OFDM_8822B,
+				       0x0e0e);
+		else
+			rtl_write_word(rtlpriv, REG_RESP_SIFS_OFDM_8822B,
+				       *((u16 *)val));
+		break;
+	case HW_VAR_SLOT_TIME: {
+		u8 e_aci;
+
+		RT_TRACE(rtlpriv, COMP_MLME, DBG_TRACE, "HW_VAR_SLOT_TIME %x\n",
+			 val[0]);
+
+		rtl_write_byte(rtlpriv, REG_SLOT_8822B, val[0]);
+
+		for (e_aci = 0; e_aci < AC_MAX; e_aci++) {
+			rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_AC_PARAM,
+						      (u8 *)(&e_aci));
+		}
+	} break;
+	case HW_VAR_ACK_PREAMBLE: {
+		u8 reg_tmp;
+		u8 short_preamble = (bool)(*(u8 *)val);
+
+		reg_tmp = (rtlpriv->mac80211.cur_40_prime_sc) << 5;
+		if (short_preamble)
+			reg_tmp |= 0x80;
+		rtl_write_byte(rtlpriv, REG_RRSR_8822B + 2, reg_tmp);
+		rtlpriv->mac80211.short_preamble = short_preamble;
+	} break;
+	case HW_VAR_WPA_CONFIG:
+		rtl_write_byte(rtlpriv, REG_SECCFG_8822B, *((u8 *)val));
+		break;
+	case HW_VAR_AMPDU_FACTOR: {
+		u32 ampdu_len = (*((u8 *)val));
+
+		ampdu_len = (0x2000 << ampdu_len) - 1;
+		rtl_write_dword(rtlpriv, REG_AMPDU_MAX_LENGTH_8822B, ampdu_len);
+	} break;
+	case HW_VAR_AC_PARAM: {
+		u8 e_aci = *((u8 *)val);
+
+		if (mac->vif && mac->vif->bss_conf.assoc && !mac->act_scanning)
+			rtl8822be_set_qos(hw, e_aci);
+
+		if (rtlpci->acm_method != EACMWAY2_SW)
+			rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_ACM_CTRL,
+						      (u8 *)(&e_aci));
+	} break;
+	case HW_VAR_ACM_CTRL: {
+		u8 e_aci = *((u8 *)val);
+		union aci_aifsn *aifs = (union aci_aifsn *)&mac->ac[0].aifs;
+
+		u8 acm = aifs->f.acm;
+		u8 acm_ctrl = rtl_read_byte(rtlpriv, REG_ACMHWCTRL_8822B);
+
+		acm_ctrl = acm_ctrl | ((rtlpci->acm_method == 2) ? 0x0 : 0x1);
+
+		if (acm) {
+			switch (e_aci) {
+			case AC0_BE:
+				acm_ctrl |= ACMHW_BEQ_EN;
+				break;
+			case AC2_VI:
+				acm_ctrl |= ACMHW_VIQ_EN;
+				break;
+			case AC3_VO:
+				acm_ctrl |= ACMHW_VOQ_EN;
+				break;
+			default:
+				RT_TRACE(
+					rtlpriv, COMP_ERR, DBG_WARNING,
+					"HW_VAR_ACM_CTRL acm set failed: eACI is %d\n",
+					acm);
+				break;
+			}
+		} else {
+			switch (e_aci) {
+			case AC0_BE:
+				acm_ctrl &= (~ACMHW_BEQ_EN);
+				break;
+			case AC2_VI:
+				acm_ctrl &= (~ACMHW_VIQ_EN);
+				break;
+			case AC3_VO:
+				acm_ctrl &= (~ACMHW_VOQ_EN);
+				break;
+			default:
+				RT_TRACE(rtlpriv, COMP_ERR, DBG_DMESG,
+					 "switch case not process\n");
+				break;
+			}
+		}
+
+		RT_TRACE(rtlpriv, COMP_QOS, DBG_TRACE,
+			 "SetHwReg8190pci(): [HW_VAR_ACM_CTRL] Write 0x%X\n",
+			 acm_ctrl);
+		rtl_write_byte(rtlpriv, REG_ACMHWCTRL_8822B, acm_ctrl);
+	} break;
+	case HW_VAR_RCR: {
+		rtl_write_dword(rtlpriv, REG_RCR_8822B, ((u32 *)(val))[0]);
+		rtlpci->receive_config = ((u32 *)(val))[0];
+	} break;
+	case HW_VAR_RETRY_LIMIT: {
+		u8 retry_limit = ((u8 *)(val))[0];
+
+		rtl_write_word(rtlpriv, REG_RETRY_LIMIT_8822B,
+			       retry_limit << RETRY_LIMIT_SHORT_SHIFT |
+				       retry_limit << RETRY_LIMIT_LONG_SHIFT);
+	} break;
+	case HW_VAR_DUAL_TSF_RST:
+		rtl_write_byte(rtlpriv, REG_DUAL_TSF_RST_8822B,
+			       (BIT(0) | BIT(1)));
+		break;
+	case HW_VAR_EFUSE_BYTES:
+		efuse->efuse_usedbytes = *((u16 *)val);
+		break;
+	case HW_VAR_EFUSE_USAGE:
+		efuse->efuse_usedpercentage = *((u8 *)val);
+		break;
+	case HW_VAR_IO_CMD:
+		rtl8822be_phy_set_io_cmd(hw, (*(enum io_type *)val));
+		break;
+	case HW_VAR_SET_RPWM:
+		break;
+	case HW_VAR_H2C_FW_PWRMODE:
+		rtl8822be_set_fw_pwrmode_cmd(hw, (*(u8 *)val));
+		break;
+	case HW_VAR_FW_PSMODE_STATUS:
+		ppsc->fw_current_inpsmode = *((bool *)val);
+		break;
+	case HW_VAR_RESUME_CLK_ON:
+		_rtl8822be_set_fw_ps_rf_on(hw);
+		break;
+	case HW_VAR_FW_LPS_ACTION: {
+		bool b_enter_fwlps = *((bool *)val);
+
+		if (b_enter_fwlps)
+			_rtl8822be_fwlps_enter(hw);
+		else
+			_rtl8822be_fwlps_leave(hw);
+	} break;
+	case HW_VAR_H2C_FW_JOINBSSRPT: {
+		u8 mstatus = (*(u8 *)val);
+
+		if (mstatus == RT_MEDIA_CONNECT) {
+			rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_AID, NULL);
+			_rtl8822be_download_rsvd_page(hw);
+		}
+		rtl8822be_set_default_port_id_cmd(hw);
+		rtl8822be_set_fw_media_status_rpt_cmd(hw, mstatus);
+	} break;
+	case HW_VAR_H2C_FW_P2P_PS_OFFLOAD:
+		rtl8822be_set_p2p_ps_offload_cmd(hw, (*(u8 *)val));
+		break;
+	case HW_VAR_AID: {
+		u16 u2btmp;
+
+		u2btmp = rtl_read_word(rtlpriv, REG_BCN_PSR_RPT_8822B);
+		u2btmp &= 0xC000;
+		rtl_write_word(rtlpriv, REG_BCN_PSR_RPT_8822B,
+			       (u2btmp | mac->assoc_id));
+	} break;
+	case HW_VAR_CORRECT_TSF: {
+		u8 btype_ibss = ((u8 *)(val))[0];
+
+		if (btype_ibss)
+			_rtl8822be_stop_tx_beacon(hw);
+
+		_rtl8822be_set_bcn_ctrl_reg(hw, 0, BIT(3));
+
+		rtl_write_dword(rtlpriv, REG_TSFTR_8822B,
+				(u32)(mac->tsf & 0xffffffff));
+		rtl_write_dword(rtlpriv, REG_TSFTR_8822B + 4,
+				(u32)((mac->tsf >> 32) & 0xffffffff));
+
+		_rtl8822be_set_bcn_ctrl_reg(hw, BIT(3), 0);
+
+		if (btype_ibss)
+			_rtl8822be_resume_tx_beacon(hw);
+	} break;
+	case HW_VAR_KEEP_ALIVE: {
+		u8 array[2];
+
+		array[0] = 0xff;
+		array[1] = *((u8 *)val);
+		rtl8822be_fill_h2c_cmd(hw, H2C_8822B_KEEP_ALIVE_CTRL, 2, array);
+	} break;
+	default:
+		RT_TRACE(rtlpriv, COMP_ERR, DBG_DMESG,
+			 "switch case not process %x\n", variable);
+		break;
+	}
+}
+
+static void _rtl8822be_gen_refresh_led_state(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci_priv *pcipriv = rtl_pcipriv(hw);
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	struct rtl_led *led0 = &pcipriv->ledctl.sw_led0;
+
+	if (rtlpriv->rtlhal.up_first_time)
+		return;
+
+	if (ppsc->rfoff_reason == RF_CHANGE_BY_IPS)
+		rtl8822be_sw_led_on(hw, led0);
+	else if (ppsc->rfoff_reason == RF_CHANGE_BY_INIT)
+		rtl8822be_sw_led_on(hw, led0);
+	else
+		rtl8822be_sw_led_off(hw, led0);
+}
+
+static bool _rtl8822be_init_trxbd(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	/*struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));*/
+
+	u8 bytetmp;
+	/*u16 wordtmp;*/
+	u32 dwordtmp;
+
+	/* Set TX/RX descriptor physical address -- HI part */
+	if (!rtlpriv->cfg->mod_params->dma64)
+		goto dma64_end;
+
+	rtl_write_dword(rtlpriv, REG_H2CQ_TXBD_DESA_8822B + 4,
+			((u64)rtlpci->tx_ring[H2C_QUEUE].buffer_desc_dma) >>
+				32);
+	rtl_write_dword(rtlpriv, REG_BCNQ_TXBD_DESA_8822B + 4,
+			((u64)rtlpci->tx_ring[BEACON_QUEUE].buffer_desc_dma) >>
+				32);
+	rtl_write_dword(rtlpriv, REG_MGQ_TXBD_DESA_8822B + 4,
+			(u64)rtlpci->tx_ring[MGNT_QUEUE].buffer_desc_dma >> 32);
+	rtl_write_dword(rtlpriv, REG_VOQ_TXBD_DESA_8822B + 4,
+			(u64)rtlpci->tx_ring[VO_QUEUE].buffer_desc_dma >> 32);
+	rtl_write_dword(rtlpriv, REG_VIQ_TXBD_DESA_8822B + 4,
+			(u64)rtlpci->tx_ring[VI_QUEUE].buffer_desc_dma >> 32);
+	rtl_write_dword(rtlpriv, REG_BEQ_TXBD_DESA_8822B + 4,
+			(u64)rtlpci->tx_ring[BE_QUEUE].buffer_desc_dma >> 32);
+	rtl_write_dword(rtlpriv, REG_BKQ_TXBD_DESA_8822B + 4,
+			(u64)rtlpci->tx_ring[BK_QUEUE].buffer_desc_dma >> 32);
+	rtl_write_dword(rtlpriv, REG_HI0Q_TXBD_DESA_8822B + 4,
+			(u64)rtlpci->tx_ring[HIGH_QUEUE].buffer_desc_dma >> 32);
+
+	rtl_write_dword(rtlpriv, REG_RXQ_RXBD_DESA_8822B + 4,
+			(u64)rtlpci->rx_ring[RX_MPDU_QUEUE].dma >> 32);
+
+dma64_end:
+	/* Set TX/RX descriptor physical address(from OS API). */
+	rtl_write_dword(rtlpriv, REG_H2CQ_TXBD_DESA_8822B,
+			((u64)rtlpci->tx_ring[H2C_QUEUE].buffer_desc_dma) &
+				DMA_BIT_MASK(32));
+	rtl_write_dword(rtlpriv, REG_BCNQ_TXBD_DESA_8822B,
+			((u64)rtlpci->tx_ring[BEACON_QUEUE].buffer_desc_dma) &
+				DMA_BIT_MASK(32));
+	rtl_write_dword(rtlpriv, REG_MGQ_TXBD_DESA_8822B,
+			(u64)rtlpci->tx_ring[MGNT_QUEUE].buffer_desc_dma &
+				DMA_BIT_MASK(32));
+	rtl_write_dword(rtlpriv, REG_VOQ_TXBD_DESA_8822B,
+			(u64)rtlpci->tx_ring[VO_QUEUE].buffer_desc_dma &
+				DMA_BIT_MASK(32));
+	rtl_write_dword(rtlpriv, REG_VIQ_TXBD_DESA_8822B,
+			(u64)rtlpci->tx_ring[VI_QUEUE].buffer_desc_dma &
+				DMA_BIT_MASK(32));
+	rtl_write_dword(rtlpriv, REG_BEQ_TXBD_DESA_8822B,
+			(u64)rtlpci->tx_ring[BE_QUEUE].buffer_desc_dma &
+				DMA_BIT_MASK(32));
+	dwordtmp = rtl_read_dword(rtlpriv, REG_BEQ_TXBD_DESA_8822B); /* need? */
+	rtl_write_dword(rtlpriv, REG_BKQ_TXBD_DESA_8822B,
+			(u64)rtlpci->tx_ring[BK_QUEUE].buffer_desc_dma &
+				DMA_BIT_MASK(32));
+	rtl_write_dword(rtlpriv, REG_HI0Q_TXBD_DESA_8822B,
+			(u64)rtlpci->tx_ring[HIGH_QUEUE].buffer_desc_dma &
+				DMA_BIT_MASK(32));
+
+	rtl_write_dword(rtlpriv, REG_RXQ_RXBD_DESA_8822B,
+			(u64)rtlpci->rx_ring[RX_MPDU_QUEUE].dma &
+				DMA_BIT_MASK(32));
+
+	/* Reset R/W point */
+	rtl_write_dword(rtlpriv, REG_BD_RWPTR_CLR_8822B, 0x3fffffff);
+
+	/* Reset the H2CQ R/W point index to 0 */
+	dwordtmp = rtl_read_dword(rtlpriv, REG_H2CQ_CSR_8822B);
+	rtl_write_dword(rtlpriv, REG_H2CQ_CSR_8822B,
+			(dwordtmp | BIT(8) | BIT(16)));
+
+	bytetmp = rtl_read_byte(rtlpriv, REG_PCIE_CTRL_8822B + 3);
+	rtl_write_byte(rtlpriv, REG_PCIE_CTRL_8822B + 3, bytetmp | 0xF7);
+
+	rtl_write_dword(rtlpriv, REG_INT_MIG_8822B, 0);
+
+	rtl_write_dword(rtlpriv, REG_MCUTST_I_8822B, 0x0);
+
+	rtl_write_word(rtlpriv, REG_H2CQ_TXBD_NUM_8822B,
+		       TX_DESC_NUM_8822B |
+			       ((RTL8822BE_SEG_NUM << 12) & 0x3000));
+	rtl_write_word(rtlpriv, REG_MGQ_TXBD_NUM_8822B,
+		       TX_DESC_NUM_8822B |
+			       ((RTL8822BE_SEG_NUM << 12) & 0x3000));
+	rtl_write_word(rtlpriv, REG_VOQ_TXBD_NUM_8822B,
+		       TX_DESC_NUM_8822B |
+			       ((RTL8822BE_SEG_NUM << 12) & 0x3000));
+	rtl_write_word(rtlpriv, REG_VIQ_TXBD_NUM_8822B,
+		       TX_DESC_NUM_8822B |
+			       ((RTL8822BE_SEG_NUM << 12) & 0x3000));
+	rtl_write_word(rtlpriv, REG_BEQ_TXBD_NUM_8822B,
+		       TX_DESC_NUM_8822B |
+			       ((RTL8822BE_SEG_NUM << 12) & 0x3000));
+	rtl_write_word(rtlpriv, REG_VOQ_TXBD_NUM_8822B,
+		       TX_DESC_NUM_8822B |
+			       ((RTL8822BE_SEG_NUM << 12) & 0x3000));
+	rtl_write_word(rtlpriv, REG_BKQ_TXBD_NUM_8822B,
+		       TX_DESC_NUM_8822B |
+			       ((RTL8822BE_SEG_NUM << 12) & 0x3000));
+	rtl_write_word(rtlpriv, REG_HI0Q_TXBD_NUM_8822B,
+		       TX_DESC_NUM_8822B |
+			       ((RTL8822BE_SEG_NUM << 12) & 0x3000));
+	rtl_write_word(rtlpriv, REG_HI1Q_TXBD_NUM_8822B,
+		       TX_DESC_NUM_8822B |
+			       ((RTL8822BE_SEG_NUM << 12) & 0x3000));
+	rtl_write_word(rtlpriv, REG_HI2Q_TXBD_NUM_8822B,
+		       TX_DESC_NUM_8822B |
+			       ((RTL8822BE_SEG_NUM << 12) & 0x3000));
+	rtl_write_word(rtlpriv, REG_HI3Q_TXBD_NUM_8822B,
+		       TX_DESC_NUM_8822B |
+			       ((RTL8822BE_SEG_NUM << 12) & 0x3000));
+	rtl_write_word(rtlpriv, REG_HI4Q_TXBD_NUM_8822B,
+		       TX_DESC_NUM_8822B |
+			       ((RTL8822BE_SEG_NUM << 12) & 0x3000));
+	rtl_write_word(rtlpriv, REG_HI5Q_TXBD_NUM_8822B,
+		       TX_DESC_NUM_8822B |
+			       ((RTL8822BE_SEG_NUM << 12) & 0x3000));
+	rtl_write_word(rtlpriv, REG_HI6Q_TXBD_NUM_8822B,
+		       TX_DESC_NUM_8822B |
+			       ((RTL8822BE_SEG_NUM << 12) & 0x3000));
+	rtl_write_word(rtlpriv, REG_HI7Q_TXBD_NUM_8822B,
+		       TX_DESC_NUM_8822B |
+			       ((RTL8822BE_SEG_NUM << 12) & 0x3000));
+	/*Rx*/
+	rtl_write_word(rtlpriv, REG_RX_RXBD_NUM_8822B,
+		       RX_DESC_NUM_8822BE |
+		       ((RTL8822BE_SEG_NUM << 13) & 0x6000) | 0x8000);
+
+	rtl_write_dword(rtlpriv, REG_BD_RWPTR_CLR_8822B, 0XFFFFFFFF);
+
+	_rtl8822be_gen_refresh_led_state(hw);
+
+	return true;
+}
+
+static void _rtl8822be_enable_aspm_back_door(struct ieee80211_hw *hw)
+{
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	u8 tmp;
+
+	if (!ppsc->support_backdoor)
+		return;
+
+	pci_read_config_byte(rtlpci->pdev, 0x70f, &tmp);
+	pci_write_config_byte(rtlpci->pdev, 0x70f, tmp | BIT(7));
+
+	pci_read_config_byte(rtlpci->pdev, 0x719, &tmp);
+	pci_write_config_byte(rtlpci->pdev, 0x719, tmp | BIT(3) | BIT(4));
+}
+
+void rtl8822be_enable_hw_security_config(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 sec_reg_value;
+	u8 tmp;
+
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
+		 "PairwiseEncAlgorithm = %d GroupEncAlgorithm = %d\n",
+		 rtlpriv->sec.pairwise_enc_algorithm,
+		 rtlpriv->sec.group_enc_algorithm);
+
+	if (rtlpriv->cfg->mod_params->sw_crypto || rtlpriv->sec.use_sw_sec) {
+		RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG,
+			 "not open hw encryption\n");
+		return;
+	}
+
+	sec_reg_value = SCR_TX_ENC_ENABLE | SRC_RX_DEC_ENABLE;
+
+	if (rtlpriv->sec.use_defaultkey) {
+		sec_reg_value |= SCR_TX_USE_DK;
+		sec_reg_value |= SCR_RX_USE_DK;
+	}
+
+	sec_reg_value |= (SCR_RXBCUSEDK | SCR_TXBCUSEDK);
+
+	tmp = rtl_read_byte(rtlpriv, REG_CR_8822B + 1);
+	rtl_write_byte(rtlpriv, REG_CR_8822B + 1, tmp | BIT(1));
+
+	RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG, "The SECR-value %x\n",
+		 sec_reg_value);
+
+	rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_WPA_CONFIG, &sec_reg_value);
+}
+
+static bool _rtl8822be_check_pcie_dma_hang(struct rtl_priv *rtlpriv)
+{
+	u8 tmp;
+
+	/* write reg 0x350 Bit[26]=1. Enable debug port. */
+	tmp = rtl_read_byte(rtlpriv, REG_DBI_FLAG_V1_8822B + 3);
+	if (!(tmp & BIT(2))) {
+		rtl_write_byte(rtlpriv, REG_DBI_FLAG_V1_8822B + 3,
+			       (tmp | BIT(2)));
+		mdelay(100); /* Suggested by DD Justin_tsai. */
+	}
+
+	/* read reg 0x350 Bit[25] if 1 : RX hang
+	 * read reg 0x350 Bit[24] if 1 : TX hang
+	 */
+	tmp = rtl_read_byte(rtlpriv, REG_DBI_FLAG_V1_8822B + 3);
+	if ((tmp & BIT(0)) || (tmp & BIT(1))) {
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+			 "CheckPcieDMAHang8822BE(): true!!\n");
+		return true;
+	} else {
+		return false;
+	}
+}
+
+static void _rtl8822be_reset_pcie_interface_dma(struct rtl_priv *rtlpriv,
+						bool mac_power_on)
+{
+	u8 tmp;
+	bool release_mac_rx_pause;
+	u8 backup_pcie_dma_pause;
+
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+		 "ResetPcieInterfaceDMA8822BE()\n");
+
+	/* Revise Note: Follow the document "PCIe RX DMA Hang Reset Flow_v03"
+	 * released by SD1 Alan.
+	 * 2013.05.07, by tynli.
+	 */
+
+	/* 1. disable register write lock
+	 *	write 0x1C bit[1:0] = 2'h0
+	 *	write 0xCC bit[2] = 1'b1
+	 */
+	tmp = rtl_read_byte(rtlpriv, REG_RSV_CTRL_8822B);
+	tmp &= ~(BIT(1) | BIT(0));
+	rtl_write_byte(rtlpriv, REG_RSV_CTRL_8822B, tmp);
+	tmp = rtl_read_byte(rtlpriv, REG_PMC_DBG_CTRL2_8822B);
+	tmp |= BIT(2);
+	rtl_write_byte(rtlpriv, REG_PMC_DBG_CTRL2_8822B, tmp);
+
+	/* 2. Check and pause TRX DMA
+	 *	write 0x284 bit[18] = 1'b1
+	 *	write 0x301 = 0xFF
+	 */
+	tmp = rtl_read_byte(rtlpriv, REG_RXDMA_CONTROL_8822B);
+	if (tmp & BIT(2)) {
+		/* Already pause before the function for another purpose. */
+		release_mac_rx_pause = false;
+	} else {
+		rtl_write_byte(rtlpriv, REG_RXDMA_CONTROL_8822B,
+			       (tmp | BIT(2)));
+		release_mac_rx_pause = true;
+	}
+
+	backup_pcie_dma_pause = rtl_read_byte(rtlpriv, REG_PCIE_CTRL_8822B + 1);
+	if (backup_pcie_dma_pause != 0xFF)
+		rtl_write_byte(rtlpriv, REG_PCIE_CTRL_8822B + 1, 0xFF);
+
+	if (mac_power_on) {
+		/* 3. reset TRX function
+		 *	write 0x100 = 0x00
+		 */
+		rtl_write_byte(rtlpriv, REG_CR_8822B, 0);
+	}
+
+	/* 4. Reset PCIe DMA
+	 *	write 0x003 bit[0] = 0
+	 */
+	tmp = rtl_read_byte(rtlpriv, REG_SYS_FUNC_EN_8822B + 1);
+	tmp &= ~(BIT(0));
+	rtl_write_byte(rtlpriv, REG_SYS_FUNC_EN_8822B + 1, tmp);
+
+	/* 5. Enable PCIe DMA
+	 *	write 0x003 bit[0] = 1
+	 */
+	tmp = rtl_read_byte(rtlpriv, REG_SYS_FUNC_EN_8822B + 1);
+	tmp |= BIT(0);
+	rtl_write_byte(rtlpriv, REG_SYS_FUNC_EN_8822B + 1, tmp);
+
+	if (mac_power_on) {
+		/* 6. enable TRX function
+		 *	write 0x100 = 0xFF
+		 */
+		rtl_write_byte(rtlpriv, REG_CR_8822B, 0xFF);
+
+		/* We should init LLT & RQPN and
+		 * prepare Tx/Rx descrptor address later
+		 * because MAC function is reset.
+		 */
+	}
+
+	/* 7. Restore PCIe autoload down bit
+	 *	write 0xF8 bit[17] = 1'b1
+	 */
+	tmp = rtl_read_byte(rtlpriv, REG_SYS_STATUS2_8822B + 2);
+	tmp |= BIT(1);
+	rtl_write_byte(rtlpriv, REG_SYS_STATUS2_8822B + 2, tmp);
+
+	/* In MAC power on state, BB and RF maybe in ON state,
+	 * if we release TRx DMA here
+	 * it will cause packets to be started to Tx/Rx,
+	 * so we release Tx/Rx DMA later.
+	 */
+	if (!mac_power_on) {
+		/* 8. release TRX DMA
+		 *	write 0x284 bit[18] = 1'b0
+		 *	write 0x301 = 0x00
+		 */
+		if (release_mac_rx_pause) {
+			tmp = rtl_read_byte(rtlpriv, REG_RXDMA_CONTROL_8822B);
+			rtl_write_byte(rtlpriv, REG_RXDMA_CONTROL_8822B,
+				       (tmp & (~BIT(2))));
+		}
+		rtl_write_byte(rtlpriv, REG_PCIE_CTRL_8822B + 1,
+			       backup_pcie_dma_pause);
+	}
+
+	/* 9. lock system register
+	 *	write 0xCC bit[2] = 1'b0
+	 */
+	tmp = rtl_read_byte(rtlpriv, REG_PMC_DBG_CTRL2_8822B);
+	tmp &= ~(BIT(2));
+	rtl_write_byte(rtlpriv, REG_PMC_DBG_CTRL2_8822B, tmp);
+}
+
+int rtl8822be_hw_init(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	int err = 0;
+	u8 tmp_u1b;
+
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD, " Rtl8822BE hw init\n");
+	rtlpriv->rtlhal.being_init_adapter = true;
+	rtlpriv->intf_ops->disable_aspm(hw);
+
+	if (_rtl8822be_check_pcie_dma_hang(rtlpriv)) {
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "8822be dma hang!\n");
+		_rtl8822be_reset_pcie_interface_dma(rtlpriv,
+						    rtlhal->mac_func_enable);
+		rtlhal->mac_func_enable = false;
+	}
+
+	/* init TRX BD */
+	_rtl8822be_init_trxbd(hw);
+
+	/* use halmac to init */
+	err = rtlpriv->halmac.ops->halmac_init_hal(rtlpriv);
+	if (err) {
+		pr_err("halmac_init_hal failed\n");
+		rtlhal->fw_ready = false;
+		return err;
+	}
+
+	rtlhal->fw_ready = true;
+
+	/* have to init after halmac init */
+	tmp_u1b = rtl_read_byte(rtlpriv, REG_PCIE_CTRL_8822B + 2);
+	rtl_write_byte(rtlpriv, REG_PCIE_CTRL_8822B + 2, (tmp_u1b | BIT(4)));
+
+	/*rtl_write_word(rtlpriv, REG_PCIE_CTRL_8822B, 0x8000);*/
+	rtlhal->rx_tag = 0;
+
+	rtl_write_byte(rtlpriv, REG_RX_DRVINFO_SZ_8822B, 0x4);
+
+	/*fw related variable initialize */
+	ppsc->fw_current_inpsmode = false;
+	rtlhal->fw_ps_state = FW_PS_STATE_ALL_ON_8822B;
+	rtlhal->fw_clk_change_in_progress = false;
+	rtlhal->allow_sw_to_change_hwclc = false;
+	rtlhal->last_hmeboxnum = 0;
+
+	rtlphy->rfreg_chnlval[0] =
+		rtl_get_rfreg(hw, RF90_PATH_A, RF_CHNLBW, RFREG_OFFSET_MASK);
+	rtlphy->rfreg_chnlval[1] =
+		rtl_get_rfreg(hw, RF90_PATH_B, RF_CHNLBW, RFREG_OFFSET_MASK);
+	rtlphy->backup_rf_0x1a = (u32)rtl_get_rfreg(hw, RF90_PATH_A, RF_RX_G1,
+						    RFREG_OFFSET_MASK);
+	rtlphy->rfreg_chnlval[0] =
+		(rtlphy->rfreg_chnlval[0] & 0xfffff3ff) | BIT(10) | BIT(11);
+
+	rtlhal->mac_func_enable = true;
+
+	if (rtlpriv->cfg->ops->get_btc_status())
+		rtlpriv->btcoexist.btc_ops->btc_power_on_setting(rtlpriv);
+
+	/* reset cam / set security */
+	rtl_cam_reset_all_entry(hw);
+	rtl8822be_enable_hw_security_config(hw);
+
+	/* check RCR/ICV bit */
+	rtlpci->receive_config &= ~(RCR_ACRC32 | RCR_AICV);
+	rtl_write_dword(rtlpriv, REG_RCR_8822B, rtlpci->receive_config);
+
+	/* clear rx ctrl frame */
+	rtl_write_word(rtlpriv, REG_RXFLTMAP1_8822B, 0);
+
+	ppsc->rfpwr_state = ERFON;
+
+	rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_ETHER_ADDR, mac->mac_addr);
+	_rtl8822be_enable_aspm_back_door(hw);
+	rtlpriv->intf_ops->enable_aspm(hw);
+
+	if (rtlpriv->cfg->ops->get_btc_status())
+		rtlpriv->btcoexist.btc_ops->btc_init_hw_config(rtlpriv);
+	else
+		rtlpriv->btcoexist.btc_ops->btc_init_hw_config_wifi_only(
+								rtlpriv);
+
+	rtlpriv->rtlhal.being_init_adapter = false;
+
+	rtlpriv->phydm.ops->phydm_init_dm(rtlpriv);
+
+	/* clear ISR, and IMR will be on later */
+	rtl_write_dword(rtlpriv, REG_HISR0_8822B,
+			rtl_read_dword(rtlpriv, REG_HISR0_8822B));
+
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD, "end of Rtl8822BE hw init %x\n",
+		 err);
+	return 0;
+}
+
+static u32 _rtl8822be_read_chip_version(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	/*enum version_8822b version = VERSION_UNKNOWN;*/
+	u32 version;
+	u32 value32;
+
+	rtlphy->rf_type = RF_2T2R;
+
+	value32 = rtl_read_dword(rtlpriv, REG_SYS_CFG1_8822B);
+
+	version = value32;
+
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD, "Chip RF Type: %s\n",
+		 (rtlphy->rf_type == RF_2T2R) ? "RF_2T2R" : "RF_1T1R");
+
+	return version;
+}
+
+static int _rtl8822be_set_media_status(struct ieee80211_hw *hw,
+				       enum nl80211_iftype type)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 bt_msr = rtl_read_byte(rtlpriv, MSR);
+	enum led_ctl_mode ledaction = LED_CTL_NO_LINK;
+	u8 mode = MSR_NOLINK;
+
+	bt_msr &= 0xfc;
+
+	switch (type) {
+	case NL80211_IFTYPE_UNSPECIFIED:
+		mode = MSR_NOLINK;
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE,
+			 "Set Network type to NO LINK!\n");
+		break;
+	case NL80211_IFTYPE_ADHOC:
+	case NL80211_IFTYPE_MESH_POINT:
+		mode = MSR_ADHOC;
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE,
+			 "Set Network type to Ad Hoc!\n");
+		break;
+	case NL80211_IFTYPE_STATION:
+		mode = MSR_INFRA;
+		ledaction = LED_CTL_LINK;
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE,
+			 "Set Network type to STA!\n");
+		break;
+	case NL80211_IFTYPE_AP:
+		mode = MSR_AP;
+		ledaction = LED_CTL_LINK;
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE,
+			 "Set Network type to AP!\n");
+		break;
+	default:
+		pr_err("Network type %d not support!\n", type);
+		return 1;
+	}
+
+	/* MSR_INFRA == Link in infrastructure network;
+	 * MSR_ADHOC == Link in ad hoc network;
+	 * Therefore, check link state is necessary.
+	 *
+	 * MSR_AP == AP mode; link state is not cared here.
+	 */
+	if (mode != MSR_AP && rtlpriv->mac80211.link_state < MAC80211_LINKED) {
+		mode = MSR_NOLINK;
+		ledaction = LED_CTL_NO_LINK;
+	}
+
+	if (mode == MSR_NOLINK || mode == MSR_INFRA) {
+		_rtl8822be_stop_tx_beacon(hw);
+		_rtl8822be_enable_bcn_sub_func(hw);
+	} else if (mode == MSR_ADHOC || mode == MSR_AP) {
+		_rtl8822be_resume_tx_beacon(hw);
+		_rtl8822be_disable_bcn_sub_func(hw);
+	} else {
+		RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING,
+			 "Set HW_VAR_MEDIA_STATUS: No such media status(%x).\n",
+			 mode);
+	}
+
+	rtl_write_byte(rtlpriv, (MSR), bt_msr | mode);
+	rtlpriv->cfg->ops->led_control(hw, ledaction);
+	if (mode == MSR_AP)
+		rtl_write_byte(rtlpriv, REG_BCNTCFG_8822B + 1, 0x00);
+	else
+		rtl_write_byte(rtlpriv, REG_BCNTCFG_8822B + 1, 0x66);
+	return 0;
+}
+
+void rtl8822be_set_check_bssid(struct ieee80211_hw *hw, bool check_bssid)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	u32 reg_rcr = rtlpci->receive_config;
+
+	if (rtlpriv->psc.rfpwr_state != ERFON)
+		return;
+
+	if (check_bssid) {
+		reg_rcr |= (RCR_CBSSID_DATA | RCR_CBSSID_BCN);
+		rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_RCR, (u8 *)(&reg_rcr));
+		_rtl8822be_set_bcn_ctrl_reg(hw, 0, BIT(4));
+	} else if (!check_bssid) {
+		reg_rcr &= (~(RCR_CBSSID_DATA | RCR_CBSSID_BCN));
+		_rtl8822be_set_bcn_ctrl_reg(hw, BIT(4), 0);
+		rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_RCR, (u8 *)(&reg_rcr));
+	}
+}
+
+int rtl8822be_set_network_type(struct ieee80211_hw *hw,
+			       enum nl80211_iftype type)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	if (_rtl8822be_set_media_status(hw, type))
+		return -EOPNOTSUPP;
+
+	if (rtlpriv->mac80211.link_state == MAC80211_LINKED) {
+		if (type != NL80211_IFTYPE_AP &&
+		    type != NL80211_IFTYPE_MESH_POINT)
+			rtl8822be_set_check_bssid(hw, true);
+	} else {
+		rtl8822be_set_check_bssid(hw, false);
+	}
+
+	return 0;
+}
+
+void rtl8822be_set_qos(struct ieee80211_hw *hw, int aci)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtlpriv);
+	u32 ac_param;
+
+	ac_param = rtl_get_hal_edca_param(hw, mac->vif, mac->mode,
+					  &mac->edca_param[aci]);
+
+	switch (aci) {
+	case AC1_BK:
+		rtl_write_dword(rtlpriv, REG_EDCA_BK_PARAM_8822B, ac_param);
+		break;
+	case AC0_BE:
+		rtl_write_dword(rtlpriv, REG_EDCA_BE_PARAM_8822B, ac_param);
+		break;
+	case AC2_VI:
+		rtl_write_dword(rtlpriv, REG_EDCA_VI_PARAM_8822B, ac_param);
+		break;
+	case AC3_VO:
+		rtl_write_dword(rtlpriv, REG_EDCA_VO_PARAM_8822B, ac_param);
+		break;
+	default:
+		WARN_ONCE(true, "invalid aci: %d !\n", aci);
+		break;
+	}
+}
+
+void rtl8822be_enable_interrupt(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+
+	rtl_write_dword(rtlpriv, REG_HIMR0_8822B,
+			rtlpci->irq_mask[0] & 0xFFFFFFFF);
+	rtl_write_dword(rtlpriv, REG_HIMR1_8822B,
+			rtlpci->irq_mask[1] & 0xFFFFFFFF);
+	rtl_write_dword(rtlpriv, REG_HIMR3_8822B,
+			rtlpci->irq_mask[3] & 0xFFFFFFFF);
+	rtlpci->irq_enabled = true;
+}
+
+void rtl8822be_disable_interrupt(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+
+	rtl_write_dword(rtlpriv, REG_HIMR0_8822B, IMR_DISABLED);
+	rtl_write_dword(rtlpriv, REG_HIMR1_8822B, IMR_DISABLED);
+	rtl_write_dword(rtlpriv, REG_HIMR3_8822B, IMR_DISABLED);
+	rtlpci->irq_enabled = false;
+	/*synchronize_irq(rtlpci->pdev->irq);*/
+}
+
+void rtl8822be_card_disable(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	enum nl80211_iftype opmode;
+
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD, "RTL8822be card disable\n");
+
+	RT_SET_PS_LEVEL(ppsc, RT_RF_OFF_LEVL_HALT_NIC);
+
+	mac->link_state = MAC80211_NOLINK;
+	opmode = NL80211_IFTYPE_UNSPECIFIED;
+
+	_rtl8822be_set_media_status(hw, opmode);
+
+	if (rtlpriv->rtlhal.driver_is_goingto_unload ||
+	    ppsc->rfoff_reason > RF_CHANGE_BY_PS)
+		rtlpriv->cfg->ops->led_control(hw, LED_CTL_POWER_OFF);
+
+	rtlpriv->phydm.ops->phydm_deinit_dm(rtlpriv);
+
+	rtlpriv->halmac.ops->halmac_deinit_hal(rtlpriv);
+
+	/* after power off we should do iqk again */
+	if (!rtlpriv->cfg->ops->get_btc_status())
+		rtlpriv->phy.iqk_initialized = false;
+}
+
+void rtl8822be_interrupt_recognized(struct ieee80211_hw *hw, u32 *p_inta,
+				    u32 *p_intb, u32 *p_intc, u32 *p_intd)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+
+	*p_inta =
+		rtl_read_dword(rtlpriv, REG_HISR0_8822B) & rtlpci->irq_mask[0];
+	rtl_write_dword(rtlpriv, REG_HISR0_8822B, *p_inta);
+
+	*p_intb =
+		rtl_read_dword(rtlpriv, REG_HISR1_8822B) & rtlpci->irq_mask[1];
+	rtl_write_dword(rtlpriv, REG_HISR1_8822B, *p_intb);
+
+	*p_intd =
+		rtl_read_dword(rtlpriv, REG_HISR3_8822B) & rtlpci->irq_mask[3];
+	rtl_write_dword(rtlpriv, REG_HISR3_8822B, *p_intd);
+}
+
+void rtl8822be_set_beacon_related_registers(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	u16 bcn_interval, atim_window;
+
+	bcn_interval = mac->beacon_interval;
+	atim_window = 2; /*FIX MERGE */
+	rtl8822be_disable_interrupt(hw);
+	rtl_write_word(rtlpriv, REG_ATIMWND_8822B, atim_window);
+	rtl_write_word(rtlpriv, REG_MBSSID_BCN_SPACE_8822B, bcn_interval);
+	rtl_write_word(rtlpriv, REG_BCNTCFG_8822B, 0x660f);
+	rtl_write_byte(rtlpriv, REG_RXTSF_OFFSET_CCK_8822B, 0x18);
+	rtl_write_byte(rtlpriv, REG_RXTSF_OFFSET_OFDM_8822B, 0x18);
+	rtl_write_byte(rtlpriv, 0x606, 0x30);
+	rtlpci->reg_bcn_ctrl_val |= BIT(3);
+	rtl_write_byte(rtlpriv, REG_BCN_CTRL_8822B,
+		       (u8)rtlpci->reg_bcn_ctrl_val);
+}
+
+void rtl8822be_set_beacon_interval(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	u16 bcn_interval = mac->beacon_interval;
+
+	RT_TRACE(rtlpriv, COMP_BEACON, DBG_DMESG, "beacon_interval:%d\n",
+		 bcn_interval);
+	rtl_write_word(rtlpriv, REG_MBSSID_BCN_SPACE_8822B, bcn_interval);
+}
+
+void rtl8822be_update_interrupt_mask(struct ieee80211_hw *hw, u32 add_msr,
+				     u32 rm_msr)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+
+	RT_TRACE(rtlpriv, COMP_INTR, DBG_LOUD, "add_msr:%x, rm_msr:%x\n",
+		 add_msr, rm_msr);
+
+	if (add_msr)
+		rtlpci->irq_mask[0] |= add_msr;
+	if (rm_msr)
+		rtlpci->irq_mask[0] &= (~rm_msr);
+	rtl8822be_disable_interrupt(hw);
+	rtl8822be_enable_interrupt(hw);
+}
+
+static bool _rtl8822be_get_chnl_group(u8 chnl, u8 *group)
+{
+	bool in_24g;
+
+	if (chnl <= 14) {
+		in_24g = true;
+
+		if (chnl >= 1 && chnl <= 2)
+			*group = 0;
+		else if (chnl >= 3 && chnl <= 5)
+			*group = 1;
+		else if (chnl >= 6 && chnl <= 8)
+			*group = 2;
+		else if (chnl >= 9 && chnl <= 11)
+			*group = 3;
+		else if (chnl >= 12 && chnl <= 14)
+			*group = 4;
+	} else {
+		in_24g = false;
+
+		if (chnl >= 36 && chnl <= 42)
+			*group = 0;
+		else if (chnl >= 44 && chnl <= 48)
+			*group = 1;
+		else if (chnl >= 50 && chnl <= 58)
+			*group = 2;
+		else if (chnl >= 60 && chnl <= 64)
+			*group = 3;
+		else if (chnl >= 100 && chnl <= 106)
+			*group = 4;
+		else if (chnl >= 108 && chnl <= 114)
+			*group = 5;
+		else if (chnl >= 116 && chnl <= 122)
+			*group = 6;
+		else if (chnl >= 124 && chnl <= 130)
+			*group = 7;
+		else if (chnl >= 132 && chnl <= 138)
+			*group = 8;
+		else if (chnl >= 140 && chnl <= 144)
+			*group = 9;
+		else if (chnl >= 149 && chnl <= 155)
+			*group = 10;
+		else if (chnl >= 157 && chnl <= 161)
+			*group = 11;
+		else if (chnl >= 165 && chnl <= 171)
+			*group = 12;
+		else if (chnl >= 173 && chnl <= 177)
+			*group = 13;
+	}
+	return in_24g;
+}
+
+static inline bool power_valid(u8 power)
+{
+	if (power <= 63)
+		return true;
+
+	return false;
+}
+
+static inline s8 power_diff(s8 diff)
+{
+	/* bit sign number to 8 bit sign number */
+	if (diff & BIT(3))
+		diff |= 0xF0;
+
+	return diff;
+}
+
+static void _rtl8822be_read_power_value_fromprom(struct ieee80211_hw *hw,
+						 struct txpower_info_2g *pwr2g,
+						 struct txpower_info_5g *pwr5g,
+						 bool autoload_fail, u8 *hwinfo)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u32 rf, addr = EEPROM_TX_PWR_INX_8822B, group, i = 0;
+	u8 power;
+	s8 diff;
+
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+		 "hal_ReadPowerValueFromPROM8822B(): PROMContent[0x%x]=0x%x\n",
+		 (addr + 1), hwinfo[addr + 1]);
+	if (hwinfo[addr + 1] == 0xFF) /*YJ,add,120316*/
+		autoload_fail = true;
+
+	memset(pwr2g, 0, sizeof(struct txpower_info_2g));
+	memset(pwr5g, 0, sizeof(struct txpower_info_5g));
+
+	if (autoload_fail) {
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+			 "auto load fail : Use Default value!\n");
+		for (rf = 0; rf < MAX_RF_PATH; rf++) {
+			/* 2.4G default value */
+			for (group = 0; group < MAX_CHNL_GROUP_24G; group++) {
+				pwr2g->index_cck_base[rf][group] = 0x2D;
+				pwr2g->index_bw40_base[rf][group] = 0x2D;
+			}
+			for (i = 0; i < MAX_TX_COUNT; i++) {
+				if (i == 0) {
+					pwr2g->bw20_diff[rf][0] = 0x02;
+					pwr2g->ofdm_diff[rf][0] = 0x04;
+				} else {
+					pwr2g->bw20_diff[rf][i] = 0xFE;
+					pwr2g->bw40_diff[rf][i] = 0xFE;
+					pwr2g->cck_diff[rf][i] = 0xFE;
+					pwr2g->ofdm_diff[rf][i] = 0xFE;
+				}
+			}
+
+			/*5G default value*/
+			for (group = 0; group < MAX_CHNL_GROUP_5G; group++)
+				pwr5g->index_bw40_base[rf][group] = 0x2A;
+
+			for (i = 0; i < MAX_TX_COUNT; i++) {
+				if (i == 0) {
+					pwr5g->ofdm_diff[rf][0] = 0x04;
+					pwr5g->bw20_diff[rf][0] = 0x00;
+					pwr5g->bw80_diff[rf][0] = 0xFE;
+					pwr5g->bw160_diff[rf][0] = 0xFE;
+				} else {
+					pwr5g->ofdm_diff[rf][i] = 0xFE;
+					pwr5g->bw20_diff[rf][i] = 0xFE;
+					pwr5g->bw40_diff[rf][i] = 0xFE;
+					pwr5g->bw80_diff[rf][i] = 0xFE;
+					pwr5g->bw160_diff[rf][i] = 0xFE;
+				}
+			}
+		}
+		return;
+	}
+
+	rtl_priv(hw)->efuse.txpwr_fromeprom = true;
+
+	for (rf = 0; rf < 2 /*MAX_RF_PATH*/; rf++) {
+		/*2.4G default value*/
+		for (group = 0; group < MAX_CHNL_GROUP_24G; group++) {
+			power = hwinfo[addr++];
+			if (power_valid(power))
+				pwr2g->index_cck_base[rf][group] = power;
+		}
+		for (group = 0; group < MAX_CHNL_GROUP_24G - 1; group++) {
+			power = hwinfo[addr++];
+			if (power_valid(power))
+				pwr2g->index_bw40_base[rf][group] = power;
+		}
+		for (i = 0; i < MAX_TX_COUNT; i++) {
+			if (i == 0) {
+				pwr2g->bw40_diff[rf][i] = 0;
+
+				diff = (hwinfo[addr] & 0xF0) >> 4;
+				pwr2g->bw20_diff[rf][i] = power_diff(diff);
+
+				diff = hwinfo[addr] & 0x0F;
+				pwr2g->ofdm_diff[rf][i] = power_diff(diff);
+
+				pwr2g->cck_diff[rf][i] = 0;
+
+				addr++;
+			} else {
+				diff = (hwinfo[addr] & 0xF0) >> 4;
+				pwr2g->bw40_diff[rf][i] = power_diff(diff);
+
+				diff = hwinfo[addr] & 0x0F;
+				pwr2g->bw20_diff[rf][i] = power_diff(diff);
+
+				addr++;
+
+				diff = (hwinfo[addr] & 0xF0) >> 4;
+				pwr2g->ofdm_diff[rf][i] = power_diff(diff);
+
+				diff = hwinfo[addr] & 0x0F;
+				pwr2g->cck_diff[rf][i] = power_diff(diff);
+
+				addr++;
+			}
+		}
+
+		/*5G default value*/
+		for (group = 0; group < MAX_CHNL_GROUP_5G; group++) {
+			power = hwinfo[addr++];
+			if (power_valid(power))
+				pwr5g->index_bw40_base[rf][group] = power;
+		}
+
+		for (i = 0; i < MAX_TX_COUNT; i++) {
+			if (i == 0) {
+				pwr5g->bw40_diff[rf][i] = 0;
+
+				diff = (hwinfo[addr] & 0xF0) >> 4;
+				pwr5g->bw20_diff[rf][i] = power_diff(diff);
+
+				diff = hwinfo[addr] & 0x0F;
+				pwr5g->ofdm_diff[rf][i] = power_diff(diff);
+
+				addr++;
+			} else {
+				diff = (hwinfo[addr] & 0xF0) >> 4;
+				pwr5g->bw40_diff[rf][i] = power_diff(diff);
+
+				diff = hwinfo[addr] & 0x0F;
+				pwr5g->bw20_diff[rf][i] = power_diff(diff);
+
+				addr++;
+			}
+		}
+
+		diff = (hwinfo[addr] & 0xF0) >> 4;
+		pwr5g->ofdm_diff[rf][1] = power_diff(diff);
+
+		diff = hwinfo[addr] & 0x0F;
+		pwr5g->ofdm_diff[rf][2] = power_diff(diff);
+
+		addr++;
+
+		diff = hwinfo[addr] & 0x0F;
+		pwr5g->ofdm_diff[rf][3] = power_diff(diff);
+
+		addr++;
+
+		for (i = 0; i < MAX_TX_COUNT; i++) {
+			diff = (hwinfo[addr] & 0xF0) >> 4;
+			pwr5g->bw80_diff[rf][i] = power_diff(diff);
+
+			diff = hwinfo[addr] & 0x0F;
+			pwr5g->bw160_diff[rf][i] = power_diff(diff);
+
+			addr++;
+		}
+	}
+}
+
+static void _rtl8822be_read_txpower_info_from_hwpg(struct ieee80211_hw *hw,
+						   bool autoload_fail,
+						   u8 *hwinfo)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_efuse *efu = rtl_efuse(rtl_priv(hw));
+	struct txpower_info_2g pwr2g;
+	struct txpower_info_5g pwr5g;
+	u8 channel5g[CHANNEL_MAX_NUMBER_5G] = {
+		36,  38,  40,  42,  44,  46,  48, /* Band 1 */
+		52,  54,  56,  58,  60,  62,  64, /* Band 2 */
+		100, 102, 104, 106, 108, 110, 112, /* Band 3 */
+		116, 118, 120, 122, 124, 126, 128, /* Band 3 */
+		132, 134, 136, 138, 140, 142, 144, /* Band 3 */
+		149, 151, 153, 155, 157, 159, 161, /* Band 4 */
+		165, 167, 169, 171, 173, 175, 177}; /* Band 4 */
+	u8 channel5g_80m[CHANNEL_MAX_NUMBER_5G_80M] = {42,  58,  106, 122,
+						       138, 155, 171};
+	u8 rf, group;
+	u8 i;
+
+	_rtl8822be_read_power_value_fromprom(hw, &pwr2g, &pwr5g, autoload_fail,
+					     hwinfo);
+
+	for (rf = 0; rf < MAX_RF_PATH; rf++) {
+		for (i = 0; i < CHANNEL_MAX_NUMBER_2G; i++) {
+			_rtl8822be_get_chnl_group(i + 1, &group);
+
+			if (i == CHANNEL_MAX_NUMBER_2G - 1) {
+				efu->txpwrlevel_cck[rf][i] =
+					pwr2g.index_cck_base[rf][5];
+				efu->txpwrlevel_ht40_1s[rf][i] =
+					pwr2g.index_bw40_base[rf][group];
+			} else {
+				efu->txpwrlevel_cck[rf][i] =
+					pwr2g.index_cck_base[rf][group];
+				efu->txpwrlevel_ht40_1s[rf][i] =
+					pwr2g.index_bw40_base[rf][group];
+			}
+		}
+		for (i = 0; i < CHANNEL_MAX_NUMBER_5G; i++) {
+			_rtl8822be_get_chnl_group(channel5g[i], &group);
+			efu->txpwr_5g_bw40base[rf][i] =
+				pwr5g.index_bw40_base[rf][group];
+		}
+		for (i = 0; i < CHANNEL_MAX_NUMBER_5G_80M; i++) {
+			u8 upper, lower;
+
+			_rtl8822be_get_chnl_group(channel5g_80m[i], &group);
+			upper = pwr5g.index_bw40_base[rf][group];
+			lower = pwr5g.index_bw40_base[rf][group + 1];
+
+			efu->txpwr_5g_bw80base[rf][i] = (upper + lower) / 2;
+		}
+		for (i = 0; i < MAX_TX_COUNT; i++) {
+			efu->txpwr_cckdiff[rf][i] = pwr2g.cck_diff[rf][i];
+			efu->txpwr_legacyhtdiff[rf][i] = pwr2g.ofdm_diff[rf][i];
+			efu->txpwr_ht20diff[rf][i] = pwr2g.bw20_diff[rf][i];
+			efu->txpwr_ht40diff[rf][i] = pwr2g.bw40_diff[rf][i];
+
+			efu->txpwr_5g_ofdmdiff[rf][i] = pwr5g.ofdm_diff[rf][i];
+			efu->txpwr_5g_bw20diff[rf][i] = pwr5g.bw20_diff[rf][i];
+			efu->txpwr_5g_bw40diff[rf][i] = pwr5g.bw40_diff[rf][i];
+			efu->txpwr_5g_bw80diff[rf][i] = pwr5g.bw80_diff[rf][i];
+		}
+	}
+
+	if (!autoload_fail)
+		efu->eeprom_thermalmeter = hwinfo[EEPROM_THERMAL_METER_8822B];
+	else
+		efu->eeprom_thermalmeter = EEPROM_DEFAULT_THERMALMETER;
+
+	if (efu->eeprom_thermalmeter == 0xff || autoload_fail) {
+		efu->apk_thermalmeterignore = true;
+		efu->eeprom_thermalmeter = EEPROM_DEFAULT_THERMALMETER;
+	}
+
+	efu->thermalmeter[0] = efu->eeprom_thermalmeter;
+	RTPRINT(rtlpriv, FINIT, INIT_TXPOWER, "thermalmeter = 0x%x\n",
+		efu->eeprom_thermalmeter);
+
+	if (!autoload_fail) {
+		efu->eeprom_regulatory =
+			hwinfo[EEPROM_RF_BOARD_OPTION_8822B] & 0x07;
+		if (hwinfo[EEPROM_RF_BOARD_OPTION_8822B] == 0xFF)
+			efu->eeprom_regulatory = 0;
+	} else {
+		efu->eeprom_regulatory = 0;
+	}
+	RTPRINT(rtlpriv, FINIT, INIT_TXPOWER, "eeprom_regulatory = 0x%x\n",
+		efu->eeprom_regulatory);
+}
+
+static void _rtl8822be_read_pa_type(struct ieee80211_hw *hw, u8 *hwinfo,
+				    bool autoload_fail)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+
+	if (!autoload_fail) {
+		rtlhal->pa_type_2g = hwinfo[EEPROM_2G_5G_PA_TYPE_8822B];
+		rtlhal->lna_type_2g =
+			hwinfo[EEPROM_2G_LNA_TYPE_GAIN_SEL_AB_8822B];
+		if (rtlhal->pa_type_2g == 0xFF)
+			rtlhal->pa_type_2g = 0;
+		if (rtlhal->lna_type_2g == 0xFF)
+			rtlhal->lna_type_2g = 0;
+
+		rtlhal->external_pa_2g = (rtlhal->pa_type_2g & BIT(4)) ? 1 : 0;
+		rtlhal->external_lna_2g =
+			(rtlhal->lna_type_2g & BIT(3)) ? 1 : 0;
+
+		rtlhal->pa_type_5g = hwinfo[EEPROM_2G_5G_PA_TYPE_8822B];
+		rtlhal->lna_type_5g =
+			hwinfo[EEPROM_5G_LNA_TYPE_GAIN_SEL_AB_8822B];
+		if (rtlhal->pa_type_5g == 0xFF)
+			rtlhal->pa_type_5g = 0;
+		if (rtlhal->lna_type_5g == 0xFF)
+			rtlhal->lna_type_5g = 0;
+
+		rtlhal->external_pa_5g = (rtlhal->pa_type_5g & BIT(0)) ? 1 : 0;
+		rtlhal->external_lna_5g =
+			(rtlhal->lna_type_5g & BIT(3)) ? 1 : 0;
+	} else {
+		rtlhal->external_pa_2g = 0;
+		rtlhal->external_lna_2g = 0;
+		rtlhal->external_pa_5g = 0;
+		rtlhal->external_lna_5g = 0;
+	}
+}
+
+static void _rtl8822be_read_amplifier_type(struct ieee80211_hw *hw, u8 *hwinfo,
+					   bool autoload_fail)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+
+	u8 ext_type_pa_2g_a =
+		(hwinfo[EEPROM_2G_LNA_TYPE_GAIN_SEL_AB_8822B] & BIT(2)) >>
+		2; /* 0xBD[2] */
+	u8 ext_type_pa_2g_b =
+		(hwinfo[EEPROM_2G_LNA_TYPE_GAIN_SEL_AB_8822B] & BIT(6)) >>
+		6; /* 0xBD[6] */
+	u8 ext_type_pa_5g_a =
+		(hwinfo[EEPROM_5G_LNA_TYPE_GAIN_SEL_AB_8822B] & BIT(2)) >>
+		2; /* 0xBF[2] */
+	u8 ext_type_pa_5g_b =
+		(hwinfo[EEPROM_5G_LNA_TYPE_GAIN_SEL_AB_8822B] & BIT(6)) >>
+		6; /* 0xBF[6] */
+	u8 ext_type_lna_2g_a = (hwinfo[EEPROM_2G_LNA_TYPE_GAIN_SEL_AB_8822B] &
+				(BIT(1) | BIT(0))) >>
+			       0; /* 0xBD[1:0] */
+	u8 ext_type_lna_2g_b = (hwinfo[EEPROM_2G_LNA_TYPE_GAIN_SEL_AB_8822B] &
+				(BIT(5) | BIT(4))) >>
+			       4; /* 0xBD[5:4] */
+	u8 ext_type_lna_5g_a = (hwinfo[EEPROM_5G_LNA_TYPE_GAIN_SEL_AB_8822B] &
+				(BIT(1) | BIT(0))) >>
+			       0; /* 0xBF[1:0] */
+	u8 ext_type_lna_5g_b = (hwinfo[EEPROM_5G_LNA_TYPE_GAIN_SEL_AB_8822B] &
+				(BIT(5) | BIT(4))) >>
+			       4; /* 0xBF[5:4] */
+
+	_rtl8822be_read_pa_type(hw, hwinfo, autoload_fail);
+
+	/* [2.4G] Path A and B are both extPA */
+	if ((rtlhal->pa_type_2g & (BIT(5) | BIT(4))) == (BIT(5) | BIT(4)))
+		rtlhal->type_gpa = ext_type_pa_2g_b << 2 | ext_type_pa_2g_a;
+
+	/* [5G] Path A and B are both extPA */
+	if ((rtlhal->pa_type_5g & (BIT(1) | BIT(0))) == (BIT(1) | BIT(0)))
+		rtlhal->type_apa = ext_type_pa_5g_b << 2 | ext_type_pa_5g_a;
+
+	/* [2.4G] Path A and B are both extLNA */
+	if ((rtlhal->lna_type_2g & (BIT(7) | BIT(3))) == (BIT(7) | BIT(3)))
+		rtlhal->type_glna = ext_type_lna_2g_b << 2 | ext_type_lna_2g_a;
+
+	/* [5G] Path A and B are both extLNA */
+	if ((rtlhal->lna_type_5g & (BIT(7) | BIT(3))) == (BIT(7) | BIT(3)))
+		rtlhal->type_alna = ext_type_lna_5g_b << 2 | ext_type_lna_5g_a;
+}
+
+static void _rtl8822be_read_rfe_type(struct ieee80211_hw *hw, u8 *hwinfo,
+				     bool autoload_fail)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+
+	if (!autoload_fail)
+		rtlhal->rfe_type = hwinfo[EEPROM_RFE_OPTION_8822B];
+	else
+		rtlhal->rfe_type = 0;
+
+	if (rtlhal->rfe_type == 0xFF)
+		rtlhal->rfe_type = 0;
+
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD, "RFE Type: 0x%2x\n",
+		 rtlhal->rfe_type);
+}
+
+static void _rtl8822be_read_adapter_info(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
+	struct rtl_halmac_ops *halmac_ops = rtlpriv->halmac.ops;
+	u16 i, usvalue;
+	u8 *hwinfo;
+	u16 eeprom_id;
+	u32 efuse_size;
+	int err;
+
+	if (rtlefuse->epromtype != EEPROM_BOOT_EFUSE) {
+		pr_err("RTL8822B Not boot from efuse!!");
+		return;
+	}
+
+	/* read logical efuse size (normalely, 0x0300) */
+	err = halmac_ops->halmac_get_logical_efuse_size(rtlpriv, &efuse_size);
+
+	if (err || !efuse_size) {
+		pr_err("halmac_get_logical_efuse_size err=%d efuse_size=0x%X",
+		       err, efuse_size);
+		efuse_size = HWSET_MAX_SIZE;
+	}
+
+	if (efuse_size > HWSET_MAX_SIZE) {
+		pr_err("halmac_get_logical_efuse_size efuse_size=0x%X > 0x%X",
+		       efuse_size, HWSET_MAX_SIZE);
+		efuse_size = HWSET_MAX_SIZE;
+	}
+
+	/* read efuse */
+	hwinfo = kzalloc(efuse_size, GFP_KERNEL);
+
+	err = halmac_ops->halmac_read_logical_efuse_map(rtlpriv, hwinfo,
+							efuse_size);
+	if (err) {
+		pr_err("%s: <ERROR> fail to get efuse map!\n", __func__);
+		goto label_end;
+	}
+
+	/* copy to efuse_map (need?) */
+	memcpy(&rtlefuse->efuse_map[EFUSE_INIT_MAP][0], hwinfo,
+	       EFUSE_MAX_LOGICAL_SIZE);
+	memcpy(&rtlefuse->efuse_map[EFUSE_MODIFY_MAP][0], hwinfo,
+	       EFUSE_MAX_LOGICAL_SIZE);
+
+	/* parse content */
+	RT_PRINT_DATA(rtlpriv, COMP_INIT, DBG_DMESG, "MAP\n", hwinfo,
+		      HWSET_MAX_SIZE);
+
+	eeprom_id = *((u16 *)&hwinfo[0]);
+	if (eeprom_id != RTL8822B_EEPROM_ID) {
+		RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING,
+			 "EEPROM ID(%#x) is invalid!!\n", eeprom_id);
+		rtlefuse->autoload_failflag = true;
+	} else {
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD, "Autoload OK\n");
+		rtlefuse->autoload_failflag = false;
+	}
+
+	if (rtlefuse->autoload_failflag)
+		goto label_end;
+
+	/*VID DID SVID SDID*/
+	rtlefuse->eeprom_vid = *(u16 *)&hwinfo[EEPROM_VID];
+	rtlefuse->eeprom_did = *(u16 *)&hwinfo[EEPROM_DID];
+	rtlefuse->eeprom_svid = *(u16 *)&hwinfo[EEPROM_SVID];
+	rtlefuse->eeprom_smid = *(u16 *)&hwinfo[EEPROM_SMID];
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD, "EEPROMId = 0x%4x\n", eeprom_id);
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD, "EEPROM VID = 0x%4x\n",
+		 rtlefuse->eeprom_vid);
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD, "EEPROM DID = 0x%4x\n",
+		 rtlefuse->eeprom_did);
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD, "EEPROM SVID = 0x%4x\n",
+		 rtlefuse->eeprom_svid);
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD, "EEPROM SMID = 0x%4x\n",
+		 rtlefuse->eeprom_smid);
+	/*customer ID*/
+	rtlefuse->eeprom_oemid = *(u8 *)&hwinfo[EEPROM_CUSTOM_ID_8822B];
+	if (rtlefuse->eeprom_oemid == 0xFF)
+		rtlefuse->eeprom_oemid = 0;
+
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD, "EEPROM Customer ID: 0x%2x\n",
+		 rtlefuse->eeprom_oemid);
+	/*EEPROM version*/
+	rtlefuse->eeprom_version = *(u8 *)&hwinfo[EEPROM_VERSION_8822B];
+	/*mac address*/
+	for (i = 0; i < 6; i += 2) {
+		usvalue = *(u16 *)&hwinfo[EEPROM_MAC_ADDR_8822BE + i];
+		*((u16 *)(&rtlefuse->dev_addr[i])) = usvalue;
+	}
+
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "dev_addr: %pM\n",
+		 rtlefuse->dev_addr);
+
+	/* channel plan */
+	rtlefuse->eeprom_channelplan =
+		*(u8 *)&hwinfo[EEPROM_CHANNEL_PLAN_8822B];
+
+	/* set channel plan from efuse */
+	rtlefuse->channel_plan = rtlefuse->eeprom_channelplan;
+	if (rtlefuse->channel_plan == 0xFF)
+		rtlefuse->channel_plan = 0x7f; /* use 2G + 5G as default */
+
+	/*tx power*/
+	_rtl8822be_read_txpower_info_from_hwpg(hw, rtlefuse->autoload_failflag,
+					       hwinfo);
+
+	rtl8822be_read_bt_coexist_info_from_hwpg(
+		hw, rtlefuse->autoload_failflag, hwinfo);
+
+	/*amplifier type*/
+	_rtl8822be_read_amplifier_type(hw, hwinfo, rtlefuse->autoload_failflag);
+
+	/*rfe type*/
+	_rtl8822be_read_rfe_type(hw, hwinfo, rtlefuse->autoload_failflag);
+
+	/*board type*/
+	rtlefuse->board_type =
+		(((*(u8 *)&hwinfo[EEPROM_RF_BOARD_OPTION_8822B]) & 0xE0) >> 5);
+	if ((*(u8 *)&hwinfo[EEPROM_RF_BOARD_OPTION_8822B]) == 0xFF)
+		rtlefuse->board_type = 0;
+
+	if (rtlpriv->btcoexist.btc_info.btcoexist == 1)
+		rtlefuse->board_type |= BIT(2); /* ODM_BOARD_BT */
+
+	/* phydm maintain rtlhal->board_type and rtlhal->package_type */
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD, "board_type = 0x%x\n",
+		 rtlefuse->board_type);
+	/*parse xtal*/
+	rtlefuse->crystalcap = hwinfo[EEPROM_XTAL_8822B];
+	if (hwinfo[EEPROM_XTAL_8822B] == 0xFF)
+		rtlefuse->crystalcap = 0; /*0x20;*/
+
+	/*antenna diversity*/
+	rtlefuse->antenna_div_type = 0;
+	rtlefuse->antenna_div_cfg = 0;
+
+label_end:
+	kfree(hwinfo);
+}
+
+static void _rtl8822be_hal_customized_behavior(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci_priv *pcipriv = rtl_pcipriv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+
+	pcipriv->ledctl.led_opendrain = true;
+
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "RT Customized ID: 0x%02X\n",
+		 rtlhal->oem_id);
+}
+
+static void _rtl8822be_read_pa_bias(struct ieee80211_hw *hw,
+				    struct rtl_phydm_params *params)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_halmac_ops *halmac_ops = rtlpriv->halmac.ops;
+	u32 size;
+	u8 *map = NULL;
+
+	/* fill default values */
+	params->efuse0x3d7 = 0xFF;
+	params->efuse0x3d8 = 0xFF;
+
+	if (halmac_ops->halmac_get_physical_efuse_size(rtlpriv, &size))
+		goto err;
+
+	map = kmalloc(size, GFP_KERNEL);
+	if (!map)
+		goto err;
+
+	if (halmac_ops->halmac_read_physical_efuse_map(rtlpriv, map, size))
+		goto err;
+
+	params->efuse0x3d7 = map[0x3d7];
+	params->efuse0x3d8 = map[0x3d8];
+
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+		 "efuse0x3d7 = 0x%2x, efuse0x3d8 = 0x%2x\n",
+		 params->efuse0x3d7, params->efuse0x3d8);
+
+err:
+	kfree(map);
+}
+
+void rtl8822be_read_eeprom_info(struct ieee80211_hw *hw,
+				struct rtl_phydm_params *params)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	u8 tmp_u1b;
+
+	rtlhal->version = _rtl8822be_read_chip_version(hw);
+
+	params->mp_chip = (rtlhal->version & BIT_RTL_ID_8822B) ? 0 : 1;
+	params->fab_ver = BIT_GET_VENDOR_ID_8822B(rtlhal->version) >> 2;
+	params->cut_ver = BIT_GET_CHIP_VER_8822B(rtlhal->version);
+
+	/* fab_ver mapping */
+	if (params->fab_ver == 2)
+		params->fab_ver = 1;
+	else if (params->fab_ver == 1)
+		params->fab_ver = 2;
+
+	/* read PA bias: params->efuse0x3d7/efuse0x3d8 */
+	_rtl8822be_read_pa_bias(hw, params);
+
+	if (get_rf_type(rtlphy) == RF_1T1R)
+		rtlpriv->dm.rfpath_rxenable[0] = true;
+	else
+		rtlpriv->dm.rfpath_rxenable[0] =
+			rtlpriv->dm.rfpath_rxenable[1] = true;
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD, "VersionID = 0x%4x\n",
+		 rtlhal->version);
+	tmp_u1b = rtl_read_byte(rtlpriv, REG_SYS_EEPROM_CTRL_8822B);
+	if (tmp_u1b & BIT(4)) {
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "Boot from EEPROM\n");
+		rtlefuse->epromtype = EEPROM_93C46;
+	} else {
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "Boot from EFUSE\n");
+		rtlefuse->epromtype = EEPROM_BOOT_EFUSE;
+	}
+	if (tmp_u1b & BIT(5)) {
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD, "Autoload OK\n");
+		rtlefuse->autoload_failflag = false;
+		_rtl8822be_read_adapter_info(hw);
+	} else {
+		pr_err("Autoload ERR!!\n");
+	}
+	_rtl8822be_hal_customized_behavior(hw);
+
+	rtlphy->rfpath_rx_enable[0] = true;
+	if (rtlphy->rf_type == RF_2T2R)
+		rtlphy->rfpath_rx_enable[1] = true;
+}
+
+void rtl8822be_read_eeprom_info_dummy(struct ieee80211_hw *hw)
+{
+	/*
+	 * 8822b use halmac, so
+	 * move rtl8822be_read_eeprom_info() to rtl8822be_init_sw_vars()
+	 * after halmac_init_adapter().
+	 */
+}
+
+static u32 _rtl8822be_rate_to_bitmap_2ssvht(__le16 vht_rate)
+{
+	u8 i, j, tmp_rate;
+	u32 rate_bitmap = 0;
+
+	for (i = j = 0; i < 4; i += 2, j += 10) {
+		tmp_rate = (le16_to_cpu(vht_rate) >> i) & 3;
+
+		switch (tmp_rate) {
+		case 2:
+			rate_bitmap = rate_bitmap | (0x03ff << j);
+			break;
+
+		case 1:
+			rate_bitmap = rate_bitmap | (0x01ff << j);
+			break;
+
+		case 0:
+			rate_bitmap = rate_bitmap | (0x00ff << j);
+			break;
+
+		default:
+			break;
+		}
+	}
+
+	return rate_bitmap;
+}
+
+static u8 _rtl8822be_get_vht_en(enum wireless_mode wirelessmode,
+				u32 ratr_bitmap)
+{
+	u8 ret = 0;
+
+	if (wirelessmode < WIRELESS_MODE_N_24G) {
+		ret = 0;
+	} else if (wirelessmode == WIRELESS_MODE_AC_24G) {
+		if (ratr_bitmap & 0xfff00000) /* Mix , 2SS */
+			ret = 3;
+		else /* Mix, 1SS */
+			ret = 2;
+	} else if (wirelessmode == WIRELESS_MODE_AC_5G) {
+		ret = 1;
+	} /* VHT */
+
+	return ret << 4;
+}
+
+static u8 _rtl8822be_get_ra_ldpc(struct ieee80211_hw *hw, u8 mac_id,
+				 struct rtl_sta_info *sta_entry,
+				 enum wireless_mode wirelessmode)
+{
+	u8 b_ldpc = 0;
+	/*not support ldpc, do not open*/
+	return b_ldpc << 2;
+}
+
+static u8 _rtl8822be_get_ra_rftype(struct ieee80211_hw *hw,
+				   enum wireless_mode wirelessmode,
+				   u32 ratr_bitmap)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	u8 rf_type = RF_1T1R;
+
+	if (rtlphy->rf_type == RF_1T1R) {
+		rf_type = RF_1T1R;
+	} else if (wirelessmode == WIRELESS_MODE_AC_5G ||
+		   wirelessmode == WIRELESS_MODE_AC_24G ||
+		   wirelessmode == WIRELESS_MODE_AC_ONLY) {
+		if (ratr_bitmap & 0xffc00000)
+			rf_type = RF_2T2R;
+	} else if (wirelessmode == WIRELESS_MODE_N_5G ||
+		   wirelessmode == WIRELESS_MODE_N_24G) {
+		if (ratr_bitmap & 0xfff00000)
+			rf_type = RF_2T2R;
+	}
+
+	return rf_type;
+}
+
+static bool _rtl8822be_get_ra_shortgi(struct ieee80211_hw *hw,
+				      struct ieee80211_sta *sta, u8 mac_id)
+{
+	bool b_short_gi = false;
+	u8 b_curshortgi_40mhz =
+		(sta->ht_cap.cap & IEEE80211_HT_CAP_SGI_40) ? 1 : 0;
+	u8 b_curshortgi_20mhz =
+		(sta->ht_cap.cap & IEEE80211_HT_CAP_SGI_20) ? 1 : 0;
+	u8 b_curshortgi_80mhz = 0;
+
+	b_curshortgi_80mhz =
+		(sta->vht_cap.cap & IEEE80211_VHT_CAP_SHORT_GI_80) ? 1 : 0;
+
+	if (mac_id == 99 /*MAC_ID_STATIC_FOR_BROADCAST_MULTICAST*/)
+		b_short_gi = false;
+
+	if (b_curshortgi_40mhz || b_curshortgi_80mhz || b_curshortgi_20mhz)
+		b_short_gi = true;
+
+	return b_short_gi;
+}
+
+static void rtl8822be_update_hal_rate_mask(struct ieee80211_hw *hw,
+					   struct ieee80211_sta *sta,
+					   u8 rssi_level, bool update_bw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct rtl_sta_info *sta_entry = NULL;
+	u32 ratr_bitmap, ratr_bitmap_msb = 0;
+	u8 ratr_index;
+	enum wireless_mode wirelessmode = 0;
+	u8 curtxbw_40mhz =
+		(sta->ht_cap.cap & IEEE80211_HT_CAP_SUP_WIDTH_20_40) ? 1 : 0;
+	bool b_shortgi = false;
+	u8 rate_mask[7];
+	u8 macid = 0;
+	u8 rf_type;
+
+	sta_entry = (struct rtl_sta_info *)sta->drv_priv;
+	wirelessmode = sta_entry->wireless_mode;
+
+	RT_TRACE(rtlpriv, COMP_RATR, DBG_LOUD, "wireless mode = 0x%x\n",
+		 wirelessmode);
+	if (mac->opmode == NL80211_IFTYPE_STATION ||
+	    mac->opmode == NL80211_IFTYPE_MESH_POINT) {
+		curtxbw_40mhz = mac->bw_40;
+	} else if (mac->opmode == NL80211_IFTYPE_AP ||
+		   mac->opmode == NL80211_IFTYPE_ADHOC)
+		macid = sta->aid + 1;
+	if (wirelessmode == WIRELESS_MODE_N_5G ||
+	    wirelessmode == WIRELESS_MODE_AC_5G ||
+	    wirelessmode == WIRELESS_MODE_A)
+		ratr_bitmap = (sta->supp_rates[NL80211_BAND_5GHZ]) << 4;
+	else
+		ratr_bitmap = sta->supp_rates[NL80211_BAND_2GHZ];
+
+	if (mac->opmode == NL80211_IFTYPE_ADHOC)
+		ratr_bitmap = 0xfff;
+
+	if (wirelessmode == WIRELESS_MODE_N_24G ||
+	    wirelessmode == WIRELESS_MODE_N_5G)
+		ratr_bitmap |= (sta->ht_cap.mcs.rx_mask[1] << 20 |
+				sta->ht_cap.mcs.rx_mask[0] << 12);
+	else if (wirelessmode == WIRELESS_MODE_AC_24G ||
+		 wirelessmode == WIRELESS_MODE_AC_5G ||
+		 wirelessmode == WIRELESS_MODE_AC_ONLY)
+		ratr_bitmap |= _rtl8822be_rate_to_bitmap_2ssvht(
+				       sta->vht_cap.vht_mcs.rx_mcs_map)
+			       << 12;
+
+	b_shortgi = _rtl8822be_get_ra_shortgi(hw, sta, macid);
+	rf_type = _rtl8822be_get_ra_rftype(hw, wirelessmode, ratr_bitmap);
+
+	ratr_index = rtlpriv->phydm.ops->phydm_rate_id_mapping(
+		rtlpriv, wirelessmode, rf_type, rtlphy->current_chan_bw);
+	sta_entry->ratr_index = ratr_index;
+
+	rtlpriv->phydm.ops->phydm_get_ra_bitmap(
+		rtlpriv, wirelessmode, rf_type, rtlphy->current_chan_bw,
+		rssi_level, &ratr_bitmap_msb, &ratr_bitmap);
+
+	RT_TRACE(rtlpriv, COMP_RATR, DBG_LOUD, "ratr_bitmap :%x\n",
+		 ratr_bitmap);
+
+	rate_mask[0] = macid;
+	rate_mask[1] = ratr_index | (b_shortgi ? 0x80 : 0x00);
+	rate_mask[2] =
+		rtlphy->current_chan_bw | ((!update_bw) << 3) |
+		_rtl8822be_get_vht_en(wirelessmode, ratr_bitmap) |
+		_rtl8822be_get_ra_ldpc(hw, macid, sta_entry, wirelessmode);
+
+	rate_mask[3] = (u8)(ratr_bitmap & 0x000000ff);
+	rate_mask[4] = (u8)((ratr_bitmap & 0x0000ff00) >> 8);
+	rate_mask[5] = (u8)((ratr_bitmap & 0x00ff0000) >> 16);
+	rate_mask[6] = (u8)((ratr_bitmap & 0xff000000) >> 24);
+
+	RT_TRACE(
+		rtlpriv, COMP_RATR, DBG_DMESG,
+		"Rate_index:%x, ratr_val:%08x, %02x:%02x:%02x:%02x:%02x:%02x:%02x\n",
+		ratr_index, ratr_bitmap, rate_mask[0], rate_mask[1],
+		rate_mask[2], rate_mask[3], rate_mask[4], rate_mask[5],
+		rate_mask[6]);
+	rtl8822be_fill_h2c_cmd(hw, H2C_8822B_MACID_CFG, 7, rate_mask);
+
+	/* for h2c cmd 0x46, only modify cmd id & ra mask */
+	/* Keep rate_mask0~2 of cmd 0x40, but clear byte3 and later */
+	/* 8822B has no 3SS, so keep it zeros. */
+	memset(rate_mask + 3, 0, 4);
+
+	rtl8822be_fill_h2c_cmd(hw, H2C_8822B_MACID_CFG_3SS, 7, rate_mask);
+
+	_rtl8822be_set_bcn_ctrl_reg(hw, BIT(3), 0);
+}
+
+void rtl8822be_update_hal_rate_tbl(struct ieee80211_hw *hw,
+				   struct ieee80211_sta *sta, u8 rssi_level,
+				   bool update_bw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	if (rtlpriv->dm.useramask)
+		rtl8822be_update_hal_rate_mask(hw, sta, rssi_level, update_bw);
+}
+
+void rtl8822be_update_channel_access_setting(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	u16 sifs_timer;
+
+	rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_SLOT_TIME,
+				      (u8 *)&mac->slot_time);
+	if (!mac->ht_enable)
+		sifs_timer = 0x0a0a;
+	else
+		sifs_timer = 0x0e0e;
+	rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_SIFS, (u8 *)&sifs_timer);
+}
+
+bool rtl8822be_gpio_radio_on_off_checking(struct ieee80211_hw *hw, u8 *valid)
+{
+	*valid = 1;
+	return true;
+}
+
+void rtl8822be_set_key(struct ieee80211_hw *hw, u32 key_index, u8 *p_macaddr,
+		       bool is_group, u8 enc_algo, bool is_wepkey,
+		       bool clear_all)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
+	u8 *macaddr = p_macaddr;
+	u32 entry_id = 0;
+	bool is_pairwise = false;
+
+	static u8 cam_const_addr[4][6] = {
+		{0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
+		{0x00, 0x00, 0x00, 0x00, 0x00, 0x01},
+		{0x00, 0x00, 0x00, 0x00, 0x00, 0x02},
+		{0x00, 0x00, 0x00, 0x00, 0x00, 0x03},
+	};
+	static u8 cam_const_broad[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+
+	if (clear_all) {
+		u8 idx = 0;
+		u8 cam_offset = 0;
+		u8 clear_number = 5;
+
+		RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG, "clear_all\n");
+
+		for (idx = 0; idx < clear_number; idx++) {
+			rtl_cam_mark_invalid(hw, cam_offset + idx);
+			rtl_cam_empty_entry(hw, cam_offset + idx);
+
+			if (idx < 5) {
+				memset(rtlpriv->sec.key_buf[idx], 0,
+				       MAX_KEY_LEN);
+				rtlpriv->sec.key_len[idx] = 0;
+			}
+		}
+
+		return;
+	}
+
+	switch (enc_algo) {
+	case WEP40_ENCRYPTION:
+		enc_algo = CAM_WEP40;
+		break;
+	case WEP104_ENCRYPTION:
+		enc_algo = CAM_WEP104;
+		break;
+	case TKIP_ENCRYPTION:
+		enc_algo = CAM_TKIP;
+		break;
+	case AESCCMP_ENCRYPTION:
+		enc_algo = CAM_AES;
+		break;
+	default:
+		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
+			 "switch case %#x not processed\n", enc_algo);
+		enc_algo = CAM_TKIP;
+		break;
+	}
+
+	if (is_wepkey || rtlpriv->sec.use_defaultkey) {
+		macaddr = cam_const_addr[key_index];
+		entry_id = key_index;
+	} else {
+		if (is_group) {
+			macaddr = cam_const_broad;
+			entry_id = key_index;
+		} else {
+			if (mac->opmode == NL80211_IFTYPE_AP) {
+				entry_id =
+					rtl_cam_get_free_entry(hw, p_macaddr);
+				if (entry_id >= TOTAL_CAM_ENTRY) {
+					pr_err("Can not find free hwsecurity cam entry\n");
+					return;
+				}
+			} else {
+				entry_id = CAM_PAIRWISE_KEY_POSITION;
+			}
+
+			key_index = PAIRWISE_KEYIDX;
+			is_pairwise = true;
+		}
+	}
+
+	if (rtlpriv->sec.key_len[key_index] == 0) {
+		RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG,
+			 "delete one entry, entry_id is %d\n", entry_id);
+		if (mac->opmode == NL80211_IFTYPE_AP)
+			rtl_cam_del_entry(hw, p_macaddr);
+		rtl_cam_delete_one_entry(hw, p_macaddr, entry_id);
+	} else {
+		RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG, "add one entry\n");
+		if (is_pairwise) {
+			RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG,
+				 "set Pairwise key\n");
+
+			rtl_cam_add_one_entry(hw, macaddr, key_index, entry_id,
+					      enc_algo, CAM_CONFIG_NO_USEDK,
+					      rtlpriv->sec.key_buf[key_index]);
+		} else {
+			RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG,
+				 "set group key\n");
+
+			if (mac->opmode == NL80211_IFTYPE_ADHOC) {
+				rtl_cam_add_one_entry(
+					hw, rtlefuse->dev_addr, PAIRWISE_KEYIDX,
+					CAM_PAIRWISE_KEY_POSITION, enc_algo,
+					CAM_CONFIG_NO_USEDK,
+					rtlpriv->sec.key_buf[entry_id]);
+			}
+
+			rtl_cam_add_one_entry(hw, macaddr, key_index, entry_id,
+					      enc_algo, CAM_CONFIG_NO_USEDK,
+					      rtlpriv->sec.key_buf[entry_id]);
+		}
+	}
+}
+
+void rtl8822be_read_bt_coexist_info_from_hwpg(struct ieee80211_hw *hw,
+					      bool auto_load_fail, u8 *hwinfo)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 value;
+	u32 val32;
+
+	val32 = rtl_read_dword(rtlpriv, REG_WL_BT_PWR_CTRL_8822B);
+	if (val32 & BIT_BT_FUNC_EN_8822B)
+		rtlpriv->btcoexist.btc_info.btcoexist = 1;
+	else
+		rtlpriv->btcoexist.btc_info.btcoexist = 0;
+
+	if (!auto_load_fail) {
+		value = hwinfo[EEPROM_RF_BT_SETTING_8822B];
+
+		rtlpriv->btcoexist.btc_info.bt_type = BT_RTL8822B;
+		rtlpriv->btcoexist.btc_info.ant_num =
+			(value & BIT(0) ? ANT_TOTAL_X1 : ANT_TOTAL_X2);
+	} else {
+		rtlpriv->btcoexist.btc_info.bt_type = BT_RTL8822B;
+		rtlpriv->btcoexist.btc_info.ant_num = ANT_TOTAL_X2;
+	}
+}
+
+void rtl8822be_bt_reg_init(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	/* 0:Low, 1:High, 2:From Efuse. */
+	rtlpriv->btcoexist.reg_bt_iso = 2;
+	/* 0:Idle, 1:None-SCO, 2:SCO, 3:From Counter. */
+	rtlpriv->btcoexist.reg_bt_sco = 3;
+	/* 0:Disable BT control A-MPDU, 1:Enable BT control A-MPDU. */
+	rtlpriv->btcoexist.reg_bt_sco = 0;
+}
+
+void rtl8822be_suspend(struct ieee80211_hw *hw) {}
+
+void rtl8822be_resume(struct ieee80211_hw *hw) {}
diff --git a/drivers/staging/rtlwifi/rtl8822be/hw.h b/drivers/staging/rtlwifi/rtl8822be/hw.h
new file mode 100644
index 000000000000..a91c276c5794
--- /dev/null
+++ b/drivers/staging/rtlwifi/rtl8822be/hw.h
@@ -0,0 +1,66 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __RTL8822B_HW_H__
+#define __RTL8822B_HW_H__
+
+extern u8 rtl_channel5g[CHANNEL_MAX_NUMBER_5G];
+extern u8 rtl_channel5g_80m[CHANNEL_MAX_NUMBER_5G_80M];
+
+void rtl8822be_get_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val);
+void rtl8822be_read_eeprom_info(struct ieee80211_hw *hw,
+				struct rtl_phydm_params *params);
+void rtl8822be_read_eeprom_info_dummy(struct ieee80211_hw *hw);
+void rtl8822be_interrupt_recognized(struct ieee80211_hw *hw, u32 *p_inta,
+				    u32 *p_intb, u32 *p_intc, u32 *p_intd);
+int rtl8822be_hw_init(struct ieee80211_hw *hw);
+void rtl8822be_card_disable(struct ieee80211_hw *hw);
+void rtl8822be_enable_interrupt(struct ieee80211_hw *hw);
+void rtl8822be_disable_interrupt(struct ieee80211_hw *hw);
+int rtl8822be_set_network_type(struct ieee80211_hw *hw,
+			       enum nl80211_iftype type);
+void rtl8822be_set_check_bssid(struct ieee80211_hw *hw, bool check_bssid);
+void rtl8822be_set_qos(struct ieee80211_hw *hw, int aci);
+void rtl8822be_set_beacon_related_registers(struct ieee80211_hw *hw);
+void rtl8822be_set_beacon_interval(struct ieee80211_hw *hw);
+void rtl8822be_update_interrupt_mask(struct ieee80211_hw *hw, u32 add_msr,
+				     u32 rm_msr);
+void rtl8822be_set_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val);
+void rtl8822be_update_hal_rate_tbl(struct ieee80211_hw *hw,
+				   struct ieee80211_sta *sta, u8 rssi_level,
+				   bool update_bw);
+void rtl8822be_update_channel_access_setting(struct ieee80211_hw *hw);
+bool rtl8822be_gpio_radio_on_off_checking(struct ieee80211_hw *hw, u8 *valid);
+void rtl8822be_set_key(struct ieee80211_hw *hw, u32 key_index, u8 *p_macaddr,
+		       bool is_group, u8 enc_algo, bool is_wepkey,
+		       bool clear_all);
+void rtl8822be_enable_hw_security_config(struct ieee80211_hw *hw);
+void rtl8822be_read_bt_coexist_info_from_hwpg(struct ieee80211_hw *hw,
+					      bool autoload_fail, u8 *hwinfo);
+void rtl8822be_bt_reg_init(struct ieee80211_hw *hw);
+void rtl8822be_suspend(struct ieee80211_hw *hw);
+void rtl8822be_resume(struct ieee80211_hw *hw);
+void rtl8822be_fw_clk_off_timer_callback(unsigned long data);
+#endif
diff --git a/drivers/staging/rtlwifi/rtl8822be/led.c b/drivers/staging/rtlwifi/rtl8822be/led.c
new file mode 100644
index 000000000000..f4b5af8ab116
--- /dev/null
+++ b/drivers/staging/rtlwifi/rtl8822be/led.c
@@ -0,0 +1,127 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#include "../wifi.h"
+#include "../pci.h"
+#include "reg.h"
+#include "led.h"
+
+static void _rtl8822be_init_led(struct ieee80211_hw *hw, struct rtl_led *pled,
+				enum rtl_led_pin ledpin)
+{
+	pled->hw = hw;
+	pled->ledpin = ledpin;
+	pled->ledon = false;
+}
+
+void rtl8822be_sw_led_on(struct ieee80211_hw *hw, struct rtl_led *pled)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	RT_TRACE(rtlpriv, COMP_LED, DBG_LOUD, "LedAddr:%X ledpin=%d\n",
+		 REG_LEDCFG2_8822B, pled->ledpin);
+
+	switch (pled->ledpin) {
+	case LED_PIN_GPIO0:
+		break;
+	case LED_PIN_LED0:
+		break;
+	case LED_PIN_LED1:
+		break;
+	default:
+		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
+			 "switch case not process\n");
+		break;
+	}
+	pled->ledon = true;
+}
+
+void rtl8822be_sw_led_off(struct ieee80211_hw *hw, struct rtl_led *pled)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	RT_TRACE(rtlpriv, COMP_LED, DBG_LOUD, "LedAddr:%X ledpin=%d\n",
+		 REG_LEDCFG2_8822B, pled->ledpin);
+
+	switch (pled->ledpin) {
+	case LED_PIN_GPIO0:
+		break;
+	case LED_PIN_LED0:
+		break;
+	case LED_PIN_LED1:
+
+		break;
+	default:
+		RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD,
+			 "switch case not process\n");
+		break;
+	}
+	pled->ledon = false;
+}
+
+void rtl8822be_init_sw_leds(struct ieee80211_hw *hw)
+{
+	struct rtl_pci_priv *pcipriv = rtl_pcipriv(hw);
+
+	_rtl8822be_init_led(hw, &pcipriv->ledctl.sw_led0, LED_PIN_LED0);
+	_rtl8822be_init_led(hw, &pcipriv->ledctl.sw_led1, LED_PIN_LED1);
+}
+
+static void _rtl8822be_sw_led_control(struct ieee80211_hw *hw,
+				      enum led_ctl_mode ledaction)
+{
+	struct rtl_pci_priv *pcipriv = rtl_pcipriv(hw);
+	struct rtl_led *led0 = &pcipriv->ledctl.sw_led0;
+
+	switch (ledaction) {
+	case LED_CTL_POWER_ON:
+	case LED_CTL_LINK:
+	case LED_CTL_NO_LINK:
+		rtl8822be_sw_led_on(hw, led0);
+		break;
+	case LED_CTL_POWER_OFF:
+		rtl8822be_sw_led_off(hw, led0);
+		break;
+	default:
+		break;
+	}
+}
+
+void rtl8822be_led_control(struct ieee80211_hw *hw, enum led_ctl_mode ledaction)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+
+	if ((ppsc->rfoff_reason > RF_CHANGE_BY_PS) &&
+	    (ledaction == LED_CTL_TX || ledaction == LED_CTL_RX ||
+	     ledaction == LED_CTL_SITE_SURVEY || ledaction == LED_CTL_LINK ||
+	     ledaction == LED_CTL_NO_LINK ||
+	     ledaction == LED_CTL_START_TO_LINK ||
+	     ledaction == LED_CTL_POWER_ON)) {
+		return;
+	}
+	RT_TRACE(rtlpriv, COMP_LED, DBG_TRACE, "ledaction %d,\n", ledaction);
+	_rtl8822be_sw_led_control(hw, ledaction);
+}
diff --git a/drivers/staging/rtlwifi/rtl8822be/led.h b/drivers/staging/rtlwifi/rtl8822be/led.h
new file mode 100644
index 000000000000..9c0a2290df7d
--- /dev/null
+++ b/drivers/staging/rtlwifi/rtl8822be/led.h
@@ -0,0 +1,34 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __RTL8822B_LED_H__
+#define __RTL8822B_LED_H__
+
+void rtl8822be_init_sw_leds(struct ieee80211_hw *hw);
+void rtl8822be_sw_led_on(struct ieee80211_hw *hw, struct rtl_led *pled);
+void rtl8822be_sw_led_off(struct ieee80211_hw *hw, struct rtl_led *pled);
+void rtl8822be_led_control(struct ieee80211_hw *hw,
+			   enum led_ctl_mode ledaction);
+#endif
diff --git a/drivers/staging/rtlwifi/rtl8822be/phy.c b/drivers/staging/rtlwifi/rtl8822be/phy.c
new file mode 100644
index 000000000000..4cba2adc3165
--- /dev/null
+++ b/drivers/staging/rtlwifi/rtl8822be/phy.c
@@ -0,0 +1,2233 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#include "../wifi.h"
+#include "../pci.h"
+#include "../ps.h"
+#include "../base.h"
+#include "reg.h"
+#include "def.h"
+#include "phy.h"
+#include "trx.h"
+#include "../btcoexist/halbt_precomp.h"
+#include "hw.h"
+#include "../efuse.h"
+
+static u32 _rtl8822be_phy_calculate_bit_shift(u32 bitmask);
+static void
+_rtl8822be_phy_init_bb_rf_register_definition(struct ieee80211_hw *hw);
+
+static long _rtl8822be_phy_txpwr_idx_to_dbm(struct ieee80211_hw *hw,
+					    enum wireless_mode wirelessmode,
+					    u8 txpwridx);
+static void rtl8822be_phy_set_rf_on(struct ieee80211_hw *hw);
+static void rtl8822be_phy_set_io(struct ieee80211_hw *hw);
+
+static u8 cck_rates[] = {DESC_RATE1M, DESC_RATE2M, DESC_RATE5_5M, DESC_RATE11M};
+static u8 sizes_of_cck_retes = 4;
+static u8 ofdm_rates[] = {DESC_RATE6M,  DESC_RATE9M,  DESC_RATE12M,
+			  DESC_RATE18M, DESC_RATE24M, DESC_RATE36M,
+			  DESC_RATE48M, DESC_RATE54M};
+static u8 sizes_of_ofdm_retes = 8;
+static u8 ht_rates_1t[] = {DESC_RATEMCS0, DESC_RATEMCS1, DESC_RATEMCS2,
+			   DESC_RATEMCS3, DESC_RATEMCS4, DESC_RATEMCS5,
+			   DESC_RATEMCS6, DESC_RATEMCS7};
+static u8 sizes_of_ht_retes_1t = 8;
+static u8 ht_rates_2t[] = {DESC_RATEMCS8,  DESC_RATEMCS9,  DESC_RATEMCS10,
+			   DESC_RATEMCS11, DESC_RATEMCS12, DESC_RATEMCS13,
+			   DESC_RATEMCS14, DESC_RATEMCS15};
+static u8 sizes_of_ht_retes_2t = 8;
+static u8 vht_rates_1t[] = {DESC_RATEVHT1SS_MCS0, DESC_RATEVHT1SS_MCS1,
+			    DESC_RATEVHT1SS_MCS2, DESC_RATEVHT1SS_MCS3,
+			    DESC_RATEVHT1SS_MCS4, DESC_RATEVHT1SS_MCS5,
+			    DESC_RATEVHT1SS_MCS6, DESC_RATEVHT1SS_MCS7,
+			    DESC_RATEVHT1SS_MCS8, DESC_RATEVHT1SS_MCS9};
+static u8 vht_rates_2t[] = {DESC_RATEVHT2SS_MCS0, DESC_RATEVHT2SS_MCS1,
+			    DESC_RATEVHT2SS_MCS2, DESC_RATEVHT2SS_MCS3,
+			    DESC_RATEVHT2SS_MCS4, DESC_RATEVHT2SS_MCS5,
+			    DESC_RATEVHT2SS_MCS6, DESC_RATEVHT2SS_MCS7,
+			    DESC_RATEVHT2SS_MCS8, DESC_RATEVHT2SS_MCS9};
+static u8 sizes_of_vht_retes = 10;
+
+u32 rtl8822be_phy_query_bb_reg(struct ieee80211_hw *hw, u32 regaddr,
+			       u32 bitmask)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u32 returnvalue, originalvalue, bitshift;
+
+	RT_TRACE(rtlpriv, COMP_RF, DBG_TRACE, "regaddr(%#x), bitmask(%#x)\n",
+		 regaddr, bitmask);
+	originalvalue = rtl_read_dword(rtlpriv, regaddr);
+	bitshift = _rtl8822be_phy_calculate_bit_shift(bitmask);
+	returnvalue = (originalvalue & bitmask) >> bitshift;
+
+	RT_TRACE(rtlpriv, COMP_RF, DBG_TRACE, "BBR MASK=0x%x Addr[0x%x]=0x%x\n",
+		 bitmask, regaddr, originalvalue);
+
+	return returnvalue;
+}
+
+void rtl8822be_phy_set_bb_reg(struct ieee80211_hw *hw, u32 regaddr, u32 bitmask,
+			      u32 data)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u32 originalvalue, bitshift;
+
+	RT_TRACE(rtlpriv, COMP_RF, DBG_TRACE,
+		 "regaddr(%#x), bitmask(%#x), data(%#x)\n", regaddr, bitmask,
+		 data);
+
+	if (bitmask != MASKDWORD) {
+		originalvalue = rtl_read_dword(rtlpriv, regaddr);
+		bitshift = _rtl8822be_phy_calculate_bit_shift(bitmask);
+		data = ((originalvalue & (~bitmask)) |
+			((data << bitshift) & bitmask));
+	}
+
+	rtl_write_dword(rtlpriv, regaddr, data);
+
+	RT_TRACE(rtlpriv, COMP_RF, DBG_TRACE,
+		 "regaddr(%#x), bitmask(%#x), data(%#x)\n", regaddr, bitmask,
+		 data);
+}
+
+u32 rtl8822be_phy_query_rf_reg(struct ieee80211_hw *hw, enum radio_path rfpath,
+			       u32 regaddr, u32 bitmask)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u32 /*original_value,*/ readback_value /*, bitshift*/;
+	unsigned long flags;
+
+	RT_TRACE(rtlpriv, COMP_RF, DBG_TRACE,
+		 "regaddr(%#x), rfpath(%#x), bitmask(%#x)\n", regaddr, rfpath,
+		 bitmask);
+
+	spin_lock_irqsave(&rtlpriv->locks.rf_lock, flags);
+
+	readback_value = rtlpriv->phydm.ops->phydm_read_rf_reg(
+		rtlpriv, rfpath, regaddr, bitmask);
+
+	spin_unlock_irqrestore(&rtlpriv->locks.rf_lock, flags);
+
+	return readback_value;
+}
+
+void rtl8822be_phy_set_rf_reg(struct ieee80211_hw *hw, enum radio_path rfpath,
+			      u32 regaddr, u32 bitmask, u32 data)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	unsigned long flags;
+
+	RT_TRACE(rtlpriv, COMP_RF, DBG_TRACE,
+		 "regaddr(%#x), bitmask(%#x), data(%#x), rfpath(%#x)\n",
+		 regaddr, bitmask, data, rfpath);
+
+	spin_lock_irqsave(&rtlpriv->locks.rf_lock, flags);
+
+	rtlpriv->phydm.ops->phydm_write_rf_reg(rtlpriv, rfpath, regaddr,
+					       bitmask, data);
+
+	spin_unlock_irqrestore(&rtlpriv->locks.rf_lock, flags);
+
+	RT_TRACE(rtlpriv, COMP_RF, DBG_TRACE,
+		 "regaddr(%#x), bitmask(%#x), data(%#x), rfpath(%#x)\n",
+		 regaddr, bitmask, data, rfpath);
+}
+
+static u32 _rtl8822be_phy_calculate_bit_shift(u32 bitmask)
+{
+	u32 i;
+
+	for (i = 0; i <= 31; i++) {
+		if (((bitmask >> i) & 0x1) == 1)
+			break;
+	}
+	return i;
+}
+
+bool rtl8822be_halmac_cb_init_mac_register(struct rtl_priv *rtlpriv)
+{
+	return rtlpriv->phydm.ops->phydm_phy_mac_config(rtlpriv);
+}
+
+bool rtl8822be_phy_bb_config(struct ieee80211_hw *hw)
+{
+	bool rtstatus = true;
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
+	u8 crystal_cap;
+	/* u32 tmp; */
+
+	rtstatus = rtlpriv->phydm.ops->phydm_phy_bb_config(rtlpriv);
+
+	/* write 0x28[6:1] = 0x24[30:25] = CrystalCap */
+	crystal_cap = rtlefuse->crystalcap & 0x3F;
+	rtl_set_bbreg(hw, REG_AFE_XTAL_CTRL_8822B, 0x7E000000, crystal_cap);
+	rtl_set_bbreg(hw, REG_AFE_PLL_CTRL_8822B, 0x7E, crystal_cap);
+
+	/*rtlphy->reg_837 = rtl_read_byte(rtlpriv, 0x837);*/ /*unused*/
+
+	return rtstatus;
+}
+
+bool rtl8822be_phy_rf_config(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+
+	if (rtlphy->rf_type == RF_1T1R)
+		rtlphy->num_total_rfpath = 1;
+	else
+		rtlphy->num_total_rfpath = 2;
+
+	return rtlpriv->phydm.ops->phydm_phy_rf_config(rtlpriv);
+}
+
+bool rtl8822be_halmac_cb_init_bb_rf_register(struct rtl_priv *rtlpriv)
+{
+	struct ieee80211_hw *hw = rtlpriv->hw;
+	enum radio_mask txpath, rxpath;
+	bool tx2path;
+	bool ret = false;
+
+	_rtl8822be_phy_init_bb_rf_register_definition(hw);
+
+	rtlpriv->halmac.ops->halmac_phy_power_switch(rtlpriv, 1);
+
+	/* beofre bb/rf config */
+	rtlpriv->phydm.ops->phydm_parameter_init(rtlpriv, 0);
+
+	/* do bb/rf config */
+	if (rtl8822be_phy_bb_config(hw) && rtl8822be_phy_rf_config(hw))
+		ret = true;
+
+	/* after bb/rf config */
+	rtlpriv->phydm.ops->phydm_parameter_init(rtlpriv, 1);
+
+	/* set trx mode (keep it to be last, r17376) */
+	txpath = RF_MASK_A | RF_MASK_B;
+	rxpath = RF_MASK_A | RF_MASK_B;
+	tx2path = false;
+	ret = rtlpriv->phydm.ops->phydm_trx_mode(rtlpriv, txpath, rxpath,
+						 tx2path);
+
+	return ret;
+}
+
+static void _rtl8822be_phy_init_tx_power_by_rate(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+
+	u8 band, rfpath, txnum, rate;
+
+	for (band = BAND_ON_2_4G; band <= BAND_ON_5G; ++band)
+		for (rfpath = 0; rfpath < TX_PWR_BY_RATE_NUM_RF; ++rfpath)
+			for (txnum = 0; txnum < TX_PWR_BY_RATE_NUM_RF; ++txnum)
+				for (rate = 0; rate < TX_PWR_BY_RATE_NUM_RATE;
+				     ++rate)
+					rtlphy->tx_power_by_rate_offset
+						[band][rfpath][txnum][rate] = 0;
+}
+
+static void _rtl8822be_phy_set_txpower_by_rate_base(struct ieee80211_hw *hw,
+						    u8 band, u8 path,
+						    u8 rate_section, u8 txnum,
+						    u8 value)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+
+	if (path > RF90_PATH_D) {
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+			 "Invalid Rf Path %d in phy_SetTxPowerByRatBase()\n",
+			 path);
+		return;
+	}
+
+	if (band != BAND_ON_2_4G && band != BAND_ON_5G) {
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+			 "Invalid band %d in phy_SetTxPowerByRatBase()\n",
+			 band);
+		return;
+	}
+
+	if (rate_section >= MAX_RATE_SECTION ||
+	    (band == BAND_ON_5G && rate_section == CCK)) {
+		RT_TRACE(
+			rtlpriv, COMP_INIT, DBG_LOUD,
+			"Invalid rate_section %d in phy_SetTxPowerByRatBase()\n",
+			rate_section);
+		return;
+	}
+
+	if (band == BAND_ON_2_4G)
+		rtlphy->txpwr_by_rate_base_24g[path][txnum][rate_section] =
+			value;
+	else /* BAND_ON_5G */
+		rtlphy->txpwr_by_rate_base_5g[path][txnum][rate_section - 1] =
+			value;
+}
+
+static u8 _rtl8822be_phy_get_txpower_by_rate_base(struct ieee80211_hw *hw,
+						  u8 band, u8 path, u8 txnum,
+						  u8 rate_section)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	u8 value;
+
+	if (path > RF90_PATH_D) {
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+			 "Invalid Rf Path %d in phy_GetTxPowerByRatBase()\n",
+			 path);
+		return 0;
+	}
+
+	if (band != BAND_ON_2_4G && band != BAND_ON_5G) {
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+			 "Invalid band %d in phy_GetTxPowerByRatBase()\n",
+			 band);
+		return 0;
+	}
+
+	if (rate_section >= MAX_RATE_SECTION ||
+	    (band == BAND_ON_5G && rate_section == CCK)) {
+		RT_TRACE(
+			rtlpriv, COMP_INIT, DBG_LOUD,
+			"Invalid rate_section %d in phy_GetTxPowerByRatBase()\n",
+			rate_section);
+		return 0;
+	}
+
+	if (band == BAND_ON_2_4G)
+		value = rtlphy->txpwr_by_rate_base_24g[path][txnum]
+						      [rate_section];
+	else /* BAND_ON_5G */
+		value = rtlphy->txpwr_by_rate_base_5g[path][txnum]
+						     [rate_section - 1];
+
+	return value;
+}
+
+static void _rtl8822be_phy_store_txpower_by_rate_base(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+
+	struct {
+		enum rtl_desc_rate rate;
+		enum rate_section section;
+	} rate_sec_base[] = {
+		{DESC_RATE11M, CCK},
+		{DESC_RATE54M, OFDM},
+		{DESC_RATEMCS7, HT_MCS0_MCS7},
+		{DESC_RATEMCS15, HT_MCS8_MCS15},
+		{DESC_RATEVHT1SS_MCS7, VHT_1SSMCS0_1SSMCS9},
+		{DESC_RATEVHT2SS_MCS7, VHT_2SSMCS0_2SSMCS9},
+	};
+
+	u8 band, path, rs, tx_num, base;
+	u8 rate, section;
+
+	for (band = BAND_ON_2_4G; band <= BAND_ON_5G; band++) {
+		for (path = RF90_PATH_A; path <= RF90_PATH_B; path++) {
+			for (rs = 0; rs < MAX_RATE_SECTION; rs++) {
+				rate = rate_sec_base[rs].rate;
+				section = rate_sec_base[rs].section;
+
+				if (IS_1T_RATE(rate))
+					tx_num = RF_1TX;
+				else
+					tx_num = RF_2TX;
+
+				if (band == BAND_ON_5G &&
+				    RX_HAL_IS_CCK_RATE(rate))
+					continue;
+
+				base = rtlphy->tx_power_by_rate_offset
+					       [band][path][tx_num][rate];
+				_rtl8822be_phy_set_txpower_by_rate_base(
+					hw, band, path, section, tx_num, base);
+			}
+		}
+	}
+}
+
+static void __rtl8822be_phy_cross_reference_core(struct ieee80211_hw *hw,
+						 u8 regulation, u8 bw,
+						 u8 channel)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	u8 rs, ref_rs;
+	s8 pwrlmt, ref_pwrlmt;
+
+	for (rs = 0; rs < MAX_RATE_SECTION_NUM; ++rs) {
+		/*5G 20M 40M VHT and HT can cross reference*/
+		if (bw != HT_CHANNEL_WIDTH_20 && bw != HT_CHANNEL_WIDTH_20_40)
+			continue;
+
+		if (rs == HT_MCS0_MCS7)
+			ref_rs = VHT_1SSMCS0_1SSMCS9;
+		else if (rs == HT_MCS8_MCS15)
+			ref_rs = VHT_2SSMCS0_2SSMCS9;
+		else if (rs == VHT_1SSMCS0_1SSMCS9)
+			ref_rs = HT_MCS0_MCS7;
+		else if (rs == VHT_2SSMCS0_2SSMCS9)
+			ref_rs = HT_MCS8_MCS15;
+		else
+			continue;
+
+		ref_pwrlmt = rtlphy->txpwr_limit_5g[regulation][bw][ref_rs]
+						   [channel][RF90_PATH_A];
+		if (ref_pwrlmt == MAX_POWER_INDEX)
+			continue;
+
+		pwrlmt = rtlphy->txpwr_limit_5g[regulation][bw][rs][channel]
+					       [RF90_PATH_A];
+		if (pwrlmt != MAX_POWER_INDEX)
+			continue;
+
+		rtlphy->txpwr_limit_5g[regulation][bw][rs][channel]
+				      [RF90_PATH_A] = ref_pwrlmt;
+	}
+}
+
+static void
+_rtl8822be_phy_cross_reference_ht_and_vht_txpower_limit(struct ieee80211_hw *hw)
+{
+	u8 regulation, bw, channel;
+
+	for (regulation = 0; regulation < MAX_REGULATION_NUM; ++regulation) {
+		for (bw = 0; bw < MAX_5G_BANDWIDTH_NUM; ++bw) {
+			for (channel = 0; channel < CHANNEL_MAX_NUMBER_5G;
+			     ++channel) {
+				__rtl8822be_phy_cross_reference_core(
+					hw, regulation, bw, channel);
+			}
+		}
+	}
+}
+
+static void __rtl8822be_txpwr_limit_to_index_2g(struct ieee80211_hw *hw,
+						u8 regulation, u8 bw,
+						u8 channel)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	u8 bw40_pwr_base_dbm2_4G;
+	u8 rate_section;
+	s8 temp_pwrlmt;
+	enum rf_tx_num txnum;
+	s8 temp_value;
+	u8 rf_path;
+
+	for (rate_section = 0; rate_section < MAX_RATE_SECTION_NUM;
+	     ++rate_section) {
+		/* obtain the base dBm values in 2.4G band
+		 * CCK => 11M, OFDM => 54M, HT 1T => MCS7, HT 2T => MCS15
+		 */
+
+		temp_pwrlmt =
+			rtlphy->txpwr_limit_2_4g[regulation][bw][rate_section]
+						[channel][RF90_PATH_A];
+		txnum = IS_1T_RATESEC(rate_section) ? RF_1TX : RF_2TX;
+
+		if (temp_pwrlmt == MAX_POWER_INDEX)
+			continue;
+
+		for (rf_path = RF90_PATH_A; rf_path < MAX_RF_PATH_NUM;
+		     ++rf_path) {
+			bw40_pwr_base_dbm2_4G =
+				_rtl8822be_phy_get_txpower_by_rate_base(
+					hw, BAND_ON_2_4G, rf_path, txnum,
+					rate_section);
+
+			temp_value = temp_pwrlmt - bw40_pwr_base_dbm2_4G;
+			rtlphy->txpwr_limit_2_4g[regulation][bw][rate_section]
+						[channel][rf_path] = temp_value;
+
+			RT_TRACE(
+				rtlpriv, COMP_INIT, DBG_TRACE,
+				"TxPwrLimit_2_4G[regulation %d][bw %d][rateSection %d][channel %d] = %d\n(TxPwrLimit in dBm %d - BW40PwrLmt2_4G[channel %d][rfPath %d] %d)\n",
+				regulation, bw, rate_section, channel,
+				rtlphy->txpwr_limit_2_4g[regulation][bw]
+							[rate_section][channel]
+							[rf_path],
+				(temp_pwrlmt == 63) ? 0 : temp_pwrlmt / 2,
+				channel, rf_path, bw40_pwr_base_dbm2_4G);
+		}
+	}
+}
+
+static void __rtl8822be_txpwr_limit_to_index_5g(struct ieee80211_hw *hw,
+						u8 regulation, u8 bw,
+						u8 channel)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	u8 bw40_pwr_base_dbm5G;
+	u8 rate_section;
+	s8 temp_pwrlmt;
+	enum rf_tx_num txnum;
+	s8 temp_value;
+	u8 rf_path;
+
+	for (rate_section = 0; rate_section < MAX_RATE_SECTION_NUM;
+	     ++rate_section) {
+		/* obtain the base dBm values in 5G band
+		 * OFDM => 54M, HT 1T => MCS7, HT 2T => MCS15,
+		 * VHT => 1SSMCS7, VHT 2T => 2SSMCS7
+		 */
+
+		temp_pwrlmt =
+			rtlphy->txpwr_limit_5g[regulation][bw][rate_section]
+					      [channel][RF90_PATH_A];
+		txnum = IS_1T_RATESEC(rate_section) ? RF_1TX : RF_2TX;
+
+		if (temp_pwrlmt == MAX_POWER_INDEX)
+			continue;
+
+		for (rf_path = RF90_PATH_A; rf_path < MAX_RF_PATH_NUM;
+		     ++rf_path) {
+			bw40_pwr_base_dbm5G =
+				_rtl8822be_phy_get_txpower_by_rate_base(
+					hw, BAND_ON_5G, rf_path, txnum,
+					rate_section);
+
+			temp_value = temp_pwrlmt - bw40_pwr_base_dbm5G;
+			rtlphy->txpwr_limit_5g[regulation][bw][rate_section]
+					      [channel][rf_path] = temp_value;
+
+			RT_TRACE(
+				rtlpriv, COMP_INIT, DBG_TRACE,
+				"TxPwrLimit_5G[regulation %d][bw %d][rateSection %d][channel %d] =%d\n(TxPwrLimit in dBm %d - BW40PwrLmt5G[chnl group %d][rfPath %d] %d)\n",
+				regulation, bw, rate_section, channel,
+				rtlphy->txpwr_limit_5g[regulation][bw]
+						      [rate_section][channel]
+						      [rf_path],
+				temp_pwrlmt, channel, rf_path,
+				bw40_pwr_base_dbm5G);
+		}
+	}
+}
+
+static void
+_rtl8822be_phy_convert_txpower_limit_to_power_index(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 regulation, bw, channel;
+
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE, "=====> %s()\n", __func__);
+
+	_rtl8822be_phy_cross_reference_ht_and_vht_txpower_limit(hw);
+
+	for (regulation = 0; regulation < MAX_REGULATION_NUM; ++regulation) {
+		for (bw = 0; bw < MAX_2_4G_BANDWIDTH_NUM; ++bw) {
+			for (channel = 0; channel < CHANNEL_MAX_NUMBER_2G;
+			     ++channel) {
+				__rtl8822be_txpwr_limit_to_index_2g(
+					hw, regulation, bw, channel);
+			}
+		}
+	}
+
+	for (regulation = 0; regulation < MAX_REGULATION_NUM; ++regulation) {
+		for (bw = 0; bw < MAX_5G_BANDWIDTH_NUM; ++bw) {
+			for (channel = 0; channel < CHANNEL_MAX_NUMBER_5G;
+			     ++channel) {
+				__rtl8822be_txpwr_limit_to_index_5g(
+					hw, regulation, bw, channel);
+			}
+		}
+	}
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE, "<===== %s()\n", __func__);
+}
+
+static void _rtl8822be_phy_init_txpower_limit(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	u8 i, j, k, l, m;
+
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD, "=====> %s()!\n", __func__);
+
+	for (i = 0; i < MAX_REGULATION_NUM; ++i) {
+		for (j = 0; j < MAX_2_4G_BANDWIDTH_NUM; ++j)
+			for (k = 0; k < MAX_RATE_SECTION_NUM; ++k)
+				for (m = 0; m < CHANNEL_MAX_NUMBER_2G; ++m)
+					for (l = 0; l < MAX_RF_PATH_NUM; ++l)
+						rtlphy->txpwr_limit_2_4g[i][j]
+									[k][m]
+									[l] =
+							MAX_POWER_INDEX;
+	}
+	for (i = 0; i < MAX_REGULATION_NUM; ++i) {
+		for (j = 0; j < MAX_5G_BANDWIDTH_NUM; ++j)
+			for (k = 0; k < MAX_RATE_SECTION_NUM; ++k)
+				for (m = 0; m < CHANNEL_MAX_NUMBER_5G; ++m)
+					for (l = 0; l < MAX_RF_PATH_NUM; ++l)
+						rtlphy->txpwr_limit_5g[i][j][k]
+								      [m][l] =
+							MAX_POWER_INDEX;
+	}
+
+	RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD, "<===== %s()!\n", __func__);
+}
+
+static void
+_rtl8822be_phy_convert_txpower_dbm_to_relative_value(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+
+	u8 base = 0, i = 0, value = 0, band = 0, path = 0, txnum = 0;
+
+	for (band = BAND_ON_2_4G; band <= BAND_ON_5G; ++band) {
+		for (path = RF90_PATH_A; path <= RF90_PATH_B; ++path) {
+			for (txnum = RF_1TX; txnum <= RF_2TX; ++txnum) {
+				/* CCK */
+				base = rtlphy->tx_power_by_rate_offset
+					       [band][path][txnum]
+					       [DESC_RATE11M];
+				for (i = 0; i < sizeof(cck_rates); ++i) {
+					value = rtlphy->tx_power_by_rate_offset
+							[band][path][txnum]
+							[cck_rates[i]];
+					rtlphy->tx_power_by_rate_offset
+						[band][path][txnum]
+						[cck_rates[i]] = value - base;
+				}
+
+				/* OFDM */
+				base = rtlphy->tx_power_by_rate_offset
+					       [band][path][txnum]
+					       [DESC_RATE54M];
+				for (i = 0; i < sizeof(ofdm_rates); ++i) {
+					value = rtlphy->tx_power_by_rate_offset
+							[band][path][txnum]
+							[ofdm_rates[i]];
+					rtlphy->tx_power_by_rate_offset
+						[band][path][txnum]
+						[ofdm_rates[i]] = value - base;
+				}
+
+				/* HT MCS0~7 */
+				base = rtlphy->tx_power_by_rate_offset
+					       [band][path][txnum]
+					       [DESC_RATEMCS7];
+				for (i = 0; i < sizeof(ht_rates_1t); ++i) {
+					value = rtlphy->tx_power_by_rate_offset
+							[band][path][txnum]
+							[ht_rates_1t[i]];
+					rtlphy->tx_power_by_rate_offset
+						[band][path][txnum]
+						[ht_rates_1t[i]] = value - base;
+				}
+
+				/* HT MCS8~15 */
+				base = rtlphy->tx_power_by_rate_offset
+					       [band][path][txnum]
+					       [DESC_RATEMCS15];
+				for (i = 0; i < sizeof(ht_rates_2t); ++i) {
+					value = rtlphy->tx_power_by_rate_offset
+							[band][path][txnum]
+							[ht_rates_2t[i]];
+					rtlphy->tx_power_by_rate_offset
+						[band][path][txnum]
+						[ht_rates_2t[i]] = value - base;
+				}
+
+				/* VHT 1SS */
+				base = rtlphy->tx_power_by_rate_offset
+					       [band][path][txnum]
+					       [DESC_RATEVHT1SS_MCS7];
+				for (i = 0; i < sizeof(vht_rates_1t); ++i) {
+					value = rtlphy->tx_power_by_rate_offset
+							[band][path][txnum]
+							[vht_rates_1t[i]];
+					rtlphy->tx_power_by_rate_offset
+						[band][path][txnum]
+						[vht_rates_1t[i]] =
+						value - base;
+				}
+
+				/* VHT 2SS */
+				base = rtlphy->tx_power_by_rate_offset
+					       [band][path][txnum]
+					       [DESC_RATEVHT2SS_MCS7];
+				for (i = 0; i < sizeof(vht_rates_2t); ++i) {
+					value = rtlphy->tx_power_by_rate_offset
+							[band][path][txnum]
+							[vht_rates_2t[i]];
+					rtlphy->tx_power_by_rate_offset
+						[band][path][txnum]
+						[vht_rates_2t[i]] =
+						value - base;
+				}
+			}
+		}
+	}
+
+	RT_TRACE(rtlpriv, COMP_POWER, DBG_TRACE, "<===%s()\n", __func__);
+}
+
+static void
+_rtl8822be_phy_txpower_by_rate_configuration(struct ieee80211_hw *hw)
+{
+	/* copy rate_section from
+	 * tx_power_by_rate_offset[][rate] to txpwr_by_rate_base_24g/_5g[][rs]
+	 */
+	_rtl8822be_phy_store_txpower_by_rate_base(hw);
+
+	/* convert tx_power_by_rate_offset[] to relative value */
+	_rtl8822be_phy_convert_txpower_dbm_to_relative_value(hw);
+}
+
+/* string is in decimal */
+static bool _rtl8822be_get_integer_from_string(char *str, u8 *pint)
+{
+	u16 i = 0;
+	*pint = 0;
+
+	while (str[i] != '\0') {
+		if (str[i] >= '0' && str[i] <= '9') {
+			*pint *= 10;
+			*pint += (str[i] - '0');
+		} else {
+			return false;
+		}
+		++i;
+	}
+
+	return true;
+}
+
+static bool _rtl8822be_eq_n_byte(u8 *str1, u8 *str2, u32 num)
+{
+	if (num == 0)
+		return false;
+	while (num > 0) {
+		num--;
+		if (str1[num] != str2[num])
+			return false;
+	}
+	return true;
+}
+
+static char _rtl8822be_phy_get_chnl_idx_of_txpwr_lmt(struct ieee80211_hw *hw,
+						     u8 band, u8 channel)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	char channel_index = -1;
+	u8 i = 0;
+
+	if (band == BAND_ON_2_4G) {
+		channel_index = channel - 1;
+	} else if (band == BAND_ON_5G) {
+		for (i = 0; i < sizeof(rtl_channel5g) / sizeof(u8); ++i) {
+			if (rtl_channel5g[i] == channel)
+				channel_index = i;
+		}
+	} else {
+		RT_TRACE(rtlpriv, COMP_POWER, DBG_LOUD, "Invalid Band %d in %s",
+			 band, __func__);
+	}
+
+	if (channel_index == -1)
+		RT_TRACE(rtlpriv, COMP_POWER, DBG_LOUD,
+			 "Invalid Channel %d of Band %d in %s", channel, band,
+			 __func__);
+
+	return channel_index;
+}
+
+void rtl8822be_phy_set_txpower_limit(struct ieee80211_hw *hw, u8 *pregulation,
+				     u8 *pband, u8 *pbandwidth,
+				     u8 *prate_section, u8 *prf_path,
+				     u8 *pchannel, u8 *ppower_limit)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	u8 regulation = 0, bandwidth = 0, rate_section = 0, channel;
+	u8 channel_index;
+	char power_limit = 0, prev_power_limit, ret;
+
+	if (!_rtl8822be_get_integer_from_string((char *)pchannel, &channel) ||
+	    !_rtl8822be_get_integer_from_string((char *)ppower_limit,
+						&power_limit)) {
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE,
+			 "Illegal index of pwr_lmt table [chnl %d][val %d]\n",
+			 channel, power_limit);
+	}
+
+	power_limit =
+		power_limit > MAX_POWER_INDEX ? MAX_POWER_INDEX : power_limit;
+
+	if (_rtl8822be_eq_n_byte(pregulation, (u8 *)("FCC"), 3))
+		regulation = 0;
+	else if (_rtl8822be_eq_n_byte(pregulation, (u8 *)("MKK"), 3))
+		regulation = 1;
+	else if (_rtl8822be_eq_n_byte(pregulation, (u8 *)("ETSI"), 4))
+		regulation = 2;
+	else if (_rtl8822be_eq_n_byte(pregulation, (u8 *)("WW13"), 4))
+		regulation = 3;
+
+	if (_rtl8822be_eq_n_byte(prate_section, (u8 *)("CCK"), 3))
+		rate_section = CCK;
+	else if (_rtl8822be_eq_n_byte(prate_section, (u8 *)("OFDM"), 4))
+		rate_section = OFDM;
+	else if (_rtl8822be_eq_n_byte(prate_section, (u8 *)("HT"), 2) &&
+		 _rtl8822be_eq_n_byte(prf_path, (u8 *)("1T"), 2))
+		rate_section = HT_MCS0_MCS7;
+	else if (_rtl8822be_eq_n_byte(prate_section, (u8 *)("HT"), 2) &&
+		 _rtl8822be_eq_n_byte(prf_path, (u8 *)("2T"), 2))
+		rate_section = HT_MCS8_MCS15;
+	else if (_rtl8822be_eq_n_byte(prate_section, (u8 *)("VHT"), 3) &&
+		 _rtl8822be_eq_n_byte(prf_path, (u8 *)("1T"), 2))
+		rate_section = VHT_1SSMCS0_1SSMCS9;
+	else if (_rtl8822be_eq_n_byte(prate_section, (u8 *)("VHT"), 3) &&
+		 _rtl8822be_eq_n_byte(prf_path, (u8 *)("2T"), 2))
+		rate_section = VHT_2SSMCS0_2SSMCS9;
+
+	if (_rtl8822be_eq_n_byte(pbandwidth, (u8 *)("20M"), 3))
+		bandwidth = HT_CHANNEL_WIDTH_20;
+	else if (_rtl8822be_eq_n_byte(pbandwidth, (u8 *)("40M"), 3))
+		bandwidth = HT_CHANNEL_WIDTH_20_40;
+	else if (_rtl8822be_eq_n_byte(pbandwidth, (u8 *)("80M"), 3))
+		bandwidth = HT_CHANNEL_WIDTH_80;
+	else if (_rtl8822be_eq_n_byte(pbandwidth, (u8 *)("160M"), 4))
+		bandwidth = 3;
+
+	if (_rtl8822be_eq_n_byte(pband, (u8 *)("2.4G"), 4)) {
+		ret = _rtl8822be_phy_get_chnl_idx_of_txpwr_lmt(hw, BAND_ON_2_4G,
+							       channel);
+
+		if (ret == -1)
+			return;
+
+		channel_index = ret;
+
+		prev_power_limit =
+			rtlphy->txpwr_limit_2_4g[regulation][bandwidth]
+						[rate_section][channel_index]
+						[RF90_PATH_A];
+
+		if (power_limit < prev_power_limit)
+			rtlphy->txpwr_limit_2_4g[regulation][bandwidth]
+						[rate_section][channel_index]
+						[RF90_PATH_A] = power_limit;
+
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE,
+			 "2.4G [regula %d][bw %d][sec %d][chnl %d][val %d]\n",
+			 regulation, bandwidth, rate_section, channel_index,
+			 rtlphy->txpwr_limit_2_4g[regulation][bandwidth]
+						 [rate_section][channel_index]
+						 [RF90_PATH_A]);
+	} else if (_rtl8822be_eq_n_byte(pband, (u8 *)("5G"), 2)) {
+		ret = _rtl8822be_phy_get_chnl_idx_of_txpwr_lmt(hw, BAND_ON_5G,
+							       channel);
+
+		if (ret == -1)
+			return;
+
+		channel_index = ret;
+
+		prev_power_limit =
+			rtlphy->txpwr_limit_5g[regulation][bandwidth]
+					      [rate_section][channel_index]
+					      [RF90_PATH_A];
+
+		if (power_limit < prev_power_limit)
+			rtlphy->txpwr_limit_5g[regulation][bandwidth]
+					      [rate_section][channel_index]
+					      [RF90_PATH_A] = power_limit;
+
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE,
+			 "5G: [regul %d][bw %d][sec %d][chnl %d][val %d]\n",
+			 regulation, bandwidth, rate_section, channel,
+			 rtlphy->txpwr_limit_5g[regulation][bandwidth]
+					       [rate_section][channel_index]
+					       [RF90_PATH_A]);
+
+	} else {
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE,
+			 "Cannot recognize the band info in %s\n", pband);
+		return;
+	}
+}
+
+bool rtl8822be_load_txpower_by_rate(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	bool rtstatus = true;
+
+	_rtl8822be_phy_init_tx_power_by_rate(hw);
+
+	rtstatus = rtlpriv->phydm.ops->phydm_load_txpower_by_rate(rtlpriv);
+
+	if (!rtstatus) {
+		pr_err("BB_PG Reg Fail!!");
+		return false;
+	}
+
+	_rtl8822be_phy_txpower_by_rate_configuration(hw);
+
+	return true;
+}
+
+bool rtl8822be_load_txpower_limit(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtlpriv);
+	bool rtstatus = true;
+
+	_rtl8822be_phy_init_txpower_limit(hw);
+
+	if (rtlefuse->eeprom_regulatory == 1)
+		;
+	else
+		return true;
+
+	rtstatus = rtlpriv->phydm.ops->phydm_load_txpower_limit(rtlpriv);
+
+	if (!rtstatus) {
+		pr_err("RF TxPwr Limit Fail!!");
+		return false;
+	}
+
+	_rtl8822be_phy_convert_txpower_limit_to_power_index(hw);
+
+	return true;
+}
+
+static void _rtl8822be_get_rate_values_of_tx_power_by_rate(
+	struct ieee80211_hw *hw, u32 reg_addr, u32 bit_mask, u32 value,
+	u8 *rate, s8 *pwr_by_rate_val, u8 *rate_num)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 /*index = 0,*/ i = 0;
+
+	switch (reg_addr) {
+	case 0xE00: /*rTxAGC_A_Rate18_06:*/
+	case 0x830: /*rTxAGC_B_Rate18_06:*/
+		rate[0] = DESC_RATE6M;
+		rate[1] = DESC_RATE9M;
+		rate[2] = DESC_RATE12M;
+		rate[3] = DESC_RATE18M;
+		for (i = 0; i < 4; ++i) {
+			pwr_by_rate_val[i] =
+				(s8)((((value >> (i * 8 + 4)) & 0xF)) * 10 +
+				     ((value >> (i * 8)) & 0xF));
+		}
+		*rate_num = 4;
+		break;
+
+	case 0xE04: /*rTxAGC_A_Rate54_24:*/
+	case 0x834: /*rTxAGC_B_Rate54_24:*/
+		rate[0] = DESC_RATE24M;
+		rate[1] = DESC_RATE36M;
+		rate[2] = DESC_RATE48M;
+		rate[3] = DESC_RATE54M;
+		for (i = 0; i < 4; ++i) {
+			pwr_by_rate_val[i] =
+				(s8)((((value >> (i * 8 + 4)) & 0xF)) * 10 +
+				     ((value >> (i * 8)) & 0xF));
+		}
+		*rate_num = 4;
+		break;
+
+	case 0xE08: /*rTxAGC_A_CCK1_Mcs32:*/
+		rate[0] = DESC_RATE1M;
+		pwr_by_rate_val[0] = (s8)((((value >> (8 + 4)) & 0xF)) * 10 +
+					  ((value >> 8) & 0xF));
+		*rate_num = 1;
+		break;
+
+	case 0x86C: /*rTxAGC_B_CCK11_A_CCK2_11:*/
+		if (bit_mask == 0xffffff00) {
+			rate[0] = DESC_RATE2M;
+			rate[1] = DESC_RATE5_5M;
+			rate[2] = DESC_RATE11M;
+			for (i = 1; i < 4; ++i) {
+				pwr_by_rate_val[i - 1] = (s8)(
+					(((value >> (i * 8 + 4)) & 0xF)) * 10 +
+					((value >> (i * 8)) & 0xF));
+			}
+			*rate_num = 3;
+		} else if (bit_mask == 0x000000ff) {
+			rate[0] = DESC_RATE11M;
+			pwr_by_rate_val[0] = (s8)((((value >> 4) & 0xF)) * 10 +
+						  (value & 0xF));
+			*rate_num = 1;
+		}
+		break;
+
+	case 0xE10: /*rTxAGC_A_Mcs03_Mcs00:*/
+	case 0x83C: /*rTxAGC_B_Mcs03_Mcs00:*/
+		rate[0] = DESC_RATEMCS0;
+		rate[1] = DESC_RATEMCS1;
+		rate[2] = DESC_RATEMCS2;
+		rate[3] = DESC_RATEMCS3;
+		for (i = 0; i < 4; ++i) {
+			pwr_by_rate_val[i] =
+				(s8)((((value >> (i * 8 + 4)) & 0xF)) * 10 +
+				     ((value >> (i * 8)) & 0xF));
+		}
+		*rate_num = 4;
+		break;
+
+	case 0xE14: /*rTxAGC_A_Mcs07_Mcs04:*/
+	case 0x848: /*rTxAGC_B_Mcs07_Mcs04:*/
+		rate[0] = DESC_RATEMCS4;
+		rate[1] = DESC_RATEMCS5;
+		rate[2] = DESC_RATEMCS6;
+		rate[3] = DESC_RATEMCS7;
+		for (i = 0; i < 4; ++i) {
+			pwr_by_rate_val[i] =
+				(s8)((((value >> (i * 8 + 4)) & 0xF)) * 10 +
+				     ((value >> (i * 8)) & 0xF));
+		}
+		*rate_num = 4;
+		break;
+
+	case 0xE18: /*rTxAGC_A_Mcs11_Mcs08:*/
+	case 0x84C: /*rTxAGC_B_Mcs11_Mcs08:*/
+		rate[0] = DESC_RATEMCS8;
+		rate[1] = DESC_RATEMCS9;
+		rate[2] = DESC_RATEMCS10;
+		rate[3] = DESC_RATEMCS11;
+		for (i = 0; i < 4; ++i) {
+			pwr_by_rate_val[i] =
+				(s8)((((value >> (i * 8 + 4)) & 0xF)) * 10 +
+				     ((value >> (i * 8)) & 0xF));
+		}
+		*rate_num = 4;
+		break;
+
+	case 0xE1C: /*rTxAGC_A_Mcs15_Mcs12:*/
+	case 0x868: /*rTxAGC_B_Mcs15_Mcs12:*/
+		rate[0] = DESC_RATEMCS12;
+		rate[1] = DESC_RATEMCS13;
+		rate[2] = DESC_RATEMCS14;
+		rate[3] = DESC_RATEMCS15;
+		for (i = 0; i < 4; ++i) {
+			pwr_by_rate_val[i] =
+				(s8)((((value >> (i * 8 + 4)) & 0xF)) * 10 +
+				     ((value >> (i * 8)) & 0xF));
+		}
+		*rate_num = 4;
+
+		break;
+
+	case 0x838: /*rTxAGC_B_CCK1_55_Mcs32:*/
+		rate[0] = DESC_RATE1M;
+		rate[1] = DESC_RATE2M;
+		rate[2] = DESC_RATE5_5M;
+		for (i = 1; i < 4; ++i) {
+			pwr_by_rate_val[i - 1] =
+				(s8)((((value >> (i * 8 + 4)) & 0xF)) * 10 +
+				     ((value >> (i * 8)) & 0xF));
+		}
+		*rate_num = 3;
+		break;
+
+	case 0xC20:
+	case 0xE20:
+	case 0x1820:
+	case 0x1a20:
+		rate[0] = DESC_RATE1M;
+		rate[1] = DESC_RATE2M;
+		rate[2] = DESC_RATE5_5M;
+		rate[3] = DESC_RATE11M;
+		for (i = 0; i < 4; ++i) {
+			pwr_by_rate_val[i] =
+				(s8)((((value >> (i * 8 + 4)) & 0xF)) * 10 +
+				     ((value >> (i * 8)) & 0xF));
+		}
+		*rate_num = 4;
+		break;
+
+	case 0xC24:
+	case 0xE24:
+	case 0x1824:
+	case 0x1a24:
+		rate[0] = DESC_RATE6M;
+		rate[1] = DESC_RATE9M;
+		rate[2] = DESC_RATE12M;
+		rate[3] = DESC_RATE18M;
+		for (i = 0; i < 4; ++i) {
+			pwr_by_rate_val[i] =
+				(s8)((((value >> (i * 8 + 4)) & 0xF)) * 10 +
+				     ((value >> (i * 8)) & 0xF));
+		}
+		*rate_num = 4;
+		break;
+
+	case 0xC28:
+	case 0xE28:
+	case 0x1828:
+	case 0x1a28:
+		rate[0] = DESC_RATE24M;
+		rate[1] = DESC_RATE36M;
+		rate[2] = DESC_RATE48M;
+		rate[3] = DESC_RATE54M;
+		for (i = 0; i < 4; ++i) {
+			pwr_by_rate_val[i] =
+				(s8)((((value >> (i * 8 + 4)) & 0xF)) * 10 +
+				     ((value >> (i * 8)) & 0xF));
+		}
+		*rate_num = 4;
+		break;
+
+	case 0xC2C:
+	case 0xE2C:
+	case 0x182C:
+	case 0x1a2C:
+		rate[0] = DESC_RATEMCS0;
+		rate[1] = DESC_RATEMCS1;
+		rate[2] = DESC_RATEMCS2;
+		rate[3] = DESC_RATEMCS3;
+		for (i = 0; i < 4; ++i) {
+			pwr_by_rate_val[i] =
+				(s8)((((value >> (i * 8 + 4)) & 0xF)) * 10 +
+				     ((value >> (i * 8)) & 0xF));
+		}
+		*rate_num = 4;
+		break;
+
+	case 0xC30:
+	case 0xE30:
+	case 0x1830:
+	case 0x1a30:
+		rate[0] = DESC_RATEMCS4;
+		rate[1] = DESC_RATEMCS5;
+		rate[2] = DESC_RATEMCS6;
+		rate[3] = DESC_RATEMCS7;
+		for (i = 0; i < 4; ++i) {
+			pwr_by_rate_val[i] =
+				(s8)((((value >> (i * 8 + 4)) & 0xF)) * 10 +
+				     ((value >> (i * 8)) & 0xF));
+		}
+		*rate_num = 4;
+		break;
+
+	case 0xC34:
+	case 0xE34:
+	case 0x1834:
+	case 0x1a34:
+		rate[0] = DESC_RATEMCS8;
+		rate[1] = DESC_RATEMCS9;
+		rate[2] = DESC_RATEMCS10;
+		rate[3] = DESC_RATEMCS11;
+		for (i = 0; i < 4; ++i) {
+			pwr_by_rate_val[i] =
+				(s8)((((value >> (i * 8 + 4)) & 0xF)) * 10 +
+				     ((value >> (i * 8)) & 0xF));
+		}
+		*rate_num = 4;
+		break;
+
+	case 0xC38:
+	case 0xE38:
+	case 0x1838:
+	case 0x1a38:
+		rate[0] = DESC_RATEMCS12;
+		rate[1] = DESC_RATEMCS13;
+		rate[2] = DESC_RATEMCS14;
+		rate[3] = DESC_RATEMCS15;
+		for (i = 0; i < 4; ++i) {
+			pwr_by_rate_val[i] =
+				(s8)((((value >> (i * 8 + 4)) & 0xF)) * 10 +
+				     ((value >> (i * 8)) & 0xF));
+		}
+		*rate_num = 4;
+		break;
+
+	case 0xC3C:
+	case 0xE3C:
+	case 0x183C:
+	case 0x1a3C:
+		rate[0] = DESC_RATEVHT1SS_MCS0;
+		rate[1] = DESC_RATEVHT1SS_MCS1;
+		rate[2] = DESC_RATEVHT1SS_MCS2;
+		rate[3] = DESC_RATEVHT1SS_MCS3;
+		for (i = 0; i < 4; ++i) {
+			pwr_by_rate_val[i] =
+				(s8)((((value >> (i * 8 + 4)) & 0xF)) * 10 +
+				     ((value >> (i * 8)) & 0xF));
+		}
+		*rate_num = 4;
+		break;
+
+	case 0xC40:
+	case 0xE40:
+	case 0x1840:
+	case 0x1a40:
+		rate[0] = DESC_RATEVHT1SS_MCS4;
+		rate[1] = DESC_RATEVHT1SS_MCS5;
+		rate[2] = DESC_RATEVHT1SS_MCS6;
+		rate[3] = DESC_RATEVHT1SS_MCS7;
+		for (i = 0; i < 4; ++i) {
+			pwr_by_rate_val[i] =
+				(s8)((((value >> (i * 8 + 4)) & 0xF)) * 10 +
+				     ((value >> (i * 8)) & 0xF));
+		}
+		*rate_num = 4;
+		break;
+
+	case 0xC44:
+	case 0xE44:
+	case 0x1844:
+	case 0x1a44:
+		rate[0] = DESC_RATEVHT1SS_MCS8;
+		rate[1] = DESC_RATEVHT1SS_MCS9;
+		rate[2] = DESC_RATEVHT2SS_MCS0;
+		rate[3] = DESC_RATEVHT2SS_MCS1;
+		for (i = 0; i < 4; ++i) {
+			pwr_by_rate_val[i] =
+				(s8)((((value >> (i * 8 + 4)) & 0xF)) * 10 +
+				     ((value >> (i * 8)) & 0xF));
+		}
+		*rate_num = 4;
+		break;
+
+	case 0xC48:
+	case 0xE48:
+	case 0x1848:
+	case 0x1a48:
+		rate[0] = DESC_RATEVHT2SS_MCS2;
+		rate[1] = DESC_RATEVHT2SS_MCS3;
+		rate[2] = DESC_RATEVHT2SS_MCS4;
+		rate[3] = DESC_RATEVHT2SS_MCS5;
+		for (i = 0; i < 4; ++i) {
+			pwr_by_rate_val[i] =
+				(s8)((((value >> (i * 8 + 4)) & 0xF)) * 10 +
+				     ((value >> (i * 8)) & 0xF));
+		}
+		*rate_num = 4;
+		break;
+
+	case 0xC4C:
+	case 0xE4C:
+	case 0x184C:
+	case 0x1a4C:
+		rate[0] = DESC_RATEVHT2SS_MCS6;
+		rate[1] = DESC_RATEVHT2SS_MCS7;
+		rate[2] = DESC_RATEVHT2SS_MCS8;
+		rate[3] = DESC_RATEVHT2SS_MCS9;
+		for (i = 0; i < 4; ++i) {
+			pwr_by_rate_val[i] =
+				(s8)((((value >> (i * 8 + 4)) & 0xF)) * 10 +
+				     ((value >> (i * 8)) & 0xF));
+		}
+		*rate_num = 4;
+		break;
+
+	default:
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
+			 "Invalid reg_addr 0x%x in %s()\n", reg_addr, __func__);
+		break;
+	};
+}
+
+void rtl8822be_store_tx_power_by_rate(struct ieee80211_hw *hw, u32 band,
+				      u32 rfpath, u32 txnum, u32 regaddr,
+				      u32 bitmask, u32 data)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	u8 i = 0, rates[4] = {0}, rate_num = 0;
+	s8 pwr_by_rate_val[4] = {0};
+
+	_rtl8822be_get_rate_values_of_tx_power_by_rate(
+		hw, regaddr, bitmask, data, rates, pwr_by_rate_val, &rate_num);
+
+	if (band != BAND_ON_2_4G && band != BAND_ON_5G) {
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_WARNING, "Invalid Band %d\n",
+			 band);
+		band = BAND_ON_2_4G;
+	}
+	if (rfpath >= MAX_RF_PATH) {
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_WARNING, "Invalid RfPath %d\n",
+			 rfpath);
+		rfpath = MAX_RF_PATH - 1;
+	}
+	if (txnum >= MAX_RF_PATH) {
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_WARNING, "Invalid TxNum %d\n",
+			 txnum);
+		txnum = MAX_RF_PATH - 1;
+	}
+
+	for (i = 0; i < rate_num; ++i) {
+		u8 rate_idx = rates[i];
+
+		if (IS_1T_RATE(rates[i]))
+			txnum = RF_1TX;
+		else if (IS_2T_RATE(rates[i]))
+			txnum = RF_2TX;
+		else
+			WARN_ON(1);
+
+		rtlphy->tx_power_by_rate_offset[band][rfpath][txnum][rate_idx] =
+			pwr_by_rate_val[i];
+
+		RT_TRACE(
+			rtlpriv, COMP_INIT, DBG_LOUD,
+			"TxPwrByRateOffset[Band %d][RfPath %d][TxNum %d][rate_idx %d] = 0x%x\n",
+			band, rfpath, txnum, rate_idx,
+			rtlphy->tx_power_by_rate_offset[band][rfpath][txnum]
+						       [rate_idx]);
+	}
+}
+
+static void
+_rtl8822be_phy_init_bb_rf_register_definition(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+
+	rtlphy->phyreg_def[RF90_PATH_A].rfintfs = RFPGA0_XAB_RFINTERFACESW;
+	rtlphy->phyreg_def[RF90_PATH_B].rfintfs = RFPGA0_XAB_RFINTERFACESW;
+
+	rtlphy->phyreg_def[RF90_PATH_A].rfintfo = RFPGA0_XA_RFINTERFACEOE;
+	rtlphy->phyreg_def[RF90_PATH_B].rfintfo = RFPGA0_XB_RFINTERFACEOE;
+
+	rtlphy->phyreg_def[RF90_PATH_A].rfintfe = RFPGA0_XA_RFINTERFACEOE;
+	rtlphy->phyreg_def[RF90_PATH_B].rfintfe = RFPGA0_XB_RFINTERFACEOE;
+
+	rtlphy->phyreg_def[RF90_PATH_A].rf3wire_offset = RA_LSSIWRITE_8822B;
+	rtlphy->phyreg_def[RF90_PATH_B].rf3wire_offset = RB_LSSIWRITE_8822B;
+
+	rtlphy->phyreg_def[RF90_PATH_A].rfhssi_para2 = RHSSIREAD_8822BE;
+	rtlphy->phyreg_def[RF90_PATH_B].rfhssi_para2 = RHSSIREAD_8822BE;
+
+	rtlphy->phyreg_def[RF90_PATH_A].rf_rb = RA_SIREAD_8822B;
+	rtlphy->phyreg_def[RF90_PATH_B].rf_rb = RB_SIREAD_8822B;
+
+	rtlphy->phyreg_def[RF90_PATH_A].rf_rbpi = RA_PIREAD_8822B;
+	rtlphy->phyreg_def[RF90_PATH_B].rf_rbpi = RB_PIREAD_8822B;
+}
+
+void rtl8822be_phy_get_txpower_level(struct ieee80211_hw *hw, long *powerlevel)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	u8 txpwr_level;
+	long txpwr_dbm;
+
+	txpwr_level = rtlphy->cur_cck_txpwridx;
+	txpwr_dbm = _rtl8822be_phy_txpwr_idx_to_dbm(hw, WIRELESS_MODE_B,
+						    txpwr_level);
+	txpwr_level = rtlphy->cur_ofdm24g_txpwridx;
+	if (_rtl8822be_phy_txpwr_idx_to_dbm(hw, WIRELESS_MODE_G, txpwr_level) >
+	    txpwr_dbm)
+		txpwr_dbm = _rtl8822be_phy_txpwr_idx_to_dbm(hw, WIRELESS_MODE_G,
+							    txpwr_level);
+	txpwr_level = rtlphy->cur_ofdm24g_txpwridx;
+	if (_rtl8822be_phy_txpwr_idx_to_dbm(hw, WIRELESS_MODE_N_24G,
+					    txpwr_level) > txpwr_dbm)
+		txpwr_dbm = _rtl8822be_phy_txpwr_idx_to_dbm(
+			hw, WIRELESS_MODE_N_24G, txpwr_level);
+	*powerlevel = txpwr_dbm;
+}
+
+static bool _rtl8822be_phy_get_chnl_index(u8 channel, u8 *chnl_index)
+{
+	u8 rtl_channel5g[CHANNEL_MAX_NUMBER_5G] = {
+		36,  38,  40,  42,  44,  46,  48, /* Band 1 */
+		52,  54,  56,  58,  60,  62,  64, /* Band 2 */
+		100, 102, 104, 106, 108, 110, 112, /* Band 3 */
+		116, 118, 120, 122, 124, 126, 128, /* Band 3 */
+		132, 134, 136, 138, 140, 142, 144, /* Band 3 */
+		149, 151, 153, 155, 157, 159, 161, /* Band 4 */
+		165, 167, 169, 171, 173, 175, 177}; /* Band 4 */
+	u8 i = 0;
+	bool in_24g = true;
+
+	if (channel <= 14) {
+		in_24g = true;
+		*chnl_index = channel - 1;
+	} else {
+		in_24g = false;
+
+		for (i = 0; i < CHANNEL_MAX_NUMBER_5G; ++i) {
+			if (rtl_channel5g[i] == channel) {
+				*chnl_index = i;
+				return in_24g;
+			}
+		}
+	}
+	return in_24g;
+}
+
+static char _rtl8822be_phy_get_world_wide_limit(char *limit_table)
+{
+	char min = limit_table[0];
+	u8 i = 0;
+
+	for (i = 0; i < MAX_REGULATION_NUM; ++i) {
+		if (limit_table[i] < min)
+			min = limit_table[i];
+	}
+	return min;
+}
+
+static char _rtl8822be_phy_get_txpower_limit(struct ieee80211_hw *hw, u8 band,
+					     enum ht_channel_width bandwidth,
+					     enum radio_path rf_path, u8 rate,
+					     u8 channel)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtlpriv);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	short regulation = -1, rate_section = -1, channel_index = -1;
+	char power_limit = MAX_POWER_INDEX;
+
+	if (rtlefuse->eeprom_regulatory == 2)
+		return MAX_POWER_INDEX;
+
+	regulation = TXPWR_LMT_WW;
+
+	switch (rate) {
+	case DESC_RATE1M:
+	case DESC_RATE2M:
+	case DESC_RATE5_5M:
+	case DESC_RATE11M:
+		rate_section = CCK;
+		break;
+
+	case DESC_RATE6M:
+	case DESC_RATE9M:
+	case DESC_RATE12M:
+	case DESC_RATE18M:
+	case DESC_RATE24M:
+	case DESC_RATE36M:
+	case DESC_RATE48M:
+	case DESC_RATE54M:
+		rate_section = OFDM;
+		break;
+
+	case DESC_RATEMCS0:
+	case DESC_RATEMCS1:
+	case DESC_RATEMCS2:
+	case DESC_RATEMCS3:
+	case DESC_RATEMCS4:
+	case DESC_RATEMCS5:
+	case DESC_RATEMCS6:
+	case DESC_RATEMCS7:
+		rate_section = HT_MCS0_MCS7;
+		break;
+
+	case DESC_RATEMCS8:
+	case DESC_RATEMCS9:
+	case DESC_RATEMCS10:
+	case DESC_RATEMCS11:
+	case DESC_RATEMCS12:
+	case DESC_RATEMCS13:
+	case DESC_RATEMCS14:
+	case DESC_RATEMCS15:
+		rate_section = HT_MCS8_MCS15;
+		break;
+
+	case DESC_RATEVHT1SS_MCS0:
+	case DESC_RATEVHT1SS_MCS1:
+	case DESC_RATEVHT1SS_MCS2:
+	case DESC_RATEVHT1SS_MCS3:
+	case DESC_RATEVHT1SS_MCS4:
+	case DESC_RATEVHT1SS_MCS5:
+	case DESC_RATEVHT1SS_MCS6:
+	case DESC_RATEVHT1SS_MCS7:
+	case DESC_RATEVHT1SS_MCS8:
+	case DESC_RATEVHT1SS_MCS9:
+		rate_section = VHT_1SSMCS0_1SSMCS9;
+		break;
+
+	case DESC_RATEVHT2SS_MCS0:
+	case DESC_RATEVHT2SS_MCS1:
+	case DESC_RATEVHT2SS_MCS2:
+	case DESC_RATEVHT2SS_MCS3:
+	case DESC_RATEVHT2SS_MCS4:
+	case DESC_RATEVHT2SS_MCS5:
+	case DESC_RATEVHT2SS_MCS6:
+	case DESC_RATEVHT2SS_MCS7:
+	case DESC_RATEVHT2SS_MCS8:
+	case DESC_RATEVHT2SS_MCS9:
+		rate_section = VHT_2SSMCS0_2SSMCS9;
+		break;
+
+	default:
+		RT_TRACE(rtlpriv, COMP_POWER, DBG_LOUD, "Wrong rate 0x%x\n",
+			 rate);
+		break;
+	}
+
+	if (band == BAND_ON_5G && rate_section == 0)
+		RT_TRACE(rtlpriv, COMP_POWER, DBG_LOUD,
+			 "Wrong rate 0x%x: No CCK in 5G Band\n", rate);
+
+	/* workaround for wrong index combination to obtain tx power limit,
+	 * OFDM only exists in BW 20M
+	 */
+	if (rate_section == 1)
+		bandwidth = 0;
+
+	/* workaround for wrong index combination to obtain tx power limit,
+	 * CCK table will only be given in BW 20M
+	 */
+	if (rate_section == 0)
+		bandwidth = 0;
+
+	/* workaround for wrong indxe combination to obtain tx power limit,
+	 * HT on 80M will reference to HT on 40M
+	 */
+	if ((rate_section == 2 || rate_section == 3) && band == BAND_ON_5G &&
+	    bandwidth == 2)
+		bandwidth = 1;
+
+	if (band == BAND_ON_2_4G)
+		channel_index = _rtl8822be_phy_get_chnl_idx_of_txpwr_lmt(
+			hw, BAND_ON_2_4G, channel);
+	else if (band == BAND_ON_5G)
+		channel_index = _rtl8822be_phy_get_chnl_idx_of_txpwr_lmt(
+			hw, BAND_ON_5G, channel);
+	else if (band == BAND_ON_BOTH)
+		; /* BAND_ON_BOTH don't care temporarily */
+
+	if (band >= BANDMAX || regulation == -1 || bandwidth == -1 ||
+	    rate_section == -1 || channel_index == -1) {
+		RT_TRACE(
+			rtlpriv, COMP_POWER, DBG_LOUD,
+			"Wrong index value to access power limit table [band %d][regulation %d][bandwidth %d][rf_path %d][rate_section %d][chnl %d]\n",
+			band, regulation, bandwidth, rf_path, rate_section,
+			channel_index);
+		return MAX_POWER_INDEX;
+	}
+
+	if (band == BAND_ON_2_4G) {
+		char limits[10] = {0};
+		u8 i = 0;
+
+		for (i = 0; i < 4; ++i)
+			limits[i] = rtlphy->txpwr_limit_2_4g[i][bandwidth]
+							    [rate_section]
+							    [channel_index]
+							    [rf_path];
+
+		power_limit =
+			(regulation == TXPWR_LMT_WW) ?
+				_rtl8822be_phy_get_world_wide_limit(limits) :
+				rtlphy->txpwr_limit_2_4g[regulation][bandwidth]
+							[rate_section]
+							[channel_index]
+							[rf_path];
+
+	} else if (band == BAND_ON_5G) {
+		char limits[10] = {0};
+		u8 i = 0;
+
+		for (i = 0; i < MAX_REGULATION_NUM; ++i)
+			limits[i] =
+				rtlphy->txpwr_limit_5g[i][bandwidth]
+						      [rate_section]
+						      [channel_index][rf_path];
+
+		power_limit =
+			(regulation == TXPWR_LMT_WW) ?
+				_rtl8822be_phy_get_world_wide_limit(limits) :
+				rtlphy->txpwr_limit_5g[regulation]
+						      [channel_index]
+						      [rate_section]
+						      [channel_index][rf_path];
+	} else
+		RT_TRACE(rtlpriv, COMP_INIT, DBG_LOUD,
+			 "No power limit table of the specified band\n");
+
+	return power_limit;
+}
+
+static char
+_rtl8822be_phy_get_txpower_by_rate(struct ieee80211_hw *hw, u8 band, u8 path,
+				   u8 rate /* enum rtl_desc8822b_rate */)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	u8 tx_num;
+	char tx_pwr_diff = 0;
+
+	if (band != BAND_ON_2_4G && band != BAND_ON_5G)
+		return tx_pwr_diff;
+
+	if (path > RF90_PATH_B)
+		return tx_pwr_diff;
+
+	if ((rate >= DESC_RATEMCS8 && rate <= DESC_RATEMCS15) ||
+	    (rate >= DESC_RATEVHT2SS_MCS0 && rate <= DESC_RATEVHT2SS_MCS9))
+		tx_num = RF_2TX;
+	else
+		tx_num = RF_1TX;
+
+	tx_pwr_diff = (char)(rtlphy->tx_power_by_rate_offset[band][path][tx_num]
+							    [rate] &
+			     0xff);
+
+	return tx_pwr_diff;
+}
+
+u8 rtl8822be_get_txpower_index(struct ieee80211_hw *hw, u8 path, u8 rate,
+			       u8 bandwidth, u8 channel)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
+	u8 index = (channel - 1);
+	u8 txpower = 0;
+	bool in_24g = false;
+	char limit;
+	char powerdiff_byrate = 0;
+
+	if (((rtlhal->current_bandtype == BAND_ON_2_4G) &&
+	     (channel > 14 || channel < 1)) ||
+	    ((rtlhal->current_bandtype == BAND_ON_5G) && (channel <= 14))) {
+		index = 0;
+		RT_TRACE(rtlpriv, COMP_POWER_TRACKING, DBG_LOUD,
+			 "Illegal channel!!\n");
+	}
+
+	/* 1. base tx power */
+	in_24g = _rtl8822be_phy_get_chnl_index(channel, &index);
+	if (in_24g) {
+		if (RX_HAL_IS_CCK_RATE(rate))
+			txpower = rtlefuse->txpwrlevel_cck[path][index];
+		else if (rate >= DESC_RATE6M)
+			txpower = rtlefuse->txpwrlevel_ht40_1s[path][index];
+		else
+			RT_TRACE(rtlpriv, COMP_POWER_TRACKING, DBG_LOUD,
+				 "invalid rate\n");
+
+		if (rate >= DESC_RATE6M && rate <= DESC_RATE54M &&
+		    !RX_HAL_IS_CCK_RATE(rate))
+			txpower += rtlefuse->txpwr_legacyhtdiff[path][TX_1S];
+
+		if (bandwidth == HT_CHANNEL_WIDTH_20) {
+			if ((rate >= DESC_RATEMCS0 && rate <= DESC_RATEMCS15) ||
+			    (rate >= DESC_RATEVHT1SS_MCS0 &&
+			     rate <= DESC_RATEVHT2SS_MCS9))
+				txpower +=
+					rtlefuse->txpwr_ht20diff[path][TX_1S];
+			if ((rate >= DESC_RATEMCS8 && rate <= DESC_RATEMCS15) ||
+			    (rate >= DESC_RATEVHT2SS_MCS0 &&
+			     rate <= DESC_RATEVHT2SS_MCS9))
+				txpower +=
+					rtlefuse->txpwr_ht20diff[path][TX_2S];
+		} else if (bandwidth == HT_CHANNEL_WIDTH_20_40) {
+			if ((rate >= DESC_RATEMCS0 && rate <= DESC_RATEMCS15) ||
+			    (rate >= DESC_RATEVHT1SS_MCS0 &&
+			     rate <= DESC_RATEVHT2SS_MCS9))
+				txpower +=
+					rtlefuse->txpwr_ht40diff[path][TX_1S];
+			if ((rate >= DESC_RATEMCS8 && rate <= DESC_RATEMCS15) ||
+			    (rate >= DESC_RATEVHT2SS_MCS0 &&
+			     rate <= DESC_RATEVHT2SS_MCS9))
+				txpower +=
+					rtlefuse->txpwr_ht40diff[path][TX_2S];
+		} else if (bandwidth == HT_CHANNEL_WIDTH_80) {
+			if ((rate >= DESC_RATEMCS0 && rate <= DESC_RATEMCS15) ||
+			    (rate >= DESC_RATEVHT1SS_MCS0 &&
+			     rate <= DESC_RATEVHT2SS_MCS9))
+				txpower +=
+					rtlefuse->txpwr_ht40diff[path][TX_1S];
+			if ((rate >= DESC_RATEMCS8 && rate <= DESC_RATEMCS15) ||
+			    (rate >= DESC_RATEVHT2SS_MCS0 &&
+			     rate <= DESC_RATEVHT2SS_MCS9))
+				txpower +=
+					rtlefuse->txpwr_ht40diff[path][TX_2S];
+		}
+
+	} else {
+		if (rate >= DESC_RATE6M)
+			txpower = rtlefuse->txpwr_5g_bw40base[path][index];
+		else
+			RT_TRACE(rtlpriv, COMP_POWER_TRACKING, DBG_WARNING,
+				 "INVALID Rate.\n");
+
+		if (rate >= DESC_RATE6M && rate <= DESC_RATE54M &&
+		    !RX_HAL_IS_CCK_RATE(rate))
+			txpower += rtlefuse->txpwr_5g_ofdmdiff[path][TX_1S];
+
+		if (bandwidth == HT_CHANNEL_WIDTH_20) {
+			if ((rate >= DESC_RATEMCS0 && rate <= DESC_RATEMCS15) ||
+			    (rate >= DESC_RATEVHT1SS_MCS0 &&
+			     rate <= DESC_RATEVHT2SS_MCS9))
+				txpower += rtlefuse->txpwr_5g_bw20diff[path]
+								      [TX_1S];
+			if ((rate >= DESC_RATEMCS8 && rate <= DESC_RATEMCS15) ||
+			    (rate >= DESC_RATEVHT2SS_MCS0 &&
+			     rate <= DESC_RATEVHT2SS_MCS9))
+				txpower += rtlefuse->txpwr_5g_bw20diff[path]
+								      [TX_2S];
+		} else if (bandwidth == HT_CHANNEL_WIDTH_20_40) {
+			if ((rate >= DESC_RATEMCS0 && rate <= DESC_RATEMCS15) ||
+			    (rate >= DESC_RATEVHT1SS_MCS0 &&
+			     rate <= DESC_RATEVHT2SS_MCS9))
+				txpower += rtlefuse->txpwr_5g_bw40diff[path]
+								      [TX_1S];
+			if ((rate >= DESC_RATEMCS8 && rate <= DESC_RATEMCS15) ||
+			    (rate >= DESC_RATEVHT2SS_MCS0 &&
+			     rate <= DESC_RATEVHT2SS_MCS9))
+				txpower += rtlefuse->txpwr_5g_bw40diff[path]
+								      [TX_2S];
+		} else if (bandwidth == HT_CHANNEL_WIDTH_80) {
+			u8 i = 0;
+
+			for (i = 0; i < sizeof(rtl_channel5g_80m) / sizeof(u8);
+			     ++i)
+				if (rtl_channel5g_80m[i] == channel)
+					index = i;
+
+			txpower = rtlefuse->txpwr_5g_bw80base[path][index];
+
+			if ((rate >= DESC_RATEMCS0 && rate <= DESC_RATEMCS15) ||
+			    (rate >= DESC_RATEVHT1SS_MCS0 &&
+			     rate <= DESC_RATEVHT2SS_MCS9))
+				txpower += rtlefuse->txpwr_5g_bw80diff[path]
+								      [TX_1S];
+			if ((rate >= DESC_RATEMCS8 && rate <= DESC_RATEMCS15) ||
+			    (rate >= DESC_RATEVHT2SS_MCS0 &&
+			     rate <= DESC_RATEVHT2SS_MCS9))
+				txpower += rtlefuse->txpwr_5g_bw80diff[path]
+								      [TX_2S];
+		}
+	}
+
+	/* 2. tx power by rate */
+	if (rtlefuse->eeprom_regulatory != 2)
+		powerdiff_byrate = _rtl8822be_phy_get_txpower_by_rate(
+			hw, (u8)(!in_24g), path, rate);
+
+	/* 3. tx power limit */
+	if (rtlefuse->eeprom_regulatory == 1)
+		limit = _rtl8822be_phy_get_txpower_limit(
+			hw, (u8)(!in_24g), bandwidth, path, rate,
+			channel);
+	else
+		limit = MAX_POWER_INDEX;
+
+	/* ----- */
+	powerdiff_byrate = powerdiff_byrate > limit ? limit : powerdiff_byrate;
+
+	txpower += powerdiff_byrate;
+
+	if (txpower > MAX_POWER_INDEX)
+		txpower = MAX_POWER_INDEX;
+
+	return txpower;
+}
+
+static void _rtl8822be_phy_set_txpower_index(struct ieee80211_hw *hw,
+					     u8 power_index, u8 path, u8 rate)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 shift = 0;
+	static u32 index;
+
+	/*
+	* For 8822B, phydm api use 4 bytes txagc value
+	* driver must combine every four 1 byte to one 4 byte and send to phydm
+	*/
+	shift = rate & 0x03;
+	index |= ((u32)power_index << (shift * 8));
+
+	if (shift == 3) {
+		rate = rate - 3;
+
+		if (!rtlpriv->phydm.ops->phydm_write_txagc(rtlpriv, index, path,
+							   rate)) {
+			RT_TRACE(rtlpriv, COMP_TXAGC, DBG_LOUD,
+				 "%s(index:%d, rfpath:%d, rate:0x%02x) fail\n",
+				 __func__, index, path, rate);
+
+			WARN_ON(1);
+		}
+		index = 0;
+	}
+}
+
+static void _rtl8822be_phy_set_txpower_level_by_path(struct ieee80211_hw *hw,
+						     u8 *array, u8 path,
+						     u8 channel, u8 size)
+{
+	struct rtl_phy *rtlphy = &(rtl_priv(hw)->phy);
+	u8 i;
+	u8 power_index;
+
+	for (i = 0; i < size; i++) {
+		power_index = rtl8822be_get_txpower_index(
+			hw, path, array[i], rtlphy->current_chan_bw, channel);
+		_rtl8822be_phy_set_txpower_index(hw, power_index, path,
+						 array[i]);
+	}
+}
+
+void rtl8822be_phy_set_txpower_level_by_path(struct ieee80211_hw *hw,
+					     u8 channel, u8 path)
+{
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+
+	/*
+	 * Below order is *VERY* important!
+	 * Because _rtl8822be_phy_set_txpower_index() do actually writing
+	 * every four power values.
+	 */
+	if (rtlhal->current_bandtype == BAND_ON_2_4G)
+		_rtl8822be_phy_set_txpower_level_by_path(
+			hw, cck_rates, path, channel, sizes_of_cck_retes);
+	_rtl8822be_phy_set_txpower_level_by_path(hw, ofdm_rates, path, channel,
+						 sizes_of_ofdm_retes);
+	_rtl8822be_phy_set_txpower_level_by_path(hw, ht_rates_1t, path, channel,
+						 sizes_of_ht_retes_1t);
+	_rtl8822be_phy_set_txpower_level_by_path(hw, ht_rates_2t, path, channel,
+						 sizes_of_ht_retes_2t);
+	_rtl8822be_phy_set_txpower_level_by_path(hw, vht_rates_1t, path,
+						 channel, sizes_of_vht_retes);
+	_rtl8822be_phy_set_txpower_level_by_path(hw, vht_rates_2t, path,
+						 channel, sizes_of_vht_retes);
+}
+
+void rtl8822be_phy_set_tx_power_index_by_rs(struct ieee80211_hw *hw, u8 channel,
+					    u8 path, enum rate_section rs)
+{
+	struct {
+		u8 *array;
+		u8 size;
+	} rs_ref[MAX_RATE_SECTION] = {
+		{cck_rates, sizes_of_cck_retes},
+		{ofdm_rates, sizes_of_ofdm_retes},
+		{ht_rates_1t, sizes_of_ht_retes_1t},
+		{ht_rates_2t, sizes_of_ht_retes_2t},
+		{vht_rates_1t, sizes_of_vht_retes},
+		{vht_rates_2t, sizes_of_vht_retes},
+	};
+
+	if (rs >= MAX_RATE_SECTION)
+		return;
+
+	_rtl8822be_phy_set_txpower_level_by_path(hw, rs_ref[rs].array, path,
+						 channel, rs_ref[rs].size);
+}
+
+void rtl8822be_phy_set_txpower_level(struct ieee80211_hw *hw, u8 channel)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	u8 path = 0;
+
+	for (path = RF90_PATH_A; path < rtlphy->num_total_rfpath; ++path)
+		rtl8822be_phy_set_txpower_level_by_path(hw, channel, path);
+}
+
+static long _rtl8822be_phy_txpwr_idx_to_dbm(struct ieee80211_hw *hw,
+					    enum wireless_mode wirelessmode,
+					    u8 txpwridx)
+{
+	long offset;
+	long pwrout_dbm;
+
+	switch (wirelessmode) {
+	case WIRELESS_MODE_B:
+		offset = -7;
+		break;
+	case WIRELESS_MODE_G:
+	case WIRELESS_MODE_N_24G:
+		offset = -8;
+		break;
+	default:
+		offset = -8;
+		break;
+	}
+	pwrout_dbm = txpwridx / 2 + offset;
+	return pwrout_dbm;
+}
+
+void rtl8822be_phy_scan_operation_backup(struct ieee80211_hw *hw, u8 operation)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	enum io_type iotype = IO_CMD_PAUSE_BAND0_DM_BY_SCAN;
+
+	if (!is_hal_stop(rtlhal)) {
+		switch (operation) {
+		case SCAN_OPT_BACKUP_BAND0:
+			iotype = IO_CMD_PAUSE_BAND0_DM_BY_SCAN;
+			rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_IO_CMD,
+						      (u8 *)&iotype);
+
+			break;
+		case SCAN_OPT_BACKUP_BAND1:
+			iotype = IO_CMD_PAUSE_BAND1_DM_BY_SCAN;
+			rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_IO_CMD,
+						      (u8 *)&iotype);
+
+			break;
+		case SCAN_OPT_RESTORE:
+			iotype = IO_CMD_RESUME_DM_BY_SCAN;
+			rtlpriv->cfg->ops->set_hw_reg(hw, HW_VAR_IO_CMD,
+						      (u8 *)&iotype);
+			break;
+		default:
+			pr_err("Unknown Scan Backup operation.\n");
+			break;
+		}
+	}
+}
+
+static u8 _rtl8822be_phy_get_pri_ch_id(struct rtl_priv *rtlpriv)
+{
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	struct rtl_mac *mac = rtl_mac(rtlpriv);
+	u8 pri_ch_idx = 0;
+
+	if (rtlphy->current_chan_bw == HT_CHANNEL_WIDTH_80) {
+		/* primary channel is at lower subband of 80MHz & 40MHz */
+		if ((mac->cur_40_prime_sc == HAL_PRIME_CHNL_OFFSET_LOWER) &&
+		    (mac->cur_80_prime_sc == HAL_PRIME_CHNL_OFFSET_LOWER)) {
+			pri_ch_idx = VHT_DATA_SC_20_LOWEST_OF_80MHZ;
+		/* primary channel is at
+		 * lower subband of 80MHz & upper subband of 40MHz
+		 */
+		} else if ((mac->cur_40_prime_sc ==
+			    HAL_PRIME_CHNL_OFFSET_UPPER) &&
+			   (mac->cur_80_prime_sc ==
+			    HAL_PRIME_CHNL_OFFSET_LOWER)) {
+			pri_ch_idx = VHT_DATA_SC_20_LOWER_OF_80MHZ;
+		/* primary channel is at
+		 * upper subband of 80MHz & lower subband of 40MHz
+		 */
+		} else if ((mac->cur_40_prime_sc ==
+			  HAL_PRIME_CHNL_OFFSET_LOWER) &&
+			 (mac->cur_80_prime_sc ==
+			  HAL_PRIME_CHNL_OFFSET_UPPER)) {
+			pri_ch_idx = VHT_DATA_SC_20_UPPER_OF_80MHZ;
+		/* primary channel is at
+		 * upper subband of 80MHz & upper subband of 40MHz
+		 */
+		} else if ((mac->cur_40_prime_sc ==
+			    HAL_PRIME_CHNL_OFFSET_UPPER) &&
+			   (mac->cur_80_prime_sc ==
+			    HAL_PRIME_CHNL_OFFSET_UPPER)) {
+			pri_ch_idx = VHT_DATA_SC_20_UPPERST_OF_80MHZ;
+		} else {
+			if (mac->cur_80_prime_sc == HAL_PRIME_CHNL_OFFSET_LOWER)
+				pri_ch_idx = VHT_DATA_SC_40_LOWER_OF_80MHZ;
+			else if (mac->cur_80_prime_sc ==
+				 HAL_PRIME_CHNL_OFFSET_UPPER)
+				pri_ch_idx = VHT_DATA_SC_40_UPPER_OF_80MHZ;
+		}
+	} else if (rtlphy->current_chan_bw == HT_CHANNEL_WIDTH_20_40) {
+		/* primary channel is at upper subband of 40MHz */
+		if (mac->cur_40_prime_sc == HAL_PRIME_CHNL_OFFSET_UPPER)
+			pri_ch_idx = VHT_DATA_SC_20_UPPER_OF_80MHZ;
+		/* primary channel is at lower subband of 40MHz */
+		else if (mac->cur_40_prime_sc == HAL_PRIME_CHNL_OFFSET_LOWER)
+			pri_ch_idx = VHT_DATA_SC_20_LOWER_OF_80MHZ;
+		else
+			;
+	}
+
+	return pri_ch_idx;
+}
+
+void rtl8822be_phy_set_bw_mode(struct ieee80211_hw *hw,
+			       enum nl80211_channel_type ch_type)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	u8 tmp_bw = rtlphy->current_chan_bw;
+
+	if (rtlphy->set_bwmode_inprogress)
+		return;
+	rtlphy->set_bwmode_inprogress = true;
+	if ((!is_hal_stop(rtlhal)) && !(RT_CANNOT_IO(hw))) {
+		/* get primary channel index */
+		u8 pri_ch_idx = _rtl8822be_phy_get_pri_ch_id(rtlpriv);
+
+		/* 3.1 set MAC register */
+		rtlpriv->halmac.ops->halmac_set_bandwidth(
+			rtlpriv, rtlphy->current_channel, pri_ch_idx,
+			rtlphy->current_chan_bw);
+
+		/* 3.2 set BB/RF registet */
+		rtlpriv->phydm.ops->phydm_switch_bandwidth(
+			rtlpriv, pri_ch_idx, rtlphy->current_chan_bw);
+
+		if (!mac->act_scanning)
+			rtlpriv->phydm.ops->phydm_iq_calibrate(rtlpriv);
+
+		rtlphy->set_bwmode_inprogress = false;
+	} else {
+		RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING,
+			 "FALSE driver sleep or unload\n");
+		rtlphy->set_bwmode_inprogress = false;
+		rtlphy->current_chan_bw = tmp_bw;
+	}
+}
+
+u8 rtl8822be_phy_sw_chnl(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	u32 timeout = 1000, timecount = 0;
+	u8 channel = rtlphy->current_channel;
+
+	if (rtlphy->sw_chnl_inprogress)
+		return 0;
+	if (rtlphy->set_bwmode_inprogress)
+		return 0;
+
+	if ((is_hal_stop(rtlhal)) || (RT_CANNOT_IO(hw))) {
+		RT_TRACE(rtlpriv, COMP_CHAN, DBG_LOUD,
+			 "sw_chnl_inprogress false driver sleep or unload\n");
+		return 0;
+	}
+	while (rtlphy->lck_inprogress && timecount < timeout) {
+		mdelay(50);
+		timecount += 50;
+	}
+
+	if (rtlphy->current_channel > 14)
+		rtlhal->current_bandtype = BAND_ON_5G;
+	else if (rtlphy->current_channel <= 14)
+		rtlhal->current_bandtype = BAND_ON_2_4G;
+
+	if (rtlpriv->cfg->ops->get_btc_status())
+		rtlpriv->btcoexist.btc_ops->btc_switch_band_notify(
+			rtlpriv, rtlhal->current_bandtype, mac->act_scanning);
+	else
+		rtlpriv->btcoexist.btc_ops->btc_switch_band_notify_wifi_only(
+			rtlpriv, rtlhal->current_bandtype, mac->act_scanning);
+
+	rtlpriv->phydm.ops->phydm_switch_band(rtlpriv, rtlphy->current_channel);
+
+	rtlphy->sw_chnl_inprogress = true;
+	if (channel == 0)
+		channel = 1;
+
+	RT_TRACE(rtlpriv, COMP_SCAN, DBG_TRACE,
+		 "switch to channel%d, band type is %d\n",
+		 rtlphy->current_channel, rtlhal->current_bandtype);
+
+	rtlpriv->phydm.ops->phydm_switch_channel(rtlpriv,
+						 rtlphy->current_channel);
+
+	rtlpriv->phydm.ops->phydm_clear_txpowertracking_state(rtlpriv);
+
+	rtl8822be_phy_set_txpower_level(hw, rtlphy->current_channel);
+
+	RT_TRACE(rtlpriv, COMP_SCAN, DBG_TRACE, "\n");
+	rtlphy->sw_chnl_inprogress = false;
+	return 1;
+}
+
+bool rtl8822be_phy_set_io_cmd(struct ieee80211_hw *hw, enum io_type iotype)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	bool postprocessing = false;
+
+	RT_TRACE(rtlpriv, COMP_CMD, DBG_TRACE,
+		 "-->IO Cmd(%#x), set_io_inprogress(%d)\n", iotype,
+		 rtlphy->set_io_inprogress);
+	do {
+		switch (iotype) {
+		case IO_CMD_RESUME_DM_BY_SCAN:
+			RT_TRACE(rtlpriv, COMP_CMD, DBG_TRACE,
+				 "[IO CMD] Resume DM after scan.\n");
+			postprocessing = true;
+			break;
+		case IO_CMD_PAUSE_BAND0_DM_BY_SCAN:
+		case IO_CMD_PAUSE_BAND1_DM_BY_SCAN:
+			RT_TRACE(rtlpriv, COMP_CMD, DBG_TRACE,
+				 "[IO CMD] Pause DM before scan.\n");
+			postprocessing = true;
+			break;
+		default:
+			pr_err("switch case not process\n");
+			break;
+		}
+	} while (false);
+	if (postprocessing && !rtlphy->set_io_inprogress) {
+		rtlphy->set_io_inprogress = true;
+		rtlphy->current_io_type = iotype;
+	} else {
+		return false;
+	}
+	rtl8822be_phy_set_io(hw);
+	RT_TRACE(rtlpriv, COMP_CMD, DBG_TRACE, "IO Type(%#x)\n", iotype);
+	return true;
+}
+
+static void rtl8822be_phy_set_io(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+
+	RT_TRACE(rtlpriv, COMP_CMD, DBG_TRACE,
+		 "--->Cmd(%#x), set_io_inprogress(%d)\n",
+		 rtlphy->current_io_type, rtlphy->set_io_inprogress);
+	switch (rtlphy->current_io_type) {
+	case IO_CMD_RESUME_DM_BY_SCAN:
+		break;
+	case IO_CMD_PAUSE_BAND0_DM_BY_SCAN:
+		break;
+	case IO_CMD_PAUSE_BAND1_DM_BY_SCAN:
+		break;
+	default:
+		pr_err("switch case not process\n");
+		break;
+	}
+	rtlphy->set_io_inprogress = false;
+	RT_TRACE(rtlpriv, COMP_CMD, DBG_TRACE, "(%#x)\n",
+		 rtlphy->current_io_type);
+}
+
+static void rtl8822be_phy_set_rf_on(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	rtl_write_byte(rtlpriv, REG_SPS0_CTRL_8822B, 0x2b);
+	rtl_write_byte(rtlpriv, REG_SYS_FUNC_EN_8822B, 0xE3);
+	rtl_write_byte(rtlpriv, REG_SYS_FUNC_EN_8822B, 0xE2);
+	rtl_write_byte(rtlpriv, REG_SYS_FUNC_EN_8822B, 0xE3);
+	rtl_write_byte(rtlpriv, REG_TXPAUSE_8822B, 0x00);
+}
+
+static bool _rtl8822be_phy_set_rf_power_state(struct ieee80211_hw *hw,
+					      enum rf_pwrstate rfpwr_state)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci_priv *pcipriv = rtl_pcipriv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+	bool bresult = true;
+	u8 i, queue_id;
+	struct rtl8192_tx_ring *ring = NULL;
+
+	switch (rfpwr_state) {
+	case ERFON:
+		if ((ppsc->rfpwr_state == ERFOFF) &&
+		    RT_IN_PS_LEVEL(ppsc, RT_RF_OFF_LEVL_HALT_NIC)) {
+			bool rtstatus = false;
+			u32 initialize_count = 0;
+
+			do {
+				initialize_count++;
+				RT_TRACE(rtlpriv, COMP_RF, DBG_DMESG,
+					 "IPS Set eRf nic enable\n");
+				rtstatus = rtl_ps_enable_nic(hw);
+			} while ((!rtstatus) && (initialize_count < 10));
+			RT_CLEAR_PS_LEVEL(ppsc, RT_RF_OFF_LEVL_HALT_NIC);
+		} else {
+			RT_TRACE(rtlpriv, COMP_RF, DBG_DMESG,
+				 "Set ERFON slept:%d ms\n",
+				 jiffies_to_msecs(jiffies -
+						  ppsc->last_sleep_jiffies));
+			ppsc->last_awake_jiffies = jiffies;
+			rtl8822be_phy_set_rf_on(hw);
+		}
+		if (mac->link_state == MAC80211_LINKED)
+			rtlpriv->cfg->ops->led_control(hw, LED_CTL_LINK);
+		else
+			rtlpriv->cfg->ops->led_control(hw, LED_CTL_NO_LINK);
+		break;
+	case ERFOFF:
+		for (queue_id = 0, i = 0;
+		     queue_id < RTL_PCI_MAX_TX_QUEUE_COUNT;) {
+			ring = &pcipriv->dev.tx_ring[queue_id];
+			if (queue_id == BEACON_QUEUE ||
+			    skb_queue_len(&ring->queue) == 0) {
+				queue_id++;
+				continue;
+			} else {
+				RT_TRACE(
+					rtlpriv, COMP_ERR, DBG_WARNING,
+					"eRf Off/Sleep: %d times TcbBusyQueue[%d] =%d before doze!\n",
+					(i + 1), queue_id,
+					skb_queue_len(&ring->queue));
+
+				udelay(10);
+				i++;
+			}
+			if (i >= MAX_DOZE_WAITING_TIMES_9x) {
+				RT_TRACE(
+					rtlpriv, COMP_ERR, DBG_WARNING,
+					"\n ERFSLEEP: %d times TcbBusyQueue[%d] = %d !\n",
+					MAX_DOZE_WAITING_TIMES_9x, queue_id,
+					skb_queue_len(&ring->queue));
+				break;
+			}
+		}
+
+		if (ppsc->reg_rfps_level & RT_RF_OFF_LEVL_HALT_NIC) {
+			RT_TRACE(rtlpriv, COMP_RF, DBG_DMESG,
+				 "IPS Set eRf nic disable\n");
+			rtl_ps_disable_nic(hw);
+			RT_SET_PS_LEVEL(ppsc, RT_RF_OFF_LEVL_HALT_NIC);
+		} else {
+			if (ppsc->rfoff_reason == RF_CHANGE_BY_IPS) {
+				rtlpriv->cfg->ops->led_control(hw,
+							       LED_CTL_NO_LINK);
+			} else {
+				rtlpriv->cfg->ops->led_control(
+					hw, LED_CTL_POWER_OFF);
+			}
+		}
+		break;
+	default:
+		pr_err("switch case not process\n");
+		bresult = false;
+		break;
+	}
+	if (bresult)
+		ppsc->rfpwr_state = rfpwr_state;
+	return bresult;
+}
+
+bool rtl8822be_phy_set_rf_power_state(struct ieee80211_hw *hw,
+				      enum rf_pwrstate rfpwr_state)
+{
+	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
+
+	bool bresult = false;
+
+	if (rfpwr_state == ppsc->rfpwr_state)
+		return bresult;
+	bresult = _rtl8822be_phy_set_rf_power_state(hw, rfpwr_state);
+	return bresult;
+}
diff --git a/drivers/staging/rtlwifi/rtl8822be/phy.h b/drivers/staging/rtlwifi/rtl8822be/phy.h
new file mode 100644
index 000000000000..5c33f16bcaa4
--- /dev/null
+++ b/drivers/staging/rtlwifi/rtl8822be/phy.h
@@ -0,0 +1,145 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __RTL8822BE_PHY_H__
+#define __RTL8822BE_PHY_H__
+
+/* It must always set to 4, otherwise read
+ * efuse table sequence will be wrong.
+ */
+#define MAX_TX_COUNT	4
+#define TX_1S	0
+#define TX_2S	1
+#define TX_3S	2
+#define TX_4S	3
+
+#define MAX_POWER_INDEX	0x3F
+
+#define MAX_PRECMD_CNT	16
+#define MAX_RFDEPENDCMD_CNT	16
+#define MAX_POSTCMD_CNT	16
+
+#define MAX_DOZE_WAITING_TIMES_9x 64
+
+#define RT_CANNOT_IO(hw) false
+#define HIGHPOWER_RADIOA_ARRAYLEN	22
+
+#define IQK_ADDA_REG_NUM	16
+#define IQK_BB_REG_NUM	9
+#define MAX_TOLERANCE	5
+#define IQK_DELAY_TIME	10
+#define index_mapping_NUM 15
+
+#define APK_BB_REG_NUM	5
+#define APK_AFE_REG_NUM	16
+#define APK_CURVE_REG_NUM	4
+#define PATH_NUM	2
+
+#define LOOP_LIMIT	5
+#define MAX_STALL_TIME	50
+#define ANTENNA_DIVERSITY_VALUE	0x80
+#define MAX_TXPWR_IDX_NMODE_92S	63
+#define RESET_CNT_LIMIT	3
+
+#define IQK_ADDA_REG_NUM	16
+#define IQK_MAC_REG_NUM	4
+
+#define RF6052_MAX_PATH	2
+
+#define CT_OFFSET_MAC_ADDR	0X16
+
+#define CT_OFFSET_CCK_TX_PWR_IDX	0x5A
+#define CT_OFFSET_HT401S_TX_PWR_IDX	0x60
+#define CT_OFFSET_HT402S_TX_PWR_IDX_DIFF	0x66
+#define CT_OFFSET_HT20_TX_PWR_IDX_DIFF	0x69
+#define CT_OFFSET_OFDM_TX_PWR_IDX_DIFF	0x6C
+
+#define CT_OFFSET_HT40_MAX_PWR_OFFSET	0x6F
+#define CT_OFFSET_HT20_MAX_PWR_OFFSET	0x72
+
+#define CT_OFFSET_CHANNEL_PLAH	0x75
+#define CT_OFFSET_THERMAL_METER	0x78
+#define CT_OFFSET_RF_OPTION	0x79
+#define CT_OFFSET_VERSION	0x7E
+#define CT_OFFSET_CUSTOMER_ID	0x7F
+
+#define RTL8822BE_MAX_PATH_NUM	2
+
+#define TARGET_CHNL_NUM_2G_5G_8822B	59
+
+u32 rtl8822be_phy_query_bb_reg(struct ieee80211_hw *hw, u32 regaddr,
+			       u32 bitmask);
+void rtl8822be_phy_set_bb_reg(struct ieee80211_hw *hw, u32 regaddr, u32 bitmask,
+			      u32 data);
+u32 rtl8822be_phy_query_rf_reg(struct ieee80211_hw *hw, enum radio_path rfpath,
+			       u32 regaddr, u32 bitmask);
+void rtl8822be_phy_set_rf_reg(struct ieee80211_hw *hw, enum radio_path rfpath,
+			      u32 regaddr, u32 bitmask, u32 data);
+bool rtl8822be_phy_bb_config(struct ieee80211_hw *hw);
+bool rtl8822be_phy_rf_config(struct ieee80211_hw *hw);
+bool rtl8822be_halmac_cb_init_mac_register(struct rtl_priv *rtlpriv);
+bool rtl8822be_halmac_cb_init_bb_rf_register(struct rtl_priv *rtlpriv);
+void rtl8822be_phy_get_txpower_level(struct ieee80211_hw *hw, long *powerlevel);
+void rtl8822be_phy_set_txpower_level(struct ieee80211_hw *hw, u8 channel);
+void rtl8822be_phy_scan_operation_backup(struct ieee80211_hw *hw, u8 operation);
+void rtl8822be_phy_set_bw_mode_callback(struct ieee80211_hw *hw);
+void rtl8822be_phy_set_bw_mode(struct ieee80211_hw *hw,
+			       enum nl80211_channel_type ch_type);
+u8 rtl8822be_phy_sw_chnl(struct ieee80211_hw *hw);
+void rtl8822be_phy_iq_calibrate(struct ieee80211_hw *hw, bool b_recovery);
+void rtl8822be_phy_iq_calibrate(struct ieee80211_hw *hw, bool b_recovery);
+void rtl8822be_phy_ap_calibrate(struct ieee80211_hw *hw, char delta);
+void rtl8822be_phy_lc_calibrate(struct ieee80211_hw *hw);
+void rtl8822be_phy_set_rfpath_switch(struct ieee80211_hw *hw, bool bmain);
+bool rtl8822be_phy_config_rf_with_headerfile(struct ieee80211_hw *hw,
+					     enum radio_path rfpath);
+bool rtl8822be_phy_config_rf_with_headerfile(struct ieee80211_hw *hw,
+					     enum radio_path rfpath);
+bool rtl8822be_phy_set_io_cmd(struct ieee80211_hw *hw, enum io_type iotype);
+bool rtl8822be_phy_set_rf_power_state(struct ieee80211_hw *hw,
+				      enum rf_pwrstate rfpwr_state);
+void rtl8822be_phy_set_txpower_level_by_path(struct ieee80211_hw *hw,
+					     u8 channel, u8 path);
+void rtl8822be_do_iqk(struct ieee80211_hw *hw, u8 delta_thermal_index,
+		      u8 thermal_value, u8 threshold);
+void rtl8822be_do_iqk(struct ieee80211_hw *hw, u8 delta_thermal_index,
+		      u8 thermal_value, u8 threshold);
+void rtl8822be_reset_iqk_result(struct ieee80211_hw *hw);
+
+u8 rtl8822be_get_txpower_index(struct ieee80211_hw *hw, u8 path, u8 rate,
+			       u8 bandwidth, u8 channel);
+void rtl8822be_phy_set_tx_power_index_by_rs(struct ieee80211_hw *hw, u8 channel,
+					    u8 path, enum rate_section rs);
+void rtl8822be_store_tx_power_by_rate(struct ieee80211_hw *hw, u32 band,
+				      u32 rfpath, u32 txnum, u32 regaddr,
+				      u32 bitmask, u32 data);
+void rtl8822be_phy_set_txpower_limit(struct ieee80211_hw *hw, u8 *pregulation,
+				     u8 *pband, u8 *pbandwidth,
+				     u8 *prate_section, u8 *prf_path,
+				     u8 *pchannel, u8 *ppower_limit);
+bool rtl8822be_load_txpower_by_rate(struct ieee80211_hw *hw);
+bool rtl8822be_load_txpower_limit(struct ieee80211_hw *hw);
+
+#endif
diff --git a/drivers/staging/rtlwifi/rtl8822be/reg.h b/drivers/staging/rtlwifi/rtl8822be/reg.h
new file mode 100644
index 000000000000..0dca5dccf49a
--- /dev/null
+++ b/drivers/staging/rtlwifi/rtl8822be/reg.h
@@ -0,0 +1,1653 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __RTL8822B_REG_H__
+#define __RTL8822B_REG_H__
+
+#include "../halmac/halmac_reg_8822b.h"
+#include "../halmac/halmac_bit_8822b.h"
+
+#define TXPKT_BUF_SELECT	0x69
+#define RXPKT_BUF_SELECT	0xA5
+#define DISABLE_TRXPKT_BUF_ACCESS	0x0
+
+/* Page 0 */
+#define REG_LEDCFG2_8822B	0x004E /* need review */
+#define REG_SPS0_CTRL_8822B	0x0011 /* need review: swlps */
+
+#define REG_EFUSE_ACCESS_8822B (REG_PMC_DBG_CTRL2_8822B + 3) /*0x00CF*/
+#define REG_AFE_XTAL_CTRL_8822B	REG_AFE_CTRL1_8822B
+#define REG_AFE_PLL_CTRL_8822B	REG_AFE_CTRL2_8822B
+
+/* Page 1 */
+
+#define MSR (REG_CR_8822B + 2)
+
+/* for MSR 0x102 */
+#define MSR_NOLINK	0x00
+#define MSR_ADHOC	0x01
+#define MSR_INFRA	0x02
+#define MSR_AP	0x03
+
+/*-----------------------------------------------------
+ *
+ *	0x0200h ~ 0x027Fh	TXDMA Configuration
+ *
+ *-----------------------------------------------------
+ */
+
+/*-----------------------------------------------------
+ *
+ *	0x0280h ~ 0x02FFh	RXDMA Configuration
+ *
+ *-----------------------------------------------------
+ */
+#define REG_RXDMA_CONTROL_8822B (REG_RXPKT_NUM_8822B + 2) /* 0x0286 */
+
+/*-----------------------------------------------------
+ *
+ *	0x0300h ~ 0x03FFh	PCIe
+ *
+ *-----------------------------------------------------
+ */
+
+/* REG_HIMR3_8822B */
+#define IMR_H2CDOK	BIT_SETH2CDOK_MASK_8822B
+
+/* spec version 11
+ *-----------------------------------------------------
+ *
+ *	0x0400h ~ 0x047Fh	Protocol Configuration
+ *
+ *-----------------------------------------------------
+ */
+
+#define REG_MAX_AGGR_NUM_8822B (REG_PROT_MODE_CTRL_8822B + 2) /*0x04CA*/
+
+/* for RRSR 0x440 */
+#define RRSR_RSC_OFFSET	21
+#define RRSR_SHORT_OFFSET	23
+#define RRSR_RSC_BW_40M	0x600000
+#define RRSR_RSC_UPSUBCHNL	0x400000
+#define RRSR_RSC_LOWSUBCHNL	0x200000
+#define RRSR_1M	BIT(0)
+#define RRSR_2M	BIT(1)
+#define RRSR_5_5M	BIT(2)
+#define RRSR_11M	BIT(3)
+#define RRSR_6M	BIT(4)
+#define RRSR_9M	BIT(5)
+#define RRSR_12M	BIT(6)
+#define RRSR_18M	BIT(7)
+#define RRSR_24M	BIT(8)
+#define RRSR_36M	BIT(9)
+#define RRSR_48M	BIT(10)
+#define RRSR_54M	BIT(11)
+#define RRSR_MCS0	BIT(12)
+#define RRSR_MCS1	BIT(13)
+#define RRSR_MCS2	BIT(14)
+#define RRSR_MCS3	BIT(15)
+#define RRSR_MCS4	BIT(16)
+#define RRSR_MCS5	BIT(17)
+#define RRSR_MCS6	BIT(18)
+#define RRSR_MCS7	BIT(19)
+
+#define RRSR_ALL_CCK (RRSR_1M | RRSR_2M | RRSR_5_5M | RRSR_11M)
+#define RRSR_ALL_OFDM_AG                                                       \
+	(RRSR_6M | RRSR_9M | RRSR_12M | RRSR_18M | RRSR_24M | RRSR_36M |       \
+	 RRSR_48M | RRSR_54M)
+
+/*-----------------------------------------------------
+ *
+ *	0x0500h ~ 0x05FFh	EDCA Configuration
+ *
+ *-----------------------------------------------------
+ */
+
+#define REG_SIFS_TRX_8822B (REG_SIFS_8822B + 2) /*0x0516*/
+
+/*-----------------------------------------------------
+ *
+ *	0x0600h ~ 0x07FFh	WMAC Configuration
+ *
+ *-----------------------------------------------------
+ */
+
+#define RATR_1M	0x00000001
+#define RATR_2M	0x00000002
+#define RATR_55M	0x00000004
+#define RATR_11M	0x00000008
+#define RATR_6M	0x00000010
+#define RATR_9M	0x00000020
+#define RATR_12M	0x00000040
+#define RATR_18M	0x00000080
+#define RATR_24M	0x00000100
+#define RATR_36M	0x00000200
+#define RATR_48M	0x00000400
+#define RATR_54M	0x00000800
+#define RATR_MCS0	0x00001000
+#define RATR_MCS1	0x00002000
+#define RATR_MCS2	0x00004000
+#define RATR_MCS3	0x00008000
+#define RATR_MCS4	0x00010000
+#define RATR_MCS5	0x00020000
+#define RATR_MCS6	0x00040000
+#define RATR_MCS7	0x00080000
+#define RATR_MCS8	0x00100000
+#define RATR_MCS9	0x00200000
+#define RATR_MCS10	0x00400000
+#define RATR_MCS11	0x00800000
+#define RATR_MCS12	0x01000000
+#define RATR_MCS13	0x02000000
+#define RATR_MCS14	0x04000000
+#define RATR_MCS15	0x08000000
+
+#define RATE_1M	BIT(0)
+#define RATE_2M	BIT(1)
+#define RATE_5_5M	BIT(2)
+#define RATE_11M	BIT(3)
+#define RATE_6M	BIT(4)
+#define RATE_9M	BIT(5)
+#define RATE_12M	BIT(6)
+#define RATE_18M	BIT(7)
+#define RATE_24M	BIT(8)
+#define RATE_36M	BIT(9)
+#define RATE_48M	BIT(10)
+#define RATE_54M	BIT(11)
+#define RATE_MCS0	BIT(12)
+#define RATE_MCS1	BIT(13)
+#define RATE_MCS2	BIT(14)
+#define RATE_MCS3	BIT(15)
+#define RATE_MCS4	BIT(16)
+#define RATE_MCS5	BIT(17)
+#define RATE_MCS6	BIT(18)
+#define RATE_MCS7	BIT(19)
+#define RATE_MCS8	BIT(20)
+#define RATE_MCS9	BIT(21)
+#define RATE_MCS10	BIT(22)
+#define RATE_MCS11	BIT(23)
+#define RATE_MCS12	BIT(24)
+#define RATE_MCS13	BIT(25)
+#define RATE_MCS14	BIT(26)
+#define RATE_MCS15	BIT(27)
+
+/* CAM definition */
+
+#define CAM_NONE	0x0
+#define CAM_WEP40	0x01
+#define CAM_TKIP	0x02
+#define CAM_AES	0x04
+#define CAM_WEP104	0x05
+
+/*#define	TOTAL_CAM_ENTRY				64*/
+/*#define	HALF_CAM_ENTRY				32*/
+
+#define CAM_WRITE	BIT(16)
+#define CAM_READ	0x00000000
+#define CAM_POLLINIG	BIT(31)
+
+/*********************************************
+ *       8822BE IMR/ISR bits
+ *********************************************
+ */
+#define IMR_DISABLED	0x0
+/* IMR DW0(0x0060-0063) Bit 0-31 */
+#define IMR_TIMER2	BIT(31)
+#define IMR_TIMER1	BIT(30)
+#define IMR_PSTIMEOUT	BIT(29)
+#define IMR_GTINT4	BIT(28)
+#define IMR_GTINT3	BIT(27)
+#define IMR_TBDER	BIT(26)
+#define IMR_TBDOK	BIT(25)
+#define IMR_TSF_BIT32_TOGGLE	BIT(24)
+#define IMR_BCNDMAINT0	BIT(20)
+#define IMR_BCNDOK0	BIT(16)
+#define IMR_HSISR_IND_ON_INT	BIT(15)
+#define IMR_BCNDMAINT_E	BIT(14)
+#define IMR_ATIMEND	BIT(12)
+#define IMR_HISR1_IND_INT	BIT(11)
+#define IMR_C2HCMD	BIT(10)
+#define IMR_CPWM2	BIT(9)
+#define IMR_CPWM	BIT(8)
+#define IMR_HIGHDOK	BIT(7)
+#define IMR_MGNTDOK	BIT(6)
+#define IMR_BKDOK	BIT(5)
+#define IMR_BEDOK	BIT(4)
+#define IMR_VIDOK	BIT(3)
+#define IMR_VODOK	BIT(2)
+#define IMR_RDU	BIT(1)
+#define IMR_ROK	BIT(0)
+
+/* IMR DW1(0x00B4-00B7) Bit 0-31 */
+#define IMR_TXFIFO_TH_INT_8822B	BIT_TXFIFO_TH_INT_8822B
+#define IMR_BTON_STS_UPDATE_MASK_8822B	BIT_BTON_STS_UPDATE_MASK_8822B
+#define IMR_MCUERR	BIT(28)
+#define IMR_BCNDMAINT7	BIT(27)
+#define IMR_BCNDMAINT6	BIT(26)
+#define IMR_BCNDMAINT5	BIT(25)
+#define IMR_BCNDMAINT4	BIT(24)
+#define IMR_BCNDMAINT3	BIT(23)
+#define IMR_BCNDMAINT2	BIT(22)
+#define IMR_BCNDMAINT1	BIT(21)
+#define IMR_BCNDOK7	BIT(20)
+#define IMR_BCNDOK6	BIT(19)
+#define IMR_BCNDOK5	BIT(18)
+#define IMR_BCNDOK4	BIT(17)
+#define IMR_BCNDOK3	BIT(16)
+#define IMR_BCNDOK2	BIT(15)
+#define IMR_BCNDOK1	BIT(14)
+#define IMR_ATIMEND_E	BIT(13)
+#define IMR_ATIMEND	BIT(12)
+#define IMR_TXERR	BIT(11)
+#define IMR_RXERR	BIT(10)
+#define IMR_TXFOVW	BIT(9)
+#define IMR_RXFOVW	BIT(8)
+#define IMR_CPU_MGQ_TXDONE_MSK_8822B	BIT_CPU_MGQ_TXDONE_MSK_8822B
+#define IMR_PS_TIMER_C_MSK_8822B	BIT_PS_TIMER_C_MSK_8822B
+#define IMR_PS_TIMER_B_MSK_8822B	BIT_PS_TIMER_B_MSK_8822B
+#define IMR_PS_TIMER_A_MSK_8822B	BIT_PS_TIMER_A_MSK_8822B
+#define IMR_CPUMGQ_TX_TIMER_MSK_8822B	BIT_CPUMGQ_TX_TIMER_MSK_8822B
+
+/*********************************************
+ *       8822BE EFUSE definition
+ *********************************************
+ */
+#define HWSET_MAX_SIZE	1024
+#define EFUSE_MAX_SECTION	64
+#define EFUSE_REAL_CONTENT_LEN	1024
+#define EFUSE_OOB_PROTECT_BYTES	18
+
+#define EEPROM_DEFAULT_THERMALMETER	0x12
+
+#define RTL8822B_EEPROM_ID	0x8129
+
+#define PPG_BB_GAIN_2G_TXA_OFFSET_8822B	0xEE
+#define PPG_THERMAL_OFFSET_8822B	0xEF
+
+#define EEPROM_TX_PWR_INX_8822B	0x10
+
+#define EEPROM_CHANNEL_PLAN_8822B	0xB8
+#define EEPROM_XTAL_8822B	0xB9
+#define EEPROM_THERMAL_METER_8822B	0xBA
+#define EEPROM_IQK_LCK_8822B	0xBB
+#define EEPROM_2G_5G_PA_TYPE_8822B	0xBC
+/* PATH A & PATH B */
+#define EEPROM_2G_LNA_TYPE_GAIN_SEL_AB_8822B	0xBD
+/* PATH C & PATH D */
+#define EEPROM_2G_LNA_TYPE_GAIN_SEL_CD_8822B	0xBE
+/* PATH A & PATH B */
+#define EEPROM_5G_LNA_TYPE_GAIN_SEL_AB_8822B	0xBF
+/* PATH C & PATH D */
+#define EEPROM_5G_LNA_TYPE_GAIN_SEL_CD_8822B	0xC0
+
+#define EEPROM_RF_BOARD_OPTION_8822B	0xC1
+#define EEPROM_FEATURE_OPTION_8822B	0xC2
+#define EEPROM_RF_BT_SETTING_8822B	0xC3
+#define EEPROM_VERSION_8822B	0xC4
+#define EEPROM_CUSTOM_ID_8822B	0xC5
+#define EEPROM_TX_BBSWING_2G_8822B	0xC6
+#define EEPROM_TX_PWR_CALIBRATE_RATE_8822B	0xC8
+#define EEPROM_RF_ANTENNA_OPT_8822B	0xC9
+#define EEPROM_RFE_OPTION_8822B	0xCA
+#define EEPROM_COUNTRY_CODE_8822B	0xCB
+
+#define EEPROM_VID	0xD6
+#define EEPROM_DID	0xD8
+#define EEPROM_SVID	0xDA
+#define EEPROM_SMID	0xDC
+
+/* RTL8822BU */
+#define EEPROM_MAC_ADDR_8822BU	0x107
+#define EEPROM_VID_8822BU	0x100
+#define EEPROM_PID_8822BU	0x102
+#define EEPROM_USB_OPTIONAL_FUNCTION0_8822BU	0x104
+#define EEPROM_USB_MODE_8822BU	0x06
+
+/* RTL8822BS */
+#define EEPROM_MAC_ADDR_8822BS	0x11A
+
+/* RTL8822BE */
+#define EEPROM_MAC_ADDR_8822BE	0xD0
+
+/* ------------------------- */
+
+#define STOPBECON	BIT(6)
+#define STOPHIGHT	BIT(5)
+#define STOPMGT	BIT(4)
+#define STOPVO	BIT(3)
+#define STOPVI	BIT(2)
+#define STOPBE	BIT(1)
+#define STOPBK	BIT(0)
+
+#define RCR_APPFCS	BIT(31)
+#define RCR_APP_MIC	BIT(30)
+#define RCR_APP_ICV	BIT(29)
+#define RCR_APP_PHYST_RXFF	BIT(28)
+#define RCR_APP_BA_SSN	BIT(27)
+#define RCR_VHT_DACK	BIT(26)
+#define RCR_ENMBID	BIT(24)
+#define RCR_LSIGEN	BIT(23)
+#define RCR_MFBEN	BIT(22)
+#define RCR_HTC_LOC_CTRL	BIT(14)
+#define RCR_AMF	BIT(13)
+#define RCR_ACF	BIT(12)
+#define RCR_ADF	BIT(11)
+#define RCR_AICV	BIT(9)
+#define RCR_ACRC32	BIT(8)
+#define RCR_CBSSID_BCN	BIT(7)
+#define RCR_CBSSID_DATA	BIT(6)
+#define RCR_CBSSID	RCR_CBSSID_DATA
+#define RCR_APWRMGT	BIT(5)
+#define RCR_ADD3	BIT(4)
+#define RCR_AB	BIT(3)
+#define RCR_AM	BIT(2)
+#define RCR_APM	BIT(1)
+#define RCR_AAP	BIT(0)
+#define RCR_MXDMA_OFFSET	8
+#define RCR_FIFO_OFFSET	13
+
+#define RSV_CTRL	0x001C
+#define RD_CTRL	0x0524
+
+#define REG_USB_INFO_8822B	0xFE17
+#define REG_USB_SPECIAL_OPTION_8822B	0xFE55
+#define REG_USB_DMA_AGG_TO_8822B	0xFE5B
+#define REG_USB_AGG_TO_8822B	0xFE5C
+#define REG_USB_AGG_TH_8822B	0xFE5D
+
+#define REG_USB_VID_8822B	0xFE60
+#define REG_USB_PID_8822B	0xFE62
+#define REG_USB_OPTIONAL_8822B	0xFE64
+#define REG_USB_CHIRP_K_8822B	0xFE65
+#define REG_USB_PHY_8822B	0xFE66
+#define REG_USB_MAC_ADDR_8822B	0xFE70
+#define REG_USB_HRPWM_8822B	0xFE58
+#define REG_USB_HCPWM_8822B	0xFE57
+
+#define SW18_FPWM	BIT(3)
+
+#define ISO_MD2PP	BIT(0)
+#define ISO_UA2USB	BIT(1)
+#define ISO_UD2CORE	BIT(2)
+#define ISO_PA2PCIE	BIT(3)
+#define ISO_PD2CORE	BIT(4)
+#define ISO_IP2MAC	BIT(5)
+#define ISO_DIOP	BIT(6)
+#define ISO_DIOE	BIT(7)
+#define ISO_EB2CORE	BIT(8)
+#define ISO_DIOR	BIT(9)
+
+#define PWC_EV25V	BIT(14)
+#define PWC_EV12V	BIT(15)
+
+#define FEN_BBRSTB	BIT(0)
+#define FEN_BB_GLB_RSTN	BIT(1)
+#define FEN_USBA	BIT(2)
+#define FEN_UPLL	BIT(3)
+#define FEN_USBD	BIT(4)
+#define FEN_DIO_PCIE	BIT(5)
+#define FEN_PCIEA	BIT(6)
+#define FEN_PPLL	BIT(7)
+#define FEN_PCIED	BIT(8)
+#define FEN_DIOE	BIT(9)
+#define FEN_CPUEN	BIT(10)
+#define FEN_DCORE	BIT(11)
+#define FEN_ELDR	BIT(12)
+#define FEN_DIO_RF	BIT(13)
+#define FEN_HWPDN	BIT(14)
+#define FEN_MREGEN	BIT(15)
+
+#define PFM_LDALL	BIT(0)
+#define PFM_ALDN	BIT(1)
+#define PFM_LDKP	BIT(2)
+#define PFM_WOWL	BIT(3)
+#define EN_PDN	BIT(4)
+#define PDN_PL	BIT(5)
+#define APFM_ONMAC	BIT(8)
+#define APFM_OFF	BIT(9)
+#define APFM_RSM	BIT(10)
+#define AFSM_HSUS	BIT(11)
+#define AFSM_PCIE	BIT(12)
+#define APDM_MAC	BIT(13)
+#define APDM_HOST	BIT(14)
+#define APDM_HPDN	BIT(15)
+#define RDY_MACON	BIT(16)
+#define SUS_HOST	BIT(17)
+#define ROP_ALD	BIT(20)
+#define ROP_PWR	BIT(21)
+#define ROP_SPS	BIT(22)
+#define SOP_MRST	BIT(25)
+#define SOP_FUSE	BIT(26)
+#define SOP_ABG	BIT(27)
+#define SOP_AMB	BIT(28)
+#define SOP_RCK	BIT(29)
+#define SOP_A8M	BIT(30)
+#define XOP_BTCK	BIT(31)
+
+#define ANAD16V_EN	BIT(0)
+#define ANA8M	BIT(1)
+#define MACSLP	BIT(4)
+#define LOADER_CLK_EN	BIT(5)
+#define _80M_SSC_DIS	BIT(7)
+#define _80M_SSC_EN_HO	BIT(8)
+#define PHY_SSC_RSTB	BIT(9)
+#define SEC_CLK_EN	BIT(10)
+#define MAC_CLK_EN	BIT(11)
+#define SYS_CLK_EN	BIT(12)
+#define RING_CLK_EN	BIT(13)
+
+#define BOOT_FROM_EEPROM	BIT(4)
+#define EEPROM_EN	BIT(5)
+
+#define AFE_BGEN	BIT(0)
+#define AFE_MBEN	BIT(1)
+#define MAC_ID_EN	BIT(7)
+
+#define WLOCK_ALL	BIT(0)
+#define WLOCK_00	BIT(1)
+#define WLOCK_04	BIT(2)
+#define WLOCK_08	BIT(3)
+#define WLOCK_40	BIT(4)
+#define R_DIS_PRST_0	BIT(5)
+#define R_DIS_PRST_1	BIT(6)
+#define LOCK_ALL_EN	BIT(7)
+
+#define RF_EN	BIT(0)
+#define RF_RSTB	BIT(1)
+#define RF_SDMRSTB	BIT(2)
+
+#define LDA15_EN	BIT(0)
+#define LDA15_STBY	BIT(1)
+#define LDA15_OBUF	BIT(2)
+#define LDA15_REG_VOS	BIT(3)
+#define _LDA15_VOADJ(x) (((x) & 0x7) << 4)
+
+#define LDV12_EN	BIT(0)
+#define LDV12_SDBY	BIT(1)
+#define LPLDO_HSM	BIT(2)
+#define LPLDO_LSM_DIS	BIT(3)
+#define _LDV12_VADJ(x) (((x) & 0xF) << 4)
+
+#define XTAL_EN	BIT(0)
+#define XTAL_BSEL	BIT(1)
+#define _XTAL_BOSC(x) (((x) & 0x3) << 2)
+#define _XTAL_CADJ(x) (((x) & 0xF) << 4)
+#define XTAL_GATE_USB	BIT(8)
+#define _XTAL_USB_DRV(x) (((x) & 0x3) << 9)
+#define XTAL_GATE_AFE	BIT(11)
+#define _XTAL_AFE_DRV(x) (((x) & 0x3) << 12)
+#define XTAL_RF_GATE	BIT(14)
+#define _XTAL_RF_DRV(x) (((x) & 0x3) << 15)
+#define XTAL_GATE_DIG	BIT(17)
+#define _XTAL_DIG_DRV(x) (((x) & 0x3) << 18)
+#define XTAL_BT_GATE	BIT(20)
+#define _XTAL_BT_DRV(x) (((x) & 0x3) << 21)
+#define _XTAL_GPIO(x) (((x) & 0x7) << 23)
+
+#define CKDLY_AFE	BIT(26)
+#define CKDLY_USB	BIT(27)
+#define CKDLY_DIG	BIT(28)
+#define CKDLY_BT	BIT(29)
+
+#define APLL_EN	BIT(0)
+#define APLL_320_EN	BIT(1)
+#define APLL_FREF_SEL	BIT(2)
+#define APLL_EDGE_SEL	BIT(3)
+#define APLL_WDOGB	BIT(4)
+#define APLL_LPFEN	BIT(5)
+
+#define APLL_REF_CLK_13MHZ	0x1
+#define APLL_REF_CLK_19_2MHZ	0x2
+#define APLL_REF_CLK_20MHZ	0x3
+#define APLL_REF_CLK_25MHZ	0x4
+#define APLL_REF_CLK_26MHZ	0x5
+#define APLL_REF_CLK_38_4MHZ	0x6
+#define APLL_REF_CLK_40MHZ	0x7
+
+#define APLL_320EN	BIT(14)
+#define APLL_80EN	BIT(15)
+#define APLL_1MEN	BIT(24)
+
+#define ALD_EN	BIT(18)
+#define EF_PD	BIT(19)
+#define EF_FLAG	BIT(31)
+
+#define EF_TRPT	BIT(7)
+#define LDOE25_EN	BIT(31)
+
+#define RSM_EN	BIT(0)
+#define TIMER_EN	BIT(4)
+
+#define TRSW0EN	BIT(2)
+#define TRSW1EN	BIT(3)
+#define EROM_EN	BIT(4)
+#define EN_BT	BIT(5)
+#define EN_UART	BIT(8)
+#define UART_910	BIT(9)
+#define EN_PMAC	BIT(10)
+#define SIC_SWRST	BIT(11)
+#define EN_SIC	BIT(12)
+#define SIC_23	BIT(13)
+#define EN_HDP	BIT(14)
+#define SIC_LBK	BIT(15)
+
+#define LED0PL	BIT(4)
+#define LED1PL	BIT(12)
+#define LED0DIS	BIT(7)
+
+#define MCUFWDL_EN	BIT(0)
+#define MCUFWDL_RDY	BIT(1)
+#define FWDL_CHKSUM_RPT	BIT(2)
+#define MACINI_RDY	BIT(3)
+#define BBINI_RDY	BIT(4)
+#define RFINI_RDY	BIT(5)
+#define WINTINI_RDY	BIT(6)
+#define CPRST	BIT(23)
+
+#define XCLK_VLD	BIT(0)
+#define ACLK_VLD	BIT(1)
+#define UCLK_VLD	BIT(2)
+#define PCLK_VLD	BIT(3)
+#define PCIRSTB	BIT(4)
+#define V15_VLD	BIT(5)
+#define TRP_B15V_EN	BIT(7)
+#define SIC_IDLE	BIT(8)
+#define BD_MAC2	BIT(9)
+#define BD_MAC1	BIT(10)
+#define IC_MACPHY_MODE	BIT(11)
+#define VENDOR_ID	BIT(19)
+#define PAD_HWPD_IDN	BIT(22)
+#define TRP_VAUX_EN	BIT(23)
+#define TRP_BT_EN	BIT(24)
+#define BD_PKG_SEL	BIT(25)
+#define BD_HCI_SEL	BIT(26)
+#define TYPE_ID	BIT(27)
+
+#define CHIP_VER_RTL_MASK	0xF000
+#define CHIP_VER_RTL_SHIFT	12
+
+#define REG_LBMODE_8822B (REG_CR_8822B + 3)
+
+#define HCI_TXDMA_EN	BIT(0)
+#define HCI_RXDMA_EN	BIT(1)
+#define TXDMA_EN	BIT(2)
+#define RXDMA_EN	BIT(3)
+#define PROTOCOL_EN	BIT(4)
+#define SCHEDULE_EN	BIT(5)
+#define MACTXEN	BIT(6)
+#define MACRXEN	BIT(7)
+#define ENSWBCN	BIT(8)
+#define ENSEC	BIT(9)
+
+#define _NETTYPE(x) (((x) & 0x3) << 16)
+#define MASK_NETTYPE	0x30000
+#define NT_NO_LINK	0x0
+#define NT_LINK_AD_HOC	0x1
+#define NT_LINK_AP	0x2
+#define NT_AS_AP	0x3
+
+#define _LBMODE(x) (((x) & 0xF) << 24)
+#define MASK_LBMODE	0xF000000
+#define LOOPBACK_NORMAL	0x0
+#define LOOPBACK_IMMEDIATELY	0xB
+#define LOOPBACK_MAC_DELAY	0x3
+#define LOOPBACK_PHY	0x1
+#define LOOPBACK_DMA	0x7
+
+#define GET_RX_PAGE_SIZE(value) ((value) & 0xF)
+#define GET_TX_PAGE_SIZE(value) (((value) & 0xF0) >> 4)
+#define _PSRX_MASK	0xF
+#define _PSTX_MASK	0xF0
+#define _PSRX(x) (x)
+#define _PSTX(x) ((x) << 4)
+
+#define PBP_64	0x0
+#define PBP_128	0x1
+#define PBP_256	0x2
+#define PBP_512	0x3
+#define PBP_1024	0x4
+
+#define RXDMA_ARBBW_EN	BIT(0)
+#define RXSHFT_EN	BIT(1)
+#define RXDMA_AGG_EN	BIT(2)
+#define QS_VO_QUEUE	BIT(8)
+#define QS_VI_QUEUE	BIT(9)
+#define QS_BE_QUEUE	BIT(10)
+#define QS_BK_QUEUE	BIT(11)
+#define QS_MANAGER_QUEUE	BIT(12)
+#define QS_HIGH_QUEUE	BIT(13)
+
+#define HQSEL_VOQ	BIT(0)
+#define HQSEL_VIQ	BIT(1)
+#define HQSEL_BEQ	BIT(2)
+#define HQSEL_BKQ	BIT(3)
+#define HQSEL_MGTQ	BIT(4)
+#define HQSEL_HIQ	BIT(5)
+
+#define _TXDMA_HIQ_MAP(x) (((x) & 0x3) << 14)
+#define _TXDMA_MGQ_MAP(x) (((x) & 0x3) << 12)
+#define _TXDMA_BKQ_MAP(x) (((x) & 0x3) << 10)
+#define _TXDMA_BEQ_MAP(x) (((x) & 0x3) << 8)
+#define _TXDMA_VIQ_MAP(x) (((x) & 0x3) << 6)
+#define _TXDMA_VOQ_MAP(x) (((x) & 0x3) << 4)
+
+#define QUEUE_LOW	1
+#define QUEUE_NORMAL	2
+#define QUEUE_HIGH	3
+
+#define _LLT_NO_ACTIVE	0x0
+#define _LLT_WRITE_ACCESS	0x1
+#define _LLT_READ_ACCESS	0x2
+
+#define _LLT_INIT_DATA(x) ((x) & 0xFF)
+#define _LLT_INIT_ADDR(x) (((x) & 0xFF) << 8)
+#define _LLT_OP(x) (((x) & 0x3) << 30)
+#define _LLT_OP_VALUE(x) (((x) >> 30) & 0x3)
+
+#define BB_WRITE_READ_MASK (BIT(31) | BIT(30))
+#define BB_WRITE_EN	BIT(30)
+#define BB_READ_EN	BIT(31)
+
+#define _HPQ(x) ((x) & 0xFF)
+#define _LPQ(x) (((x) & 0xFF) << 8)
+#define _PUBQ(x) (((x) & 0xFF) << 16)
+#define _NPQ(x) ((x) & 0xFF)
+
+#define HPQ_PUBLIC_DIS	BIT(24)
+#define LPQ_PUBLIC_DIS	BIT(25)
+#define LD_RQPN	BIT(31)
+
+#define BCN_VALID	BIT(16)
+#define BCN_HEAD(x) (((x) & 0xFF) << 8)
+#define BCN_HEAD_MASK	0xFF00
+
+#define BLK_DESC_NUM_SHIFT	4
+#define BLK_DESC_NUM_MASK	0xF
+
+#define DROP_DATA_EN	BIT(9)
+
+#define EN_AMPDU_RTY_NEW	BIT(7)
+
+#define _INIRTSMCS_SEL(x) ((x) & 0x3F)
+
+#define _SPEC_SIFS_CCK(x) ((x) & 0xFF)
+#define _SPEC_SIFS_OFDM(x) (((x) & 0xFF) << 8)
+
+#define RATE_REG_BITMAP_ALL	0xFFFFF
+
+#define _RRSC_BITMAP(x) ((x) & 0xFFFFF)
+
+#define _RRSR_RSC(x) (((x) & 0x3) << 21)
+#define RRSR_RSC_RESERVED	0x0
+#define RRSR_RSC_UPPER_SUBCHANNEL	0x1
+#define RRSR_RSC_LOWER_SUBCHANNEL	0x2
+#define RRSR_RSC_DUPLICATE_MODE	0x3
+
+#define USE_SHORT_G1	BIT(20)
+
+#define _AGGLMT_MCS0(x) ((x) & 0xF)
+#define _AGGLMT_MCS1(x) (((x) & 0xF) << 4)
+#define _AGGLMT_MCS2(x) (((x) & 0xF) << 8)
+#define _AGGLMT_MCS3(x) (((x) & 0xF) << 12)
+#define _AGGLMT_MCS4(x) (((x) & 0xF) << 16)
+#define _AGGLMT_MCS5(x) (((x) & 0xF) << 20)
+#define _AGGLMT_MCS6(x) (((x) & 0xF) << 24)
+#define _AGGLMT_MCS7(x) (((x) & 0xF) << 28)
+
+#define RETRY_LIMIT_SHORT_SHIFT	8
+#define RETRY_LIMIT_LONG_SHIFT	0
+
+#define _DARF_RC1(x) ((x) & 0x1F)
+#define _DARF_RC2(x) (((x) & 0x1F) << 8)
+#define _DARF_RC3(x) (((x) & 0x1F) << 16)
+#define _DARF_RC4(x) (((x) & 0x1F) << 24)
+#define _DARF_RC5(x) ((x) & 0x1F)
+#define _DARF_RC6(x) (((x) & 0x1F) << 8)
+#define _DARF_RC7(x) (((x) & 0x1F) << 16)
+#define _DARF_RC8(x) (((x) & 0x1F) << 24)
+
+#define _RARF_RC1(x) ((x) & 0x1F)
+#define _RARF_RC2(x) (((x) & 0x1F) << 8)
+#define _RARF_RC3(x) (((x) & 0x1F) << 16)
+#define _RARF_RC4(x) (((x) & 0x1F) << 24)
+#define _RARF_RC5(x) ((x) & 0x1F)
+#define _RARF_RC6(x) (((x) & 0x1F) << 8)
+#define _RARF_RC7(x) (((x) & 0x1F) << 16)
+#define _RARF_RC8(x) (((x) & 0x1F) << 24)
+
+#define AC_PARAM_TXOP_LIMIT_OFFSET	16
+#define AC_PARAM_ECW_MAX_OFFSET	12
+#define AC_PARAM_ECW_MIN_OFFSET	8
+#define AC_PARAM_AIFS_OFFSET	0
+
+#define _AIFS(x) (x)
+#define _ECW_MAX_MIN(x) ((x) << 8)
+#define _TXOP_LIMIT(x) ((x) << 16)
+
+#define _BCNIFS(x) ((x) & 0xFF)
+#define _BCNECW(x) ((((x) & 0xF)) << 8)
+
+#define _LRL(x) ((x) & 0x3F)
+#define _SRL(x) (((x) & 0x3F) << 8)
+
+#define _SIFS_CCK_CTX(x) ((x) & 0xFF)
+#define _SIFS_CCK_TRX(x) (((x) & 0xFF) << 8)
+
+#define _SIFS_OFDM_CTX(x) ((x) & 0xFF)
+#define _SIFS_OFDM_TRX(x) (((x) & 0xFF) << 8)
+
+#define _TBTT_PROHIBIT_HOLD(x) (((x) & 0xFF) << 8)
+
+#define DIS_EDCA_CNT_DWN	BIT(11)
+
+#define EN_MBSSID	BIT(1)
+#define EN_TXBCN_RPT	BIT(2)
+#define EN_BCN_FUNCTION	BIT(3)
+
+#define TSFTR_RST	BIT(0)
+#define TSFTR1_RST	BIT(1)
+
+#define STOP_BCNQ	BIT(6)
+
+#define DIS_TSF_UDT0_NORMAL_CHIP	BIT(4)
+#define DIS_TSF_UDT0_TEST_CHIP	BIT(5)
+
+#define ACMHW_HW_EN	BIT(0)
+#define ACMHW_BEQ_EN	BIT(1)
+#define ACMHW_VIQ_EN	BIT(2)
+#define ACMHW_VOQ_EN	BIT(3)
+#define ACMHW_BEQ_STATUS	BIT(4)
+#define ACMHW_VIQ_STATUS	BIT(5)
+#define ACMHW_VOQ_STATUS	BIT(6)
+
+#define APSDOFF	BIT(6)
+#define APSDOFF_STATUS	BIT(7)
+
+#define BW_20MHZ	BIT(2)
+
+#define RATE_BITMAP_ALL	0xFFFFF
+
+#define RATE_RRSR_CCK_ONLY_1M	0xFFFF1
+
+#define TSFRST	BIT(0)
+#define DIS_GCLK	BIT(1)
+#define PAD_SEL	BIT(2)
+#define PWR_ST	BIT(6)
+#define PWRBIT_OW_EN	BIT(7)
+#define ACRC	BIT(8)
+#define CFENDFORM	BIT(9)
+#define ICV	BIT(10)
+
+#define AAP	BIT(0)
+#define APM	BIT(1)
+#define AM	BIT(2)
+#define AB	BIT(3)
+#define ADD3	BIT(4)
+#define APWRMGT	BIT(5)
+#define CBSSID	BIT(6)
+#define CBSSID_DATA	BIT(6)
+#define CBSSID_BCN	BIT(7)
+#define ACRC32	BIT(8)
+#define AICV	BIT(9)
+#define ADF	BIT(11)
+#define ACF	BIT(12)
+#define AMF	BIT(13)
+#define HTC_LOC_CTRL	BIT(14)
+#define UC_DATA_EN	BIT(16)
+#define BM_DATA_EN	BIT(17)
+#define MFBEN	BIT(22)
+#define LSIGEN	BIT(23)
+#define EN_MBID	BIT(24)
+#define APP_BASSN	BIT(27)
+#define APP_PHYSTS	BIT(28)
+#define APP_ICV	BIT(29)
+#define APP_MIC	BIT(30)
+#define APP_FCS	BIT(31)
+
+#define _MIN_SPACE(x) ((x) & 0x7)
+#define _SHORT_GI_PADDING(x) (((x) & 0x1F) << 3)
+
+#define RXERR_TYPE_OFDM_PPDU	0
+#define RXERR_TYPE_OFDM_FALSE_ALARM	1
+#define RXERR_TYPE_OFDM_MPDU_OK	2
+#define RXERR_TYPE_OFDM_MPDU_FAIL	3
+#define RXERR_TYPE_CCK_PPDU	4
+#define RXERR_TYPE_CCK_FALSE_ALARM	5
+#define RXERR_TYPE_CCK_MPDU_OK	6
+#define RXERR_TYPE_CCK_MPDU_FAIL	7
+#define RXERR_TYPE_HT_PPDU	8
+#define RXERR_TYPE_HT_FALSE_ALARM	9
+#define RXERR_TYPE_HT_MPDU_TOTAL	10
+#define RXERR_TYPE_HT_MPDU_OK	11
+#define RXERR_TYPE_HT_MPDU_FAIL	12
+#define RXERR_TYPE_RX_FULL_DROP	15
+
+#define RXERR_COUNTER_MASK	0xFFFFF
+#define RXERR_RPT_RST	BIT(27)
+#define _RXERR_RPT_SEL(type) ((type) << 28)
+
+#define SCR_TX_USE_DK	BIT(0)
+#define SCR_RX_USE_DK	BIT(1)
+#define SCR_TX_ENC_ENABLE	BIT(2)
+#define SRC_RX_DEC_ENABLE	BIT(3)
+#define SCR_SK_BY_A2	BIT(4)
+#define SCR_NO_SKMC	BIT(5)
+#define SCR_TXBCUSEDK	BIT(6)
+#define SCR_RXBCUSEDK	BIT(7)
+
+#define USB_IS_HIGH_SPEED	0
+#define USB_IS_FULL_SPEED	1
+#define USB_SPEED_MASK	BIT(5)
+
+#define USB_NORMAL_SIE_EP_MASK	0xF
+#define USB_NORMAL_SIE_EP_SHIFT	4
+
+#define USB_TEST_EP_MASK	0x30
+#define USB_TEST_EP_SHIFT	4
+
+#define USB_AGG_EN	BIT(3)
+
+#define MAC_ADDR_LEN	6
+#define LAST_ENTRY_OF_TX_PKT_BUFFER	175
+
+#define POLLING_LLT_THRESHOLD	20
+#define POLLING_READY_TIMEOUT_COUNT	3000
+
+#define MAX_MSS_DENSITY_2T	0x13
+#define MAX_MSS_DENSITY_1T	0x0A
+
+#define EPROM_CMD_OPERATING_MODE_MASK ((1 << 7) | (1 << 6))
+#define EPROM_CMD_CONFIG	0x3
+#define EPROM_CMD_LOAD	1
+
+#define HAL_8822B_HW_GPIO_WPS_BIT	BIT(2)
+
+/*-----------------------------------------------------
+ * BB / RF register
+ *-----------------------------------------------------
+ */
+
+#define RFPGA0_XA_HSSIPARAMETER1	0x820
+#define RFPGA0_XA_HSSIPARAMETER2	0x824
+#define RFPGA0_XB_HSSIPARAMETER1	0x828
+#define RFPGA0_XB_HSSIPARAMETER2 0x82c
+#define RCCAONSEC	0x838
+
+#define RFPGA0_XA_LSSIPARAMETER	0x840
+#define RFPGA0_XB_LSSIPARAMETER	0x844
+#define RL1PEAKTH	0x848
+
+#define RFPGA0_RFWAKEUPPARAMETER	0x850
+#define RFPGA0_RFSLEEPUPPARAMETER	0x854
+
+#define RFPGA0_XAB_SWITCHCONTROL	0x858
+#define RFPGA0_XCD_SWITCHCONTROL 0x85c
+
+#define RFPGA0_XA_RFINTERFACEOE	0x860
+#define RFC_AREA	0x860
+#define RFPGA0_XB_RFINTERFACEOE	0x864
+
+#define RFPGA0_XAB_RFINTERFACESW	0x870
+#define RFPGA0_XCD_RFINTERFACESW	0x874
+
+#define RFPGA0_XAB_RF_PARA_METER	0x878
+#define RFPGA0_XCD_RF_PARA_METER 0x87c
+
+#define RFPGA0_ANALOGPARAMETER1	0x880
+#define RFPGA0_ANALOGPARAMETER2	0x884
+#define RFPGA0_ANALOGPARAMETER3	0x888
+#define RFPGA0_ANALOGPARAMETER4 0x88c
+
+#define RFPGA0_XA_LSSIREADBACK 0x8a0
+#define RFPGA0_XB_LSSIREADBACK 0x8a4
+#define RFPGA0_XC_LSSIREADBACK 0x8a8
+/*#define	RFPGA0_XD_LSSIREADBACK			0x8ac*/
+#define RRFMOD 0x8ac
+#define RHSSIREAD_8822BE 0x8b0
+
+#define RFPGA0_PSDREPORT 0x8b4
+#define TRANSCEIVEA_HSPI_READBACK 0x8b8
+#define TRANSCEIVEB_HSPI_READBACK 0x8bc
+/*#define	REG_SC_CNT_8822B				0x8c4*/
+#define RADC_BUF_CLK 0x8c4
+#define RFPGA0_XAB_RFINTERFACERB 0x8e0
+#define RFPGA0_XCD_RFINTERFACERB 0x8e4
+
+/* PageB(0xB00) */
+
+/*Page C*/
+
+#define RA_TXPWRTRAING 0xc54
+#define RB_TXPWRTRAING 0xe54
+
+#define RA_LSSIWRITE_8822B 0xc90
+#define RB_LSSIWRITE_8822B 0xe90
+
+#define RA_PIREAD_8822B 0xd04
+#define RB_PIREAD_8822B 0xd44
+#define RA_SIREAD_8822B 0xd08
+#define RB_SIREAD_8822B 0xd48
+
+#define RZEBRA1_HSSIENABLE	0x0
+#define RZEBRA1_TRXENABLE1	0x1
+#define RZEBRA1_TRXENABLE2	0x2
+#define RZEBRA1_AGC	0x4
+#define RZEBRA1_CHARGEPUMP	0x5
+#define RZEBRA1_CHANNEL	0x7
+
+#define RZEBRA1_TXGAIN	0x8
+#define RZEBRA1_TXLPF	0x9
+#define RZEBRA1_RXLPF 0xb
+#define RZEBRA1_RXHPFCORNER 0xc
+
+#define RGLOBALCTRL	0
+#define RRTL8256_TXLPF	19
+#define RRTL8256_RXLPF	11
+#define RRTL8258_TXLPF	0x11
+#define RRTL8258_RXLPF	0x13
+#define RRTL8258_RSSILPF 0xa
+
+#define RF_AC	0x00
+
+#define RF_IQADJ_G1	0x01
+#define RF_IQADJ_G2	0x02
+#define RF_POW_TRSW	0x05
+
+#define RF_GAIN_RX	0x06
+#define RF_GAIN_TX	0x07
+
+#define RF_TXM_IDAC	0x08
+#define RF_BS_IQGEN	0x0F
+
+#define RF_MODE1	0x10
+#define RF_MODE2	0x11
+
+#define RF_RX_AGC_HP	0x12
+#define RF_TX_AGC	0x13
+#define RF_BIAS	0x14
+#define RF_IPA	0x15
+#define RF_POW_ABILITY	0x17
+#define RF_MODE_AG	0x18
+#define RRFCHANNEL	0x18
+#define RF_CHNLBW	0x18
+#define RF_TOP	0x19
+
+#define RF_RX_G1	0x1A
+#define RF_RX_G2	0x1B
+
+#define RF_RX_BB2	0x1C
+#define RF_RX_BB1	0x1D
+
+#define RF_RCK1	0x1E
+#define RF_RCK2	0x1F
+
+#define RF_TX_G1	0x20
+#define RF_TX_G2	0x21
+#define RF_TX_G3	0x22
+
+#define RF_TX_BB1	0x23
+#define RF_T_METER	0x42
+
+#define RF_SYN_G1	0x25
+#define RF_SYN_G2	0x26
+#define RF_SYN_G3	0x27
+#define RF_SYN_G4	0x28
+#define RF_SYN_G5	0x29
+#define RF_SYN_G6	0x2A
+#define RF_SYN_G7	0x2B
+#define RF_SYN_G8	0x2C
+
+#define RF_RCK_OS	0x30
+#define RF_TXPA_G1	0x31
+#define RF_TXPA_G2	0x32
+#define RF_TXPA_G3	0x33
+
+#define RF_TX_BIAS_A	0x35
+#define RF_TX_BIAS_D	0x36
+#define RF_LOBF_9	0x38
+#define RF_RXRF_A3	0x3C
+#define RF_TRSW	0x3F
+
+#define RF_TXRF_A2	0x41
+#define RF_TXPA_G4	0x46
+#define RF_TXPA_A4	0x4B
+
+#define RF_APK	0x63
+
+#define RF_WE_LUT	0xEF
+
+#define BBBRESETB	0x100
+#define BGLOBALRESETB	0x200
+#define BOFDMTXSTART	0x4
+#define BCCKTXSTART	0x8
+#define BCRC32DEBUG	0x100
+#define BPMACLOOPBACK	0x10
+#define BTXLSIG 0xffffff
+#define BOFDMTXRATE 0xf
+#define BOFDMTXRESERVED	0x10
+#define BOFDMTXLENGTH 0x1ffe0
+#define BOFDMTXPARITY	0x20000
+#define BTXHTSIG1 0xffffff
+#define BTXHTMCSRATE 0x7f
+#define BTXHTBW	0x80
+#define BTXHTLENGTH 0xffff00
+#define BTXHTSIG2 0xffffff
+#define BTXHTSMOOTHING	0x1
+#define BTXHTSOUNDING	0x2
+#define BTXHTRESERVED	0x4
+#define BTXHTAGGREATION	0x8
+#define BTXHTSTBC	0x30
+#define BTXHTADVANCECODING	0x40
+#define BTXHTSHORTGI	0x80
+#define BTXHTNUMBERHT_LTF	0x300
+#define BTXHTCRC8 0x3fc00
+#define BCOUNTERRESET	0x10000
+#define BNUMOFOFDMTX 0xffff
+#define BNUMOFCCKTX 0xffff0000
+#define BTXIDLEINTERVAL 0xffff
+#define BOFDMSERVICE 0xffff0000
+#define BTXMACHEADER 0xffffffff
+#define BTXDATAINIT 0xff
+#define BTXHTMODE	0x100
+#define BTXDATATYPE	0x30000
+#define BTXRANDOMSEED 0xffffffff
+#define BCCKTXPREAMBLE	0x1
+#define BCCKTXSFD 0xffff0000
+#define BCCKTXSIG 0xff
+#define BCCKTXSERVICE 0xff00
+#define BCCKLENGTHEXT	0x8000
+#define BCCKTXLENGHT 0xffff0000
+#define BCCKTXCRC16 0xffff
+#define BCCKTXSTATUS	0x1
+#define BOFDMTXSTATUS	0x2
+#define IS_BB_REG_OFFSET_92S(_offset) ((_offset >= 0x800) && (_offset <= 0xfff))
+
+#define BRFMOD	0x1
+#define BJAPANMODE	0x2
+#define BCCKTXSC	0x30
+/* Block & Path enable*/
+#define ROFDMCCKEN	0x808
+#define BCCKEN	0x10000000
+#define BOFDMEN	0x20000000
+/* Rx antenna*/
+#define RRXPATH	0x808
+#define BRXPATH 0xff
+/* Tx antenna*/
+#define RTXPATH 0x80c
+#define BTXPATH 0x0fffffff
+/* for cck rx path selection*/
+#define RCCK_RX 0xa04
+#define BCCK_RX 0x0c000000
+/* Use LSIG for VHT length*/
+#define RVHTLEN_USE_LSIG 0x8c3
+
+#define BOFDMRXADCPHASE	0x10000
+#define BOFDMTXDACPHASE	0x40000
+#define BXATXAGC 0x3f
+
+#define BXBTXAGC 0xf00
+#define BXCTXAGC 0xf000
+#define BXDTXAGC 0xf0000
+
+#define BPASTART 0xf0000000
+#define BTRSTART 0x00f00000
+#define BRFSTART 0x0000f000
+#define BBBSTART 0x000000f0
+#define BBBCCKSTART 0x0000000f
+#define BPAEND 0xf
+#define BTREND 0x0f000000
+#define BRFEND 0x000f0000
+#define BCCAMASK 0x000000f0
+#define BR2RCCAMASK 0x00000f00
+#define BHSSI_R2TDELAY 0xf8000000
+#define BHSSI_T2RDELAY 0xf80000
+#define BCONTXHSSI	0x400
+#define BIGFROMCCK	0x200
+#define BAGCADDRESS 0x3f
+#define BRXHPTX	0x7000
+#define BRXHP2RX	0x38000
+#define BRXHPCCKINI 0xc0000
+#define BAGCTXCODE 0xc00000
+#define BAGCRXCODE	0x300000
+
+#define B3WIREDATALENGTH	0x800
+#define B3WIREADDREAALENGTH	0x400
+
+#define B3WIRERFPOWERDOWN	0x1
+#define B5GPAPEPOLARITY	0x40000000
+#define B2GPAPEPOLARITY	0x80000000
+#define BRFSW_TXDEFAULTANT	0x3
+#define BRFSW_TXOPTIONANT	0x30
+#define BRFSW_RXDEFAULTANT	0x300
+#define BRFSW_RXOPTIONANT	0x3000
+#define BRFSI_3WIREDATA	0x1
+#define BRFSI_3WIRECLOCK	0x2
+#define BRFSI_3WIRELOAD	0x4
+#define BRFSI_3WIRERW	0x8
+#define BRFSI_3WIRE 0xf
+
+#define BRFSI_RFENV	0x10
+
+#define BRFSI_TRSW	0x20
+#define BRFSI_TRSWB	0x40
+#define BRFSI_ANTSW	0x100
+#define BRFSI_ANTSWB	0x200
+#define BRFSI_PAPE	0x400
+#define BRFSI_PAPE5G	0x800
+#define BBANDSELECT	0x1
+#define BHTSIG2_GI	0x80
+#define BHTSIG2_SMOOTHING	0x01
+#define BHTSIG2_SOUNDING	0x02
+#define BHTSIG2_AGGREATON	0x08
+#define BHTSIG2_STBC	0x30
+#define BHTSIG2_ADVCODING	0x40
+#define BHTSIG2_NUMOFHTLTF	0x300
+#define BHTSIG2_CRC8 0x3fc
+#define BHTSIG1_MCS 0x7f
+#define BHTSIG1_BANDWIDTH	0x80
+#define BHTSIG1_HTLENGTH 0xffff
+#define BLSIG_RATE 0xf
+#define BLSIG_RESERVED	0x10
+#define BLSIG_LENGTH 0x1fffe
+#define BLSIG_PARITY	0x20
+#define BCCKRXPHASE	0x4
+
+#define BLSSIREADADDRESS 0x7f800000
+#define BLSSIREADEDGE	0x80000000
+
+#define BLSSIREADBACKDATA 0xfffff
+
+#define BLSSIREADOKFLAG	0x1000
+#define BCCKSAMPLERATE	0x8
+#define BREGULATOR0STANDBY	0x1
+#define BREGULATORPLLSTANDBY	0x2
+#define BREGULATOR1STANDBY	0x4
+#define BPLLPOWERUP	0x8
+#define BDPLLPOWERUP	0x10
+#define BDA10POWERUP	0x20
+#define BAD7POWERUP	0x200
+#define BDA6POWERUP	0x2000
+#define BXTALPOWERUP	0x4000
+#define B40MDCLKPOWERUP	0x8000
+#define BDA6DEBUGMODE	0x20000
+#define BDA6SWING	0x380000
+
+#define BADCLKPHASE	0x4000000
+#define B80MCLKDELAY	0x18000000
+#define BAFEWATCHDOGENABLE	0x20000000
+
+#define BXTALCAP01 0xc0000000
+#define BXTALCAP23	0x3
+#define BXTALCAP92X 0x0f000000
+#define BXTALCAP 0x0f000000
+
+#define BINTDIFCLKENABLE	0x400
+#define BEXTSIGCLKENABLE	0x800
+#define BBANDGAP_MBIAS_POWERUP	0x10000
+#define BAD11SH_GAIN 0xc0000
+#define BAD11NPUT_RANGE	0x700000
+#define BAD110P_CURRENT	0x3800000
+#define BLPATH_LOOPBACK	0x4000000
+#define BQPATH_LOOPBACK	0x8000000
+#define BAFE_LOOPBACK	0x10000000
+#define BDA10_SWING 0x7e0
+#define BDA10_REVERSE	0x800
+#define BDA_CLK_SOURCE	0x1000
+#define BDA7INPUT_RANGE	0x6000
+#define BDA7_GAIN	0x38000
+#define BDA7OUTPUT_CM_MODE	0x40000
+#define BDA7INPUT_CM_MODE	0x380000
+#define BDA7CURRENT 0xc00000
+#define BREGULATOR_ADJUST	0x7000000
+#define BAD11POWERUP_ATTX	0x1
+#define BDA10PS_ATTX	0x10
+#define BAD11POWERUP_ATRX	0x100
+#define BDA10PS_ATRX	0x1000
+#define BCCKRX_AGC_FORMAT	0x200
+#define BPSDFFT_SAMPLE_POINT 0xc000
+#define BPSD_AVERAGE_NUM	0x3000
+#define BIQPATH_CONTROL 0xc00
+#define BPSD_FREQ 0x3ff
+#define BPSD_ANTENNA_PATH	0x30
+#define BPSD_IQ_SWITCH	0x40
+#define BPSD_RX_TRIGGER	0x400000
+#define BPSD_TX_TRIGGER	0x80000000
+#define BPSD_SINE_TONE_SCALE 0x7f000000
+#define BPSD_REPORT 0xffff
+
+#define BOFDM_TXSC	0x30000000
+#define BCCK_TXON	0x1
+#define BOFDM_TXON	0x2
+#define BDEBUG_PAGE 0xfff
+#define BDEBUG_ITEM 0xff
+#define BANTL	0x10
+#define BANT_NONHT	0x100
+#define BANT_HT1	0x1000
+#define BANT_HT2	0x10000
+#define BANT_HT1S1	0x100000
+#define BANT_NONHTS1	0x1000000
+
+#define BCCK_BBMODE	0x3
+#define BCCK_TXPOWERSAVING	0x80
+#define BCCK_RXPOWERSAVING	0x40
+
+#define BCCK_SIDEBAND	0x10
+
+#define BCCK_SCRAMBLE	0x8
+#define BCCK_ANTDIVERSITY	0x8000
+#define BCCK_CARRIER_RECOVERY	0x4000
+#define BCCK_TXRATE	0x3000
+#define BCCK_DCCANCEL	0x0800
+#define BCCK_ISICANCEL	0x0400
+#define BCCK_MATCH_FILTER	0x0200
+#define BCCK_EQUALIZER	0x0100
+#define BCCK_PREAMBLE_DETECT	0x800000
+#define BCCK_FAST_FALSECCA	0x400000
+#define BCCK_CH_ESTSTART	0x300000
+#define BCCK_CCA_COUNT	0x080000
+#define BCCK_CS_LIM	0x070000
+#define BCCK_BIST_MODE	0x80000000
+#define BCCK_CCAMASK	0x40000000
+#define BCCK_TX_DAC_PHASE	0x4
+#define BCCK_RX_ADC_PHASE	0x20000000
+#define BCCKR_CP_MODE	0x0100
+#define BCCK_TXDC_OFFSET 0xf0
+#define BCCK_RXDC_OFFSET 0xf
+#define BCCK_CCA_MODE 0xc000
+#define BCCK_FALSECS_LIM 0x3f00
+#define BCCK_CS_RATIO 0xc00000
+#define BCCK_CORGBIT_SEL	0x300000
+#define BCCK_PD_LIM 0x0f0000
+#define BCCK_NEWCCA	0x80000000
+#define BCCK_RXHP_OF_IG	0x8000
+#define BCCK_RXIG 0x7f00
+#define BCCK_LNA_POLARITY	0x800000
+#define BCCK_RX1ST_BAIN 0x7f0000
+#define BCCK_RF_EXTEND	0x20000000
+#define BCCK_RXAGC_SATLEVEL 0x1f000000
+#define BCCK_RXAGC_SATCOUNT 0xe0
+#define BCCK_RX_RF_SETTLE 0x1f
+#define BCCK_FIXED_RXAGC	0x8000
+#define BCCK_ANTENNA_POLARITY	0x2000
+#define BCCK_TXFILTER_TYPE 0x0c00
+#define BCCK_RXAGC_REPORTTYPE	0x0300
+#define BCCK_RXDAGC_EN	0x80000000
+#define BCCK_RXDAGC_PERIOD	0x20000000
+#define BCCK_RXDAGC_SATLEVEL 0x1f000000
+#define BCCK_TIMING_RECOVERY	0x800000
+#define BCCK_TXC0 0x3f0000
+#define BCCK_TXC1 0x3f000000
+#define BCCK_TXC2 0x3f
+#define BCCK_TXC3 0x3f00
+#define BCCK_TXC4 0x3f0000
+#define BCCK_TXC5 0x3f000000
+#define BCCK_TXC6 0x3f
+#define BCCK_TXC7 0x3f00
+#define BCCK_DEBUGPORT 0xff0000
+#define BCCK_DAC_DEBUG 0x0f000000
+#define BCCK_FALSEALARM_ENABLE	0x8000
+#define BCCK_FALSEALARM_READ	0x4000
+#define BCCK_TRSSI 0x7f
+#define BCCK_RXAGC_REPORT 0xfe
+#define BCCK_RXREPORT_ANTSEL	0x80000000
+#define BCCK_RXREPORT_MFOFF	0x40000000
+#define BCCK_RXREPORT_SQLOSS	0x20000000
+#define BCCK_RXREPORT_PKTLOSS	0x10000000
+#define BCCK_RXREPORT_LOCKEDBIT	0x08000000
+#define BCCK_RXREPORT_RATEERROR	0x04000000
+#define BCCK_RXREPORT_RXRATE	0x03000000
+#define BCCK_RXFA_COUNTER_LOWER 0xff
+#define BCCK_RXFA_COUNTER_UPPER 0xff000000
+#define BCCK_RXHPAGC_START 0xe000
+#define BCCK_RXHPAGC_FINAL 0x1c00
+#define BCCK_RXFALSEALARM_ENABLE	0x8000
+#define BCCK_FACOUNTER_FREEZE	0x4000
+#define BCCK_TXPATH_SEL	0x10000000
+#define BCCK_DEFAULT_RXPATH 0xc000000
+#define BCCK_OPTION_RXPATH	0x3000000
+
+#define BNUM_OFSTF	0x3
+#define BSHIFT_L 0xc0
+#define BGI_TH 0xc
+#define BRXPATH_A	0x1
+#define BRXPATH_B	0x2
+#define BRXPATH_C	0x4
+#define BRXPATH_D	0x8
+#define BTXPATH_A	0x1
+#define BTXPATH_B	0x2
+#define BTXPATH_C	0x4
+#define BTXPATH_D	0x8
+#define BTRSSI_FREQ	0x200
+#define BADC_BACKOFF	0x3000
+#define BDFIR_BACKOFF 0xc000
+#define BTRSSI_LATCH_PHASE	0x10000
+#define BRX_LDC_OFFSET 0xff
+#define BRX_QDC_OFFSET 0xff00
+#define BRX_DFIR_MODE	0x1800000
+#define BRX_DCNF_TYPE 0xe000000
+#define BRXIQIMB_A 0x3ff
+#define BRXIQIMB_B 0xfc00
+#define BRXIQIMB_C 0x3f0000
+#define BRXIQIMB_D 0xffc00000
+#define BDC_DC_NOTCH	0x60000
+#define BRXNB_NOTCH 0x1f000000
+#define BPD_TH 0xf
+#define BPD_TH_OPT2 0xc000
+#define BPWED_TH	0x700
+#define BIFMF_WIN_L	0x800
+#define BPD_OPTION	0x1000
+#define BMF_WIN_L 0xe000
+#define BBW_SEARCH_L	0x30000
+#define BWIN_ENH_L 0xc0000
+#define BBW_TH	0x700000
+#define BED_TH2	0x3800000
+#define BBW_OPTION	0x4000000
+#define BRADIO_TH	0x18000000
+#define BWINDOW_L 0xe0000000
+#define BSBD_OPTION	0x1
+#define BFRAME_TH 0x1c
+#define BFS_OPTION	0x60
+#define BDC_SLOPE_CHECK	0x80
+#define BFGUARD_COUNTER_DC_L 0xe00
+#define BFRAME_WEIGHT_SHORT	0x7000
+#define BSUB_TUNE 0xe00000
+#define BFRAME_DC_LENGTH 0xe000000
+#define BSBD_START_OFFSET	0x30000000
+#define BFRAME_TH_2	0x7
+#define BFRAME_GI2_TH	0x38
+#define BGI2_SYNC_EN	0x40
+#define BSARCH_SHORT_EARLY	0x300
+#define BSARCH_SHORT_LATE 0xc00
+#define BSARCH_GI2_LATE	0x70000
+#define BCFOANTSUM	0x1
+#define BCFOACC	0x2
+#define BCFOSTARTOFFSET 0xc
+#define BCFOLOOPBACK	0x70
+#define BCFOSUMWEIGHT	0x80
+#define BDAGCENABLE	0x10000
+#define BTXIQIMB_A 0x3ff
+#define BTXIQIMB_b 0xfc00
+#define BTXIQIMB_C 0x3f0000
+#define BTXIQIMB_D 0xffc00000
+#define BTXIDCOFFSET 0xff
+#define BTXIQDCOFFSET 0xff00
+#define BTXDFIRMODE	0x10000
+#define BTXPESUDO_NOISEON	0x4000000
+#define BTXPESUDO_NOISE_A 0xff
+#define BTXPESUDO_NOISE_B 0xff00
+#define BTXPESUDO_NOISE_C 0xff0000
+#define BTXPESUDO_NOISE_D 0xff000000
+#define BCCA_DROPOPTION	0x20000
+#define BCCA_DROPTHRES 0xfff00000
+#define BEDCCA_H 0xf
+#define BEDCCA_L 0xf0
+#define BLAMBDA_ED	0x300
+#define BRX_INITIALGAIN 0x7f
+#define BRX_ANTDIV_EN	0x80
+#define BRX_AGC_ADDRESS_FOR_LNA 0x7f00
+#define BRX_HIGHPOWER_FLOW	0x8000
+#define BRX_AGC_FREEZE_THRES 0xc0000
+#define BRX_FREEZESTEP_AGC1	0x300000
+#define BRX_FREEZESTEP_AGC2 0xc00000
+#define BRX_FREEZESTEP_AGC3	0x3000000
+#define BRX_FREEZESTEP_AGC0 0xc000000
+#define BRXRSSI_CMP_EN	0x10000000
+#define BRXQUICK_AGCEN	0x20000000
+#define BRXAGC_FREEZE_THRES_MODE	0x40000000
+#define BRX_OVERFLOW_CHECKTYPE	0x80000000
+#define BRX_AGCSHIFT 0x7f
+#define BTRSW_TRI_ONLY	0x80
+#define BPOWER_THRES	0x300
+#define BRXAGC_EN	0x1
+#define BRXAGC_TOGETHER_EN	0x2
+#define BRXAGC_MIN	0x4
+#define BRXHP_INI	0x7
+#define BRXHP_TRLNA	0x70
+#define BRXHP_RSSI	0x700
+#define BRXHP_BBP1	0x7000
+#define BRXHP_BBP2	0x70000
+#define BRXHP_BBP3	0x700000
+#define BRSSI_H 0x7f0000
+#define BRSSI_GEN 0x7f000000
+#define BRXSETTLE_TRSW	0x7
+#define BRXSETTLE_LNA	0x38
+#define BRXSETTLE_RSSI 0x1c0
+#define BRXSETTLE_BBP 0xe00
+#define BRXSETTLE_RXHP	0x7000
+#define BRXSETTLE_ANTSW_RSSI	0x38000
+#define BRXSETTLE_ANTSW 0xc0000
+#define BRXPROCESS_TIME_DAGC	0x300000
+#define BRXSETTLE_HSSI	0x400000
+#define BRXPROCESS_TIME_BBPPW	0x800000
+#define BRXANTENNA_POWER_SHIFT	0x3000000
+#define BRSSI_TABLE_SELECT 0xc000000
+#define BRXHP_FINAL	0x7000000
+#define BRXHPSETTLE_BBP	0x7
+#define BRXHTSETTLE_HSSI	0x8
+#define BRXHTSETTLE_RXHP	0x70
+#define BRXHTSETTLE_BBPPW	0x80
+#define BRXHTSETTLE_IDLE	0x300
+#define BRXHTSETTLE_RESERVED 0x1c00
+#define BRXHT_RXHP_EN	0x8000
+#define BRXAGC_FREEZE_THRES	0x30000
+#define BRXAGC_TOGETHEREN	0x40000
+#define BRXHTAGC_MIN	0x80000
+#define BRXHTAGC_EN	0x100000
+#define BRXHTDAGC_EN	0x200000
+#define BRXHT_RXHP_BBP 0x1c00000
+#define BRXHT_RXHP_FINAL 0xe0000000
+#define BRXPW_RADIO_TH	0x3
+#define BRXPW_RADIO_EN	0x4
+#define BRXMF_HOLD	0x3800
+#define BRXPD_DELAY_TH1	0x38
+#define BRXPD_DELAY_TH2 0x1c0
+#define BRXPD_DC_COUNT_MAX	0x600
+#define BRXPD_DELAY_TH	0x8000
+#define BRXPROCESS_DELAY 0xf0000
+#define BRXSEARCHRANGE_GI2_EARLY	0x700000
+#define BRXFRAME_FUARD_COUNTER_L	0x3800000
+#define BRXSGI_GUARD_L 0xc000000
+#define BRXSGI_SEARCH_L	0x30000000
+#define BRXSGI_TH 0xc0000000
+#define BDFSCNT0 0xff
+#define BDFSCNT1 0xff00
+#define BDFSFLAG 0xf0000
+#define BMF_WEIGHT_SUM	0x300000
+#define BMINIDX_TH 0x7f000000
+#define BDAFORMAT	0x40000
+#define BTXCH_EMU_ENABLE	0x01000000
+#define BTRSW_ISOLATION_A 0x7f
+#define BTRSW_ISOLATION_B 0x7f00
+#define BTRSW_ISOLATION_C 0x7f0000
+#define BTRSW_ISOLATION_D 0x7f000000
+#define BEXT_LNA_GAIN 0x7c00
+
+#define BSTBC_EN	0x4
+#define BANTENNA_MAPPING	0x10
+#define BNSS	0x20
+#define BCFO_ANTSUM_ID	0x200
+#define BPHY_COUNTER_RESET	0x8000000
+#define BCFO_REPORT_GET	0x4000000
+#define BOFDM_CONTINUE_TX	0x10000000
+#define BOFDM_SINGLE_CARRIER	0x20000000
+#define BOFDM_SINGLE_TONE	0x40000000
+#define BHT_DETECT	0x100
+#define BCFOEN	0x10000
+#define BCFOVALUE 0xfff00000
+#define BSIGTONE_RE 0x3f
+#define BSIGTONE_IM 0x7f00
+#define BCOUNTER_CCA 0xffff
+#define BCOUNTER_PARITYFAIL 0xffff0000
+#define BCOUNTER_RATEILLEGAL 0xffff
+#define BCOUNTER_CRC8FAIL 0xffff0000
+#define BCOUNTER_MCSNOSUPPORT 0xffff
+#define BCOUNTER_FASTSYNC 0xffff
+#define BSHORTCFO 0xfff
+#define BSHORTCFOT_LENGTH	12
+#define BSHORTCFOF_LENGTH	11
+#define BLONGCFO 0x7ff
+#define BLONGCFOT_LENGTH	11
+#define BLONGCFOF_LENGTH	11
+#define BTAILCFO 0x1fff
+#define BTAILCFOT_LENGTH	13
+#define BTAILCFOF_LENGTH	12
+#define BNOISE_EN_PWDB 0xffff
+#define BCC_POWER_DB 0xffff0000
+#define BMOISE_PWDB 0xffff
+#define BPOWERMEAST_LENGTH	10
+#define BPOWERMEASF_LENGTH	3
+#define BRX_HT_BW	0x1
+#define BRXSC	0x6
+#define BRX_HT	0x8
+#define BNB_INTF_DET_ON	0x1
+#define BINTF_WIN_LEN_CFG	0x30
+#define BNB_INTF_TH_CFG 0x1c0
+#define BRFGAIN 0x3f
+#define BTABLESEL	0x40
+#define BTRSW	0x80
+#define BRXSNR_A 0xff
+#define BRXSNR_B 0xff00
+#define BRXSNR_C 0xff0000
+#define BRXSNR_D 0xff000000
+#define BSNR_EVMT_LENGTH	8
+#define BSNR_EVMF_LENGTH	1
+#define BCSI1ST 0xff
+#define BCSI2ND 0xff00
+#define BRXEVM1ST 0xff0000
+#define BRXEVM2ND 0xff000000
+#define BSIGEVM 0xff
+#define BPWDB 0xff00
+#define BSGIEN	0x10000
+
+#define BSFACTOR_QMA1 0xf
+#define BSFACTOR_QMA2 0xf0
+#define BSFACTOR_QMA3 0xf00
+#define BSFACTOR_QMA4 0xf000
+#define BSFACTOR_QMA5 0xf0000
+#define BSFACTOR_QMA6 0xf0000
+#define BSFACTOR_QMA7 0xf00000
+#define BSFACTOR_QMA8 0xf000000
+#define BSFACTOR_QMA9 0xf0000000
+#define BCSI_SCHEME	0x100000
+
+#define BNOISE_LVL_TOP_SET	0x3
+#define BCHSMOOTH	0x4
+#define BCHSMOOTH_CFG1	0x38
+#define BCHSMOOTH_CFG2 0x1c0
+#define BCHSMOOTH_CFG3 0xe00
+#define BCHSMOOTH_CFG4	0x7000
+#define BMRCMODE	0x800000
+#define BTHEVMCFG	0x7000000
+
+#define BLOOP_FIT_TYPE	0x1
+#define BUPD_CFO	0x40
+#define BUPD_CFO_OFFDATA	0x80
+#define BADV_UPD_CFO	0x100
+#define BADV_TIME_CTRL	0x800
+#define BUPD_CLKO	0x1000
+#define BFC	0x6000
+#define BTRACKING_MODE	0x8000
+#define BPHCMP_ENABLE	0x10000
+#define BUPD_CLKO_LTF	0x20000
+#define BCOM_CH_CFO	0x40000
+#define BCSI_ESTI_MODE	0x80000
+#define BADV_UPD_EQZ	0x100000
+#define BUCHCFG	0x7000000
+#define BUPDEQZ	0x8000000
+
+#define BRX_PESUDO_NOISE_ON	0x20000000
+#define BRX_PESUDO_NOISE_A 0xff
+#define BRX_PESUDO_NOISE_B 0xff00
+#define BRX_PESUDO_NOISE_C 0xff0000
+#define BRX_PESUDO_NOISE_D 0xff000000
+#define BRX_PESUDO_NOISESTATE_A 0xffff
+#define BRX_PESUDO_NOISESTATE_B 0xffff0000
+#define BRX_PESUDO_NOISESTATE_C 0xffff
+#define BRX_PESUDO_NOISESTATE_D 0xffff0000
+
+#define BZEBRA1_HSSIENABLE	0x8
+#define BZEBRA1_TRXCONTROL 0xc00
+#define BZEBRA1_TRXGAINSETTING 0x07f
+#define BZEBRA1_RXCOUNTER 0xc00
+#define BZEBRA1_TXCHANGEPUMP	0x38
+#define BZEBRA1_RXCHANGEPUMP	0x7
+#define BZEBRA1_CHANNEL_NUM 0xf80
+#define BZEBRA1_TXLPFBW	0x400
+#define BZEBRA1_RXLPFBW	0x600
+
+#define BRTL8256REG_MODE_CTRL1	0x100
+#define BRTL8256REG_MODE_CTRL0	0x40
+#define BRTL8256REG_TXLPFBW	0x18
+#define BRTL8256REG_RXLPFBW	0x600
+
+#define BRTL8258_TXLPFBW 0xc
+#define BRTL8258_RXLPFBW 0xc00
+#define BRTL8258_RSSILPFBW 0xc0
+
+#define BBYTE0	0x1
+#define BBYTE1	0x2
+#define BBYTE2	0x4
+#define BBYTE3	0x8
+#define BWORD0	0x3
+#define BWORD1 0xc
+#define BWORD 0xf
+
+#define MASKBYTE0 0xff
+#define MASKBYTE1 0xff00
+#define MASKBYTE2 0xff0000
+#define MASKBYTE3 0xff000000
+#define MASKHWORD 0xffff0000
+#define MASKLWORD 0x0000ffff
+#define MASKDWORD 0xffffffff
+#define MASK12BITS 0xfff
+#define MASKH4BITS 0xf0000000
+#define MASKOFDM_D 0xffc00000
+#define MASKCCK 0x3f3f3f3f
+
+#define MASK4BITS 0x0f
+#define MASK20BITS 0xfffff
+#define RFREG_OFFSET_MASK 0xfffff
+
+#define BMASKBYTE0 0xff
+#define BMASKBYTE1 0xff00
+#define BMASKBYTE2 0xff0000
+#define BMASKBYTE3 0xff000000
+#define BMASKHWORD 0xffff0000
+#define BMASKLWORD 0x0000ffff
+#define BMASKDWORD 0xffffffff
+#define BMASK12BITS 0xfff
+#define BMASKH4BITS 0xf0000000
+#define BMASKOFDM_D 0xffc00000
+#define BMASKCCK 0x3f3f3f3f
+
+#define BRFREGOFFSETMASK 0xfffff
+
+/* WOL bit information */
+#define WOL_REASON_PTK_UPDATE	BIT(0)
+#define WOL_REASON_GTK_UPDATE	BIT(1)
+#define WOL_REASON_DISASSOC	BIT(2)
+#define WOL_REASON_DEAUTH	BIT(3)
+#define WOL_REASON_FW_DISCONNECT	BIT(4)
+
+#endif
diff --git a/drivers/staging/rtlwifi/rtl8822be/sw.c b/drivers/staging/rtlwifi/rtl8822be/sw.c
new file mode 100644
index 000000000000..91b784b6d1c5
--- /dev/null
+++ b/drivers/staging/rtlwifi/rtl8822be/sw.c
@@ -0,0 +1,481 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#include "../wifi.h"
+#include "../core.h"
+#include "../pci.h"
+#include "../base.h"
+#include "reg.h"
+#include "def.h"
+#include "phy.h"
+#include "hw.h"
+#include "sw.h"
+#include "fw.h"
+#include "trx.h"
+#include "led.h"
+#include "../btcoexist/rtl_btc.h"
+#include "../halmac/rtl_halmac.h"
+#include "../phydm/rtl_phydm.h"
+#include <linux/vmalloc.h>
+#include <linux/module.h>
+
+static void rtl8822be_init_aspm_vars(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+
+	/*close ASPM for AMD defaultly */
+	rtlpci->const_amdpci_aspm = 0;
+
+	/*
+	 * ASPM PS mode.
+	 * 0 - Disable ASPM,
+	 * 1 - Enable ASPM without Clock Req,
+	 * 2 - Enable ASPM with Clock Req,
+	 * 3 - Alwyas Enable ASPM with Clock Req,
+	 * 4 - Always Enable ASPM without Clock Req.
+	 * set default to RTL8822BE:3 RTL8822B:2
+	 *
+	 */
+	rtlpci->const_pci_aspm = 3;
+
+	/*Setting for PCI-E device */
+	rtlpci->const_devicepci_aspm_setting = 0x03;
+
+	/*Setting for PCI-E bridge */
+	rtlpci->const_hostpci_aspm_setting = 0x02;
+
+	/*
+	 * In Hw/Sw Radio Off situation.
+	 * 0 - Default,
+	 * 1 - From ASPM setting without low Mac Pwr,
+	 * 2 - From ASPM setting with low Mac Pwr,
+	 * 3 - Bus D3
+	 * set default to RTL8822BE:0 RTL8192SE:2
+	 */
+	rtlpci->const_hwsw_rfoff_d3 = 0;
+
+	/*
+	 * This setting works for those device with
+	 * backdoor ASPM setting such as EPHY setting.
+	 * 0 - Not support ASPM,
+	 * 1 - Support ASPM,
+	 * 2 - According to chipset.
+	 */
+	rtlpci->const_support_pciaspm = rtlpriv->cfg->mod_params->aspm_support;
+}
+
+int rtl8822be_init_sw_vars(struct ieee80211_hw *hw)
+{
+	int err = 0;
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	const char *fw_name;
+	struct rtl_phydm_params params;
+
+	rtl8822be_bt_reg_init(hw);
+	rtlpci->msi_support = rtlpriv->cfg->mod_params->msi_support;
+	rtlpriv->btcoexist.btc_ops = rtl_btc_get_ops_pointer();
+	rtlpriv->halmac.ops = rtl_halmac_get_ops_pointer();
+	rtlpriv->halmac.ops->halmac_init_adapter(rtlpriv);
+
+	/* should after halmac_init_adapter() */
+	rtl8822be_read_eeprom_info(hw, &params);
+
+	/* need eeprom info */
+	rtlpriv->phydm.ops = rtl_phydm_get_ops_pointer();
+	rtlpriv->phydm.ops->phydm_init_priv(rtlpriv, &params);
+
+	rtlpriv->dm.dm_initialgain_enable = 1;
+	rtlpriv->dm.dm_flag = 0;
+	rtlpriv->dm.disable_framebursting = 0;
+	/*rtlpriv->dm.thermalvalue = 0;*/
+	rtlpriv->dm.useramask = 1; /* turn on RA */
+	rtlpci->transmit_config = CFENDFORM | BIT(15);
+
+	rtlpriv->rtlhal.current_bandtype = BAND_ON_2_4G;
+	/*following 2 is for register 5G band, refer to _rtl_init_mac80211()*/
+	rtlpriv->rtlhal.bandset = BAND_ON_BOTH;
+	rtlpriv->rtlhal.macphymode = SINGLEMAC_SINGLEPHY;
+
+	rtlpci->receive_config = (RCR_APPFCS			|
+				  RCR_APP_MIC			|
+				  RCR_APP_ICV			|
+				  RCR_APP_PHYST_RXFF		|
+				  RCR_VHT_DACK			|
+				  RCR_HTC_LOC_CTRL		|
+				  /*RCR_AMF			|*/
+				  RCR_CBSSID_BCN		|
+				  RCR_CBSSID_DATA		|
+				  /*RCR_ACF			|*/
+				  /*RCR_ADF			|*/
+				  /*RCR_AICV			|*/
+				  /*RCR_ACRC32			|*/
+				  RCR_AB			|
+				  RCR_AM			|
+				  RCR_APM			|
+				  0);
+
+	rtlpci->irq_mask[0] = (u32)(IMR_PSTIMEOUT		|
+				    /*IMR_TBDER			|*/
+				    /*IMR_TBDOK			|*/
+				    /*IMR_BCNDMAINT0		|*/
+				    IMR_GTINT3			|
+				    IMR_HSISR_IND_ON_INT	|
+				    IMR_C2HCMD			|
+				    IMR_HIGHDOK			|
+				    IMR_MGNTDOK			|
+				    IMR_BKDOK			|
+				    IMR_BEDOK			|
+				    IMR_VIDOK			|
+				    IMR_VODOK			|
+				    IMR_RDU			|
+				    IMR_ROK			|
+				    0);
+
+	rtlpci->irq_mask[1] = (u32)(IMR_RXFOVW | IMR_TXFOVW | 0);
+	rtlpci->irq_mask[3] = (u32)(BIT_SETH2CDOK_MASK | 0);
+
+	/* for LPS & IPS */
+	rtlpriv->psc.inactiveps = rtlpriv->cfg->mod_params->inactiveps;
+	rtlpriv->psc.swctrl_lps = rtlpriv->cfg->mod_params->swctrl_lps;
+	rtlpriv->psc.fwctrl_lps = rtlpriv->cfg->mod_params->fwctrl_lps;
+	rtlpci->msi_support = rtlpriv->cfg->mod_params->msi_support;
+	if (rtlpriv->cfg->mod_params->disable_watchdog)
+		pr_info("watchdog disabled\n");
+	rtlpriv->psc.reg_fwctrl_lps = 2;
+	rtlpriv->psc.reg_max_lps_awakeintvl = 2;
+	/* for ASPM, you can close aspm through
+	 * set const_support_pciaspm = 0
+	 */
+	rtl8822be_init_aspm_vars(hw);
+
+	if (rtlpriv->psc.reg_fwctrl_lps == 1)
+		rtlpriv->psc.fwctrl_psmode = FW_PS_MIN_MODE;
+	else if (rtlpriv->psc.reg_fwctrl_lps == 2)
+		rtlpriv->psc.fwctrl_psmode = FW_PS_MAX_MODE;
+	else if (rtlpriv->psc.reg_fwctrl_lps == 3)
+		rtlpriv->psc.fwctrl_psmode = FW_PS_DTIM_MODE;
+
+	/* for early mode */
+	rtlpriv->rtlhal.earlymode_enable = false;
+
+	/*low power */
+	rtlpriv->psc.low_power_enable = false;
+
+	/* for firmware buf */
+	rtlpriv->rtlhal.pfirmware = vzalloc(0x40000);
+	if (!rtlpriv->rtlhal.pfirmware) {
+		/*pr_err("Can't alloc buffer for fw\n");*/
+		return 1;
+	}
+
+	/* request fw */
+	fw_name = "rtlwifi/rtl8822befw.bin";
+
+	rtlpriv->max_fw_size = 0x40000;
+	pr_info("Using firmware %s\n", fw_name);
+	err = request_firmware_nowait(THIS_MODULE, 1, fw_name, rtlpriv->io.dev,
+				      GFP_KERNEL, hw, rtl_fw_cb);
+	if (err) {
+		pr_err("Failed to request firmware!\n");
+		return 1;
+	}
+
+	/* init table of tx power by rate & limit */
+	rtl8822be_load_txpower_by_rate(hw);
+	rtl8822be_load_txpower_limit(hw);
+
+	return 0;
+}
+
+void rtl8822be_deinit_sw_vars(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	rtlpriv->halmac.ops->halmac_deinit_adapter(rtlpriv);
+	rtlpriv->phydm.ops->phydm_deinit_priv(rtlpriv);
+
+	if (rtlpriv->rtlhal.pfirmware) {
+		vfree(rtlpriv->rtlhal.pfirmware);
+		rtlpriv->rtlhal.pfirmware = NULL;
+	}
+}
+
+/* get bt coexist status */
+bool rtl8822be_get_btc_status(void)
+{
+	return true;
+}
+
+static void rtl8822be_phydm_watchdog(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u32 tmp;
+
+	tmp = rtl_read_dword(rtlpriv, 0xc00);
+	if (tmp & 0xFF000000) { /* Recover 0xC00: 0xF800000C --> 0x0000000C */
+		RT_TRACE(rtlpriv, COMP_RF, DBG_DMESG,
+			 "found regaddr_c00=%08X\n", tmp);
+		tmp &= ~0xFF000000;
+		rtl_write_dword(rtlpriv, 0xc00, tmp);
+		RT_TRACE(rtlpriv, COMP_RF, DBG_DMESG,
+			 "apply regaddr_c00=%08X\n", tmp);
+	}
+
+	rtlpriv->phydm.ops->phydm_watchdog(rtlpriv);
+}
+
+static struct rtl_hal_ops rtl8822be_hal_ops = {
+	.init_sw_vars = rtl8822be_init_sw_vars,
+	.deinit_sw_vars = rtl8822be_deinit_sw_vars,
+	.read_eeprom_info = rtl8822be_read_eeprom_info_dummy,
+	.interrupt_recognized = rtl8822be_interrupt_recognized,
+	.hw_init = rtl8822be_hw_init,
+	.hw_disable = rtl8822be_card_disable,
+	.hw_suspend = rtl8822be_suspend,
+	.hw_resume = rtl8822be_resume,
+	.enable_interrupt = rtl8822be_enable_interrupt,
+	.disable_interrupt = rtl8822be_disable_interrupt,
+	.set_network_type = rtl8822be_set_network_type,
+	.set_chk_bssid = rtl8822be_set_check_bssid,
+	.set_qos = rtl8822be_set_qos,
+	.set_bcn_reg = rtl8822be_set_beacon_related_registers,
+	.set_bcn_intv = rtl8822be_set_beacon_interval,
+	.update_interrupt_mask = rtl8822be_update_interrupt_mask,
+	.get_hw_reg = rtl8822be_get_hw_reg,
+	.set_hw_reg = rtl8822be_set_hw_reg,
+	.update_rate_tbl = rtl8822be_update_hal_rate_tbl,
+	.pre_fill_tx_bd_desc = rtl8822be_pre_fill_tx_bd_desc,
+	.rx_desc_buff_remained_cnt = rtl8822be_rx_desc_buff_remained_cnt,
+	.rx_check_dma_ok = rtl8822be_rx_check_dma_ok,
+	.fill_tx_desc = rtl8822be_tx_fill_desc,
+	.fill_tx_special_desc = rtl8822be_tx_fill_special_desc,
+	.query_rx_desc = rtl8822be_rx_query_desc,
+	.radio_onoff_checking = rtl8822be_gpio_radio_on_off_checking,
+	.switch_channel = rtl8822be_phy_sw_chnl,
+	.set_channel_access = rtl8822be_update_channel_access_setting,
+	.set_bw_mode = rtl8822be_phy_set_bw_mode,
+	.dm_watchdog = rtl8822be_phydm_watchdog,
+	.scan_operation_backup = rtl8822be_phy_scan_operation_backup,
+	.set_rf_power_state = rtl8822be_phy_set_rf_power_state,
+	.led_control = rtl8822be_led_control,
+	.set_desc = rtl8822be_set_desc,
+	.get_desc = rtl8822be_get_desc,
+	.is_tx_desc_closed = rtl8822be_is_tx_desc_closed,
+	.get_available_desc = rtl8822be_get_available_desc,
+	.tx_polling = rtl8822be_tx_polling,
+	.enable_hw_sec = rtl8822be_enable_hw_security_config,
+	.set_key = rtl8822be_set_key,
+	.init_sw_leds = rtl8822be_init_sw_leds,
+	.get_bbreg = rtl8822be_phy_query_bb_reg,
+	.set_bbreg = rtl8822be_phy_set_bb_reg,
+	.get_rfreg = rtl8822be_phy_query_rf_reg,
+	.set_rfreg = rtl8822be_phy_set_rf_reg,
+	.fill_h2c_cmd = rtl8822be_fill_h2c_cmd,
+	.set_default_port_id_cmd = rtl8822be_set_default_port_id_cmd,
+	.get_btc_status = rtl8822be_get_btc_status,
+	.rx_command_packet = rtl8822be_rx_command_packet,
+	.c2h_content_parsing = rtl8822be_c2h_content_parsing,
+	/* ops for halmac cb */
+	.halmac_cb_init_mac_register = rtl8822be_halmac_cb_init_mac_register,
+	.halmac_cb_init_bb_rf_register =
+		rtl8822be_halmac_cb_init_bb_rf_register,
+	.halmac_cb_write_data_rsvd_page =
+		rtl8822b_halmac_cb_write_data_rsvd_page,
+	.halmac_cb_write_data_h2c = rtl8822b_halmac_cb_write_data_h2c,
+	/* ops for phydm cb */
+	.get_txpower_index = rtl8822be_get_txpower_index,
+	.set_tx_power_index_by_rs = rtl8822be_phy_set_tx_power_index_by_rs,
+	.store_tx_power_by_rate = rtl8822be_store_tx_power_by_rate,
+	.phy_set_txpower_limit = rtl8822be_phy_set_txpower_limit,
+};
+
+static struct rtl_mod_params rtl8822be_mod_params = {
+	.sw_crypto = false,
+	.inactiveps = true,
+	.swctrl_lps = false,
+	.fwctrl_lps = true,
+	.msi_support = true,
+	.dma64 = false,
+	.aspm_support = 1,
+	.disable_watchdog = false,
+	.debug_level = 0,
+	.debug_mask = 0,
+};
+
+static struct rtl_hal_cfg rtl8822be_hal_cfg = {
+	.bar_id = 2,
+	.write_readback = false,
+	.name = "rtl8822be_pci",
+	.ops = &rtl8822be_hal_ops,
+	.mod_params = &rtl8822be_mod_params,
+	.spec_ver = RTL_SPEC_NEW_RATEID | RTL_SPEC_SUPPORT_VHT |
+		    RTL_SPEC_NEW_FW_C2H,
+	.maps[SYS_ISO_CTRL] = REG_SYS_ISO_CTRL_8822B,
+	.maps[SYS_FUNC_EN] = REG_SYS_FUNC_EN_8822B,
+	.maps[SYS_CLK] = REG_SYS_CLK_CTRL_8822B,
+	.maps[MAC_RCR_AM] = AM,
+	.maps[MAC_RCR_AB] = AB,
+	.maps[MAC_RCR_ACRC32] = ACRC32,
+	.maps[MAC_RCR_ACF] = ACF,
+	.maps[MAC_RCR_AAP] = AAP,
+	.maps[MAC_HIMR] = REG_HIMR0_8822B,
+	.maps[MAC_HIMRE] = REG_HIMR1_8822B,
+
+	.maps[EFUSE_ACCESS] = REG_EFUSE_ACCESS_8822B,
+
+	.maps[EFUSE_TEST] = REG_LDO_EFUSE_CTRL_8822B,
+	.maps[EFUSE_CTRL] = REG_EFUSE_CTRL_8822B,
+	.maps[EFUSE_CLK] = 0,
+	.maps[EFUSE_CLK_CTRL] = REG_EFUSE_CTRL_8822B,
+	.maps[EFUSE_PWC_EV12V] = PWC_EV12V,
+	.maps[EFUSE_FEN_ELDR] = FEN_ELDR,
+	.maps[EFUSE_LOADER_CLK_EN] = LOADER_CLK_EN,
+	.maps[EFUSE_ANA8M] = ANA8M,
+	.maps[EFUSE_HWSET_MAX_SIZE] = HWSET_MAX_SIZE,
+	.maps[EFUSE_MAX_SECTION_MAP] = EFUSE_MAX_SECTION,
+	.maps[EFUSE_REAL_CONTENT_SIZE] = EFUSE_REAL_CONTENT_LEN,
+	.maps[EFUSE_OOB_PROTECT_BYTES_LEN] = EFUSE_OOB_PROTECT_BYTES,
+
+	.maps[RWCAM] = REG_CAMCMD_8822B,
+	.maps[WCAMI] = REG_CAMWRITE_8822B,
+	.maps[RCAMO] = REG_CAMREAD_8822B,
+	.maps[CAMDBG] = REG_CAMDBG_8822B,
+	.maps[SECR] = REG_SECCFG_8822B,
+	.maps[SEC_CAM_NONE] = CAM_NONE,
+	.maps[SEC_CAM_WEP40] = CAM_WEP40,
+	.maps[SEC_CAM_TKIP] = CAM_TKIP,
+	.maps[SEC_CAM_AES] = CAM_AES,
+	.maps[SEC_CAM_WEP104] = CAM_WEP104,
+
+	.maps[RTL_IMR_BCNDMAINT6] = IMR_BCNDMAINT6,
+	.maps[RTL_IMR_BCNDMAINT5] = IMR_BCNDMAINT5,
+	.maps[RTL_IMR_BCNDMAINT4] = IMR_BCNDMAINT4,
+	.maps[RTL_IMR_BCNDMAINT3] = IMR_BCNDMAINT3,
+	.maps[RTL_IMR_BCNDMAINT2] = IMR_BCNDMAINT2,
+	.maps[RTL_IMR_BCNDMAINT1] = IMR_BCNDMAINT1,
+	/*	.maps[RTL_IMR_BCNDOK8] = IMR_BCNDOK8,     */ /*need check*/
+	.maps[RTL_IMR_BCNDOK7] = IMR_BCNDOK7,
+	.maps[RTL_IMR_BCNDOK6] = IMR_BCNDOK6,
+	.maps[RTL_IMR_BCNDOK5] = IMR_BCNDOK5,
+	.maps[RTL_IMR_BCNDOK4] = IMR_BCNDOK4,
+	.maps[RTL_IMR_BCNDOK3] = IMR_BCNDOK3,
+	.maps[RTL_IMR_BCNDOK2] = IMR_BCNDOK2,
+	.maps[RTL_IMR_BCNDOK1] = IMR_BCNDOK1,
+	/*	.maps[RTL_IMR_TIMEOUT2] = IMR_TIMEOUT2,*/
+	/*	.maps[RTL_IMR_TIMEOUT1] = IMR_TIMEOUT1,*/
+
+	.maps[RTL_IMR_TXFOVW] = IMR_TXFOVW,
+	.maps[RTL_IMR_PSTIMEOUT] = IMR_PSTIMEOUT,
+	.maps[RTL_IMR_BCNINT] = IMR_BCNDMAINT0,
+	.maps[RTL_IMR_RXFOVW] = IMR_RXFOVW,
+	.maps[RTL_IMR_RDU] = IMR_RDU,
+	.maps[RTL_IMR_ATIMEND] = IMR_ATIMEND,
+	.maps[RTL_IMR_H2CDOK] = IMR_H2CDOK,
+	.maps[RTL_IMR_BDOK] = IMR_BCNDOK0,
+	.maps[RTL_IMR_MGNTDOK] = IMR_MGNTDOK,
+	.maps[RTL_IMR_TBDER] = IMR_TBDER,
+	.maps[RTL_IMR_HIGHDOK] = IMR_HIGHDOK,
+	.maps[RTL_IMR_TBDOK] = IMR_TBDOK,
+	.maps[RTL_IMR_BKDOK] = IMR_BKDOK,
+	.maps[RTL_IMR_BEDOK] = IMR_BEDOK,
+	.maps[RTL_IMR_VIDOK] = IMR_VIDOK,
+	.maps[RTL_IMR_VODOK] = IMR_VODOK,
+	.maps[RTL_IMR_ROK] = IMR_ROK,
+	.maps[RTL_IBSS_INT_MASKS] = (IMR_BCNDMAINT0 | IMR_TBDOK | IMR_TBDER),
+
+	.maps[RTL_RC_CCK_RATE1M] = DESC_RATE1M,
+	.maps[RTL_RC_CCK_RATE2M] = DESC_RATE2M,
+	.maps[RTL_RC_CCK_RATE5_5M] = DESC_RATE5_5M,
+	.maps[RTL_RC_CCK_RATE11M] = DESC_RATE11M,
+	.maps[RTL_RC_OFDM_RATE6M] = DESC_RATE6M,
+	.maps[RTL_RC_OFDM_RATE9M] = DESC_RATE9M,
+	.maps[RTL_RC_OFDM_RATE12M] = DESC_RATE12M,
+	.maps[RTL_RC_OFDM_RATE18M] = DESC_RATE18M,
+	.maps[RTL_RC_OFDM_RATE24M] = DESC_RATE24M,
+	.maps[RTL_RC_OFDM_RATE36M] = DESC_RATE36M,
+	.maps[RTL_RC_OFDM_RATE48M] = DESC_RATE48M,
+	.maps[RTL_RC_OFDM_RATE54M] = DESC_RATE54M,
+
+	.maps[RTL_RC_HT_RATEMCS7] = DESC_RATEMCS7,
+	.maps[RTL_RC_HT_RATEMCS15] = DESC_RATEMCS15,
+
+	/*VHT hightest rate*/
+	.maps[RTL_RC_VHT_RATE_1SS_MCS7] = DESC_RATEVHT1SS_MCS7,
+	.maps[RTL_RC_VHT_RATE_1SS_MCS8] = DESC_RATEVHT1SS_MCS8,
+	.maps[RTL_RC_VHT_RATE_1SS_MCS9] = DESC_RATEVHT1SS_MCS9,
+	.maps[RTL_RC_VHT_RATE_2SS_MCS7] = DESC_RATEVHT2SS_MCS7,
+	.maps[RTL_RC_VHT_RATE_2SS_MCS8] = DESC_RATEVHT2SS_MCS8,
+	.maps[RTL_RC_VHT_RATE_2SS_MCS9] = DESC_RATEVHT2SS_MCS9,
+};
+
+static const struct pci_device_id rtl8822be_pci_ids[] = {
+	{RTL_PCI_DEVICE(PCI_VENDOR_ID_REALTEK, 0xB822, rtl8822be_hal_cfg)},
+	{},
+};
+
+MODULE_DEVICE_TABLE(pci, rtl8822be_pci_ids);
+
+MODULE_AUTHOR("Realtek WlanFAE	<wlanfae@realtek.com>");
+MODULE_AUTHOR("Larry Finger	<Larry.Finger@lwfinger.net>");
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Realtek 8822BE 802.11n PCI wireless");
+MODULE_FIRMWARE("rtlwifi/rtl8822befw.bin");
+
+module_param_named(swenc, rtl8822be_mod_params.sw_crypto, bool, 0444);
+module_param_named(debug_level, rtl8822be_mod_params.debug_level, int, 0644);
+module_param_named(debug_mask, rtl8822be_mod_params.debug_mask, ullong, 0644);
+module_param_named(ips, rtl8822be_mod_params.inactiveps, bool, 0444);
+module_param_named(swlps, rtl8822be_mod_params.swctrl_lps, bool, 0444);
+module_param_named(fwlps, rtl8822be_mod_params.fwctrl_lps, bool, 0444);
+module_param_named(msi, rtl8822be_mod_params.msi_support, bool, 0444);
+module_param_named(dma64, rtl8822be_mod_params.dma64, bool, 0444);
+module_param_named(aspm, rtl8822be_mod_params.aspm_support, int, 0444);
+module_param_named(disable_watchdog, rtl8822be_mod_params.disable_watchdog,
+		   bool, 0444);
+MODULE_PARM_DESC(swenc, "Set to 1 for software crypto (default 0)\n");
+MODULE_PARM_DESC(ips, "Set to 0 to not use link power save (default 1)\n");
+MODULE_PARM_DESC(swlps, "Set to 1 to use SW control power save (default 0)\n");
+MODULE_PARM_DESC(fwlps, "Set to 1 to use FW control power save (default 1)\n");
+MODULE_PARM_DESC(msi, "Set to 1 to use MSI interrupts mode (default 1)\n");
+MODULE_PARM_DESC(dma64, "Set to 1 to use DMA 64 (default 0)\n");
+MODULE_PARM_DESC(aspm, "Set to 1 to enable ASPM (default 1)\n");
+MODULE_PARM_DESC(debug, "Set debug level (0-5) (default 0)");
+MODULE_PARM_DESC(debug_mask, "Set debug mask (default 0)");
+MODULE_PARM_DESC(disable_watchdog,
+		 "Set to 1 to disable the watchdog (default 0)\n");
+
+static SIMPLE_DEV_PM_OPS(rtlwifi_pm_ops, rtl_pci_suspend, rtl_pci_resume);
+
+static struct pci_driver rtl8822be_driver = {
+	.name = KBUILD_MODNAME,
+	.id_table = rtl8822be_pci_ids,
+	.probe = rtl_pci_probe,
+	.remove = rtl_pci_disconnect,
+	.driver.pm = &rtlwifi_pm_ops,
+};
+
+module_pci_driver(rtl8822be_driver);
diff --git a/drivers/staging/rtlwifi/rtl8822be/sw.h b/drivers/staging/rtlwifi/rtl8822be/sw.h
new file mode 100644
index 000000000000..931eba98bd80
--- /dev/null
+++ b/drivers/staging/rtlwifi/rtl8822be/sw.h
@@ -0,0 +1,32 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __RTL8822B_SW_H__
+#define __RTL8822B_SW_H__
+
+int rtl8822be_init_sw_vars(struct ieee80211_hw *hw);
+void rtl8822be_deinit_sw_vars(struct ieee80211_hw *hw);
+bool rtl8822be_get_btc_status(void);
+#endif
diff --git a/drivers/staging/rtlwifi/rtl8822be/trx.c b/drivers/staging/rtlwifi/rtl8822be/trx.c
new file mode 100644
index 000000000000..38f80e48a399
--- /dev/null
+++ b/drivers/staging/rtlwifi/rtl8822be/trx.c
@@ -0,0 +1,1015 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#include "../wifi.h"
+#include "../pci.h"
+#include "../base.h"
+#include "../stats.h"
+#include "reg.h"
+#include "def.h"
+#include "phy.h"
+#include "trx.h"
+#include "led.h"
+#include "fw.h"
+
+#include <linux/vermagic.h>
+
+static u8 _rtl8822be_map_hwqueue_to_fwqueue(struct sk_buff *skb, u8 hw_queue)
+{
+	switch (hw_queue) {
+	case BEACON_QUEUE:
+		return QSLT_BEACON;
+	case H2C_QUEUE:
+		return QSLT_CMD;
+	case MGNT_QUEUE:
+		return QSLT_MGNT;
+	case HIGH_QUEUE:
+		return QSLT_HIGH;
+	default:
+		return skb->priority;
+	}
+}
+
+static void _rtl8822be_query_rxphystatus(struct ieee80211_hw *hw, u8 *phystrpt,
+					 struct ieee80211_hdr *hdr,
+					 struct rtl_stats *pstatus)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	rtlpriv->phydm.ops->phydm_query_phy_status(rtlpriv, phystrpt, hdr,
+						   pstatus);
+
+	/* UI BSS List signal strength(in percentage),
+	 * make it good looking, from 0~100.
+	 */
+	pstatus->signalstrength =
+		(u8)(rtl_signal_scale_mapping(hw, pstatus->rx_pwdb_all));
+}
+
+static void _rtl8822be_translate_rx_signal_stuff(struct ieee80211_hw *hw,
+						 struct sk_buff *skb,
+						 struct rtl_stats *pstatus,
+						 u8 *p_phystrpt)
+{
+	struct ieee80211_hdr *hdr;
+	u8 *tmp_buf;
+
+	tmp_buf = skb->data + pstatus->rx_drvinfo_size + pstatus->rx_bufshift +
+		  24;
+
+	hdr = (struct ieee80211_hdr *)tmp_buf;
+
+	/* query phy status */
+	_rtl8822be_query_rxphystatus(hw, p_phystrpt, hdr, pstatus);
+
+	/* packet statistics */
+	if (pstatus->packet_beacon && pstatus->packet_matchbssid)
+		rtl_priv(hw)->dm.dbginfo.num_qry_beacon_pkt++;
+
+	if (pstatus->packet_matchbssid &&
+	    ieee80211_is_data_qos(hdr->frame_control) &&
+	    !is_multicast_ether_addr(ieee80211_get_DA(hdr))) {
+		struct ieee80211_qos_hdr *hdr_qos =
+			(struct ieee80211_qos_hdr *)tmp_buf;
+		u16 tid = le16_to_cpu(hdr_qos->qos_ctrl) & 0xf;
+
+		if (tid != 0 && tid != 3)
+			rtl_priv(hw)->dm.dbginfo.num_non_be_pkt++;
+	}
+
+	/* signal statistics */
+	if (p_phystrpt)
+		rtl_process_phyinfo(hw, tmp_buf, pstatus);
+}
+
+static void _rtl8822be_insert_emcontent(struct rtl_tcb_desc *ptcb_desc,
+					u8 *virtualaddress)
+{
+	u32 dwtmp = 0;
+
+	memset(virtualaddress, 0, 8);
+
+	SET_EARLYMODE_PKTNUM(virtualaddress, ptcb_desc->empkt_num);
+	if (ptcb_desc->empkt_num == 1) {
+		dwtmp = ptcb_desc->empkt_len[0];
+	} else {
+		dwtmp = ptcb_desc->empkt_len[0];
+		dwtmp += ((dwtmp % 4) ? (4 - dwtmp % 4) : 0) + 4;
+		dwtmp += ptcb_desc->empkt_len[1];
+	}
+	SET_EARLYMODE_LEN0(virtualaddress, dwtmp);
+
+	if (ptcb_desc->empkt_num <= 3) {
+		dwtmp = ptcb_desc->empkt_len[2];
+	} else {
+		dwtmp = ptcb_desc->empkt_len[2];
+		dwtmp += ((dwtmp % 4) ? (4 - dwtmp % 4) : 0) + 4;
+		dwtmp += ptcb_desc->empkt_len[3];
+	}
+	SET_EARLYMODE_LEN1(virtualaddress, dwtmp);
+	if (ptcb_desc->empkt_num <= 5) {
+		dwtmp = ptcb_desc->empkt_len[4];
+	} else {
+		dwtmp = ptcb_desc->empkt_len[4];
+		dwtmp += ((dwtmp % 4) ? (4 - dwtmp % 4) : 0) + 4;
+		dwtmp += ptcb_desc->empkt_len[5];
+	}
+	SET_EARLYMODE_LEN2_1(virtualaddress, dwtmp & 0xF);
+	SET_EARLYMODE_LEN2_2(virtualaddress, dwtmp >> 4);
+	if (ptcb_desc->empkt_num <= 7) {
+		dwtmp = ptcb_desc->empkt_len[6];
+	} else {
+		dwtmp = ptcb_desc->empkt_len[6];
+		dwtmp += ((dwtmp % 4) ? (4 - dwtmp % 4) : 0) + 4;
+		dwtmp += ptcb_desc->empkt_len[7];
+	}
+	SET_EARLYMODE_LEN3(virtualaddress, dwtmp);
+	if (ptcb_desc->empkt_num <= 9) {
+		dwtmp = ptcb_desc->empkt_len[8];
+	} else {
+		dwtmp = ptcb_desc->empkt_len[8];
+		dwtmp += ((dwtmp % 4) ? (4 - dwtmp % 4) : 0) + 4;
+		dwtmp += ptcb_desc->empkt_len[9];
+	}
+	SET_EARLYMODE_LEN4(virtualaddress, dwtmp);
+}
+
+static bool rtl8822be_get_rxdesc_is_ht(struct ieee80211_hw *hw, u8 *pdesc)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 rx_rate = 0;
+
+	rx_rate = GET_RX_DESC_RX_RATE(pdesc);
+
+	RT_TRACE(rtlpriv, COMP_RXDESC, DBG_LOUD, "rx_rate=0x%02x.\n", rx_rate);
+
+	if ((rx_rate >= DESC_RATEMCS0) && (rx_rate <= DESC_RATEMCS15))
+		return true;
+	else
+		return false;
+}
+
+static bool rtl8822be_get_rxdesc_is_vht(struct ieee80211_hw *hw, u8 *pdesc)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 rx_rate = 0;
+
+	rx_rate = GET_RX_DESC_RX_RATE(pdesc);
+
+	RT_TRACE(rtlpriv, COMP_RXDESC, DBG_LOUD, "rx_rate=0x%02x.\n", rx_rate);
+
+	if (rx_rate >= DESC_RATEVHT1SS_MCS0)
+		return true;
+	else
+		return false;
+}
+
+static u8 rtl8822be_get_rx_vht_nss(struct ieee80211_hw *hw, u8 *pdesc)
+{
+	u8 rx_rate = 0;
+	u8 vht_nss = 0;
+
+	rx_rate = GET_RX_DESC_RX_RATE(pdesc);
+
+	if ((rx_rate >= DESC_RATEVHT1SS_MCS0) &&
+	    (rx_rate <= DESC_RATEVHT1SS_MCS9))
+		vht_nss = 1;
+	else if ((rx_rate >= DESC_RATEVHT2SS_MCS0) &&
+		 (rx_rate <= DESC_RATEVHT2SS_MCS9))
+		vht_nss = 2;
+
+	return vht_nss;
+}
+
+bool rtl8822be_rx_query_desc(struct ieee80211_hw *hw, struct rtl_stats *status,
+			     struct ieee80211_rx_status *rx_status, u8 *pdesc,
+			     struct sk_buff *skb)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 *p_phystrpt = NULL;
+	struct ieee80211_hdr *hdr;
+
+	u32 phystatus = GET_RX_DESC_PHYST(pdesc);
+
+	if (GET_RX_DESC_C2H(pdesc) == 0)
+		status->packet_report_type = NORMAL_RX;
+	else
+		status->packet_report_type = C2H_PACKET;
+
+	status->length = (u16)GET_RX_DESC_PKT_LEN(pdesc);
+	status->rx_drvinfo_size =
+		(u8)GET_RX_DESC_DRV_INFO_SIZE(pdesc) * RX_DRV_INFO_SIZE_UNIT;
+	status->rx_bufshift = (u8)(GET_RX_DESC_SHIFT(pdesc) & 0x03);
+	status->icv = (u16)GET_RX_DESC_ICV_ERR(pdesc);
+	status->crc = (u16)GET_RX_DESC_CRC32(pdesc);
+	status->hwerror = (status->crc | status->icv);
+	status->decrypted = !GET_RX_DESC_SWDEC(pdesc);
+	status->rate = (u8)GET_RX_DESC_RX_RATE(pdesc);
+	status->isampdu = (bool)(GET_RX_DESC_PAGGR(pdesc) == 1);
+	status->isfirst_ampdu = (bool)(GET_RX_DESC_PAGGR(pdesc) == 1);
+	status->timestamp_low = GET_RX_DESC_TSFL(pdesc);
+	status->is_ht = rtl8822be_get_rxdesc_is_ht(hw, pdesc);
+	status->is_vht = rtl8822be_get_rxdesc_is_vht(hw, pdesc);
+	status->vht_nss = rtl8822be_get_rx_vht_nss(hw, pdesc);
+	status->is_cck = RX_HAL_IS_CCK_RATE(status->rate);
+
+	status->macid = GET_RX_DESC_MACID(pdesc);
+	if (GET_RX_DESC_PATTERN_MATCH(pdesc))
+		status->wake_match = BIT(2);
+	else if (GET_RX_DESC_MAGIC_WAKE(pdesc))
+		status->wake_match = BIT(1);
+	else if (GET_RX_DESC_UNICAST_WAKE(pdesc))
+		status->wake_match = BIT(0);
+	else
+		status->wake_match = 0;
+	if (status->wake_match)
+		RT_TRACE(rtlpriv, COMP_RXDESC, DBG_LOUD,
+			 "GGGGGGGGGGGGGet Wakeup Packet!! WakeMatch=%d\n",
+			 status->wake_match);
+	rx_status->freq = hw->conf.chandef.chan->center_freq;
+	rx_status->band = hw->conf.chandef.chan->band;
+
+	if (phystatus)
+		p_phystrpt = (skb->data + status->rx_bufshift + 24);
+
+	hdr = (struct ieee80211_hdr *)(skb->data + status->rx_drvinfo_size +
+				       status->rx_bufshift + 24);
+
+	if (status->crc)
+		rx_status->flag |= RX_FLAG_FAILED_FCS_CRC;
+
+	if (status->is_ht)
+		rx_status->encoding = RX_ENC_HT;
+	if (status->is_vht)
+		rx_status->encoding = RX_ENC_VHT;
+
+	rx_status->nss = status->vht_nss;
+
+	rx_status->flag |= RX_FLAG_MACTIME_START;
+
+	/* hw will set status->decrypted true, if it finds the
+	 * frame is open data frame or mgmt frame.
+	 */
+	/* So hw will not decryption robust management frame
+	 * for IEEE80211w but still set status->decrypted
+	 * true, so here we should set it back to undecrypted
+	 * for IEEE80211w frame, and mac80211 sw will help
+	 * to decrypt it
+	 */
+	if (status->decrypted) {
+		if ((!_ieee80211_is_robust_mgmt_frame(hdr)) &&
+		    (ieee80211_has_protected(hdr->frame_control)))
+			rx_status->flag |= RX_FLAG_DECRYPTED;
+		else
+			rx_status->flag &= ~RX_FLAG_DECRYPTED;
+	}
+
+	/* rate_idx: index of data rate into band's
+	 * supported rates or MCS index if HT rates
+	 * are use (RX_FLAG_HT)
+	 */
+	/* Notice: this is diff with windows define */
+	rx_status->rate_idx = rtlwifi_rate_mapping(
+		hw, status->is_ht, status->is_vht, status->rate);
+
+	rx_status->mactime = status->timestamp_low;
+
+	_rtl8822be_translate_rx_signal_stuff(hw, skb, status, p_phystrpt);
+
+	/* below info. are filled by _rtl8822be_translate_rx_signal_stuff() */
+	if (!p_phystrpt)
+		goto label_no_physt;
+
+	rx_status->signal = status->recvsignalpower;
+
+	if (status->rx_packet_bw == HT_CHANNEL_WIDTH_20_40)
+		rx_status->bw = RATE_INFO_BW_40;
+	else if (status->rx_packet_bw == HT_CHANNEL_WIDTH_80)
+		rx_status->bw = RATE_INFO_BW_80;
+
+label_no_physt:
+
+	return true;
+}
+
+void rtl8822be_rx_check_dma_ok(struct ieee80211_hw *hw, u8 *header_desc,
+			       u8 queue_index)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 first_seg;
+	u8 last_seg;
+	u16 total_len;
+	u16 read_cnt = 0;
+
+	if (!header_desc)
+		return;
+
+	do {
+		total_len = (u16)GET_RX_BUFFER_DESC_TOTAL_LENGTH(header_desc);
+		first_seg = (u8)GET_RX_BUFFER_DESC_FS(header_desc);
+		last_seg = (u8)GET_RX_BUFFER_DESC_LS(header_desc);
+
+		if (read_cnt++ > 20) {
+			RT_TRACE(rtlpriv, COMP_RECV, DBG_DMESG,
+				 "RX chk DMA over %d times\n", read_cnt);
+			break;
+		}
+
+	} while (total_len == 0 && first_seg == 0 && last_seg == 0);
+}
+
+u16 rtl8822be_rx_desc_buff_remained_cnt(struct ieee80211_hw *hw, u8 queue_index)
+{
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u16 desc_idx_hw = 0, desc_idx_host = 0, remind_cnt = 0;
+	u32 tmp_4byte = 0;
+
+	u32 rw_mask = 0x1ff;
+
+	tmp_4byte = rtl_read_dword(rtlpriv, REG_RXQ_RXBD_IDX_8822B);
+	desc_idx_hw = (u16)((tmp_4byte >> 16) & rw_mask);
+	desc_idx_host = (u16)(tmp_4byte & rw_mask);
+
+	/* may be no data, donot rx */
+	if (desc_idx_hw == desc_idx_host)
+		return 0;
+
+	remind_cnt =
+		(desc_idx_hw > desc_idx_host) ?
+			(desc_idx_hw - desc_idx_host) :
+			(RX_DESC_NUM_8822BE - (desc_idx_host - desc_idx_hw));
+
+	rtlpci->rx_ring[queue_index].next_rx_rp = desc_idx_host;
+
+	return remind_cnt;
+}
+
+static u16 get_desc_address_from_queue_index(u16 queue_index)
+{
+	/*
+	 * Note: Access these registers will take a lot of cost.
+	 */
+	u16 desc_address = REG_BEQ_TXBD_IDX_8822B;
+
+	switch (queue_index) {
+	case BK_QUEUE:
+		desc_address = REG_BKQ_TXBD_IDX_8822B;
+		break;
+	case BE_QUEUE:
+		desc_address = REG_BEQ_TXBD_IDX_8822B;
+		break;
+	case VI_QUEUE:
+		desc_address = REG_VIQ_TXBD_IDX_8822B;
+		break;
+	case VO_QUEUE:
+		desc_address = REG_VOQ_TXBD_IDX_8822B;
+		break;
+	case BEACON_QUEUE:
+		desc_address = REG_BEQ_TXBD_IDX_8822B;
+		break;
+	case H2C_QUEUE:
+		desc_address = REG_H2CQ_TXBD_IDX_8822B;
+		break;
+	case MGNT_QUEUE:
+		desc_address = REG_MGQ_TXBD_IDX_8822B;
+		break;
+	case HIGH_QUEUE:
+		desc_address = REG_HI0Q_TXBD_IDX_8822B;
+		break;
+	case HCCA_QUEUE:
+		desc_address = REG_BEQ_TXBD_IDX_8822B;
+		break;
+	default:
+		break;
+	}
+	return desc_address;
+}
+
+/*free  desc that can be used */
+u16 rtl8822be_get_available_desc(struct ieee80211_hw *hw, u8 q_idx)
+{
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	struct rtl8192_tx_ring *ring = &rtlpci->tx_ring[q_idx];
+
+	return calc_fifo_space(ring->cur_tx_rp, ring->cur_tx_wp,
+			       TX_DESC_NUM_8822B);
+}
+
+void rtl8822be_pre_fill_tx_bd_desc(struct ieee80211_hw *hw, u8 *tx_bd_desc,
+				   u8 *desc, u8 queue_index,
+				   struct sk_buff *skb, dma_addr_t data_addr)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	u32 pkt_len = skb->len;
+	u16 desc_size = 48; /*tx desc size*/
+	u32 psblen = 0;
+	u32 total_packet_size = 0;
+	u16 current_bd_desc;
+	u8 i = 0;
+	/*u16 real_desc_size = 0x28;*/
+	u16 append_early_mode_size = 0;
+	u8 segmentnum = 1 << (RTL8822BE_SEG_NUM + 1);
+	dma_addr_t desc_dma_addr;
+	bool dma64 = rtlpriv->cfg->mod_params->dma64;
+
+	current_bd_desc = rtlpci->tx_ring[queue_index].cur_tx_wp;
+
+	total_packet_size = desc_size + pkt_len;
+
+	if (rtlpriv->rtlhal.earlymode_enable) {
+		if (queue_index < BEACON_QUEUE) {
+			append_early_mode_size = 8;
+			total_packet_size += append_early_mode_size;
+		}
+	}
+
+	/* page number (round up) */
+	psblen = (total_packet_size - 1) / 128 + 1;
+
+	/* tx desc addr */
+	desc_dma_addr = rtlpci->tx_ring[queue_index].dma +
+			(current_bd_desc * TX_DESC_SIZE);
+
+	/* Reset */
+	SET_TX_BUFF_DESC_LEN_0(tx_bd_desc, 0);
+	SET_TX_BUFF_DESC_PSB(tx_bd_desc, 0);
+	SET_TX_BUFF_DESC_OWN(tx_bd_desc, 0);
+
+	for (i = 1; i < segmentnum; i++) {
+		SET_TXBUFFER_DESC_LEN_WITH_OFFSET(tx_bd_desc, i, 0);
+		SET_TXBUFFER_DESC_AMSDU_WITH_OFFSET(tx_bd_desc, i, 0);
+		SET_TXBUFFER_DESC_ADD_LOW_WITH_OFFSET(tx_bd_desc, i, 0);
+		SET_TXBUFFER_DESC_ADD_HIGH_WITH_OFFSET(tx_bd_desc, i, 0, dma64);
+	}
+
+	/* Clear all status */
+	CLEAR_PCI_TX_DESC_CONTENT(desc, TX_DESC_SIZE);
+
+	if (rtlpriv->rtlhal.earlymode_enable) {
+		if (queue_index < BEACON_QUEUE)
+			SET_TX_BUFF_DESC_LEN_0(tx_bd_desc, desc_size + 8);
+		else
+			SET_TX_BUFF_DESC_LEN_0(tx_bd_desc, desc_size);
+	} else {
+		SET_TX_BUFF_DESC_LEN_0(tx_bd_desc, desc_size);
+	}
+	SET_TX_BUFF_DESC_PSB(tx_bd_desc, psblen);
+	SET_TX_BUFF_DESC_ADDR_LOW_0(tx_bd_desc, desc_dma_addr);
+	SET_TX_BUFF_DESC_ADDR_HIGH_0(tx_bd_desc, ((u64)desc_dma_addr >> 32),
+				     dma64);
+
+	SET_TXBUFFER_DESC_LEN_WITH_OFFSET(tx_bd_desc, 1, pkt_len);
+	SET_TXBUFFER_DESC_AMSDU_WITH_OFFSET(tx_bd_desc, 1, 0);
+	SET_TXBUFFER_DESC_ADD_LOW_WITH_OFFSET(tx_bd_desc, 1, data_addr);
+	SET_TXBUFFER_DESC_ADD_HIGH_WITH_OFFSET(tx_bd_desc, 1,
+					       ((u64)data_addr >> 32), dma64);
+
+	SET_TX_DESC_TXPKTSIZE(desc, (u16)(pkt_len));
+}
+
+static u8 rtl8822be_bw_mapping(struct ieee80211_hw *hw,
+			       struct rtl_tcb_desc *ptcb_desc)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	u8 bw_setting_of_desc = 0;
+
+	RT_TRACE(rtlpriv, COMP_SEND, DBG_TRACE,
+		 "%s, current_chan_bw %d, packet_bw %d\n", __func__,
+		 rtlphy->current_chan_bw, ptcb_desc->packet_bw);
+
+	if (rtlphy->current_chan_bw == HT_CHANNEL_WIDTH_80) {
+		if (ptcb_desc->packet_bw == HT_CHANNEL_WIDTH_80)
+			bw_setting_of_desc = 2;
+		else if (ptcb_desc->packet_bw == HT_CHANNEL_WIDTH_20_40)
+			bw_setting_of_desc = 1;
+		else
+			bw_setting_of_desc = 0;
+	} else if (rtlphy->current_chan_bw == HT_CHANNEL_WIDTH_20_40) {
+		if ((ptcb_desc->packet_bw == HT_CHANNEL_WIDTH_20_40) ||
+		    (ptcb_desc->packet_bw == HT_CHANNEL_WIDTH_80))
+			bw_setting_of_desc = 1;
+		else
+			bw_setting_of_desc = 0;
+	} else {
+		bw_setting_of_desc = 0;
+	}
+
+	return bw_setting_of_desc;
+}
+
+static u8 rtl8822be_sc_mapping(struct ieee80211_hw *hw,
+			       struct rtl_tcb_desc *ptcb_desc)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	struct rtl_mac *mac = rtl_mac(rtlpriv);
+	u8 sc_setting_of_desc = 0;
+
+	if (rtlphy->current_chan_bw == HT_CHANNEL_WIDTH_80) {
+		if (ptcb_desc->packet_bw == HT_CHANNEL_WIDTH_80) {
+			sc_setting_of_desc = VHT_DATA_SC_DONOT_CARE;
+		} else if (ptcb_desc->packet_bw == HT_CHANNEL_WIDTH_20_40) {
+			if (mac->cur_80_prime_sc == HAL_PRIME_CHNL_OFFSET_LOWER)
+				sc_setting_of_desc =
+					VHT_DATA_SC_40_LOWER_OF_80MHZ;
+			else if (mac->cur_80_prime_sc ==
+				 HAL_PRIME_CHNL_OFFSET_UPPER)
+				sc_setting_of_desc =
+					VHT_DATA_SC_40_UPPER_OF_80MHZ;
+			else
+				RT_TRACE(rtlpriv, COMP_SEND, DBG_LOUD,
+					 "%s: Not Correct Primary40MHz Setting\n",
+					 __func__);
+		} else {
+			if ((mac->cur_40_prime_sc ==
+			     HAL_PRIME_CHNL_OFFSET_LOWER) &&
+			    (mac->cur_80_prime_sc ==
+			     HAL_PRIME_CHNL_OFFSET_LOWER))
+				sc_setting_of_desc =
+					VHT_DATA_SC_20_LOWEST_OF_80MHZ;
+			else if ((mac->cur_40_prime_sc ==
+				  HAL_PRIME_CHNL_OFFSET_UPPER) &&
+				 (mac->cur_80_prime_sc ==
+				  HAL_PRIME_CHNL_OFFSET_LOWER))
+				sc_setting_of_desc =
+					VHT_DATA_SC_20_LOWER_OF_80MHZ;
+			else if ((mac->cur_40_prime_sc ==
+				  HAL_PRIME_CHNL_OFFSET_LOWER) &&
+				 (mac->cur_80_prime_sc ==
+				  HAL_PRIME_CHNL_OFFSET_UPPER))
+				sc_setting_of_desc =
+					VHT_DATA_SC_20_UPPER_OF_80MHZ;
+			else if ((mac->cur_40_prime_sc ==
+				  HAL_PRIME_CHNL_OFFSET_UPPER) &&
+				 (mac->cur_80_prime_sc ==
+				  HAL_PRIME_CHNL_OFFSET_UPPER))
+				sc_setting_of_desc =
+					VHT_DATA_SC_20_UPPERST_OF_80MHZ;
+			else
+				RT_TRACE(
+					rtlpriv, COMP_SEND, DBG_LOUD,
+					"rtl8822be_sc_mapping: Not Correct Primary40MHz Setting\n");
+		}
+	} else if (rtlphy->current_chan_bw == HT_CHANNEL_WIDTH_20_40) {
+		if (ptcb_desc->packet_bw == HT_CHANNEL_WIDTH_20_40) {
+			sc_setting_of_desc = VHT_DATA_SC_DONOT_CARE;
+		} else if (ptcb_desc->packet_bw == HT_CHANNEL_WIDTH_20) {
+			if (mac->cur_40_prime_sc ==
+			    HAL_PRIME_CHNL_OFFSET_UPPER) {
+				sc_setting_of_desc =
+					VHT_DATA_SC_20_UPPER_OF_80MHZ;
+			} else if (mac->cur_40_prime_sc ==
+				   HAL_PRIME_CHNL_OFFSET_LOWER) {
+				sc_setting_of_desc =
+					VHT_DATA_SC_20_LOWER_OF_80MHZ;
+			} else {
+				sc_setting_of_desc = VHT_DATA_SC_DONOT_CARE;
+			}
+		}
+	} else {
+		sc_setting_of_desc = VHT_DATA_SC_DONOT_CARE;
+	}
+
+	return sc_setting_of_desc;
+}
+
+void rtl8822be_tx_fill_desc(struct ieee80211_hw *hw, struct ieee80211_hdr *hdr,
+			    u8 *pdesc_tx, u8 *pbd_desc_tx,
+			    struct ieee80211_tx_info *info,
+			    struct ieee80211_sta *sta, struct sk_buff *skb,
+			    u8 hw_queue, struct rtl_tcb_desc *ptcb_desc)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	u8 *pdesc = (u8 *)pdesc_tx;
+	u16 seq_number;
+	__le16 fc = hdr->frame_control;
+	u8 fw_qsel = _rtl8822be_map_hwqueue_to_fwqueue(skb, hw_queue);
+	bool firstseg =
+		((hdr->seq_ctrl & cpu_to_le16(IEEE80211_SCTL_FRAG)) == 0);
+	bool lastseg = ((hdr->frame_control &
+			 cpu_to_le16(IEEE80211_FCTL_MOREFRAGS)) == 0);
+	dma_addr_t mapping;
+	u8 short_gi = 0;
+
+	seq_number = (le16_to_cpu(hdr->seq_ctrl) & IEEE80211_SCTL_SEQ) >> 4;
+	rtl_get_tcb_desc(hw, info, sta, skb, ptcb_desc);
+	/* reserve 8 byte for AMPDU early mode */
+	if (rtlhal->earlymode_enable) {
+		skb_push(skb, EM_HDR_LEN);
+		memset(skb->data, 0, EM_HDR_LEN);
+	}
+	mapping = pci_map_single(rtlpci->pdev, skb->data, skb->len,
+				 PCI_DMA_TODEVICE);
+	if (pci_dma_mapping_error(rtlpci->pdev, mapping)) {
+		RT_TRACE(rtlpriv, COMP_SEND, DBG_DMESG, "DMA mapping error");
+		return;
+	}
+
+	if (pbd_desc_tx)
+		rtl8822be_pre_fill_tx_bd_desc(hw, pbd_desc_tx, pdesc, hw_queue,
+					      skb, mapping);
+
+	if (ieee80211_is_nullfunc(fc) || ieee80211_is_ctl(fc)) {
+		firstseg = true;
+		lastseg = true;
+	}
+	if (firstseg) {
+		if (rtlhal->earlymode_enable) {
+			SET_TX_DESC_PKT_OFFSET(pdesc, 1);
+			SET_TX_DESC_OFFSET(pdesc,
+					   USB_HWDESC_HEADER_LEN + EM_HDR_LEN);
+			if (ptcb_desc->empkt_num) {
+				RT_TRACE(rtlpriv, COMP_SEND, DBG_TRACE,
+					 "Insert 8 byte.pTcb->EMPktNum:%d\n",
+					 ptcb_desc->empkt_num);
+				_rtl8822be_insert_emcontent(ptcb_desc,
+							    (u8 *)(skb->data));
+			}
+		} else {
+			SET_TX_DESC_OFFSET(pdesc, USB_HWDESC_HEADER_LEN);
+		}
+
+		/* tx report */
+		rtl_get_tx_report(ptcb_desc, pdesc, hw);
+
+		if (rtlpriv->rtlhal.current_bandtype == BAND_ON_5G &&
+		    ptcb_desc->hw_rate < DESC_RATE6M) {
+			RT_TRACE(rtlpriv, COMP_SEND, DBG_WARNING,
+				 "hw_rate=0x%X is invalid in 5G\n",
+				 ptcb_desc->hw_rate);
+			ptcb_desc->hw_rate = DESC_RATE6M;
+		}
+		SET_TX_DESC_DATARATE(pdesc, ptcb_desc->hw_rate);
+
+		if (ptcb_desc->hw_rate > DESC_RATEMCS0)
+			short_gi = (ptcb_desc->use_shortgi) ? 1 : 0;
+		else
+			short_gi = (ptcb_desc->use_shortpreamble) ? 1 : 0;
+
+		if (info->flags & IEEE80211_TX_CTL_AMPDU) {
+			SET_TX_DESC_AGG_EN(pdesc, 1);
+			SET_TX_DESC_MAX_AGG_NUM(pdesc, 0x1F);
+		}
+		SET_TX_DESC_SW_SEQ(pdesc, seq_number);
+		SET_TX_DESC_RTSEN(pdesc, ((ptcb_desc->rts_enable &&
+					   !ptcb_desc->cts_enable) ?
+						  1 :
+						  0));
+		SET_TX_DESC_HW_RTS_EN(pdesc, 0);
+		SET_TX_DESC_CTS2SELF(pdesc, ((ptcb_desc->cts_enable) ? 1 : 0));
+
+		SET_TX_DESC_RTSRATE(pdesc, ptcb_desc->rts_rate);
+		SET_TX_DESC_RTS_SC(pdesc, ptcb_desc->rts_sc);
+		SET_TX_DESC_RTS_SHORT(
+			pdesc,
+			((ptcb_desc->rts_rate <= DESC_RATE54M) ?
+				 (ptcb_desc->rts_use_shortpreamble ? 1 : 0) :
+				 (ptcb_desc->rts_use_shortgi ? 1 : 0)));
+
+		if (ptcb_desc->tx_enable_sw_calc_duration)
+			SET_TX_DESC_NAVUSEHDR(pdesc, 1);
+
+		SET_TX_DESC_DATA_BW(pdesc, rtl8822be_bw_mapping(hw, ptcb_desc));
+		SET_TX_DESC_DATA_SC(pdesc, rtl8822be_sc_mapping(hw, ptcb_desc));
+
+		if (sta) {
+			u8 ampdu_density = sta->ht_cap.ampdu_density;
+
+			SET_TX_DESC_AMPDU_DENSITY(pdesc, ampdu_density);
+		}
+		if (info->control.hw_key) {
+			struct ieee80211_key_conf *key = info->control.hw_key;
+
+			switch (key->cipher) {
+			case WLAN_CIPHER_SUITE_WEP40:
+			case WLAN_CIPHER_SUITE_WEP104:
+			case WLAN_CIPHER_SUITE_TKIP:
+				SET_TX_DESC_SEC_TYPE(pdesc, 0x1);
+				break;
+			case WLAN_CIPHER_SUITE_CCMP:
+				SET_TX_DESC_SEC_TYPE(pdesc, 0x3);
+				break;
+			default:
+				SET_TX_DESC_SEC_TYPE(pdesc, 0x0);
+				break;
+			}
+		}
+
+		SET_TX_DESC_QSEL(pdesc, fw_qsel);
+
+		if (rtlphy->current_channel > 14) {
+			/* OFDM 6M */
+			SET_TX_DESC_DATA_RTY_LOWEST_RATE(pdesc, 4);
+			SET_TX_DESC_RTS_RTY_LOWEST_RATE(pdesc, 4);
+		} else {
+			/* CCK 1M */
+			SET_TX_DESC_DATA_RTY_LOWEST_RATE(pdesc, 0);
+			SET_TX_DESC_RTS_RTY_LOWEST_RATE(pdesc, 0);
+		}
+		SET_TX_DESC_DISDATAFB(pdesc,
+				      ptcb_desc->disable_ratefallback ? 1 : 0);
+		SET_TX_DESC_USE_RATE(pdesc, ptcb_desc->use_driver_rate ? 1 : 0);
+
+		/*SET_TX_DESC_PWR_STATUS(pdesc, pwr_status);*/
+		/* Set TxRate and RTSRate in TxDesc  */
+		/* This prevent Tx initial rate of new-coming packets */
+		/* from being overwritten by retried  packet rate.*/
+		if (!ptcb_desc->use_driver_rate) {
+			/*SET_TX_DESC_RTS_RATE(pdesc, 0x08); */
+			/* SET_TX_DESC_TX_RATE(pdesc, 0x0b); */
+		}
+		if (ieee80211_is_data_qos(fc)) {
+			if (mac->rdg_en) {
+				RT_TRACE(rtlpriv, COMP_SEND, DBG_TRACE,
+					 "Enable RDG function.\n");
+				SET_TX_DESC_RDG_EN(pdesc, 1);
+				SET_TX_DESC_HTC(pdesc, 1);
+			}
+		}
+
+		SET_TX_DESC_PORT_ID(pdesc, 0);
+		SET_TX_DESC_MULTIPLE_PORT(pdesc, 0);
+	}
+
+	SET_TX_DESC_LS(pdesc, (lastseg ? 1 : 0));
+	if (rtlpriv->dm.useramask) {
+		SET_TX_DESC_RATE_ID(pdesc, ptcb_desc->ratr_index);
+		SET_TX_DESC_MACID(pdesc, ptcb_desc->mac_id);
+	} else {
+		SET_TX_DESC_RATE_ID(pdesc, 0xC + ptcb_desc->ratr_index);
+		SET_TX_DESC_MACID(pdesc, ptcb_desc->ratr_index);
+	}
+
+	SET_TX_DESC_MOREFRAG(pdesc, (lastseg ? 0 : 1));
+	if (ptcb_desc->multicast || ptcb_desc->broadcast) {
+		SET_TX_DESC_BMC(pdesc, 1);
+		/* BMC must be not AGG */
+		SET_TX_DESC_AGG_EN(pdesc, 0);
+	}
+	RT_TRACE(rtlpriv, COMP_SEND, DBG_TRACE, "\n");
+
+	/* debug purpose: used to check tx desc is correct or not */
+	/*rtlpriv->halmac.ops->halmac_chk_txdesc(rtlpriv, pdesc,
+	 *			skb->len + USB_HWDESC_HEADER_LEN);
+	 */
+}
+
+void rtl8822be_tx_fill_special_desc(struct ieee80211_hw *hw, u8 *pdesc,
+				    u8 *pbd_desc, struct sk_buff *skb,
+				    u8 hw_queue)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	u8 fw_queue;
+	u8 txdesc_len = 48;
+
+	dma_addr_t mapping = pci_map_single(rtlpci->pdev, skb->data, skb->len,
+					    PCI_DMA_TODEVICE);
+
+	if (pci_dma_mapping_error(rtlpci->pdev, mapping)) {
+		RT_TRACE(rtlpriv, COMP_SEND, DBG_DMESG, "DMA mapping error");
+		return;
+	}
+
+	rtl8822be_pre_fill_tx_bd_desc(hw, pbd_desc, pdesc, hw_queue, skb,
+				      mapping);
+
+	/* it should be BEACON_QUEUE or H2C_QUEUE,
+	 * so skb=NULL is safe to assert
+	 */
+	fw_queue = _rtl8822be_map_hwqueue_to_fwqueue(NULL, hw_queue);
+
+	CLEAR_PCI_TX_DESC_CONTENT(pdesc, txdesc_len);
+
+	/* common part for BEACON and H2C */
+	SET_TX_DESC_TXPKTSIZE((u8 *)pdesc, (u16)(skb->len));
+
+	SET_TX_DESC_QSEL(pdesc, fw_queue);
+
+	if (hw_queue == H2C_QUEUE) {
+		/* fill H2C */
+		SET_TX_DESC_OFFSET(pdesc, 0);
+
+	} else {
+		/* fill beacon */
+		SET_TX_DESC_OFFSET(pdesc, txdesc_len);
+
+		SET_TX_DESC_DATARATE(pdesc, DESC_RATE1M);
+
+		SET_TX_DESC_SW_SEQ(pdesc, 0);
+
+		SET_TX_DESC_RATE_ID(pdesc, 7);
+		SET_TX_DESC_MACID(pdesc, 0);
+
+		SET_TX_DESC_LS(pdesc, 1);
+
+		SET_TX_DESC_OFFSET(pdesc, 48);
+
+		SET_TX_DESC_USE_RATE(pdesc, 1);
+	}
+
+	RT_PRINT_DATA(rtlpriv, COMP_CMD, DBG_LOUD, "H2C Tx Cmd Content\n",
+		      pdesc, txdesc_len);
+}
+
+void rtl8822be_set_desc(struct ieee80211_hw *hw, u8 *pdesc, bool istx,
+			u8 desc_name, u8 *val)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u8 q_idx = *val;
+	bool dma64 = rtlpriv->cfg->mod_params->dma64;
+
+	if (istx) {
+		switch (desc_name) {
+		case HW_DESC_OWN: {
+			struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+			struct rtl8192_tx_ring *ring = &rtlpci->tx_ring[q_idx];
+			u16 max_tx_desc = ring->entries;
+
+			if (q_idx == BEACON_QUEUE) {
+				/* in case of beacon, pdesc is BD desc. */
+				u8 *pbd_desc = pdesc;
+
+				ring->cur_tx_wp = 0;
+				ring->cur_tx_rp = 0;
+				SET_TX_BUFF_DESC_OWN(pbd_desc, 1);
+				return;
+			}
+
+			/* make sure tx desc is available by caller */
+			ring->cur_tx_wp = ((ring->cur_tx_wp + 1) % max_tx_desc);
+
+			rtl_write_word(
+				rtlpriv,
+				get_desc_address_from_queue_index(
+					q_idx),
+				ring->cur_tx_wp);
+		} break;
+		}
+	} else {
+		switch (desc_name) {
+		case HW_DESC_RX_PREPARE:
+			SET_RX_BUFFER_DESC_LS(pdesc, 0);
+			SET_RX_BUFFER_DESC_FS(pdesc, 0);
+			SET_RX_BUFFER_DESC_TOTAL_LENGTH(pdesc, 0);
+
+			SET_RX_BUFFER_DESC_DATA_LENGTH(
+				pdesc, MAX_RECEIVE_BUFFER_SIZE + RX_DESC_SIZE);
+
+			SET_RX_BUFFER_PHYSICAL_LOW(
+				pdesc, (*(dma_addr_t *)val) & DMA_BIT_MASK(32));
+			SET_RX_BUFFER_PHYSICAL_HIGH(
+				pdesc, ((u64)(*(dma_addr_t *)val) >> 32),
+				dma64);
+			break;
+		default:
+			WARN_ONCE(true, "ERR rxdesc :%d not process\n",
+				  desc_name);
+			break;
+		}
+	}
+}
+
+u64 rtl8822be_get_desc(struct ieee80211_hw *hw,
+		       u8 *pdesc, bool istx, u8 desc_name)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u64 ret = 0;
+	u8 *pbd_desc = pdesc;
+	bool dma64 = rtlpriv->cfg->mod_params->dma64;
+
+	if (istx) {
+		switch (desc_name) {
+		case HW_DESC_TXBUFF_ADDR:
+			ret = GET_TXBUFFER_DESC_ADDR_LOW(pbd_desc, 1);
+			ret |= (u64)GET_TXBUFFER_DESC_ADDR_HIGH(pbd_desc, 1,
+								dma64) << 32;
+			break;
+		default:
+			WARN_ONCE(true, "ERR txdesc :%d not process\n",
+				  desc_name);
+			break;
+		}
+	} else {
+		switch (desc_name) {
+		case HW_DESC_RXPKT_LEN:
+			ret = GET_RX_DESC_PKT_LEN(pdesc);
+			break;
+		default:
+			WARN_ONCE(true, "ERR rxdesc :%d not process\n",
+				  desc_name);
+			break;
+		}
+	}
+	return ret;
+}
+
+bool rtl8822be_is_tx_desc_closed(struct ieee80211_hw *hw, u8 hw_queue,
+				 u16 index)
+{
+	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	bool ret = false;
+	struct rtl8192_tx_ring *ring = &rtlpci->tx_ring[hw_queue];
+	u16 cur_tx_rp, cur_tx_wp;
+	u16 tmp16;
+
+	/*
+	 * design rule:
+	 *     idx <= cur_tx_rp <= hw_rp <= cur_tx_wp = hw_wp
+	 */
+
+	if (index == ring->cur_tx_rp) {
+		/* update only if sw_rp reach hw_rp */
+		tmp16 = rtl_read_word(
+			    rtlpriv,
+			    get_desc_address_from_queue_index(hw_queue) + 2);
+
+		cur_tx_rp = tmp16 & 0x01ff;
+		cur_tx_wp = ring->cur_tx_wp;
+
+		/* don't need to update ring->cur_tx_wp */
+		ring->cur_tx_rp = cur_tx_rp;
+	}
+
+	if (index == ring->cur_tx_rp)
+		ret = false;	/* no more */
+	else
+		ret = true;	/* more */
+
+	if (hw_queue == BEACON_QUEUE)
+		ret = true;
+
+	if (rtlpriv->rtlhal.driver_is_goingto_unload ||
+	    rtlpriv->psc.rfoff_reason > RF_CHANGE_BY_PS)
+		ret = true;
+
+	return ret;
+}
+
+void rtl8822be_tx_polling(struct ieee80211_hw *hw, u8 hw_queue)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	if (hw_queue == BEACON_QUEUE) {
+		/* kick start */
+		rtl_write_byte(
+			rtlpriv, REG_RX_RXBD_NUM_8822B + 1,
+			rtl_read_byte(rtlpriv, REG_RX_RXBD_NUM_8822B + 1) |
+				BIT(4));
+	}
+}
+
+u32 rtl8822be_rx_command_packet(struct ieee80211_hw *hw,
+				const struct rtl_stats *status,
+				struct sk_buff *skb)
+{
+	u32 result = 0;
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	switch (status->packet_report_type) {
+	case NORMAL_RX:
+		result = 0;
+		break;
+	case C2H_PACKET:
+		rtl8822be_c2h_packet_handler(hw, skb->data, (u8)skb->len);
+		result = 1;
+		break;
+	default:
+		RT_TRACE(rtlpriv, COMP_RECV, DBG_TRACE,
+			 "Unknown packet type %d\n",
+			 status->packet_report_type);
+		break;
+	}
+
+	return result;
+}
diff --git a/drivers/staging/rtlwifi/rtl8822be/trx.h b/drivers/staging/rtlwifi/rtl8822be/trx.h
new file mode 100644
index 000000000000..db769f3c4cd6
--- /dev/null
+++ b/drivers/staging/rtlwifi/rtl8822be/trx.h
@@ -0,0 +1,165 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2016  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __RTL8822B_TRX_H__
+#define __RTL8822B_TRX_H__
+
+#include "../halmac/halmac_tx_desc_nic.h"
+#include "../halmac/halmac_rx_desc_nic.h"
+
+#define TX_DESC_SIZE	64
+
+#define RX_DRV_INFO_SIZE_UNIT	8
+
+#define TX_DESC_NEXT_DESC_OFFSET	48
+#define USB_HWDESC_HEADER_LEN	48
+
+#define RX_DESC_SIZE	24
+#define MAX_RECEIVE_BUFFER_SIZE	8192
+
+#define SET_EARLYMODE_PKTNUM(__paddr, __val)                                   \
+	SET_BITS_TO_LE_4BYTE(__paddr, 0, 4, __val)
+#define SET_EARLYMODE_LEN0(__paddr, __val)                                     \
+	SET_BITS_TO_LE_4BYTE(__paddr, 4, 15, __val)
+#define SET_EARLYMODE_LEN1(__paddr, __val)                                     \
+	SET_BITS_TO_LE_4BYTE(__paddr, 16, 2, __val)
+#define SET_EARLYMODE_LEN1_1(__paddr, __val)                                   \
+	SET_BITS_TO_LE_4BYTE(__paddr, 19, 13, __val)
+#define SET_EARLYMODE_LEN1_2(__paddr, __val)                                   \
+	SET_BITS_TO_LE_4BYTE(__paddr + 4, 0, 2, __val)
+#define SET_EARLYMODE_LEN2(__paddr, __val)                                     \
+	SET_BITS_TO_LE_4BYTE(__paddr + 4, 2, 15, __val)
+#define SET_EARLYMODE_LEN2_1(__paddr, __val)                                   \
+	SET_BITS_TO_LE_4BYTE(__paddr, 2, 4, __val)
+#define SET_EARLYMODE_LEN2_2(__paddr, __val)                                   \
+	SET_BITS_TO_LE_4BYTE(__paddr + 4, 0, 8, __val)
+#define SET_EARLYMODE_LEN3(__paddr, __val)                                     \
+	SET_BITS_TO_LE_4BYTE(__paddr + 4, 17, 15, __val)
+#define SET_EARLYMODE_LEN4(__paddr, __val)                                     \
+	SET_BITS_TO_LE_4BYTE(__paddr + 4, 20, 12, __val)
+
+/* TX/RX buffer descriptor */
+
+/* for Txfilldescroptor8822be, fill the desc content. */
+#define SET_TXBUFFER_DESC_LEN_WITH_OFFSET(__pdesc, __offset, __val)            \
+	SET_BITS_TO_LE_4BYTE((__pdesc) + ((__offset) * 16), 0, 16, __val)
+#define SET_TXBUFFER_DESC_AMSDU_WITH_OFFSET(__pdesc, __offset, __val)          \
+	SET_BITS_TO_LE_4BYTE((__pdesc) + ((__offset) * 16), 31, 1, __val)
+#define SET_TXBUFFER_DESC_ADD_LOW_WITH_OFFSET(__pdesc, __offset, __val)        \
+	SET_BITS_TO_LE_4BYTE((__pdesc) + ((__offset) * 16) + 4, 0, 32, __val)
+#define SET_TXBUFFER_DESC_ADD_HIGH_WITH_OFFSET(pbd, off, val, dma64)	       \
+	(dma64 ? SET_BITS_TO_LE_4BYTE((pbd) + ((off) * 16) + 8, 0, 32, val) : 0)
+#define GET_TXBUFFER_DESC_ADDR_LOW(__pdesc, __offset)                          \
+	LE_BITS_TO_4BYTE((__pdesc) + ((__offset) * 16) + 4, 0, 32)
+#define GET_TXBUFFER_DESC_ADDR_HIGH(pbd, off, dma64)			       \
+	(dma64 ? LE_BITS_TO_4BYTE((pbd) + ((off) * 16) + 8, 0, 32) : 0)
+
+/* Dword 0 */
+#define SET_TX_BUFF_DESC_LEN_0(__pdesc, __val)                                 \
+	SET_BITS_TO_LE_4BYTE(__pdesc, 0, 14, __val)
+#define SET_TX_BUFF_DESC_PSB(__pdesc, __val)                                   \
+	SET_BITS_TO_LE_4BYTE(__pdesc, 16, 15, __val)
+#define SET_TX_BUFF_DESC_OWN(__pdesc, __val)                                   \
+	SET_BITS_TO_LE_4BYTE(__pdesc, 31, 1, __val)
+
+/* Dword 1 */
+#define SET_TX_BUFF_DESC_ADDR_LOW_0(__pdesc, __val)                            \
+	SET_BITS_TO_LE_4BYTE((__pdesc) + 4, 0, 32, __val)
+/* Dword 2 */
+#define SET_TX_BUFF_DESC_ADDR_HIGH_0(bdesc, val, dma64)			       \
+	SET_TXBUFFER_DESC_ADD_HIGH_WITH_OFFSET(bdesc, 0, val, dma64)
+/* Dword 3 / RESERVED 0 */
+
+/* RX buffer  */
+
+/* DWORD 0 */
+#define SET_RX_BUFFER_DESC_DATA_LENGTH(__rx_status_desc, __val)                \
+	SET_BITS_TO_LE_4BYTE(__rx_status_desc, 0, 14, __val)
+#define SET_RX_BUFFER_DESC_LS(__rx_status_desc, __val)                         \
+	SET_BITS_TO_LE_4BYTE(__rx_status_desc, 15, 1, __val)
+#define SET_RX_BUFFER_DESC_FS(__rx_status_desc, __val)                         \
+	SET_BITS_TO_LE_4BYTE(__rx_status_desc, 16, 1, __val)
+#define SET_RX_BUFFER_DESC_TOTAL_LENGTH(__rx_status_desc, __val)               \
+	SET_BITS_TO_LE_4BYTE(__rx_status_desc, 16, 15, __val)
+
+#define GET_RX_BUFFER_DESC_OWN(__rx_status_desc)                               \
+	LE_BITS_TO_4BYTE(__rx_status_desc, 31, 1)
+#define GET_RX_BUFFER_DESC_LS(__rx_status_desc)                                \
+	LE_BITS_TO_4BYTE(__rx_status_desc, 15, 1)
+#define GET_RX_BUFFER_DESC_FS(__rx_status_desc)                                \
+	LE_BITS_TO_4BYTE(__rx_status_desc, 16, 1)
+#define GET_RX_BUFFER_DESC_TOTAL_LENGTH(__rx_status_desc)                      \
+	LE_BITS_TO_4BYTE(__rx_status_desc, 16, 15)
+
+/* DWORD 1 */
+#define SET_RX_BUFFER_PHYSICAL_LOW(__rx_status_desc, __val)                    \
+	SET_BITS_TO_LE_4BYTE(__rx_status_desc + 4, 0, 32, __val)
+
+/* DWORD 2 */
+#define SET_RX_BUFFER_PHYSICAL_HIGH(__rx_status_desc, __val, dma64)            \
+	(dma64 ? SET_BITS_TO_LE_4BYTE((__rx_status_desc) + 8, 0, 32, __val) : 0)
+
+#define CLEAR_PCI_TX_DESC_CONTENT(__pdesc, _size)                              \
+	do {                                                                   \
+		if (_size > TX_DESC_NEXT_DESC_OFFSET)                          \
+			memset(__pdesc, 0, TX_DESC_NEXT_DESC_OFFSET);          \
+		else                                                           \
+			memset(__pdesc, 0, _size);                             \
+	} while (0)
+
+void rtl8822be_rx_check_dma_ok(struct ieee80211_hw *hw, u8 *header_desc,
+			       u8 queue_index);
+u16 rtl8822be_rx_desc_buff_remained_cnt(struct ieee80211_hw *hw,
+					u8 queue_index);
+u16 rtl8822be_get_available_desc(struct ieee80211_hw *hw, u8 queue_index);
+void rtl8822be_pre_fill_tx_bd_desc(struct ieee80211_hw *hw, u8 *tx_bd_desc,
+				   u8 *desc, u8 queue_index,
+				   struct sk_buff *skb, dma_addr_t addr);
+
+void rtl8822be_tx_fill_desc(struct ieee80211_hw *hw, struct ieee80211_hdr *hdr,
+			    u8 *pdesc_tx, u8 *pbd_desc_tx,
+			    struct ieee80211_tx_info *info,
+			    struct ieee80211_sta *sta, struct sk_buff *skb,
+			    u8 hw_queue, struct rtl_tcb_desc *ptcb_desc);
+void rtl8822be_tx_fill_special_desc(struct ieee80211_hw *hw, u8 *pdesc,
+				    u8 *pbd_desc, struct sk_buff *skb,
+				    u8 hw_queue);
+bool rtl8822be_rx_query_desc(struct ieee80211_hw *hw, struct rtl_stats *status,
+			     struct ieee80211_rx_status *rx_status, u8 *pdesc,
+			     struct sk_buff *skb);
+void rtl8822be_set_desc(struct ieee80211_hw *hw, u8 *pdesc, bool istx,
+			u8 desc_name, u8 *val);
+u64 rtl8822be_get_desc(struct ieee80211_hw *hw,
+		       u8 *pdesc, bool istx, u8 desc_name);
+bool rtl8822be_is_tx_desc_closed(struct ieee80211_hw *hw, u8 hw_queue,
+				 u16 index);
+void rtl8822be_tx_polling(struct ieee80211_hw *hw, u8 hw_queue);
+void rtl8822be_tx_fill_cmddesc(struct ieee80211_hw *hw, u8 *pdesc,
+			       bool firstseg, bool lastseg,
+			       struct sk_buff *skb);
+u32 rtl8822be_rx_command_packet(struct ieee80211_hw *hw,
+				const struct rtl_stats *status,
+				struct sk_buff *skb);
+#endif
diff --git a/drivers/staging/rtlwifi/stats.c b/drivers/staging/rtlwifi/stats.c
new file mode 100644
index 000000000000..96eb14c92c01
--- /dev/null
+++ b/drivers/staging/rtlwifi/stats.c
@@ -0,0 +1,260 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2012  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+#include "wifi.h"
+#include "stats.h"
+#include <linux/export.h>
+
+u8 rtl_query_rxpwrpercentage(s8 antpower)
+{
+	if ((antpower <= -100) || (antpower >= 20))
+		return 0;
+	else if (antpower >= 0)
+		return 100;
+	else
+		return 100 + antpower;
+}
+
+u8 rtl_evm_db_to_percentage(s8 value)
+{
+	s8 ret_val = clamp(-value, 0, 33) * 3;
+
+	if (ret_val == 99)
+		ret_val = 100;
+
+	return ret_val;
+}
+
+static long rtl_translate_todbm(struct ieee80211_hw *hw,
+				u8 signal_strength_index)
+{
+	long signal_power;
+
+	signal_power = (long)((signal_strength_index + 1) >> 1);
+	signal_power -= 95;
+	return signal_power;
+}
+
+long rtl_signal_scale_mapping(struct ieee80211_hw *hw, long currsig)
+{
+	long retsig;
+
+	if (currsig >= 61 && currsig <= 100)
+		retsig = 90 + ((currsig - 60) / 4);
+	else if (currsig >= 41 && currsig <= 60)
+		retsig = 78 + ((currsig - 40) / 2);
+	else if (currsig >= 31 && currsig <= 40)
+		retsig = 66 + (currsig - 30);
+	else if (currsig >= 21 && currsig <= 30)
+		retsig = 54 + (currsig - 20);
+	else if (currsig >= 5 && currsig <= 20)
+		retsig = 42 + (((currsig - 5) * 2) / 3);
+	else if (currsig == 4)
+		retsig = 36;
+	else if (currsig == 3)
+		retsig = 27;
+	else if (currsig == 2)
+		retsig = 18;
+	else if (currsig == 1)
+		retsig = 9;
+	else
+		retsig = currsig;
+
+	return retsig;
+}
+
+static void rtl_process_ui_rssi(struct ieee80211_hw *hw,
+				struct rtl_stats *pstatus)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	u8 rfpath;
+	u32 last_rssi, tmpval;
+
+	if (!pstatus->packet_toself && !pstatus->packet_beacon)
+		return;
+
+	rtlpriv->stats.pwdb_all_cnt += pstatus->rx_pwdb_all;
+	rtlpriv->stats.rssi_calculate_cnt++;
+
+	if (rtlpriv->stats.ui_rssi.total_num++ >= PHY_RSSI_SLID_WIN_MAX) {
+		rtlpriv->stats.ui_rssi.total_num = PHY_RSSI_SLID_WIN_MAX;
+		last_rssi = rtlpriv->stats.ui_rssi.elements[
+			rtlpriv->stats.ui_rssi.index];
+		rtlpriv->stats.ui_rssi.total_val -= last_rssi;
+	}
+	rtlpriv->stats.ui_rssi.total_val += pstatus->signalstrength;
+	rtlpriv->stats.ui_rssi.elements[rtlpriv->stats.ui_rssi.index++] =
+	    pstatus->signalstrength;
+	if (rtlpriv->stats.ui_rssi.index >= PHY_RSSI_SLID_WIN_MAX)
+		rtlpriv->stats.ui_rssi.index = 0;
+	tmpval = rtlpriv->stats.ui_rssi.total_val /
+		rtlpriv->stats.ui_rssi.total_num;
+	rtlpriv->stats.signal_strength = rtl_translate_todbm(hw, (u8)tmpval);
+	pstatus->rssi = rtlpriv->stats.signal_strength;
+
+	if (pstatus->is_cck)
+		return;
+
+	for (rfpath = RF90_PATH_A; rfpath < rtlphy->num_total_rfpath;
+	     rfpath++) {
+		if (rtlpriv->stats.rx_rssi_percentage[rfpath] == 0) {
+			rtlpriv->stats.rx_rssi_percentage[rfpath] =
+			    pstatus->rx_mimo_signalstrength[rfpath];
+		}
+		if (pstatus->rx_mimo_signalstrength[rfpath] >
+		    rtlpriv->stats.rx_rssi_percentage[rfpath]) {
+			rtlpriv->stats.rx_rssi_percentage[rfpath] =
+			    ((rtlpriv->stats.rx_rssi_percentage[rfpath] *
+			      (RX_SMOOTH_FACTOR - 1)) +
+			     (pstatus->rx_mimo_signalstrength[rfpath])) /
+			    (RX_SMOOTH_FACTOR);
+			rtlpriv->stats.rx_rssi_percentage[rfpath] =
+			    rtlpriv->stats.rx_rssi_percentage[rfpath] + 1;
+		} else {
+			rtlpriv->stats.rx_rssi_percentage[rfpath] =
+			    ((rtlpriv->stats.rx_rssi_percentage[rfpath] *
+			      (RX_SMOOTH_FACTOR - 1)) +
+			     (pstatus->rx_mimo_signalstrength[rfpath])) /
+			    (RX_SMOOTH_FACTOR);
+		}
+		rtlpriv->stats.rx_snr_db[rfpath] = pstatus->rx_snr[rfpath];
+		rtlpriv->stats.rx_evm_dbm[rfpath] =
+					pstatus->rx_mimo_evm_dbm[rfpath];
+		rtlpriv->stats.rx_cfo_short[rfpath] =
+					pstatus->cfo_short[rfpath];
+		rtlpriv->stats.rx_cfo_tail[rfpath] = pstatus->cfo_tail[rfpath];
+	}
+}
+
+static void rtl_update_rxsignalstatistics(struct ieee80211_hw *hw,
+					  struct rtl_stats *pstatus)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	int weighting = 0;
+
+	if (rtlpriv->stats.recv_signal_power == 0)
+		rtlpriv->stats.recv_signal_power = pstatus->recvsignalpower;
+	if (pstatus->recvsignalpower > rtlpriv->stats.recv_signal_power)
+		weighting = 5;
+	else if (pstatus->recvsignalpower < rtlpriv->stats.recv_signal_power)
+		weighting = (-5);
+	rtlpriv->stats.recv_signal_power = (rtlpriv->stats.recv_signal_power *
+		5 + pstatus->recvsignalpower + weighting) / 6;
+}
+
+static void rtl_process_pwdb(struct ieee80211_hw *hw, struct rtl_stats *pstatus)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_sta_info *drv_priv = NULL;
+	struct ieee80211_sta *sta = NULL;
+	long undec_sm_pwdb;
+
+	rcu_read_lock();
+	if (rtlpriv->mac80211.opmode != NL80211_IFTYPE_STATION)
+		sta = rtl_find_sta(hw, pstatus->psaddr);
+
+	/* adhoc or ap mode */
+	if (sta) {
+		drv_priv = (struct rtl_sta_info *)sta->drv_priv;
+		undec_sm_pwdb = drv_priv->rssi_stat.undec_sm_pwdb;
+	} else {
+		undec_sm_pwdb = rtlpriv->dm.undec_sm_pwdb;
+	}
+
+	if (undec_sm_pwdb < 0)
+		undec_sm_pwdb = pstatus->rx_pwdb_all;
+	if (pstatus->rx_pwdb_all > (u32)undec_sm_pwdb) {
+		undec_sm_pwdb = (((undec_sm_pwdb) *
+		      (RX_SMOOTH_FACTOR - 1)) +
+		     (pstatus->rx_pwdb_all)) / (RX_SMOOTH_FACTOR);
+		undec_sm_pwdb = undec_sm_pwdb + 1;
+	} else {
+		undec_sm_pwdb = (((undec_sm_pwdb) *
+		      (RX_SMOOTH_FACTOR - 1)) +
+		     (pstatus->rx_pwdb_all)) / (RX_SMOOTH_FACTOR);
+	}
+
+	if (sta)
+		drv_priv->rssi_stat.undec_sm_pwdb = undec_sm_pwdb;
+	else
+		rtlpriv->dm.undec_sm_pwdb = undec_sm_pwdb;
+	rcu_read_unlock();
+
+	rtl_update_rxsignalstatistics(hw, pstatus);
+}
+
+static void rtl_process_ui_link_quality(struct ieee80211_hw *hw,
+					struct rtl_stats *pstatus)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	u32 last_evm, n_stream, tmpval;
+
+	if (pstatus->signalquality == 0)
+		return;
+
+	if (rtlpriv->stats.ui_link_quality.total_num++ >=
+	    PHY_LINKQUALITY_SLID_WIN_MAX) {
+		rtlpriv->stats.ui_link_quality.total_num =
+		    PHY_LINKQUALITY_SLID_WIN_MAX;
+		last_evm = rtlpriv->stats.ui_link_quality.elements[
+			rtlpriv->stats.ui_link_quality.index];
+		rtlpriv->stats.ui_link_quality.total_val -= last_evm;
+	}
+	rtlpriv->stats.ui_link_quality.total_val += pstatus->signalquality;
+	rtlpriv->stats.ui_link_quality.elements[
+		rtlpriv->stats.ui_link_quality.index++] =
+							pstatus->signalquality;
+	if (rtlpriv->stats.ui_link_quality.index >=
+	    PHY_LINKQUALITY_SLID_WIN_MAX)
+		rtlpriv->stats.ui_link_quality.index = 0;
+	tmpval = rtlpriv->stats.ui_link_quality.total_val /
+	    rtlpriv->stats.ui_link_quality.total_num;
+	rtlpriv->stats.signal_quality = tmpval;
+	rtlpriv->stats.last_sigstrength_inpercent = tmpval;
+	for (n_stream = 0; n_stream < 2; n_stream++) {
+		if (pstatus->rx_mimo_sig_qual[n_stream] != -1) {
+			if (rtlpriv->stats.rx_evm_percentage[n_stream] == 0) {
+				rtlpriv->stats.rx_evm_percentage[n_stream] =
+				    pstatus->rx_mimo_sig_qual[n_stream];
+			}
+			rtlpriv->stats.rx_evm_percentage[n_stream] =
+			    ((rtlpriv->stats.rx_evm_percentage[n_stream]
+			      * (RX_SMOOTH_FACTOR - 1)) +
+			     (pstatus->rx_mimo_sig_qual[n_stream] * 1)) /
+			    (RX_SMOOTH_FACTOR);
+		}
+	}
+}
+
+void rtl_process_phyinfo(struct ieee80211_hw *hw, u8 *buffer,
+			 struct rtl_stats *pstatus)
+{
+	if (!pstatus->packet_matchbssid)
+		return;
+
+	rtl_process_ui_rssi(hw, pstatus);
+	rtl_process_pwdb(hw, pstatus);
+	rtl_process_ui_link_quality(hw, pstatus);
+}
diff --git a/drivers/staging/rtlwifi/stats.h b/drivers/staging/rtlwifi/stats.h
new file mode 100644
index 000000000000..bd0108f93182
--- /dev/null
+++ b/drivers/staging/rtlwifi/stats.h
@@ -0,0 +1,42 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2012  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __RTL_STATS_H__
+#define __RTL_STATS_H__
+
+#define	PHY_RSSI_SLID_WIN_MAX			100
+#define	PHY_LINKQUALITY_SLID_WIN_MAX		20
+#define	PHY_BEACON_RSSI_SLID_WIN_MAX		10
+
+/* Rx smooth factor */
+#define	RX_SMOOTH_FACTOR			20
+
+u8 rtl_query_rxpwrpercentage(s8 antpower);
+u8 rtl_evm_db_to_percentage(s8 value);
+long rtl_signal_scale_mapping(struct ieee80211_hw *hw, long currsig);
+void rtl_process_phyinfo(struct ieee80211_hw *hw, u8 *buffer,
+			 struct rtl_stats *pstatus);
+
+#endif
diff --git a/drivers/staging/rtlwifi/wifi.h b/drivers/staging/rtlwifi/wifi.h
new file mode 100644
index 000000000000..eb91c130b245
--- /dev/null
+++ b/drivers/staging/rtlwifi/wifi.h
@@ -0,0 +1,3375 @@
+/******************************************************************************
+ *
+ * Copyright(c) 2009-2012  Realtek Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called LICENSE.
+ *
+ * Contact Information:
+ * wlanfae <wlanfae@realtek.com>
+ * Realtek Corporation, No. 2, Innovation Road II, Hsinchu Science Park,
+ * Hsinchu 300, Taiwan.
+ *
+ * Larry Finger <Larry.Finger@lwfinger.net>
+ *
+ *****************************************************************************/
+
+#ifndef __RTL_WIFI_H__
+#define __RTL_WIFI_H__
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/sched.h>
+#include <linux/firmware.h>
+#include <linux/etherdevice.h>
+#include <linux/vmalloc.h>
+#include <linux/usb.h>
+#include <net/mac80211.h>
+#include <linux/completion.h>
+#include "debug.h"
+
+#define	MASKBYTE0				0xff
+#define	MASKBYTE1				0xff00
+#define	MASKBYTE2				0xff0000
+#define	MASKBYTE3				0xff000000
+#define	MASKHWORD				0xffff0000
+#define	MASKLWORD				0x0000ffff
+#define	MASKDWORD				0xffffffff
+#define	MASK12BITS				0xfff
+#define	MASKH4BITS				0xf0000000
+#define MASKOFDM_D				0xffc00000
+#define	MASKCCK					0x3f3f3f3f
+
+#define	MASK4BITS				0x0f
+#define	MASK20BITS				0xfffff
+#define RFREG_OFFSET_MASK			0xfffff
+
+#define	MASKBYTE0				0xff
+#define	MASKBYTE1				0xff00
+#define	MASKBYTE2				0xff0000
+#define	MASKBYTE3				0xff000000
+#define	MASKHWORD				0xffff0000
+#define	MASKLWORD				0x0000ffff
+#define	MASKDWORD				0xffffffff
+#define	MASK12BITS				0xfff
+#define	MASKH4BITS				0xf0000000
+#define MASKOFDM_D				0xffc00000
+#define	MASKCCK					0x3f3f3f3f
+
+#define	MASK4BITS				0x0f
+#define	MASK20BITS				0xfffff
+#define RFREG_OFFSET_MASK			0xfffff
+
+#define RF_CHANGE_BY_INIT			0
+#define RF_CHANGE_BY_IPS			BIT(28)
+#define RF_CHANGE_BY_PS				BIT(29)
+#define RF_CHANGE_BY_HW				BIT(30)
+#define RF_CHANGE_BY_SW				BIT(31)
+
+#define IQK_ADDA_REG_NUM			16
+#define IQK_MAC_REG_NUM				4
+#define IQK_THRESHOLD				8
+
+#define MAX_KEY_LEN				61
+#define KEY_BUF_SIZE				5
+
+/* QoS related. */
+/*aci: 0x00	Best Effort*/
+/*aci: 0x01	Background*/
+/*aci: 0x10	Video*/
+/*aci: 0x11	Voice*/
+/*Max: define total number.*/
+#define AC0_BE					0
+#define AC1_BK					1
+#define AC2_VI					2
+#define AC3_VO					3
+#define AC_MAX					4
+#define QOS_QUEUE_NUM				4
+#define RTL_MAC80211_NUM_QUEUE			5
+#define REALTEK_USB_VENQT_MAX_BUF_SIZE		254
+#define RTL_USB_MAX_RX_COUNT			100
+#define QBSS_LOAD_SIZE				5
+#define MAX_WMMELE_LENGTH			64
+
+#define TOTAL_CAM_ENTRY				32
+
+/*slot time for 11g. */
+#define RTL_SLOT_TIME_9				9
+#define RTL_SLOT_TIME_20			20
+
+/*related to tcp/ip. */
+#define SNAP_SIZE		6
+#define PROTOC_TYPE_SIZE	2
+
+/*related with 802.11 frame*/
+#define MAC80211_3ADDR_LEN			24
+#define MAC80211_4ADDR_LEN			30
+
+#define CHANNEL_MAX_NUMBER	(14 + 24 + 21)	/* 14 is the max channel no */
+#define CHANNEL_MAX_NUMBER_2G		14
+#define CHANNEL_MAX_NUMBER_5G		49 /* Please refer to
+					    *"phy_GetChnlGroup8812A" and
+					    * "Hal_ReadTxPowerInfo8812A"
+					    */
+#define CHANNEL_MAX_NUMBER_5G_80M	7
+#define CHANNEL_GROUP_MAX	(3 + 9)	/*  ch1~3, 4~9, 10~14 = three groups */
+#define MAX_PG_GROUP			13
+#define	CHANNEL_GROUP_MAX_2G		3
+#define	CHANNEL_GROUP_IDX_5GL		3
+#define	CHANNEL_GROUP_IDX_5GM		6
+#define	CHANNEL_GROUP_IDX_5GH		9
+#define	CHANNEL_GROUP_MAX_5G		9
+#define CHANNEL_MAX_NUMBER_2G		14
+#define AVG_THERMAL_NUM			8
+#define AVG_THERMAL_NUM_88E		4
+#define AVG_THERMAL_NUM_8723BE		4
+#define MAX_TID_COUNT			9
+
+/* for early mode */
+#define FCS_LEN				4
+#define EM_HDR_LEN			8
+
+enum rtl8192c_h2c_cmd {
+	H2C_AP_OFFLOAD = 0,
+	H2C_SETPWRMODE = 1,
+	H2C_JOINBSSRPT = 2,
+	H2C_RSVDPAGE = 3,
+	H2C_RSSI_REPORT = 5,
+	H2C_RA_MASK = 6,
+	H2C_MACID_PS_MODE = 7,
+	H2C_P2P_PS_OFFLOAD = 8,
+	H2C_MAC_MODE_SEL = 9,
+	H2C_PWRM = 15,
+	H2C_P2P_PS_CTW_CMD = 24,
+	MAX_H2CCMD
+};
+
+#define MAX_TX_COUNT			4
+#define MAX_REGULATION_NUM		4
+#define MAX_RF_PATH_NUM			4
+#define MAX_RATE_SECTION_NUM		6	/* = MAX_RATE_SECTION */
+#define MAX_2_4G_BANDWIDTH_NUM		4
+#define MAX_5G_BANDWIDTH_NUM		4
+#define	MAX_RF_PATH			4
+#define	MAX_CHNL_GROUP_24G		6
+#define	MAX_CHNL_GROUP_5G		14
+
+#define TX_PWR_BY_RATE_NUM_BAND		2
+#define TX_PWR_BY_RATE_NUM_RF		4
+#define TX_PWR_BY_RATE_NUM_SECTION	12
+/* compatible with TX_PWR_BY_RATE_NUM_SECTION */
+#define TX_PWR_BY_RATE_NUM_RATE		84
+#define MAX_BASE_NUM_IN_PHY_REG_PG_24G  6	/* MAX_RATE_SECTION */
+#define MAX_BASE_NUM_IN_PHY_REG_PG_5G	5	/* MAX_RATE_SECTION -1 */
+
+#define BUFDESC_SEG_NUM		1 /* 0:2 seg, 1: 4 seg, 2: 8 seg */
+
+#define DEL_SW_IDX_SZ		30
+
+/* For now, it's just for 8192ee
+ * but not OK yet, keep it 0
+ */
+#define RTL8192EE_SEG_NUM		BUFDESC_SEG_NUM
+#define RTL8822BE_SEG_NUM		BUFDESC_SEG_NUM
+
+enum rf_tx_num {
+	RF_1TX = 0,
+	RF_2TX,
+	RF_MAX_TX_NUM,
+	RF_TX_NUM_NONIMPLEMENT,
+};
+
+#define PACKET_NORMAL			0
+#define PACKET_DHCP			1
+#define PACKET_ARP			2
+#define PACKET_EAPOL			3
+
+#define	MAX_SUPPORT_WOL_PATTERN_NUM	16
+#define	RSVD_WOL_PATTERN_NUM		1
+#define	WKFMCAM_ADDR_NUM		6
+#define	WKFMCAM_SIZE			24
+
+#define	MAX_WOL_BIT_MASK_SIZE		16
+/* MIN LEN keeps 13 here */
+#define	MIN_WOL_PATTERN_SIZE		13
+#define	MAX_WOL_PATTERN_SIZE		128
+
+#define	WAKE_ON_MAGIC_PACKET		BIT(0)
+#define	WAKE_ON_PATTERN_MATCH		BIT(1)
+
+#define	WOL_REASON_PTK_UPDATE		BIT(0)
+#define	WOL_REASON_GTK_UPDATE		BIT(1)
+#define	WOL_REASON_DISASSOC		BIT(2)
+#define	WOL_REASON_DEAUTH		BIT(3)
+#define	WOL_REASON_AP_LOST		BIT(4)
+#define	WOL_REASON_MAGIC_PKT		BIT(5)
+#define	WOL_REASON_UNICAST_PKT		BIT(6)
+#define	WOL_REASON_PATTERN_PKT		BIT(7)
+#define	WOL_REASON_RTD3_SSID_MATCH	BIT(8)
+#define	WOL_REASON_REALWOW_V2_WAKEUPPKT	BIT(9)
+#define	WOL_REASON_REALWOW_V2_ACKLOST	BIT(10)
+
+struct rtlwifi_firmware_header {
+	__le16 signature;
+	u8 category;
+	u8 function;
+	__le16 version;
+	u8 subversion;
+	u8 rsvd1;
+	u8 month;
+	u8 date;
+	u8 hour;
+	u8 minute;
+	__le16 ramcodesize;
+	__le16 rsvd2;
+	__le32 svnindex;
+	__le32 rsvd3;
+	__le32 rsvd4;
+	__le32 rsvd5;
+};
+
+struct txpower_info_2g {
+	u8 index_cck_base[MAX_RF_PATH][MAX_CHNL_GROUP_24G];
+	u8 index_bw40_base[MAX_RF_PATH][MAX_CHNL_GROUP_24G];
+	/*If only one tx, only BW20 and OFDM are used.*/
+	u8 cck_diff[MAX_RF_PATH][MAX_TX_COUNT];
+	u8 ofdm_diff[MAX_RF_PATH][MAX_TX_COUNT];
+	u8 bw20_diff[MAX_RF_PATH][MAX_TX_COUNT];
+	u8 bw40_diff[MAX_RF_PATH][MAX_TX_COUNT];
+	u8 bw80_diff[MAX_RF_PATH][MAX_TX_COUNT];
+	u8 bw160_diff[MAX_RF_PATH][MAX_TX_COUNT];
+};
+
+struct txpower_info_5g {
+	u8 index_bw40_base[MAX_RF_PATH][MAX_CHNL_GROUP_5G];
+	/*If only one tx, only BW20, OFDM, BW80 and BW160 are used.*/
+	u8 ofdm_diff[MAX_RF_PATH][MAX_TX_COUNT];
+	u8 bw20_diff[MAX_RF_PATH][MAX_TX_COUNT];
+	u8 bw40_diff[MAX_RF_PATH][MAX_TX_COUNT];
+	u8 bw80_diff[MAX_RF_PATH][MAX_TX_COUNT];
+	u8 bw160_diff[MAX_RF_PATH][MAX_TX_COUNT];
+};
+
+enum rate_section {
+	CCK = 0,
+	OFDM,
+	HT_MCS0_MCS7,
+	HT_MCS8_MCS15,
+	VHT_1SSMCS0_1SSMCS9,
+	VHT_2SSMCS0_2SSMCS9,
+	MAX_RATE_SECTION,
+};
+
+enum intf_type {
+	INTF_PCI = 0,
+	INTF_USB = 1,
+};
+
+enum radio_path {
+	RF90_PATH_A = 0,
+	RF90_PATH_B = 1,
+	RF90_PATH_C = 2,
+	RF90_PATH_D = 3,
+};
+
+enum radio_mask {
+	RF_MASK_A = BIT(0),
+	RF_MASK_B = BIT(1),
+	RF_MASK_C = BIT(2),
+	RF_MASK_D = BIT(3),
+};
+
+enum regulation_txpwr_lmt {
+	TXPWR_LMT_FCC = 0,
+	TXPWR_LMT_MKK = 1,
+	TXPWR_LMT_ETSI = 2,
+	TXPWR_LMT_WW = 3,
+
+	TXPWR_LMT_MAX_REGULATION_NUM = 4
+};
+
+enum rt_eeprom_type {
+	EEPROM_93C46,
+	EEPROM_93C56,
+	EEPROM_BOOT_EFUSE,
+};
+
+enum ttl_status {
+	RTL_STATUS_INTERFACE_START = 0,
+};
+
+enum hardware_type {
+	HARDWARE_TYPE_RTL8192E,
+	HARDWARE_TYPE_RTL8192U,
+	HARDWARE_TYPE_RTL8192SE,
+	HARDWARE_TYPE_RTL8192SU,
+	HARDWARE_TYPE_RTL8192CE,
+	HARDWARE_TYPE_RTL8192CU,
+	HARDWARE_TYPE_RTL8192DE,
+	HARDWARE_TYPE_RTL8192DU,
+	HARDWARE_TYPE_RTL8723AE,
+	HARDWARE_TYPE_RTL8723U,
+	HARDWARE_TYPE_RTL8188EE,
+	HARDWARE_TYPE_RTL8723BE,
+	HARDWARE_TYPE_RTL8192EE,
+	HARDWARE_TYPE_RTL8821AE,
+	HARDWARE_TYPE_RTL8812AE,
+	HARDWARE_TYPE_RTL8822BE,
+
+	/* keep it last */
+	HARDWARE_TYPE_NUM
+};
+
+#define RTL_HW_TYPE(rtlpriv)	(rtl_hal((struct rtl_priv *)rtlpriv)->hw_type)
+#define IS_NEW_GENERATION_IC(rtlpriv)			\
+			(RTL_HW_TYPE(rtlpriv) >= HARDWARE_TYPE_RTL8192EE)
+#define IS_HARDWARE_TYPE_8192CE(rtlpriv)		\
+			(RTL_HW_TYPE(rtlpriv) == HARDWARE_TYPE_RTL8192CE)
+#define IS_HARDWARE_TYPE_8812(rtlpriv)			\
+			(RTL_HW_TYPE(rtlpriv) == HARDWARE_TYPE_RTL8812AE)
+#define IS_HARDWARE_TYPE_8821(rtlpriv)			\
+			(RTL_HW_TYPE(rtlpriv) == HARDWARE_TYPE_RTL8821AE)
+#define IS_HARDWARE_TYPE_8723A(rtlpriv)			\
+			(RTL_HW_TYPE(rtlpriv) == HARDWARE_TYPE_RTL8723AE)
+#define IS_HARDWARE_TYPE_8723B(rtlpriv)			\
+			(RTL_HW_TYPE(rtlpriv) == HARDWARE_TYPE_RTL8723BE)
+#define IS_HARDWARE_TYPE_8192E(rtlpriv)			\
+			(RTL_HW_TYPE(rtlpriv) == HARDWARE_TYPE_RTL8192EE)
+#define IS_HARDWARE_TYPE_8822B(rtlpriv)			\
+			(RTL_HW_TYPE(rtlpriv) == HARDWARE_TYPE_RTL8822BE)
+
+#define RX_HAL_IS_CCK_RATE(rxmcs)			\
+	((rxmcs) == DESC_RATE1M ||			\
+	 (rxmcs) == DESC_RATE2M ||			\
+	 (rxmcs) == DESC_RATE5_5M ||			\
+	 (rxmcs) == DESC_RATE11M)
+
+enum scan_operation_backup_opt {
+	SCAN_OPT_BACKUP = 0,
+	SCAN_OPT_BACKUP_BAND0 = 0,
+	SCAN_OPT_BACKUP_BAND1,
+	SCAN_OPT_RESTORE,
+	SCAN_OPT_MAX
+};
+
+/*RF state.*/
+enum rf_pwrstate {
+	ERFON,
+	ERFSLEEP,
+	ERFOFF
+};
+
+struct bb_reg_def {
+	u32 rfintfs;
+	u32 rfintfi;
+	u32 rfintfo;
+	u32 rfintfe;
+	u32 rf3wire_offset;
+	u32 rflssi_select;
+	u32 rftxgain_stage;
+	u32 rfhssi_para1;
+	u32 rfhssi_para2;
+	u32 rfsw_ctrl;
+	u32 rfagc_control1;
+	u32 rfagc_control2;
+	u32 rfrxiq_imbal;
+	u32 rfrx_afe;
+	u32 rftxiq_imbal;
+	u32 rftx_afe;
+	u32 rf_rb;		/* rflssi_readback */
+	u32 rf_rbpi;		/* rflssi_readbackpi */
+};
+
+enum io_type {
+	IO_CMD_PAUSE_DM_BY_SCAN = 0,
+	IO_CMD_PAUSE_BAND0_DM_BY_SCAN = 0,
+	IO_CMD_PAUSE_BAND1_DM_BY_SCAN = 1,
+	IO_CMD_RESUME_DM_BY_SCAN = 2,
+};
+
+enum hw_variables {
+	HW_VAR_ETHER_ADDR = 0x0,
+	HW_VAR_MULTICAST_REG = 0x1,
+	HW_VAR_BASIC_RATE = 0x2,
+	HW_VAR_BSSID = 0x3,
+	HW_VAR_MEDIA_STATUS = 0x4,
+	HW_VAR_SECURITY_CONF = 0x5,
+	HW_VAR_BEACON_INTERVAL = 0x6,
+	HW_VAR_ATIM_WINDOW = 0x7,
+	HW_VAR_LISTEN_INTERVAL = 0x8,
+	HW_VAR_CS_COUNTER = 0x9,
+	HW_VAR_DEFAULTKEY0 = 0xa,
+	HW_VAR_DEFAULTKEY1 = 0xb,
+	HW_VAR_DEFAULTKEY2 = 0xc,
+	HW_VAR_DEFAULTKEY3 = 0xd,
+	HW_VAR_SIFS = 0xe,
+	HW_VAR_R2T_SIFS = 0xf,
+	HW_VAR_DIFS = 0x10,
+	HW_VAR_EIFS = 0x11,
+	HW_VAR_SLOT_TIME = 0x12,
+	HW_VAR_ACK_PREAMBLE = 0x13,
+	HW_VAR_CW_CONFIG = 0x14,
+	HW_VAR_CW_VALUES = 0x15,
+	HW_VAR_RATE_FALLBACK_CONTROL = 0x16,
+	HW_VAR_CONTENTION_WINDOW = 0x17,
+	HW_VAR_RETRY_COUNT = 0x18,
+	HW_VAR_TR_SWITCH = 0x19,
+	HW_VAR_COMMAND = 0x1a,
+	HW_VAR_WPA_CONFIG = 0x1b,
+	HW_VAR_AMPDU_MIN_SPACE = 0x1c,
+	HW_VAR_SHORTGI_DENSITY = 0x1d,
+	HW_VAR_AMPDU_FACTOR = 0x1e,
+	HW_VAR_MCS_RATE_AVAILABLE = 0x1f,
+	HW_VAR_AC_PARAM = 0x20,
+	HW_VAR_ACM_CTRL = 0x21,
+	HW_VAR_DIS_REQ_QSIZE = 0x22,
+	HW_VAR_CCX_CHNL_LOAD = 0x23,
+	HW_VAR_CCX_NOISE_HISTOGRAM = 0x24,
+	HW_VAR_CCX_CLM_NHM = 0x25,
+	HW_VAR_TXOPLIMIT = 0x26,
+	HW_VAR_TURBO_MODE = 0x27,
+	HW_VAR_RF_STATE = 0x28,
+	HW_VAR_RF_OFF_BY_HW = 0x29,
+	HW_VAR_BUS_SPEED = 0x2a,
+	HW_VAR_SET_DEV_POWER = 0x2b,
+
+	HW_VAR_RCR = 0x2c,
+	HW_VAR_RATR_0 = 0x2d,
+	HW_VAR_RRSR = 0x2e,
+	HW_VAR_CPU_RST = 0x2f,
+	HW_VAR_CHECK_BSSID = 0x30,
+	HW_VAR_LBK_MODE = 0x31,
+	HW_VAR_AES_11N_FIX = 0x32,
+	HW_VAR_USB_RX_AGGR = 0x33,
+	HW_VAR_USER_CONTROL_TURBO_MODE = 0x34,
+	HW_VAR_RETRY_LIMIT = 0x35,
+	HW_VAR_INIT_TX_RATE = 0x36,
+	HW_VAR_TX_RATE_REG = 0x37,
+	HW_VAR_EFUSE_USAGE = 0x38,
+	HW_VAR_EFUSE_BYTES = 0x39,
+	HW_VAR_AUTOLOAD_STATUS = 0x3a,
+	HW_VAR_RF_2R_DISABLE = 0x3b,
+	HW_VAR_SET_RPWM = 0x3c,
+	HW_VAR_H2C_FW_PWRMODE = 0x3d,
+	HW_VAR_H2C_FW_JOINBSSRPT = 0x3e,
+	HW_VAR_H2C_FW_MEDIASTATUSRPT = 0x3f,
+	HW_VAR_H2C_FW_P2P_PS_OFFLOAD = 0x40,
+	HW_VAR_FW_PSMODE_STATUS = 0x41,
+	HW_VAR_INIT_RTS_RATE = 0x42,
+	HW_VAR_RESUME_CLK_ON = 0x43,
+	HW_VAR_FW_LPS_ACTION = 0x44,
+	HW_VAR_1X1_RECV_COMBINE = 0x45,
+	HW_VAR_STOP_SEND_BEACON = 0x46,
+	HW_VAR_TSF_TIMER = 0x47,
+	HW_VAR_IO_CMD = 0x48,
+
+	HW_VAR_RF_RECOVERY = 0x49,
+	HW_VAR_H2C_FW_UPDATE_GTK = 0x4a,
+	HW_VAR_WF_MASK = 0x4b,
+	HW_VAR_WF_CRC = 0x4c,
+	HW_VAR_WF_IS_MAC_ADDR = 0x4d,
+	HW_VAR_H2C_FW_OFFLOAD = 0x4e,
+	HW_VAR_RESET_WFCRC = 0x4f,
+
+	HW_VAR_HANDLE_FW_C2H = 0x50,
+	HW_VAR_DL_FW_RSVD_PAGE = 0x51,
+	HW_VAR_AID = 0x52,
+	HW_VAR_HW_SEQ_ENABLE = 0x53,
+	HW_VAR_CORRECT_TSF = 0x54,
+	HW_VAR_BCN_VALID = 0x55,
+	HW_VAR_FWLPS_RF_ON = 0x56,
+	HW_VAR_DUAL_TSF_RST = 0x57,
+	HW_VAR_SWITCH_EPHY_WOWLAN = 0x58,
+	HW_VAR_INT_MIGRATION = 0x59,
+	HW_VAR_INT_AC = 0x5a,
+	HW_VAR_RF_TIMING = 0x5b,
+
+	HAL_DEF_WOWLAN = 0x5c,
+	HW_VAR_MRC = 0x5d,
+	HW_VAR_KEEP_ALIVE = 0x5e,
+	HW_VAR_NAV_UPPER = 0x5f,
+
+	HW_VAR_MGT_FILTER = 0x60,
+	HW_VAR_CTRL_FILTER = 0x61,
+	HW_VAR_DATA_FILTER = 0x62,
+};
+
+enum rt_media_status {
+	RT_MEDIA_DISCONNECT = 0,
+	RT_MEDIA_CONNECT = 1
+};
+
+enum rt_oem_id {
+	RT_CID_DEFAULT = 0,
+	RT_CID_8187_ALPHA0 = 1,
+	RT_CID_8187_SERCOMM_PS = 2,
+	RT_CID_8187_HW_LED = 3,
+	RT_CID_8187_NETGEAR = 4,
+	RT_CID_WHQL = 5,
+	RT_CID_819X_CAMEO = 6,
+	RT_CID_819X_RUNTOP = 7,
+	RT_CID_819X_SENAO = 8,
+	RT_CID_TOSHIBA = 9,
+	RT_CID_819X_NETCORE = 10,
+	RT_CID_NETTRONIX = 11,
+	RT_CID_DLINK = 12,
+	RT_CID_PRONET = 13,
+	RT_CID_COREGA = 14,
+	RT_CID_819X_ALPHA = 15,
+	RT_CID_819X_SITECOM = 16,
+	RT_CID_CCX = 17,
+	RT_CID_819X_LENOVO = 18,
+	RT_CID_819X_QMI = 19,
+	RT_CID_819X_EDIMAX_BELKIN = 20,
+	RT_CID_819X_SERCOMM_BELKIN = 21,
+	RT_CID_819X_CAMEO1 = 22,
+	RT_CID_819X_MSI = 23,
+	RT_CID_819X_ACER = 24,
+	RT_CID_819X_HP = 27,
+	RT_CID_819X_CLEVO = 28,
+	RT_CID_819X_ARCADYAN_BELKIN = 29,
+	RT_CID_819X_SAMSUNG = 30,
+	RT_CID_819X_WNC_COREGA = 31,
+	RT_CID_819X_FOXCOON = 32,
+	RT_CID_819X_DELL = 33,
+	RT_CID_819X_PRONETS = 34,
+	RT_CID_819X_EDIMAX_ASUS = 35,
+	RT_CID_NETGEAR = 36,
+	RT_CID_PLANEX = 37,
+	RT_CID_CC_C = 38,
+};
+
+enum hw_descs {
+	HW_DESC_OWN,
+	HW_DESC_RXOWN,
+	HW_DESC_TX_NEXTDESC_ADDR,
+	HW_DESC_TXBUFF_ADDR,
+	HW_DESC_RXBUFF_ADDR,
+	HW_DESC_RXPKT_LEN,
+	HW_DESC_RXERO,
+	HW_DESC_RX_PREPARE,
+};
+
+enum prime_sc {
+	PRIME_CHNL_OFFSET_DONT_CARE = 0,
+	PRIME_CHNL_OFFSET_LOWER = 1,
+	PRIME_CHNL_OFFSET_UPPER = 2,
+};
+
+enum rf_type {
+	RF_1T1R = 0,
+	RF_1T2R = 1,
+	RF_2T2R = 2,
+	RF_2T2R_GREEN = 3,
+	RF_2T3R = 4,
+	RF_2T4R = 5,
+	RF_3T3R = 6,
+	RF_3T4R = 7,
+	RF_4T4R = 8,
+};
+
+enum ht_channel_width {
+	HT_CHANNEL_WIDTH_20 = 0,
+	HT_CHANNEL_WIDTH_20_40 = 1,
+	HT_CHANNEL_WIDTH_80 = 2,
+	HT_CHANNEL_WIDTH_MAX,
+};
+
+/* Ref: 802.11i spec D10.0 7.3.2.25.1
+ * Cipher Suites Encryption Algorithms
+ */
+enum rt_enc_alg {
+	NO_ENCRYPTION = 0,
+	WEP40_ENCRYPTION = 1,
+	TKIP_ENCRYPTION = 2,
+	RSERVED_ENCRYPTION = 3,
+	AESCCMP_ENCRYPTION = 4,
+	WEP104_ENCRYPTION = 5,
+	AESCMAC_ENCRYPTION = 6,	/*IEEE802.11w */
+};
+
+enum rtl_hal_state {
+	_HAL_STATE_STOP = 0,
+	_HAL_STATE_START = 1,
+};
+
+enum rtl_desc_rate {
+	DESC_RATE1M = 0x00,
+	DESC_RATE2M = 0x01,
+	DESC_RATE5_5M = 0x02,
+	DESC_RATE11M = 0x03,
+
+	DESC_RATE6M = 0x04,
+	DESC_RATE9M = 0x05,
+	DESC_RATE12M = 0x06,
+	DESC_RATE18M = 0x07,
+	DESC_RATE24M = 0x08,
+	DESC_RATE36M = 0x09,
+	DESC_RATE48M = 0x0a,
+	DESC_RATE54M = 0x0b,
+
+	DESC_RATEMCS0 = 0x0c,
+	DESC_RATEMCS1 = 0x0d,
+	DESC_RATEMCS2 = 0x0e,
+	DESC_RATEMCS3 = 0x0f,
+	DESC_RATEMCS4 = 0x10,
+	DESC_RATEMCS5 = 0x11,
+	DESC_RATEMCS6 = 0x12,
+	DESC_RATEMCS7 = 0x13,
+	DESC_RATEMCS8 = 0x14,
+	DESC_RATEMCS9 = 0x15,
+	DESC_RATEMCS10 = 0x16,
+	DESC_RATEMCS11 = 0x17,
+	DESC_RATEMCS12 = 0x18,
+	DESC_RATEMCS13 = 0x19,
+	DESC_RATEMCS14 = 0x1a,
+	DESC_RATEMCS15 = 0x1b,
+	DESC_RATEMCS15_SG = 0x1c,
+	DESC_RATEMCS32 = 0x20,
+
+	DESC_RATEVHT1SS_MCS0 = 0x2c,
+	DESC_RATEVHT1SS_MCS1 = 0x2d,
+	DESC_RATEVHT1SS_MCS2 = 0x2e,
+	DESC_RATEVHT1SS_MCS3 = 0x2f,
+	DESC_RATEVHT1SS_MCS4 = 0x30,
+	DESC_RATEVHT1SS_MCS5 = 0x31,
+	DESC_RATEVHT1SS_MCS6 = 0x32,
+	DESC_RATEVHT1SS_MCS7 = 0x33,
+	DESC_RATEVHT1SS_MCS8 = 0x34,
+	DESC_RATEVHT1SS_MCS9 = 0x35,
+	DESC_RATEVHT2SS_MCS0 = 0x36,
+	DESC_RATEVHT2SS_MCS1 = 0x37,
+	DESC_RATEVHT2SS_MCS2 = 0x38,
+	DESC_RATEVHT2SS_MCS3 = 0x39,
+	DESC_RATEVHT2SS_MCS4 = 0x3a,
+	DESC_RATEVHT2SS_MCS5 = 0x3b,
+	DESC_RATEVHT2SS_MCS6 = 0x3c,
+	DESC_RATEVHT2SS_MCS7 = 0x3d,
+	DESC_RATEVHT2SS_MCS8 = 0x3e,
+	DESC_RATEVHT2SS_MCS9 = 0x3f,
+};
+
+enum rtl_var_map {
+	/*reg map */
+	SYS_ISO_CTRL = 0,
+	SYS_FUNC_EN,
+	SYS_CLK,
+	MAC_RCR_AM,
+	MAC_RCR_AB,
+	MAC_RCR_ACRC32,
+	MAC_RCR_ACF,
+	MAC_RCR_AAP,
+	MAC_HIMR,
+	MAC_HIMRE,
+	MAC_HSISR,
+
+	/*efuse map */
+	EFUSE_TEST,
+	EFUSE_CTRL,
+	EFUSE_CLK,
+	EFUSE_CLK_CTRL,
+	EFUSE_PWC_EV12V,
+	EFUSE_FEN_ELDR,
+	EFUSE_LOADER_CLK_EN,
+	EFUSE_ANA8M,
+	EFUSE_HWSET_MAX_SIZE,
+	EFUSE_MAX_SECTION_MAP,
+	EFUSE_REAL_CONTENT_SIZE,
+	EFUSE_OOB_PROTECT_BYTES_LEN,
+	EFUSE_ACCESS,
+
+	/*CAM map */
+	RWCAM,
+	WCAMI,
+	RCAMO,
+	CAMDBG,
+	SECR,
+	SEC_CAM_NONE,
+	SEC_CAM_WEP40,
+	SEC_CAM_TKIP,
+	SEC_CAM_AES,
+	SEC_CAM_WEP104,
+
+	/*IMR map */
+	RTL_IMR_BCNDMAINT6,	/*Beacon DMA Interrupt 6 */
+	RTL_IMR_BCNDMAINT5,	/*Beacon DMA Interrupt 5 */
+	RTL_IMR_BCNDMAINT4,	/*Beacon DMA Interrupt 4 */
+	RTL_IMR_BCNDMAINT3,	/*Beacon DMA Interrupt 3 */
+	RTL_IMR_BCNDMAINT2,	/*Beacon DMA Interrupt 2 */
+	RTL_IMR_BCNDMAINT1,	/*Beacon DMA Interrupt 1 */
+	RTL_IMR_BCNDOK8,	/*Beacon Queue DMA OK Interrupt 8 */
+	RTL_IMR_BCNDOK7,	/*Beacon Queue DMA OK Interrupt 7 */
+	RTL_IMR_BCNDOK6,	/*Beacon Queue DMA OK Interrupt 6 */
+	RTL_IMR_BCNDOK5,	/*Beacon Queue DMA OK Interrupt 5 */
+	RTL_IMR_BCNDOK4,	/*Beacon Queue DMA OK Interrupt 4 */
+	RTL_IMR_BCNDOK3,	/*Beacon Queue DMA OK Interrupt 3 */
+	RTL_IMR_BCNDOK2,	/*Beacon Queue DMA OK Interrupt 2 */
+	RTL_IMR_BCNDOK1,	/*Beacon Queue DMA OK Interrupt 1 */
+	RTL_IMR_TIMEOUT2,	/*Timeout interrupt 2 */
+	RTL_IMR_TIMEOUT1,	/*Timeout interrupt 1 */
+	RTL_IMR_TXFOVW,		/*Transmit FIFO Overflow */
+	RTL_IMR_PSTIMEOUT,	/*Power save time out interrupt */
+	RTL_IMR_BCNINT,		/*Beacon DMA Interrupt 0 */
+	RTL_IMR_RXFOVW,		/*Receive FIFO Overflow */
+	RTL_IMR_RDU,		/*Receive Descriptor Unavailable */
+	RTL_IMR_ATIMEND,	/*For 92C,ATIM Window End Interrupt */
+	RTL_IMR_H2CDOK,		/*H2C Queue DMA OK Interrupt */
+	RTL_IMR_BDOK,		/*Beacon Queue DMA OK Interrupt */
+	RTL_IMR_HIGHDOK,	/*High Queue DMA OK Interrupt */
+	RTL_IMR_COMDOK,		/*Command Queue DMA OK Interrupt*/
+	RTL_IMR_TBDOK,		/*Transmit Beacon OK interrupt */
+	RTL_IMR_MGNTDOK,	/*Management Queue DMA OK Interrupt */
+	RTL_IMR_TBDER,		/*For 92C,Transmit Beacon Error Interrupt */
+	RTL_IMR_BKDOK,		/*AC_BK DMA OK Interrupt */
+	RTL_IMR_BEDOK,		/*AC_BE DMA OK Interrupt */
+	RTL_IMR_VIDOK,		/*AC_VI DMA OK Interrupt */
+	RTL_IMR_VODOK,		/*AC_VO DMA Interrupt */
+	RTL_IMR_ROK,		/*Receive DMA OK Interrupt */
+	RTL_IMR_HSISR_IND,	/*HSISR Interrupt*/
+	RTL_IBSS_INT_MASKS,	/*(RTL_IMR_BCNINT | RTL_IMR_TBDOK |
+				 * RTL_IMR_TBDER)
+				 */
+	RTL_IMR_C2HCMD,		/*fw interrupt*/
+
+	/*CCK Rates, TxHT = 0 */
+	RTL_RC_CCK_RATE1M,
+	RTL_RC_CCK_RATE2M,
+	RTL_RC_CCK_RATE5_5M,
+	RTL_RC_CCK_RATE11M,
+
+	/*OFDM Rates, TxHT = 0 */
+	RTL_RC_OFDM_RATE6M,
+	RTL_RC_OFDM_RATE9M,
+	RTL_RC_OFDM_RATE12M,
+	RTL_RC_OFDM_RATE18M,
+	RTL_RC_OFDM_RATE24M,
+	RTL_RC_OFDM_RATE36M,
+	RTL_RC_OFDM_RATE48M,
+	RTL_RC_OFDM_RATE54M,
+
+	RTL_RC_HT_RATEMCS7,
+	RTL_RC_HT_RATEMCS15,
+
+	RTL_RC_VHT_RATE_1SS_MCS7,
+	RTL_RC_VHT_RATE_1SS_MCS8,
+	RTL_RC_VHT_RATE_1SS_MCS9,
+	RTL_RC_VHT_RATE_2SS_MCS7,
+	RTL_RC_VHT_RATE_2SS_MCS8,
+	RTL_RC_VHT_RATE_2SS_MCS9,
+
+	/*keep it last */
+	RTL_VAR_MAP_MAX,
+};
+
+/*Firmware PS mode for control LPS.*/
+enum _fw_ps_mode {
+	FW_PS_ACTIVE_MODE = 0,
+	FW_PS_MIN_MODE = 1,
+	FW_PS_MAX_MODE = 2,
+	FW_PS_DTIM_MODE = 3,
+	FW_PS_VOIP_MODE = 4,
+	FW_PS_UAPSD_WMM_MODE = 5,
+	FW_PS_UAPSD_MODE = 6,
+	FW_PS_IBSS_MODE = 7,
+	FW_PS_WWLAN_MODE = 8,
+	FW_PS_PM_RADIO_OFF = 9,
+	FW_PS_PM_CARD_DISABLE = 10,
+};
+
+enum rt_psmode {
+	EACTIVE,		/*Active/Continuous access. */
+	EMAXPS,			/*Max power save mode. */
+	EFASTPS,		/*Fast power save mode. */
+	EAUTOPS,		/*Auto power save mode. */
+};
+
+/*LED related.*/
+enum led_ctl_mode {
+	LED_CTL_POWER_ON = 1,
+	LED_CTL_LINK = 2,
+	LED_CTL_NO_LINK = 3,
+	LED_CTL_TX = 4,
+	LED_CTL_RX = 5,
+	LED_CTL_SITE_SURVEY = 6,
+	LED_CTL_POWER_OFF = 7,
+	LED_CTL_START_TO_LINK = 8,
+	LED_CTL_START_WPS = 9,
+	LED_CTL_STOP_WPS = 10,
+};
+
+enum rtl_led_pin {
+	LED_PIN_GPIO0,
+	LED_PIN_LED0,
+	LED_PIN_LED1,
+	LED_PIN_LED2
+};
+
+/* QoS related.*/
+/* acm implementation method.*/
+enum acm_method {
+	EACMWAY0_SWANDHW = 0,
+	EACMWAY1_HW = 1,
+	EACMWAY2_SW = 2,
+};
+
+enum macphy_mode {
+	SINGLEMAC_SINGLEPHY = 0,
+	DUALMAC_DUALPHY,
+	DUALMAC_SINGLEPHY,
+};
+
+enum band_type {
+	BAND_ON_2_4G = 0,
+	BAND_ON_5G,
+	BAND_ON_BOTH,
+	BANDMAX
+};
+
+/* aci/aifsn Field.
+ * Ref: WMM spec 2.2.2: WME Parameter Element, p.12.
+ */
+union aci_aifsn {
+	u8 char_data;
+
+	struct {
+		u8 aifsn:4;
+		u8 acm:1;
+		u8 aci:2;
+		u8 reserved:1;
+	} f;			/* Field */
+};
+
+/*mlme related.*/
+enum wireless_mode {
+	WIRELESS_MODE_UNKNOWN = 0x00,
+	WIRELESS_MODE_A = 0x01,
+	WIRELESS_MODE_B = 0x02,
+	WIRELESS_MODE_G = 0x04,
+	WIRELESS_MODE_AUTO = 0x08,
+	WIRELESS_MODE_N_24G = 0x10,
+	WIRELESS_MODE_N_5G = 0x20,
+	WIRELESS_MODE_AC_5G = 0x40,
+	WIRELESS_MODE_AC_24G  = 0x80,
+	WIRELESS_MODE_AC_ONLY = 0x100,
+	WIRELESS_MODE_MAX = 0x800
+};
+
+#define IS_WIRELESS_MODE_A(wirelessmode)	\
+	(wirelessmode == WIRELESS_MODE_A)
+#define IS_WIRELESS_MODE_B(wirelessmode)	\
+	(wirelessmode == WIRELESS_MODE_B)
+#define IS_WIRELESS_MODE_G(wirelessmode)	\
+	(wirelessmode == WIRELESS_MODE_G)
+#define IS_WIRELESS_MODE_N_24G(wirelessmode)	\
+	(wirelessmode == WIRELESS_MODE_N_24G)
+#define IS_WIRELESS_MODE_N_5G(wirelessmode)	\
+	(wirelessmode == WIRELESS_MODE_N_5G)
+
+enum ratr_table_mode {
+	RATR_INX_WIRELESS_NGB = 0,
+	RATR_INX_WIRELESS_NG = 1,
+	RATR_INX_WIRELESS_NB = 2,
+	RATR_INX_WIRELESS_N = 3,
+	RATR_INX_WIRELESS_GB = 4,
+	RATR_INX_WIRELESS_G = 5,
+	RATR_INX_WIRELESS_B = 6,
+	RATR_INX_WIRELESS_MC = 7,
+	RATR_INX_WIRELESS_A = 8,
+	RATR_INX_WIRELESS_AC_5N = 8,
+	RATR_INX_WIRELESS_AC_24N = 9,
+};
+
+enum ratr_table_mode_new {
+	RATEID_IDX_BGN_40M_2SS = 0,
+	RATEID_IDX_BGN_40M_1SS = 1,
+	RATEID_IDX_BGN_20M_2SS_BN = 2,
+	RATEID_IDX_BGN_20M_1SS_BN = 3,
+	RATEID_IDX_GN_N2SS = 4,
+	RATEID_IDX_GN_N1SS = 5,
+	RATEID_IDX_BG = 6,
+	RATEID_IDX_G = 7,
+	RATEID_IDX_B = 8,
+	RATEID_IDX_VHT_2SS = 9,
+	RATEID_IDX_VHT_1SS = 10,
+	RATEID_IDX_MIX1 = 11,
+	RATEID_IDX_MIX2 = 12,
+	RATEID_IDX_VHT_3SS = 13,
+	RATEID_IDX_BGN_3SS = 14,
+};
+
+enum rtl_link_state {
+	MAC80211_NOLINK = 0,
+	MAC80211_LINKING = 1,
+	MAC80211_LINKED = 2,
+	MAC80211_LINKED_SCANNING = 3,
+};
+
+enum act_category {
+	ACT_CAT_QOS = 1,
+	ACT_CAT_DLS = 2,
+	ACT_CAT_BA = 3,
+	ACT_CAT_HT = 7,
+	ACT_CAT_WMM = 17,
+};
+
+enum ba_action {
+	ACT_ADDBAREQ = 0,
+	ACT_ADDBARSP = 1,
+	ACT_DELBA = 2,
+};
+
+enum rt_polarity_ctl {
+	RT_POLARITY_LOW_ACT = 0,
+	RT_POLARITY_HIGH_ACT = 1,
+};
+
+/* After 8188E, we use V2 reason define. 88C/8723A use V1 reason. */
+enum fw_wow_reason_v2 {
+	FW_WOW_V2_PTK_UPDATE_EVENT = 0x01,
+	FW_WOW_V2_GTK_UPDATE_EVENT = 0x02,
+	FW_WOW_V2_DISASSOC_EVENT = 0x04,
+	FW_WOW_V2_DEAUTH_EVENT = 0x08,
+	FW_WOW_V2_FW_DISCONNECT_EVENT = 0x10,
+	FW_WOW_V2_MAGIC_PKT_EVENT = 0x21,
+	FW_WOW_V2_UNICAST_PKT_EVENT = 0x22,
+	FW_WOW_V2_PATTERN_PKT_EVENT = 0x23,
+	FW_WOW_V2_RTD3_SSID_MATCH_EVENT = 0x24,
+	FW_WOW_V2_REALWOW_V2_WAKEUPPKT = 0x30,
+	FW_WOW_V2_REALWOW_V2_ACKLOST = 0x31,
+	FW_WOW_V2_REASON_MAX = 0xff,
+};
+
+enum wolpattern_type {
+	UNICAST_PATTERN = 0,
+	MULTICAST_PATTERN = 1,
+	BROADCAST_PATTERN = 2,
+	DONT_CARE_DA = 3,
+	UNKNOWN_TYPE = 4,
+};
+
+enum package_type {
+	PACKAGE_DEFAULT,
+	PACKAGE_QFN68,
+	PACKAGE_TFBGA90,
+	PACKAGE_TFBGA80,
+	PACKAGE_TFBGA79
+};
+
+enum rtl_spec_ver {
+	RTL_SPEC_NEW_RATEID = BIT(0),	/* use ratr_table_mode_new */
+	RTL_SPEC_SUPPORT_VHT = BIT(1),	/* support VHT */
+	RTL_SPEC_NEW_FW_C2H = BIT(2),	/* new FW C2H (e.g. TX REPORT) */
+};
+
+struct octet_string {
+	u8 *octet;
+	u16 length;
+};
+
+struct rtl_hdr_3addr {
+	__le16 frame_ctl;
+	__le16 duration_id;
+	u8 addr1[ETH_ALEN];
+	u8 addr2[ETH_ALEN];
+	u8 addr3[ETH_ALEN];
+	__le16 seq_ctl;
+	u8 payload[0];
+} __packed;
+
+struct rtl_info_element {
+	u8 id;
+	u8 len;
+	u8 data[0];
+} __packed;
+
+struct rtl_probe_rsp {
+	struct rtl_hdr_3addr header;
+	u32 time_stamp[2];
+	__le16 beacon_interval;
+	__le16 capability;
+	/* SSID, supported rates, FH params, DS params,
+	 * CF params, IBSS params, TIM (if beacon), RSN
+	 */
+	struct rtl_info_element info_element[0];
+} __packed;
+
+struct rtl_beacon_keys {
+	/*u8 ssid[32];*/
+	/*u32 ssid_len;*/
+	u8 bcn_channel;
+	__le16 ht_cap_info;
+	u8 ht_info_infos_0_sco; /* bit0 & bit1 in infos[0] is 2nd ch offset */
+	bool valid;
+};
+
+/*LED related.*/
+/*ledpin Identify how to implement this SW led.*/
+struct rtl_led {
+	void *hw;
+	enum rtl_led_pin ledpin;
+	bool ledon;
+};
+
+struct rtl_led_ctl {
+	bool led_opendrain;
+	struct rtl_led sw_led0;
+	struct rtl_led sw_led1;
+};
+
+struct rtl_qos_parameters {
+	__le16 cw_min;
+	__le16 cw_max;
+	u8 aifs;
+	u8 flag;
+	__le16 tx_op;
+} __packed;
+
+struct rt_smooth_data {
+	u32 elements[100];	/*array to store values */
+	u32 index;		/*index to current array to store */
+	u32 total_num;		/*num of valid elements */
+	u32 total_val;		/*sum of valid elements */
+};
+
+struct false_alarm_statistics {
+	u32 cnt_parity_fail;
+	u32 cnt_rate_illegal;
+	u32 cnt_crc8_fail;
+	u32 cnt_mcs_fail;
+	u32 cnt_fast_fsync_fail;
+	u32 cnt_sb_search_fail;
+	u32 cnt_ofdm_fail;
+	u32 cnt_cck_fail;
+	u32 cnt_all;
+	u32 cnt_ofdm_cca;
+	u32 cnt_cck_cca;
+	u32 cnt_cca_all;
+	u32 cnt_bw_usc;
+	u32 cnt_bw_lsc;
+};
+
+struct init_gain {
+	u8 xaagccore1;
+	u8 xbagccore1;
+	u8 xcagccore1;
+	u8 xdagccore1;
+	u8 cca;
+
+};
+
+struct wireless_stats {
+	u64 txbytesunicast;
+	u64 txbytesmulticast;
+	u64 txbytesbroadcast;
+	u64 rxbytesunicast;
+
+	u64 txbytesunicast_inperiod;
+	u64 rxbytesunicast_inperiod;
+	u32 txbytesunicast_inperiod_tp;
+	u32 rxbytesunicast_inperiod_tp;
+	u64 txbytesunicast_last;
+	u64 rxbytesunicast_last;
+
+	long rx_snr_db[4];
+	/* Correct smoothed ss in Dbm, only used
+	 * in driver to report real power now.
+	 */
+	long recv_signal_power;
+	long signal_quality;
+	long last_sigstrength_inpercent;
+
+	u32 rssi_calculate_cnt;
+	u32 pwdb_all_cnt;
+
+	/* Transformed, in dbm. Beautified signal
+	 * strength for UI, not correct.
+	 */
+	long signal_strength;
+
+	u8 rx_rssi_percentage[4];
+	u8 rx_evm_dbm[4];
+	u8 rx_evm_percentage[2];
+
+	u16 rx_cfo_short[4];
+	u16 rx_cfo_tail[4];
+
+	struct rt_smooth_data ui_rssi;
+	struct rt_smooth_data ui_link_quality;
+};
+
+struct rate_adaptive {
+	u8 rate_adaptive_disabled;
+	u8 ratr_state;
+	u16 reserve;
+
+	u32 high_rssi_thresh_for_ra;
+	u32 high2low_rssi_thresh_for_ra;
+	u8 low2high_rssi_thresh_for_ra40m;
+	u32 low_rssi_thresh_for_ra40m;
+	u8 low2high_rssi_thresh_for_ra20m;
+	u32 low_rssi_thresh_for_ra20m;
+	u32 upper_rssi_threshold_ratr;
+	u32 middleupper_rssi_threshold_ratr;
+	u32 middle_rssi_threshold_ratr;
+	u32 middlelow_rssi_threshold_ratr;
+	u32 low_rssi_threshold_ratr;
+	u32 ultralow_rssi_threshold_ratr;
+	u32 low_rssi_threshold_ratr_40m;
+	u32 low_rssi_threshold_ratr_20m;
+	u8 ping_rssi_enable;
+	u32 ping_rssi_ratr;
+	u32 ping_rssi_thresh_for_ra;
+	u32 last_ratr;
+	u8 pre_ratr_state;
+	u8 ldpc_thres;
+	bool use_ldpc;
+	bool lower_rts_rate;
+	bool is_special_data;
+};
+
+struct regd_pair_mapping {
+	u16 reg_dmnenum;
+	u16 reg_5ghz_ctl;
+	u16 reg_2ghz_ctl;
+};
+
+struct dynamic_primary_cca {
+	u8 pricca_flag;
+	u8 intf_flag;
+	u8 intf_type;
+	u8 dup_rts_flag;
+	u8 monitor_flag;
+	u8 ch_offset;
+	u8 mf_state;
+};
+
+struct rtl_regulatory {
+	s8 alpha2[2];
+	u16 country_code;
+	u16 max_power_level;
+	u32 tp_scale;
+	u16 current_rd;
+	u16 current_rd_ext;
+	s16 power_limit;
+	struct regd_pair_mapping *regpair;
+};
+
+struct rtl_rfkill {
+	bool rfkill_state;	/*0 is off, 1 is on */
+};
+
+/*for P2P PS**/
+#define	P2P_MAX_NOA_NUM		2
+
+enum p2p_role {
+	P2P_ROLE_DISABLE = 0,
+	P2P_ROLE_DEVICE = 1,
+	P2P_ROLE_CLIENT = 2,
+	P2P_ROLE_GO = 3
+};
+
+enum p2p_ps_state {
+	P2P_PS_DISABLE = 0,
+	P2P_PS_ENABLE = 1,
+	P2P_PS_SCAN = 2,
+	P2P_PS_SCAN_DONE = 3,
+	P2P_PS_ALLSTASLEEP = 4, /* for P2P GO */
+};
+
+enum p2p_ps_mode {
+	P2P_PS_NONE = 0,
+	P2P_PS_CTWINDOW = 1,
+	P2P_PS_NOA	 = 2,
+	P2P_PS_MIX = 3, /* CTWindow and NoA */
+};
+
+struct rtl_p2p_ps_info {
+	enum p2p_ps_mode p2p_ps_mode; /* indicate p2p ps mode */
+	enum p2p_ps_state p2p_ps_state; /*  indicate p2p ps state */
+	u8 noa_index; /*  Identifies instance of Notice of Absence timing. */
+	/*  Client traffic window. A period of time in TU after TBTT. */
+	u8 ctwindow;
+	u8 opp_ps; /*  opportunistic power save. */
+	u8 noa_num; /*  number of NoA descriptor in P2P IE. */
+	/*  Count for owner, Type of client. */
+	u8 noa_count_type[P2P_MAX_NOA_NUM];
+	/*  Max duration for owner, preferred or min acceptable duration
+	 * for client.
+	 */
+	u32 noa_duration[P2P_MAX_NOA_NUM];
+	/*  Length of interval for owner, preferred or max acceptable intervali
+	 * of client.
+	 */
+	u32 noa_interval[P2P_MAX_NOA_NUM];
+	/*  schedule in terms of the lower 4 bytes of the TSF timer. */
+	u32 noa_start_time[P2P_MAX_NOA_NUM];
+};
+
+struct p2p_ps_offload_t {
+	u8 offload_en:1;
+	u8 role:1; /* 1: Owner, 0: Client */
+	u8 ctwindow_en:1;
+	u8 noa0_en:1;
+	u8 noa1_en:1;
+	u8 allstasleep:1;
+	u8 discovery:1;
+	u8 reserved:1;
+};
+
+#define IQK_MATRIX_REG_NUM	8
+#define IQK_MATRIX_SETTINGS_NUM	(1 + 24 + 21)
+
+struct iqk_matrix_regs {
+	bool iqk_done;
+	long value[1][IQK_MATRIX_REG_NUM];
+};
+
+struct phy_parameters {
+	u16 length;
+	u32 *pdata;
+};
+
+enum hw_param_tab_index {
+	PHY_REG_2T,
+	PHY_REG_1T,
+	PHY_REG_PG,
+	RADIOA_2T,
+	RADIOB_2T,
+	RADIOA_1T,
+	RADIOB_1T,
+	MAC_REG,
+	AGCTAB_2T,
+	AGCTAB_1T,
+	MAX_TAB
+};
+
+struct rtl_phy {
+	struct bb_reg_def phyreg_def[4];	/*Radio A/B/C/D */
+	struct init_gain initgain_backup;
+	enum io_type current_io_type;
+
+	u8 rf_mode;
+	u8 rf_type;
+	u8 current_chan_bw;
+	u8 max_ht_chan_bw;
+	u8 max_vht_chan_bw;
+	u8 set_bwmode_inprogress;
+	u8 sw_chnl_inprogress;
+	u8 sw_chnl_stage;
+	u8 sw_chnl_step;
+	u8 current_channel;
+	u8 h2c_box_num;
+	u8 set_io_inprogress;
+	u8 lck_inprogress;
+
+	/* record for power tracking */
+	s32 reg_e94;
+	s32 reg_e9c;
+	s32 reg_ea4;
+	s32 reg_eac;
+	s32 reg_eb4;
+	s32 reg_ebc;
+	s32 reg_ec4;
+	s32 reg_ecc;
+	u8 rfpienable;
+	u8 reserve_0;
+	u16 reserve_1;
+	u32 reg_c04, reg_c08, reg_874;
+	u32 adda_backup[16];
+	u32 iqk_mac_backup[IQK_MAC_REG_NUM];
+	u32 iqk_bb_backup[10];
+	bool iqk_initialized;
+
+	bool rfpath_rx_enable[MAX_RF_PATH];
+	u8 reg_837;
+	/* Dual mac */
+	bool need_iqk;
+	struct iqk_matrix_regs iqk_matrix[IQK_MATRIX_SETTINGS_NUM];
+
+	bool rfpi_enable;
+	bool iqk_in_progress;
+
+	u8 pwrgroup_cnt;
+	u8 cck_high_power;
+	/* this is for 88E & 8723A */
+	u32 mcs_txpwrlevel_origoffset[MAX_PG_GROUP][16];
+	/* MAX_PG_GROUP groups of pwr diff by rates */
+	u32 mcs_offset[MAX_PG_GROUP][16];
+	u32 tx_power_by_rate_offset[TX_PWR_BY_RATE_NUM_BAND]
+				   [TX_PWR_BY_RATE_NUM_RF]
+				   [TX_PWR_BY_RATE_NUM_RF]
+				   [TX_PWR_BY_RATE_NUM_RATE];
+	/* compatible with TX_PWR_BY_RATE_NUM_SECTION*/
+	u8 txpwr_by_rate_base_24g[TX_PWR_BY_RATE_NUM_RF]
+				 [TX_PWR_BY_RATE_NUM_RF]
+				 [MAX_BASE_NUM_IN_PHY_REG_PG_24G];
+	u8 txpwr_by_rate_base_5g[TX_PWR_BY_RATE_NUM_RF]
+				[TX_PWR_BY_RATE_NUM_RF]
+				[MAX_BASE_NUM_IN_PHY_REG_PG_5G];
+	u8 default_initialgain[4];
+
+	/* the current Tx power level */
+	u8 cur_cck_txpwridx;
+	u8 cur_ofdm24g_txpwridx;
+	u8 cur_bw20_txpwridx;
+	u8 cur_bw40_txpwridx;
+
+	s8 txpwr_limit_2_4g[MAX_REGULATION_NUM]
+			   [MAX_2_4G_BANDWIDTH_NUM]
+			   [MAX_RATE_SECTION_NUM]
+			   [CHANNEL_MAX_NUMBER_2G]
+			   [MAX_RF_PATH_NUM];
+	s8 txpwr_limit_5g[MAX_REGULATION_NUM]
+			 [MAX_5G_BANDWIDTH_NUM]
+			 [MAX_RATE_SECTION_NUM]
+			 [CHANNEL_MAX_NUMBER_5G]
+			 [MAX_RF_PATH_NUM];
+
+	u32 rfreg_chnlval[2];
+	bool apk_done;
+	u32 reg_rf3c[2];	/* pathA / pathB  */
+
+	u32 backup_rf_0x1a;/*92ee*/
+	/* bfsync */
+	u8 framesync;
+	u32 framesync_c34;
+
+	u8 num_total_rfpath;
+	struct phy_parameters hwparam_tables[MAX_TAB];
+	u16 rf_pathmap;
+
+	u8 hw_rof_enable; /*Enable GPIO[9] as WL RF HW PDn source*/
+	enum rt_polarity_ctl polarity_ctl;
+};
+
+#define MAX_TID_COUNT				9
+#define RTL_AGG_STOP				0
+#define RTL_AGG_PROGRESS			1
+#define RTL_AGG_START				2
+#define RTL_AGG_OPERATIONAL			3
+#define RTL_AGG_OFF				0
+#define RTL_AGG_ON				1
+#define RTL_RX_AGG_START			1
+#define RTL_RX_AGG_STOP				0
+#define RTL_AGG_EMPTYING_HW_QUEUE_ADDBA		2
+#define RTL_AGG_EMPTYING_HW_QUEUE_DELBA		3
+
+struct rtl_ht_agg {
+	u16 txq_id;
+	u16 wait_for_ba;
+	u16 start_idx;
+	u64 bitmap;
+	u32 rate_n_flags;
+	u8 agg_state;
+	u8 rx_agg_state;
+};
+
+struct rssi_sta {
+	/* for old dm */
+	long undec_sm_pwdb;
+	long undec_sm_cck;
+
+	/* for new phydm_mod */
+	s32 undecorated_smoothed_pwdb;
+	s32 undecorated_smoothed_cck;
+	s32 undecorated_smoothed_ofdm;
+	u8 ofdm_pkt;
+	u8 cck_pkt;
+	u16 cck_sum_power;
+	u8 is_send_rssi;
+	u64 packet_map;
+	u8 valid_bit;
+};
+
+struct rtl_tid_data {
+	u16 seq_number;
+	struct rtl_ht_agg agg;
+};
+
+struct rtl_sta_info {
+	struct list_head list;
+	struct rtl_tid_data tids[MAX_TID_COUNT];
+	/* just used for ap adhoc or mesh*/
+	struct rssi_sta rssi_stat;
+	u8 rssi_level;
+	u16 wireless_mode;
+	u8 ratr_index;
+	u8 mimo_ps;
+	u8 mac_addr[ETH_ALEN];
+} __packed;
+
+struct rtl_priv;
+struct rtl_io {
+	struct device *dev;
+	struct mutex bb_mutex;
+
+	/*PCI MEM map */
+	unsigned long pci_mem_end;	/*shared mem end        */
+	unsigned long pci_mem_start;	/*shared mem start */
+
+	/*PCI IO map */
+	unsigned long pci_base_addr;	/*device I/O address */
+
+	void (*write8_async)(struct rtl_priv *rtlpriv, u32 addr, u8 val);
+	void (*write16_async)(struct rtl_priv *rtlpriv, u32 addr, u16 val);
+	void (*write32_async)(struct rtl_priv *rtlpriv, u32 addr, u32 val);
+	void (*writeN_sync)(struct rtl_priv *rtlpriv, u32 addr, void *buf,
+			    u16 len);
+
+	u8 (*read8_sync)(struct rtl_priv *rtlpriv, u32 addr);
+	u16 (*read16_sync)(struct rtl_priv *rtlpriv, u32 addr);
+	u32 (*read32_sync)(struct rtl_priv *rtlpriv, u32 addr);
+
+};
+
+struct rtl_mac {
+	u8 mac_addr[ETH_ALEN];
+	u8 mac80211_registered;
+	u8 beacon_enabled;
+
+	u32 tx_ss_num;
+	u32 rx_ss_num;
+
+	struct ieee80211_supported_band bands[NUM_NL80211_BANDS];
+	struct ieee80211_hw *hw;
+	struct ieee80211_vif *vif;
+	enum nl80211_iftype opmode;
+
+	/*Probe Beacon management */
+	struct rtl_tid_data tids[MAX_TID_COUNT];
+	enum rtl_link_state link_state;
+	struct rtl_beacon_keys cur_beacon_keys;
+	u8 new_beacon_cnt;
+
+	int n_channels;
+	int n_bitrates;
+
+	bool offchan_delay;
+	u8 p2p;	/*using p2p role*/
+	bool p2p_in_use;
+
+	/*filters */
+	u32 rx_conf;
+	u16 rx_mgt_filter;
+	u16 rx_ctrl_filter;
+	u16 rx_data_filter;
+
+	bool act_scanning;
+	u8 cnt_after_linked;
+	bool skip_scan;
+
+	/* early mode */
+	/* skb wait queue */
+	struct sk_buff_head skb_waitq[MAX_TID_COUNT];
+
+	u8 ht_stbc_cap;
+	u8 ht_cur_stbc;
+
+	/*vht support*/
+	u8 vht_enable;
+	u8 bw_80;
+	u8 vht_cur_ldpc;
+	u8 vht_cur_stbc;
+	u8 vht_stbc_cap;
+	u8 vht_ldpc_cap;
+
+	/*RDG*/
+	bool rdg_en;
+
+	/*AP*/
+	u8 bssid[ETH_ALEN] __aligned(2);
+	u32 vendor;
+	u8 mcs[16];	/* 16 bytes mcs for HT rates. */
+	u32 basic_rates; /* b/g rates */
+	u8 ht_enable;
+	u8 sgi_40;
+	u8 sgi_20;
+	u8 bw_40;
+	u16 mode;		/* wireless mode */
+	u8 slot_time;
+	u8 short_preamble;
+	u8 use_cts_protect;
+	u8 cur_40_prime_sc;
+	u8 cur_40_prime_sc_bk;
+	u8 cur_80_prime_sc;
+	u64 tsf;
+	u8 retry_short;
+	u8 retry_long;
+	u16 assoc_id;
+	bool hiddenssid;
+
+	/*IBSS*/
+	int beacon_interval;
+
+	/*AMPDU*/
+	u8 min_space_cfg;	/*For Min spacing configurations */
+	u8 max_mss_density;
+	u8 current_ampdu_factor;
+	u8 current_ampdu_density;
+
+	/*QOS & EDCA */
+	struct ieee80211_tx_queue_params edca_param[RTL_MAC80211_NUM_QUEUE];
+	struct rtl_qos_parameters ac[AC_MAX];
+
+	/* counters */
+	u64 last_txok_cnt;
+	u64 last_rxok_cnt;
+	u32 last_bt_edca_ul;
+	u32 last_bt_edca_dl;
+};
+
+struct btdm_8723 {
+	bool all_off;
+	bool agc_table_en;
+	bool adc_back_off_on;
+	bool b2_ant_hid_en;
+	bool low_penalty_rate_adaptive;
+	bool rf_rx_lpf_shrink;
+	bool reject_aggre_pkt;
+	bool tra_tdma_on;
+	u8 tra_tdma_nav;
+	u8 tra_tdma_ant;
+	bool tdma_on;
+	u8 tdma_ant;
+	u8 tdma_nav;
+	u8 tdma_dac_swing;
+	u8 fw_dac_swing_lvl;
+	bool ps_tdma_on;
+	u8 ps_tdma_byte[5];
+	bool pta_on;
+	u32 val_0x6c0;
+	u32 val_0x6c8;
+	u32 val_0x6cc;
+	bool sw_dac_swing_on;
+	u32 sw_dac_swing_lvl;
+	u32 wlan_act_hi;
+	u32 wlan_act_lo;
+	u32 bt_retry_index;
+	bool dec_bt_pwr;
+	bool ignore_wlan_act;
+};
+
+struct bt_coexist_8723 {
+	u32 high_priority_tx;
+	u32 high_priority_rx;
+	u32 low_priority_tx;
+	u32 low_priority_rx;
+	u8 c2h_bt_info;
+	bool c2h_bt_info_req_sent;
+	bool c2h_bt_inquiry_page;
+	u32 bt_inq_page_start_time;
+	u8 bt_retry_cnt;
+	u8 c2h_bt_info_original;
+	u8 bt_inquiry_page_cnt;
+	struct btdm_8723 btdm;
+};
+
+struct rtl_hal {
+	struct ieee80211_hw *hw;
+	bool driver_is_goingto_unload;
+	bool up_first_time;
+	bool first_init;
+	bool being_init_adapter;
+	bool bbrf_ready;
+	bool mac_func_enable;
+	bool pre_edcca_enable;
+	struct bt_coexist_8723 hal_coex_8723;
+
+	enum intf_type interface;
+	u16 hw_type;		/*92c or 92d or 92s and so on */
+	u8 ic_class;
+	u8 oem_id;
+	u32 version;		/*version of chip */
+	u8 state;		/*stop 0, start 1 */
+	u8 board_type;
+	u8 package_type;
+	u8 external_pa;
+
+	u8 pa_mode;
+	u8 pa_type_2g;
+	u8 pa_type_5g;
+	u8 lna_type_2g;
+	u8 lna_type_5g;
+	u8 external_pa_2g;
+	u8 external_lna_2g;
+	u8 external_pa_5g;
+	u8 external_lna_5g;
+	u8 type_glna;
+	u8 type_gpa;
+	u8 type_alna;
+	u8 type_apa;
+	u8 rfe_type;
+
+	/*firmware */
+	u32 fwsize;
+	u8 *pfirmware;
+	u16 fw_version;
+	u16 fw_subversion;
+	bool h2c_setinprogress;
+	u8 last_hmeboxnum;
+	bool fw_ready;
+	/*Reserve page start offset except beacon in TxQ. */
+	u8 fw_rsvdpage_startoffset;
+	u8 h2c_txcmd_seq;
+	u8 current_ra_rate;
+
+	/* FW Cmd IO related */
+	u16 fwcmd_iomap;
+	u32 fwcmd_ioparam;
+	bool set_fwcmd_inprogress;
+	u8 current_fwcmd_io;
+
+	struct p2p_ps_offload_t p2p_ps_offload;
+	bool fw_clk_change_in_progress;
+	bool allow_sw_to_change_hwclc;
+	u8 fw_ps_state;
+	/**/
+	bool driver_going2unload;
+
+	/*AMPDU init min space*/
+	u8 minspace_cfg;	/*For Min spacing configurations */
+
+	/* Dual mac */
+	enum macphy_mode macphymode;
+	enum band_type current_bandtype;	/* 0:2.4G, 1:5G */
+	enum band_type current_bandtypebackup;
+	enum band_type bandset;
+	/* dual MAC 0--Mac0 1--Mac1 */
+	u32 interfaceindex;
+	/* just for DualMac S3S4 */
+	u8 macphyctl_reg;
+	bool earlymode_enable;
+	u8 max_earlymode_num;
+	/* Dual mac*/
+	bool during_mac0init_radiob;
+	bool during_mac1init_radioa;
+	bool reloadtxpowerindex;
+	/* True if IMR or IQK  have done
+	 * for 2.4G in scan progress
+	 */
+	bool load_imrandiqk_setting_for2g;
+
+	bool disable_amsdu_8k;
+	bool master_of_dmsp;
+	bool slave_of_dmsp;
+
+	u16 rx_tag;/*for 92ee*/
+	u8 rts_en;
+
+	/*for wowlan*/
+	bool wow_enable;
+	bool enter_pnp_sleep;
+	bool wake_from_pnp_sleep;
+	bool wow_enabled;
+	__kernel_time_t last_suspend_sec;
+	u32 wowlan_fwsize;
+	u8 *wowlan_firmware;
+
+	u8 hw_rof_enable; /*Enable GPIO[9] as WL RF HW PDn source*/
+
+	bool real_wow_v2_enable;
+	bool re_init_llt_table;
+};
+
+struct rtl_security {
+	/*default 0 */
+	bool use_sw_sec;
+
+	bool being_setkey;
+	bool use_defaultkey;
+	/*Encryption Algorithm for Unicast Packet */
+	enum rt_enc_alg pairwise_enc_algorithm;
+	/*Encryption Algorithm for Brocast/Multicast */
+	enum rt_enc_alg group_enc_algorithm;
+	/*Cam Entry Bitmap */
+	u32 hwsec_cam_bitmap;
+	u8 hwsec_cam_sta_addr[TOTAL_CAM_ENTRY][ETH_ALEN];
+	/* local Key buffer, indx 0 is for
+	 * pairwise key 1-4 is for agoup key.
+	 */
+	u8 key_buf[KEY_BUF_SIZE][MAX_KEY_LEN];
+	u8 key_len[KEY_BUF_SIZE];
+
+	/* The pointer of Pairwise Key,
+	 * it always points to KeyBuf[4]
+	 */
+	u8 *pairwise_key;
+};
+
+#define ASSOCIATE_ENTRY_NUM	33
+
+struct fast_ant_training {
+	u8	bssid[6];
+	u8	antsel_rx_keep_0;
+	u8	antsel_rx_keep_1;
+	u8	antsel_rx_keep_2;
+	u32	ant_sum[7];
+	u32	ant_cnt[7];
+	u32	ant_ave[7];
+	u8	fat_state;
+	u32	train_idx;
+	u8	antsel_a[ASSOCIATE_ENTRY_NUM];
+	u8	antsel_b[ASSOCIATE_ENTRY_NUM];
+	u8	antsel_c[ASSOCIATE_ENTRY_NUM];
+	u32	main_ant_sum[ASSOCIATE_ENTRY_NUM];
+	u32	aux_ant_sum[ASSOCIATE_ENTRY_NUM];
+	u32	main_ant_cnt[ASSOCIATE_ENTRY_NUM];
+	u32	aux_ant_cnt[ASSOCIATE_ENTRY_NUM];
+	u8	rx_idle_ant;
+	bool	becomelinked;
+};
+
+struct dm_phy_dbg_info {
+	s8 rx_snrdb[4];
+	u64 num_qry_phy_status;
+	u64 num_qry_phy_status_cck;
+	u64 num_qry_phy_status_ofdm;
+	u16 num_qry_beacon_pkt;
+	u16 num_non_be_pkt;
+	s32 rx_evm[4];
+};
+
+struct rtl_dm {
+	/*PHY status for Dynamic Management */
+	long entry_min_undec_sm_pwdb;
+	long undec_sm_cck;
+	long undec_sm_pwdb;	/*out dm */
+	long entry_max_undec_sm_pwdb;
+	s32 ofdm_pkt_cnt;
+	bool dm_initialgain_enable;
+	bool dynamic_txpower_enable;
+	bool current_turbo_edca;
+	bool is_any_nonbepkts;	/*out dm */
+	bool is_cur_rdlstate;
+	bool txpower_trackinginit;
+	bool disable_framebursting;
+	bool cck_inch14;
+	bool txpower_tracking;
+	bool useramask;
+	bool rfpath_rxenable[4];
+	bool inform_fw_driverctrldm;
+	bool current_mrc_switch;
+	u8 txpowercount;
+	u8 powerindex_backup[6];
+
+	u8 thermalvalue_rxgain;
+	u8 thermalvalue_iqk;
+	u8 thermalvalue_lck;
+	u8 thermalvalue;
+	u8 last_dtp_lvl;
+	u8 thermalvalue_avg[AVG_THERMAL_NUM];
+	u8 thermalvalue_avg_index;
+	u8 tm_trigger;
+	bool done_txpower;
+	u8 dynamic_txhighpower_lvl;	/*Tx high power level */
+	u8 dm_flag;		/*Indicate each dynamic mechanism's status. */
+	u8 dm_flag_tmp;
+	u8 dm_type;
+	u8 dm_rssi_sel;
+	u8 txpower_track_control;
+	bool interrupt_migration;
+	bool disable_tx_int;
+	s8 ofdm_index[MAX_RF_PATH];
+	u8 default_ofdm_index;
+	u8 default_cck_index;
+	s8 cck_index;
+	s8 delta_power_index[MAX_RF_PATH];
+	s8 delta_power_index_last[MAX_RF_PATH];
+	s8 power_index_offset[MAX_RF_PATH];
+	s8 absolute_ofdm_swing_idx[MAX_RF_PATH];
+	s8 remnant_ofdm_swing_idx[MAX_RF_PATH];
+	s8 remnant_cck_idx;
+	bool modify_txagc_flag_path_a;
+	bool modify_txagc_flag_path_b;
+
+	bool one_entry_only;
+	struct dm_phy_dbg_info dbginfo;
+
+	/* Dynamic ATC switch */
+	bool atc_status;
+	bool large_cfo_hit;
+	bool is_freeze;
+	int cfo_tail[2];
+	int cfo_ave_pre;
+	int crystal_cap;
+	u8 cfo_threshold;
+	u32 packet_count;
+	u32 packet_count_pre;
+	u8 tx_rate;
+
+	/*88e tx power tracking*/
+	u8	swing_idx_ofdm[MAX_RF_PATH];
+	u8	swing_idx_ofdm_cur;
+	u8	swing_idx_ofdm_base[MAX_RF_PATH];
+	bool	swing_flag_ofdm;
+	u8	swing_idx_cck;
+	u8	swing_idx_cck_cur;
+	u8	swing_idx_cck_base;
+	bool	swing_flag_cck;
+
+	s8	swing_diff_2g;
+	s8	swing_diff_5g;
+
+	/* DMSP */
+	bool supp_phymode_switch;
+
+	/* DulMac */
+	struct fast_ant_training fat_table;
+
+	u8	resp_tx_path;
+	u8	path_sel;
+	u32	patha_sum;
+	u32	pathb_sum;
+	u32	patha_cnt;
+	u32	pathb_cnt;
+
+	u8 pre_channel;
+	u8 *p_channel;
+	u8 linked_interval;
+
+	u64 last_tx_ok_cnt;
+	u64 last_rx_ok_cnt;
+};
+
+#define	EFUSE_MAX_LOGICAL_SIZE			512
+
+struct rtl_efuse {
+	bool autoload_ok;
+	bool bootfromefuse;
+	u16 max_physical_size;
+
+	u8 efuse_map[2][EFUSE_MAX_LOGICAL_SIZE];
+	u16 efuse_usedbytes;
+	u8 efuse_usedpercentage;
+#ifdef EFUSE_REPG_WORKAROUND
+	bool efuse_re_pg_sec1flag;
+	u8 efuse_re_pg_data[8];
+#endif
+
+	u8 autoload_failflag;
+	u8 autoload_status;
+
+	short epromtype;
+	u16 eeprom_vid;
+	u16 eeprom_did;
+	u16 eeprom_svid;
+	u16 eeprom_smid;
+	u8 eeprom_oemid;
+	u16 eeprom_channelplan;
+	u8 eeprom_version;
+	u8 board_type;
+	u8 external_pa;
+
+	u8 dev_addr[6];
+	u8 wowlan_enable;
+	u8 antenna_div_cfg;
+	u8 antenna_div_type;
+
+	bool txpwr_fromeprom;
+	u8 eeprom_crystalcap;
+	u8 eeprom_tssi[2];
+	u8 eeprom_tssi_5g[3][2]; /* for 5GL/5GM/5GH band. */
+	u8 eeprom_pwrlimit_ht20[CHANNEL_GROUP_MAX];
+	u8 eeprom_pwrlimit_ht40[CHANNEL_GROUP_MAX];
+	u8 eeprom_chnlarea_txpwr_cck[MAX_RF_PATH][CHANNEL_GROUP_MAX_2G];
+	u8 eeprom_chnlarea_txpwr_ht40_1s[MAX_RF_PATH][CHANNEL_GROUP_MAX];
+	u8 eprom_chnl_txpwr_ht40_2sdf[MAX_RF_PATH][CHANNEL_GROUP_MAX];
+
+	u8 internal_pa_5g[2];	/* pathA / pathB */
+	u8 eeprom_c9;
+	u8 eeprom_cc;
+
+	/*For power group */
+	u8 eeprom_pwrgroup[2][3];
+	u8 pwrgroup_ht20[2][CHANNEL_MAX_NUMBER];
+	u8 pwrgroup_ht40[2][CHANNEL_MAX_NUMBER];
+
+	u8 txpwrlevel_cck[MAX_RF_PATH][CHANNEL_MAX_NUMBER_2G];
+	/*For HT 40MHZ pwr */
+	u8 txpwrlevel_ht40_1s[MAX_RF_PATH][CHANNEL_MAX_NUMBER];
+	/*For HT 40MHZ pwr */
+	u8 txpwrlevel_ht40_2s[MAX_RF_PATH][CHANNEL_MAX_NUMBER];
+
+	/*--------------------------------------------------------*
+	 * 8192CE\8192SE\8192DE\8723AE use the following 4 arrays,
+	 * other ICs (8188EE\8723BE\8192EE\8812AE...)
+	 * define new arrays in Windows code.
+	 * BUT, in linux code, we use the same array for all ICs.
+	 *
+	 * The Correspondance relation between two arrays is:
+	 * txpwr_cckdiff[][] == CCK_24G_Diff[][]
+	 * txpwr_ht20diff[][] == BW20_24G_Diff[][]
+	 * txpwr_ht40diff[][] == BW40_24G_Diff[][]
+	 * txpwr_legacyhtdiff[][] == OFDM_24G_Diff[][]
+	 *
+	 * Sizes of these arrays are decided by the larger ones.
+	 */
+	s8 txpwr_cckdiff[MAX_RF_PATH][CHANNEL_MAX_NUMBER];
+	s8 txpwr_ht20diff[MAX_RF_PATH][CHANNEL_MAX_NUMBER];
+	s8 txpwr_ht40diff[MAX_RF_PATH][CHANNEL_MAX_NUMBER];
+	s8 txpwr_legacyhtdiff[MAX_RF_PATH][CHANNEL_MAX_NUMBER];
+
+	u8 txpwr_5g_bw40base[MAX_RF_PATH][CHANNEL_MAX_NUMBER];
+	u8 txpwr_5g_bw80base[MAX_RF_PATH][CHANNEL_MAX_NUMBER_5G_80M];
+	s8 txpwr_5g_ofdmdiff[MAX_RF_PATH][MAX_TX_COUNT];
+	s8 txpwr_5g_bw20diff[MAX_RF_PATH][MAX_TX_COUNT];
+	s8 txpwr_5g_bw40diff[MAX_RF_PATH][MAX_TX_COUNT];
+	s8 txpwr_5g_bw80diff[MAX_RF_PATH][MAX_TX_COUNT];
+
+	u8 txpwr_safetyflag;			/* Band edge enable flag */
+	u16 eeprom_txpowerdiff;
+	u8 legacy_httxpowerdiff;	/* Legacy to HT rate power diff */
+	u8 antenna_txpwdiff[3];
+
+	u8 eeprom_regulatory;
+	u8 eeprom_thermalmeter;
+	u8 thermalmeter[2]; /*ThermalMeter, index 0 for RFIC0, 1 for RFIC1 */
+	u16 tssi_13dbm;
+	u8 crystalcap;		/* CrystalCap. */
+	u8 delta_iqk;
+	u8 delta_lck;
+
+	u8 legacy_ht_txpowerdiff;	/*Legacy to HT rate power diff */
+	bool apk_thermalmeterignore;
+
+	bool b1x1_recvcombine;
+	bool b1ss_support;
+
+	/*channel plan */
+	u8 channel_plan;
+};
+
+struct rtl_tx_report {
+	atomic_t sn;
+	u16 last_sent_sn;
+	unsigned long last_sent_time;
+	u16 last_recv_sn;
+};
+
+struct rtl_ps_ctl {
+	bool pwrdomain_protect;
+	bool in_powersavemode;
+	bool rfchange_inprogress;
+	bool swrf_processing;
+	bool hwradiooff;
+	/* just for PCIE ASPM
+	 * If it supports ASPM, Offset[560h] = 0x40,
+	 * otherwise Offset[560h] = 0x00.
+	 */
+	bool support_aspm;
+	bool support_backdoor;
+
+	/*for LPS */
+	enum rt_psmode dot11_psmode;	/*Power save mode configured. */
+	bool swctrl_lps;
+	bool leisure_ps;
+	bool fwctrl_lps;
+	u8 fwctrl_psmode;
+	/*For Fw control LPS mode */
+	u8 reg_fwctrl_lps;
+	/*Record Fw PS mode status. */
+	bool fw_current_inpsmode;
+	u8 reg_max_lps_awakeintvl;
+	bool report_linked;
+	bool low_power_enable;/*for 32k*/
+
+	/*for IPS */
+	bool inactiveps;
+
+	u32 rfoff_reason;
+
+	/*RF OFF Level */
+	u32 cur_ps_level;
+	u32 reg_rfps_level;
+
+	/*just for PCIE ASPM */
+	u8 const_amdpci_aspm;
+	bool pwrdown_mode;
+
+	enum rf_pwrstate inactive_pwrstate;
+	enum rf_pwrstate rfpwr_state;	/*cur power state */
+
+	/* for SW LPS*/
+	bool sw_ps_enabled;
+	bool state;
+	bool state_inap;
+	bool multi_buffered;
+	u16 nullfunc_seq;
+	unsigned int dtim_counter;
+	unsigned int sleep_ms;
+	unsigned long last_sleep_jiffies;
+	unsigned long last_awake_jiffies;
+	unsigned long last_delaylps_stamp_jiffies;
+	unsigned long last_dtim;
+	unsigned long last_beacon;
+	unsigned long last_action;
+	unsigned long last_slept;
+
+	/*For P2P PS */
+	struct rtl_p2p_ps_info p2p_ps_info;
+	u8 pwr_mode;
+	u8 smart_ps;
+
+	/* wake up on line */
+	u8 wo_wlan_mode;
+	u8 arp_offload_enable;
+	u8 gtk_offload_enable;
+	/* Used for WOL, indicates the reason for waking event.*/
+	u32 wakeup_reason;
+	/* Record the last waking time for comparison with setting key. */
+	u64 last_wakeup_time;
+};
+
+struct rtl_stats {
+	u8 psaddr[ETH_ALEN];
+	u32 mac_time[2];
+	s8 rssi;
+	u8 signal;
+	u8 noise;
+	u8 rate;		/* hw desc rate */
+	u8 received_channel;
+	u8 control;
+	u8 mask;
+	u8 freq;
+	u16 len;
+	u64 tsf;
+	u32 beacon_time;
+	u8 nic_type;
+	u16 length;
+	u8 signalquality;	/*in 0-100 index. */
+	/*
+	 * Real power in dBm for this packet,
+	 * no beautification and aggregation.
+	 */
+	s32 recvsignalpower;
+	s8 rxpower;		/*in dBm Translate from PWdB */
+	u8 signalstrength;	/*in 0-100 index. */
+	u16 hwerror:1;
+	u16 crc:1;
+	u16 icv:1;
+	u16 shortpreamble:1;
+	u16 antenna:1;
+	u16 decrypted:1;
+	u16 wakeup:1;
+	u32 timestamp_low;
+	u32 timestamp_high;
+	bool shift;
+
+	u8 rx_drvinfo_size;
+	u8 rx_bufshift;
+	bool isampdu;
+	bool isfirst_ampdu;
+	bool rx_is40mhzpacket;
+	u8 rx_packet_bw;
+	u32 rx_pwdb_all;
+	u8 rx_mimo_signalstrength[4];	/*in 0~100 index */
+	s8 rx_mimo_signalquality[4];
+	u8 rx_mimo_evm_dbm[4];
+	u16 cfo_short[4];		/* per-path's Cfo_short */
+	u16 cfo_tail[4];
+
+	s8 rx_mimo_sig_qual[4];
+	u8 rx_pwr[4]; /* per-path's pwdb */
+	u8 rx_snr[4]; /* per-path's SNR */
+	u8 bandwidth;
+	u8 bt_coex_pwr_adjust;
+	bool packet_matchbssid;
+	bool is_cck;
+	bool is_ht;
+	bool packet_toself;
+	bool packet_beacon;	/*for rssi */
+	s8 cck_adc_pwdb[4];	/*for rx path selection */
+
+	bool is_vht;
+	bool is_short_gi;
+	u8 vht_nss;
+
+	u8 packet_report_type;
+
+	u32 macid;
+	u8 wake_match;
+	u32 bt_rx_rssi_percentage;
+	u32 macid_valid_entry[2];
+};
+
+struct rt_link_detect {
+	/* count for roaming */
+	u32 bcn_rx_inperiod;
+	u32 roam_times;
+
+	u32 num_tx_in4period[4];
+	u32 num_rx_in4period[4];
+
+	u32 num_tx_inperiod;
+	u32 num_rx_inperiod;
+
+	bool busytraffic;
+	bool tx_busy_traffic;
+	bool rx_busy_traffic;
+	bool higher_busytraffic;
+	bool higher_busyrxtraffic;
+
+	u32 tidtx_in4period[MAX_TID_COUNT][4];
+	u32 tidtx_inperiod[MAX_TID_COUNT];
+	bool higher_busytxtraffic[MAX_TID_COUNT];
+};
+
+struct rtl_tcb_desc {
+	u8 packet_bw:2;
+	u8 multicast:1;
+	u8 broadcast:1;
+
+	u8 rts_stbc:1;
+	u8 rts_enable:1;
+	u8 cts_enable:1;
+	u8 rts_use_shortpreamble:1;
+	u8 rts_use_shortgi:1;
+	u8 rts_sc:1;
+	u8 rts_bw:1;
+	u8 rts_rate;
+
+	u8 use_shortgi:1;
+	u8 use_shortpreamble:1;
+	u8 use_driver_rate:1;
+	u8 disable_ratefallback:1;
+
+	u8 use_spe_rpt:1;
+
+	u8 ratr_index;
+	u8 mac_id;
+	u8 hw_rate;
+
+	u8 last_inipkt:1;
+	u8 cmd_or_init:1;
+	u8 queue_index;
+
+	/* early mode */
+	u8 empkt_num;
+	/* The max value by HW */
+	u32 empkt_len[10];
+	bool tx_enable_sw_calc_duration;
+};
+
+struct rtl_wow_pattern {
+	u8 type;
+	u16 crc;
+	u32 mask[4];
+};
+
+struct rtl_hal_ops {
+	int (*init_sw_vars)(struct ieee80211_hw *hw);
+	void (*deinit_sw_vars)(struct ieee80211_hw *hw);
+	void (*read_chip_version)(struct ieee80211_hw *hw);
+	void (*read_eeprom_info)(struct ieee80211_hw *hw);
+	void (*interrupt_recognized)(struct ieee80211_hw *hw,
+				     u32 *p_inta, u32 *p_intb,
+				     u32 *p_intc, u32 *p_intd);
+	int (*hw_init)(struct ieee80211_hw *hw);
+	void (*hw_disable)(struct ieee80211_hw *hw);
+	void (*hw_suspend)(struct ieee80211_hw *hw);
+	void (*hw_resume)(struct ieee80211_hw *hw);
+	void (*enable_interrupt)(struct ieee80211_hw *hw);
+	void (*disable_interrupt)(struct ieee80211_hw *hw);
+	int (*set_network_type)(struct ieee80211_hw *hw,
+				enum nl80211_iftype type);
+	void (*set_chk_bssid)(struct ieee80211_hw *hw,
+			      bool check_bssid);
+	void (*set_bw_mode)(struct ieee80211_hw *hw,
+			    enum nl80211_channel_type ch_type);
+	 u8 (*switch_channel)(struct ieee80211_hw *hw);
+	void (*set_qos)(struct ieee80211_hw *hw, int aci);
+	void (*set_bcn_reg)(struct ieee80211_hw *hw);
+	void (*set_bcn_intv)(struct ieee80211_hw *hw);
+	void (*update_interrupt_mask)(struct ieee80211_hw *hw,
+				      u32 add_msr, u32 rm_msr);
+	void (*get_hw_reg)(struct ieee80211_hw *hw, u8 variable, u8 *val);
+	void (*set_hw_reg)(struct ieee80211_hw *hw, u8 variable, u8 *val);
+	void (*update_rate_tbl)(struct ieee80211_hw *hw,
+				struct ieee80211_sta *sta, u8 rssi_leve,
+				bool update_bw);
+	void (*pre_fill_tx_bd_desc)(struct ieee80211_hw *hw, u8 *tx_bd_desc,
+				    u8 *desc, u8 queue_index,
+				    struct sk_buff *skb, dma_addr_t addr);
+	void (*update_rate_mask)(struct ieee80211_hw *hw, u8 rssi_level);
+	u16 (*rx_desc_buff_remained_cnt)(struct ieee80211_hw *hw,
+					 u8 queue_index);
+	void (*rx_check_dma_ok)(struct ieee80211_hw *hw, u8 *header_desc,
+				u8 queue_index);
+	void (*fill_tx_desc)(struct ieee80211_hw *hw,
+			     struct ieee80211_hdr *hdr, u8 *pdesc_tx,
+			     u8 *pbd_desc_tx,
+			     struct ieee80211_tx_info *info,
+			     struct ieee80211_sta *sta,
+			     struct sk_buff *skb, u8 hw_queue,
+			     struct rtl_tcb_desc *ptcb_desc);
+	void (*fill_fake_txdesc)(struct ieee80211_hw *hw, u8 *pdesc,
+				 u32 buffer_len, bool bispspoll);
+	void (*fill_tx_cmddesc)(struct ieee80211_hw *hw, u8 *pdesc,
+				bool firstseg, bool lastseg,
+				struct sk_buff *skb);
+	void (*fill_tx_special_desc)(struct ieee80211_hw *hw,
+				     u8 *pdesc, u8 *pbd_desc,
+				     struct sk_buff *skb, u8 hw_queue);
+	bool (*query_rx_desc)(struct ieee80211_hw *hw,
+			      struct rtl_stats *stats,
+			      struct ieee80211_rx_status *rx_status,
+			      u8 *pdesc, struct sk_buff *skb);
+	void (*set_channel_access)(struct ieee80211_hw *hw);
+	bool (*radio_onoff_checking)(struct ieee80211_hw *hw, u8 *valid);
+	void (*dm_watchdog)(struct ieee80211_hw *hw);
+	void (*scan_operation_backup)(struct ieee80211_hw *hw, u8 operation);
+	bool (*set_rf_power_state)(struct ieee80211_hw *hw,
+				   enum rf_pwrstate rfpwr_state);
+	void (*led_control)(struct ieee80211_hw *hw,
+			    enum led_ctl_mode ledaction);
+	void (*set_desc)(struct ieee80211_hw *hw, u8 *pdesc, bool istx,
+			 u8 desc_name, u8 *val);
+	u64 (*get_desc)(struct ieee80211_hw *hw, u8 *pdesc, bool istx,
+			u8 desc_name);
+	bool (*is_tx_desc_closed)(struct ieee80211_hw *hw,
+				  u8 hw_queue, u16 index);
+	void (*tx_polling)(struct ieee80211_hw *hw, u8 hw_queue);
+	void (*enable_hw_sec)(struct ieee80211_hw *hw);
+	void (*set_key)(struct ieee80211_hw *hw, u32 key_index,
+			u8 *macaddr, bool is_group, u8 enc_algo,
+			bool is_wepkey, bool clear_all);
+	void (*init_sw_leds)(struct ieee80211_hw *hw);
+	void (*deinit_sw_leds)(struct ieee80211_hw *hw);
+	u32 (*get_bbreg)(struct ieee80211_hw *hw, u32 regaddr, u32 bitmask);
+	void (*set_bbreg)(struct ieee80211_hw *hw, u32 regaddr, u32 bitmask,
+			  u32 data);
+	u32 (*get_rfreg)(struct ieee80211_hw *hw, enum radio_path rfpath,
+			 u32 regaddr, u32 bitmask);
+	void (*set_rfreg)(struct ieee80211_hw *hw, enum radio_path rfpath,
+			  u32 regaddr, u32 bitmask, u32 data);
+	void (*linked_set_reg)(struct ieee80211_hw *hw);
+	void (*chk_switch_dmdp)(struct ieee80211_hw *hw);
+	void (*dualmac_easy_concurrent)(struct ieee80211_hw *hw);
+	void (*dualmac_switch_to_dmdp)(struct ieee80211_hw *hw);
+	bool (*phy_rf6052_config)(struct ieee80211_hw *hw);
+	void (*phy_rf6052_set_cck_txpower)(struct ieee80211_hw *hw,
+					   u8 *powerlevel);
+	void (*phy_rf6052_set_ofdm_txpower)(struct ieee80211_hw *hw,
+					    u8 *ppowerlevel, u8 channel);
+	bool (*config_bb_with_headerfile)(struct ieee80211_hw *hw,
+					  u8 configtype);
+	bool (*config_bb_with_pgheaderfile)(struct ieee80211_hw *hw,
+					    u8 configtype);
+	void (*phy_lc_calibrate)(struct ieee80211_hw *hw, bool is2t);
+	void (*phy_set_bw_mode_callback)(struct ieee80211_hw *hw);
+	void (*dm_dynamic_txpower)(struct ieee80211_hw *hw);
+	void (*c2h_command_handle)(struct ieee80211_hw *hw);
+	void (*bt_wifi_media_status_notify)(struct ieee80211_hw *hw,
+					    bool mstate);
+	void (*bt_coex_off_before_lps)(struct ieee80211_hw *hw);
+	void (*fill_h2c_cmd)(struct ieee80211_hw *hw, u8 element_id,
+			     u32 cmd_len, u8 *p_cmdbuffer);
+	void (*set_default_port_id_cmd)(struct ieee80211_hw *hw);
+	bool (*get_btc_status)(void);
+	bool (*is_fw_header)(struct rtlwifi_firmware_header *hdr);
+	u32 (*rx_command_packet)(struct ieee80211_hw *hw,
+				 const struct rtl_stats *status,
+				 struct sk_buff *skb);
+	void (*add_wowlan_pattern)(struct ieee80211_hw *hw,
+				   struct rtl_wow_pattern *rtl_pattern,
+				   u8 index);
+	u16 (*get_available_desc)(struct ieee80211_hw *hw, u8 q_idx);
+	void (*c2h_content_parsing)(struct ieee80211_hw *hw, u8 tag, u8 len,
+				    u8 *val);
+	/* ops for halmac cb */
+	bool (*halmac_cb_init_mac_register)(struct rtl_priv *rtlpriv);
+	bool (*halmac_cb_init_bb_rf_register)(struct rtl_priv *rtlpriv);
+	bool (*halmac_cb_write_data_rsvd_page)(struct rtl_priv *rtlpriv,
+					       u8 *buf, u32 size);
+	bool (*halmac_cb_write_data_h2c)(struct rtl_priv *rtlpriv, u8 *buf,
+					 u32 size);
+	/* ops for phydm cb */
+	u8 (*get_txpower_index)(struct ieee80211_hw *hw, u8 path,
+				u8 rate, u8 bandwidth, u8 channel);
+	void (*set_tx_power_index_by_rs)(struct ieee80211_hw *hw,
+					 u8 channel, u8 path,
+					 enum rate_section rs);
+	void (*store_tx_power_by_rate)(struct ieee80211_hw *hw,
+				       u32 band, u32 rfpath,
+				       u32 txnum, u32 regaddr,
+				       u32 bitmask, u32 data);
+	void (*phy_set_txpower_limit)(struct ieee80211_hw *hw, u8 *pregulation,
+				      u8 *pband, u8 *pbandwidth,
+				      u8 *prate_section, u8 *prf_path,
+				      u8 *pchannel, u8 *ppower_limit);
+};
+
+struct rtl_intf_ops {
+	/*com */
+	void (*read_efuse_byte)(struct ieee80211_hw *hw, u16 _offset, u8 *pbuf);
+	int (*adapter_start)(struct ieee80211_hw *hw);
+	void (*adapter_stop)(struct ieee80211_hw *hw);
+	bool (*check_buddy_priv)(struct ieee80211_hw *hw,
+				 struct rtl_priv **buddy_priv);
+
+	int (*adapter_tx)(struct ieee80211_hw *hw,
+			  struct ieee80211_sta *sta,
+			  struct sk_buff *skb,
+			  struct rtl_tcb_desc *ptcb_desc);
+	void (*flush)(struct ieee80211_hw *hw, u32 queues, bool drop);
+	int (*reset_trx_ring)(struct ieee80211_hw *hw);
+	bool (*waitq_insert)(struct ieee80211_hw *hw,
+			     struct ieee80211_sta *sta,
+			     struct sk_buff *skb);
+
+	/*pci */
+	void (*disable_aspm)(struct ieee80211_hw *hw);
+	void (*enable_aspm)(struct ieee80211_hw *hw);
+
+	/*usb */
+};
+
+struct rtl_mod_params {
+	/* default: 0,0 */
+	u64 debug_mask;
+	/* default: 0 = using hardware encryption */
+	bool sw_crypto;
+
+	/* default: 0 = DBG_EMERG (0)*/
+	int debug_level;
+
+	/* default: 1 = using no linked power save */
+	bool inactiveps;
+
+	/* default: 1 = using linked sw power save */
+	bool swctrl_lps;
+
+	/* default: 1 = using linked fw power save */
+	bool fwctrl_lps;
+
+	/* default: 0 = not using MSI interrupts mode
+	 * submodules should set their own default value
+	 */
+	bool msi_support;
+
+	/* default: 0 = dma 32 */
+	bool dma64;
+
+	/* default: 1 = enable aspm */
+	int aspm_support;
+
+	/* default 0: 1 means disable */
+	bool disable_watchdog;
+
+	/* default 0: 1 means do not disable interrupts */
+	bool int_clear;
+
+	/* select antenna */
+	int ant_sel;
+};
+
+struct rtl_hal_usbint_cfg {
+	/* data - rx */
+	u32 in_ep_num;
+	u32 rx_urb_num;
+	u32 rx_max_size;
+
+	/* op - rx */
+	void (*usb_rx_hdl)(struct ieee80211_hw *, struct sk_buff *);
+	void (*usb_rx_segregate_hdl)(struct ieee80211_hw *, struct sk_buff *,
+				     struct sk_buff_head *);
+
+	/* tx */
+	void (*usb_tx_cleanup)(struct ieee80211_hw *, struct sk_buff *);
+	int (*usb_tx_post_hdl)(struct ieee80211_hw *, struct urb *,
+			       struct sk_buff *);
+	struct sk_buff *(*usb_tx_aggregate_hdl)(struct ieee80211_hw *,
+						struct sk_buff_head *);
+
+	/* endpoint mapping */
+	int (*usb_endpoint_mapping)(struct ieee80211_hw *hw);
+	u16 (*usb_mq_to_hwq)(__le16 fc, u16 mac80211_queue_index);
+};
+
+struct rtl_hal_cfg {
+	u8 bar_id;
+	bool write_readback;
+	char *name;
+	char *alt_fw_name;
+	struct rtl_hal_ops *ops;
+	struct rtl_mod_params *mod_params;
+	struct rtl_hal_usbint_cfg *usb_interface_cfg;
+	enum rtl_spec_ver spec_ver;
+
+	/* this map used for some registers or vars
+	 * defined int HAL but used in MAIN
+	 */
+	u32 maps[RTL_VAR_MAP_MAX];
+
+};
+
+struct rtl_locks {
+	/* mutex */
+	struct mutex conf_mutex;
+	struct mutex ips_mutex;	/* mutex for enter/leave IPS */
+	struct mutex lps_mutex;	/* mutex for enter/leave LPS */
+
+	/*spin lock */
+	spinlock_t irq_th_lock;
+	spinlock_t h2c_lock;
+	spinlock_t rf_ps_lock;
+	spinlock_t rf_lock;
+	spinlock_t waitq_lock;
+	spinlock_t entry_list_lock;
+	spinlock_t usb_lock;
+	spinlock_t c2hcmd_lock;
+	spinlock_t scan_list_lock; /* lock for the scan list */
+
+	/*FW clock change */
+	spinlock_t fw_ps_lock;
+
+	/*Dual mac*/
+	spinlock_t cck_and_rw_pagea_lock;
+
+	spinlock_t iqk_lock;
+};
+
+struct rtl_works {
+	struct ieee80211_hw *hw;
+
+	/*timer */
+	struct timer_list watchdog_timer;
+	struct timer_list dualmac_easyconcurrent_retrytimer;
+	struct timer_list fw_clockoff_timer;
+	struct timer_list fast_antenna_training_timer;
+	/*task */
+	struct tasklet_struct irq_tasklet;
+	struct tasklet_struct irq_prepare_bcn_tasklet;
+
+	/*work queue */
+	struct workqueue_struct *rtl_wq;
+	struct delayed_work watchdog_wq;
+	struct delayed_work ips_nic_off_wq;
+	struct delayed_work c2hcmd_wq;
+
+	/* For SW LPS */
+	struct delayed_work ps_work;
+	struct delayed_work ps_rfon_wq;
+	struct delayed_work fwevt_wq;
+
+	struct work_struct lps_change_work;
+	struct work_struct fill_h2c_cmd;
+};
+
+struct rtl_debug {
+	/* add for debug */
+	struct dentry *debugfs_dir;
+	char debugfs_name[20];
+
+	char *msg_buf;
+};
+
+#define MIMO_PS_STATIC			0
+#define MIMO_PS_DYNAMIC			1
+#define MIMO_PS_NOLIMIT			3
+
+struct rtl_dualmac_easy_concurrent_ctl {
+	enum band_type currentbandtype_backfordmdp;
+	bool close_bbandrf_for_dmsp;
+	bool change_to_dmdp;
+	bool change_to_dmsp;
+	bool switch_in_process;
+};
+
+struct rtl_dmsp_ctl {
+	bool activescan_for_slaveofdmsp;
+	bool scan_for_anothermac_fordmsp;
+	bool scan_for_itself_fordmsp;
+	bool writedig_for_anothermacofdmsp;
+	u32 curdigvalue_for_anothermacofdmsp;
+	bool changecckpdstate_for_anothermacofdmsp;
+	u8 curcckpdstate_for_anothermacofdmsp;
+	bool changetxhighpowerlvl_for_anothermacofdmsp;
+	u8 curtxhighlvl_for_anothermacofdmsp;
+	long rssivalmin_for_anothermacofdmsp;
+};
+
+struct ps_t {
+	u8 pre_ccastate;
+	u8 cur_ccasate;
+	u8 pre_rfstate;
+	u8 cur_rfstate;
+	u8 initialize;
+	long rssi_val_min;
+};
+
+struct dig_t {
+	u32 rssi_lowthresh;
+	u32 rssi_highthresh;
+	u32 fa_lowthresh;
+	u32 fa_highthresh;
+	long last_min_undec_pwdb_for_dm;
+	long rssi_highpower_lowthresh;
+	long rssi_highpower_highthresh;
+	u32 recover_cnt;
+	u32 pre_igvalue;
+	u32 cur_igvalue;
+	long rssi_val;
+	u8 dig_enable_flag;
+	u8 dig_ext_port_stage;
+	u8 dig_algorithm;
+	u8 dig_twoport_algorithm;
+	u8 dig_dbgmode;
+	u8 dig_slgorithm_switch;
+	u8 cursta_cstate;
+	u8 presta_cstate;
+	u8 curmultista_cstate;
+	u8 stop_dig;
+	s8 back_val;
+	s8 back_range_max;
+	s8 back_range_min;
+	u8 rx_gain_max;
+	u8 rx_gain_min;
+	u8 min_undec_pwdb_for_dm;
+	u8 rssi_val_min;
+	u8 pre_cck_cca_thres;
+	u8 cur_cck_cca_thres;
+	u8 pre_cck_pd_state;
+	u8 cur_cck_pd_state;
+	u8 pre_cck_fa_state;
+	u8 cur_cck_fa_state;
+	u8 pre_ccastate;
+	u8 cur_ccasate;
+	u8 large_fa_hit;
+	u8 forbidden_igi;
+	u8 dig_state;
+	u8 dig_highpwrstate;
+	u8 cur_sta_cstate;
+	u8 pre_sta_cstate;
+	u8 cur_ap_cstate;
+	u8 pre_ap_cstate;
+	u8 cur_pd_thstate;
+	u8 pre_pd_thstate;
+	u8 cur_cs_ratiostate;
+	u8 pre_cs_ratiostate;
+	u8 backoff_enable_flag;
+	s8 backoffval_range_max;
+	s8 backoffval_range_min;
+	u8 dig_min_0;
+	u8 dig_min_1;
+	u8 bt30_cur_igi;
+	bool media_connect_0;
+	bool media_connect_1;
+
+	u32 antdiv_rssi_max;
+	u32 rssi_max;
+};
+
+struct rtl_global_var {
+	/* from this list we can get
+	 * other adapter's rtl_priv
+	 */
+	struct list_head glb_priv_list;
+	spinlock_t glb_list_lock;
+};
+
+#define IN_4WAY_TIMEOUT_TIME	(30 * MSEC_PER_SEC)	/* 30 seconds */
+
+struct rtl_btc_info {
+	u8 bt_type;
+	u8 btcoexist;
+	u8 ant_num;
+	u8 single_ant_path;
+
+	u8 ap_num;
+	bool in_4way;
+	unsigned long in_4way_ts;
+};
+
+struct bt_coexist_info {
+	struct rtl_btc_ops *btc_ops;
+	struct rtl_btc_info btc_info;
+	/* btc context */
+	void *btc_context;
+	void *wifi_only_context;
+	/* EEPROM BT info. */
+	u8 eeprom_bt_coexist;
+	u8 eeprom_bt_type;
+	u8 eeprom_bt_ant_num;
+	u8 eeprom_bt_ant_isol;
+	u8 eeprom_bt_radio_shared;
+
+	u8 bt_coexistence;
+	u8 bt_ant_num;
+	u8 bt_coexist_type;
+	u8 bt_state;
+	u8 bt_cur_state;	/* 0:on, 1:off */
+	u8 bt_ant_isolation;	/* 0:good, 1:bad */
+	u8 bt_pape_ctrl;	/* 0:SW, 1:SW/HW dynamic */
+	u8 bt_service;
+	u8 bt_radio_shared_type;
+	u8 bt_rfreg_origin_1e;
+	u8 bt_rfreg_origin_1f;
+	u8 bt_rssi_state;
+	u32 ratio_tx;
+	u32 ratio_pri;
+	u32 bt_edca_ul;
+	u32 bt_edca_dl;
+
+	bool init_set;
+	bool bt_busy_traffic;
+	bool bt_traffic_mode_set;
+	bool bt_non_traffic_mode_set;
+
+	bool fw_coexist_all_off;
+	bool sw_coexist_all_off;
+	bool hw_coexist_all_off;
+	u32 cstate;
+	u32 previous_state;
+	u32 cstate_h;
+	u32 previous_state_h;
+
+	u8 bt_pre_rssi_state;
+	u8 bt_pre_rssi_state1;
+
+	u8 reg_bt_iso;
+	u8 reg_bt_sco;
+	bool balance_on;
+	u8 bt_active_zero_cnt;
+	bool cur_bt_disabled;
+	bool pre_bt_disabled;
+
+	u8 bt_profile_case;
+	u8 bt_profile_action;
+	bool bt_busy;
+	bool hold_for_bt_operation;
+	u8 lps_counter;
+};
+
+struct rtl_btc_ops {
+	void (*btc_init_variables)(struct rtl_priv *rtlpriv);
+	void (*btc_init_variables_wifi_only)(struct rtl_priv *rtlpriv);
+	void (*btc_deinit_variables)(struct rtl_priv *rtlpriv);
+	void (*btc_init_hal_vars)(struct rtl_priv *rtlpriv);
+	void (*btc_power_on_setting)(struct rtl_priv *rtlpriv);
+	void (*btc_init_hw_config)(struct rtl_priv *rtlpriv);
+	void (*btc_init_hw_config_wifi_only)(struct rtl_priv *rtlpriv);
+	void (*btc_ips_notify)(struct rtl_priv *rtlpriv, u8 type);
+	void (*btc_lps_notify)(struct rtl_priv *rtlpriv, u8 type);
+	void (*btc_scan_notify)(struct rtl_priv *rtlpriv, u8 scantype);
+	void (*btc_scan_notify_wifi_only)(struct rtl_priv *rtlpriv,
+					  u8 scantype);
+	void (*btc_connect_notify)(struct rtl_priv *rtlpriv, u8 action);
+	void (*btc_mediastatus_notify)(struct rtl_priv *rtlpriv,
+				       enum rt_media_status mstatus);
+	void (*btc_periodical)(struct rtl_priv *rtlpriv);
+	void (*btc_halt_notify)(struct rtl_priv *rtlpriv);
+	void (*btc_btinfo_notify)(struct rtl_priv *rtlpriv,
+				  u8 *tmp_buf, u8 length);
+	void (*btc_btmpinfo_notify)(struct rtl_priv *rtlpriv,
+				    u8 *tmp_buf, u8 length);
+	bool (*btc_is_limited_dig)(struct rtl_priv *rtlpriv);
+	bool (*btc_is_disable_edca_turbo)(struct rtl_priv *rtlpriv);
+	bool (*btc_is_bt_disabled)(struct rtl_priv *rtlpriv);
+	void (*btc_special_packet_notify)(struct rtl_priv *rtlpriv,
+					  u8 pkt_type);
+	void (*btc_switch_band_notify)(struct rtl_priv *rtlpriv, u8 type,
+				       bool scanning);
+	void (*btc_switch_band_notify_wifi_only)(struct rtl_priv *rtlpriv,
+						 u8 type, bool scanning);
+	void (*btc_display_bt_coex_info)(struct rtl_priv *rtlpriv,
+					 struct seq_file *m);
+	void (*btc_record_pwr_mode)(struct rtl_priv *rtlpriv, u8 *buf, u8 len);
+	u8   (*btc_get_lps_val)(struct rtl_priv *rtlpriv);
+	u8   (*btc_get_rpwm_val)(struct rtl_priv *rtlpriv);
+	bool (*btc_is_bt_ctrl_lps)(struct rtl_priv *rtlpriv);
+	void (*btc_get_ampdu_cfg)(struct rtl_priv *rtlpriv, u8 *reject_agg,
+				  u8 *ctrl_agg_size, u8 *agg_size);
+	bool (*btc_is_bt_lps_on)(struct rtl_priv *rtlpriv);
+};
+
+struct rtl_halmac_ops {
+	int (*halmac_init_adapter)(struct rtl_priv *);
+	int (*halmac_deinit_adapter)(struct rtl_priv *);
+	int (*halmac_init_hal)(struct rtl_priv *);
+	int (*halmac_deinit_hal)(struct rtl_priv *);
+	int (*halmac_poweron)(struct rtl_priv *);
+	int (*halmac_poweroff)(struct rtl_priv *);
+
+	int (*halmac_phy_power_switch)(struct rtl_priv *rtlpriv, u8 enable);
+	int (*halmac_set_mac_address)(struct rtl_priv *rtlpriv, u8 hwport,
+				      u8 *addr);
+	int (*halmac_set_bssid)(struct rtl_priv *rtlpriv, u8 hwport, u8 *addr);
+
+	int (*halmac_get_physical_efuse_size)(struct rtl_priv *rtlpriv,
+					      u32 *size);
+	int (*halmac_read_physical_efuse_map)(struct rtl_priv *rtlpriv,
+					      u8 *map, u32 size);
+	int (*halmac_get_logical_efuse_size)(struct rtl_priv *rtlpriv,
+					     u32 *size);
+	int (*halmac_read_logical_efuse_map)(struct rtl_priv *rtlpriv, u8 *map,
+					     u32 size);
+
+	int (*halmac_set_bandwidth)(struct rtl_priv *rtlpriv, u8 channel,
+				    u8 pri_ch_idx, u8 bw);
+
+	int (*halmac_c2h_handle)(struct rtl_priv *rtlpriv, u8 *c2h, u32 size);
+
+	int (*halmac_chk_txdesc)(struct rtl_priv *rtlpriv, u8 *txdesc,
+				 u32 size);
+};
+
+struct rtl_halmac_indicator {
+	struct completion *comp;
+	u32 wait_ms;
+
+	u8 *buffer;
+	u32 buf_size;
+	u32 ret_size;
+	u32 status;
+};
+
+struct rtl_halmac {
+	struct rtl_halmac_ops *ops; /* halmac ops (halmac.ko own this object) */
+	void *internal;	/* internal context of halmac, i.e. PHALMAC_ADAPTER */
+	struct rtl_halmac_indicator *indicator;	/* size=10 */
+
+	/* flags */
+	/*
+	 * send_general_info
+	 *	0: no need to call halmac_send_general_info()
+	 *	1: need to call halmac_send_general_info()
+	 */
+	u8 send_general_info;
+};
+
+struct rtl_phydm_params {
+	u8 mp_chip;	/* 1: MP chip, 0: test chip */
+	u8 fab_ver;	/* 0: TSMC, 1: UMC, ...*/
+	u8 cut_ver;	/* 0: A, 1: B, ..., 10: K */
+	u8 efuse0x3d7;	/* default: 0xff */
+	u8 efuse0x3d8;	/* default: 0xff */
+};
+
+struct rtl_phydm_ops {
+	/* init/deinit priv */
+	int (*phydm_init_priv)(struct rtl_priv *rtlpriv,
+			       struct rtl_phydm_params *params);
+	int (*phydm_deinit_priv)(struct rtl_priv *rtlpriv);
+	bool (*phydm_load_txpower_by_rate)(struct rtl_priv *rtlpriv);
+	bool (*phydm_load_txpower_limit)(struct rtl_priv *rtlpriv);
+
+	/* init hw */
+	int  (*phydm_init_dm)(struct rtl_priv *rtlpriv);
+	int  (*phydm_deinit_dm)(struct rtl_priv *rtlpriv);
+	int  (*phydm_reset_dm)(struct rtl_priv *rtlpriv);
+	bool (*phydm_parameter_init)(struct rtl_priv *rtlpriv, bool post);
+	bool (*phydm_phy_bb_config)(struct rtl_priv *rtlpriv);
+	bool (*phydm_phy_rf_config)(struct rtl_priv *rtlpriv);
+	bool (*phydm_phy_mac_config)(struct rtl_priv *rtlpriv);
+	bool (*phydm_trx_mode)(struct rtl_priv *rtlpriv,
+			       enum radio_mask tx_path, enum radio_mask rx_path,
+			       bool is_tx2_path);
+	/* watchdog */
+	bool (*phydm_watchdog)(struct rtl_priv *rtlpriv);
+
+	/* channel */
+	bool (*phydm_switch_band)(struct rtl_priv *rtlpriv, u8 central_ch);
+	bool (*phydm_switch_channel)(struct rtl_priv *rtlpriv, u8 central_ch);
+	bool (*phydm_switch_bandwidth)(struct rtl_priv *rtlpriv,
+				       u8 primary_ch_idx,
+				       enum ht_channel_width width);
+	bool (*phydm_iq_calibrate)(struct rtl_priv *rtlpriv);
+	bool (*phydm_clear_txpowertracking_state)(struct rtl_priv *rtlpriv);
+	bool (*phydm_pause_dig)(struct rtl_priv *rtlpriv, bool pause);
+
+	/* read/write reg */
+	u32  (*phydm_read_rf_reg)(struct rtl_priv *rtlpriv,
+				  enum radio_path rfpath,
+				  u32 addr, u32 mask);
+	bool (*phydm_write_rf_reg)(struct rtl_priv *rtlpriv,
+				   enum radio_path rfpath,
+				   u32 addr, u32 mask, u32 data);
+	u8   (*phydm_read_txagc)(struct rtl_priv *rtlpriv,
+				 enum radio_path rfpath, u8 hw_rate);
+	bool (*phydm_write_txagc)(struct rtl_priv *rtlpriv, u32 power_index,
+				  enum radio_path rfpath, u8 hw_rate);
+
+	/* RX */
+	bool (*phydm_c2h_content_parsing)(struct rtl_priv *rtlpriv, u8 cmd_id,
+					  u8 cmd_len, u8 *content);
+	bool (*phydm_query_phy_status)(struct rtl_priv *rtlpriv, u8 *phystrpt,
+				       struct ieee80211_hdr *hdr,
+				       struct rtl_stats *pstatus);
+
+	/* TX */
+	u8 (*phydm_rate_id_mapping)(struct rtl_priv *rtlpriv,
+				    enum wireless_mode wireless_mode,
+				    enum rf_type rf_type,
+				    enum ht_channel_width bw);
+	bool (*phydm_get_ra_bitmap)(struct rtl_priv *rtlpriv,
+				    enum wireless_mode wireless_mode,
+				    enum rf_type rf_type,
+				    enum ht_channel_width bw,
+				    u8 tx_rate_level, /* 0~6 */
+				    u32 *tx_bitmap_msb,
+				    u32 *tx_bitmap_lsb);
+
+	/* STA */
+	bool (*phydm_add_sta)(struct rtl_priv *rtlpriv,
+			      struct ieee80211_sta *sta);
+	bool (*phydm_del_sta)(struct rtl_priv *rtlpriv,
+			      struct ieee80211_sta *sta);
+
+	/* BTC */
+	u32  (*phydm_get_version)(struct rtl_priv *rtlpriv);
+	bool (*phydm_modify_ra_pcr_threshold)(struct rtl_priv *rtlpriv,
+					      u8 ra_offset_direction,
+					      u8 ra_threshold_offset);
+	u32  (*phydm_query_counter)(struct rtl_priv *rtlpriv,
+				    const char *info_type);
+
+	/* debug */
+	bool (*phydm_debug_cmd)(struct rtl_priv *rtlpriv, char *in, u32 in_len,
+				char *out, u32 out_len);
+
+};
+
+struct rtl_phydm {
+	struct rtl_phydm_ops *ops;/* phydm ops (phydm_mod.ko own this object) */
+	void *internal;	/* internal context of phydm, i.e. PHY_DM_STRUCT */
+
+	u8 adaptivity_en;
+	/* debug */
+	u16 forced_data_rate;
+	u8 forced_igi_lb;
+	u8 antenna_test;
+};
+
+struct proxim {
+	bool proxim_on;
+
+	void *proximity_priv;
+	int (*proxim_rx)(struct ieee80211_hw *hw, struct rtl_stats *status,
+			 struct sk_buff *skb);
+	u8  (*proxim_get_var)(struct ieee80211_hw *hw, u8 type);
+};
+
+struct rtl_c2hcmd {
+	struct list_head list;
+	u8 tag;
+	u8 len;
+	u8 *val;
+};
+
+struct rtl_bssid_entry {
+	struct list_head list;
+	u8 bssid[ETH_ALEN];
+	u32 age;
+};
+
+struct rtl_scan_list {
+	int num;
+	struct list_head list;	/* sort by age */
+};
+
+struct rtl_priv {
+	struct ieee80211_hw *hw;
+	struct completion firmware_loading_complete;
+	struct list_head list;
+	struct rtl_priv *buddy_priv;
+	struct rtl_global_var *glb_var;
+	struct rtl_dualmac_easy_concurrent_ctl easy_concurrent_ctl;
+	struct rtl_dmsp_ctl dmsp_ctl;
+	struct rtl_locks locks;
+	struct rtl_works works;
+	struct rtl_mac mac80211;
+	struct rtl_hal rtlhal;
+	struct rtl_regulatory regd;
+	struct rtl_rfkill rfkill;
+	struct rtl_io io;
+	struct rtl_phy phy;
+	struct rtl_dm dm;
+	struct rtl_security sec;
+	struct rtl_efuse efuse;
+	struct rtl_led_ctl ledctl;
+	struct rtl_tx_report tx_report;
+	struct rtl_scan_list scan_list;
+	struct rtl_ps_ctl psc;
+	struct rate_adaptive ra;
+	struct dynamic_primary_cca primarycca;
+	struct wireless_stats stats;
+	struct rt_link_detect link_info;
+	struct false_alarm_statistics falsealm_cnt;
+	struct rtl_rate_priv *rate_priv;
+	/* sta entry list for ap adhoc or mesh */
+	struct list_head entry_list;
+	/* c2hcmd list for kthread level access */
+	struct list_head c2hcmd_list;
+	struct rtl_debug dbg;
+	int max_fw_size;
+
+	/*hal_cfg : for diff cards
+	 *intf_ops : for diff interface usb/pcie
+	 */
+	struct rtl_hal_cfg *cfg;
+	const struct rtl_intf_ops *intf_ops;
+
+	/* this var will be set by set_bit,
+	 * and was used to indicate status of
+	 * interface or hardware
+	 */
+	unsigned long status;
+
+	/* tables for dm */
+	struct dig_t dm_digtable;
+	struct ps_t dm_pstable;
+
+	u32 reg_874;
+	u32 reg_c70;
+	u32 reg_85c;
+	u32 reg_a74;
+	bool reg_init;	/* true if regs saved */
+	bool bt_operation_on;
+	__le32 *usb_data;
+	int usb_data_index;
+	bool initialized;
+	bool enter_ps;	/* true when entering PS */
+	u8 rate_mask[5];
+
+	/* intel Proximity, should be alloc mem
+	 * in intel Proximity module and can only
+	 * be used in intel Proximity mode
+	 */
+	struct proxim proximity;
+
+	/*for bt coexist use*/
+	struct bt_coexist_info btcoexist;
+
+	/* halmac for newer IC. (e.g. 8822B) */
+	struct rtl_halmac halmac;
+
+	/* phydm for newer IC. (e.g. 8822B) */
+	struct rtl_phydm phydm;
+
+	/* separate 92ee from other ICs,
+	 * 92ee use new trx flow.
+	 */
+	bool use_new_trx_flow;
+
+#ifdef CONFIG_PM
+	struct wiphy_wowlan_support wowlan;
+#endif
+	/* This must be the last item so
+	 * that it points to the data allocated
+	 * beyond  this structure like:
+	 * rtl_pci_priv or rtl_usb_priv
+	 */
+	u8 priv[0] __aligned(sizeof(void *));
+};
+
+#define rtl_priv(hw)		(((struct rtl_priv *)(hw)->priv))
+#define rtl_mac(rtlpriv)	(&((rtlpriv)->mac80211))
+#define rtl_hal(rtlpriv)	(&((rtlpriv)->rtlhal))
+#define rtl_efuse(rtlpriv)	(&((rtlpriv)->efuse))
+#define rtl_psc(rtlpriv)	(&((rtlpriv)->psc))
+
+/***************************************
+ *    Bluetooth Co-existence Related
+ ***************************************/
+
+enum bt_ant_num {
+	ANT_X2 = 0,
+	ANT_X1 = 1,
+};
+
+enum bt_co_type {
+	BT_2WIRE = 0,
+	BT_ISSC_3WIRE = 1,
+	BT_ACCEL = 2,
+	BT_CSR_BC4 = 3,
+	BT_CSR_BC8 = 4,
+	BT_RTL8756 = 5,
+	BT_RTL8723A = 6,
+	BT_RTL8821A = 7,
+	BT_RTL8723B = 8,
+	BT_RTL8192E = 9,
+	BT_RTL8812A = 11,
+	BT_RTL8822B = 12,
+};
+
+enum bt_total_ant_num {
+	ANT_TOTAL_X2 = 0,
+	ANT_TOTAL_X1 = 1
+};
+
+enum bt_cur_state {
+	BT_OFF = 0,
+	BT_ON = 1,
+};
+
+enum bt_service_type {
+	BT_SCO = 0,
+	BT_A2DP = 1,
+	BT_HID = 2,
+	BT_HID_IDLE = 3,
+	BT_SCAN = 4,
+	BT_IDLE = 5,
+	BT_OTHER_ACTION = 6,
+	BT_BUSY = 7,
+	BT_OTHERBUSY = 8,
+	BT_PAN = 9,
+};
+
+enum bt_radio_shared {
+	BT_RADIO_SHARED = 0,
+	BT_RADIO_INDIVIDUAL = 1,
+};
+
+/****************************************
+ *	mem access macro define start
+ *	Call endian free function when
+ *	1. Read/write packet content.
+ *	2. Before write integer to IO.
+ *	3. After read integer from IO.
+ ***************************************/
+/* Convert little data endian to host ordering */
+#define EF1BYTE(_val)		\
+	((u8)(_val))
+#define EF2BYTE(_val)		\
+	(le16_to_cpu(_val))
+#define EF4BYTE(_val)		\
+	(le32_to_cpu(_val))
+
+/* Read data from memory */
+#define READEF1BYTE(_ptr)      \
+	EF1BYTE(*((u8 *)(_ptr)))
+/* Read le16 data from memory and convert to host ordering */
+#define READEF2BYTE(_ptr)      \
+	EF2BYTE(*(_ptr))
+#define READEF4BYTE(_ptr)      \
+	EF4BYTE(*(_ptr))
+
+/* Create a bit mask
+ * Examples:
+ * BIT_LEN_MASK_32(0) => 0x00000000
+ * BIT_LEN_MASK_32(1) => 0x00000001
+ * BIT_LEN_MASK_32(2) => 0x00000003
+ * BIT_LEN_MASK_32(32) => 0xFFFFFFFF
+ */
+#define BIT_LEN_MASK_32(__bitlen)	 \
+	(0xFFFFFFFF >> (32 - (__bitlen)))
+#define BIT_LEN_MASK_16(__bitlen)	 \
+	(0xFFFF >> (16 - (__bitlen)))
+#define BIT_LEN_MASK_8(__bitlen) \
+	(0xFF >> (8 - (__bitlen)))
+
+/* Create an offset bit mask
+ * Examples:
+ * BIT_OFFSET_LEN_MASK_32(0, 2) => 0x00000003
+ * BIT_OFFSET_LEN_MASK_32(16, 2) => 0x00030000
+ */
+#define BIT_OFFSET_LEN_MASK_32(__bitoffset, __bitlen) \
+	(BIT_LEN_MASK_32(__bitlen) << (__bitoffset))
+#define BIT_OFFSET_LEN_MASK_16(__bitoffset, __bitlen) \
+	(BIT_LEN_MASK_16(__bitlen) << (__bitoffset))
+#define BIT_OFFSET_LEN_MASK_8(__bitoffset, __bitlen) \
+	(BIT_LEN_MASK_8(__bitlen) << (__bitoffset))
+
+/*Description:
+ * Return 4-byte value in host byte ordering from
+ * 4-byte pointer in little-endian system.
+ */
+#define LE_P4BYTE_TO_HOST_4BYTE(__pstart) \
+	(EF4BYTE(*((__le32 *)(__pstart))))
+#define LE_P2BYTE_TO_HOST_2BYTE(__pstart) \
+	(EF2BYTE(*((__le16 *)(__pstart))))
+#define LE_P1BYTE_TO_HOST_1BYTE(__pstart) \
+	(EF1BYTE(*((u8 *)(__pstart))))
+
+/* Description:
+ * Translate subfield (continuous bits in little-endian) of 4-byte
+ * value to host byte ordering.
+ */
+#define LE_BITS_TO_4BYTE(__pstart, __bitoffset, __bitlen) \
+	( \
+		(LE_P4BYTE_TO_HOST_4BYTE(__pstart) >> (__bitoffset))  & \
+		BIT_LEN_MASK_32(__bitlen) \
+	)
+#define LE_BITS_TO_2BYTE(__pstart, __bitoffset, __bitlen) \
+	( \
+		(LE_P2BYTE_TO_HOST_2BYTE(__pstart) >> (__bitoffset)) & \
+		BIT_LEN_MASK_16(__bitlen) \
+	)
+#define LE_BITS_TO_1BYTE(__pstart, __bitoffset, __bitlen) \
+	( \
+		(LE_P1BYTE_TO_HOST_1BYTE(__pstart) >> (__bitoffset)) & \
+		BIT_LEN_MASK_8(__bitlen) \
+	)
+
+/* Description:
+ * Mask subfield (continuous bits in little-endian) of 4-byte value
+ * and return the result in 4-byte value in host byte ordering.
+ */
+#define LE_BITS_CLEARED_TO_4BYTE(__pstart, __bitoffset, __bitlen) \
+	( \
+		LE_P4BYTE_TO_HOST_4BYTE(__pstart)  & \
+		(~BIT_OFFSET_LEN_MASK_32(__bitoffset, __bitlen)) \
+	)
+#define LE_BITS_CLEARED_TO_2BYTE(__pstart, __bitoffset, __bitlen) \
+	( \
+		LE_P2BYTE_TO_HOST_2BYTE(__pstart) & \
+		(~BIT_OFFSET_LEN_MASK_16(__bitoffset, __bitlen)) \
+	)
+#define LE_BITS_CLEARED_TO_1BYTE(__pstart, __bitoffset, __bitlen) \
+	( \
+		LE_P1BYTE_TO_HOST_1BYTE(__pstart) & \
+		(~BIT_OFFSET_LEN_MASK_8(__bitoffset, __bitlen)) \
+	)
+
+/* Description:
+ * Set subfield of little-endian 4-byte value to specified value.
+ */
+#define SET_BITS_TO_LE_4BYTE(__pstart, __bitoffset, __bitlen, __val) \
+	(*((__le32 *)(__pstart)) = \
+	cpu_to_le32( \
+		LE_BITS_CLEARED_TO_4BYTE(__pstart, __bitoffset, __bitlen) | \
+		((((u32)__val) & BIT_LEN_MASK_32(__bitlen)) << (__bitoffset)) \
+	))
+#define SET_BITS_TO_LE_2BYTE(__pstart, __bitoffset, __bitlen, __val) \
+	(*((__le16 *)(__pstart)) = \
+	cpu_to_le16( \
+		LE_BITS_CLEARED_TO_2BYTE(__pstart, __bitoffset, __bitlen) | \
+		((((u16)__val) & BIT_LEN_MASK_16(__bitlen)) << (__bitoffset)) \
+	))
+#define SET_BITS_TO_LE_1BYTE(__pstart, __bitoffset, __bitlen, __val) \
+	(*((u8 *)(__pstart)) = EF1BYTE \
+	( \
+		LE_BITS_CLEARED_TO_1BYTE(__pstart, __bitoffset, __bitlen) | \
+		((((u8)__val) & BIT_LEN_MASK_8(__bitlen)) << (__bitoffset)) \
+	))
+
+#define	N_BYTE_ALIGNMENT(__value, __alignment) ((__alignment == 1) ? \
+	(__value) : (((__value + __alignment - 1) /		    \
+	 __alignment) * __alignment))
+
+/****************************************
+ *	mem access macro define end
+ ****************************************/
+
+#define byte(x, n) ((x >> (8 * n)) & 0xff)
+
+#define packet_get_type(_packet) (EF1BYTE((_packet).octet[0]) & 0xFC)
+#define RTL_WATCH_DOG_TIME	2000
+#define MSECS(t)		msecs_to_jiffies(t)
+#define WLAN_FC_GET_VERS(fc)	(le16_to_cpu(fc) & IEEE80211_FCTL_VERS)
+#define WLAN_FC_GET_TYPE(fc)	(le16_to_cpu(fc) & IEEE80211_FCTL_FTYPE)
+#define WLAN_FC_GET_STYPE(fc)	(le16_to_cpu(fc) & IEEE80211_FCTL_STYPE)
+#define WLAN_FC_MORE_DATA(fc)	(le16_to_cpu(fc) & IEEE80211_FCTL_MOREDATA)
+#define rtl_dm(rtlpriv)		(&((rtlpriv)->dm))
+
+#define	RT_RF_OFF_LEVL_ASPM		BIT(0)	/*PCI ASPM */
+#define	RT_RF_OFF_LEVL_CLK_REQ		BIT(1)	/*PCI clock request */
+#define	RT_RF_OFF_LEVL_PCI_D3		BIT(2)	/*PCI D3 mode */
+/*NIC halt, re-initialize hw parameters*/
+#define	RT_RF_OFF_LEVL_HALT_NIC		BIT(3)
+#define	RT_RF_OFF_LEVL_FREE_FW		BIT(4)	/*FW free, re-download the FW */
+#define	RT_RF_OFF_LEVL_FW_32K		BIT(5)	/*FW in 32k */
+/*Always enable ASPM and Clock Req in initialization.*/
+#define	RT_RF_PS_LEVEL_ALWAYS_ASPM	BIT(6)
+/* no matter RFOFF or SLEEP we set PS_ASPM_LEVL*/
+#define	RT_PS_LEVEL_ASPM		BIT(7)
+/*When LPS is on, disable 2R if no packet is received or transmitted.*/
+#define	RT_RF_LPS_DISALBE_2R		BIT(30)
+#define	RT_RF_LPS_LEVEL_ASPM		BIT(31)	/*LPS with ASPM */
+#define	RT_IN_PS_LEVEL(ppsc, _ps_flg)		\
+	((ppsc->cur_ps_level & _ps_flg) ? true : false)
+#define	RT_CLEAR_PS_LEVEL(ppsc, _ps_flg)	\
+	(ppsc->cur_ps_level &= (~(_ps_flg)))
+#define	RT_SET_PS_LEVEL(ppsc, _ps_flg)		\
+	(ppsc->cur_ps_level |= _ps_flg)
+
+#define container_of_dwork_rtl(x, y, z) \
+	container_of(to_delayed_work(x), y, z)
+
+#define FILL_OCTET_STRING(_os, _octet, _len)	\
+		(_os).octet = (u8 *)(_octet);		\
+		(_os).length = (_len)
+
+#define CP_MACADDR(des, src)	\
+	((des)[0] = (src)[0], (des)[1] = (src)[1],\
+	(des)[2] = (src)[2], (des)[3] = (src)[3],\
+	(des)[4] = (src)[4], (des)[5] = (src)[5])
+
+#define	LDPC_HT_ENABLE_RX			BIT(0)
+#define	LDPC_HT_ENABLE_TX			BIT(1)
+#define	LDPC_HT_TEST_TX_ENABLE			BIT(2)
+#define	LDPC_HT_CAP_TX				BIT(3)
+
+#define	STBC_HT_ENABLE_RX			BIT(0)
+#define	STBC_HT_ENABLE_TX			BIT(1)
+#define	STBC_HT_TEST_TX_ENABLE			BIT(2)
+#define	STBC_HT_CAP_TX				BIT(3)
+
+#define	LDPC_VHT_ENABLE_RX			BIT(0)
+#define	LDPC_VHT_ENABLE_TX			BIT(1)
+#define	LDPC_VHT_TEST_TX_ENABLE			BIT(2)
+#define	LDPC_VHT_CAP_TX				BIT(3)
+
+#define	STBC_VHT_ENABLE_RX			BIT(0)
+#define	STBC_VHT_ENABLE_TX			BIT(1)
+#define	STBC_VHT_TEST_TX_ENABLE			BIT(2)
+#define	STBC_VHT_CAP_TX				BIT(3)
+
+extern u8 channel5g[CHANNEL_MAX_NUMBER_5G];
+
+extern u8 channel5g_80m[CHANNEL_MAX_NUMBER_5G_80M];
+
+static inline u8 rtl_read_byte(struct rtl_priv *rtlpriv, u32 addr)
+{
+	return rtlpriv->io.read8_sync(rtlpriv, addr);
+}
+
+static inline u16 rtl_read_word(struct rtl_priv *rtlpriv, u32 addr)
+{
+	return rtlpriv->io.read16_sync(rtlpriv, addr);
+}
+
+static inline u32 rtl_read_dword(struct rtl_priv *rtlpriv, u32 addr)
+{
+	return rtlpriv->io.read32_sync(rtlpriv, addr);
+}
+
+static inline void rtl_write_byte(struct rtl_priv *rtlpriv, u32 addr, u8 val8)
+{
+	rtlpriv->io.write8_async(rtlpriv, addr, val8);
+
+	if (rtlpriv->cfg->write_readback)
+		rtlpriv->io.read8_sync(rtlpriv, addr);
+}
+
+static inline void rtl_write_byte_with_val32(struct ieee80211_hw *hw,
+					     u32 addr, u32 val8)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	rtl_write_byte(rtlpriv, addr, (u8)val8);
+}
+
+static inline void rtl_write_word(struct rtl_priv *rtlpriv, u32 addr, u16 val16)
+{
+	rtlpriv->io.write16_async(rtlpriv, addr, val16);
+
+	if (rtlpriv->cfg->write_readback)
+		rtlpriv->io.read16_sync(rtlpriv, addr);
+}
+
+static inline void rtl_write_dword(struct rtl_priv *rtlpriv,
+				   u32 addr, u32 val32)
+{
+	rtlpriv->io.write32_async(rtlpriv, addr, val32);
+
+	if (rtlpriv->cfg->write_readback)
+		rtlpriv->io.read32_sync(rtlpriv, addr);
+}
+
+static inline u32 rtl_get_bbreg(struct ieee80211_hw *hw,
+				u32 regaddr, u32 bitmask)
+{
+	struct rtl_priv *rtlpriv = hw->priv;
+
+	return rtlpriv->cfg->ops->get_bbreg(hw, regaddr, bitmask);
+}
+
+static inline void rtl_set_bbreg(struct ieee80211_hw *hw, u32 regaddr,
+				 u32 bitmask, u32 data)
+{
+	struct rtl_priv *rtlpriv = hw->priv;
+
+	rtlpriv->cfg->ops->set_bbreg(hw, regaddr, bitmask, data);
+}
+
+static inline void rtl_set_bbreg_with_dwmask(struct ieee80211_hw *hw,
+					     u32 regaddr, u32 data)
+{
+	rtl_set_bbreg(hw, regaddr, 0xffffffff, data);
+}
+
+static inline u32 rtl_get_rfreg(struct ieee80211_hw *hw,
+				enum radio_path rfpath, u32 regaddr,
+				u32 bitmask)
+{
+	struct rtl_priv *rtlpriv = hw->priv;
+
+	return rtlpriv->cfg->ops->get_rfreg(hw, rfpath, regaddr, bitmask);
+}
+
+static inline void rtl_set_rfreg(struct ieee80211_hw *hw,
+				 enum radio_path rfpath, u32 regaddr,
+				 u32 bitmask, u32 data)
+{
+	struct rtl_priv *rtlpriv = hw->priv;
+
+	rtlpriv->cfg->ops->set_rfreg(hw, rfpath, regaddr, bitmask, data);
+}
+
+static inline bool is_hal_stop(struct rtl_hal *rtlhal)
+{
+	return (rtlhal->state == _HAL_STATE_STOP);
+}
+
+static inline void set_hal_start(struct rtl_hal *rtlhal)
+{
+	rtlhal->state = _HAL_STATE_START;
+}
+
+static inline void set_hal_stop(struct rtl_hal *rtlhal)
+{
+	rtlhal->state = _HAL_STATE_STOP;
+}
+
+static inline u8 get_rf_type(struct rtl_phy *rtlphy)
+{
+	return rtlphy->rf_type;
+}
+
+static inline struct ieee80211_hdr *rtl_get_hdr(struct sk_buff *skb)
+{
+	return (struct ieee80211_hdr *)(skb->data);
+}
+
+static inline __le16 rtl_get_fc(struct sk_buff *skb)
+{
+	return rtl_get_hdr(skb)->frame_control;
+}
+
+static inline u16 rtl_get_tid_h(struct ieee80211_hdr *hdr)
+{
+	return (ieee80211_get_qos_ctl(hdr))[0] & IEEE80211_QOS_CTL_TID_MASK;
+}
+
+static inline u16 rtl_get_tid(struct sk_buff *skb)
+{
+	return rtl_get_tid_h(rtl_get_hdr(skb));
+}
+
+static inline struct ieee80211_sta *get_sta(struct ieee80211_hw *hw,
+					    struct ieee80211_vif *vif,
+					    const u8 *bssid)
+{
+	return ieee80211_find_sta(vif, bssid);
+}
+
+static inline struct ieee80211_sta *rtl_find_sta(struct ieee80211_hw *hw,
+						 u8 *mac_addr)
+{
+	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
+
+	return ieee80211_find_sta(mac->vif, mac_addr);
+}
+
+#endif
diff --git a/drivers/staging/rts5208/ms.c b/drivers/staging/rts5208/ms.c
index 482a29dd06f8..7cdce87f3051 100644
--- a/drivers/staging/rts5208/ms.c
+++ b/drivers/staging/rts5208/ms.c
@@ -3064,7 +3064,8 @@ static int mspro_rw_multi_sector(struct scsi_cmnd *srb,
 
 		if (detect_card_cd(chip, MS_CARD) != STATUS_SUCCESS) {
 			chip->rw_need_retry = 0;
-			dev_dbg(rtsx_dev(chip), "No card exist, exit mspro_rw_multi_sector\n");
+			dev_dbg(rtsx_dev(chip), "No card exist, exit %s\n",
+				__func__);
 			rtsx_trace(chip);
 			return STATUS_FAIL;
 		}
@@ -3101,7 +3102,7 @@ static int mspro_read_format_progress(struct rtsx_chip *chip,
 	u8 cnt, tmp;
 	u8 data[8];
 
-	dev_dbg(rtsx_dev(chip), "mspro_read_format_progress, short_data_len = %d\n",
+	dev_dbg(rtsx_dev(chip), "%s, short_data_len = %d\n", __func__,
 		short_data_len);
 
 	retval = ms_switch_clock(chip);
diff --git a/drivers/staging/rts5208/rtsx.c b/drivers/staging/rts5208/rtsx.c
index b8177f50fabc..89e2cfe7d1cc 100644
--- a/drivers/staging/rts5208/rtsx.c
+++ b/drivers/staging/rts5208/rtsx.c
@@ -205,16 +205,6 @@ static int device_reset(struct scsi_cmnd *srb)
 	return SUCCESS;
 }
 
-/* Simulate a SCSI bus reset by resetting the device's USB port. */
-static int bus_reset(struct scsi_cmnd *srb)
-{
-	struct rtsx_dev *dev = host_to_rtsx(srb->device->host);
-
-	dev_info(&dev->pci->dev, "%s called\n", __func__);
-
-	return SUCCESS;
-}
-
 /*
  * this defines our host template, with which we'll allocate hosts
  */
@@ -231,7 +221,6 @@ static struct scsi_host_template rtsx_host_template = {
 	/* error and abort handlers */
 	.eh_abort_handler =		command_abort,
 	.eh_device_reset_handler =	device_reset,
-	.eh_bus_reset_handler =		bus_reset,
 
 	/* queue commands only, only one command per LUN */
 	.can_queue =			1,
@@ -999,7 +988,7 @@ static int rtsx_probe(struct pci_dev *pci,
 
 	/* We come here if there are any problems */
 errout:
-	dev_err(&pci->dev, "rtsx_probe() failed\n");
+	dev_err(&pci->dev, "%s failed\n", __func__);
 	release_everything(dev);
 
 	return err;
@@ -1009,7 +998,7 @@ static void rtsx_remove(struct pci_dev *pci)
 {
 	struct rtsx_dev *dev = pci_get_drvdata(pci);
 
-	dev_info(&pci->dev, "rtsx_remove() called\n");
+	dev_info(&pci->dev, "%s called\n", __func__);
 
 	quiesce_and_remove_host(dev);
 	release_everything(dev);
diff --git a/drivers/staging/rts5208/rtsx_chip.c b/drivers/staging/rts5208/rtsx_chip.c
index 7f4107bffe31..4ad472dd9daf 100644
--- a/drivers/staging/rts5208/rtsx_chip.c
+++ b/drivers/staging/rts5208/rtsx_chip.c
@@ -616,8 +616,8 @@ int rtsx_reset_chip(struct rtsx_chip *chip)
 		else
 			retval = rtsx_pre_handle_sdio_new(chip);
 
-		dev_dbg(rtsx_dev(chip), "chip->need_reset = 0x%x (rtsx_reset_chip)\n",
-			(unsigned int)(chip->need_reset));
+		dev_dbg(rtsx_dev(chip), "chip->need_reset = 0x%x (%s)\n",
+			(unsigned int)(chip->need_reset), __func__);
 #else  /* HW_AUTO_SWITCH_SD_BUS */
 		retval = rtsx_pre_handle_sdio_old(chip);
 #endif  /* HW_AUTO_SWITCH_SD_BUS */
diff --git a/drivers/staging/rts5208/rtsx_scsi.c b/drivers/staging/rts5208/rtsx_scsi.c
index 36b5a11f21d2..a401b13f5f5e 100644
--- a/drivers/staging/rts5208/rtsx_scsi.c
+++ b/drivers/staging/rts5208/rtsx_scsi.c
@@ -414,7 +414,7 @@ void set_sense_data(struct rtsx_chip *chip, unsigned int lun, u8 err_code,
 	sense->ascq = ascq;
 	if (sns_key_info0 != 0) {
 		sense->sns_key_info[0] = SKSV | sns_key_info0;
-		sense->sns_key_info[1] = (sns_key_info1 & 0xf0) >> 8;
+		sense->sns_key_info[1] = (sns_key_info1 & 0xf0) >> 4;
 		sense->sns_key_info[2] = sns_key_info1 & 0x0f;
 	}
 }
diff --git a/drivers/staging/rts5208/sd.c b/drivers/staging/rts5208/sd.c
index c2eb072cbe1d..4033a2cf7ac9 100644
--- a/drivers/staging/rts5208/sd.c
+++ b/drivers/staging/rts5208/sd.c
@@ -910,8 +910,8 @@ static int sd_change_phase(struct rtsx_chip *chip, u8 sample_point, u8 tune_dir)
 	int retval;
 	bool ddr_rx = false;
 
-	dev_dbg(rtsx_dev(chip), "sd_change_phase (sample_point = %d, tune_dir = %d)\n",
-		sample_point, tune_dir);
+	dev_dbg(rtsx_dev(chip), "%s (sample_point = %d, tune_dir = %d)\n",
+		__func__, sample_point, tune_dir);
 
 	if (tune_dir == TUNE_RX) {
 		SD_VP_CTL = SD_VPRX_CTL;
@@ -1225,8 +1225,8 @@ static int sd_check_switch_mode(struct rtsx_chip *chip, u8 mode, u8 func_group,
 	int retval;
 	u8 cmd[5], buf[64];
 
-	dev_dbg(rtsx_dev(chip), "sd_check_switch_mode (mode = %d, func_group = %d, func_to_switch = %d)\n",
-		mode, func_group, func_to_switch);
+	dev_dbg(rtsx_dev(chip), "%s (mode = %d, func_group = %d, func_to_switch = %d)\n",
+		__func__, mode, func_group, func_to_switch);
 
 	cmd[0] = 0x40 | SWITCH;
 	cmd[1] = mode;
@@ -1654,7 +1654,7 @@ static int sd_ddr_tuning_rx_cmd(struct rtsx_chip *chip, u8 sample_point)
 	return STATUS_SUCCESS;
 }
 
-static int mmc_ddr_tunning_rx_cmd(struct rtsx_chip *chip, u8 sample_point)
+static int mmc_ddr_tuning_rx_cmd(struct rtsx_chip *chip, u8 sample_point)
 {
 	struct sd_info *sd_card = &chip->sd_card;
 	int retval;
@@ -1933,7 +1933,7 @@ static int sd_tuning_rx(struct rtsx_chip *chip)
 
 	} else {
 		if (CHK_MMC_DDR52(sd_card)) {
-			tuning_cmd = mmc_ddr_tunning_rx_cmd;
+			tuning_cmd = mmc_ddr_tuning_rx_cmd;
 		} else {
 			rtsx_trace(chip);
 			return STATUS_FAIL;
@@ -3575,8 +3575,8 @@ static int reset_mmc_only(struct rtsx_chip *chip)
 		return STATUS_FAIL;
 	}
 
-	dev_dbg(rtsx_dev(chip), "In reset_mmc_only, sd_card->sd_type = 0x%x\n",
-		sd_card->sd_type);
+	dev_dbg(rtsx_dev(chip), "In %s, sd_card->sd_type = 0x%x\n",
+		__func__, sd_card->sd_type);
 
 	return STATUS_SUCCESS;
 }
@@ -3699,11 +3699,11 @@ int sd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip, u32 start_sector,
 	int retval;
 
 	if (srb->sc_data_direction == DMA_FROM_DEVICE) {
-		dev_dbg(rtsx_dev(chip), "sd_rw: Read %d %s from 0x%x\n",
+		dev_dbg(rtsx_dev(chip), "%s: Read %d %s from 0x%x\n", __func__,
 			sector_cnt, (sector_cnt > 1) ? "sectors" : "sector",
 			start_sector);
 	} else {
-		dev_dbg(rtsx_dev(chip), "sd_rw: Write %d %s to 0x%x\n",
+		dev_dbg(rtsx_dev(chip), "%s: Write %d %s to 0x%x\n", __func__,
 			sector_cnt, (sector_cnt > 1) ? "sectors" : "sector",
 			start_sector);
 	}
@@ -3921,7 +3921,8 @@ int sd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip, u32 start_sector,
 		rtsx_clear_sd_error(chip);
 		if (detect_card_cd(chip, SD_CARD) != STATUS_SUCCESS) {
 			chip->rw_need_retry = 0;
-			dev_dbg(rtsx_dev(chip), "No card exist, exit sd_rw\n");
+			dev_dbg(rtsx_dev(chip), "No card exist, exit %s\n",
+				__func__);
 			rtsx_trace(chip);
 			return STATUS_FAIL;
 		}
@@ -3964,7 +3965,7 @@ RW_FAIL:
 
 	if (detect_card_cd(chip, SD_CARD) != STATUS_SUCCESS) {
 		chip->rw_need_retry = 0;
-		dev_dbg(rtsx_dev(chip), "No card exist, exit sd_rw\n");
+		dev_dbg(rtsx_dev(chip), "No card exist, exit %s\n", __func__);
 		rtsx_trace(chip);
 		return STATUS_FAIL;
 	}
diff --git a/drivers/staging/rts5208/spi.c b/drivers/staging/rts5208/spi.c
index 8b8cd955dfeb..b5646b62ec9e 100644
--- a/drivers/staging/rts5208/spi.c
+++ b/drivers/staging/rts5208/spi.c
@@ -520,7 +520,7 @@ int spi_get_status(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 {
 	struct spi_info *spi = &chip->spi;
 
-	dev_dbg(rtsx_dev(chip), "spi_get_status: err_code = 0x%x\n",
+	dev_dbg(rtsx_dev(chip), "%s: err_code = 0x%x\n", __func__,
 		spi->err_code);
 	rtsx_stor_set_xfer_buf(&spi->err_code,
 			       min_t(int, scsi_bufflen(srb), 1), srb);
@@ -543,8 +543,10 @@ int spi_set_parameter(struct scsi_cmnd *srb, struct rtsx_chip *chip)
 	spi->clk_div = ((u16)(srb->cmnd[4]) << 8) | srb->cmnd[5];
 	spi->write_en = srb->cmnd[6];
 
-	dev_dbg(rtsx_dev(chip), "spi_set_parameter: spi_clock = %d, clk_div = %d, write_en = %d\n",
-		spi->spi_clock, spi->clk_div, spi->write_en);
+	dev_dbg(rtsx_dev(chip), "%s: ", __func__);
+	dev_dbg(rtsx_dev(chip), "spi_clock = %d, ", spi->spi_clock);
+	dev_dbg(rtsx_dev(chip), "clk_div = %d, ", spi->clk_div);
+	dev_dbg(rtsx_dev(chip), "write_en = %d\n", spi->write_en);
 
 	return STATUS_SUCCESS;
 }
diff --git a/drivers/staging/rts5208/xd.c b/drivers/staging/rts5208/xd.c
index 74d36f9a4c1d..11ea0c658e28 100644
--- a/drivers/staging/rts5208/xd.c
+++ b/drivers/staging/rts5208/xd.c
@@ -885,7 +885,7 @@ static int xd_init_l2p_tbl(struct rtsx_chip *chip)
 	struct xd_info *xd_card = &chip->xd_card;
 	int size, i;
 
-	dev_dbg(rtsx_dev(chip), "xd_init_l2p_tbl: zone_cnt = %d\n",
+	dev_dbg(rtsx_dev(chip), "%s: zone_cnt = %d\n", __func__,
 		xd_card->zone_cnt);
 
 	if (xd_card->zone_cnt < 1) {
@@ -1026,7 +1026,8 @@ static u32 xd_get_l2p_tbl(struct rtsx_chip *chip, int zone_no, u16 log_off)
 #ifdef XD_DELAY_WRITE
 		retval = xd_delay_write(chip);
 		if (retval != STATUS_SUCCESS) {
-			dev_dbg(rtsx_dev(chip), "In xd_get_l2p_tbl, delay write fail!\n");
+			dev_dbg(rtsx_dev(chip), "In %s, delay write fail!\n",
+				__func__);
 			return BLK_NOT_FOUND;
 		}
 #endif
@@ -1434,7 +1435,7 @@ static int xd_build_l2p_tbl(struct rtsx_chip *chip, int zone_no)
 	u16 cur_lst_page_logoff, ent_lst_page_logoff;
 	u8 redunt[11];
 
-	dev_dbg(rtsx_dev(chip), "xd_build_l2p_tbl: %d\n", zone_no);
+	dev_dbg(rtsx_dev(chip), "%s: %d\n", __func__, zone_no);
 
 	if (!xd_card->zone) {
 		retval = xd_init_l2p_tbl(chip);
@@ -1774,8 +1775,10 @@ static int xd_finish_write(struct rtsx_chip *chip,
 	int retval, zone_no;
 	u16 log_off;
 
-	dev_dbg(rtsx_dev(chip), "xd_finish_write, old_blk = 0x%x, new_blk = 0x%x, log_blk = 0x%x\n",
-		old_blk, new_blk, log_blk);
+	dev_dbg(rtsx_dev(chip), "%s ", __func__);
+	dev_dbg(rtsx_dev(chip), "old_blk = 0x%x, ", old_blk);
+	dev_dbg(rtsx_dev(chip),	"new_blk = 0x%x, ", new_blk);
+	dev_dbg(rtsx_dev(chip), "log_blk = 0x%x\n", log_blk);
 
 	if (page_off > xd_card->page_off) {
 		rtsx_trace(chip);
@@ -1960,7 +1963,7 @@ int xd_delay_write(struct rtsx_chip *chip)
 	int retval;
 
 	if (delay_write->delay_write_flag) {
-		dev_dbg(rtsx_dev(chip), "xd_delay_write\n");
+		dev_dbg(rtsx_dev(chip), "%s\n", __func__);
 		retval = xd_switch_clock(chip);
 		if (retval != STATUS_SUCCESS) {
 			rtsx_trace(chip);
@@ -2002,7 +2005,7 @@ int xd_rw(struct scsi_cmnd *srb, struct rtsx_chip *chip,
 
 	xd_card->cleanup_counter = 0;
 
-	dev_dbg(rtsx_dev(chip), "xd_rw: scsi_sg_count = %d\n",
+	dev_dbg(rtsx_dev(chip), "%s: scsi_sg_count = %d\n", __func__,
 		scsi_sg_count(srb));
 
 	ptr = (u8 *)scsi_sglist(srb);
diff --git a/drivers/staging/skein/skein_block.c b/drivers/staging/skein/skein_block.c
index 256657077a46..3bc25e149034 100644
--- a/drivers/staging/skein/skein_block.c
+++ b/drivers/staging/skein/skein_block.c
@@ -21,329 +21,6 @@
 #include "skein_base.h"
 #include "skein_block.h"
 
-#ifndef SKEIN_USE_ASM
-#define SKEIN_USE_ASM   (0) /* default is all C code (no ASM) */
-#endif
-
-#ifndef SKEIN_LOOP
-#define SKEIN_LOOP 001 /* default: unroll 256 and 512, but not 1024 */
-#endif
-
-#define BLK_BITS        (WCNT * 64) /* some useful definitions for code here */
-#define KW_TWK_BASE     (0)
-#define KW_KEY_BASE     (3)
-#define ks              (kw + KW_KEY_BASE)
-#define ts              (kw + KW_TWK_BASE)
-
-#ifdef SKEIN_DEBUG
-#define debug_save_tweak(ctx)       \
-{                                   \
-	ctx->h.tweak[0] = ts[0];    \
-	ctx->h.tweak[1] = ts[1];    \
-}
-#else
-#define debug_save_tweak(ctx)
-#endif
-
-#if !(SKEIN_USE_ASM & 256)
-#undef  RCNT
-#define RCNT (SKEIN_256_ROUNDS_TOTAL / 8)
-#ifdef SKEIN_LOOP /* configure how much to unroll the loop */
-#define SKEIN_UNROLL_256 (((SKEIN_LOOP) / 100) % 10)
-#else
-#define SKEIN_UNROLL_256 (0)
-#endif
-
-#if SKEIN_UNROLL_256
-#if (RCNT % SKEIN_UNROLL_256)
-#error "Invalid SKEIN_UNROLL_256" /* sanity check on unroll count */
-#endif
-#endif
-#define ROUND256(p0, p1, p2, p3, ROT, r_num)         \
-	do {                                         \
-		X##p0 += X##p1;                      \
-		X##p1 = rol64(X##p1, ROT##_0);       \
-		X##p1 ^= X##p0;                      \
-		X##p2 += X##p3;                      \
-		X##p3 = rol64(X##p3, ROT##_1);       \
-		X##p3 ^= X##p2;                      \
-	} while (0)
-
-#if SKEIN_UNROLL_256 == 0
-#define R256(p0, p1, p2, p3, ROT, r_num) /* fully unrolled */ \
-	ROUND256(p0, p1, p2, p3, ROT, r_num)
-
-#define I256(R)                                                         \
-	do {                                                            \
-		/* inject the key schedule value */                     \
-		X0   += ks[((R) + 1) % 5];                              \
-		X1   += ks[((R) + 2) % 5] + ts[((R) + 1) % 3];          \
-		X2   += ks[((R) + 3) % 5] + ts[((R) + 2) % 3];          \
-		X3   += ks[((R) + 4) % 5] + (R) + 1;                    \
-	} while (0)
-#else
-/* looping version */
-#define R256(p0, p1, p2, p3, ROT, r_num) ROUND256(p0, p1, p2, p3, ROT, r_num)
-
-#define I256(R)                                         \
-	do {                                            \
-		/* inject the key schedule value */     \
-		X0 += ks[r + (R) + 0];                  \
-		X1 += ks[r + (R) + 1] + ts[r + (R) + 0];\
-		X2 += ks[r + (R) + 2] + ts[r + (R) + 1];\
-		X3 += ks[r + (R) + 3] + r + (R);        \
-		/* rotate key schedule */               \
-		ks[r + (R) + 4] = ks[r + (R) - 1];      \
-		ts[r + (R) + 2] = ts[r + (R) - 1];      \
-	} while (0)
-#endif
-#define R256_8_ROUNDS(R)                                \
-	do {                                            \
-		R256(0, 1, 2, 3, R_256_0, 8 * (R) + 1); \
-		R256(0, 3, 2, 1, R_256_1, 8 * (R) + 2); \
-		R256(0, 1, 2, 3, R_256_2, 8 * (R) + 3); \
-		R256(0, 3, 2, 1, R_256_3, 8 * (R) + 4); \
-		I256(2 * (R));                          \
-		R256(0, 1, 2, 3, R_256_4, 8 * (R) + 5); \
-		R256(0, 3, 2, 1, R_256_5, 8 * (R) + 6); \
-		R256(0, 1, 2, 3, R_256_6, 8 * (R) + 7); \
-		R256(0, 3, 2, 1, R_256_7, 8 * (R) + 8); \
-		I256(2 * (R) + 1);                      \
-	} while (0)
-
-#define R256_UNROLL_R(NN)                     \
-	((SKEIN_UNROLL_256 == 0 &&            \
-	SKEIN_256_ROUNDS_TOTAL / 8 > (NN)) || \
-	(SKEIN_UNROLL_256 > (NN)))
-
-#if  (SKEIN_UNROLL_256 > 14)
-#error  "need more unrolling in skein_256_process_block"
-#endif
-#endif
-
-#if !(SKEIN_USE_ASM & 512)
-#undef  RCNT
-#define RCNT  (SKEIN_512_ROUNDS_TOTAL / 8)
-
-#ifdef SKEIN_LOOP /* configure how much to unroll the loop */
-#define SKEIN_UNROLL_512 (((SKEIN_LOOP) / 10) % 10)
-#else
-#define SKEIN_UNROLL_512 (0)
-#endif
-
-#if SKEIN_UNROLL_512
-#if (RCNT % SKEIN_UNROLL_512)
-#error "Invalid SKEIN_UNROLL_512" /* sanity check on unroll count */
-#endif
-#endif
-#define ROUND512(p0, p1, p2, p3, p4, p5, p6, p7, ROT, r_num)    \
-	do {                                                    \
-		X##p0 += X##p1;                                 \
-		X##p1 = rol64(X##p1, ROT##_0);                  \
-		X##p1 ^= X##p0;                                 \
-		X##p2 += X##p3;                                 \
-		X##p3 = rol64(X##p3, ROT##_1);                  \
-		X##p3 ^= X##p2;                                 \
-		X##p4 += X##p5;                                 \
-		X##p5 = rol64(X##p5, ROT##_2);                  \
-		X##p5 ^= X##p4;                                 \
-		X##p6 += X##p7;                                 \
-		X##p7 = rol64(X##p7, ROT##_3);			\
-		X##p7 ^= X##p6;                                 \
-	} while (0)
-
-#if SKEIN_UNROLL_512 == 0
-#define R512(p0, p1, p2, p3, p4, p5, p6, p7, ROT, r_num) /* unrolled */ \
-	ROUND512(p0, p1, p2, p3, p4, p5, p6, p7, ROT, r_num)
-
-#define I512(R)                                                         \
-	do {                                                            \
-		/* inject the key schedule value */                     \
-		X0   += ks[((R) + 1) % 9];                              \
-		X1   += ks[((R) + 2) % 9];                              \
-		X2   += ks[((R) + 3) % 9];                              \
-		X3   += ks[((R) + 4) % 9];                              \
-		X4   += ks[((R) + 5) % 9];                              \
-		X5   += ks[((R) + 6) % 9] + ts[((R) + 1) % 3];          \
-		X6   += ks[((R) + 7) % 9] + ts[((R) + 2) % 3];          \
-		X7   += ks[((R) + 8) % 9] + (R) + 1;                    \
-	} while (0)
-
-#else /* looping version */
-#define R512(p0, p1, p2, p3, p4, p5, p6, p7, ROT, r_num)                 \
-	ROUND512(p0, p1, p2, p3, p4, p5, p6, p7, ROT, r_num)             \
-
-#define I512(R)                                                         \
-	do {                                                            \
-		/* inject the key schedule value */                     \
-		X0   += ks[r + (R) + 0];                                \
-		X1   += ks[r + (R) + 1];                                \
-		X2   += ks[r + (R) + 2];                                \
-		X3   += ks[r + (R) + 3];                                \
-		X4   += ks[r + (R) + 4];                                \
-		X5   += ks[r + (R) + 5] + ts[r + (R) + 0];              \
-		X6   += ks[r + (R) + 6] + ts[r + (R) + 1];              \
-		X7   += ks[r + (R) + 7] + r + (R);                      \
-		/* rotate key schedule */                               \
-		ks[r + (R) + 8] = ks[r + (R) - 1];                      \
-		ts[r + (R) + 2] = ts[r + (R) - 1];                      \
-	} while (0)
-#endif /* end of looped code definitions */
-#define R512_8_ROUNDS(R)  /* do 8 full rounds */                        \
-	do {                                                            \
-		R512(0, 1, 2, 3, 4, 5, 6, 7, R_512_0, 8 * (R) + 1);     \
-		R512(2, 1, 4, 7, 6, 5, 0, 3, R_512_1, 8 * (R) + 2);     \
-		R512(4, 1, 6, 3, 0, 5, 2, 7, R_512_2, 8 * (R) + 3);     \
-		R512(6, 1, 0, 7, 2, 5, 4, 3, R_512_3, 8 * (R) + 4);     \
-		I512(2 * (R));                                          \
-		R512(0, 1, 2, 3, 4, 5, 6, 7, R_512_4, 8 * (R) + 5);     \
-		R512(2, 1, 4, 7, 6, 5, 0, 3, R_512_5, 8 * (R) + 6);     \
-		R512(4, 1, 6, 3, 0, 5, 2, 7, R_512_6, 8 * (R) + 7);     \
-		R512(6, 1, 0, 7, 2, 5, 4, 3, R_512_7, 8 * (R) + 8);     \
-		I512(2 * (R) + 1); /* and key injection */              \
-	} while (0)
-#define R512_UNROLL_R(NN)                             \
-		((SKEIN_UNROLL_512 == 0 &&            \
-		SKEIN_512_ROUNDS_TOTAL / 8 > (NN)) || \
-		(SKEIN_UNROLL_512 > (NN)))
-
-#if  (SKEIN_UNROLL_512 > 14)
-#error  "need more unrolling in skein_512_process_block"
-#endif
-#endif
-
-#if !(SKEIN_USE_ASM & 1024)
-#undef  RCNT
-#define RCNT  (SKEIN_1024_ROUNDS_TOTAL / 8)
-#ifdef SKEIN_LOOP /* configure how much to unroll the loop */
-#define SKEIN_UNROLL_1024 ((SKEIN_LOOP) % 10)
-#else
-#define SKEIN_UNROLL_1024 (0)
-#endif
-
-#if (SKEIN_UNROLL_1024 != 0)
-#if (RCNT % SKEIN_UNROLL_1024)
-#error "Invalid SKEIN_UNROLL_1024" /* sanity check on unroll count */
-#endif
-#endif
-#define ROUND1024(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, pA, pB, pC, pD, pE, \
-		  pF, ROT, r_num)                                             \
-	do {                                                                  \
-		X##p0 += X##p1;                                               \
-		X##p1 = rol64(X##p1, ROT##_0);                                \
-		X##p1 ^= X##p0;                                               \
-		X##p2 += X##p3;                                               \
-		X##p3 = rol64(X##p3, ROT##_1);                                \
-		X##p3 ^= X##p2;                                               \
-		X##p4 += X##p5;                                               \
-		X##p5 = rol64(X##p5, ROT##_2);                                \
-		X##p5 ^= X##p4;                                               \
-		X##p6 += X##p7;                                               \
-		X##p7 = rol64(X##p7, ROT##_3);                                \
-		X##p7 ^= X##p6;                                               \
-		X##p8 += X##p9;                                               \
-		X##p9 = rol64(X##p9, ROT##_4);                                \
-		X##p9 ^= X##p8;                                               \
-		X##pA += X##pB;                                               \
-		X##pB = rol64(X##pB, ROT##_5);                                \
-		X##pB ^= X##pA;                                               \
-		X##pC += X##pD;                                               \
-		X##pD = rol64(X##pD, ROT##_6);                                \
-		X##pD ^= X##pC;                                               \
-		X##pE += X##pF;                                               \
-		X##pF = rol64(X##pF, ROT##_7);                                \
-		X##pF ^= X##pE;                                               \
-	} while (0)
-
-#if SKEIN_UNROLL_1024 == 0
-#define R1024(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, pA, pB, pC, pD, pE, pF, \
-	      ROT, rn)                                                        \
-	ROUND1024(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, pA, pB, pC, pD, pE, \
-		  pF, ROT, rn)                                                \
-
-#define I1024(R)                                                \
-	do {                                                    \
-		/* inject the key schedule value */             \
-		X00 += ks[((R) + 1) % 17];                      \
-		X01 += ks[((R) + 2) % 17];                      \
-		X02 += ks[((R) + 3) % 17];                      \
-		X03 += ks[((R) + 4) % 17];                      \
-		X04 += ks[((R) + 5) % 17];                      \
-		X05 += ks[((R) + 6) % 17];                      \
-		X06 += ks[((R) + 7) % 17];                      \
-		X07 += ks[((R) + 8) % 17];                      \
-		X08 += ks[((R) + 9) % 17];                      \
-		X09 += ks[((R) + 10) % 17];                     \
-		X10 += ks[((R) + 11) % 17];                     \
-		X11 += ks[((R) + 12) % 17];                     \
-		X12 += ks[((R) + 13) % 17];                     \
-		X13 += ks[((R) + 14) % 17] + ts[((R) + 1) % 3]; \
-		X14 += ks[((R) + 15) % 17] + ts[((R) + 2) % 3]; \
-		X15 += ks[((R) + 16) % 17] + (R) + 1;           \
-	} while (0)
-#else /* looping version */
-#define R1024(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, pA, pB, pC, pD, pE, pF, \
-	      ROT, rn)                                                        \
-	ROUND1024(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, pA, pB, pC, pD, pE, \
-		  pF, ROT, rn)                                                \
-
-#define I1024(R)                                                        \
-	do {                                                            \
-		/* inject the key schedule value */                     \
-		X00 += ks[r + (R) + 0];                                 \
-		X01 += ks[r + (R) + 1];                                 \
-		X02 += ks[r + (R) + 2];                                 \
-		X03 += ks[r + (R) + 3];                                 \
-		X04 += ks[r + (R) + 4];                                 \
-		X05 += ks[r + (R) + 5];                                 \
-		X06 += ks[r + (R) + 6];                                 \
-		X07 += ks[r + (R) + 7];                                 \
-		X08 += ks[r + (R) + 8];                                 \
-		X09 += ks[r + (R) + 9];                                 \
-		X10 += ks[r + (R) + 10];                                \
-		X11 += ks[r + (R) + 11];                                \
-		X12 += ks[r + (R) + 12];                                \
-		X13 += ks[r + (R) + 13] + ts[r + (R) + 0];              \
-		X14 += ks[r + (R) + 14] + ts[r + (R) + 1];              \
-		X15 += ks[r + (R) + 15] + r + (R);                      \
-		/* rotate key schedule */                               \
-		ks[r + (R) + 16] = ks[r + (R) - 1];                     \
-		ts[r + (R) + 2] = ts[r + (R) - 1];                      \
-	} while (0)
-
-#endif
-#define R1024_8_ROUNDS(R)                                                 \
-	do {                                                              \
-		R1024(00, 01, 02, 03, 04, 05, 06, 07, 08, 09, 10, 11, 12, \
-		      13, 14, 15, R1024_0, 8 * (R) + 1);                  \
-		R1024(00, 09, 02, 13, 06, 11, 04, 15, 10, 07, 12, 03, 14, \
-		      05, 08, 01, R1024_1, 8 * (R) + 2);                  \
-		R1024(00, 07, 02, 05, 04, 03, 06, 01, 12, 15, 14, 13, 08, \
-		      11, 10, 09, R1024_2, 8 * (R) + 3);                  \
-		R1024(00, 15, 02, 11, 06, 13, 04, 09, 14, 01, 08, 05, 10, \
-		      03, 12, 07, R1024_3, 8 * (R) + 4);                  \
-		I1024(2 * (R));                                           \
-		R1024(00, 01, 02, 03, 04, 05, 06, 07, 08, 09, 10, 11, 12, \
-		      13, 14, 15, R1024_4, 8 * (R) + 5);                  \
-		R1024(00, 09, 02, 13, 06, 11, 04, 15, 10, 07, 12, 03, 14, \
-		      05, 08, 01, R1024_5, 8 * (R) + 6);                  \
-		R1024(00, 07, 02, 05, 04, 03, 06, 01, 12, 15, 14, 13, 08, \
-		      11, 10, 09, R1024_6, 8 * (R) + 7);                  \
-		R1024(00, 15, 02, 11, 06, 13, 04, 09, 14, 01, 08, 05, 10, \
-		      03, 12, 07, R1024_7, 8 * (R) + 8);                  \
-		I1024(2 * (R) + 1);                                       \
-	} while (0)
-
-#define R1024_UNROLL_R(NN)                              \
-		((SKEIN_UNROLL_1024 == 0 &&             \
-		SKEIN_1024_ROUNDS_TOTAL / 8 > (NN)) ||  \
-		(SKEIN_UNROLL_1024 > (NN)))
-
-#if  (SKEIN_UNROLL_1024 > 14)
-#error  "need more unrolling in Skein_1024_Process_Block"
-#endif
-#endif
-
 /*****************************  SKEIN_256 ******************************/
 #if !(SKEIN_USE_ASM & 256)
 void skein_256_process_block(struct skein_256_ctx *ctx, const u8 *blk_ptr,
diff --git a/drivers/staging/skein/skein_block.h b/drivers/staging/skein/skein_block.h
index ec1baea25c9e..b3bb3d24273b 100644
--- a/drivers/staging/skein/skein_block.h
+++ b/drivers/staging/skein/skein_block.h
@@ -14,6 +14,329 @@
 
 #include "skein_base.h" /* get the Skein API definitions   */
 
+#ifndef SKEIN_USE_ASM
+#define SKEIN_USE_ASM   (0) /* default is all C code (no ASM) */
+#endif
+
+#ifndef SKEIN_LOOP
+#define SKEIN_LOOP 001 /* default: unroll 256 and 512, but not 1024 */
+#endif
+
+#define BLK_BITS        (WCNT * 64) /* some useful definitions for code here */
+#define KW_TWK_BASE     (0)
+#define KW_KEY_BASE     (3)
+#define ks              (kw + KW_KEY_BASE)
+#define ts              (kw + KW_TWK_BASE)
+
+#ifdef SKEIN_DEBUG
+#define debug_save_tweak(ctx)       \
+{                                   \
+	ctx->h.tweak[0] = ts[0];    \
+	ctx->h.tweak[1] = ts[1];    \
+}
+#else
+#define debug_save_tweak(ctx)
+#endif
+
+#if !(SKEIN_USE_ASM & 256)
+#undef  RCNT
+#define RCNT (SKEIN_256_ROUNDS_TOTAL / 8)
+#ifdef SKEIN_LOOP /* configure how much to unroll the loop */
+#define SKEIN_UNROLL_256 (((SKEIN_LOOP) / 100) % 10)
+#else
+#define SKEIN_UNROLL_256 (0)
+#endif
+
+#if SKEIN_UNROLL_256
+#if (RCNT % SKEIN_UNROLL_256)
+#error "Invalid SKEIN_UNROLL_256" /* sanity check on unroll count */
+#endif
+#endif
+#define ROUND256(p0, p1, p2, p3, ROT, r_num)         \
+	do {                                         \
+		X##p0 += X##p1;                      \
+		X##p1 = rol64(X##p1, ROT##_0);       \
+		X##p1 ^= X##p0;                      \
+		X##p2 += X##p3;                      \
+		X##p3 = rol64(X##p3, ROT##_1);       \
+		X##p3 ^= X##p2;                      \
+	} while (0)
+
+#if SKEIN_UNROLL_256 == 0
+#define R256(p0, p1, p2, p3, ROT, r_num) /* fully unrolled */ \
+	ROUND256(p0, p1, p2, p3, ROT, r_num)
+
+#define I256(R)                                                         \
+	do {                                                            \
+		/* inject the key schedule value */                     \
+		X0   += ks[((R) + 1) % 5];                              \
+		X1   += ks[((R) + 2) % 5] + ts[((R) + 1) % 3];          \
+		X2   += ks[((R) + 3) % 5] + ts[((R) + 2) % 3];          \
+		X3   += ks[((R) + 4) % 5] + (R) + 1;                    \
+	} while (0)
+#else
+/* looping version */
+#define R256(p0, p1, p2, p3, ROT, r_num) ROUND256(p0, p1, p2, p3, ROT, r_num)
+
+#define I256(R)                                         \
+	do {                                            \
+		/* inject the key schedule value */     \
+		X0 += ks[r + (R) + 0];                  \
+		X1 += ks[r + (R) + 1] + ts[r + (R) + 0];\
+		X2 += ks[r + (R) + 2] + ts[r + (R) + 1];\
+		X3 += ks[r + (R) + 3] + r + (R);        \
+		/* rotate key schedule */               \
+		ks[r + (R) + 4] = ks[r + (R) - 1];      \
+		ts[r + (R) + 2] = ts[r + (R) - 1];      \
+	} while (0)
+#endif
+#define R256_8_ROUNDS(R)                                \
+	do {                                            \
+		R256(0, 1, 2, 3, R_256_0, 8 * (R) + 1); \
+		R256(0, 3, 2, 1, R_256_1, 8 * (R) + 2); \
+		R256(0, 1, 2, 3, R_256_2, 8 * (R) + 3); \
+		R256(0, 3, 2, 1, R_256_3, 8 * (R) + 4); \
+		I256(2 * (R));                          \
+		R256(0, 1, 2, 3, R_256_4, 8 * (R) + 5); \
+		R256(0, 3, 2, 1, R_256_5, 8 * (R) + 6); \
+		R256(0, 1, 2, 3, R_256_6, 8 * (R) + 7); \
+		R256(0, 3, 2, 1, R_256_7, 8 * (R) + 8); \
+		I256(2 * (R) + 1);                      \
+	} while (0)
+
+#define R256_UNROLL_R(NN)                     \
+	((SKEIN_UNROLL_256 == 0 &&            \
+	SKEIN_256_ROUNDS_TOTAL / 8 > (NN)) || \
+	(SKEIN_UNROLL_256 > (NN)))
+
+#if  (SKEIN_UNROLL_256 > 14)
+#error  "need more unrolling in skein_256_process_block"
+#endif
+#endif
+
+#if !(SKEIN_USE_ASM & 512)
+#undef  RCNT
+#define RCNT  (SKEIN_512_ROUNDS_TOTAL / 8)
+
+#ifdef SKEIN_LOOP /* configure how much to unroll the loop */
+#define SKEIN_UNROLL_512 (((SKEIN_LOOP) / 10) % 10)
+#else
+#define SKEIN_UNROLL_512 (0)
+#endif
+
+#if SKEIN_UNROLL_512
+#if (RCNT % SKEIN_UNROLL_512)
+#error "Invalid SKEIN_UNROLL_512" /* sanity check on unroll count */
+#endif
+#endif
+#define ROUND512(p0, p1, p2, p3, p4, p5, p6, p7, ROT, r_num)    \
+	do {                                                    \
+		X##p0 += X##p1;                                 \
+		X##p1 = rol64(X##p1, ROT##_0);                  \
+		X##p1 ^= X##p0;                                 \
+		X##p2 += X##p3;                                 \
+		X##p3 = rol64(X##p3, ROT##_1);                  \
+		X##p3 ^= X##p2;                                 \
+		X##p4 += X##p5;                                 \
+		X##p5 = rol64(X##p5, ROT##_2);                  \
+		X##p5 ^= X##p4;                                 \
+		X##p6 += X##p7;                                 \
+		X##p7 = rol64(X##p7, ROT##_3);			\
+		X##p7 ^= X##p6;                                 \
+	} while (0)
+
+#if SKEIN_UNROLL_512 == 0
+#define R512(p0, p1, p2, p3, p4, p5, p6, p7, ROT, r_num) /* unrolled */ \
+	ROUND512(p0, p1, p2, p3, p4, p5, p6, p7, ROT, r_num)
+
+#define I512(R)                                                         \
+	do {                                                            \
+		/* inject the key schedule value */                     \
+		X0   += ks[((R) + 1) % 9];                              \
+		X1   += ks[((R) + 2) % 9];                              \
+		X2   += ks[((R) + 3) % 9];                              \
+		X3   += ks[((R) + 4) % 9];                              \
+		X4   += ks[((R) + 5) % 9];                              \
+		X5   += ks[((R) + 6) % 9] + ts[((R) + 1) % 3];          \
+		X6   += ks[((R) + 7) % 9] + ts[((R) + 2) % 3];          \
+		X7   += ks[((R) + 8) % 9] + (R) + 1;                    \
+	} while (0)
+
+#else /* looping version */
+#define R512(p0, p1, p2, p3, p4, p5, p6, p7, ROT, r_num)                 \
+	ROUND512(p0, p1, p2, p3, p4, p5, p6, p7, ROT, r_num)             \
+
+#define I512(R)                                                         \
+	do {                                                            \
+		/* inject the key schedule value */                     \
+		X0   += ks[r + (R) + 0];                                \
+		X1   += ks[r + (R) + 1];                                \
+		X2   += ks[r + (R) + 2];                                \
+		X3   += ks[r + (R) + 3];                                \
+		X4   += ks[r + (R) + 4];                                \
+		X5   += ks[r + (R) + 5] + ts[r + (R) + 0];              \
+		X6   += ks[r + (R) + 6] + ts[r + (R) + 1];              \
+		X7   += ks[r + (R) + 7] + r + (R);                      \
+		/* rotate key schedule */                               \
+		ks[r + (R) + 8] = ks[r + (R) - 1];                      \
+		ts[r + (R) + 2] = ts[r + (R) - 1];                      \
+	} while (0)
+#endif /* end of looped code definitions */
+#define R512_8_ROUNDS(R)  /* do 8 full rounds */                        \
+	do {                                                            \
+		R512(0, 1, 2, 3, 4, 5, 6, 7, R_512_0, 8 * (R) + 1);     \
+		R512(2, 1, 4, 7, 6, 5, 0, 3, R_512_1, 8 * (R) + 2);     \
+		R512(4, 1, 6, 3, 0, 5, 2, 7, R_512_2, 8 * (R) + 3);     \
+		R512(6, 1, 0, 7, 2, 5, 4, 3, R_512_3, 8 * (R) + 4);     \
+		I512(2 * (R));                                          \
+		R512(0, 1, 2, 3, 4, 5, 6, 7, R_512_4, 8 * (R) + 5);     \
+		R512(2, 1, 4, 7, 6, 5, 0, 3, R_512_5, 8 * (R) + 6);     \
+		R512(4, 1, 6, 3, 0, 5, 2, 7, R_512_6, 8 * (R) + 7);     \
+		R512(6, 1, 0, 7, 2, 5, 4, 3, R_512_7, 8 * (R) + 8);     \
+		I512(2 * (R) + 1); /* and key injection */              \
+	} while (0)
+#define R512_UNROLL_R(NN)                             \
+		((SKEIN_UNROLL_512 == 0 &&            \
+		SKEIN_512_ROUNDS_TOTAL / 8 > (NN)) || \
+		(SKEIN_UNROLL_512 > (NN)))
+
+#if  (SKEIN_UNROLL_512 > 14)
+#error  "need more unrolling in skein_512_process_block"
+#endif
+#endif
+
+#if !(SKEIN_USE_ASM & 1024)
+#undef  RCNT
+#define RCNT  (SKEIN_1024_ROUNDS_TOTAL / 8)
+#ifdef SKEIN_LOOP /* configure how much to unroll the loop */
+#define SKEIN_UNROLL_1024 ((SKEIN_LOOP) % 10)
+#else
+#define SKEIN_UNROLL_1024 (0)
+#endif
+
+#if (SKEIN_UNROLL_1024 != 0)
+#if (RCNT % SKEIN_UNROLL_1024)
+#error "Invalid SKEIN_UNROLL_1024" /* sanity check on unroll count */
+#endif
+#endif
+#define ROUND1024(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, pA, pB, pC, pD, pE, \
+		  pF, ROT, r_num)                                             \
+	do {                                                                  \
+		X##p0 += X##p1;                                               \
+		X##p1 = rol64(X##p1, ROT##_0);                                \
+		X##p1 ^= X##p0;                                               \
+		X##p2 += X##p3;                                               \
+		X##p3 = rol64(X##p3, ROT##_1);                                \
+		X##p3 ^= X##p2;                                               \
+		X##p4 += X##p5;                                               \
+		X##p5 = rol64(X##p5, ROT##_2);                                \
+		X##p5 ^= X##p4;                                               \
+		X##p6 += X##p7;                                               \
+		X##p7 = rol64(X##p7, ROT##_3);                                \
+		X##p7 ^= X##p6;                                               \
+		X##p8 += X##p9;                                               \
+		X##p9 = rol64(X##p9, ROT##_4);                                \
+		X##p9 ^= X##p8;                                               \
+		X##pA += X##pB;                                               \
+		X##pB = rol64(X##pB, ROT##_5);                                \
+		X##pB ^= X##pA;                                               \
+		X##pC += X##pD;                                               \
+		X##pD = rol64(X##pD, ROT##_6);                                \
+		X##pD ^= X##pC;                                               \
+		X##pE += X##pF;                                               \
+		X##pF = rol64(X##pF, ROT##_7);                                \
+		X##pF ^= X##pE;                                               \
+	} while (0)
+
+#if SKEIN_UNROLL_1024 == 0
+#define R1024(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, pA, pB, pC, pD, pE, pF, \
+	      ROT, rn)                                                        \
+	ROUND1024(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, pA, pB, pC, pD, pE, \
+		  pF, ROT, rn)                                                \
+
+#define I1024(R)                                                \
+	do {                                                    \
+		/* inject the key schedule value */             \
+		X00 += ks[((R) + 1) % 17];                      \
+		X01 += ks[((R) + 2) % 17];                      \
+		X02 += ks[((R) + 3) % 17];                      \
+		X03 += ks[((R) + 4) % 17];                      \
+		X04 += ks[((R) + 5) % 17];                      \
+		X05 += ks[((R) + 6) % 17];                      \
+		X06 += ks[((R) + 7) % 17];                      \
+		X07 += ks[((R) + 8) % 17];                      \
+		X08 += ks[((R) + 9) % 17];                      \
+		X09 += ks[((R) + 10) % 17];                     \
+		X10 += ks[((R) + 11) % 17];                     \
+		X11 += ks[((R) + 12) % 17];                     \
+		X12 += ks[((R) + 13) % 17];                     \
+		X13 += ks[((R) + 14) % 17] + ts[((R) + 1) % 3]; \
+		X14 += ks[((R) + 15) % 17] + ts[((R) + 2) % 3]; \
+		X15 += ks[((R) + 16) % 17] + (R) + 1;           \
+	} while (0)
+#else /* looping version */
+#define R1024(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, pA, pB, pC, pD, pE, pF, \
+	      ROT, rn)                                                        \
+	ROUND1024(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, pA, pB, pC, pD, pE, \
+		  pF, ROT, rn)                                                \
+
+#define I1024(R)                                                        \
+	do {                                                            \
+		/* inject the key schedule value */                     \
+		X00 += ks[r + (R) + 0];                                 \
+		X01 += ks[r + (R) + 1];                                 \
+		X02 += ks[r + (R) + 2];                                 \
+		X03 += ks[r + (R) + 3];                                 \
+		X04 += ks[r + (R) + 4];                                 \
+		X05 += ks[r + (R) + 5];                                 \
+		X06 += ks[r + (R) + 6];                                 \
+		X07 += ks[r + (R) + 7];                                 \
+		X08 += ks[r + (R) + 8];                                 \
+		X09 += ks[r + (R) + 9];                                 \
+		X10 += ks[r + (R) + 10];                                \
+		X11 += ks[r + (R) + 11];                                \
+		X12 += ks[r + (R) + 12];                                \
+		X13 += ks[r + (R) + 13] + ts[r + (R) + 0];              \
+		X14 += ks[r + (R) + 14] + ts[r + (R) + 1];              \
+		X15 += ks[r + (R) + 15] + r + (R);                      \
+		/* rotate key schedule */                               \
+		ks[r + (R) + 16] = ks[r + (R) - 1];                     \
+		ts[r + (R) + 2] = ts[r + (R) - 1];                      \
+	} while (0)
+
+#endif
+#define R1024_8_ROUNDS(R)                                                 \
+	do {                                                              \
+		R1024(00, 01, 02, 03, 04, 05, 06, 07, 08, 09, 10, 11, 12, \
+		      13, 14, 15, R1024_0, 8 * (R) + 1);                  \
+		R1024(00, 09, 02, 13, 06, 11, 04, 15, 10, 07, 12, 03, 14, \
+		      05, 08, 01, R1024_1, 8 * (R) + 2);                  \
+		R1024(00, 07, 02, 05, 04, 03, 06, 01, 12, 15, 14, 13, 08, \
+		      11, 10, 09, R1024_2, 8 * (R) + 3);                  \
+		R1024(00, 15, 02, 11, 06, 13, 04, 09, 14, 01, 08, 05, 10, \
+		      03, 12, 07, R1024_3, 8 * (R) + 4);                  \
+		I1024(2 * (R));                                           \
+		R1024(00, 01, 02, 03, 04, 05, 06, 07, 08, 09, 10, 11, 12, \
+		      13, 14, 15, R1024_4, 8 * (R) + 5);                  \
+		R1024(00, 09, 02, 13, 06, 11, 04, 15, 10, 07, 12, 03, 14, \
+		      05, 08, 01, R1024_5, 8 * (R) + 6);                  \
+		R1024(00, 07, 02, 05, 04, 03, 06, 01, 12, 15, 14, 13, 08, \
+		      11, 10, 09, R1024_6, 8 * (R) + 7);                  \
+		R1024(00, 15, 02, 11, 06, 13, 04, 09, 14, 01, 08, 05, 10, \
+		      03, 12, 07, R1024_7, 8 * (R) + 8);                  \
+		I1024(2 * (R) + 1);                                       \
+	} while (0)
+
+#define R1024_UNROLL_R(NN)                              \
+		((SKEIN_UNROLL_1024 == 0 &&             \
+		SKEIN_1024_ROUNDS_TOTAL / 8 > (NN)) ||  \
+		(SKEIN_UNROLL_1024 > (NN)))
+
+#if  (SKEIN_UNROLL_1024 > 14)
+#error  "need more unrolling in Skein_1024_Process_Block"
+#endif
+#endif
+
 void skein_256_process_block(struct skein_256_ctx *ctx, const u8 *blk_ptr,
 			     size_t blk_cnt, size_t byte_cnt_add);
 void skein_512_process_block(struct skein_512_ctx *ctx, const u8 *blk_ptr,
diff --git a/drivers/staging/speakup/spk_ttyio.c b/drivers/staging/speakup/spk_ttyio.c
index fe340b07c482..4d7d8f2f66ea 100644
--- a/drivers/staging/speakup/spk_ttyio.c
+++ b/drivers/staging/speakup/spk_ttyio.c
@@ -7,11 +7,6 @@
 #include "spk_types.h"
 #include "spk_priv.h"
 
-#define DEV_PREFIX_LP "lp"
-
-static const char * const lp_supported[] = { "acntsa", "bns", "dummy",
-	"txprt" };
-
 struct spk_ldisc_data {
 	char buf;
 	struct semaphore sem;
@@ -20,6 +15,11 @@ struct spk_ldisc_data {
 
 static struct spk_synth *spk_ttyio_synth;
 static struct tty_struct *speakup_tty;
+/* mutex to protect against speakup_tty disappearing from underneath us while
+ * we are using it. this can happen when the device physically unplugged,
+ * while in use. it also serialises access to speakup_tty.
+ */
+static DEFINE_MUTEX(speakup_tty_mutex);
 
 static int ser_to_dev(int ser, dev_t *dev_no)
 {
@@ -36,24 +36,8 @@ static int get_dev_to_use(struct spk_synth *synth, dev_t *dev_no)
 {
 	/* use ser only when dev is not specified */
 	if (strcmp(synth->dev_name, SYNTH_DEFAULT_DEV) ||
-	    synth->ser == SYNTH_DEFAULT_SER) {
-		/* for /dev/lp* check if synth is supported */
-		if (strncmp(synth->dev_name, DEV_PREFIX_LP,
-		    strlen(DEV_PREFIX_LP)) == 0)
-			if (match_string(lp_supported, ARRAY_SIZE(lp_supported),
-			    synth->name) < 0)  {
-				int i;
-
-				pr_err("speakup: lp* is only supported on:");
-				for (i = 0; i < ARRAY_SIZE(lp_supported); i++)
-					pr_cont(" %s", lp_supported[i]);
-				pr_cont("\n");
-
-				return -ENOTSUPP;
-			}
-
+	    synth->ser == SYNTH_DEFAULT_SER)
 		return tty_dev_name_to_number(synth->dev_name, dev_no);
-	}
 
 	return ser_to_dev(synth->ser, dev_no);
 }
@@ -81,8 +65,10 @@ static int spk_ttyio_ldisc_open(struct tty_struct *tty)
 
 static void spk_ttyio_ldisc_close(struct tty_struct *tty)
 {
+	mutex_lock(&speakup_tty_mutex);
 	kfree(speakup_tty->disc_data);
 	speakup_tty = NULL;
+	mutex_unlock(&speakup_tty_mutex);
 }
 
 static int spk_ttyio_receive_buf2(struct tty_struct *tty,
@@ -158,7 +144,7 @@ static int spk_ttyio_initialise_ldisc(struct spk_synth *synth)
 	if (ret)
 		return ret;
 
-	tty = tty_open_by_driver(dev, NULL, NULL);
+	tty = tty_kopen(dev);
 	if (IS_ERR(tty))
 		return PTR_ERR(tty);
 
@@ -210,9 +196,11 @@ void spk_ttyio_unregister_ldisc(void)
 
 static int spk_ttyio_out(struct spk_synth *in_synth, const char ch)
 {
+	mutex_lock(&speakup_tty_mutex);
 	if (in_synth->alive && speakup_tty && speakup_tty->ops->write) {
 		int ret = speakup_tty->ops->write(speakup_tty, &ch, 1);
 
+		mutex_unlock(&speakup_tty_mutex);
 		if (ret == 0)
 			/* No room */
 			return 0;
@@ -228,17 +216,50 @@ static int spk_ttyio_out(struct spk_synth *in_synth, const char ch)
 		}
 		return 1;
 	}
+
+	mutex_unlock(&speakup_tty_mutex);
+	return 0;
+}
+
+static int check_tty(struct tty_struct *tty)
+{
+	if (!tty) {
+		pr_warn("%s: I/O error, deactivating speakup\n",
+			spk_ttyio_synth->long_name);
+		/* No synth any more, so nobody will restart TTYs, and we thus
+		 * need to do it ourselves.  Now that there is no synth we can
+		 * let application flood anyway
+		 */
+		spk_ttyio_synth->alive = 0;
+		speakup_start_ttys();
+		return 1;
+	}
+
 	return 0;
 }
 
 static void spk_ttyio_send_xchar(char ch)
 {
+	mutex_lock(&speakup_tty_mutex);
+	if (check_tty(speakup_tty)) {
+		mutex_unlock(&speakup_tty_mutex);
+		return;
+	}
+
 	speakup_tty->ops->send_xchar(speakup_tty, ch);
+	mutex_unlock(&speakup_tty_mutex);
 }
 
 static void spk_ttyio_tiocmset(unsigned int set, unsigned int clear)
 {
+	mutex_lock(&speakup_tty_mutex);
+	if (check_tty(speakup_tty)) {
+		mutex_unlock(&speakup_tty_mutex);
+		return;
+	}
+
 	speakup_tty->ops->tiocmset(speakup_tty, set, clear);
+	mutex_unlock(&speakup_tty_mutex);
 }
 
 static unsigned char ttyio_in(int timeout)
@@ -278,8 +299,16 @@ static unsigned char spk_ttyio_in_nowait(void)
 
 static void spk_ttyio_flush_buffer(void)
 {
+	mutex_lock(&speakup_tty_mutex);
+	if (check_tty(speakup_tty)) {
+		mutex_unlock(&speakup_tty_mutex);
+		return;
+	}
+
 	if (speakup_tty->ops->flush_buffer)
 		speakup_tty->ops->flush_buffer(speakup_tty);
+
+	mutex_unlock(&speakup_tty_mutex);
 }
 
 int spk_ttyio_synth_probe(struct spk_synth *synth)
@@ -308,7 +337,7 @@ void spk_ttyio_release(void)
 
 	tty_ldisc_flush(speakup_tty);
 	tty_unlock(speakup_tty);
-	tty_release_struct(speakup_tty, speakup_tty->index);
+	tty_kclose(speakup_tty);
 }
 EXPORT_SYMBOL_GPL(spk_ttyio_release);
 
diff --git a/drivers/staging/typec/fusb302/Kconfig b/drivers/staging/typec/fusb302/Kconfig
index fce099ff39fe..48a4f2fcee03 100644
--- a/drivers/staging/typec/fusb302/Kconfig
+++ b/drivers/staging/typec/fusb302/Kconfig
@@ -1,6 +1,6 @@
 config TYPEC_FUSB302
 	tristate "Fairchild FUSB302 Type-C chip driver"
-	depends on I2C
+	depends on I2C && POWER_SUPPLY
 	help
 	  The Fairchild FUSB302 Type-C chip driver that works with
 	  Type-C Port Controller Manager to provide USB PD and USB
diff --git a/drivers/staging/typec/fusb302/TODO b/drivers/staging/typec/fusb302/TODO
index 4933a1d92c32..19b466eb585d 100644
--- a/drivers/staging/typec/fusb302/TODO
+++ b/drivers/staging/typec/fusb302/TODO
@@ -4,3 +4,7 @@ fusb302:
 - Find a non-hacky way to coordinate between PM and I2C access
 - Documentation? The FUSB302 datasheet provides information on the chip to help
   understand the code. But it may still be helpful to have a documentation.
+- We may want to replace the  "fcs,max-snk-microvolt", "fcs,max-snk-microamp",
+  "fcs,max-snk-microwatt" and "fcs,operating-snk-microwatt" device(tree)
+  properties with properties which are part of a generic type-c controller
+  devicetree binding.
diff --git a/drivers/staging/typec/fusb302/fusb302.c b/drivers/staging/typec/fusb302/fusb302.c
index 03a3809d18f0..fc6a3cf74eb3 100644
--- a/drivers/staging/typec/fusb302/fusb302.c
+++ b/drivers/staging/typec/fusb302/fusb302.c
@@ -17,6 +17,7 @@
 #include <linux/debugfs.h>
 #include <linux/delay.h>
 #include <linux/errno.h>
+#include <linux/extcon.h>
 #include <linux/gpio.h>
 #include <linux/i2c.h>
 #include <linux/interrupt.h>
@@ -27,6 +28,7 @@
 #include <linux/of_device.h>
 #include <linux/of_gpio.h>
 #include <linux/pinctrl/consumer.h>
+#include <linux/power_supply.h>
 #include <linux/proc_fs.h>
 #include <linux/regulator/consumer.h>
 #include <linux/sched/clock.h>
@@ -90,11 +92,13 @@ struct fusb302_chip {
 	struct i2c_client *i2c_client;
 	struct tcpm_port *tcpm_port;
 	struct tcpc_dev tcpc_dev;
+	struct tcpc_config tcpc_config;
 
 	struct regulator *vbus;
 
 	int gpio_int_n;
 	int gpio_int_n_irq;
+	struct extcon_dev *extcon;
 
 	struct workqueue_struct *wq;
 	struct delayed_work bc_lvl_handler;
@@ -105,6 +109,11 @@ struct fusb302_chip {
 	/* lock for sharing chip states */
 	struct mutex lock;
 
+	/* psy + psy status */
+	struct power_supply *psy;
+	u32 current_limit;
+	u32 supply_voltage;
+
 	/* chip status */
 	enum toggling_mode toggling_mode;
 	enum src_current_status src_current_status;
@@ -515,6 +524,38 @@ static int tcpm_get_vbus(struct tcpc_dev *dev)
 	return ret;
 }
 
+static int tcpm_get_current_limit(struct tcpc_dev *dev)
+{
+	struct fusb302_chip *chip = container_of(dev, struct fusb302_chip,
+						 tcpc_dev);
+	int current_limit = 0;
+	unsigned long timeout;
+
+	if (!chip->extcon)
+		return 0;
+
+	/*
+	 * USB2 Charger detection may still be in progress when we get here,
+	 * this can take upto 600ms, wait 800ms max.
+	 */
+	timeout = jiffies + msecs_to_jiffies(800);
+	do {
+		if (extcon_get_state(chip->extcon, EXTCON_CHG_USB_SDP) == 1)
+			current_limit = 500;
+
+		if (extcon_get_state(chip->extcon, EXTCON_CHG_USB_CDP) == 1 ||
+		    extcon_get_state(chip->extcon, EXTCON_CHG_USB_ACA) == 1)
+			current_limit = 1500;
+
+		if (extcon_get_state(chip->extcon, EXTCON_CHG_USB_DCP) == 1)
+			current_limit = 2000;
+
+		msleep(50);
+	} while (current_limit == 0 && time_before(jiffies, timeout));
+
+	return current_limit;
+}
+
 static int fusb302_set_cc_pull(struct fusb302_chip *chip,
 			       bool pull_up, bool pull_down)
 {
@@ -841,11 +882,13 @@ static int tcpm_set_vbus(struct tcpc_dev *dev, bool on, bool charge)
 		chip->vbus_on = on;
 		fusb302_log(chip, "vbus := %s", on ? "On" : "Off");
 	}
-	if (chip->charge_on == charge)
+	if (chip->charge_on == charge) {
 		fusb302_log(chip, "charge is already %s",
 			    charge ? "On" : "Off");
-	else
+	} else {
 		chip->charge_on = charge;
+		power_supply_changed(chip->psy);
+	}
 
 done:
 	mutex_unlock(&chip->lock);
@@ -861,6 +904,11 @@ static int tcpm_set_current_limit(struct tcpc_dev *dev, u32 max_ma, u32 mv)
 	fusb302_log(chip, "current limit: %d ma, %d mv (not implemented)",
 		    max_ma, mv);
 
+	chip->supply_voltage = mv;
+	chip->current_limit = max_ma;
+
+	power_supply_changed(chip->psy);
+
 	return 0;
 }
 
@@ -1187,9 +1235,9 @@ static const struct tcpc_config fusb302_tcpc_config = {
 	.nr_src_pdo = ARRAY_SIZE(src_pdo),
 	.snk_pdo = snk_pdo,
 	.nr_snk_pdo = ARRAY_SIZE(snk_pdo),
-	.max_snk_mv = 9000,
+	.max_snk_mv = 5000,
 	.max_snk_ma = 3000,
-	.max_snk_mw = 27000,
+	.max_snk_mw = 15000,
 	.operating_snk_mw = 2500,
 	.type = TYPEC_PORT_DRP,
 	.default_role = TYPEC_SINK,
@@ -1198,9 +1246,9 @@ static const struct tcpc_config fusb302_tcpc_config = {
 
 static void init_tcpc_dev(struct tcpc_dev *fusb302_tcpc_dev)
 {
-	fusb302_tcpc_dev->config = &fusb302_tcpc_config;
 	fusb302_tcpc_dev->init = tcpm_init;
 	fusb302_tcpc_dev->get_vbus = tcpm_get_vbus;
+	fusb302_tcpc_dev->get_current_limit = tcpm_get_current_limit;
 	fusb302_tcpc_dev->set_cc = tcpm_set_cc;
 	fusb302_tcpc_dev->get_cc = tcpm_get_cc;
 	fusb302_tcpc_dev->set_polarity = tcpm_set_polarity;
@@ -1646,6 +1694,43 @@ done:
 	return IRQ_HANDLED;
 }
 
+static int fusb302_psy_get_property(struct power_supply *psy,
+				    enum power_supply_property psp,
+				    union power_supply_propval *val)
+{
+	struct fusb302_chip *chip = power_supply_get_drvdata(psy);
+
+	switch (psp) {
+	case POWER_SUPPLY_PROP_ONLINE:
+		val->intval = chip->charge_on;
+		break;
+	case POWER_SUPPLY_PROP_VOLTAGE_NOW:
+		val->intval = chip->supply_voltage * 1000; /* mV -> µV */
+		break;
+	case POWER_SUPPLY_PROP_CURRENT_MAX:
+		val->intval = chip->current_limit * 1000; /* mA -> µA */
+		break;
+	default:
+		return -ENODATA;
+	}
+
+	return 0;
+}
+
+static enum power_supply_property fusb302_psy_properties[] = {
+	POWER_SUPPLY_PROP_ONLINE,
+	POWER_SUPPLY_PROP_VOLTAGE_NOW,
+	POWER_SUPPLY_PROP_CURRENT_MAX,
+};
+
+static const struct power_supply_desc fusb302_psy_desc = {
+	.name		= "fusb302-typec-source",
+	.type		= POWER_SUPPLY_TYPE_USB_TYPE_C,
+	.properties	= fusb302_psy_properties,
+	.num_properties	= ARRAY_SIZE(fusb302_psy_properties),
+	.get_property	= fusb302_psy_get_property,
+};
+
 static int init_gpio(struct fusb302_chip *chip)
 {
 	struct device_node *node;
@@ -1684,7 +1769,11 @@ static int fusb302_probe(struct i2c_client *client,
 {
 	struct fusb302_chip *chip;
 	struct i2c_adapter *adapter;
+	struct device *dev = &client->dev;
+	struct power_supply_config cfg = {};
+	const char *name;
 	int ret = 0;
+	u32 v;
 
 	adapter = to_i2c_adapter(client->dev.parent);
 	if (!i2c_check_functionality(adapter, I2C_FUNC_SMBUS_I2C_BLOCK)) {
@@ -1699,8 +1788,43 @@ static int fusb302_probe(struct i2c_client *client,
 	chip->i2c_client = client;
 	i2c_set_clientdata(client, chip);
 	chip->dev = &client->dev;
+	chip->tcpc_config = fusb302_tcpc_config;
+	chip->tcpc_dev.config = &chip->tcpc_config;
 	mutex_init(&chip->lock);
 
+	if (!device_property_read_u32(dev, "fcs,max-sink-microvolt", &v))
+		chip->tcpc_config.max_snk_mv = v / 1000;
+
+	if (!device_property_read_u32(dev, "fcs,max-sink-microamp", &v))
+		chip->tcpc_config.max_snk_ma = v / 1000;
+
+	if (!device_property_read_u32(dev, "fcs,max-sink-microwatt", &v))
+		chip->tcpc_config.max_snk_mw = v / 1000;
+
+	if (!device_property_read_u32(dev, "fcs,operating-sink-microwatt", &v))
+		chip->tcpc_config.operating_snk_mw = v / 1000;
+
+	/*
+	 * Devicetree platforms should get extcon via phandle (not yet
+	 * supported). On ACPI platforms, we get the name from a device prop.
+	 * This device prop is for kernel internal use only and is expected
+	 * to be set by the platform code which also registers the i2c client
+	 * for the fusb302.
+	 */
+	if (device_property_read_string(dev, "fcs,extcon-name", &name) == 0) {
+		chip->extcon = extcon_get_extcon_dev(name);
+		if (!chip->extcon)
+			return -EPROBE_DEFER;
+	}
+
+	cfg.drv_data = chip;
+	chip->psy = devm_power_supply_register(dev, &fusb302_psy_desc, &cfg);
+	if (IS_ERR(chip->psy)) {
+		ret = PTR_ERR(chip->psy);
+		dev_err(chip->dev, "Error registering power-supply: %d\n", ret);
+		return ret;
+	}
+
 	ret = fusb302_debugfs_init(chip);
 	if (ret < 0)
 		return ret;
@@ -1719,9 +1843,13 @@ static int fusb302_probe(struct i2c_client *client,
 		goto destroy_workqueue;
 	}
 
-	ret = init_gpio(chip);
-	if (ret < 0)
-		goto destroy_workqueue;
+	if (client->irq) {
+		chip->gpio_int_n_irq = client->irq;
+	} else {
+		ret = init_gpio(chip);
+		if (ret < 0)
+			goto destroy_workqueue;
+	}
 
 	chip->tcpm_port = tcpm_register_port(&client->dev, &chip->tcpc_dev);
 	if (IS_ERR(chip->tcpm_port)) {
diff --git a/drivers/staging/typec/pd.h b/drivers/staging/typec/pd.h
index 510ef7279900..30b32ad72acd 100644
--- a/drivers/staging/typec/pd.h
+++ b/drivers/staging/typec/pd.h
@@ -278,6 +278,8 @@ static inline unsigned int rdo_max_power(u32 rdo)
 #define PD_T_VCONN_SOURCE_ON	100
 #define PD_T_SINK_REQUEST	100	/* 100 ms minimum */
 #define PD_T_ERROR_RECOVERY	100	/* minimum 25 is insufficient */
+#define PD_T_SRCSWAPSTDBY      625     /* Maximum of 650ms */
+#define PD_T_NEWSRC            250     /* Maximum of 275ms */
 
 #define PD_T_DRP_TRY		100	/* 75 - 150 ms */
 #define PD_T_DRP_TRYWAIT	600	/* 400 - 800 ms */
diff --git a/drivers/staging/typec/tcpm.c b/drivers/staging/typec/tcpm.c
index 20eb4ebcf8c3..8af62e74d54c 100644
--- a/drivers/staging/typec/tcpm.c
+++ b/drivers/staging/typec/tcpm.c
@@ -17,6 +17,7 @@
 #include <linux/completion.h>
 #include <linux/debugfs.h>
 #include <linux/device.h>
+#include <linux/jiffies.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
@@ -89,9 +90,11 @@
 	S(PR_SWAP_START),			\
 	S(PR_SWAP_SRC_SNK_TRANSITION_OFF),	\
 	S(PR_SWAP_SRC_SNK_SOURCE_OFF),		\
+	S(PR_SWAP_SRC_SNK_SOURCE_OFF_CC_DEBOUNCED), \
 	S(PR_SWAP_SRC_SNK_SINK_ON),		\
 	S(PR_SWAP_SNK_SRC_SINK_OFF),		\
 	S(PR_SWAP_SNK_SRC_SOURCE_ON),		\
+	S(PR_SWAP_SNK_SRC_SOURCE_ON_VBUS_RAMPED_UP),    \
 						\
 	S(VCONN_SWAP_ACCEPT),			\
 	S(VCONN_SWAP_SEND),			\
@@ -104,10 +107,14 @@
 						\
 	S(SNK_TRY),				\
 	S(SNK_TRY_WAIT),			\
+	S(SNK_TRY_WAIT_DEBOUNCE),               \
+	S(SNK_TRY_WAIT_DEBOUNCE_CHECK_VBUS),    \
 	S(SRC_TRYWAIT),				\
+	S(SRC_TRYWAIT_DEBOUNCE),		\
 	S(SRC_TRYWAIT_UNATTACHED),		\
 						\
 	S(SRC_TRY),				\
+	S(SRC_TRY_WAIT),                        \
 	S(SRC_TRY_DEBOUNCE),			\
 	S(SNK_TRYWAIT),				\
 	S(SNK_TRYWAIT_DEBOUNCE),		\
@@ -115,7 +122,8 @@
 	S(BIST_RX),				\
 						\
 	S(ERROR_RECOVERY),			\
-	S(ERROR_RECOVERY_WAIT_OFF)
+	S(PORT_RESET),				\
+	S(PORT_RESET_WAIT_OFF)
 
 #define GENERATE_ENUM(e)	e
 #define GENERATE_STRING(s)	#s
@@ -196,6 +204,7 @@ struct tcpm_port {
 
 	bool attached;
 	bool connected;
+	enum typec_port_type port_type;
 	bool vbus_present;
 	bool vbus_never_low;
 	bool vbus_source;
@@ -230,6 +239,7 @@ struct tcpm_port {
 
 	struct mutex swap_lock;		/* swap command lock */
 	bool swap_pending;
+	bool non_pd_role_swap;
 	struct completion swap_complete;
 	int swap_status;
 
@@ -281,6 +291,9 @@ struct tcpm_port {
 	struct typec_altmode *partner_altmode[SVID_DISCOVERY_MAX];
 	struct typec_altmode *port_altmode[SVID_DISCOVERY_MAX];
 
+	/* Deadline in jiffies to exit src_try_wait state */
+	unsigned long max_wait;
+
 #ifdef CONFIG_DEBUG_FS
 	struct dentry *dentry;
 	struct mutex logbuffer_lock;	/* log buffer access lock */
@@ -325,19 +338,26 @@ struct pd_rx_event {
 	 (tcpm_cc_is_audio((port)->cc2) && tcpm_cc_is_open((port)->cc1)))
 
 #define tcpm_try_snk(port) \
-	((port)->try_snk_count == 0 && (port)->try_role == TYPEC_SINK)
+	((port)->try_snk_count == 0 && (port)->try_role == TYPEC_SINK && \
+	(port)->port_type == TYPEC_PORT_DRP)
 
 #define tcpm_try_src(port) \
-	((port)->try_src_count == 0 && (port)->try_role == TYPEC_SOURCE)
+	((port)->try_src_count == 0 && (port)->try_role == TYPEC_SOURCE && \
+	(port)->port_type == TYPEC_PORT_DRP)
 
 static enum tcpm_state tcpm_default_state(struct tcpm_port *port)
 {
-	if (port->try_role == TYPEC_SINK)
-		return SNK_UNATTACHED;
-	else if (port->try_role == TYPEC_SOURCE)
-		return SRC_UNATTACHED;
-	else if (port->tcpc->config->default_role == TYPEC_SINK)
+	if (port->port_type == TYPEC_PORT_DRP) {
+		if (port->try_role == TYPEC_SINK)
+			return SNK_UNATTACHED;
+		else if (port->try_role == TYPEC_SOURCE)
+			return SRC_UNATTACHED;
+		else if (port->tcpc->config->default_role == TYPEC_SINK)
+			return SNK_UNATTACHED;
+		/* Fall through to return SRC_UNATTACHED */
+	} else if (port->port_type == TYPEC_PORT_UFP) {
 		return SNK_UNATTACHED;
+	}
 	return SRC_UNATTACHED;
 }
 
@@ -369,6 +389,7 @@ static bool tcpm_log_full(struct tcpm_port *port)
 		(port->logbuffer_head + 1) % LOG_BUFFER_ENTRIES;
 }
 
+__printf(2, 0)
 static void _tcpm_log(struct tcpm_port *port, const char *fmt, va_list args)
 {
 	char tmpbuffer[LOG_BUFFER_ENTRY_SIZE];
@@ -415,6 +436,7 @@ abort:
 	mutex_unlock(&port->logbuffer_lock);
 }
 
+__printf(2, 3)
 static void tcpm_log(struct tcpm_port *port, const char *fmt, ...)
 {
 	va_list args;
@@ -430,6 +452,7 @@ static void tcpm_log(struct tcpm_port *port, const char *fmt, ...)
 	va_end(args);
 }
 
+__printf(2, 3)
 static void tcpm_log_force(struct tcpm_port *port, const char *fmt, ...)
 {
 	va_list args;
@@ -546,7 +569,9 @@ static void tcpm_debugfs_exit(struct tcpm_port *port)
 
 #else
 
+__printf(2, 3)
 static void tcpm_log(const struct tcpm_port *port, const char *fmt, ...) { }
+__printf(2, 3)
 static void tcpm_log_force(struct tcpm_port *port, const char *fmt, ...) { }
 static void tcpm_log_source_caps(struct tcpm_port *port) { }
 static int tcpm_debugfs_init(const struct tcpm_port *port) { return 0; }
@@ -660,7 +685,10 @@ static u32 tcpm_get_current_limit(struct tcpm_port *port)
 		break;
 	case TYPEC_CC_RP_DEF:
 	default:
-		limit = 0;
+		if (port->tcpc->get_current_limit)
+			limit = port->tcpc->get_current_limit(port->tcpc);
+		else
+			limit = 0;
 		break;
 	}
 
@@ -1015,8 +1043,7 @@ static int tcpm_pd_svdm(struct tcpm_port *port, const __le32 *payload, int cnt,
 			if (port->data_role == TYPEC_DEVICE &&
 			    port->nr_snk_vdo) {
 				for (i = 0; i <  port->nr_snk_vdo; i++)
-					response[i + 1]
-						= cpu_to_le32(port->snk_vdo[i]);
+					response[i + 1] = port->snk_vdo[i];
 				rlen = port->nr_snk_vdo + 1;
 			}
 			break;
@@ -1367,6 +1394,7 @@ static void tcpm_pd_ctrl_request(struct tcpm_port *port,
 				tcpm_set_current_limit(port,
 						       port->current_limit,
 						       port->supply_voltage);
+				port->explicit_contract = true;
 				tcpm_set_state(port, SNK_READY, 0);
 			} else {
 				/*
@@ -1377,7 +1405,7 @@ static void tcpm_pd_ctrl_request(struct tcpm_port *port,
 					       SNK_TRANSITION_SINK_VBUS, 0);
 			}
 			break;
-		case PR_SWAP_SRC_SNK_SOURCE_OFF:
+		case PR_SWAP_SRC_SNK_SOURCE_OFF_CC_DEBOUNCED:
 			tcpm_set_state(port, PR_SWAP_SRC_SNK_SINK_ON, 0);
 			break;
 		case PR_SWAP_SNK_SRC_SINK_OFF:
@@ -1451,7 +1479,7 @@ static void tcpm_pd_ctrl_request(struct tcpm_port *port,
 		tcpm_set_state(port, SOFT_RESET, 0);
 		break;
 	case PD_CTRL_DR_SWAP:
-		if (port->typec_caps.type != TYPEC_PORT_DRP) {
+		if (port->port_type != TYPEC_PORT_DRP) {
 			tcpm_queue_message(port, PD_MSG_CTRL_REJECT);
 			break;
 		}
@@ -1471,7 +1499,7 @@ static void tcpm_pd_ctrl_request(struct tcpm_port *port,
 		}
 		break;
 	case PD_CTRL_PR_SWAP:
-		if (port->typec_caps.type != TYPEC_PORT_DRP) {
+		if (port->port_type != TYPEC_PORT_DRP) {
 			tcpm_queue_message(port, PD_MSG_CTRL_REJECT);
 			break;
 		}
@@ -1846,7 +1874,7 @@ static bool tcpm_start_drp_toggling(struct tcpm_port *port)
 	int ret;
 
 	if (port->tcpc->start_drp_toggling &&
-	    port->typec_caps.type == TYPEC_PORT_DRP) {
+	    port->port_type == TYPEC_PORT_DRP) {
 		tcpm_log_force(port, "Start DRP toggling");
 		ret = port->tcpc->start_drp_toggling(port->tcpc,
 						     tcpm_rp_cc(port));
@@ -2099,10 +2127,16 @@ static inline enum tcpm_state ready_state(struct tcpm_port *port)
 
 static inline enum tcpm_state unattached_state(struct tcpm_port *port)
 {
-	if (port->pwr_role == TYPEC_SOURCE)
+	if (port->port_type == TYPEC_PORT_DRP) {
+		if (port->pwr_role == TYPEC_SOURCE)
+			return SRC_UNATTACHED;
+		else
+			return SNK_UNATTACHED;
+	} else if (port->port_type == TYPEC_PORT_DFP) {
 		return SRC_UNATTACHED;
-	else
-		return SNK_UNATTACHED;
+	}
+
+	return SNK_UNATTACHED;
 }
 
 static void tcpm_check_send_discover(struct tcpm_port *port)
@@ -2119,13 +2153,29 @@ static void tcpm_swap_complete(struct tcpm_port *port, int result)
 	if (port->swap_pending) {
 		port->swap_status = result;
 		port->swap_pending = false;
+		port->non_pd_role_swap = false;
 		complete(&port->swap_complete);
 	}
 }
 
+static enum typec_pwr_opmode tcpm_get_pwr_opmode(enum typec_cc_status cc)
+{
+	switch (cc) {
+	case TYPEC_CC_RP_1_5:
+		return TYPEC_PWR_MODE_1_5A;
+	case TYPEC_CC_RP_3_0:
+		return TYPEC_PWR_MODE_3_0A;
+	case TYPEC_CC_RP_DEF:
+	default:
+		return TYPEC_PWR_MODE_USB;
+	}
+}
+
 static void run_state_machine(struct tcpm_port *port)
 {
 	int ret;
+	enum typec_pwr_opmode opmode;
+	unsigned int msecs;
 
 	port->enter_state = port->state;
 	switch (port->state) {
@@ -2133,14 +2183,15 @@ static void run_state_machine(struct tcpm_port *port)
 		break;
 	/* SRC states */
 	case SRC_UNATTACHED:
-		tcpm_swap_complete(port, -ENOTCONN);
+		if (!port->non_pd_role_swap)
+			tcpm_swap_complete(port, -ENOTCONN);
 		tcpm_src_detach(port);
 		if (tcpm_start_drp_toggling(port)) {
 			tcpm_set_state(port, DRP_TOGGLING, 0);
 			break;
 		}
 		tcpm_set_cc(port, tcpm_rp_cc(port));
-		if (port->typec_caps.type == TYPEC_PORT_DRP)
+		if (port->port_type == TYPEC_PORT_DRP)
 			tcpm_set_state(port, SNK_UNATTACHED, PD_T_DRP_SNK);
 		break;
 	case SRC_ATTACH_WAIT:
@@ -2171,25 +2222,43 @@ static void run_state_machine(struct tcpm_port *port)
 		tcpm_set_state(port, SNK_TRY_WAIT, PD_T_DRP_TRY);
 		break;
 	case SNK_TRY_WAIT:
+		if (tcpm_port_is_sink(port)) {
+			tcpm_set_state(port, SNK_TRY_WAIT_DEBOUNCE, 0);
+		} else {
+			tcpm_set_state(port, SRC_TRYWAIT, 0);
+			port->max_wait = 0;
+		}
+		break;
+	case SNK_TRY_WAIT_DEBOUNCE:
+		tcpm_set_state(port, SNK_TRY_WAIT_DEBOUNCE_CHECK_VBUS,
+			       PD_T_PD_DEBOUNCE);
+		break;
+	case SNK_TRY_WAIT_DEBOUNCE_CHECK_VBUS:
 		if (port->vbus_present && tcpm_port_is_sink(port)) {
 			tcpm_set_state(port, SNK_ATTACHED, 0);
-			break;
-		}
-		if (!tcpm_port_is_sink(port)) {
-			tcpm_set_state(port, SRC_TRYWAIT,
-				       PD_T_PD_DEBOUNCE);
-			break;
+		} else {
+			tcpm_set_state(port, SRC_TRYWAIT, 0);
+			port->max_wait = 0;
 		}
-		/* No vbus, cc state is sink or open */
-		tcpm_set_state(port, SRC_TRYWAIT_UNATTACHED, PD_T_DRP_TRYWAIT);
 		break;
 	case SRC_TRYWAIT:
 		tcpm_set_cc(port, tcpm_rp_cc(port));
-		if (!port->vbus_present && tcpm_port_is_source(port))
-			tcpm_set_state(port, SRC_ATTACHED, PD_T_CC_DEBOUNCE);
-		else
+		if (port->max_wait == 0) {
+			port->max_wait = jiffies +
+					 msecs_to_jiffies(PD_T_DRP_TRY);
 			tcpm_set_state(port, SRC_TRYWAIT_UNATTACHED,
 				       PD_T_DRP_TRY);
+		} else {
+			if (time_is_after_jiffies(port->max_wait))
+				tcpm_set_state(port, SRC_TRYWAIT_UNATTACHED,
+					       jiffies_to_msecs(port->max_wait -
+								jiffies));
+			else
+				tcpm_set_state(port, SNK_UNATTACHED, 0);
+		}
+		break;
+	case SRC_TRYWAIT_DEBOUNCE:
+		tcpm_set_state(port, SRC_ATTACHED, PD_T_CC_DEBOUNCE);
 		break;
 	case SRC_TRYWAIT_UNATTACHED:
 		tcpm_set_state(port, SNK_UNATTACHED, 0);
@@ -2201,7 +2270,8 @@ static void run_state_machine(struct tcpm_port *port)
 			       ret < 0 ? 0 : PD_T_PS_SOURCE_ON);
 		break;
 	case SRC_STARTUP:
-		typec_set_pwr_opmode(port->typec_port, TYPEC_PWR_MODE_USB);
+		opmode =  tcpm_get_pwr_opmode(tcpm_rp_cc(port));
+		typec_set_pwr_opmode(port->typec_port, opmode);
 		port->pwr_opmode = TYPEC_PWR_MODE_USB;
 		port->caps_count = 0;
 		port->message_id = 0;
@@ -2262,8 +2332,8 @@ static void run_state_machine(struct tcpm_port *port)
 #endif
 		port->try_src_count = 0;
 
+		tcpm_swap_complete(port, 0);
 		tcpm_typec_connect(port);
-
 		tcpm_check_send_discover(port);
 		/*
 		 * 6.3.5
@@ -2273,14 +2343,11 @@ static void run_state_machine(struct tcpm_port *port)
 		 * - The system is not operating in PD mode
 		 * or
 		 * - Both partners are connected using a Type-C connector
-		 *   XXX How do we know that ?
+		 *
+		 * There is no actual need to send PD messages since the local
+		 * port type-c and the spec does not clearly say whether PD is
+		 * possible when type-c is connected to Type-A/B
 		 */
-		if (port->pwr_opmode == TYPEC_PWR_MODE_PD &&
-		    !port->op_vsafe5v) {
-			tcpm_pd_send_control(port, PD_CTRL_PING);
-			tcpm_set_state_cond(port, SRC_READY,
-					    PD_T_SOURCE_ACTIVITY);
-		}
 		break;
 	case SRC_WAIT_NEW_CAPABILITIES:
 		/* Nothing to do... */
@@ -2288,14 +2355,15 @@ static void run_state_machine(struct tcpm_port *port)
 
 	/* SNK states */
 	case SNK_UNATTACHED:
-		tcpm_swap_complete(port, -ENOTCONN);
+		if (!port->non_pd_role_swap)
+			tcpm_swap_complete(port, -ENOTCONN);
 		tcpm_snk_detach(port);
 		if (tcpm_start_drp_toggling(port)) {
 			tcpm_set_state(port, DRP_TOGGLING, 0);
 			break;
 		}
 		tcpm_set_cc(port, TYPEC_CC_RD);
-		if (port->typec_caps.type == TYPEC_PORT_DRP)
+		if (port->port_type == TYPEC_PORT_DRP)
 			tcpm_set_state(port, SRC_UNATTACHED, PD_T_DRP_SRC);
 		break;
 	case SNK_ATTACH_WAIT:
@@ -2320,39 +2388,52 @@ static void run_state_machine(struct tcpm_port *port)
 				       0);
 		else
 			/* Wait for VBUS, but not forever */
-			tcpm_set_state(port, SNK_UNATTACHED, PD_T_PS_SOURCE_ON);
+			tcpm_set_state(port, PORT_RESET, PD_T_PS_SOURCE_ON);
 		break;
 
 	case SRC_TRY:
 		port->try_src_count++;
 		tcpm_set_cc(port, tcpm_rp_cc(port));
-		tcpm_set_state(port, SNK_TRYWAIT, PD_T_DRP_TRY);
+		port->max_wait = 0;
+		tcpm_set_state(port, SRC_TRY_WAIT, 0);
+		break;
+	case SRC_TRY_WAIT:
+		if (port->max_wait == 0) {
+			port->max_wait = jiffies +
+					 msecs_to_jiffies(PD_T_DRP_TRY);
+			msecs = PD_T_DRP_TRY;
+		} else {
+			if (time_is_after_jiffies(port->max_wait))
+				msecs = jiffies_to_msecs(port->max_wait -
+							 jiffies);
+			else
+				msecs = 0;
+		}
+		tcpm_set_state(port, SNK_TRYWAIT, msecs);
 		break;
 	case SRC_TRY_DEBOUNCE:
 		tcpm_set_state(port, SRC_ATTACHED, PD_T_PD_DEBOUNCE);
 		break;
 	case SNK_TRYWAIT:
 		tcpm_set_cc(port, TYPEC_CC_RD);
-		tcpm_set_state(port, SNK_TRYWAIT_DEBOUNCE, PD_T_CC_DEBOUNCE);
+		tcpm_set_state(port, SNK_TRYWAIT_VBUS, PD_T_CC_DEBOUNCE);
 		break;
-	case SNK_TRYWAIT_DEBOUNCE:
-		if (port->vbus_present) {
+	case SNK_TRYWAIT_VBUS:
+		/*
+		 * TCPM stays in this state indefinitely until VBUS
+		 * is detected as long as Rp is not detected for
+		 * more than a time period of tPDDebounce.
+		 */
+		if (port->vbus_present && tcpm_port_is_sink(port)) {
 			tcpm_set_state(port, SNK_ATTACHED, 0);
 			break;
 		}
-		if (tcpm_port_is_disconnected(port)) {
-			tcpm_set_state(port, SNK_UNATTACHED,
-				       PD_T_PD_DEBOUNCE);
-			break;
-		}
-		if (tcpm_port_is_source(port))
-			tcpm_set_state(port, SRC_ATTACHED, 0);
-		/* XXX Are we supposed to stay in this state ? */
+		if (!tcpm_port_is_sink(port))
+			tcpm_set_state(port, SNK_TRYWAIT_DEBOUNCE, 0);
 		break;
-	case SNK_TRYWAIT_VBUS:
-		tcpm_set_state(port, SNK_ATTACHED, PD_T_CC_DEBOUNCE);
+	case SNK_TRYWAIT_DEBOUNCE:
+		tcpm_set_state(port, SNK_UNATTACHED, PD_T_PD_DEBOUNCE);
 		break;
-
 	case SNK_ATTACHED:
 		ret = tcpm_snk_attach(port);
 		if (ret < 0)
@@ -2362,7 +2443,9 @@ static void run_state_machine(struct tcpm_port *port)
 		break;
 	case SNK_STARTUP:
 		/* XXX: callback into infrastructure */
-		typec_set_pwr_opmode(port->typec_port, TYPEC_PWR_MODE_USB);
+		opmode =  tcpm_get_pwr_opmode(port->polarity ?
+					      port->cc2 : port->cc1);
+		typec_set_pwr_opmode(port->typec_port, opmode);
 		port->pwr_opmode = TYPEC_PWR_MODE_USB;
 		port->message_id = 0;
 		port->rx_msgid = -1;
@@ -2384,7 +2467,7 @@ static void run_state_machine(struct tcpm_port *port)
 		 * see USB power delivery specification, section 8.3.3.6.1.5.1).
 		 */
 		tcpm_set_state(port, hard_reset_state(port),
-			       port->typec_caps.type == TYPEC_PORT_DRP ?
+			       port->port_type == TYPEC_PORT_DRP ?
 					PD_T_DB_DETECT : PD_T_NO_RESPONSE);
 		break;
 	case SNK_DISCOVERY_DEBOUNCE:
@@ -2441,12 +2524,14 @@ static void run_state_machine(struct tcpm_port *port)
 		break;
 	case SNK_READY:
 		port->try_snk_count = 0;
-		port->explicit_contract = true;
-		typec_set_pwr_opmode(port->typec_port, TYPEC_PWR_MODE_PD);
-		port->pwr_opmode = TYPEC_PWR_MODE_PD;
+		if (port->explicit_contract) {
+			typec_set_pwr_opmode(port->typec_port,
+					     TYPEC_PWR_MODE_PD);
+			port->pwr_opmode = TYPEC_PWR_MODE_PD;
+		}
 
+		tcpm_swap_complete(port, 0);
 		tcpm_typec_connect(port);
-
 		tcpm_check_send_discover(port);
 		break;
 
@@ -2574,7 +2659,6 @@ static void run_state_machine(struct tcpm_port *port)
 				       TYPEC_HOST);
 			port->send_discover = true;
 		}
-		tcpm_swap_complete(port, 0);
 		tcpm_set_state(port, ready_state(port), 0);
 		break;
 
@@ -2602,11 +2686,17 @@ static void run_state_machine(struct tcpm_port *port)
 	case PR_SWAP_SRC_SNK_TRANSITION_OFF:
 		tcpm_set_vbus(port, false);
 		port->explicit_contract = false;
+		/* allow time for Vbus discharge, must be < tSrcSwapStdby */
 		tcpm_set_state(port, PR_SWAP_SRC_SNK_SOURCE_OFF,
-			       PD_T_PS_SOURCE_OFF);
+			       PD_T_SRCSWAPSTDBY);
 		break;
 	case PR_SWAP_SRC_SNK_SOURCE_OFF:
 		tcpm_set_cc(port, TYPEC_CC_RD);
+		/* allow CC debounce */
+		tcpm_set_state(port, PR_SWAP_SRC_SNK_SOURCE_OFF_CC_DEBOUNCED,
+			       PD_T_CC_DEBOUNCE);
+		break;
+	case PR_SWAP_SRC_SNK_SOURCE_OFF_CC_DEBOUNCED:
 		/*
 		 * USB-PD standard, 6.2.1.4, Port Power Role:
 		 * "During the Power Role Swap Sequence, for the initial Source
@@ -2622,7 +2712,6 @@ static void run_state_machine(struct tcpm_port *port)
 		tcpm_set_state_cond(port, SNK_UNATTACHED, PD_T_PS_SOURCE_ON);
 		break;
 	case PR_SWAP_SRC_SNK_SINK_ON:
-		tcpm_swap_complete(port, 0);
 		tcpm_set_state(port, SNK_STARTUP, 0);
 		break;
 	case PR_SWAP_SNK_SRC_SINK_OFF:
@@ -2634,6 +2723,15 @@ static void run_state_machine(struct tcpm_port *port)
 		tcpm_set_cc(port, tcpm_rp_cc(port));
 		tcpm_set_vbus(port, true);
 		/*
+		 * allow time VBUS ramp-up, must be < tNewSrc
+		 * Also, this window overlaps with CC debounce as well.
+		 * So, Wait for the max of two which is PD_T_NEWSRC
+		 */
+		tcpm_set_state(port, PR_SWAP_SNK_SRC_SOURCE_ON_VBUS_RAMPED_UP,
+			       PD_T_NEWSRC);
+		break;
+	case PR_SWAP_SNK_SRC_SOURCE_ON_VBUS_RAMPED_UP:
+		/*
 		 * USB PD standard, 6.2.1.4:
 		 * "Subsequent Messages initiated by the Policy Engine,
 		 * such as the PS_RDY Message sent to indicate that Vbus
@@ -2642,7 +2740,6 @@ static void run_state_machine(struct tcpm_port *port)
 		 */
 		tcpm_set_pwr_role(port, TYPEC_SOURCE);
 		tcpm_pd_send_control(port, PD_CTRL_PS_RDY);
-		tcpm_swap_complete(port, 0);
 		tcpm_set_state(port, SRC_STARTUP, 0);
 		break;
 
@@ -2672,12 +2769,10 @@ static void run_state_machine(struct tcpm_port *port)
 	case VCONN_SWAP_TURN_ON_VCONN:
 		tcpm_set_vconn(port, true);
 		tcpm_pd_send_control(port, PD_CTRL_PS_RDY);
-		tcpm_swap_complete(port, 0);
 		tcpm_set_state(port, ready_state(port), 0);
 		break;
 	case VCONN_SWAP_TURN_OFF_VCONN:
 		tcpm_set_vconn(port, false);
-		tcpm_swap_complete(port, 0);
 		tcpm_set_state(port, ready_state(port), 0);
 		break;
 
@@ -2704,13 +2799,15 @@ static void run_state_machine(struct tcpm_port *port)
 		break;
 	case ERROR_RECOVERY:
 		tcpm_swap_complete(port, -EPROTO);
+		tcpm_set_state(port, PORT_RESET, 0);
+		break;
+	case PORT_RESET:
 		tcpm_reset_port(port);
-
 		tcpm_set_cc(port, TYPEC_CC_OPEN);
-		tcpm_set_state(port, ERROR_RECOVERY_WAIT_OFF,
+		tcpm_set_state(port, PORT_RESET_WAIT_OFF,
 			       PD_T_ERROR_RECOVERY);
 		break;
-	case ERROR_RECOVERY_WAIT_OFF:
+	case PORT_RESET_WAIT_OFF:
 		tcpm_set_state(port,
 			       tcpm_default_state(port),
 			       port->vbus_present ? PD_T_PS_SOURCE_OFF : 0);
@@ -2799,10 +2896,12 @@ static void _tcpm_cc_change(struct tcpm_port *port, enum typec_cc_status cc1,
 			tcpm_set_state(port, SRC_ATTACH_WAIT, 0);
 		break;
 	case SRC_ATTACHED:
-		if (tcpm_port_is_disconnected(port))
+	case SRC_SEND_CAPABILITIES:
+	case SRC_READY:
+		if (tcpm_port_is_disconnected(port) ||
+		    !tcpm_port_is_source(port))
 			tcpm_set_state(port, SRC_UNATTACHED, 0);
 		break;
-
 	case SNK_UNATTACHED:
 		if (tcpm_port_is_sink(port))
 			tcpm_set_state(port, SNK_ATTACH_WAIT, 0);
@@ -2869,52 +2968,43 @@ static void _tcpm_cc_change(struct tcpm_port *port, enum typec_cc_status cc1,
 	case SRC_TRYWAIT:
 		/* Hand over to state machine if needed */
 		if (!port->vbus_present && tcpm_port_is_source(port))
-			new_state = SRC_ATTACHED;
-		else
-			new_state = SRC_TRYWAIT_UNATTACHED;
-
-		if (new_state != port->delayed_state)
+			tcpm_set_state(port, SRC_TRYWAIT_DEBOUNCE, 0);
+		break;
+	case SRC_TRYWAIT_DEBOUNCE:
+		if (port->vbus_present || !tcpm_port_is_source(port))
 			tcpm_set_state(port, SRC_TRYWAIT, 0);
 		break;
-	case SNK_TRY_WAIT:
-		if (port->vbus_present && tcpm_port_is_sink(port)) {
-			tcpm_set_state(port, SNK_ATTACHED, 0);
-			break;
+	case SNK_TRY_WAIT_DEBOUNCE:
+		if (!tcpm_port_is_sink(port)) {
+			port->max_wait = 0;
+			tcpm_set_state(port, SRC_TRYWAIT, 0);
 		}
-		if (!tcpm_port_is_sink(port))
-			new_state = SRC_TRYWAIT;
-		else
-			new_state = SRC_TRYWAIT_UNATTACHED;
-
-		if (new_state != port->delayed_state)
-			tcpm_set_state(port, SNK_TRY_WAIT, 0);
 		break;
-
-	case SRC_TRY:
-		tcpm_set_state(port, SRC_TRY_DEBOUNCE, 0);
+	case SRC_TRY_WAIT:
+		if (tcpm_port_is_source(port))
+			tcpm_set_state(port, SRC_TRY_DEBOUNCE, 0);
 		break;
 	case SRC_TRY_DEBOUNCE:
-		tcpm_set_state(port, SRC_TRY, 0);
+		tcpm_set_state(port, SRC_TRY_WAIT, 0);
 		break;
 	case SNK_TRYWAIT_DEBOUNCE:
-		if (port->vbus_present) {
-			tcpm_set_state(port, SNK_ATTACHED, 0);
-			break;
-		}
-		if (tcpm_port_is_source(port)) {
-			tcpm_set_state(port, SRC_ATTACHED, 0);
-			break;
-		}
-		if (tcpm_port_is_disconnected(port) &&
-		    port->delayed_state != SNK_UNATTACHED)
+		if (tcpm_port_is_sink(port))
+			tcpm_set_state(port, SNK_TRYWAIT_VBUS, 0);
+		break;
+	case SNK_TRYWAIT_VBUS:
+		if (!tcpm_port_is_sink(port))
 			tcpm_set_state(port, SNK_TRYWAIT_DEBOUNCE, 0);
 		break;
-
+	case SNK_TRYWAIT:
+		/* Do nothing, waiting for tCCDebounce */
+		break;
 	case PR_SWAP_SNK_SRC_SINK_OFF:
 	case PR_SWAP_SRC_SNK_TRANSITION_OFF:
 	case PR_SWAP_SRC_SNK_SOURCE_OFF:
+	case PR_SWAP_SRC_SNK_SOURCE_OFF_CC_DEBOUNCED:
+	case PR_SWAP_SNK_SRC_SOURCE_ON:
 		/*
-		 * CC state change is expected here; we just turned off power.
+		 * CC state change is expected in PR_SWAP
 		 * Ignore it.
 		 */
 		break;
@@ -2928,12 +3018,11 @@ static void _tcpm_cc_change(struct tcpm_port *port, enum typec_cc_status cc1,
 
 static void _tcpm_pd_vbus_on(struct tcpm_port *port)
 {
-	enum tcpm_state new_state;
-
 	tcpm_log_force(port, "VBUS on");
 	port->vbus_present = true;
 	switch (port->state) {
 	case SNK_TRANSITION_SINK_VBUS:
+		port->explicit_contract = true;
 		tcpm_set_state(port, SNK_READY, 0);
 		break;
 	case SNK_DISCOVERY:
@@ -2959,27 +3048,28 @@ static void _tcpm_pd_vbus_on(struct tcpm_port *port)
 		/* Do nothing, waiting for timeout */
 		break;
 	case SRC_TRYWAIT:
-		/* Hand over to state machine if needed */
-		if (port->delayed_state != SRC_TRYWAIT_UNATTACHED)
-			tcpm_set_state(port, SRC_TRYWAIT, 0);
+		/* Do nothing, Waiting for Rd to be detected */
 		break;
-	case SNK_TRY_WAIT:
-		if (tcpm_port_is_sink(port)) {
-			tcpm_set_state(port, SNK_ATTACHED, 0);
-			break;
-		}
-		if (!tcpm_port_is_sink(port))
-			new_state = SRC_TRYWAIT;
-		else
-			new_state = SRC_TRYWAIT_UNATTACHED;
-
-		if (new_state != port->delayed_state)
-			tcpm_set_state(port, SNK_TRY_WAIT, 0);
+	case SRC_TRYWAIT_DEBOUNCE:
+		tcpm_set_state(port, SRC_TRYWAIT, 0);
+		break;
+	case SNK_TRY_WAIT_DEBOUNCE:
+		/* Do nothing, waiting for PD_DEBOUNCE to do be done */
 		break;
 	case SNK_TRYWAIT:
-		tcpm_set_state(port, SNK_TRYWAIT_VBUS, 0);
+		/* Do nothing, waiting for tCCDebounce */
+		break;
+	case SNK_TRYWAIT_VBUS:
+		if (tcpm_port_is_sink(port))
+			tcpm_set_state(port, SNK_ATTACHED, 0);
+		break;
+	case SNK_TRYWAIT_DEBOUNCE:
+		/* Do nothing, waiting for Rp */
+		break;
+	case SRC_TRY_WAIT:
+	case SRC_TRY_DEBOUNCE:
+		/* Do nothing, waiting for sink detection */
 		break;
-
 	default:
 		break;
 	}
@@ -2987,8 +3077,6 @@ static void _tcpm_pd_vbus_on(struct tcpm_port *port)
 
 static void _tcpm_pd_vbus_off(struct tcpm_port *port)
 {
-	enum tcpm_state new_state;
-
 	tcpm_log_force(port, "VBUS off");
 	port->vbus_present = false;
 	port->vbus_never_low = false;
@@ -3008,25 +3096,15 @@ static void _tcpm_pd_vbus_off(struct tcpm_port *port)
 	case SRC_TRYWAIT:
 		/* Hand over to state machine if needed */
 		if (tcpm_port_is_source(port))
-			new_state = SRC_ATTACHED;
-		else
-			new_state = SRC_TRYWAIT_UNATTACHED;
-		if (new_state != port->delayed_state)
-			tcpm_set_state(port, SRC_TRYWAIT, 0);
+			tcpm_set_state(port, SRC_TRYWAIT_DEBOUNCE, 0);
 		break;
-	case SNK_TRY_WAIT:
-		if (!tcpm_port_is_sink(port))
-			new_state = SRC_TRYWAIT;
-		else
-			new_state = SRC_TRYWAIT_UNATTACHED;
-
-		if (new_state != port->delayed_state)
-			tcpm_set_state(port, SNK_TRY_WAIT, 0);
+	case SNK_TRY_WAIT_DEBOUNCE:
+		/* Do nothing, waiting for PD_DEBOUNCE to do be done */
 		break;
+	case SNK_TRYWAIT:
 	case SNK_TRYWAIT_VBUS:
-		tcpm_set_state(port, SNK_TRYWAIT, 0);
+	case SNK_TRYWAIT_DEBOUNCE:
 		break;
-
 	case SNK_ATTACH_WAIT:
 		tcpm_set_state(port, SNK_UNATTACHED, 0);
 		break;
@@ -3042,13 +3120,13 @@ static void _tcpm_pd_vbus_off(struct tcpm_port *port)
 		/* Do nothing, expected */
 		break;
 
-	case ERROR_RECOVERY_WAIT_OFF:
-		tcpm_set_state(port,
-			       port->pwr_role == TYPEC_SOURCE ?
-					SRC_UNATTACHED : SNK_UNATTACHED,
-			       0);
+	case PORT_RESET_WAIT_OFF:
+		tcpm_set_state(port, tcpm_default_state(port), 0);
+		break;
+	case SRC_TRY_WAIT:
+	case SRC_TRY_DEBOUNCE:
+		/* Do nothing, waiting for sink detection */
 		break;
-
 	default:
 		if (port->pwr_role == TYPEC_SINK &&
 		    port->attached)
@@ -3142,7 +3220,7 @@ static int tcpm_dr_set(const struct typec_capability *cap,
 	mutex_lock(&port->swap_lock);
 	mutex_lock(&port->lock);
 
-	if (port->typec_caps.type != TYPEC_PORT_DRP || !port->pd_capable) {
+	if (port->port_type != TYPEC_PORT_DRP) {
 		ret = -EINVAL;
 		goto port_unlock;
 	}
@@ -3163,15 +3241,35 @@ static int tcpm_dr_set(const struct typec_capability *cap,
 	 * Reject data role swap request in this case.
 	 */
 
+	if (!port->pd_capable) {
+		/*
+		 * If the partner is not PD capable, reset the port to
+		 * trigger a role change. This can only work if a preferred
+		 * role is configured, and if it matches the requested role.
+		 */
+		if (port->try_role == TYPEC_NO_PREFERRED_ROLE ||
+		    port->try_role == port->pwr_role) {
+			ret = -EINVAL;
+			goto port_unlock;
+		}
+		port->non_pd_role_swap = true;
+		tcpm_set_state(port, PORT_RESET, 0);
+	} else {
+		tcpm_set_state(port, DR_SWAP_SEND, 0);
+	}
+
 	port->swap_status = 0;
 	port->swap_pending = true;
 	reinit_completion(&port->swap_complete);
-	tcpm_set_state(port, DR_SWAP_SEND, 0);
 	mutex_unlock(&port->lock);
 
-	wait_for_completion(&port->swap_complete);
+	if (!wait_for_completion_timeout(&port->swap_complete,
+				msecs_to_jiffies(PD_ROLE_SWAP_TIMEOUT)))
+		ret = -ETIMEDOUT;
+	else
+		ret = port->swap_status;
 
-	ret = port->swap_status;
+	port->non_pd_role_swap = false;
 	goto swap_unlock;
 
 port_unlock:
@@ -3190,7 +3288,7 @@ static int tcpm_pr_set(const struct typec_capability *cap,
 	mutex_lock(&port->swap_lock);
 	mutex_lock(&port->lock);
 
-	if (port->typec_caps.type != TYPEC_PORT_DRP) {
+	if (port->port_type != TYPEC_PORT_DRP) {
 		ret = -EINVAL;
 		goto port_unlock;
 	}
@@ -3204,31 +3302,18 @@ static int tcpm_pr_set(const struct typec_capability *cap,
 		goto port_unlock;
 	}
 
-	if (!port->pd_capable) {
-		/*
-		 * If the partner is not PD capable, reset the port to
-		 * trigger a role change. This can only work if a preferred
-		 * role is configured, and if it matches the requested role.
-		 */
-		if (port->try_role == TYPEC_NO_PREFERRED_ROLE ||
-		    port->try_role == port->pwr_role) {
-			ret = -EINVAL;
-			goto port_unlock;
-		}
-		tcpm_set_state(port, HARD_RESET_SEND, 0);
-		ret = 0;
-		goto port_unlock;
-	}
-
 	port->swap_status = 0;
 	port->swap_pending = true;
 	reinit_completion(&port->swap_complete);
 	tcpm_set_state(port, PR_SWAP_SEND, 0);
 	mutex_unlock(&port->lock);
 
-	wait_for_completion(&port->swap_complete);
+	if (!wait_for_completion_timeout(&port->swap_complete,
+				msecs_to_jiffies(PD_ROLE_SWAP_TIMEOUT)))
+		ret = -ETIMEDOUT;
+	else
+		ret = port->swap_status;
 
-	ret = port->swap_status;
 	goto swap_unlock;
 
 port_unlock:
@@ -3263,9 +3348,12 @@ static int tcpm_vconn_set(const struct typec_capability *cap,
 	tcpm_set_state(port, VCONN_SWAP_SEND, 0);
 	mutex_unlock(&port->lock);
 
-	wait_for_completion(&port->swap_complete);
+	if (!wait_for_completion_timeout(&port->swap_complete,
+				msecs_to_jiffies(PD_ROLE_SWAP_TIMEOUT)))
+		ret = -ETIMEDOUT;
+	else
+		ret = port->swap_status;
 
-	ret = port->swap_status;
 	goto swap_unlock;
 
 port_unlock:
@@ -3319,7 +3407,35 @@ static void tcpm_init(struct tcpm_port *port)
 	 * Some adapters need a clean slate at startup, and won't recover
 	 * otherwise. So do not try to be fancy and force a clean disconnect.
 	 */
-	tcpm_set_state(port, ERROR_RECOVERY, 0);
+	tcpm_set_state(port, PORT_RESET, 0);
+}
+
+static int tcpm_port_type_set(const struct typec_capability *cap,
+			      enum typec_port_type type)
+{
+	struct tcpm_port *port = typec_cap_to_tcpm(cap);
+
+	mutex_lock(&port->lock);
+	if (type == port->port_type)
+		goto port_unlock;
+
+	port->port_type = type;
+
+	if (!port->connected) {
+		tcpm_set_state(port, PORT_RESET, 0);
+	} else if (type == TYPEC_PORT_UFP) {
+		if (!(port->pwr_role == TYPEC_SINK &&
+		      port->data_role == TYPEC_DEVICE))
+			tcpm_set_state(port, PORT_RESET, 0);
+	} else if (type == TYPEC_PORT_DFP) {
+		if (!(port->pwr_role == TYPEC_SOURCE &&
+		      port->data_role == TYPEC_HOST))
+			tcpm_set_state(port, PORT_RESET, 0);
+	}
+
+port_unlock:
+	mutex_unlock(&port->lock);
+	return 0;
 }
 
 void tcpm_tcpc_reset(struct tcpm_port *port)
@@ -3469,9 +3585,10 @@ struct tcpm_port *tcpm_register_port(struct device *dev, struct tcpc_dev *tcpc)
 	port->typec_caps.pr_set = tcpm_pr_set;
 	port->typec_caps.vconn_set = tcpm_vconn_set;
 	port->typec_caps.try_role = tcpm_try_role;
+	port->typec_caps.port_type_set = tcpm_port_type_set;
 
 	port->partner_desc.identity = &port->partner_ident;
-
+	port->port_type = tcpc->config->type;
 	/*
 	 * TODO:
 	 *  - alt_modes, set_alt_mode
@@ -3485,7 +3602,7 @@ struct tcpm_port *tcpm_register_port(struct device *dev, struct tcpc_dev *tcpc)
 	}
 
 	if (tcpc->config->alt_modes) {
-		struct typec_altmode_desc *paltmode = tcpc->config->alt_modes;
+		const struct typec_altmode_desc *paltmode = tcpc->config->alt_modes;
 
 		i = 0;
 		while (paltmode->svid && i < ARRAY_SIZE(port->port_altmode)) {
diff --git a/drivers/staging/typec/tcpm.h b/drivers/staging/typec/tcpm.h
index 19c307d31a5a..7e9a6b7b5cd6 100644
--- a/drivers/staging/typec/tcpm.h
+++ b/drivers/staging/typec/tcpm.h
@@ -34,7 +34,8 @@ enum typec_cc_polarity {
 };
 
 /* Time to wait for TCPC to complete transmit */
-#define PD_T_TCPC_TX_TIMEOUT  100
+#define PD_T_TCPC_TX_TIMEOUT	100		/* in ms	*/
+#define PD_ROLE_SWAP_TIMEOUT	(MSEC_PER_SEC * 10)
 
 enum tcpm_transmit_status {
 	TCPC_TX_SUCCESS = 0,
@@ -72,7 +73,7 @@ struct tcpc_config {
 	enum typec_role default_role;
 	bool try_role_hw;	/* try.{src,snk} implemented in hardware */
 
-	struct typec_altmode_desc *alt_modes;
+	const struct typec_altmode_desc *alt_modes;
 };
 
 enum tcpc_usb_switch {
@@ -108,6 +109,13 @@ struct tcpc_dev {
 
 	int (*init)(struct tcpc_dev *dev);
 	int (*get_vbus)(struct tcpc_dev *dev);
+	/*
+	 * This optional callback gets called by the tcpm core when configured
+	 * as a snk and cc=Rp-def. This allows the tcpm to provide a fallback
+	 * current-limit detection method for the cc=Rp-def case. E.g. some
+	 * tcpcs may include BC1.2 charger detection and use that in this case.
+	 */
+	int (*get_current_limit)(struct tcpc_dev *dev);
 	int (*set_cc)(struct tcpc_dev *dev, enum typec_cc_status cc);
 	int (*get_cc)(struct tcpc_dev *dev, enum typec_cc_status *cc1,
 		      enum typec_cc_status *cc2);
diff --git a/drivers/staging/unisys/Documentation/overview.txt b/drivers/staging/unisys/Documentation/overview.txt
index e0466bfada2f..9ab30af265a5 100644
--- a/drivers/staging/unisys/Documentation/overview.txt
+++ b/drivers/staging/unisys/Documentation/overview.txt
@@ -221,7 +221,7 @@ The following files exist under /sys/devices/visorbus<x>/vbus<x>:dev<y>:
 
 The visorhba driver registers with visorbus as the function driver to
 handle virtual scsi disk devices, specified using the
-VISOR_VHBA_CHANNEL_UUID type in the visorbus_register_visor_driver()
+VISOR_VHBA_CHANNEL_GUID type in the visorbus_register_visor_driver()
 call. visorhba uses scsi_add_host() to expose a Linux block device
 (e.g., /sys/block/) in the guest environment for each s-Par virtual device.
 
@@ -240,7 +240,7 @@ When compiled as a module, visorhba can be autoloaded by visorbus in
 standard udev/systemd environments, as it includes the modules.alias
 definition:
 
-    "visorbus:"+VISOR_VHBA_CHANNEL_UUID_STR
+    "visorbus:"+VISOR_VHBA_CHANNEL_GUID_STR
 
 i.e.:
 
@@ -252,7 +252,7 @@ i.e.:
 
 The visornic driver registers with visorbus as the function driver to
 handle virtual network devices, specified using the
-VISOR_VNIC_CHANNEL_UUID type in the visorbus_register_visor_driver()
+VISOR_VNIC_CHANNEL_GUID type in the visorbus_register_visor_driver()
 call. visornic uses register_netdev() to expose a Linux device of class net
 (e.g., /sys/class/net/) in the guest environment for each s-Par virtual
 device.
@@ -270,7 +270,7 @@ When compiled as a module, visornic can be autoloaded by visorbus in
 standard udev/systemd environments, as it includes the modules.alias
 definition:
 
-    "visorbus:"+VISOR_VNIC_CHANNEL_UUID_STR
+    "visorbus:"+VISOR_VNIC_CHANNEL_GUID_STR
 
 i.e.:
 
@@ -282,7 +282,7 @@ i.e.:
 
 The visorinput driver registers with visorbus as the function driver to
 handle human input devices, specified using the
-VISOR_KEYBOARD_CHANNEL_UUID and VISOR_MOUSE_CHANNEL_UUID
+VISOR_KEYBOARD_CHANNEL_GUID and VISOR_MOUSE_CHANNEL_GUID
 types in the visorbus_register_visor_driver() call. visorinput uses
 input_register_device() to expose devices of class input
 (e.g., /sys/class/input/) for virtual keyboard and virtual mouse devices.
@@ -307,8 +307,8 @@ When compiled as a module, visorinput can be autoloaded by visorbus in
 standard udev/systemd environments, as it includes the modules.alias
 definition:
 
-    "visorbus:"+VISOR_MOUSE_CHANNEL_UUID_STR
-    "visorbus:"+VISOR_KEYBOARD_CHANNEL_UUID_STR
+    "visorbus:"+VISOR_MOUSE_CHANNEL_GUID_STR
+    "visorbus:"+VISOR_KEYBOARD_CHANNEL_GUID_STR
 
 i.e.:
 
diff --git a/drivers/staging/unisys/include/channel.h b/drivers/staging/unisys/include/channel.h
index 692efcb38245..2babe93631f3 100644
--- a/drivers/staging/unisys/include/channel.h
+++ b/drivers/staging/unisys/include/channel.h
@@ -1,4 +1,5 @@
-/* Copyright (C) 2010 - 2013 UNISYS CORPORATION
+/*
+ * Copyright (C) 2010 - 2013 UNISYS CORPORATION
  * All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify
@@ -20,87 +21,48 @@
 #include <linux/io.h>
 #include <linux/uuid.h>
 
-#define __SUPERVISOR_CHANNEL_H__
-
 #define SIGNATURE_16(A, B) ((A) | ((B) << 8))
 #define SIGNATURE_32(A, B, C, D) \
 	(SIGNATURE_16(A, B) | (SIGNATURE_16(C, D) << 16))
-#define SIGNATURE_64(A, B, C, D, E, F, G, H) \
-	(SIGNATURE_32(A, B, C, D) | ((u64)(SIGNATURE_32(E, F, G, H)) << 32))
-
-#ifndef COVER
-#define COVER(v, d) ((d) * DIV_ROUND_UP(v, d))
-#endif
-
 #define VISOR_CHANNEL_SIGNATURE SIGNATURE_32('E', 'C', 'N', 'L')
 
+/*
+ * enum channel_serverstate
+ * @CHANNELSRV_UNINITIALIZED: Channel is in an undefined state.
+ * @CHANNELSRV_READY:	      Channel has been initialized by server.
+ */
 enum channel_serverstate {
-	CHANNELSRV_UNINITIALIZED = 0,	/* channel is in an undefined state */
-	CHANNELSRV_READY = 1	/* channel has been initialized by server */
+	CHANNELSRV_UNINITIALIZED = 0,
+	CHANNELSRV_READY = 1
 };
 
+/*
+ * enum channel_clientstate
+ * @CHANNELCLI_DETACHED:
+ * @CHANNELCLI_DISABLED:  Client can see channel but is NOT allowed to use it
+ *			  unless given TBD* explicit request
+ *			  (should actually be < DETACHED).
+ * @CHANNELCLI_ATTACHING: Legacy EFI client request for EFI server to attach.
+ * @CHANNELCLI_ATTACHED:  Idle, but client may want to use channel any time.
+ * @CHANNELCLI_BUSY:	  Client either wants to use or is using channel.
+ * @CHANNELCLI_OWNED:	  "No worries" state - client can access channel
+ *			  anytime.
+ */
 enum channel_clientstate {
 	CHANNELCLI_DETACHED = 0,
-	CHANNELCLI_DISABLED = 1,	/* client can see channel but is NOT
-					 * allowed to use it unless given TBD
-					 * explicit request (should actually be
-					 * < DETACHED)
-					 */
-	CHANNELCLI_ATTACHING = 2,	/* legacy EFI client request
-					 * for EFI server to attach
-					 */
-	CHANNELCLI_ATTACHED = 3,	/* idle, but client may want
-					 * to use channel any time
-					 */
-	CHANNELCLI_BUSY = 4,	/* client either wants to use or is
-				 * using channel
-				 */
-	CHANNELCLI_OWNED = 5	/* "no worries" state - client can */
-				/* access channel anytime */
+	CHANNELCLI_DISABLED = 1,
+	CHANNELCLI_ATTACHING = 2,
+	CHANNELCLI_ATTACHED = 3,
+	CHANNELCLI_BUSY = 4,
+	CHANNELCLI_OWNED = 5
 };
 
-#define VISOR_CHANNEL_SERVER_READY(ch) \
-	(readl(&(ch)->srv_state) == CHANNELSRV_READY)
-
-#define VISOR_VALID_CHANNELCLI_TRANSITION(o, n) \
-	(((((o) == CHANNELCLI_DETACHED) && ((n) == CHANNELCLI_DISABLED)) || \
-	  (((o) == CHANNELCLI_ATTACHING) && ((n) == CHANNELCLI_DISABLED)) || \
-	  (((o) == CHANNELCLI_ATTACHED) && ((n) == CHANNELCLI_DISABLED)) || \
-	  (((o) == CHANNELCLI_ATTACHING) && ((n) == CHANNELCLI_DETACHED)) || \
-	  (((o) == CHANNELCLI_ATTACHED) && ((n) == CHANNELCLI_DETACHED)) || \
-	  (((o) == CHANNELCLI_DETACHED) && ((n) == CHANNELCLI_ATTACHING)) || \
-	  (((o) == CHANNELCLI_ATTACHING) && ((n) == CHANNELCLI_ATTACHED)) || \
-	  (((o) == CHANNELCLI_DETACHED) && ((n) == CHANNELCLI_ATTACHED)) || \
-	  (((o) == CHANNELCLI_BUSY) && ((n) == CHANNELCLI_ATTACHED)) ||	\
-	  (((o) == CHANNELCLI_ATTACHED) && ((n) == CHANNELCLI_BUSY)) ||	\
-	  (((o) == CHANNELCLI_DETACHED) && ((n) == CHANNELCLI_OWNED)) || \
-	  (((o) == CHANNELCLI_DISABLED) && ((n) == CHANNELCLI_OWNED)) || \
-	  (((o) == CHANNELCLI_ATTACHING) && ((n) == CHANNELCLI_OWNED)) || \
-	  (((o) == CHANNELCLI_ATTACHED) && ((n) == CHANNELCLI_OWNED)) || \
-	  (((o) == CHANNELCLI_BUSY) && ((n) == CHANNELCLI_OWNED)) || (0)) \
-	 ? (1) : (0))
-
-/* Values for VISORA_CHANNEL_PROTOCOL.CliErrorBoot: */
-/* throttling invalid boot channel statetransition error due to client
- * disabled
- */
-#define VISOR_CLIERRORBOOT_THROTTLEMSG_DISABLED 0x01
-
-/* throttling invalid boot channel statetransition error due to client
- * not attached
- */
-#define VISOR_CLIERRORBOOT_THROTTLEMSG_NOTATTACHED 0x02
-
-/* throttling invalid boot channel statetransition error due to busy channel */
-#define VISOR_CLIERRORBOOT_THROTTLEMSG_BUSY 0x04
-
-/* Values for VISOR_CHANNEL_PROTOCOL.Features: This define exists so
- * that windows guest can look at the FeatureFlags in the io channel,
- * and configure the windows driver to use interrupts or not based on
- * this setting.  This flag is set in uislib after the
- * VISOR_VHBA_init_channel is called.  All feature bits for all
- * channels should be defined here.  The io channel feature bits are
- * defined right here
+/*
+ * Values for VISOR_CHANNEL_PROTOCOL.Features: This define exists so that
+ * a guest can look at the FeatureFlags in the io channel, and configure the
+ * driver to use interrupts or not based on this setting. All feature bits for
+ * all channels should be defined here. The io channel feature bits are defined
+ * below.
  */
 #define VISOR_DRIVER_ENABLES_INTS (0x1ULL << 1)
 #define VISOR_CHANNEL_IS_POLLING (0x1ULL << 3)
@@ -108,171 +70,124 @@ enum channel_clientstate {
 #define VISOR_DRIVER_DISABLES_INTS (0x1ULL << 5)
 #define VISOR_DRIVER_ENHANCED_RCVBUF_CHECKING (0x1ULL << 6)
 
-/* Common Channel Header */
+/*
+ * struct channel_header - Common Channel Header
+ * @signature:	       Signature.
+ * @legacy_state:      DEPRECATED - being replaced by.
+ * @header_size:       sizeof(struct channel_header).
+ * @size:	       Total size of this channel in bytes.
+ * @features:	       Flags to modify behavior.
+ * @chtype:	       Channel type: data, bus, control, etc..
+ * @partition_handle:  ID of guest partition.
+ * @handle:	       Device number of this channel in client.
+ * @ch_space_offset:   Offset in bytes to channel specific area.
+ * @version_id:	       Struct channel_header Version ID.
+ * @partition_index:   Index of guest partition.
+ * @zone_uuid:	       Guid of Channel's zone.
+ * @cli_str_offset:    Offset from channel header to null-terminated
+ *		       ClientString (0 if ClientString not present).
+ * @cli_state_boot:    CHANNEL_CLIENTSTATE of pre-boot EFI client of this
+ *		       channel.
+ * @cmd_state_cli:     CHANNEL_COMMANDSTATE (overloaded in Windows drivers, see
+ *		       ServerStateUp, ServerStateDown, etc).
+ * @cli_state_os:      CHANNEL_CLIENTSTATE of Guest OS client of this channel.
+ * @ch_characteristic: CHANNEL_CHARACTERISTIC_<xxx>.
+ * @cmd_state_srv:     CHANNEL_COMMANDSTATE (overloaded in Windows drivers, see
+ *		       ServerStateUp, ServerStateDown, etc).
+ * @srv_state:	       CHANNEL_SERVERSTATE.
+ * @cli_error_boot:    Bits to indicate err states for boot clients, so err
+ *		       messages can be throttled.
+ * @cli_error_os:      Bits to indicate err states for OS clients, so err
+ *		       messages can be throttled.
+ * @filler:	       Pad out to 128 byte cacheline.
+ * @recover_channel:   Please add all new single-byte values below here.
+ */
 struct channel_header {
-	u64 signature;		/* Signature */
-	u32 legacy_state;	/* DEPRECATED - being replaced by */
-			/* SrvState, CliStateBoot, and CliStateOS below */
-	u32 header_size;	/* sizeof(struct channel_header) */
-	u64 size;		/* Total size of this channel in bytes */
-	u64 features;		/* Flags to modify behavior */
-	uuid_le chtype;		/* Channel type: data, bus, control, etc. */
-	u64 partition_handle;	/* ID of guest partition */
-	u64 handle;		/* Device number of this channel in client */
-	u64 ch_space_offset;	/* Offset in bytes to channel specific area */
-	u32 version_id;		/* struct channel_header Version ID */
-	u32 partition_index;	/* Index of guest partition */
-	uuid_le zone_uuid;	/* Guid of Channel's zone */
-	u32 cli_str_offset;	/* offset from channel header to
-				 * nul-terminated ClientString (0 if
-				 * ClientString not present)
-				 */
-	u32 cli_state_boot;	/* CHANNEL_CLIENTSTATE of pre-boot
-				 * EFI client of this channel
-				 */
-	u32 cmd_state_cli;	/* CHANNEL_COMMANDSTATE (overloaded in
-				 * Windows drivers, see ServerStateUp,
-				 * ServerStateDown, etc)
-				 */
-	u32 cli_state_os;	/* CHANNEL_CLIENTSTATE of Guest OS
-				 * client of this channel
-				 */
-	u32 ch_characteristic;	/* CHANNEL_CHARACTERISTIC_<xxx> */
-	u32 cmd_state_srv;	/* CHANNEL_COMMANDSTATE (overloaded in
-				 * Windows drivers, see ServerStateUp,
-				 * ServerStateDown, etc)
-				 */
-	u32 srv_state;		/* CHANNEL_SERVERSTATE */
-	u8 cli_error_boot;	/* bits to indicate err states for
-				 * boot clients, so err messages can
-				 * be throttled
-				 */
-	u8 cli_error_os;	/* bits to indicate err states for OS
-				 * clients, so err messages can be
-				 * throttled
-				 */
-	u8 filler[1];		/* Pad out to 128 byte cacheline */
-	/* Please add all new single-byte values below here */
+	u64 signature;
+	u32 legacy_state;
+	/* SrvState, CliStateBoot, and CliStateOS below */
+	u32 header_size;
+	u64 size;
+	u64 features;
+	guid_t chtype;
+	u64 partition_handle;
+	u64 handle;
+	u64 ch_space_offset;
+	u32 version_id;
+	u32 partition_index;
+	guid_t zone_guid;
+	u32 cli_str_offset;
+	u32 cli_state_boot;
+	u32 cmd_state_cli;
+	u32 cli_state_os;
+	u32 ch_characteristic;
+	u32 cmd_state_srv;
+	u32 srv_state;
+	u8 cli_error_boot;
+	u8 cli_error_os;
+	u8 filler[1];
 	u8 recover_channel;
 } __packed;
 
 #define VISOR_CHANNEL_ENABLE_INTS (0x1ULL << 0)
 
-/* Subheader for the Signal Type variation of the Common Channel */
+/*
+ * struct signal_queue_header - Subheader for the Signal Type variation of the
+ *                              Common Channel.
+ * @version:	      SIGNAL_QUEUE_HEADER Version ID.
+ * @chtype:	      Queue type: storage, network.
+ * @size:	      Total size of this queue in bytes.
+ * @sig_base_offset:  Offset to signal queue area.
+ * @features:	      Flags to modify behavior.
+ * @num_sent:	      Total # of signals placed in this queue.
+ * @num_overflows:    Total # of inserts failed due to full queue.
+ * @signal_size:      Total size of a signal for this queue.
+ * @max_slots:        Max # of slots in queue, 1 slot is always empty.
+ * @max_signals:      Max # of signals in queue (MaxSignalSlots-1).
+ * @head:	      Queue head signal #.
+ * @num_received:     Total # of signals removed from this queue.
+ * @tail:	      Queue tail signal.
+ * @reserved1:	      Reserved field.
+ * @reserved2:	      Reserved field.
+ * @client_queue:
+ * @num_irq_received: Total # of Interrupts received. This is incremented by the
+ *		      ISR in the guest windows driver.
+ * @num_empty:	      Number of times that visor_signal_remove is called and
+ *		      returned Empty Status.
+ * @errorflags:	      Error bits set during SignalReinit to denote trouble with
+ *		      client's fields.
+ * @filler:	      Pad out to 64 byte cacheline.
+ */
 struct signal_queue_header {
 	/* 1st cache line */
-	u32 version;		/* SIGNAL_QUEUE_HEADER Version ID */
-	u32 chtype;		/* Queue type: storage, network */
-	u64 size;		/* Total size of this queue in bytes */
-	u64 sig_base_offset;	/* Offset to signal queue area */
-	u64 features;		/* Flags to modify behavior */
-	u64 num_sent;		/* Total # of signals placed in this queue */
-	u64 num_overflows;	/* Total # of inserts failed due to
-				 * full queue
-				 */
-	u32 signal_size;	/* Total size of a signal for this queue */
-	u32 max_slots;		/* Max # of slots in queue, 1 slot is
-				 * always empty
-				 */
-	u32 max_signals;	/* Max # of signals in queue
-				 * (MaxSignalSlots-1)
-				 */
-	u32 head;		/* Queue head signal # */
+	u32 version;
+	u32 chtype;
+	u64 size;
+	u64 sig_base_offset;
+	u64 features;
+	u64 num_sent;
+	u64 num_overflows;
+	u32 signal_size;
+	u32 max_slots;
+	u32 max_signals;
+	u32 head;
 	/* 2nd cache line */
-	u64 num_received;	/* Total # of signals removed from this queue */
-	u32 tail;		/* Queue tail signal */
-	u32 reserved1;		/* Reserved field */
-	u64 reserved2;		/* Reserved field */
+	u64 num_received;
+	u32 tail;
+	u32 reserved1;
+	u64 reserved2;
 	u64 client_queue;
-	u64 num_irq_received;	/* Total # of Interrupts received.  This
-				 * is incremented by the ISR in the
-				 * guest windows driver
-				 */
-	u64 num_empty;		/* Number of times that visor_signal_remove
-				 * is called and returned Empty Status.
-				 */
-	u32 errorflags;		/* Error bits set during SignalReinit
-				 * to denote trouble with client's
-				 * fields
-				 */
-	u8 filler[12];		/* Pad out to 64 byte cacheline */
+	u64 num_irq_received;
+	u64 num_empty;
+	u32 errorflags;
+	u8 filler[12];
 } __packed;
 
-/* Generic function useful for validating any type of channel when it is
- * received by the client that will be accessing the channel.
- * Note that <logCtx> is only needed for callers in the EFI environment, and
- * is used to pass the EFI_DIAG_CAPTURE_PROTOCOL needed to log messages.
- */
-static inline int
-visor_check_channel(struct channel_header *ch,
-		    uuid_le expected_uuid,
-		    char *chname,
-		    u64 expected_min_bytes,
-		    u32 expected_version,
-		    u64 expected_signature)
-{
-	if (uuid_le_cmp(expected_uuid, NULL_UUID_LE) != 0) {
-		/* caller wants us to verify type GUID */
-		if (uuid_le_cmp(ch->chtype, expected_uuid) != 0) {
-			pr_err("Channel mismatch on channel=%s(%pUL) field=type expected=%pUL actual=%pUL\n",
-			       chname, &expected_uuid,
-			       &expected_uuid, &ch->chtype);
-			return 0;
-		}
-	}
-	if (expected_min_bytes > 0) {	/* verify channel size */
-		if (ch->size < expected_min_bytes) {
-			pr_err("Channel mismatch on channel=%s(%pUL) field=size expected=0x%-8.8Lx actual=0x%-8.8Lx\n",
-			       chname, &expected_uuid,
-			       (unsigned long long)expected_min_bytes,
-			       ch->size);
-			return 0;
-		}
-	}
-	if (expected_version > 0) {	/* verify channel version */
-		if (ch->version_id != expected_version) {
-			pr_err("Channel mismatch on channel=%s(%pUL) field=version expected=0x%-8.8lx actual=0x%-8.8x\n",
-			       chname, &expected_uuid,
-			       (unsigned long)expected_version,
-			       ch->version_id);
-			return 0;
-		}
-	}
-	if (expected_signature > 0) {	/* verify channel signature */
-		if (ch->signature != expected_signature) {
-			pr_err("Channel mismatch on channel=%s(%pUL) field=signature expected=0x%-8.8Lx actual=0x%-8.8Lx\n",
-			       chname, &expected_uuid,
-			       expected_signature, ch->signature);
-			return 0;
-		}
-	}
-	return 1;
-}
-
-/*
- * CHANNEL Guids
- */
-
+/* CHANNEL Guids */
 /* {414815ed-c58c-11da-95a9-00e08161165f} */
-#define VISOR_VHBA_CHANNEL_UUID \
-	UUID_LE(0x414815ed, 0xc58c, 0x11da, \
-		0x95, 0xa9, 0x0, 0xe0, 0x81, 0x61, 0x16, 0x5f)
-static const uuid_le visor_vhba_channel_uuid = VISOR_VHBA_CHANNEL_UUID;
-#define VISOR_VHBA_CHANNEL_UUID_STR \
+#define VISOR_VHBA_CHANNEL_GUID \
+	GUID_INIT(0x414815ed, 0xc58c, 0x11da, \
+		  0x95, 0xa9, 0x0, 0xe0, 0x81, 0x61, 0x16, 0x5f)
+#define VISOR_VHBA_CHANNEL_GUID_STR \
 	"414815ed-c58c-11da-95a9-00e08161165f"
-
-/* {8cd5994d-c58e-11da-95a9-00e08161165f} */
-#define VISOR_VNIC_CHANNEL_UUID \
-	UUID_LE(0x8cd5994d, 0xc58e, 0x11da, \
-		0x95, 0xa9, 0x0, 0xe0, 0x81, 0x61, 0x16, 0x5f)
-static const uuid_le visor_vnic_channel_uuid = VISOR_VNIC_CHANNEL_UUID;
-#define VISOR_VNIC_CHANNEL_UUID_STR \
-	"8cd5994d-c58e-11da-95a9-00e08161165f"
-
-/* {72120008-4AAB-11DC-8530-444553544200} */
-#define VISOR_SIOVM_UUID \
-	UUID_LE(0x72120008, 0x4AAB, 0x11DC, \
-		0x85, 0x30, 0x44, 0x45, 0x53, 0x54, 0x42, 0x00)
-static const uuid_le visor_siovm_uuid = VISOR_SIOVM_UUID;
-
 #endif
diff --git a/drivers/staging/unisys/include/iochannel.h b/drivers/staging/unisys/include/iochannel.h
index c7cb3fbde7b2..a70760f48566 100644
--- a/drivers/staging/unisys/include/iochannel.h
+++ b/drivers/staging/unisys/include/iochannel.h
@@ -1,5 +1,19 @@
-/* Copyright (C) 2010 - 2016 UNISYS CORPORATION */
-/* All rights reserved. */
+/*
+ * Copyright (C) 2010 - 2016 UNISYS CORPORATION
+ * All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or (at
+ * your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
+ * NON INFRINGEMENT.  See the GNU General Public License for more
+ * details.
+ */
+
 #ifndef __IOCHANNEL_H__
 #define __IOCHANNEL_H__
 
@@ -7,9 +21,7 @@
  * Everything needed for IOPart-GuestPart communication is define in
  * this file. Note: Everything is OS-independent because this file is
  * used by Windows, Linux and possible EFI drivers.
- */
-
-/*
+ *
  * Communication flow between the IOPart and GuestPart uses the channel headers
  * channel state. The following states are currently being used:
  *       UNINIT(All Zeroes), CHANNEL_ATTACHING, CHANNEL_ATTACHED, CHANNEL_OPENED
@@ -30,14 +42,10 @@
  */
 
 #include <linux/uuid.h>
+#include <linux/skbuff.h>
 
-#include <linux/dma-direction.h>
 #include "channel.h"
 
-#define VISOR_VHBA_CHANNEL_SIGNATURE VISOR_CHANNEL_SIGNATURE
-#define VISOR_VNIC_CHANNEL_SIGNATURE VISOR_CHANNEL_SIGNATURE
-#define VISOR_VSWITCH_CHANNEL_SIGNATURE VISOR_CHANNEL_SIGNATURE
-
 /*
  * Must increment these whenever you insert or delete fields within this channel
  * struct. Also increment whenever you change the meaning of fields within this
@@ -47,82 +55,73 @@
  */
 #define VISOR_VHBA_CHANNEL_VERSIONID 2
 #define VISOR_VNIC_CHANNEL_VERSIONID 2
-#define VISOR_VSWITCH_CHANNEL_VERSIONID 1
-
-#define VISOR_VHBA_CHANNEL_OK_CLIENT(ch) \
-	(visor_check_channel(ch, visor_vhba_channel_uuid, \
-			     "vhba", MIN_IO_CHANNEL_SIZE, \
-			     VISOR_VHBA_CHANNEL_VERSIONID, \
-			     VISOR_VHBA_CHANNEL_SIGNATURE))
-
-#define VISOR_VNIC_CHANNEL_OK_CLIENT(ch) \
-	(visor_check_channel(ch, visor_vnic_channel_uuid, \
-			     "vnic", MIN_IO_CHANNEL_SIZE, \
-			     VISOR_VNIC_CHANNEL_VERSIONID, \
-			     VISOR_VNIC_CHANNEL_SIGNATURE))
 
 /*
  * Everything necessary to handle SCSI & NIC traffic between Guest Partition and
  * IO Partition is defined below.
  */
 
-/* Defines and enums. */
-#define MINNUM(a, b) (((a) < (b)) ? (a) : (b))
-#define MAXNUM(a, b) (((a) > (b)) ? (a) : (b))
-
-/* Define the two queues per data channel between iopart and ioguestparts. */
-/* Used by ioguestpart to 'insert' signals to iopart. */
+/*
+ * Define the two queues per data channel between iopart and ioguestparts.
+ *	IOCHAN_TO_IOPART -- used by guest to 'insert' signals to iopart.
+ *	IOCHAN_FROM_IOPART -- used by guest to 'remove' signals from IO part.
+ */
 #define IOCHAN_TO_IOPART 0
-/* Used by ioguestpart to 'remove' signals from iopart, same previous queue. */
 #define IOCHAN_FROM_IOPART 1
 
 /* Size of cdb - i.e., SCSI cmnd */
 #define MAX_CMND_SIZE 16
 
-#define MAX_SENSE_SIZE 64
+/* Unisys-specific DMA direction values */
+enum uis_dma_data_direction {
+	UIS_DMA_BIDIRECTIONAL = 0,
+	UIS_DMA_TO_DEVICE = 1,
+	UIS_DMA_FROM_DEVICE = 2,
+	UIS_DMA_NONE = 3
+};
 
+#define MAX_SENSE_SIZE 64
 #define MAX_PHYS_INFO 64
 
-/* Various types of network packets that can be sent in cmdrsp. */
+/*
+ * enum net_types - Various types of network packets that can be sent in cmdrsp.
+ * @NET_RCV_POST:	Submit buffer to hold receiving incoming packet.
+ * @NET_RCV:		visornic -> uisnic. Incoming packet received.
+ * @NET_XMIT:		uisnic -> visornic. For outgoing packet.
+ * @NET_XMIT_DONE:	visornic -> uisnic. Outgoing packet xmitted.
+ * @NET_RCV_ENBDIS:	uisnic -> visornic. Enable/Disable packet reception.
+ * @NET_RCV_ENBDIS_ACK:	visornic -> uisnic. Acknowledge enable/disable packet.
+ * @NET_RCV_PROMISC:	uisnic -> visornic. Enable/Disable promiscuous mode.
+ * @NET_CONNECT_STATUS:	visornic -> uisnic. Indicate the loss or restoration of
+ *			a network connection.
+ * @NET_MACADDR:	uisnic -> visornic. Indicates the client has requested
+ *			to update it's MAC address.
+ * @NET_MACADDR_ACK:	MAC address acknowledge.
+ */
 enum net_types {
-	NET_RCV_POST = 0,	/*
-				 * Submit buffer to hold receiving
-				 * incoming packet
-				 */
-	/* visornic -> uisnic */
-	NET_RCV,		/* incoming packet received */
-	/* uisnic -> visornic */
-	NET_XMIT,		/* for outgoing net packets */
-	/* visornic -> uisnic */
-	NET_XMIT_DONE,		/* outgoing packet xmitted */
-	/* uisnic -> visornic */
-	NET_RCV_ENBDIS,		/* enable/disable packet reception */
-	/* visornic -> uisnic */
-	NET_RCV_ENBDIS_ACK,	/* acknowledge enable/disable packet */
-				/* reception */
-	/* uisnic -> visornic */
-	NET_RCV_PROMISC,	/* enable/disable promiscuous mode */
-	/* visornic -> uisnic */
-	NET_CONNECT_STATUS,	/*
-				 * indicate the loss or restoration of a network
-				 * connection
-				 */
-	/* uisnic -> visornic */
-	NET_MACADDR,		/*
-				 * Indicates the client has requested to update
-				 * it's MAC address
-				 */
-	NET_MACADDR_ACK,	/* MAC address acknowledge */
-
+	NET_RCV_POST = 0,
+	NET_RCV,
+	NET_XMIT,
+	NET_XMIT_DONE,
+	NET_RCV_ENBDIS,
+	NET_RCV_ENBDIS_ACK,
+	/* Reception */
+	NET_RCV_PROMISC,
+	NET_CONNECT_STATUS,
+	NET_MACADDR,
+	NET_MACADDR_ACK,
 };
 
-#define ETH_MIN_DATA_SIZE 46	/* minimum eth data size */
+/* Minimum eth data size */
+#define ETH_MIN_DATA_SIZE 46
 #define ETH_MIN_PACKET_SIZE (ETH_HLEN + ETH_MIN_DATA_SIZE)
 
-#define VISOR_ETH_MAX_MTU 16384	/* maximum data size */
+/* Maximum data size */
+#define VISOR_ETH_MAX_MTU 16384
 
 #ifndef MAX_MACADDR_LEN
-#define MAX_MACADDR_LEN 6	/* number of bytes in MAC address */
+/* Number of bytes in MAC address */
+#define MAX_MACADDR_LEN 6
 #endif
 
 /* Various types of scsi task mgmt commands. */
@@ -154,12 +153,16 @@ struct guest_phys_info {
 	u64 length;
 } __packed;
 
-#define GPI_ENTRIES_PER_PAGE (PAGE_SIZE / sizeof(struct guest_phys_info))
-
+/*
+ * struct uisscsi_dest
+ * @channel: Bus number.
+ * @id:      Target number.
+ * @lun:     Logical unit number.
+ */
 struct uisscsi_dest {
-	u32 channel;	/* channel == bus number */
-	u32 id;		/* id == target number */
-	u32 lun;	/* lun == logical unit number */
+	u32 channel;
+	u32 id;
+	u32 lun;
 } __packed;
 
 struct vhba_wwnn {
@@ -168,61 +171,77 @@ struct vhba_wwnn {
 } __packed;
 
 /*
- * WARNING: Values stired in this structure must contain maximum counts (not
+ * struct vhba_config_max
+ * @max_channel: Maximum channel for devices attached to this bus.
+ * @max_id:	 Maximum SCSI ID for devices attached to bus.
+ * @max_lun:	 Maximum SCSI LUN for devices attached to bus.
+ * @cmd_per_lun: Maximum number of outstanding commands per LUN.
+ * @max_io_size: Maximum io size for devices attached to this bus. Max io size
+ *		 is often determined by the resource of the hba.
+ *		 e.g Max scatter gather list length * page size / sector size.
+ *
+ * WARNING: Values stored in this structure must contain maximum counts (not
  * maximum values).
+ *
+ * 20 bytes
  */
-struct vhba_config_max {/* 20 bytes */
-	u32 max_channel;/* maximum channel for devices attached to this bus */
-	u32 max_id;	/* maximum SCSI ID for devices attached to bus */
-	u32 max_lun;	/* maximum SCSI LUN for devices attached to bus */
-	u32 cmd_per_lun;/* maximum number of outstanding commands per LUN */
-	u32 max_io_size;/* maximum io size for devices attached to this bus */
-	/* max io size is often determined by the resource of the hba. e.g */
-	/* max scatter gather list length * page size / sector size */
+struct vhba_config_max {
+	u32 max_channel;
+	u32 max_id;
+	u32 max_lun;
+	u32 cmd_per_lun;
+	u32 max_io_size;
 } __packed;
 
+/*
+ * struct uiscmdrsp_scsi
+ *
+ * @handle:		The handle to the cmd that was received. Send it back as
+ *			is in the rsp packet.
+ * @cmnd:		The cdb for the command.
+ * @bufflen:		Length of data to be transferred out or in.
+ * @guest_phys_entries:	Number of entries in scatter-gather list.
+ * @struct gpi_list:	Physical address information for each fragment.
+ * @data_dir:		Direction of the data, if any.
+ * @struct vdest:	Identifies the virtual hba, id, channel, lun to which
+ *			cmd was sent.
+ * @linuxstat:		Original Linux status used by Linux vdisk.
+ * @scsistat:		The scsi status.
+ * @addlstat:		Non-scsi status.
+ * @sensebuf:		Sense info in case cmd failed. sensebuf holds the
+ *			sense_data struct. See sense_data struct for more
+ *			details.
+ * @*vdisk:		Pointer to the vdisk to clean up when IO completes.
+ * @no_disk_result:	Used to return no disk inquiry result when
+ *			no_disk_result is set to 1
+ *			scsi.scsistat is SAM_STAT_GOOD
+ *			scsi.addlstat is 0
+ *			scsi.linuxstat is SAM_STAT_GOOD
+ *			That is, there is NO error.
+ */
 struct uiscmdrsp_scsi {
-	u64 handle;		/* the handle to the cmd that was received */
-				/* send it back as is in the rsp packet.  */
-	u8 cmnd[MAX_CMND_SIZE];	/* the cdb for the command */
-	u32 bufflen;		/* length of data to be transferred out or in */
-	u16 guest_phys_entries;	/* Number of entries in scatter-gather list */
-	struct guest_phys_info gpi_list[MAX_PHYS_INFO];	/* physical address
-							 * information for each
-							 * fragment
-							 */
-	enum dma_data_direction data_dir; /* direction of the data, if any */
-	struct uisscsi_dest vdest;	/* identifies the virtual hba, id, */
-					/* channel, lun to which cmd was sent */
-
+	u64 handle;
+	u8 cmnd[MAX_CMND_SIZE];
+	u32 bufflen;
+	u16 guest_phys_entries;
+	struct guest_phys_info gpi_list[MAX_PHYS_INFO];
+	u32 data_dir;
+	struct uisscsi_dest vdest;
 	/* Needed to queue the rsp back to cmd originator. */
-	int linuxstat;		/* original Linux status used by Linux vdisk */
-	u8 scsistat;		/* the scsi status */
-	u8 addlstat;		/* non-scsi status */
+	int linuxstat;
+	u8 scsistat;
+	u8 addlstat;
 #define ADDL_SEL_TIMEOUT 4
-
 	/* The following fields are need to determine the result of command. */
-	 u8 sensebuf[MAX_SENSE_SIZE];	/* sense info in case cmd failed; */
-	/* sensebuf holds the sense_data struct; */
-	/* See sense_data struct for more details. */
-	void *vdisk; /* Pointer to the vdisk to clean up when IO completes. */
+	u8 sensebuf[MAX_SENSE_SIZE];
+	void *vdisk;
 	int no_disk_result;
-	/*
-	 * Used to return no disk inquiry result
-	 * when no_disk_result is set to 1,
-	 * scsi.scsistat is SAM_STAT_GOOD
-	 * scsi.addlstat is 0
-	 * scsi.linuxstat is SAM_STAT_GOOD
-	 * That is, there is NO error.
-	 */
 } __packed;
 
 /*
  * Defines to support sending correct inquiry result when no disk is
  * configured.
- */
-
-/*
+ *
  * From SCSI SPC2 -
  *
  * If the target is not capable of supporting a device on this logical unit, the
@@ -234,22 +253,35 @@ struct uiscmdrsp_scsi {
  * connected to this logical unit.
  */
 
-#define DEV_NOT_CAPABLE 0x7f	/*
-				 * peripheral qualifier of 0x3
-				 * peripheral type of 0x1f
-				 * specifies no device but target present
-				 */
-
-#define DEV_DISK_CAPABLE_NOT_PRESENT 0x20	/* peripheral qualifier of 0x1
-						 * peripheral type of 0 - disk
-						 * Specifies device capable, but
-						 * not present
-						 */
+/*
+ * Peripheral qualifier of 0x3
+ * Peripheral type of 0x1f
+ * Specifies no device but target present
+ */
+#define DEV_NOT_CAPABLE 0x7f
+/*
+ * Peripheral qualifier of 0x1
+ * Peripheral type of 0 - disk
+ * Specifies device capable, but not present
+ */
+#define DEV_DISK_CAPABLE_NOT_PRESENT 0x20
+/* HiSup = 1; shows support for report luns must be returned for lun 0. */
+#define DEV_HISUPPORT 0x10
 
-#define DEV_HISUPPORT 0x10	/*
-				 * HiSup = 1; shows support for report luns
-				 * must be returned for lun 0.
-				 */
+/*
+ * Peripheral qualifier of 0x3
+ * Peripheral type of 0x1f
+ * Specifies no device but target present
+ */
+#define DEV_NOT_CAPABLE 0x7f
+/*
+ * Peripheral qualifier of 0x1
+ * Peripheral type of 0 - disk
+ * Specifies device capable, but not present
+ */
+#define DEV_DISK_CAPABLE_NOT_PRESENT 0x20
+/* HiSup = 1; shows support for report luns must be returned for lun 0. */
+#define DEV_HISUPPORT 0x10
 
 /*
  * NOTE: Linux code assumes inquiry contains 36 bytes. Without checking length
@@ -258,11 +290,12 @@ struct uiscmdrsp_scsi {
  * inquiry result.
  */
 #define NO_DISK_INQUIRY_RESULT_LEN 36
-
-#define MIN_INQUIRY_RESULT_LEN 5 /* 5 bytes minimum for inquiry result */
+/* 5 bytes minimum for inquiry result */
+#define MIN_INQUIRY_RESULT_LEN 5
 
 /* SCSI device version for no disk inquiry result */
-#define SCSI_SPC2_VER 4	/* indicates SCSI SPC2 (SPC3 is 5) */
+/* indicates SCSI SPC2 (SPC3 is 5) */
+#define SCSI_SPC2_VER 4
 
 /* Struct and Defines to support sense information. */
 
@@ -297,35 +330,48 @@ struct sense_data {
 	u8 sense_key_specific[3];
 } __packed;
 
+/*
+ * struct net_pkt_xmt
+ * @len:		    Full length of data in the packet.
+ * @num_frags:		    Number of fragments in frags containing data.
+ * @struct phys_info frags: Physical page information.
+ * @ethhdr:		    The ethernet header.
+ * @struct lincsum:	    These are needed for csum at uisnic end.
+ *      @valid:	    1 = struct is valid - else ignore.
+ *      @hrawoffv:  1 = hwrafoff is valid.
+ *      @nhrawoffv: 1 = nhwrafoff is valid.
+ *      @protocol:  Specifies packet protocol.
+ *      @csum:	    Value used to set skb->csum at IOPart.
+ *      @hrawoff:   Value used to set skb->h.raw at IOPart. hrawoff points to
+ *		    the start of the TRANSPORT LAYER HEADER.
+ *      @nhrawoff:  Value used to set skb->nh.raw at IOPart. nhrawoff points to
+ *		    the start of the NETWORK LAYER HEADER.
+ *
+ * NOTE:
+ * The full packet is described in frags but the ethernet header is
+ * separately kept in ethhdr so that uisnic doesn't have "MAP" the
+ * guest memory to get to the header. uisnic needs ethhdr to
+ * determine how to route the packet.
+ */
 struct net_pkt_xmt {
-	int len;	/* full length of data in the packet */
-	int num_frags;	/* number of fragments in frags containing data */
-	struct phys_info frags[MAX_PHYS_INFO];	/* physical page information */
-	char ethhdr[ETH_HLEN];	/* the ethernet header  */
+	int len;
+	int num_frags;
+	struct phys_info frags[MAX_PHYS_INFO];
+	char ethhdr[ETH_HLEN];
 	struct {
-		/* These are needed for csum at uisnic end */
-		u8 valid;	/* 1 = struct is valid - else ignore */
-		u8 hrawoffv;	/* 1 = hwrafoff is valid */
-		u8 nhrawoffv;	/* 1 = nhwrafoff is valid */
-		__be16 protocol;	/* specifies packet protocol */
-		__wsum csum;	/* value used to set skb->csum at IOPart */
-		u32 hrawoff;	/* value used to set skb->h.raw at IOPart */
-		/* hrawoff points to the start of the TRANSPORT LAYER HEADER */
-		u32 nhrawoff;	/* value used to set skb->nh.raw at IOPart */
-		/* nhrawoff points to the start of the NETWORK LAYER HEADER */
+		u8 valid;
+		u8 hrawoffv;
+		u8 nhrawoffv;
+		__be16 protocol;
+		__wsum csum;
+		u32 hrawoff;
+		u32 nhrawoff;
 	} lincsum;
-
-	    /*
-	     * NOTE:
-	     * The full packet is described in frags but the ethernet header is
-	     * separately kept in ethhdr so that uisnic doesn't have "MAP" the
-	     * guest memory to get to the header. uisnic needs ethhdr to
-	     * determine how to route the packet.
-	     */
 } __packed;
 
 struct net_pkt_xmtdone {
-	u32 xmt_done_result; /* result of NET_XMIT */
+	/* Result of NET_XMIT */
+	u32 xmt_done_result;
 } __packed;
 
 /*
@@ -341,14 +387,12 @@ struct net_pkt_xmtdone {
 	((VISOR_ETH_MAX_MTU + ETH_HLEN + RCVPOST_BUF_SIZE - 1) \
 	 / RCVPOST_BUF_SIZE)
 
-/*
- * rcv buf size must be large enough to include ethernet data len + ethernet
+/* rcv buf size must be large enough to include ethernet data len + ethernet
  * header len - we are choosing 2K because it is guaranteed to be describable.
  */
 struct net_pkt_rcvpost {
 	/* Physical page information for the single fragment 2K rcv buf */
 	struct phys_info frag;
-
 	/*
 	 * Ensures that receive posts are returned to the adapter which we sent
 	 * them from originally.
@@ -358,172 +402,157 @@ struct net_pkt_rcvpost {
 } __packed;
 
 /*
+ * struct net_pkt_rcv
+ * @rcv_done_len:	Length of the received data.
+ * @numrcvbufs:		Contains the incoming data. Guest side MUST chain these
+ *			together.
+ * @*rcvbuf:		List of chained rcvbufa. Each entry is a receive buffer
+ *			provided by NET_RCV_POST. NOTE: First rcvbuf in the
+ *			chain will also be provided in net.buf.
+ * @unique_num:
+ * @rcvs_dropped_delta:
+ *
  * The number of rcvbuf that can be chained is based on max mtu and size of each
  * rcvbuf.
  */
 struct net_pkt_rcv {
-	u32 rcv_done_len; /* length of received data */
-
-	/*
-	 * numrcvbufs: contain the incoming data; guest side MUST chain these
-	 * together.
-	 */
+	u32 rcv_done_len;
 	u8 numrcvbufs;
-
-	void *rcvbuf[MAX_NET_RCV_CHAIN]; /* list of chained rcvbufs */
-
-	/* Each entry is a receive buffer provided by NET_RCV_POST. */
-	/* NOTE: first rcvbuf in the chain will also be provided in net.buf. */
+	void *rcvbuf[MAX_NET_RCV_CHAIN];
 	u64 unique_num;
 	u32 rcvs_dropped_delta;
 } __packed;
 
 struct net_pkt_enbdis {
 	void *context;
-	u16 enable; /* 1 = enable, 0 = disable */
+	/* 1 = enable, 0 = disable */
+	u16 enable;
 } __packed;
 
 struct net_pkt_macaddr {
 	void *context;
-	u8 macaddr[MAX_MACADDR_LEN]; /* 6 bytes */
+	/* 6 bytes */
+	u8 macaddr[MAX_MACADDR_LEN];
 } __packed;
 
-/* cmd rsp packet used for VNIC network traffic  */
+/*
+ * struct uiscmdrsp_net - cmd rsp packet used for VNIC network traffic.
+ * @enum type:
+ * @*buf:
+ * @union:
+ *	@struct xmt:	 Used for NET_XMIT.
+ *	@struct xmtdone: Used for NET_XMIT_DONE.
+ *	@struct rcvpost: Used for NET_RCV_POST.
+ *	@struct rcv:	 Used for NET_RCV.
+ *	@struct enbdis:	 Used for NET_RCV_ENBDIS, NET_RCV_ENBDIS_ACK,
+ *			 NET_RCV_PROMSIC, and NET_CONNECT_STATUS.
+ *	@struct macaddr:
+ */
 struct uiscmdrsp_net {
 	enum net_types type;
 	void *buf;
 	union {
-		struct net_pkt_xmt xmt;		/* used for NET_XMIT */
-		struct net_pkt_xmtdone xmtdone;	/* used for NET_XMIT_DONE */
-		struct net_pkt_rcvpost rcvpost;	/* used for NET_RCV_POST */
-		struct net_pkt_rcv rcv;		/* used for NET_RCV */
-		struct net_pkt_enbdis enbdis;	/* used for NET_RCV_ENBDIS, */
-						/* NET_RCV_ENBDIS_ACK,  */
-						/* NET_RCV_PROMSIC, */
-						/* and NET_CONNECT_STATUS */
+		struct net_pkt_xmt xmt;
+		struct net_pkt_xmtdone xmtdone;
+		struct net_pkt_rcvpost rcvpost;
+		struct net_pkt_rcv rcv;
+		struct net_pkt_enbdis enbdis;
 		struct net_pkt_macaddr macaddr;
 	};
 } __packed;
 
+/*
+ * struct uiscmdrsp_scsitaskmgmt
+ * @enum tasktype:	 The type of task.
+ * @struct vdest:	 The vdisk for which this task mgmt is generated.
+ * @handle:		 This is a handle that the guest has saved off for its
+ *			 own use. The handle value is preserved by iopart and
+ *			 returned as in task mgmt rsp.
+ * @notify_handle:	 For Linux guests, this is a pointer to wait_queue_head
+ *			 that a thread is waiting on to see if the taskmgmt
+ *			 command has completed. When the rsp is received by
+ *			 guest, the thread receiving the response uses this to
+ *			 notify the thread waiting for taskmgmt command
+ *			 completion. It's value is preserved by iopart and
+ *			 returned as in the task mgmt rsp.
+ * @notifyresult_handle: This is a handle to the location in the guest where
+ *			 the result of the taskmgmt command (result field) is
+ *			 saved to when the response is handled. It's value is
+ *			 preserved by iopart and returned as is in the task mgmt
+ *			 rsp.
+ * @result:		 Result of taskmgmt command - set by IOPart.
+ */
 struct uiscmdrsp_scsitaskmgmt {
-	/* The type of task. */
 	enum task_mgmt_types tasktype;
-
-	/* The vdisk for which this task mgmt is generated. */
 	struct uisscsi_dest vdest;
-
-	/*
-	 * This is a handle that the guest has saved off for its own use.
-	 * The handle value is preserved by iopart and returned as in task
-	 * mgmt rsp.
-	 */
 	u64 handle;
-
-	/*
-	 * For Linux guests, this is a pointer to wait_queue_head that a
-	 * thread is waiting on to see if the taskmgmt command has completed.
-	 * When the rsp is received by guest, the thread receiving the
-	 * response uses this to notify the thread waiting for taskmgmt
-	 * command completion. It's value is preserved by iopart and returned
-	 * as in the task mgmt rsp.
-	 */
 	u64 notify_handle;
-
-	/*
-	 * This is a handle to the location in the guest where the result of
-	 * the taskmgmt command (result field) is saved to when the response
-	 * is handled. It's value is preserved by iopart and returned as in
-	 * the task mgmt rsp.
-	 */
 	u64 notifyresult_handle;
-
-	/* Result of taskmgmt command - set by IOPart - values are: */
 	char result;
 
 #define TASK_MGMT_FAILED 0
 } __packed;
 
-/* Used by uissd to send disk add/remove notifications to Guest. */
-/* Note that the vHba pointer is not used by the Client/Guest side. */
-struct uiscmdrsp_disknotify {
-	u8 add;			/* 0-remove, 1-add */
-	void *v_hba;		/* channel info to route msg */
-	u32 channel, id, lun;	/* SCSI Path of Disk to added or removed */
-} __packed;
-
 /*
- * The following is used by virthba/vSCSI to send the Acquire/Release commands
- * to the IOVM.
+ * struct uiscmdrsp_disknotify - Used by uissd to send disk add/remove
+ *				 notifications to Guest.
+ * @add:     0-remove, 1-add.
+ * @*v_hba:  Channel info to route msg.
+ * @channel: SCSI Path of Disk to added or removed.
+ * @id:	     SCSI Path of Disk to added or removed.
+ * @lun:     SCSI Path of Disk to added or removed.
+ *
+ * Note that the vHba pointer is not used by the Client/Guest side.
  */
-struct uiscmdrsp_vdiskmgmt {
-	/* The type of task */
-	enum vdisk_mgmt_types vdisktype;
-
-	/* The vdisk for which this task mgmt is generated */
-	struct uisscsi_dest vdest;
-
-	/*
-	 * This is a handle that the guest has saved off for its own use. It's
-	 * value is preserved by iopart and returned as in the task mgmt rsp.
-	 */
-	u64 handle;
-
-	/*
-	 * For Linux guests, this is a pointer to wait_queue_head that a
-	 * thread is waiting on to see if the tskmgmt command has completed.
-	 * When the rsp is received by guest, the thread receiving the
-	 * response uses this to notify the thread waiting for taskmgmt
-	 * command completion. It's value is preserved by iopart and returned
-	 * as in the task mgmt rsp.
-	 */
-	u64 notify_handle;
-
-	/*
-	 * Handle to the location in guest where the result of the
-	 * taskmgmt command (result field) is saved to when the response
-	 * is handled. It's value is preserved by iopart and returned as in
-	 * the task mgmt rsp.
-	 */
-	u64 notifyresult_handle;
-
-	/* Result of taskmgmt command - set by IOPart - values are: */
-	char result;
+struct uiscmdrsp_disknotify {
+	u8 add;
+	void *v_hba;
+	u32 channel, id, lun;
 } __packed;
 
 /* Keeping cmd and rsp info in one structure for now cmd rsp packet for SCSI */
 struct uiscmdrsp {
 	char cmdtype;
-
-/* Describes what type of information is in the struct */
+	/* Describes what type of information is in the struct */
 #define CMD_SCSI_TYPE	      1
 #define CMD_NET_TYPE	      2
 #define CMD_SCSITASKMGMT_TYPE 3
 #define CMD_NOTIFYGUEST_TYPE  4
-#define CMD_VDISKMGMT_TYPE    5
 	union {
 		struct uiscmdrsp_scsi scsi;
 		struct uiscmdrsp_net net;
 		struct uiscmdrsp_scsitaskmgmt scsitaskmgmt;
 		struct uiscmdrsp_disknotify disknotify;
-		struct uiscmdrsp_vdiskmgmt vdiskmgmt;
 	};
 	/* Send the response when the cmd is done (scsi and scsittaskmgmt). */
 	void *private_data;
-	struct uiscmdrsp *next;	/* General Purpose Queue Link */
-	struct uiscmdrsp *activeQ_next;	/* Pointer to the nextactive commands */
-	struct uiscmdrsp *activeQ_prev;	/* Pointer to the prevactive commands */
+	/* General Purpose Queue Link */
+	struct uiscmdrsp *next;
+	/* Pointer to the nextactive commands */
+	struct uiscmdrsp *activeQ_next;
+	/* Pointer to the prevactive commands */
+	struct uiscmdrsp *activeQ_prev;
 } __packed;
 
+/* total = 28 bytes */
 struct iochannel_vhba {
-	struct vhba_wwnn wwnn;		/* 8 bytes */
-	struct vhba_config_max max;	/* 20 bytes */
-} __packed;				/* total = 28 bytes */
+	/* 8 bytes */
+	struct vhba_wwnn wwnn;
+	/* 20 bytes */
+	struct vhba_config_max max;
+} __packed;
+
 struct iochannel_vnic {
-	u8 macaddr[6];			/* 6 bytes */
-	u32 num_rcv_bufs;		/* 4 bytes */
-	u32 mtu;			/* 4 bytes */
-	uuid_le zone_uuid;		/* 16 bytes */
+	/* 6 bytes */
+	u8 macaddr[6];
+	/* 4 bytes */
+	u32 num_rcv_bufs;
+	/* 4 bytes */
+	u32 mtu;
+	/* 16 bytes */
+	guid_t zone_guid;
 } __packed;
+
 /*
  * This is just the header of the IO channel. It is assumed that directly after
  * this header there is a large region of memory which contains the command and
@@ -544,10 +573,11 @@ struct visor_io_channel {
 } __packed;
 
 /* INLINE functions for initializing and accessing I/O data channels. */
-#define SIZEOF_CMDRSP (COVER(sizeof(struct uiscmdrsp), 64))
+#define SIZEOF_CMDRSP (64 * DIV_ROUND_UP(sizeof(struct uiscmdrsp), 64))
 
 /* Use 4K page sizes when passing page info between Guest and IOPartition. */
 #define PI_PAGE_SIZE 0x1000
 #define PI_PAGE_MASK 0x0FFF
 
-#endif /* __IOCHANNEL_H__ */
+/* __IOCHANNEL_H__ */
+#endif
diff --git a/drivers/staging/unisys/include/visorbus.h b/drivers/staging/unisys/include/visorbus.h
index de0635542fbd..e4ee38c3dbe4 100644
--- a/drivers/staging/unisys/include/visorbus.h
+++ b/drivers/staging/unisys/include/visorbus.h
@@ -1,5 +1,4 @@
-/* visorbus.h
- *
+/*
  * Copyright (C) 2010 - 2013 UNISYS CORPORATION
  * All rights reserved.
  *
@@ -23,7 +22,6 @@
  *
  *  There should be nothing in this file that is private to the visorbus
  *  bus implementation itself.
- *
  */
 
 #ifndef __VISORBUS_H__
@@ -31,20 +29,16 @@
 
 #include <linux/device.h>
 #include <linux/module.h>
-#include <linux/poll.h>
-#include <linux/kernel.h>
-#include <linux/uuid.h>
-#include <linux/seq_file.h>
 #include <linux/slab.h>
 
 #include "channel.h"
 
-struct visor_driver;
 struct visor_device;
 extern struct bus_type visorbus_type;
 
 typedef void (*visorbus_state_complete_func) (struct visor_device *dev,
 					      int status);
+
 struct visorchipset_state {
 	u32 created:1;
 	u32 attached:1;
@@ -54,11 +48,12 @@ struct visorchipset_state {
 	/* Remaining bits in this 32-bit word are unused. */
 };
 
-/** This struct describes a specific Supervisor channel, by providing its
- *  GUID, name, and sizes.
+/*
+ * This struct describes a specific Supervisor channel, by providing its
+ * GUID, name, and sizes.
  */
 struct visor_channeltype_descriptor {
-	const uuid_le guid;
+	const guid_t guid;
 	const char *name;
 };
 
@@ -109,8 +104,7 @@ struct visor_driver {
 	struct device_driver driver;
 };
 
-#define to_visor_driver(x) ((x) ? \
-	(container_of(x, struct visor_driver, driver)) : (NULL))
+#define to_visor_driver(x) (container_of(x, struct visor_driver, driver))
 
 /**
  * struct visor_device - A device type for things "plugged" into the visorbus
@@ -125,8 +119,8 @@ struct visor_driver {
  *				activity.
  * @being_removed:		Indicates that the device is being removed from
  *				the bus. Private bus driver use only.
- * @visordriver_callback_lock:	Used by the bus driver to lock when handling
- *				channel events.
+ * @visordriver_callback_lock:	Used by the bus driver to lock when adding and
+ *				removing devices.
  * @pausing:			Indicates that a change towards a paused state.
  *				is in progress. Only modified by the bus driver.
  * @resuming:			Indicates that a change towards a running state
@@ -141,37 +135,42 @@ struct visor_driver {
  *				hypervisor requests.
  * @vbus_hdr_info:		A pointer to header info. Private use by bus
  *				driver.
- * @partition_uuid:		Indicates client partion id. This should be the
+ * @partition_guid:		Indicates client partion id. This should be the
  *				same across all visor_devices in the current
  *				guest. Private use by bus driver only.
  */
 
 struct visor_device {
 	struct visorchannel *visorchannel;
-	uuid_le channel_type_guid;
+	guid_t channel_type_guid;
 	/* These fields are for private use by the bus driver only. */
 	struct device device;
 	struct list_head list_all;
 	struct timer_list timer;
 	bool timer_active;
 	bool being_removed;
-	struct mutex visordriver_callback_lock;
+	struct mutex visordriver_callback_lock; /* synchronize probe/remove */
 	bool pausing;
 	bool resuming;
 	u32 chipset_bus_no;
 	u32 chipset_dev_no;
 	struct visorchipset_state state;
-	uuid_le inst;
+	guid_t inst;
 	u8 *name;
 	struct controlvm_message_header *pending_msg_hdr;
 	void *vbus_hdr_info;
-	uuid_le partition_uuid;
+	guid_t partition_guid;
 	struct dentry *debugfs_dir;
 	struct dentry *debugfs_client_bus_info;
 };
 
 #define to_visor_device(x) container_of(x, struct visor_device, device)
 
+int visor_check_channel(struct channel_header *ch, struct device *dev,
+			const guid_t *expected_uuid, char *chname,
+			u64 expected_min_bytes,	u32 expected_version,
+			u64 expected_signature);
+
 int visorbus_register_visor_driver(struct visor_driver *drv);
 void visorbus_unregister_visor_driver(struct visor_driver *drv);
 int visorbus_read_channel(struct visor_device *dev,
@@ -183,7 +182,8 @@ int visorbus_write_channel(struct visor_device *dev,
 int visorbus_enable_channel_interrupts(struct visor_device *dev);
 void visorbus_disable_channel_interrupts(struct visor_device *dev);
 
-/* Levels of severity for diagnostic events, in order from lowest severity to
+/*
+ * Levels of severity for diagnostic events, in order from lowest severity to
  * highest (i.e. fatal errors are the most severe, and should always be logged,
  * but info events rarely need to be logged except during debugging). The
  * values DIAG_SEVERITY_ENUM_BEGIN and DIAG_SEVERITY_ENUM_END are not valid
@@ -207,7 +207,7 @@ int visorchannel_signalremove(struct visorchannel *channel, u32 queue,
 int visorchannel_signalinsert(struct visorchannel *channel, u32 queue,
 			      void *msg);
 bool visorchannel_signalempty(struct visorchannel *channel, u32 queue);
-uuid_le visorchannel_get_uuid(struct visorchannel *channel);
+const guid_t *visorchannel_get_guid(struct visorchannel *channel);
 
 #define BUS_ROOT_DEVICE UINT_MAX
 struct visor_device *visorbus_get_device_by_id(u32 bus_no, u32 dev_no,
diff --git a/drivers/staging/unisys/visorbus/controlvmchannel.h b/drivers/staging/unisys/visorbus/controlvmchannel.h
index ed045eff0e33..32ff5c1bb6ba 100644
--- a/drivers/staging/unisys/visorbus/controlvmchannel.h
+++ b/drivers/staging/unisys/visorbus/controlvmchannel.h
@@ -1,4 +1,5 @@
-/* Copyright (C) 2010 - 2015 UNISYS CORPORATION
+/*
+ * Copyright (C) 2010 - 2015 UNISYS CORPORATION
  * All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
@@ -19,60 +20,55 @@
 #include "channel.h"
 
 /* {2B3C2D10-7EF5-4ad8-B966-3448B7386B3D} */
-#define VISOR_CONTROLVM_CHANNEL_UUID \
-	UUID_LE(0x2b3c2d10, 0x7ef5, 0x4ad8, \
-		0xb9, 0x66, 0x34, 0x48, 0xb7, 0x38, 0x6b, 0x3d)
+#define VISOR_CONTROLVM_CHANNEL_GUID \
+	GUID_INIT(0x2b3c2d10, 0x7ef5, 0x4ad8, \
+		  0xb9, 0x66, 0x34, 0x48, 0xb7, 0x38, 0x6b, 0x3d)
 
-#define VISOR_CONTROLVM_CHANNEL_SIGNATURE VISOR_CHANNEL_SIGNATURE
 #define CONTROLVM_MESSAGE_MAX 64
 
-/* Must increment this whenever you insert or delete fields within
- * this channel struct.  Also increment whenever you change the meaning
- * of fields within this channel struct so as to break pre-existing
- * software.  Note that you can usually add fields to the END of the
- * channel struct withOUT needing to increment this.
+/*
+ * Must increment this whenever you insert or delete fields within this channel
+ * struct.  Also increment whenever you change the meaning of fields within this
+ * channel struct so as to break pre-existing software. Note that you can
+ * usually add fields to the END of the channel struct withOUT needing to
+ * increment this.
  */
 #define VISOR_CONTROLVM_CHANNEL_VERSIONID 1
 
-#define VISOR_CONTROLVM_CHANNEL_OK_CLIENT(ch) \
-	(visor_check_channel(ch, \
-			     VISOR_CONTROLVM_CHANNEL_UUID, \
-			     "controlvm", \
-			     sizeof(struct visor_controlvm_channel), \
-			     VISOR_CONTROLVM_CHANNEL_VERSIONID, \
-			     VISOR_CONTROLVM_CHANNEL_SIGNATURE))
-
 /* Defines for various channel queues */
-#define CONTROLVM_QUEUE_REQUEST	 0
-#define CONTROLVM_QUEUE_RESPONSE 1
-#define CONTROLVM_QUEUE_EVENT	 2
-#define CONTROLVM_QUEUE_ACK	 3
+#define CONTROLVM_QUEUE_REQUEST		0
+#define CONTROLVM_QUEUE_RESPONSE	1
+#define CONTROLVM_QUEUE_EVENT		2
+#define CONTROLVM_QUEUE_ACK		3
 
 /* Max num of messages stored during IOVM creation to be reused after crash */
 #define CONTROLVM_CRASHMSG_MAX 2
 
+/*
+ * struct visor_segment_state
+ * @enabled:   May enter other states.
+ * @active:    Assigned to active partition.
+ * @alive:     Configure message sent to service/server.
+ * @revoked:   Similar to partition state ShuttingDown.
+ * @allocated: Memory (device/port number) has been selected by Command.
+ * @known:     Has been introduced to the service/guest partition.
+ * @ready:     Service/Guest partition has responded to introduction.
+ * @operating: Resource is configured and operating.
+ * @reserved:  Natural alignment.
+ *
+ * Note: Don't use high bit unless we need to switch to ushort which is
+ * non-compliant.
+ */
 struct visor_segment_state  {
-	/* Bit 0: May enter other states */
 	u16 enabled:1;
-	/* Bit 1: Assigned to active partition */
 	u16 active:1;
-	/* Bit 2: Configure message sent to service/server */
 	u16 alive:1;
-	/* Bit 3: similar to partition state ShuttingDown */
 	u16 revoked:1;
-	/* Bit 4: memory (device/port number) has been selected by Command */
 	u16 allocated:1;
-	/* Bit 5: has been introduced to the service/guest partition */
 	u16 known:1;
-	/* Bit 6: service/Guest partition has responded to introduction */
 	u16 ready:1;
-	/* Bit 7: resource is configured and operating */
 	u16 operating:1;
-	/* Natural alignment*/
 	u16 reserved:8;
-/* Note: don't use high bit unless we need to switch to ushort
- * which is non-compliant
- */
 } __packed;
 
 static const struct visor_segment_state segment_state_running = {
@@ -87,74 +83,101 @@ static const struct visor_segment_state segment_state_standby = {
 	1, 1, 0, 0, 1, 1, 1, 0
 };
 
-/* Ids for commands that may appear in either queue of a ControlVm channel.
+/*
+ * enum controlvm_id
+ * @CONTROLVM_INVALID:
+ * @CONTROLVM_BUS_CREATE:		CP --> SP, GP.
+ * @CONTROLVM_BUS_DESTROY:		CP --> SP, GP.
+ * @CONTROLVM_BUS_CONFIGURE:		CP --> SP.
+ * @CONTROLVM_BUS_CHANGESTATE:		CP --> SP, GP.
+ * @CONTROLVM_BUS_CHANGESTATE_EVENT:	SP, GP --> CP.
+ * @CONTROLVM_DEVICE_CREATE:		CP --> SP, GP.
+ * @CONTROLVM_DEVICE_DESTROY:		CP --> SP, GP.
+ * @CONTROLVM_DEVICE_CONFIGURE:		CP --> SP.
+ * @CONTROLVM_DEVICE_CHANGESTATE:	CP --> SP, GP.
+ * @CONTROLVM_DEVICE_CHANGESTATE_EVENT:	SP, GP --> CP.
+ * @CONTROLVM_DEVICE_RECONFIGURE:	CP --> Boot.
+ * @CONTROLVM_CHIPSET_INIT:		CP --> SP, GP.
+ * @CONTROLVM_CHIPSET_STOP:		CP --> SP, GP.
+ * @CONTROLVM_CHIPSET_READY:		CP --> SP.
+ * @CONTROLVM_CHIPSET_SELFTEST:		CP --> SP.
  *
- *  Commands that are initiated by the command partition (CP), by an IO or
- *  console service partition (SP), or by a guest partition (GP)are:
- *  - issued on the RequestQueue queue (q #0) in the ControlVm channel
- *  - responded to on the ResponseQueue queue (q #1) in the ControlVm channel
+ * Ids for commands that may appear in either queue of a ControlVm channel.
  *
- *  Events that are initiated by an IO or console service partition (SP) or
- *  by a guest partition (GP) are:
- *  - issued on the EventQueue queue (q #2) in the ControlVm channel
- *  - responded to on the EventAckQueue queue (q #3) in the ControlVm channel
+ * Commands that are initiated by the command partition (CP), by an IO or
+ * console service partition (SP), or by a guest partition (GP) are:
+ * - issued on the RequestQueue queue (q #0) in the ControlVm channel
+ * - responded to on the ResponseQueue queue (q #1) in the ControlVm channel
+ *
+ * Events that are initiated by an IO or console service partition (SP) or
+ * by a guest partition (GP) are:
+ * - issued on the EventQueue queue (q #2) in the ControlVm channel
+ * - responded to on the EventAckQueue queue (q #3) in the ControlVm channel
  */
 enum controlvm_id {
 	CONTROLVM_INVALID = 0,
-	/* SWITCH commands required Parameter: SwitchNumber */
-	/* BUS commands required Parameter: BusNumber */
-	CONTROLVM_BUS_CREATE = 0x101,			/* CP --> SP, GP */
-	CONTROLVM_BUS_DESTROY = 0x102,			/* CP --> SP, GP */
-	CONTROLVM_BUS_CONFIGURE = 0x104,		/* CP --> SP */
-	CONTROLVM_BUS_CHANGESTATE = 0x105,		/* CP --> SP, GP */
-	CONTROLVM_BUS_CHANGESTATE_EVENT = 0x106,	/* SP, GP --> CP */
-/* DEVICE commands required Parameter: BusNumber, DeviceNumber */
-
-	CONTROLVM_DEVICE_CREATE = 0x201,		/* CP --> SP, GP */
-	CONTROLVM_DEVICE_DESTROY = 0x202,		/* CP --> SP, GP */
-	CONTROLVM_DEVICE_CONFIGURE = 0x203,		/* CP --> SP */
-	CONTROLVM_DEVICE_CHANGESTATE = 0x204,		/* CP --> SP, GP */
-	CONTROLVM_DEVICE_CHANGESTATE_EVENT = 0x205,	/* SP, GP --> CP */
-	CONTROLVM_DEVICE_RECONFIGURE = 0x206,		/* CP --> Boot */
-/* CHIPSET commands */
-	CONTROLVM_CHIPSET_INIT = 0x301,			/* CP --> SP, GP */
-	CONTROLVM_CHIPSET_STOP = 0x302,			/* CP --> SP, GP */
-	CONTROLVM_CHIPSET_READY = 0x304,		/* CP --> SP */
-	CONTROLVM_CHIPSET_SELFTEST = 0x305,		/* CP --> SP */
-
+	/*
+	 * SWITCH commands required Parameter: SwitchNumber.
+	 * BUS commands required Parameter: BusNumber
+	 */
+	CONTROLVM_BUS_CREATE = 0x101,
+	CONTROLVM_BUS_DESTROY = 0x102,
+	CONTROLVM_BUS_CONFIGURE = 0x104,
+	CONTROLVM_BUS_CHANGESTATE = 0x105,
+	CONTROLVM_BUS_CHANGESTATE_EVENT = 0x106,
+	/* DEVICE commands required Parameter: BusNumber, DeviceNumber */
+	CONTROLVM_DEVICE_CREATE = 0x201,
+	CONTROLVM_DEVICE_DESTROY = 0x202,
+	CONTROLVM_DEVICE_CONFIGURE = 0x203,
+	CONTROLVM_DEVICE_CHANGESTATE = 0x204,
+	CONTROLVM_DEVICE_CHANGESTATE_EVENT = 0x205,
+	CONTROLVM_DEVICE_RECONFIGURE = 0x206,
+	/* CHIPSET commands */
+	CONTROLVM_CHIPSET_INIT = 0x301,
+	CONTROLVM_CHIPSET_STOP = 0x302,
+	CONTROLVM_CHIPSET_READY = 0x304,
+	CONTROLVM_CHIPSET_SELFTEST = 0x305,
 };
 
+/*
+ * struct irq_info
+ * @reserved1:	     Natural alignment purposes
+ * @recv_irq_handle: Specifies interrupt handle. It is used to retrieve the
+ *		     corresponding interrupt pin from Monitor; and the interrupt
+ *		     pin is used to connect to the corresponding interrupt.
+ *		     Used by IOPart-GP only.
+ * @recv_irq_vector: Specifies interrupt vector. It, interrupt pin, and shared
+ *		     are used to connect to the corresponding interrupt.
+ *		     Used by IOPart-GP only.
+ * @recv_irq_shared: Specifies if the recvInterrupt is shared.  It, interrupt
+ *		     pin and vector are used to connect to 0 = not shared;
+ *		     1 = shared the corresponding interrupt.
+ *		     Used by IOPart-GP only.
+ * @reserved:	     Natural alignment purposes
+ */
 struct irq_info {
 	u64 reserved1;
-
-	 /* specifies interrupt handle. It is used to retrieve the
-	  *   corresponding interrupt pin from Monitor; and the
-	  *   interrupt pin is used to connect to the corresponding
-	  *   interrupt.  Used by IOPart-GP only.
-	  */
 	u64 recv_irq_handle;
-
-	 /* specifies interrupt vector. It, interrupt pin, and shared are
-	  *   used to connect to the corresponding interrupt.  Used by
-	  *   IOPart-GP only.
-	  */
 	u32 recv_irq_vector;
-
-    /* specifies if the recvInterrupt is shared.  It, interrupt pin
-     * and vector are used to connect to 0 = not shared; 1 = shared.
-     * the corresponding interrupt.  Used by IOPart-GP only.
-     */
 	u8 recv_irq_shared;
-	u8 reserved[3];	/* Natural alignment purposes */
+	u8 reserved[3];
 } __packed;
 
+/*
+ * struct efi_visor_indication
+ * @boot_to_fw_ui: Stop in UEFI UI
+ * @clear_nvram:   Clear NVRAM
+ * @clear_cmos:	   Clear CMOS
+ * @boot_to_tool:  Run install tool
+ * @reserved:	   Natural alignment
+ */
 struct efi_visor_indication  {
-	u64 boot_to_fw_ui:1;		/* Bit 0: Stop in uefi ui */
-	u64 clear_nvram:1;		/* Bit 1: Clear NVRAM */
-	u64 clear_cmos:1;		/* Bit 2: Clear CMOS */
-	u64 boot_to_tool:1;		/* Bit 3: Run install tool */
-	/* remaining bits are available */
-	u64 reserved:60;		/* Natural alignment */
+	u64 boot_to_fw_ui:1;
+	u64 clear_nvram:1;
+	u64 clear_cmos:1;
+	u64 boot_to_tool:1;
+	/* Remaining bits are available */
+	u64 reserved:60;
 } __packed;
 
 enum visor_chipset_feature {
@@ -162,182 +185,248 @@ enum visor_chipset_feature {
 	VISOR_CHIPSET_FEATURE_PARA_HOTPLUG = 0x00000002,
 };
 
-/* This is the common structure that is at the beginning of every
- *  ControlVm message (both commands and responses) in any ControlVm
- *  queue.  Commands are easily distinguished from responses by
- *  looking at the flags.response field.
+/*
+ * struct controlvm_message_header
+ * @id:		       See CONTROLVM_ID.
+ * @message_size:      Includes size of this struct + size of message.
+ * @segment_index:     Index of segment containing Vm message/information.
+ * @completion_status: Error status code or result of  message completion.
+ * @struct flags:
+ *	@failed:	     =1 in a response to signify failure.
+ *	@response_expected:  =1 in all messages that expect a response.
+ *	@server:	     =1 in all bus & device-related messages where the
+ *			     message receiver is to act as the bus or device
+ *			     server.
+ *	@test_message:	     =1 for testing use only (Control and Command
+ *			     ignore this).
+ *	@partial_completion: =1 if there are forthcoming responses/acks
+ *                           associated with this message.
+ *      @preserve:	     =1 this is to let us know to preserve channel
+ *			     contents.
+ *	@writer_in_diag:     =1 the DiagWriter is active in the Diagnostic
+ *			     Partition.
+ *	@reserve:	     Natural alignment.
+ * @reserved:	       Natural alignment.
+ * @message_handle:    Identifies the particular message instance.
+ * @payload_vm_offset: Offset of payload area from start of this instance.
+ * @payload_max_bytes: Maximum bytes allocated in payload area of ControlVm
+ *		       segment.
+ * @payload_bytes:     Actual number of bytes of payload area to copy between
+ *		       IO/Command. If non-zero, there is a payload to copy.
+ *
+ * This is the common structure that is at the beginning of every
+ * ControlVm message (both commands and responses) in any ControlVm
+ * queue.  Commands are easily distinguished from responses by
+ * looking at the flags.response field.
  */
 struct controlvm_message_header  {
-	u32 id;		/* See CONTROLVM_ID. */
-	/* For requests, indicates the message type. */
-	/* For responses, indicates the type of message we are responding to. */
-
-	/* Includes size of this struct + size of message */
+	u32 id;
+	/*
+	 * For requests, indicates the message type. For responses, indicates
+	 * the type of message we are responding to.
+	 */
 	u32 message_size;
-	/* Index of segment containing Vm message/information */
 	u32 segment_index;
-	/* Error status code or result of  message completion */
 	u32 completion_status;
 	struct  {
-		/* =1 in a response to signify failure */
 		u32 failed:1;
-		/* =1 in all messages that expect a response */
 		u32 response_expected:1;
-		/* =1 in all bus & device-related messages where the message
-		 * receiver is to act as the bus or device server
-		 */
 		u32 server:1;
-		/* =1 for testing use only (Control and Command ignore this */
 		u32 test_message:1;
-		/* =1 if there are forthcoming responses/acks associated
-		 * with this message
-		 */
 		u32 partial_completion:1;
-		/* =1 this is to let us know to preserve channel contents */
 		u32 preserve:1;
-		/* =1 the DiagWriter is active in the Diagnostic Partition */
 		u32 writer_in_diag:1;
-		/* Natural alignment */
 		u32 reserve:25;
 	} __packed flags;
-	/* Natural alignment */
 	u32 reserved;
-	/* Identifies the particular message instance */
 	u64 message_handle;
-	/* request instances with the corresponding response instance. */
-	/* Offset of payload area from start of this instance */
 	u64 payload_vm_offset;
-	/* Maximum bytes allocated in payload area of ControlVm segment */
 	u32 payload_max_bytes;
-	/* Actual number of bytes of payload area to copy between IO/Command */
 	u32 payload_bytes;
-	/* if non-zero, there is a payload to copy. */
 } __packed;
 
+/*
+ * struct controlvm_packet_device_create - For CONTROLVM_DEVICE_CREATE
+ * @bus_no:	    Bus # (0..n-1) from the msg receiver's end.
+ * @dev_no:	    Bus-relative (0..n-1) device number.
+ * @channel_addr:   Guest physical address of the channel, which can be
+ *		    dereferenced by the receiver of this ControlVm command.
+ * @channel_bytes:  Specifies size of the channel in bytes.
+ * @data_type_uuid: Specifies format of data in channel.
+ * @dev_inst_uuid:  Instance guid for the device.
+ * @irq_info intr:  Specifies interrupt information.
+ */
 struct controlvm_packet_device_create  {
-	u32 bus_no;		/* bus # (0..n-1) from the msg receiver's end */
-	u32 dev_no;		/* bus-relative (0..n-1) device number */
-	/* Guest physical address of the channel, which can be dereferenced by
-	 * the receiver of this ControlVm command
-	 */
+	u32 bus_no;
+	u32 dev_no;
 	u64 channel_addr;
-	u64 channel_bytes;	/* specifies size of the channel in bytes */
-	uuid_le data_type_uuid;	/* specifies format of data in channel */
-	uuid_le dev_inst_uuid;	/* instance guid for the device */
-	struct irq_info intr;	/* specifies interrupt information */
-} __packed;	/* for CONTROLVM_DEVICE_CREATE */
+	u64 channel_bytes;
+	guid_t data_type_guid;
+	guid_t dev_inst_guid;
+	struct irq_info intr;
+} __packed;
 
+/*
+ * struct controlvm_packet_device_configure - For CONTROLVM_DEVICE_CONFIGURE
+ * @bus_no: Bus number (0..n-1) from the msg receiver's perspective.
+ * @dev_no: Bus-relative (0..n-1) device number.
+ */
 struct controlvm_packet_device_configure  {
-	/* bus # (0..n-1) from the msg receiver's perspective */
 	u32 bus_no;
-	/* Control uses header SegmentIndex field to access bus number... */
-	u32 dev_no;	      /* bus-relative (0..n-1) device number */
-} __packed;	/* for CONTROLVM_DEVICE_CONFIGURE */
+	u32 dev_no;
+} __packed;
 
+/* Total 128 bytes */
 struct controlvm_message_device_create {
 	struct controlvm_message_header header;
 	struct controlvm_packet_device_create packet;
-} __packed;	/* total 128 bytes */
+} __packed;
 
+/* Total 56 bytes */
 struct controlvm_message_device_configure  {
 	struct controlvm_message_header header;
 	struct controlvm_packet_device_configure packet;
-} __packed;	/* total 56 bytes */
+} __packed;
 
-/* This is the format for a message in any ControlVm queue. */
+/*
+ * struct controlvm_message_packet - This is the format for a message in any
+ *                                   ControlVm queue.
+ * @struct create_bus:		For CONTROLVM_BUS_CREATE.
+ *	@bus_no:	     Bus # (0..n-1) from the msg receiver's perspective.
+ *	@dev_count:	     Indicates the max number of devices on this bus.
+ *	@channel_addr:	     Guest physical address of the channel, which can be
+ *			     dereferenced by the receiver of this ControlVM
+ *			     command.
+ *	@channel_bytes:	     Size of the channel.
+ *	@bus_data_type_uuid: Indicates format of data in bus channel.
+ *	@bus_inst_uuid:	     Instance uuid for the bus.
+ *
+ * @struct destroy_bus:		For CONTROLVM_BUS_DESTROY.
+ *	@bus_no: Bus # (0..n-1) from the msg receiver's perspective.
+ *	@reserved: Natural alignment purposes.
+ *
+ * @struct configure_bus:	For CONTROLVM_BUS_CONFIGURE.
+ *	@bus_no:	      Bus # (0..n-1) from the receiver's perspective.
+ *	@reserved1:	      For alignment purposes.
+ *	@guest_handle:	      This is used to convert guest physical address to
+ *			      physical address.
+ *	@recv_bus_irq_handle: Specifies interrupt info. It is used by SP to
+ *			      register to receive interrupts from the CP. This
+ *			      interrupt is used for bus level notifications.
+ *			      The corresponding sendBusInterruptHandle is kept
+ *			      in CP.
+ *
+ * @struct create_device:	For CONTROLVM_DEVICE_CREATE.
+ *
+ * @struct destroy_device:	For CONTROLVM_DEVICE_DESTROY.
+ *	@bus_no: Bus # (0..n-1) from the msg receiver's perspective.
+ *	@dev_no: Bus-relative (0..n-1) device number.
+ *
+ * @struct configure_device:	For CONTROLVM_DEVICE_CONFIGURE.
+ *
+ * @struct reconfigure_device:	For CONTROLVM_DEVICE_RECONFIGURE.
+ *	@bus_no: Bus # (0..n-1) from the msg receiver's perspective.
+ *	@dev_no: Bus-relative (0..n-1) device number.
+ *
+ * @struct bus_change_state:	For CONTROLVM_BUS_CHANGESTATE.
+ *	@bus_no:
+ *	@struct state:
+ *	@reserved: Natural alignment purposes.
+ *
+ * @struct device_change_state:	For CONTROLVM_DEVICE_CHANGESTATE.
+ *	@bus_no:
+ *	@dev_no:
+ *	@struct state:
+ *	@struct flags:
+ *		@phys_device: =1 if message is for a physical device.
+ *		@reserved:    Natural alignment.
+ *		@reserved1:   Natural alignment.
+ *	@reserved:    Natural alignment purposes.
+ *
+ * @struct device_change_state_event:	For CONTROLVM_DEVICE_CHANGESTATE_EVENT.
+ *	@bus_no:
+ *	@dev_no:
+ *	@struct state:
+ *	@reserved:     Natural alignment purposes.
+ *
+ * @struct init_chipset:	For CONTROLVM_CHIPSET_INIT.
+ *	@bus_count:	  Indicates the max number of busses.
+ *	@switch_count:    Indicates the max number of switches.
+ *	@enum features:
+ *	@platform_number:
+ *
+ * @struct chipset_selftest:	For CONTROLVM_CHIPSET_SELFTEST.
+ *      @options: Reserved.
+ *      @test:	  Bit 0 set to run embedded selftest.
+ *
+ * @addr:   A physical address of something, that can be dereferenced by the
+ *	    receiver of this ControlVm command.
+ *
+ * @handle: A handle of something (depends on command id).
+ */
 struct controlvm_message_packet  {
 	union  {
 		struct  {
-	/* bus # (0..n-1) from the msg receiver's perspective */
 			u32 bus_no;
-	/* indicates the max number of devices on this bus */
 			u32 dev_count;
-	/* Guest physical address of the channel, which can be
-	 * dereferenced by the receiver of this ControlVm command
-	 */
 			u64 channel_addr;
-			u64 channel_bytes;	/* size of the channel */
-	/* indicates format of data in bus channel*/
-			uuid_le bus_data_type_uuid;
-			uuid_le bus_inst_uuid;	/* instance uuid for the bus */
-		} __packed create_bus;	/* for CONTROLVM_BUS_CREATE */
+			u64 channel_bytes;
+			guid_t bus_data_type_guid;
+			guid_t bus_inst_guid;
+		} __packed create_bus;
 		struct  {
-	/* bus # (0..n-1) from the msg receiver's perspective */
 			u32 bus_no;
-			u32 reserved;	/* Natural alignment purposes */
-		} __packed destroy_bus;	/* for CONTROLVM_BUS_DESTROY */
+			u32 reserved;
+		} __packed destroy_bus;
 		struct  {
-	/* bus # (0..n-1) from the receiver's perspective */
 			u32 bus_no;
-			u32 reserved1;	/* for alignment purposes */
-	/* This is used to convert guest physical address to physical address */
+			u32 reserved1;
 			u64 guest_handle;
 			u64 recv_bus_irq_handle;
-				/* specifies interrupt info. It is used by SP
-				 * to register to receive interrupts from the
-				 * CP. This interrupt is used for bus level
-				 * notifications.  The corresponding
-				 * sendBusInterruptHandle is kept in CP.
-				 */
-		} __packed configure_bus;      /* for CONTROLVM_BUS_CONFIGURE */
-		/* for CONTROLVM_DEVICE_CREATE */
+		} __packed configure_bus;
 		struct controlvm_packet_device_create create_device;
 		struct  {
-		/* bus # (0..n-1) from the msg receiver's perspective */
 			u32 bus_no;
-			u32 dev_no;	/* bus-relative (0..n-1) device # */
-		} __packed destroy_device;    /* for CONTROLVM_DEVICE_DESTROY */
-		/* for CONTROLVM_DEVICE_CONFIGURE */
+			u32 dev_no;
+		} __packed destroy_device;
 		struct controlvm_packet_device_configure configure_device;
 		struct  {
-		/* bus # (0..n-1) from the msg receiver's perspective */
 			u32 bus_no;
-			u32 dev_no;	/* bus-relative (0..n-1) device # */
+			u32 dev_no;
 		} __packed reconfigure_device;
-			/* for CONTROLVM_DEVICE_RECONFIGURE */
 		struct  {
 			u32 bus_no;
 			struct visor_segment_state state;
-			u8 reserved[2];	/* Natural alignment purposes */
-		} __packed bus_change_state; /* for CONTROLVM_BUS_CHANGESTATE */
+			u8 reserved[2];
+		} __packed bus_change_state;
 		struct  {
 			u32 bus_no;
 			u32 dev_no;
 			struct visor_segment_state state;
 			struct  {
-				/* =1 if message is for a physical device */
 				u32 phys_device:1;
-				u32 reserved:31;	/* Natural alignment */
-				u32 reserved1;		/* Natural alignment */
+				u32 reserved:31;
+				u32 reserved1;
 			} __packed flags;
-			u8 reserved[2];	/* Natural alignment purposes */
+			u8 reserved[2];
 		} __packed device_change_state;
-			/* for CONTROLVM_DEVICE_CHANGESTATE */
 		struct  {
 			u32 bus_no;
 			u32 dev_no;
 			struct visor_segment_state state;
-			u8 reserved[6];	/* Natural alignment purposes */
+			u8 reserved[6];
 		} __packed device_change_state_event;
-			/* for CONTROLVM_DEVICE_CHANGESTATE_EVENT */
 		struct  {
-			/* indicates the max number of busses */
 			u32 bus_count;
-			/* indicates the max number of switches */
 			u32 switch_count;
 			enum visor_chipset_feature features;
-			u32 platform_number;	/* Platform Number */
-		} __packed init_chipset;	/* for CONTROLVM_CHIPSET_INIT */
+			u32 platform_number;
+		} __packed init_chipset;
 		struct  {
-			u32 options;	/* reserved */
-			u32 test;	/* bit 0 set to run embedded selftest */
+			u32 options;
+			u32 test;
 		} __packed chipset_selftest;
-			/* for CONTROLVM_CHIPSET_SELFTEST */
-		/* a physical address of something, that can be dereferenced
-		 * by the receiver of this ControlVm command
-		 */
 		u64 addr;
-		/* a handle of something (depends on command id) */
 		u64 handle;
 	};
 } __packed;
@@ -348,93 +437,139 @@ struct controlvm_message {
 	struct controlvm_message_packet cmd;
 } __packed;
 
+/*
+ * struct visor_controlvm_channel
+ * @struct header:
+ * @gp_controlvm:			Guest phys addr of this channel.
+ * @gp_partition_tables:		Guest phys addr of partition tables.
+ * @gp_diag_guest:			Guest phys addr of diagnostic channel.
+ * @gp_boot_romdisk:			Guest phys addr of (read* only) Boot
+ *					ROM disk.
+ * @gp_boot_ramdisk:			Guest phys addr of writable Boot RAM
+ *					disk.
+ * @gp_acpi_table:			Guest phys addr of acpi table.
+ * @gp_control_channel:			Guest phys addr of control channel.
+ * @gp_diag_romdisk:			Guest phys addr of diagnostic ROM disk.
+ * @gp_nvram:				Guest phys addr of NVRAM channel.
+ * @request_payload_offset:		Offset to request payload area.
+ * @event_payload_offset:		Offset to event payload area.
+ * @request_payload_bytes:		Bytes available in request payload area.
+ * @event_payload_bytes:		Bytes available in event payload area.
+ * @control_channel_bytes:
+ * @nvram_channel_bytes:		Bytes in PartitionNvram segment.
+ * @message_bytes:			sizeof(CONTROLVM_MESSAGE).
+ * @message_count:			CONTROLVM_MESSAGE_MAX.
+ * @gp_smbios_table:			Guest phys addr of SMBIOS tables.
+ * @gp_physical_smbios_table:		Guest phys addr of SMBIOS table.
+ * @gp_reserved:			VISOR_MAX_GUESTS_PER_SERVICE.
+ * @virtual_guest_firmware_image_base:	Guest physical address of EFI firmware
+ *					image base.
+ * @virtual_guest_firmware_entry_point:	Guest physical address of EFI firmware
+ *					entry point.
+ * @virtual_guest_firmware_image_size:	Guest EFI firmware image size.
+ * @virtual_guest_firmware_boot_base:	GPA = 1MB where EFI firmware image is
+ *					copied to.
+ * @virtual_guest_image_base:
+ * @virtual_guest_image_size:
+ * @prototype_control_channel_offset:
+ * @virtual_guest_partition_handle:
+ * @restore_action:			Restore Action field to restore the
+ *					guest partition.
+ * @dump_action:			For Windows guests it shows if the
+ *					visordisk is in dump mode.
+ * @nvram_fail_count:
+ * @saved_crash_message_count:		= CONTROLVM_CRASHMSG_MAX.
+ * @saved_crash_message_offset:		Offset to request payload area needed
+ *					for crash dump.
+ * @installation_error:			Type of error encountered during
+ *					installation.
+ * @installation_text_id:		Id of string to display.
+ * @installation_remaining_steps:	Number of remaining installation steps
+ *					(for progress bars).
+ * @tool_action:			VISOR_TOOL_ACTIONS Installation Action
+ *					field.
+ * @reserved: Alignment.
+ * @struct efi_visor_ind:
+ * @sp_reserved:
+ * @reserved2:				Force signals to begin on 128-byte
+ *					cache line.
+ * @struct request_queue:		Guest partition uses this queue to send
+ *					requests to Control.
+ * @struct response_queue:		Control uses this queue to respond to
+ *					service or guest partition request.
+ * @struct event_queue:			Control uses this queue to send events
+ *					to guest partition.
+ * @struct event_ack_queue:		Service or guest partition uses this
+ *					queue to ack Control events.
+ * @struct request_msg:			Request fixed-size message pool -
+ *					does not include payload.
+ * @struct response_msg:		Response fixed-size message pool -
+ *					does not include payload.
+ * @struct event_msg:			Event fixed-size message pool -
+ *					does not include payload.
+ * @struct event_ack_msg:		Ack fixed-size message pool -
+ *					does not include payload.
+ * @struct saved_crash_msg:		Message stored during IOVM creation to
+ *					be reused after crash.
+ */
 struct visor_controlvm_channel {
 	struct channel_header header;
-	u64 gp_controlvm;	/* guest phys addr of this channel */
-	u64 gp_partition_tables;/* guest phys addr of partition tables */
-	u64 gp_diag_guest;	/* guest phys addr of diagnostic channel */
-	u64 gp_boot_romdisk;/* guest phys addr of (read* only) Boot ROM disk */
-	u64 gp_boot_ramdisk;/* guest phys addr of writable Boot RAM disk */
-	u64 gp_acpi_table;	/* guest phys addr of acpi table */
-	u64 gp_control_channel;/* guest phys addr of control channel */
-	u64 gp_diag_romdisk;/* guest phys addr of diagnostic ROM disk */
-	u64 gp_nvram;	/* guest phys addr of NVRAM channel */
-	u64 request_payload_offset;	/* Offset to request payload area */
-	u64 event_payload_offset;	/* Offset to event payload area */
-	/* Bytes available in request payload area */
+	u64 gp_controlvm;
+	u64 gp_partition_tables;
+	u64 gp_diag_guest;
+	u64 gp_boot_romdisk;
+	u64 gp_boot_ramdisk;
+	u64 gp_acpi_table;
+	u64 gp_control_channel;
+	u64 gp_diag_romdisk;
+	u64 gp_nvram;
+	u64 request_payload_offset;
+	u64 event_payload_offset;
 	u32 request_payload_bytes;
-	u32 event_payload_bytes;/* Bytes available in event payload area */
+	u32 event_payload_bytes;
 	u32 control_channel_bytes;
-	u32 nvram_channel_bytes;	/* Bytes in PartitionNvram segment */
-	u32 message_bytes;		/* sizeof(CONTROLVM_MESSAGE) */
-	u32 message_count;		/* CONTROLVM_MESSAGE_MAX */
-	u64 gp_smbios_table;		/* guest phys addr of SMBIOS tables */
-	u64 gp_physical_smbios_table;	/* guest phys addr of SMBIOS table  */
-	/* VISOR_MAX_GUESTS_PER_SERVICE */
+	u32 nvram_channel_bytes;
+	u32 message_bytes;
+	u32 message_count;
+	u64 gp_smbios_table;
+	u64 gp_physical_smbios_table;
 	char gp_reserved[2688];
-
-	/* guest physical address of EFI firmware image base  */
 	u64 virtual_guest_firmware_image_base;
-
-	/* guest physical address of EFI firmware entry point  */
 	u64 virtual_guest_firmware_entry_point;
-
-	/* guest EFI firmware image size  */
 	u64 virtual_guest_firmware_image_size;
-
-	/* GPA = 1MB where EFI firmware image is copied to  */
 	u64 virtual_guest_firmware_boot_base;
 	u64 virtual_guest_image_base;
 	u64 virtual_guest_image_size;
 	u64 prototype_control_channel_offset;
 	u64 virtual_guest_partition_handle;
-	/* Restore Action field to restore the guest partition */
 	u16 restore_action;
-	/* For Windows guests it shows if the visordisk is in dump mode */
 	u16 dump_action;
 	u16 nvram_fail_count;
-	u16 saved_crash_message_count;	/* = CONTROLVM_CRASHMSG_MAX */
-	/* Offset to request payload area needed for crash dump */
+	u16 saved_crash_message_count;
 	u32 saved_crash_message_offset;
-	/* Type of error encountered during installation */
 	u32 installation_error;
-	u32 installation_text_id;	/* Id of string to display */
-	/* Number of remaining installation  steps (for progress bars) */
+	u32 installation_text_id;
 	u16 installation_remaining_steps;
-	/* VISOR_TOOL_ACTIONS Installation Action field */
 	u8 tool_action;
-	u8 reserved;		/* alignment */
+	u8 reserved;
 	struct efi_visor_indication efi_visor_ind;
 	u32 sp_reserved;
-	/* Force signals to begin on 128-byte cache line */
 	u8 reserved2[28];
-	/* guest partition uses this queue to send requests to Control */
 	struct signal_queue_header request_queue;
-	/* Control uses this queue to respond to service or guest
-	 * partition requests
-	 */
 	struct signal_queue_header response_queue;
-	/* Control uses this queue to send events to guest partition */
 	struct signal_queue_header event_queue;
-	/* Service or guest partition  uses this queue to ack Control events */
 	struct signal_queue_header event_ack_queue;
-	 /* Request fixed-size message pool - does not include payload */
-	 struct controlvm_message request_msg[CONTROLVM_MESSAGE_MAX];
-
-	 /* Response fixed-size message pool - does not include payload */
-	 struct controlvm_message response_msg[CONTROLVM_MESSAGE_MAX];
-
-	 /* Event fixed-size message pool - does not include payload */
-	 struct controlvm_message event_msg[CONTROLVM_MESSAGE_MAX];
-
-	 /* Ack fixed-size message pool - does not include payload */
-	 struct controlvm_message event_ack_msg[CONTROLVM_MESSAGE_MAX];
-
-	 /* Message stored during IOVM creation to be reused after crash */
-	 struct controlvm_message saved_crash_msg[CONTROLVM_CRASHMSG_MAX];
+	struct controlvm_message request_msg[CONTROLVM_MESSAGE_MAX];
+	struct controlvm_message response_msg[CONTROLVM_MESSAGE_MAX];
+	struct controlvm_message event_msg[CONTROLVM_MESSAGE_MAX];
+	struct controlvm_message event_ack_msg[CONTROLVM_MESSAGE_MAX];
+	struct controlvm_message saved_crash_msg[CONTROLVM_CRASHMSG_MAX];
 } __packed;
 
-/* The following header will be located at the beginning of PayloadVmOffset for
+/*
+ * struct visor_controlvm_parameters_header
+ *
+ * The following header will be located at the beginning of PayloadVmOffset for
  * various ControlVm commands. The receiver of a ControlVm command with a
  * PayloadVmOffset will dereference this address and then use connection_offset,
  * initiator_offset, and target_offset to get the location of UTF-8 formatted
@@ -455,9 +590,10 @@ struct visor_controlvm_parameters_header {
 	u32 client_length;
 	u32 name_offset;
 	u32 name_length;
-	uuid_le id;
+	guid_t id;
 	u32 revision;
-	u32 reserved;		/* Natural alignment */
+	/* Natural alignment */
+	u32 reserved;
 } __packed;
 
 /* General Errors------------------------------------------------------[0-99] */
@@ -467,72 +603,57 @@ struct visor_controlvm_parameters_header {
 #define CONTROLVM_RESP_KMALLOC_FAILED		   3
 #define CONTROLVM_RESP_ID_UNKNOWN		   4
 #define CONTROLVM_RESP_ID_INVALID_FOR_CLIENT	   5
-
 /* CONTROLVM_INIT_CHIPSET-------------------------------------------[100-199] */
 #define CONTROLVM_RESP_CLIENT_SWITCHCOUNT_NONZERO  100
 #define CONTROLVM_RESP_EXPECTED_CHIPSET_INIT	   101
-
 /* Maximum Limit----------------------------------------------------[200-299] */
-#define CONTROLVM_RESP_ERROR_MAX_BUSES		   201 /* BUS_CREATE */
-#define CONTROLVM_RESP_ERROR_MAX_DEVICES	   202 /* DEVICE_CREATE */
+/* BUS_CREATE */
+#define CONTROLVM_RESP_ERROR_MAX_BUSES		   201
+/* DEVICE_CREATE */
+#define CONTROLVM_RESP_ERROR_MAX_DEVICES	   202
 /* Payload and Parameter Related------------------------------------[400-499] */
-#define CONTROLVM_RESP_PAYLOAD_INVALID		   400 /* SWITCH_ATTACHEXTPORT,
-							* DEVICE_CONFIGURE
-							*/
-#define CONTROLVM_RESP_INITIATOR_PARAMETER_INVALID 401 /* Multiple */
-#define CONTROLVM_RESP_TARGET_PARAMETER_INVALID	   402 /* DEVICE_CONFIGURE */
-#define CONTROLVM_RESP_CLIENT_PARAMETER_INVALID	   403 /* DEVICE_CONFIGURE */
-/* Specified[Packet Structure] Value-------------------------------[500-599] */
-#define CONTROLVM_RESP_BUS_INVALID		   500 /* SWITCH_ATTACHINTPORT,
-							* BUS_CONFIGURE,
-							* DEVICE_CREATE,
-							* DEVICE_CONFIG
-							* DEVICE_DESTROY
-							*/
-#define CONTROLVM_RESP_DEVICE_INVALID		   501 /* SWITCH_ATTACHINTPORT*/
-						       /* DEVICE_CREATE,
-							* DEVICE_CONFIGURE,
-							* DEVICE_DESTROY
-							*/
-#define CONTROLVM_RESP_CHANNEL_INVALID		   502 /* DEVICE_CREATE,
-							* DEVICE_CONFIGURE
-							*/
-/* Partition Driver Callback Interface----------------------[600-699] */
-#define CONTROLVM_RESP_VIRTPCI_DRIVER_FAILURE	   604 /* BUS_CREATE,
-							* BUS_DESTROY,
-							* DEVICE_CREATE,
-							* DEVICE_DESTROY
-							*/
-/* Unable to invoke VIRTPCI callback */
-#define CONTROLVM_RESP_VIRTPCI_DRIVER_CALLBACK_ERROR   605 /* BUS_CREATE,
-							    * BUS_DESTROY,
-							    * DEVICE_CREATE,
-							    * DEVICE_DESTROY
-							    */
-/* VIRTPCI Callback returned error */
+/* SWITCH_ATTACHEXTPORT, DEVICE_CONFIGURE */
+#define CONTROLVM_RESP_PAYLOAD_INVALID		   400
+/* Multiple */
+#define CONTROLVM_RESP_INITIATOR_PARAMETER_INVALID 401
+/* DEVICE_CONFIGURE */
+#define CONTROLVM_RESP_TARGET_PARAMETER_INVALID	   402
+/* DEVICE_CONFIGURE */
+#define CONTROLVM_RESP_CLIENT_PARAMETER_INVALID	   403
+/* Specified[Packet Structure] Value--------------------------------[500-599] */
+/* SWITCH_ATTACHINTPORT */
+/* BUS_CONFIGURE, DEVICE_CREATE, DEVICE_CONFIG, DEVICE_DESTROY */
+#define CONTROLVM_RESP_BUS_INVALID		   500
+/* SWITCH_ATTACHINTPORT*/
+/* DEVICE_CREATE, DEVICE_CONFIGURE, DEVICE_DESTROY */
+#define CONTROLVM_RESP_DEVICE_INVALID		   501
+/* DEVICE_CREATE, DEVICE_CONFIGURE */
+#define CONTROLVM_RESP_CHANNEL_INVALID		   502
+/* Partition Driver Callback Interface------------------------------[600-699] */
+/* BUS_CREATE, BUS_DESTROY, DEVICE_CREATE, DEVICE_DESTROY */
+#define CONTROLVM_RESP_VIRTPCI_DRIVER_FAILURE	   604
+/* Unable to invoke VIRTPCI callback. VIRTPCI Callback returned error. */
+/* BUS_CREATE, BUS_DESTROY, DEVICE_CREATE, DEVICE_DESTROY */
+#define CONTROLVM_RESP_VIRTPCI_DRIVER_CALLBACK_ERROR   605
+/* Generic device callback returned error. */
+/* SWITCH_ATTACHEXTPORT, SWITCH_DETACHEXTPORT, DEVICE_CONFIGURE */
 #define CONTROLVM_RESP_GENERIC_DRIVER_CALLBACK_ERROR   606
-						       /* SWITCH_ATTACHEXTPORT,
-							* SWITCH_DETACHEXTPORT
-							* DEVICE_CONFIGURE
-							*/
-
-/* generic device callback returned error */
 /* Bus Related------------------------------------------------------[700-799] */
-#define CONTROLVM_RESP_ERROR_BUS_DEVICE_ATTACHED       700 /* BUS_DESTROY */
+/* BUS_DESTROY */
+#define CONTROLVM_RESP_ERROR_BUS_DEVICE_ATTACHED       700
 /* Channel Related--------------------------------------------------[800-899] */
-#define CONTROLVM_RESP_CHANNEL_TYPE_UNKNOWN	       800 /* GET_CHANNELINFO,
-							    * DEVICE_DESTROY
-							    */
-#define CONTROLVM_RESP_CHANNEL_SIZE_TOO_SMALL	       801 /* DEVICE_CREATE */
+/* GET_CHANNELINFO, DEVICE_DESTROY */
+#define CONTROLVM_RESP_CHANNEL_TYPE_UNKNOWN	       800
+/* DEVICE_CREATE */
+#define CONTROLVM_RESP_CHANNEL_SIZE_TOO_SMALL	       801
 /* Chipset Shutdown Related---------------------------------------[1000-1099] */
 #define CONTROLVM_RESP_CHIPSET_SHUTDOWN_FAILED	       1000
 #define CONTROLVM_RESP_CHIPSET_SHUTDOWN_ALREADY_ACTIVE 1001
-
 /* Chipset Stop Related-------------------------------------------[1100-1199] */
 #define CONTROLVM_RESP_CHIPSET_STOP_FAILED_BUS	       1100
 #define CONTROLVM_RESP_CHIPSET_STOP_FAILED_SWITCH      1101
-
 /* Device Related-------------------------------------------------[1400-1499] */
 #define CONTROLVM_RESP_DEVICE_UDEV_TIMEOUT	       1400
 
-#endif				/* __CONTROLVMCHANNEL_H__ */
+/* __CONTROLVMCHANNEL_H__ */
+#endif
diff --git a/drivers/staging/unisys/visorbus/vbuschannel.h b/drivers/staging/unisys/visorbus/vbuschannel.h
index 01d7d517dba7..27e04de14818 100644
--- a/drivers/staging/unisys/visorbus/vbuschannel.h
+++ b/drivers/staging/unisys/visorbus/vbuschannel.h
@@ -1,4 +1,5 @@
-/* Copyright (C) 2010 - 2015 UNISYS CORPORATION
+/*
+ * Copyright (C) 2010 - 2015 UNISYS CORPORATION
  * All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
@@ -15,26 +16,26 @@
 #ifndef __VBUSCHANNEL_H__
 #define __VBUSCHANNEL_H__
 
-/*  The vbus channel is the channel area provided via the BUS_CREATE controlvm
- *  message for each virtual bus.  This channel area is provided to both server
- *  and client ends of the bus.  The channel header area is initialized by
- *  the server, and the remaining information is filled in by the client.
- *  We currently use this for the client to provide various information about
- *  the client devices and client drivers for the server end to see.
+/*
+ * The vbus channel is the channel area provided via the BUS_CREATE controlvm
+ * message for each virtual bus.  This channel area is provided to both server
+ * and client ends of the bus.  The channel header area is initialized by
+ * the server, and the remaining information is filled in by the client.
+ * We currently use this for the client to provide various information about
+ * the client devices and client drivers for the server end to see.
  */
+
 #include <linux/uuid.h>
 #include <linux/ctype.h>
 #include "channel.h"
 
 /* {193b331b-c58f-11da-95a9-00e08161165f} */
-#define VISOR_VBUS_CHANNEL_UUID \
-	UUID_LE(0x193b331b, 0xc58f, 0x11da, \
-		0x95, 0xa9, 0x0, 0xe0, 0x81, 0x61, 0x16, 0x5f)
-static const uuid_le visor_vbus_channel_uuid = VISOR_VBUS_CHANNEL_UUID;
+#define VISOR_VBUS_CHANNEL_GUID						\
+	GUID_INIT(0x193b331b, 0xc58f, 0x11da,				\
+		  0x95, 0xa9, 0x0, 0xe0, 0x81, 0x61, 0x16, 0x5f)
 
-#define VISOR_VBUS_CHANNEL_SIGNATURE VISOR_CHANNEL_SIGNATURE
-
-/* Must increment this whenever you insert or delete fields within this channel
+/*
+ * Must increment this whenever you insert or delete fields within this channel
  * struct.  Also increment whenever you change the meaning of fields within this
  * channel struct so as to break pre-existing software.  Note that you can
  * usually add fields to the END of the channel struct withOUT needing to
@@ -43,42 +44,63 @@ static const uuid_le visor_vbus_channel_uuid = VISOR_VBUS_CHANNEL_UUID;
 #define VISOR_VBUS_CHANNEL_VERSIONID 1
 
 /*
+ * struct visor_vbus_deviceinfo
+ * @devtype:  Short string identifying the device type.
+ * @drvname:  Driver .sys file name.
+ * @infostrs: Kernel vversion.
+ * @reserved: Pad size to 256 bytes.
+ *
  * An array of this struct is present in the channel area for each vbus.
- * (See vbuschannel.h.)
- * It is filled in by the client side to provide info about the device
- * and driver from the client's perspective.
+ * (See vbuschannel.h.). It is filled in by the client side to provide info
+ * about the device and driver from the client's perspective.
  */
 struct visor_vbus_deviceinfo {
-	u8 devtype[16];		/* short string identifying the device type */
-	u8 drvname[16];		/* driver .sys file name */
-	u8 infostrs[96];	/* kernel version */
-	u8 reserved[128];	/* pad size to 256 bytes */
+	u8 devtype[16];
+	u8 drvname[16];
+	u8 infostrs[96];
+	u8 reserved[128];
 } __packed;
 
+/*
+ * struct visor_vbus_headerinfo
+ * @struct_bytes:	      Size of this struct in bytes.
+ * @device_info_struct_bytes: Size of VISOR_VBUS_DEVICEINFO.
+ * @dev_info_count:	      Num of items in DevInfo member. This is the
+ *			      allocated size.
+ * @chp_info_offset:	      Byte offset from beginning of this struct to the
+ *			      ChpInfo struct.
+ * @bus_info_offset:	      Byte offset from beginning of this struct to the
+ *			      BusInfo struct.
+ * @dev_info_offset:	      Byte offset from beginning of this struct to the
+ *			      DevInfo array.
+ * @reserved:		      Natural Alignment
+ */
 struct visor_vbus_headerinfo {
-	u32 struct_bytes;	/* size of this struct in bytes */
-	u32 device_info_struct_bytes;	/* sizeof(VISOR_VBUS_DEVICEINFO) */
-	u32 dev_info_count;	/* num of items in DevInfo member */
-	/* (this is the allocated size) */
-	u32 chp_info_offset;	/* byte offset from beginning of this struct */
-	/* to the ChpInfo struct (below) */
-	u32 bus_info_offset;	/* byte offset from beginning of this struct */
-	/* to the BusInfo struct (below) */
-	u32 dev_info_offset;	/* byte offset from beginning of this struct */
-	/* to the DevInfo array (below) */
+	u32 struct_bytes;
+	u32 device_info_struct_bytes;
+	u32 dev_info_count;
+	u32 chp_info_offset;
+	u32 bus_info_offset;
+	u32 dev_info_offset;
 	u8 reserved[104];
 } __packed;
 
+/*
+ * struct visor_vbus_channel
+ * @channel_header: Initialized by server.
+ * @hdr_info:	    Initialized by server.
+ * @chp_info:	    Describes client chipset device and driver.
+ * @bus_info:	    Describes client bus device and driver.
+ * @dev_info:	    Describes client device and driver for each device on the
+ *		    bus.
+ */
 struct visor_vbus_channel {
-	struct channel_header channel_header;	/* initialized by server */
-	struct visor_vbus_headerinfo hdr_info;	/* initialized by server */
-	/* the remainder of this channel is filled in by the client */
+	struct channel_header channel_header;
+	struct visor_vbus_headerinfo hdr_info;
+	/* The remainder of this channel is filled in by the client */
 	struct visor_vbus_deviceinfo chp_info;
-	/* describes client chipset device and driver */
 	struct visor_vbus_deviceinfo bus_info;
-	/* describes client bus device and driver */
 	struct visor_vbus_deviceinfo dev_info[0];
-	/* describes client device and driver for each device on the bus */
 } __packed;
 
 #endif
diff --git a/drivers/staging/unisys/visorbus/visorbus_main.c b/drivers/staging/unisys/visorbus/visorbus_main.c
index 1c785dd19ddd..2bc7ff7bb96a 100644
--- a/drivers/staging/unisys/visorbus/visorbus_main.c
+++ b/drivers/staging/unisys/visorbus/visorbus_main.c
@@ -1,5 +1,4 @@
-/* visorbus_main.c
- *
+/*
  * Copyright � 2010 - 2015 UNISYS CORPORATION
  * All rights reserved.
  *
@@ -20,15 +19,14 @@
 #include "visorbus.h"
 #include "visorbus_private.h"
 
-#define MYDRVNAME "visorbus"
+static const guid_t visor_vbus_channel_guid = VISOR_VBUS_CHANNEL_GUID;
 
-/* Display string that is guaranteed to be no longer the 99 characters*/
+/* Display string that is guaranteed to be no longer the 99 characters */
 #define LINESIZE 99
-
-#define CURRENT_FILE_PC VISOR_BUS_PC_visorbus_main_c
 #define POLLJIFFIES_NORMALCHANNEL 10
 
-static bool initialized; /* stores whether bus_registration was successful */
+/* stores whether bus_registration was successful */
+static bool initialized;
 static struct dentry *visorbus_debugfs_dir;
 
 /*
@@ -40,11 +38,11 @@ static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
 			     char *buf)
 {
 	struct visor_device *vdev;
-	uuid_le guid;
+	const guid_t *guid;
 
 	vdev = to_visor_device(dev);
-	guid = visorchannel_get_uuid(vdev->visorchannel);
-	return sprintf(buf, "visorbus:%pUl\n", &guid);
+	guid = visorchannel_get_guid(vdev->visorchannel);
+	return sprintf(buf, "visorbus:%pUl\n", guid);
 }
 static DEVICE_ATTR_RO(modalias);
 
@@ -53,15 +51,7 @@ static struct attribute *visorbus_dev_attrs[] = {
 	NULL,
 };
 
-/* sysfs example for bridge-only sysfs files using device_type's */
-static const struct attribute_group visorbus_dev_group = {
-	.attrs = visorbus_dev_attrs,
-};
-
-static const struct attribute_group *visorbus_dev_groups[] = {
-	&visorbus_dev_group,
-	NULL,
-};
+ATTRIBUTE_GROUPS(visorbus_dev);
 
 /* filled in with info about parent chipset driver when we register with it */
 static struct visor_vbus_deviceinfo chipset_driverinfo;
@@ -73,16 +63,70 @@ static LIST_HEAD(list_all_bus_instances);
 /* list of visor_device structs, linked via .list_all */
 static LIST_HEAD(list_all_device_instances);
 
-static int
-visorbus_uevent(struct device *xdev, struct kobj_uevent_env *env)
+/*
+ * Generic function useful for validating any type of channel when it is
+ * received by the client that will be accessing the channel.
+ * Note that <logCtx> is only needed for callers in the EFI environment, and
+ * is used to pass the EFI_DIAG_CAPTURE_PROTOCOL needed to log messages.
+ */
+int visor_check_channel(struct channel_header *ch,
+			struct device *dev,
+			const guid_t *expected_guid,
+			char *chname,
+			u64 expected_min_bytes,
+			u32 expected_version,
+			u64 expected_signature)
+{
+	if (!guid_is_null(expected_guid)) {
+		/* caller wants us to verify type GUID */
+		if (!guid_equal(&ch->chtype, expected_guid)) {
+			dev_err(dev, "Channel mismatch on channel=%s(%pUL) field=type expected=%pUL actual=%pUL\n",
+				chname, expected_guid, expected_guid,
+				&ch->chtype);
+			return 0;
+		}
+	}
+	/* verify channel size */
+	if (expected_min_bytes > 0) {
+		if (ch->size < expected_min_bytes) {
+			dev_err(dev, "Channel mismatch on channel=%s(%pUL) field=size expected=0x%-8.8Lx actual=0x%-8.8Lx\n",
+				chname, expected_guid,
+				(unsigned long long)expected_min_bytes,
+				ch->size);
+			return 0;
+		}
+	}
+	/* verify channel version */
+	if (expected_version > 0) {
+		if (ch->version_id != expected_version) {
+			dev_err(dev, "Channel mismatch on channel=%s(%pUL) field=version expected=0x%-8.8lx actual=0x%-8.8x\n",
+				chname, expected_guid,
+				(unsigned long)expected_version,
+				ch->version_id);
+			return 0;
+		}
+	}
+	/* verify channel signature */
+	if (expected_signature > 0) {
+		if (ch->signature != expected_signature) {
+			dev_err(dev, "Channel mismatch on channel=%s(%pUL) field=signature expected=0x%-8.8Lx actual=0x%-8.8Lx\n",
+				chname, expected_guid,	expected_signature,
+				ch->signature);
+			return 0;
+		}
+	}
+	return 1;
+}
+
+static int visorbus_uevent(struct device *xdev, struct kobj_uevent_env *env)
 {
 	struct visor_device *dev;
-	uuid_le guid;
+	const guid_t *guid;
 
 	dev = to_visor_device(xdev);
-	guid = visorchannel_get_uuid(dev->visorchannel);
+	guid = visorchannel_get_guid(dev->visorchannel);
 
-	return add_uevent_var(env, "MODALIAS=visorbus:%pUl", &guid);
+	return add_uevent_var(env, "MODALIAS=visorbus:%pUl", guid);
 }
 
 /*
@@ -94,27 +138,21 @@ visorbus_uevent(struct device *xdev, struct kobj_uevent_env *env)
  *
  * Return: 1 iff the provided driver can control the specified device
  */
-static int
-visorbus_match(struct device *xdev, struct device_driver *xdrv)
+static int visorbus_match(struct device *xdev, struct device_driver *xdrv)
 {
-	uuid_le channel_type;
+	const guid_t *channel_type;
 	int i;
 	struct visor_device *dev;
 	struct visor_driver *drv;
 
 	dev = to_visor_device(xdev);
+	channel_type = visorchannel_get_guid(dev->visorchannel);
 	drv = to_visor_driver(xdrv);
-	channel_type = visorchannel_get_uuid(dev->visorchannel);
-
 	if (!drv->channel_types)
 		return 0;
 
-	for (i = 0;
-	     (uuid_le_cmp(drv->channel_types[i].guid, NULL_UUID_LE) != 0) ||
-	     (drv->channel_types[i].name);
-	     i++)
-		if (uuid_le_cmp(drv->channel_types[i].guid,
-				channel_type) == 0)
+	for (i = 0; !guid_is_null(&drv->channel_types[i].guid); i++)
+		if (guid_equal(&drv->channel_types[i].guid, channel_type))
 			return i + 1;
 
 	return 0;
@@ -122,7 +160,7 @@ visorbus_match(struct device *xdev, struct device_driver *xdrv)
 
 /*
  * This describes the TYPE of bus.
- *  (Don't confuse this with an INSTANCE of the bus.)
+ * (Don't confuse this with an INSTANCE of the bus.)
  */
 struct bus_type visorbus_type = {
 	.name = "visorbus",
@@ -137,8 +175,7 @@ struct bus_type visorbus_type = {
  *                                involved with destroying the dev are complete
  * @xdev: struct device for the bus being released
  */
-static void
-visorbus_release_busdevice(struct device *xdev)
+static void visorbus_release_busdevice(struct device *xdev)
 {
 	struct visor_device *dev = dev_get_drvdata(xdev);
 
@@ -152,15 +189,11 @@ visorbus_release_busdevice(struct device *xdev)
  *                             each child device instance
  * @xdev: struct device for the visor device being released
  */
-static void
-visorbus_release_device(struct device *xdev)
+static void visorbus_release_device(struct device *xdev)
 {
 	struct visor_device *dev = to_visor_device(xdev);
 
-	if (dev->visorchannel) {
-		visorchannel_destroy(dev->visorchannel);
-		dev->visorchannel = NULL;
-	}
+	visorchannel_destroy(dev->visorchannel);
 	kfree(dev);
 }
 
@@ -229,7 +262,7 @@ static ssize_t typename_show(struct device *dev, struct device_attribute *attr,
 	struct device_driver *xdrv = dev->driver;
 	struct visor_driver *drv = NULL;
 
-	if (!xbus || !xdrv)
+	if (!xdrv)
 		return 0;
 	i = xbus->match(dev, xdrv);
 	if (!i)
@@ -240,24 +273,16 @@ static ssize_t typename_show(struct device *dev, struct device_attribute *attr,
 static DEVICE_ATTR_RO(typename);
 
 static struct attribute *channel_attrs[] = {
-		&dev_attr_physaddr.attr,
-		&dev_attr_nbytes.attr,
-		&dev_attr_clientpartition.attr,
-		&dev_attr_typeguid.attr,
-		&dev_attr_zoneguid.attr,
-		&dev_attr_typename.attr,
-		NULL
+	&dev_attr_physaddr.attr,
+	&dev_attr_nbytes.attr,
+	&dev_attr_clientpartition.attr,
+	&dev_attr_typeguid.attr,
+	&dev_attr_zoneguid.attr,
+	&dev_attr_typename.attr,
+	NULL
 };
 
-static struct attribute_group channel_attr_grp = {
-		.name = "channel",
-		.attrs = channel_attrs,
-};
-
-static const struct attribute_group *visorbus_channel_groups[] = {
-		&channel_attr_grp,
-		NULL
-};
+ATTRIBUTE_GROUPS(channel);
 
 /* end implementation of specific channel attributes */
 
@@ -270,7 +295,8 @@ static const struct attribute_group *visorbus_channel_groups[] = {
 
 static ssize_t partition_handle_show(struct device *dev,
 				     struct device_attribute *attr,
-				     char *buf) {
+				     char *buf)
+{
 	struct visor_device *vdev = to_visor_device(dev);
 	u64 handle = visorchannel_get_clientpartition(vdev->visorchannel);
 
@@ -280,16 +306,18 @@ static DEVICE_ATTR_RO(partition_handle);
 
 static ssize_t partition_guid_show(struct device *dev,
 				   struct device_attribute *attr,
-				   char *buf) {
+				   char *buf)
+{
 	struct visor_device *vdev = to_visor_device(dev);
 
-	return sprintf(buf, "{%pUb}\n", &vdev->partition_uuid);
+	return sprintf(buf, "{%pUb}\n", &vdev->partition_guid);
 }
 static DEVICE_ATTR_RO(partition_guid);
 
 static ssize_t partition_name_show(struct device *dev,
 				   struct device_attribute *attr,
-				   char *buf) {
+				   char *buf)
+{
 	struct visor_device *vdev = to_visor_device(dev);
 
 	return sprintf(buf, "%s\n", vdev->name);
@@ -298,7 +326,8 @@ static DEVICE_ATTR_RO(partition_name);
 
 static ssize_t channel_addr_show(struct device *dev,
 				 struct device_attribute *attr,
-				 char *buf) {
+				 char *buf)
+{
 	struct visor_device *vdev = to_visor_device(dev);
 	u64 addr = visorchannel_get_physaddr(vdev->visorchannel);
 
@@ -308,7 +337,8 @@ static DEVICE_ATTR_RO(channel_addr);
 
 static ssize_t channel_bytes_show(struct device *dev,
 				  struct device_attribute *attr,
-				  char *buf) {
+				  char *buf)
+{
 	struct visor_device *vdev = to_visor_device(dev);
 	u64 nbytes = visorchannel_get_nbytes(vdev->visorchannel);
 
@@ -318,7 +348,8 @@ static DEVICE_ATTR_RO(channel_bytes);
 
 static ssize_t channel_id_show(struct device *dev,
 			       struct device_attribute *attr,
-			       char *buf) {
+			       char *buf)
+{
 	struct visor_device *vdev = to_visor_device(dev);
 	int len = 0;
 
@@ -330,24 +361,17 @@ static ssize_t channel_id_show(struct device *dev,
 }
 static DEVICE_ATTR_RO(channel_id);
 
-static struct attribute *dev_attrs[] = {
-		&dev_attr_partition_handle.attr,
-		&dev_attr_partition_guid.attr,
-		&dev_attr_partition_name.attr,
-		&dev_attr_channel_addr.attr,
-		&dev_attr_channel_bytes.attr,
-		&dev_attr_channel_id.attr,
-		NULL
-};
-
-static struct attribute_group dev_attr_grp = {
-		.attrs = dev_attrs,
+static struct attribute *visorbus_attrs[] = {
+	&dev_attr_partition_handle.attr,
+	&dev_attr_partition_guid.attr,
+	&dev_attr_partition_name.attr,
+	&dev_attr_channel_addr.attr,
+	&dev_attr_channel_bytes.attr,
+	&dev_attr_channel_id.attr,
+	NULL
 };
 
-static const struct attribute_group *visorbus_groups[] = {
-		&dev_attr_grp,
-		NULL
-};
+ATTRIBUTE_GROUPS(visorbus);
 
 /*
  *  BUS debugfs entries
@@ -355,6 +379,7 @@ static const struct attribute_group *visorbus_groups[] = {
  *  define & implement display of debugfs attributes under
  *  /sys/kernel/debug/visorbus/visorbus<n>.
  */
+
 /*
  * vbuschannel_print_devinfo() - format a struct visor_vbus_deviceinfo
  *                               and write it to a seq_file
@@ -365,12 +390,12 @@ static const struct attribute_group *visorbus_groups[] = {
  *
  * Reads @devInfo, and writes it in human-readable notation to @seq.
  */
-static void
-vbuschannel_print_devinfo(struct visor_vbus_deviceinfo *devinfo,
-			  struct seq_file *seq, int devix)
+static void vbuschannel_print_devinfo(struct visor_vbus_deviceinfo *devinfo,
+				      struct seq_file *seq, int devix)
 {
+	/* uninitialized vbus device entry */
 	if (!isprint(devinfo->devtype[0]))
-		return; /* uninitialized vbus device entry */
+		return;
 
 	if (devix >= 0)
 		seq_printf(seq, "[%d]", devix);
@@ -392,12 +417,11 @@ vbuschannel_print_devinfo(struct visor_vbus_deviceinfo *devinfo,
 
 static int client_bus_info_debugfs_show(struct seq_file *seq, void *v)
 {
-	struct visor_device *vdev = seq->private;
-	struct visorchannel *channel = vdev->visorchannel;
-
-	int i;
+	int i = 0;
 	unsigned long off;
 	struct visor_vbus_deviceinfo dev_info;
+	struct visor_device *vdev = seq->private;
+	struct visorchannel *channel = vdev->visorchannel;
 
 	if (!channel)
 		return 0;
@@ -406,6 +430,7 @@ static int client_bus_info_debugfs_show(struct seq_file *seq, void *v)
 		   "Client device / client driver info for %s partition (vbus #%u):\n",
 		   ((vdev->name) ? (char *)(vdev->name) : ""),
 		   vdev->chipset_bus_no);
+
 	if (visorchannel_read(channel,
 			      offsetof(struct visor_vbus_channel, chp_info),
 			      &dev_info, sizeof(dev_info)) >= 0)
@@ -414,8 +439,8 @@ static int client_bus_info_debugfs_show(struct seq_file *seq, void *v)
 			      offsetof(struct visor_vbus_channel, bus_info),
 			      &dev_info, sizeof(dev_info)) >= 0)
 		vbuschannel_print_devinfo(&dev_info, seq, -1);
+
 	off = offsetof(struct visor_vbus_channel, dev_info);
-	i = 0;
 	while (off + sizeof(dev_info) <= visorchannel_get_nbytes(channel)) {
 		if (visorchannel_read(channel, off, &dev_info,
 				      sizeof(dev_info)) >= 0)
@@ -441,8 +466,7 @@ static const struct file_operations client_bus_info_debugfs_fops = {
 	.release = single_release,
 };
 
-static void
-dev_periodic_work(unsigned long __opaque)
+static void dev_periodic_work(unsigned long __opaque)
 {
 	struct visor_device *dev = (struct visor_device *)__opaque;
 	struct visor_driver *drv = to_visor_driver(dev->device.driver);
@@ -451,8 +475,7 @@ dev_periodic_work(unsigned long __opaque)
 	mod_timer(&dev->timer, jiffies + POLLJIFFIES_NORMALCHANNEL);
 }
 
-static int
-dev_start_periodic_work(struct visor_device *dev)
+static int dev_start_periodic_work(struct visor_device *dev)
 {
 	if (dev->being_removed || dev->timer_active)
 		return -EINVAL;
@@ -464,8 +487,7 @@ dev_start_periodic_work(struct visor_device *dev)
 	return 0;
 }
 
-static void
-dev_stop_periodic_work(struct visor_device *dev)
+static void dev_stop_periodic_work(struct visor_device *dev)
 {
 	if (!dev->timer_active)
 		return;
@@ -484,40 +506,39 @@ dev_stop_periodic_work(struct visor_device *dev)
  *
  * Return: 0 iff successful
  */
-static int
-visordriver_remove_device(struct device *xdev)
+static int visordriver_remove_device(struct device *xdev)
 {
 	struct visor_device *dev;
 	struct visor_driver *drv;
 
 	dev = to_visor_device(xdev);
 	drv = to_visor_driver(xdev->driver);
+
 	mutex_lock(&dev->visordriver_callback_lock);
 	dev->being_removed = true;
-	if (drv->remove)
-		drv->remove(dev);
+	drv->remove(dev);
 	mutex_unlock(&dev->visordriver_callback_lock);
-	dev_stop_periodic_work(dev);
 
+	dev_stop_periodic_work(dev);
 	put_device(&dev->device);
+
 	return 0;
 }
 
-/**
+/*
  * visorbus_unregister_visor_driver() - unregisters the provided driver
  * @drv: the driver to unregister
  *
  * A visor function driver calls this function to unregister the driver,
  * i.e., within its module_exit function.
  */
-void
-visorbus_unregister_visor_driver(struct visor_driver *drv)
+void visorbus_unregister_visor_driver(struct visor_driver *drv)
 {
 	driver_unregister(&drv->driver);
 }
 EXPORT_SYMBOL_GPL(visorbus_unregister_visor_driver);
 
-/**
+/*
  * visorbus_read_channel() - reads from the designated channel into
  *                           the provided buffer
  * @dev:    the device whose channel is read from
@@ -530,15 +551,14 @@ EXPORT_SYMBOL_GPL(visorbus_unregister_visor_driver);
  *
  * Return: integer indicating success (zero) or failure (non-zero)
  */
-int
-visorbus_read_channel(struct visor_device *dev, unsigned long offset,
-		      void *dest, unsigned long nbytes)
+int visorbus_read_channel(struct visor_device *dev, unsigned long offset,
+			  void *dest, unsigned long nbytes)
 {
 	return visorchannel_read(dev->visorchannel, offset, dest, nbytes);
 }
 EXPORT_SYMBOL_GPL(visorbus_read_channel);
 
-/**
+/*
  * visorbus_write_channel() - writes the provided buffer into the designated
  *                            channel
  * @dev:    the device whose channel is written to
@@ -551,15 +571,14 @@ EXPORT_SYMBOL_GPL(visorbus_read_channel);
  *
  * Return: integer indicating success (zero) or failure (non-zero)
  */
-int
-visorbus_write_channel(struct visor_device *dev, unsigned long offset,
-		       void *src, unsigned long nbytes)
+int visorbus_write_channel(struct visor_device *dev, unsigned long offset,
+			   void *src, unsigned long nbytes)
 {
 	return visorchannel_write(dev->visorchannel, offset, src, nbytes);
 }
 EXPORT_SYMBOL_GPL(visorbus_write_channel);
 
-/**
+/*
  * visorbus_enable_channel_interrupts() - enables interrupts on the
  *                                        designated device
  * @dev: the device on which to enable interrupts
@@ -567,8 +586,7 @@ EXPORT_SYMBOL_GPL(visorbus_write_channel);
  * Currently we don't yet have a real interrupt, so for now we just call the
  * interrupt function periodically via a timer.
  */
-int
-visorbus_enable_channel_interrupts(struct visor_device *dev)
+int visorbus_enable_channel_interrupts(struct visor_device *dev)
 {
 	struct visor_driver *drv = to_visor_driver(dev->device.driver);
 
@@ -581,13 +599,12 @@ visorbus_enable_channel_interrupts(struct visor_device *dev)
 }
 EXPORT_SYMBOL_GPL(visorbus_enable_channel_interrupts);
 
-/**
+/*
  * visorbus_disable_channel_interrupts() - disables interrupts on the
  *                                         designated device
  * @dev: the device on which to disable interrupts
  */
-void
-visorbus_disable_channel_interrupts(struct visor_device *dev)
+void visorbus_disable_channel_interrupts(struct visor_device *dev)
 {
 	dev_stop_periodic_work(dev);
 }
@@ -616,8 +633,7 @@ EXPORT_SYMBOL_GPL(visorbus_disable_channel_interrupts);
  * Return: 0 if successful, otherwise the negative value returned by
  *         device_add() indicating the reason for failure
  */
-static int
-create_visor_device(struct visor_device *dev)
+int create_visor_device(struct visor_device *dev)
 {
 	int err;
 	u32 chipset_bus_no = dev->chipset_bus_no;
@@ -625,7 +641,7 @@ create_visor_device(struct visor_device *dev)
 
 	mutex_init(&dev->visordriver_callback_lock);
 	dev->device.bus = &visorbus_type;
-	dev->device.groups = visorbus_channel_groups;
+	dev->device.groups = channel_groups;
 	device_initialize(&dev->device);
 	dev->device.release = visorbus_release_device;
 	/* keep a reference just for us (now 2) */
@@ -664,7 +680,10 @@ create_visor_device(struct visor_device *dev)
 		goto err_put;
 
 	list_add_tail(&dev->list_all, &list_all_device_instances);
-	return 0; /* success: reference kept via unmatched get_device() */
+	dev->state.created = 1;
+	visorbus_response(dev, err, CONTROLVM_DEVICE_CREATE);
+	/* success: reference kept via unmatched get_device() */
+	return 0;
 
 err_put:
 	put_device(&dev->device);
@@ -672,26 +691,27 @@ err_put:
 	return err;
 }
 
-static void
-remove_visor_device(struct visor_device *dev)
+void remove_visor_device(struct visor_device *dev)
 {
 	list_del(&dev->list_all);
 	put_device(&dev->device);
 	device_unregister(&dev->device);
+	visorbus_response(dev, 0, CONTROLVM_DEVICE_DESTROY);
 }
 
-static int
-get_vbus_header_info(struct visorchannel *chan,
-		     struct visor_vbus_headerinfo *hdr_info)
+static int get_vbus_header_info(struct visorchannel *chan,
+				struct device *dev,
+				struct visor_vbus_headerinfo *hdr_info)
 {
 	int err;
 
 	if (!visor_check_channel(visorchannel_get_header(chan),
-				 visor_vbus_channel_uuid,
+				 dev,
+				 &visor_vbus_channel_guid,
 				 "vbus",
 				 sizeof(struct visor_vbus_channel),
 				 VISOR_VBUS_CHANNEL_VERSIONID,
-				 VISOR_VBUS_CHANNEL_SIGNATURE))
+				 VISOR_CHANNEL_SIGNATURE))
 		return -EINVAL;
 
 	err = visorchannel_read(chan, sizeof(struct channel_header), hdr_info,
@@ -722,10 +742,9 @@ get_vbus_header_info(struct visorchannel *chan,
  * Returns no value since this is debug information and not needed for
  * device functionality.
  */
-static void
-write_vbus_chp_info(struct visorchannel *chan,
-		    struct visor_vbus_headerinfo *hdr_info,
-		    struct visor_vbus_deviceinfo *info)
+static void write_vbus_chp_info(struct visorchannel *chan,
+				struct visor_vbus_headerinfo *hdr_info,
+				struct visor_vbus_deviceinfo *info)
 {
 	int off = sizeof(struct channel_header) + hdr_info->chp_info_offset;
 
@@ -748,10 +767,9 @@ write_vbus_chp_info(struct visorchannel *chan,
  * Returns no value since this is debug information and not needed for
  * device functionality.
  */
-static void
-write_vbus_bus_info(struct visorchannel *chan,
-		    struct visor_vbus_headerinfo *hdr_info,
-		    struct visor_vbus_deviceinfo *info)
+static void write_vbus_bus_info(struct visorchannel *chan,
+				struct visor_vbus_headerinfo *hdr_info,
+				struct visor_vbus_deviceinfo *info)
 {
 	int off = sizeof(struct channel_header) + hdr_info->bus_info_offset;
 
@@ -775,10 +793,10 @@ write_vbus_bus_info(struct visorchannel *chan,
  * Returns no value since this is debug information and not needed for
  * device functionality.
  */
-static void
-write_vbus_dev_info(struct visorchannel *chan,
-		    struct visor_vbus_headerinfo *hdr_info,
-		    struct visor_vbus_deviceinfo *info, unsigned int devix)
+static void write_vbus_dev_info(struct visorchannel *chan,
+				struct visor_vbus_headerinfo *hdr_info,
+				struct visor_vbus_deviceinfo *info,
+				unsigned int devix)
 {
 	int off =
 	    (sizeof(struct channel_header) + hdr_info->dev_info_offset) +
@@ -807,14 +825,13 @@ static void bus_device_info_init(
 }
 
 /*
- * fix_vbus_dev_info() - for a child device just created on a client bus, fill
- *                       in information about the driver that is controlling
- *                       this device into the appropriate slot within the
- *                       vbus channel of the bus instance
+ * publish_vbus_dev_info() - for a child device just created on a client bus,
+ *			     fill in information about the driver that is
+ *			     controlling this device into the appropriate slot
+ *			     within the vbus channel of the bus instance
  * @visordev: struct visor_device for the desired device
  */
-static void
-fix_vbus_dev_info(struct visor_device *visordev)
+static void publish_vbus_dev_info(struct visor_device *visordev)
 {
 	int i;
 	struct visor_device *bdev;
@@ -853,7 +870,6 @@ fix_vbus_dev_info(struct visor_device *visordev)
 
 	bus_device_info_init(&dev_info, chan_type_name, visordrv->name);
 	write_vbus_dev_info(bdev->visorchannel, hdr_info, &dev_info, dev_no);
-
 	write_vbus_chp_info(bdev->visorchannel, hdr_info, &chipset_driverinfo);
 	write_vbus_bus_info(bdev->visorchannel, hdr_info,
 			    &clientbus_driverinfo);
@@ -874,18 +890,14 @@ fix_vbus_dev_info(struct visor_device *visordev)
  *         was successful with this device, otherwise a negative errno
  *         value indicating failure reason
  */
-static int
-visordriver_probe_device(struct device *xdev)
+static int visordriver_probe_device(struct device *xdev)
 {
 	int res;
 	struct visor_driver *drv;
 	struct visor_device *dev;
 
-	drv = to_visor_driver(xdev->driver);
 	dev = to_visor_device(xdev);
-
-	if (!drv->probe)
-		return -ENODEV;
+	drv = to_visor_driver(xdev->driver);
 
 	mutex_lock(&dev->visordriver_callback_lock);
 	dev->being_removed = false;
@@ -894,14 +906,14 @@ visordriver_probe_device(struct device *xdev)
 	if (res >= 0) {
 		/* success: reference kept via unmatched get_device() */
 		get_device(&dev->device);
-		fix_vbus_dev_info(dev);
+		publish_vbus_dev_info(dev);
 	}
 
 	mutex_unlock(&dev->visordriver_callback_lock);
 	return res;
 }
 
-/**
+/*
  * visorbus_register_visor_driver() - registers the provided visor driver
  *                                    for handling one or more visor device
  *                                    types (channel_types)
@@ -952,8 +964,21 @@ visordriver_probe_device(struct device *xdev)
  */
 int visorbus_register_visor_driver(struct visor_driver *drv)
 {
+	/* can't register on a nonexistent bus */
 	if (!initialized)
-		return -ENODEV; /* can't register on a nonexistent bus */
+		return -ENODEV;
+
+	if (!drv->probe)
+		return -EINVAL;
+
+	if (!drv->remove)
+		return -EINVAL;
+
+	if (!drv->pause)
+		return -EINVAL;
+
+	if (!drv->resume)
+		return -EINVAL;
 
 	drv->driver.name = drv->name;
 	drv->driver.bus = &visorbus_type;
@@ -985,8 +1010,7 @@ EXPORT_SYMBOL_GPL(visorbus_register_visor_driver);
  * Return: 0 for success, otherwise negative errno value indicating reason for
  *         failure
  */
-static int
-visorbus_create_instance(struct visor_device *dev)
+int visorbus_create_instance(struct visor_device *dev)
 {
 	int id = dev->chipset_bus_no;
 	int err;
@@ -1009,7 +1033,7 @@ visorbus_create_instance(struct visor_device *dev)
 				    &client_bus_info_debugfs_fops);
 
 	dev_set_drvdata(&dev->device, dev);
-	err = get_vbus_header_info(dev->visorchannel, hdr_info);
+	err = get_vbus_header_info(dev->visorchannel, &dev->device, hdr_info);
 	if (err < 0)
 		goto err_debugfs_dir;
 
@@ -1019,18 +1043,21 @@ visorbus_create_instance(struct visor_device *dev)
 
 	list_add_tail(&dev->list_all, &list_all_bus_instances);
 
+	dev->state.created = 1;
 	dev->vbus_hdr_info = (void *)hdr_info;
 	write_vbus_chp_info(dev->visorchannel, hdr_info,
 			    &chipset_driverinfo);
 	write_vbus_bus_info(dev->visorchannel, hdr_info,
 			    &clientbus_driverinfo);
 
+	visorbus_response(dev, err, CONTROLVM_BUS_CREATE);
+
 	return 0;
 
 err_debugfs_dir:
 	debugfs_remove_recursive(dev->debugfs_dir);
 	kfree(hdr_info);
-	dev_err(&dev->device, "visorbus_create_instance failed: %d\n", err);
+	dev_err(&dev->device, "%s failed: %d\n", __func__, err);
 	return err;
 }
 
@@ -1038,8 +1065,7 @@ err_debugfs_dir:
  * visorbus_remove_instance() - remove a device instance for the visorbus itself
  * @dev: struct visor_device indentifying the bus to remove
  */
-static void
-visorbus_remove_instance(struct visor_device *dev)
+void visorbus_remove_instance(struct visor_device *dev)
 {
 	/*
 	 * Note that this will result in the release method for
@@ -1049,20 +1075,17 @@ visorbus_remove_instance(struct visor_device *dev)
 	 * successfully been able to trace thru the code to see where/how
 	 * release() gets called.  But I know it does.
 	 */
-	if (dev->visorchannel) {
-		visorchannel_destroy(dev->visorchannel);
-		dev->visorchannel = NULL;
-	}
+	visorchannel_destroy(dev->visorchannel);
 	kfree(dev->vbus_hdr_info);
 	list_del(&dev->list_all);
 	device_unregister(&dev->device);
+	visorbus_response(dev, 0, CONTROLVM_BUS_DESTROY);
 }
 
 /*
  * remove_all_visor_devices() - remove all child visorbus device instances
  */
-static void
-remove_all_visor_devices(void)
+static void remove_all_visor_devices(void)
 {
 	struct list_head *listentry, *listtmp;
 
@@ -1074,50 +1097,6 @@ remove_all_visor_devices(void)
 	}
 }
 
-int
-visorchipset_bus_create(struct visor_device *dev)
-{
-	int err;
-
-	err = visorbus_create_instance(dev);
-
-	if (err < 0)
-		return err;
-
-	visorbus_create_response(dev, err);
-
-	return 0;
-}
-
-void
-visorchipset_bus_destroy(struct visor_device *dev)
-{
-	visorbus_remove_instance(dev);
-	visorbus_destroy_response(dev, 0);
-}
-
-int
-visorchipset_device_create(struct visor_device *dev_info)
-{
-	int err;
-
-	err = create_visor_device(dev_info);
-	if (err < 0)
-		return err;
-
-	visorbus_device_create_response(dev_info, err);
-
-	return 0;
-}
-
-void
-visorchipset_device_destroy(struct visor_device *dev_info)
-{
-	remove_visor_device(dev_info);
-
-	visorbus_device_destroy_response(dev_info, 0);
-}
-
 /*
  * pause_state_change_complete() - the callback function to be called by a
  *                                 visorbus function driver when a
@@ -1127,15 +1106,14 @@ visorchipset_device_destroy(struct visor_device *dev_info)
  * @status: 0 iff the pause state change completed successfully, otherwise
  *          a negative errno value indicating the reason for failure
  */
-static void
-pause_state_change_complete(struct visor_device *dev, int status)
+static void pause_state_change_complete(struct visor_device *dev, int status)
 {
 	if (!dev->pausing)
 		return;
 
 	dev->pausing = false;
-
-	visorbus_device_pause_response(dev, status);
+	visorbus_device_changestate_response(dev, status,
+					     segment_state_standby);
 }
 
 /*
@@ -1147,8 +1125,7 @@ pause_state_change_complete(struct visor_device *dev, int status)
  * @status: 0 iff the resume state change completed successfully, otherwise
  *          a negative errno value indicating the reason for failure
  */
-static void
-resume_state_change_complete(struct visor_device *dev, int status)
+static void resume_state_change_complete(struct visor_device *dev, int status)
 {
 	if (!dev->resuming)
 		return;
@@ -1160,7 +1137,8 @@ resume_state_change_complete(struct visor_device *dev, int status)
 	 * which will presumably want to send some sort of response to
 	 * the initiator.
 	 */
-	visorbus_device_resume_response(dev, status);
+	visorbus_device_changestate_response(dev, status,
+					     segment_state_running);
 }
 
 /*
@@ -1174,34 +1152,28 @@ resume_state_change_complete(struct visor_device *dev, int status)
  * via a callback function; see pause_state_change_complete() and
  * resume_state_change_complete().
  */
-static int
-visorchipset_initiate_device_pause_resume(struct visor_device *dev,
-					  bool is_pause)
+static int visorchipset_initiate_device_pause_resume(struct visor_device *dev,
+						     bool is_pause)
 {
 	int err;
 	struct visor_driver *drv = NULL;
 
-	drv = to_visor_driver(dev->device.driver);
-	if (!drv)
-		return -ENODEV;
-
+	/* If no driver associated with the device nothing to pause/resume */
+	if (!dev->device.driver)
+		return 0;
 	if (dev->pausing || dev->resuming)
 		return -EBUSY;
 
+	drv = to_visor_driver(dev->device.driver);
 	if (is_pause) {
-		if (!drv->pause)
-			return -EINVAL;
-
 		dev->pausing = true;
 		err = drv->pause(dev, pause_state_change_complete);
 	} else {
-		/* The vbus_dev_info structure in the channel was been
-		 * cleared, make sure it is valid.
+		/*
+		 * The vbus_dev_info structure in the channel was been cleared,
+		 * make sure it is valid.
 		 */
-		fix_vbus_dev_info(dev);
-		if (!drv->resume)
-			return -EINVAL;
-
+		publish_vbus_dev_info(dev);
 		dev->resuming = true;
 		err = drv->resume(dev, resume_state_change_complete);
 	}
@@ -1209,7 +1181,7 @@ visorchipset_initiate_device_pause_resume(struct visor_device *dev,
 	return err;
 }
 
-/**
+/*
  * visorchipset_device_pause() - start a pause operation for a visor device
  * @dev_info: struct visor_device identifying the device being paused
  *
@@ -1217,13 +1189,11 @@ visorchipset_initiate_device_pause_resume(struct visor_device *dev,
  * that device.  Success/failure result is returned asynchronously
  * via a callback function; see pause_state_change_complete().
  */
-int
-visorchipset_device_pause(struct visor_device *dev_info)
+int visorchipset_device_pause(struct visor_device *dev_info)
 {
 	int err;
 
 	err = visorchipset_initiate_device_pause_resume(dev_info, true);
-
 	if (err < 0) {
 		dev_info->pausing = false;
 		return err;
@@ -1232,7 +1202,7 @@ visorchipset_device_pause(struct visor_device *dev_info)
 	return 0;
 }
 
-/**
+/*
  * visorchipset_device_resume() - start a resume operation for a visor device
  * @dev_info: struct visor_device identifying the device being resumed
  *
@@ -1240,13 +1210,11 @@ visorchipset_device_pause(struct visor_device *dev_info)
  * that device.  Success/failure result is returned asynchronously
  * via a callback function; see resume_state_change_complete().
  */
-int
-visorchipset_device_resume(struct visor_device *dev_info)
+int visorchipset_device_resume(struct visor_device *dev_info)
 {
 	int err;
 
 	err = visorchipset_initiate_device_pause_resume(dev_info, false);
-
 	if (err < 0) {
 		dev_info->resuming = false;
 		return err;
@@ -1255,8 +1223,7 @@ visorchipset_device_resume(struct visor_device *dev_info)
 	return 0;
 }
 
-int
-visorbus_init(void)
+int visorbus_init(void)
 {
 	int err;
 
@@ -1271,14 +1238,12 @@ visorbus_init(void)
 		return err;
 
 	initialized = true;
-
 	bus_device_info_init(&chipset_driverinfo, "chipset", "visorchipset");
 
 	return 0;
 }
 
-void
-visorbus_exit(void)
+void visorbus_exit(void)
 {
 	struct list_head *listentry, *listtmp;
 
diff --git a/drivers/staging/unisys/visorbus/visorbus_private.h b/drivers/staging/unisys/visorbus/visorbus_private.h
index 98a5af19189d..e878d65ab668 100644
--- a/drivers/staging/unisys/visorbus/visorbus_private.h
+++ b/drivers/staging/unisys/visorbus/visorbus_private.h
@@ -1,5 +1,4 @@
-/* visorbus_private.h
- *
+/*
  * Copyright (C) 2010 - 2015 UNISYS CORPORATION
  * All rights reserved.
  *
@@ -22,36 +21,27 @@
 
 #include "controlvmchannel.h"
 #include "vbuschannel.h"
+#include "visorbus.h"
 
-/* TARGET_HOSTNAME specified as -DTARGET_HOSTNAME=\"thename\" on the
- * command line
- */
-
-int visorchipset_bus_create(struct visor_device *bus_info);
-void visorchipset_bus_destroy(struct visor_device *bus_info);
-int visorchipset_device_create(struct visor_device *dev_info);
-void visorchipset_device_destroy(struct visor_device *dev_info);
+int visorbus_create_instance(struct visor_device *dev);
+void visorbus_remove_instance(struct visor_device *bus_info);
+int create_visor_device(struct visor_device *dev_info);
+void remove_visor_device(struct visor_device *dev_info);
 int visorchipset_device_pause(struct visor_device *dev_info);
 int visorchipset_device_resume(struct visor_device *dev_info);
 
-void visorbus_create_response(struct visor_device *p, int response);
-void visorbus_destroy_response(struct visor_device *p, int response);
-void visorbus_device_create_response(struct visor_device *p, int response);
-void visorbus_device_destroy_response(struct visor_device *p, int response);
-void visorbus_device_resume_response(struct visor_device *p, int response);
-void visorbus_device_pause_response(struct visor_device *p, int response);
+void visorbus_response(struct visor_device *p, int response, int controlvm_id);
+void visorbus_device_changestate_response(struct visor_device *p, int response,
+					  struct visor_segment_state state);
 
 int visorbus_init(void);
 void visorbus_exit(void);
 
 /* visorchannel access functions */
-
-struct visorchannel *visorchannel_create(u64 physaddr,
-					 unsigned long channel_bytes,
-					 gfp_t gfp, uuid_le guid);
-struct visorchannel *visorchannel_create_with_lock(u64 physaddr,
-						   unsigned long channel_bytes,
-						   gfp_t gfp, uuid_le guid);
+struct visorchannel *visorchannel_create(u64 physaddr, gfp_t gfp,
+					 const guid_t *guid);
+struct visorchannel *visorchannel_create_with_lock(u64 physaddr, gfp_t gfp,
+						   const guid_t *guid);
 void visorchannel_destroy(struct visorchannel *channel);
 int visorchannel_read(struct visorchannel *channel, ulong offset,
 		      void *dest, ulong nbytes);
@@ -64,6 +54,6 @@ char *visorchannel_zoneid(struct visorchannel *channel, char *s);
 u64 visorchannel_get_clientpartition(struct visorchannel *channel);
 int visorchannel_set_clientpartition(struct visorchannel *channel,
 				     u64 partition_handle);
-char *visorchannel_uuid_id(uuid_le *guid, char *s);
+char *visorchannel_guid_id(const guid_t *guid, char *s);
 void *visorchannel_get_header(struct visorchannel *channel);
 #endif
diff --git a/drivers/staging/unisys/visorbus/visorchannel.c b/drivers/staging/unisys/visorbus/visorchannel.c
index 6885c2cb7135..2a000fee3119 100644
--- a/drivers/staging/unisys/visorbus/visorchannel.c
+++ b/drivers/staging/unisys/visorbus/visorchannel.c
@@ -1,5 +1,4 @@
-/* visorchannel_funcs.c
- *
+/*
  * Copyright (C) 2010 - 2015 UNISYS CORPORATION
  * All rights reserved.
  *
@@ -26,13 +25,13 @@
 #include "visorbus_private.h"
 #include "controlvmchannel.h"
 
-#define MYDRVNAME "visorchannel"
+#define VISOR_DRV_NAME "visorchannel"
 
 #define VISOR_CONSOLEVIDEO_CHANNEL_GUID \
-	UUID_LE(0x3cd6e705, 0xd6a2, 0x4aa5, \
-		0xad, 0x5c, 0x7b, 0x8, 0x88, 0x9d, 0xff, 0xe2)
+	GUID_INIT(0x3cd6e705, 0xd6a2, 0x4aa5, \
+		  0xad, 0x5c, 0x7b, 0x8, 0x88, 0x9d, 0xff, 0xe2)
 
-static const uuid_le visor_video_guid = VISOR_CONSOLEVIDEO_CHANNEL_GUID;
+static const guid_t visor_video_guid = VISOR_CONSOLEVIDEO_CHANNEL_GUID;
 
 struct visorchannel {
 	u64 physaddr;
@@ -40,18 +39,21 @@ struct visorchannel {
 	void *mapped;
 	bool requested;
 	struct channel_header chan_hdr;
-	uuid_le guid;
-	bool needs_lock;	/* channel creator knows if more than one */
-				/* thread will be inserting or removing */
-	spinlock_t insert_lock; /* protect head writes in chan_hdr */
-	spinlock_t remove_lock;	/* protect tail writes in chan_hdr */
-
-	uuid_le type;
-	uuid_le inst;
+	guid_t guid;
+	/*
+	 * channel creator knows if more than one
+	 * thread will be inserting or removing
+	 */
+	bool needs_lock;
+	/* protect head writes in chan_hdr */
+	spinlock_t insert_lock;
+	/* protect tail writes in chan_hdr */
+	spinlock_t remove_lock;
+	guid_t type;
+	guid_t inst;
 };
 
-void
-visorchannel_destroy(struct visorchannel *channel)
+void visorchannel_destroy(struct visorchannel *channel)
 {
 	if (!channel)
 		return;
@@ -63,67 +65,58 @@ visorchannel_destroy(struct visorchannel *channel)
 	kfree(channel);
 }
 
-u64
-visorchannel_get_physaddr(struct visorchannel *channel)
+u64 visorchannel_get_physaddr(struct visorchannel *channel)
 {
 	return channel->physaddr;
 }
 
-ulong
-visorchannel_get_nbytes(struct visorchannel *channel)
+ulong visorchannel_get_nbytes(struct visorchannel *channel)
 {
 	return channel->nbytes;
 }
 
-char *
-visorchannel_uuid_id(uuid_le *guid, char *s)
+char *visorchannel_guid_id(const guid_t *guid, char *s)
 {
 	sprintf(s, "%pUL", guid);
 	return s;
 }
 
-char *
-visorchannel_id(struct visorchannel *channel, char *s)
+char *visorchannel_id(struct visorchannel *channel, char *s)
 {
-	return visorchannel_uuid_id(&channel->guid, s);
+	return visorchannel_guid_id(&channel->guid, s);
 }
 
-char *
-visorchannel_zoneid(struct visorchannel *channel, char *s)
+char *visorchannel_zoneid(struct visorchannel *channel, char *s)
 {
-	return visorchannel_uuid_id(&channel->chan_hdr.zone_uuid, s);
+	return visorchannel_guid_id(&channel->chan_hdr.zone_guid, s);
 }
 
-u64
-visorchannel_get_clientpartition(struct visorchannel *channel)
+u64 visorchannel_get_clientpartition(struct visorchannel *channel)
 {
 	return channel->chan_hdr.partition_handle;
 }
 
-int
-visorchannel_set_clientpartition(struct visorchannel *channel,
-				 u64 partition_handle)
+int visorchannel_set_clientpartition(struct visorchannel *channel,
+				     u64 partition_handle)
 {
 	channel->chan_hdr.partition_handle = partition_handle;
 	return 0;
 }
 
 /**
- * visorchannel_get_uuid() - queries the UUID of the designated channel
+ * visorchannel_get_guid() - queries the GUID of the designated channel
  * @channel: the channel to query
  *
- * Return: the UUID of the provided channel
+ * Return: the GUID of the provided channel
  */
-uuid_le
-visorchannel_get_uuid(struct visorchannel *channel)
+const guid_t *visorchannel_get_guid(struct visorchannel *channel)
 {
-	return channel->guid;
+	return &channel->guid;
 }
-EXPORT_SYMBOL_GPL(visorchannel_get_uuid);
+EXPORT_SYMBOL_GPL(visorchannel_get_guid);
 
-int
-visorchannel_read(struct visorchannel *channel, ulong offset,
-		  void *dest, ulong nbytes)
+int visorchannel_read(struct visorchannel *channel, ulong offset, void *dest,
+		      ulong nbytes)
 {
 	if (offset + nbytes > channel->nbytes)
 		return -EIO;
@@ -133,9 +126,8 @@ visorchannel_read(struct visorchannel *channel, ulong offset,
 	return 0;
 }
 
-int
-visorchannel_write(struct visorchannel *channel, ulong offset,
-		   void *dest, ulong nbytes)
+int visorchannel_write(struct visorchannel *channel, ulong offset, void *dest,
+		       ulong nbytes)
 {
 	size_t chdr_size = sizeof(struct channel_header);
 	size_t copy_size;
@@ -154,8 +146,7 @@ visorchannel_write(struct visorchannel *channel, ulong offset,
 	return 0;
 }
 
-void *
-visorchannel_get_header(struct visorchannel *channel)
+void *visorchannel_get_header(struct visorchannel *channel)
 {
 	return &channel->chan_hdr;
 }
@@ -164,17 +155,22 @@ visorchannel_get_header(struct visorchannel *channel)
  * Return offset of a specific SIGNAL_QUEUE_HEADER from the beginning of a
  * channel header
  */
-#define SIG_QUEUE_OFFSET(chan_hdr, q) \
-	((chan_hdr)->ch_space_offset + \
-	 ((q) * sizeof(struct signal_queue_header)))
+static int sig_queue_offset(struct channel_header *chan_hdr, int q)
+{
+	return ((chan_hdr)->ch_space_offset +
+	       ((q) * sizeof(struct signal_queue_header)));
+}
 
 /*
  * Return offset of a specific queue entry (data) from the beginning of a
  * channel header
  */
-#define SIG_DATA_OFFSET(chan_hdr, q, sig_hdr, slot) \
-	(SIG_QUEUE_OFFSET(chan_hdr, q) + (sig_hdr)->sig_base_offset + \
-	 ((slot) * (sig_hdr)->signal_size))
+static int sig_data_offset(struct channel_header *chan_hdr, int q,
+			   struct signal_queue_header *sig_hdr, int slot)
+{
+	return (sig_queue_offset(chan_hdr, q) + sig_hdr->sig_base_offset +
+	       (slot * sig_hdr->signal_size));
+}
 
 /*
  * Write the contents of a specific field within a SIGNAL_QUEUE_HEADER back
@@ -182,48 +178,47 @@ visorchannel_get_header(struct visorchannel *channel)
  */
 #define SIG_WRITE_FIELD(channel, queue, sig_hdr, FIELD) \
 	visorchannel_write(channel, \
-			   SIG_QUEUE_OFFSET(&channel->chan_hdr, queue) + \
+			   sig_queue_offset(&channel->chan_hdr, queue) + \
 			   offsetof(struct signal_queue_header, FIELD), \
 			   &((sig_hdr)->FIELD), \
 			   sizeof((sig_hdr)->FIELD))
 
-static int
-sig_read_header(struct visorchannel *channel, u32 queue,
-		struct signal_queue_header *sig_hdr)
+static int sig_read_header(struct visorchannel *channel, u32 queue,
+			   struct signal_queue_header *sig_hdr)
 {
 	if (channel->chan_hdr.ch_space_offset < sizeof(struct channel_header))
 		return -EINVAL;
 
 	/* Read the appropriate SIGNAL_QUEUE_HEADER into local memory. */
 	return visorchannel_read(channel,
-				 SIG_QUEUE_OFFSET(&channel->chan_hdr, queue),
+				 sig_queue_offset(&channel->chan_hdr, queue),
 				 sig_hdr, sizeof(struct signal_queue_header));
 }
 
-static int
-sig_read_data(struct visorchannel *channel, u32 queue,
-	      struct signal_queue_header *sig_hdr, u32 slot, void *data)
+static int sig_read_data(struct visorchannel *channel, u32 queue,
+			 struct signal_queue_header *sig_hdr, u32 slot,
+			 void *data)
 {
-	int signal_data_offset = SIG_DATA_OFFSET(&channel->chan_hdr, queue,
+	int signal_data_offset = sig_data_offset(&channel->chan_hdr, queue,
 						 sig_hdr, slot);
 
 	return visorchannel_read(channel, signal_data_offset,
 				 data, sig_hdr->signal_size);
 }
 
-static int
-sig_write_data(struct visorchannel *channel, u32 queue,
-	       struct signal_queue_header *sig_hdr, u32 slot, void *data)
+static int sig_write_data(struct visorchannel *channel, u32 queue,
+			  struct signal_queue_header *sig_hdr, u32 slot,
+			  void *data)
 {
-	int signal_data_offset = SIG_DATA_OFFSET(&channel->chan_hdr, queue,
+	int signal_data_offset = sig_data_offset(&channel->chan_hdr, queue,
 						 sig_hdr, slot);
 
 	return visorchannel_write(channel, signal_data_offset,
 				  data, sig_hdr->signal_size);
 }
 
-static int
-signalremove_inner(struct visorchannel *channel, u32 queue, void *msg)
+static int signalremove_inner(struct visorchannel *channel, u32 queue,
+			      void *msg)
 {
 	struct signal_queue_header sig_hdr;
 	int error;
@@ -246,9 +241,9 @@ signalremove_inner(struct visorchannel *channel, u32 queue, void *msg)
 
 	/*
 	 * For each data field in SIGNAL_QUEUE_HEADER that was modified,
-	 * update host memory.
+	 * update host memory. Required for channel sync.
 	 */
-	mb(); /* required for channel synch */
+	mb();
 
 	error = SIG_WRITE_FIELD(channel, queue, &sig_hdr, tail);
 	if (error)
@@ -269,8 +264,8 @@ signalremove_inner(struct visorchannel *channel, u32 queue, void *msg)
  *
  * Return: integer error code indicating the status of the removal
  */
-int
-visorchannel_signalremove(struct visorchannel *channel, u32 queue, void *msg)
+int visorchannel_signalremove(struct visorchannel *channel, u32 queue,
+			      void *msg)
 {
 	int rc;
 	unsigned long flags;
@@ -287,8 +282,7 @@ visorchannel_signalremove(struct visorchannel *channel, u32 queue, void *msg)
 }
 EXPORT_SYMBOL_GPL(visorchannel_signalremove);
 
-static bool
-queue_empty(struct visorchannel *channel, u32 queue)
+static bool queue_empty(struct visorchannel *channel, u32 queue)
 {
 	struct signal_queue_header sig_hdr;
 
@@ -307,8 +301,7 @@ queue_empty(struct visorchannel *channel, u32 queue)
  * Return: boolean indicating whether any messages in the designated
  *         channel/queue are present
  */
-bool
-visorchannel_signalempty(struct visorchannel *channel, u32 queue)
+bool visorchannel_signalempty(struct visorchannel *channel, u32 queue)
 {
 	bool rc;
 	unsigned long flags;
@@ -324,8 +317,8 @@ visorchannel_signalempty(struct visorchannel *channel, u32 queue)
 }
 EXPORT_SYMBOL_GPL(visorchannel_signalempty);
 
-static int
-signalinsert_inner(struct visorchannel *channel, u32 queue, void *msg)
+static int signalinsert_inner(struct visorchannel *channel, u32 queue,
+			      void *msg)
 {
 	struct signal_queue_header sig_hdr;
 	int err;
@@ -351,9 +344,9 @@ signalinsert_inner(struct visorchannel *channel, u32 queue, void *msg)
 
 	/*
 	 * For each data field in SIGNAL_QUEUE_HEADER that was modified,
-	 * update host memory.
+	 * update host memory. Required for channel sync.
 	 */
-	mb(); /* required for channel synch */
+	mb();
 
 	err = SIG_WRITE_FIELD(channel, queue, &sig_hdr, head);
 	if (err)
@@ -370,17 +363,8 @@ signalinsert_inner(struct visorchannel *channel, u32 queue, void *msg)
  *                              for a data area in memory, but does NOT modify
  *                              this data area
  * @physaddr:      physical address of start of channel
- * @channel_bytes: size of the channel in bytes; this may 0 if the channel has
- *                 already been initialized in memory (which is true for all
- *                 channels provided to guest environments by the s-Par
- *                 back-end), in which case the actual channel size will be
- *                 read from the channel header in memory
  * @gfp:           gfp_t to use when allocating memory for the data struct
- * @guid:          uuid that identifies channel type; this may 0 if the channel
- *                 has already been initialized in memory (which is true for all
- *                 channels provided to guest environments by the s-Par
- *                 back-end), in which case the actual channel guid will be
- *                 read from the channel header in memory
+ * @guid:          GUID that identifies channel type;
  * @needs_lock:    must specify true if you have multiple threads of execution
  *                 that will be calling visorchannel methods of this
  *                 visorchannel at the same time
@@ -388,9 +372,9 @@ signalinsert_inner(struct visorchannel *channel, u32 queue, void *msg)
  * Return: pointer to visorchannel that was created if successful,
  *         otherwise NULL
  */
-static struct visorchannel *
-visorchannel_create_guts(u64 physaddr, unsigned long channel_bytes,
-			 gfp_t gfp, uuid_le guid, bool needs_lock)
+static struct visorchannel *visorchannel_create_guts(u64 physaddr, gfp_t gfp,
+						     const guid_t *guid,
+						     bool needs_lock)
 {
 	struct visorchannel *channel;
 	int err;
@@ -414,8 +398,8 @@ visorchannel_create_guts(u64 physaddr, unsigned long channel_bytes,
 	 * this. Remember that we haven't requested it so we don't try to
 	 * release later on.
 	 */
-	channel->requested = request_mem_region(physaddr, size, MYDRVNAME);
-	if (!channel->requested && uuid_le_cmp(guid, visor_video_guid))
+	channel->requested = request_mem_region(physaddr, size, VISOR_DRV_NAME);
+	if (!channel->requested && !guid_equal(guid, &visor_video_guid))
 		/* we only care about errors if this is not the video channel */
 		goto err_destroy_channel;
 
@@ -428,36 +412,29 @@ visorchannel_create_guts(u64 physaddr, unsigned long channel_bytes,
 	channel->physaddr = physaddr;
 	channel->nbytes = size;
 
-	err = visorchannel_read(channel, 0, &channel->chan_hdr,
-				sizeof(struct channel_header));
+	err = visorchannel_read(channel, 0, &channel->chan_hdr, size);
 	if (err)
 		goto err_destroy_channel;
-
-	/* we had better be a CLIENT of this channel */
-	if (channel_bytes == 0)
-		channel_bytes = (ulong)channel->chan_hdr.size;
-	if (uuid_le_cmp(guid, NULL_UUID_LE) == 0)
-		guid = channel->chan_hdr.chtype;
+	size = (ulong)channel->chan_hdr.size;
 
 	memunmap(channel->mapped);
 	if (channel->requested)
 		release_mem_region(channel->physaddr, channel->nbytes);
 	channel->mapped = NULL;
-	channel->requested = request_mem_region(channel->physaddr,
-						channel_bytes, MYDRVNAME);
-	if (!channel->requested && uuid_le_cmp(guid, visor_video_guid))
+	channel->requested = request_mem_region(channel->physaddr, size,
+						VISOR_DRV_NAME);
+	if (!channel->requested && !guid_equal(guid, &visor_video_guid))
 		/* we only care about errors if this is not the video channel */
 		goto err_destroy_channel;
 
-	channel->mapped = memremap(channel->physaddr, channel_bytes,
-			MEMREMAP_WB);
+	channel->mapped = memremap(channel->physaddr, size, MEMREMAP_WB);
 	if (!channel->mapped) {
-		release_mem_region(channel->physaddr, channel_bytes);
+		release_mem_region(channel->physaddr, size);
 		goto err_destroy_channel;
 	}
 
-	channel->nbytes = channel_bytes;
-	channel->guid = guid;
+	channel->nbytes = size;
+	guid_copy(&channel->guid, guid);
 	return channel;
 
 err_destroy_channel:
@@ -465,20 +442,16 @@ err_destroy_channel:
 	return NULL;
 }
 
-struct visorchannel *
-visorchannel_create(u64 physaddr, unsigned long channel_bytes,
-		    gfp_t gfp, uuid_le guid)
+struct visorchannel *visorchannel_create(u64 physaddr, gfp_t gfp,
+					 const guid_t *guid)
 {
-	return visorchannel_create_guts(physaddr, channel_bytes, gfp, guid,
-					false);
+	return visorchannel_create_guts(physaddr, gfp, guid, false);
 }
 
-struct visorchannel *
-visorchannel_create_with_lock(u64 physaddr, unsigned long channel_bytes,
-			      gfp_t gfp, uuid_le guid)
+struct visorchannel *visorchannel_create_with_lock(u64 physaddr, gfp_t gfp,
+						   const guid_t *guid)
 {
-	return visorchannel_create_guts(physaddr, channel_bytes, gfp, guid,
-					true);
+	return visorchannel_create_guts(physaddr, gfp, guid, true);
 }
 
 /**
@@ -490,8 +463,8 @@ visorchannel_create_with_lock(u64 physaddr, unsigned long channel_bytes,
  *
  * Return: integer error code indicating the status of the insertion
  */
-int
-visorchannel_signalinsert(struct visorchannel *channel, u32 queue, void *msg)
+int visorchannel_signalinsert(struct visorchannel *channel, u32 queue,
+			      void *msg)
 {
 	int rc;
 	unsigned long flags;
diff --git a/drivers/staging/unisys/visorbus/visorchipset.c b/drivers/staging/unisys/visorbus/visorchipset.c
index 22150564b4fb..74cce4f1a7bd 100644
--- a/drivers/staging/unisys/visorbus/visorchipset.c
+++ b/drivers/staging/unisys/visorbus/visorchipset.c
@@ -1,5 +1,4 @@
-/* visorchipset_main.c
- *
+/*
  * Copyright (C) 2010 - 2015 UNISYS CORPORATION
  * All rights reserved.
  *
@@ -15,19 +14,18 @@
  */
 
 #include <linux/acpi.h>
-#include <linux/ctype.h>
-#include <linux/fs.h>
-#include <linux/mm.h>
-#include <linux/nls.h>
-#include <linux/netdevice.h>
-#include <linux/uuid.h>
 #include <linux/crash_dump.h>
 
 #include "visorbus.h"
 #include "visorbus_private.h"
-#include "vmcallinterface.h"
 
-#define CURRENT_FILE_PC VISOR_BUS_PC_visorchipset_c
+/* {72120008-4AAB-11DC-8530-444553544200} */
+#define VISOR_SIOVM_GUID GUID_INIT(0x72120008, 0x4AAB, 0x11DC, 0x85, 0x30, \
+				   0x44, 0x45, 0x53, 0x54, 0x42, 0x00)
+
+static const guid_t visor_vhba_channel_guid = VISOR_VHBA_CHANNEL_GUID;
+static const guid_t visor_siovm_guid = VISOR_SIOVM_GUID;
+static const guid_t visor_controlvm_channel_guid = VISOR_CONTROLVM_CHANNEL_GUID;
 
 #define POLLJIFFIES_CONTROLVMCHANNEL_FAST 1
 #define POLLJIFFIES_CONTROLVMCHANNEL_SLOW 100
@@ -42,9 +40,9 @@
 #define UNISYS_VISOR_ID_EDX 0x34367261
 
 /*
- * When the controlvm channel is idle for at least MIN_IDLE_SECONDS,
- * we switch to slow polling mode. As soon as we get a controlvm
- * message, we switch back to fast polling mode.
+ * When the controlvm channel is idle for at least MIN_IDLE_SECONDS, we switch
+ * to slow polling mode. As soon as we get a controlvm message, we switch back
+ * to fast polling mode.
  */
 #define MIN_IDLE_SECONDS 10
 
@@ -54,15 +52,39 @@ struct parser_context {
 	u8 *curr;
 	unsigned long bytes_remaining;
 	bool byte_stream;
-	char data[0];
+	struct visor_controlvm_parameters_header data;
 };
 
-struct vmcall_controlvm_addr {
-	struct vmcall_io_controlvm_addr_params params;
-	int err;
-	u64 physaddr;
+/* VMCALL_CONTROLVM_ADDR: Used by all guests, not just IO. */
+#define VMCALL_CONTROLVM_ADDR 0x0501
+
+enum vmcall_result {
+	VMCALL_RESULT_SUCCESS = 0,
+	VMCALL_RESULT_INVALID_PARAM = 1,
+	VMCALL_RESULT_DATA_UNAVAILABLE = 2,
+	VMCALL_RESULT_FAILURE_UNAVAILABLE = 3,
+	VMCALL_RESULT_DEVICE_ERROR = 4,
+	VMCALL_RESULT_DEVICE_NOT_READY = 5
 };
 
+/*
+ * struct vmcall_io_controlvm_addr_params - Structure for IO VMCALLS. Has
+ *					    parameters to VMCALL_CONTROLVM_ADDR
+ *					    interface.
+ * @address:	   The Guest-relative physical address of the ControlVm channel.
+ *		   This VMCall fills this in with the appropriate address.
+ *		   Contents provided by this VMCALL (OUT).
+ * @channel_bytes: The size of the ControlVm channel in bytes This VMCall fills
+ *		   this in with the appropriate address. Contents provided by
+ *		   this VMCALL (OUT).
+ * @unused:	   Unused Bytes in the 64-Bit Aligned Struct.
+ */
+struct vmcall_io_controlvm_addr_params {
+	u64 address;
+	u32 channel_bytes;
+	u8 unused[4];
+} __packed;
+
 struct visorchipset_device {
 	struct acpi_device *acpi_device;
 	unsigned long poll_jiffies;
@@ -80,7 +102,7 @@ struct visorchipset_device {
 	 */
 	struct controlvm_message controlvm_pending_msg;
 	bool controlvm_pending_msg_valid;
-	struct vmcall_controlvm_addr controlvm_addr;
+	struct vmcall_io_controlvm_addr_params controlvm_params;
 };
 
 static struct visorchipset_device *chipset_dev;
@@ -124,7 +146,6 @@ static ssize_t toolaction_store(struct device *dev,
 				 offsetof(struct visor_controlvm_channel,
 					  tool_action),
 				 &tool_action, sizeof(u8));
-
 	if (err)
 		return err;
 	return count;
@@ -143,7 +164,6 @@ static ssize_t boottotool_show(struct device *dev,
 					 efi_visor_ind),
 				&efi_visor_indication,
 				sizeof(struct efi_visor_indication));
-
 	if (err)
 		return err;
 	return sprintf(buf, "%u\n", efi_visor_indication.boot_to_tool);
@@ -165,7 +185,6 @@ static ssize_t boottotool_store(struct device *dev,
 					  efi_visor_ind),
 				 &(efi_visor_indication),
 				 sizeof(struct efi_visor_indication));
-
 	if (err)
 		return err;
 	return count;
@@ -184,7 +203,7 @@ static ssize_t error_show(struct device *dev, struct device_attribute *attr,
 				&error, sizeof(u32));
 	if (err)
 		return err;
-	return sprintf(buf, "%i\n", error);
+	return sprintf(buf, "%u\n", error);
 }
 
 static ssize_t error_store(struct device *dev, struct device_attribute *attr,
@@ -219,7 +238,7 @@ static ssize_t textid_show(struct device *dev, struct device_attribute *attr,
 	if (err)
 		return err;
 
-	return sprintf(buf, "%i\n", text_id);
+	return sprintf(buf, "%u\n", text_id);
 }
 
 static ssize_t textid_store(struct device *dev, struct device_attribute *attr,
@@ -277,67 +296,6 @@ static ssize_t remaining_steps_store(struct device *dev,
 }
 static DEVICE_ATTR_RW(remaining_steps);
 
-static uuid_le
-parser_id_get(struct parser_context *ctx)
-{
-	struct visor_controlvm_parameters_header *phdr = NULL;
-
-	phdr = (struct visor_controlvm_parameters_header *)(ctx->data);
-	return phdr->id;
-}
-
-static void parser_done(struct parser_context *ctx)
-{
-	chipset_dev->controlvm_payload_bytes_buffered -= ctx->param_bytes;
-	kfree(ctx);
-}
-
-static void *
-parser_string_get(struct parser_context *ctx)
-{
-	u8 *pscan;
-	unsigned long nscan;
-	int value_length = -1;
-	void *value = NULL;
-	int i;
-
-	pscan = ctx->curr;
-	nscan = ctx->bytes_remaining;
-	if (nscan == 0)
-		return NULL;
-	if (!pscan)
-		return NULL;
-	for (i = 0, value_length = -1; i < nscan; i++)
-		if (pscan[i] == '\0') {
-			value_length = i;
-			break;
-		}
-	if (value_length < 0)	/* '\0' was not included in the length */
-		value_length = nscan;
-	value = kmalloc(value_length + 1, GFP_KERNEL);
-	if (!value)
-		return NULL;
-	if (value_length > 0)
-		memcpy(value, pscan, value_length);
-	((u8 *)(value))[value_length] = '\0';
-	return value;
-}
-
-static void *
-parser_name_get(struct parser_context *ctx)
-{
-	struct visor_controlvm_parameters_header *phdr = NULL;
-
-	phdr = (struct visor_controlvm_parameters_header *)(ctx->data);
-
-	if (phdr->name_offset + phdr->name_length > ctx->param_bytes)
-		return NULL;
-
-	ctx->curr = ctx->data + phdr->name_offset;
-	ctx->bytes_remaining = phdr->name_length;
-	return parser_string_get(ctx);
-}
-
 struct visor_busdev {
 	u32 bus_no;
 	u32 dev_no;
@@ -347,11 +305,9 @@ static int match_visorbus_dev_by_id(struct device *dev, void *data)
 {
 	struct visor_device *vdev = to_visor_device(dev);
 	struct visor_busdev *id = data;
-	u32 bus_no = id->bus_no;
-	u32 dev_no = id->dev_no;
 
-	if ((vdev->chipset_bus_no == bus_no) &&
-	    (vdev->chipset_dev_no == dev_no))
+	if ((vdev->chipset_bus_no == id->bus_no) &&
+	    (vdev->chipset_dev_no == id->dev_no))
 		return 1;
 
 	return 0;
@@ -364,9 +320,9 @@ struct visor_device *visorbus_get_device_by_id(u32 bus_no, u32 dev_no,
 	struct device *dev_start = NULL;
 	struct visor_device *vdev = NULL;
 	struct visor_busdev id = {
-			.bus_no = bus_no,
-			.dev_no = dev_no
-		};
+		.bus_no = bus_no,
+		.dev_no = dev_no
+	};
 
 	if (from)
 		dev_start = &from->device;
@@ -377,9 +333,9 @@ struct visor_device *visorbus_get_device_by_id(u32 bus_no, u32 dev_no,
 	return vdev;
 }
 
-static void
-controlvm_init_response(struct controlvm_message *msg,
-			struct controlvm_message_header *msg_hdr, int response)
+static void controlvm_init_response(struct controlvm_message *msg,
+				    struct controlvm_message_header *msg_hdr,
+				    int response)
 {
 	memset(msg, 0, sizeof(struct controlvm_message));
 	memcpy(&msg->hdr, msg_hdr, sizeof(struct controlvm_message_header));
@@ -392,10 +348,10 @@ controlvm_init_response(struct controlvm_message *msg,
 	}
 }
 
-static int
-controlvm_respond_chipset_init(struct controlvm_message_header *msg_hdr,
-			       int response,
-			       enum visor_chipset_feature features)
+static int controlvm_respond_chipset_init(
+				struct controlvm_message_header *msg_hdr,
+				int response,
+				enum visor_chipset_feature features)
 {
 	struct controlvm_message outmsg;
 
@@ -405,8 +361,7 @@ controlvm_respond_chipset_init(struct controlvm_message_header *msg_hdr,
 					 CONTROLVM_QUEUE_REQUEST, &outmsg);
 }
 
-static int
-chipset_init(struct controlvm_message *inmsg)
+static int chipset_init(struct controlvm_message *inmsg)
 {
 	static int chipset_inited;
 	enum visor_chipset_feature features = 0;
@@ -421,15 +376,15 @@ chipset_init(struct controlvm_message *inmsg)
 	chipset_inited = 1;
 
 	/*
-	 * Set features to indicate we support parahotplug (if Command
-	 * also supports it).
+	 * Set features to indicate we support parahotplug (if Command also
+	 * supports it).
 	 */
 	features = inmsg->cmd.init_chipset.features &
 		   VISOR_CHIPSET_FEATURE_PARA_HOTPLUG;
 
 	/*
-	 * Set the "reply" bit so Command knows this is a
-	 * features-aware driver.
+	 * Set the "reply" bit so Command knows this is a features-aware
+	 * driver.
 	 */
 	features |= VISOR_CHIPSET_FEATURE_REPLY;
 
@@ -440,9 +395,9 @@ out_respond:
 	return res;
 }
 
-static int
-controlvm_respond(struct controlvm_message_header *msg_hdr, int response,
-		  struct visor_segment_state *state)
+static int controlvm_respond(struct controlvm_message_header *msg_hdr,
+			     int response,
+			     struct visor_segment_state *state)
 {
 	struct controlvm_message outmsg;
 
@@ -464,8 +419,8 @@ enum crash_obj_type {
 	CRASH_BUS,
 };
 
-static int
-save_crash_message(struct controlvm_message *msg, enum crash_obj_type cr_type)
+static int save_crash_message(struct controlvm_message *msg,
+			      enum crash_obj_type cr_type)
 {
 	u32 local_crash_msg_offset;
 	u16 local_crash_msg_count;
@@ -529,10 +484,9 @@ save_crash_message(struct controlvm_message *msg, enum crash_obj_type cr_type)
 	return 0;
 }
 
-static int
-controlvm_responder(enum controlvm_id cmd_id,
-		    struct controlvm_message_header *pending_msg_hdr,
-		    int response)
+static int controlvm_responder(enum controlvm_id cmd_id,
+			       struct controlvm_message_header *pending_msg_hdr,
+			       int response)
 {
 	if (!pending_msg_hdr)
 		return -EIO;
@@ -543,14 +497,12 @@ controlvm_responder(enum controlvm_id cmd_id,
 	return controlvm_respond(pending_msg_hdr, response, NULL);
 }
 
-static int
-device_changestate_responder(enum controlvm_id cmd_id,
-			     struct visor_device *p, int response,
-			     struct visor_segment_state response_state)
+static int device_changestate_responder(
+				enum controlvm_id cmd_id,
+				struct visor_device *p, int response,
+				struct visor_segment_state response_state)
 {
 	struct controlvm_message outmsg;
-	u32 bus_no = p->chipset_bus_no;
-	u32 dev_no = p->chipset_dev_no;
 
 	if (!p->pending_msg_hdr)
 		return -EIO;
@@ -559,16 +511,15 @@ device_changestate_responder(enum controlvm_id cmd_id,
 
 	controlvm_init_response(&outmsg, p->pending_msg_hdr, response);
 
-	outmsg.cmd.device_change_state.bus_no = bus_no;
-	outmsg.cmd.device_change_state.dev_no = dev_no;
+	outmsg.cmd.device_change_state.bus_no = p->chipset_bus_no;
+	outmsg.cmd.device_change_state.dev_no = p->chipset_dev_no;
 	outmsg.cmd.device_change_state.state = response_state;
 
 	return visorchannel_signalinsert(chipset_dev->controlvm_channel,
 					 CONTROLVM_QUEUE_REQUEST, &outmsg);
 }
 
-static int
-visorbus_create(struct controlvm_message *inmsg)
+static int visorbus_create(struct controlvm_message *inmsg)
 {
 	struct controlvm_message_packet *cmd = &inmsg->cmd;
 	struct controlvm_message_header *pmsg_hdr = NULL;
@@ -580,7 +531,7 @@ visorbus_create(struct controlvm_message *inmsg)
 	bus_info = visorbus_get_device_by_id(bus_no, BUS_ROOT_DEVICE, NULL);
 	if (bus_info && (bus_info->state.created == 1)) {
 		dev_err(&chipset_dev->acpi_device->dev,
-			"failed visorbus_create: already exists\n");
+			"failed %s: already exists\n", __func__);
 		err = -EEXIST;
 		goto err_respond;
 	}
@@ -595,7 +546,7 @@ visorbus_create(struct controlvm_message *inmsg)
 	bus_info->chipset_bus_no = bus_no;
 	bus_info->chipset_dev_no = BUS_ROOT_DEVICE;
 
-	if (uuid_le_cmp(cmd->create_bus.bus_inst_uuid, visor_siovm_uuid) == 0) {
+	if (guid_equal(&cmd->create_bus.bus_inst_guid, &visor_siovm_guid)) {
 		err = save_crash_message(inmsg, CRASH_BUS);
 		if (err)
 			goto err_free_bus_info;
@@ -615,19 +566,17 @@ visorbus_create(struct controlvm_message *inmsg)
 	}
 
 	visorchannel = visorchannel_create(cmd->create_bus.channel_addr,
-					   cmd->create_bus.channel_bytes,
 					   GFP_KERNEL,
-					   cmd->create_bus.bus_data_type_uuid);
-
+					   &cmd->create_bus.bus_data_type_guid);
 	if (!visorchannel) {
 		err = -ENOMEM;
 		goto err_free_pending_msg;
 	}
+
 	bus_info->visorchannel = visorchannel;
 
-	/* Response will be handled by visorchipset_bus_create */
-	err = visorchipset_bus_create(bus_info);
-	/* If visorchipset_bus_create didn't respond, need to respond here */
+	/* Response will be handled by visorbus_create_instance on success */
+	err = visorbus_create_instance(bus_info);
 	if (err)
 		goto err_destroy_channel;
 
@@ -648,8 +597,7 @@ err_respond:
 	return err;
 }
 
-static int
-visorbus_destroy(struct controlvm_message *inmsg)
+static int visorbus_destroy(struct controlvm_message *inmsg)
 {
 	struct controlvm_message_packet *cmd = &inmsg->cmd;
 	struct controlvm_message_header *pmsg_hdr = NULL;
@@ -683,8 +631,8 @@ visorbus_destroy(struct controlvm_message *inmsg)
 		bus_info->pending_msg_hdr = pmsg_hdr;
 	}
 
-	/* Response will be handled by visorchipset_bus_destroy */
-	visorchipset_bus_destroy(bus_info);
+	/* Response will be handled by visorbus_remove_instance */
+	visorbus_remove_instance(bus_info);
 	return 0;
 
 err_respond:
@@ -693,9 +641,60 @@ err_respond:
 	return err;
 }
 
-static int
-visorbus_configure(struct controlvm_message *inmsg,
-		   struct parser_context *parser_ctx)
+static const guid_t *parser_id_get(struct parser_context *ctx)
+{
+	return &ctx->data.id;
+}
+
+static void *parser_string_get(struct parser_context *ctx)
+{
+	u8 *pscan;
+	unsigned long nscan;
+	int value_length;
+	void *value;
+	int i;
+
+	pscan = ctx->curr;
+	if (!pscan)
+		return NULL;
+	nscan = ctx->bytes_remaining;
+	if (nscan == 0)
+		return NULL;
+
+	for (i = 0, value_length = -1; i < nscan; i++)
+		if (pscan[i] == '\0') {
+			value_length = i;
+			break;
+		}
+	/* '\0' was not included in the length */
+	if (value_length < 0)
+		value_length = nscan;
+
+	value = kmalloc(value_length + 1, GFP_KERNEL);
+	if (!value)
+		return NULL;
+	if (value_length > 0)
+		memcpy(value, pscan, value_length);
+	((u8 *)(value))[value_length] = '\0';
+	return value;
+}
+
+static void *parser_name_get(struct parser_context *ctx)
+{
+	struct visor_controlvm_parameters_header *phdr = NULL;
+
+	phdr = &ctx->data;
+
+	if (phdr->name_offset + phdr->name_length > ctx->param_bytes)
+		return NULL;
+
+	ctx->curr = (char *)&phdr + phdr->name_offset;
+	ctx->bytes_remaining = phdr->name_length;
+	return parser_string_get(ctx);
+}
+
+static int visorbus_configure(struct controlvm_message *inmsg,
+			      struct parser_context *parser_ctx)
 {
 	struct controlvm_message_packet *cmd = &inmsg->cmd;
 	u32 bus_no;
@@ -707,10 +706,12 @@ visorbus_configure(struct controlvm_message *inmsg,
 	if (!bus_info) {
 		err = -EINVAL;
 		goto err_respond;
-	} else if (bus_info->state.created == 0) {
+	}
+	if (bus_info->state.created == 0) {
 		err = -EINVAL;
 		goto err_respond;
-	} else if (bus_info->pending_msg_hdr) {
+	}
+	if (bus_info->pending_msg_hdr) {
 		err = -EIO;
 		goto err_respond;
 	}
@@ -722,7 +723,9 @@ visorbus_configure(struct controlvm_message *inmsg,
 		goto err_respond;
 
 	if (parser_ctx) {
-		bus_info->partition_uuid = parser_id_get(parser_ctx);
+		const guid_t *partition_guid = parser_id_get(parser_ctx);
+
+		guid_copy(&bus_info->partition_guid, partition_guid);
 		bus_info->name = parser_name_get(parser_ctx);
 	}
 
@@ -732,14 +735,13 @@ visorbus_configure(struct controlvm_message *inmsg,
 
 err_respond:
 	dev_err(&chipset_dev->acpi_device->dev,
-		"visorbus_configure exited with err: %d\n", err);
+		"%s exited with err: %d\n", __func__, err);
 	if (inmsg->hdr.flags.response_expected == 1)
 		controlvm_responder(inmsg->hdr.id, &inmsg->hdr, err);
 	return err;
 }
 
-static int
-visorbus_device_create(struct controlvm_message *inmsg)
+static int visorbus_device_create(struct controlvm_message *inmsg)
 {
 	struct controlvm_message_packet *cmd = &inmsg->cmd;
 	struct controlvm_message_header *pmsg_hdr = NULL;
@@ -757,7 +759,6 @@ visorbus_device_create(struct controlvm_message *inmsg)
 		err = -ENODEV;
 		goto err_respond;
 	}
-
 	if (bus_info->state.created == 0) {
 		dev_err(&chipset_dev->acpi_device->dev,
 			"bus not created, id: %d\n", bus_no);
@@ -781,17 +782,13 @@ visorbus_device_create(struct controlvm_message *inmsg)
 
 	dev_info->chipset_bus_no = bus_no;
 	dev_info->chipset_dev_no = dev_no;
-	dev_info->inst = cmd->create_device.dev_inst_uuid;
-
-	/* not sure where the best place to set the 'parent' */
+	guid_copy(&dev_info->inst, &cmd->create_device.dev_inst_guid);
 	dev_info->device.parent = &bus_info->device;
 
 	visorchannel =
 	       visorchannel_create_with_lock(cmd->create_device.channel_addr,
-					     cmd->create_device.channel_bytes,
 					     GFP_KERNEL,
-					     cmd->create_device.data_type_uuid);
-
+					     &cmd->create_device.data_type_guid);
 	if (!visorchannel) {
 		dev_err(&chipset_dev->acpi_device->dev,
 			"failed to create visorchannel: %d/%d\n",
@@ -800,9 +797,8 @@ visorbus_device_create(struct controlvm_message *inmsg)
 		goto err_free_dev_info;
 	}
 	dev_info->visorchannel = visorchannel;
-	dev_info->channel_type_guid = cmd->create_device.data_type_uuid;
-	if (uuid_le_cmp(cmd->create_device.data_type_uuid,
-			visor_vhba_channel_uuid) == 0) {
+	guid_copy(&dev_info->channel_type_guid, &cmd->create_device.data_type_guid);
+	if (guid_equal(&cmd->create_device.data_type_guid, &visor_vhba_channel_guid)) {
 		err = save_crash_message(inmsg, CRASH_DEV);
 		if (err)
 			goto err_destroy_visorchannel;
@@ -819,8 +815,8 @@ visorbus_device_create(struct controlvm_message *inmsg)
 		       sizeof(struct controlvm_message_header));
 		dev_info->pending_msg_hdr = pmsg_hdr;
 	}
-	/* visorchipset_device_create will send response */
-	err = visorchipset_device_create(dev_info);
+	/* create_visor_device will send response */
+	err = create_visor_device(dev_info);
 	if (err)
 		goto err_destroy_visorchannel;
 
@@ -838,8 +834,7 @@ err_respond:
 	return err;
 }
 
-static int
-visorbus_device_changestate(struct controlvm_message *inmsg)
+static int visorbus_device_changestate(struct controlvm_message *inmsg)
 {
 	struct controlvm_message_packet *cmd = &inmsg->cmd;
 	struct controlvm_message_header *pmsg_hdr = NULL;
@@ -899,8 +894,7 @@ err_respond:
 	return err;
 }
 
-static int
-visorbus_device_destroy(struct controlvm_message *inmsg)
+static int visorbus_device_destroy(struct controlvm_message *inmsg)
 {
 	struct controlvm_message_packet *cmd = &inmsg->cmd;
 	struct controlvm_message_header *pmsg_hdr = NULL;
@@ -918,7 +912,6 @@ visorbus_device_destroy(struct controlvm_message *inmsg)
 		err = -EINVAL;
 		goto err_respond;
 	}
-
 	if (dev_info->pending_msg_hdr) {
 		/* only non-NULL if dev is still waiting on a response */
 		err = -EIO;
@@ -936,7 +929,8 @@ visorbus_device_destroy(struct controlvm_message *inmsg)
 		dev_info->pending_msg_hdr = pmsg_hdr;
 	}
 
-	visorchipset_device_destroy(dev_info);
+	kfree(dev_info->name);
+	remove_visor_device(dev_info);
 	return 0;
 
 err_respond:
@@ -954,8 +948,7 @@ err_respond:
  * disable the specified device. The udev script then writes to
  * /sys/devices/platform/visorchipset/parahotplug, which causes the
  * parahotplug store functions to get called, at which point the
- * appropriate CONTROLVM message is retrieved from the list and responded
- * to.
+ * appropriate CONTROLVM message is retrieved from the list and responded to.
  */
 
 #define PARAHOTPLUG_TIMEOUT_MS 2000
@@ -967,8 +960,7 @@ err_respond:
  *
  * Return: a unique integer value
  */
-static int
-parahotplug_next_id(void)
+static int parahotplug_next_id(void)
 {
 	static atomic_t id = ATOMIC_INIT(0);
 
@@ -982,8 +974,7 @@ parahotplug_next_id(void)
  *
  * Return: expected expiration time (in jiffies)
  */
-static unsigned long
-parahotplug_next_expiration(void)
+static unsigned long parahotplug_next_expiration(void)
 {
 	return jiffies + msecs_to_jiffies(PARAHOTPLUG_TIMEOUT_MS);
 }
@@ -996,8 +987,8 @@ parahotplug_next_expiration(void)
  *
  * Return: the request containing the provided message
  */
-static struct parahotplug_request *
-parahotplug_request_create(struct controlvm_message *msg)
+static struct parahotplug_request *parahotplug_request_create(
+						struct controlvm_message *msg)
 {
 	struct parahotplug_request *req;
 
@@ -1016,14 +1007,14 @@ parahotplug_request_create(struct controlvm_message *msg)
  * parahotplug_request_destroy() - free a parahotplug_request
  * @req: the request to deallocate
  */
-static void
-parahotplug_request_destroy(struct parahotplug_request *req)
+static void parahotplug_request_destroy(struct parahotplug_request *req)
 {
 	kfree(req);
 }
 
 static LIST_HEAD(parahotplug_request_list);
-static DEFINE_SPINLOCK(parahotplug_request_list_lock);	/* lock for above */
+/* lock for above */
+static DEFINE_SPINLOCK(parahotplug_request_list_lock);
 
 /*
  * parahotplug_request_complete() - mark request as complete
@@ -1036,8 +1027,7 @@ static DEFINE_SPINLOCK(parahotplug_request_list_lock);	/* lock for above */
  *
  * Return: 0 on success or -EINVAL on failure
  */
-static int
-parahotplug_request_complete(int id, u16 active)
+static int parahotplug_request_complete(int id, u16 active)
 {
 	struct list_head *pos;
 	struct list_head *tmp;
@@ -1146,7 +1136,7 @@ static struct attribute *visorchipset_parahotplug_attrs[] = {
 	NULL
 };
 
-static struct attribute_group visorchipset_parahotplug_group = {
+static const struct attribute_group visorchipset_parahotplug_group = {
 	.name = "parahotplug",
 	.attrs = visorchipset_parahotplug_attrs
 };
@@ -1164,8 +1154,7 @@ static const struct attribute_group *visorchipset_dev_groups[] = {
  * Cause uevent to run the user level script to do the disable/enable specified
  * in the parahotplug_request.
  */
-static int
-parahotplug_request_kickoff(struct parahotplug_request *req)
+static int parahotplug_request_kickoff(struct parahotplug_request *req)
 {
 	struct controlvm_message_packet *cmd = &req->msg.cmd;
 	char env_cmd[40], env_id[40], env_state[40], env_bus[40], env_dev[40],
@@ -1194,14 +1183,12 @@ parahotplug_request_kickoff(struct parahotplug_request *req)
  *                                 off a udev script
  * @inmsg: the message indicating whether to enable or disable
  */
-static int
-parahotplug_process_message(struct controlvm_message *inmsg)
+static int parahotplug_process_message(struct controlvm_message *inmsg)
 {
 	struct parahotplug_request *req;
 	int err;
 
 	req = parahotplug_request_create(inmsg);
-
 	if (!req)
 		return -ENOMEM;
 
@@ -1220,10 +1207,9 @@ parahotplug_process_message(struct controlvm_message *inmsg)
 	}
 
 	/*
-	 * For disable messages, add the request to the
-	 * request list before kicking off the udev script. It
-	 * won't get responded to until the script has
-	 * indicated it's done.
+	 * For disable messages, add the request to the request list before
+	 * kicking off the udev script. It won't get responded to until the
+	 * script has indicated it's done.
 	 */
 	spin_lock(&parahotplug_request_list_lock);
 	list_add_tail(&req->list, &parahotplug_request_list);
@@ -1247,8 +1233,7 @@ err_respond:
  *
  * Return: 0 on success, negative on failure
  */
-static int
-chipset_ready_uevent(struct controlvm_message_header *msg_hdr)
+static int chipset_ready_uevent(struct controlvm_message_header *msg_hdr)
 {
 	int res;
 
@@ -1268,8 +1253,7 @@ chipset_ready_uevent(struct controlvm_message_header *msg_hdr)
  *
  * Return: 0 on success, negative on failure
  */
-static int
-chipset_selftest_uevent(struct controlvm_message_header *msg_hdr)
+static int chipset_selftest_uevent(struct controlvm_message_header *msg_hdr)
 {
 	char env_selftest[20];
 	char *envp[] = { env_selftest, NULL };
@@ -1292,13 +1276,11 @@ chipset_selftest_uevent(struct controlvm_message_header *msg_hdr)
  *
  * Return: 0 on success, negative on failure
  */
-static int
-chipset_notready_uevent(struct controlvm_message_header *msg_hdr)
+static int chipset_notready_uevent(struct controlvm_message_header *msg_hdr)
 {
-	int res;
-
-	res = kobject_uevent(&chipset_dev->acpi_device->dev.kobj,
+	int res = kobject_uevent(&chipset_dev->acpi_device->dev.kobj,
 			     KOBJ_OFFLINE);
+
 	if (msg_hdr->flags.response_expected)
 		controlvm_respond(msg_hdr, res, NULL);
 
@@ -1321,13 +1303,12 @@ static int unisys_vmcall(unsigned long tuple, unsigned long param)
 
 	__asm__ __volatile__(".byte 0x00f, 0x001, 0x0c1" : "=a"(result) :
 		"a"(tuple), "b"(reg_ebx), "c"(reg_ecx));
-
 	if (result)
 		goto error;
 
 	return 0;
-
-error: /* Need to convert from VMCALL error codes to Linux */
+/* Need to convert from VMCALL error codes to Linux */
+error:
 	switch (result) {
 	case VMCALL_RESULT_INVALID_PARAM:
 		return -EINVAL;
@@ -1337,35 +1318,27 @@ error: /* Need to convert from VMCALL error codes to Linux */
 		return -EFAULT;
 	}
 }
-static unsigned int
-issue_vmcall_io_controlvm_addr(u64 *control_addr, u32 *control_bytes)
-{
-	chipset_dev->controlvm_addr.physaddr = virt_to_phys(
-					   &chipset_dev->controlvm_addr.params);
-	chipset_dev->controlvm_addr.err = unisys_vmcall(VMCALL_CONTROLVM_ADDR,
-					  chipset_dev->controlvm_addr.physaddr);
-	if (chipset_dev->controlvm_addr.err)
-		return chipset_dev->controlvm_addr.err;
-
-	*control_addr = chipset_dev->controlvm_addr.params.address;
-	*control_bytes = chipset_dev->controlvm_addr.params.channel_bytes;
-
-	return 0;
-}
 
-static u64 controlvm_get_channel_address(void)
+static int controlvm_channel_create(struct visorchipset_device *dev)
 {
-	u64 addr = 0;
-	u32 size = 0;
-
-	if (issue_vmcall_io_controlvm_addr(&addr, &size))
-		return 0;
+	struct visorchannel *chan;
+	u64 addr;
+	int err;
 
-	return addr;
+	err = unisys_vmcall(VMCALL_CONTROLVM_ADDR,
+			    virt_to_phys(&dev->controlvm_params));
+	if (err)
+		return err;
+	addr = dev->controlvm_params.address;
+	chan = visorchannel_create_with_lock(addr, GFP_KERNEL,
+					     &visor_controlvm_channel_guid);
+	if (!chan)
+		return -ENOMEM;
+	dev->controlvm_channel = chan;
+	return 0;
 }
 
-static void
-setup_crash_devices_work_queue(struct work_struct *work)
+static void setup_crash_devices_work_queue(struct work_struct *work)
 {
 	struct controlvm_message local_crash_bus_msg;
 	struct controlvm_message local_crash_dev_msg;
@@ -1444,87 +1417,44 @@ setup_crash_devices_work_queue(struct work_struct *work)
 	visorbus_device_create(&local_crash_dev_msg);
 }
 
-void
-visorbus_create_response(struct visor_device *bus_info, int response)
+void visorbus_response(struct visor_device *bus_info, int response,
+		       int controlvm_id)
 {
-	if (response >= 0)
-		bus_info->state.created = 1;
-
-	controlvm_responder(CONTROLVM_BUS_CREATE, bus_info->pending_msg_hdr,
-			    response);
+	controlvm_responder(controlvm_id, bus_info->pending_msg_hdr, response);
 
 	kfree(bus_info->pending_msg_hdr);
 	bus_info->pending_msg_hdr = NULL;
 }
 
-void
-visorbus_destroy_response(struct visor_device *bus_info, int response)
-{
-	controlvm_responder(CONTROLVM_BUS_DESTROY, bus_info->pending_msg_hdr,
-			    response);
-
-	kfree(bus_info->pending_msg_hdr);
-	bus_info->pending_msg_hdr = NULL;
-}
-
-void
-visorbus_device_create_response(struct visor_device *dev_info, int response)
-{
-	if (response >= 0)
-		dev_info->state.created = 1;
-
-	controlvm_responder(CONTROLVM_DEVICE_CREATE, dev_info->pending_msg_hdr,
-			    response);
-
-	kfree(dev_info->pending_msg_hdr);
-	dev_info->pending_msg_hdr = NULL;
-}
-
-void
-visorbus_device_destroy_response(struct visor_device *dev_info, int response)
-{
-	controlvm_responder(CONTROLVM_DEVICE_DESTROY, dev_info->pending_msg_hdr,
-			    response);
-
-	kfree(dev_info->pending_msg_hdr);
-	dev_info->pending_msg_hdr = NULL;
-}
-
-void
-visorbus_device_pause_response(struct visor_device *dev_info, int response)
+void visorbus_device_changestate_response(struct visor_device *dev_info,
+					  int response,
+					  struct visor_segment_state state)
 {
 	device_changestate_responder(CONTROLVM_DEVICE_CHANGESTATE,
-				     dev_info, response,
-				     segment_state_standby);
+				     dev_info, response, state);
 
 	kfree(dev_info->pending_msg_hdr);
 	dev_info->pending_msg_hdr = NULL;
 }
 
-void
-visorbus_device_resume_response(struct visor_device *dev_info, int response)
+static void parser_done(struct parser_context *ctx)
 {
-	device_changestate_responder(CONTROLVM_DEVICE_CHANGESTATE,
-				     dev_info, response,
-				     segment_state_running);
-
-	kfree(dev_info->pending_msg_hdr);
-	dev_info->pending_msg_hdr = NULL;
+	chipset_dev->controlvm_payload_bytes_buffered -= ctx->param_bytes;
+	kfree(ctx);
 }
 
-static struct parser_context *
-parser_init_byte_stream(u64 addr, u32 bytes, bool local, bool *retry)
+static struct parser_context *parser_init_stream(u64 addr, u32 bytes,
+						 bool *retry)
 {
-	int allocbytes = sizeof(struct parser_context) + bytes;
+	int allocbytes;
 	struct parser_context *ctx;
+	void *mapping;
 
 	*retry = false;
 
-	/*
-	 * alloc an 0 extra byte to ensure payload is
-	 * '\0'-terminated
-	 */
-	allocbytes++;
+	/* alloc an extra byte to ensure payload is \0 terminated */
+	allocbytes = bytes + 1 + (sizeof(struct parser_context) -
+		     sizeof(struct visor_controlvm_parameters_header));
 	if ((chipset_dev->controlvm_payload_bytes_buffered + bytes)
 	    > MAX_CONTROLVM_PAYLOAD_BYTES) {
 		*retry = true;
@@ -1538,32 +1468,18 @@ parser_init_byte_stream(u64 addr, u32 bytes, bool local, bool *retry)
 
 	ctx->allocbytes = allocbytes;
 	ctx->param_bytes = bytes;
-	ctx->curr = NULL;
-	ctx->bytes_remaining = 0;
-	ctx->byte_stream = false;
-	if (local) {
-		void *p;
-
-		if (addr > virt_to_phys(high_memory - 1))
-			goto err_finish_ctx;
-		p = __va((unsigned long)(addr));
-		memcpy(ctx->data, p, bytes);
-	} else {
-		void *mapping = memremap(addr, bytes, MEMREMAP_WB);
-
-		if (!mapping)
-			goto err_finish_ctx;
-		memcpy(ctx->data, mapping, bytes);
-		memunmap(mapping);
-	}
-
+	mapping = memremap(addr, bytes, MEMREMAP_WB);
+	if (!mapping)
+		goto err_finish_ctx;
+	memcpy(&ctx->data, mapping, bytes);
+	memunmap(mapping);
 	ctx->byte_stream = true;
 	chipset_dev->controlvm_payload_bytes_buffered += ctx->param_bytes;
 
 	return ctx;
 
 err_finish_ctx:
-	parser_done(ctx);
+	kfree(ctx);
 	return NULL;
 }
 
@@ -1580,19 +1496,16 @@ err_finish_ctx:
  *		  which to copy out controlvm payload data.
  *	< 0	- error: ControlVM message was processed but an error occurred.
  */
-static int
-handle_command(struct controlvm_message inmsg, u64 channel_addr)
+static int handle_command(struct controlvm_message inmsg, u64 channel_addr)
 {
 	struct controlvm_message_packet *cmd = &inmsg.cmd;
 	u64 parm_addr;
 	u32 parm_bytes;
 	struct parser_context *parser_ctx = NULL;
-	bool local_addr;
 	struct controlvm_message ackmsg;
 	int err = 0;
 
 	/* create parsing context if necessary */
-	local_addr = (inmsg.hdr.flags.test_message == 1);
 	parm_addr = channel_addr + inmsg.hdr.payload_vm_offset;
 	parm_bytes = inmsg.hdr.payload_bytes;
 
@@ -1601,25 +1514,19 @@ handle_command(struct controlvm_message inmsg, u64 channel_addr)
 	 * within our OS-controlled memory. We need to know that, because it
 	 * makes a difference in how we compute the virtual address.
 	 */
-	if (parm_addr && parm_bytes) {
+	if (parm_bytes) {
 		bool retry = false;
 
-		parser_ctx =
-		    parser_init_byte_stream(parm_addr, parm_bytes,
-					    local_addr, &retry);
+		parser_ctx = parser_init_stream(parm_addr, parm_bytes, &retry);
 		if (!parser_ctx && retry)
 			return -EAGAIN;
 	}
+	controlvm_init_response(&ackmsg, &inmsg.hdr, CONTROLVM_RESP_SUCCESS);
+	err = visorchannel_signalinsert(chipset_dev->controlvm_channel,
+					CONTROLVM_QUEUE_ACK, &ackmsg);
+	if (err)
+		return err;
 
-	if (!local_addr) {
-		controlvm_init_response(&ackmsg, &inmsg.hdr,
-					CONTROLVM_RESP_SUCCESS);
-		err = visorchannel_signalinsert(chipset_dev->controlvm_channel,
-						CONTROLVM_QUEUE_ACK,
-						&ackmsg);
-		if (err)
-			return err;
-	}
 	switch (inmsg.hdr.id) {
 	case CONTROLVM_CHIPSET_INIT:
 		err = chipset_init(&inmsg);
@@ -1641,8 +1548,8 @@ handle_command(struct controlvm_message inmsg, u64 channel_addr)
 			err = parahotplug_process_message(&inmsg);
 		} else {
 			/*
-			 * save the hdr and cmd structures for later use
-			 * when sending back the response to Command
+			 * save the hdr and cmd structures for later use when
+			 * sending back the response to Command
 			 */
 			err = visorbus_device_changestate(&inmsg);
 			break;
@@ -1689,12 +1596,9 @@ handle_command(struct controlvm_message inmsg, u64 channel_addr)
  *
  * Return: 0 if valid message was retrieved or -error
  */
-static int
-read_controlvm_event(struct controlvm_message *msg)
+static int read_controlvm_event(struct controlvm_message *msg)
 {
-	int err;
-
-	err = visorchannel_signalremove(chipset_dev->controlvm_channel,
+	int err = visorchannel_signalremove(chipset_dev->controlvm_channel,
 					CONTROLVM_QUEUE_EVENT, msg);
 	if (err)
 		return err;
@@ -1710,8 +1614,7 @@ read_controlvm_event(struct controlvm_message *msg)
  * parahotplug_process_list() - remove any request from the list that's been on
  *                              there too long and respond with an error
  */
-static void
-parahotplug_process_list(void)
+static void parahotplug_process_list(void)
 {
 	struct list_head *pos;
 	struct list_head *tmp;
@@ -1737,8 +1640,7 @@ parahotplug_process_list(void)
 	spin_unlock(&parahotplug_request_list_lock);
 }
 
-static void
-controlvm_periodic_work(struct work_struct *work)
+static void controlvm_periodic_work(struct work_struct *work)
 {
 	struct controlvm_message inmsg;
 	int count = 0;
@@ -1756,9 +1658,8 @@ controlvm_periodic_work(struct work_struct *work)
 
 	if (chipset_dev->controlvm_pending_msg_valid) {
 		/*
-		 * we throttled processing of a prior
-		 * msg, so try to process it again
-		 * rather than reading a new one
+		 * we throttled processing of a prior msg, so try to process
+		 * it again rather than reading a new one
 		 */
 		inmsg = chipset_dev->controlvm_pending_msg;
 		chipset_dev->controlvm_pending_msg_valid = false;
@@ -1793,9 +1694,8 @@ schedule_out:
 	if (time_after(jiffies, chipset_dev->most_recent_message_jiffies +
 				(HZ * MIN_IDLE_SECONDS))) {
 		/*
-		 * it's been longer than MIN_IDLE_SECONDS since we
-		 * processed our last controlvm message; slow down the
-		 * polling
+		 * it's been longer than MIN_IDLE_SECONDS since we processed
+		 * our last controlvm message; slow down the polling
 		 */
 		if (chipset_dev->poll_jiffies !=
 					      POLLJIFFIES_CONTROLVMCHANNEL_SLOW)
@@ -1812,41 +1712,36 @@ schedule_out:
 			      chipset_dev->poll_jiffies);
 }
 
-static int
-visorchipset_init(struct acpi_device *acpi_device)
+static int visorchipset_init(struct acpi_device *acpi_device)
 {
 	int err = -ENODEV;
-	u64 addr;
-	uuid_le uuid = VISOR_CONTROLVM_CHANNEL_UUID;
 	struct visorchannel *controlvm_channel;
 
 	chipset_dev = kzalloc(sizeof(*chipset_dev), GFP_KERNEL);
 	if (!chipset_dev)
 		goto error;
 
-	addr = controlvm_get_channel_address();
-	if (!addr)
-		goto error;
+	err = controlvm_channel_create(chipset_dev);
+	if (err)
+		goto error_free_chipset_dev;
 
 	acpi_device->driver_data = chipset_dev;
-
 	chipset_dev->acpi_device = acpi_device;
 	chipset_dev->poll_jiffies = POLLJIFFIES_CONTROLVMCHANNEL_FAST;
-	controlvm_channel = visorchannel_create_with_lock(addr,
-							  0, GFP_KERNEL, uuid);
-
-	if (!controlvm_channel)
-		goto error_free_chipset_dev;
-
-	chipset_dev->controlvm_channel = controlvm_channel;
 
 	err = sysfs_create_groups(&chipset_dev->acpi_device->dev.kobj,
 				  visorchipset_dev_groups);
 	if (err < 0)
 		goto error_destroy_channel;
 
-	if (!VISOR_CONTROLVM_CHANNEL_OK_CLIENT(
-				visorchannel_get_header(controlvm_channel)))
+	controlvm_channel = chipset_dev->controlvm_channel;
+	if (!visor_check_channel(visorchannel_get_header(controlvm_channel),
+				 &chipset_dev->acpi_device->dev,
+				 &visor_controlvm_channel_guid,
+				 "controlvm",
+				 sizeof(struct visor_controlvm_channel),
+				 VISOR_CONTROLVM_CHANNEL_VERSIONID,
+				 VISOR_CHANNEL_SIGNATURE))
 		goto error_delete_groups;
 
 	/* if booting in a crash kernel */
@@ -1886,8 +1781,7 @@ error:
 	return err;
 }
 
-static int
-visorchipset_exit(struct acpi_device *acpi_device)
+static int visorchipset_exit(struct acpi_device *acpi_device)
 {
 	visorbus_exit();
 	cancel_delayed_work_sync(&chipset_dev->periodic_controlvm_work);
@@ -1928,9 +1822,8 @@ static __init int visorutil_spar_detect(void)
 		return  (ebx == UNISYS_VISOR_ID_EBX) &&
 			(ecx == UNISYS_VISOR_ID_ECX) &&
 			(edx == UNISYS_VISOR_ID_EDX);
-	} else {
-		return 0;
 	}
+	return 0;
 }
 
 static int init_unisys(void)
diff --git a/drivers/staging/unisys/visorbus/vmcallinterface.h b/drivers/staging/unisys/visorbus/vmcallinterface.h
deleted file mode 100644
index cc70e1b16bda..000000000000
--- a/drivers/staging/unisys/visorbus/vmcallinterface.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/* Copyright (C) 2010 - 2015 UNISYS CORPORATION
- * All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- * NON INFRINGEMENT.  See the GNU General Public License for more
- * details.
- */
-
-#ifndef __VMCALLINTERFACE_H__
-#define __VMCALLINTERFACE_H__
-
-enum vmcall_monitor_interface_method_tuple { /* VMCALL identification tuples  */
-	    /* Note: when a new VMCALL is added:
-	     * - the 1st 2 hex digits correspond to one of the
-	     *   VMCALL_MONITOR_INTERFACE types and
-	     * - the next 2 hex digits are the nth relative instance of within a
-	     *   type
-	     * E.G. for VMCALL_VIRTPART_RECYCLE_PART,
-	     * - the 0x02 identifies it as a VMCALL_VIRTPART type and
-	     * - the 0x01 identifies it as the 1st instance of a VMCALL_VIRTPART
-	     *   type of VMCALL
-	     */
-	/* used by all Guests, not just IO */
-	VMCALL_CONTROLVM_ADDR = 0x0501,
-};
-
-enum vmcall_result {
-	VMCALL_RESULT_SUCCESS = 0,
-	VMCALL_RESULT_INVALID_PARAM = 1,
-	VMCALL_RESULT_DATA_UNAVAILABLE = 2,
-	VMCALL_RESULT_FAILURE_UNAVAILABLE = 3,
-	VMCALL_RESULT_DEVICE_ERROR = 4,
-	VMCALL_RESULT_DEVICE_NOT_READY = 5
-};
-
-/* Structures for IO VMCALLs */
-/* Parameters to VMCALL_CONTROLVM_ADDR interface */
-struct vmcall_io_controlvm_addr_params {
-	/* The Guest-relative physical address of the ControlVm channel. */
-	/* This VMCall fills this in with the appropriate address. */
-	u64 address;	/* contents provided by this VMCALL (OUT) */
-	/* the size of the ControlVm channel in bytes This VMCall fills this */
-	/* in with the appropriate address. */
-	u32 channel_bytes;	/* contents provided by this VMCALL (OUT) */
-	u8 unused[4];		/* Unused Bytes in the 64-Bit Aligned Struct */
-} __packed;
-
-#endif /* __VMCALLINTERFACE_H__ */
diff --git a/drivers/staging/unisys/visorhba/visorhba_main.c b/drivers/staging/unisys/visorhba/visorhba_main.c
index a6e7a6bbc428..419dba89af06 100644
--- a/drivers/staging/unisys/visorhba/visorhba_main.c
+++ b/drivers/staging/unisys/visorhba/visorhba_main.c
@@ -1,4 +1,5 @@
-/* Copyright (c) 2012 - 2015 UNISYS CORPORATION
+/*
+ * Copyright (c) 2012 - 2015 UNISYS CORPORATION
  * All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify
@@ -14,7 +15,6 @@
  */
 
 #include <linux/debugfs.h>
-#include <linux/skbuff.h>
 #include <linux/kthread.h>
 #include <linux/idr.h>
 #include <linux/seq_file.h>
@@ -39,16 +39,16 @@ static struct visor_channeltype_descriptor visorhba_channel_types[] = {
 	/* Note that the only channel type we expect to be reported by the
 	 * bus driver is the VISOR_VHBA channel.
 	 */
-	{ VISOR_VHBA_CHANNEL_UUID, "sparvhba" },
-	{ NULL_UUID_LE, NULL }
+	{ VISOR_VHBA_CHANNEL_GUID, "sparvhba" },
+	{}
 };
 
 MODULE_DEVICE_TABLE(visorbus, visorhba_channel_types);
-MODULE_ALIAS("visorbus:" VISOR_VHBA_CHANNEL_UUID_STR);
+MODULE_ALIAS("visorbus:" VISOR_VHBA_CHANNEL_GUID_STR);
 
 struct visordisk_info {
+	struct scsi_device *sdev;
 	u32 valid;
-	u32 channel, id, lun;	/* Disk Path */
 	atomic_t ios_threshold;
 	atomic_t error_count;
 	struct visordisk_info *next;
@@ -56,8 +56,10 @@ struct visordisk_info {
 
 struct scsipending {
 	struct uiscmdrsp cmdrsp;
-	void *sent;		/* The Data being tracked */
-	char cmdtype;		/* Type of pointer that is being stored */
+	/* The Data being tracked */
+	void *sent;
+	/* Type of pointer that is being stored */
+	char cmdtype;
 };
 
 /* Each scsi_host has a host_data area that contains this struct. */
@@ -71,7 +73,8 @@ struct visorhba_devdata {
 	struct scsipending pending[MAX_PENDING_REQUESTS];
 	/* Start search for next pending free slot here */
 	unsigned int nextinsert;
-	spinlock_t privlock; /* lock to protect data in devdata */
+	/* lock to protect data in devdata */
+	spinlock_t privlock;
 	bool serverdown;
 	bool serverchangingstate;
 	unsigned long long acquire_failed_cnt;
@@ -101,25 +104,19 @@ struct visorhba_devices_open {
 	struct visorhba_devdata *devdata;
 };
 
-#define for_each_vdisk_match(iter, list, match) \
-	for (iter = &list->head; iter->next; iter = iter->next) \
-		if ((iter->channel == match->channel) && \
-		    (iter->id == match->id) && \
-		    (iter->lun == match->lun))
-
 /*
- *	visor_thread_start - starts a thread for the device
- *	@threadfn: Function the thread starts
- *	@thrcontext: Context to pass to the thread, i.e. devdata
- *	@name: string describing name of thread
+ * visor_thread_start - Starts a thread for the device
+ * @threadfn:   Function the thread starts
+ * @thrcontext: Context to pass to the thread, i.e. devdata
+ * @name:	String describing name of thread
  *
- *	Starts a thread for the device.
+ * Starts a thread for the device.
  *
- *	Return the task_struct * denoting the thread on success,
- *             or NULL on failure
+ * Return: The task_struct * denoting the thread on success,
+ *	   or NULL on failure
  */
-static struct task_struct *visor_thread_start
-(int (*threadfn)(void *), void *thrcontext, char *name)
+static struct task_struct *visor_thread_start(int (*threadfn)(void *),
+					      void *thrcontext, char *name)
 {
 	struct task_struct *task;
 
@@ -132,27 +129,27 @@ static struct task_struct *visor_thread_start
 }
 
 /*
- *      visor_thread_stop - stops the thread if it is running
+ * visor_thread_stop - Stops the thread if it is running
+ * @task: Description of process to stop
  */
 static void visor_thread_stop(struct task_struct *task)
 {
-	if (!task)
-		return;  /* no thread running */
 	kthread_stop(task);
 }
 
 /*
- *	add_scsipending_entry - save off io command that is pending in
- *				Service Partition
- *	@devdata: Pointer to devdata
- *	@cmdtype: Specifies the type of command pending
- *	@new:	The command to be saved
+ * add_scsipending_entry - Save off io command that is pending in
+ *			   Service Partition
+ * @devdata: Pointer to devdata
+ * @cmdtype: Specifies the type of command pending
+ * @new:     The command to be saved
  *
- *	Saves off the io command that is being handled by the Service
- *	Partition so that it can be handled when it completes. If new is
- *	NULL it is assumed the entry refers only to the cmdrsp.
- *	Returns insert_location where entry was added,
- *	-EBUSY if it can't
+ * Saves off the io command that is being handled by the Service
+ * Partition so that it can be handled when it completes. If new is
+ * NULL it is assumed the entry refers only to the cmdrsp.
+ *
+ * Return: Insert_location where entry was added on success,
+ *	   -EBUSY if it can't
  */
 static int add_scsipending_entry(struct visorhba_devdata *devdata,
 				 char cmdtype, void *new)
@@ -176,7 +173,8 @@ static int add_scsipending_entry(struct visorhba_devdata *devdata,
 	entry->cmdtype = cmdtype;
 	if (new)
 		entry->sent = new;
-	else /* wants to send cmdrsp */
+	/* wants to send cmdrsp */
+	else
 		entry->sent = &entry->cmdrsp;
 	devdata->nextinsert = (insert_location + 1) % MAX_PENDING_REQUESTS;
 	spin_unlock_irqrestore(&devdata->privlock, flags);
@@ -185,15 +183,15 @@ static int add_scsipending_entry(struct visorhba_devdata *devdata,
 }
 
 /*
- *	del_scsipending_ent - removes an entry from the pending array
- *	@devdata: Device holding the pending array
- *	@del: Entry to remove
+ * del_scsipending_ent - Removes an entry from the pending array
+ * @devdata: Device holding the pending array
+ * @del:     Entry to remove
  *
- *	Removes the entry pointed at by del and returns it.
- *	Returns the scsipending entry pointed at
+ * Removes the entry pointed at by del and returns it.
+ *
+ * Return: The scsipending entry pointed to on success, NULL on failure
  */
-static void *del_scsipending_ent(struct visorhba_devdata *devdata,
-				 int del)
+static void *del_scsipending_ent(struct visorhba_devdata *devdata, int del)
 {
 	unsigned long flags;
 	void *sent;
@@ -203,7 +201,6 @@ static void *del_scsipending_ent(struct visorhba_devdata *devdata,
 
 	spin_lock_irqsave(&devdata->privlock, flags);
 	sent = devdata->pending[del].sent;
-
 	devdata->pending[del].cmdtype = 0;
 	devdata->pending[del].sent = NULL;
 	spin_unlock_irqrestore(&devdata->privlock, flags);
@@ -212,13 +209,14 @@ static void *del_scsipending_ent(struct visorhba_devdata *devdata,
 }
 
 /*
- *	get_scsipending_cmdrsp - return the cmdrsp stored in a pending entry
- *	@ddata: Device holding the pending array
- *	@ent: Entry that stores the cmdrsp
+ * get_scsipending_cmdrsp - Return the cmdrsp stored in a pending entry
+ * @ddata: Device holding the pending array
+ * @ent:   Entry that stores the cmdrsp
  *
- *	Each scsipending entry has a cmdrsp in it. The cmdrsp is only valid
- *	if the "sent" field is not NULL
- *	Returns a pointer to the cmdrsp.
+ * Each scsipending entry has a cmdrsp in it. The cmdrsp is only valid
+ * if the "sent" field is not NULL.
+ *
+ * Return: A pointer to the cmdrsp, NULL on failure
  */
 static struct uiscmdrsp *get_scsipending_cmdrsp(struct visorhba_devdata *ddata,
 						int ent)
@@ -230,13 +228,15 @@ static struct uiscmdrsp *get_scsipending_cmdrsp(struct visorhba_devdata *ddata,
 }
 
 /*
- *      simple_idr_get - associate a provided pointer with an int value
- *                       1 <= value <= INT_MAX, and return this int value;
- *                       the pointer value can be obtained later by passing
- *                       this int value to idr_find()
- *      @idrtable: the data object maintaining the pointer<-->int mappings
- *      @p: the pointer value to be remembered
- *      @lock: a spinlock used when exclusive access to idrtable is needed
+ * simple_idr_get - Associate a provided pointer with an int value
+ *		    1 <= value <= INT_MAX, and return this int value;
+ *		    the pointer value can be obtained later by passing
+ *		    this int value to idr_find()
+ * @idrtable: The data object maintaining the pointer<-->int mappings
+ * @p:	      The pointer value to be remembered
+ * @lock:     A spinlock used when exclusive access to idrtable is needed
+ *
+ * Return: The id number mapped to pointer 'p', 0 on failure
  */
 static unsigned int simple_idr_get(struct idr *idrtable, void *p,
 				   spinlock_t *lock)
@@ -249,16 +249,23 @@ static unsigned int simple_idr_get(struct idr *idrtable, void *p,
 	id = idr_alloc(idrtable, p, 1, INT_MAX, GFP_NOWAIT);
 	spin_unlock_irqrestore(lock, flags);
 	idr_preload_end();
+	/* failure */
 	if (id < 0)
-		return 0;  /* failure */
-	return (unsigned int)(id);  /* idr_alloc() guarantees > 0 */
+		return 0;
+	/* idr_alloc() guarantees > 0 */
+	return (unsigned int)(id);
 }
 
 /*
- *      setup_scsitaskmgmt_handles - stash the necessary handles so that the
- *                                   completion processing logic for a taskmgmt
- *                                   cmd will be able to find who to wake up
- *                                   and where to stash the result
+ * setup_scsitaskmgmt_handles - Stash the necessary handles so that the
+ *				completion processing logic for a taskmgmt
+ *				cmd will be able to find who to wake up
+ *				and where to stash the result
+ * @idrtable: The data object maintaining the pointer<-->int mappings
+ * @lock:     A spinlock used when exclusive access to idrtable is needed
+ * @cmdrsp:   Response from the IOVM
+ * @event:    The event handle to associate with an id
+ * @result:   The location to place the result of the event handle into
  */
 static void setup_scsitaskmgmt_handles(struct idr *idrtable, spinlock_t *lock,
 				       struct uiscmdrsp *cmdrsp,
@@ -273,8 +280,10 @@ static void setup_scsitaskmgmt_handles(struct idr *idrtable, spinlock_t *lock,
 }
 
 /*
- *      cleanup_scsitaskmgmt_handles - forget handles created by
- *                                     setup_scsitaskmgmt_handles()
+ * cleanup_scsitaskmgmt_handles - Forget handles created by
+ *				  setup_scsitaskmgmt_handles()
+ * @idrtable: The data object maintaining the pointer<-->int mappings
+ * @cmdrsp:   Response from the IOVM
  */
 static void cleanup_scsitaskmgmt_handles(struct idr *idrtable,
 					 struct uiscmdrsp *cmdrsp)
@@ -286,20 +295,20 @@ static void cleanup_scsitaskmgmt_handles(struct idr *idrtable,
 }
 
 /*
- *	forward_taskmgmt_command - send taskmegmt command to the Service
- *				   Partition
- *	@tasktype: Type of taskmgmt command
- *	@scsidev: Scsidev that issued command
+ * forward_taskmgmt_command - Send taskmegmt command to the Service
+ *			      Partition
+ * @tasktype: Type of taskmgmt command
+ * @scsidev:  Scsidev that issued command
  *
- *	Create a cmdrsp packet and send it to the Serivce Partition
- *	that will service this request.
- *	Returns whether the command was queued successfully or not.
+ * Create a cmdrsp packet and send it to the Serivce Partition
+ * that will service this request.
+ *
+ * Return: Int representing whether command was queued successfully or not
  */
 static int forward_taskmgmt_command(enum task_mgmt_types tasktype,
-				    struct scsi_cmnd *scsicmd)
+				    struct scsi_device *scsidev)
 {
 	struct uiscmdrsp *cmdrsp;
-	struct scsi_device *scsidev = scsicmd->device;
 	struct visorhba_devdata *devdata =
 		(struct visorhba_devdata *)scsidev->host->hostdata;
 	int notifyresult = 0xffff;
@@ -347,12 +356,6 @@ static int forward_taskmgmt_command(enum task_mgmt_types tasktype,
 	dev_dbg(&scsidev->sdev_gendev,
 		"visorhba: taskmgmt type=%d success; result=0x%x\n",
 		 tasktype, notifyresult);
-	if (tasktype == TASK_MGMT_ABORT_TASK)
-		scsicmd->result = DID_ABORT << 16;
-	else
-		scsicmd->result = DID_RESET << 16;
-
-	scsicmd->scsi_done(scsicmd);
 	cleanup_scsitaskmgmt_handles(&devdata->idr, cmdrsp);
 	return SUCCESS;
 
@@ -365,97 +368,105 @@ err_del_scsipending_ent:
 }
 
 /*
- *	visorhba_abort_handler - Send TASK_MGMT_ABORT_TASK
- *	@scsicmd: The scsicmd that needs aborted
- *
- *	Returns SUCCESS if inserted, failure otherwise
+ * visorhba_abort_handler - Send TASK_MGMT_ABORT_TASK
+ * @scsicmd: The scsicmd that needs aborted
  *
+ * Return: SUCCESS if inserted, FAILED otherwise
  */
 static int visorhba_abort_handler(struct scsi_cmnd *scsicmd)
 {
 	/* issue TASK_MGMT_ABORT_TASK */
 	struct scsi_device *scsidev;
 	struct visordisk_info *vdisk;
-	struct visorhba_devdata *devdata;
+	int rtn;
 
 	scsidev = scsicmd->device;
-	devdata = (struct visorhba_devdata *)scsidev->host->hostdata;
-	for_each_vdisk_match(vdisk, devdata, scsidev) {
-		if (atomic_read(&vdisk->error_count) < VISORHBA_ERROR_COUNT)
-			atomic_inc(&vdisk->error_count);
-		else
-			atomic_set(&vdisk->ios_threshold, IOS_ERROR_THRESHOLD);
+	vdisk = scsidev->hostdata;
+	if (atomic_read(&vdisk->error_count) < VISORHBA_ERROR_COUNT)
+		atomic_inc(&vdisk->error_count);
+	else
+		atomic_set(&vdisk->ios_threshold, IOS_ERROR_THRESHOLD);
+	rtn = forward_taskmgmt_command(TASK_MGMT_ABORT_TASK, scsidev);
+	if (rtn == SUCCESS) {
+		scsicmd->result = DID_ABORT << 16;
+		scsicmd->scsi_done(scsicmd);
 	}
-	return forward_taskmgmt_command(TASK_MGMT_ABORT_TASK, scsicmd);
+	return rtn;
 }
 
 /*
- *	visorhba_device_reset_handler - Send TASK_MGMT_LUN_RESET
- *	@scsicmd: The scsicmd that needs aborted
+ * visorhba_device_reset_handler - Send TASK_MGMT_LUN_RESET
+ * @scsicmd: The scsicmd that needs aborted
  *
- *	Returns SUCCESS if inserted, failure otherwise
+ * Return: SUCCESS if inserted, FAILED otherwise
  */
 static int visorhba_device_reset_handler(struct scsi_cmnd *scsicmd)
 {
 	/* issue TASK_MGMT_LUN_RESET */
 	struct scsi_device *scsidev;
 	struct visordisk_info *vdisk;
-	struct visorhba_devdata *devdata;
+	int rtn;
 
 	scsidev = scsicmd->device;
-	devdata = (struct visorhba_devdata *)scsidev->host->hostdata;
-	for_each_vdisk_match(vdisk, devdata, scsidev) {
-		if (atomic_read(&vdisk->error_count) < VISORHBA_ERROR_COUNT)
-			atomic_inc(&vdisk->error_count);
-		else
-			atomic_set(&vdisk->ios_threshold, IOS_ERROR_THRESHOLD);
+	vdisk = scsidev->hostdata;
+	if (atomic_read(&vdisk->error_count) < VISORHBA_ERROR_COUNT)
+		atomic_inc(&vdisk->error_count);
+	else
+		atomic_set(&vdisk->ios_threshold, IOS_ERROR_THRESHOLD);
+	rtn = forward_taskmgmt_command(TASK_MGMT_LUN_RESET, scsidev);
+	if (rtn == SUCCESS) {
+		scsicmd->result = DID_RESET << 16;
+		scsicmd->scsi_done(scsicmd);
 	}
-	return forward_taskmgmt_command(TASK_MGMT_LUN_RESET, scsicmd);
+	return rtn;
 }
 
 /*
- *	visorhba_bus_reset_handler - Send TASK_MGMT_TARGET_RESET for each
- *				     target on the bus
- *	@scsicmd: The scsicmd that needs aborted
+ * visorhba_bus_reset_handler - Send TASK_MGMT_TARGET_RESET for each
+ *				target on the bus
+ * @scsicmd: The scsicmd that needs aborted
  *
- *	Returns SUCCESS
+ * Return: SUCCESS if inserted, FAILED otherwise
  */
 static int visorhba_bus_reset_handler(struct scsi_cmnd *scsicmd)
 {
 	struct scsi_device *scsidev;
 	struct visordisk_info *vdisk;
-	struct visorhba_devdata *devdata;
+	int rtn;
 
 	scsidev = scsicmd->device;
-	devdata = (struct visorhba_devdata *)scsidev->host->hostdata;
-	for_each_vdisk_match(vdisk, devdata, scsidev) {
+	shost_for_each_device(scsidev, scsidev->host) {
+		vdisk = scsidev->hostdata;
 		if (atomic_read(&vdisk->error_count) < VISORHBA_ERROR_COUNT)
 			atomic_inc(&vdisk->error_count);
 		else
 			atomic_set(&vdisk->ios_threshold, IOS_ERROR_THRESHOLD);
 	}
-	return forward_taskmgmt_command(TASK_MGMT_BUS_RESET, scsicmd);
+	rtn = forward_taskmgmt_command(TASK_MGMT_BUS_RESET, scsidev);
+	if (rtn == SUCCESS) {
+		scsicmd->result = DID_RESET << 16;
+		scsicmd->scsi_done(scsicmd);
+	}
+	return rtn;
 }
 
 /*
- *	visorhba_host_reset_handler - Not supported
- *	@scsicmd: The scsicmd that needs aborted
+ * visorhba_host_reset_handler - Not supported
+ * @scsicmd: The scsicmd that needs to be aborted
  *
- *	Not supported, return SUCCESS
- *	Returns SUCCESS
+ * Return: Not supported, return SUCCESS
  */
-static int
-visorhba_host_reset_handler(struct scsi_cmnd *scsicmd)
+static int visorhba_host_reset_handler(struct scsi_cmnd *scsicmd)
 {
 	/* issue TASK_MGMT_TARGET_RESET for each target on each bus for host */
 	return SUCCESS;
 }
 
 /*
- *	visorhba_get_info
- *	@shp: Scsi host that is requesting information
+ * visorhba_get_info - Get information about SCSI device
+ * @shp: Scsi host that is requesting information
  *
- *	Returns string with info
+ * Return: String with visorhba information
  */
 static const char *visorhba_get_info(struct Scsi_Host *shp)
 {
@@ -464,19 +475,42 @@ static const char *visorhba_get_info(struct Scsi_Host *shp)
 }
 
 /*
- *	visorhba_queue_command_lck -- queues command to the Service Partition
- *	@scsicmd: Command to be queued
- *	@vsiorhba_cmnd_done: Done command to call when scsicmd is returned
+ * dma_data_dir_linux_to_spar - convert dma_data_direction value to
+ *				Unisys-specific equivalent
+ * @d: dma direction value to convert
  *
- *	Queues to scsicmd to the ServicePartition after converting it to a
- *	uiscmdrsp structure.
+ * Returns the Unisys-specific dma direction value corresponding to @d
+ */
+static u32 dma_data_dir_linux_to_spar(enum dma_data_direction d)
+{
+	switch (d) {
+	case DMA_BIDIRECTIONAL:
+		return UIS_DMA_BIDIRECTIONAL;
+	case DMA_TO_DEVICE:
+		return UIS_DMA_TO_DEVICE;
+	case DMA_FROM_DEVICE:
+		return UIS_DMA_FROM_DEVICE;
+	case DMA_NONE:
+		return UIS_DMA_NONE;
+	default:
+		return UIS_DMA_NONE;
+	}
+}
+
+/*
+ * visorhba_queue_command_lck - Queues command to the Service Partition
+ * @scsicmd:		Command to be queued
+ * @vsiorhba_cmnd_done: Done command to call when scsicmd is returned
  *
- *	Returns success if queued to the Service Partition, otherwise
- *	failure.
+ * Queues to scsicmd to the ServicePartition after converting it to a
+ * uiscmdrsp structure.
+ *
+ * Return: 0 if successfully queued to the Service Partition, otherwise
+ *	   error code
  */
-static int
-visorhba_queue_command_lck(struct scsi_cmnd *scsicmd,
-			   void (*visorhba_cmnd_done)(struct scsi_cmnd *))
+static int visorhba_queue_command_lck(struct scsi_cmnd *scsicmd,
+				      void (*visorhba_cmnd_done)
+					   (struct scsi_cmnd *))
 {
 	struct uiscmdrsp *cmdrsp;
 	struct scsi_device *scsidev = scsicmd->device;
@@ -494,12 +528,10 @@ visorhba_queue_command_lck(struct scsi_cmnd *scsicmd,
 
 	insert_location = add_scsipending_entry(devdata, CMD_SCSI_TYPE,
 						(void *)scsicmd);
-
 	if (insert_location < 0)
 		return SCSI_MLQUEUE_DEVICE_BUSY;
 
 	cmdrsp = get_scsipending_cmdrsp(devdata, insert_location);
-
 	cmdrsp->cmdtype = CMD_SCSI_TYPE;
 	/* save the pending insertion location. Deletion from pending
 	 * will return the scsicmd pointer for completion
@@ -513,9 +545,9 @@ visorhba_queue_command_lck(struct scsi_cmnd *scsicmd,
 	cmdrsp->scsi.vdest.id = scsidev->id;
 	cmdrsp->scsi.vdest.lun = scsidev->lun;
 	/* save datadir */
-	cmdrsp->scsi.data_dir = scsicmd->sc_data_direction;
+	cmdrsp->scsi.data_dir =
+		dma_data_dir_linux_to_spar(scsicmd->sc_data_direction);
 	memcpy(cmdrsp->scsi.cmnd, cdb, MAX_CMND_SIZE);
-
 	cmdrsp->scsi.bufflen = scsi_bufflen(scsicmd);
 
 	/* keep track of the max buffer length so far. */
@@ -555,13 +587,13 @@ static DEF_SCSI_QCMD(visorhba_queue_command)
 #endif
 
 /*
- *	visorhba_slave_alloc - called when new disk is discovered
- *	@scsidev: New disk
+ * visorhba_slave_alloc - Called when new disk is discovered
+ * @scsidev: New disk
  *
- *	Create a new visordisk_info structure and add it to our
- *	list of vdisks.
+ * Create a new visordisk_info structure and add it to our
+ * list of vdisks.
  *
- *	Returns success when created, otherwise error.
+ * Return: 0 on success, -ENOMEM on failure.
  */
 static int visorhba_slave_alloc(struct scsi_device *scsidev)
 {
@@ -569,51 +601,41 @@ static int visorhba_slave_alloc(struct scsi_device *scsidev)
 	 * LLD can alloc any struct & do init if needed.
 	 */
 	struct visordisk_info *vdisk;
-	struct visordisk_info *tmpvdisk;
 	struct visorhba_devdata *devdata;
 	struct Scsi_Host *scsihost = (struct Scsi_Host *)scsidev->host;
 
+	/* already allocated return success */
+	if (scsidev->hostdata)
+		return 0;
+
+	/* even though we errored, treat as success */
 	devdata = (struct visorhba_devdata *)scsihost->hostdata;
 	if (!devdata)
-		return 0; /* even though we errored, treat as success */
-
-	for_each_vdisk_match(vdisk, devdata, scsidev)
-		return 0; /* already allocated return success */
+		return 0;
 
-	tmpvdisk = kzalloc(sizeof(*tmpvdisk), GFP_ATOMIC);
-	if (!tmpvdisk)
+	vdisk = kzalloc(sizeof(*vdisk), GFP_ATOMIC);
+	if (!vdisk)
 		return -ENOMEM;
 
-	tmpvdisk->channel = scsidev->channel;
-	tmpvdisk->id = scsidev->id;
-	tmpvdisk->lun = scsidev->lun;
-	vdisk->next = tmpvdisk;
+	vdisk->sdev = scsidev;
+	scsidev->hostdata = vdisk;
 	return 0;
 }
 
 /*
- *	visorhba_slave_destroy - disk is going away
- *	@scsidev: scsi device going away
- *
- *	Disk is going away, clean up resources.
- *	Returns void.
+ * visorhba_slave_destroy - Disk is going away, clean up resources.
+ * @scsidev: Scsi device to destroy
  */
 static void visorhba_slave_destroy(struct scsi_device *scsidev)
 {
 	/* midlevel calls this after device has been quiesced and
 	 * before it is to be deleted.
 	 */
-	struct visordisk_info *vdisk, *delvdisk;
-	struct visorhba_devdata *devdata;
-	struct Scsi_Host *scsihost = (struct Scsi_Host *)scsidev->host;
+	struct visordisk_info *vdisk;
 
-	devdata = (struct visorhba_devdata *)scsihost->hostdata;
-	for_each_vdisk_match(vdisk, devdata, scsidev) {
-		delvdisk = vdisk->next;
-		vdisk->next = delvdisk->next;
-		kfree(delvdisk);
-		return;
-	}
+	vdisk = scsidev->hostdata;
+	scsidev->hostdata = NULL;
+	kfree(vdisk);
 }
 
 static struct scsi_host_template visorhba_driver_template = {
@@ -635,10 +657,13 @@ static struct scsi_host_template visorhba_driver_template = {
 };
 
 /*
- *	info_debugfs_show - debugfs interface to dump visorhba states
+ * info_debugfs_show - Debugfs interface to dump visorhba states
+ * @seq: The sequence file to write information to
+ * @v:   Unused, but needed for use with seq file single_open invocation
+ *
+ * Presents a file in the debugfs tree named: /visorhba/vbus<x>:dev<y>/info.
  *
- *      This presents a file in the debugfs tree named:
- *          /visorhba/vbus<x>:dev<y>/info
+ * Return: SUCCESS
  */
 static int info_debugfs_show(struct seq_file *seq, void *v)
 {
@@ -679,12 +704,13 @@ static const struct file_operations info_debugfs_fops = {
 };
 
 /*
- *	complete_taskmgmt_command - complete task management
- *	@cmdrsp: Response from the IOVM
+ * complete_taskmgmt_command - Complete task management
+ * @idrtable: The data object maintaining the pointer<-->int mappings
+ * @cmdrsp:   Response from the IOVM
+ * @result:   The result of the task management command
  *
- *	Service Partition returned the result of the task management
- *	command. Wake up anyone waiting for it.
- *	Returns void
+ * Service Partition returned the result of the task management
+ * command. Wake up anyone waiting for it.
  */
 static void complete_taskmgmt_command(struct idr *idrtable,
 				      struct uiscmdrsp *cmdrsp, int result)
@@ -693,7 +719,6 @@ static void complete_taskmgmt_command(struct idr *idrtable,
 		idr_find(idrtable, cmdrsp->scsitaskmgmt.notify_handle);
 	int *scsi_result_ptr =
 		idr_find(idrtable, cmdrsp->scsitaskmgmt.notifyresult_handle);
-
 	if (unlikely(!(wq && scsi_result_ptr))) {
 		pr_err("visorhba: no completion context; cmd will time out\n");
 		return;
@@ -708,13 +733,12 @@ static void complete_taskmgmt_command(struct idr *idrtable,
 }
 
 /*
- *	visorhba_serverdown_complete - Called when we are done cleaning up
- *				       from serverdown
- *	@work: work structure for this serverdown request
+ * visorhba_serverdown_complete - Called when we are done cleaning up
+ *				  from serverdown
+ * @devdata: Visorhba instance on which to complete serverdown
  *
- *	Called when we are done cleanning up from serverdown, stop processing
- *	queue, fail pending IOs.
- *	Returns void when finished cleaning up
+ * Called when we are done cleanning up from serverdown, stop processing
+ * queue, fail pending IOs.
  */
 static void visorhba_serverdown_complete(struct visorhba_devdata *devdata)
 {
@@ -758,12 +782,13 @@ static void visorhba_serverdown_complete(struct visorhba_devdata *devdata)
 }
 
 /*
- *	visorhba_serverdown - Got notified that the IOVM is down
- *	@devdata: visorhba that is being serviced by downed IOVM.
+ * visorhba_serverdown - Got notified that the IOVM is down
+ * @devdata: Visorhba that is being serviced by downed IOVM
  *
- *	Something happened to the IOVM, return immediately and
- *	schedule work cleanup work.
- *	Return SUCCESS or EINVAL
+ * Something happened to the IOVM, return immediately and
+ * schedule cleanup work.
+ *
+ * Return: 0 on success, -EINVAL on failure
  */
 static int visorhba_serverdown(struct visorhba_devdata *devdata)
 {
@@ -777,17 +802,15 @@ static int visorhba_serverdown(struct visorhba_devdata *devdata)
 }
 
 /*
- *	do_scsi_linuxstat - scsi command returned linuxstat
- *	@cmdrsp: response from IOVM
- *	@scsicmd: Command issued.
+ * do_scsi_linuxstat - Scsi command returned linuxstat
+ * @cmdrsp:  Response from IOVM
+ * @scsicmd: Command issued
  *
- *	Don't log errors for disk-not-present inquiries
- *	Returns void
+ * Don't log errors for disk-not-present inquiries.
  */
-static void
-do_scsi_linuxstat(struct uiscmdrsp *cmdrsp, struct scsi_cmnd *scsicmd)
+static void do_scsi_linuxstat(struct uiscmdrsp *cmdrsp,
+			      struct scsi_cmnd *scsicmd)
 {
-	struct visorhba_devdata *devdata;
 	struct visordisk_info *vdisk;
 	struct scsi_device *scsidev;
 
@@ -800,19 +823,17 @@ do_scsi_linuxstat(struct uiscmdrsp *cmdrsp, struct scsi_cmnd *scsicmd)
 	    (cmdrsp->scsi.addlstat == ADDL_SEL_TIMEOUT))
 		return;
 	/* Okay see what our error_count is here.... */
-	devdata = (struct visorhba_devdata *)scsidev->host->hostdata;
-	for_each_vdisk_match(vdisk, devdata, scsidev) {
-		if (atomic_read(&vdisk->error_count) < VISORHBA_ERROR_COUNT) {
-			atomic_inc(&vdisk->error_count);
-			atomic_set(&vdisk->ios_threshold, IOS_ERROR_THRESHOLD);
-		}
+	vdisk = scsidev->hostdata;
+	if (atomic_read(&vdisk->error_count) < VISORHBA_ERROR_COUNT) {
+		atomic_inc(&vdisk->error_count);
+		atomic_set(&vdisk->ios_threshold, IOS_ERROR_THRESHOLD);
 	}
 }
 
-static int set_no_disk_inquiry_result(unsigned char *buf,
-				      size_t len, bool is_lun0)
+static int set_no_disk_inquiry_result(unsigned char *buf, size_t len,
+				      bool is_lun0)
 {
-	if (!buf || len < NO_DISK_INQUIRY_RESULT_LEN)
+	if (len < NO_DISK_INQUIRY_RESULT_LEN)
 		return -EINVAL;
 	memset(buf, 0, NO_DISK_INQUIRY_RESULT_LEN);
 	buf[2] = SCSI_SPC2_VER;
@@ -828,15 +849,14 @@ static int set_no_disk_inquiry_result(unsigned char *buf,
 }
 
 /*
- *	do_scsi_nolinuxstat - scsi command didn't have linuxstat
- *	@cmdrsp: response from IOVM
- *	@scsicmd: Command issued.
+ * do_scsi_nolinuxstat - Scsi command didn't have linuxstat
+ * @cmdrsp:  Response from IOVM
+ * @scsicmd: Command issued
  *
- *	Handle response when no linuxstat was returned
- *	Returns void
+ * Handle response when no linuxstat was returned.
  */
-static void
-do_scsi_nolinuxstat(struct uiscmdrsp *cmdrsp, struct scsi_cmnd *scsicmd)
+static void do_scsi_nolinuxstat(struct uiscmdrsp *cmdrsp,
+				struct scsi_cmnd *scsicmd)
 {
 	struct scsi_device *scsidev;
 	unsigned char *buf;
@@ -846,7 +866,6 @@ do_scsi_nolinuxstat(struct uiscmdrsp *cmdrsp, struct scsi_cmnd *scsicmd)
 	char *this_page_orig;
 	int bufind = 0;
 	struct visordisk_info *vdisk;
-	struct visorhba_devdata *devdata;
 
 	scsidev = scsicmd->device;
 	if ((cmdrsp->scsi.cmnd[0] == INQUIRY) &&
@@ -883,28 +902,25 @@ do_scsi_nolinuxstat(struct uiscmdrsp *cmdrsp, struct scsi_cmnd *scsicmd)
 		}
 		kfree(buf);
 	} else {
-		devdata = (struct visorhba_devdata *)scsidev->host->hostdata;
-		for_each_vdisk_match(vdisk, devdata, scsidev) {
-			if (atomic_read(&vdisk->ios_threshold) > 0) {
-				atomic_dec(&vdisk->ios_threshold);
-				if (atomic_read(&vdisk->ios_threshold) == 0)
-					atomic_set(&vdisk->error_count, 0);
-			}
+		vdisk = scsidev->hostdata;
+		if (atomic_read(&vdisk->ios_threshold) > 0) {
+			atomic_dec(&vdisk->ios_threshold);
+			if (atomic_read(&vdisk->ios_threshold) == 0)
+				atomic_set(&vdisk->error_count, 0);
 		}
 	}
 }
 
 /*
- *	complete_scsi_command - complete a scsi command
- *	@uiscmdrsp: Response from Service Partition
- *	@scsicmd: The scsi command
+ * complete_scsi_command - Complete a scsi command
+ * @uiscmdrsp: Response from Service Partition
+ * @scsicmd:   The scsi command
  *
- *	Response returned by the Service Partition, finish it and send
- *	completion to the scsi midlayer.
- *	Returns void.
+ * Response was returned by the Service Partition. Finish it and send
+ * completion to the scsi midlayer.
  */
-static void
-complete_scsi_command(struct uiscmdrsp *cmdrsp, struct scsi_cmnd *scsicmd)
+static void complete_scsi_command(struct uiscmdrsp *cmdrsp,
+				  struct scsi_cmnd *scsicmd)
 {
 	/* take what we need out of cmdrsp and complete the scsicmd */
 	scsicmd->result = cmdrsp->scsi.linuxstat;
@@ -917,24 +933,23 @@ complete_scsi_command(struct uiscmdrsp *cmdrsp, struct scsi_cmnd *scsicmd)
 }
 
 /*
- *	drain_queue - pull responses out of iochannel
- *	@cmdrsp: Response from the IOSP
- *	@devdata: device that owns this iochannel
+ * drain_queue - Pull responses out of iochannel
+ * @cmdrsp:  Response from the IOSP
+ * @devdata: Device that owns this iochannel
  *
- *	Pulls responses out of the iochannel and process the responses.
- *	Restuns void
+ * Pulls responses out of the iochannel and process the responses.
  */
-static void
-drain_queue(struct uiscmdrsp *cmdrsp, struct visorhba_devdata *devdata)
+static void drain_queue(struct uiscmdrsp *cmdrsp,
+			struct visorhba_devdata *devdata)
 {
 	struct scsi_cmnd *scsicmd;
 
 	while (1) {
+		/* queue empty */
 		if (visorchannel_signalremove(devdata->dev->visorchannel,
 					      IOCHAN_FROM_IOPART,
 					      cmdrsp))
-			break; /* queue empty */
-
+			break;
 		if (cmdrsp->cmdtype == CMD_SCSI_TYPE) {
 			/* scsicmd location is returned by the
 			 * deletion
@@ -959,12 +974,14 @@ drain_queue(struct uiscmdrsp *cmdrsp, struct visorhba_devdata *devdata)
 }
 
 /*
- *	process_incoming_rsps - Process responses from IOSP
- *	@v: void pointer to visorhba_devdata
+ * process_incoming_rsps - Process responses from IOSP
+ * @v:  Void pointer to visorhba_devdata
  *
- *	Main function for the thread that processes the responses
- *	from the IO Service Partition. When the queue is empty, wait
- *	to check to see if it is full again.
+ * Main function for the thread that processes the responses
+ * from the IO Service Partition. When the queue is empty, wait
+ * to check to see if it is full again.
+ *
+ * Return: 0 on success, -ENOMEM on failure
  */
 static int process_incoming_rsps(void *v)
 {
@@ -991,14 +1008,15 @@ static int process_incoming_rsps(void *v)
 }
 
 /*
- *	visorhba_pause - function to handle visorbus pause messages
- *	@dev: device that is pausing.
- *	@complete_func: function to call when finished
+ * visorhba_pause - Function to handle visorbus pause messages
+ * @dev:	   Device that is pausing
+ * @complete_func: Function to call when finished
+ *
+ * Something has happened to the IO Service Partition that is
+ * handling this device. Quiet this device and reset commands
+ * so that the Service Partition can be corrected.
  *
- *	Something has happened to the IO Service Partition that is
- *	handling this device. Quiet this device and reset commands
- *	so that the Service Partition can be corrected.
- *	Returns SUCCESS
+ * Return: SUCCESS
  */
 static int visorhba_pause(struct visor_device *dev,
 			  visorbus_state_complete_func complete_func)
@@ -1011,13 +1029,14 @@ static int visorhba_pause(struct visor_device *dev,
 }
 
 /*
- *	visorhba_resume - function called when the IO Service Partition is back
- *	@dev: device that is pausing.
- *	@complete_func: function to call when finished
+ * visorhba_resume - Function called when the IO Service Partition is back
+ * @dev:	   Device that is pausing
+ * @complete_func: Function to call when finished
  *
- *	Yay! The IO Service Partition is back, the channel has been wiped
- *	so lets re-establish connection and start processing responses.
- *	Returns 0 on success, error on failure.
+ * Yay! The IO Service Partition is back, the channel has been wiped
+ * so lets re-establish connection and start processing responses.
+ *
+ * Return: 0 on success, -EINVAL on failure
  */
 static int visorhba_resume(struct visor_device *dev,
 			   visorbus_state_complete_func complete_func)
@@ -1033,7 +1052,6 @@ static int visorhba_resume(struct visor_device *dev,
 
 	devdata->thread = visor_thread_start(process_incoming_rsps, devdata,
 					     "vhba_incming");
-
 	devdata->serverdown = false;
 	devdata->serverchangingstate = false;
 
@@ -1041,11 +1059,12 @@ static int visorhba_resume(struct visor_device *dev,
 }
 
 /*
- *	visorhba_probe - device has been discovered, do acquire
- *	@dev: visor_device that was discovered
+ * visorhba_probe - Device has been discovered; do acquire
+ * @dev: visor_device that was discovered
  *
- *	A new HBA was discovered, do the initial connections of it.
- *	Return 0 on success, otherwise error.
+ * A new HBA was discovered; do the initial connections of it.
+ *
+ * Return: 0 on success, otherwise error code
  */
 static int visorhba_probe(struct visor_device *dev)
 {
@@ -1139,11 +1158,10 @@ err_scsi_host_put:
 }
 
 /*
- *	visorhba_remove - remove a visorhba device
- *	@dev: Device to remove
+ * visorhba_remove - Remove a visorhba device
+ * @dev: Device to remove
  *
- *	Removes the visorhba device.
- *	Returns void.
+ * Removes the visorhba device.
  */
 static void visorhba_remove(struct visor_device *dev)
 {
@@ -1181,10 +1199,12 @@ static struct visor_driver visorhba_driver = {
 };
 
 /*
- *	visorhba_init		- driver init routine
+ * visorhba_init - Driver init routine
+ *
+ * Initialize the visorhba driver and register it with visorbus
+ * to handle s-Par virtual host bus adapter.
  *
- *	Initialize the visorhba driver and register it with visorbus
- *	to handle s-Par virtual host bus adapter.
+ * Return: 0 on success, error code otherwise
  */
 static int visorhba_init(void)
 {
@@ -1207,9 +1227,9 @@ cleanup_debugfs:
 }
 
 /*
- *	visorhba_exit	- driver exit routine
+ * visorhba_exit - Driver exit routine
  *
- *	Unregister driver from the bus and free up memory.
+ * Unregister driver from the bus and free up memory.
  */
 static void visorhba_exit(void)
 {
diff --git a/drivers/staging/unisys/visorinput/ultrainputreport.h b/drivers/staging/unisys/visorinput/ultrainputreport.h
index a4baea53c518..53975a09535f 100644
--- a/drivers/staging/unisys/visorinput/ultrainputreport.h
+++ b/drivers/staging/unisys/visorinput/ultrainputreport.h
@@ -1,4 +1,5 @@
-/* Copyright (C) 2010 - 2015 UNISYS CORPORATION
+/*
+ * Copyright (C) 2010 - 2015 UNISYS CORPORATION
  * All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
@@ -20,44 +21,35 @@
 /* These defines identify mouse and keyboard activity which is specified by the
  * firmware to the host using the cmsimpleinput protocol.  @ingroup coretypes
  */
-#define INPUTACTION_XY_MOTION 1		/* only motion; arg1=x, arg2=y */
-#define INPUTACTION_MOUSE_BUTTON_DOWN 2 /* arg1: 1=left,2=center,3=right */
-#define INPUTACTION_MOUSE_BUTTON_UP 3	/* arg1: 1=left,2=center,3=right */
-#define INPUTACTION_MOUSE_BUTTON_CLICK 4 /* arg1: 1=left,2=center,3=right */
-#define INPUTACTION_MOUSE_BUTTON_DCLICK 5 /* arg1: 1=left,2=center,
-					   * 3=right
-					   */
-#define INPUTACTION_WHEEL_ROTATE_AWAY 6  /* arg1: wheel rotation away from
-					  * user
-					  */
-#define INPUTACTION_WHEEL_ROTATE_TOWARD 7 /* arg1: wheel rotation toward
-					   * user
-					   */
-#define INPUTACTION_KEY_DOWN 64		/* arg1: scancode, as follows:
-					 * If arg1 <= 0xff, it's a 1-byte
-					 * scancode and arg1 is that scancode.
-					 * If arg1 > 0xff, it's a 2-byte
-					 * scanecode, with the 1st byte in the
-					 * low 8 bits, and the 2nd byte in the
-					 * high 8 bits.  E.g., the right ALT key
-					 * would appear as x'38e0'.
-					 */
-#define INPUTACTION_KEY_UP 65		/* arg1: scancode (in same format as
-					 * inputaction_keyDown)
-					 */
+	/* only motion; arg1=x, arg2=y */
+#define INPUTACTION_XY_MOTION 1
+/* arg1: 1=left,2=center,3=right */
+#define INPUTACTION_MOUSE_BUTTON_DOWN 2
+/* arg1: 1=left,2=center,3=right */
+#define INPUTACTION_MOUSE_BUTTON_UP 3
+/* arg1: 1=left,2=center,3=right */
+#define INPUTACTION_MOUSE_BUTTON_CLICK 4
+/* arg1: 1=left,2=center 3=right */
+#define INPUTACTION_MOUSE_BUTTON_DCLICK 5
+/* arg1: wheel rotation away from user */
+#define INPUTACTION_WHEEL_ROTATE_AWAY 6
+/* arg1: wheel rotation toward user */
+#define INPUTACTION_WHEEL_ROTATE_TOWARD 7
+/* arg1: scancode, as follows: If arg1 <= 0xff, it's a 1-byte scancode and arg1
+ *	 is that scancode. If arg1 > 0xff, it's a 2-byte scanecode, with the 1st
+ *	 byte in the low 8 bits, and the 2nd byte in the high 8 bits.
+ *	 E.g., the right ALT key would appear as x'38e0'.
+ */
+#define INPUTACTION_KEY_DOWN 64
+/* arg1: scancode (in same format as inputaction_keyDown) */
+#define INPUTACTION_KEY_UP 65
+/* arg1: scancode (in same format as inputaction_keyDown); MUST refer to one of
+ *	 the locking keys, like capslock, numlock, or scrolllock.
+ * arg2: 1 iff locking key should be in the LOCKED position (e.g., light is ON)
+ */
 #define INPUTACTION_SET_LOCKING_KEY_STATE 66
-					/* arg1: scancode (in same format
-					 *	 as inputaction_keyDown);
-					 *	 MUST refer to one of the
-					 *	 locking keys, like capslock,
-					 *	 numlock, or scrolllock
-					 * arg2: 1 iff locking key should be
-					 *	 in the LOCKED position
-					 *	 (e.g., light is ON)
-					 */
-#define INPUTACTION_KEY_DOWN_UP 67	/* arg1: scancode (in same format
-					 *	 as inputaction_keyDown)
-					 */
+/* arg1: scancode (in same format as inputaction_keyDown */
+#define INPUTACTION_KEY_DOWN_UP 67
 
 struct visor_inputactivity {
 	u16 action;
diff --git a/drivers/staging/unisys/visorinput/visorinput.c b/drivers/staging/unisys/visorinput/visorinput.c
index 45bc340d4e9d..9d8cbc52de8b 100644
--- a/drivers/staging/unisys/visorinput/visorinput.c
+++ b/drivers/staging/unisys/visorinput/visorinput.c
@@ -1,5 +1,4 @@
-/* visorinput.c
- *
+/*
  * Copyright (C) 2011 - 2015 UNISYS CORPORATION
  * All rights reserved.
  *
@@ -21,11 +20,8 @@
  * standard way the Linux expects for input drivers.
  */
 
-#include <linux/buffer_head.h>
 #include <linux/fb.h>
-#include <linux/fs.h>
 #include <linux/input.h>
-#include <linux/uaccess.h>
 #include <linux/kernel.h>
 #include <linux/uuid.h>
 
@@ -33,16 +29,16 @@
 #include "ultrainputreport.h"
 
 /* Keyboard channel {c73416d0-b0b8-44af-b304-9d2ae99f1b3d} */
-#define VISOR_KEYBOARD_CHANNEL_UUID \
-	UUID_LE(0xc73416d0, 0xb0b8, 0x44af, \
-		0xb3, 0x4, 0x9d, 0x2a, 0xe9, 0x9f, 0x1b, 0x3d)
-#define VISOR_KEYBOARD_CHANNEL_UUID_STR "c73416d0-b0b8-44af-b304-9d2ae99f1b3d"
+#define VISOR_KEYBOARD_CHANNEL_GUID \
+	GUID_INIT(0xc73416d0, 0xb0b8, 0x44af, \
+		  0xb3, 0x4, 0x9d, 0x2a, 0xe9, 0x9f, 0x1b, 0x3d)
+#define VISOR_KEYBOARD_CHANNEL_GUID_STR "c73416d0-b0b8-44af-b304-9d2ae99f1b3d"
 
 /* Mouse channel {addf07d4-94a9-46e2-81c3-61abcdbdbd87} */
-#define VISOR_MOUSE_CHANNEL_UUID \
-	UUID_LE(0xaddf07d4, 0x94a9, 0x46e2, \
-		0x81, 0xc3, 0x61, 0xab, 0xcd, 0xbd, 0xbd, 0x87)
-#define VISOR_MOUSE_CHANNEL_UUID_STR "addf07d4-94a9-46e2-81c3-61abcdbdbd87"
+#define VISOR_MOUSE_CHANNEL_GUID \
+	GUID_INIT(0xaddf07d4, 0x94a9, 0x46e2, \
+		  0x81, 0xc3, 0x61, 0xab, 0xcd, 0xbd, 0xbd, 0x87)
+#define VISOR_MOUSE_CHANNEL_GUID_STR "addf07d4-94a9-46e2-81c3-61abcdbdbd87"
 
 #define PIXELS_ACROSS_DEFAULT 800
 #define PIXELS_DOWN_DEFAULT   600
@@ -60,17 +56,19 @@ enum visorinput_device_type {
  */
 struct visorinput_devdata {
 	struct visor_device *dev;
-	struct mutex lock_visor_dev; /* lock for dev */
+	/* lock for dev */
+	struct mutex lock_visor_dev;
 	struct input_dev *visorinput_dev;
 	bool paused;
 	bool interrupts_enabled;
-	unsigned int keycode_table_bytes; /* size of following array */
+	/* size of following array */
+	unsigned int keycode_table_bytes;
 	/* for keyboard devices: visorkbd_keycode[] + visorkbd_ext_keycode[] */
 	unsigned char keycode_table[0];
 };
 
-static const uuid_le visor_keyboard_channel_uuid = VISOR_KEYBOARD_CHANNEL_UUID;
-static const uuid_le visor_mouse_channel_uuid = VISOR_MOUSE_CHANNEL_UUID;
+static const guid_t visor_keyboard_channel_guid = VISOR_KEYBOARD_CHANNEL_GUID;
+static const guid_t visor_mouse_channel_guid = VISOR_MOUSE_CHANNEL_GUID;
 
 /*
  * Borrowed from drivers/input/keyboard/atakbd.c
@@ -162,9 +160,8 @@ static const unsigned char visorkbd_keycode[KEYCODE_TABLE_BYTES] = {
 	[81] = KEY_KP3,
 	[82] = KEY_KP0,
 	[83] = KEY_KPDOT,
-	[86] = KEY_102ND, /* enables UK backslash+pipe key,
-			   * and FR lessthan+greaterthan key
-			   */
+	/* enables UK backslash+pipe key and FR lessthan+greaterthan key */
+	[86] = KEY_102ND,
 	[87] = KEY_F11,
 	[88] = KEY_F12,
 	[90] = KEY_KPLEFTPAREN,
@@ -260,7 +257,6 @@ static void visorinput_close(struct input_dev *visorinput_dev)
 	 * interrupts should be disabled so when we resume we will
 	 * not re-enable them.
 	 */
-
 	mutex_lock(&devdata->lock_visor_dev);
 	devdata->interrupts_enabled = false;
 	if (devdata->paused)
@@ -276,15 +272,13 @@ out_unlock:
  * we can use to deliver keyboard inputs to Linux.  We of course do this when
  * we see keyboard inputs coming in on a keyboard channel.
  */
-static struct input_dev *
-setup_client_keyboard(void *devdata,  /* opaque on purpose */
-		      unsigned char *keycode_table)
+static struct input_dev *setup_client_keyboard(void *devdata,
+					       unsigned char *keycode_table)
 
 {
 	int i;
-	struct input_dev *visorinput_dev;
+	struct input_dev *visorinput_dev = input_allocate_device();
 
-	visorinput_dev = input_allocate_device();
 	if (!visorinput_dev)
 		return NULL;
 
@@ -302,7 +296,8 @@ setup_client_keyboard(void *devdata,  /* opaque on purpose */
 				    BIT_MASK(LED_SCROLLL) |
 				    BIT_MASK(LED_NUML);
 	visorinput_dev->keycode = keycode_table;
-	visorinput_dev->keycodesize = 1; /* sizeof(unsigned char) */
+	/* sizeof(unsigned char) */
+	visorinput_dev->keycodesize = 1;
 	visorinput_dev->keycodemax = KEYCODE_TABLE_BYTES;
 
 	for (i = 1; i < visorinput_dev->keycodemax; i++)
@@ -313,19 +308,18 @@ setup_client_keyboard(void *devdata,  /* opaque on purpose */
 
 	visorinput_dev->open = visorinput_open;
 	visorinput_dev->close = visorinput_close;
-	input_set_drvdata(visorinput_dev, devdata); /* pre input_register! */
+	/* pre input_register! */
+	input_set_drvdata(visorinput_dev, devdata);
 
 	return visorinput_dev;
 }
 
-static struct input_dev *
-setup_client_mouse(void *devdata /* opaque on purpose */)
+static struct input_dev *setup_client_mouse(void *devdata)
 {
-	struct input_dev *visorinput_dev = NULL;
 	int xres, yres;
 	struct fb_info *fb0;
+	struct input_dev *visorinput_dev = input_allocate_device();
 
-	visorinput_dev = input_allocate_device();
 	if (!visorinput_dev)
 		return NULL;
 
@@ -354,14 +348,16 @@ setup_client_mouse(void *devdata /* opaque on purpose */)
 
 	visorinput_dev->open = visorinput_open;
 	visorinput_dev->close = visorinput_close;
-	input_set_drvdata(visorinput_dev, devdata); /* pre input_register! */
+	/* pre input_register! */
+	input_set_drvdata(visorinput_dev, devdata);
 	input_set_capability(visorinput_dev, EV_REL, REL_WHEEL);
 
 	return visorinput_dev;
 }
 
-static struct visorinput_devdata *
-devdata_create(struct visor_device *dev, enum visorinput_device_type devtype)
+static struct visorinput_devdata *devdata_create(
+					struct visor_device *dev,
+					enum visorinput_device_type devtype)
 {
 	struct visorinput_devdata *devdata = NULL;
 	unsigned int extra_bytes = 0;
@@ -446,16 +442,15 @@ err_kfree_devdata:
 	return NULL;
 }
 
-static int
-visorinput_probe(struct visor_device *dev)
+static int visorinput_probe(struct visor_device *dev)
 {
-	uuid_le guid;
+	const guid_t *guid;
 	enum visorinput_device_type devtype;
 
-	guid = visorchannel_get_uuid(dev->visorchannel);
-	if (uuid_le_cmp(guid, visor_mouse_channel_uuid) == 0)
+	guid = visorchannel_get_guid(dev->visorchannel);
+	if (guid_equal(guid, &visor_mouse_channel_guid))
 		devtype = visorinput_mouse;
-	else if (uuid_le_cmp(guid, visor_keyboard_channel_uuid) == 0)
+	else if (guid_equal(guid, &visor_keyboard_channel_guid))
 		devtype = visorinput_keyboard;
 	else
 		return -ENODEV;
@@ -465,15 +460,13 @@ visorinput_probe(struct visor_device *dev)
 	return 0;
 }
 
-static void
-unregister_client_input(struct input_dev *visorinput_dev)
+static void unregister_client_input(struct input_dev *visorinput_dev)
 {
 	if (visorinput_dev)
 		input_unregister_device(visorinput_dev);
 }
 
-static void
-visorinput_remove(struct visor_device *dev)
+static void visorinput_remove(struct visor_device *dev)
 {
 	struct visorinput_devdata *devdata = dev_get_drvdata(&dev->device);
 
@@ -499,9 +492,8 @@ visorinput_remove(struct visor_device *dev)
  * Make it so the current locking state of the locking key indicated by
  * <keycode> is as indicated by <desired_state> (1=locked, 0=unlocked).
  */
-static void
-handle_locking_key(struct input_dev *visorinput_dev,
-		   int keycode, int desired_state)
+static void handle_locking_key(struct input_dev *visorinput_dev, int keycode,
+			       int desired_state)
 {
 	int led;
 
@@ -533,17 +525,15 @@ handle_locking_key(struct input_dev *visorinput_dev,
  * with 0xE0 in the low byte and the extended scancode value in the next
  * higher byte.
  */
-static int
-scancode_to_keycode(int scancode)
+static int scancode_to_keycode(int scancode)
 {
 	if (scancode > 0xff)
 		return visorkbd_ext_keycode[(scancode >> 8) & 0xff];
 
-	return  visorkbd_keycode[scancode];
+	return visorkbd_keycode[scancode];
 }
 
-static int
-calc_button(int x)
+static int calc_button(int x)
 {
 	switch (x) {
 	case 1:
@@ -562,15 +552,13 @@ calc_button(int x)
  * client guest partition.  It is called periodically so we can obtain inputs
  * from the channel, and deliver them to the guest OS.
  */
-static void
-visorinput_channel_interrupt(struct visor_device *dev)
+static void visorinput_channel_interrupt(struct visor_device *dev)
 {
 	struct visor_inputreport r;
 	int scancode, keycode;
 	struct input_dev *visorinput_dev;
 	int xmotion, ymotion, button;
 	int i;
-
 	struct visorinput_devdata *devdata = dev_get_drvdata(&dev->device);
 
 	if (!devdata)
@@ -626,7 +614,6 @@ visorinput_channel_interrupt(struct visor_device *dev)
 			if (button < 0)
 				break;
 			input_report_key(visorinput_dev, button, 1);
-
 			input_sync(visorinput_dev);
 			input_report_key(visorinput_dev, button, 0);
 			input_sync(visorinput_dev);
@@ -657,9 +644,8 @@ visorinput_channel_interrupt(struct visor_device *dev)
 	}
 }
 
-static int
-visorinput_pause(struct visor_device *dev,
-		 visorbus_state_complete_func complete_func)
+static int visorinput_pause(struct visor_device *dev,
+			    visorbus_state_complete_func complete_func)
 {
 	int rc;
 	struct visorinput_devdata *devdata = dev_get_drvdata(&dev->device);
@@ -681,7 +667,6 @@ visorinput_pause(struct visor_device *dev,
 	 * due to above, at this time no thread of execution will be
 	 * in visorinput_channel_interrupt()
 	 */
-
 	devdata->paused = true;
 	complete_func(dev, 0);
 	rc = 0;
@@ -691,9 +676,8 @@ out:
 	return rc;
 }
 
-static int
-visorinput_resume(struct visor_device *dev,
-		  visorbus_state_complete_func complete_func)
+static int visorinput_resume(struct visor_device *dev,
+			     visorbus_state_complete_func complete_func)
 {
 	int rc;
 	struct visorinput_devdata *devdata = dev_get_drvdata(&dev->device);
@@ -727,9 +711,9 @@ out:
 
 /* GUIDS for all channel types supported by this driver. */
 static struct visor_channeltype_descriptor visorinput_channel_types[] = {
-	{ VISOR_KEYBOARD_CHANNEL_UUID, "keyboard"},
-	{ VISOR_MOUSE_CHANNEL_UUID, "mouse"},
-	{ NULL_UUID_LE, NULL }
+	{ VISOR_KEYBOARD_CHANNEL_GUID, "keyboard"},
+	{ VISOR_MOUSE_CHANNEL_GUID, "mouse"},
+	{}
 };
 
 static struct visor_driver visorinput_driver = {
@@ -743,20 +727,8 @@ static struct visor_driver visorinput_driver = {
 	.resume = visorinput_resume,
 };
 
-static int
-visorinput_init(void)
-{
-	return visorbus_register_visor_driver(&visorinput_driver);
-}
-
-static void
-visorinput_cleanup(void)
-{
-	visorbus_unregister_visor_driver(&visorinput_driver);
-}
-
-module_init(visorinput_init);
-module_exit(visorinput_cleanup);
+module_driver(visorinput_driver, visorbus_register_visor_driver,
+	      visorbus_unregister_visor_driver);
 
 MODULE_DEVICE_TABLE(visorbus, visorinput_channel_types);
 
@@ -764,5 +736,5 @@ MODULE_AUTHOR("Unisys");
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("s-Par human input driver for virtual keyboard/mouse");
 
-MODULE_ALIAS("visorbus:" VISOR_MOUSE_CHANNEL_UUID_STR);
-MODULE_ALIAS("visorbus:" VISOR_KEYBOARD_CHANNEL_UUID_STR);
+MODULE_ALIAS("visorbus:" VISOR_MOUSE_CHANNEL_GUID_STR);
+MODULE_ALIAS("visorbus:" VISOR_KEYBOARD_CHANNEL_GUID_STR);
diff --git a/drivers/staging/unisys/visornic/visornic_main.c b/drivers/staging/unisys/visornic/visornic_main.c
index 2891622eef18..dc390eae2960 100644
--- a/drivers/staging/unisys/visornic/visornic_main.c
+++ b/drivers/staging/unisys/visornic/visornic_main.c
@@ -37,22 +37,28 @@
 #define NAPI_WEIGHT 64
 
 /* GUIDS for director channel type supported by this driver.  */
+/* {8cd5994d-c58e-11da-95a9-00e08161165f} */
+#define VISOR_VNIC_CHANNEL_GUID \
+	GUID_INIT(0x8cd5994d, 0xc58e, 0x11da, \
+		0x95, 0xa9, 0x0, 0xe0, 0x81, 0x61, 0x16, 0x5f)
+#define VISOR_VNIC_CHANNEL_GUID_STR \
+	"8cd5994d-c58e-11da-95a9-00e08161165f"
+
 static struct visor_channeltype_descriptor visornic_channel_types[] = {
 	/* Note that the only channel type we expect to be reported by the
 	 * bus driver is the VISOR_VNIC channel.
 	 */
-	{ VISOR_VNIC_CHANNEL_UUID, "ultravnic" },
-	{ NULL_UUID_LE, NULL }
+	{ VISOR_VNIC_CHANNEL_GUID, "ultravnic" },
+	{}
 };
 MODULE_DEVICE_TABLE(visorbus, visornic_channel_types);
-/*
- * FIXME XXX: This next line of code must be fixed and removed before
+/* FIXME XXX: This next line of code must be fixed and removed before
  * acceptance into the 'normal' part of the kernel.  It is only here as a place
  * holder to get module autoloading functionality working for visorbus.  Code
  * must be added to scripts/mode/file2alias.c, etc., to get this working
  * properly.
  */
-MODULE_ALIAS("visorbus:" VISOR_VNIC_CHANNEL_UUID_STR);
+MODULE_ALIAS("visorbus:" VISOR_VNIC_CHANNEL_GUID_STR);
 
 struct chanstat {
 	unsigned long got_rcv;
@@ -68,10 +74,67 @@ struct chanstat {
 	unsigned long extra_rcvbufs_sent;
 };
 
+/* struct visornic_devdata
+ * @enabled:                        0 disabled 1 enabled to receive.
+ * @enab_dis_acked:                 NET_RCV_ENABLE/DISABLE acked by IOPART.
+ * @struct *dev:
+ * @struct *netdev:
+ * @struct net_stats:
+ * @interrupt_rcvd:
+ * @rsp_queue:
+ * @struct **rcvbuf:
+ * @incarnation_id:                 incarnation_id lets IOPART know about
+ *                                  re-birth.
+ * @old_flags:                      flags as they were prior to
+ *                                  set_multicast_list.
+ * @usage:                          count of users.
+ * @num_rcv_bufs:                   number of rcv buffers the vnic will post.
+ * @num_rcv_bufs_could_not_alloc:
+ * @num_rcvbuf_in_iovm:
+ * @alloc_failed_in_if_needed_cnt:
+ * @alloc_failed_in_repost_rtn_cnt:
+ * @max_outstanding_net_xmits:      absolute max number of outstanding xmits
+ *                                  - should never hit this.
+ * @upper_threshold_net_xmits:      high water mark for calling
+ *                                  netif_stop_queue().
+ * @lower_threshold_net_xmits:      high water mark for calling
+ *                                  netif_wake_queue().
+ * @struct xmitbufhead:             xmitbufhead - head of the xmit buffer list
+ *                                  sent to the IOPART end.
+ * @server_down_complete_func:
+ * @struct timeout_reset:
+ * @struct *cmdrsp_rcv:             cmdrsp_rcv is used for posting/unposting rcv
+ *                                  buffers.
+ * @struct *xmit_cmdrsp:            xmit_cmdrsp - issues NET_XMIT - only one
+ *                                  active xmit at a time.
+ * @server_down:                    IOPART is down.
+ * @server_change_state:            Processing SERVER_CHANGESTATE msg.
+ * @going_away:                     device is being torn down.
+ * @struct *eth_debugfs_dir:
+ * @interrupts_rcvd:
+ * @interrupts_notme:
+ * @interrupts_disabled:
+ * @busy_cnt:
+ * @priv_lock:                      spinlock to access devdata structures.
+ * @flow_control_upper_hits:
+ * @flow_control_lower_hits:
+ * @n_rcv0:                         # rcvs of 0 buffers.
+ * @n_rcv1:                         # rcvs of 1 buffers.
+ * @n_rcv2:                         # rcvs of 2 buffers.
+ * @n_rcvx:                         # rcvs of >2 buffers.
+ * @found_repost_rcvbuf_cnt:        # repost_rcvbuf_cnt.
+ * @repost_found_skb_cnt:           # of found the skb.
+ * @n_repost_deficit:               # of lost rcv buffers.
+ * @bad_rcv_buf:                    # of unknown rcv skb not freed.
+ * @n_rcv_packets_not_accepted:     # bogs rcv packets.
+ * @queuefullmsg_logged:
+ * @struct chstat:
+ * @struct irq_poll_timer:
+ * @struct napi:
+ * @struct cmdrsp:
+ */
 struct visornic_devdata {
-	/* 0 disabled 1 enabled to receive */
 	unsigned short enabled;
-	/* NET_RCV_ENABLE/DISABLE acked by IOPART */
 	unsigned short enab_dis_acked;
 
 	struct visor_device *dev;
@@ -80,59 +143,50 @@ struct visornic_devdata {
 	atomic_t interrupt_rcvd;
 	wait_queue_head_t rsp_queue;
 	struct sk_buff **rcvbuf;
-	/* incarnation_id lets IOPART know about re-birth */
 	u64 incarnation_id;
-	/* flags as they were prior to set_multicast_list */
 	unsigned short old_flags;
-	atomic_t usage;	/* count of users */
+	atomic_t usage;
 
-	/* number of rcv buffers the vnic will post */
 	int num_rcv_bufs;
 	int num_rcv_bufs_could_not_alloc;
 	atomic_t num_rcvbuf_in_iovm;
 	unsigned long alloc_failed_in_if_needed_cnt;
 	unsigned long alloc_failed_in_repost_rtn_cnt;
 
-	/* absolute max number of outstanding xmits - should never hit this */
 	unsigned long max_outstanding_net_xmits;
-	/* high water mark for calling netif_stop_queue() */
 	unsigned long upper_threshold_net_xmits;
-	/* high water mark for calling netif_wake_queue() */
 	unsigned long lower_threshold_net_xmits;
-	/* xmitbufhead - head of the xmit buffer list sent to the IOPART end */
 	struct sk_buff_head xmitbufhead;
 
 	visorbus_state_complete_func server_down_complete_func;
 	struct work_struct timeout_reset;
-	/* cmdrsp_rcv is used for posting/unposting rcv buffers  */
 	struct uiscmdrsp *cmdrsp_rcv;
-	/* xmit_cmdrsp - issues NET_XMIT - only one active xmit at a time */
 	struct uiscmdrsp *xmit_cmdrsp;
-
-	bool server_down;		 /* IOPART is down */
-	bool server_change_state;	 /* Processing SERVER_CHANGESTATE msg */
-	bool going_away;		 /* device is being torn down */
+	bool server_down;
+	bool server_change_state;
+	bool going_away;
 	struct dentry *eth_debugfs_dir;
 	u64 interrupts_rcvd;
 	u64 interrupts_notme;
 	u64 interrupts_disabled;
 	u64 busy_cnt;
-	spinlock_t priv_lock;  /* spinlock to access devdata structures */
+	/* spinlock to access devdata structures. */
+	spinlock_t priv_lock;
 
 	/* flow control counter */
 	u64 flow_control_upper_hits;
 	u64 flow_control_lower_hits;
 
 	/* debug counters */
-	unsigned long n_rcv0;			/* # rcvs of 0 buffers */
-	unsigned long n_rcv1;			/* # rcvs of 1 buffers */
-	unsigned long n_rcv2;			/* # rcvs of 2 buffers */
-	unsigned long n_rcvx;			/* # rcvs of >2 buffers */
-	unsigned long found_repost_rcvbuf_cnt;	/* # repost_rcvbuf_cnt */
-	unsigned long repost_found_skb_cnt;	/* # of found the skb */
-	unsigned long n_repost_deficit;		/* # of lost rcv buffers */
-	unsigned long bad_rcv_buf; /* # of unknown rcv skb  not freed */
-	unsigned long n_rcv_packets_not_accepted;/* # bogs rcv packets */
+	unsigned long n_rcv0;
+	unsigned long n_rcv1;
+	unsigned long n_rcv2;
+	unsigned long n_rcvx;
+	unsigned long found_repost_rcvbuf_cnt;
+	unsigned long repost_found_skb_cnt;
+	unsigned long n_repost_deficit;
+	unsigned long bad_rcv_buf;
+	unsigned long n_rcv_packets_not_accepted;
 
 	int queuefullmsg_logged;
 	struct chanstat chstat;
@@ -142,12 +196,11 @@ struct visornic_devdata {
 };
 
 /* Returns next non-zero index on success or 0 on failure (i.e. out of room). */
-static u16
-add_physinfo_entries(u64 inp_pfn, u16 inp_off, u32 inp_len, u16 index,
-		     u16 max_pi_arr_entries, struct phys_info pi_arr[])
+static u16 add_physinfo_entries(u64 inp_pfn, u16 inp_off, u16 inp_len,
+				u16 index, u16 max_pi_arr_entries,
+				struct phys_info pi_arr[])
 {
-	u32 len;
-	u16 i, firstlen;
+	u16 i, len, firstlen;
 
 	firstlen = PI_PAGE_SIZE - inp_off;
 	if (inp_len <= firstlen) {
@@ -171,29 +224,27 @@ add_physinfo_entries(u64 inp_pfn, u16 inp_off, u32 inp_len, u16 index,
 			pi_arr[index].pi_len = firstlen;
 		} else {
 			pi_arr[index + i].pi_off = 0;
-			pi_arr[index + i].pi_len =
-			    (u16)MINNUM(len, (u32)PI_PAGE_SIZE);
+			pi_arr[index + i].pi_len = min_t(u16, len,
+							 PI_PAGE_SIZE);
 		}
 	}
 	return index + i;
 }
 
-/*
- *	visor_copy_fragsinfo_from_skb(
- *	@skb_in: skbuff that we are pulling the frags from
- *	@firstfraglen: length of first fragment in skb
- *	@frags_max: max len of frags array
- *	@frags: frags array filled in on output
+/* visor_copy_fragsinfo_from_skb - copy fragment list in the SKB to a phys_info
+ *				   array that the IOPART understands
+ * @skb:	  Skbuff that we are pulling the frags from.
+ * @firstfraglen: Length of first fragment in skb.
+ * @frags_max:	  Max len of frags array.
+ * @frags:	  Frags array filled in on output.
  *
- *	Copy the fragment list in the SKB to a phys_info
- *	array that the IOPART understands.
- *	Return value indicates number of entries filled in frags
- *	Negative values indicate an error.
+ * Return: Positive integer indicating number of entries filled in frags on
+ *         success, negative integer on error.
  */
-static int
-visor_copy_fragsinfo_from_skb(struct sk_buff *skb, unsigned int firstfraglen,
-			      unsigned int frags_max,
-			      struct phys_info frags[])
+static int visor_copy_fragsinfo_from_skb(struct sk_buff *skb,
+					 unsigned int firstfraglen,
+					 unsigned int frags_max,
+					 struct phys_info frags[])
 {
 	unsigned int count = 0, frag, size, offset = 0, numfrags;
 	unsigned int total_count;
@@ -239,11 +290,10 @@ visor_copy_fragsinfo_from_skb(struct sk_buff *skb, unsigned int firstfraglen,
 
 		for (frag = 0; frag < numfrags; frag++) {
 			count = add_physinfo_entries(page_to_pfn(
-				skb_frag_page(&skb_shinfo(skb)->frags[frag])),
-					      skb_shinfo(skb)->frags[frag].
-					      page_offset,
-					      skb_shinfo(skb)->frags[frag].
-					      size, count, frags_max, frags);
+				  skb_frag_page(&skb_shinfo(skb)->frags[frag])),
+				  skb_shinfo(skb)->frags[frag].page_offset,
+				  skb_shinfo(skb)->frags[frag].size, count,
+				  frags_max, frags);
 			/* add_physinfo_entries only returns
 			 * zero if the frags array is out of room
 			 * That should never happen because we
@@ -287,21 +337,15 @@ static const struct file_operations debugfs_enable_ints_fops = {
 	.write = enable_ints_write,
 };
 
-/*
- *	visornic_serverdown_complete - IOPART went down, pause device
- *	@work: Work queue it was scheduled on
+/* visornic_serverdown_complete - pause device following IOPART going down
+ * @devdata: Device managed by IOPART.
  *
- *	The IO partition has gone down and we need to do some cleanup
- *	for when it comes back. Treat the IO partition as the link
- *	being down.
- *	Returns void.
+ * The IO partition has gone down, and we need to do some cleanup for when it
+ * comes back. Treat the IO partition as the link being down.
  */
-static void
-visornic_serverdown_complete(struct visornic_devdata *devdata)
+static void visornic_serverdown_complete(struct visornic_devdata *devdata)
 {
-	struct net_device *netdev;
-
-	netdev = devdata->netdev;
+	struct net_device *netdev = devdata->netdev;
 
 	/* Stop polling for interrupts */
 	del_timer_sync(&devdata->irq_poll_timer);
@@ -322,17 +366,17 @@ visornic_serverdown_complete(struct visornic_devdata *devdata)
 	devdata->server_down_complete_func = NULL;
 }
 
-/*
- *	visornic_serverdown - Command has notified us that IOPART is down
- *	@devdata: device that is being managed by IOPART
+/* visornic_serverdown - Command has notified us that IOPART is down
+ * @devdata:	   Device managed by IOPART.
+ * @complete_func: Function to call when finished.
+ *
+ * Schedule the work needed to handle the server down request. Make sure we
+ * haven't already handled the server change state event.
  *
- *	Schedule the work needed to handle the server down request. Make
- *	sure we haven't already handled the server change state event.
- *	Returns 0 if we scheduled the work, -EINVAL on error.
+ * Return: 0 if we scheduled the work, negative integer on error.
  */
-static int
-visornic_serverdown(struct visornic_devdata *devdata,
-		    visorbus_state_complete_func complete_func)
+static int visornic_serverdown(struct visornic_devdata *devdata,
+			       visorbus_state_complete_func complete_func)
 {
 	unsigned long flags;
 	int err;
@@ -369,16 +413,15 @@ err_unlock:
 	return err;
 }
 
-/*
- *	alloc_rcv_buf	- alloc rcv buffer to be given to the IO Partition.
- *	@netdev: network adapter the rcv bufs are attached too.
+/* alloc_rcv_buf - alloc rcv buffer to be given to the IO Partition
+ * @netdev: Network adapter the rcv bufs are attached too.
  *
- *	Create an sk_buff (rcv_buf) that will be passed to the IO Partition
- *	so that it can write rcv data into our memory space.
- *	Return pointer to sk_buff
+ * Create an sk_buff (rcv_buf) that will be passed to the IO Partition
+ * so that it can write rcv data into our memory space.
+ *
+ * Return: Pointer to sk_buff.
  */
-static struct sk_buff *
-alloc_rcv_buf(struct net_device *netdev)
+static struct sk_buff *alloc_rcv_buf(struct net_device *netdev)
 {
 	struct sk_buff *skb;
 
@@ -400,18 +443,15 @@ alloc_rcv_buf(struct net_device *netdev)
 	return skb;
 }
 
-/*
- *	post_skb	- post a skb to the IO Partition.
- *	@cmdrsp: cmdrsp packet to be send to the IO Partition
- *	@devdata: visornic_devdata to post the skb too
- *	@skb: skb to give to the IO partition
+/* post_skb - post a skb to the IO Partition
+ * @cmdrsp:  Cmdrsp packet to be send to the IO Partition.
+ * @devdata: visornic_devdata to post the skb to.
+ * @skb:     Skb to give to the IO partition.
  *
- *	Send the skb to the IO Partition.
- *	Returns 0 or error
+ * Return: 0 on success, negative integer on error.
  */
-static int
-post_skb(struct uiscmdrsp *cmdrsp,
-	 struct visornic_devdata *devdata, struct sk_buff *skb)
+static int post_skb(struct uiscmdrsp *cmdrsp, struct visornic_devdata *devdata,
+		    struct sk_buff *skb)
 {
 	int err;
 
@@ -437,23 +477,20 @@ post_skb(struct uiscmdrsp *cmdrsp,
 
 	atomic_inc(&devdata->num_rcvbuf_in_iovm);
 	devdata->chstat.sent_post++;
-
 	return 0;
 }
 
-/*
- *	send_enbdis	- send NET_RCV_ENBDIS to IO Partition
- *	@netdev: netdevice we are enable/disable, used as context
- *		 return value
- *	@state: enable = 1/disable = 0
- *	@devdata: visornic device we are enabling/disabling
+/* send_enbdis - Send NET_RCV_ENBDIS to IO Partition
+ * @netdev:  Netdevice we are enabling/disabling, used as context return value.
+ * @state:   Enable = 1/disable = 0.
+ * @devdata: Visornic device we are enabling/disabling.
+ *
+ * Send the enable/disable message to the IO Partition.
  *
- *	Send the enable/disable message to the IO Partition.
- *	Returns 0 or error
+ * Return: 0 on success, negative integer on error.
  */
-static int
-send_enbdis(struct net_device *netdev, int state,
-	    struct visornic_devdata *devdata)
+static int send_enbdis(struct net_device *netdev, int state,
+		       struct visornic_devdata *devdata)
 {
 	int err;
 
@@ -470,19 +507,17 @@ send_enbdis(struct net_device *netdev, int state,
 	return 0;
 }
 
-/*
- *	visornic_disable_with_timeout - Disable network adapter
- *	@netdev: netdevice to disable
- *	@timeout: timeout to wait for disable
+/* visornic_disable_with_timeout - disable network adapter
+ * @netdev:  netdevice to disable.
+ * @timeout: Timeout to wait for disable.
  *
- *	Disable the network adapter and inform the IO Partition that we
- *	are disabled, reclaim memory from rcv bufs.
- *	Returns 0 on success, negative for failure of IO Partition
- *	responding.
+ * Disable the network adapter and inform the IO Partition that we are disabled.
+ * Reclaim memory from rcv bufs.
  *
+ * Return: 0 on success, negative integer on failure of IO Partition responding.
  */
-static int
-visornic_disable_with_timeout(struct net_device *netdev, const int timeout)
+static int visornic_disable_with_timeout(struct net_device *netdev,
+					 const int timeout)
 {
 	struct visornic_devdata *devdata = netdev_priv(netdev);
 	int i;
@@ -493,7 +528,8 @@ visornic_disable_with_timeout(struct net_device *netdev, const int timeout)
 	/* send a msg telling the other end we are stopping incoming pkts */
 	spin_lock_irqsave(&devdata->priv_lock, flags);
 	devdata->enabled = 0;
-	devdata->enab_dis_acked = 0; /* must wait for ack */
+	/* must wait for ack */
+	devdata->enab_dis_acked = 0;
 	spin_unlock_irqrestore(&devdata->priv_lock, flags);
 
 	/* send disable and wait for ack -- don't hold lock when sending
@@ -560,16 +596,16 @@ visornic_disable_with_timeout(struct net_device *netdev, const int timeout)
 	return 0;
 }
 
-/*
- *	init_rcv_bufs  -- initialize receive bufs and send them to the IO Part
- *	@netdev: struct netdevice
- *	@devdata: visornic_devdata
+/* init_rcv_bufs - initialize receive buffs and send them to the IO Partition
+ * @netdev:  struct netdevice.
+ * @devdata: visornic_devdata.
+ *
+ * Allocate rcv buffers and post them to the IO Partition.
  *
- *	Allocate rcv buffers and post them to the IO Partition.
- *	Return 0 for success, and negative for failure.
+ * Return: 0 on success, negative integer on failure.
  */
-static int
-init_rcv_bufs(struct net_device *netdev, struct visornic_devdata *devdata)
+static int init_rcv_bufs(struct net_device *netdev,
+			 struct visornic_devdata *devdata)
 {
 	int i, j, count, err;
 
@@ -578,10 +614,12 @@ init_rcv_bufs(struct net_device *netdev, struct visornic_devdata *devdata)
 	 */
 	for (i = 0; i < devdata->num_rcv_bufs; i++) {
 		devdata->rcvbuf[i] = alloc_rcv_buf(netdev);
+		/* if we failed to allocate one let us stop */
 		if (!devdata->rcvbuf[i])
-			break; /* if we failed to allocate one let us stop */
+			break;
 	}
-	if (i == 0) /* couldn't even allocate one -- bail out */
+	/* couldn't even allocate one -- bail out */
+	if (i == 0)
 		return -ENOMEM;
 	count = i;
 
@@ -624,17 +662,17 @@ init_rcv_bufs(struct net_device *netdev, struct visornic_devdata *devdata)
 	return 0;
 }
 
-/*
- *	visornic_enable_with_timeout	- send enable to IO Part
- *	@netdev: struct net_device
- *	@timeout: Time to wait for the ACK from the enable
+/* visornic_enable_with_timeout	- send enable to IO Partition
+ * @netdev:  struct net_device.
+ * @timeout: Time to wait for the ACK from the enable.
  *
- *	Sends enable to IOVM, inits, and posts receive buffers to IOVM
- *	timeout is defined in msecs (timeout of 0 specifies infinite wait)
- *	Return 0 for success, negative for failure.
+ * Sends enable to IOVM and inits, and posts receive buffers to IOVM. Timeout is
+ * defined in msecs (timeout of 0 specifies infinite wait).
+ *
+ * Return: 0 on success, negative integer on failure.
  */
-static int
-visornic_enable_with_timeout(struct net_device *netdev, const int timeout)
+static int visornic_enable_with_timeout(struct net_device *netdev,
+					const int timeout)
 {
 	int err = 0;
 	struct visornic_devdata *devdata = netdev_priv(netdev);
@@ -695,20 +733,17 @@ visornic_enable_with_timeout(struct net_device *netdev, const int timeout)
 	}
 
 	netif_start_queue(netdev);
-
 	return 0;
 }
 
-/*
- *	visornic_timeout_reset	- handle xmit timeout resets
- *	@work	work item that scheduled the work
+/* visornic_timeout_reset - handle xmit timeout resets
+ * @work: Work item that scheduled the work.
  *
- *	Transmit Timeouts are typically handled by resetting the
- *	device for our virtual NIC we will send a Disable and Enable
- *	to the IOVM. If it doesn't respond we will trigger a serverdown.
+ * Transmit timeouts are typically handled by resetting the device for our
+ * virtual NIC; we will send a disable and enable to the IOVM. If it doesn't
+ * respond, we will trigger a serverdown.
  */
-static void
-visornic_timeout_reset(struct work_struct *work)
+static void visornic_timeout_reset(struct work_struct *work)
 {
 	struct visornic_devdata *devdata;
 	struct net_device *netdev;
@@ -742,41 +777,36 @@ call_serverdown:
 	rtnl_unlock();
 }
 
-/*
- *	visornic_open - Enable the visornic device and mark the queue started
- *	@netdev: netdevice to start
+/* visornic_open - enable the visornic device and mark the queue started
+ * @netdev: netdevice to start.
  *
- *      Enable the device and start the transmit queue.
- *      Return 0 for success
+ * Enable the device and start the transmit queue.
+ *
+ * Return: 0 on success.
  */
-static int
-visornic_open(struct net_device *netdev)
+static int visornic_open(struct net_device *netdev)
 {
 	visornic_enable_with_timeout(netdev, VISORNIC_INFINITE_RSP_WAIT);
-
 	return 0;
 }
 
-/*
- *	visornic_close - Disables the visornic device and stops the queues
- *	@netdev: netdevice to start
+/* visornic_close - disables the visornic device and stops the queues
+ * @netdev: netdevice to stop.
  *
- *      Disable the device and stop the transmit queue.
- *      Return 0 for success
+ * Disable the device and stop the transmit queue.
+ *
+ * Return 0 on success.
  */
-static int
-visornic_close(struct net_device *netdev)
+static int visornic_close(struct net_device *netdev)
 {
 	visornic_disable_with_timeout(netdev, VISORNIC_INFINITE_RSP_WAIT);
-
 	return 0;
 }
 
-/*
- *	devdata_xmits_outstanding - compute outstanding xmits
- *	@devdata: visornic_devdata for device
+/* devdata_xmits_outstanding - compute outstanding xmits
+ * @devdata: visornic_devdata for device
  *
- *	Return value is the number of outstanding xmits.
+ * Return: Long integer representing the number of outstanding xmits.
  */
 static unsigned long devdata_xmits_outstanding(struct visornic_devdata *devdata)
 {
@@ -787,14 +817,13 @@ static unsigned long devdata_xmits_outstanding(struct visornic_devdata *devdata)
 		+ devdata->chstat.sent_xmit + 1);
 }
 
-/*
- *	vnic_hit_high_watermark
- *	@devdata: indicates visornic device we are checking
- *	@high_watermark: max num of unacked xmits we will tolerate,
- *                       before we will start throttling
+/* vnic_hit_high_watermark
+ * @devdata:	    Indicates visornic device we are checking.
+ * @high_watermark: Max num of unacked xmits we will tolerate before we will
+ *		    start throttling.
  *
- *      Returns true iff the number of unacked xmits sent to
- *      the IO partition is >= high_watermark.
+ * Return: True iff the number of unacked xmits sent to the IO Partition is >=
+ *	   high_watermark. False otherwise.
  */
 static bool vnic_hit_high_watermark(struct visornic_devdata *devdata,
 				    ulong high_watermark)
@@ -802,15 +831,13 @@ static bool vnic_hit_high_watermark(struct visornic_devdata *devdata,
 	return (devdata_xmits_outstanding(devdata) >= high_watermark);
 }
 
-/*
- *	vnic_hit_low_watermark
- *	@devdata: indicates visornic device we are checking
- *	@low_watermark: we will wait until the num of unacked xmits
- *                      drops to this value or lower before we start
- *                      transmitting again
+/* vnic_hit_low_watermark
+ * @devdata:	   Indicates visornic device we are checking.
+ * @low_watermark: We will wait until the num of unacked xmits drops to this
+ *		   value or lower before we start transmitting again.
  *
- *      Returns true iff the number of unacked xmits sent to
- *      the IO partition is <= low_watermark.
+ * Return: True iff the number of unacked xmits sent to the IO Partition is <=
+ *	   low_watermark.
  */
 static bool vnic_hit_low_watermark(struct visornic_devdata *devdata,
 				   ulong low_watermark)
@@ -818,20 +845,18 @@ static bool vnic_hit_low_watermark(struct visornic_devdata *devdata,
 	return (devdata_xmits_outstanding(devdata) <= low_watermark);
 }
 
-/*
- *	visornic_xmit - send a packet to the IO Partition
- *	@skb: Packet to be sent
- *	@netdev: net device the packet is being sent from
+/* visornic_xmit - send a packet to the IO Partition
+ * @skb:    Packet to be sent.
+ * @netdev: Net device the packet is being sent from.
+ *
+ * Convert the skb to a cmdrsp so the IO Partition can understand it, and send
+ * the XMIT command to the IO Partition for processing. This function is
+ * protected from concurrent calls by a spinlock xmit_lock in the net_device
+ * struct. As soon as the function returns, it can be called again.
  *
- *	Convert the skb to a cmdrsp so the IO Partition can understand it.
- *	Send the XMIT command to the IO Partition for processing. This
- *	function is protected from concurrent calls by a spinlock xmit_lock
- *	in the net_device struct, but as soon as the function returns it
- *	can be called again.
- *	Returns NETDEV_TX_OK.
+ * Return: NETDEV_TX_OK.
  */
-static int
-visornic_xmit(struct sk_buff *skb, struct net_device *netdev)
+static int visornic_xmit(struct sk_buff *skb, struct net_device *netdev)
 {
 	struct visornic_devdata *devdata;
 	int len, firstfraglen, padlen;
@@ -938,6 +963,7 @@ visornic_xmit(struct sk_buff *skb, struct net_device *netdev)
 	 * - everything else will be pass in frags & DMA'ed
 	 */
 	memcpy(cmdrsp->net.xmt.ethhdr, skb->data, ETH_HLEN);
+
 	/* copy frags info - from skb->data we need to only provide access
 	 * beyond eth header
 	 */
@@ -991,46 +1017,39 @@ visornic_xmit(struct sk_buff *skb, struct net_device *netdev)
 	return NETDEV_TX_OK;
 }
 
-/*
- *	visornic_get_stats - returns net_stats of the visornic device
- *	@netdev: netdevice
+/* visornic_get_stats - returns net_stats of the visornic device
+ * @netdev: netdevice.
  *
- *	Returns the net_device_stats for the device
+ * Return: Pointer to the net_device_stats struct for the device.
  */
-static struct net_device_stats *
-visornic_get_stats(struct net_device *netdev)
+static struct net_device_stats *visornic_get_stats(struct net_device *netdev)
 {
 	struct visornic_devdata *devdata = netdev_priv(netdev);
 
 	return &devdata->net_stats;
 }
 
-/*
- *	visornic_change_mtu - changes mtu of device.
- *	@netdev: netdevice
- *	@new_mtu: value of new mtu
+/* visornic_change_mtu - changes mtu of device
+ * @netdev: netdevice.
+ * @new_mtu: Value of new mtu.
+ *
+ * The device's MTU cannot be changed by system; it must be changed via a
+ * CONTROLVM message. All vnics and pnics in a switch have to have the same MTU
+ * for everything to work. Currently not supported.
  *
- *	MTU cannot be changed by system, must be changed via
- *	CONTROLVM message. All vnics and pnics in a switch have
- *	to have the same MTU for everything to work.
- *	Currently not supported.
- *	Returns EINVAL
+ * Return: -EINVAL.
  */
-static int
-visornic_change_mtu(struct net_device *netdev, int new_mtu)
+static int visornic_change_mtu(struct net_device *netdev, int new_mtu)
 {
 	return -EINVAL;
 }
 
-/*
- *	visornic_set_multi - changes mtu of device.
- *	@netdev: netdevice
+/* visornic_set_multi - set visornic device flags
+ * @netdev: netdevice.
  *
- *	Only flag we support currently is IFF_PROMISC
- *	Returns void
+ * The only flag we currently support is IFF_PROMISC.
  */
-static void
-visornic_set_multi(struct net_device *netdev)
+static void visornic_set_multi(struct net_device *netdev)
 {
 	struct uiscmdrsp *cmdrsp;
 	struct visornic_devdata *devdata = netdev_priv(netdev);
@@ -1062,16 +1081,13 @@ out_save_flags:
 	devdata->old_flags = netdev->flags;
 }
 
-/*
- *	visornic_xmit_timeout - request to timeout the xmit
- *	@netdev
+/* visornic_xmit_timeout - request to timeout the xmit
+ * @netdev: netdevice.
  *
- *	Queue the work and return. Make sure we have not already
- *	been informed the IO Partition is gone, if it is gone
- *	we will already timeout the xmits.
+ * Queue the work and return. Make sure we have not already been informed that
+ * the IO Partition is gone; if so, we will have already timed-out the xmits.
  */
-static void
-visornic_xmit_timeout(struct net_device *netdev)
+static void visornic_xmit_timeout(struct net_device *netdev)
 {
 	struct visornic_devdata *devdata = netdev_priv(netdev);
 	unsigned long flags;
@@ -1097,20 +1113,20 @@ visornic_xmit_timeout(struct net_device *netdev)
 	spin_unlock_irqrestore(&devdata->priv_lock, flags);
 }
 
-/*
- *	repost_return	- repost rcv bufs that have come back
- *	@cmdrsp: io channel command struct to post
- *	@devdata: visornic devdata for the device
- *	@skb: skb
- *	@netdev: netdevice
+/* repost_return - repost rcv bufs that have come back
+ * @cmdrsp: IO channel command struct to post.
+ * @devdata: Visornic devdata for the device.
+ * @skb: Socket buffer.
+ * @netdev: netdevice.
+ *
+ * Repost rcv buffers that have been returned to us when we are finished
+ * with them.
  *
- *	Repost rcv buffers that have been returned to us when
- *	we are finished with them.
- *	Returns 0 for success, -1 for error.
+ * Return: 0 for success, negative integer on error.
  */
-static int
-repost_return(struct uiscmdrsp *cmdrsp, struct visornic_devdata *devdata,
-	      struct sk_buff *skb, struct net_device *netdev)
+static int repost_return(struct uiscmdrsp *cmdrsp,
+			 struct visornic_devdata *devdata,
+			 struct sk_buff *skb, struct net_device *netdev)
 {
 	struct net_pkt_rcv copy;
 	int i = 0, cc, numreposted;
@@ -1174,16 +1190,15 @@ repost_return(struct uiscmdrsp *cmdrsp, struct visornic_devdata *devdata,
 	return status;
 }
 
-/*
- *	visornic_rx - Handle receive packets coming back from IO Part
- *	@cmdrsp: Receive packet returned from IO Part
+/* visornic_rx - handle receive packets coming back from IO Partition
+ * @cmdrsp: Receive packet returned from IO Partition.
  *
- *	Got a receive packet back from the IO Part, handle it and send
- *	it up the stack.
- *	Returns 1 iff an skb was received, otherwise 0
+ * Got a receive packet back from the IO Partition; handle it and send it up
+ * the stack.
+
+ * Return: 1 iff an skb was received, otherwise 0.
  */
-static int
-visornic_rx(struct uiscmdrsp *cmdrsp)
+static int visornic_rx(struct uiscmdrsp *cmdrsp)
 {
 	struct visornic_devdata *devdata;
 	struct sk_buff *skb, *prev, *curr;
@@ -1236,7 +1251,8 @@ visornic_rx(struct uiscmdrsp *cmdrsp)
 	 * firstfrag & set data_len to show rest see if we have to chain
 	 * frag_list.
 	 */
-	if (skb->len > RCVPOST_BUF_SIZE) {	/* do PRECAUTIONARY check */
+	/* do PRECAUTIONARY check */
+	if (skb->len > RCVPOST_BUF_SIZE) {
 		if (cmdrsp->net.rcv.numrcvbufs < 2) {
 			if (repost_return(cmdrsp, devdata, skb, netdev) < 0)
 				dev_err(&devdata->netdev->dev,
@@ -1244,23 +1260,24 @@ visornic_rx(struct uiscmdrsp *cmdrsp)
 			return 0;
 		}
 		/* length rcvd is greater than firstfrag in this skb rcv buf  */
-		skb->tail += RCVPOST_BUF_SIZE;	/* amount in skb->data */
-		skb->data_len = skb->len - RCVPOST_BUF_SIZE;	/* amount that
-								 *  will be in
-								 * frag_list
-								 */
+		/* amount in skb->data */
+		skb->tail += RCVPOST_BUF_SIZE;
+		/* amount that will be in frag_list */
+		skb->data_len = skb->len - RCVPOST_BUF_SIZE;
 	} else {
 		/* data fits in this skb - no chaining - do
 		 * PRECAUTIONARY check
 		 */
-		if (cmdrsp->net.rcv.numrcvbufs != 1) {	/* should be 1 */
+		/* should be 1 */
+		if (cmdrsp->net.rcv.numrcvbufs != 1) {
 			if (repost_return(cmdrsp, devdata, skb, netdev) < 0)
 				dev_err(&devdata->netdev->dev,
 					"repost_return failed");
 			return 0;
 		}
 		skb->tail += skb->len;
-		skb->data_len = 0;	/* nothing rcvd in frag_list */
+		/* nothing rcvd in frag_list */
+		skb->data_len = 0;
 	}
 	off = skb_tail_pointer(skb) - skb->data;
 
@@ -1286,7 +1303,8 @@ visornic_rx(struct uiscmdrsp *cmdrsp)
 		     cc < cmdrsp->net.rcv.numrcvbufs; cc++) {
 			curr = (struct sk_buff *)cmdrsp->net.rcv.rcvbuf[cc];
 			curr->next = NULL;
-			if (!prev)	/* start of list- set head */
+			/* start of list- set head */
+			if (!prev)
 				skb_shinfo(skb)->frag_list = curr;
 			else
 				prev->next = curr;
@@ -1314,18 +1332,18 @@ visornic_rx(struct uiscmdrsp *cmdrsp)
 	 * sets up skb->pkt_type & it also PULLS out the eth header
 	 */
 	skb->protocol = eth_type_trans(skb, netdev);
-
 	eth = eth_hdr(skb);
-
 	skb->csum = 0;
 	skb->ip_summed = CHECKSUM_NONE;
 
 	do {
+		/* accept all packets */
 		if (netdev->flags & IFF_PROMISC)
-			break;	/* accept all packets */
+			break;
 		if (skb->pkt_type == PACKET_BROADCAST) {
+			/* accept all broadcast packets */
 			if (netdev->flags & IFF_BROADCAST)
-				break;	/* accept all broadcast packets */
+				break;
 		} else if (skb->pkt_type == PACKET_MULTICAST) {
 			if ((netdev->flags & IFF_MULTICAST) &&
 			    (netdev_mc_count(netdev))) {
@@ -1367,8 +1385,7 @@ visornic_rx(struct uiscmdrsp *cmdrsp)
 	 */
 
 	skb = NULL;
-	/*
-	 * whether the packet got dropped or handled, the skb is freed by
+	/* whether the packet got dropped or handled, the skb is freed by
 	 * kernel code, so we shouldn't free it. but we should repost a
 	 * new rcv buffer.
 	 */
@@ -1376,29 +1393,25 @@ visornic_rx(struct uiscmdrsp *cmdrsp)
 	return 1;
 }
 
-/*
- *	devdata_initialize	- Initialize devdata structure
- *	@devdata: visornic_devdata structure to initialize
- *	#dev: visorbus_deviced it belongs to
+/* devdata_initialize - initialize devdata structure
+ * @devdata: visornic_devdata structure to initialize.
+ * @dev:     visorbus_device it belongs to.
  *
- *	Setup initial values for the visornic based on channel and default
- *	values.
- *	Returns a pointer to the devdata structure
+ * Setup initial values for the visornic, based on channel and default values.
+ *
+ * Return: A pointer to the devdata structure.
  */
-static struct visornic_devdata *
-devdata_initialize(struct visornic_devdata *devdata, struct visor_device *dev)
+static struct visornic_devdata *devdata_initialize(
+					struct visornic_devdata *devdata,
+					struct visor_device *dev)
 {
 	devdata->dev = dev;
 	devdata->incarnation_id = get_jiffies_64();
 	return devdata;
 }
 
-/*
- *	devdata_release	- Frees up references in devdata
- *	@devdata: struct to clean up
- *
- *	Frees up references in devdata.
- *	Returns void
+/* devdata_release - free up references in devdata
+ * @devdata: Struct to clean up.
  */
 static void devdata_release(struct visornic_devdata *devdata)
 {
@@ -1570,15 +1583,10 @@ static const struct file_operations debugfs_info_fops = {
 	.read = info_debugfs_read,
 };
 
-/*
- *	send_rcv_posts_if_needed
- *	@devdata: visornic device
- *
- *	Send receive buffers to the IO Partition.
- *	Returns void
+/* send_rcv_posts_if_needed - send receive buffers to the IO Partition.
+ * @devdata: Visornic device.
  */
-static int
-send_rcv_posts_if_needed(struct visornic_devdata *devdata)
+static void send_rcv_posts_if_needed(struct visornic_devdata *devdata)
 {
 	int i;
 	struct net_device *netdev;
@@ -1588,7 +1596,7 @@ send_rcv_posts_if_needed(struct visornic_devdata *devdata)
 
 	/* don't do this until vnic is marked ready */
 	if (!(devdata->enabled && devdata->enab_dis_acked))
-		return 0;
+		return;
 
 	netdev = devdata->netdev;
 	rcv_bufs_allocated = 0;
@@ -1617,16 +1625,14 @@ send_rcv_posts_if_needed(struct visornic_devdata *devdata)
 		}
 	}
 	devdata->num_rcv_bufs_could_not_alloc -= rcv_bufs_allocated;
-	return 0;
 }
 
-/*
- *	drain_resp_queue  - drains and ignores all messages from the resp queue
- *	@cmdrsp: io channel command response message
- *	@devdata: visornic device to drain
+/* drain_resp_queue - drains and ignores all messages from the resp queue
+ * @cmdrsp:  IO channel command response message.
+ * @devdata: Visornic device to drain.
  */
-static void
-drain_resp_queue(struct uiscmdrsp *cmdrsp, struct visornic_devdata *devdata)
+static void drain_resp_queue(struct uiscmdrsp *cmdrsp,
+			     struct visornic_devdata *devdata)
 {
 	while (!visorchannel_signalremove(devdata->dev->visorchannel,
 					  IOCHAN_FROM_IOPART,
@@ -1634,30 +1640,31 @@ drain_resp_queue(struct uiscmdrsp *cmdrsp, struct visornic_devdata *devdata)
 		;
 }
 
-/*
- *	service_resp_queue	- drains the response queue
- *	@cmdrsp: io channel command response message
- *	@devdata: visornic device to drain
+/* service_resp_queue - drain the response queue
+ * @cmdrsp:  IO channel command response message.
+ * @devdata: Visornic device to drain.
+ * @rx_work_done:
+ * @budget:
  *
- *	Drain the response queue of any responses from the IO partition.
- *	Process the responses as we get them.
- *	Returns when response queue is empty or when the thread stops.
+ * Drain the response queue of any responses from the IO Partition. Process the
+ * responses as we get them.
  */
-static void
-service_resp_queue(struct uiscmdrsp *cmdrsp, struct visornic_devdata *devdata,
-		   int *rx_work_done, int budget)
+static void service_resp_queue(struct uiscmdrsp *cmdrsp,
+			       struct visornic_devdata *devdata,
+			       int *rx_work_done, int budget)
 {
 	unsigned long flags;
 	struct net_device *netdev;
 
 	while (*rx_work_done < budget) {
-	/* TODO: CLIENT ACQUIRE -- Don't really need this at the
-	 * moment
-	 */
+		/* TODO: CLIENT ACQUIRE -- Don't really need this at the
+		 * moment
+		 */
+		/* queue empty */
 		if (visorchannel_signalremove(devdata->dev->visorchannel,
 					      IOCHAN_FROM_IOPART,
 					      cmdrsp))
-			break; /* queue empty */
+			break;
 
 		switch (cmdrsp->net.type) {
 		case NET_RCV:
@@ -1740,12 +1747,8 @@ static int visornic_poll(struct napi_struct *napi, int budget)
 							struct visornic_devdata,
 							napi);
 	int rx_count = 0;
-	int err;
-
-	err = send_rcv_posts_if_needed(devdata);
-	if (err)
-		return err;
 
+	send_rcv_posts_if_needed(devdata);
 	service_resp_queue(devdata->cmdrsp, devdata, &rx_count, budget);
 
 	/* If there aren't any more packets to receive stop the poll */
@@ -1755,16 +1758,13 @@ static int visornic_poll(struct napi_struct *napi, int budget)
 	return rx_count;
 }
 
-/*
- *	poll_for_irq	- Checks the status of the response queue.
- *	@v: void pointer to the visronic devdata
+/* poll_for_irq	- checks the status of the response queue
+ * @v: Void pointer to the visronic devdata struct.
  *
- *	Main function of the vnic_incoming thread. Periodically check the
- *	response queue and drain it if needed.
- *	Returns when thread has stopped.
+ * Main function of the vnic_incoming thread. Periodically check the response
+ * queue and drain it if needed.
  */
-static void
-poll_for_irq(unsigned long v)
+static void poll_for_irq(unsigned long v)
 {
 	struct visornic_devdata *devdata = (struct visornic_devdata *)v;
 
@@ -1778,13 +1778,13 @@ poll_for_irq(unsigned long v)
 	mod_timer(&devdata->irq_poll_timer, msecs_to_jiffies(2));
 }
 
-/*
- *	visornic_probe	- probe function for visornic devices
- *	@dev: The visor device discovered
+/* visornic_probe - probe function for visornic devices
+ * @dev: The visor device discovered.
+ *
+ * Called when visorbus discovers a visornic device on its bus. It creates a new
+ * visornic ethernet adapter.
  *
- *	Called when visorbus discovers a visornic device on its
- *	bus. It creates a new visornic ethernet adapter.
- *	Returns 0 or negative for error.
+ * Return: 0 on success, or negative integer on error.
  */
 static int visornic_probe(struct visor_device *dev)
 {
@@ -1831,7 +1831,8 @@ static int visornic_probe(struct visor_device *dev)
 	dev_set_drvdata(&dev->device, devdata);
 	init_waitqueue_head(&devdata->rsp_queue);
 	spin_lock_init(&devdata->priv_lock);
-	devdata->enabled = 0; /* not yet */
+	/* not yet */
+	devdata->enabled = 0;
 	atomic_set(&devdata->usage, 1);
 
 	/* Setup rcv bufs */
@@ -1852,9 +1853,10 @@ static int visornic_probe(struct visor_device *dev)
 		goto cleanup_netdev;
 	}
 
-	/* set the net_xmit outstanding threshold */
-	/* always leave two slots open but you should have 3 at a minimum */
-	/* note that max_outstanding_net_xmits must be > 0 */
+	/* set the net_xmit outstanding threshold
+	 * always leave two slots open but you should have 3 at a minimum
+	 * note that max_outstanding_net_xmits must be > 0
+	 */
 	devdata->max_outstanding_net_xmits =
 		max_t(unsigned long, 3, ((devdata->num_rcv_bufs / 3) - 2));
 	devdata->upper_threshold_net_xmits =
@@ -1972,28 +1974,25 @@ cleanup_netdev:
 	return err;
 }
 
-/*
- *	host_side_disappeared	- IO part is gone.
- *	@devdata: device object
+/* host_side_disappeared - IO Partition is gone
+ * @devdata: Device object.
  *
- *	IO partition servicing this device is gone, do cleanup
- *	Returns void.
+ * IO partition servicing this device is gone; do cleanup.
  */
 static void host_side_disappeared(struct visornic_devdata *devdata)
 {
 	unsigned long flags;
 
 	spin_lock_irqsave(&devdata->priv_lock, flags);
-	devdata->dev = NULL;   /* indicate device destroyed */
+	/* indicate device destroyed */
+	devdata->dev = NULL;
 	spin_unlock_irqrestore(&devdata->priv_lock, flags);
 }
 
-/*
- *	visornic_remove		- Called when visornic dev goes away
- *	@dev: visornic device that is being removed
+/* visornic_remove - called when visornic dev goes away
+ * @dev: Visornic device that is being removed.
  *
- *	Called when DEVICE_DESTROY gets called to remove device.
- *	Returns void
+ * Called when DEVICE_DESTROY gets called to remove device.
  */
 static void visornic_remove(struct visor_device *dev)
 {
@@ -2023,8 +2022,8 @@ static void visornic_remove(struct visor_device *dev)
 	cancel_work_sync(&devdata->timeout_reset);
 
 	debugfs_remove_recursive(devdata->eth_debugfs_dir);
-
-	unregister_netdev(netdev);  /* this will call visornic_close() */
+	/* this will call visornic_close() */
+	unregister_netdev(netdev);
 
 	del_timer_sync(&devdata->irq_poll_timer);
 	netif_napi_del(&devdata->napi);
@@ -2035,18 +2034,17 @@ static void visornic_remove(struct visor_device *dev)
 	free_netdev(netdev);
 }
 
-/*
- *	visornic_pause		- Called when IO Part disappears
- *	@dev: visornic device that is being serviced
- *	@complete_func: call when finished.
+/* visornic_pause - called when IO Part disappears
+ * @dev:	   Visornic device that is being serviced.
+ * @complete_func: Call when finished.
+ *
+ * Called when the IO Partition has gone down. Need to free up resources and
+ * wait for IO partition to come back. Mark link as down and don't attempt any
+ * DMA. When we have freed memory, call the complete_func so that Command knows
+ * we are done. If we don't call complete_func, the IO Partition will never
+ * come back.
  *
- *	Called when the IO Partition has gone down. Need to free
- *	up resources and wait for IO partition to come back. Mark
- *	link as down and don't attempt any DMA. When we have freed
- *	memory call the complete_func so that Command knows we are
- *	done. If we don't call complete_func, IO part will never
- *	come back.
- *	Returns 0 for success.
+ * Return: 0 on success.
  */
 static int visornic_pause(struct visor_device *dev,
 			  visorbus_state_complete_func complete_func)
@@ -2057,15 +2055,14 @@ static int visornic_pause(struct visor_device *dev,
 	return 0;
 }
 
-/*
- *	visornic_resume		- Called when IO part has recovered
- *	@dev: visornic device that is being serviced
- *	@compelte_func: call when finished
+/* visornic_resume - called when IO Partition has recovered
+ * @dev:	   Visornic device that is being serviced.
+ * @compelte_func: Call when finished.
+ *
+ * Called when the IO partition has recovered. Re-establish connection to the IO
+ * Partition and set the link up. Okay to do DMA again.
  *
- *	Called when the IO partition has recovered. Reestablish
- *	connection to the IO part and set the link up. Okay to do
- *	DMA again.
- *	Returns 0 for success.
+ * Returns 0 for success, negative integer on error.
  */
 static int visornic_resume(struct visor_device *dev,
 			   visorbus_state_complete_func complete_func)
@@ -2127,12 +2124,12 @@ static struct visor_driver visornic_driver = {
 	.channel_interrupt = NULL,
 };
 
-/*
- *	visornic_init	- Init function
+/* visornic_init - init function
  *
- *	Init function for the visornic driver. Do initial driver setup
- *	and wait for devices.
- *	Returns 0 for success, negative for error.
+ * Init function for the visornic driver. Do initial driver setup and wait
+ * for devices.
+ *
+ * Return: 0 on success, negative integer on error.
  */
 static int visornic_init(void)
 {
@@ -2160,19 +2157,16 @@ static int visornic_init(void)
 
 cleanup_debugfs:
 	debugfs_remove_recursive(visornic_debugfs_dir);
-
 	return err;
 }
 
-/*
- *	visornic_cleanup	- driver exit routine
+/* visornic_cleanup - driver exit routine
  *
- *	Unregister driver from the bus and free up memory.
+ * Unregister driver from the bus and free up memory.
  */
 static void visornic_cleanup(void)
 {
 	visorbus_unregister_visor_driver(&visornic_driver);
-
 	debugfs_remove_recursive(visornic_debugfs_dir);
 }
 
diff --git a/drivers/staging/vboxvideo/Kconfig b/drivers/staging/vboxvideo/Kconfig
index a52746f9a670..1f4182e2e980 100644
--- a/drivers/staging/vboxvideo/Kconfig
+++ b/drivers/staging/vboxvideo/Kconfig
@@ -2,11 +2,14 @@ config DRM_VBOXVIDEO
 	tristate "Virtual Box Graphics Card"
 	depends on DRM && X86 && PCI
 	select DRM_KMS_HELPER
+	select DRM_TTM
+	select GENERIC_ALLOCATOR
 	help
 	  This is a KMS driver for the virtual Graphics Card used in
 	  Virtual Box virtual machines.
 
-	  Although it is possible to builtin this module, it is advised
-	  to build this driver as a module, so that it can be updated
-	  independently of the kernel. Select M to built this driver as a
-	  module and add support for these devices via drm/kms interfaces.
+	  Although it is possible to build this driver built-in to the
+	  kernel, it is advised to build it as a module, so that it can
+	  be updated independently of the kernel. Select M to build this
+	  driver as a module and add support for these devices via drm/kms
+	  interfaces.
diff --git a/drivers/staging/vboxvideo/TODO b/drivers/staging/vboxvideo/TODO
index ce764309b079..bd381d861ab3 100644
--- a/drivers/staging/vboxvideo/TODO
+++ b/drivers/staging/vboxvideo/TODO
@@ -5,5 +5,5 @@ TODO:
 -Extend this TODO with the results of that review
 
 Please send any patches to Greg Kroah-Hartman <gregkh@linuxfoundation.org>,
-Hans de Goede <hdegoede@redhat.com> and
-Michael Thayer <michael.thayer@oracle.com>.
+Hans de Goede <hdegoede@redhat.com>, Michael Thayer <michael.thayer@oracle.com>
+and dri-devel@lists.freedesktop.org .
diff --git a/drivers/staging/vboxvideo/vbox_drv.c b/drivers/staging/vboxvideo/vbox_drv.c
index 92ae1560a16d..e18642e5027e 100644
--- a/drivers/staging/vboxvideo/vbox_drv.c
+++ b/drivers/staging/vboxvideo/vbox_drv.c
@@ -36,7 +36,7 @@
 
 #include "vbox_drv.h"
 
-int vbox_modeset = -1;
+static int vbox_modeset = -1;
 
 MODULE_PARM_DESC(modeset, "Disable/Enable modesetting");
 module_param_named(modeset, vbox_modeset, int, 0400);
@@ -232,7 +232,6 @@ static struct drm_driver driver = {
 	.lastclose = vbox_driver_lastclose,
 	.master_set = vbox_master_set,
 	.master_drop = vbox_master_drop,
-	.set_busid = drm_pci_set_busid,
 
 	.fops = &vbox_fops,
 	.irq_handler = vbox_irq_handler,
@@ -270,12 +269,12 @@ static int __init vbox_init(void)
 	if (vbox_modeset == 0)
 		return -EINVAL;
 
-	return drm_pci_init(&driver, &vbox_pci_driver);
+	return pci_register_driver(&vbox_pci_driver);
 }
 
 static void __exit vbox_exit(void)
 {
-	drm_pci_exit(&driver, &vbox_pci_driver);
+	pci_unregister_driver(&vbox_pci_driver);
 }
 
 module_init(vbox_init);
diff --git a/drivers/staging/vboxvideo/vbox_fb.c b/drivers/staging/vboxvideo/vbox_fb.c
index 35f6d9f8c203..8aed248db6e2 100644
--- a/drivers/staging/vboxvideo/vbox_fb.c
+++ b/drivers/staging/vboxvideo/vbox_fb.c
@@ -45,144 +45,20 @@
 #include "vbox_drv.h"
 #include "vboxvideo.h"
 
-#define VBOX_DIRTY_DELAY (HZ / 30)
-/**
- * Tell the host about dirty rectangles to update.
- */
-static void vbox_dirty_update(struct vbox_fbdev *fbdev,
-			      int x, int y, int width, int height)
-{
-	struct drm_gem_object *obj;
-	struct vbox_bo *bo;
-	int ret = -EBUSY;
-	bool store_for_later = false;
-	int x2, y2;
-	unsigned long flags;
-	struct drm_clip_rect rect;
-
-	obj = fbdev->afb.obj;
-	bo = gem_to_vbox_bo(obj);
-
-	/*
-	 * try and reserve the BO, if we fail with busy
-	 * then the BO is being moved and we should
-	 * store up the damage until later.
-	 */
-	if (drm_can_sleep())
-		ret = vbox_bo_reserve(bo, true);
-	if (ret) {
-		if (ret != -EBUSY)
-			return;
-
-		store_for_later = true;
-	}
-
-	x2 = x + width - 1;
-	y2 = y + height - 1;
-	spin_lock_irqsave(&fbdev->dirty_lock, flags);
-
-	if (fbdev->y1 < y)
-		y = fbdev->y1;
-	if (fbdev->y2 > y2)
-		y2 = fbdev->y2;
-	if (fbdev->x1 < x)
-		x = fbdev->x1;
-	if (fbdev->x2 > x2)
-		x2 = fbdev->x2;
-
-	if (store_for_later) {
-		fbdev->x1 = x;
-		fbdev->x2 = x2;
-		fbdev->y1 = y;
-		fbdev->y2 = y2;
-		spin_unlock_irqrestore(&fbdev->dirty_lock, flags);
-		return;
-	}
-
-	fbdev->x1 = INT_MAX;
-	fbdev->y1 = INT_MAX;
-	fbdev->x2 = 0;
-	fbdev->y2 = 0;
-
-	spin_unlock_irqrestore(&fbdev->dirty_lock, flags);
-
-	/*
-	 * Not sure why the original code subtracted 1 here, but I will keep
-	 * it that way to avoid unnecessary differences.
-	 */
-	rect.x1 = x;
-	rect.x2 = x2 + 1;
-	rect.y1 = y;
-	rect.y2 = y2 + 1;
-	vbox_framebuffer_dirty_rectangles(&fbdev->afb.base, &rect, 1);
-
-	vbox_bo_unreserve(bo);
-}
-
-#ifdef CONFIG_FB_DEFERRED_IO
-static void vbox_deferred_io(struct fb_info *info, struct list_head *pagelist)
-{
-	struct vbox_fbdev *fbdev = info->par;
-	unsigned long start, end, min, max;
-	struct page *page;
-	int y1, y2;
-
-	min = ULONG_MAX;
-	max = 0;
-	list_for_each_entry(page, pagelist, lru) {
-		start = page->index << PAGE_SHIFT;
-		end = start + PAGE_SIZE - 1;
-		min = min(min, start);
-		max = max(max, end);
-	}
-
-	if (min < max) {
-		y1 = min / info->fix.line_length;
-		y2 = (max / info->fix.line_length) + 1;
-		DRM_INFO("%s: Calling dirty update: 0, %d, %d, %d\n",
-			 __func__, y1, info->var.xres, y2 - y1 - 1);
-		vbox_dirty_update(fbdev, 0, y1, info->var.xres, y2 - y1 - 1);
-	}
-}
-
+#ifdef CONFIG_DRM_KMS_FB_HELPER
 static struct fb_deferred_io vbox_defio = {
-	.delay = VBOX_DIRTY_DELAY,
-	.deferred_io = vbox_deferred_io,
+	.delay = HZ / 30,
+	.deferred_io = drm_fb_helper_deferred_io,
 };
 #endif
 
-static void vbox_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
-{
-	struct vbox_fbdev *fbdev = info->par;
-
-	sys_fillrect(info, rect);
-	vbox_dirty_update(fbdev, rect->dx, rect->dy, rect->width, rect->height);
-}
-
-static void vbox_copyarea(struct fb_info *info, const struct fb_copyarea *area)
-{
-	struct vbox_fbdev *fbdev = info->par;
-
-	sys_copyarea(info, area);
-	vbox_dirty_update(fbdev, area->dx, area->dy, area->width, area->height);
-}
-
-static void vbox_imageblit(struct fb_info *info, const struct fb_image *image)
-{
-	struct vbox_fbdev *fbdev = info->par;
-
-	sys_imageblit(info, image);
-	vbox_dirty_update(fbdev, image->dx, image->dy, image->width,
-			  image->height);
-}
-
 static struct fb_ops vboxfb_ops = {
 	.owner = THIS_MODULE,
 	.fb_check_var = drm_fb_helper_check_var,
 	.fb_set_par = drm_fb_helper_set_par,
-	.fb_fillrect = vbox_fillrect,
-	.fb_copyarea = vbox_copyarea,
-	.fb_imageblit = vbox_imageblit,
+	.fb_fillrect = drm_fb_helper_sys_fillrect,
+	.fb_copyarea = drm_fb_helper_sys_copyarea,
+	.fb_imageblit = drm_fb_helper_sys_imageblit,
 	.fb_pan_display = drm_fb_helper_pan_display,
 	.fb_blank = drm_fb_helper_blank,
 	.fb_setcmap = drm_fb_helper_setcmap,
@@ -219,7 +95,6 @@ static int vboxfb_create(struct drm_fb_helper *helper,
 	struct DRM_MODE_FB_CMD mode_cmd;
 	struct drm_framebuffer *fb;
 	struct fb_info *info;
-	struct device *device = &dev->pdev->dev;
 	struct drm_gem_object *gobj;
 	struct vbox_bo *bo;
 	int size, ret;
@@ -263,16 +138,16 @@ static int vboxfb_create(struct drm_fb_helper *helper,
 		return ret;
 	}
 
-	info = framebuffer_alloc(0, device);
-	if (!info)
-		return -ENOMEM;
+	info = drm_fb_helper_alloc_fbi(helper);
+	if (IS_ERR(info))
+		return -PTR_ERR(info);
+
 	info->par = fbdev;
 
 	fbdev->size = size;
 
 	fb = &fbdev->afb.base;
 	fbdev->helper.fb = fb;
-	fbdev->helper.fbdev = info;
 
 	strcpy(info->fix.id, "vboxdrmfb");
 
@@ -284,17 +159,10 @@ static int vboxfb_create(struct drm_fb_helper *helper,
 		      FBINFO_MISC_ALWAYS_SETPAR;
 	info->fbops = &vboxfb_ops;
 
-	ret = fb_alloc_cmap(&info->cmap, 256, 0);
-	if (ret)
-		return -ENOMEM;
-
 	/*
 	 * This seems to be done for safety checking that the framebuffer
 	 * is not registered twice by different drivers.
 	 */
-	info->apertures = alloc_apertures(1);
-	if (!info->apertures)
-		return -ENOMEM;
 	info->apertures->ranges[0].base = pci_resource_start(dev->pdev, 0);
 	info->apertures->ranges[0].size = pci_resource_len(dev->pdev, 0);
 
@@ -305,7 +173,7 @@ static int vboxfb_create(struct drm_fb_helper *helper,
 	info->screen_base = bo->kmap.virtual;
 	info->screen_size = size;
 
-#ifdef CONFIG_FB_DEFERRED_IO
+#ifdef CONFIG_DRM_KMS_FB_HELPER
 	info->fbdefio = &vbox_defio;
 	fb_deferred_io_init(info);
 #endif
@@ -317,22 +185,7 @@ static int vboxfb_create(struct drm_fb_helper *helper,
 	return 0;
 }
 
-static void vbox_fb_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
-			      u16 blue, int regno)
-{
-}
-
-static void vbox_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green,
-			      u16 *blue, int regno)
-{
-	*red = regno;
-	*green = regno;
-	*blue = regno;
-}
-
 static struct drm_fb_helper_funcs vbox_fb_helper_funcs = {
-	.gamma_set = vbox_fb_gamma_set,
-	.gamma_get = vbox_fb_gamma_get,
 	.fb_probe = vboxfb_create,
 };
 
@@ -342,6 +195,11 @@ void vbox_fbdev_fini(struct drm_device *dev)
 	struct vbox_fbdev *fbdev = vbox->fbdev;
 	struct vbox_framebuffer *afb = &fbdev->afb;
 
+#ifdef CONFIG_DRM_KMS_FB_HELPER
+	if (fbdev->helper.fbdev && fbdev->helper.fbdev->fbdefio)
+		fb_deferred_io_cleanup(fbdev->helper.fbdev);
+#endif
+
 	drm_fb_helper_unregister_fbi(&fbdev->helper);
 
 	if (afb->obj) {
@@ -358,7 +216,7 @@ void vbox_fbdev_fini(struct drm_device *dev)
 				vbox_bo_unpin(bo);
 			vbox_bo_unreserve(bo);
 		}
-		drm_gem_object_unreference_unlocked(afb->obj);
+		drm_gem_object_put_unlocked(afb->obj);
 		afb->obj = NULL;
 	}
 	drm_fb_helper_fini(&fbdev->helper);
diff --git a/drivers/staging/vboxvideo/vbox_main.c b/drivers/staging/vboxvideo/vbox_main.c
index d0c6ec75a3c7..80bd039fa08e 100644
--- a/drivers/staging/vboxvideo/vbox_main.c
+++ b/drivers/staging/vboxvideo/vbox_main.c
@@ -40,7 +40,7 @@ static void vbox_user_framebuffer_destroy(struct drm_framebuffer *fb)
 	struct vbox_framebuffer *vbox_fb = to_vbox_framebuffer(fb);
 
 	if (vbox_fb->obj)
-		drm_gem_object_unreference_unlocked(vbox_fb->obj);
+		drm_gem_object_put_unlocked(vbox_fb->obj);
 
 	drm_framebuffer_cleanup(fb);
 	kfree(fb);
@@ -198,7 +198,7 @@ static struct drm_framebuffer *vbox_user_framebuffer_create(
 err_free_vbox_fb:
 	kfree(vbox_fb);
 err_unref_obj:
-	drm_gem_object_unreference_unlocked(obj);
+	drm_gem_object_put_unlocked(obj);
 	return ERR_PTR(ret);
 }
 
@@ -472,7 +472,7 @@ int vbox_dumb_create(struct drm_file *file,
 		return ret;
 
 	ret = drm_gem_handle_create(file, gobj, &handle);
-	drm_gem_object_unreference_unlocked(gobj);
+	drm_gem_object_put_unlocked(gobj);
 	if (ret)
 		return ret;
 
@@ -525,7 +525,7 @@ vbox_dumb_mmap_offset(struct drm_file *file,
 	bo = gem_to_vbox_bo(obj);
 	*offset = vbox_bo_mmap_offset(bo);
 
-	drm_gem_object_unreference(obj);
+	drm_gem_object_put(obj);
 	ret = 0;
 
 out_unlock:
diff --git a/drivers/staging/vboxvideo/vbox_mode.c b/drivers/staging/vboxvideo/vbox_mode.c
index f2b85f3256fa..257a77830410 100644
--- a/drivers/staging/vboxvideo/vbox_mode.c
+++ b/drivers/staging/vboxvideo/vbox_mode.c
@@ -54,14 +54,12 @@ static void vbox_do_modeset(struct drm_crtc *crtc,
 	struct vbox_crtc *vbox_crtc = to_vbox_crtc(crtc);
 	struct vbox_private *vbox;
 	int width, height, bpp, pitch;
-	unsigned int crtc_id;
 	u16 flags;
 	s32 x_offset, y_offset;
 
 	vbox = crtc->dev->dev_private;
 	width = mode->hdisplay ? mode->hdisplay : 640;
 	height = mode->vdisplay ? mode->vdisplay : 480;
-	crtc_id = vbox_crtc->crtc_id;
 	bpp = crtc->enabled ? CRTC_FB(crtc)->format->cpp[0] * 8 : 32;
 	pitch = crtc->enabled ? CRTC_FB(crtc)->pitches[0] : width * bpp / 8;
 	x_offset = vbox->single_framebuffer ? crtc->x : vbox_crtc->x_hint;
@@ -134,10 +132,6 @@ static int vbox_set_view(struct drm_crtc *crtc)
 	return 0;
 }
 
-static void vbox_crtc_load_lut(struct drm_crtc *crtc)
-{
-}
-
 static void vbox_crtc_dpms(struct drm_crtc *crtc, int mode)
 {
 	struct vbox_crtc *vbox_crtc = to_vbox_crtc(crtc);
@@ -330,7 +324,6 @@ static const struct drm_crtc_helper_funcs vbox_crtc_helper_funcs = {
 	.mode_set = vbox_crtc_mode_set,
 	/* .mode_set_base = vbox_crtc_mode_set_base, */
 	.disable = vbox_crtc_disable,
-	.load_lut = vbox_crtc_load_lut,
 	.prepare = vbox_crtc_prepare,
 	.commit = vbox_crtc_commit,
 };
@@ -578,9 +571,6 @@ static int vbox_mode_valid(struct drm_connector *connector,
 
 static void vbox_connector_destroy(struct drm_connector *connector)
 {
-	struct vbox_connector *vbox_connector;
-
-	vbox_connector = to_vbox_connector(connector);
 	drm_connector_unregister(connector);
 	drm_connector_cleanup(connector);
 	kfree(connector);
@@ -817,7 +807,7 @@ out_unmap_bo:
 out_unreserve_bo:
 	vbox_bo_unreserve(bo);
 out_unref_obj:
-	drm_gem_object_unreference_unlocked(obj);
+	drm_gem_object_put_unlocked(obj);
 
 	return ret;
 }
diff --git a/drivers/staging/vboxvideo/vbox_ttm.c b/drivers/staging/vboxvideo/vbox_ttm.c
index 34a905d40735..4eb410a2a1a8 100644
--- a/drivers/staging/vboxvideo/vbox_ttm.c
+++ b/drivers/staging/vboxvideo/vbox_ttm.c
@@ -230,7 +230,7 @@ static void vbox_ttm_tt_unpopulate(struct ttm_tt *ttm)
 	ttm_pool_unpopulate(ttm);
 }
 
-struct ttm_bo_driver vbox_bo_driver = {
+static struct ttm_bo_driver vbox_bo_driver = {
 	.ttm_tt_create = vbox_ttm_tt_create,
 	.ttm_tt_populate = vbox_ttm_tt_populate,
 	.ttm_tt_unpopulate = vbox_ttm_tt_unpopulate,
diff --git a/drivers/staging/vc04_services/bcm2835-audio/bcm2835-pcm.c b/drivers/staging/vc04_services/bcm2835-audio/bcm2835-pcm.c
index 3637ddf909a4..94654c0c7bba 100644
--- a/drivers/staging/vc04_services/bcm2835-audio/bcm2835-pcm.c
+++ b/drivers/staging/vc04_services/bcm2835-audio/bcm2835-pcm.c
@@ -20,7 +20,7 @@
 #include "bcm2835.h"
 
 /* hardware definition */
-static struct snd_pcm_hardware snd_bcm2835_playback_hw = {
+static const struct snd_pcm_hardware snd_bcm2835_playback_hw = {
 	.info = (SNDRV_PCM_INFO_INTERLEAVED | SNDRV_PCM_INFO_BLOCK_TRANSFER |
 	SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID),
 	.formats = SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S16_LE,
@@ -36,7 +36,7 @@ static struct snd_pcm_hardware snd_bcm2835_playback_hw = {
 	.periods_max = 128,
 };
 
-static struct snd_pcm_hardware snd_bcm2835_playback_spdif_hw = {
+static const struct snd_pcm_hardware snd_bcm2835_playback_spdif_hw = {
 	.info = (SNDRV_PCM_INFO_INTERLEAVED | SNDRV_PCM_INFO_BLOCK_TRANSFER |
 	SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID),
 	.formats = SNDRV_PCM_FMTBIT_S16_LE,
@@ -438,7 +438,7 @@ static int snd_bcm2835_pcm_lib_ioctl(struct snd_pcm_substream *substream,
 }
 
 /* operators */
-static struct snd_pcm_ops snd_bcm2835_playback_ops = {
+static const struct snd_pcm_ops snd_bcm2835_playback_ops = {
 	.open = snd_bcm2835_playback_open,
 	.close = snd_bcm2835_playback_close,
 	.ioctl = snd_bcm2835_pcm_lib_ioctl,
@@ -450,7 +450,7 @@ static struct snd_pcm_ops snd_bcm2835_playback_ops = {
 	.ack = snd_bcm2835_pcm_ack,
 };
 
-static struct snd_pcm_ops snd_bcm2835_playback_spdif_ops = {
+static const struct snd_pcm_ops snd_bcm2835_playback_spdif_ops = {
 	.open = snd_bcm2835_playback_spdif_open,
 	.close = snd_bcm2835_playback_close,
 	.ioctl = snd_bcm2835_pcm_lib_ioctl,
diff --git a/drivers/staging/vc04_services/bcm2835-camera/bcm2835-camera.c b/drivers/staging/vc04_services/bcm2835-camera/bcm2835-camera.c
index a11e047734f9..be936b8fe317 100644
--- a/drivers/staging/vc04_services/bcm2835-camera/bcm2835-camera.c
+++ b/drivers/staging/vc04_services/bcm2835-camera/bcm2835-camera.c
@@ -642,7 +642,7 @@ static void bm2835_mmal_unlock(struct vb2_queue *vq)
 	mutex_unlock(&dev->mutex);
 }
 
-static struct vb2_ops bm2835_mmal_video_qops = {
+static const struct vb2_ops bm2835_mmal_video_qops = {
 	.queue_setup = queue_setup,
 	.buf_prepare = buffer_prepare,
 	.buf_queue = buffer_queue,
@@ -1456,7 +1456,7 @@ static const struct v4l2_file_operations camera0_fops = {
 	.mmap = vb2_fop_mmap,
 };
 
-static struct video_device vdev_template = {
+static const struct video_device vdev_template = {
 	.name = "camera0",
 	.fops = &camera0_fops,
 	.ioctl_ops = &camera0_ioctl_ops,
diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_util.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_util.c
index 7fa0310e7b9e..2e52f07bbaa9 100644
--- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_util.c
+++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_util.c
@@ -51,7 +51,7 @@ int vchiu_queue_init(VCHIU_QUEUE_T *queue, int size)
 	sema_init(&queue->pop, 0);
 	sema_init(&queue->push, 0);
 
-	queue->storage = kzalloc(size * sizeof(VCHIQ_HEADER_T *), GFP_KERNEL);
+	queue->storage = kcalloc(size, sizeof(VCHIQ_HEADER_T *), GFP_KERNEL);
 	if (!queue->storage) {
 		vchiu_queue_delete(queue);
 		return 0;
diff --git a/drivers/staging/vt6655/card.c b/drivers/staging/vt6655/card.c
index f5db2b3d9045..14034e342aa6 100644
--- a/drivers/staging/vt6655/card.c
+++ b/drivers/staging/vt6655/card.c
@@ -649,19 +649,19 @@ static unsigned short CARDwGetOFDMControlRate(struct vnt_private *priv,
 	pr_debug("BASIC RATE: %X\n", priv->basic_rates);
 
 	if (!CARDbIsOFDMinBasicRate((void *)priv)) {
-		pr_debug("CARDwGetOFDMControlRate:(NO OFDM) %d\n", wRateIdx);
+		pr_debug("%s:(NO OFDM) %d\n", __func__, wRateIdx);
 		if (wRateIdx > RATE_24M)
 			wRateIdx = RATE_24M;
 		return wRateIdx;
 	}
 	while (ui > RATE_11M) {
 		if (priv->basic_rates & ((u32)0x1 << ui)) {
-			pr_debug("CARDwGetOFDMControlRate : %d\n", ui);
+			pr_debug("%s : %d\n", __func__, ui);
 			return (unsigned short)ui;
 		}
 		ui--;
 	}
-	pr_debug("CARDwGetOFDMControlRate: 6M\n");
+	pr_debug("%s: 6M\n", __func__);
 	return (unsigned short)RATE_24M;
 }
 
diff --git a/drivers/staging/vt6655/mac.c b/drivers/staging/vt6655/mac.c
index 4aaa99bafcda..f7550b215f72 100644
--- a/drivers/staging/vt6655/mac.c
+++ b/drivers/staging/vt6655/mac.c
@@ -809,7 +809,7 @@ void MACvSetKeyEntry(struct vnt_private *priv, unsigned short wKeyCtl,
 	if (byLocalID <= 1)
 		return;
 
-	pr_debug("MACvSetKeyEntry\n");
+	pr_debug("%s\n", __func__);
 	offset = MISCFIFO_KEYETRY0;
 	offset += (uEntryIdx * MISCFIFO_KEYENTRYSIZE);
 
diff --git a/drivers/staging/vt6656/device.h b/drivers/staging/vt6656/device.h
index 4832666cc580..74715c854856 100644
--- a/drivers/staging/vt6656/device.h
+++ b/drivers/staging/vt6656/device.h
@@ -83,7 +83,7 @@
 #define CONFIG_PATH			"/etc/vntconfiguration.dat"
 
 #define MAX_UINTS			8
-#define OPTION_DEFAULT			{ [0 ... MAX_UINTS-1] = -1}
+#define OPTION_DEFAULT			{ [0 ... MAX_UINTS - 1] = -1}
 
 #define DUPLICATE_RX_CACHE_LENGTH       5
 
diff --git a/drivers/staging/vt6656/firmware.c b/drivers/staging/vt6656/firmware.c
index 282f665aacfa..093a6048bd22 100644
--- a/drivers/staging/vt6656/firmware.c
+++ b/drivers/staging/vt6656/firmware.c
@@ -65,7 +65,7 @@ int vnt_download_firmware(struct vnt_private *priv)
 
 		status = vnt_control_out(priv,
 					 0,
-					 0x1200+ii,
+					 0x1200 + ii,
 					 0x0000,
 					 length,
 					 buffer);
diff --git a/drivers/staging/vt6656/key.h b/drivers/staging/vt6656/key.h
index 906d3454591d..cfc6c2131536 100644
--- a/drivers/staging/vt6656/key.h
+++ b/drivers/staging/vt6656/key.h
@@ -46,6 +46,6 @@
 int vnt_key_init_table(struct vnt_private *priv);
 
 int vnt_set_keys(struct ieee80211_hw *hw, struct ieee80211_sta *sta,
-		  struct ieee80211_vif *vif, struct ieee80211_key_conf *key);
+		 struct ieee80211_vif *vif, struct ieee80211_key_conf *key);
 
 #endif /* __KEY_H__ */
diff --git a/drivers/staging/vt6656/main_usb.c b/drivers/staging/vt6656/main_usb.c
index 095b85567306..cc6d8778fe5b 100644
--- a/drivers/staging/vt6656/main_usb.c
+++ b/drivers/staging/vt6656/main_usb.c
@@ -419,8 +419,7 @@ static bool vnt_alloc_bufs(struct vnt_private *priv)
 	int ii;
 
 	for (ii = 0; ii < priv->num_tx_context; ii++) {
-		tx_context = kmalloc(sizeof(struct vnt_usb_send_context),
-				     GFP_KERNEL);
+		tx_context = kmalloc(sizeof(*tx_context), GFP_KERNEL);
 		if (!tx_context)
 			goto free_tx;
 
@@ -437,7 +436,7 @@ static bool vnt_alloc_bufs(struct vnt_private *priv)
 	}
 
 	for (ii = 0; ii < priv->num_rcb; ii++) {
-		priv->rcb[ii] = kzalloc(sizeof(struct vnt_rcb), GFP_KERNEL);
+		priv->rcb[ii] = kzalloc(sizeof(*priv->rcb[ii]), GFP_KERNEL);
 		if (!priv->rcb[ii]) {
 			dev_err(&priv->usb->dev,
 				"failed to allocate rcb no %d\n", ii);
diff --git a/drivers/staging/vt6656/power.c b/drivers/staging/vt6656/power.c
index e322b7d8c617..c466e0614bc4 100644
--- a/drivers/staging/vt6656/power.c
+++ b/drivers/staging/vt6656/power.c
@@ -74,16 +74,15 @@ void vnt_enable_power_saving(struct vnt_private *priv, u16 listen_interval)
 	vnt_mac_reg_bits_on(priv, MAC_REG_PSCTL, PSCTL_GO2DOZE);
 
 	if (listen_interval >= 2) {
-
 		/* clear always listen beacon */
 		vnt_mac_reg_bits_off(priv, MAC_REG_PSCTL, PSCTL_ALBCN);
 
 		/* first time set listen next beacon */
 		vnt_mac_reg_bits_on(priv, MAC_REG_PSCTL, PSCTL_LNBCN);
-	} else
-
+	} else {
 		/* always listen beacon */
 		vnt_mac_reg_bits_on(priv, MAC_REG_PSCTL, PSCTL_ALBCN);
+	}
 
 	dev_dbg(&priv->usb->dev,  "PS:Power Saving Mode Enable...\n");
 }
@@ -100,7 +99,6 @@ void vnt_enable_power_saving(struct vnt_private *priv, u16 listen_interval)
 
 void vnt_disable_power_saving(struct vnt_private *priv)
 {
-
 	/* disable power saving hw function */
 	vnt_control_out(priv, MESSAGE_TYPE_DISABLE_PS, 0,
 			0, 0, NULL);
diff --git a/drivers/staging/vt6656/rf.c b/drivers/staging/vt6656/rf.c
index 23581afb4211..3a9d19a0b842 100644
--- a/drivers/staging/vt6656/rf.c
+++ b/drivers/staging/vt6656/rf.c
@@ -611,7 +611,7 @@ int vnt_rf_write_embedded(struct vnt_private *priv, u32 data)
 	reg_data[3] = (u8)(data >> 24);
 
 	vnt_control_out(priv, MESSAGE_TYPE_WRITE_IFRF,
-				0, 0, ARRAY_SIZE(reg_data), reg_data);
+			0, 0, ARRAY_SIZE(reg_data), reg_data);
 
 	return true;
 }
@@ -643,9 +643,9 @@ int vnt_rf_setpower(struct vnt_private *priv, u32 rate, u32 channel)
 	case RATE_48M:
 	case RATE_54M:
 		if (channel > CB_MAX_CHANNEL_24G)
-			power = priv->ofdm_a_pwr_tbl[channel-15];
+			power = priv->ofdm_a_pwr_tbl[channel - 15];
 		else
-			power = priv->ofdm_pwr_tbl[channel-1];
+			power = priv->ofdm_pwr_tbl[channel - 1];
 		break;
 	}
 
diff --git a/drivers/staging/vt6656/usbpipe.c b/drivers/staging/vt6656/usbpipe.c
index dc11a05be8c4..23eaef458556 100644
--- a/drivers/staging/vt6656/usbpipe.c
+++ b/drivers/staging/vt6656/usbpipe.c
@@ -44,7 +44,7 @@
 #define USB_CTL_WAIT	500 /* ms */
 
 int vnt_control_out(struct vnt_private *priv, u8 request, u16 value,
-		     u16 index, u16 length, u8 *buffer)
+		    u16 index, u16 length, u8 *buffer)
 {
 	int status = 0;
 	u8 *usb_buffer;
@@ -82,7 +82,7 @@ void vnt_control_out_u8(struct vnt_private *priv, u8 reg, u8 reg_off, u8 data)
 }
 
 int vnt_control_in(struct vnt_private *priv, u8 request, u16 value,
-		    u16 index, u16 length, u8 *buffer)
+		   u16 index, u16 length, u8 *buffer)
 {
 	int status;
 	u8 *usb_buffer;
diff --git a/drivers/staging/wilc1000/host_interface.c b/drivers/staging/wilc1000/host_interface.c
index 2568dfc15181..7b620658ec38 100644
--- a/drivers/staging/wilc1000/host_interface.c
+++ b/drivers/staging/wilc1000/host_interface.c
@@ -1963,7 +1963,7 @@ static s32 Handle_Get_InActiveTime(struct wilc_vif *vif,
 				      wilc_get_vif_idx(vif));
 
 	if (result) {
-		netdev_err(vif->ndev, "Failed to SET incative time\n");
+		netdev_err(vif->ndev, "Failed to SET inactive time\n");
 		return -EFAULT;
 	}
 
@@ -1976,7 +1976,7 @@ static s32 Handle_Get_InActiveTime(struct wilc_vif *vif,
 				      wilc_get_vif_idx(vif));
 
 	if (result) {
-		netdev_err(vif->ndev, "Failed to get incative time\n");
+		netdev_err(vif->ndev, "Failed to get inactive time\n");
 		return -EFAULT;
 	}
 
diff --git a/drivers/staging/wilc1000/linux_wlan.c b/drivers/staging/wilc1000/linux_wlan.c
index dbb3e24615be..119f3459b5bb 100644
--- a/drivers/staging/wilc1000/linux_wlan.c
+++ b/drivers/staging/wilc1000/linux_wlan.c
@@ -283,7 +283,8 @@ int wilc_wlan_get_num_conn_ifcs(struct wilc *wilc)
 
 static int linux_wlan_txq_task(void *vp)
 {
-	int ret, txq_count;
+	int ret;
+	u32 txq_count;
 	struct wilc_vif *vif;
 	struct wilc *wl;
 	struct net_device *dev = vp;
@@ -812,7 +813,7 @@ _fail_wilc_wlan_:
 		wilc_wlan_cleanup(dev);
 _fail_locks_:
 		wlan_deinit_locks(dev);
-		netdev_err(dev, "WLAN Iinitialization FAILED\n");
+		netdev_err(dev, "WLAN initialization FAILED\n");
 	} else {
 		netdev_dbg(dev, "wilc1000 already initialized\n");
 	}
diff --git a/drivers/staging/wilc1000/wilc_wfi_cfgoperations.c b/drivers/staging/wilc1000/wilc_wfi_cfgoperations.c
index 68fd5b3b8b2d..ac5aaafa461c 100644
--- a/drivers/staging/wilc1000/wilc_wfi_cfgoperations.c
+++ b/drivers/staging/wilc1000/wilc_wfi_cfgoperations.c
@@ -214,48 +214,39 @@ static u32 get_rssi_avg(struct network_info *network_info)
 	return rssi_v;
 }
 
-static void refresh_scan(void *user_void, u8 all, bool direct_scan)
+static void refresh_scan(struct wilc_priv *priv, bool direct_scan)
 {
-	struct wilc_priv *priv;
-	struct wiphy *wiphy;
-	struct cfg80211_bss *bss = NULL;
+	struct wiphy *wiphy = priv->dev->ieee80211_ptr->wiphy;
 	int i;
-	int rssi = 0;
-
-	priv = user_void;
-	wiphy = priv->dev->ieee80211_ptr->wiphy;
 
 	for (i = 0; i < last_scanned_cnt; i++) {
 		struct network_info *network_info;
+		s32 freq;
+		struct ieee80211_channel *channel;
+		int rssi;
+		struct cfg80211_bss *bss;
 
 		network_info = &last_scanned_shadow[i];
 
-		if (!network_info->found || all) {
-			s32 freq;
-			struct ieee80211_channel *channel;
-
-			if (network_info) {
-				freq = ieee80211_channel_to_frequency((s32)network_info->ch, NL80211_BAND_2GHZ);
-				channel = ieee80211_get_channel(wiphy, freq);
-
-				rssi = get_rssi_avg(network_info);
-				if (memcmp("DIRECT-", network_info->ssid, 7) ||
-				    direct_scan) {
-					bss = cfg80211_inform_bss(wiphy,
-								  channel,
-								  CFG80211_BSS_FTYPE_UNKNOWN,
-								  network_info->bssid,
-								  network_info->tsf_hi,
-								  network_info->cap_info,
-								  network_info->beacon_period,
-								  (const u8 *)network_info->ies,
-								  (size_t)network_info->ies_len,
-								  (s32)rssi * 100,
-								  GFP_KERNEL);
-					cfg80211_put_bss(wiphy, bss);
-				}
-			}
-		}
+		if (!memcmp("DIRECT-", network_info->ssid, 7) && !direct_scan)
+			continue;
+
+		freq = ieee80211_channel_to_frequency((s32)network_info->ch,
+						      NL80211_BAND_2GHZ);
+		channel = ieee80211_get_channel(wiphy, freq);
+		rssi = get_rssi_avg(network_info);
+		bss = cfg80211_inform_bss(wiphy,
+					  channel,
+					  CFG80211_BSS_FTYPE_UNKNOWN,
+					  network_info->bssid,
+					  network_info->tsf_hi,
+					  network_info->cap_info,
+					  network_info->beacon_period,
+					  (const u8 *)network_info->ies,
+					  (size_t)network_info->ies_len,
+					  (s32)rssi * 100,
+					  GFP_KERNEL);
+		cfg80211_put_bss(wiphy, bss);
 	}
 }
 
@@ -442,7 +433,7 @@ static void CfgScanResult(enum scan_event scan_event,
 				}
 			}
 		} else if (scan_event == SCAN_EVENT_DONE) {
-			refresh_scan(priv, 1, false);
+			refresh_scan(priv, false);
 
 			mutex_lock(&priv->scan_req_lock);
 
@@ -466,7 +457,7 @@ static void CfgScanResult(enum scan_event scan_event,
 				};
 
 				update_scan_time();
-				refresh_scan(priv, 1, false);
+				refresh_scan(priv, false);
 
 				cfg80211_scan_done(priv->pstrScanReq, &info);
 				priv->bCfgScanning = false;
@@ -540,7 +531,7 @@ static void CfgConnectResult(enum conn_event enuConnDisconnEvent,
 			}
 
 			if (bNeedScanRefresh)
-				refresh_scan(priv, 1, true);
+				refresh_scan(priv, true);
 		}
 
 		cfg80211_connect_result(dev, pstrConnectInfo->bssid,
diff --git a/drivers/staging/wilc1000/wilc_wfi_netdevice.h b/drivers/staging/wilc1000/wilc_wfi_netdevice.h
index c89bf4301096..7a36561a599e 100644
--- a/drivers/staging/wilc1000/wilc_wfi_netdevice.h
+++ b/drivers/staging/wilc1000/wilc_wfi_netdevice.h
@@ -227,8 +227,8 @@ int wilc1000_wlan_init(struct net_device *dev, struct wilc_vif *vif);
 void wilc_frmw_to_linux(struct wilc *wilc, u8 *buff, u32 size, u32 pkt_offset);
 void wilc_mac_indicate(struct wilc *wilc, int flag);
 void wilc_netdev_cleanup(struct wilc *wilc);
-int wilc_netdev_init(struct wilc **wilc, struct device *, int io_type, int gpio,
-		     const struct wilc_hif_func *ops);
+int wilc_netdev_init(struct wilc **wilc, struct device *dev, int io_type,
+		     int gpio, const struct wilc_hif_func *ops);
 void wilc1000_wlan_deinit(struct net_device *dev);
 void WILC_WFI_mgmt_rx(struct wilc *wilc, u8 *buff, u32 size);
 int wilc_wlan_get_firmware(struct net_device *dev);
diff --git a/drivers/staging/wlan-ng/hfa384x.h b/drivers/staging/wlan-ng/hfa384x.h
index 018db2299d0c..f5a3a1ce21ce 100644
--- a/drivers/staging/wlan-ng/hfa384x.h
+++ b/drivers/staging/wlan-ng/hfa384x.h
@@ -413,8 +413,8 @@ struct hfa384x_join_request_data {
 /*-- Configuration Record: authenticateStation (data portion only) --*/
 struct hfa384x_authenticate_station_data {
 	u8 address[ETH_ALEN];
-	u16 status;
-	u16 algorithm;
+	__le16 status;
+	__le16 algorithm;
 } __packed;
 
 /*-- Configuration Record: WPAData       (data portion only) --*/
@@ -445,9 +445,9 @@ struct hfa384x_downloadbuffer {
 
 /*-- Information Record: commsquality --*/
 struct hfa384x_commsquality {
-	u16 cq_curr_bss;
-	u16 asl_curr_bss;
-	u16 anl_curr_fc;
+	__le16 cq_curr_bss;
+	__le16 asl_curr_bss;
+	__le16 anl_curr_fc;
 } __packed;
 
 /*-- Information Record: dmbcommsquality --*/
@@ -598,51 +598,51 @@ struct hfa384x_rx_frame {
 
 /*--  Inquiry Frame, Diagnose: Communication Tallies --*/
 struct hfa384x_comm_tallies_16 {
-	u16 txunicastframes;
-	u16 txmulticastframes;
-	u16 txfragments;
-	u16 txunicastoctets;
-	u16 txmulticastoctets;
-	u16 txdeferredtrans;
-	u16 txsingleretryframes;
-	u16 txmultipleretryframes;
-	u16 txretrylimitexceeded;
-	u16 txdiscards;
-	u16 rxunicastframes;
-	u16 rxmulticastframes;
-	u16 rxfragments;
-	u16 rxunicastoctets;
-	u16 rxmulticastoctets;
-	u16 rxfcserrors;
-	u16 rxdiscardsnobuffer;
-	u16 txdiscardswrongsa;
-	u16 rxdiscardswepundecr;
-	u16 rxmsginmsgfrag;
-	u16 rxmsginbadmsgfrag;
+	__le16 txunicastframes;
+	__le16 txmulticastframes;
+	__le16 txfragments;
+	__le16 txunicastoctets;
+	__le16 txmulticastoctets;
+	__le16 txdeferredtrans;
+	__le16 txsingleretryframes;
+	__le16 txmultipleretryframes;
+	__le16 txretrylimitexceeded;
+	__le16 txdiscards;
+	__le16 rxunicastframes;
+	__le16 rxmulticastframes;
+	__le16 rxfragments;
+	__le16 rxunicastoctets;
+	__le16 rxmulticastoctets;
+	__le16 rxfcserrors;
+	__le16 rxdiscardsnobuffer;
+	__le16 txdiscardswrongsa;
+	__le16 rxdiscardswepundecr;
+	__le16 rxmsginmsgfrag;
+	__le16 rxmsginbadmsgfrag;
 } __packed;
 
 struct hfa384x_comm_tallies_32 {
-	u32 txunicastframes;
-	u32 txmulticastframes;
-	u32 txfragments;
-	u32 txunicastoctets;
-	u32 txmulticastoctets;
-	u32 txdeferredtrans;
-	u32 txsingleretryframes;
-	u32 txmultipleretryframes;
-	u32 txretrylimitexceeded;
-	u32 txdiscards;
-	u32 rxunicastframes;
-	u32 rxmulticastframes;
-	u32 rxfragments;
-	u32 rxunicastoctets;
-	u32 rxmulticastoctets;
-	u32 rxfcserrors;
-	u32 rxdiscardsnobuffer;
-	u32 txdiscardswrongsa;
-	u32 rxdiscardswepundecr;
-	u32 rxmsginmsgfrag;
-	u32 rxmsginbadmsgfrag;
+	__le32 txunicastframes;
+	__le32 txmulticastframes;
+	__le32 txfragments;
+	__le32 txunicastoctets;
+	__le32 txmulticastoctets;
+	__le32 txdeferredtrans;
+	__le32 txsingleretryframes;
+	__le32 txmultipleretryframes;
+	__le32 txretrylimitexceeded;
+	__le32 txdiscards;
+	__le32 rxunicastframes;
+	__le32 rxmulticastframes;
+	__le32 rxfragments;
+	__le32 rxunicastoctets;
+	__le32 rxmulticastoctets;
+	__le32 rxfcserrors;
+	__le32 rxdiscardsnobuffer;
+	__le32 txdiscardswrongsa;
+	__le32 rxdiscardswepundecr;
+	__le32 rxmsginmsgfrag;
+	__le32 rxmsginbadmsgfrag;
 } __packed;
 
 /*--  Inquiry Frame, Diagnose: Scan Results & Subfields--*/
@@ -711,7 +711,7 @@ struct hfa384x_hscan_result {
 #define HFA384x_LINK_ASSOCFAIL		((u16)6)
 
 struct hfa384x_link_status {
-	u16 linkstatus;
+	__le16 linkstatus;
 } __packed;
 
 /*--  Unsolicited Frame, MAC Mgmt: AssociationStatus (--*/
@@ -733,13 +733,13 @@ struct hfa384x_assoc_status {
 
 struct hfa384x_auth_request {
 	u8 sta_addr[ETH_ALEN];
-	u16 algorithm;
+	__le16 algorithm;
 } __packed;
 
 /*--  Unsolicited Frame, MAC Mgmt: PSUserCount (AP Only) --*/
 
 struct hfa384x_ps_user_count {
-	u16 usercnt;
+	__le16 usercnt;
 } __packed;
 
 struct hfa384x_key_id_changed {
diff --git a/drivers/staging/wlan-ng/hfa384x_usb.c b/drivers/staging/wlan-ng/hfa384x_usb.c
index ee5fa86e941d..d1e8218f96fb 100644
--- a/drivers/staging/wlan-ng/hfa384x_usb.c
+++ b/drivers/staging/wlan-ng/hfa384x_usb.c
@@ -1344,16 +1344,14 @@ hfa384x_docmd(struct hfa384x *hw,
 	if (result != 0) {
 		kfree(ctlx);
 	} else if (mode == DOWAIT) {
-		struct usbctlx_cmd_completor completor;
+		struct usbctlx_cmd_completor cmd_completor;
+		struct usbctlx_completor *completor;
 
-		result =
-		    hfa384x_usbctlx_complete_sync(hw, ctlx,
-						  init_cmd_completor(&completor,
-								     &ctlx->
-								     inbuf.
-								     cmdresp,
-								     &cmd->
-								     result));
+		completor = init_cmd_completor(&cmd_completor,
+					       &ctlx->inbuf.cmdresp,
+					       &cmd->result);
+
+		result = hfa384x_usbctlx_complete_sync(hw, ctlx, completor);
 	}
 
 done:
diff --git a/drivers/staging/wlan-ng/p80211conv.c b/drivers/staging/wlan-ng/p80211conv.c
index fc8ad33ade9f..c1b6d426bcad 100644
--- a/drivers/staging/wlan-ng/p80211conv.c
+++ b/drivers/staging/wlan-ng/p80211conv.c
@@ -213,6 +213,7 @@ int skb_ether_to_p80211(struct wlandevice *wlandev, u32 ethconv,
 			netdev_warn(wlandev->netdev,
 				    "Host en-WEP failed, dropping frame (%d).\n",
 				    foo);
+			kfree(p80211_wep->data);
 			return 2;
 		}
 		fc |= cpu_to_le16(WLAN_SET_FC_ISWEP(1));
diff --git a/drivers/staging/wlan-ng/p80211netdev.c b/drivers/staging/wlan-ng/p80211netdev.c
index 021fb23ae9ba..0f503652740f 100644
--- a/drivers/staging/wlan-ng/p80211netdev.c
+++ b/drivers/staging/wlan-ng/p80211netdev.c
@@ -258,7 +258,7 @@ static int p80211_convert_to_ether(struct wlandevice *wlandev,
 		return 0;
 	}
 
-	netdev_dbg(wlandev->netdev, "p80211_convert_to_ether failed.\n");
+	netdev_dbg(wlandev->netdev, "%s failed.\n", __func__);
 	return CONV_TO_ETHER_FAILED;
 }
 
diff --git a/drivers/staging/wlan-ng/prism2fw.c b/drivers/staging/wlan-ng/prism2fw.c
index 1a0c786c7616..344bec8cc31b 100644
--- a/drivers/staging/wlan-ng/prism2fw.c
+++ b/drivers/staging/wlan-ng/prism2fw.c
@@ -1016,7 +1016,8 @@ static int writeimage(struct wlandevice *wlandev, struct imgchunk *fchunk,
 		kfree(rstmsg);
 		kfree(rwrmsg);
 		netdev_err(wlandev->netdev,
-			   "writeimage: no memory for firmware download, aborting download\n");
+			   "%s: no memory for firmware download, aborting download\n",
+			   __func__);
 		return -ENOMEM;
 	}
 
@@ -1058,15 +1059,15 @@ static int writeimage(struct wlandevice *wlandev, struct imgchunk *fchunk,
 	result = prism2mgmt_ramdl_state(wlandev, rstmsg);
 	if (result) {
 		netdev_err(wlandev->netdev,
-			   "writeimage state enable failed w/ result=%d, aborting download\n",
-			   result);
+			   "%s state enable failed w/ result=%d, aborting download\n",
+			   __func__, result);
 		goto free_result;
 	}
 	resultcode = rstmsg->resultcode.data;
 	if (resultcode != P80211ENUM_resultcode_success) {
 		netdev_err(wlandev->netdev,
-			   "writeimage()->xxxdl_state msg indicates failure, w/ resultcode=%d, aborting download.\n",
-			   resultcode);
+			   "%s()->xxxdl_state msg indicates failure, w/ resultcode=%d, aborting download.\n",
+			   __func__, resultcode);
 		result = 1;
 		goto free_result;
 	}
@@ -1102,14 +1103,14 @@ static int writeimage(struct wlandevice *wlandev, struct imgchunk *fchunk,
 			/* Check the results */
 			if (result) {
 				netdev_err(wlandev->netdev,
-					   "writeimage chunk write failed w/ result=%d, aborting download\n",
-					   result);
+					   "%s chunk write failed w/ result=%d, aborting download\n",
+					   __func__, result);
 				goto free_result;
 			}
 			resultcode = rstmsg->resultcode.data;
 			if (resultcode != P80211ENUM_resultcode_success) {
-				pr_err("writeimage()->xxxdl_write msg indicates failure, w/ resultcode=%d, aborting download.\n",
-				       resultcode);
+				pr_err("%s()->xxxdl_write msg indicates failure, w/ resultcode=%d, aborting download.\n",
+				       __func__, resultcode);
 				result = 1;
 				goto free_result;
 			}
@@ -1124,15 +1125,15 @@ static int writeimage(struct wlandevice *wlandev, struct imgchunk *fchunk,
 	result = prism2mgmt_ramdl_state(wlandev, rstmsg);
 	if (result) {
 		netdev_err(wlandev->netdev,
-			   "writeimage state disable failed w/ result=%d, aborting download\n",
-			   result);
+			   "%s state disable failed w/ result=%d, aborting download\n",
+			   __func__, result);
 		goto free_result;
 	}
 	resultcode = rstmsg->resultcode.data;
 	if (resultcode != P80211ENUM_resultcode_success) {
 		netdev_err(wlandev->netdev,
-			   "writeimage()->xxxdl_state msg indicates failure, w/ resultcode=%d, aborting download.\n",
-			   resultcode);
+			   "%s()->xxxdl_state msg indicates failure, w/ resultcode=%d, aborting download.\n",
+			   __func__, resultcode);
 		result = 1;
 		goto free_result;
 	}
diff --git a/drivers/staging/wlan-ng/prism2sta.c b/drivers/staging/wlan-ng/prism2sta.c
index e16da34389cd..c9df45063ab3 100644
--- a/drivers/staging/wlan-ng/prism2sta.c
+++ b/drivers/staging/wlan-ng/prism2sta.c
@@ -991,9 +991,9 @@ static void prism2sta_inf_tallies(struct wlandevice *wlandev,
 				  struct hfa384x_inf_frame *inf)
 {
 	struct hfa384x *hw = wlandev->priv;
-	u16 *src16;
+	__le16 *src16;
 	u32 *dst;
-	u32 *src32;
+	__le32 *src32;
 	int i;
 	int cnt;
 
@@ -1005,12 +1005,12 @@ static void prism2sta_inf_tallies(struct wlandevice *wlandev,
 	cnt = sizeof(struct hfa384x_comm_tallies_32) / sizeof(u32);
 	if (inf->framelen > 22) {
 		dst = (u32 *)&hw->tallies;
-		src32 = (u32 *)&inf->info.commtallies32;
+		src32 = (__le32 *)&inf->info.commtallies32;
 		for (i = 0; i < cnt; i++, dst++, src32++)
 			*dst += le32_to_cpu(*src32);
 	} else {
 		dst = (u32 *)&hw->tallies;
-		src16 = (u16 *)&inf->info.commtallies16;
+		src16 = (__le16 *)&inf->info.commtallies16;
 		for (i = 0; i < cnt; i++, dst++, src16++)
 			*dst += le16_to_cpu(*src16);
 	}
@@ -1136,7 +1136,7 @@ static void prism2sta_inf_chinforesults(struct wlandevice *wlandev,
 	unsigned int i, n;
 
 	hw->channel_info.results.scanchannels =
-	    le16_to_cpu(inf->info.chinforesult.scanchannels);
+	    inf->info.chinforesult.scanchannels;
 
 	for (i = 0, n = 0; i < HFA384x_CHINFORESULT_MAX; i++) {
 		struct hfa384x_ch_info_result_sub *result;
@@ -1147,16 +1147,16 @@ static void prism2sta_inf_chinforesults(struct wlandevice *wlandev,
 			continue;
 
 		result = &inf->info.chinforesult.result[n];
-		chan = le16_to_cpu(result->chid) - 1;
+		chan = result->chid - 1;
 
 		if (chan < 0 || chan >= HFA384x_CHINFORESULT_MAX)
 			continue;
 
 		chinforesult = &hw->channel_info.results.result[chan];
 		chinforesult->chid = chan;
-		chinforesult->anl = le16_to_cpu(result->anl);
-		chinforesult->pnl = le16_to_cpu(result->pnl);
-		chinforesult->active = le16_to_cpu(result->active);
+		chinforesult->anl = result->anl;
+		chinforesult->pnl = result->pnl;
+		chinforesult->active = result->active;
 
 		pr_debug("chinfo: channel %d, %s level (avg/peak)=%d/%d dB, pcf %d\n",
 			 chan + 1,
@@ -1447,7 +1447,7 @@ static void prism2sta_inf_linkstatus(struct wlandevice *wlandev,
 {
 	struct hfa384x *hw = wlandev->priv;
 
-	hw->link_status_new = le16_to_cpu(inf->info.linkstatus.linkstatus);
+	hw->link_status_new = inf->info.linkstatus.linkstatus;
 
 	schedule_work(&hw->link_bh);
 }
@@ -1561,7 +1561,7 @@ static void prism2sta_inf_authreq_defer(struct wlandevice *wlandev,
 	 */
 
 	ether_addr_copy(rec.address, inf->info.authreq.sta_addr);
-	rec.status = P80211ENUM_status_unspec_failure;
+	rec.status = cpu_to_le16(P80211ENUM_status_unspec_failure);
 
 	/*
 	 * Authenticate based on the access mode.
@@ -1578,7 +1578,7 @@ static void prism2sta_inf_authreq_defer(struct wlandevice *wlandev,
 		for (i = 0; i < hw->authlist.cnt; i++)
 			if (ether_addr_equal(rec.address,
 					     hw->authlist.addr[i])) {
-				rec.status = P80211ENUM_status_successful;
+				rec.status = cpu_to_le16(P80211ENUM_status_successful);
 				break;
 			}
 
@@ -1590,7 +1590,7 @@ static void prism2sta_inf_authreq_defer(struct wlandevice *wlandev,
 		 * Allow all authentications.
 		 */
 
-		rec.status = P80211ENUM_status_successful;
+		rec.status = cpu_to_le16(P80211ENUM_status_successful);
 		break;
 
 	case WLAN_ACCESS_ALLOW:
@@ -1615,7 +1615,7 @@ static void prism2sta_inf_authreq_defer(struct wlandevice *wlandev,
 
 		for (i = 0; i < cnt; i++, addr += ETH_ALEN)
 			if (ether_addr_equal(rec.address, addr)) {
-				rec.status = P80211ENUM_status_successful;
+				rec.status = cpu_to_le16(P80211ENUM_status_successful);
 				break;
 			}
 
@@ -1641,11 +1641,11 @@ static void prism2sta_inf_authreq_defer(struct wlandevice *wlandev,
 			addr = hw->deny.addr1[0];
 		}
 
-		rec.status = P80211ENUM_status_successful;
+		rec.status = cpu_to_le16(P80211ENUM_status_successful);
 
 		for (i = 0; i < cnt; i++, addr += ETH_ALEN)
 			if (ether_addr_equal(rec.address, addr)) {
-				rec.status = P80211ENUM_status_unspec_failure;
+				rec.status = cpu_to_le16(P80211ENUM_status_unspec_failure);
 				break;
 			}
 
@@ -1663,7 +1663,7 @@ static void prism2sta_inf_authreq_defer(struct wlandevice *wlandev,
 
 	added = 0;
 
-	if (rec.status == P80211ENUM_status_successful) {
+	if (rec.status == cpu_to_le16(P80211ENUM_status_successful)) {
 		for (i = 0; i < hw->authlist.cnt; i++)
 			if (ether_addr_equal(rec.address,
 					     hw->authlist.addr[i]))
@@ -1671,7 +1671,7 @@ static void prism2sta_inf_authreq_defer(struct wlandevice *wlandev,
 
 		if (i >= hw->authlist.cnt) {
 			if (hw->authlist.cnt >= WLAN_AUTH_MAX) {
-				rec.status = P80211ENUM_status_ap_full;
+				rec.status = cpu_to_le16(P80211ENUM_status_ap_full);
 			} else {
 				ether_addr_copy(
 					hw->authlist.addr[hw->authlist.cnt],
@@ -1688,7 +1688,6 @@ static void prism2sta_inf_authreq_defer(struct wlandevice *wlandev,
 	 * it was added.
 	 */
 
-	rec.status = cpu_to_le16(rec.status);
 	rec.algorithm = inf->info.authreq.algorithm;
 
 	result = hfa384x_drvr_setconfig(hw, HFA384x_RID_AUTHENTICATESTA,
diff --git a/drivers/target/target_core_alua.c b/drivers/target/target_core_alua.c
index a91b7c25ffd4..928127642574 100644
--- a/drivers/target/target_core_alua.c
+++ b/drivers/target/target_core_alua.c
@@ -896,13 +896,14 @@ static int core_alua_write_tpg_metadata(
 	u32 md_buf_len)
 {
 	struct file *file = filp_open(path, O_RDWR | O_CREAT | O_TRUNC, 0600);
+	loff_t pos = 0;
 	int ret;
 
 	if (IS_ERR(file)) {
 		pr_err("filp_open(%s) for ALUA metadata failed\n", path);
 		return -ENODEV;
 	}
-	ret = kernel_write(file, md_buf, md_buf_len, 0);
+	ret = kernel_write(file, md_buf, md_buf_len, &pos);
 	if (ret < 0)
 		pr_err("Error writing ALUA metadata file: %s\n", path);
 	fput(file);
diff --git a/drivers/target/target_core_file.c b/drivers/target/target_core_file.c
index 24cf11d9e50a..c629817a8854 100644
--- a/drivers/target/target_core_file.c
+++ b/drivers/target/target_core_file.c
@@ -443,7 +443,7 @@ fd_do_prot_fill(struct se_device *se_dev, sector_t lba, sector_t nolb,
 
 	for (prot = 0; prot < prot_length;) {
 		sector_t len = min_t(sector_t, bufsize, prot_length - prot);
-		ssize_t ret = kernel_write(prot_fd, buf, len, pos + prot);
+		ssize_t ret = kernel_write(prot_fd, buf, len, &pos);
 
 		if (ret != len) {
 			pr_err("vfs_write to prot file failed: %zd\n", ret);
diff --git a/drivers/target/target_core_iblock.c b/drivers/target/target_core_iblock.c
index ee7c7fa55dad..07c814c42648 100644
--- a/drivers/target/target_core_iblock.c
+++ b/drivers/target/target_core_iblock.c
@@ -338,7 +338,7 @@ iblock_get_bio(struct se_cmd *cmd, sector_t lba, u32 sg_num, int op,
 		return NULL;
 	}
 
-	bio->bi_bdev = ib_dev->ibd_bd;
+	bio_set_dev(bio, ib_dev->ibd_bd);
 	bio->bi_private = cmd;
 	bio->bi_end_io = &iblock_bio_done;
 	bio->bi_iter.bi_sector = lba;
@@ -395,7 +395,7 @@ iblock_execute_sync_cache(struct se_cmd *cmd)
 
 	bio = bio_alloc(GFP_KERNEL, 0);
 	bio->bi_end_io = iblock_end_io_flush;
-	bio->bi_bdev = ib_dev->ibd_bd;
+	bio_set_dev(bio, ib_dev->ibd_bd);
 	bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
 	if (!immed)
 		bio->bi_private = cmd;
diff --git a/drivers/target/target_core_pr.c b/drivers/target/target_core_pr.c
index 6d5def64db61..dd2cd8048582 100644
--- a/drivers/target/target_core_pr.c
+++ b/drivers/target/target_core_pr.c
@@ -1974,6 +1974,7 @@ static int __core_scsi3_write_aptpl_to_file(
 	char path[512];
 	u32 pr_aptpl_buf_len;
 	int ret;
+	loff_t pos = 0;
 
 	memset(path, 0, 512);
 
@@ -1993,7 +1994,7 @@ static int __core_scsi3_write_aptpl_to_file(
 
 	pr_aptpl_buf_len = (strlen(buf) + 1); /* Add extra for NULL */
 
-	ret = kernel_write(file, buf, pr_aptpl_buf_len, 0);
+	ret = kernel_write(file, buf, pr_aptpl_buf_len, &pos);
 
 	if (ret < 0)
 		pr_debug("Error writing APTPL metadata file: %s\n", path);
diff --git a/drivers/tee/optee/core.c b/drivers/tee/optee/core.c
index 58169e519422..7952357df9c8 100644
--- a/drivers/tee/optee/core.c
+++ b/drivers/tee/optee/core.c
@@ -224,13 +224,14 @@ static void optee_release(struct tee_context *ctx)
 	if (!IS_ERR(shm)) {
 		arg = tee_shm_get_va(shm, 0);
 		/*
-		 * If va2pa fails for some reason, we can't call
-		 * optee_close_session(), only free the memory. Secure OS
-		 * will leak sessions and finally refuse more sessions, but
-		 * we will at least let normal world reclaim its memory.
+		 * If va2pa fails for some reason, we can't call into
+		 * secure world, only free the memory. Secure OS will leak
+		 * sessions and finally refuse more sessions, but we will
+		 * at least let normal world reclaim its memory.
 		 */
 		if (!IS_ERR(arg))
-			tee_shm_va2pa(shm, arg, &parg);
+			if (tee_shm_va2pa(shm, arg, &parg))
+				arg = NULL; /* prevent usage of parg below */
 	}
 
 	list_for_each_entry_safe(sess, sess_tmp, &ctxdata->sess_list,
@@ -258,7 +259,7 @@ static void optee_release(struct tee_context *ctx)
 	}
 }
 
-static struct tee_driver_ops optee_ops = {
+static const struct tee_driver_ops optee_ops = {
 	.get_version = optee_get_version,
 	.open = optee_open,
 	.release = optee_release,
@@ -268,13 +269,13 @@ static struct tee_driver_ops optee_ops = {
 	.cancel_req = optee_cancel_req,
 };
 
-static struct tee_desc optee_desc = {
+static const struct tee_desc optee_desc = {
 	.name = DRIVER_NAME "-clnt",
 	.ops = &optee_ops,
 	.owner = THIS_MODULE,
 };
 
-static struct tee_driver_ops optee_supp_ops = {
+static const struct tee_driver_ops optee_supp_ops = {
 	.get_version = optee_get_version,
 	.open = optee_open,
 	.release = optee_release,
@@ -282,7 +283,7 @@ static struct tee_driver_ops optee_supp_ops = {
 	.supp_send = optee_supp_send,
 };
 
-static struct tee_desc optee_supp_desc = {
+static const struct tee_desc optee_supp_desc = {
 	.name = DRIVER_NAME "-supp",
 	.ops = &optee_supp_ops,
 	.owner = THIS_MODULE,
diff --git a/drivers/tee/optee/optee_smc.h b/drivers/tee/optee/optee_smc.h
index 13b7c98cdf25..069c8e1429de 100644
--- a/drivers/tee/optee/optee_smc.h
+++ b/drivers/tee/optee/optee_smc.h
@@ -298,7 +298,7 @@ struct optee_smc_disable_shm_cache_result {
 	OPTEE_SMC_FAST_CALL_VAL(OPTEE_SMC_FUNCID_ENABLE_SHM_CACHE)
 
 /*
- * Resume from RPC (for example after processing an IRQ)
+ * Resume from RPC (for example after processing a foreign interrupt)
  *
  * Call register usage:
  * a0	SMC Function ID, OPTEE_SMC_CALL_RETURN_FROM_RPC
@@ -383,19 +383,19 @@ struct optee_smc_disable_shm_cache_result {
 	OPTEE_SMC_RPC_VAL(OPTEE_SMC_RPC_FUNC_FREE)
 
 /*
- * Deliver an IRQ in normal world.
+ * Deliver foreign interrupt to normal world.
  *
  * "Call" register usage:
- * a0	OPTEE_SMC_RETURN_RPC_IRQ
+ * a0	OPTEE_SMC_RETURN_RPC_FOREIGN_INTR
  * a1-7	Resume information, must be preserved
  *
  * "Return" register usage:
  * a0	SMC Function ID, OPTEE_SMC_CALL_RETURN_FROM_RPC.
  * a1-7	Preserved
  */
-#define OPTEE_SMC_RPC_FUNC_IRQ		4
-#define OPTEE_SMC_RETURN_RPC_IRQ \
-	OPTEE_SMC_RPC_VAL(OPTEE_SMC_RPC_FUNC_IRQ)
+#define OPTEE_SMC_RPC_FUNC_FOREIGN_INTR		4
+#define OPTEE_SMC_RETURN_RPC_FOREIGN_INTR \
+	OPTEE_SMC_RPC_VAL(OPTEE_SMC_RPC_FUNC_FOREIGN_INTR)
 
 /*
  * Do an RPC request. The supplied struct optee_msg_arg tells which
diff --git a/drivers/tee/optee/rpc.c b/drivers/tee/optee/rpc.c
index 8814eca06021..cef417f4f4d2 100644
--- a/drivers/tee/optee/rpc.c
+++ b/drivers/tee/optee/rpc.c
@@ -140,11 +140,8 @@ static void handle_rpc_func_cmd_wait(struct optee_msg_arg *arg)
 
 	msec_to_wait = arg->params[0].u.value.a;
 
-	/* set task's state to interruptible sleep */
-	set_current_state(TASK_INTERRUPTIBLE);
-
-	/* take a nap */
-	msleep(msec_to_wait);
+	/* Go to interruptible sleep */
+	msleep_interruptible(msec_to_wait);
 
 	arg->ret = TEEC_SUCCESS;
 	return;
@@ -374,11 +371,11 @@ void optee_handle_rpc(struct tee_context *ctx, struct optee_rpc_param *param)
 		shm = reg_pair_to_ptr(param->a1, param->a2);
 		tee_shm_free(shm);
 		break;
-	case OPTEE_SMC_RPC_FUNC_IRQ:
+	case OPTEE_SMC_RPC_FUNC_FOREIGN_INTR:
 		/*
-		 * An IRQ was raised while secure world was executing,
-		 * since all IRQs are handled in Linux a dummy RPC is
-		 * performed to let Linux take the IRQ through the normal
+		 * A foreign interrupt was raised while secure world was
+		 * executing, since they are handled in Linux a dummy RPC is
+		 * performed to let Linux take the interrupt through the normal
 		 * vector.
 		 */
 		break;
diff --git a/drivers/tee/tee_core.c b/drivers/tee/tee_core.c
index 5c60bf4423e6..58a5009eacc3 100644
--- a/drivers/tee/tee_core.c
+++ b/drivers/tee/tee_core.c
@@ -90,8 +90,13 @@ static int tee_ioctl_version(struct tee_context *ctx,
 	struct tee_ioctl_version_data vers;
 
 	ctx->teedev->desc->ops->get_version(ctx->teedev, &vers);
+
+	if (ctx->teedev->desc->flags & TEE_DESC_PRIVILEGED)
+		vers.gen_caps |= TEE_GEN_CAP_PRIVILEGED;
+
 	if (copy_to_user(uvers, &vers, sizeof(vers)))
 		return -EFAULT;
+
 	return 0;
 }
 
diff --git a/drivers/tee/tee_shm.c b/drivers/tee/tee_shm.c
index d356d7f025eb..4bc7956cefc4 100644
--- a/drivers/tee/tee_shm.c
+++ b/drivers/tee/tee_shm.c
@@ -80,7 +80,7 @@ static int tee_shm_op_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
 			       size, vma->vm_page_prot);
 }
 
-static struct dma_buf_ops tee_shm_dma_buf_ops = {
+static const struct dma_buf_ops tee_shm_dma_buf_ops = {
 	.map_dma_buf = tee_shm_op_map_dma_buf,
 	.unmap_dma_buf = tee_shm_op_unmap_dma_buf,
 	.release = tee_shm_op_release,
diff --git a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig
index b5b5facb8747..07002df4f83a 100644
--- a/drivers/thermal/Kconfig
+++ b/drivers/thermal/Kconfig
@@ -342,7 +342,7 @@ config X86_PKG_TEMP_THERMAL
 
 config INTEL_SOC_DTS_IOSF_CORE
 	tristate
-	depends on X86
+	depends on X86 && PCI
 	select IOSF_MBI
 	help
 	  This is becoming a common feature for Intel SoCs to expose the additional
@@ -352,7 +352,7 @@ config INTEL_SOC_DTS_IOSF_CORE
 
 config INTEL_SOC_DTS_THERMAL
 	tristate "Intel SoCs DTS thermal driver"
-	depends on X86
+	depends on X86 && PCI
 	select INTEL_SOC_DTS_IOSF_CORE
 	select THERMAL_WRITABLE_TRIPS
 	help
@@ -473,4 +473,12 @@ config ZX2967_THERMAL
 	  the primitive temperature sensor embedded in zx2967 SoCs.
 	  This sensor generates the real time die temperature.
 
+config UNIPHIER_THERMAL
+	tristate "Socionext UniPhier thermal driver"
+	depends on ARCH_UNIPHIER || COMPILE_TEST
+	depends on THERMAL_OF && MFD_SYSCON
+	help
+	  Enable this to plug in UniPhier on-chip PVT thermal driver into the
+	  thermal framework. The driver supports CPU thermal zone temperature
+	  reporting and a couple of trip points.
 endif
diff --git a/drivers/thermal/Makefile b/drivers/thermal/Makefile
index 094d7039981c..8b79bca23536 100644
--- a/drivers/thermal/Makefile
+++ b/drivers/thermal/Makefile
@@ -59,3 +59,4 @@ obj-$(CONFIG_HISI_THERMAL)     += hisi_thermal.o
 obj-$(CONFIG_MTK_THERMAL)	+= mtk_thermal.o
 obj-$(CONFIG_GENERIC_ADC_THERMAL)	+= thermal-generic-adc.o
 obj-$(CONFIG_ZX2967_THERMAL)	+= zx2967_thermal.o
+obj-$(CONFIG_UNIPHIER_THERMAL)	+= uniphier_thermal.o
diff --git a/drivers/thermal/broadcom/bcm2835_thermal.c b/drivers/thermal/broadcom/bcm2835_thermal.c
index e6863c841662..a4d6a0e2e993 100644
--- a/drivers/thermal/broadcom/bcm2835_thermal.c
+++ b/drivers/thermal/broadcom/bcm2835_thermal.c
@@ -145,7 +145,7 @@ static void bcm2835_thermal_debugfs(struct platform_device *pdev)
 	debugfs_create_regset32("regset", 0444, data->debugfsdir, regset);
 }
 
-static struct thermal_zone_of_device_ops bcm2835_thermal_ops = {
+static const struct thermal_zone_of_device_ops bcm2835_thermal_ops = {
 	.get_temp = bcm2835_thermal_get_temp,
 };
 
diff --git a/drivers/thermal/hisi_thermal.c b/drivers/thermal/hisi_thermal.c
index 9c3ce341eb97..bd3572c41585 100644
--- a/drivers/thermal/hisi_thermal.c
+++ b/drivers/thermal/hisi_thermal.c
@@ -206,7 +206,7 @@ static int hisi_thermal_get_temp(void *_sensor, int *temp)
 	return 0;
 }
 
-static struct thermal_zone_of_device_ops hisi_of_thermal_ops = {
+static const struct thermal_zone_of_device_ops hisi_of_thermal_ops = {
 	.get_temp = hisi_thermal_get_temp,
 };
 
diff --git a/drivers/thermal/int340x_thermal/acpi_thermal_rel.c b/drivers/thermal/int340x_thermal/acpi_thermal_rel.c
index 51ceb80212a7..c719167e9f28 100644
--- a/drivers/thermal/int340x_thermal/acpi_thermal_rel.c
+++ b/drivers/thermal/int340x_thermal/acpi_thermal_rel.c
@@ -228,7 +228,7 @@ static void get_single_name(acpi_handle handle, char *name)
 	struct acpi_buffer buffer = {ACPI_ALLOCATE_BUFFER};
 
 	if (ACPI_FAILURE(acpi_get_name(handle, ACPI_SINGLE_NAME, &buffer)))
-		pr_warn("Failed get name from handle\n");
+		pr_warn("Failed to get device name from acpi handle\n");
 	else {
 		memcpy(name, buffer.pointer, ACPI_NAME_SIZE);
 		kfree(buffer.pointer);
diff --git a/drivers/thermal/int340x_thermal/acpi_thermal_rel.h b/drivers/thermal/int340x_thermal/acpi_thermal_rel.h
index f00700bc9d79..65075b174329 100644
--- a/drivers/thermal/int340x_thermal/acpi_thermal_rel.h
+++ b/drivers/thermal/int340x_thermal/acpi_thermal_rel.h
@@ -34,10 +34,10 @@ struct trt {
 	acpi_handle target;
 	u64 influence;
 	u64 sample_period;
-	u64 reverved1;
-	u64 reverved2;
-	u64 reverved3;
-	u64 reverved4;
+	u64 reserved1;
+	u64 reserved2;
+	u64 reserved3;
+	u64 reserved4;
 } __packed;
 
 #define ACPI_NR_ART_ELEMENTS 13
diff --git a/drivers/thermal/int340x_thermal/int3400_thermal.c b/drivers/thermal/int340x_thermal/int3400_thermal.c
index a9ec94ed7a42..8ee38f55c7f3 100644
--- a/drivers/thermal/int340x_thermal/int3400_thermal.c
+++ b/drivers/thermal/int340x_thermal/int3400_thermal.c
@@ -16,6 +16,8 @@
 #include <linux/thermal.h>
 #include "acpi_thermal_rel.h"
 
+#define INT3400_THERMAL_TABLE_CHANGED 0x83
+
 enum int3400_thermal_uuid {
 	INT3400_THERMAL_PASSIVE_1,
 	INT3400_THERMAL_ACTIVE,
@@ -104,7 +106,7 @@ static struct attribute *uuid_attrs[] = {
 	NULL
 };
 
-static struct attribute_group uuid_attribute_group = {
+static const struct attribute_group uuid_attribute_group = {
 	.attrs = uuid_attrs,
 	.name = "uuids"
 };
@@ -185,6 +187,35 @@ static int int3400_thermal_run_osc(acpi_handle handle,
 	return result;
 }
 
+static void int3400_notify(acpi_handle handle,
+			u32 event,
+			void *data)
+{
+	struct int3400_thermal_priv *priv = data;
+	char *thermal_prop[5];
+
+	if (!priv)
+		return;
+
+	switch (event) {
+	case INT3400_THERMAL_TABLE_CHANGED:
+		thermal_prop[0] = kasprintf(GFP_KERNEL, "NAME=%s",
+				priv->thermal->type);
+		thermal_prop[1] = kasprintf(GFP_KERNEL, "TEMP=%d",
+				priv->thermal->temperature);
+		thermal_prop[2] = kasprintf(GFP_KERNEL, "TRIP=");
+		thermal_prop[3] = kasprintf(GFP_KERNEL, "EVENT=%d",
+				THERMAL_TABLE_CHANGED);
+		thermal_prop[4] = NULL;
+		kobject_uevent_env(&priv->thermal->device.kobj, KOBJ_CHANGE,
+				thermal_prop);
+		break;
+	default:
+		dev_err(&priv->adev->dev, "Unsupported event [0x%x]\n", event);
+		break;
+	}
+}
+
 static int int3400_thermal_get_temp(struct thermal_zone_device *thermal,
 			int *temp)
 {
@@ -290,6 +321,12 @@ static int int3400_thermal_probe(struct platform_device *pdev)
 	if (result)
 		goto free_zone;
 
+	result = acpi_install_notify_handler(
+			priv->adev->handle, ACPI_DEVICE_NOTIFY, int3400_notify,
+			(void *)priv);
+	if (result)
+		goto free_zone;
+
 	return 0;
 
 free_zone:
@@ -306,6 +343,10 @@ static int int3400_thermal_remove(struct platform_device *pdev)
 {
 	struct int3400_thermal_priv *priv = platform_get_drvdata(pdev);
 
+	acpi_remove_notify_handler(
+			priv->adev->handle, ACPI_DEVICE_NOTIFY,
+			int3400_notify);
+
 	if (!priv->rel_misc_dev_res)
 		acpi_thermal_rel_misc_device_remove(priv->adev->handle);
 
diff --git a/drivers/thermal/int340x_thermal/int3406_thermal.c b/drivers/thermal/int340x_thermal/int3406_thermal.c
index 1891f34ab7fc..f69ab026ba24 100644
--- a/drivers/thermal/int340x_thermal/int3406_thermal.c
+++ b/drivers/thermal/int340x_thermal/int3406_thermal.c
@@ -21,39 +21,33 @@
 
 struct int3406_thermal_data {
 	int upper_limit;
-	int upper_limit_index;
 	int lower_limit;
-	int lower_limit_index;
 	acpi_handle handle;
 	struct acpi_video_device_brightness *br;
 	struct backlight_device *raw_bd;
 	struct thermal_cooling_device *cooling_dev;
 };
 
-static int int3406_thermal_to_raw(int level, struct int3406_thermal_data *d)
-{
-	int max_level = d->br->levels[d->br->count - 1];
-	int raw_max = d->raw_bd->props.max_brightness;
-
-	return level * raw_max / max_level;
-}
-
-static int int3406_thermal_to_acpi(int level, struct int3406_thermal_data *d)
-{
-	int raw_max = d->raw_bd->props.max_brightness;
-	int max_level = d->br->levels[d->br->count - 1];
-
-	return level * max_level / raw_max;
-}
+/*
+ * According to the ACPI spec,
+ * "Each brightness level is represented by a number between 0 and 100,
+ * and can be thought of as a percentage. For example, 50 can be 50%
+ * power consumption or 50% brightness, as defined by the OEM."
+ *
+ * As int3406 device uses this value to communicate with the native
+ * graphics driver, we make the assumption that it represents
+ * the percentage of brightness only
+ */
+#define ACPI_TO_RAW(v, d) (d->raw_bd->props.max_brightness * v / 100)
+#define RAW_TO_ACPI(v, d) (v * 100 / d->raw_bd->props.max_brightness)
 
 static int
 int3406_thermal_get_max_state(struct thermal_cooling_device *cooling_dev,
 			      unsigned long *state)
 {
 	struct int3406_thermal_data *d = cooling_dev->devdata;
-	int index = d->lower_limit_index ? d->lower_limit_index : 2;
 
-	*state = d->br->count - 1 - index;
+	*state = d->upper_limit - d->lower_limit;
 	return 0;
 }
 
@@ -62,19 +56,15 @@ int3406_thermal_set_cur_state(struct thermal_cooling_device *cooling_dev,
 			      unsigned long state)
 {
 	struct int3406_thermal_data *d = cooling_dev->devdata;
-	int level, raw_level;
+	int acpi_level, raw_level;
 
-	if (state > d->br->count - 3)
+	if (state > d->upper_limit - d->lower_limit)
 		return -EINVAL;
 
-	state = d->br->count - 1 - state;
-	level = d->br->levels[state];
+	acpi_level = d->br->levels[d->upper_limit - state];
 
-	if ((d->upper_limit && level > d->upper_limit) ||
-	    (d->lower_limit && level < d->lower_limit))
-		return -EINVAL;
+	raw_level = ACPI_TO_RAW(acpi_level, d);
 
-	raw_level = int3406_thermal_to_raw(level, d);
 	return backlight_device_set_brightness(d->raw_bd, raw_level);
 }
 
@@ -83,27 +73,22 @@ int3406_thermal_get_cur_state(struct thermal_cooling_device *cooling_dev,
 			      unsigned long *state)
 {
 	struct int3406_thermal_data *d = cooling_dev->devdata;
-	int raw_level, level, i;
-	int *levels = d->br->levels;
+	int acpi_level;
+	int index;
 
-	raw_level = d->raw_bd->props.brightness;
-	level = int3406_thermal_to_acpi(raw_level, d);
+	acpi_level = RAW_TO_ACPI(d->raw_bd->props.brightness, d);
 
 	/*
-	 * There is no 1:1 mapping between the firmware interface level with the
-	 * raw interface level, we will have to find one that is close enough.
+	 * There is no 1:1 mapping between the firmware interface level
+	 * with the raw interface level, we will have to find one that is
+	 * right above it.
 	 */
-	for (i = 2; i < d->br->count; i++) {
-		if (level < levels[i]) {
-			if (i == 2)
-				break;
-			if ((level - levels[i - 1]) < (levels[i] - level))
-				i--;
+	for (index = d->lower_limit; index < d->upper_limit; index++) {
+		if (acpi_level <= d->br->levels[index])
 			break;
-		}
 	}
 
-	*state = d->br->count - 1 - i;
+	*state = d->upper_limit - index;
 	return 0;
 }
 
@@ -117,7 +102,7 @@ static int int3406_thermal_get_index(int *array, int nr, int value)
 {
 	int i;
 
-	for (i = 0; i < nr; i++) {
+	for (i = 2; i < nr; i++) {
 		if (array[i] == value)
 			break;
 	}
@@ -128,27 +113,20 @@ static void int3406_thermal_get_limit(struct int3406_thermal_data *d)
 {
 	acpi_status status;
 	unsigned long long lower_limit, upper_limit;
-	int index;
 
 	status = acpi_evaluate_integer(d->handle, "DDDL", NULL, &lower_limit);
-	if (ACPI_SUCCESS(status)) {
-		index = int3406_thermal_get_index(d->br->levels, d->br->count,
-						  lower_limit);
-		if (index > 0) {
-			d->lower_limit = (int)lower_limit;
-			d->lower_limit_index = index;
-		}
-	}
+	if (ACPI_SUCCESS(status))
+		d->lower_limit = int3406_thermal_get_index(d->br->levels,
+					d->br->count, lower_limit);
 
 	status = acpi_evaluate_integer(d->handle, "DDPC", NULL, &upper_limit);
-	if (ACPI_SUCCESS(status)) {
-		index = int3406_thermal_get_index(d->br->levels, d->br->count,
-						  upper_limit);
-		if (index > 0) {
-			d->upper_limit = (int)upper_limit;
-			d->upper_limit_index = index;
-		}
-	}
+	if (ACPI_SUCCESS(status))
+		d->upper_limit = int3406_thermal_get_index(d->br->levels,
+					d->br->count, upper_limit);
+
+	/* lower_limit and upper_limit should be always set */
+	d->lower_limit = d->lower_limit > 0 ? d->lower_limit : 2;
+	d->upper_limit = d->upper_limit > 0 ? d->upper_limit : d->br->count - 1;
 }
 
 static void int3406_notify(acpi_handle handle, u32 event, void *data)
diff --git a/drivers/thermal/int340x_thermal/processor_thermal_device.c b/drivers/thermal/int340x_thermal/processor_thermal_device.c
index ff3b36f339e3..f02341f7134d 100644
--- a/drivers/thermal/int340x_thermal/processor_thermal_device.c
+++ b/drivers/thermal/int340x_thermal/processor_thermal_device.c
@@ -127,7 +127,7 @@ static struct attribute *power_limit_attrs[] = {
 	NULL
 };
 
-static struct attribute_group power_limit_attribute_group = {
+static const struct attribute_group power_limit_attribute_group = {
 	.attrs = power_limit_attrs,
 	.name = "power_limits"
 };
diff --git a/drivers/thermal/intel_pch_thermal.c b/drivers/thermal/intel_pch_thermal.c
index 2b49e8d0fe9e..c60b1cfcc64e 100644
--- a/drivers/thermal/intel_pch_thermal.c
+++ b/drivers/thermal/intel_pch_thermal.c
@@ -49,7 +49,7 @@
 #define WPT_TSGPEN	0x84	/* General Purpose Event Enables */
 
 /*  Wildcat Point-LP  PCH Thermal Register bit definitions */
-#define WPT_TEMP_TSR	0x00ff	/* Temp TS Reading */
+#define WPT_TEMP_TSR	0x01ff	/* Temp TS Reading */
 #define WPT_TSC_CPDE	0x01	/* Catastrophic Power-Down Enable */
 #define WPT_TSS_TSDSS	0x10	/* Thermal Sensor Dynamic Shutdown Status */
 #define WPT_TSS_GPES	0x08	/* GPE status */
@@ -125,7 +125,7 @@ static int pch_wpt_init(struct pch_thermal_device *ptd, int *nr_trips)
 	*nr_trips = 0;
 
 	/* Check if BIOS has already enabled thermal sensor */
-	if (WPT_TSS_TSDSS & readb(ptd->hw_base + WPT_TSS)) {
+	if (WPT_TSEL_ETS & readb(ptd->hw_base + WPT_TSEL)) {
 		ptd->bios_enabled = true;
 		goto read_trips;
 	}
@@ -141,7 +141,7 @@ static int pch_wpt_init(struct pch_thermal_device *ptd, int *nr_trips)
 	}
 
 	writeb(tsel|WPT_TSEL_ETS, ptd->hw_base + WPT_TSEL);
-	if (!(WPT_TSS_TSDSS & readb(ptd->hw_base + WPT_TSS))) {
+	if (!(WPT_TSEL_ETS & readb(ptd->hw_base + WPT_TSEL))) {
 		dev_err(&ptd->pdev->dev, "Sensor can't be enabled\n");
 		return -ENODEV;
 	}
@@ -174,9 +174,9 @@ read_trips:
 
 static int pch_wpt_get_temp(struct pch_thermal_device *ptd, int *temp)
 {
-	u8 wpt_temp;
+	u16 wpt_temp;
 
-	wpt_temp = WPT_TEMP_TSR & readl(ptd->hw_base + WPT_TEMP);
+	wpt_temp = WPT_TEMP_TSR & readw(ptd->hw_base + WPT_TEMP);
 
 	/* Resolution of 1/2 degree C and an offset of -50C */
 	*temp = (wpt_temp * 1000 / 2 - 50000);
@@ -387,7 +387,7 @@ static int intel_pch_thermal_resume(struct device *device)
 	return ptd->ops->resume(ptd);
 }
 
-static struct pci_device_id intel_pch_thermal_id[] = {
+static const struct pci_device_id intel_pch_thermal_id[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCH_THERMAL_DID_HSW_1),
 		.driver_data = board_hsw, },
 	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCH_THERMAL_DID_HSW_2),
diff --git a/drivers/thermal/mtk_thermal.c b/drivers/thermal/mtk_thermal.c
index 7737f14846f9..1e61c09153c9 100644
--- a/drivers/thermal/mtk_thermal.c
+++ b/drivers/thermal/mtk_thermal.c
@@ -3,6 +3,7 @@
  * Author: Hanyi Wu <hanyi.wu@mediatek.com>
  *         Sascha Hauer <s.hauer@pengutronix.de>
  *         Dawei Chien <dawei.chien@mediatek.com>
+ *         Louis Yu <louis.yu@mediatek.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -111,9 +112,10 @@
 
 /*
  * Layout of the fuses providing the calibration data
- * These macros could be used for both MT8173 and MT2701.
- * MT8173 has five sensors and need five VTS calibration data,
- * and MT2701 has three sensors and need three VTS calibration data.
+ * These macros could be used for MT8173, MT2701, and MT2712.
+ * MT8173 has 5 sensors and needs 5 VTS calibration data.
+ * MT2701 has 3 sensors and needs 3 VTS calibration data.
+ * MT2712 has 4 sensors and needs 4 VTS calibration data.
  */
 #define MT8173_CALIB_BUF0_VALID		BIT(0)
 #define MT8173_CALIB_BUF1_ADC_GE(x)	(((x) >> 22) & 0x3ff)
@@ -124,6 +126,8 @@
 #define MT8173_CALIB_BUF2_VTS_TSABB(x)	(((x) >> 14) & 0x1ff)
 #define MT8173_CALIB_BUF0_DEGC_CALI(x)	(((x) >> 1) & 0x3f)
 #define MT8173_CALIB_BUF0_O_SLOPE(x)	(((x) >> 26) & 0x3f)
+#define MT8173_CALIB_BUF0_O_SLOPE_SIGN(x)	(((x) >> 7) & 0x1)
+#define MT8173_CALIB_BUF1_ID(x)	(((x) >> 9) & 0x1)
 
 /* MT2701 thermal sensors */
 #define MT2701_TS1	0
@@ -136,11 +140,26 @@
 /* The total number of temperature sensors in the MT2701 */
 #define MT2701_NUM_SENSORS	3
 
-#define THERMAL_NAME    "mtk-thermal"
-
 /* The number of sensing points per bank */
 #define MT2701_NUM_SENSORS_PER_ZONE	3
 
+/* MT2712 thermal sensors */
+#define MT2712_TS1	0
+#define MT2712_TS2	1
+#define MT2712_TS3	2
+#define MT2712_TS4	3
+
+/* AUXADC channel 11 is used for the temperature sensors */
+#define MT2712_TEMP_AUXADC_CHANNEL	11
+
+/* The total number of temperature sensors in the MT2712 */
+#define MT2712_NUM_SENSORS	4
+
+/* The number of sensing points per bank */
+#define MT2712_NUM_SENSORS_PER_ZONE	4
+
+#define THERMAL_NAME    "mtk-thermal"
+
 struct mtk_thermal;
 
 struct thermal_bank_cfg {
@@ -215,6 +234,21 @@ static const int mt2701_adcpnp[MT2701_NUM_SENSORS_PER_ZONE] = {
 
 static const int mt2701_mux_values[MT2701_NUM_SENSORS] = { 0, 1, 16 };
 
+/* MT2712 thermal sensor data */
+static const int mt2712_bank_data[MT2712_NUM_SENSORS] = {
+	MT2712_TS1, MT2712_TS2, MT2712_TS3, MT2712_TS4
+};
+
+static const int mt2712_msr[MT2712_NUM_SENSORS_PER_ZONE] = {
+	TEMP_MSR0, TEMP_MSR1, TEMP_MSR2, TEMP_MSR3
+};
+
+static const int mt2712_adcpnp[MT2712_NUM_SENSORS_PER_ZONE] = {
+	TEMP_ADCPNP0, TEMP_ADCPNP1, TEMP_ADCPNP2, TEMP_ADCPNP3
+};
+
+static const int mt2712_mux_values[MT2712_NUM_SENSORS] = { 0, 1, 2, 3 };
+
 /**
  * The MT8173 thermal controller has four banks. Each bank can read up to
  * four temperature sensors simultaneously. The MT8173 has a total of 5
@@ -278,6 +312,31 @@ static const struct mtk_thermal_data mt2701_thermal_data = {
 };
 
 /**
+ * The MT2712 thermal controller has one bank, which can read up to
+ * four temperature sensors simultaneously. The MT2712 has a total of 4
+ * temperature sensors.
+ *
+ * The thermal core only gets the maximum temperature of this one bank,
+ * so the bank concept wouldn't be necessary here. However, the SVS (Smart
+ * Voltage Scaling) unit makes its decisions based on the same bank
+ * data.
+ */
+static const struct mtk_thermal_data mt2712_thermal_data = {
+	.auxadc_channel = MT2712_TEMP_AUXADC_CHANNEL,
+	.num_banks = 1,
+	.num_sensors = MT2712_NUM_SENSORS,
+	.bank_data = {
+		{
+			.num_sensors = 4,
+			.sensors = mt2712_bank_data,
+		},
+	},
+	.msr = mt2712_msr,
+	.adcpnp = mt2712_adcpnp,
+	.sensor_mux_values = mt2712_mux_values,
+};
+
+/**
  * raw_to_mcelsius - convert a raw ADC value to mcelsius
  * @mt:		The thermal controller
  * @raw:	raw ADC value
@@ -552,7 +611,11 @@ static int mtk_thermal_get_calibration_data(struct device *dev,
 		mt->vts[MT8173_TS4] = MT8173_CALIB_BUF2_VTS_TS4(buf[2]);
 		mt->vts[MT8173_TSABB] = MT8173_CALIB_BUF2_VTS_TSABB(buf[2]);
 		mt->degc_cali = MT8173_CALIB_BUF0_DEGC_CALI(buf[0]);
-		mt->o_slope = MT8173_CALIB_BUF0_O_SLOPE(buf[0]);
+		if (MT8173_CALIB_BUF1_ID(buf[1]) &
+		    MT8173_CALIB_BUF0_O_SLOPE_SIGN(buf[0]))
+			mt->o_slope = -MT8173_CALIB_BUF0_O_SLOPE(buf[0]);
+		else
+			mt->o_slope = MT8173_CALIB_BUF0_O_SLOPE(buf[0]);
 	} else {
 		dev_info(dev, "Device not calibrated, using default calibration values\n");
 	}
@@ -571,6 +634,10 @@ static const struct of_device_id mtk_thermal_of_match[] = {
 	{
 		.compatible = "mediatek,mt2701-thermal",
 		.data = (void *)&mt2701_thermal_data,
+	},
+	{
+		.compatible = "mediatek,mt2712-thermal",
+		.data = (void *)&mt2712_thermal_data,
 	}, {
 	},
 };
@@ -645,16 +712,16 @@ static int mtk_thermal_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
+	ret = device_reset(&pdev->dev);
+	if (ret)
+		return ret;
+
 	ret = clk_prepare_enable(mt->clk_auxadc);
 	if (ret) {
 		dev_err(&pdev->dev, "Can't enable auxadc clk: %d\n", ret);
 		return ret;
 	}
 
-	ret = device_reset(&pdev->dev);
-	if (ret)
-		goto err_disable_clk_auxadc;
-
 	ret = clk_prepare_enable(mt->clk_peri_therm);
 	if (ret) {
 		dev_err(&pdev->dev, "Can't enable peri clk: %d\n", ret);
@@ -705,6 +772,7 @@ static struct platform_driver mtk_thermal_driver = {
 
 module_platform_driver(mtk_thermal_driver);
 
+MODULE_AUTHOR("Louis Yu <louis.yu@mediatek.com>");
 MODULE_AUTHOR("Dawei Chien <dawei.chien@mediatek.com>");
 MODULE_AUTHOR("Sascha Hauer <s.hauer@pengutronix.de>");
 MODULE_AUTHOR("Hanyi Wu <hanyi.wu@mediatek.com>");
diff --git a/drivers/thermal/qoriq_thermal.c b/drivers/thermal/qoriq_thermal.c
index 4362a69ac88d..c866cc165960 100644
--- a/drivers/thermal/qoriq_thermal.c
+++ b/drivers/thermal/qoriq_thermal.c
@@ -188,7 +188,7 @@ static void qoriq_tmu_init_device(struct qoriq_tmu_data *data)
 	tmu_write(data, TMR_DISABLE, &data->regs->tmr);
 }
 
-static struct thermal_zone_of_device_ops tmu_tz_ops = {
+static const struct thermal_zone_of_device_ops tmu_tz_ops = {
 	.get_temp = tmu_get_temp,
 };
 
diff --git a/drivers/thermal/rcar_gen3_thermal.c b/drivers/thermal/rcar_gen3_thermal.c
index 37fcefd06d9f..203aca44a2bb 100644
--- a/drivers/thermal/rcar_gen3_thermal.c
+++ b/drivers/thermal/rcar_gen3_thermal.c
@@ -225,7 +225,7 @@ static int rcar_gen3_thermal_set_trips(void *devdata, int low, int high)
 	return 0;
 }
 
-static struct thermal_zone_of_device_ops rcar_gen3_tz_of_ops = {
+static const struct thermal_zone_of_device_ops rcar_gen3_tz_of_ops = {
 	.get_temp	= rcar_gen3_thermal_get_temp,
 	.set_trips	= rcar_gen3_thermal_set_trips,
 };
diff --git a/drivers/thermal/rockchip_thermal.c b/drivers/thermal/rockchip_thermal.c
index 4c7796512453..206035139110 100644
--- a/drivers/thermal/rockchip_thermal.c
+++ b/drivers/thermal/rockchip_thermal.c
@@ -320,6 +320,44 @@ static const struct tsadc_table rk3288_code_table[] = {
 	{0, 125000},
 };
 
+static const struct tsadc_table rk3328_code_table[] = {
+	{0, -40000},
+	{296, -40000},
+	{304, -35000},
+	{313, -30000},
+	{331, -20000},
+	{340, -15000},
+	{349, -10000},
+	{359, -5000},
+	{368, 0},
+	{378, 5000},
+	{388, 10000},
+	{398, 15000},
+	{408, 20000},
+	{418, 25000},
+	{429, 30000},
+	{440, 35000},
+	{451, 40000},
+	{462, 45000},
+	{473, 50000},
+	{485, 55000},
+	{496, 60000},
+	{508, 65000},
+	{521, 70000},
+	{533, 75000},
+	{546, 80000},
+	{559, 85000},
+	{572, 90000},
+	{586, 95000},
+	{600, 100000},
+	{614, 105000},
+	{629, 110000},
+	{644, 115000},
+	{659, 120000},
+	{675, 125000},
+	{TSADCV2_DATA_MASK, 125000},
+};
+
 static const struct tsadc_table rk3368_code_table[] = {
 	{0, -40000},
 	{106, -40000},
@@ -790,6 +828,29 @@ static const struct rockchip_tsadc_chip rk3288_tsadc_data = {
 	},
 };
 
+static const struct rockchip_tsadc_chip rk3328_tsadc_data = {
+	.chn_id[SENSOR_CPU] = 0, /* cpu sensor is channel 0 */
+	.chn_num = 1, /* one channels for tsadc */
+
+	.tshut_mode = TSHUT_MODE_CRU, /* default TSHUT via CRU */
+	.tshut_temp = 95000,
+
+	.initialize = rk_tsadcv2_initialize,
+	.irq_ack = rk_tsadcv3_irq_ack,
+	.control = rk_tsadcv3_control,
+	.get_temp = rk_tsadcv2_get_temp,
+	.set_alarm_temp = rk_tsadcv2_alarm_temp,
+	.set_tshut_temp = rk_tsadcv2_tshut_temp,
+	.set_tshut_mode = rk_tsadcv2_tshut_mode,
+
+	.table = {
+		.id = rk3328_code_table,
+		.length = ARRAY_SIZE(rk3328_code_table),
+		.data_mask = TSADCV2_DATA_MASK,
+		.mode = ADC_INCREMENT,
+	},
+};
+
 static const struct rockchip_tsadc_chip rk3366_tsadc_data = {
 	.chn_id[SENSOR_CPU] = 0, /* cpu sensor is channel 0 */
 	.chn_id[SENSOR_GPU] = 1, /* gpu sensor is channel 1 */
@@ -875,6 +936,10 @@ static const struct of_device_id of_rockchip_thermal_match[] = {
 		.data = (void *)&rk3288_tsadc_data,
 	},
 	{
+		.compatible = "rockchip,rk3328-tsadc",
+		.data = (void *)&rk3328_tsadc_data,
+	},
+	{
 		.compatible = "rockchip,rk3366-tsadc",
 		.data = (void *)&rk3366_tsadc_data,
 	},
diff --git a/drivers/thermal/samsung/exynos_tmu.c b/drivers/thermal/samsung/exynos_tmu.c
index 7b8ef09d2b3c..ed805c7c5ace 100644
--- a/drivers/thermal/samsung/exynos_tmu.c
+++ b/drivers/thermal/samsung/exynos_tmu.c
@@ -1286,7 +1286,7 @@ static int exynos_map_dt_data(struct platform_device *pdev)
 	return 0;
 }
 
-static struct thermal_zone_of_device_ops exynos_sensor_ops = {
+static const struct thermal_zone_of_device_ops exynos_sensor_ops = {
 	.get_temp = exynos_get_temp,
 	.set_emul_temp = exynos_tmu_set_emulation,
 };
diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c
index 5a51c740e372..2b1b0ba393a4 100644
--- a/drivers/thermal/thermal_core.c
+++ b/drivers/thermal/thermal_core.c
@@ -390,7 +390,7 @@ static void handle_critical_trips(struct thermal_zone_device *tz,
 
 	if (trip_type == THERMAL_TRIP_CRITICAL) {
 		dev_emerg(&tz->device,
-			  "critical temperature reached(%d C),shutting down\n",
+			  "critical temperature reached (%d C), shutting down\n",
 			  tz->temperature / 1000);
 		mutex_lock(&poweroff_lock);
 		if (!power_off_triggered) {
@@ -836,11 +836,7 @@ static void thermal_release(struct device *dev)
 	if (!strncmp(dev_name(dev), "thermal_zone",
 		     sizeof("thermal_zone") - 1)) {
 		tz = to_thermal_zone(dev);
-		kfree(tz->trip_type_attrs);
-		kfree(tz->trip_temp_attrs);
-		kfree(tz->trip_hyst_attrs);
-		kfree(tz->trips_attribute_group.attrs);
-		kfree(tz->device.groups);
+		thermal_zone_destroy_device_groups(tz);
 		kfree(tz);
 	} else if (!strncmp(dev_name(dev), "cooling_device",
 			    sizeof("cooling_device") - 1)) {
@@ -1213,10 +1209,8 @@ thermal_zone_device_register(const char *type, int trips, int mask,
 	ida_init(&tz->ida);
 	mutex_init(&tz->lock);
 	result = ida_simple_get(&thermal_tz_ida, 0, 0, GFP_KERNEL);
-	if (result < 0) {
-		kfree(tz);
-		return ERR_PTR(result);
-	}
+	if (result < 0)
+		goto free_tz;
 
 	tz->id = result;
 	strlcpy(tz->type, type, sizeof(tz->type));
@@ -1232,18 +1226,15 @@ thermal_zone_device_register(const char *type, int trips, int mask,
 	/* Add nodes that are always present via .groups */
 	result = thermal_zone_create_device_groups(tz, mask);
 	if (result)
-		goto unregister;
+		goto remove_id;
 
 	/* A new thermal zone needs to be updated anyway. */
 	atomic_set(&tz->need_update, 1);
 
 	dev_set_name(&tz->device, "thermal_zone%d", tz->id);
 	result = device_register(&tz->device);
-	if (result) {
-		ida_simple_remove(&thermal_tz_ida, tz->id);
-		kfree(tz);
-		return ERR_PTR(result);
-	}
+	if (result)
+		goto remove_device_groups;
 
 	for (count = 0; count < trips; count++) {
 		if (tz->ops->get_trip_type(tz, count, &trip_type))
@@ -1297,6 +1288,14 @@ unregister:
 	ida_simple_remove(&thermal_tz_ida, tz->id);
 	device_unregister(&tz->device);
 	return ERR_PTR(result);
+
+remove_device_groups:
+	thermal_zone_destroy_device_groups(tz);
+remove_id:
+	ida_simple_remove(&thermal_tz_ida, tz->id);
+free_tz:
+	kfree(tz);
+	return ERR_PTR(result);
 }
 EXPORT_SYMBOL_GPL(thermal_zone_device_register);
 
diff --git a/drivers/thermal/thermal_core.h b/drivers/thermal/thermal_core.h
index 2412b3759e16..27e3b1df7360 100644
--- a/drivers/thermal/thermal_core.h
+++ b/drivers/thermal/thermal_core.h
@@ -71,6 +71,7 @@ int thermal_build_list_of_policies(char *buf);
 
 /* sysfs I/F */
 int thermal_zone_create_device_groups(struct thermal_zone_device *, int);
+void thermal_zone_destroy_device_groups(struct thermal_zone_device *);
 void thermal_cooling_device_setup_sysfs(struct thermal_cooling_device *);
 /* used only at binding time */
 ssize_t
diff --git a/drivers/thermal/thermal_sysfs.c b/drivers/thermal/thermal_sysfs.c
index a694de907a26..fb80c96d8f73 100644
--- a/drivers/thermal/thermal_sysfs.c
+++ b/drivers/thermal/thermal_sysfs.c
@@ -605,6 +605,24 @@ static int create_trip_attrs(struct thermal_zone_device *tz, int mask)
 	return 0;
 }
 
+/**
+ * destroy_trip_attrs() - destroy attributes for trip points
+ * @tz:		the thermal zone device
+ *
+ * helper function to free resources allocated by create_trip_attrs()
+ */
+static void destroy_trip_attrs(struct thermal_zone_device *tz)
+{
+	if (!tz)
+		return;
+
+	kfree(tz->trip_type_attrs);
+	kfree(tz->trip_temp_attrs);
+	if (tz->ops->get_trip_hyst)
+		kfree(tz->trip_hyst_attrs);
+	kfree(tz->trips_attribute_group.attrs);
+}
+
 int thermal_zone_create_device_groups(struct thermal_zone_device *tz,
 				      int mask)
 {
@@ -637,6 +655,17 @@ int thermal_zone_create_device_groups(struct thermal_zone_device *tz,
 	return 0;
 }
 
+void thermal_zone_destroy_device_groups(struct thermal_zone_device *tz)
+{
+	if (!tz)
+		return;
+
+	if (tz->trips)
+		destroy_trip_attrs(tz);
+
+	kfree(tz->device.groups);
+}
+
 /* sys I/F for cooling device */
 static ssize_t
 thermal_cooling_device_type_show(struct device *dev,
diff --git a/drivers/thermal/uniphier_thermal.c b/drivers/thermal/uniphier_thermal.c
new file mode 100644
index 000000000000..95704732f760
--- /dev/null
+++ b/drivers/thermal/uniphier_thermal.c
@@ -0,0 +1,384 @@
+/**
+ * uniphier_thermal.c - Socionext UniPhier thermal driver
+ *
+ * Copyright 2014      Panasonic Corporation
+ * Copyright 2016-2017 Socionext Inc.
+ * All rights reserved.
+ *
+ * Author:
+ *	Kunihiko Hayashi <hayashi.kunihiko@socionext.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2  of
+ * the License as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/bitops.h>
+#include <linux/interrupt.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/thermal.h>
+
+#include "thermal_core.h"
+
+/*
+ * block registers
+ * addresses are the offset from .block_base
+ */
+#define PVTCTLEN			0x0000
+#define PVTCTLEN_EN			BIT(0)
+
+#define PVTCTLMODE			0x0004
+#define PVTCTLMODE_MASK			0xf
+#define PVTCTLMODE_TEMPMON		0x5
+
+#define EMONREPEAT			0x0040
+#define EMONREPEAT_ENDLESS		BIT(24)
+#define EMONREPEAT_PERIOD		GENMASK(3, 0)
+#define EMONREPEAT_PERIOD_1000000	0x9
+
+/*
+ * common registers
+ * addresses are the offset from .map_base
+ */
+#define PVTCTLSEL			0x0900
+#define PVTCTLSEL_MASK			GENMASK(2, 0)
+#define PVTCTLSEL_MONITOR		0
+
+#define SETALERT0			0x0910
+#define SETALERT1			0x0914
+#define SETALERT2			0x0918
+#define SETALERT_TEMP_OVF		(GENMASK(7, 0) << 16)
+#define SETALERT_TEMP_OVF_VALUE(val)	(((val) & GENMASK(7, 0)) << 16)
+#define SETALERT_EN			BIT(0)
+
+#define PMALERTINTCTL			0x0920
+#define PMALERTINTCTL_CLR(ch)		BIT(4 * (ch) + 2)
+#define PMALERTINTCTL_SET(ch)		BIT(4 * (ch) + 1)
+#define PMALERTINTCTL_EN(ch)		BIT(4 * (ch) + 0)
+#define PMALERTINTCTL_MASK		(GENMASK(10, 8) | GENMASK(6, 4) | \
+					 GENMASK(2, 0))
+
+#define TMOD				0x0928
+#define TMOD_WIDTH			9
+
+#define TMODCOEF			0x0e5c
+
+#define TMODSETUP0_EN			BIT(30)
+#define TMODSETUP0_VAL(val)		(((val) & GENMASK(13, 0)) << 16)
+#define TMODSETUP1_EN			BIT(15)
+#define TMODSETUP1_VAL(val)		((val) & GENMASK(14, 0))
+
+/* SoC critical temperature */
+#define CRITICAL_TEMP_LIMIT		(120 * 1000)
+
+/* Max # of alert channels */
+#define ALERT_CH_NUM			3
+
+/* SoC specific thermal sensor data */
+struct uniphier_tm_soc_data {
+	u32 map_base;
+	u32 block_base;
+	u32 tmod_setup_addr;
+};
+
+struct uniphier_tm_dev {
+	struct regmap *regmap;
+	struct device *dev;
+	bool alert_en[ALERT_CH_NUM];
+	struct thermal_zone_device *tz_dev;
+	const struct uniphier_tm_soc_data *data;
+};
+
+static int uniphier_tm_initialize_sensor(struct uniphier_tm_dev *tdev)
+{
+	struct regmap *map = tdev->regmap;
+	u32 val;
+	u32 tmod_calib[2];
+	int ret;
+
+	/* stop PVT */
+	regmap_write_bits(map, tdev->data->block_base + PVTCTLEN,
+			  PVTCTLEN_EN, 0);
+
+	/*
+	 * Since SoC has a calibrated value that was set in advance,
+	 * TMODCOEF shows non-zero and PVT refers the value internally.
+	 *
+	 * If TMODCOEF shows zero, the boards don't have the calibrated
+	 * value, and the driver has to set default value from DT.
+	 */
+	ret = regmap_read(map, tdev->data->map_base + TMODCOEF, &val);
+	if (ret)
+		return ret;
+	if (!val) {
+		/* look for the default values in DT */
+		ret = of_property_read_u32_array(tdev->dev->of_node,
+						 "socionext,tmod-calibration",
+						 tmod_calib,
+						 ARRAY_SIZE(tmod_calib));
+		if (ret)
+			return ret;
+
+		regmap_write(map, tdev->data->tmod_setup_addr,
+			TMODSETUP0_EN | TMODSETUP0_VAL(tmod_calib[0]) |
+			TMODSETUP1_EN | TMODSETUP1_VAL(tmod_calib[1]));
+	}
+
+	/* select temperature mode */
+	regmap_write_bits(map, tdev->data->block_base + PVTCTLMODE,
+			  PVTCTLMODE_MASK, PVTCTLMODE_TEMPMON);
+
+	/* set monitoring period */
+	regmap_write_bits(map, tdev->data->block_base + EMONREPEAT,
+			  EMONREPEAT_ENDLESS | EMONREPEAT_PERIOD,
+			  EMONREPEAT_ENDLESS | EMONREPEAT_PERIOD_1000000);
+
+	/* set monitor mode */
+	regmap_write_bits(map, tdev->data->map_base + PVTCTLSEL,
+			  PVTCTLSEL_MASK, PVTCTLSEL_MONITOR);
+
+	return 0;
+}
+
+static void uniphier_tm_set_alert(struct uniphier_tm_dev *tdev, u32 ch,
+				  u32 temp)
+{
+	struct regmap *map = tdev->regmap;
+
+	/* set alert temperature */
+	regmap_write_bits(map, tdev->data->map_base + SETALERT0 + (ch << 2),
+			  SETALERT_EN | SETALERT_TEMP_OVF,
+			  SETALERT_EN |
+			  SETALERT_TEMP_OVF_VALUE(temp / 1000));
+}
+
+static void uniphier_tm_enable_sensor(struct uniphier_tm_dev *tdev)
+{
+	struct regmap *map = tdev->regmap;
+	int i;
+	u32 bits = 0;
+
+	for (i = 0; i < ALERT_CH_NUM; i++)
+		if (tdev->alert_en[i])
+			bits |= PMALERTINTCTL_EN(i);
+
+	/* enable alert interrupt */
+	regmap_write_bits(map, tdev->data->map_base + PMALERTINTCTL,
+			  PMALERTINTCTL_MASK, bits);
+
+	/* start PVT */
+	regmap_write_bits(map, tdev->data->block_base + PVTCTLEN,
+			  PVTCTLEN_EN, PVTCTLEN_EN);
+
+	usleep_range(700, 1500);	/* The spec note says at least 700us */
+}
+
+static void uniphier_tm_disable_sensor(struct uniphier_tm_dev *tdev)
+{
+	struct regmap *map = tdev->regmap;
+
+	/* disable alert interrupt */
+	regmap_write_bits(map, tdev->data->map_base + PMALERTINTCTL,
+			  PMALERTINTCTL_MASK, 0);
+
+	/* stop PVT */
+	regmap_write_bits(map, tdev->data->block_base + PVTCTLEN,
+			  PVTCTLEN_EN, 0);
+
+	usleep_range(1000, 2000);	/* The spec note says at least 1ms */
+}
+
+static int uniphier_tm_get_temp(void *data, int *out_temp)
+{
+	struct uniphier_tm_dev *tdev = data;
+	struct regmap *map = tdev->regmap;
+	int ret;
+	u32 temp;
+
+	ret = regmap_read(map, tdev->data->map_base + TMOD, &temp);
+	if (ret)
+		return ret;
+
+	/* MSB of the TMOD field is a sign bit */
+	*out_temp = sign_extend32(temp, TMOD_WIDTH - 1) * 1000;
+
+	return 0;
+}
+
+static const struct thermal_zone_of_device_ops uniphier_of_thermal_ops = {
+	.get_temp = uniphier_tm_get_temp,
+};
+
+static void uniphier_tm_irq_clear(struct uniphier_tm_dev *tdev)
+{
+	u32 mask = 0, bits = 0;
+	int i;
+
+	for (i = 0; i < ALERT_CH_NUM; i++) {
+		mask |= (PMALERTINTCTL_CLR(i) | PMALERTINTCTL_SET(i));
+		bits |= PMALERTINTCTL_CLR(i);
+	}
+
+	/* clear alert interrupt */
+	regmap_write_bits(tdev->regmap,
+			  tdev->data->map_base + PMALERTINTCTL, mask, bits);
+}
+
+static irqreturn_t uniphier_tm_alarm_irq(int irq, void *_tdev)
+{
+	struct uniphier_tm_dev *tdev = _tdev;
+
+	disable_irq_nosync(irq);
+	uniphier_tm_irq_clear(tdev);
+
+	return IRQ_WAKE_THREAD;
+}
+
+static irqreturn_t uniphier_tm_alarm_irq_thread(int irq, void *_tdev)
+{
+	struct uniphier_tm_dev *tdev = _tdev;
+
+	thermal_zone_device_update(tdev->tz_dev, THERMAL_EVENT_UNSPECIFIED);
+
+	return IRQ_HANDLED;
+}
+
+static int uniphier_tm_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct regmap *regmap;
+	struct device_node *parent;
+	struct uniphier_tm_dev *tdev;
+	const struct thermal_trip *trips;
+	int i, ret, irq, ntrips, crit_temp = INT_MAX;
+
+	tdev = devm_kzalloc(dev, sizeof(*tdev), GFP_KERNEL);
+	if (!tdev)
+		return -ENOMEM;
+	tdev->dev = dev;
+
+	tdev->data = of_device_get_match_data(dev);
+	if (WARN_ON(!tdev->data))
+		return -EINVAL;
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return irq;
+
+	/* get regmap from syscon node */
+	parent = of_get_parent(dev->of_node); /* parent should be syscon node */
+	regmap = syscon_node_to_regmap(parent);
+	of_node_put(parent);
+	if (IS_ERR(regmap)) {
+		dev_err(dev, "failed to get regmap (error %ld)\n",
+			PTR_ERR(regmap));
+		return PTR_ERR(regmap);
+	}
+	tdev->regmap = regmap;
+
+	ret = uniphier_tm_initialize_sensor(tdev);
+	if (ret) {
+		dev_err(dev, "failed to initialize sensor\n");
+		return ret;
+	}
+
+	ret = devm_request_threaded_irq(dev, irq, uniphier_tm_alarm_irq,
+					uniphier_tm_alarm_irq_thread,
+					0, "thermal", tdev);
+	if (ret)
+		return ret;
+
+	platform_set_drvdata(pdev, tdev);
+
+	tdev->tz_dev = devm_thermal_zone_of_sensor_register(dev, 0, tdev,
+						&uniphier_of_thermal_ops);
+	if (IS_ERR(tdev->tz_dev)) {
+		dev_err(dev, "failed to register sensor device\n");
+		return PTR_ERR(tdev->tz_dev);
+	}
+
+	/* get trip points */
+	trips = of_thermal_get_trip_points(tdev->tz_dev);
+	ntrips = of_thermal_get_ntrips(tdev->tz_dev);
+	if (ntrips > ALERT_CH_NUM) {
+		dev_err(dev, "thermal zone has too many trips\n");
+		return -E2BIG;
+	}
+
+	/* set alert temperatures */
+	for (i = 0; i < ntrips; i++) {
+		if (trips[i].type == THERMAL_TRIP_CRITICAL &&
+		    trips[i].temperature < crit_temp)
+			crit_temp = trips[i].temperature;
+		uniphier_tm_set_alert(tdev, i, trips[i].temperature);
+		tdev->alert_en[i] = true;
+	}
+	if (crit_temp > CRITICAL_TEMP_LIMIT) {
+		dev_err(dev, "critical trip is over limit(>%d), or not set\n",
+			CRITICAL_TEMP_LIMIT);
+		return -EINVAL;
+	}
+
+	uniphier_tm_enable_sensor(tdev);
+
+	return 0;
+}
+
+static int uniphier_tm_remove(struct platform_device *pdev)
+{
+	struct uniphier_tm_dev *tdev = platform_get_drvdata(pdev);
+
+	/* disable sensor */
+	uniphier_tm_disable_sensor(tdev);
+
+	return 0;
+}
+
+static const struct uniphier_tm_soc_data uniphier_pxs2_tm_data = {
+	.map_base        = 0xe000,
+	.block_base      = 0xe000,
+	.tmod_setup_addr = 0xe904,
+};
+
+static const struct uniphier_tm_soc_data uniphier_ld20_tm_data = {
+	.map_base        = 0xe000,
+	.block_base      = 0xe800,
+	.tmod_setup_addr = 0xe938,
+};
+
+static const struct of_device_id uniphier_tm_dt_ids[] = {
+	{
+		.compatible = "socionext,uniphier-pxs2-thermal",
+		.data       = &uniphier_pxs2_tm_data,
+	},
+	{
+		.compatible = "socionext,uniphier-ld20-thermal",
+		.data       = &uniphier_ld20_tm_data,
+	},
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, uniphier_tm_dt_ids);
+
+static struct platform_driver uniphier_tm_driver = {
+	.probe = uniphier_tm_probe,
+	.remove = uniphier_tm_remove,
+	.driver = {
+		.name = "uniphier-thermal",
+		.of_match_table = uniphier_tm_dt_ids,
+	},
+};
+module_platform_driver(uniphier_tm_driver);
+
+MODULE_AUTHOR("Kunihiko Hayashi <hayashi.kunihiko@socionext.com>");
+MODULE_DESCRIPTION("UniPhier thermal driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/thermal/zx2967_thermal.c b/drivers/thermal/zx2967_thermal.c
index a5670ad2cfc8..6acce0bce7c0 100644
--- a/drivers/thermal/zx2967_thermal.c
+++ b/drivers/thermal/zx2967_thermal.c
@@ -111,7 +111,7 @@ unlock:
 	return ret;
 }
 
-static struct thermal_zone_of_device_ops zx2967_of_thermal_ops = {
+static const struct thermal_zone_of_device_ops zx2967_of_thermal_ops = {
 	.get_temp = zx2967_thermal_get_temp,
 };
 
diff --git a/drivers/thunderbolt/ctl.c b/drivers/thunderbolt/ctl.c
index 69c0232a22f8..fb40dd0588b9 100644
--- a/drivers/thunderbolt/ctl.c
+++ b/drivers/thunderbolt/ctl.c
@@ -804,7 +804,7 @@ struct tb_cfg_result tb_cfg_reset(struct tb_ctl *ctl, u64 route,
 	req->request_type = TB_CFG_PKG_RESET;
 	req->response = &reply;
 	req->response_size = sizeof(reply);
-	req->response_type = sizeof(TB_CFG_PKG_RESET);
+	req->response_type = TB_CFG_PKG_RESET;
 
 	res = tb_cfg_request_sync(ctl, req, timeout_msec);
 
diff --git a/drivers/thunderbolt/icm.c b/drivers/thunderbolt/icm.c
index bdaac1ff00a5..53250fc057e1 100644
--- a/drivers/thunderbolt/icm.c
+++ b/drivers/thunderbolt/icm.c
@@ -13,9 +13,9 @@
  */
 
 #include <linux/delay.h>
-#include <linux/dmi.h>
 #include <linux/mutex.h>
 #include <linux/pci.h>
+#include <linux/platform_data/x86/apple.h>
 #include <linux/sizes.h>
 #include <linux/slab.h>
 #include <linux/workqueue.h>
@@ -102,11 +102,6 @@ static inline u64 get_route(u32 route_hi, u32 route_lo)
 	return (u64)route_hi << 32 | route_lo;
 }
 
-static inline bool is_apple(void)
-{
-	return dmi_match(DMI_BOARD_VENDOR, "Apple Inc.");
-}
-
 static bool icm_match(const struct tb_cfg_request *req,
 		      const struct ctl_pkg *pkg)
 {
@@ -176,7 +171,7 @@ static int icm_request(struct tb *tb, const void *request, size_t request_size,
 
 static bool icm_fr_is_supported(struct tb *tb)
 {
-	return !is_apple();
+	return !x86_apple_machine;
 }
 
 static inline int icm_fr_get_switch_index(u32 port)
@@ -517,7 +512,7 @@ static bool icm_ar_is_supported(struct tb *tb)
 	 * Starting from Alpine Ridge we can use ICM on Apple machines
 	 * as well. We just need to reset and re-enable it first.
 	 */
-	if (!is_apple())
+	if (!x86_apple_machine)
 		return true;
 
 	/*
@@ -1011,7 +1006,7 @@ static int icm_start(struct tb *tb)
 	 * don't provide images publicly either. To be on the safe side
 	 * prevent root switch NVM upgrade on Macs for now.
 	 */
-	tb->root_switch->no_nvm_upgrade = is_apple();
+	tb->root_switch->no_nvm_upgrade = x86_apple_machine;
 
 	ret = tb_switch_add(tb->root_switch);
 	if (ret)
diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c
index e9391bbd4036..53f40c57df59 100644
--- a/drivers/thunderbolt/switch.c
+++ b/drivers/thunderbolt/switch.c
@@ -807,11 +807,11 @@ static ssize_t key_store(struct device *dev, struct device_attribute *attr,
 	struct tb_switch *sw = tb_to_switch(dev);
 	u8 key[TB_SWITCH_KEY_SIZE];
 	ssize_t ret = count;
+	bool clear = false;
 
-	if (count < 64)
-		return -EINVAL;
-
-	if (hex2bin(key, buf, sizeof(key)))
+	if (!strcmp(buf, "\n"))
+		clear = true;
+	else if (hex2bin(key, buf, sizeof(key)))
 		return -EINVAL;
 
 	if (mutex_lock_interruptible(&switch_lock))
@@ -821,15 +821,19 @@ static ssize_t key_store(struct device *dev, struct device_attribute *attr,
 		ret = -EBUSY;
 	} else {
 		kfree(sw->key);
-		sw->key = kmemdup(key, sizeof(key), GFP_KERNEL);
-		if (!sw->key)
-			ret = -ENOMEM;
+		if (clear) {
+			sw->key = NULL;
+		} else {
+			sw->key = kmemdup(key, sizeof(key), GFP_KERNEL);
+			if (!sw->key)
+				ret = -ENOMEM;
+		}
 	}
 
 	mutex_unlock(&switch_lock);
 	return ret;
 }
-static DEVICE_ATTR_RW(key);
+static DEVICE_ATTR(key, 0600, key_show, key_store);
 
 static ssize_t nvm_authenticate_show(struct device *dev,
 	struct device_attribute *attr, char *buf)
diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c
index 1b02ca0b6129..0b22ad9d68b4 100644
--- a/drivers/thunderbolt/tb.c
+++ b/drivers/thunderbolt/tb.c
@@ -7,7 +7,7 @@
 #include <linux/slab.h>
 #include <linux/errno.h>
 #include <linux/delay.h>
-#include <linux/dmi.h>
+#include <linux/platform_data/x86/apple.h>
 
 #include "tb.h"
 #include "tb_regs.h"
@@ -453,7 +453,7 @@ struct tb *tb_probe(struct tb_nhi *nhi)
 	struct tb_cm *tcm;
 	struct tb *tb;
 
-	if (!dmi_match(DMI_BOARD_VENDOR, "Apple Inc."))
+	if (!x86_apple_machine)
 		return NULL;
 
 	tb = tb_domain_alloc(nhi, sizeof(*tcm));
diff --git a/drivers/tty/Kconfig b/drivers/tty/Kconfig
index 95103054c0e4..cc2b4d9433ed 100644
--- a/drivers/tty/Kconfig
+++ b/drivers/tty/Kconfig
@@ -392,6 +392,9 @@ config PPC_EARLY_DEBUG_EHV_BC_HANDLE
 config GOLDFISH_TTY
 	tristate "Goldfish TTY Driver"
 	depends on GOLDFISH
+	select SERIAL_CORE
+	select SERIAL_CORE_CONSOLE
+	select SERIAL_EARLYCON
 	help
 	  Console and system TTY driver for the Goldfish virtual platform.
 
@@ -455,4 +458,9 @@ config MIPS_EJTAG_FDC_KGDB_CHAN
 	help
 	  FDC channel number to use for KGDB.
 
+config VCC
+	tristate "Sun Virtual Console Concentrator"
+	depends on SUN_LDOMS
+	help
+	  Support for Sun logical domain consoles.
 endif # TTY
diff --git a/drivers/tty/Makefile b/drivers/tty/Makefile
index 8689279afdf1..16330a819685 100644
--- a/drivers/tty/Makefile
+++ b/drivers/tty/Makefile
@@ -33,5 +33,6 @@ obj-$(CONFIG_PPC_EPAPR_HV_BYTECHAN) += ehv_bytechan.o
 obj-$(CONFIG_GOLDFISH_TTY)	+= goldfish.o
 obj-$(CONFIG_DA_TTY)		+= metag_da.o
 obj-$(CONFIG_MIPS_EJTAG_FDC_TTY) += mips_ejtag_fdc.o
+obj-$(CONFIG_VCC)		+= vcc.o
 
 obj-y += ipwireless/
diff --git a/drivers/tty/ehv_bytechan.c b/drivers/tty/ehv_bytechan.c
index 61fe8d6fd24e..a1c7125cb968 100644
--- a/drivers/tty/ehv_bytechan.c
+++ b/drivers/tty/ehv_bytechan.c
@@ -122,7 +122,7 @@ static int find_console_handle(void)
 
 	stdout_irq = irq_of_parse_and_map(np, 0);
 	if (stdout_irq == NO_IRQ) {
-		pr_err("ehv-bc: no 'interrupts' property in %s node\n", np->full_name);
+		pr_err("ehv-bc: no 'interrupts' property in %pOF node\n", np);
 		return 0;
 	}
 
diff --git a/drivers/tty/goldfish.c b/drivers/tty/goldfish.c
index 996bd473dd03..381e981dee06 100644
--- a/drivers/tty/goldfish.c
+++ b/drivers/tty/goldfish.c
@@ -1,6 +1,7 @@
 /*
  * Copyright (C) 2007 Google, Inc.
  * Copyright (C) 2012 Intel, Inc.
+ * Copyright (C) 2017 Imagination Technologies Ltd.
  *
  * This software is licensed under the terms of the GNU General Public
  * License version 2, as published by the Free Software Foundation, and
@@ -22,21 +23,23 @@
 #include <linux/io.h>
 #include <linux/module.h>
 #include <linux/goldfish.h>
-
-enum {
-	GOLDFISH_TTY_PUT_CHAR       = 0x00,
-	GOLDFISH_TTY_BYTES_READY    = 0x04,
-	GOLDFISH_TTY_CMD            = 0x08,
-
-	GOLDFISH_TTY_DATA_PTR       = 0x10,
-	GOLDFISH_TTY_DATA_LEN       = 0x14,
-	GOLDFISH_TTY_DATA_PTR_HIGH  = 0x18,
-
-	GOLDFISH_TTY_CMD_INT_DISABLE    = 0,
-	GOLDFISH_TTY_CMD_INT_ENABLE     = 1,
-	GOLDFISH_TTY_CMD_WRITE_BUFFER   = 2,
-	GOLDFISH_TTY_CMD_READ_BUFFER    = 3,
-};
+#include <linux/mm.h>
+#include <linux/dma-mapping.h>
+#include <linux/serial_core.h>
+
+/* Goldfish tty register's offsets */
+#define	GOLDFISH_TTY_REG_BYTES_READY	0x04
+#define	GOLDFISH_TTY_REG_CMD		0x08
+#define	GOLDFISH_TTY_REG_DATA_PTR	0x10
+#define	GOLDFISH_TTY_REG_DATA_LEN	0x14
+#define	GOLDFISH_TTY_REG_DATA_PTR_HIGH	0x18
+#define	GOLDFISH_TTY_REG_VERSION	0x20
+
+/* Goldfish tty commands */
+#define	GOLDFISH_TTY_CMD_INT_DISABLE	0
+#define	GOLDFISH_TTY_CMD_INT_ENABLE	1
+#define	GOLDFISH_TTY_CMD_WRITE_BUFFER	2
+#define	GOLDFISH_TTY_CMD_READ_BUFFER	3
 
 struct goldfish_tty {
 	struct tty_port port;
@@ -45,6 +48,8 @@ struct goldfish_tty {
 	u32 irq;
 	int opencount;
 	struct console console;
+	u32 version;
+	struct device *dev;
 };
 
 static DEFINE_MUTEX(goldfish_tty_lock);
@@ -53,38 +58,107 @@ static u32 goldfish_tty_line_count = 8;
 static u32 goldfish_tty_current_line_count;
 static struct goldfish_tty *goldfish_ttys;
 
-static void goldfish_tty_do_write(int line, const char *buf, unsigned count)
+static void do_rw_io(struct goldfish_tty *qtty,
+		     unsigned long address,
+		     unsigned int count,
+		     int is_write)
 {
 	unsigned long irq_flags;
-	struct goldfish_tty *qtty = &goldfish_ttys[line];
 	void __iomem *base = qtty->base;
+
 	spin_lock_irqsave(&qtty->lock, irq_flags);
-	gf_write_ptr(buf, base + GOLDFISH_TTY_DATA_PTR,
-				base + GOLDFISH_TTY_DATA_PTR_HIGH);
-	writel(count, base + GOLDFISH_TTY_DATA_LEN);
-	writel(GOLDFISH_TTY_CMD_WRITE_BUFFER, base + GOLDFISH_TTY_CMD);
+	gf_write_ptr((void *)address, base + GOLDFISH_TTY_REG_DATA_PTR,
+		     base + GOLDFISH_TTY_REG_DATA_PTR_HIGH);
+	writel(count, base + GOLDFISH_TTY_REG_DATA_LEN);
+
+	if (is_write)
+		writel(GOLDFISH_TTY_CMD_WRITE_BUFFER,
+		       base + GOLDFISH_TTY_REG_CMD);
+	else
+		writel(GOLDFISH_TTY_CMD_READ_BUFFER,
+		       base + GOLDFISH_TTY_REG_CMD);
+
 	spin_unlock_irqrestore(&qtty->lock, irq_flags);
 }
 
+static void goldfish_tty_rw(struct goldfish_tty *qtty,
+			    unsigned long addr,
+			    unsigned int count,
+			    int is_write)
+{
+	dma_addr_t dma_handle;
+	enum dma_data_direction dma_dir;
+
+	dma_dir = (is_write ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
+	if (qtty->version > 0) {
+		/*
+		 * Goldfish TTY for Ranchu platform uses
+		 * physical addresses and DMA for read/write operations
+		 */
+		unsigned long addr_end = addr + count;
+
+		while (addr < addr_end) {
+			unsigned long pg_end = (addr & PAGE_MASK) + PAGE_SIZE;
+			unsigned long next =
+					pg_end < addr_end ? pg_end : addr_end;
+			unsigned long avail = next - addr;
+
+			/*
+			 * Map the buffer's virtual address to the DMA address
+			 * so the buffer can be accessed by the device.
+			 */
+			dma_handle = dma_map_single(qtty->dev, (void *)addr,
+						    avail, dma_dir);
+
+			if (dma_mapping_error(qtty->dev, dma_handle)) {
+				dev_err(qtty->dev, "tty: DMA mapping error.\n");
+				return;
+			}
+			do_rw_io(qtty, dma_handle, avail, is_write);
+
+			/*
+			 * Unmap the previously mapped region after
+			 * the completion of the read/write operation.
+			 */
+			dma_unmap_single(qtty->dev, dma_handle, avail, dma_dir);
+
+			addr += avail;
+		}
+	} else {
+		/*
+		 * Old style Goldfish TTY used on the Goldfish platform
+		 * uses virtual addresses.
+		 */
+		do_rw_io(qtty, addr, count, is_write);
+	}
+}
+
+static void goldfish_tty_do_write(int line, const char *buf,
+				  unsigned int count)
+{
+	struct goldfish_tty *qtty = &goldfish_ttys[line];
+	unsigned long address = (unsigned long)(void *)buf;
+
+	goldfish_tty_rw(qtty, address, count, 1);
+}
+
 static irqreturn_t goldfish_tty_interrupt(int irq, void *dev_id)
 {
 	struct goldfish_tty *qtty = dev_id;
 	void __iomem *base = qtty->base;
-	unsigned long irq_flags;
+	unsigned long address;
 	unsigned char *buf;
 	u32 count;
 
-	count = readl(base + GOLDFISH_TTY_BYTES_READY);
+	count = readl(base + GOLDFISH_TTY_REG_BYTES_READY);
 	if (count == 0)
 		return IRQ_NONE;
 
 	count = tty_prepare_flip_string(&qtty->port, &buf, count);
-	spin_lock_irqsave(&qtty->lock, irq_flags);
-	gf_write_ptr(buf, base + GOLDFISH_TTY_DATA_PTR,
-				base + GOLDFISH_TTY_DATA_PTR_HIGH);
-	writel(count, base + GOLDFISH_TTY_DATA_LEN);
-	writel(GOLDFISH_TTY_CMD_READ_BUFFER, base + GOLDFISH_TTY_CMD);
-	spin_unlock_irqrestore(&qtty->lock, irq_flags);
+
+	address = (unsigned long)(void *)buf;
+	goldfish_tty_rw(qtty, address, count, 0);
+
 	tty_schedule_flip(&qtty->port);
 	return IRQ_HANDLED;
 }
@@ -93,7 +167,7 @@ static int goldfish_tty_activate(struct tty_port *port, struct tty_struct *tty)
 {
 	struct goldfish_tty *qtty = container_of(port, struct goldfish_tty,
 									port);
-	writel(GOLDFISH_TTY_CMD_INT_ENABLE, qtty->base + GOLDFISH_TTY_CMD);
+	writel(GOLDFISH_TTY_CMD_INT_ENABLE, qtty->base + GOLDFISH_TTY_REG_CMD);
 	return 0;
 }
 
@@ -101,7 +175,7 @@ static void goldfish_tty_shutdown(struct tty_port *port)
 {
 	struct goldfish_tty *qtty = container_of(port, struct goldfish_tty,
 									port);
-	writel(GOLDFISH_TTY_CMD_INT_DISABLE, qtty->base + GOLDFISH_TTY_CMD);
+	writel(GOLDFISH_TTY_CMD_INT_DISABLE, qtty->base + GOLDFISH_TTY_REG_CMD);
 }
 
 static int goldfish_tty_open(struct tty_struct *tty, struct file *filp)
@@ -136,7 +210,7 @@ static int goldfish_tty_chars_in_buffer(struct tty_struct *tty)
 {
 	struct goldfish_tty *qtty = &goldfish_ttys[tty->index];
 	void __iomem *base = qtty->base;
-	return readl(base + GOLDFISH_TTY_BYTES_READY);
+	return readl(base + GOLDFISH_TTY_REG_BYTES_READY);
 }
 
 static void goldfish_tty_console_write(struct console *co, const char *b,
@@ -227,7 +301,7 @@ static void goldfish_tty_delete_driver(void)
 static int goldfish_tty_probe(struct platform_device *pdev)
 {
 	struct goldfish_tty *qtty;
-	int ret = -EINVAL;
+	int ret = -ENODEV;
 	struct resource *r;
 	struct device *ttydev;
 	void __iomem *base;
@@ -235,16 +309,22 @@ static int goldfish_tty_probe(struct platform_device *pdev)
 	unsigned int line;
 
 	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (r == NULL)
-		return -EINVAL;
+	if (!r) {
+		pr_err("goldfish_tty: No MEM resource available!\n");
+		return -ENOMEM;
+	}
 
 	base = ioremap(r->start, 0x1000);
-	if (base == NULL)
-		pr_err("goldfish_tty: unable to remap base\n");
+	if (!base) {
+		pr_err("goldfish_tty: Unable to ioremap base!\n");
+		return -ENOMEM;
+	}
 
 	r = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
-	if (r == NULL)
+	if (!r) {
+		pr_err("goldfish_tty: No IRQ resource available!\n");
 		goto err_unmap;
+	}
 
 	irq = r->start;
 
@@ -255,13 +335,17 @@ static int goldfish_tty_probe(struct platform_device *pdev)
 	else
 		line = pdev->id;
 
-	if (line >= goldfish_tty_line_count)
-		goto err_create_driver_failed;
+	if (line >= goldfish_tty_line_count) {
+		pr_err("goldfish_tty: Reached maximum tty number of %d.\n",
+		       goldfish_tty_current_line_count);
+		ret = -ENOMEM;
+		goto err_unlock;
+	}
 
 	if (goldfish_tty_current_line_count == 0) {
 		ret = goldfish_tty_create_driver();
 		if (ret)
-			goto err_create_driver_failed;
+			goto err_unlock;
 	}
 	goldfish_tty_current_line_count++;
 
@@ -271,17 +355,45 @@ static int goldfish_tty_probe(struct platform_device *pdev)
 	qtty->port.ops = &goldfish_port_ops;
 	qtty->base = base;
 	qtty->irq = irq;
+	qtty->dev = &pdev->dev;
+
+	/*
+	 * Goldfish TTY device used by the Goldfish emulator
+	 * should identify itself with 0, forcing the driver
+	 * to use virtual addresses. Goldfish TTY device
+	 * on Ranchu emulator (qemu2) returns 1 here and
+	 * driver will use physical addresses.
+	 */
+	qtty->version = readl(base + GOLDFISH_TTY_REG_VERSION);
+
+	/*
+	 * Goldfish TTY device on Ranchu emulator (qemu2)
+	 * will use DMA for read/write IO operations.
+	 */
+	if (qtty->version > 0) {
+		/*
+		 * Initialize dma_mask to 32-bits.
+		 */
+		if (!pdev->dev.dma_mask)
+			pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask;
+		ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
+		if (ret) {
+			dev_err(&pdev->dev, "No suitable DMA available.\n");
+			goto err_dec_line_count;
+		}
+	}
 
-	writel(GOLDFISH_TTY_CMD_INT_DISABLE, base + GOLDFISH_TTY_CMD);
+	writel(GOLDFISH_TTY_CMD_INT_DISABLE, base + GOLDFISH_TTY_REG_CMD);
 
 	ret = request_irq(irq, goldfish_tty_interrupt, IRQF_SHARED,
-						"goldfish_tty", qtty);
-	if (ret)
-		goto err_request_irq_failed;
-
+			  "goldfish_tty", qtty);
+	if (ret) {
+		pr_err("goldfish_tty: No IRQ available!\n");
+		goto err_dec_line_count;
+	}
 
 	ttydev = tty_port_register_device(&qtty->port, goldfish_tty_driver,
-							line, &pdev->dev);
+					  line, &pdev->dev);
 	if (IS_ERR(ttydev)) {
 		ret = PTR_ERR(ttydev);
 		goto err_tty_register_device_failed;
@@ -301,11 +413,11 @@ static int goldfish_tty_probe(struct platform_device *pdev)
 
 err_tty_register_device_failed:
 	free_irq(irq, qtty);
-err_request_irq_failed:
+err_dec_line_count:
 	goldfish_tty_current_line_count--;
 	if (goldfish_tty_current_line_count == 0)
 		goldfish_tty_delete_driver();
-err_create_driver_failed:
+err_unlock:
 	mutex_unlock(&goldfish_tty_lock);
 err_unmap:
 	iounmap(base);
@@ -330,6 +442,30 @@ static int goldfish_tty_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static void gf_early_console_putchar(struct uart_port *port, int ch)
+{
+	__raw_writel(ch, port->membase);
+}
+
+static void gf_early_write(struct console *con, const char *s, unsigned int n)
+{
+	struct earlycon_device *dev = con->data;
+
+	uart_console_write(&dev->port, s, n, gf_early_console_putchar);
+}
+
+static int __init gf_earlycon_setup(struct earlycon_device *device,
+				    const char *opt)
+{
+	if (!device->port.membase)
+		return -ENODEV;
+
+	device->con->write = gf_early_write;
+	return 0;
+}
+
+OF_EARLYCON_DECLARE(early_gf_tty, "google,goldfish-tty", gf_earlycon_setup);
+
 static const struct of_device_id goldfish_tty_of_match[] = {
 	{ .compatible = "google,goldfish-tty", },
 	{},
diff --git a/drivers/tty/hvc/Kconfig b/drivers/tty/hvc/Kconfig
index b8d5ea0ae26b..fec457edad14 100644
--- a/drivers/tty/hvc/Kconfig
+++ b/drivers/tty/hvc/Kconfig
@@ -4,7 +4,7 @@ config HVC_DRIVER
 	bool
 	help
 	  Generic "hypervisor virtual console" infrastructure for various
-	  hypervisors (pSeries, iSeries, Xen, lguest).
+	  hypervisors (pSeries, iSeries, Xen).
 	  It will automatically be selected if one of the back-end console drivers
 	  is selected.
 
diff --git a/drivers/tty/hvc/hvc_opal.c b/drivers/tty/hvc/hvc_opal.c
index 510799311099..16331a90c1e8 100644
--- a/drivers/tty/hvc/hvc_opal.c
+++ b/drivers/tty/hvc/hvc_opal.c
@@ -179,8 +179,8 @@ static int hvc_opal_probe(struct platform_device *dev)
 		proto = HV_PROTOCOL_HVSI;
 		ops = &hvc_opal_hvsi_ops;
 	} else {
-		pr_err("hvc_opal: Unknown protocol for %s\n",
-		       dev->dev.of_node->full_name);
+		pr_err("hvc_opal: Unknown protocol for %pOF\n",
+		       dev->dev.of_node);
 		return -ENXIO;
 	}
 
@@ -204,14 +204,14 @@ static int hvc_opal_probe(struct platform_device *dev)
 		/* Instanciate now to establish a mapping index==vtermno */
 		hvc_instantiate(termno, termno, ops);
 	} else {
-		pr_err("hvc_opal: Device %s has duplicate terminal number #%d\n",
-		       dev->dev.of_node->full_name, termno);
+		pr_err("hvc_opal: Device %pOF has duplicate terminal number #%d\n",
+		       dev->dev.of_node, termno);
 		return -ENXIO;
 	}
 
-	pr_info("hvc%d: %s protocol on %s%s\n", termno,
+	pr_info("hvc%d: %s protocol on %pOF%s\n", termno,
 		proto == HV_PROTOCOL_RAW ? "raw" : "hvsi",
-		dev->dev.of_node->full_name,
+		dev->dev.of_node,
 		boot ? " (boot console)" : "");
 
 	irq = irq_of_parse_and_map(dev->dev.of_node, 0);
@@ -222,8 +222,8 @@ static int hvc_opal_probe(struct platform_device *dev)
 	}
 
 	if (!irq) {
-		pr_err("hvc_opal: Unable to map interrupt for device %s\n",
-			dev->dev.of_node->full_name);
+		pr_err("hvc_opal: Unable to map interrupt for device %pOF\n",
+			dev->dev.of_node);
 		return irq;
 	}
 
diff --git a/drivers/tty/hvc/hvc_vio.c b/drivers/tty/hvc/hvc_vio.c
index b05dc5086627..a1d272ac82bb 100644
--- a/drivers/tty/hvc/hvc_vio.c
+++ b/drivers/tty/hvc/hvc_vio.c
@@ -53,7 +53,7 @@
 
 static const char hvc_driver_name[] = "hvc_console";
 
-static struct vio_device_id hvc_driver_table[] = {
+static const struct vio_device_id hvc_driver_table[] = {
 	{"serial", "hvterm1"},
 #ifndef HVC_OLD_HVSI
 	{"serial", "hvterm-protocol"},
@@ -312,12 +312,12 @@ static int hvc_vio_probe(struct vio_dev *vdev,
 		proto = HV_PROTOCOL_HVSI;
 		ops = &hvterm_hvsi_ops;
 	} else {
-		pr_err("hvc_vio: Unknown protocol for %s\n", vdev->dev.of_node->full_name);
+		pr_err("hvc_vio: Unknown protocol for %pOF\n", vdev->dev.of_node);
 		return -ENXIO;
 	}
 
-	pr_devel("hvc_vio_probe() device %s, using %s protocol\n",
-		 vdev->dev.of_node->full_name,
+	pr_devel("hvc_vio_probe() device %pOF, using %s protocol\n",
+		 vdev->dev.of_node,
 		 proto == HV_PROTOCOL_RAW ? "raw" : "hvsi");
 
 	/* Is it our boot one ? */
@@ -442,6 +442,14 @@ void __init hvc_vio_init_early(void)
 #ifdef CONFIG_PPC_EARLY_DEBUG_LPAR
 void __init udbg_init_debug_lpar(void)
 {
+	/*
+	 * If we're running as a hypervisor then we definitely can't call the
+	 * hypervisor to print debug output (we *are* the hypervisor), so don't
+	 * register if we detect that MSR_HV=1.
+	 */
+	if (mfmsr() & MSR_HV)
+		return;
+
 	hvterm_privs[0] = &hvterm_priv0;
 	hvterm_priv0.termno = 0;
 	hvterm_priv0.proto = HV_PROTOCOL_RAW;
@@ -455,6 +463,10 @@ void __init udbg_init_debug_lpar(void)
 #ifdef CONFIG_PPC_EARLY_DEBUG_LPAR_HVSI
 void __init udbg_init_debug_lpar_hvsi(void)
 {
+	/* See comment above in udbg_init_debug_lpar() */
+	if (mfmsr() & MSR_HV)
+		return;
+
 	hvterm_privs[0] = &hvterm_priv0;
 	hvterm_priv0.termno = CONFIG_PPC_EARLY_DEBUG_HVSI_VTERMNO;
 	hvterm_priv0.proto = HV_PROTOCOL_HVSI;
diff --git a/drivers/tty/hvc/hvcs.c b/drivers/tty/hvc/hvcs.c
index 79cc5beea2da..63c29fe9d21f 100644
--- a/drivers/tty/hvc/hvcs.c
+++ b/drivers/tty/hvc/hvcs.c
@@ -189,7 +189,7 @@ MODULE_VERSION(HVCS_DRIVER_VERSION);
  * that will cause echoing or we'll go into recursive loop echoing chars back
  * and forth with the console drivers.
  */
-static struct ktermios hvcs_tty_termios = {
+static const struct ktermios hvcs_tty_termios = {
 	.c_iflag = IGNBRK | IGNPAR,
 	.c_oflag = OPOST,
 	.c_cflag = B38400 | CS8 | CREAD | HUPCL,
@@ -675,7 +675,7 @@ static int khvcsd(void *unused)
 	return 0;
 }
 
-static struct vio_device_id hvcs_driver_table[] = {
+static const struct vio_device_id hvcs_driver_table[] = {
 	{"serial-server", "hvterm2"},
 	{ "", "" }
 };
diff --git a/drivers/tty/isicom.c b/drivers/tty/isicom.c
index b70187b46d9d..61ecdd6b2fc2 100644
--- a/drivers/tty/isicom.c
+++ b/drivers/tty/isicom.c
@@ -150,7 +150,7 @@
 static int isicom_probe(struct pci_dev *, const struct pci_device_id *);
 static void isicom_remove(struct pci_dev *);
 
-static struct pci_device_id isicom_pci_tbl[] = {
+static const struct pci_device_id isicom_pci_tbl[] = {
 	{ PCI_DEVICE(VENDOR_ID, 0x2028) },
 	{ PCI_DEVICE(VENDOR_ID, 0x2051) },
 	{ PCI_DEVICE(VENDOR_ID, 0x2052) },
diff --git a/drivers/tty/mips_ejtag_fdc.c b/drivers/tty/mips_ejtag_fdc.c
index 234123b0c642..a2dab3fb8751 100644
--- a/drivers/tty/mips_ejtag_fdc.c
+++ b/drivers/tty/mips_ejtag_fdc.c
@@ -1110,7 +1110,7 @@ out:
 	return ret;
 }
 
-static struct mips_cdmm_device_id mips_ejtag_fdc_tty_ids[] = {
+static const struct mips_cdmm_device_id mips_ejtag_fdc_tty_ids[] = {
 	{ .type = 0xfd },
 	{ }
 };
diff --git a/drivers/tty/moxa.c b/drivers/tty/moxa.c
index 3b251f4e5df0..7f3d4cb0341b 100644
--- a/drivers/tty/moxa.c
+++ b/drivers/tty/moxa.c
@@ -88,7 +88,7 @@ static char *moxa_brdname[] =
 };
 
 #ifdef CONFIG_PCI
-static struct pci_device_id moxa_pcibrds[] = {
+static const struct pci_device_id moxa_pcibrds[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_MOXA, PCI_DEVICE_ID_MOXA_C218),
 		.driver_data = MOXA_BOARD_C218_PCI },
 	{ PCI_DEVICE(PCI_VENDOR_ID_MOXA, PCI_DEVICE_ID_MOXA_C320),
diff --git a/drivers/tty/mxser.c b/drivers/tty/mxser.c
index 8bd6fb6d9391..1c0c9553bc05 100644
--- a/drivers/tty/mxser.c
+++ b/drivers/tty/mxser.c
@@ -145,7 +145,7 @@ static const struct mxser_cardinfo mxser_cards[] = {
 
 /* driver_data correspond to the lines in the structure above
    see also ISA probe function before you change something */
-static struct pci_device_id mxser_pcibrds[] = {
+static const struct pci_device_id mxser_pcibrds[] = {
 	{ PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_C168),	.driver_data = 3 },
 	{ PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_C104),	.driver_data = 4 },
 	{ PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP132),	.driver_data = 8 },
diff --git a/drivers/tty/n_gsm.c b/drivers/tty/n_gsm.c
index 2afe5fce68e3..0a3c9665e015 100644
--- a/drivers/tty/n_gsm.c
+++ b/drivers/tty/n_gsm.c
@@ -2607,6 +2607,14 @@ static int gsmld_ioctl(struct tty_struct *tty, struct file *file,
 	}
 }
 
+#ifdef CONFIG_COMPAT
+static long gsmld_compat_ioctl(struct tty_struct *tty, struct file *file,
+		       unsigned int cmd, unsigned long arg)
+{
+	return gsmld_ioctl(tty, file, cmd, arg);
+}
+#endif
+
 /*
  *	Network interface
  *
@@ -2818,6 +2826,9 @@ static struct tty_ldisc_ops tty_ldisc_packet = {
 	.flush_buffer    = gsmld_flush_buffer,
 	.read            = gsmld_read,
 	.write           = gsmld_write,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl    = gsmld_compat_ioctl,
+#endif
 	.ioctl           = gsmld_ioctl,
 	.poll            = gsmld_poll,
 	.receive_buf     = gsmld_receive_buf,
diff --git a/drivers/tty/pty.c b/drivers/tty/pty.c
index 284749fb0f6b..26dcb3b60fb9 100644
--- a/drivers/tty/pty.c
+++ b/drivers/tty/pty.c
@@ -69,13 +69,8 @@ static void pty_close(struct tty_struct *tty, struct file *filp)
 #ifdef CONFIG_UNIX98_PTYS
 		if (tty->driver == ptm_driver) {
 			mutex_lock(&devpts_mutex);
-			if (tty->link->driver_data) {
-				struct path *path = tty->link->driver_data;
-
-				devpts_pty_kill(path->dentry);
-				path_put(path);
-				kfree(path);
-			}
+			if (tty->link->driver_data)
+				devpts_pty_kill(tty->link->driver_data);
 			mutex_unlock(&devpts_mutex);
 		}
 #endif
@@ -607,25 +602,24 @@ static inline void legacy_pty_init(void) { }
 static struct cdev ptmx_cdev;
 
 /**
- *	pty_open_peer - open the peer of a pty
- *	@tty: the peer of the pty being opened
+ *	ptm_open_peer - open the peer of a pty
+ *	@master: the open struct file of the ptmx device node
+ *	@tty: the master of the pty being opened
+ *	@flags: the flags for open
  *
- *	Open the cached dentry in tty->link, providing a safe way for userspace
- *	to get the slave end of a pty (where they have the master fd and cannot
- *	access or trust the mount namespace /dev/pts was mounted inside).
+ *	Provide a race free way for userspace to open the slave end of a pty
+ *	(where they have the master fd and cannot access or trust the mount
+ *	namespace /dev/pts was mounted inside).
  */
-static struct file *pty_open_peer(struct tty_struct *tty, int flags)
-{
-	if (tty->driver->subtype != PTY_TYPE_MASTER)
-		return ERR_PTR(-EIO);
-	return dentry_open(tty->link->driver_data, flags, current_cred());
-}
-
-static int pty_get_peer(struct tty_struct *tty, int flags)
+int ptm_open_peer(struct file *master, struct tty_struct *tty, int flags)
 {
 	int fd = -1;
-	struct file *filp = NULL;
+	struct file *filp;
 	int retval = -EINVAL;
+	struct path path;
+
+	if (tty->driver != ptm_driver)
+		return -EIO;
 
 	fd = get_unused_fd_flags(0);
 	if (fd < 0) {
@@ -633,7 +627,16 @@ static int pty_get_peer(struct tty_struct *tty, int flags)
 		goto err;
 	}
 
-	filp = pty_open_peer(tty, flags);
+	/* Compute the slave's path */
+	path.mnt = devpts_mntget(master, tty->driver_data);
+	if (IS_ERR(path.mnt)) {
+		retval = PTR_ERR(path.mnt);
+		goto err_put;
+	}
+	path.dentry = tty->link->driver_data;
+
+	filp = dentry_open(&path, flags, current_cred());
+	mntput(path.mnt);
 	if (IS_ERR(filp)) {
 		retval = PTR_ERR(filp);
 		goto err_put;
@@ -662,8 +665,6 @@ static int pty_unix98_ioctl(struct tty_struct *tty,
 		return pty_get_pktmode(tty, (int __user *)arg);
 	case TIOCGPTN: /* Get PT Number */
 		return put_user(tty->index, (unsigned int __user *)arg);
-	case TIOCGPTPEER: /* Open the other end */
-		return pty_get_peer(tty, (int) arg);
 	case TIOCSIG:    /* Send signal to other side of pty */
 		return pty_signal(tty, (int) arg);
 	}
@@ -741,6 +742,11 @@ static void pty_unix98_remove(struct tty_driver *driver, struct tty_struct *tty)
 	}
 }
 
+static void pty_show_fdinfo(struct tty_struct *tty, struct seq_file *m)
+{
+	seq_printf(m, "tty-index:\t%d\n", tty->index);
+}
+
 static const struct tty_operations ptm_unix98_ops = {
 	.lookup = ptm_unix98_lookup,
 	.install = pty_unix98_install,
@@ -755,7 +761,8 @@ static const struct tty_operations ptm_unix98_ops = {
 	.ioctl = pty_unix98_ioctl,
 	.compat_ioctl = pty_unix98_compat_ioctl,
 	.resize = pty_resize,
-	.cleanup = pty_cleanup
+	.cleanup = pty_cleanup,
+	.show_fdinfo = pty_show_fdinfo,
 };
 
 static const struct tty_operations pty_unix98_ops = {
@@ -791,7 +798,6 @@ static int ptmx_open(struct inode *inode, struct file *filp)
 {
 	struct pts_fs_info *fsi;
 	struct tty_struct *tty;
-	struct path *pts_path;
 	struct dentry *dentry;
 	int retval;
 	int index;
@@ -845,26 +851,16 @@ static int ptmx_open(struct inode *inode, struct file *filp)
 		retval = PTR_ERR(dentry);
 		goto err_release;
 	}
-	/* We need to cache a fake path for TIOCGPTPEER. */
-	pts_path = kmalloc(sizeof(struct path), GFP_KERNEL);
-	if (!pts_path)
-		goto err_release;
-	pts_path->mnt = filp->f_path.mnt;
-	pts_path->dentry = dentry;
-	path_get(pts_path);
-	tty->link->driver_data = pts_path;
+	tty->link->driver_data = dentry;
 
 	retval = ptm_driver->ops->open(tty, filp);
 	if (retval)
-		goto err_path_put;
+		goto err_release;
 
 	tty_debug_hangup(tty, "opening (count=%d)\n", tty->count);
 
 	tty_unlock(tty);
 	return 0;
-err_path_put:
-	path_put(pts_path);
-	kfree(pts_path);
 err_release:
 	tty_unlock(tty);
 	// This will also put-ref the fsi
diff --git a/drivers/tty/serdev/core.c b/drivers/tty/serdev/core.c
index ae1aaa0075d1..c68fb3a8ea1c 100644
--- a/drivers/tty/serdev/core.c
+++ b/drivers/tty/serdev/core.c
@@ -363,7 +363,7 @@ static int of_serdev_register_devices(struct serdev_controller *ctrl)
 		if (!of_get_property(node, "compatible", NULL))
 			continue;
 
-		dev_dbg(&ctrl->dev, "adding child %s\n", node->full_name);
+		dev_dbg(&ctrl->dev, "adding child %pOF\n", node);
 
 		serdev = serdev_device_alloc(ctrl);
 		if (!serdev)
diff --git a/drivers/tty/serial/21285.c b/drivers/tty/serial/21285.c
index 9b208bd686e6..804632b4a929 100644
--- a/drivers/tty/serial/21285.c
+++ b/drivers/tty/serial/21285.c
@@ -334,7 +334,7 @@ static int serial21285_verify_port(struct uart_port *port, struct serial_struct
 	return ret;
 }
 
-static struct uart_ops serial21285_ops = {
+static const struct uart_ops serial21285_ops = {
 	.tx_empty	= serial21285_tx_empty,
 	.get_mctrl	= serial21285_get_mctrl,
 	.set_mctrl	= serial21285_set_mctrl,
diff --git a/drivers/tty/serial/8250/8250_aspeed_vuart.c b/drivers/tty/serial/8250/8250_aspeed_vuart.c
index 822be4906763..33a801353114 100644
--- a/drivers/tty/serial/8250/8250_aspeed_vuart.c
+++ b/drivers/tty/serial/8250/8250_aspeed_vuart.c
@@ -223,12 +223,13 @@ static int aspeed_vuart_probe(struct platform_device *pdev)
 		if (IS_ERR(vuart->clk)) {
 			dev_warn(&pdev->dev,
 				"clk or clock-frequency not defined\n");
-			return PTR_ERR(vuart->clk);
+			rc = PTR_ERR(vuart->clk);
+			goto err_sysfs_remove;
 		}
 
 		rc = clk_prepare_enable(vuart->clk);
 		if (rc < 0)
-			return rc;
+			goto err_sysfs_remove;
 
 		clk = clk_get_rate(vuart->clk);
 	}
@@ -286,6 +287,8 @@ static int aspeed_vuart_probe(struct platform_device *pdev)
 err_clk_disable:
 	clk_disable_unprepare(vuart->clk);
 	irq_dispose_mapping(port.port.irq);
+err_sysfs_remove:
+	sysfs_remove_group(&vuart->dev->kobj, &aspeed_vuart_attr_group);
 	return rc;
 }
 
diff --git a/drivers/tty/serial/8250/8250_core.c b/drivers/tty/serial/8250/8250_core.c
index 1aab3010fbfa..d29b512a7d9f 100644
--- a/drivers/tty/serial/8250/8250_core.c
+++ b/drivers/tty/serial/8250/8250_core.c
@@ -497,6 +497,11 @@ static void univ8250_rsa_support(struct uart_ops *ops)
 #define univ8250_rsa_support(x)		do { } while (0)
 #endif /* CONFIG_SERIAL_8250_RSA */
 
+static inline void serial8250_apply_quirks(struct uart_8250_port *up)
+{
+	up->port.quirks |= skip_txen_test ? UPQ_NO_TXEN_TEST : 0;
+}
+
 static void __init serial8250_isa_init_ports(void)
 {
 	struct uart_8250_port *up;
@@ -577,9 +582,7 @@ serial8250_register_ports(struct uart_driver *drv, struct device *dev)
 
 		up->port.dev = dev;
 
-		if (skip_txen_test)
-			up->port.flags |= UPF_NO_TXEN_TEST;
-
+		serial8250_apply_quirks(up);
 		uart_add_one_port(drv, &up->port);
 	}
 }
@@ -1006,9 +1009,6 @@ int serial8250_register_8250_port(struct uart_8250_port *up)
 		if (up->port.dev)
 			uart->port.dev = up->port.dev;
 
-		if (skip_txen_test)
-			uart->port.flags |= UPF_NO_TXEN_TEST;
-
 		if (up->port.flags & UPF_FIXED_TYPE)
 			uart->port.type = up->port.type;
 
@@ -1048,6 +1048,7 @@ int serial8250_register_8250_port(struct uart_8250_port *up)
 				serial8250_isa_config(0, &uart->port,
 						&uart->capabilities);
 
+			serial8250_apply_quirks(uart);
 			ret = uart_add_one_port(&serial8250_reg,
 						&uart->port);
 			if (ret == 0)
@@ -1092,11 +1093,10 @@ void serial8250_unregister_port(int line)
 	uart_remove_one_port(&serial8250_reg, &uart->port);
 	if (serial8250_isa_devs) {
 		uart->port.flags &= ~UPF_BOOT_AUTOCONF;
-		if (skip_txen_test)
-			uart->port.flags |= UPF_NO_TXEN_TEST;
 		uart->port.type = PORT_UNKNOWN;
 		uart->port.dev = &serial8250_isa_devs->dev;
 		uart->capabilities = 0;
+		serial8250_apply_quirks(uart);
 		uart_add_one_port(&serial8250_reg, &uart->port);
 	} else {
 		uart->port.dev = NULL;
diff --git a/drivers/tty/serial/8250/8250_dw.c b/drivers/tty/serial/8250/8250_dw.c
index 787b1160d3a5..7e638997bfc2 100644
--- a/drivers/tty/serial/8250/8250_dw.c
+++ b/drivers/tty/serial/8250/8250_dw.c
@@ -529,7 +529,7 @@ static int dw8250_probe(struct platform_device *pdev)
 		}
 	}
 
-	data->rst = devm_reset_control_get_optional(dev, NULL);
+	data->rst = devm_reset_control_get_optional_exclusive(dev, NULL);
 	if (IS_ERR(data->rst)) {
 		err = PTR_ERR(data->rst);
 		goto err_pclk;
diff --git a/drivers/tty/serial/8250/8250_early.c b/drivers/tty/serial/8250/8250_early.c
index 82fc48eca1df..af72ec32e404 100644
--- a/drivers/tty/serial/8250/8250_early.c
+++ b/drivers/tty/serial/8250/8250_early.c
@@ -37,7 +37,7 @@
 #include <asm/io.h>
 #include <asm/serial.h>
 
-static unsigned int __init serial8250_early_in(struct uart_port *port, int offset)
+static unsigned int serial8250_early_in(struct uart_port *port, int offset)
 {
 	int reg_offset = offset;
 	offset <<= port->regshift;
@@ -60,7 +60,7 @@ static unsigned int __init serial8250_early_in(struct uart_port *port, int offse
 	}
 }
 
-static void __init serial8250_early_out(struct uart_port *port, int offset, int value)
+static void serial8250_early_out(struct uart_port *port, int offset, int value)
 {
 	int reg_offset = offset;
 	offset <<= port->regshift;
@@ -89,7 +89,7 @@ static void __init serial8250_early_out(struct uart_port *port, int offset, int
 
 #define BOTH_EMPTY (UART_LSR_TEMT | UART_LSR_THRE)
 
-static void __init serial_putc(struct uart_port *port, int c)
+static void serial_putc(struct uart_port *port, int c)
 {
 	unsigned int status;
 
@@ -103,7 +103,7 @@ static void __init serial_putc(struct uart_port *port, int c)
 	}
 }
 
-static void __init early_serial8250_write(struct console *console,
+static void early_serial8250_write(struct console *console,
 					const char *s, unsigned int count)
 {
 	struct earlycon_device *device = console->data;
diff --git a/drivers/tty/serial/8250/8250_exar.c b/drivers/tty/serial/8250/8250_exar.c
index c6360fbdf808..c55624703fdf 100644
--- a/drivers/tty/serial/8250/8250_exar.c
+++ b/drivers/tty/serial/8250/8250_exar.c
@@ -601,7 +601,7 @@ static const struct exar8250_board pbn_exar_XR17V8358 = {
 		(kernel_ulong_t)&bd			\
 	}
 
-static struct pci_device_id exar_pci_tbl[] = {
+static const struct pci_device_id exar_pci_tbl[] = {
 	CONNECT_DEVICE(XR17C152, UART_2_232, pbn_connect),
 	CONNECT_DEVICE(XR17C154, UART_4_232, pbn_connect),
 	CONNECT_DEVICE(XR17C158, UART_8_232, pbn_connect),
diff --git a/drivers/tty/serial/8250/8250_gsc.c b/drivers/tty/serial/8250/8250_gsc.c
index 63306de4390d..df2931e1e086 100644
--- a/drivers/tty/serial/8250/8250_gsc.c
+++ b/drivers/tty/serial/8250/8250_gsc.c
@@ -80,7 +80,7 @@ static int __init serial_init_chip(struct parisc_device *dev)
 	return 0;
 }
 
-static struct parisc_device_id serial_tbl[] = {
+static const struct parisc_device_id serial_tbl[] __initconst = {
 	{ HPHW_FIO, HVERSION_REV_ANY_ID, HVERSION_ANY_ID, 0x00075 },
 	{ HPHW_FIO, HVERSION_REV_ANY_ID, HVERSION_ANY_ID, 0x0008c },
 	{ HPHW_FIO, HVERSION_REV_ANY_ID, HVERSION_ANY_ID, 0x0008d },
@@ -94,7 +94,7 @@ static struct parisc_device_id serial_tbl[] = {
  * which only knows about Lasi and then a second which will find all the
  * other serial ports.  HPUX ignores this problem.
  */
-static struct parisc_device_id lasi_tbl[] = {
+static const struct parisc_device_id lasi_tbl[] __initconst = {
 	{ HPHW_FIO, HVERSION_REV_ANY_ID, 0x03B, 0x0008C }, /* C1xx/C1xxL */
 	{ HPHW_FIO, HVERSION_REV_ANY_ID, 0x03C, 0x0008C }, /* B132L */
 	{ HPHW_FIO, HVERSION_REV_ANY_ID, 0x03D, 0x0008C }, /* B160L */
@@ -110,13 +110,13 @@ static struct parisc_device_id lasi_tbl[] = {
 
 MODULE_DEVICE_TABLE(parisc, serial_tbl);
 
-static struct parisc_driver lasi_driver = {
+static struct parisc_driver lasi_driver __refdata = {
 	.name		= "serial_1",
 	.id_table	= lasi_tbl,
 	.probe		= serial_init_chip,
 };
 
-static struct parisc_driver serial_driver = {
+static struct parisc_driver serial_driver __refdata = {
 	.name		= "serial",
 	.id_table	= serial_tbl,
 	.probe		= serial_init_chip,
diff --git a/drivers/tty/serial/8250/8250_ingenic.c b/drivers/tty/serial/8250/8250_ingenic.c
index 4d9dc10e265c..464389b28900 100644
--- a/drivers/tty/serial/8250/8250_ingenic.c
+++ b/drivers/tty/serial/8250/8250_ingenic.c
@@ -50,17 +50,17 @@ static const struct of_device_id of_match[];
 
 static struct earlycon_device *early_device;
 
-static uint8_t __init early_in(struct uart_port *port, int offset)
+static uint8_t early_in(struct uart_port *port, int offset)
 {
 	return readl(port->membase + (offset << 2));
 }
 
-static void __init early_out(struct uart_port *port, int offset, uint8_t value)
+static void early_out(struct uart_port *port, int offset, uint8_t value)
 {
 	writel(value, port->membase + (offset << 2));
 }
 
-static void __init ingenic_early_console_putc(struct uart_port *port, int c)
+static void ingenic_early_console_putc(struct uart_port *port, int c)
 {
 	uint8_t lsr;
 
@@ -71,7 +71,7 @@ static void __init ingenic_early_console_putc(struct uart_port *port, int c)
 	early_out(port, UART_TX, c);
 }
 
-static void __init ingenic_early_console_write(struct console *console,
+static void ingenic_early_console_write(struct console *console,
 					      const char *s, unsigned int count)
 {
 	uart_console_write(&early_device->port, s, count,
diff --git a/drivers/tty/serial/8250/8250_men_mcb.c b/drivers/tty/serial/8250/8250_men_mcb.c
new file mode 100644
index 000000000000..308977807994
--- /dev/null
+++ b/drivers/tty/serial/8250/8250_men_mcb.c
@@ -0,0 +1,118 @@
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mcb.h>
+#include <linux/serial.h>
+#include <linux/serial_core.h>
+#include <linux/serial_8250.h>
+#include <uapi/linux/serial_core.h>
+
+struct serial_8250_men_mcb_data {
+	struct uart_8250_port uart;
+	int line;
+};
+
+/*
+ * The Z125 16550-compatible UART has no fixed base clock assigned
+ * So, depending on the board we're on, we need to adjust the
+ * parameter in order to really set the correct baudrate, and
+ * do so if possible without user interaction
+ */
+static u32 men_z125_lookup_uartclk(struct mcb_device *mdev)
+{
+	/* use default value if board is not available below */
+	u32 clkval = 1041666;
+
+	dev_info(&mdev->dev, "%s on board %s\n",
+		dev_name(&mdev->dev),
+		mdev->bus->name);
+	if  (strncmp(mdev->bus->name, "F075", 4) == 0)
+		clkval = 1041666;
+	else if  (strncmp(mdev->bus->name, "F216", 4) == 0)
+		clkval = 1843200;
+	else if (strncmp(mdev->bus->name, "G215", 4) == 0)
+		clkval = 1843200;
+	else
+		dev_info(&mdev->dev,
+			 "board not detected, using default uartclk\n");
+
+	clkval = clkval  << 4;
+
+	return clkval;
+}
+
+static int serial_8250_men_mcb_probe(struct mcb_device *mdev,
+				     const struct mcb_device_id *id)
+{
+	struct serial_8250_men_mcb_data *data;
+	struct resource *mem;
+
+	data = devm_kzalloc(&mdev->dev,
+			    sizeof(struct serial_8250_men_mcb_data),
+			    GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	mcb_set_drvdata(mdev, data);
+	data->uart.port.dev = mdev->dma_dev;
+	spin_lock_init(&data->uart.port.lock);
+
+	data->uart.port.type = PORT_16550;
+	data->uart.port.flags = UPF_SKIP_TEST | UPF_SHARE_IRQ | UPF_FIXED_TYPE;
+	data->uart.port.iotype = UPIO_MEM;
+	data->uart.port.uartclk = men_z125_lookup_uartclk(mdev);
+	data->uart.port.regshift = 0;
+	data->uart.port.fifosize = 60;
+
+	mem = mcb_get_resource(mdev, IORESOURCE_MEM);
+	if (mem == NULL)
+		return -ENXIO;
+
+	data->uart.port.irq = mcb_get_irq(mdev);
+
+	data->uart.port.membase = devm_ioremap_resource(&mdev->dev, mem);
+	if (IS_ERR(data->uart.port.membase))
+		return PTR_ERR_OR_ZERO(data->uart.port.membase);
+
+	data->uart.port.mapbase = (unsigned long) mem->start;
+	data->uart.port.iobase = data->uart.port.mapbase;
+
+	/* ok, register the port */
+	data->line = serial8250_register_8250_port(&data->uart);
+	if (data->line < 0)
+		return data->line;
+
+	dev_info(&mdev->dev, "found 16Z125 UART: ttyS%d\n", data->line);
+
+	return 0;
+}
+
+static void serial_8250_men_mcb_remove(struct mcb_device *mdev)
+{
+	struct serial_8250_men_mcb_data *data = mcb_get_drvdata(mdev);
+
+	if (data)
+		serial8250_unregister_port(data->line);
+}
+
+static const struct mcb_device_id serial_8250_men_mcb_ids[] = {
+	{ .device = 0x7d },
+	{ }
+};
+MODULE_DEVICE_TABLE(mcb, serial_8250_men_mcb_ids);
+
+static struct mcb_driver mcb_driver = {
+	.driver = {
+		.name = "8250_men_mcb",
+		.owner = THIS_MODULE,
+	},
+	.probe = serial_8250_men_mcb_probe,
+	.remove = serial_8250_men_mcb_remove,
+	.id_table = serial_8250_men_mcb_ids,
+};
+module_mcb_driver(mcb_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("MEN 16z125 8250 UART driver");
+MODULE_AUTHOR("Michael Moese <michael.moese@men.de");
+MODULE_ALIAS("mcb:16z125");
diff --git a/drivers/tty/serial/8250/8250_mtk.c b/drivers/tty/serial/8250/8250_mtk.c
index ce0cc471bfc3..fb45770d47aa 100644
--- a/drivers/tty/serial/8250/8250_mtk.c
+++ b/drivers/tty/serial/8250/8250_mtk.c
@@ -171,10 +171,7 @@ static int mtk8250_probe_of(struct platform_device *pdev, struct uart_port *p,
 	}
 
 	data->bus_clk = devm_clk_get(&pdev->dev, "bus");
-	if (IS_ERR(data->bus_clk))
-		return PTR_ERR(data->bus_clk);
-
-	return 0;
+	return PTR_ERR_OR_ZERO(data->bus_clk);
 }
 
 static int mtk8250_probe(struct platform_device *pdev)
diff --git a/drivers/tty/serial/8250/8250_of.c b/drivers/tty/serial/8250/8250_of.c
index 0cf95fddccfc..1222c005fb98 100644
--- a/drivers/tty/serial/8250/8250_of.c
+++ b/drivers/tty/serial/8250/8250_of.c
@@ -18,6 +18,7 @@
 #include <linux/of_address.h>
 #include <linux/of_irq.h>
 #include <linux/of_platform.h>
+#include <linux/pm_runtime.h>
 #include <linux/clk.h>
 #include <linux/reset.h>
 
@@ -65,6 +66,10 @@ static int of_platform_serial_setup(struct platform_device *ofdev,
 	int ret;
 
 	memset(port, 0, sizeof *port);
+
+	pm_runtime_enable(&ofdev->dev);
+	pm_runtime_get_sync(&ofdev->dev);
+
 	if (of_property_read_u32(np, "clock-frequency", &clk)) {
 
 		/* Get clk rate through clk driver if present */
@@ -72,12 +77,13 @@ static int of_platform_serial_setup(struct platform_device *ofdev,
 		if (IS_ERR(info->clk)) {
 			dev_warn(&ofdev->dev,
 				"clk or clock-frequency not defined\n");
-			return PTR_ERR(info->clk);
+			ret = PTR_ERR(info->clk);
+			goto err_pmruntime;
 		}
 
 		ret = clk_prepare_enable(info->clk);
 		if (ret < 0)
-			return ret;
+			goto err_pmruntime;
 
 		clk = clk_get_rate(info->clk);
 	}
@@ -88,7 +94,7 @@ static int of_platform_serial_setup(struct platform_device *ofdev,
 	ret = of_address_to_resource(np, 0, &resource);
 	if (ret) {
 		dev_warn(&ofdev->dev, "invalid address\n");
-		goto out;
+		goto err_unprepare;
 	}
 
 	spin_lock_init(&port->lock);
@@ -130,23 +136,23 @@ static int of_platform_serial_setup(struct platform_device *ofdev,
 			dev_warn(&ofdev->dev, "unsupported reg-io-width (%d)\n",
 				 prop);
 			ret = -EINVAL;
-			goto out;
+			goto err_dispose;
 		}
 	}
 
 	info->rst = devm_reset_control_get_optional_shared(&ofdev->dev, NULL);
 	if (IS_ERR(info->rst))
-		goto out;
+		goto err_dispose;
 	ret = reset_control_deassert(info->rst);
 	if (ret)
-		goto out;
+		goto err_dispose;
 
 	port->type = type;
 	port->uartclk = clk;
 	port->flags = UPF_SHARE_IRQ | UPF_BOOT_AUTOCONF | UPF_IOREMAP
 		| UPF_FIXED_PORT | UPF_FIXED_TYPE;
 
-	if (of_find_property(np, "no-loopback-test", NULL))
+	if (of_property_read_bool(np, "no-loopback-test"))
 		port->flags |= UPF_SKIP_TEST;
 
 	port->dev = &ofdev->dev;
@@ -167,9 +173,13 @@ static int of_platform_serial_setup(struct platform_device *ofdev,
 		port->handle_irq = fsl8250_handle_irq;
 
 	return 0;
-out:
-	if (info->clk)
-		clk_disable_unprepare(info->clk);
+err_dispose:
+	irq_dispose_mapping(port->irq);
+err_unprepare:
+	clk_disable_unprepare(info->clk);
+err_pmruntime:
+	pm_runtime_put_sync(&ofdev->dev);
+	pm_runtime_disable(&ofdev->dev);
 	return ret;
 }
 
@@ -190,7 +200,7 @@ static int of_platform_serial_probe(struct platform_device *ofdev)
 	if (!match)
 		return -EINVAL;
 
-	if (of_find_property(ofdev->dev.of_node, "used-by-rtas", NULL))
+	if (of_property_read_bool(ofdev->dev.of_node, "used-by-rtas"))
 		return -EBUSY;
 
 	info = kzalloc(sizeof(*info), GFP_KERNEL);
@@ -201,7 +211,7 @@ static int of_platform_serial_probe(struct platform_device *ofdev)
 	memset(&port8250, 0, sizeof(port8250));
 	ret = of_platform_serial_setup(ofdev, port_type, &port8250.port, info);
 	if (ret)
-		goto out;
+		goto err_free;
 
 	if (port8250.port.fifosize)
 		port8250.capabilities = UART_CAP_FIFO;
@@ -217,15 +227,19 @@ static int of_platform_serial_probe(struct platform_device *ofdev)
 
 	ret = serial8250_register_8250_port(&port8250);
 	if (ret < 0)
-		goto out;
+		goto err_dispose;
 
 	info->type = port_type;
 	info->line = ret;
 	platform_set_drvdata(ofdev, info);
 	return 0;
-out:
-	kfree(info);
+err_dispose:
 	irq_dispose_mapping(port8250.port.irq);
+	pm_runtime_put_sync(&ofdev->dev);
+	pm_runtime_disable(&ofdev->dev);
+	clk_disable_unprepare(info->clk);
+err_free:
+	kfree(info);
 	return ret;
 }
 
@@ -239,8 +253,9 @@ static int of_platform_serial_remove(struct platform_device *ofdev)
 	serial8250_unregister_port(info->line);
 
 	reset_control_assert(info->rst);
-	if (info->clk)
-		clk_disable_unprepare(info->clk);
+	pm_runtime_put_sync(&ofdev->dev);
+	pm_runtime_disable(&ofdev->dev);
+	clk_disable_unprepare(info->clk);
 	kfree(info);
 	return 0;
 }
@@ -254,9 +269,10 @@ static int of_serial_suspend(struct device *dev)
 
 	serial8250_suspend_port(info->line);
 
-	if (info->clk && (!uart_console(port) || console_suspend_enabled))
+	if (!uart_console(port) || console_suspend_enabled) {
+		pm_runtime_put_sync(dev);
 		clk_disable_unprepare(info->clk);
-
+	}
 	return 0;
 }
 
@@ -266,8 +282,10 @@ static int of_serial_resume(struct device *dev)
 	struct uart_8250_port *port8250 = serial8250_get_port(info->line);
 	struct uart_port *port = &port8250->port;
 
-	if (info->clk && (!uart_console(port) || console_suspend_enabled))
+	if (!uart_console(port) || console_suspend_enabled) {
+		pm_runtime_get_sync(dev);
 		clk_prepare_enable(info->clk);
+	}
 
 	serial8250_resume_port(info->line);
 
@@ -295,6 +313,8 @@ static const struct of_device_id of_platform_serial_table[] = {
 		.data = (void *)PORT_ALTR_16550_F64, },
 	{ .compatible = "altr,16550-FIFO128",
 		.data = (void *)PORT_ALTR_16550_F128, },
+	{ .compatible = "mediatek,mtk-btif",
+		.data = (void *)PORT_MTK_BTIF, },
 	{ .compatible = "mrvl,mmp-uart",
 		.data = (void *)PORT_XSCALE, },
 	{ .compatible = "ti,da830-uart", .data = (void *)PORT_DA830, },
diff --git a/drivers/tty/serial/8250/8250_pci.c b/drivers/tty/serial/8250/8250_pci.c
index 00e51a064388..0c101a7470b0 100644
--- a/drivers/tty/serial/8250/8250_pci.c
+++ b/drivers/tty/serial/8250/8250_pci.c
@@ -1548,7 +1548,7 @@ static int skip_tx_en_setup(struct serial_private *priv,
 			const struct pciserial_board *board,
 			struct uart_8250_port *port, int idx)
 {
-	port->port.flags |= UPF_NO_TXEN_TEST;
+	port->port.quirks |= UPQ_NO_TXEN_TEST;
 	dev_dbg(&priv->dev->dev,
 		"serial8250: skipping TxEn test for device [%04x:%04x] subsystem [%04x:%04x]\n",
 		priv->dev->vendor, priv->dev->device,
@@ -3384,17 +3384,8 @@ static const struct pci_device_id blacklist[] = {
 	{ PCI_VDEVICE(COMMTECH, PCI_ANY_ID), },
 };
 
-/*
- * Given a complete unknown PCI device, try to use some heuristics to
- * guess what the configuration might be, based on the pitiful PCI
- * serial specs.  Returns 0 on success, 1 on failure.
- */
-static int
-serial_pci_guess_board(struct pci_dev *dev, struct pciserial_board *board)
+static int serial_pci_is_class_communication(struct pci_dev *dev)
 {
-	const struct pci_device_id *bldev;
-	int num_iomem, num_port, first_port = -1, i;
-
 	/*
 	 * If it is not a communications device or the programming
 	 * interface is greater than 6, give up.
@@ -3407,6 +3398,13 @@ serial_pci_guess_board(struct pci_dev *dev, struct pciserial_board *board)
 	    (dev->class & 0xff) > 6)
 		return -ENODEV;
 
+	return 0;
+}
+
+static int serial_pci_is_blacklisted(struct pci_dev *dev)
+{
+	const struct pci_device_id *bldev;
+
 	/*
 	 * Do not access blacklisted devices that are known not to
 	 * feature serial ports or are handled by other modules.
@@ -3419,6 +3417,19 @@ serial_pci_guess_board(struct pci_dev *dev, struct pciserial_board *board)
 			return -ENODEV;
 	}
 
+	return 0;
+}
+
+/*
+ * Given a complete unknown PCI device, try to use some heuristics to
+ * guess what the configuration might be, based on the pitiful PCI
+ * serial specs.  Returns 0 on success, -ENODEV on failure.
+ */
+static int
+serial_pci_guess_board(struct pci_dev *dev, struct pciserial_board *board)
+{
+	int num_iomem, num_port, first_port = -1, i;
+
 	num_iomem = num_port = 0;
 	for (i = 0; i < PCI_NUM_BAR_RESOURCES; i++) {
 		if (pci_resource_flags(dev, i) & IORESOURCE_IO) {
@@ -3639,6 +3650,14 @@ pciserial_init_one(struct pci_dev *dev, const struct pci_device_id *ent)
 
 	board = &pci_boards[ent->driver_data];
 
+	rc = serial_pci_is_class_communication(dev);
+	if (rc)
+		return rc;
+
+	rc = serial_pci_is_blacklisted(dev);
+	if (rc)
+		return rc;
+
 	rc = pcim_enable_device(dev);
 	pci_save_state(dev);
 	if (rc)
@@ -3723,7 +3742,7 @@ static int pciserial_resume_one(struct device *dev)
 static SIMPLE_DEV_PM_OPS(pciserial_pm_ops, pciserial_suspend_one,
 			 pciserial_resume_one);
 
-static struct pci_device_id serial_pci_tbl[] = {
+static const struct pci_device_id serial_pci_tbl[] = {
 	/* Advantech use PCI_DEVICE_ID_ADVANTECH_PCI3620 (0x3620) as 'PCI_SUBVENDOR_ID' */
 	{	PCI_VENDOR_ID_ADVANTECH, PCI_DEVICE_ID_ADVANTECH_PCI3620,
 		PCI_DEVICE_ID_ADVANTECH_PCI3620, 0x0001, 0, 0,
diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c
index a5fe0e66c607..f0cc04f62b67 100644
--- a/drivers/tty/serial/8250/8250_port.c
+++ b/drivers/tty/serial/8250/8250_port.c
@@ -37,7 +37,7 @@
 #include <linux/slab.h>
 #include <linux/uaccess.h>
 #include <linux/pm_runtime.h>
-#include <linux/timer.h>
+#include <linux/ktime.h>
 
 #include <asm/io.h>
 #include <asm/irq.h>
@@ -289,6 +289,14 @@ static const struct serial8250_config uart_config[] = {
 		.rxtrig_bytes	= {1, 4, 8, 14},
 		.flags		= UART_CAP_FIFO | UART_CAP_AFE,
 	},
+	[PORT_MTK_BTIF] = {
+		.name		= "MediaTek BTIF",
+		.fifo_size	= 16,
+		.tx_loadsz	= 16,
+		.fcr		= UART_FCR_ENABLE_FIFO |
+				  UART_FCR_CLEAR_RCVR | UART_FCR_CLEAR_XMIT,
+		.flags		= UART_CAP_FIFO,
+	},
 };
 
 /* Uart divisor latch read */
@@ -553,8 +561,8 @@ static inline void serial8250_em485_rts_after_send(struct uart_8250_port *p)
 	serial8250_out_MCR(p, mcr);
 }
 
-static void serial8250_em485_handle_start_tx(unsigned long arg);
-static void serial8250_em485_handle_stop_tx(unsigned long arg);
+static enum hrtimer_restart serial8250_em485_handle_start_tx(struct hrtimer *t);
+static enum hrtimer_restart serial8250_em485_handle_stop_tx(struct hrtimer *t);
 
 void serial8250_clear_and_reinit_fifos(struct uart_8250_port *p)
 {
@@ -609,12 +617,14 @@ int serial8250_em485_init(struct uart_8250_port *p)
 	if (!p->em485)
 		return -ENOMEM;
 
-	setup_timer(&p->em485->stop_tx_timer,
-		serial8250_em485_handle_stop_tx, (unsigned long)p);
-	setup_timer(&p->em485->start_tx_timer,
-		serial8250_em485_handle_start_tx, (unsigned long)p);
+	hrtimer_init(&p->em485->stop_tx_timer, CLOCK_MONOTONIC,
+		     HRTIMER_MODE_REL);
+	hrtimer_init(&p->em485->start_tx_timer, CLOCK_MONOTONIC,
+		     HRTIMER_MODE_REL);
+	p->em485->stop_tx_timer.function = &serial8250_em485_handle_stop_tx;
+	p->em485->start_tx_timer.function = &serial8250_em485_handle_start_tx;
+	p->em485->port = p;
 	p->em485->active_timer = NULL;
-
 	serial8250_em485_rts_after_send(p);
 
 	return 0;
@@ -639,8 +649,8 @@ void serial8250_em485_destroy(struct uart_8250_port *p)
 	if (!p->em485)
 		return;
 
-	del_timer(&p->em485->start_tx_timer);
-	del_timer(&p->em485->stop_tx_timer);
+	hrtimer_cancel(&p->em485->start_tx_timer);
+	hrtimer_cancel(&p->em485->stop_tx_timer);
 
 	kfree(p->em485);
 	p->em485 = NULL;
@@ -1435,22 +1445,33 @@ static void __do_stop_tx_rs485(struct uart_8250_port *p)
 		serial_port_out(&p->port, UART_IER, p->ier);
 	}
 }
-
-static void serial8250_em485_handle_stop_tx(unsigned long arg)
+static enum hrtimer_restart serial8250_em485_handle_stop_tx(struct hrtimer *t)
 {
-	struct uart_8250_port *p = (struct uart_8250_port *)arg;
-	struct uart_8250_em485 *em485 = p->em485;
+	struct uart_8250_em485 *em485;
+	struct uart_8250_port *p;
 	unsigned long flags;
 
+	em485 = container_of(t, struct uart_8250_em485, stop_tx_timer);
+	p = em485->port;
+
 	serial8250_rpm_get(p);
 	spin_lock_irqsave(&p->port.lock, flags);
-	if (em485 &&
-	    em485->active_timer == &em485->stop_tx_timer) {
+	if (em485->active_timer == &em485->stop_tx_timer) {
 		__do_stop_tx_rs485(p);
 		em485->active_timer = NULL;
 	}
 	spin_unlock_irqrestore(&p->port.lock, flags);
 	serial8250_rpm_put(p);
+	return HRTIMER_NORESTART;
+}
+
+static void start_hrtimer_ms(struct hrtimer *hrt, unsigned long msec)
+{
+	long sec = msec / 1000;
+	long nsec = (msec % 1000) * 1000000;
+	ktime_t t = ktime_set(sec, nsec);
+
+	hrtimer_start(hrt, t, HRTIMER_MODE_REL);
 }
 
 static void __stop_tx_rs485(struct uart_8250_port *p)
@@ -1463,8 +1484,8 @@ static void __stop_tx_rs485(struct uart_8250_port *p)
 	 */
 	if (p->port.rs485.delay_rts_after_send > 0) {
 		em485->active_timer = &em485->stop_tx_timer;
-		mod_timer(&em485->stop_tx_timer, jiffies +
-			p->port.rs485.delay_rts_after_send * HZ / 1000);
+		start_hrtimer_ms(&em485->stop_tx_timer,
+				   p->port.rs485.delay_rts_after_send);
 	} else {
 		__do_stop_tx_rs485(p);
 	}
@@ -1494,8 +1515,8 @@ static inline void __stop_tx(struct uart_8250_port *p)
 		if ((lsr & BOTH_EMPTY) != BOTH_EMPTY)
 			return;
 
-		del_timer(&em485->start_tx_timer);
 		em485->active_timer = NULL;
+		hrtimer_cancel(&em485->start_tx_timer);
 
 		__stop_tx_rs485(p);
 	}
@@ -1558,8 +1579,9 @@ static inline void start_tx_rs485(struct uart_port *port)
 	if (!(up->port.rs485.flags & SER_RS485_RX_DURING_TX))
 		serial8250_stop_rx(&up->port);
 
-	del_timer(&em485->stop_tx_timer);
 	em485->active_timer = NULL;
+	if (hrtimer_is_queued(&em485->stop_tx_timer))
+		hrtimer_cancel(&em485->stop_tx_timer);
 
 	mcr = serial8250_in_MCR(up);
 	if (!!(up->port.rs485.flags & SER_RS485_RTS_ON_SEND) !=
@@ -1572,8 +1594,8 @@ static inline void start_tx_rs485(struct uart_port *port)
 
 		if (up->port.rs485.delay_rts_before_send > 0) {
 			em485->active_timer = &em485->start_tx_timer;
-			mod_timer(&em485->start_tx_timer, jiffies +
-				up->port.rs485.delay_rts_before_send * HZ / 1000);
+			start_hrtimer_ms(&em485->start_tx_timer,
+					 up->port.rs485.delay_rts_before_send);
 			return;
 		}
 	}
@@ -1581,19 +1603,22 @@ static inline void start_tx_rs485(struct uart_port *port)
 	__start_tx(port);
 }
 
-static void serial8250_em485_handle_start_tx(unsigned long arg)
+static enum hrtimer_restart serial8250_em485_handle_start_tx(struct hrtimer *t)
 {
-	struct uart_8250_port *p = (struct uart_8250_port *)arg;
-	struct uart_8250_em485 *em485 = p->em485;
+	struct uart_8250_em485 *em485;
+	struct uart_8250_port *p;
 	unsigned long flags;
 
+	em485 = container_of(t, struct uart_8250_em485, start_tx_timer);
+	p = em485->port;
+
 	spin_lock_irqsave(&p->port.lock, flags);
-	if (em485 &&
-	    em485->active_timer == &em485->start_tx_timer) {
+	if (em485->active_timer == &em485->start_tx_timer) {
 		__start_tx(&p->port);
 		em485->active_timer = NULL;
 	}
 	spin_unlock_irqrestore(&p->port.lock, flags);
+	return HRTIMER_NORESTART;
 }
 
 static void serial8250_start_tx(struct uart_port *port)
@@ -2304,7 +2329,7 @@ int serial8250_do_startup(struct uart_port *port)
 	 * test if we receive TX irq.  This way, we'll never enable
 	 * UART_BUG_TXEN.
 	 */
-	if (up->port.flags & UPF_NO_TXEN_TEST)
+	if (up->port.quirks & UPQ_NO_TXEN_TEST)
 		goto dont_test_tx_en;
 
 	/*
diff --git a/drivers/tty/serial/8250/8250_uniphier.c b/drivers/tty/serial/8250/8250_uniphier.c
index 746680ebf90c..8a10b10e27aa 100644
--- a/drivers/tty/serial/8250/8250_uniphier.c
+++ b/drivers/tty/serial/8250/8250_uniphier.c
@@ -29,12 +29,13 @@
  *   - MMIO32 (regshift = 2)
  *   - FCR is not at 2, but 3
  *   - LCR and MCR are not at 3 and 4, they share 4
+ *   - No SCR (Instead, CHAR can be used as a scratch register)
  *   - Divisor latch at 9, no divisor latch access bit
  */
 
 #define UNIPHIER_UART_REGSHIFT		2
 
-/* bit[15:8] = CHAR (not used), bit[7:0] = FCR */
+/* bit[15:8] = CHAR, bit[7:0] = FCR */
 #define UNIPHIER_UART_CHAR_FCR		(3 << (UNIPHIER_UART_REGSHIFT))
 /* bit[15:8] = LCR, bit[7:0] = MCR */
 #define UNIPHIER_UART_LCR_MCR		(4 << (UNIPHIER_UART_REGSHIFT))
@@ -72,13 +73,18 @@ OF_EARLYCON_DECLARE(uniphier, "socionext,uniphier-uart",
 
 /*
  * The register map is slightly different from that of 8250.
- * IO callbacks must be overridden for correct access to FCR, LCR, and MCR.
+ * IO callbacks must be overridden for correct access to FCR, LCR, MCR and SCR.
  */
 static unsigned int uniphier_serial_in(struct uart_port *p, int offset)
 {
 	unsigned int valshift = 0;
 
 	switch (offset) {
+	case UART_SCR:
+		/* No SCR for this hardware.  Use CHAR as a scratch register */
+		valshift = 8;
+		offset = UNIPHIER_UART_CHAR_FCR;
+		break;
 	case UART_LCR:
 		valshift = 8;
 		/* fall through */
@@ -91,8 +97,8 @@ static unsigned int uniphier_serial_in(struct uart_port *p, int offset)
 	}
 
 	/*
-	 * The return value must be masked with 0xff because LCR and MCR reside
-	 * in the same register that must be accessed by 32-bit write/read.
+	 * The return value must be masked with 0xff because some registers
+	 * share the same offset that must be accessed by 32-bit write/read.
 	 * 8 or 16 bit access to this hardware result in unexpected behavior.
 	 */
 	return (readl(p->membase + offset) >> valshift) & 0xff;
@@ -101,9 +107,13 @@ static unsigned int uniphier_serial_in(struct uart_port *p, int offset)
 static void uniphier_serial_out(struct uart_port *p, int offset, int value)
 {
 	unsigned int valshift = 0;
-	bool normal = true;
+	bool normal = false;
 
 	switch (offset) {
+	case UART_SCR:
+		/* No SCR for this hardware.  Use CHAR as a scratch register */
+		valshift = 8;
+		/* fall through */
 	case UART_FCR:
 		offset = UNIPHIER_UART_CHAR_FCR;
 		break;
@@ -114,10 +124,10 @@ static void uniphier_serial_out(struct uart_port *p, int offset, int value)
 		/* fall through */
 	case UART_MCR:
 		offset = UNIPHIER_UART_LCR_MCR;
-		normal = false;
 		break;
 	default:
 		offset <<= UNIPHIER_UART_REGSHIFT;
+		normal = true;
 		break;
 	}
 
@@ -169,7 +179,7 @@ static int uniphier_of_serial_setup(struct device *dev, struct uart_port *port,
 		dev_err(dev, "failed to get alias id\n");
 		return ret;
 	}
-	port->line = priv->line = ret;
+	port->line = ret;
 
 	/* Get clk rate through clk driver */
 	priv->clk = devm_clk_get(dev, NULL);
@@ -249,8 +259,8 @@ static int uniphier_uart_probe(struct platform_device *pdev)
 	up.dl_read = uniphier_serial_dl_read;
 	up.dl_write = uniphier_serial_dl_write;
 
-	ret = serial8250_register_8250_port(&up);
-	if (ret < 0) {
+	priv->line = serial8250_register_8250_port(&up);
+	if (priv->line < 0) {
 		dev_err(dev, "failed to register 8250 port\n");
 		clk_disable_unprepare(priv->clk);
 		return ret;
@@ -271,6 +281,40 @@ static int uniphier_uart_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static int __maybe_unused uniphier_uart_suspend(struct device *dev)
+{
+	struct uniphier8250_priv *priv = dev_get_drvdata(dev);
+	struct uart_8250_port *up = serial8250_get_port(priv->line);
+
+	serial8250_suspend_port(priv->line);
+
+	if (!uart_console(&up->port) || console_suspend_enabled)
+		clk_disable_unprepare(priv->clk);
+
+	return 0;
+}
+
+static int __maybe_unused uniphier_uart_resume(struct device *dev)
+{
+	struct uniphier8250_priv *priv = dev_get_drvdata(dev);
+	struct uart_8250_port *up = serial8250_get_port(priv->line);
+	int ret;
+
+	if (!uart_console(&up->port) || console_suspend_enabled) {
+		ret = clk_prepare_enable(priv->clk);
+		if (ret)
+			return ret;
+	}
+
+	serial8250_resume_port(priv->line);
+
+	return 0;
+}
+
+static const struct dev_pm_ops uniphier_uart_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(uniphier_uart_suspend, uniphier_uart_resume)
+};
+
 static const struct of_device_id uniphier_uart_match[] = {
 	{ .compatible = "socionext,uniphier-uart" },
 	{ /* sentinel */ }
@@ -283,6 +327,7 @@ static struct platform_driver uniphier_uart_platform_driver = {
 	.driver = {
 		.name	= "uniphier-uart",
 		.of_match_table = uniphier_uart_match,
+		.pm = &uniphier_uart_pm_ops,
 	},
 };
 module_platform_driver(uniphier_uart_platform_driver);
diff --git a/drivers/tty/serial/8250/Kconfig b/drivers/tty/serial/8250/Kconfig
index a1161ec0256f..a5c0ef1e7695 100644
--- a/drivers/tty/serial/8250/Kconfig
+++ b/drivers/tty/serial/8250/Kconfig
@@ -155,6 +155,17 @@ config SERIAL_8250_CS
 
 	  If unsure, say N.
 
+config SERIAL_8250_MEN_MCB
+	tristate "MEN Z125 UART device support"
+	depends on MCB && SERIAL_8250
+	help
+	  This enables support for FPGA based UARTs found on many MEN
+	  boards. This driver enables support for the Z125 UARTs.
+
+	  To compile this driver as a module, chose M here: the
+	  module will be called 8250_men_mcb.
+
+
 config SERIAL_8250_NR_UARTS
 	int "Maximum number of 8250/16550 serial ports"
 	depends on SERIAL_8250
diff --git a/drivers/tty/serial/8250/Makefile b/drivers/tty/serial/8250/Makefile
index a44a99a3e623..6a18d2d768fe 100644
--- a/drivers/tty/serial/8250/Makefile
+++ b/drivers/tty/serial/8250/Makefile
@@ -23,6 +23,7 @@ obj-$(CONFIG_SERIAL_8250_BOCA)		+= 8250_boca.o
 obj-$(CONFIG_SERIAL_8250_EXAR_ST16C554)	+= 8250_exar_st16c554.o
 obj-$(CONFIG_SERIAL_8250_HUB6)		+= 8250_hub6.o
 obj-$(CONFIG_SERIAL_8250_FSL)		+= 8250_fsl.o
+obj-$(CONFIG_SERIAL_8250_MEN_MCB)	+= 8250_men_mcb.o
 obj-$(CONFIG_SERIAL_8250_DW)		+= 8250_dw.o
 obj-$(CONFIG_SERIAL_8250_EM)		+= 8250_em.o
 obj-$(CONFIG_SERIAL_8250_OMAP)		+= 8250_omap.o
diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig
index 1f096e2bb398..b788fee54249 100644
--- a/drivers/tty/serial/Kconfig
+++ b/drivers/tty/serial/Kconfig
@@ -1689,7 +1689,7 @@ config SERIAL_MVEBU_CONSOLE
 	  Otherwise, say 'N'.
 
 config SERIAL_OWL
-	bool "Actions Semi Owl serial port support"
+	tristate "Actions Semi Owl serial port support"
 	depends on ARCH_ACTIONS || COMPILE_TEST
 	select SERIAL_CORE
 	help
@@ -1705,7 +1705,7 @@ config SERIAL_OWL_CONSOLE
 	default y
 	help
 	  Say 'Y' here if you wish to use Actions Semiconductor S500/S900 UART
-	  as the system console. Only earlycon is implemented currently.
+	  as the system console.
 
 endmenu
 
diff --git a/drivers/tty/serial/amba-pl010.c b/drivers/tty/serial/amba-pl010.c
index 24180adb1cbb..9ec4b8d2879f 100644
--- a/drivers/tty/serial/amba-pl010.c
+++ b/drivers/tty/serial/amba-pl010.c
@@ -814,7 +814,7 @@ static int pl010_resume(struct device *dev)
 
 static SIMPLE_DEV_PM_OPS(pl010_dev_pm_ops, pl010_suspend, pl010_resume);
 
-static struct amba_id pl010_ids[] = {
+static const struct amba_id pl010_ids[] = {
 	{
 		.id	= 0x00041010,
 		.mask	= 0x000fffff,
diff --git a/drivers/tty/serial/amba-pl011.c b/drivers/tty/serial/amba-pl011.c
index 1888d168a41c..111e6a950779 100644
--- a/drivers/tty/serial/amba-pl011.c
+++ b/drivers/tty/serial/amba-pl011.c
@@ -128,7 +128,7 @@ static struct vendor_data vendor_arm = {
 	.get_fifosize		= get_fifosize_arm,
 };
 
-static struct vendor_data vendor_sbsa = {
+static const struct vendor_data vendor_sbsa = {
 	.reg_offset		= pl011_std_offsets,
 	.fr_busy		= UART01x_FR_BUSY,
 	.fr_dsr			= UART01x_FR_DSR,
@@ -143,7 +143,7 @@ static struct vendor_data vendor_sbsa = {
 };
 
 #ifdef CONFIG_ACPI_SPCR_TABLE
-static struct vendor_data vendor_qdt_qdf2400_e44 = {
+static const struct vendor_data vendor_qdt_qdf2400_e44 = {
 	.reg_offset		= pl011_std_offsets,
 	.fr_busy		= UART011_FR_TXFE,
 	.fr_dsr			= UART01x_FR_DSR,
@@ -2787,7 +2787,7 @@ static struct platform_driver arm_sbsa_uart_platform_driver = {
 	},
 };
 
-static struct amba_id pl011_ids[] = {
+static const struct amba_id pl011_ids[] = {
 	{
 		.id	= 0x00041011,
 		.mask	= 0x000fffff,
diff --git a/drivers/tty/serial/apbuart.c b/drivers/tty/serial/apbuart.c
index 75eb083b3361..dd60ed96a0ad 100644
--- a/drivers/tty/serial/apbuart.c
+++ b/drivers/tty/serial/apbuart.c
@@ -325,7 +325,7 @@ static int apbuart_verify_port(struct uart_port *port,
 	return ret;
 }
 
-static struct uart_ops grlib_apbuart_ops = {
+static const struct uart_ops grlib_apbuart_ops = {
 	.tx_empty = apbuart_tx_empty,
 	.set_mctrl = apbuart_set_mctrl,
 	.get_mctrl = apbuart_get_mctrl,
diff --git a/drivers/tty/serial/arc_uart.c b/drivers/tty/serial/arc_uart.c
index 5ac06fcaa9c6..77fe306690c4 100644
--- a/drivers/tty/serial/arc_uart.c
+++ b/drivers/tty/serial/arc_uart.c
@@ -549,8 +549,8 @@ static struct console arc_console = {
 	.data	= &arc_uart_driver
 };
 
-static __init void arc_early_serial_write(struct console *con, const char *s,
-					  unsigned int n)
+static void arc_early_serial_write(struct console *con, const char *s,
+				   unsigned int n)
 {
 	struct earlycon_device *dev = con->data;
 
diff --git a/drivers/tty/serial/cpm_uart/cpm_uart_core.c b/drivers/tty/serial/cpm_uart/cpm_uart_core.c
index f6bcc19c99d5..9ac142cfc1f1 100644
--- a/drivers/tty/serial/cpm_uart/cpm_uart_core.c
+++ b/drivers/tty/serial/cpm_uart/cpm_uart_core.c
@@ -1123,7 +1123,7 @@ static void cpm_put_poll_char(struct uart_port *port,
 }
 #endif /* CONFIG_CONSOLE_POLL */
 
-static struct uart_ops cpm_uart_pops = {
+static const struct uart_ops cpm_uart_pops = {
 	.tx_empty	= cpm_uart_tx_empty,
 	.set_mctrl	= cpm_uart_set_mctrl,
 	.get_mctrl	= cpm_uart_get_mctrl,
diff --git a/drivers/tty/serial/earlycon.c b/drivers/tty/serial/earlycon.c
index c3651540e1ba..98928f082d87 100644
--- a/drivers/tty/serial/earlycon.c
+++ b/drivers/tty/serial/earlycon.c
@@ -220,7 +220,7 @@ static int __init param_setup_earlycon(char *buf)
 		if (IS_ENABLED(CONFIG_ACPI_SPCR_TABLE)) {
 			earlycon_init_is_deferred = true;
 			return 0;
-		} else {
+		} else if (!buf) {
 			return early_init_dt_scan_chosen_stdout();
 		}
 	}
@@ -282,7 +282,12 @@ int __init of_setup_earlycon(const struct earlycon_id *match,
 		}
 	}
 
+	val = of_get_flat_dt_prop(node, "current-speed", NULL);
+	if (val)
+		early_console_dev.baud = be32_to_cpu(*val);
+
 	if (options) {
+		early_console_dev.baud = simple_strtoul(options, NULL, 0);
 		strlcpy(early_console_dev.options, options,
 			sizeof(early_console_dev.options));
 	}
diff --git a/drivers/tty/serial/fsl_lpuart.c b/drivers/tty/serial/fsl_lpuart.c
index 898dcb091a27..849c1f9991ce 100644
--- a/drivers/tty/serial/fsl_lpuart.c
+++ b/drivers/tty/serial/fsl_lpuart.c
@@ -117,7 +117,7 @@
 #define UARTSFIFO_TXOF		0x02
 #define UARTSFIFO_RXUF		0x01
 
-/* 32-bit register defination */
+/* 32-bit register definition */
 #define UARTBAUD		0x00
 #define UARTSTAT		0x04
 #define UARTCTRL		0x08
@@ -521,6 +521,57 @@ static int lpuart_poll_get_char(struct uart_port *port)
 	return readb(port->membase + UARTDR);
 }
 
+static int lpuart32_poll_init(struct uart_port *port)
+{
+	unsigned long flags;
+	struct lpuart_port *sport = container_of(port, struct lpuart_port, port);
+	u32 temp;
+
+	sport->port.fifosize = 0;
+
+	spin_lock_irqsave(&sport->port.lock, flags);
+
+	/* Disable Rx & Tx */
+	writel(0, sport->port.membase + UARTCTRL);
+
+	temp = readl(sport->port.membase + UARTFIFO);
+
+	/* Enable Rx and Tx FIFO */
+	writel(temp | UARTFIFO_RXFE | UARTFIFO_TXFE,
+		   sport->port.membase + UARTFIFO);
+
+	/* flush Tx and Rx FIFO */
+	writel(UARTFIFO_TXFLUSH | UARTFIFO_RXFLUSH,
+			sport->port.membase + UARTFIFO);
+
+	/* explicitly clear RDRF */
+	if (readl(sport->port.membase + UARTSTAT) & UARTSTAT_RDRF) {
+		readl(sport->port.membase + UARTDATA);
+		writel(UARTFIFO_RXUF, sport->port.membase + UARTFIFO);
+	}
+
+	/* Enable Rx and Tx */
+	writel(UARTCTRL_RE | UARTCTRL_TE, sport->port.membase + UARTCTRL);
+	spin_unlock_irqrestore(&sport->port.lock, flags);
+
+	return 0;
+}
+
+static void lpuart32_poll_put_char(struct uart_port *port, unsigned char c)
+{
+	while (!(readl(port->membase + UARTSTAT) & UARTSTAT_TDRE))
+		barrier();
+
+	writel(c, port->membase + UARTDATA);
+}
+
+static int lpuart32_poll_get_char(struct uart_port *port)
+{
+	if (!(readl(port->membase + UARTSTAT) & UARTSTAT_RDRF))
+		return NO_POLL_CHAR;
+
+	return readl(port->membase + UARTDATA);
+}
 #endif
 
 static inline void lpuart_transmit_buffer(struct lpuart_port *sport)
@@ -1025,6 +1076,11 @@ static int lpuart_config_rs485(struct uart_port *port,
 		~(UARTMODEM_TXRTSPOL | UARTMODEM_TXRTSE);
 	writeb(modem, sport->port.membase + UARTMODEM);
 
+	/* clear unsupported configurations */
+	rs485->delay_rts_before_send = 0;
+	rs485->delay_rts_after_send = 0;
+	rs485->flags &= ~SER_RS485_RX_DURING_TX;
+
 	if (rs485->flags & SER_RS485_ENABLED) {
 		/* Enable auto RS-485 RTS mode */
 		modem |= UARTMODEM_TXRTSE;
@@ -1782,6 +1838,11 @@ static const struct uart_ops lpuart32_pops = {
 	.config_port	= lpuart_config_port,
 	.verify_port	= lpuart_verify_port,
 	.flush_buffer	= lpuart_flush_buffer,
+#if defined(CONFIG_CONSOLE_POLL)
+	.poll_init	= lpuart32_poll_init,
+	.poll_get_char	= lpuart32_poll_get_char,
+	.poll_put_char	= lpuart32_poll_put_char,
+#endif
 };
 
 static struct lpuart_port *lpuart_ports[UART_NR];
@@ -2203,6 +2264,7 @@ static int lpuart_suspend(struct device *dev)
 {
 	struct lpuart_port *sport = dev_get_drvdata(dev);
 	unsigned long temp;
+	bool irq_wake;
 
 	if (lpuart_is_32(sport)) {
 		/* disable Rx/Tx and interrupts */
@@ -2218,6 +2280,9 @@ static int lpuart_suspend(struct device *dev)
 
 	uart_suspend_port(&lpuart_reg, &sport->port);
 
+	/* uart_suspend_port() might set wakeup flag */
+	irq_wake = irqd_is_wakeup_set(irq_get_irq_data(sport->port.irq));
+
 	if (sport->lpuart_dma_rx_use) {
 		/*
 		 * EDMA driver during suspend will forcefully release any
@@ -2226,7 +2291,7 @@ static int lpuart_suspend(struct device *dev)
 		 * cannot resume as as expected, hence gracefully release the
 		 * Rx DMA path before suspend and start Rx DMA path on resume.
 		 */
-		if (sport->port.irq_wake) {
+		if (irq_wake) {
 			del_timer_sync(&sport->lpuart_timer);
 			lpuart_dma_rx_free(&sport->port);
 		}
@@ -2241,7 +2306,7 @@ static int lpuart_suspend(struct device *dev)
 		dmaengine_terminate_all(sport->dma_tx_chan);
 	}
 
-	if (sport->port.suspended && !sport->port.irq_wake)
+	if (sport->port.suspended && !irq_wake)
 		clk_disable_unprepare(sport->clk);
 
 	return 0;
@@ -2250,9 +2315,10 @@ static int lpuart_suspend(struct device *dev)
 static int lpuart_resume(struct device *dev)
 {
 	struct lpuart_port *sport = dev_get_drvdata(dev);
+	bool irq_wake = irqd_is_wakeup_set(irq_get_irq_data(sport->port.irq));
 	unsigned long temp;
 
-	if (sport->port.suspended && !sport->port.irq_wake)
+	if (sport->port.suspended && !irq_wake)
 		clk_prepare_enable(sport->clk);
 
 	if (lpuart_is_32(sport)) {
@@ -2269,7 +2335,7 @@ static int lpuart_resume(struct device *dev)
 	}
 
 	if (sport->lpuart_dma_rx_use) {
-		if (sport->port.irq_wake) {
+		if (irq_wake) {
 			if (!lpuart_start_rx_dma(sport))
 				rx_dma_timer_init(sport);
 			else
diff --git a/drivers/tty/serial/imx.c b/drivers/tty/serial/imx.c
index 80934e7bd67f..dfeff3951f93 100644
--- a/drivers/tty/serial/imx.c
+++ b/drivers/tty/serial/imx.c
@@ -226,7 +226,6 @@ struct imx_port {
 	dma_cookie_t		rx_cookie;
 	unsigned int		tx_bytes;
 	unsigned int		dma_tx_nents;
-	wait_queue_head_t	dma_wait;
 	unsigned int            saved_reg[10];
 	bool			context_saved;
 };
@@ -458,7 +457,10 @@ static inline void imx_transmit_buffer(struct imx_port *sport)
 		}
 	}
 
-	while (!uart_circ_empty(xmit) && !sport->dma_is_txing &&
+	if (sport->dma_is_txing)
+		return;
+
+	while (!uart_circ_empty(xmit) &&
 	       !(readl(sport->port.membase + uts_reg(sport)) & UTS_TXFULL)) {
 		/* send xmit->buf[xmit->tail]
 		 * out the port here */
@@ -498,20 +500,12 @@ static void dma_tx_callback(void *data)
 
 	sport->dma_is_txing = 0;
 
-	spin_unlock_irqrestore(&sport->port.lock, flags);
-
 	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
 		uart_write_wakeup(&sport->port);
 
-	if (waitqueue_active(&sport->dma_wait)) {
-		wake_up(&sport->dma_wait);
-		dev_dbg(sport->port.dev, "exit in %s.\n", __func__);
-		return;
-	}
-
-	spin_lock_irqsave(&sport->port.lock, flags);
 	if (!uart_circ_empty(xmit) && !uart_tx_stopped(&sport->port))
 		imx_dma_tx(sport);
+
 	spin_unlock_irqrestore(&sport->port.lock, flags);
 }
 
@@ -1208,8 +1202,6 @@ static void imx_enable_dma(struct imx_port *sport)
 {
 	unsigned long temp;
 
-	init_waitqueue_head(&sport->dma_wait);
-
 	/* set UCR1 */
 	temp = readl(sport->port.membase + UCR1);
 	temp |= UCR1_RDMAEN | UCR1_TDMAEN | UCR1_ATDMAEN;
@@ -2332,6 +2324,7 @@ static int imx_serial_port_suspend(struct device *dev)
 	serial_imx_enable_wakeup(sport, true);
 
 	uart_suspend_port(&imx_reg, &sport->port);
+	disable_irq(sport->port.irq);
 
 	/* Needed to enable clock in suspend_noirq */
 	return clk_prepare(sport->clk_ipg);
@@ -2346,6 +2339,7 @@ static int imx_serial_port_resume(struct device *dev)
 	serial_imx_enable_wakeup(sport, false);
 
 	uart_resume_port(&imx_reg, &sport->port);
+	enable_irq(sport->port.irq);
 
 	clk_unprepare(sport->clk_ipg);
 
diff --git a/drivers/tty/serial/jsm/jsm_driver.c b/drivers/tty/serial/jsm/jsm_driver.c
index a119f11bf2f4..102d499814ac 100644
--- a/drivers/tty/serial/jsm/jsm_driver.c
+++ b/drivers/tty/serial/jsm/jsm_driver.c
@@ -304,7 +304,7 @@ static void jsm_remove_one(struct pci_dev *pdev)
 	kfree(brd);
 }
 
-static struct pci_device_id jsm_pci_tbl[] = {
+static const struct pci_device_id jsm_pci_tbl[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_DIGI, PCI_DEVICE_ID_NEO_2DB9), 0, 0, 0 },
 	{ PCI_DEVICE(PCI_VENDOR_ID_DIGI, PCI_DEVICE_ID_NEO_2DB9PRI), 0, 0, 1 },
 	{ PCI_DEVICE(PCI_VENDOR_ID_DIGI, PCI_DEVICE_ID_NEO_2RJ45), 0, 0, 2 },
diff --git a/drivers/tty/serial/m32r_sio.c b/drivers/tty/serial/m32r_sio.c
index 218b7118e85d..5b3bd9511993 100644
--- a/drivers/tty/serial/m32r_sio.c
+++ b/drivers/tty/serial/m32r_sio.c
@@ -854,7 +854,7 @@ m32r_sio_verify_port(struct uart_port *port, struct serial_struct *ser)
 	return 0;
 }
 
-static struct uart_ops m32r_sio_pops = {
+static const struct uart_ops m32r_sio_pops = {
 	.tx_empty	= m32r_sio_tx_empty,
 	.set_mctrl	= m32r_sio_set_mctrl,
 	.get_mctrl	= m32r_sio_get_mctrl,
diff --git a/drivers/tty/serial/meson_uart.c b/drivers/tty/serial/meson_uart.c
index 42e4a4c7597f..07c0f98be3ac 100644
--- a/drivers/tty/serial/meson_uart.c
+++ b/drivers/tty/serial/meson_uart.c
@@ -424,7 +424,7 @@ static void meson_uart_config_port(struct uart_port *port, int flags)
 	}
 }
 
-static struct uart_ops meson_uart_ops = {
+static const struct uart_ops meson_uart_ops = {
 	.set_mctrl      = meson_uart_set_mctrl,
 	.get_mctrl      = meson_uart_get_mctrl,
 	.tx_empty	= meson_uart_tx_empty,
diff --git a/drivers/tty/serial/mpc52xx_uart.c b/drivers/tty/serial/mpc52xx_uart.c
index 3970d6a9aaca..791c4c74f6d6 100644
--- a/drivers/tty/serial/mpc52xx_uart.c
+++ b/drivers/tty/serial/mpc52xx_uart.c
@@ -1347,7 +1347,7 @@ mpc52xx_uart_verify_port(struct uart_port *port, struct serial_struct *ser)
 }
 
 
-static struct uart_ops mpc52xx_uart_ops = {
+static const struct uart_ops mpc52xx_uart_ops = {
 	.tx_empty	= mpc52xx_uart_tx_empty,
 	.set_mctrl	= mpc52xx_uart_set_mctrl,
 	.get_mctrl	= mpc52xx_uart_get_mctrl,
@@ -1634,8 +1634,8 @@ mpc52xx_console_setup(struct console *co, char *options)
 		return -EINVAL;
 	}
 
-	pr_debug("Console on ttyPSC%x is %s\n",
-		 co->index, mpc52xx_uart_nodes[co->index]->full_name);
+	pr_debug("Console on ttyPSC%x is %pOF\n",
+		 co->index, mpc52xx_uart_nodes[co->index]);
 
 	/* Fetch register locations */
 	ret = of_address_to_resource(np, 0, &res);
@@ -1755,8 +1755,8 @@ static int mpc52xx_uart_of_probe(struct platform_device *op)
 			break;
 	if (idx >= MPC52xx_PSC_MAXNUM)
 		return -EINVAL;
-	pr_debug("Found %s assigned to ttyPSC%x\n",
-		 mpc52xx_uart_nodes[idx]->full_name, idx);
+	pr_debug("Found %pOF assigned to ttyPSC%x\n",
+		 mpc52xx_uart_nodes[idx], idx);
 
 	/* set the uart clock to the input clock of the psc, the different
 	 * prescalers are taken into account in the set_baudrate() methods
@@ -1881,8 +1881,8 @@ mpc52xx_uart_of_enumerate(void)
 
 	for (i = 0; i < MPC52xx_PSC_MAXNUM; i++) {
 		if (mpc52xx_uart_nodes[i])
-			pr_debug("%s assigned to ttyPSC%x\n",
-				 mpc52xx_uart_nodes[i]->full_name, i);
+			pr_debug("%pOF assigned to ttyPSC%x\n",
+				 mpc52xx_uart_nodes[i], i);
 	}
 }
 
diff --git a/drivers/tty/serial/msm_serial.c b/drivers/tty/serial/msm_serial.c
index 6788e7532dff..1db79ee8a886 100644
--- a/drivers/tty/serial/msm_serial.c
+++ b/drivers/tty/serial/msm_serial.c
@@ -1175,11 +1175,6 @@ static int msm_startup(struct uart_port *port)
 	snprintf(msm_port->name, sizeof(msm_port->name),
 		 "msm_serial%d", port->line);
 
-	ret = request_irq(port->irq, msm_uart_irq, IRQF_TRIGGER_HIGH,
-			  msm_port->name, port);
-	if (unlikely(ret))
-		return ret;
-
 	msm_init_clock(port);
 
 	if (likely(port->fifosize > 12))
@@ -1206,7 +1201,21 @@ static int msm_startup(struct uart_port *port)
 		msm_request_rx_dma(msm_port, msm_port->uart.mapbase);
 	}
 
+	ret = request_irq(port->irq, msm_uart_irq, IRQF_TRIGGER_HIGH,
+			  msm_port->name, port);
+	if (unlikely(ret))
+		goto err_irq;
+
 	return 0;
+
+err_irq:
+	if (msm_port->is_uartdm)
+		msm_release_dma(msm_port);
+
+	clk_disable_unprepare(msm_port->pclk);
+	clk_disable_unprepare(msm_port->clk);
+
+	return ret;
 }
 
 static void msm_shutdown(struct uart_port *port)
diff --git a/drivers/tty/serial/mux.c b/drivers/tty/serial/mux.c
index 8a4be4b73723..2bff69e70e4b 100644
--- a/drivers/tty/serial/mux.c
+++ b/drivers/tty/serial/mux.c
@@ -427,7 +427,7 @@ static struct console mux_console = {
 #define MUX_CONSOLE	NULL
 #endif
 
-static struct uart_ops mux_pops = {
+static const struct uart_ops mux_pops = {
 	.tx_empty =		mux_tx_empty,
 	.set_mctrl =		mux_set_mctrl,
 	.get_mctrl =		mux_get_mctrl,
@@ -503,7 +503,7 @@ static int __init mux_probe(struct parisc_device *dev)
 	return 0;
 }
 
-static int mux_remove(struct parisc_device *dev)
+static int __exit mux_remove(struct parisc_device *dev)
 {
 	int i, j;
 	int port_count = (long)dev_get_drvdata(&dev->dev);
@@ -536,13 +536,13 @@ static int mux_remove(struct parisc_device *dev)
  * This table only contains the parisc_device_id of known builtin mux
  * devices.  All other mux cards will be detected by the generic mux_tbl.
  */
-static struct parisc_device_id builtin_mux_tbl[] = {
+static const struct parisc_device_id builtin_mux_tbl[] __initconst = {
 	{ HPHW_A_DIRECT, HVERSION_REV_ANY_ID, 0x15, 0x0000D }, /* All K-class */
 	{ HPHW_A_DIRECT, HVERSION_REV_ANY_ID, 0x44, 0x0000D }, /* E35, E45, and E55 */
 	{ 0, }
 };
 
-static struct parisc_device_id mux_tbl[] = {
+static const struct parisc_device_id mux_tbl[] __initconst = {
 	{ HPHW_A_DIRECT, HVERSION_REV_ANY_ID, HVERSION_ANY_ID, 0x0000D },
 	{ 0, }
 };
@@ -550,18 +550,18 @@ static struct parisc_device_id mux_tbl[] = {
 MODULE_DEVICE_TABLE(parisc, builtin_mux_tbl);
 MODULE_DEVICE_TABLE(parisc, mux_tbl);
 
-static struct parisc_driver builtin_serial_mux_driver = {
+static struct parisc_driver builtin_serial_mux_driver __refdata = {
 	.name =		"builtin_serial_mux",
 	.id_table =	builtin_mux_tbl,
 	.probe =	mux_probe,
-	.remove =       mux_remove,
+	.remove =       __exit_p(mux_remove),
 };
 
-static struct parisc_driver serial_mux_driver = {
+static struct parisc_driver serial_mux_driver __refdata = {
 	.name =		"serial_mux",
 	.id_table =	mux_tbl,
 	.probe =	mux_probe,
-	.remove =       mux_remove,
+	.remove =       __exit_p(mux_remove),
 };
 
 /**
diff --git a/drivers/tty/serial/omap-serial.c b/drivers/tty/serial/omap-serial.c
index 1ea05ac57aa7..7754053deeda 100644
--- a/drivers/tty/serial/omap-serial.c
+++ b/drivers/tty/serial/omap-serial.c
@@ -1235,21 +1235,20 @@ out:
 #ifdef CONFIG_SERIAL_OMAP_CONSOLE
 
 #ifdef CONFIG_SERIAL_EARLYCON
-static unsigned int __init omap_serial_early_in(struct uart_port *port,
-						int offset)
+static unsigned int omap_serial_early_in(struct uart_port *port, int offset)
 {
 	offset <<= port->regshift;
 	return readw(port->membase + offset);
 }
 
-static void __init omap_serial_early_out(struct uart_port *port, int offset,
-					 int value)
+static void omap_serial_early_out(struct uart_port *port, int offset,
+				  int value)
 {
 	offset <<= port->regshift;
 	writew(value, port->membase + offset);
 }
 
-static void __init omap_serial_early_putc(struct uart_port *port, int c)
+static void omap_serial_early_putc(struct uart_port *port, int c)
 {
 	unsigned int status;
 
@@ -1262,8 +1261,8 @@ static void __init omap_serial_early_putc(struct uart_port *port, int c)
 	omap_serial_early_out(port, UART_TX, c);
 }
 
-static void __init early_omap_serial_write(struct console *console,
-					   const char *s, unsigned int count)
+static void early_omap_serial_write(struct console *console, const char *s,
+				    unsigned int count)
 {
 	struct earlycon_device *device = console->data;
 	struct uart_port *port = &device->port;
diff --git a/drivers/tty/serial/owl-uart.c b/drivers/tty/serial/owl-uart.c
index 1b8008797a1b..b9c859365334 100644
--- a/drivers/tty/serial/owl-uart.c
+++ b/drivers/tty/serial/owl-uart.c
@@ -20,6 +20,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <linux/clk.h>
 #include <linux/console.h>
 #include <linux/delay.h>
 #include <linux/io.h>
@@ -28,22 +29,66 @@
 #include <linux/platform_device.h>
 #include <linux/serial.h>
 #include <linux/serial_core.h>
+#include <linux/tty.h>
+#include <linux/tty_flip.h>
+
+#define OWL_UART_PORT_NUM 7
+#define OWL_UART_DEV_NAME "ttyOWL"
 
 #define OWL_UART_CTL	0x000
+#define OWL_UART_RXDAT	0x004
 #define OWL_UART_TXDAT	0x008
 #define OWL_UART_STAT	0x00c
 
+#define OWL_UART_CTL_DWLS_MASK		GENMASK(1, 0)
+#define OWL_UART_CTL_DWLS_5BITS		(0x0 << 0)
+#define OWL_UART_CTL_DWLS_6BITS		(0x1 << 0)
+#define OWL_UART_CTL_DWLS_7BITS		(0x2 << 0)
+#define OWL_UART_CTL_DWLS_8BITS		(0x3 << 0)
+#define OWL_UART_CTL_STPS_2BITS		BIT(2)
+#define OWL_UART_CTL_PRS_MASK		GENMASK(6, 4)
+#define OWL_UART_CTL_PRS_NONE		(0x0 << 4)
+#define OWL_UART_CTL_PRS_ODD		(0x4 << 4)
+#define OWL_UART_CTL_PRS_MARK		(0x5 << 4)
+#define OWL_UART_CTL_PRS_EVEN		(0x6 << 4)
+#define OWL_UART_CTL_PRS_SPACE		(0x7 << 4)
+#define OWL_UART_CTL_AFE		BIT(12)
 #define OWL_UART_CTL_TRFS_TX		BIT(14)
 #define OWL_UART_CTL_EN			BIT(15)
+#define OWL_UART_CTL_RXDE		BIT(16)
+#define OWL_UART_CTL_TXDE		BIT(17)
 #define OWL_UART_CTL_RXIE		BIT(18)
 #define OWL_UART_CTL_TXIE		BIT(19)
+#define OWL_UART_CTL_LBEN		BIT(20)
 
 #define OWL_UART_STAT_RIP		BIT(0)
 #define OWL_UART_STAT_TIP		BIT(1)
+#define OWL_UART_STAT_RXER		BIT(2)
+#define OWL_UART_STAT_TFER		BIT(3)
+#define OWL_UART_STAT_RXST		BIT(4)
+#define OWL_UART_STAT_RFEM		BIT(5)
 #define OWL_UART_STAT_TFFU		BIT(6)
-#define OWL_UART_STAT_TRFL_MASK		(0x1f << 11)
+#define OWL_UART_STAT_CTSS		BIT(7)
+#define OWL_UART_STAT_RTSS		BIT(8)
+#define OWL_UART_STAT_TFES		BIT(10)
+#define OWL_UART_STAT_TRFL_MASK		GENMASK(16, 11)
 #define OWL_UART_STAT_UTBB		BIT(17)
 
+static struct uart_driver owl_uart_driver;
+
+struct owl_uart_info {
+	unsigned int tx_fifosize;
+};
+
+struct owl_uart_port {
+	struct uart_port port;
+	struct clk *clk;
+};
+
+#define to_owl_uart_port(prt) container_of(prt, struct owl_uart_port, prt)
+
+static struct owl_uart_port *owl_uart_ports[OWL_UART_PORT_NUM];
+
 static inline void owl_uart_write(struct uart_port *port, u32 val, unsigned int off)
 {
 	writel(val, port->membase + off);
@@ -54,6 +99,397 @@ static inline u32 owl_uart_read(struct uart_port *port, unsigned int off)
 	return readl(port->membase + off);
 }
 
+static void owl_uart_set_mctrl(struct uart_port *port, unsigned int mctrl)
+{
+	u32 ctl;
+
+	ctl = owl_uart_read(port, OWL_UART_CTL);
+
+	if (mctrl & TIOCM_LOOP)
+		ctl |= OWL_UART_CTL_LBEN;
+	else
+		ctl &= ~OWL_UART_CTL_LBEN;
+
+	owl_uart_write(port, ctl, OWL_UART_CTL);
+}
+
+static unsigned int owl_uart_get_mctrl(struct uart_port *port)
+{
+	unsigned int mctrl = TIOCM_CAR | TIOCM_DSR;
+	u32 stat, ctl;
+
+	ctl = owl_uart_read(port, OWL_UART_CTL);
+	stat = owl_uart_read(port, OWL_UART_STAT);
+	if (stat & OWL_UART_STAT_RTSS)
+		mctrl |= TIOCM_RTS;
+	if ((stat & OWL_UART_STAT_CTSS) || !(ctl & OWL_UART_CTL_AFE))
+		mctrl |= TIOCM_CTS;
+	return mctrl;
+}
+
+static unsigned int owl_uart_tx_empty(struct uart_port *port)
+{
+	unsigned long flags;
+	u32 val;
+	unsigned int ret;
+
+	spin_lock_irqsave(&port->lock, flags);
+
+	val = owl_uart_read(port, OWL_UART_STAT);
+	ret = (val & OWL_UART_STAT_TFES) ? TIOCSER_TEMT : 0;
+
+	spin_unlock_irqrestore(&port->lock, flags);
+
+	return ret;
+}
+
+static void owl_uart_stop_rx(struct uart_port *port)
+{
+	u32 val;
+
+	val = owl_uart_read(port, OWL_UART_CTL);
+	val &= ~(OWL_UART_CTL_RXIE | OWL_UART_CTL_RXDE);
+	owl_uart_write(port, val, OWL_UART_CTL);
+
+	val = owl_uart_read(port, OWL_UART_STAT);
+	val |= OWL_UART_STAT_RIP;
+	owl_uart_write(port, val, OWL_UART_STAT);
+}
+
+static void owl_uart_stop_tx(struct uart_port *port)
+{
+	u32 val;
+
+	val = owl_uart_read(port, OWL_UART_CTL);
+	val &= ~(OWL_UART_CTL_TXIE | OWL_UART_CTL_TXDE);
+	owl_uart_write(port, val, OWL_UART_CTL);
+
+	val = owl_uart_read(port, OWL_UART_STAT);
+	val |= OWL_UART_STAT_TIP;
+	owl_uart_write(port, val, OWL_UART_STAT);
+}
+
+static void owl_uart_start_tx(struct uart_port *port)
+{
+	u32 val;
+
+	if (uart_tx_stopped(port)) {
+		owl_uart_stop_tx(port);
+		return;
+	}
+
+	val = owl_uart_read(port, OWL_UART_STAT);
+	val |= OWL_UART_STAT_TIP;
+	owl_uart_write(port, val, OWL_UART_STAT);
+
+	val = owl_uart_read(port, OWL_UART_CTL);
+	val |= OWL_UART_CTL_TXIE;
+	owl_uart_write(port, val, OWL_UART_CTL);
+}
+
+static void owl_uart_send_chars(struct uart_port *port)
+{
+	struct circ_buf *xmit = &port->state->xmit;
+	unsigned int ch;
+
+	if (uart_tx_stopped(port))
+		return;
+
+	if (port->x_char) {
+		while (!(owl_uart_read(port, OWL_UART_STAT) & OWL_UART_STAT_TFFU))
+			cpu_relax();
+		owl_uart_write(port, port->x_char, OWL_UART_TXDAT);
+		port->icount.tx++;
+		port->x_char = 0;
+	}
+
+	while (!(owl_uart_read(port, OWL_UART_STAT) & OWL_UART_STAT_TFFU)) {
+		if (uart_circ_empty(xmit))
+			break;
+
+		ch = xmit->buf[xmit->tail];
+		owl_uart_write(port, ch, OWL_UART_TXDAT);
+		xmit->tail = (xmit->tail + 1) & (SERIAL_XMIT_SIZE - 1);
+		port->icount.tx++;
+	}
+
+	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
+		uart_write_wakeup(port);
+
+	if (uart_circ_empty(xmit))
+		owl_uart_stop_tx(port);
+}
+
+static void owl_uart_receive_chars(struct uart_port *port)
+{
+	u32 stat, val;
+
+	val = owl_uart_read(port, OWL_UART_CTL);
+	val &= ~OWL_UART_CTL_TRFS_TX;
+	owl_uart_write(port, val, OWL_UART_CTL);
+
+	stat = owl_uart_read(port, OWL_UART_STAT);
+	while (!(stat & OWL_UART_STAT_RFEM)) {
+		char flag = TTY_NORMAL;
+
+		if (stat & OWL_UART_STAT_RXER)
+			port->icount.overrun++;
+
+		if (stat & OWL_UART_STAT_RXST) {
+			/* We are not able to distinguish the error type. */
+			port->icount.brk++;
+			port->icount.frame++;
+
+			stat &= port->read_status_mask;
+			if (stat & OWL_UART_STAT_RXST)
+				flag = TTY_PARITY;
+		} else
+			port->icount.rx++;
+
+		val = owl_uart_read(port, OWL_UART_RXDAT);
+		val &= 0xff;
+
+		if ((stat & port->ignore_status_mask) == 0)
+			tty_insert_flip_char(&port->state->port, val, flag);
+
+		stat = owl_uart_read(port, OWL_UART_STAT);
+	}
+
+	spin_unlock(&port->lock);
+	tty_flip_buffer_push(&port->state->port);
+	spin_lock(&port->lock);
+}
+
+static irqreturn_t owl_uart_irq(int irq, void *dev_id)
+{
+	struct uart_port *port = dev_id;
+	unsigned long flags;
+	u32 stat;
+
+	spin_lock_irqsave(&port->lock, flags);
+
+	stat = owl_uart_read(port, OWL_UART_STAT);
+
+	if (stat & OWL_UART_STAT_RIP)
+		owl_uart_receive_chars(port);
+
+	if (stat & OWL_UART_STAT_TIP)
+		owl_uart_send_chars(port);
+
+	stat = owl_uart_read(port, OWL_UART_STAT);
+	stat |= OWL_UART_STAT_RIP | OWL_UART_STAT_TIP;
+	owl_uart_write(port, stat, OWL_UART_STAT);
+
+	spin_unlock_irqrestore(&port->lock, flags);
+
+	return IRQ_HANDLED;
+}
+
+static void owl_uart_shutdown(struct uart_port *port)
+{
+	u32 val;
+	unsigned long flags;
+
+	spin_lock_irqsave(&port->lock, flags);
+
+	val = owl_uart_read(port, OWL_UART_CTL);
+	val &= ~(OWL_UART_CTL_TXIE | OWL_UART_CTL_RXIE
+		| OWL_UART_CTL_TXDE | OWL_UART_CTL_RXDE | OWL_UART_CTL_EN);
+	owl_uart_write(port, val, OWL_UART_CTL);
+
+	spin_unlock_irqrestore(&port->lock, flags);
+
+	free_irq(port->irq, port);
+}
+
+static int owl_uart_startup(struct uart_port *port)
+{
+	u32 val;
+	unsigned long flags;
+	int ret;
+
+	ret = request_irq(port->irq, owl_uart_irq, IRQF_TRIGGER_HIGH,
+			"owl-uart", port);
+	if (ret)
+		return ret;
+
+	spin_lock_irqsave(&port->lock, flags);
+
+	val = owl_uart_read(port, OWL_UART_STAT);
+	val |= OWL_UART_STAT_RIP | OWL_UART_STAT_TIP
+		| OWL_UART_STAT_RXER | OWL_UART_STAT_TFER | OWL_UART_STAT_RXST;
+	owl_uart_write(port, val, OWL_UART_STAT);
+
+	val = owl_uart_read(port, OWL_UART_CTL);
+	val |= OWL_UART_CTL_RXIE | OWL_UART_CTL_TXIE;
+	val |= OWL_UART_CTL_EN;
+	owl_uart_write(port, val, OWL_UART_CTL);
+
+	spin_unlock_irqrestore(&port->lock, flags);
+
+	return 0;
+}
+
+static void owl_uart_change_baudrate(struct owl_uart_port *owl_port,
+				     unsigned long baud)
+{
+	clk_set_rate(owl_port->clk, baud * 8);
+}
+
+static void owl_uart_set_termios(struct uart_port *port,
+				 struct ktermios *termios,
+				 struct ktermios *old)
+{
+	struct owl_uart_port *owl_port = to_owl_uart_port(port);
+	unsigned int baud;
+	u32 ctl;
+	unsigned long flags;
+
+	spin_lock_irqsave(&port->lock, flags);
+
+	ctl = owl_uart_read(port, OWL_UART_CTL);
+
+	ctl &= ~OWL_UART_CTL_DWLS_MASK;
+	switch (termios->c_cflag & CSIZE) {
+	case CS5:
+		ctl |= OWL_UART_CTL_DWLS_5BITS;
+		break;
+	case CS6:
+		ctl |= OWL_UART_CTL_DWLS_6BITS;
+		break;
+	case CS7:
+		ctl |= OWL_UART_CTL_DWLS_7BITS;
+		break;
+	case CS8:
+	default:
+		ctl |= OWL_UART_CTL_DWLS_8BITS;
+		break;
+	}
+
+	if (termios->c_cflag & CSTOPB)
+		ctl |= OWL_UART_CTL_STPS_2BITS;
+	else
+		ctl &= ~OWL_UART_CTL_STPS_2BITS;
+
+	ctl &= ~OWL_UART_CTL_PRS_MASK;
+	if (termios->c_cflag & PARENB) {
+		if (termios->c_cflag & CMSPAR) {
+			if (termios->c_cflag & PARODD)
+				ctl |= OWL_UART_CTL_PRS_MARK;
+			else
+				ctl |= OWL_UART_CTL_PRS_SPACE;
+		} else if (termios->c_cflag & PARODD)
+			ctl |= OWL_UART_CTL_PRS_ODD;
+		else
+			ctl |= OWL_UART_CTL_PRS_EVEN;
+	} else
+		ctl |= OWL_UART_CTL_PRS_NONE;
+
+	if (termios->c_cflag & CRTSCTS)
+		ctl |= OWL_UART_CTL_AFE;
+	else
+		ctl &= ~OWL_UART_CTL_AFE;
+
+	owl_uart_write(port, ctl, OWL_UART_CTL);
+
+	baud = uart_get_baud_rate(port, termios, old, 9600, 3200000);
+	owl_uart_change_baudrate(owl_port, baud);
+
+	/* Don't rewrite B0 */
+	if (tty_termios_baud_rate(termios))
+		tty_termios_encode_baud_rate(termios, baud, baud);
+
+	port->read_status_mask |= OWL_UART_STAT_RXER;
+	if (termios->c_iflag & INPCK)
+		port->read_status_mask |= OWL_UART_STAT_RXST;
+
+	uart_update_timeout(port, termios->c_cflag, baud);
+
+	spin_unlock_irqrestore(&port->lock, flags);
+}
+
+static void owl_uart_release_port(struct uart_port *port)
+{
+	struct platform_device *pdev = to_platform_device(port->dev);
+	struct resource *res;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res)
+		return;
+
+	if (port->flags & UPF_IOREMAP) {
+		devm_release_mem_region(port->dev, port->mapbase,
+			resource_size(res));
+		devm_iounmap(port->dev, port->membase);
+		port->membase = NULL;
+	}
+}
+
+static int owl_uart_request_port(struct uart_port *port)
+{
+	struct platform_device *pdev = to_platform_device(port->dev);
+	struct resource *res;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res)
+		return -ENXIO;
+
+	if (!devm_request_mem_region(port->dev, port->mapbase,
+			resource_size(res), dev_name(port->dev)))
+		return -EBUSY;
+
+	if (port->flags & UPF_IOREMAP) {
+		port->membase = devm_ioremap_nocache(port->dev, port->mapbase,
+				resource_size(res));
+		if (!port->membase)
+			return -EBUSY;
+	}
+
+	return 0;
+}
+
+static const char *owl_uart_type(struct uart_port *port)
+{
+	return (port->type == PORT_OWL) ? "owl-uart" : NULL;
+}
+
+static int owl_uart_verify_port(struct uart_port *port,
+				struct serial_struct *ser)
+{
+	if (port->type != PORT_OWL)
+		return -EINVAL;
+
+	if (port->irq != ser->irq)
+		return -EINVAL;
+
+	return 0;
+}
+
+static void owl_uart_config_port(struct uart_port *port, int flags)
+{
+	if (flags & UART_CONFIG_TYPE) {
+		port->type = PORT_OWL;
+		owl_uart_request_port(port);
+	}
+}
+
+static const struct uart_ops owl_uart_ops = {
+	.set_mctrl = owl_uart_set_mctrl,
+	.get_mctrl = owl_uart_get_mctrl,
+	.tx_empty = owl_uart_tx_empty,
+	.start_tx = owl_uart_start_tx,
+	.stop_rx = owl_uart_stop_rx,
+	.stop_tx = owl_uart_stop_tx,
+	.startup = owl_uart_startup,
+	.shutdown = owl_uart_shutdown,
+	.set_termios = owl_uart_set_termios,
+	.type = owl_uart_type,
+	.config_port = owl_uart_config_port,
+	.request_port = owl_uart_request_port,
+	.release_port = owl_uart_release_port,
+	.verify_port = owl_uart_verify_port,
+};
+
 #ifdef CONFIG_SERIAL_OWL_CONSOLE
 
 static void owl_console_putchar(struct uart_port *port, int ch)
@@ -110,6 +546,57 @@ static void owl_uart_port_write(struct uart_port *port, const char *s,
 	local_irq_restore(flags);
 }
 
+static void owl_uart_console_write(struct console *co, const char *s,
+				   u_int count)
+{
+	struct owl_uart_port *owl_port;
+
+	owl_port = owl_uart_ports[co->index];
+	if (!owl_port)
+		return;
+
+	owl_uart_port_write(&owl_port->port, s, count);
+}
+
+static int owl_uart_console_setup(struct console *co, char *options)
+{
+	struct owl_uart_port *owl_port;
+	int baud = 115200;
+	int bits = 8;
+	int parity = 'n';
+	int flow = 'n';
+
+	if (co->index < 0 || co->index >= OWL_UART_PORT_NUM)
+		return -EINVAL;
+
+	owl_port = owl_uart_ports[co->index];
+	if (!owl_port || !owl_port->port.membase)
+		return -ENODEV;
+
+	if (options)
+		uart_parse_options(options, &baud, &parity, &bits, &flow);
+
+	return uart_set_options(&owl_port->port, co, baud, parity, bits, flow);
+}
+
+static struct console owl_uart_console = {
+	.name = OWL_UART_DEV_NAME,
+	.write = owl_uart_console_write,
+	.device = uart_console_device,
+	.setup = owl_uart_console_setup,
+	.flags = CON_PRINTBUFFER,
+	.index = -1,
+	.data = &owl_uart_driver,
+};
+
+static int __init owl_uart_console_init(void)
+{
+	register_console(&owl_uart_console);
+
+	return 0;
+}
+console_initcall(owl_uart_console_init);
+
 static void owl_uart_early_console_write(struct console *co,
 					 const char *s,
 					 u_int count)
@@ -132,4 +619,148 @@ owl_uart_early_console_setup(struct earlycon_device *device, const char *opt)
 OF_EARLYCON_DECLARE(owl, "actions,owl-uart",
 		    owl_uart_early_console_setup);
 
-#endif /* CONFIG_SERIAL_OWL_CONSOLE */
+#define OWL_UART_CONSOLE (&owl_uart_console)
+#else
+#define OWL_UART_CONSOLE NULL
+#endif
+
+static struct uart_driver owl_uart_driver = {
+	.owner = THIS_MODULE,
+	.driver_name = "owl-uart",
+	.dev_name = OWL_UART_DEV_NAME,
+	.nr = OWL_UART_PORT_NUM,
+	.cons = OWL_UART_CONSOLE,
+};
+
+static const struct owl_uart_info owl_s500_info = {
+	.tx_fifosize = 16,
+};
+
+static const struct owl_uart_info owl_s900_info = {
+	.tx_fifosize = 32,
+};
+
+static const struct of_device_id owl_uart_dt_matches[] = {
+	{ .compatible = "actions,s500-uart", .data = &owl_s500_info },
+	{ .compatible = "actions,s900-uart", .data = &owl_s900_info },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, owl_uart_dt_matches);
+
+static int owl_uart_probe(struct platform_device *pdev)
+{
+	const struct of_device_id *match;
+	const struct owl_uart_info *info = NULL;
+	struct resource *res_mem;
+	struct owl_uart_port *owl_port;
+	int ret, irq;
+
+	if (pdev->dev.of_node) {
+		pdev->id = of_alias_get_id(pdev->dev.of_node, "serial");
+		match = of_match_node(owl_uart_dt_matches, pdev->dev.of_node);
+		if (match)
+			info = match->data;
+	}
+
+	if (pdev->id < 0 || pdev->id >= OWL_UART_PORT_NUM) {
+		dev_err(&pdev->dev, "id %d out of range\n", pdev->id);
+		return -EINVAL;
+	}
+
+	res_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res_mem) {
+		dev_err(&pdev->dev, "could not get mem\n");
+		return -ENODEV;
+	}
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		dev_err(&pdev->dev, "could not get irq\n");
+		return irq;
+	}
+
+	if (owl_uart_ports[pdev->id]) {
+		dev_err(&pdev->dev, "port %d already allocated\n", pdev->id);
+		return -EBUSY;
+	}
+
+	owl_port = devm_kzalloc(&pdev->dev, sizeof(*owl_port), GFP_KERNEL);
+	if (!owl_port)
+		return -ENOMEM;
+
+	owl_port->clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(owl_port->clk)) {
+		dev_err(&pdev->dev, "could not get clk\n");
+		return PTR_ERR(owl_port->clk);
+	}
+
+	owl_port->port.dev = &pdev->dev;
+	owl_port->port.line = pdev->id;
+	owl_port->port.type = PORT_OWL;
+	owl_port->port.iotype = UPIO_MEM;
+	owl_port->port.mapbase = res_mem->start;
+	owl_port->port.irq = irq;
+	owl_port->port.uartclk = clk_get_rate(owl_port->clk);
+	if (owl_port->port.uartclk == 0) {
+		dev_err(&pdev->dev, "clock rate is zero\n");
+		return -EINVAL;
+	}
+	owl_port->port.flags = UPF_BOOT_AUTOCONF | UPF_IOREMAP | UPF_LOW_LATENCY;
+	owl_port->port.x_char = 0;
+	owl_port->port.fifosize = (info) ? info->tx_fifosize : 16;
+	owl_port->port.ops = &owl_uart_ops;
+
+	owl_uart_ports[pdev->id] = owl_port;
+	platform_set_drvdata(pdev, owl_port);
+
+	ret = uart_add_one_port(&owl_uart_driver, &owl_port->port);
+	if (ret)
+		owl_uart_ports[pdev->id] = NULL;
+
+	return ret;
+}
+
+static int owl_uart_remove(struct platform_device *pdev)
+{
+	struct owl_uart_port *owl_port = platform_get_drvdata(pdev);
+
+	uart_remove_one_port(&owl_uart_driver, &owl_port->port);
+	owl_uart_ports[pdev->id] = NULL;
+
+	return 0;
+}
+
+static struct platform_driver owl_uart_platform_driver = {
+	.probe = owl_uart_probe,
+	.remove = owl_uart_remove,
+	.driver = {
+		.name = "owl-uart",
+		.of_match_table = owl_uart_dt_matches,
+	},
+};
+
+static int __init owl_uart_init(void)
+{
+	int ret;
+
+	ret = uart_register_driver(&owl_uart_driver);
+	if (ret)
+		return ret;
+
+	ret = platform_driver_register(&owl_uart_platform_driver);
+	if (ret)
+		uart_unregister_driver(&owl_uart_driver);
+
+	return ret;
+}
+
+static void __init owl_uart_exit(void)
+{
+	platform_driver_unregister(&owl_uart_platform_driver);
+	uart_unregister_driver(&owl_uart_driver);
+}
+
+module_init(owl_uart_init);
+module_exit(owl_uart_exit);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/tty/serial/pch_uart.c b/drivers/tty/serial/pch_uart.c
index d3796dc26fa9..d9123f995705 100644
--- a/drivers/tty/serial/pch_uart.c
+++ b/drivers/tty/serial/pch_uart.c
@@ -46,11 +46,6 @@ enum {
 	PCH_UART_HANDLED_LS_INT_SHIFT,
 };
 
-enum {
-	PCH_UART_8LINE,
-	PCH_UART_2LINE,
-};
-
 #define PCH_UART_DRIVER_DEVICE "ttyPCH"
 
 /* Set the max number of UART port
@@ -267,7 +262,7 @@ struct eg20t_port {
 
 /**
  * struct pch_uart_driver_data - private data structure for UART-DMA
- * @port_type:			The number of DMA channel
+ * @port_type:			The type of UART port
  * @line_no:			UART port line number (0, 1, 2...)
  */
 struct pch_uart_driver_data {
@@ -290,17 +285,17 @@ enum pch_uart_num_t {
 };
 
 static struct pch_uart_driver_data drv_dat[] = {
-	[pch_et20t_uart0] = {PCH_UART_8LINE, 0},
-	[pch_et20t_uart1] = {PCH_UART_2LINE, 1},
-	[pch_et20t_uart2] = {PCH_UART_2LINE, 2},
-	[pch_et20t_uart3] = {PCH_UART_2LINE, 3},
-	[pch_ml7213_uart0] = {PCH_UART_8LINE, 0},
-	[pch_ml7213_uart1] = {PCH_UART_2LINE, 1},
-	[pch_ml7213_uart2] = {PCH_UART_2LINE, 2},
-	[pch_ml7223_uart0] = {PCH_UART_8LINE, 0},
-	[pch_ml7223_uart1] = {PCH_UART_2LINE, 1},
-	[pch_ml7831_uart0] = {PCH_UART_8LINE, 0},
-	[pch_ml7831_uart1] = {PCH_UART_2LINE, 1},
+	[pch_et20t_uart0] = {PORT_PCH_8LINE, 0},
+	[pch_et20t_uart1] = {PORT_PCH_2LINE, 1},
+	[pch_et20t_uart2] = {PORT_PCH_2LINE, 2},
+	[pch_et20t_uart3] = {PORT_PCH_2LINE, 3},
+	[pch_ml7213_uart0] = {PORT_PCH_8LINE, 0},
+	[pch_ml7213_uart1] = {PORT_PCH_2LINE, 1},
+	[pch_ml7213_uart2] = {PORT_PCH_2LINE, 2},
+	[pch_ml7223_uart0] = {PORT_PCH_8LINE, 0},
+	[pch_ml7223_uart1] = {PORT_PCH_2LINE, 1},
+	[pch_ml7831_uart0] = {PORT_PCH_8LINE, 0},
+	[pch_ml7831_uart1] = {PORT_PCH_2LINE, 1},
 };
 
 #ifdef CONFIG_SERIAL_PCH_UART_CONSOLE
@@ -376,7 +371,7 @@ static const struct file_operations port_regs_ops = {
 };
 #endif	/* CONFIG_DEBUG_FS */
 
-static struct dmi_system_id pch_uart_dmi_table[] = {
+static const struct dmi_system_id pch_uart_dmi_table[] = {
 	{
 		.ident = "CM-iTC",
 		{
@@ -1777,10 +1772,10 @@ static struct eg20t_port *pch_uart_init_port(struct pci_dev *pdev,
 		goto init_port_free_txbuf;
 
 	switch (port_type) {
-	case PORT_UNKNOWN:
+	case PORT_PCH_8LINE:
 		fifosize = 256; /* EG20T/ML7213: UART0 */
 		break;
-	case PORT_8250:
+	case PORT_PCH_2LINE:
 		fifosize = 64; /* EG20T:UART1~3  ML7213: UART1~2*/
 		break;
 	default:
@@ -1804,7 +1799,7 @@ static struct eg20t_port *pch_uart_init_port(struct pci_dev *pdev,
 
 	priv->fifo_size = fifosize;
 	priv->uartclk = pch_uart_get_uartclk();
-	priv->port_type = PORT_MAX_8250 + port_type + 1;
+	priv->port_type = port_type;
 	priv->port.dev = &pdev->dev;
 	priv->port.iobase = iobase;
 	priv->port.membase = NULL;
@@ -1862,8 +1857,7 @@ static void pch_uart_exit_port(struct eg20t_port *priv)
 {
 
 #ifdef CONFIG_DEBUG_FS
-	if (priv->debugfs)
-		debugfs_remove(priv->debugfs);
+	debugfs_remove(priv->debugfs);
 #endif
 	uart_remove_one_port(&pch_uart_driver, &priv->port);
 	free_page((unsigned long)priv->rxbuf.buf);
diff --git a/drivers/tty/serial/pmac_zilog.c b/drivers/tty/serial/pmac_zilog.c
index 0da52947e59e..6ccdd018fb45 100644
--- a/drivers/tty/serial/pmac_zilog.c
+++ b/drivers/tty/serial/pmac_zilog.c
@@ -1671,8 +1671,8 @@ static int __init pmz_probe(void)
 		if (!node_a && !node_b) {
 			of_node_put(node_a);
 			of_node_put(node_b);
-			printk(KERN_ERR "pmac_zilog: missing node %c for escc %s\n",
-				(!node_a) ? 'a' : 'b', node_p->full_name);
+			printk(KERN_ERR "pmac_zilog: missing node %c for escc %pOF\n",
+				(!node_a) ? 'a' : 'b', node_p);
 			continue;
 		}
 
diff --git a/drivers/tty/serial/serial-tegra.c b/drivers/tty/serial/serial-tegra.c
index d92a150c8733..cf9b736f26f8 100644
--- a/drivers/tty/serial/serial-tegra.c
+++ b/drivers/tty/serial/serial-tegra.c
@@ -1310,7 +1310,7 @@ static int tegra_uart_probe(struct platform_device *pdev)
 		return PTR_ERR(tup->uart_clk);
 	}
 
-	tup->rst = devm_reset_control_get(&pdev->dev, "serial");
+	tup->rst = devm_reset_control_get_exclusive(&pdev->dev, "serial");
 	if (IS_ERR(tup->rst)) {
 		dev_err(&pdev->dev, "Couldn't get the reset\n");
 		return PTR_ERR(tup->rst);
diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c
index f534a40aebde..3a14cccbd7ff 100644
--- a/drivers/tty/serial/serial_core.c
+++ b/drivers/tty/serial/serial_core.c
@@ -36,7 +36,7 @@
 #include <linux/delay.h>
 #include <linux/mutex.h>
 
-#include <asm/irq.h>
+#include <linux/irq.h>
 #include <linux/uaccess.h>
 
 /*
@@ -165,6 +165,27 @@ uart_update_mctrl(struct uart_port *port, unsigned int set, unsigned int clear)
 #define uart_set_mctrl(port, set)	uart_update_mctrl(port, set, 0)
 #define uart_clear_mctrl(port, clear)	uart_update_mctrl(port, 0, clear)
 
+static void uart_port_dtr_rts(struct uart_port *uport, int raise)
+{
+	int rs485_on = uport->rs485_config &&
+		(uport->rs485.flags & SER_RS485_ENABLED);
+	int RTS_after_send = !!(uport->rs485.flags & SER_RS485_RTS_AFTER_SEND);
+
+	if (raise) {
+		if (rs485_on && !RTS_after_send) {
+			uart_set_mctrl(uport, TIOCM_DTR);
+			uart_clear_mctrl(uport, TIOCM_RTS);
+		} else {
+			uart_set_mctrl(uport, TIOCM_DTR | TIOCM_RTS);
+		}
+	} else {
+		unsigned int clear = TIOCM_DTR;
+
+		clear |= (!rs485_on || !RTS_after_send) ? TIOCM_RTS : 0;
+		uart_clear_mctrl(uport, clear);
+	}
+}
+
 /*
  * Startup the port.  This will be called once per open.  All calls
  * will be serialised by the per-port mutex.
@@ -214,7 +235,7 @@ static int uart_port_startup(struct tty_struct *tty, struct uart_state *state,
 		 * port is open and ready to respond.
 		 */
 		if (init_hw && C_BAUD(tty))
-			uart_set_mctrl(uport, TIOCM_RTS | TIOCM_DTR);
+			uart_port_dtr_rts(uport, 1);
 	}
 
 	/*
@@ -272,7 +293,7 @@ static void uart_shutdown(struct tty_struct *tty, struct uart_state *state)
 			uport->cons->cflag = tty->termios.c_cflag;
 
 		if (!tty || C_HUPCL(tty))
-			uart_clear_mctrl(uport, TIOCM_DTR | TIOCM_RTS);
+			uart_port_dtr_rts(uport, 0);
 
 		uart_port_shutdown(port);
 	}
@@ -744,7 +765,7 @@ static int uart_get_info(struct tty_port *port, struct serial_struct *retinfo)
 	if (HIGH_BITS_OFFSET)
 		retinfo->port_high = (long) uport->iobase >> HIGH_BITS_OFFSET;
 	retinfo->irq		    = uport->irq;
-	retinfo->flags	    = uport->flags;
+	retinfo->flags	    = (__force int)uport->flags;
 	retinfo->xmit_fifo_size  = uport->fifosize;
 	retinfo->baud_base	    = uport->uartclk / 16;
 	retinfo->close_delay	    = jiffies_to_msecs(port->close_delay) / 10;
@@ -818,7 +839,7 @@ static int uart_set_info(struct tty_struct *tty, struct tty_port *port,
 		    new_info->type != uport->type);
 
 	old_flags = uport->flags;
-	new_flags = new_info->flags;
+	new_flags = (__force upf_t)new_info->flags;
 	old_custom_divisor = uport->custom_divisor;
 
 	if (!capable(CAP_SYS_ADMIN)) {
@@ -1658,7 +1679,7 @@ static int uart_carrier_raised(struct tty_port *port)
 	return 0;
 }
 
-static void uart_dtr_rts(struct tty_port *port, int onoff)
+static void uart_dtr_rts(struct tty_port *port, int raise)
 {
 	struct uart_state *state = container_of(port, struct uart_state, port);
 	struct uart_port *uport;
@@ -1666,12 +1687,7 @@ static void uart_dtr_rts(struct tty_port *port, int onoff)
 	uport = uart_port_ref(state);
 	if (!uport)
 		return;
-
-	if (onoff)
-		uart_set_mctrl(uport, TIOCM_DTR | TIOCM_RTS);
-	else
-		uart_clear_mctrl(uport, TIOCM_DTR | TIOCM_RTS);
-
+	uart_port_dtr_rts(uport, raise);
 	uart_port_deref(uport);
 }
 
@@ -2083,8 +2099,7 @@ int uart_suspend_port(struct uart_driver *drv, struct uart_port *uport)
 
 	tty_dev = device_find_child(uport->dev, &match, serial_match_port);
 	if (tty_dev && device_may_wakeup(tty_dev)) {
-		if (!enable_irq_wake(uport->irq))
-			uport->irq_wake = 1;
+		enable_irq_wake(uport->irq);
 		put_device(tty_dev);
 		mutex_unlock(&port->mutex);
 		return 0;
@@ -2147,10 +2162,8 @@ int uart_resume_port(struct uart_driver *drv, struct uart_port *uport)
 
 	tty_dev = device_find_child(uport->dev, &match, serial_match_port);
 	if (!uport->suspended && device_may_wakeup(tty_dev)) {
-		if (uport->irq_wake) {
+		if (irqd_is_wakeup_set(irq_get_irq_data((uport->irq))))
 			disable_irq_wake(uport->irq);
-			uport->irq_wake = 0;
-		}
 		put_device(tty_dev);
 		mutex_unlock(&port->mutex);
 		return 0;
diff --git a/drivers/tty/serial/sh-sci.c b/drivers/tty/serial/sh-sci.c
index e08b16b070c0..784dd42002ea 100644
--- a/drivers/tty/serial/sh-sci.c
+++ b/drivers/tty/serial/sh-sci.c
@@ -3073,8 +3073,7 @@ static struct plat_sci_port *sci_parse_dt(struct platform_device *pdev,
 	p->type = SCI_OF_TYPE(match->data);
 	p->regtype = SCI_OF_REGTYPE(match->data);
 
-	if (of_find_property(np, "uart-has-rtscts", NULL))
-		sp->has_rtscts = true;
+	sp->has_rtscts = of_property_read_bool(np, "uart-has-rtscts");
 
 	return p;
 }
diff --git a/drivers/tty/serial/sprd_serial.c b/drivers/tty/serial/sprd_serial.c
index 90996ad97b37..e902494ebbd5 100644
--- a/drivers/tty/serial/sprd_serial.c
+++ b/drivers/tty/serial/sprd_serial.c
@@ -63,6 +63,7 @@
 
 /* interrupt clear register */
 #define SPRD_ICLR		0x0014
+#define SPRD_ICLR_TIMEOUT	BIT(13)
 
 /* line control register */
 #define SPRD_LCR		0x0018
@@ -298,7 +299,8 @@ static irqreturn_t sprd_handle_irq(int irq, void *dev_id)
 		return IRQ_NONE;
 	}
 
-	serial_out(port, SPRD_ICLR, ~0);
+	if (ims & SPRD_IMSR_TIMEOUT)
+		serial_out(port, SPRD_ICLR, SPRD_ICLR_TIMEOUT);
 
 	if (ims & (SPRD_IMSR_RX_FIFO_FULL |
 		SPRD_IMSR_BREAK_DETECT | SPRD_IMSR_TIMEOUT))
@@ -729,8 +731,8 @@ static int sprd_probe(struct platform_device *pdev)
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
-		dev_err(&pdev->dev, "not provide irq resource\n");
-		return -ENODEV;
+		dev_err(&pdev->dev, "not provide irq resource: %d\n", irq);
+		return irq;
 	}
 	up->irq = irq;
 
diff --git a/drivers/tty/serial/st-asc.c b/drivers/tty/serial/st-asc.c
index 6b0ca65027d0..b313a792b149 100644
--- a/drivers/tty/serial/st-asc.c
+++ b/drivers/tty/serial/st-asc.c
@@ -310,7 +310,7 @@ static void asc_receive_chars(struct uart_port *port)
 	if (mode == ASC_CTL_MODE_8BIT || mode == ASC_CTL_MODE_8BIT_PAR)
 		ignore_pe = true;
 
-	if (port->irq_wake)
+	if (irqd_is_wakeup_set(irq_get_irq_data(port->irq)))
 		pm_wakeup_event(tport->tty->dev, 0);
 
 	while ((status = asc_in(port, ASC_STA)) & ASC_STA_RBF) {
diff --git a/drivers/tty/serial/stm32-usart.c b/drivers/tty/serial/stm32-usart.c
index 033856287ca2..03a583264d9e 100644
--- a/drivers/tty/serial/stm32-usart.c
+++ b/drivers/tty/serial/stm32-usart.c
@@ -1,5 +1,6 @@
 /*
  * Copyright (C) Maxime Coquelin 2015
+ * Copyright (C) STMicroelectronics SA 2017
  * Authors:  Maxime Coquelin <mcoquelin.stm32@gmail.com>
  *	     Gerald Baeza <gerald.baeza@st.com>
  * License terms:  GNU General Public License (GPL), version 2
@@ -25,6 +26,7 @@
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
+#include <linux/pm_wakeirq.h>
 #include <linux/serial_core.h>
 #include <linux/serial.h>
 #include <linux/spinlock.h>
@@ -110,14 +112,13 @@ static void stm32_receive_chars(struct uart_port *port, bool threaded)
 	unsigned long c;
 	u32 sr;
 	char flag;
-	static int last_res = RX_BUF_L;
 
-	if (port->irq_wake)
+	if (irqd_is_wakeup_set(irq_get_irq_data(port->irq)))
 		pm_wakeup_event(tport->tty->dev, 0);
 
-	while (stm32_pending_rx(port, &sr, &last_res, threaded)) {
+	while (stm32_pending_rx(port, &sr, &stm32_port->last_res, threaded)) {
 		sr |= USART_SR_DUMMY_RX;
-		c = stm32_get_char(port, &sr, &last_res);
+		c = stm32_get_char(port, &sr, &stm32_port->last_res);
 		flag = TTY_NORMAL;
 		port->icount.rx++;
 
@@ -202,7 +203,7 @@ static void stm32_transmit_chars_pio(struct uart_port *port)
 	ret = readl_relaxed_poll_timeout_atomic(port->membase + ofs->isr,
 						isr,
 						(isr & USART_SR_TXE),
-						10, 100);
+						10, 100000);
 
 	if (ret)
 		dev_err(port->dev, "tx empty not set\n");
@@ -326,6 +327,10 @@ static irqreturn_t stm32_interrupt(int irq, void *ptr)
 
 	sr = readl_relaxed(port->membase + ofs->isr);
 
+	if ((sr & USART_SR_WUF) && (ofs->icr != UNDEF_REG))
+		writel_relaxed(USART_ICR_WUCF,
+			       port->membase + ofs->icr);
+
 	if ((sr & USART_SR_RXNE) && !(stm32_port->rx_ch))
 		stm32_receive_chars(port, false);
 
@@ -442,6 +447,7 @@ static int stm32_startup(struct uart_port *port)
 {
 	struct stm32_port *stm32_port = to_stm32_port(port);
 	struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
+	struct stm32_usart_config *cfg = &stm32_port->info->cfg;
 	const char *name = to_platform_device(port->dev)->name;
 	u32 val;
 	int ret;
@@ -452,7 +458,18 @@ static int stm32_startup(struct uart_port *port)
 	if (ret)
 		return ret;
 
+	if (cfg->has_wakeup && stm32_port->wakeirq >= 0) {
+		ret = dev_pm_set_dedicated_wake_irq(port->dev,
+						    stm32_port->wakeirq);
+		if (ret) {
+			free_irq(port->irq, port);
+			return ret;
+		}
+	}
+
 	val = USART_CR1_RXNEIE | USART_CR1_TE | USART_CR1_RE;
+	if (stm32_port->fifoen)
+		val |= USART_CR1_FIFOEN;
 	stm32_set_bits(port, ofs->cr1, val);
 
 	return 0;
@@ -467,8 +484,11 @@ static void stm32_shutdown(struct uart_port *port)
 
 	val = USART_CR1_TXEIE | USART_CR1_RXNEIE | USART_CR1_TE | USART_CR1_RE;
 	val |= BIT(cfg->uart_enable_bit);
+	if (stm32_port->fifoen)
+		val |= USART_CR1_FIFOEN;
 	stm32_clr_bits(port, ofs->cr1, val);
 
+	dev_pm_clear_wake_irq(port->dev);
 	free_irq(port->irq, port);
 }
 
@@ -496,6 +516,8 @@ static void stm32_set_termios(struct uart_port *port, struct ktermios *termios,
 
 	cr1 = USART_CR1_TE | USART_CR1_RE | USART_CR1_RXNEIE;
 	cr1 |= BIT(cfg->uart_enable_bit);
+	if (stm32_port->fifoen)
+		cr1 |= USART_CR1_FIFOEN;
 	cr2 = 0;
 	cr3 = 0;
 
@@ -518,7 +540,7 @@ static void stm32_set_termios(struct uart_port *port, struct ktermios *termios,
 	port->status &= ~(UPSTAT_AUTOCTS | UPSTAT_AUTORTS);
 	if (cflag & CRTSCTS) {
 		port->status |= UPSTAT_AUTOCTS | UPSTAT_AUTORTS;
-		cr3 |= USART_CR3_CTSE;
+		cr3 |= USART_CR3_CTSE | USART_CR3_RTSE;
 	}
 
 	usartdiv = DIV_ROUND_CLOSEST(port->uartclk, baud);
@@ -659,6 +681,8 @@ static int stm32_init_port(struct stm32_port *stm32port,
 	port->ops	= &stm32_uart_ops;
 	port->dev	= &pdev->dev;
 	port->irq	= platform_get_irq(pdev, 0);
+	stm32port->wakeirq = platform_get_irq(pdev, 1);
+	stm32port->fifoen = stm32port->info->cfg.has_fifo;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	port->membase = devm_ioremap_resource(&pdev->dev, res);
@@ -678,8 +702,10 @@ static int stm32_init_port(struct stm32_port *stm32port,
 		return ret;
 
 	stm32port->port.uartclk = clk_get_rate(stm32port->clk);
-	if (!stm32port->port.uartclk)
+	if (!stm32port->port.uartclk) {
+		clk_disable_unprepare(stm32port->clk);
 		ret = -EINVAL;
+	}
 
 	return ret;
 }
@@ -693,8 +719,10 @@ static struct stm32_port *stm32_of_get_stm32_port(struct platform_device *pdev)
 		return NULL;
 
 	id = of_alias_get_id(np, "serial");
-	if (id < 0)
-		id = 0;
+	if (id < 0) {
+		dev_err(&pdev->dev, "failed to get alias id, errno %d\n", id);
+		return NULL;
+	}
 
 	if (WARN_ON(id >= STM32_MAX_PORTS))
 		return NULL;
@@ -702,6 +730,7 @@ static struct stm32_port *stm32_of_get_stm32_port(struct platform_device *pdev)
 	stm32_ports[id].hw_flow_control = of_property_read_bool(np,
 							"st,hw-flow-ctrl");
 	stm32_ports[id].port.line = id;
+	stm32_ports[id].last_res = RX_BUF_L;
 	return &stm32_ports[id];
 }
 
@@ -711,6 +740,8 @@ static const struct of_device_id stm32_match[] = {
 	{ .compatible = "st,stm32-uart", .data = &stm32f4_info},
 	{ .compatible = "st,stm32f7-usart", .data = &stm32f7_info},
 	{ .compatible = "st,stm32f7-uart", .data = &stm32f7_info},
+	{ .compatible = "st,stm32h7-usart", .data = &stm32h7_info},
+	{ .compatible = "st,stm32h7-uart", .data = &stm32h7_info},
 	{},
 };
 
@@ -860,9 +891,15 @@ static int stm32_serial_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
+	if (stm32port->info->cfg.has_wakeup && stm32port->wakeirq >= 0) {
+		ret = device_init_wakeup(&pdev->dev, true);
+		if (ret)
+			goto err_uninit;
+	}
+
 	ret = uart_add_one_port(&stm32_usart_driver, &stm32port->port);
 	if (ret)
-		return ret;
+		goto err_nowup;
 
 	ret = stm32_of_dma_rx_probe(stm32port, pdev);
 	if (ret)
@@ -875,6 +912,15 @@ static int stm32_serial_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, &stm32port->port);
 
 	return 0;
+
+err_nowup:
+	if (stm32port->info->cfg.has_wakeup && stm32port->wakeirq >= 0)
+		device_init_wakeup(&pdev->dev, false);
+
+err_uninit:
+	clk_disable_unprepare(stm32port->clk);
+
+	return ret;
 }
 
 static int stm32_serial_remove(struct platform_device *pdev)
@@ -882,6 +928,7 @@ static int stm32_serial_remove(struct platform_device *pdev)
 	struct uart_port *port = platform_get_drvdata(pdev);
 	struct stm32_port *stm32_port = to_stm32_port(port);
 	struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
+	struct stm32_usart_config *cfg = &stm32_port->info->cfg;
 
 	stm32_clr_bits(port, ofs->cr3, USART_CR3_DMAR);
 
@@ -903,6 +950,9 @@ static int stm32_serial_remove(struct platform_device *pdev)
 				  TX_BUF_L, stm32_port->tx_buf,
 				  stm32_port->tx_dma_buf);
 
+	if (cfg->has_wakeup && stm32_port->wakeirq >= 0)
+		device_init_wakeup(&pdev->dev, false);
+
 	clk_disable_unprepare(stm32_port->clk);
 
 	return uart_remove_one_port(&stm32_usart_driver, port);
@@ -1008,11 +1058,66 @@ static struct uart_driver stm32_usart_driver = {
 	.cons		= STM32_SERIAL_CONSOLE,
 };
 
+#ifdef CONFIG_PM_SLEEP
+static void stm32_serial_enable_wakeup(struct uart_port *port, bool enable)
+{
+	struct stm32_port *stm32_port = to_stm32_port(port);
+	struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
+	struct stm32_usart_config *cfg = &stm32_port->info->cfg;
+	u32 val;
+
+	if (!cfg->has_wakeup || stm32_port->wakeirq < 0)
+		return;
+
+	if (enable) {
+		stm32_clr_bits(port, ofs->cr1, BIT(cfg->uart_enable_bit));
+		stm32_set_bits(port, ofs->cr1, USART_CR1_UESM);
+		val = readl_relaxed(port->membase + ofs->cr3);
+		val &= ~USART_CR3_WUS_MASK;
+		/* Enable Wake up interrupt from low power on start bit */
+		val |= USART_CR3_WUS_START_BIT | USART_CR3_WUFIE;
+		writel_relaxed(val, port->membase + ofs->cr3);
+		stm32_set_bits(port, ofs->cr1, BIT(cfg->uart_enable_bit));
+	} else {
+		stm32_clr_bits(port, ofs->cr1, USART_CR1_UESM);
+	}
+}
+
+static int stm32_serial_suspend(struct device *dev)
+{
+	struct uart_port *port = dev_get_drvdata(dev);
+
+	uart_suspend_port(&stm32_usart_driver, port);
+
+	if (device_may_wakeup(dev))
+		stm32_serial_enable_wakeup(port, true);
+	else
+		stm32_serial_enable_wakeup(port, false);
+
+	return 0;
+}
+
+static int stm32_serial_resume(struct device *dev)
+{
+	struct uart_port *port = dev_get_drvdata(dev);
+
+	if (device_may_wakeup(dev))
+		stm32_serial_enable_wakeup(port, false);
+
+	return uart_resume_port(&stm32_usart_driver, port);
+}
+#endif /* CONFIG_PM_SLEEP */
+
+static const struct dev_pm_ops stm32_serial_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(stm32_serial_suspend, stm32_serial_resume)
+};
+
 static struct platform_driver stm32_serial_driver = {
 	.probe		= stm32_serial_probe,
 	.remove		= stm32_serial_remove,
 	.driver	= {
 		.name	= DRIVER_NAME,
+		.pm	= &stm32_serial_pm_ops,
 		.of_match_table = of_match_ptr(stm32_match),
 	},
 };
diff --git a/drivers/tty/serial/stm32-usart.h b/drivers/tty/serial/stm32-usart.h
index cd97ceb76e4f..ffc0c5285e51 100644
--- a/drivers/tty/serial/stm32-usart.h
+++ b/drivers/tty/serial/stm32-usart.h
@@ -1,5 +1,6 @@
 /*
  * Copyright (C) Maxime Coquelin 2015
+ * Copyright (C) STMicroelectronics SA 2017
  * Authors:  Maxime Coquelin <mcoquelin.stm32@gmail.com>
  *	     Gerald Baeza <gerald_baeza@yahoo.fr>
  * License terms:  GNU General Public License (GPL), version 2
@@ -24,6 +25,8 @@ struct stm32_usart_offsets {
 struct stm32_usart_config {
 	u8 uart_enable_bit; /* USART_CR1_UE */
 	bool has_7bits_data;
+	bool has_wakeup;
+	bool has_fifo;
 };
 
 struct stm32_usart_info {
@@ -74,6 +77,28 @@ struct stm32_usart_info stm32f7_info = {
 	}
 };
 
+struct stm32_usart_info stm32h7_info = {
+	.ofs = {
+		.cr1	= 0x00,
+		.cr2	= 0x04,
+		.cr3	= 0x08,
+		.brr	= 0x0c,
+		.gtpr	= 0x10,
+		.rtor	= 0x14,
+		.rqr	= 0x18,
+		.isr	= 0x1c,
+		.icr	= 0x20,
+		.rdr	= 0x24,
+		.tdr	= 0x28,
+	},
+	.cfg = {
+		.uart_enable_bit = 0,
+		.has_7bits_data = true,
+		.has_wakeup = true,
+		.has_fifo = true,
+	}
+};
+
 /* USART_SR (F4) / USART_ISR (F7) */
 #define USART_SR_PE		BIT(0)
 #define USART_SR_FE		BIT(1)
@@ -93,6 +118,7 @@ struct stm32_usart_info stm32f7_info = {
 #define USART_SR_BUSY		BIT(16)		/* F7 */
 #define USART_SR_CMF		BIT(17)		/* F7 */
 #define USART_SR_SBKF		BIT(18)		/* F7 */
+#define USART_SR_WUF		BIT(20)		/* H7 */
 #define USART_SR_TEACK		BIT(21)		/* F7 */
 #define USART_SR_ERR_MASK	(USART_SR_LBD | USART_SR_ORE | \
 				 USART_SR_FE | USART_SR_PE)
@@ -113,6 +139,7 @@ struct stm32_usart_info stm32f7_info = {
 /* USART_CR1 */
 #define USART_CR1_SBK		BIT(0)
 #define USART_CR1_RWU		BIT(1)		/* F4 */
+#define USART_CR1_UESM		BIT(1)		/* H7 */
 #define USART_CR1_RE		BIT(2)
 #define USART_CR1_TE		BIT(3)
 #define USART_CR1_IDLEIE	BIT(4)
@@ -134,6 +161,7 @@ struct stm32_usart_info stm32f7_info = {
 #define USART_CR1_EOBIE		BIT(27)		/* F7 */
 #define USART_CR1_M1		BIT(28)		/* F7 */
 #define USART_CR1_IE_MASK	(GENMASK(8, 4) | BIT(14) | BIT(26) | BIT(27))
+#define USART_CR1_FIFOEN	BIT(29)		/* H7 */
 
 /* USART_CR2 */
 #define USART_CR2_ADD_MASK	GENMASK(3, 0)	/* F4 */
@@ -175,6 +203,9 @@ struct stm32_usart_info stm32f7_info = {
 #define USART_CR3_DEM		BIT(14)		/* F7 */
 #define USART_CR3_DEP		BIT(15)		/* F7 */
 #define USART_CR3_SCARCNT_MASK	GENMASK(19, 17)	/* F7 */
+#define USART_CR3_WUS_MASK	GENMASK(21, 20)	/* H7 */
+#define USART_CR3_WUS_START_BIT	BIT(21)		/* H7 */
+#define USART_CR3_WUFIE		BIT(22)		/* H7 */
 
 /* USART_GTPR */
 #define USART_GTPR_PSC_MASK	GENMASK(7, 0)
@@ -203,9 +234,10 @@ struct stm32_usart_info stm32f7_info = {
 #define USART_ICR_RTOCF		BIT(11)		/* F7 */
 #define USART_ICR_EOBCF		BIT(12)		/* F7 */
 #define USART_ICR_CMCF		BIT(17)		/* F7 */
+#define USART_ICR_WUCF		BIT(20)		/* H7 */
 
 #define STM32_SERIAL_NAME "ttyS"
-#define STM32_MAX_PORTS 6
+#define STM32_MAX_PORTS 8
 
 #define RX_BUF_L 200		 /* dma rx buffer length     */
 #define RX_BUF_P RX_BUF_L	 /* dma rx buffer period     */
@@ -221,8 +253,11 @@ struct stm32_port {
 	struct dma_chan *tx_ch;  /* dma tx channel            */
 	dma_addr_t tx_dma_buf;   /* dma tx buffer bus address */
 	unsigned char *tx_buf;   /* dma tx buffer cpu address */
+	int last_res;
 	bool tx_dma_busy;	 /* dma tx busy               */
 	bool hw_flow_control;
+	bool fifoen;
+	int wakeirq;
 };
 
 static struct stm32_port stm32_ports[STM32_MAX_PORTS];
diff --git a/drivers/tty/serial/sunsab.c b/drivers/tty/serial/sunsab.c
index b5e3195b3697..653a076d89d3 100644
--- a/drivers/tty/serial/sunsab.c
+++ b/drivers/tty/serial/sunsab.c
@@ -819,7 +819,7 @@ static int sunsab_verify_port(struct uart_port *port, struct serial_struct *ser)
 	return -EINVAL;
 }
 
-static struct uart_ops sunsab_pops = {
+static const struct uart_ops sunsab_pops = {
 	.tx_empty	= sunsab_tx_empty,
 	.set_mctrl	= sunsab_set_mctrl,
 	.get_mctrl	= sunsab_get_mctrl,
diff --git a/drivers/tty/serial/sunsu.c b/drivers/tty/serial/sunsu.c
index 72df2e1b88af..95d34d7565c9 100644
--- a/drivers/tty/serial/sunsu.c
+++ b/drivers/tty/serial/sunsu.c
@@ -958,7 +958,7 @@ sunsu_type(struct uart_port *port)
 	return uart_config[type].name;
 }
 
-static struct uart_ops sunsu_pops = {
+static const struct uart_ops sunsu_pops = {
 	.tx_empty	= sunsu_tx_empty,
 	.set_mctrl	= sunsu_set_mctrl,
 	.get_mctrl	= sunsu_get_mctrl,
@@ -1212,8 +1212,8 @@ static int sunsu_kbd_ms_init(struct uart_sunsu_port *up)
 	if (up->port.type == PORT_UNKNOWN)
 		return -ENODEV;
 
-	printk("%s: %s port at %llx, irq %u\n",
-	       up->port.dev->of_node->full_name,
+	printk("%pOF: %s port at %llx, irq %u\n",
+	       up->port.dev->of_node,
 	       (up->su_type == SU_PORT_KBD) ? "Keyboard" : "Mouse",
 	       (unsigned long long) up->port.mapbase,
 	       up->port.irq);
diff --git a/drivers/tty/serial/ucc_uart.c b/drivers/tty/serial/ucc_uart.c
index 481eb2989a1e..55b702775786 100644
--- a/drivers/tty/serial/ucc_uart.c
+++ b/drivers/tty/serial/ucc_uart.c
@@ -1085,7 +1085,7 @@ static int qe_uart_verify_port(struct uart_port *port,
  *
  * Details on these functions can be found in Documentation/serial/driver
  */
-static struct uart_ops qe_uart_pops = {
+static const struct uart_ops qe_uart_pops = {
 	.tx_empty       = qe_uart_tx_empty,
 	.set_mctrl      = qe_uart_set_mctrl,
 	.get_mctrl      = qe_uart_get_mctrl,
diff --git a/drivers/tty/serial/xilinx_uartps.c b/drivers/tty/serial/xilinx_uartps.c
index fde55dcdea5a..31a630ae0870 100644
--- a/drivers/tty/serial/xilinx_uartps.c
+++ b/drivers/tty/serial/xilinx_uartps.c
@@ -1163,7 +1163,7 @@ static void cdns_uart_console_putchar(struct uart_port *port, int ch)
 	writel(ch, port->membase + CDNS_UART_FIFO);
 }
 
-static void __init cdns_early_write(struct console *con, const char *s,
+static void cdns_early_write(struct console *con, const char *s,
 				    unsigned n)
 {
 	struct earlycon_device *dev = con->data;
diff --git a/drivers/tty/synclink.c b/drivers/tty/synclink.c
index 3fafc5a1b2e0..3be981101297 100644
--- a/drivers/tty/synclink.c
+++ b/drivers/tty/synclink.c
@@ -884,7 +884,7 @@ static int synclink_init_one (struct pci_dev *dev,
 				     const struct pci_device_id *ent);
 static void synclink_remove_one (struct pci_dev *dev);
 
-static struct pci_device_id synclink_pci_tbl[] = {
+static const struct pci_device_id synclink_pci_tbl[] = {
 	{ PCI_VENDOR_ID_MICROGATE, PCI_DEVICE_ID_MICROGATE_USC, PCI_ANY_ID, PCI_ANY_ID, },
 	{ PCI_VENDOR_ID_MICROGATE, 0x0210, PCI_ANY_ID, PCI_ANY_ID, },
 	{ 0, }, /* terminate list */
diff --git a/drivers/tty/synclink_gt.c b/drivers/tty/synclink_gt.c
index 529c6e3cd537..636b8ae29b46 100644
--- a/drivers/tty/synclink_gt.c
+++ b/drivers/tty/synclink_gt.c
@@ -95,7 +95,7 @@ MODULE_LICENSE("GPL");
 #define MGSL_MAGIC 0x5401
 #define MAX_DEVICES 32
 
-static struct pci_device_id pci_table[] = {
+static const struct pci_device_id pci_table[] = {
 	{PCI_VENDOR_ID_MICROGATE, SYNCLINK_GT_DEVICE_ID, PCI_ANY_ID, PCI_ANY_ID,},
 	{PCI_VENDOR_ID_MICROGATE, SYNCLINK_GT2_DEVICE_ID, PCI_ANY_ID, PCI_ANY_ID,},
 	{PCI_VENDOR_ID_MICROGATE, SYNCLINK_GT4_DEVICE_ID, PCI_ANY_ID, PCI_ANY_ID,},
diff --git a/drivers/tty/synclinkmp.c b/drivers/tty/synclinkmp.c
index 9b4fb0251c1a..4fed9e7b281f 100644
--- a/drivers/tty/synclinkmp.c
+++ b/drivers/tty/synclinkmp.c
@@ -479,7 +479,7 @@ static char *driver_version = "$Revision: 4.38 $";
 static int synclinkmp_init_one(struct pci_dev *dev,const struct pci_device_id *ent);
 static void synclinkmp_remove_one(struct pci_dev *dev);
 
-static struct pci_device_id synclinkmp_pci_tbl[] = {
+static const struct pci_device_id synclinkmp_pci_tbl[] = {
 	{ PCI_VENDOR_ID_MICROGATE, PCI_DEVICE_ID_MICROGATE_SCA, PCI_ANY_ID, PCI_ANY_ID, },
 	{ 0, }, /* terminate list */
 };
diff --git a/drivers/tty/tty_buffer.c b/drivers/tty/tty_buffer.c
index 4e7a4e9dcf4d..f8eba1c5412f 100644
--- a/drivers/tty/tty_buffer.c
+++ b/drivers/tty/tty_buffer.c
@@ -362,6 +362,32 @@ int tty_insert_flip_string_flags(struct tty_port *port,
 EXPORT_SYMBOL(tty_insert_flip_string_flags);
 
 /**
+ *	__tty_insert_flip_char   -	Add one character to the tty buffer
+ *	@port: tty port
+ *	@ch: character
+ *	@flag: flag byte
+ *
+ *	Queue a single byte to the tty buffering, with an optional flag.
+ *	This is the slow path of tty_insert_flip_char.
+ */
+int __tty_insert_flip_char(struct tty_port *port, unsigned char ch, char flag)
+{
+	struct tty_buffer *tb;
+	int flags = (flag == TTY_NORMAL) ? TTYB_NORMAL : 0;
+
+	if (!__tty_buffer_request_room(port, 1, flags))
+		return 0;
+
+	tb = port->buf.tail;
+	if (~tb->flags & TTYB_NORMAL)
+		*flag_buf_ptr(tb, tb->used) = flag;
+	*char_buf_ptr(tb, tb->used++) = ch;
+
+	return 1;
+}
+EXPORT_SYMBOL(__tty_insert_flip_char);
+
+/**
  *	tty_schedule_flip	-	push characters to ldisc
  *	@port: tty port to push from
  *
diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c
index 974b13d24401..94cccb6efa32 100644
--- a/drivers/tty/tty_io.c
+++ b/drivers/tty/tty_io.c
@@ -280,7 +280,7 @@ static int check_tty_count(struct tty_struct *tty, const char *routine)
 {
 #ifdef CHECK_TTY_COUNT
 	struct list_head *p;
-	int count = 0;
+	int count = 0, kopen_count = 0;
 
 	spin_lock(&tty->files_lock);
 	list_for_each(p, &tty->tty_files) {
@@ -291,10 +291,12 @@ static int check_tty_count(struct tty_struct *tty, const char *routine)
 	    tty->driver->subtype == PTY_TYPE_SLAVE &&
 	    tty->link && tty->link->count)
 		count++;
-	if (tty->count != count) {
-		tty_warn(tty, "%s: tty->count(%d) != #fd's(%d)\n",
-			 routine, tty->count, count);
-		return count;
+	if (tty_port_kopened(tty->port))
+		kopen_count++;
+	if (tty->count != (count + kopen_count)) {
+		tty_warn(tty, "%s: tty->count(%d) != (#fd's(%d) + #kopen's(%d))\n",
+			 routine, tty->count, count, kopen_count);
+		return (count + kopen_count);
 	}
 #endif
 	return 0;
@@ -462,6 +464,14 @@ static int hung_up_tty_fasync(int fd, struct file *file, int on)
 	return -ENOTTY;
 }
 
+static void tty_show_fdinfo(struct seq_file *m, struct file *file)
+{
+	struct tty_struct *tty = file_tty(file);
+
+	if (tty && tty->ops && tty->ops->show_fdinfo)
+		tty->ops->show_fdinfo(tty, m);
+}
+
 static const struct file_operations tty_fops = {
 	.llseek		= no_llseek,
 	.read		= tty_read,
@@ -472,6 +482,7 @@ static const struct file_operations tty_fops = {
 	.open		= tty_open,
 	.release	= tty_release,
 	.fasync		= tty_fasync,
+	.show_fdinfo	= tty_show_fdinfo,
 };
 
 static const struct file_operations console_fops = {
@@ -1513,6 +1524,38 @@ static int tty_release_checks(struct tty_struct *tty, int idx)
 }
 
 /**
+ *      tty_kclose      -       closes tty opened by tty_kopen
+ *      @tty: tty device
+ *
+ *      Performs the final steps to release and free a tty device. It is the
+ *      same as tty_release_struct except that it also resets TTY_PORT_KOPENED
+ *      flag on tty->port.
+ */
+void tty_kclose(struct tty_struct *tty)
+{
+	/*
+	 * Ask the line discipline code to release its structures
+	 */
+	tty_ldisc_release(tty);
+
+	/* Wait for pending work before tty destruction commmences */
+	tty_flush_works(tty);
+
+	tty_debug_hangup(tty, "freeing structure\n");
+	/*
+	 * The release_tty function takes care of the details of clearing
+	 * the slots and preserving the termios structure. The tty_unlock_pair
+	 * should be safe as we keep a kref while the tty is locked (so the
+	 * unlock never unlocks a freed tty).
+	 */
+	mutex_lock(&tty_mutex);
+	tty_port_set_kopened(tty->port, 0);
+	release_tty(tty, tty->index);
+	mutex_unlock(&tty_mutex);
+}
+EXPORT_SYMBOL_GPL(tty_kclose);
+
+/**
  *	tty_release_struct	-	release a tty struct
  *	@tty: tty device
  *	@idx: index of the tty
@@ -1786,6 +1829,56 @@ static struct tty_driver *tty_lookup_driver(dev_t device, struct file *filp,
 }
 
 /**
+ *	tty_kopen	-	open a tty device for kernel
+ *	@device: dev_t of device to open
+ *
+ *	Opens tty exclusively for kernel. Performs the driver lookup,
+ *	makes sure it's not already opened and performs the first-time
+ *	tty initialization.
+ *
+ *	Returns the locked initialized &tty_struct
+ *
+ *	Claims the global tty_mutex to serialize:
+ *	  - concurrent first-time tty initialization
+ *	  - concurrent tty driver removal w/ lookup
+ *	  - concurrent tty removal from driver table
+ */
+struct tty_struct *tty_kopen(dev_t device)
+{
+	struct tty_struct *tty;
+	struct tty_driver *driver = NULL;
+	int index = -1;
+
+	mutex_lock(&tty_mutex);
+	driver = tty_lookup_driver(device, NULL, &index);
+	if (IS_ERR(driver)) {
+		mutex_unlock(&tty_mutex);
+		return ERR_CAST(driver);
+	}
+
+	/* check whether we're reopening an existing tty */
+	tty = tty_driver_lookup_tty(driver, NULL, index);
+	if (IS_ERR(tty))
+		goto out;
+
+	if (tty) {
+		/* drop kref from tty_driver_lookup_tty() */
+		tty_kref_put(tty);
+		tty = ERR_PTR(-EBUSY);
+	} else { /* tty_init_dev returns tty with the tty_lock held */
+		tty = tty_init_dev(driver, index);
+		if (IS_ERR(tty))
+			goto out;
+		tty_port_set_kopened(tty->port, 1);
+	}
+out:
+	mutex_unlock(&tty_mutex);
+	tty_driver_kref_put(driver);
+	return tty;
+}
+EXPORT_SYMBOL_GPL(tty_kopen);
+
+/**
  *	tty_open_by_driver	-	open a tty device
  *	@device: dev_t of device to open
  *	@inode: inode of device file
@@ -1801,7 +1894,7 @@ static struct tty_driver *tty_lookup_driver(dev_t device, struct file *filp,
  *	  - concurrent tty driver removal w/ lookup
  *	  - concurrent tty removal from driver table
  */
-struct tty_struct *tty_open_by_driver(dev_t device, struct inode *inode,
+static struct tty_struct *tty_open_by_driver(dev_t device, struct inode *inode,
 					     struct file *filp)
 {
 	struct tty_struct *tty;
@@ -1824,6 +1917,12 @@ struct tty_struct *tty_open_by_driver(dev_t device, struct inode *inode,
 	}
 
 	if (tty) {
+		if (tty_port_kopened(tty->port)) {
+			tty_kref_put(tty);
+			mutex_unlock(&tty_mutex);
+			tty = ERR_PTR(-EBUSY);
+			goto out;
+		}
 		mutex_unlock(&tty_mutex);
 		retval = tty_lock_interruptible(tty);
 		tty_kref_put(tty);  /* drop kref from tty_driver_lookup_tty() */
@@ -1846,7 +1945,6 @@ out:
 	tty_driver_kref_put(driver);
 	return tty;
 }
-EXPORT_SYMBOL_GPL(tty_open_by_driver);
 
 /**
  *	tty_open		-	open a tty device
@@ -2518,6 +2616,9 @@ long tty_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 	case TIOCSSERIAL:
 		tty_warn_deprecated_flags(p);
 		break;
+	case TIOCGPTPEER:
+		/* Special because the struct file is needed */
+		return ptm_open_peer(file, tty, (int)arg);
 	default:
 		retval = tty_jobctrl_ioctl(tty, real_tty, file, cmd, arg);
 		if (retval != -ENOIOCTLCMD)
diff --git a/drivers/tty/vcc.c b/drivers/tty/vcc.c
new file mode 100644
index 000000000000..ef01d24858cd
--- /dev/null
+++ b/drivers/tty/vcc.c
@@ -0,0 +1,1155 @@
+/* vcc.c: sun4v virtual channel concentrator
+ *
+ * Copyright (C) 2017 Oracle. All rights reserved.
+ */
+
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/sysfs.h>
+#include <linux/tty.h>
+#include <linux/tty_flip.h>
+#include <asm/vio.h>
+#include <asm/ldc.h>
+
+#define DRV_MODULE_NAME		"vcc"
+#define DRV_MODULE_VERSION	"1.1"
+#define DRV_MODULE_RELDATE	"July 1, 2017"
+
+static char version[] =
+	DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")";
+
+MODULE_DESCRIPTION("Sun LDOM virtual console concentrator driver");
+MODULE_LICENSE("GPL");
+MODULE_VERSION(DRV_MODULE_VERSION);
+
+struct vcc_port {
+	struct vio_driver_state vio;
+
+	spinlock_t lock;
+	char *domain;
+	struct tty_struct *tty;	/* only populated while dev is open */
+	unsigned long index;	/* index into the vcc_table */
+
+	u64 refcnt;
+	bool excl_locked;
+
+	bool removed;
+
+	/* This buffer is required to support the tty write_room interface
+	 * and guarantee that any characters that the driver accepts will
+	 * be eventually sent, either immediately or later.
+	 */
+	int chars_in_buffer;
+	struct vio_vcc buffer;
+
+	struct timer_list rx_timer;
+	struct timer_list tx_timer;
+};
+
+/* Microseconds that thread will delay waiting for a vcc port ref */
+#define VCC_REF_DELAY		100
+
+#define VCC_MAX_PORTS		1024
+#define VCC_MINOR_START		0	/* must be zero */
+#define VCC_BUFF_LEN		VIO_VCC_MTU_SIZE
+
+#define VCC_CTL_BREAK		-1
+#define VCC_CTL_HUP		-2
+
+static const char vcc_driver_name[] = "vcc";
+static const char vcc_device_node[] = "vcc";
+static struct tty_driver *vcc_tty_driver;
+
+static struct vcc_port *vcc_table[VCC_MAX_PORTS];
+static DEFINE_SPINLOCK(vcc_table_lock);
+
+int vcc_dbg;
+int vcc_dbg_ldc;
+int vcc_dbg_vio;
+
+module_param(vcc_dbg, uint, 0664);
+module_param(vcc_dbg_ldc, uint, 0664);
+module_param(vcc_dbg_vio, uint, 0664);
+
+#define VCC_DBG_DRV	0x1
+#define VCC_DBG_LDC	0x2
+#define VCC_DBG_PKT	0x4
+
+#define vccdbg(f, a...)						\
+	do {							\
+		if (vcc_dbg & VCC_DBG_DRV)			\
+			pr_info(f, ## a);			\
+	} while (0)						\
+
+#define vccdbgl(l)						\
+	do {							\
+		if (vcc_dbg & VCC_DBG_LDC)			\
+			ldc_print(l);				\
+	} while (0)						\
+
+#define vccdbgp(pkt)						\
+	do {							\
+		if (vcc_dbg & VCC_DBG_PKT) {			\
+			int i;					\
+			for (i = 0; i < pkt.tag.stype; i++)	\
+				pr_info("[%c]", pkt.data[i]);	\
+		}						\
+	} while (0)						\
+
+/* Note: Be careful when adding flags to this line discipline.  Don't
+ * add anything that will cause echoing or we'll go into recursive
+ * loop echoing chars back and forth with the console drivers.
+ */
+static const struct ktermios vcc_tty_termios = {
+	.c_iflag = IGNBRK | IGNPAR,
+	.c_oflag = OPOST,
+	.c_cflag = B38400 | CS8 | CREAD | HUPCL,
+	.c_cc = INIT_C_CC,
+	.c_ispeed = 38400,
+	.c_ospeed = 38400
+};
+
+/**
+ * vcc_table_add() - Add VCC port to the VCC table
+ * @port: pointer to the VCC port
+ *
+ * Return: index of the port in the VCC table on success,
+ *	   -1 on failure
+ */
+static int vcc_table_add(struct vcc_port *port)
+{
+	unsigned long flags;
+	int i;
+
+	spin_lock_irqsave(&vcc_table_lock, flags);
+	for (i = VCC_MINOR_START; i < VCC_MAX_PORTS; i++) {
+		if (!vcc_table[i]) {
+			vcc_table[i] = port;
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&vcc_table_lock, flags);
+
+	if (i < VCC_MAX_PORTS)
+		return i;
+	else
+		return -1;
+}
+
+/**
+ * vcc_table_remove() - Removes a VCC port from the VCC table
+ * @index: Index into the VCC table
+ */
+static void vcc_table_remove(unsigned long index)
+{
+	unsigned long flags;
+
+	if (WARN_ON(index >= VCC_MAX_PORTS))
+		return;
+
+	spin_lock_irqsave(&vcc_table_lock, flags);
+	vcc_table[index] = NULL;
+	spin_unlock_irqrestore(&vcc_table_lock, flags);
+}
+
+/**
+ * vcc_get() - Gets a reference to VCC port
+ * @index: Index into the VCC table
+ * @excl: Indicates if an exclusive access is requested
+ *
+ * Return: reference to the VCC port, if found
+ *	   NULL, if port not found
+ */
+static struct vcc_port *vcc_get(unsigned long index, bool excl)
+{
+	struct vcc_port *port;
+	unsigned long flags;
+
+try_again:
+	spin_lock_irqsave(&vcc_table_lock, flags);
+
+	port = vcc_table[index];
+	if (!port) {
+		spin_unlock_irqrestore(&vcc_table_lock, flags);
+		return NULL;
+	}
+
+	if (!excl) {
+		if (port->excl_locked) {
+			spin_unlock_irqrestore(&vcc_table_lock, flags);
+			udelay(VCC_REF_DELAY);
+			goto try_again;
+		}
+		port->refcnt++;
+		spin_unlock_irqrestore(&vcc_table_lock, flags);
+		return port;
+	}
+
+	if (port->refcnt) {
+		spin_unlock_irqrestore(&vcc_table_lock, flags);
+		/* Threads wanting exclusive access will wait half the time,
+		 * probably giving them higher priority in the case of
+		 * multiple waiters.
+		 */
+		udelay(VCC_REF_DELAY/2);
+		goto try_again;
+	}
+
+	port->refcnt++;
+	port->excl_locked = true;
+	spin_unlock_irqrestore(&vcc_table_lock, flags);
+
+	return port;
+}
+
+/**
+ * vcc_put() - Returns a reference to VCC port
+ * @port: pointer to VCC port
+ * @excl: Indicates if the returned reference is an exclusive reference
+ *
+ * Note: It's the caller's responsibility to ensure the correct value
+ *	 for the excl flag
+ */
+static void vcc_put(struct vcc_port *port, bool excl)
+{
+	unsigned long flags;
+
+	if (!port)
+		return;
+
+	spin_lock_irqsave(&vcc_table_lock, flags);
+
+	/* check if caller attempted to put with the wrong flags */
+	if (WARN_ON((excl && !port->excl_locked) ||
+		    (!excl && port->excl_locked)))
+		goto done;
+
+	port->refcnt--;
+
+	if (excl)
+		port->excl_locked = false;
+
+done:
+	spin_unlock_irqrestore(&vcc_table_lock, flags);
+}
+
+/**
+ * vcc_get_ne() - Get a non-exclusive reference to VCC port
+ * @index: Index into the VCC table
+ *
+ * Gets a non-exclusive reference to VCC port, if it's not removed
+ *
+ * Return: pointer to the VCC port, if found
+ *	   NULL, if port not found
+ */
+static struct vcc_port *vcc_get_ne(unsigned long index)
+{
+	struct vcc_port *port;
+
+	port = vcc_get(index, false);
+
+	if (port && port->removed) {
+		vcc_put(port, false);
+		return NULL;
+	}
+
+	return port;
+}
+
+static void vcc_kick_rx(struct vcc_port *port)
+{
+	struct vio_driver_state *vio = &port->vio;
+
+	assert_spin_locked(&port->lock);
+
+	if (!timer_pending(&port->rx_timer) && !port->removed) {
+		disable_irq_nosync(vio->vdev->rx_irq);
+		port->rx_timer.expires = (jiffies + 1);
+		add_timer(&port->rx_timer);
+	}
+}
+
+static void vcc_kick_tx(struct vcc_port *port)
+{
+	assert_spin_locked(&port->lock);
+
+	if (!timer_pending(&port->tx_timer) && !port->removed) {
+		port->tx_timer.expires = (jiffies + 1);
+		add_timer(&port->tx_timer);
+	}
+}
+
+static int vcc_rx_check(struct tty_struct *tty, int size)
+{
+	if (WARN_ON(!tty || !tty->port))
+		return 1;
+
+	/* tty_buffer_request_room won't sleep because it uses
+	 * GFP_ATOMIC flag to allocate buffer
+	 */
+	if (test_bit(TTY_THROTTLED, &tty->flags) ||
+	    (tty_buffer_request_room(tty->port, VCC_BUFF_LEN) < VCC_BUFF_LEN))
+		return 0;
+
+	return 1;
+}
+
+static int vcc_rx(struct tty_struct *tty, char *buf, int size)
+{
+	int len = 0;
+
+	if (WARN_ON(!tty || !tty->port))
+		return len;
+
+	len = tty_insert_flip_string(tty->port, buf, size);
+	if (len)
+		tty_flip_buffer_push(tty->port);
+
+	return len;
+}
+
+static int vcc_ldc_read(struct vcc_port *port)
+{
+	struct vio_driver_state *vio = &port->vio;
+	struct tty_struct *tty;
+	struct vio_vcc pkt;
+	int rv = 0;
+
+	tty = port->tty;
+	if (!tty) {
+		rv = ldc_rx_reset(vio->lp);
+		vccdbg("VCC: reset rx q: rv=%d\n", rv);
+		goto done;
+	}
+
+	/* Read as long as LDC has incoming data. */
+	while (1) {
+		if (!vcc_rx_check(tty, VIO_VCC_MTU_SIZE)) {
+			vcc_kick_rx(port);
+			break;
+		}
+
+		vccdbgl(vio->lp);
+
+		rv = ldc_read(vio->lp, &pkt, sizeof(pkt));
+		if (rv <= 0)
+			break;
+
+		vccdbg("VCC: ldc_read()=%d\n", rv);
+		vccdbg("TAG [%02x:%02x:%04x:%08x]\n",
+		       pkt.tag.type, pkt.tag.stype,
+		       pkt.tag.stype_env, pkt.tag.sid);
+
+		if (pkt.tag.type == VIO_TYPE_DATA) {
+			vccdbgp(pkt);
+			/* vcc_rx_check ensures memory availability */
+			vcc_rx(tty, pkt.data, pkt.tag.stype);
+		} else {
+			pr_err("VCC: unknown msg [%02x:%02x:%04x:%08x]\n",
+			       pkt.tag.type, pkt.tag.stype,
+			       pkt.tag.stype_env, pkt.tag.sid);
+			rv = -ECONNRESET;
+			break;
+		}
+
+		WARN_ON(rv != LDC_PACKET_SIZE);
+	}
+
+done:
+	return rv;
+}
+
+static void vcc_rx_timer(unsigned long index)
+{
+	struct vio_driver_state *vio;
+	struct vcc_port *port;
+	unsigned long flags;
+	int rv;
+
+	port = vcc_get_ne(index);
+	if (!port)
+		return;
+
+	spin_lock_irqsave(&port->lock, flags);
+	port->rx_timer.expires = 0;
+
+	vio = &port->vio;
+
+	enable_irq(vio->vdev->rx_irq);
+
+	if (!port->tty || port->removed)
+		goto done;
+
+	rv = vcc_ldc_read(port);
+	if (rv == -ECONNRESET)
+		vio_conn_reset(vio);
+
+done:
+	spin_unlock_irqrestore(&port->lock, flags);
+	vcc_put(port, false);
+}
+
+static void vcc_tx_timer(unsigned long index)
+{
+	struct vcc_port *port;
+	struct vio_vcc *pkt;
+	unsigned long flags;
+	int tosend = 0;
+	int rv;
+
+	port = vcc_get_ne(index);
+	if (!port)
+		return;
+
+	spin_lock_irqsave(&port->lock, flags);
+	port->tx_timer.expires = 0;
+
+	if (!port->tty || port->removed)
+		goto done;
+
+	tosend = min(VCC_BUFF_LEN, port->chars_in_buffer);
+	if (!tosend)
+		goto done;
+
+	pkt = &port->buffer;
+	pkt->tag.type = VIO_TYPE_DATA;
+	pkt->tag.stype = tosend;
+	vccdbgl(port->vio.lp);
+
+	rv = ldc_write(port->vio.lp, pkt, (VIO_TAG_SIZE + tosend));
+	WARN_ON(!rv);
+
+	if (rv < 0) {
+		vccdbg("VCC: ldc_write()=%d\n", rv);
+		vcc_kick_tx(port);
+	} else {
+		struct tty_struct *tty = port->tty;
+
+		port->chars_in_buffer = 0;
+		if (tty)
+			tty_wakeup(tty);
+	}
+
+done:
+	spin_unlock_irqrestore(&port->lock, flags);
+	vcc_put(port, false);
+}
+
+/**
+ * vcc_event() - LDC event processing engine
+ * @arg: VCC private data
+ * @event: LDC event
+ *
+ * Handles LDC events for VCC
+ */
+static void vcc_event(void *arg, int event)
+{
+	struct vio_driver_state *vio;
+	struct vcc_port *port;
+	unsigned long flags;
+	int rv;
+
+	port = arg;
+	vio = &port->vio;
+
+	spin_lock_irqsave(&port->lock, flags);
+
+	switch (event) {
+	case LDC_EVENT_RESET:
+	case LDC_EVENT_UP:
+		vio_link_state_change(vio, event);
+		break;
+
+	case LDC_EVENT_DATA_READY:
+		rv = vcc_ldc_read(port);
+		if (rv == -ECONNRESET)
+			vio_conn_reset(vio);
+		break;
+
+	default:
+		pr_err("VCC: unexpected LDC event(%d)\n", event);
+	}
+
+	spin_unlock_irqrestore(&port->lock, flags);
+}
+
+static struct ldc_channel_config vcc_ldc_cfg = {
+	.event		= vcc_event,
+	.mtu		= VIO_VCC_MTU_SIZE,
+	.mode		= LDC_MODE_RAW,
+	.debug		= 0,
+};
+
+/* Ordered from largest major to lowest */
+static struct vio_version vcc_versions[] = {
+	{ .major = 1, .minor = 0 },
+};
+
+static struct tty_port_operations vcc_port_ops = { 0 };
+
+static ssize_t vcc_sysfs_domain_show(struct device *dev,
+				     struct device_attribute *attr,
+				     char *buf)
+{
+	struct vcc_port *port;
+	int rv;
+
+	port = dev_get_drvdata(dev);
+	if (!port)
+		return -ENODEV;
+
+	rv = scnprintf(buf, PAGE_SIZE, "%s\n", port->domain);
+
+	return rv;
+}
+
+static int vcc_send_ctl(struct vcc_port *port, int ctl)
+{
+	struct vio_vcc pkt;
+	int rv;
+
+	pkt.tag.type = VIO_TYPE_CTRL;
+	pkt.tag.sid = ctl;
+	pkt.tag.stype = 0;
+
+	rv = ldc_write(port->vio.lp, &pkt, sizeof(pkt.tag));
+	WARN_ON(!rv);
+	vccdbg("VCC: ldc_write(%ld)=%d\n", sizeof(pkt.tag), rv);
+
+	return rv;
+}
+
+static ssize_t vcc_sysfs_break_store(struct device *dev,
+				     struct device_attribute *attr,
+				     const char *buf, size_t count)
+{
+	struct vcc_port *port;
+	unsigned long flags;
+	int rv = count;
+	int brk;
+
+	port = dev_get_drvdata(dev);
+	if (!port)
+		return -ENODEV;
+
+	spin_lock_irqsave(&port->lock, flags);
+
+	if (sscanf(buf, "%ud", &brk) != 1 || brk != 1)
+		rv = -EINVAL;
+	else if (vcc_send_ctl(port, VCC_CTL_BREAK) < 0)
+		vcc_kick_tx(port);
+
+	spin_unlock_irqrestore(&port->lock, flags);
+
+	return rv;
+}
+
+static DEVICE_ATTR(domain, 0400, vcc_sysfs_domain_show, NULL);
+static DEVICE_ATTR(break, 0200, NULL, vcc_sysfs_break_store);
+
+static struct attribute *vcc_sysfs_entries[] = {
+	&dev_attr_domain.attr,
+	&dev_attr_break.attr,
+	NULL
+};
+
+static struct attribute_group vcc_attribute_group = {
+	.name = NULL,
+	.attrs = vcc_sysfs_entries,
+};
+
+/**
+ * vcc_probe() - Initialize VCC port
+ * @vdev: Pointer to VIO device of the new VCC port
+ * @id: VIO device ID
+ *
+ * Initializes a VCC port to receive serial console data from
+ * the guest domain. Sets up a TTY end point on the control
+ * domain. Sets up VIO/LDC link between the guest & control
+ * domain endpoints.
+ *
+ * Return: status of the probe
+ */
+static int vcc_probe(struct vio_dev *vdev, const struct vio_device_id *id)
+{
+	struct mdesc_handle *hp;
+	struct vcc_port *port;
+	struct device *dev;
+	const char *domain;
+	char *name;
+	u64 node;
+	int rv;
+
+	vccdbg("VCC: name=%s\n", dev_name(&vdev->dev));
+
+	if (!vcc_tty_driver) {
+		pr_err("VCC: TTY driver not registered\n");
+		return -ENODEV;
+	}
+
+	port = kzalloc(sizeof(struct vcc_port), GFP_KERNEL);
+	if (!port)
+		return -ENOMEM;
+
+	name = kstrdup(dev_name(&vdev->dev), GFP_KERNEL);
+
+	rv = vio_driver_init(&port->vio, vdev, VDEV_CONSOLE_CON, vcc_versions,
+			     ARRAY_SIZE(vcc_versions), NULL, name);
+	if (rv)
+		goto free_port;
+
+	port->vio.debug = vcc_dbg_vio;
+	vcc_ldc_cfg.debug = vcc_dbg_ldc;
+
+	rv = vio_ldc_alloc(&port->vio, &vcc_ldc_cfg, port);
+	if (rv)
+		goto free_port;
+
+	spin_lock_init(&port->lock);
+
+	port->index = vcc_table_add(port);
+	if (port->index == -1) {
+		pr_err("VCC: no more TTY indices left for allocation\n");
+		goto free_ldc;
+	}
+
+	/* Register the device using VCC table index as TTY index */
+	dev = tty_register_device(vcc_tty_driver, port->index, &vdev->dev);
+	if (IS_ERR(dev)) {
+		rv = PTR_ERR(dev);
+		goto free_table;
+	}
+
+	hp = mdesc_grab();
+
+	node = vio_vdev_node(hp, vdev);
+	if (node == MDESC_NODE_NULL) {
+		rv = -ENXIO;
+		mdesc_release(hp);
+		goto unreg_tty;
+	}
+
+	domain = mdesc_get_property(hp, node, "vcc-domain-name", NULL);
+	if (!domain) {
+		rv = -ENXIO;
+		mdesc_release(hp);
+		goto unreg_tty;
+	}
+	port->domain = kstrdup(domain, GFP_KERNEL);
+
+	mdesc_release(hp);
+
+	rv = sysfs_create_group(&vdev->dev.kobj, &vcc_attribute_group);
+	if (rv)
+		goto free_domain;
+
+	init_timer(&port->rx_timer);
+	port->rx_timer.function = vcc_rx_timer;
+	port->rx_timer.data = port->index;
+
+	init_timer(&port->tx_timer);
+	port->tx_timer.function = vcc_tx_timer;
+	port->tx_timer.data = port->index;
+
+	dev_set_drvdata(&vdev->dev, port);
+
+	/* It's possible to receive IRQs in the middle of vio_port_up. Disable
+	 * IRQs until the port is up.
+	 */
+	disable_irq_nosync(vdev->rx_irq);
+	vio_port_up(&port->vio);
+	enable_irq(vdev->rx_irq);
+
+	return 0;
+
+free_domain:
+	kfree(port->domain);
+unreg_tty:
+	tty_unregister_device(vcc_tty_driver, port->index);
+free_table:
+	vcc_table_remove(port->index);
+free_ldc:
+	vio_ldc_free(&port->vio);
+free_port:
+	kfree(name);
+	kfree(port);
+
+	return rv;
+}
+
+/**
+ * vcc_remove() - Terminate a VCC port
+ * @vdev: Pointer to VIO device of the VCC port
+ *
+ * Terminates a VCC port. Sets up the teardown of TTY and
+ * VIO/LDC link between guest and primary domains.
+ *
+ * Return: status of removal
+ */
+static int vcc_remove(struct vio_dev *vdev)
+{
+	struct vcc_port *port = dev_get_drvdata(&vdev->dev);
+
+	if (!port)
+		return -ENODEV;
+
+	del_timer_sync(&port->rx_timer);
+	del_timer_sync(&port->tx_timer);
+
+	/* If there's a process with the device open, do a synchronous
+	 * hangup of the TTY. This *may* cause the process to call close
+	 * asynchronously, but it's not guaranteed.
+	 */
+	if (port->tty)
+		tty_vhangup(port->tty);
+
+	/* Get exclusive reference to VCC, ensures that there are no other
+	 * clients to this port
+	 */
+	port = vcc_get(port->index, true);
+
+	if (WARN_ON(!port))
+		return -ENODEV;
+
+	tty_unregister_device(vcc_tty_driver, port->index);
+
+	del_timer_sync(&port->vio.timer);
+	vio_ldc_free(&port->vio);
+	sysfs_remove_group(&vdev->dev.kobj, &vcc_attribute_group);
+	dev_set_drvdata(&vdev->dev, NULL);
+	if (port->tty) {
+		port->removed = true;
+		vcc_put(port, true);
+	} else {
+		vcc_table_remove(port->index);
+
+		kfree(port->vio.name);
+		kfree(port->domain);
+		kfree(port);
+	}
+
+	return 0;
+}
+
+static const struct vio_device_id vcc_match[] = {
+	{
+		.type = "vcc-port",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(vio, vcc_match);
+
+static struct vio_driver vcc_driver = {
+	.id_table	= vcc_match,
+	.probe		= vcc_probe,
+	.remove		= vcc_remove,
+	.name		= "vcc",
+};
+
+static int vcc_open(struct tty_struct *tty, struct file *vcc_file)
+{
+	struct vcc_port *port;
+
+	if (unlikely(!tty)) {
+		pr_err("VCC: open: Invalid TTY handle\n");
+		return -ENXIO;
+	}
+
+	if (tty->count > 1)
+		return -EBUSY;
+
+	port = vcc_get_ne(tty->index);
+	if (unlikely(!port)) {
+		pr_err("VCC: open: Failed to find VCC port\n");
+		return -ENODEV;
+	}
+
+	if (unlikely(!port->vio.lp)) {
+		pr_err("VCC: open: LDC channel not configured\n");
+		vcc_put(port, false);
+		return -EPIPE;
+	}
+	vccdbgl(port->vio.lp);
+
+	vcc_put(port, false);
+
+	if (unlikely(!tty->port)) {
+		pr_err("VCC: open: TTY port not found\n");
+		return -ENXIO;
+	}
+
+	if (unlikely(!tty->port->ops)) {
+		pr_err("VCC: open: TTY ops not defined\n");
+		return -ENXIO;
+	}
+
+	return tty_port_open(tty->port, tty, vcc_file);
+}
+
+static void vcc_close(struct tty_struct *tty, struct file *vcc_file)
+{
+	if (unlikely(!tty)) {
+		pr_err("VCC: close: Invalid TTY handle\n");
+		return;
+	}
+
+	if (unlikely(tty->count > 1))
+		return;
+
+	if (unlikely(!tty->port)) {
+		pr_err("VCC: close: TTY port not found\n");
+		return;
+	}
+
+	tty_port_close(tty->port, tty, vcc_file);
+}
+
+static void vcc_ldc_hup(struct vcc_port *port)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&port->lock, flags);
+
+	if (vcc_send_ctl(port, VCC_CTL_HUP) < 0)
+		vcc_kick_tx(port);
+
+	spin_unlock_irqrestore(&port->lock, flags);
+}
+
+static void vcc_hangup(struct tty_struct *tty)
+{
+	struct vcc_port *port;
+
+	if (unlikely(!tty)) {
+		pr_err("VCC: hangup: Invalid TTY handle\n");
+		return;
+	}
+
+	port = vcc_get_ne(tty->index);
+	if (unlikely(!port)) {
+		pr_err("VCC: hangup: Failed to find VCC port\n");
+		return;
+	}
+
+	if (unlikely(!tty->port)) {
+		pr_err("VCC: hangup: TTY port not found\n");
+		vcc_put(port, false);
+		return;
+	}
+
+	vcc_ldc_hup(port);
+
+	vcc_put(port, false);
+
+	tty_port_hangup(tty->port);
+}
+
+static int vcc_write(struct tty_struct *tty, const unsigned char *buf,
+		     int count)
+{
+	struct vcc_port *port;
+	struct vio_vcc *pkt;
+	unsigned long flags;
+	int total_sent = 0;
+	int tosend = 0;
+	int rv = -EINVAL;
+
+	if (unlikely(!tty)) {
+		pr_err("VCC: write: Invalid TTY handle\n");
+		return -ENXIO;
+	}
+
+	port = vcc_get_ne(tty->index);
+	if (unlikely(!port)) {
+		pr_err("VCC: write: Failed to find VCC port");
+		return -ENODEV;
+	}
+
+	spin_lock_irqsave(&port->lock, flags);
+
+	pkt = &port->buffer;
+	pkt->tag.type = VIO_TYPE_DATA;
+
+	while (count > 0) {
+		/* Minimum of data to write and space available */
+		tosend = min(count, (VCC_BUFF_LEN - port->chars_in_buffer));
+
+		if (!tosend)
+			break;
+
+		memcpy(&pkt->data[port->chars_in_buffer], &buf[total_sent],
+		       tosend);
+		port->chars_in_buffer += tosend;
+		pkt->tag.stype = tosend;
+
+		vccdbg("TAG [%02x:%02x:%04x:%08x]\n", pkt->tag.type,
+		       pkt->tag.stype, pkt->tag.stype_env, pkt->tag.sid);
+		vccdbg("DATA [%s]\n", pkt->data);
+		vccdbgl(port->vio.lp);
+
+		/* Since we know we have enough room in VCC buffer for tosend
+		 * we record that it was sent regardless of whether the
+		 * hypervisor actually took it because we have it buffered.
+		 */
+		rv = ldc_write(port->vio.lp, pkt, (VIO_TAG_SIZE + tosend));
+		vccdbg("VCC: write: ldc_write(%d)=%d\n",
+		       (VIO_TAG_SIZE + tosend), rv);
+
+		total_sent += tosend;
+		count -= tosend;
+		if (rv < 0) {
+			vcc_kick_tx(port);
+			break;
+		}
+
+		port->chars_in_buffer = 0;
+	}
+
+	spin_unlock_irqrestore(&port->lock, flags);
+
+	vcc_put(port, false);
+
+	vccdbg("VCC: write: total=%d rv=%d", total_sent, rv);
+
+	return total_sent ? total_sent : rv;
+}
+
+static int vcc_write_room(struct tty_struct *tty)
+{
+	struct vcc_port *port;
+	u64 num;
+
+	if (unlikely(!tty)) {
+		pr_err("VCC: write_room: Invalid TTY handle\n");
+		return -ENXIO;
+	}
+
+	port = vcc_get_ne(tty->index);
+	if (unlikely(!port)) {
+		pr_err("VCC: write_room: Failed to find VCC port\n");
+		return -ENODEV;
+	}
+
+	num = VCC_BUFF_LEN - port->chars_in_buffer;
+
+	vcc_put(port, false);
+
+	return num;
+}
+
+static int vcc_chars_in_buffer(struct tty_struct *tty)
+{
+	struct vcc_port *port;
+	u64 num;
+
+	if (unlikely(!tty)) {
+		pr_err("VCC: chars_in_buffer: Invalid TTY handle\n");
+		return -ENXIO;
+	}
+
+	port = vcc_get_ne(tty->index);
+	if (unlikely(!port)) {
+		pr_err("VCC: chars_in_buffer: Failed to find VCC port\n");
+		return -ENODEV;
+	}
+
+	num = port->chars_in_buffer;
+
+	vcc_put(port, false);
+
+	return num;
+}
+
+static int vcc_break_ctl(struct tty_struct *tty, int state)
+{
+	struct vcc_port *port;
+	unsigned long flags;
+
+	if (unlikely(!tty)) {
+		pr_err("VCC: break_ctl: Invalid TTY handle\n");
+		return -ENXIO;
+	}
+
+	port = vcc_get_ne(tty->index);
+	if (unlikely(!port)) {
+		pr_err("VCC: break_ctl: Failed to find VCC port\n");
+		return -ENODEV;
+	}
+
+	/* Turn off break */
+	if (state == 0) {
+		vcc_put(port, false);
+		return 0;
+	}
+
+	spin_lock_irqsave(&port->lock, flags);
+
+	if (vcc_send_ctl(port, VCC_CTL_BREAK) < 0)
+		vcc_kick_tx(port);
+
+	spin_unlock_irqrestore(&port->lock, flags);
+
+	vcc_put(port, false);
+
+	return 0;
+}
+
+static int vcc_install(struct tty_driver *driver, struct tty_struct *tty)
+{
+	struct vcc_port *port_vcc;
+	struct tty_port *port_tty;
+	int ret;
+
+	if (unlikely(!tty)) {
+		pr_err("VCC: install: Invalid TTY handle\n");
+		return -ENXIO;
+	}
+
+	if (tty->index >= VCC_MAX_PORTS)
+		return -EINVAL;
+
+	ret = tty_standard_install(driver, tty);
+	if (ret)
+		return ret;
+
+	port_tty = kzalloc(sizeof(struct tty_port), GFP_KERNEL);
+	if (!port_tty)
+		return -ENOMEM;
+
+	port_vcc = vcc_get(tty->index, true);
+	if (!port_vcc) {
+		pr_err("VCC: install: Failed to find VCC port\n");
+		tty->port = NULL;
+		kfree(port_tty);
+		return -ENODEV;
+	}
+
+	tty_port_init(port_tty);
+	port_tty->ops = &vcc_port_ops;
+	tty->port = port_tty;
+
+	port_vcc->tty = tty;
+
+	vcc_put(port_vcc, true);
+
+	return 0;
+}
+
+static void vcc_cleanup(struct tty_struct *tty)
+{
+	struct vcc_port *port;
+
+	if (unlikely(!tty)) {
+		pr_err("VCC: cleanup: Invalid TTY handle\n");
+		return;
+	}
+
+	port = vcc_get(tty->index, true);
+	if (port) {
+		port->tty = NULL;
+
+		if (port->removed) {
+			vcc_table_remove(tty->index);
+			kfree(port->vio.name);
+			kfree(port->domain);
+			kfree(port);
+		} else {
+			vcc_put(port, true);
+		}
+	}
+
+	tty_port_destroy(tty->port);
+	kfree(tty->port);
+	tty->port = NULL;
+}
+
+static const struct tty_operations vcc_ops = {
+	.open			= vcc_open,
+	.close			= vcc_close,
+	.hangup			= vcc_hangup,
+	.write			= vcc_write,
+	.write_room		= vcc_write_room,
+	.chars_in_buffer	= vcc_chars_in_buffer,
+	.break_ctl		= vcc_break_ctl,
+	.install		= vcc_install,
+	.cleanup		= vcc_cleanup,
+};
+
+#define VCC_TTY_FLAGS   (TTY_DRIVER_DYNAMIC_DEV | TTY_DRIVER_REAL_RAW)
+
+static int vcc_tty_init(void)
+{
+	int rv;
+
+	pr_info("VCC: %s\n", version);
+
+	vcc_tty_driver = tty_alloc_driver(VCC_MAX_PORTS, VCC_TTY_FLAGS);
+	if (IS_ERR(vcc_tty_driver)) {
+		pr_err("VCC: TTY driver alloc failed\n");
+		return PTR_ERR(vcc_tty_driver);
+	}
+
+	vcc_tty_driver->driver_name = vcc_driver_name;
+	vcc_tty_driver->name = vcc_device_node;
+
+	vcc_tty_driver->minor_start = VCC_MINOR_START;
+	vcc_tty_driver->type = TTY_DRIVER_TYPE_SYSTEM;
+	vcc_tty_driver->init_termios = vcc_tty_termios;
+
+	tty_set_operations(vcc_tty_driver, &vcc_ops);
+
+	rv = tty_register_driver(vcc_tty_driver);
+	if (rv) {
+		pr_err("VCC: TTY driver registration failed\n");
+		put_tty_driver(vcc_tty_driver);
+		vcc_tty_driver = NULL;
+		return rv;
+	}
+
+	vccdbg("VCC: TTY driver registered\n");
+
+	return 0;
+}
+
+static void vcc_tty_exit(void)
+{
+	tty_unregister_driver(vcc_tty_driver);
+	put_tty_driver(vcc_tty_driver);
+	vccdbg("VCC: TTY driver unregistered\n");
+
+	vcc_tty_driver = NULL;
+}
+
+static int __init vcc_init(void)
+{
+	int rv;
+
+	rv = vcc_tty_init();
+	if (rv) {
+		pr_err("VCC: TTY init failed\n");
+		return rv;
+	}
+
+	rv = vio_register_driver(&vcc_driver);
+	if (rv) {
+		pr_err("VCC: VIO driver registration failed\n");
+		vcc_tty_exit();
+	} else {
+		vccdbg("VCC: VIO driver registered successfully\n");
+	}
+
+	return rv;
+}
+
+static void __exit vcc_exit(void)
+{
+	vio_unregister_driver(&vcc_driver);
+	vccdbg("VCC: VIO driver unregistered\n");
+	vcc_tty_exit();
+	vccdbg("VCC: TTY driver unregistered\n");
+}
+
+module_init(vcc_init);
+module_exit(vcc_exit);
diff --git a/drivers/usb/atm/cxacru.c b/drivers/usb/atm/cxacru.c
index d65a64c29b85..5160a4a966b3 100644
--- a/drivers/usb/atm/cxacru.c
+++ b/drivers/usb/atm/cxacru.c
@@ -43,7 +43,6 @@
 #include "usbatm.h"
 
 #define DRIVER_AUTHOR	"Roman Kagan, David Woodhouse, Duncan Sands, Simon Arlott"
-#define DRIVER_VERSION	"0.4"
 #define DRIVER_DESC	"Conexant AccessRunner ADSL USB modem driver"
 
 static const char cxacru_driver_name[] = "cxacru";
@@ -1380,4 +1379,3 @@ module_usb_driver(cxacru_usb_driver);
 MODULE_AUTHOR(DRIVER_AUTHOR);
 MODULE_DESCRIPTION(DRIVER_DESC);
 MODULE_LICENSE("GPL");
-MODULE_VERSION(DRIVER_VERSION);
diff --git a/drivers/usb/atm/speedtch.c b/drivers/usb/atm/speedtch.c
index 5083eb5b0d5e..3676adb40d89 100644
--- a/drivers/usb/atm/speedtch.c
+++ b/drivers/usb/atm/speedtch.c
@@ -40,8 +40,7 @@
 #include "usbatm.h"
 
 #define DRIVER_AUTHOR	"Johan Verrept, Duncan Sands <duncan.sands@free.fr>"
-#define DRIVER_VERSION	"1.10"
-#define DRIVER_DESC	"Alcatel SpeedTouch USB driver version " DRIVER_VERSION
+#define DRIVER_DESC	"Alcatel SpeedTouch USB driver"
 
 static const char speedtch_driver_name[] = "speedtch";
 
@@ -738,7 +737,7 @@ static int speedtch_post_reset(struct usb_interface *intf)
 **  USB  **
 **********/
 
-static struct usb_device_id speedtch_usb_ids[] = {
+static const struct usb_device_id speedtch_usb_ids[] = {
 	{USB_DEVICE(0x06b9, 0x4061)},
 	{}
 };
@@ -962,4 +961,3 @@ module_usb_driver(speedtch_usb_driver);
 MODULE_AUTHOR(DRIVER_AUTHOR);
 MODULE_DESCRIPTION(DRIVER_DESC);
 MODULE_LICENSE("GPL");
-MODULE_VERSION(DRIVER_VERSION);
diff --git a/drivers/usb/atm/ueagle-atm.c b/drivers/usb/atm/ueagle-atm.c
index df67815f74e6..ba7616395db2 100644
--- a/drivers/usb/atm/ueagle-atm.c
+++ b/drivers/usb/atm/ueagle-atm.c
@@ -2212,7 +2212,7 @@ static int uea_boot(struct uea_softc *sc)
 	ret = usb_submit_urb(sc->urb_int, GFP_KERNEL);
 	if (ret < 0) {
 		uea_err(INS_TO_USBDEV(sc),
-		       "urb submition failed with error %d\n", ret);
+		       "urb submission failed with error %d\n", ret);
 		goto err1;
 	}
 
@@ -2522,7 +2522,7 @@ static struct attribute *attrs[] = {
 	&dev_attr_stat_firmid.attr,
 	NULL,
 };
-static struct attribute_group attr_grp = {
+static const struct attribute_group attr_grp = {
 	.attrs = attrs,
 };
 
diff --git a/drivers/usb/atm/usbatm.c b/drivers/usb/atm/usbatm.c
index 3e80aa3b917a..8607af758bbd 100644
--- a/drivers/usb/atm/usbatm.c
+++ b/drivers/usb/atm/usbatm.c
@@ -93,8 +93,7 @@ static int usbatm_print_packet(struct usbatm_data *instance, const unsigned char
 #endif
 
 #define DRIVER_AUTHOR	"Johan Verrept, Duncan Sands <duncan.sands@free.fr>"
-#define DRIVER_VERSION	"1.10"
-#define DRIVER_DESC	"Generic USB ATM/DSL I/O, version " DRIVER_VERSION
+#define DRIVER_DESC	"Generic USB ATM/DSL I/O"
 
 static const char usbatm_driver_name[] = "usbatm";
 
@@ -174,7 +173,7 @@ static int usbatm_atm_ioctl(struct atm_dev *atm_dev, unsigned int cmd, void __us
 static int usbatm_atm_send(struct atm_vcc *vcc, struct sk_buff *skb);
 static int usbatm_atm_proc_read(struct atm_dev *atm_dev, loff_t *pos, char *page);
 
-static struct atmdev_ops usbatm_atm_devops = {
+static const struct atmdev_ops usbatm_atm_devops = {
 	.dev_close	= usbatm_atm_dev_close,
 	.open		= usbatm_atm_open,
 	.close		= usbatm_atm_close,
@@ -1315,7 +1314,6 @@ module_exit(usbatm_usb_exit);
 MODULE_AUTHOR(DRIVER_AUTHOR);
 MODULE_DESCRIPTION(DRIVER_DESC);
 MODULE_LICENSE("GPL");
-MODULE_VERSION(DRIVER_VERSION);
 
 /************
 **  debug  **
diff --git a/drivers/usb/atm/xusbatm.c b/drivers/usb/atm/xusbatm.c
index a87597f88a84..c73c1ec3005e 100644
--- a/drivers/usb/atm/xusbatm.c
+++ b/drivers/usb/atm/xusbatm.c
@@ -228,4 +228,3 @@ module_exit(xusbatm_exit);
 MODULE_AUTHOR("Roman Kagan, Duncan Sands");
 MODULE_DESCRIPTION("Driver for USB ADSL modems initialized in userspace");
 MODULE_LICENSE("GPL");
-MODULE_VERSION("0.1");
diff --git a/drivers/usb/c67x00/c67x00-hcd.c b/drivers/usb/c67x00/c67x00-hcd.c
index c2d13968da82..30d3f346686e 100644
--- a/drivers/usb/c67x00/c67x00-hcd.c
+++ b/drivers/usb/c67x00/c67x00-hcd.c
@@ -305,7 +305,7 @@ static int c67x00_hcd_get_frame(struct usb_hcd *hcd)
 	return temp_val ? (temp_val - 1) : HOST_FRAME_MASK;
 }
 
-static struct hc_driver c67x00_hc_driver = {
+static const struct hc_driver c67x00_hc_driver = {
 	.description	= "c67x00-hcd",
 	.product_desc	= "Cypress C67X00 Host Controller",
 	.hcd_priv_size	= sizeof(struct c67x00_hcd),
diff --git a/drivers/usb/chipidea/Makefile b/drivers/usb/chipidea/Makefile
index 39fca5715ed3..ddcbddf8361a 100644
--- a/drivers/usb/chipidea/Makefile
+++ b/drivers/usb/chipidea/Makefile
@@ -15,3 +15,4 @@ obj-$(CONFIG_USB_CHIPIDEA)	+= ci_hdrc_zevio.o
 obj-$(CONFIG_USB_CHIPIDEA_PCI)	+= ci_hdrc_pci.o
 
 obj-$(CONFIG_USB_CHIPIDEA_OF)	+= usbmisc_imx.o ci_hdrc_imx.o
+obj-$(CONFIG_USB_CHIPIDEA_OF)	+= ci_hdrc_tegra.o
diff --git a/drivers/usb/chipidea/ci_hdrc_msm.c b/drivers/usb/chipidea/ci_hdrc_msm.c
index 0bdfcdcbf7a5..bb626120296f 100644
--- a/drivers/usb/chipidea/ci_hdrc_msm.c
+++ b/drivers/usb/chipidea/ci_hdrc_msm.c
@@ -251,7 +251,7 @@ static int ci_hdrc_msm_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_mux;
 
-	ulpi_node = of_find_node_by_name(pdev->dev.of_node, "ulpi");
+	ulpi_node = of_find_node_by_name(of_node_get(pdev->dev.of_node), "ulpi");
 	if (ulpi_node) {
 		phy_node = of_get_next_available_child(ulpi_node, NULL);
 		ci->hsic = of_device_is_compatible(phy_node, "qcom,usb-hsic-phy");
diff --git a/drivers/usb/chipidea/ci_hdrc_pci.c b/drivers/usb/chipidea/ci_hdrc_pci.c
index b635ab67490d..39414e4b2d81 100644
--- a/drivers/usb/chipidea/ci_hdrc_pci.c
+++ b/drivers/usb/chipidea/ci_hdrc_pci.c
@@ -170,5 +170,4 @@ module_pci_driver(ci_hdrc_pci_driver);
 MODULE_AUTHOR("MIPS - David Lopo <dlopo@chipidea.mips.com>");
 MODULE_DESCRIPTION("MIPS CI13XXX USB Peripheral Controller");
 MODULE_LICENSE("GPL");
-MODULE_VERSION("June 2008");
 MODULE_ALIAS("platform:ci13xxx_pci");
diff --git a/drivers/usb/chipidea/ci_hdrc_tegra.c b/drivers/usb/chipidea/ci_hdrc_tegra.c
new file mode 100644
index 000000000000..bfcee2702d50
--- /dev/null
+++ b/drivers/usb/chipidea/ci_hdrc_tegra.c
@@ -0,0 +1,155 @@
+/*
+ * Copyright (c) 2016, NVIDIA Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ */
+
+#include <linux/clk.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/reset.h>
+
+#include <linux/usb/chipidea.h>
+
+#include "ci.h"
+
+struct tegra_udc {
+	struct ci_hdrc_platform_data data;
+	struct platform_device *dev;
+
+	struct usb_phy *phy;
+	struct clk *clk;
+};
+
+struct tegra_udc_soc_info {
+	unsigned long flags;
+};
+
+static const struct tegra_udc_soc_info tegra20_udc_soc_info = {
+	.flags = CI_HDRC_REQUIRES_ALIGNED_DMA,
+};
+
+static const struct tegra_udc_soc_info tegra30_udc_soc_info = {
+	.flags = 0,
+};
+
+static const struct tegra_udc_soc_info tegra114_udc_soc_info = {
+	.flags = 0,
+};
+
+static const struct tegra_udc_soc_info tegra124_udc_soc_info = {
+	.flags = 0,
+};
+
+static const struct of_device_id tegra_udc_of_match[] = {
+	{
+		.compatible = "nvidia,tegra20-udc",
+		.data = &tegra20_udc_soc_info,
+	}, {
+		.compatible = "nvidia,tegra30-udc",
+		.data = &tegra30_udc_soc_info,
+	}, {
+		.compatible = "nvidia,tegra114-udc",
+		.data = &tegra114_udc_soc_info,
+	}, {
+		.compatible = "nvidia,tegra124-udc",
+		.data = &tegra124_udc_soc_info,
+	}, {
+		/* sentinel */
+	}
+};
+MODULE_DEVICE_TABLE(of, tegra_udc_of_match);
+
+static int tegra_udc_probe(struct platform_device *pdev)
+{
+	const struct tegra_udc_soc_info *soc;
+	struct tegra_udc *udc;
+	int err;
+
+	udc = devm_kzalloc(&pdev->dev, sizeof(*udc), GFP_KERNEL);
+	if (!udc)
+		return -ENOMEM;
+
+	soc = of_device_get_match_data(&pdev->dev);
+	if (!soc) {
+		dev_err(&pdev->dev, "failed to match OF data\n");
+		return -EINVAL;
+	}
+
+	udc->phy = devm_usb_get_phy_by_phandle(&pdev->dev, "nvidia,phy", 0);
+	if (IS_ERR(udc->phy)) {
+		err = PTR_ERR(udc->phy);
+		dev_err(&pdev->dev, "failed to get PHY: %d\n", err);
+		return err;
+	}
+
+	udc->clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(udc->clk)) {
+		err = PTR_ERR(udc->clk);
+		dev_err(&pdev->dev, "failed to get clock: %d\n", err);
+		return err;
+	}
+
+	err = clk_prepare_enable(udc->clk);
+	if (err < 0) {
+		dev_err(&pdev->dev, "failed to enable clock: %d\n", err);
+		return err;
+	}
+
+	/*
+	 * Tegra's USB PHY driver doesn't implement optional phy_init()
+	 * hook, so we have to power on UDC controller before ChipIdea
+	 * driver initialization kicks in.
+	 */
+	usb_phy_set_suspend(udc->phy, 0);
+
+	/* setup and register ChipIdea HDRC device */
+	udc->data.name = "tegra-udc";
+	udc->data.flags = soc->flags;
+	udc->data.usb_phy = udc->phy;
+	udc->data.capoffset = DEF_CAPOFFSET;
+
+	udc->dev = ci_hdrc_add_device(&pdev->dev, pdev->resource,
+				      pdev->num_resources, &udc->data);
+	if (IS_ERR(udc->dev)) {
+		err = PTR_ERR(udc->dev);
+		dev_err(&pdev->dev, "failed to add HDRC device: %d\n", err);
+		goto fail_power_off;
+	}
+
+	platform_set_drvdata(pdev, udc);
+
+	return 0;
+
+fail_power_off:
+	usb_phy_set_suspend(udc->phy, 1);
+	clk_disable_unprepare(udc->clk);
+	return err;
+}
+
+static int tegra_udc_remove(struct platform_device *pdev)
+{
+	struct tegra_udc *udc = platform_get_drvdata(pdev);
+
+	usb_phy_set_suspend(udc->phy, 1);
+	clk_disable_unprepare(udc->clk);
+
+	return 0;
+}
+
+static struct platform_driver tegra_udc_driver = {
+	.driver = {
+		.name = "tegra-udc",
+		.of_match_table = tegra_udc_of_match,
+	},
+	.probe = tegra_udc_probe,
+	.remove = tegra_udc_remove,
+};
+module_platform_driver(tegra_udc_driver);
+
+MODULE_DESCRIPTION("NVIDIA Tegra USB device mode driver");
+MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
+MODULE_ALIAS("platform:tegra-udc");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/usb/chipidea/ci_hdrc_usb2.c b/drivers/usb/chipidea/ci_hdrc_usb2.c
index d162cc0bb8ce..99425db9ba62 100644
--- a/drivers/usb/chipidea/ci_hdrc_usb2.c
+++ b/drivers/usb/chipidea/ci_hdrc_usb2.c
@@ -52,6 +52,8 @@ static int ci_hdrc_usb2_probe(struct platform_device *pdev)
 
 	if (!ci_pdata) {
 		ci_pdata = devm_kmalloc(dev, sizeof(*ci_pdata), GFP_KERNEL);
+		if (!ci_pdata)
+			return -ENOMEM;
 		*ci_pdata = ci_default_pdata;	/* struct copy */
 	}
 
diff --git a/drivers/usb/chipidea/core.c b/drivers/usb/chipidea/core.c
index b17ed3a9a304..43ea5fb87b9a 100644
--- a/drivers/usb/chipidea/core.c
+++ b/drivers/usb/chipidea/core.c
@@ -736,7 +736,7 @@ static int ci_extcon_register(struct ci_hdrc *ci)
 
 	id = &ci->platdata->id_extcon;
 	id->ci = ci;
-	if (!IS_ERR(id->edev)) {
+	if (!IS_ERR_OR_NULL(id->edev)) {
 		ret = devm_extcon_register_notifier(ci->dev, id->edev,
 						EXTCON_USB_HOST, &id->nb);
 		if (ret < 0) {
@@ -747,7 +747,7 @@ static int ci_extcon_register(struct ci_hdrc *ci)
 
 	vbus = &ci->platdata->vbus_extcon;
 	vbus->ci = ci;
-	if (!IS_ERR(vbus->edev)) {
+	if (!IS_ERR_OR_NULL(vbus->edev)) {
 		ret = devm_extcon_register_notifier(ci->dev, vbus->edev,
 						EXTCON_USB, &vbus->nb);
 		if (ret < 0) {
@@ -887,7 +887,7 @@ static struct attribute *ci_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group ci_attr_group = {
+static const struct attribute_group ci_attr_group = {
 	.attrs = ci_attrs,
 };
 
diff --git a/drivers/usb/chipidea/otg_fsm.c b/drivers/usb/chipidea/otg_fsm.c
index 949183ede16f..5ea0246f650d 100644
--- a/drivers/usb/chipidea/otg_fsm.c
+++ b/drivers/usb/chipidea/otg_fsm.c
@@ -193,7 +193,7 @@ static struct attribute *inputs_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group inputs_attr_group = {
+static const struct attribute_group inputs_attr_group = {
 	.name = "inputs",
 	.attrs = inputs_attrs,
 };
diff --git a/drivers/usb/chipidea/udc.c b/drivers/usb/chipidea/udc.c
index d68b125796f9..fe8a90543ea3 100644
--- a/drivers/usb/chipidea/udc.c
+++ b/drivers/usb/chipidea/udc.c
@@ -944,7 +944,6 @@ isr_setup_status_complete(struct usb_ep *ep, struct usb_request *req)
  */
 static int isr_setup_status_phase(struct ci_hdrc *ci)
 {
-	int retval;
 	struct ci_hw_ep *hwep;
 
 	/*
@@ -960,9 +959,7 @@ static int isr_setup_status_phase(struct ci_hdrc *ci)
 	ci->status->context = ci;
 	ci->status->complete = isr_setup_status_complete;
 
-	retval = _ep_queue(&hwep->ep, ci->status, GFP_ATOMIC);
-
-	return retval;
+	return _ep_queue(&hwep->ep, ci->status, GFP_ATOMIC);
 }
 
 /**
@@ -1899,6 +1896,9 @@ static int udc_start(struct ci_hdrc *ci)
 	ci->gadget.name         = ci->platdata->name;
 	ci->gadget.otg_caps	= otg_caps;
 
+	if (ci->platdata->flags & CI_HDRC_REQUIRES_ALIGNED_DMA)
+		ci->gadget.quirk_avoids_skb_reserve = 1;
+
 	if (ci->is_otg && (otg_caps->hnp_support || otg_caps->srp_support ||
 						otg_caps->adp_support))
 		ci->gadget.is_otg = 1;
diff --git a/drivers/usb/class/cdc-wdm.c b/drivers/usb/class/cdc-wdm.c
index 8f972247b1c1..5aacea1978a5 100644
--- a/drivers/usb/class/cdc-wdm.c
+++ b/drivers/usb/class/cdc-wdm.c
@@ -26,10 +26,6 @@
 #include <asm/unaligned.h>
 #include <linux/usb/cdc-wdm.h>
 
-/*
- * Version Information
- */
-#define DRIVER_VERSION "v0.03"
 #define DRIVER_AUTHOR "Oliver Neukum"
 #define DRIVER_DESC "USB Abstract Control Model driver for USB WCM Device Management"
 
diff --git a/drivers/usb/class/usbtmc.c b/drivers/usb/class/usbtmc.c
index 578f424decc2..6ebfabfa0dc7 100644
--- a/drivers/usb/class/usbtmc.c
+++ b/drivers/usb/class/usbtmc.c
@@ -1085,7 +1085,7 @@ static struct attribute *capability_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group capability_attr_grp = {
+static const struct attribute_group capability_attr_grp = {
 	.attrs = capability_attrs,
 };
 
@@ -1151,7 +1151,7 @@ static struct attribute *data_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group data_attr_grp = {
+static const struct attribute_group data_attr_grp = {
 	.attrs = data_attrs,
 };
 
diff --git a/drivers/usb/common/common.c b/drivers/usb/common/common.c
index 5ef8da6e67c3..552ff7ac5a6b 100644
--- a/drivers/usb/common/common.c
+++ b/drivers/usb/common/common.c
@@ -190,10 +190,7 @@ EXPORT_SYMBOL_GPL(of_usb_get_dr_mode_by_phy);
  */
 bool of_usb_host_tpl_support(struct device_node *np)
 {
-	if (of_find_property(np, "tpl-support", NULL))
-		return true;
-
-	return false;
+	return of_property_read_bool(np, "tpl-support");
 }
 EXPORT_SYMBOL_GPL(of_usb_host_tpl_support);
 
@@ -227,8 +224,8 @@ int of_usb_update_otg_caps(struct device_node *np,
 				otg_caps->otg_rev = otg_rev;
 			break;
 		default:
-			pr_err("%s: unsupported otg-rev: 0x%x\n",
-						np->full_name, otg_rev);
+			pr_err("%pOF: unsupported otg-rev: 0x%x\n",
+						np, otg_rev);
 			return -EINVAL;
 		}
 	} else {
@@ -240,11 +237,11 @@ int of_usb_update_otg_caps(struct device_node *np,
 		otg_caps->otg_rev = 0;
 	}
 
-	if (of_find_property(np, "hnp-disable", NULL))
+	if (of_property_read_bool(np, "hnp-disable"))
 		otg_caps->hnp_support = false;
-	if (of_find_property(np, "srp-disable", NULL))
+	if (of_property_read_bool(np, "srp-disable"))
 		otg_caps->srp_support = false;
-	if (of_find_property(np, "adp-disable", NULL) ||
+	if (of_property_read_bool(np, "adp-disable") ||
 				(otg_caps->otg_rev < 0x0200))
 		otg_caps->adp_support = false;
 
diff --git a/drivers/usb/common/ulpi.c b/drivers/usb/common/ulpi.c
index 930e8f35f8df..4aa5195db8ea 100644
--- a/drivers/usb/common/ulpi.c
+++ b/drivers/usb/common/ulpi.c
@@ -135,7 +135,7 @@ static void ulpi_dev_release(struct device *dev)
 	kfree(to_ulpi_dev(dev));
 }
 
-static struct device_type ulpi_dev_type = {
+static const struct device_type ulpi_dev_type = {
 	.name = "ulpi_device",
 	.groups = ulpi_dev_attr_groups,
 	.release = ulpi_dev_release,
diff --git a/drivers/usb/core/devio.c b/drivers/usb/core/devio.c
index ebe27595c4af..318bb3b96687 100644
--- a/drivers/usb/core/devio.c
+++ b/drivers/usb/core/devio.c
@@ -210,7 +210,7 @@ static void usbdev_vm_close(struct vm_area_struct *vma)
 	dec_usb_memory_use_count(usbm, &usbm->vma_use_count);
 }
 
-static struct vm_operations_struct usbdev_vm_ops = {
+static const struct vm_operations_struct usbdev_vm_ops = {
 	.open = usbdev_vm_open,
 	.close = usbdev_vm_close
 };
@@ -623,6 +623,8 @@ static void async_completed(struct urb *urb)
 	if (as->status < 0 && as->bulk_addr && as->status != -ECONNRESET &&
 			as->status != -ENOENT)
 		cancel_bulk_urbs(ps, as->bulk_addr);
+
+	wake_up(&ps->wait);
 	spin_unlock(&ps->lock);
 
 	if (signr) {
@@ -630,8 +632,6 @@ static void async_completed(struct urb *urb)
 		put_pid(pid);
 		put_cred(cred);
 	}
-
-	wake_up(&ps->wait);
 }
 
 static void destroy_async(struct usb_dev_state *ps, struct list_head *list)
diff --git a/drivers/usb/core/hcd.c b/drivers/usb/core/hcd.c
index 7f277b092b5b..75ad6718858c 100644
--- a/drivers/usb/core/hcd.c
+++ b/drivers/usb/core/hcd.c
@@ -972,7 +972,7 @@ static struct attribute *usb_bus_attrs[] = {
 		NULL,
 };
 
-static struct attribute_group usb_bus_attr_group = {
+static const struct attribute_group usb_bus_attr_group = {
 	.name = NULL,	/* we want them in the same directory */
 	.attrs = usb_bus_attrs,
 };
diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c
index 822f8c50e423..41eaf0b52518 100644
--- a/drivers/usb/core/hub.c
+++ b/drivers/usb/core/hub.c
@@ -2614,7 +2614,7 @@ static unsigned hub_is_wusb(struct usb_hub *hub)
 #define SET_CONFIG_TRIES	(2 * (use_both_schemes + 1))
 #define USE_NEW_SCHEME(i)	((i) / 2 == (int)old_scheme_first)
 
-#define HUB_ROOT_RESET_TIME	50	/* times are in msec */
+#define HUB_ROOT_RESET_TIME	60	/* times are in msec */
 #define HUB_SHORT_RESET_TIME	10
 #define HUB_BH_RESET_TIME	50
 #define HUB_LONG_RESET_TIME	200
@@ -4342,6 +4342,7 @@ hub_port_init(struct usb_hub *hub, struct usb_device *udev, int port1,
 	enum usb_device_speed	oldspeed = udev->speed;
 	const char		*speed;
 	int			devnum = udev->devnum;
+	const char		*driver_name;
 
 	/* root hub ports have a slightly longer reset period
 	 * (from USB 2.0 spec, section 7.1.7.5)
@@ -4409,11 +4410,23 @@ hub_port_init(struct usb_hub *hub, struct usb_device *udev, int port1,
 	else
 		speed = usb_speed_string(udev->speed);
 
+	/*
+	 * The controller driver may be NULL if the controller device
+	 * is the middle device between platform device and roothub.
+	 * This middle device may not need a device driver due to
+	 * all hardware control can be at platform device driver, this
+	 * platform device is usually a dual-role USB controller device.
+	 */
+	if (udev->bus->controller->driver)
+		driver_name = udev->bus->controller->driver->name;
+	else
+		driver_name = udev->bus->sysdev->driver->name;
+
 	if (udev->speed < USB_SPEED_SUPER)
 		dev_info(&udev->dev,
 				"%s %s USB device number %d using %s\n",
 				(udev->config) ? "reset" : "new", speed,
-				devnum, udev->bus->controller->driver->name);
+				devnum, driver_name);
 
 	/* Set up TT records, if needed  */
 	if (hdev->tt) {
@@ -4545,7 +4558,7 @@ hub_port_init(struct usb_hub *hub, struct usb_device *udev, int port1,
 						"%s SuperSpeed%s USB device number %d using %s\n",
 						(udev->config) ? "reset" : "new",
 					 (udev->speed == USB_SPEED_SUPER_PLUS) ? "Plus" : "",
-						devnum, udev->bus->controller->driver->name);
+					 devnum, driver_name);
 			}
 
 			/* cope with hardware quirkiness:
diff --git a/drivers/usb/core/ledtrig-usbport.c b/drivers/usb/core/ledtrig-usbport.c
index 16c19a31dad1..1af877942110 100644
--- a/drivers/usb/core/ledtrig-usbport.c
+++ b/drivers/usb/core/ledtrig-usbport.c
@@ -149,8 +149,8 @@ static bool usbport_trig_port_observed(struct usbport_trig_data *usbport_data,
 	count = of_count_phandle_with_args(led_np, "trigger-sources",
 					   "#trigger-source-cells");
 	if (count < 0) {
-		dev_warn(dev, "Failed to get trigger sources for %s\n",
-			 led_np->full_name);
+		dev_warn(dev, "Failed to get trigger sources for %pOF\n",
+			 led_np);
 		return false;
 	}
 
@@ -205,6 +205,7 @@ static int usbport_trig_add_port(struct usbport_trig_data *usbport_data,
 	}
 	snprintf(port->port_name, len, "%s-port%d", hub_name, portnum);
 
+	sysfs_attr_init(&port->attr.attr);
 	port->attr.attr.name = port->port_name;
 	port->attr.attr.mode = S_IRUSR | S_IWUSR;
 	port->attr.show = usbport_trig_port_show;
diff --git a/drivers/usb/core/quirks.c b/drivers/usb/core/quirks.c
index 574da2b4529c..82806e311202 100644
--- a/drivers/usb/core/quirks.c
+++ b/drivers/usb/core/quirks.c
@@ -57,8 +57,9 @@ static const struct usb_device_id usb_quirk_list[] = {
 	/* Microsoft LifeCam-VX700 v2.0 */
 	{ USB_DEVICE(0x045e, 0x0770), .driver_info = USB_QUIRK_RESET_RESUME },
 
-	/* Logitech HD Pro Webcams C920 and C930e */
+	/* Logitech HD Pro Webcams C920, C920-C and C930e */
 	{ USB_DEVICE(0x046d, 0x082d), .driver_info = USB_QUIRK_DELAY_INIT },
+	{ USB_DEVICE(0x046d, 0x0841), .driver_info = USB_QUIRK_DELAY_INIT },
 	{ USB_DEVICE(0x046d, 0x0843), .driver_info = USB_QUIRK_DELAY_INIT },
 
 	/* Logitech ConferenceCam CC3000e */
@@ -217,6 +218,9 @@ static const struct usb_device_id usb_quirk_list[] = {
 	{ USB_DEVICE(0x1a0a, 0x0200), .driver_info =
 			USB_QUIRK_LINEAR_UFRAME_INTR_BINTERVAL },
 
+	/* Corsair Strafe RGB */
+	{ USB_DEVICE(0x1b1c, 0x1b20), .driver_info = USB_QUIRK_DELAY_INIT },
+
 	/* Acer C120 LED Projector */
 	{ USB_DEVICE(0x1de1, 0xc102), .driver_info = USB_QUIRK_NO_LPM },
 
diff --git a/drivers/usb/core/sysfs.c b/drivers/usb/core/sysfs.c
index dfc68ed24db1..d930bfda4010 100644
--- a/drivers/usb/core/sysfs.c
+++ b/drivers/usb/core/sysfs.c
@@ -113,7 +113,7 @@ static ssize_t devspec_show(struct device *dev, struct device_attribute *attr,
 {
 	struct device_node *of_node = dev->of_node;
 
-	return sprintf(buf, "%s\n", of_node_full_name(of_node));
+	return sprintf(buf, "%pOF\n", of_node);
 }
 static DEVICE_ATTR_RO(devspec);
 #endif
diff --git a/drivers/usb/dwc2/gadget.c b/drivers/usb/dwc2/gadget.c
index c4066cd77e47..0d8e09ccb59c 100644
--- a/drivers/usb/dwc2/gadget.c
+++ b/drivers/usb/dwc2/gadget.c
@@ -4179,7 +4179,7 @@ static int dwc2_hsotg_ep_sethalt_lock(struct usb_ep *ep, int value)
 	return ret;
 }
 
-static struct usb_ep_ops dwc2_hsotg_ep_ops = {
+static const struct usb_ep_ops dwc2_hsotg_ep_ops = {
 	.enable		= dwc2_hsotg_ep_enable,
 	.disable	= dwc2_hsotg_ep_disable,
 	.alloc_request	= dwc2_hsotg_ep_alloc_request,
diff --git a/drivers/usb/dwc2/hcd.c b/drivers/usb/dwc2/hcd.c
index 740c7e86d31b..c2631145f404 100644
--- a/drivers/usb/dwc2/hcd.c
+++ b/drivers/usb/dwc2/hcd.c
@@ -4388,6 +4388,9 @@ static int _dwc2_hcd_suspend(struct usb_hcd *hcd)
 
 	spin_lock_irqsave(&hsotg->lock, flags);
 
+	if (dwc2_is_device_mode(hsotg))
+		goto unlock;
+
 	if (hsotg->lx_state != DWC2_L0)
 		goto unlock;
 
@@ -4446,6 +4449,9 @@ static int _dwc2_hcd_resume(struct usb_hcd *hcd)
 
 	spin_lock_irqsave(&hsotg->lock, flags);
 
+	if (dwc2_is_device_mode(hsotg))
+		goto unlock;
+
 	if (hsotg->lx_state != DWC2_L2)
 		goto unlock;
 
diff --git a/drivers/usb/dwc3/dwc3-keystone.c b/drivers/usb/dwc3/dwc3-keystone.c
index 12ee23f53cdd..d2ed9523e77c 100644
--- a/drivers/usb/dwc3/dwc3-keystone.c
+++ b/drivers/usb/dwc3/dwc3-keystone.c
@@ -15,7 +15,6 @@
  * GNU General Public License for more details.
  */
 
-#include <linux/clk.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/interrupt.h>
@@ -23,6 +22,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/io.h>
 #include <linux/of_platform.h>
+#include <linux/pm_runtime.h>
 
 /* USBSS register offsets */
 #define USBSS_REVISION		0x0000
@@ -41,7 +41,6 @@
 
 struct dwc3_keystone {
 	struct device			*dev;
-	struct clk			*clk;
 	void __iomem			*usbss;
 };
 
@@ -106,17 +105,13 @@ static int kdwc3_probe(struct platform_device *pdev)
 	if (IS_ERR(kdwc->usbss))
 		return PTR_ERR(kdwc->usbss);
 
-	kdwc->clk = devm_clk_get(kdwc->dev, "usb");
-	if (IS_ERR(kdwc->clk)) {
-		dev_err(kdwc->dev, "unable to get usb clock\n");
-		return PTR_ERR(kdwc->clk);
-	}
+	pm_runtime_enable(kdwc->dev);
 
-	error = clk_prepare_enable(kdwc->clk);
+	error = pm_runtime_get_sync(kdwc->dev);
 	if (error < 0) {
-		dev_err(kdwc->dev, "unable to enable usb clock, error %d\n",
+		dev_err(kdwc->dev, "pm_runtime_get_sync failed, error %d\n",
 			error);
-		return error;
+		goto err_irq;
 	}
 
 	irq = platform_get_irq(pdev, 0);
@@ -147,7 +142,8 @@ static int kdwc3_probe(struct platform_device *pdev)
 err_core:
 	kdwc3_disable_irqs(kdwc);
 err_irq:
-	clk_disable_unprepare(kdwc->clk);
+	pm_runtime_put_sync(kdwc->dev);
+	pm_runtime_disable(kdwc->dev);
 
 	return error;
 }
@@ -167,7 +163,9 @@ static int kdwc3_remove(struct platform_device *pdev)
 
 	kdwc3_disable_irqs(kdwc);
 	device_for_each_child(&pdev->dev, NULL, kdwc3_remove_core);
-	clk_disable_unprepare(kdwc->clk);
+	pm_runtime_put_sync(kdwc->dev);
+	pm_runtime_disable(kdwc->dev);
+
 	platform_set_drvdata(pdev, NULL);
 
 	return 0;
diff --git a/drivers/usb/dwc3/dwc3-of-simple.c b/drivers/usb/dwc3/dwc3-of-simple.c
index fe414e7a9c78..4cef7d4f9cd0 100644
--- a/drivers/usb/dwc3/dwc3-of-simple.c
+++ b/drivers/usb/dwc3/dwc3-of-simple.c
@@ -25,7 +25,6 @@
 #include <linux/platform_device.h>
 #include <linux/dma-mapping.h>
 #include <linux/clk.h>
-#include <linux/clk-provider.h>
 #include <linux/of.h>
 #include <linux/of_platform.h>
 #include <linux/pm_runtime.h>
@@ -96,7 +95,8 @@ static int dwc3_of_simple_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, simple);
 	simple->dev = dev;
 
-	ret = dwc3_of_simple_clk_init(simple, of_clk_get_parent_count(np));
+	ret = dwc3_of_simple_clk_init(simple, of_count_phandle_with_args(np,
+						"clocks", "#clock-cells"));
 	if (ret)
 		return ret;
 
diff --git a/drivers/usb/dwc3/dwc3-omap.c b/drivers/usb/dwc3/dwc3-omap.c
index f5aaa0cf3873..3530795bbb8f 100644
--- a/drivers/usb/dwc3/dwc3-omap.c
+++ b/drivers/usb/dwc3/dwc3-omap.c
@@ -478,8 +478,8 @@ static int dwc3_omap_probe(struct platform_device *pdev)
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
-		dev_err(dev, "missing IRQ resource\n");
-		return -EINVAL;
+		dev_err(dev, "missing IRQ resource: %d\n", irq);
+		return irq;
 	}
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
diff --git a/drivers/usb/dwc3/dwc3-pci.c b/drivers/usb/dwc3/dwc3-pci.c
index 7e995df7a797..54343fbd85ee 100644
--- a/drivers/usb/dwc3/dwc3-pci.c
+++ b/drivers/usb/dwc3/dwc3-pci.c
@@ -345,7 +345,7 @@ static int dwc3_pci_resume(struct device *dev)
 }
 #endif /* CONFIG_PM_SLEEP */
 
-static struct dev_pm_ops dwc3_pci_dev_pm_ops = {
+static const struct dev_pm_ops dwc3_pci_dev_pm_ops = {
 	SET_SYSTEM_SLEEP_PM_OPS(dwc3_pci_suspend, dwc3_pci_resume)
 	SET_RUNTIME_PM_OPS(dwc3_pci_runtime_suspend, dwc3_pci_runtime_resume,
 		NULL)
diff --git a/drivers/usb/gadget/Kconfig b/drivers/usb/gadget/Kconfig
index 35cc641d9f31..31cce7805eb2 100644
--- a/drivers/usb/gadget/Kconfig
+++ b/drivers/usb/gadget/Kconfig
@@ -130,7 +130,7 @@ config USB_GADGET_STORAGE_NUM_BUFFERS
 
 config U_SERIAL_CONSOLE
 	bool "Serial gadget console support"
-	depends on USB_G_SERIAL
+	depends on USB_U_SERIAL
 	help
 	   It supports the serial gadget can be used as a console.
 
diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c
index d21874b35cf6..9990944a7245 100644
--- a/drivers/usb/gadget/function/f_fs.c
+++ b/drivers/usb/gadget/function/f_fs.c
@@ -961,10 +961,9 @@ static ssize_t ffs_epfile_io(struct file *file, struct ffs_io_data *io_data)
 		/* In the meantime, endpoint got disabled or changed. */
 		ret = -ESHUTDOWN;
 	} else if (halt) {
-		/* Halt */
-		if (likely(epfile->ep == ep) && !WARN_ON(!ep->ep))
-			usb_ep_set_halt(ep->ep);
-		ret = -EBADMSG;
+		ret = usb_ep_set_halt(ep->ep);
+		if (!ret)
+			ret = -EBADMSG;
 	} else if (unlikely(data_len == -EINVAL)) {
 		/*
 		 * Sanity Check: even though data_len can't be used
diff --git a/drivers/usb/gadget/function/f_hid.c b/drivers/usb/gadget/function/f_hid.c
index 5eea44823ca0..d8e359ef6eb1 100644
--- a/drivers/usb/gadget/function/f_hid.c
+++ b/drivers/usb/gadget/function/f_hid.c
@@ -44,6 +44,7 @@ struct f_hidg {
 	/* configuration */
 	unsigned char			bInterfaceSubClass;
 	unsigned char			bInterfaceProtocol;
+	unsigned char			protocol;
 	unsigned short			report_desc_length;
 	char				*report_desc;
 	unsigned short			report_length;
@@ -527,7 +528,9 @@ static int hidg_setup(struct usb_function *f,
 	case ((USB_DIR_IN | USB_TYPE_CLASS | USB_RECIP_INTERFACE) << 8
 		  | HID_REQ_GET_PROTOCOL):
 		VDBG(cdev, "get_protocol\n");
-		goto stall;
+		length = min_t(unsigned int, length, 1);
+		((u8 *) req->buf)[0] = hidg->protocol;
+		goto respond;
 		break;
 
 	case ((USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE) << 8
@@ -539,6 +542,17 @@ static int hidg_setup(struct usb_function *f,
 	case ((USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE) << 8
 		  | HID_REQ_SET_PROTOCOL):
 		VDBG(cdev, "set_protocol\n");
+		if (value > HID_REPORT_PROTOCOL)
+			goto stall;
+		length = 0;
+		/*
+		 * We assume that programs implementing the Boot protocol
+		 * are also compatible with the Report Protocol
+		 */
+		if (hidg->bInterfaceSubClass == USB_INTERFACE_SUBCLASS_BOOT) {
+			hidg->protocol = value;
+			goto respond;
+		}
 		goto stall;
 		break;
 
@@ -768,6 +782,7 @@ static int hidg_bind(struct usb_configuration *c, struct usb_function *f)
 	/* set descriptor dynamic values */
 	hidg_interface_desc.bInterfaceSubClass = hidg->bInterfaceSubClass;
 	hidg_interface_desc.bInterfaceProtocol = hidg->bInterfaceProtocol;
+	hidg->protocol = HID_REPORT_PROTOCOL;
 	hidg_ss_in_ep_desc.wMaxPacketSize = cpu_to_le16(hidg->report_length);
 	hidg_ss_in_comp_desc.wBytesPerInterval =
 				cpu_to_le16(hidg->report_length);
diff --git a/drivers/usb/gadget/function/f_mass_storage.c b/drivers/usb/gadget/function/f_mass_storage.c
index f95bddd6513f..d6bd0244b008 100644
--- a/drivers/usb/gadget/function/f_mass_storage.c
+++ b/drivers/usb/gadget/function/f_mass_storage.c
@@ -686,9 +686,8 @@ static int do_read(struct fsg_common *common)
 
 		/* Perform the read */
 		file_offset_tmp = file_offset;
-		nread = vfs_read(curlun->filp,
-				 (char __user *)bh->buf,
-				 amount, &file_offset_tmp);
+		nread = kernel_read(curlun->filp, bh->buf, amount,
+				&file_offset_tmp);
 		VLDBG(curlun, "file read %u @ %llu -> %d\n", amount,
 		      (unsigned long long)file_offset, (int)nread);
 		if (signal_pending(current))
@@ -883,8 +882,8 @@ static int do_write(struct fsg_common *common)
 
 		/* Perform the write */
 		file_offset_tmp = file_offset;
-		nwritten = vfs_write(curlun->filp, (char __user *)bh->buf,
-				amount, &file_offset_tmp);
+		nwritten = kernel_write(curlun->filp, bh->buf, amount,
+				&file_offset_tmp);
 		VLDBG(curlun, "file write %u @ %llu -> %d\n", amount,
 				(unsigned long long)file_offset, (int)nwritten);
 		if (signal_pending(current))
@@ -1021,9 +1020,8 @@ static int do_verify(struct fsg_common *common)
 
 		/* Perform the read */
 		file_offset_tmp = file_offset;
-		nread = vfs_read(curlun->filp,
-				(char __user *) bh->buf,
-				amount, &file_offset_tmp);
+		nread = kernel_read(curlun->filp, bh->buf, amount,
+				&file_offset_tmp);
 		VLDBG(curlun, "file read %u @ %llu -> %d\n", amount,
 				(unsigned long long) file_offset,
 				(int) nread);
@@ -2453,13 +2451,6 @@ static int fsg_main_thread(void *common_)
 	/* Allow the thread to be frozen */
 	set_freezable();
 
-	/*
-	 * Arrange for userspace references to be interpreted as kernel
-	 * pointers.  That way we can pass a kernel pointer to a routine
-	 * that expects a __user pointer and it will work okay.
-	 */
-	set_fs(get_ds());
-
 	/* The main loop */
 	while (common->state != FSG_STATE_TERMINATED) {
 		if (exception_in_progress(common) || signal_pending(current)) {
diff --git a/drivers/usb/gadget/function/f_midi.c b/drivers/usb/gadget/function/f_midi.c
index a5719f271bf0..5d3d7941d2c2 100644
--- a/drivers/usb/gadget/function/f_midi.c
+++ b/drivers/usb/gadget/function/f_midi.c
@@ -98,6 +98,7 @@ struct f_midi {
 	DECLARE_KFIFO_PTR(in_req_fifo, struct usb_request *);
 	spinlock_t transmit_lock;
 	unsigned int in_last_port;
+	unsigned char free_ref;
 
 	struct gmidi_in_port	in_ports_array[/* in_ports */];
 };
@@ -108,6 +109,7 @@ static inline struct f_midi *func_to_midi(struct usb_function *f)
 }
 
 static void f_midi_transmit(struct f_midi *midi);
+static void f_midi_rmidi_free(struct snd_rawmidi *rmidi);
 
 DECLARE_UAC_AC_HEADER_DESCRIPTOR(1);
 DECLARE_USB_MIDI_OUT_JACK_DESCRIPTOR(1);
@@ -163,6 +165,13 @@ static struct usb_endpoint_descriptor bulk_out_desc = {
 	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
 };
 
+static struct usb_ss_ep_comp_descriptor bulk_out_ss_comp_desc = {
+	.bLength                = sizeof(bulk_out_ss_comp_desc),
+	.bDescriptorType        = USB_DT_SS_ENDPOINT_COMP,
+	/* .bMaxBurst           = 0, */
+	/* .bmAttributes        = 0, */
+};
+
 /* B.5.2  Class-specific MS Bulk OUT Endpoint Descriptor */
 static struct usb_ms_endpoint_descriptor_16 ms_out_desc = {
 	/* .bLength =		DYNAMIC */
@@ -180,6 +189,13 @@ static struct usb_endpoint_descriptor bulk_in_desc = {
 	.bmAttributes =		USB_ENDPOINT_XFER_BULK,
 };
 
+static struct usb_ss_ep_comp_descriptor bulk_in_ss_comp_desc = {
+	.bLength                = sizeof(bulk_in_ss_comp_desc),
+	.bDescriptorType        = USB_DT_SS_ENDPOINT_COMP,
+	/* .bMaxBurst           = 0, */
+	/* .bmAttributes        = 0, */
+};
+
 /* B.6.2  Class-specific MS Bulk IN Endpoint Descriptor */
 static struct usb_ms_endpoint_descriptor_16 ms_in_desc = {
 	/* .bLength =		DYNAMIC */
@@ -755,13 +771,13 @@ static void f_midi_out_trigger(struct snd_rawmidi_substream *substream, int up)
 		clear_bit(substream->number, &midi->out_triggered);
 }
 
-static struct snd_rawmidi_ops gmidi_in_ops = {
+static const struct snd_rawmidi_ops gmidi_in_ops = {
 	.open = f_midi_in_open,
 	.close = f_midi_in_close,
 	.trigger = f_midi_in_trigger,
 };
 
-static struct snd_rawmidi_ops gmidi_out_ops = {
+static const struct snd_rawmidi_ops gmidi_out_ops = {
 	.open = f_midi_out_open,
 	.close = f_midi_out_close,
 	.trigger = f_midi_out_trigger
@@ -818,6 +834,8 @@ static int f_midi_register_card(struct f_midi *midi)
 			    SNDRV_RAWMIDI_INFO_INPUT |
 			    SNDRV_RAWMIDI_INFO_DUPLEX;
 	rmidi->private_data = midi;
+	rmidi->private_free = f_midi_rmidi_free;
+	midi->free_ref++;
 
 	/*
 	 * Yes, rawmidi OUTPUT = USB IN, and rawmidi INPUT = USB OUT.
@@ -853,7 +871,7 @@ static int f_midi_bind(struct usb_configuration *c, struct usb_function *f)
 	struct usb_composite_dev *cdev = c->cdev;
 	struct f_midi *midi = func_to_midi(f);
 	struct usb_string *us;
-	int status, n, jack = 1, i = 0;
+	int status, n, jack = 1, i = 0, endpoint_descriptor_index = 0;
 
 	midi->gadget = cdev->gadget;
 	tasklet_init(&midi->tasklet, f_midi_in_tasklet, (unsigned long) midi);
@@ -895,7 +913,7 @@ static int f_midi_bind(struct usb_configuration *c, struct usb_function *f)
 		goto fail;
 
 	/* allocate temporary function list */
-	midi_function = kcalloc((MAX_PORTS * 4) + 9, sizeof(*midi_function),
+	midi_function = kcalloc((MAX_PORTS * 4) + 11, sizeof(*midi_function),
 				GFP_KERNEL);
 	if (!midi_function) {
 		status = -ENOMEM;
@@ -985,6 +1003,7 @@ static int f_midi_bind(struct usb_configuration *c, struct usb_function *f)
 	ms_in_desc.bNumEmbMIDIJack = midi->out_ports;
 
 	/* ... and add them to the list */
+	endpoint_descriptor_index = i;
 	midi_function[i++] = (struct usb_descriptor_header *) &bulk_out_desc;
 	midi_function[i++] = (struct usb_descriptor_header *) &ms_out_desc;
 	midi_function[i++] = (struct usb_descriptor_header *) &bulk_in_desc;
@@ -1009,13 +1028,34 @@ static int f_midi_bind(struct usb_configuration *c, struct usb_function *f)
 			goto fail_f_midi;
 	}
 
+	if (gadget_is_superspeed(c->cdev->gadget)) {
+		bulk_in_desc.wMaxPacketSize = cpu_to_le16(1024);
+		bulk_out_desc.wMaxPacketSize = cpu_to_le16(1024);
+		i = endpoint_descriptor_index;
+		midi_function[i++] = (struct usb_descriptor_header *)
+				     &bulk_out_desc;
+		midi_function[i++] = (struct usb_descriptor_header *)
+				     &bulk_out_ss_comp_desc;
+		midi_function[i++] = (struct usb_descriptor_header *)
+				     &ms_out_desc;
+		midi_function[i++] = (struct usb_descriptor_header *)
+				     &bulk_in_desc;
+		midi_function[i++] = (struct usb_descriptor_header *)
+				     &bulk_in_ss_comp_desc;
+		midi_function[i++] = (struct usb_descriptor_header *)
+				     &ms_in_desc;
+		f->ss_descriptors = usb_copy_descriptors(midi_function);
+		if (!f->ss_descriptors)
+			goto fail_f_midi;
+	}
+
 	kfree(midi_function);
 
 	return 0;
 
 fail_f_midi:
 	kfree(midi_function);
-	usb_free_descriptors(f->hs_descriptors);
+	usb_free_all_descriptors(f);
 fail:
 	f_midi_unregister_card(midi);
 fail_register:
@@ -1197,14 +1237,21 @@ static void f_midi_free(struct usb_function *f)
 
 	midi = func_to_midi(f);
 	opts = container_of(f->fi, struct f_midi_opts, func_inst);
-	kfree(midi->id);
 	mutex_lock(&opts->lock);
-	kfifo_free(&midi->in_req_fifo);
-	kfree(midi);
-	--opts->refcnt;
+	if (!--midi->free_ref) {
+		kfree(midi->id);
+		kfifo_free(&midi->in_req_fifo);
+		kfree(midi);
+		--opts->refcnt;
+	}
 	mutex_unlock(&opts->lock);
 }
 
+static void f_midi_rmidi_free(struct snd_rawmidi *rmidi)
+{
+	f_midi_free(rmidi->private_data);
+}
+
 static void f_midi_unbind(struct usb_configuration *c, struct usb_function *f)
 {
 	struct usb_composite_dev *cdev = f->config->cdev;
@@ -1219,7 +1266,7 @@ static void f_midi_unbind(struct usb_configuration *c, struct usb_function *f)
 	card = midi->card;
 	midi->card = NULL;
 	if (card)
-		snd_card_free(card);
+		snd_card_free_when_closed(card);
 
 	usb_free_all_descriptors(f);
 }
@@ -1263,6 +1310,7 @@ static struct usb_function *f_midi_alloc(struct usb_function_instance *fi)
 	midi->buflen = opts->buflen;
 	midi->qlen = opts->qlen;
 	midi->in_last_port = 0;
+	midi->free_ref = 1;
 
 	status = kfifo_alloc(&midi->in_req_fifo, midi->qlen, GFP_KERNEL);
 	if (status)
diff --git a/drivers/usb/gadget/function/f_ncm.c b/drivers/usb/gadget/function/f_ncm.c
index 24e34cfcb4bd..45b334ceaf2e 100644
--- a/drivers/usb/gadget/function/f_ncm.c
+++ b/drivers/usb/gadget/function/f_ncm.c
@@ -925,8 +925,6 @@ static int ncm_set_alt(struct usb_function *f, unsigned intf, unsigned alt)
 			 */
 			ncm->port.is_zlp_ok =
 				gadget_is_zlp_supported(cdev->gadget);
-			ncm->port.no_skb_reserve =
-				gadget_avoids_skb_reserve(cdev->gadget);
 			ncm->port.cdc_filter = DEFAULT_FILTER;
 			DBG(cdev, "activate ncm\n");
 			net = gether_connect(&ncm->port);
diff --git a/drivers/usb/gadget/function/f_rndis.c b/drivers/usb/gadget/function/f_rndis.c
index 16562e461121..e1d5853ef1e4 100644
--- a/drivers/usb/gadget/function/f_rndis.c
+++ b/drivers/usb/gadget/function/f_rndis.c
@@ -691,6 +691,10 @@ rndis_bind(struct usb_configuration *c, struct usb_function *f)
 		f->os_desc_table[0].os_desc = &rndis_opts->rndis_os_desc;
 	}
 
+	rndis_iad_descriptor.bFunctionClass = rndis_opts->class;
+	rndis_iad_descriptor.bFunctionSubClass = rndis_opts->subclass;
+	rndis_iad_descriptor.bFunctionProtocol = rndis_opts->protocol;
+
 	/*
 	 * in drivers/usb/gadget/configfs.c:configfs_composite_bind()
 	 * configurations are bound in sequence with list_for_each_entry,
@@ -866,11 +870,23 @@ USB_ETHERNET_CONFIGFS_ITEM_ATTR_QMULT(rndis);
 /* f_rndis_opts_ifname */
 USB_ETHERNET_CONFIGFS_ITEM_ATTR_IFNAME(rndis);
 
+/* f_rndis_opts_class */
+USB_ETHER_CONFIGFS_ITEM_ATTR_U8_RW(rndis, class);
+
+/* f_rndis_opts_subclass */
+USB_ETHER_CONFIGFS_ITEM_ATTR_U8_RW(rndis, subclass);
+
+/* f_rndis_opts_protocol */
+USB_ETHER_CONFIGFS_ITEM_ATTR_U8_RW(rndis, protocol);
+
 static struct configfs_attribute *rndis_attrs[] = {
 	&rndis_opts_attr_dev_addr,
 	&rndis_opts_attr_host_addr,
 	&rndis_opts_attr_qmult,
 	&rndis_opts_attr_ifname,
+	&rndis_opts_attr_class,
+	&rndis_opts_attr_subclass,
+	&rndis_opts_attr_protocol,
 	NULL,
 };
 
@@ -916,6 +932,10 @@ static struct usb_function_instance *rndis_alloc_inst(void)
 	}
 	INIT_LIST_HEAD(&opts->rndis_os_desc.ext_prop);
 
+	opts->class = rndis_iad_descriptor.bFunctionClass;
+	opts->subclass = rndis_iad_descriptor.bFunctionSubClass;
+	opts->protocol = rndis_iad_descriptor.bFunctionProtocol;
+
 	descs[0] = &opts->rndis_os_desc;
 	names[0] = "rndis";
 	config_group_init_type_name(&opts->func_inst.group, "",
diff --git a/drivers/usb/gadget/function/u_audio.c b/drivers/usb/gadget/function/u_audio.c
index 5dd73b9e5172..3971bbab88bd 100644
--- a/drivers/usb/gadget/function/u_audio.c
+++ b/drivers/usb/gadget/function/u_audio.c
@@ -79,7 +79,7 @@ struct snd_uac_chip {
 	unsigned int p_framesize;
 };
 
-static struct snd_pcm_hardware uac_pcm_hardware = {
+static const struct snd_pcm_hardware uac_pcm_hardware = {
 	.info = SNDRV_PCM_INFO_INTERLEAVED | SNDRV_PCM_INFO_BLOCK_TRANSFER
 		 | SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID
 		 | SNDRV_PCM_INFO_PAUSE | SNDRV_PCM_INFO_RESUME,
@@ -354,7 +354,7 @@ static int uac_pcm_null(struct snd_pcm_substream *substream)
 	return 0;
 }
 
-static struct snd_pcm_ops uac_pcm_ops = {
+static const struct snd_pcm_ops uac_pcm_ops = {
 	.open = uac_pcm_open,
 	.close = uac_pcm_null,
 	.ioctl = snd_pcm_lib_ioctl,
diff --git a/drivers/usb/gadget/function/u_ether.c b/drivers/usb/gadget/function/u_ether.c
index a8b40d07e927..bdbc3fdc7c4f 100644
--- a/drivers/usb/gadget/function/u_ether.c
+++ b/drivers/usb/gadget/function/u_ether.c
@@ -1073,7 +1073,7 @@ struct net_device *gether_connect(struct gether *link)
 
 	if (result == 0) {
 		dev->zlp = link->is_zlp_ok;
-		dev->no_skb_reserve = link->no_skb_reserve;
+		dev->no_skb_reserve = gadget_avoids_skb_reserve(dev->gadget);
 		DBG(dev, "qlen %d\n", qlen(dev->gadget, dev->qmult));
 
 		dev->header_len = link->header_len;
diff --git a/drivers/usb/gadget/function/u_ether.h b/drivers/usb/gadget/function/u_ether.h
index 81d94a7ae4b4..c77145bd6b5b 100644
--- a/drivers/usb/gadget/function/u_ether.h
+++ b/drivers/usb/gadget/function/u_ether.h
@@ -64,7 +64,6 @@ struct gether {
 	struct usb_ep			*out_ep;
 
 	bool				is_zlp_ok;
-	bool				no_skb_reserve;
 
 	u16				cdc_filter;
 
diff --git a/drivers/usb/gadget/function/u_ether_configfs.h b/drivers/usb/gadget/function/u_ether_configfs.h
index c71133de17e7..e4c3f84af4c3 100644
--- a/drivers/usb/gadget/function/u_ether_configfs.h
+++ b/drivers/usb/gadget/function/u_ether_configfs.h
@@ -153,4 +153,39 @@ out:									\
 									\
 	CONFIGFS_ATTR_RO(_f_##_opts_, ifname)
 
+#define USB_ETHER_CONFIGFS_ITEM_ATTR_U8_RW(_f_, _n_)			\
+	static ssize_t _f_##_opts_##_n_##_show(struct config_item *item,\
+					       char *page)		\
+	{								\
+		struct f_##_f_##_opts *opts = to_f_##_f_##_opts(item);	\
+		int ret;						\
+									\
+		mutex_lock(&opts->lock);				\
+		ret = sprintf(page, "%02x\n", opts->_n_);		\
+		mutex_unlock(&opts->lock);				\
+									\
+		return ret;						\
+	}								\
+									\
+	static ssize_t _f_##_opts_##_n_##_store(struct config_item *item,\
+						const char *page,	\
+						size_t len)		\
+	{								\
+		struct f_##_f_##_opts *opts = to_f_##_f_##_opts(item);	\
+		int ret;						\
+		u8 val;							\
+									\
+		mutex_lock(&opts->lock);				\
+		ret = sscanf(page, "%02hhx", &val);			\
+		if (ret > 0) {						\
+			opts->_n_ = val;				\
+			ret = len;					\
+		}							\
+		mutex_unlock(&opts->lock);				\
+									\
+		return ret;						\
+	}								\
+									\
+	CONFIGFS_ATTR(_f_##_opts_, _n_)
+
 #endif /* __U_ETHER_CONFIGFS_H */
diff --git a/drivers/usb/gadget/function/u_rndis.h b/drivers/usb/gadget/function/u_rndis.h
index 4eafd5050545..a35ee3c2545d 100644
--- a/drivers/usb/gadget/function/u_rndis.h
+++ b/drivers/usb/gadget/function/u_rndis.h
@@ -29,6 +29,10 @@ struct f_rndis_opts {
 	struct usb_os_desc		rndis_os_desc;
 	char				rndis_ext_compat_id[16];
 
+	u8				class;
+	u8				subclass;
+	u8				protocol;
+
 	/*
 	 * Read/write access to configfs attributes is handled by configfs.
 	 *
diff --git a/drivers/usb/gadget/function/u_serial.c b/drivers/usb/gadget/function/u_serial.c
index 9b0805f55ad7..4176216d54be 100644
--- a/drivers/usb/gadget/function/u_serial.c
+++ b/drivers/usb/gadget/function/u_serial.c
@@ -537,7 +537,7 @@ static void gs_rx_push(unsigned long _port)
 		}
 
 		/* push data to (open) tty */
-		if (req->actual) {
+		if (req->actual && tty) {
 			char		*packet = req->buf;
 			unsigned	size = req->actual;
 			unsigned	n;
diff --git a/drivers/usb/gadget/legacy/webcam.c b/drivers/usb/gadget/legacy/webcam.c
index f9661cd627c8..82c13fce9232 100644
--- a/drivers/usb/gadget/legacy/webcam.c
+++ b/drivers/usb/gadget/legacy/webcam.c
@@ -436,5 +436,4 @@ module_usb_composite_driver(webcam_driver);
 MODULE_AUTHOR("Laurent Pinchart");
 MODULE_DESCRIPTION("Webcam Video Gadget");
 MODULE_LICENSE("GPL");
-MODULE_VERSION("0.1.0");
 
diff --git a/drivers/usb/gadget/udc/bdc/Kconfig b/drivers/usb/gadget/udc/bdc/Kconfig
index eb8b55392360..c74ac25dddcd 100644
--- a/drivers/usb/gadget/udc/bdc/Kconfig
+++ b/drivers/usb/gadget/udc/bdc/Kconfig
@@ -1,6 +1,7 @@
 config USB_BDC_UDC
 	tristate "Broadcom USB3.0 device controller IP driver(BDC)"
 	depends on USB_GADGET && HAS_DMA
+	default ARCH_BRCMSTB
 
 	help
 	BDC is Broadcom's USB3.0 device controller IP. If your SOC has a BDC IP
diff --git a/drivers/usb/gadget/udc/bdc/bdc.h b/drivers/usb/gadget/udc/bdc/bdc.h
index 916d47135cac..6df0352cdc50 100644
--- a/drivers/usb/gadget/udc/bdc/bdc.h
+++ b/drivers/usb/gadget/udc/bdc/bdc.h
@@ -27,8 +27,8 @@
 #include <linux/usb/gadget.h>
 #include <asm/unaligned.h>
 
-#define BRCM_BDC_NAME "bdc_usb3"
-#define BRCM_BDC_DESC "BDC device controller driver"
+#define BRCM_BDC_NAME "bdc"
+#define BRCM_BDC_DESC "Broadcom USB Device Controller driver"
 
 #define DMA_ADDR_INVALID        (~(dma_addr_t)0)
 
@@ -83,14 +83,14 @@
 
 #define BDC_DVCSA	0x50
 #define BDC_DVCSB	0x54
-#define BDC_EPSTS0(n)	(0x60 + (n * 0x10))
-#define BDC_EPSTS1(n)	(0x64 + (n * 0x10))
-#define BDC_EPSTS2(n)	(0x68 + (n * 0x10))
-#define BDC_EPSTS3(n)	(0x6c + (n * 0x10))
-#define BDC_EPSTS4(n)	(0x70 + (n * 0x10))
-#define BDC_EPSTS5(n)	(0x74 + (n * 0x10))
-#define BDC_EPSTS6(n)	(0x78 + (n * 0x10))
-#define BDC_EPSTS7(n)	(0x7c + (n * 0x10))
+#define BDC_EPSTS0	0x60
+#define BDC_EPSTS1	0x64
+#define BDC_EPSTS2	0x68
+#define BDC_EPSTS3	0x6c
+#define BDC_EPSTS4	0x70
+#define BDC_EPSTS5	0x74
+#define BDC_EPSTS6	0x78
+#define BDC_EPSTS7	0x7c
 #define BDC_SRRBAL(n)	(0x200 + (n * 0x10))
 #define BDC_SRRBAH(n)	(0x204 + (n * 0x10))
 #define BDC_SRRINT(n)	(0x208 + (n * 0x10))
@@ -413,6 +413,9 @@ struct bdc {
 	/* device lock */
 	spinlock_t	lock;
 
+	/* generic phy */
+	struct phy      **phys;
+	int num_phys;
 	/* num of endpoints for a particular instantiation of IP */
 	unsigned int num_eps;
 	/*
@@ -454,6 +457,7 @@ struct bdc {
 	 * Func Wake packet every 2.5 secs. Refer to USB3 spec section 8.5.6.4
 	 */
 	struct delayed_work	func_wake_notify;
+	struct clk		*clk;
 };
 
 static inline u32 bdc_readl(void __iomem *base, u32 offset)
diff --git a/drivers/usb/gadget/udc/bdc/bdc_core.c b/drivers/usb/gadget/udc/bdc/bdc_core.c
index e9bd8d4abca0..7a8af4b916cf 100644
--- a/drivers/usb/gadget/udc/bdc/bdc_core.c
+++ b/drivers/usb/gadget/udc/bdc/bdc_core.c
@@ -24,9 +24,11 @@
 #include <linux/dma-mapping.h>
 #include <linux/dmapool.h>
 #include <linux/of.h>
+#include <linux/phy/phy.h>
 #include <linux/moduleparam.h>
 #include <linux/usb/ch9.h>
 #include <linux/usb/gadget.h>
+#include <linux/clk.h>
 
 #include "bdc.h"
 #include "bdc_dbg.h"
@@ -444,6 +446,43 @@ static int bdc_hw_init(struct bdc *bdc)
 	return 0;
 }
 
+static int bdc_phy_init(struct bdc *bdc)
+{
+	int phy_num;
+	int ret;
+
+	for (phy_num = 0; phy_num < bdc->num_phys; phy_num++) {
+		ret = phy_init(bdc->phys[phy_num]);
+		if (ret)
+			goto err_exit_phy;
+		ret = phy_power_on(bdc->phys[phy_num]);
+		if (ret) {
+			phy_exit(bdc->phys[phy_num]);
+			goto err_exit_phy;
+		}
+	}
+
+	return 0;
+
+err_exit_phy:
+	while (--phy_num >= 0) {
+		phy_power_off(bdc->phys[phy_num]);
+		phy_exit(bdc->phys[phy_num]);
+	}
+
+	return ret;
+}
+
+static void bdc_phy_exit(struct bdc *bdc)
+{
+	int phy_num;
+
+	for (phy_num = 0; phy_num < bdc->num_phys; phy_num++) {
+		phy_power_off(bdc->phys[phy_num]);
+		phy_exit(bdc->phys[phy_num]);
+	}
+}
+
 static int bdc_probe(struct platform_device *pdev)
 {
 	struct bdc *bdc;
@@ -452,12 +491,29 @@ static int bdc_probe(struct platform_device *pdev)
 	int irq;
 	u32 temp;
 	struct device *dev = &pdev->dev;
+	struct clk *clk;
+	int phy_num;
 
 	dev_dbg(dev, "%s()\n", __func__);
+
+	clk = devm_clk_get(dev, "sw_usbd");
+	if (IS_ERR(clk)) {
+		dev_info(dev, "Clock not found in Device Tree\n");
+		clk = NULL;
+	}
+
+	ret = clk_prepare_enable(clk);
+	if (ret) {
+		dev_err(dev, "could not enable clock\n");
+		return ret;
+	}
+
 	bdc = devm_kzalloc(dev, sizeof(*bdc), GFP_KERNEL);
 	if (!bdc)
 		return -ENOMEM;
 
+	bdc->clk = clk;
+
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	bdc->regs = devm_ioremap_resource(dev, res);
 	if (IS_ERR(bdc->regs)) {
@@ -473,35 +529,66 @@ static int bdc_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, bdc);
 	bdc->irq = irq;
 	bdc->dev = dev;
-	dev_dbg(bdc->dev, "bdc->regs: %p irq=%d\n", bdc->regs, bdc->irq);
+	dev_dbg(dev, "bdc->regs: %p irq=%d\n", bdc->regs, bdc->irq);
+
+	bdc->num_phys = of_count_phandle_with_args(dev->of_node,
+						"phys", "#phy-cells");
+	if (bdc->num_phys > 0) {
+		bdc->phys = devm_kcalloc(dev, bdc->num_phys,
+					sizeof(struct phy *), GFP_KERNEL);
+		if (!bdc->phys)
+			return -ENOMEM;
+	} else {
+		bdc->num_phys = 0;
+	}
+	dev_info(dev, "Using %d phy(s)\n", bdc->num_phys);
+
+	for (phy_num = 0; phy_num < bdc->num_phys; phy_num++) {
+		bdc->phys[phy_num] = devm_of_phy_get_by_index(
+			dev, dev->of_node, phy_num);
+		if (IS_ERR(bdc->phys[phy_num])) {
+			ret = PTR_ERR(bdc->phys[phy_num]);
+			dev_err(bdc->dev,
+				"BDC phy specified but not found:%d\n", ret);
+			return ret;
+		}
+	}
+
+	ret = bdc_phy_init(bdc);
+	if (ret) {
+		dev_err(bdc->dev, "BDC phy init failure:%d\n", ret);
+		return ret;
+	}
 
 	temp = bdc_readl(bdc->regs, BDC_BDCCAP1);
 	if ((temp & BDC_P64) &&
 			!dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64))) {
-		dev_dbg(bdc->dev, "Using 64-bit address\n");
+		dev_dbg(dev, "Using 64-bit address\n");
 	} else {
-		ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
+		ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
 		if (ret) {
-			dev_err(bdc->dev, "No suitable DMA config available, abort\n");
+			dev_err(dev,
+				"No suitable DMA config available, abort\n");
 			return -ENOTSUPP;
 		}
-		dev_dbg(bdc->dev, "Using 32-bit address\n");
+		dev_dbg(dev, "Using 32-bit address\n");
 	}
 	ret = bdc_hw_init(bdc);
 	if (ret) {
-		dev_err(bdc->dev, "BDC init failure:%d\n", ret);
-		return ret;
+		dev_err(dev, "BDC init failure:%d\n", ret);
+		goto phycleanup;
 	}
 	ret = bdc_udc_init(bdc);
 	if (ret) {
-		dev_err(bdc->dev, "BDC Gadget init failure:%d\n", ret);
+		dev_err(dev, "BDC Gadget init failure:%d\n", ret);
 		goto cleanup;
 	}
 	return 0;
 
 cleanup:
 	bdc_hw_exit(bdc);
-
+phycleanup:
+	bdc_phy_exit(bdc);
 	return ret;
 }
 
@@ -513,13 +600,56 @@ static int bdc_remove(struct platform_device *pdev)
 	dev_dbg(bdc->dev, "%s ()\n", __func__);
 	bdc_udc_exit(bdc);
 	bdc_hw_exit(bdc);
+	bdc_phy_exit(bdc);
+	clk_disable_unprepare(bdc->clk);
+	return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int bdc_suspend(struct device *dev)
+{
+	struct bdc *bdc = dev_get_drvdata(dev);
 
+	clk_disable_unprepare(bdc->clk);
 	return 0;
 }
 
+static int bdc_resume(struct device *dev)
+{
+	struct bdc *bdc = dev_get_drvdata(dev);
+	int ret;
+
+	ret = clk_prepare_enable(bdc->clk);
+	if (ret) {
+		dev_err(bdc->dev, "err enabling the clock\n");
+		return ret;
+	}
+	ret = bdc_reinit(bdc);
+	if (ret) {
+		dev_err(bdc->dev, "err in bdc reinit\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+#endif /* CONFIG_PM_SLEEP */
+
+static SIMPLE_DEV_PM_OPS(bdc_pm_ops, bdc_suspend,
+		bdc_resume);
+
+static const struct of_device_id bdc_of_match[] = {
+	{ .compatible = "brcm,bdc-v0.16" },
+	{ .compatible = "brcm,bdc" },
+	{ /* sentinel */ }
+};
+
 static struct platform_driver bdc_driver = {
 	.driver		= {
 		.name	= BRCM_BDC_NAME,
+		.owner	= THIS_MODULE,
+		.pm = &bdc_pm_ops,
+		.of_match_table	= bdc_of_match,
 	},
 	.probe		= bdc_probe,
 	.remove		= bdc_remove,
diff --git a/drivers/usb/gadget/udc/bdc/bdc_dbg.c b/drivers/usb/gadget/udc/bdc/bdc_dbg.c
index 5945dbc47825..ac98f6f681b7 100644
--- a/drivers/usb/gadget/udc/bdc/bdc_dbg.c
+++ b/drivers/usb/gadget/udc/bdc/bdc_dbg.c
@@ -40,28 +40,28 @@ void bdc_dump_epsts(struct bdc *bdc)
 {
 	u32 temp;
 
-	temp = bdc_readl(bdc->regs, BDC_EPSTS0(0));
+	temp = bdc_readl(bdc->regs, BDC_EPSTS0);
 	dev_vdbg(bdc->dev, "BDC_EPSTS0:0x%08x\n", temp);
 
-	temp = bdc_readl(bdc->regs, BDC_EPSTS1(0));
+	temp = bdc_readl(bdc->regs, BDC_EPSTS1);
 	dev_vdbg(bdc->dev, "BDC_EPSTS1:0x%x\n", temp);
 
-	temp = bdc_readl(bdc->regs, BDC_EPSTS2(0));
+	temp = bdc_readl(bdc->regs, BDC_EPSTS2);
 	dev_vdbg(bdc->dev, "BDC_EPSTS2:0x%08x\n", temp);
 
-	temp = bdc_readl(bdc->regs, BDC_EPSTS3(0));
+	temp = bdc_readl(bdc->regs, BDC_EPSTS3);
 	dev_vdbg(bdc->dev, "BDC_EPSTS3:0x%08x\n", temp);
 
-	temp = bdc_readl(bdc->regs, BDC_EPSTS4(0));
+	temp = bdc_readl(bdc->regs, BDC_EPSTS4);
 	dev_vdbg(bdc->dev, "BDC_EPSTS4:0x%08x\n", temp);
 
-	temp = bdc_readl(bdc->regs, BDC_EPSTS5(0));
+	temp = bdc_readl(bdc->regs, BDC_EPSTS5);
 	dev_vdbg(bdc->dev, "BDC_EPSTS5:0x%08x\n", temp);
 
-	temp = bdc_readl(bdc->regs, BDC_EPSTS6(0));
+	temp = bdc_readl(bdc->regs, BDC_EPSTS6);
 	dev_vdbg(bdc->dev, "BDC_EPSTS6:0x%08x\n", temp);
 
-	temp = bdc_readl(bdc->regs, BDC_EPSTS7(0));
+	temp = bdc_readl(bdc->regs, BDC_EPSTS7);
 	dev_vdbg(bdc->dev, "BDC_EPSTS7:0x%08x\n", temp);
 }
 
diff --git a/drivers/usb/gadget/udc/bdc/bdc_ep.c b/drivers/usb/gadget/udc/bdc/bdc_ep.c
index ff1ef24d1777..bfd8f7ade935 100644
--- a/drivers/usb/gadget/udc/bdc/bdc_ep.c
+++ b/drivers/usb/gadget/udc/bdc/bdc_ep.c
@@ -777,9 +777,9 @@ static int ep_dequeue(struct bdc_ep *ep, struct bdc_req *req)
 	 */
 
 	/* The current hw dequeue pointer */
-	tmp_32 = bdc_readl(bdc->regs, BDC_EPSTS0(0));
+	tmp_32 = bdc_readl(bdc->regs, BDC_EPSTS0);
 	deq_ptr_64 = tmp_32;
-	tmp_32 = bdc_readl(bdc->regs, BDC_EPSTS1(0));
+	tmp_32 = bdc_readl(bdc->regs, BDC_EPSTS1);
 	deq_ptr_64 |= ((u64)tmp_32 << 32);
 
 	/* we have the dma addr of next bd that will be fetched by hardware */
diff --git a/drivers/usb/gadget/udc/bdc/bdc_udc.c b/drivers/usb/gadget/udc/bdc/bdc_udc.c
index aae7458d8986..c84346146456 100644
--- a/drivers/usb/gadget/udc/bdc/bdc_udc.c
+++ b/drivers/usb/gadget/udc/bdc/bdc_udc.c
@@ -249,6 +249,7 @@ void bdc_sr_uspc(struct bdc *bdc, struct bdc_sr *sreport)
 			disconn = true;
 		else if ((uspc & BDC_PCS) && !BDC_PST(uspc))
 			connected = true;
+		clear_flags |= BDC_PCC;
 	}
 
 	/* Change in VBus and VBus is present */
@@ -259,16 +260,16 @@ void bdc_sr_uspc(struct bdc *bdc, struct bdc_sr *sreport)
 			bdc_softconn(bdc);
 			usb_gadget_set_state(&bdc->gadget, USB_STATE_POWERED);
 		}
-		clear_flags = BDC_VBC;
+		clear_flags |= BDC_VBC;
 	} else if ((uspc & BDC_PRS) || (uspc & BDC_PRC) || disconn) {
 		/* Hot reset, warm reset, 2.0 bus reset or disconn */
 		dev_dbg(bdc->dev, "Port reset or disconn\n");
 		bdc_uspc_disconnected(bdc, disconn);
-		clear_flags = BDC_PCC|BDC_PCS|BDC_PRS|BDC_PRC;
+		clear_flags |= BDC_PRC;
 	} else if ((uspc & BDC_PSC) && (uspc & BDC_PCS)) {
 		/* Change in Link state */
 		handle_link_state_change(bdc, uspc);
-		clear_flags = BDC_PSC|BDC_PCS;
+		clear_flags |= BDC_PSC;
 	}
 
 	/*
diff --git a/drivers/usb/gadget/udc/core.c b/drivers/usb/gadget/udc/core.c
index e6f04eee95c4..75c51ca4ee0f 100644
--- a/drivers/usb/gadget/udc/core.c
+++ b/drivers/usb/gadget/udc/core.c
@@ -812,6 +812,8 @@ int usb_gadget_map_request_by_dev(struct device *dev,
 			dev_err(dev, "failed to map buffer\n");
 			return -EFAULT;
 		}
+
+		req->dma_mapped = 1;
 	}
 
 	return 0;
@@ -836,9 +838,10 @@ void usb_gadget_unmap_request_by_dev(struct device *dev,
 				is_in ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
 
 		req->num_mapped_sgs = 0;
-	} else {
+	} else if (req->dma_mapped) {
 		dma_unmap_single(dev, req->dma, req->length,
 				is_in ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
+		req->dma_mapped = 0;
 	}
 }
 EXPORT_SYMBOL_GPL(usb_gadget_unmap_request_by_dev);
@@ -1130,6 +1133,7 @@ static int check_pending_gadget_drivers(struct usb_udc *udc)
  * @release: a gadget release function.
  *
  * Returns zero on success, negative errno otherwise.
+ * Calls the gadget release function in the latter case.
  */
 int usb_add_gadget_udc_release(struct device *parent, struct usb_gadget *gadget,
 		void (*release)(struct device *dev))
@@ -1137,10 +1141,6 @@ int usb_add_gadget_udc_release(struct device *parent, struct usb_gadget *gadget,
 	struct usb_udc		*udc;
 	int			ret = -ENOMEM;
 
-	udc = kzalloc(sizeof(*udc), GFP_KERNEL);
-	if (!udc)
-		goto err1;
-
 	dev_set_name(&gadget->dev, "gadget");
 	INIT_WORK(&gadget->work, usb_gadget_state_work);
 	gadget->dev.parent = parent;
@@ -1150,7 +1150,13 @@ int usb_add_gadget_udc_release(struct device *parent, struct usb_gadget *gadget,
 	else
 		gadget->dev.release = usb_udc_nop_release;
 
-	ret = device_register(&gadget->dev);
+	device_initialize(&gadget->dev);
+
+	udc = kzalloc(sizeof(*udc), GFP_KERNEL);
+	if (!udc)
+		goto err1;
+
+	ret = device_add(&gadget->dev);
 	if (ret)
 		goto err2;
 
@@ -1197,10 +1203,10 @@ err3:
 	device_del(&gadget->dev);
 
 err2:
-	put_device(&gadget->dev);
 	kfree(udc);
 
 err1:
+	put_device(&gadget->dev);
 	return ret;
 }
 EXPORT_SYMBOL_GPL(usb_add_gadget_udc_release);
diff --git a/drivers/usb/gadget/udc/dummy_hcd.c b/drivers/usb/gadget/udc/dummy_hcd.c
index 3c3760315910..a030d7923d7d 100644
--- a/drivers/usb/gadget/udc/dummy_hcd.c
+++ b/drivers/usb/gadget/udc/dummy_hcd.c
@@ -2776,7 +2776,7 @@ static int __init init(void)
 		if (retval < 0) {
 			i--;
 			while (i >= 0)
-				platform_device_del(the_udc_pdev[i]);
+				platform_device_del(the_udc_pdev[i--]);
 			goto err_add_udc;
 		}
 	}
diff --git a/drivers/usb/gadget/udc/fsl_qe_udc.c b/drivers/usb/gadget/udc/fsl_qe_udc.c
index 303328ce59ee..a3e72d690eef 100644
--- a/drivers/usb/gadget/udc/fsl_qe_udc.c
+++ b/drivers/usb/gadget/udc/fsl_qe_udc.c
@@ -62,7 +62,7 @@ static const char *const ep_name[] = {
 	"ep3",
 };
 
-static struct usb_endpoint_descriptor qe_ep0_desc = {
+static const struct usb_endpoint_descriptor qe_ep0_desc = {
 	.bLength =		USB_DT_ENDPOINT_SIZE,
 	.bDescriptorType =	USB_DT_ENDPOINT,
 
diff --git a/drivers/usb/gadget/udc/mv_udc_core.c b/drivers/usb/gadget/udc/mv_udc_core.c
index 8a708d0a1042..4103bf7cf52a 100644
--- a/drivers/usb/gadget/udc/mv_udc_core.c
+++ b/drivers/usb/gadget/udc/mv_udc_core.c
@@ -39,7 +39,6 @@
 #include "mv_udc.h"
 
 #define DRIVER_DESC		"Marvell PXA USB Device Controller driver"
-#define DRIVER_VERSION		"8 Nov 2010"
 
 #define ep_dir(ep)	(((ep)->ep_num == 0) ? \
 				((ep)->udc->ep0_dir) : ((ep)->direction))
@@ -2427,5 +2426,4 @@ module_platform_driver(udc_driver);
 MODULE_ALIAS("platform:mv-udc");
 MODULE_DESCRIPTION(DRIVER_DESC);
 MODULE_AUTHOR("Chao Xie <chao.xie@marvell.com>");
-MODULE_VERSION(DRIVER_VERSION);
 MODULE_LICENSE("GPL");
diff --git a/drivers/usb/gadget/udc/renesas_usb3.c b/drivers/usb/gadget/udc/renesas_usb3.c
index e1de8fe599a3..df37c1e6e9d5 100644
--- a/drivers/usb/gadget/udc/renesas_usb3.c
+++ b/drivers/usb/gadget/udc/renesas_usb3.c
@@ -8,6 +8,7 @@
  * the Free Software Foundation; version 2 of the License.
  */
 
+#include <linux/debugfs.h>
 #include <linux/delay.h>
 #include <linux/dma-mapping.h>
 #include <linux/err.h>
@@ -20,6 +21,8 @@
 #include <linux/pm_runtime.h>
 #include <linux/sizes.h>
 #include <linux/slab.h>
+#include <linux/sys_soc.h>
+#include <linux/uaccess.h>
 #include <linux/usb/ch9.h>
 #include <linux/usb/gadget.h>
 
@@ -347,6 +350,7 @@ struct renesas_usb3 {
 	bool workaround_for_vbus;
 	bool extcon_host;		/* check id and set EXTCON_USB_HOST */
 	bool extcon_usb;		/* check vbus and set EXTCON_USB */
+	bool forced_b_device;
 };
 
 #define gadget_to_renesas_usb3(_gadget)	\
@@ -663,7 +667,9 @@ static void usb3_mode_config(struct renesas_usb3 *usb3, bool host, bool a_dev)
 	spin_lock_irqsave(&usb3->lock, flags);
 	usb3_set_mode(usb3, host);
 	usb3_vbus_out(usb3, a_dev);
-	if (!host && a_dev)		/* for A-Peripheral */
+	/* for A-Peripheral or forced B-device mode */
+	if ((!host && a_dev) ||
+	    (usb3->workaround_for_vbus && usb3->forced_b_device))
 		usb3_connect(usb3);
 	spin_unlock_irqrestore(&usb3->lock, flags);
 }
@@ -677,7 +683,7 @@ static void usb3_check_id(struct renesas_usb3 *usb3)
 {
 	usb3->extcon_host = usb3_is_a_device(usb3);
 
-	if (usb3->extcon_host)
+	if (usb3->extcon_host && !usb3->forced_b_device)
 		usb3_mode_config(usb3, true, true);
 	else
 		usb3_mode_config(usb3, false, false);
@@ -2192,7 +2198,7 @@ static void renesas_usb3_ep_fifo_flush(struct usb_ep *_ep)
 	}
 }
 
-static struct usb_ep_ops renesas_usb3_ep_ops = {
+static const struct usb_ep_ops renesas_usb3_ep_ops = {
 	.enable		= renesas_usb3_ep_enable,
 	.disable	= renesas_usb3_ep_disable,
 
@@ -2283,6 +2289,9 @@ static ssize_t role_store(struct device *dev, struct device_attribute *attr,
 	if (!usb3->driver)
 		return -ENODEV;
 
+	if (usb3->forced_b_device)
+		return -EBUSY;
+
 	if (!strncmp(buf, "host", strlen("host")))
 		new_mode_is_host = true;
 	else if (!strncmp(buf, "peripheral", strlen("peripheral")))
@@ -2310,6 +2319,70 @@ static ssize_t role_show(struct device *dev, struct device_attribute *attr,
 }
 static DEVICE_ATTR_RW(role);
 
+static int renesas_usb3_b_device_show(struct seq_file *s, void *unused)
+{
+	struct renesas_usb3 *usb3 = s->private;
+
+	seq_printf(s, "%d\n", usb3->forced_b_device);
+
+	return 0;
+}
+
+static int renesas_usb3_b_device_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, renesas_usb3_b_device_show, inode->i_private);
+}
+
+static ssize_t renesas_usb3_b_device_write(struct file *file,
+					   const char __user *ubuf,
+					   size_t count, loff_t *ppos)
+{
+	struct seq_file *s = file->private_data;
+	struct renesas_usb3 *usb3 = s->private;
+	char buf[32];
+
+	if (!usb3->driver)
+		return -ENODEV;
+
+	if (copy_from_user(&buf, ubuf, min_t(size_t, sizeof(buf) - 1, count)))
+		return -EFAULT;
+
+	if (!strncmp(buf, "1", 1))
+		usb3->forced_b_device = true;
+	else
+		usb3->forced_b_device = false;
+
+	/* Let this driver call usb3_connect() anyway */
+	usb3_check_id(usb3);
+
+	return count;
+}
+
+static const struct file_operations renesas_usb3_b_device_fops = {
+	.open = renesas_usb3_b_device_open,
+	.write = renesas_usb3_b_device_write,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+static void renesas_usb3_debugfs_init(struct renesas_usb3 *usb3,
+				      struct device *dev)
+{
+	struct dentry *root, *file;
+
+	root = debugfs_create_dir(dev_name(dev), NULL);
+	if (IS_ERR_OR_NULL(root)) {
+		dev_info(dev, "%s: Can't create the root\n", __func__);
+		return;
+	}
+
+	file = debugfs_create_file("b_device", 0644, root, usb3,
+				   &renesas_usb3_b_device_fops);
+	if (!file)
+		dev_info(dev, "%s: Can't create debugfs mode\n", __func__);
+}
+
 /*------- platform_driver ------------------------------------------------*/
 static int renesas_usb3_remove(struct platform_device *pdev)
 {
@@ -2432,22 +2505,40 @@ static void renesas_usb3_init_ram(struct renesas_usb3 *usb3, struct device *dev,
 	}
 }
 
-static const struct renesas_usb3_priv renesas_usb3_priv_r8a7795 = {
+static const struct renesas_usb3_priv renesas_usb3_priv_r8a7795_es1 = {
 	.ramsize_per_ramif = SZ_16K,
 	.num_ramif = 2,
 	.ramsize_per_pipe = SZ_4K,
 	.workaround_for_vbus = true,
 };
 
+static const struct renesas_usb3_priv renesas_usb3_priv_gen3 = {
+	.ramsize_per_ramif = SZ_16K,
+	.num_ramif = 4,
+	.ramsize_per_pipe = SZ_4K,
+};
+
 static const struct of_device_id usb3_of_match[] = {
 	{
 		.compatible = "renesas,r8a7795-usb3-peri",
-		.data = &renesas_usb3_priv_r8a7795,
+		.data = &renesas_usb3_priv_gen3,
+	},
+	{
+		.compatible = "renesas,rcar-gen3-usb3-peri",
+		.data = &renesas_usb3_priv_gen3,
 	},
 	{ },
 };
 MODULE_DEVICE_TABLE(of, usb3_of_match);
 
+static const struct soc_device_attribute renesas_usb3_quirks_match[] = {
+	{
+		.soc_id = "r8a7795", .revision = "ES1.*",
+		.data = &renesas_usb3_priv_r8a7795_es1,
+	},
+	{ /* sentinel */ },
+};
+
 static const unsigned int renesas_usb3_cable[] = {
 	EXTCON_USB,
 	EXTCON_USB_HOST,
@@ -2461,15 +2552,23 @@ static int renesas_usb3_probe(struct platform_device *pdev)
 	const struct of_device_id *match;
 	int irq, ret;
 	const struct renesas_usb3_priv *priv;
+	const struct soc_device_attribute *attr;
 
 	match = of_match_node(usb3_of_match, pdev->dev.of_node);
 	if (!match)
 		return -ENODEV;
-	priv = match->data;
+
+	attr = soc_device_match(renesas_usb3_quirks_match);
+	if (attr)
+		priv = attr->data;
+	else
+		priv = match->data;
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0)
-		return -ENODEV;
+	if (irq < 0) {
+		dev_err(&pdev->dev, "Failed to get IRQ: %d\n", irq);
+		return irq;
+	}
 
 	usb3 = devm_kzalloc(&pdev->dev, sizeof(*usb3), GFP_KERNEL);
 	if (!usb3)
@@ -2527,6 +2626,8 @@ static int renesas_usb3_probe(struct platform_device *pdev)
 
 	usb3->workaround_for_vbus = priv->workaround_for_vbus;
 
+	renesas_usb3_debugfs_init(usb3, &pdev->dev);
+
 	dev_info(&pdev->dev, "probed\n");
 
 	return 0;
diff --git a/drivers/usb/gadget/udc/s3c2410_udc.c b/drivers/usb/gadget/udc/s3c2410_udc.c
index 4643a01262b4..394abd5d65c0 100644
--- a/drivers/usb/gadget/udc/s3c2410_udc.c
+++ b/drivers/usb/gadget/udc/s3c2410_udc.c
@@ -51,7 +51,6 @@
 #include "s3c2410_udc.h"
 
 #define DRIVER_DESC	"S3C2410 USB Device Controller Gadget"
-#define DRIVER_VERSION	"29 Apr 2007"
 #define DRIVER_AUTHOR	"Herbert Pötzl <herbert@13thfloor.at>, " \
 			"Arnaud Patard <arnaud.patard@rtp-net.org>"
 
@@ -1996,7 +1995,7 @@ static int __init udc_init(void)
 {
 	int retval;
 
-	dprintk(DEBUG_NORMAL, "%s: version %s\n", gadget_name, DRIVER_VERSION);
+	dprintk(DEBUG_NORMAL, "%s\n", gadget_name);
 
 	s3c2410_udc_debugfs_root = debugfs_create_dir(gadget_name, NULL);
 	if (IS_ERR(s3c2410_udc_debugfs_root)) {
@@ -2027,5 +2026,4 @@ module_exit(udc_exit);
 
 MODULE_AUTHOR(DRIVER_AUTHOR);
 MODULE_DESCRIPTION(DRIVER_DESC);
-MODULE_VERSION(DRIVER_VERSION);
 MODULE_LICENSE("GPL");
diff --git a/drivers/usb/gadget/udc/snps_udc_plat.c b/drivers/usb/gadget/udc/snps_udc_plat.c
index f7b4d0f159e4..e8a5fdaee37d 100644
--- a/drivers/usb/gadget/udc/snps_udc_plat.c
+++ b/drivers/usb/gadget/udc/snps_udc_plat.c
@@ -184,7 +184,7 @@ static int udc_plat_probe(struct platform_device *pdev)
 			goto exit_phy;
 		}
 
-		ret = extcon_get_cable_state_(udc->edev, EXTCON_USB);
+		ret = extcon_get_state(udc->edev, EXTCON_USB);
 		if (ret < 0) {
 			dev_err(dev, "Can't get cable state\n");
 			goto exit_extcon;
@@ -273,7 +273,7 @@ static int udc_plat_suspend(struct device *dev)
 	udc = dev_get_drvdata(dev);
 	stop_udc(udc);
 
-	if (extcon_get_cable_state_(udc->edev, EXTCON_USB) > 0) {
+	if (extcon_get_state(udc->edev, EXTCON_USB) > 0) {
 		dev_dbg(udc->dev, "device -> idle\n");
 		stop_udc(udc);
 	}
@@ -303,7 +303,7 @@ static int udc_plat_resume(struct device *dev)
 		return ret;
 	}
 
-	if (extcon_get_cable_state_(udc->edev, EXTCON_USB) > 0) {
+	if (extcon_get_state(udc->edev, EXTCON_USB) > 0) {
 		dev_dbg(udc->dev, "idle -> device\n");
 		start_udc(udc);
 	}
diff --git a/drivers/usb/host/ehci-fsl.c b/drivers/usb/host/ehci-fsl.c
index 4a08b70c81aa..d025cc06dda7 100644
--- a/drivers/usb/host/ehci-fsl.c
+++ b/drivers/usb/host/ehci-fsl.c
@@ -642,7 +642,7 @@ static int ehci_start_port_reset(struct usb_hcd *hcd, unsigned port)
 #define ehci_start_port_reset	NULL
 #endif /* CONFIG_USB_OTG */
 
-static struct ehci_driver_overrides ehci_fsl_overrides __initdata = {
+static const struct ehci_driver_overrides ehci_fsl_overrides __initconst = {
 	.extra_priv_size = sizeof(struct ehci_fsl),
 	.reset = ehci_fsl_setup,
 };
diff --git a/drivers/usb/host/ehci-omap.c b/drivers/usb/host/ehci-omap.c
index 94ea9fff13e6..4d308533bc83 100644
--- a/drivers/usb/host/ehci-omap.c
+++ b/drivers/usb/host/ehci-omap.c
@@ -130,8 +130,8 @@ static int ehci_hcd_omap_probe(struct platform_device *pdev)
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
-		dev_err(dev, "EHCI irq failed\n");
-		return -ENODEV;
+		dev_err(dev, "EHCI irq failed: %d\n", irq);
+		return irq;
 	}
 
 	res =  platform_get_resource(pdev, IORESOURCE_MEM, 0);
diff --git a/drivers/usb/host/fsl-mph-dr-of.c b/drivers/usb/host/fsl-mph-dr-of.c
index e90ddb530765..ba557cdba8ef 100644
--- a/drivers/usb/host/fsl-mph-dr-of.c
+++ b/drivers/usb/host/fsl-mph-dr-of.c
@@ -55,8 +55,8 @@ static struct fsl_usb2_dev_data *get_dr_mode_data(struct device_node *np)
 				return &dr_mode_data[i];
 		}
 	}
-	pr_warn("%s: Invalid 'dr_mode' property, fallback to host mode\n",
-		np->full_name);
+	pr_warn("%pOF: Invalid 'dr_mode' property, fallback to host mode\n",
+		np);
 	return &dr_mode_data[0]; /* mode not specified, use host */
 }
 
diff --git a/drivers/usb/host/hwa-hc.c b/drivers/usb/host/hwa-hc.c
index 1db0626c8bf4..da3b18038d23 100644
--- a/drivers/usb/host/hwa-hc.c
+++ b/drivers/usb/host/hwa-hc.c
@@ -614,7 +614,7 @@ error:
 	return result;
 }
 
-static struct hc_driver hwahc_hc_driver = {
+static const struct hc_driver hwahc_hc_driver = {
 	.description = "hwa-hcd",
 	.product_desc = "Wireless USB HWA host controller",
 	.hcd_priv_size = sizeof(struct hwahc) - sizeof(struct usb_hcd),
@@ -860,7 +860,7 @@ static void hwahc_disconnect(struct usb_interface *usb_iface)
 	usb_put_hcd(usb_hcd);
 }
 
-static struct usb_device_id hwahc_id_table[] = {
+static const struct usb_device_id hwahc_id_table[] = {
 	/* Alereon 5310 */
 	{ USB_DEVICE_AND_INTERFACE_INFO(0x13dc, 0x5310, 0xe0, 0x02, 0x01),
 	  .driver_info = WUSB_QUIRK_ALEREON_HWA_CONCAT_ISOC |
diff --git a/drivers/usb/host/imx21-hcd.c b/drivers/usb/host/imx21-hcd.c
index f542045dc2a6..39ae7fb64b6f 100644
--- a/drivers/usb/host/imx21-hcd.c
+++ b/drivers/usb/host/imx21-hcd.c
@@ -1779,7 +1779,7 @@ static void imx21_hc_stop(struct usb_hcd *hcd)
 /* Driver glue		 			*/
 /* =========================================== */
 
-static struct hc_driver imx21_hc_driver = {
+static const struct hc_driver imx21_hc_driver = {
 	.description = hcd_name,
 	.product_desc = "IMX21 USB Host Controller",
 	.hcd_priv_size = sizeof(struct imx21),
@@ -1849,8 +1849,10 @@ static int imx21_probe(struct platform_device *pdev)
 	if (!res)
 		return -ENODEV;
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0)
-		return -ENXIO;
+	if (irq < 0) {
+		dev_err(&pdev->dev, "Failed to get IRQ: %d\n", irq);
+		return irq;
+	}
 
 	hcd = usb_create_hcd(&imx21_hc_driver,
 		&pdev->dev, dev_name(&pdev->dev));
diff --git a/drivers/usb/host/isp116x-hcd.c b/drivers/usb/host/isp116x-hcd.c
index d089b3fb7a13..73fec38754f9 100644
--- a/drivers/usb/host/isp116x-hcd.c
+++ b/drivers/usb/host/isp116x-hcd.c
@@ -1511,7 +1511,7 @@ static int isp116x_bus_resume(struct usb_hcd *hcd)
 
 #endif
 
-static struct hc_driver isp116x_hc_driver = {
+static const struct hc_driver isp116x_hc_driver = {
 	.description = hcd_name,
 	.product_desc = "ISP116x Host Controller",
 	.hcd_priv_size = sizeof(struct isp116x),
diff --git a/drivers/usb/host/isp1362-hcd.c b/drivers/usb/host/isp1362-hcd.c
index 0f2b4b358e1a..9b7e307e2d54 100644
--- a/drivers/usb/host/isp1362-hcd.c
+++ b/drivers/usb/host/isp1362-hcd.c
@@ -2591,7 +2591,7 @@ static int isp1362_hc_start(struct usb_hcd *hcd)
 
 /*-------------------------------------------------------------------------*/
 
-static struct hc_driver isp1362_hc_driver = {
+static const struct hc_driver isp1362_hc_driver = {
 	.description =		hcd_name,
 	.product_desc =		"ISP1362 Host Controller",
 	.hcd_priv_size =	sizeof(struct isp1362_hcd),
diff --git a/drivers/usb/host/max3421-hcd.c b/drivers/usb/host/max3421-hcd.c
index 369869a29ebd..0ece9a9341e5 100644
--- a/drivers/usb/host/max3421-hcd.c
+++ b/drivers/usb/host/max3421-hcd.c
@@ -1811,7 +1811,7 @@ max3421_unmap_urb_for_dma(struct usb_hcd *hcd, struct urb *urb)
 {
 }
 
-static struct hc_driver max3421_hcd_desc = {
+static const struct hc_driver max3421_hcd_desc = {
 	.description =		"max3421",
 	.product_desc =		DRIVER_DESC,
 	.hcd_priv_size =	sizeof(struct max3421_hcd),
diff --git a/drivers/usb/host/ohci-omap.c b/drivers/usb/host/ohci-omap.c
index a4d814b7f380..91393ec7d850 100644
--- a/drivers/usb/host/ohci-omap.c
+++ b/drivers/usb/host/ohci-omap.c
@@ -53,7 +53,7 @@
 #define DRIVER_DESC "OHCI OMAP driver"
 
 #ifdef CONFIG_TPS65010
-#include <linux/i2c/tps65010.h>
+#include <linux/mfd/tps65010.h>
 #else
 
 #define LOW	0
diff --git a/drivers/usb/host/ohci-sm501.c b/drivers/usb/host/ohci-sm501.c
index a8b8d8b8d9f3..d4e0f7cd96fa 100644
--- a/drivers/usb/host/ohci-sm501.c
+++ b/drivers/usb/host/ohci-sm501.c
@@ -123,13 +123,12 @@ static int ohci_hcd_sm501_drv_probe(struct platform_device *pdev)
 	 * regular memory. The HCD_LOCAL_MEM flag does just that.
 	 */
 
-	if (!dma_declare_coherent_memory(dev, mem->start,
+	retval = dma_declare_coherent_memory(dev, mem->start,
 					 mem->start - mem->parent->start,
 					 resource_size(mem),
-					 DMA_MEMORY_MAP |
-					 DMA_MEMORY_EXCLUSIVE)) {
+					 DMA_MEMORY_EXCLUSIVE);
+	if (retval) {
 		dev_err(dev, "cannot declare coherent memory\n");
-		retval = -ENXIO;
 		goto err1;
 	}
 
diff --git a/drivers/usb/host/ohci-tmio.c b/drivers/usb/host/ohci-tmio.c
index cfcfadfc94fc..16d081a093bb 100644
--- a/drivers/usb/host/ohci-tmio.c
+++ b/drivers/usb/host/ohci-tmio.c
@@ -227,13 +227,10 @@ static int ohci_hcd_tmio_drv_probe(struct platform_device *dev)
 		goto err_ioremap_regs;
 	}
 
-	if (!dma_declare_coherent_memory(&dev->dev, sram->start,
-				sram->start,
-				resource_size(sram),
-				DMA_MEMORY_MAP | DMA_MEMORY_EXCLUSIVE)) {
-		ret = -EBUSY;
+	ret = dma_declare_coherent_memory(&dev->dev, sram->start, sram->start,
+				resource_size(sram), DMA_MEMORY_EXCLUSIVE);
+	if (ret)
 		goto err_dma_declare;
-	}
 
 	if (cell->enable) {
 		ret = cell->enable(dev);
diff --git a/drivers/usb/host/pci-quirks.c b/drivers/usb/host/pci-quirks.c
index c8f38649f749..658d9d1f9ea3 100644
--- a/drivers/usb/host/pci-quirks.c
+++ b/drivers/usb/host/pci-quirks.c
@@ -142,29 +142,30 @@ static int amd_chipset_sb_type_init(struct amd_chipset_info *pinfo)
 			pinfo->sb_type.gen = AMD_CHIPSET_SB700;
 		else if (rev >= 0x40 && rev <= 0x4f)
 			pinfo->sb_type.gen = AMD_CHIPSET_SB800;
-	}
-	pinfo->smbus_dev = pci_get_device(PCI_VENDOR_ID_AMD,
-					  0x145c, NULL);
-	if (pinfo->smbus_dev) {
-		pinfo->sb_type.gen = AMD_CHIPSET_TAISHAN;
 	} else {
 		pinfo->smbus_dev = pci_get_device(PCI_VENDOR_ID_AMD,
 				PCI_DEVICE_ID_AMD_HUDSON2_SMBUS, NULL);
 
-		if (!pinfo->smbus_dev) {
-			pinfo->sb_type.gen = NOT_AMD_CHIPSET;
-			return 0;
+		if (pinfo->smbus_dev) {
+			rev = pinfo->smbus_dev->revision;
+			if (rev >= 0x11 && rev <= 0x14)
+				pinfo->sb_type.gen = AMD_CHIPSET_HUDSON2;
+			else if (rev >= 0x15 && rev <= 0x18)
+				pinfo->sb_type.gen = AMD_CHIPSET_BOLTON;
+			else if (rev >= 0x39 && rev <= 0x3a)
+				pinfo->sb_type.gen = AMD_CHIPSET_YANGTZE;
+		} else {
+			pinfo->smbus_dev = pci_get_device(PCI_VENDOR_ID_AMD,
+							  0x145c, NULL);
+			if (pinfo->smbus_dev) {
+				rev = pinfo->smbus_dev->revision;
+				pinfo->sb_type.gen = AMD_CHIPSET_TAISHAN;
+			} else {
+				pinfo->sb_type.gen = NOT_AMD_CHIPSET;
+				return 0;
+			}
 		}
-
-		rev = pinfo->smbus_dev->revision;
-		if (rev >= 0x11 && rev <= 0x14)
-			pinfo->sb_type.gen = AMD_CHIPSET_HUDSON2;
-		else if (rev >= 0x15 && rev <= 0x18)
-			pinfo->sb_type.gen = AMD_CHIPSET_BOLTON;
-		else if (rev >= 0x39 && rev <= 0x3a)
-			pinfo->sb_type.gen = AMD_CHIPSET_YANGTZE;
 	}
-
 	pinfo->sb_type.rev = rev;
 	return 1;
 }
diff --git a/drivers/usb/host/r8a66597-hcd.c b/drivers/usb/host/r8a66597-hcd.c
index 7bf78be1fd32..5e5fc9d7d533 100644
--- a/drivers/usb/host/r8a66597-hcd.c
+++ b/drivers/usb/host/r8a66597-hcd.c
@@ -2312,7 +2312,7 @@ static int r8a66597_bus_resume(struct usb_hcd *hcd)
 #define	r8a66597_bus_resume	NULL
 #endif
 
-static struct hc_driver r8a66597_hc_driver = {
+static const struct hc_driver r8a66597_hc_driver = {
 	.description =		hcd_name,
 	.hcd_priv_size =	sizeof(struct r8a66597),
 	.irq =			r8a66597_irq,
diff --git a/drivers/usb/host/sl811-hcd.c b/drivers/usb/host/sl811-hcd.c
index fd2a11473be7..24ad1d6cec25 100644
--- a/drivers/usb/host/sl811-hcd.c
+++ b/drivers/usb/host/sl811-hcd.c
@@ -1554,7 +1554,7 @@ sl811h_start(struct usb_hcd *hcd)
 
 /*-------------------------------------------------------------------------*/
 
-static struct hc_driver sl811h_hc_driver = {
+static const struct hc_driver sl811h_hc_driver = {
 	.description =		hcd_name,
 	.hcd_priv_size =	sizeof(struct sl811),
 
diff --git a/drivers/usb/host/u132-hcd.c b/drivers/usb/host/u132-hcd.c
index 43d52931b5bf..c38855aed62c 100644
--- a/drivers/usb/host/u132-hcd.c
+++ b/drivers/usb/host/u132-hcd.c
@@ -2941,7 +2941,7 @@ static int u132_bus_resume(struct usb_hcd *hcd)
 #define u132_bus_suspend NULL
 #define u132_bus_resume NULL
 #endif
-static struct hc_driver u132_hc_driver = {
+static const struct hc_driver u132_hc_driver = {
 	.description = hcd_name,
 	.hcd_priv_size = sizeof(struct u132),
 	.irq = NULL,
diff --git a/drivers/usb/host/whci/hcd.c b/drivers/usb/host/whci/hcd.c
index 5b3603c360ab..cf84269c3e6d 100644
--- a/drivers/usb/host/whci/hcd.c
+++ b/drivers/usb/host/whci/hcd.c
@@ -213,7 +213,7 @@ static void whc_endpoint_reset(struct usb_hcd *usb_hcd,
 }
 
 
-static struct hc_driver whc_hc_driver = {
+static const struct hc_driver whc_hc_driver = {
 	.description = "whci-hcd",
 	.product_desc = "Wireless host controller",
 	.hcd_priv_size = sizeof(struct whc) - sizeof(struct usb_hcd),
diff --git a/drivers/usb/host/xhci-hub.c b/drivers/usb/host/xhci-hub.c
index 00721e8807ab..ad89a6d4111b 100644
--- a/drivers/usb/host/xhci-hub.c
+++ b/drivers/usb/host/xhci-hub.c
@@ -1179,6 +1179,39 @@ int xhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
 				break;
 			}
 
+			/*
+			 * For xHCI 1.1 according to section 4.19.1.2.4.1 a
+			 * root hub port's transition to compliance mode upon
+			 * detecting LFPS timeout may be controlled by an
+			 * Compliance Transition Enabled (CTE) flag (not
+			 * software visible). This flag is set by writing 0xA
+			 * to PORTSC PLS field which will allow transition to
+			 * compliance mode the next time LFPS timeout is
+			 * encountered. A warm reset will clear it.
+			 *
+			 * The CTE flag is only supported if the HCCPARAMS2 CTC
+			 * flag is set, otherwise, the compliance substate is
+			 * automatically entered as on 1.0 and prior.
+			 */
+			if (link_state == USB_SS_PORT_LS_COMP_MOD) {
+				if (!HCC2_CTC(xhci->hcc_params2)) {
+					xhci_dbg(xhci, "CTC flag is 0, port already supports entering compliance mode\n");
+					break;
+				}
+
+				if ((temp & PORT_CONNECT)) {
+					xhci_warn(xhci, "Can't set compliance mode when port is connected\n");
+					goto error;
+				}
+
+				xhci_dbg(xhci, "Enable compliance mode transition for port %d\n",
+						wIndex);
+				xhci_set_link_state(xhci, port_array, wIndex,
+						link_state);
+				temp = readl(port_array[wIndex]);
+				break;
+			}
+
 			/* Software should not attempt to set
 			 * port link state above '3' (U3) and the port
 			 * must be enabled.
@@ -1521,15 +1554,14 @@ static bool xhci_port_missing_cas_quirk(int port_index,
 int xhci_bus_resume(struct usb_hcd *hcd)
 {
 	struct xhci_hcd	*xhci = hcd_to_xhci(hcd);
-	int max_ports, port_index;
-	__le32 __iomem **port_array;
 	struct xhci_bus_state *bus_state;
-	u32 temp;
+	__le32 __iomem **port_array;
 	unsigned long flags;
-	unsigned long port_was_suspended = 0;
-	bool need_usb2_u3_exit = false;
+	int max_ports, port_index;
 	int slot_id;
 	int sret;
+	u32 next_state;
+	u32 temp, portsc;
 
 	max_ports = xhci_get_ports(hcd, &port_array);
 	bus_state = &xhci->bus_state[hcd_index(hcd)];
@@ -1548,68 +1580,77 @@ int xhci_bus_resume(struct usb_hcd *hcd)
 	temp &= ~CMD_EIE;
 	writel(temp, &xhci->op_regs->command);
 
+	/* bus specific resume for ports we suspended at bus_suspend */
+	if (hcd->speed >= HCD_USB3)
+		next_state = XDEV_U0;
+	else
+		next_state = XDEV_RESUME;
+
 	port_index = max_ports;
 	while (port_index--) {
-		/* Check whether need resume ports. If needed
-		   resume port and disable remote wakeup */
-		u32 temp;
-
-		temp = readl(port_array[port_index]);
+		portsc = readl(port_array[port_index]);
 
 		/* warm reset CAS limited ports stuck in polling/compliance */
 		if ((xhci->quirks & XHCI_MISSING_CAS) &&
 		    (hcd->speed >= HCD_USB3) &&
 		    xhci_port_missing_cas_quirk(port_index, port_array)) {
 			xhci_dbg(xhci, "reset stuck port %d\n", port_index);
+			clear_bit(port_index, &bus_state->bus_suspended);
 			continue;
 		}
-		if (DEV_SUPERSPEED_ANY(temp))
-			temp &= ~(PORT_RWC_BITS | PORT_CEC | PORT_WAKE_BITS);
-		else
-			temp &= ~(PORT_RWC_BITS | PORT_WAKE_BITS);
-		if (test_bit(port_index, &bus_state->bus_suspended) &&
-		    (temp & PORT_PLS_MASK)) {
-			set_bit(port_index, &port_was_suspended);
-			if (!DEV_SUPERSPEED_ANY(temp)) {
-				xhci_set_link_state(xhci, port_array,
-						port_index, XDEV_RESUME);
-				need_usb2_u3_exit = true;
+		/* resume if we suspended the link, and it is still suspended */
+		if (test_bit(port_index, &bus_state->bus_suspended))
+			switch (portsc & PORT_PLS_MASK) {
+			case XDEV_U3:
+				portsc = xhci_port_state_to_neutral(portsc);
+				portsc &= ~PORT_PLS_MASK;
+				portsc |= PORT_LINK_STROBE | next_state;
+				break;
+			case XDEV_RESUME:
+				/* resume already initiated */
+				break;
+			default:
+				/* not in a resumeable state, ignore it */
+				clear_bit(port_index,
+					  &bus_state->bus_suspended);
+				break;
 			}
-		} else
-			writel(temp, port_array[port_index]);
-	}
-
-	if (need_usb2_u3_exit) {
-		spin_unlock_irqrestore(&xhci->lock, flags);
-		msleep(USB_RESUME_TIMEOUT);
-		spin_lock_irqsave(&xhci->lock, flags);
+		/* disable wake for all ports, write new link state if needed */
+		portsc &= ~(PORT_RWC_BITS | PORT_CEC | PORT_WAKE_BITS);
+		writel(portsc, port_array[port_index]);
 	}
 
-	port_index = max_ports;
-	while (port_index--) {
-		if (!(port_was_suspended & BIT(port_index)))
-			continue;
-		/* Clear PLC to poll it later after XDEV_U0 */
-		xhci_test_and_clear_bit(xhci, port_array, port_index, PORT_PLC);
-		xhci_set_link_state(xhci, port_array, port_index, XDEV_U0);
+	/* USB2 specific resume signaling delay and U0 link state transition */
+	if (hcd->speed < HCD_USB3) {
+		if (bus_state->bus_suspended) {
+			spin_unlock_irqrestore(&xhci->lock, flags);
+			msleep(USB_RESUME_TIMEOUT);
+			spin_lock_irqsave(&xhci->lock, flags);
+		}
+		for_each_set_bit(port_index, &bus_state->bus_suspended,
+				 BITS_PER_LONG) {
+			/* Clear PLC to poll it later for U0 transition */
+			xhci_test_and_clear_bit(xhci, port_array, port_index,
+						PORT_PLC);
+			xhci_set_link_state(xhci, port_array, port_index,
+					    XDEV_U0);
+		}
 	}
 
-	port_index = max_ports;
-	while (port_index--) {
-		if (!(port_was_suspended & BIT(port_index)))
-			continue;
-		/* Poll and Clear PLC */
+	/* poll for U0 link state complete, both USB2 and USB3 */
+	for_each_set_bit(port_index, &bus_state->bus_suspended, BITS_PER_LONG) {
 		sret = xhci_handshake(port_array[port_index], PORT_PLC,
 				      PORT_PLC, 10 * 1000);
-		if (sret)
+		if (sret) {
 			xhci_warn(xhci, "port %d resume PLC timeout\n",
 				  port_index);
+			continue;
+		}
 		xhci_test_and_clear_bit(xhci, port_array, port_index, PORT_PLC);
 		slot_id = xhci_find_slot_id_by_port(hcd, xhci, port_index + 1);
 		if (slot_id)
 			xhci_ring_device(xhci, slot_id);
 	}
-
 	(void) readl(&xhci->op_regs->command);
 
 	bus_state->next_statechange = jiffies + msecs_to_jiffies(5);
diff --git a/drivers/usb/host/xhci-mtk.c b/drivers/usb/host/xhci-mtk.c
index 67d5dc79b6b5..8fb60657ed4f 100644
--- a/drivers/usb/host/xhci-mtk.c
+++ b/drivers/usb/host/xhci-mtk.c
@@ -795,6 +795,7 @@ static const struct dev_pm_ops xhci_mtk_pm_ops = {
 #ifdef CONFIG_OF
 static const struct of_device_id mtk_xhci_of_match[] = {
 	{ .compatible = "mediatek,mt8173-xhci"},
+	{ .compatible = "mediatek,mtk-xhci"},
 	{ },
 };
 MODULE_DEVICE_TABLE(of, mtk_xhci_of_match);
diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c
index c04144b25a67..163bafde709f 100644
--- a/drivers/usb/host/xhci-plat.c
+++ b/drivers/usb/host/xhci-plat.c
@@ -107,14 +107,6 @@ static const struct xhci_plat_priv xhci_plat_renesas_rcar_gen2 = {
 };
 
 static const struct xhci_plat_priv xhci_plat_renesas_rcar_gen3 = {
-	.firmware_name = XHCI_RCAR_FIRMWARE_NAME_V2,
-	.init_quirk = xhci_rcar_init_quirk,
-	.plat_start = xhci_rcar_start,
-	.resume_quirk = xhci_rcar_resume_quirk,
-};
-
-static const struct xhci_plat_priv xhci_plat_renesas_rcar_r8a7796 = {
-	.firmware_name = XHCI_RCAR_FIRMWARE_NAME_V3,
 	.init_quirk = xhci_rcar_init_quirk,
 	.plat_start = xhci_rcar_start,
 	.resume_quirk = xhci_rcar_resume_quirk,
@@ -145,7 +137,7 @@ static const struct of_device_id usb_xhci_of_match[] = {
 		.data = &xhci_plat_renesas_rcar_gen3,
 	}, {
 		.compatible = "renesas,xhci-r8a7796",
-		.data = &xhci_plat_renesas_rcar_r8a7796,
+		.data = &xhci_plat_renesas_rcar_gen3,
 	}, {
 		.compatible = "renesas,rcar-gen2-xhci",
 		.data = &xhci_plat_renesas_rcar_gen2,
diff --git a/drivers/usb/host/xhci-rcar.c b/drivers/usb/host/xhci-rcar.c
index 07278228214b..198bc188ab25 100644
--- a/drivers/usb/host/xhci-rcar.c
+++ b/drivers/usb/host/xhci-rcar.c
@@ -13,13 +13,15 @@
 #include <linux/platform_device.h>
 #include <linux/of.h>
 #include <linux/usb/phy.h>
+#include <linux/sys_soc.h>
 
 #include "xhci.h"
 #include "xhci-plat.h"
 #include "xhci-rcar.h"
 
 /*
-* - The V3 firmware is for r8a7796 (with good performance).
+* - The V3 firmware is for r8a7796 (with good performance) and r8a7795 es2.0
+*   or later.
 * - The V2 firmware can be used on both r8a7795 (es1.x) and r8a7796.
 * - The V2 firmware is possible to use on R-Car Gen2. However, the V2 causes
 *   performance degradation. So, this driver continues to use the V1 if R-Car
@@ -67,6 +69,26 @@ MODULE_FIRMWARE(XHCI_RCAR_FIRMWARE_NAME_V3);
 #define RCAR_USB3_RX_POL_VAL	BIT(21)
 #define RCAR_USB3_TX_POL_VAL	BIT(4)
 
+/* For soc_device_attribute */
+#define RCAR_XHCI_FIRMWARE_V2   BIT(0) /* FIRMWARE V2 */
+#define RCAR_XHCI_FIRMWARE_V3   BIT(1) /* FIRMWARE V3 */
+
+static const struct soc_device_attribute rcar_quirks_match[]  = {
+	{
+		.soc_id = "r8a7795", .revision = "ES1.*",
+		.data = (void *)RCAR_XHCI_FIRMWARE_V2,
+	},
+	{
+		.soc_id = "r8a7795",
+		.data = (void *)RCAR_XHCI_FIRMWARE_V3,
+	},
+	{
+		.soc_id = "r8a7796",
+		.data = (void *)RCAR_XHCI_FIRMWARE_V3,
+	},
+	{ /* sentinel */ },
+};
+
 static void xhci_rcar_start_gen2(struct usb_hcd *hcd)
 {
 	/* LCLK Select */
@@ -122,9 +144,23 @@ static int xhci_rcar_download_firmware(struct usb_hcd *hcd)
 	int retval, index, j, time;
 	int timeout = 10000;
 	u32 data, val, temp;
+	u32 quirks = 0;
+	const struct soc_device_attribute *attr;
+	const char *firmware_name;
+
+	attr = soc_device_match(rcar_quirks_match);
+	if (attr)
+		quirks = (uintptr_t)attr->data;
+
+	if (quirks & RCAR_XHCI_FIRMWARE_V2)
+		firmware_name = XHCI_RCAR_FIRMWARE_NAME_V2;
+	else if (quirks & RCAR_XHCI_FIRMWARE_V3)
+		firmware_name = XHCI_RCAR_FIRMWARE_NAME_V3;
+	else
+		firmware_name = priv->firmware_name;
 
 	/* request R-Car USB3.0 firmware */
-	retval = request_firmware(&fw, priv->firmware_name, dev);
+	retval = request_firmware(&fw, firmware_name, dev);
 	if (retval)
 		return retval;
 
diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c
index cc368ad2b51e..a9443651ce0f 100644
--- a/drivers/usb/host/xhci-ring.c
+++ b/drivers/usb/host/xhci-ring.c
@@ -1572,7 +1572,7 @@ static void handle_port_status(struct xhci_hcd *xhci,
 {
 	struct usb_hcd *hcd;
 	u32 port_id;
-	u32 temp, temp1;
+	u32 portsc, cmd_reg;
 	int max_ports;
 	int slot_id;
 	unsigned int faked_port_index;
@@ -1636,26 +1636,28 @@ static void handle_port_status(struct xhci_hcd *xhci,
 	/* Find the faked port hub number */
 	faked_port_index = find_faked_portnum_from_hw_portnum(hcd, xhci,
 			port_id);
+	portsc = readl(port_array[faked_port_index]);
+
+	trace_xhci_handle_port_status(faked_port_index, portsc);
 
-	temp = readl(port_array[faked_port_index]);
 	if (hcd->state == HC_STATE_SUSPENDED) {
 		xhci_dbg(xhci, "resume root hub\n");
 		usb_hcd_resume_root_hub(hcd);
 	}
 
-	if (hcd->speed >= HCD_USB3 && (temp & PORT_PLS_MASK) == XDEV_INACTIVE)
+	if (hcd->speed >= HCD_USB3 && (portsc & PORT_PLS_MASK) == XDEV_INACTIVE)
 		bus_state->port_remote_wakeup &= ~(1 << faked_port_index);
 
-	if ((temp & PORT_PLC) && (temp & PORT_PLS_MASK) == XDEV_RESUME) {
+	if ((portsc & PORT_PLC) && (portsc & PORT_PLS_MASK) == XDEV_RESUME) {
 		xhci_dbg(xhci, "port resume event for port %d\n", port_id);
 
-		temp1 = readl(&xhci->op_regs->command);
-		if (!(temp1 & CMD_RUN)) {
+		cmd_reg = readl(&xhci->op_regs->command);
+		if (!(cmd_reg & CMD_RUN)) {
 			xhci_warn(xhci, "xHC is not running.\n");
 			goto cleanup;
 		}
 
-		if (DEV_SUPERSPEED_ANY(temp)) {
+		if (DEV_SUPERSPEED_ANY(portsc)) {
 			xhci_dbg(xhci, "remote wake SS port %d\n", port_id);
 			/* Set a flag to say the port signaled remote wakeup,
 			 * so we can tell the difference between the end of
@@ -1683,8 +1685,8 @@ static void handle_port_status(struct xhci_hcd *xhci,
 		}
 	}
 
-	if ((temp & PORT_PLC) && (temp & PORT_PLS_MASK) == XDEV_U0 &&
-			DEV_SUPERSPEED_ANY(temp)) {
+	if ((portsc & PORT_PLC) && (portsc & PORT_PLS_MASK) == XDEV_U0 &&
+			DEV_SUPERSPEED_ANY(portsc)) {
 		xhci_dbg(xhci, "resume SS port %d finished\n", port_id);
 		/* We've just brought the device into U0 through either the
 		 * Resume state after a device remote wakeup, or through the
@@ -1714,7 +1716,7 @@ static void handle_port_status(struct xhci_hcd *xhci,
 	 * RExit to a disconnect state).  If so, let the the driver know it's
 	 * out of the RExit state.
 	 */
-	if (!DEV_SUPERSPEED_ANY(temp) &&
+	if (!DEV_SUPERSPEED_ANY(portsc) &&
 			test_and_clear_bit(faked_port_index,
 				&bus_state->rexit_ports)) {
 		complete(&bus_state->rexit_done[faked_port_index]);
diff --git a/drivers/usb/host/xhci-trace.h b/drivers/usb/host/xhci-trace.h
index 8ce96de10e8a..f20753b99624 100644
--- a/drivers/usb/host/xhci-trace.h
+++ b/drivers/usb/host/xhci-trace.h
@@ -453,6 +453,29 @@ DEFINE_EVENT(xhci_log_ring, xhci_inc_deq,
 	TP_PROTO(struct xhci_ring *ring),
 	TP_ARGS(ring)
 );
+
+DECLARE_EVENT_CLASS(xhci_log_portsc,
+		    TP_PROTO(u32 portnum, u32 portsc),
+		    TP_ARGS(portnum, portsc),
+		    TP_STRUCT__entry(
+				     __field(u32, portnum)
+				     __field(u32, portsc)
+				     ),
+		    TP_fast_assign(
+				   __entry->portnum = portnum;
+				   __entry->portsc = portsc;
+				   ),
+		    TP_printk("port-%d: %s",
+			      __entry->portnum,
+			      xhci_decode_portsc(__entry->portsc)
+			      )
+);
+
+DEFINE_EVENT(xhci_log_portsc, xhci_handle_port_status,
+	     TP_PROTO(u32 portnum, u32 portsc),
+	     TP_ARGS(portnum, portsc)
+);
+
 #endif /* __XHCI_TRACE_H */
 
 /* this part must be outside header guard */
diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h
index e3e935291ed6..2abaa4d6d39d 100644
--- a/drivers/usb/host/xhci.h
+++ b/drivers/usb/host/xhci.h
@@ -311,12 +311,19 @@ struct xhci_op_regs {
  */
 #define PORT_PLS_MASK	(0xf << 5)
 #define XDEV_U0		(0x0 << 5)
+#define XDEV_U1		(0x1 << 5)
 #define XDEV_U2		(0x2 << 5)
 #define XDEV_U3		(0x3 << 5)
+#define XDEV_DISABLED	(0x4 << 5)
+#define XDEV_RXDETECT	(0x5 << 5)
 #define XDEV_INACTIVE	(0x6 << 5)
 #define XDEV_POLLING	(0x7 << 5)
-#define XDEV_COMP_MODE  (0xa << 5)
+#define XDEV_RECOVERY	(0x8 << 5)
+#define XDEV_HOT_RESET	(0x9 << 5)
+#define XDEV_COMP_MODE	(0xa << 5)
+#define XDEV_TEST_MODE	(0xb << 5)
 #define XDEV_RESUME	(0xf << 5)
+
 /* true: port has power (see HCC_PPC) */
 #define PORT_POWER	(1 << 9)
 /* bits 10:13 indicate device speed:
@@ -2392,6 +2399,87 @@ static inline const char *xhci_decode_slot_context(u32 info, u32 info2,
 	return str;
 }
 
+
+static inline const char *xhci_portsc_link_state_string(u32 portsc)
+{
+	switch (portsc & PORT_PLS_MASK) {
+	case XDEV_U0:
+		return "U0";
+	case XDEV_U1:
+		return "U1";
+	case XDEV_U2:
+		return "U2";
+	case XDEV_U3:
+		return "U3";
+	case XDEV_DISABLED:
+		return "Disabled";
+	case XDEV_RXDETECT:
+		return "RxDetect";
+	case XDEV_INACTIVE:
+		return "Inactive";
+	case XDEV_POLLING:
+		return "Polling";
+	case XDEV_RECOVERY:
+		return "Recovery";
+	case XDEV_HOT_RESET:
+		return "Hot Reset";
+	case XDEV_COMP_MODE:
+		return "Compliance mode";
+	case XDEV_TEST_MODE:
+		return "Test mode";
+	case XDEV_RESUME:
+		return "Resume";
+	default:
+		break;
+	}
+	return "Unknown";
+}
+
+static inline const char *xhci_decode_portsc(u32 portsc)
+{
+	static char str[256];
+	int ret;
+
+	ret = sprintf(str, "%s %s %s Link:%s ",
+		      portsc & PORT_POWER	? "Powered" : "Powered-off",
+		      portsc & PORT_CONNECT	? "Connected" : "Not-connected",
+		      portsc & PORT_PE		? "Enabled" : "Disabled",
+		      xhci_portsc_link_state_string(portsc));
+
+	if (portsc & PORT_OC)
+		ret += sprintf(str + ret, "OverCurrent ");
+	if (portsc & PORT_RESET)
+		ret += sprintf(str + ret, "In-Reset ");
+
+	ret += sprintf(str + ret, "Change: ");
+	if (portsc & PORT_CSC)
+		ret += sprintf(str + ret, "CSC ");
+	if (portsc & PORT_PEC)
+		ret += sprintf(str + ret, "PEC ");
+	if (portsc & PORT_WRC)
+		ret += sprintf(str + ret, "WRC ");
+	if (portsc & PORT_OCC)
+		ret += sprintf(str + ret, "OCC ");
+	if (portsc & PORT_RC)
+		ret += sprintf(str + ret, "PRC ");
+	if (portsc & PORT_PLC)
+		ret += sprintf(str + ret, "PLC ");
+	if (portsc & PORT_CEC)
+		ret += sprintf(str + ret, "CEC ");
+	if (portsc & PORT_CAS)
+		ret += sprintf(str + ret, "CAS ");
+
+	ret += sprintf(str + ret, "Wake: ");
+	if (portsc & PORT_WKCONN_E)
+		ret += sprintf(str + ret, "WCE ");
+	if (portsc & PORT_WKDISC_E)
+		ret += sprintf(str + ret, "WDE ");
+	if (portsc & PORT_WKOC_E)
+		ret += sprintf(str + ret, "WOE ");
+
+	return str;
+}
+
 static inline const char *xhci_ep_state_string(u8 state)
 {
 	switch (state) {
diff --git a/drivers/usb/image/microtek.c b/drivers/usb/image/microtek.c
index a4dbb0cd80da..0b21ba757bba 100644
--- a/drivers/usb/image/microtek.c
+++ b/drivers/usb/image/microtek.c
@@ -137,10 +137,6 @@
 
 #include "microtek.h"
 
-/*
- * Version Information
- */
-#define DRIVER_VERSION "v0.4.3"
 #define DRIVER_AUTHOR "John Fremlin <vii@penguinpowered.com>, Oliver Neukum <Oliver.Neukum@lrz.uni-muenchen.de>"
 #define DRIVER_DESC "Microtek Scanmaker X6 USB scanner driver"
 
diff --git a/drivers/usb/isp1760/isp1760-hcd.c b/drivers/usb/isp1760/isp1760-hcd.c
index ac31d19cc54b..8e59e0c02b8a 100644
--- a/drivers/usb/isp1760/isp1760-hcd.c
+++ b/drivers/usb/isp1760/isp1760-hcd.c
@@ -396,7 +396,6 @@ static int handshake(struct usb_hcd *hcd, u32 reg,
 /* reset a non-running (STS_HALT == 1) controller */
 static int ehci_reset(struct usb_hcd *hcd)
 {
-	int retval;
 	struct isp1760_hcd *priv = hcd_to_priv(hcd);
 
 	u32 command = reg_read32(hcd->regs, HC_USBCMD);
@@ -405,9 +404,8 @@ static int ehci_reset(struct usb_hcd *hcd)
 	reg_write32(hcd->regs, HC_USBCMD, command);
 	hcd->state = HC_STATE_HALT;
 	priv->next_statechange = jiffies;
-	retval = handshake(hcd, HC_USBCMD,
-			    CMD_RESET, 0, 250 * 1000);
-	return retval;
+
+	return handshake(hcd, HC_USBCMD, CMD_RESET, 0, 250 * 1000);
 }
 
 static struct isp1760_qh *qh_alloc(gfp_t flags)
diff --git a/drivers/usb/misc/adutux.c b/drivers/usb/misc/adutux.c
index dfd54ea4808f..1c0ada75c35d 100644
--- a/drivers/usb/misc/adutux.c
+++ b/drivers/usb/misc/adutux.c
@@ -29,8 +29,6 @@
 #include <linux/mutex.h>
 #include <linux/uaccess.h>
 
-/* Version Information */
-#define DRIVER_VERSION "v0.0.13"
 #define DRIVER_AUTHOR "John Homppi"
 #define DRIVER_DESC "adutux (see www.ontrak.net)"
 
diff --git a/drivers/usb/misc/chaoskey.c b/drivers/usb/misc/chaoskey.c
index 15d4e64d3b65..abec6e604a62 100644
--- a/drivers/usb/misc/chaoskey.c
+++ b/drivers/usb/misc/chaoskey.c
@@ -42,12 +42,10 @@ static int chaoskey_rng_read(struct hwrng *rng, void *data,
 	dev_err(&(usb_if)->dev, format, ## arg)
 
 /* Version Information */
-#define DRIVER_VERSION	"v0.1"
 #define DRIVER_AUTHOR	"Keith Packard, keithp@keithp.com"
 #define DRIVER_DESC	"Altus Metrum ChaosKey driver"
 #define DRIVER_SHORT	"chaoskey"
 
-MODULE_VERSION(DRIVER_VERSION);
 MODULE_AUTHOR(DRIVER_AUTHOR);
 MODULE_DESCRIPTION(DRIVER_DESC);
 MODULE_LICENSE("GPL");
diff --git a/drivers/usb/misc/cytherm.c b/drivers/usb/misc/cytherm.c
index 9d8bb8dacdcd..63207c42acf6 100644
--- a/drivers/usb/misc/cytherm.c
+++ b/drivers/usb/misc/cytherm.c
@@ -20,7 +20,6 @@
 #include <linux/module.h>
 #include <linux/usb.h>
 
-#define DRIVER_VERSION "v1.0"
 #define DRIVER_AUTHOR "Erik Rigtorp"
 #define DRIVER_DESC "Cypress USB Thermometer driver"
 
diff --git a/drivers/usb/misc/ftdi-elan.c b/drivers/usb/misc/ftdi-elan.c
index 8291499d0581..424ff12f3b51 100644
--- a/drivers/usb/misc/ftdi-elan.c
+++ b/drivers/usb/misc/ftdi-elan.c
@@ -305,9 +305,9 @@ static int ftdi_elan_command_engine(struct usb_ftdi *ftdi);
 static int ftdi_elan_respond_engine(struct usb_ftdi *ftdi);
 static int ftdi_elan_hcd_init(struct usb_ftdi *ftdi)
 {
-	int result;
 	if (ftdi->platform_dev.dev.parent)
 		return -EBUSY;
+
 	ftdi_elan_get_kref(ftdi);
 	ftdi->platform_data.potpg = 100;
 	ftdi->platform_data.reset = NULL;
@@ -324,8 +324,8 @@ static int ftdi_elan_hcd_init(struct usb_ftdi *ftdi)
 	request_module("u132_hcd");
 	dev_info(&ftdi->udev->dev, "registering '%s'\n",
 		 ftdi->platform_dev.name);
-	result = platform_device_register(&ftdi->platform_dev);
-	return result;
+
+	return platform_device_register(&ftdi->platform_dev);
 }
 
 static void ftdi_elan_abandon_completions(struct usb_ftdi *ftdi)
@@ -857,7 +857,7 @@ static char *have_ed_set_response(struct usb_ftdi *ftdi,
 	target->actual = 0;
 	target->non_null = (ed_length >> 15) & 0x0001;
 	target->repeat_number = (ed_length >> 11) & 0x000F;
-	if (ed_type == 0x02) {
+	if (ed_type == 0x02 || ed_type == 0x03) {
 		if (payload == 0 || target->abandoning > 0) {
 			target->abandoning = 0;
 			mutex_unlock(&ftdi->u132_lock);
@@ -873,31 +873,6 @@ static char *have_ed_set_response(struct usb_ftdi *ftdi,
 			mutex_unlock(&ftdi->u132_lock);
 			return b;
 		}
-	} else if (ed_type == 0x03) {
-		if (payload == 0 || target->abandoning > 0) {
-			target->abandoning = 0;
-			mutex_unlock(&ftdi->u132_lock);
-			ftdi_elan_do_callback(ftdi, target, 4 + ftdi->response,
-					      payload);
-			ftdi->received = 0;
-			ftdi->expected = 4;
-			ftdi->ed_found = 0;
-			return ftdi->response;
-		} else {
-			ftdi->expected = 4 + payload;
-			ftdi->ed_found = 1;
-			mutex_unlock(&ftdi->u132_lock);
-			return b;
-		}
-	} else if (ed_type == 0x01) {
-		target->abandoning = 0;
-		mutex_unlock(&ftdi->u132_lock);
-		ftdi_elan_do_callback(ftdi, target, 4 + ftdi->response,
-				      payload);
-		ftdi->received = 0;
-		ftdi->expected = 4;
-		ftdi->ed_found = 0;
-		return ftdi->response;
 	} else {
 		target->abandoning = 0;
 		mutex_unlock(&ftdi->u132_lock);
diff --git a/drivers/usb/misc/idmouse.c b/drivers/usb/misc/idmouse.c
index 81fcbf024c65..39d8fedfaf3b 100644
--- a/drivers/usb/misc/idmouse.c
+++ b/drivers/usb/misc/idmouse.c
@@ -33,8 +33,6 @@
 #define HEADER "P5 225 289 255 "
 #define IMGSIZE ((WIDTH * HEIGHT) + sizeof(HEADER)-1)
 
-/* version information */
-#define DRIVER_VERSION "0.6"
 #define DRIVER_SHORT   "idmouse"
 #define DRIVER_AUTHOR  "Florian 'Floe' Echtler <echtler@fs.tum.de>"
 #define DRIVER_DESC    "Siemens ID Mouse FingerTIP Sensor Driver"
diff --git a/drivers/usb/misc/iowarrior.c b/drivers/usb/misc/iowarrior.c
index 7ca4c7e0ea0d..be5881303681 100644
--- a/drivers/usb/misc/iowarrior.c
+++ b/drivers/usb/misc/iowarrior.c
@@ -21,10 +21,8 @@
 #include <linux/poll.h>
 #include <linux/usb/iowarrior.h>
 
-/* Version Information */
-#define DRIVER_VERSION "v0.4.0"
 #define DRIVER_AUTHOR "Christian Lucht <lucht@codemercs.com>"
-#define DRIVER_DESC "USB IO-Warrior driver (Linux 2.6.x)"
+#define DRIVER_DESC "USB IO-Warrior driver"
 
 #define USB_VENDOR_ID_CODEMERCS		1984
 /* low speed iowarrior */
diff --git a/drivers/usb/misc/ldusb.c b/drivers/usb/misc/ldusb.c
index 9d9487c66f87..680bddb3ce05 100644
--- a/drivers/usb/misc/ldusb.c
+++ b/drivers/usb/misc/ldusb.c
@@ -112,7 +112,6 @@ static const struct usb_device_id ld_usb_table[] = {
 	{ }					/* Terminating entry */
 };
 MODULE_DEVICE_TABLE(usb, ld_usb_table);
-MODULE_VERSION("V0.14");
 MODULE_AUTHOR("Michael Hund <mhund@ld-didactic.de>");
 MODULE_DESCRIPTION("LD USB Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/usb/misc/legousbtower.c b/drivers/usb/misc/legousbtower.c
index 0782ac6f5edf..5628f678ab59 100644
--- a/drivers/usb/misc/legousbtower.c
+++ b/drivers/usb/misc/legousbtower.c
@@ -88,8 +88,6 @@
 #include <linux/poll.h>
 
 
-/* Version Information */
-#define DRIVER_VERSION "v0.96"
 #define DRIVER_AUTHOR "Juergen Stuber <starblue@sourceforge.net>"
 #define DRIVER_DESC "LEGO USB Tower Driver"
 
diff --git a/drivers/usb/misc/lvstest.c b/drivers/usb/misc/lvstest.c
index 2142132a1f82..ddddd6387f66 100644
--- a/drivers/usb/misc/lvstest.c
+++ b/drivers/usb/misc/lvstest.c
@@ -178,6 +178,25 @@ static ssize_t hot_reset_store(struct device *dev,
 }
 static DEVICE_ATTR_WO(hot_reset);
 
+static ssize_t warm_reset_store(struct device *dev,
+		struct device_attribute *attr, const char *buf, size_t count)
+{
+	struct usb_interface *intf = to_usb_interface(dev);
+	struct usb_device *hdev = interface_to_usbdev(intf);
+	struct lvs_rh *lvs = usb_get_intfdata(intf);
+	int ret;
+
+	ret = lvs_rh_set_port_feature(hdev, lvs->portnum,
+			USB_PORT_FEAT_BH_PORT_RESET);
+	if (ret < 0) {
+		dev_err(dev, "can't issue warm reset %d\n", ret);
+		return ret;
+	}
+
+	return count;
+}
+static DEVICE_ATTR_WO(warm_reset);
+
 static ssize_t u2_timeout_store(struct device *dev,
 		struct device_attribute *attr, const char *buf, size_t count)
 {
@@ -274,13 +293,35 @@ free_desc:
 }
 static DEVICE_ATTR_WO(get_dev_desc);
 
+static ssize_t enable_compliance_store(struct device *dev,
+		struct device_attribute *attr, const char *buf, size_t count)
+{
+	struct usb_interface *intf = to_usb_interface(dev);
+	struct usb_device *hdev = interface_to_usbdev(intf);
+	struct lvs_rh *lvs = usb_get_intfdata(intf);
+	int ret;
+
+	ret = lvs_rh_set_port_feature(hdev,
+			lvs->portnum | USB_SS_PORT_LS_COMP_MOD << 3,
+			USB_PORT_FEAT_LINK_STATE);
+	if (ret < 0) {
+		dev_err(dev, "can't enable compliance mode %d\n", ret);
+		return ret;
+	}
+
+	return count;
+}
+static DEVICE_ATTR_WO(enable_compliance);
+
 static struct attribute *lvs_attributes[] = {
 	&dev_attr_get_dev_desc.attr,
 	&dev_attr_u1_timeout.attr,
 	&dev_attr_u2_timeout.attr,
 	&dev_attr_hot_reset.attr,
+	&dev_attr_warm_reset.attr,
 	&dev_attr_u3_entry.attr,
 	&dev_attr_u3_exit.attr,
+	&dev_attr_enable_compliance.attr,
 	NULL
 };
 
diff --git a/drivers/usb/misc/rio500.c b/drivers/usb/misc/rio500.c
index b106ce76997b..ddfebb144aaa 100644
--- a/drivers/usb/misc/rio500.c
+++ b/drivers/usb/misc/rio500.c
@@ -43,10 +43,6 @@
 
 #include "rio500_usb.h"
 
-/*
- * Version Information
- */
-#define DRIVER_VERSION "v1.1"
 #define DRIVER_AUTHOR "Cesar Miquel <miquel@df.uba.ar>"
 #define DRIVER_DESC "USB Rio 500 driver"
 
diff --git a/drivers/usb/misc/sisusbvga/sisusb.c b/drivers/usb/misc/sisusbvga/sisusb.c
index 440d7fef58cc..30774e0aeadd 100644
--- a/drivers/usb/misc/sisusbvga/sisusb.c
+++ b/drivers/usb/misc/sisusbvga/sisusb.c
@@ -610,13 +610,11 @@ static int sisusb_write_memio_byte(struct sisusb_usb_data *sisusb, int type,
 		u32 addr, u8 data)
 {
 	struct sisusb_packet packet;
-	int ret;
 
 	packet.header  = (1 << (addr & 3)) | (type << 6);
 	packet.address = addr & ~3;
 	packet.data    = data << ((addr & 3) << 3);
-	ret = sisusb_send_packet(sisusb, 10, &packet);
-	return ret;
+	return sisusb_send_packet(sisusb, 10, &packet);
 }
 
 static int sisusb_write_memio_word(struct sisusb_usb_data *sisusb, int type,
@@ -1333,13 +1331,11 @@ static int sisusb_write_pci_config(struct sisusb_usb_data *sisusb,
 		int regnum, u32 data)
 {
 	struct sisusb_packet packet;
-	int ret;
 
 	packet.header = 0x008f;
 	packet.address = regnum | 0x10000;
 	packet.data = data;
-	ret = sisusb_send_packet(sisusb, 10, &packet);
-	return ret;
+	return sisusb_send_packet(sisusb, 10, &packet);
 }
 
 static int sisusb_read_pci_config(struct sisusb_usb_data *sisusb,
@@ -2982,14 +2978,11 @@ err_out:
 static long sisusb_compat_ioctl(struct file *f, unsigned int cmd,
 		unsigned long arg)
 {
-	long retval;
-
 	switch (cmd) {
 	case SISUSB_GET_CONFIG_SIZE:
 	case SISUSB_GET_CONFIG:
 	case SISUSB_COMMAND:
-		retval = sisusb_ioctl(f, cmd, arg);
-		return retval;
+		return sisusb_ioctl(f, cmd, arg);
 
 	default:
 		return -ENOIOCTLCMD;
diff --git a/drivers/usb/misc/trancevibrator.c b/drivers/usb/misc/trancevibrator.c
index 9795457723d8..1862ed15ce28 100644
--- a/drivers/usb/misc/trancevibrator.c
+++ b/drivers/usb/misc/trancevibrator.c
@@ -25,8 +25,6 @@
 #include <linux/module.h>
 #include <linux/usb.h>
 
-/* Version Information */
-#define DRIVER_VERSION "v1.1"
 #define DRIVER_AUTHOR "Sam Hocevar, sam@zoy.org"
 #define DRIVER_DESC "PlayStation 2 Trance Vibrator driver"
 
diff --git a/drivers/usb/misc/usb251xb.c b/drivers/usb/misc/usb251xb.c
index 91f66d68bcb7..135c91c434bf 100644
--- a/drivers/usb/misc/usb251xb.c
+++ b/drivers/usb/misc/usb251xb.c
@@ -114,7 +114,6 @@
 
 #define DRIVER_NAME	"usb251xb"
 #define DRIVER_DESC	"Microchip USB 2.0 Hi-Speed Hub Controller"
-#define DRIVER_VERSION	"1.0"
 
 struct usb251xb {
 	struct device *dev;
diff --git a/drivers/usb/misc/usbsevseg.c b/drivers/usb/misc/usbsevseg.c
index 388fae6373db..3f6a28045b53 100644
--- a/drivers/usb/misc/usbsevseg.c
+++ b/drivers/usb/misc/usbsevseg.c
@@ -330,7 +330,7 @@ static struct attribute *dev_attrs[] = {
 	NULL
 };
 
-static struct attribute_group dev_attr_grp = {
+static const struct attribute_group dev_attr_grp = {
 	.attrs = dev_attrs,
 };
 
diff --git a/drivers/usb/misc/uss720.c b/drivers/usb/misc/uss720.c
index 5947373700a1..8a13b2fcf3e1 100644
--- a/drivers/usb/misc/uss720.c
+++ b/drivers/usb/misc/uss720.c
@@ -52,10 +52,6 @@
 #include <linux/slab.h>
 #include <linux/sched/signal.h>
 
-/*
- * Version Information
- */
-#define DRIVER_VERSION "v0.6"
 #define DRIVER_AUTHOR "Thomas M. Sailer, t.sailer@alumni.ethz.ch"
 #define DRIVER_DESC "USB Parport Cable driver for Cables using the Lucent Technologies USS720 Chip"
 
@@ -816,8 +812,7 @@ static int __init uss720_init(void)
 	if (retval)
 		goto out;
 
-	printk(KERN_INFO KBUILD_MODNAME ": " DRIVER_VERSION ":"
-	       DRIVER_DESC "\n");
+	printk(KERN_INFO KBUILD_MODNAME ": " DRIVER_DESC "\n");
 	printk(KERN_INFO KBUILD_MODNAME ": NOTE: this is a special purpose "
 	       "driver to allow nonstandard\n");
 	printk(KERN_INFO KBUILD_MODNAME ": protocols (eg. bitbang) over "
diff --git a/drivers/usb/mtu3/mtu3.h b/drivers/usb/mtu3/mtu3.h
index 7b6dc23d77e9..b26fffc58446 100644
--- a/drivers/usb/mtu3/mtu3.h
+++ b/drivers/usb/mtu3/mtu3.h
@@ -288,6 +288,7 @@ static inline struct ssusb_mtk *dev_to_ssusb(struct device *dev)
  *		MTU3_U3_IP_SLOT_DEFAULT for U3 IP
  * @may_wakeup: means device's remote wakeup is enabled
  * @is_self_powered: is reported in device status and the config descriptor
+ * @delayed_status: true when function drivers ask for delayed status
  * @ep0_req: dummy request used while handling standard USB requests
  *		for GET_STATUS and SET_SEL
  * @setup_buf: ep0 response buffer for GET_STATUS and SET_SEL requests
@@ -327,6 +328,7 @@ struct mtu3 {
 	unsigned u1_enable:1;
 	unsigned u2_enable:1;
 	unsigned is_u3_ip:1;
+	unsigned delayed_status:1;
 
 	u8 address;
 	u8 test_mode_nr;
diff --git a/drivers/usb/mtu3/mtu3_dr.c b/drivers/usb/mtu3/mtu3_dr.c
index 11a0d3b84c5e..560256115b23 100644
--- a/drivers/usb/mtu3/mtu3_dr.c
+++ b/drivers/usb/mtu3/mtu3_dr.c
@@ -322,23 +322,65 @@ static const struct file_operations ssusb_mode_fops = {
 	.release = single_release,
 };
 
+static int ssusb_vbus_show(struct seq_file *sf, void *unused)
+{
+	struct ssusb_mtk *ssusb = sf->private;
+	struct otg_switch_mtk *otg_sx = &ssusb->otg_switch;
+
+	seq_printf(sf, "vbus state: %s\n(echo on/off)\n",
+		regulator_is_enabled(otg_sx->vbus) ? "on" : "off");
+
+	return 0;
+}
+
+static int ssusb_vbus_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, ssusb_vbus_show, inode->i_private);
+}
+
+static ssize_t ssusb_vbus_write(struct file *file,
+	const char __user *ubuf, size_t count, loff_t *ppos)
+{
+	struct seq_file *sf = file->private_data;
+	struct ssusb_mtk *ssusb = sf->private;
+	struct otg_switch_mtk *otg_sx = &ssusb->otg_switch;
+	char buf[16];
+	bool enable;
+
+	if (copy_from_user(&buf, ubuf, min_t(size_t, sizeof(buf) - 1, count)))
+		return -EFAULT;
+
+	if (kstrtobool(buf, &enable)) {
+		dev_err(ssusb->dev, "wrong setting\n");
+		return -EINVAL;
+	}
+
+	ssusb_set_vbus(otg_sx, enable);
+
+	return count;
+}
+
+static const struct file_operations ssusb_vbus_fops = {
+	.open = ssusb_vbus_open,
+	.write = ssusb_vbus_write,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
 static void ssusb_debugfs_init(struct ssusb_mtk *ssusb)
 {
 	struct dentry *root;
-	struct dentry *file;
 
 	root = debugfs_create_dir(dev_name(ssusb->dev), usb_debug_root);
-	if (IS_ERR_OR_NULL(root)) {
-		if (!root)
-			dev_err(ssusb->dev, "create debugfs root failed\n");
+	if (!root) {
+		dev_err(ssusb->dev, "create debugfs root failed\n");
 		return;
 	}
 	ssusb->dbgfs_root = root;
 
-	file = debugfs_create_file("mode", S_IRUGO | S_IWUSR, root,
-			ssusb, &ssusb_mode_fops);
-	if (!file)
-		dev_dbg(ssusb->dev, "create debugfs mode failed\n");
+	debugfs_create_file("mode", 0644, root, ssusb, &ssusb_mode_fops);
+	debugfs_create_file("vbus", 0644, root, ssusb, &ssusb_vbus_fops);
 }
 
 static void ssusb_debugfs_exit(struct ssusb_mtk *ssusb)
diff --git a/drivers/usb/mtu3/mtu3_gadget.c b/drivers/usb/mtu3/mtu3_gadget.c
index 9dd2441b4fa1..434fca58143c 100644
--- a/drivers/usb/mtu3/mtu3_gadget.c
+++ b/drivers/usb/mtu3/mtu3_gadget.c
@@ -663,6 +663,7 @@ int mtu3_gadget_setup(struct mtu3 *mtu)
 	mtu->g.sg_supported = 0;
 	mtu->g.name = MTU3_DRIVER_NAME;
 	mtu->is_active = 0;
+	mtu->delayed_status = false;
 
 	mtu3_gadget_init_eps(mtu);
 
@@ -727,4 +728,7 @@ void mtu3_gadget_reset(struct mtu3 *mtu)
 	mtu->address = 0;
 	mtu->ep0_state = MU3D_EP0_STATE_SETUP;
 	mtu->may_wakeup = 0;
+	mtu->u1_enable = 0;
+	mtu->u2_enable = 0;
+	mtu->delayed_status = false;
 }
diff --git a/drivers/usb/mtu3/mtu3_gadget_ep0.c b/drivers/usb/mtu3/mtu3_gadget_ep0.c
index 2d7427b48775..958d74dd2b78 100644
--- a/drivers/usb/mtu3/mtu3_gadget_ep0.c
+++ b/drivers/usb/mtu3/mtu3_gadget_ep0.c
@@ -16,6 +16,8 @@
  *
  */
 
+#include <linux/usb/composite.h>
+
 #include "mtu3.h"
 
 /* ep0 is always mtu3->in_eps[0] */
@@ -150,6 +152,7 @@ static void ep0_stall_set(struct mtu3_ep *mep0, bool set, u32 pktrdy)
 		csr = (csr & ~EP0_SENDSTALL) | EP0_SENTSTALL;
 	mtu3_writel(mtu->mac_base, U3D_EP0CSR, csr);
 
+	mtu->delayed_status = false;
 	mtu->ep0_state = MU3D_EP0_STATE_SETUP;
 
 	dev_dbg(mtu->dev, "ep0: %s STALL, ep0_state: %s\n",
@@ -656,6 +659,9 @@ stall:
 finish:
 	if (mtu->test_mode) {
 		;	/* nothing to do */
+	} else if (handled == USB_GADGET_DELAYED_STATUS) {
+		/* handle the delay STATUS phase till receive ep_queue on ep0 */
+		mtu->delayed_status = true;
 	} else if (le16_to_cpu(setup.wLength) == 0) { /* no data stage */
 
 		mtu3_writel(mbase, U3D_EP0CSR,
@@ -775,9 +781,6 @@ static int ep0_queue(struct mtu3_ep *mep, struct mtu3_request *mreq)
 	dev_dbg(mtu->dev, "%s %s (ep0_state: %s), len#%d\n", __func__,
 		mep->name, decode_ep0_state(mtu), mreq->request.length);
 
-	if (!list_empty(&mep->req_list))
-		return -EBUSY;
-
 	switch (mtu->ep0_state) {
 	case MU3D_EP0_STATE_SETUP:
 	case MU3D_EP0_STATE_RX:	/* control-OUT data */
@@ -789,6 +792,20 @@ static int ep0_queue(struct mtu3_ep *mep, struct mtu3_request *mreq)
 		return -EINVAL;
 	}
 
+	if (mtu->delayed_status) {
+		u32 csr;
+
+		mtu->delayed_status = false;
+		csr = mtu3_readl(mtu->mac_base, U3D_EP0CSR) & EP0_W1C_BITS;
+		csr |= EP0_SETUPPKTRDY | EP0_DATAEND;
+		mtu3_writel(mtu->mac_base, U3D_EP0CSR, csr);
+		/* needn't giveback the request for handling delay STATUS */
+		return 0;
+	}
+
+	if (!list_empty(&mep->req_list))
+		return -EBUSY;
+
 	list_add_tail(&mreq->list, &mep->req_list);
 
 	/* sequence #1, IN ... start writing the data */
diff --git a/drivers/usb/mtu3/mtu3_host.c b/drivers/usb/mtu3/mtu3_host.c
index cd4d01087855..e42d308b8dc2 100644
--- a/drivers/usb/mtu3/mtu3_host.c
+++ b/drivers/usb/mtu3/mtu3_host.c
@@ -258,8 +258,8 @@ int ssusb_host_init(struct ssusb_mtk *ssusb, struct device_node *parent_dn)
 
 	ret = of_platform_populate(parent_dn, NULL, NULL, parent_dev);
 	if (ret) {
-		dev_dbg(parent_dev, "failed to create child devices at %s\n",
-				parent_dn->full_name);
+		dev_dbg(parent_dev, "failed to create child devices at %pOF\n",
+				parent_dn);
 		return ret;
 	}
 
diff --git a/drivers/usb/mtu3/mtu3_hw_regs.h b/drivers/usb/mtu3/mtu3_hw_regs.h
index 212367295276..06b29664470f 100644
--- a/drivers/usb/mtu3/mtu3_hw_regs.h
+++ b/drivers/usb/mtu3/mtu3_hw_regs.h
@@ -462,10 +462,12 @@
 #define SSUSB_U3_PORT_DIS		BIT(0)
 
 /* U3D_SSUSB_U2_CTRL_0P */
+#define SSUSB_U2_PORT_VBUSVALID	BIT(9)
 #define SSUSB_U2_PORT_OTG_SEL		BIT(7)
-#define SSUSB_U2_PORT_HOST_SEL		BIT(2)
+#define SSUSB_U2_PORT_HOST		BIT(2)
 #define SSUSB_U2_PORT_PDN		BIT(1)
 #define SSUSB_U2_PORT_DIS		BIT(0)
+#define SSUSB_U2_PORT_HOST_SEL	(SSUSB_U2_PORT_VBUSVALID | SSUSB_U2_PORT_HOST)
 
 /* U3D_SSUSB_DEV_RST_CTRL */
 #define SSUSB_DEV_SW_RST		BIT(0)
diff --git a/drivers/usb/mtu3/mtu3_plat.c b/drivers/usb/mtu3/mtu3_plat.c
index 0d3ebb353e08..088e3e685c4f 100644
--- a/drivers/usb/mtu3/mtu3_plat.c
+++ b/drivers/usb/mtu3/mtu3_plat.c
@@ -500,6 +500,7 @@ static const struct dev_pm_ops mtu3_pm_ops = {
 
 static const struct of_device_id mtu3_of_match[] = {
 	{.compatible = "mediatek,mt8173-mtu3",},
+	{.compatible = "mediatek,mtu3",},
 	{},
 };
 
diff --git a/drivers/usb/musb/musb_core.c b/drivers/usb/musb/musb_core.c
index 87cbd56cc761..029692053dd3 100644
--- a/drivers/usb/musb/musb_core.c
+++ b/drivers/usb/musb/musb_core.c
@@ -1156,8 +1156,8 @@ static struct musb_fifo_cfg mode_2_cfg[] = {
 { .hw_ep_num = 1, .style = FIFO_RX,   .maxpacket = 512, },
 { .hw_ep_num = 2, .style = FIFO_TX,   .maxpacket = 512, },
 { .hw_ep_num = 2, .style = FIFO_RX,   .maxpacket = 512, },
-{ .hw_ep_num = 3, .style = FIFO_RXTX, .maxpacket = 256, },
-{ .hw_ep_num = 4, .style = FIFO_RXTX, .maxpacket = 256, },
+{ .hw_ep_num = 3, .style = FIFO_RXTX, .maxpacket = 960, },
+{ .hw_ep_num = 4, .style = FIFO_RXTX, .maxpacket = 1024, },
 };
 
 /* mode 3 - fits in 4KB */
@@ -2671,6 +2671,13 @@ static int musb_suspend(struct device *dev)
 {
 	struct musb	*musb = dev_to_musb(dev);
 	unsigned long	flags;
+	int ret;
+
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0) {
+		pm_runtime_put_noidle(dev);
+		return ret;
+	}
 
 	musb_platform_disable(musb);
 	musb_disable_interrupts(musb);
@@ -2721,14 +2728,6 @@ static int musb_resume(struct device *dev)
 	if ((devctl & mask) != (musb->context.devctl & mask))
 		musb->port1_status = 0;
 
-	/*
-	 * The USB HUB code expects the device to be in RPM_ACTIVE once it came
-	 * out of suspend
-	 */
-	pm_runtime_disable(dev);
-	pm_runtime_set_active(dev);
-	pm_runtime_enable(dev);
-
 	musb_start(musb);
 
 	spin_lock_irqsave(&musb->lock, flags);
@@ -2738,6 +2737,9 @@ static int musb_resume(struct device *dev)
 			error);
 	spin_unlock_irqrestore(&musb->lock, flags);
 
+	pm_runtime_mark_last_busy(dev);
+	pm_runtime_put_autosuspend(dev);
+
 	return 0;
 }
 
diff --git a/drivers/usb/musb/musb_core.h b/drivers/usb/musb/musb_core.h
index 9f22c5b8ce37..c748f4ac1154 100644
--- a/drivers/usb/musb/musb_core.h
+++ b/drivers/usb/musb/musb_core.h
@@ -465,6 +465,30 @@ static inline struct musb *gadget_to_musb(struct usb_gadget *g)
 	return container_of(g, struct musb, g);
 }
 
+static inline char *musb_ep_xfertype_string(u8 type)
+{
+	char *s;
+
+	switch (type) {
+	case USB_ENDPOINT_XFER_CONTROL:
+		s = "ctrl";
+		break;
+	case USB_ENDPOINT_XFER_ISOC:
+		s = "iso";
+		break;
+	case USB_ENDPOINT_XFER_BULK:
+		s = "bulk";
+		break;
+	case USB_ENDPOINT_XFER_INT:
+		s = "int";
+		break;
+	default:
+		s = "";
+		break;
+	}
+	return s;
+}
+
 #ifdef CONFIG_BLACKFIN
 static inline int musb_read_fifosize(struct musb *musb,
 		struct musb_hw_ep *hw_ep, u8 epnum)
diff --git a/drivers/usb/musb/musb_dsps.c b/drivers/usb/musb/musb_dsps.c
index bc6a9be2ccc5..f6b526606ad1 100644
--- a/drivers/usb/musb/musb_dsps.c
+++ b/drivers/usb/musb/musb_dsps.c
@@ -1015,13 +1015,20 @@ static int dsps_suspend(struct device *dev)
 	const struct dsps_musb_wrapper *wrp = glue->wrp;
 	struct musb *musb = platform_get_drvdata(glue->musb);
 	void __iomem *mbase;
-
-	del_timer_sync(&glue->timer);
+	int ret;
 
 	if (!musb)
 		/* This can happen if the musb device is in -EPROBE_DEFER */
 		return 0;
 
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0) {
+		pm_runtime_put_noidle(dev);
+		return ret;
+	}
+
+	del_timer_sync(&glue->timer);
+
 	mbase = musb->ctrl_base;
 	glue->context.control = musb_readl(mbase, wrp->control);
 	glue->context.epintr = musb_readl(mbase, wrp->epintr_set);
@@ -1060,6 +1067,8 @@ static int dsps_resume(struct device *dev)
 	    musb->port_mode == MUSB_PORT_MODE_DUAL_ROLE)
 		dsps_mod_timer(glue, -1);
 
+	pm_runtime_put(dev);
+
 	return 0;
 }
 #endif
diff --git a/drivers/usb/musb/musb_gadget.c b/drivers/usb/musb/musb_gadget.c
index 1acc4864f9f6..bc6d1717c9ec 100644
--- a/drivers/usb/musb/musb_gadget.c
+++ b/drivers/usb/musb/musb_gadget.c
@@ -1105,11 +1105,7 @@ static int musb_gadget_enable(struct usb_ep *ep,
 
 	pr_debug("%s periph: enabled %s for %s %s, %smaxpacket %d\n",
 			musb_driver_name, musb_ep->end_point.name,
-			({ char *s; switch (musb_ep->type) {
-			case USB_ENDPOINT_XFER_BULK:	s = "bulk"; break;
-			case USB_ENDPOINT_XFER_INT:	s = "int"; break;
-			default:			s = "iso"; break;
-			} s; }),
+			musb_ep_xfertype_string(musb_ep->type),
 			musb_ep->is_in ? "IN" : "OUT",
 			musb_ep->dma ? "dma, " : "",
 			musb_ep->packet_sz);
diff --git a/drivers/usb/musb/musb_host.c b/drivers/usb/musb/musb_host.c
index 3344ffd5bb13..b17450a59882 100644
--- a/drivers/usb/musb/musb_host.c
+++ b/drivers/usb/musb/musb_host.c
@@ -2152,6 +2152,10 @@ static int musb_schedule(
 				(USB_SPEED_HIGH == qh->dev->speed) ? 8 : 4;
 		goto success;
 	} else if (best_end < 0) {
+		dev_err(musb->controller,
+				"%s hwep alloc failed for %dx%d\n",
+				musb_ep_xfertype_string(qh->type),
+				qh->hb_mult, qh->maxpacket);
 		return -ENOSPC;
 	}
 
@@ -2244,6 +2248,10 @@ static int musb_urb_enqueue(
 			ok = (usb_pipein(urb->pipe) && musb->hb_iso_rx)
 				|| (usb_pipeout(urb->pipe) && musb->hb_iso_tx);
 		if (!ok) {
+			dev_err(musb->controller,
+				"high bandwidth %s (%dx%d) not supported\n",
+				musb_ep_xfertype_string(qh->type),
+				qh->hb_mult, qh->maxpacket & 0x7ff);
 			ret = -EMSGSIZE;
 			goto done;
 		}
diff --git a/drivers/usb/phy/phy-isp1301-omap.c b/drivers/usb/phy/phy-isp1301-omap.c
index 042c5a8fd423..c6052c814bcc 100644
--- a/drivers/usb/phy/phy-isp1301-omap.c
+++ b/drivers/usb/phy/phy-isp1301-omap.c
@@ -96,7 +96,7 @@ struct isp1301 {
 
 #if IS_REACHABLE(CONFIG_TPS65010)
 
-#include <linux/i2c/tps65010.h>
+#include <linux/mfd/tps65010.h>
 
 #else
 
diff --git a/drivers/usb/phy/phy-mv-usb.c b/drivers/usb/phy/phy-mv-usb.c
index 697a741a0cb1..0e315694adc9 100644
--- a/drivers/usb/phy/phy-mv-usb.c
+++ b/drivers/usb/phy/phy-mv-usb.c
@@ -29,10 +29,8 @@
 #include "phy-mv-usb.h"
 
 #define	DRIVER_DESC	"Marvell USB OTG transceiver driver"
-#define	DRIVER_VERSION	"Jan 20, 2010"
 
 MODULE_DESCRIPTION(DRIVER_DESC);
-MODULE_VERSION(DRIVER_VERSION);
 MODULE_LICENSE("GPL");
 
 static const char driver_name[] = "mv-otg";
@@ -650,7 +648,7 @@ static struct attribute *inputs_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group inputs_attr_group = {
+static const struct attribute_group inputs_attr_group = {
 	.name = "inputs",
 	.attrs = inputs_attrs,
 };
diff --git a/drivers/usb/phy/phy-qcom-8x16-usb.c b/drivers/usb/phy/phy-qcom-8x16-usb.c
index b6a83a5cbad3..679afeaaa9a8 100644
--- a/drivers/usb/phy/phy-qcom-8x16-usb.c
+++ b/drivers/usb/phy/phy-qcom-8x16-usb.c
@@ -270,12 +270,9 @@ static int phy_8x16_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, qphy);
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res)
-		return -EINVAL;
-
-	qphy->regs = devm_ioremap(&pdev->dev, res->start, resource_size(res));
-	if (!qphy->regs)
-		return -ENOMEM;
+	qphy->regs = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(qphy->regs))
+		return PTR_ERR(qphy->regs);
 
 	phy			= &qphy->phy;
 	phy->dev		= &pdev->dev;
diff --git a/drivers/usb/phy/phy-tahvo.c b/drivers/usb/phy/phy-tahvo.c
index a31c8682e998..8babd318c0ed 100644
--- a/drivers/usb/phy/phy-tahvo.c
+++ b/drivers/usb/phy/phy-tahvo.c
@@ -326,7 +326,7 @@ static struct attribute *tahvo_attributes[] = {
 	NULL
 };
 
-static struct attribute_group tahvo_attr_group = {
+static const struct attribute_group tahvo_attr_group = {
 	.attrs = tahvo_attributes,
 };
 
diff --git a/drivers/usb/phy/phy-twl6030-usb.c b/drivers/usb/phy/phy-twl6030-usb.c
index 628b600b02b1..b5dc077ed7d3 100644
--- a/drivers/usb/phy/phy-twl6030-usb.c
+++ b/drivers/usb/phy/phy-twl6030-usb.c
@@ -28,7 +28,7 @@
 #include <linux/usb/musb.h>
 #include <linux/usb/phy_companion.h>
 #include <linux/phy/omap_usb.h>
-#include <linux/i2c/twl.h>
+#include <linux/mfd/twl.h>
 #include <linux/regulator/consumer.h>
 #include <linux/err.h>
 #include <linux/slab.h>
diff --git a/drivers/usb/phy/phy.c b/drivers/usb/phy/phy.c
index 032f5afaad4b..89f4ac4cd93e 100644
--- a/drivers/usb/phy/phy.c
+++ b/drivers/usb/phy/phy.c
@@ -18,6 +18,18 @@
 
 #include <linux/usb/phy.h>
 
+/* Default current range by charger type. */
+#define DEFAULT_SDP_CUR_MIN	2
+#define DEFAULT_SDP_CUR_MAX	500
+#define DEFAULT_SDP_CUR_MIN_SS	150
+#define DEFAULT_SDP_CUR_MAX_SS	900
+#define DEFAULT_DCP_CUR_MIN	500
+#define DEFAULT_DCP_CUR_MAX	5000
+#define DEFAULT_CDP_CUR_MIN	1500
+#define DEFAULT_CDP_CUR_MAX	5000
+#define DEFAULT_ACA_CUR_MIN	1500
+#define DEFAULT_ACA_CUR_MAX	5000
+
 static LIST_HEAD(phy_list);
 static LIST_HEAD(phy_bind_list);
 static DEFINE_SPINLOCK(phy_lock);
@@ -77,6 +89,221 @@ static struct usb_phy *__of_usb_find_phy(struct device_node *node)
 	return ERR_PTR(-EPROBE_DEFER);
 }
 
+static void usb_phy_set_default_current(struct usb_phy *usb_phy)
+{
+	usb_phy->chg_cur.sdp_min = DEFAULT_SDP_CUR_MIN;
+	usb_phy->chg_cur.sdp_max = DEFAULT_SDP_CUR_MAX;
+	usb_phy->chg_cur.dcp_min = DEFAULT_DCP_CUR_MIN;
+	usb_phy->chg_cur.dcp_max = DEFAULT_DCP_CUR_MAX;
+	usb_phy->chg_cur.cdp_min = DEFAULT_CDP_CUR_MIN;
+	usb_phy->chg_cur.cdp_max = DEFAULT_CDP_CUR_MAX;
+	usb_phy->chg_cur.aca_min = DEFAULT_ACA_CUR_MIN;
+	usb_phy->chg_cur.aca_max = DEFAULT_ACA_CUR_MAX;
+}
+
+/**
+ * usb_phy_notify_charger_work - notify the USB charger state
+ * @work - the charger work to notify the USB charger state
+ *
+ * This work can be issued when USB charger state has been changed or
+ * USB charger current has been changed, then we can notify the current
+ * what can be drawn to power user and the charger state to userspace.
+ *
+ * If we get the charger type from extcon subsystem, we can notify the
+ * charger state to power user automatically by usb_phy_get_charger_type()
+ * issuing from extcon subsystem.
+ *
+ * If we get the charger type from ->charger_detect() instead of extcon
+ * subsystem, the usb phy driver should issue usb_phy_set_charger_state()
+ * to set charger state when the charger state has been changed.
+ */
+static void usb_phy_notify_charger_work(struct work_struct *work)
+{
+	struct usb_phy *usb_phy = container_of(work, struct usb_phy, chg_work);
+	char uchger_state[50] = { 0 };
+	char *envp[] = { uchger_state, NULL };
+	unsigned int min, max;
+
+	switch (usb_phy->chg_state) {
+	case USB_CHARGER_PRESENT:
+		usb_phy_get_charger_current(usb_phy, &min, &max);
+
+		atomic_notifier_call_chain(&usb_phy->notifier, max, usb_phy);
+		snprintf(uchger_state, ARRAY_SIZE(uchger_state),
+			 "USB_CHARGER_STATE=%s", "USB_CHARGER_PRESENT");
+		break;
+	case USB_CHARGER_ABSENT:
+		usb_phy_set_default_current(usb_phy);
+
+		atomic_notifier_call_chain(&usb_phy->notifier, 0, usb_phy);
+		snprintf(uchger_state, ARRAY_SIZE(uchger_state),
+			 "USB_CHARGER_STATE=%s", "USB_CHARGER_ABSENT");
+		break;
+	default:
+		dev_warn(usb_phy->dev, "Unknown USB charger state: %d\n",
+			 usb_phy->chg_state);
+		return;
+	}
+
+	kobject_uevent_env(&usb_phy->dev->kobj, KOBJ_CHANGE, envp);
+}
+
+static void __usb_phy_get_charger_type(struct usb_phy *usb_phy)
+{
+	if (extcon_get_state(usb_phy->edev, EXTCON_CHG_USB_SDP) > 0) {
+		usb_phy->chg_type = SDP_TYPE;
+		usb_phy->chg_state = USB_CHARGER_PRESENT;
+	} else if (extcon_get_state(usb_phy->edev, EXTCON_CHG_USB_CDP) > 0) {
+		usb_phy->chg_type = CDP_TYPE;
+		usb_phy->chg_state = USB_CHARGER_PRESENT;
+	} else if (extcon_get_state(usb_phy->edev, EXTCON_CHG_USB_DCP) > 0) {
+		usb_phy->chg_type = DCP_TYPE;
+		usb_phy->chg_state = USB_CHARGER_PRESENT;
+	} else if (extcon_get_state(usb_phy->edev, EXTCON_CHG_USB_ACA) > 0) {
+		usb_phy->chg_type = ACA_TYPE;
+		usb_phy->chg_state = USB_CHARGER_PRESENT;
+	} else {
+		usb_phy->chg_type = UNKNOWN_TYPE;
+		usb_phy->chg_state = USB_CHARGER_ABSENT;
+	}
+
+	schedule_work(&usb_phy->chg_work);
+}
+
+/**
+ * usb_phy_get_charger_type - get charger type from extcon subsystem
+ * @nb -the notifier block to determine charger type
+ * @state - the cable state
+ * @data - private data
+ *
+ * Determin the charger type from extcon subsystem which also means the
+ * charger state has been chaned, then we should notify this event.
+ */
+static int usb_phy_get_charger_type(struct notifier_block *nb,
+				    unsigned long state, void *data)
+{
+	struct usb_phy *usb_phy = container_of(nb, struct usb_phy, type_nb);
+
+	__usb_phy_get_charger_type(usb_phy);
+	return NOTIFY_OK;
+}
+
+/**
+ * usb_phy_set_charger_current - set the USB charger current
+ * @usb_phy - the USB phy to be used
+ * @mA - the current need to be set
+ *
+ * Usually we only change the charger default current when USB finished the
+ * enumeration as one SDP charger. As one SDP charger, usb_phy_set_power()
+ * will issue this function to change charger current when after setting USB
+ * configuration, or suspend/resume USB. For other type charger, we should
+ * use the default charger current and we do not suggest to issue this function
+ * to change the charger current.
+ *
+ * When USB charger current has been changed, we need to notify the power users.
+ */
+void usb_phy_set_charger_current(struct usb_phy *usb_phy, unsigned int mA)
+{
+	switch (usb_phy->chg_type) {
+	case SDP_TYPE:
+		if (usb_phy->chg_cur.sdp_max == mA)
+			return;
+
+		usb_phy->chg_cur.sdp_max = (mA > DEFAULT_SDP_CUR_MAX_SS) ?
+			DEFAULT_SDP_CUR_MAX_SS : mA;
+		break;
+	case DCP_TYPE:
+		if (usb_phy->chg_cur.dcp_max == mA)
+			return;
+
+		usb_phy->chg_cur.dcp_max = (mA > DEFAULT_DCP_CUR_MAX) ?
+			DEFAULT_DCP_CUR_MAX : mA;
+		break;
+	case CDP_TYPE:
+		if (usb_phy->chg_cur.cdp_max == mA)
+			return;
+
+		usb_phy->chg_cur.cdp_max = (mA > DEFAULT_CDP_CUR_MAX) ?
+			DEFAULT_CDP_CUR_MAX : mA;
+		break;
+	case ACA_TYPE:
+		if (usb_phy->chg_cur.aca_max == mA)
+			return;
+
+		usb_phy->chg_cur.aca_max = (mA > DEFAULT_ACA_CUR_MAX) ?
+			DEFAULT_ACA_CUR_MAX : mA;
+		break;
+	default:
+		return;
+	}
+
+	schedule_work(&usb_phy->chg_work);
+}
+EXPORT_SYMBOL_GPL(usb_phy_set_charger_current);
+
+/**
+ * usb_phy_get_charger_current - get the USB charger current
+ * @usb_phy - the USB phy to be used
+ * @min - the minimum current
+ * @max - the maximum current
+ *
+ * Usually we will notify the maximum current to power user, but for some
+ * special case, power user also need the minimum current value. Then the
+ * power user can issue this function to get the suitable current.
+ */
+void usb_phy_get_charger_current(struct usb_phy *usb_phy,
+				 unsigned int *min, unsigned int *max)
+{
+	switch (usb_phy->chg_type) {
+	case SDP_TYPE:
+		*min = usb_phy->chg_cur.sdp_min;
+		*max = usb_phy->chg_cur.sdp_max;
+		break;
+	case DCP_TYPE:
+		*min = usb_phy->chg_cur.dcp_min;
+		*max = usb_phy->chg_cur.dcp_max;
+		break;
+	case CDP_TYPE:
+		*min = usb_phy->chg_cur.cdp_min;
+		*max = usb_phy->chg_cur.cdp_max;
+		break;
+	case ACA_TYPE:
+		*min = usb_phy->chg_cur.aca_min;
+		*max = usb_phy->chg_cur.aca_max;
+		break;
+	default:
+		*min = 0;
+		*max = 0;
+		break;
+	}
+}
+EXPORT_SYMBOL_GPL(usb_phy_get_charger_current);
+
+/**
+ * usb_phy_set_charger_state - set the USB charger state
+ * @usb_phy - the USB phy to be used
+ * @state - the new state need to be set for charger
+ *
+ * The usb phy driver can issue this function when the usb phy driver
+ * detected the charger state has been changed, in this case the charger
+ * type should be get from ->charger_detect().
+ */
+void usb_phy_set_charger_state(struct usb_phy *usb_phy,
+			       enum usb_charger_state state)
+{
+	if (usb_phy->chg_state == state || !usb_phy->charger_detect)
+		return;
+
+	usb_phy->chg_state = state;
+	if (usb_phy->chg_state == USB_CHARGER_PRESENT)
+		usb_phy->chg_type = usb_phy->charger_detect(usb_phy);
+	else
+		usb_phy->chg_type = UNKNOWN_TYPE;
+
+	schedule_work(&usb_phy->chg_work);
+}
+EXPORT_SYMBOL_GPL(usb_phy_set_charger_state);
+
 static void devm_usb_phy_release(struct device *dev, void *res)
 {
 	struct usb_phy *phy = *(struct usb_phy **)res;
@@ -124,6 +351,44 @@ static int usb_add_extcon(struct usb_phy *x)
 					"register VBUS notifier failed\n");
 				return ret;
 			}
+		} else {
+			x->type_nb.notifier_call = usb_phy_get_charger_type;
+
+			ret = devm_extcon_register_notifier(x->dev, x->edev,
+							    EXTCON_CHG_USB_SDP,
+							    &x->type_nb);
+			if (ret) {
+				dev_err(x->dev,
+					"register extcon USB SDP failed.\n");
+				return ret;
+			}
+
+			ret = devm_extcon_register_notifier(x->dev, x->edev,
+							    EXTCON_CHG_USB_CDP,
+							    &x->type_nb);
+			if (ret) {
+				dev_err(x->dev,
+					"register extcon USB CDP failed.\n");
+				return ret;
+			}
+
+			ret = devm_extcon_register_notifier(x->dev, x->edev,
+							    EXTCON_CHG_USB_DCP,
+							    &x->type_nb);
+			if (ret) {
+				dev_err(x->dev,
+					"register extcon USB DCP failed.\n");
+				return ret;
+			}
+
+			ret = devm_extcon_register_notifier(x->dev, x->edev,
+							    EXTCON_CHG_USB_ACA,
+							    &x->type_nb);
+			if (ret) {
+				dev_err(x->dev,
+					"register extcon USB ACA failed.\n");
+				return ret;
+			}
 		}
 
 		if (x->id_nb.notifier_call) {
@@ -145,6 +410,13 @@ static int usb_add_extcon(struct usb_phy *x)
 		}
 	}
 
+	usb_phy_set_default_current(x);
+	INIT_WORK(&x->chg_work, usb_phy_notify_charger_work);
+	x->chg_type = UNKNOWN_TYPE;
+	x->chg_state = USB_CHARGER_DEFAULT;
+	if (x->type_nb.notifier_call)
+		__usb_phy_get_charger_type(x);
+
 	return 0;
 }
 
@@ -302,8 +574,8 @@ struct usb_phy *devm_usb_get_phy_by_phandle(struct device *dev,
 
 	node = of_parse_phandle(dev->of_node, phandle, index);
 	if (!node) {
-		dev_dbg(dev, "failed to get %s phandle in %s node\n", phandle,
-			dev->of_node->full_name);
+		dev_dbg(dev, "failed to get %s phandle in %pOF node\n", phandle,
+			dev->of_node);
 		return ERR_PTR(-ENODEV);
 	}
 	phy = devm_usb_get_phy_by_node(dev, node, NULL);
diff --git a/drivers/usb/renesas_usbhs/mod_gadget.c b/drivers/usb/renesas_usbhs/mod_gadget.c
index 2c8161bcf5b5..c068b673420b 100644
--- a/drivers/usb/renesas_usbhs/mod_gadget.c
+++ b/drivers/usb/renesas_usbhs/mod_gadget.c
@@ -764,7 +764,7 @@ static int usbhsg_ep_set_wedge(struct usb_ep *ep)
 	return __usbhsg_ep_set_halt_wedge(ep, 1, 1);
 }
 
-static struct usb_ep_ops usbhsg_ep_ops = {
+static const struct usb_ep_ops usbhsg_ep_ops = {
 	.enable		= usbhsg_ep_enable,
 	.disable	= usbhsg_ep_disable,
 
@@ -1082,7 +1082,6 @@ int usbhs_mod_gadget_probe(struct usbhs_priv *priv)
 		ret = -ENOMEM;
 		goto usbhs_mod_gadget_probe_err_gpriv;
 	}
-	spin_lock_init(&uep->lock);
 
 	gpriv->transceiver = usb_get_phy(USB_PHY_TYPE_UNDEFINED);
 	dev_info(dev, "%stransceiver found\n",
@@ -1132,6 +1131,7 @@ int usbhs_mod_gadget_probe(struct usbhs_priv *priv)
 		uep->ep.name		= uep->ep_name;
 		uep->ep.ops		= &usbhsg_ep_ops;
 		INIT_LIST_HEAD(&uep->ep.ep_list);
+		spin_lock_init(&uep->lock);
 
 		/* init DCP */
 		if (usbhsg_is_dcp(uep)) {
diff --git a/drivers/usb/renesas_usbhs/mod_host.c b/drivers/usb/renesas_usbhs/mod_host.c
index dfb346e9bd0c..e256351cb72d 100644
--- a/drivers/usb/renesas_usbhs/mod_host.c
+++ b/drivers/usb/renesas_usbhs/mod_host.c
@@ -1285,7 +1285,7 @@ static int usbhsh_bus_nop(struct usb_hcd *hcd)
 	return 0;
 }
 
-static struct hc_driver usbhsh_driver = {
+static const struct hc_driver usbhsh_driver = {
 	.description =		usbhsh_hcd_name,
 	.hcd_priv_size =	sizeof(struct usbhsh_hpriv),
 
diff --git a/drivers/usb/renesas_usbhs/pipe.c b/drivers/usb/renesas_usbhs/pipe.c
index 9396a8c14af8..d811f0550c04 100644
--- a/drivers/usb/renesas_usbhs/pipe.c
+++ b/drivers/usb/renesas_usbhs/pipe.c
@@ -401,7 +401,7 @@ static int usbhsp_setup_pipecfg(struct usbhs_pipe *pipe, int is_host,
 	u16 dir = 0;
 	u16 epnum = 0;
 	u16 shtnak = 0;
-	u16 type_array[] = {
+	static const u16 type_array[] = {
 		[USB_ENDPOINT_XFER_BULK] = TYPE_BULK,
 		[USB_ENDPOINT_XFER_INT]  = TYPE_INT,
 		[USB_ENDPOINT_XFER_ISOC] = TYPE_ISO,
diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c
index fe123153b1a5..54bfef13966a 100644
--- a/drivers/usb/serial/option.c
+++ b/drivers/usb/serial/option.c
@@ -2016,13 +2016,11 @@ static const struct usb_device_id option_ids[] = {
 	{ USB_DEVICE(TPLINK_VENDOR_ID, 0x9000),					/* TP-Link MA260 */
 	  .driver_info = (kernel_ulong_t)&net_intf4_blacklist },
 	{ USB_DEVICE(CHANGHONG_VENDOR_ID, CHANGHONG_PRODUCT_CH690) },
-	{ USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x7d01, 0xff, 0x02, 0x01) },	/* D-Link DWM-156 (variant) */
-	{ USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x7d01, 0xff, 0x00, 0x00) },	/* D-Link DWM-156 (variant) */
-	{ USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x7d02, 0xff, 0x02, 0x01) },
-	{ USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x7d02, 0xff, 0x00, 0x00) },
-	{ USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x7d03, 0xff, 0x02, 0x01) },
-	{ USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x7d03, 0xff, 0x00, 0x00) },
+	{ USB_DEVICE_INTERFACE_CLASS(0x2001, 0x7d01, 0xff) },			/* D-Link DWM-156 (variant) */
+	{ USB_DEVICE_INTERFACE_CLASS(0x2001, 0x7d02, 0xff) },
+	{ USB_DEVICE_INTERFACE_CLASS(0x2001, 0x7d03, 0xff) },
 	{ USB_DEVICE_INTERFACE_CLASS(0x2001, 0x7d04, 0xff) },			/* D-Link DWM-158 */
+	{ USB_DEVICE_INTERFACE_CLASS(0x2001, 0x7d0e, 0xff) },			/* D-Link DWM-157 C1 */
 	{ USB_DEVICE_INTERFACE_CLASS(0x2001, 0x7e19, 0xff),			/* D-Link DWM-221 B1 */
 	  .driver_info = (kernel_ulong_t)&net_intf4_blacklist },
 	{ USB_DEVICE_INTERFACE_CLASS(0x2001, 0x7e35, 0xff),			/* D-Link DWM-222 */
diff --git a/drivers/usb/storage/realtek_cr.c b/drivers/usb/storage/realtek_cr.c
index 4176d1af9bf2..ec83b3b5efa9 100644
--- a/drivers/usb/storage/realtek_cr.c
+++ b/drivers/usb/storage/realtek_cr.c
@@ -47,7 +47,6 @@
 MODULE_DESCRIPTION("Driver for Realtek USB Card Reader");
 MODULE_AUTHOR("wwang <wei_wang@realsil.com.cn>");
 MODULE_LICENSE("GPL");
-MODULE_VERSION("1.03");
 
 static int auto_delink_en = 1;
 module_param(auto_delink_en, int, S_IRUGO | S_IWUSR);
diff --git a/drivers/usb/storage/uas.c b/drivers/usb/storage/uas.c
index 5ef014ba6ae8..cfb1e3bbd434 100644
--- a/drivers/usb/storage/uas.c
+++ b/drivers/usb/storage/uas.c
@@ -737,7 +737,7 @@ static int uas_eh_abort_handler(struct scsi_cmnd *cmnd)
 	return FAILED;
 }
 
-static int uas_eh_bus_reset_handler(struct scsi_cmnd *cmnd)
+static int uas_eh_device_reset_handler(struct scsi_cmnd *cmnd)
 {
 	struct scsi_device *sdev = cmnd->device;
 	struct uas_dev_info *devinfo = sdev->hostdata;
@@ -848,7 +848,7 @@ static struct scsi_host_template uas_host_template = {
 	.slave_alloc = uas_slave_alloc,
 	.slave_configure = uas_slave_configure,
 	.eh_abort_handler = uas_eh_abort_handler,
-	.eh_bus_reset_handler = uas_eh_bus_reset_handler,
+	.eh_device_reset_handler = uas_eh_device_reset_handler,
 	.this_id = -1,
 	.sg_tablesize = SG_NONE,
 	.skip_settle_delay = 1,
diff --git a/drivers/usb/usbip/stub_main.c b/drivers/usb/usbip/stub_main.c
index 660180a5d5c4..7170404e8979 100644
--- a/drivers/usb/usbip/stub_main.c
+++ b/drivers/usb/usbip/stub_main.c
@@ -302,7 +302,6 @@ static int __init usbip_host_init(void)
 		goto err_create_file;
 	}
 
-	pr_info(DRIVER_DESC " v" USBIP_VERSION "\n");
 	return ret;
 
 err_create_file:
@@ -335,4 +334,3 @@ module_exit(usbip_host_exit);
 MODULE_AUTHOR(DRIVER_AUTHOR);
 MODULE_DESCRIPTION(DRIVER_DESC);
 MODULE_LICENSE("GPL");
-MODULE_VERSION(USBIP_VERSION);
diff --git a/drivers/usb/usbip/usbip_common.c b/drivers/usb/usbip/usbip_common.c
index cab2b71a80d0..2281f3562870 100644
--- a/drivers/usb/usbip/usbip_common.c
+++ b/drivers/usb/usbip/usbip_common.c
@@ -763,7 +763,6 @@ static int __init usbip_core_init(void)
 {
 	int ret;
 
-	pr_info(DRIVER_DESC " v" USBIP_VERSION "\n");
 	ret = usbip_init_eh();
 	if (ret)
 		return ret;
@@ -783,4 +782,3 @@ module_exit(usbip_core_exit);
 MODULE_AUTHOR(DRIVER_AUTHOR);
 MODULE_DESCRIPTION(DRIVER_DESC);
 MODULE_LICENSE("GPL");
-MODULE_VERSION(USBIP_VERSION);
diff --git a/drivers/usb/usbip/usbip_common.h b/drivers/usb/usbip/usbip_common.h
index f8573a52e41a..3050fc99a417 100644
--- a/drivers/usb/usbip/usbip_common.h
+++ b/drivers/usb/usbip/usbip_common.h
@@ -34,8 +34,6 @@
 #include <linux/sched/task.h>
 #include <uapi/linux/usbip.h>
 
-#define USBIP_VERSION "1.0.0"
-
 #undef pr_fmt
 
 #ifdef DEBUG
diff --git a/drivers/usb/usbip/vhci_hcd.c b/drivers/usb/usbip/vhci_hcd.c
index 2c4b2fd40406..11b9a22799cc 100644
--- a/drivers/usb/usbip/vhci_hcd.c
+++ b/drivers/usb/usbip/vhci_hcd.c
@@ -1274,7 +1274,7 @@ static int vhci_free_streams(struct usb_hcd *hcd, struct usb_device *udev,
 	return 0;
 }
 
-static struct hc_driver vhci_hc_driver = {
+static const struct hc_driver vhci_hc_driver = {
 	.description	= driver_name,
 	.product_desc	= driver_desc,
 	.hcd_priv_size	= sizeof(struct vhci_hcd),
@@ -1516,7 +1516,6 @@ static int __init vhci_hcd_init(void)
 		}
 	}
 
-	pr_info(DRIVER_DESC " v" USBIP_VERSION "\n");
 	return ret;
 
 err_add_hcd:
@@ -1542,4 +1541,3 @@ module_exit(vhci_hcd_exit);
 MODULE_AUTHOR(DRIVER_AUTHOR);
 MODULE_DESCRIPTION(DRIVER_DESC);
 MODULE_LICENSE("GPL");
-MODULE_VERSION(USBIP_VERSION);
diff --git a/drivers/usb/usbip/vhci_sysfs.c b/drivers/usb/usbip/vhci_sysfs.c
index 5778b640ba9c..1b9f60a22e0b 100644
--- a/drivers/usb/usbip/vhci_sysfs.c
+++ b/drivers/usb/usbip/vhci_sysfs.c
@@ -366,7 +366,11 @@ static ssize_t store_attach(struct device *dev, struct device_attribute *attr,
 		sockfd_put(socket);
 
 		dev_err(dev, "port %d already used\n", rhport);
-		return -EINVAL;
+		/*
+		 * Will be retried from userspace
+		 * if there's another free port.
+		 */
+		return -EBUSY;
 	}
 
 	dev_info(dev, "pdev(%u) rhport(%u) sockfd(%d)\n",
diff --git a/drivers/usb/wusbcore/cbaf.c b/drivers/usb/wusbcore/cbaf.c
index fb70cbef0671..aa4e440e9975 100644
--- a/drivers/usb/wusbcore/cbaf.c
+++ b/drivers/usb/wusbcore/cbaf.c
@@ -586,7 +586,7 @@ static struct attribute *cbaf_dev_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group cbaf_dev_attr_group = {
+static const struct attribute_group cbaf_dev_attr_group = {
 	.name = NULL,	/* we want them in the same directory */
 	.attrs = cbaf_dev_attrs,
 };
diff --git a/drivers/usb/wusbcore/dev-sysfs.c b/drivers/usb/wusbcore/dev-sysfs.c
index d4de56b93d68..78212f8180ce 100644
--- a/drivers/usb/wusbcore/dev-sysfs.c
+++ b/drivers/usb/wusbcore/dev-sysfs.c
@@ -114,7 +114,7 @@ static struct attribute *wusb_dev_attrs[] = {
 		NULL,
 };
 
-static struct attribute_group wusb_dev_attr_group = {
+static const struct attribute_group wusb_dev_attr_group = {
 	.name = NULL,	/* we want them in the same directory */
 	.attrs = wusb_dev_attrs,
 };
diff --git a/drivers/usb/wusbcore/wusbhc.c b/drivers/usb/wusbcore/wusbhc.c
index a273a91cf667..5338e42533c8 100644
--- a/drivers/usb/wusbcore/wusbhc.c
+++ b/drivers/usb/wusbcore/wusbhc.c
@@ -244,7 +244,7 @@ static struct attribute *wusbhc_attrs[] = {
 		NULL,
 };
 
-static struct attribute_group wusbhc_attr_group = {
+static const struct attribute_group wusbhc_attr_group = {
 	.name = NULL,	/* we want them in the same directory */
 	.attrs = wusbhc_attrs,
 };
diff --git a/drivers/uwb/lc-rc.c b/drivers/uwb/lc-rc.c
index 97ee1b46db69..b0816c753a63 100644
--- a/drivers/uwb/lc-rc.c
+++ b/drivers/uwb/lc-rc.c
@@ -228,7 +228,7 @@ static struct attribute *rc_attrs[] = {
 		NULL,
 };
 
-static struct attribute_group rc_attr_group = {
+static const struct attribute_group rc_attr_group = {
 	.attrs = rc_attrs,
 };
 
diff --git a/drivers/vfio/platform/vfio_amba.c b/drivers/vfio/platform/vfio_amba.c
index 31372fbf6c5b..62dfbfeaabfc 100644
--- a/drivers/vfio/platform/vfio_amba.c
+++ b/drivers/vfio/platform/vfio_amba.c
@@ -93,7 +93,7 @@ static int vfio_amba_remove(struct amba_device *adev)
 	return -EINVAL;
 }
 
-static struct amba_id pl330_ids[] = {
+static const struct amba_id pl330_ids[] = {
 	{ 0, 0 },
 };
 
diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c
index 330d50582f40..f5a86f651f38 100644
--- a/drivers/vfio/vfio.c
+++ b/drivers/vfio/vfio.c
@@ -85,6 +85,7 @@ struct vfio_group {
 	struct list_head		unbound_list;
 	struct mutex			unbound_lock;
 	atomic_t			opened;
+	wait_queue_head_t		container_q;
 	bool				noiommu;
 	struct kvm			*kvm;
 	struct blocking_notifier_head	notifier;
@@ -138,9 +139,10 @@ struct iommu_group *vfio_iommu_group_get(struct device *dev)
 	iommu_group_set_name(group, "vfio-noiommu");
 	iommu_group_set_iommudata(group, &noiommu, NULL);
 	ret = iommu_group_add_device(group, dev);
-	iommu_group_put(group);
-	if (ret)
+	if (ret) {
+		iommu_group_put(group);
 		return NULL;
+	}
 
 	/*
 	 * Where to taint?  At this point we've added an IOMMU group for a
@@ -337,6 +339,7 @@ static struct vfio_group *vfio_create_group(struct iommu_group *iommu_group)
 	mutex_init(&group->unbound_lock);
 	atomic_set(&group->container_users, 0);
 	atomic_set(&group->opened, 0);
+	init_waitqueue_head(&group->container_q);
 	group->iommu_group = iommu_group;
 #ifdef CONFIG_VFIO_NOIOMMU
 	group->noiommu = (iommu_group_get_iommudata(iommu_group) == &noiommu);
@@ -993,6 +996,23 @@ void *vfio_del_group_dev(struct device *dev)
 		}
 	} while (ret <= 0);
 
+	/*
+	 * In order to support multiple devices per group, devices can be
+	 * plucked from the group while other devices in the group are still
+	 * in use.  The container persists with this group and those remaining
+	 * devices still attached.  If the user creates an isolation violation
+	 * by binding this device to another driver while the group is still in
+	 * use, that's their fault.  However, in the case of removing the last,
+	 * or potentially the only, device in the group there can be no other
+	 * in-use devices in the group.  The user has done their due diligence
+	 * and we should lay no claims to those devices.  In order to do that,
+	 * we need to make sure the group is detached from the container.
+	 * Without this stall, we're potentially racing with a user process
+	 * that may attempt to immediately bind this device to another driver.
+	 */
+	if (list_empty(&group->device_list))
+		wait_event(group->container_q, !group->container);
+
 	vfio_group_put(group);
 
 	return device_data;
@@ -1298,6 +1318,7 @@ static void __vfio_group_unset_container(struct vfio_group *group)
 					  group->iommu_group);
 
 	group->container = NULL;
+	wake_up(&group->container_q);
 	list_del(&group->container_next);
 
 	/* Detaching the last group deprivileges a container, remove iommu */
diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index 8549cb111627..92155cce926d 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -1169,13 +1169,21 @@ static bool vfio_iommu_has_sw_msi(struct iommu_group *group, phys_addr_t *base)
 	INIT_LIST_HEAD(&group_resv_regions);
 	iommu_get_group_resv_regions(group, &group_resv_regions);
 	list_for_each_entry(region, &group_resv_regions, list) {
+		/*
+		 * The presence of any 'real' MSI regions should take
+		 * precedence over the software-managed one if the
+		 * IOMMU driver happens to advertise both types.
+		 */
+		if (region->type == IOMMU_RESV_MSI) {
+			ret = false;
+			break;
+		}
+
 		if (region->type == IOMMU_RESV_SW_MSI) {
 			*base = region->start;
 			ret = true;
-			goto out;
 		}
 	}
-out:
 	list_for_each_entry_safe(region, next, &group_resv_regions, list)
 		kfree(region);
 	return ret;
@@ -1265,8 +1273,8 @@ static int vfio_iommu_type1_attach_group(void *iommu_data,
 	INIT_LIST_HEAD(&domain->group_list);
 	list_add(&group->next, &domain->group_list);
 
-	msi_remap = resv_msi ? irq_domain_check_msi_remap() :
-				iommu_capable(bus, IOMMU_CAP_INTR_REMAP);
+	msi_remap = irq_domain_check_msi_remap() ||
+		    iommu_capable(bus, IOMMU_CAP_INTR_REMAP);
 
 	if (!allow_unsafe_interrupts && !msi_remap) {
 		pr_warn("%s: No interrupt remapping support.  Use the module param \"allow_unsafe_interrupts\" to enable VFIO IOMMU support on this platform\n",
diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
index 06d044862e58..58585ec8699e 100644
--- a/drivers/vhost/net.c
+++ b/drivers/vhost/net.c
@@ -533,6 +533,7 @@ static void handle_tx(struct vhost_net *net)
 			ubuf->callback = vhost_zerocopy_callback;
 			ubuf->ctx = nvq->ubufs;
 			ubuf->desc = nvq->upend_idx;
+			refcount_set(&ubuf->refcnt, 1);
 			msg.msg_control = ubuf;
 			msg.msg_controllen = sizeof(ubuf);
 			ubufs = nvq->ubufs;
@@ -634,8 +635,13 @@ static int vhost_net_rx_peek_head_len(struct vhost_net *net, struct sock *sk)
 
 		preempt_enable();
 
-		if (vhost_enable_notify(&net->dev, vq))
+		if (!vhost_vq_avail_empty(&net->dev, vq))
 			vhost_poll_queue(&vq->poll);
+		else if (unlikely(vhost_enable_notify(&net->dev, vq))) {
+			vhost_disable_notify(&net->dev, vq);
+			vhost_poll_queue(&vq->poll);
+		}
+
 		mutex_unlock(&vq->mutex);
 
 		len = peek_head_len(rvq, sk);
diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
index 9cb3f722dce1..d6dbb28245e6 100644
--- a/drivers/vhost/vhost.c
+++ b/drivers/vhost/vhost.c
@@ -1271,7 +1271,7 @@ static struct vhost_umem *vhost_umem_alloc(void)
 	if (!umem)
 		return NULL;
 
-	umem->umem_tree = RB_ROOT;
+	umem->umem_tree = RB_ROOT_CACHED;
 	umem->numem = 0;
 	INIT_LIST_HEAD(&umem->umem_list);
 
diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h
index bb7c29b8b9fc..d59a9cc65f9d 100644
--- a/drivers/vhost/vhost.h
+++ b/drivers/vhost/vhost.h
@@ -71,7 +71,7 @@ struct vhost_umem_node {
 };
 
 struct vhost_umem {
-	struct rb_root umem_tree;
+	struct rb_root_cached umem_tree;
 	struct list_head umem_list;
 	int numem;
 };
diff --git a/drivers/video/backlight/gpio_backlight.c b/drivers/video/backlight/gpio_backlight.c
index 18134416b154..e470da95d806 100644
--- a/drivers/video/backlight/gpio_backlight.c
+++ b/drivers/video/backlight/gpio_backlight.c
@@ -9,7 +9,8 @@
 #include <linux/backlight.h>
 #include <linux/err.h>
 #include <linux/fb.h>
-#include <linux/gpio.h>
+#include <linux/gpio.h> /* Only for legacy support */
+#include <linux/gpio/consumer.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
@@ -23,8 +24,7 @@ struct gpio_backlight {
 	struct device *dev;
 	struct device *fbdev;
 
-	int gpio;
-	int active;
+	struct gpio_desc *gpiod;
 	int def_value;
 };
 
@@ -38,8 +38,7 @@ static int gpio_backlight_update_status(struct backlight_device *bl)
 	    bl->props.state & (BL_CORE_SUSPENDED | BL_CORE_FBBLANK))
 		brightness = 0;
 
-	gpio_set_value_cansleep(gbl->gpio,
-				brightness ? gbl->active : !gbl->active);
+	gpiod_set_value_cansleep(gbl->gpiod, brightness);
 
 	return 0;
 }
@@ -61,23 +60,25 @@ static const struct backlight_ops gpio_backlight_ops = {
 static int gpio_backlight_probe_dt(struct platform_device *pdev,
 				   struct gpio_backlight *gbl)
 {
-	struct device_node *np = pdev->dev.of_node;
-	enum of_gpio_flags gpio_flags;
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	enum gpiod_flags flags;
+	int ret;
+
+	gbl->def_value = of_property_read_bool(np, "default-on");
+	flags = gbl->def_value ? GPIOD_OUT_HIGH : GPIOD_OUT_LOW;
 
-	gbl->gpio = of_get_gpio_flags(np, 0, &gpio_flags);
+	gbl->gpiod = devm_gpiod_get(dev, NULL, flags);
+	if (IS_ERR(gbl->gpiod)) {
+		ret = PTR_ERR(gbl->gpiod);
 
-	if (!gpio_is_valid(gbl->gpio)) {
-		if (gbl->gpio != -EPROBE_DEFER) {
-			dev_err(&pdev->dev,
+		if (ret != -EPROBE_DEFER) {
+			dev_err(dev,
 				"Error: The gpios parameter is missing or invalid.\n");
 		}
-		return gbl->gpio;
+		return ret;
 	}
 
-	gbl->active = (gpio_flags & OF_GPIO_ACTIVE_LOW) ? 0 : 1;
-
-	gbl->def_value = of_property_read_bool(np, "default-on");
-
 	return 0;
 }
 
@@ -89,7 +90,6 @@ static int gpio_backlight_probe(struct platform_device *pdev)
 	struct backlight_device *bl;
 	struct gpio_backlight *gbl;
 	struct device_node *np = pdev->dev.of_node;
-	unsigned long flags = GPIOF_DIR_OUT;
 	int ret;
 
 	if (!pdata && !np) {
@@ -109,22 +109,26 @@ static int gpio_backlight_probe(struct platform_device *pdev)
 		if (ret)
 			return ret;
 	} else {
+		/*
+		 * Legacy platform data GPIO retrieveal. Do not expand
+		 * the use of this code path, currently only used by one
+		 * SH board.
+		 */
+		unsigned long flags = GPIOF_DIR_OUT;
+
 		gbl->fbdev = pdata->fbdev;
-		gbl->gpio = pdata->gpio;
-		gbl->active = pdata->active_low ? 0 : 1;
 		gbl->def_value = pdata->def_value;
-	}
-
-	if (gbl->active)
 		flags |= gbl->def_value ? GPIOF_INIT_HIGH : GPIOF_INIT_LOW;
-	else
-		flags |= gbl->def_value ? GPIOF_INIT_LOW : GPIOF_INIT_HIGH;
 
-	ret = devm_gpio_request_one(gbl->dev, gbl->gpio, flags,
-				    pdata ? pdata->name : "backlight");
-	if (ret < 0) {
-		dev_err(&pdev->dev, "unable to request GPIO\n");
-		return ret;
+		ret = devm_gpio_request_one(gbl->dev, pdata->gpio, flags,
+					    pdata ? pdata->name : "backlight");
+		if (ret < 0) {
+			dev_err(&pdev->dev, "unable to request GPIO\n");
+			return ret;
+		}
+		gbl->gpiod = gpio_to_desc(pdata->gpio);
+		if (!gbl->gpiod)
+			return -EINVAL;
 	}
 
 	memset(&props, 0, sizeof(props));
diff --git a/drivers/video/backlight/kb3886_bl.c b/drivers/video/backlight/kb3886_bl.c
index 84a110a719cb..96312c3afc07 100644
--- a/drivers/video/backlight/kb3886_bl.c
+++ b/drivers/video/backlight/kb3886_bl.c
@@ -78,7 +78,7 @@ static struct kb3886bl_machinfo *bl_machinfo;
 static unsigned long kb3886bl_flags;
 #define KB3886BL_SUSPENDED     0x01
 
-static struct dmi_system_id kb3886bl_device_table[] __initdata = {
+static const struct dmi_system_id kb3886bl_device_table[] __initconst = {
 	{
 		.ident = "Sahara Touch-iT",
 		.matches = {
diff --git a/drivers/video/backlight/lm3630a_bl.c b/drivers/video/backlight/lm3630a_bl.c
index 60d6c2ac87aa..2030a6b77a09 100644
--- a/drivers/video/backlight/lm3630a_bl.c
+++ b/drivers/video/backlight/lm3630a_bl.c
@@ -31,7 +31,8 @@
 #define REG_FAULT	0x0B
 #define REG_PWM_OUTLOW	0x12
 #define REG_PWM_OUTHIGH	0x13
-#define REG_MAX		0x1F
+#define REG_FILTER_STRENGTH	0x50
+#define REG_MAX		0x50
 
 #define INT_DEBOUNCE_MSEC	10
 struct lm3630a_chip {
@@ -80,7 +81,7 @@ static int lm3630a_chip_init(struct lm3630a_chip *pchip)
 
 	usleep_range(1000, 2000);
 	/* set Filter Strength Register */
-	rval = lm3630a_write(pchip, 0x50, 0x03);
+	rval = lm3630a_write(pchip, REG_FILTER_STRENGTH, 0x03);
 	/* set Cofig. register */
 	rval |= lm3630a_update(pchip, REG_CONFIG, 0x07, pdata->pwm_ctrl);
 	/* set boost control */
diff --git a/drivers/video/backlight/pandora_bl.c b/drivers/video/backlight/pandora_bl.c
index 5d8bb8b20183..a186bc677c7d 100644
--- a/drivers/video/backlight/pandora_bl.c
+++ b/drivers/video/backlight/pandora_bl.c
@@ -16,7 +16,7 @@
 #include <linux/delay.h>
 #include <linux/fb.h>
 #include <linux/backlight.h>
-#include <linux/i2c/twl.h>
+#include <linux/mfd/twl.h>
 #include <linux/err.h>
 
 #define TWL_PWM0_ON		0x00
diff --git a/drivers/video/backlight/pwm_bl.c b/drivers/video/backlight/pwm_bl.c
index 002f1ce22bd0..9bd17682655a 100644
--- a/drivers/video/backlight/pwm_bl.c
+++ b/drivers/video/backlight/pwm_bl.c
@@ -178,7 +178,7 @@ static int pwm_backlight_parse_dt(struct device *dev,
 	return 0;
 }
 
-static struct of_device_id pwm_backlight_of_match[] = {
+static const struct of_device_id pwm_backlight_of_match[] = {
 	{ .compatible = "pwm-backlight" },
 	{ }
 };
diff --git a/drivers/video/console/Kconfig b/drivers/video/console/Kconfig
index 2111d06f8c81..7f1f1fbcef9e 100644
--- a/drivers/video/console/Kconfig
+++ b/drivers/video/console/Kconfig
@@ -117,7 +117,7 @@ config DUMMY_CONSOLE_ROWS
           Select 25 if you use a 640x480 resolution by default.
 
 config FRAMEBUFFER_CONSOLE
-	tristate "Framebuffer Console support"
+	bool "Framebuffer Console support"
 	depends on FB && !UML
 	select VT_HW_CONSOLE_BINDING
 	select CRC32
diff --git a/drivers/video/console/Makefile b/drivers/video/console/Makefile
index 43bfa485db96..eb2cbec52643 100644
--- a/drivers/video/console/Makefile
+++ b/drivers/video/console/Makefile
@@ -7,13 +7,5 @@ obj-$(CONFIG_SGI_NEWPORT_CONSOLE) += newport_con.o
 obj-$(CONFIG_STI_CONSOLE)         += sticon.o sticore.o
 obj-$(CONFIG_VGA_CONSOLE)         += vgacon.o
 obj-$(CONFIG_MDA_CONSOLE)         += mdacon.o
-obj-$(CONFIG_FRAMEBUFFER_CONSOLE) += fbcon.o bitblit.o softcursor.o
-ifeq ($(CONFIG_FB_TILEBLITTING),y)
-obj-$(CONFIG_FRAMEBUFFER_CONSOLE)     += tileblit.o
-endif
-ifeq ($(CONFIG_FRAMEBUFFER_CONSOLE_ROTATION),y)
-obj-$(CONFIG_FRAMEBUFFER_CONSOLE)     += fbcon_rotate.o fbcon_cw.o fbcon_ud.o \
-                                         fbcon_ccw.o
-endif
 
 obj-$(CONFIG_FB_STI)              += sticore.o
diff --git a/drivers/video/console/sticore.c b/drivers/video/console/sticore.c
index 7da1ad03acb5..d1d3796773aa 100644
--- a/drivers/video/console/sticore.c
+++ b/drivers/video/console/sticore.c
@@ -281,7 +281,7 @@ static void sti_rom_copy(unsigned long base, unsigned long count, void *dest)
 static char default_sti_path[21] __read_mostly;
 
 #ifndef MODULE
-static int sti_setup(char *str)
+static int __init sti_setup(char *str)
 {
 	if (str)
 		strlcpy (default_sti_path, str, sizeof (default_sti_path));
@@ -941,7 +941,7 @@ static void sticore_check_for_default_sti(struct sti_struct *sti, char *path)
  * in the additional address field addr[1] while on
  * older Systems the PDC stores it in page0->proc_sti 
  */
-static int sticore_pa_init(struct parisc_device *dev)
+static int __init sticore_pa_init(struct parisc_device *dev)
 {
 	char pa_path[21];
 	struct sti_struct *sti = NULL;
@@ -1009,7 +1009,7 @@ static int sticore_pci_init(struct pci_dev *pd, const struct pci_device_id *ent)
 }
 
 
-static void sticore_pci_remove(struct pci_dev *pd)
+static void __exit sticore_pci_remove(struct pci_dev *pd)
 {
 	BUG();
 }
@@ -1029,7 +1029,7 @@ static struct pci_driver pci_sti_driver = {
 	.name		= "sti",
 	.id_table	= sti_pci_tbl,
 	.probe		= sticore_pci_init,
-	.remove		= sticore_pci_remove,
+	.remove		= __exit_p(sticore_pci_remove),
 };
 
 static struct parisc_device_id sti_pa_tbl[] = {
@@ -1037,8 +1037,9 @@ static struct parisc_device_id sti_pa_tbl[] = {
 	{ HPHW_FIO, HVERSION_REV_ANY_ID, HVERSION_ANY_ID, 0x00085 },
 	{ 0, }
 };
+MODULE_DEVICE_TABLE(parisc, sti_pa_tbl);
 
-static struct parisc_driver pa_sti_driver = {
+static struct parisc_driver pa_sti_driver __refdata = {
 	.name		= "sti",
 	.id_table	= sti_pa_tbl,
 	.probe		= sticore_pa_init,
diff --git a/drivers/video/console/vgacon.c b/drivers/video/console/vgacon.c
index dc06cb6a15dc..445b1dc5d441 100644
--- a/drivers/video/console/vgacon.c
+++ b/drivers/video/console/vgacon.c
@@ -398,9 +398,8 @@ static const char *vgacon_startup(void)
 #endif
 	}
 
-	/* boot_params.screen_info initialized? */
-	if ((screen_info.orig_video_mode  == 0) &&
-	    (screen_info.orig_video_lines == 0) &&
+	/* boot_params.screen_info reasonably initialized? */
+	if ((screen_info.orig_video_lines == 0) ||
 	    (screen_info.orig_video_cols  == 0))
 		goto no_vga;
 
diff --git a/drivers/video/fbdev/68328fb.c b/drivers/video/fbdev/68328fb.c
index c0c6b88d3839..d48e96088f76 100644
--- a/drivers/video/fbdev/68328fb.c
+++ b/drivers/video/fbdev/68328fb.c
@@ -72,7 +72,7 @@ static struct fb_var_screeninfo mc68x328fb_default __initdata = {
       	.vmode =	FB_VMODE_NONINTERLACED,
 };
 
-static struct fb_fix_screeninfo mc68x328fb_fix __initdata = {
+static const struct fb_fix_screeninfo mc68x328fb_fix __initconst = {
 	.id =		"68328fb",
 	.type =		FB_TYPE_PACKED_PIXELS,
 	.xpanstep =	1,
diff --git a/drivers/video/fbdev/Kconfig b/drivers/video/fbdev/Kconfig
index 5c6696bb56da..5e58f5ec0a28 100644
--- a/drivers/video/fbdev/Kconfig
+++ b/drivers/video/fbdev/Kconfig
@@ -2173,7 +2173,7 @@ config FB_PS3_DEFAULT_SIZE_M
 
 config FB_XILINX
 	tristate "Xilinx frame buffer support"
-	depends on FB && (XILINX_VIRTEX || MICROBLAZE || ARCH_ZYNQ)
+	depends on FB && (XILINX_VIRTEX || MICROBLAZE || ARCH_ZYNQ || ARCH_ZYNQMP)
 	select FB_CFB_FILLRECT
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
diff --git a/drivers/video/fbdev/amba-clcd.c b/drivers/video/fbdev/amba-clcd.c
index ffc2c33c6cef..36d25190b48c 100644
--- a/drivers/video/fbdev/amba-clcd.c
+++ b/drivers/video/fbdev/amba-clcd.c
@@ -1035,7 +1035,7 @@ static struct clcd_vendor_data vendor_nomadik = {
 	.init_panel = nomadik_clcd_init_panel,
 };
 
-static struct amba_id clcdfb_id_table[] = {
+static const struct amba_id clcdfb_id_table[] = {
 	{
 		.id	= 0x00041110,
 		.mask	= 0x000ffffe,
diff --git a/drivers/video/fbdev/arkfb.c b/drivers/video/fbdev/arkfb.c
index 6a317de7082c..13ba371e70aa 100644
--- a/drivers/video/fbdev/arkfb.c
+++ b/drivers/video/fbdev/arkfb.c
@@ -1157,7 +1157,7 @@ fail:
 
 /* List of boards that we are trying to support */
 
-static struct pci_device_id ark_devices[] = {
+static const struct pci_device_id ark_devices[] = {
 	{PCI_DEVICE(0xEDD8, 0xA099)},
 	{0, 0, 0, 0, 0, 0, 0}
 };
diff --git a/drivers/video/fbdev/asiliantfb.c b/drivers/video/fbdev/asiliantfb.c
index 91eea4583382..ea31054a28ca 100644
--- a/drivers/video/fbdev/asiliantfb.c
+++ b/drivers/video/fbdev/asiliantfb.c
@@ -592,7 +592,7 @@ static void asiliantfb_remove(struct pci_dev *dp)
 	framebuffer_release(p);
 }
 
-static struct pci_device_id asiliantfb_pci_tbl[] = {
+static const struct pci_device_id asiliantfb_pci_tbl[] = {
 	{ PCI_VENDOR_ID_CT, PCI_DEVICE_ID_CT_69000, PCI_ANY_ID, PCI_ANY_ID },
 	{ 0 }
 };
diff --git a/drivers/video/fbdev/atmel_lcdfb.c b/drivers/video/fbdev/atmel_lcdfb.c
index 669ecc755fa9..e06358da4b99 100644
--- a/drivers/video/fbdev/atmel_lcdfb.c
+++ b/drivers/video/fbdev/atmel_lcdfb.c
@@ -320,7 +320,7 @@ static inline void atmel_lcdfb_power_control(struct atmel_lcdfb_info *sinfo, int
 	}
 }
 
-static struct fb_fix_screeninfo atmel_lcdfb_fix __initdata = {
+static const struct fb_fix_screeninfo atmel_lcdfb_fix __initconst = {
 	.type		= FB_TYPE_PACKED_PIXELS,
 	.visual		= FB_VISUAL_TRUECOLOR,
 	.xpanstep	= 0,
diff --git a/drivers/video/fbdev/aty/aty128fb.c b/drivers/video/fbdev/aty/aty128fb.c
index fa07242a78d2..db18474607c9 100644
--- a/drivers/video/fbdev/aty/aty128fb.c
+++ b/drivers/video/fbdev/aty/aty128fb.c
@@ -116,7 +116,7 @@ static const struct fb_var_screeninfo default_var = {
 
 /* default modedb mode */
 /* 640x480, 60 Hz, Non-Interlaced (25.172 MHz dotclock) */
-static struct fb_videomode defaultmode = {
+static const struct fb_videomode defaultmode = {
 	.refresh =	60,
 	.xres =		640,
 	.yres =		480,
@@ -166,7 +166,7 @@ static int aty128_pci_resume(struct pci_dev *pdev);
 static int aty128_do_resume(struct pci_dev *pdev);
 
 /* supported Rage128 chipsets */
-static struct pci_device_id aty128_pci_tbl[] = {
+static const struct pci_device_id aty128_pci_tbl[] = {
 	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_LE,
 	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_M3_pci },
 	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_LF,
diff --git a/drivers/video/fbdev/aty/atyfb_base.c b/drivers/video/fbdev/aty/atyfb_base.c
index b55fdac9c9f5..3ec72f19114b 100644
--- a/drivers/video/fbdev/aty/atyfb_base.c
+++ b/drivers/video/fbdev/aty/atyfb_base.c
@@ -274,7 +274,7 @@ static struct fb_var_screeninfo default_var = {
 	0, FB_VMODE_NONINTERLACED
 };
 
-static struct fb_videomode defmode = {
+static const struct fb_videomode defmode = {
 	/* 640x480 @ 60 Hz, 31.5 kHz hsync */
 	NULL, 60, 640, 480, 39721, 40, 24, 32, 11, 96, 2,
 	0, FB_VMODE_NONINTERLACED
@@ -1855,7 +1855,7 @@ static int atyfb_ioctl(struct fb_info *info, u_int cmd, u_long arg)
 #if defined(DEBUG) && defined(CONFIG_FB_ATY_CT)
 	case ATYIO_CLKR:
 		if (M64_HAS(INTEGRATED)) {
-			struct atyclk clk;
+			struct atyclk clk = { 0 };
 			union aty_pll *pll = &par->pll;
 			u32 dsp_config = pll->ct.dsp_config;
 			u32 dsp_on_off = pll->ct.dsp_on_off;
@@ -3756,7 +3756,7 @@ static void atyfb_pci_remove(struct pci_dev *pdev)
 	atyfb_remove(info);
 }
 
-static struct pci_device_id atyfb_pci_tbl[] = {
+static const struct pci_device_id atyfb_pci_tbl[] = {
 #ifdef CONFIG_FB_ATY_GX
 	{ PCI_DEVICE(PCI_VENDOR_ID_ATI, PCI_CHIP_MACH64GX) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_ATI, PCI_CHIP_MACH64CX) },
diff --git a/drivers/video/fbdev/aty/radeon_base.c b/drivers/video/fbdev/aty/radeon_base.c
index 6b4c7872b375..1e2ec360f8c1 100644
--- a/drivers/video/fbdev/aty/radeon_base.c
+++ b/drivers/video/fbdev/aty/radeon_base.c
@@ -96,7 +96,7 @@
 #define CHIP_DEF(id, family, flags)					\
 	{ PCI_VENDOR_ID_ATI, id, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (flags) | (CHIP_FAMILY_##family) }
 
-static struct pci_device_id radeonfb_pci_table[] = {
+static const struct pci_device_id radeonfb_pci_table[] = {
         /* Radeon Xpress 200m */
 	CHIP_DEF(PCI_CHIP_RS480_5955,   RS480,  CHIP_HAS_CRTC2 | CHIP_IS_IGP | CHIP_IS_MOBILITY),
 	CHIP_DEF(PCI_CHIP_RS482_5975,	RS480,	CHIP_HAS_CRTC2 | CHIP_IS_IGP | CHIP_IS_MOBILITY),
@@ -2241,7 +2241,7 @@ static ssize_t radeon_show_edid2(struct file *filp, struct kobject *kobj,
 	return radeon_show_one_edid(buf, off, count, rinfo->mon2_EDID);
 }
 
-static struct bin_attribute edid1_attr = {
+static const struct bin_attribute edid1_attr = {
 	.attr   = {
 		.name	= "edid1",
 		.mode	= 0444,
@@ -2250,7 +2250,7 @@ static struct bin_attribute edid1_attr = {
 	.read	= radeon_show_edid1,
 };
 
-static struct bin_attribute edid2_attr = {
+static const struct bin_attribute edid2_attr = {
 	.attr   = {
 		.name	= "edid2",
 		.mode	= 0444,
diff --git a/drivers/video/fbdev/bfin-lq035q1-fb.c b/drivers/video/fbdev/bfin-lq035q1-fb.c
index b594a58ff21d..b459354ad940 100644
--- a/drivers/video/fbdev/bfin-lq035q1-fb.c
+++ b/drivers/video/fbdev/bfin-lq035q1-fb.c
@@ -841,7 +841,7 @@ static int bfin_lq035q1_resume(struct device *dev)
 	return 0;
 }
 
-static struct dev_pm_ops bfin_lq035q1_dev_pm_ops = {
+static const struct dev_pm_ops bfin_lq035q1_dev_pm_ops = {
 	.suspend = bfin_lq035q1_suspend,
 	.resume  = bfin_lq035q1_resume,
 };
diff --git a/drivers/video/fbdev/bw2.c b/drivers/video/fbdev/bw2.c
index 8c5b281f0b29..7aa972072357 100644
--- a/drivers/video/fbdev/bw2.c
+++ b/drivers/video/fbdev/bw2.c
@@ -333,8 +333,8 @@ static int bw2_probe(struct platform_device *op)
 
 	dev_set_drvdata(&op->dev, info);
 
-	printk(KERN_INFO "%s: bwtwo at %lx:%lx\n",
-	       dp->full_name, par->which_io, info->fix.smem_start);
+	printk(KERN_INFO "%pOF: bwtwo at %lx:%lx\n",
+	       dp, par->which_io, info->fix.smem_start);
 
 	return 0;
 
diff --git a/drivers/video/fbdev/cg14.c b/drivers/video/fbdev/cg14.c
index 43e915eaf606..8de88b129b62 100644
--- a/drivers/video/fbdev/cg14.c
+++ b/drivers/video/fbdev/cg14.c
@@ -553,8 +553,8 @@ static int cg14_probe(struct platform_device *op)
 
 	dev_set_drvdata(&op->dev, info);
 
-	printk(KERN_INFO "%s: cgfourteen at %lx:%lx, %dMB\n",
-	       dp->full_name,
+	printk(KERN_INFO "%pOF: cgfourteen at %lx:%lx, %dMB\n",
+	       dp,
 	       par->iospace, info->fix.smem_start,
 	       par->ramsize >> 20);
 
diff --git a/drivers/video/fbdev/cg3.c b/drivers/video/fbdev/cg3.c
index 716391f22e75..6c334260cf53 100644
--- a/drivers/video/fbdev/cg3.c
+++ b/drivers/video/fbdev/cg3.c
@@ -412,8 +412,8 @@ static int cg3_probe(struct platform_device *op)
 
 	dev_set_drvdata(&op->dev, info);
 
-	printk(KERN_INFO "%s: cg3 at %lx:%lx\n",
-	       dp->full_name, par->which_io, info->fix.smem_start);
+	printk(KERN_INFO "%pOF: cg3 at %lx:%lx\n",
+	       dp, par->which_io, info->fix.smem_start);
 
 	return 0;
 
diff --git a/drivers/video/fbdev/cg6.c b/drivers/video/fbdev/cg6.c
index bdf901ed5291..0296c21acc78 100644
--- a/drivers/video/fbdev/cg6.c
+++ b/drivers/video/fbdev/cg6.c
@@ -810,8 +810,8 @@ static int cg6_probe(struct platform_device *op)
 
 	dev_set_drvdata(&op->dev, info);
 
-	printk(KERN_INFO "%s: CGsix [%s] at %lx:%lx\n",
-	       dp->full_name, info->fix.id,
+	printk(KERN_INFO "%pOF: CGsix [%s] at %lx:%lx\n",
+	       dp, info->fix.id,
 	       par->which_io, info->fix.smem_start);
 
 	return 0;
diff --git a/drivers/video/fbdev/chipsfb.c b/drivers/video/fbdev/chipsfb.c
index 59abdc6a97f6..f103665cad43 100644
--- a/drivers/video/fbdev/chipsfb.c
+++ b/drivers/video/fbdev/chipsfb.c
@@ -292,7 +292,7 @@ static void chips_hw_init(void)
 		write_fr(chips_init_fr[i].addr, chips_init_fr[i].data);
 }
 
-static struct fb_fix_screeninfo chipsfb_fix = {
+static const struct fb_fix_screeninfo chipsfb_fix = {
 	.id =		"C&T 65550",
 	.type =		FB_TYPE_PACKED_PIXELS,
 	.visual =	FB_VISUAL_PSEUDOCOLOR,
@@ -309,7 +309,7 @@ static struct fb_fix_screeninfo chipsfb_fix = {
 	.smem_len =	0x100000,	/* 1MB */
 };
 
-static struct fb_var_screeninfo chipsfb_var = {
+static const struct fb_var_screeninfo chipsfb_var = {
 	.xres = 800,
 	.yres = 600,
 	.xres_virtual = 800,
diff --git a/drivers/video/fbdev/cobalt_lcdfb.c b/drivers/video/fbdev/cobalt_lcdfb.c
index 9da90bd242f4..0ef633e278a1 100644
--- a/drivers/video/fbdev/cobalt_lcdfb.c
+++ b/drivers/video/fbdev/cobalt_lcdfb.c
@@ -126,7 +126,7 @@ static void lcd_clear(struct fb_info *info)
 	lcd_write_control(info, LCD_RESET);
 }
 
-static struct fb_fix_screeninfo cobalt_lcdfb_fix = {
+static const struct fb_fix_screeninfo cobalt_lcdfb_fix = {
 	.id		= "cobalt-lcd",
 	.type		= FB_TYPE_TEXT,
 	.type_aux	= FB_AUX_TEXT_MDA,
diff --git a/drivers/video/fbdev/core/Makefile b/drivers/video/fbdev/core/Makefile
index 9e3ddf225393..73493bbd7a15 100644
--- a/drivers/video/fbdev/core/Makefile
+++ b/drivers/video/fbdev/core/Makefile
@@ -4,6 +4,20 @@ obj-$(CONFIG_FB)                  += fb.o
 fb-y                              := fbmem.o fbmon.o fbcmap.o fbsysfs.o \
                                      modedb.o fbcvt.o
 fb-$(CONFIG_FB_DEFERRED_IO)       += fb_defio.o
+
+ifeq ($(CONFIG_FRAMEBUFFER_CONSOLE),y)
+fb-y				  += fbcon.o bitblit.o softcursor.o
+ifeq ($(CONFIG_FB_TILEBLITTING),y)
+fb-y				  += tileblit.o
+endif
+ifeq ($(CONFIG_FRAMEBUFFER_CONSOLE_ROTATION),y)
+fb-y				  += fbcon_rotate.o fbcon_cw.o fbcon_ud.o \
+				     fbcon_ccw.o
+endif
+ifeq ($(CONFIG_DMI),y)
+fb-y				  += fbcon_dmi_quirks.o
+endif
+endif
 fb-objs                           := $(fb-y)
 
 obj-$(CONFIG_FB_CFB_FILLRECT)  += cfbfillrect.o
diff --git a/drivers/video/console/bitblit.c b/drivers/video/fbdev/core/bitblit.c
index dbfe4eecf12e..790900d646c0 100644
--- a/drivers/video/console/bitblit.c
+++ b/drivers/video/fbdev/core/bitblit.c
@@ -203,7 +203,7 @@ static void bit_putcs(struct vc_data *vc, struct fb_info *info,
 }
 
 static void bit_clear_margins(struct vc_data *vc, struct fb_info *info,
-			      int bottom_only)
+			      int color, int bottom_only)
 {
 	unsigned int cw = vc->vc_font.width;
 	unsigned int ch = vc->vc_font.height;
@@ -213,7 +213,7 @@ static void bit_clear_margins(struct vc_data *vc, struct fb_info *info,
 	unsigned int bs = info->var.yres - bh;
 	struct fb_fillrect region;
 
-	region.color = 0;
+	region.color = color;
 	region.rop = ROP_COPY;
 
 	if (rw && !bottom_only) {
@@ -416,7 +416,3 @@ void fbcon_set_bitops(struct fbcon_ops *ops)
 
 EXPORT_SYMBOL(fbcon_set_bitops);
 
-MODULE_AUTHOR("Antonino Daplas <adaplas@pol.net>");
-MODULE_DESCRIPTION("Bit Blitting Operation");
-MODULE_LICENSE("GPL");
-
diff --git a/drivers/video/fbdev/core/fb_defio.c b/drivers/video/fbdev/core/fb_defio.c
index 37f69c061210..487d5e336e1b 100644
--- a/drivers/video/fbdev/core/fb_defio.c
+++ b/drivers/video/fbdev/core/fb_defio.c
@@ -69,7 +69,7 @@ int fb_deferred_io_fsync(struct file *file, loff_t start, loff_t end, int datasy
 {
 	struct fb_info *info = file->private_data;
 	struct inode *inode = file_inode(file);
-	int err = filemap_write_and_wait_range(inode->i_mapping, start, end);
+	int err = file_write_and_wait_range(file, start, end);
 	if (err)
 		return err;
 
diff --git a/drivers/video/console/fbcon.c b/drivers/video/fbdev/core/fbcon.c
index 12ded23f1aaf..04612f938bab 100644
--- a/drivers/video/console/fbcon.c
+++ b/drivers/video/fbdev/core/fbcon.c
@@ -68,6 +68,7 @@
 #include <linux/kd.h>
 #include <linux/slab.h>
 #include <linux/fb.h>
+#include <linux/fbcon.h>
 #include <linux/vt_kern.h>
 #include <linux/selection.h>
 #include <linux/font.h>
@@ -135,8 +136,9 @@ static char fontname[40];
 static int info_idx = -1;
 
 /* console rotation */
-static int initial_rotation;
+static int initial_rotation = -1;
 static int fbcon_has_sysfs;
+static int margin_color;
 
 static const struct consw fb_con;
 
@@ -491,6 +493,13 @@ static int __init fb_console_setup(char *this_opt)
 				initial_rotation = 0;
 			continue;
 		}
+
+		if (!strncmp(options, "margin:", 7)) {
+			options += 7;
+			if (*options)
+				margin_color = simple_strtoul(options, &options, 0);
+			continue;
+		}
 	}
 	return 1;
 }
@@ -563,7 +572,7 @@ static void fbcon_prepare_logo(struct vc_data *vc, struct fb_info *info,
 	unsigned short *save = NULL, *r, *q;
 	int logo_height;
 
-	if (info->flags & FBINFO_MODULE) {
+	if (info->fbops->owner) {
 		logo_shown = FBCON_LOGO_DONTSHOW;
 		return;
 	}
@@ -954,7 +963,10 @@ static const char *fbcon_startup(void)
 	ops->cur_rotate = -1;
 	ops->cur_blink_jiffies = HZ / 5;
 	info->fbcon_par = ops;
-	p->con_rotate = initial_rotation;
+	if (initial_rotation != -1)
+		p->con_rotate = initial_rotation;
+	else
+		p->con_rotate = fbcon_platform_get_rotate(info);
 	set_blitting_type(vc, info);
 
 	if (info->fix.type != FB_TYPE_TEXT) {
@@ -1091,7 +1103,10 @@ static void fbcon_init(struct vc_data *vc, int init)
 
 	ops = info->fbcon_par;
 	ops->cur_blink_jiffies = msecs_to_jiffies(vc->vc_cur_blink_ms);
-	p->con_rotate = initial_rotation;
+	if (initial_rotation != -1)
+		p->con_rotate = initial_rotation;
+	else
+		p->con_rotate = fbcon_platform_get_rotate(info);
 	set_blitting_type(vc, info);
 
 	cols = vc->vc_cols;
@@ -1299,7 +1314,7 @@ static void fbcon_clear_margins(struct vc_data *vc, int bottom_only)
 	struct fbcon_ops *ops = info->fbcon_par;
 
 	if (!fbcon_is_inactive(vc, info))
-		ops->clear_margins(vc, info, bottom_only);
+		ops->clear_margins(vc, info, margin_color, bottom_only);
 }
 
 static void fbcon_cursor(struct vc_data *vc, int mode)
@@ -3606,7 +3621,7 @@ static void fbcon_exit(void)
 	fbcon_has_exited = 1;
 }
 
-static int __init fb_console_init(void)
+void __init fb_console_init(void)
 {
 	int i;
 
@@ -3628,11 +3643,8 @@ static int __init fb_console_init(void)
 
 	console_unlock();
 	fbcon_start();
-	return 0;
 }
 
-fs_initcall(fb_console_init);
-
 #ifdef MODULE
 
 static void __exit fbcon_deinit_device(void)
@@ -3647,7 +3659,7 @@ static void __exit fbcon_deinit_device(void)
 	}
 }
 
-static void __exit fb_console_exit(void)
+void __exit fb_console_exit(void)
 {
 	console_lock();
 	fb_unregister_client(&fbcon_event_notifier);
@@ -3657,9 +3669,4 @@ static void __exit fb_console_exit(void)
 	do_unregister_con_driver(&fb_con);
 	console_unlock();
 }	
-
-module_exit(fb_console_exit);
-
 #endif
-
-MODULE_LICENSE("GPL");
diff --git a/drivers/video/console/fbcon.h b/drivers/video/fbdev/core/fbcon.h
index 7aaa4eabbba0..18f3ac144237 100644
--- a/drivers/video/console/fbcon.h
+++ b/drivers/video/fbdev/core/fbcon.h
@@ -60,7 +60,7 @@ struct fbcon_ops {
 		      const unsigned short *s, int count, int yy, int xx,
 		      int fg, int bg);
 	void (*clear_margins)(struct vc_data *vc, struct fb_info *info,
-			      int bottom_only);
+			      int color, int bottom_only);
 	void (*cursor)(struct vc_data *vc, struct fb_info *info, int mode,
 		       int softback_lines, int fg, int bg);
 	int  (*update_start)(struct fb_info *info);
@@ -261,5 +261,10 @@ extern void fbcon_set_rotate(struct fbcon_ops *ops);
 #define fbcon_set_rotate(x) do {} while(0)
 #endif /* CONFIG_FRAMEBUFFER_CONSOLE_ROTATION */
 
-#endif /* _VIDEO_FBCON_H */
+#ifdef CONFIG_DMI
+int fbcon_platform_get_rotate(struct fb_info *info);
+#else
+#define fbcon_platform_get_rotate(i) FB_ROTATE_UR
+#endif /* CONFIG_DMI */
 
+#endif /* _VIDEO_FBCON_H */
diff --git a/drivers/video/console/fbcon_ccw.c b/drivers/video/fbdev/core/fbcon_ccw.c
index 5a3cbf6dff4d..37a8b0b22566 100644
--- a/drivers/video/console/fbcon_ccw.c
+++ b/drivers/video/fbdev/core/fbcon_ccw.c
@@ -189,7 +189,7 @@ static void ccw_putcs(struct vc_data *vc, struct fb_info *info,
 }
 
 static void ccw_clear_margins(struct vc_data *vc, struct fb_info *info,
-			     int bottom_only)
+			      int color, int bottom_only)
 {
 	unsigned int cw = vc->vc_font.width;
 	unsigned int ch = vc->vc_font.height;
@@ -198,7 +198,7 @@ static void ccw_clear_margins(struct vc_data *vc, struct fb_info *info,
 	unsigned int bs = vc->vc_rows*ch;
 	struct fb_fillrect region;
 
-	region.color = 0;
+	region.color = color;
 	region.rop = ROP_COPY;
 
 	if (rw && !bottom_only) {
@@ -418,7 +418,3 @@ void fbcon_rotate_ccw(struct fbcon_ops *ops)
 	ops->update_start = ccw_update_start;
 }
 EXPORT_SYMBOL(fbcon_rotate_ccw);
-
-MODULE_AUTHOR("Antonino Daplas <adaplas@pol.net>");
-MODULE_DESCRIPTION("Console Rotation (270 degrees) Support");
-MODULE_LICENSE("GPL");
diff --git a/drivers/video/console/fbcon_cw.c b/drivers/video/fbdev/core/fbcon_cw.c
index e7ee44db4e98..1888f8c866e8 100644
--- a/drivers/video/console/fbcon_cw.c
+++ b/drivers/video/fbdev/core/fbcon_cw.c
@@ -172,7 +172,7 @@ static void cw_putcs(struct vc_data *vc, struct fb_info *info,
 }
 
 static void cw_clear_margins(struct vc_data *vc, struct fb_info *info,
-			     int bottom_only)
+			     int color, int bottom_only)
 {
 	unsigned int cw = vc->vc_font.width;
 	unsigned int ch = vc->vc_font.height;
@@ -181,7 +181,7 @@ static void cw_clear_margins(struct vc_data *vc, struct fb_info *info,
 	unsigned int rs = info->var.yres - rw;
 	struct fb_fillrect region;
 
-	region.color = 0;
+	region.color = color;
 	region.rop = ROP_COPY;
 
 	if (rw && !bottom_only) {
@@ -401,7 +401,3 @@ void fbcon_rotate_cw(struct fbcon_ops *ops)
 	ops->update_start = cw_update_start;
 }
 EXPORT_SYMBOL(fbcon_rotate_cw);
-
-MODULE_AUTHOR("Antonino Daplas <adaplas@pol.net>");
-MODULE_DESCRIPTION("Console Rotation (90 degrees) Support");
-MODULE_LICENSE("GPL");
diff --git a/drivers/video/fbdev/core/fbcon_dmi_quirks.c b/drivers/video/fbdev/core/fbcon_dmi_quirks.c
new file mode 100644
index 000000000000..6904e47d1e51
--- /dev/null
+++ b/drivers/video/fbdev/core/fbcon_dmi_quirks.c
@@ -0,0 +1,145 @@
+/*
+ *  fbcon_dmi_quirks.c -- DMI based quirk detection for fbcon
+ *
+ *	Copyright (C) 2017 Hans de Goede <hdegoede@redhat.com>
+ *
+ *  This file is subject to the terms and conditions of the GNU General Public
+ *  License.  See the file COPYING in the main directory of this archive for
+ *  more details.
+ */
+
+#include <linux/dmi.h>
+#include <linux/fb.h>
+#include <linux/kernel.h>
+#include "fbcon.h"
+
+/*
+ * Some x86 clamshell design devices use portrait tablet screens and a display
+ * engine which cannot rotate in hardware, so we need to rotate the fbcon to
+ * compensate. Unfortunately these (cheap) devices also typically have quite
+ * generic DMI data, so we match on a combination of DMI data, screen resolution
+ * and a list of known BIOS dates to avoid false positives.
+ */
+
+struct fbcon_dmi_rotate_data {
+	int width;
+	int height;
+	const char * const *bios_dates;
+	int rotate;
+};
+
+static const struct fbcon_dmi_rotate_data rotate_data_asus_t100ha = {
+	.width = 800,
+	.height = 1280,
+	.rotate = FB_ROTATE_CCW,
+};
+
+static const struct fbcon_dmi_rotate_data rotate_data_gpd_pocket = {
+	.width = 1200,
+	.height = 1920,
+	.bios_dates = (const char * const []){ "05/26/2017", "06/28/2017",
+		"07/05/2017", "08/07/2017", NULL },
+	.rotate = FB_ROTATE_CW,
+};
+
+static const struct fbcon_dmi_rotate_data rotate_data_gpd_win = {
+	.width = 720,
+	.height = 1280,
+	.bios_dates = (const char * const []){
+		"10/25/2016", "11/18/2016", "12/23/2016", "12/26/2016",
+		"02/21/2017", "03/20/2017", "05/25/2017", NULL },
+	.rotate = FB_ROTATE_CW,
+};
+
+static const struct fbcon_dmi_rotate_data rotate_data_itworks_tw891 = {
+	.width = 800,
+	.height = 1280,
+	.bios_dates = (const char * const []){ "10/16/2015", NULL },
+	.rotate = FB_ROTATE_CW,
+};
+
+static const struct fbcon_dmi_rotate_data rotate_data_vios_lth17 = {
+	.width = 800,
+	.height = 1280,
+	.rotate = FB_ROTATE_CW,
+};
+
+static const struct dmi_system_id rotate_data[] = {
+	{	/* Asus T100HA */
+		.matches = {
+		  DMI_EXACT_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+		  DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "T100HAN"),
+		},
+		.driver_data = (void *)&rotate_data_asus_t100ha,
+	}, {	/*
+		 * GPD Pocket, note that the the DMI data is less generic then
+		 * it seems, devices with a board-vendor of "AMI Corporation"
+		 * are quite rare, as are devices which have both board- *and*
+		 * product-id set to "Default String"
+		 */
+		.matches = {
+		  DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "AMI Corporation"),
+		  DMI_EXACT_MATCH(DMI_BOARD_NAME, "Default string"),
+		  DMI_EXACT_MATCH(DMI_BOARD_SERIAL, "Default string"),
+		  DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Default string"),
+		},
+		.driver_data = (void *)&rotate_data_gpd_pocket,
+	}, {	/* GPD Win (same note on DMI match as GPD Pocket) */
+		.matches = {
+		  DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "AMI Corporation"),
+		  DMI_EXACT_MATCH(DMI_BOARD_NAME, "Default string"),
+		  DMI_EXACT_MATCH(DMI_BOARD_SERIAL, "Default string"),
+		  DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Default string"),
+		},
+		.driver_data = (void *)&rotate_data_gpd_win,
+	}, {	/* I.T.Works TW891 */
+		.matches = {
+		  DMI_EXACT_MATCH(DMI_SYS_VENDOR, "To be filled by O.E.M."),
+		  DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "TW891"),
+		  DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "To be filled by O.E.M."),
+		  DMI_EXACT_MATCH(DMI_BOARD_NAME, "TW891"),
+		},
+		.driver_data = (void *)&rotate_data_itworks_tw891,
+	}, {	/* VIOS LTH17 */
+		.matches = {
+		  DMI_EXACT_MATCH(DMI_SYS_VENDOR, "VIOS"),
+		  DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "LTH17"),
+		  DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "VIOS"),
+		  DMI_EXACT_MATCH(DMI_BOARD_NAME, "LTH17"),
+		},
+		.driver_data = (void *)&rotate_data_vios_lth17,
+	},
+	{}
+};
+
+int fbcon_platform_get_rotate(struct fb_info *info)
+{
+	const struct dmi_system_id *match;
+	const struct fbcon_dmi_rotate_data *data;
+	const char *bios_date;
+	int i;
+
+	for (match = dmi_first_match(rotate_data);
+	     match;
+	     match = dmi_first_match(match + 1)) {
+		data = match->driver_data;
+
+		if (data->width != info->var.xres ||
+		    data->height != info->var.yres)
+			continue;
+
+		if (!data->bios_dates)
+			return data->rotate;
+
+		bios_date = dmi_get_system_info(DMI_BIOS_DATE);
+		if (!bios_date)
+			continue;
+
+		for (i = 0; data->bios_dates[i]; i++) {
+			if (!strcmp(data->bios_dates[i], bios_date))
+				return data->rotate;
+		}
+	}
+
+	return FB_ROTATE_UR;
+}
diff --git a/drivers/video/console/fbcon_rotate.c b/drivers/video/fbdev/core/fbcon_rotate.c
index db6528f2d3f2..8a51e4d95cc5 100644
--- a/drivers/video/console/fbcon_rotate.c
+++ b/drivers/video/fbdev/core/fbcon_rotate.c
@@ -110,7 +110,3 @@ void fbcon_set_rotate(struct fbcon_ops *ops)
 	}
 }
 EXPORT_SYMBOL(fbcon_set_rotate);
-
-MODULE_AUTHOR("Antonino Daplas <adaplas@pol.net>");
-MODULE_DESCRIPTION("Console Rotation Support");
-MODULE_LICENSE("GPL");
diff --git a/drivers/video/console/fbcon_rotate.h b/drivers/video/fbdev/core/fbcon_rotate.h
index e233444cda66..e233444cda66 100644
--- a/drivers/video/console/fbcon_rotate.h
+++ b/drivers/video/fbdev/core/fbcon_rotate.h
diff --git a/drivers/video/console/fbcon_ud.c b/drivers/video/fbdev/core/fbcon_ud.c
index 19e3714abfe8..f98eee263597 100644
--- a/drivers/video/console/fbcon_ud.c
+++ b/drivers/video/fbdev/core/fbcon_ud.c
@@ -220,7 +220,7 @@ static void ud_putcs(struct vc_data *vc, struct fb_info *info,
 }
 
 static void ud_clear_margins(struct vc_data *vc, struct fb_info *info,
-			     int bottom_only)
+			     int color, int bottom_only)
 {
 	unsigned int cw = vc->vc_font.width;
 	unsigned int ch = vc->vc_font.height;
@@ -228,7 +228,7 @@ static void ud_clear_margins(struct vc_data *vc, struct fb_info *info,
 	unsigned int bh = info->var.yres - (vc->vc_rows*ch);
 	struct fb_fillrect region;
 
-	region.color = 0;
+	region.color = color;
 	region.rop = ROP_COPY;
 
 	if (rw && !bottom_only) {
@@ -446,7 +446,3 @@ void fbcon_rotate_ud(struct fbcon_ops *ops)
 	ops->update_start = ud_update_start;
 }
 EXPORT_SYMBOL(fbcon_rotate_ud);
-
-MODULE_AUTHOR("Antonino Daplas <adaplas@pol.net>");
-MODULE_DESCRIPTION("Console Rotation (180 degrees) Support");
-MODULE_LICENSE("GPL");
diff --git a/drivers/video/fbdev/core/fbmem.c b/drivers/video/fbdev/core/fbmem.c
index 7a42238db446..f741ba8df01b 100644
--- a/drivers/video/fbdev/core/fbmem.c
+++ b/drivers/video/fbdev/core/fbmem.c
@@ -32,6 +32,8 @@
 #include <linux/device.h>
 #include <linux/efi.h>
 #include <linux/fb.h>
+#include <linux/fbcon.h>
+#include <linux/mem_encrypt.h>
 
 #include <asm/fb.h>
 
@@ -315,7 +317,7 @@ static void fb_set_logo(struct fb_info *info,
 		for (i = 0; i < logo->height; i++) {
 			for (j = 0; j < logo->width; src++) {
 				d = *src ^ xor;
-				for (k = 7; k >= 0; k--) {
+				for (k = 7; k >= 0 && j < logo->width; k--) {
 					*dst++ = ((d >> k) & 1) ? fg : 0;
 					j++;
 				}
@@ -462,7 +464,7 @@ static int fb_show_logo_line(struct fb_info *info, int rotate,
 
 	/* Return if the frame buffer is not mapped or suspended */
 	if (logo == NULL || info->state != FBINFO_STATE_RUNNING ||
-	    info->flags & FBINFO_MODULE)
+	    info->fbops->owner)
 		return 0;
 
 	image.depth = 8;
@@ -600,7 +602,7 @@ int fb_prepare_logo(struct fb_info *info, int rotate)
 	memset(&fb_logo, 0, sizeof(struct logo_data));
 
 	if (info->flags & FBINFO_MISC_TILEBLITTING ||
-	    info->flags & FBINFO_MODULE)
+	    info->fbops->owner)
 		return 0;
 
 	if (info->fix.visual == FB_VISUAL_DIRECTCOLOR) {
@@ -1396,6 +1398,12 @@ fb_mmap(struct file *file, struct vm_area_struct * vma)
 	mutex_lock(&info->mm_lock);
 	if (fb->fb_mmap) {
 		int res;
+
+		/*
+		 * The framebuffer needs to be accessed decrypted, be sure
+		 * SME protection is removed ahead of the call
+		 */
+		vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
 		res = fb->fb_mmap(info, vma);
 		mutex_unlock(&info->mm_lock);
 		return res;
@@ -1421,6 +1429,11 @@ fb_mmap(struct file *file, struct vm_area_struct * vma)
 	mutex_unlock(&info->mm_lock);
 
 	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
+	/*
+	 * The framebuffer needs to be accessed decrypted, be sure
+	 * SME protection is removed
+	 */
+	vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
 	fb_pgprotect(file, vma, start);
 
 	return vm_iomap_memory(vma, start, len);
@@ -1880,6 +1893,9 @@ fbmem_init(void)
 		fb_class = NULL;
 		goto err_class;
 	}
+
+	fb_console_init();
+
 	return 0;
 
 err_class:
@@ -1894,6 +1910,8 @@ module_init(fbmem_init);
 static void __exit
 fbmem_exit(void)
 {
+	fb_console_exit();
+
 	remove_proc_entry("fb", NULL);
 	class_destroy(fb_class);
 	unregister_chrdev(FB_MAJOR, "fb");
diff --git a/drivers/video/fbdev/core/fbmon.c b/drivers/video/fbdev/core/fbmon.c
index 41d7979d81c5..2b2d67328514 100644
--- a/drivers/video/fbdev/core/fbmon.c
+++ b/drivers/video/fbdev/core/fbmon.c
@@ -1479,8 +1479,8 @@ int of_get_fb_videomode(struct device_node *np, struct fb_videomode *fb,
 	if (ret)
 		return ret;
 
-	pr_debug("%s: got %dx%d display mode from %s\n",
-		of_node_full_name(np), vm.hactive, vm.vactive, np->name);
+	pr_debug("%pOF: got %dx%d display mode from %s\n",
+		np, vm.hactive, vm.vactive, np->name);
 	dump_fb_videomode(fb);
 
 	return 0;
diff --git a/drivers/video/console/softcursor.c b/drivers/video/fbdev/core/softcursor.c
index 46dd8f5d2e9e..fc93f254498e 100644
--- a/drivers/video/console/softcursor.c
+++ b/drivers/video/fbdev/core/softcursor.c
@@ -76,7 +76,3 @@ int soft_cursor(struct fb_info *info, struct fb_cursor *cursor)
 }
 
 EXPORT_SYMBOL(soft_cursor);
-
-MODULE_AUTHOR("James Simmons <jsimmons@users.sf.net>");
-MODULE_DESCRIPTION("Generic software cursor");
-MODULE_LICENSE("GPL");
diff --git a/drivers/video/console/tileblit.c b/drivers/video/fbdev/core/tileblit.c
index 15e8e1a89c45..93390312957f 100644
--- a/drivers/video/console/tileblit.c
+++ b/drivers/video/fbdev/core/tileblit.c
@@ -74,7 +74,7 @@ static void tile_putcs(struct vc_data *vc, struct fb_info *info,
 }
 
 static void tile_clear_margins(struct vc_data *vc, struct fb_info *info,
-			       int bottom_only)
+			       int color, int bottom_only)
 {
 	return;
 }
@@ -152,8 +152,3 @@ void fbcon_set_tileops(struct vc_data *vc, struct fb_info *info)
 }
 
 EXPORT_SYMBOL(fbcon_set_tileops);
-
-MODULE_AUTHOR("Antonino Daplas <adaplas@pol.net>");
-MODULE_DESCRIPTION("Tile Blitting Operation");
-MODULE_LICENSE("GPL");
-
diff --git a/drivers/video/fbdev/cyber2000fb.c b/drivers/video/fbdev/cyber2000fb.c
index 99acf538a8b8..9a5751cb4e16 100644
--- a/drivers/video/fbdev/cyber2000fb.c
+++ b/drivers/video/fbdev/cyber2000fb.c
@@ -1336,7 +1336,7 @@ static void cyber2000fb_i2c_unregister(struct cfb_info *cfb)
  * These parameters give
  * 640x480, hsync 31.5kHz, vsync 60Hz
  */
-static struct fb_videomode cyber2000fb_default_mode = {
+static const struct fb_videomode cyber2000fb_default_mode = {
 	.refresh	= 60,
 	.xres		= 640,
 	.yres		= 480,
diff --git a/drivers/video/fbdev/da8xx-fb.c b/drivers/video/fbdev/da8xx-fb.c
index c229b1a0d13b..a74096c53cb5 100644
--- a/drivers/video/fbdev/da8xx-fb.c
+++ b/drivers/video/fbdev/da8xx-fb.c
@@ -1341,7 +1341,7 @@ static int fb_probe(struct platform_device *device)
 {
 	struct da8xx_lcdc_platform_data *fb_pdata =
 						dev_get_platdata(&device->dev);
-	static struct resource *lcdc_regs;
+	struct resource *lcdc_regs;
 	struct lcd_ctrl_config *lcd_cfg;
 	struct fb_videomode *lcdc_info;
 	struct fb_info *da8xx_fb_info;
diff --git a/drivers/video/fbdev/dnfb.c b/drivers/video/fbdev/dnfb.c
index 3526899da61b..7b1492d34e98 100644
--- a/drivers/video/fbdev/dnfb.c
+++ b/drivers/video/fbdev/dnfb.c
@@ -126,7 +126,7 @@ struct fb_var_screeninfo dnfb_var = {
 	.vmode		= FB_VMODE_NONINTERLACED,
 };
 
-static struct fb_fix_screeninfo dnfb_fix = {
+static const struct fb_fix_screeninfo dnfb_fix = {
 	.id		= "Apollo Mono",
 	.smem_start	= (FRAME_BUFFER_START + IO_BASE),
 	.smem_len	= FRAME_BUFFER_LEN,
diff --git a/drivers/video/fbdev/efifb.c b/drivers/video/fbdev/efifb.c
index 1e784adb89b1..3a010641f630 100644
--- a/drivers/video/fbdev/efifb.c
+++ b/drivers/video/fbdev/efifb.c
@@ -149,6 +149,10 @@ ATTRIBUTE_GROUPS(efifb);
 
 static bool pci_dev_disabled;	/* FB base matches BAR of a disabled device */
 
+static struct pci_dev *efifb_pci_dev;	/* dev with BAR covering the efifb */
+static struct resource *bar_resource;
+static u64 bar_offset;
+
 static int efifb_probe(struct platform_device *dev)
 {
 	struct fb_info *info;
@@ -203,6 +207,13 @@ static int efifb_probe(struct platform_device *dev)
 		efifb_fix.smem_start |= ext_lfb_base;
 	}
 
+	if (bar_resource &&
+	    bar_resource->start + bar_offset != efifb_fix.smem_start) {
+		dev_info(&efifb_pci_dev->dev,
+			 "BAR has moved, updating efifb address\n");
+		efifb_fix.smem_start = bar_resource->start + bar_offset;
+	}
+
 	efifb_defined.bits_per_pixel = screen_info.lfb_depth;
 	efifb_defined.xres = screen_info.lfb_width;
 	efifb_defined.yres = screen_info.lfb_height;
@@ -370,15 +381,13 @@ static struct platform_driver efifb_driver = {
 
 builtin_platform_driver(efifb_driver);
 
-#if defined(CONFIG_PCI) && !defined(CONFIG_X86)
-
-static bool pci_bar_found;	/* did we find a BAR matching the efifb base? */
+#if defined(CONFIG_PCI)
 
-static void claim_efifb_bar(struct pci_dev *dev, int idx)
+static void record_efifb_bar_resource(struct pci_dev *dev, int idx, u64 offset)
 {
 	u16 word;
 
-	pci_bar_found = true;
+	efifb_pci_dev = dev;
 
 	pci_read_config_word(dev, PCI_COMMAND, &word);
 	if (!(word & PCI_COMMAND_MEMORY)) {
@@ -389,12 +398,8 @@ static void claim_efifb_bar(struct pci_dev *dev, int idx)
 		return;
 	}
 
-	if (pci_claim_resource(dev, idx)) {
-		pci_dev_disabled = true;
-		dev_err(&dev->dev,
-			"BAR %d: failed to claim resource for efifb!\n", idx);
-		return;
-	}
+	bar_resource = &dev->resource[idx];
+	bar_offset = offset;
 
 	dev_info(&dev->dev, "BAR %d: assigned to efifb\n", idx);
 }
@@ -405,7 +410,7 @@ static void efifb_fixup_resources(struct pci_dev *dev)
 	u64 size = screen_info.lfb_size;
 	int i;
 
-	if (pci_bar_found || screen_info.orig_video_isVGA != VIDEO_TYPE_EFI)
+	if (efifb_pci_dev || screen_info.orig_video_isVGA != VIDEO_TYPE_EFI)
 		return;
 
 	if (screen_info.capabilities & VIDEO_CAPABILITY_64BIT_BASE)
@@ -421,7 +426,7 @@ static void efifb_fixup_resources(struct pci_dev *dev)
 			continue;
 
 		if (res->start <= base && res->end >= base + size - 1) {
-			claim_efifb_bar(dev, i);
+			record_efifb_bar_resource(dev, i, base - res->start);
 			break;
 		}
 	}
diff --git a/drivers/video/fbdev/fb-puv3.c b/drivers/video/fbdev/fb-puv3.c
index 88fa2e70a0bb..d9e816d53531 100644
--- a/drivers/video/fbdev/fb-puv3.c
+++ b/drivers/video/fbdev/fb-puv3.c
@@ -69,7 +69,7 @@ static const struct fb_videomode unifb_modes[] = {
 	  0, FB_VMODE_NONINTERLACED, FB_MODE_IS_VESA },
 };
 
-static struct fb_var_screeninfo unifb_default = {
+static const struct fb_var_screeninfo unifb_default = {
 	.xres =		640,
 	.yres =		480,
 	.xres_virtual =	640,
diff --git a/drivers/video/fbdev/ffb.c b/drivers/video/fbdev/ffb.c
index dda31e0a45af..6b1915872af1 100644
--- a/drivers/video/fbdev/ffb.c
+++ b/drivers/video/fbdev/ffb.c
@@ -997,9 +997,9 @@ static int ffb_probe(struct platform_device *op)
 
 	dev_set_drvdata(&op->dev, info);
 
-	printk(KERN_INFO "%s: %s at %016lx, type %d, "
+	printk(KERN_INFO "%pOF: %s at %016lx, type %d, "
 	       "DAC pnum[%x] rev[%d] manuf_rev[%d]\n",
-	       dp->full_name,
+	       dp,
 	       ((par->flags & FFB_FLAG_AFB) ? "AFB" : "FFB"),
 	       par->physbase, par->board_type,
 	       dac_pnum, dac_rev, dac_mrev);
diff --git a/drivers/video/fbdev/fm2fb.c b/drivers/video/fbdev/fm2fb.c
index e69d47af9932..ac7a4ebfd390 100644
--- a/drivers/video/fbdev/fm2fb.c
+++ b/drivers/video/fbdev/fm2fb.c
@@ -213,7 +213,7 @@ static int fm2fb_setcolreg(u_int regno, u_int red, u_int green, u_int blue,
 
 static int fm2fb_probe(struct zorro_dev *z, const struct zorro_device_id *id);
 
-static struct zorro_device_id fm2fb_devices[] = {
+static const struct zorro_device_id fm2fb_devices[] = {
 	{ ZORRO_PROD_BSC_FRAMEMASTER_II },
 	{ ZORRO_PROD_HELFRICH_RAINBOW_II },
 	{ 0 }
diff --git a/drivers/video/fbdev/geode/gxfb_core.c b/drivers/video/fbdev/geode/gxfb_core.c
index ec9fc9ac23de..f4f76373b2a8 100644
--- a/drivers/video/fbdev/geode/gxfb_core.c
+++ b/drivers/video/fbdev/geode/gxfb_core.c
@@ -474,7 +474,7 @@ static void gxfb_remove(struct pci_dev *pdev)
 	framebuffer_release(info);
 }
 
-static struct pci_device_id gxfb_id_table[] = {
+static const struct pci_device_id gxfb_id_table[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_NS, PCI_DEVICE_ID_NS_GX_VIDEO) },
 	{ 0, }
 };
diff --git a/drivers/video/fbdev/grvga.c b/drivers/video/fbdev/grvga.c
index b471f92969b1..8fc8f46dadeb 100644
--- a/drivers/video/fbdev/grvga.c
+++ b/drivers/video/fbdev/grvga.c
@@ -70,7 +70,7 @@ static const struct fb_videomode grvga_modedb[] = {
     }
  };
 
-static struct fb_fix_screeninfo grvga_fix = {
+static const struct fb_fix_screeninfo grvga_fix = {
 	.id =		"AG SVGACTRL",
 	.type =		FB_TYPE_PACKED_PIXELS,
 	.visual =       FB_VISUAL_PSEUDOCOLOR,
diff --git a/drivers/video/fbdev/i810/i810_main.c b/drivers/video/fbdev/i810/i810_main.c
index 2488baab7c89..d18f7b31932c 100644
--- a/drivers/video/fbdev/i810/i810_main.c
+++ b/drivers/video/fbdev/i810/i810_main.c
@@ -107,7 +107,7 @@ static const char * const i810_pci_list[] = {
 	"Intel(R) 815 (Internal Graphics with AGP) Framebuffer Device"
 };
 
-static struct pci_device_id i810fb_pci_tbl[] = {
+static const struct pci_device_id i810fb_pci_tbl[] = {
 	{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82810_IG1,
 	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
 	{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82810_IG3,
@@ -1542,7 +1542,7 @@ static int i810fb_cursor(struct fb_info *info, struct fb_cursor *cursor)
 	return 0;
 }
 
-static struct fb_ops i810fb_ops = {
+static const struct fb_ops i810fb_ops = {
 	.owner =             THIS_MODULE,
 	.fb_open =           i810fb_open,
 	.fb_release =        i810fb_release,
diff --git a/drivers/video/fbdev/imsttfb.c b/drivers/video/fbdev/imsttfb.c
index 4363c64d74e8..ecdcf358ad5e 100644
--- a/drivers/video/fbdev/imsttfb.c
+++ b/drivers/video/fbdev/imsttfb.c
@@ -1318,7 +1318,7 @@ imsttfb_ioctl(struct fb_info *info, u_int cmd, u_long arg)
 	}
 }
 
-static struct pci_device_id imsttfb_pci_tbl[] = {
+static const struct pci_device_id imsttfb_pci_tbl[] = {
 	{ PCI_VENDOR_ID_IMS, PCI_DEVICE_ID_IMS_TT128,
 	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, IBM },
 	{ PCI_VENDOR_ID_IMS, PCI_DEVICE_ID_IMS_TT3D,
diff --git a/drivers/video/fbdev/intelfb/intelfbdrv.c b/drivers/video/fbdev/intelfb/intelfbdrv.c
index ffc391208b27..d7463a2a5d83 100644
--- a/drivers/video/fbdev/intelfb/intelfbdrv.c
+++ b/drivers/video/fbdev/intelfb/intelfbdrv.c
@@ -173,7 +173,7 @@ static int intelfb_set_fbinfo(struct intelfb_info *dinfo);
 #define INTELFB_CLASS_MASK 0
 #endif
 
-static struct pci_device_id intelfb_pci_table[] = {
+static const struct pci_device_id intelfb_pci_table[] = {
 	{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_830M, PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_DISPLAY_VGA << 8, INTELFB_CLASS_MASK, INTEL_830M },
 	{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_845G, PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_DISPLAY_VGA << 8, INTELFB_CLASS_MASK, INTEL_845G },
 	{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_85XGM, PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_DISPLAY_VGA << 8, INTELFB_CLASS_MASK, INTEL_85XGM },
diff --git a/drivers/video/fbdev/kyro/fbdev.c b/drivers/video/fbdev/kyro/fbdev.c
index f77478fb3d14..a7bd9f25911b 100644
--- a/drivers/video/fbdev/kyro/fbdev.c
+++ b/drivers/video/fbdev/kyro/fbdev.c
@@ -633,7 +633,7 @@ static int kyrofb_ioctl(struct fb_info *info,
 	return 0;
 }
 
-static struct pci_device_id kyrofb_pci_tbl[] = {
+static const struct pci_device_id kyrofb_pci_tbl[] = {
 	{ PCI_VENDOR_ID_ST, PCI_DEVICE_ID_STG4000,
 	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
 	{ 0, }
diff --git a/drivers/video/fbdev/leo.c b/drivers/video/fbdev/leo.c
index 62e59dc90ee6..71862188f528 100644
--- a/drivers/video/fbdev/leo.c
+++ b/drivers/video/fbdev/leo.c
@@ -619,8 +619,8 @@ static int leo_probe(struct platform_device *op)
 
 	dev_set_drvdata(&op->dev, info);
 
-	printk(KERN_INFO "%s: leo at %lx:%lx\n",
-	       dp->full_name,
+	printk(KERN_INFO "%pOF: leo at %lx:%lx\n",
+	       dp,
 	       par->which_io, info->fix.smem_start);
 
 	return 0;
diff --git a/drivers/video/fbdev/matrox/matroxfb_base.c b/drivers/video/fbdev/matrox/matroxfb_base.c
index f6a0b9af97a9..b9b284d79631 100644
--- a/drivers/video/fbdev/matrox/matroxfb_base.c
+++ b/drivers/video/fbdev/matrox/matroxfb_base.c
@@ -1198,7 +1198,7 @@ static int matroxfb_blank(int blank, struct fb_info *info)
 	return 0;
 }
 
-static struct fb_ops matroxfb_ops = {
+static const struct fb_ops matroxfb_ops = {
 	.owner =	THIS_MODULE,
 	.fb_open =	matroxfb_open,
 	.fb_release =	matroxfb_release,
@@ -1573,14 +1573,14 @@ static struct board {
 		NULL}};
 
 #ifndef MODULE
-static struct fb_videomode defaultmode = {
+static const struct fb_videomode defaultmode = {
 	/* 640x480 @ 60Hz, 31.5 kHz */
 	NULL, 60, 640, 480, 39721, 40, 24, 32, 11, 96, 2,
 	0, FB_VMODE_NONINTERLACED
 };
-#endif /* !MODULE */
 
 static int hotplug = 0;
+#endif /* !MODULE */
 
 static void setDefaultOutputs(struct matrox_fb_info *minfo)
 {
@@ -1623,7 +1623,7 @@ static int initMatrox2(struct matrox_fb_info *minfo, struct board *b)
 	unsigned int memsize;
 	int err;
 
-	static struct pci_device_id intel_82437[] = {
+	static const struct pci_device_id intel_82437[] = {
 		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82437) },
 		{ },
 	};
@@ -1794,9 +1794,7 @@ static int initMatrox2(struct matrox_fb_info *minfo, struct board *b)
 	minfo->fbops = matroxfb_ops;
 	minfo->fbcon.fbops = &minfo->fbops;
 	minfo->fbcon.pseudo_palette = minfo->cmap;
-	/* after __init time we are like module... no logo */
-	minfo->fbcon.flags = hotplug ? FBINFO_FLAG_MODULE : FBINFO_FLAG_DEFAULT;
-	minfo->fbcon.flags |= FBINFO_PARTIAL_PAN_OK | 	 /* Prefer panning for scroll under MC viewer/edit */
+	minfo->fbcon.flags = FBINFO_PARTIAL_PAN_OK | 	 /* Prefer panning for scroll under MC viewer/edit */
 				      FBINFO_HWACCEL_COPYAREA |  /* We have hw-assisted bmove */
 				      FBINFO_HWACCEL_FILLRECT |  /* And fillrect */
 				      FBINFO_HWACCEL_IMAGEBLIT | /* And imageblit */
@@ -2116,7 +2114,7 @@ static void pci_remove_matrox(struct pci_dev* pdev) {
 	matroxfb_remove(minfo, 1);
 }
 
-static struct pci_device_id matroxfb_devices[] = {
+static const struct pci_device_id matroxfb_devices[] = {
 #ifdef CONFIG_FB_MATROX_MILLENIUM
 	{PCI_VENDOR_ID_MATROX,	PCI_DEVICE_ID_MATROX_MIL,
 		PCI_ANY_ID,	PCI_ANY_ID,	0, 0, 0},
diff --git a/drivers/video/fbdev/maxinefb.c b/drivers/video/fbdev/maxinefb.c
index cab7333208ea..5bb1b5c308a7 100644
--- a/drivers/video/fbdev/maxinefb.c
+++ b/drivers/video/fbdev/maxinefb.c
@@ -39,7 +39,7 @@
 
 static struct fb_info fb_info;
 
-static struct fb_var_screeninfo maxinefb_defined = {
+static const struct fb_var_screeninfo maxinefb_defined = {
 	.xres =		1024,
 	.yres =		768,
 	.xres_virtual =	1024,
diff --git a/drivers/video/fbdev/mb862xx/mb862xxfbdrv.c b/drivers/video/fbdev/mb862xx/mb862xxfbdrv.c
index f9ec5c0484fa..cd372527c9e4 100644
--- a/drivers/video/fbdev/mb862xx/mb862xxfbdrv.c
+++ b/drivers/video/fbdev/mb862xx/mb862xxfbdrv.c
@@ -982,7 +982,7 @@ static inline int mb862xx_pci_gdc_init(struct mb862xxfb_par *par)
 #define CHIP_ID(id)	\
 	{ PCI_DEVICE(PCI_VENDOR_ID_FUJITSU_LIMITED, id) }
 
-static struct pci_device_id mb862xx_pci_tbl[] = {
+static const struct pci_device_id mb862xx_pci_tbl[] = {
 	/* MB86295/MB86296 */
 	CHIP_ID(PCI_DEVICE_ID_FUJITSU_CORALP),
 	CHIP_ID(PCI_DEVICE_ID_FUJITSU_CORALPA),
diff --git a/drivers/video/fbdev/mbx/mbxfb.c b/drivers/video/fbdev/mbx/mbxfb.c
index 698df9543e30..539b85da0897 100644
--- a/drivers/video/fbdev/mbx/mbxfb.c
+++ b/drivers/video/fbdev/mbx/mbxfb.c
@@ -79,7 +79,7 @@ struct mbxfb_info {
 
 };
 
-static struct fb_var_screeninfo mbxfb_default = {
+static const struct fb_var_screeninfo mbxfb_default = {
 	.xres = 640,
 	.yres = 480,
 	.xres_virtual = 640,
@@ -102,7 +102,7 @@ static struct fb_var_screeninfo mbxfb_default = {
 	.sync = FB_SYNC_HOR_HIGH_ACT | FB_SYNC_VERT_HIGH_ACT,
 };
 
-static struct fb_fix_screeninfo mbxfb_fix = {
+static const struct fb_fix_screeninfo mbxfb_fix = {
 	.id = "MBX",
 	.type = FB_TYPE_PACKED_PIXELS,
 	.visual = FB_VISUAL_TRUECOLOR,
diff --git a/drivers/video/fbdev/neofb.c b/drivers/video/fbdev/neofb.c
index db023a97d1ea..5d3a444083f7 100644
--- a/drivers/video/fbdev/neofb.c
+++ b/drivers/video/fbdev/neofb.c
@@ -2138,7 +2138,7 @@ static void neofb_remove(struct pci_dev *dev)
 	}
 }
 
-static struct pci_device_id neofb_devices[] = {
+static const struct pci_device_id neofb_devices[] = {
 	{PCI_VENDOR_ID_NEOMAGIC, PCI_CHIP_NM2070,
 	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_NEOMAGIC_NM2070},
 
diff --git a/drivers/video/fbdev/nvidia/nvidia.c b/drivers/video/fbdev/nvidia/nvidia.c
index ce7dab7299fe..418a2d0d06a9 100644
--- a/drivers/video/fbdev/nvidia/nvidia.c
+++ b/drivers/video/fbdev/nvidia/nvidia.c
@@ -55,7 +55,7 @@
 /* HW cursor parameters */
 #define MAX_CURS		32
 
-static struct pci_device_id nvidiafb_pci_tbl[] = {
+static const struct pci_device_id nvidiafb_pci_tbl[] = {
 	{PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
 	 PCI_BASE_CLASS_DISPLAY << 16, 0xff0000, 0},
 	{ 0, }
diff --git a/drivers/video/fbdev/offb.c b/drivers/video/fbdev/offb.c
index 9be884b0c778..90d38de34479 100644
--- a/drivers/video/fbdev/offb.c
+++ b/drivers/video/fbdev/offb.c
@@ -383,7 +383,7 @@ static void offb_init_palette_hacks(struct fb_info *info, struct device_node *dp
 		FB_VISUAL_PSEUDOCOLOR : FB_VISUAL_STATIC_PSEUDOCOLOR;
 }
 
-static void __init offb_init_fb(const char *name, const char *full_name,
+static void __init offb_init_fb(const char *name,
 				int width, int height, int depth,
 				int pitch, unsigned long address,
 				int foreign_endian, struct device_node *dp)
@@ -402,14 +402,13 @@ static void __init offb_init_fb(const char *name, const char *full_name,
 	       "Using unsupported %dx%d %s at %lx, depth=%d, pitch=%d\n",
 	       width, height, name, address, depth, pitch);
 	if (depth != 8 && depth != 15 && depth != 16 && depth != 32) {
-		printk(KERN_ERR "%s: can't use depth = %d\n", full_name,
-		       depth);
+		printk(KERN_ERR "%pOF: can't use depth = %d\n", dp, depth);
 		release_mem_region(res_start, res_size);
 		return;
 	}
 
 	info = framebuffer_alloc(sizeof(u32) * 16, NULL);
-	
+
 	if (info == 0) {
 		release_mem_region(res_start, res_size);
 		return;
@@ -515,7 +514,7 @@ static void __init offb_init_fb(const char *name, const char *full_name,
 	if (register_framebuffer(info) < 0)
 		goto out_err;
 
-	fb_info(info, "Open Firmware frame buffer device on %s\n", full_name);
+	fb_info(info, "Open Firmware frame buffer device on %pOF\n", dp);
 	return;
 
 out_err:
@@ -644,7 +643,6 @@ static void __init offb_init_nodriver(struct device_node *dp, int no_real_node)
 		if (strcmp(dp->name, "valkyrie") == 0)
 			address += 0x1000;
 		offb_init_fb(no_real_node ? "bootx" : dp->name,
-			     no_real_node ? "display" : dp->full_name,
 			     width, height, depth, pitch, address,
 			     foreign_endian, no_real_node ? NULL : dp);
 	}
diff --git a/drivers/video/fbdev/omap/lcd_h3.c b/drivers/video/fbdev/omap/lcd_h3.c
index 9d2da146813e..796f4634c4c6 100644
--- a/drivers/video/fbdev/omap/lcd_h3.c
+++ b/drivers/video/fbdev/omap/lcd_h3.c
@@ -21,7 +21,7 @@
 
 #include <linux/module.h>
 #include <linux/platform_device.h>
-#include <linux/i2c/tps65010.h>
+#include <linux/mfd/tps65010.h>
 #include <linux/gpio.h>
 
 #include "omapfb.h"
diff --git a/drivers/video/fbdev/omap/lcd_mipid.c b/drivers/video/fbdev/omap/lcd_mipid.c
index df9e6ebcfad5..e3a85432f926 100644
--- a/drivers/video/fbdev/omap/lcd_mipid.c
+++ b/drivers/video/fbdev/omap/lcd_mipid.c
@@ -496,7 +496,7 @@ static void mipid_cleanup(struct lcd_panel *panel)
 		mipid_esd_stop_check(md);
 }
 
-static struct lcd_panel mipid_panel = {
+static const struct lcd_panel mipid_panel = {
 	.config		= OMAP_LCDC_PANEL_TFT,
 
 	.bpp		= 16,
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/panel-lgphilips-lb035q02.c b/drivers/video/fbdev/omap2/omapfb/displays/panel-lgphilips-lb035q02.c
index f14691ce8d02..6cd759c01037 100644
--- a/drivers/video/fbdev/omap2/omapfb/displays/panel-lgphilips-lb035q02.c
+++ b/drivers/video/fbdev/omap2/omapfb/displays/panel-lgphilips-lb035q02.c
@@ -18,7 +18,7 @@
 
 #include <video/omapfb_dss.h>
 
-static struct omap_video_timings lb035q02_timings = {
+static const struct omap_video_timings lb035q02_timings = {
 	.x_res = 320,
 	.y_res = 240,
 
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/panel-sony-acx565akm.c b/drivers/video/fbdev/omap2/omapfb/displays/panel-sony-acx565akm.c
index 468560a6daae..f2c2fef3db74 100644
--- a/drivers/video/fbdev/omap2/omapfb/displays/panel-sony-acx565akm.c
+++ b/drivers/video/fbdev/omap2/omapfb/displays/panel-sony-acx565akm.c
@@ -509,7 +509,7 @@ static struct attribute *bldev_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group bldev_attr_group = {
+static const struct attribute_group bldev_attr_group = {
 	.attrs = bldev_attrs,
 };
 
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td028ttec1.c b/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td028ttec1.c
index b529a8c2b652..57e9e146ff74 100644
--- a/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td028ttec1.c
+++ b/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td028ttec1.c
@@ -41,7 +41,7 @@ struct panel_drv_data {
 	struct spi_device *spi_dev;
 };
 
-static struct omap_video_timings td028ttec1_panel_timings = {
+static const struct omap_video_timings td028ttec1_panel_timings = {
 	.x_res		= 480,
 	.y_res		= 640,
 	.pixelclock	= 22153000,
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td043mtea1.c b/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td043mtea1.c
index 51e628b85f4a..ea8c79a42b41 100644
--- a/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td043mtea1.c
+++ b/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td043mtea1.c
@@ -282,7 +282,7 @@ static struct attribute *tpo_td043_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group tpo_td043_attr_group = {
+static const struct attribute_group tpo_td043_attr_group = {
 	.attrs = tpo_td043_attrs,
 };
 
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dss-of.c b/drivers/video/fbdev/omap2/omapfb/dss/dss-of.c
index d356a252ab4a..f1eb8b0f8a2a 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/dss-of.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dss-of.c
@@ -16,6 +16,7 @@
 #include <linux/err.h>
 #include <linux/module.h>
 #include <linux/of.h>
+#include <linux/of_graph.h>
 #include <linux/seq_file.h>
 
 #include <video/omapfb_dss.h>
@@ -128,7 +129,7 @@ static struct device_node *omapdss_of_get_remote_port(const struct device_node *
 {
 	struct device_node *np;
 
-	np = of_parse_phandle(node, "remote-endpoint", 0);
+	np = of_graph_get_remote_endpoint(node);
 	if (!np)
 		return NULL;
 
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/hdmi4.c b/drivers/video/fbdev/omap2/omapfb/dss/hdmi4.c
index 156a254705ea..ec78d61bc551 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/hdmi4.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/hdmi4.c
@@ -664,7 +664,7 @@ static int hdmi_audio_register(struct device *dev)
 {
 	struct omap_hdmi_audio_pdata pdata = {
 		.dev = dev,
-		.dss_version = omapdss_get_version(),
+		.version = 4,
 		.audio_dma_addr = hdmi_wp_get_audio_dma_addr(&hdmi.wp),
 		.ops = &hdmi_audio_ops,
 	};
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/hdmi5.c b/drivers/video/fbdev/omap2/omapfb/dss/hdmi5.c
index 4da36bcab977..2e2fcc3d6d4f 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/hdmi5.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/hdmi5.c
@@ -695,7 +695,7 @@ static int hdmi_audio_register(struct device *dev)
 {
 	struct omap_hdmi_audio_pdata pdata = {
 		.dev = dev,
-		.dss_version = omapdss_get_version(),
+		.version = 5,
 		.audio_dma_addr = hdmi_wp_get_audio_dma_addr(&hdmi.wp),
 		.ops = &hdmi_audio_ops,
 	};
diff --git a/drivers/video/fbdev/p9100.c b/drivers/video/fbdev/p9100.c
index 1f6ee76af878..64de5cda541d 100644
--- a/drivers/video/fbdev/p9100.c
+++ b/drivers/video/fbdev/p9100.c
@@ -304,8 +304,8 @@ static int p9100_probe(struct platform_device *op)
 
 	dev_set_drvdata(&op->dev, info);
 
-	printk(KERN_INFO "%s: p9100 at %lx:%lx\n",
-	       dp->full_name,
+	printk(KERN_INFO "%pOF: p9100 at %lx:%lx\n",
+	       dp,
 	       par->which_io, info->fix.smem_start);
 
 	return 0;
diff --git a/drivers/video/fbdev/pm2fb.c b/drivers/video/fbdev/pm2fb.c
index 1a4070f719c2..bd6c2f5f6095 100644
--- a/drivers/video/fbdev/pm2fb.c
+++ b/drivers/video/fbdev/pm2fb.c
@@ -1732,7 +1732,7 @@ static void pm2fb_remove(struct pci_dev *pdev)
 	framebuffer_release(info);
 }
 
-static struct pci_device_id pm2fb_id_table[] = {
+static const struct pci_device_id pm2fb_id_table[] = {
 	{ PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_TVP4020,
 	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
 	{ PCI_VENDOR_ID_3DLABS, PCI_DEVICE_ID_3DLABS_PERMEDIA2,
diff --git a/drivers/video/fbdev/pm3fb.c b/drivers/video/fbdev/pm3fb.c
index 6ff5077a2e15..6130aa56a1e9 100644
--- a/drivers/video/fbdev/pm3fb.c
+++ b/drivers/video/fbdev/pm3fb.c
@@ -1479,7 +1479,7 @@ static void pm3fb_remove(struct pci_dev *dev)
 	}
 }
 
-static struct pci_device_id pm3fb_id_table[] = {
+static const struct pci_device_id pm3fb_id_table[] = {
 	{ PCI_VENDOR_ID_3DLABS, 0x0a,
 	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
 	{ 0, }
diff --git a/drivers/video/fbdev/pmag-aa-fb.c b/drivers/video/fbdev/pmag-aa-fb.c
index 39922f072db4..ca7e9390d1e7 100644
--- a/drivers/video/fbdev/pmag-aa-fb.c
+++ b/drivers/video/fbdev/pmag-aa-fb.c
@@ -67,7 +67,7 @@ struct aafb_par {
 	struct bt431_regs __iomem *bt431;
 };
 
-static struct fb_var_screeninfo aafb_defined = {
+static const struct fb_var_screeninfo aafb_defined = {
 	.xres		= 1280,
 	.yres		= 1024,
 	.xres_virtual	= 2048,
@@ -90,7 +90,7 @@ static struct fb_var_screeninfo aafb_defined = {
 	.vmode		= FB_VMODE_NONINTERLACED,
 };
 
-static struct fb_fix_screeninfo aafb_fix = {
+static const struct fb_fix_screeninfo aafb_fix = {
 	.id		= "PMAG-AA",
 	.smem_len	= (2048 * 1024),
 	.type		= FB_TYPE_PACKED_PIXELS,
diff --git a/drivers/video/fbdev/pmag-ba-fb.c b/drivers/video/fbdev/pmag-ba-fb.c
index 1fd02f40708e..3b9249449ea6 100644
--- a/drivers/video/fbdev/pmag-ba-fb.c
+++ b/drivers/video/fbdev/pmag-ba-fb.c
@@ -43,7 +43,7 @@ struct pmagbafb_par {
 };
 
 
-static struct fb_var_screeninfo pmagbafb_defined = {
+static const struct fb_var_screeninfo pmagbafb_defined = {
 	.xres		= 1024,
 	.yres		= 864,
 	.xres_virtual	= 1024,
@@ -67,7 +67,7 @@ static struct fb_var_screeninfo pmagbafb_defined = {
 	.vmode		= FB_VMODE_NONINTERLACED,
 };
 
-static struct fb_fix_screeninfo pmagbafb_fix = {
+static const struct fb_fix_screeninfo pmagbafb_fix = {
 	.id		= "PMAG-BA",
 	.smem_len	= (1024 * 1024),
 	.type		= FB_TYPE_PACKED_PIXELS,
diff --git a/drivers/video/fbdev/pmagb-b-fb.c b/drivers/video/fbdev/pmagb-b-fb.c
index 46e96c451506..e58df36233c4 100644
--- a/drivers/video/fbdev/pmagb-b-fb.c
+++ b/drivers/video/fbdev/pmagb-b-fb.c
@@ -44,7 +44,7 @@ struct pmagbbfb_par {
 };
 
 
-static struct fb_var_screeninfo pmagbbfb_defined = {
+static const struct fb_var_screeninfo pmagbbfb_defined = {
 	.bits_per_pixel	= 8,
 	.red.length	= 8,
 	.green.length	= 8,
@@ -57,7 +57,7 @@ static struct fb_var_screeninfo pmagbbfb_defined = {
 	.vmode		= FB_VMODE_NONINTERLACED,
 };
 
-static struct fb_fix_screeninfo pmagbbfb_fix = {
+static const struct fb_fix_screeninfo pmagbbfb_fix = {
 	.id		= "PMAGB-BA",
 	.smem_len	= (2048 * 1024),
 	.type		= FB_TYPE_PACKED_PIXELS,
diff --git a/drivers/video/fbdev/ps3fb.c b/drivers/video/fbdev/ps3fb.c
index b269abd932aa..5ed2db39d823 100644
--- a/drivers/video/fbdev/ps3fb.c
+++ b/drivers/video/fbdev/ps3fb.c
@@ -952,7 +952,7 @@ static struct fb_ops ps3fb_ops = {
 	.fb_compat_ioctl = ps3fb_ioctl
 };
 
-static struct fb_fix_screeninfo ps3fb_fix = {
+static const struct fb_fix_screeninfo ps3fb_fix = {
 	.id =		DEVICE_NAME,
 	.type =		FB_TYPE_PACKED_PIXELS,
 	.visual =	FB_VISUAL_TRUECOLOR,
diff --git a/drivers/video/fbdev/pvr2fb.c b/drivers/video/fbdev/pvr2fb.c
index a2564ab91e62..867c5218968f 100644
--- a/drivers/video/fbdev/pvr2fb.c
+++ b/drivers/video/fbdev/pvr2fb.c
@@ -154,7 +154,7 @@ static struct fb_fix_screeninfo pvr2_fix = {
 	.accel =	FB_ACCEL_NONE,
 };
 
-static struct fb_var_screeninfo pvr2_var = {
+static const struct fb_var_screeninfo pvr2_var = {
 	.xres =		640,
 	.yres =		480,
 	.xres_virtual =	640,
@@ -966,7 +966,7 @@ static void pvr2fb_pci_remove(struct pci_dev *pdev)
 	pci_release_regions(pdev);
 }
 
-static struct pci_device_id pvr2fb_pci_tbl[] = {
+static const struct pci_device_id pvr2fb_pci_tbl[] = {
 	{ PCI_VENDOR_ID_NEC, PCI_DEVICE_ID_NEC_NEON250,
 	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
 	{ 0, },
diff --git a/drivers/video/fbdev/pxa3xx-gcu.c b/drivers/video/fbdev/pxa3xx-gcu.c
index 50bce45e7f3d..933619da1a94 100644
--- a/drivers/video/fbdev/pxa3xx-gcu.c
+++ b/drivers/video/fbdev/pxa3xx-gcu.c
@@ -626,8 +626,8 @@ static int pxa3xx_gcu_probe(struct platform_device *pdev)
 	/* request the IRQ */
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
-		dev_err(dev, "no IRQ defined\n");
-		return -ENODEV;
+		dev_err(dev, "no IRQ defined: %d\n", irq);
+		return irq;
 	}
 
 	ret = devm_request_irq(dev, irq, pxa3xx_gcu_handle_irq,
diff --git a/drivers/video/fbdev/q40fb.c b/drivers/video/fbdev/q40fb.c
index 04ea330ccf5d..0b93aa964d43 100644
--- a/drivers/video/fbdev/q40fb.c
+++ b/drivers/video/fbdev/q40fb.c
@@ -36,7 +36,7 @@ static struct fb_fix_screeninfo q40fb_fix = {
 	.accel		= FB_ACCEL_NONE,
 };
 
-static struct fb_var_screeninfo q40fb_var = {
+static const struct fb_var_screeninfo q40fb_var = {
 	.xres		= 1024,
 	.yres		= 512,
 	.xres_virtual	= 1024,
diff --git a/drivers/video/fbdev/riva/fbdev.c b/drivers/video/fbdev/riva/fbdev.c
index 2ef26ad99341..1ea78bb911fb 100644
--- a/drivers/video/fbdev/riva/fbdev.c
+++ b/drivers/video/fbdev/riva/fbdev.c
@@ -101,7 +101,7 @@ static int rivafb_blank(int blank, struct fb_info *info);
  *
  * ------------------------------------------------------------------------- */
 
-static struct pci_device_id rivafb_pci_tbl[] = {
+static const struct pci_device_id rivafb_pci_tbl[] = {
 	{ PCI_VENDOR_ID_NVIDIA_SGS, PCI_DEVICE_ID_NVIDIA_SGS_RIVA128,
 	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
 	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_TNT,
diff --git a/drivers/video/fbdev/s3fb.c b/drivers/video/fbdev/s3fb.c
index 13b109073c63..d63f23e26f7d 100644
--- a/drivers/video/fbdev/s3fb.c
+++ b/drivers/video/fbdev/s3fb.c
@@ -1483,7 +1483,7 @@ static int s3_pci_resume(struct pci_dev* dev)
 
 /* List of boards that we are trying to support */
 
-static struct pci_device_id s3_devices[] = {
+static const struct pci_device_id s3_devices[] = {
 	{PCI_DEVICE(PCI_VENDOR_ID_S3, 0x8810), .driver_data = CHIP_XXX_TRIO},
 	{PCI_DEVICE(PCI_VENDOR_ID_S3, 0x8811), .driver_data = CHIP_XXX_TRIO},
 	{PCI_DEVICE(PCI_VENDOR_ID_S3, 0x8812), .driver_data = CHIP_M65_AURORA64VP},
diff --git a/drivers/video/fbdev/savage/savagefb_driver.c b/drivers/video/fbdev/savage/savagefb_driver.c
index c30a91c1137c..c20468362f11 100644
--- a/drivers/video/fbdev/savage/savagefb_driver.c
+++ b/drivers/video/fbdev/savage/savagefb_driver.c
@@ -2429,7 +2429,7 @@ static int savagefb_resume(struct pci_dev* dev)
 }
 
 
-static struct pci_device_id savagefb_devices[] = {
+static const struct pci_device_id savagefb_devices[] = {
 	{PCI_VENDOR_ID_S3, PCI_CHIP_SUPSAV_MX128,
 	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_SUPERSAVAGE},
 
diff --git a/drivers/video/fbdev/sis/init301.c b/drivers/video/fbdev/sis/init301.c
index 20f7234e809e..1ec9c3e0e1d8 100644
--- a/drivers/video/fbdev/sis/init301.c
+++ b/drivers/video/fbdev/sis/init301.c
@@ -6848,8 +6848,6 @@ SiS_SetGroup2(struct SiS_Private *SiS_Pr, unsigned short ModeNo, unsigned short
 	   if(SiS_Pr->SiS_VGAHDE >= 1280) {
               tempch = 20;
               tempbx &= ~0x20;
-           } else if(SiS_Pr->SiS_VGAHDE >= 1024) {
-              tempch = 25;
            } else {
 	      tempch = 25; /* OK */
 	   }
@@ -7964,14 +7962,9 @@ SiS_SetCHTVReg(struct SiS_Private *SiS_Pr, unsigned short ModeNo, unsigned short
             }
          }
       } else {						/* ---- PAL ---- */
-         /* We don't play around with FSCI in PAL mode */
-         if(resindex == 0x04) {
-            SiS_SetCH70xxANDOR(SiS_Pr,0x20,0x00,0xEF);	/* loop filter off */
-            SiS_SetCH70xxANDOR(SiS_Pr,0x21,0x01,0xFE);	/* ACIV on */
-         } else {
-            SiS_SetCH70xxANDOR(SiS_Pr,0x20,0x00,0xEF);	/* loop filter off */
-            SiS_SetCH70xxANDOR(SiS_Pr,0x21,0x01,0xFE);	/* ACIV on */
-         }
+	/* We don't play around with FSCI in PAL mode */
+	SiS_SetCH70xxANDOR(SiS_Pr, 0x20, 0x00, 0xEF);	/* loop filter off */
+	SiS_SetCH70xxANDOR(SiS_Pr, 0x21, 0x01, 0xFE);	/* ACIV on */
       }
 
 #endif  /* 300 */
@@ -9657,8 +9650,6 @@ SetDelayComp(struct SiS_Private *SiS_Pr, unsigned short ModeNo)
 	      delay = 0x0a;
 	   } else if(IS_SIS740) {
 	      delay = 0x00;
-	   } else if(SiS_Pr->ChipType < SIS_330) {
-	      delay = 0x0c;
 	   } else {
 	      delay = 0x0c;
 	   }
diff --git a/drivers/video/fbdev/skeletonfb.c b/drivers/video/fbdev/skeletonfb.c
index e219a0a22077..7f4e908330bf 100644
--- a/drivers/video/fbdev/skeletonfb.c
+++ b/drivers/video/fbdev/skeletonfb.c
@@ -84,7 +84,7 @@ struct xxx_par;
  * if we don't use modedb. If we do use modedb see xxxfb_init how to use it
  * to get a fb_var_screeninfo. Otherwise define a default var as well. 
  */
-static struct fb_fix_screeninfo xxxfb_fix = {
+static const struct fb_fix_screeninfo xxxfb_fix = {
 	.id =		"FB's name", 
 	.type =		FB_TYPE_PACKED_PIXELS,
 	.visual =	FB_VISUAL_PSEUDOCOLOR,
@@ -866,7 +866,7 @@ static int xxxfb_resume(struct pci_dev *dev)
 #define xxxfb_resume NULL
 #endif /* CONFIG_PM */
 
-static struct pci_device_id xxxfb_id_table[] = {
+static const struct pci_device_id xxxfb_id_table[] = {
 	{ PCI_VENDOR_ID_XXX, PCI_DEVICE_ID_XXX,
 	  PCI_ANY_ID, PCI_ANY_ID, PCI_BASE_CLASS_DISPLAY << 16,
 	  PCI_CLASS_MASK, 0 },
diff --git a/drivers/video/fbdev/sm501fb.c b/drivers/video/fbdev/sm501fb.c
index 67e314fdd947..076dd2711630 100644
--- a/drivers/video/fbdev/sm501fb.c
+++ b/drivers/video/fbdev/sm501fb.c
@@ -46,7 +46,7 @@
 static char *fb_mode = "640x480-16@60";
 static unsigned long default_bpp = 16;
 
-static struct fb_videomode sm501_default_mode = {
+static const struct fb_videomode sm501_default_mode = {
 	.refresh	= 60,
 	.xres		= 640,
 	.yres		= 480,
diff --git a/drivers/video/fbdev/sm712fb.c b/drivers/video/fbdev/sm712fb.c
index 73cb4ffff3c5..502d0de2feec 100644
--- a/drivers/video/fbdev/sm712fb.c
+++ b/drivers/video/fbdev/sm712fb.c
@@ -33,8 +33,8 @@
 #include "sm712.h"
 
 /*
-* Private structure
-*/
+ * Private structure
+ */
 struct smtcfb_info {
 	struct pci_dev *pdev;
 	struct fb_info *fb;
@@ -785,7 +785,7 @@ static void __init sm7xx_vga_setup(char *options)
 	smtc_scr_info.lfb_height = 0;
 	smtc_scr_info.lfb_depth = 0;
 
-	pr_debug("sm7xx_vga_setup = %s\n", options);
+	pr_debug("%s = %s\n", __func__, options);
 
 	for (i = 0; i < ARRAY_SIZE(vesa_mode_table); i++) {
 		if (strstr(options, vesa_mode_table[i].index)) {
@@ -798,8 +798,8 @@ static void __init sm7xx_vga_setup(char *options)
 	}
 }
 
-static void sm712_setpalette(int regno, unsigned red, unsigned green,
-			     unsigned blue, struct fb_info *info)
+static void sm712_setpalette(int regno, unsigned int red, unsigned int green,
+			     unsigned int blue, struct fb_info *info)
 {
 	/* set bit 5:4 = 01 (write LCD RAM only) */
 	smtc_seqw(0x66, (smtc_seqr(0x66) & 0xC3) | 0x10);
@@ -896,8 +896,9 @@ static int smtc_blank(int blank_mode, struct fb_info *info)
 	return 0;
 }
 
-static int smtc_setcolreg(unsigned regno, unsigned red, unsigned green,
-			  unsigned blue, unsigned trans, struct fb_info *info)
+static int smtc_setcolreg(unsigned int regno, unsigned int red,
+			  unsigned int green, unsigned int blue,
+			  unsigned int trans, struct fb_info *info)
 {
 	struct smtcfb_info *sfb;
 	u32 val;
@@ -1477,7 +1478,7 @@ static int smtcfb_pci_probe(struct pci_dev *pdev,
 	}
 
 	/* can support 32 bpp */
-	if (15 == sfb->fb->var.bits_per_pixel)
+	if (sfb->fb->var.bits_per_pixel == 15)
 		sfb->fb->var.bits_per_pixel = 16;
 
 	sfb->fb->var.xres_virtual = sfb->fb->var.xres;
diff --git a/drivers/video/fbdev/smscufx.c b/drivers/video/fbdev/smscufx.c
index 449fceaf79d5..2275e80b5776 100644
--- a/drivers/video/fbdev/smscufx.c
+++ b/drivers/video/fbdev/smscufx.c
@@ -122,7 +122,7 @@ static const u32 smscufx_info_flags = FBINFO_DEFAULT | FBINFO_READS_FAST |
 	FBINFO_VIRTFB |	FBINFO_HWACCEL_IMAGEBLIT | FBINFO_HWACCEL_FILLRECT |
 	FBINFO_HWACCEL_COPYAREA | FBINFO_MISC_ALWAYS_SETPAR;
 
-static struct usb_device_id id_table[] = {
+static const struct usb_device_id id_table[] = {
 	{USB_DEVICE(0x0424, 0x9d00),},
 	{USB_DEVICE(0x0424, 0x9d01),},
 	{},
diff --git a/drivers/video/fbdev/sunxvr1000.c b/drivers/video/fbdev/sunxvr1000.c
index fb37f6e05391..8fe37c0ef2f5 100644
--- a/drivers/video/fbdev/sunxvr1000.c
+++ b/drivers/video/fbdev/sunxvr1000.c
@@ -33,8 +33,8 @@ static int gfb_get_props(struct gfb_info *gp)
 	gp->depth = of_getintprop_default(gp->of_node, "depth", 32);
 
 	if (!gp->width || !gp->height) {
-		printk(KERN_ERR "gfb: Critical properties missing for %s\n",
-		       gp->of_node->full_name);
+		printk(KERN_ERR "gfb: Critical properties missing for %pOF\n",
+		       gp->of_node);
 		return -EINVAL;
 	}
 
@@ -151,12 +151,12 @@ static int gfb_probe(struct platform_device *op)
 	if (err)
 		goto err_unmap_fb;
 
-	printk("gfb: Found device at %s\n", dp->full_name);
+	printk("gfb: Found device at %pOF\n", dp);
 
 	err = register_framebuffer(info);
 	if (err < 0) {
-		printk(KERN_ERR "gfb: Could not register framebuffer %s\n",
-		       dp->full_name);
+		printk(KERN_ERR "gfb: Could not register framebuffer %pOF\n",
+		       dp);
 		goto err_unmap_fb;
 	}
 
diff --git a/drivers/video/fbdev/sunxvr2500.c b/drivers/video/fbdev/sunxvr2500.c
index 1a053292f2eb..544465ba1dc0 100644
--- a/drivers/video/fbdev/sunxvr2500.c
+++ b/drivers/video/fbdev/sunxvr2500.c
@@ -220,7 +220,7 @@ err_out:
 	return err;
 }
 
-static struct pci_device_id s3d_pci_table[] = {
+static const struct pci_device_id s3d_pci_table[] = {
 	{	PCI_DEVICE(PCI_VENDOR_ID_3DLABS, 0x002c),	},
 	{	PCI_DEVICE(PCI_VENDOR_ID_3DLABS, 0x002d),	},
 	{	PCI_DEVICE(PCI_VENDOR_ID_3DLABS, 0x002e),	},
diff --git a/drivers/video/fbdev/sunxvr500.c b/drivers/video/fbdev/sunxvr500.c
index dc0d886e4e7e..bc595937df08 100644
--- a/drivers/video/fbdev/sunxvr500.c
+++ b/drivers/video/fbdev/sunxvr500.c
@@ -393,7 +393,7 @@ err_out:
 	return err;
 }
 
-static struct pci_device_id e3d_pci_table[] = {
+static const struct pci_device_id e3d_pci_table[] = {
 	{	PCI_DEVICE(PCI_VENDOR_ID_3DLABS, 0x7a0),	},
 	{	PCI_DEVICE(0x1091, 0x7a0),			},
 	{	PCI_DEVICE(PCI_VENDOR_ID_3DLABS, 0x7a2),	},
diff --git a/drivers/video/fbdev/tcx.c b/drivers/video/fbdev/tcx.c
index 54ad08854c94..c98d8a569ccd 100644
--- a/drivers/video/fbdev/tcx.c
+++ b/drivers/video/fbdev/tcx.c
@@ -467,8 +467,8 @@ static int tcx_probe(struct platform_device *op)
 
 	dev_set_drvdata(&op->dev, info);
 
-	printk(KERN_INFO "%s: TCX at %lx:%lx, %s\n",
-	       dp->full_name,
+	printk(KERN_INFO "%pOF: TCX at %lx:%lx, %s\n",
+	       dp,
 	       par->which_io,
 	       info->fix.smem_start,
 	       par->lowdepth ? "8-bit only" : "24-bit depth");
diff --git a/drivers/video/fbdev/tdfxfb.c b/drivers/video/fbdev/tdfxfb.c
index d5fa313806fe..dec1fed9880e 100644
--- a/drivers/video/fbdev/tdfxfb.c
+++ b/drivers/video/fbdev/tdfxfb.c
@@ -120,7 +120,7 @@ static const struct fb_var_screeninfo tdfx_var = {
 static int tdfxfb_probe(struct pci_dev *pdev, const struct pci_device_id *id);
 static void tdfxfb_remove(struct pci_dev *pdev);
 
-static struct pci_device_id tdfxfb_id_table[] = {
+static const struct pci_device_id tdfxfb_id_table[] = {
 	{ PCI_VENDOR_ID_3DFX, PCI_DEVICE_ID_3DFX_BANSHEE,
 	  PCI_ANY_ID, PCI_ANY_ID, PCI_BASE_CLASS_DISPLAY << 16,
 	  0xff0000, 0 },
diff --git a/drivers/video/fbdev/tridentfb.c b/drivers/video/fbdev/tridentfb.c
index 8a5bbc13082e..284706184b1b 100644
--- a/drivers/video/fbdev/tridentfb.c
+++ b/drivers/video/fbdev/tridentfb.c
@@ -1737,7 +1737,7 @@ static void trident_pci_remove(struct pci_dev *dev)
 }
 
 /* List of boards that we are trying to support */
-static struct pci_device_id trident_devices[] = {
+static const struct pci_device_id trident_devices[] = {
 	{PCI_VENDOR_ID_TRIDENT,	BLADE3D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
 	{PCI_VENDOR_ID_TRIDENT,	CYBERBLADEi7, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
 	{PCI_VENDOR_ID_TRIDENT,	CYBERBLADEi7D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
diff --git a/drivers/video/fbdev/udlfb.c b/drivers/video/fbdev/udlfb.c
index 05ef657235df..ef08a104fb42 100644
--- a/drivers/video/fbdev/udlfb.c
+++ b/drivers/video/fbdev/udlfb.c
@@ -54,7 +54,7 @@ static const u32 udlfb_info_flags = FBINFO_DEFAULT | FBINFO_READS_FAST |
  * which is compatible with all known USB 2.0 era graphics chips and firmware,
  * but allows DisplayLink to increment those for any future incompatible chips
  */
-static struct usb_device_id id_table[] = {
+static const struct usb_device_id id_table[] = {
 	{.idVendor = 0x17e9,
 	 .bInterfaceClass = 0xff,
 	 .bInterfaceSubClass = 0x00,
@@ -1465,7 +1465,7 @@ static ssize_t metrics_reset_store(struct device *fbdev,
 	return count;
 }
 
-static struct bin_attribute edid_attr = {
+static const struct bin_attribute edid_attr = {
 	.attr.name = "edid",
 	.attr.mode = 0666,
 	.size = EDID_LENGTH,
@@ -1655,7 +1655,6 @@ static int dlfb_usb_probe(struct usb_interface *interface,
 error:
 	if (dev) {
 
-		kref_put(&dev->kref, dlfb_free); /* ref for framebuffer */
 		kref_put(&dev->kref, dlfb_free); /* last ref from kref_init */
 
 		/* dev has been deallocated. Do not dereference */
diff --git a/drivers/video/fbdev/uvesafb.c b/drivers/video/fbdev/uvesafb.c
index dc0e8d90d9cc..73676eb0244a 100644
--- a/drivers/video/fbdev/uvesafb.c
+++ b/drivers/video/fbdev/uvesafb.c
@@ -1666,7 +1666,7 @@ static struct attribute *uvesafb_dev_attrs[] = {
 	NULL,
 };
 
-static struct attribute_group uvesafb_dev_attgrp = {
+static const struct attribute_group uvesafb_dev_attgrp = {
 	.name = NULL,
 	.attrs = uvesafb_dev_attrs,
 };
@@ -1860,19 +1860,18 @@ static int uvesafb_setup(char *options)
 }
 #endif /* !MODULE */
 
-static ssize_t show_v86d(struct device_driver *dev, char *buf)
+static ssize_t v86d_show(struct device_driver *dev, char *buf)
 {
 	return snprintf(buf, PAGE_SIZE, "%s\n", v86d_path);
 }
 
-static ssize_t store_v86d(struct device_driver *dev, const char *buf,
+static ssize_t v86d_store(struct device_driver *dev, const char *buf,
 		size_t count)
 {
 	strncpy(v86d_path, buf, PATH_MAX);
 	return count;
 }
-
-static DRIVER_ATTR(v86d, S_IRUGO | S_IWUSR, show_v86d, store_v86d);
+static DRIVER_ATTR_RW(v86d);
 
 static int uvesafb_init(void)
 {
diff --git a/drivers/video/fbdev/vermilion/vermilion.c b/drivers/video/fbdev/vermilion/vermilion.c
index ce4c4729a5e8..6f8d444eb0e3 100644
--- a/drivers/video/fbdev/vermilion/vermilion.c
+++ b/drivers/video/fbdev/vermilion/vermilion.c
@@ -55,7 +55,7 @@ static struct list_head global_has_mode;
 static struct fb_ops vmlfb_ops;
 static struct vml_sys *subsys = NULL;
 static char *vml_default_mode = "1024x768@60";
-static struct fb_videomode defaultmode = {
+static const struct fb_videomode defaultmode = {
 	NULL, 60, 1024, 768, 12896, 144, 24, 29, 3, 136, 6,
 	0, FB_VMODE_NONINTERLACED
 };
@@ -1044,7 +1044,7 @@ static struct fb_ops vmlfb_ops = {
 	.fb_setcolreg = vmlfb_setcolreg
 };
 
-static struct pci_device_id vml_ids[] = {
+static const struct pci_device_id vml_ids[] = {
 	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, VML_DEVICE_VDC)},
 	{0}
 };
diff --git a/drivers/video/fbdev/via/via-core.c b/drivers/video/fbdev/via/via-core.c
index 1d28e16888e9..77774d8abf94 100644
--- a/drivers/video/fbdev/via/via-core.c
+++ b/drivers/video/fbdev/via/via-core.c
@@ -724,7 +724,7 @@ static void via_pci_remove(struct pci_dev *pdev)
 }
 
 
-static struct pci_device_id via_pci_table[] = {
+static const struct pci_device_id via_pci_table[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_VIA, UNICHROME_CLE266_DID),
 	  .driver_data = UNICHROME_CLE266 },
 	{ PCI_DEVICE(PCI_VENDOR_ID_VIA, UNICHROME_K400_DID),
diff --git a/drivers/video/fbdev/vt8623fb.c b/drivers/video/fbdev/vt8623fb.c
index dd0f18e42d3e..5cac871db3ee 100644
--- a/drivers/video/fbdev/vt8623fb.c
+++ b/drivers/video/fbdev/vt8623fb.c
@@ -81,7 +81,7 @@ static struct vga_regset vt8623_line_compare_regs[]  = {{0x18, 0, 7}, {0x07, 4,
 static struct vga_regset vt8623_fetch_count_regs[]   = {{0x1C, 0, 7}, {0x1D, 0, 1}, VGA_REGSET_END};
 static struct vga_regset vt8623_start_address_regs[] = {{0x0d, 0, 7}, {0x0c, 0, 7}, {0x34, 0, 7}, {0x48, 0, 1}, VGA_REGSET_END};
 
-static struct svga_timing_regs vt8623_timing_regs     = {
+static const struct svga_timing_regs vt8623_timing_regs     = {
 	vt8623_h_total_regs, vt8623_h_display_regs, vt8623_h_blank_start_regs,
 	vt8623_h_blank_end_regs, vt8623_h_sync_start_regs, vt8623_h_sync_end_regs,
 	vt8623_v_total_regs, vt8623_v_display_regs, vt8623_v_blank_start_regs,
@@ -888,7 +888,7 @@ fail:
 
 /* List of boards that we are trying to support */
 
-static struct pci_device_id vt8623_devices[] = {
+static const struct pci_device_id vt8623_devices[] = {
 	{PCI_DEVICE(PCI_VENDOR_ID_VIA, 0x3122)},
 	{0, 0, 0, 0, 0, 0, 0}
 };
diff --git a/drivers/video/fbdev/xilinxfb.c b/drivers/video/fbdev/xilinxfb.c
index 17dc119c7a98..8628829b470d 100644
--- a/drivers/video/fbdev/xilinxfb.c
+++ b/drivers/video/fbdev/xilinxfb.c
@@ -41,7 +41,6 @@
 
 #define DRIVER_NAME		"xilinxfb"
 
-
 /*
  * Xilinx calls it "TFT LCD Controller" though it can also be used for
  * the VGA port on the Xilinx ML40x board. This is a hardware display
@@ -92,15 +91,16 @@ struct xilinxfb_platform_data {
 	u32 xvirt, yvirt;       /* resolution of memory buffer */
 
 	/* Physical address of framebuffer memory; If non-zero, driver
-	* will use provided memory address instead of allocating one from
-	* the consistent pool. */
+	 * will use provided memory address instead of allocating one from
+	 * the consistent pool.
+	 */
 	u32 fb_phys;
 };
 
 /*
  * Default xilinxfb configuration
  */
-static struct xilinxfb_platform_data xilinx_fb_default_pdata = {
+static const struct xilinxfb_platform_data xilinx_fb_default_pdata = {
 	.xres = 640,
 	.yres = 480,
 	.xvirt = 1024,
@@ -110,14 +110,14 @@ static struct xilinxfb_platform_data xilinx_fb_default_pdata = {
 /*
  * Here are the default fb_fix_screeninfo and fb_var_screeninfo structures
  */
-static struct fb_fix_screeninfo xilinx_fb_fix = {
+static const struct fb_fix_screeninfo xilinx_fb_fix = {
 	.id =		"Xilinx",
 	.type =		FB_TYPE_PACKED_PIXELS,
 	.visual =	FB_VISUAL_TRUECOLOR,
 	.accel =	FB_ACCEL_NONE
 };
 
-static struct fb_var_screeninfo xilinx_fb_var = {
+static const struct fb_var_screeninfo xilinx_fb_var = {
 	.bits_per_pixel =	BITS_PER_PIXEL,
 
 	.red =		{ RED_SHIFT, 8, 0 },
@@ -128,18 +128,18 @@ static struct fb_var_screeninfo xilinx_fb_var = {
 	.activate =	FB_ACTIVATE_NOW
 };
 
-
 #define BUS_ACCESS_FLAG		0x1 /* 1 = BUS, 0 = DCR */
 #define LITTLE_ENDIAN_ACCESS	0x2 /* LITTLE ENDIAN IO functions */
 
 struct xilinxfb_drvdata {
-
 	struct fb_info	info;		/* FB driver info record */
 
 	phys_addr_t	regs_phys;	/* phys. address of the control
-						registers */
+					 * registers
+					 */
 	void __iomem	*regs;		/* virt. address of the control
-						registers */
+					 * registers
+					 */
 #ifdef CONFIG_PPC_DCR
 	dcr_host_t      dcr_host;
 	unsigned int    dcr_len;
@@ -148,7 +148,7 @@ struct xilinxfb_drvdata {
 	dma_addr_t	fb_phys;	/* phys. address of the frame buffer */
 	int		fb_alloced;	/* Flag, was the fb memory alloced? */
 
-	u8 		flags;		/* features of the driver */
+	u8		flags;		/* features of the driver */
 
 	u32		reg_ctrl_default;
 
@@ -165,7 +165,7 @@ struct xilinxfb_drvdata {
  * which bus its connected and call the appropriate write API.
  */
 static void xilinx_fb_out32(struct xilinxfb_drvdata *drvdata, u32 offset,
-				u32 val)
+			    u32 val)
 {
 	if (drvdata->flags & BUS_ACCESS_FLAG) {
 		if (drvdata->flags & LITTLE_ENDIAN_ACCESS)
@@ -195,8 +195,8 @@ static u32 xilinx_fb_in32(struct xilinxfb_drvdata *drvdata, u32 offset)
 }
 
 static int
-xilinx_fb_setcolreg(unsigned regno, unsigned red, unsigned green, unsigned blue,
-	unsigned transp, struct fb_info *fbi)
+xilinx_fb_setcolreg(unsigned int regno, unsigned int red, unsigned int green,
+		    unsigned int blue, unsigned int transp, struct fb_info *fbi)
 {
 	u32 *palette = fbi->pseudo_palette;
 
@@ -205,9 +205,11 @@ xilinx_fb_setcolreg(unsigned regno, unsigned red, unsigned green, unsigned blue,
 
 	if (fbi->var.grayscale) {
 		/* Convert color to grayscale.
-		 * grayscale = 0.30*R + 0.59*G + 0.11*B */
-		red = green = blue =
-			(red * 77 + green * 151 + blue * 28 + 127) >> 8;
+		 * grayscale = 0.30*R + 0.59*G + 0.11*B
+		 */
+		blue = (red * 77 + green * 151 + blue * 28 + 127) >> 8;
+		green = blue;
+		red = green;
 	}
 
 	/* fbi->fix.visual is always FB_VISUAL_TRUECOLOR */
@@ -241,13 +243,11 @@ xilinx_fb_blank(int blank_mode, struct fb_info *fbi)
 		xilinx_fb_out32(drvdata, REG_CTRL, 0);
 	default:
 		break;
-
 	}
 	return 0; /* success */
 }
 
-static struct fb_ops xilinxfb_ops =
-{
+static struct fb_ops xilinxfb_ops = {
 	.owner			= THIS_MODULE,
 	.fb_setcolreg		= xilinx_fb_setcolreg,
 	.fb_blank		= xilinx_fb_blank,
@@ -286,7 +286,8 @@ static int xilinxfb_assign(struct platform_device *pdev,
 	} else {
 		drvdata->fb_alloced = 1;
 		drvdata->fb_virt = dma_alloc_coherent(dev, PAGE_ALIGN(fbsize),
-					&drvdata->fb_phys, GFP_KERNEL);
+						      &drvdata->fb_phys,
+						      GFP_KERNEL);
 	}
 
 	if (!drvdata->fb_virt) {
@@ -300,7 +301,7 @@ static int xilinxfb_assign(struct platform_device *pdev,
 	/* Tell the hardware where the frame buffer is */
 	xilinx_fb_out32(drvdata, REG_FB_ADDR, drvdata->fb_phys);
 	rc = xilinx_fb_in32(drvdata, REG_FB_ADDR);
-	/* Endianess detection */
+	/* Endianness detection */
 	if (rc != drvdata->fb_phys) {
 		drvdata->flags |= LITTLE_ENDIAN_ACCESS;
 		xilinx_fb_out32(drvdata, REG_FB_ADDR, drvdata->fb_phys);
@@ -310,8 +311,7 @@ static int xilinxfb_assign(struct platform_device *pdev,
 	drvdata->reg_ctrl_default = REG_CTRL_ENABLE;
 	if (pdata->rotate_screen)
 		drvdata->reg_ctrl_default |= REG_CTRL_ROTATE;
-	xilinx_fb_out32(drvdata, REG_CTRL,
-					drvdata->reg_ctrl_default);
+	xilinx_fb_out32(drvdata, REG_CTRL, drvdata->reg_ctrl_default);
 
 	/* Fill struct fb_info */
 	drvdata->info.device = dev;
@@ -364,7 +364,7 @@ err_regfb:
 err_cmap:
 	if (drvdata->fb_alloced)
 		dma_free_coherent(dev, PAGE_ALIGN(fbsize), drvdata->fb_virt,
-			drvdata->fb_phys);
+				  drvdata->fb_phys);
 	else
 		iounmap(drvdata->fb_virt);
 
@@ -435,12 +435,12 @@ static int xilinxfb_of_probe(struct platform_device *pdev)
 	 * Fill the resource structure if its direct BUS interface
 	 * otherwise fill the dcr_host structure.
 	 */
-	if (tft_access) {
+	if (tft_access)
 		drvdata->flags |= BUS_ACCESS_FLAG;
-	}
 #ifdef CONFIG_PPC_DCR
 	else {
 		int start;
+
 		start = dcr_resource_start(pdev->dev.of_node, 0);
 		drvdata->dcr_len = dcr_resource_len(pdev->dev.of_node, 0);
 		drvdata->dcr_host = dcr_map(pdev->dev.of_node, start, drvdata->dcr_len);
@@ -452,19 +452,19 @@ static int xilinxfb_of_probe(struct platform_device *pdev)
 #endif
 
 	prop = of_get_property(pdev->dev.of_node, "phys-size", &size);
-	if ((prop) && (size >= sizeof(u32)*2)) {
+	if ((prop) && (size >= sizeof(u32) * 2)) {
 		pdata.screen_width_mm = prop[0];
 		pdata.screen_height_mm = prop[1];
 	}
 
 	prop = of_get_property(pdev->dev.of_node, "resolution", &size);
-	if ((prop) && (size >= sizeof(u32)*2)) {
+	if ((prop) && (size >= sizeof(u32) * 2)) {
 		pdata.xres = prop[0];
 		pdata.yres = prop[1];
 	}
 
 	prop = of_get_property(pdev->dev.of_node, "virtual-resolution", &size);
-	if ((prop) && (size >= sizeof(u32)*2)) {
+	if ((prop) && (size >= sizeof(u32) * 2)) {
 		pdata.xvirt = prop[0];
 		pdata.yvirt = prop[1];
 	}
@@ -482,7 +482,7 @@ static int xilinxfb_of_remove(struct platform_device *op)
 }
 
 /* Match table for of_platform binding */
-static struct of_device_id xilinxfb_of_match[] = {
+static const struct of_device_id xilinxfb_of_match[] = {
 	{ .compatible = "xlnx,xps-tft-1.00.a", },
 	{ .compatible = "xlnx,xps-tft-2.00.a", },
 	{ .compatible = "xlnx,xps-tft-2.01.a", },
diff --git a/drivers/video/of_display_timing.c b/drivers/video/of_display_timing.c
index 32b0a7543433..8ce0a99bf17c 100644
--- a/drivers/video/of_display_timing.c
+++ b/drivers/video/of_display_timing.c
@@ -31,8 +31,7 @@ static int parse_timing_property(const struct device_node *np, const char *name,
 
 	prop = of_find_property(np, name, &length);
 	if (!prop) {
-		pr_err("%s: could not find property %s\n",
-			of_node_full_name(np), name);
+		pr_err("%pOF: could not find property %s\n", np, name);
 		return -EINVAL;
 	}
 
@@ -44,8 +43,7 @@ static int parse_timing_property(const struct device_node *np, const char *name,
 	} else if (cells == 3) {
 		ret = of_property_read_u32_array(np, name, &result->min, cells);
 	} else {
-		pr_err("%s: illegal timing specification in %s\n",
-			of_node_full_name(np), name);
+		pr_err("%pOF: illegal timing specification in %s\n", np, name);
 		return -EINVAL;
 	}
 
@@ -105,8 +103,7 @@ static int of_parse_display_timing(const struct device_node *np,
 		dt->flags |= DISPLAY_FLAGS_DOUBLECLK;
 
 	if (ret) {
-		pr_err("%s: error reading timing properties\n",
-			of_node_full_name(np));
+		pr_err("%pOF: error reading timing properties\n", np);
 		return -EINVAL;
 	}
 
@@ -129,8 +126,7 @@ int of_get_display_timing(const struct device_node *np, const char *name,
 
 	timing_np = of_get_child_by_name(np, name);
 	if (!timing_np) {
-		pr_err("%s: could not find node '%s'\n",
-			of_node_full_name(np), name);
+		pr_err("%pOF: could not find node '%s'\n", np, name);
 		return -ENOENT;
 	}
 
@@ -154,15 +150,13 @@ struct display_timings *of_get_display_timings(const struct device_node *np)
 
 	timings_np = of_get_child_by_name(np, "display-timings");
 	if (!timings_np) {
-		pr_err("%s: could not find display-timings node\n",
-			of_node_full_name(np));
+		pr_err("%pOF: could not find display-timings node\n", np);
 		return NULL;
 	}
 
 	disp = kzalloc(sizeof(*disp), GFP_KERNEL);
 	if (!disp) {
-		pr_err("%s: could not allocate struct disp'\n",
-			of_node_full_name(np));
+		pr_err("%pOF: could not allocate struct disp'\n", np);
 		goto dispfail;
 	}
 
@@ -172,28 +166,25 @@ struct display_timings *of_get_display_timings(const struct device_node *np)
 		entry = of_get_next_child(timings_np, NULL);
 	/* if there is no child, it is useless to go on */
 	if (!entry) {
-		pr_err("%s: no timing specifications given\n",
-			of_node_full_name(np));
+		pr_err("%pOF: no timing specifications given\n", np);
 		goto entryfail;
 	}
 
-	pr_debug("%s: using %s as default timing\n",
-		of_node_full_name(np), entry->name);
+	pr_debug("%pOF: using %s as default timing\n", np, entry->name);
 
 	native_mode = entry;
 
 	disp->num_timings = of_get_child_count(timings_np);
 	if (disp->num_timings == 0) {
 		/* should never happen, as entry was already found above */
-		pr_err("%s: no timings specified\n", of_node_full_name(np));
+		pr_err("%pOF: no timings specified\n", np);
 		goto entryfail;
 	}
 
 	disp->timings = kzalloc(sizeof(struct display_timing *) *
 				disp->num_timings, GFP_KERNEL);
 	if (!disp->timings) {
-		pr_err("%s: could not allocate timings array\n",
-			of_node_full_name(np));
+		pr_err("%pOF: could not allocate timings array\n", np);
 		goto entryfail;
 	}
 
@@ -206,8 +197,8 @@ struct display_timings *of_get_display_timings(const struct device_node *np)
 
 		dt = kzalloc(sizeof(*dt), GFP_KERNEL);
 		if (!dt) {
-			pr_err("%s: could not allocate display_timing struct\n",
-					of_node_full_name(np));
+			pr_err("%pOF: could not allocate display_timing struct\n",
+				np);
 			goto timingfail;
 		}
 
@@ -217,8 +208,8 @@ struct display_timings *of_get_display_timings(const struct device_node *np)
 			 * to not encourage wrong devicetrees, fail in case of
 			 * an error
 			 */
-			pr_err("%s: error in timing %d\n",
-				of_node_full_name(np), disp->num_timings + 1);
+			pr_err("%pOF: error in timing %d\n",
+				np, disp->num_timings + 1);
 			kfree(dt);
 			goto timingfail;
 		}
@@ -236,8 +227,8 @@ struct display_timings *of_get_display_timings(const struct device_node *np)
 	 */
 	of_node_put(native_mode);
 
-	pr_debug("%s: got %d timings. Using timing #%d as default\n",
-		of_node_full_name(np), disp->num_timings,
+	pr_debug("%pOF: got %d timings. Using timing #%d as default\n",
+		np, disp->num_timings,
 		disp->native_mode + 1);
 
 	return disp;
diff --git a/drivers/video/of_videomode.c b/drivers/video/of_videomode.c
index b5102aa6090d..9b5f9de88fec 100644
--- a/drivers/video/of_videomode.c
+++ b/drivers/video/of_videomode.c
@@ -36,7 +36,7 @@ int of_get_videomode(struct device_node *np, struct videomode *vm,
 
 	disp = of_get_display_timings(np);
 	if (!disp) {
-		pr_err("%s: no timings specified\n", of_node_full_name(np));
+		pr_err("%pOF: no timings specified\n", np);
 		return -EINVAL;
 	}
 
diff --git a/drivers/virt/fsl_hypervisor.c b/drivers/virt/fsl_hypervisor.c
index d3eca879a0a8..d993df5586c0 100644
--- a/drivers/virt/fsl_hypervisor.c
+++ b/drivers/virt/fsl_hypervisor.c
@@ -841,8 +841,8 @@ static int __init fsl_hypervisor_init(void)
 		handle = of_get_property(np, "interrupts", NULL);
 		irq = irq_of_parse_and_map(np, 0);
 		if (!handle || (irq == NO_IRQ)) {
-			pr_err("fsl-hv: no 'interrupts' property in %s node\n",
-				np->full_name);
+			pr_err("fsl-hv: no 'interrupts' property in %pOF node\n",
+				np);
 			continue;
 		}
 
@@ -869,8 +869,8 @@ static int __init fsl_hypervisor_init(void)
 			 */
 			dbisr->partition = ret = get_parent_handle(np);
 			if (ret < 0) {
-				pr_err("fsl-hv: node %s has missing or "
-				       "malformed parent\n", np->full_name);
+				pr_err("fsl-hv: node %pOF has missing or "
+				       "malformed parent\n", np);
 				kfree(dbisr);
 				continue;
 			}
@@ -881,8 +881,8 @@ static int __init fsl_hypervisor_init(void)
 			ret = request_irq(irq, fsl_hv_isr, 0, np->name, dbisr);
 
 		if (ret < 0) {
-			pr_err("fsl-hv: could not request irq %u for node %s\n",
-			       irq, np->full_name);
+			pr_err("fsl-hv: could not request irq %u for node %pOF\n",
+			       irq, np);
 			kfree(dbisr);
 			continue;
 		}
diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig
index 623f72334fa5..cff773f15b7e 100644
--- a/drivers/virtio/Kconfig
+++ b/drivers/virtio/Kconfig
@@ -2,8 +2,8 @@ config VIRTIO
 	tristate
 	---help---
 	  This option is selected by any driver which implements the virtio
-	  bus, such as CONFIG_VIRTIO_PCI, CONFIG_VIRTIO_MMIO, CONFIG_LGUEST,
-	  CONFIG_RPMSG or CONFIG_S390_GUEST.
+	  bus, such as CONFIG_VIRTIO_PCI, CONFIG_VIRTIO_MMIO, CONFIG_RPMSG
+	  or CONFIG_S390_GUEST.
 
 menu "Virtio drivers"
 
diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c
index 007a4f366086..1c4797e53f68 100644
--- a/drivers/virtio/virtio_pci_common.c
+++ b/drivers/virtio/virtio_pci_common.c
@@ -107,6 +107,7 @@ static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
 {
 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 	const char *name = dev_name(&vp_dev->vdev.dev);
+	unsigned flags = PCI_IRQ_MSIX;
 	unsigned i, v;
 	int err = -ENOMEM;
 
@@ -126,10 +127,13 @@ static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
 					GFP_KERNEL))
 			goto error;
 
+	if (desc) {
+		flags |= PCI_IRQ_AFFINITY;
+		desc->pre_vectors++; /* virtio config vector */
+	}
+
 	err = pci_alloc_irq_vectors_affinity(vp_dev->pci_dev, nvectors,
-					     nvectors, PCI_IRQ_MSIX |
-					     (desc ? PCI_IRQ_AFFINITY : 0),
-					     desc);
+					     nvectors, flags, desc);
 	if (err < 0)
 		goto error;
 	vp_dev->msix_enabled = 1;
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 5e1b548828e6..eb30f3e09a47 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -296,7 +296,6 @@ static inline int virtqueue_add(struct virtqueue *_vq,
 	}
 #endif
 
-	BUG_ON(total_sg > vq->vring.num);
 	BUG_ON(total_sg == 0);
 
 	head = vq->free_head;
@@ -305,8 +304,10 @@ static inline int virtqueue_add(struct virtqueue *_vq,
 	 * buffers, then go indirect. FIXME: tune this threshold */
 	if (vq->indirect && total_sg > 1 && vq->vq.num_free)
 		desc = alloc_indirect(_vq, total_sg, gfp);
-	else
+	else {
 		desc = NULL;
+		WARN_ON_ONCE(total_sg > vq->vring.num && !vq->indirect);
+	}
 
 	if (desc) {
 		/* Use a single buffer which doesn't continue */
@@ -391,7 +392,7 @@ static inline int virtqueue_add(struct virtqueue *_vq,
 	vq->desc_state[head].data = data;
 	if (indirect)
 		vq->desc_state[head].indir_desc = desc;
-	if (ctx)
+	else
 		vq->desc_state[head].indir_desc = ctx;
 
 	/* Put entry in available array (but don't update avail->idx until they
diff --git a/drivers/w1/masters/ds1wm.c b/drivers/w1/masters/ds1wm.c
index fd2e9da27c4b..f661695fb589 100644
--- a/drivers/w1/masters/ds1wm.c
+++ b/drivers/w1/masters/ds1wm.c
@@ -95,7 +95,8 @@ static struct {
 
 struct ds1wm_data {
 	void     __iomem *map;
-	int      bus_shift; /* # of shifts to calc register offsets */
+	unsigned int      bus_shift; /* # of shifts to calc register offsets */
+	bool      is_hw_big_endian;
 	struct platform_device *pdev;
 	const struct mfd_cell   *cell;
 	int      irq;
@@ -115,12 +116,65 @@ struct ds1wm_data {
 static inline void ds1wm_write_register(struct ds1wm_data *ds1wm_data, u32 reg,
 					u8 val)
 {
-	__raw_writeb(val, ds1wm_data->map + (reg << ds1wm_data->bus_shift));
+	if (ds1wm_data->is_hw_big_endian) {
+		switch (ds1wm_data->bus_shift) {
+		case 0:
+			iowrite8(val, ds1wm_data->map + (reg << 0));
+			break;
+		case 1:
+			iowrite16be((u16)val, ds1wm_data->map + (reg << 1));
+			break;
+		case 2:
+			iowrite32be((u32)val, ds1wm_data->map + (reg << 2));
+			break;
+		}
+	} else {
+		switch (ds1wm_data->bus_shift) {
+		case 0:
+			iowrite8(val, ds1wm_data->map + (reg << 0));
+			break;
+		case 1:
+			iowrite16((u16)val, ds1wm_data->map + (reg << 1));
+			break;
+		case 2:
+			iowrite32((u32)val, ds1wm_data->map + (reg << 2));
+			break;
+		}
+	}
 }
 
 static inline u8 ds1wm_read_register(struct ds1wm_data *ds1wm_data, u32 reg)
 {
-	return __raw_readb(ds1wm_data->map + (reg << ds1wm_data->bus_shift));
+	u32 val = 0;
+
+	if (ds1wm_data->is_hw_big_endian) {
+		switch (ds1wm_data->bus_shift) {
+		case 0:
+			val = ioread8(ds1wm_data->map + (reg << 0));
+			break;
+		case 1:
+			val = ioread16be(ds1wm_data->map + (reg << 1));
+			break;
+		case 2:
+			val = ioread32be(ds1wm_data->map + (reg << 2));
+			break;
+		}
+	} else {
+		switch (ds1wm_data->bus_shift) {
+		case 0:
+			val = ioread8(ds1wm_data->map + (reg << 0));
+			break;
+		case 1:
+			val = ioread16(ds1wm_data->map + (reg << 1));
+			break;
+		case 2:
+			val = ioread32(ds1wm_data->map + (reg << 2));
+			break;
+		}
+	}
+	dev_dbg(&ds1wm_data->pdev->dev,
+		"ds1wm_read_register reg: %d, 32 bit val:%x\n", reg, val);
+	return (u8)val;
 }
 
 
@@ -455,6 +509,7 @@ static int ds1wm_probe(struct platform_device *pdev)
 	struct ds1wm_driver_data *plat;
 	struct resource *res;
 	int ret;
+	u8 inten;
 
 	if (!pdev)
 		return -ENODEV;
@@ -473,9 +528,6 @@ static int ds1wm_probe(struct platform_device *pdev)
 	if (!ds1wm_data->map)
 		return -ENOMEM;
 
-	/* calculate bus shift from mem resource */
-	ds1wm_data->bus_shift = resource_size(res) >> 3;
-
 	ds1wm_data->pdev = pdev;
 	ds1wm_data->cell = mfd_get_cell(pdev);
 	if (!ds1wm_data->cell)
@@ -484,6 +536,26 @@ static int ds1wm_probe(struct platform_device *pdev)
 	if (!plat)
 		return -ENODEV;
 
+	/* how many bits to shift register number to get register offset */
+	if (plat->bus_shift > 2) {
+		dev_err(&ds1wm_data->pdev->dev,
+			"illegal bus shift %d, not written",
+			ds1wm_data->bus_shift);
+		return -EINVAL;
+	}
+
+	ds1wm_data->bus_shift = plat->bus_shift;
+	/* make sure resource has space for 8 registers */
+	if ((8 << ds1wm_data->bus_shift) > resource_size(res)) {
+		dev_err(&ds1wm_data->pdev->dev,
+			"memory resource size %d to small, should be %d\n",
+			(int)resource_size(res),
+			8 << ds1wm_data->bus_shift);
+		return -EINVAL;
+	}
+
+	ds1wm_data->is_hw_big_endian = plat->is_hw_big_endian;
+
 	res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
 	if (!res)
 		return -ENXIO;
@@ -491,15 +563,30 @@ static int ds1wm_probe(struct platform_device *pdev)
 	ds1wm_data->int_en_reg_none = (plat->active_high ? DS1WM_INTEN_IAS : 0);
 	ds1wm_data->reset_recover_delay = plat->reset_recover_delay;
 
+	/* Mask interrupts, set IAS before claiming interrupt */
+	inten = ds1wm_read_register(ds1wm_data, DS1WM_INT_EN);
+	ds1wm_write_register(ds1wm_data,
+		DS1WM_INT_EN, ds1wm_data->int_en_reg_none);
+
 	if (res->flags & IORESOURCE_IRQ_HIGHEDGE)
 		irq_set_irq_type(ds1wm_data->irq, IRQ_TYPE_EDGE_RISING);
 	if (res->flags & IORESOURCE_IRQ_LOWEDGE)
 		irq_set_irq_type(ds1wm_data->irq, IRQ_TYPE_EDGE_FALLING);
+	if (res->flags & IORESOURCE_IRQ_HIGHLEVEL)
+		irq_set_irq_type(ds1wm_data->irq, IRQ_TYPE_LEVEL_HIGH);
+	if (res->flags & IORESOURCE_IRQ_LOWLEVEL)
+		irq_set_irq_type(ds1wm_data->irq, IRQ_TYPE_LEVEL_LOW);
 
 	ret = devm_request_irq(&pdev->dev, ds1wm_data->irq, ds1wm_isr,
 			IRQF_SHARED, "ds1wm", ds1wm_data);
-	if (ret)
+	if (ret) {
+		dev_err(&ds1wm_data->pdev->dev,
+			"devm_request_irq %d failed with errno %d\n",
+			ds1wm_data->irq,
+			ret);
+
 		return ret;
+	}
 
 	ds1wm_up(ds1wm_data);
 
@@ -509,6 +596,13 @@ static int ds1wm_probe(struct platform_device *pdev)
 	if (ret)
 		goto err;
 
+	dev_dbg(&ds1wm_data->pdev->dev,
+		"ds1wm: probe successful, IAS: %d, rec.delay: %d, clockrate: %d, bus-shift: %d, is Hw Big Endian: %d\n",
+		plat->active_high,
+		plat->reset_recover_delay,
+		plat->clock_rate,
+		ds1wm_data->bus_shift,
+		ds1wm_data->is_hw_big_endian);
 	return 0;
 
 err:
diff --git a/drivers/w1/masters/ds2482.c b/drivers/w1/masters/ds2482.c
index d49681cd29af..5b3e017d9276 100644
--- a/drivers/w1/masters/ds2482.c
+++ b/drivers/w1/masters/ds2482.c
@@ -70,6 +70,8 @@ MODULE_PARM_DESC(active_pullup, "Active pullup (apply to all buses): " \
 #define DS2482_REG_CFG_PPM		0x02	/* presence pulse masking */
 #define DS2482_REG_CFG_APU		0x01	/* active pull-up */
 
+/* extra configurations - e.g. 1WS */
+int extra_config;
 
 /**
  * Write and verify codes for the CHANNEL_SELECT command (DS2482-800 only).
@@ -403,7 +405,7 @@ static u8 ds2482_w1_reset_bus(void *data)
 		/* If the chip did reset since detect, re-config it */
 		if (err & DS2482_REG_STS_RST)
 			ds2482_send_cmd_data(pdev, DS2482_CMD_WRITE_CONFIG,
-					     ds2482_calculate_config(0x00));
+					ds2482_calculate_config(extra_config));
 	}
 
 	mutex_unlock(&pdev->access_lock);
@@ -429,8 +431,7 @@ static u8 ds2482_w1_set_pullup(void *data, int delay)
 		ds2482_wait_1wire_idle(pdev);
 		/* note: it seems like both SPU and APU have to be set! */
 		retval = ds2482_send_cmd_data(pdev, DS2482_CMD_WRITE_CONFIG,
-			ds2482_calculate_config(DS2482_REG_CFG_SPU |
-						DS2482_REG_CFG_APU));
+			ds2482_calculate_config(extra_config|DS2482_REG_CFG_SPU|DS2482_REG_CFG_APU));
 		ds2482_wait_1wire_idle(pdev);
 	}
 
@@ -483,7 +484,7 @@ static int ds2482_probe(struct i2c_client *client,
 
 	/* Set all config items to 0 (off) */
 	ds2482_send_cmd_data(data, DS2482_CMD_WRITE_CONFIG,
-		ds2482_calculate_config(0x00));
+		ds2482_calculate_config(extra_config));
 
 	mutex_init(&data->access_lock);
 
@@ -558,4 +559,7 @@ module_i2c_driver(ds2482_driver);
 
 MODULE_AUTHOR("Ben Gardner <bgardner@wabtec.com>");
 MODULE_DESCRIPTION("DS2482 driver");
+module_param(extra_config, int, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(extra_config, "Extra Configuration settings 1=APU,2=PPM,3=SPU,8=1WS");
+
 MODULE_LICENSE("GPL");
diff --git a/drivers/w1/masters/ds2490.c b/drivers/w1/masters/ds2490.c
index 46ccb2fc4f60..c423bdb982bb 100644
--- a/drivers/w1/masters/ds2490.c
+++ b/drivers/w1/masters/ds2490.c
@@ -1088,7 +1088,7 @@ static void ds_disconnect(struct usb_interface *intf)
 	kfree(dev);
 }
 
-static struct usb_device_id ds_id_table [] = {
+static const struct usb_device_id ds_id_table[] = {
 	{ USB_DEVICE(0x04fa, 0x2490) },
 	{ },
 };
diff --git a/drivers/w1/slaves/Kconfig b/drivers/w1/slaves/Kconfig
index fb68465908f2..3c945f9f5f0f 100644
--- a/drivers/w1/slaves/Kconfig
+++ b/drivers/w1/slaves/Kconfig
@@ -65,6 +65,14 @@ config W1_SLAVE_DS2423
 	  Say Y here if you want to use a 1-wire
 	  counter family device (DS2423).
 
+config W1_SLAVE_DS2805
+	tristate "112-byte EEPROM support (DS28E05)"
+	help
+	  Say Y here if you want to use a 1-wire
+	  is a 112-byte user-programmable EEPROM is
+          organized as 7 pages of 16 bytes each with 64bit
+          unique number. Requires OverDrive Speed to talk to.
+
 config W1_SLAVE_DS2431
 	tristate "1kb EEPROM family support (DS2431)"
 	help
@@ -140,10 +148,4 @@ config W1_SLAVE_DS28E04
 
 	  If you are unsure, say N.
 
-config W1_SLAVE_BQ27000
-	tristate "BQ27000 slave support"
-	help
-	  Say Y here if you want to use a hdq
-	  bq27000 slave support.
-
 endmenu
diff --git a/drivers/w1/slaves/Makefile b/drivers/w1/slaves/Makefile
index 54c63e420302..36b22fb2d3a1 100644
--- a/drivers/w1/slaves/Makefile
+++ b/drivers/w1/slaves/Makefile
@@ -10,10 +10,10 @@ obj-$(CONFIG_W1_SLAVE_DS2413)	+= w1_ds2413.o
 obj-$(CONFIG_W1_SLAVE_DS2406)	+= w1_ds2406.o
 obj-$(CONFIG_W1_SLAVE_DS2423)	+= w1_ds2423.o
 obj-$(CONFIG_W1_SLAVE_DS2431)	+= w1_ds2431.o
+obj-$(CONFIG_W1_SLAVE_DS2805)	+= w1_ds2805.o
 obj-$(CONFIG_W1_SLAVE_DS2433)	+= w1_ds2433.o
 obj-$(CONFIG_W1_SLAVE_DS2438)	+= w1_ds2438.o
 obj-$(CONFIG_W1_SLAVE_DS2760)	+= w1_ds2760.o
 obj-$(CONFIG_W1_SLAVE_DS2780)	+= w1_ds2780.o
 obj-$(CONFIG_W1_SLAVE_DS2781)	+= w1_ds2781.o
-obj-$(CONFIG_W1_SLAVE_BQ27000)	+= w1_bq27000.o
 obj-$(CONFIG_W1_SLAVE_DS28E04)	+= w1_ds28e04.o
diff --git a/drivers/w1/slaves/w1_bq27000.c b/drivers/w1/slaves/w1_bq27000.c
deleted file mode 100644
index 8046ac45381a..000000000000
--- a/drivers/w1/slaves/w1_bq27000.c
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * drivers/w1/slaves/w1_bq27000.c
- *
- * Copyright (C) 2007 Texas Instruments, Inc.
- *
- * This file is licensed under the terms of the GNU General Public License
- * version 2. This program is licensed "as is" without any warranty of any
- * kind, whether express or implied.
- *
- */
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/device.h>
-#include <linux/types.h>
-#include <linux/platform_device.h>
-#include <linux/mutex.h>
-#include <linux/power/bq27xxx_battery.h>
-
-#include <linux/w1.h>
-
-#define W1_FAMILY_BQ27000	0x01
-
-#define HDQ_CMD_READ	(0)
-#define HDQ_CMD_WRITE	(1<<7)
-
-static int F_ID;
-module_param(F_ID, int, S_IRUSR);
-MODULE_PARM_DESC(F_ID, "1-wire slave FID for BQ device");
-
-static int w1_bq27000_read(struct device *dev, unsigned int reg)
-{
-	u8 val;
-	struct w1_slave *sl = container_of(dev->parent, struct w1_slave, dev);
-
-	mutex_lock(&sl->master->bus_mutex);
-	w1_write_8(sl->master, HDQ_CMD_READ | reg);
-	val = w1_read_8(sl->master);
-	mutex_unlock(&sl->master->bus_mutex);
-
-	return val;
-}
-
-static struct bq27xxx_platform_data bq27000_battery_info = {
-	.read   = w1_bq27000_read,
-	.name   = "bq27000-battery",
-	.chip   = BQ27000,
-};
-
-static int w1_bq27000_add_slave(struct w1_slave *sl)
-{
-	int ret;
-	struct platform_device *pdev;
-
-	pdev = platform_device_alloc("bq27000-battery", -1);
-	if (!pdev) {
-		ret = -ENOMEM;
-		return ret;
-	}
-	ret = platform_device_add_data(pdev,
-				       &bq27000_battery_info,
-				       sizeof(bq27000_battery_info));
-	if (ret)
-		goto pdev_add_failed;
-	pdev->dev.parent = &sl->dev;
-
-	ret = platform_device_add(pdev);
-	if (ret)
-		goto pdev_add_failed;
-
-	dev_set_drvdata(&sl->dev, pdev);
-
-	goto success;
-
-pdev_add_failed:
-	platform_device_put(pdev);
-success:
-	return ret;
-}
-
-static void w1_bq27000_remove_slave(struct w1_slave *sl)
-{
-	struct platform_device *pdev = dev_get_drvdata(&sl->dev);
-
-	platform_device_unregister(pdev);
-}
-
-static struct w1_family_ops w1_bq27000_fops = {
-	.add_slave	= w1_bq27000_add_slave,
-	.remove_slave	= w1_bq27000_remove_slave,
-};
-
-static struct w1_family w1_bq27000_family = {
-	.fid = W1_FAMILY_BQ27000,
-	.fops = &w1_bq27000_fops,
-};
-
-static int __init w1_bq27000_init(void)
-{
-	if (F_ID)
-		w1_bq27000_family.fid = F_ID;
-
-	return w1_register_family(&w1_bq27000_family);
-}
-
-static void __exit w1_bq27000_exit(void)
-{
-	w1_unregister_family(&w1_bq27000_family);
-}
-
-module_init(w1_bq27000_init);
-module_exit(w1_bq27000_exit);
-
-MODULE_AUTHOR("Texas Instruments Ltd");
-MODULE_DESCRIPTION("HDQ/1-wire slave driver bq27000 battery monitor chip");
-MODULE_LICENSE("GPL");
-MODULE_ALIAS("w1-family-" __stringify(W1_FAMILY_BQ27000));
diff --git a/drivers/w1/slaves/w1_ds2438.c b/drivers/w1/slaves/w1_ds2438.c
index 6487fb772a20..bf641a191d07 100644
--- a/drivers/w1/slaves/w1_ds2438.c
+++ b/drivers/w1/slaves/w1_ds2438.c
@@ -51,7 +51,7 @@
 #define DS2438_CURRENT_MSB		0x06
 #define DS2438_THRESHOLD		0x07
 
-int w1_ds2438_get_page(struct w1_slave *sl, int pageno, u8 *buf)
+static int w1_ds2438_get_page(struct w1_slave *sl, int pageno, u8 *buf)
 {
 	unsigned int retries = W1_DS2438_RETRIES;
 	u8 w1_buf[2];
@@ -85,7 +85,7 @@ int w1_ds2438_get_page(struct w1_slave *sl, int pageno, u8 *buf)
 	return -1;
 }
 
-int w1_ds2438_get_temperature(struct w1_slave *sl, int16_t *temperature)
+static int w1_ds2438_get_temperature(struct w1_slave *sl, int16_t *temperature)
 {
 	unsigned int retries = W1_DS2438_RETRIES;
 	u8 w1_buf[DS2438_PAGE_SIZE + 1 /*for CRC*/];
@@ -127,7 +127,7 @@ post_unlock:
 	return ret;
 }
 
-int w1_ds2438_change_config_bit(struct w1_slave *sl, u8 mask, u8 value)
+static int w1_ds2438_change_config_bit(struct w1_slave *sl, u8 mask, u8 value)
 {
 	unsigned int retries = W1_DS2438_RETRIES;
 	u8 w1_buf[3];
@@ -186,7 +186,8 @@ int w1_ds2438_change_config_bit(struct w1_slave *sl, u8 mask, u8 value)
 	return -1;
 }
 
-uint16_t w1_ds2438_get_voltage(struct w1_slave *sl, int adc_input, uint16_t *voltage)
+static uint16_t w1_ds2438_get_voltage(struct w1_slave *sl,
+				      int adc_input, uint16_t *voltage)
 {
 	unsigned int retries = W1_DS2438_RETRIES;
 	u8 w1_buf[DS2438_PAGE_SIZE + 1 /*for CRC*/];
diff --git a/drivers/w1/slaves/w1_ds2805.c b/drivers/w1/slaves/w1_ds2805.c
new file mode 100644
index 000000000000..29348d283a65
--- /dev/null
+++ b/drivers/w1/slaves/w1_ds2805.c
@@ -0,0 +1,313 @@
+/*
+ * w1_ds2805 - w1 family 0d (DS28E05) driver
+ *
+ * Copyright (c) 2016 Andrew Worsley amworsley@gmail.com
+ *
+ * This source code is licensed under the GNU General Public License,
+ * Version 2. See the file COPYING for more details.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/device.h>
+#include <linux/types.h>
+#include <linux/delay.h>
+
+#include <linux/w1.h>
+
+#define W1_EEPROM_DS2805       0x0D
+
+#define W1_F0D_EEPROM_SIZE		128
+#define W1_F0D_PAGE_BITS		3
+#define W1_F0D_PAGE_SIZE		(1<<W1_F0D_PAGE_BITS)
+#define W1_F0D_PAGE_MASK		0x0F
+
+#define W1_F0D_SCRATCH_BITS  1
+#define W1_F0D_SCRATCH_SIZE  (1<<W1_F0D_SCRATCH_BITS)
+#define W1_F0D_SCRATCH_MASK  (W1_F0D_SCRATCH_SIZE-1)
+
+#define W1_F0D_READ_EEPROM	0xF0
+#define W1_F0D_WRITE_EEPROM	0x55
+#define W1_F0D_RELEASE		0xFF
+
+#define W1_F0D_CS_OK		0xAA /* Chip Status Ok */
+
+#define W1_F0D_TPROG_MS		16
+
+#define W1_F0D_READ_RETRIES		10
+#define W1_F0D_READ_MAXLEN		W1_F0D_EEPROM_SIZE
+
+/*
+ * Check the file size bounds and adjusts count as needed.
+ * This would not be needed if the file size didn't reset to 0 after a write.
+ */
+static inline size_t w1_f0d_fix_count(loff_t off, size_t count, size_t size)
+{
+	if (off > size)
+		return 0;
+
+	if ((off + count) > size)
+		return size - off;
+
+	return count;
+}
+
+/*
+ * Read a block from W1 ROM two times and compares the results.
+ * If they are equal they are returned, otherwise the read
+ * is repeated W1_F0D_READ_RETRIES times.
+ *
+ * count must not exceed W1_F0D_READ_MAXLEN.
+ */
+static int w1_f0d_readblock(struct w1_slave *sl, int off, int count, char *buf)
+{
+	u8 wrbuf[3];
+	u8 cmp[W1_F0D_READ_MAXLEN];
+	int tries = W1_F0D_READ_RETRIES;
+
+	do {
+		wrbuf[0] = W1_F0D_READ_EEPROM;
+		wrbuf[1] = off & 0x7f;
+		wrbuf[2] = 0;
+
+		if (w1_reset_select_slave(sl))
+			return -1;
+
+		w1_write_block(sl->master, wrbuf, sizeof(wrbuf));
+		w1_read_block(sl->master, buf, count);
+
+		if (w1_reset_select_slave(sl))
+			return -1;
+
+		w1_write_block(sl->master, wrbuf, sizeof(wrbuf));
+		w1_read_block(sl->master, cmp, count);
+
+		if (!memcmp(cmp, buf, count))
+			return 0;
+	} while (--tries);
+
+	dev_err(&sl->dev, "proof reading failed %d times\n",
+			W1_F0D_READ_RETRIES);
+
+	return -1;
+}
+
+static ssize_t w1_f0d_read_bin(struct file *filp, struct kobject *kobj,
+			       struct bin_attribute *bin_attr,
+			       char *buf, loff_t off, size_t count)
+{
+	struct w1_slave *sl = kobj_to_w1_slave(kobj);
+	int todo = count;
+
+	count = w1_f0d_fix_count(off, count, W1_F0D_EEPROM_SIZE);
+	if (count == 0)
+		return 0;
+
+	mutex_lock(&sl->master->mutex);
+
+	/* read directly from the EEPROM in chunks of W1_F0D_READ_MAXLEN */
+	while (todo > 0) {
+		int block_read;
+
+		if (todo >= W1_F0D_READ_MAXLEN)
+			block_read = W1_F0D_READ_MAXLEN;
+		else
+			block_read = todo;
+
+		if (w1_f0d_readblock(sl, off, block_read, buf) < 0) {
+			count = -EIO;
+			break;
+		}
+
+		todo -= W1_F0D_READ_MAXLEN;
+		buf += W1_F0D_READ_MAXLEN;
+		off += W1_F0D_READ_MAXLEN;
+	}
+
+	mutex_unlock(&sl->master->mutex);
+
+	return count;
+}
+
+/*
+ * Writes to the scratchpad and reads it back for verification.
+ * Then copies the scratchpad to EEPROM.
+ * The data must be aligned at W1_F0D_SCRATCH_SIZE bytes and
+ * must be W1_F0D_SCRATCH_SIZE bytes long.
+ * The master must be locked.
+ *
+ * @param sl	The slave structure
+ * @param addr	Address for the write
+ * @param len   length must be <= (W1_F0D_PAGE_SIZE - (addr & W1_F0D_PAGE_MASK))
+ * @param data	The data to write
+ * @return	0=Success -1=failure
+ */
+static int w1_f0d_write(struct w1_slave *sl, int addr, int len, const u8 *data)
+{
+	int tries = W1_F0D_READ_RETRIES;
+	u8 wrbuf[3];
+	u8 rdbuf[W1_F0D_SCRATCH_SIZE];
+	u8 cs;
+
+	if ((addr & 1) || (len != 2)) {
+		dev_err(&sl->dev, "%s: bad addr/len -  addr=%#x len=%d\n",
+		    __func__, addr, len);
+		return -1;
+	}
+
+retry:
+
+	/* Write the data to the scratchpad */
+	if (w1_reset_select_slave(sl))
+		return -1;
+
+	wrbuf[0] = W1_F0D_WRITE_EEPROM;
+	wrbuf[1] = addr & 0xff;
+	wrbuf[2] = 0xff; /* ?? from Example */
+
+	w1_write_block(sl->master, wrbuf, sizeof(wrbuf));
+	w1_write_block(sl->master, data, len);
+
+	w1_read_block(sl->master, rdbuf, sizeof(rdbuf));
+	/* Compare what was read against the data written */
+	if ((rdbuf[0] != data[0]) || (rdbuf[1] != data[1])) {
+
+		if (--tries)
+			goto retry;
+
+		dev_err(&sl->dev,
+			"could not write to eeprom, scratchpad compare failed %d times\n",
+			W1_F0D_READ_RETRIES);
+		pr_info("%s: rdbuf = %#x %#x data = %#x %#x\n",
+		    __func__, rdbuf[0], rdbuf[1], data[0], data[1]);
+
+		return -1;
+	}
+
+	/* Trigger write out to EEPROM */
+	w1_write_8(sl->master, W1_F0D_RELEASE);
+
+	/* Sleep for tprog ms to wait for the write to complete */
+	msleep(W1_F0D_TPROG_MS);
+
+	/* Check CS (Command Status) == 0xAA ? */
+	cs = w1_read_8(sl->master);
+	if (cs != W1_F0D_CS_OK) {
+		dev_err(&sl->dev, "save to eeprom failed = CS=%#x\n", cs);
+		return -1;
+	}
+
+	return 0;
+}
+
+static ssize_t w1_f0d_write_bin(struct file *filp, struct kobject *kobj,
+				struct bin_attribute *bin_attr,
+				char *buf, loff_t off, size_t count)
+{
+	struct w1_slave *sl = kobj_to_w1_slave(kobj);
+	int addr, len;
+	int copy;
+
+	count = w1_f0d_fix_count(off, count, W1_F0D_EEPROM_SIZE);
+	if (count == 0)
+		return 0;
+
+	mutex_lock(&sl->master->mutex);
+
+	/* Can only write data in blocks of the size of the scratchpad */
+	addr = off;
+	len = count;
+	while (len > 0) {
+
+		/* if len too short or addr not aligned */
+		if (len < W1_F0D_SCRATCH_SIZE || addr & W1_F0D_SCRATCH_MASK) {
+			char tmp[W1_F0D_SCRATCH_SIZE];
+
+			/* read the block and update the parts to be written */
+			if (w1_f0d_readblock(sl, addr & ~W1_F0D_SCRATCH_MASK,
+					W1_F0D_SCRATCH_SIZE, tmp)) {
+				count = -EIO;
+				goto out_up;
+			}
+
+			/* copy at most to the boundary of the PAGE or len */
+			copy = W1_F0D_SCRATCH_SIZE -
+				(addr & W1_F0D_SCRATCH_MASK);
+
+			if (copy > len)
+				copy = len;
+
+			memcpy(&tmp[addr & W1_F0D_SCRATCH_MASK], buf, copy);
+			if (w1_f0d_write(sl, addr & ~W1_F0D_SCRATCH_MASK,
+					W1_F0D_SCRATCH_SIZE, tmp) < 0) {
+				count = -EIO;
+				goto out_up;
+			}
+		} else {
+
+			copy = W1_F0D_SCRATCH_SIZE;
+			if (w1_f0d_write(sl, addr, copy, buf) < 0) {
+				count = -EIO;
+				goto out_up;
+			}
+		}
+		buf += copy;
+		addr += copy;
+		len -= copy;
+	}
+
+out_up:
+	mutex_unlock(&sl->master->mutex);
+
+	return count;
+}
+
+static struct bin_attribute w1_f0d_bin_attr = {
+	.attr = {
+		.name = "eeprom",
+		.mode = S_IRUGO | S_IWUSR,
+	},
+	.size = W1_F0D_EEPROM_SIZE,
+	.read = w1_f0d_read_bin,
+	.write = w1_f0d_write_bin,
+};
+
+static int w1_f0d_add_slave(struct w1_slave *sl)
+{
+	return sysfs_create_bin_file(&sl->dev.kobj, &w1_f0d_bin_attr);
+}
+
+static void w1_f0d_remove_slave(struct w1_slave *sl)
+{
+	sysfs_remove_bin_file(&sl->dev.kobj, &w1_f0d_bin_attr);
+}
+
+static struct w1_family_ops w1_f0d_fops = {
+	.add_slave      = w1_f0d_add_slave,
+	.remove_slave   = w1_f0d_remove_slave,
+};
+
+static struct w1_family w1_family_2d = {
+	.fid = W1_EEPROM_DS2805,
+	.fops = &w1_f0d_fops,
+};
+
+static int __init w1_f0d_init(void)
+{
+	pr_info("%s()\n", __func__);
+	return w1_register_family(&w1_family_2d);
+}
+
+static void __exit w1_f0d_fini(void)
+{
+	pr_info("%s()\n", __func__);
+	w1_unregister_family(&w1_family_2d);
+}
+
+module_init(w1_f0d_init);
+module_exit(w1_f0d_fini);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Andrew Worsley amworsley@gmail.com");
+MODULE_DESCRIPTION("w1 family 0d driver for DS2805, 1kb EEPROM");
diff --git a/drivers/w1/slaves/w1_therm.c b/drivers/w1/slaves/w1_therm.c
index cb3fc3c6b0d1..259525c3382a 100644
--- a/drivers/w1/slaves/w1_therm.c
+++ b/drivers/w1/slaves/w1_therm.c
@@ -29,6 +29,7 @@
 #include <linux/types.h>
 #include <linux/slab.h>
 #include <linux/delay.h>
+#include <linux/hwmon.h>
 
 #include <linux/w1.h>
 
@@ -59,6 +60,12 @@ struct w1_therm_family_data {
 	atomic_t refcnt;
 };
 
+struct therm_info {
+	u8 rom[9];
+	u8 crc;
+	u8 verdict;
+};
+
 /* return the address of the refcnt in the family data */
 #define THERM_REFCNT(family_data) \
 	(&((struct w1_therm_family_data *)family_data)->refcnt)
@@ -107,19 +114,72 @@ static struct attribute *w1_ds28ea00_attrs[] = {
 	&dev_attr_w1_seq.attr,
 	NULL,
 };
+
 ATTRIBUTE_GROUPS(w1_therm);
 ATTRIBUTE_GROUPS(w1_ds28ea00);
 
+#if IS_REACHABLE(CONFIG_HWMON)
+static int w1_read_temp(struct device *dev, u32 attr, int channel,
+			long *val);
+
+static umode_t w1_is_visible(const void *_data, enum hwmon_sensor_types type,
+			     u32 attr, int channel)
+{
+	return attr == hwmon_temp_input ? 0444 : 0;
+}
+
+static int w1_read(struct device *dev, enum hwmon_sensor_types type,
+		   u32 attr, int channel, long *val)
+{
+	switch (type) {
+	case hwmon_temp:
+		return w1_read_temp(dev, attr, channel, val);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static const u32 w1_temp_config[] = {
+	HWMON_T_INPUT,
+	0
+};
+
+static const struct hwmon_channel_info w1_temp = {
+	.type = hwmon_temp,
+	.config = w1_temp_config,
+};
+
+static const struct hwmon_channel_info *w1_info[] = {
+	&w1_temp,
+	NULL
+};
+
+static const struct hwmon_ops w1_hwmon_ops = {
+	.is_visible = w1_is_visible,
+	.read = w1_read,
+};
+
+static const struct hwmon_chip_info w1_chip_info = {
+	.ops = &w1_hwmon_ops,
+	.info = w1_info,
+};
+#define W1_CHIPINFO	(&w1_chip_info)
+#else
+#define W1_CHIPINFO	NULL
+#endif
+
 static struct w1_family_ops w1_therm_fops = {
 	.add_slave	= w1_therm_add_slave,
 	.remove_slave	= w1_therm_remove_slave,
 	.groups		= w1_therm_groups,
+	.chip_info	= W1_CHIPINFO,
 };
 
 static struct w1_family_ops w1_ds28ea00_fops = {
 	.add_slave	= w1_therm_add_slave,
 	.remove_slave	= w1_therm_remove_slave,
 	.groups		= w1_ds28ea00_groups,
+	.chip_info	= W1_CHIPINFO,
 };
 
 static struct w1_family w1_therm_family_DS18S20 = {
@@ -422,33 +482,31 @@ static ssize_t w1_slave_store(struct device *device,
 	return ret ? : size;
 }
 
-static ssize_t w1_slave_show(struct device *device,
-	struct device_attribute *attr, char *buf)
+static ssize_t read_therm(struct device *device,
+			  struct w1_slave *sl, struct therm_info *info)
 {
-	struct w1_slave *sl = dev_to_w1_slave(device);
 	struct w1_master *dev = sl->master;
-	u8 rom[9], crc, verdict, external_power;
-	int i, ret, max_trying = 10;
-	ssize_t c = PAGE_SIZE;
+	u8 external_power;
+	int ret, max_trying = 10;
 	u8 *family_data = sl->family_data;
 
 	ret = mutex_lock_interruptible(&dev->bus_mutex);
 	if (ret != 0)
-		goto post_unlock;
+		goto error;
 
-	if (!sl->family_data) {
+	if (!family_data) {
 		ret = -ENODEV;
-		goto pre_unlock;
+		goto mt_unlock;
 	}
 
 	/* prevent the slave from going away in sleep */
 	atomic_inc(THERM_REFCNT(family_data));
-	memset(rom, 0, sizeof(rom));
+	memset(info->rom, 0, sizeof(info->rom));
 
 	while (max_trying--) {
 
-		verdict = 0;
-		crc = 0;
+		info->verdict = 0;
+		info->crc = 0;
 
 		if (!w1_reset_select_slave(sl)) {
 			int count = 0;
@@ -474,47 +532,69 @@ static ssize_t w1_slave_show(struct device *device,
 				sleep_rem = msleep_interruptible(tm);
 				if (sleep_rem != 0) {
 					ret = -EINTR;
-					goto post_unlock;
+					goto dec_refcnt;
 				}
 
 				ret = mutex_lock_interruptible(&dev->bus_mutex);
 				if (ret != 0)
-					goto post_unlock;
+					goto dec_refcnt;
 			} else if (!w1_strong_pullup) {
 				sleep_rem = msleep_interruptible(tm);
 				if (sleep_rem != 0) {
 					ret = -EINTR;
-					goto pre_unlock;
+					goto dec_refcnt;
 				}
 			}
 
 			if (!w1_reset_select_slave(sl)) {
 
 				w1_write_8(dev, W1_READ_SCRATCHPAD);
-				count = w1_read_block(dev, rom, 9);
+				count = w1_read_block(dev, info->rom, 9);
 				if (count != 9) {
 					dev_warn(device, "w1_read_block() "
 						"returned %u instead of 9.\n",
 						count);
 				}
 
-				crc = w1_calc_crc8(rom, 8);
+				info->crc = w1_calc_crc8(info->rom, 8);
 
-				if (rom[8] == crc)
-					verdict = 1;
+				if (info->rom[8] == info->crc)
+					info->verdict = 1;
 			}
 		}
 
-		if (verdict)
+		if (info->verdict)
 			break;
 	}
 
+dec_refcnt:
+	atomic_dec(THERM_REFCNT(family_data));
+mt_unlock:
+	mutex_unlock(&dev->bus_mutex);
+error:
+	return ret;
+}
+
+static ssize_t w1_slave_show(struct device *device,
+			     struct device_attribute *attr, char *buf)
+{
+	struct w1_slave *sl = dev_to_w1_slave(device);
+	struct therm_info info;
+	u8 *family_data = sl->family_data;
+	int ret, i;
+	ssize_t c = PAGE_SIZE;
+	u8 fid = sl->family->fid;
+
+	ret = read_therm(device, sl, &info);
+	if (ret)
+		return ret;
+
 	for (i = 0; i < 9; ++i)
-		c -= snprintf(buf + PAGE_SIZE - c, c, "%02x ", rom[i]);
+		c -= snprintf(buf + PAGE_SIZE - c, c, "%02x ", info.rom[i]);
 	c -= snprintf(buf + PAGE_SIZE - c, c, ": crc=%02x %s\n",
-		      crc, (verdict) ? "YES" : "NO");
-	if (verdict)
-		memcpy(family_data, rom, sizeof(rom));
+		      info.crc, (info.verdict) ? "YES" : "NO");
+	if (info.verdict)
+		memcpy(family_data, info.rom, sizeof(info.rom));
 	else
 		dev_warn(device, "Read failed CRC check\n");
 
@@ -523,16 +603,42 @@ static ssize_t w1_slave_show(struct device *device,
 			      ((u8 *)family_data)[i]);
 
 	c -= snprintf(buf + PAGE_SIZE - c, c, "t=%d\n",
-		w1_convert_temp(rom, sl->family->fid));
+			w1_convert_temp(info.rom, fid));
 	ret = PAGE_SIZE - c;
+	return ret;
+}
 
-pre_unlock:
-	mutex_unlock(&dev->bus_mutex);
+#if IS_REACHABLE(CONFIG_HWMON)
+static int w1_read_temp(struct device *device, u32 attr, int channel,
+			long *val)
+{
+	struct w1_slave *sl = dev_get_drvdata(device);
+	struct therm_info info;
+	u8 fid = sl->family->fid;
+	int ret;
+
+	switch (attr) {
+	case hwmon_temp_input:
+		ret = read_therm(device, sl, &info);
+		if (ret)
+			return ret;
+
+		if (!info.verdict) {
+			ret = -EIO;
+			return ret;
+		}
+
+		*val = w1_convert_temp(info.rom, fid);
+		ret = 0;
+		break;
+	default:
+		ret = -EOPNOTSUPP;
+		break;
+	}
 
-post_unlock:
-	atomic_dec(THERM_REFCNT(family_data));
 	return ret;
 }
+#endif
 
 #define W1_42_CHAIN	0x99
 #define W1_42_CHAIN_OFF	0x3C
diff --git a/drivers/w1/w1.c b/drivers/w1/w1.c
index 74471e7aa5cc..0c2a5a8327bd 100644
--- a/drivers/w1/w1.c
+++ b/drivers/w1/w1.c
@@ -25,6 +25,7 @@
 #include <linux/sched.h>
 #include <linux/kthread.h>
 #include <linux/freezer.h>
+#include <linux/hwmon.h>
 
 #include <linux/atomic.h>
 
@@ -568,7 +569,7 @@ static struct attribute *w1_master_default_attrs[] = {
 	NULL
 };
 
-static struct attribute_group w1_master_defattr_group = {
+static const struct attribute_group w1_master_defattr_group = {
 	.attrs = w1_master_default_attrs,
 };
 
@@ -649,9 +650,24 @@ static int w1_family_notify(unsigned long action, struct w1_slave *sl)
 				return err;
 			}
 		}
-
+		if (IS_REACHABLE(CONFIG_HWMON) && fops->chip_info) {
+			struct device *hwmon
+				= hwmon_device_register_with_info(&sl->dev,
+						"w1_slave_temp", sl,
+						fops->chip_info,
+						NULL);
+			if (IS_ERR(hwmon)) {
+				dev_warn(&sl->dev,
+					 "could not create hwmon device\n");
+			} else {
+				sl->hwmon = hwmon;
+			}
+		}
 		break;
 	case BUS_NOTIFY_DEL_DEVICE:
+		if (IS_REACHABLE(CONFIG_HWMON) && fops->chip_info &&
+			    sl->hwmon)
+			hwmon_device_unregister(sl->hwmon);
 		if (fops->remove_slave)
 			sl->family->fops->remove_slave(sl);
 		if (fops->groups)
@@ -729,6 +745,8 @@ int w1_attach_slave_device(struct w1_master *dev, struct w1_reg_num *rn)
 	atomic_set(&sl->refcnt, 1);
 	atomic_inc(&sl->master->refcnt);
 	dev->slave_count++;
+	dev_info(&dev->dev, "Attaching one wire slave %02x.%012llx crc %02x\n",
+		  rn->family, (unsigned long long)rn->id, rn->crc);
 
 	/* slave modules need to be loaded in a context with unlocked mutex */
 	mutex_unlock(&dev->mutex);
diff --git a/drivers/watchdog/asm9260_wdt.c b/drivers/watchdog/asm9260_wdt.c
index 53da001f0838..7dd0da644a7f 100644
--- a/drivers/watchdog/asm9260_wdt.c
+++ b/drivers/watchdog/asm9260_wdt.c
@@ -82,7 +82,7 @@ static unsigned int asm9260_wdt_gettimeleft(struct watchdog_device *wdd)
 
 	counter = ioread32(priv->iobase + HW_WDTV);
 
-	return DIV_ROUND_CLOSEST(counter, priv->wdt_freq);
+	return counter / priv->wdt_freq;
 }
 
 static int asm9260_wdt_updatetimeout(struct watchdog_device *wdd)
@@ -296,7 +296,7 @@ static int asm9260_wdt_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	priv->rst = devm_reset_control_get(&pdev->dev, "wdt_rst");
+	priv->rst = devm_reset_control_get_exclusive(&pdev->dev, "wdt_rst");
 	if (IS_ERR(priv->rst))
 		return PTR_ERR(priv->rst);
 
diff --git a/drivers/watchdog/aspeed_wdt.c b/drivers/watchdog/aspeed_wdt.c
index 1c652582de40..79cc766cd30f 100644
--- a/drivers/watchdog/aspeed_wdt.c
+++ b/drivers/watchdog/aspeed_wdt.c
@@ -23,9 +23,21 @@ struct aspeed_wdt {
 	u32			ctrl;
 };
 
+struct aspeed_wdt_config {
+	u32 ext_pulse_width_mask;
+};
+
+static const struct aspeed_wdt_config ast2400_config = {
+	.ext_pulse_width_mask = 0xff,
+};
+
+static const struct aspeed_wdt_config ast2500_config = {
+	.ext_pulse_width_mask = 0xfffff,
+};
+
 static const struct of_device_id aspeed_wdt_of_table[] = {
-	{ .compatible = "aspeed,ast2400-wdt" },
-	{ .compatible = "aspeed,ast2500-wdt" },
+	{ .compatible = "aspeed,ast2400-wdt", .data = &ast2400_config },
+	{ .compatible = "aspeed,ast2500-wdt", .data = &ast2500_config },
 	{ },
 };
 MODULE_DEVICE_TABLE(of, aspeed_wdt_of_table);
@@ -36,12 +48,45 @@ MODULE_DEVICE_TABLE(of, aspeed_wdt_of_table);
 #define WDT_CTRL		0x0C
 #define   WDT_CTRL_RESET_MODE_SOC	(0x00 << 5)
 #define   WDT_CTRL_RESET_MODE_FULL_CHIP	(0x01 << 5)
+#define   WDT_CTRL_RESET_MODE_ARM_CPU	(0x10 << 5)
 #define   WDT_CTRL_1MHZ_CLK		BIT(4)
 #define   WDT_CTRL_WDT_EXT		BIT(3)
 #define   WDT_CTRL_WDT_INTR		BIT(2)
 #define   WDT_CTRL_RESET_SYSTEM		BIT(1)
 #define   WDT_CTRL_ENABLE		BIT(0)
 
+/*
+ * WDT_RESET_WIDTH controls the characteristics of the external pulse (if
+ * enabled), specifically:
+ *
+ * * Pulse duration
+ * * Drive mode: push-pull vs open-drain
+ * * Polarity: Active high or active low
+ *
+ * Pulse duration configuration is available on both the AST2400 and AST2500,
+ * though the field changes between SoCs:
+ *
+ * AST2400: Bits 7:0
+ * AST2500: Bits 19:0
+ *
+ * This difference is captured in struct aspeed_wdt_config.
+ *
+ * The AST2500 exposes the drive mode and polarity options, but not in a
+ * regular fashion. For read purposes, bit 31 represents active high or low,
+ * and bit 30 represents push-pull or open-drain. With respect to write, magic
+ * values need to be written to the top byte to change the state of the drive
+ * mode and polarity bits. Any other value written to the top byte has no
+ * effect on the state of the drive mode or polarity bits. However, the pulse
+ * width value must be preserved (as desired) if written.
+ */
+#define WDT_RESET_WIDTH		0x18
+#define   WDT_RESET_WIDTH_ACTIVE_HIGH	BIT(31)
+#define     WDT_ACTIVE_HIGH_MAGIC	(0xA5 << 24)
+#define     WDT_ACTIVE_LOW_MAGIC	(0x5A << 24)
+#define   WDT_RESET_WIDTH_PUSH_PULL	BIT(30)
+#define     WDT_PUSH_PULL_MAGIC		(0xA8 << 24)
+#define     WDT_OPEN_DRAIN_MAGIC	(0x8A << 24)
+
 #define WDT_RESTART_MAGIC	0x4755
 
 /* 32 bits at 1MHz, in milliseconds */
@@ -138,8 +183,13 @@ static const struct watchdog_info aspeed_wdt_info = {
 
 static int aspeed_wdt_probe(struct platform_device *pdev)
 {
+	const struct aspeed_wdt_config *config;
+	const struct of_device_id *ofdid;
 	struct aspeed_wdt *wdt;
 	struct resource *res;
+	struct device_node *np;
+	const char *reset_type;
+	u32 duration;
 	int ret;
 
 	wdt = devm_kzalloc(&pdev->dev, sizeof(*wdt), GFP_KERNEL);
@@ -164,20 +214,88 @@ static int aspeed_wdt_probe(struct platform_device *pdev)
 	wdt->wdd.timeout = WDT_DEFAULT_TIMEOUT;
 	watchdog_init_timeout(&wdt->wdd, 0, &pdev->dev);
 
+	np = pdev->dev.of_node;
+
+	ofdid = of_match_node(aspeed_wdt_of_table, np);
+	if (!ofdid)
+		return -EINVAL;
+	config = ofdid->data;
+
+	wdt->ctrl = WDT_CTRL_1MHZ_CLK;
+
 	/*
 	 * Control reset on a per-device basis to ensure the
-	 * host is not affected by a BMC reboot, so only reset
-	 * the SOC and not the full chip
+	 * host is not affected by a BMC reboot
 	 */
-	wdt->ctrl = WDT_CTRL_RESET_MODE_SOC |
-		WDT_CTRL_1MHZ_CLK |
-		WDT_CTRL_RESET_SYSTEM;
+	ret = of_property_read_string(np, "aspeed,reset-type", &reset_type);
+	if (ret) {
+		wdt->ctrl |= WDT_CTRL_RESET_MODE_SOC | WDT_CTRL_RESET_SYSTEM;
+	} else {
+		if (!strcmp(reset_type, "cpu"))
+			wdt->ctrl |= WDT_CTRL_RESET_MODE_ARM_CPU;
+		else if (!strcmp(reset_type, "soc"))
+			wdt->ctrl |= WDT_CTRL_RESET_MODE_SOC;
+		else if (!strcmp(reset_type, "system"))
+			wdt->ctrl |= WDT_CTRL_RESET_SYSTEM;
+		else if (strcmp(reset_type, "none"))
+			return -EINVAL;
+	}
+	if (of_property_read_bool(np, "aspeed,external-signal"))
+		wdt->ctrl |= WDT_CTRL_WDT_EXT;
+
+	writel(wdt->ctrl, wdt->base + WDT_CTRL);
 
 	if (readl(wdt->base + WDT_CTRL) & WDT_CTRL_ENABLE)  {
 		aspeed_wdt_start(&wdt->wdd);
 		set_bit(WDOG_HW_RUNNING, &wdt->wdd.status);
 	}
 
+	if (of_device_is_compatible(np, "aspeed,ast2500-wdt")) {
+		u32 reg = readl(wdt->base + WDT_RESET_WIDTH);
+
+		reg &= config->ext_pulse_width_mask;
+		if (of_property_read_bool(np, "aspeed,ext-push-pull"))
+			reg |= WDT_PUSH_PULL_MAGIC;
+		else
+			reg |= WDT_OPEN_DRAIN_MAGIC;
+
+		writel(reg, wdt->base + WDT_RESET_WIDTH);
+
+		reg &= config->ext_pulse_width_mask;
+		if (of_property_read_bool(np, "aspeed,ext-active-high"))
+			reg |= WDT_ACTIVE_HIGH_MAGIC;
+		else
+			reg |= WDT_ACTIVE_LOW_MAGIC;
+
+		writel(reg, wdt->base + WDT_RESET_WIDTH);
+	}
+
+	if (!of_property_read_u32(np, "aspeed,ext-pulse-duration", &duration)) {
+		u32 max_duration = config->ext_pulse_width_mask + 1;
+
+		if (duration == 0 || duration > max_duration) {
+			dev_err(&pdev->dev, "Invalid pulse duration: %uus\n",
+					duration);
+			duration = max(1U, min(max_duration, duration));
+			dev_info(&pdev->dev, "Pulse duration set to %uus\n",
+					duration);
+		}
+
+		/*
+		 * The watchdog is always configured with a 1MHz source, so
+		 * there is no need to scale the microsecond value. However we
+		 * need to offset it - from the datasheet:
+		 *
+		 * "This register decides the asserting duration of wdt_ext and
+		 * wdt_rstarm signal. The default value is 0xFF. It means the
+		 * default asserting duration of wdt_ext and wdt_rstarm is
+		 * 256us."
+		 *
+		 * This implies a value of 0 gives a 1us pulse.
+		 */
+		writel(duration - 1, wdt->base + WDT_RESET_WIDTH);
+	}
+
 	ret = devm_watchdog_register_device(&pdev->dev, &wdt->wdd);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to register\n");
diff --git a/drivers/watchdog/bcm7038_wdt.c b/drivers/watchdog/bcm7038_wdt.c
index c1b8e534fb55..f88f546e8050 100644
--- a/drivers/watchdog/bcm7038_wdt.c
+++ b/drivers/watchdog/bcm7038_wdt.c
@@ -136,7 +136,9 @@ static int bcm7038_wdt_probe(struct platform_device *pdev)
 	wdt->clk = devm_clk_get(dev, NULL);
 	/* If unable to get clock, use default frequency */
 	if (!IS_ERR(wdt->clk)) {
-		clk_prepare_enable(wdt->clk);
+		err = clk_prepare_enable(wdt->clk);
+		if (err)
+			return err;
 		wdt->rate = clk_get_rate(wdt->clk);
 		/* Prevent divide-by-zero exception */
 		if (!wdt->rate)
diff --git a/drivers/watchdog/cadence_wdt.c b/drivers/watchdog/cadence_wdt.c
index 05c000081e9d..064cf7b6c1c5 100644
--- a/drivers/watchdog/cadence_wdt.c
+++ b/drivers/watchdog/cadence_wdt.c
@@ -52,12 +52,12 @@
 static int wdt_timeout;
 static int nowayout = WATCHDOG_NOWAYOUT;
 
-module_param(wdt_timeout, int, 0);
+module_param(wdt_timeout, int, 0644);
 MODULE_PARM_DESC(wdt_timeout,
 		 "Watchdog time in seconds. (default="
 		 __MODULE_STRING(CDNS_WDT_DEFAULT_TIMEOUT) ")");
 
-module_param(nowayout, int, 0);
+module_param(nowayout, int, 0644);
 MODULE_PARM_DESC(nowayout,
 		 "Watchdog cannot be stopped once started (default="
 		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
@@ -368,7 +368,7 @@ static int cdns_wdt_probe(struct platform_device *pdev)
 	}
 	platform_set_drvdata(pdev, wdt);
 
-	dev_dbg(&pdev->dev, "Xilinx Watchdog Timer at %p with timeout %ds%s\n",
+	dev_info(&pdev->dev, "Xilinx Watchdog Timer at %p with timeout %ds%s\n",
 		 wdt->regs, cdns_wdt_device->timeout,
 		 nowayout ? ", nowayout" : "");
 
diff --git a/drivers/watchdog/coh901327_wdt.c b/drivers/watchdog/coh901327_wdt.c
index 38dd60f0cfcc..4410337f4f7f 100644
--- a/drivers/watchdog/coh901327_wdt.c
+++ b/drivers/watchdog/coh901327_wdt.c
@@ -218,7 +218,7 @@ static const struct watchdog_info coh901327_ident = {
 	.identity = DRV_NAME,
 };
 
-static struct watchdog_ops coh901327_ops = {
+static const struct watchdog_ops coh901327_ops = {
 	.owner = THIS_MODULE,
 	.start = coh901327_start,
 	.stop = coh901327_stop,
diff --git a/drivers/watchdog/da9063_wdt.c b/drivers/watchdog/da9063_wdt.c
index 4691c5509129..2a20fc163ed0 100644
--- a/drivers/watchdog/da9063_wdt.c
+++ b/drivers/watchdog/da9063_wdt.c
@@ -36,11 +36,6 @@ static const unsigned int wdt_timeout[] = { 0, 2, 4, 8, 16, 32, 65, 131 };
 #define DA9063_WDG_TIMEOUT		wdt_timeout[3]
 #define DA9063_RESET_PROTECTION_MS	256
 
-struct da9063_watchdog {
-	struct da9063 *da9063;
-	struct watchdog_device wdtdev;
-};
-
 static unsigned int da9063_wdt_timeout_to_sel(unsigned int secs)
 {
 	unsigned int i;
@@ -61,14 +56,14 @@ static int _da9063_wdt_set_timeout(struct da9063 *da9063, unsigned int regval)
 
 static int da9063_wdt_start(struct watchdog_device *wdd)
 {
-	struct da9063_watchdog *wdt = watchdog_get_drvdata(wdd);
+	struct da9063 *da9063 = watchdog_get_drvdata(wdd);
 	unsigned int selector;
 	int ret;
 
-	selector = da9063_wdt_timeout_to_sel(wdt->wdtdev.timeout);
-	ret = _da9063_wdt_set_timeout(wdt->da9063, selector);
+	selector = da9063_wdt_timeout_to_sel(wdd->timeout);
+	ret = _da9063_wdt_set_timeout(da9063, selector);
 	if (ret)
-		dev_err(wdt->da9063->dev, "Watchdog failed to start (err = %d)\n",
+		dev_err(da9063->dev, "Watchdog failed to start (err = %d)\n",
 			ret);
 
 	return ret;
@@ -76,13 +71,13 @@ static int da9063_wdt_start(struct watchdog_device *wdd)
 
 static int da9063_wdt_stop(struct watchdog_device *wdd)
 {
-	struct da9063_watchdog *wdt = watchdog_get_drvdata(wdd);
+	struct da9063 *da9063 = watchdog_get_drvdata(wdd);
 	int ret;
 
-	ret = regmap_update_bits(wdt->da9063->regmap, DA9063_REG_CONTROL_D,
+	ret = regmap_update_bits(da9063->regmap, DA9063_REG_CONTROL_D,
 				 DA9063_TWDSCALE_MASK, DA9063_TWDSCALE_DISABLE);
 	if (ret)
-		dev_alert(wdt->da9063->dev, "Watchdog failed to stop (err = %d)\n",
+		dev_alert(da9063->dev, "Watchdog failed to stop (err = %d)\n",
 			  ret);
 
 	return ret;
@@ -90,13 +85,13 @@ static int da9063_wdt_stop(struct watchdog_device *wdd)
 
 static int da9063_wdt_ping(struct watchdog_device *wdd)
 {
-	struct da9063_watchdog *wdt = watchdog_get_drvdata(wdd);
+	struct da9063 *da9063 = watchdog_get_drvdata(wdd);
 	int ret;
 
-	ret = regmap_write(wdt->da9063->regmap, DA9063_REG_CONTROL_F,
+	ret = regmap_write(da9063->regmap, DA9063_REG_CONTROL_F,
 			   DA9063_WATCHDOG);
 	if (ret)
-		dev_alert(wdt->da9063->dev, "Failed to ping the watchdog (err = %d)\n",
+		dev_alert(da9063->dev, "Failed to ping the watchdog (err = %d)\n",
 			  ret);
 
 	return ret;
@@ -105,14 +100,14 @@ static int da9063_wdt_ping(struct watchdog_device *wdd)
 static int da9063_wdt_set_timeout(struct watchdog_device *wdd,
 				  unsigned int timeout)
 {
-	struct da9063_watchdog *wdt = watchdog_get_drvdata(wdd);
+	struct da9063 *da9063 = watchdog_get_drvdata(wdd);
 	unsigned int selector;
 	int ret;
 
 	selector = da9063_wdt_timeout_to_sel(timeout);
-	ret = _da9063_wdt_set_timeout(wdt->da9063, selector);
+	ret = _da9063_wdt_set_timeout(da9063, selector);
 	if (ret)
-		dev_err(wdt->da9063->dev, "Failed to set watchdog timeout (err = %d)\n",
+		dev_err(da9063->dev, "Failed to set watchdog timeout (err = %d)\n",
 			ret);
 	else
 		wdd->timeout = wdt_timeout[selector];
@@ -123,13 +118,13 @@ static int da9063_wdt_set_timeout(struct watchdog_device *wdd,
 static int da9063_wdt_restart(struct watchdog_device *wdd, unsigned long action,
 			      void *data)
 {
-	struct da9063_watchdog *wdt = watchdog_get_drvdata(wdd);
+	struct da9063 *da9063 = watchdog_get_drvdata(wdd);
 	int ret;
 
-	ret = regmap_write(wdt->da9063->regmap, DA9063_REG_CONTROL_F,
+	ret = regmap_write(da9063->regmap, DA9063_REG_CONTROL_F,
 			   DA9063_SHUTDOWN);
 	if (ret)
-		dev_alert(wdt->da9063->dev, "Failed to shutdown (err = %d)\n",
+		dev_alert(da9063->dev, "Failed to shutdown (err = %d)\n",
 			  ret);
 
 	return ret;
@@ -152,7 +147,7 @@ static const struct watchdog_ops da9063_watchdog_ops = {
 static int da9063_wdt_probe(struct platform_device *pdev)
 {
 	struct da9063 *da9063;
-	struct da9063_watchdog *wdt;
+	struct watchdog_device *wdd;
 
 	if (!pdev->dev.parent)
 		return -EINVAL;
@@ -161,27 +156,25 @@ static int da9063_wdt_probe(struct platform_device *pdev)
 	if (!da9063)
 		return -EINVAL;
 
-	wdt = devm_kzalloc(&pdev->dev, sizeof(*wdt), GFP_KERNEL);
-	if (!wdt)
+	wdd = devm_kzalloc(&pdev->dev, sizeof(*wdd), GFP_KERNEL);
+	if (!wdd)
 		return -ENOMEM;
 
-	wdt->da9063 = da9063;
-
-	wdt->wdtdev.info = &da9063_watchdog_info;
-	wdt->wdtdev.ops = &da9063_watchdog_ops;
-	wdt->wdtdev.min_timeout = DA9063_WDT_MIN_TIMEOUT;
-	wdt->wdtdev.max_timeout = DA9063_WDT_MAX_TIMEOUT;
-	wdt->wdtdev.min_hw_heartbeat_ms = DA9063_RESET_PROTECTION_MS;
-	wdt->wdtdev.timeout = DA9063_WDG_TIMEOUT;
-	wdt->wdtdev.parent = &pdev->dev;
+	wdd->info = &da9063_watchdog_info;
+	wdd->ops = &da9063_watchdog_ops;
+	wdd->min_timeout = DA9063_WDT_MIN_TIMEOUT;
+	wdd->max_timeout = DA9063_WDT_MAX_TIMEOUT;
+	wdd->min_hw_heartbeat_ms = DA9063_RESET_PROTECTION_MS;
+	wdd->timeout = DA9063_WDG_TIMEOUT;
+	wdd->parent = &pdev->dev;
 
-	wdt->wdtdev.status = WATCHDOG_NOWAYOUT_INIT_STATUS;
+	wdd->status = WATCHDOG_NOWAYOUT_INIT_STATUS;
 
-	watchdog_set_restart_priority(&wdt->wdtdev, 128);
+	watchdog_set_restart_priority(wdd, 128);
 
-	watchdog_set_drvdata(&wdt->wdtdev, wdt);
+	watchdog_set_drvdata(wdd, da9063);
 
-	return devm_watchdog_register_device(&pdev->dev, &wdt->wdtdev);
+	return devm_watchdog_register_device(&pdev->dev, wdd);
 }
 
 static struct platform_driver da9063_wdt_driver = {
diff --git a/drivers/watchdog/diag288_wdt.c b/drivers/watchdog/diag288_wdt.c
index 6f591084bb7a..806a04a676b7 100644
--- a/drivers/watchdog/diag288_wdt.c
+++ b/drivers/watchdog/diag288_wdt.c
@@ -213,7 +213,7 @@ static const struct watchdog_ops wdt_ops = {
 	.set_timeout = wdt_set_timeout,
 };
 
-static struct watchdog_info wdt_info = {
+static const struct watchdog_info wdt_info = {
 	.options = WDIOF_SETTIMEOUT | WDIOF_KEEPALIVEPING | WDIOF_MAGICCLOSE,
 	.firmware_version = 0,
 	.identity = "z Watchdog",
diff --git a/drivers/watchdog/iTCO_wdt.c b/drivers/watchdog/iTCO_wdt.c
index c4f65873bfa4..347f0389b089 100644
--- a/drivers/watchdog/iTCO_wdt.c
+++ b/drivers/watchdog/iTCO_wdt.c
@@ -306,15 +306,16 @@ static int iTCO_wdt_ping(struct watchdog_device *wd_dev)
 
 	iTCO_vendor_pre_keepalive(p->smi_res, wd_dev->timeout);
 
-	/* Reset the timeout status bit so that the timer
-	 * needs to count down twice again before rebooting */
-	outw(0x0008, TCO1_STS(p));	/* write 1 to clear bit */
-
 	/* Reload the timer by writing to the TCO Timer Counter register */
-	if (p->iTCO_version >= 2)
+	if (p->iTCO_version >= 2) {
 		outw(0x01, TCO_RLD(p));
-	else if (p->iTCO_version == 1)
+	} else if (p->iTCO_version == 1) {
+		/* Reset the timeout status bit so that the timer
+		 * needs to count down twice again before rebooting */
+		outw(0x0008, TCO1_STS(p));	/* write 1 to clear bit */
+
 		outb(0x01, TCO_RLD(p));
+	}
 
 	spin_unlock(&p->io_lock);
 	return 0;
@@ -327,8 +328,11 @@ static int iTCO_wdt_set_timeout(struct watchdog_device *wd_dev, unsigned int t)
 	unsigned char val8;
 	unsigned int tmrval;
 
-	/* The timer counts down twice before rebooting */
-	tmrval = seconds_to_ticks(p, t) / 2;
+	tmrval = seconds_to_ticks(p, t);
+
+	/* For TCO v1 the timer counts down twice before rebooting */
+	if (p->iTCO_version == 1)
+		tmrval /= 2;
 
 	/* from the specs: */
 	/* "Values of 0h-3h are ignored and should not be attempted" */
@@ -381,8 +385,6 @@ static unsigned int iTCO_wdt_get_timeleft(struct watchdog_device *wd_dev)
 		spin_lock(&p->io_lock);
 		val16 = inw(TCO_RLD(p));
 		val16 &= 0x3ff;
-		if (!(inw(TCO1_STS(p)) & 0x0008))
-			val16 += (inw(TCOv2_TMR(p)) & 0x3ff);
 		spin_unlock(&p->io_lock);
 
 		time_left = ticks_to_seconds(p, val16);
diff --git a/drivers/watchdog/it87_wdt.c b/drivers/watchdog/it87_wdt.c
index dd1e7eaef50f..e96faea24925 100644
--- a/drivers/watchdog/it87_wdt.c
+++ b/drivers/watchdog/it87_wdt.c
@@ -253,7 +253,7 @@ static const struct watchdog_info ident = {
 	.identity = WATCHDOG_NAME,
 };
 
-static struct watchdog_ops wdt_ops = {
+static const struct watchdog_ops wdt_ops = {
 	.owner = THIS_MODULE,
 	.start = wdt_start,
 	.stop = wdt_stop,
diff --git a/drivers/watchdog/lantiq_wdt.c b/drivers/watchdog/lantiq_wdt.c
index e0823677d8c1..7f43cefa0eae 100644
--- a/drivers/watchdog/lantiq_wdt.c
+++ b/drivers/watchdog/lantiq_wdt.c
@@ -4,6 +4,7 @@
  *  by the Free Software Foundation.
  *
  *  Copyright (C) 2010 John Crispin <john@phrozen.org>
+ *  Copyright (C) 2017 Hauke Mehrtens <hauke@hauke-m.de>
  *  Based on EP93xx wdt driver
  */
 
@@ -17,9 +18,20 @@
 #include <linux/uaccess.h>
 #include <linux/clk.h>
 #include <linux/io.h>
+#include <linux/regmap.h>
+#include <linux/mfd/syscon.h>
 
 #include <lantiq_soc.h>
 
+#define LTQ_XRX_RCU_RST_STAT		0x0014
+#define LTQ_XRX_RCU_RST_STAT_WDT	BIT(31)
+
+/* CPU0 Reset Source Register */
+#define LTQ_FALCON_SYS1_CPU0RS		0x0060
+/* reset cause mask */
+#define LTQ_FALCON_SYS1_CPU0RS_MASK	0x0007
+#define LTQ_FALCON_SYS1_CPU0RS_WDT	0x02
+
 /*
  * Section 3.4 of the datasheet
  * The password sequence protects the WDT control register from unintended
@@ -186,16 +198,70 @@ static struct miscdevice ltq_wdt_miscdev = {
 	.fops	= &ltq_wdt_fops,
 };
 
+typedef int (*ltq_wdt_bootstatus_set)(struct platform_device *pdev);
+
+static int ltq_wdt_bootstatus_xrx(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct regmap *rcu_regmap;
+	u32 val;
+	int err;
+
+	rcu_regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "regmap");
+	if (IS_ERR(rcu_regmap))
+		return PTR_ERR(rcu_regmap);
+
+	err = regmap_read(rcu_regmap, LTQ_XRX_RCU_RST_STAT, &val);
+	if (err)
+		return err;
+
+	if (val & LTQ_XRX_RCU_RST_STAT_WDT)
+		ltq_wdt_bootstatus = WDIOF_CARDRESET;
+
+	return 0;
+}
+
+static int ltq_wdt_bootstatus_falcon(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct regmap *rcu_regmap;
+	u32 val;
+	int err;
+
+	rcu_regmap = syscon_regmap_lookup_by_phandle(dev->of_node,
+						     "lantiq,rcu");
+	if (IS_ERR(rcu_regmap))
+		return PTR_ERR(rcu_regmap);
+
+	err = regmap_read(rcu_regmap, LTQ_FALCON_SYS1_CPU0RS, &val);
+	if (err)
+		return err;
+
+	if ((val & LTQ_FALCON_SYS1_CPU0RS_MASK) == LTQ_FALCON_SYS1_CPU0RS_WDT)
+		ltq_wdt_bootstatus = WDIOF_CARDRESET;
+
+	return 0;
+}
+
 static int
 ltq_wdt_probe(struct platform_device *pdev)
 {
 	struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	struct clk *clk;
+	ltq_wdt_bootstatus_set ltq_wdt_bootstatus_set;
+	int ret;
 
 	ltq_wdt_membase = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(ltq_wdt_membase))
 		return PTR_ERR(ltq_wdt_membase);
 
+	ltq_wdt_bootstatus_set = of_device_get_match_data(&pdev->dev);
+	if (ltq_wdt_bootstatus_set) {
+		ret = ltq_wdt_bootstatus_set(pdev);
+		if (ret)
+			return ret;
+	}
+
 	/* we do not need to enable the clock as it is always running */
 	clk = clk_get_io();
 	if (IS_ERR(clk)) {
@@ -205,10 +271,6 @@ ltq_wdt_probe(struct platform_device *pdev)
 	ltq_io_region_clk_rate = clk_get_rate(clk);
 	clk_put(clk);
 
-	/* find out if the watchdog caused the last reboot */
-	if (ltq_reset_cause() == LTQ_RST_CAUSE_WDTRST)
-		ltq_wdt_bootstatus = WDIOF_CARDRESET;
-
 	dev_info(&pdev->dev, "Init done\n");
 	return misc_register(&ltq_wdt_miscdev);
 }
@@ -222,7 +284,9 @@ ltq_wdt_remove(struct platform_device *pdev)
 }
 
 static const struct of_device_id ltq_wdt_match[] = {
-	{ .compatible = "lantiq,wdt" },
+	{ .compatible = "lantiq,wdt", .data = NULL},
+	{ .compatible = "lantiq,xrx100-wdt", .data = ltq_wdt_bootstatus_xrx },
+	{ .compatible = "lantiq,falcon-wdt", .data = ltq_wdt_bootstatus_falcon },
 	{},
 };
 MODULE_DEVICE_TABLE(of, ltq_wdt_match);
diff --git a/drivers/watchdog/max77620_wdt.c b/drivers/watchdog/max77620_wdt.c
index 68c41fa2be27..2c9f53eaff4f 100644
--- a/drivers/watchdog/max77620_wdt.c
+++ b/drivers/watchdog/max77620_wdt.c
@@ -201,7 +201,7 @@ static int max77620_wdt_remove(struct platform_device *pdev)
 	return 0;
 }
 
-static struct platform_device_id max77620_wdt_devtype[] = {
+static const struct platform_device_id max77620_wdt_devtype[] = {
 	{ .name = "max77620-watchdog", },
 	{ },
 };
diff --git a/drivers/watchdog/mei_wdt.c b/drivers/watchdog/mei_wdt.c
index b29c6fde7473..ea60b29494fb 100644
--- a/drivers/watchdog/mei_wdt.c
+++ b/drivers/watchdog/mei_wdt.c
@@ -670,7 +670,7 @@ static int mei_wdt_remove(struct mei_cl_device *cldev)
 #define MEI_UUID_WD UUID_LE(0x05B79A6F, 0x4628, 0x4D7F, \
 			    0x89, 0x9D, 0xA9, 0x15, 0x14, 0xCB, 0x32, 0xAB)
 
-static struct mei_cl_device_id mei_wdt_tbl[] = {
+static const struct mei_cl_device_id mei_wdt_tbl[] = {
 	{ .uuid = MEI_UUID_WD, .version = MEI_CL_VERSION_ANY },
 	/* required last entry */
 	{ }
diff --git a/drivers/watchdog/meson_wdt.c b/drivers/watchdog/meson_wdt.c
index 491b9bf13d84..304274c67735 100644
--- a/drivers/watchdog/meson_wdt.c
+++ b/drivers/watchdog/meson_wdt.c
@@ -155,7 +155,9 @@ static const struct watchdog_ops meson_wdt_ops = {
 
 static const struct of_device_id meson_wdt_dt_ids[] = {
 	{ .compatible = "amlogic,meson6-wdt", .data = &meson6_wdt_data },
+	{ .compatible = "amlogic,meson8-wdt", .data = &meson6_wdt_data },
 	{ .compatible = "amlogic,meson8b-wdt", .data = &meson8b_wdt_data },
+	{ .compatible = "amlogic,meson8m2-wdt", .data = &meson8b_wdt_data },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, meson_wdt_dt_ids);
diff --git a/drivers/watchdog/mt7621_wdt.c b/drivers/watchdog/mt7621_wdt.c
index 48a06067075d..db38f8017218 100644
--- a/drivers/watchdog/mt7621_wdt.c
+++ b/drivers/watchdog/mt7621_wdt.c
@@ -105,7 +105,7 @@ static int mt7621_wdt_bootcause(void)
 	return 0;
 }
 
-static struct watchdog_info mt7621_wdt_info = {
+static const struct watchdog_info mt7621_wdt_info = {
 	.identity = "Mediatek Watchdog",
 	.options = WDIOF_SETTIMEOUT | WDIOF_KEEPALIVEPING | WDIOF_MAGICCLOSE,
 };
@@ -135,7 +135,7 @@ static int mt7621_wdt_probe(struct platform_device *pdev)
 	if (IS_ERR(mt7621_wdt_base))
 		return PTR_ERR(mt7621_wdt_base);
 
-	mt7621_wdt_reset = devm_reset_control_get(&pdev->dev, NULL);
+	mt7621_wdt_reset = devm_reset_control_get_exclusive(&pdev->dev, NULL);
 	if (!IS_ERR(mt7621_wdt_reset))
 		reset_control_deassert(mt7621_wdt_reset);
 
diff --git a/drivers/watchdog/octeon-wdt-main.c b/drivers/watchdog/octeon-wdt-main.c
index b5cdceb36cff..0ec419a3f7ed 100644
--- a/drivers/watchdog/octeon-wdt-main.c
+++ b/drivers/watchdog/octeon-wdt-main.c
@@ -1,7 +1,7 @@
 /*
  * Octeon Watchdog driver
  *
- * Copyright (C) 2007, 2008, 2009, 2010 Cavium Networks
+ * Copyright (C) 2007-2017 Cavium, Inc.
  *
  * Converted to use WATCHDOG_CORE by Aaro Koskinen <aaro.koskinen@iki.fi>.
  *
@@ -59,20 +59,23 @@
 #include <linux/interrupt.h>
 #include <linux/watchdog.h>
 #include <linux/cpumask.h>
-#include <linux/bitops.h>
-#include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/string.h>
 #include <linux/delay.h>
 #include <linux/cpu.h>
-#include <linux/smp.h>
-#include <linux/fs.h>
 #include <linux/irq.h>
 
 #include <asm/mipsregs.h>
 #include <asm/uasm.h>
 
 #include <asm/octeon/octeon.h>
+#include <asm/octeon/cvmx-boot-vector.h>
+#include <asm/octeon/cvmx-ciu2-defs.h>
+#include <asm/octeon/cvmx-rst-defs.h>
+
+/* Watchdog interrupt major block number (8 MSBs of intsn) */
+#define WD_BLOCK_NUMBER		0x01
+
+static int divisor;
 
 /* The count needed to achieve timeout_sec. */
 static unsigned int timeout_cnt;
@@ -84,7 +87,7 @@ static unsigned int max_timeout_sec;
 static unsigned int timeout_sec;
 
 /* Set to non-zero when userspace countdown mode active */
-static int do_coundown;
+static bool do_countdown;
 static unsigned int countdown_reset;
 static unsigned int per_cpu_countdown[NR_CPUS];
 
@@ -92,152 +95,38 @@ static cpumask_t irq_enabled_cpus;
 
 #define WD_TIMO 60			/* Default heartbeat = 60 seconds */
 
+#define CVMX_GSERX_SCRATCH(offset) (CVMX_ADD_IO_SEG(0x0001180090000020ull) + ((offset) & 15) * 0x1000000ull)
+
 static int heartbeat = WD_TIMO;
-module_param(heartbeat, int, S_IRUGO);
+module_param(heartbeat, int, 0444);
 MODULE_PARM_DESC(heartbeat,
 	"Watchdog heartbeat in seconds. (0 < heartbeat, default="
 				__MODULE_STRING(WD_TIMO) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
-module_param(nowayout, bool, S_IRUGO);
+module_param(nowayout, bool, 0444);
 MODULE_PARM_DESC(nowayout,
 	"Watchdog cannot be stopped once started (default="
 				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
 
-static u32 nmi_stage1_insns[64] __initdata;
-/* We need one branch and therefore one relocation per target label. */
-static struct uasm_label labels[5] __initdata;
-static struct uasm_reloc relocs[5] __initdata;
-
-enum lable_id {
-	label_enter_bootloader = 1
-};
+static int disable;
+module_param(disable, int, 0444);
+MODULE_PARM_DESC(disable,
+	"Disable the watchdog entirely (default=0)");
 
-/* Some CP0 registers */
-#define K0		26
-#define C0_CVMMEMCTL 11, 7
-#define C0_STATUS 12, 0
-#define C0_EBASE 15, 1
-#define C0_DESAVE 31, 0
+static struct cvmx_boot_vector_element *octeon_wdt_bootvector;
 
 void octeon_wdt_nmi_stage2(void);
 
-static void __init octeon_wdt_build_stage1(void)
-{
-	int i;
-	int len;
-	u32 *p = nmi_stage1_insns;
-#ifdef CONFIG_HOTPLUG_CPU
-	struct uasm_label *l = labels;
-	struct uasm_reloc *r = relocs;
-#endif
-
-	/*
-	 * For the next few instructions running the debugger may
-	 * cause corruption of k0 in the saved registers. Since we're
-	 * about to crash, nobody probably cares.
-	 *
-	 * Save K0 into the debug scratch register
-	 */
-	uasm_i_dmtc0(&p, K0, C0_DESAVE);
-
-	uasm_i_mfc0(&p, K0, C0_STATUS);
-#ifdef CONFIG_HOTPLUG_CPU
-	if (octeon_bootloader_entry_addr)
-		uasm_il_bbit0(&p, &r, K0, ilog2(ST0_NMI),
-			      label_enter_bootloader);
-#endif
-	/* Force 64-bit addressing enabled */
-	uasm_i_ori(&p, K0, K0, ST0_UX | ST0_SX | ST0_KX);
-	uasm_i_mtc0(&p, K0, C0_STATUS);
-
-#ifdef CONFIG_HOTPLUG_CPU
-	if (octeon_bootloader_entry_addr) {
-		uasm_i_mfc0(&p, K0, C0_EBASE);
-		/* Coreid number in K0 */
-		uasm_i_andi(&p, K0, K0, 0xf);
-		/* 8 * coreid in bits 16-31 */
-		uasm_i_dsll_safe(&p, K0, K0, 3 + 16);
-		uasm_i_ori(&p, K0, K0, 0x8001);
-		uasm_i_dsll_safe(&p, K0, K0, 16);
-		uasm_i_ori(&p, K0, K0, 0x0700);
-		uasm_i_drotr_safe(&p, K0, K0, 32);
-		/*
-		 * Should result in: 0x8001,0700,0000,8*coreid which is
-		 * CVMX_CIU_WDOGX(coreid) - 0x0500
-		 *
-		 * Now ld K0, CVMX_CIU_WDOGX(coreid)
-		 */
-		uasm_i_ld(&p, K0, 0x500, K0);
-		/*
-		 * If bit one set handle the NMI as a watchdog event.
-		 * otherwise transfer control to bootloader.
-		 */
-		uasm_il_bbit0(&p, &r, K0, 1, label_enter_bootloader);
-		uasm_i_nop(&p);
-	}
-#endif
-
-	/* Clear Dcache so cvmseg works right. */
-	uasm_i_cache(&p, 1, 0, 0);
-
-	/* Use K0 to do a read/modify/write of CVMMEMCTL */
-	uasm_i_dmfc0(&p, K0, C0_CVMMEMCTL);
-	/* Clear out the size of CVMSEG	*/
-	uasm_i_dins(&p, K0, 0, 0, 6);
-	/* Set CVMSEG to its largest value */
-	uasm_i_ori(&p, K0, K0, 0x1c0 | 54);
-	/* Store the CVMMEMCTL value */
-	uasm_i_dmtc0(&p, K0, C0_CVMMEMCTL);
-
-	/* Load the address of the second stage handler */
-	UASM_i_LA(&p, K0, (long)octeon_wdt_nmi_stage2);
-	uasm_i_jr(&p, K0);
-	uasm_i_dmfc0(&p, K0, C0_DESAVE);
-
-#ifdef CONFIG_HOTPLUG_CPU
-	if (octeon_bootloader_entry_addr) {
-		uasm_build_label(&l, p, label_enter_bootloader);
-		/* Jump to the bootloader and restore K0 */
-		UASM_i_LA(&p, K0, (long)octeon_bootloader_entry_addr);
-		uasm_i_jr(&p, K0);
-		uasm_i_dmfc0(&p, K0, C0_DESAVE);
-	}
-#endif
-	uasm_resolve_relocs(relocs, labels);
-
-	len = (int)(p - nmi_stage1_insns);
-	pr_debug("Synthesized NMI stage 1 handler (%d instructions)\n", len);
-
-	pr_debug("\t.set push\n");
-	pr_debug("\t.set noreorder\n");
-	for (i = 0; i < len; i++)
-		pr_debug("\t.word 0x%08x\n", nmi_stage1_insns[i]);
-	pr_debug("\t.set pop\n");
-
-	if (len > 32)
-		panic("NMI stage 1 handler exceeds 32 instructions, was %d\n",
-		      len);
-}
-
 static int cpu2core(int cpu)
 {
 #ifdef CONFIG_SMP
-	return cpu_logical_map(cpu);
+	return cpu_logical_map(cpu) & 0x3f;
 #else
 	return cvmx_get_core_num();
 #endif
 }
 
-static int core2cpu(int coreid)
-{
-#ifdef CONFIG_SMP
-	return cpu_number_map(coreid);
-#else
-	return 0;
-#endif
-}
-
 /**
  * Poke the watchdog when an interrupt is received
  *
@@ -248,13 +137,14 @@ static int core2cpu(int coreid)
  */
 static irqreturn_t octeon_wdt_poke_irq(int cpl, void *dev_id)
 {
-	unsigned int core = cvmx_get_core_num();
-	int cpu = core2cpu(core);
+	int cpu = raw_smp_processor_id();
+	unsigned int core = cpu2core(cpu);
+	int node = cpu_to_node(cpu);
 
-	if (do_coundown) {
+	if (do_countdown) {
 		if (per_cpu_countdown[cpu] > 0) {
 			/* We're alive, poke the watchdog */
-			cvmx_write_csr(CVMX_CIU_PP_POKEX(core), 1);
+			cvmx_write_csr_node(node, CVMX_CIU_PP_POKEX(core), 1);
 			per_cpu_countdown[cpu]--;
 		} else {
 			/* Bad news, you are about to reboot. */
@@ -263,7 +153,7 @@ static irqreturn_t octeon_wdt_poke_irq(int cpl, void *dev_id)
 		}
 	} else {
 		/* Not open, just ping away... */
-		cvmx_write_csr(CVMX_CIU_PP_POKEX(core), 1);
+		cvmx_write_csr_node(node, CVMX_CIU_PP_POKEX(core), 1);
 	}
 	return IRQ_HANDLED;
 }
@@ -338,10 +228,10 @@ void octeon_wdt_nmi_stage3(u64 reg[32])
 	u64 cp0_epc = read_c0_epc();
 
 	/* Delay so output from all cores output is not jumbled together. */
-	__delay(100000000ull * coreid);
+	udelay(85000 * coreid);
 
 	octeon_wdt_write_string("\r\n*** NMI Watchdog interrupt on Core 0x");
-	octeon_wdt_write_hex(coreid, 1);
+	octeon_wdt_write_hex(coreid, 2);
 	octeon_wdt_write_string(" ***\r\n");
 	for (i = 0; i < 32; i++) {
 		octeon_wdt_write_string("\t");
@@ -364,33 +254,98 @@ void octeon_wdt_nmi_stage3(u64 reg[32])
 	octeon_wdt_write_hex(cp0_cause, 16);
 	octeon_wdt_write_string("\r\n");
 
-	octeon_wdt_write_string("\tsum0\t0x");
-	octeon_wdt_write_hex(cvmx_read_csr(CVMX_CIU_INTX_SUM0(coreid * 2)), 16);
-	octeon_wdt_write_string("\ten0\t0x");
-	octeon_wdt_write_hex(cvmx_read_csr(CVMX_CIU_INTX_EN0(coreid * 2)), 16);
-	octeon_wdt_write_string("\r\n");
+	/* The CIU register is different for each Octeon model. */
+	if (OCTEON_IS_MODEL(OCTEON_CN68XX)) {
+		octeon_wdt_write_string("\tsrc_wd\t0x");
+		octeon_wdt_write_hex(cvmx_read_csr(CVMX_CIU2_SRC_PPX_IP2_WDOG(coreid)), 16);
+		octeon_wdt_write_string("\ten_wd\t0x");
+		octeon_wdt_write_hex(cvmx_read_csr(CVMX_CIU2_EN_PPX_IP2_WDOG(coreid)), 16);
+		octeon_wdt_write_string("\r\n");
+		octeon_wdt_write_string("\tsrc_rml\t0x");
+		octeon_wdt_write_hex(cvmx_read_csr(CVMX_CIU2_SRC_PPX_IP2_RML(coreid)), 16);
+		octeon_wdt_write_string("\ten_rml\t0x");
+		octeon_wdt_write_hex(cvmx_read_csr(CVMX_CIU2_EN_PPX_IP2_RML(coreid)), 16);
+		octeon_wdt_write_string("\r\n");
+		octeon_wdt_write_string("\tsum\t0x");
+		octeon_wdt_write_hex(cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP2(coreid)), 16);
+		octeon_wdt_write_string("\r\n");
+	} else if (!octeon_has_feature(OCTEON_FEATURE_CIU3)) {
+		octeon_wdt_write_string("\tsum0\t0x");
+		octeon_wdt_write_hex(cvmx_read_csr(CVMX_CIU_INTX_SUM0(coreid * 2)), 16);
+		octeon_wdt_write_string("\ten0\t0x");
+		octeon_wdt_write_hex(cvmx_read_csr(CVMX_CIU_INTX_EN0(coreid * 2)), 16);
+		octeon_wdt_write_string("\r\n");
+	}
 
 	octeon_wdt_write_string("*** Chip soft reset soon ***\r\n");
+
+	/*
+	 * G-30204: We must trigger a soft reset before watchdog
+	 * does an incomplete job of doing it.
+	 */
+	if (OCTEON_IS_OCTEON3() && !OCTEON_IS_MODEL(OCTEON_CN70XX)) {
+		u64 scr;
+		unsigned int node = cvmx_get_node_num();
+		unsigned int lcore = cvmx_get_local_core_num();
+		union cvmx_ciu_wdogx ciu_wdog;
+
+		/*
+		 * Wait for other cores to print out information, but
+		 * not too long.  Do the soft reset before watchdog
+		 * can trigger it.
+		 */
+		do {
+			ciu_wdog.u64 = cvmx_read_csr_node(node, CVMX_CIU_WDOGX(lcore));
+		} while (ciu_wdog.s.cnt > 0x10000);
+
+		scr = cvmx_read_csr_node(0, CVMX_GSERX_SCRATCH(0));
+		scr |= 1 << 11; /* Indicate watchdog in bit 11 */
+		cvmx_write_csr_node(0, CVMX_GSERX_SCRATCH(0), scr);
+		cvmx_write_csr_node(0, CVMX_RST_SOFT_RST, 1);
+	}
+}
+
+static int octeon_wdt_cpu_to_irq(int cpu)
+{
+	unsigned int coreid;
+	int node;
+	int irq;
+
+	coreid = cpu2core(cpu);
+	node = cpu_to_node(cpu);
+
+	if (octeon_has_feature(OCTEON_FEATURE_CIU3)) {
+		struct irq_domain *domain;
+		int hwirq;
+
+		domain = octeon_irq_get_block_domain(node,
+						     WD_BLOCK_NUMBER);
+		hwirq = WD_BLOCK_NUMBER << 12 | 0x200 | coreid;
+		irq = irq_find_mapping(domain, hwirq);
+	} else {
+		irq = OCTEON_IRQ_WDOG0 + coreid;
+	}
+	return irq;
 }
 
 static int octeon_wdt_cpu_pre_down(unsigned int cpu)
 {
 	unsigned int core;
-	unsigned int irq;
+	int node;
 	union cvmx_ciu_wdogx ciu_wdog;
 
 	core = cpu2core(cpu);
 
-	irq = OCTEON_IRQ_WDOG0 + core;
+	node = cpu_to_node(cpu);
 
 	/* Poke the watchdog to clear out its state */
-	cvmx_write_csr(CVMX_CIU_PP_POKEX(core), 1);
+	cvmx_write_csr_node(node, CVMX_CIU_PP_POKEX(core), 1);
 
 	/* Disable the hardware. */
 	ciu_wdog.u64 = 0;
-	cvmx_write_csr(CVMX_CIU_WDOGX(core), ciu_wdog.u64);
+	cvmx_write_csr_node(node, CVMX_CIU_WDOGX(core), ciu_wdog.u64);
 
-	free_irq(irq, octeon_wdt_poke_irq);
+	free_irq(octeon_wdt_cpu_to_irq(cpu), octeon_wdt_poke_irq);
 	return 0;
 }
 
@@ -399,31 +354,56 @@ static int octeon_wdt_cpu_online(unsigned int cpu)
 	unsigned int core;
 	unsigned int irq;
 	union cvmx_ciu_wdogx ciu_wdog;
+	int node;
+	struct irq_domain *domain;
+	int hwirq;
 
 	core = cpu2core(cpu);
+	node = cpu_to_node(cpu);
+
+	octeon_wdt_bootvector[core].target_ptr = (u64)octeon_wdt_nmi_stage2;
 
 	/* Disable it before doing anything with the interrupts. */
 	ciu_wdog.u64 = 0;
-	cvmx_write_csr(CVMX_CIU_WDOGX(core), ciu_wdog.u64);
+	cvmx_write_csr_node(node, CVMX_CIU_WDOGX(core), ciu_wdog.u64);
 
 	per_cpu_countdown[cpu] = countdown_reset;
 
-	irq = OCTEON_IRQ_WDOG0 + core;
+	if (octeon_has_feature(OCTEON_FEATURE_CIU3)) {
+		/* Must get the domain for the watchdog block */
+		domain = octeon_irq_get_block_domain(node, WD_BLOCK_NUMBER);
+
+		/* Get a irq for the wd intsn (hardware interrupt) */
+		hwirq = WD_BLOCK_NUMBER << 12 | 0x200 | core;
+		irq = irq_create_mapping(domain, hwirq);
+		irqd_set_trigger_type(irq_get_irq_data(irq),
+				      IRQ_TYPE_EDGE_RISING);
+	} else
+		irq = OCTEON_IRQ_WDOG0 + core;
 
 	if (request_irq(irq, octeon_wdt_poke_irq,
 			IRQF_NO_THREAD, "octeon_wdt", octeon_wdt_poke_irq))
 		panic("octeon_wdt: Couldn't obtain irq %d", irq);
 
+	/* Must set the irq affinity here */
+	if (octeon_has_feature(OCTEON_FEATURE_CIU3)) {
+		cpumask_t mask;
+
+		cpumask_clear(&mask);
+		cpumask_set_cpu(cpu, &mask);
+		irq_set_affinity(irq, &mask);
+	}
+
 	cpumask_set_cpu(cpu, &irq_enabled_cpus);
 
 	/* Poke the watchdog to clear out its state */
-	cvmx_write_csr(CVMX_CIU_PP_POKEX(core), 1);
+	cvmx_write_csr_node(node, CVMX_CIU_PP_POKEX(core), 1);
 
 	/* Finally enable the watchdog now that all handlers are installed */
 	ciu_wdog.u64 = 0;
 	ciu_wdog.s.len = timeout_cnt;
 	ciu_wdog.s.mode = 3;	/* 3 = Interrupt + NMI + Soft-Reset */
-	cvmx_write_csr(CVMX_CIU_WDOGX(core), ciu_wdog.u64);
+	cvmx_write_csr_node(node, CVMX_CIU_WDOGX(core), ciu_wdog.u64);
 
 	return 0;
 }
@@ -432,17 +412,20 @@ static int octeon_wdt_ping(struct watchdog_device __always_unused *wdog)
 {
 	int cpu;
 	int coreid;
+	int node;
+
+	if (disable)
+		return 0;
 
 	for_each_online_cpu(cpu) {
 		coreid = cpu2core(cpu);
-		cvmx_write_csr(CVMX_CIU_PP_POKEX(coreid), 1);
+		node = cpu_to_node(cpu);
+		cvmx_write_csr_node(node, CVMX_CIU_PP_POKEX(coreid), 1);
 		per_cpu_countdown[cpu] = countdown_reset;
-		if ((countdown_reset || !do_coundown) &&
+		if ((countdown_reset || !do_countdown) &&
 		    !cpumask_test_cpu(cpu, &irq_enabled_cpus)) {
 			/* We have to enable the irq */
-			int irq = OCTEON_IRQ_WDOG0 + coreid;
-
-			enable_irq(irq);
+			enable_irq(octeon_wdt_cpu_to_irq(cpu));
 			cpumask_set_cpu(cpu, &irq_enabled_cpus);
 		}
 	}
@@ -472,7 +455,7 @@ static void octeon_wdt_calc_parameters(int t)
 
 	countdown_reset = periods > 2 ? periods - 2 : 0;
 	heartbeat = t;
-	timeout_cnt = ((octeon_get_io_clock_rate() >> 8) * timeout_sec) >> 8;
+	timeout_cnt = ((octeon_get_io_clock_rate() / divisor) * timeout_sec) >> 8;
 }
 
 static int octeon_wdt_set_timeout(struct watchdog_device *wdog,
@@ -481,20 +464,25 @@ static int octeon_wdt_set_timeout(struct watchdog_device *wdog,
 	int cpu;
 	int coreid;
 	union cvmx_ciu_wdogx ciu_wdog;
+	int node;
 
 	if (t <= 0)
 		return -1;
 
 	octeon_wdt_calc_parameters(t);
 
+	if (disable)
+		return 0;
+
 	for_each_online_cpu(cpu) {
 		coreid = cpu2core(cpu);
-		cvmx_write_csr(CVMX_CIU_PP_POKEX(coreid), 1);
+		node = cpu_to_node(cpu);
+		cvmx_write_csr_node(node, CVMX_CIU_PP_POKEX(coreid), 1);
 		ciu_wdog.u64 = 0;
 		ciu_wdog.s.len = timeout_cnt;
 		ciu_wdog.s.mode = 3;	/* 3 = Interrupt + NMI + Soft-Reset */
-		cvmx_write_csr(CVMX_CIU_WDOGX(coreid), ciu_wdog.u64);
-		cvmx_write_csr(CVMX_CIU_PP_POKEX(coreid), 1);
+		cvmx_write_csr_node(node, CVMX_CIU_WDOGX(coreid), ciu_wdog.u64);
+		cvmx_write_csr_node(node, CVMX_CIU_PP_POKEX(coreid), 1);
 	}
 	octeon_wdt_ping(wdog); /* Get the irqs back on. */
 	return 0;
@@ -503,13 +491,13 @@ static int octeon_wdt_set_timeout(struct watchdog_device *wdog,
 static int octeon_wdt_start(struct watchdog_device *wdog)
 {
 	octeon_wdt_ping(wdog);
-	do_coundown = 1;
+	do_countdown = 1;
 	return 0;
 }
 
 static int octeon_wdt_stop(struct watchdog_device *wdog)
 {
-	do_coundown = 0;
+	do_countdown = 0;
 	octeon_wdt_ping(wdog);
 	return 0;
 }
@@ -540,14 +528,25 @@ static enum cpuhp_state octeon_wdt_online;
  */
 static int __init octeon_wdt_init(void)
 {
-	int i;
 	int ret;
-	u64 *ptr;
+
+	octeon_wdt_bootvector = cvmx_boot_vector_get();
+	if (!octeon_wdt_bootvector) {
+		pr_err("Error: Cannot allocate boot vector.\n");
+		return -ENOMEM;
+	}
+
+	if (OCTEON_IS_MODEL(OCTEON_CN68XX))
+		divisor = 0x200;
+	else if (OCTEON_IS_MODEL(OCTEON_CN78XX))
+		divisor = 0x400;
+	else
+		divisor = 0x100;
 
 	/*
 	 * Watchdog time expiration length = The 16 bits of LEN
 	 * represent the most significant bits of a 24 bit decrementer
-	 * that decrements every 256 cycles.
+	 * that decrements every divisor cycle.
 	 *
 	 * Try for a timeout of 5 sec, if that fails a smaller number
 	 * of even seconds,
@@ -555,8 +554,7 @@ static int __init octeon_wdt_init(void)
 	max_timeout_sec = 6;
 	do {
 		max_timeout_sec--;
-		timeout_cnt = ((octeon_get_io_clock_rate() >> 8) *
-			      max_timeout_sec) >> 8;
+		timeout_cnt = ((octeon_get_io_clock_rate() / divisor) * max_timeout_sec) >> 8;
 	} while (timeout_cnt > 65535);
 
 	BUG_ON(timeout_cnt == 0);
@@ -576,16 +574,10 @@ static int __init octeon_wdt_init(void)
 		return ret;
 	}
 
-	/* Build the NMI handler ... */
-	octeon_wdt_build_stage1();
-
-	/* ... and install it. */
-	ptr = (u64 *) nmi_stage1_insns;
-	for (i = 0; i < 16; i++) {
-		cvmx_write_csr(CVMX_MIO_BOOT_LOC_ADR, i * 8);
-		cvmx_write_csr(CVMX_MIO_BOOT_LOC_DAT, ptr[i]);
+	if (disable) {
+		pr_notice("disabled\n");
+		return 0;
 	}
-	cvmx_write_csr(CVMX_MIO_BOOT_LOC_CFGX(0), 0x81fc0000);
 
 	cpumask_clear(&irq_enabled_cpus);
 
@@ -607,6 +599,10 @@ err:
 static void __exit octeon_wdt_cleanup(void)
 {
 	watchdog_unregister_device(&octeon_wdt);
+
+	if (disable)
+		return;
+
 	cpuhp_remove_state(octeon_wdt_online);
 
 	/*
@@ -617,7 +613,7 @@ static void __exit octeon_wdt_cleanup(void)
 }
 
 MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Cavium Networks <support@caviumnetworks.com>");
-MODULE_DESCRIPTION("Cavium Networks Octeon Watchdog driver.");
+MODULE_AUTHOR("Cavium Inc. <support@cavium.com>");
+MODULE_DESCRIPTION("Cavium Inc. OCTEON Watchdog driver.");
 module_init(octeon_wdt_init);
 module_exit(octeon_wdt_cleanup);
diff --git a/drivers/watchdog/octeon-wdt-nmi.S b/drivers/watchdog/octeon-wdt-nmi.S
index 8a900a5e3233..97f6eb7b5a8e 100644
--- a/drivers/watchdog/octeon-wdt-nmi.S
+++ b/drivers/watchdog/octeon-wdt-nmi.S
@@ -3,20 +3,40 @@
  * License.  See the file "COPYING" in the main directory of this archive
  * for more details.
  *
- * Copyright (C) 2007 Cavium Networks
+ * Copyright (C) 2007-2017 Cavium, Inc.
  */
 #include <asm/asm.h>
 #include <asm/regdef.h>
 
-#define SAVE_REG(r)	sd $r, -32768+6912-(32-r)*8($0)
+#define CVMSEG_BASE	-32768
+#define CVMSEG_SIZE	6912
+#define SAVE_REG(r)	sd $r, CVMSEG_BASE + CVMSEG_SIZE - ((32 - r) * 8)($0)
 
         NESTED(octeon_wdt_nmi_stage2, 0, sp)
 	.set 	push
 	.set 	noreorder
 	.set 	noat
-	/* Save all registers to the top CVMSEG. This shouldn't
+	/* Clear Dcache so cvmseg works right. */
+	cache	1,0($0)
+	/* Use K0 to do a read/modify/write of CVMMEMCTL */
+	dmfc0	k0, $11, 7
+	/* Clear out the size of CVMSEG	*/
+	dins	k0, $0, 0, 6
+	/* Set CVMSEG to its largest value */
+	ori	k0, k0, 0x1c0 | 54
+	/* Store the CVMMEMCTL value */
+	dmtc0	k0, $11, 7
+	/*
+	 * Restore K0 from the debug scratch register, it was saved in
+	 * the boot-vector code.
+	 */
+	dmfc0	k0, $31
+
+	/*
+	 * Save all registers to the top CVMSEG. This shouldn't
 	 * corrupt any state used by the kernel. Also all registers
-	 * should have the value right before the NMI. */
+	 * should have the value right before the NMI.
+	 */
 	SAVE_REG(0)
 	SAVE_REG(1)
 	SAVE_REG(2)
@@ -49,16 +69,22 @@
 	SAVE_REG(29)
 	SAVE_REG(30)
 	SAVE_REG(31)
+	/* Write zero to all CVMSEG locations per Core-15169 */
+	dli	a0, CVMSEG_SIZE - (33 * 8)
+1:	sd	zero, CVMSEG_BASE(a0)
+	daddiu	a0, a0, -8
+	bgez	a0, 1b
+	nop
 	/* Set the stack to begin right below the registers */
-	li	sp, -32768+6912-32*8
+	dli	sp, CVMSEG_BASE + CVMSEG_SIZE - (32 * 8)
 	/* Load the address of the third stage handler */
-	dla	a0, octeon_wdt_nmi_stage3
+	dla	$25, octeon_wdt_nmi_stage3
 	/* Call the third stage handler */
-	jal	a0
+	jal	$25
 	/* a0 is the address of the saved registers */
 	 move	a0, sp
 	/* Loop forvever if we get here. */
-1:	b	1b
+2:	b	2b
 	nop
 	.set pop
 	END(octeon_wdt_nmi_stage2)
diff --git a/drivers/watchdog/of_xilinx_wdt.c b/drivers/watchdog/of_xilinx_wdt.c
index fae7fe929ea3..1cf286945b7a 100644
--- a/drivers/watchdog/of_xilinx_wdt.c
+++ b/drivers/watchdog/of_xilinx_wdt.c
@@ -51,9 +51,16 @@ struct xwdt_device {
 
 static int xilinx_wdt_start(struct watchdog_device *wdd)
 {
+	int ret;
 	u32 control_status_reg;
 	struct xwdt_device *xdev = watchdog_get_drvdata(wdd);
 
+	ret = clk_enable(xdev->clk);
+	if (ret) {
+		dev_err(wdd->parent, "Failed to enable clock\n");
+		return ret;
+	}
+
 	spin_lock(&xdev->spinlock);
 
 	/* Clean previous status and enable the watchdog timer */
@@ -85,6 +92,9 @@ static int xilinx_wdt_stop(struct watchdog_device *wdd)
 	iowrite32(0, xdev->base + XWT_TWCSR1_OFFSET);
 
 	spin_unlock(&xdev->spinlock);
+
+	clk_disable(xdev->clk);
+
 	pr_info("Stopped!\n");
 
 	return 0;
@@ -167,11 +177,6 @@ static int xwdt_probe(struct platform_device *pdev)
 	if (IS_ERR(xdev->base))
 		return PTR_ERR(xdev->base);
 
-	rc = of_property_read_u32(pdev->dev.of_node, "clock-frequency", &pfreq);
-	if (rc)
-		dev_warn(&pdev->dev,
-			 "The watchdog clock frequency cannot be obtained\n");
-
 	rc = of_property_read_u32(pdev->dev.of_node, "xlnx,wdt-interval",
 				  &xdev->wdt_interval);
 	if (rc)
@@ -186,6 +191,26 @@ static int xwdt_probe(struct platform_device *pdev)
 
 	watchdog_set_nowayout(xilinx_wdt_wdd, enable_once);
 
+	xdev->clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(xdev->clk)) {
+		if (PTR_ERR(xdev->clk) != -ENOENT)
+			return PTR_ERR(xdev->clk);
+
+		/*
+		 * Clock framework support is optional, continue on
+		 * anyways if we don't find a matching clock.
+		 */
+		xdev->clk = NULL;
+
+		rc = of_property_read_u32(pdev->dev.of_node, "clock-frequency",
+					  &pfreq);
+		if (rc)
+			dev_warn(&pdev->dev,
+				 "The watchdog clock freq cannot be obtained\n");
+	} else {
+		pfreq = clk_get_rate(xdev->clk);
+	}
+
 	/*
 	 * Twice of the 2^wdt_interval / freq  because the first wdt overflow is
 	 * ignored (interrupt), reset is only generated at second wdt overflow
@@ -197,14 +222,6 @@ static int xwdt_probe(struct platform_device *pdev)
 	spin_lock_init(&xdev->spinlock);
 	watchdog_set_drvdata(xilinx_wdt_wdd, xdev);
 
-	xdev->clk = devm_clk_get(&pdev->dev, NULL);
-	if (IS_ERR(xdev->clk)) {
-		if (PTR_ERR(xdev->clk) == -ENOENT)
-			xdev->clk = NULL;
-		else
-			return PTR_ERR(xdev->clk);
-	}
-
 	rc = clk_prepare_enable(xdev->clk);
 	if (rc) {
 		dev_err(&pdev->dev, "unable to enable clock\n");
@@ -223,6 +240,8 @@ static int xwdt_probe(struct platform_device *pdev)
 		goto err_clk_disable;
 	}
 
+	clk_disable(xdev->clk);
+
 	dev_info(&pdev->dev, "Xilinx Watchdog Timer at %p with timeout %ds\n",
 		 xdev->base, xilinx_wdt_wdd->timeout);
 
@@ -245,6 +264,43 @@ static int xwdt_remove(struct platform_device *pdev)
 	return 0;
 }
 
+/**
+ * xwdt_suspend - Suspend the device.
+ *
+ * @dev: handle to the device structure.
+ * Return: 0 always.
+ */
+static int __maybe_unused xwdt_suspend(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct xwdt_device *xdev = platform_get_drvdata(pdev);
+
+	if (watchdog_active(&xdev->xilinx_wdt_wdd))
+		xilinx_wdt_stop(&xdev->xilinx_wdt_wdd);
+
+	return 0;
+}
+
+/**
+ * xwdt_resume - Resume the device.
+ *
+ * @dev: handle to the device structure.
+ * Return: 0 on success, errno otherwise.
+ */
+static int __maybe_unused xwdt_resume(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct xwdt_device *xdev = platform_get_drvdata(pdev);
+	int ret = 0;
+
+	if (watchdog_active(&xdev->xilinx_wdt_wdd))
+		ret = xilinx_wdt_start(&xdev->xilinx_wdt_wdd);
+
+	return ret;
+}
+
+static SIMPLE_DEV_PM_OPS(xwdt_pm_ops, xwdt_suspend, xwdt_resume);
+
 /* Match table for of_platform binding */
 static const struct of_device_id xwdt_of_match[] = {
 	{ .compatible = "xlnx,xps-timebase-wdt-1.00.a", },
@@ -259,6 +315,7 @@ static struct platform_driver xwdt_driver = {
 	.driver = {
 		.name  = WATCHDOG_NAME,
 		.of_match_table = xwdt_of_match,
+		.pm = &xwdt_pm_ops,
 	},
 };
 
diff --git a/drivers/watchdog/pcwd_usb.c b/drivers/watchdog/pcwd_usb.c
index 5615f4013924..b9e376c8e2e3 100644
--- a/drivers/watchdog/pcwd_usb.c
+++ b/drivers/watchdog/pcwd_usb.c
@@ -74,7 +74,7 @@ MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
 #define USB_PCWD_PRODUCT_ID	0x1140
 
 /* table of devices that work with this driver */
-static struct usb_device_id usb_pcwd_table[] = {
+static const struct usb_device_id usb_pcwd_table[] = {
 	{ USB_DEVICE(USB_PCWD_VENDOR_ID, USB_PCWD_PRODUCT_ID) },
 	{ }					/* Terminating entry */
 };
diff --git a/drivers/watchdog/qcom-wdt.c b/drivers/watchdog/qcom-wdt.c
index 4f47b5e90956..780971318810 100644
--- a/drivers/watchdog/qcom-wdt.c
+++ b/drivers/watchdog/qcom-wdt.c
@@ -162,6 +162,8 @@ static int qcom_wdt_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res)
+		return -ENOMEM;
 
 	/* We use CPU0's DGT for the watchdog */
 	if (of_property_read_u32(np, "cpu-offset", &percpu_offset))
diff --git a/drivers/watchdog/renesas_wdt.c b/drivers/watchdog/renesas_wdt.c
index cf61c92f7ecd..831ef83f6de1 100644
--- a/drivers/watchdog/renesas_wdt.c
+++ b/drivers/watchdog/renesas_wdt.c
@@ -1,8 +1,8 @@
 /*
  * Watchdog driver for Renesas WDT watchdog
  *
- * Copyright (C) 2015-16 Wolfram Sang, Sang Engineering <wsa@sang-engineering.com>
- * Copyright (C) 2015-16 Renesas Electronics Corporation
+ * Copyright (C) 2015-17 Wolfram Sang, Sang Engineering <wsa@sang-engineering.com>
+ * Copyright (C) 2015-17 Renesas Electronics Corporation
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 as published by
@@ -23,10 +23,22 @@
 #define RWTCSRA_WOVF	BIT(4)
 #define RWTCSRA_WRFLG	BIT(5)
 #define RWTCSRA_TME	BIT(7)
+#define RWTCSRB		8
 
 #define RWDT_DEFAULT_TIMEOUT 60U
 
-static const unsigned int clk_divs[] = { 1, 4, 16, 32, 64, 128, 1024 };
+/*
+ * In probe, clk_rate is checked to be not more than 16 bit * biggest clock
+ * divider (12 bits). d is only a factor to fully utilize the WDT counter and
+ * will not exceed its 16 bits. Thus, no overflow, we stay below 32 bits.
+ */
+#define MUL_BY_CLKS_PER_SEC(p, d) \
+	DIV_ROUND_UP((d) * (p)->clk_rate, clk_divs[(p)->cks])
+
+/* d is 16 bit, clk_divs 12 bit -> no 32 bit overflow */
+#define DIV_BY_CLKS_PER_SEC(p, d) ((d) * clk_divs[(p)->cks] / (p)->clk_rate)
+
+static const unsigned int clk_divs[] = { 1, 4, 16, 32, 64, 128, 1024, 4096 };
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
@@ -36,8 +48,7 @@ MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
 struct rwdt_priv {
 	void __iomem *base;
 	struct watchdog_device wdev;
-	struct clk *clk;
-	unsigned int clks_per_sec;
+	unsigned long clk_rate;
 	u8 cks;
 };
 
@@ -55,7 +66,7 @@ static int rwdt_init_timeout(struct watchdog_device *wdev)
 {
 	struct rwdt_priv *priv = watchdog_get_drvdata(wdev);
 
-	rwdt_write(priv, 65536 - wdev->timeout * priv->clks_per_sec, RWTCNT);
+	rwdt_write(priv, 65536 - MUL_BY_CLKS_PER_SEC(priv, wdev->timeout), RWTCNT);
 
 	return 0;
 }
@@ -64,8 +75,9 @@ static int rwdt_start(struct watchdog_device *wdev)
 {
 	struct rwdt_priv *priv = watchdog_get_drvdata(wdev);
 
-	clk_prepare_enable(priv->clk);
+	pm_runtime_get_sync(wdev->parent);
 
+	rwdt_write(priv, 0, RWTCSRB);
 	rwdt_write(priv, priv->cks, RWTCSRA);
 	rwdt_init_timeout(wdev);
 
@@ -82,7 +94,7 @@ static int rwdt_stop(struct watchdog_device *wdev)
 	struct rwdt_priv *priv = watchdog_get_drvdata(wdev);
 
 	rwdt_write(priv, priv->cks, RWTCSRA);
-	clk_disable_unprepare(priv->clk);
+	pm_runtime_put(wdev->parent);
 
 	return 0;
 }
@@ -92,7 +104,7 @@ static unsigned int rwdt_get_timeleft(struct watchdog_device *wdev)
 	struct rwdt_priv *priv = watchdog_get_drvdata(wdev);
 	u16 val = readw_relaxed(priv->base + RWTCNT);
 
-	return DIV_ROUND_CLOSEST(65536 - val, priv->clks_per_sec);
+	return DIV_BY_CLKS_PER_SEC(priv, 65536 - val);
 }
 
 static const struct watchdog_info rwdt_ident = {
@@ -112,8 +124,8 @@ static int rwdt_probe(struct platform_device *pdev)
 {
 	struct rwdt_priv *priv;
 	struct resource *res;
-	unsigned long rate;
-	unsigned int clks_per_sec;
+	struct clk *clk;
+	unsigned long clks_per_sec;
 	int ret, i;
 
 	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
@@ -125,36 +137,40 @@ static int rwdt_probe(struct platform_device *pdev)
 	if (IS_ERR(priv->base))
 		return PTR_ERR(priv->base);
 
-	priv->clk = devm_clk_get(&pdev->dev, NULL);
-	if (IS_ERR(priv->clk))
-		return PTR_ERR(priv->clk);
+	clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(clk))
+		return PTR_ERR(clk);
+
+	pm_runtime_enable(&pdev->dev);
 
-	rate = clk_get_rate(priv->clk);
-	if (!rate)
-		return -ENOENT;
+	pm_runtime_get_sync(&pdev->dev);
+	priv->clk_rate = clk_get_rate(clk);
+	pm_runtime_put(&pdev->dev);
+
+	if (!priv->clk_rate) {
+		ret = -ENOENT;
+		goto out_pm_disable;
+	}
 
 	for (i = ARRAY_SIZE(clk_divs) - 1; i >= 0; i--) {
-		clks_per_sec = DIV_ROUND_UP(rate, clk_divs[i]);
-		if (clks_per_sec) {
-			priv->clks_per_sec = clks_per_sec;
+		clks_per_sec = priv->clk_rate / clk_divs[i];
+		if (clks_per_sec && clks_per_sec < 65536) {
 			priv->cks = i;
 			break;
 		}
 	}
 
-	if (!clks_per_sec) {
+	if (i < 0) {
 		dev_err(&pdev->dev, "Can't find suitable clock divider\n");
-		return -ERANGE;
+		ret = -ERANGE;
+		goto out_pm_disable;
 	}
 
-	pm_runtime_enable(&pdev->dev);
-	pm_runtime_get_sync(&pdev->dev);
-
 	priv->wdev.info = &rwdt_ident,
 	priv->wdev.ops = &rwdt_ops,
 	priv->wdev.parent = &pdev->dev;
 	priv->wdev.min_timeout = 1;
-	priv->wdev.max_timeout = 65536 / clks_per_sec;
+	priv->wdev.max_timeout = DIV_BY_CLKS_PER_SEC(priv, 65536);
 	priv->wdev.timeout = min(priv->wdev.max_timeout, RWDT_DEFAULT_TIMEOUT);
 
 	platform_set_drvdata(pdev, priv);
@@ -167,13 +183,14 @@ static int rwdt_probe(struct platform_device *pdev)
 		dev_warn(&pdev->dev, "Specified timeout value invalid, using default\n");
 
 	ret = watchdog_register_device(&priv->wdev);
-	if (ret < 0) {
-		pm_runtime_put(&pdev->dev);
-		pm_runtime_disable(&pdev->dev);
-		return ret;
-	}
+	if (ret < 0)
+		goto out_pm_disable;
 
 	return 0;
+
+ out_pm_disable:
+	pm_runtime_disable(&pdev->dev);
+	return ret;
 }
 
 static int rwdt_remove(struct platform_device *pdev)
@@ -181,7 +198,6 @@ static int rwdt_remove(struct platform_device *pdev)
 	struct rwdt_priv *priv = platform_get_drvdata(pdev);
 
 	watchdog_unregister_device(&priv->wdev);
-	pm_runtime_put(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
 
 	return 0;
diff --git a/drivers/watchdog/rt2880_wdt.c b/drivers/watchdog/rt2880_wdt.c
index 05524baf7dcc..98967f0a7d10 100644
--- a/drivers/watchdog/rt2880_wdt.c
+++ b/drivers/watchdog/rt2880_wdt.c
@@ -119,7 +119,7 @@ static int rt288x_wdt_bootcause(void)
 	return 0;
 }
 
-static struct watchdog_info rt288x_wdt_info = {
+static const struct watchdog_info rt288x_wdt_info = {
 	.identity = "Ralink Watchdog",
 	.options = WDIOF_SETTIMEOUT | WDIOF_KEEPALIVEPING | WDIOF_MAGICCLOSE,
 };
@@ -152,7 +152,7 @@ static int rt288x_wdt_probe(struct platform_device *pdev)
 	if (IS_ERR(rt288x_wdt_clk))
 		return PTR_ERR(rt288x_wdt_clk);
 
-	rt288x_wdt_reset = devm_reset_control_get(&pdev->dev, NULL);
+	rt288x_wdt_reset = devm_reset_control_get_exclusive(&pdev->dev, NULL);
 	if (!IS_ERR(rt288x_wdt_reset))
 		reset_control_deassert(rt288x_wdt_reset);
 
diff --git a/drivers/watchdog/sc1200wdt.c b/drivers/watchdog/sc1200wdt.c
index b34d3d5ba632..8e4e2fc13f87 100644
--- a/drivers/watchdog/sc1200wdt.c
+++ b/drivers/watchdog/sc1200wdt.c
@@ -342,7 +342,7 @@ static int __init sc1200wdt_probe(void)
 
 #if defined CONFIG_PNP
 
-static struct pnp_device_id scl200wdt_pnp_devices[] = {
+static const struct pnp_device_id scl200wdt_pnp_devices[] = {
 	/* National Semiconductor PC87307/PC97307 watchdog component */
 	{.id = "NSC0800", .driver_data = 0},
 	{.id = ""},
diff --git a/drivers/watchdog/sp805_wdt.c b/drivers/watchdog/sp805_wdt.c
index e7a715e82021..03805bc5d67a 100644
--- a/drivers/watchdog/sp805_wdt.c
+++ b/drivers/watchdog/sp805_wdt.c
@@ -281,7 +281,7 @@ static int __maybe_unused sp805_wdt_resume(struct device *dev)
 static SIMPLE_DEV_PM_OPS(sp805_wdt_dev_pm_ops, sp805_wdt_suspend,
 		sp805_wdt_resume);
 
-static struct amba_id sp805_wdt_ids[] = {
+static const struct amba_id sp805_wdt_ids[] = {
 	{
 		.id	= 0x00141805,
 		.mask	= 0x00ffffff,
diff --git a/drivers/watchdog/stm32_iwdg.c b/drivers/watchdog/stm32_iwdg.c
index 6c501b7dba29..be64a8699de3 100644
--- a/drivers/watchdog/stm32_iwdg.c
+++ b/drivers/watchdog/stm32_iwdg.c
@@ -140,7 +140,7 @@ static const struct watchdog_info stm32_iwdg_info = {
 	.identity	= "STM32 Independent Watchdog",
 };
 
-static struct watchdog_ops stm32_iwdg_ops = {
+static const struct watchdog_ops stm32_iwdg_ops = {
 	.owner		= THIS_MODULE,
 	.start		= stm32_iwdg_start,
 	.ping		= stm32_iwdg_ping,
diff --git a/drivers/watchdog/ts72xx_wdt.c b/drivers/watchdog/ts72xx_wdt.c
index 17c25daebcce..811e43c39ec4 100644
--- a/drivers/watchdog/ts72xx_wdt.c
+++ b/drivers/watchdog/ts72xx_wdt.c
@@ -112,7 +112,7 @@ static const struct watchdog_info ts72xx_wdt_ident = {
 	.identity		= "TS-72XX WDT",
 };
 
-static struct watchdog_ops ts72xx_wdt_ops = {
+static const struct watchdog_ops ts72xx_wdt_ops = {
 	.owner		= THIS_MODULE,
 	.start		= ts72xx_wdt_start,
 	.stop		= ts72xx_wdt_stop,
diff --git a/drivers/watchdog/twl4030_wdt.c b/drivers/watchdog/twl4030_wdt.c
index 9bf3cc0f3961..569fe85e52da 100644
--- a/drivers/watchdog/twl4030_wdt.c
+++ b/drivers/watchdog/twl4030_wdt.c
@@ -24,7 +24,7 @@
 #include <linux/kernel.h>
 #include <linux/watchdog.h>
 #include <linux/platform_device.h>
-#include <linux/i2c/twl.h>
+#include <linux/mfd/twl.h>
 
 #define TWL4030_WATCHDOG_CFG_REG_OFFS	0x3
 
diff --git a/drivers/watchdog/w83627hf_wdt.c b/drivers/watchdog/w83627hf_wdt.c
index d9ba0496713c..7817836bff55 100644
--- a/drivers/watchdog/w83627hf_wdt.c
+++ b/drivers/watchdog/w83627hf_wdt.c
@@ -429,7 +429,7 @@ static int __init wdt_init(void)
 {
 	int ret;
 	int chip;
-	const char * const chip_name[] = {
+	static const char * const chip_name[] = {
 		"W83627HF",
 		"W83627S",
 		"W83697HF",
diff --git a/drivers/watchdog/ziirave_wdt.c b/drivers/watchdog/ziirave_wdt.c
index b4e0cea5a64e..d3594aa3a374 100644
--- a/drivers/watchdog/ziirave_wdt.c
+++ b/drivers/watchdog/ziirave_wdt.c
@@ -737,7 +737,7 @@ static int ziirave_wdt_remove(struct i2c_client *client)
 	return 0;
 }
 
-static struct i2c_device_id ziirave_wdt_id[] = {
+static const struct i2c_device_id ziirave_wdt_id[] = {
 	{ "rave-wdt", 0 },
 	{ }
 };
diff --git a/drivers/watchdog/zx2967_wdt.c b/drivers/watchdog/zx2967_wdt.c
index 69ec5855584b..9261f7c77f6d 100644
--- a/drivers/watchdog/zx2967_wdt.c
+++ b/drivers/watchdog/zx2967_wdt.c
@@ -229,7 +229,7 @@ static int zx2967_wdt_probe(struct platform_device *pdev)
 	}
 	clk_set_rate(wdt->clock, ZX2967_WDT_CLK_FREQ);
 
-	rstc = devm_reset_control_get(dev, NULL);
+	rstc = devm_reset_control_get_exclusive(dev, NULL);
 	if (IS_ERR(rstc)) {
 		dev_err(dev, "failed to get rstc");
 		ret = PTR_ERR(rstc);
diff --git a/drivers/xen/Kconfig b/drivers/xen/Kconfig
index f15bb3b789d5..4545561954ee 100644
--- a/drivers/xen/Kconfig
+++ b/drivers/xen/Kconfig
@@ -196,6 +196,18 @@ config XEN_PCIDEV_BACKEND
 
 	  If in doubt, say m.
 
+config XEN_PVCALLS_BACKEND
+	bool "XEN PV Calls backend driver"
+	depends on INET && XEN && XEN_BACKEND
+	default n
+	help
+	  Experimental backend for the Xen PV Calls protocol
+	  (https://xenbits.xen.org/docs/unstable/misc/pvcalls.html). It
+	  allows PV Calls frontends to send POSIX calls to the backend,
+	  which implements them.
+
+	  If in doubt, say n.
+
 config XEN_SCSI_BACKEND
 	tristate "XEN SCSI backend driver"
 	depends on XEN && XEN_BACKEND && TARGET_CORE
diff --git a/drivers/xen/Makefile b/drivers/xen/Makefile
index 8feab810aed9..caaa15dc37bc 100644
--- a/drivers/xen/Makefile
+++ b/drivers/xen/Makefile
@@ -7,9 +7,6 @@ obj-y	+= xenbus/
 nostackp := $(call cc-option, -fno-stack-protector)
 CFLAGS_features.o			:= $(nostackp)
 
-CFLAGS_efi.o				+= -fshort-wchar
-LDFLAGS					+= $(call ld-option, --no-wchar-size-warning)
-
 dom0-$(CONFIG_ARM64) += arm-device.o
 dom0-$(CONFIG_PCI) += pci.o
 dom0-$(CONFIG_USB_SUPPORT) += dbgp.o
@@ -38,6 +35,7 @@ obj-$(CONFIG_XEN_ACPI_PROCESSOR)	+= xen-acpi-processor.o
 obj-$(CONFIG_XEN_EFI)			+= efi.o
 obj-$(CONFIG_XEN_SCSI_BACKEND)		+= xen-scsiback.o
 obj-$(CONFIG_XEN_AUTO_XLATE)		+= xlate_mmu.o
+obj-$(CONFIG_XEN_PVCALLS_BACKEND)	+= pvcalls-back.o
 xen-evtchn-y				:= evtchn.o
 xen-gntdev-y				:= gntdev.o
 xen-gntalloc-y				:= gntalloc.o
diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c
index ab609255a0f3..f77e499afddd 100644
--- a/drivers/xen/balloon.c
+++ b/drivers/xen/balloon.c
@@ -664,9 +664,11 @@ int alloc_xenballooned_pages(int nr_pages, struct page **pages)
 			 */
 			BUILD_BUG_ON(XEN_PAGE_SIZE != PAGE_SIZE);
 
-			ret = xen_alloc_p2m_entry(page_to_pfn(page));
-			if (ret < 0)
-				goto out_undo;
+			if (!xen_feature(XENFEAT_auto_translated_physmap)) {
+				ret = xen_alloc_p2m_entry(page_to_pfn(page));
+				if (ret < 0)
+					goto out_undo;
+			}
 #endif
 		} else {
 			ret = add_ballooned_pages(nr_pages - pgno);
diff --git a/drivers/xen/biomerge.c b/drivers/xen/biomerge.c
index 4da69dbf7dca..1bdd02a6d6ac 100644
--- a/drivers/xen/biomerge.c
+++ b/drivers/xen/biomerge.c
@@ -10,8 +10,7 @@ bool xen_biovec_phys_mergeable(const struct bio_vec *vec1,
 	unsigned long bfn1 = pfn_to_bfn(page_to_pfn(vec1->bv_page));
 	unsigned long bfn2 = pfn_to_bfn(page_to_pfn(vec2->bv_page));
 
-	return __BIOVEC_PHYS_MERGEABLE(vec1, vec2) &&
-		((bfn1 == bfn2) || ((bfn1+1) == bfn2));
+	return bfn1 + PFN_DOWN(vec1->bv_offset + vec1->bv_len) == bfn2;
 #else
 	/*
 	 * XXX: Add support for merging bio_vec when using different page
diff --git a/drivers/xen/events/events_base.c b/drivers/xen/events/events_base.c
index 2d43118077e4..1ab4bd11f5f3 100644
--- a/drivers/xen/events/events_base.c
+++ b/drivers/xen/events/events_base.c
@@ -1653,10 +1653,8 @@ void xen_callback_vector(void)
 			return;
 		}
 		pr_info("Xen HVM callback vector for event delivery is enabled\n");
-		/* in the restore case the vector has already been allocated */
-		if (!test_bit(HYPERVISOR_CALLBACK_VECTOR, used_vectors))
-			alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR,
-					xen_hvm_callback_vector);
+		alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR,
+				xen_hvm_callback_vector);
 	}
 }
 #else
diff --git a/drivers/xen/events/events_fifo.c b/drivers/xen/events/events_fifo.c
index 3c41470c7fc4..76b318e88382 100644
--- a/drivers/xen/events/events_fifo.c
+++ b/drivers/xen/events/events_fifo.c
@@ -432,12 +432,12 @@ static int xen_evtchn_cpu_dead(unsigned int cpu)
 
 int __init xen_evtchn_fifo_init(void)
 {
-	int cpu = get_cpu();
+	int cpu = smp_processor_id();
 	int ret;
 
 	ret = evtchn_fifo_alloc_control_block(cpu);
 	if (ret < 0)
-		goto out;
+		return ret;
 
 	pr_info("Using FIFO-based ABI\n");
 
@@ -446,7 +446,6 @@ int __init xen_evtchn_fifo_init(void)
 	cpuhp_setup_state_nocalls(CPUHP_XEN_EVTCHN_PREPARE,
 				  "xen/evtchn:prepare",
 				  xen_evtchn_cpu_prepare, xen_evtchn_cpu_dead);
-out:
-	put_cpu();
+
 	return ret;
 }
diff --git a/drivers/xen/gntalloc.c b/drivers/xen/gntalloc.c
index 1bf55a32a4b3..3fa40c723e8e 100644
--- a/drivers/xen/gntalloc.c
+++ b/drivers/xen/gntalloc.c
@@ -294,7 +294,7 @@ static long gntalloc_ioctl_alloc(struct gntalloc_file_private_data *priv,
 		goto out;
 	}
 
-	gref_ids = kcalloc(op.count, sizeof(gref_ids[0]), GFP_TEMPORARY);
+	gref_ids = kcalloc(op.count, sizeof(gref_ids[0]), GFP_KERNEL);
 	if (!gref_ids) {
 		rc = -ENOMEM;
 		goto out;
diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c
index f3bf8f4e2d6c..82360594fa8e 100644
--- a/drivers/xen/gntdev.c
+++ b/drivers/xen/gntdev.c
@@ -484,13 +484,6 @@ static void mn_invl_range_start(struct mmu_notifier *mn,
 	mutex_unlock(&priv->lock);
 }
 
-static void mn_invl_page(struct mmu_notifier *mn,
-			 struct mm_struct *mm,
-			 unsigned long address)
-{
-	mn_invl_range_start(mn, mm, address, address + PAGE_SIZE);
-}
-
 static void mn_release(struct mmu_notifier *mn,
 		       struct mm_struct *mm)
 {
@@ -522,7 +515,6 @@ static void mn_release(struct mmu_notifier *mn,
 
 static const struct mmu_notifier_ops gntdev_mmu_ops = {
 	.release                = mn_release,
-	.invalidate_page        = mn_invl_page,
 	.invalidate_range_start = mn_invl_range_start,
 };
 
diff --git a/drivers/xen/platform-pci.c b/drivers/xen/platform-pci.c
index 1275df83070f..5d7dcad0b0a0 100644
--- a/drivers/xen/platform-pci.c
+++ b/drivers/xen/platform-pci.c
@@ -175,7 +175,7 @@ pci_out:
 	return ret;
 }
 
-static struct pci_device_id platform_pci_tbl[] = {
+static const struct pci_device_id platform_pci_tbl[] = {
 	{PCI_VENDOR_ID_XEN, PCI_DEVICE_ID_XEN_PLATFORM,
 		PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
 	{0,}
diff --git a/drivers/xen/pvcalls-back.c b/drivers/xen/pvcalls-back.c
new file mode 100644
index 000000000000..b209cd44bb8d
--- /dev/null
+++ b/drivers/xen/pvcalls-back.c
@@ -0,0 +1,1240 @@
+/*
+ * (c) 2017 Stefano Stabellini <stefano@aporeto.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/inet.h>
+#include <linux/kthread.h>
+#include <linux/list.h>
+#include <linux/radix-tree.h>
+#include <linux/module.h>
+#include <linux/semaphore.h>
+#include <linux/wait.h>
+#include <net/sock.h>
+#include <net/inet_common.h>
+#include <net/inet_connection_sock.h>
+#include <net/request_sock.h>
+
+#include <xen/events.h>
+#include <xen/grant_table.h>
+#include <xen/xen.h>
+#include <xen/xenbus.h>
+#include <xen/interface/io/pvcalls.h>
+
+#define PVCALLS_VERSIONS "1"
+#define MAX_RING_ORDER XENBUS_MAX_RING_GRANT_ORDER
+
+struct pvcalls_back_global {
+	struct list_head frontends;
+	struct semaphore frontends_lock;
+} pvcalls_back_global;
+
+/*
+ * Per-frontend data structure. It contains pointers to the command
+ * ring, its event channel, a list of active sockets and a tree of
+ * passive sockets.
+ */
+struct pvcalls_fedata {
+	struct list_head list;
+	struct xenbus_device *dev;
+	struct xen_pvcalls_sring *sring;
+	struct xen_pvcalls_back_ring ring;
+	int irq;
+	struct list_head socket_mappings;
+	struct radix_tree_root socketpass_mappings;
+	struct semaphore socket_lock;
+};
+
+struct pvcalls_ioworker {
+	struct work_struct register_work;
+	struct workqueue_struct *wq;
+};
+
+struct sock_mapping {
+	struct list_head list;
+	struct pvcalls_fedata *fedata;
+	struct sockpass_mapping *sockpass;
+	struct socket *sock;
+	uint64_t id;
+	grant_ref_t ref;
+	struct pvcalls_data_intf *ring;
+	void *bytes;
+	struct pvcalls_data data;
+	uint32_t ring_order;
+	int irq;
+	atomic_t read;
+	atomic_t write;
+	atomic_t io;
+	atomic_t release;
+	void (*saved_data_ready)(struct sock *sk);
+	struct pvcalls_ioworker ioworker;
+};
+
+struct sockpass_mapping {
+	struct list_head list;
+	struct pvcalls_fedata *fedata;
+	struct socket *sock;
+	uint64_t id;
+	struct xen_pvcalls_request reqcopy;
+	spinlock_t copy_lock;
+	struct workqueue_struct *wq;
+	struct work_struct register_work;
+	void (*saved_data_ready)(struct sock *sk);
+};
+
+static irqreturn_t pvcalls_back_conn_event(int irq, void *sock_map);
+static int pvcalls_back_release_active(struct xenbus_device *dev,
+				       struct pvcalls_fedata *fedata,
+				       struct sock_mapping *map);
+
+static void pvcalls_conn_back_read(void *opaque)
+{
+	struct sock_mapping *map = (struct sock_mapping *)opaque;
+	struct msghdr msg;
+	struct kvec vec[2];
+	RING_IDX cons, prod, size, wanted, array_size, masked_prod, masked_cons;
+	int32_t error;
+	struct pvcalls_data_intf *intf = map->ring;
+	struct pvcalls_data *data = &map->data;
+	unsigned long flags;
+	int ret;
+
+	array_size = XEN_FLEX_RING_SIZE(map->ring_order);
+	cons = intf->in_cons;
+	prod = intf->in_prod;
+	error = intf->in_error;
+	/* read the indexes first, then deal with the data */
+	virt_mb();
+
+	if (error)
+		return;
+
+	size = pvcalls_queued(prod, cons, array_size);
+	if (size >= array_size)
+		return;
+	spin_lock_irqsave(&map->sock->sk->sk_receive_queue.lock, flags);
+	if (skb_queue_empty(&map->sock->sk->sk_receive_queue)) {
+		atomic_set(&map->read, 0);
+		spin_unlock_irqrestore(&map->sock->sk->sk_receive_queue.lock,
+				flags);
+		return;
+	}
+	spin_unlock_irqrestore(&map->sock->sk->sk_receive_queue.lock, flags);
+	wanted = array_size - size;
+	masked_prod = pvcalls_mask(prod, array_size);
+	masked_cons = pvcalls_mask(cons, array_size);
+
+	memset(&msg, 0, sizeof(msg));
+	msg.msg_iter.type = ITER_KVEC|WRITE;
+	msg.msg_iter.count = wanted;
+	if (masked_prod < masked_cons) {
+		vec[0].iov_base = data->in + masked_prod;
+		vec[0].iov_len = wanted;
+		msg.msg_iter.kvec = vec;
+		msg.msg_iter.nr_segs = 1;
+	} else {
+		vec[0].iov_base = data->in + masked_prod;
+		vec[0].iov_len = array_size - masked_prod;
+		vec[1].iov_base = data->in;
+		vec[1].iov_len = wanted - vec[0].iov_len;
+		msg.msg_iter.kvec = vec;
+		msg.msg_iter.nr_segs = 2;
+	}
+
+	atomic_set(&map->read, 0);
+	ret = inet_recvmsg(map->sock, &msg, wanted, MSG_DONTWAIT);
+	WARN_ON(ret > wanted);
+	if (ret == -EAGAIN) /* shouldn't happen */
+		return;
+	if (!ret)
+		ret = -ENOTCONN;
+	spin_lock_irqsave(&map->sock->sk->sk_receive_queue.lock, flags);
+	if (ret > 0 && !skb_queue_empty(&map->sock->sk->sk_receive_queue))
+		atomic_inc(&map->read);
+	spin_unlock_irqrestore(&map->sock->sk->sk_receive_queue.lock, flags);
+
+	/* write the data, then modify the indexes */
+	virt_wmb();
+	if (ret < 0)
+		intf->in_error = ret;
+	else
+		intf->in_prod = prod + ret;
+	/* update the indexes, then notify the other end */
+	virt_wmb();
+	notify_remote_via_irq(map->irq);
+
+	return;
+}
+
+static void pvcalls_conn_back_write(struct sock_mapping *map)
+{
+	struct pvcalls_data_intf *intf = map->ring;
+	struct pvcalls_data *data = &map->data;
+	struct msghdr msg;
+	struct kvec vec[2];
+	RING_IDX cons, prod, size, array_size;
+	int ret;
+
+	cons = intf->out_cons;
+	prod = intf->out_prod;
+	/* read the indexes before dealing with the data */
+	virt_mb();
+
+	array_size = XEN_FLEX_RING_SIZE(map->ring_order);
+	size = pvcalls_queued(prod, cons, array_size);
+	if (size == 0)
+		return;
+
+	memset(&msg, 0, sizeof(msg));
+	msg.msg_flags |= MSG_DONTWAIT;
+	msg.msg_iter.type = ITER_KVEC|READ;
+	msg.msg_iter.count = size;
+	if (pvcalls_mask(prod, array_size) > pvcalls_mask(cons, array_size)) {
+		vec[0].iov_base = data->out + pvcalls_mask(cons, array_size);
+		vec[0].iov_len = size;
+		msg.msg_iter.kvec = vec;
+		msg.msg_iter.nr_segs = 1;
+	} else {
+		vec[0].iov_base = data->out + pvcalls_mask(cons, array_size);
+		vec[0].iov_len = array_size - pvcalls_mask(cons, array_size);
+		vec[1].iov_base = data->out;
+		vec[1].iov_len = size - vec[0].iov_len;
+		msg.msg_iter.kvec = vec;
+		msg.msg_iter.nr_segs = 2;
+	}
+
+	atomic_set(&map->write, 0);
+	ret = inet_sendmsg(map->sock, &msg, size);
+	if (ret == -EAGAIN || (ret >= 0 && ret < size)) {
+		atomic_inc(&map->write);
+		atomic_inc(&map->io);
+	}
+	if (ret == -EAGAIN)
+		return;
+
+	/* write the data, then update the indexes */
+	virt_wmb();
+	if (ret < 0) {
+		intf->out_error = ret;
+	} else {
+		intf->out_error = 0;
+		intf->out_cons = cons + ret;
+		prod = intf->out_prod;
+	}
+	/* update the indexes, then notify the other end */
+	virt_wmb();
+	if (prod != cons + ret)
+		atomic_inc(&map->write);
+	notify_remote_via_irq(map->irq);
+}
+
+static void pvcalls_back_ioworker(struct work_struct *work)
+{
+	struct pvcalls_ioworker *ioworker = container_of(work,
+		struct pvcalls_ioworker, register_work);
+	struct sock_mapping *map = container_of(ioworker, struct sock_mapping,
+		ioworker);
+
+	while (atomic_read(&map->io) > 0) {
+		if (atomic_read(&map->release) > 0) {
+			atomic_set(&map->release, 0);
+			return;
+		}
+
+		if (atomic_read(&map->read) > 0)
+			pvcalls_conn_back_read(map);
+		if (atomic_read(&map->write) > 0)
+			pvcalls_conn_back_write(map);
+
+		atomic_dec(&map->io);
+	}
+}
+
+static int pvcalls_back_socket(struct xenbus_device *dev,
+		struct xen_pvcalls_request *req)
+{
+	struct pvcalls_fedata *fedata;
+	int ret;
+	struct xen_pvcalls_response *rsp;
+
+	fedata = dev_get_drvdata(&dev->dev);
+
+	if (req->u.socket.domain != AF_INET ||
+	    req->u.socket.type != SOCK_STREAM ||
+	    (req->u.socket.protocol != IPPROTO_IP &&
+	     req->u.socket.protocol != AF_INET))
+		ret = -EAFNOSUPPORT;
+	else
+		ret = 0;
+
+	/* leave the actual socket allocation for later */
+
+	rsp = RING_GET_RESPONSE(&fedata->ring, fedata->ring.rsp_prod_pvt++);
+	rsp->req_id = req->req_id;
+	rsp->cmd = req->cmd;
+	rsp->u.socket.id = req->u.socket.id;
+	rsp->ret = ret;
+
+	return 0;
+}
+
+static void pvcalls_sk_state_change(struct sock *sock)
+{
+	struct sock_mapping *map = sock->sk_user_data;
+	struct pvcalls_data_intf *intf;
+
+	if (map == NULL)
+		return;
+
+	intf = map->ring;
+	intf->in_error = -ENOTCONN;
+	notify_remote_via_irq(map->irq);
+}
+
+static void pvcalls_sk_data_ready(struct sock *sock)
+{
+	struct sock_mapping *map = sock->sk_user_data;
+	struct pvcalls_ioworker *iow;
+
+	if (map == NULL)
+		return;
+
+	iow = &map->ioworker;
+	atomic_inc(&map->read);
+	atomic_inc(&map->io);
+	queue_work(iow->wq, &iow->register_work);
+}
+
+static struct sock_mapping *pvcalls_new_active_socket(
+		struct pvcalls_fedata *fedata,
+		uint64_t id,
+		grant_ref_t ref,
+		uint32_t evtchn,
+		struct socket *sock)
+{
+	int ret;
+	struct sock_mapping *map;
+	void *page;
+
+	map = kzalloc(sizeof(*map), GFP_KERNEL);
+	if (map == NULL)
+		return NULL;
+
+	map->fedata = fedata;
+	map->sock = sock;
+	map->id = id;
+	map->ref = ref;
+
+	ret = xenbus_map_ring_valloc(fedata->dev, &ref, 1, &page);
+	if (ret < 0)
+		goto out;
+	map->ring = page;
+	map->ring_order = map->ring->ring_order;
+	/* first read the order, then map the data ring */
+	virt_rmb();
+	if (map->ring_order > MAX_RING_ORDER) {
+		pr_warn("%s frontend requested ring_order %u, which is > MAX (%u)\n",
+				__func__, map->ring_order, MAX_RING_ORDER);
+		goto out;
+	}
+	ret = xenbus_map_ring_valloc(fedata->dev, map->ring->ref,
+				     (1 << map->ring_order), &page);
+	if (ret < 0)
+		goto out;
+	map->bytes = page;
+
+	ret = bind_interdomain_evtchn_to_irqhandler(fedata->dev->otherend_id,
+						    evtchn,
+						    pvcalls_back_conn_event,
+						    0,
+						    "pvcalls-backend",
+						    map);
+	if (ret < 0)
+		goto out;
+	map->irq = ret;
+
+	map->data.in = map->bytes;
+	map->data.out = map->bytes + XEN_FLEX_RING_SIZE(map->ring_order);
+
+	map->ioworker.wq = alloc_workqueue("pvcalls_io", WQ_UNBOUND, 1);
+	if (!map->ioworker.wq)
+		goto out;
+	atomic_set(&map->io, 1);
+	INIT_WORK(&map->ioworker.register_work,	pvcalls_back_ioworker);
+
+	down(&fedata->socket_lock);
+	list_add_tail(&map->list, &fedata->socket_mappings);
+	up(&fedata->socket_lock);
+
+	write_lock_bh(&map->sock->sk->sk_callback_lock);
+	map->saved_data_ready = map->sock->sk->sk_data_ready;
+	map->sock->sk->sk_user_data = map;
+	map->sock->sk->sk_data_ready = pvcalls_sk_data_ready;
+	map->sock->sk->sk_state_change = pvcalls_sk_state_change;
+	write_unlock_bh(&map->sock->sk->sk_callback_lock);
+
+	return map;
+out:
+	down(&fedata->socket_lock);
+	list_del(&map->list);
+	pvcalls_back_release_active(fedata->dev, fedata, map);
+	up(&fedata->socket_lock);
+	return NULL;
+}
+
+static int pvcalls_back_connect(struct xenbus_device *dev,
+				struct xen_pvcalls_request *req)
+{
+	struct pvcalls_fedata *fedata;
+	int ret = -EINVAL;
+	struct socket *sock;
+	struct sock_mapping *map;
+	struct xen_pvcalls_response *rsp;
+	struct sockaddr *sa = (struct sockaddr *)&req->u.connect.addr;
+
+	fedata = dev_get_drvdata(&dev->dev);
+
+	if (req->u.connect.len < sizeof(sa->sa_family) ||
+	    req->u.connect.len > sizeof(req->u.connect.addr) ||
+	    sa->sa_family != AF_INET)
+		goto out;
+
+	ret = sock_create(AF_INET, SOCK_STREAM, 0, &sock);
+	if (ret < 0)
+		goto out;
+	ret = inet_stream_connect(sock, sa, req->u.connect.len, 0);
+	if (ret < 0) {
+		sock_release(sock);
+		goto out;
+	}
+
+	map = pvcalls_new_active_socket(fedata,
+					req->u.connect.id,
+					req->u.connect.ref,
+					req->u.connect.evtchn,
+					sock);
+	if (!map) {
+		ret = -EFAULT;
+		sock_release(map->sock);
+	}
+
+out:
+	rsp = RING_GET_RESPONSE(&fedata->ring, fedata->ring.rsp_prod_pvt++);
+	rsp->req_id = req->req_id;
+	rsp->cmd = req->cmd;
+	rsp->u.connect.id = req->u.connect.id;
+	rsp->ret = ret;
+
+	return 0;
+}
+
+static int pvcalls_back_release_active(struct xenbus_device *dev,
+				       struct pvcalls_fedata *fedata,
+				       struct sock_mapping *map)
+{
+	disable_irq(map->irq);
+	if (map->sock->sk != NULL) {
+		write_lock_bh(&map->sock->sk->sk_callback_lock);
+		map->sock->sk->sk_user_data = NULL;
+		map->sock->sk->sk_data_ready = map->saved_data_ready;
+		write_unlock_bh(&map->sock->sk->sk_callback_lock);
+	}
+
+	atomic_set(&map->release, 1);
+	flush_work(&map->ioworker.register_work);
+
+	xenbus_unmap_ring_vfree(dev, map->bytes);
+	xenbus_unmap_ring_vfree(dev, (void *)map->ring);
+	unbind_from_irqhandler(map->irq, map);
+
+	sock_release(map->sock);
+	kfree(map);
+
+	return 0;
+}
+
+static int pvcalls_back_release_passive(struct xenbus_device *dev,
+					struct pvcalls_fedata *fedata,
+					struct sockpass_mapping *mappass)
+{
+	if (mappass->sock->sk != NULL) {
+		write_lock_bh(&mappass->sock->sk->sk_callback_lock);
+		mappass->sock->sk->sk_user_data = NULL;
+		mappass->sock->sk->sk_data_ready = mappass->saved_data_ready;
+		write_unlock_bh(&mappass->sock->sk->sk_callback_lock);
+	}
+	sock_release(mappass->sock);
+	flush_workqueue(mappass->wq);
+	destroy_workqueue(mappass->wq);
+	kfree(mappass);
+
+	return 0;
+}
+
+static int pvcalls_back_release(struct xenbus_device *dev,
+				struct xen_pvcalls_request *req)
+{
+	struct pvcalls_fedata *fedata;
+	struct sock_mapping *map, *n;
+	struct sockpass_mapping *mappass;
+	int ret = 0;
+	struct xen_pvcalls_response *rsp;
+
+	fedata = dev_get_drvdata(&dev->dev);
+
+	down(&fedata->socket_lock);
+	list_for_each_entry_safe(map, n, &fedata->socket_mappings, list) {
+		if (map->id == req->u.release.id) {
+			list_del(&map->list);
+			up(&fedata->socket_lock);
+			ret = pvcalls_back_release_active(dev, fedata, map);
+			goto out;
+		}
+	}
+	mappass = radix_tree_lookup(&fedata->socketpass_mappings,
+				    req->u.release.id);
+	if (mappass != NULL) {
+		radix_tree_delete(&fedata->socketpass_mappings, mappass->id);
+		up(&fedata->socket_lock);
+		ret = pvcalls_back_release_passive(dev, fedata, mappass);
+	} else
+		up(&fedata->socket_lock);
+
+out:
+	rsp = RING_GET_RESPONSE(&fedata->ring, fedata->ring.rsp_prod_pvt++);
+	rsp->req_id = req->req_id;
+	rsp->u.release.id = req->u.release.id;
+	rsp->cmd = req->cmd;
+	rsp->ret = ret;
+	return 0;
+}
+
+static void __pvcalls_back_accept(struct work_struct *work)
+{
+	struct sockpass_mapping *mappass = container_of(
+		work, struct sockpass_mapping, register_work);
+	struct sock_mapping *map;
+	struct pvcalls_ioworker *iow;
+	struct pvcalls_fedata *fedata;
+	struct socket *sock;
+	struct xen_pvcalls_response *rsp;
+	struct xen_pvcalls_request *req;
+	int notify;
+	int ret = -EINVAL;
+	unsigned long flags;
+
+	fedata = mappass->fedata;
+	/*
+	 * __pvcalls_back_accept can race against pvcalls_back_accept.
+	 * We only need to check the value of "cmd" on read. It could be
+	 * done atomically, but to simplify the code on the write side, we
+	 * use a spinlock.
+	 */
+	spin_lock_irqsave(&mappass->copy_lock, flags);
+	req = &mappass->reqcopy;
+	if (req->cmd != PVCALLS_ACCEPT) {
+		spin_unlock_irqrestore(&mappass->copy_lock, flags);
+		return;
+	}
+	spin_unlock_irqrestore(&mappass->copy_lock, flags);
+
+	sock = sock_alloc();
+	if (sock == NULL)
+		goto out_error;
+	sock->type = mappass->sock->type;
+	sock->ops = mappass->sock->ops;
+
+	ret = inet_accept(mappass->sock, sock, O_NONBLOCK, true);
+	if (ret == -EAGAIN) {
+		sock_release(sock);
+		goto out_error;
+	}
+
+	map = pvcalls_new_active_socket(fedata,
+					req->u.accept.id_new,
+					req->u.accept.ref,
+					req->u.accept.evtchn,
+					sock);
+	if (!map) {
+		ret = -EFAULT;
+		sock_release(sock);
+		goto out_error;
+	}
+
+	map->sockpass = mappass;
+	iow = &map->ioworker;
+	atomic_inc(&map->read);
+	atomic_inc(&map->io);
+	queue_work(iow->wq, &iow->register_work);
+
+out_error:
+	rsp = RING_GET_RESPONSE(&fedata->ring, fedata->ring.rsp_prod_pvt++);
+	rsp->req_id = req->req_id;
+	rsp->cmd = req->cmd;
+	rsp->u.accept.id = req->u.accept.id;
+	rsp->ret = ret;
+	RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&fedata->ring, notify);
+	if (notify)
+		notify_remote_via_irq(fedata->irq);
+
+	mappass->reqcopy.cmd = 0;
+}
+
+static void pvcalls_pass_sk_data_ready(struct sock *sock)
+{
+	struct sockpass_mapping *mappass = sock->sk_user_data;
+	struct pvcalls_fedata *fedata;
+	struct xen_pvcalls_response *rsp;
+	unsigned long flags;
+	int notify;
+
+	if (mappass == NULL)
+		return;
+
+	fedata = mappass->fedata;
+	spin_lock_irqsave(&mappass->copy_lock, flags);
+	if (mappass->reqcopy.cmd == PVCALLS_POLL) {
+		rsp = RING_GET_RESPONSE(&fedata->ring,
+					fedata->ring.rsp_prod_pvt++);
+		rsp->req_id = mappass->reqcopy.req_id;
+		rsp->u.poll.id = mappass->reqcopy.u.poll.id;
+		rsp->cmd = mappass->reqcopy.cmd;
+		rsp->ret = 0;
+
+		mappass->reqcopy.cmd = 0;
+		spin_unlock_irqrestore(&mappass->copy_lock, flags);
+
+		RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&fedata->ring, notify);
+		if (notify)
+			notify_remote_via_irq(mappass->fedata->irq);
+	} else {
+		spin_unlock_irqrestore(&mappass->copy_lock, flags);
+		queue_work(mappass->wq, &mappass->register_work);
+	}
+}
+
+static int pvcalls_back_bind(struct xenbus_device *dev,
+			     struct xen_pvcalls_request *req)
+{
+	struct pvcalls_fedata *fedata;
+	int ret;
+	struct sockpass_mapping *map;
+	struct xen_pvcalls_response *rsp;
+
+	fedata = dev_get_drvdata(&dev->dev);
+
+	map = kzalloc(sizeof(*map), GFP_KERNEL);
+	if (map == NULL) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	INIT_WORK(&map->register_work, __pvcalls_back_accept);
+	spin_lock_init(&map->copy_lock);
+	map->wq = alloc_workqueue("pvcalls_wq", WQ_UNBOUND, 1);
+	if (!map->wq) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	ret = sock_create(AF_INET, SOCK_STREAM, 0, &map->sock);
+	if (ret < 0)
+		goto out;
+
+	ret = inet_bind(map->sock, (struct sockaddr *)&req->u.bind.addr,
+			req->u.bind.len);
+	if (ret < 0)
+		goto out;
+
+	map->fedata = fedata;
+	map->id = req->u.bind.id;
+
+	down(&fedata->socket_lock);
+	ret = radix_tree_insert(&fedata->socketpass_mappings, map->id,
+				map);
+	up(&fedata->socket_lock);
+	if (ret)
+		goto out;
+
+	write_lock_bh(&map->sock->sk->sk_callback_lock);
+	map->saved_data_ready = map->sock->sk->sk_data_ready;
+	map->sock->sk->sk_user_data = map;
+	map->sock->sk->sk_data_ready = pvcalls_pass_sk_data_ready;
+	write_unlock_bh(&map->sock->sk->sk_callback_lock);
+
+out:
+	if (ret) {
+		if (map && map->sock)
+			sock_release(map->sock);
+		if (map && map->wq)
+			destroy_workqueue(map->wq);
+		kfree(map);
+	}
+	rsp = RING_GET_RESPONSE(&fedata->ring, fedata->ring.rsp_prod_pvt++);
+	rsp->req_id = req->req_id;
+	rsp->cmd = req->cmd;
+	rsp->u.bind.id = req->u.bind.id;
+	rsp->ret = ret;
+	return 0;
+}
+
+static int pvcalls_back_listen(struct xenbus_device *dev,
+			       struct xen_pvcalls_request *req)
+{
+	struct pvcalls_fedata *fedata;
+	int ret = -EINVAL;
+	struct sockpass_mapping *map;
+	struct xen_pvcalls_response *rsp;
+
+	fedata = dev_get_drvdata(&dev->dev);
+
+	down(&fedata->socket_lock);
+	map = radix_tree_lookup(&fedata->socketpass_mappings, req->u.listen.id);
+	up(&fedata->socket_lock);
+	if (map == NULL)
+		goto out;
+
+	ret = inet_listen(map->sock, req->u.listen.backlog);
+
+out:
+	rsp = RING_GET_RESPONSE(&fedata->ring, fedata->ring.rsp_prod_pvt++);
+	rsp->req_id = req->req_id;
+	rsp->cmd = req->cmd;
+	rsp->u.listen.id = req->u.listen.id;
+	rsp->ret = ret;
+	return 0;
+}
+
+static int pvcalls_back_accept(struct xenbus_device *dev,
+			       struct xen_pvcalls_request *req)
+{
+	struct pvcalls_fedata *fedata;
+	struct sockpass_mapping *mappass;
+	int ret = -EINVAL;
+	struct xen_pvcalls_response *rsp;
+	unsigned long flags;
+
+	fedata = dev_get_drvdata(&dev->dev);
+
+	down(&fedata->socket_lock);
+	mappass = radix_tree_lookup(&fedata->socketpass_mappings,
+		req->u.accept.id);
+	up(&fedata->socket_lock);
+	if (mappass == NULL)
+		goto out_error;
+
+	/*
+	 * Limitation of the current implementation: only support one
+	 * concurrent accept or poll call on one socket.
+	 */
+	spin_lock_irqsave(&mappass->copy_lock, flags);
+	if (mappass->reqcopy.cmd != 0) {
+		spin_unlock_irqrestore(&mappass->copy_lock, flags);
+		ret = -EINTR;
+		goto out_error;
+	}
+
+	mappass->reqcopy = *req;
+	spin_unlock_irqrestore(&mappass->copy_lock, flags);
+	queue_work(mappass->wq, &mappass->register_work);
+
+	/* Tell the caller we don't need to send back a notification yet */
+	return -1;
+
+out_error:
+	rsp = RING_GET_RESPONSE(&fedata->ring, fedata->ring.rsp_prod_pvt++);
+	rsp->req_id = req->req_id;
+	rsp->cmd = req->cmd;
+	rsp->u.accept.id = req->u.accept.id;
+	rsp->ret = ret;
+	return 0;
+}
+
+static int pvcalls_back_poll(struct xenbus_device *dev,
+			     struct xen_pvcalls_request *req)
+{
+	struct pvcalls_fedata *fedata;
+	struct sockpass_mapping *mappass;
+	struct xen_pvcalls_response *rsp;
+	struct inet_connection_sock *icsk;
+	struct request_sock_queue *queue;
+	unsigned long flags;
+	int ret;
+	bool data;
+
+	fedata = dev_get_drvdata(&dev->dev);
+
+	down(&fedata->socket_lock);
+	mappass = radix_tree_lookup(&fedata->socketpass_mappings,
+				    req->u.poll.id);
+	up(&fedata->socket_lock);
+	if (mappass == NULL)
+		return -EINVAL;
+
+	/*
+	 * Limitation of the current implementation: only support one
+	 * concurrent accept or poll call on one socket.
+	 */
+	spin_lock_irqsave(&mappass->copy_lock, flags);
+	if (mappass->reqcopy.cmd != 0) {
+		ret = -EINTR;
+		goto out;
+	}
+
+	mappass->reqcopy = *req;
+	icsk = inet_csk(mappass->sock->sk);
+	queue = &icsk->icsk_accept_queue;
+	data = queue->rskq_accept_head != NULL;
+	if (data) {
+		mappass->reqcopy.cmd = 0;
+		ret = 0;
+		goto out;
+	}
+	spin_unlock_irqrestore(&mappass->copy_lock, flags);
+
+	/* Tell the caller we don't need to send back a notification yet */
+	return -1;
+
+out:
+	spin_unlock_irqrestore(&mappass->copy_lock, flags);
+
+	rsp = RING_GET_RESPONSE(&fedata->ring, fedata->ring.rsp_prod_pvt++);
+	rsp->req_id = req->req_id;
+	rsp->cmd = req->cmd;
+	rsp->u.poll.id = req->u.poll.id;
+	rsp->ret = ret;
+	return 0;
+}
+
+static int pvcalls_back_handle_cmd(struct xenbus_device *dev,
+				   struct xen_pvcalls_request *req)
+{
+	int ret = 0;
+
+	switch (req->cmd) {
+	case PVCALLS_SOCKET:
+		ret = pvcalls_back_socket(dev, req);
+		break;
+	case PVCALLS_CONNECT:
+		ret = pvcalls_back_connect(dev, req);
+		break;
+	case PVCALLS_RELEASE:
+		ret = pvcalls_back_release(dev, req);
+		break;
+	case PVCALLS_BIND:
+		ret = pvcalls_back_bind(dev, req);
+		break;
+	case PVCALLS_LISTEN:
+		ret = pvcalls_back_listen(dev, req);
+		break;
+	case PVCALLS_ACCEPT:
+		ret = pvcalls_back_accept(dev, req);
+		break;
+	case PVCALLS_POLL:
+		ret = pvcalls_back_poll(dev, req);
+		break;
+	default:
+	{
+		struct pvcalls_fedata *fedata;
+		struct xen_pvcalls_response *rsp;
+
+		fedata = dev_get_drvdata(&dev->dev);
+		rsp = RING_GET_RESPONSE(
+				&fedata->ring, fedata->ring.rsp_prod_pvt++);
+		rsp->req_id = req->req_id;
+		rsp->cmd = req->cmd;
+		rsp->ret = -ENOTSUPP;
+		break;
+	}
+	}
+	return ret;
+}
+
+static void pvcalls_back_work(struct pvcalls_fedata *fedata)
+{
+	int notify, notify_all = 0, more = 1;
+	struct xen_pvcalls_request req;
+	struct xenbus_device *dev = fedata->dev;
+
+	while (more) {
+		while (RING_HAS_UNCONSUMED_REQUESTS(&fedata->ring)) {
+			RING_COPY_REQUEST(&fedata->ring,
+					  fedata->ring.req_cons++,
+					  &req);
+
+			if (!pvcalls_back_handle_cmd(dev, &req)) {
+				RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(
+					&fedata->ring, notify);
+				notify_all += notify;
+			}
+		}
+
+		if (notify_all) {
+			notify_remote_via_irq(fedata->irq);
+			notify_all = 0;
+		}
+
+		RING_FINAL_CHECK_FOR_REQUESTS(&fedata->ring, more);
+	}
+}
+
+static irqreturn_t pvcalls_back_event(int irq, void *dev_id)
+{
+	struct xenbus_device *dev = dev_id;
+	struct pvcalls_fedata *fedata = NULL;
+
+	if (dev == NULL)
+		return IRQ_HANDLED;
+
+	fedata = dev_get_drvdata(&dev->dev);
+	if (fedata == NULL)
+		return IRQ_HANDLED;
+
+	pvcalls_back_work(fedata);
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t pvcalls_back_conn_event(int irq, void *sock_map)
+{
+	struct sock_mapping *map = sock_map;
+	struct pvcalls_ioworker *iow;
+
+	if (map == NULL || map->sock == NULL || map->sock->sk == NULL ||
+		map->sock->sk->sk_user_data != map)
+		return IRQ_HANDLED;
+
+	iow = &map->ioworker;
+
+	atomic_inc(&map->write);
+	atomic_inc(&map->io);
+	queue_work(iow->wq, &iow->register_work);
+
+	return IRQ_HANDLED;
+}
+
+static int backend_connect(struct xenbus_device *dev)
+{
+	int err, evtchn;
+	grant_ref_t ring_ref;
+	struct pvcalls_fedata *fedata = NULL;
+
+	fedata = kzalloc(sizeof(struct pvcalls_fedata), GFP_KERNEL);
+	if (!fedata)
+		return -ENOMEM;
+
+	fedata->irq = -1;
+	err = xenbus_scanf(XBT_NIL, dev->otherend, "port", "%u",
+			   &evtchn);
+	if (err != 1) {
+		err = -EINVAL;
+		xenbus_dev_fatal(dev, err, "reading %s/event-channel",
+				 dev->otherend);
+		goto error;
+	}
+
+	err = xenbus_scanf(XBT_NIL, dev->otherend, "ring-ref", "%u", &ring_ref);
+	if (err != 1) {
+		err = -EINVAL;
+		xenbus_dev_fatal(dev, err, "reading %s/ring-ref",
+				 dev->otherend);
+		goto error;
+	}
+
+	err = bind_interdomain_evtchn_to_irq(dev->otherend_id, evtchn);
+	if (err < 0)
+		goto error;
+	fedata->irq = err;
+
+	err = request_threaded_irq(fedata->irq, NULL, pvcalls_back_event,
+				   IRQF_ONESHOT, "pvcalls-back", dev);
+	if (err < 0)
+		goto error;
+
+	err = xenbus_map_ring_valloc(dev, &ring_ref, 1,
+				     (void **)&fedata->sring);
+	if (err < 0)
+		goto error;
+
+	BACK_RING_INIT(&fedata->ring, fedata->sring, XEN_PAGE_SIZE * 1);
+	fedata->dev = dev;
+
+	INIT_LIST_HEAD(&fedata->socket_mappings);
+	INIT_RADIX_TREE(&fedata->socketpass_mappings, GFP_KERNEL);
+	sema_init(&fedata->socket_lock, 1);
+	dev_set_drvdata(&dev->dev, fedata);
+
+	down(&pvcalls_back_global.frontends_lock);
+	list_add_tail(&fedata->list, &pvcalls_back_global.frontends);
+	up(&pvcalls_back_global.frontends_lock);
+
+	return 0;
+
+ error:
+	if (fedata->irq >= 0)
+		unbind_from_irqhandler(fedata->irq, dev);
+	if (fedata->sring != NULL)
+		xenbus_unmap_ring_vfree(dev, fedata->sring);
+	kfree(fedata);
+	return err;
+}
+
+static int backend_disconnect(struct xenbus_device *dev)
+{
+	struct pvcalls_fedata *fedata;
+	struct sock_mapping *map, *n;
+	struct sockpass_mapping *mappass;
+	struct radix_tree_iter iter;
+	void **slot;
+
+
+	fedata = dev_get_drvdata(&dev->dev);
+
+	down(&fedata->socket_lock);
+	list_for_each_entry_safe(map, n, &fedata->socket_mappings, list) {
+		list_del(&map->list);
+		pvcalls_back_release_active(dev, fedata, map);
+	}
+
+	radix_tree_for_each_slot(slot, &fedata->socketpass_mappings, &iter, 0) {
+		mappass = radix_tree_deref_slot(slot);
+		if (!mappass)
+			continue;
+		if (radix_tree_exception(mappass)) {
+			if (radix_tree_deref_retry(mappass))
+				slot = radix_tree_iter_retry(&iter);
+		} else {
+			radix_tree_delete(&fedata->socketpass_mappings,
+					  mappass->id);
+			pvcalls_back_release_passive(dev, fedata, mappass);
+		}
+	}
+	up(&fedata->socket_lock);
+
+	unbind_from_irqhandler(fedata->irq, dev);
+	xenbus_unmap_ring_vfree(dev, fedata->sring);
+
+	list_del(&fedata->list);
+	kfree(fedata);
+	dev_set_drvdata(&dev->dev, NULL);
+
+	return 0;
+}
+
+static int pvcalls_back_probe(struct xenbus_device *dev,
+			      const struct xenbus_device_id *id)
+{
+	int err, abort;
+	struct xenbus_transaction xbt;
+
+again:
+	abort = 1;
+
+	err = xenbus_transaction_start(&xbt);
+	if (err) {
+		pr_warn("%s cannot create xenstore transaction\n", __func__);
+		return err;
+	}
+
+	err = xenbus_printf(xbt, dev->nodename, "versions", "%s",
+			    PVCALLS_VERSIONS);
+	if (err) {
+		pr_warn("%s write out 'versions' failed\n", __func__);
+		goto abort;
+	}
+
+	err = xenbus_printf(xbt, dev->nodename, "max-page-order", "%u",
+			    MAX_RING_ORDER);
+	if (err) {
+		pr_warn("%s write out 'max-page-order' failed\n", __func__);
+		goto abort;
+	}
+
+	err = xenbus_printf(xbt, dev->nodename, "function-calls",
+			    XENBUS_FUNCTIONS_CALLS);
+	if (err) {
+		pr_warn("%s write out 'function-calls' failed\n", __func__);
+		goto abort;
+	}
+
+	abort = 0;
+abort:
+	err = xenbus_transaction_end(xbt, abort);
+	if (err) {
+		if (err == -EAGAIN && !abort)
+			goto again;
+		pr_warn("%s cannot complete xenstore transaction\n", __func__);
+		return err;
+	}
+
+	if (abort)
+		return -EFAULT;
+
+	xenbus_switch_state(dev, XenbusStateInitWait);
+
+	return 0;
+}
+
+static void set_backend_state(struct xenbus_device *dev,
+			      enum xenbus_state state)
+{
+	while (dev->state != state) {
+		switch (dev->state) {
+		case XenbusStateClosed:
+			switch (state) {
+			case XenbusStateInitWait:
+			case XenbusStateConnected:
+				xenbus_switch_state(dev, XenbusStateInitWait);
+				break;
+			case XenbusStateClosing:
+				xenbus_switch_state(dev, XenbusStateClosing);
+				break;
+			default:
+				WARN_ON(1);
+			}
+			break;
+		case XenbusStateInitWait:
+		case XenbusStateInitialised:
+			switch (state) {
+			case XenbusStateConnected:
+				backend_connect(dev);
+				xenbus_switch_state(dev, XenbusStateConnected);
+				break;
+			case XenbusStateClosing:
+			case XenbusStateClosed:
+				xenbus_switch_state(dev, XenbusStateClosing);
+				break;
+			default:
+				WARN_ON(1);
+			}
+			break;
+		case XenbusStateConnected:
+			switch (state) {
+			case XenbusStateInitWait:
+			case XenbusStateClosing:
+			case XenbusStateClosed:
+				down(&pvcalls_back_global.frontends_lock);
+				backend_disconnect(dev);
+				up(&pvcalls_back_global.frontends_lock);
+				xenbus_switch_state(dev, XenbusStateClosing);
+				break;
+			default:
+				WARN_ON(1);
+			}
+			break;
+		case XenbusStateClosing:
+			switch (state) {
+			case XenbusStateInitWait:
+			case XenbusStateConnected:
+			case XenbusStateClosed:
+				xenbus_switch_state(dev, XenbusStateClosed);
+				break;
+			default:
+				WARN_ON(1);
+			}
+			break;
+		default:
+			WARN_ON(1);
+		}
+	}
+}
+
+static void pvcalls_back_changed(struct xenbus_device *dev,
+				 enum xenbus_state frontend_state)
+{
+	switch (frontend_state) {
+	case XenbusStateInitialising:
+		set_backend_state(dev, XenbusStateInitWait);
+		break;
+
+	case XenbusStateInitialised:
+	case XenbusStateConnected:
+		set_backend_state(dev, XenbusStateConnected);
+		break;
+
+	case XenbusStateClosing:
+		set_backend_state(dev, XenbusStateClosing);
+		break;
+
+	case XenbusStateClosed:
+		set_backend_state(dev, XenbusStateClosed);
+		if (xenbus_dev_is_online(dev))
+			break;
+		device_unregister(&dev->dev);
+		break;
+	case XenbusStateUnknown:
+		set_backend_state(dev, XenbusStateClosed);
+		device_unregister(&dev->dev);
+		break;
+
+	default:
+		xenbus_dev_fatal(dev, -EINVAL, "saw state %d at frontend",
+				 frontend_state);
+		break;
+	}
+}
+
+static int pvcalls_back_remove(struct xenbus_device *dev)
+{
+	return 0;
+}
+
+static int pvcalls_back_uevent(struct xenbus_device *xdev,
+			       struct kobj_uevent_env *env)
+{
+	return 0;
+}
+
+static const struct xenbus_device_id pvcalls_back_ids[] = {
+	{ "pvcalls" },
+	{ "" }
+};
+
+static struct xenbus_driver pvcalls_back_driver = {
+	.ids = pvcalls_back_ids,
+	.probe = pvcalls_back_probe,
+	.remove = pvcalls_back_remove,
+	.uevent = pvcalls_back_uevent,
+	.otherend_changed = pvcalls_back_changed,
+};
+
+static int __init pvcalls_back_init(void)
+{
+	int ret;
+
+	if (!xen_domain())
+		return -ENODEV;
+
+	ret = xenbus_register_backend(&pvcalls_back_driver);
+	if (ret < 0)
+		return ret;
+
+	sema_init(&pvcalls_back_global.frontends_lock, 1);
+	INIT_LIST_HEAD(&pvcalls_back_global.frontends);
+	return 0;
+}
+module_init(pvcalls_back_init);
+
+static void __exit pvcalls_back_fin(void)
+{
+	struct pvcalls_fedata *fedata, *nfedata;
+
+	down(&pvcalls_back_global.frontends_lock);
+	list_for_each_entry_safe(fedata, nfedata,
+				 &pvcalls_back_global.frontends, list) {
+		backend_disconnect(fedata->dev);
+	}
+	up(&pvcalls_back_global.frontends_lock);
+
+	xenbus_unregister_driver(&pvcalls_back_driver);
+}
+
+module_exit(pvcalls_back_fin);
diff --git a/drivers/xen/xen-pciback/pci_stub.c b/drivers/xen/xen-pciback/pci_stub.c
index 6331a95691a4..9e480fdebe1f 100644
--- a/drivers/xen/xen-pciback/pci_stub.c
+++ b/drivers/xen/xen-pciback/pci_stub.c
@@ -1172,8 +1172,8 @@ out:
 	return err;
 }
 
-static ssize_t pcistub_slot_add(struct device_driver *drv, const char *buf,
-				size_t count)
+static ssize_t new_slot_store(struct device_driver *drv, const char *buf,
+			      size_t count)
 {
 	int domain, bus, slot, func;
 	int err;
@@ -1189,10 +1189,10 @@ out:
 		err = count;
 	return err;
 }
-static DRIVER_ATTR(new_slot, S_IWUSR, NULL, pcistub_slot_add);
+static DRIVER_ATTR_WO(new_slot);
 
-static ssize_t pcistub_slot_remove(struct device_driver *drv, const char *buf,
-				   size_t count)
+static ssize_t remove_slot_store(struct device_driver *drv, const char *buf,
+				 size_t count)
 {
 	int domain, bus, slot, func;
 	int err;
@@ -1208,9 +1208,9 @@ out:
 		err = count;
 	return err;
 }
-static DRIVER_ATTR(remove_slot, S_IWUSR, NULL, pcistub_slot_remove);
+static DRIVER_ATTR_WO(remove_slot);
 
-static ssize_t pcistub_slot_show(struct device_driver *drv, char *buf)
+static ssize_t slots_show(struct device_driver *drv, char *buf)
 {
 	struct pcistub_device_id *pci_dev_id;
 	size_t count = 0;
@@ -1231,9 +1231,9 @@ static ssize_t pcistub_slot_show(struct device_driver *drv, char *buf)
 
 	return count;
 }
-static DRIVER_ATTR(slots, S_IRUSR, pcistub_slot_show, NULL);
+static DRIVER_ATTR_RO(slots);
 
-static ssize_t pcistub_irq_handler_show(struct device_driver *drv, char *buf)
+static ssize_t irq_handlers_show(struct device_driver *drv, char *buf)
 {
 	struct pcistub_device *psdev;
 	struct xen_pcibk_dev_data *dev_data;
@@ -1260,11 +1260,10 @@ static ssize_t pcistub_irq_handler_show(struct device_driver *drv, char *buf)
 	spin_unlock_irqrestore(&pcistub_devices_lock, flags);
 	return count;
 }
-static DRIVER_ATTR(irq_handlers, S_IRUSR, pcistub_irq_handler_show, NULL);
+static DRIVER_ATTR_RO(irq_handlers);
 
-static ssize_t pcistub_irq_handler_switch(struct device_driver *drv,
-					  const char *buf,
-					  size_t count)
+static ssize_t irq_handler_state_store(struct device_driver *drv,
+				       const char *buf, size_t count)
 {
 	struct pcistub_device *psdev;
 	struct xen_pcibk_dev_data *dev_data;
@@ -1301,11 +1300,10 @@ out:
 		err = count;
 	return err;
 }
-static DRIVER_ATTR(irq_handler_state, S_IWUSR, NULL,
-		   pcistub_irq_handler_switch);
+static DRIVER_ATTR_WO(irq_handler_state);
 
-static ssize_t pcistub_quirk_add(struct device_driver *drv, const char *buf,
-				 size_t count)
+static ssize_t quirks_store(struct device_driver *drv, const char *buf,
+			    size_t count)
 {
 	int domain, bus, slot, func, reg, size, mask;
 	int err;
@@ -1323,7 +1321,7 @@ out:
 	return err;
 }
 
-static ssize_t pcistub_quirk_show(struct device_driver *drv, char *buf)
+static ssize_t quirks_show(struct device_driver *drv, char *buf)
 {
 	int count = 0;
 	unsigned long flags;
@@ -1366,11 +1364,10 @@ out:
 
 	return count;
 }
-static DRIVER_ATTR(quirks, S_IRUSR | S_IWUSR, pcistub_quirk_show,
-		   pcistub_quirk_add);
+static DRIVER_ATTR_RW(quirks);
 
-static ssize_t permissive_add(struct device_driver *drv, const char *buf,
-			      size_t count)
+static ssize_t permissive_store(struct device_driver *drv, const char *buf,
+				size_t count)
 {
 	int domain, bus, slot, func;
 	int err;
@@ -1431,8 +1428,7 @@ static ssize_t permissive_show(struct device_driver *drv, char *buf)
 	spin_unlock_irqrestore(&pcistub_devices_lock, flags);
 	return count;
 }
-static DRIVER_ATTR(permissive, S_IRUSR | S_IWUSR, permissive_show,
-		   permissive_add);
+static DRIVER_ATTR_RW(permissive);
 
 static void pcistub_exit(void)
 {